From 4f56458d34cb13dcbf69aca650e9bf77d5497e6f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20G=C3=A4=C3=9Fler?= Date: Wed, 10 Jan 2024 01:04:33 +0100 Subject: [PATCH 001/812] Python script to compare commits with llama-bench (#4844) --- scripts/compare-llama-bench.py | 356 +++++++++++++++++++++++++++++++++ 1 file changed, 356 insertions(+) create mode 100755 scripts/compare-llama-bench.py diff --git a/scripts/compare-llama-bench.py b/scripts/compare-llama-bench.py new file mode 100755 index 000000000..bc1714487 --- /dev/null +++ b/scripts/compare-llama-bench.py @@ -0,0 +1,356 @@ +#!/usr/bin/env python3 + +import argparse +import heapq +import sys +import os +from glob import glob +import sqlite3 + +try: + import git + from tabulate import tabulate +except ImportError: + print("ERROR: the following Python libraries are required: GitPython, tabulate.") + sys.exit(1) + +# Properties by which to differentiate results per commit: +KEY_PROPERTIES = [ + "cuda", "opencl", "metal", "gpu_blas", "blas", "cpu_info", "gpu_info", "model_filename", + "model_type", "model_size", "model_n_params", "n_batch", "n_threads", "type_k", "type_v", + "n_gpu_layers", "main_gpu", "no_kv_offload", "mul_mat_q", "tensor_split", "n_prompt", "n_gen" +] + +# Properties that are boolean and are converted to Yes/No for the table: +BOOL_PROPERTIES = ["cuda", "opencl", "metal", "gpu_blas", "blas"] + +# Header names for the table: +PRETTY_NAMES = { + "cuda": "CUDA", "opencl": "OpenCL", "metal": "Metal", "gpu_blas": "GPU BLAS", "blas": "BLAS", + "cpu_info": "CPU", "gpu_info": "GPU", "model_filename": "File", "model_type": "Model", + "model_size": "Model Size [GiB]", "model_n_params": "Num. of Parameters", + "n_batch": "Batch size", "n_threads": "Threads", "type_k": "K type", "type_v": "V type", + "n_gpu_layers": "GPU layers", "main_gpu": "Main GPU", "no_kv_offload": "NKVO", + "mul_mat_q": "MMQ", "tensor_split": "Tensor split" +} + +DEFAULT_SHOW = ["model_type"] # Always show these properties by default. +DEFAULT_HIDE = ["model_filename"] # Always hide these properties by default. +GPU_NAME_STRIP = ["NVIDIA GeForce ", "Tesla ", "AMD Radeon "] # Strip prefixes for smaller tables. + +DESCRIPTION = """Creates tables from llama-bench data written to an SQLite database. Example usage (Linux): + +$ git checkout master +$ make clean && make llama-bench +$ ./llama-bench -o sql | sqlite3 llama-bench.sqlite +$ git checkout some_branch +$ make clean && make llama-bench +$ ./llama-bench -o sql | sqlite3 llama-bench.sqlite +$ ./scripts/compare-llama-bench.py + +Performance numbers from multiple runs per commit are averaged WITHOUT being weighted by the --repetitions parameter of llama-bench. +""" + +parser = argparse.ArgumentParser( + description=DESCRIPTION, formatter_class=argparse.RawDescriptionHelpFormatter) +help_b = ( + "The baseline commit to compare performance to. " + "Accepts either a branch name, tag name, or commit hash. " + "Defaults to latest master commit with data." +) +parser.add_argument("-b", "--baseline", help=help_b) +help_c = ( + "The commit whose performance is to be compared to the baseline. " + "Accepts either a branch name, tag name, or commit hash. " + "Defaults to the non-master commit for which llama-bench was run most recently." +) +parser.add_argument("-c", "--compare", help=help_c) +help_i = ( + "Input SQLite file for comparing commits. " + "Defaults to 'llama-bench.sqlite' in the current working directory. " + "If no such file is found and there is exactly one .sqlite file in the current directory, " + "that file is instead used as input." +) +parser.add_argument("-i", "--input", help=help_i) +help_o = ( + "Output format for the table. " + "Defaults to 'pipe' (GitHub compatible). " + "Also supports e.g. 'latex' or 'mediawiki'. " + "See tabulate documentation for full list." +) +parser.add_argument("-o", "--output", help=help_o, default="pipe") +help_s = ( + "Columns to add to the table. " + "Accepts a comma-separated list of values. " + f"Legal values: {', '.join(KEY_PROPERTIES[:-2])}. " + "Defaults to model name (model_type) and CPU and/or GPU name (cpu_info, gpu_info) " + "plus any column where not all data points are the same. " + "If the columns are manually specified, then the results for each unique combination of the " + "specified values are averaged WITHOUT weighing by the --repetitions parameter of llama-bench." +) +parser.add_argument("-s", "--show", help=help_s) + +known_args, unknown_args = parser.parse_known_args() + +if unknown_args: + print(f"ERROR: Received unknown args: {unknown_args}.") + print() + parser.print_help() + sys.exit(1) + +input_file = known_args.input +if input_file is None and os.path.exists("./llama-bench.sqlite"): + input_file = "llama-bench.sqlite" +if input_file is None: + sqlite_files = glob("*.sqlite") + if len(sqlite_files) == 1: + input_file = sqlite_files[0] + +if input_file is None: + print("ERROR: Cannot find a suitable input file, please provide one.") + print() + parser.print_help() + sys.exit(1) + +connection = sqlite3.connect(input_file) +cursor = connection.cursor() +builds = cursor.execute("SELECT DISTINCT build_commit FROM test;").fetchall() + +try: + repo = git.Repo(".", search_parent_directories=True) +except git.exc.InvalidGitRepositoryError: + repo = None + + +def find_parent_in_data(commit): + """Helper function to find the most recent parent measured in number of commits for which there is data.""" + heap = [(0, commit)] + seen_hexsha8 = set() + while heap: + depth, current_commit = heapq.heappop(heap) + current_hexsha8 = commit.hexsha[:8] + if (current_hexsha8,) in builds: + return current_hexsha8 + for parent in commit.parents: + parent_hexsha8 = parent.hexsha[:8] + if parent_hexsha8 not in seen_hexsha8: + seen_hexsha8.add(parent_hexsha8) + heapq.heappush(heap, (depth + 1, parent)) + return None + + +def get_all_parent_hexsha8s(commit): + """Helper function to recursively get hexsha8 values for all parents of a commit.""" + unvisited = [commit] + visited = [] + + while unvisited: + current_commit = unvisited.pop(0) + visited.append(current_commit.hexsha[:8]) + for parent in current_commit.parents: + if parent.hexsha[:8] not in visited: + unvisited.append(parent) + + return visited + + +def get_commit_name(hexsha8): + """Helper function to find a human-readable name for a commit if possible.""" + if repo is None: + return hexsha8 + for h in repo.heads: + if h.commit.hexsha[:8] == hexsha8: + return h.name + for t in repo.tags: + if t.commit.hexsha[:8] == hexsha8: + return t.name + return hexsha8 + + +def get_commit_hexsha8(name): + """Helper function to search for a commit given a human-readable name.""" + if repo is None: + return None + for h in repo.heads: + if h.name == name: + return h.commit.hexsha[:8] + for t in repo.tags: + if t.name == name: + return t.commit.hexsha[:8] + return None + + +hexsha8_baseline = name_baseline = None + +# If the user specified a baseline, try to find a commit for it: +if known_args.baseline is not None: + if (known_args.baseline,) in builds: + hexsha8_baseline = known_args.baseline + if hexsha8_baseline is None: + hexsha8_baseline = get_commit_hexsha8(known_args.baseline) + name_baseline = known_args.baseline + if hexsha8_baseline is None: + print(f"ERROR: cannot find data for baseline={known_args.baseline}.") + sys.exit(1) +# Otherwise, search for the most recent parent of master for which there is data: +elif repo is not None: + hexsha8_baseline = find_parent_in_data(repo.heads.master.commit) + + if hexsha8_baseline is None: + print("ERROR: No baseline was provided and did not find data for any master branch commits.") + print() + parser.print_help() + sys.exit(1) +else: + print( + "ERROR: No baseline was provided and the current working directory " + "is not part of a git repository from which a baseline could be inferred." + ) + print() + parser.print_help() + sys.exit(1) + + +name_baseline = get_commit_name(hexsha8_baseline) + +hexsha8_compare = name_compare = None + +# If the user has specified a compare value, try to find a corresponding commit: +if known_args.compare is not None: + if (known_args.compare,) in builds: + hexsha8_compare = known_args.compare + if hexsha8_compare is None: + hexsha8_compare = get_commit_hexsha8(known_args.compare) + name_compare = known_args.compare + if hexsha8_compare is None: + print(f"ERROR: cannot find data for baseline={known_args.compare}.") + sys.exit(1) +# Otherwise, search for the commit for llama-bench was most recently run +# and that is not a parent of master: +elif repo is not None: + hexsha8s_master = get_all_parent_hexsha8s(repo.heads.master.commit) + builds_timestamp = cursor.execute( + "SELECT build_commit, test_time FROM test ORDER BY test_time;").fetchall() + for (hexsha8, _) in reversed(builds_timestamp): + if hexsha8 not in hexsha8s_master: + hexsha8_compare = hexsha8 + break + + if hexsha8_compare is None: + print("ERROR: No compare target was provided and did not find data for any non-master commits.") + print() + parser.print_help() + sys.exit(1) +else: + print( + "ERROR: No compare target was provided and the current working directory " + "is not part of a git repository from which a compare target could be inferred." + ) + print() + parser.print_help() + sys.exit(1) + +name_compare = get_commit_name(hexsha8_compare) + + +def get_rows(properties): + """ + Helper function that gets table rows for some list of properties. + Rows are created by combining those where all provided properties are equal. + The resulting rows are then grouped by the provided properties and the t/s values are averaged. + The returned rows are unique in terms of property combinations. + """ + select_string = ", ".join( + [f"tb.{p}" for p in properties] + ["tb.n_prompt", "tb.n_gen", "AVG(tb.avg_ts)", "AVG(tc.avg_ts)"]) + equal_string = " AND ".join( + [f"tb.{p} = tc.{p}" for p in KEY_PROPERTIES] + [ + f"tb.build_commit = '{hexsha8_baseline}'", f"tc.build_commit = '{hexsha8_compare}'"] + ) + group_order_string = ", ".join([f"tb.{p}" for p in properties] + ["tb.n_gen", "tb.n_prompt"]) + query = (f"SELECT {select_string} FROM test tb JOIN test tc ON {equal_string} " + f"GROUP BY {group_order_string} ORDER BY {group_order_string};") + return cursor.execute(query).fetchall() + + +# If the user provided columns to group the results by, use them: +if known_args.show is not None: + show = known_args.show.split(",") + unknown_cols = [] + for prop in show: + if prop not in KEY_PROPERTIES[:-2]: # Last two values are n_prompt, n_gen. + unknown_cols.append(prop) + if unknown_cols: + print(f"ERROR: Unknown values for --show: {', '.join(unknown_cols)}") + print() + parser.print_usage() + sys.exit(1) + rows_show = get_rows(show) +# Otherwise, select those columns where the values are not all the same: +else: + rows_full = get_rows(KEY_PROPERTIES) + properties_different = [] + for i, kp_i in enumerate(KEY_PROPERTIES): + if kp_i in DEFAULT_SHOW or kp_i == "n_prompt" or kp_i == "n_gen": + continue + for row_full in rows_full: + if row_full[i] != rows_full[0][i]: + properties_different.append(kp_i) + break + + show = [] + # Show CPU and/or GPU by default even if the hardware for all results is the same: + if "gpu_blas" not in properties_different and "n_gpu_layers" not in properties_different: + gpu_blas = bool(rows_full[0][KEY_PROPERTIES.index("gpu_blas")]) + ngl = int(rows_full[0][KEY_PROPERTIES.index("n_gpu_layers")]) + + if not gpu_blas or ngl != 99 and "cpu_info" not in properties_different: + show.append("cpu_info") + if gpu_blas and "gpu_info" not in properties_different: + show.append("gpu_info") + + show += DEFAULT_SHOW + show += properties_different + for prop in DEFAULT_HIDE: + try: + show.remove(prop) + except ValueError: + pass + rows_show = get_rows(show) + +table = [] +for row in rows_show: + n_prompt = int(row[-4]) + n_gen = int(row[-3]) + assert n_prompt == 0 or n_gen == 0 + test_name = f"tg{n_gen}" if n_prompt == 0 else f"pp{n_prompt}" + # Regular columns test name avg t/s values Speedup + # VVVVVVVVVVVVV VVVVVVVVV VVVVVVVVVVVVVV VVVVVVV + table.append(list(row[:-4]) + [test_name] + list(row[-2:]) + [float(row[-1]) / float(row[-2])]) + +# Some a-posteriori fixes to make the table contents prettier: +for bool_property in BOOL_PROPERTIES: + if bool_property in show: + ip = show.index(bool_property) + for row_table in table: + row_table[ip] = "Yes" if int(row_table[ip]) == 1 else "No" + +if "model_size" in show: + ip = show.index("model_size") + for row_table in table: + row_table[ip] = float(row_table[ip]) / 1024 ** 3 + +if "gpu_info" in show: + ip = show.index("gpu_info") + for gns in GPU_NAME_STRIP: + for row_table in table: + row_table[ip] = row_table[ip].replace(gns, "") + +headers = [PRETTY_NAMES[p] for p in show] +headers += ["Test", f"t/s {name_baseline}", f"t/s {name_compare}", "Speedup"] + +print(tabulate( + table, + headers=headers, + floatfmt=".2f", + tablefmt=known_args.output +)) From d34633d8db6c2e400355de4862cd699154ecc73f Mon Sep 17 00:00:00 2001 From: John <78893154+cmp-nct@users.noreply.github.com> Date: Wed, 10 Jan 2024 14:37:09 +0100 Subject: [PATCH 002/812] clip : support more quantization types (#4846) Uses ggml functions instead of hardcoded names and adds support to quantize into the modern Q-K variants. This is just the bare minimum to get k-types working - a more refined choice of types would be needed to get best quality on low quantizations. I ran a few tests, it doesn't break anything I could notice and a Q6_K ViT works almost as well as Q8_0 but 3 times the inference speed. --- examples/llava/clip.cpp | 62 ++++++++++++++++------------------------- 1 file changed, 24 insertions(+), 38 deletions(-) diff --git a/examples/llava/clip.cpp b/examples/llava/clip.cpp index cfb79e789..2ae8853d3 100644 --- a/examples/llava/clip.cpp +++ b/examples/llava/clip.cpp @@ -126,24 +126,7 @@ static struct ggml_tensor * get_tensor(struct ggml_context * ctx, const std::str } static std::string get_ftype(int ftype) { - switch (ftype) { - case 0: - return "f32"; - case 1: - return "f16"; - case 2: - return "q4_0"; - case 3: - return "q4_1"; - case 6: - return "q5_0"; - case 7: - return "q5_1"; - case 8: - return "q8_0"; - default: - throw std::runtime_error(format("%s: Unrecognized file type: %d\n", __func__, ftype)); - } + return ggml_type_name(static_cast(ftype)); } // @@ -533,6 +516,7 @@ struct clip_ctx * clip_model_load(const char * fname, const int verbosity = 1) { buffer_size += n_tensors * 128 /* CLIP PADDING */; clip_ctx * new_clip = new clip_ctx; + #ifdef GGML_USE_CUBLAS new_clip->backend = ggml_backend_cuda_init(0); printf("%s: CLIP using CUDA backend\n", __func__); @@ -543,6 +527,7 @@ struct clip_ctx * clip_model_load(const char * fname, const int verbosity = 1) { printf("%s: CLIP using Metal backend\n", __func__); #endif + if (!new_clip->backend) { new_clip->backend = ggml_backend_cpu_init(); printf("%s: CLIP using CPU backend\n", __func__); @@ -931,26 +916,8 @@ bool clip_model_quantize(const char * fname_inp, const char * fname_out, const i ggml_type type = GGML_TYPE_Q4_1; - switch (itype) { - case 2: - type = GGML_TYPE_Q4_0; - break; - case 3: - type = GGML_TYPE_Q4_1; - break; - case 6: - type = GGML_TYPE_Q5_0; - break; - case 7: - type = GGML_TYPE_Q5_1; - break; - case 8: - type = GGML_TYPE_Q8_0; - break; - default: - fprintf(stderr, "%s: invalid quantization type %d\n", __func__, itype); - return false; - }; + assert(itype < GGML_TYPE_COUNT); + type = static_cast(itype); auto * ctx_clip = clip_model_load(fname_inp, 2); @@ -1010,6 +977,10 @@ bool clip_model_quantize(const char * fname_inp, const char * fname_out, const i if (quantize) { new_type = type; + if (new_type >= GGML_TYPE_Q2_K && name.find("embd") != std::string::npos) { + new_type = GGML_TYPE_Q8_0; // ggml_get_rows needs non K type + // fprintf(stderr, "%s: quantizing %s to %s\n", __func__, name.c_str(), ggml_type_name(new_type)); + } const size_t n_elms = ggml_nelements(cur); float * f32_data; @@ -1054,6 +1025,21 @@ bool clip_model_quantize(const char * fname_inp, const char * fname_out, const i case GGML_TYPE_Q8_0: { new_size = ggml_quantize_q8_0(f32_data, new_data, n_elms, cur->ne[0], hist_cur.data()); } break; + case GGML_TYPE_Q2_K: { + new_size = ggml_quantize_q2_K(f32_data, new_data, n_elms, cur->ne[0], hist_cur.data()); + } break; + case GGML_TYPE_Q3_K: { + new_size = ggml_quantize_q3_K(f32_data, new_data, n_elms, cur->ne[0], hist_cur.data()); + } break; + case GGML_TYPE_Q4_K: { + new_size = ggml_quantize_q4_K(f32_data, new_data, n_elms, cur->ne[0], hist_cur.data()); + } break; + case GGML_TYPE_Q5_K: { + new_size = ggml_quantize_q5_K(f32_data, new_data, n_elms, cur->ne[0], hist_cur.data()); + } break; + case GGML_TYPE_Q6_K: { + new_size = ggml_quantize_q6_K(f32_data, new_data, n_elms, cur->ne[0], hist_cur.data()); + } break; default: { fprintf(stderr, "%s: unsupported quantization type %d\n", __func__, new_type); return false; From 329ff615699d32f596d4ebf8baba654c30064e0d Mon Sep 17 00:00:00 2001 From: Austin <77757836+teleprint-me@users.noreply.github.com> Date: Wed, 10 Jan 2024 08:39:09 -0500 Subject: [PATCH 003/812] llama : recognize 1B phi models (#4847) This update categorizes models with 24 layers as MODEL_1B, ensuring compatibility with different Phi model variants without impacting existing Phi-2 model functionality. --- llama.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/llama.cpp b/llama.cpp index 8e0717db9..0f09d0c2b 100644 --- a/llama.cpp +++ b/llama.cpp @@ -2829,6 +2829,7 @@ static void llm_load_hparams( ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps); switch (hparams.n_layer) { + case 24: model.type = e_model::MODEL_1B; break; case 32: model.type = e_model::MODEL_3B; break; default: model.type = e_model::MODEL_UNKNOWN; } From 57d016ba2d46a6e22517a31a75cebb48f9e234b6 Mon Sep 17 00:00:00 2001 From: Brian Date: Thu, 11 Jan 2024 01:09:53 +1100 Subject: [PATCH 004/812] llama : add additional suffixes for model params (#4834) * llm_load_print_meta: Add additional suffixs for model params * Update llama.cpp model param log remove unneeded comments and convert from > to >= --- llama.cpp | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/llama.cpp b/llama.cpp index 0f09d0c2b..e1f1932ba 100644 --- a/llama.cpp +++ b/llama.cpp @@ -3146,7 +3146,15 @@ static void llm_load_print_meta(llama_model_loader & ml, llama_model & model) { LLAMA_LOG_INFO("%s: rope_finetuned = %s\n", __func__, hparams.rope_finetuned ? "yes" : "unknown"); LLAMA_LOG_INFO("%s: model type = %s\n", __func__, llama_model_type_name(model.type)); LLAMA_LOG_INFO("%s: model ftype = %s\n", __func__, llama_model_ftype_name(model.ftype).c_str()); - LLAMA_LOG_INFO("%s: model params = %.2f B\n", __func__, ml.n_elements*1e-9); + if (ml.n_elements >= 1e12) { + LLAMA_LOG_INFO("%s: model params = %.2f T\n", __func__, ml.n_elements*1e-12); + } else if (ml.n_elements >= 1e9) { + LLAMA_LOG_INFO("%s: model params = %.2f B\n", __func__, ml.n_elements*1e-9); + } else if (ml.n_elements >= 1e6) { + LLAMA_LOG_INFO("%s: model params = %.2f M\n", __func__, ml.n_elements*1e-6); + } else { + LLAMA_LOG_INFO("%s: model params = %.2f K\n", __func__, ml.n_elements*1e-3); + } if (ml.n_bytes < GiB) { LLAMA_LOG_INFO("%s: model size = %.2f MiB (%.2f BPW) \n", __func__, ml.n_bytes/1024.0/1024.0, ml.n_bytes*8.0/ml.n_elements); } else { From cd108e641dbdedd8c5641c4cec1762f751f38136 Mon Sep 17 00:00:00 2001 From: Behnam M <58621210+ibehnam@users.noreply.github.com> Date: Wed, 10 Jan 2024 14:56:05 -0500 Subject: [PATCH 005/812] server : add a `/health` endpoint (#4860) * added /health endpoint to the server * added comments on the additional /health endpoint * Better handling of server state When the model is being loaded, the server state is `LOADING_MODEL`. If model-loading fails, the server state becomes `ERROR`, otherwise it becomes `READY`. The `/health` endpoint provides more granular messages now according to the server_state value. * initialized server_state * fixed a typo * starting http server before initializing the model * Update server.cpp * Update server.cpp * fixes * fixes * fixes * made ServerState atomic and turned two-line spaces into one-line --- examples/server/server.cpp | 199 +++++++++++++++++++++---------------- 1 file changed, 113 insertions(+), 86 deletions(-) diff --git a/examples/server/server.cpp b/examples/server/server.cpp index 6c7fcd176..1cca634d5 100644 --- a/examples/server/server.cpp +++ b/examples/server/server.cpp @@ -26,6 +26,7 @@ #include #include #include +#include #ifndef SERVER_VERBOSE #define SERVER_VERBOSE 1 @@ -146,6 +147,12 @@ static std::vector base64_decode(const std::string & encoded_string) // parallel // +enum ServerState { + LOADING_MODEL, // Server is starting up, model not fully loaded yet + READY, // Server is ready and model is loaded + ERROR // An error occurred, load_model failed +}; + enum task_type { COMPLETION_TASK, CANCEL_TASK @@ -2453,7 +2460,6 @@ static void server_params_parse(int argc, char **argv, server_params &sparams, } } - static std::string random_string() { static const std::string str("0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"); @@ -2790,15 +2796,117 @@ int main(int argc, char **argv) {"system_info", llama_print_system_info()}, }); - // load the model - if (!llama.load_model(params)) + httplib::Server svr; + + std::atomic server_state{LOADING_MODEL}; + + svr.set_default_headers({{"Server", "llama.cpp"}, + {"Access-Control-Allow-Origin", "*"}, + {"Access-Control-Allow-Headers", "content-type"}}); + + svr.Get("/health", [&](const httplib::Request&, httplib::Response& res) { + ServerState current_state = server_state.load(); + switch(current_state) { + case READY: + res.set_content(R"({"status": "ok"})", "application/json"); + res.status = 200; // HTTP OK + break; + case LOADING_MODEL: + res.set_content(R"({"status": "loading model"})", "application/json"); + res.status = 503; // HTTP Service Unavailable + break; + case ERROR: + res.set_content(R"({"status": "error", "error": "Model failed to load"})", "application/json"); + res.status = 500; // HTTP Internal Server Error + break; + } + }); + + svr.set_logger(log_server_request); + + svr.set_exception_handler([](const httplib::Request &, httplib::Response &res, std::exception_ptr ep) + { + const char fmt[] = "500 Internal Server Error\n%s"; + char buf[BUFSIZ]; + try + { + std::rethrow_exception(std::move(ep)); + } + catch (std::exception &e) + { + snprintf(buf, sizeof(buf), fmt, e.what()); + } + catch (...) + { + snprintf(buf, sizeof(buf), fmt, "Unknown Exception"); + } + res.set_content(buf, "text/plain; charset=utf-8"); + res.status = 500; + }); + + svr.set_error_handler([](const httplib::Request &, httplib::Response &res) + { + if (res.status == 401) + { + res.set_content("Unauthorized", "text/plain; charset=utf-8"); + } + if (res.status == 400) + { + res.set_content("Invalid request", "text/plain; charset=utf-8"); + } + else if (res.status == 404) + { + res.set_content("File Not Found", "text/plain; charset=utf-8"); + res.status = 404; + } + }); + + // set timeouts and change hostname and port + svr.set_read_timeout (sparams.read_timeout); + svr.set_write_timeout(sparams.write_timeout); + + if (!svr.bind_to_port(sparams.hostname, sparams.port)) { + fprintf(stderr, "\ncouldn't bind to server socket: hostname=%s port=%d\n\n", sparams.hostname.c_str(), sparams.port); return 1; } - llama.initialize(); + // Set the base directory for serving static files + svr.set_base_dir(sparams.public_path); - httplib::Server svr; + // to make it ctrl+clickable: + LOG_TEE("\nllama server listening at http://%s:%d\n\n", sparams.hostname.c_str(), sparams.port); + + std::unordered_map log_data; + log_data["hostname"] = sparams.hostname; + log_data["port"] = std::to_string(sparams.port); + + if (!sparams.api_key.empty()) { + log_data["api_key"] = "api_key: ****" + sparams.api_key.substr(sparams.api_key.length() - 4); + } + + LOG_INFO("HTTP server listening", log_data); + // run the HTTP server in a thread - see comment below + std::thread t([&]() + { + if (!svr.listen_after_bind()) + { + server_state.store(ERROR); + return 1; + } + + return 0; + }); + + // load the model + if (!llama.load_model(params)) + { + server_state.store(ERROR); + return 1; + } else { + llama.initialize(); + server_state.store(READY); + } // Middleware for API key validation auto validate_api_key = [&sparams](const httplib::Request &req, httplib::Response &res) -> bool { @@ -2826,10 +2934,6 @@ int main(int argc, char **argv) return false; }; - svr.set_default_headers({{"Server", "llama.cpp"}, - {"Access-Control-Allow-Origin", "*"}, - {"Access-Control-Allow-Headers", "content-type"}}); - // this is only called if no index.html is found in the public --path svr.Get("/", [](const httplib::Request &, httplib::Response &res) { @@ -2937,8 +3041,6 @@ int main(int argc, char **argv) } }); - - svr.Get("/v1/models", [¶ms](const httplib::Request&, httplib::Response& res) { std::time_t t = std::time(0); @@ -3157,81 +3259,6 @@ int main(int argc, char **argv) return res.set_content(result.result_json.dump(), "application/json; charset=utf-8"); }); - svr.set_logger(log_server_request); - - svr.set_exception_handler([](const httplib::Request &, httplib::Response &res, std::exception_ptr ep) - { - const char fmt[] = "500 Internal Server Error\n%s"; - char buf[BUFSIZ]; - try - { - std::rethrow_exception(std::move(ep)); - } - catch (std::exception &e) - { - snprintf(buf, sizeof(buf), fmt, e.what()); - } - catch (...) - { - snprintf(buf, sizeof(buf), fmt, "Unknown Exception"); - } - res.set_content(buf, "text/plain; charset=utf-8"); - res.status = 500; - }); - - svr.set_error_handler([](const httplib::Request &, httplib::Response &res) - { - if (res.status == 401) - { - res.set_content("Unauthorized", "text/plain; charset=utf-8"); - } - if (res.status == 400) - { - res.set_content("Invalid request", "text/plain; charset=utf-8"); - } - else if (res.status == 404) - { - res.set_content("File Not Found", "text/plain; charset=utf-8"); - res.status = 404; - } - }); - - // set timeouts and change hostname and port - svr.set_read_timeout (sparams.read_timeout); - svr.set_write_timeout(sparams.write_timeout); - - if (!svr.bind_to_port(sparams.hostname, sparams.port)) - { - fprintf(stderr, "\ncouldn't bind to server socket: hostname=%s port=%d\n\n", sparams.hostname.c_str(), sparams.port); - return 1; - } - - // Set the base directory for serving static files - svr.set_base_dir(sparams.public_path); - - // to make it ctrl+clickable: - LOG_TEE("\nllama server listening at http://%s:%d\n\n", sparams.hostname.c_str(), sparams.port); - - std::unordered_map log_data; - log_data["hostname"] = sparams.hostname; - log_data["port"] = std::to_string(sparams.port); - - if (!sparams.api_key.empty()) { - log_data["api_key"] = "api_key: ****" + sparams.api_key.substr(sparams.api_key.length() - 4); - } - - LOG_INFO("HTTP server listening", log_data); - // run the HTTP server in a thread - see comment below - std::thread t([&]() - { - if (!svr.listen_after_bind()) - { - return 1; - } - - return 0; - }); - // GG: if I put the main loop inside a thread, it crashes on the first request when build in Debug!? // "Bus error: 10" - this is on macOS, it does not crash on Linux //std::thread t2([&]() From 5c1980d8d4c4e0c0af77359f81cc44d90b3f250b Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Thu, 11 Jan 2024 09:10:34 +0200 Subject: [PATCH 006/812] server : fix build + rename enums (#4870) --- examples/server/server.cpp | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/examples/server/server.cpp b/examples/server/server.cpp index 1cca634d5..4a0714997 100644 --- a/examples/server/server.cpp +++ b/examples/server/server.cpp @@ -147,15 +147,15 @@ static std::vector base64_decode(const std::string & encoded_string) // parallel // -enum ServerState { - LOADING_MODEL, // Server is starting up, model not fully loaded yet - READY, // Server is ready and model is loaded - ERROR // An error occurred, load_model failed +enum server_state { + SERVER_STATE_LOADING_MODEL, // Server is starting up, model not fully loaded yet + SERVER_STATE_READY, // Server is ready and model is loaded + SERVER_STATE_ERROR // An error occurred, load_model failed }; enum task_type { - COMPLETION_TASK, - CANCEL_TASK + TASK_TYPE_COMPLETION, + TASK_TYPE_CANCEL, }; struct task_server { @@ -1402,7 +1402,7 @@ struct llama_server_context task.data = std::move(data); task.infill_mode = infill; task.embedding_mode = embedding; - task.type = COMPLETION_TASK; + task.type = TASK_TYPE_COMPLETION; task.multitask_id = multitask_id; // when a completion task's prompt array is not a singleton, we split it into multiple requests @@ -1524,7 +1524,7 @@ struct llama_server_context std::unique_lock lock(mutex_tasks); task_server task; task.id = id_gen++; - task.type = CANCEL_TASK; + task.type = TASK_TYPE_CANCEL; task.target_id = task_id; queue_tasks.push_back(task); condition_tasks.notify_one(); @@ -1560,7 +1560,7 @@ struct llama_server_context queue_tasks.erase(queue_tasks.begin()); switch (task.type) { - case COMPLETION_TASK: { + case TASK_TYPE_COMPLETION: { llama_client_slot *slot = get_slot(json_value(task.data, "slot_id", -1)); if (slot == nullptr) { @@ -1589,7 +1589,7 @@ struct llama_server_context break; } } break; - case CANCEL_TASK: { // release slot linked with the task id + case TASK_TYPE_CANCEL: { // release slot linked with the task id for (auto & slot : slots) { if (slot.task_id == task.target_id) @@ -2798,24 +2798,24 @@ int main(int argc, char **argv) httplib::Server svr; - std::atomic server_state{LOADING_MODEL}; + std::atomic state{SERVER_STATE_LOADING_MODEL}; svr.set_default_headers({{"Server", "llama.cpp"}, {"Access-Control-Allow-Origin", "*"}, {"Access-Control-Allow-Headers", "content-type"}}); svr.Get("/health", [&](const httplib::Request&, httplib::Response& res) { - ServerState current_state = server_state.load(); + server_state current_state = state.load(); switch(current_state) { - case READY: + case SERVER_STATE_READY: res.set_content(R"({"status": "ok"})", "application/json"); res.status = 200; // HTTP OK break; - case LOADING_MODEL: + case SERVER_STATE_LOADING_MODEL: res.set_content(R"({"status": "loading model"})", "application/json"); res.status = 503; // HTTP Service Unavailable break; - case ERROR: + case SERVER_STATE_ERROR: res.set_content(R"({"status": "error", "error": "Model failed to load"})", "application/json"); res.status = 500; // HTTP Internal Server Error break; @@ -2891,7 +2891,7 @@ int main(int argc, char **argv) { if (!svr.listen_after_bind()) { - server_state.store(ERROR); + state.store(SERVER_STATE_ERROR); return 1; } @@ -2901,11 +2901,11 @@ int main(int argc, char **argv) // load the model if (!llama.load_model(params)) { - server_state.store(ERROR); + state.store(SERVER_STATE_ERROR); return 1; } else { llama.initialize(); - server_state.store(READY); + state.store(SERVER_STATE_READY); } // Middleware for API key validation From 7a9f75c38b5e62fe27b8a5a3ed823b4a3714024b Mon Sep 17 00:00:00 2001 From: Behnam M <58621210+ibehnam@users.noreply.github.com> Date: Thu, 11 Jan 2024 02:12:05 -0500 Subject: [PATCH 007/812] server : update readme to document the new `/health` endpoint (#4866) * added /health endpoint to the server * added comments on the additional /health endpoint * Better handling of server state When the model is being loaded, the server state is `LOADING_MODEL`. If model-loading fails, the server state becomes `ERROR`, otherwise it becomes `READY`. The `/health` endpoint provides more granular messages now according to the server_state value. * initialized server_state * fixed a typo * starting http server before initializing the model * Update server.cpp * Update server.cpp * fixes * fixes * fixes * made ServerState atomic and turned two-line spaces into one-line * updated `server` readme to document the `/health` endpoint too --- examples/server/README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/examples/server/README.md b/examples/server/README.md index d85a14f89..dc27e72b9 100644 --- a/examples/server/README.md +++ b/examples/server/README.md @@ -110,6 +110,10 @@ node index.js ``` ## API Endpoints +- **GET** `/health`: Returns the current state of the server: + - `{"status": "loading model"}` if the model is still being loaded. + - `{"status": "error"}` if the model failed to load. + - `{"status": "ok"}` if the model is successfully loaded and the server is ready for further requests mentioned below. - **POST** `/completion`: Given a `prompt`, it returns the predicted completion. From f34432ca1e0b288129390c1db8296a82aaf1e632 Mon Sep 17 00:00:00 2001 From: Erik Scholz Date: Fri, 5 Jan 2024 16:00:00 +0100 Subject: [PATCH 008/812] fix : cuda order of synchronization when setting a buffer (ggml/679) * fix : cuda order of synchronization when setting a buffer * also sync before memcpy --------- Co-authored-by: slaren --- ggml-cuda.cu | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ggml-cuda.cu b/ggml-cuda.cu index e26260a35..900f7ba4a 100644 --- a/ggml-cuda.cu +++ b/ggml-cuda.cu @@ -10184,8 +10184,8 @@ static void ggml_backend_cuda_buffer_set_tensor(ggml_backend_buffer_t buffer, gg ggml_cuda_set_device(ctx->device); CUDA_CHECK(cudaDeviceSynchronize()); - CUDA_CHECK(cudaMemcpy((char *)tensor->data + offset, data, size, cudaMemcpyHostToDevice)); + CUDA_CHECK(cudaDeviceSynchronize()); } static void ggml_backend_cuda_buffer_get_tensor(ggml_backend_buffer_t buffer, const ggml_tensor * tensor, void * data, size_t offset, size_t size) { From c910e3c28a1caee8cb1398143d582dd9ab697e68 Mon Sep 17 00:00:00 2001 From: Halalaluyafail3 <55773281+Halalaluyafail3@users.noreply.github.com> Date: Tue, 9 Jan 2024 11:16:37 -0500 Subject: [PATCH 009/812] Fix execlp call (ggml/689) NULL can be an integer constant expression with the value zero, in this case the behavior would be undefined because of an incorrect type being passed to the variable arguments. --- ggml.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ggml.c b/ggml.c index adb387100..4a0ec4c44 100644 --- a/ggml.c +++ b/ggml.c @@ -132,7 +132,7 @@ void ggml_print_backtrace(void) { "-ex", "bt -frame-info source-and-location", "-ex", "detach", "-ex", "quit", - NULL); + (char *) NULL); } else { waitpid(pid, NULL, 0); } From e739de790921e6abbc8c70398303cacd74913f61 Mon Sep 17 00:00:00 2001 From: leejet Date: Wed, 10 Jan 2024 21:13:42 +0800 Subject: [PATCH 010/812] ggml : change GGML_MAX_NAME at compile time (ggml/682) * change GGML_MAX_NAME to 128 * allow controlling the value of GGML_MAX_NAME through external macro definitions --- ggml.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ggml.h b/ggml.h index c55e598b4..b6cc85952 100644 --- a/ggml.h +++ b/ggml.h @@ -218,7 +218,9 @@ #define GGML_MAX_PARAMS 2048 #define GGML_MAX_CONTEXTS 64 #define GGML_MAX_SRC 10 +#ifndef GGML_MAX_NAME #define GGML_MAX_NAME 64 +#endif #define GGML_MAX_OP_PARAMS 64 #define GGML_DEFAULT_N_THREADS 4 #define GGML_DEFAULT_GRAPH_SIZE 2048 From 5362e43962e84d61e20b91f34991d7ccaef4a7d5 Mon Sep 17 00:00:00 2001 From: Jack Mousseau Date: Wed, 10 Jan 2024 06:19:19 -0800 Subject: [PATCH 011/812] metal : wrap each operation in debug group (ggml/690) --- ggml-metal.m | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ggml-metal.m b/ggml-metal.m index 6c2a8d04e..161906824 100644 --- a/ggml-metal.m +++ b/ggml-metal.m @@ -1067,6 +1067,8 @@ bool ggml_metal_graph_compute( GGML_ASSERT(!"unsupported op"); } + [encoder pushDebugGroup:[NSString stringWithCString:ggml_op_desc(dst)]]; + const int64_t ne00 = src0 ? src0->ne[0] : 0; const int64_t ne01 = src0 ? src0->ne[1] : 0; const int64_t ne02 = src0 ? src0->ne[2] : 0; @@ -2423,6 +2425,8 @@ bool ggml_metal_graph_compute( GGML_ASSERT(false); } } + + [encoder popDebugGroup]; } if (encoder != nil) { From f85a973aa139ae6f37e8b8e1966f1d278b5e0372 Mon Sep 17 00:00:00 2001 From: Timothy Cronin <40186632+4imothy@users.noreply.github.com> Date: Thu, 11 Jan 2024 02:27:48 -0500 Subject: [PATCH 012/812] ggml : remove ggml_cpy_inplace and ggml_cont_inplace (ggml/693) --- ggml.c | 30 ++++++++---------------------- ggml.h | 11 ----------- 2 files changed, 8 insertions(+), 33 deletions(-) diff --git a/ggml.c b/ggml.c index 4a0ec4c44..9c42a45e3 100644 --- a/ggml.c +++ b/ggml.c @@ -4311,13 +4311,13 @@ struct ggml_tensor * ggml_set_2d_inplace( static struct ggml_tensor * ggml_cpy_impl( struct ggml_context * ctx, struct ggml_tensor * a, - struct ggml_tensor * b, - bool inplace) { + struct ggml_tensor * b) { GGML_ASSERT(ggml_nelements(a) == ggml_nelements(b)); bool is_node = false; - if (!inplace && (a->grad || b->grad)) { + if (a->grad || b->grad) { + // inplace is false and either one have a grad is_node = true; } @@ -4341,29 +4341,21 @@ struct ggml_tensor * ggml_cpy( struct ggml_context * ctx, struct ggml_tensor * a, struct ggml_tensor * b) { - return ggml_cpy_impl(ctx, a, b, false); -} - -struct ggml_tensor * ggml_cpy_inplace( - struct ggml_context * ctx, - struct ggml_tensor * a, - struct ggml_tensor * b) { - return ggml_cpy_impl(ctx, a, b, true); + return ggml_cpy_impl(ctx, a, b); } // ggml_cont static struct ggml_tensor * ggml_cont_impl( struct ggml_context * ctx, - struct ggml_tensor * a, - bool inplace) { + struct ggml_tensor * a) { bool is_node = false; - if (!inplace && a->grad) { + if (a->grad) { is_node = true; } - struct ggml_tensor * result = inplace ? ggml_view_tensor(ctx, a) : ggml_dup_tensor(ctx, a); + struct ggml_tensor * result = ggml_dup_tensor(ctx, a); ggml_format_name(result, "%s (cont)", a->name); result->op = GGML_OP_CONT; @@ -4376,13 +4368,7 @@ static struct ggml_tensor * ggml_cont_impl( struct ggml_tensor * ggml_cont( struct ggml_context * ctx, struct ggml_tensor * a) { - return ggml_cont_impl(ctx, a, false); -} - -struct ggml_tensor * ggml_cont_inplace( - struct ggml_context * ctx, - struct ggml_tensor * a) { - return ggml_cont_impl(ctx, a, true); + return ggml_cont_impl(ctx, a); } // make contiguous, with new shape diff --git a/ggml.h b/ggml.h index b6cc85952..127dcef1d 100644 --- a/ggml.h +++ b/ggml.h @@ -1163,22 +1163,11 @@ extern "C" { struct ggml_tensor * a, struct ggml_tensor * b); - // a -> b, in-place, return view(b) - GGML_API struct ggml_tensor * ggml_cpy_inplace( - struct ggml_context * ctx, - struct ggml_tensor * a, - struct ggml_tensor * b); - // make contiguous GGML_API struct ggml_tensor * ggml_cont( struct ggml_context * ctx, struct ggml_tensor * a); - // make contiguous, in-place - GGML_API struct ggml_tensor * ggml_cont_inplace( - struct ggml_context * ctx, - struct ggml_tensor * a); - // make contiguous, with new shape GGML_API struct ggml_tensor * ggml_cont_1d( struct ggml_context * ctx, From 3267c2abc72e34608224408ace3c048831050f97 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Thu, 11 Jan 2024 09:34:59 +0200 Subject: [PATCH 013/812] metal : fix deprecation warning (ggml/690) --- ggml-metal.m | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ggml-metal.m b/ggml-metal.m index 161906824..82d68cd1b 100644 --- a/ggml-metal.m +++ b/ggml-metal.m @@ -1067,7 +1067,7 @@ bool ggml_metal_graph_compute( GGML_ASSERT(!"unsupported op"); } - [encoder pushDebugGroup:[NSString stringWithCString:ggml_op_desc(dst)]]; + [encoder pushDebugGroup:[NSString stringWithCString:ggml_op_desc(dst) encoding:NSUTF8StringEncoding]]; const int64_t ne00 = src0 ? src0->ne[0] : 0; const int64_t ne01 = src0 ? src0->ne[1] : 0; From 64802ec00d6383784a9dacf616095eaced16c3c3 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Thu, 11 Jan 2024 09:39:08 +0200 Subject: [PATCH 014/812] sync : ggml --- scripts/sync-ggml.last | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/sync-ggml.last b/scripts/sync-ggml.last index fe7f3202f..3e2c579d5 100644 --- a/scripts/sync-ggml.last +++ b/scripts/sync-ggml.last @@ -1 +1 @@ -f96711108d55bdbbd277e6be07204dce6a94fb93 +979cc23b345006504cfc1f67c0fdf627805e3319 From 2a7c94db5fb67b2f8882d2d16a11bf5d8d12d397 Mon Sep 17 00:00:00 2001 From: Paul Tsochantaris Date: Thu, 11 Jan 2024 14:31:52 +0000 Subject: [PATCH 015/812] metal : put encoder debug group behind a define (#4873) --- ggml-metal.m | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ggml-metal.m b/ggml-metal.m index 82d68cd1b..9698e5a79 100644 --- a/ggml-metal.m +++ b/ggml-metal.m @@ -1067,7 +1067,9 @@ bool ggml_metal_graph_compute( GGML_ASSERT(!"unsupported op"); } +#ifndef GGML_METAL_NDEBUG [encoder pushDebugGroup:[NSString stringWithCString:ggml_op_desc(dst) encoding:NSUTF8StringEncoding]]; +#endif const int64_t ne00 = src0 ? src0->ne[0] : 0; const int64_t ne01 = src0 ? src0->ne[1] : 0; @@ -2426,7 +2428,9 @@ bool ggml_metal_graph_compute( } } +#ifndef GGML_METAL_NDEBUG [encoder popDebugGroup]; +#endif } if (encoder != nil) { From 2f043328e3116724d15b915b5c6078e2df860a69 Mon Sep 17 00:00:00 2001 From: Isaac McFadyen Date: Thu, 11 Jan 2024 09:33:26 -0500 Subject: [PATCH 016/812] server : fix typo in model name (#4876) --- examples/server/server.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/server/server.cpp b/examples/server/server.cpp index 4a0714997..860e4e9ae 100644 --- a/examples/server/server.cpp +++ b/examples/server/server.cpp @@ -2515,7 +2515,7 @@ json oaicompat_completion_params_parse( // // https://platform.openai.com/docs/api-reference/chat/create llama_sampling_params default_sparams; - llama_params["model"] = json_value(body, "model", std::string("uknown")); + llama_params["model"] = json_value(body, "model", std::string("unknown")); llama_params["prompt"] = format_chatml(body["messages"]); // OpenAI 'messages' to llama.cpp 'prompt' llama_params["cache_prompt"] = json_value(body, "cache_prompt", false); llama_params["temperature"] = json_value(body, "temperature", 0.0); From 43f76bf1c362c067fce46bb8dcda0b64af8a9533 Mon Sep 17 00:00:00 2001 From: pudepiedj Date: Thu, 11 Jan 2024 16:14:52 +0000 Subject: [PATCH 017/812] main : print total token count and tokens consumed so far (#4874) * Token count changes * Add show token count * Updating before PR * Two requested changes * Move param def posn --- common/common.cpp | 8 ++++++++ common/common.h | 2 +- examples/main/main.cpp | 6 +++++- llama.cpp | 2 +- 4 files changed, 15 insertions(+), 3 deletions(-) diff --git a/common/common.cpp b/common/common.cpp index 4e89fe516..bfcd6d4df 100644 --- a/common/common.cpp +++ b/common/common.cpp @@ -630,6 +630,12 @@ bool gpt_params_parse_ex(int argc, char ** argv, gpt_params & params) { break; } params.ppl_stride = std::stoi(argv[i]); + } else if (arg == "-stc" || arg == "--show_token_count") { + if (++i >= argc) { + invalid_param = true; + break; + } + params.token_interval = std::stoi(argv[i]); } else if (arg == "--ppl-output-type") { if (++i >= argc) { invalid_param = true; @@ -944,6 +950,8 @@ void gpt_print_usage(int /*argc*/, char ** argv, const gpt_params & params) { printf(" --override-kv KEY=TYPE:VALUE\n"); printf(" advanced option to override model metadata by key. may be specified multiple times.\n"); printf(" types: int, float, bool. example: --override-kv tokenizer.ggml.add_bos_token=bool:false\n"); + printf(" -stc N --show_token_count N\n"); + printf(" show consumed tokens every N tokens\n"); printf("\n"); #ifndef LOG_DISABLE_LOGS log_print_usage(); diff --git a/common/common.h b/common/common.h index e2bbfc258..a295e88b0 100644 --- a/common/common.h +++ b/common/common.h @@ -64,6 +64,7 @@ struct gpt_params { int32_t n_beams = 0; // if non-zero then use beam search of given width. int32_t grp_attn_n = 1; // group-attention factor int32_t grp_attn_w = 512; // group-attention width + int32_t token_interval = 512; // show token count every 512 tokens float rope_freq_base = 0.0f; // RoPE base frequency float rope_freq_scale = 0.0f; // RoPE frequency scaling factor float yarn_ext_factor = -1.0f; // YaRN extrapolation mix factor @@ -242,4 +243,3 @@ void dump_kv_cache_view(const llama_kv_cache_view & view, int row_size = 80); // Dump the KV cache view showing individual sequences in each cell (long output). void dump_kv_cache_view_seqs(const llama_kv_cache_view & view, int row_size = 40); - diff --git a/examples/main/main.cpp b/examples/main/main.cpp index 5ea67051f..1f35febbd 100644 --- a/examples/main/main.cpp +++ b/examples/main/main.cpp @@ -500,7 +500,7 @@ int main(int argc, char ** argv) { while ((n_remain != 0 && !is_antiprompt) || params.interactive) { // predict if (!embd.empty()) { - // Note: n_ctx - 4 here is to match the logic for commandline prompt handling via + // Note: (n_ctx - 4) here is to match the logic for commandline prompt handling via // --prompt or --file which uses the same value. int max_embd_size = n_ctx - 4; @@ -650,6 +650,10 @@ int main(int argc, char ** argv) { n_past += n_eval; LOG("n_past = %d\n", n_past); + // Display total tokens alongside total time + if (n_past % params.token_interval == 0) { + printf("\n\033[31mTokens consumed so far = %d / %d \033[0m\n", n_past, n_ctx); + } } if (!embd.empty() && !path_session.empty()) { diff --git a/llama.cpp b/llama.cpp index e1f1932ba..aaadfa444 100644 --- a/llama.cpp +++ b/llama.cpp @@ -10921,7 +10921,7 @@ void llama_print_timings(struct llama_context * ctx) { __func__, timings.t_p_eval_ms, timings.n_p_eval, timings.t_p_eval_ms / timings.n_p_eval, 1e3 / timings.t_p_eval_ms * timings.n_p_eval); LLAMA_LOG_INFO("%s: eval time = %10.2f ms / %5d runs (%8.2f ms per token, %8.2f tokens per second)\n", __func__, timings.t_eval_ms, timings.n_eval, timings.t_eval_ms / timings.n_eval, 1e3 / timings.t_eval_ms * timings.n_eval); - LLAMA_LOG_INFO("%s: total time = %10.2f ms\n", __func__, (timings.t_end_ms - timings.t_start_ms)); + LLAMA_LOG_INFO("%s: total time = %10.2f ms / %5d tokens\n", __func__, (timings.t_end_ms - timings.t_start_ms), (timings.n_p_eval + timings.n_eval)); } void llama_reset_timings(struct llama_context * ctx) { From d8d90aa343c22fe01429d3540e47ded87e9dcb9d Mon Sep 17 00:00:00 2001 From: Someone Date: Thu, 11 Jan 2024 17:22:34 +0000 Subject: [PATCH 018/812] ci: nix-flake-update: new token with pr permissions (#4879) * ci: nix-flake-update: new token with pr permissions --------- Co-authored-by: Georgi Gerganov --- .github/workflows/nix-flake-update.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/nix-flake-update.yml b/.github/workflows/nix-flake-update.yml index fa9360841..3a6a96e26 100644 --- a/.github/workflows/nix-flake-update.yml +++ b/.github/workflows/nix-flake-update.yml @@ -19,4 +19,4 @@ jobs: pr-labels: | nix pr-reviewers: philiptaron,SomeoneSerge - token: ${{ secrets.GITHUB_TOKEN }} + token: ${{ secrets.FLAKE_TOKEN }} From eab67950068e4b125007d027232c47d2a5831cd0 Mon Sep 17 00:00:00 2001 From: Behnam M <58621210+ibehnam@users.noreply.github.com> Date: Thu, 11 Jan 2024 12:41:39 -0500 Subject: [PATCH 019/812] server : add `LOG_INFO` when model is successfully loaded (#4881) * added /health endpoint to the server * added comments on the additional /health endpoint * Better handling of server state When the model is being loaded, the server state is `LOADING_MODEL`. If model-loading fails, the server state becomes `ERROR`, otherwise it becomes `READY`. The `/health` endpoint provides more granular messages now according to the server_state value. * initialized server_state * fixed a typo * starting http server before initializing the model * Update server.cpp * Update server.cpp * fixes * fixes * fixes * made ServerState atomic and turned two-line spaces into one-line * updated `server` readme to document the `/health` endpoint too * used LOG_INFO after successful model loading --- examples/server/server.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/examples/server/server.cpp b/examples/server/server.cpp index 860e4e9ae..51a4b689f 100644 --- a/examples/server/server.cpp +++ b/examples/server/server.cpp @@ -2906,6 +2906,7 @@ int main(int argc, char **argv) } else { llama.initialize(); state.store(SERVER_STATE_READY); + LOG_INFO("model loaded", {}); } // Middleware for API key validation From 27379455c38cb13f24de92dbd6fcdd04eeb1b9d9 Mon Sep 17 00:00:00 2001 From: Michael Coppola Date: Thu, 11 Jan 2024 12:51:17 -0500 Subject: [PATCH 020/812] server : support for multiple api keys (#4864) * server: added support for multiple api keys, added loading api keys from file * minor: fix whitespace * added file error handling to --api-key-file, changed code to better reflect current style * server: update README.md for --api-key-file --------- Co-authored-by: Michael Coppola --- examples/server/README.md | 3 ++- examples/server/server.cpp | 36 ++++++++++++++++++++++++++++++------ 2 files changed, 32 insertions(+), 7 deletions(-) diff --git a/examples/server/README.md b/examples/server/README.md index dc27e72b9..fd3034b99 100644 --- a/examples/server/README.md +++ b/examples/server/README.md @@ -23,7 +23,8 @@ Command line options: - `--host`: Set the hostname or ip address to listen. Default `127.0.0.1`. - `--port`: Set the port to listen. Default: `8080`. - `--path`: path from which to serve static files (default examples/server/public) -- `--api-key`: Set an api key for request authorization. By default the server responds to every request. With an api key set, the requests must have the Authorization header set with the api key as Bearer token. +- `--api-key`: Set an api key for request authorization. By default the server responds to every request. With an api key set, the requests must have the Authorization header set with the api key as Bearer token. May be used multiple times to enable multiple valid keys. +- `--api-key-file`: path to file containing api keys delimited by new lines. If set, requests must include one of the keys for access. May be used in conjunction with `--api-key`'s. - `--embedding`: Enable embedding extraction, Default: disabled. - `-np N`, `--parallel N`: Set the number of slots for process requests (default: 1) - `-cb`, `--cont-batching`: enable continuous batching (a.k.a dynamic batching) (default: disabled) diff --git a/examples/server/server.cpp b/examples/server/server.cpp index 51a4b689f..345004fa1 100644 --- a/examples/server/server.cpp +++ b/examples/server/server.cpp @@ -39,7 +39,7 @@ using json = nlohmann::json; struct server_params { std::string hostname = "127.0.0.1"; - std::string api_key; + std::vector api_keys; std::string public_path = "examples/server/public"; int32_t port = 8080; int32_t read_timeout = 600; @@ -2021,6 +2021,7 @@ static void server_print_usage(const char *argv0, const gpt_params ¶ms, printf(" --port PORT port to listen (default (default: %d)\n", sparams.port); printf(" --path PUBLIC_PATH path from which to serve static files (default %s)\n", sparams.public_path.c_str()); printf(" --api-key API_KEY optional api key to enhance server security. If set, requests must include this key for access.\n"); + printf(" --api-key-file FNAME path to file containing api keys delimited by new lines. If set, requests must include one of the keys for access.\n"); printf(" -to N, --timeout N server read/write timeout in seconds (default: %d)\n", sparams.read_timeout); printf(" --embedding enable embedding vector output (default: %s)\n", params.embedding ? "enabled" : "disabled"); printf(" -np N, --parallel N number of slots for process requests (default: %d)\n", params.n_parallel); @@ -2081,7 +2082,28 @@ static void server_params_parse(int argc, char **argv, server_params &sparams, invalid_param = true; break; } - sparams.api_key = argv[i]; + sparams.api_keys.push_back(argv[i]); + } + else if (arg == "--api-key-file") + { + if (++i >= argc) + { + invalid_param = true; + break; + } + std::ifstream key_file(argv[i]); + if (!key_file) { + fprintf(stderr, "error: failed to open file '%s'\n", argv[i]); + invalid_param = true; + break; + } + std::string key; + while (std::getline(key_file, key)) { + if (key.size() > 0) { + sparams.api_keys.push_back(key); + } + } + key_file.close(); } else if (arg == "--timeout" || arg == "-to") { @@ -2881,8 +2903,10 @@ int main(int argc, char **argv) log_data["hostname"] = sparams.hostname; log_data["port"] = std::to_string(sparams.port); - if (!sparams.api_key.empty()) { - log_data["api_key"] = "api_key: ****" + sparams.api_key.substr(sparams.api_key.length() - 4); + if (sparams.api_keys.size() == 1) { + log_data["api_key"] = "api_key: ****" + sparams.api_keys[0].substr(sparams.api_keys[0].length() - 4); + } else if (sparams.api_keys.size() > 1) { + log_data["api_key"] = "api_key: " + std::to_string(sparams.api_keys.size()) + " keys loaded"; } LOG_INFO("HTTP server listening", log_data); @@ -2912,7 +2936,7 @@ int main(int argc, char **argv) // Middleware for API key validation auto validate_api_key = [&sparams](const httplib::Request &req, httplib::Response &res) -> bool { // If API key is not set, skip validation - if (sparams.api_key.empty()) { + if (sparams.api_keys.empty()) { return true; } @@ -2921,7 +2945,7 @@ int main(int argc, char **argv) std::string prefix = "Bearer "; if (auth_header.substr(0, prefix.size()) == prefix) { std::string received_api_key = auth_header.substr(prefix.size()); - if (received_api_key == sparams.api_key) { + if (std::find(sparams.api_keys.begin(), sparams.api_keys.end(), received_api_key) != sparams.api_keys.end()) { return true; // API key is valid } } From 4330bd83feb39683de4bd7a34cfcf672ff8ac3e4 Mon Sep 17 00:00:00 2001 From: Laura Date: Thu, 11 Jan 2024 19:02:48 +0100 Subject: [PATCH 021/812] server : implement credentialed CORS (#4514) * Implement credentialed CORS according to MDN * Fix syntax error * Move validate_api_key up so it is defined before its first usage --- examples/server/server.cpp | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/examples/server/server.cpp b/examples/server/server.cpp index 345004fa1..031824e14 100644 --- a/examples/server/server.cpp +++ b/examples/server/server.cpp @@ -2822,9 +2822,15 @@ int main(int argc, char **argv) std::atomic state{SERVER_STATE_LOADING_MODEL}; - svr.set_default_headers({{"Server", "llama.cpp"}, - {"Access-Control-Allow-Origin", "*"}, - {"Access-Control-Allow-Headers", "content-type"}}); + svr.set_default_headers({{"Server", "llama.cpp"}}); + + // CORS preflight + svr.Options(R"(.*)", [](const httplib::Request &req, httplib::Response &res) { + res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin")); + res.set_header("Access-Control-Allow-Credentials", "true"); + res.set_header("Access-Control-Allow-Methods", "POST"); + res.set_header("Access-Control-Allow-Headers", "*"); + }); svr.Get("/health", [&](const httplib::Request&, httplib::Response& res) { server_state current_state = state.load(); @@ -2987,9 +2993,9 @@ int main(int argc, char **argv) return false; }); - svr.Get("/props", [&llama](const httplib::Request & /*req*/, httplib::Response &res) + svr.Get("/props", [&llama](const httplib::Request & req, httplib::Response &res) { - res.set_header("Access-Control-Allow-Origin", "*"); + res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin")); json data = { { "user_name", llama.name_user.c_str() }, { "assistant_name", llama.name_assistant.c_str() } @@ -2999,6 +3005,7 @@ int main(int argc, char **argv) svr.Post("/completion", [&llama, &validate_api_key](const httplib::Request &req, httplib::Response &res) { + res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin")); if (!validate_api_key(req, res)) { return; } @@ -3066,8 +3073,9 @@ int main(int argc, char **argv) } }); - svr.Get("/v1/models", [¶ms](const httplib::Request&, httplib::Response& res) + svr.Get("/v1/models", [¶ms](const httplib::Request& req, httplib::Response& res) { + res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin")); std::time_t t = std::time(0); json models = { @@ -3085,9 +3093,11 @@ int main(int argc, char **argv) res.set_content(models.dump(), "application/json; charset=utf-8"); }); + // TODO: add mount point without "/v1" prefix -- how? svr.Post("/v1/chat/completions", [&llama, &validate_api_key](const httplib::Request &req, httplib::Response &res) { + res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin")); if (!validate_api_key(req, res)) { return; } @@ -3161,6 +3171,7 @@ int main(int argc, char **argv) svr.Post("/infill", [&llama, &validate_api_key](const httplib::Request &req, httplib::Response &res) { + res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin")); if (!validate_api_key(req, res)) { return; } @@ -3233,6 +3244,7 @@ int main(int argc, char **argv) svr.Post("/tokenize", [&llama](const httplib::Request &req, httplib::Response &res) { + res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin")); const json body = json::parse(req.body); std::vector tokens; if (body.count("content") != 0) @@ -3245,6 +3257,7 @@ int main(int argc, char **argv) svr.Post("/detokenize", [&llama](const httplib::Request &req, httplib::Response &res) { + res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin")); const json body = json::parse(req.body); std::string content; if (body.count("tokens") != 0) @@ -3259,6 +3272,7 @@ int main(int argc, char **argv) svr.Post("/embedding", [&llama](const httplib::Request &req, httplib::Response &res) { + res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin")); const json body = json::parse(req.body); json prompt; if (body.count("content") != 0) From 3ba5b8ca8e6181a5c712c5b77595a29f1d3e2b97 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Thu, 11 Jan 2024 21:31:31 +0200 Subject: [PATCH 022/812] swift : pin ggml commit + remove ggml.h from spm-headers (#4878) ggml-ci --- Package.swift | 2 +- spm-headers/ggml.h | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) delete mode 120000 spm-headers/ggml.h diff --git a/Package.swift b/Package.swift index 583e2e276..59191da45 100644 --- a/Package.swift +++ b/Package.swift @@ -14,7 +14,7 @@ let package = Package( .library(name: "llama", targets: ["llama"]), ], dependencies: [ - .package(url: "https://github.com/ggerganov/ggml.git", .branch("master")) + .package(url: "https://github.com/ggerganov/ggml.git", .revision("979cc23b345006504cfc1f67c0fdf627805e3319")) ], targets: [ .target( diff --git a/spm-headers/ggml.h b/spm-headers/ggml.h deleted file mode 120000 index 39215298f..000000000 --- a/spm-headers/ggml.h +++ /dev/null @@ -1 +0,0 @@ -../ggml.h \ No newline at end of file From 49662cbed3e95f5976c070b85b9fd53fd577038d Mon Sep 17 00:00:00 2001 From: Kawrakow <48489457+ikawrakow@users.noreply.github.com> Date: Thu, 11 Jan 2024 20:39:39 +0100 Subject: [PATCH 023/812] ggml : SOTA 2-bit quants (add IQ2_XS) (#4856) * iq2_xs: basics * iq2_xs: this should have been in the basics * iq2_xs: CUDA and scalar CPU works * iq2_xs: WIP Metal * iq2_xs: Metal now works * iq2_xs: working, but dog slow, ARM_NEON dot product * iq2_xs: better ARM_NEON dot product We are now at 19.5 t/s for TG-128 and 61 t/s for PP-512 when running on the CPU. * iq2_xs: AVX2 dot product - 19.5 t/s * iq2_xs: faster AVX2 dit product 21.4 t/s for TG-128, 59.2 t/s for PP-512. The latter is 2x compared to the previous version. * iq2_xs: had forgotten to delete iq2-data.h * Add llama enum for IQ2_XS --------- Co-authored-by: Iwan Kawrakow --- ggml-cuda.cu | 232 +++++++++++++++++++++- ggml-metal.m | 42 +++- ggml-metal.metal | 378 +++++++++++++++++++++++++++++++++++- ggml-quants.c | 360 +++++++++++++++++++++++++++++++++- ggml-quants.h | 12 ++ ggml.c | 30 ++- ggml.h | 3 + llama.cpp | 3 + llama.h | 1 + tests/test-quantize-fns.cpp | 5 +- 10 files changed, 1038 insertions(+), 28 deletions(-) diff --git a/ggml-cuda.cu b/ggml-cuda.cu index 900f7ba4a..dd19699f6 100644 --- a/ggml-cuda.cu +++ b/ggml-cuda.cu @@ -486,6 +486,15 @@ typedef struct { } block_iq2_xxs; static_assert(sizeof(block_iq2_xxs) == sizeof(ggml_fp16_t) + QK_K/8*sizeof(uint16_t), "wrong iq2_xxs block size/padding"); +#define QR2_XS 8 +#define QI2_XS (QK_K / (4*QR2_XS)) +typedef struct { + half d; + uint16_t qs[QK_K/8]; + uint8_t scales[QK_K/32]; +} block_iq2_xs; +static_assert(sizeof(block_iq2_xs) == sizeof(ggml_fp16_t) + QK_K/8*sizeof(uint16_t) + QK_K/32, "wrong iq2_xs block size/padding"); + #define WARP_SIZE 32 #define MATRIX_ROW_PADDING 512 // last row of quant. matrices is a multiple of this to avoid out-of-bounds memory accesses @@ -1328,7 +1337,7 @@ static __global__ void dequantize_block_q6_K(const void * __restrict__ vx, dst_t #endif } -static const __device__ uint64_t kgrid_iq2xxs[256] = { +static const __device__ uint64_t iq2xxs_grid[256] = { 0x0808080808080808, 0x080808080808082b, 0x0808080808081919, 0x0808080808082b08, 0x0808080808082b2b, 0x0808080808190819, 0x0808080808191908, 0x08080808082b0808, 0x08080808082b082b, 0x08080808082b2b08, 0x08080808082b2b2b, 0x0808080819080819, @@ -1395,6 +1404,137 @@ static const __device__ uint64_t kgrid_iq2xxs[256] = { 0x2b2b082b08080808, 0x2b2b190808192b08, 0x2b2b2b0819190808, 0x2b2b2b1908081908, }; +static const __device__ uint64_t iq2xs_grid[512] = { + 0x0808080808080808, 0x080808080808082b, 0x0808080808081919, 0x0808080808082b08, + 0x0808080808082b2b, 0x0808080808190819, 0x0808080808191908, 0x080808080819192b, + 0x0808080808192b19, 0x08080808082b0808, 0x08080808082b082b, 0x08080808082b1919, + 0x08080808082b2b08, 0x0808080819080819, 0x0808080819081908, 0x080808081908192b, + 0x0808080819082b19, 0x0808080819190808, 0x080808081919082b, 0x0808080819191919, + 0x0808080819192b08, 0x08080808192b0819, 0x08080808192b1908, 0x080808082b080808, + 0x080808082b08082b, 0x080808082b081919, 0x080808082b082b08, 0x080808082b190819, + 0x080808082b191908, 0x080808082b192b19, 0x080808082b2b0808, 0x0808081908080819, + 0x0808081908081908, 0x080808190808192b, 0x0808081908082b19, 0x0808081908190808, + 0x080808190819082b, 0x0808081908191919, 0x0808081908192b08, 0x0808081908192b2b, + 0x08080819082b0819, 0x08080819082b1908, 0x0808081919080808, 0x080808191908082b, + 0x0808081919081919, 0x0808081919082b08, 0x0808081919190819, 0x0808081919191908, + 0x08080819192b0808, 0x08080819192b2b08, 0x080808192b080819, 0x080808192b081908, + 0x080808192b190808, 0x0808082b08080808, 0x0808082b0808082b, 0x0808082b08081919, + 0x0808082b08082b08, 0x0808082b08190819, 0x0808082b08191908, 0x0808082b082b0808, + 0x0808082b19080819, 0x0808082b19081908, 0x0808082b19190808, 0x0808082b19191919, + 0x0808082b2b080808, 0x0808082b2b082b2b, 0x0808190808080819, 0x0808190808081908, + 0x080819080808192b, 0x0808190808082b19, 0x0808190808190808, 0x080819080819082b, + 0x0808190808191919, 0x0808190808192b08, 0x08081908082b0819, 0x08081908082b1908, + 0x0808190819080808, 0x080819081908082b, 0x0808190819081919, 0x0808190819082b08, + 0x0808190819190819, 0x0808190819191908, 0x080819081919192b, 0x08081908192b0808, + 0x080819082b080819, 0x080819082b081908, 0x080819082b190808, 0x0808191908080808, + 0x080819190808082b, 0x0808191908081919, 0x0808191908082b08, 0x0808191908190819, + 0x0808191908191908, 0x08081919082b0808, 0x0808191919080819, 0x0808191919081908, + 0x0808191919190808, 0x08081919192b0819, 0x080819192b080808, 0x0808192b08080819, + 0x0808192b08081908, 0x0808192b08190808, 0x0808192b082b192b, 0x0808192b19080808, + 0x0808192b1908082b, 0x0808192b2b081908, 0x08082b0808080808, 0x08082b080808082b, + 0x08082b0808081919, 0x08082b0808082b08, 0x08082b0808082b2b, 0x08082b0808190819, + 0x08082b0808191908, 0x08082b08082b0808, 0x08082b08082b1919, 0x08082b0819080819, + 0x08082b0819081908, 0x08082b0819190808, 0x08082b0819192b08, 0x08082b082b080808, + 0x08082b082b2b0808, 0x08082b082b2b2b2b, 0x08082b1908080819, 0x08082b1908081908, + 0x08082b1908190808, 0x08082b1919080808, 0x08082b192b080819, 0x08082b192b082b19, + 0x08082b2b08080808, 0x08082b2b082b0808, 0x08082b2b082b2b08, 0x08082b2b2b19192b, + 0x08082b2b2b2b0808, 0x0819080808080819, 0x0819080808081908, 0x081908080808192b, + 0x0819080808082b19, 0x0819080808190808, 0x081908080819082b, 0x0819080808191919, + 0x0819080808192b08, 0x08190808082b0819, 0x08190808082b1908, 0x0819080819080808, + 0x081908081908082b, 0x0819080819081919, 0x0819080819082b08, 0x0819080819190819, + 0x0819080819191908, 0x08190808192b0808, 0x08190808192b2b2b, 0x081908082b080819, + 0x081908082b081908, 0x081908082b190808, 0x0819081908080808, 0x081908190808082b, + 0x0819081908081919, 0x0819081908082b08, 0x0819081908190819, 0x0819081908191908, + 0x08190819082b0808, 0x0819081919080819, 0x0819081919081908, 0x0819081919190808, + 0x081908192b080808, 0x081908192b191908, 0x081908192b19192b, 0x0819082b08080819, + 0x0819082b08081908, 0x0819082b0808192b, 0x0819082b08190808, 0x0819082b19080808, + 0x0819082b192b0808, 0x0819190808080808, 0x081919080808082b, 0x0819190808081919, + 0x0819190808082b08, 0x0819190808190819, 0x0819190808191908, 0x08191908082b0808, + 0x0819190819080819, 0x0819190819081908, 0x0819190819082b19, 0x0819190819190808, + 0x08191908192b1908, 0x081919082b080808, 0x0819191908080819, 0x0819191908081908, + 0x0819191908190808, 0x0819191919080808, 0x0819192b08080808, 0x0819192b08191908, + 0x0819192b19082b19, 0x08192b0808080819, 0x08192b0808081908, 0x08192b0808190808, + 0x08192b080819082b, 0x08192b0819080808, 0x08192b0819191908, 0x08192b082b08192b, + 0x08192b1908080808, 0x08192b1908081919, 0x08192b19192b192b, 0x08192b2b19190819, + 0x08192b2b2b2b2b19, 0x082b080808080808, 0x082b08080808082b, 0x082b080808081919, + 0x082b080808082b08, 0x082b080808082b2b, 0x082b080808190819, 0x082b080808191908, + 0x082b0808082b0808, 0x082b080819080819, 0x082b080819081908, 0x082b080819190808, + 0x082b08082b080808, 0x082b08082b2b0808, 0x082b081908080819, 0x082b081908081908, + 0x082b081908190808, 0x082b081919080808, 0x082b081919082b08, 0x082b0819192b1919, + 0x082b082b08080808, 0x082b082b082b082b, 0x082b082b2b080808, 0x082b082b2b2b2b08, + 0x082b190808080819, 0x082b190808081908, 0x082b190808190808, 0x082b1908082b2b19, + 0x082b190819080808, 0x082b191908080808, 0x082b191919080819, 0x082b19191919082b, + 0x082b19192b192b19, 0x082b192b08080819, 0x082b192b08192b2b, 0x082b192b2b2b192b, + 0x082b2b0808080808, 0x082b2b0808082b08, 0x082b2b0808082b2b, 0x082b2b08082b0808, + 0x082b2b0819191919, 0x082b2b082b082b08, 0x082b2b082b2b082b, 0x082b2b19192b2b08, + 0x082b2b192b190808, 0x082b2b2b08082b08, 0x082b2b2b082b0808, 0x082b2b2b2b08082b, + 0x082b2b2b2b082b08, 0x082b2b2b2b082b2b, 0x1908080808080819, 0x1908080808081908, + 0x190808080808192b, 0x1908080808082b19, 0x1908080808190808, 0x190808080819082b, + 0x1908080808191919, 0x1908080808192b08, 0x19080808082b0819, 0x19080808082b1908, + 0x1908080819080808, 0x190808081908082b, 0x1908080819081919, 0x1908080819082b08, + 0x1908080819082b2b, 0x1908080819190819, 0x1908080819191908, 0x19080808192b0808, + 0x19080808192b1919, 0x190808082b080819, 0x190808082b081908, 0x190808082b190808, + 0x1908081908080808, 0x190808190808082b, 0x1908081908081919, 0x1908081908082b08, + 0x1908081908190819, 0x1908081908191908, 0x19080819082b0808, 0x1908081919080819, + 0x1908081919081908, 0x1908081919190808, 0x190808192b080808, 0x190808192b081919, + 0x190808192b2b082b, 0x1908082b08080819, 0x1908082b08081908, 0x1908082b08190808, + 0x1908082b0819082b, 0x1908082b082b2b19, 0x1908082b19080808, 0x1908190808080808, + 0x190819080808082b, 0x1908190808081919, 0x1908190808082b08, 0x1908190808190819, + 0x1908190808191908, 0x1908190808192b19, 0x19081908082b0808, 0x1908190819080819, + 0x1908190819081908, 0x1908190819190808, 0x190819082b080808, 0x190819082b191908, + 0x1908191908080819, 0x1908191908081908, 0x1908191908190808, 0x19081919082b1908, + 0x1908191919080808, 0x190819192b192b2b, 0x1908192b08080808, 0x1908192b08082b2b, + 0x1908192b19081908, 0x1908192b19190808, 0x19082b0808080819, 0x19082b0808081908, + 0x19082b0808190808, 0x19082b0819080808, 0x19082b0819081919, 0x19082b0819191908, + 0x19082b08192b082b, 0x19082b1908080808, 0x19082b1908190819, 0x19082b1919081908, + 0x19082b1919190808, 0x19082b19192b2b19, 0x19082b2b08081908, 0x1919080808080808, + 0x191908080808082b, 0x1919080808081919, 0x1919080808082b08, 0x1919080808190819, + 0x1919080808191908, 0x19190808082b0808, 0x19190808082b2b08, 0x1919080819080819, + 0x1919080819081908, 0x1919080819190808, 0x191908082b080808, 0x1919081908080819, + 0x1919081908081908, 0x1919081908190808, 0x1919081908191919, 0x1919081919080808, + 0x191908191908082b, 0x1919082b08080808, 0x1919082b19081908, 0x1919082b2b2b2b2b, + 0x1919190808080819, 0x1919190808081908, 0x1919190808190808, 0x19191908082b0819, + 0x1919190819080808, 0x19191908192b0808, 0x191919082b080819, 0x191919082b2b0819, + 0x1919191908080808, 0x1919191908082b08, 0x191919192b080808, 0x191919192b082b08, + 0x1919192b082b0819, 0x1919192b192b2b08, 0x1919192b2b2b0819, 0x19192b0808080808, + 0x19192b0808191908, 0x19192b0819080819, 0x19192b0819190808, 0x19192b082b192b19, + 0x19192b1908192b2b, 0x19192b1919080808, 0x19192b191908082b, 0x19192b2b2b081919, + 0x192b080808080819, 0x192b080808081908, 0x192b080808190808, 0x192b080819080808, + 0x192b080819191908, 0x192b0808192b082b, 0x192b08082b08192b, 0x192b08082b2b2b19, + 0x192b081908080808, 0x192b082b082b1908, 0x192b082b19082b2b, 0x192b082b2b19082b, + 0x192b190808080808, 0x192b19080819192b, 0x192b191908190808, 0x192b191919080808, + 0x192b191919081919, 0x192b19192b2b1908, 0x192b2b0808080819, 0x192b2b08192b2b2b, + 0x192b2b19082b1919, 0x192b2b2b0808192b, 0x192b2b2b19191908, 0x192b2b2b192b082b, + 0x2b08080808080808, 0x2b0808080808082b, 0x2b08080808081919, 0x2b08080808082b08, + 0x2b08080808190819, 0x2b08080808191908, 0x2b080808082b0808, 0x2b080808082b2b2b, + 0x2b08080819080819, 0x2b08080819081908, 0x2b08080819190808, 0x2b0808082b080808, + 0x2b0808082b08082b, 0x2b0808082b2b2b08, 0x2b0808082b2b2b2b, 0x2b08081908080819, + 0x2b08081908081908, 0x2b0808190808192b, 0x2b08081908190808, 0x2b08081919080808, + 0x2b08081919190819, 0x2b08081919192b19, 0x2b08082b08080808, 0x2b08082b082b0808, + 0x2b08082b2b080808, 0x2b08082b2b08082b, 0x2b08082b2b2b0808, 0x2b08082b2b2b2b08, + 0x2b08190808080819, 0x2b08190808081908, 0x2b08190808190808, 0x2b0819080819082b, + 0x2b08190808191919, 0x2b08190819080808, 0x2b081908192b0808, 0x2b0819082b082b19, + 0x2b08191908080808, 0x2b08191919081908, 0x2b0819192b2b1919, 0x2b08192b08192b08, + 0x2b08192b192b2b2b, 0x2b082b0808080808, 0x2b082b0808082b08, 0x2b082b08082b1919, + 0x2b082b0819192b2b, 0x2b082b082b080808, 0x2b082b082b08082b, 0x2b082b082b2b2b08, + 0x2b082b190808192b, 0x2b082b2b082b082b, 0x2b082b2b2b080808, 0x2b082b2b2b082b08, + 0x2b082b2b2b19192b, 0x2b082b2b2b2b2b08, 0x2b19080808080819, 0x2b19080808081908, + 0x2b19080808190808, 0x2b19080819080808, 0x2b1908081919192b, 0x2b1908082b081908, + 0x2b19081908080808, 0x2b190819082b082b, 0x2b190819192b1908, 0x2b19082b1919192b, + 0x2b19082b2b082b19, 0x2b19190808080808, 0x2b19190808081919, 0x2b19190819081908, + 0x2b19190819190808, 0x2b19190819192b08, 0x2b191919082b2b19, 0x2b1919192b190808, + 0x2b1919192b19082b, 0x2b19192b19080819, 0x2b192b0819190819, 0x2b192b082b2b192b, + 0x2b192b1919082b19, 0x2b192b2b08191919, 0x2b192b2b192b0808, 0x2b2b080808080808, + 0x2b2b08080808082b, 0x2b2b080808082b08, 0x2b2b080808082b2b, 0x2b2b0808082b0808, + 0x2b2b0808082b2b2b, 0x2b2b08082b2b0808, 0x2b2b081919190819, 0x2b2b081919192b19, + 0x2b2b08192b2b192b, 0x2b2b082b08080808, 0x2b2b082b0808082b, 0x2b2b082b08082b08, + 0x2b2b082b082b2b2b, 0x2b2b082b2b080808, 0x2b2b082b2b2b0808, 0x2b2b190819080808, + 0x2b2b19082b191919, 0x2b2b192b192b1919, 0x2b2b192b2b192b08, 0x2b2b2b0808082b2b, + 0x2b2b2b08082b0808, 0x2b2b2b08082b082b, 0x2b2b2b08082b2b08, 0x2b2b2b082b2b0808, + 0x2b2b2b082b2b2b08, 0x2b2b2b1908081908, 0x2b2b2b192b081908, 0x2b2b2b192b08192b, + 0x2b2b2b2b082b2b08, 0x2b2b2b2b082b2b2b, 0x2b2b2b2b2b190819, 0x2b2b2b2b2b2b2b2b, +}; + static const __device__ uint8_t ksigns_iq2xs[128] = { 0, 129, 130, 3, 132, 5, 6, 135, 136, 9, 10, 139, 12, 141, 142, 15, 144, 17, 18, 147, 20, 149, 150, 23, 24, 153, 154, 27, 156, 29, 30, 159, @@ -1439,7 +1579,7 @@ static __global__ void dequantize_block_iq2_xxs(const void * __restrict__ vx, ds dst_t * y = yy + i*QK_K + 32*ib + 8*il; const uint16_t * q2 = x[i].qs + 4*ib; const uint8_t * aux8 = (const uint8_t *)q2; - const uint8_t * grid = (const uint8_t *)(kgrid_iq2xxs + aux8[il]); + const uint8_t * grid = (const uint8_t *)(iq2xxs_grid + aux8[il]); const uint32_t aux32 = q2[2] | (q2[3] << 16); const float d = (float)x[i].d * (0.5f + (aux32 >> 28)) * 0.25f; const uint8_t signs = ksigns_iq2xs[(aux32 >> 7*il) & 127]; @@ -1450,6 +1590,28 @@ static __global__ void dequantize_block_iq2_xxs(const void * __restrict__ vx, ds } +template +static __global__ void dequantize_block_iq2_xs(const void * __restrict__ vx, dst_t * __restrict__ yy) { + + const int i = blockIdx.x; + const block_iq2_xs * x = (const block_iq2_xs *) vx; + + const int tid = threadIdx.x; +#if QK_K == 256 + const int il = tid/8; // 0...3 + const int ib = tid%8; // 0...7 + dst_t * y = yy + i*QK_K + 32*ib + 8*il; + const uint16_t * q2 = x[i].qs + 4*ib; + const uint8_t * grid = (const uint8_t *)(iq2xs_grid + (q2[il] & 511)); + const float d = (float)x[i].d * (0.5f + ((x[i].scales[ib] >> 4*(il/2)) & 0xf)) * 0.25f; + const uint8_t signs = ksigns_iq2xs[q2[il] >> 9]; + for (int j = 0; j < 8; ++j) y[j] = d * grid[j] * (signs & kmask_iq2xs[j] ? -1.f : 1.f); +#else + assert(false); +#endif + +} + static __global__ void dequantize_mul_mat_vec_q2_k(const void * __restrict__ vx, const float * __restrict__ yy, float * __restrict__ dst, const int ncols, int nrows) { static_assert(16%K_QUANTS_PER_ITERATION == 0, "16 must be divisible by K_QUANTS_PER_ITERATION"); @@ -3996,7 +4158,7 @@ static __device__ __forceinline__ float vec_dot_iq2_xxs_q8_1( uint32_t aux32 = q2[2] | (q2[3] << 16); int sumi = 0; for (int l = 0; l < 4; ++l) { - const uint8_t * grid = (const uint8_t *)(kgrid_iq2xxs + aux8[l]); + const uint8_t * grid = (const uint8_t *)(iq2xxs_grid + aux8[l]); const uint8_t signs = ksigns_iq2xs[aux32 & 127]; for (int j = 0; j < 8; ++j) { sumi += q8[j] * grid[j] * (signs & kmask_iq2xs[j] ? -1 : 1); @@ -4012,8 +4174,8 @@ static __device__ __forceinline__ float vec_dot_iq2_xxs_q8_1( const int il = iqs%2; const uint16_t * q2 = bq2->qs + 4*ib32; const uint8_t * aux8 = (const uint8_t *)q2; - const uint8_t * grid1 = (const uint8_t *)(kgrid_iq2xxs + aux8[2*il+0]); - const uint8_t * grid2 = (const uint8_t *)(kgrid_iq2xxs + aux8[2*il+1]); + const uint8_t * grid1 = (const uint8_t *)(iq2xxs_grid + aux8[2*il+0]); + const uint8_t * grid2 = (const uint8_t *)(iq2xxs_grid + aux8[2*il+1]); const uint32_t aux32 = q2[2] | (q2[3] << 16); const float d = (float)bq2->d * (0.5f + (aux32 >> 28)) * (float)bq8_1[ib32].ds.x * 0.25f; const uint8_t signs1 = ksigns_iq2xs[(aux32 >> 14*il) & 127]; @@ -4032,6 +4194,42 @@ static __device__ __forceinline__ float vec_dot_iq2_xxs_q8_1( #endif } +static __device__ __forceinline__ float vec_dot_iq2_xs_q8_1( + const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) { +#if QK_K == 256 + const block_iq2_xs * bq2 = (const block_iq2_xs *) vbq; + + const int ib32 = iqs; + const uint16_t * q2 = bq2->qs + 4*ib32; + const int8_t * q8 = bq8_1[ib32].qs; + const uint8_t ls1 = bq2->scales[ib32] & 0xf; + const uint8_t ls2 = bq2->scales[ib32] >> 4; + int sumi1 = 0; + for (int l = 0; l < 2; ++l) { + const uint8_t * grid = (const uint8_t *)(iq2xs_grid + (q2[l] & 511)); + const uint8_t signs = ksigns_iq2xs[q2[l] >> 9]; + for (int j = 0; j < 8; ++j) { + sumi1 += q8[j] * grid[j] * (signs & kmask_iq2xs[j] ? -1 : 1); + } + q8 += 8; + } + int sumi2 = 0; + for (int l = 2; l < 4; ++l) { + const uint8_t * grid = (const uint8_t *)(iq2xs_grid + (q2[l] & 511)); + const uint8_t signs = ksigns_iq2xs[q2[l] >> 9]; + for (int j = 0; j < 8; ++j) { + sumi2 += q8[j] * grid[j] * (signs & kmask_iq2xs[j] ? -1 : 1); + } + q8 += 8; + } + const float d = (float)bq2->d * (float)bq8_1[ib32].ds.x * 0.25f; + return d * ((0.5f + ls1) * sumi1 + (0.5f + ls2) * sumi2); +#else + assert(false); + return 0.f; +#endif +} + template static __device__ __forceinline__ void mul_mat_q( @@ -6035,6 +6233,12 @@ static void dequantize_row_iq2_xxs_cuda(const void * vx, dst_t * y, const int k, dequantize_block_iq2_xxs<<>>(vx, y); } +template +static void dequantize_row_iq2_xs_cuda(const void * vx, dst_t * y, const int k, cudaStream_t stream) { + const int nb = k / QK_K; + dequantize_block_iq2_xs<<>>(vx, y); +} + template static void convert_unary_cuda(const void * __restrict__ vx, dst_t * __restrict__ y, const int k, cudaStream_t stream) { const int num_blocks = (k + CUDA_DEQUANTIZE_BLOCK_SIZE - 1) / CUDA_DEQUANTIZE_BLOCK_SIZE; @@ -6065,6 +6269,8 @@ static to_fp16_cuda_t ggml_get_to_fp16_cuda(ggml_type type) { return dequantize_row_q6_K_cuda; case GGML_TYPE_IQ2_XXS: return dequantize_row_iq2_xxs_cuda; + case GGML_TYPE_IQ2_XS: + return dequantize_row_iq2_xs_cuda; case GGML_TYPE_F32: return convert_unary_cuda; default: @@ -6096,6 +6302,8 @@ static to_fp32_cuda_t ggml_get_to_fp32_cuda(ggml_type type) { return dequantize_row_q6_K_cuda; case GGML_TYPE_IQ2_XXS: return dequantize_row_iq2_xxs_cuda; + case GGML_TYPE_IQ2_XS: + return dequantize_row_iq2_xs_cuda; case GGML_TYPE_F16: return convert_unary_cuda; default: @@ -6299,6 +6507,15 @@ static void mul_mat_vec_iq2_xxs_q8_1_cuda(const void * vx, const void * vy, floa <<>>(vx, vy, dst, ncols, nrows); } +static void mul_mat_vec_iq2_xs_q8_1_cuda(const void * vx, const void * vy, float * dst, const int ncols, const int nrows, cudaStream_t stream) { + GGML_ASSERT(ncols % QK_K == 0); + const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y; + const dim3 block_nums(block_num_y, 1, 1); + const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1); + mul_mat_vec_q + <<>>(vx, vy, dst, ncols, nrows); +} + static void ggml_mul_mat_q4_0_q8_1_cuda( const void * vx, const void * vy, float * dst, const int ncols_x, const int nrows_x, const int ncols_y, const int nrows_y, const int nrows_dst, cudaStream_t stream) { @@ -7871,6 +8088,7 @@ static int64_t get_row_rounding(ggml_type type) { case GGML_TYPE_Q5_K: case GGML_TYPE_Q6_K: case GGML_TYPE_IQ2_XXS: + case GGML_TYPE_IQ2_XS: return max_compute_capability >= CC_RDNA2 ? 128 : 64; default: GGML_ASSERT(false); @@ -7892,6 +8110,7 @@ static int64_t get_row_rounding(ggml_type type) { case GGML_TYPE_Q4_K: case GGML_TYPE_Q5_K: case GGML_TYPE_IQ2_XXS: + case GGML_TYPE_IQ2_XS: return max_compute_capability >= CC_VOLTA ? 128 : 64; case GGML_TYPE_Q6_K: return 64; @@ -7945,6 +8164,9 @@ static void ggml_cuda_op_mul_mat_vec_q( case GGML_TYPE_IQ2_XXS: mul_mat_vec_iq2_xxs_q8_1_cuda(src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, stream); break; + case GGML_TYPE_IQ2_XS: + mul_mat_vec_iq2_xs_q8_1_cuda(src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, stream); + break; default: GGML_ASSERT(false); break; diff --git a/ggml-metal.m b/ggml-metal.m index 9698e5a79..6e5594432 100644 --- a/ggml-metal.m +++ b/ggml-metal.m @@ -89,6 +89,7 @@ struct ggml_metal_context { GGML_METAL_DECL_KERNEL(get_rows_q6_K); GGML_METAL_DECL_KERNEL(get_rows_i32); GGML_METAL_DECL_KERNEL(get_rows_iq2_xxs); + GGML_METAL_DECL_KERNEL(get_rows_iq2_xs); GGML_METAL_DECL_KERNEL(rms_norm); GGML_METAL_DECL_KERNEL(group_norm); GGML_METAL_DECL_KERNEL(norm); @@ -108,6 +109,7 @@ struct ggml_metal_context { GGML_METAL_DECL_KERNEL(mul_mv_q5_K_f32); GGML_METAL_DECL_KERNEL(mul_mv_q6_K_f32); GGML_METAL_DECL_KERNEL(mul_mv_iq2_xxs_f32); + GGML_METAL_DECL_KERNEL(mul_mv_iq2_xs_f32); GGML_METAL_DECL_KERNEL(mul_mv_id_f32_f32); //GGML_METAL_DECL_KERNEL(mul_mv_id_f16_f16); GGML_METAL_DECL_KERNEL(mul_mv_id_f16_f32); @@ -124,6 +126,7 @@ struct ggml_metal_context { GGML_METAL_DECL_KERNEL(mul_mv_id_q5_K_f32); GGML_METAL_DECL_KERNEL(mul_mv_id_q6_K_f32); GGML_METAL_DECL_KERNEL(mul_mv_id_iq2_xxs_f32); + GGML_METAL_DECL_KERNEL(mul_mv_id_iq2_xs_f32); GGML_METAL_DECL_KERNEL(mul_mm_f32_f32); GGML_METAL_DECL_KERNEL(mul_mm_f16_f32); GGML_METAL_DECL_KERNEL(mul_mm_q4_0_f32); @@ -137,6 +140,7 @@ struct ggml_metal_context { GGML_METAL_DECL_KERNEL(mul_mm_q5_K_f32); GGML_METAL_DECL_KERNEL(mul_mm_q6_K_f32); GGML_METAL_DECL_KERNEL(mul_mm_iq2_xxs_f32); + GGML_METAL_DECL_KERNEL(mul_mm_iq2_xs_f32); GGML_METAL_DECL_KERNEL(mul_mm_id_f32_f32); GGML_METAL_DECL_KERNEL(mul_mm_id_f16_f32); GGML_METAL_DECL_KERNEL(mul_mm_id_q4_0_f32); @@ -150,6 +154,7 @@ struct ggml_metal_context { GGML_METAL_DECL_KERNEL(mul_mm_id_q5_K_f32); GGML_METAL_DECL_KERNEL(mul_mm_id_q6_K_f32); GGML_METAL_DECL_KERNEL(mul_mm_id_iq2_xxs_f32); + GGML_METAL_DECL_KERNEL(mul_mm_id_iq2_xs_f32); GGML_METAL_DECL_KERNEL(rope_f32); GGML_METAL_DECL_KERNEL(rope_f16); GGML_METAL_DECL_KERNEL(alibi_f32); @@ -385,6 +390,7 @@ struct ggml_metal_context * ggml_metal_init(int n_cb) { GGML_METAL_ADD_KERNEL(get_rows_q6_K); GGML_METAL_ADD_KERNEL(get_rows_i32); GGML_METAL_ADD_KERNEL(get_rows_iq2_xxs); + GGML_METAL_ADD_KERNEL(get_rows_iq2_xs); GGML_METAL_ADD_KERNEL(rms_norm); GGML_METAL_ADD_KERNEL(group_norm); GGML_METAL_ADD_KERNEL(norm); @@ -404,6 +410,7 @@ struct ggml_metal_context * ggml_metal_init(int n_cb) { GGML_METAL_ADD_KERNEL(mul_mv_q5_K_f32); GGML_METAL_ADD_KERNEL(mul_mv_q6_K_f32); GGML_METAL_ADD_KERNEL(mul_mv_iq2_xxs_f32); + GGML_METAL_ADD_KERNEL(mul_mv_iq2_xs_f32); GGML_METAL_ADD_KERNEL(mul_mv_id_f32_f32); //GGML_METAL_ADD_KERNEL(mul_mv_id_f16_f16); GGML_METAL_ADD_KERNEL(mul_mv_id_f16_f32); @@ -420,6 +427,7 @@ struct ggml_metal_context * ggml_metal_init(int n_cb) { GGML_METAL_ADD_KERNEL(mul_mv_id_q5_K_f32); GGML_METAL_ADD_KERNEL(mul_mv_id_q6_K_f32); GGML_METAL_ADD_KERNEL(mul_mv_id_iq2_xxs_f32); + GGML_METAL_ADD_KERNEL(mul_mv_id_iq2_xs_f32); if ([ctx->device supportsFamily:MTLGPUFamilyApple7]) { GGML_METAL_ADD_KERNEL(mul_mm_f32_f32); GGML_METAL_ADD_KERNEL(mul_mm_f16_f32); @@ -434,6 +442,7 @@ struct ggml_metal_context * ggml_metal_init(int n_cb) { GGML_METAL_ADD_KERNEL(mul_mm_q5_K_f32); GGML_METAL_ADD_KERNEL(mul_mm_q6_K_f32); GGML_METAL_ADD_KERNEL(mul_mm_iq2_xxs_f32); + GGML_METAL_ADD_KERNEL(mul_mm_iq2_xs_f32); GGML_METAL_ADD_KERNEL(mul_mm_id_f32_f32); GGML_METAL_ADD_KERNEL(mul_mm_id_f16_f32); GGML_METAL_ADD_KERNEL(mul_mm_id_q4_0_f32); @@ -447,6 +456,7 @@ struct ggml_metal_context * ggml_metal_init(int n_cb) { GGML_METAL_ADD_KERNEL(mul_mm_id_q5_K_f32); GGML_METAL_ADD_KERNEL(mul_mm_id_q6_K_f32); GGML_METAL_ADD_KERNEL(mul_mm_id_iq2_xxs_f32); + GGML_METAL_ADD_KERNEL(mul_mm_id_iq2_xs_f32); } GGML_METAL_ADD_KERNEL(rope_f32); GGML_METAL_ADD_KERNEL(rope_f16); @@ -513,6 +523,7 @@ void ggml_metal_free(struct ggml_metal_context * ctx) { GGML_METAL_DEL_KERNEL(get_rows_q6_K); GGML_METAL_DEL_KERNEL(get_rows_i32); GGML_METAL_DEL_KERNEL(get_rows_iq2_xxs); + GGML_METAL_DEL_KERNEL(get_rows_iq2_xs); GGML_METAL_DEL_KERNEL(rms_norm); GGML_METAL_DEL_KERNEL(group_norm); GGML_METAL_DEL_KERNEL(norm); @@ -532,6 +543,7 @@ void ggml_metal_free(struct ggml_metal_context * ctx) { GGML_METAL_DEL_KERNEL(mul_mv_q5_K_f32); GGML_METAL_DEL_KERNEL(mul_mv_q6_K_f32); GGML_METAL_DEL_KERNEL(mul_mv_iq2_xxs_f32); + GGML_METAL_DEL_KERNEL(mul_mv_iq2_xs_f32); GGML_METAL_DEL_KERNEL(mul_mv_id_f32_f32); //GGML_METAL_DEL_KERNEL(mul_mv_id_f16_f16); GGML_METAL_DEL_KERNEL(mul_mv_id_f16_f32); @@ -548,6 +560,7 @@ void ggml_metal_free(struct ggml_metal_context * ctx) { GGML_METAL_DEL_KERNEL(mul_mv_id_q5_K_f32); GGML_METAL_DEL_KERNEL(mul_mv_id_q6_K_f32); GGML_METAL_DEL_KERNEL(mul_mv_id_iq2_xxs_f32); + GGML_METAL_DEL_KERNEL(mul_mv_id_iq2_xs_f32); if ([ctx->device supportsFamily:MTLGPUFamilyApple7]) { GGML_METAL_DEL_KERNEL(mul_mm_f32_f32); GGML_METAL_DEL_KERNEL(mul_mm_f16_f32); @@ -562,6 +575,7 @@ void ggml_metal_free(struct ggml_metal_context * ctx) { GGML_METAL_DEL_KERNEL(mul_mm_q5_K_f32); GGML_METAL_DEL_KERNEL(mul_mm_q6_K_f32); GGML_METAL_DEL_KERNEL(mul_mm_iq2_xxs_f32); + GGML_METAL_DEL_KERNEL(mul_mm_iq2_xs_f32); GGML_METAL_DEL_KERNEL(mul_mm_id_f32_f32); GGML_METAL_DEL_KERNEL(mul_mm_id_f16_f32); GGML_METAL_DEL_KERNEL(mul_mm_id_q4_0_f32); @@ -575,6 +589,7 @@ void ggml_metal_free(struct ggml_metal_context * ctx) { GGML_METAL_DEL_KERNEL(mul_mm_id_q5_K_f32); GGML_METAL_DEL_KERNEL(mul_mm_id_q6_K_f32); GGML_METAL_DEL_KERNEL(mul_mm_id_iq2_xxs_f32); + GGML_METAL_DEL_KERNEL(mul_mm_id_iq2_xs_f32); } GGML_METAL_DEL_KERNEL(rope_f32); GGML_METAL_DEL_KERNEL(rope_f16); @@ -1561,6 +1576,7 @@ bool ggml_metal_graph_compute( case GGML_TYPE_Q5_K: [encoder setComputePipelineState:ctx->pipeline_mul_mm_q5_K_f32]; break; case GGML_TYPE_Q6_K: [encoder setComputePipelineState:ctx->pipeline_mul_mm_q6_K_f32]; break; case GGML_TYPE_IQ2_XXS: [encoder setComputePipelineState:ctx->pipeline_mul_mm_iq2_xxs_f32]; break; + case GGML_TYPE_IQ2_XS : [encoder setComputePipelineState:ctx->pipeline_mul_mm_iq2_xs_f32]; break; default: GGML_ASSERT(false && "MUL MAT-MAT not implemented"); } [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0]; @@ -1679,6 +1695,12 @@ bool ggml_metal_graph_compute( nth1 = 16; [encoder setComputePipelineState:ctx->pipeline_mul_mv_iq2_xxs_f32]; } break; + case GGML_TYPE_IQ2_XS: + { + nth0 = 4; + nth1 = 16; + [encoder setComputePipelineState:ctx->pipeline_mul_mv_iq2_xs_f32]; + } break; default: { GGML_METAL_LOG_ERROR("Asserting on type %d\n", (int)src0t); @@ -1712,12 +1734,12 @@ bool ggml_metal_graph_compute( if (src0t == GGML_TYPE_Q4_0 || src0t == GGML_TYPE_Q4_1 || src0t == GGML_TYPE_Q5_0 || src0t == GGML_TYPE_Q5_1 || src0t == GGML_TYPE_Q8_0 || - //src0t == GGML_TYPE_IQ2_XXS || src0t == GGML_TYPE_Q2_K) { // || src0t == GGML_TYPE_Q4_K) { [encoder dispatchThreadgroups:MTLSizeMake((ne01 + 7)/8, ne11, ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)]; } - else if (src0t == GGML_TYPE_IQ2_XXS) { - [encoder setThreadgroupMemoryLength:(256*8+128) atIndex:0]; + else if (src0t == GGML_TYPE_IQ2_XXS || src0t == GGML_TYPE_IQ2_XS) { + const int mem_size = src0t == GGML_TYPE_IQ2_XXS ? 256*8+128 : 512*8+128; + [encoder setThreadgroupMemoryLength:mem_size atIndex:0]; [encoder dispatchThreadgroups:MTLSizeMake((ne01 + 7)/8, ne11, ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)]; } else if (src0t == GGML_TYPE_Q4_K) { @@ -1810,6 +1832,7 @@ bool ggml_metal_graph_compute( case GGML_TYPE_Q5_K: [encoder setComputePipelineState:ctx->pipeline_mul_mm_id_q5_K_f32]; break; case GGML_TYPE_Q6_K: [encoder setComputePipelineState:ctx->pipeline_mul_mm_id_q6_K_f32]; break; case GGML_TYPE_IQ2_XXS: [encoder setComputePipelineState:ctx->pipeline_mul_mm_id_iq2_xxs_f32]; break; + case GGML_TYPE_IQ2_XS : [encoder setComputePipelineState:ctx->pipeline_mul_mm_id_iq2_xs_f32]; break; default: GGML_ASSERT(false && "MUL_MAT_ID not implemented"); } [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0]; @@ -1931,6 +1954,12 @@ bool ggml_metal_graph_compute( nth1 = 16; [encoder setComputePipelineState:ctx->pipeline_mul_mv_id_iq2_xxs_f32]; } break; + case GGML_TYPE_IQ2_XS: + { + nth0 = 4; + nth1 = 16; + [encoder setComputePipelineState:ctx->pipeline_mul_mv_id_iq2_xs_f32]; + } break; default: { GGML_METAL_LOG_ERROR("Asserting on type %d\n", (int)src2t); @@ -1980,12 +2009,12 @@ bool ggml_metal_graph_compute( if (src2t == GGML_TYPE_Q4_0 || src2t == GGML_TYPE_Q4_1 || src2t == GGML_TYPE_Q5_0 || src2t == GGML_TYPE_Q5_1 || src2t == GGML_TYPE_Q8_0 || - //src2t == GGML_TYPE_IQ2_XXS || src2t == GGML_TYPE_Q2_K) { // || src2t == GGML_TYPE_Q4_K) { [encoder dispatchThreadgroups:MTLSizeMake((ne21 + 7)/8, _ne1, ne01*ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)]; } - else if (src2t == GGML_TYPE_IQ2_XXS) { - [encoder setThreadgroupMemoryLength:(256*8+128) atIndex:0]; + else if (src2t == GGML_TYPE_IQ2_XXS || src2t == GGML_TYPE_IQ2_XS) { + const int mem_size = src2t == GGML_TYPE_IQ2_XXS ? 256*8+128 : 512*8+128; + [encoder setThreadgroupMemoryLength:mem_size atIndex:0]; [encoder dispatchThreadgroups:MTLSizeMake((ne21 + 7)/8, _ne1, ne01*ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)]; } else if (src2t == GGML_TYPE_Q4_K) { @@ -2026,6 +2055,7 @@ bool ggml_metal_graph_compute( case GGML_TYPE_Q6_K: [encoder setComputePipelineState:ctx->pipeline_get_rows_q6_K]; break; case GGML_TYPE_I32: [encoder setComputePipelineState:ctx->pipeline_get_rows_i32]; break; case GGML_TYPE_IQ2_XXS: [encoder setComputePipelineState:ctx->pipeline_get_rows_iq2_xxs]; break; + case GGML_TYPE_IQ2_XS : [encoder setComputePipelineState:ctx->pipeline_get_rows_iq2_xs]; break; default: GGML_ASSERT(false && "not implemented"); } diff --git a/ggml-metal.metal b/ggml-metal.metal index 229efb8b6..029578dc5 100644 --- a/ggml-metal.metal +++ b/ggml-metal.metal @@ -2452,6 +2452,13 @@ typedef struct { } block_iq2_xxs; // 66 bytes / block for QK_K = 256, so 2.0625 bpw +typedef struct { + half d; + uint16_t qs[QK_K/8]; + uint8_t scales[QK_K/32]; +} block_iq2_xs; +// 74 bytes / block for QK_K = 256, so 2.3125 bpw + //====================================== dot products ========================= void kernel_mul_mv_q2_K_f32_impl( @@ -3476,7 +3483,7 @@ kernel void kernel_mul_mv_q6_K_f32( // ======================= "True" 2-bit -constexpr constant static uint64_t kgrid_iq2xxs[256] = { +constexpr constant static uint64_t iq2xxs_grid[256] = { 0x0808080808080808, 0x080808080808082b, 0x0808080808081919, 0x0808080808082b08, 0x0808080808082b2b, 0x0808080808190819, 0x0808080808191908, 0x08080808082b0808, 0x08080808082b082b, 0x08080808082b2b08, 0x08080808082b2b2b, 0x0808080819080819, @@ -3543,6 +3550,137 @@ constexpr constant static uint64_t kgrid_iq2xxs[256] = { 0x2b2b082b08080808, 0x2b2b190808192b08, 0x2b2b2b0819190808, 0x2b2b2b1908081908, }; +constexpr constant static uint64_t iq2xs_grid[512] = { + 0x0808080808080808, 0x080808080808082b, 0x0808080808081919, 0x0808080808082b08, + 0x0808080808082b2b, 0x0808080808190819, 0x0808080808191908, 0x080808080819192b, + 0x0808080808192b19, 0x08080808082b0808, 0x08080808082b082b, 0x08080808082b1919, + 0x08080808082b2b08, 0x0808080819080819, 0x0808080819081908, 0x080808081908192b, + 0x0808080819082b19, 0x0808080819190808, 0x080808081919082b, 0x0808080819191919, + 0x0808080819192b08, 0x08080808192b0819, 0x08080808192b1908, 0x080808082b080808, + 0x080808082b08082b, 0x080808082b081919, 0x080808082b082b08, 0x080808082b190819, + 0x080808082b191908, 0x080808082b192b19, 0x080808082b2b0808, 0x0808081908080819, + 0x0808081908081908, 0x080808190808192b, 0x0808081908082b19, 0x0808081908190808, + 0x080808190819082b, 0x0808081908191919, 0x0808081908192b08, 0x0808081908192b2b, + 0x08080819082b0819, 0x08080819082b1908, 0x0808081919080808, 0x080808191908082b, + 0x0808081919081919, 0x0808081919082b08, 0x0808081919190819, 0x0808081919191908, + 0x08080819192b0808, 0x08080819192b2b08, 0x080808192b080819, 0x080808192b081908, + 0x080808192b190808, 0x0808082b08080808, 0x0808082b0808082b, 0x0808082b08081919, + 0x0808082b08082b08, 0x0808082b08190819, 0x0808082b08191908, 0x0808082b082b0808, + 0x0808082b19080819, 0x0808082b19081908, 0x0808082b19190808, 0x0808082b19191919, + 0x0808082b2b080808, 0x0808082b2b082b2b, 0x0808190808080819, 0x0808190808081908, + 0x080819080808192b, 0x0808190808082b19, 0x0808190808190808, 0x080819080819082b, + 0x0808190808191919, 0x0808190808192b08, 0x08081908082b0819, 0x08081908082b1908, + 0x0808190819080808, 0x080819081908082b, 0x0808190819081919, 0x0808190819082b08, + 0x0808190819190819, 0x0808190819191908, 0x080819081919192b, 0x08081908192b0808, + 0x080819082b080819, 0x080819082b081908, 0x080819082b190808, 0x0808191908080808, + 0x080819190808082b, 0x0808191908081919, 0x0808191908082b08, 0x0808191908190819, + 0x0808191908191908, 0x08081919082b0808, 0x0808191919080819, 0x0808191919081908, + 0x0808191919190808, 0x08081919192b0819, 0x080819192b080808, 0x0808192b08080819, + 0x0808192b08081908, 0x0808192b08190808, 0x0808192b082b192b, 0x0808192b19080808, + 0x0808192b1908082b, 0x0808192b2b081908, 0x08082b0808080808, 0x08082b080808082b, + 0x08082b0808081919, 0x08082b0808082b08, 0x08082b0808082b2b, 0x08082b0808190819, + 0x08082b0808191908, 0x08082b08082b0808, 0x08082b08082b1919, 0x08082b0819080819, + 0x08082b0819081908, 0x08082b0819190808, 0x08082b0819192b08, 0x08082b082b080808, + 0x08082b082b2b0808, 0x08082b082b2b2b2b, 0x08082b1908080819, 0x08082b1908081908, + 0x08082b1908190808, 0x08082b1919080808, 0x08082b192b080819, 0x08082b192b082b19, + 0x08082b2b08080808, 0x08082b2b082b0808, 0x08082b2b082b2b08, 0x08082b2b2b19192b, + 0x08082b2b2b2b0808, 0x0819080808080819, 0x0819080808081908, 0x081908080808192b, + 0x0819080808082b19, 0x0819080808190808, 0x081908080819082b, 0x0819080808191919, + 0x0819080808192b08, 0x08190808082b0819, 0x08190808082b1908, 0x0819080819080808, + 0x081908081908082b, 0x0819080819081919, 0x0819080819082b08, 0x0819080819190819, + 0x0819080819191908, 0x08190808192b0808, 0x08190808192b2b2b, 0x081908082b080819, + 0x081908082b081908, 0x081908082b190808, 0x0819081908080808, 0x081908190808082b, + 0x0819081908081919, 0x0819081908082b08, 0x0819081908190819, 0x0819081908191908, + 0x08190819082b0808, 0x0819081919080819, 0x0819081919081908, 0x0819081919190808, + 0x081908192b080808, 0x081908192b191908, 0x081908192b19192b, 0x0819082b08080819, + 0x0819082b08081908, 0x0819082b0808192b, 0x0819082b08190808, 0x0819082b19080808, + 0x0819082b192b0808, 0x0819190808080808, 0x081919080808082b, 0x0819190808081919, + 0x0819190808082b08, 0x0819190808190819, 0x0819190808191908, 0x08191908082b0808, + 0x0819190819080819, 0x0819190819081908, 0x0819190819082b19, 0x0819190819190808, + 0x08191908192b1908, 0x081919082b080808, 0x0819191908080819, 0x0819191908081908, + 0x0819191908190808, 0x0819191919080808, 0x0819192b08080808, 0x0819192b08191908, + 0x0819192b19082b19, 0x08192b0808080819, 0x08192b0808081908, 0x08192b0808190808, + 0x08192b080819082b, 0x08192b0819080808, 0x08192b0819191908, 0x08192b082b08192b, + 0x08192b1908080808, 0x08192b1908081919, 0x08192b19192b192b, 0x08192b2b19190819, + 0x08192b2b2b2b2b19, 0x082b080808080808, 0x082b08080808082b, 0x082b080808081919, + 0x082b080808082b08, 0x082b080808082b2b, 0x082b080808190819, 0x082b080808191908, + 0x082b0808082b0808, 0x082b080819080819, 0x082b080819081908, 0x082b080819190808, + 0x082b08082b080808, 0x082b08082b2b0808, 0x082b081908080819, 0x082b081908081908, + 0x082b081908190808, 0x082b081919080808, 0x082b081919082b08, 0x082b0819192b1919, + 0x082b082b08080808, 0x082b082b082b082b, 0x082b082b2b080808, 0x082b082b2b2b2b08, + 0x082b190808080819, 0x082b190808081908, 0x082b190808190808, 0x082b1908082b2b19, + 0x082b190819080808, 0x082b191908080808, 0x082b191919080819, 0x082b19191919082b, + 0x082b19192b192b19, 0x082b192b08080819, 0x082b192b08192b2b, 0x082b192b2b2b192b, + 0x082b2b0808080808, 0x082b2b0808082b08, 0x082b2b0808082b2b, 0x082b2b08082b0808, + 0x082b2b0819191919, 0x082b2b082b082b08, 0x082b2b082b2b082b, 0x082b2b19192b2b08, + 0x082b2b192b190808, 0x082b2b2b08082b08, 0x082b2b2b082b0808, 0x082b2b2b2b08082b, + 0x082b2b2b2b082b08, 0x082b2b2b2b082b2b, 0x1908080808080819, 0x1908080808081908, + 0x190808080808192b, 0x1908080808082b19, 0x1908080808190808, 0x190808080819082b, + 0x1908080808191919, 0x1908080808192b08, 0x19080808082b0819, 0x19080808082b1908, + 0x1908080819080808, 0x190808081908082b, 0x1908080819081919, 0x1908080819082b08, + 0x1908080819082b2b, 0x1908080819190819, 0x1908080819191908, 0x19080808192b0808, + 0x19080808192b1919, 0x190808082b080819, 0x190808082b081908, 0x190808082b190808, + 0x1908081908080808, 0x190808190808082b, 0x1908081908081919, 0x1908081908082b08, + 0x1908081908190819, 0x1908081908191908, 0x19080819082b0808, 0x1908081919080819, + 0x1908081919081908, 0x1908081919190808, 0x190808192b080808, 0x190808192b081919, + 0x190808192b2b082b, 0x1908082b08080819, 0x1908082b08081908, 0x1908082b08190808, + 0x1908082b0819082b, 0x1908082b082b2b19, 0x1908082b19080808, 0x1908190808080808, + 0x190819080808082b, 0x1908190808081919, 0x1908190808082b08, 0x1908190808190819, + 0x1908190808191908, 0x1908190808192b19, 0x19081908082b0808, 0x1908190819080819, + 0x1908190819081908, 0x1908190819190808, 0x190819082b080808, 0x190819082b191908, + 0x1908191908080819, 0x1908191908081908, 0x1908191908190808, 0x19081919082b1908, + 0x1908191919080808, 0x190819192b192b2b, 0x1908192b08080808, 0x1908192b08082b2b, + 0x1908192b19081908, 0x1908192b19190808, 0x19082b0808080819, 0x19082b0808081908, + 0x19082b0808190808, 0x19082b0819080808, 0x19082b0819081919, 0x19082b0819191908, + 0x19082b08192b082b, 0x19082b1908080808, 0x19082b1908190819, 0x19082b1919081908, + 0x19082b1919190808, 0x19082b19192b2b19, 0x19082b2b08081908, 0x1919080808080808, + 0x191908080808082b, 0x1919080808081919, 0x1919080808082b08, 0x1919080808190819, + 0x1919080808191908, 0x19190808082b0808, 0x19190808082b2b08, 0x1919080819080819, + 0x1919080819081908, 0x1919080819190808, 0x191908082b080808, 0x1919081908080819, + 0x1919081908081908, 0x1919081908190808, 0x1919081908191919, 0x1919081919080808, + 0x191908191908082b, 0x1919082b08080808, 0x1919082b19081908, 0x1919082b2b2b2b2b, + 0x1919190808080819, 0x1919190808081908, 0x1919190808190808, 0x19191908082b0819, + 0x1919190819080808, 0x19191908192b0808, 0x191919082b080819, 0x191919082b2b0819, + 0x1919191908080808, 0x1919191908082b08, 0x191919192b080808, 0x191919192b082b08, + 0x1919192b082b0819, 0x1919192b192b2b08, 0x1919192b2b2b0819, 0x19192b0808080808, + 0x19192b0808191908, 0x19192b0819080819, 0x19192b0819190808, 0x19192b082b192b19, + 0x19192b1908192b2b, 0x19192b1919080808, 0x19192b191908082b, 0x19192b2b2b081919, + 0x192b080808080819, 0x192b080808081908, 0x192b080808190808, 0x192b080819080808, + 0x192b080819191908, 0x192b0808192b082b, 0x192b08082b08192b, 0x192b08082b2b2b19, + 0x192b081908080808, 0x192b082b082b1908, 0x192b082b19082b2b, 0x192b082b2b19082b, + 0x192b190808080808, 0x192b19080819192b, 0x192b191908190808, 0x192b191919080808, + 0x192b191919081919, 0x192b19192b2b1908, 0x192b2b0808080819, 0x192b2b08192b2b2b, + 0x192b2b19082b1919, 0x192b2b2b0808192b, 0x192b2b2b19191908, 0x192b2b2b192b082b, + 0x2b08080808080808, 0x2b0808080808082b, 0x2b08080808081919, 0x2b08080808082b08, + 0x2b08080808190819, 0x2b08080808191908, 0x2b080808082b0808, 0x2b080808082b2b2b, + 0x2b08080819080819, 0x2b08080819081908, 0x2b08080819190808, 0x2b0808082b080808, + 0x2b0808082b08082b, 0x2b0808082b2b2b08, 0x2b0808082b2b2b2b, 0x2b08081908080819, + 0x2b08081908081908, 0x2b0808190808192b, 0x2b08081908190808, 0x2b08081919080808, + 0x2b08081919190819, 0x2b08081919192b19, 0x2b08082b08080808, 0x2b08082b082b0808, + 0x2b08082b2b080808, 0x2b08082b2b08082b, 0x2b08082b2b2b0808, 0x2b08082b2b2b2b08, + 0x2b08190808080819, 0x2b08190808081908, 0x2b08190808190808, 0x2b0819080819082b, + 0x2b08190808191919, 0x2b08190819080808, 0x2b081908192b0808, 0x2b0819082b082b19, + 0x2b08191908080808, 0x2b08191919081908, 0x2b0819192b2b1919, 0x2b08192b08192b08, + 0x2b08192b192b2b2b, 0x2b082b0808080808, 0x2b082b0808082b08, 0x2b082b08082b1919, + 0x2b082b0819192b2b, 0x2b082b082b080808, 0x2b082b082b08082b, 0x2b082b082b2b2b08, + 0x2b082b190808192b, 0x2b082b2b082b082b, 0x2b082b2b2b080808, 0x2b082b2b2b082b08, + 0x2b082b2b2b19192b, 0x2b082b2b2b2b2b08, 0x2b19080808080819, 0x2b19080808081908, + 0x2b19080808190808, 0x2b19080819080808, 0x2b1908081919192b, 0x2b1908082b081908, + 0x2b19081908080808, 0x2b190819082b082b, 0x2b190819192b1908, 0x2b19082b1919192b, + 0x2b19082b2b082b19, 0x2b19190808080808, 0x2b19190808081919, 0x2b19190819081908, + 0x2b19190819190808, 0x2b19190819192b08, 0x2b191919082b2b19, 0x2b1919192b190808, + 0x2b1919192b19082b, 0x2b19192b19080819, 0x2b192b0819190819, 0x2b192b082b2b192b, + 0x2b192b1919082b19, 0x2b192b2b08191919, 0x2b192b2b192b0808, 0x2b2b080808080808, + 0x2b2b08080808082b, 0x2b2b080808082b08, 0x2b2b080808082b2b, 0x2b2b0808082b0808, + 0x2b2b0808082b2b2b, 0x2b2b08082b2b0808, 0x2b2b081919190819, 0x2b2b081919192b19, + 0x2b2b08192b2b192b, 0x2b2b082b08080808, 0x2b2b082b0808082b, 0x2b2b082b08082b08, + 0x2b2b082b082b2b2b, 0x2b2b082b2b080808, 0x2b2b082b2b2b0808, 0x2b2b190819080808, + 0x2b2b19082b191919, 0x2b2b192b192b1919, 0x2b2b192b2b192b08, 0x2b2b2b0808082b2b, + 0x2b2b2b08082b0808, 0x2b2b2b08082b082b, 0x2b2b2b08082b2b08, 0x2b2b2b082b2b0808, + 0x2b2b2b082b2b2b08, 0x2b2b2b1908081908, 0x2b2b2b192b081908, 0x2b2b2b192b08192b, + 0x2b2b2b2b082b2b08, 0x2b2b2b2b082b2b2b, 0x2b2b2b2b2b190819, 0x2b2b2b2b2b2b2b2b, +}; + constexpr constant static uint8_t ksigns_iq2xs[128] = { 0, 129, 130, 3, 132, 5, 6, 135, 136, 9, 10, 139, 12, 141, 142, 15, 144, 17, 18, 147, 20, 149, 150, 23, 24, 153, 154, 27, 156, 29, 30, 159, @@ -3600,7 +3738,7 @@ void kernel_mul_mv_iq2_xxs_f32_impl( { int nval = 4; int pos = (32*sgitg + tiisg)*nval; - for (int i = 0; i < nval; ++i) values[pos + i] = kgrid_iq2xxs[pos + i]; + for (int i = 0; i < nval; ++i) values[pos + i] = iq2xxs_grid[pos + i]; nval = 2; pos = (32*sgitg + tiisg)*nval; for (int i = 0; i < nval; ++i) shared_signs[pos+i] = ksigns_iq2xs[pos+i]; @@ -3689,6 +3827,149 @@ kernel void kernel_mul_mv_iq2_xxs_f32( kernel_mul_mv_iq2_xxs_f32_impl(src0, src1, dst, ne00, ne01, ne02, ne10, ne12, ne0, ne1, r2, r3, shared_values, tgpig, tiisg, sgitg); } +void kernel_mul_mv_iq2_xs_f32_impl( + device const void * src0, + device const float * src1, + device float * dst, + constant int64_t & ne00, + constant int64_t & ne01, + constant int64_t & ne02, + constant int64_t & ne10, + constant int64_t & ne12, + constant int64_t & ne0, + constant int64_t & ne1, + constant uint & r2, + constant uint & r3, + threadgroup int8_t * shared_values [[threadgroup(0)]], + uint3 tgpig[[threadgroup_position_in_grid]], + uint tiisg[[thread_index_in_simdgroup]], + uint sgitg[[simdgroup_index_in_threadgroup]]) { + + const int nb = ne00/QK_K; + const int r0 = tgpig.x; + const int r1 = tgpig.y; + const int im = tgpig.z; + + const int first_row = (r0 * N_SIMDGROUP + sgitg) * N_DST; + const int ib_row = first_row * nb; + + const uint i12 = im%ne12; + const uint i13 = im/ne12; + + const uint offset0 = (i12/r2)*(nb*ne01) + (i13/r3)*(nb*ne01*ne02); + + device const block_iq2_xs * x = (device const block_iq2_xs *) src0 + ib_row + offset0; + device const float * y = (device const float *) src1 + r1*ne10 + im*ne00*ne1; + + float yl[32]; + float sumf[N_DST]={0.f}, all_sum; + + const int nb32 = nb * (QK_K / 32); + + threadgroup uint64_t * values = (threadgroup uint64_t *)shared_values; + threadgroup uint8_t * shared_signs = (threadgroup uint8_t *)(values + 512); + { + int nval = 8; + int pos = (32*sgitg + tiisg)*nval; + for (int i = 0; i < nval; ++i) values[pos + i] = iq2xs_grid[pos + i]; + nval = 2; + pos = (32*sgitg + tiisg)*nval; + for (int i = 0; i < nval; ++i) shared_signs[pos+i] = ksigns_iq2xs[pos+i]; + threadgroup_barrier(mem_flags::mem_threadgroup); + } + +#if QK_K == 256 + const int ix = tiisg; + + device const float * y4 = y + 32 * ix; + + for (int ib32 = ix; ib32 < nb32; ib32 += 32) { + + for (int i = 0; i < 32; ++i) { + yl[i] = y4[i]; + } + + const int ibl = ib32 / (QK_K / 32); + const int ib = ib32 % (QK_K / 32); + + device const block_iq2_xs * xr = x + ibl; + device const uint16_t * q2 = xr->qs + 4 * ib; + device const uint8_t * sc = xr->scales + ib; + device const half * dh = &xr->d; + + for (int row = 0; row < N_DST; row++) { + + const float db = dh[0]; + const uint8_t ls1 = sc[0] & 0xf; + const uint8_t ls2 = sc[0] >> 4; + const float d1 = db * (0.5f + ls1); + const float d2 = db * (0.5f + ls2); + + float sum1 = 0, sum2 = 0; + for (int l = 0; l < 2; ++l) { + const threadgroup uint8_t * grid = (const threadgroup uint8_t *)(values + (q2[l] & 511)); + const uint8_t signs = shared_signs[(q2[l] >> 9)]; + for (int j = 0; j < 8; ++j) { + sum1 += yl[8*l + j] * grid[j] * (signs & kmask_iq2xs[j] ? -1.f : 1.f); + } + } + for (int l = 2; l < 4; ++l) { + const threadgroup uint8_t * grid = (const threadgroup uint8_t *)(values + (q2[l] & 511)); + const uint8_t signs = shared_signs[(q2[l] >> 9)]; + for (int j = 0; j < 8; ++j) { + sum2 += yl[8*l + j] * grid[j] * (signs & kmask_iq2xs[j] ? -1.f : 1.f); + } + } + sumf[row] += d1 * sum1 + d2 * sum2; + + dh += nb*sizeof(block_iq2_xs)/2; + q2 += nb*sizeof(block_iq2_xs)/2; + sc += nb*sizeof(block_iq2_xs); + } + + y4 += 32 * 32; + } +#else + // TODO +#endif + + for (int row = 0; row < N_DST; ++row) { + all_sum = simd_sum(sumf[row]); + if (tiisg == 0) { + dst[r1*ne0 + im*ne0*ne1 + first_row + row] = all_sum * 0.25f; + } + } +} + +[[host_name("kernel_mul_mv_iq2_xs_f32")]] +kernel void kernel_mul_mv_iq2_xs_f32( + device const void * src0, + device const float * src1, + device float * dst, + constant int64_t & ne00, + constant int64_t & ne01, + constant int64_t & ne02, + constant uint64_t & nb00, + constant uint64_t & nb01, + constant uint64_t & nb02, + constant int64_t & ne10, + constant int64_t & ne11, + constant int64_t & ne12, + constant uint64_t & nb10, + constant uint64_t & nb11, + constant uint64_t & nb12, + constant int64_t & ne0, + constant int64_t & ne1, + constant uint & r2, + constant uint & r3, + threadgroup int8_t * shared_values [[threadgroup(0)]], + uint3 tgpig[[threadgroup_position_in_grid]], + uint tiisg[[thread_index_in_simdgroup]], + uint sgitg[[simdgroup_index_in_threadgroup]]) { + + kernel_mul_mv_iq2_xs_f32_impl(src0, src1, dst, ne00, ne01, ne02, ne10, ne12, ne0, ne1, r2, r3, shared_values, tgpig, tiisg, sgitg); +} + //============================= templates and their specializations ============================= // NOTE: this is not dequantizing - we are simply fitting the template @@ -3973,18 +4254,39 @@ void dequantize_iq2_xxs(device const block_iq2_xxs * xb, short il, thread type4x const uint32_t aux32_s = q2[2] | (q2[3] << 16); thread const uint8_t * aux8 = (thread const uint8_t *)&aux32_g; const float dl = d * (0.5f + (aux32_s >> 28)) * 0.25f; - constant uint8_t * grid = (constant uint8_t *)(kgrid_iq2xxs + aux8[2*il+0]); + constant uint8_t * grid = (constant uint8_t *)(iq2xxs_grid + aux8[2*il+0]); uint8_t signs = ksigns_iq2xs[(aux32_s >> 14*il) & 127]; for (int i = 0; i < 8; ++i) { reg[i/4][i%4] = dl * grid[i] * (signs & kmask_iq2xs[i] ? -1.f : 1.f); } - grid = (constant uint8_t *)(kgrid_iq2xxs + aux8[2*il+1]); + grid = (constant uint8_t *)(iq2xxs_grid + aux8[2*il+1]); signs = ksigns_iq2xs[(aux32_s >> (14*il+7)) & 127]; for (int i = 0; i < 8; ++i) { reg[2+i/4][i%4] = dl * grid[i] * (signs & kmask_iq2xs[i] ? -1.f : 1.f); } } +template +void dequantize_iq2_xs(device const block_iq2_xs * xb, short il, thread type4x4 & reg) { + // il is 0...15 for QK_K = 256 => index of block of 32 is il/2 + const float d = xb->d; + const int ib32 = il/2; + il = il%2; + // il = 0 or 1. il = 0 processes the first 16 quants in a block of 32, il = 1 the second 16 + device const uint16_t * q2 = xb->qs + 4*ib32; + const float dl = d * (0.5f + ((xb->scales[ib32] >> 4*il) & 0xf)) * 0.25f; + constant uint8_t * grid = (constant uint8_t *)(iq2xs_grid + (q2[2*il+0] & 511)); + uint8_t signs = ksigns_iq2xs[q2[2*il+0] >> 9]; + for (int i = 0; i < 8; ++i) { + reg[i/4][i%4] = dl * grid[i] * (signs & kmask_iq2xs[i] ? -1.f : 1.f); + } + grid = (constant uint8_t *)(iq2xs_grid + (q2[2*il+1] & 511)); + signs = ksigns_iq2xs[q2[2*il+1] >> 9]; + for (int i = 0; i < 8; ++i) { + reg[2+i/4][i%4] = dl * grid[i] * (signs & kmask_iq2xs[i] ? -1.f : 1.f); + } +} + template kernel void kernel_get_rows( device const void * src0, @@ -4525,6 +4827,7 @@ template [[host_name("kernel_get_rows_q4_K")]] kernel get_rows_t kernel_get_rows template [[host_name("kernel_get_rows_q5_K")]] kernel get_rows_t kernel_get_rows; template [[host_name("kernel_get_rows_q6_K")]] kernel get_rows_t kernel_get_rows; template [[host_name("kernel_get_rows_iq2_xxs")]] kernel get_rows_t kernel_get_rows; +template [[host_name("kernel_get_rows_iq2_xs")]] kernel get_rows_t kernel_get_rows; // // matrix-matrix multiplication @@ -4562,6 +4865,7 @@ template [[host_name("kernel_mul_mm_q4_K_f32")]] kernel mat_mm_t kernel_mul_mm; template [[host_name("kernel_mul_mm_q6_K_f32")]] kernel mat_mm_t kernel_mul_mm; template [[host_name("kernel_mul_mm_iq2_xxs_f32")]] kernel mat_mm_t kernel_mul_mm; +template [[host_name("kernel_mul_mm_iq2_xs_f32")]] kernel mat_mm_t kernel_mul_mm; // // indirect matrix-matrix multiplication @@ -4611,6 +4915,7 @@ template [[host_name("kernel_mul_mm_id_q4_K_f32")]] kernel mat_mm_id_t kernel_mu template [[host_name("kernel_mul_mm_id_q5_K_f32")]] kernel mat_mm_id_t kernel_mul_mm_id; template [[host_name("kernel_mul_mm_id_q6_K_f32")]] kernel mat_mm_id_t kernel_mul_mm_id; template [[host_name("kernel_mul_mm_id_iq2_xxs_f32")]] kernel mat_mm_id_t kernel_mul_mm_id; +template [[host_name("kernel_mul_mm_id_iq2_xs_f32")]] kernel mat_mm_id_t kernel_mul_mm_id; // // matrix-vector multiplication @@ -5448,3 +5753,68 @@ kernel void kernel_mul_mv_id_iq2_xxs_f32( tiisg, sgitg); } + +[[host_name("kernel_mul_mv_id_iq2_xs_f32")]] +kernel void kernel_mul_mv_id_iq2_xs_f32( + device const char * ids, + device const char * src1, + device float * dst, + constant uint64_t & nbi1, + constant int64_t & ne00, + constant int64_t & ne01, + constant int64_t & ne02, + constant uint64_t & nb00, + constant uint64_t & nb01, + constant uint64_t & nb02, + constant int64_t & ne10, + constant int64_t & ne11, + constant int64_t & ne12, + constant int64_t & ne13, + constant uint64_t & nb10, + constant uint64_t & nb11, + constant uint64_t & nb12, + constant int64_t & ne0, + constant int64_t & ne1, + constant uint64_t & nb1, + constant uint & r2, + constant uint & r3, + constant int & idx, + device const char * src00, + device const char * src01, + device const char * src02, + device const char * src03, + device const char * src04, + device const char * src05, + device const char * src06, + device const char * src07, + threadgroup int8_t * shared_values [[threadgroup(0)]], + uint3 tgpig[[threadgroup_position_in_grid]], + uint tiitg[[thread_index_in_threadgroup]], + uint tiisg[[thread_index_in_simdgroup]], + uint sgitg[[simdgroup_index_in_threadgroup]]) { + device const char * src0[8] = {src00, src01, src02, src03, src04, src05, src06, src07}; + + const int64_t bid = tgpig.z/(ne12*ne13); + + tgpig.z = tgpig.z%(ne12*ne13); + + const int32_t id = ((device int32_t *) (ids + bid*nbi1))[idx]; + + kernel_mul_mv_iq2_xs_f32_impl( + src0[id], + (device const float *) (src1 + bid*nb11), + dst + bid*ne0, + ne00, + ne01, + ne02, + ne10, + ne12, + ne0, + ne1, + r2, + r3, + shared_values, + tgpig, + tiisg, + sgitg); +} diff --git a/ggml-quants.c b/ggml-quants.c index d497e6de9..a24b4b244 100644 --- a/ggml-quants.c +++ b/ggml-quants.c @@ -2342,15 +2342,7 @@ size_t ggml_quantize_q6_K(const float * src, void * dst, int n, int k, int64_t * // ====================== "True" 2-bit (de)-quantization -void quantize_row_iq2_xxs_reference(const float * restrict x, block_iq2_xxs * restrict y, int k) { - (void)x; - (void)y; - (void)k; - assert(k % QK_K == 0); - //fprintf(stderr, "=========================== %s: not implemented\n", __func__); -} - -static const uint64_t iq2xxs_grid[256] = { +static const uint64_t iq2xxs_grid[256] = { 0x0808080808080808, 0x080808080808082b, 0x0808080808081919, 0x0808080808082b08, 0x0808080808082b2b, 0x0808080808190819, 0x0808080808191908, 0x08080808082b0808, 0x08080808082b082b, 0x08080808082b2b08, 0x08080808082b2b2b, 0x0808080819080819, @@ -2417,6 +2409,137 @@ static const uint64_t iq2xxs_grid[256] = { 0x2b2b082b08080808, 0x2b2b190808192b08, 0x2b2b2b0819190808, 0x2b2b2b1908081908, }; +static const uint64_t iq2xs_grid[512] = { + 0x0808080808080808, 0x080808080808082b, 0x0808080808081919, 0x0808080808082b08, + 0x0808080808082b2b, 0x0808080808190819, 0x0808080808191908, 0x080808080819192b, + 0x0808080808192b19, 0x08080808082b0808, 0x08080808082b082b, 0x08080808082b1919, + 0x08080808082b2b08, 0x0808080819080819, 0x0808080819081908, 0x080808081908192b, + 0x0808080819082b19, 0x0808080819190808, 0x080808081919082b, 0x0808080819191919, + 0x0808080819192b08, 0x08080808192b0819, 0x08080808192b1908, 0x080808082b080808, + 0x080808082b08082b, 0x080808082b081919, 0x080808082b082b08, 0x080808082b190819, + 0x080808082b191908, 0x080808082b192b19, 0x080808082b2b0808, 0x0808081908080819, + 0x0808081908081908, 0x080808190808192b, 0x0808081908082b19, 0x0808081908190808, + 0x080808190819082b, 0x0808081908191919, 0x0808081908192b08, 0x0808081908192b2b, + 0x08080819082b0819, 0x08080819082b1908, 0x0808081919080808, 0x080808191908082b, + 0x0808081919081919, 0x0808081919082b08, 0x0808081919190819, 0x0808081919191908, + 0x08080819192b0808, 0x08080819192b2b08, 0x080808192b080819, 0x080808192b081908, + 0x080808192b190808, 0x0808082b08080808, 0x0808082b0808082b, 0x0808082b08081919, + 0x0808082b08082b08, 0x0808082b08190819, 0x0808082b08191908, 0x0808082b082b0808, + 0x0808082b19080819, 0x0808082b19081908, 0x0808082b19190808, 0x0808082b19191919, + 0x0808082b2b080808, 0x0808082b2b082b2b, 0x0808190808080819, 0x0808190808081908, + 0x080819080808192b, 0x0808190808082b19, 0x0808190808190808, 0x080819080819082b, + 0x0808190808191919, 0x0808190808192b08, 0x08081908082b0819, 0x08081908082b1908, + 0x0808190819080808, 0x080819081908082b, 0x0808190819081919, 0x0808190819082b08, + 0x0808190819190819, 0x0808190819191908, 0x080819081919192b, 0x08081908192b0808, + 0x080819082b080819, 0x080819082b081908, 0x080819082b190808, 0x0808191908080808, + 0x080819190808082b, 0x0808191908081919, 0x0808191908082b08, 0x0808191908190819, + 0x0808191908191908, 0x08081919082b0808, 0x0808191919080819, 0x0808191919081908, + 0x0808191919190808, 0x08081919192b0819, 0x080819192b080808, 0x0808192b08080819, + 0x0808192b08081908, 0x0808192b08190808, 0x0808192b082b192b, 0x0808192b19080808, + 0x0808192b1908082b, 0x0808192b2b081908, 0x08082b0808080808, 0x08082b080808082b, + 0x08082b0808081919, 0x08082b0808082b08, 0x08082b0808082b2b, 0x08082b0808190819, + 0x08082b0808191908, 0x08082b08082b0808, 0x08082b08082b1919, 0x08082b0819080819, + 0x08082b0819081908, 0x08082b0819190808, 0x08082b0819192b08, 0x08082b082b080808, + 0x08082b082b2b0808, 0x08082b082b2b2b2b, 0x08082b1908080819, 0x08082b1908081908, + 0x08082b1908190808, 0x08082b1919080808, 0x08082b192b080819, 0x08082b192b082b19, + 0x08082b2b08080808, 0x08082b2b082b0808, 0x08082b2b082b2b08, 0x08082b2b2b19192b, + 0x08082b2b2b2b0808, 0x0819080808080819, 0x0819080808081908, 0x081908080808192b, + 0x0819080808082b19, 0x0819080808190808, 0x081908080819082b, 0x0819080808191919, + 0x0819080808192b08, 0x08190808082b0819, 0x08190808082b1908, 0x0819080819080808, + 0x081908081908082b, 0x0819080819081919, 0x0819080819082b08, 0x0819080819190819, + 0x0819080819191908, 0x08190808192b0808, 0x08190808192b2b2b, 0x081908082b080819, + 0x081908082b081908, 0x081908082b190808, 0x0819081908080808, 0x081908190808082b, + 0x0819081908081919, 0x0819081908082b08, 0x0819081908190819, 0x0819081908191908, + 0x08190819082b0808, 0x0819081919080819, 0x0819081919081908, 0x0819081919190808, + 0x081908192b080808, 0x081908192b191908, 0x081908192b19192b, 0x0819082b08080819, + 0x0819082b08081908, 0x0819082b0808192b, 0x0819082b08190808, 0x0819082b19080808, + 0x0819082b192b0808, 0x0819190808080808, 0x081919080808082b, 0x0819190808081919, + 0x0819190808082b08, 0x0819190808190819, 0x0819190808191908, 0x08191908082b0808, + 0x0819190819080819, 0x0819190819081908, 0x0819190819082b19, 0x0819190819190808, + 0x08191908192b1908, 0x081919082b080808, 0x0819191908080819, 0x0819191908081908, + 0x0819191908190808, 0x0819191919080808, 0x0819192b08080808, 0x0819192b08191908, + 0x0819192b19082b19, 0x08192b0808080819, 0x08192b0808081908, 0x08192b0808190808, + 0x08192b080819082b, 0x08192b0819080808, 0x08192b0819191908, 0x08192b082b08192b, + 0x08192b1908080808, 0x08192b1908081919, 0x08192b19192b192b, 0x08192b2b19190819, + 0x08192b2b2b2b2b19, 0x082b080808080808, 0x082b08080808082b, 0x082b080808081919, + 0x082b080808082b08, 0x082b080808082b2b, 0x082b080808190819, 0x082b080808191908, + 0x082b0808082b0808, 0x082b080819080819, 0x082b080819081908, 0x082b080819190808, + 0x082b08082b080808, 0x082b08082b2b0808, 0x082b081908080819, 0x082b081908081908, + 0x082b081908190808, 0x082b081919080808, 0x082b081919082b08, 0x082b0819192b1919, + 0x082b082b08080808, 0x082b082b082b082b, 0x082b082b2b080808, 0x082b082b2b2b2b08, + 0x082b190808080819, 0x082b190808081908, 0x082b190808190808, 0x082b1908082b2b19, + 0x082b190819080808, 0x082b191908080808, 0x082b191919080819, 0x082b19191919082b, + 0x082b19192b192b19, 0x082b192b08080819, 0x082b192b08192b2b, 0x082b192b2b2b192b, + 0x082b2b0808080808, 0x082b2b0808082b08, 0x082b2b0808082b2b, 0x082b2b08082b0808, + 0x082b2b0819191919, 0x082b2b082b082b08, 0x082b2b082b2b082b, 0x082b2b19192b2b08, + 0x082b2b192b190808, 0x082b2b2b08082b08, 0x082b2b2b082b0808, 0x082b2b2b2b08082b, + 0x082b2b2b2b082b08, 0x082b2b2b2b082b2b, 0x1908080808080819, 0x1908080808081908, + 0x190808080808192b, 0x1908080808082b19, 0x1908080808190808, 0x190808080819082b, + 0x1908080808191919, 0x1908080808192b08, 0x19080808082b0819, 0x19080808082b1908, + 0x1908080819080808, 0x190808081908082b, 0x1908080819081919, 0x1908080819082b08, + 0x1908080819082b2b, 0x1908080819190819, 0x1908080819191908, 0x19080808192b0808, + 0x19080808192b1919, 0x190808082b080819, 0x190808082b081908, 0x190808082b190808, + 0x1908081908080808, 0x190808190808082b, 0x1908081908081919, 0x1908081908082b08, + 0x1908081908190819, 0x1908081908191908, 0x19080819082b0808, 0x1908081919080819, + 0x1908081919081908, 0x1908081919190808, 0x190808192b080808, 0x190808192b081919, + 0x190808192b2b082b, 0x1908082b08080819, 0x1908082b08081908, 0x1908082b08190808, + 0x1908082b0819082b, 0x1908082b082b2b19, 0x1908082b19080808, 0x1908190808080808, + 0x190819080808082b, 0x1908190808081919, 0x1908190808082b08, 0x1908190808190819, + 0x1908190808191908, 0x1908190808192b19, 0x19081908082b0808, 0x1908190819080819, + 0x1908190819081908, 0x1908190819190808, 0x190819082b080808, 0x190819082b191908, + 0x1908191908080819, 0x1908191908081908, 0x1908191908190808, 0x19081919082b1908, + 0x1908191919080808, 0x190819192b192b2b, 0x1908192b08080808, 0x1908192b08082b2b, + 0x1908192b19081908, 0x1908192b19190808, 0x19082b0808080819, 0x19082b0808081908, + 0x19082b0808190808, 0x19082b0819080808, 0x19082b0819081919, 0x19082b0819191908, + 0x19082b08192b082b, 0x19082b1908080808, 0x19082b1908190819, 0x19082b1919081908, + 0x19082b1919190808, 0x19082b19192b2b19, 0x19082b2b08081908, 0x1919080808080808, + 0x191908080808082b, 0x1919080808081919, 0x1919080808082b08, 0x1919080808190819, + 0x1919080808191908, 0x19190808082b0808, 0x19190808082b2b08, 0x1919080819080819, + 0x1919080819081908, 0x1919080819190808, 0x191908082b080808, 0x1919081908080819, + 0x1919081908081908, 0x1919081908190808, 0x1919081908191919, 0x1919081919080808, + 0x191908191908082b, 0x1919082b08080808, 0x1919082b19081908, 0x1919082b2b2b2b2b, + 0x1919190808080819, 0x1919190808081908, 0x1919190808190808, 0x19191908082b0819, + 0x1919190819080808, 0x19191908192b0808, 0x191919082b080819, 0x191919082b2b0819, + 0x1919191908080808, 0x1919191908082b08, 0x191919192b080808, 0x191919192b082b08, + 0x1919192b082b0819, 0x1919192b192b2b08, 0x1919192b2b2b0819, 0x19192b0808080808, + 0x19192b0808191908, 0x19192b0819080819, 0x19192b0819190808, 0x19192b082b192b19, + 0x19192b1908192b2b, 0x19192b1919080808, 0x19192b191908082b, 0x19192b2b2b081919, + 0x192b080808080819, 0x192b080808081908, 0x192b080808190808, 0x192b080819080808, + 0x192b080819191908, 0x192b0808192b082b, 0x192b08082b08192b, 0x192b08082b2b2b19, + 0x192b081908080808, 0x192b082b082b1908, 0x192b082b19082b2b, 0x192b082b2b19082b, + 0x192b190808080808, 0x192b19080819192b, 0x192b191908190808, 0x192b191919080808, + 0x192b191919081919, 0x192b19192b2b1908, 0x192b2b0808080819, 0x192b2b08192b2b2b, + 0x192b2b19082b1919, 0x192b2b2b0808192b, 0x192b2b2b19191908, 0x192b2b2b192b082b, + 0x2b08080808080808, 0x2b0808080808082b, 0x2b08080808081919, 0x2b08080808082b08, + 0x2b08080808190819, 0x2b08080808191908, 0x2b080808082b0808, 0x2b080808082b2b2b, + 0x2b08080819080819, 0x2b08080819081908, 0x2b08080819190808, 0x2b0808082b080808, + 0x2b0808082b08082b, 0x2b0808082b2b2b08, 0x2b0808082b2b2b2b, 0x2b08081908080819, + 0x2b08081908081908, 0x2b0808190808192b, 0x2b08081908190808, 0x2b08081919080808, + 0x2b08081919190819, 0x2b08081919192b19, 0x2b08082b08080808, 0x2b08082b082b0808, + 0x2b08082b2b080808, 0x2b08082b2b08082b, 0x2b08082b2b2b0808, 0x2b08082b2b2b2b08, + 0x2b08190808080819, 0x2b08190808081908, 0x2b08190808190808, 0x2b0819080819082b, + 0x2b08190808191919, 0x2b08190819080808, 0x2b081908192b0808, 0x2b0819082b082b19, + 0x2b08191908080808, 0x2b08191919081908, 0x2b0819192b2b1919, 0x2b08192b08192b08, + 0x2b08192b192b2b2b, 0x2b082b0808080808, 0x2b082b0808082b08, 0x2b082b08082b1919, + 0x2b082b0819192b2b, 0x2b082b082b080808, 0x2b082b082b08082b, 0x2b082b082b2b2b08, + 0x2b082b190808192b, 0x2b082b2b082b082b, 0x2b082b2b2b080808, 0x2b082b2b2b082b08, + 0x2b082b2b2b19192b, 0x2b082b2b2b2b2b08, 0x2b19080808080819, 0x2b19080808081908, + 0x2b19080808190808, 0x2b19080819080808, 0x2b1908081919192b, 0x2b1908082b081908, + 0x2b19081908080808, 0x2b190819082b082b, 0x2b190819192b1908, 0x2b19082b1919192b, + 0x2b19082b2b082b19, 0x2b19190808080808, 0x2b19190808081919, 0x2b19190819081908, + 0x2b19190819190808, 0x2b19190819192b08, 0x2b191919082b2b19, 0x2b1919192b190808, + 0x2b1919192b19082b, 0x2b19192b19080819, 0x2b192b0819190819, 0x2b192b082b2b192b, + 0x2b192b1919082b19, 0x2b192b2b08191919, 0x2b192b2b192b0808, 0x2b2b080808080808, + 0x2b2b08080808082b, 0x2b2b080808082b08, 0x2b2b080808082b2b, 0x2b2b0808082b0808, + 0x2b2b0808082b2b2b, 0x2b2b08082b2b0808, 0x2b2b081919190819, 0x2b2b081919192b19, + 0x2b2b08192b2b192b, 0x2b2b082b08080808, 0x2b2b082b0808082b, 0x2b2b082b08082b08, + 0x2b2b082b082b2b2b, 0x2b2b082b2b080808, 0x2b2b082b2b2b0808, 0x2b2b190819080808, + 0x2b2b19082b191919, 0x2b2b192b192b1919, 0x2b2b192b2b192b08, 0x2b2b2b0808082b2b, + 0x2b2b2b08082b0808, 0x2b2b2b08082b082b, 0x2b2b2b08082b2b08, 0x2b2b2b082b2b0808, + 0x2b2b2b082b2b2b08, 0x2b2b2b1908081908, 0x2b2b2b192b081908, 0x2b2b2b192b08192b, + 0x2b2b2b2b082b2b08, 0x2b2b2b2b082b2b2b, 0x2b2b2b2b2b190819, 0x2b2b2b2b2b2b2b2b, +}; + static const uint8_t ksigns_iq2xs[128] = { 0, 129, 130, 3, 132, 5, 6, 135, 136, 9, 10, 139, 12, 141, 142, 15, 144, 17, 18, 147, 20, 149, 150, 23, 24, 153, 154, 27, 156, 29, 30, 159, @@ -2427,8 +2550,17 @@ static const uint8_t ksigns_iq2xs[128] = { 96, 225, 226, 99, 228, 101, 102, 231, 232, 105, 106, 235, 108, 237, 238, 111, 240, 113, 114, 243, 116, 245, 246, 119, 120, 249, 250, 123, 252, 125, 126, 255, }; + static const uint8_t kmask_iq2xs[8] = {1, 2, 4, 8, 16, 32, 64, 128}; +void quantize_row_iq2_xxs_reference(const float * restrict x, block_iq2_xxs * restrict y, int k) { + (void)x; + (void)y; + (void)k; + assert(k % QK_K == 0); + //fprintf(stderr, "=========================== %s: not implemented\n", __func__); +} + void dequantize_row_iq2_xxs(const block_iq2_xxs * restrict x, float * restrict y, int k) { assert(k % QK_K == 0); const int nb = k / QK_K; @@ -2472,6 +2604,58 @@ size_t ggml_quantize_iq2_xxs(const float * src, void * dst, int n, int k, int64_ return (n/QK_K*sizeof(block_iq2_xxs)); } +// ====================== 2.3125 bpw (de)-quantization + +void quantize_row_iq2_xs_reference(const float * restrict x, block_iq2_xs * restrict y, int k) { + (void)x; + (void)y; + (void)k; + assert(k % QK_K == 0); + //fprintf(stderr, "=========================== %s: not implemented\n", __func__); +} + +void dequantize_row_iq2_xs(const block_iq2_xs * restrict x, float * restrict y, int k) { + assert(k % QK_K == 0); + const int nb = k / QK_K; + + float db[2]; + + for (int i = 0; i < nb; i++) { + + const float d = GGML_FP16_TO_FP32(x[i].d); + + for (int ib32 = 0; ib32 < QK_K/32; ++ib32) { + db[0] = d * (0.5f + (x[i].scales[ib32] & 0xf)) * 0.25f; + db[1] = d * (0.5f + (x[i].scales[ib32] >> 4)) * 0.25f; + for (int l = 0; l < 4; ++l) { + const uint8_t * grid = (const uint8_t *)(iq2xs_grid + (x[i].qs[4*ib32 + l] & 511)); + const uint8_t signs = ksigns_iq2xs[x[i].qs[4*ib32 + l] >> 9]; + for (int j = 0; j < 8; ++j) { + y[j] = db[l/2] * grid[j] * (signs & kmask_iq2xs[j] ? -1.f : 1.f); + } + y += 8; + } + } + } +} + +void quantize_row_iq2_xs(const float * restrict x, void * restrict vy, int k) { + assert(k % QK_K == 0); + block_iq2_xs * restrict y = vy; + quantize_row_iq2_xs_reference(x, y, k); +} + +size_t ggml_quantize_iq2_xs(const float * src, void * dst, int n, int k, int64_t * hist) { + assert(k % QK_K == 0); + (void)hist; // TODO: collect histograms + + for (int j = 0; j < n; j += k) { + block_iq2_xs * restrict y = (block_iq2_xs *)dst + j/QK_K; + quantize_row_iq2_xs_reference(src + j, y, k); + } + return (n/QK_K*sizeof(block_iq2_xs)); +} + //===================================== Q8_K ============================================== void quantize_row_q8_K_reference(const float * restrict x, block_q8_K * restrict y, int k) { @@ -7357,3 +7541,161 @@ void ggml_vec_dot_iq2_xxs_q8_K(const int n, float * restrict s, const void * res *s = 0.125f * sumf; #endif } + +void ggml_vec_dot_iq2_xs_q8_K(const int n, float * restrict s, const void * restrict vx, const void * restrict vy) { + assert(n % QK_K == 0); + + const block_iq2_xs * restrict x = vx; + const block_q8_K * restrict y = vy; + + const int nb = n / QK_K; + +#if defined(__ARM_NEON) + + const uint64_t * signs64 = (const uint64_t *)keven_signs_q2xs; + + int8x16x4_t q2u; + int8x16x4_t q2s; + int8x16x4_t q8b; + + int32x4x4_t scales32; + + float sumf = 0; + for (int i = 0; i < nb; ++i) { + const float d = GGML_FP16_TO_FP32(x[i].d) * y[i].d; + const uint16_t * restrict q2 = x[i].qs; + const int8_t * restrict q8 = y[i].qs; + const uint8x8_t scales8 = vld1_u8(x[i].scales); + const uint8x8_t scales_l = vand_u8(scales8, vdup_n_u8(0xf)); + const uint8x8_t scales_h = vshr_n_u8(scales8, 4); + uint8x16_t scales = vcombine_u8(vzip1_u8(scales_l, scales_h), vzip2_u8(scales_l, scales_h)); + scales = vaddq_u8(vshlq_n_u8(scales, 1), vdupq_n_u8(1)); + const uint16x8_t scales1 = vmovl_u8(vget_low_u8(scales)); + const uint16x8_t scales2 = vmovl_u8(vget_high_u8(scales)); + scales32.val[0] = vreinterpretq_s32_u32(vmovl_u16(vget_low_u16(scales1))); + scales32.val[1] = vreinterpretq_s32_u32(vmovl_u16(vget_high_u16(scales1))); + scales32.val[2] = vreinterpretq_s32_u32(vmovl_u16(vget_low_u16(scales2))); + scales32.val[3] = vreinterpretq_s32_u32(vmovl_u16(vget_high_u16(scales2))); + int32x4_t sumi = vdupq_n_s32(0); + for (int ib64 = 0; ib64 < QK_K/64; ++ib64) { + q8b = vld1q_s8_x4(q8); q8 += 64; + q2u.val[0] = vcombine_s8(vld1_s8((const void *)(iq2xs_grid + (q2[0] & 511))), vld1_s8((const void *)(iq2xs_grid + (q2[1] & 511)))); + q2u.val[1] = vcombine_s8(vld1_s8((const void *)(iq2xs_grid + (q2[2] & 511))), vld1_s8((const void *)(iq2xs_grid + (q2[3] & 511)))); + q2u.val[2] = vcombine_s8(vld1_s8((const void *)(iq2xs_grid + (q2[4] & 511))), vld1_s8((const void *)(iq2xs_grid + (q2[5] & 511)))); + q2u.val[3] = vcombine_s8(vld1_s8((const void *)(iq2xs_grid + (q2[6] & 511))), vld1_s8((const void *)(iq2xs_grid + (q2[7] & 511)))); + q2s.val[0] = vcombine_s8(vld1_s8((const void *)(signs64 + (q2[0] >> 9))), vld1_s8((const void *)(signs64 + (q2[1] >> 9)))); + q2s.val[1] = vcombine_s8(vld1_s8((const void *)(signs64 + (q2[2] >> 9))), vld1_s8((const void *)(signs64 + (q2[3] >> 9)))); + q2s.val[2] = vcombine_s8(vld1_s8((const void *)(signs64 + (q2[4] >> 9))), vld1_s8((const void *)(signs64 + (q2[5] >> 9)))); + q2s.val[3] = vcombine_s8(vld1_s8((const void *)(signs64 + (q2[6] >> 9))), vld1_s8((const void *)(signs64 + (q2[7] >> 9)))); + q2u.val[0] = vmulq_s8(q2u.val[0], q2s.val[0]); + q2u.val[1] = vmulq_s8(q2u.val[1], q2s.val[1]); + q2u.val[2] = vmulq_s8(q2u.val[2], q2s.val[2]); + q2u.val[3] = vmulq_s8(q2u.val[3], q2s.val[3]); + const int32x4_t p1 = ggml_vdotq_s32(vdupq_n_s32(0), q2u.val[0], q8b.val[0]); + const int32x4_t p2 = ggml_vdotq_s32(vdupq_n_s32(0), q2u.val[1], q8b.val[1]); + const int32x4_t p3 = ggml_vdotq_s32(vdupq_n_s32(0), q2u.val[2], q8b.val[2]); + const int32x4_t p4 = ggml_vdotq_s32(vdupq_n_s32(0), q2u.val[3], q8b.val[3]); + const int32x4_t p = vpaddq_s32(vpaddq_s32(p1, p2), vpaddq_s32(p3, p4)); + sumi = vmlaq_s32(sumi, p, scales32.val[ib64]); + q2 += 8; + } + sumf += d*vaddvq_s32(sumi); + } + *s = 0.125f * sumf; + +#elif defined(__AVX2__) + + const __m128i m4 = _mm_set1_epi8(0xf); + const __m128i m1 = _mm_set1_epi8(1); + const __m128i m511 = _mm_set1_epi16(511); + const __m128i m127 = _mm_set1_epi16(127); + + const uint64_t * signs64 = (const uint64_t *)keven_signs_q2xs; + + uint64_t aux64; + + // somewhat hacky, but gives a significant boost in performance + __m128i aux_gindex, aux_sindex; + const uint16_t * gindex = (const uint16_t *)&aux_gindex; + const uint16_t * sindex = (const uint16_t *)&aux_sindex; + + __m256 accumf = _mm256_setzero_ps(); + for (int i = 0; i < nb; ++i) { + const float d = GGML_FP16_TO_FP32(x[i].d) * y[i].d; + const uint16_t * restrict q2 = x[i].qs; + const int8_t * restrict q8 = y[i].qs; + + memcpy(&aux64, x[i].scales, 8); + __m128i stmp = _mm_set1_epi64x(aux64); + stmp = _mm_unpacklo_epi8(_mm_and_si128(stmp, m4), _mm_and_si128(_mm_srli_epi16(stmp, 4), m4)); + const __m128i scales = _mm_add_epi8(_mm_slli_epi16(stmp, 1), m1); + + __m256i sumi1 = _mm256_setzero_si256(); + __m256i sumi2 = _mm256_setzero_si256(); + for (int ib32 = 0; ib32 < QK_K/32; ib32 += 2) { + const __m256i q8_1 = _mm256_loadu_si256((const __m256i *)q8); q8 += 32; + const __m256i q8_2 = _mm256_loadu_si256((const __m256i *)q8); q8 += 32; + const __m128i q2_data = _mm_loadu_si128((const __m128i*)q2); q2 += 8; + aux_gindex = _mm_and_si128(q2_data, m511); + aux_sindex = _mm_and_si128(_mm_srli_epi16(q2_data, 9), m127); + const __m256i q2_1 = _mm256_set_epi64x(iq2xs_grid[gindex[3]], iq2xs_grid[gindex[2]], iq2xs_grid[gindex[1]], iq2xs_grid[gindex[0]]); + const __m256i q2_2 = _mm256_set_epi64x(iq2xs_grid[gindex[7]], iq2xs_grid[gindex[6]], iq2xs_grid[gindex[5]], iq2xs_grid[gindex[4]]); + const __m256i s2_1 = _mm256_set_epi64x(signs64[sindex[3]], signs64[sindex[2]], signs64[sindex[1]], signs64[sindex[0]]); + const __m256i s2_2 = _mm256_set_epi64x(signs64[sindex[7]], signs64[sindex[6]], signs64[sindex[5]], signs64[sindex[4]]); + const __m256i q8s_1 = _mm256_sign_epi8(q8_1, s2_1); + const __m256i q8s_2 = _mm256_sign_epi8(q8_2, s2_2); + const __m256i dot1 = _mm256_maddubs_epi16(q2_1, q8s_1); + const __m256i dot2 = _mm256_maddubs_epi16(q2_2, q8s_2); + + const __m256i sc1 = _mm256_cvtepi8_epi16(_mm_shuffle_epi8(scales, get_scale_shuffle(ib32+0))); + const __m256i sc2 = _mm256_cvtepi8_epi16(_mm_shuffle_epi8(scales, get_scale_shuffle(ib32+1))); + + sumi1 = _mm256_add_epi32(sumi1, _mm256_madd_epi16(dot1, sc1)); + sumi2 = _mm256_add_epi32(sumi2, _mm256_madd_epi16(dot2, sc2)); + } + + accumf = _mm256_fmadd_ps(_mm256_set1_ps(d), _mm256_cvtepi32_ps(_mm256_add_epi32(sumi1, sumi2)), accumf); + + } + + *s = 0.125f * hsum_float_8(accumf); + +#else + + float sumf = 0.f; + for (int i = 0; i < nb; ++i) { + const float d = GGML_FP16_TO_FP32(x[i].d) * y[i].d; + const uint16_t * restrict q2 = x[i].qs; + const uint8_t * restrict sc = x[i].scales; + const int8_t * restrict q8 = y[i].qs; + int32_t bsum = 0; + for (int ib32 = 0; ib32 < QK_K/32; ++ib32) { + const uint16_t ls1 = 2*(sc[ib32] & 0xf) + 1; + const uint16_t ls2 = 2*(sc[ib32] >> 4) + 1; + int32_t sumi = 0; + for (int l = 0; l < 2; ++l) { + const uint8_t * grid = (const uint8_t *)(iq2xs_grid + (q2[l] & 511)); + const uint8_t signs = ksigns_iq2xs[q2[l] >> 9]; + for (int j = 0; j < 8; ++j) { + sumi += grid[j] * q8[j] * (signs & kmask_iq2xs[j] ? -1 : 1); + } + q8 += 8; + } + bsum += sumi * ls1; + sumi = 0; + for (int l = 2; l < 4; ++l) { + const uint8_t * grid = (const uint8_t *)(iq2xs_grid + (q2[l] & 511)); + const uint8_t signs = ksigns_iq2xs[q2[l] >> 9]; + for (int j = 0; j < 8; ++j) { + sumi += grid[j] * q8[j] * (signs & kmask_iq2xs[j] ? -1 : 1); + } + q8 += 8; + } + bsum += sumi * ls2; + q2 += 4; + } + sumf += d * bsum; + } + *s = 0.125f * sumf; +#endif +} diff --git a/ggml-quants.h b/ggml-quants.h index 8dd911d41..df5e7ae80 100644 --- a/ggml-quants.h +++ b/ggml-quants.h @@ -174,6 +174,14 @@ typedef struct { } block_iq2_xxs; static_assert(sizeof(block_iq2_xxs) == sizeof(ggml_fp16_t) + QK_K/8*sizeof(uint16_t), "wrong iq2_xxs block size/padding"); +// 2.3125 bpw quants +typedef struct { + ggml_fp16_t d; + uint16_t qs[QK_K/8]; + uint8_t scales[QK_K/32]; +} block_iq2_xs; +static_assert(sizeof(block_iq2_xs) == sizeof(ggml_fp16_t) + QK_K/8*sizeof(uint16_t) + QK_K/32, "wrong iq2_xs block size/padding"); + // Quantization void quantize_row_q4_0_reference(const float * restrict x, block_q4_0 * restrict y, int k); void quantize_row_q4_1_reference(const float * restrict x, block_q4_1 * restrict y, int k); @@ -189,6 +197,7 @@ void quantize_row_q5_K_reference(const float * restrict x, block_q5_K * restrict void quantize_row_q6_K_reference(const float * restrict x, block_q6_K * restrict y, int k); void quantize_row_q8_K_reference(const float * restrict x, block_q8_K * restrict y, int k); void quantize_row_iq2_xxs_reference(const float * restrict x, block_iq2_xxs * restrict y, int k); +void quantize_row_iq2_xs_reference (const float * restrict x, block_iq2_xs * restrict y, int k); void quantize_row_q4_0(const float * restrict x, void * restrict y, int k); void quantize_row_q4_1(const float * restrict x, void * restrict y, int k); @@ -204,6 +213,7 @@ void quantize_row_q5_K(const float * restrict x, void * restrict y, int k); void quantize_row_q6_K(const float * restrict x, void * restrict y, int k); void quantize_row_q8_K(const float * restrict x, void * restrict y, int k); void quantize_row_iq2_xxs(const float * restrict x, void * restrict y, int k); +void quantize_row_iq2_xs (const float * restrict x, void * restrict y, int k); // Dequantization void dequantize_row_q4_0(const block_q4_0 * restrict x, float * restrict y, int k); @@ -220,6 +230,7 @@ void dequantize_row_q5_K(const block_q5_K * restrict x, float * restrict y, int void dequantize_row_q6_K(const block_q6_K * restrict x, float * restrict y, int k); void dequantize_row_q8_K(const block_q8_K * restrict x, float * restrict y, int k); void dequantize_row_iq2_xxs(const block_iq2_xxs * restrict x, float * restrict y, int k); +void dequantize_row_iq2_xs (const block_iq2_xs * restrict x, float * restrict y, int k); // Dot product void ggml_vec_dot_q4_0_q8_0(int n, float * restrict s, const void * restrict vx, const void * restrict vy); @@ -234,3 +245,4 @@ void ggml_vec_dot_q4_K_q8_K(int n, float * restrict s, const void * restrict vx, void ggml_vec_dot_q5_K_q8_K(int n, float * restrict s, const void * restrict vx, const void * restrict vy); void ggml_vec_dot_q6_K_q8_K(int n, float * restrict s, const void * restrict vx, const void * restrict vy); void ggml_vec_dot_iq2_xxs_q8_K(int n, float * restrict s, const void * restrict vx, const void * restrict vy); +void ggml_vec_dot_iq2_xs_q8_K (int n, float * restrict s, const void * restrict vx, const void * restrict vy); diff --git a/ggml.c b/ggml.c index 9c42a45e3..d2a8c0478 100644 --- a/ggml.c +++ b/ggml.c @@ -584,6 +584,17 @@ static const ggml_type_traits_t type_traits[GGML_TYPE_COUNT] = { .vec_dot = ggml_vec_dot_iq2_xxs_q8_K, .vec_dot_type = GGML_TYPE_Q8_K, }, + [GGML_TYPE_IQ2_XS] = { + .type_name = "iq2_xs", + .blck_size = QK_K, + .type_size = sizeof(block_iq2_xs), + .is_quantized = true, + .to_float = (ggml_to_float_t) dequantize_row_iq2_xs, + .from_float = quantize_row_iq2_xs, + .from_float_reference = (ggml_from_float_t) quantize_row_iq2_xs_reference, + .vec_dot = ggml_vec_dot_iq2_xs_q8_K, + .vec_dot_type = GGML_TYPE_Q8_K, + }, [GGML_TYPE_Q8_K] = { .type_name = "q8_K", .blck_size = QK_K, @@ -2123,6 +2134,7 @@ enum ggml_type ggml_ftype_to_ggml_type(enum ggml_ftype ftype) { case GGML_FTYPE_MOSTLY_Q5_K: wtype = GGML_TYPE_Q5_K; break; case GGML_FTYPE_MOSTLY_Q6_K: wtype = GGML_TYPE_Q6_K; break; case GGML_FTYPE_MOSTLY_IQ2_XXS: wtype = GGML_TYPE_IQ2_XXS; break; + case GGML_FTYPE_MOSTLY_IQ2_XS: wtype = GGML_TYPE_IQ2_XS; break; case GGML_FTYPE_UNKNOWN: wtype = GGML_TYPE_COUNT; break; case GGML_FTYPE_MOSTLY_Q4_1_SOME_F16: wtype = GGML_TYPE_COUNT; break; } @@ -7435,6 +7447,7 @@ static void ggml_compute_forward_add( case GGML_TYPE_Q5_K: case GGML_TYPE_Q6_K: case GGML_TYPE_IQ2_XXS: + case GGML_TYPE_IQ2_XS: { ggml_compute_forward_add_q_f32(params, src0, src1, dst); } break; @@ -7700,6 +7713,7 @@ static void ggml_compute_forward_add1( case GGML_TYPE_Q5_K: case GGML_TYPE_Q6_K: case GGML_TYPE_IQ2_XXS: + case GGML_TYPE_IQ2_XS: { ggml_compute_forward_add1_q_f32(params, src0, src1, dst); } break; @@ -7815,6 +7829,7 @@ static void ggml_compute_forward_acc( case GGML_TYPE_Q5_K: case GGML_TYPE_Q6_K: case GGML_TYPE_IQ2_XXS: + case GGML_TYPE_IQ2_XS: default: { GGML_ASSERT(false); @@ -10457,6 +10472,7 @@ static void ggml_compute_forward_out_prod( case GGML_TYPE_Q5_K: case GGML_TYPE_Q6_K: case GGML_TYPE_IQ2_XXS: + case GGML_TYPE_IQ2_XS: { ggml_compute_forward_out_prod_q_f32(params, src0, src1, dst); } break; @@ -10632,6 +10648,7 @@ static void ggml_compute_forward_set( case GGML_TYPE_Q5_K: case GGML_TYPE_Q6_K: case GGML_TYPE_IQ2_XXS: + case GGML_TYPE_IQ2_XS: default: { GGML_ASSERT(false); @@ -10827,6 +10844,7 @@ static void ggml_compute_forward_get_rows( case GGML_TYPE_Q5_K: case GGML_TYPE_Q6_K: case GGML_TYPE_IQ2_XXS: + case GGML_TYPE_IQ2_XS: { ggml_compute_forward_get_rows_q(params, src0, src1, dst); } break; @@ -11464,6 +11482,7 @@ static void ggml_compute_forward_alibi( case GGML_TYPE_Q5_K: case GGML_TYPE_Q6_K: case GGML_TYPE_IQ2_XXS: + case GGML_TYPE_IQ2_XS: case GGML_TYPE_Q8_K: case GGML_TYPE_I8: case GGML_TYPE_I16: @@ -11539,6 +11558,7 @@ static void ggml_compute_forward_clamp( case GGML_TYPE_Q5_K: case GGML_TYPE_Q6_K: case GGML_TYPE_IQ2_XXS: + case GGML_TYPE_IQ2_XS: case GGML_TYPE_Q8_K: case GGML_TYPE_I8: case GGML_TYPE_I16: @@ -18660,6 +18680,12 @@ size_t ggml_quantize_chunk(enum ggml_type type, const float * src, void * dst, i block_iq2_xxs * block = (block_iq2_xxs*)dst + start / QK_K; result = ggml_quantize_iq2_xxs(src + start, block, n, n, hist); } break; + case GGML_TYPE_IQ2_XS: + { + GGML_ASSERT(start % QK_K == 0); + block_iq2_xs * block = (block_iq2_xs*)dst + start / QK_K; + result = ggml_quantize_iq2_xs(src + start, block, n, n, hist); + } break; case GGML_TYPE_F16: { int elemsize = sizeof(ggml_fp16_t); @@ -19015,8 +19041,8 @@ struct gguf_context * gguf_init_from_file(const char * fname, struct gguf_init_p (int64_t) info->ne[3]; if (ne % ggml_blck_size(info->type) != 0) { - fprintf(stderr, "%s: tensor '%s' number of elements (%" PRId64 ") is not a multiple of block size (%d)\n", - __func__, info->name.data, ne, ggml_blck_size(info->type)); + fprintf(stderr, "%s: tensor '%s' of type %d (%s) number of elements (%" PRId64 ") is not a multiple of block size (%d)\n", + __func__, info->name.data, (int)info->type, ggml_type_name(info->type), ne, ggml_blck_size(info->type)); fclose(file); gguf_free(ctx); return NULL; diff --git a/ggml.h b/ggml.h index 127dcef1d..93b42a27d 100644 --- a/ggml.h +++ b/ggml.h @@ -342,6 +342,7 @@ extern "C" { GGML_TYPE_Q6_K = 14, GGML_TYPE_Q8_K = 15, GGML_TYPE_IQ2_XXS = 16, + GGML_TYPE_IQ2_XS = 17, GGML_TYPE_I8, GGML_TYPE_I16, GGML_TYPE_I32, @@ -377,6 +378,7 @@ extern "C" { GGML_FTYPE_MOSTLY_Q5_K = 13, // except 1d tensors GGML_FTYPE_MOSTLY_Q6_K = 14, // except 1d tensors GGML_FTYPE_MOSTLY_IQ2_XXS = 15, // except 1d tensors + GGML_FTYPE_MOSTLY_IQ2_XS = 16, // except 1d tensors }; // available tensor operations: @@ -2061,6 +2063,7 @@ extern "C" { GGML_API size_t ggml_quantize_q5_K(const float * src, void * dst, int n, int k, int64_t * hist); GGML_API size_t ggml_quantize_q6_K(const float * src, void * dst, int n, int k, int64_t * hist); GGML_API size_t ggml_quantize_iq2_xxs(const float * src, void * dst, int n, int k, int64_t * hist); + GGML_API size_t ggml_quantize_iq2_xs (const float * src, void * dst, int n, int k, int64_t * hist); GGML_API size_t ggml_quantize_chunk(enum ggml_type type, const float * src, void * dst, int start, int n, int64_t * hist); diff --git a/llama.cpp b/llama.cpp index aaadfa444..bd219d49c 100644 --- a/llama.cpp +++ b/llama.cpp @@ -2223,6 +2223,7 @@ struct llama_model_loader { case GGML_TYPE_Q5_K: ftype = LLAMA_FTYPE_MOSTLY_Q5_K_M; break; case GGML_TYPE_Q6_K: ftype = LLAMA_FTYPE_MOSTLY_Q6_K; break; case GGML_TYPE_IQ2_XXS: ftype = LLAMA_FTYPE_MOSTLY_IQ2_XXS; break; + case GGML_TYPE_IQ2_XS: ftype = LLAMA_FTYPE_MOSTLY_IQ2_XS; break; default: { LLAMA_LOG_WARN("%s: unknown type %s\n", __func__, ggml_type_name(type_max)); @@ -2595,6 +2596,7 @@ static std::string llama_model_ftype_name(llama_ftype ftype) { case LLAMA_FTYPE_MOSTLY_Q5_K_M: return "Q5_K - Medium"; case LLAMA_FTYPE_MOSTLY_Q6_K: return "Q6_K"; case LLAMA_FTYPE_MOSTLY_IQ2_XXS:return "IQ2_XSS - 2.0625 bpw"; + case LLAMA_FTYPE_MOSTLY_IQ2_XS: return "IQ2_XS - 2.3125 bpw"; default: return "unknown, may not work"; } @@ -9050,6 +9052,7 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s case LLAMA_FTYPE_MOSTLY_Q5_K_M: quantized_type = GGML_TYPE_Q5_K; break; case LLAMA_FTYPE_MOSTLY_Q6_K: quantized_type = GGML_TYPE_Q6_K; break; case LLAMA_FTYPE_MOSTLY_IQ2_XXS:quantized_type = GGML_TYPE_IQ2_XXS; break; + case LLAMA_FTYPE_MOSTLY_IQ2_XS :quantized_type = GGML_TYPE_IQ2_XS; break; default: throw std::runtime_error(format("invalid output file type %d\n", ftype)); } diff --git a/llama.h b/llama.h index c11075bbc..6fde113ff 100644 --- a/llama.h +++ b/llama.h @@ -104,6 +104,7 @@ extern "C" { LLAMA_FTYPE_MOSTLY_Q5_K_M = 17, // except 1d tensors LLAMA_FTYPE_MOSTLY_Q6_K = 18, // except 1d tensors LLAMA_FTYPE_MOSTLY_IQ2_XXS = 19, // except 1d tensors + LLAMA_FTYPE_MOSTLY_IQ2_XS = 20, // except 1d tensors LLAMA_FTYPE_GUESSED = 1024, // not specified in the model file }; diff --git a/tests/test-quantize-fns.cpp b/tests/test-quantize-fns.cpp index cee712618..31a78c632 100644 --- a/tests/test-quantize-fns.cpp +++ b/tests/test-quantize-fns.cpp @@ -134,8 +134,9 @@ int main(int argc, char * argv[]) { continue; } - if ((ggml_type)i == GGML_TYPE_IQ2_XXS) { - printf("Skip %s due to missing quantization functionality\n", ggml_type_name((ggml_type) i)); + const ggml_type ei = (ggml_type)i; + if (ei == GGML_TYPE_IQ2_XXS || ei == GGML_TYPE_IQ2_XS) { + printf("Skip %s due to missing quantization functionality\n", ggml_type_name(ei)); continue; } From 469e75d0a35b08de549a4fd87f082ca7a8a539ba Mon Sep 17 00:00:00 2001 From: Kawrakow <48489457+ikawrakow@users.noreply.github.com> Date: Thu, 11 Jan 2024 20:43:15 +0100 Subject: [PATCH 024/812] llama : restore intended k-quants mixes for MoE models (#4872) * Restore intended k-quants quantization mixes for MoE models * Update Q2_K_S values in the quantize tool Still using LLaMA-v1 PPL values in the quant description today does not make much sense. But let's leave this update for another PR. --------- Co-authored-by: Iwan Kawrakow Co-authored-by: Georgi Gerganov --- examples/quantize/quantize.cpp | 1 + llama.cpp | 24 +++++++++++++++--------- llama.h | 1 + 3 files changed, 17 insertions(+), 9 deletions(-) diff --git a/examples/quantize/quantize.cpp b/examples/quantize/quantize.cpp index d27ea5e91..f878f6911 100644 --- a/examples/quantize/quantize.cpp +++ b/examples/quantize/quantize.cpp @@ -18,6 +18,7 @@ static const std::vector QUANT_OPTIONS = { { "Q5_0", LLAMA_FTYPE_MOSTLY_Q5_0, " 4.33G, +0.0683 ppl @ LLaMA-v1-7B", }, { "Q5_1", LLAMA_FTYPE_MOSTLY_Q5_1, " 4.70G, +0.0349 ppl @ LLaMA-v1-7B", }, { "Q2_K", LLAMA_FTYPE_MOSTLY_Q2_K, " 2.63G, +0.6717 ppl @ LLaMA-v1-7B", }, + { "Q2_K_S", LLAMA_FTYPE_MOSTLY_Q2_K_S, " 2.16G, +9.0634 ppl @ LLaMA-v1-7B", }, { "Q3_K", LLAMA_FTYPE_MOSTLY_Q3_K_M, "alias for Q3_K_M" }, { "Q3_K_S", LLAMA_FTYPE_MOSTLY_Q3_K_S, " 2.75G, +0.5551 ppl @ LLaMA-v1-7B", }, { "Q3_K_M", LLAMA_FTYPE_MOSTLY_Q3_K_M, " 3.07G, +0.2496 ppl @ LLaMA-v1-7B", }, diff --git a/llama.cpp b/llama.cpp index bd219d49c..d39ff94c7 100644 --- a/llama.cpp +++ b/llama.cpp @@ -2586,7 +2586,8 @@ static std::string llama_model_ftype_name(llama_ftype ftype) { case LLAMA_FTYPE_MOSTLY_Q8_0: return "Q8_0"; // K-quants - case LLAMA_FTYPE_MOSTLY_Q2_K: return "Q2_K"; + case LLAMA_FTYPE_MOSTLY_Q2_K: return "Q2_K - Medium"; + case LLAMA_FTYPE_MOSTLY_Q2_K_S: return "Q2_K - Small"; case LLAMA_FTYPE_MOSTLY_Q3_K_S: return "Q3_K - Small"; case LLAMA_FTYPE_MOSTLY_Q3_K_M: return "Q3_K - Medium"; case LLAMA_FTYPE_MOSTLY_Q3_K_L: return "Q3_K - Large"; @@ -8955,10 +8956,13 @@ static ggml_type get_k_quant_type(quantize_state_internal & qs, ggml_type new_ty // TODO: explore better strategies new_type = GGML_TYPE_Q8_0; } - } else if (name.find("ffn_down.weight") != std::string::npos) { + } else if (name.find("ffn_down") != std::string::npos) { if (ftype == LLAMA_FTYPE_MOSTLY_Q2_K) new_type = GGML_TYPE_Q3_K; + else if (ftype == LLAMA_FTYPE_MOSTLY_Q2_K_S) { + if (qs.i_feed_forward_w2 < qs.n_feed_forward_w2/8) new_type = GGML_TYPE_Q4_K; + } else if (ftype == LLAMA_FTYPE_MOSTLY_Q3_K_M) { - new_type = qs.i_feed_forward_w2 < 2 ? GGML_TYPE_Q5_K + new_type = qs.i_feed_forward_w2 < qs.n_feed_forward_w2/16 ? GGML_TYPE_Q5_K : arch != LLM_ARCH_FALCON || use_more_bits(qs.i_feed_forward_w2, qs.n_feed_forward_w2) ? GGML_TYPE_Q4_K : GGML_TYPE_Q3_K; } @@ -8967,14 +8971,14 @@ static ggml_type get_k_quant_type(quantize_state_internal & qs, ggml_type new_ty } else if (ftype == LLAMA_FTYPE_MOSTLY_Q4_K_M) { if (arch == LLM_ARCH_FALCON) { - new_type = qs.i_feed_forward_w2 < 2 ? GGML_TYPE_Q6_K : + new_type = qs.i_feed_forward_w2 < qs.n_feed_forward_w2/16 ? GGML_TYPE_Q6_K : use_more_bits(qs.i_feed_forward_w2, qs.n_feed_forward_w2) ? GGML_TYPE_Q5_K : GGML_TYPE_Q4_K; } else { if (use_more_bits(qs.i_feed_forward_w2, qs.n_feed_forward_w2)) new_type = GGML_TYPE_Q6_K; } } else if (ftype == LLAMA_FTYPE_MOSTLY_Q5_K_M && use_more_bits(qs.i_feed_forward_w2, qs.n_feed_forward_w2)) new_type = GGML_TYPE_Q6_K; - else if (ftype == LLAMA_FTYPE_MOSTLY_Q4_K_S && arch != LLM_ARCH_FALCON && qs.i_feed_forward_w2 < 4) { + else if (ftype == LLAMA_FTYPE_MOSTLY_Q4_K_S && arch != LLM_ARCH_FALCON && qs.i_feed_forward_w2 < qs.n_feed_forward_w2/8) { new_type = GGML_TYPE_Q5_K; } ++qs.i_feed_forward_w2; @@ -8992,9 +8996,10 @@ static ggml_type get_k_quant_type(quantize_state_internal & qs, ggml_type new_ty else if (ftype == LLAMA_FTYPE_MOSTLY_Q4_K_M) new_type = GGML_TYPE_Q5_K; else if (ftype == LLAMA_FTYPE_MOSTLY_Q5_K_M) new_type = GGML_TYPE_Q6_K; } - else if (name.find("ffn_gate.weight") != std::string::npos || name.find("ffn_up.weight") != std::string::npos) { - if (ftype == LLAMA_FTYPE_MOSTLY_Q2_K) new_type = GGML_TYPE_Q3_K; - } + // IK: let's remove this, else Q2_K is almost the same as Q3_K_S + //else if (name.find("ffn_gate") != std::string::npos || name.find("ffn_up") != std::string::npos) { + // if (ftype == LLAMA_FTYPE_MOSTLY_Q2_K) new_type = GGML_TYPE_Q3_K; + //} // This can be used to reduce the size of the Q5_K_S model. // The associated PPL increase is fully in line with the size reduction //else { @@ -9043,6 +9048,7 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s // K-quants case LLAMA_FTYPE_MOSTLY_Q2_K: quantized_type = GGML_TYPE_Q2_K; break; + case LLAMA_FTYPE_MOSTLY_Q2_K_S: quantized_type = GGML_TYPE_Q2_K; break; case LLAMA_FTYPE_MOSTLY_Q3_K_S: case LLAMA_FTYPE_MOSTLY_Q3_K_M: case LLAMA_FTYPE_MOSTLY_Q3_K_L: quantized_type = GGML_TYPE_Q3_K; break; @@ -9101,7 +9107,7 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s if (name.find("attn_v.weight") != std::string::npos || name.find("attn_qkv.weight") != std::string::npos) { ++qs.n_attention_wv; } - else if (name.find("ffn_down.weight") != std::string::npos) { + else if (name.find("ffn_down") != std::string::npos) { ++qs.n_feed_forward_w2; } } diff --git a/llama.h b/llama.h index 6fde113ff..43d41b8f6 100644 --- a/llama.h +++ b/llama.h @@ -105,6 +105,7 @@ extern "C" { LLAMA_FTYPE_MOSTLY_Q6_K = 18, // except 1d tensors LLAMA_FTYPE_MOSTLY_IQ2_XXS = 19, // except 1d tensors LLAMA_FTYPE_MOSTLY_IQ2_XS = 20, // except 1d tensors + LLAMA_FTYPE_MOSTLY_Q2_K_S = 21, // except 1d tensors LLAMA_FTYPE_GUESSED = 1024, // not specified in the model file }; From b0377875488b33f7114138687d828da1de61775d Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Thu, 11 Jan 2024 21:58:28 +0200 Subject: [PATCH 025/812] swift : track ggml release branch (#4867) --- Package.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Package.swift b/Package.swift index 59191da45..37524edee 100644 --- a/Package.swift +++ b/Package.swift @@ -14,7 +14,7 @@ let package = Package( .library(name: "llama", targets: ["llama"]), ], dependencies: [ - .package(url: "https://github.com/ggerganov/ggml.git", .revision("979cc23b345006504cfc1f67c0fdf627805e3319")) + .package(url: "https://github.com/ggerganov/ggml.git", .branch("release")) ], targets: [ .target( From 3ca63b4538dfc78aaec88cd2c3e3f8417c1924e3 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Thu, 11 Jan 2024 22:43:05 +0200 Subject: [PATCH 026/812] main : disable token count by default (#4874) --- common/common.cpp | 6 +++--- common/common.h | 2 +- examples/main/main.cpp | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/common/common.cpp b/common/common.cpp index bfcd6d4df..287e8bd5a 100644 --- a/common/common.cpp +++ b/common/common.cpp @@ -630,7 +630,7 @@ bool gpt_params_parse_ex(int argc, char ** argv, gpt_params & params) { break; } params.ppl_stride = std::stoi(argv[i]); - } else if (arg == "-stc" || arg == "--show_token_count") { + } else if (arg == "-stc" || arg == "--show-token-count") { if (++i >= argc) { invalid_param = true; break; @@ -950,8 +950,8 @@ void gpt_print_usage(int /*argc*/, char ** argv, const gpt_params & params) { printf(" --override-kv KEY=TYPE:VALUE\n"); printf(" advanced option to override model metadata by key. may be specified multiple times.\n"); printf(" types: int, float, bool. example: --override-kv tokenizer.ggml.add_bos_token=bool:false\n"); - printf(" -stc N --show_token_count N\n"); - printf(" show consumed tokens every N tokens\n"); + printf(" -stc N --show-token-count N\n"); + printf(" show consumed tokens every N tokens (default: %d)\n", params.token_interval); printf("\n"); #ifndef LOG_DISABLE_LOGS log_print_usage(); diff --git a/common/common.h b/common/common.h index a295e88b0..82d23cf54 100644 --- a/common/common.h +++ b/common/common.h @@ -64,7 +64,7 @@ struct gpt_params { int32_t n_beams = 0; // if non-zero then use beam search of given width. int32_t grp_attn_n = 1; // group-attention factor int32_t grp_attn_w = 512; // group-attention width - int32_t token_interval = 512; // show token count every 512 tokens + int32_t token_interval = -1; // show token count every 512 tokens (-1 = disabled) float rope_freq_base = 0.0f; // RoPE base frequency float rope_freq_scale = 0.0f; // RoPE frequency scaling factor float yarn_ext_factor = -1.0f; // YaRN extrapolation mix factor diff --git a/examples/main/main.cpp b/examples/main/main.cpp index 1f35febbd..6953d107c 100644 --- a/examples/main/main.cpp +++ b/examples/main/main.cpp @@ -651,8 +651,8 @@ int main(int argc, char ** argv) { LOG("n_past = %d\n", n_past); // Display total tokens alongside total time - if (n_past % params.token_interval == 0) { - printf("\n\033[31mTokens consumed so far = %d / %d \033[0m\n", n_past, n_ctx); + if (params.token_interval > 0 && n_past % params.token_interval == 0) { + LOG_TEE("\n\033[31mTokens consumed so far = %d / %d \033[0m\n", n_past, n_ctx); } } From 7edefbd79cc6dea96640edc54c6b94b2b2496d8b Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Thu, 11 Jan 2024 22:46:26 +0200 Subject: [PATCH 027/812] main : better name for variable n_print (#4874) --- common/common.cpp | 8 ++++---- common/common.h | 2 +- examples/main/main.cpp | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/common/common.cpp b/common/common.cpp index 287e8bd5a..b2cb0e257 100644 --- a/common/common.cpp +++ b/common/common.cpp @@ -630,12 +630,12 @@ bool gpt_params_parse_ex(int argc, char ** argv, gpt_params & params) { break; } params.ppl_stride = std::stoi(argv[i]); - } else if (arg == "-stc" || arg == "--show-token-count") { + } else if (arg == "-ptc" || arg == "--print-token-count") { if (++i >= argc) { invalid_param = true; break; } - params.token_interval = std::stoi(argv[i]); + params.n_print = std::stoi(argv[i]); } else if (arg == "--ppl-output-type") { if (++i >= argc) { invalid_param = true; @@ -950,8 +950,8 @@ void gpt_print_usage(int /*argc*/, char ** argv, const gpt_params & params) { printf(" --override-kv KEY=TYPE:VALUE\n"); printf(" advanced option to override model metadata by key. may be specified multiple times.\n"); printf(" types: int, float, bool. example: --override-kv tokenizer.ggml.add_bos_token=bool:false\n"); - printf(" -stc N --show-token-count N\n"); - printf(" show consumed tokens every N tokens (default: %d)\n", params.token_interval); + printf(" -stc N --print-token-count N\n"); + printf(" print token count every N tokens (default: %d)\n", params.n_print); printf("\n"); #ifndef LOG_DISABLE_LOGS log_print_usage(); diff --git a/common/common.h b/common/common.h index 82d23cf54..1359e76ab 100644 --- a/common/common.h +++ b/common/common.h @@ -64,7 +64,7 @@ struct gpt_params { int32_t n_beams = 0; // if non-zero then use beam search of given width. int32_t grp_attn_n = 1; // group-attention factor int32_t grp_attn_w = 512; // group-attention width - int32_t token_interval = -1; // show token count every 512 tokens (-1 = disabled) + int32_t n_print = -1; // print token count every n tokens (-1 = disabled) float rope_freq_base = 0.0f; // RoPE base frequency float rope_freq_scale = 0.0f; // RoPE frequency scaling factor float yarn_ext_factor = -1.0f; // YaRN extrapolation mix factor diff --git a/examples/main/main.cpp b/examples/main/main.cpp index 6953d107c..c53b29978 100644 --- a/examples/main/main.cpp +++ b/examples/main/main.cpp @@ -651,7 +651,7 @@ int main(int argc, char ** argv) { LOG("n_past = %d\n", n_past); // Display total tokens alongside total time - if (params.token_interval > 0 && n_past % params.token_interval == 0) { + if (params.n_print > 0 && n_past % params.n_print == 0) { LOG_TEE("\n\033[31mTokens consumed so far = %d / %d \033[0m\n", n_past, n_ctx); } } From 1d118386fea031f01550f8cd47a5c86296e5333f Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Thu, 11 Jan 2024 23:23:49 +0200 Subject: [PATCH 028/812] server : fix infill when prompt is empty (#4833) --- examples/server/server.cpp | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/examples/server/server.cpp b/examples/server/server.cpp index 031824e14..1d30a15a6 100644 --- a/examples/server/server.cpp +++ b/examples/server/server.cpp @@ -1406,7 +1406,7 @@ struct llama_server_context task.multitask_id = multitask_id; // when a completion task's prompt array is not a singleton, we split it into multiple requests - if (task.data.at("prompt").size() > 1) + if (task.data.count("prompt") && task.data.at("prompt").size() > 1) { lock.unlock(); // entering new func scope return split_multiprompt_task(task); @@ -1577,9 +1577,9 @@ struct llama_server_context slot->reset(); - slot->infill = task.infill_mode; - slot->embedding = task.embedding_mode; - slot->task_id = task.id; + slot->infill = task.infill_mode; + slot->embedding = task.embedding_mode; + slot->task_id = task.id; slot->multitask_id = task.multitask_id; if (!launch_slot_with_data(slot, task.data)) @@ -1731,7 +1731,8 @@ struct llama_server_context const bool has_prompt = slot.prompt.is_array() || (slot.prompt.is_string() && !slot.prompt.get().empty()) || !slot.images.empty(); // empty prompt passed -> release the slot and send empty response - if (slot.state == IDLE && slot.command == LOAD_PROMPT && !has_prompt) + // note: infill mode allows empty prompt + if (slot.state == IDLE && slot.command == LOAD_PROMPT && !has_prompt && !slot.infill) { slot.release(); slot.print_timings(); @@ -2609,8 +2610,8 @@ static json format_final_response_oaicompat(const json &request, const task_resu {"object", streaming ? "chat.completion.chunk" : "chat.completion"}, {"usage", json{{"completion_tokens", num_tokens_predicted}, - {"prompt_tokens", num_prompt_tokens}, - {"total_tokens", num_tokens_predicted + num_prompt_tokens}}}, + {"prompt_tokens", num_prompt_tokens}, + {"total_tokens", num_tokens_predicted + num_prompt_tokens}}}, {"id", gen_chatcmplid()}}; if (server_verbose) { From 326b418b59b6d48d854c4461a2303e8ac0a311e6 Mon Sep 17 00:00:00 2001 From: Kawrakow <48489457+ikawrakow@users.noreply.github.com> Date: Fri, 12 Jan 2024 06:59:57 +0100 Subject: [PATCH 029/812] Importance Matrix calculation (#4861) * imatrix: 1st version * imatrix: WIP * Cleanup * Update examples/imatrix/imatrix.cpp Co-authored-by: Georgi Gerganov --------- Co-authored-by: Iwan Kawrakow Co-authored-by: Georgi Gerganov --- Makefile | 5 +- examples/CMakeLists.txt | 1 + examples/imatrix/CMakeLists.txt | 5 + examples/imatrix/imatrix.cpp | 380 ++++++++++++++++++++++++++++++++ ggml.c | 14 ++ ggml.h | 6 + 6 files changed, 410 insertions(+), 1 deletion(-) create mode 100644 examples/imatrix/CMakeLists.txt create mode 100644 examples/imatrix/imatrix.cpp diff --git a/Makefile b/Makefile index 4c7e175bf..05fe9a0f6 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ # Define the default target now so that it is always the first target BUILD_TARGETS = \ - main quantize quantize-stats perplexity embedding vdot q8dot train-text-from-scratch convert-llama2c-to-ggml \ + main quantize quantize-stats perplexity imatrix embedding vdot q8dot train-text-from-scratch convert-llama2c-to-ggml \ simple batched batched-bench save-load-state server gguf llama-bench libllava.a llava-cli baby-llama beam-search \ speculative infill tokenize benchmark-matmult parallel finetune export-lora lookahead lookup passkey tests/test-c.o @@ -614,6 +614,9 @@ quantize-stats: examples/quantize-stats/quantize-stats.cpp build-info.o ggml. perplexity: examples/perplexity/perplexity.cpp ggml.o llama.o $(COMMON_DEPS) $(OBJS) $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) +imatrix: examples/imatrix/imatrix.cpp ggml.o llama.o $(COMMON_DEPS) $(OBJS) + $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) + embedding: examples/embedding/embedding.cpp ggml.o llama.o $(COMMON_DEPS) $(OBJS) $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index 0c71cbdf7..fa127a3aa 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -36,6 +36,7 @@ else() add_subdirectory(lookahead) add_subdirectory(lookup) add_subdirectory(train-text-from-scratch) + add_subdirectory(imatrix) if (LLAMA_METAL) add_subdirectory(metal) endif() diff --git a/examples/imatrix/CMakeLists.txt b/examples/imatrix/CMakeLists.txt new file mode 100644 index 000000000..d688a1620 --- /dev/null +++ b/examples/imatrix/CMakeLists.txt @@ -0,0 +1,5 @@ +set(TARGET imatrix) +add_executable(${TARGET} imatrix.cpp) +install(TARGETS ${TARGET} RUNTIME) +target_link_libraries(${TARGET} PRIVATE common llama ${CMAKE_THREAD_LIBS_INIT}) +target_compile_features(${TARGET} PRIVATE cxx_std_11) diff --git a/examples/imatrix/imatrix.cpp b/examples/imatrix/imatrix.cpp new file mode 100644 index 000000000..1461bc963 --- /dev/null +++ b/examples/imatrix/imatrix.cpp @@ -0,0 +1,380 @@ +#include "common.h" +#include "llama.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if defined(_MSC_VER) +#pragma warning(disable: 4244 4267) // possible loss of data +#endif + +struct Stats { + std::vector values; + int ncall = 0; +}; + +struct StatParams { + std::string ofile = "imatrix.dat"; + int n_output_frequency = 10; + int verbosity = 1; + bool collect_output_weight = false; +}; + +class IMatrixCollector { +public: + IMatrixCollector() = default; + void set_parameters(StatParams&& params) { m_params = std::move(params); } + void collect_imatrix(const struct ggml_tensor * src0, const struct ggml_tensor * src1); + void save_imatrix() const; +private: + std::unordered_map m_stats; + StatParams m_params; + std::mutex m_mutex; + int m_last_call = 0; +}; + +void IMatrixCollector::collect_imatrix(const struct ggml_tensor * src0, const struct ggml_tensor * src1) { + if (src1->ne[1] < 16 || src1->type != GGML_TYPE_F32) return; + if (!(strncmp(src0->name, "blk.", 4) == 0 || (m_params.collect_output_weight && strcmp(src0->name, "output.weight") == 0))) return; + std::lock_guard lock(m_mutex); + auto& e = m_stats[src0->name]; + if (e.values.empty()) { + e.values.resize(src1->ne[0], 0); + } + else if (e.values.size() != (size_t)src1->ne[0]) { + fprintf(stderr, "Oops: inconsistent size for %s (%d vs %d)\n", src0->name, (int)e.values.size(), (int)src1->ne[0]); + exit(1); //GGML_ASSERT(false); + } + ++e.ncall; + if (m_params.verbosity > 1) { + printf("%s[%d]: %s, %d x %d, %d\n",__func__,m_last_call,src0->name,(int)src1->ne[0],(int)src1->ne[1],(int)src1->type); + } + for (int row = 0; row < (int)src1->ne[1]; ++row) { + const float * x = (const float *)src1->data + row * src1->ne[0]; + for (int j = 0; j < (int)src1->ne[0]; ++j) { + e.values[j] += x[j]*x[j]; + } + } + if (e.ncall > m_last_call) { + m_last_call = e.ncall; + if (m_last_call % m_params.n_output_frequency == 0) { + save_imatrix(); + } + } +} + +void IMatrixCollector::save_imatrix() const { + const char * fname = m_params.ofile.empty() ? "imatrix.dat" : m_params.ofile.c_str(); + std::ofstream out(fname, std::ios::binary); + int n_entries = m_stats.size(); + out.write((const char*)&n_entries, sizeof(n_entries)); + for (auto& p : m_stats) { + int len = p.first.size(); + out.write((const char*)&len, sizeof(len)); + out.write(p.first.c_str(), len); + out.write((const char*)&p.second.ncall, sizeof(p.second.ncall)); + int nval = p.second.values.size(); + out.write((const char*)&nval, sizeof(nval)); + if (nval > 0) out.write((const char*)p.second.values.data(), nval*sizeof(float)); + } + if (m_params.verbosity > 0) { + fprintf(stderr, "\n%s: stored collected data after %d chunks in %s\n",__func__,m_last_call,fname); + } +} + +static IMatrixCollector g_collector; + +static void ik_collect_imatrix(const struct ggml_tensor * src0, const struct ggml_tensor * src1) { + g_collector.collect_imatrix(src0, src1); +} + + +struct results_log_softmax { + double log_softmax; + float logit; + float prob; +}; + +static std::vector softmax(const std::vector& logits) { + std::vector probs(logits.size()); + float max_logit = logits[0]; + for (float v : logits) { + max_logit = std::max(max_logit, v); + } + double sum_exp = 0.0; + for (size_t i = 0; i < logits.size(); i++) { + // Subtract the maximum logit value from the current logit value for numerical stability + const float logit = logits[i] - max_logit; + const float exp_logit = expf(logit); + sum_exp += exp_logit; + probs[i] = exp_logit; + } + for (size_t i = 0; i < probs.size(); i++) { + probs[i] /= sum_exp; + } + return probs; +} + +static results_log_softmax log_softmax(int n_vocab, const float * logits, int tok) { + float max_logit = logits[0]; + for (int i = 1; i < n_vocab; ++i) { + max_logit = std::max(max_logit, logits[i]); + } + double sum_exp = 0.0; + for (int i = 0; i < n_vocab; ++i) { + sum_exp += expf(logits[i] - max_logit); + } + return {logits[tok] - max_logit - log(sum_exp), logits[tok], expf(logits[tok] - max_logit) / (float) sum_exp}; +} + +static void process_logits( + int n_vocab, const float * logits, const int * tokens, int n_token, std::vector & workers, + double & nll, double & nll2, float * logit_history, float * prob_history +) { + std::mutex mutex; + int counter = 0; + auto compute = [&mutex, &counter, &nll, &nll2, logit_history, prob_history, n_vocab, logits, tokens, n_token] () { + double local_nll = 0; + double local_nll2 = 0; + while (true) { + std::unique_lock lock(mutex); + int i = counter++; + if (i >= n_token) { + nll += local_nll; nll2 += local_nll2; + break; + } + lock.unlock(); + const results_log_softmax results = log_softmax(n_vocab, logits + i*n_vocab, tokens[i+1]); + const double v = -results.log_softmax; + local_nll += v; + local_nll2 += v*v; + + logit_history[i] = results.logit; + prob_history[i] = results.prob; + } + }; + for (auto & w : workers) { + w = std::thread(compute); + } + compute(); + for (auto & w : workers) { + w.join(); + } +} + +static bool compute_imatrix(llama_context * ctx, const gpt_params & params) { + + const bool add_bos = llama_should_add_bos_token(llama_get_model(ctx)); + const int n_ctx = llama_n_ctx(ctx); + + auto tim1 = std::chrono::high_resolution_clock::now(); + fprintf(stderr, "%s: tokenizing the input ..\n", __func__); + + std::vector tokens = ::llama_tokenize(ctx, params.prompt, add_bos); + + auto tim2 = std::chrono::high_resolution_clock::now(); + fprintf(stderr, "%s: tokenization took %g ms\n",__func__,1e-3*std::chrono::duration_cast(tim2-tim1).count()); + + if (int(tokens.size()) < 2*n_ctx) { + fprintf(stderr, "%s: you need at least %d tokens for a context of %d tokens\n",__func__,2*n_ctx, + n_ctx); + fprintf(stderr, "%s: the data file you provided tokenizes to only %zu tokens\n",__func__,tokens.size()); + return false; + } + + std::vector logit_history; + logit_history.resize(tokens.size()); + + std::vector prob_history; + prob_history.resize(tokens.size()); + + const int n_chunk_max = tokens.size() / n_ctx; + + const int n_chunk = params.n_chunks < 0 ? n_chunk_max : std::min(params.n_chunks, n_chunk_max); + const int n_vocab = llama_n_vocab(llama_get_model(ctx)); + const int n_batch = params.n_batch; + + int count = 0; + double nll = 0.0; + double nll2 = 0.0; + + fprintf(stderr, "%s: computing over %d chunks with batch_size %d\n", __func__, n_chunk, n_batch); + + std::vector workers(std::thread::hardware_concurrency() - 1); + + for (int i = 0; i < n_chunk; ++i) { + const int start = i * n_ctx; + const int end = start + n_ctx; + + const int num_batches = (n_ctx + n_batch - 1) / n_batch; + + std::vector logits; + + const auto t_start = std::chrono::high_resolution_clock::now(); + + // clear the KV cache + llama_kv_cache_clear(ctx); + + for (int j = 0; j < num_batches; ++j) { + const int batch_start = start + j * n_batch; + const int batch_size = std::min(end - batch_start, n_batch); + + // save original token and restore it after eval + const auto token_org = tokens[batch_start]; + + // add BOS token for the first batch of each chunk + if (add_bos && j == 0) { + tokens[batch_start] = llama_token_bos(llama_get_model(ctx)); + } + + if (llama_decode(ctx, llama_batch_get_one(tokens.data() + batch_start, batch_size, j * n_batch, 0))) { + fprintf(stderr, "%s : failed to eval\n", __func__); + return false; + } + + // restore the original token in case it was set to BOS + tokens[batch_start] = token_org; + + const auto * batch_logits = llama_get_logits(ctx); + logits.insert(logits.end(), batch_logits, batch_logits + batch_size * n_vocab); + } + + const auto t_end = std::chrono::high_resolution_clock::now(); + + if (i == 0) { + const float t_total = std::chrono::duration(t_end - t_start).count(); + fprintf(stderr, "%s: %.2f seconds per pass - ETA ", __func__, t_total); + int total_seconds = (int)(t_total * n_chunk); + if (total_seconds >= 60*60) { + fprintf(stderr, "%d hours ", total_seconds / (60*60)); + total_seconds = total_seconds % (60*60); + } + fprintf(stderr, "%.2f minutes\n", total_seconds / 60.0); + } + + const int first = n_ctx/2; + process_logits(n_vocab, logits.data() + first*n_vocab, tokens.data() + start + first, n_ctx - 1 - first, + workers, nll, nll2, logit_history.data() + start + first, prob_history.data() + start + first); + count += n_ctx - first - 1; + + printf("[%d]%.4lf,", i + 1, std::exp(nll / count)); + fflush(stdout); + } + printf("\n"); + + nll2 /= count; + nll /= count; + const double ppl = exp(nll); + nll2 -= nll * nll; + if (nll2 > 0) { + nll2 = sqrt(nll2/(count-1)); + printf("Final estimate: PPL = %.4lf +/- %.5lf\n", ppl, nll2*ppl); + } else { + printf("Unexpected negative standard deviation of log(prob)\n"); + } + + return true; +} + +int main(int argc, char ** argv) { + + StatParams sparams; + std::vector args; + args.push_back(argv[0]); + int iarg = 1; + for (; iarg < argc-1; ++iarg) { + std::string arg{argv[iarg]}; + if (arg == "-o" || arg == "--output-file") { + sparams.ofile = argv[++iarg]; + } + else if (arg == "-ofreq" || arg == "--output-frequency") { + sparams.n_output_frequency = std::stoi(argv[++iarg]); + } + else if (arg == "-ow" || arg == "--output-weight") { + sparams.collect_output_weight = std::stoi(argv[++iarg]); + } + else if (arg == "--verbosity") { + sparams.verbosity = std::stoi(argv[++iarg]); + } else { + args.push_back(argv[iarg]); + } + } + if (iarg < argc) { + args.push_back(argv[iarg]); + } + + gpt_params params; + params.n_batch = 512; + if (!gpt_params_parse(args.size(), args.data(), params)) { + return 1; + } + + g_collector.set_parameters(std::move(sparams)); + + ggml_set_imatrix_collection(ik_collect_imatrix); + + params.logits_all = true; + params.n_batch = std::min(params.n_batch, params.n_ctx); + + print_build_info(); + + if (params.seed == LLAMA_DEFAULT_SEED) { + params.seed = time(NULL); + } + + fprintf(stderr, "%s: seed = %u\n", __func__, params.seed); + + std::mt19937 rng(params.seed); + if (params.random_prompt) { + params.prompt = gpt_random_prompt(rng); + } + + llama_backend_init(params.numa); + + llama_model * model; + llama_context * ctx; + + // load the model and apply lora adapter, if any + std::tie(model, ctx) = llama_init_from_gpt_params(params); + if (model == NULL) { + fprintf(stderr, "%s: error: unable to load model\n", __func__); + return 1; + } + + const int n_ctx_train = llama_n_ctx_train(model); + if (params.n_ctx > n_ctx_train) { + fprintf(stderr, "%s: warning: model was trained on only %d context tokens (%d specified)\n", + __func__, n_ctx_train, params.n_ctx); + } + + // print system information + { + fprintf(stderr, "\n"); + fprintf(stderr, "%s\n", get_system_info(params).c_str()); + } + + bool OK = compute_imatrix(ctx, params); + if (!OK) { + return 1; + } + + g_collector.save_imatrix(); + + llama_print_timings(ctx); + + llama_free(ctx); + llama_free_model(model); + + llama_backend_free(); + + return 0; +} diff --git a/ggml.c b/ggml.c index d2a8c0478..f5caeba08 100644 --- a/ggml.c +++ b/ggml.c @@ -394,6 +394,12 @@ static const size_t CACHE_LINE_SIZE_F32 = CACHE_LINE_SIZE/sizeof(float); static void ggml_vec_dot_f32(const int n, float * restrict s, const float * restrict x, const float * restrict y); static void ggml_vec_dot_f16(const int n, float * restrict s, ggml_fp16_t * restrict x, ggml_fp16_t * restrict y); +ggml_collect_imatrix_t g_imatrix_collect = NULL; + +void ggml_set_imatrix_collection(ggml_collect_imatrix_t imatrix_collect) { + g_imatrix_collect = imatrix_collect; +} + static const ggml_type_traits_t type_traits[GGML_TYPE_COUNT] = { [GGML_TYPE_I8] = { .type_name = "i8", @@ -9763,6 +9769,10 @@ static void ggml_compute_forward_mul_mat( const int ith = params->ith; const int nth = params->nth; + if (ith == 1 && g_imatrix_collect) { + g_imatrix_collect(src0, src1); + } + const enum ggml_type type = src0->type; const bool src1_cont = ggml_is_contiguous(src1); @@ -10066,6 +10076,10 @@ static void ggml_compute_forward_mul_mat_id( const struct ggml_tensor * src0_cur = dst->src[cur_a + 2]; + if (ith == 1 && g_imatrix_collect) { + g_imatrix_collect(src0_cur, src1); + } + const void * wdata = (src1->type == vec_dot_type) ? src1->data : params->wdata; const size_t row_size = ggml_row_size(vec_dot_type, ne10); diff --git a/ggml.h b/ggml.h index 93b42a27d..4c2ff6c66 100644 --- a/ggml.h +++ b/ggml.h @@ -2067,6 +2067,12 @@ extern "C" { GGML_API size_t ggml_quantize_chunk(enum ggml_type type, const float * src, void * dst, int start, int n, int64_t * hist); + // + // Importance matrix + // + typedef void(*ggml_collect_imatrix_t)(const struct ggml_tensor * src0, const struct ggml_tensor * src1); + GGML_API void ggml_set_imatrix_collection(ggml_collect_imatrix_t imatrix_collect); + // // gguf // From f445c0e68cf8e1faca0b2aa8dfb9d48231cec301 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Fri, 12 Jan 2024 13:01:56 +0200 Subject: [PATCH 030/812] llama : fix llm_build_k_shift to use correct n_rot (#4889) * llama : fix llm_build_k_shift to use correct n_rot ggml-ci * llama : always use hparams.n_rot for ggml_rope_custom ggml-ci * convert : fix persimmon conversion to write correct n_rot --- common/common.cpp | 3 ++ convert-hf-to-gguf.py | 9 ++++- gguf-py/gguf/tensor_mapping.py | 7 ++++ llama.cpp | 65 +++++++++++++++++----------------- 4 files changed, 51 insertions(+), 33 deletions(-) diff --git a/common/common.cpp b/common/common.cpp index b2cb0e257..3aefed01d 100644 --- a/common/common.cpp +++ b/common/common.cpp @@ -1055,6 +1055,9 @@ struct llama_model_params llama_model_params_from_gpt_params(const gpt_params & } static ggml_type kv_cache_type_from_str(const std::string & s) { + if (s == "f32") { + return GGML_TYPE_F32; + } if (s == "f16") { return GGML_TYPE_F16; } diff --git a/convert-hf-to-gguf.py b/convert-hf-to-gguf.py index 203eaf64b..813aeeed6 100755 --- a/convert-hf-to-gguf.py +++ b/convert-hf-to-gguf.py @@ -817,10 +817,17 @@ class PersimmonModel(Model): hidden_size = self.hparams["hidden_size"] self.gguf_writer.add_name('persimmon-8b-chat') + self.gguf_writer.add_context_length(self.hparams["max_position_embeddings"]) self.gguf_writer.add_embedding_length(hidden_size) self.gguf_writer.add_block_count(block_count) self.gguf_writer.add_feed_forward_length(self.hparams["intermediate_size"]) - self.gguf_writer.add_rope_dimension_count(hidden_size // head_count) + + # NOTE: not sure about this change - why does the model not have a rope dimension count when it is smaller + # than the head size? + # ref: https://github.com/ggerganov/llama.cpp/pull/4889 + #self.gguf_writer.add_rope_dimension_count(hidden_size // head_count) + self.gguf_writer.add_rope_dimension_count(hidden_size // head_count // 2) + self.gguf_writer.add_head_count(head_count) self.gguf_writer.add_head_count_kv(head_count_kv) self.gguf_writer.add_rope_freq_base(self.hparams["rope_theta"]) diff --git a/gguf-py/gguf/tensor_mapping.py b/gguf-py/gguf/tensor_mapping.py index 80c1d5449..24a089037 100644 --- a/gguf-py/gguf/tensor_mapping.py +++ b/gguf-py/gguf/tensor_mapping.py @@ -57,6 +57,7 @@ class TensorNameMap: "transformer.norm_f", # mpt "ln_f", # refact bloom qwen gpt2 "language_model.encoder.final_layernorm", # persimmon + "model.final_layernorm", # persimmon "lm_head.ln", # phi2 ), @@ -98,6 +99,7 @@ class TensorNameMap: "transformer.h.{bid}.self_attention.query_key_value", # falcon "h.{bid}.self_attention.query_key_value", # bloom "language_model.encoder.layers.{bid}.self_attention.query_key_value", # persimmon + "model.layers.{bid}.self_attn.query_key_value", # persimmon "h.{bid}.attn.c_attn", # gpt2 "transformer.h.{bid}.mixer.Wqkv", # phi2 ), @@ -141,6 +143,7 @@ class TensorNameMap: "encoder.layer.{bid}.attention.output.dense", # bert "transformer.h.{bid}.attn.out_proj", # gpt-j "language_model.encoder.layers.{bid}.self_attention.dense", # persimmon + "model.layers.{bid}.self_attn.dense", # persimmon "h.{bid}.attn.c_proj", # gpt2 "transformer.h.{bid}.mixer.out_proj", # phi2 "model.layers.layers.{bid}.self_attn.o_proj", # plamo @@ -184,6 +187,7 @@ class TensorNameMap: "encoder.layer.{bid}.intermediate.dense", # bert "transformer.h.{bid}.mlp.fc_in", # gpt-j "language_model.encoder.layers.{bid}.mlp.dense_h_to_4h", # persimmon + "model.layers.{bid}.mlp.dense_h_to_4h", # persimmon "transformer.h.{bid}.mlp.w1", # qwen "h.{bid}.mlp.c_fc", # gpt2 "transformer.h.{bid}.mlp.fc1", # phi2 @@ -225,6 +229,7 @@ class TensorNameMap: "encoder.layer.{bid}.output.dense", # bert "transformer.h.{bid}.mlp.fc_out", # gpt-j "language_model.encoder.layers.{bid}.mlp.dense_4h_to_h", # persimmon + "model.layers.{bid}.mlp.dense_4h_to_h", # persimmon "h.{bid}.mlp.c_proj", # gpt2 "transformer.h.{bid}.mlp.fc2", # phi2 "model.layers.layers.{bid}.mlp.down_proj", # plamo @@ -237,10 +242,12 @@ class TensorNameMap: MODEL_TENSOR.ATTN_Q_NORM: ( "language_model.encoder.layers.{bid}.self_attention.q_layernorm", + "model.layers.{bid}.self_attn.q_layernorm", # persimmon ), MODEL_TENSOR.ATTN_K_NORM: ( "language_model.encoder.layers.{bid}.self_attention.k_layernorm", + "model.layers.{bid}.self_attn.k_layernorm", # persimmon ), MODEL_TENSOR.ROPE_FREQS: ( diff --git a/llama.cpp b/llama.cpp index d39ff94c7..0bab95563 100644 --- a/llama.cpp +++ b/llama.cpp @@ -4104,7 +4104,6 @@ static void llm_build_k_shift( struct ggml_cgraph * graph, llm_rope_type type, int64_t n_ctx, - int n_rot, float freq_base, float freq_scale, const llm_build_cb & cb) { @@ -4112,14 +4111,13 @@ static void llm_build_k_shift( const int64_t n_head_kv = hparams.n_head_kv; const int64_t n_embd_head_k = hparams.n_embd_head_k; const int64_t n_embd_k_gqa = hparams.n_embd_k_gqa(); + const int32_t n_rot = hparams.n_rot; const int32_t n_orig_ctx = cparams.n_yarn_orig_ctx; const float ext_factor = cparams.yarn_ext_factor; const float attn_factor = cparams.yarn_attn_factor; const float beta_fast = cparams.yarn_beta_fast; const float beta_slow = cparams.yarn_beta_slow; - GGML_ASSERT(n_embd_head_k % n_rot == 0); - struct ggml_tensor * K_shift = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, n_ctx); cb(K_shift, "K_shift", -1); @@ -4523,7 +4521,7 @@ struct llm_build_context { // shift the entire K-cache if needed if (do_rope_shift) { - llm_build_k_shift(ctx0, hparams, cparams, kv_self, gf, LLM_ROPE, n_ctx, n_embd_head, freq_base, freq_scale, cb); + llm_build_k_shift(ctx0, hparams, cparams, kv_self, gf, LLM_ROPE, n_ctx, freq_base, freq_scale, cb); } for (int il = 0; il < n_layer; ++il) { @@ -4561,14 +4559,14 @@ struct llm_build_context { Qcur = ggml_rope_custom( ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, - n_embd_head, 0, 0, n_orig_ctx, freq_base, freq_scale, + hparams.n_rot, 0, 0, n_orig_ctx, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); cb(Qcur, "Qcur", il); Kcur = ggml_rope_custom( ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, - n_embd_head, 0, 0, n_orig_ctx, freq_base, freq_scale, + hparams.n_rot, 0, 0, n_orig_ctx, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); cb(Kcur, "Kcur", il); @@ -4691,6 +4689,7 @@ struct llm_build_context { const int64_t n_embd_head = hparams.n_embd_head_v; GGML_ASSERT(n_embd_head == hparams.n_embd_head_k); + GGML_ASSERT(n_embd_head == hparams.n_rot); struct ggml_tensor * cur; struct ggml_tensor * inpL; @@ -4708,7 +4707,7 @@ struct llm_build_context { // shift the entire K-cache if needed if (do_rope_shift) { - llm_build_k_shift(ctx0, hparams, cparams, kv_self, gf, LLM_ROPE, n_ctx, n_embd_head, freq_base, freq_scale, cb); + llm_build_k_shift(ctx0, hparams, cparams, kv_self, gf, LLM_ROPE, n_ctx, freq_base, freq_scale, cb); } for (int il = 0; il < n_layer; ++il) { @@ -4734,12 +4733,12 @@ struct llm_build_context { case MODEL_7B: Qcur = ggml_rope_custom( ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, - n_embd_head, 0, 0, n_orig_ctx, freq_base, freq_scale, + hparams.n_rot, 0, 0, n_orig_ctx, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); Kcur = ggml_rope_custom( ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, - n_embd_head, 0, 0, n_orig_ctx, freq_base, freq_scale, + hparams.n_rot, 0, 0, n_orig_ctx, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); break; @@ -4812,6 +4811,7 @@ struct llm_build_context { const int64_t n_embd_head = hparams.n_embd_head_v; const int64_t n_embd_gqa = hparams.n_embd_v_gqa(); GGML_ASSERT(n_embd_head == hparams.n_embd_head_k); + GGML_ASSERT(n_embd_head == hparams.n_rot); struct ggml_tensor * cur; struct ggml_tensor * inpL; @@ -4829,7 +4829,7 @@ struct llm_build_context { // shift the entire K-cache if needed if (do_rope_shift) { - llm_build_k_shift(ctx0, hparams, cparams, kv_self, gf, LLM_ROPE_NEOX, n_ctx, n_embd_head, freq_base, freq_scale, cb); + llm_build_k_shift(ctx0, hparams, cparams, kv_self, gf, LLM_ROPE_NEOX, n_ctx, freq_base, freq_scale, cb); } for (int il = 0; il < n_layer; ++il) { @@ -4870,13 +4870,13 @@ struct llm_build_context { // using mode = 2 for neox mode Qcur = ggml_rope_custom( - ctx0, Qcur, inp_pos, n_embd_head, 2, 0, n_orig_ctx, + ctx0, Qcur, inp_pos, hparams.n_rot, 2, 0, n_orig_ctx, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); cb(Qcur, "Qcur", il); Kcur = ggml_rope_custom( - ctx0, Kcur, inp_pos, n_embd_head, 2, 0, n_orig_ctx, + ctx0, Kcur, inp_pos, hparams.n_rot, 2, 0, n_orig_ctx, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); cb(Kcur, "Kcur", il); @@ -5033,9 +5033,8 @@ struct llm_build_context { struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, LLAMA_MAX_NODES, false); const int64_t n_embd_head = hparams.n_embd_head_v; - GGML_ASSERT(n_embd_head == hparams.n_embd_head_k); - - const int64_t n_rot = n_embd_head_k / 2; + GGML_ASSERT(n_embd_head == hparams.n_embd_head_k); + GGML_ASSERT(n_embd_head/2 == hparams.n_rot); struct ggml_tensor * cur; struct ggml_tensor * inpL; @@ -5052,7 +5051,7 @@ struct llm_build_context { cb(KQ_mask, "KQ_mask", -1); if (do_rope_shift) { - llm_build_k_shift(ctx0, hparams, cparams, kv_self, gf, LLM_ROPE_NEOX, n_ctx, n_embd_head, freq_base, freq_scale, cb); + llm_build_k_shift(ctx0, hparams, cparams, kv_self, gf, LLM_ROPE_NEOX, n_ctx, freq_base, freq_scale, cb); } for (int il = 0; il < n_layer; ++il) { @@ -5112,7 +5111,7 @@ struct llm_build_context { // RoPE the first n_rot of q/k, pass the other half, and concat. struct ggml_tensor * qrot = ggml_view_3d( - ctx0, tmpq, n_rot, n_head, n_tokens, + ctx0, tmpq, hparams.n_rot, n_head, n_tokens, ggml_element_size(tmpq) * n_embd_head, ggml_element_size(tmpq) * n_embd_head * n_head, 0 @@ -5120,7 +5119,7 @@ struct llm_build_context { cb(qrot, "qrot", il); struct ggml_tensor * krot = ggml_view_3d( - ctx0, tmpk, n_rot, n_head, n_tokens, + ctx0, tmpk, hparams.n_rot, n_head, n_tokens, ggml_element_size(tmpk) * n_embd_head, ggml_element_size(tmpk) * n_embd_head * n_head, 0 @@ -5129,29 +5128,29 @@ struct llm_build_context { // get the second half of tmpq, e.g tmpq[n_rot:, :, :] struct ggml_tensor * qpass = ggml_view_3d( - ctx0, tmpq, n_rot, n_head, n_tokens, + ctx0, tmpq, hparams.n_rot, n_head, n_tokens, ggml_element_size(tmpq) * n_embd_head, ggml_element_size(tmpq) * n_embd_head * n_head, - ggml_element_size(tmpq) * n_rot + ggml_element_size(tmpq) * hparams.n_rot ); cb(qpass, "qpass", il); struct ggml_tensor * kpass = ggml_view_3d( - ctx0, tmpk, n_rot, n_head, n_tokens, + ctx0, tmpk, hparams.n_rot, n_head, n_tokens, ggml_element_size(tmpk) * n_embd_head, ggml_element_size(tmpk) * n_embd_head * n_head, - ggml_element_size(tmpk) * n_rot + ggml_element_size(tmpk) * hparams.n_rot ); cb(kpass, "kpass", il); struct ggml_tensor * qrotated = ggml_rope_custom( - ctx0, qrot, inp_pos, n_rot, 2, 0, n_orig_ctx, + ctx0, qrot, inp_pos, hparams.n_rot, 2, 0, n_orig_ctx, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); cb(qrotated, "qrotated", il); struct ggml_tensor * krotated = ggml_rope_custom( - ctx0, krot, inp_pos, n_rot, 2, 0, n_orig_ctx, + ctx0, krot, inp_pos, hparams.n_rot, 2, 0, n_orig_ctx, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); cb(krotated, "krotated", il); @@ -5531,6 +5530,7 @@ struct llm_build_context { const int64_t n_embd_head = hparams.n_embd_head_v; GGML_ASSERT(n_embd_head == hparams.n_embd_head_k); + GGML_ASSERT(n_embd_head == hparams.n_rot); struct ggml_tensor * cur; struct ggml_tensor * inpL; @@ -5548,7 +5548,7 @@ struct llm_build_context { // shift the entire K-cache if needed if (do_rope_shift) { - llm_build_k_shift(ctx0, hparams, cparams, kv_self, gf, LLM_ROPE_NEOX, n_ctx, hparams.n_rot, freq_base, freq_scale, cb); + llm_build_k_shift(ctx0, hparams, cparams, kv_self, gf, LLM_ROPE_NEOX, n_ctx, freq_base, freq_scale, cb); } for (int il = 0; il < n_layer; ++il) { @@ -5661,7 +5661,7 @@ struct llm_build_context { // shift the entire K-cache if needed if (do_rope_shift) { - llm_build_k_shift(ctx0, hparams, cparams, kv_self, gf, LLM_ROPE_NEOX, n_ctx, n_embd_head, freq_base, freq_scale, cb); + llm_build_k_shift(ctx0, hparams, cparams, kv_self, gf, LLM_ROPE_NEOX, n_ctx, freq_base, freq_scale, cb); } for (int il = 0; il < n_layer; ++il) { @@ -5693,13 +5693,13 @@ struct llm_build_context { // using mode = 2 for neox mode Qcur = ggml_rope_custom( - ctx0, Qcur, inp_pos, n_embd_head, 2, 0, n_orig_ctx, + ctx0, Qcur, inp_pos, hparams.n_rot, 2, 0, n_orig_ctx, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); cb(Qcur, "Qcur", il); Kcur = ggml_rope_custom( - ctx0, Kcur, inp_pos, n_embd_head, 2, 0, n_orig_ctx, + ctx0, Kcur, inp_pos, hparams.n_rot, 2, 0, n_orig_ctx, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); cb(Kcur, "Kcur", il); @@ -5778,7 +5778,7 @@ struct llm_build_context { // shift the entire K-cache if needed if (do_rope_shift) { - llm_build_k_shift(ctx0, hparams, cparams, kv_self, gf, LLM_ROPE_NEOX, n_ctx, n_embd_head, freq_base, freq_scale, cb); + llm_build_k_shift(ctx0, hparams, cparams, kv_self, gf, LLM_ROPE_NEOX, n_ctx, freq_base, freq_scale, cb); } for (int il = 0; il < n_layer; ++il) { @@ -5874,6 +5874,7 @@ struct llm_build_context { const int64_t n_embd_head = hparams.n_embd_head_v; GGML_ASSERT(n_embd_head == hparams.n_embd_head_k); + GGML_ASSERT(n_embd_head == hparams.n_rot); struct ggml_tensor * cur; struct ggml_tensor * inpL; @@ -5891,7 +5892,7 @@ struct llm_build_context { // shift the entire K-cache if needed if (do_rope_shift) { - llm_build_k_shift(ctx0, hparams, cparams, kv_self, gf, LLM_ROPE, n_ctx, n_embd_head, freq_base, freq_scale, cb); + llm_build_k_shift(ctx0, hparams, cparams, kv_self, gf, LLM_ROPE, n_ctx, freq_base, freq_scale, cb); } for (int il = 0; il < n_layer; ++il) { @@ -5917,13 +5918,13 @@ struct llm_build_context { cb(Vcur, "Vcur", il); Qcur = ggml_rope_custom( - ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, + ctx0, ggml_reshape_3d(ctx0, Qcur, hparams.n_rot, n_head, n_tokens), inp_pos, n_embd_head, 2, 0, n_orig_ctx, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow); cb(Qcur, "Qcur", il); Kcur = ggml_rope_custom( - ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, + ctx0, ggml_reshape_3d(ctx0, Kcur, hparams.n_rot, n_head_kv, n_tokens), inp_pos, n_embd_head, 2, 0, n_orig_ctx, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow); cb(Kcur, "Kcur", il); From 2d00741e12c5db4a33dfccd1125f5de4adec9a5b Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Fri, 12 Jan 2024 13:03:38 +0200 Subject: [PATCH 031/812] py : fix lint (#4889) --- convert-hf-to-gguf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/convert-hf-to-gguf.py b/convert-hf-to-gguf.py index 813aeeed6..a1c79fd47 100755 --- a/convert-hf-to-gguf.py +++ b/convert-hf-to-gguf.py @@ -825,7 +825,7 @@ class PersimmonModel(Model): # NOTE: not sure about this change - why does the model not have a rope dimension count when it is smaller # than the head size? # ref: https://github.com/ggerganov/llama.cpp/pull/4889 - #self.gguf_writer.add_rope_dimension_count(hidden_size // head_count) + # self.gguf_writer.add_rope_dimension_count(hidden_size // head_count) self.gguf_writer.add_rope_dimension_count(hidden_size // head_count // 2) self.gguf_writer.add_head_count(head_count) From 4315a94366708828f949f9db89d2a8d99b634459 Mon Sep 17 00:00:00 2001 From: howlger Date: Fri, 12 Jan 2024 12:05:32 +0100 Subject: [PATCH 032/812] common : streamline the formatting of help (#4890) * common : streamline the formatting of help - Separate alternative parameters by a comma - Do not indent `--version` differently * Update common/common.cpp --------- Co-authored-by: Georgi Gerganov --- common/common.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/common/common.cpp b/common/common.cpp index 3aefed01d..062a8b4de 100644 --- a/common/common.cpp +++ b/common/common.cpp @@ -818,7 +818,7 @@ void gpt_print_usage(int /*argc*/, char ** argv, const gpt_params & params) { printf("\n"); printf("options:\n"); printf(" -h, --help show this help message and exit\n"); - printf(" --version show version and build info\n"); + printf(" --version show version and build info\n"); printf(" -i, --interactive run in interactive mode\n"); printf(" --interactive-first run in interactive mode and wait for input right away\n"); printf(" -ins, --instruct run in instruction mode (use with Alpaca models)\n"); @@ -915,7 +915,7 @@ void gpt_print_usage(int /*argc*/, char ** argv, const gpt_params & params) { printf(" number of layers to store in VRAM\n"); printf(" -ngld N, --n-gpu-layers-draft N\n"); printf(" number of layers to store in VRAM for the draft model\n"); - printf(" -ts SPLIT --tensor-split SPLIT\n"); + printf(" -ts SPLIT, --tensor-split SPLIT\n"); printf(" how to split tensors across multiple GPUs, comma-separated list of proportions, e.g. 3,1\n"); printf(" -mg i, --main-gpu i the GPU to use for scratch and small tensors\n"); #ifdef GGML_USE_CUBLAS @@ -950,7 +950,7 @@ void gpt_print_usage(int /*argc*/, char ** argv, const gpt_params & params) { printf(" --override-kv KEY=TYPE:VALUE\n"); printf(" advanced option to override model metadata by key. may be specified multiple times.\n"); printf(" types: int, float, bool. example: --override-kv tokenizer.ggml.add_bos_token=bool:false\n"); - printf(" -stc N --print-token-count N\n"); + printf(" -ptc N, --print-token-count N\n"); printf(" print token count every N tokens (default: %d)\n", params.n_print); printf("\n"); #ifndef LOG_DISABLE_LOGS From 3cabe80630c7eeb57713cd02249053a8cf6894fa Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Fri, 12 Jan 2024 13:10:19 +0200 Subject: [PATCH 033/812] llama : fix typo "imp_embd" -> "inp_embd" --- llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llama.cpp b/llama.cpp index 0bab95563..29f8873f6 100644 --- a/llama.cpp +++ b/llama.cpp @@ -5040,7 +5040,7 @@ struct llm_build_context { struct ggml_tensor * inpL; inpL = llm_build_inp_embd(ctx0, hparams, batch, model.tok_embd, cb); - cb(inpL, "imp_embd", -1); + cb(inpL, "inp_embd", -1); // inp_pos - contains the positions struct ggml_tensor * inp_pos = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_tokens); From 1b280c9fffd682b6924010a4437f0275f2921fa9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20G=C3=A4=C3=9Fler?= Date: Fri, 12 Jan 2024 12:30:41 +0100 Subject: [PATCH 034/812] CUDA: fix softmax compile for old CUDA versions (#4862) --- ggml-cuda.cu | 34 ++++++++++++++++++---------------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/ggml-cuda.cu b/ggml-cuda.cu index dd19699f6..a345b0c4a 100644 --- a/ggml-cuda.cu +++ b/ggml-cuda.cu @@ -116,6 +116,8 @@ #include "ggml.h" #include "ggml-backend-impl.h" +#define CUDART_HMAX 11070 // CUDA 11.7, min. ver. for which __hmax and __hmax2 are known to work (may be higher than needed) + #define CC_PASCAL 600 #define MIN_CC_DP4A 610 // minimum compute capability for __dp4a, an intrinsic for byte-wise dot products #define CC_VOLTA 700 @@ -605,16 +607,16 @@ static __device__ __forceinline__ float2 warp_reduce_sum(float2 a) { } static __device__ __forceinline__ half2 warp_reduce_sum(half2 a) { -#if __CUDA_ARCH__ < CC_PASCAL || (defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)) - (void) a; - bad_arch(); -#else +#if !(defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)) && __CUDA_ARCH__ >= CC_PASCAL #pragma unroll for (int mask = 16; mask > 0; mask >>= 1) { a = __hadd2(a, __shfl_xor_sync(0xffffffff, a, mask, 32)); } return a; -#endif // __CUDA_ARCH__ < CC_PASCAL || (defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)) +#else + (void) a; + bad_arch(); +#endif // !(defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)) && __CUDA_ARCH__ >= CC_PASCAL } static __device__ __forceinline__ float warp_reduce_max(float x) { @@ -626,16 +628,16 @@ static __device__ __forceinline__ float warp_reduce_max(float x) { } static __device__ __forceinline__ half2 warp_reduce_max(half2 x) { -#if __CUDA_ARCH__ < CC_PASCAL || (defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)) - (void) x; - bad_arch(); -#else +#if !(defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)) && __CUDA_ARCH__ >= CC_PASCAL && CUDART_VERSION >= CUDART_HMAX #pragma unroll for (int mask = 16; mask > 0; mask >>= 1) { x = __hmax2(x, __shfl_xor_sync(0xffffffff, x, mask, 32)); } return x; -#endif // __CUDA_ARCH__ < CC_PASCAL || (defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)) +#else + (void) x; + bad_arch(); +#endif // !(defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)) && __CUDA_ARCH__ >= CC_PASCAL && CUDART_VERSION >= CUDART_HMAX } static __device__ __forceinline__ float op_repeat(const float a, const float b) { @@ -5613,7 +5615,7 @@ static __global__ void diag_mask_inf_f32(const float * x, float * dst, const int template static __global__ void soft_max_f16(const float * x, const float * y, float * dst, const int ncols_par, const int nrows_y, const float scale) { -#if !(defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)) && __CUDA_ARCH__ >= CC_PASCAL +#if !(defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)) && __CUDA_ARCH__ >= CC_PASCAL && CUDART_VERSION >= CUDART_HMAX const int ncols_data = ncols_template == 0 ? ncols_par : ncols_template; const int ncols_smem = GGML_PAD(ncols_data, 2*WARP_SIZE)/2; @@ -5738,7 +5740,7 @@ static __global__ void soft_max_f16(const float * x, const float * y, float * ds #else (void) x; (void) y; (void) dst; (void) ncols_par; (void) nrows_y; (void) scale; bad_arch(); -#endif // !(defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)) && __CUDA_ARCH__ >= CC_PASCAL +#endif // !(defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)) && __CUDA_ARCH__ >= CC_PASCAL && CUDART_VERSION >= CUDART_HMAX } template @@ -8574,15 +8576,15 @@ static void ggml_cuda_op_soft_max( float scale = 1.0f; memcpy(&scale, dst->op_params, sizeof(float)); -#if defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__) - const bool use_f16_soft_max = false; -#else +#if !(defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)) && CUDART_VERSION >= CUDART_HMAX #ifdef GGML_CUDA_F16 const bool use_f16_soft_max = true; #else const bool use_f16_soft_max = false; #endif // GGML_CUDA_F16 -#endif // defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__) +#else + const bool use_f16_soft_max = false; +#endif // defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__) && CUDART_VERSION >= CUDART_HMAX if (use_f16_soft_max) { soft_max_f16_cuda(src0_dd, src1 ? src1_dd : nullptr, dst_dd, ne00, nrows_x, nrows_y, scale, main_stream); From 5537d9d36bfdb4379555431f574d3d78ce6e7955 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Fri, 12 Jan 2024 14:33:21 +0200 Subject: [PATCH 035/812] gitignore : imatrix --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index cf1b692e9..fba207045 100644 --- a/.gitignore +++ b/.gitignore @@ -43,6 +43,7 @@ models-mnt /embedding /gguf /gguf-llama-simple +/imatrix /infill /libllama.so /llama-bench From e790eef21ce659f5c16d59f8a5c8dcf6cde0692a Mon Sep 17 00:00:00 2001 From: Zay <95888118+isaiahbjork@users.noreply.github.com> Date: Fri, 12 Jan 2024 05:48:00 -0700 Subject: [PATCH 036/812] llama.swiftui : update models layout (#4826) * Updated Models Layout - Added a models drawer - Added downloading directly from Hugging Face - Load custom models from local folder - Delete models by swiping left * trimmed trailing white space * Updated Models Layout --- .../llama.swiftui.xcodeproj/project.pbxproj | 8 +- .../llama.swiftui/Models/LlamaState.swift | 89 ++++++++ .../llama.swiftui/UI/ContentView.swift | 213 +++++++++--------- .../llama.swiftui/UI/DownloadButton.swift | 2 + .../llama.swiftui/UI/InputButton.swift | 131 +++++++++++ 5 files changed, 338 insertions(+), 105 deletions(-) create mode 100644 examples/llama.swiftui/llama.swiftui/UI/InputButton.swift diff --git a/examples/llama.swiftui/llama.swiftui.xcodeproj/project.pbxproj b/examples/llama.swiftui/llama.swiftui.xcodeproj/project.pbxproj index a8848a49f..3950b9e9d 100644 --- a/examples/llama.swiftui/llama.swiftui.xcodeproj/project.pbxproj +++ b/examples/llama.swiftui/llama.swiftui.xcodeproj/project.pbxproj @@ -8,6 +8,7 @@ /* Begin PBXBuildFile section */ 549479CB2AC9E16000E0F78B /* Metal.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 549479CA2AC9E16000E0F78B /* Metal.framework */; }; + 79E1D9CD2B4CD16E005F8E46 /* InputButton.swift in Sources */ = {isa = PBXBuildFile; fileRef = 79E1D9CC2B4CD16E005F8E46 /* InputButton.swift */; }; 7FA3D2B32B2EA2F600543F92 /* DownloadButton.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7FA3D2B22B2EA2F600543F92 /* DownloadButton.swift */; }; 8A1C83772AC328BD0096AF73 /* llama_swiftuiApp.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8A1C83762AC328BD0096AF73 /* llama_swiftuiApp.swift */; }; 8A1C83792AC328BD0096AF73 /* ContentView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8A1C83782AC328BD0096AF73 /* ContentView.swift */; }; @@ -22,6 +23,7 @@ /* Begin PBXFileReference section */ 549479CA2AC9E16000E0F78B /* Metal.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; name = Metal.framework; path = System/Library/Frameworks/Metal.framework; sourceTree = SDKROOT; }; + 79E1D9CC2B4CD16E005F8E46 /* InputButton.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = InputButton.swift; sourceTree = ""; }; 7FA3D2B22B2EA2F600543F92 /* DownloadButton.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = DownloadButton.swift; sourceTree = ""; }; 8A1C83732AC328BD0096AF73 /* llama.swiftui.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = llama.swiftui.app; sourceTree = BUILT_PRODUCTS_DIR; }; 8A1C83762AC328BD0096AF73 /* llama_swiftuiApp.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = llama_swiftuiApp.swift; sourceTree = ""; }; @@ -119,6 +121,7 @@ 7FA3D2B22B2EA2F600543F92 /* DownloadButton.swift */, 8A1C83782AC328BD0096AF73 /* ContentView.swift */, F1FE20E12B465EC900B45541 /* LoadCustomButton.swift */, + 79E1D9CC2B4CD16E005F8E46 /* InputButton.swift */, ); path = UI; sourceTree = ""; @@ -213,6 +216,7 @@ 8A1C83792AC328BD0096AF73 /* ContentView.swift in Sources */, 8A1C83772AC328BD0096AF73 /* llama_swiftuiApp.swift in Sources */, 7FA3D2B32B2EA2F600543F92 /* DownloadButton.swift in Sources */, + 79E1D9CD2B4CD16E005F8E46 /* InputButton.swift in Sources */, ); runOnlyForDeploymentPostprocessing = 0; }; @@ -345,7 +349,7 @@ CLANG_ENABLE_MODULES = YES; CODE_SIGN_STYLE = Automatic; CURRENT_PROJECT_VERSION = 1; - DEVELOPMENT_TEAM = STLSG3FG8Q; + DEVELOPMENT_TEAM = K5UQJPP73A; ENABLE_PREVIEWS = YES; GENERATE_INFOPLIST_FILE = YES; INFOPLIST_KEY_UIApplicationSceneManifest_Generation = YES; @@ -377,7 +381,7 @@ CLANG_ENABLE_MODULES = YES; CODE_SIGN_STYLE = Automatic; CURRENT_PROJECT_VERSION = 1; - DEVELOPMENT_TEAM = STLSG3FG8Q; + DEVELOPMENT_TEAM = K5UQJPP73A; ENABLE_PREVIEWS = YES; GENERATE_INFOPLIST_FILE = YES; INFOPLIST_KEY_UIApplicationSceneManifest_Generation = YES; diff --git a/examples/llama.swiftui/llama.swiftui/Models/LlamaState.swift b/examples/llama.swiftui/llama.swiftui/Models/LlamaState.swift index 17cb5b9dd..5bde18917 100644 --- a/examples/llama.swiftui/llama.swiftui/Models/LlamaState.swift +++ b/examples/llama.swiftui/llama.swiftui/Models/LlamaState.swift @@ -1,9 +1,19 @@ import Foundation +struct Model: Identifiable { + var id = UUID() + var name: String + var url: String + var filename: String + var status: String? +} + @MainActor class LlamaState: ObservableObject { @Published var messageLog = "" @Published var cacheCleared = false + @Published var downloadedModels: [Model] = [] + @Published var undownloadedModels: [Model] = [] let NS_PER_S = 1_000_000_000.0 private var llamaContext: LlamaContext? @@ -13,23 +23,102 @@ class LlamaState: ObservableObject { } init() { + loadModelsFromDisk() + loadDefaultModels() + } + + private func loadModelsFromDisk() { + do { + let documentsURL = getDocumentsDirectory() + let modelURLs = try FileManager.default.contentsOfDirectory(at: documentsURL, includingPropertiesForKeys: nil, options: [.skipsHiddenFiles, .skipsSubdirectoryDescendants]) + for modelURL in modelURLs { + let modelName = modelURL.deletingPathExtension().lastPathComponent + downloadedModels.append(Model(name: modelName, url: "", filename: modelURL.lastPathComponent, status: "downloaded")) + } + } catch { + print("Error loading models from disk: \(error)") + } + } + + private func loadDefaultModels() { do { try loadModel(modelUrl: defaultModelUrl) } catch { messageLog += "Error!\n" } + + for model in defaultModels { + let fileURL = getDocumentsDirectory().appendingPathComponent(model.filename) + if FileManager.default.fileExists(atPath: fileURL.path) { + + } else { + var undownloadedModel = model + undownloadedModel.status = "download" + undownloadedModels.append(undownloadedModel) + } + } } + func getDocumentsDirectory() -> URL { + let paths = FileManager.default.urls(for: .documentDirectory, in: .userDomainMask) + return paths[0] + } + private let defaultModels: [Model] = [ + Model(name: "TinyLlama-1.1B (Q4_0, 0.6 GiB)",url: "https://huggingface.co/TheBloke/TinyLlama-1.1B-1T-OpenOrca-GGUF/resolve/main/tinyllama-1.1b-1t-openorca.Q4_0.gguf?download=true",filename: "tinyllama-1.1b-1t-openorca.Q4_0.gguf", status: "download"), + Model( + name: "TinyLlama-1.1B Chat (Q8_0, 1.1 GiB)", + url: "https://huggingface.co/TheBloke/TinyLlama-1.1B-Chat-v1.0-GGUF/resolve/main/tinyllama-1.1b-chat-v1.0.Q8_0.gguf?download=true", + filename: "tinyllama-1.1b-chat-v1.0.Q8_0.gguf", status: "download" + ), + + Model( + name: "TinyLlama-1.1B (F16, 2.2 GiB)", + url: "https://huggingface.co/ggml-org/models/resolve/main/tinyllama-1.1b/ggml-model-f16.gguf?download=true", + filename: "tinyllama-1.1b-f16.gguf", status: "download" + ), + + Model( + name: "Phi-2.7B (Q4_0, 1.6 GiB)", + url: "https://huggingface.co/ggml-org/models/resolve/main/phi-2/ggml-model-q4_0.gguf?download=true", + filename: "phi-2-q4_0.gguf", status: "download" + ), + + Model( + name: "Phi-2.7B (Q8_0, 2.8 GiB)", + url: "https://huggingface.co/ggml-org/models/resolve/main/phi-2/ggml-model-q8_0.gguf?download=true", + filename: "phi-2-q8_0.gguf", status: "download" + ), + + Model( + name: "Mistral-7B-v0.1 (Q4_0, 3.8 GiB)", + url: "https://huggingface.co/TheBloke/Mistral-7B-v0.1-GGUF/resolve/main/mistral-7b-v0.1.Q4_0.gguf?download=true", + filename: "mistral-7b-v0.1.Q4_0.gguf", status: "download" + ), + Model( + name: "OpenHermes-2.5-Mistral-7B (Q3_K_M, 3.52 GiB)", + url: "https://huggingface.co/TheBloke/OpenHermes-2.5-Mistral-7B-GGUF/resolve/main/openhermes-2.5-mistral-7b.Q3_K_M.gguf?download=true", + filename: "openhermes-2.5-mistral-7b.Q3_K_M.gguf", status: "download" + ) + ] func loadModel(modelUrl: URL?) throws { if let modelUrl { messageLog += "Loading model...\n" llamaContext = try LlamaContext.create_context(path: modelUrl.path()) messageLog += "Loaded model \(modelUrl.lastPathComponent)\n" + + // Assuming that the model is successfully loaded, update the downloaded models + updateDownloadedModels(modelName: modelUrl.lastPathComponent, status: "downloaded") } else { messageLog += "Load a model from the list below\n" } } + + private func updateDownloadedModels(modelName: String, status: String) { + undownloadedModels.removeAll { $0.name == modelName } + } + + func complete(text: String) async { guard let llamaContext else { return diff --git a/examples/llama.swiftui/llama.swiftui/UI/ContentView.swift b/examples/llama.swiftui/llama.swiftui/UI/ContentView.swift index 7c81ea256..30c2dc431 100644 --- a/examples/llama.swiftui/llama.swiftui/UI/ContentView.swift +++ b/examples/llama.swiftui/llama.swiftui/UI/ContentView.swift @@ -2,115 +2,57 @@ import SwiftUI struct ContentView: View { @StateObject var llamaState = LlamaState() - @State private var multiLineText = "" - - private static func cleanupModelCaches() { - // Delete all models (*.gguf) - let fileManager = FileManager.default - let documentsUrl = FileManager.default.urls(for: .documentDirectory, in: .userDomainMask)[0] - do { - let fileURLs = try fileManager.contentsOfDirectory(at: documentsUrl, includingPropertiesForKeys: nil) - for fileURL in fileURLs { - if fileURL.pathExtension == "gguf" { - try fileManager.removeItem(at: fileURL) - } - } - } catch { - print("Error while enumerating files \(documentsUrl.path): \(error.localizedDescription)") - } - } + @State private var showingHelp = false // To track if Help Sheet should be shown var body: some View { - VStack { - ScrollView(.vertical, showsIndicators: true) { - Text(llamaState.messageLog) - .font(.system(size: 12)) - .frame(maxWidth: .infinity, alignment: .leading) + NavigationView { + VStack { + ScrollView(.vertical, showsIndicators: true) { + Text(llamaState.messageLog) + .font(.system(size: 12)) + .frame(maxWidth: .infinity, alignment: .leading) + .padding() + .onTapGesture { + UIApplication.shared.sendAction(#selector(UIResponder.resignFirstResponder), to: nil, from: nil, for: nil) + } + } + + TextEditor(text: $multiLineText) + .frame(height: 80) + .padding() + .border(Color.gray, width: 0.5) + + HStack { + Button("Send") { + sendText() + } + + Button("Bench") { + bench() + } + + Button("Clear") { + clear() + } + + Button("Copy") { + UIPasteboard.general.string = llamaState.messageLog + } + } + .buttonStyle(.bordered) .padding() - .onTapGesture { - UIApplication.shared.sendAction(#selector(UIResponder.resignFirstResponder), to: nil, from: nil, for: nil) - } - } - TextEditor(text: $multiLineText) - .frame(height: 80) + NavigationLink(destination: DrawerView(llamaState: llamaState)) { + Text("View Models") + } .padding() - .border(Color.gray, width: 0.5) - HStack { - Button("Send") { - sendText() - } - - Button("Bench") { - bench() - } - - Button("Clear") { - clear() - } - - Button("Copy") { - UIPasteboard.general.string = llamaState.messageLog - } - }.buttonStyle(.bordered) - - VStack(alignment: .leading) { - DownloadButton( - llamaState: llamaState, - modelName: "TinyLlama-1.1B (Q4_0, 0.6 GiB)", - modelUrl: "https://huggingface.co/TheBloke/TinyLlama-1.1B-1T-OpenOrca-GGUF/resolve/main/tinyllama-1.1b-1t-openorca.Q4_0.gguf?download=true", - filename: "tinyllama-1.1b-1t-openorca.Q4_0.gguf" - ) - - DownloadButton( - llamaState: llamaState, - modelName: "TinyLlama-1.1B (Q8_0, 1.1 GiB)", - modelUrl: "https://huggingface.co/TheBloke/TinyLlama-1.1B-1T-OpenOrca-GGUF/resolve/main/tinyllama-1.1b-1t-openorca.Q8_0.gguf?download=true", - filename: "tinyllama-1.1b-1t-openorca.Q8_0.gguf" - ) - - DownloadButton( - llamaState: llamaState, - modelName: "TinyLlama-1.1B (F16, 2.2 GiB)", - modelUrl: "https://huggingface.co/ggml-org/models/resolve/main/tinyllama-1.1b/ggml-model-f16.gguf?download=true", - filename: "tinyllama-1.1b-f16.gguf" - ) - - DownloadButton( - llamaState: llamaState, - modelName: "Phi-2.7B (Q4_0, 1.6 GiB)", - modelUrl: "https://huggingface.co/ggml-org/models/resolve/main/phi-2/ggml-model-q4_0.gguf?download=true", - filename: "phi-2-q4_0.gguf" - ) - - DownloadButton( - llamaState: llamaState, - modelName: "Phi-2.7B (Q8_0, 2.8 GiB)", - modelUrl: "https://huggingface.co/ggml-org/models/resolve/main/phi-2/ggml-model-q8_0.gguf?download=true", - filename: "phi-2-q8_0.gguf" - ) - - DownloadButton( - llamaState: llamaState, - modelName: "Mistral-7B-v0.1 (Q4_0, 3.8 GiB)", - modelUrl: "https://huggingface.co/TheBloke/Mistral-7B-v0.1-GGUF/resolve/main/mistral-7b-v0.1.Q4_0.gguf?download=true", - filename: "mistral-7b-v0.1.Q4_0.gguf" - ) - - Button("Clear downloaded models") { - ContentView.cleanupModelCaches() - llamaState.cacheCleared = true - } - - LoadCustomButton(llamaState: llamaState) } - .padding(.top, 4) - .font(.system(size: 12)) - .frame(maxWidth: .infinity, alignment: .leading) + .padding() + .navigationBarTitle("Model Settings", displayMode: .inline) + } - .padding() } func sendText() { @@ -131,8 +73,73 @@ struct ContentView: View { await llamaState.clear() } } + struct DrawerView: View { + + @ObservedObject var llamaState: LlamaState + @State private var showingHelp = false + func delete(at offsets: IndexSet) { + offsets.forEach { offset in + let model = llamaState.downloadedModels[offset] + let fileURL = getDocumentsDirectory().appendingPathComponent(model.filename) + do { + try FileManager.default.removeItem(at: fileURL) + } catch { + print("Error deleting file: \(error)") + } + } + + // Remove models from downloadedModels array + llamaState.downloadedModels.remove(atOffsets: offsets) + } + + func getDocumentsDirectory() -> URL { + let paths = FileManager.default.urls(for: .documentDirectory, in: .userDomainMask) + return paths[0] + } + var body: some View { + List { + Section(header: Text("Download Models From Hugging Face")) { + HStack { + InputButton(llamaState: llamaState) + } + } + Section(header: Text("Downloaded Models")) { + ForEach(llamaState.downloadedModels) { model in + DownloadButton(llamaState: llamaState, modelName: model.name, modelUrl: model.url, filename: model.filename) + } + .onDelete(perform: delete) + } + Section(header: Text("Default Models")) { + ForEach(llamaState.undownloadedModels) { model in + DownloadButton(llamaState: llamaState, modelName: model.name, modelUrl: model.url, filename: model.filename) + } + } + + } + .listStyle(GroupedListStyle()) + .navigationBarTitle("Model Settings", displayMode: .inline).toolbar { + ToolbarItem(placement: .navigationBarTrailing) { + Button("Help") { + showingHelp = true + } + } + }.sheet(isPresented: $showingHelp) { // Sheet for help modal + VStack(alignment: .leading) { + VStack(alignment: .leading) { + Text("1. Make sure the model is in GGUF Format") + .padding() + Text("2. Copy the download link of the quantized model") + .padding() + } + Spacer() + } + } + } + } } -//#Preview { -// ContentView() -//} +struct ContentView_Previews: PreviewProvider { + static var previews: some View { + ContentView() + } +} diff --git a/examples/llama.swiftui/llama.swiftui/UI/DownloadButton.swift b/examples/llama.swiftui/llama.swiftui/UI/DownloadButton.swift index c9f322ca1..4584d6eaa 100644 --- a/examples/llama.swiftui/llama.swiftui/UI/DownloadButton.swift +++ b/examples/llama.swiftui/llama.swiftui/UI/DownloadButton.swift @@ -53,6 +53,8 @@ struct DownloadButton: View { llamaState.cacheCleared = false + let model = Model(name: modelName, url: modelUrl, filename: filename, status: "downloaded") + llamaState.downloadedModels.append(model) status = "downloaded" } } catch let err { diff --git a/examples/llama.swiftui/llama.swiftui/UI/InputButton.swift b/examples/llama.swiftui/llama.swiftui/UI/InputButton.swift new file mode 100644 index 000000000..c5ffbad4e --- /dev/null +++ b/examples/llama.swiftui/llama.swiftui/UI/InputButton.swift @@ -0,0 +1,131 @@ +import SwiftUI + +struct InputButton: View { + @ObservedObject var llamaState: LlamaState + @State private var inputLink: String = "" + @State private var status: String = "download" + @State private var filename: String = "" + + @State private var downloadTask: URLSessionDownloadTask? + @State private var progress = 0.0 + @State private var observation: NSKeyValueObservation? + + private static func extractModelInfo(from link: String) -> (modelName: String, filename: String)? { + guard let url = URL(string: link), + let lastPathComponent = url.lastPathComponent.components(separatedBy: ".").first, + let modelName = lastPathComponent.components(separatedBy: "-").dropLast().joined(separator: "-").removingPercentEncoding, + let filename = lastPathComponent.removingPercentEncoding else { + return nil + } + + return (modelName, filename) + } + + private static func getFileURL(filename: String) -> URL { + FileManager.default.urls(for: .documentDirectory, in: .userDomainMask)[0].appendingPathComponent(filename) + } + + private func download() { + guard let extractedInfo = InputButton.extractModelInfo(from: inputLink) else { + // Handle invalid link or extraction failure + return + } + + let (modelName, filename) = extractedInfo + self.filename = filename // Set the state variable + + status = "downloading" + print("Downloading model \(modelName) from \(inputLink)") + guard let url = URL(string: inputLink) else { return } + let fileURL = InputButton.getFileURL(filename: filename) + + downloadTask = URLSession.shared.downloadTask(with: url) { temporaryURL, response, error in + if let error = error { + print("Error: \(error.localizedDescription)") + return + } + + guard let response = response as? HTTPURLResponse, (200...299).contains(response.statusCode) else { + print("Server error!") + return + } + + do { + if let temporaryURL = temporaryURL { + try FileManager.default.copyItem(at: temporaryURL, to: fileURL) + print("Writing to \(filename) completed") + + llamaState.cacheCleared = false + + let model = Model(name: modelName, url: self.inputLink, filename: filename, status: "downloaded") + llamaState.downloadedModels.append(model) + status = "downloaded" + } + } catch let err { + print("Error: \(err.localizedDescription)") + } + } + + observation = downloadTask?.progress.observe(\.fractionCompleted) { progress, _ in + self.progress = progress.fractionCompleted + } + + downloadTask?.resume() + } + + var body: some View { + VStack { + HStack { + TextField("Paste Quantized Download Link", text: $inputLink) + .textFieldStyle(RoundedBorderTextFieldStyle()) + + Button(action: { + downloadTask?.cancel() + status = "download" + }) { + Text("Cancel") + } + } + + if status == "download" { + Button(action: download) { + Text("Download Custom Model") + } + } else if status == "downloading" { + Button(action: { + downloadTask?.cancel() + status = "download" + }) { + Text("Downloading \(Int(progress * 100))%") + } + } else if status == "downloaded" { + Button(action: { + let fileURL = InputButton.getFileURL(filename: self.filename) + if !FileManager.default.fileExists(atPath: fileURL.path) { + download() + return + } + do { + try llamaState.loadModel(modelUrl: fileURL) + } catch let err { + print("Error: \(err.localizedDescription)") + } + }) { + Text("Load Custom Model") + } + } else { + Text("Unknown status") + } + } + .onDisappear() { + downloadTask?.cancel() + } + .onChange(of: llamaState.cacheCleared) { newValue in + if newValue { + downloadTask?.cancel() + let fileURL = InputButton.getFileURL(filename: self.filename) + status = FileManager.default.fileExists(atPath: fileURL.path) ? "downloaded" : "download" + } + } + } +} From 930f907d3ece1eb5b0a1ec5e209983a66dcbfa68 Mon Sep 17 00:00:00 2001 From: Daniel Bevenius Date: Fri, 12 Jan 2024 18:54:53 +0100 Subject: [PATCH 037/812] export-lora : use LLAMA_FILE_MAGIC_GGLA (#4894) This commit replaces the magic number used in export-lora.cpp with the one defined in llama.h, which is indirectly included via common.h. Signed-off-by: Daniel Bevenius --- examples/export-lora/export-lora.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/examples/export-lora/export-lora.cpp b/examples/export-lora/export-lora.cpp index 58fbe204d..4cd5d99bb 100644 --- a/examples/export-lora/export-lora.cpp +++ b/examples/export-lora/export-lora.cpp @@ -245,9 +245,8 @@ static struct lora_data * load_lora(struct lora_info * info) { params_ggml.no_alloc = true; result->ctx = ggml_init(params_ggml); - uint32_t LLAMA_FILE_MAGIC_LORA = 0x67676C61; // 'ggla' uint32_t magic = file.read_u32(); - if (magic != LLAMA_FILE_MAGIC_LORA) { + if (magic != LLAMA_FILE_MAGIC_GGLA) { die_fmt("unexpected lora header file magic in '%s'", info->filename.c_str()); } uint32_t version = file.read_u32(); From 584d674be622fbf1578694ada6e62eebedbfd377 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Fri, 12 Jan 2024 20:54:12 +0200 Subject: [PATCH 038/812] llama : remove redundant assert for StableLM (#4901) --- llama.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/llama.cpp b/llama.cpp index 29f8873f6..ce413f605 100644 --- a/llama.cpp +++ b/llama.cpp @@ -5530,7 +5530,6 @@ struct llm_build_context { const int64_t n_embd_head = hparams.n_embd_head_v; GGML_ASSERT(n_embd_head == hparams.n_embd_head_k); - GGML_ASSERT(n_embd_head == hparams.n_rot); struct ggml_tensor * cur; struct ggml_tensor * inpL; From e7e4df031b9e29d4b55a4e0b0295187f6b213db1 Mon Sep 17 00:00:00 2001 From: slaren Date: Fri, 12 Jan 2024 20:07:38 +0100 Subject: [PATCH 039/812] llama : ggml-backend integration (#4766) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * llama : ggml-backend integration * ggml-backend : add names to buffers * fix unmap after loading * batched-bench : add tensor_split param * llama : check for null tensor_split * ggml-backend : increase GGML_MAX_BACKENDS * improve graph splitting, partial fix for --no-kv-offload * cuda : add ggml-backend split buffer support * cuda : do not create buffer types for devices that don't exist (fixes usage without CUDA devices available) * ggml : fix null backend dereference (#4807) * ggml : fix null backend dereference * ggml : also check ggml_backend_is_cpu * test-backend-ops : check buffer allocation failures * llama : add cparam (split_mode) and command line argument (--split-mode, -sm) to configure the split mode (none, layer or row) * ggml : fix mul_mat_id work size * llama : rewrite session kv load/set without graphs * minor * llama : only initialize used backends, free backends on context free * llama : abort ctx if cuda backend init fails * llama : rewrite lora with ggml-backend and compute on CPU ggml-ci * llama : only map to a backend buffer the region of the file mapping containing the tensors used in the buffer * opencl : add ggml-backend buffer type * cuda : only use batched_cublas with batched mat muls (fixes fp16 tg perf) * llama : on Metal, by default offload the full model ggml-ci * metal : page align the data ptr (#4854) * Apply suggestions from code review Co-authored-by: Johannes Gäßler * cuda : fix split buffer free * address review comments * llama-bench : add split-mode parameter * fix whitespace * opencl : fix double initialization * server : add --split-mode parameter * use async copy and compute to improve multi-gpu performance ggml-ci * use async memcpys to copy the graph outputs to the CPU * fix opencl * use a host buffer for the cpu compute buffer for faster copies to the gpu --------- Co-authored-by: Georgi Gerganov Co-authored-by: Johannes Gäßler --- common/common.cpp | 65 +- common/common.h | 1 + examples/batched-bench/batched-bench.cpp | 3 + examples/llama-bench/llama-bench.cpp | 146 +- examples/server/server.cpp | 40 +- ggml-alloc.c | 34 +- ggml-alloc.h | 4 +- ggml-backend-impl.h | 38 +- ggml-backend.c | 693 ++++--- ggml-backend.h | 60 +- ggml-cuda.cu | 907 +++++---- ggml-cuda.h | 26 +- ggml-impl.h | 2 + ggml-metal.m | 55 +- ggml-opencl.cpp | 335 +++- ggml-opencl.h | 16 +- ggml.c | 30 +- ggml.h | 9 +- llama.cpp | 2320 +++++++++------------- llama.h | 18 +- tests/test-backend-ops.cpp | 26 +- 21 files changed, 2533 insertions(+), 2295 deletions(-) diff --git a/common/common.cpp b/common/common.cpp index 062a8b4de..322b9f91e 100644 --- a/common/common.cpp +++ b/common/common.cpp @@ -543,9 +543,8 @@ bool gpt_params_parse_ex(int argc, char ** argv, gpt_params & params) { invalid_param = true; break; } -#ifdef LLAMA_SUPPORTS_GPU_OFFLOAD params.n_gpu_layers = std::stoi(argv[i]); -#else +#ifndef LLAMA_SUPPORTS_GPU_OFFLOAD fprintf(stderr, "warning: not compiled with GPU offload support, --n-gpu-layers option will be ignored\n"); fprintf(stderr, "warning: see main README.md for information on enabling GPU BLAS support\n"); #endif @@ -554,9 +553,8 @@ bool gpt_params_parse_ex(int argc, char ** argv, gpt_params & params) { invalid_param = true; break; } -#ifdef LLAMA_SUPPORTS_GPU_OFFLOAD params.n_gpu_layers_draft = std::stoi(argv[i]); -#else +#ifndef LLAMA_SUPPORTS_GPU_OFFLOAD fprintf(stderr, "warning: not compiled with GPU offload support, --n-gpu-layers-draft option will be ignored\n"); fprintf(stderr, "warning: see main README.md for information on enabling GPU BLAS support\n"); #endif @@ -565,25 +563,44 @@ bool gpt_params_parse_ex(int argc, char ** argv, gpt_params & params) { invalid_param = true; break; } -#ifdef GGML_USE_CUBLAS params.main_gpu = std::stoi(argv[i]); -#else - fprintf(stderr, "warning: llama.cpp was compiled without cuBLAS. It is not possible to set a main GPU.\n"); -#endif +#ifndef GGML_USE_CUBLAS + fprintf(stderr, "warning: llama.cpp was compiled without cuBLAS. Setting the main GPU has no effect.\n"); +#endif // GGML_USE_CUBLAS + } else if (arg == "--split-mode" || arg == "-sm") { + if (++i >= argc) { + invalid_param = true; + break; + } + std::string arg_next = argv[i]; + if (arg_next == "none") { + params.split_mode = LLAMA_SPLIT_NONE; + } else if (arg_next == "layer") { + params.split_mode = LLAMA_SPLIT_LAYER; + } else if (arg_next == "row") { + params.split_mode = LLAMA_SPLIT_ROW; + } else { + invalid_param = true; + break; + } +#ifndef GGML_USE_CUBLAS + fprintf(stderr, "warning: llama.cpp was compiled without cuBLAS. Setting the split mode has no effect.\n"); +#endif // GGML_USE_CUBLAS } else if (arg == "--tensor-split" || arg == "-ts") { if (++i >= argc) { invalid_param = true; break; } -#ifdef GGML_USE_CUBLAS std::string arg_next = argv[i]; // split string by , and / const std::regex regex{R"([,/]+)"}; std::sregex_token_iterator it{arg_next.begin(), arg_next.end(), regex, -1}; std::vector split_arg{it, {}}; - GGML_ASSERT(split_arg.size() <= LLAMA_MAX_DEVICES); - + if (split_arg.size() >= LLAMA_MAX_DEVICES) { + invalid_param = true; + break; + } for (size_t i = 0; i < LLAMA_MAX_DEVICES; ++i) { if (i < split_arg.size()) { params.tensor_split[i] = std::stof(split_arg[i]); @@ -591,14 +608,8 @@ bool gpt_params_parse_ex(int argc, char ** argv, gpt_params & params) { params.tensor_split[i] = 0.0f; } } -#else - fprintf(stderr, "warning: llama.cpp was compiled without cuBLAS. It is not possible to set a tensor split.\n"); -#endif // GGML_USE_CUBLAS - } else if (arg == "--no-mul-mat-q" || arg == "-nommq") { -#ifdef GGML_USE_CUBLAS - params.mul_mat_q = false; -#else - fprintf(stderr, "warning: llama.cpp was compiled without cuBLAS. Disabling mul_mat_q kernels has no effect.\n"); +#ifndef GGML_USE_CUBLAS + fprintf(stderr, "warning: llama.cpp was compiled without cuBLAS. Setting a tensor split has no effect.\n"); #endif // GGML_USE_CUBLAS } else if (arg == "--no-mmap") { params.use_mmap = false; @@ -915,14 +926,15 @@ void gpt_print_usage(int /*argc*/, char ** argv, const gpt_params & params) { printf(" number of layers to store in VRAM\n"); printf(" -ngld N, --n-gpu-layers-draft N\n"); printf(" number of layers to store in VRAM for the draft model\n"); + printf(" -sm SPLIT_MODE, --split-mode SPLIT_MODE\n"); + printf(" how to split the model across multiple GPUs, one of:\n"); + printf(" - none: use one GPU only\n"); + printf(" - layer (default): split layers and KV across GPUs\n"); + printf(" - row: split rows across GPUs\n"); printf(" -ts SPLIT, --tensor-split SPLIT\n"); - printf(" how to split tensors across multiple GPUs, comma-separated list of proportions, e.g. 3,1\n"); - printf(" -mg i, --main-gpu i the GPU to use for scratch and small tensors\n"); -#ifdef GGML_USE_CUBLAS - printf(" -nommq, --no-mul-mat-q\n"); - printf(" use " GGML_CUBLAS_NAME " instead of custom mul_mat_q " GGML_CUDA_NAME " kernels.\n"); - printf(" Not recommended since this is both slower and uses more VRAM.\n"); -#endif // GGML_USE_CUBLAS + printf(" fraction of the model to offload to each GPU, comma-separated list of proportions, e.g. 3,1\n"); + printf(" -mg i, --main-gpu i the GPU to use for the model (with split-mode = none),\n"); + printf(" or for intermediate results and KV (with split-mode = row) (default: %d)\n", params.main_gpu); #endif printf(" -gan N, --grp-attn-n N\n"); printf(" group-attention factor (default: %d)\n", params.grp_attn_n); @@ -1041,6 +1053,7 @@ struct llama_model_params llama_model_params_from_gpt_params(const gpt_params & mparams.n_gpu_layers = params.n_gpu_layers; } mparams.main_gpu = params.main_gpu; + mparams.split_mode = params.split_mode; mparams.tensor_split = params.tensor_split; mparams.use_mmap = params.use_mmap; mparams.use_mlock = params.use_mlock; diff --git a/common/common.h b/common/common.h index 1359e76ab..f29be5b5a 100644 --- a/common/common.h +++ b/common/common.h @@ -59,6 +59,7 @@ struct gpt_params { float p_split = 0.1f; // speculative decoding split probability int32_t n_gpu_layers = -1; // number of layers to store in VRAM (-1 - use default) int32_t n_gpu_layers_draft = -1; // number of layers to store in VRAM for the draft model (-1 - use default) + llama_split_mode split_mode = LLAMA_SPLIT_LAYER; // how to split the model across GPUs int32_t main_gpu = 0; // the GPU that is used for scratch and small tensors float tensor_split[LLAMA_MAX_DEVICES] = {0}; // how split tensors should be distributed across GPUs int32_t n_beams = 0; // if non-zero then use beam search of given width. diff --git a/examples/batched-bench/batched-bench.cpp b/examples/batched-bench/batched-bench.cpp index 57596ed98..7924db267 100644 --- a/examples/batched-bench/batched-bench.cpp +++ b/examples/batched-bench/batched-bench.cpp @@ -88,7 +88,10 @@ int main(int argc, char ** argv) { llama_model_params model_params = llama_model_default_params(); + const std::vector t_split (LLAMA_MAX_DEVICES, 0.0f); + model_params.n_gpu_layers = n_gpu_layers; + model_params.tensor_split = t_split.data(); llama_model * model = llama_load_model_from_file(params.model.c_str(), model_params); diff --git a/examples/llama-bench/llama-bench.cpp b/examples/llama-bench/llama-bench.cpp index 7f7186cde..97325b5bd 100644 --- a/examples/llama-bench/llama-bench.cpp +++ b/examples/llama-bench/llama-bench.cpp @@ -128,6 +128,25 @@ static std::string get_gpu_info() { // command line params enum output_formats {CSV, JSON, MARKDOWN, SQL}; +static const char * output_format_str(output_formats format) { + switch (format) { + case CSV: return "csv"; + case JSON: return "json"; + case MARKDOWN: return "md"; + case SQL: return "sql"; + default: GGML_ASSERT(!"invalid output format"); + } +} + +static const char * split_mode_str(llama_split_mode mode) { + switch (mode) { + case LLAMA_SPLIT_NONE: return "none"; + case LLAMA_SPLIT_LAYER: return "layer"; + case LLAMA_SPLIT_ROW: return "row"; + default: GGML_ASSERT(!"invalid split mode"); + } +} + struct cmd_params { std::vector model; std::vector n_prompt; @@ -137,6 +156,7 @@ struct cmd_params { std::vector type_v; std::vector n_threads; std::vector n_gpu_layers; + std::vector split_mode; std::vector main_gpu; std::vector no_kv_offload; std::vector mul_mat_q; @@ -155,6 +175,7 @@ static const cmd_params cmd_params_defaults = { /* type_v */ {GGML_TYPE_F16}, /* n_threads */ {get_num_physical_cores()}, /* n_gpu_layers */ {99}, + /* split_mode */ {LLAMA_SPLIT_LAYER}, /* main_gpu */ {0}, /* no_kv_offload */ {false}, /* mul_mat_q */ {true}, @@ -169,21 +190,22 @@ static void print_usage(int /* argc */, char ** argv) { printf("\n"); printf("options:\n"); printf(" -h, --help\n"); - printf(" -m, --model (default: %s)\n", join(cmd_params_defaults.model, ",").c_str()); - printf(" -p, --n-prompt (default: %s)\n", join(cmd_params_defaults.n_prompt, ",").c_str()); - printf(" -n, --n-gen (default: %s)\n", join(cmd_params_defaults.n_gen, ",").c_str()); - printf(" -b, --batch-size (default: %s)\n", join(cmd_params_defaults.n_batch, ",").c_str()); - printf(" -ctk , --cache-type-k (default: %s)\n", join(transform_to_str(cmd_params_defaults.type_k, ggml_type_name), ",").c_str()); - printf(" -ctv , --cache-type-v (default: %s)\n", join(transform_to_str(cmd_params_defaults.type_v, ggml_type_name), ",").c_str()); - printf(" -t, --threads (default: %s)\n", join(cmd_params_defaults.n_threads, ",").c_str()); - printf(" -ngl, --n-gpu-layers (default: %s)\n", join(cmd_params_defaults.n_gpu_layers, ",").c_str()); - printf(" -mg, --main-gpu (default: %s)\n", join(cmd_params_defaults.main_gpu, ",").c_str()); - printf(" -nkvo, --no-kv-offload <0|1> (default: %s)\n", join(cmd_params_defaults.no_kv_offload, ",").c_str()); - printf(" -mmq, --mul-mat-q <0|1> (default: %s)\n", join(cmd_params_defaults.mul_mat_q, ",").c_str()); - printf(" -ts, --tensor_split \n"); - printf(" -r, --repetitions (default: %d)\n", cmd_params_defaults.reps); - printf(" -o, --output (default: %s)\n", cmd_params_defaults.output_format == CSV ? "csv" : cmd_params_defaults.output_format == JSON ? "json" : cmd_params_defaults.output_format == MARKDOWN ? "md" : "sql"); - printf(" -v, --verbose (default: %s)\n", cmd_params_defaults.verbose ? "1" : "0"); + printf(" -m, --model (default: %s)\n", join(cmd_params_defaults.model, ",").c_str()); + printf(" -p, --n-prompt (default: %s)\n", join(cmd_params_defaults.n_prompt, ",").c_str()); + printf(" -n, --n-gen (default: %s)\n", join(cmd_params_defaults.n_gen, ",").c_str()); + printf(" -b, --batch-size (default: %s)\n", join(cmd_params_defaults.n_batch, ",").c_str()); + printf(" -ctk , --cache-type-k (default: %s)\n", join(transform_to_str(cmd_params_defaults.type_k, ggml_type_name), ",").c_str()); + printf(" -ctv , --cache-type-v (default: %s)\n", join(transform_to_str(cmd_params_defaults.type_v, ggml_type_name), ",").c_str()); + printf(" -t, --threads (default: %s)\n", join(cmd_params_defaults.n_threads, ",").c_str()); + printf(" -ngl, --n-gpu-layers (default: %s)\n", join(cmd_params_defaults.n_gpu_layers, ",").c_str()); + printf(" -sm, --split-mode (default: %s)\n", join(transform_to_str(cmd_params_defaults.split_mode, split_mode_str), ",").c_str()); + printf(" -mg, --main-gpu (default: %s)\n", join(cmd_params_defaults.main_gpu, ",").c_str()); + printf(" -nkvo, --no-kv-offload <0|1> (default: %s)\n", join(cmd_params_defaults.no_kv_offload, ",").c_str()); + printf(" -mmq, --mul-mat-q <0|1> (default: %s)\n", join(cmd_params_defaults.mul_mat_q, ",").c_str()); + printf(" -ts, --tensor_split (default: 0)\n"); + printf(" -r, --repetitions (default: %d)\n", cmd_params_defaults.reps); + printf(" -o, --output (default: %s)\n", output_format_str(cmd_params_defaults.output_format)); + printf(" -v, --verbose (default: %s)\n", cmd_params_defaults.verbose ? "1" : "0"); printf("\n"); printf("Multiple values can be given for each parameter by separating them with ',' or by specifying the parameter multiple times.\n"); } @@ -306,6 +328,28 @@ static cmd_params parse_cmd_params(int argc, char ** argv) { } auto p = split(argv[i], split_delim); params.n_gpu_layers.insert(params.n_gpu_layers.end(), p.begin(), p.end()); + } else if (arg == "-sm" || arg == "--split-mode") { + if (++i >= argc) { + invalid_param = true; + break; + } + auto p = split(argv[i], split_delim); + std::vector modes; + for (const auto & m : p) { + llama_split_mode mode; + if (m == "none") { + mode = LLAMA_SPLIT_NONE; + } else if (m == "layer") { + mode = LLAMA_SPLIT_LAYER; + } else if (m == "row") { + mode = LLAMA_SPLIT_ROW; + } else { + invalid_param = true; + break; + } + modes.push_back(mode); + } + params.split_mode.insert(params.split_mode.end(), modes.begin(), modes.end()); } else if (arg == "-mg" || arg == "--main-gpu") { if (++i >= argc) { invalid_param = true; @@ -392,6 +436,7 @@ static cmd_params parse_cmd_params(int argc, char ** argv) { if (params.type_k.empty()) { params.type_k = cmd_params_defaults.type_k; } if (params.type_v.empty()) { params.type_v = cmd_params_defaults.type_v; } if (params.n_gpu_layers.empty()) { params.n_gpu_layers = cmd_params_defaults.n_gpu_layers; } + if (params.split_mode.empty()) { params.split_mode = cmd_params_defaults.split_mode; } if (params.main_gpu.empty()) { params.main_gpu = cmd_params_defaults.main_gpu; } if (params.no_kv_offload.empty()){ params.no_kv_offload = cmd_params_defaults.no_kv_offload; } if (params.mul_mat_q.empty()) { params.mul_mat_q = cmd_params_defaults.mul_mat_q; } @@ -410,6 +455,7 @@ struct cmd_params_instance { ggml_type type_v; int n_threads; int n_gpu_layers; + llama_split_mode split_mode; int main_gpu; bool no_kv_offload; bool mul_mat_q; @@ -419,6 +465,7 @@ struct cmd_params_instance { llama_model_params mparams = llama_model_default_params(); mparams.n_gpu_layers = n_gpu_layers; + mparams.split_mode = split_mode; mparams.main_gpu = main_gpu; mparams.tensor_split = tensor_split.data(); @@ -428,6 +475,7 @@ struct cmd_params_instance { bool equal_mparams(const cmd_params_instance & other) const { return model == other.model && n_gpu_layers == other.n_gpu_layers && + split_mode == other.split_mode && main_gpu == other.main_gpu && tensor_split == other.tensor_split; } @@ -446,45 +494,13 @@ struct cmd_params_instance { } }; -static std::vector get_cmd_params_instances_int(const cmd_params & params, int n_gen, int n_prompt) { - std::vector instances; - - for (const auto & m : params.model) - for (const auto & nl : params.n_gpu_layers) - for (const auto & mg : params.main_gpu) - for (const auto & ts : params.tensor_split) - for (const auto & nb : params.n_batch) - for (const auto & tk : params.type_k) - for (const auto & tv : params.type_v) - for (const auto & mmq : params.mul_mat_q) - for (const auto & nkvo : params.no_kv_offload) - for (const auto & nt : params.n_threads) { - cmd_params_instance instance = { - /* .model = */ m, - /* .n_prompt = */ n_prompt, - /* .n_gen = */ n_gen, - /* .n_batch = */ nb, - /* .type_k = */ tk, - /* .type_v = */ tv, - /* .n_threads = */ nt, - /* .n_gpu_layers = */ nl, - /* .main_gpu = */ mg, - /* .no_kv_offload= */ nkvo, - /* .mul_mat_q = */ mmq, - /* .tensor_split = */ ts, - }; - instances.push_back(instance); - } - return instances; -} - static std::vector get_cmd_params_instances(const cmd_params & params) { std::vector instances; -#if 1 // this ordering minimizes the number of times that each model needs to be reloaded for (const auto & m : params.model) for (const auto & nl : params.n_gpu_layers) + for (const auto & sm : params.split_mode) for (const auto & mg : params.main_gpu) for (const auto & ts : params.tensor_split) for (const auto & nb : params.n_batch) @@ -506,6 +522,7 @@ static std::vector get_cmd_params_instances(const cmd_param /* .type_v = */ tv, /* .n_threads = */ nt, /* .n_gpu_layers = */ nl, + /* .split_mode = */ sm, /* .main_gpu = */ mg, /* .no_kv_offload= */ nkvo, /* .mul_mat_q = */ mmq, @@ -527,6 +544,7 @@ static std::vector get_cmd_params_instances(const cmd_param /* .type_v = */ tv, /* .n_threads = */ nt, /* .n_gpu_layers = */ nl, + /* .split_mode = */ sm, /* .main_gpu = */ mg, /* .no_kv_offload= */ nkvo, /* .mul_mat_q = */ mmq, @@ -535,24 +553,6 @@ static std::vector get_cmd_params_instances(const cmd_param instances.push_back(instance); } } -#else - // this ordering separates the prompt and generation tests - for (const auto & n_prompt : params.n_prompt) { - if (n_prompt == 0) { - continue; - } - auto instances_prompt = get_cmd_params_instances_int(params, 0, n_prompt); - instances.insert(instances.end(), instances_prompt.begin(), instances_prompt.end()); - } - - for (const auto & n_gen : params.n_gen) { - if (n_gen == 0) { - continue; - } - auto instances_gen = get_cmd_params_instances_int(params, n_gen, 0); - instances.insert(instances.end(), instances_gen.begin(), instances_gen.end()); - } -#endif return instances; } @@ -576,6 +576,7 @@ struct test { ggml_type type_k; ggml_type type_v; int n_gpu_layers; + llama_split_mode split_mode; int main_gpu; bool no_kv_offload; bool mul_mat_q; @@ -597,6 +598,7 @@ struct test { type_k = inst.type_k; type_v = inst.type_v; n_gpu_layers = inst.n_gpu_layers; + split_mode = inst.split_mode; main_gpu = inst.main_gpu; no_kv_offload = inst.no_kv_offload; mul_mat_q = inst.mul_mat_q; @@ -660,7 +662,8 @@ struct test { "cpu_info", "gpu_info", "model_filename", "model_type", "model_size", "model_n_params", "n_batch", "n_threads", "type_k", "type_v", - "n_gpu_layers", "main_gpu", "no_kv_offload", + "n_gpu_layers", "split_mode", + "main_gpu", "no_kv_offload", "mul_mat_q", "tensor_split", "n_prompt", "n_gen", "test_time", "avg_ns", "stddev_ns", @@ -711,7 +714,8 @@ struct test { cpu_info, gpu_info, model_filename, model_type, std::to_string(model_size), std::to_string(model_n_params), std::to_string(n_batch), std::to_string(n_threads), ggml_type_name(type_k), ggml_type_name(type_v), - std::to_string(n_gpu_layers), std::to_string(main_gpu), std::to_string(no_kv_offload), + std::to_string(n_gpu_layers), split_mode_str(split_mode), + std::to_string(main_gpu), std::to_string(no_kv_offload), std::to_string(mul_mat_q), tensor_split_str, std::to_string(n_prompt), std::to_string(n_gen), test_time, std::to_string(avg_ns()), std::to_string(stdev_ns()), @@ -867,6 +871,9 @@ struct markdown_printer : public printer { if (field == "n_gpu_layers") { return "ngl"; } + if (field == "split_mode") { + return "sm"; + } if (field == "n_threads") { return "threads"; } @@ -907,6 +914,9 @@ struct markdown_printer : public printer { if (params.main_gpu.size() > 1 || params.main_gpu != cmd_params_defaults.main_gpu) { fields.push_back("main_gpu"); } + if (params.split_mode.size() > 1 || params.split_mode != cmd_params_defaults.split_mode) { + fields.push_back("split_mode"); + } if (params.mul_mat_q.size() > 1 || params.mul_mat_q != cmd_params_defaults.mul_mat_q) { fields.push_back("mul_mat_q"); } diff --git a/examples/server/server.cpp b/examples/server/server.cpp index 1d30a15a6..c1ab8f9dc 100644 --- a/examples/server/server.cpp +++ b/examples/server/server.cpp @@ -2005,12 +2005,15 @@ static void server_print_usage(const char *argv0, const gpt_params ¶ms, #ifdef LLAMA_SUPPORTS_GPU_OFFLOAD printf(" -ngl N, --n-gpu-layers N\n"); printf(" number of layers to store in VRAM\n"); + printf(" -sm SPLIT_MODE, --split-mode SPLIT_MODE\n"); + printf(" how to split the model across multiple GPUs, one of:\n"); + printf(" - none: use one GPU only\n"); + printf(" - layer (default): split layers and KV across GPUs\n"); + printf(" - row: split rows across GPUs\n"); printf(" -ts SPLIT --tensor-split SPLIT\n"); - printf(" how to split tensors across multiple GPUs, comma-separated list of proportions, e.g. 3,1\n"); - printf(" -mg i, --main-gpu i the GPU to use for scratch and small tensors\n"); - printf(" -nommq, --no-mul-mat-q\n"); - printf(" use cuBLAS instead of custom mul_mat_q CUDA kernels.\n"); - printf(" Not recommended since this is both slower and uses more VRAM.\n"); + printf(" fraction of the model to offload to each GPU, comma-separated list of proportions, e.g. 3,1\n"); + printf(" -mg i, --main-gpu i the GPU to use for the model (with split-mode = none),\n"); + printf(" or for intermediate results and KV (with split-mode = row)\n"); #endif printf(" -m FNAME, --model FNAME\n"); printf(" model path (default: %s)\n", params.model.c_str()); @@ -2253,6 +2256,33 @@ static void server_params_parse(int argc, char **argv, server_params &sparams, "See main README.md for information on enabling GPU BLAS support", {{"n_gpu_layers", params.n_gpu_layers}}); #endif + } + else if (arg == "--split-mode" || arg == "-sm") + { + if (++i >= argc) { + invalid_param = true; + break; + } + std::string arg_next = argv[i]; + if (arg_next == "none") + { + params.split_mode = LLAMA_SPLIT_NONE; + } + else if (arg_next == "layer") + { + params.split_mode = LLAMA_SPLIT_LAYER; + } + else if (arg_next == "row") + { + params.split_mode = LLAMA_SPLIT_ROW; + } + else { + invalid_param = true; + break; + } +#ifndef GGML_USE_CUBLAS + fprintf(stderr, "warning: llama.cpp was compiled without cuBLAS. Setting the split mode has no effect.\n"); +#endif // GGML_USE_CUBLAS } else if (arg == "--tensor-split" || arg == "-ts") { diff --git a/ggml-alloc.c b/ggml-alloc.c index a27dd54b0..89b85d348 100644 --- a/ggml-alloc.c +++ b/ggml-alloc.c @@ -102,8 +102,6 @@ void ggml_tallocr_alloc(ggml_tallocr_t alloc, struct ggml_tensor * tensor) { } } - AT_PRINTF("block %d\n", best_fit_block); - if (best_fit_block == -1) { // the last block is our last resort struct free_block * block = &alloc->free_blocks[alloc->n_free_blocks - 1]; @@ -117,6 +115,7 @@ void ggml_tallocr_alloc(ggml_tallocr_t alloc, struct ggml_tensor * tensor) { return; } } + struct free_block * block = &alloc->free_blocks[best_fit_block]; void * addr = block->addr; block->addr = (char*)block->addr + size; @@ -129,6 +128,8 @@ void ggml_tallocr_alloc(ggml_tallocr_t alloc, struct ggml_tensor * tensor) { } } + AT_PRINTF("block %d, addr %p\n", best_fit_block, addr); + tensor->data = addr; tensor->buffer = alloc->buffer; if (!alloc->measure) { @@ -229,6 +230,7 @@ void ggml_tallocr_reset(ggml_tallocr_t alloc) { alloc->free_blocks[0].size = SIZE_MAX/2; // restrict maximum size of a measure allocator to half size_t max to avoid overflows } else { alloc->free_blocks[0].size = ggml_backend_buffer_get_size(alloc->buffer) - align_offset; + ggml_backend_buffer_reset(alloc->buffer); } } @@ -263,9 +265,9 @@ ggml_tallocr_t ggml_tallocr_new_measure(size_t alignment) { return alloc; } -ggml_tallocr_t ggml_tallocr_new_measure_from_backend(struct ggml_backend * backend) { +ggml_tallocr_t ggml_tallocr_new_measure_from_buft(struct ggml_backend_buffer_type * buft) { // create a backend buffer to get the correct tensor allocation sizes - ggml_backend_buffer_t buffer = ggml_backend_alloc_buffer(backend, 1); + ggml_backend_buffer_t buffer = ggml_backend_buft_alloc_buffer(buft, 1); // TODO: move alloc initialization to a common ggml_tallocr_new_impl function ggml_tallocr_t alloc = ggml_tallocr_new_from_buffer(buffer); @@ -275,13 +277,22 @@ ggml_tallocr_t ggml_tallocr_new_measure_from_backend(struct ggml_backend * backe return alloc; } -ggml_tallocr_t ggml_tallocr_new_from_backend(struct ggml_backend * backend, size_t size) { - ggml_backend_buffer_t buffer = ggml_backend_alloc_buffer(backend, size); +ggml_tallocr_t ggml_tallocr_new_measure_from_backend(struct ggml_backend * backend) { + return ggml_tallocr_new_measure_from_buft(ggml_backend_get_default_buffer_type(backend)); +} + +ggml_tallocr_t ggml_tallocr_new_from_buft(struct ggml_backend_buffer_type * buft, size_t size) { + // create a backend buffer to get the correct tensor allocation sizes + ggml_backend_buffer_t buffer = ggml_backend_buft_alloc_buffer(buft, size); ggml_tallocr_t alloc = ggml_tallocr_new_from_buffer(buffer); alloc->buffer_owned = true; return alloc; } +ggml_tallocr_t ggml_tallocr_new_from_backend(struct ggml_backend * backend, size_t size) { + return ggml_tallocr_new_from_buft(ggml_backend_get_default_buffer_type(backend), size); +} + ggml_tallocr_t ggml_tallocr_new_from_buffer(struct ggml_backend_buffer * buffer) { ggml_tallocr_t alloc = (ggml_tallocr_t)malloc(sizeof(struct ggml_tallocr)); @@ -779,10 +790,21 @@ ggml_backend_buffer_t ggml_backend_alloc_ctx_tensors_from_buft(struct ggml_conte if (nbytes == 0) { // all the tensors in the context are already allocated +#ifndef NDEBUG + fprintf(stderr, "%s: all tensors in the context are already allocated\n", __func__); +#endif return NULL; } ggml_backend_buffer_t buffer = ggml_backend_buft_alloc_buffer(buft, nbytes); + if (buffer == NULL) { + // failed to allocate buffer +#ifndef NDEBUG + fprintf(stderr, "%s: failed to allocate buffer\n", __func__); +#endif + return NULL; + } + ggml_tallocr_t tallocr = ggml_tallocr_new_from_buffer(buffer); for (struct ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) { diff --git a/ggml-alloc.h b/ggml-alloc.h index 64a412468..4e5997521 100644 --- a/ggml-alloc.h +++ b/ggml-alloc.h @@ -52,8 +52,10 @@ typedef struct ggml_tallocr * ggml_tallocr_t; GGML_API ggml_tallocr_t ggml_tallocr_new(void * data, size_t size, size_t alignment); GGML_API ggml_tallocr_t ggml_tallocr_new_measure(size_t alignment); -GGML_API ggml_tallocr_t ggml_tallocr_new_from_buffer(struct ggml_backend_buffer * buffer); +GGML_API ggml_tallocr_t ggml_tallocr_new_from_buft(struct ggml_backend_buffer_type * buft, size_t size); GGML_API ggml_tallocr_t ggml_tallocr_new_from_backend(struct ggml_backend * backend, size_t size); // allocates an owned buffer +GGML_API ggml_tallocr_t ggml_tallocr_new_from_buffer(struct ggml_backend_buffer * buffer); +GGML_API ggml_tallocr_t ggml_tallocr_new_measure_from_buft(struct ggml_backend_buffer_type * buft); GGML_API ggml_tallocr_t ggml_tallocr_new_measure_from_backend(struct ggml_backend * backend); GGML_API struct ggml_backend_buffer * ggml_tallocr_get_buffer(ggml_tallocr_t talloc); diff --git a/ggml-backend-impl.h b/ggml-backend-impl.h index ca21b4743..1db32901f 100644 --- a/ggml-backend-impl.h +++ b/ggml-backend-impl.h @@ -16,9 +16,10 @@ extern "C" { typedef void * ggml_backend_buffer_type_context_t; struct ggml_backend_buffer_type_i { + const char * (*get_name) (ggml_backend_buffer_type_t buft); ggml_backend_buffer_t (*alloc_buffer) (ggml_backend_buffer_type_t buft, size_t size); size_t (*get_alignment) (ggml_backend_buffer_type_t buft); // tensor alignment - size_t (*get_alloc_size) (ggml_backend_buffer_type_t buft, struct ggml_tensor * tensor); // data size needed to allocate the tensor, including padding + size_t (*get_alloc_size) (ggml_backend_buffer_type_t buft, const struct ggml_tensor * tensor); // data size needed to allocate the tensor, including padding bool (*supports_backend)(ggml_backend_buffer_type_t buft, ggml_backend_t backend); // check if the buffer type is usable by the backend // check if tensor data is in host memory // should be equivalent to supports_backend(buft, ggml_backend_cpu_init()) @@ -34,16 +35,15 @@ extern "C" { typedef void * ggml_backend_buffer_context_t; struct ggml_backend_buffer_i { - void (*free_buffer) (ggml_backend_buffer_t buffer); - //void (*reset) (ggml_backend_buffer_t buffer); // reset any internal state due to tensor initialization, such as tensor extras - void * (*get_base) (ggml_backend_buffer_t buffer); - void (*init_tensor) (ggml_backend_buffer_t buffer, struct ggml_tensor * tensor); - void (*set_tensor) (ggml_backend_buffer_t buffer, struct ggml_tensor * tensor, const void * data, size_t offset, size_t size); - void (*get_tensor) (ggml_backend_buffer_t buffer, const struct ggml_tensor * tensor, void * data, size_t offset, size_t size); - // (optional) copy tensor between different buffer-type, allow for single-copy tranfers - void (*cpy_tensor_from)(ggml_backend_buffer_t buffer, struct ggml_tensor * src, struct ggml_tensor * dst); - void (*cpy_tensor_to) (ggml_backend_buffer_t buffer, struct ggml_tensor * src, struct ggml_tensor * dst); - void (*clear) (ggml_backend_buffer_t buffer, uint8_t value); + const char * (*get_name) (ggml_backend_buffer_t buffer); + void (*free_buffer)(ggml_backend_buffer_t buffer); + void * (*get_base) (ggml_backend_buffer_t buffer); + void (*init_tensor)(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor); + void (*set_tensor) (ggml_backend_buffer_t buffer, struct ggml_tensor * tensor, const void * data, size_t offset, size_t size); + void (*get_tensor) (ggml_backend_buffer_t buffer, const struct ggml_tensor * tensor, void * data, size_t offset, size_t size); + bool (*cpy_tensor) (ggml_backend_buffer_t buffer, const struct ggml_tensor * src, struct ggml_tensor * dst); // dst is in the buffer, src may be in any buffer + void (*clear) (ggml_backend_buffer_t buffer, uint8_t value); + void (*reset) (ggml_backend_buffer_t buffer); // reset any internal state due to tensor initialization, such as tensor extras }; struct ggml_backend_buffer { @@ -51,6 +51,7 @@ extern "C" { ggml_backend_buffer_type_t buft; ggml_backend_buffer_context_t context; size_t size; + enum ggml_backend_buffer_usage usage; }; ggml_backend_buffer_t ggml_backend_buffer_init( @@ -59,6 +60,8 @@ extern "C" { ggml_backend_buffer_context_t context, size_t size); + // do not use directly, use ggml_backend_tensor_copy instead + bool ggml_backend_buffer_copy_tensor(const struct ggml_tensor * src, struct ggml_tensor * dst); // // Backend @@ -74,22 +77,20 @@ extern "C" { // buffer allocation ggml_backend_buffer_type_t (*get_default_buffer_type)(ggml_backend_t backend); - // (optional) asynchroneous tensor data access + // (optional) asynchronous tensor data access void (*set_tensor_async)(ggml_backend_t backend, struct ggml_tensor * tensor, const void * data, size_t offset, size_t size); void (*get_tensor_async)(ggml_backend_t backend, const struct ggml_tensor * tensor, void * data, size_t offset, size_t size); + bool (*cpy_tensor_async)(ggml_backend_t backend, const struct ggml_tensor * src, struct ggml_tensor * dst); - // (optional) asynchroneous tensor copy - void (*cpy_tensor_from_async)(ggml_backend_t backend, struct ggml_tensor * src, struct ggml_tensor * dst); - void (*cpy_tensor_to_async) (ggml_backend_t backend, struct ggml_tensor * src, struct ggml_tensor * dst); - + // (optional) complete all pending operations void (*synchronize)(ggml_backend_t backend); // compute graph with a plan - ggml_backend_graph_plan_t (*graph_plan_create) (ggml_backend_t backend, struct ggml_cgraph * cgraph); + ggml_backend_graph_plan_t (*graph_plan_create) (ggml_backend_t backend, const struct ggml_cgraph * cgraph); void (*graph_plan_free) (ggml_backend_t backend, ggml_backend_graph_plan_t plan); void (*graph_plan_compute)(ggml_backend_t backend, ggml_backend_graph_plan_t plan); - // compute graph without a plan + // compute graph without a plan (async) bool (*graph_compute)(ggml_backend_t backend, struct ggml_cgraph * cgraph); // check if the backend supports an operation @@ -102,7 +103,6 @@ extern "C" { ggml_backend_context_t context; }; - // // Backend registry // diff --git a/ggml-backend.c b/ggml-backend.c index 53e741cb8..4c2d8b0b2 100644 --- a/ggml-backend.c +++ b/ggml-backend.c @@ -15,6 +15,10 @@ // backend buffer type +const char * ggml_backend_buft_name(ggml_backend_buffer_type_t buft) { + return buft->iface.get_name(buft); +} + ggml_backend_buffer_t ggml_backend_buft_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size) { return buft->iface.alloc_buffer(buft, size); } @@ -58,11 +62,16 @@ ggml_backend_buffer_t ggml_backend_buffer_init( /* .buft = */ buft, /* .context = */ context, /* .size = */ size, + /* .usage = */ GGML_BACKEND_BUFFER_USAGE_ANY }; return buffer; } +const char * ggml_backend_buffer_name(ggml_backend_buffer_t buffer) { + return buffer->iface.get_name(buffer); +} + void ggml_backend_buffer_free(ggml_backend_buffer_t buffer) { if (buffer == NULL) { return; @@ -94,11 +103,11 @@ void ggml_backend_buffer_init_tensor(ggml_backend_buffer_t buffer, struct ggml_t } size_t ggml_backend_buffer_get_alignment (ggml_backend_buffer_t buffer) { - return ggml_backend_buft_get_alignment(ggml_backend_buffer_type(buffer)); + return ggml_backend_buft_get_alignment(ggml_backend_buffer_get_type(buffer)); } size_t ggml_backend_buffer_get_alloc_size(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor) { - return ggml_backend_buft_get_alloc_size(ggml_backend_buffer_type(buffer), tensor); + return ggml_backend_buft_get_alloc_size(ggml_backend_buffer_get_type(buffer), tensor); } void ggml_backend_buffer_clear(ggml_backend_buffer_t buffer, uint8_t value) { @@ -106,13 +115,31 @@ void ggml_backend_buffer_clear(ggml_backend_buffer_t buffer, uint8_t value) { } bool ggml_backend_buffer_is_host(ggml_backend_buffer_t buffer) { - return ggml_backend_buft_is_host(ggml_backend_buffer_type(buffer)); + return ggml_backend_buft_is_host(ggml_backend_buffer_get_type(buffer)); } -ggml_backend_buffer_type_t ggml_backend_buffer_type(ggml_backend_buffer_t buffer) { +void ggml_backend_buffer_set_usage(ggml_backend_buffer_t buffer, enum ggml_backend_buffer_usage usage) { + buffer->usage = usage; +} + +ggml_backend_buffer_type_t ggml_backend_buffer_get_type(ggml_backend_buffer_t buffer) { return buffer->buft; } +void ggml_backend_buffer_reset(ggml_backend_buffer_t buffer) { + if (buffer->iface.reset) { + buffer->iface.reset(buffer); + } +} + +bool ggml_backend_buffer_copy_tensor(const struct ggml_tensor * src, struct ggml_tensor * dst) { + ggml_backend_buffer_t dst_buf = dst->view_src ? dst->view_src->buffer : dst->buffer; + if (dst_buf->iface.cpy_tensor) { + return src->buffer->iface.cpy_tensor(dst_buf, src, dst); + } + return false; +} + // backend const char * ggml_backend_name(ggml_backend_t backend) { @@ -146,30 +173,42 @@ void ggml_backend_tensor_set_async(ggml_backend_t backend, struct ggml_tensor * GGML_ASSERT(tensor->data != NULL && "tensor not allocated"); GGML_ASSERT(offset + size <= ggml_nbytes(tensor) && "tensor write out of bounds"); - backend->iface.set_tensor_async(backend, tensor, data, offset, size); + if (backend->iface.set_tensor_async == NULL) { + ggml_backend_tensor_set(tensor, data, offset, size); + } else { + backend->iface.set_tensor_async(backend, tensor, data, offset, size); + } } void ggml_backend_tensor_get_async(ggml_backend_t backend, const struct ggml_tensor * tensor, void * data, size_t offset, size_t size) { GGML_ASSERT(tensor->data != NULL && "tensor not allocated"); GGML_ASSERT(offset + size <= ggml_nbytes(tensor) && "tensor read out of bounds"); - backend->iface.get_tensor_async(backend, tensor, data, offset, size); + if (backend->iface.get_tensor_async == NULL) { + ggml_backend_tensor_get(tensor, data, offset, size); + } else { + backend->iface.get_tensor_async(backend, tensor, data, offset, size); + } } void ggml_backend_tensor_set(struct ggml_tensor * tensor, const void * data, size_t offset, size_t size) { + ggml_backend_buffer_t buf = tensor->view_src ? tensor->view_src->buffer : tensor->buffer; + GGML_ASSERT(tensor->data != NULL && "tensor not allocated"); - GGML_ASSERT(tensor->buffer != NULL && "tensor buffer not set"); + GGML_ASSERT(buf != NULL && "tensor buffer not set"); GGML_ASSERT(offset + size <= ggml_nbytes(tensor) && "tensor write out of bounds"); - tensor->buffer->iface.set_tensor(tensor->buffer, tensor, data, offset, size); + tensor->buffer->iface.set_tensor(buf, tensor, data, offset, size); } void ggml_backend_tensor_get(const struct ggml_tensor * tensor, void * data, size_t offset, size_t size) { + ggml_backend_buffer_t buf = tensor->view_src ? tensor->view_src->buffer : tensor->buffer; + GGML_ASSERT(tensor->data != NULL && "tensor not allocated"); GGML_ASSERT(tensor->buffer != NULL && "tensor buffer not set"); GGML_ASSERT(offset + size <= ggml_nbytes(tensor) && "tensor read out of bounds"); - tensor->buffer->iface.get_tensor(tensor->buffer, tensor, data, offset, size); + tensor->buffer->iface.get_tensor(buf, tensor, data, offset, size); } void ggml_backend_synchronize(ggml_backend_t backend) { @@ -190,19 +229,10 @@ void ggml_backend_graph_plan_free(ggml_backend_t backend, ggml_backend_graph_pla void ggml_backend_graph_plan_compute(ggml_backend_t backend, ggml_backend_graph_plan_t plan) { backend->iface.graph_plan_compute(backend, plan); - - // TODO: optional sync - ggml_backend_synchronize(backend); } bool ggml_backend_graph_compute(ggml_backend_t backend, struct ggml_cgraph * cgraph) { - if (!backend->iface.graph_compute(backend, cgraph)) { - return false; - } - - // TODO: optional sync - ggml_backend_synchronize(backend); - return true; + return backend->iface.graph_compute(backend, cgraph); } bool ggml_backend_supports_op(ggml_backend_t backend, const struct ggml_tensor * op) { @@ -227,28 +257,20 @@ static bool ggml_are_same_layout(const struct ggml_tensor * a, const struct ggml } void ggml_backend_tensor_copy(struct ggml_tensor * src, struct ggml_tensor * dst) { - //printf("src: %s ne: [%d %d %d %d] nb: [%d %d %d %d]\n", src->name, (int)src->ne[0], (int)src->ne[1], (int)src->ne[2], (int)src->ne[3], (int)src->nb[0], (int)src->nb[1], (int)src->nb[2], (int)src->nb[3]); - //printf("dst: %s ne: [%d %d %d %d] nb: [%d %d %d %d]\n", dst->name, (int)dst->ne[0], (int)dst->ne[1], (int)dst->ne[2], (int)dst->ne[3], (int)dst->nb[0], (int)dst->nb[1], (int)dst->nb[2], (int)dst->nb[3]); GGML_ASSERT(ggml_are_same_layout(src, dst) && "cannot copy tensors with different layouts"); - // fprintf(stderr, "cpy tensor %s from %s to %s (%lu bytes)\n", src->name, ggml_backend_name(src->backend), ggml_backend_name(dst->backend), ggml_nbytes(src)); - if (src == dst) { return; } - // TODO: allow backends to support copy to/from same backend - - if (dst->buffer->iface.cpy_tensor_from != NULL) { - dst->buffer->iface.cpy_tensor_from(dst->buffer, src, dst); - } else if (src->buffer->iface.cpy_tensor_to != NULL) { - src->buffer->iface.cpy_tensor_to(src->buffer, src, dst); - } else { - // shouldn't be hit when copying from/to CPU - #ifndef NDEBUG - fprintf(stderr, "ggml_backend_tensor_copy: neither cpy_tensor_from nor cpy_tensor_to " - "are implemented for %s and %s, falling back to get/set\n", src->name, dst->name); - #endif + if (ggml_backend_buffer_is_host(src->buffer)) { + ggml_backend_tensor_set(dst, src->data, 0, ggml_nbytes(src)); + } else if (ggml_backend_buffer_is_host(dst->buffer)) { + ggml_backend_tensor_get(src, dst->data, 0, ggml_nbytes(src)); + } else if (!ggml_backend_buffer_copy_tensor(src, dst)) { +#ifndef NDEBUG + fprintf(stderr, "%s: warning: slow copy from %s to %s\n", __func__, ggml_backend_buffer_name(src->buffer), ggml_backend_buffer_name(dst->buffer)); +#endif size_t nbytes = ggml_nbytes(src); void * data = malloc(nbytes); ggml_backend_tensor_get(src, data, 0, nbytes); @@ -257,6 +279,31 @@ void ggml_backend_tensor_copy(struct ggml_tensor * src, struct ggml_tensor * dst } } +void ggml_backend_tensor_copy_async(ggml_backend_t backend, struct ggml_tensor * src, struct ggml_tensor * dst) { + GGML_ASSERT(ggml_are_same_layout(src, dst) && "cannot copy tensors with different layouts"); + + if (src == dst) { + return; + } + + if (ggml_backend_buft_supports_backend(src->buffer->buft, backend) && ggml_backend_buft_supports_backend(dst->buffer->buft, backend)) { + if (backend->iface.cpy_tensor_async != NULL) { + if (backend->iface.cpy_tensor_async(backend, src, dst)) { + return; + } + } + } + + size_t nbytes = ggml_nbytes(src); + if (ggml_backend_buffer_is_host(src->buffer)) { + ggml_backend_tensor_set_async(backend, dst, src->data, 0, nbytes); + } + else { + ggml_backend_tensor_copy(src, dst); + } +} + + // backend registry #define GGML_MAX_BACKENDS_REG 16 @@ -392,6 +439,12 @@ ggml_backend_buffer_t ggml_backend_reg_alloc_buffer(size_t i, size_t size) { // backend CPU +static const char * ggml_backend_cpu_buffer_name(ggml_backend_buffer_t buffer) { + return "CPU"; + + GGML_UNUSED(buffer); +} + static void * ggml_backend_cpu_buffer_get_base(ggml_backend_buffer_t buffer) { return (void *)buffer->context; } @@ -412,14 +465,12 @@ static void ggml_backend_cpu_buffer_get_tensor(ggml_backend_buffer_t buffer, con GGML_UNUSED(buffer); } -static void ggml_backend_cpu_buffer_cpy_tensor_from(ggml_backend_buffer_t buffer, struct ggml_tensor * src, struct ggml_tensor * dst) { - ggml_backend_tensor_get(src, dst->data, 0, ggml_nbytes(src)); - - GGML_UNUSED(buffer); -} - -static void ggml_backend_cpu_buffer_cpy_tensor_to(ggml_backend_buffer_t buffer, struct ggml_tensor * src, struct ggml_tensor * dst) { - ggml_backend_tensor_set(dst, src->data, 0, ggml_nbytes(src)); +static bool ggml_backend_cpu_buffer_cpy_tensor(ggml_backend_buffer_t buffer, const struct ggml_tensor * src, struct ggml_tensor * dst) { + if (ggml_backend_buffer_is_host(src->buffer)) { + memcpy(dst->data, src->data, ggml_nbytes(src)); + return true; + } + return false; GGML_UNUSED(buffer); } @@ -429,30 +480,38 @@ static void ggml_backend_cpu_buffer_clear(ggml_backend_buffer_t buffer, uint8_t } static struct ggml_backend_buffer_i cpu_backend_buffer_i = { + /* .get_name = */ ggml_backend_cpu_buffer_name, /* .free_buffer = */ ggml_backend_cpu_buffer_free_buffer, /* .get_base = */ ggml_backend_cpu_buffer_get_base, /* .init_tensor = */ NULL, // no initialization required /* .set_tensor = */ ggml_backend_cpu_buffer_set_tensor, /* .get_tensor = */ ggml_backend_cpu_buffer_get_tensor, - /* .cpy_tensor_from = */ ggml_backend_cpu_buffer_cpy_tensor_from, - /* .cpy_tensor_to = */ ggml_backend_cpu_buffer_cpy_tensor_to, + /* .cpy_tensor = */ ggml_backend_cpu_buffer_cpy_tensor, /* .clear = */ ggml_backend_cpu_buffer_clear, + /* .reset = */ NULL, }; // for buffers from ptr, free is not called static struct ggml_backend_buffer_i cpu_backend_buffer_i_from_ptr = { + /* .get_name = */ ggml_backend_cpu_buffer_name, /* .free_buffer = */ NULL, // ptr is not owned by the buffer, so it does not need to be freed /* .get_base = */ ggml_backend_cpu_buffer_get_base, /* .init_tensor = */ NULL, // no initialization required /* .set_tensor = */ ggml_backend_cpu_buffer_set_tensor, /* .get_tensor = */ ggml_backend_cpu_buffer_get_tensor, - /* .cpy_tensor_from = */ ggml_backend_cpu_buffer_cpy_tensor_from, - /* .cpy_tensor_to = */ ggml_backend_cpu_buffer_cpy_tensor_to, + /* .cpy_tensor = */ ggml_backend_cpu_buffer_cpy_tensor, /* .clear = */ ggml_backend_cpu_buffer_clear, + /* .reset = */ NULL, }; static const size_t TENSOR_ALIGNMENT = 64; // should be enough for AVX 512 +static const char * ggml_backend_cpu_buffer_type_get_name(ggml_backend_buffer_type_t buft) { + return "CPU"; + + GGML_UNUSED(buft); +} + static ggml_backend_buffer_t ggml_backend_cpu_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size) { size += TENSOR_ALIGNMENT; // malloc may return an address that is not aligned void * data = malloc(size); // TODO: maybe use GGML_ALIGNED_MALLOC? @@ -483,6 +542,7 @@ static bool ggml_backend_cpu_buffer_type_is_host(ggml_backend_buffer_type_t buft ggml_backend_buffer_type_t ggml_backend_cpu_buffer_type(void) { static struct ggml_backend_buffer_type ggml_backend_cpu_buffer_type = { /* .iface = */ { + /* .get_name = */ ggml_backend_cpu_buffer_type_get_name, /* .alloc_buffer = */ ggml_backend_cpu_buffer_type_alloc_buffer, /* .get_alignment = */ ggml_backend_cpu_buffer_type_get_alignment, /* .get_alloc_size = */ NULL, // defaults to ggml_nbytes @@ -501,6 +561,18 @@ ggml_backend_buffer_type_t ggml_backend_cpu_buffer_type(void) { #include +static const char * ggml_backend_cpu_hbm_buffer_type_get_name(ggml_backend_buffer_type_t buft) { + return "CPU_HBM"; + + GGML_UNUSED(buft); +} + +static const char * ggml_backend_cpu_hbm_buffer_get_name(ggml_backend_buffer_t buf) { + return "CPU_HBM"; + + GGML_UNUSED(buf); +} + static void ggml_backend_cpu_hbm_buffer_free_buffer(ggml_backend_buffer_t buffer) { hbw_free(buffer->context); } @@ -514,17 +586,18 @@ static ggml_backend_buffer_t ggml_backend_cpu_hbm_buffer_type_alloc_buffer(ggml_ return NULL; } - // FIXME: this is a hack to avoid having to implement a new buffer type ggml_backend_buffer_t buffer = ggml_backend_cpu_buffer_from_ptr(ptr, size); buffer->buft = buft; + buffer->iface.get_name = ggml_backend_cpu_hbm_buffer_get_name; buffer->iface.free_buffer = ggml_backend_cpu_hbm_buffer_free_buffer; return buffer; } -ggml_backend_buffer_type_t ggml_backend_cpu_hbm_buffer_type() { +ggml_backend_buffer_type_t ggml_backend_cpu_hbm_buffer_type(void) { static struct ggml_backend_buffer_type ggml_backend_cpu_buffer_type_hbm = { /* .iface = */ { + /* .get_name = */ ggml_backend_cpu_hbm_buffer_type_get_name, /* .alloc_buffer = */ ggml_backend_cpu_hbm_buffer_type_alloc_buffer, /* .get_alignment = */ ggml_backend_cpu_buffer_type_get_alignment, /* .get_alloc_size = */ NULL, // defaults to ggml_nbytes @@ -568,7 +641,7 @@ struct ggml_backend_plan_cpu { struct ggml_cgraph cgraph; }; -static ggml_backend_graph_plan_t ggml_backend_cpu_graph_plan_create(ggml_backend_t backend, struct ggml_cgraph * cgraph) { +static ggml_backend_graph_plan_t ggml_backend_cpu_graph_plan_create(ggml_backend_t backend, const struct ggml_cgraph * cgraph) { struct ggml_backend_cpu_context * cpu_ctx = (struct ggml_backend_cpu_context *)backend->context; struct ggml_backend_plan_cpu * cpu_plan = malloc(sizeof(struct ggml_backend_plan_cpu)); @@ -634,8 +707,7 @@ static struct ggml_backend_i cpu_backend_i = { /* .get_default_buffer_type = */ ggml_backend_cpu_get_default_buffer_type, /* .set_tensor_async = */ NULL, /* .get_tensor_async = */ NULL, - /* .cpy_tensor_from_async = */ NULL, - /* .cpy_tensor_to_async = */ NULL, + /* .cpy_tensor_async = */ NULL, /* .synchronize = */ NULL, /* .graph_plan_create = */ ggml_backend_cpu_graph_plan_create, /* .graph_plan_free = */ ggml_backend_cpu_graph_plan_free, @@ -661,7 +733,7 @@ ggml_backend_t ggml_backend_cpu_init(void) { } bool ggml_backend_is_cpu(ggml_backend_t backend) { - return backend->iface.get_name == ggml_backend_cpu_name; + return backend && backend->iface.get_name == ggml_backend_cpu_name; } void ggml_backend_cpu_set_n_threads(ggml_backend_t backend_cpu, int n_threads) { @@ -685,7 +757,7 @@ static ggml_backend_t ggml_backend_reg_cpu_init(const char * params, void * user // scheduler -#define GGML_MAX_BACKENDS 4 +#define GGML_MAX_BACKENDS 16 #define GGML_MAX_SPLITS 256 #define GGML_MAX_SPLIT_INPUTS 16 @@ -695,21 +767,29 @@ struct ggml_backend_sched_split { int i_end; struct ggml_tensor * inputs[GGML_MAX_SPLIT_INPUTS]; int n_inputs; + // graph view of this split struct ggml_cgraph graph; }; struct ggml_backend_sched { + bool is_reset; // true if the scheduler has been reset since the last graph split + int n_backends; ggml_backend_t backends[GGML_MAX_BACKENDS]; + ggml_backend_buffer_type_t bufts[GGML_MAX_BACKENDS]; ggml_tallocr_t tallocs[GGML_MAX_BACKENDS]; ggml_gallocr_t galloc; + // hash keys of the nodes in the graph struct ggml_hash_set hash_set; - ggml_tallocr_t * node_talloc; // [hash_set.size] - struct ggml_tensor * (* node_copies)[GGML_MAX_BACKENDS]; // [hash_set.size][GGML_MAX_BACKENDS] + // hash values (arrays of [hash_set.size]) + ggml_tallocr_t * node_talloc; // tallocr assigned to each node (indirectly this is the backend) + struct ggml_tensor * (* node_copies)[GGML_MAX_BACKENDS]; // copies of each node for each destination backend + // copy of the graph with modified inputs struct ggml_cgraph * graph; + struct ggml_backend_sched_split splits[GGML_MAX_SPLITS]; int n_splits; @@ -750,14 +830,22 @@ static int sched_allocr_prio(ggml_backend_sched_t sched, ggml_tallocr_t allocr) return INT_MAX; } -static ggml_backend_t get_buffer_backend(ggml_backend_sched_t sched, ggml_backend_buffer_t buffer) { +static ggml_tallocr_t sched_allocr_from_buffer(ggml_backend_sched_t sched, ggml_backend_buffer_t buffer) { if (buffer == NULL) { return NULL; } + + // check if this is already allocate in a allocr buffer (from user manual allocations) + for (int i = 0; i < sched->n_backends; i++) { + if (ggml_tallocr_get_buffer(sched->tallocs[i]) == buffer) { + return sched->tallocs[i]; + } + } + // find highest prio backend that supports the buffer type for (int i = 0; i < sched->n_backends; i++) { if (ggml_backend_buft_supports_backend(buffer->buft, sched->backends[i])) { - return sched->backends[i]; + return sched->tallocs[i]; } } GGML_ASSERT(false && "tensor buffer type not supported by any backend"); @@ -767,7 +855,6 @@ static ggml_backend_t get_allocr_backend(ggml_backend_sched_t sched, ggml_talloc if (allocr == NULL) { return NULL; } - // find highest prio backend that supports the buffer type for (int i = 0; i < sched->n_backends; i++) { if (sched->tallocs[i] == allocr) { return sched->backends[i]; @@ -777,7 +864,7 @@ static ggml_backend_t get_allocr_backend(ggml_backend_sched_t sched, ggml_talloc } #if 0 -static char causes[GGML_DEFAULT_GRAPH_SIZE*8 + GGML_MAX_SPLITS*GGML_MAX_SPLIT_INPUTS][128]; // debug, remove +static char causes[GGML_DEFAULT_GRAPH_SIZE*16 + GGML_MAX_SPLITS*GGML_MAX_SPLIT_INPUTS][128]; // debug only #define SET_CAUSE(node, ...) sprintf(causes[hash_id(node)], __VA_ARGS__) #define GET_CAUSE(node) causes[hash_id(node)] #else @@ -786,45 +873,37 @@ static char causes[GGML_DEFAULT_GRAPH_SIZE*8 + GGML_MAX_SPLITS*GGML_MAX_SPLIT_IN #endif // returns the backend that should be used for the node based on the current locations -static ggml_backend_t sched_backend_from_cur(ggml_backend_sched_t sched, struct ggml_tensor * node) { - // if the dst tensor is already allocated in a buffer, we must assume that it is critical to keep it there - // ie. kv cache updates - // note that this doesn't allow fallback to CPU. need to add output tensors to the splits to copy the data back to the original backend. +static ggml_tallocr_t sched_allocr_from_cur(ggml_backend_sched_t sched, struct ggml_tensor * node) { + // assign pre-allocated nodes to their backend // dst - ggml_backend_t cur_backend = get_buffer_backend(sched, node->buffer); - if (cur_backend != NULL) { + ggml_tallocr_t cur_allocr = sched_allocr_from_buffer(sched, node->buffer); + if (cur_allocr != NULL) { SET_CAUSE(node, "1.dst"); - return cur_backend; + return cur_allocr; } - // view_src - if (node->view_src != NULL && get_buffer_backend(sched, node->view_src->buffer) != NULL) { - SET_CAUSE(node, "1.vsrc"); - return get_buffer_backend(sched, node->view_src->buffer); + if (node->view_src != NULL) { + cur_allocr = sched_allocr_from_buffer(sched, node->view_src->buffer); + if (cur_allocr != NULL) { + SET_CAUSE(node, "1.vsrc"); + return cur_allocr; + } } - - // src - int cur_prio = INT_MAX; - size_t cur_size = 0; - + // assign nodes that use weights to the backend of the weights for (int i = 0; i < GGML_MAX_SRC; i++) { const struct ggml_tensor * src = node->src[i]; if (src == NULL) { break; } - ggml_backend_t src_backend = get_buffer_backend(sched, src->buffer); - if (src_backend != NULL) { - int src_prio = sched_backend_prio(sched, src_backend); - size_t src_size = ggml_nbytes(src); - if (src_prio < cur_prio && src_size >= cur_size) { - cur_prio = src_prio; - cur_size = src_size; - cur_backend = src_backend; - SET_CAUSE(node, "1.src%d", i); - } + if (src->buffer != NULL && src->buffer->usage == GGML_BACKEND_BUFFER_USAGE_WEIGHTS) { + ggml_tallocr_t src_allocr = sched_allocr_from_buffer(sched, src->buffer); + // operations with weights are always run on the same backend as the weights + SET_CAUSE(node, "1.wgt%d", i); + return src_allocr; } } - return cur_backend; + + return NULL; } static char * fmt_size(size_t size) { @@ -857,7 +936,7 @@ static void sched_print_assignments(ggml_backend_sched_t sched, struct ggml_cgra } ggml_tallocr_t node_allocr = node_allocr(node); ggml_backend_t node_backend = node_allocr ? get_allocr_backend(sched, node_allocr) : NULL; // FIXME: - fprintf(stderr, "node #%3d (%10.10s): %20.20s (%4.4s) [%4.4s %8.8s]:", i, ggml_op_name(node->op), node->name, + fprintf(stderr, "node #%3d (%10.10s): %20.20s (%5.5s) [%5.5s %8.8s]:", i, ggml_op_name(node->op), node->name, fmt_size(ggml_nbytes(node)), node_allocr ? ggml_backend_name(node_backend) : "NULL", GET_CAUSE(node)); for (int j = 0; j < GGML_MAX_SRC; j++) { struct ggml_tensor * src = node->src[j]; @@ -866,7 +945,7 @@ static void sched_print_assignments(ggml_backend_sched_t sched, struct ggml_cgra } ggml_tallocr_t src_allocr = node_allocr(src); ggml_backend_t src_backend = src_allocr ? get_allocr_backend(sched, src_allocr) : NULL; - fprintf(stderr, " %20.20s (%4.4s) [%4.4s %8.8s]", src->name, + fprintf(stderr, " %20.20s (%5.5s) [%5.5s %8.8s]", src->name, fmt_size(ggml_nbytes(src)), src_backend ? ggml_backend_name(src_backend) : "NULL", GET_CAUSE(src)); } fprintf(stderr, "\n"); @@ -882,15 +961,17 @@ static struct ggml_tensor * ggml_dup_tensor_layout(struct ggml_context * ctx, co return dup; } + +//#define DEBUG_PASS1 +//#define DEBUG_PASS2 +//#define DEBUG_PASS3 +//#define DEBUG_PASS4 + // assigns backends to ops and splits the graph into subgraphs that can be computed on the same backend -// TODO: merge passes static void sched_split_graph(ggml_backend_sched_t sched, struct ggml_cgraph * graph) { - // reset state - size_t hash_size = sched->hash_set.size; - memset(sched->hash_set.keys, 0, sizeof(sched->hash_set.keys[0]) * hash_size); - memset(sched->node_talloc, 0, sizeof(sched->node_talloc[0]) * hash_size); - memset(sched->node_copies, 0, sizeof(sched->node_copies[0]) * hash_size); + // reset splits sched->n_splits = 0; + sched->is_reset = false; struct ggml_init_params params = { /* .mem_size = */ sizeof(sched->context_buffer), @@ -898,26 +979,22 @@ static void sched_split_graph(ggml_backend_sched_t sched, struct ggml_cgraph * g /* .no_alloc = */ true }; - if (sched->ctx != NULL) { - ggml_free(sched->ctx); - } + ggml_free(sched->ctx); sched->ctx = ggml_init(params); + if (sched->ctx == NULL) { + fprintf(stderr, "%s: failed to initialize context\n", __func__); + GGML_ASSERT(false); + } - // pass 1: assign backends to ops with allocated inputs + // pass 1: assign backends to ops with pre-allocated inputs for (int i = 0; i < graph->n_leafs; i++) { struct ggml_tensor * leaf = graph->leafs[i]; if (node_allocr(leaf) != NULL) { // do not overwrite user assignments continue; } - ggml_backend_t leaf_backend = get_buffer_backend(sched, leaf->buffer); - if (leaf_backend == NULL && leaf->view_src != NULL) { - leaf_backend = get_buffer_backend(sched, leaf->view_src->buffer); - } - if (leaf_backend != NULL) { - node_allocr(leaf) = ggml_backend_sched_get_tallocr(sched, leaf_backend); - } + node_allocr(leaf) = sched_allocr_from_cur(sched, leaf); } for (int i = 0; i < graph->n_nodes; i++) { @@ -926,50 +1003,102 @@ static void sched_split_graph(ggml_backend_sched_t sched, struct ggml_cgraph * g // do not overwrite user assignments continue; } - ggml_backend_t node_backend = sched_backend_from_cur(sched, node); - if (node_backend != NULL) { - node_allocr(node) = ggml_backend_sched_get_tallocr(sched, node_backend); + node_allocr(node) = sched_allocr_from_cur(sched, node); + // src + for (int j = 0; j < GGML_MAX_SRC; j++) { + struct ggml_tensor * src = node->src[j]; + if (src == NULL) { + break; + } + if (node_allocr(src) == NULL) { + node_allocr(src) = sched_allocr_from_cur(sched, src); + } } } - //printf("PASS 1 ASSIGNMENTS\n"); sched_print_assignments(sched, graph); +#ifdef DEBUG_PASS1 + fprintf(stderr, "PASS 1 ASSIGNMENTS\n"); sched_print_assignments(sched, graph); +#endif - // pass 2: assign backends to ops from current assignments - // TODO: - // - reuse sched_backend_from_cur - for (int i = 0; i < graph->n_nodes; i++) { - struct ggml_tensor * node = graph->nodes[i]; - ggml_tallocr_t node_allocr = node_allocr(node); - if (node_allocr == NULL) { - int cur_prio = INT_MAX; - size_t cur_size = 0; - for (int j = 0; j < GGML_MAX_SRC; j++) { - struct ggml_tensor * src = node->src[j]; - if (src == NULL) { - break; - } - ggml_tallocr_t src_allocr = node_allocr(src); - if (src_allocr != NULL) { - int src_prio = sched_allocr_prio(sched, src_allocr); - size_t src_size = ggml_nbytes(src); - if (src_prio < cur_prio && src_size >= cur_size) { - cur_prio = src_prio; - cur_size = src_size; - node_allocr = src_allocr; - SET_CAUSE(node, "2.src%d", j); - } - } + // pass 2: expand current backend assignments + // assign the same backend to adjacent nodes + // expand gpu backends (i.e. non last prio) up and down, ignoring cpu (the lowest priority backend) + // thus, cpu will never be used unless weights are on cpu, or there are no gpu ops between cpu ops + + // pass 2.1 expand gpu up + { + ggml_tallocr_t cur_allocr = NULL; + for (int i = graph->n_nodes - 1; i >= 0; i--) { + struct ggml_tensor * node = graph->nodes[i]; + if (ggml_is_view_op(node->op)) { + continue; } + ggml_tallocr_t node_allocr = node_allocr(node); if (node_allocr != NULL) { - node_allocr(node) = node_allocr; + if (sched_allocr_prio(sched, node_allocr) == sched->n_backends - 1) { + // skip cpu (lowest prio backend) + cur_allocr = NULL; + } else { + cur_allocr = node_allocr; + } + } else { + node_allocr(node) = cur_allocr; + SET_CAUSE(node, "2.1"); } } } - //printf("PASS 2 ASSIGNMENTS\n"); sched_print_assignments(sched, graph); - // pass 3: assign backends to remaining src from dst (should only be leafs) + // pass 2.2 expand gpu down + { + ggml_tallocr_t cur_allocr = NULL; + for (int i = 0; i < graph->n_nodes; i++) { + struct ggml_tensor * node = graph->nodes[i]; + if (ggml_is_view_op(node->op)) { + continue; + } + ggml_tallocr_t node_allocr = node_allocr(node); + if (node_allocr != NULL) { + if (sched_allocr_prio(sched, node_allocr) == sched->n_backends - 1) { + // skip cpu (lowest prio backend) + cur_allocr = NULL; + } else { + cur_allocr = node_allocr; + } + } else { + node_allocr(node) = cur_allocr; + SET_CAUSE(node, "2.2"); + } + } + } + + // pass 2.3 expand rest up + { + ggml_tallocr_t cur_allocr = NULL; + for (int i = graph->n_nodes - 1; i >= 0; i--) { + struct ggml_tensor * node = graph->nodes[i]; + if (ggml_is_view_op(node->op)) { + continue; + } + ggml_tallocr_t node_allocr = node_allocr(node); + if (node_allocr != NULL) { + cur_allocr = node_allocr; + } else { + node_allocr(node) = cur_allocr; + SET_CAUSE(node, "2.3"); + } + } + } +#ifdef DEBUG_PASS2 + fprintf(stderr, "PASS 2 ASSIGNMENTS\n"); sched_print_assignments(sched, graph); +#endif + + // pass 3: assign backends to remaining src from dst and view_src for (int i = 0; i < graph->n_nodes; i++) { struct ggml_tensor * node = graph->nodes[i]; - ggml_tallocr_t node_allocr = node_allocr(node); + ggml_tallocr_t cur_allocr = node_allocr(node); + if (node->view_src != NULL && cur_allocr == NULL) { + cur_allocr = node_allocr(node) = node_allocr(node->view_src); + SET_CAUSE(node, "3.vsrc"); + } for (int j = 0; j < GGML_MAX_SRC; j++) { struct ggml_tensor * src = node->src[j]; if (src == NULL) { @@ -977,81 +1106,105 @@ static void sched_split_graph(ggml_backend_sched_t sched, struct ggml_cgraph * g } ggml_tallocr_t src_allocr = node_allocr(src); if (src_allocr == NULL) { - node_allocr(src) = node_allocr; + if (src->view_src != NULL) { + // views are always on the same backend as the source + node_allocr(src) = node_allocr(src->view_src); + SET_CAUSE(src, "3.vsrc"); + } else { + node_allocr(src) = cur_allocr; + SET_CAUSE(src, "3.cur"); + } } } } - //printf("PASS 3 ASSIGNMENTS\n"); sched_print_assignments(sched, graph); +#ifdef DEBUG_PASS3 + fprintf(stderr, "PASS 3 ASSIGNMENTS\n"); sched_print_assignments(sched, graph); +#endif // pass 4: split graph, find tensors that need to be copied - // TODO: - // - when switching from a less preferred backend to a more preferred backend, check if it is possible to move the switch to an earlier point for the same cost - // find first backend - int cur_split = 0; - for (int i = 0; i < graph->n_nodes; i++) { - struct ggml_tensor * node = graph->nodes[i]; - if (node->view_src == NULL) { - sched->splits[0].tallocr = node_allocr(node); - break; - } - } - sched->splits[0].i_start = 0; - sched->splits[0].n_inputs = 0; - memset(sched->splits[0].inputs, 0, sizeof(sched->splits[0].inputs)); //HACK - ggml_tallocr_t cur_allocr = sched->splits[0].tallocr; - size_t cur_backend_id = sched_allocr_prio(sched, cur_allocr); - for (int i = 0; i < graph->n_nodes; i++) { - struct ggml_tensor * node = graph->nodes[i]; - - if (ggml_is_view_op(node->op)) { - continue; - } - - ggml_tallocr_t node_allocr = node_allocr(node); - - if (node_allocr != cur_allocr) { - sched->splits[cur_split].i_end = i; - cur_split++; - GGML_ASSERT(cur_split < GGML_MAX_SPLITS); - sched->splits[cur_split].tallocr = node_allocr; - sched->splits[cur_split].i_start = i; - sched->splits[cur_split].n_inputs = 0; - memset(sched->splits[cur_split].inputs, 0, sizeof(sched->splits[cur_split].inputs)); //HACK - cur_allocr = node_allocr; - cur_backend_id = sched_allocr_prio(sched, cur_allocr); - } - - // find inputs that are not on the same backend - for (int j = 0; j < GGML_MAX_SRC; j++) { - struct ggml_tensor * src = node->src[j]; - if (src == NULL) { + { + int cur_split = 0; + // find the backend of the first split, skipping view ops + for (int i = 0; i < graph->n_nodes; i++) { + struct ggml_tensor * node = graph->nodes[i]; + if (!ggml_is_view_op(node->op)) { + sched->splits[0].tallocr = node_allocr(node); break; } - ggml_tallocr_t src_allocr = node_allocr(src); - if (src_allocr != node_allocr) { - int n_inputs = sched->splits[cur_split].n_inputs++; - GGML_ASSERT(n_inputs < GGML_MAX_SPLIT_INPUTS); - sched->splits[cur_split].inputs[n_inputs] = (struct ggml_tensor *)src; + } + sched->splits[0].i_start = 0; + sched->splits[0].n_inputs = 0; + memset(sched->splits[0].inputs, 0, sizeof(sched->splits[0].inputs)); //HACK + ggml_tallocr_t cur_allocr = sched->splits[0].tallocr; + size_t cur_backend_id = sched_allocr_prio(sched, cur_allocr); + for (int i = 0; i < graph->n_nodes; i++) { + struct ggml_tensor * node = graph->nodes[i]; - // create copies - size_t id = hash_id(src); - if (sched->node_copies[id][cur_backend_id] == NULL) { - struct ggml_tensor * tensor_copy = ggml_dup_tensor_layout(sched->ctx, src); - sched->node_copies[id][cur_backend_id] = tensor_copy; - node_allocr(tensor_copy) = cur_allocr; - ggml_backend_t backend = get_allocr_backend(sched, cur_allocr); - ggml_format_name(tensor_copy, "%s#%s", ggml_backend_name(backend), src->name); + if (ggml_is_view_op(node->op)) { + continue; + } + + ggml_tallocr_t node_allocr = node_allocr(node); + + if (node_allocr != cur_allocr) { + sched->splits[cur_split].i_end = i; + cur_split++; + GGML_ASSERT(cur_split < GGML_MAX_SPLITS); + sched->splits[cur_split].tallocr = node_allocr; + sched->splits[cur_split].i_start = i; + sched->splits[cur_split].n_inputs = 0; + cur_allocr = node_allocr; + cur_backend_id = sched_allocr_prio(sched, cur_allocr); + } + + // find inputs that are not on the same backend + for (int j = 0; j < GGML_MAX_SRC; j++) { + struct ggml_tensor * src = node->src[j]; + if (src == NULL) { + break; + } + ggml_tallocr_t src_allocr = node_allocr(src); + GGML_ASSERT(src_allocr != NULL); // all inputs should be assigned by now + if (src_allocr != node_allocr) { + // check if the input is already in the split + bool found = false; + for (int k = 0; k < sched->splits[cur_split].n_inputs; k++) { + if (sched->splits[cur_split].inputs[k] == src) { + found = true; + break; + } + } + + if (!found) { + int n_inputs = sched->splits[cur_split].n_inputs++; + //printf("split %d input %d: %s (%s)\n", cur_split, n_inputs, src->name, ggml_backend_name(get_allocr_backend(sched, src_allocr))); + GGML_ASSERT(n_inputs < GGML_MAX_SPLIT_INPUTS); + sched->splits[cur_split].inputs[n_inputs] = src; + } + + // create a copy of the input in the split's backend + size_t id = hash_id(src); + if (sched->node_copies[id][cur_backend_id] == NULL) { + ggml_backend_t backend = get_allocr_backend(sched, cur_allocr); + struct ggml_tensor * tensor_copy = ggml_dup_tensor_layout(sched->ctx, src); + ggml_format_name(tensor_copy, "%s#%s", ggml_backend_name(backend), src->name); + + sched->node_copies[id][cur_backend_id] = tensor_copy; + node_allocr(tensor_copy) = cur_allocr; + SET_CAUSE(tensor_copy, "4.cpy"); + } + node->src[j] = sched->node_copies[id][cur_backend_id]; } - node->src[j] = sched->node_copies[id][cur_backend_id]; } } + sched->splits[cur_split].i_end = graph->n_nodes; + sched->n_splits = cur_split + 1; } - sched->splits[cur_split].i_end = graph->n_nodes; - sched->n_splits = cur_split + 1; +#ifdef DEBUG_PASS4 + fprintf(stderr, "PASS 4 ASSIGNMENTS\n"); sched_print_assignments(sched, graph); +#endif - //fprintf(stderr, "PASS 4 ASSIGNMENTS\n"); sched_print_assignments(sched, graph); fflush(stdout); - -#if 1 +#ifndef NDEBUG // sanity check: all sources should have the same backend as the node for (int i = 0; i < graph->n_nodes; i++) { struct ggml_tensor * node = graph->nodes[i]; @@ -1059,6 +1212,11 @@ static void sched_split_graph(ggml_backend_sched_t sched, struct ggml_cgraph * g if (node_allocr == NULL) { fprintf(stderr, "!!!!!!! %s has no backend\n", node->name); } + if (node->view_src != NULL && node_allocr != node_allocr(node->view_src)) { + fprintf(stderr, "!!!!!!! %s has backend %s, view_src %s has backend %s\n", + node->name, node_allocr ? ggml_backend_name(get_allocr_backend(sched, node_allocr)) : "NULL", + node->view_src->name, node_allocr(node->view_src) ? ggml_backend_name(get_allocr_backend(sched, node_allocr(node->view_src))) : "NULL"); + } for (int j = 0; j < GGML_MAX_SRC; j++) { struct ggml_tensor * src = node->src[j]; if (src == NULL) { @@ -1070,8 +1228,14 @@ static void sched_split_graph(ggml_backend_sched_t sched, struct ggml_cgraph * g node->name, node_allocr ? ggml_backend_name(get_allocr_backend(sched, node_allocr)) : "NULL", j, src->name, src_allocr ? ggml_backend_name(get_allocr_backend(sched, src_allocr)) : "NULL"); } + if (src->view_src != NULL && src_allocr != node_allocr(src->view_src)) { + fprintf(stderr, "!!!!!!! [src] %s has backend %s, view_src %s has backend %s\n", + src->name, src_allocr ? ggml_backend_name(get_allocr_backend(sched, src_allocr)) : "NULL", + src->view_src->name, node_allocr(src->view_src) ? ggml_backend_name(get_allocr_backend(sched, node_allocr(src->view_src))) : "NULL"); + } } } + fflush(stderr); #endif // create copies of the graph for each split @@ -1085,6 +1249,8 @@ static void sched_split_graph(ggml_backend_sched_t sched, struct ggml_cgraph * g for (int j = 0; j < split->n_inputs; j++) { struct ggml_tensor * input = split->inputs[j]; struct ggml_tensor * input_cpy = sched->node_copies[hash_id(input)][sched_allocr_prio(sched, split->tallocr)]; + // add a dependency to the input source so that it is not freed before the copy is done + GGML_ASSERT(input_cpy->src[0] == NULL || input_cpy->src[0] == input); input_cpy->src[0] = input; graph_copy->nodes[graph_copy->n_nodes++] = input_cpy; } @@ -1119,24 +1285,16 @@ static void sched_compute_splits(ggml_backend_sched_t sched) { uint64_t copy_start_us = ggml_time_us(); for (int j = 0; j < split->n_inputs; j++) { struct ggml_tensor * input = split->inputs[j]; - struct ggml_tensor * input_cpy = sched->node_copies[hash_id(input)][sched_backend_prio(sched, split_backend)]; - if (input->buffer == NULL) { - if (input->view_src == NULL) { - fprintf(stderr, "input %s has no buffer and no view_src\n", input->name); - exit(1); - } - // FIXME: may need to use the sched buffer instead - ggml_backend_view_init(input->view_src->buffer, input); - } - if (input_cpy->buffer == NULL) { - fprintf(stderr, "input_cpy %s has no buffer\n", input_cpy->name); - exit(1); - } - //GGML_ASSERT(input->buffer->backend != input_cpy->buffer->backend); - //GGML_ASSERT(input_cpy->buffer->backend == split_backend); - ggml_backend_tensor_copy(input, input_cpy); + struct ggml_tensor * input_cpy = sched->node_copies[hash_id(input)][split_backend_id]; + + GGML_ASSERT(input->buffer != NULL); + GGML_ASSERT(input_cpy->buffer != NULL); + + // TODO: avoid this copy if it was already copied in a previous split, and the input didn't change + // this is important to avoid copying constants such as KQ_mask and inp_pos multiple times + ggml_backend_tensor_copy_async(split_backend, input, input_cpy); } - // ggml_backend_synchronize(split_backend); + //ggml_backend_synchronize(split_backend); // necessary to measure copy time int64_t copy_end_us = ggml_time_us(); copy_us[split_backend_id] += copy_end_us - copy_start_us; @@ -1148,7 +1306,7 @@ static void sched_compute_splits(ggml_backend_sched_t sched) { uint64_t compute_start_us = ggml_time_us(); ggml_backend_graph_compute(split_backend, &split->graph); - // ggml_backend_synchronize(split_backend); + //ggml_backend_synchronize(split_backend); // necessary to measure compute time uint64_t compute_end_us = ggml_time_us(); compute_us[split_backend_id] += compute_end_us - compute_start_us; } @@ -1168,26 +1326,41 @@ static void sched_reset(ggml_backend_sched_t sched) { for (int i = 0; i < sched->n_backends; i++) { ggml_tallocr_reset(sched->tallocs[i]); } + // reset state for the next run + size_t hash_size = sched->hash_set.size; + memset(sched->hash_set.keys, 0, sizeof(sched->hash_set.keys[0]) * hash_size); + memset(sched->node_talloc, 0, sizeof(sched->node_talloc[0]) * hash_size); + memset(sched->node_copies, 0, sizeof(sched->node_copies[0]) * hash_size); + + sched->is_reset = true; } -ggml_backend_sched_t ggml_backend_sched_new(ggml_backend_t * backends, int n_backends) { +ggml_backend_sched_t ggml_backend_sched_new(ggml_backend_t * backends, ggml_backend_buffer_type_t * bufts, int n_backends, size_t graph_size) { + GGML_ASSERT(n_backends > 0); GGML_ASSERT(n_backends <= GGML_MAX_BACKENDS); - struct ggml_backend_sched * sched = malloc(sizeof(struct ggml_backend_sched)); - memset(sched, 0, sizeof(struct ggml_backend_sched)); + struct ggml_backend_sched * sched = calloc(sizeof(struct ggml_backend_sched), 1); + + // initialize hash table + sched->hash_set = ggml_hash_set_new(graph_size + GGML_MAX_SPLITS*GGML_MAX_SPLIT_INPUTS); + sched->node_talloc = calloc(sizeof(sched->node_talloc[0]) * sched->hash_set.size, 1); + sched->node_copies = calloc(sizeof(sched->node_copies[0]) * sched->hash_set.size, 1); sched->n_backends = n_backends; for (int i = 0; i < n_backends; i++) { sched->backends[i] = backends[i]; + sched->bufts[i] = bufts ? bufts[i] : ggml_backend_get_default_buffer_type(backends[i]); } sched->galloc = ggml_gallocr_new(); // init measure allocs for each backend for (int i = 0; i < n_backends; i++) { - sched->tallocs[i] = ggml_tallocr_new_measure_from_backend(backends[i]); + sched->tallocs[i] = ggml_tallocr_new_measure_from_buft(sched->bufts[i]); } + sched_reset(sched); + return sched; } @@ -1199,6 +1372,7 @@ void ggml_backend_sched_free(ggml_backend_sched_t sched) { ggml_tallocr_free(sched->tallocs[i]); } ggml_gallocr_free(sched->galloc); + ggml_free(sched->ctx); free(sched->hash_set.keys); free(sched->node_talloc); free(sched->node_copies); @@ -1206,12 +1380,7 @@ void ggml_backend_sched_free(ggml_backend_sched_t sched) { } void ggml_backend_sched_init_measure(ggml_backend_sched_t sched, struct ggml_cgraph * measure_graph) { - // initialize hash tables - size_t hash_size = measure_graph->visited_hash_table.size + GGML_MAX_SPLITS*GGML_MAX_SPLIT_INPUTS; - sched->hash_set.size = hash_size; - sched->hash_set.keys = malloc(sizeof(sched->hash_set.keys[0]) * hash_size); - sched->node_talloc = malloc(sizeof(sched->node_talloc[0]) * hash_size); - sched->node_copies = malloc(sizeof(sched->node_copies[0]) * hash_size); + GGML_ASSERT(ggml_tallocr_is_measure(sched->tallocs[0])); // can only be initialized once sched_split_graph(sched, measure_graph); sched_alloc_splits(sched); @@ -1220,28 +1389,41 @@ void ggml_backend_sched_init_measure(ggml_backend_sched_t sched, struct ggml_cgr for (int i = 0; i < sched->n_backends; i++) { size_t size = ggml_tallocr_max_size(sched->tallocs[i]); ggml_tallocr_free(sched->tallocs[i]); - sched->tallocs[i] = ggml_tallocr_new_from_backend(sched->backends[i], size); + sched->tallocs[i] = ggml_tallocr_new_from_buft(sched->bufts[i], size); } sched_reset(sched); } void ggml_backend_sched_graph_compute(ggml_backend_sched_t sched, struct ggml_cgraph * graph) { - GGML_ASSERT(sched->hash_set.size >= graph->visited_hash_table.size + GGML_MAX_SPLITS*GGML_MAX_SPLIT_INPUTS); + GGML_ASSERT((int)sched->hash_set.size >= graph->n_nodes + GGML_MAX_SPLITS*GGML_MAX_SPLIT_INPUTS); + + if (!sched->is_reset) { + sched_reset(sched); + } sched_split_graph(sched, graph); sched_alloc_splits(sched); sched_compute_splits(sched); +} + +void ggml_backend_sched_reset(ggml_backend_sched_t sched) { sched_reset(sched); } +int ggml_backend_sched_get_n_splits(ggml_backend_sched_t sched) { + return sched->n_splits; +} + ggml_tallocr_t ggml_backend_sched_get_tallocr(ggml_backend_sched_t sched, ggml_backend_t backend) { int backend_index = sched_backend_prio(sched, backend); + GGML_ASSERT(backend_index >= 0 && backend_index < sched->n_backends); return sched->tallocs[backend_index]; } ggml_backend_buffer_t ggml_backend_sched_get_buffer(ggml_backend_sched_t sched, ggml_backend_t backend) { int backend_index = sched_backend_prio(sched, backend); + GGML_ASSERT(backend_index >= 0 && backend_index < sched->n_backends); return ggml_tallocr_get_buffer(sched->tallocs[backend_index]); } @@ -1251,10 +1433,19 @@ void ggml_backend_sched_set_node_backend(ggml_backend_sched_t sched, struct ggml node_allocr(node) = sched->tallocs[backend_index]; } +ggml_backend_t ggml_backend_sched_get_node_backend(ggml_backend_sched_t sched, struct ggml_tensor * node) { + ggml_tallocr_t allocr = node_allocr(node); + if (allocr == NULL) { + return NULL; + } + return get_allocr_backend(sched, allocr); +} + // utils + void ggml_backend_view_init(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor) { GGML_ASSERT(tensor->buffer == NULL); - //GGML_ASSERT(tensor->data == NULL); // views of pre-allocted tensors may have the data set, but still need to be initialized + //GGML_ASSERT(tensor->data == NULL); // views of pre-allocated tensors may have the data set in ggml_new_tensor, but still need to be initialized by the backend GGML_ASSERT(tensor->view_src != NULL); GGML_ASSERT(tensor->view_src->buffer != NULL); GGML_ASSERT(tensor->view_src->data != NULL); @@ -1320,6 +1511,7 @@ static void graph_init_tensor(struct ggml_hash_set hash_set, struct ggml_tensor struct ggml_tensor * dst = node_copies[id]; if (dst->view_src != NULL) { + graph_init_tensor(hash_set, node_copies, node_init, src->view_src); ggml_backend_view_init(dst->view_src->buffer, dst); } else { @@ -1353,6 +1545,21 @@ struct ggml_backend_graph_copy ggml_backend_graph_copy(ggml_backend_t backend, s struct ggml_context * ctx_allocated = ggml_init(params); struct ggml_context * ctx_unallocated = ggml_init(params); + if (ctx_allocated == NULL || ctx_unallocated == NULL) { + fprintf(stderr, "failed to allocate context for graph copy\n"); + free(hash_set.keys); + free(node_copies); + free(node_init); + ggml_free(ctx_allocated); + ggml_free(ctx_unallocated); + return (struct ggml_backend_graph_copy) { + /* .buffer = */ NULL, + /* .ctx_allocated = */ NULL, + /* .ctx_unallocated = */ NULL, + /* .graph = */ NULL, + }; + } + // dup nodes for (int i = 0; i < graph->n_nodes; i++) { struct ggml_tensor * node = graph->nodes[i]; @@ -1361,6 +1568,20 @@ struct ggml_backend_graph_copy ggml_backend_graph_copy(ggml_backend_t backend, s // allocate nodes ggml_backend_buffer_t buffer = ggml_backend_alloc_ctx_tensors(ctx_allocated, backend); + if (buffer == NULL) { + fprintf(stderr, "failed to allocate buffer for graph copy\n"); + free(hash_set.keys); + free(node_copies); + free(node_init); + ggml_free(ctx_allocated); + ggml_free(ctx_unallocated); + return (struct ggml_backend_graph_copy) { + /* .buffer = */ NULL, + /* .ctx_allocated = */ NULL, + /* .ctx_unallocated = */ NULL, + /* .graph = */ NULL, + }; + } //printf("copy buffer size: %zu MB\n", ggml_backend_buffer_get_size(buffer) / 1024 / 1024); @@ -1397,8 +1618,12 @@ void ggml_backend_graph_copy_free(struct ggml_backend_graph_copy copy) { ggml_free(copy.ctx_unallocated); } -void ggml_backend_compare_graph_backend(ggml_backend_t backend1, ggml_backend_t backend2, struct ggml_cgraph * graph, ggml_backend_eval_callback callback, void * user_data) { +bool ggml_backend_compare_graph_backend(ggml_backend_t backend1, ggml_backend_t backend2, struct ggml_cgraph * graph, ggml_backend_eval_callback callback, void * user_data) { struct ggml_backend_graph_copy copy = ggml_backend_graph_copy(backend2, graph); + if (copy.buffer == NULL) { + return false; + } + struct ggml_cgraph * g1 = graph; struct ggml_cgraph * g2 = copy.graph; @@ -1428,4 +1653,6 @@ void ggml_backend_compare_graph_backend(ggml_backend_t backend1, ggml_backend_t } ggml_backend_graph_copy_free(copy); + + return true; } diff --git a/ggml-backend.h b/ggml-backend.h index 85ff67b0e..4eb244af1 100644 --- a/ggml-backend.h +++ b/ggml-backend.h @@ -17,22 +17,31 @@ extern "C" { // // buffer type - GGML_API ggml_backend_buffer_t ggml_backend_buft_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size); - GGML_API size_t ggml_backend_buft_get_alignment (ggml_backend_buffer_type_t buft); - GGML_API size_t ggml_backend_buft_get_alloc_size(ggml_backend_buffer_type_t buft, struct ggml_tensor * tensor); - GGML_API bool ggml_backend_buft_supports_backend(ggml_backend_buffer_type_t buft, ggml_backend_t backend); - GGML_API bool ggml_backend_buft_is_host (ggml_backend_buffer_type_t buft); + GGML_API const char * ggml_backend_buft_name (ggml_backend_buffer_type_t buft); + GGML_API ggml_backend_buffer_t ggml_backend_buft_alloc_buffer (ggml_backend_buffer_type_t buft, size_t size); + GGML_API size_t ggml_backend_buft_get_alignment (ggml_backend_buffer_type_t buft); + GGML_API size_t ggml_backend_buft_get_alloc_size (ggml_backend_buffer_type_t buft, struct ggml_tensor * tensor); + GGML_API bool ggml_backend_buft_supports_backend(ggml_backend_buffer_type_t buft, ggml_backend_t backend); + GGML_API bool ggml_backend_buft_is_host (ggml_backend_buffer_type_t buft); // buffer - GGML_API void ggml_backend_buffer_free (ggml_backend_buffer_t buffer); - GGML_API void * ggml_backend_buffer_get_base (ggml_backend_buffer_t buffer); - GGML_API size_t ggml_backend_buffer_get_size (ggml_backend_buffer_t buffer); - GGML_API void ggml_backend_buffer_init_tensor (ggml_backend_buffer_t buffer, struct ggml_tensor * tensor); - GGML_API size_t ggml_backend_buffer_get_alignment (ggml_backend_buffer_t buffer); - GGML_API size_t ggml_backend_buffer_get_alloc_size(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor); - GGML_API void ggml_backend_buffer_clear (ggml_backend_buffer_t buffer, uint8_t value); - GGML_API bool ggml_backend_buffer_is_host (ggml_backend_buffer_t buffer); - GGML_API ggml_backend_buffer_type_t ggml_backend_buffer_type(ggml_backend_buffer_t buffer); + enum ggml_backend_buffer_usage { + GGML_BACKEND_BUFFER_USAGE_ANY = 0, + GGML_BACKEND_BUFFER_USAGE_WEIGHTS = 1, + }; + + GGML_API const char * ggml_backend_buffer_name (ggml_backend_buffer_t buffer); + GGML_API void ggml_backend_buffer_free (ggml_backend_buffer_t buffer); + GGML_API void * ggml_backend_buffer_get_base (ggml_backend_buffer_t buffer); + GGML_API size_t ggml_backend_buffer_get_size (ggml_backend_buffer_t buffer); + GGML_API void ggml_backend_buffer_init_tensor (ggml_backend_buffer_t buffer, struct ggml_tensor * tensor); + GGML_API size_t ggml_backend_buffer_get_alignment (ggml_backend_buffer_t buffer); + GGML_API size_t ggml_backend_buffer_get_alloc_size(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor); + GGML_API void ggml_backend_buffer_clear (ggml_backend_buffer_t buffer, uint8_t value); + GGML_API bool ggml_backend_buffer_is_host (ggml_backend_buffer_t buffer); + GGML_API void ggml_backend_buffer_set_usage (ggml_backend_buffer_t buffer, enum ggml_backend_buffer_usage usage); + GGML_API ggml_backend_buffer_type_t ggml_backend_buffer_get_type (ggml_backend_buffer_t buffer); + GGML_API void ggml_backend_buffer_reset (ggml_backend_buffer_t buffer); // // Backend @@ -140,23 +149,24 @@ extern "C" { typedef struct ggml_backend_sched * ggml_backend_sched_t; // Initialize a backend scheduler - GGML_API ggml_backend_sched_t ggml_backend_sched_new(ggml_backend_t * backends, int n_backends); - - GGML_API void ggml_backend_sched_free(ggml_backend_sched_t sched); - + GGML_API ggml_backend_sched_t ggml_backend_sched_new(ggml_backend_t * backends, ggml_backend_buffer_type_t * bufts, int n_backends, size_t graph_size); + GGML_API void ggml_backend_sched_free(ggml_backend_sched_t sched); // Initialize backend buffers from a measure graph - GGML_API void ggml_backend_sched_init_measure(ggml_backend_sched_t sched, struct ggml_cgraph * measure_graph); + GGML_API void ggml_backend_sched_init_measure(ggml_backend_sched_t sched, struct ggml_cgraph * measure_graph); + // Get the number of splits of the last graph + GGML_API int ggml_backend_sched_get_n_splits(ggml_backend_sched_t sched); GGML_API ggml_tallocr_t ggml_backend_sched_get_tallocr(ggml_backend_sched_t sched, ggml_backend_t backend); GGML_API ggml_backend_buffer_t ggml_backend_sched_get_buffer (ggml_backend_sched_t sched, ggml_backend_t backend); - GGML_API void ggml_backend_sched_set_node_backend(ggml_backend_sched_t sched, struct ggml_tensor * node, ggml_backend_t backend); + GGML_API void ggml_backend_sched_set_node_backend(ggml_backend_sched_t sched, struct ggml_tensor * node, ggml_backend_t backend); + GGML_API ggml_backend_t ggml_backend_sched_get_node_backend(ggml_backend_sched_t sched, struct ggml_tensor * node); - // Allocate a graph on the backend scheduler - GGML_API void ggml_backend_sched_graph_compute( - ggml_backend_sched_t sched, - struct ggml_cgraph * graph); + // Allocate and compute graph on the backend scheduler + GGML_API void ggml_backend_sched_graph_compute(ggml_backend_sched_t sched, struct ggml_cgraph * graph); + // Reset all assignments and allocators - must be called before using the sched allocators to allocate inputs + GGML_API void ggml_backend_sched_reset(ggml_backend_sched_t sched); // // Utils @@ -176,7 +186,7 @@ extern "C" { typedef bool (*ggml_backend_eval_callback)(int node_index, struct ggml_tensor * t1, struct ggml_tensor * t2, void * user_data); // Compare the output of two backends - GGML_API void ggml_backend_compare_graph_backend(ggml_backend_t backend1, ggml_backend_t backend2, struct ggml_cgraph * graph, ggml_backend_eval_callback callback, void * user_data); + GGML_API bool ggml_backend_compare_graph_backend(ggml_backend_t backend1, ggml_backend_t backend2, struct ggml_cgraph * graph, ggml_backend_eval_callback callback, void * user_data); // Tensor initialization GGML_API void ggml_backend_tensor_alloc(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor, void * addr); diff --git a/ggml-cuda.cu b/ggml-cuda.cu index a345b0c4a..2db50437c 100644 --- a/ggml-cuda.cu +++ b/ggml-cuda.cu @@ -8,8 +8,13 @@ #include #include #include +#include #include - +#include +#include +#include "ggml-cuda.h" +#include "ggml.h" +#include "ggml-backend-impl.h" #if defined(GGML_USE_HIPBLAS) #include @@ -77,6 +82,7 @@ #define cudaMemcpyKind hipMemcpyKind #define cudaMemset hipMemset #define cudaMemsetAsync hipMemsetAsync +#define cudaMemGetInfo hipMemGetInfo #define cudaOccupancyMaxPotentialBlockSize hipOccupancyMaxPotentialBlockSize #define cudaSetDevice hipSetDevice #define cudaStreamCreateWithFlags hipStreamCreateWithFlags @@ -112,10 +118,6 @@ #endif // defined(GGML_USE_HIPBLAS) -#include "ggml-cuda.h" -#include "ggml.h" -#include "ggml-backend-impl.h" - #define CUDART_HMAX 11070 // CUDA 11.7, min. ver. for which __hmax and __hmax2 are known to work (may be higher than needed) #define CC_PASCAL 600 @@ -564,7 +566,7 @@ static void ggml_cuda_set_device(const int device) { static int g_device_count = -1; static int g_main_device = 0; -static float g_tensor_split[GGML_CUDA_MAX_DEVICES] = {0}; +static std::array g_default_tensor_split = {}; struct cuda_device_capabilities { int cc; // compute capability @@ -575,10 +577,6 @@ struct cuda_device_capabilities { static cuda_device_capabilities g_device_caps[GGML_CUDA_MAX_DEVICES] = { {0, 0, false, 0} }; -static void * g_scratch_buffer = nullptr; -static size_t g_scratch_size = 0; // disabled by default -static size_t g_scratch_offset = 0; - static cublasHandle_t g_cublas_handles[GGML_CUDA_MAX_DEVICES] = {nullptr}; [[noreturn]] @@ -7548,8 +7546,9 @@ void ggml_init_cublas() { CUDA_CHECK(cudaGetDeviceProperties(&prop, id)); fprintf(stderr, " Device %d: %s, compute capability %d.%d, VMM: %s\n", id, prop.name, prop.major, prop.minor, device_vmm ? "yes" : "no"); - g_tensor_split[id] = total_vram; + g_default_tensor_split[id] = total_vram; total_vram += prop.totalGlobalMem; + #if defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__) g_device_caps[id].cc = 100*prop.major + 10*prop.minor + CC_OFFSET_AMD; #else @@ -7558,7 +7557,7 @@ void ggml_init_cublas() { g_device_caps[id].smpb = prop.sharedMemPerBlock; } for (int id = 0; id < g_device_count; ++id) { - g_tensor_split[id] /= total_vram; + g_default_tensor_split[id] /= total_vram; } for (int id = 0; id < g_device_count; ++id) { @@ -7582,30 +7581,6 @@ void ggml_init_cublas() { } } -void ggml_cuda_set_tensor_split(const float * tensor_split) { - if (tensor_split == nullptr) { - return; - } - bool all_zero = true; - for (int i = 0; i < g_device_count; ++i) { - if (tensor_split[i] != 0.0f) { - all_zero = false; - break; - } - } - if (all_zero) { - return; - } - float split_sum = 0.0f; - for (int i = 0; i < g_device_count; ++i) { - g_tensor_split[i] = split_sum; - split_sum += tensor_split[i]; - } - for (int i = 0; i < g_device_count; ++i) { - g_tensor_split[i] /= split_sum; - } -} - void * ggml_cuda_host_malloc(size_t size) { if (getenv("GGML_CUDA_NO_PINNED") != nullptr) { return nullptr; @@ -8057,11 +8032,11 @@ static void ggml_cuda_op_mul_mat_q( (void) src1_ddf_i; } -static int64_t get_row_rounding(ggml_type type) { +static int64_t get_row_rounding(ggml_type type, const std::array & tensor_split) { int64_t min_compute_capability = INT_MAX; int64_t max_compute_capability = INT_MIN; for (int id = 0; id < g_device_count; ++id) { - if (g_tensor_split[id] < (id + 1 < g_device_count ? g_tensor_split[id + 1] : 1.0f)) { + if (tensor_split[id] < (id + 1 < g_device_count ? tensor_split[id + 1] : 1.0f)) { if (min_compute_capability > g_device_caps[id].cc) { min_compute_capability = g_device_caps[id].cc; } @@ -8122,6 +8097,21 @@ static int64_t get_row_rounding(ggml_type type) { #endif // defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__) } +static void get_row_split(int64_t * row_low, int64_t * row_high, const ggml_tensor * tensor, const std::array & tensor_split, int id) { + const int64_t nrows = ggml_nrows(tensor); + const int64_t rounding = get_row_rounding(tensor->type, tensor_split); + + *row_low = id == 0 ? 0 : nrows*tensor_split[id]; + *row_low -= *row_low % rounding; + + if (id == g_device_count - 1) { + *row_high = nrows; + } else { + *row_high = nrows*tensor_split[id + 1]; + *row_high -= *row_high % rounding; + } +} + static void ggml_cuda_op_mul_mat_vec_q( const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, const char * src0_dd_i, const float * src1_ddf_i, const char * src1_ddq_i, float * dst_dd_i, const int64_t row_low, const int64_t row_high, const int64_t src1_ncols, @@ -8739,6 +8729,11 @@ static void ggml_cuda_set_peer_access(const int n_tokens) { peer_access_enabled = enable_peer_access; } +// FIXME: move this somewhere else +struct ggml_backend_cuda_split_buffer_type_context { + std::array tensor_split; +}; + static void ggml_cuda_op_mul_mat( const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, ggml_cuda_op_mul_mat_t op, const bool convert_src1_to_q8_1) { @@ -8790,6 +8785,14 @@ static void ggml_cuda_op_mul_mat( GGML_ASSERT(!(split && ne03 > 1)); GGML_ASSERT(!(split && ne02 < ne12)); + std::array tensor_split; + if (split) { + // TODO: check that src0->buffer->buft is a split buffer type, replace GGML_BACKEND_GPU_SPLIT check + // GGML_ASSERT(src0->buffer != nullptr && src0->buffer->buft == ...); + ggml_backend_cuda_split_buffer_type_context * buft_ctx = (ggml_backend_cuda_split_buffer_type_context *) src0->buffer->buft->context; + tensor_split = buft_ctx->tensor_split; + } + struct dev_data { cuda_pool_alloc src0_dd_alloc; cuda_pool_alloc src1_ddf_alloc; @@ -8817,17 +8820,17 @@ static void ggml_cuda_op_mul_mat( // for multi GPU, get the row boundaries from tensor split // and round to mul_mat_q tile sizes if (split) { - const int64_t rounding = get_row_rounding(src0->type); + const int64_t rounding = get_row_rounding(src0->type, tensor_split); if (id != 0) { - dev[id].row_low = ne01*g_tensor_split[id]; + dev[id].row_low = ne01*tensor_split[id]; if (dev[id].row_low < ne01) { dev[id].row_low -= dev[id].row_low % rounding; } } if (id != g_device_count - 1) { - dev[id].row_high = ne01*g_tensor_split[id + 1]; + dev[id].row_high = ne01*tensor_split[id + 1]; if (dev[id].row_high < ne01) { dev[id].row_high -= dev[id].row_high % rounding; } @@ -9373,10 +9376,17 @@ static void ggml_cuda_mul_mat(const ggml_tensor * src0, const ggml_tensor * src1 const bool split = src0->backend == GGML_BACKEND_GPU_SPLIT; int64_t min_compute_capability = INT_MAX; - for (int id = 0; id < g_device_count; ++id) { - if (min_compute_capability > g_device_caps[id].cc && g_tensor_split[id] < (id + 1 < g_device_count ? g_tensor_split[id + 1] : 1.0f)) { - min_compute_capability = g_device_caps[id].cc; + + if (split) { + ggml_backend_cuda_split_buffer_type_context * buft_ctx = (ggml_backend_cuda_split_buffer_type_context *) src0->buffer->buft->context; + auto & tensor_split = buft_ctx->tensor_split; + for (int id = 0; id < g_device_count; ++id) { + if (min_compute_capability > g_device_caps[id].cc && tensor_split[id] < (id + 1 < g_device_count ? tensor_split[id + 1] : 1.0f)) { + min_compute_capability = g_device_caps[id].cc; + } } + } else { + min_compute_capability = g_device_caps[g_main_device].cc; } #if defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__) @@ -9415,7 +9425,7 @@ static void ggml_cuda_mul_mat(const ggml_tensor * src0, const ggml_tensor * src1 } else if (!split && all_on_device && !fp16_performance_good && src0->type == GGML_TYPE_F16 && !ggml_is_contiguous(src0) && !ggml_is_transposed(src1) && src1->ne[1] == 1) { // KQV single-batch ggml_cuda_mul_mat_vec_nc(src0, src1, dst); - } else if (!split && all_on_device && fp16_performance_good && src0->type == GGML_TYPE_F16 && !ggml_is_transposed(src0) && !ggml_is_transposed(src1)) { + } else if (!split && all_on_device && fp16_performance_good && src0->type == GGML_TYPE_F16 && !ggml_is_transposed(src0) && !ggml_is_transposed(src1) && src1->ne[2]*src1->ne[3] > 1) { // KQ + KQV multi-batch ggml_cuda_mul_mat_mat_batched_cublas(src0, src1, dst); } else if (src0->type == GGML_TYPE_F32) { @@ -9877,247 +9887,7 @@ static size_t ggml_nbytes_split(const struct ggml_tensor * tensor, int nrows_spl return nrows_split*ggml_row_size(tensor->type, tensor->ne[0]); } -void ggml_cuda_transform_tensor(void * data, struct ggml_tensor * tensor) { - const int64_t nrows = ggml_nrows(tensor); - - const int64_t ne0 = tensor->ne[0]; - - const size_t nb1 = tensor->nb[1]; - - ggml_backend_type backend = tensor->backend; - ggml_tensor_extra_gpu * extra = new struct ggml_tensor_extra_gpu; - memset(extra, 0, sizeof(*extra)); - - for (int id = 0; id < g_device_count; ++id) { - if (backend == GGML_BACKEND_GPU && id != g_main_device) { - continue; - } - - ggml_cuda_set_device(id); - - int64_t row_low, row_high; - if (backend == GGML_BACKEND_GPU) { - row_low = 0; - row_high = nrows; - } else if (backend == GGML_BACKEND_GPU_SPLIT) { - const int64_t rounding = get_row_rounding(tensor->type); - - row_low = id == 0 ? 0 : nrows*g_tensor_split[id]; - row_low -= row_low % rounding; - - if (id == g_device_count - 1) { - row_high = nrows; - } else { - row_high = nrows*g_tensor_split[id + 1]; - row_high -= row_high % rounding; - } - } else { - GGML_ASSERT(false); - } - if (row_low == row_high) { - continue; - } - - int64_t nrows_split = row_high - row_low; - - const size_t offset_split = row_low*nb1; - size_t size = ggml_nbytes_split(tensor, nrows_split); - const size_t original_size = size; - - // pad last row to a multiple of 512 elements to avoid out-of-bounds memory accesses - if (ne0 % MATRIX_ROW_PADDING != 0) { - size += ggml_row_size(tensor->type, MATRIX_ROW_PADDING - ne0 % MATRIX_ROW_PADDING); - } - - char * buf; - CUDA_CHECK(cudaMalloc(&buf, size)); - char * buf_host = (char *)data + offset_split; - - // set padding to 0 to avoid possible NaN values - if (size > original_size) { - CUDA_CHECK(cudaMemset(buf + original_size, 0, size - original_size)); - } - - CUDA_CHECK(cudaMemcpy(buf, buf_host, original_size, cudaMemcpyHostToDevice)); - - extra->data_device[id] = buf; - - if (backend == GGML_BACKEND_GPU_SPLIT) { - for (int64_t is = 0; is < MAX_STREAMS; ++is) { - CUDA_CHECK(cudaEventCreateWithFlags(&extra->events[id][is], cudaEventDisableTiming)); - } - } - } - - tensor->extra = extra; -} - -void ggml_cuda_free_data(struct ggml_tensor * tensor) { - if (!tensor || !tensor->extra || (tensor->backend != GGML_BACKEND_GPU && tensor->backend != GGML_BACKEND_GPU_SPLIT) ) { - return; - } - - ggml_tensor_extra_gpu * extra = (ggml_tensor_extra_gpu *) tensor->extra; - - for (int id = 0; id < g_device_count; ++id) { - ggml_cuda_set_device(id); - if (extra->data_device[id] != nullptr) { - CUDA_CHECK(cudaFree(extra->data_device[id])); - } - - for (int64_t is = 0; is < MAX_STREAMS; ++is) { - if (extra->events[id][is] != nullptr) { - CUDA_CHECK(cudaEventDestroy(extra->events[id][is])); - } - } - } - - delete extra; -} - -static ggml_tensor_extra_gpu * g_temp_tensor_extras = nullptr; -static size_t g_temp_tensor_extra_index = 0; - -static ggml_tensor_extra_gpu * ggml_cuda_alloc_temp_tensor_extra() { - if (g_temp_tensor_extras == nullptr) { - g_temp_tensor_extras = new ggml_tensor_extra_gpu[GGML_CUDA_MAX_NODES]; - } - - size_t alloc_index = g_temp_tensor_extra_index; - g_temp_tensor_extra_index = (g_temp_tensor_extra_index + 1) % GGML_CUDA_MAX_NODES; - ggml_tensor_extra_gpu * extra = &g_temp_tensor_extras[alloc_index]; - memset(extra, 0, sizeof(*extra)); - - return extra; -} - -static void ggml_cuda_assign_buffers_impl(struct ggml_tensor * tensor, bool scratch, bool force_inplace, bool no_alloc) { - if (scratch && g_scratch_size == 0) { - return; - } - - tensor->backend = GGML_BACKEND_GPU; - - // recursively assign CUDA buffers until a compute tensor is found - if (tensor->src[0] != nullptr && tensor->src[0]->backend == GGML_BACKEND_CPU) { - const ggml_op src0_op = tensor->src[0]->op; - if (src0_op == GGML_OP_RESHAPE || src0_op == GGML_OP_TRANSPOSE || src0_op == GGML_OP_VIEW || src0_op == GGML_OP_PERMUTE) { - ggml_cuda_assign_buffers_impl(tensor->src[0], scratch, force_inplace, no_alloc); - } - } - if (tensor->op == GGML_OP_CPY && tensor->src[1]->backend == GGML_BACKEND_CPU) { - ggml_cuda_assign_buffers_impl(tensor->src[1], scratch, force_inplace, no_alloc); - } - - if (scratch && no_alloc) { - return; - } - - ggml_tensor_extra_gpu * extra; - - const bool inplace = (tensor->src[0] != nullptr && tensor->src[0]->data == tensor->data) || - tensor->op == GGML_OP_VIEW || - force_inplace; - const size_t size = ggml_nbytes(tensor); - - ggml_cuda_set_device(g_main_device); - if (inplace && (tensor->src[0]->backend == GGML_BACKEND_GPU || tensor->src[0]->backend == GGML_BACKEND_GPU_SPLIT)) { - ggml_tensor_extra_gpu * src0_extra = (ggml_tensor_extra_gpu * ) tensor->src[0]->extra; - char * src0_ddc = (char *) src0_extra->data_device[g_main_device]; - size_t offset = 0; - if (tensor->op == GGML_OP_VIEW) { - memcpy(&offset, tensor->op_params, sizeof(size_t)); - } - extra = ggml_cuda_alloc_temp_tensor_extra(); - extra->data_device[g_main_device] = src0_ddc + offset; - } else if (tensor->op == GGML_OP_CPY) { - ggml_tensor_extra_gpu * src1_extra = (ggml_tensor_extra_gpu * ) tensor->src[1]->extra; - void * src1_ddv = src1_extra->data_device[g_main_device]; - extra = ggml_cuda_alloc_temp_tensor_extra(); - extra->data_device[g_main_device] = src1_ddv; - } else if (scratch) { - GGML_ASSERT(size <= g_scratch_size); - if (g_scratch_offset + size > g_scratch_size) { - g_scratch_offset = 0; - } - - char * data = (char *) g_scratch_buffer; - if (data == nullptr) { - CUDA_CHECK(cudaMalloc(&data, g_scratch_size)); - g_scratch_buffer = data; - } - extra = ggml_cuda_alloc_temp_tensor_extra(); - extra->data_device[g_main_device] = data + g_scratch_offset; - - g_scratch_offset += size; - - GGML_ASSERT(g_scratch_offset <= g_scratch_size); - } else { // allocate new buffers outside of scratch - void * data; - CUDA_CHECK(cudaMalloc(&data, size)); - CUDA_CHECK(cudaMemset(data, 0, size)); - extra = new ggml_tensor_extra_gpu; - memset(extra, 0, sizeof(*extra)); - extra->data_device[g_main_device] = data; - } - - tensor->extra = extra; -} - -void ggml_cuda_assign_scratch_offset(struct ggml_tensor * tensor, size_t offset) { - if (g_scratch_size == 0) { - return; - } - if (g_scratch_buffer == nullptr) { - ggml_cuda_set_device(g_main_device); - CUDA_CHECK(cudaMalloc(&g_scratch_buffer, g_scratch_size)); - } - - ggml_tensor_extra_gpu * extra = ggml_cuda_alloc_temp_tensor_extra(); - - const bool inplace = tensor->view_src != nullptr; - - if (inplace && (tensor->view_src->backend == GGML_BACKEND_GPU || tensor->view_src->backend == GGML_BACKEND_GPU_SPLIT)) { - ggml_tensor_extra_gpu * src0_extra = (ggml_tensor_extra_gpu * ) tensor->view_src->extra; - char * src0_ddc = (char *) src0_extra->data_device[g_main_device]; - size_t view_offset = 0; - if (tensor->op == GGML_OP_VIEW) { - memcpy(&view_offset, tensor->op_params, sizeof(size_t)); - } - extra->data_device[g_main_device] = src0_ddc + view_offset; - } else { - extra->data_device[g_main_device] = (char *) g_scratch_buffer + offset; - } - - tensor->extra = extra; -} - -void ggml_cuda_copy_to_device(struct ggml_tensor * tensor) { - GGML_ASSERT(tensor->backend == GGML_BACKEND_GPU); - GGML_ASSERT(ggml_is_contiguous(tensor)); - - ggml_tensor_extra_gpu * extra = (ggml_tensor_extra_gpu *) tensor->extra; - ggml_cuda_set_device(g_main_device); - CUDA_CHECK(cudaMemcpy(extra->data_device[g_main_device], tensor->data, ggml_nbytes(tensor), cudaMemcpyHostToDevice)); -} - -void ggml_cuda_assign_buffers(struct ggml_tensor * tensor) { - ggml_cuda_assign_buffers_impl(tensor, true, false, false); -} - -void ggml_cuda_assign_buffers_no_alloc(struct ggml_tensor * tensor) { - ggml_cuda_assign_buffers_impl(tensor, true, false, true); -} - -void ggml_cuda_assign_buffers_no_scratch(struct ggml_tensor * tensor) { - ggml_cuda_assign_buffers_impl(tensor, false, false, false); -} - -void ggml_cuda_assign_buffers_force_inplace(struct ggml_tensor * tensor) { - ggml_cuda_assign_buffers_impl(tensor, false, true, false); -} - -void ggml_cuda_set_main_device(const int main_device) { +static void ggml_cuda_set_main_device(const int main_device) { if (main_device >= g_device_count) { fprintf(stderr, "warning: cannot set main_device=%d because there are only %d devices. Using device %d instead.\n", main_device, g_device_count, g_main_device); @@ -10126,30 +9896,12 @@ void ggml_cuda_set_main_device(const int main_device) { if (g_main_device != main_device && g_device_count > 1) { g_main_device = main_device; - cudaDeviceProp prop; - CUDA_CHECK(cudaGetDeviceProperties(&prop, g_main_device)); - fprintf(stderr, "%s: using device %d (%s) as main device\n", __func__, g_main_device, prop.name); + //cudaDeviceProp prop; + //CUDA_CHECK(cudaGetDeviceProperties(&prop, g_main_device)); + //fprintf(stderr, "%s: using device %d (%s) as main device\n", __func__, g_main_device, prop.name); } } -void ggml_cuda_set_scratch_size(const size_t scratch_size) { - // this is a hack to not completely break llama.cpp when using multiple models or contexts simultaneously - // it still won't always work as expected, but it's better than nothing - if (scratch_size > g_scratch_size) { - ggml_cuda_free_scratch(); - } - g_scratch_size = std::max(g_scratch_size, scratch_size); -} - -void ggml_cuda_free_scratch() { - if (g_scratch_buffer == nullptr) { - return; - } - - CUDA_CHECK(cudaFree(g_scratch_buffer)); - g_scratch_buffer = nullptr; -} - bool ggml_cuda_compute_forward(struct ggml_compute_params * params, struct ggml_tensor * tensor) { if (!g_cublas_loaded) return false; @@ -10328,21 +10080,31 @@ void ggml_cuda_get_device_description(int device, char * description, size_t des #define UNUSED GGML_UNUSED +struct ggml_backend_cuda_context { + int device; + std::string name; +}; + // cuda buffer -struct ggml_backend_buffer_context_cuda { +struct ggml_backend_cuda_buffer_context { int device; void * dev_ptr = nullptr; ggml_tensor_extra_gpu * temp_tensor_extras = nullptr; size_t temp_tensor_extra_index = 0; + std::string name; - ggml_backend_buffer_context_cuda(int device, void * dev_ptr) : device(device), dev_ptr(dev_ptr) {} + ggml_backend_cuda_buffer_context(int device, void * dev_ptr) : + device(device), dev_ptr(dev_ptr), + name(GGML_CUDA_NAME + std::to_string(device)) { + } - ~ggml_backend_buffer_context_cuda() { + ~ggml_backend_cuda_buffer_context() { delete[] temp_tensor_extras; } ggml_tensor_extra_gpu * ggml_cuda_alloc_temp_tensor_extra() { + // TODO: remove GGML_CUDA_MAX_NODES, allocate dynamically and reuse in backend_buffer_reset if (temp_tensor_extras == nullptr) { temp_tensor_extras = new ggml_tensor_extra_gpu[GGML_CUDA_MAX_NODES]; } @@ -10356,19 +10118,28 @@ struct ggml_backend_buffer_context_cuda { } }; +static const char * ggml_backend_cuda_buffer_get_name(ggml_backend_buffer_t buffer) { + ggml_backend_cuda_buffer_context * ctx = (ggml_backend_cuda_buffer_context *)buffer->context; + return ctx->name.c_str(); +} + +static bool ggml_backend_buffer_is_cuda(ggml_backend_buffer_t buffer) { + return buffer->iface.get_name == ggml_backend_cuda_buffer_get_name; +} + static void ggml_backend_cuda_buffer_free_buffer(ggml_backend_buffer_t buffer) { - ggml_backend_buffer_context_cuda * ctx = (ggml_backend_buffer_context_cuda *)buffer->context; + ggml_backend_cuda_buffer_context * ctx = (ggml_backend_cuda_buffer_context *)buffer->context; CUDA_CHECK(cudaFree(ctx->dev_ptr)); delete ctx; } static void * ggml_backend_cuda_buffer_get_base(ggml_backend_buffer_t buffer) { - ggml_backend_buffer_context_cuda * ctx = (ggml_backend_buffer_context_cuda *)buffer->context; + ggml_backend_cuda_buffer_context * ctx = (ggml_backend_cuda_buffer_context *)buffer->context; return ctx->dev_ptr; } static void ggml_backend_cuda_buffer_init_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor) { - ggml_backend_buffer_context_cuda * ctx = (ggml_backend_buffer_context_cuda *)buffer->context; + ggml_backend_cuda_buffer_context * ctx = (ggml_backend_cuda_buffer_context *)buffer->context; if (tensor->view_src != NULL && tensor->view_offs == 0) { assert(tensor->view_src->buffer->buft == buffer->buft); @@ -10397,14 +10168,12 @@ static void ggml_backend_cuda_buffer_init_tensor(ggml_backend_buffer_t buffer, g CUDA_CHECK(cudaMemsetAsync((char *)tensor->data + original_size, 0, padded_size - original_size, g_cudaStreams[ctx->device][0])); } } - - UNUSED(buffer); } static void ggml_backend_cuda_buffer_set_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor, const void * data, size_t offset, size_t size) { GGML_ASSERT(tensor->backend == GGML_BACKEND_GPU); - ggml_backend_buffer_context_cuda * ctx = (ggml_backend_buffer_context_cuda *)buffer->context; + ggml_backend_cuda_buffer_context * ctx = (ggml_backend_cuda_buffer_context *)buffer->context; ggml_cuda_set_device(ctx->device); CUDA_CHECK(cudaDeviceSynchronize()); @@ -10415,49 +10184,82 @@ static void ggml_backend_cuda_buffer_set_tensor(ggml_backend_buffer_t buffer, gg static void ggml_backend_cuda_buffer_get_tensor(ggml_backend_buffer_t buffer, const ggml_tensor * tensor, void * data, size_t offset, size_t size) { GGML_ASSERT(tensor->backend == GGML_BACKEND_GPU); - ggml_backend_buffer_context_cuda * ctx = (ggml_backend_buffer_context_cuda *)buffer->context; + ggml_backend_cuda_buffer_context * ctx = (ggml_backend_cuda_buffer_context *)buffer->context; ggml_cuda_set_device(ctx->device); CUDA_CHECK(cudaDeviceSynchronize()); - CUDA_CHECK(cudaMemcpy(data, (const char *)tensor->data + offset, size, cudaMemcpyDeviceToHost)); + CUDA_CHECK(cudaDeviceSynchronize()); +} + +static bool ggml_backend_cuda_buffer_cpy_tensor(ggml_backend_buffer_t buffer, const ggml_tensor * src, ggml_tensor * dst) { + if (ggml_backend_buffer_is_cuda(src->buffer)) { + ggml_backend_cuda_buffer_context * src_ctx = (ggml_backend_cuda_buffer_context *)src->buffer->context; + ggml_backend_cuda_buffer_context * dst_ctx = (ggml_backend_cuda_buffer_context *)buffer->context; + + ggml_cuda_set_device(src_ctx->device); + CUDA_CHECK(cudaDeviceSynchronize()); + ggml_cuda_set_device(dst_ctx->device); + CUDA_CHECK(cudaDeviceSynchronize()); + CUDA_CHECK(cudaMemcpy((char *)dst->data, (const char *)src->data, ggml_nbytes(src), cudaMemcpyDeviceToDevice)); + CUDA_CHECK(cudaDeviceSynchronize()); + + return true; + } + return false; } static void ggml_backend_cuda_buffer_clear(ggml_backend_buffer_t buffer, uint8_t value) { - ggml_backend_buffer_context_cuda * ctx = (ggml_backend_buffer_context_cuda *)buffer->context; + ggml_backend_cuda_buffer_context * ctx = (ggml_backend_cuda_buffer_context *)buffer->context; ggml_cuda_set_device(ctx->device); CUDA_CHECK(cudaDeviceSynchronize()); - CUDA_CHECK(cudaMemset(ctx->dev_ptr, value, buffer->size)); + CUDA_CHECK(cudaDeviceSynchronize()); } -static struct ggml_backend_buffer_i cuda_backend_buffer_interface = { +static ggml_backend_buffer_i ggml_backend_cuda_buffer_interface = { + /* .get_name = */ ggml_backend_cuda_buffer_get_name, /* .free_buffer = */ ggml_backend_cuda_buffer_free_buffer, /* .get_base = */ ggml_backend_cuda_buffer_get_base, /* .init_tensor = */ ggml_backend_cuda_buffer_init_tensor, /* .set_tensor = */ ggml_backend_cuda_buffer_set_tensor, /* .get_tensor = */ ggml_backend_cuda_buffer_get_tensor, - /* .cpy_tensor_from = */ NULL, - /* .cpy_tensor_to = */ NULL, + /* .cpy_tensor = */ ggml_backend_cuda_buffer_cpy_tensor, /* .clear = */ ggml_backend_cuda_buffer_clear, + /* .reset = */ NULL, }; // cuda buffer type -static ggml_backend_buffer_t ggml_backend_cuda_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size) { - int device = (int) (intptr_t) buft->context; +struct ggml_backend_cuda_buffer_type_context { + int device; + std::string name; +}; - ggml_cuda_set_device(device); +static const char * ggml_backend_cuda_buffer_type_name(ggml_backend_buffer_type_t buft) { + ggml_backend_cuda_buffer_type_context * ctx = (ggml_backend_cuda_buffer_type_context *)buft->context; + + return ctx->name.c_str(); +} + +static ggml_backend_buffer_t ggml_backend_cuda_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size) { + ggml_backend_cuda_buffer_type_context * buft_ctx = (ggml_backend_cuda_buffer_type_context *)buft->context; + + ggml_cuda_set_device(buft_ctx->device); size = std::max(size, (size_t)1); // cudaMalloc returns null for size 0 void * dev_ptr; - CUDA_CHECK(cudaMalloc(&dev_ptr, size)); + cudaError_t err = cudaMalloc(&dev_ptr, size); + if (err != cudaSuccess) { + fprintf(stderr, "%s: allocating %.2f MiB on device %d: cudaMalloc failed: %s\n", __func__, size/1024.0/1024.0, buft_ctx->device, cudaGetErrorString(err)); + return nullptr; + } - ggml_backend_buffer_context_cuda * ctx = new ggml_backend_buffer_context_cuda(device, dev_ptr); + ggml_backend_cuda_buffer_context * ctx = new ggml_backend_cuda_buffer_context(buft_ctx->device, dev_ptr); - return ggml_backend_buffer_init(buft, cuda_backend_buffer_interface, ctx, size); + return ggml_backend_buffer_init(buft, ggml_backend_cuda_buffer_interface, ctx, size); } static size_t ggml_backend_cuda_buffer_type_get_alignment(ggml_backend_buffer_type_t buft) { @@ -10466,7 +10268,7 @@ static size_t ggml_backend_cuda_buffer_type_get_alignment(ggml_backend_buffer_ty UNUSED(buft); } -static size_t ggml_backend_cuda_buffer_type_get_alloc_size(ggml_backend_buffer_type_t buft, ggml_tensor * tensor) { +static size_t ggml_backend_cuda_buffer_type_get_alloc_size(ggml_backend_buffer_type_t buft, const ggml_tensor * tensor) { int64_t row_low = 0; int64_t row_high = ggml_nrows(tensor); int64_t nrows_split = row_high - row_low; @@ -10487,21 +10289,32 @@ static size_t ggml_backend_cuda_buffer_type_get_alloc_size(ggml_backend_buffer_t } static bool ggml_backend_cuda_buffer_type_supports_backend(ggml_backend_buffer_type_t buft, ggml_backend_t backend) { - return ggml_backend_is_cuda(backend); + if (!ggml_backend_is_cuda(backend)) { + return false; + } - UNUSED(buft); + ggml_backend_cuda_buffer_type_context * buft_ctx = (ggml_backend_cuda_buffer_type_context *)buft->context; + ggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *)backend->context; + + return buft_ctx->device == cuda_ctx->device; } static ggml_backend_buffer_type_i ggml_backend_cuda_buffer_type_interface = { + /* .get_name = */ ggml_backend_cuda_buffer_type_name, /* .alloc_buffer = */ ggml_backend_cuda_buffer_type_alloc_buffer, /* .get_alignment = */ ggml_backend_cuda_buffer_type_get_alignment, /* .get_alloc_size = */ ggml_backend_cuda_buffer_type_get_alloc_size, /* .supports_backend = */ ggml_backend_cuda_buffer_type_supports_backend, - /* .is_host = */ nullptr, + /* .is_host = */ NULL, }; ggml_backend_buffer_type_t ggml_backend_cuda_buffer_type(int device) { - static struct ggml_backend_buffer_type ggml_backend_cuda_buffer_types[GGML_CUDA_MAX_DEVICES]; + // FIXME: this is not thread safe + if (device >= ggml_backend_cuda_get_device_count()) { + return nullptr; + } + + static ggml_backend_buffer_type ggml_backend_cuda_buffer_types[GGML_CUDA_MAX_DEVICES]; static bool ggml_backend_cuda_buffer_type_initialized = false; @@ -10509,7 +10322,7 @@ ggml_backend_buffer_type_t ggml_backend_cuda_buffer_type(int device) { for (int i = 0; i < GGML_CUDA_MAX_DEVICES; i++) { ggml_backend_cuda_buffer_types[i] = { /* .iface = */ ggml_backend_cuda_buffer_type_interface, - /* .context = */ (ggml_backend_buffer_type_context_t) (intptr_t) i, + /* .context = */ new ggml_backend_cuda_buffer_type_context{i, GGML_CUDA_NAME + std::to_string(i)}, }; } ggml_backend_cuda_buffer_type_initialized = true; @@ -10518,8 +10331,306 @@ ggml_backend_buffer_type_t ggml_backend_cuda_buffer_type(int device) { return &ggml_backend_cuda_buffer_types[device]; } +// cuda split buffer + +struct ggml_backend_cuda_split_buffer_context { + ~ggml_backend_cuda_split_buffer_context() { + for (ggml_tensor_extra_gpu * extra : tensor_extras) { + for (int id = 0; id < g_device_count; ++id) { + for (int64_t is = 0; is < MAX_STREAMS; ++is) { + if (extra->events[id][is] != nullptr) { + CUDA_CHECK(cudaEventDestroy(extra->events[id][is])); + } + } + if (extra->data_device[id] != nullptr) { + CUDA_CHECK(cudaFree(extra->data_device[id])); + } + } + delete extra; + } + } + + std::vector tensor_extras; +}; + +static const char * ggml_backend_cuda_split_buffer_get_name(ggml_backend_buffer_t buffer) { + return GGML_CUDA_NAME "_Split"; + + UNUSED(buffer); +} + +// unused at the moment +//static bool ggml_backend_buffer_is_cuda_split(ggml_backend_buffer_t buffer) { +// return buffer->iface.get_name == ggml_backend_cuda_split_buffer_get_name; +//} + +static void ggml_backend_cuda_split_buffer_free_buffer(ggml_backend_buffer_t buffer) { + ggml_backend_cuda_split_buffer_context * ctx = (ggml_backend_cuda_split_buffer_context *)buffer->context; + delete ctx; +} + +static void * ggml_backend_cuda_split_buffer_get_base(ggml_backend_buffer_t buffer) { + // the pointers are stored in the tensor extras, this is just a dummy address and never dereferenced + return (void *)0x1000; + + UNUSED(buffer); +} + +static void ggml_backend_cuda_split_buffer_init_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor) { + GGML_ASSERT(tensor->view_src == nullptr); // views of split tensors are not supported + + ggml_backend_cuda_split_buffer_context * ctx = (ggml_backend_cuda_split_buffer_context *)buffer->context; + ggml_backend_cuda_split_buffer_type_context * buft_ctx = (ggml_backend_cuda_split_buffer_type_context *)buffer->buft->context; + + const int64_t ne0 = tensor->ne[0]; + + ggml_tensor_extra_gpu * extra = new ggml_tensor_extra_gpu{}; + + ctx->tensor_extras.push_back(extra); + + for (int id = 0; id < g_device_count; ++id) { + int64_t row_low, row_high; + get_row_split(&row_low, &row_high, tensor, buft_ctx->tensor_split, id); + + int64_t nrows_split = row_high - row_low; + if (nrows_split == 0) { + continue; + } + + size_t size = ggml_nbytes_split(tensor, nrows_split); + const size_t original_size = size; + + // pad last row to a multiple of 512 elements to avoid out-of-bounds memory accesses + if (ne0 % MATRIX_ROW_PADDING != 0) { + size += ggml_row_size(tensor->type, MATRIX_ROW_PADDING - ne0 % MATRIX_ROW_PADDING); + } + + // FIXME: do not crash if cudaMalloc fails + // currently, init_tensor cannot fail, it needs to be fixed in ggml-backend first + ggml_cuda_set_device(id); + char * buf; + CUDA_CHECK(cudaMalloc(&buf, size)); + + // set padding to 0 to avoid possible NaN values + if (size > original_size) { + CUDA_CHECK(cudaMemset(buf + original_size, 0, size - original_size)); + } + + extra->data_device[id] = buf; + + for (int64_t is = 0; is < MAX_STREAMS; ++is) { + CUDA_CHECK(cudaEventCreateWithFlags(&extra->events[id][is], cudaEventDisableTiming)); + } + } + tensor->backend = GGML_BACKEND_GPU_SPLIT; + tensor->extra = extra; +} + +static void ggml_backend_cuda_split_buffer_set_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor, const void * data, size_t offset, size_t size) { + // split tensors must always be set in their entirety at once + GGML_ASSERT(offset == 0); + GGML_ASSERT(size == ggml_nbytes(tensor)); + + ggml_backend_cuda_split_buffer_type_context * buft_ctx = (ggml_backend_cuda_split_buffer_type_context *)buffer->buft->context; + + const int64_t ne0 = tensor->ne[0]; + const size_t nb1 = tensor->nb[1]; + ggml_tensor_extra_gpu * extra = (ggml_tensor_extra_gpu *)tensor->extra; + + for (int id = 0; id < g_device_count; ++id) { + int64_t row_low, row_high; + get_row_split(&row_low, &row_high, tensor, buft_ctx->tensor_split, id); + + int64_t nrows_split = row_high - row_low; + if (nrows_split == 0) { + continue; + } + + const size_t offset_split = row_low*nb1; + size_t size = ggml_nbytes_split(tensor, nrows_split); + const size_t original_size = size; + + // pad last row to a multiple of 512 elements to avoid out-of-bounds memory accesses + if (ne0 % MATRIX_ROW_PADDING != 0) { + size += ggml_row_size(tensor->type, MATRIX_ROW_PADDING - ne0 % MATRIX_ROW_PADDING); + } + + const char * buf_host = (const char *)data + offset_split; + CUDA_CHECK(cudaMemcpy(extra->data_device[id], buf_host, original_size, cudaMemcpyHostToDevice)); + } +} + +static void ggml_backend_cuda_split_buffer_get_tensor(ggml_backend_buffer_t buffer, const ggml_tensor * tensor, void * data, size_t offset, size_t size) { + // split tensors must always be set in their entirety at once + GGML_ASSERT(offset == 0); + GGML_ASSERT(size == ggml_nbytes(tensor)); + + ggml_backend_cuda_split_buffer_type_context * buft_ctx = (ggml_backend_cuda_split_buffer_type_context *)buffer->buft->context; + + const int64_t ne0 = tensor->ne[0]; + const size_t nb1 = tensor->nb[1]; + ggml_tensor_extra_gpu * extra = (ggml_tensor_extra_gpu *)tensor->extra; + + for (int id = 0; id < g_device_count; ++id) { + int64_t row_low, row_high; + get_row_split(&row_low, &row_high, tensor, buft_ctx->tensor_split, id); + + int64_t nrows_split = row_high - row_low; + if (nrows_split == 0) { + continue; + } + + const size_t offset_split = row_low*nb1; + size_t size = ggml_nbytes_split(tensor, nrows_split); + const size_t original_size = size; + + // pad last row to a multiple of 512 elements to avoid out-of-bounds memory accesses + if (ne0 % MATRIX_ROW_PADDING != 0) { + size += ggml_row_size(tensor->type, MATRIX_ROW_PADDING - ne0 % MATRIX_ROW_PADDING); + } + + char * buf_host = (char *)data + offset_split; + CUDA_CHECK(cudaMemcpy(buf_host, extra->data_device[id], original_size, cudaMemcpyDeviceToHost)); + } +} + +static void ggml_backend_cuda_split_buffer_clear(ggml_backend_buffer_t buffer, uint8_t value) { + UNUSED(buffer); + UNUSED(value); +} + +static struct ggml_backend_buffer_i ggml_backend_cuda_split_buffer_interface = { + /* .get_name = */ ggml_backend_cuda_split_buffer_get_name, + /* .free_buffer = */ ggml_backend_cuda_split_buffer_free_buffer, + /* .get_base = */ ggml_backend_cuda_split_buffer_get_base, + /* .init_tensor = */ ggml_backend_cuda_split_buffer_init_tensor, + /* .set_tensor = */ ggml_backend_cuda_split_buffer_set_tensor, + /* .get_tensor = */ ggml_backend_cuda_split_buffer_get_tensor, + /* .cpy_tensor = */ NULL, + /* .clear = */ ggml_backend_cuda_split_buffer_clear, + /* .reset = */ NULL, +}; + +// cuda split buffer type + +static const char * ggml_backend_cuda_split_buffer_type_name(ggml_backend_buffer_type_t buft) { + return GGML_CUDA_NAME "_Split"; + + UNUSED(buft); +} + +static ggml_backend_buffer_t ggml_backend_cuda_split_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size) { + // since we don't know the exact split after rounding, we cannot allocate the device buffers at this point + // instead, we allocate them for each tensor separately in init_tensor + // however, the size still represents the maximum cumulative size of all the device buffers after the tensors are allocated, + // as returned by get_alloc_size. this limit is enforced during tensor allocation by ggml-alloc, so it must be correct. + ggml_backend_cuda_split_buffer_context * ctx = new ggml_backend_cuda_split_buffer_context(); + + return ggml_backend_buffer_init(buft, ggml_backend_cuda_split_buffer_interface, ctx, size); +} + +static size_t ggml_backend_cuda_split_buffer_type_get_alignment(ggml_backend_buffer_type_t buft) { + return 128; + + UNUSED(buft); +} + +static size_t ggml_backend_cuda_split_buffer_type_get_alloc_size(ggml_backend_buffer_type_t buft, const ggml_tensor * tensor) { + ggml_backend_cuda_split_buffer_type_context * ctx = (ggml_backend_cuda_split_buffer_type_context *)buft->context; + + size_t total_size = 0; + + const int64_t ne0 = tensor->ne[0]; + + for (int id = 0; id < g_device_count; ++id) { + int64_t row_low, row_high; + get_row_split(&row_low, &row_high, tensor, ctx->tensor_split, id); + + int64_t nrows_split = row_high - row_low; + if (nrows_split == 0) { + continue; + } + + total_size += ggml_nbytes_split(tensor, nrows_split); + + // pad last row to a multiple of 512 elements to avoid out-of-bounds memory accesses + if (ne0 % MATRIX_ROW_PADDING != 0) { + total_size += ggml_row_size(tensor->type, MATRIX_ROW_PADDING - ne0 % MATRIX_ROW_PADDING); + } + } + + return total_size; +} + +static bool ggml_backend_cuda_split_buffer_type_supports_backend(ggml_backend_buffer_type_t buft, ggml_backend_t backend) { + return ggml_backend_is_cuda(backend); + + UNUSED(buft); +} + +static bool ggml_backend_cuda_split_buffer_type_is_host(ggml_backend_buffer_type_t buft) { + return false; + + UNUSED(buft); +} + +static ggml_backend_buffer_type_i ggml_backend_cuda_split_buffer_type_interface = { + /* .get_name = */ ggml_backend_cuda_split_buffer_type_name, + /* .alloc_buffer = */ ggml_backend_cuda_split_buffer_type_alloc_buffer, + /* .get_alignment = */ ggml_backend_cuda_split_buffer_type_get_alignment, + /* .get_alloc_size = */ ggml_backend_cuda_split_buffer_type_get_alloc_size, + /* .supports_backend = */ ggml_backend_cuda_split_buffer_type_supports_backend, + /* .is_host = */ ggml_backend_cuda_split_buffer_type_is_host, +}; + +ggml_backend_buffer_type_t ggml_backend_cuda_split_buffer_type(const float * tensor_split) { + // FIXME: this is not thread safe + static std::map, struct ggml_backend_buffer_type> buft_map; + + std::array tensor_split_arr = {}; + + bool all_zero = tensor_split == nullptr || std::all_of(tensor_split, tensor_split + GGML_CUDA_MAX_DEVICES, [](float x) { return x == 0.0f; }); + if (all_zero) { + tensor_split_arr = g_default_tensor_split; + } else { + float split_sum = 0.0f; + for (int i = 0; i < g_device_count; ++i) { + tensor_split_arr[i] = split_sum; + split_sum += tensor_split[i]; + } + for (int i = 0; i < g_device_count; ++i) { + tensor_split_arr[i] /= split_sum; + } + } + + auto it = buft_map.find(tensor_split_arr); + if (it != buft_map.end()) { + return &it->second; + } + + struct ggml_backend_buffer_type buft { + /* .iface = */ ggml_backend_cuda_split_buffer_type_interface, + /* .context = */ new ggml_backend_cuda_split_buffer_type_context{tensor_split_arr}, + }; + + auto result = buft_map.emplace(tensor_split_arr, buft); + return &result.first->second; +} + // host buffer type +static const char * ggml_backend_cuda_host_buffer_type_name(ggml_backend_buffer_type_t buft) { + return GGML_CUDA_NAME "_Host"; + + UNUSED(buft); +} + +static const char * ggml_backend_cuda_host_buffer_name(ggml_backend_buffer_t buffer) { + return GGML_CUDA_NAME "_Host"; + + UNUSED(buffer); +} + static void ggml_backend_cuda_host_buffer_free_buffer(ggml_backend_buffer_t buffer) { ggml_cuda_host_free(buffer->context); } @@ -10532,9 +10643,9 @@ static ggml_backend_buffer_t ggml_backend_cuda_host_buffer_type_alloc_buffer(ggm return ggml_backend_buft_alloc_buffer(ggml_backend_cpu_buffer_type(), size); } - // FIXME: this is a hack to avoid having to implement a new buffer type ggml_backend_buffer_t buffer = ggml_backend_cpu_buffer_from_ptr(ptr, size); buffer->buft = buft; + buffer->iface.get_name = ggml_backend_cuda_host_buffer_name; buffer->iface.free_buffer = ggml_backend_cuda_host_buffer_free_buffer; return buffer; @@ -10543,6 +10654,7 @@ static ggml_backend_buffer_t ggml_backend_cuda_host_buffer_type_alloc_buffer(ggm ggml_backend_buffer_type_t ggml_backend_cuda_host_buffer_type() { static struct ggml_backend_buffer_type ggml_backend_cuda_buffer_type_host = { /* .iface = */ { + /* .get_name = */ ggml_backend_cuda_host_buffer_type_name, /* .alloc_buffer = */ ggml_backend_cuda_host_buffer_type_alloc_buffer, /* .get_alignment = */ ggml_backend_cpu_buffer_type()->iface.get_alignment, /* .get_alloc_size = */ ggml_backend_cpu_buffer_type()->iface.get_alloc_size, @@ -10557,31 +10669,27 @@ ggml_backend_buffer_type_t ggml_backend_cuda_host_buffer_type() { // backend -struct ggml_backend_context_cuda { - int device; -}; - static const char * ggml_backend_cuda_name(ggml_backend_t backend) { - return GGML_CUDA_NAME; + ggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *)backend->context; - UNUSED(backend); + return cuda_ctx->name.c_str(); } static void ggml_backend_cuda_free(ggml_backend_t backend) { - ggml_backend_context_cuda * cuda_ctx = (ggml_backend_context_cuda *)backend->context; + ggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *)backend->context; delete cuda_ctx; delete backend; } static ggml_backend_buffer_type_t ggml_backend_cuda_get_default_buffer_type(ggml_backend_t backend) { - ggml_backend_context_cuda * cuda_ctx = (ggml_backend_context_cuda *)backend->context; + ggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *)backend->context; return ggml_backend_cuda_buffer_type(cuda_ctx->device); } static void ggml_backend_cuda_set_tensor_async(ggml_backend_t backend, ggml_tensor * tensor, const void * data, size_t offset, size_t size) { - ggml_backend_context_cuda * cuda_ctx = (ggml_backend_context_cuda *)backend->context; + ggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *)backend->context; GGML_ASSERT(tensor->buffer->buft == ggml_backend_cuda_buffer_type(cuda_ctx->device) && "unsupported buffer type"); GGML_ASSERT(tensor->backend == GGML_BACKEND_GPU); @@ -10590,7 +10698,7 @@ static void ggml_backend_cuda_set_tensor_async(ggml_backend_t backend, ggml_tens } static void ggml_backend_cuda_get_tensor_async(ggml_backend_t backend, const ggml_tensor * tensor, void * data, size_t offset, size_t size) { - ggml_backend_context_cuda * cuda_ctx = (ggml_backend_context_cuda *)backend->context; + ggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *)backend->context; GGML_ASSERT(tensor->buffer->buft == ggml_backend_cuda_buffer_type(cuda_ctx->device) && "unsupported buffer type"); GGML_ASSERT(tensor->backend == GGML_BACKEND_GPU); @@ -10598,39 +10706,27 @@ static void ggml_backend_cuda_get_tensor_async(ggml_backend_t backend, const ggm CUDA_CHECK(cudaMemcpyAsync(data, (const char *)tensor->data + offset, size, cudaMemcpyDeviceToHost, g_cudaStreams[cuda_ctx->device][0])); } +static bool ggml_backend_cuda_cpy_tensor_async(ggml_backend_t backend, const ggml_tensor * src, ggml_tensor * dst) { + ggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *)backend->context; + + if (dst->buffer->buft == ggml_backend_cuda_buffer_type(cuda_ctx->device) && ggml_backend_buffer_is_cuda(src->buffer)) { + CUDA_CHECK(cudaMemcpyAsync(dst->data, src->data, ggml_nbytes(dst), cudaMemcpyDeviceToDevice, g_cudaStreams[cuda_ctx->device][0])); + return true; + } + + return false; +} + static void ggml_backend_cuda_synchronize(ggml_backend_t backend) { - ggml_backend_context_cuda * cuda_ctx = (ggml_backend_context_cuda *)backend->context; + ggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *)backend->context; CUDA_CHECK(cudaStreamSynchronize(g_cudaStreams[cuda_ctx->device][0])); UNUSED(backend); } -static ggml_backend_graph_plan_t ggml_backend_cuda_graph_plan_create(ggml_backend_t backend, ggml_cgraph * cgraph) { - GGML_ASSERT(!"not implemented"); - - return nullptr; - - UNUSED(backend); - UNUSED(cgraph); -} - -static void ggml_backend_cuda_graph_plan_free(ggml_backend_t backend, ggml_backend_graph_plan_t plan) { - GGML_ASSERT(!"not implemented"); - - UNUSED(backend); - UNUSED(plan); -} - -static void ggml_backend_cuda_graph_plan_compute(ggml_backend_t backend, ggml_backend_graph_plan_t plan) { - GGML_ASSERT(!"not implemented"); - - UNUSED(backend); - UNUSED(plan); -} - static bool ggml_backend_cuda_graph_compute(ggml_backend_t backend, ggml_cgraph * cgraph) { - ggml_backend_context_cuda * cuda_ctx = (ggml_backend_context_cuda *)backend->context; + ggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *)backend->context; ggml_cuda_set_main_device(cuda_ctx->device); @@ -10640,53 +10736,31 @@ static bool ggml_backend_cuda_graph_compute(ggml_backend_t backend, ggml_cgraph for (int i = 0; i < cgraph->n_nodes; i++) { ggml_tensor * node = cgraph->nodes[i]; - if (node->op == GGML_OP_RESHAPE || node->op == GGML_OP_TRANSPOSE || node->op == GGML_OP_VIEW || node->op == GGML_OP_PERMUTE) + if (node->op == GGML_OP_RESHAPE || node->op == GGML_OP_TRANSPOSE || node->op == GGML_OP_VIEW || node->op == GGML_OP_PERMUTE || node->op == GGML_OP_NONE) { continue; + } - assert(node->backend == GGML_BACKEND_GPU); +#ifndef NDEBUG + assert(node->backend == GGML_BACKEND_GPU || node->backend == GGML_BACKEND_GPU_SPLIT); assert(node->buffer->buft == ggml_backend_cuda_buffer_type(cuda_ctx->device)); assert(node->extra != nullptr); for (int j = 0; j < GGML_MAX_SRC; j++) { if (node->src[j] != nullptr) { - assert(node->src[j]->backend == GGML_BACKEND_GPU); + assert(node->src[j]->backend == GGML_BACKEND_GPU || node->src[j]->backend == GGML_BACKEND_GPU_SPLIT); assert(node->src[j]->buffer->buft == ggml_backend_cuda_buffer_type(cuda_ctx->device)); assert(node->src[j]->extra != nullptr); } } +#endif bool ok = ggml_cuda_compute_forward(¶ms, node); if (!ok) { fprintf(stderr, "%s: error: op not supported %s (%s)\n", __func__, node->name, ggml_op_name(node->op)); } GGML_ASSERT(ok); - -#if 0 - if (node->type == GGML_TYPE_F32) { - cudaDeviceSynchronize(); - std::vector tmp(ggml_nelements(node), 0.0f); - cudaMemcpy(tmp.data(), node->data, ggml_nelements(node)*sizeof(float), cudaMemcpyDeviceToHost); - printf("\n%s (%s) (%s %s) (%s %s): ", node->name, ggml_op_name(node->op), - ggml_type_name(node->src[0]->type), - node->src[1] ? ggml_type_name(node->src[1]->type) : "none", - node->src[0]->name, - node->src[1] ? node->src[1]->name : "none"); - double sum = 0.0; - double sq_sum = 0.0; - for (int i = 0; i < ggml_nelements(node); i++) { - printf("%f ", tmp[i]); - sum += tmp[i]; - sq_sum += tmp[i]*tmp[i]; - } - printf("\n"); - printf("sum: %f, ", sum); - printf("sq_sum: %f\n", sq_sum); - } -#endif } - UNUSED(backend); - return true; } @@ -10801,18 +10875,17 @@ static bool ggml_backend_cuda_supports_op(ggml_backend_t backend, const ggml_ten UNUSED(backend); } -static ggml_backend_i cuda_backend_i = { +static ggml_backend_i ggml_backend_cuda_interface = { /* .get_name = */ ggml_backend_cuda_name, /* .free = */ ggml_backend_cuda_free, /* .get_default_buffer_type = */ ggml_backend_cuda_get_default_buffer_type, /* .set_tensor_async = */ ggml_backend_cuda_set_tensor_async, /* .get_tensor_async = */ ggml_backend_cuda_get_tensor_async, - /* .cpy_tensor_from_async = */ NULL, - /* .cpy_tensor_to_async = */ NULL, + /* .cpy_tensor_async = */ ggml_backend_cuda_cpy_tensor_async, /* .synchronize = */ ggml_backend_cuda_synchronize, - /* .graph_plan_create = */ ggml_backend_cuda_graph_plan_create, - /* .graph_plan_free = */ ggml_backend_cuda_graph_plan_free, - /* .graph_plan_compute = */ ggml_backend_cuda_graph_plan_compute, + /* .graph_plan_create = */ NULL, + /* .graph_plan_free = */ NULL, + /* .graph_plan_compute = */ NULL, /* .graph_compute = */ ggml_backend_cuda_graph_compute, /* .supports_op = */ ggml_backend_cuda_supports_op, }; @@ -10828,12 +10901,13 @@ ggml_backend_t ggml_backend_cuda_init(int device) { // not strictly necessary, but it may reduce the overhead of the first graph_compute ggml_cuda_set_main_device(device); - ggml_backend_context_cuda * ctx = new ggml_backend_context_cuda { - /* .device = */ device + ggml_backend_cuda_context * ctx = new ggml_backend_cuda_context { + /* .device = */ device, + /* .name = */ GGML_CUDA_NAME + std::to_string(device), }; ggml_backend_t cuda_backend = new ggml_backend { - /* .interface = */ cuda_backend_i, + /* .interface = */ ggml_backend_cuda_interface, /* .context = */ ctx }; @@ -10841,9 +10915,24 @@ ggml_backend_t ggml_backend_cuda_init(int device) { } bool ggml_backend_is_cuda(ggml_backend_t backend) { - return backend->iface.get_name == ggml_backend_cuda_name; + return backend && backend->iface.get_name == ggml_backend_cuda_name; } +int ggml_backend_cuda_get_device_count() { + return ggml_cuda_get_device_count(); +} + +void ggml_backend_cuda_get_device_description(int device, char * description, size_t description_size) { + ggml_cuda_get_device_description(device, description, description_size); +} + +void ggml_backend_cuda_get_device_memory(int device, size_t * free, size_t * total) { + ggml_cuda_set_device(device); + + CUDA_CHECK(cudaMemGetInfo(free, total)); +} + +// backend registry static ggml_backend_t ggml_backend_reg_cuda_init(const char * params, void * user_data) { ggml_backend_t cuda_backend = ggml_backend_cuda_init((int) (intptr_t) user_data); return cuda_backend; diff --git a/ggml-cuda.h b/ggml-cuda.h index cdb0c0c41..d19cbf3fd 100644 --- a/ggml-cuda.h +++ b/ggml-cuda.h @@ -27,22 +27,6 @@ GGML_API void * ggml_cuda_host_malloc(size_t size); GGML_API void ggml_cuda_host_free(void * ptr); GGML_API bool ggml_cuda_can_mul_mat(const struct ggml_tensor * src0, const struct ggml_tensor * src1, struct ggml_tensor * dst); -GGML_API void ggml_cuda_set_tensor_split(const float * tensor_split); -GGML_API void ggml_cuda_transform_tensor(void * data, struct ggml_tensor * tensor); -GGML_API void ggml_cuda_free_data(struct ggml_tensor * tensor); - -GGML_API void ggml_cuda_assign_buffers(struct ggml_tensor * tensor); -GGML_API void ggml_cuda_assign_buffers_no_scratch(struct ggml_tensor * tensor); -GGML_API void ggml_cuda_assign_buffers_force_inplace(struct ggml_tensor * tensor); - -GGML_API void ggml_cuda_assign_buffers_no_alloc(struct ggml_tensor * tensor); -GGML_API void ggml_cuda_assign_scratch_offset(struct ggml_tensor * tensor, size_t offset); -GGML_API void ggml_cuda_copy_to_device(struct ggml_tensor * tensor); - -GGML_API void ggml_cuda_set_main_device(int main_device); -GGML_API void ggml_cuda_set_mul_mat_q(bool mul_mat_q); -GGML_API void ggml_cuda_set_scratch_size(size_t scratch_size); -GGML_API void ggml_cuda_free_scratch(void); GGML_API bool ggml_cuda_compute_forward(struct ggml_compute_params * params, struct ggml_tensor * tensor); GGML_API int ggml_cuda_get_device_count(void); @@ -52,13 +36,17 @@ GGML_API void ggml_cuda_get_device_description(int device, char * description, GGML_API ggml_backend_t ggml_backend_cuda_init(int device); GGML_API bool ggml_backend_is_cuda(ggml_backend_t backend); -GGML_API int ggml_backend_cuda_get_device(ggml_backend_t backend); GGML_API ggml_backend_buffer_type_t ggml_backend_cuda_buffer_type(int device); - -// pinned host buffer for use with CPU backend for faster copies between CPU and GPU +// split tensor buffer that splits matrices by rows across multiple devices +GGML_API ggml_backend_buffer_type_t ggml_backend_cuda_split_buffer_type(const float * tensor_split); +// pinned host buffer for use with the CPU backend for faster copies between CPU and GPU GGML_API ggml_backend_buffer_type_t ggml_backend_cuda_host_buffer_type(void); +GGML_API int ggml_backend_cuda_get_device_count(void); +GGML_API void ggml_backend_cuda_get_device_description(int device, char * description, size_t description_size); +GGML_API void ggml_backend_cuda_get_device_memory(int device, size_t * free, size_t * total); + #ifdef __cplusplus } #endif diff --git a/ggml-impl.h b/ggml-impl.h index 2faced080..2c58075ac 100644 --- a/ggml-impl.h +++ b/ggml-impl.h @@ -228,6 +228,8 @@ inline static float ggml_lookup_fp16_to_fp32(ggml_fp16_t f) { #define GGML_HASHTABLE_FULL ((size_t)-1) #define GGML_HASHTABLE_ALREADY_EXISTS ((size_t)-2) +struct ggml_hash_set ggml_hash_set_new(size_t size); + bool ggml_hash_contains (const struct ggml_hash_set hash_set, struct ggml_tensor * key); // returns GGML_HASHTABLE_FULL if table is full, otherwise the current index of the key or where it should be inserted diff --git a/ggml-metal.m b/ggml-metal.m index 6e5594432..c03624073 100644 --- a/ggml-metal.m +++ b/ggml-metal.m @@ -2520,10 +2520,10 @@ static void ggml_backend_metal_free_device(void) { } } -static void * ggml_backend_metal_buffer_get_base(ggml_backend_buffer_t buffer) { - struct ggml_backend_metal_buffer_context * ctx = (struct ggml_backend_metal_buffer_context *)buffer->context; +static const char * ggml_backend_metal_buffer_get_name(ggml_backend_buffer_t buffer) { + return "Metal"; - return ctx->all_data; + UNUSED(buffer); } static void ggml_backend_metal_buffer_free_buffer(ggml_backend_buffer_t buffer) { @@ -2541,6 +2541,12 @@ static void ggml_backend_metal_buffer_free_buffer(ggml_backend_buffer_t buffer) free(ctx); } +static void * ggml_backend_metal_buffer_get_base(ggml_backend_buffer_t buffer) { + struct ggml_backend_metal_buffer_context * ctx = (struct ggml_backend_metal_buffer_context *)buffer->context; + + return ctx->all_data; +} + static void ggml_backend_metal_buffer_set_tensor(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor, const void * data, size_t offset, size_t size) { memcpy((char *)tensor->data + offset, data, size); @@ -2553,14 +2559,12 @@ static void ggml_backend_metal_buffer_get_tensor(ggml_backend_buffer_t buffer, c UNUSED(buffer); } -static void ggml_backend_metal_buffer_cpy_tensor_from(ggml_backend_buffer_t buffer, struct ggml_tensor * src, struct ggml_tensor * dst) { - ggml_backend_tensor_get(src, dst->data, 0, ggml_nbytes(src)); - - UNUSED(buffer); -} - -static void ggml_backend_metal_buffer_cpy_tensor_to(ggml_backend_buffer_t buffer, struct ggml_tensor * src, struct ggml_tensor * dst) { - ggml_backend_tensor_set(dst, src->data, 0, ggml_nbytes(src)); +static bool ggml_backend_metal_buffer_cpy_tensor(ggml_backend_buffer_t buffer, const struct ggml_tensor * src, struct ggml_tensor * dst) { + if (ggml_backend_buffer_is_host(src->buffer)) { + memcpy(dst->data, src->data, ggml_nbytes(src)); + return true; + } + return false; UNUSED(buffer); } @@ -2572,18 +2576,25 @@ static void ggml_backend_metal_buffer_clear(ggml_backend_buffer_t buffer, uint8_ } static struct ggml_backend_buffer_i ggml_backend_metal_buffer_i = { + /* .get_name = */ ggml_backend_metal_buffer_get_name, /* .free_buffer = */ ggml_backend_metal_buffer_free_buffer, /* .get_base = */ ggml_backend_metal_buffer_get_base, /* .init_tensor = */ NULL, /* .set_tensor = */ ggml_backend_metal_buffer_set_tensor, /* .get_tensor = */ ggml_backend_metal_buffer_get_tensor, - /* .cpy_tensor_from = */ ggml_backend_metal_buffer_cpy_tensor_from, - /* .cpy_tensor_to = */ ggml_backend_metal_buffer_cpy_tensor_to, + /* .cpy_tensor = */ ggml_backend_metal_buffer_cpy_tensor, /* .clear = */ ggml_backend_metal_buffer_clear, + /* .reset = */ NULL, }; // default buffer type +static const char * ggml_backend_metal_buffer_type_get_name(ggml_backend_buffer_type_t buft) { + return "Metal"; + + UNUSED(buft); +} + static ggml_backend_buffer_t ggml_backend_metal_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size) { struct ggml_backend_metal_buffer_context * ctx = malloc(sizeof(struct ggml_backend_metal_buffer_context)); @@ -2656,6 +2667,7 @@ static bool ggml_backend_metal_buffer_type_is_host(ggml_backend_buffer_type_t bu ggml_backend_buffer_type_t ggml_backend_metal_buffer_type(void) { static struct ggml_backend_buffer_type ggml_backend_buffer_type_metal = { /* .iface = */ { + /* .get_name = */ ggml_backend_metal_buffer_type_get_name, /* .alloc_buffer = */ ggml_backend_metal_buffer_type_alloc_buffer, /* .get_alignment = */ ggml_backend_metal_buffer_type_get_alignment, /* .get_alloc_size = */ NULL, // defaults to ggml_nbytes @@ -2679,6 +2691,14 @@ ggml_backend_buffer_t ggml_backend_metal_buffer_from_ptr(void * data, size_t siz ctx->n_buffers = 0; const size_t size_page = sysconf(_SC_PAGESIZE); + + // page-align the data ptr + { + const uintptr_t offs = (uintptr_t) data % size_page; + data = (void *) ((char *) data - offs); + size += offs; + } + size_t size_aligned = size; if ((size_aligned % size_page) != 0) { size_aligned += (size_page - (size_aligned % size_page)); @@ -2779,14 +2799,13 @@ static bool ggml_backend_metal_supports_op(ggml_backend_t backend, const struct UNUSED(backend); } -static struct ggml_backend_i metal_backend_i = { +static struct ggml_backend_i ggml_backend_metal_i = { /* .get_name = */ ggml_backend_metal_name, /* .free = */ ggml_backend_metal_free, /* .get_default_buffer_type = */ ggml_backend_metal_get_default_buffer_type, /* .set_tensor_async = */ NULL, /* .get_tensor_async = */ NULL, - /* .cpy_tensor_from_async = */ NULL, - /* .cpy_tensor_to_async = */ NULL, + /* .cpy_tensor_async = */ NULL, /* .synchronize = */ NULL, /* .graph_plan_create = */ NULL, /* .graph_plan_free = */ NULL, @@ -2805,7 +2824,7 @@ ggml_backend_t ggml_backend_metal_init(void) { ggml_backend_t metal_backend = malloc(sizeof(struct ggml_backend)); *metal_backend = (struct ggml_backend) { - /* .interface = */ metal_backend_i, + /* .interface = */ ggml_backend_metal_i, /* .context = */ ctx, }; @@ -2813,7 +2832,7 @@ ggml_backend_t ggml_backend_metal_init(void) { } bool ggml_backend_is_metal(ggml_backend_t backend) { - return backend->iface.get_name == ggml_backend_metal_name; + return backend && backend->iface.get_name == ggml_backend_metal_name; } void ggml_backend_metal_set_n_cb(ggml_backend_t backend, int n_cb) { diff --git a/ggml-opencl.cpp b/ggml-opencl.cpp index 496f9cdca..2bb93638f 100644 --- a/ggml-opencl.cpp +++ b/ggml-opencl.cpp @@ -1,5 +1,6 @@ #include "ggml.h" #include "ggml-opencl.h" +#include "ggml-backend-impl.h" #include #include @@ -10,7 +11,7 @@ #include #include -#define CL_TARGET_OPENCL_VERSION 110 +#define CL_TARGET_OPENCL_VERSION 120 #include #if defined(_MSC_VER) @@ -929,6 +930,12 @@ static cl_program build_program_from_source(cl_context ctx, cl_device_id dev, co } void ggml_cl_init(void) { + static bool initialized = false; + if (initialized) { + return; + } + initialized = true; + cl_int err; struct cl_device; @@ -1483,8 +1490,8 @@ static void ggml_cl_mul_mat_f32(const ggml_tensor * src0, const ggml_tensor * sr } else { d_X = ggml_cl_pool_malloc(sizeof(float) * x_ne, &x_size); } - cl_mem d_Y = ggml_cl_pool_malloc(sizeof(float) * y_ne, &y_size); - cl_mem d_D = ggml_cl_pool_malloc(sizeof(float) * d_ne, &d_size); + cl_mem d_Y = src1->backend == GGML_BACKEND_GPU ? (cl_mem) src1->extra : ggml_cl_pool_malloc(sizeof(float) * y_ne, &y_size); + cl_mem d_D = dst->backend == GGML_BACKEND_GPU ? (cl_mem) dst->extra : ggml_cl_pool_malloc(sizeof(float) * d_ne, &d_size); size_t x_offset = 0; @@ -1501,7 +1508,9 @@ static void ggml_cl_mul_mat_f32(const ggml_tensor * src0, const ggml_tensor * sr for (int64_t i12 = i02 * r2, e12 = i12 + r2; i12 < e12; i12++) { // copy src1 to device - CL_CHECK(ggml_cl_h2d_tensor_2d(queue, d_Y, 0, src1, i13, i12, NULL)); + if (src1->backend == GGML_BACKEND_CPU) { + CL_CHECK(ggml_cl_h2d_tensor_2d(queue, d_Y, 0, src1, i13, i12, NULL)); + } CL_CHECK(clFinish(queue)); @@ -1522,8 +1531,10 @@ static void ggml_cl_mul_mat_f32(const ggml_tensor * src0, const ggml_tensor * sr } // copy dst to host - float * d = (float *) ((char *) dst->data + i12*nb2 + i13*nb3); - CL_CHECK(clEnqueueReadBuffer(queue, d_D, true, 0, sizeof(float) * d_ne, d, 1, &ev_sgemm, NULL)); + if (dst->backend == GGML_BACKEND_CPU) { + float * d = (float *) ((char *) dst->data + i12*nb2 + i13*nb3); + CL_CHECK(clEnqueueReadBuffer(queue, d_D, true, 0, sizeof(float) * d_ne, d, 1, &ev_sgemm, NULL)); + } } } } @@ -1532,8 +1543,12 @@ static void ggml_cl_mul_mat_f32(const ggml_tensor * src0, const ggml_tensor * sr if (src0->backend != GGML_BACKEND_GPU) { ggml_cl_pool_free(d_X, x_size); } - ggml_cl_pool_free(d_Y, y_size); - ggml_cl_pool_free(d_D, d_size); + if (src1->backend != GGML_BACKEND_GPU) { + ggml_cl_pool_free(d_Y, y_size); + } + if (dst->backend != GGML_BACKEND_GPU) { + ggml_cl_pool_free(d_D, d_size); + } } static void ggml_cl_mul_mat_f16(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, void * wdata, size_t wsize) { @@ -1598,6 +1613,8 @@ static void ggml_cl_mul_mat_f16(const ggml_tensor * src0, const ggml_tensor * sr CL_CHECK(ggml_cl_h2d_tensor_2d(queue, d_X, 0, src0, i03, i02, NULL)); } + // FIXME: convert on device + for (int64_t i12 = i02 * r2, e12 = i12 + r2; i12 < e12; i12++) { // convert src1 to fp16 // TODO: use multiple threads @@ -1643,11 +1660,13 @@ static void ggml_cl_mul_mat_f16(const ggml_tensor * src0, const ggml_tensor * sr } // copy dst to host, then convert to float - CL_CHECK(clEnqueueReadBuffer(queue, d_D, true, 0, sizeof(ggml_fp16_t) * d_ne, tmp, 1, &ev_sgemm, NULL)); - - float * d = (float *) ((char *) dst->data + i12*nb2 + i13*nb3); - - ggml_fp16_to_fp32_row(tmp, d, d_ne); + if (dst->backend == GGML_BACKEND_CPU) { + CL_CHECK(clEnqueueReadBuffer(queue, d_D, true, 0, sizeof(ggml_fp16_t) * d_ne, tmp, 1, &ev_sgemm, NULL)); + float * d = (float *) ((char *) dst->data + i12*nb2 + i13*nb3); + ggml_fp16_to_fp32_row(tmp, d, d_ne); + } else { + // FIXME: convert dst to fp32 on device + } } } } @@ -1801,7 +1820,7 @@ static void ggml_cl_mul_mat_q_f32(const ggml_tensor * src0, const ggml_tensor * } -bool ggml_cl_can_mul_mat(const struct ggml_tensor * src0, const struct ggml_tensor * src1, struct ggml_tensor * dst) { +bool ggml_cl_can_mul_mat(const struct ggml_tensor * src0, const struct ggml_tensor * src1, const struct ggml_tensor * dst) { const int64_t ne10 = src1->ne[0]; const int64_t ne0 = dst->ne[0]; @@ -1895,3 +1914,291 @@ void ggml_cl_transform_tensor(void * data, ggml_tensor * tensor) { tensor->extra = dst; GGML_ASSERT(tensor->backend == GGML_BACKEND_GPU); } + +// ggml-backend + +// buffer + +struct ggml_backend_opencl_buffer_context { + ~ggml_backend_opencl_buffer_context() { + if (buffer) { + clReleaseMemObject(buffer); + } + for (auto * sub_buffer : sub_buffers) { + clReleaseMemObject(sub_buffer); + } + } + + cl_mem buffer; + std::vector sub_buffers; +}; + +static void * const cl_ptr_base = (void *)(uintptr_t) 0x1000; + +static const char * ggml_backend_opencl_buffer_get_name(ggml_backend_buffer_t buffer) { + return "OpenCL"; + + GGML_UNUSED(buffer); +} + +static void ggml_backend_opencl_buffer_free_buffer(ggml_backend_buffer_t buffer) { + ggml_backend_opencl_buffer_context * ctx = (ggml_backend_opencl_buffer_context *) buffer->context; + delete ctx; +} + +static void * ggml_backend_opencl_buffer_get_base(ggml_backend_buffer_t buffer) { + return cl_ptr_base; + + GGML_UNUSED(buffer); +} + +static void ggml_backend_opencl_buffer_init_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor) { + if (tensor->view_src != NULL && tensor->view_offs == 0) { + tensor->extra = tensor->view_src->extra; + } else { + ggml_backend_opencl_buffer_context * ctx = (ggml_backend_opencl_buffer_context *) buffer->context; + cl_buffer_region region = {(size_t)((char *)tensor->data - (char *)cl_ptr_base), ggml_nbytes(tensor)}; + cl_int err; + cl_mem sub_buffer = clCreateSubBuffer(ctx->buffer, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); + CL_CHECK(err); + ctx->sub_buffers.push_back(sub_buffer); + tensor->extra = sub_buffer; + } + tensor->backend = GGML_BACKEND_GPU; +} + +static void ggml_backend_opencl_buffer_set_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor, const void * data, size_t offset, size_t size) { + cl_mem tensor_buffer = (cl_mem) tensor->extra; + CL_CHECK(clEnqueueWriteBuffer(queue, tensor_buffer, true, offset, size, data, 0, NULL, NULL)); + CL_CHECK(clFinish(queue)); + + GGML_UNUSED(buffer); +} + +static void ggml_backend_opencl_buffer_get_tensor(ggml_backend_buffer_t buffer, const ggml_tensor * tensor, void * data, size_t offset, size_t size) { + cl_mem tensor_buffer = (cl_mem) tensor->extra; + CL_CHECK(clEnqueueReadBuffer(queue, tensor_buffer, true, offset, size, data, 0, NULL, NULL)); + CL_CHECK(clFinish(queue)); + + GGML_UNUSED(buffer); +} + +static void ggml_backend_opencl_buffer_clear(ggml_backend_buffer_t buffer, uint8_t value) { + ggml_backend_opencl_buffer_context * ctx = (ggml_backend_opencl_buffer_context *) buffer->context; + CL_CHECK(clEnqueueFillBuffer(queue, ctx->buffer, &value, sizeof(value), 0, buffer->size, 0, NULL, NULL)); + CL_CHECK(clFinish(queue)); +} + +static void ggml_backend_opencl_buffer_reset(ggml_backend_buffer_t buffer) { + ggml_backend_opencl_buffer_context * ctx = (ggml_backend_opencl_buffer_context *) buffer->context; + for (auto * sub_buffer : ctx->sub_buffers) { + clReleaseMemObject(sub_buffer); + } + ctx->sub_buffers.clear(); +} + +static ggml_backend_buffer_i ggml_backend_opencl_buffer_interface = { + /* .get_name = */ ggml_backend_opencl_buffer_get_name, + /* .free_buffer = */ ggml_backend_opencl_buffer_free_buffer, + /* .get_base = */ ggml_backend_opencl_buffer_get_base, + /* .init_tensor = */ ggml_backend_opencl_buffer_init_tensor, + /* .set_tensor = */ ggml_backend_opencl_buffer_set_tensor, + /* .get_tensor = */ ggml_backend_opencl_buffer_get_tensor, + /* .cpy_tensor = */ NULL, + /* .clear = */ ggml_backend_opencl_buffer_clear, + /* .reset = */ ggml_backend_opencl_buffer_reset, +}; + +// buffer type + +static const char * ggml_backend_opencl_buffer_type_name(ggml_backend_buffer_type_t buffer_type) { + return "OpenCL"; + + GGML_UNUSED(buffer_type); +} + +static ggml_backend_buffer_t ggml_backend_opencl_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buffer_type, size_t size) { + ggml_cl_init(); + + cl_int err; + cl_mem mem = clCreateBuffer(context, CL_MEM_READ_WRITE, size, NULL, &err); + if (err != CL_SUCCESS) { + fprintf(stderr, "%s: failed to allocate %.2f MiB\n", __func__, size / 1024.0 / 1024.0); + return nullptr; + } + + ggml_backend_opencl_buffer_context * ctx = new ggml_backend_opencl_buffer_context{mem, {}}; + + return ggml_backend_buffer_init(buffer_type, ggml_backend_opencl_buffer_interface, ctx, size); +} + +static size_t ggml_backend_opencl_buffer_type_get_alignment(ggml_backend_buffer_type_t buffer_type) { + // FIXME: not thread safe, device may not be initialized yet + static cl_uint alignment = -1; + if (alignment == (cl_uint)-1) { + ggml_cl_init(); + clGetDeviceInfo(device, CL_DEVICE_MEM_BASE_ADDR_ALIGN, sizeof(cl_uint), &alignment, NULL); + } + return alignment; + + GGML_UNUSED(buffer_type); +} + +static bool ggml_backend_opencl_buffer_type_supports_backend(ggml_backend_buffer_type_t buffer_type, ggml_backend_t backend) { + //return ggml_backend_is_opencl(backend); // opencl must be used through the cpu backend + return ggml_backend_is_cpu(backend); + + GGML_UNUSED(buffer_type); +} + +static ggml_backend_buffer_type_i ggml_backend_opencl_buffer_type_interface = { + /* .get_name = */ ggml_backend_opencl_buffer_type_name, + /* .alloc_buffer = */ ggml_backend_opencl_buffer_type_alloc_buffer, + /* .get_alignment = */ ggml_backend_opencl_buffer_type_get_alignment, + /* .get_alloc_size = */ NULL, + /* .supports_backend = */ ggml_backend_opencl_buffer_type_supports_backend, + /* .is_host = */ NULL, +}; + + +ggml_backend_buffer_type_t ggml_backend_opencl_buffer_type() { + static ggml_backend_buffer_type buffer_type = { + /* .iface = */ ggml_backend_opencl_buffer_type_interface, + /* .context = */ nullptr, + }; + + return &buffer_type; +} + +#if 0 +// host buffer type + +static const char * ggml_backend_opencl_host_buffer_type_name(ggml_backend_buffer_type_t buft) { + return "CL_Host"; + + GGML_UNUSED(buft); +} + +static const char * ggml_backend_opencl_host_buffer_name(ggml_backend_buffer_t buffer) { + return "CL_Host"; + + GGML_UNUSED(buffer); +} + +static void ggml_backend_opencl_host_buffer_free_buffer(ggml_backend_buffer_t buffer) { + ggml_cl_host_free(buffer->context); +} + +static ggml_backend_buffer_t ggml_backend_opencl_host_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size) { + void * ptr = ggml_cl_host_malloc(size); + + if (ptr == nullptr) { + // fallback to cpu buffer + return ggml_backend_buft_alloc_buffer(ggml_backend_cpu_buffer_type(), size); + } + + ggml_backend_buffer_t buffer = ggml_backend_cpu_buffer_from_ptr(ptr, size); + buffer->buft = buft; + buffer->iface.get_name = ggml_backend_opencl_host_buffer_name; + buffer->iface.free_buffer = ggml_backend_opencl_host_buffer_free_buffer; + + return buffer; +} + +ggml_backend_buffer_type_t ggml_backend_opencl_host_buffer_type() { + static struct ggml_backend_buffer_type ggml_backend_opencl_buffer_type_host = { + /* .iface = */ { + /* .get_name = */ ggml_backend_opencl_host_buffer_type_name, + /* .alloc_buffer = */ ggml_backend_opencl_host_buffer_type_alloc_buffer, + /* .get_alignment = */ ggml_backend_cpu_buffer_type()->iface.get_alignment, + /* .get_alloc_size = */ ggml_backend_cpu_buffer_type()->iface.get_alloc_size, + /* .supports_backend = */ ggml_backend_cpu_buffer_type()->iface.supports_backend, + /* .is_host = */ ggml_backend_cpu_buffer_type()->iface.is_host, + }, + /* .context = */ nullptr, + }; + + return &ggml_backend_opencl_buffer_type_host; +} + +// backend + +static const char * ggml_backend_opencl_name(ggml_backend_t backend) { + return "OpenCL"; + + GGML_UNUSED(backend); +} + +static void ggml_backend_opencl_free(ggml_backend_t backend) { + GGML_UNUSED(backend); +} + +static ggml_backend_buffer_type_t ggml_backend_opencl_get_default_buffer_type(ggml_backend_t backend) { + return ggml_backend_opencl_buffer_type(); + + GGML_UNUSED(backend); +} + +static bool ggml_backend_opencl_graph_compute(ggml_backend_t backend, ggml_cgraph * graph) { + for (int i = 0; i < graph->n_nodes; ++i) { + ggml_tensor * node = graph->nodes[i]; + switch (node->op) { + case GGML_OP_MUL_MAT: + ggml_cl_mul_mat(node->src[0], node->src[1], node, nullptr, 0); + break; + case GGML_OP_MUL: + ggml_cl_mul(node->src[0], node->src[1], node); + break; + default: + GGML_ASSERT(false); + } + } + + return true; + + GGML_UNUSED(backend); +} + +static bool ggml_backend_opencl_supports_op(ggml_backend_t backend, const ggml_tensor * op) { + switch (op->op) { + case GGML_OP_MUL_MAT: + return ggml_cl_can_mul_mat(op->src[0], op->src[1], op); + case GGML_OP_MUL: + // return ggml_can_repeat_rows(op->src[1], op->src[0]); + return true; + default: + return false; + } + + GGML_UNUSED(backend); +} + +static ggml_backend_i opencl_backend_i = { + /* .get_name = */ ggml_backend_opencl_name, + /* .free = */ ggml_backend_opencl_free, + /* .get_default_buffer_type = */ ggml_backend_opencl_get_default_buffer_type, + /* .set_tensor_async = */ NULL, + /* .get_tensor_async = */ NULL, + /* .cpy_tensor_from_async = */ NULL, + /* .cpy_tensor_to_async = */ NULL, + /* .synchronize = */ NULL, + /* .graph_plan_create = */ NULL, + /* .graph_plan_free = */ NULL, + /* .graph_plan_compute = */ NULL, + /* .graph_compute = */ ggml_backend_opencl_graph_compute, + /* .supports_op = */ ggml_backend_opencl_supports_op, +}; + +ggml_backend_t ggml_backend_opencl_init() { + ggml_backend_t backend = new ggml_backend { + /* .interface = */ opencl_backend_i, + /* .context = */ nullptr + }; + + return backend; +} + +bool ggml_backend_is_opencl(ggml_backend_t backend) { + return backend && backend->iface.get_name == ggml_backend_opencl_name; +} +#endif diff --git a/ggml-opencl.h b/ggml-opencl.h index 44d05bd64..919b00d63 100644 --- a/ggml-opencl.h +++ b/ggml-opencl.h @@ -1,6 +1,7 @@ #pragma once #include "ggml.h" +#include "ggml-backend.h" #ifdef __cplusplus extern "C" { @@ -9,17 +10,26 @@ extern "C" { GGML_API void ggml_cl_init(void); GGML_API void ggml_cl_mul(const struct ggml_tensor * src0, const struct ggml_tensor * src1, struct ggml_tensor * dst); -GGML_API bool ggml_cl_can_mul_mat(const struct ggml_tensor * src0, const struct ggml_tensor * src1, struct ggml_tensor * dst); +GGML_API bool ggml_cl_can_mul_mat(const struct ggml_tensor * src0, const struct ggml_tensor * src1, const struct ggml_tensor * dst); GGML_API size_t ggml_cl_mul_mat_get_wsize(const struct ggml_tensor * src0, const struct ggml_tensor * src1, struct ggml_tensor * dst); GGML_API void ggml_cl_mul_mat(const struct ggml_tensor * src0, const struct ggml_tensor * src1, struct ggml_tensor * dst, void * wdata, size_t wsize); -GGML_API void * ggml_cl_host_malloc(size_t size); -GGML_API void ggml_cl_host_free(void * ptr); +// GGML_API void * ggml_cl_host_malloc(size_t size); +// GGML_API void ggml_cl_host_free(void * ptr); GGML_API void ggml_cl_free_data(const struct ggml_tensor* tensor); GGML_API void ggml_cl_transform_tensor(void * data, struct ggml_tensor * tensor); +// backend API + +// GGML_API ggml_backend_t ggml_backend_opencl_init(void); + +// GGML_API bool ggml_backend_is_opencl(ggml_backend_t backend); + +GGML_API ggml_backend_buffer_type_t ggml_backend_opencl_buffer_type(void); +// GGML_API ggml_backend_buffer_type_t ggml_backend_opencl_host_buffer_type(void); + #ifdef __cplusplus } #endif diff --git a/ggml.c b/ggml.c index f5caeba08..6dbd7626c 100644 --- a/ggml.c +++ b/ggml.c @@ -2354,6 +2354,10 @@ struct ggml_context * ggml_init(struct ggml_init_params params) { } void ggml_free(struct ggml_context * ctx) { + if (ctx == NULL) { + return; + } + // make this function thread safe ggml_critical_section_start(); @@ -4362,6 +4366,23 @@ struct ggml_tensor * ggml_cpy( return ggml_cpy_impl(ctx, a, b); } +struct ggml_tensor * ggml_cast( + struct ggml_context * ctx, + struct ggml_tensor * a, + enum ggml_type type) { + bool is_node = false; + + struct ggml_tensor * result = ggml_new_tensor(ctx, type, GGML_MAX_DIMS, a->ne); + ggml_format_name(result, "%s (copy)", a->name); + + result->op = GGML_OP_CPY; + result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL; + result->src[0] = a; + result->src[1] = result; + + return result; +} + // ggml_cont static struct ggml_tensor * ggml_cont_impl( @@ -14871,7 +14892,7 @@ size_t ggml_hash_find_or_insert(struct ggml_hash_set hash_set, struct ggml_tenso return i; } -static struct ggml_hash_set ggml_hash_set_new(size_t size) { +struct ggml_hash_set ggml_hash_set_new(size_t size) { size = ggml_hash_size(size); struct ggml_hash_set result; result.size = size; @@ -16620,7 +16641,7 @@ static thread_ret_t ggml_graph_compute_thread(void * data) { return GGML_EXIT_SUCCESS; } -struct ggml_cplan ggml_graph_plan(struct ggml_cgraph * cgraph, int n_threads) { +struct ggml_cplan ggml_graph_plan(const struct ggml_cgraph * cgraph, int n_threads) { if (n_threads <= 0) { n_threads = GGML_DEFAULT_N_THREADS; } @@ -16682,14 +16703,15 @@ struct ggml_cplan ggml_graph_plan(struct ggml_cgraph * cgraph, int n_threads) { } break; case GGML_OP_MUL_MAT_ID: { + cur = 0; const struct ggml_tensor * src0 = node->src[2]; const struct ggml_tensor * src1 = node->src[1]; const enum ggml_type vec_dot_type = type_traits[src0->type].vec_dot_type; if (src1->type != vec_dot_type) { - cur = ggml_row_size(vec_dot_type, ggml_nelements(src1)); + cur += ggml_row_size(vec_dot_type, ggml_nelements(src1)); } const int n_as = ggml_get_op_params_i32(node, 1); - cur = GGML_PAD(cur, sizeof(int64_t)); // align + cur += GGML_PAD(cur, sizeof(int64_t)); // align cur += n_as * sizeof(int64_t); // matrix_row_counts cur += n_as * src1->ne[1] * sizeof(int64_t); // matrix_rows } break; diff --git a/ggml.h b/ggml.h index 4c2ff6c66..b18ba7812 100644 --- a/ggml.h +++ b/ggml.h @@ -1165,6 +1165,11 @@ extern "C" { struct ggml_tensor * a, struct ggml_tensor * b); + GGML_API struct ggml_tensor * ggml_cast( + struct ggml_context * ctx, + struct ggml_tensor * a, + enum ggml_type type); + // make contiguous GGML_API struct ggml_tensor * ggml_cont( struct ggml_context * ctx, @@ -1842,8 +1847,8 @@ extern "C" { // ggml_graph_plan() has to be called before ggml_graph_compute() // when plan.work_size > 0, caller must allocate memory for plan.work_data - GGML_API struct ggml_cplan ggml_graph_plan (struct ggml_cgraph * cgraph, int n_threads /*= GGML_DEFAULT_N_THREADS*/); - GGML_API int ggml_graph_compute(struct ggml_cgraph * cgraph, struct ggml_cplan * cplan); + GGML_API struct ggml_cplan ggml_graph_plan (const struct ggml_cgraph * cgraph, int n_threads /*= GGML_DEFAULT_N_THREADS*/); + GGML_API int ggml_graph_compute( struct ggml_cgraph * cgraph, struct ggml_cplan * cplan); // same as ggml_graph_compute() but the work data is allocated as a part of the context // note: the drawback of this API is that you must have ensured that the context has enough memory for the work data diff --git a/llama.cpp b/llama.cpp index ce413f605..fe1d8947c 100644 --- a/llama.cpp +++ b/llama.cpp @@ -1,5 +1,4 @@ #define LLAMA_API_INTERNAL -//#define LLAMA_GGML_BACKEND_CUDA_TEST // for testing only - enables ggml-cuda through ggml-backend, disables partial offloading #include "llama.h" #include "unicode.h" @@ -152,10 +151,6 @@ static bool is_float_close(float a, float b, float abs_tol) { return std::fabs(b - a) <= abs_tol; } -#ifdef GGML_USE_CPU_HBM -#include -#endif - static void zeros(std::ofstream & file, size_t n) { char zero = 0; for (size_t i = 0; i < n; ++i) { @@ -1190,12 +1185,6 @@ struct llama_mlock { #endif }; -typedef void (*offload_func_t)(struct ggml_tensor * tensor); - -static void ggml_offload_nop(struct ggml_tensor * tensor) { - (void) tensor; -} - static std::string llama_token_to_piece(const struct llama_context * ctx, llama_token token) { std::vector result(8, 0); const int n_tokens = llama_token_to_piece(llama_get_model(ctx), token, result.data(), result.size()); @@ -1211,19 +1200,14 @@ static std::string llama_token_to_piece(const struct llama_context * ctx, llama_ return std::string(result.data(), result.size()); } -static ggml_backend_buffer_type_t llama_default_buffer_type(int n_gpu_layers) { +static ggml_backend_buffer_type_t llama_default_buffer_type_cpu(bool host_buffer) { ggml_backend_buffer_type_t buft = nullptr; -#ifdef GGML_USE_METAL - if (n_gpu_layers > 0) { - buft = ggml_backend_metal_buffer_type(); +#if defined(GGML_USE_CUBLAS) + // host buffers should only be used when data is expected to be copied to/from the GPU + if (host_buffer) { + buft = ggml_backend_cuda_host_buffer_type(); } -#elif defined(GGML_USE_CUBLAS) && defined(LLAMA_GGML_BACKEND_CUDA_TEST) - if (n_gpu_layers > 0) { - buft = ggml_backend_cuda_buffer_type(0); - } -#elif defined(GGML_USE_CUBLAS) - buft = ggml_backend_cuda_host_buffer_type(); #elif defined(GGML_USE_CPU_HBM) buft = ggml_backend_cpu_hbm_buffer_type(); #endif @@ -1231,10 +1215,45 @@ static ggml_backend_buffer_type_t llama_default_buffer_type(int n_gpu_layers) { if (buft == nullptr) { buft = ggml_backend_cpu_buffer_type(); } - return buft; - GGML_UNUSED(n_gpu_layers); + GGML_UNUSED(host_buffer); +} + +static ggml_backend_buffer_type_t llama_default_buffer_type_offload(int gpu) { + ggml_backend_buffer_type_t buft = nullptr; + +#ifdef GGML_USE_METAL + buft = ggml_backend_metal_buffer_type(); +#elif defined(GGML_USE_CUBLAS) + buft = ggml_backend_cuda_buffer_type(gpu); +#elif defined(GGML_USE_CLBLAST) + buft = ggml_backend_opencl_buffer_type(); +#endif + + if (buft == nullptr) { + buft = llama_default_buffer_type_cpu(true); + } + return buft; + + GGML_UNUSED(gpu); +} + +static ggml_backend_buffer_type_t llama_default_buffer_type_split(int fallback_gpu, const float * tensor_split) { + ggml_backend_buffer_type_t buft = nullptr; + +#ifdef GGML_USE_CUBLAS + if (ggml_backend_cuda_get_device_count() > 1) { + buft = ggml_backend_cuda_split_buffer_type(tensor_split); + } +#endif + + if (buft == nullptr) { + buft = llama_default_buffer_type_offload(fallback_gpu); + } + return buft; + + GGML_UNUSED(tensor_split); } // @@ -1445,24 +1464,24 @@ struct llama_kv_cache { std::vector k_l; // per layer std::vector v_l; - struct ggml_context * ctx = NULL; + std::vector ctxs; + std::vector bufs; - ggml_backend_buffer_t buf = NULL; + size_t total_size() const { + size_t size = 0; + for (ggml_backend_buffer_t buf : bufs) { + size += ggml_backend_buffer_get_size(buf); + } + return size; + } ~llama_kv_cache() { -#if defined(GGML_USE_CUBLAS) && !defined(LLAMA_GGML_BACKEND_CUDA_TEST) - if (ggml_cublas_loaded()) { - for (size_t i = 0; i < k_l.size(); ++i) { - ggml_cuda_free_data(k_l[i]); - ggml_cuda_free_data(v_l[i]); - } - } -#endif - if (ctx) { + for (struct ggml_context * ctx : ctxs) { ggml_free(ctx); } - - ggml_backend_buffer_free(buf); + for (ggml_backend_buffer_t buf : bufs) { + ggml_backend_buffer_free(buf); + } } }; @@ -1539,16 +1558,32 @@ struct llama_model { std::vector layers; + llama_split_mode split_mode; + int main_gpu; int n_gpu_layers; // gguf metadata std::unordered_map gguf_kv; - // context - struct ggml_context * ctx = NULL; + // layer -> buffer type mapping + struct layer_buft { + layer_buft() : buft_matrix(nullptr), buft(nullptr) {} + layer_buft(ggml_backend_buffer_type_t matrix) : buft_matrix(matrix), buft(matrix) {} + layer_buft(ggml_backend_buffer_type_t matrix, ggml_backend_buffer_type_t other) : buft_matrix(matrix), buft(other) {} - // the model memory buffer - ggml_backend_buffer_t buf = NULL; + ggml_backend_buffer_type_t buft_matrix; // matrices only - used by split buffers and backends that support only matrix multiplication + ggml_backend_buffer_type_t buft; // everything else + }; + + layer_buft buft_input; + layer_buft buft_output; + std::vector buft_layer; + + // contexts where the model tensors metadata is stored + std::vector ctxs; + + // the model memory buffers for the tensor data + std::vector bufs; // model memory mapped file std::unique_ptr mapping; @@ -1564,39 +1599,32 @@ struct llama_model { int64_t t_start_us = 0; ~llama_model() { -#if defined(GGML_USE_CUBLAS) && !defined(LLAMA_GGML_BACKEND_CUDA_TEST) - if (ggml_cublas_loaded()) { - for (size_t i = 0; i < tensors_by_name.size(); ++i) { - ggml_cuda_free_data(tensors_by_name[i].second); - } - ggml_cuda_free_scratch(); - } -#endif - -#if defined(GGML_USE_CLBLAST) - for (size_t i = 0; i < tensors_by_name.size(); ++i) { - ggml_cl_free_data(tensors_by_name[i].second); - } -#endif - if (ctx) { + for (struct ggml_context * ctx : ctxs) { ggml_free(ctx); } - - ggml_backend_buffer_free(buf); + for (ggml_backend_buffer_t buf : bufs) { + ggml_backend_buffer_free(buf); + } } }; struct llama_context { llama_context(const llama_model & model) : model(model), t_start_us(model.t_start_us), t_load_us(model.t_load_us) {} ~llama_context() { - ggml_allocr_free(alloc); - ggml_backend_buffer_free(buf_alloc); - ggml_backend_free(backend); + ggml_backend_sched_free(sched); + + for (ggml_backend_t backend : backends) { + ggml_backend_free(backend); + } } llama_cparams cparams; - ggml_backend_t backend = nullptr; + std::vector backends; +#ifdef GGML_USE_METAL + ggml_backend_t backend_metal = nullptr; +#endif + ggml_backend_t backend_cpu = nullptr; const llama_model & model; @@ -1630,8 +1658,9 @@ struct llama_context { // memory buffers used to evaluate the model std::vector buf_compute_meta; - ggml_backend_buffer_t buf_alloc = NULL; - ggml_allocr * alloc = NULL; + ggml_backend_sched_t sched = nullptr; + // allocator for the input tensors + ggml_tallocr * alloc = nullptr; // temporary buffer for copying data to/from the backend std::vector> buf_copy; @@ -1646,16 +1675,17 @@ struct llama_context { // static bool llama_kv_cache_init( - const struct llama_hparams & hparams, struct llama_kv_cache & cache, + const llama_model & model, ggml_type ktype, ggml_type vtype, uint32_t n_ctx, - int n_gpu_layers, bool offload) { + const struct llama_hparams & hparams = model.hparams; + const uint32_t n_embd_k_gqa = hparams.n_embd_k_gqa(); const uint32_t n_embd_v_gqa = hparams.n_embd_v_gqa(); - const uint32_t n_layer = hparams.n_layer; + const int64_t n_layer = hparams.n_layer; cache.has_shift = false; @@ -1666,62 +1696,65 @@ static bool llama_kv_cache_init( cache.cells.clear(); cache.cells.resize(n_ctx); - struct ggml_init_params params; - params.mem_size = 2u*n_layer*ggml_tensor_overhead(); - params.mem_buffer = NULL; - params.no_alloc = true; +#ifdef GGML_USE_CLBLAST + offload = false; +#endif - cache.ctx = ggml_init(params); + // count used buffer types + std::map buft_layer_count; + if (offload) { + for (int64_t i = 0; i < n_layer; ++i) { + buft_layer_count[model.buft_layer[i].buft]++; + } + } else { + buft_layer_count[llama_default_buffer_type_cpu(true)] = n_layer; + } - size_t vram_kv_cache = 0; - - if (!cache.ctx) { - LLAMA_LOG_ERROR("%s: failed to allocate memory for kv cache\n", __func__); - return false; + // create a context for each buffer type + std::map ctx_map; + for (auto & it : buft_layer_count) { + int n_layers = it.second; + struct ggml_init_params params = { + /*.mem_size =*/ 2u*n_layers*ggml_tensor_overhead(), + /*.mem_buffer =*/ NULL, + /*.no_alloc =*/ true, + }; + ggml_context * ctx = ggml_init(params); + if (!ctx) { + LLAMA_LOG_ERROR("%s: failed to allocate context for kv cache\n", __func__); + return false; + } + ctx_map[it.first] = ctx; + cache.ctxs.push_back(ctx); } cache.k_l.reserve(n_layer); cache.v_l.reserve(n_layer); - const int i_gpu_start = (int) n_layer - n_gpu_layers; - for (int i = 0; i < (int) n_layer; i++) { - ggml_tensor * k = ggml_new_tensor_1d(cache.ctx, ktype, n_embd_k_gqa*n_ctx); - ggml_tensor * v = ggml_new_tensor_1d(cache.ctx, vtype, n_embd_v_gqa*n_ctx); + struct ggml_context * ctx = offload ? ctx_map.at(model.buft_layer[i].buft) : cache.ctxs.front(); + ggml_tensor * k = ggml_new_tensor_1d(ctx, ktype, n_embd_k_gqa*n_ctx); + ggml_tensor * v = ggml_new_tensor_1d(ctx, vtype, n_embd_v_gqa*n_ctx); ggml_format_name(k, "cache_k_l%d", i); ggml_format_name(v, "cache_v_l%d", i); cache.k_l.push_back(k); cache.v_l.push_back(v); -#if defined(GGML_USE_CUBLAS) && !defined(LLAMA_GGML_BACKEND_CUDA_TEST) - if (i >= i_gpu_start) { - if (offload) { - ggml_cuda_assign_buffers_no_scratch(k); - ggml_cuda_assign_buffers_no_scratch(v); - vram_kv_cache += ggml_nbytes(k); - vram_kv_cache += ggml_nbytes(v); - // HACK: mark tensor as allocated - k->data = v->data = (void *)(uintptr_t)1; - } + } + + // allocate tensors and initialize the buffers to avoid NaNs in the padding + for (auto it : ctx_map) { + ggml_backend_buffer_type_t buft = it.first; + ggml_context * ctx = it.second; + ggml_backend_buffer_t buf = ggml_backend_alloc_ctx_tensors_from_buft(ctx, buft); + if (!buf) { + LLAMA_LOG_ERROR("%s: failed to allocate buffer for kv cache\n", __func__); + return false; } -#endif // GGML_USE_CUBLAS + ggml_backend_buffer_clear(buf, 0); + LLAMA_LOG_INFO("%s: %10s KV buffer size = %8.2f MiB\n", __func__, ggml_backend_buffer_name(buf), ggml_backend_buffer_get_size(buf)/1024.0/1024.0); + cache.bufs.push_back(buf); } - // allocate tensors - cache.buf = ggml_backend_alloc_ctx_tensors_from_buft(cache.ctx, llama_default_buffer_type(n_gpu_layers)); - - // buf may be NULL with full offload - if (cache.buf) { - // initialize the buffer to avoid NaNs in the padding - ggml_backend_buffer_clear(cache.buf, 0); - } - - if (vram_kv_cache > 0) { - LLAMA_LOG_INFO("%s: VRAM kv self = %.2f MB\n", __func__, vram_kv_cache / 1024.0 / 1024.0); - } - - GGML_UNUSED(i_gpu_start); - GGML_UNUSED(offload); - return true; } @@ -2354,9 +2387,8 @@ struct llama_model_loader { return get_tensor_meta(get_tensor_name(i)); } - struct ggml_tensor * create_tensor_for(struct ggml_context * ctx, struct ggml_tensor * meta, ggml_backend_type backend) { + struct ggml_tensor * create_tensor_for(struct ggml_context * ctx, struct ggml_tensor * meta) { struct ggml_tensor * tensor = ggml_dup_tensor(ctx, meta); - tensor->backend = backend; // TODO: ggml_set_backend ggml_set_name(tensor, ggml_get_name(meta)); n_created++; @@ -2364,7 +2396,7 @@ struct llama_model_loader { return tensor; } - struct ggml_tensor * create_tensor(struct ggml_context * ctx, const std::string & name, const std::vector & ne, ggml_backend_type backend, bool required = true) { + struct ggml_tensor * create_tensor(struct ggml_context * ctx, const std::string & name, const std::vector & ne, bool required = true) { struct ggml_tensor * cur = ggml_get_tensor(ctx_meta, name.c_str()); if (cur == NULL) { @@ -2374,12 +2406,6 @@ struct llama_model_loader { throw std::runtime_error(format("%s: tensor '%s' not found", __func__, name.c_str())); } - if (backend == GGML_BACKEND_GPU_SPLIT) { - if (ne.size() == 1) { - throw std::runtime_error(format("%s: 1-dimensional tensor '%s' cannot be split on the GPU", __func__, name.c_str())); - } - } - { bool is_ok = true; for (size_t i = 0; i < ne.size(); ++i) { @@ -2397,7 +2423,7 @@ struct llama_model_loader { } } - return create_tensor_for(ctx, cur, backend); + return create_tensor_for(ctx, cur); } void done_getting_tensors() const { @@ -2416,26 +2442,36 @@ struct llama_model_loader { return gguf_get_data_offset(ctx_gguf) + gguf_get_tensor_offset(ctx_gguf, idx); } - void init_mapping(bool prefetch = true) { - /* - // prefetch only CPU tensors - if (use_mmap) { - size_t size_pref = 0; // prefetch - - for (int i = 0; i < gguf_get_n_tensors(ctx_gguf); i++) { - struct ggml_tensor * cur = ggml_get_tensor(ctx, gguf_get_tensor_name(ctx_gguf, i)); - if (cur->backend == GGML_BACKEND_CPU) { - size_t tensor_end = gguf_get_tensor_offset(ctx_gguf, i) + ggml_nbytes(cur); - size_pref = std::max(size_pref, tensor_end); - } - } - mapping.reset(new llama_mmap(&file, gguf_get_data_offset(ctx_gguf) + size_pref, ggml_is_numa())); - } - */ + void init_mapping(bool prefetch = true, llama_mlock * lmlock = nullptr) { // prefetch the whole file - all the data is needed anyway if (use_mmap) { mapping.reset(new llama_mmap(&file, prefetch ? -1 : 0, ggml_is_numa())); } + + // compute the total size of all tensors for progress reporting + for (int i = 0; i < gguf_get_n_tensors(ctx_gguf); i++) { + struct ggml_tensor * cur = ggml_get_tensor(ctx_meta, gguf_get_tensor_name(ctx_gguf, i)); + size_data += ggml_nbytes(cur); + } + + if (use_mmap && mapping) { + if (lmlock) { + lmlock->init(mapping->addr); + } + mmap_used_first = mapping->size; + } + } + + void get_mapping_range(size_t * first, size_t * last, ggml_context * ctx) const { + GGML_ASSERT(mapping); + + *first = mapping->size; + *last = 0; + for (ggml_tensor * tensor = ggml_get_first_tensor(ctx); tensor; tensor = ggml_get_next_tensor(ctx, tensor)) { + const size_t offs = file_offset(ggml_get_name(tensor)); + *first = std::min(*first, offs); + *last = std::max(*last, offs + ggml_nbytes(tensor)); + } } // for backwards compatibility, does not support ggml-backend @@ -2443,8 +2479,11 @@ struct llama_model_loader { const size_t offs = file_offset(ggml_get_name(cur)); if (use_mmap && mapping) { - GGML_ASSERT(cur->data == nullptr); - cur->data = (uint8_t *)mapping->addr + offs; + if (cur->data == nullptr) { + cur->data = (uint8_t *)mapping->addr + offs; + } else { + memcpy(cur->data, (uint8_t *)mapping->addr + offs, ggml_nbytes(cur)); + } } else { GGML_ASSERT(cur->data != nullptr); file.seek(offs, SEEK_SET); @@ -2452,37 +2491,23 @@ struct llama_model_loader { } } + size_t size_done = 0; + size_t size_data = 0; + size_t mmap_used_first = -1; + size_t mmap_used_last = 0; + // Returns false if cancelled by progress_callback - bool load_all_data(struct ggml_context * ctx, llama_progress_callback progress_callback, void * progress_callback_user_data, ggml_backend_buffer_t buf_mmap, llama_mlock * lmlock) const { - size_t size_data = 0; - - for (int i = 0; i < gguf_get_n_tensors(ctx_gguf); i++) { - struct ggml_tensor * cur = ggml_get_tensor(ctx, gguf_get_tensor_name(ctx_gguf, i)); - size_data += ggml_nbytes(cur); - } - - if (use_mmap && buf_mmap) { - if (lmlock) { - lmlock->init(mapping->addr); - } - } - -#if (defined(GGML_USE_CUBLAS) && !defined(LLAMA_GGML_BACKEND_CUDA_TEST)) || defined(GGML_USE_CLBLAST) - const bool legacy_offload = true; -#else - const bool legacy_offload = false; -#endif + bool load_all_data(struct ggml_context * ctx, llama_progress_callback progress_callback, void * progress_callback_user_data, ggml_backend_buffer_t buf_mmap, llama_mlock * lmlock) { + GGML_ASSERT(size_data != 0 && "call init_mapping() first"); std::vector> read_buf; - size_t size_done = 0; - - size_t mmap_first = -1; - size_t mmap_last = 0; - for (int i = 0; i < gguf_get_n_tensors(ctx_gguf); i++) { struct ggml_tensor * cur = ggml_get_tensor(ctx, gguf_get_tensor_name(ctx_gguf, i)); - GGML_ASSERT(cur); // unused tensors should have been caught by load_data already + if (!cur) { + // some tensors may be allocated in a different context + continue; + } if (progress_callback) { if (!progress_callback((float) size_done / size_data, progress_callback_user_data)) { @@ -2492,67 +2517,48 @@ struct llama_model_loader { const size_t offs = file_offset(ggml_get_name(cur)); - if (!legacy_offload || cur->backend == GGML_BACKEND_CPU) { - if (use_mmap && mapping) { - if (buf_mmap) { - ggml_backend_tensor_alloc(buf_mmap, cur, (uint8_t *) mapping->addr + offs); - if (lmlock) { - lmlock->grow_to(offs + ggml_nbytes(cur)); - } - mmap_first = std::min(mmap_first, offs); - mmap_last = std::max(mmap_last, offs + ggml_nbytes(cur)); - } else { - ggml_backend_tensor_set(cur, (uint8_t *) mapping->addr + offs, 0, ggml_nbytes(cur)); + if (use_mmap && mapping) { + if (buf_mmap && cur->data == nullptr) { + ggml_backend_tensor_alloc(buf_mmap, cur, (uint8_t *) mapping->addr + offs); + if (lmlock) { + lmlock->grow_to(offs + ggml_nbytes(cur)); } + mmap_used_first = std::min(mmap_used_first, offs); + mmap_used_last = std::max(mmap_used_last, offs + ggml_nbytes(cur)); } else { - if (ggml_backend_buffer_is_host(cur->buffer)) { - file.seek(offs, SEEK_SET); - file.read_raw(cur->data, ggml_nbytes(cur)); - } else { - read_buf.resize(ggml_nbytes(cur)); - file.seek(offs, SEEK_SET); - file.read_raw(read_buf.data(), ggml_nbytes(cur)); - ggml_backend_tensor_set(cur, read_buf.data(), 0, ggml_nbytes(cur)); - } + ggml_backend_tensor_set(cur, (uint8_t *) mapping->addr + offs, 0, ggml_nbytes(cur)); } } else { - // HACK: mark tensor as allocated - cur->data = (void *)(uintptr_t)1; - void * data; - if (use_mmap && mapping) { - data = (uint8_t *) mapping->addr + offs; + if (ggml_backend_buffer_is_host(cur->buffer)) { + file.seek(offs, SEEK_SET); + file.read_raw(cur->data, ggml_nbytes(cur)); } else { read_buf.resize(ggml_nbytes(cur)); file.seek(offs, SEEK_SET); file.read_raw(read_buf.data(), ggml_nbytes(cur)); - data = read_buf.data(); + ggml_backend_tensor_set(cur, read_buf.data(), 0, ggml_nbytes(cur)); } - -#if defined(GGML_USE_CUBLAS) && !defined(LLAMA_GGML_BACKEND_CUDA_TEST) - ggml_cuda_transform_tensor(data, cur); -#elif defined(GGML_USE_CLBLAST) - GGML_ASSERT(cur->backend == GGML_BACKEND_GPU); - ggml_cl_transform_tensor(data, cur); -#else - GGML_ASSERT(!"GPU tensor without a GPU backend"); - GGML_UNUSED(data); -#endif } size_done += ggml_nbytes(cur); } - // unmap offloaded tensors and metadata - if (use_mmap && mapping) { - mapping->unmap_fragment(0, mmap_first); - mapping->unmap_fragment(mmap_last, mapping->size); + // check if this is the last call and do final cleanup + if (size_done >= size_data) { + // unmap offloaded tensors and metadata + if (use_mmap && mapping) { + mapping->unmap_fragment(0, mmap_used_first); + if (mmap_used_last != 0) { + mapping->unmap_fragment(mmap_used_last, mapping->size); + } + } + if (progress_callback) { + // Even though the model is done loading, we still honor + // cancellation since we need to free allocations. + return progress_callback(1.0f, progress_callback_user_data); + } } - if (progress_callback) { - // Even though the model is done loading, we still honor - // cancellation since we need to free allocations. - return progress_callback(1.0f, progress_callback_user_data); - } return true; } }; @@ -3181,6 +3187,7 @@ static bool llm_load_tensors( llama_model_loader & ml, llama_model & model, int n_gpu_layers, + enum llama_split_mode split_mode, int main_gpu, const float * tensor_split, bool use_mlock, @@ -3188,702 +3195,563 @@ static bool llm_load_tensors( void * progress_callback_user_data) { model.t_start_us = ggml_time_us(); - auto & ctx = model.ctx; auto & hparams = model.hparams; + model.split_mode = split_mode; + model.main_gpu = main_gpu; model.n_gpu_layers = n_gpu_layers; - size_t ctx_size = ggml_tensor_overhead() * ml.n_tensors; + const int64_t n_layer = hparams.n_layer; + const int64_t i_gpu_start = std::max((int64_t) hparams.n_layer - n_gpu_layers, (int64_t) 0); - LLAMA_LOG_INFO("%s: ggml ctx size = %7.2f MiB\n", __func__, ctx_size/1024.0/1024.0); + // there is very little benefit to offloading the input layer, so always keep it on the CPU + model.buft_input = llama_default_buffer_type_cpu(true); - // create the ggml context + model.buft_layer.resize(n_layer); + + // assign cpu layers + for (int64_t i = 0; i < i_gpu_start; ++i) { + model.buft_layer[i] = llama_default_buffer_type_cpu(true); + } + +#ifdef GGML_USE_CUBLAS + if (split_mode == LLAMA_SPLIT_LAYER) { + // calculate the split points + int device_count = ggml_backend_cuda_get_device_count(); + bool all_zero = tensor_split == nullptr || std::all_of(tensor_split, tensor_split + device_count, [](float x) { return x == 0.0f; }); + float splits[GGML_CUDA_MAX_DEVICES]; + if (all_zero) { + // default split, by free memory + for (int i = 0; i < device_count; ++i) { + size_t total; + size_t free; + ggml_backend_cuda_get_device_memory(i, &total, &free); + splits[i] = free; + } + } else { + std::copy(tensor_split, tensor_split + device_count, splits); + } + + // sum and normalize the splits to get the split points + float split_sum = 0.0f; + for (int i = 0; i < device_count; ++i) { + split_sum += splits[i]; + splits[i] = split_sum; + } + for (int i = 0; i < device_count; ++i) { + splits[i] /= split_sum; + } + + // assign the repeating layers to the devices according to the splits + int act_gpu_layers = std::min(n_gpu_layers, (int)n_layer + 1); + for (int64_t i = i_gpu_start; i < n_layer; ++i) { + int layer_gpu = std::upper_bound(splits, splits + device_count, float(i - i_gpu_start)/act_gpu_layers) - splits; + model.buft_layer[i] = llama_default_buffer_type_offload(layer_gpu); + } + // assign the output layer + if (n_gpu_layers > n_layer) { + int layer_gpu = std::upper_bound(splits, splits + device_count, float(act_gpu_layers - 1)/act_gpu_layers) - splits; + model.buft_output = llama_default_buffer_type_offload(layer_gpu); + } else { + model.buft_output = llama_default_buffer_type_cpu(true); + } + } else +#endif { + ggml_backend_buffer_type_t split_buft; + if (split_mode == LLAMA_SPLIT_ROW) { + split_buft = llama_default_buffer_type_split(main_gpu, tensor_split); + } else { + // LLAMA_SPLIT_NONE or LLAMA_SPLIT_LAYER in backends where it is not supported + split_buft = llama_default_buffer_type_offload(main_gpu); + } + // assign the repeating layers + for (int64_t i = i_gpu_start; i < n_layer; ++i) { + model.buft_layer[i] = { + split_buft, + llama_default_buffer_type_offload(main_gpu) + }; + } + // assign the output layer + if (n_gpu_layers > n_layer) { + model.buft_output = { + split_buft, + llama_default_buffer_type_offload(main_gpu) + }; + } else { + model.buft_output = llama_default_buffer_type_cpu(true); + } + } + + // count used buffer types + std::map buft_layer_count; + buft_layer_count[model.buft_input.buft]++; + buft_layer_count[model.buft_input.buft_matrix]++; + buft_layer_count[model.buft_output.buft]++; + buft_layer_count[model.buft_output.buft_matrix]++; + for (int64_t i = 0; i < n_layer; ++i) { + buft_layer_count[model.buft_layer[i].buft]++; + buft_layer_count[model.buft_layer[i].buft_matrix]++; + } + + // create one context per buffer type + size_t ctx_size = ggml_tensor_overhead()*ml.n_tensors; + std::map ctx_map; + for (auto & it : buft_layer_count) { struct ggml_init_params params = { /*.mem_size =*/ ctx_size, /*.mem_buffer =*/ NULL, /*.no_alloc =*/ true, }; - - model.ctx = ggml_init(params); - if (!model.ctx) { - throw std::runtime_error(format("ggml_init() failed")); + ggml_context * ctx = ggml_init(params); + if (!ctx) { + throw std::runtime_error(format("failed to create context")); } + ctx_map[it.first] = ctx; + model.ctxs.push_back(ctx); } - (void) main_gpu; - - enum ggml_backend_type llama_backend_offload = GGML_BACKEND_CPU; - enum ggml_backend_type llama_backend_offload_split = GGML_BACKEND_CPU; - -#if defined(GGML_USE_CUBLAS) && !defined(LLAMA_GGML_BACKEND_CUDA_TEST) - if (ggml_cublas_loaded()) { - LLAMA_LOG_INFO("%s: using " GGML_CUDA_NAME " for GPU acceleration\n", __func__); - ggml_cuda_set_main_device(main_gpu); - - llama_backend_offload = GGML_BACKEND_GPU; - llama_backend_offload_split = GGML_BACKEND_GPU_SPLIT; - } -#elif defined(GGML_USE_CLBLAST) - LLAMA_LOG_INFO("%s: using OpenCL for GPU acceleration\n", __func__); - llama_backend_offload = GGML_BACKEND_GPU; - llama_backend_offload_split = GGML_BACKEND_GPU; -#endif + LLAMA_LOG_INFO("%s: ggml ctx size = %7.2f MiB\n", __func__, model.ctxs.size()*ctx_size/1024.0/1024.0); // create tensors for the weights { const int64_t n_embd = hparams.n_embd; const int64_t n_embd_k_gqa = hparams.n_embd_k_gqa(); const int64_t n_embd_v_gqa = hparams.n_embd_v_gqa(); - const int64_t n_layer = hparams.n_layer; + const int64_t n_embd_gqa = n_embd_v_gqa; const int64_t n_vocab = hparams.n_vocab; + const int64_t n_ff = hparams.n_ff; + + GGML_ASSERT(n_embd_gqa == n_embd_k_gqa); + + ggml_context * ctx_input = ctx_map.at(model.buft_input.buft); + ggml_context * ctx_output = ctx_map.at(model.buft_output.buft); + ggml_context * ctx_output_split = ctx_map.at(model.buft_output.buft_matrix); + auto ctx_for_layer = [&](int i) { return ctx_map.at(model.buft_layer[i].buft); }; + auto ctx_for_layer_split = [&](int i) { return ctx_map.at(model.buft_layer[i].buft_matrix); }; + + model.layers.resize(n_layer); const auto tn = LLM_TN(model.arch); switch (model.arch) { case LLM_ARCH_LLAMA: case LLM_ARCH_REFACT: { - model.tok_embd = ml.create_tensor(ctx, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, GGML_BACKEND_CPU); + model.tok_embd = ml.create_tensor(ctx_input, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}); // output { - ggml_backend_type backend_norm; - ggml_backend_type backend_output; - - if (n_gpu_layers > int(n_layer)) { - backend_norm = llama_backend_offload; - backend_output = llama_backend_offload_split; - } else { - backend_norm = GGML_BACKEND_CPU; - backend_output = GGML_BACKEND_CPU; - } - - model.output_norm = ml.create_tensor(ctx, tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, backend_norm); - model.output = ml.create_tensor(ctx, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, backend_output); + model.output_norm = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}); + model.output = ml.create_tensor(ctx_output_split, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}); } - const uint32_t n_ff = hparams.n_ff; - const int64_t n_embd_gqa = n_embd_v_gqa; - GGML_ASSERT(n_embd_gqa == n_embd / hparams.n_gqa()); - GGML_ASSERT(n_embd_gqa == n_embd_k_gqa); - - const int i_gpu_start = n_layer - n_gpu_layers; - - model.layers.resize(n_layer); - - for (uint32_t i = 0; i < n_layer; ++i) { - const ggml_backend_type backend = int(i) < i_gpu_start ? GGML_BACKEND_CPU : llama_backend_offload; // NOLINT - const ggml_backend_type backend_split = int(i) < i_gpu_start ? GGML_BACKEND_CPU : llama_backend_offload_split; // NOLINT + for (int i = 0; i < n_layer; ++i) { + ggml_context * ctx_layer = ctx_for_layer(i); + ggml_context * ctx_split = ctx_for_layer_split(i); auto & layer = model.layers[i]; - layer.attn_norm = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, backend); + layer.attn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}); - layer.wq = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_embd}, backend_split); - layer.wk = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_K, "weight", i), {n_embd, n_embd_gqa}, backend_split); - layer.wv = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_V, "weight", i), {n_embd, n_embd_gqa}, backend_split); - layer.wo = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd}, backend_split); + layer.wq = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_embd}); + layer.wk = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_K, "weight", i), {n_embd, n_embd_gqa}); + layer.wv = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_V, "weight", i), {n_embd, n_embd_gqa}); + layer.wo = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd}); // optional bias tensors - layer.bq = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_Q, "bias", i), {n_embd}, backend, false); - layer.bk = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_K, "bias", i), {n_embd_gqa}, backend, false); - layer.bv = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_V, "bias", i), {n_embd_gqa}, backend, false); - layer.bo = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_OUT, "bias", i), {n_embd}, backend, false); + layer.bq = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_Q, "bias", i), {n_embd}, false); + layer.bk = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_K, "bias", i), {n_embd_gqa}, false); + layer.bv = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_V, "bias", i), {n_embd_gqa}, false); + layer.bo = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_OUT, "bias", i), {n_embd}, false); - layer.ffn_norm = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, backend); + layer.ffn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}); - layer.ffn_gate_inp = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_GATE_INP, "weight", i), {n_embd}, backend, false); + layer.ffn_gate_inp = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_GATE_INP, "weight", i), {n_embd}, false); if (layer.ffn_gate_inp == nullptr) { GGML_ASSERT(hparams.n_expert == 0); GGML_ASSERT(hparams.n_expert_used == 0); - layer.ffn_gate = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff}, backend_split); - layer.ffn_down = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_DOWN, "weight", i), { n_ff, n_embd}, backend_split); - layer.ffn_up = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff}, backend_split); + layer.ffn_gate = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff}); + layer.ffn_down = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_DOWN, "weight", i), { n_ff, n_embd}); + layer.ffn_up = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff}); } else { GGML_ASSERT(hparams.n_expert > 0); GGML_ASSERT(hparams.n_expert_used > 0); // MoE branch for (uint32_t x = 0; x < hparams.n_expert; ++x) { - layer.ffn_gate_exp[x] = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_GATE_EXP, "weight", i, x), {n_embd, n_ff}, backend_split); - layer.ffn_down_exp[x] = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_DOWN_EXP, "weight", i, x), { n_ff, n_embd}, backend_split); - layer.ffn_up_exp[x] = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_UP_EXP, "weight", i, x), {n_embd, n_ff}, backend_split); + layer.ffn_gate_exp[x] = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_GATE_EXP, "weight", i, x), {n_embd, n_ff}); + layer.ffn_down_exp[x] = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_DOWN_EXP, "weight", i, x), { n_ff, n_embd}); + layer.ffn_up_exp[x] = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_UP_EXP, "weight", i, x), {n_embd, n_ff}); } } } } break; case LLM_ARCH_BAICHUAN: { - model.tok_embd = ml.create_tensor(ctx, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, GGML_BACKEND_CPU); + model.tok_embd = ml.create_tensor(ctx_input, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}); { - ggml_backend_type backend_norm; - ggml_backend_type backend_output; - - if (n_gpu_layers > int(n_layer)) { - backend_norm = llama_backend_offload; - backend_output = llama_backend_offload_split; - } else { - backend_norm = GGML_BACKEND_CPU; - backend_output = GGML_BACKEND_CPU; - } - - model.output_norm = ml.create_tensor(ctx, tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, backend_norm); - model.output = ml.create_tensor(ctx, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, backend_output); + model.output_norm = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}); + model.output = ml.create_tensor(ctx_output_split, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}); } - const uint32_t n_ff = hparams.n_ff; - const int64_t n_embd_gqa = n_embd_v_gqa; - GGML_ASSERT(n_embd_gqa == n_embd / hparams.n_gqa()); - GGML_ASSERT(n_embd_gqa == n_embd_k_gqa); - - const int i_gpu_start = n_layer - n_gpu_layers; - - model.layers.resize(n_layer); - - for (uint32_t i = 0; i < n_layer; ++i) { - const ggml_backend_type backend = int(i) < i_gpu_start ? GGML_BACKEND_CPU : llama_backend_offload; // NOLINT - const ggml_backend_type backend_split = int(i) < i_gpu_start ? GGML_BACKEND_CPU : llama_backend_offload_split; // NOLINT + for (int i = 0; i < n_layer; ++i) { + ggml_context * ctx_layer = ctx_for_layer(i); + ggml_context * ctx_split = ctx_for_layer_split(i); auto & layer = model.layers[i]; - layer.attn_norm = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, backend); + layer.attn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}); - layer.wq = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_embd}, backend_split); - layer.wk = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_K, "weight", i), {n_embd, n_embd_gqa}, backend_split); - layer.wv = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_V, "weight", i), {n_embd, n_embd_gqa}, backend_split); - layer.wo = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd}, backend_split); + layer.wq = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_embd}); + layer.wk = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_K, "weight", i), {n_embd, n_embd_gqa}); + layer.wv = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_V, "weight", i), {n_embd, n_embd_gqa}); + layer.wo = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd}); - layer.ffn_norm = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, backend); + layer.ffn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}); - layer.ffn_gate = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff}, backend_split); - layer.ffn_down = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_DOWN, "weight", i), { n_ff, n_embd}, backend_split); - layer.ffn_up = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff}, backend_split); + layer.ffn_gate = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff}); + layer.ffn_down = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_DOWN, "weight", i), { n_ff, n_embd}); + layer.ffn_up = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff}); } } break; case LLM_ARCH_FALCON: { - model.tok_embd = ml.create_tensor(ctx, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, GGML_BACKEND_CPU); + model.tok_embd = ml.create_tensor(ctx_input, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}); // output { - ggml_backend_type backend_norm; - ggml_backend_type backend_output; - - if (n_gpu_layers > int(n_layer)) { - backend_norm = llama_backend_offload; - backend_output = llama_backend_offload_split; - } else { - backend_norm = GGML_BACKEND_CPU; - backend_output = GGML_BACKEND_CPU; - } - - model.output_norm = ml.create_tensor(ctx, tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, backend_norm); - model.output_norm_b = ml.create_tensor(ctx, tn(LLM_TENSOR_OUTPUT_NORM, "bias"), {n_embd}, backend_norm); - model.output = ml.create_tensor(ctx, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, backend_output); + model.output_norm = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}); + model.output_norm_b = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "bias"), {n_embd}); + model.output = ml.create_tensor(ctx_output_split, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}); } - const uint32_t n_ff = hparams.n_ff; - const int64_t n_embd_gqa = n_embd_v_gqa; - GGML_ASSERT(n_embd_gqa == n_embd / hparams.n_gqa()); - GGML_ASSERT(n_embd_gqa == n_embd_k_gqa); - - const int i_gpu_start = n_layer - n_gpu_layers; - - model.layers.resize(n_layer); - - for (uint32_t i = 0; i < n_layer; ++i) { - const ggml_backend_type backend = int(i) < i_gpu_start ? GGML_BACKEND_CPU : llama_backend_offload; // NOLINT - const ggml_backend_type backend_split = int(i) < i_gpu_start ? GGML_BACKEND_CPU : llama_backend_offload_split; // NOLINT + for (int i = 0; i < n_layer; ++i) { + ggml_context * ctx_layer = ctx_for_layer(i); + ggml_context * ctx_split = ctx_for_layer_split(i); auto & layer = model.layers[i]; - layer.attn_norm = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, backend); - layer.attn_norm_b = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_NORM, "bias", i), {n_embd}, backend); + layer.attn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}); + layer.attn_norm_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM, "bias", i), {n_embd}); if (gguf_find_tensor(ml.ctx_gguf, tn(LLM_TENSOR_ATTN_NORM_2, "weight", i).c_str()) >= 0) { - layer.attn_norm_2 = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_NORM_2, "weight", i), {n_embd}, backend); - layer.attn_norm_2_b = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_NORM_2, "bias", i), {n_embd}, backend); + layer.attn_norm_2 = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM_2, "weight", i), {n_embd}); + layer.attn_norm_2_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM_2, "bias", i), {n_embd}); } - layer.wqkv = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_QKV, "weight", i), {n_embd, n_embd + 2*n_embd_gqa}, backend_split); - layer.wo = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd}, backend_split); + layer.wqkv = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_QKV, "weight", i), {n_embd, n_embd + 2*n_embd_gqa}); + layer.wo = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd}); - layer.ffn_down = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_DOWN, "weight", i), { n_ff, n_embd}, backend_split); - layer.ffn_up = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff}, backend_split); + layer.ffn_down = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_DOWN, "weight", i), { n_ff, n_embd}); + layer.ffn_up = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff}); } } break; case LLM_ARCH_STARCODER: { - model.tok_embd = ml.create_tensor(ctx, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, GGML_BACKEND_CPU); - model.pos_embd = ml.create_tensor(ctx, tn(LLM_TENSOR_POS_EMBD, "weight"), {n_embd, hparams.n_ctx_train}, GGML_BACKEND_CPU); + model.tok_embd = ml.create_tensor(ctx_input, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}); + model.pos_embd = ml.create_tensor(ctx_input, tn(LLM_TENSOR_POS_EMBD, "weight"), {n_embd, hparams.n_ctx_train}); // output { - ggml_backend_type backend_norm; - ggml_backend_type backend_output; - - if (n_gpu_layers > int(n_layer)) { - backend_norm = llama_backend_offload; - backend_output = llama_backend_offload_split; - } else { - backend_norm = GGML_BACKEND_CPU; - backend_output = GGML_BACKEND_CPU; - } - - model.output_norm = ml.create_tensor(ctx, tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, backend_norm); - model.output_norm_b = ml.create_tensor(ctx, tn(LLM_TENSOR_OUTPUT_NORM, "bias"), {n_embd}, backend_norm); - model.output = ml.create_tensor(ctx, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, backend_output); + model.output_norm = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}); + model.output_norm_b = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "bias"), {n_embd}); + model.output = ml.create_tensor(ctx_output_split, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}); } - const uint32_t n_ff = hparams.n_ff; - const int64_t n_embd_gqa = n_embd_v_gqa; - GGML_ASSERT(n_embd_gqa == n_embd / hparams.n_gqa()); - GGML_ASSERT(n_embd_gqa == n_embd_k_gqa); - - const int i_gpu_start = n_layer - n_gpu_layers; - - model.layers.resize(n_layer); - - for (uint32_t i = 0; i < n_layer; ++i) { - const ggml_backend_type backend = int(i) < i_gpu_start ? GGML_BACKEND_CPU : llama_backend_offload; // NOLINT - const ggml_backend_type backend_split = int(i) < i_gpu_start ? GGML_BACKEND_CPU : llama_backend_offload_split; // NOLINT + for (int i = 0; i < n_layer; ++i) { + ggml_context * ctx_layer = ctx_for_layer(i); + ggml_context * ctx_split = ctx_for_layer_split(i); auto & layer = model.layers[i]; - layer.attn_norm = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, backend); - layer.attn_norm_b = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_NORM, "bias", i), {n_embd}, backend); + layer.attn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}); + layer.attn_norm_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM, "bias", i), {n_embd}); - layer.wqkv = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_QKV, "weight", i), {n_embd, n_embd + 2*n_embd_gqa}, backend_split); - layer.bqkv = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_QKV, "bias", i), {n_embd + 2*n_embd_gqa}, backend); + layer.wqkv = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_QKV, "weight", i), {n_embd, n_embd + 2*n_embd_gqa}); + layer.bqkv = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_QKV, "bias", i), {n_embd + 2*n_embd_gqa}); - layer.wo = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd}, backend_split); - layer.bo = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_OUT, "bias", i), {n_embd}, backend); + layer.wo = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd}); + layer.bo = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_OUT, "bias", i), {n_embd}); - layer.ffn_norm = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, backend); - layer.ffn_norm_b = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_NORM, "bias", i), {n_embd}, backend); + layer.ffn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}); + layer.ffn_norm_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_NORM, "bias", i), {n_embd}); - layer.ffn_down = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_DOWN, "weight", i), {n_ff, n_embd}, backend_split); - layer.ffn_down_b = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_DOWN, "bias", i), {n_embd}, backend); + layer.ffn_down = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_DOWN, "weight", i), {n_ff, n_embd}); + layer.ffn_down_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_DOWN, "bias", i), {n_embd}); - layer.ffn_up = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff}, backend_split); - layer.ffn_up_b = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_UP, "bias", i), {n_ff}, backend); + layer.ffn_up = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff}); + layer.ffn_up_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_UP, "bias", i), {n_ff}); } } break; case LLM_ARCH_PERSIMMON: { - model.tok_embd = ml.create_tensor(ctx, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, GGML_BACKEND_CPU); + model.tok_embd = ml.create_tensor(ctx_input, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}); { - ggml_backend_type backend_norm; - ggml_backend_type backend_output; - - if (n_gpu_layers > int(n_layer)) { - backend_norm = llama_backend_offload; - backend_output = llama_backend_offload_split; - } else { - backend_norm = GGML_BACKEND_CPU; - backend_output = GGML_BACKEND_CPU; - } - - model.output_norm = ml.create_tensor(ctx, tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, backend_norm); - model.output_norm_b = ml.create_tensor(ctx, tn(LLM_TENSOR_OUTPUT_NORM, "bias"), {n_embd}, backend_norm); - model.output = ml.create_tensor(ctx, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, backend_output); + model.output_norm = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}); + model.output_norm_b = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "bias"), {n_embd}); + model.output = ml.create_tensor(ctx_output_split, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}); } - const uint32_t n_ff = hparams.n_ff; - const int64_t n_embd_gqa = n_embd_v_gqa; - GGML_ASSERT(n_embd_gqa == n_embd / hparams.n_gqa()); - GGML_ASSERT(n_embd_gqa == n_embd_k_gqa); + for (int i = 0; i < n_layer; ++i) { + ggml_context * ctx_layer = ctx_for_layer(i); + ggml_context * ctx_split = ctx_for_layer_split(i); - const int i_gpu_start = n_layer - n_gpu_layers; - model.layers.resize(n_layer); - for (uint32_t i = 0; i < n_layer; ++i) { - const ggml_backend_type backend = int(i) < i_gpu_start ? GGML_BACKEND_CPU : llama_backend_offload; - const ggml_backend_type backend_split = int(i) < i_gpu_start ? GGML_BACKEND_CPU : llama_backend_offload_split; auto & layer = model.layers[i]; - layer.attn_norm = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, backend); - layer.attn_norm_b = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_NORM, "bias", i), {n_embd}, backend); - layer.wqkv = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_QKV, "weight", i), {n_embd, n_embd + 2*n_embd_gqa}, backend_split); - layer.bqkv = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_QKV, "bias", i), {n_embd + 2*n_embd_gqa}, backend); - layer.wo = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd}, backend_split); - layer.bo = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_OUT, "bias", i), {n_embd}, backend); - layer.ffn_down = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_DOWN, "weight", i), {n_ff, n_embd}, backend_split); - layer.ffn_down_b = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_DOWN, "bias", i), {n_embd}, backend); - layer.ffn_up = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff}, backend_split); - layer.ffn_up_b = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_UP, "bias", i), {n_ff}, backend); - layer.ffn_norm = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, backend); - layer.ffn_norm_b = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_NORM, "bias", i), {n_embd}, backend); - layer.attn_q_norm = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_Q_NORM, "weight", i), {64}, backend); - layer.attn_q_norm_b = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_Q_NORM, "bias", i), {64}, backend); - layer.attn_k_norm = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_K_NORM, "weight", i), {64}, backend); - layer.attn_k_norm_b = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_K_NORM, "bias", i), {64}, backend); + + layer.attn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}); + layer.attn_norm_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM, "bias", i), {n_embd}); + + layer.wqkv = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_QKV, "weight", i), {n_embd, n_embd + 2*n_embd_gqa}); + layer.bqkv = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_QKV, "bias", i), {n_embd + 2*n_embd_gqa}); + + layer.wo = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd}); + layer.bo = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_OUT, "bias", i), {n_embd}); + + layer.ffn_down = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_DOWN, "weight", i), {n_ff, n_embd}); + layer.ffn_down_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_DOWN, "bias", i), {n_embd}); + + layer.ffn_up = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff}); + layer.ffn_up_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_UP, "bias", i), {n_ff}); + + layer.ffn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}); + layer.ffn_norm_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_NORM, "bias", i), {n_embd}); + + layer.attn_q_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_Q_NORM, "weight", i), {64}); + layer.attn_q_norm_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_Q_NORM, "bias", i), {64}); + + layer.attn_k_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_K_NORM, "weight", i), {64}); + layer.attn_k_norm_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_K_NORM, "bias", i), {64}); } } break; case LLM_ARCH_BLOOM: { - model.tok_embd = ml.create_tensor(ctx, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, GGML_BACKEND_CPU); - model.tok_norm = ml.create_tensor(ctx, tn(LLM_TENSOR_TOKEN_EMBD_NORM, "weight"), {n_embd}, GGML_BACKEND_CPU); - model.tok_norm_b = ml.create_tensor(ctx, tn(LLM_TENSOR_TOKEN_EMBD_NORM, "bias"), {n_embd}, GGML_BACKEND_CPU); + model.tok_embd = ml.create_tensor(ctx_input, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}); + model.tok_norm = ml.create_tensor(ctx_input, tn(LLM_TENSOR_TOKEN_EMBD_NORM, "weight"), {n_embd}); + model.tok_norm_b = ml.create_tensor(ctx_input, tn(LLM_TENSOR_TOKEN_EMBD_NORM, "bias"), {n_embd}); // output { - ggml_backend_type backend_norm; - ggml_backend_type backend_output; - - if (n_gpu_layers > int(n_layer)) { - backend_norm = llama_backend_offload; - backend_output = llama_backend_offload_split; - } else { - backend_norm = GGML_BACKEND_CPU; - backend_output = GGML_BACKEND_CPU; - } - - model.output_norm = ml.create_tensor(ctx, tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, backend_norm); - model.output_norm_b = ml.create_tensor(ctx, tn(LLM_TENSOR_OUTPUT_NORM, "bias"), {n_embd}, backend_norm); - model.output = ml.create_tensor(ctx, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, backend_output); + model.output_norm = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}); + model.output_norm_b = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "bias"), {n_embd}); + model.output = ml.create_tensor(ctx_output_split, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}); } - const uint32_t n_ff = hparams.n_ff; - const int64_t n_embd_gqa = n_embd_v_gqa; - GGML_ASSERT(n_embd_gqa == n_embd / hparams.n_gqa()); - GGML_ASSERT(n_embd_gqa == n_embd_k_gqa); - - const int i_gpu_start = n_layer - n_gpu_layers; - - model.layers.resize(n_layer); - - for (uint32_t i = 0; i < n_layer; ++i) { - const ggml_backend_type backend = int(i) < i_gpu_start ? GGML_BACKEND_CPU : llama_backend_offload; // NOLINT - const ggml_backend_type backend_split = int(i) < i_gpu_start ? GGML_BACKEND_CPU : llama_backend_offload_split; // NOLINT + for (int i = 0; i < n_layer; ++i) { + ggml_context * ctx_layer = ctx_for_layer(i); + ggml_context * ctx_split = ctx_for_layer_split(i); auto & layer = model.layers[i]; - layer.attn_norm = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, backend); - layer.attn_norm_b = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_NORM, "bias", i), {n_embd}, backend); + layer.attn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}); + layer.attn_norm_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM, "bias", i), {n_embd}); - layer.wqkv = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_QKV, "weight", i), {n_embd, n_embd + 2*n_embd_gqa}, backend_split); - layer.bqkv = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_QKV, "bias", i), {n_embd + 2*n_embd_gqa}, backend); + layer.wqkv = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_QKV, "weight", i), {n_embd, n_embd + 2*n_embd_gqa}); + layer.bqkv = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_QKV, "bias", i), {n_embd + 2*n_embd_gqa}); - layer.wo = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd}, backend_split); - layer.bo = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_OUT, "bias", i), {n_embd}, backend); + layer.wo = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd}); + layer.bo = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_OUT, "bias", i), {n_embd}); - layer.ffn_norm = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, backend); - layer.ffn_norm_b = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_NORM, "bias", i), {n_embd}, backend); + layer.ffn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}); + layer.ffn_norm_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_NORM, "bias", i), {n_embd}); - layer.ffn_down = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_DOWN, "weight", i), {n_ff, n_embd}, backend_split); - layer.ffn_down_b = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_DOWN, "bias", i), {n_embd}, backend); + layer.ffn_down = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_DOWN, "weight", i), {n_ff, n_embd}); + layer.ffn_down_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_DOWN, "bias", i), {n_embd}); - layer.ffn_up = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff}, backend_split); - layer.ffn_up_b = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_UP, "bias", i), {n_ff}, backend); + layer.ffn_up = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff}); + layer.ffn_up_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_UP, "bias", i), {n_ff}); } } break; case LLM_ARCH_MPT: { - model.tok_embd = ml.create_tensor(ctx, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, GGML_BACKEND_CPU); + model.tok_embd = ml.create_tensor(ctx_input, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}); + // output { - ggml_backend_type backend_norm; - ggml_backend_type backend_output; - - if (n_gpu_layers > int(n_layer)) { - backend_norm = llama_backend_offload; - backend_output = llama_backend_offload_split; - } else { - backend_norm = GGML_BACKEND_CPU; - backend_output = GGML_BACKEND_CPU; - } - - model.output_norm = ml.create_tensor(ctx, tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, backend_norm); - model.output = ml.create_tensor(ctx, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, backend_output); + model.output_norm = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}); + model.output = ml.create_tensor(ctx_output_split, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}); } - const uint32_t n_ff = hparams.n_ff; - const int64_t n_embd_gqa = n_embd_v_gqa; - GGML_ASSERT(n_embd_gqa == n_embd / hparams.n_gqa()); - GGML_ASSERT(n_embd_gqa == n_embd_k_gqa); - - const int i_gpu_start = n_layer - n_gpu_layers; - - model.layers.resize(n_layer); - - for (uint32_t i = 0; i < n_layer; ++i) { - const ggml_backend_type backend = int(i) < i_gpu_start ? GGML_BACKEND_CPU : llama_backend_offload; // NOLINT - const ggml_backend_type backend_split = int(i) < i_gpu_start ? GGML_BACKEND_CPU : llama_backend_offload_split; // NOLINT + for (int i = 0; i < n_layer; ++i) { + ggml_context * ctx_layer = ctx_for_layer(i); + ggml_context * ctx_split = ctx_for_layer_split(i); auto & layer = model.layers[i]; - layer.attn_norm = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, backend); - layer.wqkv = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_QKV, "weight", i), {n_embd, n_embd + 2*n_embd_gqa}, backend_split); - layer.wo = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd}, backend_split); + layer.attn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}); - layer.ffn_norm = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, backend); + layer.wqkv = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_QKV, "weight", i), {n_embd, n_embd + 2*n_embd_gqa}); + layer.wo = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd}); - layer.ffn_down = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_DOWN, "weight", i), { n_ff, n_embd}, backend_split); - layer.ffn_up = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff}, backend_split); + layer.ffn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}); + layer.ffn_down = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_DOWN, "weight", i), { n_ff, n_embd}); + layer.ffn_up = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff}); // AWQ ScaleActivation layer - layer.ffn_act = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_ACT, "scales", i), {n_ff}, backend, false); + layer.ffn_act = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_ACT, "scales", i), {n_ff}, false); } } break; case LLM_ARCH_STABLELM: { - model.tok_embd = ml.create_tensor(ctx, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, GGML_BACKEND_CPU); + model.tok_embd = ml.create_tensor(ctx_input, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}); // output { - ggml_backend_type backend_norm; - ggml_backend_type backend_output; - - if (n_gpu_layers > int(n_layer)) { - backend_norm = llama_backend_offload; - backend_output = llama_backend_offload_split; - } else { - backend_norm = GGML_BACKEND_CPU; - backend_output = GGML_BACKEND_CPU; - } - - model.output_norm_b = ml.create_tensor(ctx, tn(LLM_TENSOR_OUTPUT_NORM, "bias"), {n_embd}, backend_norm); - model.output_norm = ml.create_tensor(ctx, tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, backend_norm); - model.output = ml.create_tensor(ctx, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, backend_output); + model.output_norm_b = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "bias"), {n_embd}); + model.output_norm = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}); + model.output = ml.create_tensor(ctx_output_split, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}); } - const uint32_t n_ff = hparams.n_ff; - const int64_t n_embd_gqa = n_embd_v_gqa; - GGML_ASSERT(n_embd_gqa == n_embd / hparams.n_gqa()); - GGML_ASSERT(n_embd_gqa == n_embd_k_gqa); - - const int i_gpu_start = n_layer - n_gpu_layers; - - model.layers.resize(n_layer); - - for (uint32_t i = 0; i < n_layer; ++i) { - /* - llama_model_loader: - tensor 4: blk.0.attn_output.weight f16 [ 2560, 2560, 1, 1 ] - */ - const ggml_backend_type backend = int(i) < i_gpu_start ? GGML_BACKEND_CPU : llama_backend_offload; // NOLINT - const ggml_backend_type backend_split = int(i) < i_gpu_start ? GGML_BACKEND_CPU : llama_backend_offload_split; // NOLINT + for (int i = 0; i < n_layer; ++i) { + ggml_context * ctx_layer = ctx_for_layer(i); + ggml_context * ctx_split = ctx_for_layer_split(i); auto & layer = model.layers[i]; - layer.attn_norm = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, backend); - layer.attn_norm_b = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_NORM, "bias", i), {n_embd}, backend); + layer.attn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}); + layer.attn_norm_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM, "bias", i), {n_embd}); - layer.wq = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_embd}, backend_split); - layer.wk = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_K, "weight", i), {n_embd, n_embd_gqa}, backend_split); - layer.wv = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_V, "weight", i), {n_embd, n_embd_gqa}, backend_split); - layer.wo = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd}, backend_split); + layer.wq = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_embd}); + layer.wk = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_K, "weight", i), {n_embd, n_embd_gqa}); + layer.wv = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_V, "weight", i), {n_embd, n_embd_gqa}); + layer.wo = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd}); - layer.ffn_norm = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, backend); - layer.ffn_norm_b = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_NORM, "bias", i), {n_embd}, backend); + layer.ffn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}); + layer.ffn_norm_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_NORM, "bias", i), {n_embd}); - layer.ffn_gate = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff}, backend_split); - layer.ffn_down = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_DOWN, "weight", i), { n_ff, n_embd}, backend_split); - layer.ffn_up = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff}, backend_split); + layer.ffn_gate = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff}); + layer.ffn_down = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_DOWN, "weight", i), { n_ff, n_embd}); + layer.ffn_up = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff}); } } break; case LLM_ARCH_QWEN: { - model.tok_embd = ml.create_tensor(ctx, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, GGML_BACKEND_CPU); + model.tok_embd = ml.create_tensor(ctx_input, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}); + + // output { - ggml_backend_type backend_norm; - ggml_backend_type backend_output; + model.output_norm = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}); + model.output = ml.create_tensor(ctx_output_split, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}); + } - if (n_gpu_layers > int(n_layer)) { - backend_norm = llama_backend_offload; - backend_output = llama_backend_offload_split; - } else { - backend_norm = GGML_BACKEND_CPU; - backend_output = GGML_BACKEND_CPU; - } - - model.output_norm = ml.create_tensor(ctx, tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, backend_norm); - model.output = ml.create_tensor(ctx, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, backend_output); - } - - const uint32_t n_ff = hparams.n_ff / 2; - - const int i_gpu_start = n_layer - n_gpu_layers; - - model.layers.resize(n_layer); - - for (uint32_t i = 0; i < n_layer; ++i) { - const ggml_backend_type backend = int(i) < i_gpu_start ? GGML_BACKEND_CPU : llama_backend_offload; // NOLINT - const ggml_backend_type backend_split = int(i) < i_gpu_start ? GGML_BACKEND_CPU : llama_backend_offload_split; // NOLINT + for (int i = 0; i < n_layer; ++i) { + ggml_context * ctx_layer = ctx_for_layer(i); + ggml_context * ctx_split = ctx_for_layer_split(i); auto & layer = model.layers[i]; - layer.attn_norm = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, backend); + layer.attn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}); - layer.wqkv = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_QKV, "weight", i), {n_embd, n_embd * 3}, backend_split); - layer.bqkv = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_QKV, "bias", i), {n_embd * 3}, backend); - layer.wo = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd}, backend_split); + layer.wqkv = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_QKV, "weight", i), {n_embd, n_embd*3}); + layer.bqkv = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_QKV, "bias", i), {n_embd*3}); + layer.wo = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd}); - layer.ffn_norm = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, backend); + layer.ffn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}); - layer.ffn_gate = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff}, backend_split); - layer.ffn_down = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_DOWN, "weight", i), { n_ff, n_embd}, backend_split); - layer.ffn_up = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff}, backend_split); + layer.ffn_gate = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff/2}); + layer.ffn_down = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_DOWN, "weight", i), {n_ff/2, n_embd}); + layer.ffn_up = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff/2}); } } break; case LLM_ARCH_PHI2: { - model.tok_embd = ml.create_tensor(ctx, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, GGML_BACKEND_CPU); + model.tok_embd = ml.create_tensor(ctx_input, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}); // output { - ggml_backend_type backend_norm; - ggml_backend_type backend_output; - - if (n_gpu_layers > int(n_layer)) { - backend_norm = llama_backend_offload; - backend_output = llama_backend_offload; - } else { - backend_norm = GGML_BACKEND_CPU; - backend_output = GGML_BACKEND_CPU; - } - - model.output_norm = ml.create_tensor(ctx, tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, backend_norm); - model.output_norm_b = ml.create_tensor(ctx, tn(LLM_TENSOR_OUTPUT_NORM, "bias"), {n_embd}, backend_norm); - model.output = ml.create_tensor(ctx, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, backend_output); - model.output_b = ml.create_tensor(ctx, tn(LLM_TENSOR_OUTPUT, "bias"), {n_vocab}, backend_output); + model.output_norm = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}); + model.output_norm_b = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "bias"), {n_embd}); + model.output = ml.create_tensor(ctx_output_split, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}); + model.output_b = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT, "bias"), {n_vocab}); } - const uint32_t n_ff = hparams.n_ff; - const int64_t n_embd_gqa = n_embd_v_gqa; - GGML_ASSERT(n_embd_gqa == n_embd / hparams.n_gqa()); - GGML_ASSERT(n_embd_gqa == n_embd_k_gqa); - - const int i_gpu_start = n_layer - n_gpu_layers; - - model.layers.resize(n_layer); - - for (uint32_t i = 0; i < n_layer; ++i) { - const ggml_backend_type backend = int(i) < i_gpu_start ? GGML_BACKEND_CPU : llama_backend_offload; // NOLINT - const ggml_backend_type backend_split = int(i) < i_gpu_start ? GGML_BACKEND_CPU : llama_backend_offload_split; // NOLINT + for (int i = 0; i < n_layer; ++i) { + ggml_context * ctx_layer = ctx_for_layer(i); + ggml_context * ctx_split = ctx_for_layer_split(i); auto & layer = model.layers[i]; - layer.attn_norm = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, backend); - layer.attn_norm_b = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_NORM, "bias", i), {n_embd}, backend); + layer.attn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}); + layer.attn_norm_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM, "bias", i), {n_embd}); - layer.wqkv = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_QKV, "weight", i), {n_embd, n_embd + 2*n_embd_gqa}, backend_split); - layer.bqkv = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_QKV, "bias", i), {n_embd + 2*n_embd_gqa}, backend); + layer.wqkv = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_QKV, "weight", i), {n_embd, n_embd + 2*n_embd_gqa}); + layer.bqkv = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_QKV, "bias", i), {n_embd + 2*n_embd_gqa}); - layer.wo = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd}, backend_split); - layer.bo = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_OUT, "bias", i), {n_embd}, backend); + layer.wo = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd}); + layer.bo = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_OUT, "bias", i), {n_embd}); - layer.ffn_down = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_DOWN, "weight", i), {n_ff, n_embd}, backend_split); - layer.ffn_down_b = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_DOWN, "bias", i), {n_embd}, backend); + layer.ffn_down = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_DOWN, "weight", i), {n_ff, n_embd}); + layer.ffn_down_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_DOWN, "bias", i), {n_embd}); - layer.ffn_up = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff}, backend_split); - layer.ffn_up_b = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_UP, "bias", i), {n_ff}, backend); + layer.ffn_up = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff}); + layer.ffn_up_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_UP, "bias", i), {n_ff}); } } break; case LLM_ARCH_PLAMO: { - model.tok_embd = ml.create_tensor(ctx, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, GGML_BACKEND_CPU); + model.tok_embd = ml.create_tensor(ctx_input, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}); // output { - ggml_backend_type backend_norm; - ggml_backend_type backend_output; - - if (n_gpu_layers > int(n_layer)) { - backend_norm = llama_backend_offload; - backend_output = llama_backend_offload_split; - } else { - backend_norm = GGML_BACKEND_CPU; - backend_output = GGML_BACKEND_CPU; - } - - model.output_norm = ml.create_tensor(ctx, tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, backend_norm); - model.output = ml.create_tensor(ctx, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, backend_output); + model.output_norm = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}); + model.output = ml.create_tensor(ctx_output_split, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}); } - const uint32_t n_ff = hparams.n_ff; - const int64_t n_embd_gqa = n_embd_v_gqa; - GGML_ASSERT(n_embd_gqa == n_embd / hparams.n_gqa()); - GGML_ASSERT(n_embd_gqa == n_embd_k_gqa); - - const int i_gpu_start = n_layer - n_gpu_layers; - - model.layers.resize(n_layer); - - for (uint32_t i = 0; i < n_layer; ++i) { - const ggml_backend_type backend = int(i) < i_gpu_start ? GGML_BACKEND_CPU : llama_backend_offload; // NOLINT - const ggml_backend_type backend_split = int(i) < i_gpu_start ? GGML_BACKEND_CPU : llama_backend_offload_split; // NOLINT + for (int i = 0; i < n_layer; ++i) { + ggml_context * ctx_layer = ctx_for_layer(i); + ggml_context * ctx_split = ctx_for_layer_split(i); auto & layer = model.layers[i]; - layer.attn_norm = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, backend); + layer.attn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}); - layer.wq = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_embd}, backend_split); - layer.wk = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_K, "weight", i), {n_embd, n_embd_gqa}, backend_split); - layer.wv = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_V, "weight", i), {n_embd, n_embd_gqa}, backend_split); - layer.wo = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd}, backend_split); + layer.wq = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_embd}); + layer.wk = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_K, "weight", i), {n_embd, n_embd_gqa}); + layer.wv = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_V, "weight", i), {n_embd, n_embd_gqa}); + layer.wo = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd}); - layer.ffn_gate = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff}, backend_split); - layer.ffn_down = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_DOWN, "weight", i), { n_ff, n_embd}, backend_split); - layer.ffn_up = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff}, backend_split); + layer.ffn_gate = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff}); + layer.ffn_down = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_DOWN, "weight", i), { n_ff, n_embd}); + layer.ffn_up = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff}); } } break; case LLM_ARCH_GPT2: { - model.tok_embd = ml.create_tensor(ctx, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, GGML_BACKEND_CPU); - model.pos_embd = ml.create_tensor(ctx, tn(LLM_TENSOR_POS_EMBD, "weight"), {n_embd, hparams.n_ctx_train}, GGML_BACKEND_CPU); + model.tok_embd = ml.create_tensor(ctx_input, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}); + model.pos_embd = ml.create_tensor(ctx_input, tn(LLM_TENSOR_POS_EMBD, "weight"), {n_embd, hparams.n_ctx_train}); // output { - ggml_backend_type backend_norm; - ggml_backend_type backend_output; - - if (n_gpu_layers > int(n_layer)) { - backend_norm = llama_backend_offload; - backend_output = llama_backend_offload_split; - } else { - backend_norm = GGML_BACKEND_CPU; - backend_output = GGML_BACKEND_CPU; - } - - model.output_norm = ml.create_tensor(ctx, tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, backend_norm); - model.output_norm_b = ml.create_tensor(ctx, tn(LLM_TENSOR_OUTPUT_NORM, "bias"), {n_embd}, backend_norm); - model.output = ml.create_tensor(ctx, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, backend_output); + model.output_norm = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}); + model.output_norm_b = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "bias"), {n_embd}); + model.output = ml.create_tensor(ctx_output_split, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}); } - const uint32_t n_ff = hparams.n_ff; - const int64_t n_embd_gqa = n_embd_v_gqa; - GGML_ASSERT(n_embd_gqa == n_embd / hparams.n_gqa()); - GGML_ASSERT(n_embd_gqa == n_embd_k_gqa); - - const int i_gpu_start = n_layer - n_gpu_layers; - - model.layers.resize(n_layer); - - for (uint32_t i = 0; i < n_layer; ++i) { - const ggml_backend_type backend = int(i) < i_gpu_start ? GGML_BACKEND_CPU : llama_backend_offload; // NOLINT - const ggml_backend_type backend_split = int(i) < i_gpu_start ? GGML_BACKEND_CPU : llama_backend_offload_split; // NOLINT + for (int i = 0; i < n_layer; ++i) { + ggml_context * ctx_layer = ctx_for_layer(i); + ggml_context * ctx_split = ctx_for_layer_split(i); auto & layer = model.layers[i]; - layer.attn_norm = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, backend); - layer.attn_norm_b = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_NORM, "bias", i), {n_embd}, backend); + layer.attn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}); + layer.attn_norm_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM, "bias", i), {n_embd}); - layer.wqkv = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_QKV, "weight", i), {n_embd, n_embd + 2*n_embd_gqa}, backend_split); - layer.bqkv = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_QKV, "bias", i), {n_embd + 2*n_embd_gqa}, backend); + layer.wqkv = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_QKV, "weight", i), {n_embd, n_embd + 2*n_embd_gqa}); + layer.bqkv = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_QKV, "bias", i), {n_embd + 2*n_embd_gqa}); - layer.wo = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd}, backend_split); - layer.bo = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_OUT, "bias", i), {n_embd}, backend); + layer.wo = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd}); + layer.bo = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_OUT, "bias", i), {n_embd}); - layer.ffn_norm = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, backend); - layer.ffn_norm_b = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_NORM, "bias", i), {n_embd}, backend); + layer.ffn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}); + layer.ffn_norm_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_NORM, "bias", i), {n_embd}); - layer.ffn_down = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_DOWN, "weight", i), {n_ff, n_embd}, backend_split); - layer.ffn_down_b = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_DOWN, "bias", i), {n_embd}, backend); + layer.ffn_down = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_DOWN, "weight", i), {n_ff, n_embd}); + layer.ffn_down_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_DOWN, "bias", i), {n_embd}); - layer.ffn_up = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff}, backend_split); - layer.ffn_up_b = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_UP, "bias", i), {n_ff}, backend); + layer.ffn_up = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff}); + layer.ffn_up_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_UP, "bias", i), {n_ff}); } } break; default: @@ -3893,78 +3761,51 @@ static bool llm_load_tensors( ml.done_getting_tensors(); - ml.init_mapping(); + ml.init_mapping(true, use_mlock ? &model.mlock_mmap : nullptr); - // allocate tensors - size_t vram_weights = 0; - size_t buf_size = 0; + // create the backend buffers + std::vector> ctx_bufs; - ggml_backend_buffer_type_t buft = llama_default_buffer_type(n_gpu_layers); + for (auto & it : ctx_map) { + ggml_backend_buffer_type_t buft = it.first; + ggml_context * ctx = it.second; + ggml_backend_buffer_t buf = nullptr; - for (struct ggml_tensor * t = ggml_get_first_tensor(ctx); t != nullptr; t = ggml_get_next_tensor(ctx, t)) { - // GGML_BACKEND_GPU tensors are for CUDA and OpenCL only, which are handled separately without ggml-backend - if (t->backend == GGML_BACKEND_CPU) { - buf_size += GGML_PAD(ggml_backend_buft_get_alloc_size(buft, t), ggml_backend_buft_get_alignment(buft)); - } else { - vram_weights += ggml_nbytes(t); + // only the mmap region containing the tensors in the model is mapped to the backend buffer + // this is important for metal with apple silicon: if the entire model could be mapped to a metal buffer, then we could just use metal for all layers + // this allows using partial offloading when the model size exceeds the metal buffer size, but not the RAM size + if (ml.use_mmap && buft == llama_default_buffer_type_cpu(true)) { + size_t first, last; + ml.get_mapping_range(&first, &last, ctx); + buf = ggml_backend_cpu_buffer_from_ptr((char *) ml.mapping->addr + first, last - first); } - } - - // create backend buffer - ggml_backend_buffer_t buf_mmap = nullptr; - #ifdef GGML_USE_METAL - if (n_gpu_layers > 0) { - if (ml.use_mmap) { + else if (ml.use_mmap && buft == ggml_backend_metal_buffer_type()) { const size_t max_size = ggml_get_max_tensor_size(ctx); - model.buf = ggml_backend_metal_buffer_from_ptr(ml.mapping->addr, ml.mapping->size, max_size); - buf_mmap = model.buf; - } else { - model.buf = ggml_backend_alloc_ctx_tensors_from_buft(ctx, ggml_backend_metal_buffer_type()); + size_t first, last; + ml.get_mapping_range(&first, &last, ctx); + buf = ggml_backend_metal_buffer_from_ptr((char *) ml.mapping->addr + first, last - first, max_size); } - } -#elif defined(GGML_USE_CUBLAS) && defined(LLAMA_GGML_BACKEND_CUDA_TEST) - // for testing only - if (n_gpu_layers > 0) { - model.buf = ggml_backend_alloc_ctx_tensors_from_buft(ctx, ggml_backend_cuda_buffer_type(0)); - } #endif - - if (model.buf == nullptr) { - // CPU backend, and indirectly CUDA and OpenCL - if (ml.use_mmap) { - model.buf = ggml_backend_cpu_buffer_from_ptr(ml.mapping->addr, ml.mapping->size); - buf_mmap = model.buf; - } else { - // allocate only CPU tensors - model.buf = ggml_backend_buft_alloc_buffer(buft, buf_size); - ggml_tallocr_t alloc = ggml_tallocr_new_from_buffer(model.buf); - for (struct ggml_tensor * t = ggml_get_first_tensor(ctx); t != nullptr; t = ggml_get_next_tensor(ctx, t)) { - if (t->backend == GGML_BACKEND_CPU) { - ggml_tallocr_alloc(alloc, t); - } + else { + buf = ggml_backend_alloc_ctx_tensors_from_buft(ctx, buft); + if (buf != nullptr && use_mlock && ggml_backend_buffer_is_host(buf)) { + model.mlock_buf.init (ggml_backend_buffer_get_base(buf)); + model.mlock_buf.grow_to(ggml_backend_buffer_get_size(buf)); } - ggml_tallocr_free(alloc); } - } - - if (use_mlock && ggml_backend_buffer_is_host(model.buf)) { - model.mlock_buf.init (ggml_backend_buffer_get_base(model.buf)); - model.mlock_buf.grow_to(ggml_backend_buffer_get_size(model.buf)); + if (buf == nullptr) { + throw std::runtime_error("failed to allocate buffer"); + } + // indicate that this buffer contains weights + // this is used by ggml_backend_sched to improve op scheduling -> ops that use a weight are preferably scheduled to the backend that contains the weight + ggml_backend_buffer_set_usage(buf, GGML_BACKEND_BUFFER_USAGE_WEIGHTS); + model.bufs.push_back(buf); + ctx_bufs.emplace_back(ctx, buf); } // print memory requirements { - size_t sys_mem_required = ctx_size + buf_size; - - if (sys_mem_required > 0) { - LLAMA_LOG_INFO("%s: system memory used = %7.2f MiB\n", __func__, sys_mem_required / 1024.0 / 1024.0); - } - if (vram_weights > 0) { - LLAMA_LOG_INFO("%s: VRAM used = %7.2f MiB\n", __func__, vram_weights / 1024.0 / 1024.0); - } - -#if (defined(GGML_USE_CUBLAS) && !defined(LLAMA_GGML_BACKEND_CUDA_TEST)) || defined(GGML_USE_CLBLAST) const int n_gpu = std::min(n_gpu_layers, int(hparams.n_layer)); LLAMA_LOG_INFO("%s: offloading %d repeating layers to GPU\n", __func__, n_gpu); @@ -3976,23 +3817,26 @@ static bool llm_load_tensors( const int max_offloadable_layers = hparams.n_layer + 1; LLAMA_LOG_INFO("%s: offloaded %d/%d layers to GPU\n", __func__, std::min(n_gpu_layers, max_offloadable_layers), max_backend_supported_layers); -#endif // defined(GGML_USE_CUBLAS) || defined(GGML_USE_CLBLAST) - } -#if defined(GGML_USE_CUBLAS) && !defined(LLAMA_GGML_BACKEND_CUDA_TEST) - ggml_cuda_set_tensor_split(tensor_split); -#else - GGML_UNUSED(tensor_split); -#endif // GGML_USE_CUBLAS + for (ggml_backend_buffer_t buf : model.bufs) { + LLAMA_LOG_INFO("%s: %10s buffer size = %8.2f MiB\n", __func__, ggml_backend_buffer_name(buf), ggml_backend_buffer_get_size(buf) / 1024.0 / 1024.0); + } + } // populate tensors_by_name - for (int i = 0; i < ml.n_tensors; ++i) { - struct ggml_tensor * cur = ggml_get_tensor(ctx, ml.get_tensor_name(i)); - model.tensors_by_name.emplace_back(ggml_get_name(cur), cur); + for (ggml_context * ctx : model.ctxs) { + for (auto * cur = ggml_get_first_tensor(ctx); cur != NULL; cur = ggml_get_next_tensor(ctx, cur)) { + model.tensors_by_name.emplace_back(ggml_get_name(cur), cur); + } } - if (!ml.load_all_data(ctx, progress_callback, progress_callback_user_data, buf_mmap, use_mlock ? &model.mlock_mmap : NULL)) { - return false; + // load tensor data + for (auto & it : ctx_bufs) { + ggml_context * ctx = it.first; + ggml_backend_buffer_t buf = it.second; + if (!ml.load_all_data(ctx, progress_callback, progress_callback_user_data, buf, use_mlock ? &model.mlock_mmap : NULL)) { + return false; + } } model.mapping = std::move(ml.mapping); @@ -4026,13 +3870,13 @@ static int llama_model_load(const std::string & fname, llama_model & model, cons } if (!llm_load_tensors( - ml, model, params.n_gpu_layers, params.main_gpu, params.tensor_split, params.use_mlock, + ml, model, params.n_gpu_layers, params.split_mode, params.main_gpu, params.tensor_split, params.use_mlock, params.progress_callback, params.progress_callback_user_data )) { return -2; } } catch (const std::exception & err) { - LLAMA_LOG_ERROR("error loading model: %s\n", err.what()); + LLAMA_LOG_ERROR("%s: error loading model: %s\n", __func__, err.what()); return -1; } @@ -4476,8 +4320,6 @@ struct llm_build_context { do_rope_shift (worst_case || kv_self.has_shift), cb (cb), buf_compute_meta (lctx.buf_compute_meta) { - GGML_ASSERT(!!kv_self.ctx); - // all initializations should be done in init() } @@ -4557,6 +4399,12 @@ struct llm_build_context { cb(Vcur, "Vcur", il); } + // these nodes are added to the graph together so that they are not reordered + // by doing so, the number of splits in the graph is reduced + ggml_build_forward_expand(gf, Qcur); + ggml_build_forward_expand(gf, Kcur); + ggml_build_forward_expand(gf, Vcur); + Qcur = ggml_rope_custom( ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, hparams.n_rot, 0, 0, n_orig_ctx, freq_base, freq_scale, @@ -6077,199 +5925,13 @@ struct llm_build_context { } }; -// -// tensor offloading helpers -// -// TODO: will be removed with backend v2 - -enum llm_offload_func_e { - OFFLOAD_FUNC_NOP, - OFFLOAD_FUNC, - OFFLOAD_FUNC_FRC, // force offload - OFFLOAD_FUNC_KQV, - OFFLOAD_FUNC_NR, - OFFLOAD_FUNC_EMB, // embeddings - OFFLOAD_FUNC_OUT, -}; - -// TODO: will be removed with backend v2 -struct llm_offload_trie { - struct node { - ~node() { - for (int i = 0; i < 256; ++i) { - if (children[i]) { - delete children[i]; - } - } - } - - node * children[256] = { nullptr }; - llm_offload_func_e func = OFFLOAD_FUNC_NOP; - }; - - llm_offload_trie() { - root = new node; - } - - llm_offload_trie(const std::unordered_map & map) { - root = new node; - - for (const auto & kv : map) { - add(kv.first, kv.second); - } - } - - ~llm_offload_trie() { - delete root; - } - - void add(const char * name, llm_offload_func_e func) { - node * cur = root; - - for (int i = 0; ; ++i) { - const uint8_t c = name[i]; - - if (!c) { - break; - } - - if (!cur->children[c]) { - cur->children[c] = new node; - } - - cur = cur->children[c]; - } - - cur->func = func; - } - - llm_offload_func_e find(const char * name) const { - const node * cur = root; - - for (int i = 0; ; ++i) { - const uint8_t c = name[i]; - - if (!c) { - break; - } - - if (!cur->children[c]) { - return OFFLOAD_FUNC_NOP; - } - - cur = cur->children[c]; - } - - return cur->func; - } - - node * root = nullptr; -}; - -// TODO: will be removed with backend v2 -static const std::unordered_map k_offload_map = { - //{ "inp_tokens", OFFLOAD_FUNC_NR }, // TODO: missing K-quants get_rows kernel - //{ "inp_embd", OFFLOAD_FUNC_NR }, // TODO: missing K-quants get_rows kernel - { "pos_embd", OFFLOAD_FUNC_NR }, - - { "inp_pos", OFFLOAD_FUNC_FRC }, // this is often used for KQ ops (e.g. rope) - { "KQ_mask", OFFLOAD_FUNC_FRC }, - { "K_shift", OFFLOAD_FUNC_FRC }, - - { "K_shifted", OFFLOAD_FUNC }, - - { "inp_norm", OFFLOAD_FUNC_NR }, - { "inp_norm_w", OFFLOAD_FUNC_NR }, - { "inp_norm_wb", OFFLOAD_FUNC_NR }, - - { "norm", OFFLOAD_FUNC }, - { "norm_w", OFFLOAD_FUNC }, - { "norm_wb", OFFLOAD_FUNC }, - - { "attn_norm", OFFLOAD_FUNC }, - { "attn_norm_2", OFFLOAD_FUNC }, - - { "wqkv", OFFLOAD_FUNC_KQV }, - { "bqkv", OFFLOAD_FUNC_KQV }, - { "wqkv_clamped", OFFLOAD_FUNC_KQV }, - - { "tmpk", OFFLOAD_FUNC_KQV }, - { "tmpq", OFFLOAD_FUNC_KQV }, - { "tmpv", OFFLOAD_FUNC_KQV }, - { "Kcur", OFFLOAD_FUNC_KQV }, - { "Qcur", OFFLOAD_FUNC_KQV }, - { "Vcur", OFFLOAD_FUNC_KQV }, - - { "krot", OFFLOAD_FUNC_KQV }, - { "qrot", OFFLOAD_FUNC_KQV }, - { "kpass", OFFLOAD_FUNC_KQV }, - { "qpass", OFFLOAD_FUNC_KQV }, - { "krotated", OFFLOAD_FUNC_KQV }, - { "qrotated", OFFLOAD_FUNC_KQV }, - - { "q", OFFLOAD_FUNC_KQV }, - { "k", OFFLOAD_FUNC_KQV }, - { "kq", OFFLOAD_FUNC_KQV }, - { "kq_scaled", OFFLOAD_FUNC_KQV }, - { "kq_scaled_alibi", OFFLOAD_FUNC_KQV }, - { "kq_masked", OFFLOAD_FUNC_KQV }, - { "kq_soft_max", OFFLOAD_FUNC_KQV }, - { "kq_soft_max_ext", OFFLOAD_FUNC_KQV }, - { "v", OFFLOAD_FUNC_KQV }, - { "kqv", OFFLOAD_FUNC_KQV }, - { "kqv_merged", OFFLOAD_FUNC_KQV }, - { "kqv_merged_cont", OFFLOAD_FUNC_KQV }, - { "kqv_wo", OFFLOAD_FUNC_KQV }, - { "kqv_out", OFFLOAD_FUNC_KQV }, - - { "ffn_inp", OFFLOAD_FUNC }, - { "ffn_norm", OFFLOAD_FUNC }, - - { "ffn_up", OFFLOAD_FUNC }, - { "ffn_up_b", OFFLOAD_FUNC }, - { "ffn_gate", OFFLOAD_FUNC }, - { "ffn_gate_b", OFFLOAD_FUNC }, - { "ffn_gate_par", OFFLOAD_FUNC }, - { "ffn_act", OFFLOAD_FUNC }, - { "ffn_down", OFFLOAD_FUNC }, - { "ffn_down_b", OFFLOAD_FUNC }, - { "ffn_out", OFFLOAD_FUNC }, - - { "ffn_silu", OFFLOAD_FUNC }, - { "ffn_gelu", OFFLOAD_FUNC }, - { "ffn_relu", OFFLOAD_FUNC }, - { "ffn_sqr(relu)", OFFLOAD_FUNC }, - - { "ffn_moe_logits", OFFLOAD_FUNC }, - { "ffn_moe_probs", OFFLOAD_FUNC }, - { "ffn_moe_argsort", OFFLOAD_FUNC }, - { "ffn_moe_weights", OFFLOAD_FUNC }, - { "ffn_moe_weights_sum", OFFLOAD_FUNC }, - { "ffn_moe_weights_norm", OFFLOAD_FUNC }, - { "ffn_moe_weighted", OFFLOAD_FUNC }, - { "ffn_moe_up", OFFLOAD_FUNC }, - { "ffn_moe_gate", OFFLOAD_FUNC }, - { "ffn_moe_silu", OFFLOAD_FUNC }, - { "ffn_moe_gate_par", OFFLOAD_FUNC }, - { "ffn_moe_down", OFFLOAD_FUNC }, - { "ffn_moe_out", OFFLOAD_FUNC }, - - { "l_out", OFFLOAD_FUNC }, - - { "result_norm", OFFLOAD_FUNC_EMB }, - { "result_output_no_bias", OFFLOAD_FUNC_EMB }, - { "result_output", OFFLOAD_FUNC_OUT }, -}; - -static llm_offload_trie k_offload_func_trie(k_offload_map); - static struct ggml_cgraph * llama_build_graph( llama_context & lctx, const llama_batch & batch) { const auto & model = lctx.model; // check if we should build the worst-case graph (for memory measurement) - const bool worst_case = ggml_allocr_is_measure(lctx.alloc); + const bool worst_case = ggml_tallocr_is_measure(lctx.alloc); // keep track of the input that has already been allocated bool alloc_inp_tokens = false; @@ -6278,16 +5940,8 @@ static struct ggml_cgraph * llama_build_graph( bool alloc_inp_KQ_mask = false; bool alloc_inp_K_shift = false; -#if defined(GGML_USE_CUBLAS) && !defined(LLAMA_GGML_BACKEND_CUDA_TEST) - const bool do_offload = true; -#else - const bool do_offload = true; // TODO: set to false after finishing refactoring -#endif - - int n_non_view = 0; // number of non-view tensors that have been processed by the callback - // this callback allows us to apply custom logic to each tensor (e.g. ggml-alloc, offloading, etc.) - // TODO: will be removed with backend v2 + // TODO: improve handling of input and output tensors, then replace this with ggml_set_name llm_build_cb cb = [&](struct ggml_tensor * cur, const char * name, int il) { if (il >= 0) { ggml_format_name(cur, "%s-%d", name, il); @@ -6298,12 +5952,11 @@ static struct ggml_cgraph * llama_build_graph( // // allocate input tensors and set input data // - // TODO: will be removed with backend v2 if (!alloc_inp_tokens && strcmp(name, "inp_tokens") == 0) { - ggml_allocr_alloc(lctx.alloc, cur); + ggml_tallocr_alloc(lctx.alloc, cur); - if (!ggml_allocr_is_measure(lctx.alloc) && batch.token) { + if (!ggml_tallocr_is_measure(lctx.alloc) && batch.token) { const int64_t n_tokens = cur->ne[0]; ggml_backend_tensor_set(cur, batch.token, 0, n_tokens*ggml_element_size(cur)); @@ -6312,10 +5965,10 @@ static struct ggml_cgraph * llama_build_graph( alloc_inp_tokens = true; } - if (!alloc_inp_embd && strcmp(name, "inp_embd") == 0) { - ggml_allocr_alloc(lctx.alloc, cur); + if (!alloc_inp_embd && strcmp(name, "inp_embd") == 0 && batch.embd) { + ggml_tallocr_alloc(lctx.alloc, cur); - if (!ggml_allocr_is_measure(lctx.alloc) && batch.embd) { + if (!ggml_tallocr_is_measure(lctx.alloc) && batch.embd) { const int64_t n_embd = cur->ne[0]; const int64_t n_tokens = cur->ne[1]; @@ -6326,9 +5979,9 @@ static struct ggml_cgraph * llama_build_graph( } if (!alloc_inp_pos && strcmp(name, "inp_pos") == 0) { - ggml_allocr_alloc(lctx.alloc, cur); + ggml_tallocr_alloc(lctx.alloc, cur); - if (!ggml_allocr_is_measure(lctx.alloc) && batch.pos) { + if (!ggml_tallocr_is_measure(lctx.alloc) && batch.pos) { const int64_t n_tokens = cur->ne[0]; static_assert(std::is_same::value, "llama_pos must be int32_t"); @@ -6339,9 +5992,9 @@ static struct ggml_cgraph * llama_build_graph( } if (!alloc_inp_KQ_mask && strcmp(name, "KQ_mask") == 0) { - ggml_allocr_alloc(lctx.alloc, cur); + ggml_tallocr_alloc(lctx.alloc, cur); - if (!ggml_allocr_is_measure(lctx.alloc)) { + if (!ggml_tallocr_is_measure(lctx.alloc)) { const int64_t n_kv = cur->ne[0]; const int64_t n_tokens = cur->ne[1]; @@ -6379,9 +6032,9 @@ static struct ggml_cgraph * llama_build_graph( } if (!alloc_inp_K_shift && strcmp(name, "K_shift") == 0) { - ggml_allocr_alloc(lctx.alloc, cur); + ggml_tallocr_alloc(lctx.alloc, cur); - if (!ggml_allocr_is_measure(lctx.alloc)) { + if (!ggml_tallocr_is_measure(lctx.alloc)) { const int64_t n_ctx = cur->ne[0]; int32_t * data; @@ -6403,136 +6056,6 @@ static struct ggml_cgraph * llama_build_graph( alloc_inp_K_shift = true; } - - // view tensors are not processed further - if (cur->view_src != nullptr) { - return; - } - - if (cur->op != GGML_OP_NONE) { - n_non_view++; - } - - // - // offload layers - // - // TODO: will be removed with backend v2 - -//#define LLAMA_OFFLOAD_DEBUG - - if (!do_offload) { - return; - } - - const int n_layer = model.hparams.n_layer; - - const int n_gpu_layers = model.n_gpu_layers; - const int i_gpu_start = n_layer - n_gpu_layers; - - // should we offload the final norm? yes if we are not computing embeddings - const bool offload_emb = lctx.embedding.empty(); - - static const std::unordered_map> k_offload_func_name = { - { OFFLOAD_FUNC_NOP, "CPU" }, - { OFFLOAD_FUNC_OUT, "CPU" }, -#if defined(GGML_USE_CUBLAS) && !defined(LLAMA_GGML_BACKEND_CUDA_TEST) - { OFFLOAD_FUNC, "GPU (CUDA)" }, - { OFFLOAD_FUNC_FRC, "GPU (CUDA) FRC" }, - { OFFLOAD_FUNC_KQV, "GPU (CUDA) KQV" }, - { OFFLOAD_FUNC_NR, "GPU (CUDA) NR" }, - { OFFLOAD_FUNC_EMB, "GPU (CUDA) EMB" }, -#else - { OFFLOAD_FUNC, "CPU" }, - { OFFLOAD_FUNC_FRC, "CPU" }, - { OFFLOAD_FUNC_KQV, "CPU" }, - { OFFLOAD_FUNC_NR, "CPU" }, - { OFFLOAD_FUNC_EMB, "CPU" }, -#endif // GGML_USE_CUBLAS - }; - - // check the global map for what offload function to use for this tensor - llm_offload_func_e func_e = k_offload_func_trie.find(name); - - if (func_e == OFFLOAD_FUNC_NOP) { -#ifdef LLAMA_OFFLOAD_DEBUG - // if a tensor hasn't been offloaded, we warn the user - if (worst_case) { - LLAMA_LOG_WARN("%s: %32s: not offloaded (ref: %s)\n", __func__, - cur->name, "https://github.com/ggerganov/llama.cpp/pull/3837"); - } -#endif - - return; - } - - // count the number of layers and respect the provided n_gpu_layers - switch (func_e) { - case OFFLOAD_FUNC_NOP: - case OFFLOAD_FUNC_OUT: - break; - case OFFLOAD_FUNC: - if (n_gpu_layers < n_layer) { - if (il < i_gpu_start) { - func_e = OFFLOAD_FUNC_NOP; - } - } - break; - case OFFLOAD_FUNC_FRC: - if (!lctx.cparams.offload_kqv) { - func_e = OFFLOAD_FUNC_NOP; - } break; - case OFFLOAD_FUNC_KQV: - if (!lctx.cparams.offload_kqv) { - func_e = OFFLOAD_FUNC_NOP; - } else { - if (n_gpu_layers < n_layer) { - if (il < i_gpu_start) { - func_e = OFFLOAD_FUNC_NOP; - } - } - } - break; - case OFFLOAD_FUNC_NR: - if (n_gpu_layers <= n_layer + 0) { - func_e = OFFLOAD_FUNC_NOP; - } - break; - case OFFLOAD_FUNC_EMB: - if (!offload_emb || n_gpu_layers < n_layer) { - func_e = OFFLOAD_FUNC_NOP; - } - break; - default: GGML_ASSERT(false); - } - - offload_func_t func = ggml_offload_nop; - - // this is needed for compatibility with Metal for example -#if defined(GGML_USE_CUBLAS) && !defined(LLAMA_GGML_BACKEND_CUDA_TEST) - static offload_func_t ggml_offload_gpu = ggml_cuda_assign_buffers_no_alloc; -#else - static offload_func_t ggml_offload_gpu = ggml_offload_nop; -#endif - - switch (func_e) { - case OFFLOAD_FUNC_NOP: - case OFFLOAD_FUNC_OUT: func = ggml_offload_nop; break; - case OFFLOAD_FUNC: - case OFFLOAD_FUNC_KQV: - case OFFLOAD_FUNC_FRC: - case OFFLOAD_FUNC_NR: - case OFFLOAD_FUNC_EMB: func = ggml_offload_gpu; break; - default: GGML_ASSERT(false); - } - - // apply offload function to the tensor - func(cur); - -#ifdef LLAMA_OFFLOAD_DEBUG - if (worst_case) { - LLAMA_LOG_INFO("%s: %32s: %s\n", __func__, cur->name, k_offload_func_name.at(func_e).c_str()); - } -#endif }; struct ggml_cgraph * result = NULL; @@ -6600,27 +6123,6 @@ static struct ggml_cgraph * llama_build_graph( llm.free(); - if (worst_case) { - int n_non_view_total = 0; - - for (int i = 0; i < result->n_nodes; ++i) { - if (result->nodes[i]->view_src == nullptr) { - n_non_view_total++; - } - } - - LLAMA_LOG_INFO("%s: non-view tensors processed: %d/%d\n", __func__, n_non_view, n_non_view_total); - - if (n_non_view != n_non_view_total) { - LLAMA_LOG_WARN("%s: ****************************************************************\n", __func__); - LLAMA_LOG_WARN("%s: not all non-view tensors have been processed with a callback\n", __func__); - LLAMA_LOG_WARN("%s: this can indicate an inefficiency in the graph implementation\n", __func__); - LLAMA_LOG_WARN("%s: build with LLAMA_OFFLOAD_DEBUG for more info\n", __func__); - LLAMA_LOG_WARN("%s: ref: https://github.com/ggerganov/llama.cpp/pull/3837\n", __func__); - LLAMA_LOG_WARN("%s: ****************************************************************\n", __func__); - } - } - return result; } @@ -6666,8 +6168,6 @@ static int llama_decode_internal( auto & kv_self = lctx.kv_self; - GGML_ASSERT(!!kv_self.ctx); - const int64_t n_embd = hparams.n_embd; const int64_t n_vocab = hparams.n_vocab; @@ -6721,12 +6221,10 @@ static int llama_decode_internal( //printf("kv_self.n = %5d, kv_self.used = %5d, kv_self.head = %5d\n", kv_self.n, kv_self.used, kv_self.head); - ggml_allocr_reset(lctx.alloc); + ggml_backend_sched_reset(lctx.sched); ggml_cgraph * gf = llama_build_graph(lctx, batch); - ggml_allocr_alloc_graph(lctx.alloc, gf); - // the output is always the last tensor in the graph struct ggml_tensor * res = gf->nodes[gf->n_nodes - 1]; GGML_ASSERT(strcmp(res->name, "result_output") == 0); @@ -6738,30 +6236,6 @@ static int llama_decode_internal( GGML_ASSERT(strcmp(embeddings->name, "result_norm") == 0); } -#if defined(GGML_USE_CUBLAS) && !defined(LLAMA_GGML_BACKEND_CUDA_TEST) - char * buf_alloc_base = (char *)ggml_backend_buffer_get_base(lctx.buf_alloc); - for (int i = 0; i < gf->n_leafs; i++) { - ggml_tensor * node = gf->leafs[i]; - if (node->backend == GGML_BACKEND_GPU && node->extra == NULL) { - ggml_cuda_assign_scratch_offset(node, (char *)node->data - buf_alloc_base); - ggml_cuda_copy_to_device(node); - } - } - - for (int i = 0; i < gf->n_nodes; i++) { - ggml_tensor * node = gf->nodes[i]; - if (node->backend == GGML_BACKEND_GPU && node->extra == NULL) { - ggml_cuda_assign_scratch_offset(node, (char *)node->data - buf_alloc_base); - } - } - - // HACK: ggml-alloc may change the tensor backend when reusing a parent, so force output to be on the CPU here if needed - if (!lctx.embedding.empty()) { - embeddings->backend = GGML_BACKEND_CPU; - } - res->backend = GGML_BACKEND_CPU; -#endif - // LLAMA_LOG_INFO("graph build time: %.3f ms (%d nodes, %d leafs)\n", (ggml_time_us() - t_start_us)/1000.0, gf->n_nodes, gf->n_leafs); // for big prompts, if BLAS is enabled, it is better to use only one thread @@ -6784,15 +6258,17 @@ static int llama_decode_internal( #endif #ifdef GGML_USE_METAL - if (ggml_backend_is_metal(lctx.backend)) { - ggml_backend_metal_set_n_cb(lctx.backend, n_threads); + if (ggml_backend_is_metal(lctx.backend_metal)) { + ggml_backend_metal_set_n_cb(lctx.backend_metal, n_threads); } #endif - if (ggml_backend_is_cpu(lctx.backend)) { - ggml_backend_cpu_set_n_threads(lctx.backend, n_threads); + if (lctx.backend_cpu != nullptr) { + ggml_backend_cpu_set_n_threads(lctx.backend_cpu, n_threads); } - ggml_backend_graph_compute(lctx.backend, gf); + ggml_backend_sched_graph_compute(lctx.sched, gf); + + // fprintf(stderr, "splits: %d\n", ggml_backend_sched_get_n_splits(lctx.sched)); #ifdef GGML_USE_MPI ggml_mpi_graph_compute_post(lctx.ctx_mpi, gf, n_layer); @@ -6840,30 +6316,33 @@ static int llama_decode_internal( logits_out.clear(); #endif + ggml_backend_t res_backend = ggml_backend_sched_get_node_backend(lctx.sched, res); + GGML_ASSERT(res_backend != nullptr); if (batch.logits) { logits_out.resize(n_vocab * n_tokens); for (uint32_t i = 0; i < n_tokens; i++) { if (batch.logits[i] == 0) { continue; } - ggml_backend_tensor_get(res, logits_out.data() + (n_vocab*i), (n_vocab*i)*sizeof(float), n_vocab*sizeof(float)); + ggml_backend_tensor_get_async(res_backend, res, logits_out.data() + (n_vocab*i), (n_vocab*i)*sizeof(float), n_vocab*sizeof(float)); #ifndef NDEBUG logits_valid[i] = true; #endif } } else if (lctx.logits_all) { logits_out.resize(n_vocab * n_tokens); - ggml_backend_tensor_get(res, logits_out.data(), 0, n_vocab*n_tokens*sizeof(float)); + ggml_backend_tensor_get_async(res_backend, res, logits_out.data(), 0, n_vocab*n_tokens*sizeof(float)); #ifndef NDEBUG std::fill(logits_valid.begin(), logits_valid.end(), true); #endif } else { logits_out.resize(n_vocab); - ggml_backend_tensor_get(res, logits_out.data(), (n_vocab*(n_tokens - 1))*sizeof(float), n_vocab*sizeof(float)); + ggml_backend_tensor_get_async(res_backend, res, logits_out.data(), (n_vocab*(n_tokens - 1))*sizeof(float), n_vocab*sizeof(float)); #ifndef NDEBUG logits_valid[0] = true; #endif } + ggml_backend_synchronize(res_backend); } // extract embeddings @@ -6871,7 +6350,9 @@ static int llama_decode_internal( auto & embedding_out = lctx.embedding; embedding_out.resize(n_embd); - ggml_backend_tensor_get(embeddings, embedding_out.data(), (n_embd*(n_tokens - 1))*sizeof(float), n_embd*sizeof(float)); + ggml_backend_t embeddings_backend = ggml_backend_sched_get_node_backend(lctx.sched, embeddings); + ggml_backend_tensor_get_async(embeddings_backend, embeddings, embedding_out.data(), (n_embd*(n_tokens - 1))*sizeof(float), n_embd*sizeof(float)); + ggml_backend_synchronize(embeddings_backend); } // measure the performance only for the single-token evals @@ -9347,48 +8828,23 @@ static int llama_apply_lora_from_file_internal( LLAMA_LOG_INFO("%s: r = %d, alpha = %d, scaling = %.2f\n", __func__, lora_r, lora_alpha, scaling); - // create a name -> tensor map of the model to accelerate lookups - // find the max tensor size to estimate the required temporary buffer size - size_t max_tensor_size = 0; - std::unordered_map model_tensors; - for (const auto & kv : model.tensors_by_name) { - model_tensors.insert(kv); - size_t f32_size = ggml_nelements(kv.second) * sizeof(float); - max_tensor_size = std::max(max_tensor_size, f32_size); - } - - // create a temporary ggml context to store the lora tensors - // TODO: use ggml-alloc - size_t lora_ctx_size = max_tensor_size * 3; - LLAMA_LOG_INFO("%s: allocating %.f MB for lora temporary buffer\n", __func__, lora_ctx_size / 1024.0 / 1024.0); - std::vector lora_buf(lora_ctx_size); - - struct ggml_init_params params; - params.mem_size = lora_buf.size(); - params.mem_buffer = lora_buf.data(); - params.no_alloc = false; - - using unique_context = std::unique_ptr; - - unique_context lora_ctx(nullptr, ggml_free); - lora_ctx.reset(ggml_init(params)); - std::unordered_map lora_tensors; - // load base model std::unique_ptr ml; - - if (path_base_model) { + if (path_base_model) { LLAMA_LOG_INFO("%s: loading base model from '%s'\n", __func__, path_base_model); ml.reset(new llama_model_loader(path_base_model, /*use_mmap*/ true, /*kv_overrides*/ nullptr)); - ml->init_mapping(false); // no prefetching + ml->init_mapping(/*prefetch*/ false); // no prefetching } - // read tensors and apply - bool warned = false; - int n_tensors = 0; - - std::vector work_buffer; + struct tensor_meta { + std::string name; + ggml_type type; + int32_t ne[2]; + size_t offset; + }; + std::map tensor_meta_map; + // load all tensor meta while (true) { if (fin.tell() == fin.size) { // eof @@ -9401,7 +8857,7 @@ static int llama_apply_lora_from_file_internal( fin.read_raw(&n_dims, sizeof(n_dims)); fin.read_raw(&name_len, sizeof(name_len)); - fin.read_raw(&ftype, sizeof(ftype)); + fin.read_raw(&ftype, sizeof(ftype)); if (n_dims != 1 && n_dims != 2) { LLAMA_LOG_ERROR("%s: unsupported tensor dimension %d\n", __func__, n_dims); @@ -9415,31 +8871,23 @@ static int llama_apply_lora_from_file_internal( std::string name; { - GGML_ASSERT(name_len <= 1024); - char buf[1024]; + GGML_ASSERT(name_len < GGML_MAX_NAME); + char buf[GGML_MAX_NAME]; fin.read_raw(buf, name_len); name = std::string(buf, name_len); } - // check for lora suffix and get the type of tensor - const std::string lora_suffix = ".lora"; - size_t pos = name.rfind(lora_suffix); - if (pos == std::string::npos) { + // check for lora suffix + std::string lora_suffix; + if (name.length() > 6) { + lora_suffix = name.substr(name.length() - 6); + } + if (lora_suffix != ".loraA" && lora_suffix != ".loraB") { LLAMA_LOG_ERROR("%s: error: '%s' is not a lora tensor\n", __func__, name.c_str()); return 1; } - std::string lora_type = name.substr(pos + lora_suffix.length()); - std::string base_name = name; - base_name.erase(pos); - // LLAMA_LOG_INFO("%s: %s => %s (lora type %s) \n", __func__, name.c_str(), base_name.c_str(), lora_type.c_str()); - - if (model_tensors.find(base_name) == model_tensors.end()) { - LLAMA_LOG_ERROR("%s: unknown tensor '%s' in lora adapter\n", __func__, name.data()); - return 1; - } - - // create ggml tensor + // tensor type ggml_type wtype; switch (ftype) { case 0: wtype = GGML_TYPE_F32; break; @@ -9451,122 +8899,177 @@ static int llama_apply_lora_from_file_internal( return false; } } - ggml_tensor * lora_tensor = ggml_new_tensor_2d(lora_ctx.get(), wtype, ne[0], ne[1]); - ggml_set_name(lora_tensor, name.c_str()); - // load tensor data + // data offset size_t offset = fin.tell(); - size_t tensor_data_size = ggml_nbytes(lora_tensor); offset = (offset + 31) & -32; - fin.seek(offset, SEEK_SET); - fin.read_raw(lora_tensor->data, tensor_data_size); - lora_tensors[name] = lora_tensor; + // skip tensor data + fin.seek(offset + ggml_row_size(wtype, ne[0]) * ne[1], SEEK_SET); - // check if we have both A and B tensors and apply - if (lora_tensors.find(base_name + ".loraA") != lora_tensors.end() && - lora_tensors.find(base_name + ".loraB") != lora_tensors.end()) { + tensor_meta_map.emplace(name, tensor_meta{ name, wtype, { ne[0], ne[1] }, offset }); + } - ggml_tensor * dest_t = model_tensors[base_name]; + bool warned = false; + int n_tensors = 0; - offload_func_t offload_func = ggml_offload_nop; - offload_func_t offload_func_force_inplace = ggml_offload_nop; + // apply + ggml_backend_t backend_cpu = ggml_backend_cpu_init(); + if (backend_cpu == nullptr) { + LLAMA_LOG_ERROR("%s: error: failed to initialize cpu backend\n", __func__); + return 1; + } + ggml_backend_cpu_set_n_threads(backend_cpu, n_threads); -#if defined(GGML_USE_CUBLAS) && !defined(LLAMA_GGML_BACKEND_CUDA_TEST) - if (dest_t->backend == GGML_BACKEND_GPU || dest_t->backend == GGML_BACKEND_GPU_SPLIT) { - if (dest_t->type != GGML_TYPE_F16) { - throw std::runtime_error(format( - "%s: error: the simultaneous use of LoRAs and GPU acceleration is only supported for f16 models. dest_t->type: %d", __func__, dest_t->type)); - } - offload_func = ggml_cuda_assign_buffers; - offload_func_force_inplace = ggml_cuda_assign_buffers_force_inplace; - } -#endif // GGML_USE_CUBLAS + std::vector> read_buf; + for (const auto & it : model.tensors_by_name) { + const std::string & base_name = it.first; + ggml_tensor * model_t = it.second; - ggml_tensor * base_t; - if (ml) { - struct gguf_context * ctx_gguf = ml->ctx_gguf; + if (tensor_meta_map.find(base_name + ".loraA") == tensor_meta_map.end() || + tensor_meta_map.find(base_name + ".loraB") == tensor_meta_map.end()) { + continue; + } - // load from base model - if (gguf_find_tensor(ctx_gguf, base_name.c_str()) < 0) { - LLAMA_LOG_ERROR("%s: error: tensor '%s' not found in base model\n", __func__, base_name.c_str()); - return 1; - } + tensor_meta & metaA = tensor_meta_map.at(base_name + ".loraA"); + tensor_meta & metaB = tensor_meta_map.at(base_name + ".loraB"); - base_t = ml->get_tensor_meta(base_name.c_str()); - ml->load_data_for(base_t); - } else { - base_t = dest_t; - } + ggml_init_params lora_init_params = { + /* .mem_size */ ggml_tensor_overhead()*128 + ggml_graph_overhead(), + /* .mem_buffer */ nullptr, + /* .no_alloc */ true, + }; + ggml_context * lora_ctx = ggml_init(lora_init_params); + if (lora_ctx == nullptr) { + LLAMA_LOG_ERROR("%s: error: failed to initialize lora context\n", __func__); + ggml_backend_free(backend_cpu); + return 1; + } - if (ggml_is_quantized(base_t->type)) { - if (!warned) { - LLAMA_LOG_WARN("%s: warning: using a lora adapter with a quantized model may result in poor quality, " - "use a f16 or f32 base model with --lora-base\n", __func__); - warned = true; - } - } + // create tensors + ggml_tensor * loraA = ggml_new_tensor_2d(lora_ctx, metaA.type, metaA.ne[0], metaA.ne[1]); + ggml_tensor * loraB = ggml_new_tensor_2d(lora_ctx, metaB.type, metaB.ne[0], metaB.ne[1]); + ggml_set_name(loraA, metaA.name.c_str()); + ggml_set_name(loraB, metaB.name.c_str()); - ggml_tensor * loraA = lora_tensors[base_name + ".loraA"]; - GGML_ASSERT(loraA->type == GGML_TYPE_F32); - ggml_set_name(loraA, "loraA"); - - ggml_tensor * loraB = lora_tensors[base_name + ".loraB"]; - GGML_ASSERT(loraB->type == GGML_TYPE_F32); - ggml_set_name(loraB, "loraB"); - - if (base_t->ne[0] != loraA->ne[1] || base_t->ne[1] != loraB->ne[1]) { - LLAMA_LOG_ERROR("%s: incompatible tensor dimensions (%" PRId64 " and %" PRId64 ");" - " are you sure that this adapter is for this model?\n", __func__, base_t->ne[0], loraA->ne[1]); + ggml_tensor * base_t; + if (ml) { + if (gguf_find_tensor(ml->ctx_gguf, base_name.c_str()) < 0) { + LLAMA_LOG_ERROR("%s: error: tensor '%s' not found in base model\n", __func__, base_name.c_str()); return 1; } + base_t = ggml_dup_tensor(lora_ctx, ml->get_tensor_meta(base_name.c_str())); + } else { + base_t = ggml_dup_tensor(lora_ctx, model_t); + } + ggml_set_name(base_t, base_name.c_str()); + // allocate in backend buffer + ggml_backend_buffer_t lora_buf = ggml_backend_alloc_ctx_tensors_from_buft(lora_ctx, ggml_backend_cpu_buffer_type()); + if (lora_buf == nullptr) { + LLAMA_LOG_ERROR("%s: error: failed to allocate lora tensors\n", __func__); + return 1; + } + + // load tensor data + auto load_tensor = [&read_buf, &fin](const tensor_meta & tensor_meta, ggml_tensor * tensor) { + read_buf.resize(ggml_nbytes(tensor)); + fin.seek(tensor_meta.offset, SEEK_SET); + fin.read_raw(read_buf.data(), ggml_nbytes(tensor)); + ggml_backend_tensor_set(tensor, read_buf.data(), 0, read_buf.size()); + }; + load_tensor(metaA, loraA); + load_tensor(metaB, loraB); + + // load base model tensor data + if (ml) { + ml->load_data_for(base_t); + } else { + ggml_backend_tensor_copy(model_t, base_t); + } + + if (ggml_is_quantized(base_t->type) && !warned) { + LLAMA_LOG_WARN("%s: warning: using a lora adapter with a quantized model may result in poor quality, " + "use a f16 or f32 base model with --lora-base\n", __func__); + warned = true; + } + + if (base_t->ne[0] != loraA->ne[1] || base_t->ne[1] != loraB->ne[1]) { + LLAMA_LOG_ERROR("%s: incompatible tensor dimensions (%" PRId64 " and %" PRId64 ");" + " are you sure that this adapter is for this model?\n", __func__, base_t->ne[0], loraA->ne[1]); + ggml_free(lora_ctx); + ggml_backend_buffer_free(lora_buf); + ggml_backend_free(backend_cpu); + return 1; + } + + auto build_lora_graph = [&]() { // w = w + BA*s - ggml_tensor * BA = ggml_mul_mat(lora_ctx.get(), loraA, loraB); - offload_func(BA); + ggml_tensor * BA = ggml_mul_mat(lora_ctx, loraA, loraB); ggml_set_name(BA, "BA"); if (scaling != 1.0f) { - BA = ggml_scale_inplace(lora_ctx.get(), BA, scaling); - offload_func(BA); + BA = ggml_scale(lora_ctx, BA, scaling); ggml_set_name(BA, "BA_scaled"); } ggml_tensor * r; - if (base_t == dest_t) { - r = ggml_add_inplace(lora_ctx.get(), dest_t, BA); - offload_func_force_inplace(r); - ggml_set_name(r, "r_add_inplace"); - } - else { - r = ggml_add(lora_ctx.get(), base_t, BA); - offload_func(r); - ggml_set_name(r, "r_add"); + r = ggml_add_inplace(lora_ctx, base_t, BA); + ggml_set_name(r, "r_add"); - r = ggml_cpy(lora_ctx.get(), r, dest_t); - offload_func(r); - ggml_set_name(r, "r_cpy"); + if (base_t->type != model_t->type) { + // convert the result to the model type + r = ggml_cast(lora_ctx, r, model_t->type); + ggml_set_name(r, "r_cast"); } - struct ggml_cgraph * gf = ggml_new_graph(lora_ctx.get()); - ggml_build_forward_expand(gf, r); + return r; + }; - ggml_graph_compute_helper(work_buffer, gf, n_threads); + ggml_cgraph * gf = ggml_new_graph(lora_ctx); + ggml_tensor * r = build_lora_graph(); + ggml_build_forward_expand(gf, r); - // the tensors in the adapter must be sorted such that loraA and loraB of the same tensor are next to each other - GGML_ASSERT(lora_tensors.size() == 2); + ggml_backend_buffer_t graph_buf = ggml_backend_alloc_ctx_tensors_from_buft(lora_ctx, ggml_backend_cpu_buffer_type()); + if (graph_buf == nullptr) { + LLAMA_LOG_ERROR("%s: error: failed to allocate graph tensors\n", __func__); + ggml_free(lora_ctx); + ggml_backend_buffer_free(lora_buf); + ggml_backend_free(backend_cpu); + return 1; + } - // we won't need these tensors again, reset the context to save memory - lora_ctx.reset(ggml_init(params)); - lora_tensors.clear(); + ggml_backend_graph_compute(backend_cpu, gf); - n_tensors++; - if (n_tensors % 4 == 0) { - LLAMA_LOG_INFO("."); - } + ggml_backend_tensor_set(model_t, r->data, 0, ggml_nbytes(r)); + +#if 0 + // TODO: use scheduler with fallback to CPU for less copies between CPU and GPU + //ggml_backend_sched_t sched = ggml_backend_sched_new(backends.data(), backends.size(), GGML_DEFAULT_GRAPH_SIZE); + + // sched compute + ggml_build_forward_expand(gf, build_graph()); + ggml_backend_sched_init_measure(sched, gf); + + // create the graph again, since the previous one was destroyed by the measure + ggml_graph_clear(gf); + ggml_build_forward_expand(gf, build_graph()); + ggml_backend_sched_graph_compute(sched, gf); + ggml_backend_sched_free(sched); +#endif + + ggml_backend_buffer_free(lora_buf); + ggml_backend_buffer_free(graph_buf); + ggml_free(lora_ctx); + + n_tensors++; + if (n_tensors % 4 == 0) { + LLAMA_LOG_INFO("."); } } + ggml_backend_free(backend_cpu); + const int64_t t_lora_us = ggml_time_us() - t_start_lora_us; LLAMA_LOG_INFO(" done (%.2f ms)\n", t_lora_us / 1000.0); @@ -9579,6 +9082,7 @@ static int llama_apply_lora_from_file_internal( struct llama_model_params llama_model_default_params() { struct llama_model_params result = { /*.n_gpu_layers =*/ 0, + /*.split_mode =*/ LLAMA_SPLIT_LAYER, /*.main_gpu =*/ 0, /*.tensor_split =*/ nullptr, /*.progress_callback =*/ nullptr, @@ -9590,7 +9094,8 @@ struct llama_model_params llama_model_default_params() { }; #ifdef GGML_USE_METAL - result.n_gpu_layers = 1; + // note: we usually have plenty of VRAM, so by default offload all layers to the GPU + result.n_gpu_layers = 999; #endif return result; @@ -9780,41 +9285,53 @@ struct llama_context * llama_new_context_with_model( GGML_ASSERT(hparams.n_embd_head_k % ggml_blck_size(type_k) == 0); GGML_ASSERT(hparams.n_embd_head_v % ggml_blck_size(type_v) == 0); - // reserve memory for context buffers if (!hparams.vocab_only) { - // initialize backend + // initialize backends #ifdef GGML_USE_METAL if (model->n_gpu_layers > 0) { - ctx->backend = ggml_backend_metal_init(); - if (ctx->backend == nullptr) { + ctx->backend_metal = ggml_backend_metal_init(); + if (ctx->backend_metal == nullptr) { LLAMA_LOG_ERROR("%s: failed to initialize Metal backend\n", __func__); + llama_free(ctx); + return nullptr; } + ctx->backends.push_back(ctx->backend_metal); } -#elif defined(GGML_USE_CUBLAS) && defined(LLAMA_GGML_BACKEND_CUDA_TEST) - // for testing only +#elif defined(GGML_USE_CUBLAS) if (model->n_gpu_layers > 0) { - ctx->backend = ggml_backend_cuda_init(0); - if (ctx->backend == nullptr) { - LLAMA_LOG_ERROR("%s: failed to initialize CUDA backend\n", __func__); + // with split_mode LLAMA_SPLIT_NONE or LLAMA_SPLIT_ROW, only the main GPU backend is used + if (model->split_mode == LLAMA_SPLIT_NONE || model->split_mode == LLAMA_SPLIT_ROW) { + ggml_backend_t backend = ggml_backend_cuda_init(model->main_gpu); + if (backend == nullptr) { + LLAMA_LOG_ERROR("%s: failed to initialize CUDA%d backend\n", __func__, model->main_gpu); + llama_free(ctx); + return nullptr; + } + ctx->backends.push_back(backend); + } else { + // LLAMA_SPLIT_LAYER requires a backend for each GPU + for (int device = 0; device < ggml_backend_cuda_get_device_count(); ++device) { + ggml_backend_t backend = ggml_backend_cuda_init(device); + if (backend == nullptr) { + LLAMA_LOG_ERROR("%s: failed to initialize CUDA%d backend\n", __func__, device); + llama_free(ctx); + return nullptr; + } + ctx->backends.push_back(backend); + } } } #endif - - if (ctx->backend == nullptr && ggml_backend_buffer_is_host(model->buf)) { - ctx->backend = ggml_backend_cpu_init(); - if (ctx->backend == nullptr) { - LLAMA_LOG_ERROR("%s: failed to initialize CPU backend\n", __func__); - } - } - - if (ctx->backend == nullptr) { - LLAMA_LOG_ERROR("%s: failed to initialize a backend\n", __func__); - delete ctx; + ctx->backend_cpu = ggml_backend_cpu_init(); + if (ctx->backend_cpu == nullptr) { + LLAMA_LOG_ERROR("%s: failed to initialize CPU backend\n", __func__); + llama_free(ctx); return nullptr; } + ctx->backends.push_back(ctx->backend_cpu); - if (!llama_kv_cache_init(ctx->model.hparams, ctx->kv_self, type_k, type_v, - cparams.n_ctx, model->n_gpu_layers, cparams.offload_kqv)) { + if (!llama_kv_cache_init(ctx->kv_self, ctx->model, type_k, type_v, + cparams.n_ctx, cparams.offload_kqv)) { LLAMA_LOG_ERROR("%s: llama_kv_cache_init() failed for self-attention cache\n", __func__); llama_free(ctx); return nullptr; @@ -9850,11 +9367,22 @@ struct llama_context * llama_new_context_with_model( } { - // the compute buffer is used to store the tensor and graph structs, while the allocator buffer is used for the tensor data + // buffer types used for the compute buffer of each backend + std::vector backend_buft; + for (auto * backend : ctx->backends) { + if (ggml_backend_is_cpu(backend)) { + // use host buffers for the CPU backend compute buffer + backend_buft.push_back(llama_default_buffer_type_cpu(true)); + } else { + backend_buft.push_back(ggml_backend_get_default_buffer_type(backend)); + } + } + + // buffer used to store the computation graph and the tensor meta data ctx->buf_compute_meta.resize(ggml_tensor_overhead()*LLAMA_MAX_NODES + ggml_graph_overhead()); - // create measure allocator - ctx->alloc = ggml_allocr_new_measure_from_backend(ctx->backend); + ctx->sched = ggml_backend_sched_new(ctx->backends.data(), backend_buft.data(), ctx->backends.size(), LLAMA_MAX_NODES); + ctx->alloc = ggml_backend_sched_get_tallocr(ctx->sched, ctx->backend_cpu); // build worst-case graph int n_tokens = (int)std::min(cparams.n_ctx, cparams.n_batch); @@ -9862,50 +9390,19 @@ struct llama_context * llama_new_context_with_model( llama_token token = llama_token_bos(&ctx->model); // not actually used by llama_build_graph, but required to choose between token and embedding inputs graph ggml_cgraph * gf = llama_build_graph(*ctx, llama_batch_get_one(&token, n_tokens, n_past, 0)); - // measure memory requirements for the graph - size_t alloc_size = ggml_allocr_alloc_graph(ctx->alloc, gf); + // initialize scheduler with the worst-case graph + ggml_backend_sched_init_measure(ctx->sched, gf); + // note: the number of splits during measure is higher than during inference due to the kv shift + int n_splits = ggml_backend_sched_get_n_splits(ctx->sched); + LLAMA_LOG_INFO("%s: graph splits (measure): %d\n", __func__, n_splits); + ctx->alloc = ggml_backend_sched_get_tallocr(ctx->sched, ctx->backend_cpu); - LLAMA_LOG_INFO("%s: compute buffer total size = %.2f MiB\n", __func__, (ctx->buf_compute_meta.size() + alloc_size) / 1024.0 / 1024.0); - - // create allocator again with exact memory requirements - ggml_allocr_free(ctx->alloc); - - ctx->buf_alloc = ggml_backend_alloc_buffer(ctx->backend, alloc_size); - ctx->alloc = ggml_allocr_new_from_buffer(ctx->buf_alloc); -#if defined(GGML_USE_CUBLAS) && !defined(LLAMA_GGML_BACKEND_CUDA_TEST) - if (model->n_gpu_layers > 0) { - // the CPU buffer adds this padding in case the malloc buffer is not aligned, so we need to do the same for the GPU buffer, since we use the same offsets - ggml_cuda_set_scratch_size(alloc_size + 64); - LLAMA_LOG_INFO("%s: VRAM scratch buffer: %.2f MiB\n", __func__, alloc_size / 1024.0 / 1024.0); - - // calculate total VRAM usage - auto add_tensor = [](const ggml_tensor * t, size_t & size) { - if (t->backend == GGML_BACKEND_GPU || t->backend == GGML_BACKEND_GPU_SPLIT) { - size += ggml_nbytes(t); - } - }; - size_t model_vram_size = 0; - for (const auto & kv : model->tensors_by_name) { - add_tensor(kv.second, model_vram_size); - } - - size_t kv_vram_size = 0; - for (auto & k : ctx->kv_self.k_l) { - add_tensor(k, kv_vram_size); - } - for (auto & v : ctx->kv_self.v_l) { - add_tensor(v, kv_vram_size); - } - - size_t ctx_vram_size = alloc_size + kv_vram_size; - size_t total_vram_size = model_vram_size + ctx_vram_size; - - LLAMA_LOG_INFO("%s: total VRAM used: %.2f MiB (model: %.2f MiB, context: %.2f MiB)\n", __func__, - total_vram_size / 1024.0 / 1024.0, - model_vram_size / 1024.0 / 1024.0, - ctx_vram_size / 1024.0 / 1024.0); + for (ggml_backend_t backend : ctx->backends) { + ggml_backend_buffer_t buf = ggml_backend_sched_get_buffer(ctx->sched, backend); + LLAMA_LOG_INFO("%s: %10s compute buffer size = %8.2f MiB\n", __func__, + ggml_backend_buffer_name(buf), + ggml_backend_buffer_get_size(buf) / 1024.0 / 1024.0); } -#endif } } @@ -10002,9 +9499,8 @@ int32_t llama_model_meta_val_str_by_index(const struct llama_model * model, int3 } int32_t llama_model_desc(const struct llama_model * model, char * buf, size_t buf_size) { - return snprintf(buf, buf_size, "%s %s%s %s", + return snprintf(buf, buf_size, "%s %s %s", llama_model_arch_name(model->arch).c_str(), - model->hparams.n_expert > 0 ? (std::to_string(model->hparams.n_expert) + "x").c_str() : "", llama_model_type_name(model->type), llama_model_ftype_name(model->ftype).c_str()); } @@ -10026,7 +9522,14 @@ uint64_t llama_model_n_params(const struct llama_model * model) { } struct ggml_tensor * llama_get_model_tensor(struct llama_model * model, const char * name) { - return ggml_get_tensor(model->ctx, name); + auto it = std::find_if(model->tensors_by_name.begin(), model->tensors_by_name.end(), + [name](const std::pair & it) { + return it.first == name; + }); + if (it == model->tensors_by_name.end()) { + return nullptr; + } + return it->second; } uint32_t llama_model_quantize( @@ -10211,7 +9714,7 @@ size_t llama_get_state_size(const struct llama_context * ctx) { const size_t s_embedding = ctx->embedding.size() * sizeof(float); const size_t s_kv_size = sizeof(size_t); const size_t s_kv_ntok = sizeof(int); - const size_t s_kv = ggml_backend_buffer_get_size(ctx->kv_self.buf); + const size_t s_kv = ctx->kv_self.total_size(); const size_t s_total = ( + s_rng_size @@ -10340,7 +9843,7 @@ static void llama_copy_state_data_internal(struct llama_context * ctx, llama_dat const auto n_embd_v_gqa = hparams.n_embd_v_gqa(); const auto n_ctx = cparams.n_ctx; - const size_t kv_buf_size = ggml_backend_buffer_get_size(kv_self.buf); + const size_t kv_buf_size = kv_self.total_size(); const uint32_t kv_head = kv_self.head; const uint32_t kv_size = kv_self.size; const uint32_t kv_used = kv_self.used; @@ -10353,46 +9856,19 @@ static void llama_copy_state_data_internal(struct llama_context * ctx, llama_dat if (kv_buf_size) { const size_t elt_size = ggml_element_size(kv_self.k_l[0]); - ggml_context * cpy_ctx = ggml_init({ 6*n_layer*ggml_tensor_overhead() + ggml_graph_overhead(), NULL, /* no_alloc */ true }); - ggml_cgraph * gf = ggml_new_graph(cpy_ctx); - - std::vector kout2d(n_layer); - std::vector vout2d(n_layer); - - for (int il = 0; il < (int) n_layer; ++il) { - kout2d[il] = ggml_new_tensor_2d(cpy_ctx, kv_self.k_l[il]->type, n_embd_k_gqa, kv_head); - vout2d[il] = ggml_new_tensor_2d(cpy_ctx, kv_self.v_l[il]->type, kv_head, n_embd_v_gqa); - - ggml_tensor * k2d = ggml_view_2d(cpy_ctx, kv_self.k_l[il], - n_embd_k_gqa, kv_head, - elt_size*n_embd_k_gqa, 0); - - ggml_tensor * v2d = ggml_view_2d(cpy_ctx, kv_self.v_l[il], - kv_head, n_embd_v_gqa, - elt_size*n_ctx, 0); - - ggml_build_forward_expand(gf, ggml_cpy(cpy_ctx, k2d, kout2d[il])); - ggml_build_forward_expand(gf, ggml_cpy(cpy_ctx, v2d, vout2d[il])); - } - - ggml_backend_buffer_t buf = ggml_backend_alloc_ctx_tensors(cpy_ctx, ctx->backend); - - ggml_backend_graph_compute(ctx->backend, gf); - std::vector tmp_buf; for (int il = 0; il < (int) n_layer; ++il) { - tmp_buf.resize(ggml_nbytes(kout2d[il])); - ggml_backend_tensor_get(kout2d[il], tmp_buf.data(), 0, tmp_buf.size()); + tmp_buf.resize(elt_size*n_embd_k_gqa*kv_head); + ggml_backend_tensor_get(kv_self.k_l[il], tmp_buf.data(), 0, tmp_buf.size()); data_ctx->write(tmp_buf.data(), tmp_buf.size()); - tmp_buf.resize(ggml_nbytes(vout2d[il])); - ggml_backend_tensor_get(vout2d[il], tmp_buf.data(), 0, tmp_buf.size()); - data_ctx->write(tmp_buf.data(), tmp_buf.size()); + // v is not contiguous, copy row by row + tmp_buf.resize(elt_size*kv_head); + for (int ir = 0; ir < (int) n_embd_v_gqa; ++ir) { + ggml_backend_tensor_get(kv_self.v_l[il], tmp_buf.data(), ir*elt_size*n_ctx, tmp_buf.size()); + data_ctx->write(tmp_buf.data(), tmp_buf.size()); + } } - - ggml_free(cpy_ctx); - - ggml_backend_buffer_free(buf); } for (uint32_t i = 0; i < kv_size; ++i) { @@ -10491,48 +9967,22 @@ size_t llama_set_state_data(struct llama_context * ctx, uint8_t * src) { memcpy(&kv_used, inp, sizeof(kv_used)); inp += sizeof(kv_used); if (kv_buf_size) { - GGML_ASSERT(ggml_backend_buffer_get_size(kv_self.buf) == kv_buf_size); + GGML_ASSERT(kv_self.total_size() == kv_buf_size); const size_t elt_size = ggml_element_size(kv_self.k_l[0]); - ggml_context * cpy_ctx = ggml_init({ 6*n_layer*ggml_tensor_overhead() + ggml_graph_overhead(), NULL, /* no_alloc */ true }); - ggml_cgraph * gf = ggml_new_graph(cpy_ctx); + for (int il = 0; il < (int) n_layer; ++il) { + size_t k_size = elt_size*n_embd_k_gqa*kv_head; + ggml_backend_tensor_set(kv_self.k_l[il], inp, 0, k_size); + inp += k_size; - std::vector kin2d(n_layer); - std::vector vin2d(n_layer); - - for (int il = 0; il < n_layer; ++il) { - kin2d[il] = ggml_new_tensor_2d(cpy_ctx, kv_self.k_l[il]->type, n_embd_k_gqa, kv_head); - vin2d[il] = ggml_new_tensor_2d(cpy_ctx, kv_self.v_l[il]->type, kv_head, n_embd_v_gqa); - - ggml_tensor * k2d = ggml_view_2d(cpy_ctx, kv_self.k_l[il], - n_embd_k_gqa, kv_head, - elt_size*n_embd_k_gqa, 0); - - ggml_tensor * v2d = ggml_view_2d(cpy_ctx, kv_self.v_l[il], - kv_head, n_embd_v_gqa, - elt_size*n_ctx, 0); - - ggml_build_forward_expand(gf, ggml_cpy(cpy_ctx, kin2d[il], k2d)); - ggml_build_forward_expand(gf, ggml_cpy(cpy_ctx, vin2d[il], v2d)); + // v is not contiguous, copy row by row + size_t v_row_size = elt_size*kv_head; + for (int ir = 0; ir < (int) n_embd_v_gqa; ++ir) { + ggml_backend_tensor_set(kv_self.v_l[il], inp, ir*elt_size*n_ctx, v_row_size); + inp += v_row_size; + } } - - ggml_backend_buffer_t buf = ggml_backend_alloc_ctx_tensors(cpy_ctx, ctx->backend); - - // load data into the tensors - for (int il = 0; il < n_layer; ++il) { - ggml_backend_tensor_set(kin2d[il], inp, 0, ggml_nbytes(kin2d[il])); - inp += ggml_nbytes(kin2d[il]); - - ggml_backend_tensor_set(vin2d[il], inp, 0, ggml_nbytes(vin2d[il])); - inp += ggml_nbytes(vin2d[il]); - } - - ggml_backend_graph_compute(ctx->backend, gf); - - ggml_free(cpy_ctx); - - ggml_backend_buffer_free(buf); } ctx->kv_self.head = kv_head; diff --git a/llama.h b/llama.h index 43d41b8f6..689e12d7c 100644 --- a/llama.h +++ b/llama.h @@ -118,6 +118,12 @@ extern "C" { LLAMA_ROPE_SCALING_MAX_VALUE = LLAMA_ROPE_SCALING_YARN, }; + enum llama_split_mode { + LLAMA_SPLIT_NONE = 0, // single GPU + LLAMA_SPLIT_LAYER = 1, // split layers and KV across GPUs + LLAMA_SPLIT_ROW = 2, // split rows across GPUs + }; + typedef struct llama_token_data { llama_token id; // token id float logit; // log-odds of the token @@ -180,8 +186,16 @@ extern "C" { struct llama_model_params { int32_t n_gpu_layers; // number of layers to store in VRAM - int32_t main_gpu; // the GPU that is used for scratch and small tensors - const float * tensor_split; // how to split layers across multiple GPUs (size: LLAMA_MAX_DEVICES) + enum llama_split_mode split_mode; // how to split the model across multiple GPUs + + // main_gpu interpretation depends on split_mode: + // LLAMA_SPLIT_NONE: the GPU that is used for the entire model + // LLAMA_SPLIT_ROW: the GPU that is used for small tensors and intermediate results + // LLAMA_SPLIT_LAYER: ignored + int32_t main_gpu; + + // proportion of the model (layers or rows) to offload to each GPU, size: LLAMA_MAX_DEVICES + const float * tensor_split; // Called with a progress value between 0.0 and 1.0. Pass NULL to disable. // If the provided progress_callback returns true, model loading continues. diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index 7a60d7743..d9b8b106a 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -376,6 +376,11 @@ struct test_case { // allocate ggml_backend_buffer_t buf = ggml_backend_alloc_ctx_tensors(ctx, backend1); + if (buf == NULL) { + printf("failed to allocate tensors [%s] ", ggml_backend_name(backend1)); + ggml_free(ctx); + return false; + } // build graph ggml_build_forward_expand(gf, out); @@ -463,19 +468,23 @@ struct test_case { GGML_UNUSED(index); }; - ggml_backend_compare_graph_backend(backend1, backend2, gf, callback, &ud); + const bool cmp_ok = ggml_backend_compare_graph_backend(backend1, backend2, gf, callback, &ud); - if (ud.ok) { - printf("\033[1;32mOK\033[0m\n"); - } else { - printf("\033[1;31mFAIL\033[0m\n"); + if (!cmp_ok) { + printf("compare failed "); } ggml_backend_buffer_free(buf); ggml_free(ctx); - return ud.ok; + if (ud.ok && cmp_ok) { + printf("\033[1;32mOK\033[0m\n"); + return true; + } + + printf("\033[1;31mFAIL\033[0m\n"); + return false; } bool eval_perf(ggml_backend_t backend, const char * op_name) { @@ -519,6 +528,11 @@ struct test_case { // allocate ggml_backend_buffer_t buf = ggml_backend_alloc_ctx_tensors(ctx, backend); + if (buf == NULL) { + printf("failed to allocate tensors\n"); + ggml_free(ctx); + return false; + } // randomize tensors initialize_tensors(ctx); From 3fe81781e3bf98b8e44946240a19f3a6ad08a11a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20G=C3=A4=C3=9Fler?= Date: Fri, 12 Jan 2024 20:38:54 +0100 Subject: [PATCH 040/812] CUDA: faster q8_0 -> f16 dequantization (#4895) --- ggml-cuda.cu | 57 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/ggml-cuda.cu b/ggml-cuda.cu index 2db50437c..bd3814c72 100644 --- a/ggml-cuda.cu +++ b/ggml-cuda.cu @@ -523,6 +523,8 @@ static_assert(sizeof(block_iq2_xs) == sizeof(ggml_fp16_t) + QK_K/8*sizeof(uint16 #define CUDA_ACC_BLOCK_SIZE 256 #define CUDA_IM2COL_BLOCK_SIZE 256 +#define CUDA_Q8_0_NE_ALIGN 2048 + // dmmv = dequantize_mul_mat_vec #ifndef GGML_CUDA_DMMV_X #define GGML_CUDA_DMMV_X 32 @@ -2327,6 +2329,45 @@ static __global__ void convert_unary(const void * __restrict__ vx, dst_t * __res y[i] = x[i]; } +template +static __global__ void dequantize_block_q8_0_f16(const void * __restrict__ vx, half * __restrict__ y, const int k) { +#if __CUDA_ARCH__ >= CC_PASCAL + constexpr int nint = CUDA_Q8_0_NE_ALIGN/sizeof(int) + WARP_SIZE; + + const int i0 = CUDA_Q8_0_NE_ALIGN*blockIdx.x; + const int * x0 = ((int *) vx) + blockIdx.x * nint; + half2 * y2 = (half2 *) (y + i0); + + __shared__ int vals[nint]; + +#pragma unroll + for (int ix0 = 0; ix0 < nint; ix0 += WARP_SIZE) { + if (need_check && i0*sizeof(block_q8_0)/QK8_0 + sizeof(int)*(ix0 + threadIdx.x) >= k*sizeof(block_q8_0)/QK8_0) { + break; + } + + const int ix = ix0 + threadIdx.x; + vals[ix] = x0[ix]; + } + +#pragma unroll + for (int iy = 0; iy < CUDA_Q8_0_NE_ALIGN; iy += 2*WARP_SIZE) { + if (need_check && i0 + iy + 2*threadIdx.x >= k) { + return; + } + + const half * b0 = ((const half *) vals) + (sizeof(block_q8_0)/sizeof(half)) * ((iy + 2*threadIdx.x)/QK8_0); + const half d = *b0; + const char2 qs = ((const char2 *) (b0 + 1))[threadIdx.x % (QK8_0/2)]; + + y2[iy/2 + threadIdx.x] = __hmul2(make_half2(qs.x, qs.y), __half2half2(d)); + } +#else + (void) vx; (void) y; (void) k; + bad_arch(); +#endif // __CUDA_ARCH__ >= CC_PASCAL +} + // VDR = vec dot ratio, how many contiguous integers each thread processes when the vec dot kernel is called // MMVQ = mul_mat_vec_q, MMQ = mul_mat_q @@ -6181,6 +6222,17 @@ static void dequantize_block_cuda(const void * __restrict__ vx, dst_t * __restri dequantize_block<<>>(vx, y, k); } +static void dequantize_block_q8_0_f16_cuda(const void * __restrict__ vx, half * __restrict__ y, const int k, cudaStream_t stream) { + const int num_blocks = (k + CUDA_Q8_0_NE_ALIGN - 1) / CUDA_Q8_0_NE_ALIGN; + if (k % CUDA_Q8_0_NE_ALIGN == 0) { + const bool need_check = false; + dequantize_block_q8_0_f16<<>>(vx, y, k); + } else { + const bool need_check = true; + dequantize_block_q8_0_f16<<>>(vx, y, k); + } +} + template static void dequantize_row_q2_K_cuda(const void * vx, dst_t * y, const int k, cudaStream_t stream) { const int nb = k / QK_K; @@ -6246,6 +6298,7 @@ static void convert_unary_cuda(const void * __restrict__ vx, dst_t * __restrict_ } static to_fp16_cuda_t ggml_get_to_fp16_cuda(ggml_type type) { + int id; switch (type) { case GGML_TYPE_Q4_0: return dequantize_block_cuda; @@ -6256,6 +6309,10 @@ static to_fp16_cuda_t ggml_get_to_fp16_cuda(ggml_type type) { case GGML_TYPE_Q5_1: return dequantize_block_cuda; case GGML_TYPE_Q8_0: + CUDA_CHECK(cudaGetDevice(&id)); + if (g_device_caps[id].cc >= CC_PASCAL) { + return dequantize_block_q8_0_f16_cuda; + } return dequantize_block_cuda; case GGML_TYPE_Q2_K: return dequantize_row_q2_K_cuda; From 52ee4540c0f2e11d52c839db6eb51d014ce060e1 Mon Sep 17 00:00:00 2001 From: Maximilian Winter Date: Fri, 12 Jan 2024 20:46:45 +0100 Subject: [PATCH 041/812] examples : add pydantic models to GBNF grammar generator (#4883) * Create pydantic-models-to-grammar.py * Added some comments for usage * Refactored Grammar Generator Added example and usage instruction. * Update pydantic_models_to_grammar.py * Update pydantic-models-to-grammar-examples.py * Renamed module and imported it. * Update pydantic-models-to-grammar.py * Renamed file and fixed grammar generator issue. --- .../pydantic-models-to-grammar-examples.py | 136 ++ examples/pydantic_models_to_grammar.py | 1151 +++++++++++++++++ 2 files changed, 1287 insertions(+) create mode 100644 examples/pydantic-models-to-grammar-examples.py create mode 100644 examples/pydantic_models_to_grammar.py diff --git a/examples/pydantic-models-to-grammar-examples.py b/examples/pydantic-models-to-grammar-examples.py new file mode 100644 index 000000000..a8a4919cf --- /dev/null +++ b/examples/pydantic-models-to-grammar-examples.py @@ -0,0 +1,136 @@ +# Function calling example using pydantic models. + +import json +from enum import Enum +from typing import Union, Optional + +import requests +from pydantic import BaseModel, Field + +import importlib +from pydantic_models_to_grammar import generate_gbnf_grammar_and_documentation + +# Function to get completion on the llama.cpp server with grammar. +def create_completion(prompt, grammar): + headers = {"Content-Type": "application/json"} + data = {"prompt": prompt, "grammar": grammar} + + response = requests.post("http://127.0.0.1:8080/completion", headers=headers, json=data) + data = response.json() + + print(data["content"]) + return data["content"] + + +# A function for the agent to send a message to the user. +class SendMessageToUser(BaseModel): + """ + Send a message to the User. + """ + chain_of_thought: str = Field(..., description="Your chain of thought while sending the message.") + message: str = Field(..., description="Message you want to send to the user.") + + def run(self): + print(self.message) + + +# Enum for the calculator function. +class MathOperation(Enum): + ADD = "add" + SUBTRACT = "subtract" + MULTIPLY = "multiply" + DIVIDE = "divide" + + +# Very simple calculator tool for the agent. +class Calculator(BaseModel): + """ + Perform a math operation on two numbers. + """ + number_one: Union[int, float] = Field(..., description="First number.") + operation: MathOperation = Field(..., description="Math operation to perform.") + number_two: Union[int, float] = Field(..., description="Second number.") + + def run(self): + if self.operation == MathOperation.ADD: + return self.number_one + self.number_two + elif self.operation == MathOperation.SUBTRACT: + return self.number_one - self.number_two + elif self.operation == MathOperation.MULTIPLY: + return self.number_one * self.number_two + elif self.operation == MathOperation.DIVIDE: + return self.number_one / self.number_two + else: + raise ValueError("Unknown operation.") + + +# Here the grammar gets generated by passing the available function models to generate_gbnf_grammar_and_documentation function. This also generates a documentation usable by the LLM. +# pydantic_model_list is the list of pydanitc models +# outer_object_name is an optional name for an outer object around the actual model object. Like a "function" object with "function_parameters" which contains the actual model object. If None, no outer object will be generated +# outer_object_content is the name of outer object content. +# model_prefix is the optional prefix for models in the documentation. (Default="Output Model") +# fields_prefix is the prefix for the model fields in the documentation. (Default="Output Fields") +gbnf_grammar, documentation = generate_gbnf_grammar_and_documentation( + pydantic_model_list=[SendMessageToUser, Calculator], outer_object_name="function", + outer_object_content="function_parameters", model_prefix="Function", fields_prefix="Parameters") + +print(gbnf_grammar) +print(documentation) + +system_message = "You are an advanced AI, tasked to assist the user by calling functions in JSON format. The following are the available functions and their parameters and types:\n\n" + documentation + +user_message = "What is 42 * 42?" +prompt = f"<|im_start|>system\n{system_message}<|im_end|>\n<|im_start|>user\n{user_message}<|im_end|>\n<|im_start|>assistant" + +text = create_completion(prompt=prompt, grammar=gbnf_grammar) +# This should output something like this: +# { +# "function": "calculator", +# "function_parameters": { +# "number_one": 42, +# "operation": "multiply", +# "number_two": 42 +# } +# } +function_dictionary = json.loads(text) +if function_dictionary["function"] == "calculator": + function_parameters = {**function_dictionary["function_parameters"]} + + print(Calculator(**function_parameters).run()) + # This should output: 1764 + + +# A example structured output based on pydantic models. The LLM will create an entry for a Book database out of an unstructured text. +class Category(Enum): + """ + The category of the book. + """ + Fiction = "Fiction" + NonFiction = "Non-Fiction" + + +class Book(BaseModel): + """ + Represents an entry about a book. + """ + title: str = Field(..., description="Title of the book.") + author: str = Field(..., description="Author of the book.") + published_year: Optional[int] = Field(..., description="Publishing year of the book.") + keywords: list[str] = Field(..., description="A list of keywords.") + category: Category = Field(..., description="Category of the book.") + summary: str = Field(..., description="Summary of the book.") + + +# We need no additional parameters other than our list of pydantic models. +gbnf_grammar, documentation = generate_gbnf_grammar_and_documentation([Book]) + +system_message = "You are an advanced AI, tasked to create a dataset entry in JSON for a Book. The following is the expected output model:\n\n" + documentation + +text = """The Feynman Lectures on Physics is a physics textbook based on some lectures by Richard Feynman, a Nobel laureate who has sometimes been called "The Great Explainer". The lectures were presented before undergraduate students at the California Institute of Technology (Caltech), during 1961–1963. The book's co-authors are Feynman, Robert B. Leighton, and Matthew Sands.""" +prompt = f"<|im_start|>system\n{system_message}<|im_end|>\n<|im_start|>user\n{text}<|im_end|>\n<|im_start|>assistant" + +text = create_completion(prompt=prompt, grammar=gbnf_grammar) + +json_data = json.loads(text) + +print(Book(**json_data)) diff --git a/examples/pydantic_models_to_grammar.py b/examples/pydantic_models_to_grammar.py new file mode 100644 index 000000000..41b98fdc1 --- /dev/null +++ b/examples/pydantic_models_to_grammar.py @@ -0,0 +1,1151 @@ +import inspect +import json +from copy import copy +from inspect import isclass, getdoc +from types import NoneType + +from pydantic import BaseModel, create_model, Field +from typing import Any, Type, List, get_args, get_origin, Tuple, Union, Optional, _GenericAlias +from enum import Enum +from typing import get_type_hints, Callable +import re + + +class PydanticDataType(Enum): + """ + Defines the data types supported by the grammar_generator. + + Attributes: + STRING (str): Represents a string data type. + BOOLEAN (str): Represents a boolean data type. + INTEGER (str): Represents an integer data type. + FLOAT (str): Represents a float data type. + OBJECT (str): Represents an object data type. + ARRAY (str): Represents an array data type. + ENUM (str): Represents an enum data type. + CUSTOM_CLASS (str): Represents a custom class data type. + """ + STRING = "string" + TRIPLE_QUOTED_STRING = "triple_quoted_string" + MARKDOWN_STRING = "markdown_string" + BOOLEAN = "boolean" + INTEGER = "integer" + FLOAT = "float" + OBJECT = "object" + ARRAY = "array" + ENUM = "enum" + ANY = "any" + NULL = "null" + CUSTOM_CLASS = "custom-class" + CUSTOM_DICT = "custom-dict" + SET = "set" + + +def map_pydantic_type_to_gbnf(pydantic_type: Type[Any]) -> str: + if isclass(pydantic_type) and issubclass(pydantic_type, str): + return PydanticDataType.STRING.value + elif isclass(pydantic_type) and issubclass(pydantic_type, bool): + return PydanticDataType.BOOLEAN.value + elif isclass(pydantic_type) and issubclass(pydantic_type, int): + return PydanticDataType.INTEGER.value + elif isclass(pydantic_type) and issubclass(pydantic_type, float): + return PydanticDataType.FLOAT.value + elif isclass(pydantic_type) and issubclass(pydantic_type, Enum): + return PydanticDataType.ENUM.value + + elif isclass(pydantic_type) and issubclass(pydantic_type, BaseModel): + return format_model_and_field_name(pydantic_type.__name__) + elif get_origin(pydantic_type) == list: + element_type = get_args(pydantic_type)[0] + return f"{map_pydantic_type_to_gbnf(element_type)}-list" + elif get_origin(pydantic_type) == set: + element_type = get_args(pydantic_type)[0] + return f"{map_pydantic_type_to_gbnf(element_type)}-set" + elif get_origin(pydantic_type) == Union: + union_types = get_args(pydantic_type) + union_rules = [map_pydantic_type_to_gbnf(ut) for ut in union_types] + return f"union-{'-or-'.join(union_rules)}" + elif get_origin(pydantic_type) == Optional: + element_type = get_args(pydantic_type)[0] + return f"optional-{map_pydantic_type_to_gbnf(element_type)}" + elif isclass(pydantic_type): + return f"{PydanticDataType.CUSTOM_CLASS.value}-{format_model_and_field_name(pydantic_type.__name__)}" + elif get_origin(pydantic_type) == dict: + key_type, value_type = get_args(pydantic_type) + return f"custom-dict-key-type-{format_model_and_field_name(map_pydantic_type_to_gbnf(key_type))}-value-type-{format_model_and_field_name(map_pydantic_type_to_gbnf(value_type))}" + else: + return "unknown" + + +def format_model_and_field_name(model_name: str) -> str: + parts = re.findall('[A-Z][^A-Z]*', model_name) + if not parts: # Check if the list is empty + return model_name.lower().replace("_", "-") + return '-'.join(part.lower().replace("_", "-") for part in parts) + + +def generate_list_rule(element_type): + """ + Generate a GBNF rule for a list of a given element type. + + :param element_type: The type of the elements in the list (e.g., 'string'). + :return: A string representing the GBNF rule for a list of the given type. + """ + rule_name = f"{map_pydantic_type_to_gbnf(element_type)}-list" + element_rule = map_pydantic_type_to_gbnf(element_type) + list_rule = fr'{rule_name} ::= "[" {element_rule} ("," {element_rule})* "]"' + return list_rule + + +def get_members_structure(cls, rule_name): + if issubclass(cls, Enum): + # Handle Enum types + members = [f'\"\\\"{member.value}\\\"\"' for name, member in cls.__members__.items()] + return f"{cls.__name__.lower()} ::= " + " | ".join(members) + if cls.__annotations__ and cls.__annotations__ != {}: + result = f'{rule_name} ::= "{{"' + type_list_rules = [] + # Modify this comprehension + members = [f' \"\\\"{name}\\\"\" ":" {map_pydantic_type_to_gbnf(param_type)}' + for name, param_type in cls.__annotations__.items() + if name != 'self'] + + result += '"," '.join(members) + result += ' "}"' + return result, type_list_rules + elif rule_name == "custom-class-any": + result = f'{rule_name} ::= ' + result += 'value' + type_list_rules = [] + return result, type_list_rules + else: + init_signature = inspect.signature(cls.__init__) + parameters = init_signature.parameters + result = f'{rule_name} ::= "{{"' + type_list_rules = [] + # Modify this comprehension too + members = [f' \"\\\"{name}\\\"\" ":" {map_pydantic_type_to_gbnf(param.annotation)}' + for name, param in parameters.items() + if name != 'self' and param.annotation != inspect.Parameter.empty] + + result += '", "'.join(members) + result += ' "}"' + return result, type_list_rules + + +def regex_to_gbnf(regex_pattern: str) -> str: + """ + Translate a basic regex pattern to a GBNF rule. + Note: This function handles only a subset of simple regex patterns. + """ + gbnf_rule = regex_pattern + + # Translate common regex components to GBNF + gbnf_rule = gbnf_rule.replace('\\d', '[0-9]') + gbnf_rule = gbnf_rule.replace('\\s', '[ \t\n]') + + # Handle quantifiers and other regex syntax that is similar in GBNF + # (e.g., '*', '+', '?', character classes) + + return gbnf_rule + + +def generate_gbnf_integer_rules(max_digit=None, min_digit=None): + """ + + Generate GBNF Integer Rules + + Generates GBNF (Generalized Backus-Naur Form) rules for integers based on the given maximum and minimum digits. + + Parameters: + max_digit (int): The maximum number of digits for the integer. Default is None. + min_digit (int): The minimum number of digits for the integer. Default is None. + + Returns: + integer_rule (str): The identifier for the integer rule generated. + additional_rules (list): A list of additional rules generated based on the given maximum and minimum digits. + + """ + additional_rules = [] + + # Define the rule identifier based on max_digit and min_digit + integer_rule = "integer-part" + if max_digit is not None: + integer_rule += f"-max{max_digit}" + if min_digit is not None: + integer_rule += f"-min{min_digit}" + + # Handling Integer Rules + if max_digit is not None or min_digit is not None: + # Start with an empty rule part + integer_rule_part = '' + + # Add mandatory digits as per min_digit + if min_digit is not None: + integer_rule_part += '[0-9] ' * min_digit + + # Add optional digits up to max_digit + if max_digit is not None: + optional_digits = max_digit - (min_digit if min_digit is not None else 0) + integer_rule_part += ''.join(['[0-9]? ' for _ in range(optional_digits)]) + + # Trim the rule part and append it to additional rules + integer_rule_part = integer_rule_part.strip() + if integer_rule_part: + additional_rules.append(f'{integer_rule} ::= {integer_rule_part}') + + return integer_rule, additional_rules + + +def generate_gbnf_float_rules(max_digit=None, min_digit=None, max_precision=None, min_precision=None): + """ + Generate GBNF float rules based on the given constraints. + + :param max_digit: Maximum number of digits in the integer part (default: None) + :param min_digit: Minimum number of digits in the integer part (default: None) + :param max_precision: Maximum number of digits in the fractional part (default: None) + :param min_precision: Minimum number of digits in the fractional part (default: None) + :return: A tuple containing the float rule and additional rules as a list + + Example Usage: + max_digit = 3 + min_digit = 1 + max_precision = 2 + min_precision = 1 + generate_gbnf_float_rules(max_digit, min_digit, max_precision, min_precision) + + Output: + ('float-3-1-2-1', ['integer-part-max3-min1 ::= [0-9] [0-9] [0-9]?', 'fractional-part-max2-min1 ::= [0-9] [0-9]?', 'float-3-1-2-1 ::= integer-part-max3-min1 "." fractional-part-max2-min + *1']) + + Note: + GBNF stands for Generalized Backus-Naur Form, which is a notation technique to specify the syntax of programming languages or other formal grammars. + """ + additional_rules = [] + + # Define the integer part rule + integer_part_rule = "integer-part" + (f"-max{max_digit}" if max_digit is not None else "") + ( + f"-min{min_digit}" if min_digit is not None else "") + + # Define the fractional part rule based on precision constraints + fractional_part_rule = "fractional-part" + fractional_rule_part = '' + if max_precision is not None or min_precision is not None: + fractional_part_rule += (f"-max{max_precision}" if max_precision is not None else "") + ( + f"-min{min_precision}" if min_precision is not None else "") + # Minimum number of digits + fractional_rule_part = '[0-9]' * (min_precision if min_precision is not None else 1) + # Optional additional digits + fractional_rule_part += ''.join([' [0-9]?'] * ( + (max_precision - (min_precision if min_precision is not None else 1)) if max_precision is not None else 0)) + additional_rules.append(f'{fractional_part_rule} ::= {fractional_rule_part}') + + # Define the float rule + float_rule = f"float-{max_digit if max_digit is not None else 'X'}-{min_digit if min_digit is not None else 'X'}-{max_precision if max_precision is not None else 'X'}-{min_precision if min_precision is not None else 'X'}" + additional_rules.append(f'{float_rule} ::= {integer_part_rule} "." {fractional_part_rule}') + + # Generating the integer part rule definition, if necessary + if max_digit is not None or min_digit is not None: + integer_rule_part = '[0-9]' + if min_digit is not None and min_digit > 1: + integer_rule_part += ' [0-9]' * (min_digit - 1) + if max_digit is not None: + integer_rule_part += ''.join([' [0-9]?'] * (max_digit - (min_digit if min_digit is not None else 1))) + additional_rules.append(f'{integer_part_rule} ::= {integer_rule_part.strip()}') + + return float_rule, additional_rules + + +def generate_gbnf_rule_for_type(model_name, field_name, + field_type, is_optional, processed_models, created_rules, + field_info=None) -> \ + Tuple[str, list]: + """ + Generate GBNF rule for a given field type. + + :param model_name: Name of the model. + + :param field_name: Name of the field. + :param field_type: Type of the field. + :param is_optional: Whether the field is optional. + :param processed_models: List of processed models. + :param created_rules: List of created rules. + :param field_info: Additional information about the field (optional). + + :return: Tuple containing the GBNF type and a list of additional rules. + :rtype: Tuple[str, list] + """ + rules = [] + + field_name = format_model_and_field_name(field_name) + gbnf_type = map_pydantic_type_to_gbnf(field_type) + + if isclass(field_type) and issubclass(field_type, BaseModel): + nested_model_name = format_model_and_field_name(field_type.__name__) + nested_model_rules = generate_gbnf_grammar(field_type, processed_models, created_rules) + rules.extend(nested_model_rules) + gbnf_type, rules = nested_model_name, rules + elif isclass(field_type) and issubclass(field_type, Enum): + enum_values = [f'\"\\\"{e.value}\\\"\"' for e in field_type] # Adding escaped quotes + enum_rule = f"{model_name}-{field_name} ::= {' | '.join(enum_values)}" + rules.append(enum_rule) + gbnf_type, rules = model_name + "-" + field_name, rules + elif get_origin(field_type) == list or field_type == list: # Array + element_type = get_args(field_type)[0] + element_rule_name, additional_rules = generate_gbnf_rule_for_type(model_name, + f"{field_name}-element", + element_type, is_optional, processed_models, + created_rules) + rules.extend(additional_rules) + array_rule = f"""{model_name}-{field_name} ::= "[" ws {element_rule_name} ("," ws {element_rule_name})* "]" """ + rules.append(array_rule) + gbnf_type, rules = model_name + "-" + field_name, rules + + elif get_origin(field_type) == set or field_type == set: # Array + element_type = get_args(field_type)[0] + element_rule_name, additional_rules = generate_gbnf_rule_for_type(model_name, + f"{field_name}-element", + element_type, is_optional, processed_models, + created_rules) + rules.extend(additional_rules) + array_rule = f"""{model_name}-{field_name} ::= "[" ws {element_rule_name} ("," ws {element_rule_name})* "]" """ + rules.append(array_rule) + gbnf_type, rules = model_name + "-" + field_name, rules + + elif gbnf_type.startswith("custom-class-"): + nested_model_rules, field_types = get_members_structure(field_type, gbnf_type) + rules.append(nested_model_rules) + elif gbnf_type.startswith("custom-dict-"): + key_type, value_type = get_args(field_type) + + additional_key_type, additional_key_rules = generate_gbnf_rule_for_type(model_name, + f"{field_name}-key-type", + key_type, is_optional, processed_models, + created_rules) + additional_value_type, additional_value_rules = generate_gbnf_rule_for_type(model_name, + f"{field_name}-value-type", + value_type, is_optional, + processed_models, created_rules) + gbnf_type = fr'{gbnf_type} ::= "{{" ( {additional_key_type} ":" {additional_value_type} ("," {additional_key_type} ":" {additional_value_type})* )? "}}" ' + + rules.extend(additional_key_rules) + rules.extend(additional_value_rules) + elif gbnf_type.startswith("union-"): + union_types = get_args(field_type) + union_rules = [] + + for union_type in union_types: + if isinstance(union_type, _GenericAlias): + union_gbnf_type, union_rules_list = generate_gbnf_rule_for_type(model_name, + field_name, union_type, + False, + processed_models, created_rules) + union_rules.append(union_gbnf_type) + rules.extend(union_rules_list) + + + elif not issubclass(union_type, NoneType): + union_gbnf_type, union_rules_list = generate_gbnf_rule_for_type(model_name, + field_name, union_type, + False, + processed_models, created_rules) + union_rules.append(union_gbnf_type) + rules.extend(union_rules_list) + + # Defining the union grammar rule separately + if len(union_rules) == 1: + union_grammar_rule = f"{model_name}-{field_name}-optional ::= {' | '.join(union_rules)} | null" + else: + union_grammar_rule = f"{model_name}-{field_name}-union ::= {' | '.join(union_rules)}" + rules.append(union_grammar_rule) + if len(union_rules) == 1: + gbnf_type = f"{model_name}-{field_name}-optional" + else: + gbnf_type = f"{model_name}-{field_name}-union" + elif isclass(field_type) and issubclass(field_type, str): + if field_info and hasattr(field_info, 'json_schema_extra') and field_info.json_schema_extra is not None: + + triple_quoted_string = field_info.json_schema_extra.get('triple_quoted_string', False) + markdown_string = field_info.json_schema_extra.get('markdown_string', False) + + gbnf_type = PydanticDataType.TRIPLE_QUOTED_STRING.value if triple_quoted_string else PydanticDataType.STRING.value + gbnf_type = PydanticDataType.MARKDOWN_STRING.value if markdown_string else gbnf_type + + elif field_info and hasattr(field_info, 'pattern'): + # Convert regex pattern to grammar rule + regex_pattern = field_info.regex.pattern + gbnf_type = f"pattern-{field_name} ::= {regex_to_gbnf(regex_pattern)}" + else: + gbnf_type = PydanticDataType.STRING.value + + elif isclass(field_type) and issubclass(field_type, float) and field_info and hasattr(field_info, + 'json_schema_extra') and field_info.json_schema_extra is not None: + # Retrieve precision attributes for floats + max_precision = field_info.json_schema_extra.get('max_precision') if field_info and hasattr(field_info, + 'json_schema_extra') else None + min_precision = field_info.json_schema_extra.get('min_precision') if field_info and hasattr(field_info, + 'json_schema_extra') else None + max_digits = field_info.json_schema_extra.get('max_digit') if field_info and hasattr(field_info, + 'json_schema_extra') else None + min_digits = field_info.json_schema_extra.get('min_digit') if field_info and hasattr(field_info, + 'json_schema_extra') else None + + # Generate GBNF rule for float with given attributes + gbnf_type, rules = generate_gbnf_float_rules(max_digit=max_digits, min_digit=min_digits, + max_precision=max_precision, + min_precision=min_precision) + + elif isclass(field_type) and issubclass(field_type, int) and field_info and hasattr(field_info, + 'json_schema_extra') and field_info.json_schema_extra is not None: + # Retrieve digit attributes for integers + max_digits = field_info.json_schema_extra.get('max_digit') if field_info and hasattr(field_info, + 'json_schema_extra') else None + min_digits = field_info.json_schema_extra.get('min_digit') if field_info and hasattr(field_info, + 'json_schema_extra') else None + + # Generate GBNF rule for integer with given attributes + gbnf_type, rules = generate_gbnf_integer_rules(max_digit=max_digits, min_digit=min_digits) + else: + gbnf_type, rules = gbnf_type, [] + + if gbnf_type not in created_rules: + return gbnf_type, rules + else: + if gbnf_type in created_rules: + return gbnf_type, rules + + +def generate_gbnf_grammar(model: Type[BaseModel], processed_models: set, created_rules: dict) -> (list, bool, bool): + """ + + Generate GBnF Grammar + + Generates a GBnF grammar for a given model. + + :param model: A Pydantic model class to generate the grammar for. Must be a subclass of BaseModel. + :param processed_models: A set of already processed models to prevent infinite recursion. + :param created_rules: A dict containing already created rules to prevent duplicates. + :return: A list of GBnF grammar rules in string format. And two booleans indicating if an extra markdown or triple quoted string is in the grammar. + Example Usage: + ``` + model = MyModel + processed_models = set() + created_rules = dict() + + gbnf_grammar = generate_gbnf_grammar(model, processed_models, created_rules) + ``` + """ + if model in processed_models: + return [] + + processed_models.add(model) + model_name = format_model_and_field_name(model.__name__) + + if not issubclass(model, BaseModel): + # For non-Pydantic classes, generate model_fields from __annotations__ or __init__ + if hasattr(model, '__annotations__') and model.__annotations__: + model_fields = {name: (typ, ...) for name, typ in model.__annotations__.items()} + else: + init_signature = inspect.signature(model.__init__) + parameters = init_signature.parameters + model_fields = {name: (param.annotation, param.default) for name, param in parameters.items() + if name != 'self'} + else: + # For Pydantic models, use model_fields and check for ellipsis (required fields) + model_fields = model.__annotations__ + + model_rule_parts = [] + nested_rules = [] + has_markdown_code_block = False + has_triple_quoted_string = False + look_for_markdown_code_block = False + look_for_triple_quoted_string = False + for field_name, field_info in model_fields.items(): + if not issubclass(model, BaseModel): + field_type, default_value = field_info + # Check if the field is optional (not required) + is_optional = (default_value is not inspect.Parameter.empty) and (default_value is not Ellipsis) + else: + field_type = field_info + field_info = model.model_fields[field_name] + is_optional = field_info.is_required is False and get_origin(field_type) is Optional + rule_name, additional_rules = generate_gbnf_rule_for_type(model_name, + format_model_and_field_name(field_name), + field_type, is_optional, + processed_models, created_rules, field_info) + look_for_markdown_code_block = True if rule_name == "markdown_string" else False + look_for_triple_quoted_string = True if rule_name == "triple_quoted_string" else False + if not look_for_markdown_code_block and not look_for_triple_quoted_string: + if rule_name not in created_rules: + created_rules[rule_name] = additional_rules + model_rule_parts.append(f' ws \"\\\"{field_name}\\\"\" ": " {rule_name}') # Adding escaped quotes + nested_rules.extend(additional_rules) + else: + has_triple_quoted_string = look_for_markdown_code_block + has_markdown_code_block = look_for_triple_quoted_string + + fields_joined = r' "," "\n" '.join(model_rule_parts) + model_rule = fr'{model_name} ::= "{{" "\n" {fields_joined} "\n" ws "}}"' + + if look_for_markdown_code_block or look_for_triple_quoted_string: + model_rule += ' ws "}"' + + if has_triple_quoted_string: + model_rule += '"\\n" triple-quoted-string' + if has_markdown_code_block: + model_rule += '"\\n" markdown-code-block' + all_rules = [model_rule] + nested_rules + + return all_rules, has_markdown_code_block, has_triple_quoted_string + + +def generate_gbnf_grammar_from_pydantic_models(models: List[Type[BaseModel]], outer_object_name: str = None, + outer_object_content: str = None, list_of_outputs: bool = False) -> str: + """ + Generate GBNF Grammar from Pydantic Models. + + This method takes a list of Pydantic models and uses them to generate a GBNF grammar string. The generated grammar string can be used for parsing and validating data using the generated + * grammar. + + Parameters: + models (List[Type[BaseModel]]): A list of Pydantic models to generate the grammar from. + outer_object_name (str): Outer object name for the GBNF grammar. If None, no outer object will be generated. Eg. "function" for function calling. + outer_object_content (str): Content for the outer rule in the GBNF grammar. Eg. "function_parameters" or "params" for function calling. + list_of_outputs (str, optional): Allows a list of output objects + Returns: + str: The generated GBNF grammar string. + + Examples: + models = [UserModel, PostModel] + grammar = generate_gbnf_grammar_from_pydantic(models) + print(grammar) + # Output: + # root ::= UserModel | PostModel + # ... + """ + processed_models = set() + all_rules = [] + created_rules = {} + if outer_object_name is None: + + for model in models: + model_rules, _, _ = generate_gbnf_grammar(model, + processed_models, created_rules) + all_rules.extend(model_rules) + + if list_of_outputs: + root_rule = r'root ::= ws "[" grammar-models ("," grammar-models)* "]"' + "\n" + else: + root_rule = r'root ::= ws grammar-models' + "\n" + root_rule += "grammar-models ::= " + " | ".join( + [format_model_and_field_name(model.__name__) for model in models]) + all_rules.insert(0, root_rule) + return "\n".join(all_rules) + elif outer_object_name is not None: + if list_of_outputs: + root_rule = fr'root ::= ws "[" {format_model_and_field_name(outer_object_name)} ("," {format_model_and_field_name(outer_object_name)})* "]"' + "\n" + else: + root_rule = f"root ::= {format_model_and_field_name(outer_object_name)}\n" + + model_rule = fr'{format_model_and_field_name(outer_object_name)} ::= ws "{{" ws "\"{outer_object_name}\"" ": " grammar-models' + + fields_joined = " | ".join( + [fr'{format_model_and_field_name(model.__name__)}-grammar-model' for model in models]) + + grammar_model_rules = f'\ngrammar-models ::= {fields_joined}' + mod_rules = [] + for model in models: + mod_rule = fr'{format_model_and_field_name(model.__name__)}-grammar-model ::= ws' + mod_rule += fr'"\"{format_model_and_field_name(model.__name__)}\"" "," ws "\"{outer_object_content}\"" ws ":" ws {format_model_and_field_name(model.__name__)}' + '\n' + mod_rules.append(mod_rule) + grammar_model_rules += "\n" + "\n".join(mod_rules) + look_for_markdown_code_block = False + look_for_triple_quoted_string = False + for model in models: + model_rules, markdown_block, triple_quoted_string = generate_gbnf_grammar(model, + processed_models, created_rules) + all_rules.extend(model_rules) + if markdown_block: + look_for_markdown_code_block = True + + if triple_quoted_string: + look_for_triple_quoted_string = True + + if not look_for_markdown_code_block and not look_for_triple_quoted_string: + model_rule += ' ws "}"' + all_rules.insert(0, root_rule + model_rule + grammar_model_rules) + return "\n".join(all_rules) + + +def get_primitive_grammar(grammar): + """ + Returns the needed GBNF primitive grammar for a given GBNF grammar string. + + Args: + grammar (str): The string containing the GBNF grammar. + + Returns: + str: GBNF primitive grammar string. + """ + type_list = [] + if "string-list" in grammar: + type_list.append(str) + if "boolean-list" in grammar: + type_list.append(bool) + if "integer-list" in grammar: + type_list.append(int) + if "float-list" in grammar: + type_list.append(float) + additional_grammar = [generate_list_rule(t) for t in type_list] + primitive_grammar = r""" +boolean ::= "true" | "false" +null ::= "null" +string ::= "\"" ( + [^"\\] | + "\\" (["\\/bfnrt] | "u" [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F]) + )* "\"" ws +ws ::= ([ \t\n] ws)? +float ::= ("-"? ([0-9] | [1-9] [0-9]*)) ("." [0-9]+)? ([eE] [-+]? [0-9]+)? ws + +integer ::= [0-9]+""" + + any_block = "" + if "custom-class-any" in grammar: + any_block = ''' +value ::= object | array | string | number | boolean | null + +object ::= + "{" ws ( + string ":" ws value + ("," ws string ":" ws value)* + )? "}" ws + +array ::= + "[" ws ( + value + ("," ws value)* + )? "]" ws + +number ::= integer | float''' + + markdown_code_block_grammar = "" + if "markdown-code-block" in grammar: + markdown_code_block_grammar = r''' +markdown-code-block ::= opening-triple-ticks markdown-code-block-content closing-triple-ticks +markdown-code-block-content ::= ( [^`] | "`" [^`] | "`" "`" [^`] )* +opening-triple-ticks ::= "```" "python" "\n" | "```" "c" "\n" | "```" "cpp" "\n" | "```" "txt" "\n" | "```" "text" "\n" | "```" "json" "\n" | "```" "javascript" "\n" | "```" "css" "\n" | "```" "html" "\n" | "```" "markdown" "\n" +closing-triple-ticks ::= "```" "\n"''' + + if "triple-quoted-string" in grammar: + markdown_code_block_grammar = r""" +triple-quoted-string ::= triple-quotes triple-quoted-string-content triple-quotes +triple-quoted-string-content ::= ( [^'] | "'" [^'] | "'" "'" [^'] )* +triple-quotes ::= "'''" """ + return "\n" + '\n'.join(additional_grammar) + any_block + primitive_grammar + markdown_code_block_grammar + + +def generate_field_markdown(field_name: str, field_type: Type[Any], model: Type[BaseModel], depth=1) -> str: + indent = ' ' * depth + field_markdown = f"{indent}- **{field_name}** (`{field_type.__name__}`): " + + # Extracting field description from Pydantic Field using __model_fields__ + field_info = model.model_fields.get(field_name) + field_description = field_info.description if field_info and field_info.description else "No description available." + + field_markdown += field_description + '\n' + + # Handling nested BaseModel fields + if isclass(field_type) and issubclass(field_type, BaseModel): + field_markdown += f"{indent} - Details:\n" + for name, type_ in field_type.__annotations__.items(): + field_markdown += generate_field_markdown(name, type_, field_type, depth + 2) + + return field_markdown + + +def generate_markdown_report(pydantic_models: List[Type[BaseModel]]) -> str: + markdown = "" + for model in pydantic_models: + markdown += f"### {format_model_and_field_name(model.__name__)}\n" + + # Check if the model's docstring is different from BaseModel's docstring + class_doc = getdoc(model) + base_class_doc = getdoc(BaseModel) + class_description = class_doc if class_doc and class_doc != base_class_doc else "No specific description available." + + markdown += f"{class_description}\n\n" + markdown += "#### Fields\n" + + if isclass(model) and issubclass(model, BaseModel): + for name, field_type in model.__annotations__.items(): + markdown += generate_field_markdown(format_model_and_field_name(name), field_type, model) + markdown += "\n" + + return markdown + + +def format_json_example(example: dict, depth: int) -> str: + """ + Format a JSON example into a readable string with indentation. + + Args: + example (dict): JSON example to be formatted. + depth (int): Indentation depth. + + Returns: + str: Formatted JSON example string. + """ + indent = ' ' * depth + formatted_example = '{\n' + for key, value in example.items(): + value_text = f"'{value}'" if isinstance(value, str) else value + formatted_example += f"{indent}{key}: {value_text},\n" + formatted_example = formatted_example.rstrip(',\n') + '\n' + indent + '}' + return formatted_example + + +def generate_text_documentation(pydantic_models: List[Type[BaseModel]], model_prefix="Model", + fields_prefix="Fields", documentation_with_field_description=True) -> str: + """ + Generate text documentation for a list of Pydantic models. + + Args: + pydantic_models (List[Type[BaseModel]]): List of Pydantic model classes. + model_prefix (str): Prefix for the model section. + fields_prefix (str): Prefix for the fields section. + documentation_with_field_description (bool): Include field descriptions in the documentation. + + Returns: + str: Generated text documentation. + """ + documentation = "" + pyd_models = [(model, True) for model in pydantic_models] + for model, add_prefix in pyd_models: + if add_prefix: + documentation += f"{model_prefix}: {format_model_and_field_name(model.__name__)}\n" + else: + documentation += f"Model: {format_model_and_field_name(model.__name__)}\n" + + # Handling multi-line model description with proper indentation + + class_doc = getdoc(model) + base_class_doc = getdoc(BaseModel) + class_description = class_doc if class_doc and class_doc != base_class_doc else "" + if class_description != "": + documentation += " Description: " + documentation += "\n" + format_multiline_description(class_description, 2) + "\n" + + if add_prefix: + # Indenting the fields section + documentation += f" {fields_prefix}:\n" + else: + documentation += f" Fields:\n" + if isclass(model) and issubclass(model, BaseModel): + for name, field_type in model.__annotations__.items(): + # if name == "markdown_code_block": + # continue + if get_origin(field_type) == list: + element_type = get_args(field_type)[0] + if isclass(element_type) and issubclass(element_type, BaseModel): + pyd_models.append((element_type, False)) + if get_origin(field_type) == Union: + element_types = get_args(field_type) + for element_type in element_types: + if isclass(element_type) and issubclass(element_type, BaseModel): + pyd_models.append((element_type, False)) + documentation += generate_field_text(name, field_type, model, + documentation_with_field_description=documentation_with_field_description) + documentation += "\n" + + if hasattr(model, 'Config') and hasattr(model.Config, + 'json_schema_extra') and 'example' in model.Config.json_schema_extra: + documentation += f" Expected Example Output for {format_model_and_field_name(model.__name__)}:\n" + json_example = json.dumps(model.Config.json_schema_extra['example']) + documentation += format_multiline_description(json_example, 2) + "\n" + + return documentation + + +def generate_field_text(field_name: str, field_type: Type[Any], model: Type[BaseModel], depth=1, + documentation_with_field_description=True) -> str: + """ + Generate text documentation for a Pydantic model field. + + Args: + field_name (str): Name of the field. + field_type (Type[Any]): Type of the field. + model (Type[BaseModel]): Pydantic model class. + depth (int): Indentation depth in the documentation. + documentation_with_field_description (bool): Include field descriptions in the documentation. + + Returns: + str: Generated text documentation for the field. + """ + indent = ' ' * depth + + field_info = model.model_fields.get(field_name) + field_description = field_info.description if field_info and field_info.description else "" + + if get_origin(field_type) == list: + element_type = get_args(field_type)[0] + field_text = f"{indent}{field_name} ({format_model_and_field_name(field_type.__name__)} of {format_model_and_field_name(element_type.__name__)})" + if field_description != "": + field_text += ":\n" + else: + field_text += "\n" + elif get_origin(field_type) == Union: + element_types = get_args(field_type) + types = [] + for element_type in element_types: + types.append(format_model_and_field_name(element_type.__name__)) + field_text = f"{indent}{field_name} ({' or '.join(types)})" + if field_description != "": + field_text += ":\n" + else: + field_text += "\n" + else: + field_text = f"{indent}{field_name} ({format_model_and_field_name(field_type.__name__)})" + if field_description != "": + field_text += ":\n" + else: + field_text += "\n" + + if not documentation_with_field_description: + return field_text + + if field_description != "": + field_text += f"{indent} Description: " + field_description + "\n" + + # Check for and include field-specific examples if available + if hasattr(model, 'Config') and hasattr(model.Config, + 'json_schema_extra') and 'example' in model.Config.json_schema_extra: + field_example = model.Config.json_schema_extra['example'].get(field_name) + if field_example is not None: + example_text = f"'{field_example}'" if isinstance(field_example, str) else field_example + field_text += f"{indent} Example: {example_text}\n" + + if isclass(field_type) and issubclass(field_type, BaseModel): + field_text += f"{indent} Details:\n" + for name, type_ in field_type.__annotations__.items(): + field_text += generate_field_text(name, type_, field_type, depth + 2) + + return field_text + + +def format_multiline_description(description: str, indent_level: int) -> str: + """ + Format a multiline description with proper indentation. + + Args: + description (str): Multiline description. + indent_level (int): Indentation level. + + Returns: + str: Formatted multiline description. + """ + indent = ' ' * indent_level + return indent + description.replace('\n', '\n' + indent) + + +def save_gbnf_grammar_and_documentation(grammar, documentation, grammar_file_path="./grammar.gbnf", + documentation_file_path="./grammar_documentation.md"): + """ + Save GBNF grammar and documentation to specified files. + + Args: + grammar (str): GBNF grammar string. + documentation (str): Documentation string. + grammar_file_path (str): File path to save the GBNF grammar. + documentation_file_path (str): File path to save the documentation. + + Returns: + None + """ + try: + with open(grammar_file_path, 'w') as file: + file.write(grammar + get_primitive_grammar(grammar)) + print(f"Grammar successfully saved to {grammar_file_path}") + except IOError as e: + print(f"An error occurred while saving the grammar file: {e}") + + try: + with open(documentation_file_path, 'w') as file: + file.write(documentation) + print(f"Documentation successfully saved to {documentation_file_path}") + except IOError as e: + print(f"An error occurred while saving the documentation file: {e}") + + +def remove_empty_lines(string): + """ + Remove empty lines from a string. + + Args: + string (str): Input string. + + Returns: + str: String with empty lines removed. + """ + lines = string.splitlines() + non_empty_lines = [line for line in lines if line.strip() != ""] + string_no_empty_lines = "\n".join(non_empty_lines) + return string_no_empty_lines + + +def generate_and_save_gbnf_grammar_and_documentation(pydantic_model_list, + grammar_file_path="./generated_grammar.gbnf", + documentation_file_path="./generated_grammar_documentation.md", + outer_object_name: str = None, + outer_object_content: str = None, + model_prefix: str = "Output Model", + fields_prefix: str = "Output Fields", + list_of_outputs: bool = False, + documentation_with_field_description=True): + """ + Generate GBNF grammar and documentation, and save them to specified files. + + Args: + pydantic_model_list: List of Pydantic model classes. + grammar_file_path (str): File path to save the generated GBNF grammar. + documentation_file_path (str): File path to save the generated documentation. + outer_object_name (str): Outer object name for the GBNF grammar. If None, no outer object will be generated. Eg. "function" for function calling. + outer_object_content (str): Content for the outer rule in the GBNF grammar. Eg. "function_parameters" or "params" for function calling. + model_prefix (str): Prefix for the model section in the documentation. + fields_prefix (str): Prefix for the fields section in the documentation. + list_of_outputs (bool): Whether the output is a list of items. + documentation_with_field_description (bool): Include field descriptions in the documentation. + + Returns: + None + """ + documentation = generate_text_documentation(pydantic_model_list, model_prefix, fields_prefix, + documentation_with_field_description=documentation_with_field_description) + grammar = generate_gbnf_grammar_from_pydantic_models(pydantic_model_list, outer_object_name, + outer_object_content, list_of_outputs) + grammar = remove_empty_lines(grammar) + save_gbnf_grammar_and_documentation(grammar, documentation, grammar_file_path, documentation_file_path) + + +def generate_gbnf_grammar_and_documentation(pydantic_model_list, outer_object_name: str = None, + outer_object_content: str = None, + model_prefix: str = "Output Model", + fields_prefix: str = "Output Fields", list_of_outputs: bool = False, + documentation_with_field_description=True): + """ + Generate GBNF grammar and documentation for a list of Pydantic models. + + Args: + pydantic_model_list: List of Pydantic model classes. + outer_object_name (str): Outer object name for the GBNF grammar. If None, no outer object will be generated. Eg. "function" for function calling. + outer_object_content (str): Content for the outer rule in the GBNF grammar. Eg. "function_parameters" or "params" for function calling. + model_prefix (str): Prefix for the model section in the documentation. + fields_prefix (str): Prefix for the fields section in the documentation. + list_of_outputs (bool): Whether the output is a list of items. + documentation_with_field_description (bool): Include field descriptions in the documentation. + + Returns: + tuple: GBNF grammar string, documentation string. + """ + documentation = generate_text_documentation(copy(pydantic_model_list), model_prefix, fields_prefix, + documentation_with_field_description=documentation_with_field_description) + grammar = generate_gbnf_grammar_from_pydantic_models(pydantic_model_list, outer_object_name, + outer_object_content, list_of_outputs) + grammar = remove_empty_lines(grammar + get_primitive_grammar(grammar)) + return grammar, documentation + + +def generate_gbnf_grammar_and_documentation_from_dictionaries(dictionaries: List[dict], + outer_object_name: str = None, + outer_object_content: str = None, + model_prefix: str = "Output Model", + fields_prefix: str = "Output Fields", + list_of_outputs: bool = False, + documentation_with_field_description=True): + """ + Generate GBNF grammar and documentation from a list of dictionaries. + + Args: + dictionaries (List[dict]): List of dictionaries representing Pydantic models. + outer_object_name (str): Outer object name for the GBNF grammar. If None, no outer object will be generated. Eg. "function" for function calling. + outer_object_content (str): Content for the outer rule in the GBNF grammar. Eg. "function_parameters" or "params" for function calling. + model_prefix (str): Prefix for the model section in the documentation. + fields_prefix (str): Prefix for the fields section in the documentation. + list_of_outputs (bool): Whether the output is a list of items. + documentation_with_field_description (bool): Include field descriptions in the documentation. + + Returns: + tuple: GBNF grammar string, documentation string. + """ + pydantic_model_list = create_dynamic_models_from_dictionaries(dictionaries) + documentation = generate_text_documentation(copy(pydantic_model_list), model_prefix, fields_prefix, + documentation_with_field_description=documentation_with_field_description) + grammar = generate_gbnf_grammar_from_pydantic_models(pydantic_model_list, outer_object_name, + outer_object_content, list_of_outputs) + grammar = remove_empty_lines(grammar + get_primitive_grammar(grammar)) + return grammar, documentation + + +def create_dynamic_model_from_function(func: Callable): + """ + Creates a dynamic Pydantic model from a given function's type hints and adds the function as a 'run' method. + + Args: + func (Callable): A function with type hints from which to create the model. + + Returns: + A dynamic Pydantic model class with the provided function as a 'run' method. + """ + # Extracting type hints from the provided function + type_hints = get_type_hints(func) + type_hints.pop('return', None) + + # Handling default values and annotations + dynamic_fields = {} + defaults = getattr(func, '__defaults__', ()) or () + defaults_index = len(type_hints) - len(defaults) + + for index, (name, typ) in enumerate(type_hints.items()): + if index >= defaults_index: + default_value = defaults[index - defaults_index] + dynamic_fields[name] = (typ, default_value) + else: + dynamic_fields[name] = (typ, ...) + + # Creating the dynamic model + dynamicModel = create_model(f'{func.__name__}', **dynamic_fields) + + dynamicModel.__doc__ = getdoc(func) + + # Wrapping the original function to handle instance 'self' + def run_method_wrapper(self): + func_args = {name: getattr(self, name) for name in type_hints} + return func(**func_args) + + # Adding the wrapped function as a 'run' method + setattr(dynamicModel, 'run', run_method_wrapper) + + return dynamicModel + + +def add_run_method_to_dynamic_model(model: Type[BaseModel], func: Callable): + """ + Add a 'run' method to a dynamic Pydantic model, using the provided function. + + Args: + - model (Type[BaseModel]): Dynamic Pydantic model class. + - func (Callable): Function to be added as a 'run' method to the model. + + Returns: + - Type[BaseModel]: Pydantic model class with the added 'run' method. + """ + + def run_method_wrapper(self): + func_args = {name: getattr(self, name) for name in model.model_fields} + return func(**func_args) + + # Adding the wrapped function as a 'run' method + setattr(model, 'run', run_method_wrapper) + + return model + + +def create_dynamic_models_from_dictionaries(dictionaries: List[dict]): + """ + Create a list of dynamic Pydantic model classes from a list of dictionaries. + + Args: + - dictionaries (List[dict]): List of dictionaries representing model structures. + + Returns: + - List[Type[BaseModel]]: List of generated dynamic Pydantic model classes. + """ + dynamic_models = [] + for func in dictionaries: + model_name = format_model_and_field_name(func.get("name", "")) + dyn_model = convert_dictionary_to_to_pydantic_model(func, model_name) + dynamic_models.append(dyn_model) + return dynamic_models + + +def map_grammar_names_to_pydantic_model_class(pydantic_model_list): + output = {} + for model in pydantic_model_list: + output[format_model_and_field_name(model.__name__)] = model + + return output + + +from enum import Enum + + +def json_schema_to_python_types(schema): + type_map = { + 'any': Any, + 'string': str, + 'number': float, + 'integer': int, + 'boolean': bool, + 'array': list, + } + return type_map[schema] + + +def list_to_enum(enum_name, values): + return Enum(enum_name, {value: value for value in values}) + + +def convert_dictionary_to_to_pydantic_model(dictionary: dict, model_name: str = 'CustomModel') -> Type[BaseModel]: + """ + Convert a dictionary to a Pydantic model class. + + Args: + - dictionary (dict): Dictionary representing the model structure. + - model_name (str): Name of the generated Pydantic model. + + Returns: + - Type[BaseModel]: Generated Pydantic model class. + """ + fields = {} + + if "properties" in dictionary: + for field_name, field_data in dictionary.get("properties", {}).items(): + if field_data == 'object': + submodel = convert_dictionary_to_to_pydantic_model(dictionary, f'{model_name}_{field_name}') + fields[field_name] = (submodel, ...) + else: + field_type = field_data.get('type', 'str') + + if field_data.get("enum", []): + fields[field_name] = (list_to_enum(field_name, field_data.get("enum", [])), ...) + if field_type == "array": + items = field_data.get("items", {}) + if items != {}: + array = {"properties": items} + array_type = convert_dictionary_to_to_pydantic_model(array, f'{model_name}_{field_name}_items') + fields[field_name] = (List[array_type], ...) + else: + fields[field_name] = (list, ...) + elif field_type == 'object': + submodel = convert_dictionary_to_to_pydantic_model(field_data, f'{model_name}_{field_name}') + fields[field_name] = (submodel, ...) + else: + field_type = json_schema_to_python_types(field_type) + fields[field_name] = (field_type, ...) + if "function" in dictionary: + + for field_name, field_data in dictionary.get("function", {}).items(): + if field_name == "name": + model_name = field_data + elif field_name == "description": + fields["__doc__"] = field_data + elif field_name == "parameters": + return convert_dictionary_to_to_pydantic_model(field_data, f'{model_name}') + if "parameters" in dictionary: + field_data = {"function": dictionary} + return convert_dictionary_to_to_pydantic_model(field_data, f'{model_name}') + + custom_model = create_model(model_name, **fields) + return custom_model + + + From fa5c1fb44a2724292da545d6b7cf2a1ac0e0b989 Mon Sep 17 00:00:00 2001 From: slaren Date: Fri, 12 Jan 2024 20:38:34 +0100 Subject: [PATCH 042/812] backend_sched : fix assignments ggml-ci --- ggml-backend.c | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/ggml-backend.c b/ggml-backend.c index 4c2d8b0b2..505dbba47 100644 --- a/ggml-backend.c +++ b/ggml-backend.c @@ -1087,6 +1087,24 @@ static void sched_split_graph(ggml_backend_sched_t sched, struct ggml_cgraph * g } } } + + // pass 2.4 expand rest down + { + ggml_tallocr_t cur_allocr = NULL; + for (int i = 0; i < graph->n_nodes; i++) { + struct ggml_tensor * node = graph->nodes[i]; + if (ggml_is_view_op(node->op)) { + continue; + } + ggml_tallocr_t node_allocr = node_allocr(node); + if (node_allocr != NULL) { + cur_allocr = node_allocr; + } else { + node_allocr(node) = cur_allocr; + SET_CAUSE(node, "2.4"); + } + } + } #ifdef DEBUG_PASS2 fprintf(stderr, "PASS 2 ASSIGNMENTS\n"); sched_print_assignments(sched, graph); #endif @@ -1146,6 +1164,8 @@ static void sched_split_graph(ggml_backend_sched_t sched, struct ggml_cgraph * g ggml_tallocr_t node_allocr = node_allocr(node); + GGML_ASSERT(node_allocr != NULL); // all nodes should be assigned by now + if (node_allocr != cur_allocr) { sched->splits[cur_split].i_end = i; cur_split++; From f238461236f4e0e18cac1a554af23c7deadc9b01 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Fri, 12 Jan 2024 14:02:30 +0200 Subject: [PATCH 043/812] ggml : fix 32-bit ARM compat for IQ2_XS (whisper/1758) * ggml : fix 32-bit ARM compat * ggml : fix fix * ggml : fix fix fix --- ggml-quants.c | 39 +++++++++++++++++++++++++++++++++++---- 1 file changed, 35 insertions(+), 4 deletions(-) diff --git a/ggml-quants.c b/ggml-quants.c index a24b4b244..601d155d7 100644 --- a/ggml-quants.c +++ b/ggml-quants.c @@ -272,10 +272,13 @@ static inline float hsum_float_4x4(const __m128 a, const __m128 b, const __m128 // vaddvq_s16 // vpaddq_s16 +// vpaddq_s32 // vaddvq_s32 // vaddvq_f32 // vmaxvq_f32 // vcvtnq_s32_f32 +// vzip1_u8 +// vzip2_u8 inline static int32_t vaddvq_s16(int16x8_t v) { return @@ -291,6 +294,12 @@ inline static int16x8_t vpaddq_s16(int16x8_t a, int16x8_t b) { return vcombine_s16(a0, b0); } +inline static int32x4_t vpaddq_s32(int32x4_t a, int32x4_t b) { + int32x2_t a0 = vpadd_s32(vget_low_s32(a), vget_high_s32(a)); + int32x2_t b0 = vpadd_s32(vget_low_s32(b), vget_high_s32(b)); + return vcombine_s32(a0, b0); +} + inline static int32_t vaddvq_s32(int32x4_t v) { return vgetq_lane_s32(v, 0) + vgetq_lane_s32(v, 1) + vgetq_lane_s32(v, 2) + vgetq_lane_s32(v, 3); } @@ -316,6 +325,28 @@ inline static int32x4_t vcvtnq_s32_f32(float32x4_t v) { return res; } +inline static uint8x8_t vzip1_u8(uint8x8_t a, uint8x8_t b) { + uint8x8_t res; + + res[0] = a[0]; res[1] = b[0]; + res[2] = a[1]; res[3] = b[1]; + res[4] = a[2]; res[5] = b[2]; + res[6] = a[3]; res[7] = b[3]; + + return res; +} + +inline static uint8x8_t vzip2_u8(uint8x8_t a, uint8x8_t b) { + uint8x8_t res; + + res[0] = a[4]; res[1] = b[4]; + res[2] = a[5]; res[3] = b[5]; + res[4] = a[6]; res[5] = b[6]; + res[6] = a[7]; res[7] = b[7]; + + return res; +} + // vld1q_s16_x2 // vld1q_u8_x2 // vld1q_u8_x4 @@ -7554,9 +7585,9 @@ void ggml_vec_dot_iq2_xs_q8_K(const int n, float * restrict s, const void * rest const uint64_t * signs64 = (const uint64_t *)keven_signs_q2xs; - int8x16x4_t q2u; - int8x16x4_t q2s; - int8x16x4_t q8b; + ggml_int8x16x4_t q2u; + ggml_int8x16x4_t q2s; + ggml_int8x16x4_t q8b; int32x4x4_t scales32; @@ -7578,7 +7609,7 @@ void ggml_vec_dot_iq2_xs_q8_K(const int n, float * restrict s, const void * rest scales32.val[3] = vreinterpretq_s32_u32(vmovl_u16(vget_high_u16(scales2))); int32x4_t sumi = vdupq_n_s32(0); for (int ib64 = 0; ib64 < QK_K/64; ++ib64) { - q8b = vld1q_s8_x4(q8); q8 += 64; + q8b = ggml_vld1q_s8_x4(q8); q8 += 64; q2u.val[0] = vcombine_s8(vld1_s8((const void *)(iq2xs_grid + (q2[0] & 511))), vld1_s8((const void *)(iq2xs_grid + (q2[1] & 511)))); q2u.val[1] = vcombine_s8(vld1_s8((const void *)(iq2xs_grid + (q2[2] & 511))), vld1_s8((const void *)(iq2xs_grid + (q2[3] & 511)))); q2u.val[2] = vcombine_s8(vld1_s8((const void *)(iq2xs_grid + (q2[4] & 511))), vld1_s8((const void *)(iq2xs_grid + (q2[5] & 511)))); From de473f5f8e19ba5e659cdf5af65fb9251dce16c5 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Fri, 12 Jan 2024 22:02:43 +0200 Subject: [PATCH 044/812] sync : ggml --- scripts/sync-ggml.last | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/sync-ggml.last b/scripts/sync-ggml.last index 3e2c579d5..edcdb530a 100644 --- a/scripts/sync-ggml.last +++ b/scripts/sync-ggml.last @@ -1 +1 @@ -979cc23b345006504cfc1f67c0fdf627805e3319 +400c07f00508e6f60fb25405444b5669c365b0a9 From 15ebe59210e7fd9817ff67f51fa1a5ee2d004294 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Sat, 13 Jan 2024 13:44:37 +0200 Subject: [PATCH 045/812] convert : update phi-2 to latest HF repo (#4903) * convert : update phi-2 to latest HF repo ggml-ci * py : try to fix flake stuff --- convert-hf-to-gguf.py | 39 +++++++++++++++++++++---------- gguf-py/gguf/constants.py | 3 +++ gguf-py/gguf/tensor_mapping.py | 2 ++ llama.cpp | 42 ++++++++++++++++++++++++++-------- 4 files changed, 65 insertions(+), 21 deletions(-) diff --git a/convert-hf-to-gguf.py b/convert-hf-to-gguf.py index a1c79fd47..b133f3b49 100755 --- a/convert-hf-to-gguf.py +++ b/convert-hf-to-gguf.py @@ -23,6 +23,15 @@ if 'NO_LOCAL_GGUF' not in os.environ: import gguf +# check for any of the given keys in the dictionary and return the value of the first key found +def get_key_opts(d, keys): + for k in keys: + if k in d: + return d[k] + print(f"Could not find any of {keys}") + sys.exit() + + ###### MODEL DEFINITIONS ###### class SentencePieceTokenTypes(IntEnum): @@ -257,10 +266,11 @@ class Model: toktypes.append(gguf.TokenType.USER_DEFINED) elif reverse_vocab[i] in added_vocab: tokens.append(reverse_vocab[i]) - if tokenizer.added_tokens_decoder[i].special: - toktypes.append(gguf.TokenType.CONTROL) - else: - toktypes.append(gguf.TokenType.USER_DEFINED) + if hasattr(tokenizer, "added_tokens_decoder"): + if tokenizer.added_tokens_decoder[i].special: + toktypes.append(gguf.TokenType.CONTROL) + else: + toktypes.append(gguf.TokenType.USER_DEFINED) else: tokens.append(reverse_vocab[i]) toktypes.append(gguf.TokenType.NORMAL) @@ -1068,17 +1078,22 @@ class GPT2Model(Model): class Phi2Model(Model): def set_gguf_parameters(self): - block_count = self.hparams["n_layer"] + block_count = get_key_opts(self.hparams, ["num_hidden_layers", "n_layer"]) + + rot_pct = get_key_opts(self.hparams, ["partial_rotary_factor"]) + n_embd = get_key_opts(self.hparams, ["hidden_size", "n_embd"]) + n_head = get_key_opts(self.hparams, ["num_attention_heads", "n_head"]) self.gguf_writer.add_name("Phi2") - self.gguf_writer.add_context_length(self.hparams["n_positions"]) - self.gguf_writer.add_embedding_length(self.hparams["n_embd"]) - self.gguf_writer.add_feed_forward_length(4 * self.hparams["n_embd"]) + self.gguf_writer.add_context_length(get_key_opts(self.hparams, ["n_positions", "max_position_embeddings"])) + + self.gguf_writer.add_embedding_length(n_embd) + self.gguf_writer.add_feed_forward_length(4 * n_embd) self.gguf_writer.add_block_count(block_count) - self.gguf_writer.add_head_count(self.hparams["n_head"]) - self.gguf_writer.add_head_count_kv(self.hparams["n_head"]) - self.gguf_writer.add_layer_norm_eps(self.hparams["layer_norm_epsilon"]) - self.gguf_writer.add_rope_dimension_count(self.hparams["rotary_dim"]) + self.gguf_writer.add_head_count(n_head) + self.gguf_writer.add_head_count_kv(n_head) + self.gguf_writer.add_layer_norm_eps(get_key_opts(self.hparams, ["layer_norm_epsilon", "layer_norm_eps"])) + self.gguf_writer.add_rope_dimension_count(int(rot_pct * n_embd) // n_head) self.gguf_writer.add_file_type(self.ftype) self.gguf_writer.add_add_bos_token(False) diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py index f0a1c51f8..972b4e9a7 100644 --- a/gguf-py/gguf/constants.py +++ b/gguf-py/gguf/constants.py @@ -389,6 +389,9 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = { MODEL_TENSOR.OUTPUT, MODEL_TENSOR.ATTN_NORM, MODEL_TENSOR.ATTN_QKV, + MODEL_TENSOR.ATTN_Q, + MODEL_TENSOR.ATTN_K, + MODEL_TENSOR.ATTN_V, MODEL_TENSOR.ATTN_OUT, MODEL_TENSOR.FFN_NORM, MODEL_TENSOR.FFN_DOWN, diff --git a/gguf-py/gguf/tensor_mapping.py b/gguf-py/gguf/tensor_mapping.py index 24a089037..e5b146106 100644 --- a/gguf-py/gguf/tensor_mapping.py +++ b/gguf-py/gguf/tensor_mapping.py @@ -191,6 +191,7 @@ class TensorNameMap: "transformer.h.{bid}.mlp.w1", # qwen "h.{bid}.mlp.c_fc", # gpt2 "transformer.h.{bid}.mlp.fc1", # phi2 + "model.layers.{bid}.mlp.fc1", # phi2 "model.layers.layers.{bid}.mlp.up_proj", # plamo ), @@ -232,6 +233,7 @@ class TensorNameMap: "model.layers.{bid}.mlp.dense_4h_to_h", # persimmon "h.{bid}.mlp.c_proj", # gpt2 "transformer.h.{bid}.mlp.fc2", # phi2 + "model.layers.{bid}.mlp.fc2", # phi2 "model.layers.layers.{bid}.mlp.down_proj", # plamo ), diff --git a/llama.cpp b/llama.cpp index fe1d8947c..1d2eb569f 100644 --- a/llama.cpp +++ b/llama.cpp @@ -574,6 +574,9 @@ static std::map> LLM_TENSOR_NAMES = { LLM_TENSOR_OUTPUT, "output" }, { LLM_TENSOR_ATTN_NORM, "blk.%d.attn_norm" }, { LLM_TENSOR_ATTN_QKV, "blk.%d.attn_qkv" }, + { LLM_TENSOR_ATTN_Q, "blk.%d.attn_q" }, + { LLM_TENSOR_ATTN_K, "blk.%d.attn_k" }, + { LLM_TENSOR_ATTN_V, "blk.%d.attn_v" }, { LLM_TENSOR_ATTN_OUT, "blk.%d.attn_output" }, { LLM_TENSOR_FFN_DOWN, "blk.%d.ffn_down" }, { LLM_TENSOR_FFN_UP, "blk.%d.ffn_up" }, @@ -3676,8 +3679,19 @@ static bool llm_load_tensors( layer.attn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}); layer.attn_norm_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM, "bias", i), {n_embd}); - layer.wqkv = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_QKV, "weight", i), {n_embd, n_embd + 2*n_embd_gqa}); - layer.bqkv = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_QKV, "bias", i), {n_embd + 2*n_embd_gqa}); + layer.wqkv = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_QKV, "weight", i), {n_embd, n_embd + 2*n_embd_gqa}, false); + layer.bqkv = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_QKV, "bias", i), {n_embd + 2*n_embd_gqa}, false); + + if (layer.wqkv == nullptr) { + layer.wq = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_embd}); + layer.bq = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_Q, "bias", i), {n_embd}); + + layer.wk = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_K, "weight", i), {n_embd, n_embd_gqa}); + layer.bk = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_K, "bias", i), {n_embd_gqa}); + + layer.wv = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_V, "weight", i), {n_embd, n_embd_gqa}); + layer.bv = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_V, "bias", i), {n_embd_gqa}); + } layer.wo = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd}); layer.bo = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_OUT, "bias", i), {n_embd}); @@ -5637,15 +5651,25 @@ struct llm_build_context { // self-attention { - cur = ggml_mul_mat(ctx0, model.layers[il].wqkv, attn_norm_output); - cb(cur, "wqkv", il); + struct ggml_tensor * Qcur = nullptr; + struct ggml_tensor * Kcur = nullptr; + struct ggml_tensor * Vcur = nullptr; - cur = ggml_add(ctx0, cur, model.layers[il].bqkv); - cb(cur, "bqkv", il); + if (model.layers[il].wqkv) { + cur = ggml_mul_mat(ctx0, model.layers[il].wqkv, attn_norm_output); + cb(cur, "wqkv", il); - struct ggml_tensor * Qcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd, n_tokens, cur->nb[1], 0*sizeof(float)*(n_embd))); - struct ggml_tensor * Kcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd_gqa, n_tokens, cur->nb[1], 1*sizeof(float)*(n_embd))); - struct ggml_tensor * Vcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd_gqa, n_tokens, cur->nb[1], 1*sizeof(float)*(n_embd + n_embd_gqa))); + cur = ggml_add(ctx0, cur, model.layers[il].bqkv); + cb(cur, "bqkv", il); + + Qcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd, n_tokens, cur->nb[1], 0*sizeof(float)*(n_embd))); + Kcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd_gqa, n_tokens, cur->nb[1], 1*sizeof(float)*(n_embd))); + Vcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd_gqa, n_tokens, cur->nb[1], 1*sizeof(float)*(n_embd + n_embd_gqa))); + } else { + Qcur = ggml_add(ctx0, ggml_mul_mat(ctx0, model.layers[il].wq, attn_norm_output), model.layers[il].bq); + Kcur = ggml_add(ctx0, ggml_mul_mat(ctx0, model.layers[il].wk, attn_norm_output), model.layers[il].bk); + Vcur = ggml_add(ctx0, ggml_mul_mat(ctx0, model.layers[il].wv, attn_norm_output), model.layers[il].bv); + } cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); From ee8243adaa9a9f51ff449213383874e49efe368f Mon Sep 17 00:00:00 2001 From: makomk Date: Sat, 13 Jan 2024 14:16:11 +0000 Subject: [PATCH 046/812] server : fix crash with multimodal models without BOS token (#4904) --- examples/server/server.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/server/server.cpp b/examples/server/server.cpp index c1ab8f9dc..7b33aea1f 100644 --- a/examples/server/server.cpp +++ b/examples/server/server.cpp @@ -1835,7 +1835,7 @@ struct llama_server_context slot.cache_tokens = prompt_tokens; - if (slot.n_past == slot.num_prompt_tokens) + if (slot.n_past == slot.num_prompt_tokens && slot.n_past > 0) { // we have to evaluate at least 1 token to generate logits. LOG_TEE("slot %d : we have to evaluate at least 1 token to generate logits\n", slot.id); From 356327feb3f66980ab687040495d722696d98970 Mon Sep 17 00:00:00 2001 From: Ziad Ben Hadj-Alouane Date: Sat, 13 Jan 2024 09:20:46 -0500 Subject: [PATCH 047/812] server : fix deadlock that occurs in multi-prompt scenarios (#4905) * * fix deadlock * * dont ruint all whitespace --- examples/server/server.cpp | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/examples/server/server.cpp b/examples/server/server.cpp index 7b33aea1f..79eacf828 100644 --- a/examples/server/server.cpp +++ b/examples/server/server.cpp @@ -1350,14 +1350,17 @@ struct llama_server_context res.result_json["model"] = slot.oaicompat_model; } + queue_results.push_back(res); + condition_results.notify_all(); + + // done with results, unlock + lock.unlock(); + // parent multitask, if any, needs to be updated if (slot.multitask_id != -1) { update_multi_task(slot.multitask_id, slot.task_id, res); } - - queue_results.push_back(res); - condition_results.notify_all(); } void send_embedding(llama_client_slot &slot) @@ -1603,6 +1606,7 @@ struct llama_server_context } // remove finished multitasks from the queue of multitasks, and add the corresponding result to the result queue + std::vector agg_results; auto queue_iterator = queue_multitasks.begin(); while (queue_iterator != queue_multitasks.end()) { @@ -1623,8 +1627,9 @@ struct llama_server_context } aggregate_result.result_json = json{ "results", result_jsons }; - std::lock_guard lock(mutex_results); - queue_results.push_back(aggregate_result); + + agg_results.push_back(aggregate_result); + condition_results.notify_all(); queue_iterator = queue_multitasks.erase(queue_iterator); @@ -1634,6 +1639,13 @@ struct llama_server_context ++queue_iterator; } } + + // done with tasks, unlock + lock.unlock(); + + // copy aggregate results of complete multi-tasks to the results queue + std::lock_guard lock_results(mutex_results); + queue_results.insert(queue_results.end(), agg_results.begin(), agg_results.end()); } bool update_slots() { From 7dc78764e2ff86512e6e31cb0fcb8087df4b4708 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20G=C3=A4=C3=9Fler?= Date: Sat, 13 Jan 2024 15:52:53 +0100 Subject: [PATCH 048/812] compare-llama-bench: tweak output format (#4910) --- scripts/compare-llama-bench.py | 34 ++++++++++++++++++++++++++-------- 1 file changed, 26 insertions(+), 8 deletions(-) diff --git a/scripts/compare-llama-bench.py b/scripts/compare-llama-bench.py index bc1714487..70737f976 100755 --- a/scripts/compare-llama-bench.py +++ b/scripts/compare-llama-bench.py @@ -10,15 +10,15 @@ import sqlite3 try: import git from tabulate import tabulate -except ImportError: +except ImportError as e: print("ERROR: the following Python libraries are required: GitPython, tabulate.") - sys.exit(1) + raise e # Properties by which to differentiate results per commit: KEY_PROPERTIES = [ - "cuda", "opencl", "metal", "gpu_blas", "blas", "cpu_info", "gpu_info", "model_filename", - "model_type", "model_size", "model_n_params", "n_batch", "n_threads", "type_k", "type_v", - "n_gpu_layers", "main_gpu", "no_kv_offload", "mul_mat_q", "tensor_split", "n_prompt", "n_gen" + "cpu_info", "gpu_info", "n_gpu_layers", "main_gpu", "cuda", "opencl", "metal", "gpu_blas", + "blas", "model_filename", "model_type", "model_size", "model_n_params", "n_batch", "n_threads", + "type_k", "type_v", "no_kv_offload", "mul_mat_q", "tensor_split", "n_prompt", "n_gen" ] # Properties that are boolean and are converted to Yes/No for the table: @@ -37,6 +37,7 @@ PRETTY_NAMES = { DEFAULT_SHOW = ["model_type"] # Always show these properties by default. DEFAULT_HIDE = ["model_filename"] # Always hide these properties by default. GPU_NAME_STRIP = ["NVIDIA GeForce ", "Tesla ", "AMD Radeon "] # Strip prefixes for smaller tables. +MODEL_SUFFIX_REPLACE = {" - Small": "_S", " - Medium": "_M", " - Large": "_L"} DESCRIPTION = """Creates tables from llama-bench data written to an SQLite database. Example usage (Linux): @@ -308,8 +309,13 @@ else: if gpu_blas and "gpu_info" not in properties_different: show.append("gpu_info") - show += DEFAULT_SHOW show += properties_different + + index_default = 0 + for prop in ["cpu_info", "gpu_info", "n_gpu_layers", "main_gpu"]: + if prop in show: + index_default += 1 + show = show[:index_default] + DEFAULT_SHOW + show[index_default:] for prop in DEFAULT_HIDE: try: show.remove(prop) @@ -334,6 +340,12 @@ for bool_property in BOOL_PROPERTIES: for row_table in table: row_table[ip] = "Yes" if int(row_table[ip]) == 1 else "No" +if "model_type" in show: + ip = show.index("model_type") + for (old, new) in MODEL_SUFFIX_REPLACE.items(): + for row_table in table: + row_table[ip] = row_table[ip].replace(old, new) + if "model_size" in show: ip = show.index("model_size") for row_table in table: @@ -341,10 +353,16 @@ if "model_size" in show: if "gpu_info" in show: ip = show.index("gpu_info") - for gns in GPU_NAME_STRIP: - for row_table in table: + for row_table in table: + for gns in GPU_NAME_STRIP: row_table[ip] = row_table[ip].replace(gns, "") + gpu_names = row_table[ip].split("/") + num_gpus = len(gpu_names) + all_names_the_same = len(set(gpu_names)) == 1 + if len(gpu_names) >= 2 and all_names_the_same: + row_table[ip] = f"{num_gpus}x {gpu_names[0]}" + headers = [PRETTY_NAMES[p] for p in show] headers += ["Test", f"t/s {name_baseline}", f"t/s {name_compare}", "Speedup"] From b38b5e93ae31019e87f692b69d27124eae6aac02 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Sat, 13 Jan 2024 18:03:45 +0200 Subject: [PATCH 049/812] metal : refactor kernel loading code (#4794) * metal : detect more GPU families * metal : refactor kernel loading * metal : set kernel family requirements * metal : fix kernel init + fix compile options * metal : take into account simdgroup reduction support * metal : print only skipped kernels * metal : fix check for simdgroup reduction support * metal : check for Metal 3 * metal : free allocations * metal : normalize encoder:setComputePipelineStatus calls ggml-ci * metal : fix Metal3 family check ggml-ci * metal : check for simdgroup matrix mul. feature ggml-ci --- ggml-metal.m | 1048 +++++++++++++++++++++++++------------------------- 1 file changed, 530 insertions(+), 518 deletions(-) diff --git a/ggml-metal.m b/ggml-metal.m index c03624073..6c28a7ee3 100644 --- a/ggml-metal.m +++ b/ggml-metal.m @@ -26,6 +26,8 @@ #define GGML_MAX_CONCUR (2*GGML_DEFAULT_GRAPH_SIZE) +#define GGML_METAL_MAX_KERNELS 256 + struct ggml_metal_buffer { const char * name; @@ -35,6 +37,134 @@ struct ggml_metal_buffer { id metal; }; +struct ggml_metal_kernel { + id function; + id pipeline; +}; + +enum ggml_metal_kernel_type { + GGML_METAL_KERNEL_TYPE_ADD, + GGML_METAL_KERNEL_TYPE_ADD_ROW, + GGML_METAL_KERNEL_TYPE_MUL, + GGML_METAL_KERNEL_TYPE_MUL_ROW, + GGML_METAL_KERNEL_TYPE_DIV, + GGML_METAL_KERNEL_TYPE_DIV_ROW, + GGML_METAL_KERNEL_TYPE_SCALE, + GGML_METAL_KERNEL_TYPE_SCALE_4, + GGML_METAL_KERNEL_TYPE_TANH, + GGML_METAL_KERNEL_TYPE_RELU, + GGML_METAL_KERNEL_TYPE_GELU, + GGML_METAL_KERNEL_TYPE_GELU_QUICK, + GGML_METAL_KERNEL_TYPE_SILU, + GGML_METAL_KERNEL_TYPE_SOFT_MAX, + GGML_METAL_KERNEL_TYPE_SOFT_MAX_4, + GGML_METAL_KERNEL_TYPE_DIAG_MASK_INF, + GGML_METAL_KERNEL_TYPE_DIAG_MASK_INF_8, + GGML_METAL_KERNEL_TYPE_GET_ROWS_F32, + GGML_METAL_KERNEL_TYPE_GET_ROWS_F16, + GGML_METAL_KERNEL_TYPE_GET_ROWS_Q4_0, + GGML_METAL_KERNEL_TYPE_GET_ROWS_Q4_1, + GGML_METAL_KERNEL_TYPE_GET_ROWS_Q5_0, + GGML_METAL_KERNEL_TYPE_GET_ROWS_Q5_1, + GGML_METAL_KERNEL_TYPE_GET_ROWS_Q8_0, + GGML_METAL_KERNEL_TYPE_GET_ROWS_Q2_K, + GGML_METAL_KERNEL_TYPE_GET_ROWS_Q3_K, + GGML_METAL_KERNEL_TYPE_GET_ROWS_Q4_K, + GGML_METAL_KERNEL_TYPE_GET_ROWS_Q5_K, + GGML_METAL_KERNEL_TYPE_GET_ROWS_Q6_K, + GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ2_XXS, + GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ2_XS, + GGML_METAL_KERNEL_TYPE_GET_ROWS_I32, + GGML_METAL_KERNEL_TYPE_RMS_NORM, + GGML_METAL_KERNEL_TYPE_GROUP_NORM, + GGML_METAL_KERNEL_TYPE_NORM, + GGML_METAL_KERNEL_TYPE_MUL_MV_F32_F32, + GGML_METAL_KERNEL_TYPE_MUL_MV_F16_F16, + GGML_METAL_KERNEL_TYPE_MUL_MV_F16_F32, + GGML_METAL_KERNEL_TYPE_MUL_MV_F16_F32_1ROW, + GGML_METAL_KERNEL_TYPE_MUL_MV_F16_F32_L4, + GGML_METAL_KERNEL_TYPE_MUL_MV_Q4_0_F32, + GGML_METAL_KERNEL_TYPE_MUL_MV_Q4_1_F32, + GGML_METAL_KERNEL_TYPE_MUL_MV_Q5_0_F32, + GGML_METAL_KERNEL_TYPE_MUL_MV_Q5_1_F32, + GGML_METAL_KERNEL_TYPE_MUL_MV_Q8_0_F32, + GGML_METAL_KERNEL_TYPE_MUL_MV_Q2_K_F32, + GGML_METAL_KERNEL_TYPE_MUL_MV_Q3_K_F32, + GGML_METAL_KERNEL_TYPE_MUL_MV_Q4_K_F32, + GGML_METAL_KERNEL_TYPE_MUL_MV_Q5_K_F32, + GGML_METAL_KERNEL_TYPE_MUL_MV_Q6_K_F32, + GGML_METAL_KERNEL_TYPE_MUL_MV_IQ2_XXS_F32, + GGML_METAL_KERNEL_TYPE_MUL_MV_IQ2_XS_F32, + GGML_METAL_KERNEL_TYPE_MUL_MV_ID_F32_F32, + //GGML_METAL_KERNEL_TYPE_MUL_MV_ID_F16_F16, + GGML_METAL_KERNEL_TYPE_MUL_MV_ID_F16_F32, + //GGML_METAL_KERNEL_TYPE_MUL_MV_ID_F16_F32_1ROW, + //GGML_METAL_KERNEL_TYPE_MUL_MV_ID_F16_F32_L4, + GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q4_0_F32, + GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q4_1_F32, + GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q5_0_F32, + GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q5_1_F32, + GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q8_0_F32, + GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q2_K_F32, + GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q3_K_F32, + GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q4_K_F32, + GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q5_K_F32, + GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q6_K_F32, + GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ2_XXS_F32, + GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ2_XS_F32, + GGML_METAL_KERNEL_TYPE_MUL_MM_F32_F32, + GGML_METAL_KERNEL_TYPE_MUL_MM_F16_F32, + GGML_METAL_KERNEL_TYPE_MUL_MM_Q4_0_F32, + GGML_METAL_KERNEL_TYPE_MUL_MM_Q4_1_F32, + GGML_METAL_KERNEL_TYPE_MUL_MM_Q5_0_F32, + GGML_METAL_KERNEL_TYPE_MUL_MM_Q5_1_F32, + GGML_METAL_KERNEL_TYPE_MUL_MM_Q8_0_F32, + GGML_METAL_KERNEL_TYPE_MUL_MM_Q2_K_F32, + GGML_METAL_KERNEL_TYPE_MUL_MM_Q3_K_F32, + GGML_METAL_KERNEL_TYPE_MUL_MM_Q4_K_F32, + GGML_METAL_KERNEL_TYPE_MUL_MM_Q5_K_F32, + GGML_METAL_KERNEL_TYPE_MUL_MM_Q6_K_F32, + GGML_METAL_KERNEL_TYPE_MUL_MM_IQ2_XXS_F32, + GGML_METAL_KERNEL_TYPE_MUL_MM_IQ2_XS_F32, + GGML_METAL_KERNEL_TYPE_MUL_MM_ID_F32_F32, + GGML_METAL_KERNEL_TYPE_MUL_MM_ID_F16_F32, + GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q4_0_F32, + GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q4_1_F32, + GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q5_0_F32, + GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q5_1_F32, + GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q8_0_F32, + GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q2_K_F32, + GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q3_K_F32, + GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q4_K_F32, + GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q5_K_F32, + GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q6_K_F32, + GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ2_XXS_F32, + GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ2_XS_F32, + GGML_METAL_KERNEL_TYPE_ROPE_F32, + GGML_METAL_KERNEL_TYPE_ROPE_F16, + GGML_METAL_KERNEL_TYPE_ALIBI_F32, + GGML_METAL_KERNEL_TYPE_IM2COL_F16, + GGML_METAL_KERNEL_TYPE_UPSCALE_F32, + GGML_METAL_KERNEL_TYPE_PAD_F32, + GGML_METAL_KERNEL_TYPE_ARGSORT_F32_I32_ASC, + GGML_METAL_KERNEL_TYPE_ARGSORT_F32_I32_DESC, + GGML_METAL_KERNEL_TYPE_LEAKY_RELU_F32, + GGML_METAL_KERNEL_TYPE_CPY_F32_F16, + GGML_METAL_KERNEL_TYPE_CPY_F32_F32, + GGML_METAL_KERNEL_TYPE_CPY_F32_Q8_0, + GGML_METAL_KERNEL_TYPE_CPY_F32_Q4_0, + GGML_METAL_KERNEL_TYPE_CPY_F32_Q4_1, + //GGML_METAL_KERNEL_TYPE_CPY_F32_Q5_0, + //GGML_METAL_KERNEL_TYPE_CPY_F32_Q5_1, + GGML_METAL_KERNEL_TYPE_CPY_F16_F16, + GGML_METAL_KERNEL_TYPE_CPY_F16_F32, + GGML_METAL_KERNEL_TYPE_CONCAT, + GGML_METAL_KERNEL_TYPE_SQR, + GGML_METAL_KERNEL_TYPE_SUM_ROWS, + + GGML_METAL_KERNEL_TYPE_COUNT +}; + struct ggml_metal_context { int n_cb; @@ -50,134 +180,13 @@ struct ggml_metal_context { int n_buffers; struct ggml_metal_buffer buffers[GGML_METAL_MAX_BUFFERS]; + struct ggml_metal_kernel kernels[GGML_METAL_MAX_KERNELS]; + int concur_list[GGML_MAX_CONCUR]; int concur_list_len; - // custom kernels -#define GGML_METAL_DECL_KERNEL(name) \ - id function_##name; \ - id pipeline_##name - - GGML_METAL_DECL_KERNEL(add); - GGML_METAL_DECL_KERNEL(add_row); // TODO: avoid this extra kernel, instead extend the "add" kernel to support broadcast - GGML_METAL_DECL_KERNEL(mul); - GGML_METAL_DECL_KERNEL(mul_row); // TODO: avoid this extra kernel, instead extend the "mul" kernel to support broadcast - GGML_METAL_DECL_KERNEL(div); - GGML_METAL_DECL_KERNEL(div_row); - GGML_METAL_DECL_KERNEL(scale); - GGML_METAL_DECL_KERNEL(scale_4); - GGML_METAL_DECL_KERNEL(tanh); - GGML_METAL_DECL_KERNEL(relu); - GGML_METAL_DECL_KERNEL(gelu); - GGML_METAL_DECL_KERNEL(gelu_quick); - GGML_METAL_DECL_KERNEL(silu); - GGML_METAL_DECL_KERNEL(soft_max); - GGML_METAL_DECL_KERNEL(soft_max_4); - GGML_METAL_DECL_KERNEL(diag_mask_inf); - GGML_METAL_DECL_KERNEL(diag_mask_inf_8); - GGML_METAL_DECL_KERNEL(get_rows_f32); - GGML_METAL_DECL_KERNEL(get_rows_f16); - GGML_METAL_DECL_KERNEL(get_rows_q4_0); - GGML_METAL_DECL_KERNEL(get_rows_q4_1); - GGML_METAL_DECL_KERNEL(get_rows_q5_0); - GGML_METAL_DECL_KERNEL(get_rows_q5_1); - GGML_METAL_DECL_KERNEL(get_rows_q8_0); - GGML_METAL_DECL_KERNEL(get_rows_q2_K); - GGML_METAL_DECL_KERNEL(get_rows_q3_K); - GGML_METAL_DECL_KERNEL(get_rows_q4_K); - GGML_METAL_DECL_KERNEL(get_rows_q5_K); - GGML_METAL_DECL_KERNEL(get_rows_q6_K); - GGML_METAL_DECL_KERNEL(get_rows_i32); - GGML_METAL_DECL_KERNEL(get_rows_iq2_xxs); - GGML_METAL_DECL_KERNEL(get_rows_iq2_xs); - GGML_METAL_DECL_KERNEL(rms_norm); - GGML_METAL_DECL_KERNEL(group_norm); - GGML_METAL_DECL_KERNEL(norm); - GGML_METAL_DECL_KERNEL(mul_mv_f32_f32); - GGML_METAL_DECL_KERNEL(mul_mv_f16_f16); - GGML_METAL_DECL_KERNEL(mul_mv_f16_f32); - GGML_METAL_DECL_KERNEL(mul_mv_f16_f32_1row); - GGML_METAL_DECL_KERNEL(mul_mv_f16_f32_l4); - GGML_METAL_DECL_KERNEL(mul_mv_q4_0_f32); - GGML_METAL_DECL_KERNEL(mul_mv_q4_1_f32); - GGML_METAL_DECL_KERNEL(mul_mv_q5_0_f32); - GGML_METAL_DECL_KERNEL(mul_mv_q5_1_f32); - GGML_METAL_DECL_KERNEL(mul_mv_q8_0_f32); - GGML_METAL_DECL_KERNEL(mul_mv_q2_K_f32); - GGML_METAL_DECL_KERNEL(mul_mv_q3_K_f32); - GGML_METAL_DECL_KERNEL(mul_mv_q4_K_f32); - GGML_METAL_DECL_KERNEL(mul_mv_q5_K_f32); - GGML_METAL_DECL_KERNEL(mul_mv_q6_K_f32); - GGML_METAL_DECL_KERNEL(mul_mv_iq2_xxs_f32); - GGML_METAL_DECL_KERNEL(mul_mv_iq2_xs_f32); - GGML_METAL_DECL_KERNEL(mul_mv_id_f32_f32); - //GGML_METAL_DECL_KERNEL(mul_mv_id_f16_f16); - GGML_METAL_DECL_KERNEL(mul_mv_id_f16_f32); - //GGML_METAL_DECL_KERNEL(mul_mv_id_f16_f32_1row); - //GGML_METAL_DECL_KERNEL(mul_mv_id_f16_f32_l4); - GGML_METAL_DECL_KERNEL(mul_mv_id_q4_0_f32); - GGML_METAL_DECL_KERNEL(mul_mv_id_q4_1_f32); - GGML_METAL_DECL_KERNEL(mul_mv_id_q5_0_f32); - GGML_METAL_DECL_KERNEL(mul_mv_id_q5_1_f32); - GGML_METAL_DECL_KERNEL(mul_mv_id_q8_0_f32); - GGML_METAL_DECL_KERNEL(mul_mv_id_q2_K_f32); - GGML_METAL_DECL_KERNEL(mul_mv_id_q3_K_f32); - GGML_METAL_DECL_KERNEL(mul_mv_id_q4_K_f32); - GGML_METAL_DECL_KERNEL(mul_mv_id_q5_K_f32); - GGML_METAL_DECL_KERNEL(mul_mv_id_q6_K_f32); - GGML_METAL_DECL_KERNEL(mul_mv_id_iq2_xxs_f32); - GGML_METAL_DECL_KERNEL(mul_mv_id_iq2_xs_f32); - GGML_METAL_DECL_KERNEL(mul_mm_f32_f32); - GGML_METAL_DECL_KERNEL(mul_mm_f16_f32); - GGML_METAL_DECL_KERNEL(mul_mm_q4_0_f32); - GGML_METAL_DECL_KERNEL(mul_mm_q4_1_f32); - GGML_METAL_DECL_KERNEL(mul_mm_q5_0_f32); - GGML_METAL_DECL_KERNEL(mul_mm_q5_1_f32); - GGML_METAL_DECL_KERNEL(mul_mm_q8_0_f32); - GGML_METAL_DECL_KERNEL(mul_mm_q2_K_f32); - GGML_METAL_DECL_KERNEL(mul_mm_q3_K_f32); - GGML_METAL_DECL_KERNEL(mul_mm_q4_K_f32); - GGML_METAL_DECL_KERNEL(mul_mm_q5_K_f32); - GGML_METAL_DECL_KERNEL(mul_mm_q6_K_f32); - GGML_METAL_DECL_KERNEL(mul_mm_iq2_xxs_f32); - GGML_METAL_DECL_KERNEL(mul_mm_iq2_xs_f32); - GGML_METAL_DECL_KERNEL(mul_mm_id_f32_f32); - GGML_METAL_DECL_KERNEL(mul_mm_id_f16_f32); - GGML_METAL_DECL_KERNEL(mul_mm_id_q4_0_f32); - GGML_METAL_DECL_KERNEL(mul_mm_id_q4_1_f32); - GGML_METAL_DECL_KERNEL(mul_mm_id_q5_0_f32); - GGML_METAL_DECL_KERNEL(mul_mm_id_q5_1_f32); - GGML_METAL_DECL_KERNEL(mul_mm_id_q8_0_f32); - GGML_METAL_DECL_KERNEL(mul_mm_id_q2_K_f32); - GGML_METAL_DECL_KERNEL(mul_mm_id_q3_K_f32); - GGML_METAL_DECL_KERNEL(mul_mm_id_q4_K_f32); - GGML_METAL_DECL_KERNEL(mul_mm_id_q5_K_f32); - GGML_METAL_DECL_KERNEL(mul_mm_id_q6_K_f32); - GGML_METAL_DECL_KERNEL(mul_mm_id_iq2_xxs_f32); - GGML_METAL_DECL_KERNEL(mul_mm_id_iq2_xs_f32); - GGML_METAL_DECL_KERNEL(rope_f32); - GGML_METAL_DECL_KERNEL(rope_f16); - GGML_METAL_DECL_KERNEL(alibi_f32); - GGML_METAL_DECL_KERNEL(im2col_f16); - GGML_METAL_DECL_KERNEL(upscale_f32); - GGML_METAL_DECL_KERNEL(pad_f32); - GGML_METAL_DECL_KERNEL(argsort_f32_i32_asc); - GGML_METAL_DECL_KERNEL(argsort_f32_i32_desc); - GGML_METAL_DECL_KERNEL(leaky_relu_f32); - GGML_METAL_DECL_KERNEL(cpy_f32_f16); - GGML_METAL_DECL_KERNEL(cpy_f32_f32); - GGML_METAL_DECL_KERNEL(cpy_f32_q8_0); - GGML_METAL_DECL_KERNEL(cpy_f32_q4_0); - GGML_METAL_DECL_KERNEL(cpy_f32_q4_1); - //GGML_METAL_DECL_KERNEL(cpy_f32_q5_0); - //GGML_METAL_DECL_KERNEL(cpy_f32_q5_1); - GGML_METAL_DECL_KERNEL(cpy_f16_f16); - GGML_METAL_DECL_KERNEL(cpy_f16_f32); - GGML_METAL_DECL_KERNEL(concat); - GGML_METAL_DECL_KERNEL(sqr); - GGML_METAL_DECL_KERNEL(sum_rows); - -#undef GGML_METAL_DECL_KERNEL + bool support_simdgroup_reduction; + bool support_simdgroup_mm; }; // MSL code @@ -298,19 +307,22 @@ struct ggml_metal_context * ggml_metal_init(int n_cb) { return NULL; } - MTLCompileOptions* options = nil; + // dictionary of preprocessor macros + NSMutableDictionary * prep = [NSMutableDictionary dictionary]; + #ifdef GGML_QKK_64 - options = [MTLCompileOptions new]; - options.preprocessorMacros = @{ @"QK_K" : @(64) }; + prep[@"QK_K"] = @(64); #endif - // try to disable fast-math - // NOTE: this seems to have no effect whatsoever - // instead, in order to disable fast-math, we have to build default.metallib from the command line - // using xcrun -sdk macosx metal -fno-fast-math -c ggml-metal.metal -o ggml-metal.air - // and go through the "pre-compiled library found" path above + + MTLCompileOptions* options = [MTLCompileOptions new]; + options.preprocessorMacros = prep; + //[options setFastMathEnabled:false]; ctx->library = [ctx->device newLibraryWithSource:src options:options error:&error]; + + [options release]; + [prep release]; } if (error) { @@ -323,16 +335,41 @@ struct ggml_metal_context * ggml_metal_init(int n_cb) { // print MTL GPU family: GGML_METAL_LOG_INFO("%s: GPU name: %s\n", __func__, [[ctx->device name] UTF8String]); + const NSInteger MTLGPUFamilyMetal3 = 5001; + // determine max supported GPU family // https://developer.apple.com/metal/Metal-Shading-Language-Specification.pdf // https://developer.apple.com/metal/Metal-Feature-Set-Tables.pdf - for (int i = MTLGPUFamilyApple1 + 20; i >= MTLGPUFamilyApple1; --i) { - if ([ctx->device supportsFamily:i]) { - GGML_METAL_LOG_INFO("%s: GPU family: MTLGPUFamilyApple%d (%d)\n", __func__, i - (int) MTLGPUFamilyApple1 + 1, i); - break; + { + for (int i = MTLGPUFamilyApple1 + 20; i >= MTLGPUFamilyApple1; --i) { + if ([ctx->device supportsFamily:i]) { + GGML_METAL_LOG_INFO("%s: GPU family: MTLGPUFamilyApple%d (%d)\n", __func__, i - (int) MTLGPUFamilyApple1 + 1, i); + break; + } + } + + for (int i = MTLGPUFamilyCommon1 + 5; i >= MTLGPUFamilyCommon1; --i) { + if ([ctx->device supportsFamily:i]) { + GGML_METAL_LOG_INFO("%s: GPU family: MTLGPUFamilyCommon%d (%d)\n", __func__, i - (int) MTLGPUFamilyCommon1 + 1, i); + break; + } + } + + for (int i = MTLGPUFamilyMetal3 + 5; i >= MTLGPUFamilyMetal3; --i) { + if ([ctx->device supportsFamily:i]) { + GGML_METAL_LOG_INFO("%s: GPU family: MTLGPUFamilyMetal%d (%d)\n", __func__, i - (int) MTLGPUFamilyMetal3 + 3, i); + break; + } } } + ctx->support_simdgroup_reduction = [ctx->device supportsFamily:MTLGPUFamilyApple7]; + ctx->support_simdgroup_reduction |= [ctx->device supportsFamily:MTLGPUFamilyMetal3]; + + ctx->support_simdgroup_mm = [ctx->device supportsFamily:MTLGPUFamilyApple7]; + + GGML_METAL_LOG_INFO("%s: simdgroup reduction support = %s\n", __func__, ctx->support_simdgroup_reduction ? "true" : "false"); + GGML_METAL_LOG_INFO("%s: simdgroup matrix mul. support = %s\n", __func__, ctx->support_simdgroup_mm ? "true" : "false"); GGML_METAL_LOG_INFO("%s: hasUnifiedMemory = %s\n", __func__, ctx->device.hasUnifiedMemory ? "true" : "false"); GGML_METAL_LOG_INFO("%s: recommendedMaxWorkingSetSize = %8.2f MB\n", __func__, ctx->device.recommendedMaxWorkingSetSize / 1e6); if (ctx->device.maxTransferRate != 0) { @@ -346,141 +383,152 @@ struct ggml_metal_context * ggml_metal_init(int n_cb) { { NSError * error = nil; + for (int i = 0; i < GGML_METAL_MAX_KERNELS; ++i) { + ctx->kernels[i].function = nil; + ctx->kernels[i].pipeline = nil; + } + /* - GGML_METAL_LOG_INFO("%s: loaded %-32s %16p | th_max = %4d | th_width = %4d\n", __func__, "kernel_"#name, (void *) ctx->pipeline_##name, \ - (int) ctx->pipeline_##name.maxTotalThreadsPerThreadgroup, \ - (int) ctx->pipeline_##name.threadExecutionWidth); \ + GGML_METAL_LOG_INFO("%s: loaded %-32s %16p | th_max = %4d | th_width = %4d\n", __func__, "kernel_"#name, (void *) kernel->pipeline, \ + (int) kernel->pipeline.maxTotalThreadsPerThreadgroup, \ + (int) kernel->pipeline.threadExecutionWidth); \ */ -#define GGML_METAL_ADD_KERNEL(name) \ - ctx->function_##name = [ctx->library newFunctionWithName:@"kernel_"#name]; \ - ctx->pipeline_##name = [ctx->device newComputePipelineStateWithFunction:ctx->function_##name error:&error]; \ - if (error) { \ - GGML_METAL_LOG_ERROR("%s: error: load pipeline error: %s\n", __func__, [[error description] UTF8String]); \ - return NULL; \ +#define GGML_METAL_ADD_KERNEL(e, name, supported) \ + if (supported) { \ + struct ggml_metal_kernel * kernel = &ctx->kernels[e]; \ + kernel->function = [ctx->library newFunctionWithName:@"kernel_"#name]; \ + kernel->pipeline = [ctx->device newComputePipelineStateWithFunction:kernel->function error:&error]; \ + GGML_METAL_LOG_INFO("%s: loaded %-32s %16p | th_max = %4d | th_width = %4d\n", __func__, "kernel_"#name, (void *) kernel->pipeline, \ + (int) kernel->pipeline.maxTotalThreadsPerThreadgroup, \ + (int) kernel->pipeline.threadExecutionWidth); \ + if (error) { \ + GGML_METAL_LOG_ERROR("%s: error: load pipeline error: %s\n", __func__, [[error description] UTF8String]); \ + return NULL; \ + } \ + } else { \ + GGML_METAL_LOG_WARN("%s: skipping %-32s (not supported)\n", __func__, "kernel_"#name); \ } - GGML_METAL_ADD_KERNEL(add); - GGML_METAL_ADD_KERNEL(add_row); - GGML_METAL_ADD_KERNEL(mul); - GGML_METAL_ADD_KERNEL(mul_row); - GGML_METAL_ADD_KERNEL(div); - GGML_METAL_ADD_KERNEL(div_row); - GGML_METAL_ADD_KERNEL(scale); - GGML_METAL_ADD_KERNEL(scale_4); - GGML_METAL_ADD_KERNEL(tanh); - GGML_METAL_ADD_KERNEL(relu); - GGML_METAL_ADD_KERNEL(gelu); - GGML_METAL_ADD_KERNEL(gelu_quick); - GGML_METAL_ADD_KERNEL(silu); - GGML_METAL_ADD_KERNEL(soft_max); - GGML_METAL_ADD_KERNEL(soft_max_4); - GGML_METAL_ADD_KERNEL(diag_mask_inf); - GGML_METAL_ADD_KERNEL(diag_mask_inf_8); - GGML_METAL_ADD_KERNEL(get_rows_f32); - GGML_METAL_ADD_KERNEL(get_rows_f16); - GGML_METAL_ADD_KERNEL(get_rows_q4_0); - GGML_METAL_ADD_KERNEL(get_rows_q4_1); - GGML_METAL_ADD_KERNEL(get_rows_q5_0); - GGML_METAL_ADD_KERNEL(get_rows_q5_1); - GGML_METAL_ADD_KERNEL(get_rows_q8_0); - GGML_METAL_ADD_KERNEL(get_rows_q2_K); - GGML_METAL_ADD_KERNEL(get_rows_q3_K); - GGML_METAL_ADD_KERNEL(get_rows_q4_K); - GGML_METAL_ADD_KERNEL(get_rows_q5_K); - GGML_METAL_ADD_KERNEL(get_rows_q6_K); - GGML_METAL_ADD_KERNEL(get_rows_i32); - GGML_METAL_ADD_KERNEL(get_rows_iq2_xxs); - GGML_METAL_ADD_KERNEL(get_rows_iq2_xs); - GGML_METAL_ADD_KERNEL(rms_norm); - GGML_METAL_ADD_KERNEL(group_norm); - GGML_METAL_ADD_KERNEL(norm); - GGML_METAL_ADD_KERNEL(mul_mv_f32_f32); - GGML_METAL_ADD_KERNEL(mul_mv_f16_f16); - GGML_METAL_ADD_KERNEL(mul_mv_f16_f32); - GGML_METAL_ADD_KERNEL(mul_mv_f16_f32_1row); - GGML_METAL_ADD_KERNEL(mul_mv_f16_f32_l4); - GGML_METAL_ADD_KERNEL(mul_mv_q4_0_f32); - GGML_METAL_ADD_KERNEL(mul_mv_q4_1_f32); - GGML_METAL_ADD_KERNEL(mul_mv_q5_0_f32); - GGML_METAL_ADD_KERNEL(mul_mv_q5_1_f32); - GGML_METAL_ADD_KERNEL(mul_mv_q8_0_f32); - GGML_METAL_ADD_KERNEL(mul_mv_q2_K_f32); - GGML_METAL_ADD_KERNEL(mul_mv_q3_K_f32); - GGML_METAL_ADD_KERNEL(mul_mv_q4_K_f32); - GGML_METAL_ADD_KERNEL(mul_mv_q5_K_f32); - GGML_METAL_ADD_KERNEL(mul_mv_q6_K_f32); - GGML_METAL_ADD_KERNEL(mul_mv_iq2_xxs_f32); - GGML_METAL_ADD_KERNEL(mul_mv_iq2_xs_f32); - GGML_METAL_ADD_KERNEL(mul_mv_id_f32_f32); - //GGML_METAL_ADD_KERNEL(mul_mv_id_f16_f16); - GGML_METAL_ADD_KERNEL(mul_mv_id_f16_f32); - //GGML_METAL_ADD_KERNEL(mul_mv_id_f16_f32_1row); - //GGML_METAL_ADD_KERNEL(mul_mv_id_f16_f32_l4); - GGML_METAL_ADD_KERNEL(mul_mv_id_q4_0_f32); - GGML_METAL_ADD_KERNEL(mul_mv_id_q4_1_f32); - GGML_METAL_ADD_KERNEL(mul_mv_id_q5_0_f32); - GGML_METAL_ADD_KERNEL(mul_mv_id_q5_1_f32); - GGML_METAL_ADD_KERNEL(mul_mv_id_q8_0_f32); - GGML_METAL_ADD_KERNEL(mul_mv_id_q2_K_f32); - GGML_METAL_ADD_KERNEL(mul_mv_id_q3_K_f32); - GGML_METAL_ADD_KERNEL(mul_mv_id_q4_K_f32); - GGML_METAL_ADD_KERNEL(mul_mv_id_q5_K_f32); - GGML_METAL_ADD_KERNEL(mul_mv_id_q6_K_f32); - GGML_METAL_ADD_KERNEL(mul_mv_id_iq2_xxs_f32); - GGML_METAL_ADD_KERNEL(mul_mv_id_iq2_xs_f32); - if ([ctx->device supportsFamily:MTLGPUFamilyApple7]) { - GGML_METAL_ADD_KERNEL(mul_mm_f32_f32); - GGML_METAL_ADD_KERNEL(mul_mm_f16_f32); - GGML_METAL_ADD_KERNEL(mul_mm_q4_0_f32); - GGML_METAL_ADD_KERNEL(mul_mm_q4_1_f32); - GGML_METAL_ADD_KERNEL(mul_mm_q5_0_f32); - GGML_METAL_ADD_KERNEL(mul_mm_q5_1_f32); - GGML_METAL_ADD_KERNEL(mul_mm_q8_0_f32); - GGML_METAL_ADD_KERNEL(mul_mm_q2_K_f32); - GGML_METAL_ADD_KERNEL(mul_mm_q3_K_f32); - GGML_METAL_ADD_KERNEL(mul_mm_q4_K_f32); - GGML_METAL_ADD_KERNEL(mul_mm_q5_K_f32); - GGML_METAL_ADD_KERNEL(mul_mm_q6_K_f32); - GGML_METAL_ADD_KERNEL(mul_mm_iq2_xxs_f32); - GGML_METAL_ADD_KERNEL(mul_mm_iq2_xs_f32); - GGML_METAL_ADD_KERNEL(mul_mm_id_f32_f32); - GGML_METAL_ADD_KERNEL(mul_mm_id_f16_f32); - GGML_METAL_ADD_KERNEL(mul_mm_id_q4_0_f32); - GGML_METAL_ADD_KERNEL(mul_mm_id_q4_1_f32); - GGML_METAL_ADD_KERNEL(mul_mm_id_q5_0_f32); - GGML_METAL_ADD_KERNEL(mul_mm_id_q5_1_f32); - GGML_METAL_ADD_KERNEL(mul_mm_id_q8_0_f32); - GGML_METAL_ADD_KERNEL(mul_mm_id_q2_K_f32); - GGML_METAL_ADD_KERNEL(mul_mm_id_q3_K_f32); - GGML_METAL_ADD_KERNEL(mul_mm_id_q4_K_f32); - GGML_METAL_ADD_KERNEL(mul_mm_id_q5_K_f32); - GGML_METAL_ADD_KERNEL(mul_mm_id_q6_K_f32); - GGML_METAL_ADD_KERNEL(mul_mm_id_iq2_xxs_f32); - GGML_METAL_ADD_KERNEL(mul_mm_id_iq2_xs_f32); - } - GGML_METAL_ADD_KERNEL(rope_f32); - GGML_METAL_ADD_KERNEL(rope_f16); - GGML_METAL_ADD_KERNEL(alibi_f32); - GGML_METAL_ADD_KERNEL(im2col_f16); - GGML_METAL_ADD_KERNEL(upscale_f32); - GGML_METAL_ADD_KERNEL(pad_f32); - GGML_METAL_ADD_KERNEL(argsort_f32_i32_asc); - GGML_METAL_ADD_KERNEL(argsort_f32_i32_desc); - GGML_METAL_ADD_KERNEL(leaky_relu_f32); - GGML_METAL_ADD_KERNEL(cpy_f32_f16); - GGML_METAL_ADD_KERNEL(cpy_f32_f32); - GGML_METAL_ADD_KERNEL(cpy_f32_q8_0); - GGML_METAL_ADD_KERNEL(cpy_f32_q4_0); - GGML_METAL_ADD_KERNEL(cpy_f32_q4_1); - //GGML_METAL_ADD_KERNEL(cpy_f32_q5_0); - //GGML_METAL_ADD_KERNEL(cpy_f32_q5_1); - GGML_METAL_ADD_KERNEL(cpy_f16_f16); - GGML_METAL_ADD_KERNEL(cpy_f16_f32); - GGML_METAL_ADD_KERNEL(concat); - GGML_METAL_ADD_KERNEL(sqr); - GGML_METAL_ADD_KERNEL(sum_rows); + // simd_sum and simd_max requires MTLGPUFamilyApple7 -#undef GGML_METAL_ADD_KERNEL + GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_ADD, add, true); + GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_ADD_ROW, add_row, true); + GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL, mul, true); + GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_ROW, mul_row, true); + GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_DIV, div, true); + GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_DIV_ROW, div_row, true); + GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_SCALE, scale, true); + GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_SCALE_4, scale_4, true); + GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_TANH, tanh, true); + GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_RELU, relu, true); + GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GELU, gelu, true); + GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GELU_QUICK, gelu_quick, true); + GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_SILU, silu, true); + GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_SOFT_MAX, soft_max, ctx->support_simdgroup_reduction); + GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_SOFT_MAX_4, soft_max_4, ctx->support_simdgroup_reduction); + GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_DIAG_MASK_INF, diag_mask_inf, true); + GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_DIAG_MASK_INF_8, diag_mask_inf_8, true); + GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_F32, get_rows_f32, true); + GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_F16, get_rows_f16, true); + GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_Q4_0, get_rows_q4_0, true); + GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_Q4_1, get_rows_q4_1, true); + GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_Q5_0, get_rows_q5_0, true); + GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_Q5_1, get_rows_q5_1, true); + GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_Q8_0, get_rows_q8_0, true); + GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_Q2_K, get_rows_q2_K, true); + GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_Q3_K, get_rows_q3_K, true); + GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_Q4_K, get_rows_q4_K, true); + GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_Q5_K, get_rows_q5_K, true); + GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_Q6_K, get_rows_q6_K, true); + GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ2_XXS, get_rows_iq2_xxs, true); + GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ2_XS, get_rows_iq2_xs, true); + GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_I32, get_rows_i32, true); + GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_RMS_NORM, rms_norm, ctx->support_simdgroup_reduction); + GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GROUP_NORM, group_norm, ctx->support_simdgroup_reduction); + GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_NORM, norm, true); + GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_F32_F32, mul_mv_f32_f32, ctx->support_simdgroup_reduction); + GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_F16_F16, mul_mv_f16_f16, ctx->support_simdgroup_reduction); + GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_F16_F32, mul_mv_f16_f32, ctx->support_simdgroup_reduction); + GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_F16_F32_1ROW, mul_mv_f16_f32_1row, ctx->support_simdgroup_reduction); + GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_F16_F32_L4, mul_mv_f16_f32_l4, ctx->support_simdgroup_reduction); + GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_Q4_0_F32, mul_mv_q4_0_f32, ctx->support_simdgroup_reduction); + GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_Q4_1_F32, mul_mv_q4_1_f32, ctx->support_simdgroup_reduction); + GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_Q5_0_F32, mul_mv_q5_0_f32, ctx->support_simdgroup_reduction); + GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_Q5_1_F32, mul_mv_q5_1_f32, ctx->support_simdgroup_reduction); + GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_Q8_0_F32, mul_mv_q8_0_f32, ctx->support_simdgroup_reduction); + GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_Q2_K_F32, mul_mv_q2_K_f32, ctx->support_simdgroup_reduction); + GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_Q3_K_F32, mul_mv_q3_K_f32, ctx->support_simdgroup_reduction); + GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_Q4_K_F32, mul_mv_q4_K_f32, ctx->support_simdgroup_reduction); + GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_Q5_K_F32, mul_mv_q5_K_f32, ctx->support_simdgroup_reduction); + GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_Q6_K_F32, mul_mv_q6_K_f32, ctx->support_simdgroup_reduction); + GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_IQ2_XXS_F32, mul_mv_iq2_xxs_f32, ctx->support_simdgroup_reduction); + GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_IQ2_XS_F32, mul_mv_iq2_xs_f32, ctx->support_simdgroup_reduction); + GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_F32_F32, mul_mv_id_f32_f32, ctx->support_simdgroup_reduction); + //GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_F16_F16, mul_mv_id_f16_f16, ctx->support_simdgroup_reduction); + GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_F16_F32, mul_mv_id_f16_f32, ctx->support_simdgroup_reduction); + //GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_F16_F32_1ROW, mul_mv_id_f16_f32_1row, ctx->support_simdgroup_reduction); + //GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_F16_F32_L4, mul_mv_id_f16_f32_l4, ctx->support_simdgroup_reduction); + GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q4_0_F32, mul_mv_id_q4_0_f32, ctx->support_simdgroup_reduction); + GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q4_1_F32, mul_mv_id_q4_1_f32, ctx->support_simdgroup_reduction); + GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q5_0_F32, mul_mv_id_q5_0_f32, ctx->support_simdgroup_reduction); + GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q5_1_F32, mul_mv_id_q5_1_f32, ctx->support_simdgroup_reduction); + GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q8_0_F32, mul_mv_id_q8_0_f32, ctx->support_simdgroup_reduction); + GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q2_K_F32, mul_mv_id_q2_K_f32, ctx->support_simdgroup_reduction); + GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q3_K_F32, mul_mv_id_q3_K_f32, ctx->support_simdgroup_reduction); + GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q4_K_F32, mul_mv_id_q4_K_f32, ctx->support_simdgroup_reduction); + GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q5_K_F32, mul_mv_id_q5_K_f32, ctx->support_simdgroup_reduction); + GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q6_K_F32, mul_mv_id_q6_K_f32, ctx->support_simdgroup_reduction); + GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ2_XXS_F32, mul_mv_id_iq2_xxs_f32, ctx->support_simdgroup_reduction); + GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ2_XS_F32, mul_mv_id_iq2_xs_f32, ctx->support_simdgroup_reduction); + GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_F32_F32, mul_mm_f32_f32, ctx->support_simdgroup_mm); + GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_F16_F32, mul_mm_f16_f32, ctx->support_simdgroup_mm); + GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_Q4_0_F32, mul_mm_q4_0_f32, ctx->support_simdgroup_mm); + GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_Q4_1_F32, mul_mm_q4_1_f32, ctx->support_simdgroup_mm); + GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_Q5_0_F32, mul_mm_q5_0_f32, ctx->support_simdgroup_mm); + GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_Q5_1_F32, mul_mm_q5_1_f32, ctx->support_simdgroup_mm); + GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_Q8_0_F32, mul_mm_q8_0_f32, ctx->support_simdgroup_mm); + GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_Q2_K_F32, mul_mm_q2_K_f32, ctx->support_simdgroup_mm); + GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_Q3_K_F32, mul_mm_q3_K_f32, ctx->support_simdgroup_mm); + GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_Q4_K_F32, mul_mm_q4_K_f32, ctx->support_simdgroup_mm); + GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_Q5_K_F32, mul_mm_q5_K_f32, ctx->support_simdgroup_mm); + GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_Q6_K_F32, mul_mm_q6_K_f32, ctx->support_simdgroup_mm); + GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_IQ2_XXS_F32, mul_mm_iq2_xxs_f32, ctx->support_simdgroup_mm); + GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_IQ2_XS_F32, mul_mm_iq2_xs_f32, ctx->support_simdgroup_mm); + GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_F32_F32, mul_mm_id_f32_f32, ctx->support_simdgroup_mm); + GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_F16_F32, mul_mm_id_f16_f32, ctx->support_simdgroup_mm); + GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q4_0_F32, mul_mm_id_q4_0_f32, ctx->support_simdgroup_mm); + GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q4_1_F32, mul_mm_id_q4_1_f32, ctx->support_simdgroup_mm); + GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q5_0_F32, mul_mm_id_q5_0_f32, ctx->support_simdgroup_mm); + GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q5_1_F32, mul_mm_id_q5_1_f32, ctx->support_simdgroup_mm); + GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q8_0_F32, mul_mm_id_q8_0_f32, ctx->support_simdgroup_mm); + GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q2_K_F32, mul_mm_id_q2_K_f32, ctx->support_simdgroup_mm); + GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q3_K_F32, mul_mm_id_q3_K_f32, ctx->support_simdgroup_mm); + GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q4_K_F32, mul_mm_id_q4_K_f32, ctx->support_simdgroup_mm); + GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q5_K_F32, mul_mm_id_q5_K_f32, ctx->support_simdgroup_mm); + GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q6_K_F32, mul_mm_id_q6_K_f32, ctx->support_simdgroup_mm); + GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ2_XXS_F32, mul_mm_id_iq2_xxs_f32, ctx->support_simdgroup_mm); + GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ2_XS_F32, mul_mm_id_iq2_xs_f32, ctx->support_simdgroup_mm); + GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_ROPE_F32, rope_f32, true); + GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_ROPE_F16, rope_f16, true); + GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_ALIBI_F32, alibi_f32, true); + GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_IM2COL_F16, im2col_f16, true); + GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_UPSCALE_F32, upscale_f32, true); + GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_PAD_F32, pad_f32, true); + GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_ARGSORT_F32_I32_ASC, argsort_f32_i32_asc, true); + GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_ARGSORT_F32_I32_DESC, argsort_f32_i32_desc, true); + GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_LEAKY_RELU_F32, leaky_relu_f32, true); + GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_CPY_F32_F16, cpy_f32_f16, true); + GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_CPY_F32_F32, cpy_f32_f32, true); + GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_CPY_F32_Q8_0, cpy_f32_q8_0, true); + GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_CPY_F32_Q4_0, cpy_f32_q4_0, true); + GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_CPY_F32_Q4_1, cpy_f32_q4_1, true); + //GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_CPY_F32_Q5_0, cpy_f32_q5_0, true); + //GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_CPY_F32_Q5_1, cpy_f32_q5_1, true); + GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_CPY_F16_F16, cpy_f16_f16, true); + GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_CPY_F16_F32, cpy_f16_f32, true); + GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_CONCAT, concat, true); + GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_SQR, sqr, true); + GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_SUM_ROWS, sum_rows, true); } return ctx; @@ -488,137 +536,21 @@ struct ggml_metal_context * ggml_metal_init(int n_cb) { void ggml_metal_free(struct ggml_metal_context * ctx) { GGML_METAL_LOG_INFO("%s: deallocating\n", __func__); -#define GGML_METAL_DEL_KERNEL(name) \ - [ctx->function_##name release]; \ - [ctx->pipeline_##name release]; - - GGML_METAL_DEL_KERNEL(add); - GGML_METAL_DEL_KERNEL(add_row); - GGML_METAL_DEL_KERNEL(mul); - GGML_METAL_DEL_KERNEL(mul_row); - GGML_METAL_DEL_KERNEL(div); - GGML_METAL_DEL_KERNEL(div_row); - GGML_METAL_DEL_KERNEL(scale); - GGML_METAL_DEL_KERNEL(scale_4); - GGML_METAL_DEL_KERNEL(tanh); - GGML_METAL_DEL_KERNEL(relu); - GGML_METAL_DEL_KERNEL(gelu); - GGML_METAL_DEL_KERNEL(gelu_quick); - GGML_METAL_DEL_KERNEL(silu); - GGML_METAL_DEL_KERNEL(soft_max); - GGML_METAL_DEL_KERNEL(soft_max_4); - GGML_METAL_DEL_KERNEL(diag_mask_inf); - GGML_METAL_DEL_KERNEL(diag_mask_inf_8); - GGML_METAL_DEL_KERNEL(get_rows_f32); - GGML_METAL_DEL_KERNEL(get_rows_f16); - GGML_METAL_DEL_KERNEL(get_rows_q4_0); - GGML_METAL_DEL_KERNEL(get_rows_q4_1); - GGML_METAL_DEL_KERNEL(get_rows_q5_0); - GGML_METAL_DEL_KERNEL(get_rows_q5_1); - GGML_METAL_DEL_KERNEL(get_rows_q8_0); - GGML_METAL_DEL_KERNEL(get_rows_q2_K); - GGML_METAL_DEL_KERNEL(get_rows_q3_K); - GGML_METAL_DEL_KERNEL(get_rows_q4_K); - GGML_METAL_DEL_KERNEL(get_rows_q5_K); - GGML_METAL_DEL_KERNEL(get_rows_q6_K); - GGML_METAL_DEL_KERNEL(get_rows_i32); - GGML_METAL_DEL_KERNEL(get_rows_iq2_xxs); - GGML_METAL_DEL_KERNEL(get_rows_iq2_xs); - GGML_METAL_DEL_KERNEL(rms_norm); - GGML_METAL_DEL_KERNEL(group_norm); - GGML_METAL_DEL_KERNEL(norm); - GGML_METAL_DEL_KERNEL(mul_mv_f32_f32); - GGML_METAL_DEL_KERNEL(mul_mv_f16_f16); - GGML_METAL_DEL_KERNEL(mul_mv_f16_f32); - GGML_METAL_DEL_KERNEL(mul_mv_f16_f32_1row); - GGML_METAL_DEL_KERNEL(mul_mv_f16_f32_l4); - GGML_METAL_DEL_KERNEL(mul_mv_q4_0_f32); - GGML_METAL_DEL_KERNEL(mul_mv_q4_1_f32); - GGML_METAL_DEL_KERNEL(mul_mv_q5_0_f32); - GGML_METAL_DEL_KERNEL(mul_mv_q5_1_f32); - GGML_METAL_DEL_KERNEL(mul_mv_q8_0_f32); - GGML_METAL_DEL_KERNEL(mul_mv_q2_K_f32); - GGML_METAL_DEL_KERNEL(mul_mv_q3_K_f32); - GGML_METAL_DEL_KERNEL(mul_mv_q4_K_f32); - GGML_METAL_DEL_KERNEL(mul_mv_q5_K_f32); - GGML_METAL_DEL_KERNEL(mul_mv_q6_K_f32); - GGML_METAL_DEL_KERNEL(mul_mv_iq2_xxs_f32); - GGML_METAL_DEL_KERNEL(mul_mv_iq2_xs_f32); - GGML_METAL_DEL_KERNEL(mul_mv_id_f32_f32); - //GGML_METAL_DEL_KERNEL(mul_mv_id_f16_f16); - GGML_METAL_DEL_KERNEL(mul_mv_id_f16_f32); - //GGML_METAL_DEL_KERNEL(mul_mv_id_f16_f32_1row); - //GGML_METAL_DEL_KERNEL(mul_mv_id_f16_f32_l4); - GGML_METAL_DEL_KERNEL(mul_mv_id_q4_0_f32); - GGML_METAL_DEL_KERNEL(mul_mv_id_q4_1_f32); - GGML_METAL_DEL_KERNEL(mul_mv_id_q5_0_f32); - GGML_METAL_DEL_KERNEL(mul_mv_id_q5_1_f32); - GGML_METAL_DEL_KERNEL(mul_mv_id_q8_0_f32); - GGML_METAL_DEL_KERNEL(mul_mv_id_q2_K_f32); - GGML_METAL_DEL_KERNEL(mul_mv_id_q3_K_f32); - GGML_METAL_DEL_KERNEL(mul_mv_id_q4_K_f32); - GGML_METAL_DEL_KERNEL(mul_mv_id_q5_K_f32); - GGML_METAL_DEL_KERNEL(mul_mv_id_q6_K_f32); - GGML_METAL_DEL_KERNEL(mul_mv_id_iq2_xxs_f32); - GGML_METAL_DEL_KERNEL(mul_mv_id_iq2_xs_f32); - if ([ctx->device supportsFamily:MTLGPUFamilyApple7]) { - GGML_METAL_DEL_KERNEL(mul_mm_f32_f32); - GGML_METAL_DEL_KERNEL(mul_mm_f16_f32); - GGML_METAL_DEL_KERNEL(mul_mm_q4_0_f32); - GGML_METAL_DEL_KERNEL(mul_mm_q4_1_f32); - GGML_METAL_DEL_KERNEL(mul_mm_q5_0_f32); - GGML_METAL_DEL_KERNEL(mul_mm_q5_1_f32); - GGML_METAL_DEL_KERNEL(mul_mm_q8_0_f32); - GGML_METAL_DEL_KERNEL(mul_mm_q2_K_f32); - GGML_METAL_DEL_KERNEL(mul_mm_q3_K_f32); - GGML_METAL_DEL_KERNEL(mul_mm_q4_K_f32); - GGML_METAL_DEL_KERNEL(mul_mm_q5_K_f32); - GGML_METAL_DEL_KERNEL(mul_mm_q6_K_f32); - GGML_METAL_DEL_KERNEL(mul_mm_iq2_xxs_f32); - GGML_METAL_DEL_KERNEL(mul_mm_iq2_xs_f32); - GGML_METAL_DEL_KERNEL(mul_mm_id_f32_f32); - GGML_METAL_DEL_KERNEL(mul_mm_id_f16_f32); - GGML_METAL_DEL_KERNEL(mul_mm_id_q4_0_f32); - GGML_METAL_DEL_KERNEL(mul_mm_id_q4_1_f32); - GGML_METAL_DEL_KERNEL(mul_mm_id_q5_0_f32); - GGML_METAL_DEL_KERNEL(mul_mm_id_q5_1_f32); - GGML_METAL_DEL_KERNEL(mul_mm_id_q8_0_f32); - GGML_METAL_DEL_KERNEL(mul_mm_id_q2_K_f32); - GGML_METAL_DEL_KERNEL(mul_mm_id_q3_K_f32); - GGML_METAL_DEL_KERNEL(mul_mm_id_q4_K_f32); - GGML_METAL_DEL_KERNEL(mul_mm_id_q5_K_f32); - GGML_METAL_DEL_KERNEL(mul_mm_id_q6_K_f32); - GGML_METAL_DEL_KERNEL(mul_mm_id_iq2_xxs_f32); - GGML_METAL_DEL_KERNEL(mul_mm_id_iq2_xs_f32); - } - GGML_METAL_DEL_KERNEL(rope_f32); - GGML_METAL_DEL_KERNEL(rope_f16); - GGML_METAL_DEL_KERNEL(alibi_f32); - GGML_METAL_DEL_KERNEL(im2col_f16); - GGML_METAL_DEL_KERNEL(upscale_f32); - GGML_METAL_DEL_KERNEL(pad_f32); - GGML_METAL_DEL_KERNEL(argsort_f32_i32_asc); - GGML_METAL_DEL_KERNEL(argsort_f32_i32_desc); - GGML_METAL_DEL_KERNEL(leaky_relu_f32); - GGML_METAL_DEL_KERNEL(cpy_f32_f16); - GGML_METAL_DEL_KERNEL(cpy_f32_f32); - GGML_METAL_DEL_KERNEL(cpy_f32_q8_0); - GGML_METAL_DEL_KERNEL(cpy_f32_q4_0); - GGML_METAL_DEL_KERNEL(cpy_f32_q4_1); - //GGML_METAL_DEL_KERNEL(cpy_f32_q5_0); - //GGML_METAL_DEL_KERNEL(cpy_f32_q5_1); - GGML_METAL_DEL_KERNEL(cpy_f16_f16); - GGML_METAL_DEL_KERNEL(cpy_f16_f32); - GGML_METAL_DEL_KERNEL(concat); - GGML_METAL_DEL_KERNEL(sqr); - GGML_METAL_DEL_KERNEL(sum_rows); - -#undef GGML_METAL_DEL_KERNEL for (int i = 0; i < ctx->n_buffers; ++i) { [ctx->buffers[i].metal release]; } + for (int i = 0; i < GGML_METAL_MAX_KERNELS; ++i) { + if (ctx->kernels[i].pipeline) { + [ctx->kernels[i].pipeline release]; + } + + if (ctx->kernels[i].function) { + [ctx->kernels[i].function release]; + } + } + [ctx->library release]; [ctx->queue release]; [ctx->device release]; @@ -930,7 +862,7 @@ void ggml_metal_graph_find_concurrency( } } -static bool ggml_metal_supports_op(const struct ggml_tensor * op) { +static bool ggml_metal_supports_op(const struct ggml_metal_context * ctx, const struct ggml_tensor * op) { switch (op->op) { case GGML_OP_UNARY: switch (ggml_get_unary_op(op)) { @@ -956,9 +888,11 @@ static bool ggml_metal_supports_op(const struct ggml_tensor * op) { case GGML_OP_SCALE: case GGML_OP_SQR: case GGML_OP_SUM_ROWS: + return true; case GGML_OP_SOFT_MAX: case GGML_OP_RMS_NORM: case GGML_OP_GROUP_NORM: + return ctx->support_simdgroup_reduction; case GGML_OP_NORM: case GGML_OP_ALIBI: case GGML_OP_ROPE: @@ -967,9 +901,10 @@ static bool ggml_metal_supports_op(const struct ggml_tensor * op) { case GGML_OP_PAD: case GGML_OP_ARGSORT: case GGML_OP_LEAKY_RELU: + return true; case GGML_OP_MUL_MAT: case GGML_OP_MUL_MAT_ID: - return true; + return ctx->support_simdgroup_reduction; case GGML_OP_CPY: case GGML_OP_DUP: case GGML_OP_CONT: @@ -1007,6 +942,7 @@ static bool ggml_metal_supports_op(const struct ggml_tensor * op) { return false; } } + bool ggml_metal_graph_compute( struct ggml_metal_context * ctx, struct ggml_cgraph * gf) { @@ -1077,7 +1013,7 @@ bool ggml_metal_graph_compute( } break; } - if (!ggml_metal_supports_op(dst)) { + if (!ggml_metal_supports_op(ctx, dst)) { GGML_METAL_LOG_ERROR("%s: error: unsupported op '%s'\n", __func__, ggml_op_desc(dst)); GGML_ASSERT(!"unsupported op"); } @@ -1143,7 +1079,9 @@ bool ggml_metal_graph_compute( { const int64_t nb = ne00; - [encoder setComputePipelineState:ctx->pipeline_concat]; + id pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_CONCAT].pipeline; + + [encoder setComputePipelineState:pipeline]; [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0]; [encoder setBuffer:id_src1 offset:offs_src1 atIndex:1]; [encoder setBuffer:id_dst offset:offs_dst atIndex:2]; @@ -1197,18 +1135,18 @@ bool ggml_metal_graph_compute( nb = ne00 / 4; switch (dst->op) { - case GGML_OP_ADD: pipeline = ctx->pipeline_add_row; break; - case GGML_OP_MUL: pipeline = ctx->pipeline_mul_row; break; - case GGML_OP_DIV: pipeline = ctx->pipeline_div_row; break; + case GGML_OP_ADD: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_ADD_ROW].pipeline; break; + case GGML_OP_MUL: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_ROW].pipeline; break; + case GGML_OP_DIV: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_DIV_ROW].pipeline; break; default: GGML_ASSERT(false); } bcast_row = true; } else { switch (dst->op) { - case GGML_OP_ADD: pipeline = ctx->pipeline_add; break; - case GGML_OP_MUL: pipeline = ctx->pipeline_mul; break; - case GGML_OP_DIV: pipeline = ctx->pipeline_div; break; + case GGML_OP_ADD: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_ADD].pipeline; break; + case GGML_OP_MUL: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL].pipeline; break; + case GGML_OP_DIV: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_DIV].pipeline; break; default: GGML_ASSERT(false); } } @@ -1275,9 +1213,9 @@ bool ggml_metal_graph_compute( // not sure how to avoid this // TODO: make a simpler cpy_bytes kernel - const int nth = MIN((int) ctx->pipeline_cpy_f32_f32.maxTotalThreadsPerThreadgroup, ne00); + const id pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_CPY_F32_F32].pipeline; - [encoder setComputePipelineState:ctx->pipeline_cpy_f32_f32]; + [encoder setComputePipelineState:pipeline]; [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0]; [encoder setBuffer:id_dst offset:offs_dst atIndex:1]; [encoder setBytes:&ne00 length:sizeof( int64_t) atIndex:2]; @@ -1297,10 +1235,14 @@ bool ggml_metal_graph_compute( [encoder setBytes:&nb2 length:sizeof(uint64_t) atIndex:16]; [encoder setBytes:&nb3 length:sizeof(uint64_t) atIndex:17]; + const int nth = MIN((int) pipeline.maxTotalThreadsPerThreadgroup, ne00); + [encoder dispatchThreadgroups:MTLSizeMake(ne01, ne02, ne03) threadsPerThreadgroup:MTLSizeMake(nth, 1, 1)]; } - [encoder setComputePipelineState:ctx->pipeline_add]; + const id pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_ADD].pipeline; + + [encoder setComputePipelineState:pipeline]; [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0]; [encoder setBuffer:id_src1 offset:offs_src1 atIndex:1]; [encoder setBuffer:id_dst offset:offs_dst atIndex:2]; @@ -1330,7 +1272,7 @@ bool ggml_metal_graph_compute( [encoder setBytes:&pnb3 length:sizeof(pnb3) atIndex:26]; [encoder setBytes:&offs length:sizeof(offs) atIndex:27]; - const int nth = MIN((int) ctx->pipeline_add.maxTotalThreadsPerThreadgroup, ne00); + const int nth = MIN((int) pipeline.maxTotalThreadsPerThreadgroup, ne00); [encoder dispatchThreadgroups:MTLSizeMake(ne11, ne12, ne13) threadsPerThreadgroup:MTLSizeMake(nth, 1, 1)]; } break; @@ -1342,13 +1284,16 @@ bool ggml_metal_graph_compute( int64_t n = ggml_nelements(dst); + id pipeline = nil; + if (n % 4 == 0) { n /= 4; - [encoder setComputePipelineState:ctx->pipeline_scale_4]; + pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_SCALE_4].pipeline; } else { - [encoder setComputePipelineState:ctx->pipeline_scale]; + pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_SCALE].pipeline; } + [encoder setComputePipelineState:pipeline]; [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0]; [encoder setBuffer:id_dst offset:offs_dst atIndex:1]; [encoder setBytes:&scale length:sizeof(scale) atIndex:2]; @@ -1359,7 +1304,9 @@ bool ggml_metal_graph_compute( switch (ggml_get_unary_op(gf->nodes[i])) { case GGML_UNARY_OP_TANH: { - [encoder setComputePipelineState:ctx->pipeline_tanh]; + id pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_TANH].pipeline; + + [encoder setComputePipelineState:pipeline]; [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0]; [encoder setBuffer:id_dst offset:offs_dst atIndex:1]; @@ -1369,7 +1316,9 @@ bool ggml_metal_graph_compute( } break; case GGML_UNARY_OP_RELU: { - [encoder setComputePipelineState:ctx->pipeline_relu]; + id pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_RELU].pipeline; + + [encoder setComputePipelineState:pipeline]; [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0]; [encoder setBuffer:id_dst offset:offs_dst atIndex:1]; @@ -1379,7 +1328,9 @@ bool ggml_metal_graph_compute( } break; case GGML_UNARY_OP_GELU: { - [encoder setComputePipelineState:ctx->pipeline_gelu]; + id pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GELU].pipeline; + + [encoder setComputePipelineState:pipeline]; [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0]; [encoder setBuffer:id_dst offset:offs_dst atIndex:1]; @@ -1390,7 +1341,9 @@ bool ggml_metal_graph_compute( } break; case GGML_UNARY_OP_GELU_QUICK: { - [encoder setComputePipelineState:ctx->pipeline_gelu_quick]; + id pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GELU_QUICK].pipeline; + + [encoder setComputePipelineState:pipeline]; [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0]; [encoder setBuffer:id_dst offset:offs_dst atIndex:1]; @@ -1401,7 +1354,9 @@ bool ggml_metal_graph_compute( } break; case GGML_UNARY_OP_SILU: { - [encoder setComputePipelineState:ctx->pipeline_silu]; + id pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_SILU].pipeline; + + [encoder setComputePipelineState:pipeline]; [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0]; [encoder setBuffer:id_dst offset:offs_dst atIndex:1]; @@ -1420,18 +1375,23 @@ bool ggml_metal_graph_compute( { GGML_ASSERT(ggml_is_contiguous(src0)); - [encoder setComputePipelineState:ctx->pipeline_sqr]; + id pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_SQR].pipeline; + + [encoder setComputePipelineState:pipeline]; [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0]; [encoder setBuffer:id_dst offset:offs_dst atIndex:1]; const int64_t n = ggml_nelements(dst); + [encoder dispatchThreadgroups:MTLSizeMake(n, 1, 1) threadsPerThreadgroup:MTLSizeMake(1, 1, 1)]; } break; case GGML_OP_SUM_ROWS: { GGML_ASSERT(src0->nb[0] == ggml_type_size(src0->type)); - [encoder setComputePipelineState:ctx->pipeline_sum_rows]; + id pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_SUM_ROWS].pipeline; + + [encoder setComputePipelineState:pipeline]; [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0]; [encoder setBuffer:id_dst offset:offs_dst atIndex:1]; [encoder setBytes:&ne00 length:sizeof(ne00) atIndex:2]; @@ -1465,20 +1425,23 @@ bool ggml_metal_graph_compute( { int nth = 32; // SIMD width + id pipeline = nil; + if (ne00%4 == 0) { while (nth < ne00/4 && nth < 256) { nth *= 2; } - [encoder setComputePipelineState:ctx->pipeline_soft_max_4]; + pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_SOFT_MAX_4].pipeline; } else { while (nth < ne00 && nth < 1024) { nth *= 2; } - [encoder setComputePipelineState:ctx->pipeline_soft_max]; + pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_SOFT_MAX].pipeline; } const float scale = ((float *) dst->op_params)[0]; + [encoder setComputePipelineState:pipeline]; [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0]; if (id_src1) { [encoder setBuffer:id_src1 offset:offs_src1 atIndex:1]; @@ -1498,11 +1461,15 @@ bool ggml_metal_graph_compute( { const int n_past = ((int32_t *)(dst->op_params))[0]; + id pipeline = nil; + if (ne00%8 == 0) { - [encoder setComputePipelineState:ctx->pipeline_diag_mask_inf_8]; + pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_DIAG_MASK_INF_8].pipeline; } else { - [encoder setComputePipelineState:ctx->pipeline_diag_mask_inf]; + pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_DIAG_MASK_INF].pipeline; } + + [encoder setComputePipelineState:pipeline]; [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0]; [encoder setBuffer:id_dst offset:offs_dst atIndex:1]; [encoder setBytes:&ne00 length:sizeof(ne00) atIndex:2]; @@ -1562,23 +1529,28 @@ bool ggml_metal_graph_compute( ne00 % 32 == 0 && ne00 >= 64 && (ne11 > ne11_mm_min || (ggml_is_quantized(src0t) && ne12 > 1))) { //printf("matrix: ne00 = %6d, ne01 = %6d, ne02 = %6d, ne11 = %6d, ne12 = %6d\n", ne00, ne01, ne02, ne11, ne12); + + id pipeline = nil; + switch (src0->type) { - case GGML_TYPE_F32: [encoder setComputePipelineState:ctx->pipeline_mul_mm_f32_f32]; break; - case GGML_TYPE_F16: [encoder setComputePipelineState:ctx->pipeline_mul_mm_f16_f32]; break; - case GGML_TYPE_Q4_0: [encoder setComputePipelineState:ctx->pipeline_mul_mm_q4_0_f32]; break; - case GGML_TYPE_Q4_1: [encoder setComputePipelineState:ctx->pipeline_mul_mm_q4_1_f32]; break; - case GGML_TYPE_Q5_0: [encoder setComputePipelineState:ctx->pipeline_mul_mm_q5_0_f32]; break; - case GGML_TYPE_Q5_1: [encoder setComputePipelineState:ctx->pipeline_mul_mm_q5_1_f32]; break; - case GGML_TYPE_Q8_0: [encoder setComputePipelineState:ctx->pipeline_mul_mm_q8_0_f32]; break; - case GGML_TYPE_Q2_K: [encoder setComputePipelineState:ctx->pipeline_mul_mm_q2_K_f32]; break; - case GGML_TYPE_Q3_K: [encoder setComputePipelineState:ctx->pipeline_mul_mm_q3_K_f32]; break; - case GGML_TYPE_Q4_K: [encoder setComputePipelineState:ctx->pipeline_mul_mm_q4_K_f32]; break; - case GGML_TYPE_Q5_K: [encoder setComputePipelineState:ctx->pipeline_mul_mm_q5_K_f32]; break; - case GGML_TYPE_Q6_K: [encoder setComputePipelineState:ctx->pipeline_mul_mm_q6_K_f32]; break; - case GGML_TYPE_IQ2_XXS: [encoder setComputePipelineState:ctx->pipeline_mul_mm_iq2_xxs_f32]; break; - case GGML_TYPE_IQ2_XS : [encoder setComputePipelineState:ctx->pipeline_mul_mm_iq2_xs_f32]; break; + case GGML_TYPE_F32: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_F32_F32 ].pipeline; break; + case GGML_TYPE_F16: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_F16_F32 ].pipeline; break; + case GGML_TYPE_Q4_0: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_Q4_0_F32 ].pipeline; break; + case GGML_TYPE_Q4_1: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_Q4_1_F32 ].pipeline; break; + case GGML_TYPE_Q5_0: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_Q5_0_F32 ].pipeline; break; + case GGML_TYPE_Q5_1: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_Q5_1_F32 ].pipeline; break; + case GGML_TYPE_Q8_0: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_Q8_0_F32 ].pipeline; break; + case GGML_TYPE_Q2_K: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_Q2_K_F32 ].pipeline; break; + case GGML_TYPE_Q3_K: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_Q3_K_F32 ].pipeline; break; + case GGML_TYPE_Q4_K: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_Q4_K_F32 ].pipeline; break; + case GGML_TYPE_Q5_K: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_Q5_K_F32 ].pipeline; break; + case GGML_TYPE_Q6_K: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_Q6_K_F32 ].pipeline; break; + case GGML_TYPE_IQ2_XXS: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_IQ2_XXS_F32].pipeline; break; + case GGML_TYPE_IQ2_XS: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_IQ2_XS_F32 ].pipeline; break; default: GGML_ASSERT(false && "MUL MAT-MAT not implemented"); } + + [encoder setComputePipelineState:pipeline]; [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0]; [encoder setBuffer:id_src1 offset:offs_src1 atIndex:1]; [encoder setBuffer:id_dst offset:offs_dst atIndex:2]; @@ -1602,12 +1574,14 @@ bool ggml_metal_graph_compute( int nrows = 1; //printf("vector: ne00 = %6d, ne01 = %6d, ne02 = %6d, ne11 = %6d, ne12 = %6d\n", ne00, ne01, ne02, ne11, ne12); + id pipeline = nil; + // use custom matrix x vector kernel switch (src0t) { case GGML_TYPE_F32: { GGML_ASSERT(src1t == GGML_TYPE_F32); - [encoder setComputePipelineState:ctx->pipeline_mul_mv_f32_f32]; + pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_F32_F32].pipeline; nrows = 4; } break; case GGML_TYPE_F16: @@ -1616,16 +1590,16 @@ bool ggml_metal_graph_compute( nth1 = 1; if (src1t == GGML_TYPE_F32) { if (ne11 * ne12 < 4) { - [encoder setComputePipelineState:ctx->pipeline_mul_mv_f16_f32_1row]; + pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_F16_F32_1ROW].pipeline; } else if (ne00 >= 128 && ne01 >= 8 && ne00%4 == 0) { - [encoder setComputePipelineState:ctx->pipeline_mul_mv_f16_f32_l4]; + pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_F16_F32_L4].pipeline; nrows = ne11; } else { - [encoder setComputePipelineState:ctx->pipeline_mul_mv_f16_f32]; + pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_F16_F32].pipeline; nrows = 4; } } else { - [encoder setComputePipelineState:ctx->pipeline_mul_mv_f16_f16]; + pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_F16_F16].pipeline; nrows = 4; } } break; @@ -1633,73 +1607,73 @@ bool ggml_metal_graph_compute( { nth0 = 8; nth1 = 8; - [encoder setComputePipelineState:ctx->pipeline_mul_mv_q4_0_f32]; + pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_Q4_0_F32].pipeline; } break; case GGML_TYPE_Q4_1: { nth0 = 8; nth1 = 8; - [encoder setComputePipelineState:ctx->pipeline_mul_mv_q4_1_f32]; + pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_Q4_1_F32].pipeline; } break; case GGML_TYPE_Q5_0: { nth0 = 8; nth1 = 8; - [encoder setComputePipelineState:ctx->pipeline_mul_mv_q5_0_f32]; + pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_Q5_0_F32].pipeline; } break; case GGML_TYPE_Q5_1: { nth0 = 8; nth1 = 8; - [encoder setComputePipelineState:ctx->pipeline_mul_mv_q5_1_f32]; + pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_Q5_1_F32].pipeline; } break; case GGML_TYPE_Q8_0: { nth0 = 8; nth1 = 8; - [encoder setComputePipelineState:ctx->pipeline_mul_mv_q8_0_f32]; + pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_Q8_0_F32].pipeline; } break; case GGML_TYPE_Q2_K: { nth0 = 2; nth1 = 32; - [encoder setComputePipelineState:ctx->pipeline_mul_mv_q2_K_f32]; + pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_Q2_K_F32].pipeline; } break; case GGML_TYPE_Q3_K: { nth0 = 2; nth1 = 32; - [encoder setComputePipelineState:ctx->pipeline_mul_mv_q3_K_f32]; + pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_Q3_K_F32].pipeline; } break; case GGML_TYPE_Q4_K: { nth0 = 4; //1; nth1 = 8; //32; - [encoder setComputePipelineState:ctx->pipeline_mul_mv_q4_K_f32]; + pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_Q4_K_F32].pipeline; } break; case GGML_TYPE_Q5_K: { nth0 = 2; nth1 = 32; - [encoder setComputePipelineState:ctx->pipeline_mul_mv_q5_K_f32]; + pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_Q5_K_F32].pipeline; } break; case GGML_TYPE_Q6_K: { nth0 = 2; nth1 = 32; - [encoder setComputePipelineState:ctx->pipeline_mul_mv_q6_K_f32]; + pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_Q6_K_F32].pipeline; } break; case GGML_TYPE_IQ2_XXS: { nth0 = 4; nth1 = 16; - [encoder setComputePipelineState:ctx->pipeline_mul_mv_iq2_xxs_f32]; + pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_IQ2_XXS_F32].pipeline; } break; case GGML_TYPE_IQ2_XS: { nth0 = 4; nth1 = 16; - [encoder setComputePipelineState:ctx->pipeline_mul_mv_iq2_xs_f32]; + pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_IQ2_XS_F32].pipeline; } break; default: { @@ -1712,6 +1686,7 @@ bool ggml_metal_graph_compute( GGML_ASSERT(ne00 >= nth0*nth1); } + [encoder setComputePipelineState:pipeline]; [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0]; [encoder setBuffer:id_src1 offset:offs_src1 atIndex:1]; [encoder setBuffer:id_dst offset:offs_dst atIndex:2]; @@ -1818,23 +1793,28 @@ bool ggml_metal_graph_compute( if ([ctx->device supportsFamily:MTLGPUFamilyApple7] && ne20 % 32 == 0 && ne20 >= 64 && ne11 > ne11_mm_min) { + + id pipeline = nil; + switch (src2->type) { - case GGML_TYPE_F32: [encoder setComputePipelineState:ctx->pipeline_mul_mm_id_f32_f32]; break; - case GGML_TYPE_F16: [encoder setComputePipelineState:ctx->pipeline_mul_mm_id_f16_f32]; break; - case GGML_TYPE_Q4_0: [encoder setComputePipelineState:ctx->pipeline_mul_mm_id_q4_0_f32]; break; - case GGML_TYPE_Q4_1: [encoder setComputePipelineState:ctx->pipeline_mul_mm_id_q4_1_f32]; break; - case GGML_TYPE_Q5_0: [encoder setComputePipelineState:ctx->pipeline_mul_mm_id_q5_0_f32]; break; - case GGML_TYPE_Q5_1: [encoder setComputePipelineState:ctx->pipeline_mul_mm_id_q5_1_f32]; break; - case GGML_TYPE_Q8_0: [encoder setComputePipelineState:ctx->pipeline_mul_mm_id_q8_0_f32]; break; - case GGML_TYPE_Q2_K: [encoder setComputePipelineState:ctx->pipeline_mul_mm_id_q2_K_f32]; break; - case GGML_TYPE_Q3_K: [encoder setComputePipelineState:ctx->pipeline_mul_mm_id_q3_K_f32]; break; - case GGML_TYPE_Q4_K: [encoder setComputePipelineState:ctx->pipeline_mul_mm_id_q4_K_f32]; break; - case GGML_TYPE_Q5_K: [encoder setComputePipelineState:ctx->pipeline_mul_mm_id_q5_K_f32]; break; - case GGML_TYPE_Q6_K: [encoder setComputePipelineState:ctx->pipeline_mul_mm_id_q6_K_f32]; break; - case GGML_TYPE_IQ2_XXS: [encoder setComputePipelineState:ctx->pipeline_mul_mm_id_iq2_xxs_f32]; break; - case GGML_TYPE_IQ2_XS : [encoder setComputePipelineState:ctx->pipeline_mul_mm_id_iq2_xs_f32]; break; + case GGML_TYPE_F32: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_ID_F32_F32 ].pipeline; break; + case GGML_TYPE_F16: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_ID_F16_F32 ].pipeline; break; + case GGML_TYPE_Q4_0: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q4_0_F32 ].pipeline; break; + case GGML_TYPE_Q4_1: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q4_1_F32 ].pipeline; break; + case GGML_TYPE_Q5_0: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q5_0_F32 ].pipeline; break; + case GGML_TYPE_Q5_1: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q5_1_F32 ].pipeline; break; + case GGML_TYPE_Q8_0: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q8_0_F32 ].pipeline; break; + case GGML_TYPE_Q2_K: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q2_K_F32 ].pipeline; break; + case GGML_TYPE_Q3_K: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q3_K_F32 ].pipeline; break; + case GGML_TYPE_Q4_K: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q4_K_F32 ].pipeline; break; + case GGML_TYPE_Q5_K: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q5_K_F32 ].pipeline; break; + case GGML_TYPE_Q6_K: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q6_K_F32 ].pipeline; break; + case GGML_TYPE_IQ2_XXS: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ2_XXS_F32].pipeline; break; + case GGML_TYPE_IQ2_XS: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ2_XS_F32 ].pipeline; break; default: GGML_ASSERT(false && "MUL_MAT_ID not implemented"); } + + [encoder setComputePipelineState:pipeline]; [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0]; [encoder setBuffer:id_src1 offset:offs_src1 atIndex:1]; [encoder setBuffer:id_dst offset:offs_dst atIndex:2]; @@ -1874,91 +1854,93 @@ bool ggml_metal_graph_compute( int nrows = 1; //printf("vector: ne00 = %6d, ne01 = %6d, ne02 = %6d, ne11 = %6d, ne12 = %6d\n", ne00, ne01, ne02, ne11, ne12); + id pipeline = nil; + // use custom matrix x vector kernel switch (src2t) { case GGML_TYPE_F32: { GGML_ASSERT(src1t == GGML_TYPE_F32); - [encoder setComputePipelineState:ctx->pipeline_mul_mv_id_f32_f32]; + pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_ID_F32_F32].pipeline; } break; case GGML_TYPE_F16: { GGML_ASSERT(src1t == GGML_TYPE_F32); nth0 = 32; nth1 = 1; - [encoder setComputePipelineState:ctx->pipeline_mul_mv_id_f16_f32]; + pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_ID_F16_F32].pipeline; } break; case GGML_TYPE_Q4_0: { nth0 = 8; nth1 = 8; - [encoder setComputePipelineState:ctx->pipeline_mul_mv_id_q4_0_f32]; + pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q4_0_F32].pipeline; } break; case GGML_TYPE_Q4_1: { nth0 = 8; nth1 = 8; - [encoder setComputePipelineState:ctx->pipeline_mul_mv_id_q4_1_f32]; + pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q4_1_F32].pipeline; } break; case GGML_TYPE_Q5_0: { nth0 = 8; nth1 = 8; - [encoder setComputePipelineState:ctx->pipeline_mul_mv_id_q5_0_f32]; + pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q5_0_F32].pipeline; } break; case GGML_TYPE_Q5_1: { nth0 = 8; nth1 = 8; - [encoder setComputePipelineState:ctx->pipeline_mul_mv_id_q5_1_f32]; + pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q5_1_F32].pipeline; } break; case GGML_TYPE_Q8_0: { nth0 = 8; nth1 = 8; - [encoder setComputePipelineState:ctx->pipeline_mul_mv_id_q8_0_f32]; + pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q8_0_F32].pipeline; } break; case GGML_TYPE_Q2_K: { nth0 = 2; nth1 = 32; - [encoder setComputePipelineState:ctx->pipeline_mul_mv_id_q2_K_f32]; + pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q2_K_F32].pipeline; } break; case GGML_TYPE_Q3_K: { nth0 = 2; nth1 = 32; - [encoder setComputePipelineState:ctx->pipeline_mul_mv_id_q3_K_f32]; + pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q3_K_F32].pipeline; } break; case GGML_TYPE_Q4_K: { nth0 = 4; //1; nth1 = 8; //32; - [encoder setComputePipelineState:ctx->pipeline_mul_mv_id_q4_K_f32]; + pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q4_K_F32].pipeline; } break; case GGML_TYPE_Q5_K: { nth0 = 2; nth1 = 32; - [encoder setComputePipelineState:ctx->pipeline_mul_mv_id_q5_K_f32]; + pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q5_K_F32].pipeline; } break; case GGML_TYPE_Q6_K: { nth0 = 2; nth1 = 32; - [encoder setComputePipelineState:ctx->pipeline_mul_mv_id_q6_K_f32]; + pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q6_K_F32].pipeline; } break; case GGML_TYPE_IQ2_XXS: { nth0 = 4; nth1 = 16; - [encoder setComputePipelineState:ctx->pipeline_mul_mv_id_iq2_xxs_f32]; + pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ2_XXS_F32].pipeline; } break; case GGML_TYPE_IQ2_XS: { nth0 = 4; nth1 = 16; - [encoder setComputePipelineState:ctx->pipeline_mul_mv_id_iq2_xs_f32]; + pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ2_XS_F32].pipeline; } break; default: { @@ -1973,6 +1955,7 @@ bool ggml_metal_graph_compute( const int64_t _ne1 = 1; // kernels needs a reference in constant memory + [encoder setComputePipelineState:pipeline]; [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0]; [encoder setBuffer:id_src1 offset:offs_src1 atIndex:1]; [encoder setBuffer:id_dst offset:offs_dst atIndex:2]; @@ -2040,25 +2023,28 @@ bool ggml_metal_graph_compute( } break; case GGML_OP_GET_ROWS: { + id pipeline = nil; + switch (src0->type) { - case GGML_TYPE_F32: [encoder setComputePipelineState:ctx->pipeline_get_rows_f32]; break; - case GGML_TYPE_F16: [encoder setComputePipelineState:ctx->pipeline_get_rows_f16]; break; - case GGML_TYPE_Q4_0: [encoder setComputePipelineState:ctx->pipeline_get_rows_q4_0]; break; - case GGML_TYPE_Q4_1: [encoder setComputePipelineState:ctx->pipeline_get_rows_q4_1]; break; - case GGML_TYPE_Q5_0: [encoder setComputePipelineState:ctx->pipeline_get_rows_q5_0]; break; - case GGML_TYPE_Q5_1: [encoder setComputePipelineState:ctx->pipeline_get_rows_q5_1]; break; - case GGML_TYPE_Q8_0: [encoder setComputePipelineState:ctx->pipeline_get_rows_q8_0]; break; - case GGML_TYPE_Q2_K: [encoder setComputePipelineState:ctx->pipeline_get_rows_q2_K]; break; - case GGML_TYPE_Q3_K: [encoder setComputePipelineState:ctx->pipeline_get_rows_q3_K]; break; - case GGML_TYPE_Q4_K: [encoder setComputePipelineState:ctx->pipeline_get_rows_q4_K]; break; - case GGML_TYPE_Q5_K: [encoder setComputePipelineState:ctx->pipeline_get_rows_q5_K]; break; - case GGML_TYPE_Q6_K: [encoder setComputePipelineState:ctx->pipeline_get_rows_q6_K]; break; - case GGML_TYPE_I32: [encoder setComputePipelineState:ctx->pipeline_get_rows_i32]; break; - case GGML_TYPE_IQ2_XXS: [encoder setComputePipelineState:ctx->pipeline_get_rows_iq2_xxs]; break; - case GGML_TYPE_IQ2_XS : [encoder setComputePipelineState:ctx->pipeline_get_rows_iq2_xs]; break; + case GGML_TYPE_F32: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GET_ROWS_F32 ].pipeline; break; + case GGML_TYPE_F16: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GET_ROWS_F16 ].pipeline; break; + case GGML_TYPE_Q4_0: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GET_ROWS_Q4_0 ].pipeline; break; + case GGML_TYPE_Q4_1: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GET_ROWS_Q4_1 ].pipeline; break; + case GGML_TYPE_Q5_0: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GET_ROWS_Q5_0 ].pipeline; break; + case GGML_TYPE_Q5_1: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GET_ROWS_Q5_1 ].pipeline; break; + case GGML_TYPE_Q8_0: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GET_ROWS_Q8_0 ].pipeline; break; + case GGML_TYPE_Q2_K: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GET_ROWS_Q2_K ].pipeline; break; + case GGML_TYPE_Q3_K: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GET_ROWS_Q3_K ].pipeline; break; + case GGML_TYPE_Q4_K: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GET_ROWS_Q4_K ].pipeline; break; + case GGML_TYPE_Q5_K: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GET_ROWS_Q5_K ].pipeline; break; + case GGML_TYPE_Q6_K: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GET_ROWS_Q6_K ].pipeline; break; + case GGML_TYPE_IQ2_XXS: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ2_XXS].pipeline; break; + case GGML_TYPE_IQ2_XS: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ2_XS ].pipeline; break; + case GGML_TYPE_I32: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GET_ROWS_I32 ].pipeline; break; default: GGML_ASSERT(false && "not implemented"); } + [encoder setComputePipelineState:pipeline]; [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0]; [encoder setBuffer:id_src1 offset:offs_src1 atIndex:1]; [encoder setBuffer:id_dst offset:offs_dst atIndex:2]; @@ -2086,7 +2072,9 @@ bool ggml_metal_graph_compute( nth *= 2; } - [encoder setComputePipelineState:ctx->pipeline_rms_norm]; + id pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_RMS_NORM].pipeline; + + [encoder setComputePipelineState:pipeline]; [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0]; [encoder setBuffer:id_dst offset:offs_dst atIndex:1]; [encoder setBytes:&ne00 length:sizeof( int64_t) atIndex:2]; @@ -2115,7 +2103,9 @@ bool ggml_metal_graph_compute( // nth *= 2; //} - [encoder setComputePipelineState:ctx->pipeline_group_norm]; + id pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GROUP_NORM].pipeline; + + [encoder setComputePipelineState:pipeline]; [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0]; [encoder setBuffer:id_dst offset:offs_dst atIndex:1]; [encoder setBytes:&ne00 length:sizeof( int64_t) atIndex:2]; @@ -2137,7 +2127,9 @@ bool ggml_metal_graph_compute( const int nth = MIN(256, ne00); - [encoder setComputePipelineState:ctx->pipeline_norm]; + id pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_NORM].pipeline; + + [encoder setComputePipelineState:pipeline]; [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0]; [encoder setBuffer:id_dst offset:offs_dst atIndex:1]; [encoder setBytes:&ne00 length:sizeof( int64_t) atIndex:2]; @@ -2164,7 +2156,9 @@ bool ggml_metal_graph_compute( const float m0 = powf(2.0f, -(max_bias) / n_heads_log2_floor); const float m1 = powf(2.0f, -(max_bias / 2.0f) / n_heads_log2_floor); - [encoder setComputePipelineState:ctx->pipeline_alibi_f32]; + id pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_ALIBI_F32].pipeline; + + [encoder setComputePipelineState:pipeline]; [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0]; [encoder setBuffer:id_dst offset:offs_dst atIndex:1]; [encoder setBytes:&ne00 length:sizeof( int64_t) atIndex:2]; @@ -2209,12 +2203,15 @@ bool ggml_metal_graph_compute( memcpy(&beta_fast, (int32_t *) dst->op_params + 9, sizeof(float)); memcpy(&beta_slow, (int32_t *) dst->op_params + 10, sizeof(float)); + id pipeline = nil; + switch (src0->type) { - case GGML_TYPE_F32: [encoder setComputePipelineState:ctx->pipeline_rope_f32]; break; - case GGML_TYPE_F16: [encoder setComputePipelineState:ctx->pipeline_rope_f16]; break; + case GGML_TYPE_F32: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_ROPE_F32].pipeline; break; + case GGML_TYPE_F16: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_ROPE_F16].pipeline; break; default: GGML_ASSERT(false); }; + [encoder setComputePipelineState:pipeline]; [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0]; [encoder setBuffer:id_src1 offset:offs_src1 atIndex:1]; [encoder setBuffer:id_dst offset:offs_dst atIndex:2]; @@ -2277,12 +2274,15 @@ bool ggml_metal_graph_compute( const int32_t ofs0 = src1->nb[is_2D ? 3 : 2] / 4; const int32_t ofs1 = src1->nb[is_2D ? 2 : 1] / 4; + id pipeline = nil; + switch (src0->type) { case GGML_TYPE_F32: GGML_ASSERT(false && "not implemented"); break; - case GGML_TYPE_F16: [encoder setComputePipelineState:ctx->pipeline_im2col_f16]; break; + case GGML_TYPE_F16: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_IM2COL_F16].pipeline; break; default: GGML_ASSERT(false); }; + [encoder setComputePipelineState:pipeline]; [encoder setBuffer:id_src1 offset:offs_src1 atIndex:0]; [encoder setBuffer:id_dst offset:offs_dst atIndex:1]; [encoder setBytes:&ofs0 length:sizeof( int32_t) atIndex:2]; @@ -2305,7 +2305,9 @@ bool ggml_metal_graph_compute( const int sf = dst->op_params[0]; - [encoder setComputePipelineState:ctx->pipeline_upscale_f32]; + const id pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_UPSCALE_F32].pipeline; + + [encoder setComputePipelineState:pipeline]; [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0]; [encoder setBuffer:id_dst offset:offs_dst atIndex:1]; [encoder setBytes:&ne00 length:sizeof(ne00) atIndex:2]; @@ -2326,7 +2328,7 @@ bool ggml_metal_graph_compute( [encoder setBytes:&nb3 length:sizeof(nb3) atIndex:17]; [encoder setBytes:&sf length:sizeof(sf) atIndex:18]; - const int nth = MIN((int) ctx->pipeline_upscale_f32.maxTotalThreadsPerThreadgroup, ne0); + const int nth = MIN((int) pipeline.maxTotalThreadsPerThreadgroup, ne0); [encoder dispatchThreadgroups:MTLSizeMake(ne1, ne2, ne3) threadsPerThreadgroup:MTLSizeMake(nth, 1, 1)]; } break; @@ -2334,7 +2336,9 @@ bool ggml_metal_graph_compute( { GGML_ASSERT(src0->type == GGML_TYPE_F32); - [encoder setComputePipelineState:ctx->pipeline_pad_f32]; + id pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_PAD_F32].pipeline; + + [encoder setComputePipelineState:pipeline]; [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0]; [encoder setBuffer:id_dst offset:offs_dst atIndex:1]; [encoder setBytes:&ne00 length:sizeof(ne00) atIndex:2]; @@ -2367,12 +2371,15 @@ bool ggml_metal_graph_compute( enum ggml_sort_order order = (enum ggml_sort_order) dst->op_params[0]; + id pipeline = nil; + switch (order) { - case GGML_SORT_ASC: [encoder setComputePipelineState:ctx->pipeline_argsort_f32_i32_asc]; break; - case GGML_SORT_DESC: [encoder setComputePipelineState:ctx->pipeline_argsort_f32_i32_desc]; break; + case GGML_SORT_ASC: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_ARGSORT_F32_I32_ASC].pipeline; break; + case GGML_SORT_DESC: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_ARGSORT_F32_I32_DESC].pipeline; break; default: GGML_ASSERT(false); }; + [encoder setComputePipelineState:pipeline]; [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0]; [encoder setBuffer:id_dst offset:offs_dst atIndex:1]; [encoder setBytes:&ne00 length:sizeof( int64_t) atIndex:2]; @@ -2386,7 +2393,9 @@ bool ggml_metal_graph_compute( float slope; memcpy(&slope, dst->op_params, sizeof(float)); - [encoder setComputePipelineState:ctx->pipeline_leaky_relu_f32]; + id pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_LEAKY_RELU_F32].pipeline; + + [encoder setComputePipelineState:pipeline]; [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0]; [encoder setBuffer:id_dst offset:offs_dst atIndex:1]; [encoder setBytes:&slope length:sizeof(slope) atIndex:2]; @@ -2403,33 +2412,36 @@ bool ggml_metal_graph_compute( int nth = MIN(1024, ne00/ggml_blck_size(src0->type)); + id pipeline = nil; + switch (src0t) { case GGML_TYPE_F32: { GGML_ASSERT(ne0 % ggml_blck_size(dst->type) == 0); switch (dstt) { - case GGML_TYPE_F16: [encoder setComputePipelineState:ctx->pipeline_cpy_f32_f16]; break; - case GGML_TYPE_F32: [encoder setComputePipelineState:ctx->pipeline_cpy_f32_f32]; break; - case GGML_TYPE_Q8_0: [encoder setComputePipelineState:ctx->pipeline_cpy_f32_q8_0]; break; - case GGML_TYPE_Q4_0: [encoder setComputePipelineState:ctx->pipeline_cpy_f32_q4_0]; break; - case GGML_TYPE_Q4_1: [encoder setComputePipelineState:ctx->pipeline_cpy_f32_q4_1]; break; - //case GGML_TYPE_Q5_0: [encoder setComputePipelineState:ctx->pipeline_cpy_f32_q5_0]; break; - //case GGML_TYPE_Q5_1: [encoder setComputePipelineState:ctx->pipeline_cpy_f32_q5_1]; break; + case GGML_TYPE_F16: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_CPY_F32_F16].pipeline; break; + case GGML_TYPE_F32: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_CPY_F32_F32].pipeline; break; + case GGML_TYPE_Q8_0: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_CPY_F32_Q8_0].pipeline; break; + case GGML_TYPE_Q4_0: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_CPY_F32_Q4_0].pipeline; break; + case GGML_TYPE_Q4_1: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_CPY_F32_Q4_1].pipeline; break; + //case GGML_TYPE_Q5_0: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_CPY_F32_Q5_0].pipeline; break; + //case GGML_TYPE_Q5_1: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_CPY_F32_Q5_1].pipeline; break; default: GGML_ASSERT(false && "not implemented"); }; } break; case GGML_TYPE_F16: { switch (dstt) { - case GGML_TYPE_F16: [encoder setComputePipelineState:ctx->pipeline_cpy_f16_f16]; break; - case GGML_TYPE_F32: [encoder setComputePipelineState:ctx->pipeline_cpy_f16_f32]; break; + case GGML_TYPE_F16: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_CPY_F16_F16].pipeline; break; + case GGML_TYPE_F32: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_CPY_F16_F32].pipeline; break; default: GGML_ASSERT(false && "not implemented"); }; } break; default: GGML_ASSERT(false && "not implemented"); } + [encoder setComputePipelineState:pipeline]; [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0]; [encoder setBuffer:id_dst offset:offs_dst atIndex:1]; [encoder setBytes:&ne00 length:sizeof( int64_t) atIndex:2]; @@ -2794,9 +2806,9 @@ static bool ggml_backend_metal_graph_compute(ggml_backend_t backend, struct ggml } static bool ggml_backend_metal_supports_op(ggml_backend_t backend, const struct ggml_tensor * op) { - return ggml_metal_supports_op(op); + struct ggml_metal_context * metal_ctx = (struct ggml_metal_context *)backend->context; - UNUSED(backend); + return ggml_metal_supports_op(metal_ctx, op); } static struct ggml_backend_i ggml_backend_metal_i = { From c30b1ef39aeba497a943416d2897d69fee055b96 Mon Sep 17 00:00:00 2001 From: texmex76 <40733439+texmex76@users.noreply.github.com> Date: Sat, 13 Jan 2024 17:06:20 +0100 Subject: [PATCH 050/812] gguf : fix potential infinite for-loop (#4600) Co-authored-by: Bernhard Gstrein --- ggml.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ggml.c b/ggml.c index 6dbd7626c..de6ef34bd 100644 --- a/ggml.c +++ b/ggml.c @@ -19184,7 +19184,7 @@ void gguf_free(struct gguf_context * ctx) { if (ctx->kv) { // free string memory - not great.. - for (uint32_t i = 0; i < ctx->header.n_kv; ++i) { + for (uint64_t i = 0; i < ctx->header.n_kv; ++i) { struct gguf_kv * kv = &ctx->kv[i]; if (kv->key.data) { @@ -19200,7 +19200,7 @@ void gguf_free(struct gguf_context * ctx) { if (kv->type == GGUF_TYPE_ARRAY) { if (kv->value.arr.data) { if (kv->value.arr.type == GGUF_TYPE_STRING) { - for (uint32_t j = 0; j < kv->value.arr.n; ++j) { + for (uint64_t j = 0; j < kv->value.arr.n; ++j) { struct gguf_str * str = &((struct gguf_str *) kv->value.arr.data)[j]; if (str->data) { free(str->data); @@ -19216,7 +19216,7 @@ void gguf_free(struct gguf_context * ctx) { } if (ctx->infos) { - for (uint32_t i = 0; i < ctx->header.n_tensors; ++i) { + for (uint64_t i = 0; i < ctx->header.n_tensors; ++i) { struct gguf_tensor_info * info = &ctx->infos[i]; if (info->name.data) { From 722d33f34ec74c6f7046109f936d0928ffe171bc Mon Sep 17 00:00:00 2001 From: Yann Follet <131855179+YannFollet@users.noreply.github.com> Date: Sun, 14 Jan 2024 00:09:08 +0800 Subject: [PATCH 051/812] main : add parameter --no-display-prompt (#4541) * add the parameter : --no-display-prompt , combine with --log-disable it will display only the generated tokens * remove empty line --------- Co-authored-by: Georgi Gerganov --- common/common.cpp | 6 +++++- common/common.h | 1 + examples/main/main.cpp | 7 ++++++- 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/common/common.cpp b/common/common.cpp index 322b9f91e..c11006bcb 100644 --- a/common/common.cpp +++ b/common/common.cpp @@ -617,6 +617,8 @@ bool gpt_params_parse_ex(int argc, char ** argv, gpt_params & params) { params.numa = true; } else if (arg == "--verbose-prompt") { params.verbose_prompt = true; + } else if (arg == "--no-display-prompt") { + params.display_prompt = false; } else if (arg == "-r" || arg == "--reverse-prompt") { if (++i >= argc) { invalid_param = true; @@ -936,11 +938,12 @@ void gpt_print_usage(int /*argc*/, char ** argv, const gpt_params & params) { printf(" -mg i, --main-gpu i the GPU to use for the model (with split-mode = none),\n"); printf(" or for intermediate results and KV (with split-mode = row) (default: %d)\n", params.main_gpu); #endif + printf(" --verbose-prompt print a verbose prompt before generation (default: %s)\n", params.verbose_prompt ? "true" : "false"); + printf(" --no-display-prompt don't print prompt at generation (default: %s)\n", !params.display_prompt ? "true" : "false"); printf(" -gan N, --grp-attn-n N\n"); printf(" group-attention factor (default: %d)\n", params.grp_attn_n); printf(" -gaw N, --grp-attn-w N\n"); printf(" group-attention width (default: %.1f)\n", (double)params.grp_attn_w); - printf(" --verbose-prompt print prompt before generation\n"); printf(" -dkvc, --dump-kv-cache\n"); printf(" verbose print of the KV cache\n"); printf(" -nkvo, --no-kv-offload\n"); @@ -1582,6 +1585,7 @@ void dump_non_result_info_yaml(FILE * stream, const gpt_params & params, const l fprintf(stream, "min_p: %f # default: 0.0\n", sparams.min_p); fprintf(stream, "typical_p: %f # default: 1.0\n", sparams.typical_p); fprintf(stream, "verbose_prompt: %s # default: false\n", params.verbose_prompt ? "true" : "false"); + fprintf(stream, "display_prompt: %s # default: true\n", params.display_prompt ? "true" : "false"); } // diff --git a/common/common.h b/common/common.h index f29be5b5a..096468243 100644 --- a/common/common.h +++ b/common/common.h @@ -126,6 +126,7 @@ struct gpt_params { bool use_mlock = false; // use mlock to keep model in memory bool numa = false; // attempt optimizations that help on some NUMA systems bool verbose_prompt = false; // print prompt tokens before generation + bool display_prompt = true; // print prompt before generation bool infill = false; // use infill mode bool dump_kv_cache = false; // dump the KV cache contents for debugging purposes bool no_kv_offload = false; // disable KV offloading diff --git a/examples/main/main.cpp b/examples/main/main.cpp index c53b29978..58b7f807a 100644 --- a/examples/main/main.cpp +++ b/examples/main/main.cpp @@ -477,6 +477,7 @@ int main(int argc, char ** argv) { bool is_antiprompt = false; bool input_echo = true; + bool display = true; bool need_to_save_session = !path_session.empty() && n_matching_session_tokens < embd_inp.size(); int n_past = 0; @@ -491,6 +492,7 @@ int main(int argc, char ** argv) { // the first thing we will do is to output the prompt, so set color accordingly console::set_display(console::prompt); + display = params.display_prompt; std::vector embd; std::vector embd_guidance; @@ -707,7 +709,7 @@ int main(int argc, char ** argv) { } // display text - if (input_echo) { + if (input_echo && display) { for (auto id : embd) { const std::string token_str = llama_token_to_piece(ctx, id); printf("%s", token_str.c_str()); @@ -724,6 +726,7 @@ int main(int argc, char ** argv) { // reset color to default if there is no pending user input if (input_echo && (int) embd_inp.size() == n_consumed) { console::set_display(console::reset); + display = true; } // if not currently processing queued inputs; @@ -796,6 +799,7 @@ int main(int argc, char ** argv) { // color user input only console::set_display(console::user_input); + display = params.display_prompt; std::string line; bool another_line = true; @@ -806,6 +810,7 @@ int main(int argc, char ** argv) { // done taking input, reset color console::set_display(console::reset); + display = true; // Add tokens to embd only if the input buffer is non-empty // Entering a empty line lets the user pass control back From 6b48ed089377330cdb362970a51c1c89b6d857a8 Mon Sep 17 00:00:00 2001 From: Someone Date: Sat, 13 Jan 2024 16:29:16 +0000 Subject: [PATCH 052/812] workflows: unbreak nix-build-aarch64, and split it out (#4915) The fix should be just the `sudo apt-get update` --- .github/workflows/nix-ci-aarch64.yml | 55 ++++++++++++++++++++++++++++ .github/workflows/nix-ci.yml | 41 --------------------- 2 files changed, 55 insertions(+), 41 deletions(-) create mode 100644 .github/workflows/nix-ci-aarch64.yml diff --git a/.github/workflows/nix-ci-aarch64.yml b/.github/workflows/nix-ci-aarch64.yml new file mode 100644 index 000000000..be7c26d40 --- /dev/null +++ b/.github/workflows/nix-ci-aarch64.yml @@ -0,0 +1,55 @@ +name: Nix aarch64 builds + +on: + workflow_dispatch: # allows manual triggering + push: + branches: + - master + paths: ['.github/workflows/**', '**/CMakeLists.txt', '**/Makefile', '**/*.h', '**/*.hpp', '**/*.c', '**/*.cpp', '**/*.cu', '**/*.swift', '**/*.m', '**/*.sh', '**/*.py', '**/*.nix'] + pull_request: + types: [opened, synchronize, reopened] + paths: ['**/CMakeLists.txt', '**/Makefile', '**/*.h', '**/*.hpp', '**/*.c', '**/*.cpp', '**/*.cu', '**/*.swift', '**/*.m', '**/*.sh', '**/*.py', '**/*.nix'] + +jobs: + nix-build-aarch64: + if: ${{ vars.CACHIX_NAME != '' }} + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + - name: Install QEMU + # Copy-paste from https://github.com/orgs/community/discussions/8305#discussioncomment-5888654 + run: | + sudo apt-get update + sudo apt-get install -y qemu-user-static qemu-system-aarch64 + sudo usermod -a -G kvm $USER + - name: Install Nix + uses: DeterminateSystems/nix-installer-action@v9 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + extra-conf: | + extra-platforms = aarch64-linux + extra-system-features = nixos-test kvm + extra-substituters = https://${{ vars.CACHIX_NAME }}.cachix.org https://cuda-maintainers.cachix.org + extra-trusted-public-keys = ${{ vars.CACHIX_PUBLIC_KEY }} cuda-maintainers.cachix.org-1:0dq3bujKpuEPMCX6U4WylrUDZ9JyUG0VpVZa7CNfq5E= + - uses: DeterminateSystems/magic-nix-cache-action@v2 + with: + upstream-cache: https://${{ matrix.cachixName }}.cachix.org + - name: Set-up cachix to push the results to + uses: cachix/cachix-action@v13 + with: + authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}' + name: ${{ vars.CACHIX_NAME }} + - name: Show all output paths + run: > + nix run github:nix-community/nix-eval-jobs + -- --gc-roots-dir gcroot + --flake + ".#packages.aarch64-linux" + - name: Build + run: > + nix run github:Mic92/nix-fast-build + -- --skip-cached --no-nom + --systems aarch64-linux + --flake + ".#checks.aarch64-linux" diff --git a/.github/workflows/nix-ci.yml b/.github/workflows/nix-ci.yml index a38c6ead4..845b93bfb 100644 --- a/.github/workflows/nix-ci.yml +++ b/.github/workflows/nix-ci.yml @@ -69,44 +69,3 @@ jobs: -- --skip-cached --no-nom --flake ".#checks.$(nix eval --raw --impure --expr builtins.currentSystem)" - nix-build-aarch64: - if: ${{ vars.CACHIX_NAME != '' }} - runs-on: ubuntu-latest - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - name: Install QEMU - # Copy-paste from https://github.com/orgs/community/discussions/8305#discussioncomment-5888654 - run: | - sudo apt-get install -y qemu-user-static qemu-system-aarch64 - sudo usermod -a -G kvm $USER - - name: Install Nix - uses: DeterminateSystems/nix-installer-action@v9 - with: - github-token: ${{ secrets.GITHUB_TOKEN }} - extra-conf: | - extra-platforms = aarch64-linux - extra-system-features = nixos-test kvm - extra-substituters = https://${{ vars.CACHIX_NAME }}.cachix.org https://cuda-maintainers.cachix.org - extra-trusted-public-keys = ${{ vars.CACHIX_PUBLIC_KEY }} cuda-maintainers.cachix.org-1:0dq3bujKpuEPMCX6U4WylrUDZ9JyUG0VpVZa7CNfq5E= - - uses: DeterminateSystems/magic-nix-cache-action@v2 - with: - upstream-cache: https://${{ matrix.cachixName }}.cachix.org - - name: Set-up cachix to push the results to - uses: cachix/cachix-action@v13 - with: - authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}' - name: ${{ vars.CACHIX_NAME }} - - name: Show all output paths - run: > - nix run github:nix-community/nix-eval-jobs - -- --gc-roots-dir gcroot - --flake - ".#packages.aarch64-linux" - - name: Build - run: > - nix run github:Mic92/nix-fast-build - -- --skip-cached --no-nom - --systems aarch64-linux - --flake - ".#checks.aarch64-linux" From df845cc982e7e2ea7b9900e29d55b15338faa78d Mon Sep 17 00:00:00 2001 From: David Friehs Date: Sat, 13 Jan 2024 17:29:43 +0100 Subject: [PATCH 053/812] llama : minimize size used for state save/load (#4820) * examples : save-load-state: save only required state * llama : only reserve n_vocab * n_batch at most for logits llama_decode asserts that only n_batch tokens are passed each call, and n_ctx is expected to be bigger than n_batch. * llama : always reserve n_vocab * n_batch for logits llama_context de-serialization breaks if the contexts have differing capacity for logits and llama_decode will at maximum resize to n_vocab * n_batch. * llama : only save and restore used logits for batch sizes of 512 this reduces save state in the best case by around 62 MB, which can be a lot if planning to save on each message to allow regenerating messages. * llama : use ostringstream and istringstream for save and load * llama : serialize rng into minimum amount of space required * llama : break session version due to serialization changes --- examples/save-load-state/save-load-state.cpp | 21 ++++---- llama.cpp | 53 +++++++------------- llama.h | 2 +- 3 files changed, 29 insertions(+), 47 deletions(-) diff --git a/examples/save-load-state/save-load-state.cpp b/examples/save-load-state/save-load-state.cpp index 48d801110..ef952e2bd 100644 --- a/examples/save-load-state/save-load-state.cpp +++ b/examples/save-load-state/save-load-state.cpp @@ -45,13 +45,13 @@ int main(int argc, char ** argv) { // save state (rng, logits, embedding and kv_cache) to file { std::vector state_mem(llama_get_state_size(ctx)); + const size_t written = llama_copy_state_data(ctx, state_mem.data()); - { - FILE *fp_write = fopen("dump_state.bin", "wb"); - llama_copy_state_data(ctx, state_mem.data()); // could also copy directly to memory mapped file - fwrite(state_mem.data(), 1, state_mem.size(), fp_write); - fclose(fp_write); - } + FILE *fp_write = fopen("dump_state.bin", "wb"); + fwrite(state_mem.data(), 1, written, fp_write); + fclose(fp_write); + + fprintf(stderr, "%s : serialized state into %zd out of a maximum of %zd bytes\n", __func__, written, state_mem.size()); } // save state (last tokens) @@ -100,18 +100,17 @@ int main(int argc, char ** argv) { std::vector state_mem(llama_get_state_size(ctx2)); FILE * fp_read = fopen("dump_state.bin", "rb"); + const size_t read = fread(state_mem.data(), 1, state_mem.size(), fp_read); + fclose(fp_read); - const size_t ret = fread(state_mem.data(), 1, state_mem.size(), fp_read); - if (ret != state_mem.size()) { + if (read != llama_set_state_data(ctx2, state_mem.data())) { fprintf(stderr, "\n%s : failed to read state\n", __func__); llama_free(ctx2); llama_free_model(model); return 1; } - llama_set_state_data(ctx2, state_mem.data()); - - fclose(fp_read); + fprintf(stderr, "%s : deserialized state from %zd out of a maximum of %zd bytes\n", __func__, read, state_mem.size()); } // restore state (last tokens) diff --git a/llama.cpp b/llama.cpp index 1d2eb569f..275456088 100644 --- a/llama.cpp +++ b/llama.cpp @@ -9379,12 +9379,8 @@ struct llama_context * llama_new_context_with_model( ggml_type_name(type_v), (float)memory_size_v / (1024.0f * 1024.0f)); } - // resized during inference - if (params.logits_all) { - ctx->logits.reserve(cparams.n_ctx*hparams.n_vocab); - } else { - ctx->logits.reserve(hparams.n_vocab); - } + // resized during inference, reserve maximum + ctx->logits.reserve(hparams.n_vocab*cparams.n_batch); if (params.embedding){ ctx->embedding.resize(hparams.n_embd); @@ -9731,8 +9727,8 @@ size_t llama_get_state_size(const struct llama_context * ctx) { // for reference, std::mt19937(1337) serializes to 6701 bytes. const size_t s_rng_size = sizeof(size_t); const size_t s_rng = LLAMA_MAX_RNG_STATE; - const size_t s_logits_capacity = sizeof(size_t); const size_t s_logits_size = sizeof(size_t); + // assume worst case for logits although only currently set ones are serialized const size_t s_logits = ctx->logits.capacity() * sizeof(float); const size_t s_embedding_size = sizeof(size_t); const size_t s_embedding = ctx->embedding.size() * sizeof(float); @@ -9743,7 +9739,6 @@ size_t llama_get_state_size(const struct llama_context * ctx) { const size_t s_total = ( + s_rng_size + s_rng - + s_logits_capacity + s_logits_size + s_logits + s_embedding_size @@ -9812,37 +9807,27 @@ struct llama_data_file_context : llama_data_context { static void llama_copy_state_data_internal(struct llama_context * ctx, llama_data_context * data_ctx) { // copy rng { - std::stringstream rng_ss; + std::ostringstream rng_ss; rng_ss << ctx->rng; - const size_t rng_size = rng_ss.str().size(); - char rng_buf[LLAMA_MAX_RNG_STATE]; + const std::string & rng_str = rng_ss.str(); + const size_t rng_size = rng_str.size(); - memset(&rng_buf[0], 0, LLAMA_MAX_RNG_STATE); - memcpy(&rng_buf[0], rng_ss.str().data(), rng_ss.str().size()); + GGML_ASSERT(rng_size <= LLAMA_MAX_RNG_STATE); - data_ctx->write(&rng_size, sizeof(rng_size)); - data_ctx->write(&rng_buf[0], LLAMA_MAX_RNG_STATE); + data_ctx->write(&rng_size, sizeof(rng_size)); + data_ctx->write(rng_str.data(), rng_size); } // copy logits { - const size_t logits_cap = ctx->logits.capacity(); const size_t logits_size = ctx->logits.size(); - data_ctx->write(&logits_cap, sizeof(logits_cap)); data_ctx->write(&logits_size, sizeof(logits_size)); if (logits_size) { data_ctx->write(ctx->logits.data(), logits_size * sizeof(float)); } - - // If there is a gap between the size and the capacity, write padding - size_t padding_size = (logits_cap - logits_size) * sizeof(float); - if (padding_size > 0) { - std::vector padding(padding_size, 0); // Create a buffer filled with zeros - data_ctx->write(padding.data(), padding_size); - } } // copy embeddings @@ -9925,13 +9910,13 @@ size_t llama_set_state_data(struct llama_context * ctx, uint8_t * src) { // set rng { size_t rng_size; - char rng_buf[LLAMA_MAX_RNG_STATE]; + memcpy(&rng_size, inp, sizeof(rng_size)); inp += sizeof(rng_size); - memcpy(&rng_size, inp, sizeof(rng_size)); inp += sizeof(rng_size); - memcpy(&rng_buf[0], inp, LLAMA_MAX_RNG_STATE); inp += LLAMA_MAX_RNG_STATE; + GGML_ASSERT(rng_size <= LLAMA_MAX_RNG_STATE); - std::stringstream rng_ss; - rng_ss.str(std::string(&rng_buf[0], rng_size)); + std::string rng_str((char *)inp, rng_size); inp += rng_size; + + std::istringstream rng_ss(rng_str); rng_ss >> ctx->rng; GGML_ASSERT(!rng_ss.fail()); @@ -9939,20 +9924,18 @@ size_t llama_set_state_data(struct llama_context * ctx, uint8_t * src) { // set logits { - size_t logits_cap; size_t logits_size; - memcpy(&logits_cap, inp, sizeof(logits_cap)); inp += sizeof(logits_cap); memcpy(&logits_size, inp, sizeof(logits_size)); inp += sizeof(logits_size); - GGML_ASSERT(ctx->logits.capacity() == logits_cap); + GGML_ASSERT(ctx->logits.capacity() >= logits_size); if (logits_size) { ctx->logits.resize(logits_size); - memcpy(ctx->logits.data(), inp, logits_size * sizeof(float)); - } - inp += logits_cap * sizeof(float); + memcpy(ctx->logits.data(), inp, logits_size * sizeof(float)); + inp += logits_size * sizeof(float); + } } // set embeddings diff --git a/llama.h b/llama.h index 689e12d7c..01d6fafaa 100644 --- a/llama.h +++ b/llama.h @@ -43,7 +43,7 @@ #define LLAMA_FILE_MAGIC_GGSN 0x6767736eu // 'ggsn' #define LLAMA_SESSION_MAGIC LLAMA_FILE_MAGIC_GGSN -#define LLAMA_SESSION_VERSION 3 +#define LLAMA_SESSION_VERSION 4 #if defined(GGML_USE_CUBLAS) || defined(GGML_USE_CLBLAST) || defined(GGML_USE_METAL) // Defined when llama.cpp is compiled with support for offloading model layers to GPU. From 2d57de525541247132e354f561ff48775fba5d85 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Sat, 13 Jan 2024 18:46:37 +0200 Subject: [PATCH 054/812] metal : disable log for loaded kernels (#4794) --- ggml-metal.m | 3 --- 1 file changed, 3 deletions(-) diff --git a/ggml-metal.m b/ggml-metal.m index 6c28a7ee3..57e444827 100644 --- a/ggml-metal.m +++ b/ggml-metal.m @@ -398,9 +398,6 @@ struct ggml_metal_context * ggml_metal_init(int n_cb) { struct ggml_metal_kernel * kernel = &ctx->kernels[e]; \ kernel->function = [ctx->library newFunctionWithName:@"kernel_"#name]; \ kernel->pipeline = [ctx->device newComputePipelineStateWithFunction:kernel->function error:&error]; \ - GGML_METAL_LOG_INFO("%s: loaded %-32s %16p | th_max = %4d | th_width = %4d\n", __func__, "kernel_"#name, (void *) kernel->pipeline, \ - (int) kernel->pipeline.maxTotalThreadsPerThreadgroup, \ - (int) kernel->pipeline.threadExecutionWidth); \ if (error) { \ GGML_METAL_LOG_ERROR("%s: error: load pipeline error: %s\n", __func__, [[error description] UTF8String]); \ return NULL; \ From f172de03f11465dc6c5a0fc3a22f8ec254c6832c Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Sat, 13 Jan 2024 18:47:38 +0200 Subject: [PATCH 055/812] llama : fix detokenization of non-special added-tokens (#4916) Co-authored-by: goerch --- llama.cpp | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/llama.cpp b/llama.cpp index 275456088..2190ea7aa 100644 --- a/llama.cpp +++ b/llama.cpp @@ -10305,6 +10305,8 @@ int32_t llama_token_to_piece(const struct llama_model * model, llama_token token if (0 <= token && token < llama_n_vocab(model)) { switch (llama_vocab_get_type(model->vocab)) { case LLAMA_VOCAB_TYPE_SPM: { + // NOTE: we accept all unsupported token types, + // suppressing them like CONTROL tokens. if (llama_is_normal_token(model->vocab, token)) { std::string result = model->vocab.id_to_token[token].text; llama_unescape_whitespace(result); @@ -10313,6 +10315,13 @@ int32_t llama_token_to_piece(const struct llama_model * model, llama_token token } memcpy(buf, result.c_str(), result.length()); return result.length(); + } else if (llama_is_user_defined_token(model->vocab, token)) { + std::string result = model->vocab.id_to_token[token].text; + if (length < (int) result.length()) { + return -result.length(); + } + memcpy(buf, result.c_str(), result.length()); + return result.length(); } else if (llama_is_unknown_token(model->vocab, token)) { // NOLINT if (length < 3) { return -3; @@ -10327,14 +10336,12 @@ int32_t llama_token_to_piece(const struct llama_model * model, llama_token token } buf[0] = llama_token_to_byte(model->vocab, token); return 1; - } else { - // TODO: for now we accept all unsupported token types, - // suppressing them like CONTROL tokens. - // GGML_ASSERT(false); } break; } case LLAMA_VOCAB_TYPE_BPE: { + // NOTE: we accept all unsupported token types, + // suppressing them like CONTROL tokens. if (llama_is_normal_token(model->vocab, token)) { std::string result = model->vocab.id_to_token[token].text; result = llama_decode_text(result); @@ -10343,12 +10350,15 @@ int32_t llama_token_to_piece(const struct llama_model * model, llama_token token } memcpy(buf, result.c_str(), result.length()); return result.length(); + } else if (llama_is_user_defined_token(model->vocab, token)) { + std::string result = model->vocab.id_to_token[token].text; + if (length < (int) result.length()) { + return -result.length(); + } + memcpy(buf, result.c_str(), result.length()); + return result.length(); } else if (llama_is_control_token(model->vocab, token)) { ; - } else { - // TODO: for now we accept all unsupported token types, - // suppressing them like CONTROL tokens. - // GGML_ASSERT(false); } break; } From 0ea069b87bd296c556824e57455433b6c0357340 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Sat, 13 Jan 2024 19:31:26 +0200 Subject: [PATCH 056/812] server : fix prompt caching with system prompt (#4914) --- examples/server/server.cpp | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/examples/server/server.cpp b/examples/server/server.cpp index 79eacf828..93f999298 100644 --- a/examples/server/server.cpp +++ b/examples/server/server.cpp @@ -1180,8 +1180,9 @@ struct llama_server_context return slot.images.size() > 0; } - void send_error(task_server& task, std::string error) + void send_error(task_server& task, const std::string &error) { + LOG_TEE("task %i - error: %s\n", task.id, error.c_str()); std::unique_lock lock(mutex_results); task_result res; res.id = task.id; @@ -1570,12 +1571,22 @@ struct llama_server_context LOG_TEE("slot unavailable\n"); // send error result send_error(task, "slot unavailable"); - return; + break; } if (task.data.contains("system_prompt")) { + if (!all_slots_are_idle) { + send_error(task, "system prompt can only be updated when all slots are idle"); + break; + } process_system_prompt_data(task.data["system_prompt"]); + + // reset cache_tokens for all slots + for (llama_client_slot &slot : slots) + { + slot.cache_tokens.clear(); + } } slot->reset(); @@ -1652,8 +1663,7 @@ struct llama_server_context // attend tasks process_tasks(); - // update the system prompt wait until all slots are idle state - if (system_need_update && all_slots_are_idle) + if (system_need_update) { LOG_TEE("updating system prompt\n"); update_system_prompt(); From 4be5ef556de830c5c4f6e45c05ef4427823fe607 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Sat, 13 Jan 2024 20:45:45 +0200 Subject: [PATCH 057/812] metal : remove old API (#4919) ggml-ci --- Makefile | 9 -- examples/CMakeLists.txt | 3 - examples/metal/CMakeLists.txt | 4 - examples/metal/metal.cpp | 103 ------------- ggml-metal.h | 55 +------ ggml-metal.m | 276 +++------------------------------- llama.cpp | 4 +- 7 files changed, 27 insertions(+), 427 deletions(-) delete mode 100644 examples/metal/CMakeLists.txt delete mode 100644 examples/metal/metal.cpp diff --git a/Makefile b/Makefile index 05fe9a0f6..995b89f7a 100644 --- a/Makefile +++ b/Makefile @@ -43,10 +43,6 @@ ifeq ($(UNAME_S),Darwin) endif endif -ifneq '' '$(or $(filter clean,$(MAKECMDGOALS)),$(LLAMA_METAL))' -BUILD_TARGETS += metal -endif - default: $(BUILD_TARGETS) test: $(TEST_TARGETS) @@ -671,11 +667,6 @@ lookup: examples/lookup/lookup.cpp ggml.o llama.o $(COMMON_DEPS) $(OBJS) passkey: examples/passkey/passkey.cpp ggml.o llama.o $(COMMON_DEPS) $(OBJS) $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) -ifdef LLAMA_METAL -metal: examples/metal/metal.cpp ggml.o $(OBJS) - $(CXX) $(CXXFLAGS) $^ -o $@ $(LDFLAGS) -endif - ifeq ($(UNAME_S),Darwin) swift: examples/batched.swift (cd examples/batched.swift; make build) diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index fa127a3aa..f67d74c55 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -37,9 +37,6 @@ else() add_subdirectory(lookup) add_subdirectory(train-text-from-scratch) add_subdirectory(imatrix) - if (LLAMA_METAL) - add_subdirectory(metal) - endif() if (LLAMA_BUILD_SERVER) add_subdirectory(server) endif() diff --git a/examples/metal/CMakeLists.txt b/examples/metal/CMakeLists.txt deleted file mode 100644 index f16d49165..000000000 --- a/examples/metal/CMakeLists.txt +++ /dev/null @@ -1,4 +0,0 @@ -set(TEST_TARGET metal) -add_executable(${TEST_TARGET} metal.cpp) -install(TARGETS ${TARGET} RUNTIME) -target_link_libraries(${TEST_TARGET} PRIVATE ggml) diff --git a/examples/metal/metal.cpp b/examples/metal/metal.cpp deleted file mode 100644 index 16c1146f9..000000000 --- a/examples/metal/metal.cpp +++ /dev/null @@ -1,103 +0,0 @@ -// Evaluate a statically exported ggml computation graph with Metal -// -// - First, export a LLaMA graph: -// -// $ ./bin/main -m ../models/7B/ggml-model-q4_0.gguf --export -// -// - Run this tool to evaluate the exported graph: -// -// $ ./bin/metal llama.ggml -// -// The purpose of this tool is mostly for debugging and demonstration purposes. -// The main limitation of exporting computation graphs is that their sizes are static which often -// can be a problem for real-world applications. -// - -#include "ggml.h" -#include "ggml-metal.h" - -#include -#include -#include - -int main(int argc, char ** argv) { - ggml_time_init(); - - if (argc != 2) { - fprintf(stderr, "Usage: %s llama.ggml\n", argv[0]); - return -1; - } - - const char * fname_cgraph = argv[1]; - - // load the compute graph - struct ggml_context * ctx_data = NULL; - struct ggml_context * ctx_eval = NULL; - - struct ggml_cgraph * gf = ggml_graph_import(fname_cgraph, &ctx_data, &ctx_eval); - - // this allocates all Metal resources and memory buffers - auto * ctx_metal = ggml_metal_init(1); - - const size_t max_size_data = ggml_get_max_tensor_size(ctx_data); - const size_t max_size_eval = ggml_get_max_tensor_size(ctx_eval); - ggml_metal_add_buffer(ctx_metal, "data", ggml_get_mem_buffer(ctx_data), ggml_get_mem_size(ctx_data), max_size_data); - ggml_metal_add_buffer(ctx_metal, "eval", ggml_get_mem_buffer(ctx_eval), ggml_get_mem_size(ctx_eval), max_size_eval); - - // main - { - struct ggml_tensor * input = ggml_graph_get_tensor(gf, "embd"); - *(int32_t *) input->data = 1; // BOS - - ggml_metal_set_tensor(ctx_metal, input); - - // warmup - ggml_metal_graph_compute(ctx_metal, gf); - - const int n_iter = 16; - - const int64_t t0 = ggml_time_us(); - - // the actual inference happens here - for (int i = 0; i < n_iter; ++i) { - ggml_metal_graph_compute(ctx_metal, gf); - } - - const int64_t t1 = ggml_time_us(); - - printf("time: %.2f ms, %.2f ms/tok\n", (t1 - t0) / 1000.0, (t1 - t0) / 1000.0 / n_iter); - } - - // debug output - { - struct ggml_tensor * logits = gf->nodes[gf->n_nodes - 1]; - ggml_metal_get_tensor(ctx_metal, logits); - - float * ptr = (float *) ggml_get_data(logits); - - printf("logits: "); - for (int i = 0; i < 10; i++) { - printf("%8.4f ", ptr[i]); - } - printf("\n"); - int imax = 0; - double sum = 0.0; - double vmax = -1e9; - for (int i = 0; i < 32000; i++) { - sum += (double) ptr[i]; - if (ptr[i] > vmax) { - vmax = ptr[i]; - imax = i; - } - } - printf("sum: %f, imax = %d, vmax = %f\n", sum, imax, vmax); - } - - ggml_metal_free(ctx_metal); - - ggml_free(ctx_data); - ggml_free(ctx_eval); - - return 0; -} - diff --git a/ggml-metal.h b/ggml-metal.h index c4b7325da..cd5e2995f 100644 --- a/ggml-metal.h +++ b/ggml-metal.h @@ -36,64 +36,13 @@ struct ggml_cgraph; extern "C" { #endif -// -// internal API -// temporary exposed to user-code -// - -struct ggml_metal_context; - -void ggml_metal_log_set_callback(ggml_log_callback log_callback, void * user_data); - -// number of command buffers to use -struct ggml_metal_context * ggml_metal_init(int n_cb); -void ggml_metal_free(struct ggml_metal_context * ctx); - -void * ggml_metal_host_malloc(size_t n); -void ggml_metal_host_free (void * data); - -// set the number of command buffers to use -void ggml_metal_set_n_cb(struct ggml_metal_context * ctx, int n_cb); - -// creates a mapping between a host memory buffer and a device memory buffer -// - make sure to map all buffers used in the graph before calling ggml_metal_graph_compute -// - the mapping is used during computation to determine the arguments of the compute kernels -// - you don't need to keep the host memory buffer allocated as it is never accessed by Metal -// - max_size specifies the maximum size of a tensor and is used to create shared views such -// that it is guaranteed that the tensor will fit in at least one of the views -// -bool ggml_metal_add_buffer( - struct ggml_metal_context * ctx, - const char * name, - void * data, - size_t size, - size_t max_size); - -// set data from host memory into the device -void ggml_metal_set_tensor(struct ggml_metal_context * ctx, struct ggml_tensor * t); - -// get data from the device into host memory -void ggml_metal_get_tensor(struct ggml_metal_context * ctx, struct ggml_tensor * t); - -// try to find operations that can be run concurrently in the graph -// you should run it again if the topology of your graph changes -void ggml_metal_graph_find_concurrency(struct ggml_metal_context * ctx, struct ggml_cgraph * gf, bool check_mem); - -// if the graph has been optimized for concurrently dispatch, return length of the concur_list if optimized -int ggml_metal_if_optimized(struct ggml_metal_context * ctx); - -// output the concur_list for ggml_alloc -int * ggml_metal_get_concur_list(struct ggml_metal_context * ctx); - -// same as ggml_graph_compute but uses Metal -// creates gf->n_threads command buffers in parallel -bool ggml_metal_graph_compute(struct ggml_metal_context * ctx, struct ggml_cgraph * gf); - // // backend API // user-code should use only these functions // +GGML_API void ggml_backend_metal_log_set_callback(ggml_log_callback log_callback, void * user_data); + GGML_API ggml_backend_t ggml_backend_metal_init(void); GGML_API bool ggml_backend_is_metal(ggml_backend_t backend); diff --git a/ggml-metal.m b/ggml-metal.m index 57e444827..cae52c983 100644 --- a/ggml-metal.m +++ b/ggml-metal.m @@ -24,8 +24,6 @@ #define UNUSED(x) (void)(x) -#define GGML_MAX_CONCUR (2*GGML_DEFAULT_GRAPH_SIZE) - #define GGML_METAL_MAX_KERNELS 256 struct ggml_metal_buffer { @@ -182,9 +180,6 @@ struct ggml_metal_context { struct ggml_metal_kernel kernels[GGML_METAL_MAX_KERNELS]; - int concur_list[GGML_MAX_CONCUR]; - int concur_list_len; - bool support_simdgroup_reduction; bool support_simdgroup_mm; }; @@ -200,7 +195,6 @@ struct ggml_metal_context { @implementation GGMLMetalClass @end - static void ggml_metal_default_log_callback(enum ggml_log_level level, const char * msg, void * user_data) { fprintf(stderr, "%s", msg); @@ -211,11 +205,6 @@ static void ggml_metal_default_log_callback(enum ggml_log_level level, const cha ggml_log_callback ggml_metal_log_callback = ggml_metal_default_log_callback; void * ggml_metal_log_user_data = NULL; -void ggml_metal_log_set_callback(ggml_log_callback log_callback, void * user_data) { - ggml_metal_log_callback = log_callback; - ggml_metal_log_user_data = user_data; -} - GGML_ATTRIBUTE_FORMAT(2, 3) static void ggml_metal_log(enum ggml_log_level level, const char * format, ...){ if (ggml_metal_log_callback != NULL) { @@ -238,7 +227,18 @@ static void ggml_metal_log(enum ggml_log_level level, const char * format, ...){ } } -struct ggml_metal_context * ggml_metal_init(int n_cb) { +static void * ggml_metal_host_malloc(size_t n) { + void * data = NULL; + const int result = posix_memalign((void **) &data, sysconf(_SC_PAGESIZE), n); + if (result != 0) { + GGML_METAL_LOG_ERROR("%s: error: posix_memalign failed\n", __func__); + return NULL; + } + + return data; +} + +static struct ggml_metal_context * ggml_metal_init(int n_cb) { GGML_METAL_LOG_INFO("%s: allocating\n", __func__); id device; @@ -264,7 +264,6 @@ struct ggml_metal_context * ggml_metal_init(int n_cb) { ctx->n_cb = MIN(n_cb, GGML_METAL_MAX_BUFFERS); ctx->queue = [ctx->device newCommandQueue]; ctx->n_buffers = 0; - ctx->concur_list_len = 0; ctx->d_queue = dispatch_queue_create("ggml-metal", DISPATCH_QUEUE_CONCURRENT); @@ -531,7 +530,7 @@ struct ggml_metal_context * ggml_metal_init(int n_cb) { return ctx; } -void ggml_metal_free(struct ggml_metal_context * ctx) { +static void ggml_metal_free(struct ggml_metal_context * ctx) { GGML_METAL_LOG_INFO("%s: deallocating\n", __func__); for (int i = 0; i < ctx->n_buffers; ++i) { @@ -557,33 +556,6 @@ void ggml_metal_free(struct ggml_metal_context * ctx) { free(ctx); } -void * ggml_metal_host_malloc(size_t n) { - void * data = NULL; - const int result = posix_memalign((void **) &data, sysconf(_SC_PAGESIZE), n); - if (result != 0) { - GGML_METAL_LOG_ERROR("%s: error: posix_memalign failed\n", __func__); - return NULL; - } - - return data; -} - -void ggml_metal_host_free(void * data) { - free(data); -} - -void ggml_metal_set_n_cb(struct ggml_metal_context * ctx, int n_cb) { - ctx->n_cb = MIN(n_cb, GGML_METAL_MAX_BUFFERS); -} - -int ggml_metal_if_optimized(struct ggml_metal_context * ctx) { - return ctx->concur_list_len; -} - -int * ggml_metal_get_concur_list(struct ggml_metal_context * ctx) { - return ctx->concur_list; -} - // temporarily defined here for compatibility between ggml-backend and the old API struct ggml_backend_metal_buffer { @@ -656,209 +628,6 @@ static id ggml_metal_get_buffer(struct ggml_metal_context * ctx, stru return nil; } -bool ggml_metal_add_buffer( - struct ggml_metal_context * ctx, - const char * name, - void * data, - size_t size, - size_t max_size) { - if (ctx->n_buffers >= GGML_METAL_MAX_BUFFERS) { - GGML_METAL_LOG_ERROR("%s: error: too many buffers\n", __func__); - return false; - } - - if (data) { - // verify that the buffer does not overlap with any of the existing buffers - for (int i = 0; i < ctx->n_buffers; ++i) { - const int64_t ioffs = (int64_t) data - (int64_t) ctx->buffers[i].data; - - if (ioffs >= 0 && ioffs < (int64_t) ctx->buffers[i].size) { - GGML_METAL_LOG_ERROR("%s: error: buffer '%s' overlaps with '%s'\n", __func__, name, ctx->buffers[i].name); - return false; - } - } - - const size_t size_page = sysconf(_SC_PAGESIZE); - - size_t size_aligned = size; - if ((size_aligned % size_page) != 0) { - size_aligned += (size_page - (size_aligned % size_page)); - } - - // the buffer fits into the max buffer size allowed by the device - if (size_aligned <= ctx->device.maxBufferLength) { - ctx->buffers[ctx->n_buffers].name = name; - ctx->buffers[ctx->n_buffers].data = data; - ctx->buffers[ctx->n_buffers].size = size; - - ctx->buffers[ctx->n_buffers].metal = [ctx->device newBufferWithBytesNoCopy:data length:size_aligned options:MTLResourceStorageModeShared deallocator:nil]; - - if (ctx->buffers[ctx->n_buffers].metal == nil) { - GGML_METAL_LOG_ERROR("%s: error: failed to allocate '%-16s' buffer, size = %8.2f MiB\n", __func__, name, size_aligned / 1024.0 / 1024.0); - return false; - } - - GGML_METAL_LOG_INFO("%s: allocated '%-16s' buffer, size = %8.2f MiB", __func__, name, size_aligned / 1024.0 / 1024.0); - - ++ctx->n_buffers; - } else { - // this overlap between the views will guarantee that the tensor with the maximum size will fully fit into - // one of the views - const size_t size_ovlp = ((max_size + size_page - 1) / size_page + 1) * size_page; // round-up 2 pages just in case - const size_t size_step = ctx->device.maxBufferLength - size_ovlp; - const size_t size_view = ctx->device.maxBufferLength; - - for (size_t i = 0; i < size; i += size_step) { - const size_t size_step_aligned = (i + size_view <= size) ? size_view : (size_aligned - i); - - ctx->buffers[ctx->n_buffers].name = name; - ctx->buffers[ctx->n_buffers].data = (void *) ((uint8_t *) data + i); - ctx->buffers[ctx->n_buffers].size = size_step_aligned; - - ctx->buffers[ctx->n_buffers].metal = [ctx->device newBufferWithBytesNoCopy:(void *) ((uint8_t *) data + i) length:size_step_aligned options:MTLResourceStorageModeShared deallocator:nil]; - - if (ctx->buffers[ctx->n_buffers].metal == nil) { - GGML_METAL_LOG_ERROR("%s: error: failed to allocate '%-16s' buffer, size = %8.2f MiB\n", __func__, name, size_step_aligned / 1024.0 / 1024.0); - return false; - } - - GGML_METAL_LOG_INFO("%s: allocated '%-16s' buffer, size = %8.2f MiB, offs = %12ld", __func__, name, size_step_aligned / 1024.0 / 1024.0, i); - if (i + size_step < size) { - GGML_METAL_LOG_INFO("\n"); - } - - ++ctx->n_buffers; - } - } - -#if TARGET_OS_OSX - GGML_METAL_LOG_INFO(", (%8.2f / %8.2f)", - ctx->device.currentAllocatedSize / 1024.0 / 1024.0, - ctx->device.recommendedMaxWorkingSetSize / 1024.0 / 1024.0); - - if (ctx->device.currentAllocatedSize > ctx->device.recommendedMaxWorkingSetSize) { - GGML_METAL_LOG_WARN("%s: warning: current allocated size is greater than the recommended max working set size\n", __func__); - } else { - GGML_METAL_LOG_INFO("\n"); - } -#else - GGML_METAL_LOG_INFO(", (%8.2f)\n", ctx->device.currentAllocatedSize / 1024.0 / 1024.0); -#endif - } - - return true; -} - -void ggml_metal_set_tensor( - struct ggml_metal_context * ctx, - struct ggml_tensor * t) { - size_t offs; - id id_dst = ggml_metal_get_buffer(ctx, t, &offs); - - memcpy((void *) ((uint8_t *) id_dst.contents + offs), t->data, ggml_nbytes(t)); -} - -void ggml_metal_get_tensor( - struct ggml_metal_context * ctx, - struct ggml_tensor * t) { - size_t offs; - id id_src = ggml_metal_get_buffer(ctx, t, &offs); - - memcpy(t->data, (void *) ((uint8_t *) id_src.contents + offs), ggml_nbytes(t)); -} - -void ggml_metal_graph_find_concurrency( - struct ggml_metal_context * ctx, - struct ggml_cgraph * gf, bool check_mem) { - int search_depth = gf->n_nodes; //we only find concurrency in this range to avoid wasting too much time - int nodes_unused[GGML_MAX_CONCUR]; - - for (int i = 0; i < GGML_MAX_CONCUR; i++) { ctx->concur_list[i] = 0; } - for (int i = 0; i < gf->n_nodes; i++) { nodes_unused[i] = 1; } - ctx->concur_list_len = 0; - - int n_left = gf->n_nodes; - int n_start = 0; // all nodes before n_start at nodes_unused array have been sorted and store back to ctx->concur_list - int level_pos = 0; // at ctx->concur_list, the last layer (level) ends at level_pos - - while (n_left > 0) { - // number of nodes at a layer (that can be issued concurrently) - int concurrency = 0; - for (int i = n_start; i < ((n_start + search_depth > gf->n_nodes) ? gf->n_nodes : n_start + search_depth); i++) { - if (nodes_unused[i]) { - // if the requirements for gf->nodes[i] are satisfied - int exe_flag = 1; - - // scan all srcs - for (int src_ind = 0; src_ind < GGML_MAX_SRC; src_ind++) { - struct ggml_tensor * src_cur = gf->nodes[i]->src[src_ind]; - if (src_cur) { - // if is leaf nodes it's satisfied. - // TODO: ggml_is_leaf() - if (src_cur->op == GGML_OP_NONE && src_cur->grad == NULL) { - continue; - } - - // otherwise this src should be the output from previous nodes. - int is_found = 0; - - // scan 2*search_depth back because we inserted barrier. - //for (int j = ((level_pos - 2*search_depth) < 0 ? 0 : (level_pos - 2*search_depth)); j < level_pos; j++) { - for (int j = MAX(0, level_pos - 2*search_depth); j < level_pos; j++) { - if (ctx->concur_list[j] >= 0 && gf->nodes[ctx->concur_list[j]] == src_cur) { - is_found = 1; - break; - } - } - if (is_found == 0) { - exe_flag = 0; - break; - } - } - } - if (exe_flag && check_mem) { - // check if nodes[i]'s data will be overwritten by a node before nodes[i]. - // if node[5] and node[3] write to the same memory region, then we can't issue node[5] before node[3] - int64_t data_start = (int64_t) gf->nodes[i]->data; - int64_t length = (int64_t) ggml_nbytes(gf->nodes[i]); - for (int j = n_start; j < i; j++) { - if (nodes_unused[j] && gf->nodes[j]->op != GGML_OP_RESHAPE \ - && gf->nodes[j]->op != GGML_OP_VIEW \ - && gf->nodes[j]->op != GGML_OP_TRANSPOSE \ - && gf->nodes[j]->op != GGML_OP_PERMUTE) { - if (((int64_t)gf->nodes[j]->data) >= data_start + length || \ - ((int64_t)gf->nodes[j]->data) + (int64_t) ggml_nbytes(gf->nodes[j]) <= data_start) { - continue; - } - - exe_flag = 0; - } - } - } - if (exe_flag) { - ctx->concur_list[level_pos + concurrency] = i; - nodes_unused[i] = 0; - concurrency++; - ctx->concur_list_len++; - } - } - } - n_left -= concurrency; - // adding a barrier different layer - ctx->concur_list[level_pos + concurrency] = -1; - ctx->concur_list_len++; - // jump all sorted nodes at nodes_bak - while (!nodes_unused[n_start]) { - n_start++; - } - level_pos += concurrency + 1; - } - - if (ctx->concur_list_len > GGML_MAX_CONCUR) { - GGML_METAL_LOG_WARN("%s: too many elements for metal ctx->concur_list!\n", __func__); - } -} - static bool ggml_metal_supports_op(const struct ggml_metal_context * ctx, const struct ggml_tensor * op) { switch (op->op) { case GGML_OP_UNARY: @@ -940,19 +709,15 @@ static bool ggml_metal_supports_op(const struct ggml_metal_context * ctx, const } } -bool ggml_metal_graph_compute( +static bool ggml_metal_graph_compute( struct ggml_metal_context * ctx, struct ggml_cgraph * gf) { @autoreleasepool { - // if there is ctx->concur_list, dispatch concurrently - // else fallback to serial dispatch MTLComputePassDescriptor * edesc = MTLComputePassDescriptor.computePassDescriptor; - const bool has_concur = ctx->concur_list_len && ctx->concur_list_len <= GGML_MAX_CONCUR; - - const int n_nodes = has_concur ? ctx->concur_list_len : gf->n_nodes; - edesc.dispatchType = has_concur ? MTLDispatchTypeConcurrent : MTLDispatchTypeSerial; + const int n_nodes = gf->n_nodes; + edesc.dispatchType = MTLDispatchTypeSerial; // create multiple command buffers and enqueue them // then, we encode the graph into the command buffers in parallel @@ -983,7 +748,7 @@ bool ggml_metal_graph_compute( const int node_end = MIN((cb_idx == n_cb - 1) ? n_nodes : (cb_idx + 1) * n_nodes_per_cb, n_nodes); for (int ind = node_start; ind < node_end; ++ind) { - const int i = has_concur ? ctx->concur_list[ind] : ind; + const int i = ind; if (i == -1) { [encoder memoryBarrierWithScope:MTLBarrierScopeBuffers]; @@ -2823,6 +2588,11 @@ static struct ggml_backend_i ggml_backend_metal_i = { /* .supports_op = */ ggml_backend_metal_supports_op, }; +void ggml_backend_metal_log_set_callback(ggml_log_callback log_callback, void * user_data) { + ggml_metal_log_callback = log_callback; + ggml_metal_log_user_data = user_data; +} + ggml_backend_t ggml_backend_metal_init(void) { struct ggml_metal_context * ctx = ggml_metal_init(GGML_DEFAULT_N_THREADS); @@ -2849,7 +2619,7 @@ void ggml_backend_metal_set_n_cb(ggml_backend_t backend, int n_cb) { struct ggml_metal_context * ctx = (struct ggml_metal_context *)backend->context; - ggml_metal_set_n_cb(ctx, n_cb); + ctx->n_cb = MIN(n_cb, GGML_METAL_MAX_BUFFERS); } bool ggml_backend_metal_supports_family(ggml_backend_t backend, int family) { diff --git a/llama.cpp b/llama.cpp index 2190ea7aa..66494974a 100644 --- a/llama.cpp +++ b/llama.cpp @@ -1266,7 +1266,7 @@ static ggml_backend_buffer_type_t llama_default_buffer_type_split(int fallback_g struct llama_state { llama_state() { #ifdef GGML_USE_METAL - ggml_metal_log_set_callback(log_callback, log_callback_user_data); + ggml_backend_metal_log_set_callback(log_callback, log_callback_user_data); #endif } @@ -10470,7 +10470,7 @@ void llama_log_set(ggml_log_callback log_callback, void * user_data) { g_state.log_callback = log_callback ? log_callback : llama_log_callback_default; g_state.log_callback_user_data = user_data; #ifdef GGML_USE_METAL - ggml_metal_log_set_callback(g_state.log_callback, g_state.log_callback_user_data); + ggml_backend_metal_log_set_callback(g_state.log_callback, g_state.log_callback_user_data); #endif } From c71d608ce7a1584bf5072f197919dd24f3a6163f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20G=C3=A4=C3=9Fler?= Date: Sat, 13 Jan 2024 21:41:37 +0100 Subject: [PATCH 058/812] ggml: cache sin/cos for RoPE (#4908) --- ggml.c | 46 ++++++++++++++++++++++++++++++++-------------- 1 file changed, 32 insertions(+), 14 deletions(-) diff --git a/ggml.c b/ggml.c index de6ef34bd..bcfb6652c 100644 --- a/ggml.c +++ b/ggml.c @@ -11638,6 +11638,21 @@ static float ggml_rope_yarn_corr_dim(int n_dims, int n_orig_ctx, float n_rot, fl return n_dims * logf(n_orig_ctx / (n_rot * 2 * (float)M_PI)) / (2 * logf(base)); } +static void ggml_rope_cache_init( + float theta_base, float freq_scale, float corr_dims[2], int64_t ne0, float ext_factor, float mscale, + float * cache, float sin_sign, float theta_scale +) { + float theta = theta_base; + for (int64_t i0 = 0; i0 < ne0; i0 += 2) { + rope_yarn( + theta, freq_scale, corr_dims, i0, ext_factor, mscale, &cache[i0 + 0], &cache[i0 + 1] + ); + cache[i0 + 1] *= sin_sign; + + theta *= theta_scale; + } +} + void ggml_rope_yarn_corr_dims( int n_dims, int n_orig_ctx, float freq_base, float beta_fast, float beta_slow, float dims[2] ) { @@ -11720,6 +11735,12 @@ static void ggml_compute_forward_rope_f32( for (int64_t i3 = 0; i3 < ne3; i3++) { for (int64_t i2 = 0; i2 < ne2; i2++) { const int64_t p = pos[i2]; + + float * cache = (float *) params->wdata + (ne0 + CACHE_LINE_SIZE_F32)*ith; + if (!is_glm && !is_neox) { // TODO: cache sin/cos for glm, neox + ggml_rope_cache_init(p, freq_scale, corr_dims, ne0, ext_factor, attn_factor, cache, sin_sign, theta_scale); + } + for (int64_t i1 = 0; i1 < ne1; i1++) { if (ir++ < ir0) continue; if (ir > ir1) break; @@ -11753,18 +11774,13 @@ static void ggml_compute_forward_rope_f32( } } else if (!is_neox) { for (int64_t i0 = 0; i0 < ne0; i0 += 2) { - float cos_theta, sin_theta; - rope_yarn( - theta_base, freq_scale, corr_dims, i0, ext_factor, attn_factor, &cos_theta, &sin_theta - ); - sin_theta *= sin_sign; + const float cos_theta = cache[i0 + 0]; + const float sin_theta = cache[i0 + 1]; // zeta scaling for xPos only: float zeta = xpos_base != 0.0f ? powf((i0 + 0.4f * ne0) / (1.4f * ne0), p / xpos_base) : 1.0f; if (xpos_down) zeta = 1.0f / zeta; - theta_base *= theta_scale; - const float * const src = (float *)((char *) src0->data + i3*nb03 + i2*nb02 + i1*nb01 + i0*nb00); float * dst_data = (float *)((char *) dst->data + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0); @@ -11888,6 +11904,12 @@ static void ggml_compute_forward_rope_f16( for (int64_t i3 = 0; i3 < ne3; i3++) { for (int64_t i2 = 0; i2 < ne2; i2++) { const int64_t p = pos[i2]; + + float * cache = (float *) params->wdata + (ne0 + CACHE_LINE_SIZE_F32)*ith; + if (!is_glm && !is_neox) { // TODO: cache sin/cos for glm, neox + ggml_rope_cache_init(p, freq_scale, corr_dims, ne0, ext_factor, attn_factor, cache, sin_sign, theta_scale); + } + for (int64_t i1 = 0; i1 < ne1; i1++) { if (ir++ < ir0) continue; if (ir > ir1) break; @@ -11921,13 +11943,8 @@ static void ggml_compute_forward_rope_f16( } } else if (!is_neox) { for (int64_t i0 = 0; i0 < ne0; i0 += 2) { - float cos_theta, sin_theta; - rope_yarn( - theta_base, freq_scale, corr_dims, i0, ext_factor, attn_factor, &cos_theta, &sin_theta - ); - sin_theta *= sin_sign; - - theta_base *= theta_scale; + const float cos_theta = cache[i0 + 0]; + const float sin_theta = cache[i0 + 1]; const ggml_fp16_t * const src = (ggml_fp16_t *)((char *) src0->data + i3*nb03 + i2*nb02 + i1*nb01 + i0*nb00); ggml_fp16_t * dst_data = (ggml_fp16_t *)((char *) dst->data + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0); @@ -16722,6 +16739,7 @@ struct ggml_cplan ggml_graph_plan(const struct ggml_cgraph * cgraph, int n_threa } } break; case GGML_OP_SOFT_MAX: + case GGML_OP_ROPE: { cur = ggml_type_size(GGML_TYPE_F32) * node->ne[0] * n_tasks; } break; From 76484fbfd355df388f71d6edaa98e1692a74de7e Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Sun, 14 Jan 2024 00:14:46 +0200 Subject: [PATCH 059/812] sync : ggml --- scripts/sync-ggml.last | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/sync-ggml.last b/scripts/sync-ggml.last index edcdb530a..753d227a7 100644 --- a/scripts/sync-ggml.last +++ b/scripts/sync-ggml.last @@ -1 +1 @@ -400c07f00508e6f60fb25405444b5669c365b0a9 +1890780da4ea10db88736fcde85f285abf6c64b0 From 807179ec583dcb882f97d9704577c06beb2c5ec9 Mon Sep 17 00:00:00 2001 From: Kawrakow <48489457+ikawrakow@users.noreply.github.com> Date: Sun, 14 Jan 2024 09:44:30 +0200 Subject: [PATCH 060/812] Make Q3_K_S be the same as olf Q3_K_L for Mixtral-8x7B (#4906) Co-authored-by: Iwan Kawrakow --- llama.cpp | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/llama.cpp b/llama.cpp index 66494974a..8e20e72a2 100644 --- a/llama.cpp +++ b/llama.cpp @@ -8489,9 +8489,16 @@ static ggml_type get_k_quant_type(quantize_state_internal & qs, ggml_type new_ty ++qs.i_feed_forward_w2; } else if (name.find("attn_output.weight") != std::string::npos) { if (arch != LLM_ARCH_FALCON) { - if (ftype == LLAMA_FTYPE_MOSTLY_Q2_K ) new_type = GGML_TYPE_Q3_K; - else if (ftype == LLAMA_FTYPE_MOSTLY_Q3_K_M) new_type = GGML_TYPE_Q4_K; - else if (ftype == LLAMA_FTYPE_MOSTLY_Q3_K_L) new_type = GGML_TYPE_Q5_K; + if (qs.model.hparams.n_expert == 8) { + if (ftype == LLAMA_FTYPE_MOSTLY_Q2_K || ftype == LLAMA_FTYPE_MOSTLY_Q3_K_S || ftype == LLAMA_FTYPE_MOSTLY_Q3_K_M || + ftype == LLAMA_FTYPE_MOSTLY_Q4_K_S || ftype == LLAMA_FTYPE_MOSTLY_Q4_K_M) { + new_type = GGML_TYPE_Q5_K; + } + } else { + if (ftype == LLAMA_FTYPE_MOSTLY_Q2_K ) new_type = GGML_TYPE_Q3_K; + else if (ftype == LLAMA_FTYPE_MOSTLY_Q3_K_M) new_type = GGML_TYPE_Q4_K; + else if (ftype == LLAMA_FTYPE_MOSTLY_Q3_K_L) new_type = GGML_TYPE_Q5_K; + } } else { if (ftype == LLAMA_FTYPE_MOSTLY_Q3_K_L) new_type = GGML_TYPE_Q4_K; } From 147b17ac94a24d524e367cda26a9ff6245689f34 Mon Sep 17 00:00:00 2001 From: Kawrakow <48489457+ikawrakow@users.noreply.github.com> Date: Sun, 14 Jan 2024 09:45:56 +0200 Subject: [PATCH 061/812] 2-bit quantizations (#4897) * imatrix: load * imatrix: WIP * imatrix: Add Q2_K quantization * imatrix: also guard against Q2_K_S quantization without importance matrix * imatrix: guard even more against low-bit quantization misuse --------- Co-authored-by: Iwan Kawrakow --- examples/benchmark/benchmark-matmult.cpp | 4 +- examples/quantize/quantize.cpp | 133 +++- ggml-quants.c | 950 +++++++++++++++++++++-- ggml-quants.h | 12 +- ggml.c | 36 +- ggml.h | 9 +- llama.cpp | 84 +- llama.h | 1 + tests/test-backend-ops.cpp | 2 +- 9 files changed, 1149 insertions(+), 82 deletions(-) diff --git a/examples/benchmark/benchmark-matmult.cpp b/examples/benchmark/benchmark-matmult.cpp index 434e1d6bd..e89f3de2f 100644 --- a/examples/benchmark/benchmark-matmult.cpp +++ b/examples/benchmark/benchmark-matmult.cpp @@ -194,7 +194,7 @@ int main(int argc, char ** argv) { // Set up a the benchmark matrices // printf("Creating new tensor q11 & Running quantize\n"); struct ggml_tensor * q11 = ggml_new_tensor_2d(ctx, qtype, sizex, sizey); - ggml_quantize_chunk(qtype, (const float *) m11->data, q11->data, 0, nelements, hist_cur.data()); + ggml_quantize_chunk(qtype, (const float *) m11->data, q11->data, 0, nelements/m11->ne[0], m11->ne[0], hist_cur.data(), nullptr); // Set up a the compute graph // printf("Creating new tensor q31\n"); @@ -207,7 +207,7 @@ int main(int argc, char ** argv) { // Set up a second graph computation to make sure we override the CPU cache lines // printf("Creating new tensor q12 & Running quantize\n"); struct ggml_tensor * q12 = ggml_new_tensor_2d(ctx, qtype, sizex, sizey); - ggml_quantize_chunk(qtype, (const float *) m12->data, q12->data, 0, nelements, hist_cur.data()); + ggml_quantize_chunk(qtype, (const float *) m12->data, q12->data, 0, nelements/m12->ne[0], m12->ne[0], hist_cur.data(), nullptr); // printf("Creating new tensor q32\n"); struct ggml_tensor * q32 = ggml_mul_mat(ctx, q12, m2); diff --git a/examples/quantize/quantize.cpp b/examples/quantize/quantize.cpp index f878f6911..f4e2175f1 100644 --- a/examples/quantize/quantize.cpp +++ b/examples/quantize/quantize.cpp @@ -5,6 +5,10 @@ #include #include #include +#include +#include +#include +#include struct quant_option { std::string name; @@ -17,6 +21,8 @@ static const std::vector QUANT_OPTIONS = { { "Q4_1", LLAMA_FTYPE_MOSTLY_Q4_1, " 3.90G, +0.1585 ppl @ LLaMA-v1-7B", }, { "Q5_0", LLAMA_FTYPE_MOSTLY_Q5_0, " 4.33G, +0.0683 ppl @ LLaMA-v1-7B", }, { "Q5_1", LLAMA_FTYPE_MOSTLY_Q5_1, " 4.70G, +0.0349 ppl @ LLaMA-v1-7B", }, + { "IQ2_XXS",LLAMA_FTYPE_MOSTLY_IQ2_XXS," 2.06 bpw quantization", }, + { "IQ2_XS", LLAMA_FTYPE_MOSTLY_IQ2_XS, " 2.31 bpw quantization", }, { "Q2_K", LLAMA_FTYPE_MOSTLY_Q2_K, " 2.63G, +0.6717 ppl @ LLaMA-v1-7B", }, { "Q2_K_S", LLAMA_FTYPE_MOSTLY_Q2_K_S, " 2.16G, +9.0634 ppl @ LLaMA-v1-7B", }, { "Q3_K", LLAMA_FTYPE_MOSTLY_Q3_K_M, "alias for Q3_K_M" }, @@ -72,10 +78,14 @@ static bool try_parse_ftype(const std::string & ftype_str_in, llama_ftype & ftyp // [[noreturn]] static void usage(const char * executable) { - printf("usage: %s [--help] [--allow-requantize] [--leave-output-tensor] [--pure] model-f32.gguf [model-quant.gguf] type [nthreads]\n\n", executable); + printf("usage: %s [--help] [--allow-requantize] [--leave-output-tensor] [--pure] [--imatrix] [--include-weights] [--exclude-weights] model-f32.gguf [model-quant.gguf] type [nthreads]\n\n", executable); printf(" --allow-requantize: Allows requantizing tensors that have already been quantized. Warning: This can severely reduce quality compared to quantizing from 16bit or 32bit\n"); printf(" --leave-output-tensor: Will leave output.weight un(re)quantized. Increases model size but may also increase quality, especially when requantizing\n"); printf(" --pure: Disable k-quant mixtures and quantize all tensors to the same type\n"); + printf(" --imatrixfile_name: use data in file_name as importance matrix for quant optimizations\n"); + printf(" --include-weights tensor_name: use importance matrix for this/these tensor(s)\n"); + printf(" --exclude-weights tensor_name: use importance matrix for this/these tensor(s)\n"); + printf("Note: --include-weights and --exclude-weights cannot be used together\n"); printf("\nAllowed quantization types:\n"); for (auto & it : QUANT_OPTIONS) { if (it.name != "COPY") { @@ -83,11 +93,93 @@ static void usage(const char * executable) { } else { printf(" "); } - printf("%-6s : %s\n", it.name.c_str(), it.desc.c_str()); + printf("%-7s : %s\n", it.name.c_str(), it.desc.c_str()); } exit(1); } +static void load_imatrix(const std::string& imatrix_file, std::unordered_map>& imatrix_data) { + std::ifstream in(imatrix_file.c_str(), std::ios::binary); + if (!in) { + printf("%s: failed to open %s\n",__func__,imatrix_file.c_str()); + return; + } + int n_entries; + in.read((char*)&n_entries, sizeof(n_entries)); + if (in.fail() || n_entries < 1) { + printf("%s: no data in file %s\n", __func__, imatrix_file.c_str()); + return; + } + for (int i = 0; i < n_entries; ++i) { + int len; in.read((char *)&len, sizeof(len)); + std::vector name_as_vec(len+1); + in.read((char *)name_as_vec.data(), len); + if (in.fail()) { + printf("%s: failed reading name for entry %d from %s\n",__func__,i+1,imatrix_file.c_str()); + return; + } + name_as_vec[len] = 0; + std::string name{name_as_vec.data()}; + auto& e = imatrix_data[std::move(name)]; + int ncall; + in.read((char*)&ncall, sizeof(ncall)); + int nval; + in.read((char *)&nval, sizeof(nval)); + if (in.fail() || nval < 1) { + printf("%s: failed reading number of values for entry %d\n",__func__,i); + imatrix_data = {}; + return; + } + e.resize(nval); + in.read((char*)e.data(), nval*sizeof(float)); + if (in.fail()) { + printf("%s: failed reading data for entry %d\n",__func__,i); + imatrix_data = {}; + return; + } + if (ncall > 0) { + for (auto& v : e) v /= ncall; + } + } + printf("%s: loaded %d importance matrix entries from %s\n",__func__,int(imatrix_data.size()),imatrix_file.c_str()); +} + +static void prepare_imatrix(const std::string& imatrix_file, + const std::vector& included_weights, + const std::vector& excluded_weights, + std::unordered_map>& imatrix_data) { + if (!imatrix_file.empty()) { + load_imatrix(imatrix_file, imatrix_data); + } + if (imatrix_data.empty()) { + return; + } + if (!excluded_weights.empty()) { + for (auto& name : excluded_weights) { + for (auto it = imatrix_data.begin(); it != imatrix_data.end(); ) { + auto pos = it->first.find(name); + if (pos != std::string::npos) it = imatrix_data.erase(it); + else ++it; + } + } + } + if (!included_weights.empty()) { + std::unordered_map> tmp; + for (auto& name : included_weights) { + for (auto& e : imatrix_data) { + auto pos = e.first.find(name); + if (pos != std::string::npos) { + tmp.emplace(std::move(e)); + } + } + } + imatrix_data = std::move(tmp); + } + if (!imatrix_data.empty()) { + printf("%s: have %d importance matrix entries\n", __func__, int(imatrix_data.size())); + } +} + int main(int argc, char ** argv) { if (argc < 3) { usage(argv[0]); @@ -96,6 +188,8 @@ int main(int argc, char ** argv) { llama_model_quantize_params params = llama_model_quantize_default_params(); int arg_idx = 1; + std::string imatrix_file; + std::vector included_weights, excluded_weights; for (; arg_idx < argc && strncmp(argv[arg_idx], "--", 2) == 0; arg_idx++) { if (strcmp(argv[arg_idx], "--leave-output-tensor") == 0) { @@ -104,14 +198,42 @@ int main(int argc, char ** argv) { params.allow_requantize = true; } else if (strcmp(argv[arg_idx], "--pure") == 0) { params.pure = true; + } else if (strcmp(argv[arg_idx], "--imatrix") == 0) { + if (arg_idx < argc-1) { + imatrix_file = argv[++arg_idx]; + } else { + usage(argv[0]); + } + } else if (strcmp(argv[arg_idx], "--include-weights") == 0) { + if (arg_idx < argc-1) { + included_weights.push_back(argv[++arg_idx]); + } else { + usage(argv[0]); + } + } else if (strcmp(argv[arg_idx], "--exclude-weights") == 0) { + if (arg_idx < argc-1) { + excluded_weights.push_back(argv[++arg_idx]); + } else { + usage(argv[0]); + } } else { usage(argv[0]); } } if (argc - arg_idx < 2) { + printf("%s: bad arguments\n", argv[0]); usage(argv[0]); } + if (!included_weights.empty() && !excluded_weights.empty()) { + usage(argv[0]); + } + + std::unordered_map> imatrix_data; + prepare_imatrix(imatrix_file, included_weights, excluded_weights, imatrix_data); + if (!imatrix_data.empty()) { + params.imatrix = &imatrix_data; + } llama_backend_init(false); @@ -163,6 +285,13 @@ int main(int argc, char ** argv) { } } + if ((params.ftype == LLAMA_FTYPE_MOSTLY_IQ2_XS || params.ftype == LLAMA_FTYPE_MOSTLY_IQ2_XXS || params.ftype == LLAMA_FTYPE_MOSTLY_Q2_K_S) && imatrix_data.empty()) { + fprintf(stderr, "\n===============================================================================================\n"); + fprintf(stderr, "Please do not use IQ2_XXS, IQ2_XS or Q2_K_S quantization without an importance matrix\n"); + fprintf(stderr, "===============================================================================================\n\n\n"); + return 1; + } + print_build_info(); fprintf(stderr, "%s: quantizing '%s' to '%s' as %s", __func__, fname_inp.c_str(), fname_out.c_str(), ftype_str.c_str()); diff --git a/ggml-quants.c b/ggml-quants.c index 601d155d7..9290d54cf 100644 --- a/ggml-quants.c +++ b/ggml-quants.c @@ -5,6 +5,8 @@ #include #include #include +#include // for qsort +#include // for GGML_ASSERT #ifdef __ARM_NEON @@ -1639,6 +1641,241 @@ size_t ggml_quantize_q2_K(const float * restrict src, void * restrict dst, int n return (n/QK_K*sizeof(block_q2_K)); } +static float make_qkx3_quants(int n, int nmax, const float * restrict x, const float * restrict weights, + uint8_t * restrict L, float * restrict the_min, uint8_t * restrict Laux, + float rmin, float rdelta, int nstep, bool use_mad) { + float min = x[0]; + float max = x[0]; + float sum_w = weights ? weights[0] : x[0]*x[0]; + float sum_x = sum_w * x[0]; + for (int i = 1; i < n; ++i) { + if (x[i] < min) min = x[i]; + if (x[i] > max) max = x[i]; + float w = weights ? weights[i] : x[i]*x[i]; + sum_w += w; + sum_x += w * x[i]; + } + if (min > 0) { + min = 0; + } + if (max <= min) { + for (int i = 0; i < n; ++i) L[i] = 0; + *the_min = -min; + return 0.f; + } + float iscale = nmax/(max - min); + float scale = 1/iscale; + float best_mad = 0; + for (int i = 0; i < n; ++i) { + int l = nearest_int(iscale*(x[i] - min)); + L[i] = MAX(0, MIN(nmax, l)); + float diff = scale * L[i] + min - x[i]; + diff = use_mad ? fabsf(diff) : diff*diff; + float w = weights ? weights[i] : x[i]*x[i]; + best_mad += w * diff; + } + if (nstep < 1) { + *the_min = -min; + return scale; + } + for (int is = 0; is <= nstep; ++is) { + iscale = (rmin + rdelta*is + nmax)/(max - min); + float sum_l = 0, sum_l2 = 0, sum_xl = 0; + for (int i = 0; i < n; ++i) { + int l = nearest_int(iscale*(x[i] - min)); + l = MAX(0, MIN(nmax, l)); + Laux[i] = l; + float w = weights ? weights[i] : x[i]*x[i]; + sum_l += w*l; + sum_l2 += w*l*l; + sum_xl += w*l*x[i]; + } + float D = sum_w * sum_l2 - sum_l * sum_l; + if (D > 0) { + float this_scale = (sum_w * sum_xl - sum_x * sum_l)/D; + float this_min = (sum_l2 * sum_x - sum_l * sum_xl)/D; + if (this_min > 0) { + this_min = 0; + this_scale = sum_xl / sum_l2; + } + float mad = 0; + for (int i = 0; i < n; ++i) { + float diff = this_scale * Laux[i] + this_min - x[i]; + diff = use_mad ? fabsf(diff) : diff*diff; + float w = weights ? weights[i] : x[i]*x[i]; + mad += w * diff; + } + if (mad < best_mad) { + for (int i = 0; i < n; ++i) { + L[i] = Laux[i]; + } + best_mad = mad; + scale = this_scale; + min = this_min; + } + } + } + *the_min = -min; + return scale; +} + +static float make_qp_quants(int n, int nmax, const float * restrict x, uint8_t * restrict L, const float * quant_weights) { + float max = 0; + for (int i = 0; i < n; ++i) { + max = MAX(max, x[i]); + } + if (!max) { // all zero + for (int i = 0; i < n; ++i) { L[i] = 0; } + return 0.f; + } + float iscale = nmax / max; + for (int i = 0; i < n; ++i) { + L[i] = nearest_int(iscale * x[i]); + } + float scale = 1/iscale; + float best_mse = 0; + for (int i = 0; i < n; ++i) { + float diff = x[i] - scale*L[i]; + float w = quant_weights[i]; + best_mse += w*diff*diff; + } + for (int is = -4; is <= 4; ++is) { + if (is == 0) continue; + float iscale_is = (0.1f*is + nmax)/max; + float scale_is = 1/iscale_is; + float mse = 0; + for (int i = 0; i < n; ++i) { + int l = nearest_int(iscale_is*x[i]); + l = MIN(nmax, l); + float diff = x[i] - scale_is*l; + float w = quant_weights[i]; + mse += w*diff*diff; + } + if (mse < best_mse) { + best_mse = mse; + iscale = iscale_is; + } + } + float sumlx = 0; + float suml2 = 0; + for (int i = 0; i < n; ++i) { + int l = nearest_int(iscale * x[i]); + l = MIN(nmax, l); + L[i] = l; + float w = quant_weights[i]; + sumlx += w*x[i]*l; + suml2 += w*l*l; + } + for (int itry = 0; itry < 5; ++itry) { + int n_changed = 0; + for (int i = 0; i < n; ++i) { + float w = quant_weights[i]; + float slx = sumlx - w*x[i]*L[i]; + float sl2 = suml2 - w*L[i]*L[i]; + if (slx > 0 && sl2 > 0) { + int new_l = nearest_int(x[i] * sl2 / slx); + new_l = MIN(nmax, new_l); + if (new_l != L[i]) { + slx += w*x[i]*new_l; + sl2 += w*new_l*new_l; + if (slx*slx*suml2 > sumlx*sumlx*sl2) { + L[i] = new_l; sumlx = slx; suml2 = sl2; + ++n_changed; + } + } + } + } + if (!n_changed) { + break; + } + } + return sumlx / suml2; +} + +static void quantize_row_q2_K_impl(const float * restrict x, block_q2_K * restrict y, int k, const float * restrict quant_weights) { + GGML_ASSERT(quant_weights); + assert(k % QK_K == 0); + const int nb = k / QK_K; + const bool requantize = true; + + uint8_t L[QK_K]; + uint8_t Laux[16]; + float mins[QK_K/16]; + float scales[QK_K/16]; + float sw[QK_K/16]; + float weight[QK_K/16]; + uint8_t Ls[QK_K/16], Lm[QK_K/16]; + + for (int i = 0; i < nb; i++) { + memset(sw, 0, QK_K/16*sizeof(float)); + float sumx2 = 0; + for (int j = 0; j < QK_K; ++j) sumx2 += x[j]*x[j]; + float sigma2 = sumx2/QK_K; + for (int j = 0; j < QK_K/16; ++j) { + const float * restrict qw = quant_weights + QK_K * i + 16*j; + for (int l = 0; l < 16; ++l) weight[l] = qw[l] * sqrtf(sigma2 + x[16*j + l]*x[16*j + l]); + for (int l = 0; l < 16; ++l) sw[j] += weight[l]; + scales[j] = make_qkx3_quants(16, 3, x + 16*j, weight, L + 16*j, &mins[j], Laux, -0.9f, 0.05f, 36, false); + } + + float dm = make_qp_quants(QK_K/16, 15, scales, Ls, sw); + float mm = make_qp_quants(QK_K/16, 15, mins, Lm, sw); + y[i].d = GGML_FP32_TO_FP16(dm); + y[i].dmin = GGML_FP32_TO_FP16(mm); + dm = GGML_FP16_TO_FP32(y[i].d); + mm = GGML_FP16_TO_FP32(y[i].dmin); + + for (int j = 0; j < QK_K/16; ++j) { + y[i].scales[j] = Ls[j] | (Lm[j] << 4); + } + + if (requantize) { + for (int j = 0; j < QK_K/16; ++j) { + const float d = dm * (y[i].scales[j] & 0xF); + if (!d) continue; + const float m = mm * (y[i].scales[j] >> 4); + for (int ii = 0; ii < 16; ++ii) { + int l = nearest_int((x[16*j + ii] + m)/d); + l = MAX(0, MIN(3, l)); + L[16*j + ii] = l; + } + } + } + +#if QK_K == 256 + for (int j = 0; j < QK_K; j += 128) { + for (int l = 0; l < 32; ++l) { + y[i].qs[j/4 + l] = L[j + l] | (L[j + l + 32] << 2) | (L[j + l + 64] << 4) | (L[j + l + 96] << 6); + } + } +#else + for (int l = 0; l < 16; ++l) { + y[i].qs[l] = L[l] | (L[l + 16] << 2) | (L[l + 32] << 4) | (L[l + 48] << 6); + } +#endif + + x += QK_K; + + } +} + +size_t quantize_q2_K(const float * src, void * dst, int nrow, int n_per_row, int64_t * hist, const float * quant_weights) { + (void)hist; + int row_size = ggml_row_size(GGML_TYPE_Q2_K, n_per_row); + if (!quant_weights) { + quantize_row_q2_K_reference(src, dst, nrow*n_per_row); + } + else { + char * qrow = (char *)dst; + for (int row = 0; row < nrow; ++row) { + quantize_row_q2_K_impl(src, (block_q2_K*)qrow, n_per_row, quant_weights); + src += n_per_row; + qrow += row_size; + } + } + return nrow * row_size; +} + //========================= 3-bit (de)-quantization void quantize_row_q3_K_reference(const float * restrict x, block_q3_K * restrict y, int k) { @@ -2584,14 +2821,6 @@ static const uint8_t ksigns_iq2xs[128] = { static const uint8_t kmask_iq2xs[8] = {1, 2, 4, 8, 16, 32, 64, 128}; -void quantize_row_iq2_xxs_reference(const float * restrict x, block_iq2_xxs * restrict y, int k) { - (void)x; - (void)y; - (void)k; - assert(k % QK_K == 0); - //fprintf(stderr, "=========================== %s: not implemented\n", __func__); -} - void dequantize_row_iq2_xxs(const block_iq2_xxs * restrict x, float * restrict y, int k) { assert(k % QK_K == 0); const int nb = k / QK_K; @@ -2618,33 +2847,8 @@ void dequantize_row_iq2_xxs(const block_iq2_xxs * restrict x, float * restrict y } } -void quantize_row_iq2_xxs(const float * restrict x, void * restrict vy, int k) { - assert(k % QK_K == 0); - block_iq2_xxs * restrict y = vy; - quantize_row_iq2_xxs_reference(x, y, k); -} - -size_t ggml_quantize_iq2_xxs(const float * src, void * dst, int n, int k, int64_t * hist) { - assert(k % QK_K == 0); - (void)hist; // TODO: collect histograms - - for (int j = 0; j < n; j += k) { - block_iq2_xxs * restrict y = (block_iq2_xxs *)dst + j/QK_K; - quantize_row_iq2_xxs_reference(src + j, y, k); - } - return (n/QK_K*sizeof(block_iq2_xxs)); -} - // ====================== 2.3125 bpw (de)-quantization -void quantize_row_iq2_xs_reference(const float * restrict x, block_iq2_xs * restrict y, int k) { - (void)x; - (void)y; - (void)k; - assert(k % QK_K == 0); - //fprintf(stderr, "=========================== %s: not implemented\n", __func__); -} - void dequantize_row_iq2_xs(const block_iq2_xs * restrict x, float * restrict y, int k) { assert(k % QK_K == 0); const int nb = k / QK_K; @@ -2670,23 +2874,6 @@ void dequantize_row_iq2_xs(const block_iq2_xs * restrict x, float * restrict y, } } -void quantize_row_iq2_xs(const float * restrict x, void * restrict vy, int k) { - assert(k % QK_K == 0); - block_iq2_xs * restrict y = vy; - quantize_row_iq2_xs_reference(x, y, k); -} - -size_t ggml_quantize_iq2_xs(const float * src, void * dst, int n, int k, int64_t * hist) { - assert(k % QK_K == 0); - (void)hist; // TODO: collect histograms - - for (int j = 0; j < n; j += k) { - block_iq2_xs * restrict y = (block_iq2_xs *)dst + j/QK_K; - quantize_row_iq2_xs_reference(src + j, y, k); - } - return (n/QK_K*sizeof(block_iq2_xs)); -} - //===================================== Q8_K ============================================== void quantize_row_q8_K_reference(const float * restrict x, block_q8_K * restrict y, int k) { @@ -7730,3 +7917,666 @@ void ggml_vec_dot_iq2_xs_q8_K(const int n, float * restrict s, const void * rest *s = 0.125f * sumf; #endif } + +// ================================ IQ2 quantization ============================================= + +typedef struct { + uint64_t * grid; + int * map; + uint16_t * neighbours; +} iq2_entry_t; + +static iq2_entry_t iq2_data[2] = { + {NULL, NULL, NULL}, + {NULL, NULL, NULL}, +}; + +static inline int iq2_data_index(int grid_size) { + GGML_ASSERT(grid_size == 256 || grid_size == 512); + return grid_size == 256 ? 0 : 1; +} + +static int iq2_compare_func(const void * left, const void * right) { + const int * l = (const int *)left; + const int * r = (const int *)right; + return l[0] < r[0] ? -1 : l[0] > r[0] ? 1 : l[1] < r[1] ? -1 : l[1] > r[1] ? 1 : 0; +} + +static void q2xs_init_impl(int grid_size) { + const int gindex = iq2_data_index(grid_size); + if (iq2_data[gindex].grid) { + return; + } + static const uint16_t kgrid_256[256] = { + 0, 2, 5, 8, 10, 17, 20, 32, 34, 40, 42, 65, 68, 80, 88, 97, + 100, 128, 130, 138, 162, 257, 260, 272, 277, 320, 388, 408, 512, 514, 546, 642, + 1025, 1028, 1040, 1057, 1060, 1088, 1090, 1096, 1120, 1153, 1156, 1168, 1188, 1280, 1282, 1288, + 1312, 1350, 1385, 1408, 1425, 1545, 1552, 1600, 1668, 1700, 2048, 2053, 2056, 2068, 2088, 2113, + 2116, 2128, 2130, 2184, 2308, 2368, 2562, 2580, 4097, 4100, 4112, 4129, 4160, 4192, 4228, 4240, + 4245, 4352, 4360, 4384, 4432, 4442, 4480, 4644, 4677, 5120, 5128, 5152, 5157, 5193, 5248, 5400, + 5474, 5632, 5654, 6145, 6148, 6160, 6208, 6273, 6400, 6405, 6560, 6737, 8192, 8194, 8202, 8260, + 8289, 8320, 8322, 8489, 8520, 8704, 8706, 9217, 9220, 9232, 9280, 9302, 9472, 9537, 9572, 9872, + 10248, 10272, 10388, 10820, 16385, 16388, 16400, 16408, 16417, 16420, 16448, 16456, 16470, 16480, 16513, 16516, + 16528, 16640, 16672, 16737, 16768, 16773, 16897, 16912, 16968, 16982, 17000, 17408, 17416, 17440, 17536, 17561, + 17682, 17700, 17920, 18433, 18436, 18448, 18496, 18501, 18688, 18776, 18785, 18818, 19013, 19088, 20480, 20488, + 20497, 20505, 20512, 20608, 20616, 20740, 20802, 20900, 21137, 21648, 21650, 21770, 22017, 22100, 22528, 22545, + 22553, 22628, 22848, 23048, 24580, 24592, 24640, 24680, 24832, 24917, 25112, 25184, 25600, 25605, 25872, 25874, + 25988, 26690, 32768, 32770, 32778, 32833, 32898, 33028, 33048, 33088, 33297, 33793, 33796, 33808, 33813, 33856, + 33888, 34048, 34118, 34196, 34313, 34368, 34400, 34818, 35076, 35345, 36868, 36880, 36900, 36928, 37025, 37142, + 37248, 37445, 37888, 37922, 37956, 38225, 39041, 39200, 40962, 41040, 41093, 41225, 41472, 42008, 43088, 43268, + }; + static const uint16_t kgrid_512[512] = { + 0, 2, 5, 8, 10, 17, 20, 22, 25, 32, 34, 37, 40, 65, 68, 70, + 73, 80, 82, 85, 88, 97, 100, 128, 130, 133, 136, 145, 148, 153, 160, 257, + 260, 262, 265, 272, 274, 277, 280, 282, 289, 292, 320, 322, 325, 328, 337, 340, + 352, 360, 385, 388, 400, 512, 514, 517, 520, 529, 532, 544, 577, 580, 592, 597, + 640, 650, 1025, 1028, 1030, 1033, 1040, 1042, 1045, 1048, 1057, 1060, 1088, 1090, 1093, 1096, + 1105, 1108, 1110, 1120, 1153, 1156, 1168, 1280, 1282, 1285, 1288, 1297, 1300, 1312, 1345, 1348, + 1360, 1377, 1408, 1537, 1540, 1552, 1574, 1600, 1602, 1668, 2048, 2050, 2053, 2056, 2058, 2065, + 2068, 2080, 2085, 2113, 2116, 2128, 2136, 2176, 2208, 2218, 2305, 2308, 2320, 2368, 2433, 2441, + 2560, 2592, 2600, 2710, 2720, 4097, 4100, 4102, 4105, 4112, 4114, 4117, 4120, 4129, 4132, 4160, + 4162, 4165, 4168, 4177, 4180, 4192, 4202, 4225, 4228, 4240, 4352, 4354, 4357, 4360, 4369, 4372, + 4384, 4417, 4420, 4432, 4480, 4500, 4502, 4609, 4612, 4614, 4624, 4672, 4704, 5120, 5122, 5125, + 5128, 5137, 5140, 5152, 5185, 5188, 5193, 5200, 5220, 5248, 5377, 5380, 5392, 5440, 5632, 5652, + 5705, 6145, 6148, 6160, 6162, 6208, 6228, 6278, 6400, 6405, 6502, 6737, 6825, 8192, 8194, 8197, + 8200, 8202, 8209, 8212, 8224, 8257, 8260, 8272, 8320, 8352, 8449, 8452, 8464, 8512, 8520, 8549, + 8704, 8738, 8832, 8872, 9217, 9220, 9232, 9257, 9280, 9472, 9537, 9554, 9625, 9729, 9754, 9894, + 10240, 10248, 10250, 10272, 10325, 10376, 10402, 10600, 10640, 10760, 10784, 10882, 10888, 10890, 16385, 16388, + 16390, 16393, 16400, 16402, 16405, 16408, 16417, 16420, 16448, 16450, 16453, 16456, 16458, 16465, 16468, 16480, + 16485, 16513, 16516, 16528, 16640, 16642, 16645, 16648, 16657, 16660, 16672, 16705, 16708, 16720, 16768, 16773, + 16802, 16897, 16900, 16912, 16914, 16937, 16960, 17408, 17410, 17413, 17416, 17425, 17428, 17433, 17440, 17473, + 17476, 17488, 17536, 17556, 17665, 17668, 17680, 17700, 17728, 17818, 17920, 17930, 17988, 18000, 18433, 18436, + 18448, 18496, 18501, 18516, 18530, 18688, 18705, 18756, 18768, 18793, 18948, 20480, 20482, 20485, 20488, 20497, + 20500, 20512, 20520, 20545, 20548, 20560, 20608, 20737, 20740, 20752, 20757, 20800, 20802, 20992, 21060, 21162, + 21505, 21508, 21520, 21537, 21568, 21600, 21633, 21665, 21760, 21768, 21888, 21896, 22049, 22120, 22177, 22528, + 22548, 22593, 22608, 22681, 22810, 22848, 22850, 23173, 24577, 24580, 24592, 24640, 24660, 24674, 24710, 24745, + 24832, 25124, 25162, 25234, 25600, 25622, 25872, 25920, 25925, 26020, 26625, 26730, 26917, 27142, 27220, 27234, + 32768, 32770, 32773, 32776, 32785, 32788, 32800, 32810, 32833, 32836, 32848, 32896, 32898, 32936, 32938, 33025, + 33028, 33030, 33040, 33088, 33105, 33113, 33280, 33312, 33408, 33410, 33440, 33448, 33793, 33796, 33808, 33810, + 33813, 33856, 33888, 33929, 34048, 34116, 34213, 34328, 34410, 34816, 34824, 34853, 34906, 34944, 34946, 34984, + 35078, 35362, 35456, 35464, 35478, 35496, 36865, 36868, 36880, 36928, 36950, 36996, 37120, 37154, 37220, 37462, + 37513, 37888, 37893, 37956, 37968, 37976, 38185, 38288, 38290, 38465, 38993, 39078, 39241, 39445, 39520, 40960, + 40962, 40968, 40970, 40992, 41002, 41120, 41297, 41305, 41382, 41472, 41474, 41480, 41514, 41600, 41632, 42048, + 42133, 42597, 42648, 43018, 43040, 43042, 43048, 43168, 43176, 43268, 43396, 43398, 43560, 43562, 43665, 43690, + }; + const int kmap_size = 43692; + const int nwant = 2; + const uint16_t * kgrid = grid_size == 256 ? kgrid_256 : kgrid_512; + uint64_t * kgrid_q2xs; + int * kmap_q2xs; + uint16_t * kneighbors_q2xs; + + printf("================================================================= %s(grid_size = %d)\n", __func__, grid_size); + uint64_t * the_grid = (uint64_t *)malloc(grid_size*sizeof(uint64_t)); + for (int k = 0; k < grid_size; ++k) { + int8_t * pos = (int8_t *)(the_grid + k); + for (int i = 0; i < 8; ++i) { + int l = (kgrid[k] >> 2*i) & 0x3; + pos[i] = 2*l + 1; + } + } + kgrid_q2xs = the_grid; + iq2_data[gindex].grid = the_grid; + kmap_q2xs = (int *)malloc(kmap_size*sizeof(int)); + iq2_data[gindex].map = kmap_q2xs; + for (int i = 0; i < kmap_size; ++i) kmap_q2xs[i] = -1; + uint64_t aux64; + uint8_t * aux8 = (uint8_t *)&aux64; + for (int i = 0; i < grid_size; ++i) { + aux64 = kgrid_q2xs[i]; + uint16_t index = 0; + for (int k=0; k<8; ++k) { + uint16_t q = (aux8[k] - 1)/2; + index |= (q << 2*k); + } + kmap_q2xs[index] = i; + } + int8_t pos[8]; + int * dist2 = (int *)malloc(2*grid_size*sizeof(int)); + int num_neighbors = 0, num_not_in_map = 0; + for (int i = 0; i < kmap_size; ++i) { + if (kmap_q2xs[i] >= 0) continue; + ++num_not_in_map; + for (int k = 0; k < 8; ++k) { + int l = (i >> 2*k) & 0x3; + pos[k] = 2*l + 1; + } + for (int j = 0; j < grid_size; ++j) { + const int8_t * pg = (const int8_t *)(kgrid_q2xs + j); + int d2 = 0; + for (int k = 0; k < 8; ++k) d2 += (pg[k] - pos[k])*(pg[k] - pos[k]); + dist2[2*j+0] = d2; + dist2[2*j+1] = j; + } + qsort(dist2, grid_size, 2*sizeof(int), iq2_compare_func); + int n = 0; int d2 = dist2[0]; + int nhave = 1; + for (int j = 0; j < grid_size; ++j) { + if (dist2[2*j] > d2) { + if (nhave == nwant) break; + d2 = dist2[2*j]; + ++nhave; + } + ++n; + } + num_neighbors += n; + } + printf("%s: %d neighbours in total\n", __func__, num_neighbors); + kneighbors_q2xs = (uint16_t *)malloc((num_neighbors + num_not_in_map)*sizeof(uint16_t)); + iq2_data[gindex].neighbours = kneighbors_q2xs; + int counter = 0; + for (int i = 0; i < kmap_size; ++i) { + if (kmap_q2xs[i] >= 0) continue; + for (int k = 0; k < 8; ++k) { + int l = (i >> 2*k) & 0x3; + pos[k] = 2*l + 1; + } + for (int j = 0; j < grid_size; ++j) { + const int8_t * pg = (const int8_t *)(kgrid_q2xs + j); + int d2 = 0; + for (int k = 0; k < 8; ++k) d2 += (pg[k] - pos[k])*(pg[k] - pos[k]); + dist2[2*j+0] = d2; + dist2[2*j+1] = j; + } + qsort(dist2, grid_size, 2*sizeof(int), iq2_compare_func); + kmap_q2xs[i] = -(counter + 1); + int d2 = dist2[0]; + uint16_t * start = &kneighbors_q2xs[counter++]; + int n = 0, nhave = 1; + for (int j = 0; j < grid_size; ++j) { + if (dist2[2*j] > d2) { + if (nhave == nwant) break; + d2 = dist2[2*j]; + ++nhave; + } + kneighbors_q2xs[counter++] = dist2[2*j+1]; + ++n; + } + *start = n; + } + free(dist2); +} + +void ggml_init_iq2_quantization(enum ggml_type type) { + if (type == GGML_TYPE_IQ2_XXS) { + q2xs_init_impl(256); + } + else if (type == GGML_TYPE_IQ2_XS) { + q2xs_init_impl(512); + } + else { + fprintf(stderr, "======================== Why are you calling %s with type %d?\n", __func__, (int)type); + } +} + +static void q2xs_deinit_impl(int grid_size) { + GGML_ASSERT(grid_size == 256 || grid_size == 512 || grid_size == 1024); + const int gindex = iq2_data_index(grid_size); + if (iq2_data[gindex].grid) { + free(iq2_data[gindex].grid); iq2_data[gindex].grid = NULL; + free(iq2_data[gindex].map); iq2_data[gindex].map = NULL; + free(iq2_data[gindex].neighbours); iq2_data[gindex].neighbours = NULL; + } +} + +void ggml_deinit_iq2_quantization(enum ggml_type type) { + if (type == GGML_TYPE_IQ2_XXS) { + q2xs_deinit_impl(256); + } + else if (type == GGML_TYPE_IQ2_XS) { + q2xs_deinit_impl(512); + } + else { + fprintf(stderr, "======================== Why are you calling %s with type %d?\n", __func__, (int)type); + } +} + +static int iq2_find_best_neighbour(const uint16_t * restrict neighbours, const uint64_t * restrict grid, + const float * restrict xval, const float * restrict weight, float scale, int8_t * restrict L) { + int num_neighbors = neighbours[0]; + GGML_ASSERT(num_neighbors > 0); + float best_d2 = FLT_MAX; + int grid_index = -1; + for (int j = 1; j <= num_neighbors; ++j) { + const int8_t * pg = (const int8_t *)(grid + neighbours[j]); + float d2 = 0; + for (int i = 0; i < 8; ++i) { + float q = pg[i]; + float diff = scale*q - xval[i]; + d2 += weight[i]*diff*diff; + } + if (d2 < best_d2) { + best_d2 = d2; grid_index = neighbours[j]; + } + } + GGML_ASSERT(grid_index >= 0); + const int8_t * pg = (const int8_t *)(grid + grid_index); + for (int i = 0; i < 8; ++i) L[i] = (pg[i] - 1)/2; + return grid_index; +} + +static void quantize_row_iq2_xxs_impl(const float * restrict x, void * restrict vy, int n, const float * restrict quant_weights) { + + const int gindex = iq2_data_index(256); + + const uint64_t * kgrid_q2xs = iq2_data[gindex].grid; + const int * kmap_q2xs = iq2_data[gindex].map; + const uint16_t * kneighbors_q2xs = iq2_data[gindex].neighbours; + + GGML_ASSERT(quant_weights); + GGML_ASSERT(kgrid_q2xs); + GGML_ASSERT(kmap_q2xs); + GGML_ASSERT(kneighbors_q2xs); + GGML_ASSERT(n%QK_K == 0); + + const int kMaxQ = 3; + + const int nbl = n/256; + + block_iq2_xxs * y = vy; + + float scales[QK_K/32]; + float weight[32]; + float xval[32]; + int8_t L[32]; + int8_t Laux[32]; + float waux[32]; + bool is_on_grid[4]; + bool is_on_grid_aux[4]; + uint8_t block_signs[4]; + uint32_t q2[2*(QK_K/32)]; + + for (int ibl = 0; ibl < nbl; ++ibl) { + + y[ibl].d = GGML_FP32_TO_FP16(0.f); + memset(q2, 0, QK_K/4); + + float max_scale = 0; + + const float * xbl = x + QK_K*ibl; + float sumx2 = 0; + for (int i = 0; i < QK_K; ++i) sumx2 += xbl[i]*xbl[i]; + float sigma2 = sumx2/QK_K; + + for (int ib = 0; ib < QK_K/32; ++ib) { + const float * xb = xbl + 32*ib; + const float * qw = quant_weights + QK_K*ibl + 32*ib; + for (int i = 0; i < 32; ++i) weight[i] = qw[i] * sqrtf(sigma2 + xb[i]*xb[i]); + for (int i = 0; i < 32; ++i) waux[i] = sqrtf(weight[i]); + for (int k = 0; k < 4; ++k) { + int nflip = 0; + uint8_t s = 0; + for (int i = 0; i < 8; ++i) { + if (xb[8*k + i] >= 0) xval[8*k + i] = xb[8*k + i]; + else { + xval[8*k + i] = -xb[8*k + i]; ++nflip; s |= (1 << i); + } + } + if (nflip%2) { + int imin = 0; float min = weight[8*k+imin]*xb[8*k+imin]*xb[8*k+imin]; + for (int i = 1; i < 8; ++i) { + float ax = weight[8*k+i]*xb[8*k+i]*xb[8*k+i]; + if (ax < min) { + min = ax; imin = i; + } + } + xval[8*k+imin] = -xval[8*k+imin]; + s ^= (1 << imin); + } + block_signs[k] = s & 127; + } + float max = xval[0]; + for (int i = 1; i < 32; ++i) max = MAX(max, xval[i]); + if (!max) { + scales[ib] = 0; + memset(L, 0, 32); + continue; + } + float best = 0; + float scale = max/(2*kMaxQ-1); + for (int is = -9; is <= 9; ++is) { + float id = (2*kMaxQ-1+is*0.1f)/max; + float this_scale = 1/id; + for (int k = 0; k < 4; ++k) { + for (int i = 0; i < 8; ++i) { + int l = nearest_int(0.5f*(id*xval[8*k+i]-1)); + Laux[8*k+i] = MAX(0, MIN(kMaxQ-1, l)); + } + uint16_t u = 0; + for (int i = 0; i < 8; ++i) u |= (Laux[8*k+i] << 2*i); + int grid_index = kmap_q2xs[u]; + is_on_grid_aux[k] = true; + if (grid_index < 0) { + is_on_grid_aux[k] = false; + const uint16_t * neighbours = kneighbors_q2xs - kmap_q2xs[u] - 1; + grid_index = iq2_find_best_neighbour(neighbours, kgrid_q2xs, xval + 8*k, waux + 8*k, this_scale, Laux + 8*k); + } + } + float sumqx = 0, sumq2 = 0; + for (int i = 0; i < 32; ++i) { + float w = weight[i]; + float q = 2*Laux[i] + 1; + sumqx += w*xval[i]*q; + sumq2 += w*q*q; + } + if (sumq2 > 0 && sumqx*sumqx > best*sumq2) { + scale = sumqx/sumq2; best = scale*sumqx; + for (int i = 0; i < 32; ++i) L[i] = Laux[i]; + for (int k = 0; k < 4; ++k) is_on_grid[k] = is_on_grid_aux[k]; + } + } + int n_not_ongrid = 0; + for (int k = 0; k < 4; ++k) if (!is_on_grid[k]) ++n_not_ongrid; + if (n_not_ongrid > 0 && scale > 0) { + float id = 1/scale; + for (int k = 0; k < 4; ++k) { + if (is_on_grid[k]) continue; + uint16_t u = 0; + for (int i = 0; i < 8; ++i) { + int l = nearest_int(0.5f*(id*xval[8*k+i]-1)); + l = MAX(0, MIN(kMaxQ-1, l)); + u |= (l << 2*i); + } + int grid_index = kmap_q2xs[u]; + if (grid_index < 0) { + const uint16_t * neighbours = kneighbors_q2xs - kmap_q2xs[u] - 1; + grid_index = iq2_find_best_neighbour(neighbours, kgrid_q2xs, xval + 8*k, waux + 8*k, scale, L + 8*k); + } + const int8_t * pg = (const int8_t *)(kgrid_q2xs + grid_index); + for (int i = 0; i < 8; ++i) L[8*k+i] = (pg[i] - 1)/2; + } + float sumqx = 0, sumq2 = 0; + for (int i = 0; i < 32; ++i) { + float w = weight[i]; + float q = 2*L[i] + 1; + sumqx += w*xval[i]*q; + sumq2 += w*q*q; + } + if (sumq2 > 0) scale = sumqx/sumq2; + } + if (scale < 0) { + // This should never happen, but just in case, flip scale so that it is positive (we use uint's to encode the scale) + // and correspondingly flip quant signs. + scale = -scale; + for (int k = 0; k < 4; ++k) block_signs[k] = (~block_signs[k]) & 127; + } + for (int k = 0; k < 4; ++k) { + uint16_t u = 0; + for (int i = 0; i < 8; ++i) u |= (L[8*k+i] << 2*i); + int grid_index = kmap_q2xs[u]; + if (grid_index < 0) { + printf("Oops: found point %u not on grid:", u); + for (int i = 0; i < 8; ++i) printf(" %d", L[8*k+i]); + printf("\n"); + GGML_ASSERT(false); + } + q2[2*ib+0] |= (grid_index << 8*k); + q2[2*ib+1] |= (block_signs[k] << 7*k); + } + GGML_ASSERT(scale >= 0); + scales[ib] = scale; + max_scale = MAX(max_scale, scale); + } + + if (!max_scale) { + memset(y[ibl].qs, 0, QK_K/4); + continue; + } + + float d = max_scale/31; + y[ibl].d = GGML_FP32_TO_FP16(d); + float id = 1/d; + float sumqx = 0, sumq2 = 0; + for (int ib = 0; ib < QK_K/32; ++ib) { + int l = nearest_int(0.5f*(id*scales[ib]-1)); + l = MAX(0, MIN(15, l)); + q2[2*ib+1] |= ((uint32_t)l << 28); + const float * xb = xbl + 32*ib; + const float * qw = quant_weights + QK_K*ibl + 32*ib; + for (int i = 0; i < 32; ++i) weight[i] = qw[i] * sqrtf(sigma2 + xb[i]*xb[i]); + const uint8_t * aux8 = (const uint8_t *)(q2 + 2*ib); + const float db = d * (1 + 2*l); + uint32_t u = 0; + for (int k = 0; k < 4; ++k) { + const int8_t * signs = keven_signs_q2xs + 8*((q2[2*ib+1] >> 7*k) & 127); + const float * xk = xb + 8*k; + const float * wk = weight + 8*k; + const uint8_t * grid = (const uint8_t *)(kgrid_q2xs + aux8[k]); + float best_mse = 0; int best_index = aux8[k]; + for (int j = 0; j < 8; ++j) { + float diff = db * grid[j] * signs[j] - xk[j]; + best_mse += wk[j] * diff * diff; + } + for (int idx = 0; idx < 256; ++idx) { + grid = (const uint8_t *)(kgrid_q2xs + idx); + float mse = 0; + for (int j = 0; j < 8; ++j) { + float diff = db * grid[j] * signs[j] - xk[j]; + mse += wk[j] * diff * diff; + } + if (mse < best_mse) { + best_mse = mse; best_index = idx; + } + } + u |= (best_index << 8*k); + grid = (const uint8_t *)(kgrid_q2xs + best_index); + //grid = (const uint8_t *)(kgrid_q2xs + aux8[k]); + for (int j = 0; j < 8; ++j) { + float q = db * grid[j] * signs[j]; + sumqx += wk[j] * q * xk[j]; + sumq2 += wk[j] * q * q; + } + } + q2[2*ib] = u; + if (sumq2 > 0) y[ibl].d = GGML_FP32_TO_FP16(d*sumqx/sumq2); + } + memcpy(y[ibl].qs, q2, QK_K/4); + } +} + +static void quantize_row_iq2_xs_impl(const float * restrict x, void * restrict vy, int n, const float * restrict quant_weights) { + + const int gindex = iq2_data_index(512); + + const uint64_t * kgrid_q2xs = iq2_data[gindex].grid; + const int * kmap_q2xs = iq2_data[gindex].map; + const uint16_t * kneighbors_q2xs = iq2_data[gindex].neighbours; + + GGML_ASSERT(quant_weights); + GGML_ASSERT(kmap_q2xs); + GGML_ASSERT(kgrid_q2xs); + GGML_ASSERT(kneighbors_q2xs); + GGML_ASSERT(n%QK_K == 0); + + const int kMaxQ = 3; + + const int nbl = n/256; + + block_iq2_xs * y = vy; + + float scales[QK_K/16]; + float weight[16]; + float xval[16]; + int8_t L[16]; + int8_t Laux[16]; + float waux[16]; + bool is_on_grid[2]; + bool is_on_grid_aux[2]; + uint8_t block_signs[2]; + uint16_t q2[2*(QK_K/16)]; + + for (int ibl = 0; ibl < nbl; ++ibl) { + + y[ibl].d = GGML_FP32_TO_FP16(0.f); + memset(q2, 0, QK_K/4); + memset(y[ibl].scales, 0, QK_K/32); + + float max_scale = 0; + + const float * xbl = x + QK_K*ibl; + float sumx2 = 0; + for (int i = 0; i < QK_K; ++i) sumx2 += xbl[i]*xbl[i]; + float sigma2 = sumx2/QK_K; + + for (int ib = 0; ib < QK_K/16; ++ib) { + const float * xb = xbl + 16*ib; + const float * qw = quant_weights + QK_K*ibl + 16*ib; + for (int i = 0; i < 16; ++i) weight[i] = qw[i] * sqrtf(sigma2 + xb[i]*xb[i]); + for (int i = 0; i < 16; ++i) waux[i] = sqrtf(weight[i]); + for (int k = 0; k < 2; ++k) { + int nflip = 0; + uint8_t s = 0; + for (int i = 0; i < 8; ++i) { + if (xb[8*k + i] >= 0) xval[8*k + i] = xb[8*k + i]; + else { + xval[8*k + i] = -xb[8*k + i]; ++nflip; s |= (1 << i); + } + } + if (nflip%2) { + int imin = 0; float min = weight[8*k+imin]*xb[8*k+imin]*xb[8*k+imin]; + for (int i = 1; i < 8; ++i) { + float ax = weight[8*k+i]*xb[8*k+i]*xb[8*k+i]; + if (ax < min) { + min = ax; imin = i; + } + } + xval[8*k+imin] = -xval[8*k+imin]; + s ^= (1 << imin); + } + block_signs[k] = s & 127; + } + float max = xval[0]; + for (int i = 1; i < 16; ++i) max = MAX(max, xval[i]); + if (!max) { + scales[ib] = 0; + memset(L, 0, 16); + continue; + } + float best = 0; + float scale = max/(2*kMaxQ-1); + is_on_grid[0] = is_on_grid[1] = true; + for (int is = -9; is <= 9; ++is) { + float id = (2*kMaxQ-1+is*0.1f)/max; + float this_scale = 1/id; + for (int k = 0; k < 2; ++k) { + for (int i = 0; i < 8; ++i) { + int l = nearest_int(0.5f*(id*xval[8*k+i]-1)); + Laux[8*k+i] = MAX(0, MIN(kMaxQ-1, l)); + } + uint16_t u = 0; + for (int i = 0; i < 8; ++i) u |= (Laux[8*k+i] << 2*i); + int grid_index = kmap_q2xs[u]; + is_on_grid_aux[k] = true; + if (grid_index < 0) { + is_on_grid_aux[k] = false; + const uint16_t * neighbours = kneighbors_q2xs - kmap_q2xs[u] - 1; + grid_index = iq2_find_best_neighbour(neighbours, kgrid_q2xs, xval + 8*k, waux + 8*k, this_scale, Laux + 8*k); + } + } + float sumqx = 0, sumq2 = 0; + for (int i = 0; i < 16; ++i) { + float w = weight[i]; + float q = 2*Laux[i] + 1; + sumqx += w*xval[i]*q; + sumq2 += w*q*q; + } + if (sumq2 > 0 && sumqx*sumqx > best*sumq2) { + scale = sumqx/sumq2; best = scale*sumqx; + for (int i = 0; i < 16; ++i) L[i] = Laux[i]; + for (int k = 0; k < 2; ++k) is_on_grid[k] = is_on_grid_aux[k]; + } + } + int n_not_ongrid = 0; + for (int k = 0; k < 2; ++k) if (!is_on_grid[k]) ++n_not_ongrid; + if (n_not_ongrid > 0 && scale > 0) { + float id = 1/scale; + for (int k = 0; k < 2; ++k) { + if (is_on_grid[k]) continue; + uint16_t u = 0; + for (int i = 0; i < 8; ++i) { + int l = nearest_int(0.5f*(id*xval[8*k+i]-1)); + l = MAX(0, MIN(kMaxQ-1, l)); + u |= (l << 2*i); + L[8*k + i] = l; + } + int grid_index = kmap_q2xs[u]; + if (grid_index < 0) { + const uint16_t * neighbours = kneighbors_q2xs - kmap_q2xs[u] - 1; + grid_index = iq2_find_best_neighbour(neighbours, kgrid_q2xs, xval + 8*k, waux + 8*k, scale, L + 8*k); + } + } + float sumqx = 0, sumq2 = 0; + for (int i = 0; i < 16; ++i) { + float w = weight[i]; + float q = 2*L[i] + 1; + sumqx += w*xval[i]*q; + sumq2 += w*q*q; + } + if (sumq2 > 0) scale = sumqx/sumq2; + } + if (scale < 0) { + scale = -scale; + for (int k = 0; k < 2; ++k) block_signs[k] = (~block_signs[k]) & 127; + } + for (int k = 0; k < 2; ++k) { + uint16_t u = 0; + for (int i = 0; i < 8; ++i) u |= (L[8*k+i] << 2*i); + int grid_index = kmap_q2xs[u]; + if (grid_index < 0) { + printf("Oops: found point %u not on grid:", u); + for (int i = 0; i < 8; ++i) printf(" %d", L[8*k+i]); + printf("\n"); + GGML_ASSERT(false); + } + q2[2*ib+k] = grid_index | (block_signs[k] << 9); + } + GGML_ASSERT(scale >= 0); + scales[ib] = scale; + max_scale = MAX(max_scale, scale); + } + + if (!max_scale) { + memset(y[ibl].qs, 0, QK_K/4); + continue; + } + + float d = max_scale/31; + y[ibl].d = GGML_FP32_TO_FP16(d); + float id = 1/d; + for (int ib = 0; ib < QK_K/16; ++ib) { + int l = nearest_int(0.5f*(id*scales[ib]-1)); + l = MAX(0, MIN(15, l)); + if (ib%2 == 0) y[ibl].scales[ib/2] = l; + else y[ibl].scales[ib/2] |= (l << 4); + } + memcpy(y[ibl].qs, q2, QK_K/4); + + } +} + +size_t quantize_iq2_xxs(const float * src, void * dst, int nrow, int n_per_row, int64_t * hist, const float * quant_weights) { + (void)hist; + GGML_ASSERT(n_per_row%QK_K == 0); + int nblock = n_per_row/QK_K; + char * qrow = (char *)dst; + for (int row = 0; row < nrow; ++row) { + quantize_row_iq2_xxs_impl(src, qrow, n_per_row, quant_weights); + src += n_per_row; + qrow += nblock*sizeof(block_iq2_xxs); + } + return nrow * nblock * sizeof(block_iq2_xxs); +} + +size_t quantize_iq2_xs(const float * src, void * dst, int nrow, int n_per_row, int64_t * hist, const float * quant_weights) { + (void)hist; + GGML_ASSERT(n_per_row%QK_K == 0); + int nblock = n_per_row/QK_K; + char * qrow = (char *)dst; + for (int row = 0; row < nrow; ++row) { + quantize_row_iq2_xs_impl(src, qrow, n_per_row, quant_weights); + src += n_per_row; + qrow += nblock*sizeof(block_iq2_xs); + } + return nrow * nblock * sizeof(block_iq2_xs); +} + diff --git a/ggml-quants.h b/ggml-quants.h index df5e7ae80..e5d110230 100644 --- a/ggml-quants.h +++ b/ggml-quants.h @@ -196,8 +196,6 @@ void quantize_row_q4_K_reference(const float * restrict x, block_q4_K * restrict void quantize_row_q5_K_reference(const float * restrict x, block_q5_K * restrict y, int k); void quantize_row_q6_K_reference(const float * restrict x, block_q6_K * restrict y, int k); void quantize_row_q8_K_reference(const float * restrict x, block_q8_K * restrict y, int k); -void quantize_row_iq2_xxs_reference(const float * restrict x, block_iq2_xxs * restrict y, int k); -void quantize_row_iq2_xs_reference (const float * restrict x, block_iq2_xs * restrict y, int k); void quantize_row_q4_0(const float * restrict x, void * restrict y, int k); void quantize_row_q4_1(const float * restrict x, void * restrict y, int k); @@ -212,8 +210,6 @@ void quantize_row_q4_K(const float * restrict x, void * restrict y, int k); void quantize_row_q5_K(const float * restrict x, void * restrict y, int k); void quantize_row_q6_K(const float * restrict x, void * restrict y, int k); void quantize_row_q8_K(const float * restrict x, void * restrict y, int k); -void quantize_row_iq2_xxs(const float * restrict x, void * restrict y, int k); -void quantize_row_iq2_xs (const float * restrict x, void * restrict y, int k); // Dequantization void dequantize_row_q4_0(const block_q4_0 * restrict x, float * restrict y, int k); @@ -246,3 +242,11 @@ void ggml_vec_dot_q5_K_q8_K(int n, float * restrict s, const void * restrict vx, void ggml_vec_dot_q6_K_q8_K(int n, float * restrict s, const void * restrict vx, const void * restrict vy); void ggml_vec_dot_iq2_xxs_q8_K(int n, float * restrict s, const void * restrict vx, const void * restrict vy); void ggml_vec_dot_iq2_xs_q8_K (int n, float * restrict s, const void * restrict vx, const void * restrict vy); + +// +// Quantization utilizing an importance matrix (a.k.a. "Activation aWare Quantization") +// +size_t quantize_iq2_xxs(const float * src, void * dst, int nrows, int n_per_row, int64_t * hist, const float * imatrix); +size_t quantize_iq2_xs (const float * src, void * dst, int nrows, int n_per_row, int64_t * hist, const float * imatrix); +size_t quantize_q2_K (const float * src, void * dst, int nrows, int n_per_row, int64_t * hist, const float * imatrix); + diff --git a/ggml.c b/ggml.c index bcfb6652c..52467475a 100644 --- a/ggml.c +++ b/ggml.c @@ -585,8 +585,8 @@ static const ggml_type_traits_t type_traits[GGML_TYPE_COUNT] = { .type_size = sizeof(block_iq2_xxs), .is_quantized = true, .to_float = (ggml_to_float_t) dequantize_row_iq2_xxs, - .from_float = quantize_row_iq2_xxs, - .from_float_reference = (ggml_from_float_t) quantize_row_iq2_xxs_reference, + .from_float = NULL, + .from_float_reference = NULL, .vec_dot = ggml_vec_dot_iq2_xxs_q8_K, .vec_dot_type = GGML_TYPE_Q8_K, }, @@ -596,8 +596,8 @@ static const ggml_type_traits_t type_traits[GGML_TYPE_COUNT] = { .type_size = sizeof(block_iq2_xs), .is_quantized = true, .to_float = (ggml_to_float_t) dequantize_row_iq2_xs, - .from_float = quantize_row_iq2_xs, - .from_float_reference = (ggml_from_float_t) quantize_row_iq2_xs_reference, + .from_float = NULL, + .from_float_reference = NULL, .vec_dot = ggml_vec_dot_iq2_xs_q8_K, .vec_dot_type = GGML_TYPE_Q8_K, }, @@ -18665,8 +18665,11 @@ size_t ggml_quantize_q8_0(const float * src, void * dst, int n, int k, int64_t * return (n/QK8_0*sizeof(block_q8_0)); } -size_t ggml_quantize_chunk(enum ggml_type type, const float * src, void * dst, int start, int n, int64_t * hist) { +size_t ggml_quantize_chunk(enum ggml_type type, const float * src, void * dst, int start, + int nrows, int n_per_row, int64_t * hist, const float * imatrix) { + (void)imatrix; size_t result = 0; + int n = nrows * n_per_row; switch (type) { case GGML_TYPE_Q4_0: { @@ -18701,8 +18704,11 @@ size_t ggml_quantize_chunk(enum ggml_type type, const float * src, void * dst, i case GGML_TYPE_Q2_K: { GGML_ASSERT(start % QK_K == 0); - block_q2_K * block = (block_q2_K*)dst + start / QK_K; - result = ggml_quantize_q2_K(src + start, block, n, n, hist); + GGML_ASSERT(start % n_per_row == 0); + size_t start_row = start / n_per_row; + size_t row_size = ggml_row_size(type, n_per_row); + result = quantize_q2_K(src + start, (char *)dst + start_row * row_size, nrows, n_per_row, hist, imatrix); + GGML_ASSERT(result == row_size * nrows); } break; case GGML_TYPE_Q3_K: { @@ -18731,14 +18737,22 @@ size_t ggml_quantize_chunk(enum ggml_type type, const float * src, void * dst, i case GGML_TYPE_IQ2_XXS: { GGML_ASSERT(start % QK_K == 0); - block_iq2_xxs * block = (block_iq2_xxs*)dst + start / QK_K; - result = ggml_quantize_iq2_xxs(src + start, block, n, n, hist); + GGML_ASSERT(start % n_per_row == 0); + GGML_ASSERT(imatrix); + size_t start_row = start / n_per_row; + size_t row_size = ggml_row_size(type, n_per_row); + result = quantize_iq2_xxs(src + start, (char *)dst + start_row * row_size, nrows, n_per_row, hist, imatrix); + GGML_ASSERT(result == row_size * nrows); } break; case GGML_TYPE_IQ2_XS: { GGML_ASSERT(start % QK_K == 0); - block_iq2_xs * block = (block_iq2_xs*)dst + start / QK_K; - result = ggml_quantize_iq2_xs(src + start, block, n, n, hist); + GGML_ASSERT(start % n_per_row == 0); + GGML_ASSERT(imatrix); + size_t start_row = start / n_per_row; + size_t row_size = ggml_row_size(type, n_per_row); + result = quantize_iq2_xs(src + start, (char *)dst + start_row * row_size, nrows, n_per_row, hist, imatrix); + GGML_ASSERT(result == row_size * nrows); } break; case GGML_TYPE_F16: { diff --git a/ggml.h b/ggml.h index b18ba7812..1187074f7 100644 --- a/ggml.h +++ b/ggml.h @@ -2067,10 +2067,13 @@ extern "C" { GGML_API size_t ggml_quantize_q4_K(const float * src, void * dst, int n, int k, int64_t * hist); GGML_API size_t ggml_quantize_q5_K(const float * src, void * dst, int n, int k, int64_t * hist); GGML_API size_t ggml_quantize_q6_K(const float * src, void * dst, int n, int k, int64_t * hist); - GGML_API size_t ggml_quantize_iq2_xxs(const float * src, void * dst, int n, int k, int64_t * hist); - GGML_API size_t ggml_quantize_iq2_xs (const float * src, void * dst, int n, int k, int64_t * hist); - GGML_API size_t ggml_quantize_chunk(enum ggml_type type, const float * src, void * dst, int start, int n, int64_t * hist); + GGML_API size_t ggml_quantize_chunk(enum ggml_type type, const float * src, void * dst, + int start, int nrows, int n_per_row, int64_t * hist, const float * imatrix); + + // These are needed for IQ2_XS and IQ2_XXS quantizations + GGML_API void ggml_init_iq2_quantization(enum ggml_type type); + GGML_API void ggml_deinit_iq2_quantization(enum ggml_type type); // // Importance matrix diff --git a/llama.cpp b/llama.cpp index 8e20e72a2..107b05114 100644 --- a/llama.cpp +++ b/llama.cpp @@ -8429,9 +8429,23 @@ static ggml_type get_k_quant_type(quantize_state_internal & qs, ggml_type new_ty if (arch == LLM_ARCH_FALCON || nx % QK_K != 0) { new_type = GGML_TYPE_Q8_0; } + else if (ftype == LLAMA_FTYPE_MOSTLY_IQ2_XXS || ftype == LLAMA_FTYPE_MOSTLY_IQ2_XS) { + new_type = GGML_TYPE_Q5_K; + } else if (new_type != GGML_TYPE_Q8_0) { new_type = GGML_TYPE_Q6_K; } + } else if (ftype == LLAMA_FTYPE_MOSTLY_IQ2_XXS || ftype == LLAMA_FTYPE_MOSTLY_IQ2_XS) { + if (name.find("attn_v.weight") != std::string::npos) { + if (qs.model.hparams.n_gqa() >= 4 || qs.model.hparams.n_expert >= 4) new_type = GGML_TYPE_Q4_K; + else new_type = GGML_TYPE_Q2_K; + ++qs.i_attention_wv; + } + else if (name.find("ffn_down") != std::string::npos) { + if (qs.i_feed_forward_w2 < qs.n_feed_forward_w2/8) new_type = GGML_TYPE_Q2_K; + ++qs.i_feed_forward_w2; + } + else if (name == "token_embd.weight") new_type = GGML_TYPE_Q2_K; } else if (name.find("attn_v.weight") != std::string::npos) { if (ftype == LLAMA_FTYPE_MOSTLY_Q2_K) new_type = GGML_TYPE_Q3_K; else if (ftype == LLAMA_FTYPE_MOSTLY_Q3_K_M) { @@ -8601,6 +8615,13 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s if (params->only_copy) { ftype = model.ftype; } + const std::unordered_map> * imatrix_data = nullptr; + if (params->imatrix) { + imatrix_data = static_cast>*>(params->imatrix); + if (imatrix_data) { + printf("================================ Have weights data with %d entries\n",int(imatrix_data->size())); + } + } const size_t align = GGUF_DEFAULT_ALIGNMENT; struct gguf_context * ctx_out = gguf_init_empty(); @@ -8658,6 +8679,8 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s // placeholder for the meta data ::zeros(fout, meta_size); + std::set used_iq2; + for (int i = 0; i < ml.n_tensors; ++i) { struct ggml_tensor * tensor = ml.get_tensor_meta(i); @@ -8710,6 +8733,35 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s } else { const size_t nelements = ggml_nelements(tensor); + if ((new_type == GGML_TYPE_IQ2_XXS || new_type == GGML_TYPE_IQ2_XS) && used_iq2.find(new_type) == used_iq2.end()) { + ggml_init_iq2_quantization(new_type); + used_iq2.insert(new_type); + } + + const float * imatrix = nullptr; + if (imatrix_data) { + auto it = imatrix_data->find(tensor->name); + if (it == imatrix_data->end()) { + printf("\n====== %s: did not find weights for %s\n", __func__, tensor->name); + } else { + if (it->second.size() == (size_t)tensor->ne[0]) { + imatrix = it->second.data(); + } else { + printf("\n====== %s: imatrix size %d is different from tensor size %d for %s\n", __func__, + int(it->second.size()), int(tensor->ne[0]), tensor->name); + } + } + } + if ((new_type == GGML_TYPE_IQ2_XXS || + new_type == GGML_TYPE_IQ2_XS || + (new_type == GGML_TYPE_Q2_K && params->ftype == LLAMA_FTYPE_MOSTLY_Q2_K_S && strcmp(tensor->name, "token_embd.weight") != 0)) && !imatrix) { + fprintf(stderr, "\n\n============================================================\n"); + fprintf(stderr, "Missing importance matrix for tensor %s in a very low-bit quantization\n", tensor->name); + fprintf(stderr, "The result will be garbage, so bailing out\n"); + fprintf(stderr, "============================================================\n\n"); + throw std::runtime_error(format("Missing importance matrix for tensor %s in a very low-bit quantization", tensor->name)); + } + float * f32_data; if (tensor->type == GGML_TYPE_F32) { @@ -8730,21 +8782,28 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s new_data = work.data(); std::array hist_cur = {}; - static const int chunk_size = 32 * 512; + const int n_per_row = tensor->ne[0]; + const int nrows = nelements / n_per_row; + + static const int min_chunk_size = 32 * 512; + const int chunk_size = n_per_row >= min_chunk_size ? n_per_row : n_per_row * ((min_chunk_size + n_per_row - 1)/n_per_row); + const int nchunk = (nelements + chunk_size - 1)/chunk_size; const int nthread_use = nthread > 1 ? std::max(1, std::min(nthread, nchunk)) : 1; if (nthread_use < 2) { - new_size = ggml_quantize_chunk(new_type, f32_data, new_data, 0, nelements, hist_cur.data()); + new_size = ggml_quantize_chunk(new_type, f32_data, new_data, 0, nrows, n_per_row, hist_cur.data(), imatrix); } else { - size_t counter = 0; + int counter = 0; new_size = 0; - auto compute = [&mutex, &counter, &hist_cur, &new_size, new_type, f32_data, new_data, nelements]() { + auto compute = [&mutex, &counter, &hist_cur, &new_size, new_type, f32_data, new_data, chunk_size, + nrows, n_per_row, imatrix]() { std::array local_hist = {}; + const int nrows_per_chunk = chunk_size / n_per_row; size_t local_size = 0; while (true) { std::unique_lock lock(mutex); - size_t first = counter; counter += chunk_size; - if (first >= nelements) { + int first_row = counter; counter += nrows_per_chunk; + if (first_row >= nrows) { if (local_size > 0) { for (int j=0; j %8.2f MiB | hist: ", ggml_nbytes(tensor)/1024.0/1024.0, new_size/1024.0/1024.0); + LLAMA_LOG_INFO("size = %8.2f MiB -> %8.2f MiB", ggml_nbytes(tensor)/1024.0/1024.0, new_size/1024.0/1024.0); int64_t tot_count = 0; for (size_t i = 0; i < hist_cur.size(); i++) { hist_all[i] += hist_cur[i]; @@ -8774,6 +8834,7 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s } if (tot_count > 0) { + LLAMA_LOG_INFO(" | hist: "); for (size_t i = 0; i < hist_cur.size(); i++) { LLAMA_LOG_INFO("%5.3f ", hist_cur[i] / float(nelements)); } @@ -8802,6 +8863,10 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s fout.close(); + for (auto type : used_iq2) { + ggml_deinit_iq2_quantization(type); + } + gguf_free(ctx_out); LLAMA_LOG_INFO("%s: model size = %8.2f MB\n", __func__, total_size_org/1024.0/1024.0); @@ -9166,6 +9231,7 @@ struct llama_model_quantize_params llama_model_quantize_default_params() { /*.quantize_output_tensor =*/ true, /*.only_copy =*/ false, /*.pure =*/ false, + /*.imatrix =*/ nullptr, }; return result; diff --git a/llama.h b/llama.h index 01d6fafaa..79c8335b6 100644 --- a/llama.h +++ b/llama.h @@ -249,6 +249,7 @@ extern "C" { bool quantize_output_tensor; // quantize output.weight bool only_copy; // only copy tensors - ftype, allow_requantize and quantize_output_tensor are ignored bool pure; // disable k-quant mixtures and quantize all tensors to the same type + void * imatrix; // pointer to importance matrix data } llama_model_quantize_params; // grammar types diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index d9b8b106a..22a7856d4 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -56,7 +56,7 @@ static void init_tensor_uniform(ggml_tensor * tensor, float min = -1.0f, float m GGML_ASSERT(size % ggml_blck_size(tensor->type) == 0); std::vector dataq(ggml_row_size(tensor->type, size)); int64_t hist[16]; - ggml_quantize_chunk(tensor->type, data.data(), dataq.data(), 0, size, hist); + ggml_quantize_chunk(tensor->type, data.data(), dataq.data(), 0, size/tensor->ne[0], tensor->ne[0], hist, nullptr); ggml_backend_tensor_set(tensor, dataq.data(), 0, dataq.size()); } else if (tensor->type == GGML_TYPE_I8 || tensor->type == GGML_TYPE_I16 || tensor->type == GGML_TYPE_I32) { // This is going to create some weird integers though. From ac32902a87147f78d63c931aa8a23dee762660e7 Mon Sep 17 00:00:00 2001 From: Karthik Kumar Viswanathan <195178+guilt@users.noreply.github.com> Date: Sun, 14 Jan 2024 00:41:44 -0800 Subject: [PATCH 062/812] llama : support WinXP build with MinGW 8.1.0 (#3419) --- CMakeLists.txt | 8 ++++++-- llama.cpp | 4 ++++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 668669c6d..2741568ed 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.13) # for add_link_options +cmake_minimum_required(VERSION 3.14) # for add_link_options and implicit target directories. project("llama.cpp" C CXX) set(CMAKE_EXPORT_COMPILE_COMMANDS ON) @@ -76,6 +76,10 @@ if (NOT MSVC) option(LLAMA_F16C "llama: enable F16C" ${INS_ENB}) endif() +if (WIN32) + option(LLAMA_WIN_VER "llama: Windows Version" 0x602) +endif() + # 3rd party libs option(LLAMA_ACCELERATE "llama: enable Accelerate framework" ON) option(LLAMA_BLAS "llama: use BLAS" OFF) @@ -686,7 +690,7 @@ endif() if (MINGW) # Target Windows 8 for PrefetchVirtualMemory - add_compile_definitions(_WIN32_WINNT=0x602) + add_compile_definitions(_WIN32_WINNT=${LLAMA_WIN_VER}) endif() # diff --git a/llama.cpp b/llama.cpp index 107b05114..51e9bdaed 100644 --- a/llama.cpp +++ b/llama.cpp @@ -987,6 +987,7 @@ struct llama_mmap { } if (prefetch > 0) { +#if _WIN32_WINNT >= 0x602 // PrefetchVirtualMemory is only present on Windows 8 and above, so we dynamically load it BOOL (WINAPI *pPrefetchVirtualMemory) (HANDLE, ULONG_PTR, PWIN32_MEMORY_RANGE_ENTRY, ULONG); HMODULE hKernel32 = GetModuleHandleW(L"kernel32.dll"); @@ -1004,6 +1005,9 @@ struct llama_mmap { llama_format_win_err(GetLastError()).c_str()); } } +#else + throw std::runtime_error("PrefetchVirtualMemory unavailable"); +#endif } } From 5f5fe1bd608fa2ed42af97b5f2ea31be6625fc48 Mon Sep 17 00:00:00 2001 From: Alex Azarov Date: Sun, 14 Jan 2024 09:44:39 +0100 Subject: [PATCH 063/812] metal : correctly set SIMD support flags on iOS (#4923) * Correctly set support_simdgroup_reduction and support_simdgroup_mm on iPhone/iPad * log a little bit more info on iOS --- ggml-metal.m | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ggml-metal.m b/ggml-metal.m index cae52c983..2ca726055 100644 --- a/ggml-metal.m +++ b/ggml-metal.m @@ -330,7 +330,6 @@ static struct ggml_metal_context * ggml_metal_init(int n_cb) { } } -#if TARGET_OS_OSX // print MTL GPU family: GGML_METAL_LOG_INFO("%s: GPU name: %s\n", __func__, [[ctx->device name] UTF8String]); @@ -370,6 +369,7 @@ static struct ggml_metal_context * ggml_metal_init(int n_cb) { GGML_METAL_LOG_INFO("%s: simdgroup reduction support = %s\n", __func__, ctx->support_simdgroup_reduction ? "true" : "false"); GGML_METAL_LOG_INFO("%s: simdgroup matrix mul. support = %s\n", __func__, ctx->support_simdgroup_mm ? "true" : "false"); GGML_METAL_LOG_INFO("%s: hasUnifiedMemory = %s\n", __func__, ctx->device.hasUnifiedMemory ? "true" : "false"); +#if TARGET_OS_OSX GGML_METAL_LOG_INFO("%s: recommendedMaxWorkingSetSize = %8.2f MB\n", __func__, ctx->device.recommendedMaxWorkingSetSize / 1e6); if (ctx->device.maxTransferRate != 0) { GGML_METAL_LOG_INFO("%s: maxTransferRate = %8.2f MB/s\n", __func__, ctx->device.maxTransferRate / 1e6); From a128c38de862431f1aae9ccc40b792fbc1b8b682 Mon Sep 17 00:00:00 2001 From: Kawrakow <48489457+ikawrakow@users.noreply.github.com> Date: Sun, 14 Jan 2024 10:53:39 +0200 Subject: [PATCH 064/812] Fix ffn_down quantization mix for MoE models (#4927) * Fix ffn_down quantization mix for MoE models In #4872 I did not consider the part where every third tensor is quantized with more bits. Fir MoE this leads to tensors of the same layer being quantized with different number of bits, which is not considered as a possibility in the inference implementation (it is assumed all experts use the same quantization). * Fix the fix * Review suggestion --------- Co-authored-by: Iwan Kawrakow --- llama.cpp | 34 ++++++++++++++++++++++++++-------- 1 file changed, 26 insertions(+), 8 deletions(-) diff --git a/llama.cpp b/llama.cpp index 51e9bdaed..b1d6015e2 100644 --- a/llama.cpp +++ b/llama.cpp @@ -8480,13 +8480,31 @@ static ggml_type get_k_quant_type(quantize_state_internal & qs, ggml_type new_ty new_type = GGML_TYPE_Q8_0; } } else if (name.find("ffn_down") != std::string::npos) { + const int n_expert = std::max(1, (int)qs.model.hparams.n_expert); + int i_layer, n_layer; + if (n_expert == 1) { + i_layer = qs.i_feed_forward_w2; + n_layer = qs.n_feed_forward_w2; + } else { + // Believe it or not, "experts" in the FFN of Mixtral-8x7B are not consecutive, but iccasionally randomly + // sprinkled in the model. Hence, simply dividing i_feed_forward_w2 by n_expert does not work + // for getting the current layer as I initially thought, and we need to resort to parsing the + // tensor name. + n_layer = qs.n_feed_forward_w2 / n_expert; + if (sscanf(name.c_str(), "blk.%d.ffn_down", &i_layer) != 1) { + throw std::runtime_error(format("Failed to determine layer for tensor %s", name.c_str())); + } + if (i_layer < 0 || i_layer >= n_layer) { + throw std::runtime_error(format("Bad layer %d for tensor %s. Must be in [0, %d)", i_layer, name.c_str(), n_layer)); + } + } if (ftype == LLAMA_FTYPE_MOSTLY_Q2_K) new_type = GGML_TYPE_Q3_K; else if (ftype == LLAMA_FTYPE_MOSTLY_Q2_K_S) { - if (qs.i_feed_forward_w2 < qs.n_feed_forward_w2/8) new_type = GGML_TYPE_Q4_K; + if (i_layer < n_layer/8) new_type = GGML_TYPE_Q4_K; } else if (ftype == LLAMA_FTYPE_MOSTLY_Q3_K_M) { - new_type = qs.i_feed_forward_w2 < qs.n_feed_forward_w2/16 ? GGML_TYPE_Q5_K - : arch != LLM_ARCH_FALCON || use_more_bits(qs.i_feed_forward_w2, qs.n_feed_forward_w2) ? GGML_TYPE_Q4_K + new_type = i_layer < n_layer/16 ? GGML_TYPE_Q5_K + : arch != LLM_ARCH_FALCON || use_more_bits(i_layer, n_layer) ? GGML_TYPE_Q4_K : GGML_TYPE_Q3_K; } else if (ftype == LLAMA_FTYPE_MOSTLY_Q3_K_L) { @@ -8494,14 +8512,14 @@ static ggml_type get_k_quant_type(quantize_state_internal & qs, ggml_type new_ty } else if (ftype == LLAMA_FTYPE_MOSTLY_Q4_K_M) { if (arch == LLM_ARCH_FALCON) { - new_type = qs.i_feed_forward_w2 < qs.n_feed_forward_w2/16 ? GGML_TYPE_Q6_K : - use_more_bits(qs.i_feed_forward_w2, qs.n_feed_forward_w2) ? GGML_TYPE_Q5_K : GGML_TYPE_Q4_K; + new_type = i_layer < n_layer/16 ? GGML_TYPE_Q6_K : + use_more_bits(i_layer, n_layer) ? GGML_TYPE_Q5_K : GGML_TYPE_Q4_K; } else { - if (use_more_bits(qs.i_feed_forward_w2, qs.n_feed_forward_w2)) new_type = GGML_TYPE_Q6_K; + if (use_more_bits(i_layer, n_layer)) new_type = GGML_TYPE_Q6_K; } } - else if (ftype == LLAMA_FTYPE_MOSTLY_Q5_K_M && use_more_bits(qs.i_feed_forward_w2, qs.n_feed_forward_w2)) new_type = GGML_TYPE_Q6_K; - else if (ftype == LLAMA_FTYPE_MOSTLY_Q4_K_S && arch != LLM_ARCH_FALCON && qs.i_feed_forward_w2 < qs.n_feed_forward_w2/8) { + else if (ftype == LLAMA_FTYPE_MOSTLY_Q5_K_M && use_more_bits(i_layer, n_layer)) new_type = GGML_TYPE_Q6_K; + else if (ftype == LLAMA_FTYPE_MOSTLY_Q4_K_S && arch != LLM_ARCH_FALCON && i_layer < n_layer/8) { new_type = GGML_TYPE_Q5_K; } ++qs.i_feed_forward_w2; From 03c526749041c863b0cd842b26b8907e1ea0e0b1 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Sun, 14 Jan 2024 11:03:19 +0200 Subject: [PATCH 065/812] llama : use LLAMA_LOG_ macros for logging --- llama.cpp | 46 +++++++++++++++++++++++----------------------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/llama.cpp b/llama.cpp index b1d6015e2..51821965e 100644 --- a/llama.cpp +++ b/llama.cpp @@ -1114,7 +1114,7 @@ struct llama_mlock { suggest = false; } - fprintf(stderr, "warning: failed to mlock %zu-byte buffer (after previously locking %zu bytes): %s\n%s", + LLAMA_LOG_WARN("warning: failed to mlock %zu-byte buffer (after previously locking %zu bytes): %s\n%s", size, this->size, errmsg, suggest ? MLOCK_SUGGESTION : ""); return false; } @@ -1123,7 +1123,7 @@ struct llama_mlock { static void raw_unlock(void * addr, size_t size) { if (munlock(addr, size)) { - fprintf(stderr, "warning: failed to munlock buffer: %s\n", std::strerror(errno)); + LLAMA_LOG_WARN("warning: failed to munlock buffer: %s\n", std::strerror(errno)); } } #elif defined(_WIN32) @@ -1141,7 +1141,7 @@ struct llama_mlock { return true; } if (tries == 2) { - fprintf(stderr, "warning: failed to VirtualLock %zu-byte buffer (after previously locking %zu bytes): %s\n", + LLAMA_LOG_WARN("warning: failed to VirtualLock %zu-byte buffer (after previously locking %zu bytes): %s\n", len, size, llama_format_win_err(GetLastError()).c_str()); return false; } @@ -1150,7 +1150,7 @@ struct llama_mlock { // set size and try again. SIZE_T min_ws_size, max_ws_size; if (!GetProcessWorkingSetSize(GetCurrentProcess(), &min_ws_size, &max_ws_size)) { - fprintf(stderr, "warning: GetProcessWorkingSetSize failed: %s\n", + LLAMA_LOG_WARN("warning: GetProcessWorkingSetSize failed: %s\n", llama_format_win_err(GetLastError()).c_str()); return false; } @@ -1163,7 +1163,7 @@ struct llama_mlock { min_ws_size += increment; max_ws_size += increment; if (!SetProcessWorkingSetSize(GetCurrentProcess(), min_ws_size, max_ws_size)) { - fprintf(stderr, "warning: SetProcessWorkingSetSize failed: %s\n", + LLAMA_LOG_WARN("warning: SetProcessWorkingSetSize failed: %s\n", llama_format_win_err(GetLastError()).c_str()); return false; } @@ -1172,7 +1172,7 @@ struct llama_mlock { static void raw_unlock(void * ptr, size_t len) { if (!VirtualUnlock(ptr, len)) { - fprintf(stderr, "warning: failed to VirtualUnlock buffer: %s\n", + LLAMA_LOG_WARN("warning: failed to VirtualUnlock buffer: %s\n", llama_format_win_err(GetLastError()).c_str()); } } @@ -1184,7 +1184,7 @@ struct llama_mlock { } bool raw_lock(const void * addr, size_t len) const { - fprintf(stderr, "warning: mlock not supported on this system\n"); + LLAMA_LOG_WARN("warning: mlock not supported on this system\n"); return false; } @@ -2085,13 +2085,13 @@ namespace GGUFMeta { __func__, override_type_to_str(override->tag), override->key); switch (override->tag) { case LLAMA_KV_OVERRIDE_BOOL: { - printf("%s\n", override->bool_value ? "true" : "false"); + LLAMA_LOG_INFO("%s\n", override->bool_value ? "true" : "false"); } break; case LLAMA_KV_OVERRIDE_INT: { - printf("%" PRId64 "\n", override->int_value); + LLAMA_LOG_INFO("%" PRId64 "\n", override->int_value); } break; case LLAMA_KV_OVERRIDE_FLOAT: { - printf("%.6f\n", override->float_value); + LLAMA_LOG_INFO("%.6f\n", override->float_value); } break; default: // Shouldn't be possible to end up here, but just in case... @@ -6993,7 +6993,7 @@ static void tokenizer_st_partition(const llama_vocab & vocab, std::forward_list< if (match + special_token.length() > raw_text_base_offset + raw_text_base_length) break; #ifdef PRETOKENIZERDEBUG - fprintf(stderr, "FF: (%ld %ld %ld) '%s'\n", raw_text->length(), raw_text_base_offset, raw_text_base_length, raw_text->substr(raw_text_base_offset, raw_text_base_length).c_str()); + LLAMA_LOG_WARN("FF: (%ld %ld %ld) '%s'\n", raw_text->length(), raw_text_base_offset, raw_text_base_length, raw_text->substr(raw_text_base_offset, raw_text_base_length).c_str()); #endif auto source = std::distance(buffer.begin(), it); @@ -7006,7 +7006,7 @@ static void tokenizer_st_partition(const llama_vocab & vocab, std::forward_list< buffer.emplace_after(it, (*raw_text), left_reminder_offset, left_reminder_length); #ifdef PRETOKENIZERDEBUG - fprintf(stderr, "FL: (%ld %ld) '%s'\n", left_reminder_offset, left_reminder_length, raw_text->substr(left_reminder_offset, left_reminder_length).c_str()); + LLAMA_LOG_WARN("FL: (%ld %ld) '%s'\n", left_reminder_offset, left_reminder_length, raw_text->substr(left_reminder_offset, left_reminder_length).c_str()); #endif it++; } @@ -7022,7 +7022,7 @@ static void tokenizer_st_partition(const llama_vocab & vocab, std::forward_list< buffer.emplace_after(it, (*raw_text), right_reminder_offset, right_reminder_length); #ifdef PRETOKENIZERDEBUG - fprintf(stderr, "FR: (%ld %ld) '%s'\n", right_reminder_offset, right_reminder_length, raw_text->substr(right_reminder_offset, right_reminder_length).c_str()); + LLAMA_LOG_WARN("FR: (%ld %ld) '%s'\n", right_reminder_offset, right_reminder_length, raw_text->substr(right_reminder_offset, right_reminder_length).c_str()); #endif it++; @@ -7038,7 +7038,7 @@ static void tokenizer_st_partition(const llama_vocab & vocab, std::forward_list< raw_text_base_length = right_reminder_length; #ifdef PRETOKENIZERDEBUG - fprintf(stderr, "RR: (%ld %ld) '%s'\n", raw_text_base_offset, raw_text_base_length, raw_text->substr(raw_text_base_offset, raw_text_base_length).c_str()); + LLAMA_LOG_WARN("RR: (%ld %ld) '%s'\n", raw_text_base_offset, raw_text_base_length, raw_text->substr(raw_text_base_offset, raw_text_base_length).c_str()); #endif } else { if (source == 0) { @@ -7095,7 +7095,7 @@ static std::vector llama_tokenize_internal(const llama_vocab & } #ifdef PRETOKENIZERDEBUG - fprintf(stderr,"TT: (%ld %ld %ld) '%s'\n", raw_text.length(), fragment.offset, fragment.length, raw_text.c_str()); + LLAMA_LOG_WARN(TT: (%ld %ld %ld) '%s'\n", raw_text.length(), fragment.offset, fragment.length, raw_text.c_str()); #endif llm_tokenizer_spm tokenizer(vocab); llama_escape_whitespace(raw_text); @@ -7116,7 +7116,7 @@ static std::vector llama_tokenize_internal(const llama_vocab & auto raw_text = fragment.raw_text.substr(fragment.offset, fragment.length); #ifdef PRETOKENIZERDEBUG - fprintf(stderr,"TT: (%ld %ld %ld) '%s'\n", raw_text.length(), fragment.offset, fragment.length, raw_text.c_str()); + LLAMA_LOG_WARN(TT: (%ld %ld %ld) '%s'\n", raw_text.length(), fragment.offset, fragment.length, raw_text.c_str()); #endif llm_tokenizer_bpe tokenizer(vocab); tokenizer.tokenize(raw_text, output); @@ -8641,7 +8641,7 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s if (params->imatrix) { imatrix_data = static_cast>*>(params->imatrix); if (imatrix_data) { - printf("================================ Have weights data with %d entries\n",int(imatrix_data->size())); + LLAMA_LOG_INFO("================================ Have weights data with %d entries\n",int(imatrix_data->size())); } } @@ -8764,12 +8764,12 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s if (imatrix_data) { auto it = imatrix_data->find(tensor->name); if (it == imatrix_data->end()) { - printf("\n====== %s: did not find weights for %s\n", __func__, tensor->name); + LLAMA_LOG_INFO("\n====== %s: did not find weights for %s\n", __func__, tensor->name); } else { if (it->second.size() == (size_t)tensor->ne[0]) { imatrix = it->second.data(); } else { - printf("\n====== %s: imatrix size %d is different from tensor size %d for %s\n", __func__, + LLAMA_LOG_INFO("\n====== %s: imatrix size %d is different from tensor size %d for %s\n", __func__, int(it->second.size()), int(tensor->ne[0]), tensor->name); } } @@ -8777,10 +8777,10 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s if ((new_type == GGML_TYPE_IQ2_XXS || new_type == GGML_TYPE_IQ2_XS || (new_type == GGML_TYPE_Q2_K && params->ftype == LLAMA_FTYPE_MOSTLY_Q2_K_S && strcmp(tensor->name, "token_embd.weight") != 0)) && !imatrix) { - fprintf(stderr, "\n\n============================================================\n"); - fprintf(stderr, "Missing importance matrix for tensor %s in a very low-bit quantization\n", tensor->name); - fprintf(stderr, "The result will be garbage, so bailing out\n"); - fprintf(stderr, "============================================================\n\n"); + LLAMA_LOG_ERROR("\n\n============================================================\n"); + LLAMA_LOG_ERROR("Missing importance matrix for tensor %s in a very low-bit quantization\n", tensor->name); + LLAMA_LOG_ERROR("The result will be garbage, so bailing out\n"); + LLAMA_LOG_ERROR("============================================================\n\n"); throw std::runtime_error(format("Missing importance matrix for tensor %s in a very low-bit quantization", tensor->name)); } From 9408cfdad6b1c090a7e1419d4434edc260b7e47e Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Sun, 14 Jan 2024 11:08:09 +0200 Subject: [PATCH 066/812] scripts : sync-ggml-am.sh option to skip commits --- scripts/sync-ggml-am.sh | 14 +++++++++++++- scripts/sync-ggml.last | 2 +- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/scripts/sync-ggml-am.sh b/scripts/sync-ggml-am.sh index 248cf1023..6b2514a11 100755 --- a/scripts/sync-ggml-am.sh +++ b/scripts/sync-ggml-am.sh @@ -5,7 +5,7 @@ # Usage: # # $ cd /path/to/llama.cpp -# $ ./scripts/sync-ggml-am.sh +# $ ./scripts/sync-ggml-am.sh -skip hash0,hash1,hash2... # set -e @@ -24,6 +24,11 @@ fi lc=$(cat $SRC_LLAMA/scripts/sync-ggml.last) echo "Syncing ggml changes since commit $lc" +to_skip="" +if [ "$1" == "-skip" ]; then + to_skip=$2 +fi + cd $SRC_GGML git log --oneline $lc..HEAD @@ -40,6 +45,13 @@ if [ -f $SRC_LLAMA/ggml-src.patch ]; then fi while read c; do + if [ -n "$to_skip" ]; then + if [[ $to_skip == *"$c"* ]]; then + echo "Skipping $c" + continue + fi + fi + git format-patch -k $c~1..$c --stdout -- \ include/ggml/ggml*.h \ src/ggml*.h \ diff --git a/scripts/sync-ggml.last b/scripts/sync-ggml.last index 753d227a7..be9e408fb 100644 --- a/scripts/sync-ggml.last +++ b/scripts/sync-ggml.last @@ -1 +1 @@ -1890780da4ea10db88736fcde85f285abf6c64b0 +b306d6e996ec0ace77118fa5098822cdc7f9c88f From bb0c1392479398f9aba86d9ec98db0b95ede6e6d Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Sun, 14 Jan 2024 13:26:53 +0200 Subject: [PATCH 067/812] llama : check LLAMA_TRACE env for extra logging (#4929) * llama : minor fix indent * llama : check LLAMA_TRACE env for extra logging ggml-ci --- llama.cpp | 32 ++++++++++++++++++-------------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/llama.cpp b/llama.cpp index 51821965e..63f37ecdb 100644 --- a/llama.cpp +++ b/llama.cpp @@ -2190,6 +2190,11 @@ struct llama_model_loader { LLM_KV llm_kv = LLM_KV(LLM_ARCH_UNKNOWN); llama_model_loader(const std::string & fname, bool use_mmap, const struct llama_model_kv_override * param_overrides_p) : file(fname.c_str(), "rb") { + int trace = 0; + if (getenv("LLAMA_TRACE")) { + trace = atoi(getenv("LLAMA_TRACE")); + } + struct gguf_init_params params = { /*.no_alloc = */ true, /*.ctx = */ &ctx_meta, @@ -2242,11 +2247,10 @@ struct llama_model_loader { type_max = type; } - // TODO: make runtime configurable -#if 0 - struct ggml_tensor * meta = ggml_get_tensor(ctx_meta, gguf_get_tensor_name(ctx_gguf, i)); - LLAMA_LOG_INFO("%s: - tensor %4d: %32s %-8s [ %s ]\n", __func__, i, ggml_get_name(meta), ggml_type_name(type), llama_format_tensor_shape(meta).c_str()); -#endif + if (trace > 0) { + struct ggml_tensor * meta = ggml_get_tensor(ctx_meta, gguf_get_tensor_name(ctx_gguf, i)); + LLAMA_LOG_INFO("%s: - tensor %4d: %32s %-8s [ %s ]\n", __func__, i, ggml_get_name(meta), ggml_type_name(type), llama_format_tensor_shape(meta).c_str()); + } } switch (type_max) { @@ -6451,15 +6455,15 @@ static uint8_t llama_token_to_byte(const llama_vocab& vocab, llama_token id) { static llama_token llama_byte_to_token(const llama_vocab & vocab, uint8_t ch) { static const char * hex = "0123456789ABCDEF"; switch (llama_vocab_get_type(vocab)) { - case LLAMA_VOCAB_TYPE_SPM: { - const char buf[7] = { '<', '0', 'x', hex[ch >> 4], hex[ch & 15], '>', 0 }; - return vocab.token_to_id.at(buf); - } - case LLAMA_VOCAB_TYPE_BPE: { - return vocab.token_to_id.at(bytes_to_unicode_bpe(ch)); - } - default: - GGML_ASSERT(false); + case LLAMA_VOCAB_TYPE_SPM: { + const char buf[7] = { '<', '0', 'x', hex[ch >> 4], hex[ch & 15], '>', 0 }; + return vocab.token_to_id.at(buf); + } + case LLAMA_VOCAB_TYPE_BPE: { + return vocab.token_to_id.at(bytes_to_unicode_bpe(ch)); + } + default: + GGML_ASSERT(false); } } From 467a882fd2e5b6172897b49aa45aa29bd3f27685 Mon Sep 17 00:00:00 2001 From: Kawrakow <48489457+ikawrakow@users.noreply.github.com> Date: Sun, 14 Jan 2024 16:21:12 +0200 Subject: [PATCH 068/812] Add ability to use importance matrix for all k-quants (#4930) Co-authored-by: Iwan Kawrakow --- examples/quantize/quantize.cpp | 2 +- ggml-quants.c | 443 ++++++++++++++++++++++++++++++++- ggml-quants.h | 5 +- ggml.c | 28 ++- 4 files changed, 462 insertions(+), 16 deletions(-) diff --git a/examples/quantize/quantize.cpp b/examples/quantize/quantize.cpp index f4e2175f1..2ae046933 100644 --- a/examples/quantize/quantize.cpp +++ b/examples/quantize/quantize.cpp @@ -82,7 +82,7 @@ static void usage(const char * executable) { printf(" --allow-requantize: Allows requantizing tensors that have already been quantized. Warning: This can severely reduce quality compared to quantizing from 16bit or 32bit\n"); printf(" --leave-output-tensor: Will leave output.weight un(re)quantized. Increases model size but may also increase quality, especially when requantizing\n"); printf(" --pure: Disable k-quant mixtures and quantize all tensors to the same type\n"); - printf(" --imatrixfile_name: use data in file_name as importance matrix for quant optimizations\n"); + printf(" --imatrix file_name: use data in file_name as importance matrix for quant optimizations\n"); printf(" --include-weights tensor_name: use importance matrix for this/these tensor(s)\n"); printf(" --exclude-weights tensor_name: use importance matrix for this/these tensor(s)\n"); printf("Note: --include-weights and --exclude-weights cannot be used together\n"); diff --git a/ggml-quants.c b/ggml-quants.c index 9290d54cf..0750fe1bb 100644 --- a/ggml-quants.c +++ b/ggml-quants.c @@ -1244,7 +1244,8 @@ static inline int nearest_int(float fval) { return (i & 0x007fffff) - 0x00400000; } -static float make_qx_quants(int n, int nmax, const float * restrict x, int8_t * restrict L, int rmse_type) { +static float make_qx_quants(int n, int nmax, const float * restrict x, int8_t * restrict L, int rmse_type, + const float * restrict qw) { float max = 0; float amax = 0; for (int i = 0; i < n; ++i) { @@ -1270,14 +1271,13 @@ static float make_qx_quants(int n, int nmax, const float * restrict x, int8_t * rmse_type = -rmse_type; return_early = true; } - int weight_type = rmse_type%2; float sumlx = 0; float suml2 = 0; for (int i = 0; i < n; ++i) { int l = nearest_int(iscale * x[i]); l = MAX(-nmax, MIN(nmax-1, l)); L[i] = l + nmax; - float w = weight_type == 1 ? x[i] * x[i] : 1; + float w = qw ? qw[i] : rmse_type == 1 ? x[i] * x[i] : rmse_type == 2 ? 1 : rmse_type == 3 ? fabsf(x[i]) : sqrtf(fabsf(x[i])); sumlx += w*x[i]*l; suml2 += w*l*l; } @@ -1293,7 +1293,7 @@ static float make_qx_quants(int n, int nmax, const float * restrict x, int8_t * for (int i = 0; i < n; ++i) { int l = nearest_int(iscale * x[i]); l = MAX(-nmax, MIN(nmax-1, l)); - float w = weight_type == 1 ? x[i] * x[i] : 1; + float w = qw ? qw[i] : rmse_type == 1 ? x[i] * x[i] : rmse_type == 2 ? 1 : rmse_type == 3 ? fabsf(x[i]) : sqrtf(fabsf(x[i])); sumlx += w*x[i]*l; suml2 += w*l*l; } @@ -2089,6 +2089,112 @@ size_t ggml_quantize_q3_K(const float * restrict src, void * restrict dst, int n return (n/QK_K*sizeof(block_q3_K)); } +static void quantize_row_q3_K_impl(const float * restrict x, block_q3_K * restrict y, int n_per_row, const float * restrict quant_weights) { +#if QK_K != 256 + (void)quant_weights; + quantize_row_q3_K_reference(x, y, n_per_row); +#else + assert(n_per_row % QK_K == 0); + const int nb = n_per_row / QK_K; + + int8_t L[QK_K]; + float scales[QK_K / 16]; + float weight[16]; + float sw[QK_K / 16]; + int8_t Ls[QK_K / 16]; + + for (int i = 0; i < nb; i++) { + + float sumx2 = 0; + for (int j = 0; j < QK_K; ++j) sumx2 += x[j]*x[j]; + float sigma2 = 2*sumx2/QK_K; + + for (int j = 0; j < QK_K/16; ++j) { + if (quant_weights) { + const float * qw = quant_weights ? quant_weights + QK_K * i + 16*j : NULL; + for (int l = 0; l < 16; ++l) weight[l] = qw[l] * sqrtf(sigma2 + x[16*j+l]*x[16*j+l]); + } else { + for (int l = 0; l < 16; ++l) weight[l] = x[16*j+l]*x[16*j+l]; + } + float sumw = 0; + for (int l = 0; l < 16; ++l) sumw += weight[l]; + sw[j] = sumw; + + scales[j] = make_qx_quants(16, 4, x + 16*j, L + 16*j, 1, weight); + + } + + memset(y[i].scales, 0, 12); + + float d_block = make_qx_quants(QK_K/16, 32, scales, Ls, 1, sw); + for (int j = 0; j < QK_K/16; ++j) { + int l = Ls[j]; + if (j < 8) { + y[i].scales[j] = l & 0xF; + } else { + y[i].scales[j-8] |= ((l & 0xF) << 4); + } + l >>= 4; + y[i].scales[j%4 + 8] |= (l << (2*(j/4))); + } + y[i].d = GGML_FP32_TO_FP16(d_block); + + int8_t sc; + for (int j = 0; j < QK_K/16; ++j) { + sc = j < 8 ? y[i].scales[j] & 0xF : y[i].scales[j-8] >> 4; + sc = (sc | (((y[i].scales[8 + j%4] >> (2*(j/4))) & 3) << 4)) - 32; + float d = GGML_FP16_TO_FP32(y[i].d) * sc; + if (!d) { + continue; + } + for (int ii = 0; ii < 16; ++ii) { + int l = nearest_int(x[16*j + ii]/d); + l = MAX(-4, MIN(3, l)); + L[16*j + ii] = l + 4; + } + } + + memset(y[i].hmask, 0, QK_K/8); + // We put the high-bit for the 1st 8 quants into bit 0, the next 8 into bit 1, etc. + int m = 0; + uint8_t hm = 1; + for (int j = 0; j < QK_K; ++j) { + if (L[j] > 3) { + y[i].hmask[m] |= hm; + L[j] -= 4; + } + if (++m == QK_K/8) { + m = 0; hm <<= 1; + } + } + for (int j = 0; j < QK_K; j += 128) { + for (int l = 0; l < 32; ++l) { + y[i].qs[j/4 + l] = L[j + l] | (L[j + l + 32] << 2) | (L[j + l + 64] << 4) | (L[j + l + 96] << 6); + } + } + + x += QK_K; + } +#endif +} + +size_t quantize_q3_K(const float * src, void * dst, int nrow, int n_per_row, int64_t * hist, const float * quant_weights) { + (void)hist; + int row_size = ggml_row_size(GGML_TYPE_Q3_K, n_per_row); + if (!quant_weights) { + quantize_row_q3_K_reference(src, dst, nrow*n_per_row); + } + else { + char * qrow = (char *)dst; + for (int row = 0; row < nrow; ++row) { + quantize_row_q3_K_impl(src, (block_q3_K*)qrow, n_per_row, quant_weights); + src += n_per_row; + qrow += row_size; + } + } + return nrow * row_size; +} + // ====================== 4-bit (de)-quantization void quantize_row_q4_K_reference(const float * restrict x, block_q4_K * restrict y, int k) { @@ -2254,6 +2360,108 @@ size_t ggml_quantize_q4_K(const float * restrict src, void * restrict dst, int n return (n/QK_K*sizeof(block_q4_K)); } +static void quantize_row_q4_K_impl(const float * restrict x, block_q4_K * restrict y, int n_per_row, const float * quant_weights) { +#if QK_K != 256 + (void)quant_weights; + quantize_row_q4_K_reference(x, y, n_per_row); +#else + assert(n_per_row % QK_K == 0); + const int nb = n_per_row / QK_K; + + uint8_t L[QK_K]; + uint8_t Laux[32]; + float weights[32]; + float mins[QK_K/32]; + float scales[QK_K/32]; + + for (int i = 0; i < nb; i++) { + + float sum_x2 = 0; + for (int l = 0; l < QK_K; ++l) sum_x2 += x[l] * x[l]; + float sigma2 = sum_x2/QK_K; + float av_x = sqrtf(sigma2); + + float max_scale = 0; // as we are deducting the min, scales are always positive + float max_min = 0; + for (int j = 0; j < QK_K/32; ++j) { + if (quant_weights) { + const float * qw = quant_weights + QK_K*i + 32*j; + for (int l = 0; l < 32; ++l) weights[l] = qw[l] * sqrtf(sigma2 + x[32*j + l]*x[32*j + l]); + } else { + for (int l = 0; l < 32; ++l) weights[l] = av_x + fabsf(x[32*j + l]); + } + scales[j] = make_qkx3_quants(32, 15, x + 32*j, weights, L + 32*j, &mins[j], Laux, -0.9f, 0.05f, 36, false); + //scales[j] = make_qkx2_quants(32, 15, x + 32*j, weights, L + 32*j, &mins[j], Laux, -1.f, 0.1f, 20, false); + float scale = scales[j]; + if (scale > max_scale) { + max_scale = scale; + } + float min = mins[j]; + if (min > max_min) { + max_min = min; + } + } + + float inv_scale = max_scale > 0 ? 63.f/max_scale : 0.f; + float inv_min = max_min > 0 ? 63.f/max_min : 0.f; + for (int j = 0; j < QK_K/32; ++j) { + uint8_t ls = nearest_int(inv_scale*scales[j]); + uint8_t lm = nearest_int(inv_min*mins[j]); + ls = MIN(63, ls); + lm = MIN(63, lm); + if (j < 4) { + y[i].scales[j] = ls; + y[i].scales[j+4] = lm; + } else { + y[i].scales[j+4] = (ls & 0xF) | ((lm & 0xF) << 4); + y[i].scales[j-4] |= ((ls >> 4) << 6); + y[i].scales[j-0] |= ((lm >> 4) << 6); + } + } + y[i].d = GGML_FP32_TO_FP16(max_scale/63.f); + y[i].dmin = GGML_FP32_TO_FP16(max_min/63.f); + + uint8_t sc, m; + for (int j = 0; j < QK_K/32; ++j) { + get_scale_min_k4(j, y[i].scales, &sc, &m); + const float d = GGML_FP16_TO_FP32(y[i].d) * sc; + if (!d) continue; + const float dm = GGML_FP16_TO_FP32(y[i].dmin) * m; + for (int ii = 0; ii < 32; ++ii) { + int l = nearest_int((x[32*j + ii] + dm)/d); + l = MAX(0, MIN(15, l)); + L[32*j + ii] = l; + } + } + uint8_t * q = y[i].qs; + for (int j = 0; j < QK_K; j += 64) { + for (int l = 0; l < 32; ++l) q[l] = L[j + l] | (L[j + l + 32] << 4); + q += 32; + } + + x += QK_K; + + } +#endif +} + +size_t quantize_q4_K(const float * src, void * dst, int nrow, int n_per_row, int64_t * hist, const float * quant_weights) { + (void)hist; + int row_size = ggml_row_size(GGML_TYPE_Q4_K, n_per_row); + if (!quant_weights) { + quantize_row_q4_K_reference(src, dst, nrow*n_per_row); + } + else { + char * qrow = (char *)dst; + for (int row = 0; row < nrow; ++row) { + quantize_row_q4_K_impl(src, (block_q4_K*)qrow, n_per_row, quant_weights); + src += n_per_row; + qrow += row_size; + } + } + return nrow * row_size; +} + // ====================== 5-bit (de)-quantization void quantize_row_q5_K_reference(const float * restrict x, block_q5_K * restrict y, int k) { @@ -2349,7 +2557,7 @@ void quantize_row_q5_K_reference(const float * restrict x, block_q5_K * restrict #else float max_scale = 0, amax = 0; for (int j = 0; j < QK_K/16; ++j) { - scales[j] = make_qx_quants(16, 16, x + 16*j, L + 16*j, 1); + scales[j] = make_qx_quants(16, 16, x + 16*j, L + 16*j, 1, NULL); float abs_scale = fabsf(scales[j]); if (abs_scale > amax) { amax = abs_scale; @@ -2460,6 +2668,123 @@ size_t ggml_quantize_q5_K(const float * restrict src, void * restrict dst, int n return (n/QK_K*sizeof(block_q5_K)); } +static void quantize_row_q5_K_impl(const float * restrict x, block_q5_K * restrict y, int n_per_row, const float * quant_weights) { +#if QK_K != 256 + (void)quant_weights; + quantize_row_q5_K_reference(x, y, n_per_row); +#else + assert(n_per_row % QK_K == 0); + const int nb = n_per_row / QK_K; + + uint8_t L[QK_K]; + float mins[QK_K/32]; + float scales[QK_K/32]; + float weights[32]; + uint8_t Laux[32]; + + for (int i = 0; i < nb; i++) { + + float sum_x2 = 0; + for (int l = 0; l < QK_K; ++l) sum_x2 += x[l] * x[l]; + float sigma2 = sum_x2/QK_K; + float av_x = sqrtf(sigma2); + + float max_scale = 0; // as we are deducting the min, scales are always positive + float max_min = 0; + for (int j = 0; j < QK_K/32; ++j) { + if (quant_weights) { + const float * qw = quant_weights + QK_K*i + 32*j; + for (int l = 0; l < 32; ++l) weights[l] = qw[l] * sqrtf(sigma2 + x[32*j + l]*x[32*j + l]); + } else { + for (int l = 0; l < 32; ++l) weights[l] = av_x + fabsf(x[32*j + l]); + } + scales[j] = make_qkx3_quants(32, 31, x + 32*j, weights, L + 32*j, &mins[j], Laux, -0.9f, 0.05f, 36, false); + float scale = scales[j]; + if (scale > max_scale) { + max_scale = scale; + } + float min = mins[j]; + if (min > max_min) { + max_min = min; + } + } + + float inv_scale = max_scale > 0 ? 63.f/max_scale : 0.f; + float inv_min = max_min > 0 ? 63.f/max_min : 0.f; + for (int j = 0; j < QK_K/32; ++j) { + uint8_t ls = nearest_int(inv_scale*scales[j]); + uint8_t lm = nearest_int(inv_min*mins[j]); + ls = MIN(63, ls); + lm = MIN(63, lm); + if (j < 4) { + y[i].scales[j] = ls; + y[i].scales[j+4] = lm; + } else { + y[i].scales[j+4] = (ls & 0xF) | ((lm & 0xF) << 4); + y[i].scales[j-4] |= ((ls >> 4) << 6); + y[i].scales[j-0] |= ((lm >> 4) << 6); + } + } + y[i].d = GGML_FP32_TO_FP16(max_scale/63.f); + y[i].dmin = GGML_FP32_TO_FP16(max_min/63.f); + + uint8_t sc, m; + for (int j = 0; j < QK_K/32; ++j) { + get_scale_min_k4(j, y[i].scales, &sc, &m); + const float d = GGML_FP16_TO_FP32(y[i].d) * sc; + if (!d) continue; + const float dm = GGML_FP16_TO_FP32(y[i].dmin) * m; + for (int ii = 0; ii < 32; ++ii) { + int l = nearest_int((x[32*j + ii] + dm)/d); + l = MAX(0, MIN(31, l)); + L[32*j + ii] = l; + } + } + + uint8_t * restrict qh = y[i].qh; + uint8_t * restrict ql = y[i].qs; + memset(qh, 0, QK_K/8); + + uint8_t m1 = 1, m2 = 2; + for (int n = 0; n < QK_K; n += 64) { + for (int j = 0; j < 32; ++j) { + int l1 = L[n + j]; + if (l1 > 15) { + l1 -= 16; qh[j] |= m1; + } + int l2 = L[n + j + 32]; + if (l2 > 15) { + l2 -= 16; qh[j] |= m2; + } + ql[j] = l1 | (l2 << 4); + } + m1 <<= 2; m2 <<= 2; + ql += 32; + } + + x += QK_K; + + } +#endif +} + +size_t quantize_q5_K(const float * src, void * dst, int nrow, int n_per_row, int64_t * hist, const float * quant_weights) { + (void)hist; + int row_size = ggml_row_size(GGML_TYPE_Q5_K, n_per_row); + if (!quant_weights) { + quantize_row_q5_K_reference(src, dst, nrow*n_per_row); + } + else { + char * qrow = (char *)dst; + for (int row = 0; row < nrow; ++row) { + quantize_row_q5_K_impl(src, (block_q5_K*)qrow, n_per_row, quant_weights); + src += n_per_row; + qrow += row_size; + } + } + return nrow * row_size; +} + // ====================== 6-bit (de)-quantization void quantize_row_q6_K_reference(const float * restrict x, block_q6_K * restrict y, int k) { @@ -2476,7 +2801,7 @@ void quantize_row_q6_K_reference(const float * restrict x, block_q6_K * restrict for (int ib = 0; ib < QK_K/16; ++ib) { - const float scale = make_qx_quants(16, 32, x + 16*ib, L + 16*ib, 1); + const float scale = make_qx_quants(16, 32, x + 16*ib, L + 16*ib, 1, NULL); scales[ib] = scale; const float abs_scale = fabsf(scale); @@ -2608,6 +2933,112 @@ size_t ggml_quantize_q6_K(const float * src, void * dst, int n, int k, int64_t * return (n/QK_K*sizeof(block_q6_K)); } +static void quantize_row_q6_K_impl(const float * restrict x, block_q6_K * restrict y, int n_per_row, const float * quant_weights) { +#if QK_K != 256 + (void)quant_weights; + quantize_row_q6_K_reference(x, y, n_per_row); +#else + assert(n_per_row % QK_K == 0); + const int nb = n_per_row / QK_K; + + int8_t L[QK_K]; + float scales[QK_K/16]; + //float weights[16]; + + for (int i = 0; i < nb; i++) { + + //float sum_x2 = 0; + //for (int j = 0; j < QK_K; ++j) sum_x2 += x[j]*x[j]; + //float sigma2 = sum_x2/QK_K; + + float max_scale = 0; + float max_abs_scale = 0; + + for (int ib = 0; ib < QK_K/16; ++ib) { + + float scale; + if (quant_weights) { + const float * qw = quant_weights + QK_K*i + 16*ib; + //for (int j = 0; j < 16; ++j) weights[j] = qw[j] * sqrtf(sigma2 + x[16*ib + j]*x[16*ib + j]); + //scale = make_qx_quants(16, 32, x + 16*ib, L + 16*ib, 1, weights); + scale = make_qx_quants(16, 32, x + 16*ib, L + 16*ib, 1, qw); + } else { + scale = make_qx_quants(16, 32, x + 16*ib, L + 16*ib, 1, NULL); + } + scales[ib] = scale; + + const float abs_scale = fabsf(scale); + if (abs_scale > max_abs_scale) { + max_abs_scale = abs_scale; + max_scale = scale; + } + + } + + if (!max_abs_scale) { + memset(&y[i], 0, sizeof(block_q6_K)); + y[i].d = GGML_FP32_TO_FP16(0.f); + x += QK_K; + continue; + } + + float iscale = -128.f/max_scale; + y[i].d = GGML_FP32_TO_FP16(1/iscale); + for (int ib = 0; ib < QK_K/16; ++ib) { + y[i].scales[ib] = MIN(127, nearest_int(iscale*scales[ib])); + } + + for (int j = 0; j < QK_K/16; ++j) { + float d = GGML_FP16_TO_FP32(y[i].d) * y[i].scales[j]; + if (!d) { + continue; + } + for (int ii = 0; ii < 16; ++ii) { + int l = nearest_int(x[16*j + ii]/d); + l = MAX(-32, MIN(31, l)); + L[16*j + ii] = l + 32; + } + } + + uint8_t * restrict ql = y[i].ql; + uint8_t * restrict qh = y[i].qh; + for (int j = 0; j < QK_K; j += 128) { + for (int l = 0; l < 32; ++l) { + const uint8_t q1 = L[j + l + 0] & 0xF; + const uint8_t q2 = L[j + l + 32] & 0xF; + const uint8_t q3 = L[j + l + 64] & 0xF; + const uint8_t q4 = L[j + l + 96] & 0xF; + ql[l+ 0] = q1 | (q3 << 4); + ql[l+32] = q2 | (q4 << 4); + qh[l] = (L[j + l] >> 4) | ((L[j + l + 32] >> 4) << 2) | ((L[j + l + 64] >> 4) << 4) | ((L[j + l + 96] >> 4) << 6); + } + ql += 64; + qh += 32; + } + + x += QK_K; + + } +#endif +} + +size_t quantize_q6_K(const float * src, void * dst, int nrow, int n_per_row, int64_t * hist, const float * quant_weights) { + (void)hist; + int row_size = ggml_row_size(GGML_TYPE_Q6_K, n_per_row); + if (!quant_weights) { + quantize_row_q6_K_reference(src, dst, nrow*n_per_row); + } + else { + char * qrow = (char *)dst; + for (int row = 0; row < nrow; ++row) { + quantize_row_q6_K_impl(src, (block_q6_K*)qrow, n_per_row, quant_weights); + src += n_per_row; + qrow += row_size; + } + } + return nrow * row_size; +} + // ====================== "True" 2-bit (de)-quantization static const uint64_t iq2xxs_grid[256] = { diff --git a/ggml-quants.h b/ggml-quants.h index e5d110230..99467936a 100644 --- a/ggml-quants.h +++ b/ggml-quants.h @@ -249,4 +249,7 @@ void ggml_vec_dot_iq2_xs_q8_K (int n, float * restrict s, const void * restrict size_t quantize_iq2_xxs(const float * src, void * dst, int nrows, int n_per_row, int64_t * hist, const float * imatrix); size_t quantize_iq2_xs (const float * src, void * dst, int nrows, int n_per_row, int64_t * hist, const float * imatrix); size_t quantize_q2_K (const float * src, void * dst, int nrows, int n_per_row, int64_t * hist, const float * imatrix); - +size_t quantize_q3_K (const float * src, void * dst, int nrows, int n_per_row, int64_t * hist, const float * imatrix); +size_t quantize_q4_K (const float * src, void * dst, int nrows, int n_per_row, int64_t * hist, const float * imatrix); +size_t quantize_q5_K (const float * src, void * dst, int nrows, int n_per_row, int64_t * hist, const float * imatrix); +size_t quantize_q6_K (const float * src, void * dst, int nrows, int n_per_row, int64_t * hist, const float * imatrix); diff --git a/ggml.c b/ggml.c index 52467475a..ef5888ab2 100644 --- a/ggml.c +++ b/ggml.c @@ -18713,26 +18713,38 @@ size_t ggml_quantize_chunk(enum ggml_type type, const float * src, void * dst, i case GGML_TYPE_Q3_K: { GGML_ASSERT(start % QK_K == 0); - block_q3_K * block = (block_q3_K*)dst + start / QK_K; - result = ggml_quantize_q3_K(src + start, block, n, n, hist); + GGML_ASSERT(start % n_per_row == 0); + size_t start_row = start / n_per_row; + size_t row_size = ggml_row_size(type, n_per_row); + result = quantize_q3_K(src + start, (char *)dst + start_row * row_size, nrows, n_per_row, hist, imatrix); + GGML_ASSERT(result == row_size * nrows); } break; case GGML_TYPE_Q4_K: { GGML_ASSERT(start % QK_K == 0); - block_q4_K * block = (block_q4_K*)dst + start / QK_K; - result = ggml_quantize_q4_K(src + start, block, n, n, hist); + GGML_ASSERT(start % n_per_row == 0); + size_t start_row = start / n_per_row; + size_t row_size = ggml_row_size(type, n_per_row); + result = quantize_q4_K(src + start, (char *)dst + start_row * row_size, nrows, n_per_row, hist, imatrix); + GGML_ASSERT(result == row_size * nrows); } break; case GGML_TYPE_Q5_K: { GGML_ASSERT(start % QK_K == 0); - block_q5_K * block = (block_q5_K*)dst + start / QK_K; - result = ggml_quantize_q5_K(src + start, block, n, n, hist); + GGML_ASSERT(start % n_per_row == 0); + size_t start_row = start / n_per_row; + size_t row_size = ggml_row_size(type, n_per_row); + result = quantize_q5_K(src + start, (char *)dst + start_row * row_size, nrows, n_per_row, hist, imatrix); + GGML_ASSERT(result == row_size * nrows); } break; case GGML_TYPE_Q6_K: { GGML_ASSERT(start % QK_K == 0); - block_q6_K * block = (block_q6_K*)dst + start / QK_K; - result = ggml_quantize_q6_K(src + start, block, n, n, hist); + GGML_ASSERT(start % n_per_row == 0); + size_t start_row = start / n_per_row; + size_t row_size = ggml_row_size(type, n_per_row); + result = quantize_q6_K(src + start, (char *)dst + start_row * row_size, nrows, n_per_row, hist, imatrix); + GGML_ASSERT(result == row_size * nrows); } break; case GGML_TYPE_IQ2_XXS: { From a836c8f534ab789b02da149fbdaf7735500bff74 Mon Sep 17 00:00:00 2001 From: David Pflug Date: Sun, 14 Jan 2024 10:46:00 -0500 Subject: [PATCH 069/812] llama : fix missing quotes (#4937) --- llama.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/llama.cpp b/llama.cpp index 63f37ecdb..7af38718c 100644 --- a/llama.cpp +++ b/llama.cpp @@ -7099,7 +7099,7 @@ static std::vector llama_tokenize_internal(const llama_vocab & } #ifdef PRETOKENIZERDEBUG - LLAMA_LOG_WARN(TT: (%ld %ld %ld) '%s'\n", raw_text.length(), fragment.offset, fragment.length, raw_text.c_str()); + LLAMA_LOG_WARN("TT: (%ld %ld %ld) '%s'\n", raw_text.length(), fragment.offset, fragment.length, raw_text.c_str()); #endif llm_tokenizer_spm tokenizer(vocab); llama_escape_whitespace(raw_text); @@ -7120,7 +7120,7 @@ static std::vector llama_tokenize_internal(const llama_vocab & auto raw_text = fragment.raw_text.substr(fragment.offset, fragment.length); #ifdef PRETOKENIZERDEBUG - LLAMA_LOG_WARN(TT: (%ld %ld %ld) '%s'\n", raw_text.length(), fragment.offset, fragment.length, raw_text.c_str()); + LLAMA_LOG_WARN("TT: (%ld %ld %ld) '%s'\n", raw_text.length(), fragment.offset, fragment.length, raw_text.c_str()); #endif llm_tokenizer_bpe tokenizer(vocab); tokenizer.tokenize(raw_text, output); From 4a3156de2fac9a8ee4279de7804d4e352dcfe121 Mon Sep 17 00:00:00 2001 From: Kawrakow <48489457+ikawrakow@users.noreply.github.com> Date: Mon, 15 Jan 2024 07:48:06 +0200 Subject: [PATCH 070/812] CUDA: faster dequantize kernels for Q4_0 and Q4_1 (#4938) Co-authored-by: Iwan Kawrakow --- ggml-cuda.cu | 77 +++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 73 insertions(+), 4 deletions(-) diff --git a/ggml-cuda.cu b/ggml-cuda.cu index bd3814c72..a870718a7 100644 --- a/ggml-cuda.cu +++ b/ggml-cuda.cu @@ -1105,6 +1105,61 @@ static __device__ __forceinline__ void dequantize_q8_0(const void * vx, const in #endif // GGML_CUDA_F16 } +template +static __global__ void dequantize_block_q4_0(const void * __restrict__ vx, dst_t * __restrict__ yy, int nb32) { + + const int i = blockIdx.x; + + // assume 32 threads + const int tid = threadIdx.x; + const int il = tid/8; + const int ir = tid%8; + const int ib = 8*i + ir; + if (ib >= nb32) { + return; + } + + dst_t * y = yy + 256*i + 32*ir + 4*il; + + const block_q4_0 * x = (const block_q4_0 *)vx + ib; + const float d = __half2float(x->d); + const float dm = -8*d; + + const uint8_t * q = x->qs + 4*il; + + for (int l = 0; l < 4; ++l) { + y[l+ 0] = d * (q[l] & 0xF) + dm; + y[l+16] = d * (q[l] >> 4) + dm; + } +} + +template +static __global__ void dequantize_block_q4_1(const void * __restrict__ vx, dst_t * __restrict__ yy, int nb32) { + + const int i = blockIdx.x; + + // assume 32 threads + const int tid = threadIdx.x; + const int il = tid/8; + const int ir = tid%8; + const int ib = 8*i + ir; + if (ib >= nb32) { + return; + } + + dst_t * y = yy + 256*i + 32*ir + 4*il; + + const block_q4_1 * x = (const block_q4_1 *)vx + ib; + const float2 d = __half22float2(x->dm); + + const uint8_t * q = x->qs + 4*il; + + for (int l = 0; l < 4; ++l) { + y[l+ 0] = d.x * (q[l] & 0xF) + d.y; + y[l+16] = d.x * (q[l] >> 4) + d.y; + } +} + //================================== k-quants template @@ -6253,6 +6308,20 @@ static void dequantize_row_q3_K_cuda(const void * vx, dst_t * y, const int k, cu #endif } +template +static void dequantize_q4_0_cuda(const void * vx, dst_t * y, const int k, cudaStream_t stream) { + const int nb32 = k / 32; + const int nb = (k + 255) / 256; + dequantize_block_q4_0<<>>(vx, y, nb32); +} + +template +static void dequantize_q4_1_cuda(const void * vx, dst_t * y, const int k, cudaStream_t stream) { + const int nb32 = k / 32; + const int nb = (k + 255) / 256; + dequantize_block_q4_1<<>>(vx, y, nb32); +} + template static void dequantize_row_q4_K_cuda(const void * vx, dst_t * y, const int k, cudaStream_t stream) { const int nb = k / QK_K; @@ -6301,9 +6370,9 @@ static to_fp16_cuda_t ggml_get_to_fp16_cuda(ggml_type type) { int id; switch (type) { case GGML_TYPE_Q4_0: - return dequantize_block_cuda; + return dequantize_q4_0_cuda; case GGML_TYPE_Q4_1: - return dequantize_block_cuda; + return dequantize_q4_1_cuda; case GGML_TYPE_Q5_0: return dequantize_block_cuda; case GGML_TYPE_Q5_1: @@ -6338,9 +6407,9 @@ static to_fp16_cuda_t ggml_get_to_fp16_cuda(ggml_type type) { static to_fp32_cuda_t ggml_get_to_fp32_cuda(ggml_type type) { switch (type) { case GGML_TYPE_Q4_0: - return dequantize_block_cuda; + return dequantize_q4_0_cuda; case GGML_TYPE_Q4_1: - return dequantize_block_cuda; + return dequantize_q4_1_cuda; case GGML_TYPE_Q5_0: return dequantize_block_cuda; case GGML_TYPE_Q5_1: From 2faaef39799c97a53bec3898141478700da25757 Mon Sep 17 00:00:00 2001 From: Kawrakow <48489457+ikawrakow@users.noreply.github.com> Date: Mon, 15 Jan 2024 10:09:38 +0200 Subject: [PATCH 071/812] llama : check for 256 divisibility for IQ2_XS, IQ2_XXS (#4950) Co-authored-by: Iwan Kawrakow --- llama.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/llama.cpp b/llama.cpp index 7af38718c..f9718060d 100644 --- a/llama.cpp +++ b/llama.cpp @@ -8559,7 +8559,8 @@ static ggml_type get_k_quant_type(quantize_state_internal & qs, ggml_type new_ty //} bool convert_incompatible_tensor = false; if (new_type == GGML_TYPE_Q2_K || new_type == GGML_TYPE_Q3_K || new_type == GGML_TYPE_Q4_K || - new_type == GGML_TYPE_Q5_K || new_type == GGML_TYPE_Q6_K) { + new_type == GGML_TYPE_Q5_K || new_type == GGML_TYPE_Q6_K || + new_type == GGML_TYPE_IQ2_XS || new_type == GGML_TYPE_IQ2_XXS) { int nx = tensor->ne[0]; int ny = tensor->ne[1]; if (nx % QK_K != 0) { @@ -8571,6 +8572,8 @@ static ggml_type get_k_quant_type(quantize_state_internal & qs, ggml_type new_ty } if (convert_incompatible_tensor) { switch (new_type) { + case GGML_TYPE_IQ2_XXS: + case GGML_TYPE_IQ2_XS: case GGML_TYPE_Q2_K: new_type = GGML_TYPE_Q4_0; break; case GGML_TYPE_Q3_K: new_type = GGML_TYPE_Q4_1; break; case GGML_TYPE_Q4_K: new_type = GGML_TYPE_Q5_0; break; From ddb008d845cd50bb090bf051f570130524042936 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Mon, 15 Jan 2024 13:27:00 +0200 Subject: [PATCH 072/812] cuda : fix dequantize kernel names (#4938) --- ggml-cuda.cu | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/ggml-cuda.cu b/ggml-cuda.cu index a870718a7..c3e14bc96 100644 --- a/ggml-cuda.cu +++ b/ggml-cuda.cu @@ -6309,14 +6309,14 @@ static void dequantize_row_q3_K_cuda(const void * vx, dst_t * y, const int k, cu } template -static void dequantize_q4_0_cuda(const void * vx, dst_t * y, const int k, cudaStream_t stream) { +static void dequantize_row_q4_0_cuda(const void * vx, dst_t * y, const int k, cudaStream_t stream) { const int nb32 = k / 32; const int nb = (k + 255) / 256; dequantize_block_q4_0<<>>(vx, y, nb32); } template -static void dequantize_q4_1_cuda(const void * vx, dst_t * y, const int k, cudaStream_t stream) { +static void dequantize_row_q4_1_cuda(const void * vx, dst_t * y, const int k, cudaStream_t stream) { const int nb32 = k / 32; const int nb = (k + 255) / 256; dequantize_block_q4_1<<>>(vx, y, nb32); @@ -6370,9 +6370,9 @@ static to_fp16_cuda_t ggml_get_to_fp16_cuda(ggml_type type) { int id; switch (type) { case GGML_TYPE_Q4_0: - return dequantize_q4_0_cuda; + return dequantize_row_q4_0_cuda; case GGML_TYPE_Q4_1: - return dequantize_q4_1_cuda; + return dequantize_row_q4_1_cuda; case GGML_TYPE_Q5_0: return dequantize_block_cuda; case GGML_TYPE_Q5_1: @@ -6407,9 +6407,9 @@ static to_fp16_cuda_t ggml_get_to_fp16_cuda(ggml_type type) { static to_fp32_cuda_t ggml_get_to_fp32_cuda(ggml_type type) { switch (type) { case GGML_TYPE_Q4_0: - return dequantize_q4_0_cuda; + return dequantize_row_q4_0_cuda; case GGML_TYPE_Q4_1: - return dequantize_q4_1_cuda; + return dequantize_row_q4_1_cuda; case GGML_TYPE_Q5_0: return dequantize_block_cuda; case GGML_TYPE_Q5_1: From d9aa4ffa6e0296d42f1f676dd85de97c8491eb73 Mon Sep 17 00:00:00 2001 From: "Victor Z. Peng" Date: Mon, 15 Jan 2024 04:41:46 -0800 Subject: [PATCH 073/812] awq-py : fix typo in awq-py/README.md (#4947) --- awq-py/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/awq-py/README.md b/awq-py/README.md index 59354f4e3..16e68d027 100644 --- a/awq-py/README.md +++ b/awq-py/README.md @@ -43,7 +43,7 @@ Example for llama model # For llama7b and llama2 models python convert.py models/llama-7b/ --awq-path awq_cache/llama-7b-w4-g128.pt --outfile models/llama_7b_fp16.gguf # For mistral and mpt models -python convert-hf-to-gguf.py models/mpt-7b/ --awq-path awq_cache/llama-7b-w4-g128.pt --outfile models/mpt_7b_fp16.gguf +python convert-hf-to-gguf.py models/mpt-7b/ --awq-path awq_cache/mpt-7b-w4-g128.pt --outfile models/mpt_7b_fp16.gguf ``` ## Quantize From 4483396751c79dea540808b9cb9238245d06da2b Mon Sep 17 00:00:00 2001 From: David Friehs Date: Mon, 15 Jan 2024 14:06:52 +0100 Subject: [PATCH 074/812] llama : apply classifier-free guidance to logits directly (#4951) --- common/sampling.cpp | 9 ++++--- llama.cpp | 60 ++++++++++++++++++++++++++++++--------------- llama.h | 17 +++++++++---- 3 files changed, 57 insertions(+), 29 deletions(-) diff --git a/common/sampling.cpp b/common/sampling.cpp index 8e45909f1..dd1ffeb1b 100644 --- a/common/sampling.cpp +++ b/common/sampling.cpp @@ -190,6 +190,11 @@ static llama_token llama_sampling_sample_impl( logits[it->first] += it->second; } + if (ctx_cfg) { + float * logits_guidance = llama_get_logits_ith(ctx_cfg, idx); + llama_sample_apply_guidance(ctx_main, logits, logits_guidance, params.cfg_scale); + } + cur.clear(); for (llama_token token_id = 0; token_id < n_vocab; token_id++) { @@ -198,10 +203,6 @@ static llama_token llama_sampling_sample_impl( llama_token_data_array cur_p = { cur.data(), cur.size(), false }; - if (ctx_cfg) { - llama_sample_classifier_free_guidance(ctx_main, &cur_p, ctx_cfg, params.cfg_scale); - } - // apply penalties const auto& penalty_tokens = params.use_penalty_prompt_tokens ? params.penalty_prompt_tokens : prev; const int penalty_tokens_used_size = std::min((int)penalty_tokens.size(), penalty_last_n); diff --git a/llama.cpp b/llama.cpp index f9718060d..46c4d11c8 100644 --- a/llama.cpp +++ b/llama.cpp @@ -7898,39 +7898,59 @@ static void llama_log_softmax(float * array, size_t size) { } } +void llama_sample_apply_guidance( + struct llama_context * ctx, + float * logits, + float * logits_guidance, + float scale) { + GGML_ASSERT(ctx); + + const auto t_start_sample_us = ggml_time_us(); + const auto n_vocab = llama_n_vocab(llama_get_model(ctx)); + + llama_log_softmax(logits, n_vocab); + llama_log_softmax(logits_guidance, n_vocab); + + for (int i = 0; i < n_vocab; ++i) { + auto & l = logits[i]; + const auto & g = logits_guidance[i]; + + l = scale * (l - g) + g; + } + + ctx->t_sample_us += ggml_time_us() - t_start_sample_us; +} + void llama_sample_classifier_free_guidance( struct llama_context * ctx, llama_token_data_array * candidates, struct llama_context * guidance_ctx, float scale) { - int64_t t_start_sample_us = ggml_time_us(); - GGML_ASSERT(ctx); + int64_t t_start_sample_us; - auto n_vocab = llama_n_vocab(llama_get_model(ctx)); + t_start_sample_us = ggml_time_us(); + const size_t n_vocab = llama_n_vocab(llama_get_model(ctx)); - GGML_ASSERT(n_vocab == (int)candidates->size); + GGML_ASSERT(n_vocab == candidates->size); GGML_ASSERT(!candidates->sorted); - std::vector logits_base; - logits_base.reserve(candidates->size); - for (size_t i = 0; i < candidates->size; ++i) { - logits_base.push_back(candidates->data[i].logit); - } - llama_log_softmax(logits_base.data(), candidates->size); - - float* logits_guidance = llama_get_logits(guidance_ctx); - llama_log_softmax(logits_guidance, n_vocab); - - for (int i = 0; i < n_vocab; ++i) { - float logit_guidance = logits_guidance[i]; - float logit_base = logits_base[i]; - candidates->data[i].logit = scale * (logit_base - logit_guidance) + logit_guidance; + std::vector logits_base(n_vocab); + for (size_t i = 0; i < n_vocab; ++i) { + logits_base[i] = candidates->data[i].logit; } - if (ctx) { - ctx->t_sample_us += ggml_time_us() - t_start_sample_us; + float * logits_guidance = llama_get_logits(guidance_ctx); + + ctx->t_sample_us += ggml_time_us() - t_start_sample_us; + llama_sample_apply_guidance(ctx, logits_base.data(), logits_guidance, scale); + t_start_sample_us = ggml_time_us(); + + for (size_t i = 0; i < n_vocab; ++i) { + candidates->data[i].logit = logits_base[i]; } + + ctx->t_sample_us += ggml_time_us() - t_start_sample_us; } llama_token llama_sample_token_mirostat(struct llama_context * ctx, llama_token_data_array * candidates, float tau, float eta, int32_t m, float * mu) { diff --git a/llama.h b/llama.h index 79c8335b6..a570b0d69 100644 --- a/llama.h +++ b/llama.h @@ -714,14 +714,21 @@ extern "C" { float penalty_present); /// @details Apply classifier-free guidance to the logits as described in academic paper "Stay on topic with Classifier-Free Guidance" https://arxiv.org/abs/2306.17806 - /// @param candidates A vector of `llama_token_data` containing the candidate tokens, the logits must be directly extracted from the original generation context without being sorted. - /// @params guidance_ctx A separate context from the same model. Other than a negative prompt at the beginning, it should have all generated and user input tokens copied from the main context. - /// @params scale Guidance strength. 1.0f means no guidance. Higher values mean stronger guidance. - LLAMA_API void llama_sample_classifier_free_guidance( + /// @param logits Logits extracted from the original generation context. + /// @param logits_guidance Logits extracted from a separate context from the same model. Other than a negative prompt at the beginning, it should have all generated and user input tokens copied from the main context. + /// @param scale Guidance strength. 1.0f means no guidance. Higher values mean stronger guidance. + LLAMA_API void llama_sample_apply_guidance( + struct llama_context * ctx, + float * logits, + float * logits_guidance, + float scale); + + LLAMA_API DEPRECATED(void llama_sample_classifier_free_guidance( struct llama_context * ctx, llama_token_data_array * candidates, struct llama_context * guidance_ctx, - float scale); + float scale), + "use llama_sample_apply_guidance() instead"); /// @details Sorts candidate tokens by their logits in descending order and calculate probabilities based on logits. LLAMA_API void llama_sample_softmax( From 3e5ca7931c68152e4ec18d126e9c832dd84914c8 Mon Sep 17 00:00:00 2001 From: ngc92 <7938269+ngc92@users.noreply.github.com> Date: Mon, 15 Jan 2024 20:40:48 +0200 Subject: [PATCH 075/812] pass cpu-architecture arguments only to host code (C;C++) (#4943) --- CMakeLists.txt | 34 +++++++++++++++++++--------------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 2741568ed..7bd640966 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -594,6 +594,13 @@ if (NOT MSVC) endif() endif() +function(add_compile_option_cpp ARG) + # Adds a compile option to C/C++ only, but not for Cuda. + # Use, e.g., for CPU-architecture flags. + add_compile_options($<$:${ARG}>) + add_compile_options($<$:${ARG}>) +endfunction() + if ((${CMAKE_SYSTEM_PROCESSOR} MATCHES "arm") OR (${CMAKE_SYSTEM_PROCESSOR} MATCHES "aarch64") OR ("${CMAKE_GENERATOR_PLATFORM_LWR}" MATCHES "arm64")) message(STATUS "ARM detected") if (MSVC) @@ -628,8 +635,7 @@ elseif (${CMAKE_SYSTEM_PROCESSOR} MATCHES "^(x86_64|i686|AMD64)$" OR "${CMAKE_GE include(cmake/FindSIMD.cmake) endif () if (LLAMA_AVX512) - add_compile_options($<$:/arch:AVX512>) - add_compile_options($<$:/arch:AVX512>) + add_compile_option_cpp(/arch:AVX512) # MSVC has no compile-time flags enabling specific # AVX512 extensions, neither it defines the # macros corresponding to the extensions. @@ -643,37 +649,35 @@ elseif (${CMAKE_SYSTEM_PROCESSOR} MATCHES "^(x86_64|i686|AMD64)$" OR "${CMAKE_GE add_compile_definitions($<$:__AVX512VNNI__>) endif() elseif (LLAMA_AVX2) - add_compile_options($<$:/arch:AVX2>) - add_compile_options($<$:/arch:AVX2>) + add_compile_option_cpp(/arch:AVX2) elseif (LLAMA_AVX) - add_compile_options($<$:/arch:AVX>) - add_compile_options($<$:/arch:AVX>) + add_compile_option_cpp(/arch:AVX) endif() else() if (LLAMA_NATIVE) - add_compile_options(-march=native) + add_compile_option_cpp(-march=native) endif() if (LLAMA_F16C) - add_compile_options(-mf16c) + add_compile_option_cpp(-mf16c) endif() if (LLAMA_FMA) - add_compile_options(-mfma) + add_compile_option_cpp(-mfma) endif() if (LLAMA_AVX) - add_compile_options(-mavx) + add_compile_option_cpp(-mavx) endif() if (LLAMA_AVX2) - add_compile_options(-mavx2) + add_compile_option_cpp(-mavx2) endif() if (LLAMA_AVX512) - add_compile_options(-mavx512f) - add_compile_options(-mavx512bw) + add_compile_option_cpp(-mavx512f) + add_compile_option_cpp(-mavx512bw) endif() if (LLAMA_AVX512_VBMI) - add_compile_options(-mavx512vbmi) + add_compile_option_cpp(-mavx512vbmi) endif() if (LLAMA_AVX512_VNNI) - add_compile_options(-mavx512vnni) + add_compile_option_cpp(-mavx512vnni) endif() endif() elseif (${CMAKE_SYSTEM_PROCESSOR} MATCHES "ppc64") From e0324285a569d0583cf2f4a07a2402221ee25f58 Mon Sep 17 00:00:00 2001 From: stduhpf Date: Tue, 16 Jan 2024 12:04:32 +0100 Subject: [PATCH 076/812] speculative : threading options (#4959) * speculative: expose draft threading * fix usage format * accept -td and -tbd args * speculative: revert default behavior when -td is unspecified * fix trailing whitespace --- common/common.cpp | 22 ++++++++++++++++++++++ common/common.h | 2 ++ examples/speculative/speculative.cpp | 4 ++++ 3 files changed, 28 insertions(+) diff --git a/common/common.cpp b/common/common.cpp index c11006bcb..2b0865fff 100644 --- a/common/common.cpp +++ b/common/common.cpp @@ -167,6 +167,24 @@ bool gpt_params_parse_ex(int argc, char ** argv, gpt_params & params) { if (params.n_threads_batch <= 0) { params.n_threads_batch = std::thread::hardware_concurrency(); } + } else if (arg == "-td" || arg == "--threads-draft") { + if (++i >= argc) { + invalid_param = true; + break; + } + params.n_threads_draft = std::stoi(argv[i]); + if (params.n_threads_draft <= 0) { + params.n_threads_draft = std::thread::hardware_concurrency(); + } + } else if (arg == "-tbd" || arg == "--threads-batch-draft") { + if (++i >= argc) { + invalid_param = true; + break; + } + params.n_threads_batch_draft = std::stoi(argv[i]); + if (params.n_threads_batch_draft <= 0) { + params.n_threads_batch_draft = std::thread::hardware_concurrency(); + } } else if (arg == "-p" || arg == "--prompt") { if (++i >= argc) { invalid_param = true; @@ -845,6 +863,10 @@ void gpt_print_usage(int /*argc*/, char ** argv, const gpt_params & params) { printf(" -t N, --threads N number of threads to use during generation (default: %d)\n", params.n_threads); printf(" -tb N, --threads-batch N\n"); printf(" number of threads to use during batch and prompt processing (default: same as --threads)\n"); + printf(" -td N, --threads-draft N"); + printf(" number of threads to use during generation (default: same as --threads)"); + printf(" -tbd N, --threads-batch-draft N\n"); + printf(" number of threads to use during batch and prompt processing (default: same as --threads-draft)\n"); printf(" -p PROMPT, --prompt PROMPT\n"); printf(" prompt to start generation with (default: empty)\n"); printf(" -e, --escape process prompt escapes sequences (\\n, \\r, \\t, \\', \\\", \\\\)\n"); diff --git a/common/common.h b/common/common.h index 096468243..1f43e6282 100644 --- a/common/common.h +++ b/common/common.h @@ -46,7 +46,9 @@ struct gpt_params { uint32_t seed = -1; // RNG seed int32_t n_threads = get_num_physical_cores(); + int32_t n_threads_draft = -1; int32_t n_threads_batch = -1; // number of threads to use for batch processing (-1 = use n_threads) + int32_t n_threads_batch_draft = -1; int32_t n_predict = -1; // new tokens to predict int32_t n_ctx = 512; // context size int32_t n_batch = 512; // batch size for prompt processing (must be >=32 to use BLAS) diff --git a/examples/speculative/speculative.cpp b/examples/speculative/speculative.cpp index 20f1fb5bf..7b3af01f3 100644 --- a/examples/speculative/speculative.cpp +++ b/examples/speculative/speculative.cpp @@ -65,6 +65,10 @@ int main(int argc, char ** argv) { // load the draft model params.model = params.model_draft; params.n_gpu_layers = params.n_gpu_layers_draft; + if (params.n_threads_draft > 0) { + params.n_threads = params.n_threads_draft; + } + params.n_threads_batch = params.n_threads_batch_draft; std::tie(model_dft, ctx_dft) = llama_init_from_gpt_params(params); { From d75c232e1da56f19ac4d2530dadbe0ab3a11fde5 Mon Sep 17 00:00:00 2001 From: Daniel Bevenius Date: Tue, 16 Jan 2024 12:14:19 +0100 Subject: [PATCH 077/812] finetune : use LLAMA_FILE_MAGIC_GGLA (#4961) This commit replaces the magic number LLAMA_FILE_MAGIC_LORA used in finetune.cpp with LLAMA_FILE_MAGIC_GGLA defined in llama.h. Signed-off-by: Daniel Bevenius --- examples/finetune/finetune.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/examples/finetune/finetune.cpp b/examples/finetune/finetune.cpp index eaca42fc1..a6620fd73 100644 --- a/examples/finetune/finetune.cpp +++ b/examples/finetune/finetune.cpp @@ -1138,9 +1138,8 @@ static void save_as_llama_lora(const char * filename, struct my_llama_lora * lor return tn_buf.data(); }; - uint32_t LLAMA_FILE_MAGIC_LORA = 0x67676C61; // 'ggla' // write_magic - file.write_u32(LLAMA_FILE_MAGIC_LORA); // magic + file.write_u32(LLAMA_FILE_MAGIC_GGLA); // magic file.write_u32(1); // version // write_hparams file.write_u32(lora->hparams.lora_r); From a0b3ac8c48b66206b9c5921ce57bd5c0ea6557c3 Mon Sep 17 00:00:00 2001 From: Justine Tunney Date: Tue, 16 Jan 2024 03:16:33 -0800 Subject: [PATCH 078/812] ggml : introduce GGML_CALL function annotation (#4850) This change makes it possible to build ggml-cuda.cu and ggml-metal.m as independent dynamic shared objects, that may be conditionally linked at runtime in a multiplatform binary. It introduces a GGML_CALL annotation that documents which functions have a cyclic call relationship, between the application code and GPU modules. This change does nothing, unless the build defines -DGGML_MULTIPLATFORM which causes back-references and function pointers to conform to MS ABI which is supported by NVCC, ROCm, XCode, GCC and Clang across platforms --- ggml-backend-impl.h | 60 +++++++++++----------- ggml-backend.c | 80 ++++++++++++++--------------- ggml-backend.h | 50 +++++++++--------- ggml-cuda.cu | 121 ++++++++++++++++++++++---------------------- ggml-cuda.h | 32 ++++++------ ggml-metal.h | 4 +- ggml-metal.m | 42 +++++++-------- ggml.c | 32 ++++++------ ggml.h | 58 ++++++++++++--------- 9 files changed, 244 insertions(+), 235 deletions(-) diff --git a/ggml-backend-impl.h b/ggml-backend-impl.h index 1db32901f..1397828d9 100644 --- a/ggml-backend-impl.h +++ b/ggml-backend-impl.h @@ -16,14 +16,14 @@ extern "C" { typedef void * ggml_backend_buffer_type_context_t; struct ggml_backend_buffer_type_i { - const char * (*get_name) (ggml_backend_buffer_type_t buft); - ggml_backend_buffer_t (*alloc_buffer) (ggml_backend_buffer_type_t buft, size_t size); - size_t (*get_alignment) (ggml_backend_buffer_type_t buft); // tensor alignment - size_t (*get_alloc_size) (ggml_backend_buffer_type_t buft, const struct ggml_tensor * tensor); // data size needed to allocate the tensor, including padding - bool (*supports_backend)(ggml_backend_buffer_type_t buft, ggml_backend_t backend); // check if the buffer type is usable by the backend + const char * (*GGML_CALL get_name) (ggml_backend_buffer_type_t buft); + ggml_backend_buffer_t (*GGML_CALL alloc_buffer) (ggml_backend_buffer_type_t buft, size_t size); + size_t (*GGML_CALL get_alignment) (ggml_backend_buffer_type_t buft); // tensor alignment + size_t (*GGML_CALL get_alloc_size) (ggml_backend_buffer_type_t buft, const struct ggml_tensor * tensor); // data size needed to allocate the tensor, including padding + bool (*GGML_CALL supports_backend)(ggml_backend_buffer_type_t buft, ggml_backend_t backend); // check if the buffer type is usable by the backend // check if tensor data is in host memory // should be equivalent to supports_backend(buft, ggml_backend_cpu_init()) - bool (*is_host) (ggml_backend_buffer_type_t buft); + bool (*GGML_CALL is_host) (ggml_backend_buffer_type_t buft); }; struct ggml_backend_buffer_type { @@ -35,15 +35,15 @@ extern "C" { typedef void * ggml_backend_buffer_context_t; struct ggml_backend_buffer_i { - const char * (*get_name) (ggml_backend_buffer_t buffer); - void (*free_buffer)(ggml_backend_buffer_t buffer); - void * (*get_base) (ggml_backend_buffer_t buffer); - void (*init_tensor)(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor); - void (*set_tensor) (ggml_backend_buffer_t buffer, struct ggml_tensor * tensor, const void * data, size_t offset, size_t size); - void (*get_tensor) (ggml_backend_buffer_t buffer, const struct ggml_tensor * tensor, void * data, size_t offset, size_t size); - bool (*cpy_tensor) (ggml_backend_buffer_t buffer, const struct ggml_tensor * src, struct ggml_tensor * dst); // dst is in the buffer, src may be in any buffer - void (*clear) (ggml_backend_buffer_t buffer, uint8_t value); - void (*reset) (ggml_backend_buffer_t buffer); // reset any internal state due to tensor initialization, such as tensor extras + const char * (*GGML_CALL get_name) (ggml_backend_buffer_t buffer); + void (*GGML_CALL free_buffer)(ggml_backend_buffer_t buffer); + void * (*GGML_CALL get_base) (ggml_backend_buffer_t buffer); + void (*GGML_CALL init_tensor)(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor); + void (*GGML_CALL set_tensor) (ggml_backend_buffer_t buffer, struct ggml_tensor * tensor, const void * data, size_t offset, size_t size); + void (*GGML_CALL get_tensor) (ggml_backend_buffer_t buffer, const struct ggml_tensor * tensor, void * data, size_t offset, size_t size); + bool (*GGML_CALL cpy_tensor) (ggml_backend_buffer_t buffer, const struct ggml_tensor * src, struct ggml_tensor * dst); // dst is in the buffer, src may be in any buffer + void (*GGML_CALL clear) (ggml_backend_buffer_t buffer, uint8_t value); + void (*GGML_CALL reset) (ggml_backend_buffer_t buffer); // reset any internal state due to tensor initialization, such as tensor extras }; struct ggml_backend_buffer { @@ -54,7 +54,7 @@ extern "C" { enum ggml_backend_buffer_usage usage; }; - ggml_backend_buffer_t ggml_backend_buffer_init( + GGML_CALL ggml_backend_buffer_t ggml_backend_buffer_init( ggml_backend_buffer_type_t buft, struct ggml_backend_buffer_i iface, ggml_backend_buffer_context_t context, @@ -70,31 +70,31 @@ extern "C" { typedef void * ggml_backend_context_t; struct ggml_backend_i { - const char * (*get_name)(ggml_backend_t backend); + const char * (*GGML_CALL get_name)(ggml_backend_t backend); - void (*free)(ggml_backend_t backend); + void (*GGML_CALL free)(ggml_backend_t backend); // buffer allocation - ggml_backend_buffer_type_t (*get_default_buffer_type)(ggml_backend_t backend); + ggml_backend_buffer_type_t (*GGML_CALL get_default_buffer_type)(ggml_backend_t backend); // (optional) asynchronous tensor data access - void (*set_tensor_async)(ggml_backend_t backend, struct ggml_tensor * tensor, const void * data, size_t offset, size_t size); - void (*get_tensor_async)(ggml_backend_t backend, const struct ggml_tensor * tensor, void * data, size_t offset, size_t size); - bool (*cpy_tensor_async)(ggml_backend_t backend, const struct ggml_tensor * src, struct ggml_tensor * dst); + void (*GGML_CALL set_tensor_async)(ggml_backend_t backend, struct ggml_tensor * tensor, const void * data, size_t offset, size_t size); + void (*GGML_CALL get_tensor_async)(ggml_backend_t backend, const struct ggml_tensor * tensor, void * data, size_t offset, size_t size); + bool (*GGML_CALL cpy_tensor_async)(ggml_backend_t backend, const struct ggml_tensor * src, struct ggml_tensor * dst); // (optional) complete all pending operations - void (*synchronize)(ggml_backend_t backend); + void (*GGML_CALL synchronize)(ggml_backend_t backend); // compute graph with a plan - ggml_backend_graph_plan_t (*graph_plan_create) (ggml_backend_t backend, const struct ggml_cgraph * cgraph); - void (*graph_plan_free) (ggml_backend_t backend, ggml_backend_graph_plan_t plan); - void (*graph_plan_compute)(ggml_backend_t backend, ggml_backend_graph_plan_t plan); + ggml_backend_graph_plan_t (*GGML_CALL graph_plan_create) (ggml_backend_t backend, const struct ggml_cgraph * cgraph); + void (*GGML_CALL graph_plan_free) (ggml_backend_t backend, ggml_backend_graph_plan_t plan); + void (*GGML_CALL graph_plan_compute)(ggml_backend_t backend, ggml_backend_graph_plan_t plan); // compute graph without a plan (async) - bool (*graph_compute)(ggml_backend_t backend, struct ggml_cgraph * cgraph); + bool (*GGML_CALL graph_compute)(ggml_backend_t backend, struct ggml_cgraph * cgraph); // check if the backend supports an operation - bool (*supports_op)(ggml_backend_t backend, const struct ggml_tensor * op); + bool (*GGML_CALL supports_op)(ggml_backend_t backend, const struct ggml_tensor * op); }; struct ggml_backend { @@ -107,9 +107,9 @@ extern "C" { // Backend registry // - typedef ggml_backend_t (*ggml_backend_init_fn)(const char * params, void * user_data); + typedef ggml_backend_t (*GGML_CALL ggml_backend_init_fn)(const char * params, void * user_data); - void ggml_backend_register(const char * name, ggml_backend_init_fn init_fn, ggml_backend_buffer_type_t default_buffer_type, void * user_data); + GGML_CALL void ggml_backend_register(const char * name, ggml_backend_init_fn init_fn, ggml_backend_buffer_type_t default_buffer_type, void * user_data); #ifdef __cplusplus } diff --git a/ggml-backend.c b/ggml-backend.c index 505dbba47..f5424fb90 100644 --- a/ggml-backend.c +++ b/ggml-backend.c @@ -19,7 +19,7 @@ const char * ggml_backend_buft_name(ggml_backend_buffer_type_t buft) { return buft->iface.get_name(buft); } -ggml_backend_buffer_t ggml_backend_buft_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size) { +GGML_CALL ggml_backend_buffer_t ggml_backend_buft_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size) { return buft->iface.alloc_buffer(buft, size); } @@ -27,7 +27,7 @@ size_t ggml_backend_buft_get_alignment(ggml_backend_buffer_type_t buft) { return buft->iface.get_alignment(buft); } -size_t ggml_backend_buft_get_alloc_size(ggml_backend_buffer_type_t buft, struct ggml_tensor * tensor) { +GGML_CALL size_t ggml_backend_buft_get_alloc_size(ggml_backend_buffer_type_t buft, struct ggml_tensor * tensor) { // get_alloc_size is optional, defaults to ggml_nbytes if (buft->iface.get_alloc_size) { return buft->iface.get_alloc_size(buft, tensor); @@ -48,7 +48,7 @@ bool ggml_backend_buft_is_host(ggml_backend_buffer_type_t buft) { // backend buffer -ggml_backend_buffer_t ggml_backend_buffer_init( +GGML_CALL ggml_backend_buffer_t ggml_backend_buffer_init( ggml_backend_buffer_type_t buft, struct ggml_backend_buffer_i iface, ggml_backend_buffer_context_t context, @@ -95,7 +95,7 @@ void * ggml_backend_buffer_get_base(ggml_backend_buffer_t buffer) { return base; } -void ggml_backend_buffer_init_tensor(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor) { +GGML_CALL void ggml_backend_buffer_init_tensor(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor) { // init_tensor is optional if (buffer->iface.init_tensor) { buffer->iface.init_tensor(buffer, tensor); @@ -191,7 +191,7 @@ void ggml_backend_tensor_get_async(ggml_backend_t backend, const struct ggml_ten } } -void ggml_backend_tensor_set(struct ggml_tensor * tensor, const void * data, size_t offset, size_t size) { +GGML_CALL void ggml_backend_tensor_set(struct ggml_tensor * tensor, const void * data, size_t offset, size_t size) { ggml_backend_buffer_t buf = tensor->view_src ? tensor->view_src->buffer : tensor->buffer; GGML_ASSERT(tensor->data != NULL && "tensor not allocated"); @@ -201,7 +201,7 @@ void ggml_backend_tensor_set(struct ggml_tensor * tensor, const void * data, siz tensor->buffer->iface.set_tensor(buf, tensor, data, offset, size); } -void ggml_backend_tensor_get(const struct ggml_tensor * tensor, void * data, size_t offset, size_t size) { +GGML_CALL void ggml_backend_tensor_get(const struct ggml_tensor * tensor, void * data, size_t offset, size_t size) { ggml_backend_buffer_t buf = tensor->view_src ? tensor->view_src->buffer : tensor->buffer; GGML_ASSERT(tensor->data != NULL && "tensor not allocated"); @@ -318,9 +318,9 @@ struct ggml_backend_reg { static struct ggml_backend_reg ggml_backend_registry[GGML_MAX_BACKENDS_REG]; static size_t ggml_backend_registry_count = 0; -static ggml_backend_t ggml_backend_reg_cpu_init(const char * params, void * user_data); +GGML_CALL static ggml_backend_t ggml_backend_reg_cpu_init(const char * params, void * user_data); -static void ggml_backend_registry_init(void) { +GGML_CALL static void ggml_backend_registry_init(void) { static bool initialized = false; if (initialized) { @@ -333,18 +333,18 @@ static void ggml_backend_registry_init(void) { // add forward decls here to avoid including the backend headers #ifdef GGML_USE_CUBLAS - extern void ggml_backend_cuda_reg_devices(void); + extern GGML_CALL void ggml_backend_cuda_reg_devices(void); ggml_backend_cuda_reg_devices(); #endif #ifdef GGML_USE_METAL - extern ggml_backend_t ggml_backend_reg_metal_init(const char * params, void * user_data); - extern ggml_backend_buffer_type_t ggml_backend_metal_buffer_type(void); + extern GGML_CALL ggml_backend_t ggml_backend_reg_metal_init(const char * params, void * user_data); + extern GGML_CALL ggml_backend_buffer_type_t ggml_backend_metal_buffer_type(void); ggml_backend_register("Metal", ggml_backend_reg_metal_init, ggml_backend_metal_buffer_type(), NULL); #endif } -void ggml_backend_register(const char * name, ggml_backend_init_fn init_fn, ggml_backend_buffer_type_t default_buffer_type, void * user_data) { +GGML_CALL void ggml_backend_register(const char * name, ggml_backend_init_fn init_fn, ggml_backend_buffer_type_t default_buffer_type, void * user_data) { GGML_ASSERT(ggml_backend_registry_count < GGML_MAX_BACKENDS_REG); size_t id = ggml_backend_registry_count; @@ -439,33 +439,33 @@ ggml_backend_buffer_t ggml_backend_reg_alloc_buffer(size_t i, size_t size) { // backend CPU -static const char * ggml_backend_cpu_buffer_name(ggml_backend_buffer_t buffer) { +GGML_CALL static const char * ggml_backend_cpu_buffer_name(ggml_backend_buffer_t buffer) { return "CPU"; GGML_UNUSED(buffer); } -static void * ggml_backend_cpu_buffer_get_base(ggml_backend_buffer_t buffer) { +GGML_CALL static void * ggml_backend_cpu_buffer_get_base(ggml_backend_buffer_t buffer) { return (void *)buffer->context; } -static void ggml_backend_cpu_buffer_free_buffer(ggml_backend_buffer_t buffer) { +GGML_CALL static void ggml_backend_cpu_buffer_free_buffer(ggml_backend_buffer_t buffer) { free(buffer->context); } -static void ggml_backend_cpu_buffer_set_tensor(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor, const void * data, size_t offset, size_t size) { +GGML_CALL static void ggml_backend_cpu_buffer_set_tensor(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor, const void * data, size_t offset, size_t size) { memcpy((char *)tensor->data + offset, data, size); GGML_UNUSED(buffer); } -static void ggml_backend_cpu_buffer_get_tensor(ggml_backend_buffer_t buffer, const struct ggml_tensor * tensor, void * data, size_t offset, size_t size) { +GGML_CALL static void ggml_backend_cpu_buffer_get_tensor(ggml_backend_buffer_t buffer, const struct ggml_tensor * tensor, void * data, size_t offset, size_t size) { memcpy(data, (const char *)tensor->data + offset, size); GGML_UNUSED(buffer); } -static bool ggml_backend_cpu_buffer_cpy_tensor(ggml_backend_buffer_t buffer, const struct ggml_tensor * src, struct ggml_tensor * dst) { +GGML_CALL static bool ggml_backend_cpu_buffer_cpy_tensor(ggml_backend_buffer_t buffer, const struct ggml_tensor * src, struct ggml_tensor * dst) { if (ggml_backend_buffer_is_host(src->buffer)) { memcpy(dst->data, src->data, ggml_nbytes(src)); return true; @@ -475,7 +475,7 @@ static bool ggml_backend_cpu_buffer_cpy_tensor(ggml_backend_buffer_t buffer, con GGML_UNUSED(buffer); } -static void ggml_backend_cpu_buffer_clear(ggml_backend_buffer_t buffer, uint8_t value) { +GGML_CALL static void ggml_backend_cpu_buffer_clear(ggml_backend_buffer_t buffer, uint8_t value) { memset(buffer->context, value, buffer->size); } @@ -506,13 +506,13 @@ static struct ggml_backend_buffer_i cpu_backend_buffer_i_from_ptr = { static const size_t TENSOR_ALIGNMENT = 64; // should be enough for AVX 512 -static const char * ggml_backend_cpu_buffer_type_get_name(ggml_backend_buffer_type_t buft) { +GGML_CALL static const char * ggml_backend_cpu_buffer_type_get_name(ggml_backend_buffer_type_t buft) { return "CPU"; GGML_UNUSED(buft); } -static ggml_backend_buffer_t ggml_backend_cpu_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size) { +GGML_CALL static ggml_backend_buffer_t ggml_backend_cpu_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size) { size += TENSOR_ALIGNMENT; // malloc may return an address that is not aligned void * data = malloc(size); // TODO: maybe use GGML_ALIGNED_MALLOC? @@ -521,25 +521,25 @@ static ggml_backend_buffer_t ggml_backend_cpu_buffer_type_alloc_buffer(ggml_back return ggml_backend_buffer_init(buft, cpu_backend_buffer_i, data, size); } -static size_t ggml_backend_cpu_buffer_type_get_alignment(ggml_backend_buffer_type_t buft) { +GGML_CALL static size_t ggml_backend_cpu_buffer_type_get_alignment(ggml_backend_buffer_type_t buft) { return TENSOR_ALIGNMENT; GGML_UNUSED(buft); } -static bool ggml_backend_cpu_buffer_type_supports_backend(ggml_backend_buffer_type_t buft, ggml_backend_t backend) { +GGML_CALL static bool ggml_backend_cpu_buffer_type_supports_backend(ggml_backend_buffer_type_t buft, ggml_backend_t backend) { return ggml_backend_is_cpu(backend); GGML_UNUSED(buft); } -static bool ggml_backend_cpu_buffer_type_is_host(ggml_backend_buffer_type_t buft) { +GGML_CALL static bool ggml_backend_cpu_buffer_type_is_host(ggml_backend_buffer_type_t buft) { return true; GGML_UNUSED(buft); } -ggml_backend_buffer_type_t ggml_backend_cpu_buffer_type(void) { +GGML_CALL ggml_backend_buffer_type_t ggml_backend_cpu_buffer_type(void) { static struct ggml_backend_buffer_type ggml_backend_cpu_buffer_type = { /* .iface = */ { /* .get_name = */ ggml_backend_cpu_buffer_type_get_name, @@ -561,23 +561,23 @@ ggml_backend_buffer_type_t ggml_backend_cpu_buffer_type(void) { #include -static const char * ggml_backend_cpu_hbm_buffer_type_get_name(ggml_backend_buffer_type_t buft) { +GGML_CALL static const char * ggml_backend_cpu_hbm_buffer_type_get_name(ggml_backend_buffer_type_t buft) { return "CPU_HBM"; GGML_UNUSED(buft); } -static const char * ggml_backend_cpu_hbm_buffer_get_name(ggml_backend_buffer_t buf) { +GGML_CALL static const char * ggml_backend_cpu_hbm_buffer_get_name(ggml_backend_buffer_t buf) { return "CPU_HBM"; GGML_UNUSED(buf); } -static void ggml_backend_cpu_hbm_buffer_free_buffer(ggml_backend_buffer_t buffer) { +GGML_CALL static void ggml_backend_cpu_hbm_buffer_free_buffer(ggml_backend_buffer_t buffer) { hbw_free(buffer->context); } -static ggml_backend_buffer_t ggml_backend_cpu_hbm_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size) { +GGML_CALL static ggml_backend_buffer_t ggml_backend_cpu_hbm_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size) { //void * ptr = hbw_malloc(size); void * ptr; int result = hbw_posix_memalign(&ptr, ggml_backend_cpu_buffer_type_get_alignment(buft), size); @@ -617,20 +617,20 @@ struct ggml_backend_cpu_context { size_t work_size; }; -static const char * ggml_backend_cpu_name(ggml_backend_t backend) { +GGML_CALL static const char * ggml_backend_cpu_name(ggml_backend_t backend) { return "CPU"; GGML_UNUSED(backend); } -static void ggml_backend_cpu_free(ggml_backend_t backend) { +GGML_CALL static void ggml_backend_cpu_free(ggml_backend_t backend) { struct ggml_backend_cpu_context * cpu_ctx = (struct ggml_backend_cpu_context *)backend->context; free(cpu_ctx->work_data); free(cpu_ctx); free(backend); } -static ggml_backend_buffer_type_t ggml_backend_cpu_get_default_buffer_type(ggml_backend_t backend) { +GGML_CALL static ggml_backend_buffer_type_t ggml_backend_cpu_get_default_buffer_type(ggml_backend_t backend) { return ggml_backend_cpu_buffer_type(); GGML_UNUSED(backend); @@ -641,7 +641,7 @@ struct ggml_backend_plan_cpu { struct ggml_cgraph cgraph; }; -static ggml_backend_graph_plan_t ggml_backend_cpu_graph_plan_create(ggml_backend_t backend, const struct ggml_cgraph * cgraph) { +GGML_CALL static ggml_backend_graph_plan_t ggml_backend_cpu_graph_plan_create(ggml_backend_t backend, const struct ggml_cgraph * cgraph) { struct ggml_backend_cpu_context * cpu_ctx = (struct ggml_backend_cpu_context *)backend->context; struct ggml_backend_plan_cpu * cpu_plan = malloc(sizeof(struct ggml_backend_plan_cpu)); @@ -656,7 +656,7 @@ static ggml_backend_graph_plan_t ggml_backend_cpu_graph_plan_create(ggml_backend return cpu_plan; } -static void ggml_backend_cpu_graph_plan_free(ggml_backend_t backend, ggml_backend_graph_plan_t plan) { +GGML_CALL static void ggml_backend_cpu_graph_plan_free(ggml_backend_t backend, ggml_backend_graph_plan_t plan) { struct ggml_backend_plan_cpu * cpu_plan = (struct ggml_backend_plan_cpu *)plan; free(cpu_plan->cplan.work_data); @@ -665,7 +665,7 @@ static void ggml_backend_cpu_graph_plan_free(ggml_backend_t backend, ggml_backen GGML_UNUSED(backend); } -static void ggml_backend_cpu_graph_plan_compute(ggml_backend_t backend, ggml_backend_graph_plan_t plan) { +GGML_CALL static void ggml_backend_cpu_graph_plan_compute(ggml_backend_t backend, ggml_backend_graph_plan_t plan) { struct ggml_backend_plan_cpu * cpu_plan = (struct ggml_backend_plan_cpu *)plan; ggml_graph_compute(&cpu_plan->cgraph, &cpu_plan->cplan); @@ -673,7 +673,7 @@ static void ggml_backend_cpu_graph_plan_compute(ggml_backend_t backend, ggml_bac GGML_UNUSED(backend); } -static bool ggml_backend_cpu_graph_compute(ggml_backend_t backend, struct ggml_cgraph * cgraph) { +GGML_CALL static bool ggml_backend_cpu_graph_compute(ggml_backend_t backend, struct ggml_cgraph * cgraph) { struct ggml_backend_cpu_context * cpu_ctx = (struct ggml_backend_cpu_context *)backend->context; struct ggml_cplan cplan = ggml_graph_plan(cgraph, cpu_ctx->n_threads); @@ -690,7 +690,7 @@ static bool ggml_backend_cpu_graph_compute(ggml_backend_t backend, struct ggml_c return true; } -static bool ggml_backend_cpu_supports_op(ggml_backend_t backend, const struct ggml_tensor * op) { +GGML_CALL static bool ggml_backend_cpu_supports_op(ggml_backend_t backend, const struct ggml_tensor * op) { switch (op->op) { case GGML_OP_MUL_MAT: return op->src[1]->type == GGML_TYPE_F32 || op->src[1]->type == ggml_internal_get_type_traits(op->src[0]->type).vec_dot_type; @@ -732,7 +732,7 @@ ggml_backend_t ggml_backend_cpu_init(void) { return cpu_backend; } -bool ggml_backend_is_cpu(ggml_backend_t backend) { +GGML_CALL bool ggml_backend_is_cpu(ggml_backend_t backend) { return backend && backend->iface.get_name == ggml_backend_cpu_name; } @@ -743,11 +743,11 @@ void ggml_backend_cpu_set_n_threads(ggml_backend_t backend_cpu, int n_threads) { ctx->n_threads = n_threads; } -ggml_backend_buffer_t ggml_backend_cpu_buffer_from_ptr(void * ptr, size_t size) { +GGML_CALL ggml_backend_buffer_t ggml_backend_cpu_buffer_from_ptr(void * ptr, size_t size) { return ggml_backend_buffer_init(ggml_backend_cpu_buffer_type(), cpu_backend_buffer_i_from_ptr, ptr, size); } -static ggml_backend_t ggml_backend_reg_cpu_init(const char * params, void * user_data) { +GGML_CALL static ggml_backend_t ggml_backend_reg_cpu_init(const char * params, void * user_data) { return ggml_backend_cpu_init(); GGML_UNUSED(params); diff --git a/ggml-backend.h b/ggml-backend.h index 4eb244af1..12b4b4ab7 100644 --- a/ggml-backend.h +++ b/ggml-backend.h @@ -17,12 +17,12 @@ extern "C" { // // buffer type - GGML_API const char * ggml_backend_buft_name (ggml_backend_buffer_type_t buft); - GGML_API ggml_backend_buffer_t ggml_backend_buft_alloc_buffer (ggml_backend_buffer_type_t buft, size_t size); - GGML_API size_t ggml_backend_buft_get_alignment (ggml_backend_buffer_type_t buft); - GGML_API size_t ggml_backend_buft_get_alloc_size (ggml_backend_buffer_type_t buft, struct ggml_tensor * tensor); - GGML_API bool ggml_backend_buft_supports_backend(ggml_backend_buffer_type_t buft, ggml_backend_t backend); - GGML_API bool ggml_backend_buft_is_host (ggml_backend_buffer_type_t buft); + GGML_API const char * ggml_backend_buft_name (ggml_backend_buffer_type_t buft); + GGML_API GGML_CALL ggml_backend_buffer_t ggml_backend_buft_alloc_buffer (ggml_backend_buffer_type_t buft, size_t size); + GGML_API size_t ggml_backend_buft_get_alignment (ggml_backend_buffer_type_t buft); + GGML_API GGML_CALL size_t ggml_backend_buft_get_alloc_size (ggml_backend_buffer_type_t buft, struct ggml_tensor * tensor); + GGML_API bool ggml_backend_buft_supports_backend(ggml_backend_buffer_type_t buft, ggml_backend_t backend); + GGML_API bool ggml_backend_buft_is_host (ggml_backend_buffer_type_t buft); // buffer enum ggml_backend_buffer_usage { @@ -30,18 +30,18 @@ extern "C" { GGML_BACKEND_BUFFER_USAGE_WEIGHTS = 1, }; - GGML_API const char * ggml_backend_buffer_name (ggml_backend_buffer_t buffer); - GGML_API void ggml_backend_buffer_free (ggml_backend_buffer_t buffer); - GGML_API void * ggml_backend_buffer_get_base (ggml_backend_buffer_t buffer); - GGML_API size_t ggml_backend_buffer_get_size (ggml_backend_buffer_t buffer); - GGML_API void ggml_backend_buffer_init_tensor (ggml_backend_buffer_t buffer, struct ggml_tensor * tensor); - GGML_API size_t ggml_backend_buffer_get_alignment (ggml_backend_buffer_t buffer); - GGML_API size_t ggml_backend_buffer_get_alloc_size(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor); - GGML_API void ggml_backend_buffer_clear (ggml_backend_buffer_t buffer, uint8_t value); - GGML_API bool ggml_backend_buffer_is_host (ggml_backend_buffer_t buffer); - GGML_API void ggml_backend_buffer_set_usage (ggml_backend_buffer_t buffer, enum ggml_backend_buffer_usage usage); - GGML_API ggml_backend_buffer_type_t ggml_backend_buffer_get_type (ggml_backend_buffer_t buffer); - GGML_API void ggml_backend_buffer_reset (ggml_backend_buffer_t buffer); + GGML_API const char * ggml_backend_buffer_name (ggml_backend_buffer_t buffer); + GGML_API void ggml_backend_buffer_free (ggml_backend_buffer_t buffer); + GGML_API void * ggml_backend_buffer_get_base (ggml_backend_buffer_t buffer); + GGML_API size_t ggml_backend_buffer_get_size (ggml_backend_buffer_t buffer); + GGML_API GGML_CALL void ggml_backend_buffer_init_tensor (ggml_backend_buffer_t buffer, struct ggml_tensor * tensor); + GGML_API size_t ggml_backend_buffer_get_alignment (ggml_backend_buffer_t buffer); + GGML_API size_t ggml_backend_buffer_get_alloc_size(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor); + GGML_API void ggml_backend_buffer_clear (ggml_backend_buffer_t buffer, uint8_t value); + GGML_API bool ggml_backend_buffer_is_host (ggml_backend_buffer_t buffer); + GGML_API void ggml_backend_buffer_set_usage (ggml_backend_buffer_t buffer, enum ggml_backend_buffer_usage usage); + GGML_API ggml_backend_buffer_type_t ggml_backend_buffer_get_type (ggml_backend_buffer_t buffer); + GGML_API void ggml_backend_buffer_reset (ggml_backend_buffer_t buffer); // // Backend @@ -58,8 +58,8 @@ extern "C" { GGML_API void ggml_backend_tensor_set_async(ggml_backend_t backend, struct ggml_tensor * tensor, const void * data, size_t offset, size_t size); GGML_API void ggml_backend_tensor_get_async(ggml_backend_t backend, const struct ggml_tensor * tensor, void * data, size_t offset, size_t size); - GGML_API void ggml_backend_tensor_set( struct ggml_tensor * tensor, const void * data, size_t offset, size_t size); - GGML_API void ggml_backend_tensor_get(const struct ggml_tensor * tensor, void * data, size_t offset, size_t size); + GGML_API GGML_CALL void ggml_backend_tensor_set( struct ggml_tensor * tensor, const void * data, size_t offset, size_t size); + GGML_API GGML_CALL void ggml_backend_tensor_get(const struct ggml_tensor * tensor, void * data, size_t offset, size_t size); GGML_API void ggml_backend_synchronize(ggml_backend_t backend); @@ -80,13 +80,13 @@ extern "C" { GGML_API ggml_backend_t ggml_backend_cpu_init(void); - GGML_API bool ggml_backend_is_cpu(ggml_backend_t backend); - GGML_API void ggml_backend_cpu_set_n_threads(ggml_backend_t backend_cpu, int n_threads); + GGML_API GGML_CALL bool ggml_backend_is_cpu (ggml_backend_t backend); + GGML_API void ggml_backend_cpu_set_n_threads(ggml_backend_t backend_cpu, int n_threads); // Create a backend buffer from an existing pointer - GGML_API ggml_backend_buffer_t ggml_backend_cpu_buffer_from_ptr(void * ptr, size_t size); + GGML_API GGML_CALL ggml_backend_buffer_t ggml_backend_cpu_buffer_from_ptr(void * ptr, size_t size); - GGML_API ggml_backend_buffer_type_t ggml_backend_cpu_buffer_type(void); + GGML_API GGML_CALL ggml_backend_buffer_type_t ggml_backend_cpu_buffer_type(void); #ifdef GGML_USE_CPU_HBM GGML_API ggml_backend_buffer_type_t ggml_backend_cpu_hbm_buffer_type(void); @@ -183,7 +183,7 @@ extern "C" { GGML_API struct ggml_backend_graph_copy ggml_backend_graph_copy(ggml_backend_t backend, struct ggml_cgraph * graph); GGML_API void ggml_backend_graph_copy_free(struct ggml_backend_graph_copy copy); - typedef bool (*ggml_backend_eval_callback)(int node_index, struct ggml_tensor * t1, struct ggml_tensor * t2, void * user_data); + typedef bool (*GGML_CALL ggml_backend_eval_callback)(int node_index, struct ggml_tensor * t1, struct ggml_tensor * t2, void * user_data); // Compare the output of two backends GGML_API bool ggml_backend_compare_graph_backend(ggml_backend_t backend1, ggml_backend_t backend2, struct ggml_cgraph * graph, ggml_backend_eval_callback callback, void * user_data); diff --git a/ggml-cuda.cu b/ggml-cuda.cu index c3e14bc96..568c411af 100644 --- a/ggml-cuda.cu +++ b/ggml-cuda.cu @@ -7615,11 +7615,11 @@ struct cuda_pool_alloc { static bool g_cublas_loaded = false; -bool ggml_cublas_loaded(void) { +GGML_CALL bool ggml_cublas_loaded(void) { return g_cublas_loaded; } -void ggml_init_cublas() { +GGML_CALL void ggml_init_cublas() { static bool initialized = false; if (!initialized) { @@ -7707,7 +7707,7 @@ void ggml_init_cublas() { } } -void * ggml_cuda_host_malloc(size_t size) { +GGML_CALL void * ggml_cuda_host_malloc(size_t size) { if (getenv("GGML_CUDA_NO_PINNED") != nullptr) { return nullptr; } @@ -7725,7 +7725,7 @@ void * ggml_cuda_host_malloc(size_t size) { return ptr; } -void ggml_cuda_host_free(void * ptr) { +GGML_CALL void ggml_cuda_host_free(void * ptr) { CUDA_CHECK(cudaFreeHost(ptr)); } @@ -9242,7 +9242,7 @@ static void ggml_cuda_rms_norm(const ggml_tensor * src0, const ggml_tensor * src ggml_cuda_op_flatten(src0, src1, dst, ggml_cuda_op_rms_norm); } -bool ggml_cuda_can_mul_mat(const struct ggml_tensor * src0, const struct ggml_tensor * src1, struct ggml_tensor * dst) { +GGML_CALL bool ggml_cuda_can_mul_mat(const struct ggml_tensor * src0, const struct ggml_tensor * src1, struct ggml_tensor * dst) { if (!g_cublas_loaded) return false; const int64_t ne10 = src1->ne[0]; @@ -10013,7 +10013,7 @@ static size_t ggml_nbytes_split(const struct ggml_tensor * tensor, int nrows_spl return nrows_split*ggml_row_size(tensor->type, tensor->ne[0]); } -static void ggml_cuda_set_main_device(const int main_device) { +GGML_CALL static void ggml_cuda_set_main_device(const int main_device) { if (main_device >= g_device_count) { fprintf(stderr, "warning: cannot set main_device=%d because there are only %d devices. Using device %d instead.\n", main_device, g_device_count, g_main_device); @@ -10028,7 +10028,7 @@ static void ggml_cuda_set_main_device(const int main_device) { } } -bool ggml_cuda_compute_forward(struct ggml_compute_params * params, struct ggml_tensor * tensor) { +GGML_CALL bool ggml_cuda_compute_forward(struct ggml_compute_params * params, struct ggml_tensor * tensor) { if (!g_cublas_loaded) return false; ggml_cuda_func_t func; @@ -10186,7 +10186,7 @@ bool ggml_cuda_compute_forward(struct ggml_compute_params * params, struct ggml_ return true; } -int ggml_cuda_get_device_count() { +GGML_CALL int ggml_cuda_get_device_count() { int device_count; if (cudaGetDeviceCount(&device_count) != cudaSuccess) { return 0; @@ -10194,7 +10194,7 @@ int ggml_cuda_get_device_count() { return device_count; } -void ggml_cuda_get_device_description(int device, char * description, size_t description_size) { +GGML_CALL void ggml_cuda_get_device_description(int device, char * description, size_t description_size) { cudaDeviceProp prop; CUDA_CHECK(cudaGetDeviceProperties(&prop, device)); snprintf(description, description_size, "%s", prop.name); @@ -10244,27 +10244,27 @@ struct ggml_backend_cuda_buffer_context { } }; -static const char * ggml_backend_cuda_buffer_get_name(ggml_backend_buffer_t buffer) { +GGML_CALL static const char * ggml_backend_cuda_buffer_get_name(ggml_backend_buffer_t buffer) { ggml_backend_cuda_buffer_context * ctx = (ggml_backend_cuda_buffer_context *)buffer->context; return ctx->name.c_str(); } -static bool ggml_backend_buffer_is_cuda(ggml_backend_buffer_t buffer) { +GGML_CALL static bool ggml_backend_buffer_is_cuda(ggml_backend_buffer_t buffer) { return buffer->iface.get_name == ggml_backend_cuda_buffer_get_name; } -static void ggml_backend_cuda_buffer_free_buffer(ggml_backend_buffer_t buffer) { +GGML_CALL static void ggml_backend_cuda_buffer_free_buffer(ggml_backend_buffer_t buffer) { ggml_backend_cuda_buffer_context * ctx = (ggml_backend_cuda_buffer_context *)buffer->context; CUDA_CHECK(cudaFree(ctx->dev_ptr)); delete ctx; } -static void * ggml_backend_cuda_buffer_get_base(ggml_backend_buffer_t buffer) { +GGML_CALL static void * ggml_backend_cuda_buffer_get_base(ggml_backend_buffer_t buffer) { ggml_backend_cuda_buffer_context * ctx = (ggml_backend_cuda_buffer_context *)buffer->context; return ctx->dev_ptr; } -static void ggml_backend_cuda_buffer_init_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor) { +GGML_CALL static void ggml_backend_cuda_buffer_init_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor) { ggml_backend_cuda_buffer_context * ctx = (ggml_backend_cuda_buffer_context *)buffer->context; if (tensor->view_src != NULL && tensor->view_offs == 0) { @@ -10296,7 +10296,7 @@ static void ggml_backend_cuda_buffer_init_tensor(ggml_backend_buffer_t buffer, g } } -static void ggml_backend_cuda_buffer_set_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor, const void * data, size_t offset, size_t size) { +GGML_CALL static void ggml_backend_cuda_buffer_set_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor, const void * data, size_t offset, size_t size) { GGML_ASSERT(tensor->backend == GGML_BACKEND_GPU); ggml_backend_cuda_buffer_context * ctx = (ggml_backend_cuda_buffer_context *)buffer->context; @@ -10307,7 +10307,7 @@ static void ggml_backend_cuda_buffer_set_tensor(ggml_backend_buffer_t buffer, gg CUDA_CHECK(cudaDeviceSynchronize()); } -static void ggml_backend_cuda_buffer_get_tensor(ggml_backend_buffer_t buffer, const ggml_tensor * tensor, void * data, size_t offset, size_t size) { +GGML_CALL static void ggml_backend_cuda_buffer_get_tensor(ggml_backend_buffer_t buffer, const ggml_tensor * tensor, void * data, size_t offset, size_t size) { GGML_ASSERT(tensor->backend == GGML_BACKEND_GPU); ggml_backend_cuda_buffer_context * ctx = (ggml_backend_cuda_buffer_context *)buffer->context; @@ -10318,7 +10318,7 @@ static void ggml_backend_cuda_buffer_get_tensor(ggml_backend_buffer_t buffer, co CUDA_CHECK(cudaDeviceSynchronize()); } -static bool ggml_backend_cuda_buffer_cpy_tensor(ggml_backend_buffer_t buffer, const ggml_tensor * src, ggml_tensor * dst) { +GGML_CALL static bool ggml_backend_cuda_buffer_cpy_tensor(ggml_backend_buffer_t buffer, const ggml_tensor * src, ggml_tensor * dst) { if (ggml_backend_buffer_is_cuda(src->buffer)) { ggml_backend_cuda_buffer_context * src_ctx = (ggml_backend_cuda_buffer_context *)src->buffer->context; ggml_backend_cuda_buffer_context * dst_ctx = (ggml_backend_cuda_buffer_context *)buffer->context; @@ -10335,7 +10335,7 @@ static bool ggml_backend_cuda_buffer_cpy_tensor(ggml_backend_buffer_t buffer, co return false; } -static void ggml_backend_cuda_buffer_clear(ggml_backend_buffer_t buffer, uint8_t value) { +GGML_CALL static void ggml_backend_cuda_buffer_clear(ggml_backend_buffer_t buffer, uint8_t value) { ggml_backend_cuda_buffer_context * ctx = (ggml_backend_cuda_buffer_context *)buffer->context; ggml_cuda_set_device(ctx->device); @@ -10357,19 +10357,18 @@ static ggml_backend_buffer_i ggml_backend_cuda_buffer_interface = { }; // cuda buffer type - struct ggml_backend_cuda_buffer_type_context { int device; std::string name; }; -static const char * ggml_backend_cuda_buffer_type_name(ggml_backend_buffer_type_t buft) { +GGML_CALL static const char * ggml_backend_cuda_buffer_type_name(ggml_backend_buffer_type_t buft) { ggml_backend_cuda_buffer_type_context * ctx = (ggml_backend_cuda_buffer_type_context *)buft->context; return ctx->name.c_str(); } -static ggml_backend_buffer_t ggml_backend_cuda_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size) { +GGML_CALL static ggml_backend_buffer_t ggml_backend_cuda_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size) { ggml_backend_cuda_buffer_type_context * buft_ctx = (ggml_backend_cuda_buffer_type_context *)buft->context; ggml_cuda_set_device(buft_ctx->device); @@ -10388,13 +10387,13 @@ static ggml_backend_buffer_t ggml_backend_cuda_buffer_type_alloc_buffer(ggml_bac return ggml_backend_buffer_init(buft, ggml_backend_cuda_buffer_interface, ctx, size); } -static size_t ggml_backend_cuda_buffer_type_get_alignment(ggml_backend_buffer_type_t buft) { +GGML_CALL static size_t ggml_backend_cuda_buffer_type_get_alignment(ggml_backend_buffer_type_t buft) { return 128; UNUSED(buft); } -static size_t ggml_backend_cuda_buffer_type_get_alloc_size(ggml_backend_buffer_type_t buft, const ggml_tensor * tensor) { +GGML_CALL static size_t ggml_backend_cuda_buffer_type_get_alloc_size(ggml_backend_buffer_type_t buft, const ggml_tensor * tensor) { int64_t row_low = 0; int64_t row_high = ggml_nrows(tensor); int64_t nrows_split = row_high - row_low; @@ -10414,7 +10413,7 @@ static size_t ggml_backend_cuda_buffer_type_get_alloc_size(ggml_backend_buffer_t UNUSED(buft); } -static bool ggml_backend_cuda_buffer_type_supports_backend(ggml_backend_buffer_type_t buft, ggml_backend_t backend) { +GGML_CALL static bool ggml_backend_cuda_buffer_type_supports_backend(ggml_backend_buffer_type_t buft, ggml_backend_t backend) { if (!ggml_backend_is_cuda(backend)) { return false; } @@ -10434,7 +10433,7 @@ static ggml_backend_buffer_type_i ggml_backend_cuda_buffer_type_interface = { /* .is_host = */ NULL, }; -ggml_backend_buffer_type_t ggml_backend_cuda_buffer_type(int device) { +GGML_CALL ggml_backend_buffer_type_t ggml_backend_cuda_buffer_type(int device) { // FIXME: this is not thread safe if (device >= ggml_backend_cuda_get_device_count()) { return nullptr; @@ -10479,7 +10478,7 @@ struct ggml_backend_cuda_split_buffer_context { std::vector tensor_extras; }; -static const char * ggml_backend_cuda_split_buffer_get_name(ggml_backend_buffer_t buffer) { +GGML_CALL static const char * ggml_backend_cuda_split_buffer_get_name(ggml_backend_buffer_t buffer) { return GGML_CUDA_NAME "_Split"; UNUSED(buffer); @@ -10490,19 +10489,19 @@ static const char * ggml_backend_cuda_split_buffer_get_name(ggml_backend_buffer_ // return buffer->iface.get_name == ggml_backend_cuda_split_buffer_get_name; //} -static void ggml_backend_cuda_split_buffer_free_buffer(ggml_backend_buffer_t buffer) { +GGML_CALL static void ggml_backend_cuda_split_buffer_free_buffer(ggml_backend_buffer_t buffer) { ggml_backend_cuda_split_buffer_context * ctx = (ggml_backend_cuda_split_buffer_context *)buffer->context; delete ctx; } -static void * ggml_backend_cuda_split_buffer_get_base(ggml_backend_buffer_t buffer) { +GGML_CALL static void * ggml_backend_cuda_split_buffer_get_base(ggml_backend_buffer_t buffer) { // the pointers are stored in the tensor extras, this is just a dummy address and never dereferenced return (void *)0x1000; UNUSED(buffer); } -static void ggml_backend_cuda_split_buffer_init_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor) { +GGML_CALL static void ggml_backend_cuda_split_buffer_init_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor) { GGML_ASSERT(tensor->view_src == nullptr); // views of split tensors are not supported ggml_backend_cuda_split_buffer_context * ctx = (ggml_backend_cuda_split_buffer_context *)buffer->context; @@ -10552,7 +10551,7 @@ static void ggml_backend_cuda_split_buffer_init_tensor(ggml_backend_buffer_t buf tensor->extra = extra; } -static void ggml_backend_cuda_split_buffer_set_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor, const void * data, size_t offset, size_t size) { +GGML_CALL static void ggml_backend_cuda_split_buffer_set_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor, const void * data, size_t offset, size_t size) { // split tensors must always be set in their entirety at once GGML_ASSERT(offset == 0); GGML_ASSERT(size == ggml_nbytes(tensor)); @@ -10586,7 +10585,7 @@ static void ggml_backend_cuda_split_buffer_set_tensor(ggml_backend_buffer_t buff } } -static void ggml_backend_cuda_split_buffer_get_tensor(ggml_backend_buffer_t buffer, const ggml_tensor * tensor, void * data, size_t offset, size_t size) { +GGML_CALL static void ggml_backend_cuda_split_buffer_get_tensor(ggml_backend_buffer_t buffer, const ggml_tensor * tensor, void * data, size_t offset, size_t size) { // split tensors must always be set in their entirety at once GGML_ASSERT(offset == 0); GGML_ASSERT(size == ggml_nbytes(tensor)); @@ -10620,7 +10619,7 @@ static void ggml_backend_cuda_split_buffer_get_tensor(ggml_backend_buffer_t buff } } -static void ggml_backend_cuda_split_buffer_clear(ggml_backend_buffer_t buffer, uint8_t value) { +GGML_CALL static void ggml_backend_cuda_split_buffer_clear(ggml_backend_buffer_t buffer, uint8_t value) { UNUSED(buffer); UNUSED(value); } @@ -10639,13 +10638,13 @@ static struct ggml_backend_buffer_i ggml_backend_cuda_split_buffer_interface = { // cuda split buffer type -static const char * ggml_backend_cuda_split_buffer_type_name(ggml_backend_buffer_type_t buft) { +GGML_CALL static const char * ggml_backend_cuda_split_buffer_type_name(ggml_backend_buffer_type_t buft) { return GGML_CUDA_NAME "_Split"; UNUSED(buft); } -static ggml_backend_buffer_t ggml_backend_cuda_split_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size) { +GGML_CALL static ggml_backend_buffer_t ggml_backend_cuda_split_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size) { // since we don't know the exact split after rounding, we cannot allocate the device buffers at this point // instead, we allocate them for each tensor separately in init_tensor // however, the size still represents the maximum cumulative size of all the device buffers after the tensors are allocated, @@ -10655,13 +10654,13 @@ static ggml_backend_buffer_t ggml_backend_cuda_split_buffer_type_alloc_buffer(gg return ggml_backend_buffer_init(buft, ggml_backend_cuda_split_buffer_interface, ctx, size); } -static size_t ggml_backend_cuda_split_buffer_type_get_alignment(ggml_backend_buffer_type_t buft) { +GGML_CALL static size_t ggml_backend_cuda_split_buffer_type_get_alignment(ggml_backend_buffer_type_t buft) { return 128; UNUSED(buft); } -static size_t ggml_backend_cuda_split_buffer_type_get_alloc_size(ggml_backend_buffer_type_t buft, const ggml_tensor * tensor) { +GGML_CALL static size_t ggml_backend_cuda_split_buffer_type_get_alloc_size(ggml_backend_buffer_type_t buft, const ggml_tensor * tensor) { ggml_backend_cuda_split_buffer_type_context * ctx = (ggml_backend_cuda_split_buffer_type_context *)buft->context; size_t total_size = 0; @@ -10688,13 +10687,13 @@ static size_t ggml_backend_cuda_split_buffer_type_get_alloc_size(ggml_backend_bu return total_size; } -static bool ggml_backend_cuda_split_buffer_type_supports_backend(ggml_backend_buffer_type_t buft, ggml_backend_t backend) { +GGML_CALL static bool ggml_backend_cuda_split_buffer_type_supports_backend(ggml_backend_buffer_type_t buft, ggml_backend_t backend) { return ggml_backend_is_cuda(backend); UNUSED(buft); } -static bool ggml_backend_cuda_split_buffer_type_is_host(ggml_backend_buffer_type_t buft) { +GGML_CALL static bool ggml_backend_cuda_split_buffer_type_is_host(ggml_backend_buffer_type_t buft) { return false; UNUSED(buft); @@ -10709,7 +10708,7 @@ static ggml_backend_buffer_type_i ggml_backend_cuda_split_buffer_type_interface /* .is_host = */ ggml_backend_cuda_split_buffer_type_is_host, }; -ggml_backend_buffer_type_t ggml_backend_cuda_split_buffer_type(const float * tensor_split) { +GGML_CALL ggml_backend_buffer_type_t ggml_backend_cuda_split_buffer_type(const float * tensor_split) { // FIXME: this is not thread safe static std::map, struct ggml_backend_buffer_type> buft_map; @@ -10745,23 +10744,23 @@ ggml_backend_buffer_type_t ggml_backend_cuda_split_buffer_type(const float * ten // host buffer type -static const char * ggml_backend_cuda_host_buffer_type_name(ggml_backend_buffer_type_t buft) { +GGML_CALL static const char * ggml_backend_cuda_host_buffer_type_name(ggml_backend_buffer_type_t buft) { return GGML_CUDA_NAME "_Host"; UNUSED(buft); } -static const char * ggml_backend_cuda_host_buffer_name(ggml_backend_buffer_t buffer) { +GGML_CALL static const char * ggml_backend_cuda_host_buffer_name(ggml_backend_buffer_t buffer) { return GGML_CUDA_NAME "_Host"; UNUSED(buffer); } -static void ggml_backend_cuda_host_buffer_free_buffer(ggml_backend_buffer_t buffer) { +GGML_CALL static void ggml_backend_cuda_host_buffer_free_buffer(ggml_backend_buffer_t buffer) { ggml_cuda_host_free(buffer->context); } -static ggml_backend_buffer_t ggml_backend_cuda_host_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size) { +GGML_CALL static ggml_backend_buffer_t ggml_backend_cuda_host_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size) { void * ptr = ggml_cuda_host_malloc(size); if (ptr == nullptr) { @@ -10777,7 +10776,7 @@ static ggml_backend_buffer_t ggml_backend_cuda_host_buffer_type_alloc_buffer(ggm return buffer; } -ggml_backend_buffer_type_t ggml_backend_cuda_host_buffer_type() { +GGML_CALL ggml_backend_buffer_type_t ggml_backend_cuda_host_buffer_type() { static struct ggml_backend_buffer_type ggml_backend_cuda_buffer_type_host = { /* .iface = */ { /* .get_name = */ ggml_backend_cuda_host_buffer_type_name, @@ -10795,26 +10794,26 @@ ggml_backend_buffer_type_t ggml_backend_cuda_host_buffer_type() { // backend -static const char * ggml_backend_cuda_name(ggml_backend_t backend) { +GGML_CALL static const char * ggml_backend_cuda_name(ggml_backend_t backend) { ggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *)backend->context; return cuda_ctx->name.c_str(); } -static void ggml_backend_cuda_free(ggml_backend_t backend) { +GGML_CALL static void ggml_backend_cuda_free(ggml_backend_t backend) { ggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *)backend->context; delete cuda_ctx; delete backend; } -static ggml_backend_buffer_type_t ggml_backend_cuda_get_default_buffer_type(ggml_backend_t backend) { +GGML_CALL static ggml_backend_buffer_type_t ggml_backend_cuda_get_default_buffer_type(ggml_backend_t backend) { ggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *)backend->context; return ggml_backend_cuda_buffer_type(cuda_ctx->device); } -static void ggml_backend_cuda_set_tensor_async(ggml_backend_t backend, ggml_tensor * tensor, const void * data, size_t offset, size_t size) { +GGML_CALL static void ggml_backend_cuda_set_tensor_async(ggml_backend_t backend, ggml_tensor * tensor, const void * data, size_t offset, size_t size) { ggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *)backend->context; GGML_ASSERT(tensor->buffer->buft == ggml_backend_cuda_buffer_type(cuda_ctx->device) && "unsupported buffer type"); @@ -10823,7 +10822,7 @@ static void ggml_backend_cuda_set_tensor_async(ggml_backend_t backend, ggml_tens CUDA_CHECK(cudaMemcpyAsync((char *)tensor->data + offset, data, size, cudaMemcpyHostToDevice, g_cudaStreams[cuda_ctx->device][0])); } -static void ggml_backend_cuda_get_tensor_async(ggml_backend_t backend, const ggml_tensor * tensor, void * data, size_t offset, size_t size) { +GGML_CALL static void ggml_backend_cuda_get_tensor_async(ggml_backend_t backend, const ggml_tensor * tensor, void * data, size_t offset, size_t size) { ggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *)backend->context; GGML_ASSERT(tensor->buffer->buft == ggml_backend_cuda_buffer_type(cuda_ctx->device) && "unsupported buffer type"); @@ -10832,7 +10831,7 @@ static void ggml_backend_cuda_get_tensor_async(ggml_backend_t backend, const ggm CUDA_CHECK(cudaMemcpyAsync(data, (const char *)tensor->data + offset, size, cudaMemcpyDeviceToHost, g_cudaStreams[cuda_ctx->device][0])); } -static bool ggml_backend_cuda_cpy_tensor_async(ggml_backend_t backend, const ggml_tensor * src, ggml_tensor * dst) { +GGML_CALL static bool ggml_backend_cuda_cpy_tensor_async(ggml_backend_t backend, const ggml_tensor * src, ggml_tensor * dst) { ggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *)backend->context; if (dst->buffer->buft == ggml_backend_cuda_buffer_type(cuda_ctx->device) && ggml_backend_buffer_is_cuda(src->buffer)) { @@ -10843,7 +10842,7 @@ static bool ggml_backend_cuda_cpy_tensor_async(ggml_backend_t backend, const ggm return false; } -static void ggml_backend_cuda_synchronize(ggml_backend_t backend) { +GGML_CALL static void ggml_backend_cuda_synchronize(ggml_backend_t backend) { ggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *)backend->context; CUDA_CHECK(cudaStreamSynchronize(g_cudaStreams[cuda_ctx->device][0])); @@ -10851,7 +10850,7 @@ static void ggml_backend_cuda_synchronize(ggml_backend_t backend) { UNUSED(backend); } -static bool ggml_backend_cuda_graph_compute(ggml_backend_t backend, ggml_cgraph * cgraph) { +GGML_CALL static bool ggml_backend_cuda_graph_compute(ggml_backend_t backend, ggml_cgraph * cgraph) { ggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *)backend->context; ggml_cuda_set_main_device(cuda_ctx->device); @@ -10890,7 +10889,7 @@ static bool ggml_backend_cuda_graph_compute(ggml_backend_t backend, ggml_cgraph return true; } -static bool ggml_backend_cuda_supports_op(ggml_backend_t backend, const ggml_tensor * op) { +GGML_CALL static bool ggml_backend_cuda_supports_op(ggml_backend_t backend, const ggml_tensor * op) { switch (op->op) { case GGML_OP_UNARY: switch (ggml_get_unary_op(op)) { @@ -11016,7 +11015,7 @@ static ggml_backend_i ggml_backend_cuda_interface = { /* .supports_op = */ ggml_backend_cuda_supports_op, }; -ggml_backend_t ggml_backend_cuda_init(int device) { +GGML_CALL ggml_backend_t ggml_backend_cuda_init(int device) { ggml_init_cublas(); // TODO: remove from ggml.c if (device < 0 || device >= ggml_cuda_get_device_count()) { @@ -11040,35 +11039,35 @@ ggml_backend_t ggml_backend_cuda_init(int device) { return cuda_backend; } -bool ggml_backend_is_cuda(ggml_backend_t backend) { +GGML_CALL bool ggml_backend_is_cuda(ggml_backend_t backend) { return backend && backend->iface.get_name == ggml_backend_cuda_name; } -int ggml_backend_cuda_get_device_count() { +GGML_CALL int ggml_backend_cuda_get_device_count() { return ggml_cuda_get_device_count(); } -void ggml_backend_cuda_get_device_description(int device, char * description, size_t description_size) { +GGML_CALL void ggml_backend_cuda_get_device_description(int device, char * description, size_t description_size) { ggml_cuda_get_device_description(device, description, description_size); } -void ggml_backend_cuda_get_device_memory(int device, size_t * free, size_t * total) { +GGML_CALL void ggml_backend_cuda_get_device_memory(int device, size_t * free, size_t * total) { ggml_cuda_set_device(device); CUDA_CHECK(cudaMemGetInfo(free, total)); } // backend registry -static ggml_backend_t ggml_backend_reg_cuda_init(const char * params, void * user_data) { +GGML_CALL static ggml_backend_t ggml_backend_reg_cuda_init(const char * params, void * user_data) { ggml_backend_t cuda_backend = ggml_backend_cuda_init((int) (intptr_t) user_data); return cuda_backend; UNUSED(params); } -extern "C" int ggml_backend_cuda_reg_devices(); +extern "C" GGML_CALL int ggml_backend_cuda_reg_devices(); -int ggml_backend_cuda_reg_devices() { +GGML_CALL int ggml_backend_cuda_reg_devices() { int device_count = ggml_cuda_get_device_count(); //int device_count = 1; // DEBUG: some tools require delaying CUDA initialization for (int i = 0; i < device_count; i++) { diff --git a/ggml-cuda.h b/ggml-cuda.h index d19cbf3fd..b1ebd61d7 100644 --- a/ggml-cuda.h +++ b/ggml-cuda.h @@ -18,34 +18,34 @@ extern "C" { #define GGML_CUDA_MAX_DEVICES 16 // Always success. To check if CUDA is actually loaded, use `ggml_cublas_loaded`. -GGML_API void ggml_init_cublas(void); +GGML_API GGML_CALL void ggml_init_cublas(void); // Returns `true` if there are available CUDA devices and cublas loads successfully; otherwise, it returns `false`. -GGML_API bool ggml_cublas_loaded(void); +GGML_API GGML_CALL bool ggml_cublas_loaded(void); -GGML_API void * ggml_cuda_host_malloc(size_t size); -GGML_API void ggml_cuda_host_free(void * ptr); +GGML_API GGML_CALL void * ggml_cuda_host_malloc(size_t size); +GGML_API GGML_CALL void ggml_cuda_host_free(void * ptr); -GGML_API bool ggml_cuda_can_mul_mat(const struct ggml_tensor * src0, const struct ggml_tensor * src1, struct ggml_tensor * dst); -GGML_API bool ggml_cuda_compute_forward(struct ggml_compute_params * params, struct ggml_tensor * tensor); +GGML_API GGML_CALL bool ggml_cuda_can_mul_mat(const struct ggml_tensor * src0, const struct ggml_tensor * src1, struct ggml_tensor * dst); +GGML_API GGML_CALL bool ggml_cuda_compute_forward(struct ggml_compute_params * params, struct ggml_tensor * tensor); -GGML_API int ggml_cuda_get_device_count(void); -GGML_API void ggml_cuda_get_device_description(int device, char * description, size_t description_size); +GGML_API GGML_CALL int ggml_cuda_get_device_count(void); +GGML_API GGML_CALL void ggml_cuda_get_device_description(int device, char * description, size_t description_size); // backend API -GGML_API ggml_backend_t ggml_backend_cuda_init(int device); +GGML_API GGML_CALL ggml_backend_t ggml_backend_cuda_init(int device); -GGML_API bool ggml_backend_is_cuda(ggml_backend_t backend); +GGML_API GGML_CALL bool ggml_backend_is_cuda(ggml_backend_t backend); -GGML_API ggml_backend_buffer_type_t ggml_backend_cuda_buffer_type(int device); +GGML_API GGML_CALL ggml_backend_buffer_type_t ggml_backend_cuda_buffer_type(int device); // split tensor buffer that splits matrices by rows across multiple devices -GGML_API ggml_backend_buffer_type_t ggml_backend_cuda_split_buffer_type(const float * tensor_split); +GGML_API GGML_CALL ggml_backend_buffer_type_t ggml_backend_cuda_split_buffer_type(const float * tensor_split); // pinned host buffer for use with the CPU backend for faster copies between CPU and GPU -GGML_API ggml_backend_buffer_type_t ggml_backend_cuda_host_buffer_type(void); +GGML_API GGML_CALL ggml_backend_buffer_type_t ggml_backend_cuda_host_buffer_type(void); -GGML_API int ggml_backend_cuda_get_device_count(void); -GGML_API void ggml_backend_cuda_get_device_description(int device, char * description, size_t description_size); -GGML_API void ggml_backend_cuda_get_device_memory(int device, size_t * free, size_t * total); +GGML_API GGML_CALL int ggml_backend_cuda_get_device_count(void); +GGML_API GGML_CALL void ggml_backend_cuda_get_device_description(int device, char * description, size_t description_size); +GGML_API GGML_CALL void ggml_backend_cuda_get_device_memory(int device, size_t * free, size_t * total); #ifdef __cplusplus } diff --git a/ggml-metal.h b/ggml-metal.h index cd5e2995f..8b0bfc5f1 100644 --- a/ggml-metal.h +++ b/ggml-metal.h @@ -47,11 +47,11 @@ GGML_API ggml_backend_t ggml_backend_metal_init(void); GGML_API bool ggml_backend_is_metal(ggml_backend_t backend); -GGML_API ggml_backend_buffer_t ggml_backend_metal_buffer_from_ptr(void * data, size_t size, size_t max_size); +GGML_API GGML_CALL ggml_backend_buffer_t ggml_backend_metal_buffer_from_ptr(void * data, size_t size, size_t max_size); GGML_API void ggml_backend_metal_set_n_cb(ggml_backend_t backend, int n_cb); -GGML_API ggml_backend_buffer_type_t ggml_backend_metal_buffer_type(void); +GGML_API GGML_CALL ggml_backend_buffer_type_t ggml_backend_metal_buffer_type(void); // helper to check if the device supports a specific family // ideally, the user code should be doing these checks diff --git a/ggml-metal.m b/ggml-metal.m index 2ca726055..867f2fd48 100644 --- a/ggml-metal.m +++ b/ggml-metal.m @@ -2294,13 +2294,13 @@ static void ggml_backend_metal_free_device(void) { } } -static const char * ggml_backend_metal_buffer_get_name(ggml_backend_buffer_t buffer) { +GGML_CALL static const char * ggml_backend_metal_buffer_get_name(ggml_backend_buffer_t buffer) { return "Metal"; UNUSED(buffer); } -static void ggml_backend_metal_buffer_free_buffer(ggml_backend_buffer_t buffer) { +GGML_CALL static void ggml_backend_metal_buffer_free_buffer(ggml_backend_buffer_t buffer) { struct ggml_backend_metal_buffer_context * ctx = (struct ggml_backend_metal_buffer_context *)buffer->context; for (int i = 0; i < ctx->n_buffers; i++) { @@ -2315,25 +2315,25 @@ static void ggml_backend_metal_buffer_free_buffer(ggml_backend_buffer_t buffer) free(ctx); } -static void * ggml_backend_metal_buffer_get_base(ggml_backend_buffer_t buffer) { +GGML_CALL static void * ggml_backend_metal_buffer_get_base(ggml_backend_buffer_t buffer) { struct ggml_backend_metal_buffer_context * ctx = (struct ggml_backend_metal_buffer_context *)buffer->context; return ctx->all_data; } -static void ggml_backend_metal_buffer_set_tensor(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor, const void * data, size_t offset, size_t size) { +GGML_CALL static void ggml_backend_metal_buffer_set_tensor(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor, const void * data, size_t offset, size_t size) { memcpy((char *)tensor->data + offset, data, size); UNUSED(buffer); } -static void ggml_backend_metal_buffer_get_tensor(ggml_backend_buffer_t buffer, const struct ggml_tensor * tensor, void * data, size_t offset, size_t size) { +GGML_CALL static void ggml_backend_metal_buffer_get_tensor(ggml_backend_buffer_t buffer, const struct ggml_tensor * tensor, void * data, size_t offset, size_t size) { memcpy(data, (const char *)tensor->data + offset, size); UNUSED(buffer); } -static bool ggml_backend_metal_buffer_cpy_tensor(ggml_backend_buffer_t buffer, const struct ggml_tensor * src, struct ggml_tensor * dst) { +GGML_CALL static bool ggml_backend_metal_buffer_cpy_tensor(ggml_backend_buffer_t buffer, const struct ggml_tensor * src, struct ggml_tensor * dst) { if (ggml_backend_buffer_is_host(src->buffer)) { memcpy(dst->data, src->data, ggml_nbytes(src)); return true; @@ -2343,7 +2343,7 @@ static bool ggml_backend_metal_buffer_cpy_tensor(ggml_backend_buffer_t buffer, c UNUSED(buffer); } -static void ggml_backend_metal_buffer_clear(ggml_backend_buffer_t buffer, uint8_t value) { +GGML_CALL static void ggml_backend_metal_buffer_clear(ggml_backend_buffer_t buffer, uint8_t value) { struct ggml_backend_metal_buffer_context * ctx = (struct ggml_backend_metal_buffer_context *)buffer->context; memset(ctx->all_data, value, ctx->all_size); @@ -2363,13 +2363,13 @@ static struct ggml_backend_buffer_i ggml_backend_metal_buffer_i = { // default buffer type -static const char * ggml_backend_metal_buffer_type_get_name(ggml_backend_buffer_type_t buft) { +GGML_CALL static const char * ggml_backend_metal_buffer_type_get_name(ggml_backend_buffer_type_t buft) { return "Metal"; UNUSED(buft); } -static ggml_backend_buffer_t ggml_backend_metal_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size) { +GGML_CALL static ggml_backend_buffer_t ggml_backend_metal_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size) { struct ggml_backend_metal_buffer_context * ctx = malloc(sizeof(struct ggml_backend_metal_buffer_context)); const size_t size_page = sysconf(_SC_PAGESIZE); @@ -2421,24 +2421,24 @@ static ggml_backend_buffer_t ggml_backend_metal_buffer_type_alloc_buffer(ggml_ba return ggml_backend_buffer_init(buft, ggml_backend_metal_buffer_i, ctx, size); } -static size_t ggml_backend_metal_buffer_type_get_alignment(ggml_backend_buffer_type_t buft) { +GGML_CALL static size_t ggml_backend_metal_buffer_type_get_alignment(ggml_backend_buffer_type_t buft) { return 32; UNUSED(buft); } -static bool ggml_backend_metal_buffer_type_supports_backend(ggml_backend_buffer_type_t buft, ggml_backend_t backend) { +GGML_CALL static bool ggml_backend_metal_buffer_type_supports_backend(ggml_backend_buffer_type_t buft, ggml_backend_t backend) { return ggml_backend_is_metal(backend) || ggml_backend_is_cpu(backend); UNUSED(buft); } -static bool ggml_backend_metal_buffer_type_is_host(ggml_backend_buffer_type_t buft) { +GGML_CALL static bool ggml_backend_metal_buffer_type_is_host(ggml_backend_buffer_type_t buft) { return true; UNUSED(buft); } -ggml_backend_buffer_type_t ggml_backend_metal_buffer_type(void) { +GGML_CALL ggml_backend_buffer_type_t ggml_backend_metal_buffer_type(void) { static struct ggml_backend_buffer_type ggml_backend_buffer_type_metal = { /* .iface = */ { /* .get_name = */ ggml_backend_metal_buffer_type_get_name, @@ -2456,7 +2456,7 @@ ggml_backend_buffer_type_t ggml_backend_metal_buffer_type(void) { // buffer from ptr -ggml_backend_buffer_t ggml_backend_metal_buffer_from_ptr(void * data, size_t size, size_t max_size) { +GGML_CALL ggml_backend_buffer_t ggml_backend_metal_buffer_from_ptr(void * data, size_t size, size_t max_size) { struct ggml_backend_metal_buffer_context * ctx = malloc(sizeof(struct ggml_backend_metal_buffer_context)); ctx->all_data = data; @@ -2543,31 +2543,31 @@ ggml_backend_buffer_t ggml_backend_metal_buffer_from_ptr(void * data, size_t siz // backend -static const char * ggml_backend_metal_name(ggml_backend_t backend) { +GGML_CALL static const char * ggml_backend_metal_name(ggml_backend_t backend) { return "Metal"; UNUSED(backend); } -static void ggml_backend_metal_free(ggml_backend_t backend) { +GGML_CALL static void ggml_backend_metal_free(ggml_backend_t backend) { struct ggml_metal_context * ctx = (struct ggml_metal_context *)backend->context; ggml_metal_free(ctx); free(backend); } -static ggml_backend_buffer_type_t ggml_backend_metal_get_default_buffer_type(ggml_backend_t backend) { +GGML_CALL static ggml_backend_buffer_type_t ggml_backend_metal_get_default_buffer_type(ggml_backend_t backend) { return ggml_backend_metal_buffer_type(); UNUSED(backend); } -static bool ggml_backend_metal_graph_compute(ggml_backend_t backend, struct ggml_cgraph * cgraph) { +GGML_CALL static bool ggml_backend_metal_graph_compute(ggml_backend_t backend, struct ggml_cgraph * cgraph) { struct ggml_metal_context * metal_ctx = (struct ggml_metal_context *)backend->context; return ggml_metal_graph_compute(metal_ctx, cgraph); } -static bool ggml_backend_metal_supports_op(ggml_backend_t backend, const struct ggml_tensor * op) { +GGML_CALL static bool ggml_backend_metal_supports_op(ggml_backend_t backend, const struct ggml_tensor * op) { struct ggml_metal_context * metal_ctx = (struct ggml_metal_context *)backend->context; return ggml_metal_supports_op(metal_ctx, op); @@ -2630,9 +2630,9 @@ bool ggml_backend_metal_supports_family(ggml_backend_t backend, int family) { return [ctx->device supportsFamily:(MTLGPUFamilyApple1 + family - 1)]; } -ggml_backend_t ggml_backend_reg_metal_init(const char * params, void * user_data); // silence warning +GGML_CALL ggml_backend_t ggml_backend_reg_metal_init(const char * params, void * user_data); // silence warning -ggml_backend_t ggml_backend_reg_metal_init(const char * params, void * user_data) { +GGML_CALL ggml_backend_t ggml_backend_reg_metal_init(const char * params, void * user_data) { return ggml_backend_metal_init(); GGML_UNUSED(params); diff --git a/ggml.c b/ggml.c index ef5888ab2..5779f32d2 100644 --- a/ggml.c +++ b/ggml.c @@ -1990,19 +1990,19 @@ void ggml_print_objects(const struct ggml_context * ctx) { GGML_PRINT("%s: --- end ---\n", __func__); } -int64_t ggml_nelements(const struct ggml_tensor * tensor) { +GGML_CALL int64_t ggml_nelements(const struct ggml_tensor * tensor) { static_assert(GGML_MAX_DIMS == 4, "GGML_MAX_DIMS is not 4 - update this function"); return tensor->ne[0]*tensor->ne[1]*tensor->ne[2]*tensor->ne[3]; } -int64_t ggml_nrows(const struct ggml_tensor * tensor) { +GGML_CALL int64_t ggml_nrows(const struct ggml_tensor * tensor) { static_assert(GGML_MAX_DIMS == 4, "GGML_MAX_DIMS is not 4 - update this function"); return tensor->ne[1]*tensor->ne[2]*tensor->ne[3]; } -size_t ggml_nbytes(const struct ggml_tensor * tensor) { +GGML_CALL size_t ggml_nbytes(const struct ggml_tensor * tensor) { size_t nbytes; size_t blck_size = ggml_blck_size(tensor->type); if (blck_size == 1) { @@ -2025,15 +2025,15 @@ size_t ggml_nbytes_pad(const struct ggml_tensor * tensor) { return GGML_PAD(ggml_nbytes(tensor), GGML_MEM_ALIGN); } -int ggml_blck_size(enum ggml_type type) { +GGML_CALL int ggml_blck_size(enum ggml_type type) { return type_traits[type].blck_size; } -size_t ggml_type_size(enum ggml_type type) { +GGML_CALL size_t ggml_type_size(enum ggml_type type) { return type_traits[type].type_size; } -size_t ggml_row_size(enum ggml_type type, int64_t ne) { +GGML_CALL size_t ggml_row_size(enum ggml_type type, int64_t ne) { assert(ne % ggml_blck_size(type) == 0); return ggml_type_size(type)*ne/ggml_blck_size(type); } @@ -2042,15 +2042,15 @@ double ggml_type_sizef(enum ggml_type type) { return ((double)(type_traits[type].type_size))/type_traits[type].blck_size; } -const char * ggml_type_name(enum ggml_type type) { +GGML_CALL const char * ggml_type_name(enum ggml_type type) { return type_traits[type].type_name; } -bool ggml_is_quantized(enum ggml_type type) { +GGML_CALL bool ggml_is_quantized(enum ggml_type type) { return type_traits[type].is_quantized; } -const char * ggml_op_name(enum ggml_op op) { +GGML_CALL const char * ggml_op_name(enum ggml_op op) { return GGML_OP_NAME[op]; } @@ -2062,7 +2062,7 @@ const char * ggml_unary_op_name(enum ggml_unary_op op) { return GGML_UNARY_OP_NAME[op]; } -const char * ggml_op_desc(const struct ggml_tensor * t) { +GGML_CALL const char * ggml_op_desc(const struct ggml_tensor * t) { if (t->op == GGML_OP_UNARY) { enum ggml_unary_op uop = ggml_get_unary_op(t); return ggml_unary_op_name(uop); @@ -2072,7 +2072,7 @@ const char * ggml_op_desc(const struct ggml_tensor * t) { } } -size_t ggml_element_size(const struct ggml_tensor * tensor) { +GGML_CALL size_t ggml_element_size(const struct ggml_tensor * tensor) { return ggml_type_size(tensor->type); } @@ -2154,11 +2154,11 @@ size_t ggml_tensor_overhead(void) { return GGML_OBJECT_SIZE + GGML_TENSOR_SIZE; } -bool ggml_is_transposed(const struct ggml_tensor * tensor) { +GGML_CALL bool ggml_is_transposed(const struct ggml_tensor * tensor) { return tensor->nb[0] > tensor->nb[1]; } -bool ggml_is_contiguous(const struct ggml_tensor * tensor) { +GGML_CALL bool ggml_is_contiguous(const struct ggml_tensor * tensor) { static_assert(GGML_MAX_DIMS == 4, "GGML_MAX_DIMS is not 4 - update this function"); return @@ -2177,7 +2177,7 @@ static inline bool ggml_is_contiguous_except_dim_1(const struct ggml_tensor * te tensor->nb[3] == tensor->nb[2]*tensor->ne[2]; } -bool ggml_is_permuted(const struct ggml_tensor * tensor) { +GGML_CALL bool ggml_is_permuted(const struct ggml_tensor * tensor) { static_assert(GGML_MAX_DIMS == 4, "GGML_MAX_DIMS is not 4 - update this function"); return tensor->nb[0] > tensor->nb[1] || tensor->nb[1] > tensor->nb[2] || tensor->nb[2] > tensor->nb[3]; @@ -3079,7 +3079,7 @@ float * ggml_get_data_f32(const struct ggml_tensor * tensor) { return (float *)(tensor->data); } -enum ggml_unary_op ggml_get_unary_op(const struct ggml_tensor * tensor) { +GGML_CALL enum ggml_unary_op ggml_get_unary_op(const struct ggml_tensor * tensor) { GGML_ASSERT(tensor->op == GGML_OP_UNARY); return (enum ggml_unary_op) ggml_get_op_params_i32(tensor, 0); } @@ -11653,7 +11653,7 @@ static void ggml_rope_cache_init( } } -void ggml_rope_yarn_corr_dims( +GGML_CALL void ggml_rope_yarn_corr_dims( int n_dims, int n_orig_ctx, float freq_base, float beta_fast, float beta_slow, float dims[2] ) { // start and end correction dims diff --git a/ggml.h b/ggml.h index 1187074f7..837c52e68 100644 --- a/ggml.h +++ b/ggml.h @@ -187,6 +187,16 @@ # define GGML_API #endif +#ifdef GGML_MULTIPLATFORM +# if defined(_WIN32) +# define GGML_CALL +# else +# define GGML_CALL __attribute__((__ms_abi__)) +# endif +#else +# define GGML_CALL +#endif + // TODO: support for clang #ifdef __GNUC__ # define GGML_DEPRECATED(func, hint) func __attribute__((deprecated(hint))) @@ -649,41 +659,41 @@ extern "C" { GGML_API void ggml_print_object (const struct ggml_object * obj); GGML_API void ggml_print_objects(const struct ggml_context * ctx); - GGML_API int64_t ggml_nelements (const struct ggml_tensor * tensor); - GGML_API int64_t ggml_nrows (const struct ggml_tensor * tensor); - GGML_API size_t ggml_nbytes (const struct ggml_tensor * tensor); - GGML_API size_t ggml_nbytes_pad (const struct ggml_tensor * tensor); // same as ggml_nbytes() but padded to GGML_MEM_ALIGN + GGML_API GGML_CALL int64_t ggml_nelements (const struct ggml_tensor * tensor); + GGML_API GGML_CALL int64_t ggml_nrows (const struct ggml_tensor * tensor); + GGML_API GGML_CALL size_t ggml_nbytes (const struct ggml_tensor * tensor); + GGML_API size_t ggml_nbytes_pad (const struct ggml_tensor * tensor); // same as ggml_nbytes() but padded to GGML_MEM_ALIGN - GGML_API int ggml_blck_size(enum ggml_type type); - GGML_API size_t ggml_type_size(enum ggml_type type); // size in bytes for all elements in a block - GGML_API size_t ggml_row_size (enum ggml_type type, int64_t ne); // size in bytes for all elements in a row + GGML_API GGML_CALL int ggml_blck_size(enum ggml_type type); + GGML_API GGML_CALL size_t ggml_type_size(enum ggml_type type); // size in bytes for all elements in a block + GGML_API GGML_CALL size_t ggml_row_size (enum ggml_type type, int64_t ne); // size in bytes for all elements in a row GGML_DEPRECATED( GGML_API double ggml_type_sizef(enum ggml_type type), // ggml_type_size()/ggml_blck_size() as float "use ggml_row_size() instead"); - GGML_API const char * ggml_type_name(enum ggml_type type); - GGML_API const char * ggml_op_name (enum ggml_op op); - GGML_API const char * ggml_op_symbol(enum ggml_op op); + GGML_API GGML_CALL const char * ggml_type_name(enum ggml_type type); + GGML_API GGML_CALL const char * ggml_op_name (enum ggml_op op); + GGML_API const char * ggml_op_symbol(enum ggml_op op); - GGML_API const char * ggml_unary_op_name(enum ggml_unary_op op); - GGML_API const char * ggml_op_desc(const struct ggml_tensor * t); // unary or op name + GGML_API const char * ggml_unary_op_name(enum ggml_unary_op op); + GGML_API GGML_CALL const char * ggml_op_desc(const struct ggml_tensor * t); // unary or op name - GGML_API size_t ggml_element_size(const struct ggml_tensor * tensor); + GGML_API GGML_CALL size_t ggml_element_size(const struct ggml_tensor * tensor); - GGML_API bool ggml_is_quantized(enum ggml_type type); + GGML_API GGML_CALL bool ggml_is_quantized(enum ggml_type type); // TODO: temporary until model loading of ggml examples is refactored GGML_API enum ggml_type ggml_ftype_to_ggml_type(enum ggml_ftype ftype); - GGML_API bool ggml_is_transposed(const struct ggml_tensor * tensor); - GGML_API bool ggml_is_contiguous(const struct ggml_tensor * tensor); - GGML_API bool ggml_is_permuted (const struct ggml_tensor * tensor); - GGML_API bool ggml_is_scalar (const struct ggml_tensor * tensor); - GGML_API bool ggml_is_vector (const struct ggml_tensor * tensor); - GGML_API bool ggml_is_matrix (const struct ggml_tensor * tensor); - GGML_API bool ggml_is_3d (const struct ggml_tensor * tensor); - GGML_API int ggml_n_dims (const struct ggml_tensor * tensor); // returns 1 for scalars + GGML_API GGML_CALL bool ggml_is_transposed(const struct ggml_tensor * tensor); + GGML_API GGML_CALL bool ggml_is_contiguous(const struct ggml_tensor * tensor); + GGML_API GGML_CALL bool ggml_is_permuted (const struct ggml_tensor * tensor); + GGML_API bool ggml_is_scalar (const struct ggml_tensor * tensor); + GGML_API bool ggml_is_vector (const struct ggml_tensor * tensor); + GGML_API bool ggml_is_matrix (const struct ggml_tensor * tensor); + GGML_API bool ggml_is_3d (const struct ggml_tensor * tensor); + GGML_API int ggml_n_dims (const struct ggml_tensor * tensor); // returns 1 for scalars GGML_API bool ggml_are_same_shape(const struct ggml_tensor * t0, const struct ggml_tensor * t1); @@ -770,7 +780,7 @@ extern "C" { GGML_API void * ggml_get_data (const struct ggml_tensor * tensor); GGML_API float * ggml_get_data_f32(const struct ggml_tensor * tensor); - GGML_API enum ggml_unary_op ggml_get_unary_op(const struct ggml_tensor * tensor); + GGML_API GGML_CALL enum ggml_unary_op ggml_get_unary_op(const struct ggml_tensor * tensor); GGML_API const char * ggml_get_name (const struct ggml_tensor * tensor); GGML_API struct ggml_tensor * ggml_set_name ( struct ggml_tensor * tensor, const char * name); @@ -1413,7 +1423,7 @@ extern "C" { float beta_slow); // compute correction dims for YaRN RoPE scaling - void ggml_rope_yarn_corr_dims( + GGML_CALL void ggml_rope_yarn_corr_dims( int n_dims, int n_orig_ctx, float freq_base, float beta_fast, float beta_slow, float dims[2]); // xPos RoPE, in-place, returns view(a) From 122ed4840cc6d209df6043e027f9f8a03aee01da Mon Sep 17 00:00:00 2001 From: Maximilian Winter Date: Tue, 16 Jan 2024 13:10:48 +0100 Subject: [PATCH 079/812] examples : fix and improv docs for the grammar generator (#4909) * Create pydantic-models-to-grammar.py * Added some comments for usage * Refactored Grammar Generator Added example and usage instruction. * Update pydantic_models_to_grammar.py * Update pydantic-models-to-grammar-examples.py * Renamed module and imported it. * Update pydantic-models-to-grammar.py * Renamed file and fixed grammar generator issue. * Fixed some issues and bugs of the grammar generator. Imporved Documentation * Update pydantic_models_to_grammar.py --- examples/pydantic_models_to_grammar.py | 877 +++++++++++++++---------- 1 file changed, 519 insertions(+), 358 deletions(-) diff --git a/examples/pydantic_models_to_grammar.py b/examples/pydantic_models_to_grammar.py index 41b98fdc1..848c1c367 100644 --- a/examples/pydantic_models_to_grammar.py +++ b/examples/pydantic_models_to_grammar.py @@ -4,6 +4,7 @@ from copy import copy from inspect import isclass, getdoc from types import NoneType +from docstring_parser import parse from pydantic import BaseModel, create_model, Field from typing import Any, Type, List, get_args, get_origin, Tuple, Union, Optional, _GenericAlias from enum import Enum @@ -25,9 +26,10 @@ class PydanticDataType(Enum): ENUM (str): Represents an enum data type. CUSTOM_CLASS (str): Represents a custom class data type. """ + STRING = "string" TRIPLE_QUOTED_STRING = "triple_quoted_string" - MARKDOWN_STRING = "markdown_string" + MARKDOWN_CODE_BLOCK = "markdown_code_block" BOOLEAN = "boolean" INTEGER = "integer" FLOAT = "float" @@ -78,10 +80,10 @@ def map_pydantic_type_to_gbnf(pydantic_type: Type[Any]) -> str: def format_model_and_field_name(model_name: str) -> str: - parts = re.findall('[A-Z][^A-Z]*', model_name) + parts = re.findall("[A-Z][^A-Z]*", model_name) if not parts: # Check if the list is empty return model_name.lower().replace("_", "-") - return '-'.join(part.lower().replace("_", "-") for part in parts) + return "-".join(part.lower().replace("_", "-") for part in parts) def generate_list_rule(element_type): @@ -93,29 +95,31 @@ def generate_list_rule(element_type): """ rule_name = f"{map_pydantic_type_to_gbnf(element_type)}-list" element_rule = map_pydantic_type_to_gbnf(element_type) - list_rule = fr'{rule_name} ::= "[" {element_rule} ("," {element_rule})* "]"' + list_rule = rf'{rule_name} ::= "[" {element_rule} ("," {element_rule})* "]"' return list_rule def get_members_structure(cls, rule_name): if issubclass(cls, Enum): # Handle Enum types - members = [f'\"\\\"{member.value}\\\"\"' for name, member in cls.__members__.items()] + members = [f'"\\"{member.value}\\""' for name, member in cls.__members__.items()] return f"{cls.__name__.lower()} ::= " + " | ".join(members) if cls.__annotations__ and cls.__annotations__ != {}: result = f'{rule_name} ::= "{{"' type_list_rules = [] # Modify this comprehension - members = [f' \"\\\"{name}\\\"\" ":" {map_pydantic_type_to_gbnf(param_type)}' - for name, param_type in cls.__annotations__.items() - if name != 'self'] + members = [ + f' "\\"{name}\\"" ":" {map_pydantic_type_to_gbnf(param_type)}' + for name, param_type in cls.__annotations__.items() + if name != "self" + ] result += '"," '.join(members) result += ' "}"' return result, type_list_rules elif rule_name == "custom-class-any": - result = f'{rule_name} ::= ' - result += 'value' + result = f"{rule_name} ::= " + result += "value" type_list_rules = [] return result, type_list_rules else: @@ -124,9 +128,11 @@ def get_members_structure(cls, rule_name): result = f'{rule_name} ::= "{{"' type_list_rules = [] # Modify this comprehension too - members = [f' \"\\\"{name}\\\"\" ":" {map_pydantic_type_to_gbnf(param.annotation)}' - for name, param in parameters.items() - if name != 'self' and param.annotation != inspect.Parameter.empty] + members = [ + f' "\\"{name}\\"" ":" {map_pydantic_type_to_gbnf(param.annotation)}' + for name, param in parameters.items() + if name != "self" and param.annotation != inspect.Parameter.empty + ] result += '", "'.join(members) result += ' "}"' @@ -141,8 +147,8 @@ def regex_to_gbnf(regex_pattern: str) -> str: gbnf_rule = regex_pattern # Translate common regex components to GBNF - gbnf_rule = gbnf_rule.replace('\\d', '[0-9]') - gbnf_rule = gbnf_rule.replace('\\s', '[ \t\n]') + gbnf_rule = gbnf_rule.replace("\\d", "[0-9]") + gbnf_rule = gbnf_rule.replace("\\s", "[ \t\n]") # Handle quantifiers and other regex syntax that is similar in GBNF # (e.g., '*', '+', '?', character classes) @@ -158,12 +164,12 @@ def generate_gbnf_integer_rules(max_digit=None, min_digit=None): Generates GBNF (Generalized Backus-Naur Form) rules for integers based on the given maximum and minimum digits. Parameters: - max_digit (int): The maximum number of digits for the integer. Default is None. - min_digit (int): The minimum number of digits for the integer. Default is None. + max_digit (int): The maximum number of digits for the integer. Default is None. + min_digit (int): The minimum number of digits for the integer. Default is None. Returns: - integer_rule (str): The identifier for the integer rule generated. - additional_rules (list): A list of additional rules generated based on the given maximum and minimum digits. + integer_rule (str): The identifier for the integer rule generated. + additional_rules (list): A list of additional rules generated based on the given maximum and minimum digits. """ additional_rules = [] @@ -178,21 +184,21 @@ def generate_gbnf_integer_rules(max_digit=None, min_digit=None): # Handling Integer Rules if max_digit is not None or min_digit is not None: # Start with an empty rule part - integer_rule_part = '' + integer_rule_part = "" # Add mandatory digits as per min_digit if min_digit is not None: - integer_rule_part += '[0-9] ' * min_digit + integer_rule_part += "[0-9] " * min_digit # Add optional digits up to max_digit if max_digit is not None: optional_digits = max_digit - (min_digit if min_digit is not None else 0) - integer_rule_part += ''.join(['[0-9]? ' for _ in range(optional_digits)]) + integer_rule_part += "".join(["[0-9]? " for _ in range(optional_digits)]) # Trim the rule part and append it to additional rules integer_rule_part = integer_rule_part.strip() if integer_rule_part: - additional_rules.append(f'{integer_rule} ::= {integer_rule_part}') + additional_rules.append(f"{integer_rule} ::= {integer_rule_part}") return integer_rule, additional_rules @@ -224,21 +230,26 @@ def generate_gbnf_float_rules(max_digit=None, min_digit=None, max_precision=None additional_rules = [] # Define the integer part rule - integer_part_rule = "integer-part" + (f"-max{max_digit}" if max_digit is not None else "") + ( + integer_part_rule = ( + "integer-part" + (f"-max{max_digit}" if max_digit is not None else "") + ( f"-min{min_digit}" if min_digit is not None else "") + ) # Define the fractional part rule based on precision constraints fractional_part_rule = "fractional-part" - fractional_rule_part = '' + fractional_rule_part = "" if max_precision is not None or min_precision is not None: fractional_part_rule += (f"-max{max_precision}" if max_precision is not None else "") + ( - f"-min{min_precision}" if min_precision is not None else "") + f"-min{min_precision}" if min_precision is not None else "" + ) # Minimum number of digits - fractional_rule_part = '[0-9]' * (min_precision if min_precision is not None else 1) + fractional_rule_part = "[0-9]" * (min_precision if min_precision is not None else 1) # Optional additional digits - fractional_rule_part += ''.join([' [0-9]?'] * ( - (max_precision - (min_precision if min_precision is not None else 1)) if max_precision is not None else 0)) - additional_rules.append(f'{fractional_part_rule} ::= {fractional_rule_part}') + fractional_rule_part += "".join( + [" [0-9]?"] * ((max_precision - ( + min_precision if min_precision is not None else 1)) if max_precision is not None else 0) + ) + additional_rules.append(f"{fractional_part_rule} ::= {fractional_rule_part}") # Define the float rule float_rule = f"float-{max_digit if max_digit is not None else 'X'}-{min_digit if min_digit is not None else 'X'}-{max_precision if max_precision is not None else 'X'}-{min_precision if min_precision is not None else 'X'}" @@ -246,20 +257,19 @@ def generate_gbnf_float_rules(max_digit=None, min_digit=None, max_precision=None # Generating the integer part rule definition, if necessary if max_digit is not None or min_digit is not None: - integer_rule_part = '[0-9]' + integer_rule_part = "[0-9]" if min_digit is not None and min_digit > 1: - integer_rule_part += ' [0-9]' * (min_digit - 1) + integer_rule_part += " [0-9]" * (min_digit - 1) if max_digit is not None: - integer_rule_part += ''.join([' [0-9]?'] * (max_digit - (min_digit if min_digit is not None else 1))) - additional_rules.append(f'{integer_part_rule} ::= {integer_rule_part.strip()}') + integer_rule_part += "".join([" [0-9]?"] * (max_digit - (min_digit if min_digit is not None else 1))) + additional_rules.append(f"{integer_part_rule} ::= {integer_rule_part.strip()}") return float_rule, additional_rules -def generate_gbnf_rule_for_type(model_name, field_name, - field_type, is_optional, processed_models, created_rules, - field_info=None) -> \ - Tuple[str, list]: +def generate_gbnf_rule_for_type( + model_name, field_name, field_type, is_optional, processed_models, created_rules, field_info=None +) -> Tuple[str, list]: """ Generate GBNF rule for a given field type. @@ -282,20 +292,19 @@ def generate_gbnf_rule_for_type(model_name, field_name, if isclass(field_type) and issubclass(field_type, BaseModel): nested_model_name = format_model_and_field_name(field_type.__name__) - nested_model_rules = generate_gbnf_grammar(field_type, processed_models, created_rules) + nested_model_rules, _ = generate_gbnf_grammar(field_type, processed_models, created_rules) rules.extend(nested_model_rules) gbnf_type, rules = nested_model_name, rules elif isclass(field_type) and issubclass(field_type, Enum): - enum_values = [f'\"\\\"{e.value}\\\"\"' for e in field_type] # Adding escaped quotes + enum_values = [f'"\\"{e.value}\\""' for e in field_type] # Adding escaped quotes enum_rule = f"{model_name}-{field_name} ::= {' | '.join(enum_values)}" rules.append(enum_rule) gbnf_type, rules = model_name + "-" + field_name, rules - elif get_origin(field_type) == list or field_type == list: # Array + elif get_origin(field_type) == list: # Array element_type = get_args(field_type)[0] - element_rule_name, additional_rules = generate_gbnf_rule_for_type(model_name, - f"{field_name}-element", - element_type, is_optional, processed_models, - created_rules) + element_rule_name, additional_rules = generate_gbnf_rule_for_type( + model_name, f"{field_name}-element", element_type, is_optional, processed_models, created_rules + ) rules.extend(additional_rules) array_rule = f"""{model_name}-{field_name} ::= "[" ws {element_rule_name} ("," ws {element_rule_name})* "]" """ rules.append(array_rule) @@ -303,10 +312,9 @@ def generate_gbnf_rule_for_type(model_name, field_name, elif get_origin(field_type) == set or field_type == set: # Array element_type = get_args(field_type)[0] - element_rule_name, additional_rules = generate_gbnf_rule_for_type(model_name, - f"{field_name}-element", - element_type, is_optional, processed_models, - created_rules) + element_rule_name, additional_rules = generate_gbnf_rule_for_type( + model_name, f"{field_name}-element", element_type, is_optional, processed_models, created_rules + ) rules.extend(additional_rules) array_rule = f"""{model_name}-{field_name} ::= "[" ws {element_rule_name} ("," ws {element_rule_name})* "]" """ rules.append(array_rule) @@ -318,15 +326,13 @@ def generate_gbnf_rule_for_type(model_name, field_name, elif gbnf_type.startswith("custom-dict-"): key_type, value_type = get_args(field_type) - additional_key_type, additional_key_rules = generate_gbnf_rule_for_type(model_name, - f"{field_name}-key-type", - key_type, is_optional, processed_models, - created_rules) - additional_value_type, additional_value_rules = generate_gbnf_rule_for_type(model_name, - f"{field_name}-value-type", - value_type, is_optional, - processed_models, created_rules) - gbnf_type = fr'{gbnf_type} ::= "{{" ( {additional_key_type} ":" {additional_value_type} ("," {additional_key_type} ":" {additional_value_type})* )? "}}" ' + additional_key_type, additional_key_rules = generate_gbnf_rule_for_type( + model_name, f"{field_name}-key-type", key_type, is_optional, processed_models, created_rules + ) + additional_value_type, additional_value_rules = generate_gbnf_rule_for_type( + model_name, f"{field_name}-value-type", value_type, is_optional, processed_models, created_rules + ) + gbnf_type = rf'{gbnf_type} ::= "{{" ( {additional_key_type} ": " {additional_value_type} ("," "\n" ws {additional_key_type} ":" {additional_value_type})* )? "}}" ' rules.extend(additional_key_rules) rules.extend(additional_value_rules) @@ -336,19 +342,16 @@ def generate_gbnf_rule_for_type(model_name, field_name, for union_type in union_types: if isinstance(union_type, _GenericAlias): - union_gbnf_type, union_rules_list = generate_gbnf_rule_for_type(model_name, - field_name, union_type, - False, - processed_models, created_rules) + union_gbnf_type, union_rules_list = generate_gbnf_rule_for_type( + model_name, field_name, union_type, False, processed_models, created_rules + ) union_rules.append(union_gbnf_type) rules.extend(union_rules_list) - elif not issubclass(union_type, NoneType): - union_gbnf_type, union_rules_list = generate_gbnf_rule_for_type(model_name, - field_name, union_type, - False, - processed_models, created_rules) + union_gbnf_type, union_rules_list = generate_gbnf_rule_for_type( + model_name, field_name, union_type, False, processed_models, created_rules + ) union_rules.append(union_gbnf_type) rules.extend(union_rules_list) @@ -363,45 +366,58 @@ def generate_gbnf_rule_for_type(model_name, field_name, else: gbnf_type = f"{model_name}-{field_name}-union" elif isclass(field_type) and issubclass(field_type, str): - if field_info and hasattr(field_info, 'json_schema_extra') and field_info.json_schema_extra is not None: - - triple_quoted_string = field_info.json_schema_extra.get('triple_quoted_string', False) - markdown_string = field_info.json_schema_extra.get('markdown_string', False) + if field_info and hasattr(field_info, "json_schema_extra") and field_info.json_schema_extra is not None: + triple_quoted_string = field_info.json_schema_extra.get("triple_quoted_string", False) + markdown_string = field_info.json_schema_extra.get("markdown_code_block", False) gbnf_type = PydanticDataType.TRIPLE_QUOTED_STRING.value if triple_quoted_string else PydanticDataType.STRING.value - gbnf_type = PydanticDataType.MARKDOWN_STRING.value if markdown_string else gbnf_type + gbnf_type = PydanticDataType.MARKDOWN_CODE_BLOCK.value if markdown_string else gbnf_type - elif field_info and hasattr(field_info, 'pattern'): + elif field_info and hasattr(field_info, "pattern"): # Convert regex pattern to grammar rule regex_pattern = field_info.regex.pattern gbnf_type = f"pattern-{field_name} ::= {regex_to_gbnf(regex_pattern)}" else: gbnf_type = PydanticDataType.STRING.value - elif isclass(field_type) and issubclass(field_type, float) and field_info and hasattr(field_info, - 'json_schema_extra') and field_info.json_schema_extra is not None: + elif ( + isclass(field_type) + and issubclass(field_type, float) + and field_info + and hasattr(field_info, "json_schema_extra") + and field_info.json_schema_extra is not None + ): # Retrieve precision attributes for floats - max_precision = field_info.json_schema_extra.get('max_precision') if field_info and hasattr(field_info, - 'json_schema_extra') else None - min_precision = field_info.json_schema_extra.get('min_precision') if field_info and hasattr(field_info, - 'json_schema_extra') else None - max_digits = field_info.json_schema_extra.get('max_digit') if field_info and hasattr(field_info, - 'json_schema_extra') else None - min_digits = field_info.json_schema_extra.get('min_digit') if field_info and hasattr(field_info, - 'json_schema_extra') else None + max_precision = ( + field_info.json_schema_extra.get("max_precision") if field_info and hasattr(field_info, + "json_schema_extra") else None + ) + min_precision = ( + field_info.json_schema_extra.get("min_precision") if field_info and hasattr(field_info, + "json_schema_extra") else None + ) + max_digits = field_info.json_schema_extra.get("max_digit") if field_info and hasattr(field_info, + "json_schema_extra") else None + min_digits = field_info.json_schema_extra.get("min_digit") if field_info and hasattr(field_info, + "json_schema_extra") else None # Generate GBNF rule for float with given attributes - gbnf_type, rules = generate_gbnf_float_rules(max_digit=max_digits, min_digit=min_digits, - max_precision=max_precision, - min_precision=min_precision) + gbnf_type, rules = generate_gbnf_float_rules( + max_digit=max_digits, min_digit=min_digits, max_precision=max_precision, min_precision=min_precision + ) - elif isclass(field_type) and issubclass(field_type, int) and field_info and hasattr(field_info, - 'json_schema_extra') and field_info.json_schema_extra is not None: + elif ( + isclass(field_type) + and issubclass(field_type, int) + and field_info + and hasattr(field_info, "json_schema_extra") + and field_info.json_schema_extra is not None + ): # Retrieve digit attributes for integers - max_digits = field_info.json_schema_extra.get('max_digit') if field_info and hasattr(field_info, - 'json_schema_extra') else None - min_digits = field_info.json_schema_extra.get('min_digit') if field_info and hasattr(field_info, - 'json_schema_extra') else None + max_digits = field_info.json_schema_extra.get("max_digit") if field_info and hasattr(field_info, + "json_schema_extra") else None + min_digits = field_info.json_schema_extra.get("min_digit") if field_info and hasattr(field_info, + "json_schema_extra") else None # Generate GBNF rule for integer with given attributes gbnf_type, rules = generate_gbnf_integer_rules(max_digit=max_digits, min_digit=min_digits) @@ -443,13 +459,13 @@ def generate_gbnf_grammar(model: Type[BaseModel], processed_models: set, created if not issubclass(model, BaseModel): # For non-Pydantic classes, generate model_fields from __annotations__ or __init__ - if hasattr(model, '__annotations__') and model.__annotations__: + if hasattr(model, "__annotations__") and model.__annotations__: model_fields = {name: (typ, ...) for name, typ in model.__annotations__.items()} else: init_signature = inspect.signature(model.__init__) parameters = init_signature.parameters - model_fields = {name: (param.annotation, param.default) for name, param in parameters.items() - if name != 'self'} + model_fields = {name: (param.annotation, param.default) for name, param in parameters.items() if + name != "self"} else: # For Pydantic models, use model_fields and check for ellipsis (required fields) model_fields = model.__annotations__ @@ -469,51 +485,55 @@ def generate_gbnf_grammar(model: Type[BaseModel], processed_models: set, created field_type = field_info field_info = model.model_fields[field_name] is_optional = field_info.is_required is False and get_origin(field_type) is Optional - rule_name, additional_rules = generate_gbnf_rule_for_type(model_name, - format_model_and_field_name(field_name), - field_type, is_optional, - processed_models, created_rules, field_info) - look_for_markdown_code_block = True if rule_name == "markdown_string" else False + rule_name, additional_rules = generate_gbnf_rule_for_type( + model_name, format_model_and_field_name(field_name), field_type, is_optional, processed_models, + created_rules, field_info + ) + look_for_markdown_code_block = True if rule_name == "markdown_code_block" else False look_for_triple_quoted_string = True if rule_name == "triple_quoted_string" else False if not look_for_markdown_code_block and not look_for_triple_quoted_string: if rule_name not in created_rules: created_rules[rule_name] = additional_rules - model_rule_parts.append(f' ws \"\\\"{field_name}\\\"\" ": " {rule_name}') # Adding escaped quotes + model_rule_parts.append(f' ws "\\"{field_name}\\"" ":" ws {rule_name}') # Adding escaped quotes nested_rules.extend(additional_rules) else: - has_triple_quoted_string = look_for_markdown_code_block - has_markdown_code_block = look_for_triple_quoted_string + has_triple_quoted_string = look_for_triple_quoted_string + has_markdown_code_block = look_for_markdown_code_block fields_joined = r' "," "\n" '.join(model_rule_parts) - model_rule = fr'{model_name} ::= "{{" "\n" {fields_joined} "\n" ws "}}"' - - if look_for_markdown_code_block or look_for_triple_quoted_string: - model_rule += ' ws "}"' + model_rule = rf'{model_name} ::= "{{" "\n" {fields_joined} "\n" ws "}}"' + has_special_string = False if has_triple_quoted_string: + model_rule += '"\\n" ws "}"' model_rule += '"\\n" triple-quoted-string' + has_special_string = True if has_markdown_code_block: + model_rule += '"\\n" ws "}"' model_rule += '"\\n" markdown-code-block' + has_special_string = True all_rules = [model_rule] + nested_rules - return all_rules, has_markdown_code_block, has_triple_quoted_string + return all_rules, has_special_string -def generate_gbnf_grammar_from_pydantic_models(models: List[Type[BaseModel]], outer_object_name: str = None, - outer_object_content: str = None, list_of_outputs: bool = False) -> str: +def generate_gbnf_grammar_from_pydantic_models( + models: List[Type[BaseModel]], outer_object_name: str = None, outer_object_content: str = None, + list_of_outputs: bool = False +) -> str: """ Generate GBNF Grammar from Pydantic Models. This method takes a list of Pydantic models and uses them to generate a GBNF grammar string. The generated grammar string can be used for parsing and validating data using the generated * grammar. - Parameters: - models (List[Type[BaseModel]]): A list of Pydantic models to generate the grammar from. - outer_object_name (str): Outer object name for the GBNF grammar. If None, no outer object will be generated. Eg. "function" for function calling. - outer_object_content (str): Content for the outer rule in the GBNF grammar. Eg. "function_parameters" or "params" for function calling. - list_of_outputs (str, optional): Allows a list of output objects + Args: + models (List[Type[BaseModel]]): A list of Pydantic models to generate the grammar from. + outer_object_name (str): Outer object name for the GBNF grammar. If None, no outer object will be generated. Eg. "function" for function calling. + outer_object_content (str): Content for the outer rule in the GBNF grammar. Eg. "function_parameters" or "params" for function calling. + list_of_outputs (str, optional): Allows a list of output objects Returns: - str: The generated GBNF grammar string. + str: The generated GBNF grammar string. Examples: models = [UserModel, PostModel] @@ -527,52 +547,53 @@ def generate_gbnf_grammar_from_pydantic_models(models: List[Type[BaseModel]], ou all_rules = [] created_rules = {} if outer_object_name is None: - for model in models: - model_rules, _, _ = generate_gbnf_grammar(model, - processed_models, created_rules) + model_rules, _ = generate_gbnf_grammar(model, processed_models, created_rules) all_rules.extend(model_rules) if list_of_outputs: - root_rule = r'root ::= ws "[" grammar-models ("," grammar-models)* "]"' + "\n" + root_rule = r'root ::= (" "| "\n") "[" ws grammar-models ("," ws grammar-models)* ws "]"' + "\n" else: - root_rule = r'root ::= ws grammar-models' + "\n" + root_rule = r'root ::= (" "| "\n") grammar-models' + "\n" root_rule += "grammar-models ::= " + " | ".join( [format_model_and_field_name(model.__name__) for model in models]) all_rules.insert(0, root_rule) return "\n".join(all_rules) elif outer_object_name is not None: if list_of_outputs: - root_rule = fr'root ::= ws "[" {format_model_and_field_name(outer_object_name)} ("," {format_model_and_field_name(outer_object_name)})* "]"' + "\n" + root_rule = ( + rf'root ::= (" "| "\n") "[" ws {format_model_and_field_name(outer_object_name)} ("," ws {format_model_and_field_name(outer_object_name)})* ws "]"' + + "\n" + ) else: root_rule = f"root ::= {format_model_and_field_name(outer_object_name)}\n" - model_rule = fr'{format_model_and_field_name(outer_object_name)} ::= ws "{{" ws "\"{outer_object_name}\"" ": " grammar-models' + model_rule = ( + rf'{format_model_and_field_name(outer_object_name)} ::= (" "| "\n") "{{" ws "\"{outer_object_name}\"" ":" ws grammar-models' + ) fields_joined = " | ".join( - [fr'{format_model_and_field_name(model.__name__)}-grammar-model' for model in models]) + [rf"{format_model_and_field_name(model.__name__)}-grammar-model" for model in models]) - grammar_model_rules = f'\ngrammar-models ::= {fields_joined}' + grammar_model_rules = f"\ngrammar-models ::= {fields_joined}" mod_rules = [] for model in models: - mod_rule = fr'{format_model_and_field_name(model.__name__)}-grammar-model ::= ws' - mod_rule += fr'"\"{format_model_and_field_name(model.__name__)}\"" "," ws "\"{outer_object_content}\"" ws ":" ws {format_model_and_field_name(model.__name__)}' + '\n' + mod_rule = rf"{format_model_and_field_name(model.__name__)}-grammar-model ::= " + mod_rule += ( + rf'"\"{model.__name__}\"" "," ws "\"{outer_object_content}\"" ":" ws {format_model_and_field_name(model.__name__)}' + "\n" + ) mod_rules.append(mod_rule) grammar_model_rules += "\n" + "\n".join(mod_rules) - look_for_markdown_code_block = False - look_for_triple_quoted_string = False + for model in models: - model_rules, markdown_block, triple_quoted_string = generate_gbnf_grammar(model, - processed_models, created_rules) + model_rules, has_special_string = generate_gbnf_grammar(model, processed_models, + created_rules) + + if not has_special_string: + model_rules[0] += r'"\n" ws "}"' + all_rules.extend(model_rules) - if markdown_block: - look_for_markdown_code_block = True - if triple_quoted_string: - look_for_triple_quoted_string = True - - if not look_for_markdown_code_block and not look_for_triple_quoted_string: - model_rule += ' ws "}"' all_rules.insert(0, root_rule + model_rule + grammar_model_rules) return "\n".join(all_rules) @@ -582,10 +603,10 @@ def get_primitive_grammar(grammar): Returns the needed GBNF primitive grammar for a given GBNF grammar string. Args: - grammar (str): The string containing the GBNF grammar. + grammar (str): The string containing the GBNF grammar. Returns: - str: GBNF primitive grammar string. + str: GBNF primitive grammar string. """ type_list = [] if "string-list" in grammar: @@ -611,7 +632,7 @@ integer ::= [0-9]+""" any_block = "" if "custom-class-any" in grammar: - any_block = ''' + any_block = """ value ::= object | array | string | number | boolean | null object ::= @@ -626,7 +647,7 @@ array ::= ("," ws value)* )? "]" ws -number ::= integer | float''' +number ::= integer | float""" markdown_code_block_grammar = "" if "markdown-code-block" in grammar: @@ -641,90 +662,32 @@ closing-triple-ticks ::= "```" "\n"''' triple-quoted-string ::= triple-quotes triple-quoted-string-content triple-quotes triple-quoted-string-content ::= ( [^'] | "'" [^'] | "'" "'" [^'] )* triple-quotes ::= "'''" """ - return "\n" + '\n'.join(additional_grammar) + any_block + primitive_grammar + markdown_code_block_grammar + return "\n" + "\n".join(additional_grammar) + any_block + primitive_grammar + markdown_code_block_grammar -def generate_field_markdown(field_name: str, field_type: Type[Any], model: Type[BaseModel], depth=1) -> str: - indent = ' ' * depth - field_markdown = f"{indent}- **{field_name}** (`{field_type.__name__}`): " - - # Extracting field description from Pydantic Field using __model_fields__ - field_info = model.model_fields.get(field_name) - field_description = field_info.description if field_info and field_info.description else "No description available." - - field_markdown += field_description + '\n' - - # Handling nested BaseModel fields - if isclass(field_type) and issubclass(field_type, BaseModel): - field_markdown += f"{indent} - Details:\n" - for name, type_ in field_type.__annotations__.items(): - field_markdown += generate_field_markdown(name, type_, field_type, depth + 2) - - return field_markdown - - -def generate_markdown_report(pydantic_models: List[Type[BaseModel]]) -> str: - markdown = "" - for model in pydantic_models: - markdown += f"### {format_model_and_field_name(model.__name__)}\n" - - # Check if the model's docstring is different from BaseModel's docstring - class_doc = getdoc(model) - base_class_doc = getdoc(BaseModel) - class_description = class_doc if class_doc and class_doc != base_class_doc else "No specific description available." - - markdown += f"{class_description}\n\n" - markdown += "#### Fields\n" - - if isclass(model) and issubclass(model, BaseModel): - for name, field_type in model.__annotations__.items(): - markdown += generate_field_markdown(format_model_and_field_name(name), field_type, model) - markdown += "\n" - - return markdown - - -def format_json_example(example: dict, depth: int) -> str: +def generate_markdown_documentation( + pydantic_models: List[Type[BaseModel]], model_prefix="Model", fields_prefix="Fields", + documentation_with_field_description=True +) -> str: """ - Format a JSON example into a readable string with indentation. + Generate markdown documentation for a list of Pydantic models. Args: - example (dict): JSON example to be formatted. - depth (int): Indentation depth. + pydantic_models (List[Type[BaseModel]]): List of Pydantic model classes. + model_prefix (str): Prefix for the model section. + fields_prefix (str): Prefix for the fields section. + documentation_with_field_description (bool): Include field descriptions in the documentation. Returns: - str: Formatted JSON example string. - """ - indent = ' ' * depth - formatted_example = '{\n' - for key, value in example.items(): - value_text = f"'{value}'" if isinstance(value, str) else value - formatted_example += f"{indent}{key}: {value_text},\n" - formatted_example = formatted_example.rstrip(',\n') + '\n' + indent + '}' - return formatted_example - - -def generate_text_documentation(pydantic_models: List[Type[BaseModel]], model_prefix="Model", - fields_prefix="Fields", documentation_with_field_description=True) -> str: - """ - Generate text documentation for a list of Pydantic models. - - Args: - pydantic_models (List[Type[BaseModel]]): List of Pydantic model classes. - model_prefix (str): Prefix for the model section. - fields_prefix (str): Prefix for the fields section. - documentation_with_field_description (bool): Include field descriptions in the documentation. - - Returns: - str: Generated text documentation. + str: Generated text documentation. """ documentation = "" pyd_models = [(model, True) for model in pydantic_models] for model, add_prefix in pyd_models: if add_prefix: - documentation += f"{model_prefix}: {format_model_and_field_name(model.__name__)}\n" + documentation += f"{model_prefix}: {model.__name__}\n" else: - documentation += f"Model: {format_model_and_field_name(model.__name__)}\n" + documentation += f"Model: {model.__name__}\n" # Handling multi-line model description with proper indentation @@ -733,7 +696,7 @@ def generate_text_documentation(pydantic_models: List[Type[BaseModel]], model_pr class_description = class_doc if class_doc and class_doc != base_class_doc else "" if class_description != "": documentation += " Description: " - documentation += "\n" + format_multiline_description(class_description, 2) + "\n" + documentation += format_multiline_description(class_description, 0) + "\n" if add_prefix: # Indenting the fields section @@ -753,35 +716,192 @@ def generate_text_documentation(pydantic_models: List[Type[BaseModel]], model_pr for element_type in element_types: if isclass(element_type) and issubclass(element_type, BaseModel): pyd_models.append((element_type, False)) - documentation += generate_field_text(name, field_type, model, - documentation_with_field_description=documentation_with_field_description) + documentation += generate_field_markdown( + name, field_type, model, documentation_with_field_description=documentation_with_field_description + ) documentation += "\n" - if hasattr(model, 'Config') and hasattr(model.Config, - 'json_schema_extra') and 'example' in model.Config.json_schema_extra: + if hasattr(model, "Config") and hasattr(model.Config, + "json_schema_extra") and "example" in model.Config.json_schema_extra: documentation += f" Expected Example Output for {format_model_and_field_name(model.__name__)}:\n" - json_example = json.dumps(model.Config.json_schema_extra['example']) + json_example = json.dumps(model.Config.json_schema_extra["example"]) documentation += format_multiline_description(json_example, 2) + "\n" return documentation -def generate_field_text(field_name: str, field_type: Type[Any], model: Type[BaseModel], depth=1, - documentation_with_field_description=True) -> str: +def generate_field_markdown( + field_name: str, field_type: Type[Any], model: Type[BaseModel], depth=1, + documentation_with_field_description=True +) -> str: + """ + Generate markdown documentation for a Pydantic model field. + + Args: + field_name (str): Name of the field. + field_type (Type[Any]): Type of the field. + model (Type[BaseModel]): Pydantic model class. + depth (int): Indentation depth in the documentation. + documentation_with_field_description (bool): Include field descriptions in the documentation. + + Returns: + str: Generated text documentation for the field. + """ + indent = " " * depth + + field_info = model.model_fields.get(field_name) + field_description = field_info.description if field_info and field_info.description else "" + + if get_origin(field_type) == list: + element_type = get_args(field_type)[0] + field_text = f"{indent}{field_name} ({format_model_and_field_name(field_type.__name__)} of {format_model_and_field_name(element_type.__name__)})" + if field_description != "": + field_text += ":\n" + else: + field_text += "\n" + elif get_origin(field_type) == Union: + element_types = get_args(field_type) + types = [] + for element_type in element_types: + types.append(format_model_and_field_name(element_type.__name__)) + field_text = f"{indent}{field_name} ({' or '.join(types)})" + if field_description != "": + field_text += ":\n" + else: + field_text += "\n" + else: + field_text = f"{indent}{field_name} ({format_model_and_field_name(field_type.__name__)})" + if field_description != "": + field_text += ":\n" + else: + field_text += "\n" + + if not documentation_with_field_description: + return field_text + + if field_description != "": + field_text += f" Description: " + field_description + "\n" + + # Check for and include field-specific examples if available + if hasattr(model, "Config") and hasattr(model.Config, + "json_schema_extra") and "example" in model.Config.json_schema_extra: + field_example = model.Config.json_schema_extra["example"].get(field_name) + if field_example is not None: + example_text = f"'{field_example}'" if isinstance(field_example, str) else field_example + field_text += f"{indent} Example: {example_text}\n" + + if isclass(field_type) and issubclass(field_type, BaseModel): + field_text += f"{indent} Details:\n" + for name, type_ in field_type.__annotations__.items(): + field_text += generate_field_markdown(name, type_, field_type, depth + 2) + + return field_text + + +def format_json_example(example: dict, depth: int) -> str: + """ + Format a JSON example into a readable string with indentation. + + Args: + example (dict): JSON example to be formatted. + depth (int): Indentation depth. + + Returns: + str: Formatted JSON example string. + """ + indent = " " * depth + formatted_example = "{\n" + for key, value in example.items(): + value_text = f"'{value}'" if isinstance(value, str) else value + formatted_example += f"{indent}{key}: {value_text},\n" + formatted_example = formatted_example.rstrip(",\n") + "\n" + indent + "}" + return formatted_example + + +def generate_text_documentation( + pydantic_models: List[Type[BaseModel]], model_prefix="Model", fields_prefix="Fields", + documentation_with_field_description=True +) -> str: + """ + Generate text documentation for a list of Pydantic models. + + Args: + pydantic_models (List[Type[BaseModel]]): List of Pydantic model classes. + model_prefix (str): Prefix for the model section. + fields_prefix (str): Prefix for the fields section. + documentation_with_field_description (bool): Include field descriptions in the documentation. + + Returns: + str: Generated text documentation. + """ + documentation = "" + pyd_models = [(model, True) for model in pydantic_models] + for model, add_prefix in pyd_models: + if add_prefix: + documentation += f"{model_prefix}: {model.__name__}\n" + else: + documentation += f"Model: {model.__name__}\n" + + # Handling multi-line model description with proper indentation + + class_doc = getdoc(model) + base_class_doc = getdoc(BaseModel) + class_description = class_doc if class_doc and class_doc != base_class_doc else "" + if class_description != "": + documentation += " Description: " + documentation += "\n" + format_multiline_description(class_description, 2) + "\n" + + if isclass(model) and issubclass(model, BaseModel): + documentation_fields = "" + for name, field_type in model.__annotations__.items(): + # if name == "markdown_code_block": + # continue + if get_origin(field_type) == list: + element_type = get_args(field_type)[0] + if isclass(element_type) and issubclass(element_type, BaseModel): + pyd_models.append((element_type, False)) + if get_origin(field_type) == Union: + element_types = get_args(field_type) + for element_type in element_types: + if isclass(element_type) and issubclass(element_type, BaseModel): + pyd_models.append((element_type, False)) + documentation_fields += generate_field_text( + name, field_type, model, documentation_with_field_description=documentation_with_field_description + ) + if documentation_fields != "": + if add_prefix: + documentation += f" {fields_prefix}:\n{documentation_fields}" + else: + documentation += f" Fields:\n{documentation_fields}" + documentation += "\n" + + if hasattr(model, "Config") and hasattr(model.Config, + "json_schema_extra") and "example" in model.Config.json_schema_extra: + documentation += f" Expected Example Output for {format_model_and_field_name(model.__name__)}:\n" + json_example = json.dumps(model.Config.json_schema_extra["example"]) + documentation += format_multiline_description(json_example, 2) + "\n" + + return documentation + + +def generate_field_text( + field_name: str, field_type: Type[Any], model: Type[BaseModel], depth=1, + documentation_with_field_description=True +) -> str: """ Generate text documentation for a Pydantic model field. Args: - field_name (str): Name of the field. - field_type (Type[Any]): Type of the field. - model (Type[BaseModel]): Pydantic model class. - depth (int): Indentation depth in the documentation. - documentation_with_field_description (bool): Include field descriptions in the documentation. + field_name (str): Name of the field. + field_type (Type[Any]): Type of the field. + model (Type[BaseModel]): Pydantic model class. + depth (int): Indentation depth in the documentation. + documentation_with_field_description (bool): Include field descriptions in the documentation. Returns: - str: Generated text documentation for the field. + str: Generated text documentation for the field. """ - indent = ' ' * depth + indent = " " * depth field_info = model.model_fields.get(field_name) field_description = field_info.description if field_info and field_info.description else "" @@ -817,9 +937,9 @@ def generate_field_text(field_name: str, field_type: Type[Any], model: Type[Base field_text += f"{indent} Description: " + field_description + "\n" # Check for and include field-specific examples if available - if hasattr(model, 'Config') and hasattr(model.Config, - 'json_schema_extra') and 'example' in model.Config.json_schema_extra: - field_example = model.Config.json_schema_extra['example'].get(field_name) + if hasattr(model, "Config") and hasattr(model.Config, + "json_schema_extra") and "example" in model.Config.json_schema_extra: + field_example = model.Config.json_schema_extra["example"].get(field_name) if field_example is not None: example_text = f"'{field_example}'" if isinstance(field_example, str) else field_example field_text += f"{indent} Example: {example_text}\n" @@ -837,39 +957,40 @@ def format_multiline_description(description: str, indent_level: int) -> str: Format a multiline description with proper indentation. Args: - description (str): Multiline description. - indent_level (int): Indentation level. + description (str): Multiline description. + indent_level (int): Indentation level. Returns: - str: Formatted multiline description. + str: Formatted multiline description. """ - indent = ' ' * indent_level - return indent + description.replace('\n', '\n' + indent) + indent = " " * indent_level + return indent + description.replace("\n", "\n" + indent) -def save_gbnf_grammar_and_documentation(grammar, documentation, grammar_file_path="./grammar.gbnf", - documentation_file_path="./grammar_documentation.md"): +def save_gbnf_grammar_and_documentation( + grammar, documentation, grammar_file_path="./grammar.gbnf", documentation_file_path="./grammar_documentation.md" +): """ Save GBNF grammar and documentation to specified files. Args: - grammar (str): GBNF grammar string. - documentation (str): Documentation string. - grammar_file_path (str): File path to save the GBNF grammar. - documentation_file_path (str): File path to save the documentation. + grammar (str): GBNF grammar string. + documentation (str): Documentation string. + grammar_file_path (str): File path to save the GBNF grammar. + documentation_file_path (str): File path to save the documentation. Returns: - None + None """ try: - with open(grammar_file_path, 'w') as file: + with open(grammar_file_path, "w") as file: file.write(grammar + get_primitive_grammar(grammar)) print(f"Grammar successfully saved to {grammar_file_path}") except IOError as e: print(f"An error occurred while saving the grammar file: {e}") try: - with open(documentation_file_path, 'w') as file: + with open(documentation_file_path, "w") as file: file.write(documentation) print(f"Documentation successfully saved to {documentation_file_path}") except IOError as e: @@ -881,10 +1002,10 @@ def remove_empty_lines(string): Remove empty lines from a string. Args: - string (str): Input string. + string (str): Input string. Returns: - str: String with empty lines removed. + str: String with empty lines removed. """ lines = string.splitlines() non_empty_lines = [line for line in lines if line.strip() != ""] @@ -892,95 +1013,109 @@ def remove_empty_lines(string): return string_no_empty_lines -def generate_and_save_gbnf_grammar_and_documentation(pydantic_model_list, - grammar_file_path="./generated_grammar.gbnf", - documentation_file_path="./generated_grammar_documentation.md", - outer_object_name: str = None, - outer_object_content: str = None, - model_prefix: str = "Output Model", - fields_prefix: str = "Output Fields", - list_of_outputs: bool = False, - documentation_with_field_description=True): +def generate_and_save_gbnf_grammar_and_documentation( + pydantic_model_list, + grammar_file_path="./generated_grammar.gbnf", + documentation_file_path="./generated_grammar_documentation.md", + outer_object_name: str = None, + outer_object_content: str = None, + model_prefix: str = "Output Model", + fields_prefix: str = "Output Fields", + list_of_outputs: bool = False, + documentation_with_field_description=True, +): """ Generate GBNF grammar and documentation, and save them to specified files. Args: - pydantic_model_list: List of Pydantic model classes. - grammar_file_path (str): File path to save the generated GBNF grammar. - documentation_file_path (str): File path to save the generated documentation. - outer_object_name (str): Outer object name for the GBNF grammar. If None, no outer object will be generated. Eg. "function" for function calling. - outer_object_content (str): Content for the outer rule in the GBNF grammar. Eg. "function_parameters" or "params" for function calling. - model_prefix (str): Prefix for the model section in the documentation. - fields_prefix (str): Prefix for the fields section in the documentation. - list_of_outputs (bool): Whether the output is a list of items. - documentation_with_field_description (bool): Include field descriptions in the documentation. + pydantic_model_list: List of Pydantic model classes. + grammar_file_path (str): File path to save the generated GBNF grammar. + documentation_file_path (str): File path to save the generated documentation. + outer_object_name (str): Outer object name for the GBNF grammar. If None, no outer object will be generated. Eg. "function" for function calling. + outer_object_content (str): Content for the outer rule in the GBNF grammar. Eg. "function_parameters" or "params" for function calling. + model_prefix (str): Prefix for the model section in the documentation. + fields_prefix (str): Prefix for the fields section in the documentation. + list_of_outputs (bool): Whether the output is a list of items. + documentation_with_field_description (bool): Include field descriptions in the documentation. Returns: - None + None """ - documentation = generate_text_documentation(pydantic_model_list, model_prefix, fields_prefix, - documentation_with_field_description=documentation_with_field_description) - grammar = generate_gbnf_grammar_from_pydantic_models(pydantic_model_list, outer_object_name, - outer_object_content, list_of_outputs) + documentation = generate_markdown_documentation( + pydantic_model_list, model_prefix, fields_prefix, + documentation_with_field_description=documentation_with_field_description + ) + grammar = generate_gbnf_grammar_from_pydantic_models(pydantic_model_list, outer_object_name, outer_object_content, + list_of_outputs) grammar = remove_empty_lines(grammar) save_gbnf_grammar_and_documentation(grammar, documentation, grammar_file_path, documentation_file_path) -def generate_gbnf_grammar_and_documentation(pydantic_model_list, outer_object_name: str = None, - outer_object_content: str = None, - model_prefix: str = "Output Model", - fields_prefix: str = "Output Fields", list_of_outputs: bool = False, - documentation_with_field_description=True): +def generate_gbnf_grammar_and_documentation( + pydantic_model_list, + outer_object_name: str = None, + outer_object_content: str = None, + model_prefix: str = "Output Model", + fields_prefix: str = "Output Fields", + list_of_outputs: bool = False, + documentation_with_field_description=True, +): """ Generate GBNF grammar and documentation for a list of Pydantic models. Args: - pydantic_model_list: List of Pydantic model classes. - outer_object_name (str): Outer object name for the GBNF grammar. If None, no outer object will be generated. Eg. "function" for function calling. - outer_object_content (str): Content for the outer rule in the GBNF grammar. Eg. "function_parameters" or "params" for function calling. - model_prefix (str): Prefix for the model section in the documentation. - fields_prefix (str): Prefix for the fields section in the documentation. - list_of_outputs (bool): Whether the output is a list of items. - documentation_with_field_description (bool): Include field descriptions in the documentation. + pydantic_model_list: List of Pydantic model classes. + outer_object_name (str): Outer object name for the GBNF grammar. If None, no outer object will be generated. Eg. "function" for function calling. + outer_object_content (str): Content for the outer rule in the GBNF grammar. Eg. "function_parameters" or "params" for function calling. + model_prefix (str): Prefix for the model section in the documentation. + fields_prefix (str): Prefix for the fields section in the documentation. + list_of_outputs (bool): Whether the output is a list of items. + documentation_with_field_description (bool): Include field descriptions in the documentation. Returns: - tuple: GBNF grammar string, documentation string. + tuple: GBNF grammar string, documentation string. """ - documentation = generate_text_documentation(copy(pydantic_model_list), model_prefix, fields_prefix, - documentation_with_field_description=documentation_with_field_description) - grammar = generate_gbnf_grammar_from_pydantic_models(pydantic_model_list, outer_object_name, - outer_object_content, list_of_outputs) + documentation = generate_markdown_documentation( + copy(pydantic_model_list), model_prefix, fields_prefix, + documentation_with_field_description=documentation_with_field_description + ) + grammar = generate_gbnf_grammar_from_pydantic_models(pydantic_model_list, outer_object_name, outer_object_content, + list_of_outputs) grammar = remove_empty_lines(grammar + get_primitive_grammar(grammar)) return grammar, documentation -def generate_gbnf_grammar_and_documentation_from_dictionaries(dictionaries: List[dict], - outer_object_name: str = None, - outer_object_content: str = None, - model_prefix: str = "Output Model", - fields_prefix: str = "Output Fields", - list_of_outputs: bool = False, - documentation_with_field_description=True): +def generate_gbnf_grammar_and_documentation_from_dictionaries( + dictionaries: List[dict], + outer_object_name: str = None, + outer_object_content: str = None, + model_prefix: str = "Output Model", + fields_prefix: str = "Output Fields", + list_of_outputs: bool = False, + documentation_with_field_description=True, +): """ Generate GBNF grammar and documentation from a list of dictionaries. Args: - dictionaries (List[dict]): List of dictionaries representing Pydantic models. - outer_object_name (str): Outer object name for the GBNF grammar. If None, no outer object will be generated. Eg. "function" for function calling. - outer_object_content (str): Content for the outer rule in the GBNF grammar. Eg. "function_parameters" or "params" for function calling. - model_prefix (str): Prefix for the model section in the documentation. - fields_prefix (str): Prefix for the fields section in the documentation. - list_of_outputs (bool): Whether the output is a list of items. - documentation_with_field_description (bool): Include field descriptions in the documentation. + dictionaries (List[dict]): List of dictionaries representing Pydantic models. + outer_object_name (str): Outer object name for the GBNF grammar. If None, no outer object will be generated. Eg. "function" for function calling. + outer_object_content (str): Content for the outer rule in the GBNF grammar. Eg. "function_parameters" or "params" for function calling. + model_prefix (str): Prefix for the model section in the documentation. + fields_prefix (str): Prefix for the fields section in the documentation. + list_of_outputs (bool): Whether the output is a list of items. + documentation_with_field_description (bool): Include field descriptions in the documentation. Returns: - tuple: GBNF grammar string, documentation string. + tuple: GBNF grammar string, documentation string. """ pydantic_model_list = create_dynamic_models_from_dictionaries(dictionaries) - documentation = generate_text_documentation(copy(pydantic_model_list), model_prefix, fields_prefix, - documentation_with_field_description=documentation_with_field_description) - grammar = generate_gbnf_grammar_from_pydantic_models(pydantic_model_list, outer_object_name, - outer_object_content, list_of_outputs) + documentation = generate_markdown_documentation( + copy(pydantic_model_list), model_prefix, fields_prefix, + documentation_with_field_description=documentation_with_field_description + ) + grammar = generate_gbnf_grammar_from_pydantic_models(pydantic_model_list, outer_object_name, outer_object_content, + list_of_outputs) grammar = remove_empty_lines(grammar + get_primitive_grammar(grammar)) return grammar, documentation @@ -990,41 +1125,61 @@ def create_dynamic_model_from_function(func: Callable): Creates a dynamic Pydantic model from a given function's type hints and adds the function as a 'run' method. Args: - func (Callable): A function with type hints from which to create the model. + func (Callable): A function with type hints from which to create the model. Returns: - A dynamic Pydantic model class with the provided function as a 'run' method. + A dynamic Pydantic model class with the provided function as a 'run' method. """ - # Extracting type hints from the provided function - type_hints = get_type_hints(func) - type_hints.pop('return', None) - # Handling default values and annotations + # Get the signature of the function + sig = inspect.signature(func) + + # Parse the docstring + docstring = parse(func.__doc__) + dynamic_fields = {} - defaults = getattr(func, '__defaults__', ()) or () - defaults_index = len(type_hints) - len(defaults) + param_docs = [] + for param in sig.parameters.values(): + # Exclude 'self' parameter + if param.name == "self": + continue - for index, (name, typ) in enumerate(type_hints.items()): - if index >= defaults_index: - default_value = defaults[index - defaults_index] - dynamic_fields[name] = (typ, default_value) + # Assert that the parameter has a type annotation + if param.annotation == inspect.Parameter.empty: + raise TypeError(f"Parameter '{param.name}' in function '{func.__name__}' lacks a type annotation") + + # Find the parameter's description in the docstring + param_doc = next((d for d in docstring.params if d.arg_name == param.name), None) + + # Assert that the parameter has a description + if not param_doc or not param_doc.description: + raise ValueError( + f"Parameter '{param.name}' in function '{func.__name__}' lacks a description in the docstring") + + # Add parameter details to the schema + param_doc = next((d for d in docstring.params if d.arg_name == param.name), None) + param_docs.append((param.name, param_doc)) + if param.default == inspect.Parameter.empty: + default_value = ... else: - dynamic_fields[name] = (typ, ...) - + default_value = param.default + dynamic_fields[param.name] = ( + param.annotation if param.annotation != inspect.Parameter.empty else str, default_value) # Creating the dynamic model - dynamicModel = create_model(f'{func.__name__}', **dynamic_fields) + dynamic_model = create_model(f"{func.__name__}", **dynamic_fields) - dynamicModel.__doc__ = getdoc(func) + for param_doc in param_docs: + dynamic_model.model_fields[param_doc[0]].description = param_doc[1].description + + dynamic_model.__doc__ = docstring.short_description - # Wrapping the original function to handle instance 'self' def run_method_wrapper(self): - func_args = {name: getattr(self, name) for name in type_hints} + func_args = {name: getattr(self, name) for name, _ in dynamic_fields.items()} return func(**func_args) # Adding the wrapped function as a 'run' method - setattr(dynamicModel, 'run', run_method_wrapper) - - return dynamicModel + setattr(dynamic_model, "run", run_method_wrapper) + return dynamic_model def add_run_method_to_dynamic_model(model: Type[BaseModel], func: Callable): @@ -1032,11 +1187,11 @@ def add_run_method_to_dynamic_model(model: Type[BaseModel], func: Callable): Add a 'run' method to a dynamic Pydantic model, using the provided function. Args: - - model (Type[BaseModel]): Dynamic Pydantic model class. - - func (Callable): Function to be added as a 'run' method to the model. + model (Type[BaseModel]): Dynamic Pydantic model class. + func (Callable): Function to be added as a 'run' method to the model. Returns: - - Type[BaseModel]: Pydantic model class with the added 'run' method. + Type[BaseModel]: Pydantic model class with the added 'run' method. """ def run_method_wrapper(self): @@ -1044,7 +1199,7 @@ def add_run_method_to_dynamic_model(model: Type[BaseModel], func: Callable): return func(**func_args) # Adding the wrapped function as a 'run' method - setattr(model, 'run', run_method_wrapper) + setattr(model, "run", run_method_wrapper) return model @@ -1054,15 +1209,15 @@ def create_dynamic_models_from_dictionaries(dictionaries: List[dict]): Create a list of dynamic Pydantic model classes from a list of dictionaries. Args: - - dictionaries (List[dict]): List of dictionaries representing model structures. + dictionaries (List[dict]): List of dictionaries representing model structures. Returns: - - List[Type[BaseModel]]: List of generated dynamic Pydantic model classes. + List[Type[BaseModel]]: List of generated dynamic Pydantic model classes. """ dynamic_models = [] for func in dictionaries: model_name = format_model_and_field_name(func.get("name", "")) - dyn_model = convert_dictionary_to_to_pydantic_model(func, model_name) + dyn_model = convert_dictionary_to_pydantic_model(func, model_name) dynamic_models.append(dyn_model) return dynamic_models @@ -1080,12 +1235,12 @@ from enum import Enum def json_schema_to_python_types(schema): type_map = { - 'any': Any, - 'string': str, - 'number': float, - 'integer': int, - 'boolean': bool, - 'array': list, + "any": Any, + "string": str, + "number": float, + "integer": int, + "boolean": bool, + "array": list, } return type_map[schema] @@ -1094,58 +1249,64 @@ def list_to_enum(enum_name, values): return Enum(enum_name, {value: value for value in values}) -def convert_dictionary_to_to_pydantic_model(dictionary: dict, model_name: str = 'CustomModel') -> Type[BaseModel]: +def convert_dictionary_to_pydantic_model(dictionary: dict, model_name: str = "CustomModel") -> Type[BaseModel]: """ Convert a dictionary to a Pydantic model class. Args: - - dictionary (dict): Dictionary representing the model structure. - - model_name (str): Name of the generated Pydantic model. + dictionary (dict): Dictionary representing the model structure. + model_name (str): Name of the generated Pydantic model. Returns: - - Type[BaseModel]: Generated Pydantic model class. + Type[BaseModel]: Generated Pydantic model class. """ fields = {} if "properties" in dictionary: for field_name, field_data in dictionary.get("properties", {}).items(): - if field_data == 'object': - submodel = convert_dictionary_to_to_pydantic_model(dictionary, f'{model_name}_{field_name}') + if field_data == "object": + submodel = convert_dictionary_to_pydantic_model(dictionary, f"{model_name}_{field_name}") fields[field_name] = (submodel, ...) else: - field_type = field_data.get('type', 'str') + field_type = field_data.get("type", "str") if field_data.get("enum", []): fields[field_name] = (list_to_enum(field_name, field_data.get("enum", [])), ...) - if field_type == "array": + elif field_type == "array": items = field_data.get("items", {}) if items != {}: array = {"properties": items} - array_type = convert_dictionary_to_to_pydantic_model(array, f'{model_name}_{field_name}_items') + array_type = convert_dictionary_to_pydantic_model(array, f"{model_name}_{field_name}_items") fields[field_name] = (List[array_type], ...) else: fields[field_name] = (list, ...) - elif field_type == 'object': - submodel = convert_dictionary_to_to_pydantic_model(field_data, f'{model_name}_{field_name}') + elif field_type == "object": + submodel = convert_dictionary_to_pydantic_model(field_data, f"{model_name}_{field_name}") fields[field_name] = (submodel, ...) + elif field_type == "required": + required = field_data.get("enum", []) + for key, field in fields.items(): + if key not in required: + fields[key] = (Optional[fields[key][0]], ...) else: field_type = json_schema_to_python_types(field_type) fields[field_name] = (field_type, ...) if "function" in dictionary: - for field_name, field_data in dictionary.get("function", {}).items(): if field_name == "name": model_name = field_data elif field_name == "description": fields["__doc__"] = field_data elif field_name == "parameters": - return convert_dictionary_to_to_pydantic_model(field_data, f'{model_name}') + return convert_dictionary_to_pydantic_model(field_data, f"{model_name}") + if "parameters" in dictionary: field_data = {"function": dictionary} - return convert_dictionary_to_to_pydantic_model(field_data, f'{model_name}') - + return convert_dictionary_to_pydantic_model(field_data, f"{model_name}") + if "required" in dictionary: + required = dictionary.get("required", []) + for key, field in fields.items(): + if key not in required: + fields[key] = (Optional[fields[key][0]], ...) custom_model = create_model(model_name, **fields) return custom_model - - - From 7c8d3abd1a17c28fc56b1a4814bc4b29f91d7454 Mon Sep 17 00:00:00 2001 From: Alex Azarov Date: Tue, 16 Jan 2024 14:33:02 +0100 Subject: [PATCH 080/812] metal : log `recommendedMaxWorkingSetSize` on iOS 16+ (#4936) * metal: Log `recommendedMaxWorkingSetSize` on iOS 16+ * Only log on iOS and macOS, ignoring tvOS and other platforms * Check for Xcode version before using recommendedMaxWorkingSetSize --------- Co-authored-by: Georgi Gerganov --- ggml-metal.m | 58 ++++++++++++++++++++++++---------------------------- 1 file changed, 27 insertions(+), 31 deletions(-) diff --git a/ggml-metal.m b/ggml-metal.m index 867f2fd48..44134d1d9 100644 --- a/ggml-metal.m +++ b/ggml-metal.m @@ -369,8 +369,12 @@ static struct ggml_metal_context * ggml_metal_init(int n_cb) { GGML_METAL_LOG_INFO("%s: simdgroup reduction support = %s\n", __func__, ctx->support_simdgroup_reduction ? "true" : "false"); GGML_METAL_LOG_INFO("%s: simdgroup matrix mul. support = %s\n", __func__, ctx->support_simdgroup_mm ? "true" : "false"); GGML_METAL_LOG_INFO("%s: hasUnifiedMemory = %s\n", __func__, ctx->device.hasUnifiedMemory ? "true" : "false"); -#if TARGET_OS_OSX - GGML_METAL_LOG_INFO("%s: recommendedMaxWorkingSetSize = %8.2f MB\n", __func__, ctx->device.recommendedMaxWorkingSetSize / 1e6); + +#if TARGET_OS_OSX || (TARGET_OS_IOS && __clang_major__ >= 15) + if (@available(macOS 10.12, iOS 16.0, *)) { + GGML_METAL_LOG_INFO("%s: recommendedMaxWorkingSetSize = %8.2f MB\n", __func__, ctx->device.recommendedMaxWorkingSetSize / 1e6); + } +#elif TARGET_OS_OSX if (ctx->device.maxTransferRate != 0) { GGML_METAL_LOG_INFO("%s: maxTransferRate = %8.2f MB/s\n", __func__, ctx->device.maxTransferRate / 1e6); } else { @@ -2369,6 +2373,25 @@ GGML_CALL static const char * ggml_backend_metal_buffer_type_get_name(ggml_backe UNUSED(buft); } +static void ggml_backend_metal_log_allocated_size(id device) { +#if TARGET_OS_OSX || (TARGET_OS_IOS && __clang_major__ >= 15) + if (@available(macOS 10.12, iOS 16.0, *)) { + GGML_METAL_LOG_INFO(", (%8.2f / %8.2f)", + device.currentAllocatedSize / 1024.0 / 1024.0, + device.recommendedMaxWorkingSetSize / 1024.0 / 1024.0); + + if (device.currentAllocatedSize > device.recommendedMaxWorkingSetSize) { + GGML_METAL_LOG_WARN("%s: warning: current allocated size is greater than the recommended max working set size\n", __func__); + } else { + GGML_METAL_LOG_INFO("\n"); + } + } else { + GGML_METAL_LOG_INFO(", (%8.2f)\n", device.currentAllocatedSize / 1024.0 / 1024.0); + } +#endif + UNUSED(device); +} + GGML_CALL static ggml_backend_buffer_t ggml_backend_metal_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size) { struct ggml_backend_metal_buffer_context * ctx = malloc(sizeof(struct ggml_backend_metal_buffer_context)); @@ -2401,22 +2424,7 @@ GGML_CALL static ggml_backend_buffer_t ggml_backend_metal_buffer_type_alloc_buff } GGML_METAL_LOG_INFO("%s: allocated buffer, size = %8.2f MiB", __func__, size_aligned / 1024.0 / 1024.0); - - -#if TARGET_OS_OSX - GGML_METAL_LOG_INFO(", (%8.2f / %8.2f)", - device.currentAllocatedSize / 1024.0 / 1024.0, - device.recommendedMaxWorkingSetSize / 1024.0 / 1024.0); - - if (device.currentAllocatedSize > device.recommendedMaxWorkingSetSize) { - GGML_METAL_LOG_WARN("%s: warning: current allocated size is greater than the recommended max working set size\n", __func__); - } else { - GGML_METAL_LOG_INFO("\n"); - } -#else - GGML_METAL_LOG_INFO(", (%8.2f)\n", device.currentAllocatedSize / 1024.0 / 1024.0); -#endif - + ggml_backend_metal_log_allocated_size(device); return ggml_backend_buffer_init(buft, ggml_backend_metal_buffer_i, ctx, size); } @@ -2524,19 +2532,7 @@ GGML_CALL ggml_backend_buffer_t ggml_backend_metal_buffer_from_ptr(void * data, } } -#if TARGET_OS_OSX - GGML_METAL_LOG_INFO(", (%8.2f / %8.2f)", - device.currentAllocatedSize / 1024.0 / 1024.0, - device.recommendedMaxWorkingSetSize / 1024.0 / 1024.0); - - if (device.currentAllocatedSize > device.recommendedMaxWorkingSetSize) { - GGML_METAL_LOG_WARN("%s: warning: current allocated size is greater than the recommended max working set size\n", __func__); - } else { - GGML_METAL_LOG_INFO("\n"); - } -#else - GGML_METAL_LOG_INFO(", (%8.2f)\n", device.currentAllocatedSize / 1024.0 / 1024.0); -#endif + ggml_backend_metal_log_allocated_size(device); return ggml_backend_buffer_init(ggml_backend_metal_buffer_type(), ggml_backend_metal_buffer_i, ctx, size); } From 3a48d558a69c88ac17efcaa5900cd9eb19596ac4 Mon Sep 17 00:00:00 2001 From: Alex Azarov Date: Tue, 16 Jan 2024 14:41:27 +0100 Subject: [PATCH 081/812] metal : replace loop of dispatch_async with dispatch_apply (#4934) * Replace loop of dispatch_async with dispatch_apply * Update ggml-metal.m --------- Co-authored-by: Georgi Gerganov --- ggml-metal.m | 2882 +++++++++++++++++++++++++------------------------- 1 file changed, 1439 insertions(+), 1443 deletions(-) diff --git a/ggml-metal.m b/ggml-metal.m index 44134d1d9..c21dc465a 100644 --- a/ggml-metal.m +++ b/ggml-metal.m @@ -737,1475 +737,249 @@ static bool ggml_metal_graph_compute( ctx->command_encoders[i] = [ctx->command_buffers[i] computeCommandEncoderWithDescriptor: edesc]; } - for (int cb_idx = 0; cb_idx < n_cb; ++cb_idx) { - const int n_nodes_per_cb = (n_nodes + n_cb - 1) / n_cb; + const int n_nodes_per_cb = (n_nodes + n_cb - 1) / n_cb; + dispatch_apply(n_cb, ctx->d_queue, ^(size_t iter) { + const int cb_idx = iter; - dispatch_async(ctx->d_queue, ^{ - size_t offs_src0 = 0; - size_t offs_src1 = 0; - size_t offs_dst = 0; + size_t offs_src0 = 0; + size_t offs_src1 = 0; + size_t offs_dst = 0; - id command_buffer = ctx->command_buffers[cb_idx]; - id encoder = ctx->command_encoders[cb_idx]; + id command_buffer = ctx->command_buffers[cb_idx]; + id encoder = ctx->command_encoders[cb_idx]; - const int node_start = (cb_idx + 0) * n_nodes_per_cb; - const int node_end = MIN((cb_idx == n_cb - 1) ? n_nodes : (cb_idx + 1) * n_nodes_per_cb, n_nodes); + const int node_start = (cb_idx + 0) * n_nodes_per_cb; + const int node_end = MIN((cb_idx == n_cb - 1) ? n_nodes : (cb_idx + 1) * n_nodes_per_cb, n_nodes); - for (int ind = node_start; ind < node_end; ++ind) { - const int i = ind; + for (int ind = node_start; ind < node_end; ++ind) { + const int i = ind; - if (i == -1) { - [encoder memoryBarrierWithScope:MTLBarrierScopeBuffers]; - continue; - } + if (i == -1) { + [encoder memoryBarrierWithScope:MTLBarrierScopeBuffers]; + continue; + } - //GGML_METAL_LOG_INFO("%s: encoding node %3d, op = %8s\n", __func__, i, ggml_op_name(gf->nodes[i]->op)); + //GGML_METAL_LOG_INFO("%s: encoding node %3d, op = %8s\n", __func__, i, ggml_op_name(gf->nodes[i]->op)); - struct ggml_tensor * src0 = gf->nodes[i]->src[0]; - struct ggml_tensor * src1 = gf->nodes[i]->src[1]; - struct ggml_tensor * dst = gf->nodes[i]; + struct ggml_tensor * src0 = gf->nodes[i]->src[0]; + struct ggml_tensor * src1 = gf->nodes[i]->src[1]; + struct ggml_tensor * dst = gf->nodes[i]; - switch (dst->op) { - case GGML_OP_NONE: - case GGML_OP_RESHAPE: - case GGML_OP_VIEW: - case GGML_OP_TRANSPOSE: - case GGML_OP_PERMUTE: - { - // noop -> next node - } continue; - default: - { - } break; - } + switch (dst->op) { + case GGML_OP_NONE: + case GGML_OP_RESHAPE: + case GGML_OP_VIEW: + case GGML_OP_TRANSPOSE: + case GGML_OP_PERMUTE: + { + // noop -> next node + } continue; + default: + { + } break; + } - if (!ggml_metal_supports_op(ctx, dst)) { - GGML_METAL_LOG_ERROR("%s: error: unsupported op '%s'\n", __func__, ggml_op_desc(dst)); - GGML_ASSERT(!"unsupported op"); - } + if (!ggml_metal_supports_op(ctx, dst)) { + GGML_METAL_LOG_ERROR("%s: error: unsupported op '%s'\n", __func__, ggml_op_desc(dst)); + GGML_ASSERT(!"unsupported op"); + } #ifndef GGML_METAL_NDEBUG - [encoder pushDebugGroup:[NSString stringWithCString:ggml_op_desc(dst) encoding:NSUTF8StringEncoding]]; + [encoder pushDebugGroup:[NSString stringWithCString:ggml_op_desc(dst) encoding:NSUTF8StringEncoding]]; #endif - const int64_t ne00 = src0 ? src0->ne[0] : 0; - const int64_t ne01 = src0 ? src0->ne[1] : 0; - const int64_t ne02 = src0 ? src0->ne[2] : 0; - const int64_t ne03 = src0 ? src0->ne[3] : 0; + const int64_t ne00 = src0 ? src0->ne[0] : 0; + const int64_t ne01 = src0 ? src0->ne[1] : 0; + const int64_t ne02 = src0 ? src0->ne[2] : 0; + const int64_t ne03 = src0 ? src0->ne[3] : 0; - const uint64_t nb00 = src0 ? src0->nb[0] : 0; - const uint64_t nb01 = src0 ? src0->nb[1] : 0; - const uint64_t nb02 = src0 ? src0->nb[2] : 0; - const uint64_t nb03 = src0 ? src0->nb[3] : 0; + const uint64_t nb00 = src0 ? src0->nb[0] : 0; + const uint64_t nb01 = src0 ? src0->nb[1] : 0; + const uint64_t nb02 = src0 ? src0->nb[2] : 0; + const uint64_t nb03 = src0 ? src0->nb[3] : 0; - const int64_t ne10 = src1 ? src1->ne[0] : 0; - const int64_t ne11 = src1 ? src1->ne[1] : 0; - const int64_t ne12 = src1 ? src1->ne[2] : 0; - const int64_t ne13 = src1 ? src1->ne[3] : 0; UNUSED(ne13); + const int64_t ne10 = src1 ? src1->ne[0] : 0; + const int64_t ne11 = src1 ? src1->ne[1] : 0; + const int64_t ne12 = src1 ? src1->ne[2] : 0; + const int64_t ne13 = src1 ? src1->ne[3] : 0; UNUSED(ne13); - const uint64_t nb10 = src1 ? src1->nb[0] : 0; - const uint64_t nb11 = src1 ? src1->nb[1] : 0; - const uint64_t nb12 = src1 ? src1->nb[2] : 0; - const uint64_t nb13 = src1 ? src1->nb[3] : 0; UNUSED(nb13); + const uint64_t nb10 = src1 ? src1->nb[0] : 0; + const uint64_t nb11 = src1 ? src1->nb[1] : 0; + const uint64_t nb12 = src1 ? src1->nb[2] : 0; + const uint64_t nb13 = src1 ? src1->nb[3] : 0; UNUSED(nb13); - const int64_t ne0 = dst ? dst->ne[0] : 0; - const int64_t ne1 = dst ? dst->ne[1] : 0; - const int64_t ne2 = dst ? dst->ne[2] : 0; - const int64_t ne3 = dst ? dst->ne[3] : 0; + const int64_t ne0 = dst ? dst->ne[0] : 0; + const int64_t ne1 = dst ? dst->ne[1] : 0; + const int64_t ne2 = dst ? dst->ne[2] : 0; + const int64_t ne3 = dst ? dst->ne[3] : 0; - const uint64_t nb0 = dst ? dst->nb[0] : 0; - const uint64_t nb1 = dst ? dst->nb[1] : 0; - const uint64_t nb2 = dst ? dst->nb[2] : 0; - const uint64_t nb3 = dst ? dst->nb[3] : 0; + const uint64_t nb0 = dst ? dst->nb[0] : 0; + const uint64_t nb1 = dst ? dst->nb[1] : 0; + const uint64_t nb2 = dst ? dst->nb[2] : 0; + const uint64_t nb3 = dst ? dst->nb[3] : 0; - const enum ggml_type src0t = src0 ? src0->type : GGML_TYPE_COUNT; - const enum ggml_type src1t = src1 ? src1->type : GGML_TYPE_COUNT; - const enum ggml_type dstt = dst ? dst->type : GGML_TYPE_COUNT; + const enum ggml_type src0t = src0 ? src0->type : GGML_TYPE_COUNT; + const enum ggml_type src1t = src1 ? src1->type : GGML_TYPE_COUNT; + const enum ggml_type dstt = dst ? dst->type : GGML_TYPE_COUNT; - id id_src0 = src0 ? ggml_metal_get_buffer(ctx, src0, &offs_src0) : nil; - id id_src1 = src1 ? ggml_metal_get_buffer(ctx, src1, &offs_src1) : nil; - id id_dst = dst ? ggml_metal_get_buffer(ctx, dst, &offs_dst) : nil; + id id_src0 = src0 ? ggml_metal_get_buffer(ctx, src0, &offs_src0) : nil; + id id_src1 = src1 ? ggml_metal_get_buffer(ctx, src1, &offs_src1) : nil; + id id_dst = dst ? ggml_metal_get_buffer(ctx, dst, &offs_dst) : nil; - //GGML_METAL_LOG_INFO("%s: op - %s\n", __func__, ggml_op_name(dst->op)); - //if (src0) { - // GGML_METAL_LOG_INFO("%s: src0 - %4s [%5lld, %5lld, %5lld], %d, %s\n", __func__, ggml_type_name(src0t), ne00, ne01, ne02, - // ggml_is_contiguous(src0), src0->name); - //} - //if (src1) { - // GGML_METAL_LOG_INFO("%s: src1 - %4s [%5lld, %5lld, %5lld], %d, %s\n", __func__, ggml_type_name(src1t), ne10, ne11, ne12, - // ggml_is_contiguous(src1), src1->name); - //} - //if (dst) { - // GGML_METAL_LOG_INFO("%s: dst - %4s [%5lld, %5lld, %5lld], 1, %s\n", __func__, ggml_type_name(dstt), ne0, ne1, ne2, - // dst->name); - //} + //GGML_METAL_LOG_INFO("%s: op - %s\n", __func__, ggml_op_name(dst->op)); + //if (src0) { + // GGML_METAL_LOG_INFO("%s: src0 - %4s [%5lld, %5lld, %5lld], %d, %s\n", __func__, ggml_type_name(src0t), ne00, ne01, ne02, + // ggml_is_contiguous(src0), src0->name); + //} + //if (src1) { + // GGML_METAL_LOG_INFO("%s: src1 - %4s [%5lld, %5lld, %5lld], %d, %s\n", __func__, ggml_type_name(src1t), ne10, ne11, ne12, + // ggml_is_contiguous(src1), src1->name); + //} + //if (dst) { + // GGML_METAL_LOG_INFO("%s: dst - %4s [%5lld, %5lld, %5lld], 1, %s\n", __func__, ggml_type_name(dstt), ne0, ne1, ne2, + // dst->name); + //} - switch (dst->op) { - case GGML_OP_CONCAT: - { - const int64_t nb = ne00; + switch (dst->op) { + case GGML_OP_CONCAT: + { + const int64_t nb = ne00; - id pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_CONCAT].pipeline; + id pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_CONCAT].pipeline; - [encoder setComputePipelineState:pipeline]; - [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0]; - [encoder setBuffer:id_src1 offset:offs_src1 atIndex:1]; - [encoder setBuffer:id_dst offset:offs_dst atIndex:2]; - [encoder setBytes:&ne00 length:sizeof(ne00) atIndex:3]; - [encoder setBytes:&ne01 length:sizeof(ne01) atIndex:4]; - [encoder setBytes:&ne02 length:sizeof(ne02) atIndex:5]; - [encoder setBytes:&ne03 length:sizeof(ne03) atIndex:6]; - [encoder setBytes:&nb00 length:sizeof(nb00) atIndex:7]; - [encoder setBytes:&nb01 length:sizeof(nb01) atIndex:8]; - [encoder setBytes:&nb02 length:sizeof(nb02) atIndex:9]; - [encoder setBytes:&nb03 length:sizeof(nb03) atIndex:10]; - [encoder setBytes:&ne10 length:sizeof(ne10) atIndex:11]; - [encoder setBytes:&ne11 length:sizeof(ne11) atIndex:12]; - [encoder setBytes:&ne12 length:sizeof(ne12) atIndex:13]; - [encoder setBytes:&ne13 length:sizeof(ne13) atIndex:14]; - [encoder setBytes:&nb10 length:sizeof(nb10) atIndex:15]; - [encoder setBytes:&nb11 length:sizeof(nb11) atIndex:16]; - [encoder setBytes:&nb12 length:sizeof(nb12) atIndex:17]; - [encoder setBytes:&nb13 length:sizeof(nb13) atIndex:18]; - [encoder setBytes:&ne0 length:sizeof(ne0) atIndex:19]; - [encoder setBytes:&ne1 length:sizeof(ne1) atIndex:20]; - [encoder setBytes:&ne2 length:sizeof(ne2) atIndex:21]; - [encoder setBytes:&ne3 length:sizeof(ne3) atIndex:22]; - [encoder setBytes:&nb0 length:sizeof(nb0) atIndex:23]; - [encoder setBytes:&nb1 length:sizeof(nb1) atIndex:24]; - [encoder setBytes:&nb2 length:sizeof(nb2) atIndex:25]; - [encoder setBytes:&nb3 length:sizeof(nb3) atIndex:26]; - [encoder setBytes:&nb length:sizeof(nb) atIndex:27]; + [encoder setComputePipelineState:pipeline]; + [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0]; + [encoder setBuffer:id_src1 offset:offs_src1 atIndex:1]; + [encoder setBuffer:id_dst offset:offs_dst atIndex:2]; + [encoder setBytes:&ne00 length:sizeof(ne00) atIndex:3]; + [encoder setBytes:&ne01 length:sizeof(ne01) atIndex:4]; + [encoder setBytes:&ne02 length:sizeof(ne02) atIndex:5]; + [encoder setBytes:&ne03 length:sizeof(ne03) atIndex:6]; + [encoder setBytes:&nb00 length:sizeof(nb00) atIndex:7]; + [encoder setBytes:&nb01 length:sizeof(nb01) atIndex:8]; + [encoder setBytes:&nb02 length:sizeof(nb02) atIndex:9]; + [encoder setBytes:&nb03 length:sizeof(nb03) atIndex:10]; + [encoder setBytes:&ne10 length:sizeof(ne10) atIndex:11]; + [encoder setBytes:&ne11 length:sizeof(ne11) atIndex:12]; + [encoder setBytes:&ne12 length:sizeof(ne12) atIndex:13]; + [encoder setBytes:&ne13 length:sizeof(ne13) atIndex:14]; + [encoder setBytes:&nb10 length:sizeof(nb10) atIndex:15]; + [encoder setBytes:&nb11 length:sizeof(nb11) atIndex:16]; + [encoder setBytes:&nb12 length:sizeof(nb12) atIndex:17]; + [encoder setBytes:&nb13 length:sizeof(nb13) atIndex:18]; + [encoder setBytes:&ne0 length:sizeof(ne0) atIndex:19]; + [encoder setBytes:&ne1 length:sizeof(ne1) atIndex:20]; + [encoder setBytes:&ne2 length:sizeof(ne2) atIndex:21]; + [encoder setBytes:&ne3 length:sizeof(ne3) atIndex:22]; + [encoder setBytes:&nb0 length:sizeof(nb0) atIndex:23]; + [encoder setBytes:&nb1 length:sizeof(nb1) atIndex:24]; + [encoder setBytes:&nb2 length:sizeof(nb2) atIndex:25]; + [encoder setBytes:&nb3 length:sizeof(nb3) atIndex:26]; + [encoder setBytes:&nb length:sizeof(nb) atIndex:27]; - const int nth = MIN(1024, ne0); + const int nth = MIN(1024, ne0); - [encoder dispatchThreadgroups:MTLSizeMake(ne1, ne2, ne3) threadsPerThreadgroup:MTLSizeMake(nth, 1, 1)]; - } break; - case GGML_OP_ADD: - case GGML_OP_MUL: - case GGML_OP_DIV: - { - const size_t offs = 0; + [encoder dispatchThreadgroups:MTLSizeMake(ne1, ne2, ne3) threadsPerThreadgroup:MTLSizeMake(nth, 1, 1)]; + } break; + case GGML_OP_ADD: + case GGML_OP_MUL: + case GGML_OP_DIV: + { + const size_t offs = 0; - bool bcast_row = false; + bool bcast_row = false; - int64_t nb = ne00; + int64_t nb = ne00; - id pipeline = nil; + id pipeline = nil; - if (ggml_nelements(src1) == ne10 && ggml_is_contiguous(src1) && ne00 % 4 == 0 && ne10 % 4 == 0) { - GGML_ASSERT(ggml_is_contiguous(src0)); - - // src1 is a row - GGML_ASSERT(ne11 == 1); - - nb = ne00 / 4; - switch (dst->op) { - case GGML_OP_ADD: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_ADD_ROW].pipeline; break; - case GGML_OP_MUL: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_ROW].pipeline; break; - case GGML_OP_DIV: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_DIV_ROW].pipeline; break; - default: GGML_ASSERT(false); - } - - bcast_row = true; - } else { - switch (dst->op) { - case GGML_OP_ADD: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_ADD].pipeline; break; - case GGML_OP_MUL: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL].pipeline; break; - case GGML_OP_DIV: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_DIV].pipeline; break; - default: GGML_ASSERT(false); - } - } - - [encoder setComputePipelineState:pipeline]; - [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0]; - [encoder setBuffer:id_src1 offset:offs_src1 atIndex:1]; - [encoder setBuffer:id_dst offset:offs_dst atIndex:2]; - [encoder setBytes:&ne00 length:sizeof(ne00) atIndex:3]; - [encoder setBytes:&ne01 length:sizeof(ne01) atIndex:4]; - [encoder setBytes:&ne02 length:sizeof(ne02) atIndex:5]; - [encoder setBytes:&ne03 length:sizeof(ne03) atIndex:6]; - [encoder setBytes:&nb00 length:sizeof(nb00) atIndex:7]; - [encoder setBytes:&nb01 length:sizeof(nb01) atIndex:8]; - [encoder setBytes:&nb02 length:sizeof(nb02) atIndex:9]; - [encoder setBytes:&nb03 length:sizeof(nb03) atIndex:10]; - [encoder setBytes:&ne10 length:sizeof(ne10) atIndex:11]; - [encoder setBytes:&ne11 length:sizeof(ne11) atIndex:12]; - [encoder setBytes:&ne12 length:sizeof(ne12) atIndex:13]; - [encoder setBytes:&ne13 length:sizeof(ne13) atIndex:14]; - [encoder setBytes:&nb10 length:sizeof(nb10) atIndex:15]; - [encoder setBytes:&nb11 length:sizeof(nb11) atIndex:16]; - [encoder setBytes:&nb12 length:sizeof(nb12) atIndex:17]; - [encoder setBytes:&nb13 length:sizeof(nb13) atIndex:18]; - [encoder setBytes:&ne0 length:sizeof(ne0) atIndex:19]; - [encoder setBytes:&ne1 length:sizeof(ne1) atIndex:20]; - [encoder setBytes:&ne2 length:sizeof(ne2) atIndex:21]; - [encoder setBytes:&ne3 length:sizeof(ne3) atIndex:22]; - [encoder setBytes:&nb0 length:sizeof(nb0) atIndex:23]; - [encoder setBytes:&nb1 length:sizeof(nb1) atIndex:24]; - [encoder setBytes:&nb2 length:sizeof(nb2) atIndex:25]; - [encoder setBytes:&nb3 length:sizeof(nb3) atIndex:26]; - [encoder setBytes:&offs length:sizeof(offs) atIndex:27]; - [encoder setBytes:&nb length:sizeof(nb) atIndex:28]; - - if (bcast_row) { - const int64_t n = ggml_nelements(dst)/4; - - [encoder dispatchThreadgroups:MTLSizeMake(n, 1, 1) threadsPerThreadgroup:MTLSizeMake(1, 1, 1)]; - } else { - const int nth = MIN((int) pipeline.maxTotalThreadsPerThreadgroup, ne0); - - [encoder dispatchThreadgroups:MTLSizeMake(ne01, ne02, ne03) threadsPerThreadgroup:MTLSizeMake(nth, 1, 1)]; - } - } break; - case GGML_OP_ACC: - { - GGML_ASSERT(src0t == GGML_TYPE_F32); - GGML_ASSERT(src1t == GGML_TYPE_F32); - GGML_ASSERT(dstt == GGML_TYPE_F32); - - GGML_ASSERT(ggml_is_contiguous(src0)); - GGML_ASSERT(ggml_is_contiguous(src1)); - - const size_t pnb1 = ((int32_t *) dst->op_params)[0]; - const size_t pnb2 = ((int32_t *) dst->op_params)[1]; - const size_t pnb3 = ((int32_t *) dst->op_params)[2]; - const size_t offs = ((int32_t *) dst->op_params)[3]; - - const bool inplace = (bool) ((int32_t *) dst->op_params)[4]; - - if (!inplace) { - // run a separete kernel to cpy src->dst - // not sure how to avoid this - // TODO: make a simpler cpy_bytes kernel - - const id pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_CPY_F32_F32].pipeline; - - [encoder setComputePipelineState:pipeline]; - [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0]; - [encoder setBuffer:id_dst offset:offs_dst atIndex:1]; - [encoder setBytes:&ne00 length:sizeof( int64_t) atIndex:2]; - [encoder setBytes:&ne01 length:sizeof( int64_t) atIndex:3]; - [encoder setBytes:&ne02 length:sizeof( int64_t) atIndex:4]; - [encoder setBytes:&ne03 length:sizeof( int64_t) atIndex:5]; - [encoder setBytes:&nb00 length:sizeof(uint64_t) atIndex:6]; - [encoder setBytes:&nb01 length:sizeof(uint64_t) atIndex:7]; - [encoder setBytes:&nb02 length:sizeof(uint64_t) atIndex:8]; - [encoder setBytes:&nb03 length:sizeof(uint64_t) atIndex:9]; - [encoder setBytes:&ne0 length:sizeof( int64_t) atIndex:10]; - [encoder setBytes:&ne1 length:sizeof( int64_t) atIndex:11]; - [encoder setBytes:&ne2 length:sizeof( int64_t) atIndex:12]; - [encoder setBytes:&ne3 length:sizeof( int64_t) atIndex:13]; - [encoder setBytes:&nb0 length:sizeof(uint64_t) atIndex:14]; - [encoder setBytes:&nb1 length:sizeof(uint64_t) atIndex:15]; - [encoder setBytes:&nb2 length:sizeof(uint64_t) atIndex:16]; - [encoder setBytes:&nb3 length:sizeof(uint64_t) atIndex:17]; - - const int nth = MIN((int) pipeline.maxTotalThreadsPerThreadgroup, ne00); - - [encoder dispatchThreadgroups:MTLSizeMake(ne01, ne02, ne03) threadsPerThreadgroup:MTLSizeMake(nth, 1, 1)]; - } - - const id pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_ADD].pipeline; - - [encoder setComputePipelineState:pipeline]; - [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0]; - [encoder setBuffer:id_src1 offset:offs_src1 atIndex:1]; - [encoder setBuffer:id_dst offset:offs_dst atIndex:2]; - [encoder setBytes:&ne00 length:sizeof(ne00) atIndex:3]; - [encoder setBytes:&ne01 length:sizeof(ne01) atIndex:4]; - [encoder setBytes:&ne02 length:sizeof(ne02) atIndex:5]; - [encoder setBytes:&ne03 length:sizeof(ne03) atIndex:6]; - [encoder setBytes:&nb00 length:sizeof(nb00) atIndex:7]; - [encoder setBytes:&pnb1 length:sizeof(pnb1) atIndex:8]; - [encoder setBytes:&pnb2 length:sizeof(pnb2) atIndex:9]; - [encoder setBytes:&pnb3 length:sizeof(pnb3) atIndex:10]; - [encoder setBytes:&ne10 length:sizeof(ne10) atIndex:11]; - [encoder setBytes:&ne11 length:sizeof(ne11) atIndex:12]; - [encoder setBytes:&ne12 length:sizeof(ne12) atIndex:13]; - [encoder setBytes:&ne13 length:sizeof(ne13) atIndex:14]; - [encoder setBytes:&nb10 length:sizeof(nb10) atIndex:15]; - [encoder setBytes:&nb11 length:sizeof(nb11) atIndex:16]; - [encoder setBytes:&nb12 length:sizeof(nb12) atIndex:17]; - [encoder setBytes:&nb13 length:sizeof(nb13) atIndex:18]; - [encoder setBytes:&ne0 length:sizeof(ne0) atIndex:19]; - [encoder setBytes:&ne1 length:sizeof(ne1) atIndex:20]; - [encoder setBytes:&ne2 length:sizeof(ne2) atIndex:21]; - [encoder setBytes:&ne3 length:sizeof(ne3) atIndex:22]; - [encoder setBytes:&nb0 length:sizeof(nb0) atIndex:23]; - [encoder setBytes:&pnb1 length:sizeof(pnb1) atIndex:24]; - [encoder setBytes:&pnb2 length:sizeof(pnb2) atIndex:25]; - [encoder setBytes:&pnb3 length:sizeof(pnb3) atIndex:26]; - [encoder setBytes:&offs length:sizeof(offs) atIndex:27]; - - const int nth = MIN((int) pipeline.maxTotalThreadsPerThreadgroup, ne00); - - [encoder dispatchThreadgroups:MTLSizeMake(ne11, ne12, ne13) threadsPerThreadgroup:MTLSizeMake(nth, 1, 1)]; - } break; - case GGML_OP_SCALE: - { + if (ggml_nelements(src1) == ne10 && ggml_is_contiguous(src1) && ne00 % 4 == 0 && ne10 % 4 == 0) { GGML_ASSERT(ggml_is_contiguous(src0)); - const float scale = *(const float *) dst->op_params; + // src1 is a row + GGML_ASSERT(ne11 == 1); - int64_t n = ggml_nelements(dst); - - id pipeline = nil; - - if (n % 4 == 0) { - n /= 4; - pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_SCALE_4].pipeline; - } else { - pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_SCALE].pipeline; + nb = ne00 / 4; + switch (dst->op) { + case GGML_OP_ADD: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_ADD_ROW].pipeline; break; + case GGML_OP_MUL: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_ROW].pipeline; break; + case GGML_OP_DIV: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_DIV_ROW].pipeline; break; + default: GGML_ASSERT(false); } - [encoder setComputePipelineState:pipeline]; - [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0]; - [encoder setBuffer:id_dst offset:offs_dst atIndex:1]; - [encoder setBytes:&scale length:sizeof(scale) atIndex:2]; + bcast_row = true; + } else { + switch (dst->op) { + case GGML_OP_ADD: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_ADD].pipeline; break; + case GGML_OP_MUL: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL].pipeline; break; + case GGML_OP_DIV: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_DIV].pipeline; break; + default: GGML_ASSERT(false); + } + } + + [encoder setComputePipelineState:pipeline]; + [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0]; + [encoder setBuffer:id_src1 offset:offs_src1 atIndex:1]; + [encoder setBuffer:id_dst offset:offs_dst atIndex:2]; + [encoder setBytes:&ne00 length:sizeof(ne00) atIndex:3]; + [encoder setBytes:&ne01 length:sizeof(ne01) atIndex:4]; + [encoder setBytes:&ne02 length:sizeof(ne02) atIndex:5]; + [encoder setBytes:&ne03 length:sizeof(ne03) atIndex:6]; + [encoder setBytes:&nb00 length:sizeof(nb00) atIndex:7]; + [encoder setBytes:&nb01 length:sizeof(nb01) atIndex:8]; + [encoder setBytes:&nb02 length:sizeof(nb02) atIndex:9]; + [encoder setBytes:&nb03 length:sizeof(nb03) atIndex:10]; + [encoder setBytes:&ne10 length:sizeof(ne10) atIndex:11]; + [encoder setBytes:&ne11 length:sizeof(ne11) atIndex:12]; + [encoder setBytes:&ne12 length:sizeof(ne12) atIndex:13]; + [encoder setBytes:&ne13 length:sizeof(ne13) atIndex:14]; + [encoder setBytes:&nb10 length:sizeof(nb10) atIndex:15]; + [encoder setBytes:&nb11 length:sizeof(nb11) atIndex:16]; + [encoder setBytes:&nb12 length:sizeof(nb12) atIndex:17]; + [encoder setBytes:&nb13 length:sizeof(nb13) atIndex:18]; + [encoder setBytes:&ne0 length:sizeof(ne0) atIndex:19]; + [encoder setBytes:&ne1 length:sizeof(ne1) atIndex:20]; + [encoder setBytes:&ne2 length:sizeof(ne2) atIndex:21]; + [encoder setBytes:&ne3 length:sizeof(ne3) atIndex:22]; + [encoder setBytes:&nb0 length:sizeof(nb0) atIndex:23]; + [encoder setBytes:&nb1 length:sizeof(nb1) atIndex:24]; + [encoder setBytes:&nb2 length:sizeof(nb2) atIndex:25]; + [encoder setBytes:&nb3 length:sizeof(nb3) atIndex:26]; + [encoder setBytes:&offs length:sizeof(offs) atIndex:27]; + [encoder setBytes:&nb length:sizeof(nb) atIndex:28]; + + if (bcast_row) { + const int64_t n = ggml_nelements(dst)/4; [encoder dispatchThreadgroups:MTLSizeMake(n, 1, 1) threadsPerThreadgroup:MTLSizeMake(1, 1, 1)]; - } break; - case GGML_OP_UNARY: - switch (ggml_get_unary_op(gf->nodes[i])) { - case GGML_UNARY_OP_TANH: - { - id pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_TANH].pipeline; - - [encoder setComputePipelineState:pipeline]; - [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0]; - [encoder setBuffer:id_dst offset:offs_dst atIndex:1]; - - const int64_t n = ggml_nelements(dst); - - [encoder dispatchThreadgroups:MTLSizeMake(n, 1, 1) threadsPerThreadgroup:MTLSizeMake(1, 1, 1)]; - } break; - case GGML_UNARY_OP_RELU: - { - id pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_RELU].pipeline; - - [encoder setComputePipelineState:pipeline]; - [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0]; - [encoder setBuffer:id_dst offset:offs_dst atIndex:1]; - - const int64_t n = ggml_nelements(dst); - - [encoder dispatchThreadgroups:MTLSizeMake(n, 1, 1) threadsPerThreadgroup:MTLSizeMake(1, 1, 1)]; - } break; - case GGML_UNARY_OP_GELU: - { - id pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GELU].pipeline; - - [encoder setComputePipelineState:pipeline]; - [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0]; - [encoder setBuffer:id_dst offset:offs_dst atIndex:1]; - - const int64_t n = ggml_nelements(dst); - GGML_ASSERT(n % 4 == 0); - - [encoder dispatchThreadgroups:MTLSizeMake(n/4, 1, 1) threadsPerThreadgroup:MTLSizeMake(1, 1, 1)]; - } break; - case GGML_UNARY_OP_GELU_QUICK: - { - id pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GELU_QUICK].pipeline; - - [encoder setComputePipelineState:pipeline]; - [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0]; - [encoder setBuffer:id_dst offset:offs_dst atIndex:1]; - - const int64_t n = ggml_nelements(dst); - GGML_ASSERT(n % 4 == 0); - - [encoder dispatchThreadgroups:MTLSizeMake(n/4, 1, 1) threadsPerThreadgroup:MTLSizeMake(1, 1, 1)]; - } break; - case GGML_UNARY_OP_SILU: - { - id pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_SILU].pipeline; - - [encoder setComputePipelineState:pipeline]; - [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0]; - [encoder setBuffer:id_dst offset:offs_dst atIndex:1]; - - const int64_t n = ggml_nelements(dst); - GGML_ASSERT(n % 4 == 0); - - [encoder dispatchThreadgroups:MTLSizeMake(n/4, 1, 1) threadsPerThreadgroup:MTLSizeMake(1, 1, 1)]; - } break; - default: - { - GGML_METAL_LOG_WARN("%s: node %3d, op = %8s not implemented\n", __func__, i, ggml_op_name(dst->op)); - GGML_ASSERT(false); - } - } break; - case GGML_OP_SQR: - { - GGML_ASSERT(ggml_is_contiguous(src0)); - - id pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_SQR].pipeline; - - [encoder setComputePipelineState:pipeline]; - [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0]; - [encoder setBuffer:id_dst offset:offs_dst atIndex:1]; - - const int64_t n = ggml_nelements(dst); - - [encoder dispatchThreadgroups:MTLSizeMake(n, 1, 1) threadsPerThreadgroup:MTLSizeMake(1, 1, 1)]; - } break; - case GGML_OP_SUM_ROWS: - { - GGML_ASSERT(src0->nb[0] == ggml_type_size(src0->type)); - - id pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_SUM_ROWS].pipeline; - - [encoder setComputePipelineState:pipeline]; - [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0]; - [encoder setBuffer:id_dst offset:offs_dst atIndex:1]; - [encoder setBytes:&ne00 length:sizeof(ne00) atIndex:2]; - [encoder setBytes:&ne01 length:sizeof(ne01) atIndex:3]; - [encoder setBytes:&ne02 length:sizeof(ne02) atIndex:4]; - [encoder setBytes:&ne03 length:sizeof(ne03) atIndex:5]; - [encoder setBytes:&nb00 length:sizeof(nb00) atIndex:6]; - [encoder setBytes:&nb01 length:sizeof(nb01) atIndex:7]; - [encoder setBytes:&nb02 length:sizeof(nb02) atIndex:8]; - [encoder setBytes:&nb03 length:sizeof(nb03) atIndex:9]; - [encoder setBytes:&ne10 length:sizeof(ne10) atIndex:10]; - [encoder setBytes:&ne11 length:sizeof(ne11) atIndex:11]; - [encoder setBytes:&ne12 length:sizeof(ne12) atIndex:12]; - [encoder setBytes:&ne13 length:sizeof(ne13) atIndex:13]; - [encoder setBytes:&nb10 length:sizeof(nb10) atIndex:14]; - [encoder setBytes:&nb11 length:sizeof(nb11) atIndex:15]; - [encoder setBytes:&nb12 length:sizeof(nb12) atIndex:16]; - [encoder setBytes:&nb13 length:sizeof(nb13) atIndex:17]; - [encoder setBytes:&ne0 length:sizeof(ne0) atIndex:18]; - [encoder setBytes:&ne1 length:sizeof(ne1) atIndex:19]; - [encoder setBytes:&ne2 length:sizeof(ne2) atIndex:20]; - [encoder setBytes:&ne3 length:sizeof(ne3) atIndex:21]; - [encoder setBytes:&nb0 length:sizeof(nb0) atIndex:22]; - [encoder setBytes:&nb1 length:sizeof(nb1) atIndex:23]; - [encoder setBytes:&nb2 length:sizeof(nb2) atIndex:24]; - [encoder setBytes:&nb3 length:sizeof(nb3) atIndex:25]; - - [encoder dispatchThreadgroups:MTLSizeMake(ne01, ne02, ne03) threadsPerThreadgroup:MTLSizeMake(1, 1, 1)]; - } break; - case GGML_OP_SOFT_MAX: - { - int nth = 32; // SIMD width - - id pipeline = nil; - - if (ne00%4 == 0) { - while (nth < ne00/4 && nth < 256) { - nth *= 2; - } - pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_SOFT_MAX_4].pipeline; - } else { - while (nth < ne00 && nth < 1024) { - nth *= 2; - } - pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_SOFT_MAX].pipeline; - } - - const float scale = ((float *) dst->op_params)[0]; - - [encoder setComputePipelineState:pipeline]; - [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0]; - if (id_src1) { - [encoder setBuffer:id_src1 offset:offs_src1 atIndex:1]; - } else { - [encoder setBuffer:id_src0 offset:offs_src0 atIndex:1]; - } - [encoder setBuffer:id_dst offset:offs_dst atIndex:2]; - [encoder setBytes:&ne00 length:sizeof(ne00) atIndex:3]; - [encoder setBytes:&ne01 length:sizeof(ne01) atIndex:4]; - [encoder setBytes:&ne02 length:sizeof(ne02) atIndex:5]; - [encoder setBytes:&scale length:sizeof(scale) atIndex:6]; - [encoder setThreadgroupMemoryLength:32*sizeof(float) atIndex:0]; - - [encoder dispatchThreadgroups:MTLSizeMake(ne01*ne02*ne03, 1, 1) threadsPerThreadgroup:MTLSizeMake(nth, 1, 1)]; - } break; - case GGML_OP_DIAG_MASK_INF: - { - const int n_past = ((int32_t *)(dst->op_params))[0]; - - id pipeline = nil; - - if (ne00%8 == 0) { - pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_DIAG_MASK_INF_8].pipeline; - } else { - pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_DIAG_MASK_INF].pipeline; - } - - [encoder setComputePipelineState:pipeline]; - [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0]; - [encoder setBuffer:id_dst offset:offs_dst atIndex:1]; - [encoder setBytes:&ne00 length:sizeof(ne00) atIndex:2]; - [encoder setBytes:&ne01 length:sizeof(ne01) atIndex:3]; - [encoder setBytes:&n_past length:sizeof(int) atIndex:4]; - - if (ne00%8 == 0) { - [encoder dispatchThreadgroups:MTLSizeMake(ne00*ne01*ne02/8, 1, 1) threadsPerThreadgroup:MTLSizeMake(1, 1, 1)]; - } - else { - [encoder dispatchThreadgroups:MTLSizeMake(ne00, ne01, ne02) threadsPerThreadgroup:MTLSizeMake(1, 1, 1)]; - } - } break; - case GGML_OP_MUL_MAT: - { - GGML_ASSERT(ne00 == ne10); - - // TODO: assert that dim2 and dim3 are contiguous - GGML_ASSERT(ne12 % ne02 == 0); - GGML_ASSERT(ne13 % ne03 == 0); - - const uint r2 = ne12/ne02; - const uint r3 = ne13/ne03; - - // find the break-even point where the matrix-matrix kernel becomes more efficient compared - // to the matrix-vector kernel - int ne11_mm_min = 1; - -#if 0 - // the numbers below are measured on M2 Ultra for 7B and 13B models - // these numbers do not translate to other devices or model sizes - // TODO: need to find a better approach - if ([ctx->device.name isEqualToString:@"Apple M2 Ultra"]) { - switch (src0t) { - case GGML_TYPE_F16: ne11_mm_min = 2; break; - case GGML_TYPE_Q8_0: ne11_mm_min = 7; break; - case GGML_TYPE_Q2_K: ne11_mm_min = 15; break; - case GGML_TYPE_Q3_K: ne11_mm_min = 7; break; - case GGML_TYPE_Q4_0: - case GGML_TYPE_Q4_1: ne11_mm_min = 15; break; - case GGML_TYPE_Q4_K: ne11_mm_min = 11; break; - case GGML_TYPE_Q5_0: // not tested yet - case GGML_TYPE_Q5_1: ne11_mm_min = 13; break; // not tested yet - case GGML_TYPE_Q5_K: ne11_mm_min = 7; break; - case GGML_TYPE_Q6_K: ne11_mm_min = 7; break; - default: ne11_mm_min = 1; break; - } - } -#endif - - // for now the matrix-matrix multiplication kernel only works on A14+/M1+ SoCs - // AMD GPU and older A-chips will reuse matrix-vector multiplication kernel - if ([ctx->device supportsFamily:MTLGPUFamilyApple7] && - !ggml_is_transposed(src0) && - !ggml_is_transposed(src1) && - src1t == GGML_TYPE_F32 && - ne00 % 32 == 0 && ne00 >= 64 && - (ne11 > ne11_mm_min || (ggml_is_quantized(src0t) && ne12 > 1))) { - //printf("matrix: ne00 = %6d, ne01 = %6d, ne02 = %6d, ne11 = %6d, ne12 = %6d\n", ne00, ne01, ne02, ne11, ne12); - - id pipeline = nil; - - switch (src0->type) { - case GGML_TYPE_F32: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_F32_F32 ].pipeline; break; - case GGML_TYPE_F16: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_F16_F32 ].pipeline; break; - case GGML_TYPE_Q4_0: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_Q4_0_F32 ].pipeline; break; - case GGML_TYPE_Q4_1: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_Q4_1_F32 ].pipeline; break; - case GGML_TYPE_Q5_0: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_Q5_0_F32 ].pipeline; break; - case GGML_TYPE_Q5_1: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_Q5_1_F32 ].pipeline; break; - case GGML_TYPE_Q8_0: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_Q8_0_F32 ].pipeline; break; - case GGML_TYPE_Q2_K: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_Q2_K_F32 ].pipeline; break; - case GGML_TYPE_Q3_K: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_Q3_K_F32 ].pipeline; break; - case GGML_TYPE_Q4_K: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_Q4_K_F32 ].pipeline; break; - case GGML_TYPE_Q5_K: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_Q5_K_F32 ].pipeline; break; - case GGML_TYPE_Q6_K: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_Q6_K_F32 ].pipeline; break; - case GGML_TYPE_IQ2_XXS: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_IQ2_XXS_F32].pipeline; break; - case GGML_TYPE_IQ2_XS: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_IQ2_XS_F32 ].pipeline; break; - default: GGML_ASSERT(false && "MUL MAT-MAT not implemented"); - } - - [encoder setComputePipelineState:pipeline]; - [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0]; - [encoder setBuffer:id_src1 offset:offs_src1 atIndex:1]; - [encoder setBuffer:id_dst offset:offs_dst atIndex:2]; - [encoder setBytes:&ne00 length:sizeof(ne00) atIndex:3]; - [encoder setBytes:&ne02 length:sizeof(ne02) atIndex:4]; - [encoder setBytes:&nb01 length:sizeof(nb01) atIndex:5]; - [encoder setBytes:&nb02 length:sizeof(nb02) atIndex:6]; - [encoder setBytes:&ne12 length:sizeof(ne12) atIndex:7]; - [encoder setBytes:&nb10 length:sizeof(nb10) atIndex:8]; - [encoder setBytes:&nb11 length:sizeof(nb11) atIndex:9]; - [encoder setBytes:&nb12 length:sizeof(nb12) atIndex:10]; - [encoder setBytes:&ne0 length:sizeof(ne0) atIndex:11]; - [encoder setBytes:&ne1 length:sizeof(ne1) atIndex:12]; - [encoder setBytes:&r2 length:sizeof(r2) atIndex:13]; - [encoder setBytes:&r3 length:sizeof(r3) atIndex:14]; - [encoder setThreadgroupMemoryLength:8192 atIndex:0]; - [encoder dispatchThreadgroups:MTLSizeMake( (ne11 + 31)/32, (ne01 + 63)/64, ne12*ne13) threadsPerThreadgroup:MTLSizeMake(128, 1, 1)]; - } else { - int nth0 = 32; - int nth1 = 1; - int nrows = 1; - //printf("vector: ne00 = %6d, ne01 = %6d, ne02 = %6d, ne11 = %6d, ne12 = %6d\n", ne00, ne01, ne02, ne11, ne12); - - id pipeline = nil; - - // use custom matrix x vector kernel - switch (src0t) { - case GGML_TYPE_F32: - { - GGML_ASSERT(src1t == GGML_TYPE_F32); - pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_F32_F32].pipeline; - nrows = 4; - } break; - case GGML_TYPE_F16: - { - nth0 = 32; - nth1 = 1; - if (src1t == GGML_TYPE_F32) { - if (ne11 * ne12 < 4) { - pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_F16_F32_1ROW].pipeline; - } else if (ne00 >= 128 && ne01 >= 8 && ne00%4 == 0) { - pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_F16_F32_L4].pipeline; - nrows = ne11; - } else { - pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_F16_F32].pipeline; - nrows = 4; - } - } else { - pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_F16_F16].pipeline; - nrows = 4; - } - } break; - case GGML_TYPE_Q4_0: - { - nth0 = 8; - nth1 = 8; - pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_Q4_0_F32].pipeline; - } break; - case GGML_TYPE_Q4_1: - { - nth0 = 8; - nth1 = 8; - pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_Q4_1_F32].pipeline; - } break; - case GGML_TYPE_Q5_0: - { - nth0 = 8; - nth1 = 8; - pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_Q5_0_F32].pipeline; - } break; - case GGML_TYPE_Q5_1: - { - nth0 = 8; - nth1 = 8; - pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_Q5_1_F32].pipeline; - } break; - case GGML_TYPE_Q8_0: - { - nth0 = 8; - nth1 = 8; - pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_Q8_0_F32].pipeline; - } break; - case GGML_TYPE_Q2_K: - { - nth0 = 2; - nth1 = 32; - pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_Q2_K_F32].pipeline; - } break; - case GGML_TYPE_Q3_K: - { - nth0 = 2; - nth1 = 32; - pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_Q3_K_F32].pipeline; - } break; - case GGML_TYPE_Q4_K: - { - nth0 = 4; //1; - nth1 = 8; //32; - pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_Q4_K_F32].pipeline; - } break; - case GGML_TYPE_Q5_K: - { - nth0 = 2; - nth1 = 32; - pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_Q5_K_F32].pipeline; - } break; - case GGML_TYPE_Q6_K: - { - nth0 = 2; - nth1 = 32; - pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_Q6_K_F32].pipeline; - } break; - case GGML_TYPE_IQ2_XXS: - { - nth0 = 4; - nth1 = 16; - pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_IQ2_XXS_F32].pipeline; - } break; - case GGML_TYPE_IQ2_XS: - { - nth0 = 4; - nth1 = 16; - pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_IQ2_XS_F32].pipeline; - } break; - default: - { - GGML_METAL_LOG_ERROR("Asserting on type %d\n", (int)src0t); - GGML_ASSERT(false && "not implemented"); - } - }; - - if (ggml_is_quantized(src0t)) { - GGML_ASSERT(ne00 >= nth0*nth1); - } - - [encoder setComputePipelineState:pipeline]; - [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0]; - [encoder setBuffer:id_src1 offset:offs_src1 atIndex:1]; - [encoder setBuffer:id_dst offset:offs_dst atIndex:2]; - [encoder setBytes:&ne00 length:sizeof(ne00) atIndex:3]; - [encoder setBytes:&ne01 length:sizeof(ne01) atIndex:4]; - [encoder setBytes:&ne02 length:sizeof(ne02) atIndex:5]; - [encoder setBytes:&nb00 length:sizeof(nb00) atIndex:6]; - [encoder setBytes:&nb01 length:sizeof(nb01) atIndex:7]; - [encoder setBytes:&nb02 length:sizeof(nb02) atIndex:8]; - [encoder setBytes:&ne10 length:sizeof(ne10) atIndex:9]; - [encoder setBytes:&ne11 length:sizeof(ne11) atIndex:10]; - [encoder setBytes:&ne12 length:sizeof(ne12) atIndex:11]; - [encoder setBytes:&nb10 length:sizeof(nb10) atIndex:12]; - [encoder setBytes:&nb11 length:sizeof(nb11) atIndex:13]; - [encoder setBytes:&nb12 length:sizeof(nb12) atIndex:14]; - [encoder setBytes:&ne0 length:sizeof(ne0) atIndex:15]; - [encoder setBytes:&ne1 length:sizeof(ne1) atIndex:16]; - [encoder setBytes:&r2 length:sizeof(r2) atIndex:17]; - [encoder setBytes:&r3 length:sizeof(r3) atIndex:18]; - - if (src0t == GGML_TYPE_Q4_0 || src0t == GGML_TYPE_Q4_1 || - src0t == GGML_TYPE_Q5_0 || src0t == GGML_TYPE_Q5_1 || src0t == GGML_TYPE_Q8_0 || - src0t == GGML_TYPE_Q2_K) { // || src0t == GGML_TYPE_Q4_K) { - [encoder dispatchThreadgroups:MTLSizeMake((ne01 + 7)/8, ne11, ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)]; - } - else if (src0t == GGML_TYPE_IQ2_XXS || src0t == GGML_TYPE_IQ2_XS) { - const int mem_size = src0t == GGML_TYPE_IQ2_XXS ? 256*8+128 : 512*8+128; - [encoder setThreadgroupMemoryLength:mem_size atIndex:0]; - [encoder dispatchThreadgroups:MTLSizeMake((ne01 + 7)/8, ne11, ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)]; - } - else if (src0t == GGML_TYPE_Q4_K) { - [encoder dispatchThreadgroups:MTLSizeMake((ne01 + 3)/4, ne11, ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)]; - } - else if (src0t == GGML_TYPE_Q3_K) { -#ifdef GGML_QKK_64 - [encoder dispatchThreadgroups:MTLSizeMake((ne01 + 1)/2, ne11, ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)]; -#else - [encoder dispatchThreadgroups:MTLSizeMake((ne01 + 3)/4, ne11, ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)]; -#endif - } - else if (src0t == GGML_TYPE_Q5_K) { - [encoder dispatchThreadgroups:MTLSizeMake((ne01 + 3)/4, ne11, ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)]; - } - else if (src0t == GGML_TYPE_Q6_K) { - [encoder dispatchThreadgroups:MTLSizeMake((ne01 + 1)/2, ne11, ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)]; - } else { - const int64_t ny = (ne11 + nrows - 1)/nrows; - [encoder dispatchThreadgroups:MTLSizeMake(ne01, ny, ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)]; - } - } - } break; - case GGML_OP_MUL_MAT_ID: - { - //GGML_ASSERT(ne00 == ne10); - //GGML_ASSERT(ne03 == ne13); - - GGML_ASSERT(src0t == GGML_TYPE_I32); - - const int n_as = ((int32_t *) dst->op_params)[1]; - - // TODO: make this more general - GGML_ASSERT(n_as <= 8); - - // max size of the src1ids array in the kernel stack - GGML_ASSERT(ne11 <= 512); - - struct ggml_tensor * src2 = gf->nodes[i]->src[2]; - - const int64_t ne20 = src2 ? src2->ne[0] : 0; - const int64_t ne21 = src2 ? src2->ne[1] : 0; - const int64_t ne22 = src2 ? src2->ne[2] : 0; - const int64_t ne23 = src2 ? src2->ne[3] : 0; GGML_UNUSED(ne23); - - const uint64_t nb20 = src2 ? src2->nb[0] : 0; GGML_UNUSED(nb20); - const uint64_t nb21 = src2 ? src2->nb[1] : 0; - const uint64_t nb22 = src2 ? src2->nb[2] : 0; - const uint64_t nb23 = src2 ? src2->nb[3] : 0; GGML_UNUSED(nb23); - - const enum ggml_type src2t = src2 ? src2->type : GGML_TYPE_COUNT; GGML_UNUSED(src2t); - - GGML_ASSERT(!ggml_is_transposed(src2)); - GGML_ASSERT(!ggml_is_transposed(src1)); - - GGML_ASSERT(src1t == GGML_TYPE_F32); - - const uint r2 = ne12/ne22; - const uint r3 = ne13/ne23; - - // find the break-even point where the matrix-matrix kernel becomes more efficient compared - // to the matrix-vector kernel - int ne11_mm_min = n_as; - - const int idx = ((int32_t *) dst->op_params)[0]; - - // batch size - GGML_ASSERT(ne01 == ne11); - - // for now the matrix-matrix multiplication kernel only works on A14+/M1+ SoCs - // AMD GPU and older A-chips will reuse matrix-vector multiplication kernel - // !!! - // TODO: for now, always use mat-vec kernels until we figure out how to improve the - // indirect matrix multiplication - // !!! - if ([ctx->device supportsFamily:MTLGPUFamilyApple7] && - ne20 % 32 == 0 && ne20 >= 64 && - ne11 > ne11_mm_min) { - - id pipeline = nil; - - switch (src2->type) { - case GGML_TYPE_F32: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_ID_F32_F32 ].pipeline; break; - case GGML_TYPE_F16: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_ID_F16_F32 ].pipeline; break; - case GGML_TYPE_Q4_0: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q4_0_F32 ].pipeline; break; - case GGML_TYPE_Q4_1: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q4_1_F32 ].pipeline; break; - case GGML_TYPE_Q5_0: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q5_0_F32 ].pipeline; break; - case GGML_TYPE_Q5_1: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q5_1_F32 ].pipeline; break; - case GGML_TYPE_Q8_0: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q8_0_F32 ].pipeline; break; - case GGML_TYPE_Q2_K: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q2_K_F32 ].pipeline; break; - case GGML_TYPE_Q3_K: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q3_K_F32 ].pipeline; break; - case GGML_TYPE_Q4_K: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q4_K_F32 ].pipeline; break; - case GGML_TYPE_Q5_K: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q5_K_F32 ].pipeline; break; - case GGML_TYPE_Q6_K: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q6_K_F32 ].pipeline; break; - case GGML_TYPE_IQ2_XXS: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ2_XXS_F32].pipeline; break; - case GGML_TYPE_IQ2_XS: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ2_XS_F32 ].pipeline; break; - default: GGML_ASSERT(false && "MUL_MAT_ID not implemented"); - } - - [encoder setComputePipelineState:pipeline]; - [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0]; - [encoder setBuffer:id_src1 offset:offs_src1 atIndex:1]; - [encoder setBuffer:id_dst offset:offs_dst atIndex:2]; - [encoder setBytes:&nb01 length:sizeof(nb01) atIndex:3]; - [encoder setBytes:&ne20 length:sizeof(ne20) atIndex:4]; - [encoder setBytes:&ne22 length:sizeof(ne22) atIndex:5]; - [encoder setBytes:&nb21 length:sizeof(nb21) atIndex:6]; - [encoder setBytes:&nb22 length:sizeof(nb22) atIndex:7]; - [encoder setBytes:&ne12 length:sizeof(ne12) atIndex:8]; - [encoder setBytes:&ne13 length:sizeof(ne13) atIndex:9]; - [encoder setBytes:&nb10 length:sizeof(nb10) atIndex:10]; - [encoder setBytes:&nb11 length:sizeof(nb11) atIndex:11]; - [encoder setBytes:&nb12 length:sizeof(nb12) atIndex:12]; - [encoder setBytes:&ne0 length:sizeof(ne0) atIndex:13]; - [encoder setBytes:&ne1 length:sizeof(ne1) atIndex:14]; - [encoder setBytes:&nb1 length:sizeof(nb1) atIndex:15]; - [encoder setBytes:&r2 length:sizeof(r2) atIndex:16]; - [encoder setBytes:&r3 length:sizeof(r3) atIndex:17]; - [encoder setBytes:&idx length:sizeof(idx) atIndex:18]; - // TODO: how to make this an array? read Metal docs - for (int j = 0; j < 8; ++j) { - // NOTE: this is done like this to avoid uninitialized kernel arguments when n_as < 8 - struct ggml_tensor * src_cur = dst->src[2 + (j % n_as)]; - - size_t offs_src_cur = 0; - id id_src_cur = ggml_metal_get_buffer(ctx, src_cur, &offs_src_cur); - - [encoder setBuffer:id_src_cur offset:offs_src_cur atIndex:19 + j]; - } - - [encoder setThreadgroupMemoryLength:8192 atIndex:0]; - - [encoder dispatchThreadgroups:MTLSizeMake((ne11 + 31)/32, (ne21 + 63)/64, n_as*ne12*ne13) threadsPerThreadgroup:MTLSizeMake(128, 1, 1)]; - } else { - int nth0 = 32; - int nth1 = 1; - int nrows = 1; - //printf("vector: ne00 = %6d, ne01 = %6d, ne02 = %6d, ne11 = %6d, ne12 = %6d\n", ne00, ne01, ne02, ne11, ne12); - - id pipeline = nil; - - // use custom matrix x vector kernel - switch (src2t) { - case GGML_TYPE_F32: - { - GGML_ASSERT(src1t == GGML_TYPE_F32); - pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_ID_F32_F32].pipeline; - } break; - case GGML_TYPE_F16: - { - GGML_ASSERT(src1t == GGML_TYPE_F32); - nth0 = 32; - nth1 = 1; - pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_ID_F16_F32].pipeline; - } break; - case GGML_TYPE_Q4_0: - { - nth0 = 8; - nth1 = 8; - pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q4_0_F32].pipeline; - } break; - case GGML_TYPE_Q4_1: - { - nth0 = 8; - nth1 = 8; - pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q4_1_F32].pipeline; - } break; - case GGML_TYPE_Q5_0: - { - nth0 = 8; - nth1 = 8; - pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q5_0_F32].pipeline; - } break; - case GGML_TYPE_Q5_1: - { - nth0 = 8; - nth1 = 8; - pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q5_1_F32].pipeline; - } break; - case GGML_TYPE_Q8_0: - { - nth0 = 8; - nth1 = 8; - pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q8_0_F32].pipeline; - } break; - case GGML_TYPE_Q2_K: - { - nth0 = 2; - nth1 = 32; - pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q2_K_F32].pipeline; - } break; - case GGML_TYPE_Q3_K: - { - nth0 = 2; - nth1 = 32; - pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q3_K_F32].pipeline; - } break; - case GGML_TYPE_Q4_K: - { - nth0 = 4; //1; - nth1 = 8; //32; - pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q4_K_F32].pipeline; - } break; - case GGML_TYPE_Q5_K: - { - nth0 = 2; - nth1 = 32; - pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q5_K_F32].pipeline; - } break; - case GGML_TYPE_Q6_K: - { - nth0 = 2; - nth1 = 32; - pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q6_K_F32].pipeline; - } break; - case GGML_TYPE_IQ2_XXS: - { - nth0 = 4; - nth1 = 16; - pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ2_XXS_F32].pipeline; - } break; - case GGML_TYPE_IQ2_XS: - { - nth0 = 4; - nth1 = 16; - pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ2_XS_F32].pipeline; - } break; - default: - { - GGML_METAL_LOG_ERROR("Asserting on type %d\n", (int)src2t); - GGML_ASSERT(false && "not implemented"); - } - }; - - if (ggml_is_quantized(src2t)) { - GGML_ASSERT(ne20 >= nth0*nth1); - } - - const int64_t _ne1 = 1; // kernels needs a reference in constant memory - - [encoder setComputePipelineState:pipeline]; - [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0]; - [encoder setBuffer:id_src1 offset:offs_src1 atIndex:1]; - [encoder setBuffer:id_dst offset:offs_dst atIndex:2]; - [encoder setBytes:&nb01 length:sizeof(nb01) atIndex:3]; - [encoder setBytes:&ne20 length:sizeof(ne20) atIndex:4]; - [encoder setBytes:&ne21 length:sizeof(ne21) atIndex:5]; - [encoder setBytes:&ne22 length:sizeof(ne22) atIndex:6]; - [encoder setBytes:&nb20 length:sizeof(nb20) atIndex:7]; - [encoder setBytes:&nb21 length:sizeof(nb21) atIndex:8]; - [encoder setBytes:&nb22 length:sizeof(nb22) atIndex:9]; - [encoder setBytes:&ne10 length:sizeof(ne10) atIndex:10]; - [encoder setBytes:&_ne1 length:sizeof(_ne1) atIndex:11]; - [encoder setBytes:&ne12 length:sizeof(ne12) atIndex:12]; - [encoder setBytes:&ne13 length:sizeof(ne13) atIndex:13]; - [encoder setBytes:&nb10 length:sizeof(nb10) atIndex:14]; - [encoder setBytes:&nb11 length:sizeof(nb11) atIndex:15]; - [encoder setBytes:&nb12 length:sizeof(nb12) atIndex:16]; - [encoder setBytes:&ne0 length:sizeof(ne0) atIndex:17]; - [encoder setBytes:&_ne1 length:sizeof(_ne1) atIndex:18]; - [encoder setBytes:&nb1 length:sizeof(nb1) atIndex:19]; - [encoder setBytes:&r2 length:sizeof(r2) atIndex:20]; - [encoder setBytes:&r3 length:sizeof(r3) atIndex:21]; - [encoder setBytes:&idx length:sizeof(idx) atIndex:22]; - // TODO: how to make this an array? read Metal docs - for (int j = 0; j < 8; ++j) { - // NOTE: this is done like this to avoid uninitialized kernel arguments when n_as < 8 - struct ggml_tensor * src_cur = dst->src[2 + (j % n_as)]; - - size_t offs_src_cur = 0; - id id_src_cur = ggml_metal_get_buffer(ctx, src_cur, &offs_src_cur); - - [encoder setBuffer:id_src_cur offset:offs_src_cur atIndex:23 + j]; - } - - if (src2t == GGML_TYPE_Q4_0 || src2t == GGML_TYPE_Q4_1 || - src2t == GGML_TYPE_Q5_0 || src2t == GGML_TYPE_Q5_1 || src2t == GGML_TYPE_Q8_0 || - src2t == GGML_TYPE_Q2_K) { // || src2t == GGML_TYPE_Q4_K) { - [encoder dispatchThreadgroups:MTLSizeMake((ne21 + 7)/8, _ne1, ne01*ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)]; - } - else if (src2t == GGML_TYPE_IQ2_XXS || src2t == GGML_TYPE_IQ2_XS) { - const int mem_size = src2t == GGML_TYPE_IQ2_XXS ? 256*8+128 : 512*8+128; - [encoder setThreadgroupMemoryLength:mem_size atIndex:0]; - [encoder dispatchThreadgroups:MTLSizeMake((ne21 + 7)/8, _ne1, ne01*ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)]; - } - else if (src2t == GGML_TYPE_Q4_K) { - [encoder dispatchThreadgroups:MTLSizeMake((ne21 + 3)/4, _ne1, ne01*ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)]; - } - else if (src2t == GGML_TYPE_Q3_K) { -#ifdef GGML_QKK_64 - [encoder dispatchThreadgroups:MTLSizeMake((ne21 + 1)/2, _ne1, ne01*ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)]; -#else - [encoder dispatchThreadgroups:MTLSizeMake((ne21 + 3)/4, _ne1, ne01*ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)]; -#endif - } - else if (src2t == GGML_TYPE_Q5_K) { - [encoder dispatchThreadgroups:MTLSizeMake((ne21 + 3)/4, _ne1, ne01*ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)]; - } - else if (src2t == GGML_TYPE_Q6_K) { - [encoder dispatchThreadgroups:MTLSizeMake((ne21 + 1)/2, _ne1, ne01*ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)]; - } else { - const int64_t ny = (_ne1 + nrows - 1)/nrows; - [encoder dispatchThreadgroups:MTLSizeMake(ne21, ny, ne01*ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)]; - } - } - } break; - case GGML_OP_GET_ROWS: - { - id pipeline = nil; - - switch (src0->type) { - case GGML_TYPE_F32: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GET_ROWS_F32 ].pipeline; break; - case GGML_TYPE_F16: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GET_ROWS_F16 ].pipeline; break; - case GGML_TYPE_Q4_0: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GET_ROWS_Q4_0 ].pipeline; break; - case GGML_TYPE_Q4_1: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GET_ROWS_Q4_1 ].pipeline; break; - case GGML_TYPE_Q5_0: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GET_ROWS_Q5_0 ].pipeline; break; - case GGML_TYPE_Q5_1: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GET_ROWS_Q5_1 ].pipeline; break; - case GGML_TYPE_Q8_0: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GET_ROWS_Q8_0 ].pipeline; break; - case GGML_TYPE_Q2_K: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GET_ROWS_Q2_K ].pipeline; break; - case GGML_TYPE_Q3_K: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GET_ROWS_Q3_K ].pipeline; break; - case GGML_TYPE_Q4_K: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GET_ROWS_Q4_K ].pipeline; break; - case GGML_TYPE_Q5_K: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GET_ROWS_Q5_K ].pipeline; break; - case GGML_TYPE_Q6_K: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GET_ROWS_Q6_K ].pipeline; break; - case GGML_TYPE_IQ2_XXS: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ2_XXS].pipeline; break; - case GGML_TYPE_IQ2_XS: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ2_XS ].pipeline; break; - case GGML_TYPE_I32: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GET_ROWS_I32 ].pipeline; break; - default: GGML_ASSERT(false && "not implemented"); - } - - [encoder setComputePipelineState:pipeline]; - [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0]; - [encoder setBuffer:id_src1 offset:offs_src1 atIndex:1]; - [encoder setBuffer:id_dst offset:offs_dst atIndex:2]; - [encoder setBytes:&ne00 length:sizeof( int64_t) atIndex:3]; - [encoder setBytes:&nb01 length:sizeof(uint64_t) atIndex:4]; - [encoder setBytes:&nb02 length:sizeof(uint64_t) atIndex:5]; - [encoder setBytes:&ne10 length:sizeof( int64_t) atIndex:6]; - [encoder setBytes:&nb10 length:sizeof( int64_t) atIndex:7]; - [encoder setBytes:&nb11 length:sizeof( int64_t) atIndex:8]; - [encoder setBytes:&nb1 length:sizeof(uint64_t) atIndex:9]; - [encoder setBytes:&nb2 length:sizeof(uint64_t) atIndex:10]; - - [encoder dispatchThreadgroups:MTLSizeMake(ne10, ne11, 1) threadsPerThreadgroup:MTLSizeMake(32, 1, 1)]; - } break; - case GGML_OP_RMS_NORM: - { - GGML_ASSERT(ne00 % 4 == 0); - - float eps; - memcpy(&eps, dst->op_params, sizeof(float)); - - int nth = 32; // SIMD width - - while (nth < ne00/4 && nth < 1024) { - nth *= 2; - } - - id pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_RMS_NORM].pipeline; - - [encoder setComputePipelineState:pipeline]; - [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0]; - [encoder setBuffer:id_dst offset:offs_dst atIndex:1]; - [encoder setBytes:&ne00 length:sizeof( int64_t) atIndex:2]; - [encoder setBytes:&nb01 length:sizeof(uint64_t) atIndex:3]; - [encoder setBytes:&eps length:sizeof( float) atIndex:4]; - [encoder setThreadgroupMemoryLength:32*sizeof(float) atIndex:0]; - - const int64_t nrows = ggml_nrows(src0); - - [encoder dispatchThreadgroups:MTLSizeMake(nrows, 1, 1) threadsPerThreadgroup:MTLSizeMake(nth, 1, 1)]; - } break; - case GGML_OP_GROUP_NORM: - { - GGML_ASSERT(ne00 % 4 == 0); - - //float eps; - //memcpy(&eps, dst->op_params, sizeof(float)); - - const float eps = 1e-6f; // TODO: temporarily hardcoded - - const int32_t n_groups = ((int32_t *) dst->op_params)[0]; - - int nth = 32; // SIMD width - - //while (nth < ne00/4 && nth < 1024) { - // nth *= 2; - //} - - id pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GROUP_NORM].pipeline; - - [encoder setComputePipelineState:pipeline]; - [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0]; - [encoder setBuffer:id_dst offset:offs_dst atIndex:1]; - [encoder setBytes:&ne00 length:sizeof( int64_t) atIndex:2]; - [encoder setBytes:&ne01 length:sizeof( int64_t) atIndex:3]; - [encoder setBytes:&ne02 length:sizeof( int64_t) atIndex:4]; - [encoder setBytes:&nb00 length:sizeof(uint64_t) atIndex:5]; - [encoder setBytes:&nb01 length:sizeof(uint64_t) atIndex:6]; - [encoder setBytes:&nb02 length:sizeof(uint64_t) atIndex:7]; - [encoder setBytes:&n_groups length:sizeof( int32_t) atIndex:8]; - [encoder setBytes:&eps length:sizeof( float) atIndex:9]; - [encoder setThreadgroupMemoryLength:32*sizeof(float) atIndex:0]; - - [encoder dispatchThreadgroups:MTLSizeMake(n_groups, 1, 1) threadsPerThreadgroup:MTLSizeMake(nth, 1, 1)]; - } break; - case GGML_OP_NORM: - { - float eps; - memcpy(&eps, dst->op_params, sizeof(float)); - - const int nth = MIN(256, ne00); - - id pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_NORM].pipeline; - - [encoder setComputePipelineState:pipeline]; - [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0]; - [encoder setBuffer:id_dst offset:offs_dst atIndex:1]; - [encoder setBytes:&ne00 length:sizeof( int64_t) atIndex:2]; - [encoder setBytes:&nb01 length:sizeof(uint64_t) atIndex:3]; - [encoder setBytes:&eps length:sizeof( float) atIndex:4]; - [encoder setThreadgroupMemoryLength:GGML_PAD(nth*sizeof(float), 16) atIndex:0]; - - const int64_t nrows = ggml_nrows(src0); - - [encoder dispatchThreadgroups:MTLSizeMake(nrows, 1, 1) threadsPerThreadgroup:MTLSizeMake(nth, 1, 1)]; - } break; - case GGML_OP_ALIBI: - { - GGML_ASSERT((src0t == GGML_TYPE_F32)); - - const int nth = MIN(1024, ne00); - - //const int n_past = ((int32_t *) dst->op_params)[0]; - const int n_head = ((int32_t *) dst->op_params)[1]; - float max_bias; - memcpy(&max_bias, (int32_t *) dst->op_params + 2, sizeof(float)); - - const int n_heads_log2_floor = 1 << (int) floor(log2(n_head)); - const float m0 = powf(2.0f, -(max_bias) / n_heads_log2_floor); - const float m1 = powf(2.0f, -(max_bias / 2.0f) / n_heads_log2_floor); - - id pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_ALIBI_F32].pipeline; - - [encoder setComputePipelineState:pipeline]; - [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0]; - [encoder setBuffer:id_dst offset:offs_dst atIndex:1]; - [encoder setBytes:&ne00 length:sizeof( int64_t) atIndex:2]; - [encoder setBytes:&ne01 length:sizeof( int64_t) atIndex:3]; - [encoder setBytes:&ne02 length:sizeof( int64_t) atIndex:4]; - [encoder setBytes:&ne03 length:sizeof( int64_t) atIndex:5]; - [encoder setBytes:&nb00 length:sizeof(uint64_t) atIndex:6]; - [encoder setBytes:&nb01 length:sizeof(uint64_t) atIndex:7]; - [encoder setBytes:&nb02 length:sizeof(uint64_t) atIndex:8]; - [encoder setBytes:&nb03 length:sizeof(uint64_t) atIndex:9]; - [encoder setBytes:&ne0 length:sizeof( int64_t) atIndex:10]; - [encoder setBytes:&ne1 length:sizeof( int64_t) atIndex:11]; - [encoder setBytes:&ne2 length:sizeof( int64_t) atIndex:12]; - [encoder setBytes:&ne3 length:sizeof( int64_t) atIndex:13]; - [encoder setBytes:&nb0 length:sizeof(uint64_t) atIndex:14]; - [encoder setBytes:&nb1 length:sizeof(uint64_t) atIndex:15]; - [encoder setBytes:&nb2 length:sizeof(uint64_t) atIndex:16]; - [encoder setBytes:&nb3 length:sizeof(uint64_t) atIndex:17]; - [encoder setBytes:&m0 length:sizeof( float) atIndex:18]; - [encoder setBytes:&m1 length:sizeof( float) atIndex:19]; - [encoder setBytes:&n_heads_log2_floor length:sizeof(int) atIndex:20]; - - [encoder dispatchThreadgroups:MTLSizeMake(ne01, ne02, ne03) threadsPerThreadgroup:MTLSizeMake(nth, 1, 1)]; - } break; - case GGML_OP_ROPE: - { - GGML_ASSERT(ne10 == ne02); - - const int nth = MIN(1024, ne00); - - const int n_past = ((int32_t *) dst->op_params)[0]; - const int n_dims = ((int32_t *) dst->op_params)[1]; - const int mode = ((int32_t *) dst->op_params)[2]; - // skip 3, n_ctx, used in GLM RoPE, unimplemented in metal - const int n_orig_ctx = ((int32_t *) dst->op_params)[4]; - - float freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow; - memcpy(&freq_base, (int32_t *) dst->op_params + 5, sizeof(float)); - memcpy(&freq_scale, (int32_t *) dst->op_params + 6, sizeof(float)); - memcpy(&ext_factor, (int32_t *) dst->op_params + 7, sizeof(float)); - memcpy(&attn_factor, (int32_t *) dst->op_params + 8, sizeof(float)); - memcpy(&beta_fast, (int32_t *) dst->op_params + 9, sizeof(float)); - memcpy(&beta_slow, (int32_t *) dst->op_params + 10, sizeof(float)); - - id pipeline = nil; - - switch (src0->type) { - case GGML_TYPE_F32: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_ROPE_F32].pipeline; break; - case GGML_TYPE_F16: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_ROPE_F16].pipeline; break; - default: GGML_ASSERT(false); - }; - - [encoder setComputePipelineState:pipeline]; - [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0]; - [encoder setBuffer:id_src1 offset:offs_src1 atIndex:1]; - [encoder setBuffer:id_dst offset:offs_dst atIndex:2]; - [encoder setBytes:&ne00 length:sizeof( int64_t) atIndex:3]; - [encoder setBytes:&ne01 length:sizeof( int64_t) atIndex:4]; - [encoder setBytes:&ne02 length:sizeof( int64_t) atIndex:5]; - [encoder setBytes:&ne03 length:sizeof( int64_t) atIndex:6]; - [encoder setBytes:&nb00 length:sizeof(uint64_t) atIndex:7]; - [encoder setBytes:&nb01 length:sizeof(uint64_t) atIndex:8]; - [encoder setBytes:&nb02 length:sizeof(uint64_t) atIndex:9]; - [encoder setBytes:&nb03 length:sizeof(uint64_t) atIndex:10]; - [encoder setBytes:&ne0 length:sizeof( int64_t) atIndex:11]; - [encoder setBytes:&ne1 length:sizeof( int64_t) atIndex:12]; - [encoder setBytes:&ne2 length:sizeof( int64_t) atIndex:13]; - [encoder setBytes:&ne3 length:sizeof( int64_t) atIndex:14]; - [encoder setBytes:&nb0 length:sizeof(uint64_t) atIndex:15]; - [encoder setBytes:&nb1 length:sizeof(uint64_t) atIndex:16]; - [encoder setBytes:&nb2 length:sizeof(uint64_t) atIndex:17]; - [encoder setBytes:&nb3 length:sizeof(uint64_t) atIndex:18]; - [encoder setBytes:&n_past length:sizeof( int) atIndex:19]; - [encoder setBytes:&n_dims length:sizeof( int) atIndex:20]; - [encoder setBytes:&mode length:sizeof( int) atIndex:21]; - [encoder setBytes:&n_orig_ctx length:sizeof( int) atIndex:22]; - [encoder setBytes:&freq_base length:sizeof( float) atIndex:23]; - [encoder setBytes:&freq_scale length:sizeof( float) atIndex:24]; - [encoder setBytes:&ext_factor length:sizeof( float) atIndex:25]; - [encoder setBytes:&attn_factor length:sizeof( float) atIndex:26]; - [encoder setBytes:&beta_fast length:sizeof( float) atIndex:27]; - [encoder setBytes:&beta_slow length:sizeof( float) atIndex:28]; - - [encoder dispatchThreadgroups:MTLSizeMake(ne01, ne02, ne03) threadsPerThreadgroup:MTLSizeMake(nth, 1, 1)]; - } break; - case GGML_OP_IM2COL: - { - GGML_ASSERT(src0->type == GGML_TYPE_F16); - GGML_ASSERT(src1->type == GGML_TYPE_F32); - GGML_ASSERT( dst->type == GGML_TYPE_F16); - - const int32_t s0 = ((const int32_t *)(dst->op_params))[0]; - const int32_t s1 = ((const int32_t *)(dst->op_params))[1]; - const int32_t p0 = ((const int32_t *)(dst->op_params))[2]; - const int32_t p1 = ((const int32_t *)(dst->op_params))[3]; - const int32_t d0 = ((const int32_t *)(dst->op_params))[4]; - const int32_t d1 = ((const int32_t *)(dst->op_params))[5]; - const bool is_2D = ((const int32_t *)(dst->op_params))[6] == 1; - - const int32_t N = src1->ne[is_2D ? 3 : 2]; - const int32_t IC = src1->ne[is_2D ? 2 : 1]; - const int32_t IH = is_2D ? src1->ne[1] : 1; - const int32_t IW = src1->ne[0]; - - const int32_t KH = is_2D ? src0->ne[1] : 1; - const int32_t KW = src0->ne[0]; - - const int32_t OH = is_2D ? dst->ne[2] : 1; - const int32_t OW = dst->ne[1]; - - const int32_t CHW = IC * KH * KW; - - const int32_t ofs0 = src1->nb[is_2D ? 3 : 2] / 4; - const int32_t ofs1 = src1->nb[is_2D ? 2 : 1] / 4; - - id pipeline = nil; - - switch (src0->type) { - case GGML_TYPE_F32: GGML_ASSERT(false && "not implemented"); break; - case GGML_TYPE_F16: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_IM2COL_F16].pipeline; break; - default: GGML_ASSERT(false); - }; - - [encoder setComputePipelineState:pipeline]; - [encoder setBuffer:id_src1 offset:offs_src1 atIndex:0]; - [encoder setBuffer:id_dst offset:offs_dst atIndex:1]; - [encoder setBytes:&ofs0 length:sizeof( int32_t) atIndex:2]; - [encoder setBytes:&ofs1 length:sizeof( int32_t) atIndex:3]; - [encoder setBytes:&IW length:sizeof( int32_t) atIndex:4]; - [encoder setBytes:&IH length:sizeof( int32_t) atIndex:5]; - [encoder setBytes:&CHW length:sizeof( int32_t) atIndex:6]; - [encoder setBytes:&s0 length:sizeof( int32_t) atIndex:7]; - [encoder setBytes:&s1 length:sizeof( int32_t) atIndex:8]; - [encoder setBytes:&p0 length:sizeof( int32_t) atIndex:9]; - [encoder setBytes:&p1 length:sizeof( int32_t) atIndex:10]; - [encoder setBytes:&d0 length:sizeof( int32_t) atIndex:11]; - [encoder setBytes:&d1 length:sizeof( int32_t) atIndex:12]; - - [encoder dispatchThreadgroups:MTLSizeMake(IC, OH, OW) threadsPerThreadgroup:MTLSizeMake(N, KH, KW)]; - } break; - case GGML_OP_UPSCALE: - { - GGML_ASSERT(src0->type == GGML_TYPE_F32); - - const int sf = dst->op_params[0]; - - const id pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_UPSCALE_F32].pipeline; - - [encoder setComputePipelineState:pipeline]; - [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0]; - [encoder setBuffer:id_dst offset:offs_dst atIndex:1]; - [encoder setBytes:&ne00 length:sizeof(ne00) atIndex:2]; - [encoder setBytes:&ne01 length:sizeof(ne01) atIndex:3]; - [encoder setBytes:&ne02 length:sizeof(ne02) atIndex:4]; - [encoder setBytes:&ne03 length:sizeof(ne03) atIndex:5]; - [encoder setBytes:&nb00 length:sizeof(nb00) atIndex:6]; - [encoder setBytes:&nb01 length:sizeof(nb01) atIndex:7]; - [encoder setBytes:&nb02 length:sizeof(nb02) atIndex:8]; - [encoder setBytes:&nb03 length:sizeof(nb03) atIndex:9]; - [encoder setBytes:&ne0 length:sizeof(ne0) atIndex:10]; - [encoder setBytes:&ne1 length:sizeof(ne1) atIndex:11]; - [encoder setBytes:&ne2 length:sizeof(ne2) atIndex:12]; - [encoder setBytes:&ne3 length:sizeof(ne3) atIndex:13]; - [encoder setBytes:&nb0 length:sizeof(nb0) atIndex:14]; - [encoder setBytes:&nb1 length:sizeof(nb1) atIndex:15]; - [encoder setBytes:&nb2 length:sizeof(nb2) atIndex:16]; - [encoder setBytes:&nb3 length:sizeof(nb3) atIndex:17]; - [encoder setBytes:&sf length:sizeof(sf) atIndex:18]; - + } else { const int nth = MIN((int) pipeline.maxTotalThreadsPerThreadgroup, ne0); - [encoder dispatchThreadgroups:MTLSizeMake(ne1, ne2, ne3) threadsPerThreadgroup:MTLSizeMake(nth, 1, 1)]; - } break; - case GGML_OP_PAD: - { - GGML_ASSERT(src0->type == GGML_TYPE_F32); + [encoder dispatchThreadgroups:MTLSizeMake(ne01, ne02, ne03) threadsPerThreadgroup:MTLSizeMake(nth, 1, 1)]; + } + } break; + case GGML_OP_ACC: + { + GGML_ASSERT(src0t == GGML_TYPE_F32); + GGML_ASSERT(src1t == GGML_TYPE_F32); + GGML_ASSERT(dstt == GGML_TYPE_F32); - id pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_PAD_F32].pipeline; + GGML_ASSERT(ggml_is_contiguous(src0)); + GGML_ASSERT(ggml_is_contiguous(src1)); - [encoder setComputePipelineState:pipeline]; - [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0]; - [encoder setBuffer:id_dst offset:offs_dst atIndex:1]; - [encoder setBytes:&ne00 length:sizeof(ne00) atIndex:2]; - [encoder setBytes:&ne01 length:sizeof(ne01) atIndex:3]; - [encoder setBytes:&ne02 length:sizeof(ne02) atIndex:4]; - [encoder setBytes:&ne03 length:sizeof(ne03) atIndex:5]; - [encoder setBytes:&nb00 length:sizeof(nb00) atIndex:6]; - [encoder setBytes:&nb01 length:sizeof(nb01) atIndex:7]; - [encoder setBytes:&nb02 length:sizeof(nb02) atIndex:8]; - [encoder setBytes:&nb03 length:sizeof(nb03) atIndex:9]; - [encoder setBytes:&ne0 length:sizeof(ne0) atIndex:10]; - [encoder setBytes:&ne1 length:sizeof(ne1) atIndex:11]; - [encoder setBytes:&ne2 length:sizeof(ne2) atIndex:12]; - [encoder setBytes:&ne3 length:sizeof(ne3) atIndex:13]; - [encoder setBytes:&nb0 length:sizeof(nb0) atIndex:14]; - [encoder setBytes:&nb1 length:sizeof(nb1) atIndex:15]; - [encoder setBytes:&nb2 length:sizeof(nb2) atIndex:16]; - [encoder setBytes:&nb3 length:sizeof(nb3) atIndex:17]; + const size_t pnb1 = ((int32_t *) dst->op_params)[0]; + const size_t pnb2 = ((int32_t *) dst->op_params)[1]; + const size_t pnb3 = ((int32_t *) dst->op_params)[2]; + const size_t offs = ((int32_t *) dst->op_params)[3]; - const int nth = MIN(1024, ne0); + const bool inplace = (bool) ((int32_t *) dst->op_params)[4]; - [encoder dispatchThreadgroups:MTLSizeMake(ne1, ne2, ne3) threadsPerThreadgroup:MTLSizeMake(nth, 1, 1)]; - } break; - case GGML_OP_ARGSORT: - { - GGML_ASSERT(src0->type == GGML_TYPE_F32); - GGML_ASSERT( dst->type == GGML_TYPE_I32); + if (!inplace) { + // run a separete kernel to cpy src->dst + // not sure how to avoid this + // TODO: make a simpler cpy_bytes kernel - const int nrows = ggml_nrows(src0); - - enum ggml_sort_order order = (enum ggml_sort_order) dst->op_params[0]; - - id pipeline = nil; - - switch (order) { - case GGML_SORT_ASC: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_ARGSORT_F32_I32_ASC].pipeline; break; - case GGML_SORT_DESC: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_ARGSORT_F32_I32_DESC].pipeline; break; - default: GGML_ASSERT(false); - }; - - [encoder setComputePipelineState:pipeline]; - [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0]; - [encoder setBuffer:id_dst offset:offs_dst atIndex:1]; - [encoder setBytes:&ne00 length:sizeof( int64_t) atIndex:2]; - - [encoder dispatchThreadgroups:MTLSizeMake(1, nrows, 1) threadsPerThreadgroup:MTLSizeMake(ne00, 1, 1)]; - } break; - case GGML_OP_LEAKY_RELU: - { - GGML_ASSERT(src0->type == GGML_TYPE_F32); - - float slope; - memcpy(&slope, dst->op_params, sizeof(float)); - - id pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_LEAKY_RELU_F32].pipeline; - - [encoder setComputePipelineState:pipeline]; - [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0]; - [encoder setBuffer:id_dst offset:offs_dst atIndex:1]; - [encoder setBytes:&slope length:sizeof(slope) atIndex:2]; - - const int64_t n = ggml_nelements(dst); - - [encoder dispatchThreadgroups:MTLSizeMake(n, 1, 1) threadsPerThreadgroup:MTLSizeMake(1, 1, 1)]; - } break; - case GGML_OP_DUP: - case GGML_OP_CPY: - case GGML_OP_CONT: - { - GGML_ASSERT(ne00 % ggml_blck_size(src0->type) == 0); - - int nth = MIN(1024, ne00/ggml_blck_size(src0->type)); - - id pipeline = nil; - - switch (src0t) { - case GGML_TYPE_F32: - { - GGML_ASSERT(ne0 % ggml_blck_size(dst->type) == 0); - - switch (dstt) { - case GGML_TYPE_F16: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_CPY_F32_F16].pipeline; break; - case GGML_TYPE_F32: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_CPY_F32_F32].pipeline; break; - case GGML_TYPE_Q8_0: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_CPY_F32_Q8_0].pipeline; break; - case GGML_TYPE_Q4_0: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_CPY_F32_Q4_0].pipeline; break; - case GGML_TYPE_Q4_1: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_CPY_F32_Q4_1].pipeline; break; - //case GGML_TYPE_Q5_0: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_CPY_F32_Q5_0].pipeline; break; - //case GGML_TYPE_Q5_1: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_CPY_F32_Q5_1].pipeline; break; - default: GGML_ASSERT(false && "not implemented"); - }; - } break; - case GGML_TYPE_F16: - { - switch (dstt) { - case GGML_TYPE_F16: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_CPY_F16_F16].pipeline; break; - case GGML_TYPE_F32: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_CPY_F16_F32].pipeline; break; - default: GGML_ASSERT(false && "not implemented"); - }; - } break; - default: GGML_ASSERT(false && "not implemented"); - } + const id pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_CPY_F32_F32].pipeline; [encoder setComputePipelineState:pipeline]; [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0]; @@ -2227,31 +1001,1253 @@ static bool ggml_metal_graph_compute( [encoder setBytes:&nb2 length:sizeof(uint64_t) atIndex:16]; [encoder setBytes:&nb3 length:sizeof(uint64_t) atIndex:17]; + const int nth = MIN((int) pipeline.maxTotalThreadsPerThreadgroup, ne00); + [encoder dispatchThreadgroups:MTLSizeMake(ne01, ne02, ne03) threadsPerThreadgroup:MTLSizeMake(nth, 1, 1)]; - } break; - default: - { - GGML_METAL_LOG_ERROR("%s: error: node %3d, op = %8s not implemented\n", __func__, i, ggml_op_name(dst->op)); - GGML_ASSERT(false); } - } + + const id pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_ADD].pipeline; + + [encoder setComputePipelineState:pipeline]; + [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0]; + [encoder setBuffer:id_src1 offset:offs_src1 atIndex:1]; + [encoder setBuffer:id_dst offset:offs_dst atIndex:2]; + [encoder setBytes:&ne00 length:sizeof(ne00) atIndex:3]; + [encoder setBytes:&ne01 length:sizeof(ne01) atIndex:4]; + [encoder setBytes:&ne02 length:sizeof(ne02) atIndex:5]; + [encoder setBytes:&ne03 length:sizeof(ne03) atIndex:6]; + [encoder setBytes:&nb00 length:sizeof(nb00) atIndex:7]; + [encoder setBytes:&pnb1 length:sizeof(pnb1) atIndex:8]; + [encoder setBytes:&pnb2 length:sizeof(pnb2) atIndex:9]; + [encoder setBytes:&pnb3 length:sizeof(pnb3) atIndex:10]; + [encoder setBytes:&ne10 length:sizeof(ne10) atIndex:11]; + [encoder setBytes:&ne11 length:sizeof(ne11) atIndex:12]; + [encoder setBytes:&ne12 length:sizeof(ne12) atIndex:13]; + [encoder setBytes:&ne13 length:sizeof(ne13) atIndex:14]; + [encoder setBytes:&nb10 length:sizeof(nb10) atIndex:15]; + [encoder setBytes:&nb11 length:sizeof(nb11) atIndex:16]; + [encoder setBytes:&nb12 length:sizeof(nb12) atIndex:17]; + [encoder setBytes:&nb13 length:sizeof(nb13) atIndex:18]; + [encoder setBytes:&ne0 length:sizeof(ne0) atIndex:19]; + [encoder setBytes:&ne1 length:sizeof(ne1) atIndex:20]; + [encoder setBytes:&ne2 length:sizeof(ne2) atIndex:21]; + [encoder setBytes:&ne3 length:sizeof(ne3) atIndex:22]; + [encoder setBytes:&nb0 length:sizeof(nb0) atIndex:23]; + [encoder setBytes:&pnb1 length:sizeof(pnb1) atIndex:24]; + [encoder setBytes:&pnb2 length:sizeof(pnb2) atIndex:25]; + [encoder setBytes:&pnb3 length:sizeof(pnb3) atIndex:26]; + [encoder setBytes:&offs length:sizeof(offs) atIndex:27]; + + const int nth = MIN((int) pipeline.maxTotalThreadsPerThreadgroup, ne00); + + [encoder dispatchThreadgroups:MTLSizeMake(ne11, ne12, ne13) threadsPerThreadgroup:MTLSizeMake(nth, 1, 1)]; + } break; + case GGML_OP_SCALE: + { + GGML_ASSERT(ggml_is_contiguous(src0)); + + const float scale = *(const float *) dst->op_params; + + int64_t n = ggml_nelements(dst); + + id pipeline = nil; + + if (n % 4 == 0) { + n /= 4; + pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_SCALE_4].pipeline; + } else { + pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_SCALE].pipeline; + } + + [encoder setComputePipelineState:pipeline]; + [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0]; + [encoder setBuffer:id_dst offset:offs_dst atIndex:1]; + [encoder setBytes:&scale length:sizeof(scale) atIndex:2]; + + [encoder dispatchThreadgroups:MTLSizeMake(n, 1, 1) threadsPerThreadgroup:MTLSizeMake(1, 1, 1)]; + } break; + case GGML_OP_UNARY: + switch (ggml_get_unary_op(gf->nodes[i])) { + case GGML_UNARY_OP_TANH: + { + id pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_TANH].pipeline; + + [encoder setComputePipelineState:pipeline]; + [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0]; + [encoder setBuffer:id_dst offset:offs_dst atIndex:1]; + + const int64_t n = ggml_nelements(dst); + + [encoder dispatchThreadgroups:MTLSizeMake(n, 1, 1) threadsPerThreadgroup:MTLSizeMake(1, 1, 1)]; + } break; + case GGML_UNARY_OP_RELU: + { + id pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_RELU].pipeline; + + [encoder setComputePipelineState:pipeline]; + [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0]; + [encoder setBuffer:id_dst offset:offs_dst atIndex:1]; + + const int64_t n = ggml_nelements(dst); + + [encoder dispatchThreadgroups:MTLSizeMake(n, 1, 1) threadsPerThreadgroup:MTLSizeMake(1, 1, 1)]; + } break; + case GGML_UNARY_OP_GELU: + { + id pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GELU].pipeline; + + [encoder setComputePipelineState:pipeline]; + [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0]; + [encoder setBuffer:id_dst offset:offs_dst atIndex:1]; + + const int64_t n = ggml_nelements(dst); + GGML_ASSERT(n % 4 == 0); + + [encoder dispatchThreadgroups:MTLSizeMake(n/4, 1, 1) threadsPerThreadgroup:MTLSizeMake(1, 1, 1)]; + } break; + case GGML_UNARY_OP_GELU_QUICK: + { + id pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GELU_QUICK].pipeline; + + [encoder setComputePipelineState:pipeline]; + [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0]; + [encoder setBuffer:id_dst offset:offs_dst atIndex:1]; + + const int64_t n = ggml_nelements(dst); + GGML_ASSERT(n % 4 == 0); + + [encoder dispatchThreadgroups:MTLSizeMake(n/4, 1, 1) threadsPerThreadgroup:MTLSizeMake(1, 1, 1)]; + } break; + case GGML_UNARY_OP_SILU: + { + id pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_SILU].pipeline; + + [encoder setComputePipelineState:pipeline]; + [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0]; + [encoder setBuffer:id_dst offset:offs_dst atIndex:1]; + + const int64_t n = ggml_nelements(dst); + GGML_ASSERT(n % 4 == 0); + + [encoder dispatchThreadgroups:MTLSizeMake(n/4, 1, 1) threadsPerThreadgroup:MTLSizeMake(1, 1, 1)]; + } break; + default: + { + GGML_METAL_LOG_WARN("%s: node %3d, op = %8s not implemented\n", __func__, i, ggml_op_name(dst->op)); + GGML_ASSERT(false); + } + } break; + case GGML_OP_SQR: + { + GGML_ASSERT(ggml_is_contiguous(src0)); + + id pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_SQR].pipeline; + + [encoder setComputePipelineState:pipeline]; + [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0]; + [encoder setBuffer:id_dst offset:offs_dst atIndex:1]; + + const int64_t n = ggml_nelements(dst); + + [encoder dispatchThreadgroups:MTLSizeMake(n, 1, 1) threadsPerThreadgroup:MTLSizeMake(1, 1, 1)]; + } break; + case GGML_OP_SUM_ROWS: + { + GGML_ASSERT(src0->nb[0] == ggml_type_size(src0->type)); + + id pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_SUM_ROWS].pipeline; + + [encoder setComputePipelineState:pipeline]; + [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0]; + [encoder setBuffer:id_dst offset:offs_dst atIndex:1]; + [encoder setBytes:&ne00 length:sizeof(ne00) atIndex:2]; + [encoder setBytes:&ne01 length:sizeof(ne01) atIndex:3]; + [encoder setBytes:&ne02 length:sizeof(ne02) atIndex:4]; + [encoder setBytes:&ne03 length:sizeof(ne03) atIndex:5]; + [encoder setBytes:&nb00 length:sizeof(nb00) atIndex:6]; + [encoder setBytes:&nb01 length:sizeof(nb01) atIndex:7]; + [encoder setBytes:&nb02 length:sizeof(nb02) atIndex:8]; + [encoder setBytes:&nb03 length:sizeof(nb03) atIndex:9]; + [encoder setBytes:&ne10 length:sizeof(ne10) atIndex:10]; + [encoder setBytes:&ne11 length:sizeof(ne11) atIndex:11]; + [encoder setBytes:&ne12 length:sizeof(ne12) atIndex:12]; + [encoder setBytes:&ne13 length:sizeof(ne13) atIndex:13]; + [encoder setBytes:&nb10 length:sizeof(nb10) atIndex:14]; + [encoder setBytes:&nb11 length:sizeof(nb11) atIndex:15]; + [encoder setBytes:&nb12 length:sizeof(nb12) atIndex:16]; + [encoder setBytes:&nb13 length:sizeof(nb13) atIndex:17]; + [encoder setBytes:&ne0 length:sizeof(ne0) atIndex:18]; + [encoder setBytes:&ne1 length:sizeof(ne1) atIndex:19]; + [encoder setBytes:&ne2 length:sizeof(ne2) atIndex:20]; + [encoder setBytes:&ne3 length:sizeof(ne3) atIndex:21]; + [encoder setBytes:&nb0 length:sizeof(nb0) atIndex:22]; + [encoder setBytes:&nb1 length:sizeof(nb1) atIndex:23]; + [encoder setBytes:&nb2 length:sizeof(nb2) atIndex:24]; + [encoder setBytes:&nb3 length:sizeof(nb3) atIndex:25]; + + [encoder dispatchThreadgroups:MTLSizeMake(ne01, ne02, ne03) threadsPerThreadgroup:MTLSizeMake(1, 1, 1)]; + } break; + case GGML_OP_SOFT_MAX: + { + int nth = 32; // SIMD width + + id pipeline = nil; + + if (ne00%4 == 0) { + while (nth < ne00/4 && nth < 256) { + nth *= 2; + } + pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_SOFT_MAX_4].pipeline; + } else { + while (nth < ne00 && nth < 1024) { + nth *= 2; + } + pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_SOFT_MAX].pipeline; + } + + const float scale = ((float *) dst->op_params)[0]; + + [encoder setComputePipelineState:pipeline]; + [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0]; + if (id_src1) { + [encoder setBuffer:id_src1 offset:offs_src1 atIndex:1]; + } else { + [encoder setBuffer:id_src0 offset:offs_src0 atIndex:1]; + } + [encoder setBuffer:id_dst offset:offs_dst atIndex:2]; + [encoder setBytes:&ne00 length:sizeof(ne00) atIndex:3]; + [encoder setBytes:&ne01 length:sizeof(ne01) atIndex:4]; + [encoder setBytes:&ne02 length:sizeof(ne02) atIndex:5]; + [encoder setBytes:&scale length:sizeof(scale) atIndex:6]; + [encoder setThreadgroupMemoryLength:32*sizeof(float) atIndex:0]; + + [encoder dispatchThreadgroups:MTLSizeMake(ne01*ne02*ne03, 1, 1) threadsPerThreadgroup:MTLSizeMake(nth, 1, 1)]; + } break; + case GGML_OP_DIAG_MASK_INF: + { + const int n_past = ((int32_t *)(dst->op_params))[0]; + + id pipeline = nil; + + if (ne00%8 == 0) { + pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_DIAG_MASK_INF_8].pipeline; + } else { + pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_DIAG_MASK_INF].pipeline; + } + + [encoder setComputePipelineState:pipeline]; + [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0]; + [encoder setBuffer:id_dst offset:offs_dst atIndex:1]; + [encoder setBytes:&ne00 length:sizeof(ne00) atIndex:2]; + [encoder setBytes:&ne01 length:sizeof(ne01) atIndex:3]; + [encoder setBytes:&n_past length:sizeof(int) atIndex:4]; + + if (ne00%8 == 0) { + [encoder dispatchThreadgroups:MTLSizeMake(ne00*ne01*ne02/8, 1, 1) threadsPerThreadgroup:MTLSizeMake(1, 1, 1)]; + } + else { + [encoder dispatchThreadgroups:MTLSizeMake(ne00, ne01, ne02) threadsPerThreadgroup:MTLSizeMake(1, 1, 1)]; + } + } break; + case GGML_OP_MUL_MAT: + { + GGML_ASSERT(ne00 == ne10); + + // TODO: assert that dim2 and dim3 are contiguous + GGML_ASSERT(ne12 % ne02 == 0); + GGML_ASSERT(ne13 % ne03 == 0); + + const uint r2 = ne12/ne02; + const uint r3 = ne13/ne03; + + // find the break-even point where the matrix-matrix kernel becomes more efficient compared + // to the matrix-vector kernel + int ne11_mm_min = 1; + +#if 0 + // the numbers below are measured on M2 Ultra for 7B and 13B models + // these numbers do not translate to other devices or model sizes + // TODO: need to find a better approach + if ([ctx->device.name isEqualToString:@"Apple M2 Ultra"]) { + switch (src0t) { + case GGML_TYPE_F16: ne11_mm_min = 2; break; + case GGML_TYPE_Q8_0: ne11_mm_min = 7; break; + case GGML_TYPE_Q2_K: ne11_mm_min = 15; break; + case GGML_TYPE_Q3_K: ne11_mm_min = 7; break; + case GGML_TYPE_Q4_0: + case GGML_TYPE_Q4_1: ne11_mm_min = 15; break; + case GGML_TYPE_Q4_K: ne11_mm_min = 11; break; + case GGML_TYPE_Q5_0: // not tested yet + case GGML_TYPE_Q5_1: ne11_mm_min = 13; break; // not tested yet + case GGML_TYPE_Q5_K: ne11_mm_min = 7; break; + case GGML_TYPE_Q6_K: ne11_mm_min = 7; break; + default: ne11_mm_min = 1; break; + } + } +#endif + + // for now the matrix-matrix multiplication kernel only works on A14+/M1+ SoCs + // AMD GPU and older A-chips will reuse matrix-vector multiplication kernel + if ([ctx->device supportsFamily:MTLGPUFamilyApple7] && + !ggml_is_transposed(src0) && + !ggml_is_transposed(src1) && + src1t == GGML_TYPE_F32 && + ne00 % 32 == 0 && ne00 >= 64 && + (ne11 > ne11_mm_min || (ggml_is_quantized(src0t) && ne12 > 1))) { + //printf("matrix: ne00 = %6d, ne01 = %6d, ne02 = %6d, ne11 = %6d, ne12 = %6d\n", ne00, ne01, ne02, ne11, ne12); + + id pipeline = nil; + + switch (src0->type) { + case GGML_TYPE_F32: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_F32_F32 ].pipeline; break; + case GGML_TYPE_F16: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_F16_F32 ].pipeline; break; + case GGML_TYPE_Q4_0: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_Q4_0_F32 ].pipeline; break; + case GGML_TYPE_Q4_1: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_Q4_1_F32 ].pipeline; break; + case GGML_TYPE_Q5_0: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_Q5_0_F32 ].pipeline; break; + case GGML_TYPE_Q5_1: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_Q5_1_F32 ].pipeline; break; + case GGML_TYPE_Q8_0: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_Q8_0_F32 ].pipeline; break; + case GGML_TYPE_Q2_K: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_Q2_K_F32 ].pipeline; break; + case GGML_TYPE_Q3_K: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_Q3_K_F32 ].pipeline; break; + case GGML_TYPE_Q4_K: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_Q4_K_F32 ].pipeline; break; + case GGML_TYPE_Q5_K: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_Q5_K_F32 ].pipeline; break; + case GGML_TYPE_Q6_K: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_Q6_K_F32 ].pipeline; break; + case GGML_TYPE_IQ2_XXS: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_IQ2_XXS_F32].pipeline; break; + case GGML_TYPE_IQ2_XS: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_IQ2_XS_F32 ].pipeline; break; + default: GGML_ASSERT(false && "MUL MAT-MAT not implemented"); + } + + [encoder setComputePipelineState:pipeline]; + [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0]; + [encoder setBuffer:id_src1 offset:offs_src1 atIndex:1]; + [encoder setBuffer:id_dst offset:offs_dst atIndex:2]; + [encoder setBytes:&ne00 length:sizeof(ne00) atIndex:3]; + [encoder setBytes:&ne02 length:sizeof(ne02) atIndex:4]; + [encoder setBytes:&nb01 length:sizeof(nb01) atIndex:5]; + [encoder setBytes:&nb02 length:sizeof(nb02) atIndex:6]; + [encoder setBytes:&ne12 length:sizeof(ne12) atIndex:7]; + [encoder setBytes:&nb10 length:sizeof(nb10) atIndex:8]; + [encoder setBytes:&nb11 length:sizeof(nb11) atIndex:9]; + [encoder setBytes:&nb12 length:sizeof(nb12) atIndex:10]; + [encoder setBytes:&ne0 length:sizeof(ne0) atIndex:11]; + [encoder setBytes:&ne1 length:sizeof(ne1) atIndex:12]; + [encoder setBytes:&r2 length:sizeof(r2) atIndex:13]; + [encoder setBytes:&r3 length:sizeof(r3) atIndex:14]; + [encoder setThreadgroupMemoryLength:8192 atIndex:0]; + [encoder dispatchThreadgroups:MTLSizeMake( (ne11 + 31)/32, (ne01 + 63)/64, ne12*ne13) threadsPerThreadgroup:MTLSizeMake(128, 1, 1)]; + } else { + int nth0 = 32; + int nth1 = 1; + int nrows = 1; + //printf("vector: ne00 = %6d, ne01 = %6d, ne02 = %6d, ne11 = %6d, ne12 = %6d\n", ne00, ne01, ne02, ne11, ne12); + + id pipeline = nil; + + // use custom matrix x vector kernel + switch (src0t) { + case GGML_TYPE_F32: + { + GGML_ASSERT(src1t == GGML_TYPE_F32); + pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_F32_F32].pipeline; + nrows = 4; + } break; + case GGML_TYPE_F16: + { + nth0 = 32; + nth1 = 1; + if (src1t == GGML_TYPE_F32) { + if (ne11 * ne12 < 4) { + pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_F16_F32_1ROW].pipeline; + } else if (ne00 >= 128 && ne01 >= 8 && ne00%4 == 0) { + pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_F16_F32_L4].pipeline; + nrows = ne11; + } else { + pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_F16_F32].pipeline; + nrows = 4; + } + } else { + pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_F16_F16].pipeline; + nrows = 4; + } + } break; + case GGML_TYPE_Q4_0: + { + nth0 = 8; + nth1 = 8; + pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_Q4_0_F32].pipeline; + } break; + case GGML_TYPE_Q4_1: + { + nth0 = 8; + nth1 = 8; + pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_Q4_1_F32].pipeline; + } break; + case GGML_TYPE_Q5_0: + { + nth0 = 8; + nth1 = 8; + pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_Q5_0_F32].pipeline; + } break; + case GGML_TYPE_Q5_1: + { + nth0 = 8; + nth1 = 8; + pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_Q5_1_F32].pipeline; + } break; + case GGML_TYPE_Q8_0: + { + nth0 = 8; + nth1 = 8; + pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_Q8_0_F32].pipeline; + } break; + case GGML_TYPE_Q2_K: + { + nth0 = 2; + nth1 = 32; + pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_Q2_K_F32].pipeline; + } break; + case GGML_TYPE_Q3_K: + { + nth0 = 2; + nth1 = 32; + pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_Q3_K_F32].pipeline; + } break; + case GGML_TYPE_Q4_K: + { + nth0 = 4; //1; + nth1 = 8; //32; + pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_Q4_K_F32].pipeline; + } break; + case GGML_TYPE_Q5_K: + { + nth0 = 2; + nth1 = 32; + pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_Q5_K_F32].pipeline; + } break; + case GGML_TYPE_Q6_K: + { + nth0 = 2; + nth1 = 32; + pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_Q6_K_F32].pipeline; + } break; + case GGML_TYPE_IQ2_XXS: + { + nth0 = 4; + nth1 = 16; + pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_IQ2_XXS_F32].pipeline; + } break; + case GGML_TYPE_IQ2_XS: + { + nth0 = 4; + nth1 = 16; + pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_IQ2_XS_F32].pipeline; + } break; + default: + { + GGML_METAL_LOG_ERROR("Asserting on type %d\n", (int)src0t); + GGML_ASSERT(false && "not implemented"); + } + }; + + if (ggml_is_quantized(src0t)) { + GGML_ASSERT(ne00 >= nth0*nth1); + } + + [encoder setComputePipelineState:pipeline]; + [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0]; + [encoder setBuffer:id_src1 offset:offs_src1 atIndex:1]; + [encoder setBuffer:id_dst offset:offs_dst atIndex:2]; + [encoder setBytes:&ne00 length:sizeof(ne00) atIndex:3]; + [encoder setBytes:&ne01 length:sizeof(ne01) atIndex:4]; + [encoder setBytes:&ne02 length:sizeof(ne02) atIndex:5]; + [encoder setBytes:&nb00 length:sizeof(nb00) atIndex:6]; + [encoder setBytes:&nb01 length:sizeof(nb01) atIndex:7]; + [encoder setBytes:&nb02 length:sizeof(nb02) atIndex:8]; + [encoder setBytes:&ne10 length:sizeof(ne10) atIndex:9]; + [encoder setBytes:&ne11 length:sizeof(ne11) atIndex:10]; + [encoder setBytes:&ne12 length:sizeof(ne12) atIndex:11]; + [encoder setBytes:&nb10 length:sizeof(nb10) atIndex:12]; + [encoder setBytes:&nb11 length:sizeof(nb11) atIndex:13]; + [encoder setBytes:&nb12 length:sizeof(nb12) atIndex:14]; + [encoder setBytes:&ne0 length:sizeof(ne0) atIndex:15]; + [encoder setBytes:&ne1 length:sizeof(ne1) atIndex:16]; + [encoder setBytes:&r2 length:sizeof(r2) atIndex:17]; + [encoder setBytes:&r3 length:sizeof(r3) atIndex:18]; + + if (src0t == GGML_TYPE_Q4_0 || src0t == GGML_TYPE_Q4_1 || + src0t == GGML_TYPE_Q5_0 || src0t == GGML_TYPE_Q5_1 || src0t == GGML_TYPE_Q8_0 || + src0t == GGML_TYPE_Q2_K) { // || src0t == GGML_TYPE_Q4_K) { + [encoder dispatchThreadgroups:MTLSizeMake((ne01 + 7)/8, ne11, ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)]; + } + else if (src0t == GGML_TYPE_IQ2_XXS || src0t == GGML_TYPE_IQ2_XS) { + const int mem_size = src0t == GGML_TYPE_IQ2_XXS ? 256*8+128 : 512*8+128; + [encoder setThreadgroupMemoryLength:mem_size atIndex:0]; + [encoder dispatchThreadgroups:MTLSizeMake((ne01 + 7)/8, ne11, ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)]; + } + else if (src0t == GGML_TYPE_Q4_K) { + [encoder dispatchThreadgroups:MTLSizeMake((ne01 + 3)/4, ne11, ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)]; + } + else if (src0t == GGML_TYPE_Q3_K) { +#ifdef GGML_QKK_64 + [encoder dispatchThreadgroups:MTLSizeMake((ne01 + 1)/2, ne11, ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)]; +#else + [encoder dispatchThreadgroups:MTLSizeMake((ne01 + 3)/4, ne11, ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)]; +#endif + } + else if (src0t == GGML_TYPE_Q5_K) { + [encoder dispatchThreadgroups:MTLSizeMake((ne01 + 3)/4, ne11, ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)]; + } + else if (src0t == GGML_TYPE_Q6_K) { + [encoder dispatchThreadgroups:MTLSizeMake((ne01 + 1)/2, ne11, ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)]; + } else { + const int64_t ny = (ne11 + nrows - 1)/nrows; + [encoder dispatchThreadgroups:MTLSizeMake(ne01, ny, ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)]; + } + } + } break; + case GGML_OP_MUL_MAT_ID: + { + //GGML_ASSERT(ne00 == ne10); + //GGML_ASSERT(ne03 == ne13); + + GGML_ASSERT(src0t == GGML_TYPE_I32); + + const int n_as = ((int32_t *) dst->op_params)[1]; + + // TODO: make this more general + GGML_ASSERT(n_as <= 8); + + // max size of the src1ids array in the kernel stack + GGML_ASSERT(ne11 <= 512); + + struct ggml_tensor * src2 = gf->nodes[i]->src[2]; + + const int64_t ne20 = src2 ? src2->ne[0] : 0; + const int64_t ne21 = src2 ? src2->ne[1] : 0; + const int64_t ne22 = src2 ? src2->ne[2] : 0; + const int64_t ne23 = src2 ? src2->ne[3] : 0; GGML_UNUSED(ne23); + + const uint64_t nb20 = src2 ? src2->nb[0] : 0; GGML_UNUSED(nb20); + const uint64_t nb21 = src2 ? src2->nb[1] : 0; + const uint64_t nb22 = src2 ? src2->nb[2] : 0; + const uint64_t nb23 = src2 ? src2->nb[3] : 0; GGML_UNUSED(nb23); + + const enum ggml_type src2t = src2 ? src2->type : GGML_TYPE_COUNT; GGML_UNUSED(src2t); + + GGML_ASSERT(!ggml_is_transposed(src2)); + GGML_ASSERT(!ggml_is_transposed(src1)); + + GGML_ASSERT(src1t == GGML_TYPE_F32); + + const uint r2 = ne12/ne22; + const uint r3 = ne13/ne23; + + // find the break-even point where the matrix-matrix kernel becomes more efficient compared + // to the matrix-vector kernel + int ne11_mm_min = n_as; + + const int idx = ((int32_t *) dst->op_params)[0]; + + // batch size + GGML_ASSERT(ne01 == ne11); + + // for now the matrix-matrix multiplication kernel only works on A14+/M1+ SoCs + // AMD GPU and older A-chips will reuse matrix-vector multiplication kernel + // !!! + // TODO: for now, always use mat-vec kernels until we figure out how to improve the + // indirect matrix multiplication + // !!! + if ([ctx->device supportsFamily:MTLGPUFamilyApple7] && + ne20 % 32 == 0 && ne20 >= 64 && + ne11 > ne11_mm_min) { + + id pipeline = nil; + + switch (src2->type) { + case GGML_TYPE_F32: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_ID_F32_F32 ].pipeline; break; + case GGML_TYPE_F16: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_ID_F16_F32 ].pipeline; break; + case GGML_TYPE_Q4_0: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q4_0_F32 ].pipeline; break; + case GGML_TYPE_Q4_1: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q4_1_F32 ].pipeline; break; + case GGML_TYPE_Q5_0: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q5_0_F32 ].pipeline; break; + case GGML_TYPE_Q5_1: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q5_1_F32 ].pipeline; break; + case GGML_TYPE_Q8_0: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q8_0_F32 ].pipeline; break; + case GGML_TYPE_Q2_K: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q2_K_F32 ].pipeline; break; + case GGML_TYPE_Q3_K: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q3_K_F32 ].pipeline; break; + case GGML_TYPE_Q4_K: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q4_K_F32 ].pipeline; break; + case GGML_TYPE_Q5_K: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q5_K_F32 ].pipeline; break; + case GGML_TYPE_Q6_K: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q6_K_F32 ].pipeline; break; + case GGML_TYPE_IQ2_XXS: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ2_XXS_F32].pipeline; break; + case GGML_TYPE_IQ2_XS: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ2_XS_F32 ].pipeline; break; + default: GGML_ASSERT(false && "MUL_MAT_ID not implemented"); + } + + [encoder setComputePipelineState:pipeline]; + [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0]; + [encoder setBuffer:id_src1 offset:offs_src1 atIndex:1]; + [encoder setBuffer:id_dst offset:offs_dst atIndex:2]; + [encoder setBytes:&nb01 length:sizeof(nb01) atIndex:3]; + [encoder setBytes:&ne20 length:sizeof(ne20) atIndex:4]; + [encoder setBytes:&ne22 length:sizeof(ne22) atIndex:5]; + [encoder setBytes:&nb21 length:sizeof(nb21) atIndex:6]; + [encoder setBytes:&nb22 length:sizeof(nb22) atIndex:7]; + [encoder setBytes:&ne12 length:sizeof(ne12) atIndex:8]; + [encoder setBytes:&ne13 length:sizeof(ne13) atIndex:9]; + [encoder setBytes:&nb10 length:sizeof(nb10) atIndex:10]; + [encoder setBytes:&nb11 length:sizeof(nb11) atIndex:11]; + [encoder setBytes:&nb12 length:sizeof(nb12) atIndex:12]; + [encoder setBytes:&ne0 length:sizeof(ne0) atIndex:13]; + [encoder setBytes:&ne1 length:sizeof(ne1) atIndex:14]; + [encoder setBytes:&nb1 length:sizeof(nb1) atIndex:15]; + [encoder setBytes:&r2 length:sizeof(r2) atIndex:16]; + [encoder setBytes:&r3 length:sizeof(r3) atIndex:17]; + [encoder setBytes:&idx length:sizeof(idx) atIndex:18]; + // TODO: how to make this an array? read Metal docs + for (int j = 0; j < 8; ++j) { + // NOTE: this is done like this to avoid uninitialized kernel arguments when n_as < 8 + struct ggml_tensor * src_cur = dst->src[2 + (j % n_as)]; + + size_t offs_src_cur = 0; + id id_src_cur = ggml_metal_get_buffer(ctx, src_cur, &offs_src_cur); + + [encoder setBuffer:id_src_cur offset:offs_src_cur atIndex:19 + j]; + } + + [encoder setThreadgroupMemoryLength:8192 atIndex:0]; + + [encoder dispatchThreadgroups:MTLSizeMake((ne11 + 31)/32, (ne21 + 63)/64, n_as*ne12*ne13) threadsPerThreadgroup:MTLSizeMake(128, 1, 1)]; + } else { + int nth0 = 32; + int nth1 = 1; + int nrows = 1; + //printf("vector: ne00 = %6d, ne01 = %6d, ne02 = %6d, ne11 = %6d, ne12 = %6d\n", ne00, ne01, ne02, ne11, ne12); + + id pipeline = nil; + + // use custom matrix x vector kernel + switch (src2t) { + case GGML_TYPE_F32: + { + GGML_ASSERT(src1t == GGML_TYPE_F32); + pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_ID_F32_F32].pipeline; + } break; + case GGML_TYPE_F16: + { + GGML_ASSERT(src1t == GGML_TYPE_F32); + nth0 = 32; + nth1 = 1; + pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_ID_F16_F32].pipeline; + } break; + case GGML_TYPE_Q4_0: + { + nth0 = 8; + nth1 = 8; + pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q4_0_F32].pipeline; + } break; + case GGML_TYPE_Q4_1: + { + nth0 = 8; + nth1 = 8; + pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q4_1_F32].pipeline; + } break; + case GGML_TYPE_Q5_0: + { + nth0 = 8; + nth1 = 8; + pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q5_0_F32].pipeline; + } break; + case GGML_TYPE_Q5_1: + { + nth0 = 8; + nth1 = 8; + pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q5_1_F32].pipeline; + } break; + case GGML_TYPE_Q8_0: + { + nth0 = 8; + nth1 = 8; + pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q8_0_F32].pipeline; + } break; + case GGML_TYPE_Q2_K: + { + nth0 = 2; + nth1 = 32; + pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q2_K_F32].pipeline; + } break; + case GGML_TYPE_Q3_K: + { + nth0 = 2; + nth1 = 32; + pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q3_K_F32].pipeline; + } break; + case GGML_TYPE_Q4_K: + { + nth0 = 4; //1; + nth1 = 8; //32; + pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q4_K_F32].pipeline; + } break; + case GGML_TYPE_Q5_K: + { + nth0 = 2; + nth1 = 32; + pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q5_K_F32].pipeline; + } break; + case GGML_TYPE_Q6_K: + { + nth0 = 2; + nth1 = 32; + pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q6_K_F32].pipeline; + } break; + case GGML_TYPE_IQ2_XXS: + { + nth0 = 4; + nth1 = 16; + pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ2_XXS_F32].pipeline; + } break; + case GGML_TYPE_IQ2_XS: + { + nth0 = 4; + nth1 = 16; + pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ2_XS_F32].pipeline; + } break; + default: + { + GGML_METAL_LOG_ERROR("Asserting on type %d\n", (int)src2t); + GGML_ASSERT(false && "not implemented"); + } + }; + + if (ggml_is_quantized(src2t)) { + GGML_ASSERT(ne20 >= nth0*nth1); + } + + const int64_t _ne1 = 1; // kernels needs a reference in constant memory + + [encoder setComputePipelineState:pipeline]; + [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0]; + [encoder setBuffer:id_src1 offset:offs_src1 atIndex:1]; + [encoder setBuffer:id_dst offset:offs_dst atIndex:2]; + [encoder setBytes:&nb01 length:sizeof(nb01) atIndex:3]; + [encoder setBytes:&ne20 length:sizeof(ne20) atIndex:4]; + [encoder setBytes:&ne21 length:sizeof(ne21) atIndex:5]; + [encoder setBytes:&ne22 length:sizeof(ne22) atIndex:6]; + [encoder setBytes:&nb20 length:sizeof(nb20) atIndex:7]; + [encoder setBytes:&nb21 length:sizeof(nb21) atIndex:8]; + [encoder setBytes:&nb22 length:sizeof(nb22) atIndex:9]; + [encoder setBytes:&ne10 length:sizeof(ne10) atIndex:10]; + [encoder setBytes:&_ne1 length:sizeof(_ne1) atIndex:11]; + [encoder setBytes:&ne12 length:sizeof(ne12) atIndex:12]; + [encoder setBytes:&ne13 length:sizeof(ne13) atIndex:13]; + [encoder setBytes:&nb10 length:sizeof(nb10) atIndex:14]; + [encoder setBytes:&nb11 length:sizeof(nb11) atIndex:15]; + [encoder setBytes:&nb12 length:sizeof(nb12) atIndex:16]; + [encoder setBytes:&ne0 length:sizeof(ne0) atIndex:17]; + [encoder setBytes:&_ne1 length:sizeof(_ne1) atIndex:18]; + [encoder setBytes:&nb1 length:sizeof(nb1) atIndex:19]; + [encoder setBytes:&r2 length:sizeof(r2) atIndex:20]; + [encoder setBytes:&r3 length:sizeof(r3) atIndex:21]; + [encoder setBytes:&idx length:sizeof(idx) atIndex:22]; + // TODO: how to make this an array? read Metal docs + for (int j = 0; j < 8; ++j) { + // NOTE: this is done like this to avoid uninitialized kernel arguments when n_as < 8 + struct ggml_tensor * src_cur = dst->src[2 + (j % n_as)]; + + size_t offs_src_cur = 0; + id id_src_cur = ggml_metal_get_buffer(ctx, src_cur, &offs_src_cur); + + [encoder setBuffer:id_src_cur offset:offs_src_cur atIndex:23 + j]; + } + + if (src2t == GGML_TYPE_Q4_0 || src2t == GGML_TYPE_Q4_1 || + src2t == GGML_TYPE_Q5_0 || src2t == GGML_TYPE_Q5_1 || src2t == GGML_TYPE_Q8_0 || + src2t == GGML_TYPE_Q2_K) { // || src2t == GGML_TYPE_Q4_K) { + [encoder dispatchThreadgroups:MTLSizeMake((ne21 + 7)/8, _ne1, ne01*ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)]; + } + else if (src2t == GGML_TYPE_IQ2_XXS || src2t == GGML_TYPE_IQ2_XS) { + const int mem_size = src2t == GGML_TYPE_IQ2_XXS ? 256*8+128 : 512*8+128; + [encoder setThreadgroupMemoryLength:mem_size atIndex:0]; + [encoder dispatchThreadgroups:MTLSizeMake((ne21 + 7)/8, _ne1, ne01*ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)]; + } + else if (src2t == GGML_TYPE_Q4_K) { + [encoder dispatchThreadgroups:MTLSizeMake((ne21 + 3)/4, _ne1, ne01*ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)]; + } + else if (src2t == GGML_TYPE_Q3_K) { +#ifdef GGML_QKK_64 + [encoder dispatchThreadgroups:MTLSizeMake((ne21 + 1)/2, _ne1, ne01*ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)]; +#else + [encoder dispatchThreadgroups:MTLSizeMake((ne21 + 3)/4, _ne1, ne01*ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)]; +#endif + } + else if (src2t == GGML_TYPE_Q5_K) { + [encoder dispatchThreadgroups:MTLSizeMake((ne21 + 3)/4, _ne1, ne01*ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)]; + } + else if (src2t == GGML_TYPE_Q6_K) { + [encoder dispatchThreadgroups:MTLSizeMake((ne21 + 1)/2, _ne1, ne01*ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)]; + } else { + const int64_t ny = (_ne1 + nrows - 1)/nrows; + [encoder dispatchThreadgroups:MTLSizeMake(ne21, ny, ne01*ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)]; + } + } + } break; + case GGML_OP_GET_ROWS: + { + id pipeline = nil; + + switch (src0->type) { + case GGML_TYPE_F32: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GET_ROWS_F32 ].pipeline; break; + case GGML_TYPE_F16: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GET_ROWS_F16 ].pipeline; break; + case GGML_TYPE_Q4_0: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GET_ROWS_Q4_0 ].pipeline; break; + case GGML_TYPE_Q4_1: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GET_ROWS_Q4_1 ].pipeline; break; + case GGML_TYPE_Q5_0: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GET_ROWS_Q5_0 ].pipeline; break; + case GGML_TYPE_Q5_1: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GET_ROWS_Q5_1 ].pipeline; break; + case GGML_TYPE_Q8_0: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GET_ROWS_Q8_0 ].pipeline; break; + case GGML_TYPE_Q2_K: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GET_ROWS_Q2_K ].pipeline; break; + case GGML_TYPE_Q3_K: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GET_ROWS_Q3_K ].pipeline; break; + case GGML_TYPE_Q4_K: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GET_ROWS_Q4_K ].pipeline; break; + case GGML_TYPE_Q5_K: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GET_ROWS_Q5_K ].pipeline; break; + case GGML_TYPE_Q6_K: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GET_ROWS_Q6_K ].pipeline; break; + case GGML_TYPE_IQ2_XXS: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ2_XXS].pipeline; break; + case GGML_TYPE_IQ2_XS: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ2_XS ].pipeline; break; + case GGML_TYPE_I32: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GET_ROWS_I32 ].pipeline; break; + default: GGML_ASSERT(false && "not implemented"); + } + + [encoder setComputePipelineState:pipeline]; + [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0]; + [encoder setBuffer:id_src1 offset:offs_src1 atIndex:1]; + [encoder setBuffer:id_dst offset:offs_dst atIndex:2]; + [encoder setBytes:&ne00 length:sizeof( int64_t) atIndex:3]; + [encoder setBytes:&nb01 length:sizeof(uint64_t) atIndex:4]; + [encoder setBytes:&nb02 length:sizeof(uint64_t) atIndex:5]; + [encoder setBytes:&ne10 length:sizeof( int64_t) atIndex:6]; + [encoder setBytes:&nb10 length:sizeof( int64_t) atIndex:7]; + [encoder setBytes:&nb11 length:sizeof( int64_t) atIndex:8]; + [encoder setBytes:&nb1 length:sizeof(uint64_t) atIndex:9]; + [encoder setBytes:&nb2 length:sizeof(uint64_t) atIndex:10]; + + [encoder dispatchThreadgroups:MTLSizeMake(ne10, ne11, 1) threadsPerThreadgroup:MTLSizeMake(32, 1, 1)]; + } break; + case GGML_OP_RMS_NORM: + { + GGML_ASSERT(ne00 % 4 == 0); + + float eps; + memcpy(&eps, dst->op_params, sizeof(float)); + + int nth = 32; // SIMD width + + while (nth < ne00/4 && nth < 1024) { + nth *= 2; + } + + id pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_RMS_NORM].pipeline; + + [encoder setComputePipelineState:pipeline]; + [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0]; + [encoder setBuffer:id_dst offset:offs_dst atIndex:1]; + [encoder setBytes:&ne00 length:sizeof( int64_t) atIndex:2]; + [encoder setBytes:&nb01 length:sizeof(uint64_t) atIndex:3]; + [encoder setBytes:&eps length:sizeof( float) atIndex:4]; + [encoder setThreadgroupMemoryLength:32*sizeof(float) atIndex:0]; + + const int64_t nrows = ggml_nrows(src0); + + [encoder dispatchThreadgroups:MTLSizeMake(nrows, 1, 1) threadsPerThreadgroup:MTLSizeMake(nth, 1, 1)]; + } break; + case GGML_OP_GROUP_NORM: + { + GGML_ASSERT(ne00 % 4 == 0); + + //float eps; + //memcpy(&eps, dst->op_params, sizeof(float)); + + const float eps = 1e-6f; // TODO: temporarily hardcoded + + const int32_t n_groups = ((int32_t *) dst->op_params)[0]; + + int nth = 32; // SIMD width + + //while (nth < ne00/4 && nth < 1024) { + // nth *= 2; + //} + + id pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GROUP_NORM].pipeline; + + [encoder setComputePipelineState:pipeline]; + [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0]; + [encoder setBuffer:id_dst offset:offs_dst atIndex:1]; + [encoder setBytes:&ne00 length:sizeof( int64_t) atIndex:2]; + [encoder setBytes:&ne01 length:sizeof( int64_t) atIndex:3]; + [encoder setBytes:&ne02 length:sizeof( int64_t) atIndex:4]; + [encoder setBytes:&nb00 length:sizeof(uint64_t) atIndex:5]; + [encoder setBytes:&nb01 length:sizeof(uint64_t) atIndex:6]; + [encoder setBytes:&nb02 length:sizeof(uint64_t) atIndex:7]; + [encoder setBytes:&n_groups length:sizeof( int32_t) atIndex:8]; + [encoder setBytes:&eps length:sizeof( float) atIndex:9]; + [encoder setThreadgroupMemoryLength:32*sizeof(float) atIndex:0]; + + [encoder dispatchThreadgroups:MTLSizeMake(n_groups, 1, 1) threadsPerThreadgroup:MTLSizeMake(nth, 1, 1)]; + } break; + case GGML_OP_NORM: + { + float eps; + memcpy(&eps, dst->op_params, sizeof(float)); + + const int nth = MIN(256, ne00); + + id pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_NORM].pipeline; + + [encoder setComputePipelineState:pipeline]; + [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0]; + [encoder setBuffer:id_dst offset:offs_dst atIndex:1]; + [encoder setBytes:&ne00 length:sizeof( int64_t) atIndex:2]; + [encoder setBytes:&nb01 length:sizeof(uint64_t) atIndex:3]; + [encoder setBytes:&eps length:sizeof( float) atIndex:4]; + [encoder setThreadgroupMemoryLength:GGML_PAD(nth*sizeof(float), 16) atIndex:0]; + + const int64_t nrows = ggml_nrows(src0); + + [encoder dispatchThreadgroups:MTLSizeMake(nrows, 1, 1) threadsPerThreadgroup:MTLSizeMake(nth, 1, 1)]; + } break; + case GGML_OP_ALIBI: + { + GGML_ASSERT((src0t == GGML_TYPE_F32)); + + const int nth = MIN(1024, ne00); + + //const int n_past = ((int32_t *) dst->op_params)[0]; + const int n_head = ((int32_t *) dst->op_params)[1]; + float max_bias; + memcpy(&max_bias, (int32_t *) dst->op_params + 2, sizeof(float)); + + const int n_heads_log2_floor = 1 << (int) floor(log2(n_head)); + const float m0 = powf(2.0f, -(max_bias) / n_heads_log2_floor); + const float m1 = powf(2.0f, -(max_bias / 2.0f) / n_heads_log2_floor); + + id pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_ALIBI_F32].pipeline; + + [encoder setComputePipelineState:pipeline]; + [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0]; + [encoder setBuffer:id_dst offset:offs_dst atIndex:1]; + [encoder setBytes:&ne00 length:sizeof( int64_t) atIndex:2]; + [encoder setBytes:&ne01 length:sizeof( int64_t) atIndex:3]; + [encoder setBytes:&ne02 length:sizeof( int64_t) atIndex:4]; + [encoder setBytes:&ne03 length:sizeof( int64_t) atIndex:5]; + [encoder setBytes:&nb00 length:sizeof(uint64_t) atIndex:6]; + [encoder setBytes:&nb01 length:sizeof(uint64_t) atIndex:7]; + [encoder setBytes:&nb02 length:sizeof(uint64_t) atIndex:8]; + [encoder setBytes:&nb03 length:sizeof(uint64_t) atIndex:9]; + [encoder setBytes:&ne0 length:sizeof( int64_t) atIndex:10]; + [encoder setBytes:&ne1 length:sizeof( int64_t) atIndex:11]; + [encoder setBytes:&ne2 length:sizeof( int64_t) atIndex:12]; + [encoder setBytes:&ne3 length:sizeof( int64_t) atIndex:13]; + [encoder setBytes:&nb0 length:sizeof(uint64_t) atIndex:14]; + [encoder setBytes:&nb1 length:sizeof(uint64_t) atIndex:15]; + [encoder setBytes:&nb2 length:sizeof(uint64_t) atIndex:16]; + [encoder setBytes:&nb3 length:sizeof(uint64_t) atIndex:17]; + [encoder setBytes:&m0 length:sizeof( float) atIndex:18]; + [encoder setBytes:&m1 length:sizeof( float) atIndex:19]; + [encoder setBytes:&n_heads_log2_floor length:sizeof(int) atIndex:20]; + + [encoder dispatchThreadgroups:MTLSizeMake(ne01, ne02, ne03) threadsPerThreadgroup:MTLSizeMake(nth, 1, 1)]; + } break; + case GGML_OP_ROPE: + { + GGML_ASSERT(ne10 == ne02); + + const int nth = MIN(1024, ne00); + + const int n_past = ((int32_t *) dst->op_params)[0]; + const int n_dims = ((int32_t *) dst->op_params)[1]; + const int mode = ((int32_t *) dst->op_params)[2]; + // skip 3, n_ctx, used in GLM RoPE, unimplemented in metal + const int n_orig_ctx = ((int32_t *) dst->op_params)[4]; + + float freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow; + memcpy(&freq_base, (int32_t *) dst->op_params + 5, sizeof(float)); + memcpy(&freq_scale, (int32_t *) dst->op_params + 6, sizeof(float)); + memcpy(&ext_factor, (int32_t *) dst->op_params + 7, sizeof(float)); + memcpy(&attn_factor, (int32_t *) dst->op_params + 8, sizeof(float)); + memcpy(&beta_fast, (int32_t *) dst->op_params + 9, sizeof(float)); + memcpy(&beta_slow, (int32_t *) dst->op_params + 10, sizeof(float)); + + id pipeline = nil; + + switch (src0->type) { + case GGML_TYPE_F32: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_ROPE_F32].pipeline; break; + case GGML_TYPE_F16: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_ROPE_F16].pipeline; break; + default: GGML_ASSERT(false); + }; + + [encoder setComputePipelineState:pipeline]; + [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0]; + [encoder setBuffer:id_src1 offset:offs_src1 atIndex:1]; + [encoder setBuffer:id_dst offset:offs_dst atIndex:2]; + [encoder setBytes:&ne00 length:sizeof( int64_t) atIndex:3]; + [encoder setBytes:&ne01 length:sizeof( int64_t) atIndex:4]; + [encoder setBytes:&ne02 length:sizeof( int64_t) atIndex:5]; + [encoder setBytes:&ne03 length:sizeof( int64_t) atIndex:6]; + [encoder setBytes:&nb00 length:sizeof(uint64_t) atIndex:7]; + [encoder setBytes:&nb01 length:sizeof(uint64_t) atIndex:8]; + [encoder setBytes:&nb02 length:sizeof(uint64_t) atIndex:9]; + [encoder setBytes:&nb03 length:sizeof(uint64_t) atIndex:10]; + [encoder setBytes:&ne0 length:sizeof( int64_t) atIndex:11]; + [encoder setBytes:&ne1 length:sizeof( int64_t) atIndex:12]; + [encoder setBytes:&ne2 length:sizeof( int64_t) atIndex:13]; + [encoder setBytes:&ne3 length:sizeof( int64_t) atIndex:14]; + [encoder setBytes:&nb0 length:sizeof(uint64_t) atIndex:15]; + [encoder setBytes:&nb1 length:sizeof(uint64_t) atIndex:16]; + [encoder setBytes:&nb2 length:sizeof(uint64_t) atIndex:17]; + [encoder setBytes:&nb3 length:sizeof(uint64_t) atIndex:18]; + [encoder setBytes:&n_past length:sizeof( int) atIndex:19]; + [encoder setBytes:&n_dims length:sizeof( int) atIndex:20]; + [encoder setBytes:&mode length:sizeof( int) atIndex:21]; + [encoder setBytes:&n_orig_ctx length:sizeof( int) atIndex:22]; + [encoder setBytes:&freq_base length:sizeof( float) atIndex:23]; + [encoder setBytes:&freq_scale length:sizeof( float) atIndex:24]; + [encoder setBytes:&ext_factor length:sizeof( float) atIndex:25]; + [encoder setBytes:&attn_factor length:sizeof( float) atIndex:26]; + [encoder setBytes:&beta_fast length:sizeof( float) atIndex:27]; + [encoder setBytes:&beta_slow length:sizeof( float) atIndex:28]; + + [encoder dispatchThreadgroups:MTLSizeMake(ne01, ne02, ne03) threadsPerThreadgroup:MTLSizeMake(nth, 1, 1)]; + } break; + case GGML_OP_IM2COL: + { + GGML_ASSERT(src0->type == GGML_TYPE_F16); + GGML_ASSERT(src1->type == GGML_TYPE_F32); + GGML_ASSERT( dst->type == GGML_TYPE_F16); + + const int32_t s0 = ((const int32_t *)(dst->op_params))[0]; + const int32_t s1 = ((const int32_t *)(dst->op_params))[1]; + const int32_t p0 = ((const int32_t *)(dst->op_params))[2]; + const int32_t p1 = ((const int32_t *)(dst->op_params))[3]; + const int32_t d0 = ((const int32_t *)(dst->op_params))[4]; + const int32_t d1 = ((const int32_t *)(dst->op_params))[5]; + const bool is_2D = ((const int32_t *)(dst->op_params))[6] == 1; + + const int32_t N = src1->ne[is_2D ? 3 : 2]; + const int32_t IC = src1->ne[is_2D ? 2 : 1]; + const int32_t IH = is_2D ? src1->ne[1] : 1; + const int32_t IW = src1->ne[0]; + + const int32_t KH = is_2D ? src0->ne[1] : 1; + const int32_t KW = src0->ne[0]; + + const int32_t OH = is_2D ? dst->ne[2] : 1; + const int32_t OW = dst->ne[1]; + + const int32_t CHW = IC * KH * KW; + + const int32_t ofs0 = src1->nb[is_2D ? 3 : 2] / 4; + const int32_t ofs1 = src1->nb[is_2D ? 2 : 1] / 4; + + id pipeline = nil; + + switch (src0->type) { + case GGML_TYPE_F32: GGML_ASSERT(false && "not implemented"); break; + case GGML_TYPE_F16: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_IM2COL_F16].pipeline; break; + default: GGML_ASSERT(false); + }; + + [encoder setComputePipelineState:pipeline]; + [encoder setBuffer:id_src1 offset:offs_src1 atIndex:0]; + [encoder setBuffer:id_dst offset:offs_dst atIndex:1]; + [encoder setBytes:&ofs0 length:sizeof( int32_t) atIndex:2]; + [encoder setBytes:&ofs1 length:sizeof( int32_t) atIndex:3]; + [encoder setBytes:&IW length:sizeof( int32_t) atIndex:4]; + [encoder setBytes:&IH length:sizeof( int32_t) atIndex:5]; + [encoder setBytes:&CHW length:sizeof( int32_t) atIndex:6]; + [encoder setBytes:&s0 length:sizeof( int32_t) atIndex:7]; + [encoder setBytes:&s1 length:sizeof( int32_t) atIndex:8]; + [encoder setBytes:&p0 length:sizeof( int32_t) atIndex:9]; + [encoder setBytes:&p1 length:sizeof( int32_t) atIndex:10]; + [encoder setBytes:&d0 length:sizeof( int32_t) atIndex:11]; + [encoder setBytes:&d1 length:sizeof( int32_t) atIndex:12]; + + [encoder dispatchThreadgroups:MTLSizeMake(IC, OH, OW) threadsPerThreadgroup:MTLSizeMake(N, KH, KW)]; + } break; + case GGML_OP_UPSCALE: + { + GGML_ASSERT(src0->type == GGML_TYPE_F32); + + const int sf = dst->op_params[0]; + + const id pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_UPSCALE_F32].pipeline; + + [encoder setComputePipelineState:pipeline]; + [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0]; + [encoder setBuffer:id_dst offset:offs_dst atIndex:1]; + [encoder setBytes:&ne00 length:sizeof(ne00) atIndex:2]; + [encoder setBytes:&ne01 length:sizeof(ne01) atIndex:3]; + [encoder setBytes:&ne02 length:sizeof(ne02) atIndex:4]; + [encoder setBytes:&ne03 length:sizeof(ne03) atIndex:5]; + [encoder setBytes:&nb00 length:sizeof(nb00) atIndex:6]; + [encoder setBytes:&nb01 length:sizeof(nb01) atIndex:7]; + [encoder setBytes:&nb02 length:sizeof(nb02) atIndex:8]; + [encoder setBytes:&nb03 length:sizeof(nb03) atIndex:9]; + [encoder setBytes:&ne0 length:sizeof(ne0) atIndex:10]; + [encoder setBytes:&ne1 length:sizeof(ne1) atIndex:11]; + [encoder setBytes:&ne2 length:sizeof(ne2) atIndex:12]; + [encoder setBytes:&ne3 length:sizeof(ne3) atIndex:13]; + [encoder setBytes:&nb0 length:sizeof(nb0) atIndex:14]; + [encoder setBytes:&nb1 length:sizeof(nb1) atIndex:15]; + [encoder setBytes:&nb2 length:sizeof(nb2) atIndex:16]; + [encoder setBytes:&nb3 length:sizeof(nb3) atIndex:17]; + [encoder setBytes:&sf length:sizeof(sf) atIndex:18]; + + const int nth = MIN((int) pipeline.maxTotalThreadsPerThreadgroup, ne0); + + [encoder dispatchThreadgroups:MTLSizeMake(ne1, ne2, ne3) threadsPerThreadgroup:MTLSizeMake(nth, 1, 1)]; + } break; + case GGML_OP_PAD: + { + GGML_ASSERT(src0->type == GGML_TYPE_F32); + + id pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_PAD_F32].pipeline; + + [encoder setComputePipelineState:pipeline]; + [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0]; + [encoder setBuffer:id_dst offset:offs_dst atIndex:1]; + [encoder setBytes:&ne00 length:sizeof(ne00) atIndex:2]; + [encoder setBytes:&ne01 length:sizeof(ne01) atIndex:3]; + [encoder setBytes:&ne02 length:sizeof(ne02) atIndex:4]; + [encoder setBytes:&ne03 length:sizeof(ne03) atIndex:5]; + [encoder setBytes:&nb00 length:sizeof(nb00) atIndex:6]; + [encoder setBytes:&nb01 length:sizeof(nb01) atIndex:7]; + [encoder setBytes:&nb02 length:sizeof(nb02) atIndex:8]; + [encoder setBytes:&nb03 length:sizeof(nb03) atIndex:9]; + [encoder setBytes:&ne0 length:sizeof(ne0) atIndex:10]; + [encoder setBytes:&ne1 length:sizeof(ne1) atIndex:11]; + [encoder setBytes:&ne2 length:sizeof(ne2) atIndex:12]; + [encoder setBytes:&ne3 length:sizeof(ne3) atIndex:13]; + [encoder setBytes:&nb0 length:sizeof(nb0) atIndex:14]; + [encoder setBytes:&nb1 length:sizeof(nb1) atIndex:15]; + [encoder setBytes:&nb2 length:sizeof(nb2) atIndex:16]; + [encoder setBytes:&nb3 length:sizeof(nb3) atIndex:17]; + + const int nth = MIN(1024, ne0); + + [encoder dispatchThreadgroups:MTLSizeMake(ne1, ne2, ne3) threadsPerThreadgroup:MTLSizeMake(nth, 1, 1)]; + } break; + case GGML_OP_ARGSORT: + { + GGML_ASSERT(src0->type == GGML_TYPE_F32); + GGML_ASSERT( dst->type == GGML_TYPE_I32); + + const int nrows = ggml_nrows(src0); + + enum ggml_sort_order order = (enum ggml_sort_order) dst->op_params[0]; + + id pipeline = nil; + + switch (order) { + case GGML_SORT_ASC: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_ARGSORT_F32_I32_ASC].pipeline; break; + case GGML_SORT_DESC: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_ARGSORT_F32_I32_DESC].pipeline; break; + default: GGML_ASSERT(false); + }; + + [encoder setComputePipelineState:pipeline]; + [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0]; + [encoder setBuffer:id_dst offset:offs_dst atIndex:1]; + [encoder setBytes:&ne00 length:sizeof( int64_t) atIndex:2]; + + [encoder dispatchThreadgroups:MTLSizeMake(1, nrows, 1) threadsPerThreadgroup:MTLSizeMake(ne00, 1, 1)]; + } break; + case GGML_OP_LEAKY_RELU: + { + GGML_ASSERT(src0->type == GGML_TYPE_F32); + + float slope; + memcpy(&slope, dst->op_params, sizeof(float)); + + id pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_LEAKY_RELU_F32].pipeline; + + [encoder setComputePipelineState:pipeline]; + [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0]; + [encoder setBuffer:id_dst offset:offs_dst atIndex:1]; + [encoder setBytes:&slope length:sizeof(slope) atIndex:2]; + + const int64_t n = ggml_nelements(dst); + + [encoder dispatchThreadgroups:MTLSizeMake(n, 1, 1) threadsPerThreadgroup:MTLSizeMake(1, 1, 1)]; + } break; + case GGML_OP_DUP: + case GGML_OP_CPY: + case GGML_OP_CONT: + { + GGML_ASSERT(ne00 % ggml_blck_size(src0->type) == 0); + + int nth = MIN(1024, ne00/ggml_blck_size(src0->type)); + + id pipeline = nil; + + switch (src0t) { + case GGML_TYPE_F32: + { + GGML_ASSERT(ne0 % ggml_blck_size(dst->type) == 0); + + switch (dstt) { + case GGML_TYPE_F16: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_CPY_F32_F16].pipeline; break; + case GGML_TYPE_F32: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_CPY_F32_F32].pipeline; break; + case GGML_TYPE_Q8_0: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_CPY_F32_Q8_0].pipeline; break; + case GGML_TYPE_Q4_0: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_CPY_F32_Q4_0].pipeline; break; + case GGML_TYPE_Q4_1: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_CPY_F32_Q4_1].pipeline; break; + //case GGML_TYPE_Q5_0: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_CPY_F32_Q5_0].pipeline; break; + //case GGML_TYPE_Q5_1: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_CPY_F32_Q5_1].pipeline; break; + default: GGML_ASSERT(false && "not implemented"); + }; + } break; + case GGML_TYPE_F16: + { + switch (dstt) { + case GGML_TYPE_F16: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_CPY_F16_F16].pipeline; break; + case GGML_TYPE_F32: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_CPY_F16_F32].pipeline; break; + default: GGML_ASSERT(false && "not implemented"); + }; + } break; + default: GGML_ASSERT(false && "not implemented"); + } + + [encoder setComputePipelineState:pipeline]; + [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0]; + [encoder setBuffer:id_dst offset:offs_dst atIndex:1]; + [encoder setBytes:&ne00 length:sizeof( int64_t) atIndex:2]; + [encoder setBytes:&ne01 length:sizeof( int64_t) atIndex:3]; + [encoder setBytes:&ne02 length:sizeof( int64_t) atIndex:4]; + [encoder setBytes:&ne03 length:sizeof( int64_t) atIndex:5]; + [encoder setBytes:&nb00 length:sizeof(uint64_t) atIndex:6]; + [encoder setBytes:&nb01 length:sizeof(uint64_t) atIndex:7]; + [encoder setBytes:&nb02 length:sizeof(uint64_t) atIndex:8]; + [encoder setBytes:&nb03 length:sizeof(uint64_t) atIndex:9]; + [encoder setBytes:&ne0 length:sizeof( int64_t) atIndex:10]; + [encoder setBytes:&ne1 length:sizeof( int64_t) atIndex:11]; + [encoder setBytes:&ne2 length:sizeof( int64_t) atIndex:12]; + [encoder setBytes:&ne3 length:sizeof( int64_t) atIndex:13]; + [encoder setBytes:&nb0 length:sizeof(uint64_t) atIndex:14]; + [encoder setBytes:&nb1 length:sizeof(uint64_t) atIndex:15]; + [encoder setBytes:&nb2 length:sizeof(uint64_t) atIndex:16]; + [encoder setBytes:&nb3 length:sizeof(uint64_t) atIndex:17]; + + [encoder dispatchThreadgroups:MTLSizeMake(ne01, ne02, ne03) threadsPerThreadgroup:MTLSizeMake(nth, 1, 1)]; + } break; + default: + { + GGML_METAL_LOG_ERROR("%s: error: node %3d, op = %8s not implemented\n", __func__, i, ggml_op_name(dst->op)); + GGML_ASSERT(false); + } + } #ifndef GGML_METAL_NDEBUG - [encoder popDebugGroup]; + [encoder popDebugGroup]; #endif - } + } - if (encoder != nil) { - [encoder endEncoding]; - encoder = nil; - } + if (encoder != nil) { + [encoder endEncoding]; + encoder = nil; + } - [command_buffer commit]; - }); - } - - // wait for all threads to finish - dispatch_barrier_sync(ctx->d_queue, ^{}); + [command_buffer commit]; + }); // check status of command buffers // needed to detect if the device ran out-of-memory for example (#1881) From 862f5e41ab1fdf12d6f59455aad3f5dd8258f805 Mon Sep 17 00:00:00 2001 From: Neuman Vong Date: Wed, 17 Jan 2024 00:47:34 +1100 Subject: [PATCH 082/812] android : introduce starter project example (#4926) * Introduce starter project for Android Based on examples/llama.swiftui. * Add github workflow * Set NDK version * Only build arm64-v8a in CI * Sync bench code * Rename CI prop to skip-armeabi-v7a * Remove unused tests --- .github/workflows/build.yml | 25 ++ examples/llama.android/.gitignore | 33 ++ examples/llama.android/README.md | 0 examples/llama.android/app/.gitignore | 1 + examples/llama.android/app/build.gradle.kts | 91 ++++ examples/llama.android/app/proguard-rules.pro | 21 + .../app/src/main/AndroidManifest.xml | 30 ++ .../app/src/main/cpp/CMakeLists.txt | 50 +++ .../app/src/main/cpp/llama-android.cpp | 394 ++++++++++++++++++ .../java/com/example/llama/Downloadable.kt | 119 ++++++ .../src/main/java/com/example/llama/Llm.kt | 172 ++++++++ .../java/com/example/llama/MainActivity.kt | 154 +++++++ .../java/com/example/llama/MainViewModel.kt | 104 +++++ .../java/com/example/llama/ui/theme/Color.kt | 11 + .../java/com/example/llama/ui/theme/Theme.kt | 70 ++++ .../java/com/example/llama/ui/theme/Type.kt | 34 ++ .../res/drawable/ic_launcher_background.xml | 170 ++++++++ .../res/drawable/ic_launcher_foreground.xml | 30 ++ .../main/res/mipmap-anydpi/ic_launcher.xml | 6 + .../res/mipmap-anydpi/ic_launcher_round.xml | 6 + .../src/main/res/mipmap-hdpi/ic_launcher.webp | Bin 0 -> 1404 bytes .../res/mipmap-hdpi/ic_launcher_round.webp | Bin 0 -> 2898 bytes .../src/main/res/mipmap-mdpi/ic_launcher.webp | Bin 0 -> 982 bytes .../res/mipmap-mdpi/ic_launcher_round.webp | Bin 0 -> 1772 bytes .../main/res/mipmap-xhdpi/ic_launcher.webp | Bin 0 -> 1900 bytes .../res/mipmap-xhdpi/ic_launcher_round.webp | Bin 0 -> 3918 bytes .../main/res/mipmap-xxhdpi/ic_launcher.webp | Bin 0 -> 2884 bytes .../res/mipmap-xxhdpi/ic_launcher_round.webp | Bin 0 -> 5914 bytes .../main/res/mipmap-xxxhdpi/ic_launcher.webp | Bin 0 -> 3844 bytes .../res/mipmap-xxxhdpi/ic_launcher_round.webp | Bin 0 -> 7778 bytes .../app/src/main/res/values/colors.xml | 10 + .../app/src/main/res/values/strings.xml | 3 + .../app/src/main/res/values/themes.xml | 5 + .../app/src/main/res/xml/backup_rules.xml | 13 + .../main/res/xml/data_extraction_rules.xml | 19 + examples/llama.android/build.gradle.kts | 5 + examples/llama.android/gradle.properties | 23 + .../gradle/wrapper/gradle-wrapper.jar | Bin 0 -> 59203 bytes .../gradle/wrapper/gradle-wrapper.properties | 6 + examples/llama.android/gradlew | 185 ++++++++ examples/llama.android/settings.gradle.kts | 17 + 41 files changed, 1807 insertions(+) create mode 100644 examples/llama.android/.gitignore create mode 100644 examples/llama.android/README.md create mode 100644 examples/llama.android/app/.gitignore create mode 100644 examples/llama.android/app/build.gradle.kts create mode 100644 examples/llama.android/app/proguard-rules.pro create mode 100644 examples/llama.android/app/src/main/AndroidManifest.xml create mode 100644 examples/llama.android/app/src/main/cpp/CMakeLists.txt create mode 100644 examples/llama.android/app/src/main/cpp/llama-android.cpp create mode 100644 examples/llama.android/app/src/main/java/com/example/llama/Downloadable.kt create mode 100644 examples/llama.android/app/src/main/java/com/example/llama/Llm.kt create mode 100644 examples/llama.android/app/src/main/java/com/example/llama/MainActivity.kt create mode 100644 examples/llama.android/app/src/main/java/com/example/llama/MainViewModel.kt create mode 100644 examples/llama.android/app/src/main/java/com/example/llama/ui/theme/Color.kt create mode 100644 examples/llama.android/app/src/main/java/com/example/llama/ui/theme/Theme.kt create mode 100644 examples/llama.android/app/src/main/java/com/example/llama/ui/theme/Type.kt create mode 100644 examples/llama.android/app/src/main/res/drawable/ic_launcher_background.xml create mode 100644 examples/llama.android/app/src/main/res/drawable/ic_launcher_foreground.xml create mode 100644 examples/llama.android/app/src/main/res/mipmap-anydpi/ic_launcher.xml create mode 100644 examples/llama.android/app/src/main/res/mipmap-anydpi/ic_launcher_round.xml create mode 100644 examples/llama.android/app/src/main/res/mipmap-hdpi/ic_launcher.webp create mode 100644 examples/llama.android/app/src/main/res/mipmap-hdpi/ic_launcher_round.webp create mode 100644 examples/llama.android/app/src/main/res/mipmap-mdpi/ic_launcher.webp create mode 100644 examples/llama.android/app/src/main/res/mipmap-mdpi/ic_launcher_round.webp create mode 100644 examples/llama.android/app/src/main/res/mipmap-xhdpi/ic_launcher.webp create mode 100644 examples/llama.android/app/src/main/res/mipmap-xhdpi/ic_launcher_round.webp create mode 100644 examples/llama.android/app/src/main/res/mipmap-xxhdpi/ic_launcher.webp create mode 100644 examples/llama.android/app/src/main/res/mipmap-xxhdpi/ic_launcher_round.webp create mode 100644 examples/llama.android/app/src/main/res/mipmap-xxxhdpi/ic_launcher.webp create mode 100644 examples/llama.android/app/src/main/res/mipmap-xxxhdpi/ic_launcher_round.webp create mode 100644 examples/llama.android/app/src/main/res/values/colors.xml create mode 100644 examples/llama.android/app/src/main/res/values/strings.xml create mode 100644 examples/llama.android/app/src/main/res/values/themes.xml create mode 100644 examples/llama.android/app/src/main/res/xml/backup_rules.xml create mode 100644 examples/llama.android/app/src/main/res/xml/data_extraction_rules.xml create mode 100644 examples/llama.android/build.gradle.kts create mode 100644 examples/llama.android/gradle.properties create mode 100644 examples/llama.android/gradle/wrapper/gradle-wrapper.jar create mode 100644 examples/llama.android/gradle/wrapper/gradle-wrapper.properties create mode 100755 examples/llama.android/gradlew create mode 100644 examples/llama.android/settings.gradle.kts diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 0a28a1111..367df07a7 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -515,6 +515,31 @@ jobs: - name: Build Xcode project run: xcodebuild -project examples/llama.swiftui/llama.swiftui.xcodeproj -scheme llama.swiftui -sdk iphoneos CODE_SIGNING_REQUIRED=NO CODE_SIGN_IDENTITY= -destination 'generic/platform=iOS' build + android-build: + runs-on: ubuntu-latest + + steps: + - name: Clone + uses: actions/checkout@v3 + + - name: Set up JDK + uses: actions/setup-java@v3 + with: + java-version: 17 + distribution: zulu + + - name: Setup Android SDK + uses: android-actions/setup-android@v3 + with: + log-accepted-android-sdk-licenses: false + + - name: Build + run: | + cd examples/llama.android + + # Skip armeabi-v7a for now (https://github.com/llvm/llvm-project/issues/65820). + ./gradlew build --no-daemon -Pskip-armeabi-v7a + # freeBSD-latest: # runs-on: macos-12 # steps: diff --git a/examples/llama.android/.gitignore b/examples/llama.android/.gitignore new file mode 100644 index 000000000..347e252ef --- /dev/null +++ b/examples/llama.android/.gitignore @@ -0,0 +1,33 @@ +# Gradle files +.gradle/ +build/ + +# Local configuration file (sdk path, etc) +local.properties + +# Log/OS Files +*.log + +# Android Studio generated files and folders +captures/ +.externalNativeBuild/ +.cxx/ +*.apk +output.json + +# IntelliJ +*.iml +.idea/ +misc.xml +deploymentTargetDropDown.xml +render.experimental.xml + +# Keystore files +*.jks +*.keystore + +# Google Services (e.g. APIs or Firebase) +google-services.json + +# Android Profiling +*.hprof diff --git a/examples/llama.android/README.md b/examples/llama.android/README.md new file mode 100644 index 000000000..e69de29bb diff --git a/examples/llama.android/app/.gitignore b/examples/llama.android/app/.gitignore new file mode 100644 index 000000000..796b96d1c --- /dev/null +++ b/examples/llama.android/app/.gitignore @@ -0,0 +1 @@ +/build diff --git a/examples/llama.android/app/build.gradle.kts b/examples/llama.android/app/build.gradle.kts new file mode 100644 index 000000000..7815a8025 --- /dev/null +++ b/examples/llama.android/app/build.gradle.kts @@ -0,0 +1,91 @@ +plugins { + id("com.android.application") + id("org.jetbrains.kotlin.android") +} + +android { + namespace = "com.example.llama" + compileSdk = 34 + + ndkVersion = "26.1.10909125" + + defaultConfig { + applicationId = "com.example.llama" + minSdk = 33 + targetSdk = 34 + versionCode = 1 + versionName = "1.0" + + testInstrumentationRunner = "androidx.test.runner.AndroidJUnitRunner" + vectorDrawables { + useSupportLibrary = true + } + ndk { + // Workaround for https://github.com/llvm/llvm-project/issues/65820 + // affecting armeabi-v7a. Skip armeabi-v7a when invoked with + // -Pskip-armeabi-v7a (e.g., ./gradlew build -Pskip-armeabi-v7a). + if (project.hasProperty("skip-armeabi-v7a")) { + abiFilters += listOf("arm64-v8a", "x86_64", "x86") + } + } + externalNativeBuild { + cmake { + cppFlags += listOf() + arguments += listOf() + } + } + } + + buildTypes { + release { + isMinifyEnabled = false + proguardFiles( + getDefaultProguardFile("proguard-android-optimize.txt"), + "proguard-rules.pro" + ) + } + } + compileOptions { + sourceCompatibility = JavaVersion.VERSION_1_8 + targetCompatibility = JavaVersion.VERSION_1_8 + } + kotlinOptions { + jvmTarget = "1.8" + } + buildFeatures { + compose = true + } + composeOptions { + kotlinCompilerExtensionVersion = "1.5.1" + } + packaging { + resources { + excludes += "/META-INF/{AL2.0,LGPL2.1}" + } + } + externalNativeBuild { + cmake { + path = file("src/main/cpp/CMakeLists.txt") + version = "3.22.1" + } + } +} + +dependencies { + + implementation("androidx.core:core-ktx:1.12.0") + implementation("androidx.lifecycle:lifecycle-runtime-ktx:2.6.2") + implementation("androidx.activity:activity-compose:1.8.2") + implementation(platform("androidx.compose:compose-bom:2023.08.00")) + implementation("androidx.compose.ui:ui") + implementation("androidx.compose.ui:ui-graphics") + implementation("androidx.compose.ui:ui-tooling-preview") + implementation("androidx.compose.material3:material3") + testImplementation("junit:junit:4.13.2") + androidTestImplementation("androidx.test.ext:junit:1.1.5") + androidTestImplementation("androidx.test.espresso:espresso-core:3.5.1") + androidTestImplementation(platform("androidx.compose:compose-bom:2023.08.00")) + androidTestImplementation("androidx.compose.ui:ui-test-junit4") + debugImplementation("androidx.compose.ui:ui-tooling") + debugImplementation("androidx.compose.ui:ui-test-manifest") +} diff --git a/examples/llama.android/app/proguard-rules.pro b/examples/llama.android/app/proguard-rules.pro new file mode 100644 index 000000000..f1b424510 --- /dev/null +++ b/examples/llama.android/app/proguard-rules.pro @@ -0,0 +1,21 @@ +# Add project specific ProGuard rules here. +# You can control the set of applied configuration files using the +# proguardFiles setting in build.gradle. +# +# For more details, see +# http://developer.android.com/guide/developing/tools/proguard.html + +# If your project uses WebView with JS, uncomment the following +# and specify the fully qualified class name to the JavaScript interface +# class: +#-keepclassmembers class fqcn.of.javascript.interface.for.webview { +# public *; +#} + +# Uncomment this to preserve the line number information for +# debugging stack traces. +#-keepattributes SourceFile,LineNumberTable + +# If you keep the line number information, uncomment this to +# hide the original source file name. +#-renamesourcefileattribute SourceFile diff --git a/examples/llama.android/app/src/main/AndroidManifest.xml b/examples/llama.android/app/src/main/AndroidManifest.xml new file mode 100644 index 000000000..41a358a29 --- /dev/null +++ b/examples/llama.android/app/src/main/AndroidManifest.xml @@ -0,0 +1,30 @@ + + + + + + + + + + + + + + + + + diff --git a/examples/llama.android/app/src/main/cpp/CMakeLists.txt b/examples/llama.android/app/src/main/cpp/CMakeLists.txt new file mode 100644 index 000000000..85139329a --- /dev/null +++ b/examples/llama.android/app/src/main/cpp/CMakeLists.txt @@ -0,0 +1,50 @@ + +# For more information about using CMake with Android Studio, read the +# documentation: https://d.android.com/studio/projects/add-native-code.html. +# For more examples on how to use CMake, see https://github.com/android/ndk-samples. + +# Sets the minimum CMake version required for this project. +cmake_minimum_required(VERSION 3.22.1) + +# Declares the project name. The project name can be accessed via ${ PROJECT_NAME}, +# Since this is the top level CMakeLists.txt, the project name is also accessible +# with ${CMAKE_PROJECT_NAME} (both CMake variables are in-sync within the top level +# build script scope). +project("llama-android") + +include(FetchContent) +FetchContent_Declare( + llama + GIT_REPOSITORY https://github.com/ggerganov/llama.cpp + GIT_TAG master +) + +# Also provides "common" +FetchContent_MakeAvailable(llama) + +# Creates and names a library, sets it as either STATIC +# or SHARED, and provides the relative paths to its source code. +# You can define multiple libraries, and CMake builds them for you. +# Gradle automatically packages shared libraries with your APK. +# +# In this top level CMakeLists.txt, ${CMAKE_PROJECT_NAME} is used to define +# the target library name; in the sub-module's CMakeLists.txt, ${PROJECT_NAME} +# is preferred for the same purpose. +# +# In order to load a library into your app from Java/Kotlin, you must call +# System.loadLibrary() and pass the name of the library defined here; +# for GameActivity/NativeActivity derived applications, the same library name must be +# used in the AndroidManifest.xml file. +add_library(${CMAKE_PROJECT_NAME} SHARED + # List C/C++ source files with relative paths to this CMakeLists.txt. + llama-android.cpp) + +# Specifies libraries CMake should link to your target library. You +# can link libraries from various origins, such as libraries defined in this +# build script, prebuilt third-party libraries, or Android system libraries. +target_link_libraries(${CMAKE_PROJECT_NAME} + # List libraries link to the target library + llama + common + android + log) diff --git a/examples/llama.android/app/src/main/cpp/llama-android.cpp b/examples/llama.android/app/src/main/cpp/llama-android.cpp new file mode 100644 index 000000000..d5e705dce --- /dev/null +++ b/examples/llama.android/app/src/main/cpp/llama-android.cpp @@ -0,0 +1,394 @@ +#include +#include +#include +#include +#include +#include +#include "llama.h" +#include "common/common.h" + +// Write C++ code here. +// +// Do not forget to dynamically load the C++ library into your application. +// +// For instance, +// +// In MainActivity.java: +// static { +// System.loadLibrary("llama-android"); +// } +// +// Or, in MainActivity.kt: +// companion object { +// init { +// System.loadLibrary("llama-android") +// } +// } + +#define TAG "llama-android.cpp" +#define LOGi(...) __android_log_print(ANDROID_LOG_INFO, TAG, __VA_ARGS__) +#define LOGe(...) __android_log_print(ANDROID_LOG_ERROR, TAG, __VA_ARGS__) + +jclass la_int_var; +jmethodID la_int_var_value; +jmethodID la_int_var_inc; + +static void log_callback(ggml_log_level level, const char * fmt, void * data) { + if (level == GGML_LOG_LEVEL_ERROR) __android_log_print(ANDROID_LOG_ERROR, TAG, fmt, data); + else if (level == GGML_LOG_LEVEL_INFO) __android_log_print(ANDROID_LOG_INFO, TAG, fmt, data); + else if (level == GGML_LOG_LEVEL_WARN) __android_log_print(ANDROID_LOG_WARN, TAG, fmt, data); + else __android_log_print(ANDROID_LOG_DEFAULT, TAG, fmt, data); +} + +extern "C" +JNIEXPORT jlong JNICALL +Java_com_example_llama_Llm_load_1model(JNIEnv *env, jobject, jstring filename) { + llama_model_params model_params = llama_model_default_params(); + + auto path_to_model = env->GetStringUTFChars(filename, 0); + LOGi("Loading model from %s", path_to_model); + + auto model = llama_load_model_from_file(path_to_model, model_params); + env->ReleaseStringUTFChars(filename, path_to_model); + + if (!model) { + LOGe("load_model() failed"); + env->ThrowNew(env->FindClass("java/lang/IllegalStateException"), "load_model() failed"); + return 0; + } + + return reinterpret_cast(model); +} + +extern "C" +JNIEXPORT void JNICALL +Java_com_example_llama_Llm_free_1model(JNIEnv *, jobject, jlong model) { + llama_free_model(reinterpret_cast(model)); +} + +extern "C" +JNIEXPORT jlong JNICALL +Java_com_example_llama_Llm_new_1context(JNIEnv *env, jobject, jlong jmodel) { + auto model = reinterpret_cast(jmodel); + + if (!model) { + LOGe("new_context(): model cannot be null"); + env->ThrowNew(env->FindClass("java/lang/IllegalArgumentException"), "Model cannot be null"); + return 0; + } + + int n_threads = std::max(1, std::min(8, (int) sysconf(_SC_NPROCESSORS_ONLN) - 2)); + LOGi("Using %d threads", n_threads); + + llama_context_params ctx_params = llama_context_default_params(); + ctx_params.seed = 1234; + ctx_params.n_ctx = 2048; + ctx_params.n_threads = n_threads; + ctx_params.n_threads_batch = n_threads; + + llama_context * context = llama_new_context_with_model(model, ctx_params); + + if (!context) { + LOGe("llama_new_context_with_model() returned null)"); + env->ThrowNew(env->FindClass("java/lang/IllegalStateException"), + "llama_new_context_with_model() returned null)"); + return 0; + } + + return reinterpret_cast(context); +} + +extern "C" +JNIEXPORT void JNICALL +Java_com_example_llama_Llm_free_1context(JNIEnv *, jobject, jlong context) { + llama_free(reinterpret_cast(context)); +} + +extern "C" +JNIEXPORT void JNICALL +Java_com_example_llama_Llm_backend_1free(JNIEnv *, jobject) { + llama_backend_free(); +} + +extern "C" +JNIEXPORT void JNICALL +Java_com_example_llama_Llm_log_1to_1android(JNIEnv *, jobject) { + llama_log_set(log_callback, NULL); +} + +extern "C" +JNIEXPORT jstring JNICALL +Java_com_example_llama_Llm_bench_1model( + JNIEnv *env, + jobject, + jlong context_pointer, + jlong model_pointer, + jlong batch_pointer, + jint pp, + jint tg, + jint pl, + jint nr + ) { + auto pp_avg = 0.0; + auto tg_avg = 0.0; + auto pp_std = 0.0; + auto tg_std = 0.0; + + const auto context = reinterpret_cast(context_pointer); + const auto model = reinterpret_cast(model_pointer); + const auto batch = reinterpret_cast(batch_pointer); + + const int n_ctx = llama_n_ctx(context); + + LOGi("n_ctx = %d", n_ctx); + + int i, j; + int nri; + for (nri = 0; nri < nr; nri++) { + LOGi("Benchmark prompt processing (pp)"); + + llama_batch_clear(*batch); + + const int n_tokens = pp; + for (i = 0; i < n_tokens; i++) { + llama_batch_add(*batch, 0, i, { 0 }, false); + } + + batch->logits[batch->n_tokens - 1] = true; + llama_kv_cache_clear(context); + + const auto t_pp_start = ggml_time_us(); + if (llama_decode(context, *batch) != 0) { + LOGi("llama_decode() failed during prompt processing"); + } + const auto t_pp_end = ggml_time_us(); + + // bench text generation + + LOGi("Benchmark text generation (tg)"); + + llama_kv_cache_clear(context); + const auto t_tg_start = ggml_time_us(); + for (i = 0; i < tg; i++) { + + llama_batch_clear(*batch); + for (j = 0; j < pl; j++) { + llama_batch_add(*batch, 0, i, { j }, true); + } + + LOGi("llama_decode() text generation: %d", i); + if (llama_decode(context, *batch) != 0) { + LOGi("llama_decode() failed during text generation"); + } + } + + const auto t_tg_end = ggml_time_us(); + + llama_kv_cache_clear(context); + + const auto t_pp = double(t_pp_end - t_pp_start) / 1000000.0; + const auto t_tg = double(t_tg_end - t_tg_start) / 1000000.0; + + const auto speed_pp = double(pp) / t_pp; + const auto speed_tg = double(pl * tg) / t_tg; + + pp_avg += speed_pp; + tg_avg += speed_tg; + + pp_std += speed_pp * speed_pp; + tg_std += speed_tg * speed_tg; + + LOGi("pp %f t/s, tg %f t/s", speed_pp, speed_tg); + } + + pp_avg /= double(nr); + tg_avg /= double(nr); + + if (nr > 1) { + pp_std = sqrt(pp_std / double(nr - 1) - pp_avg * pp_avg * double(nr) / double(nr - 1)); + tg_std = sqrt(tg_std / double(nr - 1) - tg_avg * tg_avg * double(nr) / double(nr - 1)); + } else { + pp_std = 0; + tg_std = 0; + } + + char model_desc[128]; + llama_model_desc(model, model_desc, sizeof(model_desc)); + + const auto model_size = double(llama_model_size(model)) / 1024.0 / 1024.0 / 1024.0; + const auto model_n_params = double(llama_model_n_params(model)) / 1e9; + + const auto backend = "(Android)"; // TODO: What should this be? + + std::stringstream result; + result << std::setprecision(2); + result << "| model | size | params | backend | test | t/s |\n"; + result << "| --- | --- | --- | --- | --- | --- |\n"; + result << "| " << model_desc << " | " << model_size << "GiB | " << model_n_params << "B | " << backend << " | pp " << pp << " | " << pp_avg << " ± " << pp_std << " |\n"; + result << "| " << model_desc << " | " << model_size << "GiB | " << model_n_params << "B | " << backend << " | tg " << tg << " | " << tg_avg << " ± " << tg_std << " |\n"; + + return env->NewStringUTF(result.str().c_str()); +} + +extern "C" +JNIEXPORT void JNICALL +Java_com_example_llama_Llm_free_1batch(JNIEnv *, jobject, jlong batch_pointer) { + llama_batch_free(*reinterpret_cast(batch_pointer)); +} + +extern "C" +JNIEXPORT jlong JNICALL +Java_com_example_llama_Llm_new_1batch(JNIEnv *, jobject, jint n_tokens, jint embd, jint n_seq_max) { + + // Source: Copy of llama.cpp:llama_batch_init but heap-allocated. + + llama_batch *batch = new llama_batch { + 0, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + 0, + 0, + 0, + }; + + if (embd) { + batch->embd = (float *) malloc(sizeof(float) * n_tokens * embd); + } else { + batch->token = (llama_token *) malloc(sizeof(llama_token) * n_tokens); + } + + batch->pos = (llama_pos *) malloc(sizeof(llama_pos) * n_tokens); + batch->n_seq_id = (int32_t *) malloc(sizeof(int32_t) * n_tokens); + batch->seq_id = (llama_seq_id **) malloc(sizeof(llama_seq_id *) * n_tokens); + for (int i = 0; i < n_tokens; ++i) { + batch->seq_id[i] = (llama_seq_id *) malloc(sizeof(llama_seq_id) * n_seq_max); + } + batch->logits = (int8_t *) malloc(sizeof(int8_t) * n_tokens); + + return reinterpret_cast(batch); +} + +extern "C" +JNIEXPORT void JNICALL +Java_com_example_llama_Llm_backend_1init(JNIEnv *, jobject, jboolean numa) { + llama_backend_init(numa); +} + +extern "C" +JNIEXPORT jstring JNICALL +Java_com_example_llama_Llm_system_1info(JNIEnv *env, jobject) { + return env->NewStringUTF(llama_print_system_info()); +} + +extern "C" +JNIEXPORT jint JNICALL +Java_com_example_llama_Llm_completion_1init( + JNIEnv *env, + jobject, + jlong context_pointer, + jlong batch_pointer, + jstring jtext, + jint n_len + ) { + + const auto text = env->GetStringUTFChars(jtext, 0); + const auto context = reinterpret_cast(context_pointer); + const auto batch = reinterpret_cast(batch_pointer); + + const auto tokens_list = llama_tokenize(context, text, 1); + + auto n_ctx = llama_n_ctx(context); + auto n_kv_req = tokens_list.size() + (n_len - tokens_list.size()); + + LOGi("n_len = %d, n_ctx = %d, n_kv_req = %d", n_len, n_ctx, n_kv_req); + + if (n_kv_req > n_ctx) { + LOGe("error: n_kv_req > n_ctx, the required KV cache size is not big enough"); + } + + for (auto id : tokens_list) { + LOGi("%s", llama_token_to_piece(context, id).c_str()); + } + + llama_batch_clear(*batch); + + // evaluate the initial prompt + for (auto i = 0; i < tokens_list.size(); i++) { + llama_batch_add(*batch, tokens_list[i], i, { 0 }, false); + } + + // llama_decode will output logits only for the last token of the prompt + batch->logits[batch->n_tokens - 1] = true; + + if (llama_decode(context, *batch) != 0) { + LOGe("llama_decode() failed"); + } + + env->ReleaseStringUTFChars(jtext, text); + + return batch->n_tokens; +} + +extern "C" +JNIEXPORT jstring JNICALL +Java_com_example_llama_Llm_completion_1loop( + JNIEnv * env, + jobject, + jlong context_pointer, + jlong batch_pointer, + jint n_len, + jobject intvar_ncur +) { + const auto context = reinterpret_cast(context_pointer); + const auto batch = reinterpret_cast(batch_pointer); + const auto model = llama_get_model(context); + + if (!la_int_var) la_int_var = env->GetObjectClass(intvar_ncur); + if (!la_int_var_value) la_int_var_value = env->GetMethodID(la_int_var, "getValue", "()I"); + if (!la_int_var_inc) la_int_var_inc = env->GetMethodID(la_int_var, "inc", "()V"); + + auto n_vocab = llama_n_vocab(model); + auto logits = llama_get_logits_ith(context, batch->n_tokens - 1); + + std::vector candidates; + candidates.reserve(n_vocab); + + for (llama_token token_id = 0; token_id < n_vocab; token_id++) { + candidates.emplace_back(llama_token_data{ token_id, logits[token_id], 0.0f }); + } + + llama_token_data_array candidates_p = { candidates.data(), candidates.size(), false }; + + // sample the most likely token + const auto new_token_id = llama_sample_token_greedy(context, &candidates_p); + + const auto n_cur = env->CallIntMethod(intvar_ncur, la_int_var_value); + if (new_token_id == llama_token_eos(model) || n_cur == n_len) { + return env->NewStringUTF(""); + } + + auto new_token_chars = llama_token_to_piece(context, new_token_id); + LOGi("new_token_chars: `%s`", new_token_chars.c_str()); + auto new_token = env->NewStringUTF(new_token_chars.c_str()); + + llama_batch_clear(*batch); + llama_batch_add(*batch, new_token_id, n_cur, { 0 }, true); + + env->CallVoidMethod(intvar_ncur, la_int_var_inc); + + if (llama_decode(context, *batch) != 0) { + LOGe("llama_decode() returned null"); + } + + return new_token; +} + +extern "C" +JNIEXPORT void JNICALL +Java_com_example_llama_Llm_kv_1cache_1clear(JNIEnv *, jobject, jlong context) { + llama_kv_cache_clear(reinterpret_cast(context)); +} diff --git a/examples/llama.android/app/src/main/java/com/example/llama/Downloadable.kt b/examples/llama.android/app/src/main/java/com/example/llama/Downloadable.kt new file mode 100644 index 000000000..78c231ae5 --- /dev/null +++ b/examples/llama.android/app/src/main/java/com/example/llama/Downloadable.kt @@ -0,0 +1,119 @@ +package com.example.llama + +import android.app.DownloadManager +import android.net.Uri +import android.util.Log +import androidx.compose.material3.Button +import androidx.compose.material3.Text +import androidx.compose.runtime.Composable +import androidx.compose.runtime.getValue +import androidx.compose.runtime.mutableDoubleStateOf +import androidx.compose.runtime.mutableStateOf +import androidx.compose.runtime.remember +import androidx.compose.runtime.rememberCoroutineScope +import androidx.compose.runtime.setValue +import androidx.core.database.getLongOrNull +import androidx.core.net.toUri +import kotlinx.coroutines.delay +import kotlinx.coroutines.launch +import java.io.File + +data class Downloadable(val name: String, val source: Uri, val destination: File) { + companion object { + @JvmStatic + private val tag: String? = this::class.qualifiedName + + sealed interface State + data object Ready: State + data class Downloading(val id: Long): State + data class Downloaded(val downloadable: Downloadable): State + data class Error(val message: String): State + + @JvmStatic + @Composable + fun Button(viewModel: MainViewModel, dm: DownloadManager, item: Downloadable) { + var status: State by remember { + mutableStateOf( + if (item.destination.exists()) Downloaded(item) + else Ready + ) + } + var progress by remember { mutableDoubleStateOf(0.0) } + + val coroutineScope = rememberCoroutineScope() + + suspend fun waitForDownload(result: Downloading, item: Downloadable): State { + while (true) { + val cursor = dm.query(DownloadManager.Query().setFilterById(result.id)) + + if (cursor == null) { + Log.e(tag, "dm.query() returned null") + return Error("dm.query() returned null") + } + + if (!cursor.moveToFirst() || cursor.count < 1) { + cursor.close() + Log.i(tag, "cursor.moveToFirst() returned false or cursor.count < 1, download canceled?") + return Ready + } + + val pix = cursor.getColumnIndex(DownloadManager.COLUMN_BYTES_DOWNLOADED_SO_FAR) + val tix = cursor.getColumnIndex(DownloadManager.COLUMN_TOTAL_SIZE_BYTES) + val sofar = cursor.getLongOrNull(pix) ?: 0 + val total = cursor.getLongOrNull(tix) ?: 1 + cursor.close() + + if (sofar == total) { + return Downloaded(item) + } + + progress = (sofar * 1.0) / total + + delay(1000L) + } + } + + fun onClick() { + when (val s = status) { + is Downloaded -> { + viewModel.load(item.destination.path) + } + + is Downloading -> { + coroutineScope.launch { + status = waitForDownload(s, item) + } + } + + else -> { + item.destination.delete() + + val request = DownloadManager.Request(item.source).apply { + setTitle("Downloading model") + setDescription("Downloading model: ${item.name}") + setAllowedNetworkTypes(DownloadManager.Request.NETWORK_WIFI) + setDestinationUri(item.destination.toUri()) + } + + viewModel.log("Saving ${item.name} to ${item.destination.path}") + Log.i(tag, "Saving ${item.name} to ${item.destination.path}") + + val id = dm.enqueue(request) + status = Downloading(id) + onClick() + } + } + } + + Button(onClick = { onClick() }, enabled = status !is Downloading) { + when (status) { + is Downloading -> Text(text = "Downloading ${(progress * 100).toInt()}%") + is Downloaded -> Text("Load ${item.name}") + is Ready -> Text("Download ${item.name}") + is Error -> Text("Download ${item.name}") + } + } + } + + } +} diff --git a/examples/llama.android/app/src/main/java/com/example/llama/Llm.kt b/examples/llama.android/app/src/main/java/com/example/llama/Llm.kt new file mode 100644 index 000000000..5f3270372 --- /dev/null +++ b/examples/llama.android/app/src/main/java/com/example/llama/Llm.kt @@ -0,0 +1,172 @@ +package com.example.llama + +import android.util.Log +import kotlinx.coroutines.CoroutineDispatcher +import kotlinx.coroutines.asCoroutineDispatcher +import kotlinx.coroutines.flow.Flow +import kotlinx.coroutines.flow.flow +import kotlinx.coroutines.flow.flowOn +import kotlinx.coroutines.withContext +import java.util.concurrent.Executors +import kotlin.concurrent.thread + +class Llm { + private val tag: String? = this::class.simpleName + + private val threadLocalState: ThreadLocal = ThreadLocal.withInitial { State.Idle } + + private val runLoop: CoroutineDispatcher = Executors.newSingleThreadExecutor { + thread(start = false, name = "Llm-RunLoop") { + Log.d(tag, "Dedicated thread for native code: ${Thread.currentThread().name}") + + // No-op if called more than once. + System.loadLibrary("llama-android") + + // Set llama log handler to Android + log_to_android() + backend_init(false) + + Log.d(tag, system_info()) + + it.run() + }.apply { + uncaughtExceptionHandler = Thread.UncaughtExceptionHandler { _, exception: Throwable -> + Log.e(tag, "Unhandled exception", exception) + } + } + }.asCoroutineDispatcher() + + private val nlen: Int = 64 + + private external fun log_to_android() + private external fun load_model(filename: String): Long + private external fun free_model(model: Long) + private external fun new_context(model: Long): Long + private external fun free_context(context: Long) + private external fun backend_init(numa: Boolean) + private external fun backend_free() + private external fun free_batch(batch: Long) + private external fun new_batch(nTokens: Int, embd: Int, nSeqMax: Int): Long + private external fun bench_model( + context: Long, + model: Long, + batch: Long, + pp: Int, + tg: Int, + pl: Int, + nr: Int + ): String + + private external fun system_info(): String + + private external fun completion_init( + context: Long, + batch: Long, + text: String, + nLen: Int + ): Int + + private external fun completion_loop( + context: Long, + batch: Long, + nLen: Int, + ncur: IntVar + ): String + + private external fun kv_cache_clear(context: Long) + + suspend fun bench(pp: Int, tg: Int, pl: Int, nr: Int = 1): String { + return withContext(runLoop) { + when (val state = threadLocalState.get()) { + is State.Loaded -> { + Log.d(tag, "bench(): $state") + bench_model(state.context, state.model, state.batch, pp, tg, pl, nr) + } + + else -> throw IllegalStateException("No model loaded") + } + } + } + + suspend fun load(pathToModel: String) { + withContext(runLoop) { + when (threadLocalState.get()) { + is State.Idle -> { + val model = load_model(pathToModel) + if (model == 0L) throw IllegalStateException("load_model() failed") + + val context = new_context(model) + if (context == 0L) throw IllegalStateException("new_context() failed") + + val batch = new_batch(512, 0, 1) + if (batch == 0L) throw IllegalStateException("new_batch() failed") + + Log.i(tag, "Loaded model $pathToModel") + threadLocalState.set(State.Loaded(model, context, batch)) + } + else -> throw IllegalStateException("Model already loaded") + } + } + } + + fun send(message: String): Flow = flow { + when (val state = threadLocalState.get()) { + is State.Loaded -> { + val ncur = IntVar(completion_init(state.context, state.batch, message, nlen)) + while (ncur.value <= nlen) { + val str = completion_loop(state.context, state.batch, nlen, ncur) + if (str.isEmpty()) { + break + } + emit(str) + } + kv_cache_clear(state.context) + } + else -> {} + } + }.flowOn(runLoop) + + /** + * Unloads the model and frees resources. + * + * This is a no-op if there's no model loaded. + */ + suspend fun unload() { + withContext(runLoop) { + when (val state = threadLocalState.get()) { + is State.Loaded -> { + free_context(state.context) + free_model(state.model) + free_batch(state.batch) + + threadLocalState.set(State.Idle) + } + else -> {} + } + } + } + + companion object { + private class IntVar(value: Int) { + @Volatile + var value: Int = value + private set + + fun inc() { + synchronized(this) { + value += 1 + } + } + } + + private sealed interface State { + data object Idle: State + data class Loaded(val model: Long, val context: Long, val batch: Long): State + } + + // Enforce only one instance of Llm. + private val _instance: Llm = Llm() + + fun instance(): Llm = _instance + } +} diff --git a/examples/llama.android/app/src/main/java/com/example/llama/MainActivity.kt b/examples/llama.android/app/src/main/java/com/example/llama/MainActivity.kt new file mode 100644 index 000000000..9da04f7d3 --- /dev/null +++ b/examples/llama.android/app/src/main/java/com/example/llama/MainActivity.kt @@ -0,0 +1,154 @@ +package com.example.llama + +import android.app.ActivityManager +import android.app.DownloadManager +import android.content.ClipData +import android.content.ClipboardManager +import android.net.Uri +import android.os.Bundle +import android.os.StrictMode +import android.os.StrictMode.VmPolicy +import android.text.format.Formatter +import androidx.activity.ComponentActivity +import androidx.activity.compose.setContent +import androidx.activity.viewModels +import androidx.compose.foundation.layout.Box +import androidx.compose.foundation.layout.Column +import androidx.compose.foundation.layout.Row +import androidx.compose.foundation.layout.fillMaxSize +import androidx.compose.foundation.layout.padding +import androidx.compose.foundation.lazy.LazyColumn +import androidx.compose.foundation.lazy.items +import androidx.compose.foundation.lazy.rememberLazyListState +import androidx.compose.material3.Button +import androidx.compose.material3.LocalContentColor +import androidx.compose.material3.MaterialTheme +import androidx.compose.material3.OutlinedTextField +import androidx.compose.material3.Surface +import androidx.compose.material3.Text +import androidx.compose.runtime.Composable +import androidx.compose.ui.Modifier +import androidx.compose.ui.unit.dp +import androidx.core.content.getSystemService +import com.example.llama.ui.theme.LlamaAndroidTheme +import java.io.File + +class MainActivity( + activityManager: ActivityManager? = null, + downloadManager: DownloadManager? = null, + clipboardManager: ClipboardManager? = null, +): ComponentActivity() { + private val tag: String? = this::class.simpleName + + private val activityManager by lazy { activityManager ?: getSystemService()!! } + private val downloadManager by lazy { downloadManager ?: getSystemService()!! } + private val clipboardManager by lazy { clipboardManager ?: getSystemService()!! } + + private val viewModel: MainViewModel by viewModels() + + // Get a MemoryInfo object for the device's current memory status. + private fun availableMemory(): ActivityManager.MemoryInfo { + return ActivityManager.MemoryInfo().also { memoryInfo -> + activityManager.getMemoryInfo(memoryInfo) + } + } + + override fun onCreate(savedInstanceState: Bundle?) { + super.onCreate(savedInstanceState) + + StrictMode.setVmPolicy( + VmPolicy.Builder(StrictMode.getVmPolicy()) + .detectLeakedClosableObjects() + .build() + ) + + val free = Formatter.formatFileSize(this, availableMemory().availMem) + val total = Formatter.formatFileSize(this, availableMemory().totalMem) + + viewModel.log("Current memory: $free / $total") + viewModel.log("Downloads directory: ${getExternalFilesDir(null)}") + + val extFilesDir = getExternalFilesDir(null) + + val models = listOf( + Downloadable( + "Phi-2 7B (Q4_0, 1.6 GiB)", + Uri.parse("https://huggingface.co/ggml-org/models/resolve/main/phi-2/ggml-model-q4_0.gguf?download=true"), + File(extFilesDir, "phi-2-q4_0.gguf"), + ), + Downloadable( + "TinyLlama 1.1B (f16, 2.2 GiB)", + Uri.parse("https://huggingface.co/ggml-org/models/resolve/main/tinyllama-1.1b/ggml-model-f16.gguf?download=true"), + File(extFilesDir, "tinyllama-1.1-f16.gguf"), + ), + Downloadable( + "Phi 2 DPO (Q3_K_M, 1.48 GiB)", + Uri.parse("https://huggingface.co/TheBloke/phi-2-dpo-GGUF/resolve/main/phi-2-dpo.Q3_K_M.gguf?download=true"), + File(extFilesDir, "phi-2-dpo.Q3_K_M.gguf") + ), + ) + + setContent { + LlamaAndroidTheme { + // A surface container using the 'background' color from the theme + Surface( + modifier = Modifier.fillMaxSize(), + color = MaterialTheme.colorScheme.background + ) { + MainCompose( + viewModel, + clipboardManager, + downloadManager, + models, + ) + } + + } + } + } +} + +@Composable +fun MainCompose( + viewModel: MainViewModel, + clipboard: ClipboardManager, + dm: DownloadManager, + models: List +) { + Column { + val scrollState = rememberLazyListState() + + Box(modifier = Modifier.weight(1f)) { + LazyColumn(state = scrollState) { + items(viewModel.messages) { + Text( + it, + style = MaterialTheme.typography.bodyLarge.copy(color = LocalContentColor.current), + modifier = Modifier.padding(16.dp) + ) + } + } + } + OutlinedTextField( + value = viewModel.message, + onValueChange = { viewModel.updateMessage(it) }, + label = { Text("Message") }, + ) + Row { + Button({ viewModel.send() }) { Text("Send") } + Button({ viewModel.bench(8, 4, 1) }) { Text("Bench") } + Button({ viewModel.clear() }) { Text("Clear") } + Button({ + viewModel.messages.joinToString("\n").let { + clipboard.setPrimaryClip(ClipData.newPlainText("", it)) + } + }) { Text("Copy") } + } + + Column { + for (model in models) { + Downloadable.Button(viewModel, dm, model) + } + } + } +} diff --git a/examples/llama.android/app/src/main/java/com/example/llama/MainViewModel.kt b/examples/llama.android/app/src/main/java/com/example/llama/MainViewModel.kt new file mode 100644 index 000000000..be95e2221 --- /dev/null +++ b/examples/llama.android/app/src/main/java/com/example/llama/MainViewModel.kt @@ -0,0 +1,104 @@ +package com.example.llama + +import android.util.Log +import androidx.compose.runtime.getValue +import androidx.compose.runtime.mutableStateOf +import androidx.compose.runtime.setValue +import androidx.lifecycle.ViewModel +import androidx.lifecycle.viewModelScope +import kotlinx.coroutines.flow.catch +import kotlinx.coroutines.launch + +class MainViewModel(private val llm: Llm = Llm.instance()): ViewModel() { + companion object { + @JvmStatic + private val NanosPerSecond = 1_000_000_000.0 + } + + private val tag: String? = this::class.simpleName + + var messages by mutableStateOf(listOf("Initializing...")) + private set + + var message by mutableStateOf("") + private set + + override fun onCleared() { + super.onCleared() + + viewModelScope.launch { + try { + llm.unload() + } catch (exc: IllegalStateException) { + messages += exc.message!! + } + } + } + + fun send() { + val text = message + message = "" + + // Add to messages console. + messages += text + messages += "" + + viewModelScope.launch { + llm.send(text) + .catch { + Log.e(tag, "send() failed", it) + messages += it.message!! + } + .collect { messages = messages.dropLast(1) + (messages.last() + it) } + } + } + + fun bench(pp: Int, tg: Int, pl: Int, nr: Int = 1) { + viewModelScope.launch { + try { + val start = System.nanoTime() + val warmupResult = llm.bench(pp, tg, pl, nr) + val end = System.nanoTime() + + messages += warmupResult + + val warmup = (end - start).toDouble() / NanosPerSecond + messages += "Warm up time: $warmup seconds, please wait..." + + if (warmup > 5.0) { + messages += "Warm up took too long, aborting benchmark" + return@launch + } + + messages += llm.bench(512, 128, 1, 3) + } catch (exc: IllegalStateException) { + Log.e(tag, "bench() failed", exc) + messages += exc.message!! + } + } + } + + fun load(pathToModel: String) { + viewModelScope.launch { + try { + llm.load(pathToModel) + messages += "Loaded $pathToModel" + } catch (exc: IllegalStateException) { + Log.e(tag, "load() failed", exc) + messages += exc.message!! + } + } + } + + fun updateMessage(newMessage: String) { + message = newMessage + } + + fun clear() { + messages = listOf() + } + + fun log(message: String) { + messages += message + } +} diff --git a/examples/llama.android/app/src/main/java/com/example/llama/ui/theme/Color.kt b/examples/llama.android/app/src/main/java/com/example/llama/ui/theme/Color.kt new file mode 100644 index 000000000..40c30e8d9 --- /dev/null +++ b/examples/llama.android/app/src/main/java/com/example/llama/ui/theme/Color.kt @@ -0,0 +1,11 @@ +package com.example.llama.ui.theme + +import androidx.compose.ui.graphics.Color + +val Purple80 = Color(0xFFD0BCFF) +val PurpleGrey80 = Color(0xFFCCC2DC) +val Pink80 = Color(0xFFEFB8C8) + +val Purple40 = Color(0xFF6650a4) +val PurpleGrey40 = Color(0xFF625b71) +val Pink40 = Color(0xFF7D5260) diff --git a/examples/llama.android/app/src/main/java/com/example/llama/ui/theme/Theme.kt b/examples/llama.android/app/src/main/java/com/example/llama/ui/theme/Theme.kt new file mode 100644 index 000000000..e742220a8 --- /dev/null +++ b/examples/llama.android/app/src/main/java/com/example/llama/ui/theme/Theme.kt @@ -0,0 +1,70 @@ +package com.example.llama.ui.theme + +import android.app.Activity +import android.os.Build +import androidx.compose.foundation.isSystemInDarkTheme +import androidx.compose.material3.MaterialTheme +import androidx.compose.material3.darkColorScheme +import androidx.compose.material3.dynamicDarkColorScheme +import androidx.compose.material3.dynamicLightColorScheme +import androidx.compose.material3.lightColorScheme +import androidx.compose.runtime.Composable +import androidx.compose.runtime.SideEffect +import androidx.compose.ui.graphics.toArgb +import androidx.compose.ui.platform.LocalContext +import androidx.compose.ui.platform.LocalView +import androidx.core.view.WindowCompat + +private val DarkColorScheme = darkColorScheme( + primary = Purple80, + secondary = PurpleGrey80, + tertiary = Pink80 +) + +private val LightColorScheme = lightColorScheme( + primary = Purple40, + secondary = PurpleGrey40, + tertiary = Pink40 + + /* Other default colors to override + background = Color(0xFFFFFBFE), + surface = Color(0xFFFFFBFE), + onPrimary = Color.White, + onSecondary = Color.White, + onTertiary = Color.White, + onBackground = Color(0xFF1C1B1F), + onSurface = Color(0xFF1C1B1F), + */ +) + +@Composable +fun LlamaAndroidTheme( + darkTheme: Boolean = isSystemInDarkTheme(), + // Dynamic color is available on Android 12+ + dynamicColor: Boolean = true, + content: @Composable () -> Unit +) { + val colorScheme = when { + dynamicColor && Build.VERSION.SDK_INT >= Build.VERSION_CODES.S -> { + val context = LocalContext.current + if (darkTheme) dynamicDarkColorScheme(context) else dynamicLightColorScheme(context) + } + + darkTheme -> DarkColorScheme + else -> LightColorScheme + } + val view = LocalView.current + if (!view.isInEditMode) { + SideEffect { + val window = (view.context as Activity).window + window.statusBarColor = colorScheme.primary.toArgb() + WindowCompat.getInsetsController(window, view).isAppearanceLightStatusBars = darkTheme + } + } + + MaterialTheme( + colorScheme = colorScheme, + typography = Typography, + content = content + ) +} diff --git a/examples/llama.android/app/src/main/java/com/example/llama/ui/theme/Type.kt b/examples/llama.android/app/src/main/java/com/example/llama/ui/theme/Type.kt new file mode 100644 index 000000000..0b87946ca --- /dev/null +++ b/examples/llama.android/app/src/main/java/com/example/llama/ui/theme/Type.kt @@ -0,0 +1,34 @@ +package com.example.llama.ui.theme + +import androidx.compose.material3.Typography +import androidx.compose.ui.text.TextStyle +import androidx.compose.ui.text.font.FontFamily +import androidx.compose.ui.text.font.FontWeight +import androidx.compose.ui.unit.sp + +// Set of Material typography styles to start with +val Typography = Typography( + bodyLarge = TextStyle( + fontFamily = FontFamily.Default, + fontWeight = FontWeight.Normal, + fontSize = 16.sp, + lineHeight = 24.sp, + letterSpacing = 0.5.sp + ) + /* Other default text styles to override + titleLarge = TextStyle( + fontFamily = FontFamily.Default, + fontWeight = FontWeight.Normal, + fontSize = 22.sp, + lineHeight = 28.sp, + letterSpacing = 0.sp + ), + labelSmall = TextStyle( + fontFamily = FontFamily.Default, + fontWeight = FontWeight.Medium, + fontSize = 11.sp, + lineHeight = 16.sp, + letterSpacing = 0.5.sp + ) + */ +) diff --git a/examples/llama.android/app/src/main/res/drawable/ic_launcher_background.xml b/examples/llama.android/app/src/main/res/drawable/ic_launcher_background.xml new file mode 100644 index 000000000..07d5da9cb --- /dev/null +++ b/examples/llama.android/app/src/main/res/drawable/ic_launcher_background.xml @@ -0,0 +1,170 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/examples/llama.android/app/src/main/res/drawable/ic_launcher_foreground.xml b/examples/llama.android/app/src/main/res/drawable/ic_launcher_foreground.xml new file mode 100644 index 000000000..7706ab9e6 --- /dev/null +++ b/examples/llama.android/app/src/main/res/drawable/ic_launcher_foreground.xml @@ -0,0 +1,30 @@ + + + + + + + + + + + diff --git a/examples/llama.android/app/src/main/res/mipmap-anydpi/ic_launcher.xml b/examples/llama.android/app/src/main/res/mipmap-anydpi/ic_launcher.xml new file mode 100644 index 000000000..b3e26b4c6 --- /dev/null +++ b/examples/llama.android/app/src/main/res/mipmap-anydpi/ic_launcher.xml @@ -0,0 +1,6 @@ + + + + + + diff --git a/examples/llama.android/app/src/main/res/mipmap-anydpi/ic_launcher_round.xml b/examples/llama.android/app/src/main/res/mipmap-anydpi/ic_launcher_round.xml new file mode 100644 index 000000000..b3e26b4c6 --- /dev/null +++ b/examples/llama.android/app/src/main/res/mipmap-anydpi/ic_launcher_round.xml @@ -0,0 +1,6 @@ + + + + + + diff --git a/examples/llama.android/app/src/main/res/mipmap-hdpi/ic_launcher.webp b/examples/llama.android/app/src/main/res/mipmap-hdpi/ic_launcher.webp new file mode 100644 index 0000000000000000000000000000000000000000..c209e78ecd372343283f4157dcfd918ec5165bb3 GIT binary patch literal 1404 zcmV-?1%vuhNk&F=1pok7MM6+kP&il$0000G0000-002h-06|PpNX!5L00Dqw+t%{r zzW2vH!KF=w&cMnnN@{whkTw+#mAh0SV?YL=)3MimFYCWp#fpdtz~8$hD5VPuQgtcN zXl<@<#Cme5f5yr2h%@8TWh?)bSK`O z^Z@d={gn7J{iyxL_y_%J|L>ep{dUxUP8a{byupH&!UNR*OutO~0{*T4q5R6@ApLF! z5{w?Z150gC7#>(VHFJZ-^6O@PYp{t!jH(_Z*nzTK4 zkc{fLE4Q3|mA2`CWQ3{8;gxGizgM!zccbdQoOLZc8hThi-IhN90RFT|zlxh3Ty&VG z?Fe{#9RrRnxzsu|Lg2ddugg7k%>0JeD+{XZ7>Z~{=|M+sh1MF7~ zz>To~`~LVQe1nNoR-gEzkpe{Ak^7{{ZBk2i_<+`Bq<^GB!RYG+z)h;Y3+<{zlMUYd zrd*W4w&jZ0%kBuDZ1EW&KLpyR7r2=}fF2%0VwHM4pUs}ZI2egi#DRMYZPek*^H9YK zay4Iy3WXFG(F14xYsoDA|KXgGc5%2DhmQ1gFCkrgHBm!lXG8I5h*uf{rn48Z!_@ z4Bk6TJAB2CKYqPjiX&mWoW>OPFGd$wqroa($ne7EUK;#3VYkXaew%Kh^3OrMhtjYN?XEoY`tRPQsAkH-DSL^QqyN0>^ zmC>{#F14jz4GeW{pJoRpLFa_*GI{?T93^rX7SPQgT@LbLqpNA}<@2wH;q493)G=1Y z#-sCiRNX~qf3KgiFzB3I>4Z%AfS(3$`-aMIBU+6?gbgDb!)L~A)je+;fR0jWLL-Fu z4)P{c7{B4Hp91&%??2$v9iRSFnuckHUm}or9seH6 z>%NbT+5*@L5(I9j@06@(!{ZI?U0=pKn8uwIg&L{JV14+8s2hnvbRrU|hZCd}IJu7*;;ECgO%8_*W Kmw_-CKmY()leWbG literal 0 HcmV?d00001 diff --git a/examples/llama.android/app/src/main/res/mipmap-hdpi/ic_launcher_round.webp b/examples/llama.android/app/src/main/res/mipmap-hdpi/ic_launcher_round.webp new file mode 100644 index 0000000000000000000000000000000000000000..b2dfe3d1ba5cf3ee31b3ecc1ced89044a1f3b7a9 GIT binary patch literal 2898 zcmV-Y3$650Nk&FW3jhFDMM6+kP&il$0000G0000-002h-06|PpNWB9900E$G+qN-D z+81ABX7q?;bwx%xBg?kcwr$(C-Tex-ZCkHUw(Y9#+`E5-zuONG5fgw~E2WDng@Bc@ z24xy+R1n%~6xI#u9vJ8zREI)sb<&Il(016}Z~V1n^PU3-_H17A*Bf^o)&{_uBv}Py zulRfeE8g(g6HFhk_?o_;0@tz?1I+l+Y#Q*;RVC?(ud`_cU-~n|AX-b`JHrOIqn(-t&rOg-o`#C zh0LPxmbOAEb;zHTu!R3LDh1QO zZTf-|lJNUxi-PpcbRjw3n~n-pG;$+dIF6eqM5+L();B2O2tQ~|p{PlpNcvDbd1l%c zLtXn%lu(3!aNK!V#+HNn_D3lp z2%l+hK-nsj|Bi9;V*WIcQRTt5j90A<=am+cc`J zTYIN|PsYAhJ|=&h*4wI4ebv-C=Be#u>}%m;a{IGmJDU`0snWS&$9zdrT(z8#{OZ_Y zxwJx!ZClUi%YJjD6Xz@OP8{ieyJB=tn?>zaI-4JN;rr`JQbb%y5h2O-?_V@7pG_+y z(lqAsqYr!NyVb0C^|uclHaeecG)Sz;WV?rtoqOdAAN{j%?Uo%owya(F&qps@Id|Of zo@~Y-(YmfB+chv^%*3g4k3R0WqvuYUIA+8^SGJ{2Bl$X&X&v02>+0$4?di(34{pt* zG=f#yMs@Y|b&=HyH3k4yP&goF2LJ#tBLJNNDo6lG06r}ghC-pC4Q*=x3;|+W04zte zAl>l4kzUBQFYF(E`KJy?ZXd1tnfbH+Z~SMmA21KokJNs#eqcXWKUIC>{TuoKe^vhF z);H)o`t9j~`$h1D`#bxe@E`oE`cM9w(@)5Bp8BNukIwM>wZHfd0S;5bcXA*5KT3bj zc&_~`&{z7u{Et!Z_k78H75gXf4g8<_ul!H$eVspPeU3j&&Au=2R*Zp#M9$9s;fqwgzfiX=E_?BwVcfx3tG9Q-+<5fw z%Hs64z)@Q*%s3_Xd5>S4dg$s>@rN^ixeVj*tqu3ZV)biDcFf&l?lGwsa zWj3rvK}?43c{IruV2L`hUU0t^MemAn3U~x3$4mFDxj=Byowu^Q+#wKRPrWywLjIAp z9*n}eQ9-gZmnd9Y0WHtwi2sn6n~?i#n9VN1B*074_VbZZ=WrpkMYr{RsI ztM_8X1)J*DZejxkjOTRJ&a*lrvMKBQURNP#K)a5wIitfu(CFYV4FT?LUB$jVwJSZz zNBFTWg->Yk0j&h3e*a5>B=-xM7dE`IuOQna!u$OoxLlE;WdrNlN)1 z7**de7-hZ!(%_ZllHBLg`Ir#|t>2$*xVOZ-ADZKTN?{(NUeLU9GbuG-+Axf*AZ-P1 z0ZZ*fx+ck4{XtFsbcc%GRStht@q!m*ImssGwuK+P@%gEK!f5dHymg<9nSCXsB6 zQ*{<`%^bxB($Z@5286^-A(tR;r+p7B%^%$N5h%lb*Vlz-?DL9x;!j<5>~kmXP$E}m zQV|7uv4SwFs0jUervsxVUm>&9Y3DBIzc1XW|CUZrUdb<&{@D5yuLe%Xniw^x&{A2s z0q1+owDSfc3Gs?ht;3jw49c#mmrViUfX-yvc_B*wY|Lo7; zGh!t2R#BHx{1wFXReX*~`NS-LpSX z#TV*miO^~B9PF%O0huw!1Zv>^d0G3$^8dsC6VI!$oKDKiXdJt{mGkyA`+Gwd4D-^1qtNTUK)`N*=NTG-6}=5k6suNfdLt*dt8D| z%H#$k)z#ZRcf|zDWB|pn<3+7Nz>?WW9WdkO5(a^m+D4WRJ9{wc>Y}IN)2Kbgn;_O? zGqdr&9~|$Y0tP=N(k7^Eu;iO*w+f%W`20BNo)=Xa@M_)+o$4LXJyiw{F?a633SC{B zl~9FH%?^Rm*LVz`lkULs)%idDX^O)SxQol(3jDRyBVR!7d`;ar+D7do)jQ}m`g$TevUD5@?*P8)voa?kEe@_hl{_h8j&5eB-5FrYW&*FHVt$ z$kRF9Nstj%KRzpjdd_9wO=4zO8ritN*NPk_9avYrsF(!4))tm{Ga#OY z(r{0buexOzu7+rw8E08Gxd`LTOID{*AC1m*6Nw@osfB%0oBF5sf<~wH1kL;sd zo)k6^VyRFU`)dt*iX^9&QtWbo6yE8XXH?`ztvpiOLgI3R+=MOBQ9=rMVgi<*CU%+d1PQQ0a1U=&b0vkF207%xU0ssI2 literal 0 HcmV?d00001 diff --git a/examples/llama.android/app/src/main/res/mipmap-mdpi/ic_launcher.webp b/examples/llama.android/app/src/main/res/mipmap-mdpi/ic_launcher.webp new file mode 100644 index 0000000000000000000000000000000000000000..4f0f1d64e58ba64d180ce43ee13bf9a17835fbca GIT binary patch literal 982 zcmV;{11bDcNk&G_0{{S5MM6+kP&il$0000G0000l001ul06|PpNU8t;00Dqo+t#w^ z^1csucXz7-Qrhzl9HuHB%l>&>1tG2^vb*E&k^T3$FG1eQZ51g$uv4V+kI`0<^1Z@N zk?Jjh$olyC%l>)Xq;7!>{iBj&BjJ`P&$fsCfpve_epJOBkTF?nu-B7D!hO=2ZR}

C%4 zc_9eOXvPbC4kzU8YowIA8cW~Uv|eB&yYwAObSwL2vY~UYI7NXPvf3b+c^?wcs~_t{ ze_m66-0)^{JdOMKPwjpQ@Sna!*?$wTZ~su*tNv7o!gXT!GRgivP}ec?5>l1!7<(rT zds|8x(qGc673zrvYIz;J23FG{9nHMnAuP}NpAED^laz3mAN1sy+NXK)!6v1FxQ;lh zOBLA>$~P3r4b*NcqR;y6pwyhZ3_PiDb|%n1gGjl3ZU}ujInlP{eks-#oA6>rh&g+!f`hv#_%JrgYPu z(U^&XLW^QX7F9Z*SRPpQl{B%x)_AMp^}_v~?j7 zapvHMKxSf*Mtyx8I}-<*UGn3)oHd(nn=)BZ`d$lDBwq_GL($_TPaS{UeevT(AJ`p0 z9%+hQb6z)U9qjbuXjg|dExCLjpS8$VKQ55VsIC%@{N5t{NsW)=hNGI`J=x97_kbz@ E0Of=7!TQj4N+cqN`nQhxvX7dAV-`K|Ub$-q+H-5I?Tx0g9jWxd@A|?POE8`3b8fO$T))xP* z(X?&brZw({`)WU&rdAs1iTa0x6F@PIxJ&&L|dpySV!ID|iUhjCcKz(@mE z!x@~W#3H<)4Ae(4eQJRk`Iz3<1)6^m)0b_4_TRZ+cz#eD3f8V;2r-1fE!F}W zEi0MEkTTx}8i1{`l_6vo0(Vuh0HD$I4SjZ=?^?k82R51bC)2D_{y8mi_?X^=U?2|F{Vr7s!k(AZC$O#ZMyavHhlQ7 zUR~QXuH~#o#>(b$u4?s~HLF*3IcF7023AlwAYudn0FV~|odGH^05AYPEfR)8p`i{n zwg3zPVp{+wOsxKc>)(pMupKF!Y2HoUqQ3|Yu|8lwR=?5zZuhG6J?H`bSNk_wPoM{u zSL{c@pY7+c2kck>`^q1^^gR0QB7Y?KUD{vz-uVX~;V-rW)PDcI)$_UjgVV?S?=oLR zf4}zz{#*R_{LkiJ#0RdQLNC^2Vp%JPEUvG9ra2BVZ92(p9h7Ka@!yf9(lj#}>+|u* z;^_?KWdzkM`6gqPo9;;r6&JEa)}R3X{(CWv?NvgLeOTq$cZXqf7|sPImi-7cS8DCN zGf;DVt3Am`>hH3{4-WzH43Ftx)SofNe^-#|0HdCo<+8Qs!}TZP{HH8~z5n`ExcHuT zDL1m&|DVpIy=xsLO>8k92HcmfSKhflQ0H~9=^-{#!I1g(;+44xw~=* zxvNz35vfsQE)@)Zsp*6_GjYD};Squ83<_?^SbALb{a`j<0Gn%6JY!zhp=Fg}Ga2|8 z52e1WU%^L1}15Ex0fF$e@eCT(()_P zvV?CA%#Sy08_U6VPt4EtmVQraWJX` zh=N|WQ>LgrvF~R&qOfB$!%D3cGv?;Xh_z$z7k&s4N)$WYf*k=|*jCEkO19{h_(%W4 zPuOqbCw`SeAX*R}UUsbVsgtuG?xs(#Ikx9`JZoQFz0n*7ZG@Fv@kZk`gzO$HoA9kN z8U5{-yY zvV{`&WKU2$mZeoBmiJrEdzUZAv1sRxpePdg1)F*X^Y)zp^Y*R;;z~vOv-z&)&G)JQ{m!C9cmziu1^nHA z`#`0c>@PnQ9CJKgC5NjJD8HM3|KC(g5nnCq$n0Gsu_DXk36@ql%npEye|?%RmG)

FJ$wK}0tWNB{uH;AM~i literal 0 HcmV?d00001 diff --git a/examples/llama.android/app/src/main/res/mipmap-xhdpi/ic_launcher.webp b/examples/llama.android/app/src/main/res/mipmap-xhdpi/ic_launcher.webp new file mode 100644 index 0000000000000000000000000000000000000000..948a3070fe34c611c42c0d3ad3013a0dce358be0 GIT binary patch literal 1900 zcmV-y2b1_xNk&Fw2LJ$9MM6+kP&il$0000G0001A003VA06|PpNH75a00DqwTbm-~ zullQTcXxO9ki!OCRx^i?oR|n!<8G0=kI^!JSjFi-LL*`V;ET0H2IXfU0*i>o6o6Gy zRq6Ap5(_{XLdXcL-MzlN`ugSdZY_`jXhcENAu)N_0?GhF))9R;E`!bo9p?g?SRgw_ zEXHhFG$0{qYOqhdX<(wE4N@es3VIo$%il%6xP9gjiBri+2pI6aY4 zJbgh-Ud|V%3O!IcHKQx1FQH(_*TK;1>FQWbt^$K1zNn^cczkBs=QHCYZ8b&l!UV{K z{L0$KCf_&KR^}&2Fe|L&?1I7~pBENnCtCuH3sjcx6$c zwqkNkru);ie``q+_QI;IYLD9OV0ZxkuyBz|5<$1BH|vtey$> z5oto4=l-R-Aaq`Dk0}o9N0VrkqW_#;!u{!bJLDq%0092{Ghe=F;(kn} z+sQ@1=UlX30+2nWjkL$B^b!H2^QYO@iFc0{(-~yXj2TWz?VG{v`Jg zg}WyYnwGgn>{HFaG7E~pt=)sOO}*yd(UU-D(E&x{xKEl6OcU?pl)K%#U$dn1mDF19 zSw@l8G!GNFB3c3VVK0?uyqN&utT-D5%NM4g-3@Sii9tSXKtwce~uF zS&Jn746EW^wV~8zdQ1XC28~kXu8+Yo9p!<8h&(Q({J*4DBglPdpe4M_mD8AguZFn~ ztiuO~{6Bx?SfO~_ZV(GIboeR9~hAym{{fV|VM=77MxDrbW6`ujX z<3HF(>Zr;#*uCvC*bpoSr~C$h?_%nXps@A)=l_;({Fo#6Y1+Zv`!T5HB+)#^-Ud_; zBwftPN=d8Vx)*O1Mj+0oO=mZ+NVH*ptNDC-&zZ7Hwho6UQ#l-yNvc0Cm+2$$6YUk2D2t#vdZX-u3>-Be1u9gtTBiMB^xwWQ_rgvGpZ6(C@e23c!^K=>ai-Rqu zhqT`ZQof;9Bu!AD(i^PCbYV%yha9zuoKMp`U^z;3!+&d@Hud&_iy!O-$b9ZLcSRh? z)R|826w}TU!J#X6P%@Zh=La$I6zXa#h!B;{qfug}O%z@K{EZECu6zl)7CiNi%xti0 zB{OKfAj83~iJvmpTU|&q1^?^cIMn2RQ?jeSB95l}{DrEPTW{_gmU_pqTc)h@4T>~& zluq3)GM=xa(#^VU5}@FNqpc$?#SbVsX!~RH*5p0p@w z;~v{QMX0^bFT1!cXGM8K9FP+=9~-d~#TK#ZE{4umGT=;dfvWi?rYj;^l_Zxywze`W z^Cr{55U@*BalS}K%Czii_80e0#0#Zkhlij4-~I@}`-JFJ7$5{>LnoJSs??J8kWVl6|8A}RCGAu9^rAsfCE=2}tHwl93t0C?#+jMpvr7O3`2=tr{Hg$=HlnjVG^ewm|Js0J*kfPa6*GhtB>`fN!m#9J(sU!?(OSfzY*zS(FJ<-Vb zfAIg+`U)YaXv#sY(c--|X zEB+TVyZ%Ie4L$gi#Fc++`h6%vzsS$pjz9aLt+ZL(g;n$Dzy5=m=_TV(3H8^C{r0xd zp#a%}ht55dOq?yhwYPrtp-m1xXp;4X;)NhxxUpgP%XTLmO zcjaFva^}dP3$&sfFTIR_jC=2pHh9kpI@2(6V*GQo7Ws)`j)hd+tr@P~gR*2gO@+1? zG<`_tB+LJuF|SZ9tIec;h%}}6WClT`L>HSW?E{Hp1h^+mlbf_$9zA>!ug>NALJsO{ mU%z=YwVD?}XMya)Bp;vlyE5&E_6!fzx9pwrdz474!~g(M6R?N? literal 0 HcmV?d00001 diff --git a/examples/llama.android/app/src/main/res/mipmap-xhdpi/ic_launcher_round.webp b/examples/llama.android/app/src/main/res/mipmap-xhdpi/ic_launcher_round.webp new file mode 100644 index 0000000000000000000000000000000000000000..1b9a6956b3acdc11f40ce2bb3f6efbd845cc243f GIT binary patch literal 3918 zcmV-U53%r4Nk&FS4*&pHMM6+kP&il$0000G0001A003VA06|PpNSy@$00HoY|G(*G z+qV7x14$dSO^Re!iqt-AAIE9iwr$(CZQJL$blA4B`>;C3fBY6Q8_YSjb2%a=fc}4E zrSzssacq<^nmW|Rs93PJni30R<8w<(bK_$LO4L?!_OxLl$}K$MUEllnMK|rg=f3;y z*?;3j|Nh>)p0JQ3A~rf(MibH2r+)3cyV1qF&;8m{w-S*y+0mM){KTK^M5}ksc`qX3 zy>rf^b>~l>SSHds8(I@hz3&PD@LmEs4&prkT=BjsBCXTMhN$_)+kvnl0bLKW5rEsj z*d#KXGDB4P&>etx0X+`R19yC=LS)j!mgs5M0L~+o-T~Jl!p!AJxnGAhV%~rhYUL4hlWhgES3Kb5oA&X z{}?3OBSS-{!v$nCIGj->(-TAG)8LR{htr41^gxsT8yqt2@DEG6Yl`Uma3Nd4;YUoW zTbkYl3CMU5ypMF3EIkYmWL|*BknM`0+Kq6CpvO(y$#j94e+q{vI{Zp8cV_6RK!`&C zob$*5Q|$IZ09dW=L!V zw@#2wviu|<#3lgGE8GEhcx+zBt`} zOwP8j9X%^f7i_bth4PiJ$LYtFJSCN$3xwDN;8mr*B;CJwBP2G0TMq0uNt7S^DO_wE zepk!Wrn#Z#03j{`c*Rf~y3o7?J}w?tEELRUR2cgxB*Y{LzA#pxHgf}q?u5idu>077 zd^=p)`nA}6e`|@`p?u}YU66PP_MA}Zqqe!c{nK&z%Jwq1N4e_q<#4g^xaz=ao;u|6 zwpRcW2Lax=ZGbx=Q*HhlJ`Ns#Y*r0*%!T?P*TTiX;rb)$CGLz=rSUum$)3Qyv{BL2 zO*=OI2|%(Yz~`pNEOnLp>+?T@glq-DujlIp?hdJeZ7ctP4_OKx|5@EOps3rr(pWzg zK4d3&oN-X2qN(d_MkfwB4I)_)!I_6nj2iA9u^pQ{;GckGLxBGrJUM2Wdda!k)Y>lq zmjws>dVQ*vW9lvEMkiN3wE-__6OWD0txS&Qn0n22cyj4Q*8(nG4!G{6OOwNvsrPIL zCl-$W9UwkEUVuLwyD%|inbOF*xMODZ4VMEVAq_zUxZ+K#Gdqf!DW$5f)?7UNOFMz! zrB~tuu=6X2FE(p^iqgxr+?ZK;=yz`e;C$#_@D9Lj-+TDVOrva>(#*PVbaHO>A)mhl z07OJWCqYC60518$!&c`eNBcBW%GnfaQ*$eazV^2_AW?j)h;J1nUjN(I9=0+!RVx~% z3@Tf!P0TE+98jA?WceK-}A1% zW!K)lyKcGqy#M~})315-A#2NXQ`?6NR#Apo=S!oF=JfpX>iR*49ec{7AN$xxpK{D$ z2d%Fz&rdfSqourN$~Y^NFIMV1CZ?J*bMx~H3k&meGtH@q9ra2vZxmA$S(#jaaj-g4 ztJmxG+DLV<*q<|sDXPp$X>E)#S}Vm&sRaO5P&goh2><}FEdZSXDqsL$06sAkh(e+v zAsBhKSRexgwg6tIy~GFJzaTxXD(}|+0eOwFDA%rn`X;MVwDHT9=4=g%OaJ9s%3b9>9EUTnnp0t;2Zpa{*>mk~hZqItE_!dQ zOtC>8`$l|mV43Jbudf0N6&&X;{=z}Zi}d1`2qmJ}i|0*GsulD3>GgQXHN)pkR6sf1 z?5ZU%&xtL}oH;YiAA)d*^Ndw2T$+Mjuzyzz@-SM`9df7LqTxLuIwC~S0092~+=qYv z@*ja;?Wt!T!{U?c*Z0YtGe)XbI&y-?B&G2$`JDM)(dIV9G`Sc#6?sI60de6kv+)Qb zUW~2|WjvJq3TA8`0+sWA3zRhY9a~ow)O~&StBkG2{*{TGiY~S8ep{V&Vo2l<6LWsu z^#p0-v*t2?3&aA1)ozu|%efSR=XnpX$lvTeRdKlvM!@|pM5p2w3u-6 zU>}t2xiYLS+{|%C65AzX+23Mtlq?BS&YdYcYsVjoiE&rT>;Necn6l^K)T^lmE`5u{ zm1i+-a-gc;Z&v-{;8r)z6NYfBUv+=_L}ef}qa9FX01)+Aaf+;xj(mL6|JUzGJR1|fnanb%?BPPIp>SCjP|8qE5qJ{=n5ZGw?81z3(k;pzH%1CtlX50{E7h)$h{qGKfzC`e2o`*IqA#tjA z`Fz&^%$b9F*N`)U-#6>a)Z`55`$Dd0cfcs0$d13^ONrdCu9xcv_=n#WQo8stcz3jP9|2EvdI-RhJM3%Q%oM&!OlShM|0 z?gz?wHZSnm45njLtsz8PVT1S&jAlbKg5kVam$p16=EK@Sj4EP0OtH zmJDmdc^v)x>56Qg_wmYHz6h)>kl_h$>0@J!ypv%APmjZTAQVLy6Fu50RGY&JAVNhx zrF_qG6`x9MkT;1SFWo$)l{M$;3qUDn9JwE}z zRl#E_bDRJFii61kPgBybIgp8dNW!Cc1b*^YYk-#oWLJvtM_v^hQx~9?8LD4VFFxBF z3MlrsSC%f9Oupn*ctPL0U1fwfX?`tRhPD{PSLFPQOmIt$mDy0SgpNVvHS+f#Do>h1Gn?LZU9(KaN>Q_=Y*_T zvtD7%_u^^+{g`0VGzg(VZrpVQ6Ub5M=tI_p7T93R8@3Zulu3|#{iNcu!oiHxZ4Rf*( zfmiN$$ru(*_Zqn=`Gq#OuHRTSwp7uH_SokR&|)RuW5yo=Z|_4?qU-JU+tpt>!B&Is z@N(=SG;bpVc;AO@zbmMM zScqq1)b-ZQIrs={oD}|?6y{$HNB1U0^LsBh8JI&3!GBZxOXI<}&5-$lgkAaYqhOTb z?2vEnZ$-kk;*M_17(upJF3%+iH*s0-r{vttXVB2OUwI1s^+G(Ft(U8gYFXC}#P&E^ z>T@C^tS`Z7{6HT4_nF~n>JlZtk5&qDBl6r|^kzQYe`wq!C)n@$c>WOPA61NDFj<<6 zGW71NMMhwAl!U-yqrq2xrSFqRCI8acw7?}3j;ynxo*-b7Co;g5r%^j=H@9({PXXBf z@r>U>>N;E)81wx`B4f%{PB~MHka_);%kBCb(d|Jy5!MqJ%2p`t&@L)4$T2j&-WHvG zv3(uyA_gwqNu(k?jQTtv3dgPKRZoH8prxe7>pQBW5L&dpumS&5Ld2?(sCpJjvc4L5 zEnh&?91WVm)ZdTj=fjJ$pPDdgAttLXuke+?KdKxu*;kTC(r!tQk6;gxj4h%FdHAt(^M3YvYj(!tOeN)+Hvj6+< zzyJRG?^lZfWuR#t!tUKP&(?%3v&Zd$R2YN>lB(Lq`OInY48%4%yTv2 zYe1{G`3)(PDEio5Y@-I5tUf`c%%OCJMtSW56g3iEg%3`$7XSJJHyA z<|7&N)5Xrlgv~%BO24eFd;Hd;uiK%D`EdK|quUeRZDqbh9l)%j%J#0lfrZumvA<_w zu&=AVvdChf6}eqh(bUz`(`Ue*p01{fBAcTgKyDYLs_I+YyJEk+rM@avU~>fB$n)HS zM7pfJydu`i%gfS<{PF94kZDv$t>06sAkheDzu40NJ$5CMW%n^Lls?8^p^QGWURbKu3ZduZQZ((s2? zzE`}<{;Zt7<$C|9R8A~DJ~@%x>TfP zF>TX8)@v|t)q4GjRt<}5s6hLHwRel7>V@&r-O|Av(yh;Q1A{E>Ir>p+%dHD|=l+lT zpr(Dg&>#Nu=!)6bCLr-ZS%|;h)Ij$+e@r8_{qO19QvDe=&1tmpY*0lcA^Cc-#{9fQ z<~$*<&P$Q<_jy#<$40PMofM7aQ}C=jphI`4kLg}Z7CIN#26D{-4v-_CA-LiE@(%{y!BzsU%gG`Q?sjLUf%qFSl0y)2#ae*+EI>s|i`d^V$Dn)qmzqRq6VJRY|{4ujsIU%#bnqU6MR&-1I_43=|5(6Jr;Jvert) zE?S|Tmn}Tv<-??sxV5@9t}3D=>YZ0JrQe$CO~|EY=Lj9RM&4svQHPQL6%pV5fPFiH zfXDx;l@~et{*{U*#c#Dvzu)|znDO7$#CRx)Z&yp-}SrD{&|(MQtfUz~n35@RLfUy=aqrhCX0M}J_r5QsK~NmRCR|Nm&L z41UdsLjWxSUlL41r^0K&nCCK>fdR-!MYjFg(z9_mF^C|#ZQw?`)f6uVzF^`bRnVY& zo}@M06J&_+>w9@jpaO4snmU;0t-(zYW1qVBHtuD!d?%?AtN7Plp><-1Y8Rqb20ZaP zTCgn*-Sri4Q8Xn>=gNaWQ57%!D35UkA@ksOlPB*Dvw}t02ENAqw|kFhn%ZyyW%+t{ zNdM!uqEM^;2}f+tECHbwLmH*!nZVrb$-az%t50Y2pg(HqhvY-^-lb}>^6l{$jOI6} zo_kBzj%8aX|6H5M0Y<)7pzz_wLkIpRm!;PzY)9+24wk2&TT{w--phDGDCOz{cN_ca zpnm7`$oDy=HX%0i-`769*0M6(e5j-?(?24%)<)&46y0e&6@HCDZAm9W6Ib#Y#BF6- z=30crHGg+RRTe%VBC>T00OV6F+gQDAK38Ne3N9bm|62tPccBJi)5{B z4zc^Db72XiBd}v$CF|yU{Z=M|DZ%-(XarYNclODlb1Kz1_EKLy(NSLCN`eUl(rBCL zT*jx@wNvze0|TSqgE(QArOZU)_?qH(sj#TwzElLs9q)(0u!_P|R%Cy_0JFQxgGV>1 zz4?_uq<8_gM0`c*Hh|;UMz~vrg1gQXp{ufg`hM_qU;U>+zmvc5blCLSq@PrEBSGR# z&8=2Z4uXN`F3p73ueD1l{s{k$WipAvSh5W7ABe?4)t;r@V?y`bNB5FvBuE|0VRTb< zM1Hn^?DSsJY+sX@T5xW=#>T9VEV|?<(=6|ge$X6Sb05!LFdjDcoq*gM(Zq=t;_)Le&jyt(&9jzR73noru`a# zN*<`KwGa^gZU3-)MSLF0aFag#f0<>E(bYTeHmtdbns#|I)-$)mJ`q9ctQ8g0=ET?| zdO}eZ*b_p>ygRTtR^5Ggdam=Zb5wmd{}np+Jn1d_=M`~P=M67jj})fH4ztb5yQqQW z^C|C&^LHAK-u+ooIK)yM)QM?t;|<{P;;{`p=BclzAN#JzL4jCwXkQB1Dy{=^KR`=~ zTrr)y7eiYBzSNs_DvO=4A6#EgGS-zY%Vi)N*Yb`U;6o}KR}dq{r9pT5wqZ@3NOE8- z9-(}D|Nc5732CSYQbL)!gPQ#RbD8BhK3dl{sUuPvei0tkvnJBxDEAYTesU8H$)g(Plra{VH(v3u^CO1~(+ zU0O7#)jaS4{NcwA+LuSm&VBcX2#Im3xg)W}ySNw%->orn1taZ&+d)}8gJTqA!u|5P z{yv?zol_3|(1(%M(EVU=cp?L`{Pi|ixk{U)*guFML3P!OSlz;zGA#T+E@8@cgQ_mv1o7RSU=Zo_82F?&&2r;WE z@wk}JHYEZ9nYUc(Vv~iTCa3u8e4q(yq<29VoNbKk|`mq%I6u)My=gPIDuUb&lzf4`MEA9^g8u z)vp8|$$HE9m_BTV?lOosIGa4jud=jIbw)O2eCMfyw2*S8?hjWw^nqws$O*M$3I1)x zR0PWFb3$ySOcGTe1dz%N0l;RPc`x%05FtT^f^j{YCP}*Q=lvp4$ZXrTZQHhO+w%wJn3c8j%+5C3UAFD&%8dBl_qi9D5g8fry}6Ev z2_Q~)5^N$!IU`BPh1O|=BxQ#*C5*}`lluC515$lxc-vNC)IgW=K|=z7o%cWFpndn= zX}f{`!VK02_kU+Q5a3m37J;c} zTzbxteE{GNf?yLt5X=Bzc-mio^Up0nunMCgp*ZJ;%MJvPM3QK)BryP(_v@ei4UvHr z6+sbCifQaOkL6-;5fL8$W($zZ_;CZp305C;~$hhRquZr-r)jjd1z z31%ZK{-(`P#|Um_Sivn@p$-vz46uqT>QG0B1w9znfS9A8PB2LaHdzA|_)yjXVR*l{ zkcu3@vEf7bxH0nkh`q?8FmoO_Ucui*>_a~P?qQrlZ9@+D7%MTpSnztpylXrt5!-k8_QPB?YL8Kx_On8WD zgT+111d(Op$^$&KLAN5+@?>f7F4~wFi(8TL8+szgVmcMDTp5l&k6~=rA{Dt}!gb^r zSWY<)M7D|Z2P0cEodj6E42PV>&>DFmQpgt)E-|#sSUU@uKed+F680H@<;-x{p|nuH4!_mn85rx>wz;0mPi2ZkL#k6;sznu?cXh!T0S>{w6 zL^gvR05NY64l*<+_L>On$rjx9!US;l;LX6@z}yi#2XHh)F@Oo+l)h%fq$v}DNmF2> zfs^_t0)3N-W<9-N?uedVv{)-J0W5mh#29QM5R5h&KuiRM=0Zvnf#lF=K#WlCgc#9c zS;qvh(P$!_a8JwyhI^ZJV2k+B6Z^64?w|1?5gyo6y{}923CRZfYVe1#?F% z7h2SUiNO3;T#JUOyovSs@@C1GtwipycA=*x5{BpIZ_#GCMuV8XK=x;qCNy{d7?wA~ zC+=vjls;ci&zW=6$H~4^K%v{p}Ab?U%C6Z4p%eC<3ExqU$XR<}LLF67A$Sr20DR_pJ3yeBa~ z^sw{V0FI5;UpwXsScYuhbqGQ`YQ25;6p6W^+tgL&;Ml;>S3CGpSZ>VrTn0m1$y$HU z&65)I!c?oREz};c=nLCliriqQX->4uivHTgd${GqeAlf*!P^B|jkU|*IdNP(&6C>4 zqOW$)Nw9nvjy^&`?E|gotDV{JmJ9Q~vuhy<`^C4XIUDt|j4o6rK^e8_(=YqC zuaR6TRVf@tUFHB079o4MBIh{M~4>WwnGgesQH*3?w(RA%hCZ*7)b!aNV=yOQ%o_Y=Lt0Sl*(9^jfRnC210Om$=y>*o|3z} zAR&vAdrB#mWoaB0fJSw9xw|Am$fzK>rx-~R#7IFSAwdu_EI|SRfB*yl0w8oX09H^q zAjl2?0I)v*odGJ40FVGaF&2qJq9Gv`>V>2r0|c`GX8h>CX8eHcOy>S0@<;M3<_6UM z7yCEpug5NZL!H_0>Hg_HasQGxR`rY&Z{geOy?N92Z z{lER^um|$*?*G63*njwc(R?NT)Bei*3jVzR>FWUDb^gKhtL4A=kE_1p-%Fo2`!8M} z(0AjuCiS;G{?*^1tB-uY%=)SRx&D)pK4u@>f6@KPe3}2j_har$>HqzH;UCR^ssFD0 z7h+VLO4o@_Yt>>AeaZKUxqyvxWCAjKB>qjQ30UA)#w z&=RmdwlT`7a8J8Yae=7*c8XL|{@%wA8uvCqfsNX^?UZsS>wX}QD{K}ad4y~iO*p%4 z_cS{u7Ek%?WV6em2(U9#d8(&JDirb^u~7wK4+xP$iiI6IlD|a&S)6o=kG;59N|>K1 zn(0mUqbG3YIY7dQd+*4~)`!S9m7H6HP6YcKHhBc#b%1L}VIisp%;TckEkcu0>lo@u995$<*Em;XNodjTiCdC%R+TX|_ZR#|1`RR|`^@Teh zl#w@8fI1FTx2Dy+{blUT{`^kY*V-AZUd?ZZqCS4gW(kY5?retkLbF=>p=59Nl|=sf zo1Pc|{{N4>5nt#627ylGF`3n>X%`w%bw-Y~zWM_{Si$dc82|=YhISal{N7OY?O`C4 zD|qb}6nLWJ`hUyL+E>-;ricg9J@ZNYP(x(Sct&OI$Y!QWr*=^VN;G3#i>^1n4e#Je zOVhbFbLpXVu*16enDM+ic;97@R~u&kh__kgP#!R`*rQEnA+_dLkNP~L`0alC|J;c; zeiK=s8;BsLE)KbG3BD&Br@(Ha@SBT&$?xX`=$;eeel=|R_dIr6-Ro?=HEjnsJ_b`1 zK6Yg^-6;^2aW!xeTK)A~3Rm|L^FCHB_I>jIju7ZGo&N_1*QHkxH2!!%@o4iZ?vntS;&zJdPe1dH#04YD93A44o-MpfD zP{rn_aq>U%RDvC2+bp;xPlsOzauIi3*Lf42`jVKKZCRuKdYhi>FDuL2l=v{$BCN#Q6796s%r-AG$Q^t(3c@ zD?w0UhYr11@feiyl9kY_@H8~|xlmO<8PfQmj1!$@WieW@VxR@Psxfe-v9WCi1+f>F4VL?0O~K7T?m4-u|pSkBpUJZZe*16_wAp zSYZ@;k`3;W3UHKUWc8QeI}0jH5Ly=cGWQPw(Kr2fm=-5L(d`lcXofy8tJY3@Tuadz zYWXR{mW7XT!RF#RVCe%}=tM*O6!AD3^(!8un~opNI%Uko7$5t@<8+?; zTxDys(MyyGsUjtSu9$+|_-t!U3fVb1dkK?l`17<+jfl=hrBHnDSV>^R1=TnQeyqbW z>ov#l%!1|S!1>8UUxIdhQq`_klcHVx0{?#>K3#$4GlXncwldt!g17TcvKq-jo_996 z>oA=tH9CqRl6Yw?Uc`am!V?lHJbizOJaVaScf1UP5e7Dbgabq=b!B~T&_F6?ooU>w%x0A zH~&MHJ=q`fCH{U<7MDXE4SD32cDZA)WJeWkllJ`UspWaS#eDe^kg^oU_A14UE9zG-a^g{xaXf$})Wik>gT zl#dkzGr(;h0JZDuFn(+k8wNq?PZ5grQ<+sM?wBGt@JnH6v0#or-5wBQWKU~(S_> zkE!tc*ZJ1Y&*p(xX84POb3cClRMd!^qJ#CAZfIepEj-<`VURS_yCz0(?*Ixcj4 z-!zV1_QZhpm=0<;*(nm+F>T=)o?ep@CK5I%g^VAA+RB25ab?7)A~z~egru=I1S|@v zH7tXV!0wmGS^qj#e+MY;C5eUjEAp$Y?LDkS^QPZ}8WN85?r$u<-Epi;yZ1|J2J`se z$D6DpH~2F=eI0B&=UFAUnJvZAmClJlK)sutJ?M>xpZiWV&0=G4MZP+x+p>EX=HbCz zxls%Mw?*u^;LbHWIWCyq+yi)`GmFn9J112CZda_u@YIP%i;srFg_paU02Ifij*7}l z&CF-(3|>*a|+vbNR`^RP=9G?ymEJ0Z~)d&c*UE$UMepZ zcITr{0WqhxkjUnM15js_gW=e3Uh|y6ZReaXHIz-=p`x5VvB&rH9y>Amv@^WmXFEw) zQXYrk3feir=a{jMQ+wDIkkFnZ$k{sJakHn*?u za%4b!00ev8NVLM1TY=cl?KB&55BY_MU-sg?c>=Dbz_W{(Z~c?HJi*XpYL)C6Bd8WH zt+v-#0&o~@t4qESi*)+eW%@VD0|o^yF)n0hME$UtXF$*Lvh}7sso{`|pn*JDIy5^Fm3s$5*zEE=?u5<=l8FJc3r%+H} zdfoNl2J0^~!-*mOL5o-x32|e0Im*E!yY7F7E5N)W3>+v_LBydlEx?4$RL5f2oYRD# zaR0wv(-p~wO0eLDl3K=%`{5+0Gd$ktO=W)gWlGZJ0`K z$_RNA=ckrfa;H0KA~dR^p�(p-{x$&=IACIfoAR!za)F-^da-t3#0Dycnp zwO~NVXwXCl;jE<}>%@xz|=8fIJAB?>+E{7)|4l${4ngA3G|=r z2Dyv;VVWSgZx9Wj>qUjleGl3Ei9K4>h!(lPS%8VOG>Xu0%6VDz^O=bjJmuP7>DeUv zrbI}MlHB^^d?{zv6d=@_ZD2lg1&G7UjnVN{1}9WkaM3H~btX0GtSzB+tZ^qRgWo4m z!GmimlG$=wgXCnr6j@m<1gAL46#T~5Bnm=2{^@>|t&`9mkEPddj zAvG~@Tv~TAm2i%VW}R-g(Z0)z-Y|szHr@rk>4MAyG*Ma*7Yh#H7(!-5>DZ@8r;_dx z{prSe<>~099F8vsYd2xff7uAS%7{S)f(|@me3t2$iy&NEc7OUEchp@9A|X;;IA>8!oX+y(BKJ$EzV* znR$z;!L$s7uy@{OT~nG#B!NRraT8(X##Ho!0r_o@gg0CA-9H^;-uE&?$2$nHv_00o z%cbuUc-tCx$Uh&EZ4Nf4Zgqv)Y6>usG3>GeQnxx_Z6+PcbX-+ysbt1hQ`K1LDpOE? zrAhIZhSN9yVIAOa22gn577tbc&i3|3V8NWy&!tw##`}9*x}gtI^h1DzZRA>UuaJG) zaZ7j)dq!O}{?#8Y7~7i6fHh4{`pL?>-18|p!S75Y#^DM>-S3)vuZG+Q7l@ek zQP~#cBpWgg#mApc_sPYjpw8odQuRokmTkzcNl`^CcKB7e&;zViV;{Y{o^Y$%7i0m# z62%#1Lq!RC?}lK>%mp}T!3Xv;L*0v*>USLm``N%>w>@fwC+#T&Tx2bN4w(20JB}oU zuSa6v^kXi0xPs?pbaOHnyiqq6By1EZY9OZ^^QA>{q-Hsd&m`pbQ%8121aWG-F5xf zlZ%;B{;C>X19|`^_?dVyCq>n+41w7|!tUS!{9rHlbhX=SZO5CQ^;!Du_E7*`GiR^Q w)2!4MKjfSAeNo!9>IaV6aUZ*?W>} zs4%E?srLW`CJh0GCIK@hTkrW7A15Iu%N&?Q^$0+!{Tv&|t^Y@u%!L zglTg&?Q5q#ijZ;&HBQ?FNPp;k3J5!&{^+SGq?AX~SiOM9jJMRpyP?RCr@z38AQyy&WRMaC;n4una$~nJKSp?q|s8F00c9?Q! zY_ovvjTFm+DeQM^LXJ#v0}6HRt3R1%5PT*}W!k8BEM;Jrj8dIceFo2fhzTqaB3KKk zGlCLI)gU25(#u6ch6GeB1k@eHq7l{EHXv0n6xE#ws#ri}08kkCf8hUt{|Ejb`2YW* zvg}0nSSX1m=76s?sZhRY$K=3dpJ+y*eDULGnL2}4>4nvW^7_<~wIM_5fjvwt4h1|g z)g0Z6ZFq9j<~9~b8((~TN{Z?ZQfw|is&Xp~AC61sj;xItKyCHdI|tCMC_LbXF>~vR z=w6V3^H=W4CbAgR4#xw}ETTwu2guW~=Crl@SMXv85jQ=%y!s^?m4PI0My7MWICO;- z175jm%&PcPWh8QdOU(#8bp4!N7ET-+)N}N2zk2)8ch|4Q&lPFNQgT-thu053`r*h3 z_8dI@G;`zn;lH$zX3RzIk`E8~`J=BBdR}qD%n@vVG1834)!pS1Y?zVkJGtsa(sB~y zNfMYKsOJb%5J(0ivK8d+l2D2y&5X!cg3BG!AJ}910|_${nF}sC1QF^nLIhzXk-Y#x z0)&1iK!O;Og0Ky!;`b~v%b$`S4E&fB)1NB4v@8wr( z&+NX4e^&o)ecb=)dd~C!{(1e6t?&9j{l8%U*k4)?`(L3;Qjw z#w7FS+U(94MaJKS!J9O8^$)36_J8;thW#2$y9i{bB{?M{QS_inZIJ!jwqAbfXYVd$ zQ5fC$6Nc9hFi8m^;oI-%C#BS|c8vy+@{jx6hFcf^_;2VRgkoN(0h!_VSGmgNPRsxI z8$rTo0LaYq-H5i&gtj81=&xU?H-Y2==G@uQV7E`@+2E9XQW@{&j`?EOktk|Ho{HU>ZqDzvgjwBmdex z&uZNd2C1h{{}2k6Ys9$*nFP3;K%u!MhW`uZy7Sn`1M1zs@Es&;z*Z>Gsh@-3Fe6pE zQD2@cqF((NrRevgvLsvM_8;;iNyJ5nyPyy?e!kvKjGj`6diRFBEe49Oa7wwkJFV7Z z$YT&DWloYu-H?3<0BKn9L&JYDT-SK~*6c5pi18P26$JESKRYj{T7Zk6KiRJcbvOO*{P56Q6s8msbeI3>|j>K9}Q9UBeq*inXKemCm`-<5|-$ZyN4u$(3 z&HcvqehFD%5Yrmykg-^d`=BSa8(i=>ZoC77^mWY{evp(km@aHqhUECBz76YiR+VYK zY_avFC~V3$=`6C4JhfHAQ@DZtUOwH`L;oYX6zK0-uI^?hS$ALfq}A7evR;ohJHij} zHSZdW?EKv9U1s4oD*<(0oQ*;MaQ6@cvGL zuHCPgm_NhVsgp^sfr*ia^Db}swo1?O(_Q2)y+S$CBm+g=9wCOUPbz(x)_GbaKa@A7 zuI&!ynLiZRT#V%_y_-D`0Z5lT*auoe{(U5NylTzFSJW()W-#F6*&A`LNO1bV#Y;QJ zSbLBnp|B^dtK|KIWC|No>JjWBWE@n7O)x{&^E(WMeMvp57#qA8m* zeTow*U@_86B#Fm*rxyYu5PRWaWHx8y> z*qmHEp(AMDl0v)ij(AY8fnH=~ZwwjVAbu*m5;xPfidh@ov6d8g zfJsi&!QyK53Es%sC39ts;54V68koALD4b|%tNHW0bIkZAJKa=W&FomJSEDT>W1xIX z1x%Z>AvNIsSPLcn3RTcHXb@KB?cuM)=x6fcIx>&(GxqZ8w3p#jJ(GVgc*`c0HG}dv zIop&Qim!K1NFwic%07KcjWgHBPUkq7f~lj;TPqVGTiT#cUeim>;nY`>h@a*S{qQex zQ`z62WK|Mj)Y{tfF{;T4P;c8$Q|KU?Joh zIkA^z%X7z|r>4aTh@|StTi!-r1D!g=zb#3d#{{&K3CqE$Iz-UH<%37c zRfkO`&uM%#AD3PHv`g5t0e^O%nVL0d{Xlx^EjEC3#skF@`zl-7PF^0oxW)1!C!JxR zWvuAHH?)61FKA1QeT*_sY7;_Id#!GmV4n`MO{~sv}VLSK` zXRw=Y=Clz*00B(5y^K;gCZMAzjT5+c3IC=)l(9VIDdatpxj3y89WwI|bH&$!ZEvp` zPR!T@#!(|KfI-w?!&+7$N3F6>tD{YO4Qg$d_`nNEdfVCha9vaPn0jI0`)`@*72hq! zpU5ND^P*RoEkbD5o#az(-g=Y)L>HH>Oc%}$ zT3Rs_ih0;4+Lv4Y;@Iv(;fUbQ=i-G(#>vghec~*j(I#r|5mqFiJBpzi&hzEcD{u$< zRsm0BVYn=pT;0>R(itW|*D&;O%bOc7et9ACaH#J>z3A1A~6fdP>pmbM%xzm4>|;c_?B+%sl;Qs2{t!60$^u zH1t@9^6>;?!FuusnISi$f5CL&;z?EqJN$FBuWDA#D5`cy_UvCFIVvf{c?4N0teh;d zET$7aVbj08KTQS!x?Nd1Is8q8qFzs}a=!@nJ;7FSfCY^T@D-gpw`w<6e#X3+;O}1h z$%I!M)0bg|EKUA04Qjn@+x{Rj8vt6Wn!R|3A92z}^$KfF5(#CWr4y#~re1CN4i4w0 z#GsypBR{xA3Er7sgAi(|}1-W?s~n$7?K|9WL8kpVfw-;#b9 z+mn;=ep!162U5R>_t}fOt~tE?s#m( zO-S$7>Ay6*hHdZ)7_oU915WYYCIX;hFI-U2EWYX!pllONr@Q--2o~`!isi6vTPLJ4@(|o=%NHYjo0_S&q*UQIROw@*N-By@PaQ&;YxFZ0aR zX&}LeOEz);#m~Hwm^VAY8DK}b$F4bo{jMN?d!lxKPhNklzr^Cd`0f4oJr^z=I|l`* zm8AHm*fPV`0=lF3Pnnp}&J0N1X@}-D94YvmUabFrLGSnTz7Mu^21F#O5tN#CuY9Vh zUZBH=ez%h*wkf0hBtXJh1SN3d+IF{gzT7lp)j}n?03lt;XSQRAh7qd&v;RwTYDuQ# zbI2*r<>?x-G0@hM{;%{VBD7nLKt~D`T~-HAt5;h%i0_=Ifs=yHma5dhJ+QMG?Ux(a z|E?1CMy1!~oA`FP!k~iG=t&5#>bVdz=peT8HMB6Y)#7PpETtNryT^+Rv3vpJaF^zP z{H}0-LyV9Fu21ID%wO9f1IKlFr1p4c{o-?03vyB-tr5duk^&L$;m_|f$vs`^Sl{j2 z95}oY{LlY+=ZS%J+tZoXCd0*sSU7w^gjovXn+g7uyra5{cU49@yHf#Z^Jl-$9cIfo z+AJuxH$VLb=#+uBbVmUjnx zxb1pZ@-O9=AIk4@S)m6fJ2?{HrNYwwnL3a45muuNjr;6$O`bGEM0T4A2_S$t=86*- zcO+0mywg*j#A4mU}enR_!cGmIYQ;qwfchWtFEXL)AK%*;=j znYne+hS4EMy3S)C*mZ1KI>!+)0V@9!N6H$Y}~MJ{rYuf zz^KljIWvFi-?#?V@LPR&c6Nn{!=XM z>}-h$S76;$H{E{Y%@^zlmOl^efBwa%UU+jJD9UVukQ3ti_kH-?H*RC0?M1W%FCvMB zM_+v6fk$6X2sx)-p~B3&Kl{nscK}pNLM*qjtpaf9>AU{-iPKQZR8yCg!TY}Qg*(;) z)gdvCcB%kppZc$VdvsK@)3l1{&DG!d_6OHOS`y=ITLEVu`unSKA2E%JD*DVX{LJ}K z9l>hMRDqxQh0lnpGHpVYneX}eA3Pt|2v%=q;rt)``R|#bDyB)OXY&vI_@|*}h}G?^ z@aZ4_!7cQPX`!fW_?{oT1NTwHs#l5L-0`E|y@48<3Q^HFf8=Idi zpJYD%1MkII!~|7I^WGo)IF=?{>ACnjJ_WUi39C}!Q{QnheVJqeKKqq5^o5CBde(g9 zvw$X6^jz_^E2$wSw4!q5*RG(C2_^XO$HBn_55vbl44OnTTRwRaePP0vo{K)U1#99& z<>rq7V&V(<&@I%MFoN5zrY}sz=(*-L&}1QQ*a%`u25h{cFj===17eB_uGuzG&byQ< zrm8BJZl4r_E$3k|Wo6FW0-6M7>qac5uFQsQcmkLWGfeH74S3Z_rJ!jgN++!@i=HW8 zkyjI(oPH-+-N#Qc^-mpNO`bc6r=2-<%&Wy5K1vfFJB(L_IkpS6fY^NmuL8qsgj>MD zn~BHH9WM~32_3vd=W&B)k7F9q%stJx+b_L_X-4zr^LVUMCmyCTA3sWtkvsmME?Xiy z?xOSfB=_$oY06~J-HcCq&)qcW{j;uP;?Dm}=hkq?zh&n!;m((-G-u_t|6x399Q;>A zgNpxoJNj{u|MFDH7Rhq@FCAl0dE|ddnl!oh9{Lq?@JDoR6L;C941IK`ISfdE$4S zE0AUQ8+2|Ncl_q5QkSp#AODp~(^mfP&%Au@@|TBQwoP`UU+V{6u8|)6ZA{~uKmQ*M zmrMTDU8S~8Eqi{^v0Ug&5Upcm#y7Z1(RbgZAG8jB$eRwCspQ)>5;U)oGZ&E5aeR*K z8Yt`Y0$G))Yd(Y3KH}tA4`-_QmNke5hU_|nq=xtyjwW(_o?itz>B>WM&^63bNdQ)k@-IgDHW*RW$Xo9#RzrTrCn7L2H{9Amq|qNg@#eZY=|P zCoI?2s+L)zsM%WX(NbVEY^`C>lFjIBYmJ6@DKJ0ZT4&F&WHW!dwa%QzOG!?jY_2(S zDcEzZbz*2Q!43|z))9yOP9X1Xt%DXzwY(3tl-TR=Qb_MbZYRrooh;dYYmS!U_as1(=YVB?Q_A|tNu5Ut&_q3jbfDM zoFxT^uEuH`nX3*sB%K?GuHUkweYReBwnHqh3P)~`+s3+Tj!rDA1e)8vuBv5J*IsxC zkd^~b(aGzArj08{>cnzOuy04C+C`}gb|Yz-1avxeWzev3NzcHbz_&4W@QCr$z3~w=8Ua- z`;vfG1~BP8CyLb=F7t1am~ph_#|O%$khSJ9%Vtcn)YmpgQxF?xM^_Vb+5fnpB^W0I`f%X8gb9#X{Q-yJG0{Z56aWeI&zPxnf5pdJA38bM`cYnS#x)% z`n1tFf$i)W-hGm(f9mde^=X@NcV_lFb=P`4&CI&H=IArijGwdCk&X@uQ$5xmj!~^? z#$ROCI)V-~t%L%GS#wo@U27ddR`4`3)WoB{R-4snfNrfee|kI8^bu#yDgYqOwas9# zmcb`3!kRJ`Cr=_tq)8aMt{aGtUZsqwVlj6DgCGre>AEt&x8H_in!x@uwgExIh|-mA zjdaC(29~CTVSaaF7HPbql&*9Uo8P@f)>LqCXclr}peS7_1BQ28u9PO8Eq1@`l3q9o zkfKCaO2?T?ZyA6loW<#9_c^O=m<&h}CA!ineAD@=(gbq`vyT|tiJ6#^B1$P;;qax` z55k&Q?wEh#87niLo*+n4L@65J(Nz~=Ya%7^(miLb(E>A3B@|Jjl;FU&D>o|9#7PJH z?|ago!o;WC^h=|T7PVBg(DAB}72cyUS zb(f>Bwbr!F1eTCO5fpj<{PqhY5>143p?~5ZA5H40);=@M#MYvrB6gqHbU_!GSY??i z%s=>-ciA4*zOOZHds0a(kWewZ4h(k8h(ua7HX)Au&mY~H8KY6(_cb$_&fA@QjIW-*heP3%$d!m5^AdnT}`12qA^c@!g3DOwZ5WwE2?)-yU z!)Vx#Mtxt?FzFTwK!77sy7)sMzUd->w4^bxtpM2j!b1pjgyk zGKwWGeb4)^zjy{9Es&PU1}gwg?|J#L$KJB7ett9@4M%-nGtIQr0>Fl@8-yh`-+1ed zS6r}(MeSvgSoFmH*_WPu@i?}!AB~2?;i&IxrkNg~cQ9Som98tcq)k^|eeER|Zl77t za-TVUc;DNvzVXJ%w52+#weN?+;i#{f#!Oc&z?81*N>^e~ltRS%ZI@lR{rs()HmqG! zx*}ZrI-EZ}ckJMiy>A^oofwDfC~IH)z8{VHKGT@#E5I(Ll&+MnMCl>~AV7+>Gi%mF zkU1QlKASdR0B80!YhP<$Ywi0?W2Ux45oPfxv9QolWzJPD^weBfvo4SONxP35106sAmh(e+vAs0GboFD@PvNs)jNPvarhW}0YliZEg{Gazv z+JDIpoojRVPr<*C|BTq<`6ga{5q^8^!|0cxe=rZ!zxH3%f5ZO0cQ*Z<^$Yt2{|Ek0 zyT|*F+CO@K;(owBKtGg!S^xj-Z~rga2m6nxKl9J=fBSuNKW_dLKWhJKeg^-Xe`^1? z`TyJj)8E!#>_3Y?uKrwqq3LJ#SGU>AzUO|6`nR^u&3FNN_jGOc zw)Nw`wr3yIKhgcee6IaN=ws>M{6677%)hPwx&HzC(f&u~&)6@b2kNRzBDQAP0*H73 zq%McOmRk{B3i47qRe=DA*$&odrbEJZ*pV9XXa&p@wlW~@Yfs>V{yiTtplMhgM*-Bz zsSnlq&pG;z0OUN%$~$3=g1UF+G*>+17eRbBf3=y79J}KR8owon@$1Z7MIrvvWWH)34nK2SD)GsrJ{l z1Cl#oVo3A8qY3e=aF)qzms~FG#2$LzT=gs&aVMOj>(%{y<&O0cG!nCiESl~x=^dF{ zKvj8F1K8Ng171wwM5Fh4KoQw`_c6#y$(5cAm7e}~nJ#A*fx+c9;y#&W!#VukR)ugk zKp3=+;Ut+IYn%m+r4d*<`L2h%aDnX5}^!5R|H;(34AoVWjRx(msBZvk;rCI*|~ zdOijqI@9Z{Vu!~jvHW{lBa$rnl4+!s_5sfK3bCGk-B%iDe&@-}+%fOKU|(9?V1 zHE8&@4z)Kx!RAvAs z!Wic9=o#(bg?kc-G68-m(jZ`^=XGUXb)}t(%&~sjFnV^sEX%hSy6UKC4iOhgV=BHV z2w`4g7Y=s#Vu2B_?#VQ|hP39@eArgfX>-0S+dd&^mx0*wp}>)x;c4RUgxz%;oNe?& z-7-lJ@Y^2^C;=qJsxx5|xF)*pTGhch2B&kxtn;f!7=gznk}I3}Dh}(CoMXgA5-p&kS202!l?!fT3t|HG*rIP~mS* z$Wjo}jq3}z$Qq!9yrtd3fM0N629ZM?LU$nv@Tv9b7I;D|;0H2dsA~g7Z7zp1| zB)XmrkMgF6OQr|R)HHD^TE{Y#j!~SR?b`Xt3Qs`B+x<hxexYeAjMUWdZ-*n9%(1)Wb(n2U<><7&9dwGJmrob)4%H? zlQ%z+L-^$dFhhH|@u$%97Qz?*Ynh2VG@q|?8vY&L74&fs&_b&3$x&Oyjl~LQDRRap zJU4U*R+(2Dd!G+lh8!V{pT_UJn+^1Qg6$` zqkNm(a#hWyc6SP+p5=C4HL8-m`pO`5o~`-LI?_h5CsH?F_%?nDodmz&pWR20WTpJE z?N|wSzLjMUK8E)a2tI}Lf;+;*M|h3Y(U#>)g1>zk9|Hd}oZAa2 zLYBWBoSW!Ts!RwXr^8h+U*@{9{zqS^iH)Op<;r`Uw~nc}<^$V~_i%$GFjaG?X1@E|M`h)nekvFKt`Dh-f>@|0-`Xoq)o` zx;JmzDfOV9qCx|EVpogEe0LK~tGS?5$$L_i6P$P6wIsCQaP_;d{{N=iV@+8LI}o#( zvo*Ejy=IIn{rdIQh1&q-{EuohpVOjJ^Q3lD*YTp37$^RRgn8ihpdu5{Ct%5-KO!VL zcNB6dUajXI9jkm-P|i3~GB-A(X`P1Oqqb$tcku)UJw0w3GeUijb__#QT4j%64z%EeB7S?jlWwx_7&+EEvB|6N=kV}DwnyAlX=?j`) zmU#!$*^@NIu#n_d7;WoJV@*Fbv9|yJO4;n|BNF2xy(54RyB>t~8lUOUW$&2%Nwi1y zx6JxW88>U2$#qhl^6KUbtmg9}D0o5vYDT7kWJthLGkpGnN4T>{St^_EU>4;DmLF9o zr|LqsA8_MoNLQ=}w?8u!ziSZ@PC#Y<#9uJFo-ozVo6D;<8j^1$c|qAE3ZTE5i~zmE z$BU5lw6l=EWsg^y^;8>r9qH{xfL|~PZYK#md$zZ0?o11gV<*WSW~cgy2GYGQir%wf zt4iW8D+;s*;RGrmd(-T<@2&j(Cb9xhV*l-x`TpK`xq|7p?5R%5*s!69?2c!cC*VY* z2DE^9pvOPLU!1e}wA8S8opcTJ3`NB>hY=JQnL~QFXR4K8A$BqJnoEB$wn-%u@E6Mh zCfMF4kusv3N!(aHC}4)Xs^xoOwXd%e^6pi5|DZo=Q25j+6HlJ^7FodH6y1bMROR^q zGu6)fopS`h%Sw<;ZH%TEPf+#81-#_v+@8nlR0jLcIDKQtLleOC)6yLZgC!D9X3GgS zohwU{v$jl=quD#Go^hB{`@Qw*a%`(^jyT~=q^bWgGzRj;|12J55HWdCWV}EB|K=%N z3Nq-qxJJ`>^|1MNN+q}zTB&ooE3j==AgK@^UW<^oSbeALa2peF)Th6{@sj0KyMNHZ zksk1+MXN2tv+22A%cQOGpS9)77(uP9mh+!5T5ERLvF@b}$+WvXM45Z?-kCa)fb~f1 znVbTD$Gx-0Zxc`0D@YgHakge6SL0H`-vN_x?AP0>iGH0_EE&=v83hMJgaKAI0jJXm zVxVz;X<$v6WW7}fxROO7vr#YLP;;lij5VrX{;>7kK6TtOH&6|Ar^xo>00%+u$C4@# z>!jOt6*3><171+WxoZnKDTzJtDRw+T030;yI}~uV@9fCnei^I*j>Bp&mzP2d=FPb_ zCM*l_+$LDR3B*a!A$g#>xsrZvw0lckxmMg>0aQd7tPyN=t{dgXb;Ie+T8{fZH=gdu zM7Rg9c(kg(Jg0?ARRRl=AONFKrvFj)lTY$KfT%6^6s`mk*ABGhsce*LsoD>K{z_M2 ziPpnu+lw22PfF!CoId^6n*G4H(Ix+#+N{C(da7t1BYMGEaE#PdpOLxsVD5riQXHp@OX;`S`8VnpM~)I920w~<3|mo0 zf8~Az`*?2?H&gZ&*K&bRkV@qzvMlRHXys8*Ze2+1c?5o!^+$&MHxB@4Ee5cke52R! zmn7AZtY6ST%ixgU5)%$%QcwHj7Es-Qu^kLAPwy%7pGBw_4Q9#da^W2$}axNHr03)_nw z5?yuNmXrI5HgS46)c5&}B)Tts49oU92>3xBLLy}FMUW=84DQbVq^;7_e7|(Sdz|&J z73N+M`rc2rt*oSWu#7S{*s~nH6HRHJS1SmzeXk|;CA)FI4bat3<%}nkB%;;?=F>B7ms9QSxv#@+69;@>QaR?REYX4&)=itG>rM{<{A79Rmk)`5ON#GL`*KX%}Ihk3w(RtM-WLt z?f&FLF}4N^yE!(pZ&Yj&Bc`~K0@4_}*0Om?wN|}4WJ>WL;G^H2*QpgEkGA~OET-Km zkwz|5{6dnz1U<2Pe9DNL>3g5FEIvp1jzP&2K#z~j%g6!7B;^zF+o95?fV{3mnB8*RMhCDNp>Am-3e@jNfMj?jHV$MWjk!DDKP zkAz$Y?Sr)!GUOX}qTQ5aMh|wq1uq}~joWyKl=b_LboM#wi{CMuz5x6BKlA-qy++cM01D3b7`uD z#l6M4pI;JCypO8JZ6?U&wNxR!{4oB_ zlV!x9+-&Qy6{%MQ{~yoZGkKiTSC`YS_j22~G;xUV855g2&C(zm^V!(wpcm@zn{%!g z4}JGo(sGZ1O~to-}le

UmY2RIYtNPVDpE$%vda+HD#3m z&VuXJ{BK&Qe+rBa7eq}Q(bq|tn(RrJAk|ztj2(i{d>nmQnM?;HF2k&9sA6up5tmjl z7lySlzMbifH17-m-Lwa_F&e7nOH?ESi3#ckR3tsM+jsck3`oG!uMS}|eAwVXv>}qxwq?QY%QJ0}r@^;fhuUA9W z*BVl>TGo&N004@xSiwDUXUvp51sVmqO3m)=B55aPwf@0=e}cN+$-BdKxY`YrT_4)0 z_d10#i44Q*rFr8MC>*)v$EJvz``(pb{e&*6k+b zsMz%($|1+8hn8c2?P(l@;Rb&CsZeYoCI3?2!LqjbwPXW3z4G$Qfj=cT5Yb%vY0(AX oeb?AaKtwrnc|$|zzw9vfvn^aJJ!zd)XFXqqy0000001=f@-~a#s literal 0 HcmV?d00001 diff --git a/examples/llama.android/app/src/main/res/values/colors.xml b/examples/llama.android/app/src/main/res/values/colors.xml new file mode 100644 index 000000000..ca1931bca --- /dev/null +++ b/examples/llama.android/app/src/main/res/values/colors.xml @@ -0,0 +1,10 @@ + + + #FFBB86FC + #FF6200EE + #FF3700B3 + #FF03DAC5 + #FF018786 + #FF000000 + #FFFFFFFF + diff --git a/examples/llama.android/app/src/main/res/values/strings.xml b/examples/llama.android/app/src/main/res/values/strings.xml new file mode 100644 index 000000000..7a9d314e2 --- /dev/null +++ b/examples/llama.android/app/src/main/res/values/strings.xml @@ -0,0 +1,3 @@ + + LlamaAndroid + diff --git a/examples/llama.android/app/src/main/res/values/themes.xml b/examples/llama.android/app/src/main/res/values/themes.xml new file mode 100644 index 000000000..8a24fda56 --- /dev/null +++ b/examples/llama.android/app/src/main/res/values/themes.xml @@ -0,0 +1,5 @@ + + + + + + + + + +

+ +
+
+ + + + +)LITERAL"; +unsigned int index_html_len = sizeof(index_html); diff --git a/examples/server/index.js.hpp b/examples/server/index.js.hpp index e09b3c8c5..647abe116 100644 --- a/examples/server/index.js.hpp +++ b/examples/server/index.js.hpp @@ -1,1903 +1,4 @@ -unsigned char index_js[] = { - 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x74, 0x28, 0x29, - 0x7b, 0x74, 0x68, 0x72, 0x6f, 0x77, 0x20, 0x6e, 0x65, 0x77, 0x20, 0x45, - 0x72, 0x72, 0x6f, 0x72, 0x28, 0x22, 0x43, 0x79, 0x63, 0x6c, 0x65, 0x20, - 0x64, 0x65, 0x74, 0x65, 0x63, 0x74, 0x65, 0x64, 0x22, 0x29, 0x7d, 0x63, - 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x6e, 0x3d, 0x53, 0x79, 0x6d, 0x62, 0x6f, - 0x6c, 0x2e, 0x66, 0x6f, 0x72, 0x28, 0x22, 0x70, 0x72, 0x65, 0x61, 0x63, - 0x74, 0x2d, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x73, 0x22, 0x29, 0x3b, - 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x65, 0x28, 0x29, - 0x7b, 0x69, 0x66, 0x28, 0x66, 0x3e, 0x31, 0x29, 0x7b, 0x66, 0x2d, 0x2d, - 0x3b, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x7d, 0x6c, 0x65, 0x74, 0x20, - 0x74, 0x2c, 0x6e, 0x3d, 0x21, 0x31, 0x3b, 0x77, 0x68, 0x69, 0x6c, 0x65, - 0x28, 0x76, 0x6f, 0x69, 0x64, 0x20, 0x30, 0x21, 0x3d, 0x3d, 0x6f, 0x29, - 0x7b, 0x6c, 0x65, 0x74, 0x20, 0x5f, 0x3d, 0x6f, 0x3b, 0x6f, 0x3d, 0x76, - 0x6f, 0x69, 0x64, 0x20, 0x30, 0x3b, 0x73, 0x2b, 0x2b, 0x3b, 0x77, 0x68, - 0x69, 0x6c, 0x65, 0x28, 0x76, 0x6f, 0x69, 0x64, 0x20, 0x30, 0x21, 0x3d, - 0x3d, 0x5f, 0x29, 0x7b, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x69, 0x3d, - 0x5f, 0x2e, 0x6f, 0x3b, 0x5f, 0x2e, 0x6f, 0x3d, 0x76, 0x6f, 0x69, 0x64, - 0x20, 0x30, 0x3b, 0x5f, 0x2e, 0x66, 0x26, 0x3d, 0x2d, 0x33, 0x3b, 0x69, - 0x66, 0x28, 0x21, 0x28, 0x38, 0x26, 0x5f, 0x2e, 0x66, 0x29, 0x26, 0x26, - 0x70, 0x28, 0x5f, 0x29, 0x29, 0x74, 0x72, 0x79, 0x7b, 0x5f, 0x2e, 0x63, - 0x28, 0x29, 0x7d, 0x63, 0x61, 0x74, 0x63, 0x68, 0x28, 0x65, 0x29, 0x7b, - 0x69, 0x66, 0x28, 0x21, 0x6e, 0x29, 0x7b, 0x74, 0x3d, 0x65, 0x3b, 0x6e, - 0x3d, 0x21, 0x30, 0x7d, 0x7d, 0x5f, 0x3d, 0x69, 0x7d, 0x7d, 0x73, 0x3d, - 0x30, 0x3b, 0x66, 0x2d, 0x2d, 0x3b, 0x69, 0x66, 0x28, 0x6e, 0x29, 0x74, - 0x68, 0x72, 0x6f, 0x77, 0x20, 0x74, 0x7d, 0x66, 0x75, 0x6e, 0x63, 0x74, - 0x69, 0x6f, 0x6e, 0x20, 0x5f, 0x28, 0x74, 0x29, 0x7b, 0x69, 0x66, 0x28, - 0x66, 0x3e, 0x30, 0x29, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x74, - 0x28, 0x29, 0x3b, 0x66, 0x2b, 0x2b, 0x3b, 0x74, 0x72, 0x79, 0x7b, 0x72, - 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x74, 0x28, 0x29, 0x7d, 0x66, 0x69, - 0x6e, 0x61, 0x6c, 0x6c, 0x79, 0x7b, 0x65, 0x28, 0x29, 0x7d, 0x7d, 0x6c, - 0x65, 0x74, 0x20, 0x69, 0x2c, 0x6f, 0x2c, 0x72, 0x3d, 0x30, 0x3b, 0x66, - 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x75, 0x28, 0x74, 0x29, - 0x7b, 0x69, 0x66, 0x28, 0x72, 0x3e, 0x30, 0x29, 0x72, 0x65, 0x74, 0x75, - 0x72, 0x6e, 0x20, 0x74, 0x28, 0x29, 0x3b, 0x63, 0x6f, 0x6e, 0x73, 0x74, - 0x20, 0x6e, 0x3d, 0x69, 0x3b, 0x69, 0x3d, 0x76, 0x6f, 0x69, 0x64, 0x20, - 0x30, 0x3b, 0x72, 0x2b, 0x2b, 0x3b, 0x74, 0x72, 0x79, 0x7b, 0x72, 0x65, - 0x74, 0x75, 0x72, 0x6e, 0x20, 0x74, 0x28, 0x29, 0x7d, 0x66, 0x69, 0x6e, - 0x61, 0x6c, 0x6c, 0x79, 0x7b, 0x72, 0x2d, 0x2d, 0x3b, 0x69, 0x3d, 0x6e, - 0x7d, 0x7d, 0x6c, 0x65, 0x74, 0x20, 0x66, 0x3d, 0x30, 0x2c, 0x73, 0x3d, - 0x30, 0x2c, 0x6c, 0x3d, 0x30, 0x3b, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, - 0x6f, 0x6e, 0x20, 0x63, 0x28, 0x74, 0x29, 0x7b, 0x69, 0x66, 0x28, 0x76, - 0x6f, 0x69, 0x64, 0x20, 0x30, 0x3d, 0x3d, 0x3d, 0x69, 0x29, 0x72, 0x65, - 0x74, 0x75, 0x72, 0x6e, 0x3b, 0x6c, 0x65, 0x74, 0x20, 0x6e, 0x3d, 0x74, - 0x2e, 0x6e, 0x3b, 0x69, 0x66, 0x28, 0x76, 0x6f, 0x69, 0x64, 0x20, 0x30, - 0x3d, 0x3d, 0x3d, 0x6e, 0x7c, 0x7c, 0x6e, 0x2e, 0x74, 0x21, 0x3d, 0x3d, - 0x69, 0x29, 0x7b, 0x6e, 0x3d, 0x7b, 0x69, 0x3a, 0x30, 0x2c, 0x53, 0x3a, - 0x74, 0x2c, 0x70, 0x3a, 0x69, 0x2e, 0x73, 0x2c, 0x6e, 0x3a, 0x76, 0x6f, - 0x69, 0x64, 0x20, 0x30, 0x2c, 0x74, 0x3a, 0x69, 0x2c, 0x65, 0x3a, 0x76, - 0x6f, 0x69, 0x64, 0x20, 0x30, 0x2c, 0x78, 0x3a, 0x76, 0x6f, 0x69, 0x64, - 0x20, 0x30, 0x2c, 0x72, 0x3a, 0x6e, 0x7d, 0x3b, 0x69, 0x66, 0x28, 0x76, - 0x6f, 0x69, 0x64, 0x20, 0x30, 0x21, 0x3d, 0x3d, 0x69, 0x2e, 0x73, 0x29, - 0x69, 0x2e, 0x73, 0x2e, 0x6e, 0x3d, 0x6e, 0x3b, 0x69, 0x2e, 0x73, 0x3d, - 0x6e, 0x3b, 0x74, 0x2e, 0x6e, 0x3d, 0x6e, 0x3b, 0x69, 0x66, 0x28, 0x33, - 0x32, 0x26, 0x69, 0x2e, 0x66, 0x29, 0x74, 0x2e, 0x53, 0x28, 0x6e, 0x29, - 0x3b, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x6e, 0x7d, 0x65, 0x6c, - 0x73, 0x65, 0x20, 0x69, 0x66, 0x28, 0x2d, 0x31, 0x3d, 0x3d, 0x3d, 0x6e, - 0x2e, 0x69, 0x29, 0x7b, 0x6e, 0x2e, 0x69, 0x3d, 0x30, 0x3b, 0x69, 0x66, - 0x28, 0x76, 0x6f, 0x69, 0x64, 0x20, 0x30, 0x21, 0x3d, 0x3d, 0x6e, 0x2e, - 0x6e, 0x29, 0x7b, 0x6e, 0x2e, 0x6e, 0x2e, 0x70, 0x3d, 0x6e, 0x2e, 0x70, - 0x3b, 0x69, 0x66, 0x28, 0x76, 0x6f, 0x69, 0x64, 0x20, 0x30, 0x21, 0x3d, - 0x3d, 0x6e, 0x2e, 0x70, 0x29, 0x6e, 0x2e, 0x70, 0x2e, 0x6e, 0x3d, 0x6e, - 0x2e, 0x6e, 0x3b, 0x6e, 0x2e, 0x70, 0x3d, 0x69, 0x2e, 0x73, 0x3b, 0x6e, - 0x2e, 0x6e, 0x3d, 0x76, 0x6f, 0x69, 0x64, 0x20, 0x30, 0x3b, 0x69, 0x2e, - 0x73, 0x2e, 0x6e, 0x3d, 0x6e, 0x3b, 0x69, 0x2e, 0x73, 0x3d, 0x6e, 0x7d, - 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x6e, 0x7d, 0x7d, 0x66, 0x75, - 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x68, 0x28, 0x74, 0x29, 0x7b, - 0x74, 0x68, 0x69, 0x73, 0x2e, 0x76, 0x3d, 0x74, 0x3b, 0x74, 0x68, 0x69, - 0x73, 0x2e, 0x69, 0x3d, 0x30, 0x3b, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x6e, - 0x3d, 0x76, 0x6f, 0x69, 0x64, 0x20, 0x30, 0x3b, 0x74, 0x68, 0x69, 0x73, - 0x2e, 0x74, 0x3d, 0x76, 0x6f, 0x69, 0x64, 0x20, 0x30, 0x7d, 0x68, 0x2e, - 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x74, 0x79, 0x70, 0x65, 0x2e, 0x62, 0x72, - 0x61, 0x6e, 0x64, 0x3d, 0x6e, 0x3b, 0x68, 0x2e, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x74, 0x79, 0x70, 0x65, 0x2e, 0x68, 0x3d, 0x66, 0x75, 0x6e, 0x63, - 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x29, 0x7b, 0x72, 0x65, 0x74, 0x75, 0x72, - 0x6e, 0x21, 0x30, 0x7d, 0x3b, 0x68, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, - 0x74, 0x79, 0x70, 0x65, 0x2e, 0x53, 0x3d, 0x66, 0x75, 0x6e, 0x63, 0x74, - 0x69, 0x6f, 0x6e, 0x28, 0x74, 0x29, 0x7b, 0x69, 0x66, 0x28, 0x74, 0x68, - 0x69, 0x73, 0x2e, 0x74, 0x21, 0x3d, 0x3d, 0x74, 0x26, 0x26, 0x76, 0x6f, - 0x69, 0x64, 0x20, 0x30, 0x3d, 0x3d, 0x3d, 0x74, 0x2e, 0x65, 0x29, 0x7b, - 0x74, 0x2e, 0x78, 0x3d, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x74, 0x3b, 0x69, - 0x66, 0x28, 0x76, 0x6f, 0x69, 0x64, 0x20, 0x30, 0x21, 0x3d, 0x3d, 0x74, - 0x68, 0x69, 0x73, 0x2e, 0x74, 0x29, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x74, - 0x2e, 0x65, 0x3d, 0x74, 0x3b, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x74, 0x3d, - 0x74, 0x7d, 0x7d, 0x3b, 0x68, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x74, - 0x79, 0x70, 0x65, 0x2e, 0x55, 0x3d, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, - 0x6f, 0x6e, 0x28, 0x74, 0x29, 0x7b, 0x69, 0x66, 0x28, 0x76, 0x6f, 0x69, - 0x64, 0x20, 0x30, 0x21, 0x3d, 0x3d, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x74, - 0x29, 0x7b, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x6e, 0x3d, 0x74, 0x2e, - 0x65, 0x2c, 0x65, 0x3d, 0x74, 0x2e, 0x78, 0x3b, 0x69, 0x66, 0x28, 0x76, - 0x6f, 0x69, 0x64, 0x20, 0x30, 0x21, 0x3d, 0x3d, 0x6e, 0x29, 0x7b, 0x6e, - 0x2e, 0x78, 0x3d, 0x65, 0x3b, 0x74, 0x2e, 0x65, 0x3d, 0x76, 0x6f, 0x69, - 0x64, 0x20, 0x30, 0x7d, 0x69, 0x66, 0x28, 0x76, 0x6f, 0x69, 0x64, 0x20, - 0x30, 0x21, 0x3d, 0x3d, 0x65, 0x29, 0x7b, 0x65, 0x2e, 0x65, 0x3d, 0x6e, - 0x3b, 0x74, 0x2e, 0x78, 0x3d, 0x76, 0x6f, 0x69, 0x64, 0x20, 0x30, 0x7d, - 0x69, 0x66, 0x28, 0x74, 0x3d, 0x3d, 0x3d, 0x74, 0x68, 0x69, 0x73, 0x2e, - 0x74, 0x29, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x74, 0x3d, 0x65, 0x7d, 0x7d, - 0x3b, 0x68, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x74, 0x79, 0x70, 0x65, - 0x2e, 0x73, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x62, 0x65, 0x3d, 0x66, - 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x74, 0x29, 0x7b, 0x63, - 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x6e, 0x3d, 0x74, 0x68, 0x69, 0x73, 0x3b, - 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x77, 0x28, 0x28, 0x66, 0x75, - 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x29, 0x7b, 0x63, 0x6f, 0x6e, - 0x73, 0x74, 0x20, 0x65, 0x3d, 0x6e, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, - 0x2c, 0x5f, 0x3d, 0x33, 0x32, 0x26, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x66, - 0x3b, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x66, 0x26, 0x3d, 0x2d, 0x33, 0x33, - 0x3b, 0x74, 0x72, 0x79, 0x7b, 0x74, 0x28, 0x65, 0x29, 0x7d, 0x66, 0x69, - 0x6e, 0x61, 0x6c, 0x6c, 0x79, 0x7b, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x66, - 0x7c, 0x3d, 0x5f, 0x7d, 0x7d, 0x29, 0x29, 0x7d, 0x3b, 0x68, 0x2e, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x74, 0x79, 0x70, 0x65, 0x2e, 0x76, 0x61, 0x6c, - 0x75, 0x65, 0x4f, 0x66, 0x3d, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, - 0x6e, 0x28, 0x29, 0x7b, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x74, - 0x68, 0x69, 0x73, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x7d, 0x3b, 0x68, - 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x74, 0x79, 0x70, 0x65, 0x2e, 0x74, - 0x6f, 0x53, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x3d, 0x66, 0x75, 0x6e, 0x63, - 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x29, 0x7b, 0x72, 0x65, 0x74, 0x75, 0x72, - 0x6e, 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, - 0x2b, 0x22, 0x22, 0x7d, 0x3b, 0x68, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, - 0x74, 0x79, 0x70, 0x65, 0x2e, 0x74, 0x6f, 0x4a, 0x53, 0x4f, 0x4e, 0x3d, - 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x29, 0x7b, 0x72, - 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x76, - 0x61, 0x6c, 0x75, 0x65, 0x7d, 0x3b, 0x68, 0x2e, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x74, 0x79, 0x70, 0x65, 0x2e, 0x70, 0x65, 0x65, 0x6b, 0x3d, 0x66, - 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x29, 0x7b, 0x72, 0x65, - 0x74, 0x75, 0x72, 0x6e, 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x76, 0x7d, - 0x3b, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x2e, 0x64, 0x65, 0x66, 0x69, - 0x6e, 0x65, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x79, 0x28, 0x68, - 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x74, 0x79, 0x70, 0x65, 0x2c, 0x22, - 0x76, 0x61, 0x6c, 0x75, 0x65, 0x22, 0x2c, 0x7b, 0x67, 0x65, 0x74, 0x28, - 0x29, 0x7b, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x74, 0x3d, 0x63, 0x28, - 0x74, 0x68, 0x69, 0x73, 0x29, 0x3b, 0x69, 0x66, 0x28, 0x76, 0x6f, 0x69, - 0x64, 0x20, 0x30, 0x21, 0x3d, 0x3d, 0x74, 0x29, 0x74, 0x2e, 0x69, 0x3d, - 0x74, 0x68, 0x69, 0x73, 0x2e, 0x69, 0x3b, 0x72, 0x65, 0x74, 0x75, 0x72, - 0x6e, 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x76, 0x7d, 0x2c, 0x73, 0x65, - 0x74, 0x28, 0x6e, 0x29, 0x7b, 0x69, 0x66, 0x28, 0x69, 0x20, 0x69, 0x6e, - 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x6f, 0x66, 0x20, 0x79, 0x29, 0x21, - 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x29, 0x7b, 0x74, - 0x68, 0x72, 0x6f, 0x77, 0x20, 0x6e, 0x65, 0x77, 0x20, 0x45, 0x72, 0x72, - 0x6f, 0x72, 0x28, 0x22, 0x43, 0x6f, 0x6d, 0x70, 0x75, 0x74, 0x65, 0x64, - 0x20, 0x63, 0x61, 0x6e, 0x6e, 0x6f, 0x74, 0x20, 0x68, 0x61, 0x76, 0x65, - 0x20, 0x73, 0x69, 0x64, 0x65, 0x2d, 0x65, 0x66, 0x66, 0x65, 0x63, 0x74, - 0x73, 0x22, 0x29, 0x7d, 0x28, 0x29, 0x3b, 0x69, 0x66, 0x28, 0x6e, 0x21, - 0x3d, 0x3d, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x76, 0x29, 0x7b, 0x69, 0x66, - 0x28, 0x73, 0x3e, 0x31, 0x30, 0x30, 0x29, 0x74, 0x28, 0x29, 0x3b, 0x74, - 0x68, 0x69, 0x73, 0x2e, 0x76, 0x3d, 0x6e, 0x3b, 0x74, 0x68, 0x69, 0x73, - 0x2e, 0x69, 0x2b, 0x2b, 0x3b, 0x6c, 0x2b, 0x2b, 0x3b, 0x66, 0x2b, 0x2b, - 0x3b, 0x74, 0x72, 0x79, 0x7b, 0x66, 0x6f, 0x72, 0x28, 0x6c, 0x65, 0x74, - 0x20, 0x74, 0x3d, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x74, 0x3b, 0x76, 0x6f, - 0x69, 0x64, 0x20, 0x30, 0x21, 0x3d, 0x3d, 0x74, 0x3b, 0x74, 0x3d, 0x74, - 0x2e, 0x78, 0x29, 0x74, 0x2e, 0x74, 0x2e, 0x4e, 0x28, 0x29, 0x7d, 0x66, - 0x69, 0x6e, 0x61, 0x6c, 0x6c, 0x79, 0x7b, 0x65, 0x28, 0x29, 0x7d, 0x7d, - 0x7d, 0x7d, 0x29, 0x3b, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, - 0x20, 0x61, 0x28, 0x74, 0x29, 0x7b, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, - 0x20, 0x6e, 0x65, 0x77, 0x20, 0x68, 0x28, 0x74, 0x29, 0x7d, 0x66, 0x75, - 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x70, 0x28, 0x74, 0x29, 0x7b, - 0x66, 0x6f, 0x72, 0x28, 0x6c, 0x65, 0x74, 0x20, 0x6e, 0x3d, 0x74, 0x2e, - 0x73, 0x3b, 0x76, 0x6f, 0x69, 0x64, 0x20, 0x30, 0x21, 0x3d, 0x3d, 0x6e, - 0x3b, 0x6e, 0x3d, 0x6e, 0x2e, 0x6e, 0x29, 0x69, 0x66, 0x28, 0x6e, 0x2e, - 0x53, 0x2e, 0x69, 0x21, 0x3d, 0x3d, 0x6e, 0x2e, 0x69, 0x7c, 0x7c, 0x21, - 0x6e, 0x2e, 0x53, 0x2e, 0x68, 0x28, 0x29, 0x7c, 0x7c, 0x6e, 0x2e, 0x53, - 0x2e, 0x69, 0x21, 0x3d, 0x3d, 0x6e, 0x2e, 0x69, 0x29, 0x72, 0x65, 0x74, - 0x75, 0x72, 0x6e, 0x21, 0x30, 0x3b, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, - 0x21, 0x31, 0x7d, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, - 0x64, 0x28, 0x74, 0x29, 0x7b, 0x66, 0x6f, 0x72, 0x28, 0x6c, 0x65, 0x74, - 0x20, 0x6e, 0x3d, 0x74, 0x2e, 0x73, 0x3b, 0x76, 0x6f, 0x69, 0x64, 0x20, - 0x30, 0x21, 0x3d, 0x3d, 0x6e, 0x3b, 0x6e, 0x3d, 0x6e, 0x2e, 0x6e, 0x29, - 0x7b, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x65, 0x3d, 0x6e, 0x2e, 0x53, - 0x2e, 0x6e, 0x3b, 0x69, 0x66, 0x28, 0x76, 0x6f, 0x69, 0x64, 0x20, 0x30, - 0x21, 0x3d, 0x3d, 0x65, 0x29, 0x6e, 0x2e, 0x72, 0x3d, 0x65, 0x3b, 0x6e, - 0x2e, 0x53, 0x2e, 0x6e, 0x3d, 0x6e, 0x3b, 0x6e, 0x2e, 0x69, 0x3d, 0x2d, - 0x31, 0x3b, 0x69, 0x66, 0x28, 0x76, 0x6f, 0x69, 0x64, 0x20, 0x30, 0x3d, - 0x3d, 0x3d, 0x6e, 0x2e, 0x6e, 0x29, 0x7b, 0x74, 0x2e, 0x73, 0x3d, 0x6e, - 0x3b, 0x62, 0x72, 0x65, 0x61, 0x6b, 0x7d, 0x7d, 0x7d, 0x66, 0x75, 0x6e, - 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x76, 0x28, 0x74, 0x29, 0x7b, 0x6c, - 0x65, 0x74, 0x20, 0x6e, 0x2c, 0x65, 0x3d, 0x74, 0x2e, 0x73, 0x3b, 0x77, - 0x68, 0x69, 0x6c, 0x65, 0x28, 0x76, 0x6f, 0x69, 0x64, 0x20, 0x30, 0x21, - 0x3d, 0x3d, 0x65, 0x29, 0x7b, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x74, - 0x3d, 0x65, 0x2e, 0x70, 0x3b, 0x69, 0x66, 0x28, 0x2d, 0x31, 0x3d, 0x3d, - 0x3d, 0x65, 0x2e, 0x69, 0x29, 0x7b, 0x65, 0x2e, 0x53, 0x2e, 0x55, 0x28, - 0x65, 0x29, 0x3b, 0x69, 0x66, 0x28, 0x76, 0x6f, 0x69, 0x64, 0x20, 0x30, - 0x21, 0x3d, 0x3d, 0x74, 0x29, 0x74, 0x2e, 0x6e, 0x3d, 0x65, 0x2e, 0x6e, - 0x3b, 0x69, 0x66, 0x28, 0x76, 0x6f, 0x69, 0x64, 0x20, 0x30, 0x21, 0x3d, - 0x3d, 0x65, 0x2e, 0x6e, 0x29, 0x65, 0x2e, 0x6e, 0x2e, 0x70, 0x3d, 0x74, - 0x7d, 0x65, 0x6c, 0x73, 0x65, 0x20, 0x6e, 0x3d, 0x65, 0x3b, 0x65, 0x2e, - 0x53, 0x2e, 0x6e, 0x3d, 0x65, 0x2e, 0x72, 0x3b, 0x69, 0x66, 0x28, 0x76, - 0x6f, 0x69, 0x64, 0x20, 0x30, 0x21, 0x3d, 0x3d, 0x65, 0x2e, 0x72, 0x29, - 0x65, 0x2e, 0x72, 0x3d, 0x76, 0x6f, 0x69, 0x64, 0x20, 0x30, 0x3b, 0x65, - 0x3d, 0x74, 0x7d, 0x74, 0x2e, 0x73, 0x3d, 0x6e, 0x7d, 0x66, 0x75, 0x6e, - 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x79, 0x28, 0x74, 0x29, 0x7b, 0x68, - 0x2e, 0x63, 0x61, 0x6c, 0x6c, 0x28, 0x74, 0x68, 0x69, 0x73, 0x2c, 0x76, - 0x6f, 0x69, 0x64, 0x20, 0x30, 0x29, 0x3b, 0x74, 0x68, 0x69, 0x73, 0x2e, - 0x78, 0x3d, 0x74, 0x3b, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x73, 0x3d, 0x76, - 0x6f, 0x69, 0x64, 0x20, 0x30, 0x3b, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x67, - 0x3d, 0x6c, 0x2d, 0x31, 0x3b, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x66, 0x3d, - 0x34, 0x7d, 0x28, 0x79, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x74, 0x79, - 0x70, 0x65, 0x3d, 0x6e, 0x65, 0x77, 0x20, 0x68, 0x29, 0x2e, 0x68, 0x3d, - 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x29, 0x7b, 0x74, - 0x68, 0x69, 0x73, 0x2e, 0x66, 0x26, 0x3d, 0x2d, 0x33, 0x3b, 0x69, 0x66, - 0x28, 0x31, 0x26, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x66, 0x29, 0x72, 0x65, - 0x74, 0x75, 0x72, 0x6e, 0x21, 0x31, 0x3b, 0x69, 0x66, 0x28, 0x33, 0x32, - 0x3d, 0x3d, 0x28, 0x33, 0x36, 0x26, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x66, - 0x29, 0x29, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x21, 0x30, 0x3b, 0x74, - 0x68, 0x69, 0x73, 0x2e, 0x66, 0x26, 0x3d, 0x2d, 0x35, 0x3b, 0x69, 0x66, - 0x28, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x67, 0x3d, 0x3d, 0x3d, 0x6c, 0x29, - 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x21, 0x30, 0x3b, 0x74, 0x68, 0x69, - 0x73, 0x2e, 0x67, 0x3d, 0x6c, 0x3b, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x66, - 0x7c, 0x3d, 0x31, 0x3b, 0x69, 0x66, 0x28, 0x74, 0x68, 0x69, 0x73, 0x2e, - 0x69, 0x3e, 0x30, 0x26, 0x26, 0x21, 0x70, 0x28, 0x74, 0x68, 0x69, 0x73, - 0x29, 0x29, 0x7b, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x66, 0x26, 0x3d, 0x2d, - 0x32, 0x3b, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x21, 0x30, 0x7d, 0x63, - 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x74, 0x3d, 0x69, 0x3b, 0x74, 0x72, 0x79, - 0x7b, 0x64, 0x28, 0x74, 0x68, 0x69, 0x73, 0x29, 0x3b, 0x69, 0x3d, 0x74, - 0x68, 0x69, 0x73, 0x3b, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x74, 0x3d, - 0x74, 0x68, 0x69, 0x73, 0x2e, 0x78, 0x28, 0x29, 0x3b, 0x69, 0x66, 0x28, - 0x31, 0x36, 0x26, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x66, 0x7c, 0x7c, 0x74, - 0x68, 0x69, 0x73, 0x2e, 0x76, 0x21, 0x3d, 0x3d, 0x74, 0x7c, 0x7c, 0x30, - 0x3d, 0x3d, 0x3d, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x69, 0x29, 0x7b, 0x74, - 0x68, 0x69, 0x73, 0x2e, 0x76, 0x3d, 0x74, 0x3b, 0x74, 0x68, 0x69, 0x73, - 0x2e, 0x66, 0x26, 0x3d, 0x2d, 0x31, 0x37, 0x3b, 0x74, 0x68, 0x69, 0x73, - 0x2e, 0x69, 0x2b, 0x2b, 0x7d, 0x7d, 0x63, 0x61, 0x74, 0x63, 0x68, 0x28, - 0x74, 0x29, 0x7b, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x76, 0x3d, 0x74, 0x3b, - 0x74, 0x68, 0x69, 0x73, 0x2e, 0x66, 0x7c, 0x3d, 0x31, 0x36, 0x3b, 0x74, - 0x68, 0x69, 0x73, 0x2e, 0x69, 0x2b, 0x2b, 0x7d, 0x69, 0x3d, 0x74, 0x3b, - 0x76, 0x28, 0x74, 0x68, 0x69, 0x73, 0x29, 0x3b, 0x74, 0x68, 0x69, 0x73, - 0x2e, 0x66, 0x26, 0x3d, 0x2d, 0x32, 0x3b, 0x72, 0x65, 0x74, 0x75, 0x72, - 0x6e, 0x21, 0x30, 0x7d, 0x3b, 0x79, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, - 0x74, 0x79, 0x70, 0x65, 0x2e, 0x53, 0x3d, 0x66, 0x75, 0x6e, 0x63, 0x74, - 0x69, 0x6f, 0x6e, 0x28, 0x74, 0x29, 0x7b, 0x69, 0x66, 0x28, 0x76, 0x6f, - 0x69, 0x64, 0x20, 0x30, 0x3d, 0x3d, 0x3d, 0x74, 0x68, 0x69, 0x73, 0x2e, - 0x74, 0x29, 0x7b, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x66, 0x7c, 0x3d, 0x33, - 0x36, 0x3b, 0x66, 0x6f, 0x72, 0x28, 0x6c, 0x65, 0x74, 0x20, 0x74, 0x3d, - 0x74, 0x68, 0x69, 0x73, 0x2e, 0x73, 0x3b, 0x76, 0x6f, 0x69, 0x64, 0x20, - 0x30, 0x21, 0x3d, 0x3d, 0x74, 0x3b, 0x74, 0x3d, 0x74, 0x2e, 0x6e, 0x29, - 0x74, 0x2e, 0x53, 0x2e, 0x53, 0x28, 0x74, 0x29, 0x7d, 0x68, 0x2e, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x74, 0x79, 0x70, 0x65, 0x2e, 0x53, 0x2e, 0x63, - 0x61, 0x6c, 0x6c, 0x28, 0x74, 0x68, 0x69, 0x73, 0x2c, 0x74, 0x29, 0x7d, - 0x3b, 0x79, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x74, 0x79, 0x70, 0x65, - 0x2e, 0x55, 0x3d, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x28, - 0x74, 0x29, 0x7b, 0x69, 0x66, 0x28, 0x76, 0x6f, 0x69, 0x64, 0x20, 0x30, - 0x21, 0x3d, 0x3d, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x74, 0x29, 0x7b, 0x68, - 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x74, 0x79, 0x70, 0x65, 0x2e, 0x55, - 0x2e, 0x63, 0x61, 0x6c, 0x6c, 0x28, 0x74, 0x68, 0x69, 0x73, 0x2c, 0x74, - 0x29, 0x3b, 0x69, 0x66, 0x28, 0x76, 0x6f, 0x69, 0x64, 0x20, 0x30, 0x3d, - 0x3d, 0x3d, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x74, 0x29, 0x7b, 0x74, 0x68, - 0x69, 0x73, 0x2e, 0x66, 0x26, 0x3d, 0x2d, 0x33, 0x33, 0x3b, 0x66, 0x6f, - 0x72, 0x28, 0x6c, 0x65, 0x74, 0x20, 0x74, 0x3d, 0x74, 0x68, 0x69, 0x73, - 0x2e, 0x73, 0x3b, 0x76, 0x6f, 0x69, 0x64, 0x20, 0x30, 0x21, 0x3d, 0x3d, - 0x74, 0x3b, 0x74, 0x3d, 0x74, 0x2e, 0x6e, 0x29, 0x74, 0x2e, 0x53, 0x2e, - 0x55, 0x28, 0x74, 0x29, 0x7d, 0x7d, 0x7d, 0x3b, 0x79, 0x2e, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x74, 0x79, 0x70, 0x65, 0x2e, 0x4e, 0x3d, 0x66, 0x75, - 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x29, 0x7b, 0x69, 0x66, 0x28, - 0x21, 0x28, 0x32, 0x26, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x66, 0x29, 0x29, - 0x7b, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x66, 0x7c, 0x3d, 0x36, 0x3b, 0x66, - 0x6f, 0x72, 0x28, 0x6c, 0x65, 0x74, 0x20, 0x74, 0x3d, 0x74, 0x68, 0x69, - 0x73, 0x2e, 0x74, 0x3b, 0x76, 0x6f, 0x69, 0x64, 0x20, 0x30, 0x21, 0x3d, - 0x3d, 0x74, 0x3b, 0x74, 0x3d, 0x74, 0x2e, 0x78, 0x29, 0x74, 0x2e, 0x74, - 0x2e, 0x4e, 0x28, 0x29, 0x7d, 0x7d, 0x3b, 0x79, 0x2e, 0x70, 0x72, 0x6f, - 0x74, 0x6f, 0x74, 0x79, 0x70, 0x65, 0x2e, 0x70, 0x65, 0x65, 0x6b, 0x3d, - 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x29, 0x7b, 0x69, - 0x66, 0x28, 0x21, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x68, 0x28, 0x29, 0x29, - 0x74, 0x28, 0x29, 0x3b, 0x69, 0x66, 0x28, 0x31, 0x36, 0x26, 0x74, 0x68, - 0x69, 0x73, 0x2e, 0x66, 0x29, 0x74, 0x68, 0x72, 0x6f, 0x77, 0x20, 0x74, - 0x68, 0x69, 0x73, 0x2e, 0x76, 0x3b, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, - 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x76, 0x7d, 0x3b, 0x4f, 0x62, 0x6a, - 0x65, 0x63, 0x74, 0x2e, 0x64, 0x65, 0x66, 0x69, 0x6e, 0x65, 0x50, 0x72, - 0x6f, 0x70, 0x65, 0x72, 0x74, 0x79, 0x28, 0x79, 0x2e, 0x70, 0x72, 0x6f, - 0x74, 0x6f, 0x74, 0x79, 0x70, 0x65, 0x2c, 0x22, 0x76, 0x61, 0x6c, 0x75, - 0x65, 0x22, 0x2c, 0x7b, 0x67, 0x65, 0x74, 0x28, 0x29, 0x7b, 0x69, 0x66, - 0x28, 0x31, 0x26, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x66, 0x29, 0x74, 0x28, - 0x29, 0x3b, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x6e, 0x3d, 0x63, 0x28, - 0x74, 0x68, 0x69, 0x73, 0x29, 0x3b, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x68, - 0x28, 0x29, 0x3b, 0x69, 0x66, 0x28, 0x76, 0x6f, 0x69, 0x64, 0x20, 0x30, - 0x21, 0x3d, 0x3d, 0x6e, 0x29, 0x6e, 0x2e, 0x69, 0x3d, 0x74, 0x68, 0x69, - 0x73, 0x2e, 0x69, 0x3b, 0x69, 0x66, 0x28, 0x31, 0x36, 0x26, 0x74, 0x68, - 0x69, 0x73, 0x2e, 0x66, 0x29, 0x74, 0x68, 0x72, 0x6f, 0x77, 0x20, 0x74, - 0x68, 0x69, 0x73, 0x2e, 0x76, 0x3b, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, - 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x76, 0x7d, 0x7d, 0x29, 0x3b, 0x66, - 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x6d, 0x28, 0x74, 0x29, - 0x7b, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x6e, 0x65, 0x77, 0x20, - 0x79, 0x28, 0x74, 0x29, 0x7d, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, - 0x6e, 0x20, 0x67, 0x28, 0x74, 0x29, 0x7b, 0x63, 0x6f, 0x6e, 0x73, 0x74, - 0x20, 0x6e, 0x3d, 0x74, 0x2e, 0x75, 0x3b, 0x74, 0x2e, 0x75, 0x3d, 0x76, - 0x6f, 0x69, 0x64, 0x20, 0x30, 0x3b, 0x69, 0x66, 0x28, 0x22, 0x66, 0x75, - 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x3d, 0x3d, 0x74, 0x79, 0x70, - 0x65, 0x6f, 0x66, 0x20, 0x6e, 0x29, 0x7b, 0x66, 0x2b, 0x2b, 0x3b, 0x63, - 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x5f, 0x3d, 0x69, 0x3b, 0x69, 0x3d, 0x76, - 0x6f, 0x69, 0x64, 0x20, 0x30, 0x3b, 0x74, 0x72, 0x79, 0x7b, 0x6e, 0x28, - 0x29, 0x7d, 0x63, 0x61, 0x74, 0x63, 0x68, 0x28, 0x6e, 0x29, 0x7b, 0x74, - 0x2e, 0x66, 0x26, 0x3d, 0x2d, 0x32, 0x3b, 0x74, 0x2e, 0x66, 0x7c, 0x3d, - 0x38, 0x3b, 0x62, 0x28, 0x74, 0x29, 0x3b, 0x74, 0x68, 0x72, 0x6f, 0x77, - 0x20, 0x6e, 0x7d, 0x66, 0x69, 0x6e, 0x61, 0x6c, 0x6c, 0x79, 0x7b, 0x69, - 0x3d, 0x5f, 0x3b, 0x65, 0x28, 0x29, 0x7d, 0x7d, 0x7d, 0x66, 0x75, 0x6e, - 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x62, 0x28, 0x74, 0x29, 0x7b, 0x66, - 0x6f, 0x72, 0x28, 0x6c, 0x65, 0x74, 0x20, 0x6e, 0x3d, 0x74, 0x2e, 0x73, - 0x3b, 0x76, 0x6f, 0x69, 0x64, 0x20, 0x30, 0x21, 0x3d, 0x3d, 0x6e, 0x3b, - 0x6e, 0x3d, 0x6e, 0x2e, 0x6e, 0x29, 0x6e, 0x2e, 0x53, 0x2e, 0x55, 0x28, - 0x6e, 0x29, 0x3b, 0x74, 0x2e, 0x78, 0x3d, 0x76, 0x6f, 0x69, 0x64, 0x20, - 0x30, 0x3b, 0x74, 0x2e, 0x73, 0x3d, 0x76, 0x6f, 0x69, 0x64, 0x20, 0x30, - 0x3b, 0x67, 0x28, 0x74, 0x29, 0x7d, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, - 0x6f, 0x6e, 0x20, 0x6b, 0x28, 0x74, 0x29, 0x7b, 0x69, 0x66, 0x28, 0x69, - 0x21, 0x3d, 0x3d, 0x74, 0x68, 0x69, 0x73, 0x29, 0x74, 0x68, 0x72, 0x6f, - 0x77, 0x20, 0x6e, 0x65, 0x77, 0x20, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x28, - 0x22, 0x4f, 0x75, 0x74, 0x2d, 0x6f, 0x66, 0x2d, 0x6f, 0x72, 0x64, 0x65, - 0x72, 0x20, 0x65, 0x66, 0x66, 0x65, 0x63, 0x74, 0x22, 0x29, 0x3b, 0x76, - 0x28, 0x74, 0x68, 0x69, 0x73, 0x29, 0x3b, 0x69, 0x3d, 0x74, 0x3b, 0x74, - 0x68, 0x69, 0x73, 0x2e, 0x66, 0x26, 0x3d, 0x2d, 0x32, 0x3b, 0x69, 0x66, - 0x28, 0x38, 0x26, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x66, 0x29, 0x62, 0x28, - 0x74, 0x68, 0x69, 0x73, 0x29, 0x3b, 0x65, 0x28, 0x29, 0x7d, 0x66, 0x75, - 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x53, 0x28, 0x74, 0x29, 0x7b, - 0x74, 0x68, 0x69, 0x73, 0x2e, 0x78, 0x3d, 0x74, 0x3b, 0x74, 0x68, 0x69, - 0x73, 0x2e, 0x75, 0x3d, 0x76, 0x6f, 0x69, 0x64, 0x20, 0x30, 0x3b, 0x74, - 0x68, 0x69, 0x73, 0x2e, 0x73, 0x3d, 0x76, 0x6f, 0x69, 0x64, 0x20, 0x30, - 0x3b, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x6f, 0x3d, 0x76, 0x6f, 0x69, 0x64, - 0x20, 0x30, 0x3b, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x66, 0x3d, 0x33, 0x32, - 0x7d, 0x53, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x74, 0x79, 0x70, 0x65, - 0x2e, 0x63, 0x3d, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x28, - 0x29, 0x7b, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x74, 0x3d, 0x74, 0x68, - 0x69, 0x73, 0x2e, 0x53, 0x28, 0x29, 0x3b, 0x74, 0x72, 0x79, 0x7b, 0x69, - 0x66, 0x28, 0x38, 0x26, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x66, 0x29, 0x72, - 0x65, 0x74, 0x75, 0x72, 0x6e, 0x3b, 0x69, 0x66, 0x28, 0x76, 0x6f, 0x69, - 0x64, 0x20, 0x30, 0x3d, 0x3d, 0x3d, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x78, - 0x29, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x3b, 0x63, 0x6f, 0x6e, 0x73, - 0x74, 0x20, 0x6e, 0x3d, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x78, 0x28, 0x29, - 0x3b, 0x69, 0x66, 0x28, 0x22, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, - 0x6e, 0x22, 0x3d, 0x3d, 0x74, 0x79, 0x70, 0x65, 0x6f, 0x66, 0x20, 0x6e, - 0x29, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x75, 0x3d, 0x6e, 0x7d, 0x66, 0x69, - 0x6e, 0x61, 0x6c, 0x6c, 0x79, 0x7b, 0x74, 0x28, 0x29, 0x7d, 0x7d, 0x3b, - 0x53, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x74, 0x79, 0x70, 0x65, 0x2e, - 0x53, 0x3d, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x29, - 0x7b, 0x69, 0x66, 0x28, 0x31, 0x26, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x66, - 0x29, 0x74, 0x28, 0x29, 0x3b, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x66, 0x7c, - 0x3d, 0x31, 0x3b, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x66, 0x26, 0x3d, 0x2d, - 0x39, 0x3b, 0x67, 0x28, 0x74, 0x68, 0x69, 0x73, 0x29, 0x3b, 0x64, 0x28, - 0x74, 0x68, 0x69, 0x73, 0x29, 0x3b, 0x66, 0x2b, 0x2b, 0x3b, 0x63, 0x6f, - 0x6e, 0x73, 0x74, 0x20, 0x6e, 0x3d, 0x69, 0x3b, 0x69, 0x3d, 0x74, 0x68, - 0x69, 0x73, 0x3b, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x6b, 0x2e, - 0x62, 0x69, 0x6e, 0x64, 0x28, 0x74, 0x68, 0x69, 0x73, 0x2c, 0x6e, 0x29, - 0x7d, 0x3b, 0x53, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x74, 0x79, 0x70, - 0x65, 0x2e, 0x4e, 0x3d, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, - 0x28, 0x29, 0x7b, 0x69, 0x66, 0x28, 0x21, 0x28, 0x32, 0x26, 0x74, 0x68, - 0x69, 0x73, 0x2e, 0x66, 0x29, 0x29, 0x7b, 0x74, 0x68, 0x69, 0x73, 0x2e, - 0x66, 0x7c, 0x3d, 0x32, 0x3b, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x6f, 0x3d, - 0x6f, 0x3b, 0x6f, 0x3d, 0x74, 0x68, 0x69, 0x73, 0x7d, 0x7d, 0x3b, 0x53, - 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x74, 0x79, 0x70, 0x65, 0x2e, 0x64, - 0x3d, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x29, 0x7b, - 0x74, 0x68, 0x69, 0x73, 0x2e, 0x66, 0x7c, 0x3d, 0x38, 0x3b, 0x69, 0x66, - 0x28, 0x21, 0x28, 0x31, 0x26, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x66, 0x29, - 0x29, 0x62, 0x28, 0x74, 0x68, 0x69, 0x73, 0x29, 0x7d, 0x3b, 0x66, 0x75, - 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x77, 0x28, 0x74, 0x29, 0x7b, - 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x6e, 0x3d, 0x6e, 0x65, 0x77, 0x20, - 0x53, 0x28, 0x74, 0x29, 0x3b, 0x74, 0x72, 0x79, 0x7b, 0x6e, 0x2e, 0x63, - 0x28, 0x29, 0x7d, 0x63, 0x61, 0x74, 0x63, 0x68, 0x28, 0x74, 0x29, 0x7b, - 0x6e, 0x2e, 0x64, 0x28, 0x29, 0x3b, 0x74, 0x68, 0x72, 0x6f, 0x77, 0x20, - 0x74, 0x7d, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x6e, 0x2e, 0x64, - 0x2e, 0x62, 0x69, 0x6e, 0x64, 0x28, 0x6e, 0x29, 0x7d, 0x76, 0x61, 0x72, - 0x20, 0x78, 0x2c, 0x43, 0x2c, 0x45, 0x2c, 0x55, 0x2c, 0x48, 0x2c, 0x50, - 0x2c, 0x4e, 0x2c, 0x24, 0x2c, 0x44, 0x2c, 0x54, 0x3d, 0x7b, 0x7d, 0x2c, - 0x56, 0x3d, 0x5b, 0x5d, 0x2c, 0x41, 0x3d, 0x2f, 0x61, 0x63, 0x69, 0x74, - 0x7c, 0x65, 0x78, 0x28, 0x3f, 0x3a, 0x73, 0x7c, 0x67, 0x7c, 0x6e, 0x7c, - 0x70, 0x7c, 0x24, 0x29, 0x7c, 0x72, 0x70, 0x68, 0x7c, 0x67, 0x72, 0x69, - 0x64, 0x7c, 0x6f, 0x77, 0x73, 0x7c, 0x6d, 0x6e, 0x63, 0x7c, 0x6e, 0x74, - 0x77, 0x7c, 0x69, 0x6e, 0x65, 0x5b, 0x63, 0x68, 0x5d, 0x7c, 0x7a, 0x6f, - 0x6f, 0x7c, 0x5e, 0x6f, 0x72, 0x64, 0x7c, 0x69, 0x74, 0x65, 0x72, 0x61, - 0x2f, 0x69, 0x2c, 0x46, 0x3d, 0x41, 0x72, 0x72, 0x61, 0x79, 0x2e, 0x69, - 0x73, 0x41, 0x72, 0x72, 0x61, 0x79, 0x3b, 0x66, 0x75, 0x6e, 0x63, 0x74, - 0x69, 0x6f, 0x6e, 0x20, 0x4d, 0x28, 0x74, 0x2c, 0x6e, 0x29, 0x7b, 0x66, - 0x6f, 0x72, 0x28, 0x76, 0x61, 0x72, 0x20, 0x65, 0x20, 0x69, 0x6e, 0x20, - 0x6e, 0x29, 0x74, 0x5b, 0x65, 0x5d, 0x3d, 0x6e, 0x5b, 0x65, 0x5d, 0x3b, - 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x74, 0x7d, 0x66, 0x75, 0x6e, - 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x57, 0x28, 0x74, 0x29, 0x7b, 0x76, - 0x61, 0x72, 0x20, 0x6e, 0x3d, 0x74, 0x2e, 0x70, 0x61, 0x72, 0x65, 0x6e, - 0x74, 0x4e, 0x6f, 0x64, 0x65, 0x3b, 0x6e, 0x26, 0x26, 0x6e, 0x2e, 0x72, - 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x68, 0x69, 0x6c, 0x64, 0x28, 0x74, - 0x29, 0x7d, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x4c, - 0x28, 0x74, 0x2c, 0x6e, 0x2c, 0x65, 0x29, 0x7b, 0x76, 0x61, 0x72, 0x20, - 0x5f, 0x2c, 0x69, 0x2c, 0x6f, 0x2c, 0x72, 0x3d, 0x7b, 0x7d, 0x3b, 0x66, - 0x6f, 0x72, 0x28, 0x6f, 0x20, 0x69, 0x6e, 0x20, 0x6e, 0x29, 0x22, 0x6b, - 0x65, 0x79, 0x22, 0x3d, 0x3d, 0x6f, 0x3f, 0x5f, 0x3d, 0x6e, 0x5b, 0x6f, - 0x5d, 0x3a, 0x22, 0x72, 0x65, 0x66, 0x22, 0x3d, 0x3d, 0x6f, 0x3f, 0x69, - 0x3d, 0x6e, 0x5b, 0x6f, 0x5d, 0x3a, 0x72, 0x5b, 0x6f, 0x5d, 0x3d, 0x6e, - 0x5b, 0x6f, 0x5d, 0x3b, 0x69, 0x66, 0x28, 0x61, 0x72, 0x67, 0x75, 0x6d, - 0x65, 0x6e, 0x74, 0x73, 0x2e, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x3e, - 0x32, 0x26, 0x26, 0x28, 0x72, 0x2e, 0x63, 0x68, 0x69, 0x6c, 0x64, 0x72, - 0x65, 0x6e, 0x3d, 0x61, 0x72, 0x67, 0x75, 0x6d, 0x65, 0x6e, 0x74, 0x73, - 0x2e, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x3e, 0x33, 0x3f, 0x78, 0x2e, - 0x63, 0x61, 0x6c, 0x6c, 0x28, 0x61, 0x72, 0x67, 0x75, 0x6d, 0x65, 0x6e, - 0x74, 0x73, 0x2c, 0x32, 0x29, 0x3a, 0x65, 0x29, 0x2c, 0x22, 0x66, 0x75, - 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x3d, 0x3d, 0x74, 0x79, 0x70, - 0x65, 0x6f, 0x66, 0x20, 0x74, 0x26, 0x26, 0x6e, 0x75, 0x6c, 0x6c, 0x21, - 0x3d, 0x74, 0x2e, 0x64, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x50, 0x72, - 0x6f, 0x70, 0x73, 0x29, 0x66, 0x6f, 0x72, 0x28, 0x6f, 0x20, 0x69, 0x6e, - 0x20, 0x74, 0x2e, 0x64, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x50, 0x72, - 0x6f, 0x70, 0x73, 0x29, 0x76, 0x6f, 0x69, 0x64, 0x20, 0x30, 0x3d, 0x3d, - 0x3d, 0x72, 0x5b, 0x6f, 0x5d, 0x26, 0x26, 0x28, 0x72, 0x5b, 0x6f, 0x5d, - 0x3d, 0x74, 0x2e, 0x64, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x50, 0x72, - 0x6f, 0x70, 0x73, 0x5b, 0x6f, 0x5d, 0x29, 0x3b, 0x72, 0x65, 0x74, 0x75, - 0x72, 0x6e, 0x20, 0x4f, 0x28, 0x74, 0x2c, 0x72, 0x2c, 0x5f, 0x2c, 0x69, - 0x2c, 0x6e, 0x75, 0x6c, 0x6c, 0x29, 0x7d, 0x66, 0x75, 0x6e, 0x63, 0x74, - 0x69, 0x6f, 0x6e, 0x20, 0x4f, 0x28, 0x74, 0x2c, 0x6e, 0x2c, 0x65, 0x2c, - 0x5f, 0x2c, 0x69, 0x29, 0x7b, 0x76, 0x61, 0x72, 0x20, 0x6f, 0x3d, 0x7b, - 0x74, 0x79, 0x70, 0x65, 0x3a, 0x74, 0x2c, 0x70, 0x72, 0x6f, 0x70, 0x73, - 0x3a, 0x6e, 0x2c, 0x6b, 0x65, 0x79, 0x3a, 0x65, 0x2c, 0x72, 0x65, 0x66, - 0x3a, 0x5f, 0x2c, 0x5f, 0x5f, 0x6b, 0x3a, 0x6e, 0x75, 0x6c, 0x6c, 0x2c, - 0x5f, 0x5f, 0x3a, 0x6e, 0x75, 0x6c, 0x6c, 0x2c, 0x5f, 0x5f, 0x62, 0x3a, - 0x30, 0x2c, 0x5f, 0x5f, 0x65, 0x3a, 0x6e, 0x75, 0x6c, 0x6c, 0x2c, 0x5f, - 0x5f, 0x64, 0x3a, 0x76, 0x6f, 0x69, 0x64, 0x20, 0x30, 0x2c, 0x5f, 0x5f, - 0x63, 0x3a, 0x6e, 0x75, 0x6c, 0x6c, 0x2c, 0x63, 0x6f, 0x6e, 0x73, 0x74, - 0x72, 0x75, 0x63, 0x74, 0x6f, 0x72, 0x3a, 0x76, 0x6f, 0x69, 0x64, 0x20, - 0x30, 0x2c, 0x5f, 0x5f, 0x76, 0x3a, 0x6e, 0x75, 0x6c, 0x6c, 0x3d, 0x3d, - 0x69, 0x3f, 0x2b, 0x2b, 0x45, 0x3a, 0x69, 0x2c, 0x5f, 0x5f, 0x69, 0x3a, - 0x2d, 0x31, 0x2c, 0x5f, 0x5f, 0x75, 0x3a, 0x30, 0x7d, 0x3b, 0x72, 0x65, - 0x74, 0x75, 0x72, 0x6e, 0x20, 0x6e, 0x75, 0x6c, 0x6c, 0x3d, 0x3d, 0x69, - 0x26, 0x26, 0x6e, 0x75, 0x6c, 0x6c, 0x21, 0x3d, 0x43, 0x2e, 0x76, 0x6e, - 0x6f, 0x64, 0x65, 0x26, 0x26, 0x43, 0x2e, 0x76, 0x6e, 0x6f, 0x64, 0x65, - 0x28, 0x6f, 0x29, 0x2c, 0x6f, 0x7d, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, - 0x6f, 0x6e, 0x20, 0x52, 0x28, 0x29, 0x7b, 0x72, 0x65, 0x74, 0x75, 0x72, - 0x6e, 0x7b, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x74, 0x3a, 0x6e, 0x75, - 0x6c, 0x6c, 0x7d, 0x7d, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, - 0x20, 0x6a, 0x28, 0x74, 0x29, 0x7b, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, - 0x20, 0x74, 0x2e, 0x63, 0x68, 0x69, 0x6c, 0x64, 0x72, 0x65, 0x6e, 0x7d, - 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x49, 0x28, 0x74, - 0x2c, 0x6e, 0x29, 0x7b, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x70, 0x72, 0x6f, - 0x70, 0x73, 0x3d, 0x74, 0x2c, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x63, 0x6f, - 0x6e, 0x74, 0x65, 0x78, 0x74, 0x3d, 0x6e, 0x7d, 0x66, 0x75, 0x6e, 0x63, - 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x71, 0x28, 0x74, 0x2c, 0x6e, 0x29, 0x7b, - 0x69, 0x66, 0x28, 0x6e, 0x75, 0x6c, 0x6c, 0x3d, 0x3d, 0x6e, 0x29, 0x72, - 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x74, 0x2e, 0x5f, 0x5f, 0x3f, 0x71, - 0x28, 0x74, 0x2e, 0x5f, 0x5f, 0x2c, 0x74, 0x2e, 0x5f, 0x5f, 0x69, 0x2b, - 0x31, 0x29, 0x3a, 0x6e, 0x75, 0x6c, 0x6c, 0x3b, 0x66, 0x6f, 0x72, 0x28, - 0x76, 0x61, 0x72, 0x20, 0x65, 0x3b, 0x6e, 0x3c, 0x74, 0x2e, 0x5f, 0x5f, - 0x6b, 0x2e, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x3b, 0x6e, 0x2b, 0x2b, - 0x29, 0x69, 0x66, 0x28, 0x6e, 0x75, 0x6c, 0x6c, 0x21, 0x3d, 0x28, 0x65, - 0x3d, 0x74, 0x2e, 0x5f, 0x5f, 0x6b, 0x5b, 0x6e, 0x5d, 0x29, 0x26, 0x26, - 0x6e, 0x75, 0x6c, 0x6c, 0x21, 0x3d, 0x65, 0x2e, 0x5f, 0x5f, 0x65, 0x29, - 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x65, 0x2e, 0x5f, 0x5f, 0x65, - 0x3b, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x22, 0x66, 0x75, 0x6e, 0x63, - 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x3d, 0x3d, 0x74, 0x79, 0x70, 0x65, 0x6f, - 0x66, 0x20, 0x74, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x3f, 0x71, 0x28, 0x74, - 0x29, 0x3a, 0x6e, 0x75, 0x6c, 0x6c, 0x7d, 0x66, 0x75, 0x6e, 0x63, 0x74, - 0x69, 0x6f, 0x6e, 0x20, 0x42, 0x28, 0x74, 0x29, 0x7b, 0x76, 0x61, 0x72, - 0x20, 0x6e, 0x2c, 0x65, 0x3b, 0x69, 0x66, 0x28, 0x6e, 0x75, 0x6c, 0x6c, - 0x21, 0x3d, 0x28, 0x74, 0x3d, 0x74, 0x2e, 0x5f, 0x5f, 0x29, 0x26, 0x26, - 0x6e, 0x75, 0x6c, 0x6c, 0x21, 0x3d, 0x74, 0x2e, 0x5f, 0x5f, 0x63, 0x29, - 0x7b, 0x66, 0x6f, 0x72, 0x28, 0x74, 0x2e, 0x5f, 0x5f, 0x65, 0x3d, 0x74, - 0x2e, 0x5f, 0x5f, 0x63, 0x2e, 0x62, 0x61, 0x73, 0x65, 0x3d, 0x6e, 0x75, - 0x6c, 0x6c, 0x2c, 0x6e, 0x3d, 0x30, 0x3b, 0x6e, 0x3c, 0x74, 0x2e, 0x5f, - 0x5f, 0x6b, 0x2e, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x3b, 0x6e, 0x2b, - 0x2b, 0x29, 0x69, 0x66, 0x28, 0x6e, 0x75, 0x6c, 0x6c, 0x21, 0x3d, 0x28, - 0x65, 0x3d, 0x74, 0x2e, 0x5f, 0x5f, 0x6b, 0x5b, 0x6e, 0x5d, 0x29, 0x26, - 0x26, 0x6e, 0x75, 0x6c, 0x6c, 0x21, 0x3d, 0x65, 0x2e, 0x5f, 0x5f, 0x65, - 0x29, 0x7b, 0x74, 0x2e, 0x5f, 0x5f, 0x65, 0x3d, 0x74, 0x2e, 0x5f, 0x5f, - 0x63, 0x2e, 0x62, 0x61, 0x73, 0x65, 0x3d, 0x65, 0x2e, 0x5f, 0x5f, 0x65, - 0x3b, 0x62, 0x72, 0x65, 0x61, 0x6b, 0x7d, 0x72, 0x65, 0x74, 0x75, 0x72, - 0x6e, 0x20, 0x42, 0x28, 0x74, 0x29, 0x7d, 0x7d, 0x66, 0x75, 0x6e, 0x63, - 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x47, 0x28, 0x74, 0x29, 0x7b, 0x28, 0x21, - 0x74, 0x2e, 0x5f, 0x5f, 0x64, 0x26, 0x26, 0x28, 0x74, 0x2e, 0x5f, 0x5f, - 0x64, 0x3d, 0x21, 0x30, 0x29, 0x26, 0x26, 0x48, 0x2e, 0x70, 0x75, 0x73, - 0x68, 0x28, 0x74, 0x29, 0x26, 0x26, 0x21, 0x7a, 0x2e, 0x5f, 0x5f, 0x72, - 0x2b, 0x2b, 0x7c, 0x7c, 0x50, 0x21, 0x3d, 0x3d, 0x43, 0x2e, 0x64, 0x65, - 0x62, 0x6f, 0x75, 0x6e, 0x63, 0x65, 0x52, 0x65, 0x6e, 0x64, 0x65, 0x72, - 0x69, 0x6e, 0x67, 0x29, 0x26, 0x26, 0x28, 0x28, 0x50, 0x3d, 0x43, 0x2e, - 0x64, 0x65, 0x62, 0x6f, 0x75, 0x6e, 0x63, 0x65, 0x52, 0x65, 0x6e, 0x64, - 0x65, 0x72, 0x69, 0x6e, 0x67, 0x29, 0x7c, 0x7c, 0x4e, 0x29, 0x28, 0x7a, - 0x29, 0x7d, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x7a, - 0x28, 0x29, 0x7b, 0x76, 0x61, 0x72, 0x20, 0x74, 0x2c, 0x6e, 0x2c, 0x65, - 0x2c, 0x5f, 0x2c, 0x69, 0x2c, 0x6f, 0x2c, 0x72, 0x2c, 0x75, 0x2c, 0x66, - 0x3b, 0x66, 0x6f, 0x72, 0x28, 0x48, 0x2e, 0x73, 0x6f, 0x72, 0x74, 0x28, - 0x24, 0x29, 0x3b, 0x74, 0x3d, 0x48, 0x2e, 0x73, 0x68, 0x69, 0x66, 0x74, - 0x28, 0x29, 0x3b, 0x29, 0x74, 0x2e, 0x5f, 0x5f, 0x64, 0x26, 0x26, 0x28, - 0x6e, 0x3d, 0x48, 0x2e, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x2c, 0x5f, - 0x3d, 0x76, 0x6f, 0x69, 0x64, 0x20, 0x30, 0x2c, 0x6f, 0x3d, 0x28, 0x69, - 0x3d, 0x28, 0x65, 0x3d, 0x74, 0x29, 0x2e, 0x5f, 0x5f, 0x76, 0x29, 0x2e, - 0x5f, 0x5f, 0x65, 0x2c, 0x75, 0x3d, 0x5b, 0x5d, 0x2c, 0x66, 0x3d, 0x5b, - 0x5d, 0x2c, 0x28, 0x72, 0x3d, 0x65, 0x2e, 0x5f, 0x5f, 0x50, 0x29, 0x26, - 0x26, 0x28, 0x28, 0x5f, 0x3d, 0x4d, 0x28, 0x7b, 0x7d, 0x2c, 0x69, 0x29, - 0x29, 0x2e, 0x5f, 0x5f, 0x76, 0x3d, 0x69, 0x2e, 0x5f, 0x5f, 0x76, 0x2b, - 0x31, 0x2c, 0x43, 0x2e, 0x76, 0x6e, 0x6f, 0x64, 0x65, 0x26, 0x26, 0x43, - 0x2e, 0x76, 0x6e, 0x6f, 0x64, 0x65, 0x28, 0x5f, 0x29, 0x2c, 0x5f, 0x74, - 0x28, 0x72, 0x2c, 0x5f, 0x2c, 0x69, 0x2c, 0x65, 0x2e, 0x5f, 0x5f, 0x6e, - 0x2c, 0x76, 0x6f, 0x69, 0x64, 0x20, 0x30, 0x21, 0x3d, 0x3d, 0x72, 0x2e, - 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x53, 0x56, 0x47, 0x45, 0x6c, 0x65, 0x6d, - 0x65, 0x6e, 0x74, 0x2c, 0x33, 0x32, 0x26, 0x69, 0x2e, 0x5f, 0x5f, 0x75, - 0x3f, 0x5b, 0x6f, 0x5d, 0x3a, 0x6e, 0x75, 0x6c, 0x6c, 0x2c, 0x75, 0x2c, - 0x6e, 0x75, 0x6c, 0x6c, 0x3d, 0x3d, 0x6f, 0x3f, 0x71, 0x28, 0x69, 0x29, - 0x3a, 0x6f, 0x2c, 0x21, 0x21, 0x28, 0x33, 0x32, 0x26, 0x69, 0x2e, 0x5f, - 0x5f, 0x75, 0x29, 0x2c, 0x66, 0x29, 0x2c, 0x5f, 0x2e, 0x5f, 0x5f, 0x2e, - 0x5f, 0x5f, 0x6b, 0x5b, 0x5f, 0x2e, 0x5f, 0x5f, 0x69, 0x5d, 0x3d, 0x5f, - 0x2c, 0x69, 0x74, 0x28, 0x75, 0x2c, 0x5f, 0x2c, 0x66, 0x29, 0x2c, 0x5f, - 0x2e, 0x5f, 0x5f, 0x65, 0x21, 0x3d, 0x6f, 0x26, 0x26, 0x42, 0x28, 0x5f, - 0x29, 0x29, 0x2c, 0x48, 0x2e, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x3e, - 0x6e, 0x26, 0x26, 0x48, 0x2e, 0x73, 0x6f, 0x72, 0x74, 0x28, 0x24, 0x29, - 0x29, 0x3b, 0x7a, 0x2e, 0x5f, 0x5f, 0x72, 0x3d, 0x30, 0x7d, 0x66, 0x75, - 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x4a, 0x28, 0x74, 0x2c, 0x6e, - 0x2c, 0x65, 0x2c, 0x5f, 0x2c, 0x69, 0x2c, 0x6f, 0x2c, 0x72, 0x2c, 0x75, - 0x2c, 0x66, 0x2c, 0x73, 0x2c, 0x6c, 0x29, 0x7b, 0x76, 0x61, 0x72, 0x20, - 0x63, 0x2c, 0x68, 0x2c, 0x61, 0x2c, 0x70, 0x2c, 0x64, 0x2c, 0x76, 0x3d, - 0x5f, 0x26, 0x26, 0x5f, 0x2e, 0x5f, 0x5f, 0x6b, 0x7c, 0x7c, 0x56, 0x2c, - 0x79, 0x3d, 0x6e, 0x2e, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x3b, 0x66, - 0x6f, 0x72, 0x28, 0x65, 0x2e, 0x5f, 0x5f, 0x64, 0x3d, 0x66, 0x2c, 0x4b, - 0x28, 0x65, 0x2c, 0x6e, 0x2c, 0x76, 0x29, 0x2c, 0x66, 0x3d, 0x65, 0x2e, - 0x5f, 0x5f, 0x64, 0x2c, 0x63, 0x3d, 0x30, 0x3b, 0x63, 0x3c, 0x79, 0x3b, - 0x63, 0x2b, 0x2b, 0x29, 0x6e, 0x75, 0x6c, 0x6c, 0x21, 0x3d, 0x28, 0x61, - 0x3d, 0x65, 0x2e, 0x5f, 0x5f, 0x6b, 0x5b, 0x63, 0x5d, 0x29, 0x26, 0x26, - 0x22, 0x62, 0x6f, 0x6f, 0x6c, 0x65, 0x61, 0x6e, 0x22, 0x21, 0x3d, 0x74, - 0x79, 0x70, 0x65, 0x6f, 0x66, 0x20, 0x61, 0x26, 0x26, 0x22, 0x66, 0x75, - 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x21, 0x3d, 0x74, 0x79, 0x70, - 0x65, 0x6f, 0x66, 0x20, 0x61, 0x26, 0x26, 0x28, 0x68, 0x3d, 0x2d, 0x31, - 0x3d, 0x3d, 0x3d, 0x61, 0x2e, 0x5f, 0x5f, 0x69, 0x3f, 0x54, 0x3a, 0x76, - 0x5b, 0x61, 0x2e, 0x5f, 0x5f, 0x69, 0x5d, 0x7c, 0x7c, 0x54, 0x2c, 0x61, - 0x2e, 0x5f, 0x5f, 0x69, 0x3d, 0x63, 0x2c, 0x5f, 0x74, 0x28, 0x74, 0x2c, - 0x61, 0x2c, 0x68, 0x2c, 0x69, 0x2c, 0x6f, 0x2c, 0x72, 0x2c, 0x75, 0x2c, - 0x66, 0x2c, 0x73, 0x2c, 0x6c, 0x29, 0x2c, 0x70, 0x3d, 0x61, 0x2e, 0x5f, - 0x5f, 0x65, 0x2c, 0x61, 0x2e, 0x72, 0x65, 0x66, 0x26, 0x26, 0x68, 0x2e, - 0x72, 0x65, 0x66, 0x21, 0x3d, 0x61, 0x2e, 0x72, 0x65, 0x66, 0x26, 0x26, - 0x28, 0x68, 0x2e, 0x72, 0x65, 0x66, 0x26, 0x26, 0x72, 0x74, 0x28, 0x68, - 0x2e, 0x72, 0x65, 0x66, 0x2c, 0x6e, 0x75, 0x6c, 0x6c, 0x2c, 0x61, 0x29, - 0x2c, 0x6c, 0x2e, 0x70, 0x75, 0x73, 0x68, 0x28, 0x61, 0x2e, 0x72, 0x65, - 0x66, 0x2c, 0x61, 0x2e, 0x5f, 0x5f, 0x63, 0x7c, 0x7c, 0x70, 0x2c, 0x61, - 0x29, 0x29, 0x2c, 0x6e, 0x75, 0x6c, 0x6c, 0x3d, 0x3d, 0x64, 0x26, 0x26, - 0x6e, 0x75, 0x6c, 0x6c, 0x21, 0x3d, 0x70, 0x26, 0x26, 0x28, 0x64, 0x3d, - 0x70, 0x29, 0x2c, 0x36, 0x35, 0x35, 0x33, 0x36, 0x26, 0x61, 0x2e, 0x5f, - 0x5f, 0x75, 0x7c, 0x7c, 0x68, 0x2e, 0x5f, 0x5f, 0x6b, 0x3d, 0x3d, 0x3d, - 0x61, 0x2e, 0x5f, 0x5f, 0x6b, 0x3f, 0x66, 0x3d, 0x51, 0x28, 0x61, 0x2c, - 0x66, 0x2c, 0x74, 0x29, 0x3a, 0x22, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, - 0x6f, 0x6e, 0x22, 0x3d, 0x3d, 0x74, 0x79, 0x70, 0x65, 0x6f, 0x66, 0x20, - 0x61, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x26, 0x26, 0x76, 0x6f, 0x69, 0x64, - 0x20, 0x30, 0x21, 0x3d, 0x3d, 0x61, 0x2e, 0x5f, 0x5f, 0x64, 0x3f, 0x66, - 0x3d, 0x61, 0x2e, 0x5f, 0x5f, 0x64, 0x3a, 0x70, 0x26, 0x26, 0x28, 0x66, - 0x3d, 0x70, 0x2e, 0x6e, 0x65, 0x78, 0x74, 0x53, 0x69, 0x62, 0x6c, 0x69, - 0x6e, 0x67, 0x29, 0x2c, 0x61, 0x2e, 0x5f, 0x5f, 0x64, 0x3d, 0x76, 0x6f, - 0x69, 0x64, 0x20, 0x30, 0x2c, 0x61, 0x2e, 0x5f, 0x5f, 0x75, 0x26, 0x3d, - 0x2d, 0x31, 0x39, 0x36, 0x36, 0x30, 0x39, 0x29, 0x3b, 0x65, 0x2e, 0x5f, - 0x5f, 0x64, 0x3d, 0x66, 0x2c, 0x65, 0x2e, 0x5f, 0x5f, 0x65, 0x3d, 0x64, - 0x7d, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x4b, 0x28, - 0x74, 0x2c, 0x6e, 0x2c, 0x65, 0x29, 0x7b, 0x76, 0x61, 0x72, 0x20, 0x5f, - 0x2c, 0x69, 0x2c, 0x6f, 0x2c, 0x72, 0x2c, 0x75, 0x2c, 0x66, 0x3d, 0x6e, - 0x2e, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x2c, 0x73, 0x3d, 0x65, 0x2e, - 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x2c, 0x6c, 0x3d, 0x73, 0x2c, 0x63, - 0x3d, 0x30, 0x3b, 0x66, 0x6f, 0x72, 0x28, 0x74, 0x2e, 0x5f, 0x5f, 0x6b, - 0x3d, 0x5b, 0x5d, 0x2c, 0x5f, 0x3d, 0x30, 0x3b, 0x5f, 0x3c, 0x66, 0x3b, - 0x5f, 0x2b, 0x2b, 0x29, 0x6e, 0x75, 0x6c, 0x6c, 0x21, 0x3d, 0x28, 0x69, - 0x3d, 0x74, 0x2e, 0x5f, 0x5f, 0x6b, 0x5b, 0x5f, 0x5d, 0x3d, 0x6e, 0x75, - 0x6c, 0x6c, 0x3d, 0x3d, 0x28, 0x69, 0x3d, 0x6e, 0x5b, 0x5f, 0x5d, 0x29, - 0x7c, 0x7c, 0x22, 0x62, 0x6f, 0x6f, 0x6c, 0x65, 0x61, 0x6e, 0x22, 0x3d, - 0x3d, 0x74, 0x79, 0x70, 0x65, 0x6f, 0x66, 0x20, 0x69, 0x7c, 0x7c, 0x22, - 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x3d, 0x3d, 0x74, - 0x79, 0x70, 0x65, 0x6f, 0x66, 0x20, 0x69, 0x3f, 0x6e, 0x75, 0x6c, 0x6c, - 0x3a, 0x22, 0x73, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x22, 0x3d, 0x3d, 0x74, - 0x79, 0x70, 0x65, 0x6f, 0x66, 0x20, 0x69, 0x7c, 0x7c, 0x22, 0x6e, 0x75, - 0x6d, 0x62, 0x65, 0x72, 0x22, 0x3d, 0x3d, 0x74, 0x79, 0x70, 0x65, 0x6f, - 0x66, 0x20, 0x69, 0x7c, 0x7c, 0x22, 0x62, 0x69, 0x67, 0x69, 0x6e, 0x74, - 0x22, 0x3d, 0x3d, 0x74, 0x79, 0x70, 0x65, 0x6f, 0x66, 0x20, 0x69, 0x7c, - 0x7c, 0x69, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x72, 0x75, 0x63, 0x74, - 0x6f, 0x72, 0x3d, 0x3d, 0x53, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x3f, 0x4f, - 0x28, 0x6e, 0x75, 0x6c, 0x6c, 0x2c, 0x69, 0x2c, 0x6e, 0x75, 0x6c, 0x6c, - 0x2c, 0x6e, 0x75, 0x6c, 0x6c, 0x2c, 0x69, 0x29, 0x3a, 0x46, 0x28, 0x69, - 0x29, 0x3f, 0x4f, 0x28, 0x6a, 0x2c, 0x7b, 0x63, 0x68, 0x69, 0x6c, 0x64, - 0x72, 0x65, 0x6e, 0x3a, 0x69, 0x7d, 0x2c, 0x6e, 0x75, 0x6c, 0x6c, 0x2c, - 0x6e, 0x75, 0x6c, 0x6c, 0x2c, 0x6e, 0x75, 0x6c, 0x6c, 0x29, 0x3a, 0x76, - 0x6f, 0x69, 0x64, 0x20, 0x30, 0x3d, 0x3d, 0x3d, 0x69, 0x2e, 0x63, 0x6f, - 0x6e, 0x73, 0x74, 0x72, 0x75, 0x63, 0x74, 0x6f, 0x72, 0x26, 0x26, 0x69, - 0x2e, 0x5f, 0x5f, 0x62, 0x3e, 0x30, 0x3f, 0x4f, 0x28, 0x69, 0x2e, 0x74, - 0x79, 0x70, 0x65, 0x2c, 0x69, 0x2e, 0x70, 0x72, 0x6f, 0x70, 0x73, 0x2c, - 0x69, 0x2e, 0x6b, 0x65, 0x79, 0x2c, 0x69, 0x2e, 0x72, 0x65, 0x66, 0x3f, - 0x69, 0x2e, 0x72, 0x65, 0x66, 0x3a, 0x6e, 0x75, 0x6c, 0x6c, 0x2c, 0x69, - 0x2e, 0x5f, 0x5f, 0x76, 0x29, 0x3a, 0x69, 0x29, 0x3f, 0x28, 0x69, 0x2e, - 0x5f, 0x5f, 0x3d, 0x74, 0x2c, 0x69, 0x2e, 0x5f, 0x5f, 0x62, 0x3d, 0x74, - 0x2e, 0x5f, 0x5f, 0x62, 0x2b, 0x31, 0x2c, 0x75, 0x3d, 0x59, 0x28, 0x69, - 0x2c, 0x65, 0x2c, 0x72, 0x3d, 0x5f, 0x2b, 0x63, 0x2c, 0x6c, 0x29, 0x2c, - 0x69, 0x2e, 0x5f, 0x5f, 0x69, 0x3d, 0x75, 0x2c, 0x6f, 0x3d, 0x6e, 0x75, - 0x6c, 0x6c, 0x2c, 0x2d, 0x31, 0x21, 0x3d, 0x3d, 0x75, 0x26, 0x26, 0x28, - 0x6c, 0x2d, 0x2d, 0x2c, 0x28, 0x6f, 0x3d, 0x65, 0x5b, 0x75, 0x5d, 0x29, - 0x26, 0x26, 0x28, 0x6f, 0x2e, 0x5f, 0x5f, 0x75, 0x7c, 0x3d, 0x31, 0x33, - 0x31, 0x30, 0x37, 0x32, 0x29, 0x29, 0x2c, 0x6e, 0x75, 0x6c, 0x6c, 0x3d, - 0x3d, 0x6f, 0x7c, 0x7c, 0x6e, 0x75, 0x6c, 0x6c, 0x3d, 0x3d, 0x3d, 0x6f, - 0x2e, 0x5f, 0x5f, 0x76, 0x3f, 0x28, 0x2d, 0x31, 0x3d, 0x3d, 0x75, 0x26, - 0x26, 0x63, 0x2d, 0x2d, 0x2c, 0x22, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, - 0x6f, 0x6e, 0x22, 0x21, 0x3d, 0x74, 0x79, 0x70, 0x65, 0x6f, 0x66, 0x20, - 0x69, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x26, 0x26, 0x28, 0x69, 0x2e, 0x5f, - 0x5f, 0x75, 0x7c, 0x3d, 0x36, 0x35, 0x35, 0x33, 0x36, 0x29, 0x29, 0x3a, - 0x75, 0x21, 0x3d, 0x3d, 0x72, 0x26, 0x26, 0x28, 0x75, 0x3d, 0x3d, 0x3d, - 0x72, 0x2b, 0x31, 0x3f, 0x63, 0x2b, 0x2b, 0x3a, 0x75, 0x3e, 0x72, 0x3f, - 0x6c, 0x3e, 0x66, 0x2d, 0x72, 0x3f, 0x63, 0x2b, 0x3d, 0x75, 0x2d, 0x72, - 0x3a, 0x63, 0x2d, 0x2d, 0x3a, 0x63, 0x3d, 0x75, 0x3c, 0x72, 0x26, 0x26, - 0x75, 0x3d, 0x3d, 0x72, 0x2d, 0x31, 0x3f, 0x75, 0x2d, 0x72, 0x3a, 0x30, - 0x2c, 0x75, 0x21, 0x3d, 0x3d, 0x5f, 0x2b, 0x63, 0x26, 0x26, 0x28, 0x69, - 0x2e, 0x5f, 0x5f, 0x75, 0x7c, 0x3d, 0x36, 0x35, 0x35, 0x33, 0x36, 0x29, - 0x29, 0x29, 0x3a, 0x28, 0x6f, 0x3d, 0x65, 0x5b, 0x5f, 0x5d, 0x29, 0x26, - 0x26, 0x6e, 0x75, 0x6c, 0x6c, 0x3d, 0x3d, 0x6f, 0x2e, 0x6b, 0x65, 0x79, - 0x26, 0x26, 0x6f, 0x2e, 0x5f, 0x5f, 0x65, 0x26, 0x26, 0x28, 0x6f, 0x2e, - 0x5f, 0x5f, 0x65, 0x3d, 0x3d, 0x74, 0x2e, 0x5f, 0x5f, 0x64, 0x26, 0x26, - 0x28, 0x74, 0x2e, 0x5f, 0x5f, 0x64, 0x3d, 0x71, 0x28, 0x6f, 0x29, 0x29, - 0x2c, 0x75, 0x74, 0x28, 0x6f, 0x2c, 0x6f, 0x2c, 0x21, 0x31, 0x29, 0x2c, - 0x65, 0x5b, 0x5f, 0x5d, 0x3d, 0x6e, 0x75, 0x6c, 0x6c, 0x2c, 0x6c, 0x2d, - 0x2d, 0x29, 0x3b, 0x69, 0x66, 0x28, 0x6c, 0x29, 0x66, 0x6f, 0x72, 0x28, - 0x5f, 0x3d, 0x30, 0x3b, 0x5f, 0x3c, 0x73, 0x3b, 0x5f, 0x2b, 0x2b, 0x29, - 0x6e, 0x75, 0x6c, 0x6c, 0x21, 0x3d, 0x28, 0x6f, 0x3d, 0x65, 0x5b, 0x5f, - 0x5d, 0x29, 0x26, 0x26, 0x30, 0x3d, 0x3d, 0x28, 0x31, 0x33, 0x31, 0x30, - 0x37, 0x32, 0x26, 0x6f, 0x2e, 0x5f, 0x5f, 0x75, 0x29, 0x26, 0x26, 0x28, - 0x6f, 0x2e, 0x5f, 0x5f, 0x65, 0x3d, 0x3d, 0x74, 0x2e, 0x5f, 0x5f, 0x64, - 0x26, 0x26, 0x28, 0x74, 0x2e, 0x5f, 0x5f, 0x64, 0x3d, 0x71, 0x28, 0x6f, - 0x29, 0x29, 0x2c, 0x75, 0x74, 0x28, 0x6f, 0x2c, 0x6f, 0x29, 0x29, 0x7d, - 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x51, 0x28, 0x74, - 0x2c, 0x6e, 0x2c, 0x65, 0x29, 0x7b, 0x76, 0x61, 0x72, 0x20, 0x5f, 0x2c, - 0x69, 0x3b, 0x69, 0x66, 0x28, 0x22, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, - 0x6f, 0x6e, 0x22, 0x3d, 0x3d, 0x74, 0x79, 0x70, 0x65, 0x6f, 0x66, 0x20, - 0x74, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x29, 0x7b, 0x66, 0x6f, 0x72, 0x28, - 0x5f, 0x3d, 0x74, 0x2e, 0x5f, 0x5f, 0x6b, 0x2c, 0x69, 0x3d, 0x30, 0x3b, - 0x5f, 0x26, 0x26, 0x69, 0x3c, 0x5f, 0x2e, 0x6c, 0x65, 0x6e, 0x67, 0x74, - 0x68, 0x3b, 0x69, 0x2b, 0x2b, 0x29, 0x5f, 0x5b, 0x69, 0x5d, 0x26, 0x26, - 0x28, 0x5f, 0x5b, 0x69, 0x5d, 0x2e, 0x5f, 0x5f, 0x3d, 0x74, 0x2c, 0x6e, - 0x3d, 0x51, 0x28, 0x5f, 0x5b, 0x69, 0x5d, 0x2c, 0x6e, 0x2c, 0x65, 0x29, - 0x29, 0x3b, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x6e, 0x7d, 0x72, - 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x74, 0x2e, 0x5f, 0x5f, 0x65, 0x21, - 0x3d, 0x6e, 0x26, 0x26, 0x28, 0x65, 0x2e, 0x69, 0x6e, 0x73, 0x65, 0x72, - 0x74, 0x42, 0x65, 0x66, 0x6f, 0x72, 0x65, 0x28, 0x74, 0x2e, 0x5f, 0x5f, - 0x65, 0x2c, 0x6e, 0x7c, 0x7c, 0x6e, 0x75, 0x6c, 0x6c, 0x29, 0x2c, 0x6e, - 0x3d, 0x74, 0x2e, 0x5f, 0x5f, 0x65, 0x29, 0x2c, 0x6e, 0x26, 0x26, 0x6e, - 0x2e, 0x6e, 0x65, 0x78, 0x74, 0x53, 0x69, 0x62, 0x6c, 0x69, 0x6e, 0x67, - 0x7d, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x58, 0x28, - 0x74, 0x2c, 0x6e, 0x29, 0x7b, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, - 0x6e, 0x3d, 0x6e, 0x7c, 0x7c, 0x5b, 0x5d, 0x2c, 0x6e, 0x75, 0x6c, 0x6c, - 0x3d, 0x3d, 0x74, 0x7c, 0x7c, 0x22, 0x62, 0x6f, 0x6f, 0x6c, 0x65, 0x61, - 0x6e, 0x22, 0x3d, 0x3d, 0x74, 0x79, 0x70, 0x65, 0x6f, 0x66, 0x20, 0x74, - 0x7c, 0x7c, 0x28, 0x46, 0x28, 0x74, 0x29, 0x3f, 0x74, 0x2e, 0x73, 0x6f, - 0x6d, 0x65, 0x28, 0x28, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, - 0x28, 0x74, 0x29, 0x7b, 0x58, 0x28, 0x74, 0x2c, 0x6e, 0x29, 0x7d, 0x29, - 0x29, 0x3a, 0x6e, 0x2e, 0x70, 0x75, 0x73, 0x68, 0x28, 0x74, 0x29, 0x29, - 0x2c, 0x6e, 0x7d, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, - 0x59, 0x28, 0x74, 0x2c, 0x6e, 0x2c, 0x65, 0x2c, 0x5f, 0x29, 0x7b, 0x76, - 0x61, 0x72, 0x20, 0x69, 0x3d, 0x74, 0x2e, 0x6b, 0x65, 0x79, 0x2c, 0x6f, - 0x3d, 0x74, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x2c, 0x72, 0x3d, 0x65, 0x2d, - 0x31, 0x2c, 0x75, 0x3d, 0x65, 0x2b, 0x31, 0x2c, 0x66, 0x3d, 0x6e, 0x5b, - 0x65, 0x5d, 0x3b, 0x69, 0x66, 0x28, 0x6e, 0x75, 0x6c, 0x6c, 0x3d, 0x3d, - 0x3d, 0x66, 0x7c, 0x7c, 0x66, 0x26, 0x26, 0x69, 0x3d, 0x3d, 0x66, 0x2e, - 0x6b, 0x65, 0x79, 0x26, 0x26, 0x6f, 0x3d, 0x3d, 0x3d, 0x66, 0x2e, 0x74, - 0x79, 0x70, 0x65, 0x29, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x65, - 0x3b, 0x69, 0x66, 0x28, 0x5f, 0x3e, 0x28, 0x6e, 0x75, 0x6c, 0x6c, 0x21, - 0x3d, 0x66, 0x26, 0x26, 0x30, 0x3d, 0x3d, 0x28, 0x31, 0x33, 0x31, 0x30, - 0x37, 0x32, 0x26, 0x66, 0x2e, 0x5f, 0x5f, 0x75, 0x29, 0x3f, 0x31, 0x3a, - 0x30, 0x29, 0x29, 0x66, 0x6f, 0x72, 0x28, 0x3b, 0x72, 0x3e, 0x3d, 0x30, - 0x7c, 0x7c, 0x75, 0x3c, 0x6e, 0x2e, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, - 0x3b, 0x29, 0x7b, 0x69, 0x66, 0x28, 0x72, 0x3e, 0x3d, 0x30, 0x29, 0x7b, - 0x69, 0x66, 0x28, 0x28, 0x66, 0x3d, 0x6e, 0x5b, 0x72, 0x5d, 0x29, 0x26, - 0x26, 0x30, 0x3d, 0x3d, 0x28, 0x31, 0x33, 0x31, 0x30, 0x37, 0x32, 0x26, - 0x66, 0x2e, 0x5f, 0x5f, 0x75, 0x29, 0x26, 0x26, 0x69, 0x3d, 0x3d, 0x66, - 0x2e, 0x6b, 0x65, 0x79, 0x26, 0x26, 0x6f, 0x3d, 0x3d, 0x3d, 0x66, 0x2e, - 0x74, 0x79, 0x70, 0x65, 0x29, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, - 0x72, 0x3b, 0x72, 0x2d, 0x2d, 0x7d, 0x69, 0x66, 0x28, 0x75, 0x3c, 0x6e, - 0x2e, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x29, 0x7b, 0x69, 0x66, 0x28, - 0x28, 0x66, 0x3d, 0x6e, 0x5b, 0x75, 0x5d, 0x29, 0x26, 0x26, 0x30, 0x3d, - 0x3d, 0x28, 0x31, 0x33, 0x31, 0x30, 0x37, 0x32, 0x26, 0x66, 0x2e, 0x5f, - 0x5f, 0x75, 0x29, 0x26, 0x26, 0x69, 0x3d, 0x3d, 0x66, 0x2e, 0x6b, 0x65, - 0x79, 0x26, 0x26, 0x6f, 0x3d, 0x3d, 0x3d, 0x66, 0x2e, 0x74, 0x79, 0x70, - 0x65, 0x29, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x75, 0x3b, 0x75, - 0x2b, 0x2b, 0x7d, 0x7d, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x2d, 0x31, - 0x7d, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x5a, 0x28, - 0x74, 0x2c, 0x6e, 0x2c, 0x65, 0x29, 0x7b, 0x22, 0x2d, 0x22, 0x3d, 0x3d, - 0x3d, 0x6e, 0x5b, 0x30, 0x5d, 0x3f, 0x74, 0x2e, 0x73, 0x65, 0x74, 0x50, - 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x79, 0x28, 0x6e, 0x2c, 0x6e, 0x75, - 0x6c, 0x6c, 0x3d, 0x3d, 0x65, 0x3f, 0x22, 0x22, 0x3a, 0x65, 0x29, 0x3a, - 0x74, 0x5b, 0x6e, 0x5d, 0x3d, 0x6e, 0x75, 0x6c, 0x6c, 0x3d, 0x3d, 0x65, - 0x3f, 0x22, 0x22, 0x3a, 0x22, 0x6e, 0x75, 0x6d, 0x62, 0x65, 0x72, 0x22, - 0x21, 0x3d, 0x74, 0x79, 0x70, 0x65, 0x6f, 0x66, 0x20, 0x65, 0x7c, 0x7c, - 0x41, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x28, 0x6e, 0x29, 0x3f, 0x65, 0x3a, - 0x65, 0x2b, 0x22, 0x70, 0x78, 0x22, 0x7d, 0x66, 0x75, 0x6e, 0x63, 0x74, - 0x69, 0x6f, 0x6e, 0x20, 0x74, 0x74, 0x28, 0x74, 0x2c, 0x6e, 0x2c, 0x65, - 0x2c, 0x5f, 0x2c, 0x69, 0x29, 0x7b, 0x76, 0x61, 0x72, 0x20, 0x6f, 0x3b, - 0x74, 0x3a, 0x69, 0x66, 0x28, 0x22, 0x73, 0x74, 0x79, 0x6c, 0x65, 0x22, - 0x3d, 0x3d, 0x3d, 0x6e, 0x29, 0x69, 0x66, 0x28, 0x22, 0x73, 0x74, 0x72, - 0x69, 0x6e, 0x67, 0x22, 0x3d, 0x3d, 0x74, 0x79, 0x70, 0x65, 0x6f, 0x66, - 0x20, 0x65, 0x29, 0x74, 0x2e, 0x73, 0x74, 0x79, 0x6c, 0x65, 0x2e, 0x63, - 0x73, 0x73, 0x54, 0x65, 0x78, 0x74, 0x3d, 0x65, 0x3b, 0x65, 0x6c, 0x73, - 0x65, 0x7b, 0x69, 0x66, 0x28, 0x22, 0x73, 0x74, 0x72, 0x69, 0x6e, 0x67, - 0x22, 0x3d, 0x3d, 0x74, 0x79, 0x70, 0x65, 0x6f, 0x66, 0x20, 0x5f, 0x26, - 0x26, 0x28, 0x74, 0x2e, 0x73, 0x74, 0x79, 0x6c, 0x65, 0x2e, 0x63, 0x73, - 0x73, 0x54, 0x65, 0x78, 0x74, 0x3d, 0x5f, 0x3d, 0x22, 0x22, 0x29, 0x2c, - 0x5f, 0x29, 0x66, 0x6f, 0x72, 0x28, 0x6e, 0x20, 0x69, 0x6e, 0x20, 0x5f, - 0x29, 0x65, 0x26, 0x26, 0x6e, 0x20, 0x69, 0x6e, 0x20, 0x65, 0x7c, 0x7c, - 0x5a, 0x28, 0x74, 0x2e, 0x73, 0x74, 0x79, 0x6c, 0x65, 0x2c, 0x6e, 0x2c, - 0x22, 0x22, 0x29, 0x3b, 0x69, 0x66, 0x28, 0x65, 0x29, 0x66, 0x6f, 0x72, - 0x28, 0x6e, 0x20, 0x69, 0x6e, 0x20, 0x65, 0x29, 0x5f, 0x26, 0x26, 0x65, - 0x5b, 0x6e, 0x5d, 0x3d, 0x3d, 0x3d, 0x5f, 0x5b, 0x6e, 0x5d, 0x7c, 0x7c, - 0x5a, 0x28, 0x74, 0x2e, 0x73, 0x74, 0x79, 0x6c, 0x65, 0x2c, 0x6e, 0x2c, - 0x65, 0x5b, 0x6e, 0x5d, 0x29, 0x7d, 0x65, 0x6c, 0x73, 0x65, 0x20, 0x69, - 0x66, 0x28, 0x22, 0x6f, 0x22, 0x3d, 0x3d, 0x3d, 0x6e, 0x5b, 0x30, 0x5d, - 0x26, 0x26, 0x22, 0x6e, 0x22, 0x3d, 0x3d, 0x3d, 0x6e, 0x5b, 0x31, 0x5d, - 0x29, 0x6f, 0x3d, 0x6e, 0x21, 0x3d, 0x3d, 0x28, 0x6e, 0x3d, 0x6e, 0x2e, - 0x72, 0x65, 0x70, 0x6c, 0x61, 0x63, 0x65, 0x28, 0x2f, 0x28, 0x50, 0x6f, - 0x69, 0x6e, 0x74, 0x65, 0x72, 0x43, 0x61, 0x70, 0x74, 0x75, 0x72, 0x65, - 0x29, 0x24, 0x7c, 0x43, 0x61, 0x70, 0x74, 0x75, 0x72, 0x65, 0x24, 0x2f, - 0x2c, 0x22, 0x24, 0x31, 0x22, 0x29, 0x29, 0x2c, 0x6e, 0x3d, 0x6e, 0x2e, - 0x74, 0x6f, 0x4c, 0x6f, 0x77, 0x65, 0x72, 0x43, 0x61, 0x73, 0x65, 0x28, - 0x29, 0x69, 0x6e, 0x20, 0x74, 0x3f, 0x6e, 0x2e, 0x74, 0x6f, 0x4c, 0x6f, - 0x77, 0x65, 0x72, 0x43, 0x61, 0x73, 0x65, 0x28, 0x29, 0x2e, 0x73, 0x6c, - 0x69, 0x63, 0x65, 0x28, 0x32, 0x29, 0x3a, 0x6e, 0x2e, 0x73, 0x6c, 0x69, - 0x63, 0x65, 0x28, 0x32, 0x29, 0x2c, 0x74, 0x2e, 0x6c, 0x7c, 0x7c, 0x28, - 0x74, 0x2e, 0x6c, 0x3d, 0x7b, 0x7d, 0x29, 0x2c, 0x74, 0x2e, 0x6c, 0x5b, - 0x6e, 0x2b, 0x6f, 0x5d, 0x3d, 0x65, 0x2c, 0x65, 0x3f, 0x5f, 0x3f, 0x65, - 0x2e, 0x75, 0x3d, 0x5f, 0x2e, 0x75, 0x3a, 0x28, 0x65, 0x2e, 0x75, 0x3d, - 0x44, 0x61, 0x74, 0x65, 0x2e, 0x6e, 0x6f, 0x77, 0x28, 0x29, 0x2c, 0x74, - 0x2e, 0x61, 0x64, 0x64, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x4c, 0x69, 0x73, - 0x74, 0x65, 0x6e, 0x65, 0x72, 0x28, 0x6e, 0x2c, 0x6f, 0x3f, 0x65, 0x74, - 0x3a, 0x6e, 0x74, 0x2c, 0x6f, 0x29, 0x29, 0x3a, 0x74, 0x2e, 0x72, 0x65, - 0x6d, 0x6f, 0x76, 0x65, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x4c, 0x69, 0x73, - 0x74, 0x65, 0x6e, 0x65, 0x72, 0x28, 0x6e, 0x2c, 0x6f, 0x3f, 0x65, 0x74, - 0x3a, 0x6e, 0x74, 0x2c, 0x6f, 0x29, 0x3b, 0x65, 0x6c, 0x73, 0x65, 0x7b, - 0x69, 0x66, 0x28, 0x69, 0x29, 0x6e, 0x3d, 0x6e, 0x2e, 0x72, 0x65, 0x70, - 0x6c, 0x61, 0x63, 0x65, 0x28, 0x2f, 0x78, 0x6c, 0x69, 0x6e, 0x6b, 0x28, - 0x48, 0x7c, 0x3a, 0x68, 0x29, 0x2f, 0x2c, 0x22, 0x68, 0x22, 0x29, 0x2e, - 0x72, 0x65, 0x70, 0x6c, 0x61, 0x63, 0x65, 0x28, 0x2f, 0x73, 0x4e, 0x61, - 0x6d, 0x65, 0x24, 0x2f, 0x2c, 0x22, 0x73, 0x22, 0x29, 0x3b, 0x65, 0x6c, - 0x73, 0x65, 0x20, 0x69, 0x66, 0x28, 0x22, 0x77, 0x69, 0x64, 0x74, 0x68, - 0x22, 0x21, 0x3d, 0x3d, 0x6e, 0x26, 0x26, 0x22, 0x68, 0x65, 0x69, 0x67, - 0x68, 0x74, 0x22, 0x21, 0x3d, 0x3d, 0x6e, 0x26, 0x26, 0x22, 0x68, 0x72, - 0x65, 0x66, 0x22, 0x21, 0x3d, 0x3d, 0x6e, 0x26, 0x26, 0x22, 0x6c, 0x69, - 0x73, 0x74, 0x22, 0x21, 0x3d, 0x3d, 0x6e, 0x26, 0x26, 0x22, 0x66, 0x6f, - 0x72, 0x6d, 0x22, 0x21, 0x3d, 0x3d, 0x6e, 0x26, 0x26, 0x22, 0x74, 0x61, - 0x62, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x22, 0x21, 0x3d, 0x3d, 0x6e, 0x26, - 0x26, 0x22, 0x64, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x22, 0x21, - 0x3d, 0x3d, 0x6e, 0x26, 0x26, 0x22, 0x72, 0x6f, 0x77, 0x53, 0x70, 0x61, - 0x6e, 0x22, 0x21, 0x3d, 0x3d, 0x6e, 0x26, 0x26, 0x22, 0x63, 0x6f, 0x6c, - 0x53, 0x70, 0x61, 0x6e, 0x22, 0x21, 0x3d, 0x3d, 0x6e, 0x26, 0x26, 0x22, - 0x72, 0x6f, 0x6c, 0x65, 0x22, 0x21, 0x3d, 0x3d, 0x6e, 0x26, 0x26, 0x6e, - 0x20, 0x69, 0x6e, 0x20, 0x74, 0x29, 0x74, 0x72, 0x79, 0x7b, 0x74, 0x5b, - 0x6e, 0x5d, 0x3d, 0x6e, 0x75, 0x6c, 0x6c, 0x3d, 0x3d, 0x65, 0x3f, 0x22, - 0x22, 0x3a, 0x65, 0x3b, 0x62, 0x72, 0x65, 0x61, 0x6b, 0x20, 0x74, 0x7d, - 0x63, 0x61, 0x74, 0x63, 0x68, 0x28, 0x74, 0x29, 0x7b, 0x7d, 0x22, 0x66, - 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x3d, 0x3d, 0x74, 0x79, - 0x70, 0x65, 0x6f, 0x66, 0x20, 0x65, 0x7c, 0x7c, 0x28, 0x6e, 0x75, 0x6c, - 0x6c, 0x3d, 0x3d, 0x65, 0x7c, 0x7c, 0x21, 0x31, 0x3d, 0x3d, 0x3d, 0x65, - 0x26, 0x26, 0x22, 0x2d, 0x22, 0x21, 0x3d, 0x3d, 0x6e, 0x5b, 0x34, 0x5d, - 0x3f, 0x74, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x41, 0x74, 0x74, - 0x72, 0x69, 0x62, 0x75, 0x74, 0x65, 0x28, 0x6e, 0x29, 0x3a, 0x74, 0x2e, - 0x73, 0x65, 0x74, 0x41, 0x74, 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, 0x65, - 0x28, 0x6e, 0x2c, 0x65, 0x29, 0x29, 0x7d, 0x7d, 0x66, 0x75, 0x6e, 0x63, - 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x6e, 0x74, 0x28, 0x74, 0x29, 0x7b, 0x76, - 0x61, 0x72, 0x20, 0x6e, 0x3d, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x6c, 0x5b, - 0x74, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x2b, 0x21, 0x31, 0x5d, 0x3b, 0x69, - 0x66, 0x28, 0x74, 0x2e, 0x74, 0x29, 0x7b, 0x69, 0x66, 0x28, 0x74, 0x2e, - 0x74, 0x3c, 0x3d, 0x6e, 0x2e, 0x75, 0x29, 0x72, 0x65, 0x74, 0x75, 0x72, - 0x6e, 0x7d, 0x65, 0x6c, 0x73, 0x65, 0x20, 0x74, 0x2e, 0x74, 0x3d, 0x44, - 0x61, 0x74, 0x65, 0x2e, 0x6e, 0x6f, 0x77, 0x28, 0x29, 0x3b, 0x72, 0x65, - 0x74, 0x75, 0x72, 0x6e, 0x20, 0x6e, 0x28, 0x43, 0x2e, 0x65, 0x76, 0x65, - 0x6e, 0x74, 0x3f, 0x43, 0x2e, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x28, 0x74, - 0x29, 0x3a, 0x74, 0x29, 0x7d, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, - 0x6e, 0x20, 0x65, 0x74, 0x28, 0x74, 0x29, 0x7b, 0x72, 0x65, 0x74, 0x75, - 0x72, 0x6e, 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x6c, 0x5b, 0x74, 0x2e, - 0x74, 0x79, 0x70, 0x65, 0x2b, 0x21, 0x30, 0x5d, 0x28, 0x43, 0x2e, 0x65, - 0x76, 0x65, 0x6e, 0x74, 0x3f, 0x43, 0x2e, 0x65, 0x76, 0x65, 0x6e, 0x74, - 0x28, 0x74, 0x29, 0x3a, 0x74, 0x29, 0x7d, 0x66, 0x75, 0x6e, 0x63, 0x74, - 0x69, 0x6f, 0x6e, 0x20, 0x5f, 0x74, 0x28, 0x74, 0x2c, 0x6e, 0x2c, 0x65, - 0x2c, 0x5f, 0x2c, 0x69, 0x2c, 0x6f, 0x2c, 0x72, 0x2c, 0x75, 0x2c, 0x66, - 0x2c, 0x73, 0x29, 0x7b, 0x76, 0x61, 0x72, 0x20, 0x6c, 0x2c, 0x63, 0x2c, - 0x68, 0x2c, 0x61, 0x2c, 0x70, 0x2c, 0x64, 0x2c, 0x76, 0x2c, 0x79, 0x2c, - 0x6d, 0x2c, 0x67, 0x2c, 0x62, 0x2c, 0x6b, 0x2c, 0x53, 0x2c, 0x77, 0x2c, - 0x78, 0x2c, 0x45, 0x3d, 0x6e, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x3b, 0x69, - 0x66, 0x28, 0x76, 0x6f, 0x69, 0x64, 0x20, 0x30, 0x21, 0x3d, 0x3d, 0x6e, - 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x72, 0x75, 0x63, 0x74, 0x6f, 0x72, - 0x29, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x6e, 0x75, 0x6c, 0x6c, - 0x3b, 0x31, 0x32, 0x38, 0x26, 0x65, 0x2e, 0x5f, 0x5f, 0x75, 0x26, 0x26, - 0x28, 0x66, 0x3d, 0x21, 0x21, 0x28, 0x33, 0x32, 0x26, 0x65, 0x2e, 0x5f, - 0x5f, 0x75, 0x29, 0x2c, 0x6f, 0x3d, 0x5b, 0x75, 0x3d, 0x6e, 0x2e, 0x5f, - 0x5f, 0x65, 0x3d, 0x65, 0x2e, 0x5f, 0x5f, 0x65, 0x5d, 0x29, 0x2c, 0x28, - 0x6c, 0x3d, 0x43, 0x2e, 0x5f, 0x5f, 0x62, 0x29, 0x26, 0x26, 0x6c, 0x28, - 0x6e, 0x29, 0x3b, 0x74, 0x3a, 0x69, 0x66, 0x28, 0x22, 0x66, 0x75, 0x6e, - 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x3d, 0x3d, 0x74, 0x79, 0x70, 0x65, - 0x6f, 0x66, 0x20, 0x45, 0x29, 0x74, 0x72, 0x79, 0x7b, 0x69, 0x66, 0x28, - 0x79, 0x3d, 0x6e, 0x2e, 0x70, 0x72, 0x6f, 0x70, 0x73, 0x2c, 0x6d, 0x3d, - 0x28, 0x6c, 0x3d, 0x45, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, - 0x54, 0x79, 0x70, 0x65, 0x29, 0x26, 0x26, 0x5f, 0x5b, 0x6c, 0x2e, 0x5f, - 0x5f, 0x63, 0x5d, 0x2c, 0x67, 0x3d, 0x6c, 0x3f, 0x6d, 0x3f, 0x6d, 0x2e, - 0x70, 0x72, 0x6f, 0x70, 0x73, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, - 0x6c, 0x2e, 0x5f, 0x5f, 0x3a, 0x5f, 0x2c, 0x65, 0x2e, 0x5f, 0x5f, 0x63, - 0x3f, 0x76, 0x3d, 0x28, 0x63, 0x3d, 0x6e, 0x2e, 0x5f, 0x5f, 0x63, 0x3d, - 0x65, 0x2e, 0x5f, 0x5f, 0x63, 0x29, 0x2e, 0x5f, 0x5f, 0x3d, 0x63, 0x2e, - 0x5f, 0x5f, 0x45, 0x3a, 0x28, 0x22, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x74, - 0x79, 0x70, 0x65, 0x22, 0x69, 0x6e, 0x20, 0x45, 0x26, 0x26, 0x45, 0x2e, - 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x74, 0x79, 0x70, 0x65, 0x2e, 0x72, 0x65, - 0x6e, 0x64, 0x65, 0x72, 0x3f, 0x6e, 0x2e, 0x5f, 0x5f, 0x63, 0x3d, 0x63, - 0x3d, 0x6e, 0x65, 0x77, 0x20, 0x45, 0x28, 0x79, 0x2c, 0x67, 0x29, 0x3a, - 0x28, 0x6e, 0x2e, 0x5f, 0x5f, 0x63, 0x3d, 0x63, 0x3d, 0x6e, 0x65, 0x77, - 0x20, 0x49, 0x28, 0x79, 0x2c, 0x67, 0x29, 0x2c, 0x63, 0x2e, 0x63, 0x6f, - 0x6e, 0x73, 0x74, 0x72, 0x75, 0x63, 0x74, 0x6f, 0x72, 0x3d, 0x45, 0x2c, - 0x63, 0x2e, 0x72, 0x65, 0x6e, 0x64, 0x65, 0x72, 0x3d, 0x66, 0x74, 0x29, - 0x2c, 0x6d, 0x26, 0x26, 0x6d, 0x2e, 0x73, 0x75, 0x62, 0x28, 0x63, 0x29, - 0x2c, 0x63, 0x2e, 0x70, 0x72, 0x6f, 0x70, 0x73, 0x3d, 0x79, 0x2c, 0x63, - 0x2e, 0x73, 0x74, 0x61, 0x74, 0x65, 0x7c, 0x7c, 0x28, 0x63, 0x2e, 0x73, - 0x74, 0x61, 0x74, 0x65, 0x3d, 0x7b, 0x7d, 0x29, 0x2c, 0x63, 0x2e, 0x63, - 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x3d, 0x67, 0x2c, 0x63, 0x2e, 0x5f, - 0x5f, 0x6e, 0x3d, 0x5f, 0x2c, 0x68, 0x3d, 0x63, 0x2e, 0x5f, 0x5f, 0x64, - 0x3d, 0x21, 0x30, 0x2c, 0x63, 0x2e, 0x5f, 0x5f, 0x68, 0x3d, 0x5b, 0x5d, - 0x2c, 0x63, 0x2e, 0x5f, 0x73, 0x62, 0x3d, 0x5b, 0x5d, 0x29, 0x2c, 0x6e, - 0x75, 0x6c, 0x6c, 0x3d, 0x3d, 0x63, 0x2e, 0x5f, 0x5f, 0x73, 0x26, 0x26, - 0x28, 0x63, 0x2e, 0x5f, 0x5f, 0x73, 0x3d, 0x63, 0x2e, 0x73, 0x74, 0x61, - 0x74, 0x65, 0x29, 0x2c, 0x6e, 0x75, 0x6c, 0x6c, 0x21, 0x3d, 0x45, 0x2e, - 0x67, 0x65, 0x74, 0x44, 0x65, 0x72, 0x69, 0x76, 0x65, 0x64, 0x53, 0x74, - 0x61, 0x74, 0x65, 0x46, 0x72, 0x6f, 0x6d, 0x50, 0x72, 0x6f, 0x70, 0x73, - 0x26, 0x26, 0x28, 0x63, 0x2e, 0x5f, 0x5f, 0x73, 0x3d, 0x3d, 0x63, 0x2e, - 0x73, 0x74, 0x61, 0x74, 0x65, 0x26, 0x26, 0x28, 0x63, 0x2e, 0x5f, 0x5f, - 0x73, 0x3d, 0x4d, 0x28, 0x7b, 0x7d, 0x2c, 0x63, 0x2e, 0x5f, 0x5f, 0x73, - 0x29, 0x29, 0x2c, 0x4d, 0x28, 0x63, 0x2e, 0x5f, 0x5f, 0x73, 0x2c, 0x45, - 0x2e, 0x67, 0x65, 0x74, 0x44, 0x65, 0x72, 0x69, 0x76, 0x65, 0x64, 0x53, - 0x74, 0x61, 0x74, 0x65, 0x46, 0x72, 0x6f, 0x6d, 0x50, 0x72, 0x6f, 0x70, - 0x73, 0x28, 0x79, 0x2c, 0x63, 0x2e, 0x5f, 0x5f, 0x73, 0x29, 0x29, 0x29, - 0x2c, 0x61, 0x3d, 0x63, 0x2e, 0x70, 0x72, 0x6f, 0x70, 0x73, 0x2c, 0x70, - 0x3d, 0x63, 0x2e, 0x73, 0x74, 0x61, 0x74, 0x65, 0x2c, 0x63, 0x2e, 0x5f, - 0x5f, 0x76, 0x3d, 0x6e, 0x2c, 0x68, 0x29, 0x6e, 0x75, 0x6c, 0x6c, 0x3d, - 0x3d, 0x45, 0x2e, 0x67, 0x65, 0x74, 0x44, 0x65, 0x72, 0x69, 0x76, 0x65, - 0x64, 0x53, 0x74, 0x61, 0x74, 0x65, 0x46, 0x72, 0x6f, 0x6d, 0x50, 0x72, - 0x6f, 0x70, 0x73, 0x26, 0x26, 0x6e, 0x75, 0x6c, 0x6c, 0x21, 0x3d, 0x63, - 0x2e, 0x63, 0x6f, 0x6d, 0x70, 0x6f, 0x6e, 0x65, 0x6e, 0x74, 0x57, 0x69, - 0x6c, 0x6c, 0x4d, 0x6f, 0x75, 0x6e, 0x74, 0x26, 0x26, 0x63, 0x2e, 0x63, - 0x6f, 0x6d, 0x70, 0x6f, 0x6e, 0x65, 0x6e, 0x74, 0x57, 0x69, 0x6c, 0x6c, - 0x4d, 0x6f, 0x75, 0x6e, 0x74, 0x28, 0x29, 0x2c, 0x6e, 0x75, 0x6c, 0x6c, - 0x21, 0x3d, 0x63, 0x2e, 0x63, 0x6f, 0x6d, 0x70, 0x6f, 0x6e, 0x65, 0x6e, - 0x74, 0x44, 0x69, 0x64, 0x4d, 0x6f, 0x75, 0x6e, 0x74, 0x26, 0x26, 0x63, - 0x2e, 0x5f, 0x5f, 0x68, 0x2e, 0x70, 0x75, 0x73, 0x68, 0x28, 0x63, 0x2e, - 0x63, 0x6f, 0x6d, 0x70, 0x6f, 0x6e, 0x65, 0x6e, 0x74, 0x44, 0x69, 0x64, - 0x4d, 0x6f, 0x75, 0x6e, 0x74, 0x29, 0x3b, 0x65, 0x6c, 0x73, 0x65, 0x7b, - 0x69, 0x66, 0x28, 0x6e, 0x75, 0x6c, 0x6c, 0x3d, 0x3d, 0x45, 0x2e, 0x67, - 0x65, 0x74, 0x44, 0x65, 0x72, 0x69, 0x76, 0x65, 0x64, 0x53, 0x74, 0x61, - 0x74, 0x65, 0x46, 0x72, 0x6f, 0x6d, 0x50, 0x72, 0x6f, 0x70, 0x73, 0x26, - 0x26, 0x79, 0x21, 0x3d, 0x3d, 0x61, 0x26, 0x26, 0x6e, 0x75, 0x6c, 0x6c, - 0x21, 0x3d, 0x63, 0x2e, 0x63, 0x6f, 0x6d, 0x70, 0x6f, 0x6e, 0x65, 0x6e, - 0x74, 0x57, 0x69, 0x6c, 0x6c, 0x52, 0x65, 0x63, 0x65, 0x69, 0x76, 0x65, - 0x50, 0x72, 0x6f, 0x70, 0x73, 0x26, 0x26, 0x63, 0x2e, 0x63, 0x6f, 0x6d, - 0x70, 0x6f, 0x6e, 0x65, 0x6e, 0x74, 0x57, 0x69, 0x6c, 0x6c, 0x52, 0x65, - 0x63, 0x65, 0x69, 0x76, 0x65, 0x50, 0x72, 0x6f, 0x70, 0x73, 0x28, 0x79, - 0x2c, 0x67, 0x29, 0x2c, 0x21, 0x63, 0x2e, 0x5f, 0x5f, 0x65, 0x26, 0x26, - 0x28, 0x6e, 0x75, 0x6c, 0x6c, 0x21, 0x3d, 0x63, 0x2e, 0x73, 0x68, 0x6f, - 0x75, 0x6c, 0x64, 0x43, 0x6f, 0x6d, 0x70, 0x6f, 0x6e, 0x65, 0x6e, 0x74, - 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x26, 0x26, 0x21, 0x31, 0x3d, 0x3d, - 0x3d, 0x63, 0x2e, 0x73, 0x68, 0x6f, 0x75, 0x6c, 0x64, 0x43, 0x6f, 0x6d, - 0x70, 0x6f, 0x6e, 0x65, 0x6e, 0x74, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, - 0x28, 0x79, 0x2c, 0x63, 0x2e, 0x5f, 0x5f, 0x73, 0x2c, 0x67, 0x29, 0x7c, - 0x7c, 0x6e, 0x2e, 0x5f, 0x5f, 0x76, 0x3d, 0x3d, 0x3d, 0x65, 0x2e, 0x5f, - 0x5f, 0x76, 0x29, 0x29, 0x7b, 0x66, 0x6f, 0x72, 0x28, 0x6e, 0x2e, 0x5f, - 0x5f, 0x76, 0x21, 0x3d, 0x3d, 0x65, 0x2e, 0x5f, 0x5f, 0x76, 0x26, 0x26, - 0x28, 0x63, 0x2e, 0x70, 0x72, 0x6f, 0x70, 0x73, 0x3d, 0x79, 0x2c, 0x63, - 0x2e, 0x73, 0x74, 0x61, 0x74, 0x65, 0x3d, 0x63, 0x2e, 0x5f, 0x5f, 0x73, - 0x2c, 0x63, 0x2e, 0x5f, 0x5f, 0x64, 0x3d, 0x21, 0x31, 0x29, 0x2c, 0x6e, - 0x2e, 0x5f, 0x5f, 0x65, 0x3d, 0x65, 0x2e, 0x5f, 0x5f, 0x65, 0x2c, 0x6e, - 0x2e, 0x5f, 0x5f, 0x6b, 0x3d, 0x65, 0x2e, 0x5f, 0x5f, 0x6b, 0x2c, 0x6e, - 0x2e, 0x5f, 0x5f, 0x6b, 0x2e, 0x66, 0x6f, 0x72, 0x45, 0x61, 0x63, 0x68, - 0x28, 0x28, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x74, - 0x29, 0x7b, 0x74, 0x26, 0x26, 0x28, 0x74, 0x2e, 0x5f, 0x5f, 0x3d, 0x6e, - 0x29, 0x7d, 0x29, 0x29, 0x2c, 0x62, 0x3d, 0x30, 0x3b, 0x62, 0x3c, 0x63, - 0x2e, 0x5f, 0x73, 0x62, 0x2e, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x3b, - 0x62, 0x2b, 0x2b, 0x29, 0x63, 0x2e, 0x5f, 0x5f, 0x68, 0x2e, 0x70, 0x75, - 0x73, 0x68, 0x28, 0x63, 0x2e, 0x5f, 0x73, 0x62, 0x5b, 0x62, 0x5d, 0x29, - 0x3b, 0x63, 0x2e, 0x5f, 0x73, 0x62, 0x3d, 0x5b, 0x5d, 0x2c, 0x63, 0x2e, - 0x5f, 0x5f, 0x68, 0x2e, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x26, 0x26, - 0x72, 0x2e, 0x70, 0x75, 0x73, 0x68, 0x28, 0x63, 0x29, 0x3b, 0x62, 0x72, - 0x65, 0x61, 0x6b, 0x20, 0x74, 0x7d, 0x6e, 0x75, 0x6c, 0x6c, 0x21, 0x3d, - 0x63, 0x2e, 0x63, 0x6f, 0x6d, 0x70, 0x6f, 0x6e, 0x65, 0x6e, 0x74, 0x57, - 0x69, 0x6c, 0x6c, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x26, 0x26, 0x63, - 0x2e, 0x63, 0x6f, 0x6d, 0x70, 0x6f, 0x6e, 0x65, 0x6e, 0x74, 0x57, 0x69, - 0x6c, 0x6c, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x28, 0x79, 0x2c, 0x63, - 0x2e, 0x5f, 0x5f, 0x73, 0x2c, 0x67, 0x29, 0x2c, 0x6e, 0x75, 0x6c, 0x6c, - 0x21, 0x3d, 0x63, 0x2e, 0x63, 0x6f, 0x6d, 0x70, 0x6f, 0x6e, 0x65, 0x6e, - 0x74, 0x44, 0x69, 0x64, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x26, 0x26, - 0x63, 0x2e, 0x5f, 0x5f, 0x68, 0x2e, 0x70, 0x75, 0x73, 0x68, 0x28, 0x28, - 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x29, 0x7b, 0x63, - 0x2e, 0x63, 0x6f, 0x6d, 0x70, 0x6f, 0x6e, 0x65, 0x6e, 0x74, 0x44, 0x69, - 0x64, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x28, 0x61, 0x2c, 0x70, 0x2c, - 0x64, 0x29, 0x7d, 0x29, 0x29, 0x7d, 0x69, 0x66, 0x28, 0x63, 0x2e, 0x63, - 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x3d, 0x67, 0x2c, 0x63, 0x2e, 0x70, - 0x72, 0x6f, 0x70, 0x73, 0x3d, 0x79, 0x2c, 0x63, 0x2e, 0x5f, 0x5f, 0x50, - 0x3d, 0x74, 0x2c, 0x63, 0x2e, 0x5f, 0x5f, 0x65, 0x3d, 0x21, 0x31, 0x2c, - 0x6b, 0x3d, 0x43, 0x2e, 0x5f, 0x5f, 0x72, 0x2c, 0x53, 0x3d, 0x30, 0x2c, - 0x22, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x74, 0x79, 0x70, 0x65, 0x22, 0x69, - 0x6e, 0x20, 0x45, 0x26, 0x26, 0x45, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, - 0x74, 0x79, 0x70, 0x65, 0x2e, 0x72, 0x65, 0x6e, 0x64, 0x65, 0x72, 0x29, - 0x7b, 0x66, 0x6f, 0x72, 0x28, 0x63, 0x2e, 0x73, 0x74, 0x61, 0x74, 0x65, - 0x3d, 0x63, 0x2e, 0x5f, 0x5f, 0x73, 0x2c, 0x63, 0x2e, 0x5f, 0x5f, 0x64, - 0x3d, 0x21, 0x31, 0x2c, 0x6b, 0x26, 0x26, 0x6b, 0x28, 0x6e, 0x29, 0x2c, - 0x6c, 0x3d, 0x63, 0x2e, 0x72, 0x65, 0x6e, 0x64, 0x65, 0x72, 0x28, 0x63, - 0x2e, 0x70, 0x72, 0x6f, 0x70, 0x73, 0x2c, 0x63, 0x2e, 0x73, 0x74, 0x61, - 0x74, 0x65, 0x2c, 0x63, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, - 0x29, 0x2c, 0x77, 0x3d, 0x30, 0x3b, 0x77, 0x3c, 0x63, 0x2e, 0x5f, 0x73, - 0x62, 0x2e, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x3b, 0x77, 0x2b, 0x2b, - 0x29, 0x63, 0x2e, 0x5f, 0x5f, 0x68, 0x2e, 0x70, 0x75, 0x73, 0x68, 0x28, - 0x63, 0x2e, 0x5f, 0x73, 0x62, 0x5b, 0x77, 0x5d, 0x29, 0x3b, 0x63, 0x2e, - 0x5f, 0x73, 0x62, 0x3d, 0x5b, 0x5d, 0x7d, 0x65, 0x6c, 0x73, 0x65, 0x20, - 0x64, 0x6f, 0x7b, 0x63, 0x2e, 0x5f, 0x5f, 0x64, 0x3d, 0x21, 0x31, 0x2c, - 0x6b, 0x26, 0x26, 0x6b, 0x28, 0x6e, 0x29, 0x2c, 0x6c, 0x3d, 0x63, 0x2e, - 0x72, 0x65, 0x6e, 0x64, 0x65, 0x72, 0x28, 0x63, 0x2e, 0x70, 0x72, 0x6f, - 0x70, 0x73, 0x2c, 0x63, 0x2e, 0x73, 0x74, 0x61, 0x74, 0x65, 0x2c, 0x63, - 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x29, 0x2c, 0x63, 0x2e, - 0x73, 0x74, 0x61, 0x74, 0x65, 0x3d, 0x63, 0x2e, 0x5f, 0x5f, 0x73, 0x7d, - 0x77, 0x68, 0x69, 0x6c, 0x65, 0x28, 0x63, 0x2e, 0x5f, 0x5f, 0x64, 0x26, - 0x26, 0x2b, 0x2b, 0x53, 0x3c, 0x32, 0x35, 0x29, 0x3b, 0x63, 0x2e, 0x73, - 0x74, 0x61, 0x74, 0x65, 0x3d, 0x63, 0x2e, 0x5f, 0x5f, 0x73, 0x2c, 0x6e, - 0x75, 0x6c, 0x6c, 0x21, 0x3d, 0x63, 0x2e, 0x67, 0x65, 0x74, 0x43, 0x68, - 0x69, 0x6c, 0x64, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x26, 0x26, - 0x28, 0x5f, 0x3d, 0x4d, 0x28, 0x4d, 0x28, 0x7b, 0x7d, 0x2c, 0x5f, 0x29, - 0x2c, 0x63, 0x2e, 0x67, 0x65, 0x74, 0x43, 0x68, 0x69, 0x6c, 0x64, 0x43, - 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x28, 0x29, 0x29, 0x29, 0x2c, 0x68, - 0x7c, 0x7c, 0x6e, 0x75, 0x6c, 0x6c, 0x3d, 0x3d, 0x63, 0x2e, 0x67, 0x65, - 0x74, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x42, 0x65, 0x66, - 0x6f, 0x72, 0x65, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x7c, 0x7c, 0x28, - 0x64, 0x3d, 0x63, 0x2e, 0x67, 0x65, 0x74, 0x53, 0x6e, 0x61, 0x70, 0x73, - 0x68, 0x6f, 0x74, 0x42, 0x65, 0x66, 0x6f, 0x72, 0x65, 0x55, 0x70, 0x64, - 0x61, 0x74, 0x65, 0x28, 0x61, 0x2c, 0x70, 0x29, 0x29, 0x2c, 0x4a, 0x28, - 0x74, 0x2c, 0x46, 0x28, 0x78, 0x3d, 0x6e, 0x75, 0x6c, 0x6c, 0x21, 0x3d, - 0x6c, 0x26, 0x26, 0x6c, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x3d, 0x3d, 0x3d, - 0x6a, 0x26, 0x26, 0x6e, 0x75, 0x6c, 0x6c, 0x3d, 0x3d, 0x6c, 0x2e, 0x6b, - 0x65, 0x79, 0x3f, 0x6c, 0x2e, 0x70, 0x72, 0x6f, 0x70, 0x73, 0x2e, 0x63, - 0x68, 0x69, 0x6c, 0x64, 0x72, 0x65, 0x6e, 0x3a, 0x6c, 0x29, 0x3f, 0x78, - 0x3a, 0x5b, 0x78, 0x5d, 0x2c, 0x6e, 0x2c, 0x65, 0x2c, 0x5f, 0x2c, 0x69, - 0x2c, 0x6f, 0x2c, 0x72, 0x2c, 0x75, 0x2c, 0x66, 0x2c, 0x73, 0x29, 0x2c, - 0x63, 0x2e, 0x62, 0x61, 0x73, 0x65, 0x3d, 0x6e, 0x2e, 0x5f, 0x5f, 0x65, - 0x2c, 0x6e, 0x2e, 0x5f, 0x5f, 0x75, 0x26, 0x3d, 0x2d, 0x31, 0x36, 0x31, - 0x2c, 0x63, 0x2e, 0x5f, 0x5f, 0x68, 0x2e, 0x6c, 0x65, 0x6e, 0x67, 0x74, - 0x68, 0x26, 0x26, 0x72, 0x2e, 0x70, 0x75, 0x73, 0x68, 0x28, 0x63, 0x29, - 0x2c, 0x76, 0x26, 0x26, 0x28, 0x63, 0x2e, 0x5f, 0x5f, 0x45, 0x3d, 0x63, - 0x2e, 0x5f, 0x5f, 0x3d, 0x6e, 0x75, 0x6c, 0x6c, 0x29, 0x7d, 0x63, 0x61, - 0x74, 0x63, 0x68, 0x28, 0x74, 0x29, 0x7b, 0x6e, 0x2e, 0x5f, 0x5f, 0x76, - 0x3d, 0x6e, 0x75, 0x6c, 0x6c, 0x2c, 0x66, 0x7c, 0x7c, 0x6e, 0x75, 0x6c, - 0x6c, 0x21, 0x3d, 0x6f, 0x3f, 0x28, 0x6e, 0x2e, 0x5f, 0x5f, 0x65, 0x3d, - 0x75, 0x2c, 0x6e, 0x2e, 0x5f, 0x5f, 0x75, 0x7c, 0x3d, 0x66, 0x3f, 0x31, - 0x36, 0x30, 0x3a, 0x33, 0x32, 0x2c, 0x6f, 0x5b, 0x6f, 0x2e, 0x69, 0x6e, - 0x64, 0x65, 0x78, 0x4f, 0x66, 0x28, 0x75, 0x29, 0x5d, 0x3d, 0x6e, 0x75, - 0x6c, 0x6c, 0x29, 0x3a, 0x28, 0x6e, 0x2e, 0x5f, 0x5f, 0x65, 0x3d, 0x65, - 0x2e, 0x5f, 0x5f, 0x65, 0x2c, 0x6e, 0x2e, 0x5f, 0x5f, 0x6b, 0x3d, 0x65, - 0x2e, 0x5f, 0x5f, 0x6b, 0x29, 0x2c, 0x43, 0x2e, 0x5f, 0x5f, 0x65, 0x28, - 0x74, 0x2c, 0x6e, 0x2c, 0x65, 0x29, 0x7d, 0x65, 0x6c, 0x73, 0x65, 0x20, - 0x6e, 0x75, 0x6c, 0x6c, 0x3d, 0x3d, 0x6f, 0x26, 0x26, 0x6e, 0x2e, 0x5f, - 0x5f, 0x76, 0x3d, 0x3d, 0x3d, 0x65, 0x2e, 0x5f, 0x5f, 0x76, 0x3f, 0x28, - 0x6e, 0x2e, 0x5f, 0x5f, 0x6b, 0x3d, 0x65, 0x2e, 0x5f, 0x5f, 0x6b, 0x2c, - 0x6e, 0x2e, 0x5f, 0x5f, 0x65, 0x3d, 0x65, 0x2e, 0x5f, 0x5f, 0x65, 0x29, - 0x3a, 0x6e, 0x2e, 0x5f, 0x5f, 0x65, 0x3d, 0x6f, 0x74, 0x28, 0x65, 0x2e, - 0x5f, 0x5f, 0x65, 0x2c, 0x6e, 0x2c, 0x65, 0x2c, 0x5f, 0x2c, 0x69, 0x2c, - 0x6f, 0x2c, 0x72, 0x2c, 0x66, 0x2c, 0x73, 0x29, 0x3b, 0x28, 0x6c, 0x3d, - 0x43, 0x2e, 0x64, 0x69, 0x66, 0x66, 0x65, 0x64, 0x29, 0x26, 0x26, 0x6c, - 0x28, 0x6e, 0x29, 0x7d, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, - 0x20, 0x69, 0x74, 0x28, 0x74, 0x2c, 0x6e, 0x2c, 0x65, 0x29, 0x7b, 0x6e, - 0x2e, 0x5f, 0x5f, 0x64, 0x3d, 0x76, 0x6f, 0x69, 0x64, 0x20, 0x30, 0x3b, - 0x66, 0x6f, 0x72, 0x28, 0x76, 0x61, 0x72, 0x20, 0x5f, 0x3d, 0x30, 0x3b, - 0x5f, 0x3c, 0x65, 0x2e, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x3b, 0x5f, - 0x2b, 0x2b, 0x29, 0x72, 0x74, 0x28, 0x65, 0x5b, 0x5f, 0x5d, 0x2c, 0x65, - 0x5b, 0x2b, 0x2b, 0x5f, 0x5d, 0x2c, 0x65, 0x5b, 0x2b, 0x2b, 0x5f, 0x5d, - 0x29, 0x3b, 0x43, 0x2e, 0x5f, 0x5f, 0x63, 0x26, 0x26, 0x43, 0x2e, 0x5f, - 0x5f, 0x63, 0x28, 0x6e, 0x2c, 0x74, 0x29, 0x2c, 0x74, 0x2e, 0x73, 0x6f, - 0x6d, 0x65, 0x28, 0x28, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, - 0x28, 0x6e, 0x29, 0x7b, 0x74, 0x72, 0x79, 0x7b, 0x74, 0x3d, 0x6e, 0x2e, - 0x5f, 0x5f, 0x68, 0x2c, 0x6e, 0x2e, 0x5f, 0x5f, 0x68, 0x3d, 0x5b, 0x5d, - 0x2c, 0x74, 0x2e, 0x73, 0x6f, 0x6d, 0x65, 0x28, 0x28, 0x66, 0x75, 0x6e, - 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x74, 0x29, 0x7b, 0x74, 0x2e, 0x63, - 0x61, 0x6c, 0x6c, 0x28, 0x6e, 0x29, 0x7d, 0x29, 0x29, 0x7d, 0x63, 0x61, - 0x74, 0x63, 0x68, 0x28, 0x74, 0x29, 0x7b, 0x43, 0x2e, 0x5f, 0x5f, 0x65, - 0x28, 0x74, 0x2c, 0x6e, 0x2e, 0x5f, 0x5f, 0x76, 0x29, 0x7d, 0x7d, 0x29, - 0x29, 0x7d, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x6f, - 0x74, 0x28, 0x74, 0x2c, 0x6e, 0x2c, 0x65, 0x2c, 0x5f, 0x2c, 0x69, 0x2c, - 0x6f, 0x2c, 0x72, 0x2c, 0x75, 0x2c, 0x66, 0x29, 0x7b, 0x76, 0x61, 0x72, - 0x20, 0x73, 0x2c, 0x6c, 0x2c, 0x63, 0x2c, 0x68, 0x2c, 0x61, 0x2c, 0x70, - 0x2c, 0x64, 0x2c, 0x76, 0x3d, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x70, 0x73, - 0x2c, 0x79, 0x3d, 0x6e, 0x2e, 0x70, 0x72, 0x6f, 0x70, 0x73, 0x2c, 0x6d, - 0x3d, 0x6e, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x3b, 0x69, 0x66, 0x28, 0x22, - 0x73, 0x76, 0x67, 0x22, 0x3d, 0x3d, 0x3d, 0x6d, 0x26, 0x26, 0x28, 0x69, - 0x3d, 0x21, 0x30, 0x29, 0x2c, 0x6e, 0x75, 0x6c, 0x6c, 0x21, 0x3d, 0x6f, - 0x29, 0x66, 0x6f, 0x72, 0x28, 0x73, 0x3d, 0x30, 0x3b, 0x73, 0x3c, 0x6f, - 0x2e, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x3b, 0x73, 0x2b, 0x2b, 0x29, - 0x69, 0x66, 0x28, 0x28, 0x61, 0x3d, 0x6f, 0x5b, 0x73, 0x5d, 0x29, 0x26, - 0x26, 0x22, 0x73, 0x65, 0x74, 0x41, 0x74, 0x74, 0x72, 0x69, 0x62, 0x75, - 0x74, 0x65, 0x22, 0x69, 0x6e, 0x20, 0x61, 0x3d, 0x3d, 0x21, 0x21, 0x6d, - 0x26, 0x26, 0x28, 0x6d, 0x3f, 0x61, 0x2e, 0x6c, 0x6f, 0x63, 0x61, 0x6c, - 0x4e, 0x61, 0x6d, 0x65, 0x3d, 0x3d, 0x3d, 0x6d, 0x3a, 0x33, 0x3d, 0x3d, - 0x3d, 0x61, 0x2e, 0x6e, 0x6f, 0x64, 0x65, 0x54, 0x79, 0x70, 0x65, 0x29, - 0x29, 0x7b, 0x74, 0x3d, 0x61, 0x2c, 0x6f, 0x5b, 0x73, 0x5d, 0x3d, 0x6e, - 0x75, 0x6c, 0x6c, 0x3b, 0x62, 0x72, 0x65, 0x61, 0x6b, 0x7d, 0x69, 0x66, - 0x28, 0x6e, 0x75, 0x6c, 0x6c, 0x3d, 0x3d, 0x74, 0x29, 0x7b, 0x69, 0x66, - 0x28, 0x6e, 0x75, 0x6c, 0x6c, 0x3d, 0x3d, 0x3d, 0x6d, 0x29, 0x72, 0x65, - 0x74, 0x75, 0x72, 0x6e, 0x20, 0x64, 0x6f, 0x63, 0x75, 0x6d, 0x65, 0x6e, - 0x74, 0x2e, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x54, 0x65, 0x78, 0x74, - 0x4e, 0x6f, 0x64, 0x65, 0x28, 0x79, 0x29, 0x3b, 0x74, 0x3d, 0x69, 0x3f, - 0x64, 0x6f, 0x63, 0x75, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x63, 0x72, 0x65, - 0x61, 0x74, 0x65, 0x45, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x4e, 0x53, - 0x28, 0x22, 0x68, 0x74, 0x74, 0x70, 0x3a, 0x2f, 0x2f, 0x77, 0x77, 0x77, - 0x2e, 0x77, 0x33, 0x2e, 0x6f, 0x72, 0x67, 0x2f, 0x32, 0x30, 0x30, 0x30, - 0x2f, 0x73, 0x76, 0x67, 0x22, 0x2c, 0x6d, 0x29, 0x3a, 0x64, 0x6f, 0x63, - 0x75, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, - 0x45, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x28, 0x6d, 0x2c, 0x79, 0x2e, - 0x69, 0x73, 0x26, 0x26, 0x79, 0x29, 0x2c, 0x6f, 0x3d, 0x6e, 0x75, 0x6c, - 0x6c, 0x2c, 0x75, 0x3d, 0x21, 0x31, 0x7d, 0x69, 0x66, 0x28, 0x6e, 0x75, - 0x6c, 0x6c, 0x3d, 0x3d, 0x3d, 0x6d, 0x29, 0x76, 0x3d, 0x3d, 0x3d, 0x79, - 0x7c, 0x7c, 0x75, 0x26, 0x26, 0x74, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x3d, - 0x3d, 0x3d, 0x79, 0x7c, 0x7c, 0x28, 0x74, 0x2e, 0x64, 0x61, 0x74, 0x61, - 0x3d, 0x79, 0x29, 0x3b, 0x65, 0x6c, 0x73, 0x65, 0x7b, 0x69, 0x66, 0x28, - 0x6f, 0x3d, 0x6f, 0x26, 0x26, 0x78, 0x2e, 0x63, 0x61, 0x6c, 0x6c, 0x28, - 0x74, 0x2e, 0x63, 0x68, 0x69, 0x6c, 0x64, 0x4e, 0x6f, 0x64, 0x65, 0x73, - 0x29, 0x2c, 0x76, 0x3d, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x70, 0x73, 0x7c, - 0x7c, 0x54, 0x2c, 0x21, 0x75, 0x26, 0x26, 0x6e, 0x75, 0x6c, 0x6c, 0x21, - 0x3d, 0x6f, 0x29, 0x66, 0x6f, 0x72, 0x28, 0x76, 0x3d, 0x7b, 0x7d, 0x2c, - 0x73, 0x3d, 0x30, 0x3b, 0x73, 0x3c, 0x74, 0x2e, 0x61, 0x74, 0x74, 0x72, - 0x69, 0x62, 0x75, 0x74, 0x65, 0x73, 0x2e, 0x6c, 0x65, 0x6e, 0x67, 0x74, - 0x68, 0x3b, 0x73, 0x2b, 0x2b, 0x29, 0x76, 0x5b, 0x28, 0x61, 0x3d, 0x74, - 0x2e, 0x61, 0x74, 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, 0x65, 0x73, 0x5b, - 0x73, 0x5d, 0x29, 0x2e, 0x6e, 0x61, 0x6d, 0x65, 0x5d, 0x3d, 0x61, 0x2e, - 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3b, 0x66, 0x6f, 0x72, 0x28, 0x73, 0x20, - 0x69, 0x6e, 0x20, 0x76, 0x29, 0x61, 0x3d, 0x76, 0x5b, 0x73, 0x5d, 0x2c, - 0x22, 0x63, 0x68, 0x69, 0x6c, 0x64, 0x72, 0x65, 0x6e, 0x22, 0x3d, 0x3d, - 0x73, 0x7c, 0x7c, 0x28, 0x22, 0x64, 0x61, 0x6e, 0x67, 0x65, 0x72, 0x6f, - 0x75, 0x73, 0x6c, 0x79, 0x53, 0x65, 0x74, 0x49, 0x6e, 0x6e, 0x65, 0x72, - 0x48, 0x54, 0x4d, 0x4c, 0x22, 0x3d, 0x3d, 0x73, 0x3f, 0x63, 0x3d, 0x61, - 0x3a, 0x22, 0x6b, 0x65, 0x79, 0x22, 0x3d, 0x3d, 0x3d, 0x73, 0x7c, 0x7c, - 0x73, 0x20, 0x69, 0x6e, 0x20, 0x79, 0x7c, 0x7c, 0x74, 0x74, 0x28, 0x74, - 0x2c, 0x73, 0x2c, 0x6e, 0x75, 0x6c, 0x6c, 0x2c, 0x61, 0x2c, 0x69, 0x29, - 0x29, 0x3b, 0x66, 0x6f, 0x72, 0x28, 0x73, 0x20, 0x69, 0x6e, 0x20, 0x79, - 0x29, 0x61, 0x3d, 0x79, 0x5b, 0x73, 0x5d, 0x2c, 0x22, 0x63, 0x68, 0x69, - 0x6c, 0x64, 0x72, 0x65, 0x6e, 0x22, 0x3d, 0x3d, 0x73, 0x3f, 0x68, 0x3d, - 0x61, 0x3a, 0x22, 0x64, 0x61, 0x6e, 0x67, 0x65, 0x72, 0x6f, 0x75, 0x73, - 0x6c, 0x79, 0x53, 0x65, 0x74, 0x49, 0x6e, 0x6e, 0x65, 0x72, 0x48, 0x54, - 0x4d, 0x4c, 0x22, 0x3d, 0x3d, 0x73, 0x3f, 0x6c, 0x3d, 0x61, 0x3a, 0x22, - 0x76, 0x61, 0x6c, 0x75, 0x65, 0x22, 0x3d, 0x3d, 0x73, 0x3f, 0x70, 0x3d, - 0x61, 0x3a, 0x22, 0x63, 0x68, 0x65, 0x63, 0x6b, 0x65, 0x64, 0x22, 0x3d, - 0x3d, 0x73, 0x3f, 0x64, 0x3d, 0x61, 0x3a, 0x22, 0x6b, 0x65, 0x79, 0x22, - 0x3d, 0x3d, 0x3d, 0x73, 0x7c, 0x7c, 0x75, 0x26, 0x26, 0x22, 0x66, 0x75, - 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x21, 0x3d, 0x74, 0x79, 0x70, - 0x65, 0x6f, 0x66, 0x20, 0x61, 0x7c, 0x7c, 0x76, 0x5b, 0x73, 0x5d, 0x3d, - 0x3d, 0x3d, 0x61, 0x7c, 0x7c, 0x74, 0x74, 0x28, 0x74, 0x2c, 0x73, 0x2c, - 0x61, 0x2c, 0x76, 0x5b, 0x73, 0x5d, 0x2c, 0x69, 0x29, 0x3b, 0x69, 0x66, - 0x28, 0x6c, 0x29, 0x75, 0x7c, 0x7c, 0x63, 0x26, 0x26, 0x28, 0x6c, 0x2e, - 0x5f, 0x5f, 0x68, 0x74, 0x6d, 0x6c, 0x3d, 0x3d, 0x3d, 0x63, 0x2e, 0x5f, - 0x5f, 0x68, 0x74, 0x6d, 0x6c, 0x7c, 0x7c, 0x6c, 0x2e, 0x5f, 0x5f, 0x68, - 0x74, 0x6d, 0x6c, 0x3d, 0x3d, 0x3d, 0x74, 0x2e, 0x69, 0x6e, 0x6e, 0x65, - 0x72, 0x48, 0x54, 0x4d, 0x4c, 0x29, 0x7c, 0x7c, 0x28, 0x74, 0x2e, 0x69, - 0x6e, 0x6e, 0x65, 0x72, 0x48, 0x54, 0x4d, 0x4c, 0x3d, 0x6c, 0x2e, 0x5f, - 0x5f, 0x68, 0x74, 0x6d, 0x6c, 0x29, 0x2c, 0x6e, 0x2e, 0x5f, 0x5f, 0x6b, - 0x3d, 0x5b, 0x5d, 0x3b, 0x65, 0x6c, 0x73, 0x65, 0x20, 0x69, 0x66, 0x28, - 0x63, 0x26, 0x26, 0x28, 0x74, 0x2e, 0x69, 0x6e, 0x6e, 0x65, 0x72, 0x48, - 0x54, 0x4d, 0x4c, 0x3d, 0x22, 0x22, 0x29, 0x2c, 0x4a, 0x28, 0x74, 0x2c, - 0x46, 0x28, 0x68, 0x29, 0x3f, 0x68, 0x3a, 0x5b, 0x68, 0x5d, 0x2c, 0x6e, - 0x2c, 0x65, 0x2c, 0x5f, 0x2c, 0x69, 0x26, 0x26, 0x22, 0x66, 0x6f, 0x72, - 0x65, 0x69, 0x67, 0x6e, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x22, 0x21, - 0x3d, 0x3d, 0x6d, 0x2c, 0x6f, 0x2c, 0x72, 0x2c, 0x6f, 0x3f, 0x6f, 0x5b, - 0x30, 0x5d, 0x3a, 0x65, 0x2e, 0x5f, 0x5f, 0x6b, 0x26, 0x26, 0x71, 0x28, - 0x65, 0x2c, 0x30, 0x29, 0x2c, 0x75, 0x2c, 0x66, 0x29, 0x2c, 0x6e, 0x75, - 0x6c, 0x6c, 0x21, 0x3d, 0x6f, 0x29, 0x66, 0x6f, 0x72, 0x28, 0x73, 0x3d, - 0x6f, 0x2e, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x3b, 0x73, 0x2d, 0x2d, - 0x3b, 0x29, 0x6e, 0x75, 0x6c, 0x6c, 0x21, 0x3d, 0x6f, 0x5b, 0x73, 0x5d, - 0x26, 0x26, 0x57, 0x28, 0x6f, 0x5b, 0x73, 0x5d, 0x29, 0x3b, 0x75, 0x7c, - 0x7c, 0x28, 0x73, 0x3d, 0x22, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x22, 0x2c, - 0x76, 0x6f, 0x69, 0x64, 0x20, 0x30, 0x21, 0x3d, 0x3d, 0x70, 0x26, 0x26, - 0x28, 0x70, 0x21, 0x3d, 0x3d, 0x74, 0x5b, 0x73, 0x5d, 0x7c, 0x7c, 0x22, - 0x70, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x22, 0x3d, 0x3d, 0x3d, - 0x6d, 0x26, 0x26, 0x21, 0x70, 0x7c, 0x7c, 0x22, 0x6f, 0x70, 0x74, 0x69, - 0x6f, 0x6e, 0x22, 0x3d, 0x3d, 0x3d, 0x6d, 0x26, 0x26, 0x70, 0x21, 0x3d, - 0x3d, 0x76, 0x5b, 0x73, 0x5d, 0x29, 0x26, 0x26, 0x74, 0x74, 0x28, 0x74, - 0x2c, 0x73, 0x2c, 0x70, 0x2c, 0x76, 0x5b, 0x73, 0x5d, 0x2c, 0x21, 0x31, - 0x29, 0x2c, 0x73, 0x3d, 0x22, 0x63, 0x68, 0x65, 0x63, 0x6b, 0x65, 0x64, - 0x22, 0x2c, 0x76, 0x6f, 0x69, 0x64, 0x20, 0x30, 0x21, 0x3d, 0x3d, 0x64, - 0x26, 0x26, 0x64, 0x21, 0x3d, 0x3d, 0x74, 0x5b, 0x73, 0x5d, 0x26, 0x26, - 0x74, 0x74, 0x28, 0x74, 0x2c, 0x73, 0x2c, 0x64, 0x2c, 0x76, 0x5b, 0x73, - 0x5d, 0x2c, 0x21, 0x31, 0x29, 0x29, 0x7d, 0x72, 0x65, 0x74, 0x75, 0x72, - 0x6e, 0x20, 0x74, 0x7d, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, - 0x20, 0x72, 0x74, 0x28, 0x74, 0x2c, 0x6e, 0x2c, 0x65, 0x29, 0x7b, 0x74, - 0x72, 0x79, 0x7b, 0x22, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, - 0x22, 0x3d, 0x3d, 0x74, 0x79, 0x70, 0x65, 0x6f, 0x66, 0x20, 0x74, 0x3f, - 0x74, 0x28, 0x6e, 0x29, 0x3a, 0x74, 0x2e, 0x63, 0x75, 0x72, 0x72, 0x65, - 0x6e, 0x74, 0x3d, 0x6e, 0x7d, 0x63, 0x61, 0x74, 0x63, 0x68, 0x28, 0x74, - 0x29, 0x7b, 0x43, 0x2e, 0x5f, 0x5f, 0x65, 0x28, 0x74, 0x2c, 0x65, 0x29, - 0x7d, 0x7d, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x75, - 0x74, 0x28, 0x74, 0x2c, 0x6e, 0x2c, 0x65, 0x29, 0x7b, 0x76, 0x61, 0x72, - 0x20, 0x5f, 0x2c, 0x69, 0x3b, 0x69, 0x66, 0x28, 0x43, 0x2e, 0x75, 0x6e, - 0x6d, 0x6f, 0x75, 0x6e, 0x74, 0x26, 0x26, 0x43, 0x2e, 0x75, 0x6e, 0x6d, - 0x6f, 0x75, 0x6e, 0x74, 0x28, 0x74, 0x29, 0x2c, 0x28, 0x5f, 0x3d, 0x74, - 0x2e, 0x72, 0x65, 0x66, 0x29, 0x26, 0x26, 0x28, 0x5f, 0x2e, 0x63, 0x75, - 0x72, 0x72, 0x65, 0x6e, 0x74, 0x26, 0x26, 0x5f, 0x2e, 0x63, 0x75, 0x72, - 0x72, 0x65, 0x6e, 0x74, 0x21, 0x3d, 0x3d, 0x74, 0x2e, 0x5f, 0x5f, 0x65, - 0x7c, 0x7c, 0x72, 0x74, 0x28, 0x5f, 0x2c, 0x6e, 0x75, 0x6c, 0x6c, 0x2c, - 0x6e, 0x29, 0x29, 0x2c, 0x6e, 0x75, 0x6c, 0x6c, 0x21, 0x3d, 0x28, 0x5f, - 0x3d, 0x74, 0x2e, 0x5f, 0x5f, 0x63, 0x29, 0x29, 0x7b, 0x69, 0x66, 0x28, - 0x5f, 0x2e, 0x63, 0x6f, 0x6d, 0x70, 0x6f, 0x6e, 0x65, 0x6e, 0x74, 0x57, - 0x69, 0x6c, 0x6c, 0x55, 0x6e, 0x6d, 0x6f, 0x75, 0x6e, 0x74, 0x29, 0x74, - 0x72, 0x79, 0x7b, 0x5f, 0x2e, 0x63, 0x6f, 0x6d, 0x70, 0x6f, 0x6e, 0x65, - 0x6e, 0x74, 0x57, 0x69, 0x6c, 0x6c, 0x55, 0x6e, 0x6d, 0x6f, 0x75, 0x6e, - 0x74, 0x28, 0x29, 0x7d, 0x63, 0x61, 0x74, 0x63, 0x68, 0x28, 0x74, 0x29, - 0x7b, 0x43, 0x2e, 0x5f, 0x5f, 0x65, 0x28, 0x74, 0x2c, 0x6e, 0x29, 0x7d, - 0x5f, 0x2e, 0x62, 0x61, 0x73, 0x65, 0x3d, 0x5f, 0x2e, 0x5f, 0x5f, 0x50, - 0x3d, 0x6e, 0x75, 0x6c, 0x6c, 0x2c, 0x74, 0x2e, 0x5f, 0x5f, 0x63, 0x3d, - 0x76, 0x6f, 0x69, 0x64, 0x20, 0x30, 0x7d, 0x69, 0x66, 0x28, 0x5f, 0x3d, - 0x74, 0x2e, 0x5f, 0x5f, 0x6b, 0x29, 0x66, 0x6f, 0x72, 0x28, 0x69, 0x3d, - 0x30, 0x3b, 0x69, 0x3c, 0x5f, 0x2e, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, - 0x3b, 0x69, 0x2b, 0x2b, 0x29, 0x5f, 0x5b, 0x69, 0x5d, 0x26, 0x26, 0x75, - 0x74, 0x28, 0x5f, 0x5b, 0x69, 0x5d, 0x2c, 0x6e, 0x2c, 0x65, 0x7c, 0x7c, - 0x22, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x21, 0x3d, - 0x74, 0x79, 0x70, 0x65, 0x6f, 0x66, 0x20, 0x74, 0x2e, 0x74, 0x79, 0x70, - 0x65, 0x29, 0x3b, 0x65, 0x7c, 0x7c, 0x6e, 0x75, 0x6c, 0x6c, 0x3d, 0x3d, - 0x74, 0x2e, 0x5f, 0x5f, 0x65, 0x7c, 0x7c, 0x57, 0x28, 0x74, 0x2e, 0x5f, - 0x5f, 0x65, 0x29, 0x2c, 0x74, 0x2e, 0x5f, 0x5f, 0x3d, 0x74, 0x2e, 0x5f, - 0x5f, 0x65, 0x3d, 0x74, 0x2e, 0x5f, 0x5f, 0x64, 0x3d, 0x76, 0x6f, 0x69, - 0x64, 0x20, 0x30, 0x7d, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, - 0x20, 0x66, 0x74, 0x28, 0x74, 0x2c, 0x6e, 0x2c, 0x65, 0x29, 0x7b, 0x72, - 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x63, - 0x6f, 0x6e, 0x73, 0x74, 0x72, 0x75, 0x63, 0x74, 0x6f, 0x72, 0x28, 0x74, - 0x2c, 0x65, 0x29, 0x7d, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, - 0x20, 0x73, 0x74, 0x28, 0x74, 0x2c, 0x6e, 0x2c, 0x65, 0x29, 0x7b, 0x76, - 0x61, 0x72, 0x20, 0x5f, 0x2c, 0x69, 0x2c, 0x6f, 0x2c, 0x72, 0x3b, 0x43, - 0x2e, 0x5f, 0x5f, 0x26, 0x26, 0x43, 0x2e, 0x5f, 0x5f, 0x28, 0x74, 0x2c, - 0x6e, 0x29, 0x2c, 0x69, 0x3d, 0x28, 0x5f, 0x3d, 0x22, 0x66, 0x75, 0x6e, - 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x3d, 0x3d, 0x74, 0x79, 0x70, 0x65, - 0x6f, 0x66, 0x20, 0x65, 0x29, 0x3f, 0x6e, 0x75, 0x6c, 0x6c, 0x3a, 0x65, - 0x26, 0x26, 0x65, 0x2e, 0x5f, 0x5f, 0x6b, 0x7c, 0x7c, 0x6e, 0x2e, 0x5f, - 0x5f, 0x6b, 0x2c, 0x6f, 0x3d, 0x5b, 0x5d, 0x2c, 0x72, 0x3d, 0x5b, 0x5d, - 0x2c, 0x5f, 0x74, 0x28, 0x6e, 0x2c, 0x74, 0x3d, 0x28, 0x21, 0x5f, 0x26, - 0x26, 0x65, 0x7c, 0x7c, 0x6e, 0x29, 0x2e, 0x5f, 0x5f, 0x6b, 0x3d, 0x4c, - 0x28, 0x6a, 0x2c, 0x6e, 0x75, 0x6c, 0x6c, 0x2c, 0x5b, 0x74, 0x5d, 0x29, - 0x2c, 0x69, 0x7c, 0x7c, 0x54, 0x2c, 0x54, 0x2c, 0x76, 0x6f, 0x69, 0x64, - 0x20, 0x30, 0x21, 0x3d, 0x3d, 0x6e, 0x2e, 0x6f, 0x77, 0x6e, 0x65, 0x72, - 0x53, 0x56, 0x47, 0x45, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2c, 0x21, - 0x5f, 0x26, 0x26, 0x65, 0x3f, 0x5b, 0x65, 0x5d, 0x3a, 0x69, 0x3f, 0x6e, - 0x75, 0x6c, 0x6c, 0x3a, 0x6e, 0x2e, 0x66, 0x69, 0x72, 0x73, 0x74, 0x43, - 0x68, 0x69, 0x6c, 0x64, 0x3f, 0x78, 0x2e, 0x63, 0x61, 0x6c, 0x6c, 0x28, - 0x6e, 0x2e, 0x63, 0x68, 0x69, 0x6c, 0x64, 0x4e, 0x6f, 0x64, 0x65, 0x73, - 0x29, 0x3a, 0x6e, 0x75, 0x6c, 0x6c, 0x2c, 0x6f, 0x2c, 0x21, 0x5f, 0x26, - 0x26, 0x65, 0x3f, 0x65, 0x3a, 0x69, 0x3f, 0x69, 0x2e, 0x5f, 0x5f, 0x65, - 0x3a, 0x6e, 0x2e, 0x66, 0x69, 0x72, 0x73, 0x74, 0x43, 0x68, 0x69, 0x6c, - 0x64, 0x2c, 0x5f, 0x2c, 0x72, 0x29, 0x2c, 0x69, 0x74, 0x28, 0x6f, 0x2c, - 0x74, 0x2c, 0x72, 0x29, 0x7d, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, - 0x6e, 0x20, 0x6c, 0x74, 0x28, 0x74, 0x2c, 0x6e, 0x29, 0x7b, 0x73, 0x74, - 0x28, 0x74, 0x2c, 0x6e, 0x2c, 0x6c, 0x74, 0x29, 0x7d, 0x66, 0x75, 0x6e, - 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x63, 0x74, 0x28, 0x74, 0x2c, 0x6e, - 0x2c, 0x65, 0x29, 0x7b, 0x76, 0x61, 0x72, 0x20, 0x5f, 0x2c, 0x69, 0x2c, - 0x6f, 0x2c, 0x72, 0x2c, 0x75, 0x3d, 0x4d, 0x28, 0x7b, 0x7d, 0x2c, 0x74, - 0x2e, 0x70, 0x72, 0x6f, 0x70, 0x73, 0x29, 0x3b, 0x66, 0x6f, 0x72, 0x28, - 0x6f, 0x20, 0x69, 0x6e, 0x20, 0x74, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x26, - 0x26, 0x74, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x2e, 0x64, 0x65, 0x66, 0x61, - 0x75, 0x6c, 0x74, 0x50, 0x72, 0x6f, 0x70, 0x73, 0x26, 0x26, 0x28, 0x72, - 0x3d, 0x74, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x2e, 0x64, 0x65, 0x66, 0x61, - 0x75, 0x6c, 0x74, 0x50, 0x72, 0x6f, 0x70, 0x73, 0x29, 0x2c, 0x6e, 0x29, - 0x22, 0x6b, 0x65, 0x79, 0x22, 0x3d, 0x3d, 0x6f, 0x3f, 0x5f, 0x3d, 0x6e, - 0x5b, 0x6f, 0x5d, 0x3a, 0x22, 0x72, 0x65, 0x66, 0x22, 0x3d, 0x3d, 0x6f, - 0x3f, 0x69, 0x3d, 0x6e, 0x5b, 0x6f, 0x5d, 0x3a, 0x75, 0x5b, 0x6f, 0x5d, - 0x3d, 0x76, 0x6f, 0x69, 0x64, 0x20, 0x30, 0x3d, 0x3d, 0x3d, 0x6e, 0x5b, - 0x6f, 0x5d, 0x26, 0x26, 0x76, 0x6f, 0x69, 0x64, 0x20, 0x30, 0x21, 0x3d, - 0x3d, 0x72, 0x3f, 0x72, 0x5b, 0x6f, 0x5d, 0x3a, 0x6e, 0x5b, 0x6f, 0x5d, - 0x3b, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x61, 0x72, 0x67, 0x75, - 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x2e, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, - 0x3e, 0x32, 0x26, 0x26, 0x28, 0x75, 0x2e, 0x63, 0x68, 0x69, 0x6c, 0x64, - 0x72, 0x65, 0x6e, 0x3d, 0x61, 0x72, 0x67, 0x75, 0x6d, 0x65, 0x6e, 0x74, - 0x73, 0x2e, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x3e, 0x33, 0x3f, 0x78, - 0x2e, 0x63, 0x61, 0x6c, 0x6c, 0x28, 0x61, 0x72, 0x67, 0x75, 0x6d, 0x65, - 0x6e, 0x74, 0x73, 0x2c, 0x32, 0x29, 0x3a, 0x65, 0x29, 0x2c, 0x4f, 0x28, - 0x74, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x2c, 0x75, 0x2c, 0x5f, 0x7c, 0x7c, - 0x74, 0x2e, 0x6b, 0x65, 0x79, 0x2c, 0x69, 0x7c, 0x7c, 0x74, 0x2e, 0x72, - 0x65, 0x66, 0x2c, 0x6e, 0x75, 0x6c, 0x6c, 0x29, 0x7d, 0x66, 0x75, 0x6e, - 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x68, 0x74, 0x28, 0x74, 0x2c, 0x6e, - 0x29, 0x7b, 0x76, 0x61, 0x72, 0x20, 0x65, 0x3d, 0x7b, 0x5f, 0x5f, 0x63, - 0x3a, 0x6e, 0x3d, 0x22, 0x5f, 0x5f, 0x63, 0x43, 0x22, 0x2b, 0x44, 0x2b, - 0x2b, 0x2c, 0x5f, 0x5f, 0x3a, 0x74, 0x2c, 0x43, 0x6f, 0x6e, 0x73, 0x75, - 0x6d, 0x65, 0x72, 0x3a, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, - 0x28, 0x74, 0x2c, 0x6e, 0x29, 0x7b, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, - 0x20, 0x74, 0x2e, 0x63, 0x68, 0x69, 0x6c, 0x64, 0x72, 0x65, 0x6e, 0x28, - 0x6e, 0x29, 0x7d, 0x2c, 0x50, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, - 0x3a, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x74, 0x29, - 0x7b, 0x76, 0x61, 0x72, 0x20, 0x65, 0x2c, 0x5f, 0x3b, 0x72, 0x65, 0x74, - 0x75, 0x72, 0x6e, 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x67, 0x65, 0x74, - 0x43, 0x68, 0x69, 0x6c, 0x64, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, - 0x7c, 0x7c, 0x28, 0x65, 0x3d, 0x5b, 0x5d, 0x2c, 0x28, 0x5f, 0x3d, 0x7b, - 0x7d, 0x29, 0x5b, 0x6e, 0x5d, 0x3d, 0x74, 0x68, 0x69, 0x73, 0x2c, 0x74, - 0x68, 0x69, 0x73, 0x2e, 0x67, 0x65, 0x74, 0x43, 0x68, 0x69, 0x6c, 0x64, - 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x3d, 0x66, 0x75, 0x6e, 0x63, - 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x29, 0x7b, 0x72, 0x65, 0x74, 0x75, 0x72, - 0x6e, 0x20, 0x5f, 0x7d, 0x2c, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x73, 0x68, - 0x6f, 0x75, 0x6c, 0x64, 0x43, 0x6f, 0x6d, 0x70, 0x6f, 0x6e, 0x65, 0x6e, - 0x74, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x3d, 0x66, 0x75, 0x6e, 0x63, - 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x74, 0x29, 0x7b, 0x74, 0x68, 0x69, 0x73, - 0x2e, 0x70, 0x72, 0x6f, 0x70, 0x73, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, - 0x21, 0x3d, 0x3d, 0x74, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x26, 0x26, - 0x65, 0x2e, 0x73, 0x6f, 0x6d, 0x65, 0x28, 0x28, 0x66, 0x75, 0x6e, 0x63, - 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x74, 0x29, 0x7b, 0x74, 0x2e, 0x5f, 0x5f, - 0x65, 0x3d, 0x21, 0x30, 0x2c, 0x47, 0x28, 0x74, 0x29, 0x7d, 0x29, 0x29, - 0x7d, 0x2c, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x73, 0x75, 0x62, 0x3d, 0x66, - 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x74, 0x29, 0x7b, 0x65, - 0x2e, 0x70, 0x75, 0x73, 0x68, 0x28, 0x74, 0x29, 0x3b, 0x76, 0x61, 0x72, - 0x20, 0x6e, 0x3d, 0x74, 0x2e, 0x63, 0x6f, 0x6d, 0x70, 0x6f, 0x6e, 0x65, - 0x6e, 0x74, 0x57, 0x69, 0x6c, 0x6c, 0x55, 0x6e, 0x6d, 0x6f, 0x75, 0x6e, - 0x74, 0x3b, 0x74, 0x2e, 0x63, 0x6f, 0x6d, 0x70, 0x6f, 0x6e, 0x65, 0x6e, - 0x74, 0x57, 0x69, 0x6c, 0x6c, 0x55, 0x6e, 0x6d, 0x6f, 0x75, 0x6e, 0x74, - 0x3d, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x29, 0x7b, - 0x65, 0x2e, 0x73, 0x70, 0x6c, 0x69, 0x63, 0x65, 0x28, 0x65, 0x2e, 0x69, - 0x6e, 0x64, 0x65, 0x78, 0x4f, 0x66, 0x28, 0x74, 0x29, 0x2c, 0x31, 0x29, - 0x2c, 0x6e, 0x26, 0x26, 0x6e, 0x2e, 0x63, 0x61, 0x6c, 0x6c, 0x28, 0x74, - 0x29, 0x7d, 0x7d, 0x29, 0x2c, 0x74, 0x2e, 0x63, 0x68, 0x69, 0x6c, 0x64, - 0x72, 0x65, 0x6e, 0x7d, 0x7d, 0x3b, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, - 0x20, 0x65, 0x2e, 0x50, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x2e, - 0x5f, 0x5f, 0x3d, 0x65, 0x2e, 0x43, 0x6f, 0x6e, 0x73, 0x75, 0x6d, 0x65, - 0x72, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x54, 0x79, 0x70, - 0x65, 0x3d, 0x65, 0x7d, 0x78, 0x3d, 0x56, 0x2e, 0x73, 0x6c, 0x69, 0x63, - 0x65, 0x2c, 0x43, 0x3d, 0x7b, 0x5f, 0x5f, 0x65, 0x3a, 0x66, 0x75, 0x6e, - 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x74, 0x2c, 0x6e, 0x2c, 0x65, 0x2c, - 0x5f, 0x29, 0x7b, 0x66, 0x6f, 0x72, 0x28, 0x76, 0x61, 0x72, 0x20, 0x69, - 0x2c, 0x6f, 0x2c, 0x72, 0x3b, 0x6e, 0x3d, 0x6e, 0x2e, 0x5f, 0x5f, 0x3b, - 0x29, 0x69, 0x66, 0x28, 0x28, 0x69, 0x3d, 0x6e, 0x2e, 0x5f, 0x5f, 0x63, - 0x29, 0x26, 0x26, 0x21, 0x69, 0x2e, 0x5f, 0x5f, 0x29, 0x74, 0x72, 0x79, - 0x7b, 0x69, 0x66, 0x28, 0x28, 0x6f, 0x3d, 0x69, 0x2e, 0x63, 0x6f, 0x6e, - 0x73, 0x74, 0x72, 0x75, 0x63, 0x74, 0x6f, 0x72, 0x29, 0x26, 0x26, 0x6e, - 0x75, 0x6c, 0x6c, 0x21, 0x3d, 0x6f, 0x2e, 0x67, 0x65, 0x74, 0x44, 0x65, - 0x72, 0x69, 0x76, 0x65, 0x64, 0x53, 0x74, 0x61, 0x74, 0x65, 0x46, 0x72, - 0x6f, 0x6d, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x26, 0x26, 0x28, 0x69, 0x2e, - 0x73, 0x65, 0x74, 0x53, 0x74, 0x61, 0x74, 0x65, 0x28, 0x6f, 0x2e, 0x67, - 0x65, 0x74, 0x44, 0x65, 0x72, 0x69, 0x76, 0x65, 0x64, 0x53, 0x74, 0x61, - 0x74, 0x65, 0x46, 0x72, 0x6f, 0x6d, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x28, - 0x74, 0x29, 0x29, 0x2c, 0x72, 0x3d, 0x69, 0x2e, 0x5f, 0x5f, 0x64, 0x29, - 0x2c, 0x6e, 0x75, 0x6c, 0x6c, 0x21, 0x3d, 0x69, 0x2e, 0x63, 0x6f, 0x6d, - 0x70, 0x6f, 0x6e, 0x65, 0x6e, 0x74, 0x44, 0x69, 0x64, 0x43, 0x61, 0x74, - 0x63, 0x68, 0x26, 0x26, 0x28, 0x69, 0x2e, 0x63, 0x6f, 0x6d, 0x70, 0x6f, - 0x6e, 0x65, 0x6e, 0x74, 0x44, 0x69, 0x64, 0x43, 0x61, 0x74, 0x63, 0x68, - 0x28, 0x74, 0x2c, 0x5f, 0x7c, 0x7c, 0x7b, 0x7d, 0x29, 0x2c, 0x72, 0x3d, - 0x69, 0x2e, 0x5f, 0x5f, 0x64, 0x29, 0x2c, 0x72, 0x29, 0x72, 0x65, 0x74, - 0x75, 0x72, 0x6e, 0x20, 0x69, 0x2e, 0x5f, 0x5f, 0x45, 0x3d, 0x69, 0x7d, - 0x63, 0x61, 0x74, 0x63, 0x68, 0x28, 0x6e, 0x29, 0x7b, 0x74, 0x3d, 0x6e, - 0x7d, 0x74, 0x68, 0x72, 0x6f, 0x77, 0x20, 0x74, 0x7d, 0x7d, 0x2c, 0x45, - 0x3d, 0x30, 0x2c, 0x55, 0x3d, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, - 0x6e, 0x28, 0x74, 0x29, 0x7b, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, - 0x6e, 0x75, 0x6c, 0x6c, 0x21, 0x3d, 0x74, 0x26, 0x26, 0x6e, 0x75, 0x6c, - 0x6c, 0x3d, 0x3d, 0x74, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x72, 0x75, - 0x63, 0x74, 0x6f, 0x72, 0x7d, 0x2c, 0x49, 0x2e, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x74, 0x79, 0x70, 0x65, 0x2e, 0x73, 0x65, 0x74, 0x53, 0x74, 0x61, - 0x74, 0x65, 0x3d, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x28, - 0x74, 0x2c, 0x6e, 0x29, 0x7b, 0x76, 0x61, 0x72, 0x20, 0x65, 0x3b, 0x65, - 0x3d, 0x6e, 0x75, 0x6c, 0x6c, 0x21, 0x3d, 0x74, 0x68, 0x69, 0x73, 0x2e, - 0x5f, 0x5f, 0x73, 0x26, 0x26, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x5f, 0x5f, - 0x73, 0x21, 0x3d, 0x3d, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x73, 0x74, 0x61, - 0x74, 0x65, 0x3f, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x5f, 0x5f, 0x73, 0x3a, - 0x74, 0x68, 0x69, 0x73, 0x2e, 0x5f, 0x5f, 0x73, 0x3d, 0x4d, 0x28, 0x7b, - 0x7d, 0x2c, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x73, 0x74, 0x61, 0x74, 0x65, - 0x29, 0x2c, 0x22, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x22, - 0x3d, 0x3d, 0x74, 0x79, 0x70, 0x65, 0x6f, 0x66, 0x20, 0x74, 0x26, 0x26, - 0x28, 0x74, 0x3d, 0x74, 0x28, 0x4d, 0x28, 0x7b, 0x7d, 0x2c, 0x65, 0x29, - 0x2c, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x70, 0x73, 0x29, - 0x29, 0x2c, 0x74, 0x26, 0x26, 0x4d, 0x28, 0x65, 0x2c, 0x74, 0x29, 0x2c, - 0x6e, 0x75, 0x6c, 0x6c, 0x21, 0x3d, 0x74, 0x26, 0x26, 0x74, 0x68, 0x69, - 0x73, 0x2e, 0x5f, 0x5f, 0x76, 0x26, 0x26, 0x28, 0x6e, 0x26, 0x26, 0x74, - 0x68, 0x69, 0x73, 0x2e, 0x5f, 0x73, 0x62, 0x2e, 0x70, 0x75, 0x73, 0x68, - 0x28, 0x6e, 0x29, 0x2c, 0x47, 0x28, 0x74, 0x68, 0x69, 0x73, 0x29, 0x29, - 0x7d, 0x2c, 0x49, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x74, 0x79, 0x70, - 0x65, 0x2e, 0x66, 0x6f, 0x72, 0x63, 0x65, 0x55, 0x70, 0x64, 0x61, 0x74, - 0x65, 0x3d, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x74, - 0x29, 0x7b, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x5f, 0x5f, 0x76, 0x26, 0x26, - 0x28, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x5f, 0x5f, 0x65, 0x3d, 0x21, 0x30, - 0x2c, 0x74, 0x26, 0x26, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x5f, 0x5f, 0x68, - 0x2e, 0x70, 0x75, 0x73, 0x68, 0x28, 0x74, 0x29, 0x2c, 0x47, 0x28, 0x74, - 0x68, 0x69, 0x73, 0x29, 0x29, 0x7d, 0x2c, 0x49, 0x2e, 0x70, 0x72, 0x6f, - 0x74, 0x6f, 0x74, 0x79, 0x70, 0x65, 0x2e, 0x72, 0x65, 0x6e, 0x64, 0x65, - 0x72, 0x3d, 0x6a, 0x2c, 0x48, 0x3d, 0x5b, 0x5d, 0x2c, 0x4e, 0x3d, 0x22, - 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x3d, 0x3d, 0x74, - 0x79, 0x70, 0x65, 0x6f, 0x66, 0x20, 0x50, 0x72, 0x6f, 0x6d, 0x69, 0x73, - 0x65, 0x3f, 0x50, 0x72, 0x6f, 0x6d, 0x69, 0x73, 0x65, 0x2e, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x74, 0x79, 0x70, 0x65, 0x2e, 0x74, 0x68, 0x65, 0x6e, - 0x2e, 0x62, 0x69, 0x6e, 0x64, 0x28, 0x50, 0x72, 0x6f, 0x6d, 0x69, 0x73, - 0x65, 0x2e, 0x72, 0x65, 0x73, 0x6f, 0x6c, 0x76, 0x65, 0x28, 0x29, 0x29, - 0x3a, 0x73, 0x65, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x2c, - 0x24, 0x3d, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x74, - 0x2c, 0x6e, 0x29, 0x7b, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x74, - 0x2e, 0x5f, 0x5f, 0x76, 0x2e, 0x5f, 0x5f, 0x62, 0x2d, 0x6e, 0x2e, 0x5f, - 0x5f, 0x76, 0x2e, 0x5f, 0x5f, 0x62, 0x7d, 0x2c, 0x7a, 0x2e, 0x5f, 0x5f, - 0x72, 0x3d, 0x30, 0x2c, 0x44, 0x3d, 0x30, 0x3b, 0x76, 0x61, 0x72, 0x20, - 0x61, 0x74, 0x2c, 0x70, 0x74, 0x2c, 0x64, 0x74, 0x2c, 0x76, 0x74, 0x2c, - 0x79, 0x74, 0x3d, 0x30, 0x2c, 0x6d, 0x74, 0x3d, 0x5b, 0x5d, 0x2c, 0x67, - 0x74, 0x3d, 0x5b, 0x5d, 0x2c, 0x62, 0x74, 0x3d, 0x43, 0x2e, 0x5f, 0x5f, - 0x62, 0x2c, 0x6b, 0x74, 0x3d, 0x43, 0x2e, 0x5f, 0x5f, 0x72, 0x2c, 0x53, - 0x74, 0x3d, 0x43, 0x2e, 0x64, 0x69, 0x66, 0x66, 0x65, 0x64, 0x2c, 0x77, - 0x74, 0x3d, 0x43, 0x2e, 0x5f, 0x5f, 0x63, 0x2c, 0x78, 0x74, 0x3d, 0x43, - 0x2e, 0x75, 0x6e, 0x6d, 0x6f, 0x75, 0x6e, 0x74, 0x3b, 0x66, 0x75, 0x6e, - 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x43, 0x74, 0x28, 0x74, 0x2c, 0x6e, - 0x29, 0x7b, 0x43, 0x2e, 0x5f, 0x5f, 0x68, 0x26, 0x26, 0x43, 0x2e, 0x5f, - 0x5f, 0x68, 0x28, 0x70, 0x74, 0x2c, 0x74, 0x2c, 0x79, 0x74, 0x7c, 0x7c, - 0x6e, 0x29, 0x2c, 0x79, 0x74, 0x3d, 0x30, 0x3b, 0x76, 0x61, 0x72, 0x20, - 0x65, 0x3d, 0x70, 0x74, 0x2e, 0x5f, 0x5f, 0x48, 0x7c, 0x7c, 0x28, 0x70, - 0x74, 0x2e, 0x5f, 0x5f, 0x48, 0x3d, 0x7b, 0x5f, 0x5f, 0x3a, 0x5b, 0x5d, - 0x2c, 0x5f, 0x5f, 0x68, 0x3a, 0x5b, 0x5d, 0x7d, 0x29, 0x3b, 0x72, 0x65, - 0x74, 0x75, 0x72, 0x6e, 0x20, 0x74, 0x3e, 0x3d, 0x65, 0x2e, 0x5f, 0x5f, - 0x2e, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x26, 0x26, 0x65, 0x2e, 0x5f, - 0x5f, 0x2e, 0x70, 0x75, 0x73, 0x68, 0x28, 0x7b, 0x5f, 0x5f, 0x56, 0x3a, - 0x67, 0x74, 0x7d, 0x29, 0x2c, 0x65, 0x2e, 0x5f, 0x5f, 0x5b, 0x74, 0x5d, - 0x7d, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x45, 0x74, - 0x28, 0x74, 0x29, 0x7b, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x79, - 0x74, 0x3d, 0x31, 0x2c, 0x55, 0x74, 0x28, 0x71, 0x74, 0x2c, 0x74, 0x29, - 0x7d, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x55, 0x74, - 0x28, 0x74, 0x2c, 0x6e, 0x2c, 0x65, 0x29, 0x7b, 0x76, 0x61, 0x72, 0x20, - 0x5f, 0x3d, 0x43, 0x74, 0x28, 0x61, 0x74, 0x2b, 0x2b, 0x2c, 0x32, 0x29, - 0x3b, 0x69, 0x66, 0x28, 0x5f, 0x2e, 0x74, 0x3d, 0x74, 0x2c, 0x21, 0x5f, - 0x2e, 0x5f, 0x5f, 0x63, 0x26, 0x26, 0x28, 0x5f, 0x2e, 0x5f, 0x5f, 0x3d, - 0x5b, 0x65, 0x3f, 0x65, 0x28, 0x6e, 0x29, 0x3a, 0x71, 0x74, 0x28, 0x76, - 0x6f, 0x69, 0x64, 0x20, 0x30, 0x2c, 0x6e, 0x29, 0x2c, 0x66, 0x75, 0x6e, - 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x74, 0x29, 0x7b, 0x76, 0x61, 0x72, - 0x20, 0x6e, 0x3d, 0x5f, 0x2e, 0x5f, 0x5f, 0x4e, 0x3f, 0x5f, 0x2e, 0x5f, - 0x5f, 0x4e, 0x5b, 0x30, 0x5d, 0x3a, 0x5f, 0x2e, 0x5f, 0x5f, 0x5b, 0x30, - 0x5d, 0x2c, 0x65, 0x3d, 0x5f, 0x2e, 0x74, 0x28, 0x6e, 0x2c, 0x74, 0x29, - 0x3b, 0x6e, 0x21, 0x3d, 0x3d, 0x65, 0x26, 0x26, 0x28, 0x5f, 0x2e, 0x5f, - 0x5f, 0x4e, 0x3d, 0x5b, 0x65, 0x2c, 0x5f, 0x2e, 0x5f, 0x5f, 0x5b, 0x31, - 0x5d, 0x5d, 0x2c, 0x5f, 0x2e, 0x5f, 0x5f, 0x63, 0x2e, 0x73, 0x65, 0x74, - 0x53, 0x74, 0x61, 0x74, 0x65, 0x28, 0x7b, 0x7d, 0x29, 0x29, 0x7d, 0x5d, - 0x2c, 0x5f, 0x2e, 0x5f, 0x5f, 0x63, 0x3d, 0x70, 0x74, 0x2c, 0x21, 0x70, - 0x74, 0x2e, 0x75, 0x29, 0x29, 0x7b, 0x76, 0x61, 0x72, 0x20, 0x69, 0x3d, - 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x74, 0x2c, 0x6e, - 0x2c, 0x65, 0x29, 0x7b, 0x69, 0x66, 0x28, 0x21, 0x5f, 0x2e, 0x5f, 0x5f, - 0x63, 0x2e, 0x5f, 0x5f, 0x48, 0x29, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, - 0x21, 0x30, 0x3b, 0x76, 0x61, 0x72, 0x20, 0x69, 0x3d, 0x5f, 0x2e, 0x5f, - 0x5f, 0x63, 0x2e, 0x5f, 0x5f, 0x48, 0x2e, 0x5f, 0x5f, 0x2e, 0x66, 0x69, - 0x6c, 0x74, 0x65, 0x72, 0x28, 0x28, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, - 0x6f, 0x6e, 0x28, 0x74, 0x29, 0x7b, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, - 0x20, 0x74, 0x2e, 0x5f, 0x5f, 0x63, 0x7d, 0x29, 0x29, 0x3b, 0x69, 0x66, - 0x28, 0x69, 0x2e, 0x65, 0x76, 0x65, 0x72, 0x79, 0x28, 0x28, 0x66, 0x75, - 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x74, 0x29, 0x7b, 0x72, 0x65, - 0x74, 0x75, 0x72, 0x6e, 0x21, 0x74, 0x2e, 0x5f, 0x5f, 0x4e, 0x7d, 0x29, - 0x29, 0x29, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x21, 0x6f, 0x7c, 0x7c, - 0x6f, 0x2e, 0x63, 0x61, 0x6c, 0x6c, 0x28, 0x74, 0x68, 0x69, 0x73, 0x2c, - 0x74, 0x2c, 0x6e, 0x2c, 0x65, 0x29, 0x3b, 0x76, 0x61, 0x72, 0x20, 0x72, - 0x3d, 0x21, 0x31, 0x3b, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x69, - 0x2e, 0x66, 0x6f, 0x72, 0x45, 0x61, 0x63, 0x68, 0x28, 0x28, 0x66, 0x75, - 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x74, 0x29, 0x7b, 0x69, 0x66, - 0x28, 0x74, 0x2e, 0x5f, 0x5f, 0x4e, 0x29, 0x7b, 0x76, 0x61, 0x72, 0x20, - 0x6e, 0x3d, 0x74, 0x2e, 0x5f, 0x5f, 0x5b, 0x30, 0x5d, 0x3b, 0x74, 0x2e, - 0x5f, 0x5f, 0x3d, 0x74, 0x2e, 0x5f, 0x5f, 0x4e, 0x2c, 0x74, 0x2e, 0x5f, - 0x5f, 0x4e, 0x3d, 0x76, 0x6f, 0x69, 0x64, 0x20, 0x30, 0x2c, 0x6e, 0x21, - 0x3d, 0x3d, 0x74, 0x2e, 0x5f, 0x5f, 0x5b, 0x30, 0x5d, 0x26, 0x26, 0x28, - 0x72, 0x3d, 0x21, 0x30, 0x29, 0x7d, 0x7d, 0x29, 0x29, 0x2c, 0x21, 0x28, - 0x21, 0x72, 0x26, 0x26, 0x5f, 0x2e, 0x5f, 0x5f, 0x63, 0x2e, 0x70, 0x72, - 0x6f, 0x70, 0x73, 0x3d, 0x3d, 0x3d, 0x74, 0x29, 0x26, 0x26, 0x28, 0x21, - 0x6f, 0x7c, 0x7c, 0x6f, 0x2e, 0x63, 0x61, 0x6c, 0x6c, 0x28, 0x74, 0x68, - 0x69, 0x73, 0x2c, 0x74, 0x2c, 0x6e, 0x2c, 0x65, 0x29, 0x29, 0x7d, 0x3b, - 0x70, 0x74, 0x2e, 0x75, 0x3d, 0x21, 0x30, 0x3b, 0x76, 0x61, 0x72, 0x20, - 0x6f, 0x3d, 0x70, 0x74, 0x2e, 0x73, 0x68, 0x6f, 0x75, 0x6c, 0x64, 0x43, - 0x6f, 0x6d, 0x70, 0x6f, 0x6e, 0x65, 0x6e, 0x74, 0x55, 0x70, 0x64, 0x61, - 0x74, 0x65, 0x2c, 0x72, 0x3d, 0x70, 0x74, 0x2e, 0x63, 0x6f, 0x6d, 0x70, - 0x6f, 0x6e, 0x65, 0x6e, 0x74, 0x57, 0x69, 0x6c, 0x6c, 0x55, 0x70, 0x64, - 0x61, 0x74, 0x65, 0x3b, 0x70, 0x74, 0x2e, 0x63, 0x6f, 0x6d, 0x70, 0x6f, - 0x6e, 0x65, 0x6e, 0x74, 0x57, 0x69, 0x6c, 0x6c, 0x55, 0x70, 0x64, 0x61, - 0x74, 0x65, 0x3d, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x28, - 0x74, 0x2c, 0x6e, 0x2c, 0x65, 0x29, 0x7b, 0x69, 0x66, 0x28, 0x74, 0x68, - 0x69, 0x73, 0x2e, 0x5f, 0x5f, 0x65, 0x29, 0x7b, 0x76, 0x61, 0x72, 0x20, - 0x5f, 0x3d, 0x6f, 0x3b, 0x6f, 0x3d, 0x76, 0x6f, 0x69, 0x64, 0x20, 0x30, - 0x2c, 0x69, 0x28, 0x74, 0x2c, 0x6e, 0x2c, 0x65, 0x29, 0x2c, 0x6f, 0x3d, - 0x5f, 0x7d, 0x72, 0x26, 0x26, 0x72, 0x2e, 0x63, 0x61, 0x6c, 0x6c, 0x28, - 0x74, 0x68, 0x69, 0x73, 0x2c, 0x74, 0x2c, 0x6e, 0x2c, 0x65, 0x29, 0x7d, - 0x2c, 0x70, 0x74, 0x2e, 0x73, 0x68, 0x6f, 0x75, 0x6c, 0x64, 0x43, 0x6f, - 0x6d, 0x70, 0x6f, 0x6e, 0x65, 0x6e, 0x74, 0x55, 0x70, 0x64, 0x61, 0x74, - 0x65, 0x3d, 0x69, 0x7d, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x5f, - 0x2e, 0x5f, 0x5f, 0x4e, 0x7c, 0x7c, 0x5f, 0x2e, 0x5f, 0x5f, 0x7d, 0x66, - 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x48, 0x74, 0x28, 0x74, - 0x2c, 0x6e, 0x29, 0x7b, 0x76, 0x61, 0x72, 0x20, 0x65, 0x3d, 0x43, 0x74, - 0x28, 0x61, 0x74, 0x2b, 0x2b, 0x2c, 0x33, 0x29, 0x3b, 0x21, 0x43, 0x2e, - 0x5f, 0x5f, 0x73, 0x26, 0x26, 0x49, 0x74, 0x28, 0x65, 0x2e, 0x5f, 0x5f, - 0x48, 0x2c, 0x6e, 0x29, 0x26, 0x26, 0x28, 0x65, 0x2e, 0x5f, 0x5f, 0x3d, - 0x74, 0x2c, 0x65, 0x2e, 0x69, 0x3d, 0x6e, 0x2c, 0x70, 0x74, 0x2e, 0x5f, - 0x5f, 0x48, 0x2e, 0x5f, 0x5f, 0x68, 0x2e, 0x70, 0x75, 0x73, 0x68, 0x28, - 0x65, 0x29, 0x29, 0x7d, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, - 0x20, 0x50, 0x74, 0x28, 0x74, 0x2c, 0x6e, 0x29, 0x7b, 0x76, 0x61, 0x72, - 0x20, 0x65, 0x3d, 0x43, 0x74, 0x28, 0x61, 0x74, 0x2b, 0x2b, 0x2c, 0x34, - 0x29, 0x3b, 0x21, 0x43, 0x2e, 0x5f, 0x5f, 0x73, 0x26, 0x26, 0x49, 0x74, - 0x28, 0x65, 0x2e, 0x5f, 0x5f, 0x48, 0x2c, 0x6e, 0x29, 0x26, 0x26, 0x28, - 0x65, 0x2e, 0x5f, 0x5f, 0x3d, 0x74, 0x2c, 0x65, 0x2e, 0x69, 0x3d, 0x6e, - 0x2c, 0x70, 0x74, 0x2e, 0x5f, 0x5f, 0x68, 0x2e, 0x70, 0x75, 0x73, 0x68, - 0x28, 0x65, 0x29, 0x29, 0x7d, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, - 0x6e, 0x20, 0x4e, 0x74, 0x28, 0x74, 0x29, 0x7b, 0x72, 0x65, 0x74, 0x75, - 0x72, 0x6e, 0x20, 0x79, 0x74, 0x3d, 0x35, 0x2c, 0x44, 0x74, 0x28, 0x28, - 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x29, 0x7b, 0x72, - 0x65, 0x74, 0x75, 0x72, 0x6e, 0x7b, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, - 0x74, 0x3a, 0x74, 0x7d, 0x7d, 0x29, 0x2c, 0x5b, 0x5d, 0x29, 0x7d, 0x66, - 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x24, 0x74, 0x28, 0x74, - 0x2c, 0x6e, 0x2c, 0x65, 0x29, 0x7b, 0x79, 0x74, 0x3d, 0x36, 0x2c, 0x50, - 0x74, 0x28, 0x28, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x28, - 0x29, 0x7b, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x22, 0x66, 0x75, 0x6e, - 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x3d, 0x3d, 0x74, 0x79, 0x70, 0x65, - 0x6f, 0x66, 0x20, 0x74, 0x3f, 0x28, 0x74, 0x28, 0x6e, 0x28, 0x29, 0x29, - 0x2c, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x29, 0x7b, - 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x74, 0x28, 0x6e, 0x75, 0x6c, - 0x6c, 0x29, 0x7d, 0x29, 0x3a, 0x74, 0x3f, 0x28, 0x74, 0x2e, 0x63, 0x75, - 0x72, 0x72, 0x65, 0x6e, 0x74, 0x3d, 0x6e, 0x28, 0x29, 0x2c, 0x66, 0x75, - 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x29, 0x7b, 0x72, 0x65, 0x74, - 0x75, 0x72, 0x6e, 0x20, 0x74, 0x2e, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, - 0x74, 0x3d, 0x6e, 0x75, 0x6c, 0x6c, 0x7d, 0x29, 0x3a, 0x76, 0x6f, 0x69, - 0x64, 0x20, 0x30, 0x7d, 0x29, 0x2c, 0x6e, 0x75, 0x6c, 0x6c, 0x3d, 0x3d, - 0x65, 0x3f, 0x65, 0x3a, 0x65, 0x2e, 0x63, 0x6f, 0x6e, 0x63, 0x61, 0x74, - 0x28, 0x74, 0x29, 0x29, 0x7d, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, - 0x6e, 0x20, 0x44, 0x74, 0x28, 0x74, 0x2c, 0x6e, 0x29, 0x7b, 0x76, 0x61, - 0x72, 0x20, 0x65, 0x3d, 0x43, 0x74, 0x28, 0x61, 0x74, 0x2b, 0x2b, 0x2c, - 0x37, 0x29, 0x3b, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x49, 0x74, - 0x28, 0x65, 0x2e, 0x5f, 0x5f, 0x48, 0x2c, 0x6e, 0x29, 0x3f, 0x28, 0x65, - 0x2e, 0x5f, 0x5f, 0x56, 0x3d, 0x74, 0x28, 0x29, 0x2c, 0x65, 0x2e, 0x69, - 0x3d, 0x6e, 0x2c, 0x65, 0x2e, 0x5f, 0x5f, 0x68, 0x3d, 0x74, 0x2c, 0x65, - 0x2e, 0x5f, 0x5f, 0x56, 0x29, 0x3a, 0x65, 0x2e, 0x5f, 0x5f, 0x7d, 0x66, - 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x54, 0x74, 0x28, 0x74, - 0x2c, 0x6e, 0x29, 0x7b, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x79, - 0x74, 0x3d, 0x38, 0x2c, 0x44, 0x74, 0x28, 0x28, 0x66, 0x75, 0x6e, 0x63, - 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x29, 0x7b, 0x72, 0x65, 0x74, 0x75, 0x72, - 0x6e, 0x20, 0x74, 0x7d, 0x29, 0x2c, 0x6e, 0x29, 0x7d, 0x66, 0x75, 0x6e, - 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x56, 0x74, 0x28, 0x74, 0x29, 0x7b, - 0x76, 0x61, 0x72, 0x20, 0x6e, 0x3d, 0x70, 0x74, 0x2e, 0x63, 0x6f, 0x6e, - 0x74, 0x65, 0x78, 0x74, 0x5b, 0x74, 0x2e, 0x5f, 0x5f, 0x63, 0x5d, 0x2c, - 0x65, 0x3d, 0x43, 0x74, 0x28, 0x61, 0x74, 0x2b, 0x2b, 0x2c, 0x39, 0x29, - 0x3b, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x65, 0x2e, 0x63, 0x3d, - 0x74, 0x2c, 0x6e, 0x3f, 0x28, 0x6e, 0x75, 0x6c, 0x6c, 0x3d, 0x3d, 0x65, - 0x2e, 0x5f, 0x5f, 0x26, 0x26, 0x28, 0x65, 0x2e, 0x5f, 0x5f, 0x3d, 0x21, - 0x30, 0x2c, 0x6e, 0x2e, 0x73, 0x75, 0x62, 0x28, 0x70, 0x74, 0x29, 0x29, - 0x2c, 0x6e, 0x2e, 0x70, 0x72, 0x6f, 0x70, 0x73, 0x2e, 0x76, 0x61, 0x6c, - 0x75, 0x65, 0x29, 0x3a, 0x74, 0x2e, 0x5f, 0x5f, 0x7d, 0x66, 0x75, 0x6e, - 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x41, 0x74, 0x28, 0x74, 0x2c, 0x6e, - 0x29, 0x7b, 0x43, 0x2e, 0x75, 0x73, 0x65, 0x44, 0x65, 0x62, 0x75, 0x67, - 0x56, 0x61, 0x6c, 0x75, 0x65, 0x26, 0x26, 0x43, 0x2e, 0x75, 0x73, 0x65, - 0x44, 0x65, 0x62, 0x75, 0x67, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x28, 0x6e, - 0x3f, 0x6e, 0x28, 0x74, 0x29, 0x3a, 0x74, 0x29, 0x7d, 0x66, 0x75, 0x6e, - 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x46, 0x74, 0x28, 0x74, 0x29, 0x7b, - 0x76, 0x61, 0x72, 0x20, 0x6e, 0x3d, 0x43, 0x74, 0x28, 0x61, 0x74, 0x2b, - 0x2b, 0x2c, 0x31, 0x30, 0x29, 0x2c, 0x65, 0x3d, 0x45, 0x74, 0x28, 0x29, - 0x3b, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x6e, 0x2e, 0x5f, 0x5f, - 0x3d, 0x74, 0x2c, 0x70, 0x74, 0x2e, 0x63, 0x6f, 0x6d, 0x70, 0x6f, 0x6e, - 0x65, 0x6e, 0x74, 0x44, 0x69, 0x64, 0x43, 0x61, 0x74, 0x63, 0x68, 0x7c, - 0x7c, 0x28, 0x70, 0x74, 0x2e, 0x63, 0x6f, 0x6d, 0x70, 0x6f, 0x6e, 0x65, - 0x6e, 0x74, 0x44, 0x69, 0x64, 0x43, 0x61, 0x74, 0x63, 0x68, 0x3d, 0x66, - 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x74, 0x2c, 0x5f, 0x29, - 0x7b, 0x6e, 0x2e, 0x5f, 0x5f, 0x26, 0x26, 0x6e, 0x2e, 0x5f, 0x5f, 0x28, - 0x74, 0x2c, 0x5f, 0x29, 0x2c, 0x65, 0x5b, 0x31, 0x5d, 0x28, 0x74, 0x29, - 0x7d, 0x29, 0x2c, 0x5b, 0x65, 0x5b, 0x30, 0x5d, 0x2c, 0x66, 0x75, 0x6e, - 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x29, 0x7b, 0x65, 0x5b, 0x31, 0x5d, - 0x28, 0x76, 0x6f, 0x69, 0x64, 0x20, 0x30, 0x29, 0x7d, 0x5d, 0x7d, 0x66, - 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x4d, 0x74, 0x28, 0x29, - 0x7b, 0x76, 0x61, 0x72, 0x20, 0x74, 0x3d, 0x43, 0x74, 0x28, 0x61, 0x74, - 0x2b, 0x2b, 0x2c, 0x31, 0x31, 0x29, 0x3b, 0x69, 0x66, 0x28, 0x21, 0x74, - 0x2e, 0x5f, 0x5f, 0x29, 0x7b, 0x66, 0x6f, 0x72, 0x28, 0x76, 0x61, 0x72, - 0x20, 0x6e, 0x3d, 0x70, 0x74, 0x2e, 0x5f, 0x5f, 0x76, 0x3b, 0x6e, 0x75, - 0x6c, 0x6c, 0x21, 0x3d, 0x3d, 0x6e, 0x26, 0x26, 0x21, 0x6e, 0x2e, 0x5f, - 0x5f, 0x6d, 0x26, 0x26, 0x6e, 0x75, 0x6c, 0x6c, 0x21, 0x3d, 0x3d, 0x6e, - 0x2e, 0x5f, 0x5f, 0x3b, 0x29, 0x6e, 0x3d, 0x6e, 0x2e, 0x5f, 0x5f, 0x3b, - 0x76, 0x61, 0x72, 0x20, 0x65, 0x3d, 0x6e, 0x2e, 0x5f, 0x5f, 0x6d, 0x7c, - 0x7c, 0x28, 0x6e, 0x2e, 0x5f, 0x5f, 0x6d, 0x3d, 0x5b, 0x30, 0x2c, 0x30, - 0x5d, 0x29, 0x3b, 0x74, 0x2e, 0x5f, 0x5f, 0x3d, 0x22, 0x50, 0x22, 0x2b, - 0x65, 0x5b, 0x30, 0x5d, 0x2b, 0x22, 0x2d, 0x22, 0x2b, 0x65, 0x5b, 0x31, - 0x5d, 0x2b, 0x2b, 0x7d, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x74, - 0x2e, 0x5f, 0x5f, 0x7d, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, - 0x20, 0x57, 0x74, 0x28, 0x29, 0x7b, 0x66, 0x6f, 0x72, 0x28, 0x76, 0x61, - 0x72, 0x20, 0x74, 0x3b, 0x74, 0x3d, 0x6d, 0x74, 0x2e, 0x73, 0x68, 0x69, - 0x66, 0x74, 0x28, 0x29, 0x3b, 0x29, 0x69, 0x66, 0x28, 0x74, 0x2e, 0x5f, - 0x5f, 0x50, 0x26, 0x26, 0x74, 0x2e, 0x5f, 0x5f, 0x48, 0x29, 0x74, 0x72, - 0x79, 0x7b, 0x74, 0x2e, 0x5f, 0x5f, 0x48, 0x2e, 0x5f, 0x5f, 0x68, 0x2e, - 0x66, 0x6f, 0x72, 0x45, 0x61, 0x63, 0x68, 0x28, 0x52, 0x74, 0x29, 0x2c, - 0x74, 0x2e, 0x5f, 0x5f, 0x48, 0x2e, 0x5f, 0x5f, 0x68, 0x2e, 0x66, 0x6f, - 0x72, 0x45, 0x61, 0x63, 0x68, 0x28, 0x6a, 0x74, 0x29, 0x2c, 0x74, 0x2e, - 0x5f, 0x5f, 0x48, 0x2e, 0x5f, 0x5f, 0x68, 0x3d, 0x5b, 0x5d, 0x7d, 0x63, - 0x61, 0x74, 0x63, 0x68, 0x28, 0x75, 0x29, 0x7b, 0x74, 0x2e, 0x5f, 0x5f, - 0x48, 0x2e, 0x5f, 0x5f, 0x68, 0x3d, 0x5b, 0x5d, 0x2c, 0x43, 0x2e, 0x5f, - 0x5f, 0x65, 0x28, 0x75, 0x2c, 0x74, 0x2e, 0x5f, 0x5f, 0x76, 0x29, 0x7d, - 0x7d, 0x43, 0x2e, 0x5f, 0x5f, 0x62, 0x3d, 0x66, 0x75, 0x6e, 0x63, 0x74, - 0x69, 0x6f, 0x6e, 0x28, 0x74, 0x29, 0x7b, 0x70, 0x74, 0x3d, 0x6e, 0x75, - 0x6c, 0x6c, 0x2c, 0x62, 0x74, 0x26, 0x26, 0x62, 0x74, 0x28, 0x74, 0x29, - 0x7d, 0x2c, 0x43, 0x2e, 0x5f, 0x5f, 0x72, 0x3d, 0x66, 0x75, 0x6e, 0x63, - 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x74, 0x29, 0x7b, 0x6b, 0x74, 0x26, 0x26, - 0x6b, 0x74, 0x28, 0x74, 0x29, 0x2c, 0x61, 0x74, 0x3d, 0x30, 0x3b, 0x76, - 0x61, 0x72, 0x20, 0x6e, 0x3d, 0x28, 0x70, 0x74, 0x3d, 0x74, 0x2e, 0x5f, - 0x5f, 0x63, 0x29, 0x2e, 0x5f, 0x5f, 0x48, 0x3b, 0x6e, 0x26, 0x26, 0x28, - 0x64, 0x74, 0x3d, 0x3d, 0x3d, 0x70, 0x74, 0x3f, 0x28, 0x6e, 0x2e, 0x5f, - 0x5f, 0x68, 0x3d, 0x5b, 0x5d, 0x2c, 0x70, 0x74, 0x2e, 0x5f, 0x5f, 0x68, - 0x3d, 0x5b, 0x5d, 0x2c, 0x6e, 0x2e, 0x5f, 0x5f, 0x2e, 0x66, 0x6f, 0x72, - 0x45, 0x61, 0x63, 0x68, 0x28, 0x28, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, - 0x6f, 0x6e, 0x28, 0x74, 0x29, 0x7b, 0x74, 0x2e, 0x5f, 0x5f, 0x4e, 0x26, - 0x26, 0x28, 0x74, 0x2e, 0x5f, 0x5f, 0x3d, 0x74, 0x2e, 0x5f, 0x5f, 0x4e, - 0x29, 0x2c, 0x74, 0x2e, 0x5f, 0x5f, 0x56, 0x3d, 0x67, 0x74, 0x2c, 0x74, - 0x2e, 0x5f, 0x5f, 0x4e, 0x3d, 0x74, 0x2e, 0x69, 0x3d, 0x76, 0x6f, 0x69, - 0x64, 0x20, 0x30, 0x7d, 0x29, 0x29, 0x29, 0x3a, 0x28, 0x6e, 0x2e, 0x5f, - 0x5f, 0x68, 0x2e, 0x66, 0x6f, 0x72, 0x45, 0x61, 0x63, 0x68, 0x28, 0x52, - 0x74, 0x29, 0x2c, 0x6e, 0x2e, 0x5f, 0x5f, 0x68, 0x2e, 0x66, 0x6f, 0x72, - 0x45, 0x61, 0x63, 0x68, 0x28, 0x6a, 0x74, 0x29, 0x2c, 0x6e, 0x2e, 0x5f, - 0x5f, 0x68, 0x3d, 0x5b, 0x5d, 0x2c, 0x61, 0x74, 0x3d, 0x30, 0x29, 0x29, - 0x2c, 0x64, 0x74, 0x3d, 0x70, 0x74, 0x7d, 0x2c, 0x43, 0x2e, 0x64, 0x69, - 0x66, 0x66, 0x65, 0x64, 0x3d, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, - 0x6e, 0x28, 0x74, 0x29, 0x7b, 0x53, 0x74, 0x26, 0x26, 0x53, 0x74, 0x28, - 0x74, 0x29, 0x3b, 0x76, 0x61, 0x72, 0x20, 0x6e, 0x3d, 0x74, 0x2e, 0x5f, - 0x5f, 0x63, 0x3b, 0x6e, 0x26, 0x26, 0x6e, 0x2e, 0x5f, 0x5f, 0x48, 0x26, - 0x26, 0x28, 0x6e, 0x2e, 0x5f, 0x5f, 0x48, 0x2e, 0x5f, 0x5f, 0x68, 0x2e, - 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x26, 0x26, 0x28, 0x31, 0x21, 0x3d, - 0x3d, 0x6d, 0x74, 0x2e, 0x70, 0x75, 0x73, 0x68, 0x28, 0x6e, 0x29, 0x26, - 0x26, 0x76, 0x74, 0x3d, 0x3d, 0x3d, 0x43, 0x2e, 0x72, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x41, 0x6e, 0x69, 0x6d, 0x61, 0x74, 0x69, 0x6f, 0x6e, - 0x46, 0x72, 0x61, 0x6d, 0x65, 0x7c, 0x7c, 0x28, 0x28, 0x76, 0x74, 0x3d, - 0x43, 0x2e, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x41, 0x6e, 0x69, - 0x6d, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x46, 0x72, 0x61, 0x6d, 0x65, 0x29, - 0x7c, 0x7c, 0x4f, 0x74, 0x29, 0x28, 0x57, 0x74, 0x29, 0x29, 0x2c, 0x6e, - 0x2e, 0x5f, 0x5f, 0x48, 0x2e, 0x5f, 0x5f, 0x2e, 0x66, 0x6f, 0x72, 0x45, - 0x61, 0x63, 0x68, 0x28, 0x28, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, - 0x6e, 0x28, 0x74, 0x29, 0x7b, 0x74, 0x2e, 0x69, 0x26, 0x26, 0x28, 0x74, - 0x2e, 0x5f, 0x5f, 0x48, 0x3d, 0x74, 0x2e, 0x69, 0x29, 0x2c, 0x74, 0x2e, - 0x5f, 0x5f, 0x56, 0x21, 0x3d, 0x3d, 0x67, 0x74, 0x26, 0x26, 0x28, 0x74, - 0x2e, 0x5f, 0x5f, 0x3d, 0x74, 0x2e, 0x5f, 0x5f, 0x56, 0x29, 0x2c, 0x74, - 0x2e, 0x69, 0x3d, 0x76, 0x6f, 0x69, 0x64, 0x20, 0x30, 0x2c, 0x74, 0x2e, - 0x5f, 0x5f, 0x56, 0x3d, 0x67, 0x74, 0x7d, 0x29, 0x29, 0x29, 0x2c, 0x64, - 0x74, 0x3d, 0x70, 0x74, 0x3d, 0x6e, 0x75, 0x6c, 0x6c, 0x7d, 0x2c, 0x43, - 0x2e, 0x5f, 0x5f, 0x63, 0x3d, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, - 0x6e, 0x28, 0x74, 0x2c, 0x6e, 0x29, 0x7b, 0x6e, 0x2e, 0x73, 0x6f, 0x6d, - 0x65, 0x28, 0x28, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x28, - 0x74, 0x29, 0x7b, 0x74, 0x72, 0x79, 0x7b, 0x74, 0x2e, 0x5f, 0x5f, 0x68, - 0x2e, 0x66, 0x6f, 0x72, 0x45, 0x61, 0x63, 0x68, 0x28, 0x52, 0x74, 0x29, - 0x2c, 0x74, 0x2e, 0x5f, 0x5f, 0x68, 0x3d, 0x74, 0x2e, 0x5f, 0x5f, 0x68, - 0x2e, 0x66, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x28, 0x28, 0x66, 0x75, 0x6e, - 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x74, 0x29, 0x7b, 0x72, 0x65, 0x74, - 0x75, 0x72, 0x6e, 0x21, 0x74, 0x2e, 0x5f, 0x5f, 0x7c, 0x7c, 0x6a, 0x74, - 0x28, 0x74, 0x29, 0x7d, 0x29, 0x29, 0x7d, 0x63, 0x61, 0x74, 0x63, 0x68, - 0x28, 0x6c, 0x29, 0x7b, 0x6e, 0x2e, 0x73, 0x6f, 0x6d, 0x65, 0x28, 0x28, - 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x74, 0x29, 0x7b, - 0x74, 0x2e, 0x5f, 0x5f, 0x68, 0x26, 0x26, 0x28, 0x74, 0x2e, 0x5f, 0x5f, - 0x68, 0x3d, 0x5b, 0x5d, 0x29, 0x7d, 0x29, 0x29, 0x2c, 0x6e, 0x3d, 0x5b, - 0x5d, 0x2c, 0x43, 0x2e, 0x5f, 0x5f, 0x65, 0x28, 0x6c, 0x2c, 0x74, 0x2e, - 0x5f, 0x5f, 0x76, 0x29, 0x7d, 0x7d, 0x29, 0x29, 0x2c, 0x77, 0x74, 0x26, - 0x26, 0x77, 0x74, 0x28, 0x74, 0x2c, 0x6e, 0x29, 0x7d, 0x2c, 0x43, 0x2e, - 0x75, 0x6e, 0x6d, 0x6f, 0x75, 0x6e, 0x74, 0x3d, 0x66, 0x75, 0x6e, 0x63, - 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x74, 0x29, 0x7b, 0x78, 0x74, 0x26, 0x26, - 0x78, 0x74, 0x28, 0x74, 0x29, 0x3b, 0x76, 0x61, 0x72, 0x20, 0x6e, 0x2c, - 0x65, 0x3d, 0x74, 0x2e, 0x5f, 0x5f, 0x63, 0x3b, 0x65, 0x26, 0x26, 0x65, - 0x2e, 0x5f, 0x5f, 0x48, 0x26, 0x26, 0x28, 0x65, 0x2e, 0x5f, 0x5f, 0x48, - 0x2e, 0x5f, 0x5f, 0x2e, 0x66, 0x6f, 0x72, 0x45, 0x61, 0x63, 0x68, 0x28, - 0x28, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x74, 0x29, - 0x7b, 0x74, 0x72, 0x79, 0x7b, 0x52, 0x74, 0x28, 0x74, 0x29, 0x7d, 0x63, - 0x61, 0x74, 0x63, 0x68, 0x28, 0x74, 0x29, 0x7b, 0x6e, 0x3d, 0x74, 0x7d, - 0x7d, 0x29, 0x29, 0x2c, 0x65, 0x2e, 0x5f, 0x5f, 0x48, 0x3d, 0x76, 0x6f, - 0x69, 0x64, 0x20, 0x30, 0x2c, 0x6e, 0x26, 0x26, 0x43, 0x2e, 0x5f, 0x5f, - 0x65, 0x28, 0x6e, 0x2c, 0x65, 0x2e, 0x5f, 0x5f, 0x76, 0x29, 0x29, 0x7d, - 0x3b, 0x76, 0x61, 0x72, 0x20, 0x4c, 0x74, 0x3d, 0x22, 0x66, 0x75, 0x6e, - 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x3d, 0x3d, 0x74, 0x79, 0x70, 0x65, - 0x6f, 0x66, 0x20, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x41, 0x6e, - 0x69, 0x6d, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x46, 0x72, 0x61, 0x6d, 0x65, - 0x3b, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x4f, 0x74, - 0x28, 0x74, 0x29, 0x7b, 0x76, 0x61, 0x72, 0x20, 0x6e, 0x2c, 0x65, 0x3d, - 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x29, 0x7b, 0x63, - 0x6c, 0x65, 0x61, 0x72, 0x54, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x28, - 0x5f, 0x29, 0x2c, 0x4c, 0x74, 0x26, 0x26, 0x63, 0x61, 0x6e, 0x63, 0x65, - 0x6c, 0x41, 0x6e, 0x69, 0x6d, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x46, 0x72, - 0x61, 0x6d, 0x65, 0x28, 0x6e, 0x29, 0x2c, 0x73, 0x65, 0x74, 0x54, 0x69, - 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x28, 0x74, 0x29, 0x7d, 0x2c, 0x5f, 0x3d, - 0x73, 0x65, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x28, 0x65, - 0x2c, 0x31, 0x30, 0x30, 0x29, 0x3b, 0x4c, 0x74, 0x26, 0x26, 0x28, 0x6e, - 0x3d, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x41, 0x6e, 0x69, 0x6d, - 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x46, 0x72, 0x61, 0x6d, 0x65, 0x28, 0x65, - 0x29, 0x29, 0x7d, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, - 0x52, 0x74, 0x28, 0x74, 0x29, 0x7b, 0x76, 0x61, 0x72, 0x20, 0x6e, 0x3d, - 0x70, 0x74, 0x2c, 0x65, 0x3d, 0x74, 0x2e, 0x5f, 0x5f, 0x63, 0x3b, 0x22, - 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x3d, 0x3d, 0x74, - 0x79, 0x70, 0x65, 0x6f, 0x66, 0x20, 0x65, 0x26, 0x26, 0x28, 0x74, 0x2e, - 0x5f, 0x5f, 0x63, 0x3d, 0x76, 0x6f, 0x69, 0x64, 0x20, 0x30, 0x2c, 0x65, - 0x28, 0x29, 0x29, 0x2c, 0x70, 0x74, 0x3d, 0x6e, 0x7d, 0x66, 0x75, 0x6e, - 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x6a, 0x74, 0x28, 0x74, 0x29, 0x7b, - 0x76, 0x61, 0x72, 0x20, 0x6e, 0x3d, 0x70, 0x74, 0x3b, 0x74, 0x2e, 0x5f, - 0x5f, 0x63, 0x3d, 0x74, 0x2e, 0x5f, 0x5f, 0x28, 0x29, 0x2c, 0x70, 0x74, - 0x3d, 0x6e, 0x7d, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, - 0x49, 0x74, 0x28, 0x74, 0x2c, 0x6e, 0x29, 0x7b, 0x72, 0x65, 0x74, 0x75, - 0x72, 0x6e, 0x21, 0x74, 0x7c, 0x7c, 0x74, 0x2e, 0x6c, 0x65, 0x6e, 0x67, - 0x74, 0x68, 0x21, 0x3d, 0x3d, 0x6e, 0x2e, 0x6c, 0x65, 0x6e, 0x67, 0x74, - 0x68, 0x7c, 0x7c, 0x6e, 0x2e, 0x73, 0x6f, 0x6d, 0x65, 0x28, 0x28, 0x66, - 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x6e, 0x2c, 0x65, 0x29, - 0x7b, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x6e, 0x21, 0x3d, 0x3d, - 0x74, 0x5b, 0x65, 0x5d, 0x7d, 0x29, 0x29, 0x7d, 0x66, 0x75, 0x6e, 0x63, - 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x71, 0x74, 0x28, 0x74, 0x2c, 0x6e, 0x29, - 0x7b, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x22, 0x66, 0x75, 0x6e, 0x63, - 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x3d, 0x3d, 0x74, 0x79, 0x70, 0x65, 0x6f, - 0x66, 0x20, 0x6e, 0x3f, 0x6e, 0x28, 0x74, 0x29, 0x3a, 0x6e, 0x7d, 0x66, - 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x42, 0x74, 0x28, 0x74, - 0x2c, 0x6e, 0x29, 0x7b, 0x43, 0x5b, 0x74, 0x5d, 0x3d, 0x6e, 0x2e, 0x62, - 0x69, 0x6e, 0x64, 0x28, 0x6e, 0x75, 0x6c, 0x6c, 0x2c, 0x43, 0x5b, 0x74, - 0x5d, 0x7c, 0x7c, 0x28, 0x28, 0x29, 0x3d, 0x3e, 0x7b, 0x7d, 0x29, 0x29, - 0x7d, 0x6c, 0x65, 0x74, 0x20, 0x47, 0x74, 0x2c, 0x7a, 0x74, 0x3b, 0x66, - 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x4a, 0x74, 0x28, 0x74, - 0x29, 0x7b, 0x69, 0x66, 0x28, 0x7a, 0x74, 0x29, 0x7a, 0x74, 0x28, 0x29, - 0x3b, 0x7a, 0x74, 0x3d, 0x74, 0x26, 0x26, 0x74, 0x2e, 0x53, 0x28, 0x29, - 0x7d, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x4b, 0x74, - 0x28, 0x7b, 0x64, 0x61, 0x74, 0x61, 0x3a, 0x74, 0x7d, 0x29, 0x7b, 0x63, - 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x6e, 0x3d, 0x58, 0x74, 0x28, 0x74, 0x29, - 0x3b, 0x6e, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3d, 0x74, 0x3b, 0x63, - 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x65, 0x3d, 0x44, 0x74, 0x28, 0x28, 0x29, - 0x3d, 0x3e, 0x7b, 0x6c, 0x65, 0x74, 0x20, 0x74, 0x3d, 0x74, 0x68, 0x69, - 0x73, 0x2e, 0x5f, 0x5f, 0x76, 0x3b, 0x77, 0x68, 0x69, 0x6c, 0x65, 0x28, - 0x74, 0x3d, 0x74, 0x2e, 0x5f, 0x5f, 0x29, 0x69, 0x66, 0x28, 0x74, 0x2e, - 0x5f, 0x5f, 0x63, 0x29, 0x7b, 0x74, 0x2e, 0x5f, 0x5f, 0x63, 0x2e, 0x5f, - 0x5f, 0x24, 0x66, 0x7c, 0x3d, 0x34, 0x3b, 0x62, 0x72, 0x65, 0x61, 0x6b, - 0x7d, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x5f, 0x5f, 0x24, 0x75, 0x2e, 0x63, - 0x3d, 0x28, 0x29, 0x3d, 0x3e, 0x7b, 0x76, 0x61, 0x72, 0x20, 0x74, 0x3b, - 0x69, 0x66, 0x28, 0x21, 0x55, 0x28, 0x65, 0x2e, 0x70, 0x65, 0x65, 0x6b, - 0x28, 0x29, 0x29, 0x26, 0x26, 0x33, 0x3d, 0x3d, 0x3d, 0x28, 0x6e, 0x75, - 0x6c, 0x6c, 0x3d, 0x3d, 0x28, 0x74, 0x3d, 0x74, 0x68, 0x69, 0x73, 0x2e, - 0x62, 0x61, 0x73, 0x65, 0x29, 0x3f, 0x76, 0x6f, 0x69, 0x64, 0x20, 0x30, - 0x3a, 0x74, 0x2e, 0x6e, 0x6f, 0x64, 0x65, 0x54, 0x79, 0x70, 0x65, 0x29, - 0x29, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x62, 0x61, 0x73, 0x65, 0x2e, 0x64, - 0x61, 0x74, 0x61, 0x3d, 0x65, 0x2e, 0x70, 0x65, 0x65, 0x6b, 0x28, 0x29, - 0x3b, 0x65, 0x6c, 0x73, 0x65, 0x7b, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x5f, - 0x5f, 0x24, 0x66, 0x7c, 0x3d, 0x31, 0x3b, 0x74, 0x68, 0x69, 0x73, 0x2e, - 0x73, 0x65, 0x74, 0x53, 0x74, 0x61, 0x74, 0x65, 0x28, 0x7b, 0x7d, 0x29, - 0x7d, 0x7d, 0x3b, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x6d, 0x28, - 0x28, 0x29, 0x3d, 0x3e, 0x7b, 0x6c, 0x65, 0x74, 0x20, 0x74, 0x3d, 0x6e, - 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, - 0x3b, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x30, 0x3d, 0x3d, 0x3d, - 0x74, 0x3f, 0x30, 0x3a, 0x21, 0x30, 0x3d, 0x3d, 0x3d, 0x74, 0x3f, 0x22, - 0x22, 0x3a, 0x74, 0x7c, 0x7c, 0x22, 0x22, 0x7d, 0x29, 0x7d, 0x2c, 0x5b, - 0x5d, 0x29, 0x3b, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x65, 0x2e, - 0x76, 0x61, 0x6c, 0x75, 0x65, 0x7d, 0x4b, 0x74, 0x2e, 0x64, 0x69, 0x73, - 0x70, 0x6c, 0x61, 0x79, 0x4e, 0x61, 0x6d, 0x65, 0x3d, 0x22, 0x5f, 0x73, - 0x74, 0x22, 0x3b, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x2e, 0x64, 0x65, - 0x66, 0x69, 0x6e, 0x65, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, - 0x65, 0x73, 0x28, 0x68, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x74, 0x79, - 0x70, 0x65, 0x2c, 0x7b, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x72, 0x75, 0x63, - 0x74, 0x6f, 0x72, 0x3a, 0x7b, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, - 0x72, 0x61, 0x62, 0x6c, 0x65, 0x3a, 0x21, 0x30, 0x2c, 0x76, 0x61, 0x6c, - 0x75, 0x65, 0x3a, 0x76, 0x6f, 0x69, 0x64, 0x20, 0x30, 0x7d, 0x2c, 0x74, - 0x79, 0x70, 0x65, 0x3a, 0x7b, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, - 0x72, 0x61, 0x62, 0x6c, 0x65, 0x3a, 0x21, 0x30, 0x2c, 0x76, 0x61, 0x6c, - 0x75, 0x65, 0x3a, 0x4b, 0x74, 0x7d, 0x2c, 0x70, 0x72, 0x6f, 0x70, 0x73, - 0x3a, 0x7b, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x61, 0x62, - 0x6c, 0x65, 0x3a, 0x21, 0x30, 0x2c, 0x67, 0x65, 0x74, 0x28, 0x29, 0x7b, - 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x7b, 0x64, 0x61, 0x74, 0x61, 0x3a, - 0x74, 0x68, 0x69, 0x73, 0x7d, 0x7d, 0x7d, 0x2c, 0x5f, 0x5f, 0x62, 0x3a, - 0x7b, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x61, 0x62, 0x6c, - 0x65, 0x3a, 0x21, 0x30, 0x2c, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x31, - 0x7d, 0x7d, 0x29, 0x3b, 0x42, 0x74, 0x28, 0x22, 0x5f, 0x5f, 0x62, 0x22, - 0x2c, 0x28, 0x74, 0x2c, 0x6e, 0x29, 0x3d, 0x3e, 0x7b, 0x69, 0x66, 0x28, - 0x22, 0x73, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x22, 0x3d, 0x3d, 0x74, 0x79, - 0x70, 0x65, 0x6f, 0x66, 0x20, 0x6e, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x29, - 0x7b, 0x6c, 0x65, 0x74, 0x20, 0x74, 0x2c, 0x65, 0x3d, 0x6e, 0x2e, 0x70, - 0x72, 0x6f, 0x70, 0x73, 0x3b, 0x66, 0x6f, 0x72, 0x28, 0x6c, 0x65, 0x74, - 0x20, 0x5f, 0x20, 0x69, 0x6e, 0x20, 0x65, 0x29, 0x7b, 0x69, 0x66, 0x28, - 0x22, 0x63, 0x68, 0x69, 0x6c, 0x64, 0x72, 0x65, 0x6e, 0x22, 0x3d, 0x3d, - 0x3d, 0x5f, 0x29, 0x63, 0x6f, 0x6e, 0x74, 0x69, 0x6e, 0x75, 0x65, 0x3b, - 0x6c, 0x65, 0x74, 0x20, 0x69, 0x3d, 0x65, 0x5b, 0x5f, 0x5d, 0x3b, 0x69, - 0x66, 0x28, 0x69, 0x20, 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, - 0x6f, 0x66, 0x20, 0x68, 0x29, 0x7b, 0x69, 0x66, 0x28, 0x21, 0x74, 0x29, - 0x6e, 0x2e, 0x5f, 0x5f, 0x6e, 0x70, 0x3d, 0x74, 0x3d, 0x7b, 0x7d, 0x3b, - 0x74, 0x5b, 0x5f, 0x5d, 0x3d, 0x69, 0x3b, 0x65, 0x5b, 0x5f, 0x5d, 0x3d, - 0x69, 0x2e, 0x70, 0x65, 0x65, 0x6b, 0x28, 0x29, 0x7d, 0x7d, 0x7d, 0x74, - 0x28, 0x6e, 0x29, 0x7d, 0x29, 0x3b, 0x42, 0x74, 0x28, 0x22, 0x5f, 0x5f, - 0x72, 0x22, 0x2c, 0x28, 0x74, 0x2c, 0x6e, 0x29, 0x3d, 0x3e, 0x7b, 0x4a, - 0x74, 0x28, 0x29, 0x3b, 0x6c, 0x65, 0x74, 0x20, 0x65, 0x2c, 0x5f, 0x3d, - 0x6e, 0x2e, 0x5f, 0x5f, 0x63, 0x3b, 0x69, 0x66, 0x28, 0x5f, 0x29, 0x7b, - 0x5f, 0x2e, 0x5f, 0x5f, 0x24, 0x66, 0x26, 0x3d, 0x2d, 0x32, 0x3b, 0x65, - 0x3d, 0x5f, 0x2e, 0x5f, 0x5f, 0x24, 0x75, 0x3b, 0x69, 0x66, 0x28, 0x76, - 0x6f, 0x69, 0x64, 0x20, 0x30, 0x3d, 0x3d, 0x3d, 0x65, 0x29, 0x5f, 0x2e, - 0x5f, 0x5f, 0x24, 0x75, 0x3d, 0x65, 0x3d, 0x66, 0x75, 0x6e, 0x63, 0x74, - 0x69, 0x6f, 0x6e, 0x28, 0x74, 0x29, 0x7b, 0x6c, 0x65, 0x74, 0x20, 0x6e, - 0x3b, 0x77, 0x28, 0x28, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, - 0x28, 0x29, 0x7b, 0x6e, 0x3d, 0x74, 0x68, 0x69, 0x73, 0x7d, 0x29, 0x29, - 0x3b, 0x6e, 0x2e, 0x63, 0x3d, 0x28, 0x29, 0x3d, 0x3e, 0x7b, 0x5f, 0x2e, - 0x5f, 0x5f, 0x24, 0x66, 0x7c, 0x3d, 0x31, 0x3b, 0x5f, 0x2e, 0x73, 0x65, - 0x74, 0x53, 0x74, 0x61, 0x74, 0x65, 0x28, 0x7b, 0x7d, 0x29, 0x7d, 0x3b, - 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x6e, 0x7d, 0x28, 0x29, 0x7d, - 0x47, 0x74, 0x3d, 0x5f, 0x3b, 0x4a, 0x74, 0x28, 0x65, 0x29, 0x3b, 0x74, - 0x28, 0x6e, 0x29, 0x7d, 0x29, 0x3b, 0x42, 0x74, 0x28, 0x22, 0x5f, 0x5f, - 0x65, 0x22, 0x2c, 0x28, 0x74, 0x2c, 0x6e, 0x2c, 0x65, 0x2c, 0x5f, 0x29, - 0x3d, 0x3e, 0x7b, 0x4a, 0x74, 0x28, 0x29, 0x3b, 0x47, 0x74, 0x3d, 0x76, - 0x6f, 0x69, 0x64, 0x20, 0x30, 0x3b, 0x74, 0x28, 0x6e, 0x2c, 0x65, 0x2c, - 0x5f, 0x29, 0x7d, 0x29, 0x3b, 0x42, 0x74, 0x28, 0x22, 0x64, 0x69, 0x66, - 0x66, 0x65, 0x64, 0x22, 0x2c, 0x28, 0x74, 0x2c, 0x6e, 0x29, 0x3d, 0x3e, - 0x7b, 0x4a, 0x74, 0x28, 0x29, 0x3b, 0x47, 0x74, 0x3d, 0x76, 0x6f, 0x69, - 0x64, 0x20, 0x30, 0x3b, 0x6c, 0x65, 0x74, 0x20, 0x65, 0x3b, 0x69, 0x66, - 0x28, 0x22, 0x73, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x22, 0x3d, 0x3d, 0x74, - 0x79, 0x70, 0x65, 0x6f, 0x66, 0x20, 0x6e, 0x2e, 0x74, 0x79, 0x70, 0x65, - 0x26, 0x26, 0x28, 0x65, 0x3d, 0x6e, 0x2e, 0x5f, 0x5f, 0x65, 0x29, 0x29, - 0x7b, 0x6c, 0x65, 0x74, 0x20, 0x74, 0x3d, 0x6e, 0x2e, 0x5f, 0x5f, 0x6e, - 0x70, 0x2c, 0x5f, 0x3d, 0x6e, 0x2e, 0x70, 0x72, 0x6f, 0x70, 0x73, 0x3b, - 0x69, 0x66, 0x28, 0x74, 0x29, 0x7b, 0x6c, 0x65, 0x74, 0x20, 0x6e, 0x3d, - 0x65, 0x2e, 0x55, 0x3b, 0x69, 0x66, 0x28, 0x6e, 0x29, 0x66, 0x6f, 0x72, - 0x28, 0x6c, 0x65, 0x74, 0x20, 0x65, 0x20, 0x69, 0x6e, 0x20, 0x6e, 0x29, - 0x7b, 0x6c, 0x65, 0x74, 0x20, 0x5f, 0x3d, 0x6e, 0x5b, 0x65, 0x5d, 0x3b, - 0x69, 0x66, 0x28, 0x76, 0x6f, 0x69, 0x64, 0x20, 0x30, 0x21, 0x3d, 0x3d, - 0x5f, 0x26, 0x26, 0x21, 0x28, 0x65, 0x20, 0x69, 0x6e, 0x20, 0x74, 0x29, - 0x29, 0x7b, 0x5f, 0x2e, 0x64, 0x28, 0x29, 0x3b, 0x6e, 0x5b, 0x65, 0x5d, - 0x3d, 0x76, 0x6f, 0x69, 0x64, 0x20, 0x30, 0x7d, 0x7d, 0x65, 0x6c, 0x73, - 0x65, 0x7b, 0x6e, 0x3d, 0x7b, 0x7d, 0x3b, 0x65, 0x2e, 0x55, 0x3d, 0x6e, - 0x7d, 0x66, 0x6f, 0x72, 0x28, 0x6c, 0x65, 0x74, 0x20, 0x69, 0x20, 0x69, - 0x6e, 0x20, 0x74, 0x29, 0x7b, 0x6c, 0x65, 0x74, 0x20, 0x6f, 0x3d, 0x6e, - 0x5b, 0x69, 0x5d, 0x2c, 0x72, 0x3d, 0x74, 0x5b, 0x69, 0x5d, 0x3b, 0x69, - 0x66, 0x28, 0x76, 0x6f, 0x69, 0x64, 0x20, 0x30, 0x3d, 0x3d, 0x3d, 0x6f, - 0x29, 0x7b, 0x6f, 0x3d, 0x51, 0x74, 0x28, 0x65, 0x2c, 0x69, 0x2c, 0x72, - 0x2c, 0x5f, 0x29, 0x3b, 0x6e, 0x5b, 0x69, 0x5d, 0x3d, 0x6f, 0x7d, 0x65, - 0x6c, 0x73, 0x65, 0x20, 0x6f, 0x2e, 0x6f, 0x28, 0x72, 0x2c, 0x5f, 0x29, - 0x7d, 0x7d, 0x7d, 0x74, 0x28, 0x6e, 0x29, 0x7d, 0x29, 0x3b, 0x66, 0x75, - 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x51, 0x74, 0x28, 0x74, 0x2c, - 0x6e, 0x2c, 0x65, 0x2c, 0x5f, 0x29, 0x7b, 0x63, 0x6f, 0x6e, 0x73, 0x74, - 0x20, 0x69, 0x3d, 0x6e, 0x20, 0x69, 0x6e, 0x20, 0x74, 0x26, 0x26, 0x76, - 0x6f, 0x69, 0x64, 0x20, 0x30, 0x3d, 0x3d, 0x3d, 0x74, 0x2e, 0x6f, 0x77, - 0x6e, 0x65, 0x72, 0x53, 0x56, 0x47, 0x45, 0x6c, 0x65, 0x6d, 0x65, 0x6e, - 0x74, 0x2c, 0x6f, 0x3d, 0x61, 0x28, 0x65, 0x29, 0x3b, 0x72, 0x65, 0x74, - 0x75, 0x72, 0x6e, 0x7b, 0x6f, 0x3a, 0x28, 0x74, 0x2c, 0x6e, 0x29, 0x3d, - 0x3e, 0x7b, 0x6f, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3d, 0x74, 0x3b, - 0x5f, 0x3d, 0x6e, 0x7d, 0x2c, 0x64, 0x3a, 0x77, 0x28, 0x28, 0x29, 0x3d, - 0x3e, 0x7b, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x65, 0x3d, 0x6f, 0x2e, - 0x76, 0x61, 0x6c, 0x75, 0x65, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3b, - 0x69, 0x66, 0x28, 0x5f, 0x5b, 0x6e, 0x5d, 0x21, 0x3d, 0x3d, 0x65, 0x29, - 0x7b, 0x5f, 0x5b, 0x6e, 0x5d, 0x3d, 0x65, 0x3b, 0x69, 0x66, 0x28, 0x69, - 0x29, 0x74, 0x5b, 0x6e, 0x5d, 0x3d, 0x65, 0x3b, 0x65, 0x6c, 0x73, 0x65, - 0x20, 0x69, 0x66, 0x28, 0x65, 0x29, 0x74, 0x2e, 0x73, 0x65, 0x74, 0x41, - 0x74, 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, 0x65, 0x28, 0x6e, 0x2c, 0x65, - 0x29, 0x3b, 0x65, 0x6c, 0x73, 0x65, 0x20, 0x74, 0x2e, 0x72, 0x65, 0x6d, - 0x6f, 0x76, 0x65, 0x41, 0x74, 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, 0x65, - 0x28, 0x6e, 0x29, 0x7d, 0x7d, 0x29, 0x7d, 0x7d, 0x42, 0x74, 0x28, 0x22, - 0x75, 0x6e, 0x6d, 0x6f, 0x75, 0x6e, 0x74, 0x22, 0x2c, 0x28, 0x74, 0x2c, - 0x6e, 0x29, 0x3d, 0x3e, 0x7b, 0x69, 0x66, 0x28, 0x22, 0x73, 0x74, 0x72, - 0x69, 0x6e, 0x67, 0x22, 0x3d, 0x3d, 0x74, 0x79, 0x70, 0x65, 0x6f, 0x66, - 0x20, 0x6e, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x29, 0x7b, 0x6c, 0x65, 0x74, - 0x20, 0x74, 0x3d, 0x6e, 0x2e, 0x5f, 0x5f, 0x65, 0x3b, 0x69, 0x66, 0x28, - 0x74, 0x29, 0x7b, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x6e, 0x3d, 0x74, - 0x2e, 0x55, 0x3b, 0x69, 0x66, 0x28, 0x6e, 0x29, 0x7b, 0x74, 0x2e, 0x55, - 0x3d, 0x76, 0x6f, 0x69, 0x64, 0x20, 0x30, 0x3b, 0x66, 0x6f, 0x72, 0x28, - 0x6c, 0x65, 0x74, 0x20, 0x74, 0x20, 0x69, 0x6e, 0x20, 0x6e, 0x29, 0x7b, - 0x6c, 0x65, 0x74, 0x20, 0x65, 0x3d, 0x6e, 0x5b, 0x74, 0x5d, 0x3b, 0x69, - 0x66, 0x28, 0x65, 0x29, 0x65, 0x2e, 0x64, 0x28, 0x29, 0x7d, 0x7d, 0x7d, - 0x7d, 0x65, 0x6c, 0x73, 0x65, 0x7b, 0x6c, 0x65, 0x74, 0x20, 0x74, 0x3d, - 0x6e, 0x2e, 0x5f, 0x5f, 0x63, 0x3b, 0x69, 0x66, 0x28, 0x74, 0x29, 0x7b, - 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x6e, 0x3d, 0x74, 0x2e, 0x5f, 0x5f, - 0x24, 0x75, 0x3b, 0x69, 0x66, 0x28, 0x6e, 0x29, 0x7b, 0x74, 0x2e, 0x5f, - 0x5f, 0x24, 0x75, 0x3d, 0x76, 0x6f, 0x69, 0x64, 0x20, 0x30, 0x3b, 0x6e, - 0x2e, 0x64, 0x28, 0x29, 0x7d, 0x7d, 0x7d, 0x74, 0x28, 0x6e, 0x29, 0x7d, - 0x29, 0x3b, 0x42, 0x74, 0x28, 0x22, 0x5f, 0x5f, 0x68, 0x22, 0x2c, 0x28, - 0x74, 0x2c, 0x6e, 0x2c, 0x65, 0x2c, 0x5f, 0x29, 0x3d, 0x3e, 0x7b, 0x69, - 0x66, 0x28, 0x5f, 0x3c, 0x33, 0x7c, 0x7c, 0x39, 0x3d, 0x3d, 0x3d, 0x5f, - 0x29, 0x6e, 0x2e, 0x5f, 0x5f, 0x24, 0x66, 0x7c, 0x3d, 0x32, 0x3b, 0x74, - 0x28, 0x6e, 0x2c, 0x65, 0x2c, 0x5f, 0x29, 0x7d, 0x29, 0x3b, 0x49, 0x2e, - 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x74, 0x79, 0x70, 0x65, 0x2e, 0x73, 0x68, - 0x6f, 0x75, 0x6c, 0x64, 0x43, 0x6f, 0x6d, 0x70, 0x6f, 0x6e, 0x65, 0x6e, - 0x74, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x3d, 0x66, 0x75, 0x6e, 0x63, - 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x74, 0x2c, 0x6e, 0x29, 0x7b, 0x63, 0x6f, - 0x6e, 0x73, 0x74, 0x20, 0x65, 0x3d, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x5f, - 0x5f, 0x24, 0x75, 0x3b, 0x69, 0x66, 0x28, 0x21, 0x28, 0x65, 0x26, 0x26, - 0x76, 0x6f, 0x69, 0x64, 0x20, 0x30, 0x21, 0x3d, 0x3d, 0x65, 0x2e, 0x73, - 0x7c, 0x7c, 0x34, 0x26, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x5f, 0x5f, 0x24, - 0x66, 0x29, 0x29, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x21, 0x30, 0x3b, - 0x69, 0x66, 0x28, 0x33, 0x26, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x5f, 0x5f, - 0x24, 0x66, 0x29, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x21, 0x30, 0x3b, - 0x66, 0x6f, 0x72, 0x28, 0x6c, 0x65, 0x74, 0x20, 0x5f, 0x20, 0x69, 0x6e, - 0x20, 0x6e, 0x29, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x21, 0x30, 0x3b, - 0x66, 0x6f, 0x72, 0x28, 0x6c, 0x65, 0x74, 0x20, 0x5f, 0x20, 0x69, 0x6e, - 0x20, 0x74, 0x29, 0x69, 0x66, 0x28, 0x22, 0x5f, 0x5f, 0x73, 0x6f, 0x75, - 0x72, 0x63, 0x65, 0x22, 0x21, 0x3d, 0x3d, 0x5f, 0x26, 0x26, 0x74, 0x5b, - 0x5f, 0x5d, 0x21, 0x3d, 0x3d, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x70, 0x72, - 0x6f, 0x70, 0x73, 0x5b, 0x5f, 0x5d, 0x29, 0x72, 0x65, 0x74, 0x75, 0x72, - 0x6e, 0x21, 0x30, 0x3b, 0x66, 0x6f, 0x72, 0x28, 0x6c, 0x65, 0x74, 0x20, - 0x5f, 0x20, 0x69, 0x6e, 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x70, 0x72, - 0x6f, 0x70, 0x73, 0x29, 0x69, 0x66, 0x28, 0x21, 0x28, 0x5f, 0x20, 0x69, - 0x6e, 0x20, 0x74, 0x29, 0x29, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x21, - 0x30, 0x3b, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x21, 0x31, 0x7d, 0x3b, - 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x58, 0x74, 0x28, - 0x74, 0x29, 0x7b, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x44, 0x74, - 0x28, 0x28, 0x29, 0x3d, 0x3e, 0x61, 0x28, 0x74, 0x29, 0x2c, 0x5b, 0x5d, - 0x29, 0x7d, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x59, - 0x74, 0x28, 0x74, 0x29, 0x7b, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x6e, - 0x3d, 0x4e, 0x74, 0x28, 0x74, 0x29, 0x3b, 0x6e, 0x2e, 0x63, 0x75, 0x72, - 0x72, 0x65, 0x6e, 0x74, 0x3d, 0x74, 0x3b, 0x47, 0x74, 0x2e, 0x5f, 0x5f, - 0x24, 0x66, 0x7c, 0x3d, 0x34, 0x3b, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, - 0x20, 0x44, 0x74, 0x28, 0x28, 0x29, 0x3d, 0x3e, 0x6d, 0x28, 0x28, 0x29, - 0x3d, 0x3e, 0x6e, 0x2e, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x74, 0x28, - 0x29, 0x29, 0x2c, 0x5b, 0x5d, 0x29, 0x7d, 0x66, 0x75, 0x6e, 0x63, 0x74, - 0x69, 0x6f, 0x6e, 0x20, 0x5a, 0x74, 0x28, 0x74, 0x29, 0x7b, 0x63, 0x6f, - 0x6e, 0x73, 0x74, 0x20, 0x6e, 0x3d, 0x4e, 0x74, 0x28, 0x74, 0x29, 0x3b, - 0x6e, 0x2e, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x74, 0x3d, 0x74, 0x3b, - 0x48, 0x74, 0x28, 0x28, 0x29, 0x3d, 0x3e, 0x77, 0x28, 0x28, 0x29, 0x3d, - 0x3e, 0x6e, 0x2e, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x74, 0x28, 0x29, - 0x29, 0x2c, 0x5b, 0x5d, 0x29, 0x7d, 0x76, 0x61, 0x72, 0x20, 0x74, 0x6e, - 0x3d, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x74, 0x2c, - 0x6e, 0x2c, 0x65, 0x2c, 0x5f, 0x29, 0x7b, 0x76, 0x61, 0x72, 0x20, 0x69, - 0x3b, 0x6e, 0x5b, 0x30, 0x5d, 0x3d, 0x30, 0x3b, 0x66, 0x6f, 0x72, 0x28, - 0x76, 0x61, 0x72, 0x20, 0x6f, 0x3d, 0x31, 0x3b, 0x6f, 0x3c, 0x6e, 0x2e, - 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x3b, 0x6f, 0x2b, 0x2b, 0x29, 0x7b, - 0x76, 0x61, 0x72, 0x20, 0x72, 0x3d, 0x6e, 0x5b, 0x6f, 0x2b, 0x2b, 0x5d, - 0x2c, 0x75, 0x3d, 0x6e, 0x5b, 0x6f, 0x5d, 0x3f, 0x28, 0x6e, 0x5b, 0x30, - 0x5d, 0x7c, 0x3d, 0x72, 0x3f, 0x31, 0x3a, 0x32, 0x2c, 0x65, 0x5b, 0x6e, - 0x5b, 0x6f, 0x2b, 0x2b, 0x5d, 0x5d, 0x29, 0x3a, 0x6e, 0x5b, 0x2b, 0x2b, - 0x6f, 0x5d, 0x3b, 0x33, 0x3d, 0x3d, 0x3d, 0x72, 0x3f, 0x5f, 0x5b, 0x30, - 0x5d, 0x3d, 0x75, 0x3a, 0x34, 0x3d, 0x3d, 0x3d, 0x72, 0x3f, 0x5f, 0x5b, - 0x31, 0x5d, 0x3d, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x2e, 0x61, 0x73, - 0x73, 0x69, 0x67, 0x6e, 0x28, 0x5f, 0x5b, 0x31, 0x5d, 0x7c, 0x7c, 0x7b, - 0x7d, 0x2c, 0x75, 0x29, 0x3a, 0x35, 0x3d, 0x3d, 0x3d, 0x72, 0x3f, 0x28, - 0x5f, 0x5b, 0x31, 0x5d, 0x3d, 0x5f, 0x5b, 0x31, 0x5d, 0x7c, 0x7c, 0x7b, - 0x7d, 0x29, 0x5b, 0x6e, 0x5b, 0x2b, 0x2b, 0x6f, 0x5d, 0x5d, 0x3d, 0x75, - 0x3a, 0x36, 0x3d, 0x3d, 0x3d, 0x72, 0x3f, 0x5f, 0x5b, 0x31, 0x5d, 0x5b, - 0x6e, 0x5b, 0x2b, 0x2b, 0x6f, 0x5d, 0x5d, 0x2b, 0x3d, 0x75, 0x2b, 0x22, - 0x22, 0x3a, 0x72, 0x3f, 0x28, 0x69, 0x3d, 0x74, 0x2e, 0x61, 0x70, 0x70, - 0x6c, 0x79, 0x28, 0x75, 0x2c, 0x74, 0x6e, 0x28, 0x74, 0x2c, 0x75, 0x2c, - 0x65, 0x2c, 0x5b, 0x22, 0x22, 0x2c, 0x6e, 0x75, 0x6c, 0x6c, 0x5d, 0x29, - 0x29, 0x2c, 0x5f, 0x2e, 0x70, 0x75, 0x73, 0x68, 0x28, 0x69, 0x29, 0x2c, - 0x75, 0x5b, 0x30, 0x5d, 0x3f, 0x6e, 0x5b, 0x30, 0x5d, 0x7c, 0x3d, 0x32, - 0x3a, 0x28, 0x6e, 0x5b, 0x6f, 0x2d, 0x32, 0x5d, 0x3d, 0x30, 0x2c, 0x6e, - 0x5b, 0x6f, 0x5d, 0x3d, 0x69, 0x29, 0x29, 0x3a, 0x5f, 0x2e, 0x70, 0x75, - 0x73, 0x68, 0x28, 0x75, 0x29, 0x7d, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, - 0x20, 0x5f, 0x7d, 0x2c, 0x6e, 0x6e, 0x3d, 0x6e, 0x65, 0x77, 0x20, 0x4d, - 0x61, 0x70, 0x3b, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, - 0x65, 0x6e, 0x28, 0x74, 0x29, 0x7b, 0x76, 0x61, 0x72, 0x20, 0x6e, 0x3d, - 0x6e, 0x6e, 0x2e, 0x67, 0x65, 0x74, 0x28, 0x74, 0x68, 0x69, 0x73, 0x29, - 0x3b, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x6e, 0x7c, 0x7c, 0x28, - 0x6e, 0x3d, 0x6e, 0x65, 0x77, 0x20, 0x4d, 0x61, 0x70, 0x2c, 0x6e, 0x6e, - 0x2e, 0x73, 0x65, 0x74, 0x28, 0x74, 0x68, 0x69, 0x73, 0x2c, 0x6e, 0x29, - 0x29, 0x2c, 0x28, 0x6e, 0x3d, 0x74, 0x6e, 0x28, 0x74, 0x68, 0x69, 0x73, - 0x2c, 0x6e, 0x2e, 0x67, 0x65, 0x74, 0x28, 0x74, 0x29, 0x7c, 0x7c, 0x28, - 0x6e, 0x2e, 0x73, 0x65, 0x74, 0x28, 0x74, 0x2c, 0x6e, 0x3d, 0x66, 0x75, - 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x74, 0x29, 0x7b, 0x66, 0x6f, - 0x72, 0x28, 0x76, 0x61, 0x72, 0x20, 0x6e, 0x2c, 0x65, 0x2c, 0x5f, 0x3d, - 0x31, 0x2c, 0x69, 0x3d, 0x22, 0x22, 0x2c, 0x6f, 0x3d, 0x22, 0x22, 0x2c, - 0x72, 0x3d, 0x5b, 0x30, 0x5d, 0x2c, 0x75, 0x3d, 0x66, 0x75, 0x6e, 0x63, - 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x74, 0x29, 0x7b, 0x31, 0x3d, 0x3d, 0x3d, - 0x5f, 0x26, 0x26, 0x28, 0x74, 0x7c, 0x7c, 0x28, 0x69, 0x3d, 0x69, 0x2e, - 0x72, 0x65, 0x70, 0x6c, 0x61, 0x63, 0x65, 0x28, 0x2f, 0x5e, 0x5c, 0x73, - 0x2a, 0x5c, 0x6e, 0x5c, 0x73, 0x2a, 0x7c, 0x5c, 0x73, 0x2a, 0x5c, 0x6e, - 0x5c, 0x73, 0x2a, 0x24, 0x2f, 0x67, 0x2c, 0x22, 0x22, 0x29, 0x29, 0x29, - 0x3f, 0x72, 0x2e, 0x70, 0x75, 0x73, 0x68, 0x28, 0x30, 0x2c, 0x74, 0x2c, - 0x69, 0x29, 0x3a, 0x33, 0x3d, 0x3d, 0x3d, 0x5f, 0x26, 0x26, 0x28, 0x74, - 0x7c, 0x7c, 0x69, 0x29, 0x3f, 0x28, 0x72, 0x2e, 0x70, 0x75, 0x73, 0x68, - 0x28, 0x33, 0x2c, 0x74, 0x2c, 0x69, 0x29, 0x2c, 0x5f, 0x3d, 0x32, 0x29, - 0x3a, 0x32, 0x3d, 0x3d, 0x3d, 0x5f, 0x26, 0x26, 0x22, 0x2e, 0x2e, 0x2e, - 0x22, 0x3d, 0x3d, 0x3d, 0x69, 0x26, 0x26, 0x74, 0x3f, 0x72, 0x2e, 0x70, - 0x75, 0x73, 0x68, 0x28, 0x34, 0x2c, 0x74, 0x2c, 0x30, 0x29, 0x3a, 0x32, - 0x3d, 0x3d, 0x3d, 0x5f, 0x26, 0x26, 0x69, 0x26, 0x26, 0x21, 0x74, 0x3f, - 0x72, 0x2e, 0x70, 0x75, 0x73, 0x68, 0x28, 0x35, 0x2c, 0x30, 0x2c, 0x21, - 0x30, 0x2c, 0x69, 0x29, 0x3a, 0x5f, 0x3e, 0x3d, 0x35, 0x26, 0x26, 0x28, - 0x28, 0x69, 0x7c, 0x7c, 0x21, 0x74, 0x26, 0x26, 0x35, 0x3d, 0x3d, 0x3d, - 0x5f, 0x29, 0x26, 0x26, 0x28, 0x72, 0x2e, 0x70, 0x75, 0x73, 0x68, 0x28, - 0x5f, 0x2c, 0x30, 0x2c, 0x69, 0x2c, 0x65, 0x29, 0x2c, 0x5f, 0x3d, 0x36, - 0x29, 0x2c, 0x74, 0x26, 0x26, 0x28, 0x72, 0x2e, 0x70, 0x75, 0x73, 0x68, - 0x28, 0x5f, 0x2c, 0x74, 0x2c, 0x30, 0x2c, 0x65, 0x29, 0x2c, 0x5f, 0x3d, - 0x36, 0x29, 0x29, 0x2c, 0x69, 0x3d, 0x22, 0x22, 0x7d, 0x2c, 0x66, 0x3d, - 0x30, 0x3b, 0x66, 0x3c, 0x74, 0x2e, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, - 0x3b, 0x66, 0x2b, 0x2b, 0x29, 0x7b, 0x66, 0x26, 0x26, 0x28, 0x31, 0x3d, - 0x3d, 0x3d, 0x5f, 0x26, 0x26, 0x75, 0x28, 0x29, 0x2c, 0x75, 0x28, 0x66, - 0x29, 0x29, 0x3b, 0x66, 0x6f, 0x72, 0x28, 0x76, 0x61, 0x72, 0x20, 0x73, - 0x3d, 0x30, 0x3b, 0x73, 0x3c, 0x74, 0x5b, 0x66, 0x5d, 0x2e, 0x6c, 0x65, - 0x6e, 0x67, 0x74, 0x68, 0x3b, 0x73, 0x2b, 0x2b, 0x29, 0x6e, 0x3d, 0x74, - 0x5b, 0x66, 0x5d, 0x5b, 0x73, 0x5d, 0x2c, 0x31, 0x3d, 0x3d, 0x3d, 0x5f, - 0x3f, 0x22, 0x3c, 0x22, 0x3d, 0x3d, 0x3d, 0x6e, 0x3f, 0x28, 0x75, 0x28, - 0x29, 0x2c, 0x72, 0x3d, 0x5b, 0x72, 0x5d, 0x2c, 0x5f, 0x3d, 0x33, 0x29, - 0x3a, 0x69, 0x2b, 0x3d, 0x6e, 0x3a, 0x34, 0x3d, 0x3d, 0x3d, 0x5f, 0x3f, - 0x22, 0x2d, 0x2d, 0x22, 0x3d, 0x3d, 0x3d, 0x69, 0x26, 0x26, 0x22, 0x3e, - 0x22, 0x3d, 0x3d, 0x3d, 0x6e, 0x3f, 0x28, 0x5f, 0x3d, 0x31, 0x2c, 0x69, - 0x3d, 0x22, 0x22, 0x29, 0x3a, 0x69, 0x3d, 0x6e, 0x2b, 0x69, 0x5b, 0x30, - 0x5d, 0x3a, 0x6f, 0x3f, 0x6e, 0x3d, 0x3d, 0x3d, 0x6f, 0x3f, 0x6f, 0x3d, - 0x22, 0x22, 0x3a, 0x69, 0x2b, 0x3d, 0x6e, 0x3a, 0x27, 0x22, 0x27, 0x3d, - 0x3d, 0x3d, 0x6e, 0x7c, 0x7c, 0x22, 0x27, 0x22, 0x3d, 0x3d, 0x3d, 0x6e, - 0x3f, 0x6f, 0x3d, 0x6e, 0x3a, 0x22, 0x3e, 0x22, 0x3d, 0x3d, 0x3d, 0x6e, - 0x3f, 0x28, 0x75, 0x28, 0x29, 0x2c, 0x5f, 0x3d, 0x31, 0x29, 0x3a, 0x5f, - 0x26, 0x26, 0x28, 0x22, 0x3d, 0x22, 0x3d, 0x3d, 0x3d, 0x6e, 0x3f, 0x28, - 0x5f, 0x3d, 0x35, 0x2c, 0x65, 0x3d, 0x69, 0x2c, 0x69, 0x3d, 0x22, 0x22, - 0x29, 0x3a, 0x22, 0x2f, 0x22, 0x3d, 0x3d, 0x3d, 0x6e, 0x26, 0x26, 0x28, - 0x5f, 0x3c, 0x35, 0x7c, 0x7c, 0x22, 0x3e, 0x22, 0x3d, 0x3d, 0x3d, 0x74, - 0x5b, 0x66, 0x5d, 0x5b, 0x73, 0x2b, 0x31, 0x5d, 0x29, 0x3f, 0x28, 0x75, - 0x28, 0x29, 0x2c, 0x33, 0x3d, 0x3d, 0x3d, 0x5f, 0x26, 0x26, 0x28, 0x72, - 0x3d, 0x72, 0x5b, 0x30, 0x5d, 0x29, 0x2c, 0x5f, 0x3d, 0x72, 0x2c, 0x28, - 0x72, 0x3d, 0x72, 0x5b, 0x30, 0x5d, 0x29, 0x2e, 0x70, 0x75, 0x73, 0x68, - 0x28, 0x32, 0x2c, 0x30, 0x2c, 0x5f, 0x29, 0x2c, 0x5f, 0x3d, 0x30, 0x29, - 0x3a, 0x22, 0x20, 0x22, 0x3d, 0x3d, 0x3d, 0x6e, 0x7c, 0x7c, 0x22, 0x5c, - 0x74, 0x22, 0x3d, 0x3d, 0x3d, 0x6e, 0x7c, 0x7c, 0x22, 0x5c, 0x6e, 0x22, - 0x3d, 0x3d, 0x3d, 0x6e, 0x7c, 0x7c, 0x22, 0x5c, 0x72, 0x22, 0x3d, 0x3d, - 0x3d, 0x6e, 0x3f, 0x28, 0x75, 0x28, 0x29, 0x2c, 0x5f, 0x3d, 0x32, 0x29, - 0x3a, 0x69, 0x2b, 0x3d, 0x6e, 0x29, 0x2c, 0x33, 0x3d, 0x3d, 0x3d, 0x5f, - 0x26, 0x26, 0x22, 0x21, 0x2d, 0x2d, 0x22, 0x3d, 0x3d, 0x3d, 0x69, 0x26, - 0x26, 0x28, 0x5f, 0x3d, 0x34, 0x2c, 0x72, 0x3d, 0x72, 0x5b, 0x30, 0x5d, - 0x29, 0x7d, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x75, 0x28, 0x29, - 0x2c, 0x72, 0x7d, 0x28, 0x74, 0x29, 0x29, 0x2c, 0x6e, 0x29, 0x2c, 0x61, - 0x72, 0x67, 0x75, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x2c, 0x5b, 0x5d, 0x29, - 0x29, 0x2e, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x3e, 0x31, 0x3f, 0x6e, - 0x3a, 0x6e, 0x5b, 0x30, 0x5d, 0x7d, 0x76, 0x61, 0x72, 0x20, 0x5f, 0x6e, - 0x3d, 0x65, 0x6e, 0x2e, 0x62, 0x69, 0x6e, 0x64, 0x28, 0x4c, 0x29, 0x3b, - 0x65, 0x78, 0x70, 0x6f, 0x72, 0x74, 0x7b, 0x49, 0x20, 0x61, 0x73, 0x20, - 0x43, 0x6f, 0x6d, 0x70, 0x6f, 0x6e, 0x65, 0x6e, 0x74, 0x2c, 0x6a, 0x20, - 0x61, 0x73, 0x20, 0x46, 0x72, 0x61, 0x67, 0x6d, 0x65, 0x6e, 0x74, 0x2c, - 0x68, 0x20, 0x61, 0x73, 0x20, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x2c, - 0x5f, 0x20, 0x61, 0x73, 0x20, 0x62, 0x61, 0x74, 0x63, 0x68, 0x2c, 0x63, - 0x74, 0x20, 0x61, 0x73, 0x20, 0x63, 0x6c, 0x6f, 0x6e, 0x65, 0x45, 0x6c, - 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2c, 0x6d, 0x20, 0x61, 0x73, 0x20, 0x63, - 0x6f, 0x6d, 0x70, 0x75, 0x74, 0x65, 0x64, 0x2c, 0x68, 0x74, 0x20, 0x61, - 0x73, 0x20, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x43, 0x6f, 0x6e, 0x74, - 0x65, 0x78, 0x74, 0x2c, 0x4c, 0x20, 0x61, 0x73, 0x20, 0x63, 0x72, 0x65, - 0x61, 0x74, 0x65, 0x45, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2c, 0x52, - 0x20, 0x61, 0x73, 0x20, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x52, 0x65, - 0x66, 0x2c, 0x77, 0x20, 0x61, 0x73, 0x20, 0x65, 0x66, 0x66, 0x65, 0x63, - 0x74, 0x2c, 0x4c, 0x20, 0x61, 0x73, 0x20, 0x68, 0x2c, 0x5f, 0x6e, 0x20, - 0x61, 0x73, 0x20, 0x68, 0x74, 0x6d, 0x6c, 0x2c, 0x6c, 0x74, 0x20, 0x61, - 0x73, 0x20, 0x68, 0x79, 0x64, 0x72, 0x61, 0x74, 0x65, 0x2c, 0x55, 0x20, - 0x61, 0x73, 0x20, 0x69, 0x73, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x45, 0x6c, - 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2c, 0x43, 0x20, 0x61, 0x73, 0x20, 0x6f, - 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x2c, 0x73, 0x74, 0x20, 0x61, 0x73, - 0x20, 0x72, 0x65, 0x6e, 0x64, 0x65, 0x72, 0x2c, 0x61, 0x20, 0x61, 0x73, - 0x20, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x2c, 0x58, 0x20, 0x61, 0x73, - 0x20, 0x74, 0x6f, 0x43, 0x68, 0x69, 0x6c, 0x64, 0x41, 0x72, 0x72, 0x61, - 0x79, 0x2c, 0x75, 0x20, 0x61, 0x73, 0x20, 0x75, 0x6e, 0x74, 0x72, 0x61, - 0x63, 0x6b, 0x65, 0x64, 0x2c, 0x54, 0x74, 0x20, 0x61, 0x73, 0x20, 0x75, - 0x73, 0x65, 0x43, 0x61, 0x6c, 0x6c, 0x62, 0x61, 0x63, 0x6b, 0x2c, 0x59, - 0x74, 0x20, 0x61, 0x73, 0x20, 0x75, 0x73, 0x65, 0x43, 0x6f, 0x6d, 0x70, - 0x75, 0x74, 0x65, 0x64, 0x2c, 0x56, 0x74, 0x20, 0x61, 0x73, 0x20, 0x75, - 0x73, 0x65, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x2c, 0x41, 0x74, - 0x20, 0x61, 0x73, 0x20, 0x75, 0x73, 0x65, 0x44, 0x65, 0x62, 0x75, 0x67, - 0x56, 0x61, 0x6c, 0x75, 0x65, 0x2c, 0x48, 0x74, 0x20, 0x61, 0x73, 0x20, - 0x75, 0x73, 0x65, 0x45, 0x66, 0x66, 0x65, 0x63, 0x74, 0x2c, 0x46, 0x74, - 0x20, 0x61, 0x73, 0x20, 0x75, 0x73, 0x65, 0x45, 0x72, 0x72, 0x6f, 0x72, - 0x42, 0x6f, 0x75, 0x6e, 0x64, 0x61, 0x72, 0x79, 0x2c, 0x4d, 0x74, 0x20, - 0x61, 0x73, 0x20, 0x75, 0x73, 0x65, 0x49, 0x64, 0x2c, 0x24, 0x74, 0x20, - 0x61, 0x73, 0x20, 0x75, 0x73, 0x65, 0x49, 0x6d, 0x70, 0x65, 0x72, 0x61, - 0x74, 0x69, 0x76, 0x65, 0x48, 0x61, 0x6e, 0x64, 0x6c, 0x65, 0x2c, 0x50, - 0x74, 0x20, 0x61, 0x73, 0x20, 0x75, 0x73, 0x65, 0x4c, 0x61, 0x79, 0x6f, - 0x75, 0x74, 0x45, 0x66, 0x66, 0x65, 0x63, 0x74, 0x2c, 0x44, 0x74, 0x20, - 0x61, 0x73, 0x20, 0x75, 0x73, 0x65, 0x4d, 0x65, 0x6d, 0x6f, 0x2c, 0x55, - 0x74, 0x20, 0x61, 0x73, 0x20, 0x75, 0x73, 0x65, 0x52, 0x65, 0x64, 0x75, - 0x63, 0x65, 0x72, 0x2c, 0x4e, 0x74, 0x20, 0x61, 0x73, 0x20, 0x75, 0x73, - 0x65, 0x52, 0x65, 0x66, 0x2c, 0x58, 0x74, 0x20, 0x61, 0x73, 0x20, 0x75, - 0x73, 0x65, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x2c, 0x5a, 0x74, 0x20, - 0x61, 0x73, 0x20, 0x75, 0x73, 0x65, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x6c, - 0x45, 0x66, 0x66, 0x65, 0x63, 0x74, 0x2c, 0x45, 0x74, 0x20, 0x61, 0x73, - 0x20, 0x75, 0x73, 0x65, 0x53, 0x74, 0x61, 0x74, 0x65, 0x7d, 0x3b, 0x0a -}; -unsigned int index_js_len = 22800; +const char index_js[] = R"LITERAL( +function t(){throw new Error("Cycle detected")}const n=Symbol.for("preact-signals");function e(){if(f>1){f--;return}let t,n=!1;while(void 0!==o){let _=o;o=void 0;s++;while(void 0!==_){const i=_.o;_.o=void 0;_.f&=-3;if(!(8&_.f)&&p(_))try{_.c()}catch(e){if(!n){t=e;n=!0}}_=i}}s=0;f--;if(n)throw t}function _(t){if(f>0)return t();f++;try{return t()}finally{e()}}let i,o,r=0;function u(t){if(r>0)return t();const n=i;i=void 0;r++;try{return t()}finally{r--;i=n}}let f=0,s=0,l=0;function c(t){if(void 0===i)return;let n=t.n;if(void 0===n||n.t!==i){n={i:0,S:t,p:i.s,n:void 0,t:i,e:void 0,x:void 0,r:n};if(void 0!==i.s)i.s.n=n;i.s=n;t.n=n;if(32&i.f)t.S(n);return n}else if(-1===n.i){n.i=0;if(void 0!==n.n){n.n.p=n.p;if(void 0!==n.p)n.p.n=n.n;n.p=i.s;n.n=void 0;i.s.n=n;i.s=n}return n}}function h(t){this.v=t;this.i=0;this.n=void 0;this.t=void 0}h.prototype.brand=n;h.prototype.h=function(){return!0};h.prototype.S=function(t){if(this.t!==t&&void 0===t.e){t.x=this.t;if(void 0!==this.t)this.t.e=t;this.t=t}};h.prototype.U=function(t){if(void 0!==this.t){const n=t.e,e=t.x;if(void 0!==n){n.x=e;t.e=void 0}if(void 0!==e){e.e=n;t.x=void 0}if(t===this.t)this.t=e}};h.prototype.subscribe=function(t){const n=this;return w((function(){const e=n.value,_=32&this.f;this.f&=-33;try{t(e)}finally{this.f|=_}}))};h.prototype.valueOf=function(){return this.value};h.prototype.toString=function(){return this.value+""};h.prototype.toJSON=function(){return this.value};h.prototype.peek=function(){return this.v};Object.defineProperty(h.prototype,"value",{get(){const t=c(this);if(void 0!==t)t.i=this.i;return this.v},set(n){if(i instanceof y)!function(){throw new Error("Computed cannot have side-effects")}();if(n!==this.v){if(s>100)t();this.v=n;this.i++;l++;f++;try{for(let t=this.t;void 0!==t;t=t.x)t.t.N()}finally{e()}}}});function a(t){return new h(t)}function p(t){for(let n=t.s;void 0!==n;n=n.n)if(n.S.i!==n.i||!n.S.h()||n.S.i!==n.i)return!0;return!1}function d(t){for(let n=t.s;void 0!==n;n=n.n){const e=n.S.n;if(void 0!==e)n.r=e;n.S.n=n;n.i=-1;if(void 0===n.n){t.s=n;break}}}function v(t){let n,e=t.s;while(void 0!==e){const t=e.p;if(-1===e.i){e.S.U(e);if(void 0!==t)t.n=e.n;if(void 0!==e.n)e.n.p=t}else n=e;e.S.n=e.r;if(void 0!==e.r)e.r=void 0;e=t}t.s=n}function y(t){h.call(this,void 0);this.x=t;this.s=void 0;this.g=l-1;this.f=4}(y.prototype=new h).h=function(){this.f&=-3;if(1&this.f)return!1;if(32==(36&this.f))return!0;this.f&=-5;if(this.g===l)return!0;this.g=l;this.f|=1;if(this.i>0&&!p(this)){this.f&=-2;return!0}const t=i;try{d(this);i=this;const t=this.x();if(16&this.f||this.v!==t||0===this.i){this.v=t;this.f&=-17;this.i++}}catch(t){this.v=t;this.f|=16;this.i++}i=t;v(this);this.f&=-2;return!0};y.prototype.S=function(t){if(void 0===this.t){this.f|=36;for(let t=this.s;void 0!==t;t=t.n)t.S.S(t)}h.prototype.S.call(this,t)};y.prototype.U=function(t){if(void 0!==this.t){h.prototype.U.call(this,t);if(void 0===this.t){this.f&=-33;for(let t=this.s;void 0!==t;t=t.n)t.S.U(t)}}};y.prototype.N=function(){if(!(2&this.f)){this.f|=6;for(let t=this.t;void 0!==t;t=t.x)t.t.N()}};y.prototype.peek=function(){if(!this.h())t();if(16&this.f)throw this.v;return this.v};Object.defineProperty(y.prototype,"value",{get(){if(1&this.f)t();const n=c(this);this.h();if(void 0!==n)n.i=this.i;if(16&this.f)throw this.v;return this.v}});function m(t){return new y(t)}function g(t){const n=t.u;t.u=void 0;if("function"==typeof n){f++;const _=i;i=void 0;try{n()}catch(n){t.f&=-2;t.f|=8;b(t);throw n}finally{i=_;e()}}}function b(t){for(let n=t.s;void 0!==n;n=n.n)n.S.U(n);t.x=void 0;t.s=void 0;g(t)}function k(t){if(i!==this)throw new Error("Out-of-order effect");v(this);i=t;this.f&=-2;if(8&this.f)b(this);e()}function S(t){this.x=t;this.u=void 0;this.s=void 0;this.o=void 0;this.f=32}S.prototype.c=function(){const t=this.S();try{if(8&this.f)return;if(void 0===this.x)return;const n=this.x();if("function"==typeof n)this.u=n}finally{t()}};S.prototype.S=function(){if(1&this.f)t();this.f|=1;this.f&=-9;g(this);d(this);f++;const n=i;i=this;return k.bind(this,n)};S.prototype.N=function(){if(!(2&this.f)){this.f|=2;this.o=o;o=this}};S.prototype.d=function(){this.f|=8;if(!(1&this.f))b(this)};function w(t){const n=new S(t);try{n.c()}catch(t){n.d();throw t}return n.d.bind(n)}var x,C,E,U,H,P,N,$,D,T={},V=[],A=/acit|ex(?:s|g|n|p|$)|rph|grid|ows|mnc|ntw|ine[ch]|zoo|^ord|itera/i,F=Array.isArray;function M(t,n){for(var e in n)t[e]=n[e];return t}function W(t){var n=t.parentNode;n&&n.removeChild(t)}function L(t,n,e){var _,i,o,r={};for(o in n)"key"==o?_=n[o]:"ref"==o?i=n[o]:r[o]=n[o];if(arguments.length>2&&(r.children=arguments.length>3?x.call(arguments,2):e),"function"==typeof t&&null!=t.defaultProps)for(o in t.defaultProps)void 0===r[o]&&(r[o]=t.defaultProps[o]);return O(t,r,_,i,null)}function O(t,n,e,_,i){var o={type:t,props:n,key:e,ref:_,__k:null,__:null,__b:0,__e:null,__d:void 0,__c:null,constructor:void 0,__v:null==i?++E:i,__i:-1,__u:0};return null==i&&null!=C.vnode&&C.vnode(o),o}function R(){return{current:null}}function j(t){return t.children}function I(t,n){this.props=t,this.context=n}function q(t,n){if(null==n)return t.__?q(t.__,t.__i+1):null;for(var e;nn&&H.sort($));z.__r=0}function J(t,n,e,_,i,o,r,u,f,s,l){var c,h,a,p,d,v=_&&_.__k||V,y=n.length;for(e.__d=f,K(e,n,v),f=e.__d,c=0;c0?O(i.type,i.props,i.key,i.ref?i.ref:null,i.__v):i)?(i.__=t,i.__b=t.__b+1,u=Y(i,e,r=_+c,l),i.__i=u,o=null,-1!==u&&(l--,(o=e[u])&&(o.__u|=131072)),null==o||null===o.__v?(-1==u&&c--,"function"!=typeof i.type&&(i.__u|=65536)):u!==r&&(u===r+1?c++:u>r?l>f-r?c+=u-r:c--:c=u(null!=f&&0==(131072&f.__u)?1:0))for(;r>=0||u=0){if((f=n[r])&&0==(131072&f.__u)&&i==f.key&&o===f.type)return r;r--}if(u2&&(u.children=arguments.length>3?x.call(arguments,2):e),O(t.type,u,_||t.key,i||t.ref,null)}function ht(t,n){var e={__c:n="__cC"+D++,__:t,Consumer:function(t,n){return t.children(n)},Provider:function(t){var e,_;return this.getChildContext||(e=[],(_={})[n]=this,this.getChildContext=function(){return _},this.shouldComponentUpdate=function(t){this.props.value!==t.value&&e.some((function(t){t.__e=!0,G(t)}))},this.sub=function(t){e.push(t);var n=t.componentWillUnmount;t.componentWillUnmount=function(){e.splice(e.indexOf(t),1),n&&n.call(t)}}),t.children}};return e.Provider.__=e.Consumer.contextType=e}x=V.slice,C={__e:function(t,n,e,_){for(var i,o,r;n=n.__;)if((i=n.__c)&&!i.__)try{if((o=i.constructor)&&null!=o.getDerivedStateFromError&&(i.setState(o.getDerivedStateFromError(t)),r=i.__d),null!=i.componentDidCatch&&(i.componentDidCatch(t,_||{}),r=i.__d),r)return i.__E=i}catch(n){t=n}throw t}},E=0,U=function(t){return null!=t&&null==t.constructor},I.prototype.setState=function(t,n){var e;e=null!=this.__s&&this.__s!==this.state?this.__s:this.__s=M({},this.state),"function"==typeof t&&(t=t(M({},e),this.props)),t&&M(e,t),null!=t&&this.__v&&(n&&this._sb.push(n),G(this))},I.prototype.forceUpdate=function(t){this.__v&&(this.__e=!0,t&&this.__h.push(t),G(this))},I.prototype.render=j,H=[],N="function"==typeof Promise?Promise.prototype.then.bind(Promise.resolve()):setTimeout,$=function(t,n){return t.__v.__b-n.__v.__b},z.__r=0,D=0;var at,pt,dt,vt,yt=0,mt=[],gt=[],bt=C.__b,kt=C.__r,St=C.diffed,wt=C.__c,xt=C.unmount;function Ct(t,n){C.__h&&C.__h(pt,t,yt||n),yt=0;var e=pt.__H||(pt.__H={__:[],__h:[]});return t>=e.__.length&&e.__.push({__V:gt}),e.__[t]}function Et(t){return yt=1,Ut(qt,t)}function Ut(t,n,e){var _=Ct(at++,2);if(_.t=t,!_.__c&&(_.__=[e?e(n):qt(void 0,n),function(t){var n=_.__N?_.__N[0]:_.__[0],e=_.t(n,t);n!==e&&(_.__N=[e,_.__[1]],_.__c.setState({}))}],_.__c=pt,!pt.u)){var i=function(t,n,e){if(!_.__c.__H)return!0;var i=_.__c.__H.__.filter((function(t){return t.__c}));if(i.every((function(t){return!t.__N})))return!o||o.call(this,t,n,e);var r=!1;return i.forEach((function(t){if(t.__N){var n=t.__[0];t.__=t.__N,t.__N=void 0,n!==t.__[0]&&(r=!0)}})),!(!r&&_.__c.props===t)&&(!o||o.call(this,t,n,e))};pt.u=!0;var o=pt.shouldComponentUpdate,r=pt.componentWillUpdate;pt.componentWillUpdate=function(t,n,e){if(this.__e){var _=o;o=void 0,i(t,n,e),o=_}r&&r.call(this,t,n,e)},pt.shouldComponentUpdate=i}return _.__N||_.__}function Ht(t,n){var e=Ct(at++,3);!C.__s&&It(e.__H,n)&&(e.__=t,e.i=n,pt.__H.__h.push(e))}function Pt(t,n){var e=Ct(at++,4);!C.__s&&It(e.__H,n)&&(e.__=t,e.i=n,pt.__h.push(e))}function Nt(t){return yt=5,Dt((function(){return{current:t}}),[])}function $t(t,n,e){yt=6,Pt((function(){return"function"==typeof t?(t(n()),function(){return t(null)}):t?(t.current=n(),function(){return t.current=null}):void 0}),null==e?e:e.concat(t))}function Dt(t,n){var e=Ct(at++,7);return It(e.__H,n)?(e.__V=t(),e.i=n,e.__h=t,e.__V):e.__}function Tt(t,n){return yt=8,Dt((function(){return t}),n)}function Vt(t){var n=pt.context[t.__c],e=Ct(at++,9);return e.c=t,n?(null==e.__&&(e.__=!0,n.sub(pt)),n.props.value):t.__}function At(t,n){C.useDebugValue&&C.useDebugValue(n?n(t):t)}function Ft(t){var n=Ct(at++,10),e=Et();return n.__=t,pt.componentDidCatch||(pt.componentDidCatch=function(t,_){n.__&&n.__(t,_),e[1](t)}),[e[0],function(){e[1](void 0)}]}function Mt(){var t=Ct(at++,11);if(!t.__){for(var n=pt.__v;null!==n&&!n.__m&&null!==n.__;)n=n.__;var e=n.__m||(n.__m=[0,0]);t.__="P"+e[0]+"-"+e[1]++}return t.__}function Wt(){for(var t;t=mt.shift();)if(t.__P&&t.__H)try{t.__H.__h.forEach(Rt),t.__H.__h.forEach(jt),t.__H.__h=[]}catch(u){t.__H.__h=[],C.__e(u,t.__v)}}C.__b=function(t){pt=null,bt&&bt(t)},C.__r=function(t){kt&&kt(t),at=0;var n=(pt=t.__c).__H;n&&(dt===pt?(n.__h=[],pt.__h=[],n.__.forEach((function(t){t.__N&&(t.__=t.__N),t.__V=gt,t.__N=t.i=void 0}))):(n.__h.forEach(Rt),n.__h.forEach(jt),n.__h=[],at=0)),dt=pt},C.diffed=function(t){St&&St(t);var n=t.__c;n&&n.__H&&(n.__H.__h.length&&(1!==mt.push(n)&&vt===C.requestAnimationFrame||((vt=C.requestAnimationFrame)||Ot)(Wt)),n.__H.__.forEach((function(t){t.i&&(t.__H=t.i),t.__V!==gt&&(t.__=t.__V),t.i=void 0,t.__V=gt}))),dt=pt=null},C.__c=function(t,n){n.some((function(t){try{t.__h.forEach(Rt),t.__h=t.__h.filter((function(t){return!t.__||jt(t)}))}catch(l){n.some((function(t){t.__h&&(t.__h=[])})),n=[],C.__e(l,t.__v)}})),wt&&wt(t,n)},C.unmount=function(t){xt&&xt(t);var n,e=t.__c;e&&e.__H&&(e.__H.__.forEach((function(t){try{Rt(t)}catch(t){n=t}})),e.__H=void 0,n&&C.__e(n,e.__v))};var Lt="function"==typeof requestAnimationFrame;function Ot(t){var n,e=function(){clearTimeout(_),Lt&&cancelAnimationFrame(n),setTimeout(t)},_=setTimeout(e,100);Lt&&(n=requestAnimationFrame(e))}function Rt(t){var n=pt,e=t.__c;"function"==typeof e&&(t.__c=void 0,e()),pt=n}function jt(t){var n=pt;t.__c=t.__(),pt=n}function It(t,n){return!t||t.length!==n.length||n.some((function(n,e){return n!==t[e]}))}function qt(t,n){return"function"==typeof n?n(t):n}function Bt(t,n){C[t]=n.bind(null,C[t]||(()=>{}))}let Gt,zt;function Jt(t){if(zt)zt();zt=t&&t.S()}function Kt({data:t}){const n=Xt(t);n.value=t;const e=Dt(()=>{let t=this.__v;while(t=t.__)if(t.__c){t.__c.__$f|=4;break}this.__$u.c=()=>{var t;if(!U(e.peek())&&3===(null==(t=this.base)?void 0:t.nodeType))this.base.data=e.peek();else{this.__$f|=1;this.setState({})}};return m(()=>{let t=n.value.value;return 0===t?0:!0===t?"":t||""})},[]);return e.value}Kt.displayName="_st";Object.defineProperties(h.prototype,{constructor:{configurable:!0,value:void 0},type:{configurable:!0,value:Kt},props:{configurable:!0,get(){return{data:this}}},__b:{configurable:!0,value:1}});Bt("__b",(t,n)=>{if("string"==typeof n.type){let t,e=n.props;for(let _ in e){if("children"===_)continue;let i=e[_];if(i instanceof h){if(!t)n.__np=t={};t[_]=i;e[_]=i.peek()}}}t(n)});Bt("__r",(t,n)=>{Jt();let e,_=n.__c;if(_){_.__$f&=-2;e=_.__$u;if(void 0===e)_.__$u=e=function(t){let n;w((function(){n=this}));n.c=()=>{_.__$f|=1;_.setState({})};return n}()}Gt=_;Jt(e);t(n)});Bt("__e",(t,n,e,_)=>{Jt();Gt=void 0;t(n,e,_)});Bt("diffed",(t,n)=>{Jt();Gt=void 0;let e;if("string"==typeof n.type&&(e=n.__e)){let t=n.__np,_=n.props;if(t){let n=e.U;if(n)for(let e in n){let _=n[e];if(void 0!==_&&!(e in t)){_.d();n[e]=void 0}}else{n={};e.U=n}for(let i in t){let o=n[i],r=t[i];if(void 0===o){o=Qt(e,i,r,_);n[i]=o}else o.o(r,_)}}}t(n)});function Qt(t,n,e,_){const i=n in t&&void 0===t.ownerSVGElement,o=a(e);return{o:(t,n)=>{o.value=t;_=n},d:w(()=>{const e=o.value.value;if(_[n]!==e){_[n]=e;if(i)t[n]=e;else if(e)t.setAttribute(n,e);else t.removeAttribute(n)}})}}Bt("unmount",(t,n)=>{if("string"==typeof n.type){let t=n.__e;if(t){const n=t.U;if(n){t.U=void 0;for(let t in n){let e=n[t];if(e)e.d()}}}}else{let t=n.__c;if(t){const n=t.__$u;if(n){t.__$u=void 0;n.d()}}}t(n)});Bt("__h",(t,n,e,_)=>{if(_<3||9===_)n.__$f|=2;t(n,e,_)});I.prototype.shouldComponentUpdate=function(t,n){const e=this.__$u;if(!(e&&void 0!==e.s||4&this.__$f))return!0;if(3&this.__$f)return!0;for(let _ in n)return!0;for(let _ in t)if("__source"!==_&&t[_]!==this.props[_])return!0;for(let _ in this.props)if(!(_ in t))return!0;return!1};function Xt(t){return Dt(()=>a(t),[])}function Yt(t){const n=Nt(t);n.current=t;Gt.__$f|=4;return Dt(()=>m(()=>n.current()),[])}function Zt(t){const n=Nt(t);n.current=t;Ht(()=>w(()=>n.current()),[])}var tn=function(t,n,e,_){var i;n[0]=0;for(var o=1;o=5&&((i||!t&&5===_)&&(r.push(_,0,i,e),_=6),t&&(r.push(_,t,0,e),_=6)),i=""},f=0;f"===n?(_=1,i=""):i=n+i[0]:o?n===o?o="":i+=n:'"'===n||"'"===n?o=n:">"===n?(u(),_=1):_&&("="===n?(_=5,e=i,i=""):"/"===n&&(_<5||">"===t[f][s+1])?(u(),3===_&&(r=r[0]),_=r,(r=r[0]).push(2,0,_),_=0):" "===n||"\t"===n||"\n"===n||"\r"===n?(u(),_=2):i+=n),3===_&&"!--"===i&&(_=4,r=r[0])}return u(),r}(t)),n),arguments,[])).length>1?n:n[0]}var _n=en.bind(L);export{I as Component,j as Fragment,h as Signal,_ as batch,ct as cloneElement,m as computed,ht as createContext,L as createElement,R as createRef,w as effect,L as h,_n as html,lt as hydrate,U as isValidElement,C as options,st as render,a as signal,X as toChildArray,u as untracked,Tt as useCallback,Yt as useComputed,Vt as useContext,At as useDebugValue,Ht as useEffect,Ft as useErrorBoundary,Mt as useId,$t as useImperativeHandle,Pt as useLayoutEffect,Dt as useMemo,Ut as useReducer,Nt as useRef,Xt as useSignal,Zt as useSignalEffect,Et as useState}; +)LITERAL"; +unsigned int index_js_len = sizeof(index_js); diff --git a/examples/server/json-schema-to-grammar.mjs.hpp b/examples/server/json-schema-to-grammar.mjs.hpp index 0a05c369d..83b22d670 100644 --- a/examples/server/json-schema-to-grammar.mjs.hpp +++ b/examples/server/json-schema-to-grammar.mjs.hpp @@ -1,311 +1,115 @@ -unsigned char json_schema_to_grammar_mjs[] = { - 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x53, 0x50, 0x41, 0x43, 0x45, 0x5f, - 0x52, 0x55, 0x4c, 0x45, 0x20, 0x3d, 0x20, 0x27, 0x22, 0x20, 0x22, 0x3f, - 0x27, 0x3b, 0x0a, 0x0a, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x50, 0x52, - 0x49, 0x4d, 0x49, 0x54, 0x49, 0x56, 0x45, 0x5f, 0x52, 0x55, 0x4c, 0x45, - 0x53, 0x20, 0x3d, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x62, 0x6f, 0x6f, 0x6c, - 0x65, 0x61, 0x6e, 0x3a, 0x20, 0x27, 0x28, 0x22, 0x74, 0x72, 0x75, 0x65, - 0x22, 0x20, 0x7c, 0x20, 0x22, 0x66, 0x61, 0x6c, 0x73, 0x65, 0x22, 0x29, - 0x20, 0x73, 0x70, 0x61, 0x63, 0x65, 0x27, 0x2c, 0x0a, 0x20, 0x20, 0x6e, - 0x75, 0x6d, 0x62, 0x65, 0x72, 0x3a, 0x20, 0x27, 0x28, 0x22, 0x2d, 0x22, - 0x3f, 0x20, 0x28, 0x5b, 0x30, 0x2d, 0x39, 0x5d, 0x20, 0x7c, 0x20, 0x5b, - 0x31, 0x2d, 0x39, 0x5d, 0x20, 0x5b, 0x30, 0x2d, 0x39, 0x5d, 0x2a, 0x29, - 0x29, 0x20, 0x28, 0x22, 0x2e, 0x22, 0x20, 0x5b, 0x30, 0x2d, 0x39, 0x5d, - 0x2b, 0x29, 0x3f, 0x20, 0x28, 0x5b, 0x65, 0x45, 0x5d, 0x20, 0x5b, 0x2d, - 0x2b, 0x5d, 0x3f, 0x20, 0x5b, 0x30, 0x2d, 0x39, 0x5d, 0x2b, 0x29, 0x3f, - 0x20, 0x73, 0x70, 0x61, 0x63, 0x65, 0x27, 0x2c, 0x0a, 0x20, 0x20, 0x69, - 0x6e, 0x74, 0x65, 0x67, 0x65, 0x72, 0x3a, 0x20, 0x27, 0x28, 0x22, 0x2d, - 0x22, 0x3f, 0x20, 0x28, 0x5b, 0x30, 0x2d, 0x39, 0x5d, 0x20, 0x7c, 0x20, - 0x5b, 0x31, 0x2d, 0x39, 0x5d, 0x20, 0x5b, 0x30, 0x2d, 0x39, 0x5d, 0x2a, - 0x29, 0x29, 0x20, 0x73, 0x70, 0x61, 0x63, 0x65, 0x27, 0x2c, 0x0a, 0x20, - 0x20, 0x73, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x3a, 0x20, 0x60, 0x20, 0x22, - 0x5c, 0x5c, 0x22, 0x22, 0x20, 0x28, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x5b, 0x5e, 0x22, 0x5c, 0x5c, 0x5c, 0x5c, 0x5d, 0x20, - 0x7c, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x22, 0x5c, - 0x5c, 0x5c, 0x5c, 0x22, 0x20, 0x28, 0x5b, 0x22, 0x5c, 0x5c, 0x5c, 0x5c, - 0x2f, 0x62, 0x66, 0x6e, 0x72, 0x74, 0x5d, 0x20, 0x7c, 0x20, 0x22, 0x75, - 0x22, 0x20, 0x5b, 0x30, 0x2d, 0x39, 0x61, 0x2d, 0x66, 0x41, 0x2d, 0x46, - 0x5d, 0x20, 0x5b, 0x30, 0x2d, 0x39, 0x61, 0x2d, 0x66, 0x41, 0x2d, 0x46, - 0x5d, 0x20, 0x5b, 0x30, 0x2d, 0x39, 0x61, 0x2d, 0x66, 0x41, 0x2d, 0x46, - 0x5d, 0x20, 0x5b, 0x30, 0x2d, 0x39, 0x61, 0x2d, 0x66, 0x41, 0x2d, 0x46, - 0x5d, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x29, 0x2a, 0x20, - 0x22, 0x5c, 0x5c, 0x22, 0x22, 0x20, 0x73, 0x70, 0x61, 0x63, 0x65, 0x60, - 0x2c, 0x0a, 0x20, 0x20, 0x6e, 0x75, 0x6c, 0x6c, 0x3a, 0x20, 0x27, 0x22, - 0x6e, 0x75, 0x6c, 0x6c, 0x22, 0x20, 0x73, 0x70, 0x61, 0x63, 0x65, 0x27, - 0x2c, 0x0a, 0x7d, 0x3b, 0x0a, 0x0a, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, - 0x49, 0x4e, 0x56, 0x41, 0x4c, 0x49, 0x44, 0x5f, 0x52, 0x55, 0x4c, 0x45, - 0x5f, 0x43, 0x48, 0x41, 0x52, 0x53, 0x5f, 0x52, 0x45, 0x20, 0x3d, 0x20, - 0x2f, 0x5b, 0x5e, 0x5c, 0x64, 0x41, 0x2d, 0x5a, 0x61, 0x2d, 0x7a, 0x2d, - 0x5d, 0x2b, 0x2f, 0x67, 0x3b, 0x0a, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, - 0x47, 0x52, 0x41, 0x4d, 0x4d, 0x41, 0x52, 0x5f, 0x4c, 0x49, 0x54, 0x45, - 0x52, 0x41, 0x4c, 0x5f, 0x45, 0x53, 0x43, 0x41, 0x50, 0x45, 0x5f, 0x52, - 0x45, 0x20, 0x3d, 0x20, 0x2f, 0x5b, 0x5c, 0x6e, 0x5c, 0x72, 0x22, 0x5d, - 0x2f, 0x67, 0x3b, 0x0a, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x47, 0x52, - 0x41, 0x4d, 0x4d, 0x41, 0x52, 0x5f, 0x4c, 0x49, 0x54, 0x45, 0x52, 0x41, - 0x4c, 0x5f, 0x45, 0x53, 0x43, 0x41, 0x50, 0x45, 0x53, 0x20, 0x3d, 0x20, - 0x7b, 0x27, 0x5c, 0x72, 0x27, 0x3a, 0x20, 0x27, 0x5c, 0x5c, 0x72, 0x27, - 0x2c, 0x20, 0x27, 0x5c, 0x6e, 0x27, 0x3a, 0x20, 0x27, 0x5c, 0x5c, 0x6e, - 0x27, 0x2c, 0x20, 0x27, 0x22, 0x27, 0x3a, 0x20, 0x27, 0x5c, 0x5c, 0x22, - 0x27, 0x7d, 0x3b, 0x0a, 0x0a, 0x65, 0x78, 0x70, 0x6f, 0x72, 0x74, 0x20, - 0x63, 0x6c, 0x61, 0x73, 0x73, 0x20, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, - 0x43, 0x6f, 0x6e, 0x76, 0x65, 0x72, 0x74, 0x65, 0x72, 0x20, 0x7b, 0x0a, - 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x72, 0x75, 0x63, 0x74, 0x6f, - 0x72, 0x28, 0x70, 0x72, 0x6f, 0x70, 0x4f, 0x72, 0x64, 0x65, 0x72, 0x29, - 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, - 0x5f, 0x70, 0x72, 0x6f, 0x70, 0x4f, 0x72, 0x64, 0x65, 0x72, 0x20, 0x3d, - 0x20, 0x70, 0x72, 0x6f, 0x70, 0x4f, 0x72, 0x64, 0x65, 0x72, 0x20, 0x7c, - 0x7c, 0x20, 0x7b, 0x7d, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x74, 0x68, - 0x69, 0x73, 0x2e, 0x5f, 0x72, 0x75, 0x6c, 0x65, 0x73, 0x20, 0x3d, 0x20, - 0x6e, 0x65, 0x77, 0x20, 0x4d, 0x61, 0x70, 0x28, 0x29, 0x3b, 0x0a, 0x20, - 0x20, 0x20, 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x5f, 0x72, 0x75, 0x6c, - 0x65, 0x73, 0x2e, 0x73, 0x65, 0x74, 0x28, 0x27, 0x73, 0x70, 0x61, 0x63, - 0x65, 0x27, 0x2c, 0x20, 0x53, 0x50, 0x41, 0x43, 0x45, 0x5f, 0x52, 0x55, - 0x4c, 0x45, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x7d, 0x0a, 0x0a, 0x20, 0x20, - 0x5f, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x4c, 0x69, 0x74, 0x65, 0x72, - 0x61, 0x6c, 0x28, 0x6c, 0x69, 0x74, 0x65, 0x72, 0x61, 0x6c, 0x29, 0x20, - 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, - 0x65, 0x73, 0x63, 0x61, 0x70, 0x65, 0x64, 0x20, 0x3d, 0x20, 0x4a, 0x53, - 0x4f, 0x4e, 0x2e, 0x73, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x69, 0x66, 0x79, - 0x28, 0x6c, 0x69, 0x74, 0x65, 0x72, 0x61, 0x6c, 0x29, 0x2e, 0x72, 0x65, - 0x70, 0x6c, 0x61, 0x63, 0x65, 0x28, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x47, 0x52, 0x41, 0x4d, 0x4d, 0x41, 0x52, 0x5f, 0x4c, 0x49, 0x54, - 0x45, 0x52, 0x41, 0x4c, 0x5f, 0x45, 0x53, 0x43, 0x41, 0x50, 0x45, 0x5f, - 0x52, 0x45, 0x2c, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x6d, 0x20, - 0x3d, 0x3e, 0x20, 0x47, 0x52, 0x41, 0x4d, 0x4d, 0x41, 0x52, 0x5f, 0x4c, - 0x49, 0x54, 0x45, 0x52, 0x41, 0x4c, 0x5f, 0x45, 0x53, 0x43, 0x41, 0x50, - 0x45, 0x53, 0x5b, 0x6d, 0x5d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x29, 0x3b, - 0x0a, 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, - 0x60, 0x22, 0x24, 0x7b, 0x65, 0x73, 0x63, 0x61, 0x70, 0x65, 0x64, 0x7d, - 0x22, 0x60, 0x3b, 0x0a, 0x20, 0x20, 0x7d, 0x0a, 0x0a, 0x20, 0x20, 0x5f, - 0x61, 0x64, 0x64, 0x52, 0x75, 0x6c, 0x65, 0x28, 0x6e, 0x61, 0x6d, 0x65, - 0x2c, 0x20, 0x72, 0x75, 0x6c, 0x65, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, - 0x20, 0x20, 0x6c, 0x65, 0x74, 0x20, 0x65, 0x73, 0x63, 0x4e, 0x61, 0x6d, - 0x65, 0x20, 0x3d, 0x20, 0x6e, 0x61, 0x6d, 0x65, 0x2e, 0x72, 0x65, 0x70, - 0x6c, 0x61, 0x63, 0x65, 0x28, 0x49, 0x4e, 0x56, 0x41, 0x4c, 0x49, 0x44, - 0x5f, 0x52, 0x55, 0x4c, 0x45, 0x5f, 0x43, 0x48, 0x41, 0x52, 0x53, 0x5f, - 0x52, 0x45, 0x2c, 0x20, 0x27, 0x2d, 0x27, 0x29, 0x3b, 0x0a, 0x20, 0x20, - 0x20, 0x20, 0x6c, 0x65, 0x74, 0x20, 0x6b, 0x65, 0x79, 0x20, 0x3d, 0x20, - 0x65, 0x73, 0x63, 0x4e, 0x61, 0x6d, 0x65, 0x3b, 0x0a, 0x0a, 0x20, 0x20, - 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x5f, - 0x72, 0x75, 0x6c, 0x65, 0x73, 0x2e, 0x68, 0x61, 0x73, 0x28, 0x65, 0x73, - 0x63, 0x4e, 0x61, 0x6d, 0x65, 0x29, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, 0x74, 0x68, 0x69, 0x73, - 0x2e, 0x5f, 0x72, 0x75, 0x6c, 0x65, 0x73, 0x2e, 0x67, 0x65, 0x74, 0x28, - 0x65, 0x73, 0x63, 0x4e, 0x61, 0x6d, 0x65, 0x29, 0x20, 0x3d, 0x3d, 0x3d, - 0x20, 0x72, 0x75, 0x6c, 0x65, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, - 0x6b, 0x65, 0x79, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, - 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x6c, 0x65, 0x74, 0x20, - 0x69, 0x20, 0x3d, 0x20, 0x30, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x77, 0x68, 0x69, 0x6c, 0x65, 0x20, 0x28, 0x74, 0x68, 0x69, 0x73, - 0x2e, 0x5f, 0x72, 0x75, 0x6c, 0x65, 0x73, 0x2e, 0x68, 0x61, 0x73, 0x28, - 0x60, 0x24, 0x7b, 0x65, 0x73, 0x63, 0x4e, 0x61, 0x6d, 0x65, 0x7d, 0x24, - 0x7b, 0x69, 0x7d, 0x60, 0x29, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x20, 0x2b, 0x3d, 0x20, 0x31, 0x3b, - 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x6b, 0x65, 0x79, 0x20, 0x3d, 0x20, 0x60, 0x24, 0x7b, - 0x65, 0x73, 0x63, 0x4e, 0x61, 0x6d, 0x65, 0x7d, 0x24, 0x7b, 0x69, 0x7d, - 0x60, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x0a, 0x20, 0x20, - 0x20, 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x5f, 0x72, 0x75, 0x6c, 0x65, - 0x73, 0x2e, 0x73, 0x65, 0x74, 0x28, 0x6b, 0x65, 0x79, 0x2c, 0x20, 0x72, - 0x75, 0x6c, 0x65, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, - 0x74, 0x75, 0x72, 0x6e, 0x20, 0x6b, 0x65, 0x79, 0x3b, 0x0a, 0x20, 0x20, - 0x7d, 0x0a, 0x0a, 0x20, 0x20, 0x76, 0x69, 0x73, 0x69, 0x74, 0x28, 0x73, - 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2c, 0x20, 0x6e, 0x61, 0x6d, 0x65, 0x29, - 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, - 0x20, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x54, 0x79, 0x70, 0x65, 0x20, - 0x3d, 0x20, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2e, 0x74, 0x79, 0x70, - 0x65, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, - 0x20, 0x72, 0x75, 0x6c, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x20, 0x3d, 0x20, - 0x6e, 0x61, 0x6d, 0x65, 0x20, 0x7c, 0x7c, 0x20, 0x27, 0x72, 0x6f, 0x6f, - 0x74, 0x27, 0x3b, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, - 0x28, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2e, 0x6f, 0x6e, 0x65, 0x4f, - 0x66, 0x20, 0x7c, 0x7c, 0x20, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2e, - 0x61, 0x6e, 0x79, 0x4f, 0x66, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x72, 0x75, 0x6c, - 0x65, 0x20, 0x3d, 0x20, 0x28, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2e, - 0x6f, 0x6e, 0x65, 0x4f, 0x66, 0x20, 0x7c, 0x7c, 0x20, 0x73, 0x63, 0x68, - 0x65, 0x6d, 0x61, 0x2e, 0x61, 0x6e, 0x79, 0x4f, 0x66, 0x29, 0x2e, 0x6d, - 0x61, 0x70, 0x28, 0x28, 0x61, 0x6c, 0x74, 0x53, 0x63, 0x68, 0x65, 0x6d, - 0x61, 0x2c, 0x20, 0x69, 0x29, 0x20, 0x3d, 0x3e, 0x0a, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x76, 0x69, - 0x73, 0x69, 0x74, 0x28, 0x61, 0x6c, 0x74, 0x53, 0x63, 0x68, 0x65, 0x6d, - 0x61, 0x2c, 0x20, 0x60, 0x24, 0x7b, 0x6e, 0x61, 0x6d, 0x65, 0x7d, 0x24, - 0x7b, 0x6e, 0x61, 0x6d, 0x65, 0x20, 0x3f, 0x20, 0x22, 0x2d, 0x22, 0x20, - 0x3a, 0x20, 0x22, 0x22, 0x7d, 0x24, 0x7b, 0x69, 0x7d, 0x60, 0x29, 0x0a, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x29, 0x2e, 0x6a, 0x6f, 0x69, 0x6e, - 0x28, 0x27, 0x20, 0x7c, 0x20, 0x27, 0x29, 0x3b, 0x0a, 0x0a, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x74, - 0x68, 0x69, 0x73, 0x2e, 0x5f, 0x61, 0x64, 0x64, 0x52, 0x75, 0x6c, 0x65, - 0x28, 0x72, 0x75, 0x6c, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x2c, 0x20, 0x72, - 0x75, 0x6c, 0x65, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x20, - 0x65, 0x6c, 0x73, 0x65, 0x20, 0x69, 0x66, 0x20, 0x28, 0x27, 0x63, 0x6f, - 0x6e, 0x73, 0x74, 0x27, 0x20, 0x69, 0x6e, 0x20, 0x73, 0x63, 0x68, 0x65, - 0x6d, 0x61, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, - 0x5f, 0x61, 0x64, 0x64, 0x52, 0x75, 0x6c, 0x65, 0x28, 0x72, 0x75, 0x6c, - 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x2c, 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, - 0x5f, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x4c, 0x69, 0x74, 0x65, 0x72, - 0x61, 0x6c, 0x28, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2e, 0x63, 0x6f, - 0x6e, 0x73, 0x74, 0x29, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x7d, - 0x20, 0x65, 0x6c, 0x73, 0x65, 0x20, 0x69, 0x66, 0x20, 0x28, 0x27, 0x65, - 0x6e, 0x75, 0x6d, 0x27, 0x20, 0x69, 0x6e, 0x20, 0x73, 0x63, 0x68, 0x65, - 0x6d, 0x61, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x72, 0x75, 0x6c, 0x65, 0x20, 0x3d, - 0x20, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2e, 0x65, 0x6e, 0x75, 0x6d, - 0x2e, 0x6d, 0x61, 0x70, 0x28, 0x76, 0x20, 0x3d, 0x3e, 0x20, 0x74, 0x68, - 0x69, 0x73, 0x2e, 0x5f, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x4c, 0x69, - 0x74, 0x65, 0x72, 0x61, 0x6c, 0x28, 0x76, 0x29, 0x29, 0x2e, 0x6a, 0x6f, - 0x69, 0x6e, 0x28, 0x27, 0x20, 0x7c, 0x20, 0x27, 0x29, 0x3b, 0x0a, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, - 0x74, 0x68, 0x69, 0x73, 0x2e, 0x5f, 0x61, 0x64, 0x64, 0x52, 0x75, 0x6c, - 0x65, 0x28, 0x72, 0x75, 0x6c, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x2c, 0x20, - 0x72, 0x75, 0x6c, 0x65, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x7d, - 0x20, 0x65, 0x6c, 0x73, 0x65, 0x20, 0x69, 0x66, 0x20, 0x28, 0x73, 0x63, - 0x68, 0x65, 0x6d, 0x61, 0x54, 0x79, 0x70, 0x65, 0x20, 0x3d, 0x3d, 0x3d, - 0x20, 0x27, 0x6f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x27, 0x20, 0x26, 0x26, - 0x20, 0x27, 0x70, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, - 0x27, 0x20, 0x69, 0x6e, 0x20, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x29, - 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x2f, 0x2f, 0x20, - 0x54, 0x4f, 0x44, 0x4f, 0x3a, 0x20, 0x60, 0x72, 0x65, 0x71, 0x75, 0x69, - 0x72, 0x65, 0x64, 0x60, 0x20, 0x6b, 0x65, 0x79, 0x77, 0x6f, 0x72, 0x64, - 0x20, 0x28, 0x66, 0x72, 0x6f, 0x6d, 0x20, 0x70, 0x79, 0x74, 0x68, 0x6f, - 0x6e, 0x20, 0x69, 0x6d, 0x70, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x61, - 0x74, 0x69, 0x6f, 0x6e, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x70, 0x72, 0x6f, 0x70, 0x4f, 0x72, - 0x64, 0x65, 0x72, 0x20, 0x3d, 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x5f, - 0x70, 0x72, 0x6f, 0x70, 0x4f, 0x72, 0x64, 0x65, 0x72, 0x3b, 0x0a, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x70, - 0x72, 0x6f, 0x70, 0x50, 0x61, 0x69, 0x72, 0x73, 0x20, 0x3d, 0x20, 0x4f, - 0x62, 0x6a, 0x65, 0x63, 0x74, 0x2e, 0x65, 0x6e, 0x74, 0x72, 0x69, 0x65, - 0x73, 0x28, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2e, 0x70, 0x72, 0x6f, - 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x29, 0x2e, 0x73, 0x6f, 0x72, - 0x74, 0x28, 0x28, 0x61, 0x2c, 0x20, 0x62, 0x29, 0x20, 0x3d, 0x3e, 0x20, - 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x2f, 0x2f, - 0x20, 0x73, 0x6f, 0x72, 0x74, 0x20, 0x62, 0x79, 0x20, 0x70, 0x6f, 0x73, - 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x69, 0x6e, 0x20, 0x70, 0x72, 0x6f, - 0x70, 0x5f, 0x6f, 0x72, 0x64, 0x65, 0x72, 0x20, 0x28, 0x69, 0x66, 0x20, - 0x73, 0x70, 0x65, 0x63, 0x69, 0x66, 0x69, 0x65, 0x64, 0x29, 0x20, 0x74, - 0x68, 0x65, 0x6e, 0x20, 0x62, 0x79, 0x20, 0x6b, 0x65, 0x79, 0x0a, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, - 0x20, 0x6f, 0x72, 0x64, 0x65, 0x72, 0x41, 0x20, 0x3d, 0x20, 0x74, 0x79, - 0x70, 0x65, 0x6f, 0x66, 0x20, 0x70, 0x72, 0x6f, 0x70, 0x4f, 0x72, 0x64, - 0x65, 0x72, 0x5b, 0x61, 0x5b, 0x30, 0x5d, 0x5d, 0x20, 0x3d, 0x3d, 0x3d, - 0x20, 0x27, 0x6e, 0x75, 0x6d, 0x62, 0x65, 0x72, 0x27, 0x20, 0x3f, 0x20, - 0x70, 0x72, 0x6f, 0x70, 0x4f, 0x72, 0x64, 0x65, 0x72, 0x5b, 0x61, 0x5b, - 0x30, 0x5d, 0x5d, 0x20, 0x3a, 0x20, 0x49, 0x6e, 0x66, 0x69, 0x6e, 0x69, - 0x74, 0x79, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x6f, 0x72, 0x64, 0x65, 0x72, 0x42, - 0x20, 0x3d, 0x20, 0x74, 0x79, 0x70, 0x65, 0x6f, 0x66, 0x20, 0x70, 0x72, - 0x6f, 0x70, 0x4f, 0x72, 0x64, 0x65, 0x72, 0x5b, 0x62, 0x5b, 0x30, 0x5d, - 0x5d, 0x20, 0x3d, 0x3d, 0x3d, 0x20, 0x27, 0x6e, 0x75, 0x6d, 0x62, 0x65, - 0x72, 0x27, 0x20, 0x3f, 0x20, 0x70, 0x72, 0x6f, 0x70, 0x4f, 0x72, 0x64, - 0x65, 0x72, 0x5b, 0x62, 0x5b, 0x30, 0x5d, 0x5d, 0x20, 0x3a, 0x20, 0x49, - 0x6e, 0x66, 0x69, 0x6e, 0x69, 0x74, 0x79, 0x3b, 0x0a, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, - 0x6f, 0x72, 0x64, 0x65, 0x72, 0x41, 0x20, 0x2d, 0x20, 0x6f, 0x72, 0x64, - 0x65, 0x72, 0x42, 0x20, 0x7c, 0x7c, 0x20, 0x61, 0x5b, 0x30, 0x5d, 0x2e, - 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x65, 0x43, 0x6f, 0x6d, 0x70, 0x61, 0x72, - 0x65, 0x28, 0x62, 0x5b, 0x30, 0x5d, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x7d, 0x29, 0x3b, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x6c, 0x65, 0x74, 0x20, 0x72, 0x75, 0x6c, 0x65, 0x20, 0x3d, - 0x20, 0x27, 0x22, 0x7b, 0x22, 0x20, 0x73, 0x70, 0x61, 0x63, 0x65, 0x27, - 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x70, 0x72, 0x6f, 0x70, - 0x50, 0x61, 0x69, 0x72, 0x73, 0x2e, 0x66, 0x6f, 0x72, 0x45, 0x61, 0x63, - 0x68, 0x28, 0x28, 0x5b, 0x70, 0x72, 0x6f, 0x70, 0x4e, 0x61, 0x6d, 0x65, - 0x2c, 0x20, 0x70, 0x72, 0x6f, 0x70, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, - 0x5d, 0x2c, 0x20, 0x69, 0x29, 0x20, 0x3d, 0x3e, 0x20, 0x7b, 0x0a, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, - 0x20, 0x70, 0x72, 0x6f, 0x70, 0x52, 0x75, 0x6c, 0x65, 0x4e, 0x61, 0x6d, - 0x65, 0x20, 0x3d, 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x76, 0x69, 0x73, - 0x69, 0x74, 0x28, 0x70, 0x72, 0x6f, 0x70, 0x53, 0x63, 0x68, 0x65, 0x6d, - 0x61, 0x2c, 0x20, 0x60, 0x24, 0x7b, 0x6e, 0x61, 0x6d, 0x65, 0x7d, 0x24, - 0x7b, 0x6e, 0x61, 0x6d, 0x65, 0x20, 0x3f, 0x20, 0x22, 0x2d, 0x22, 0x20, - 0x3a, 0x20, 0x22, 0x22, 0x7d, 0x24, 0x7b, 0x70, 0x72, 0x6f, 0x70, 0x4e, - 0x61, 0x6d, 0x65, 0x7d, 0x60, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, 0x69, 0x20, 0x3e, 0x20, - 0x30, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x72, 0x75, 0x6c, 0x65, 0x20, 0x2b, 0x3d, 0x20, 0x27, - 0x20, 0x22, 0x2c, 0x22, 0x20, 0x73, 0x70, 0x61, 0x63, 0x65, 0x27, 0x3b, - 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, 0x75, 0x6c, 0x65, 0x20, - 0x2b, 0x3d, 0x20, 0x60, 0x20, 0x24, 0x7b, 0x74, 0x68, 0x69, 0x73, 0x2e, - 0x5f, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x4c, 0x69, 0x74, 0x65, 0x72, - 0x61, 0x6c, 0x28, 0x70, 0x72, 0x6f, 0x70, 0x4e, 0x61, 0x6d, 0x65, 0x29, - 0x7d, 0x20, 0x73, 0x70, 0x61, 0x63, 0x65, 0x20, 0x22, 0x3a, 0x22, 0x20, - 0x73, 0x70, 0x61, 0x63, 0x65, 0x20, 0x24, 0x7b, 0x70, 0x72, 0x6f, 0x70, - 0x52, 0x75, 0x6c, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x7d, 0x60, 0x3b, 0x0a, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x29, 0x3b, 0x0a, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x72, 0x75, 0x6c, 0x65, 0x20, 0x2b, 0x3d, 0x20, - 0x27, 0x20, 0x22, 0x7d, 0x22, 0x20, 0x73, 0x70, 0x61, 0x63, 0x65, 0x27, - 0x3b, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, 0x74, - 0x75, 0x72, 0x6e, 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x5f, 0x61, 0x64, - 0x64, 0x52, 0x75, 0x6c, 0x65, 0x28, 0x72, 0x75, 0x6c, 0x65, 0x4e, 0x61, - 0x6d, 0x65, 0x2c, 0x20, 0x72, 0x75, 0x6c, 0x65, 0x29, 0x3b, 0x0a, 0x20, - 0x20, 0x20, 0x20, 0x7d, 0x20, 0x65, 0x6c, 0x73, 0x65, 0x20, 0x69, 0x66, - 0x20, 0x28, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x54, 0x79, 0x70, 0x65, - 0x20, 0x3d, 0x3d, 0x3d, 0x20, 0x27, 0x61, 0x72, 0x72, 0x61, 0x79, 0x27, - 0x20, 0x26, 0x26, 0x20, 0x27, 0x69, 0x74, 0x65, 0x6d, 0x73, 0x27, 0x20, - 0x69, 0x6e, 0x20, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x29, 0x20, 0x7b, - 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x2f, 0x2f, 0x20, 0x54, 0x4f, - 0x44, 0x4f, 0x20, 0x60, 0x70, 0x72, 0x65, 0x66, 0x69, 0x78, 0x49, 0x74, - 0x65, 0x6d, 0x73, 0x60, 0x20, 0x6b, 0x65, 0x79, 0x77, 0x6f, 0x72, 0x64, - 0x20, 0x28, 0x66, 0x72, 0x6f, 0x6d, 0x20, 0x70, 0x79, 0x74, 0x68, 0x6f, - 0x6e, 0x20, 0x69, 0x6d, 0x70, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x61, - 0x74, 0x69, 0x6f, 0x6e, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x69, 0x74, 0x65, 0x6d, 0x52, 0x75, - 0x6c, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x20, 0x3d, 0x20, 0x74, 0x68, 0x69, - 0x73, 0x2e, 0x76, 0x69, 0x73, 0x69, 0x74, 0x28, 0x73, 0x63, 0x68, 0x65, - 0x6d, 0x61, 0x2e, 0x69, 0x74, 0x65, 0x6d, 0x73, 0x2c, 0x20, 0x60, 0x24, - 0x7b, 0x6e, 0x61, 0x6d, 0x65, 0x7d, 0x24, 0x7b, 0x6e, 0x61, 0x6d, 0x65, - 0x20, 0x3f, 0x20, 0x22, 0x2d, 0x22, 0x20, 0x3a, 0x20, 0x22, 0x22, 0x7d, - 0x69, 0x74, 0x65, 0x6d, 0x60, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x72, 0x75, 0x6c, 0x65, - 0x20, 0x3d, 0x20, 0x60, 0x22, 0x5b, 0x22, 0x20, 0x73, 0x70, 0x61, 0x63, - 0x65, 0x20, 0x28, 0x24, 0x7b, 0x69, 0x74, 0x65, 0x6d, 0x52, 0x75, 0x6c, - 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x7d, 0x20, 0x28, 0x22, 0x2c, 0x22, 0x20, - 0x73, 0x70, 0x61, 0x63, 0x65, 0x20, 0x24, 0x7b, 0x69, 0x74, 0x65, 0x6d, - 0x52, 0x75, 0x6c, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x7d, 0x29, 0x2a, 0x29, - 0x3f, 0x20, 0x22, 0x5d, 0x22, 0x20, 0x73, 0x70, 0x61, 0x63, 0x65, 0x60, - 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, - 0x72, 0x6e, 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x5f, 0x61, 0x64, 0x64, - 0x52, 0x75, 0x6c, 0x65, 0x28, 0x72, 0x75, 0x6c, 0x65, 0x4e, 0x61, 0x6d, - 0x65, 0x2c, 0x20, 0x72, 0x75, 0x6c, 0x65, 0x29, 0x3b, 0x0a, 0x20, 0x20, - 0x20, 0x20, 0x7d, 0x20, 0x65, 0x6c, 0x73, 0x65, 0x20, 0x7b, 0x0a, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, 0x21, 0x50, 0x52, - 0x49, 0x4d, 0x49, 0x54, 0x49, 0x56, 0x45, 0x5f, 0x52, 0x55, 0x4c, 0x45, - 0x53, 0x5b, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x54, 0x79, 0x70, 0x65, - 0x5d, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x74, 0x68, 0x72, 0x6f, 0x77, 0x20, 0x6e, 0x65, 0x77, 0x20, 0x45, - 0x72, 0x72, 0x6f, 0x72, 0x28, 0x60, 0x55, 0x6e, 0x72, 0x65, 0x63, 0x6f, - 0x67, 0x6e, 0x69, 0x7a, 0x65, 0x64, 0x20, 0x73, 0x63, 0x68, 0x65, 0x6d, - 0x61, 0x3a, 0x20, 0x24, 0x7b, 0x4a, 0x53, 0x4f, 0x4e, 0x2e, 0x73, 0x74, - 0x72, 0x69, 0x6e, 0x67, 0x69, 0x66, 0x79, 0x28, 0x73, 0x63, 0x68, 0x65, - 0x6d, 0x61, 0x29, 0x7d, 0x60, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, - 0x74, 0x75, 0x72, 0x6e, 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x5f, 0x61, - 0x64, 0x64, 0x52, 0x75, 0x6c, 0x65, 0x28, 0x0a, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x72, 0x75, 0x6c, 0x65, 0x4e, 0x61, 0x6d, 0x65, - 0x20, 0x3d, 0x3d, 0x3d, 0x20, 0x27, 0x72, 0x6f, 0x6f, 0x74, 0x27, 0x20, - 0x3f, 0x20, 0x27, 0x72, 0x6f, 0x6f, 0x74, 0x27, 0x20, 0x3a, 0x20, 0x73, - 0x63, 0x68, 0x65, 0x6d, 0x61, 0x54, 0x79, 0x70, 0x65, 0x2c, 0x0a, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x50, 0x52, 0x49, 0x4d, 0x49, - 0x54, 0x49, 0x56, 0x45, 0x5f, 0x52, 0x55, 0x4c, 0x45, 0x53, 0x5b, 0x73, - 0x63, 0x68, 0x65, 0x6d, 0x61, 0x54, 0x79, 0x70, 0x65, 0x5d, 0x0a, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, - 0x7d, 0x0a, 0x20, 0x20, 0x7d, 0x0a, 0x0a, 0x20, 0x20, 0x66, 0x6f, 0x72, - 0x6d, 0x61, 0x74, 0x47, 0x72, 0x61, 0x6d, 0x6d, 0x61, 0x72, 0x28, 0x29, - 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x6c, 0x65, 0x74, 0x20, 0x67, - 0x72, 0x61, 0x6d, 0x6d, 0x61, 0x72, 0x20, 0x3d, 0x20, 0x27, 0x27, 0x3b, - 0x0a, 0x20, 0x20, 0x20, 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x5f, 0x72, - 0x75, 0x6c, 0x65, 0x73, 0x2e, 0x66, 0x6f, 0x72, 0x45, 0x61, 0x63, 0x68, - 0x28, 0x28, 0x72, 0x75, 0x6c, 0x65, 0x2c, 0x20, 0x6e, 0x61, 0x6d, 0x65, - 0x29, 0x20, 0x3d, 0x3e, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x67, 0x72, 0x61, 0x6d, 0x6d, 0x61, 0x72, 0x20, 0x2b, 0x3d, 0x20, - 0x60, 0x24, 0x7b, 0x6e, 0x61, 0x6d, 0x65, 0x7d, 0x20, 0x3a, 0x3a, 0x3d, - 0x20, 0x24, 0x7b, 0x72, 0x75, 0x6c, 0x65, 0x7d, 0x5c, 0x6e, 0x60, 0x3b, - 0x0a, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, - 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x67, 0x72, 0x61, 0x6d, - 0x6d, 0x61, 0x72, 0x3b, 0x0a, 0x20, 0x20, 0x7d, 0x0a, 0x7d, 0x0a +const char json_schema_to_grammar_mjs[] = R"LITERAL( +const SPACE_RULE = '" "?'; + +const PRIMITIVE_RULES = { + boolean: '("true" | "false") space', + number: '("-"? ([0-9] | [1-9] [0-9]*)) ("." [0-9]+)? ([eE] [-+]? [0-9]+)? space', + integer: '("-"? ([0-9] | [1-9] [0-9]*)) space', + string: ` "\\"" ( + [^"\\\\] | + "\\\\" (["\\\\/bfnrt] | "u" [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F]) + )* "\\"" space`, + null: '"null" space', }; -unsigned int json_schema_to_grammar_mjs_len = 3695; + +const INVALID_RULE_CHARS_RE = /[^\dA-Za-z-]+/g; +const GRAMMAR_LITERAL_ESCAPE_RE = /[\n\r"]/g; +const GRAMMAR_LITERAL_ESCAPES = {'\r': '\\r', '\n': '\\n', '"': '\\"'}; + +export class SchemaConverter { + constructor(propOrder) { + this._propOrder = propOrder || {}; + this._rules = new Map(); + this._rules.set('space', SPACE_RULE); + } + + _formatLiteral(literal) { + const escaped = JSON.stringify(literal).replace( + GRAMMAR_LITERAL_ESCAPE_RE, + m => GRAMMAR_LITERAL_ESCAPES[m] + ); + return `"${escaped}"`; + } + + _addRule(name, rule) { + let escName = name.replace(INVALID_RULE_CHARS_RE, '-'); + let key = escName; + + if (this._rules.has(escName)) { + if (this._rules.get(escName) === rule) { + return key; + } + + let i = 0; + while (this._rules.has(`${escName}${i}`)) { + i += 1; + } + key = `${escName}${i}`; + } + + this._rules.set(key, rule); + return key; + } + + visit(schema, name) { + const schemaType = schema.type; + const ruleName = name || 'root'; + + if (schema.oneOf || schema.anyOf) { + const rule = (schema.oneOf || schema.anyOf).map((altSchema, i) => + this.visit(altSchema, `${name}${name ? "-" : ""}${i}`) + ).join(' | '); + + return this._addRule(ruleName, rule); + } else if ('const' in schema) { + return this._addRule(ruleName, this._formatLiteral(schema.const)); + } else if ('enum' in schema) { + const rule = schema.enum.map(v => this._formatLiteral(v)).join(' | '); + return this._addRule(ruleName, rule); + } else if (schemaType === 'object' && 'properties' in schema) { + // TODO: `required` keyword (from python implementation) + const propOrder = this._propOrder; + const propPairs = Object.entries(schema.properties).sort((a, b) => { + // sort by position in prop_order (if specified) then by key + const orderA = typeof propOrder[a[0]] === 'number' ? propOrder[a[0]] : Infinity; + const orderB = typeof propOrder[b[0]] === 'number' ? propOrder[b[0]] : Infinity; + return orderA - orderB || a[0].localeCompare(b[0]); + }); + + let rule = '"{" space'; + propPairs.forEach(([propName, propSchema], i) => { + const propRuleName = this.visit(propSchema, `${name}${name ? "-" : ""}${propName}`); + if (i > 0) { + rule += ' "," space'; + } + rule += ` ${this._formatLiteral(propName)} space ":" space ${propRuleName}`; + }); + rule += ' "}" space'; + + return this._addRule(ruleName, rule); + } else if (schemaType === 'array' && 'items' in schema) { + // TODO `prefixItems` keyword (from python implementation) + const itemRuleName = this.visit(schema.items, `${name}${name ? "-" : ""}item`); + const rule = `"[" space (${itemRuleName} ("," space ${itemRuleName})*)? "]" space`; + return this._addRule(ruleName, rule); + } else { + if (!PRIMITIVE_RULES[schemaType]) { + throw new Error(`Unrecognized schema: ${JSON.stringify(schema)}`); + } + return this._addRule( + ruleName === 'root' ? 'root' : schemaType, + PRIMITIVE_RULES[schemaType] + ); + } + } + + formatGrammar() { + let grammar = ''; + this._rules.forEach((rule, name) => { + grammar += `${name} ::= ${rule}\n`; + }); + return grammar; + } +} +)LITERAL"; +unsigned int json_schema_to_grammar_mjs_len = sizeof(json_schema_to_grammar_mjs); From e6f291d15844398f8326940fe5ad7f2e02b5aa56 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Tue, 30 Jan 2024 20:17:30 +0200 Subject: [PATCH 221/812] server : fix context shift (#5195) * server : fix context shift + simplify self-extend * server : take system_tokens into account * server : more n_past fixes * server : rever n_past_se changes --- examples/server/chat.sh | 1 + examples/server/server.cpp | 109 ++++++++++++++++++++----------------- 2 files changed, 60 insertions(+), 50 deletions(-) diff --git a/examples/server/chat.sh b/examples/server/chat.sh index 014360121..da0a6ca68 100755 --- a/examples/server/chat.sh +++ b/examples/server/chat.sh @@ -48,6 +48,7 @@ chat_completion() { top_p: 0.9, n_keep: $n_keep, n_predict: 256, + cache_prompt: true, stop: ["\n### Human:"], stream: true }')" diff --git a/examples/server/server.cpp b/examples/server/server.cpp index 11dd82c33..21bdce8ed 100644 --- a/examples/server/server.cpp +++ b/examples/server/server.cpp @@ -185,7 +185,7 @@ struct llama_client_slot llama_sampling_context *ctx_sampling = nullptr; int32_t ga_i = 0; // group-attention state - int32_t ga_n = 1;// group-attention factor + int32_t ga_n = 1; // group-attention factor int32_t ga_w = 512; // group-attention width int32_t n_past_se = 0; // self-extend @@ -219,7 +219,8 @@ struct llama_client_slot sent_token_probs_index = 0; infill = false; ga_i = 0; - n_past_se = 0; + n_past_se = 0; + generated_token_probs.clear(); for (slot_image & img : images) @@ -1227,7 +1228,7 @@ struct llama_server_context std::vector append_tokens = tokenize(json_prompt, false); // has next image for (int i = 0; i < (int) append_tokens.size(); ++i) { - llama_batch_add(batch, append_tokens[i], slot.n_past, { slot.id }, true); + llama_batch_add(batch, append_tokens[i], system_tokens.size() + slot.n_past, { slot.id }, true); slot.n_past += 1; } } @@ -1295,6 +1296,8 @@ struct llama_server_context for (llama_client_slot &slot : slots) { slot.cache_tokens.clear(); + slot.n_past = 0; + slot.n_past_se = 0; } } @@ -1364,26 +1367,26 @@ struct llama_server_context kv_cache_clear(); } return true; - } else { - task_server task; - task.type = TASK_TYPE_NEXT_RESPONSE; - task.target_id = -1; - queue_tasks.post(task); } + task_server task; + task.type = TASK_TYPE_NEXT_RESPONSE; + task.target_id = -1; + queue_tasks.post(task); + for (llama_client_slot &slot : slots) { if (slot.ga_n == 1) { - if (slot.is_processing() && slot.cache_tokens.size() >= (size_t) slot.n_ctx) + if (slot.is_processing() && system_tokens.size() + slot.cache_tokens.size() >= (size_t) slot.n_ctx) { // Shift context - const int n_left = slot.n_past - slot.params.n_keep - 1; + const int n_left = system_tokens.size() + slot.n_past - slot.params.n_keep - 1; const int n_discard = n_left / 2; LOG_TEE("slot %d: context shift - n_keep = %d, n_left = %d, n_discard = %d\n", slot.id, slot.params.n_keep, n_left, n_discard); llama_kv_cache_seq_rm (ctx, slot.id, slot.params.n_keep + 1 , slot.params.n_keep + n_discard + 1); - llama_kv_cache_seq_shift(ctx, slot.id, slot.params.n_keep + 1 + n_discard, slot.n_past, -n_discard); + llama_kv_cache_seq_shift(ctx, slot.id, slot.params.n_keep + 1 + n_discard, system_tokens.size() + slot.n_past, -n_discard); for (size_t i = slot.params.n_keep + 1 + n_discard; i < slot.cache_tokens.size(); i++) { @@ -1429,8 +1432,10 @@ struct llama_server_context slot.i_batch = batch.n_tokens; const int32_t slot_npast = slot.n_past_se > 0 ? slot.n_past_se : slot.n_past; - llama_batch_add(batch, slot.sampled, system_tokens.size() + slot_npast, { slot.id }, true); + // TODO: we always have to take into account the "system_tokens" + // this is not great and needs to be improved somehow + llama_batch_add(batch, slot.sampled, system_tokens.size() + slot_npast, { slot.id }, true); slot.n_past += 1; } @@ -1481,8 +1486,8 @@ struct llama_server_context prefix_tokens.insert(prefix_tokens.begin(), llama_token_prefix(model)); prefix_tokens.insert(prefix_tokens.begin(), llama_token_bos(model)); // always add BOS - prefix_tokens.insert(prefix_tokens.end(), llama_token_suffix(model)); - prefix_tokens.insert(prefix_tokens.end(), suffix_tokens.begin(), suffix_tokens.end()); + prefix_tokens.insert(prefix_tokens.end(), llama_token_suffix(model)); + prefix_tokens.insert(prefix_tokens.end(), suffix_tokens.begin(), suffix_tokens.end()); prefix_tokens.push_back(llama_token_middle(model)); prompt_tokens = prefix_tokens; } @@ -1582,8 +1587,8 @@ struct llama_server_context } LOG_VERBOSE("prompt ingested", { - {"n_past", slot.n_past}, - {"cached", tokens_to_str(ctx, slot.cache_tokens.cbegin(), slot.cache_tokens.cbegin() + slot.n_past)}, + {"n_past", slot.n_past}, + {"cached", tokens_to_str(ctx, slot.cache_tokens.cbegin(), slot.cache_tokens.cbegin() + slot.n_past)}, {"to_eval", tokens_to_str(ctx, slot.cache_tokens.cbegin() + slot.n_past, slot.cache_tokens.cend())}, }); @@ -1591,10 +1596,13 @@ struct llama_server_context // process the prefix of first image std::vector prefix_tokens = has_images ? tokenize(slot.images[0].prefix_prompt, add_bos_token) : prompt_tokens; + int32_t slot_npast = slot.n_past_se > 0 ? slot.n_past_se : slot.n_past; - int ga_i = slot.ga_i; + + int32_t ga_i = slot.ga_i; int32_t ga_n = slot.ga_n; int32_t ga_w = slot.ga_w; + for (; slot.n_past < (int) prefix_tokens.size(); ++slot.n_past) { if (slot.ga_n != 1) @@ -1606,7 +1614,7 @@ struct llama_server_context } } llama_batch_add(batch, prefix_tokens[slot.n_past], system_tokens.size() + slot_npast, {slot.id }, false); - slot_npast += 1; + slot_npast++; } if (has_images && !ingest_images(slot, n_batch)) @@ -1666,6 +1674,7 @@ struct llama_server_context slot.n_past_se += n_tokens; } } + llama_batch batch_view = { n_tokens, @@ -1782,51 +1791,51 @@ static void server_print_usage(const char *argv0, const gpt_params ¶ms, printf(" not recommended: doubles context memory required and no measurable increase in quality\n"); if (llama_mlock_supported()) { - printf(" --mlock force system to keep model in RAM rather than swapping or compressing\n"); + printf(" --mlock force system to keep model in RAM rather than swapping or compressing\n"); } if (llama_mmap_supported()) { - printf(" --no-mmap do not memory-map model (slower load but may reduce pageouts if not using mlock)\n"); + printf(" --no-mmap do not memory-map model (slower load but may reduce pageouts if not using mlock)\n"); } - printf(" --numa attempt optimizations that help on some NUMA systems\n"); + printf(" --numa attempt optimizations that help on some NUMA systems\n"); #ifdef LLAMA_SUPPORTS_GPU_OFFLOAD printf(" -ngl N, --n-gpu-layers N\n"); - printf(" number of layers to store in VRAM\n"); + printf(" number of layers to store in VRAM\n"); printf(" -sm SPLIT_MODE, --split-mode SPLIT_MODE\n"); - printf(" how to split the model across multiple GPUs, one of:\n"); - printf(" - none: use one GPU only\n"); - printf(" - layer (default): split layers and KV across GPUs\n"); - printf(" - row: split rows across GPUs\n"); + printf(" how to split the model across multiple GPUs, one of:\n"); + printf(" - none: use one GPU only\n"); + printf(" - layer (default): split layers and KV across GPUs\n"); + printf(" - row: split rows across GPUs\n"); printf(" -ts SPLIT --tensor-split SPLIT\n"); - printf(" fraction of the model to offload to each GPU, comma-separated list of proportions, e.g. 3,1\n"); - printf(" -mg i, --main-gpu i the GPU to use for the model (with split-mode = none),\n"); - printf(" or for intermediate results and KV (with split-mode = row)\n"); + printf(" fraction of the model to offload to each GPU, comma-separated list of proportions, e.g. 3,1\n"); + printf(" -mg i, --main-gpu i the GPU to use for the model (with split-mode = none),\n"); + printf(" or for intermediate results and KV (with split-mode = row)\n"); #endif printf(" -m FNAME, --model FNAME\n"); - printf(" model path (default: %s)\n", params.model.c_str()); + printf(" model path (default: %s)\n", params.model.c_str()); printf(" -a ALIAS, --alias ALIAS\n"); - printf(" set an alias for the model, will be added as `model` field in completion response\n"); - printf(" --lora FNAME apply LoRA adapter (implies --no-mmap)\n"); - printf(" --lora-base FNAME optional model to use as a base for the layers modified by the LoRA adapter\n"); - printf(" --host ip address to listen (default (default: %s)\n", sparams.hostname.c_str()); - printf(" --port PORT port to listen (default (default: %d)\n", sparams.port); - printf(" --path PUBLIC_PATH path from which to serve static files (default %s)\n", sparams.public_path.c_str()); - printf(" --api-key API_KEY optional api key to enhance server security. If set, requests must include this key for access.\n"); - printf(" --api-key-file FNAME path to file containing api keys delimited by new lines. If set, requests must include one of the keys for access.\n"); - printf(" -to N, --timeout N server read/write timeout in seconds (default: %d)\n", sparams.read_timeout); - printf(" --embedding enable embedding vector output (default: %s)\n", params.embedding ? "enabled" : "disabled"); - printf(" -np N, --parallel N number of slots for process requests (default: %d)\n", params.n_parallel); - printf(" -cb, --cont-batching enable continuous batching (a.k.a dynamic batching) (default: disabled)\n"); - printf(" -spf FNAME, --system-prompt-file FNAME\n"); - printf(" Set a file to load a system prompt (initial prompt of all slots), this is useful for chat applications.\n"); - printf(" --mmproj MMPROJ_FILE path to a multimodal projector file for LLaVA.\n"); - printf(" --log-disable disables logging to a file.\n"); + printf(" set an alias for the model, will be added as `model` field in completion response\n"); + printf(" --lora FNAME apply LoRA adapter (implies --no-mmap)\n"); + printf(" --lora-base FNAME optional model to use as a base for the layers modified by the LoRA adapter\n"); + printf(" --host ip address to listen (default (default: %s)\n", sparams.hostname.c_str()); + printf(" --port PORT port to listen (default (default: %d)\n", sparams.port); + printf(" --path PUBLIC_PATH path from which to serve static files (default %s)\n", sparams.public_path.c_str()); + printf(" --api-key API_KEY optional api key to enhance server security. If set, requests must include this key for access.\n"); + printf(" --api-key-file FNAME path to file containing api keys delimited by new lines. If set, requests must include one of the keys for access.\n"); + printf(" -to N, --timeout N server read/write timeout in seconds (default: %d)\n", sparams.read_timeout); + printf(" --embedding enable embedding vector output (default: %s)\n", params.embedding ? "enabled" : "disabled"); + printf(" -np N, --parallel N number of slots for process requests (default: %d)\n", params.n_parallel); + printf(" -cb, --cont-batching enable continuous batching (a.k.a dynamic batching) (default: disabled)\n"); + printf(" -spf FNAME, --system-prompt-file FNAME\n"); + printf(" set a file to load a system prompt (initial prompt of all slots), this is useful for chat applications.\n"); + printf(" --mmproj MMPROJ_FILE path to a multimodal projector file for LLaVA.\n"); + printf(" --log-disable disables logging to a file.\n"); printf("\n"); printf(" --override-kv KEY=TYPE:VALUE\n"); - printf(" advanced option to override model metadata by key. may be specified multiple times.\n"); - printf(" types: int, float, bool. example: --override-kv tokenizer.ggml.add_bos_token=bool:false\n"); - printf(" -gan N, --grp-attn-n N Set the group attention factor to extend context size through self-extend(default: 1=disabled), used together with group attention width `--grp-attn-w`"); - printf(" -gaw N, --grp-attn-w N Set the group attention width to extend context size through self-extend(default: 512), used together with group attention factor `--grp-attn-n`"); + printf(" advanced option to override model metadata by key. may be specified multiple times.\n"); + printf(" types: int, float, bool. example: --override-kv tokenizer.ggml.add_bos_token=bool:false\n"); + printf(" -gan N, --grp-attn-n N set the group attention factor to extend context size through self-extend(default: 1=disabled), used together with group attention width `--grp-attn-w`"); + printf(" -gaw N, --grp-attn-w N set the group attention width to extend context size through self-extend(default: 512), used together with group attention factor `--grp-attn-n`"); printf("\n"); } From e0085fdf7c758f0bc2746fc106fb29dd9df959de Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Tue, 30 Jan 2024 21:19:26 +0200 Subject: [PATCH 222/812] Revert "server : change deps.sh xxd files to string literals (#5221)" This reverts commit 4003be0e5feef320f3707786f22722b73cff9356. --- examples/server/completion.js.hpp | 651 ++- examples/server/deps.sh | 11 +- examples/server/index.html.hpp | 3829 ++++++++++++----- examples/server/index.js.hpp | 1907 +++++++- .../server/json-schema-to-grammar.mjs.hpp | 424 +- 5 files changed, 5454 insertions(+), 1368 deletions(-) diff --git a/examples/server/completion.js.hpp b/examples/server/completion.js.hpp index 5609ee3bf..fe5f81228 100644 --- a/examples/server/completion.js.hpp +++ b/examples/server/completion.js.hpp @@ -1,204 +1,449 @@ -const char completion_js[] = R"LITERAL( -const paramDefaults = { - stream: true, - n_predict: 500, - temperature: 0.2, - stop: ["
"] +unsigned char completion_js[] = { + 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x44, + 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x73, 0x20, 0x3d, 0x20, 0x7b, 0x0a, + 0x20, 0x20, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x3a, 0x20, 0x74, 0x72, + 0x75, 0x65, 0x2c, 0x0a, 0x20, 0x20, 0x6e, 0x5f, 0x70, 0x72, 0x65, 0x64, + 0x69, 0x63, 0x74, 0x3a, 0x20, 0x35, 0x30, 0x30, 0x2c, 0x0a, 0x20, 0x20, + 0x74, 0x65, 0x6d, 0x70, 0x65, 0x72, 0x61, 0x74, 0x75, 0x72, 0x65, 0x3a, + 0x20, 0x30, 0x2e, 0x32, 0x2c, 0x0a, 0x20, 0x20, 0x73, 0x74, 0x6f, 0x70, + 0x3a, 0x20, 0x5b, 0x22, 0x3c, 0x2f, 0x73, 0x3e, 0x22, 0x5d, 0x0a, 0x7d, + 0x3b, 0x0a, 0x0a, 0x6c, 0x65, 0x74, 0x20, 0x67, 0x65, 0x6e, 0x65, 0x72, + 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x73, 0x65, 0x74, 0x74, 0x69, 0x6e, + 0x67, 0x73, 0x20, 0x3d, 0x20, 0x6e, 0x75, 0x6c, 0x6c, 0x3b, 0x0a, 0x0a, + 0x0a, 0x2f, 0x2f, 0x20, 0x43, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65, + 0x73, 0x20, 0x74, 0x68, 0x65, 0x20, 0x70, 0x72, 0x6f, 0x6d, 0x70, 0x74, + 0x20, 0x61, 0x73, 0x20, 0x61, 0x20, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, + 0x74, 0x6f, 0x72, 0x2e, 0x20, 0x52, 0x65, 0x63, 0x6f, 0x6d, 0x6d, 0x65, + 0x6e, 0x64, 0x65, 0x64, 0x20, 0x66, 0x6f, 0x72, 0x20, 0x6d, 0x6f, 0x73, + 0x74, 0x20, 0x75, 0x73, 0x65, 0x20, 0x63, 0x61, 0x73, 0x65, 0x73, 0x2e, + 0x0a, 0x2f, 0x2f, 0x0a, 0x2f, 0x2f, 0x20, 0x45, 0x78, 0x61, 0x6d, 0x70, + 0x6c, 0x65, 0x3a, 0x0a, 0x2f, 0x2f, 0x0a, 0x2f, 0x2f, 0x20, 0x20, 0x20, + 0x20, 0x69, 0x6d, 0x70, 0x6f, 0x72, 0x74, 0x20, 0x7b, 0x20, 0x6c, 0x6c, + 0x61, 0x6d, 0x61, 0x20, 0x7d, 0x20, 0x66, 0x72, 0x6f, 0x6d, 0x20, 0x27, + 0x2f, 0x63, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x69, 0x6f, 0x6e, 0x2e, + 0x6a, 0x73, 0x27, 0x0a, 0x2f, 0x2f, 0x0a, 0x2f, 0x2f, 0x20, 0x20, 0x20, + 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x72, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x20, 0x3d, 0x20, 0x6c, 0x6c, 0x61, 0x6d, 0x61, 0x28, 0x22, + 0x54, 0x65, 0x6c, 0x6c, 0x20, 0x6d, 0x65, 0x20, 0x61, 0x20, 0x6a, 0x6f, + 0x6b, 0x65, 0x22, 0x2c, 0x20, 0x7b, 0x6e, 0x5f, 0x70, 0x72, 0x65, 0x64, + 0x69, 0x63, 0x74, 0x3a, 0x20, 0x38, 0x30, 0x30, 0x7d, 0x29, 0x0a, 0x2f, + 0x2f, 0x20, 0x20, 0x20, 0x20, 0x66, 0x6f, 0x72, 0x20, 0x61, 0x77, 0x61, + 0x69, 0x74, 0x20, 0x28, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x63, 0x68, + 0x75, 0x6e, 0x6b, 0x20, 0x6f, 0x66, 0x20, 0x72, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x29, 0x20, 0x7b, 0x0a, 0x2f, 0x2f, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x64, 0x6f, 0x63, 0x75, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x77, + 0x72, 0x69, 0x74, 0x65, 0x28, 0x63, 0x68, 0x75, 0x6e, 0x6b, 0x2e, 0x64, + 0x61, 0x74, 0x61, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x29, + 0x0a, 0x2f, 0x2f, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x2f, 0x2f, 0x0a, + 0x65, 0x78, 0x70, 0x6f, 0x72, 0x74, 0x20, 0x61, 0x73, 0x79, 0x6e, 0x63, + 0x20, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x2a, 0x20, 0x6c, + 0x6c, 0x61, 0x6d, 0x61, 0x28, 0x70, 0x72, 0x6f, 0x6d, 0x70, 0x74, 0x2c, + 0x20, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x20, 0x3d, 0x20, 0x7b, 0x7d, + 0x2c, 0x20, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x20, 0x3d, 0x20, 0x7b, + 0x7d, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x6c, 0x65, 0x74, 0x20, 0x63, + 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x20, 0x3d, 0x20, + 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, + 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x3b, 0x0a, 0x0a, 0x20, 0x20, 0x69, 0x66, + 0x20, 0x28, 0x21, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, + 0x72, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, + 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x20, 0x3d, 0x20, 0x6e, 0x65, + 0x77, 0x20, 0x41, 0x62, 0x6f, 0x72, 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x72, + 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x28, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x7d, + 0x0a, 0x0a, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x63, 0x6f, + 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x69, 0x6f, 0x6e, 0x50, 0x61, 0x72, 0x61, + 0x6d, 0x73, 0x20, 0x3d, 0x20, 0x7b, 0x20, 0x2e, 0x2e, 0x2e, 0x70, 0x61, + 0x72, 0x61, 0x6d, 0x44, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x73, 0x2c, + 0x20, 0x2e, 0x2e, 0x2e, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x2c, 0x20, + 0x70, 0x72, 0x6f, 0x6d, 0x70, 0x74, 0x20, 0x7d, 0x3b, 0x0a, 0x0a, 0x20, + 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x72, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x20, 0x3d, 0x20, 0x61, 0x77, 0x61, 0x69, 0x74, 0x20, + 0x66, 0x65, 0x74, 0x63, 0x68, 0x28, 0x22, 0x2f, 0x63, 0x6f, 0x6d, 0x70, + 0x6c, 0x65, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x2c, 0x20, 0x7b, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x6d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x3a, 0x20, 0x27, + 0x50, 0x4f, 0x53, 0x54, 0x27, 0x2c, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x62, + 0x6f, 0x64, 0x79, 0x3a, 0x20, 0x4a, 0x53, 0x4f, 0x4e, 0x2e, 0x73, 0x74, + 0x72, 0x69, 0x6e, 0x67, 0x69, 0x66, 0x79, 0x28, 0x63, 0x6f, 0x6d, 0x70, + 0x6c, 0x65, 0x74, 0x69, 0x6f, 0x6e, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, + 0x29, 0x2c, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x68, 0x65, 0x61, 0x64, 0x65, + 0x72, 0x73, 0x3a, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x27, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x27, + 0x3a, 0x20, 0x27, 0x6b, 0x65, 0x65, 0x70, 0x2d, 0x61, 0x6c, 0x69, 0x76, + 0x65, 0x27, 0x2c, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x27, 0x43, + 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x2d, 0x54, 0x79, 0x70, 0x65, 0x27, + 0x3a, 0x20, 0x27, 0x61, 0x70, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, + 0x6f, 0x6e, 0x2f, 0x6a, 0x73, 0x6f, 0x6e, 0x27, 0x2c, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x27, 0x41, 0x63, 0x63, 0x65, 0x70, 0x74, 0x27, + 0x3a, 0x20, 0x27, 0x74, 0x65, 0x78, 0x74, 0x2f, 0x65, 0x76, 0x65, 0x6e, + 0x74, 0x2d, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x27, 0x2c, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x2e, 0x2e, 0x2e, 0x28, 0x70, 0x61, 0x72, + 0x61, 0x6d, 0x73, 0x2e, 0x61, 0x70, 0x69, 0x5f, 0x6b, 0x65, 0x79, 0x20, + 0x3f, 0x20, 0x7b, 0x27, 0x41, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69, 0x7a, + 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x27, 0x3a, 0x20, 0x60, 0x42, 0x65, 0x61, + 0x72, 0x65, 0x72, 0x20, 0x24, 0x7b, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, + 0x2e, 0x61, 0x70, 0x69, 0x5f, 0x6b, 0x65, 0x79, 0x7d, 0x60, 0x7d, 0x20, + 0x3a, 0x20, 0x7b, 0x7d, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x2c, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x3a, + 0x20, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x2e, + 0x73, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x2c, 0x0a, 0x20, 0x20, 0x7d, 0x29, + 0x3b, 0x0a, 0x0a, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x72, + 0x65, 0x61, 0x64, 0x65, 0x72, 0x20, 0x3d, 0x20, 0x72, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x62, 0x6f, 0x64, 0x79, 0x2e, 0x67, 0x65, + 0x74, 0x52, 0x65, 0x61, 0x64, 0x65, 0x72, 0x28, 0x29, 0x3b, 0x0a, 0x20, + 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x64, 0x65, 0x63, 0x6f, 0x64, + 0x65, 0x72, 0x20, 0x3d, 0x20, 0x6e, 0x65, 0x77, 0x20, 0x54, 0x65, 0x78, + 0x74, 0x44, 0x65, 0x63, 0x6f, 0x64, 0x65, 0x72, 0x28, 0x29, 0x3b, 0x0a, + 0x0a, 0x20, 0x20, 0x6c, 0x65, 0x74, 0x20, 0x63, 0x6f, 0x6e, 0x74, 0x65, + 0x6e, 0x74, 0x20, 0x3d, 0x20, 0x22, 0x22, 0x3b, 0x0a, 0x20, 0x20, 0x6c, + 0x65, 0x74, 0x20, 0x6c, 0x65, 0x66, 0x74, 0x6f, 0x76, 0x65, 0x72, 0x20, + 0x3d, 0x20, 0x22, 0x22, 0x3b, 0x20, 0x2f, 0x2f, 0x20, 0x42, 0x75, 0x66, + 0x66, 0x65, 0x72, 0x20, 0x66, 0x6f, 0x72, 0x20, 0x70, 0x61, 0x72, 0x74, + 0x69, 0x61, 0x6c, 0x6c, 0x79, 0x20, 0x72, 0x65, 0x61, 0x64, 0x20, 0x6c, + 0x69, 0x6e, 0x65, 0x73, 0x0a, 0x0a, 0x20, 0x20, 0x74, 0x72, 0x79, 0x20, + 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x6c, 0x65, 0x74, 0x20, 0x63, 0x6f, + 0x6e, 0x74, 0x20, 0x3d, 0x20, 0x74, 0x72, 0x75, 0x65, 0x3b, 0x0a, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x77, 0x68, 0x69, 0x6c, 0x65, 0x20, 0x28, 0x63, + 0x6f, 0x6e, 0x74, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x72, 0x65, 0x73, 0x75, 0x6c, + 0x74, 0x20, 0x3d, 0x20, 0x61, 0x77, 0x61, 0x69, 0x74, 0x20, 0x72, 0x65, + 0x61, 0x64, 0x65, 0x72, 0x2e, 0x72, 0x65, 0x61, 0x64, 0x28, 0x29, 0x3b, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, 0x72, + 0x65, 0x73, 0x75, 0x6c, 0x74, 0x2e, 0x64, 0x6f, 0x6e, 0x65, 0x29, 0x20, + 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x62, 0x72, + 0x65, 0x61, 0x6b, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, + 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x2f, 0x2f, 0x20, 0x41, + 0x64, 0x64, 0x20, 0x61, 0x6e, 0x79, 0x20, 0x6c, 0x65, 0x66, 0x74, 0x6f, + 0x76, 0x65, 0x72, 0x20, 0x64, 0x61, 0x74, 0x61, 0x20, 0x74, 0x6f, 0x20, + 0x74, 0x68, 0x65, 0x20, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x74, 0x20, + 0x63, 0x68, 0x75, 0x6e, 0x6b, 0x20, 0x6f, 0x66, 0x20, 0x64, 0x61, 0x74, + 0x61, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, + 0x74, 0x20, 0x74, 0x65, 0x78, 0x74, 0x20, 0x3d, 0x20, 0x6c, 0x65, 0x66, + 0x74, 0x6f, 0x76, 0x65, 0x72, 0x20, 0x2b, 0x20, 0x64, 0x65, 0x63, 0x6f, + 0x64, 0x65, 0x72, 0x2e, 0x64, 0x65, 0x63, 0x6f, 0x64, 0x65, 0x28, 0x72, + 0x65, 0x73, 0x75, 0x6c, 0x74, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x29, + 0x3b, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x2f, 0x2f, 0x20, + 0x43, 0x68, 0x65, 0x63, 0x6b, 0x20, 0x69, 0x66, 0x20, 0x74, 0x68, 0x65, + 0x20, 0x6c, 0x61, 0x73, 0x74, 0x20, 0x63, 0x68, 0x61, 0x72, 0x61, 0x63, + 0x74, 0x65, 0x72, 0x20, 0x69, 0x73, 0x20, 0x61, 0x20, 0x6c, 0x69, 0x6e, + 0x65, 0x20, 0x62, 0x72, 0x65, 0x61, 0x6b, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x65, 0x6e, 0x64, 0x73, + 0x57, 0x69, 0x74, 0x68, 0x4c, 0x69, 0x6e, 0x65, 0x42, 0x72, 0x65, 0x61, + 0x6b, 0x20, 0x3d, 0x20, 0x74, 0x65, 0x78, 0x74, 0x2e, 0x65, 0x6e, 0x64, + 0x73, 0x57, 0x69, 0x74, 0x68, 0x28, 0x27, 0x5c, 0x6e, 0x27, 0x29, 0x3b, + 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x2f, 0x2f, 0x20, 0x53, + 0x70, 0x6c, 0x69, 0x74, 0x20, 0x74, 0x68, 0x65, 0x20, 0x74, 0x65, 0x78, + 0x74, 0x20, 0x69, 0x6e, 0x74, 0x6f, 0x20, 0x6c, 0x69, 0x6e, 0x65, 0x73, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x6c, 0x65, 0x74, 0x20, 0x6c, + 0x69, 0x6e, 0x65, 0x73, 0x20, 0x3d, 0x20, 0x74, 0x65, 0x78, 0x74, 0x2e, + 0x73, 0x70, 0x6c, 0x69, 0x74, 0x28, 0x27, 0x5c, 0x6e, 0x27, 0x29, 0x3b, + 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x2f, 0x2f, 0x20, 0x49, + 0x66, 0x20, 0x74, 0x68, 0x65, 0x20, 0x74, 0x65, 0x78, 0x74, 0x20, 0x64, + 0x6f, 0x65, 0x73, 0x6e, 0x27, 0x74, 0x20, 0x65, 0x6e, 0x64, 0x20, 0x77, + 0x69, 0x74, 0x68, 0x20, 0x61, 0x20, 0x6c, 0x69, 0x6e, 0x65, 0x20, 0x62, + 0x72, 0x65, 0x61, 0x6b, 0x2c, 0x20, 0x74, 0x68, 0x65, 0x6e, 0x20, 0x74, + 0x68, 0x65, 0x20, 0x6c, 0x61, 0x73, 0x74, 0x20, 0x6c, 0x69, 0x6e, 0x65, + 0x20, 0x69, 0x73, 0x20, 0x69, 0x6e, 0x63, 0x6f, 0x6d, 0x70, 0x6c, 0x65, + 0x74, 0x65, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x2f, 0x2f, 0x20, + 0x53, 0x74, 0x6f, 0x72, 0x65, 0x20, 0x69, 0x74, 0x20, 0x69, 0x6e, 0x20, + 0x6c, 0x65, 0x66, 0x74, 0x6f, 0x76, 0x65, 0x72, 0x20, 0x74, 0x6f, 0x20, + 0x62, 0x65, 0x20, 0x61, 0x64, 0x64, 0x65, 0x64, 0x20, 0x74, 0x6f, 0x20, + 0x74, 0x68, 0x65, 0x20, 0x6e, 0x65, 0x78, 0x74, 0x20, 0x63, 0x68, 0x75, + 0x6e, 0x6b, 0x20, 0x6f, 0x66, 0x20, 0x64, 0x61, 0x74, 0x61, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, 0x21, 0x65, 0x6e, + 0x64, 0x73, 0x57, 0x69, 0x74, 0x68, 0x4c, 0x69, 0x6e, 0x65, 0x42, 0x72, + 0x65, 0x61, 0x6b, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x6c, 0x65, 0x66, 0x74, 0x6f, 0x76, 0x65, 0x72, 0x20, + 0x3d, 0x20, 0x6c, 0x69, 0x6e, 0x65, 0x73, 0x2e, 0x70, 0x6f, 0x70, 0x28, + 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x20, 0x65, + 0x6c, 0x73, 0x65, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x6c, 0x65, 0x66, 0x74, 0x6f, 0x76, 0x65, 0x72, 0x20, 0x3d, + 0x20, 0x22, 0x22, 0x3b, 0x20, 0x2f, 0x2f, 0x20, 0x52, 0x65, 0x73, 0x65, + 0x74, 0x20, 0x6c, 0x65, 0x66, 0x74, 0x6f, 0x76, 0x65, 0x72, 0x20, 0x69, + 0x66, 0x20, 0x77, 0x65, 0x20, 0x68, 0x61, 0x76, 0x65, 0x20, 0x61, 0x20, + 0x6c, 0x69, 0x6e, 0x65, 0x20, 0x62, 0x72, 0x65, 0x61, 0x6b, 0x20, 0x61, + 0x74, 0x20, 0x74, 0x68, 0x65, 0x20, 0x65, 0x6e, 0x64, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x2f, 0x2f, 0x20, 0x50, 0x61, 0x72, 0x73, 0x65, 0x20, 0x61, 0x6c, + 0x6c, 0x20, 0x73, 0x73, 0x65, 0x20, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x73, + 0x20, 0x61, 0x6e, 0x64, 0x20, 0x61, 0x64, 0x64, 0x20, 0x74, 0x68, 0x65, + 0x6d, 0x20, 0x74, 0x6f, 0x20, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, + 0x72, 0x65, 0x67, 0x65, 0x78, 0x20, 0x3d, 0x20, 0x2f, 0x5e, 0x28, 0x5c, + 0x53, 0x2b, 0x29, 0x3a, 0x5c, 0x73, 0x28, 0x2e, 0x2a, 0x29, 0x24, 0x2f, + 0x67, 0x6d, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x66, 0x6f, + 0x72, 0x20, 0x28, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x6c, 0x69, 0x6e, + 0x65, 0x20, 0x6f, 0x66, 0x20, 0x6c, 0x69, 0x6e, 0x65, 0x73, 0x29, 0x20, + 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, + 0x6e, 0x73, 0x74, 0x20, 0x6d, 0x61, 0x74, 0x63, 0x68, 0x20, 0x3d, 0x20, + 0x72, 0x65, 0x67, 0x65, 0x78, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x28, 0x6c, + 0x69, 0x6e, 0x65, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, 0x6d, 0x61, 0x74, 0x63, 0x68, 0x29, + 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x5b, 0x6d, 0x61, 0x74, 0x63, + 0x68, 0x5b, 0x31, 0x5d, 0x5d, 0x20, 0x3d, 0x20, 0x6d, 0x61, 0x74, 0x63, + 0x68, 0x5b, 0x32, 0x5d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x2f, 0x2f, 0x20, 0x73, 0x69, 0x6e, 0x63, 0x65, 0x20, + 0x77, 0x65, 0x20, 0x6b, 0x6e, 0x6f, 0x77, 0x20, 0x74, 0x68, 0x69, 0x73, + 0x20, 0x69, 0x73, 0x20, 0x6c, 0x6c, 0x61, 0x6d, 0x61, 0x2e, 0x63, 0x70, + 0x70, 0x2c, 0x20, 0x6c, 0x65, 0x74, 0x27, 0x73, 0x20, 0x6a, 0x75, 0x73, + 0x74, 0x20, 0x64, 0x65, 0x63, 0x6f, 0x64, 0x65, 0x20, 0x74, 0x68, 0x65, + 0x20, 0x6a, 0x73, 0x6f, 0x6e, 0x20, 0x69, 0x6e, 0x20, 0x64, 0x61, 0x74, + 0x61, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x69, 0x66, 0x20, 0x28, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x2e, 0x64, + 0x61, 0x74, 0x61, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, 0x73, 0x75, 0x6c, + 0x74, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x20, 0x3d, 0x20, 0x4a, 0x53, 0x4f, + 0x4e, 0x2e, 0x70, 0x61, 0x72, 0x73, 0x65, 0x28, 0x72, 0x65, 0x73, 0x75, + 0x6c, 0x74, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x29, 0x3b, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, + 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x20, 0x2b, 0x3d, 0x20, 0x72, 0x65, 0x73, + 0x75, 0x6c, 0x74, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x63, 0x6f, 0x6e, + 0x74, 0x65, 0x6e, 0x74, 0x3b, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x2f, 0x2f, 0x20, 0x79, 0x69, + 0x65, 0x6c, 0x64, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x79, 0x69, 0x65, 0x6c, 0x64, 0x20, 0x72, 0x65, + 0x73, 0x75, 0x6c, 0x74, 0x3b, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x2f, 0x2f, 0x20, 0x69, 0x66, + 0x20, 0x77, 0x65, 0x20, 0x67, 0x6f, 0x74, 0x20, 0x61, 0x20, 0x73, 0x74, + 0x6f, 0x70, 0x20, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x20, 0x66, 0x72, 0x6f, + 0x6d, 0x20, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x2c, 0x20, 0x77, 0x65, + 0x20, 0x77, 0x69, 0x6c, 0x6c, 0x20, 0x62, 0x72, 0x65, 0x61, 0x6b, 0x20, + 0x68, 0x65, 0x72, 0x65, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, 0x72, 0x65, 0x73, + 0x75, 0x6c, 0x74, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x73, 0x74, 0x6f, + 0x70, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, 0x72, + 0x65, 0x73, 0x75, 0x6c, 0x74, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x67, + 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x73, 0x65, + 0x74, 0x74, 0x69, 0x6e, 0x67, 0x73, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, + 0x5f, 0x73, 0x65, 0x74, 0x74, 0x69, 0x6e, 0x67, 0x73, 0x20, 0x3d, 0x20, + 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x2e, + 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x73, + 0x65, 0x74, 0x74, 0x69, 0x6e, 0x67, 0x73, 0x3b, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x74, 0x20, 0x3d, 0x20, 0x66, 0x61, + 0x6c, 0x73, 0x65, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x62, 0x72, 0x65, 0x61, 0x6b, + 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, + 0x2e, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, + 0x73, 0x75, 0x6c, 0x74, 0x2e, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x20, 0x3d, + 0x20, 0x4a, 0x53, 0x4f, 0x4e, 0x2e, 0x70, 0x61, 0x72, 0x73, 0x65, 0x28, + 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x2e, 0x65, 0x72, 0x72, 0x6f, 0x72, + 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x6f, 0x6c, 0x65, 0x2e, 0x65, + 0x72, 0x72, 0x6f, 0x72, 0x28, 0x60, 0x6c, 0x6c, 0x61, 0x6d, 0x61, 0x2e, + 0x63, 0x70, 0x70, 0x20, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x3a, 0x20, 0x24, + 0x7b, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x2e, 0x65, 0x72, 0x72, 0x6f, + 0x72, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x7d, 0x60, 0x29, + 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x7d, 0x0a, 0x20, 0x20, 0x7d, 0x20, 0x63, 0x61, 0x74, 0x63, 0x68, 0x20, + 0x28, 0x65, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, + 0x20, 0x28, 0x65, 0x2e, 0x6e, 0x61, 0x6d, 0x65, 0x20, 0x21, 0x3d, 0x3d, + 0x20, 0x27, 0x41, 0x62, 0x6f, 0x72, 0x74, 0x45, 0x72, 0x72, 0x6f, 0x72, + 0x27, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, + 0x6f, 0x6e, 0x73, 0x6f, 0x6c, 0x65, 0x2e, 0x65, 0x72, 0x72, 0x6f, 0x72, + 0x28, 0x22, 0x6c, 0x6c, 0x61, 0x6d, 0x61, 0x20, 0x65, 0x72, 0x72, 0x6f, + 0x72, 0x3a, 0x20, 0x22, 0x2c, 0x20, 0x65, 0x29, 0x3b, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x74, 0x68, 0x72, 0x6f, + 0x77, 0x20, 0x65, 0x3b, 0x0a, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x66, + 0x69, 0x6e, 0x61, 0x6c, 0x6c, 0x79, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x2e, + 0x61, 0x62, 0x6f, 0x72, 0x74, 0x28, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x7d, + 0x0a, 0x0a, 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x63, + 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x3b, 0x0a, 0x7d, 0x0a, 0x0a, 0x2f, + 0x2f, 0x20, 0x43, 0x61, 0x6c, 0x6c, 0x20, 0x6c, 0x6c, 0x61, 0x6d, 0x61, + 0x2c, 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x61, 0x6e, 0x20, + 0x65, 0x76, 0x65, 0x6e, 0x74, 0x20, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, + 0x20, 0x74, 0x68, 0x61, 0x74, 0x20, 0x79, 0x6f, 0x75, 0x20, 0x63, 0x61, + 0x6e, 0x20, 0x73, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x62, 0x65, 0x20, + 0x74, 0x6f, 0x0a, 0x2f, 0x2f, 0x0a, 0x2f, 0x2f, 0x20, 0x45, 0x78, 0x61, + 0x6d, 0x70, 0x6c, 0x65, 0x3a, 0x0a, 0x2f, 0x2f, 0x0a, 0x2f, 0x2f, 0x20, + 0x20, 0x20, 0x20, 0x69, 0x6d, 0x70, 0x6f, 0x72, 0x74, 0x20, 0x7b, 0x20, + 0x6c, 0x6c, 0x61, 0x6d, 0x61, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x54, 0x61, + 0x72, 0x67, 0x65, 0x74, 0x20, 0x7d, 0x20, 0x66, 0x72, 0x6f, 0x6d, 0x20, + 0x27, 0x2f, 0x63, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x69, 0x6f, 0x6e, + 0x2e, 0x6a, 0x73, 0x27, 0x0a, 0x2f, 0x2f, 0x0a, 0x2f, 0x2f, 0x20, 0x20, + 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x63, 0x6f, 0x6e, 0x6e, + 0x20, 0x3d, 0x20, 0x6c, 0x6c, 0x61, 0x6d, 0x61, 0x45, 0x76, 0x65, 0x6e, + 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x28, 0x70, 0x72, 0x6f, 0x6d, + 0x70, 0x74, 0x29, 0x0a, 0x2f, 0x2f, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, + 0x6e, 0x6e, 0x2e, 0x61, 0x64, 0x64, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x4c, + 0x69, 0x73, 0x74, 0x65, 0x6e, 0x65, 0x72, 0x28, 0x22, 0x6d, 0x65, 0x73, + 0x73, 0x61, 0x67, 0x65, 0x22, 0x2c, 0x20, 0x28, 0x63, 0x68, 0x75, 0x6e, + 0x6b, 0x29, 0x20, 0x3d, 0x3e, 0x20, 0x7b, 0x0a, 0x2f, 0x2f, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x64, 0x6f, 0x63, 0x75, 0x6d, 0x65, 0x6e, 0x74, + 0x2e, 0x77, 0x72, 0x69, 0x74, 0x65, 0x28, 0x63, 0x68, 0x75, 0x6e, 0x6b, + 0x2e, 0x64, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x74, + 0x65, 0x6e, 0x74, 0x29, 0x0a, 0x2f, 0x2f, 0x20, 0x20, 0x20, 0x20, 0x7d, + 0x29, 0x0a, 0x2f, 0x2f, 0x0a, 0x65, 0x78, 0x70, 0x6f, 0x72, 0x74, 0x20, + 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x6c, 0x6c, 0x61, 0x6d, 0x61, 0x45, + 0x76, 0x65, 0x6e, 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x20, 0x3d, + 0x20, 0x28, 0x70, 0x72, 0x6f, 0x6d, 0x70, 0x74, 0x2c, 0x20, 0x70, 0x61, + 0x72, 0x61, 0x6d, 0x73, 0x20, 0x3d, 0x20, 0x7b, 0x7d, 0x2c, 0x20, 0x63, + 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x20, 0x3d, 0x20, 0x7b, 0x7d, 0x29, 0x20, + 0x3d, 0x3e, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, + 0x20, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, + 0x20, 0x3d, 0x20, 0x6e, 0x65, 0x77, 0x20, 0x45, 0x76, 0x65, 0x6e, 0x74, + 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x28, 0x29, 0x3b, 0x0a, 0x20, 0x20, + 0x28, 0x61, 0x73, 0x79, 0x6e, 0x63, 0x20, 0x28, 0x29, 0x20, 0x3d, 0x3e, + 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x6c, 0x65, 0x74, 0x20, 0x63, + 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x20, 0x3d, 0x20, 0x22, 0x22, 0x3b, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x66, 0x6f, 0x72, 0x20, 0x61, 0x77, 0x61, + 0x69, 0x74, 0x20, 0x28, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x63, 0x68, + 0x75, 0x6e, 0x6b, 0x20, 0x6f, 0x66, 0x20, 0x6c, 0x6c, 0x61, 0x6d, 0x61, + 0x28, 0x70, 0x72, 0x6f, 0x6d, 0x70, 0x74, 0x2c, 0x20, 0x70, 0x61, 0x72, + 0x61, 0x6d, 0x73, 0x2c, 0x20, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x29, + 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, + 0x20, 0x28, 0x63, 0x68, 0x75, 0x6e, 0x6b, 0x2e, 0x64, 0x61, 0x74, 0x61, + 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x20, 0x2b, 0x3d, 0x20, 0x63, + 0x68, 0x75, 0x6e, 0x6b, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x63, 0x6f, + 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x54, 0x61, 0x72, 0x67, + 0x65, 0x74, 0x2e, 0x64, 0x69, 0x73, 0x70, 0x61, 0x74, 0x63, 0x68, 0x45, + 0x76, 0x65, 0x6e, 0x74, 0x28, 0x6e, 0x65, 0x77, 0x20, 0x43, 0x75, 0x73, + 0x74, 0x6f, 0x6d, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x28, 0x22, 0x6d, 0x65, + 0x73, 0x73, 0x61, 0x67, 0x65, 0x22, 0x2c, 0x20, 0x7b, 0x20, 0x64, 0x65, + 0x74, 0x61, 0x69, 0x6c, 0x3a, 0x20, 0x63, 0x68, 0x75, 0x6e, 0x6b, 0x2e, + 0x64, 0x61, 0x74, 0x61, 0x20, 0x7d, 0x29, 0x29, 0x3b, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x69, 0x66, 0x20, 0x28, 0x63, 0x68, 0x75, 0x6e, 0x6b, 0x2e, 0x64, 0x61, + 0x74, 0x61, 0x2e, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, + 0x6e, 0x5f, 0x73, 0x65, 0x74, 0x74, 0x69, 0x6e, 0x67, 0x73, 0x29, 0x20, + 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x65, 0x76, + 0x65, 0x6e, 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x2e, 0x64, 0x69, + 0x73, 0x70, 0x61, 0x74, 0x63, 0x68, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x28, + 0x6e, 0x65, 0x77, 0x20, 0x43, 0x75, 0x73, 0x74, 0x6f, 0x6d, 0x45, 0x76, + 0x65, 0x6e, 0x74, 0x28, 0x22, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, + 0x69, 0x6f, 0x6e, 0x5f, 0x73, 0x65, 0x74, 0x74, 0x69, 0x6e, 0x67, 0x73, + 0x22, 0x2c, 0x20, 0x7b, 0x20, 0x64, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x3a, + 0x20, 0x63, 0x68, 0x75, 0x6e, 0x6b, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x2e, + 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x73, + 0x65, 0x74, 0x74, 0x69, 0x6e, 0x67, 0x73, 0x20, 0x7d, 0x29, 0x29, 0x3b, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, 0x63, 0x68, 0x75, 0x6e, 0x6b, + 0x2e, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x74, 0x69, 0x6d, 0x69, 0x6e, 0x67, + 0x73, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, + 0x2e, 0x64, 0x69, 0x73, 0x70, 0x61, 0x74, 0x63, 0x68, 0x45, 0x76, 0x65, + 0x6e, 0x74, 0x28, 0x6e, 0x65, 0x77, 0x20, 0x43, 0x75, 0x73, 0x74, 0x6f, + 0x6d, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x28, 0x22, 0x74, 0x69, 0x6d, 0x69, + 0x6e, 0x67, 0x73, 0x22, 0x2c, 0x20, 0x7b, 0x20, 0x64, 0x65, 0x74, 0x61, + 0x69, 0x6c, 0x3a, 0x20, 0x63, 0x68, 0x75, 0x6e, 0x6b, 0x2e, 0x64, 0x61, + 0x74, 0x61, 0x2e, 0x74, 0x69, 0x6d, 0x69, 0x6e, 0x67, 0x73, 0x20, 0x7d, + 0x29, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x65, 0x76, + 0x65, 0x6e, 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x2e, 0x64, 0x69, + 0x73, 0x70, 0x61, 0x74, 0x63, 0x68, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x28, + 0x6e, 0x65, 0x77, 0x20, 0x43, 0x75, 0x73, 0x74, 0x6f, 0x6d, 0x45, 0x76, + 0x65, 0x6e, 0x74, 0x28, 0x22, 0x64, 0x6f, 0x6e, 0x65, 0x22, 0x2c, 0x20, + 0x7b, 0x20, 0x64, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x3a, 0x20, 0x7b, 0x20, + 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x20, 0x7d, 0x20, 0x7d, 0x29, + 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x7d, 0x29, 0x28, 0x29, 0x3b, 0x0a, 0x20, + 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x65, 0x76, 0x65, 0x6e, + 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x3b, 0x0a, 0x7d, 0x0a, 0x0a, + 0x2f, 0x2f, 0x20, 0x43, 0x61, 0x6c, 0x6c, 0x20, 0x6c, 0x6c, 0x61, 0x6d, + 0x61, 0x2c, 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x61, 0x20, + 0x70, 0x72, 0x6f, 0x6d, 0x69, 0x73, 0x65, 0x20, 0x74, 0x68, 0x61, 0x74, + 0x20, 0x72, 0x65, 0x73, 0x6f, 0x6c, 0x76, 0x65, 0x73, 0x20, 0x74, 0x6f, + 0x20, 0x74, 0x68, 0x65, 0x20, 0x63, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, + 0x65, 0x64, 0x20, 0x74, 0x65, 0x78, 0x74, 0x2e, 0x20, 0x54, 0x68, 0x69, + 0x73, 0x20, 0x64, 0x6f, 0x65, 0x73, 0x20, 0x6e, 0x6f, 0x74, 0x20, 0x73, + 0x75, 0x70, 0x70, 0x6f, 0x72, 0x74, 0x20, 0x73, 0x74, 0x72, 0x65, 0x61, + 0x6d, 0x69, 0x6e, 0x67, 0x0a, 0x2f, 0x2f, 0x0a, 0x2f, 0x2f, 0x20, 0x45, + 0x78, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x3a, 0x0a, 0x2f, 0x2f, 0x0a, 0x2f, + 0x2f, 0x20, 0x20, 0x20, 0x20, 0x20, 0x6c, 0x6c, 0x61, 0x6d, 0x61, 0x50, + 0x72, 0x6f, 0x6d, 0x69, 0x73, 0x65, 0x28, 0x70, 0x72, 0x6f, 0x6d, 0x70, + 0x74, 0x29, 0x2e, 0x74, 0x68, 0x65, 0x6e, 0x28, 0x28, 0x63, 0x6f, 0x6e, + 0x74, 0x65, 0x6e, 0x74, 0x29, 0x20, 0x3d, 0x3e, 0x20, 0x7b, 0x0a, 0x2f, + 0x2f, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x64, 0x6f, 0x63, 0x75, + 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x77, 0x72, 0x69, 0x74, 0x65, 0x28, 0x63, + 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x29, 0x0a, 0x2f, 0x2f, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x7d, 0x29, 0x0a, 0x2f, 0x2f, 0x0a, 0x2f, 0x2f, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x6f, 0x72, 0x0a, 0x2f, 0x2f, 0x0a, 0x2f, 0x2f, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x63, + 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x20, 0x3d, 0x20, 0x61, 0x77, 0x61, + 0x69, 0x74, 0x20, 0x6c, 0x6c, 0x61, 0x6d, 0x61, 0x50, 0x72, 0x6f, 0x6d, + 0x69, 0x73, 0x65, 0x28, 0x70, 0x72, 0x6f, 0x6d, 0x70, 0x74, 0x29, 0x0a, + 0x2f, 0x2f, 0x20, 0x20, 0x20, 0x20, 0x20, 0x64, 0x6f, 0x63, 0x75, 0x6d, + 0x65, 0x6e, 0x74, 0x2e, 0x77, 0x72, 0x69, 0x74, 0x65, 0x28, 0x63, 0x6f, + 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x29, 0x0a, 0x2f, 0x2f, 0x0a, 0x65, 0x78, + 0x70, 0x6f, 0x72, 0x74, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x6c, + 0x6c, 0x61, 0x6d, 0x61, 0x50, 0x72, 0x6f, 0x6d, 0x69, 0x73, 0x65, 0x20, + 0x3d, 0x20, 0x28, 0x70, 0x72, 0x6f, 0x6d, 0x70, 0x74, 0x2c, 0x20, 0x70, + 0x61, 0x72, 0x61, 0x6d, 0x73, 0x20, 0x3d, 0x20, 0x7b, 0x7d, 0x2c, 0x20, + 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x20, 0x3d, 0x20, 0x7b, 0x7d, 0x29, + 0x20, 0x3d, 0x3e, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, + 0x72, 0x6e, 0x20, 0x6e, 0x65, 0x77, 0x20, 0x50, 0x72, 0x6f, 0x6d, 0x69, + 0x73, 0x65, 0x28, 0x61, 0x73, 0x79, 0x6e, 0x63, 0x20, 0x28, 0x72, 0x65, + 0x73, 0x6f, 0x6c, 0x76, 0x65, 0x2c, 0x20, 0x72, 0x65, 0x6a, 0x65, 0x63, + 0x74, 0x29, 0x20, 0x3d, 0x3e, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x6c, 0x65, 0x74, 0x20, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x20, + 0x3d, 0x20, 0x22, 0x22, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x74, 0x72, + 0x79, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x66, 0x6f, + 0x72, 0x20, 0x61, 0x77, 0x61, 0x69, 0x74, 0x20, 0x28, 0x63, 0x6f, 0x6e, + 0x73, 0x74, 0x20, 0x63, 0x68, 0x75, 0x6e, 0x6b, 0x20, 0x6f, 0x66, 0x20, + 0x6c, 0x6c, 0x61, 0x6d, 0x61, 0x28, 0x70, 0x72, 0x6f, 0x6d, 0x70, 0x74, + 0x2c, 0x20, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x2c, 0x20, 0x63, 0x6f, + 0x6e, 0x66, 0x69, 0x67, 0x29, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, + 0x20, 0x2b, 0x3d, 0x20, 0x63, 0x68, 0x75, 0x6e, 0x6b, 0x2e, 0x64, 0x61, + 0x74, 0x61, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x3b, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x72, 0x65, 0x73, 0x6f, 0x6c, 0x76, 0x65, 0x28, 0x63, 0x6f, + 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x7d, 0x20, 0x63, 0x61, 0x74, 0x63, 0x68, 0x20, 0x28, 0x65, 0x72, 0x72, + 0x6f, 0x72, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x72, 0x65, 0x6a, 0x65, 0x63, 0x74, 0x28, 0x65, 0x72, 0x72, 0x6f, 0x72, + 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x7d, + 0x29, 0x3b, 0x0a, 0x7d, 0x3b, 0x0a, 0x0a, 0x2f, 0x2a, 0x2a, 0x0a, 0x20, + 0x2a, 0x20, 0x28, 0x64, 0x65, 0x70, 0x72, 0x65, 0x63, 0x61, 0x74, 0x65, + 0x64, 0x29, 0x0a, 0x20, 0x2a, 0x2f, 0x0a, 0x65, 0x78, 0x70, 0x6f, 0x72, + 0x74, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x6c, 0x6c, 0x61, 0x6d, + 0x61, 0x43, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65, 0x20, 0x3d, 0x20, + 0x61, 0x73, 0x79, 0x6e, 0x63, 0x20, 0x28, 0x70, 0x61, 0x72, 0x61, 0x6d, + 0x73, 0x2c, 0x20, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, + 0x72, 0x2c, 0x20, 0x63, 0x61, 0x6c, 0x6c, 0x62, 0x61, 0x63, 0x6b, 0x29, + 0x20, 0x3d, 0x3e, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x66, 0x6f, 0x72, 0x20, + 0x61, 0x77, 0x61, 0x69, 0x74, 0x20, 0x28, 0x63, 0x6f, 0x6e, 0x73, 0x74, + 0x20, 0x63, 0x68, 0x75, 0x6e, 0x6b, 0x20, 0x6f, 0x66, 0x20, 0x6c, 0x6c, + 0x61, 0x6d, 0x61, 0x28, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x2e, 0x70, + 0x72, 0x6f, 0x6d, 0x70, 0x74, 0x2c, 0x20, 0x70, 0x61, 0x72, 0x61, 0x6d, + 0x73, 0x2c, 0x20, 0x7b, 0x20, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, + 0x6c, 0x65, 0x72, 0x20, 0x7d, 0x29, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x63, 0x61, 0x6c, 0x6c, 0x62, 0x61, 0x63, 0x6b, 0x28, 0x63, + 0x68, 0x75, 0x6e, 0x6b, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x7d, 0x0a, 0x7d, + 0x0a, 0x0a, 0x2f, 0x2f, 0x20, 0x47, 0x65, 0x74, 0x20, 0x74, 0x68, 0x65, + 0x20, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x20, 0x69, 0x6e, 0x66, 0x6f, 0x20, + 0x66, 0x72, 0x6f, 0x6d, 0x20, 0x74, 0x68, 0x65, 0x20, 0x73, 0x65, 0x72, + 0x76, 0x65, 0x72, 0x2e, 0x20, 0x54, 0x68, 0x69, 0x73, 0x20, 0x69, 0x73, + 0x20, 0x75, 0x73, 0x65, 0x66, 0x75, 0x6c, 0x20, 0x66, 0x6f, 0x72, 0x20, + 0x67, 0x65, 0x74, 0x74, 0x69, 0x6e, 0x67, 0x20, 0x74, 0x68, 0x65, 0x20, + 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x20, 0x77, 0x69, 0x6e, 0x64, + 0x6f, 0x77, 0x20, 0x61, 0x6e, 0x64, 0x20, 0x73, 0x6f, 0x20, 0x6f, 0x6e, + 0x2e, 0x0a, 0x65, 0x78, 0x70, 0x6f, 0x72, 0x74, 0x20, 0x63, 0x6f, 0x6e, + 0x73, 0x74, 0x20, 0x6c, 0x6c, 0x61, 0x6d, 0x61, 0x4d, 0x6f, 0x64, 0x65, + 0x6c, 0x49, 0x6e, 0x66, 0x6f, 0x20, 0x3d, 0x20, 0x61, 0x73, 0x79, 0x6e, + 0x63, 0x20, 0x28, 0x29, 0x20, 0x3d, 0x3e, 0x20, 0x7b, 0x0a, 0x20, 0x20, + 0x69, 0x66, 0x20, 0x28, 0x21, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, + 0x69, 0x6f, 0x6e, 0x5f, 0x73, 0x65, 0x74, 0x74, 0x69, 0x6e, 0x67, 0x73, + 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x67, 0x65, 0x6e, 0x65, + 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x73, 0x65, 0x74, 0x74, 0x69, + 0x6e, 0x67, 0x73, 0x20, 0x3d, 0x20, 0x61, 0x77, 0x61, 0x69, 0x74, 0x20, + 0x66, 0x65, 0x74, 0x63, 0x68, 0x28, 0x22, 0x2f, 0x6d, 0x6f, 0x64, 0x65, + 0x6c, 0x2e, 0x6a, 0x73, 0x6f, 0x6e, 0x22, 0x29, 0x2e, 0x74, 0x68, 0x65, + 0x6e, 0x28, 0x72, 0x20, 0x3d, 0x3e, 0x20, 0x72, 0x2e, 0x6a, 0x73, 0x6f, + 0x6e, 0x28, 0x29, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, + 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x67, 0x65, 0x6e, 0x65, 0x72, + 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x73, 0x65, 0x74, 0x74, 0x69, 0x6e, + 0x67, 0x73, 0x3b, 0x0a, 0x7d, 0x0a }; - -let generation_settings = null; - - -// Completes the prompt as a generator. Recommended for most use cases. -// -// Example: -// -// import { llama } from '/completion.js' -// -// const request = llama("Tell me a joke", {n_predict: 800}) -// for await (const chunk of request) { -// document.write(chunk.data.content) -// } -// -export async function* llama(prompt, params = {}, config = {}) { - let controller = config.controller; - - if (!controller) { - controller = new AbortController(); - } - - const completionParams = { ...paramDefaults, ...params, prompt }; - - const response = await fetch("/completion", { - method: 'POST', - body: JSON.stringify(completionParams), - headers: { - 'Connection': 'keep-alive', - 'Content-Type': 'application/json', - 'Accept': 'text/event-stream', - ...(params.api_key ? {'Authorization': `Bearer ${params.api_key}`} : {}) - }, - signal: controller.signal, - }); - - const reader = response.body.getReader(); - const decoder = new TextDecoder(); - - let content = ""; - let leftover = ""; // Buffer for partially read lines - - try { - let cont = true; - - while (cont) { - const result = await reader.read(); - if (result.done) { - break; - } - - // Add any leftover data to the current chunk of data - const text = leftover + decoder.decode(result.value); - - // Check if the last character is a line break - const endsWithLineBreak = text.endsWith('\n'); - - // Split the text into lines - let lines = text.split('\n'); - - // If the text doesn't end with a line break, then the last line is incomplete - // Store it in leftover to be added to the next chunk of data - if (!endsWithLineBreak) { - leftover = lines.pop(); - } else { - leftover = ""; // Reset leftover if we have a line break at the end - } - - // Parse all sse events and add them to result - const regex = /^(\S+):\s(.*)$/gm; - for (const line of lines) { - const match = regex.exec(line); - if (match) { - result[match[1]] = match[2] - // since we know this is llama.cpp, let's just decode the json in data - if (result.data) { - result.data = JSON.parse(result.data); - content += result.data.content; - - // yield - yield result; - - // if we got a stop token from server, we will break here - if (result.data.stop) { - if (result.data.generation_settings) { - generation_settings = result.data.generation_settings; - } - cont = false; - break; - } - } - if (result.error) { - result.error = JSON.parse(result.error); - if (result.error.content.includes('slot unavailable')) { - // Throw an error to be caught by upstream callers - throw new Error('slot unavailable'); - } else { - console.error(`llama.cpp error: ${result.error.content}`); - } - } - if (result.error) { - result.error = JSON.parse(result.error); - console.error(`llama.cpp error: ${result.error.content}`); - } - } - } - } - } catch (e) { - if (e.name !== 'AbortError') { - console.error("llama error: ", e); - } - throw e; - } - finally { - controller.abort(); - } - - return content; -} - -// Call llama, return an event target that you can subscribe to -// -// Example: -// -// import { llamaEventTarget } from '/completion.js' -// -// const conn = llamaEventTarget(prompt) -// conn.addEventListener("message", (chunk) => { -// document.write(chunk.detail.content) -// }) -// -export const llamaEventTarget = (prompt, params = {}, config = {}) => { - const eventTarget = new EventTarget(); - (async () => { - let content = ""; - for await (const chunk of llama(prompt, params, config)) { - if (chunk.data) { - content += chunk.data.content; - eventTarget.dispatchEvent(new CustomEvent("message", { detail: chunk.data })); - } - if (chunk.data.generation_settings) { - eventTarget.dispatchEvent(new CustomEvent("generation_settings", { detail: chunk.data.generation_settings })); - } - if (chunk.data.timings) { - eventTarget.dispatchEvent(new CustomEvent("timings", { detail: chunk.data.timings })); - } - } - eventTarget.dispatchEvent(new CustomEvent("done", { detail: { content } })); - })(); - return eventTarget; -} - -// Call llama, return a promise that resolves to the completed text. This does not support streaming -// -// Example: -// -// llamaPromise(prompt).then((content) => { -// document.write(content) -// }) -// -// or -// -// const content = await llamaPromise(prompt) -// document.write(content) -// -export const llamaPromise = (prompt, params = {}, config = {}) => { - return new Promise(async (resolve, reject) => { - let content = ""; - try { - for await (const chunk of llama(prompt, params, config)) { - content += chunk.data.content; - } - resolve(content); - } catch (error) { - reject(error); - } - }); -}; - -/** - * (deprecated) - */ -export const llamaComplete = async (params, controller, callback) => { - for await (const chunk of llama(params.prompt, params, { controller })) { - callback(chunk); - } -} - -// Get the model info from the server. This is useful for getting the context window and so on. -export const llamaModelInfo = async () => { - if (!generation_settings) { - generation_settings = await fetch("/model.json").then(r => r.json()); - } - return generation_settings; -} -)LITERAL"; -unsigned int completion_js_len = sizeof(completion_js); +unsigned int completion_js_len = 5346; diff --git a/examples/server/deps.sh b/examples/server/deps.sh index c0a9de9f9..ea23e6450 100755 --- a/examples/server/deps.sh +++ b/examples/server/deps.sh @@ -15,13 +15,6 @@ cd $PUBLIC for FILE in $FILES; do echo "generate $FILE.hpp" - # Use C++11 string literals instead of ugly xxd. - f=$(echo $FILE | sed 's/\./_/g' -e 's/-/_/g') - echo "const char $f[] = R\"LITERAL(" > $DIR/$FILE.hpp - cat $FILE >> $DIR/$FILE.hpp - echo ")LITERAL\";" >> $DIR/$FILE.hpp - echo "unsigned int ${f}_len = sizeof($f);" >> $DIR/$FILE.hpp - - #Deprecated old xxd - #xxd -i $FILE > $DIR/$FILE.hpp + # use simple flag for old version of xxd + xxd -i $FILE > $DIR/$FILE.hpp done diff --git a/examples/server/index.html.hpp b/examples/server/index.html.hpp index 603d12068..20551520e 100644 --- a/examples/server/index.html.hpp +++ b/examples/server/index.html.hpp @@ -1,1038 +1,2791 @@ -const char index_html[] = R"LITERAL( - - - - - - - llama.cpp - chat - - - - - - - -
- -
-
- - - - -)LITERAL"; -unsigned int index_html_len = sizeof(index_html); +unsigned char index_html[] = { + 0x3c, 0x68, 0x74, 0x6d, 0x6c, 0x3e, 0x0a, 0x0a, 0x3c, 0x68, 0x65, 0x61, + 0x64, 0x3e, 0x0a, 0x20, 0x20, 0x3c, 0x6d, 0x65, 0x74, 0x61, 0x20, 0x63, + 0x68, 0x61, 0x72, 0x73, 0x65, 0x74, 0x3d, 0x22, 0x55, 0x54, 0x46, 0x2d, + 0x38, 0x22, 0x3e, 0x0a, 0x20, 0x20, 0x3c, 0x6d, 0x65, 0x74, 0x61, 0x20, + 0x6e, 0x61, 0x6d, 0x65, 0x3d, 0x22, 0x76, 0x69, 0x65, 0x77, 0x70, 0x6f, + 0x72, 0x74, 0x22, 0x20, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x3d, + 0x22, 0x77, 0x69, 0x64, 0x74, 0x68, 0x3d, 0x64, 0x65, 0x76, 0x69, 0x63, + 0x65, 0x2d, 0x77, 0x69, 0x64, 0x74, 0x68, 0x2c, 0x20, 0x69, 0x6e, 0x69, + 0x74, 0x69, 0x61, 0x6c, 0x2d, 0x73, 0x63, 0x61, 0x6c, 0x65, 0x3d, 0x31, + 0x2c, 0x20, 0x6d, 0x61, 0x78, 0x69, 0x6d, 0x75, 0x6d, 0x2d, 0x73, 0x63, + 0x61, 0x6c, 0x65, 0x3d, 0x31, 0x22, 0x20, 0x2f, 0x3e, 0x0a, 0x20, 0x20, + 0x3c, 0x6d, 0x65, 0x74, 0x61, 0x20, 0x6e, 0x61, 0x6d, 0x65, 0x3d, 0x22, + 0x63, 0x6f, 0x6c, 0x6f, 0x72, 0x2d, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x65, + 0x22, 0x20, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x3d, 0x22, 0x6c, + 0x69, 0x67, 0x68, 0x74, 0x20, 0x64, 0x61, 0x72, 0x6b, 0x22, 0x3e, 0x0a, + 0x20, 0x20, 0x3c, 0x74, 0x69, 0x74, 0x6c, 0x65, 0x3e, 0x6c, 0x6c, 0x61, + 0x6d, 0x61, 0x2e, 0x63, 0x70, 0x70, 0x20, 0x2d, 0x20, 0x63, 0x68, 0x61, + 0x74, 0x3c, 0x2f, 0x74, 0x69, 0x74, 0x6c, 0x65, 0x3e, 0x0a, 0x0a, 0x20, + 0x20, 0x3c, 0x73, 0x74, 0x79, 0x6c, 0x65, 0x3e, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x62, 0x6f, 0x64, 0x79, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x66, 0x6f, 0x6e, 0x74, 0x2d, 0x66, 0x61, 0x6d, 0x69, 0x6c, + 0x79, 0x3a, 0x20, 0x73, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x2d, 0x75, 0x69, + 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x66, 0x6f, 0x6e, 0x74, + 0x2d, 0x73, 0x69, 0x7a, 0x65, 0x3a, 0x20, 0x39, 0x30, 0x25, 0x3b, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x23, + 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x20, 0x7b, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x6d, 0x61, 0x72, 0x67, 0x69, 0x6e, + 0x3a, 0x20, 0x30, 0x65, 0x6d, 0x20, 0x61, 0x75, 0x74, 0x6f, 0x3b, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x64, 0x69, 0x73, 0x70, 0x6c, 0x61, + 0x79, 0x3a, 0x20, 0x66, 0x6c, 0x65, 0x78, 0x3b, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x66, 0x6c, 0x65, 0x78, 0x2d, 0x64, 0x69, 0x72, 0x65, + 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x3a, 0x20, 0x63, 0x6f, 0x6c, 0x75, 0x6d, + 0x6e, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x6a, 0x75, 0x73, + 0x74, 0x69, 0x66, 0x79, 0x2d, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, + 0x3a, 0x20, 0x73, 0x70, 0x61, 0x63, 0x65, 0x2d, 0x62, 0x65, 0x74, 0x77, + 0x65, 0x65, 0x6e, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x68, + 0x65, 0x69, 0x67, 0x68, 0x74, 0x3a, 0x20, 0x31, 0x30, 0x30, 0x25, 0x3b, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x6d, 0x61, 0x69, 0x6e, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x6d, 0x61, 0x72, 0x67, 0x69, 0x6e, 0x3a, 0x20, 0x33, 0x70, 0x78, + 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x64, 0x69, 0x73, 0x70, + 0x6c, 0x61, 0x79, 0x3a, 0x20, 0x66, 0x6c, 0x65, 0x78, 0x3b, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x66, 0x6c, 0x65, 0x78, 0x2d, 0x64, 0x69, + 0x72, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x3a, 0x20, 0x63, 0x6f, 0x6c, + 0x75, 0x6d, 0x6e, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x6a, + 0x75, 0x73, 0x74, 0x69, 0x66, 0x79, 0x2d, 0x63, 0x6f, 0x6e, 0x74, 0x65, + 0x6e, 0x74, 0x3a, 0x20, 0x73, 0x70, 0x61, 0x63, 0x65, 0x2d, 0x62, 0x65, + 0x74, 0x77, 0x65, 0x65, 0x6e, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x67, 0x61, 0x70, 0x3a, 0x20, 0x31, 0x65, 0x6d, 0x3b, 0x0a, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x66, 0x6c, 0x65, 0x78, 0x2d, 0x67, + 0x72, 0x6f, 0x77, 0x3a, 0x20, 0x31, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x6f, 0x76, 0x65, 0x72, 0x66, 0x6c, 0x6f, 0x77, 0x2d, 0x79, + 0x3a, 0x20, 0x61, 0x75, 0x74, 0x6f, 0x3b, 0x0a, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x62, 0x6f, 0x72, 0x64, 0x65, 0x72, 0x3a, 0x20, 0x31, + 0x70, 0x78, 0x20, 0x73, 0x6f, 0x6c, 0x69, 0x64, 0x20, 0x23, 0x63, 0x63, + 0x63, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x62, 0x6f, 0x72, + 0x64, 0x65, 0x72, 0x2d, 0x72, 0x61, 0x64, 0x69, 0x75, 0x73, 0x3a, 0x20, + 0x35, 0x70, 0x78, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x70, + 0x61, 0x64, 0x64, 0x69, 0x6e, 0x67, 0x3a, 0x20, 0x30, 0x2e, 0x35, 0x65, + 0x6d, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x62, 0x6f, 0x64, 0x79, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x6d, 0x61, 0x78, 0x2d, 0x77, 0x69, 0x64, 0x74, 0x68, + 0x3a, 0x20, 0x36, 0x30, 0x30, 0x70, 0x78, 0x3b, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x6d, 0x69, 0x6e, 0x2d, 0x77, 0x69, 0x64, 0x74, 0x68, + 0x3a, 0x20, 0x33, 0x30, 0x30, 0x70, 0x78, 0x3b, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x6c, 0x69, 0x6e, 0x65, 0x2d, 0x68, 0x65, 0x69, 0x67, + 0x68, 0x74, 0x3a, 0x20, 0x31, 0x2e, 0x32, 0x3b, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x6d, 0x61, 0x72, 0x67, 0x69, 0x6e, 0x3a, 0x20, 0x30, + 0x20, 0x61, 0x75, 0x74, 0x6f, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x70, 0x61, 0x64, 0x64, 0x69, 0x6e, 0x67, 0x3a, 0x20, 0x30, 0x20, + 0x30, 0x2e, 0x35, 0x65, 0x6d, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x7d, + 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x70, 0x20, 0x7b, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x6f, 0x76, 0x65, 0x72, 0x66, 0x6c, 0x6f, 0x77, + 0x2d, 0x77, 0x72, 0x61, 0x70, 0x3a, 0x20, 0x62, 0x72, 0x65, 0x61, 0x6b, + 0x2d, 0x77, 0x6f, 0x72, 0x64, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x77, 0x6f, 0x72, 0x64, 0x2d, 0x77, 0x72, 0x61, 0x70, 0x3a, 0x20, + 0x62, 0x72, 0x65, 0x61, 0x6b, 0x2d, 0x77, 0x6f, 0x72, 0x64, 0x3b, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x68, 0x79, 0x70, 0x68, 0x65, 0x6e, + 0x73, 0x3a, 0x20, 0x61, 0x75, 0x74, 0x6f, 0x3b, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x6d, 0x61, 0x72, 0x67, 0x69, 0x6e, 0x2d, 0x74, 0x6f, + 0x70, 0x3a, 0x20, 0x30, 0x2e, 0x35, 0x65, 0x6d, 0x3b, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x6d, 0x61, 0x72, 0x67, 0x69, 0x6e, 0x2d, 0x62, + 0x6f, 0x74, 0x74, 0x6f, 0x6d, 0x3a, 0x20, 0x30, 0x2e, 0x35, 0x65, 0x6d, + 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x23, 0x77, 0x72, 0x69, 0x74, 0x65, 0x20, 0x66, 0x6f, 0x72, 0x6d, + 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x6d, 0x61, 0x72, + 0x67, 0x69, 0x6e, 0x3a, 0x20, 0x31, 0x65, 0x6d, 0x20, 0x30, 0x20, 0x30, + 0x20, 0x30, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x64, 0x69, + 0x73, 0x70, 0x6c, 0x61, 0x79, 0x3a, 0x20, 0x66, 0x6c, 0x65, 0x78, 0x3b, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x66, 0x6c, 0x65, 0x78, 0x2d, + 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x3a, 0x20, 0x63, + 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x67, 0x61, 0x70, 0x3a, 0x20, 0x30, 0x2e, 0x35, 0x65, 0x6d, 0x3b, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x61, 0x6c, 0x69, 0x67, 0x6e, + 0x2d, 0x69, 0x74, 0x65, 0x6d, 0x73, 0x3a, 0x20, 0x73, 0x74, 0x72, 0x65, + 0x74, 0x63, 0x68, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x2e, 0x72, 0x69, 0x67, 0x68, 0x74, 0x20, 0x7b, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x64, 0x69, 0x73, 0x70, 0x6c, + 0x61, 0x79, 0x3a, 0x20, 0x66, 0x6c, 0x65, 0x78, 0x3b, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x66, 0x6c, 0x65, 0x78, 0x2d, 0x64, 0x69, 0x72, + 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x3a, 0x20, 0x72, 0x6f, 0x77, 0x3b, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x67, 0x61, 0x70, 0x3a, 0x20, + 0x30, 0x2e, 0x35, 0x65, 0x6d, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x6a, 0x75, 0x73, 0x74, 0x69, 0x66, 0x79, 0x2d, 0x63, 0x6f, 0x6e, + 0x74, 0x65, 0x6e, 0x74, 0x3a, 0x20, 0x66, 0x6c, 0x65, 0x78, 0x2d, 0x65, + 0x6e, 0x64, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x65, 0x74, 0x20, + 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x62, 0x6f, 0x72, 0x64, + 0x65, 0x72, 0x3a, 0x20, 0x6e, 0x6f, 0x6e, 0x65, 0x3b, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x70, 0x61, 0x64, 0x64, 0x69, 0x6e, 0x67, 0x3a, + 0x20, 0x30, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x6d, 0x61, + 0x72, 0x67, 0x69, 0x6e, 0x3a, 0x20, 0x30, 0x3b, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x7d, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x66, 0x69, 0x65, 0x6c, + 0x64, 0x73, 0x65, 0x74, 0x2e, 0x74, 0x77, 0x6f, 0x20, 0x7b, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x64, 0x69, 0x73, 0x70, 0x6c, 0x61, 0x79, + 0x3a, 0x20, 0x67, 0x72, 0x69, 0x64, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x67, 0x72, 0x69, 0x64, 0x2d, 0x74, 0x65, 0x6d, 0x70, 0x6c, + 0x61, 0x74, 0x65, 0x3a, 0x20, 0x22, 0x61, 0x20, 0x61, 0x22, 0x3b, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x67, 0x61, 0x70, 0x3a, 0x20, 0x31, + 0x65, 0x6d, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x65, 0x74, 0x2e, + 0x74, 0x68, 0x72, 0x65, 0x65, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x64, 0x69, 0x73, 0x70, 0x6c, 0x61, 0x79, 0x3a, 0x20, 0x67, + 0x72, 0x69, 0x64, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x67, + 0x72, 0x69, 0x64, 0x2d, 0x74, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, + 0x3a, 0x20, 0x22, 0x61, 0x20, 0x61, 0x20, 0x61, 0x22, 0x3b, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x67, 0x61, 0x70, 0x3a, 0x20, 0x31, 0x65, + 0x6d, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x64, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x73, 0x20, 0x7b, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x62, 0x6f, 0x72, 0x64, 0x65, 0x72, + 0x3a, 0x20, 0x31, 0x70, 0x78, 0x20, 0x73, 0x6f, 0x6c, 0x69, 0x64, 0x20, + 0x23, 0x61, 0x61, 0x61, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x62, 0x6f, 0x72, 0x64, 0x65, 0x72, 0x2d, 0x72, 0x61, 0x64, 0x69, 0x75, + 0x73, 0x3a, 0x20, 0x34, 0x70, 0x78, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x70, 0x61, 0x64, 0x64, 0x69, 0x6e, 0x67, 0x3a, 0x20, 0x30, + 0x2e, 0x35, 0x65, 0x6d, 0x20, 0x30, 0x2e, 0x35, 0x65, 0x6d, 0x20, 0x30, + 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x6d, 0x61, 0x72, 0x67, + 0x69, 0x6e, 0x2d, 0x74, 0x6f, 0x70, 0x3a, 0x20, 0x30, 0x2e, 0x35, 0x65, + 0x6d, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x73, 0x75, 0x6d, 0x6d, 0x61, 0x72, 0x79, 0x20, 0x7b, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x66, 0x6f, 0x6e, 0x74, 0x2d, 0x77, + 0x65, 0x69, 0x67, 0x68, 0x74, 0x3a, 0x20, 0x62, 0x6f, 0x6c, 0x64, 0x3b, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x6d, 0x61, 0x72, 0x67, 0x69, + 0x6e, 0x3a, 0x20, 0x2d, 0x30, 0x2e, 0x35, 0x65, 0x6d, 0x20, 0x2d, 0x30, + 0x2e, 0x35, 0x65, 0x6d, 0x20, 0x30, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x70, 0x61, 0x64, 0x64, 0x69, 0x6e, 0x67, 0x3a, 0x20, 0x30, + 0x2e, 0x35, 0x65, 0x6d, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x63, 0x75, 0x72, 0x73, 0x6f, 0x72, 0x3a, 0x20, 0x70, 0x6f, 0x69, 0x6e, + 0x74, 0x65, 0x72, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x64, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x73, 0x5b, + 0x6f, 0x70, 0x65, 0x6e, 0x5d, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x70, 0x61, 0x64, 0x64, 0x69, 0x6e, 0x67, 0x3a, 0x20, 0x30, + 0x2e, 0x35, 0x65, 0x6d, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x2e, 0x70, 0x72, 0x6f, 0x62, 0x2d, 0x73, + 0x65, 0x74, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x70, + 0x61, 0x64, 0x64, 0x69, 0x6e, 0x67, 0x3a, 0x20, 0x30, 0x2e, 0x33, 0x65, + 0x6d, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x62, 0x6f, 0x72, + 0x64, 0x65, 0x72, 0x2d, 0x62, 0x6f, 0x74, 0x74, 0x6f, 0x6d, 0x3a, 0x20, + 0x31, 0x70, 0x78, 0x20, 0x73, 0x6f, 0x6c, 0x69, 0x64, 0x20, 0x23, 0x63, + 0x63, 0x63, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x2e, 0x70, 0x6f, 0x70, 0x6f, 0x76, 0x65, 0x72, 0x2d, + 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x20, 0x7b, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x69, 0x6f, 0x6e, + 0x3a, 0x20, 0x61, 0x62, 0x73, 0x6f, 0x6c, 0x75, 0x74, 0x65, 0x3b, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x62, 0x61, 0x63, 0x6b, 0x67, 0x72, + 0x6f, 0x75, 0x6e, 0x64, 0x2d, 0x63, 0x6f, 0x6c, 0x6f, 0x72, 0x3a, 0x20, + 0x77, 0x68, 0x69, 0x74, 0x65, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x70, 0x61, 0x64, 0x64, 0x69, 0x6e, 0x67, 0x3a, 0x20, 0x30, 0x2e, + 0x32, 0x65, 0x6d, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x62, + 0x6f, 0x78, 0x2d, 0x73, 0x68, 0x61, 0x64, 0x6f, 0x77, 0x3a, 0x20, 0x30, + 0x20, 0x30, 0x20, 0x31, 0x30, 0x70, 0x78, 0x20, 0x72, 0x67, 0x62, 0x61, + 0x28, 0x30, 0x2c, 0x20, 0x30, 0x2c, 0x20, 0x30, 0x2c, 0x20, 0x30, 0x2e, + 0x31, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x74, 0x65, 0x78, 0x74, 0x61, 0x72, 0x65, 0x61, 0x20, + 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x70, 0x61, 0x64, 0x64, + 0x69, 0x6e, 0x67, 0x3a, 0x20, 0x35, 0x70, 0x78, 0x3b, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x66, 0x6c, 0x65, 0x78, 0x2d, 0x67, 0x72, 0x6f, + 0x77, 0x3a, 0x20, 0x31, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x77, 0x69, 0x64, 0x74, 0x68, 0x3a, 0x20, 0x31, 0x30, 0x30, 0x25, 0x3b, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x70, 0x72, 0x65, 0x20, 0x63, 0x6f, 0x64, 0x65, 0x20, 0x7b, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x64, 0x69, 0x73, 0x70, 0x6c, 0x61, 0x79, + 0x3a, 0x20, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x3b, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x62, 0x61, 0x63, 0x6b, 0x67, 0x72, 0x6f, 0x75, 0x6e, + 0x64, 0x2d, 0x63, 0x6f, 0x6c, 0x6f, 0x72, 0x3a, 0x20, 0x23, 0x32, 0x32, + 0x32, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6c, + 0x6f, 0x72, 0x3a, 0x20, 0x23, 0x64, 0x64, 0x64, 0x3b, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x7d, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x64, + 0x65, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x66, 0x6f, + 0x6e, 0x74, 0x2d, 0x66, 0x61, 0x6d, 0x69, 0x6c, 0x79, 0x3a, 0x20, 0x6d, + 0x6f, 0x6e, 0x6f, 0x73, 0x70, 0x61, 0x63, 0x65, 0x3b, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x70, 0x61, 0x64, 0x64, 0x69, 0x6e, 0x67, 0x3a, + 0x20, 0x30, 0x2e, 0x31, 0x65, 0x6d, 0x20, 0x30, 0x2e, 0x33, 0x65, 0x6d, + 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x62, 0x6f, 0x72, 0x64, + 0x65, 0x72, 0x2d, 0x72, 0x61, 0x64, 0x69, 0x75, 0x73, 0x3a, 0x20, 0x33, + 0x70, 0x78, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x65, 0x74, 0x20, + 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x6d, 0x61, 0x72, 0x67, 0x69, 0x6e, 0x3a, 0x20, 0x30, 0x2e, + 0x35, 0x65, 0x6d, 0x20, 0x30, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x64, 0x69, 0x73, 0x70, 0x6c, 0x61, 0x79, 0x3a, 0x20, 0x62, 0x6c, + 0x6f, 0x63, 0x6b, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x65, 0x74, + 0x20, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x2e, 0x73, 0x6c, 0x69, 0x6d, 0x20, + 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x6d, 0x61, 0x72, 0x67, + 0x69, 0x6e, 0x3a, 0x20, 0x30, 0x20, 0x30, 0x2e, 0x35, 0x65, 0x6d, 0x3b, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x64, 0x69, 0x73, 0x70, 0x6c, + 0x61, 0x79, 0x3a, 0x20, 0x69, 0x6e, 0x6c, 0x69, 0x6e, 0x65, 0x3b, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x68, + 0x65, 0x61, 0x64, 0x65, 0x72, 0x2c, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x66, + 0x6f, 0x6f, 0x74, 0x65, 0x72, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x74, 0x65, 0x78, 0x74, 0x2d, 0x61, 0x6c, 0x69, 0x67, 0x6e, + 0x3a, 0x20, 0x63, 0x65, 0x6e, 0x74, 0x65, 0x72, 0x3b, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x7d, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x66, 0x6f, 0x6f, + 0x74, 0x65, 0x72, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x66, 0x6f, 0x6e, 0x74, 0x2d, 0x73, 0x69, 0x7a, 0x65, 0x3a, 0x20, 0x38, + 0x30, 0x25, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, + 0x6c, 0x6f, 0x72, 0x3a, 0x20, 0x23, 0x38, 0x38, 0x38, 0x3b, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x2e, 0x6d, + 0x6f, 0x64, 0x65, 0x2d, 0x63, 0x68, 0x61, 0x74, 0x20, 0x74, 0x65, 0x78, + 0x74, 0x61, 0x72, 0x65, 0x61, 0x5b, 0x6e, 0x61, 0x6d, 0x65, 0x3d, 0x70, + 0x72, 0x6f, 0x6d, 0x70, 0x74, 0x5d, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x68, 0x65, 0x69, 0x67, 0x68, 0x74, 0x3a, 0x20, 0x34, + 0x2e, 0x35, 0x65, 0x6d, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x2d, 0x63, + 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x74, 0x65, + 0x78, 0x74, 0x61, 0x72, 0x65, 0x61, 0x5b, 0x6e, 0x61, 0x6d, 0x65, 0x3d, + 0x70, 0x72, 0x6f, 0x6d, 0x70, 0x74, 0x5d, 0x20, 0x7b, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x68, 0x65, 0x69, 0x67, 0x68, 0x74, 0x3a, 0x20, + 0x31, 0x30, 0x65, 0x6d, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x5b, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, + 0x74, 0x65, 0x64, 0x69, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x5d, 0x20, 0x7b, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x64, 0x69, 0x73, 0x70, 0x6c, + 0x61, 0x79, 0x3a, 0x20, 0x69, 0x6e, 0x6c, 0x69, 0x6e, 0x65, 0x2d, 0x62, + 0x6c, 0x6f, 0x63, 0x6b, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x77, 0x68, 0x69, 0x74, 0x65, 0x2d, 0x73, 0x70, 0x61, 0x63, 0x65, 0x3a, + 0x20, 0x70, 0x72, 0x65, 0x2d, 0x77, 0x72, 0x61, 0x70, 0x3b, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x6f, 0x75, 0x74, 0x6c, 0x69, 0x6e, 0x65, + 0x3a, 0x20, 0x30, 0x70, 0x78, 0x20, 0x73, 0x6f, 0x6c, 0x69, 0x64, 0x20, + 0x74, 0x72, 0x61, 0x6e, 0x73, 0x70, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x3b, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x40, 0x6b, 0x65, 0x79, 0x66, 0x72, 0x61, 0x6d, 0x65, 0x73, 0x20, 0x6c, + 0x6f, 0x61, 0x64, 0x69, 0x6e, 0x67, 0x2d, 0x62, 0x67, 0x2d, 0x77, 0x69, + 0x70, 0x65, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x30, + 0x25, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x62, 0x61, 0x63, 0x6b, 0x67, 0x72, 0x6f, 0x75, 0x6e, 0x64, 0x2d, 0x70, + 0x6f, 0x73, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x3a, 0x20, 0x30, 0x25, 0x3b, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x31, 0x30, 0x30, 0x25, 0x20, 0x7b, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x62, 0x61, 0x63, 0x6b, 0x67, + 0x72, 0x6f, 0x75, 0x6e, 0x64, 0x2d, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x69, + 0x6f, 0x6e, 0x3a, 0x20, 0x31, 0x30, 0x30, 0x25, 0x3b, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x2e, 0x6c, 0x6f, 0x61, 0x64, 0x69, 0x6e, + 0x67, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x2d, 0x2d, + 0x6c, 0x6f, 0x61, 0x64, 0x69, 0x6e, 0x67, 0x2d, 0x63, 0x6f, 0x6c, 0x6f, + 0x72, 0x2d, 0x31, 0x3a, 0x20, 0x23, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, + 0x30, 0x30, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x2d, 0x2d, + 0x6c, 0x6f, 0x61, 0x64, 0x69, 0x6e, 0x67, 0x2d, 0x63, 0x6f, 0x6c, 0x6f, + 0x72, 0x2d, 0x32, 0x3a, 0x20, 0x23, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, + 0x66, 0x66, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x62, 0x61, + 0x63, 0x6b, 0x67, 0x72, 0x6f, 0x75, 0x6e, 0x64, 0x2d, 0x73, 0x69, 0x7a, + 0x65, 0x3a, 0x20, 0x35, 0x30, 0x25, 0x20, 0x31, 0x30, 0x30, 0x25, 0x3b, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x62, 0x61, 0x63, 0x6b, 0x67, + 0x72, 0x6f, 0x75, 0x6e, 0x64, 0x2d, 0x69, 0x6d, 0x61, 0x67, 0x65, 0x3a, + 0x20, 0x6c, 0x69, 0x6e, 0x65, 0x61, 0x72, 0x2d, 0x67, 0x72, 0x61, 0x64, + 0x69, 0x65, 0x6e, 0x74, 0x28, 0x39, 0x30, 0x64, 0x65, 0x67, 0x2c, 0x20, + 0x76, 0x61, 0x72, 0x28, 0x2d, 0x2d, 0x6c, 0x6f, 0x61, 0x64, 0x69, 0x6e, + 0x67, 0x2d, 0x63, 0x6f, 0x6c, 0x6f, 0x72, 0x2d, 0x31, 0x29, 0x2c, 0x20, + 0x76, 0x61, 0x72, 0x28, 0x2d, 0x2d, 0x6c, 0x6f, 0x61, 0x64, 0x69, 0x6e, + 0x67, 0x2d, 0x63, 0x6f, 0x6c, 0x6f, 0x72, 0x2d, 0x32, 0x29, 0x2c, 0x20, + 0x76, 0x61, 0x72, 0x28, 0x2d, 0x2d, 0x6c, 0x6f, 0x61, 0x64, 0x69, 0x6e, + 0x67, 0x2d, 0x63, 0x6f, 0x6c, 0x6f, 0x72, 0x2d, 0x31, 0x29, 0x29, 0x3b, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x61, 0x6e, 0x69, 0x6d, 0x61, + 0x74, 0x69, 0x6f, 0x6e, 0x3a, 0x20, 0x6c, 0x6f, 0x61, 0x64, 0x69, 0x6e, + 0x67, 0x2d, 0x62, 0x67, 0x2d, 0x77, 0x69, 0x70, 0x65, 0x20, 0x32, 0x73, + 0x20, 0x6c, 0x69, 0x6e, 0x65, 0x61, 0x72, 0x20, 0x69, 0x6e, 0x66, 0x69, + 0x6e, 0x69, 0x74, 0x65, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x40, 0x6d, 0x65, 0x64, 0x69, 0x61, 0x20, + 0x28, 0x70, 0x72, 0x65, 0x66, 0x65, 0x72, 0x73, 0x2d, 0x63, 0x6f, 0x6c, + 0x6f, 0x72, 0x2d, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x65, 0x3a, 0x20, 0x64, + 0x61, 0x72, 0x6b, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x2e, 0x6c, 0x6f, 0x61, 0x64, 0x69, 0x6e, 0x67, 0x20, 0x7b, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x2d, 0x2d, 0x6c, 0x6f, + 0x61, 0x64, 0x69, 0x6e, 0x67, 0x2d, 0x63, 0x6f, 0x6c, 0x6f, 0x72, 0x2d, + 0x31, 0x3a, 0x20, 0x23, 0x32, 0x32, 0x32, 0x32, 0x32, 0x32, 0x30, 0x30, + 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x2d, 0x2d, + 0x6c, 0x6f, 0x61, 0x64, 0x69, 0x6e, 0x67, 0x2d, 0x63, 0x6f, 0x6c, 0x6f, + 0x72, 0x2d, 0x32, 0x3a, 0x20, 0x23, 0x32, 0x32, 0x32, 0x32, 0x32, 0x32, + 0x66, 0x66, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x2e, 0x70, 0x6f, 0x70, 0x6f, + 0x76, 0x65, 0x72, 0x2d, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x20, + 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x62, 0x61, + 0x63, 0x6b, 0x67, 0x72, 0x6f, 0x75, 0x6e, 0x64, 0x2d, 0x63, 0x6f, 0x6c, + 0x6f, 0x72, 0x3a, 0x20, 0x62, 0x6c, 0x61, 0x63, 0x6b, 0x3b, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x7d, + 0x0a, 0x20, 0x20, 0x3c, 0x2f, 0x73, 0x74, 0x79, 0x6c, 0x65, 0x3e, 0x0a, + 0x0a, 0x20, 0x20, 0x3c, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x20, 0x74, + 0x79, 0x70, 0x65, 0x3d, 0x22, 0x6d, 0x6f, 0x64, 0x75, 0x6c, 0x65, 0x22, + 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x69, 0x6d, 0x70, 0x6f, 0x72, 0x74, + 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x68, 0x74, 0x6d, + 0x6c, 0x2c, 0x20, 0x68, 0x2c, 0x20, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x6c, + 0x2c, 0x20, 0x65, 0x66, 0x66, 0x65, 0x63, 0x74, 0x2c, 0x20, 0x63, 0x6f, + 0x6d, 0x70, 0x75, 0x74, 0x65, 0x64, 0x2c, 0x20, 0x72, 0x65, 0x6e, 0x64, + 0x65, 0x72, 0x2c, 0x20, 0x75, 0x73, 0x65, 0x53, 0x69, 0x67, 0x6e, 0x61, + 0x6c, 0x2c, 0x20, 0x75, 0x73, 0x65, 0x45, 0x66, 0x66, 0x65, 0x63, 0x74, + 0x2c, 0x20, 0x75, 0x73, 0x65, 0x52, 0x65, 0x66, 0x2c, 0x20, 0x43, 0x6f, + 0x6d, 0x70, 0x6f, 0x6e, 0x65, 0x6e, 0x74, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x7d, 0x20, 0x66, 0x72, 0x6f, 0x6d, 0x20, 0x27, 0x2f, 0x69, 0x6e, 0x64, + 0x65, 0x78, 0x2e, 0x6a, 0x73, 0x27, 0x3b, 0x0a, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x69, 0x6d, 0x70, 0x6f, 0x72, 0x74, 0x20, 0x7b, 0x20, 0x6c, 0x6c, + 0x61, 0x6d, 0x61, 0x20, 0x7d, 0x20, 0x66, 0x72, 0x6f, 0x6d, 0x20, 0x27, + 0x2f, 0x63, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x69, 0x6f, 0x6e, 0x2e, + 0x6a, 0x73, 0x27, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x69, 0x6d, 0x70, + 0x6f, 0x72, 0x74, 0x20, 0x7b, 0x20, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, + 0x43, 0x6f, 0x6e, 0x76, 0x65, 0x72, 0x74, 0x65, 0x72, 0x20, 0x7d, 0x20, + 0x66, 0x72, 0x6f, 0x6d, 0x20, 0x27, 0x2f, 0x6a, 0x73, 0x6f, 0x6e, 0x2d, + 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2d, 0x74, 0x6f, 0x2d, 0x67, 0x72, + 0x61, 0x6d, 0x6d, 0x61, 0x72, 0x2e, 0x6d, 0x6a, 0x73, 0x27, 0x3b, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x6c, 0x65, 0x74, 0x20, 0x73, 0x65, 0x6c, 0x65, + 0x63, 0x74, 0x65, 0x64, 0x5f, 0x69, 0x6d, 0x61, 0x67, 0x65, 0x20, 0x3d, + 0x20, 0x66, 0x61, 0x6c, 0x73, 0x65, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x76, 0x61, 0x72, 0x20, 0x73, 0x6c, 0x6f, 0x74, 0x5f, 0x69, 0x64, 0x20, + 0x3d, 0x20, 0x2d, 0x31, 0x3b, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x63, + 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, + 0x20, 0x3d, 0x20, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x28, 0x7b, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x70, 0x72, 0x6f, 0x6d, 0x70, 0x74, + 0x3a, 0x20, 0x22, 0x54, 0x68, 0x69, 0x73, 0x20, 0x69, 0x73, 0x20, 0x61, + 0x20, 0x63, 0x6f, 0x6e, 0x76, 0x65, 0x72, 0x73, 0x61, 0x74, 0x69, 0x6f, + 0x6e, 0x20, 0x62, 0x65, 0x74, 0x77, 0x65, 0x65, 0x6e, 0x20, 0x55, 0x73, + 0x65, 0x72, 0x20, 0x61, 0x6e, 0x64, 0x20, 0x4c, 0x6c, 0x61, 0x6d, 0x61, + 0x2c, 0x20, 0x61, 0x20, 0x66, 0x72, 0x69, 0x65, 0x6e, 0x64, 0x6c, 0x79, + 0x20, 0x63, 0x68, 0x61, 0x74, 0x62, 0x6f, 0x74, 0x2e, 0x20, 0x4c, 0x6c, + 0x61, 0x6d, 0x61, 0x20, 0x69, 0x73, 0x20, 0x68, 0x65, 0x6c, 0x70, 0x66, + 0x75, 0x6c, 0x2c, 0x20, 0x6b, 0x69, 0x6e, 0x64, 0x2c, 0x20, 0x68, 0x6f, + 0x6e, 0x65, 0x73, 0x74, 0x2c, 0x20, 0x67, 0x6f, 0x6f, 0x64, 0x20, 0x61, + 0x74, 0x20, 0x77, 0x72, 0x69, 0x74, 0x69, 0x6e, 0x67, 0x2c, 0x20, 0x61, + 0x6e, 0x64, 0x20, 0x6e, 0x65, 0x76, 0x65, 0x72, 0x20, 0x66, 0x61, 0x69, + 0x6c, 0x73, 0x20, 0x74, 0x6f, 0x20, 0x61, 0x6e, 0x73, 0x77, 0x65, 0x72, + 0x20, 0x61, 0x6e, 0x79, 0x20, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x73, 0x20, 0x69, 0x6d, 0x6d, 0x65, 0x64, 0x69, 0x61, 0x74, 0x65, 0x6c, + 0x79, 0x20, 0x61, 0x6e, 0x64, 0x20, 0x77, 0x69, 0x74, 0x68, 0x20, 0x70, + 0x72, 0x65, 0x63, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x2e, 0x22, 0x2c, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x74, 0x65, 0x6d, 0x70, 0x6c, 0x61, + 0x74, 0x65, 0x3a, 0x20, 0x22, 0x7b, 0x7b, 0x70, 0x72, 0x6f, 0x6d, 0x70, + 0x74, 0x7d, 0x7d, 0x5c, 0x6e, 0x5c, 0x6e, 0x7b, 0x7b, 0x68, 0x69, 0x73, + 0x74, 0x6f, 0x72, 0x79, 0x7d, 0x7d, 0x5c, 0x6e, 0x7b, 0x7b, 0x63, 0x68, + 0x61, 0x72, 0x7d, 0x7d, 0x3a, 0x22, 0x2c, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x68, 0x69, 0x73, 0x74, 0x6f, 0x72, 0x79, 0x54, 0x65, 0x6d, + 0x70, 0x6c, 0x61, 0x74, 0x65, 0x3a, 0x20, 0x22, 0x7b, 0x7b, 0x6e, 0x61, + 0x6d, 0x65, 0x7d, 0x7d, 0x3a, 0x20, 0x7b, 0x7b, 0x6d, 0x65, 0x73, 0x73, + 0x61, 0x67, 0x65, 0x7d, 0x7d, 0x22, 0x2c, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, + 0x3a, 0x20, 0x5b, 0x5d, 0x2c, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x74, 0x79, 0x70, 0x65, 0x3a, 0x20, 0x22, 0x63, 0x68, 0x61, 0x74, 0x22, + 0x2c, 0x20, 0x20, 0x2f, 0x2f, 0x20, 0x22, 0x63, 0x68, 0x61, 0x74, 0x22, + 0x20, 0x7c, 0x20, 0x22, 0x63, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x69, + 0x6f, 0x6e, 0x22, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x68, + 0x61, 0x72, 0x3a, 0x20, 0x22, 0x4c, 0x6c, 0x61, 0x6d, 0x61, 0x22, 0x2c, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x75, 0x73, 0x65, 0x72, 0x3a, + 0x20, 0x22, 0x55, 0x73, 0x65, 0x72, 0x22, 0x2c, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x69, 0x6d, 0x61, 0x67, 0x65, 0x5f, 0x73, 0x65, 0x6c, + 0x65, 0x63, 0x74, 0x65, 0x64, 0x3a, 0x20, 0x27, 0x27, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x7d, 0x29, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, + 0x6e, 0x73, 0x74, 0x20, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x20, 0x3d, + 0x20, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x28, 0x7b, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x6e, 0x5f, 0x70, 0x72, 0x65, 0x64, 0x69, 0x63, + 0x74, 0x3a, 0x20, 0x34, 0x30, 0x30, 0x2c, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x74, 0x65, 0x6d, 0x70, 0x65, 0x72, 0x61, 0x74, 0x75, 0x72, + 0x65, 0x3a, 0x20, 0x30, 0x2e, 0x37, 0x2c, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x72, 0x65, 0x70, 0x65, 0x61, 0x74, 0x5f, 0x6c, 0x61, 0x73, + 0x74, 0x5f, 0x6e, 0x3a, 0x20, 0x32, 0x35, 0x36, 0x2c, 0x20, 0x2f, 0x2f, + 0x20, 0x30, 0x20, 0x3d, 0x20, 0x64, 0x69, 0x73, 0x61, 0x62, 0x6c, 0x65, + 0x20, 0x70, 0x65, 0x6e, 0x61, 0x6c, 0x74, 0x79, 0x2c, 0x20, 0x2d, 0x31, + 0x20, 0x3d, 0x20, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x20, 0x73, + 0x69, 0x7a, 0x65, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, + 0x70, 0x65, 0x61, 0x74, 0x5f, 0x70, 0x65, 0x6e, 0x61, 0x6c, 0x74, 0x79, + 0x3a, 0x20, 0x31, 0x2e, 0x31, 0x38, 0x2c, 0x20, 0x2f, 0x2f, 0x20, 0x31, + 0x2e, 0x30, 0x20, 0x3d, 0x20, 0x64, 0x69, 0x73, 0x61, 0x62, 0x6c, 0x65, + 0x64, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x74, 0x6f, 0x70, 0x5f, + 0x6b, 0x3a, 0x20, 0x34, 0x30, 0x2c, 0x20, 0x2f, 0x2f, 0x20, 0x3c, 0x3d, + 0x20, 0x30, 0x20, 0x74, 0x6f, 0x20, 0x75, 0x73, 0x65, 0x20, 0x76, 0x6f, + 0x63, 0x61, 0x62, 0x20, 0x73, 0x69, 0x7a, 0x65, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x74, 0x6f, 0x70, 0x5f, 0x70, 0x3a, 0x20, 0x30, 0x2e, + 0x39, 0x35, 0x2c, 0x20, 0x2f, 0x2f, 0x20, 0x31, 0x2e, 0x30, 0x20, 0x3d, + 0x20, 0x64, 0x69, 0x73, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x6d, 0x69, 0x6e, 0x5f, 0x70, 0x3a, 0x20, 0x30, + 0x2e, 0x30, 0x35, 0x2c, 0x20, 0x2f, 0x2f, 0x20, 0x30, 0x20, 0x3d, 0x20, + 0x64, 0x69, 0x73, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x74, 0x66, 0x73, 0x5f, 0x7a, 0x3a, 0x20, 0x31, 0x2e, + 0x30, 0x2c, 0x20, 0x2f, 0x2f, 0x20, 0x31, 0x2e, 0x30, 0x20, 0x3d, 0x20, + 0x64, 0x69, 0x73, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x74, 0x79, 0x70, 0x69, 0x63, 0x61, 0x6c, 0x5f, 0x70, + 0x3a, 0x20, 0x31, 0x2e, 0x30, 0x2c, 0x20, 0x2f, 0x2f, 0x20, 0x31, 0x2e, + 0x30, 0x20, 0x3d, 0x20, 0x64, 0x69, 0x73, 0x61, 0x62, 0x6c, 0x65, 0x64, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x70, 0x72, 0x65, 0x73, 0x65, + 0x6e, 0x63, 0x65, 0x5f, 0x70, 0x65, 0x6e, 0x61, 0x6c, 0x74, 0x79, 0x3a, + 0x20, 0x30, 0x2e, 0x30, 0x2c, 0x20, 0x2f, 0x2f, 0x20, 0x30, 0x2e, 0x30, + 0x20, 0x3d, 0x20, 0x64, 0x69, 0x73, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x66, 0x72, 0x65, 0x71, 0x75, 0x65, + 0x6e, 0x63, 0x79, 0x5f, 0x70, 0x65, 0x6e, 0x61, 0x6c, 0x74, 0x79, 0x3a, + 0x20, 0x30, 0x2e, 0x30, 0x2c, 0x20, 0x2f, 0x2f, 0x20, 0x30, 0x2e, 0x30, + 0x20, 0x3d, 0x20, 0x64, 0x69, 0x73, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x6d, 0x69, 0x72, 0x6f, 0x73, 0x74, + 0x61, 0x74, 0x3a, 0x20, 0x30, 0x2c, 0x20, 0x2f, 0x2f, 0x20, 0x30, 0x2f, + 0x31, 0x2f, 0x32, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x6d, 0x69, + 0x72, 0x6f, 0x73, 0x74, 0x61, 0x74, 0x5f, 0x74, 0x61, 0x75, 0x3a, 0x20, + 0x35, 0x2c, 0x20, 0x2f, 0x2f, 0x20, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, + 0x20, 0x65, 0x6e, 0x74, 0x72, 0x6f, 0x70, 0x79, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x6d, 0x69, 0x72, 0x6f, 0x73, 0x74, 0x61, 0x74, 0x5f, + 0x65, 0x74, 0x61, 0x3a, 0x20, 0x30, 0x2e, 0x31, 0x2c, 0x20, 0x2f, 0x2f, + 0x20, 0x6c, 0x65, 0x61, 0x72, 0x6e, 0x69, 0x6e, 0x67, 0x20, 0x72, 0x61, + 0x74, 0x65, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x67, 0x72, 0x61, + 0x6d, 0x6d, 0x61, 0x72, 0x3a, 0x20, 0x27, 0x27, 0x2c, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x6e, 0x5f, 0x70, 0x72, 0x6f, 0x62, 0x73, 0x3a, + 0x20, 0x30, 0x2c, 0x20, 0x2f, 0x2f, 0x20, 0x6e, 0x6f, 0x20, 0x63, 0x6f, + 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x70, 0x72, 0x6f, + 0x62, 0x61, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x69, 0x65, 0x73, 0x2c, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x6d, 0x61, 0x67, 0x65, 0x5f, + 0x64, 0x61, 0x74, 0x61, 0x3a, 0x20, 0x5b, 0x5d, 0x2c, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x63, 0x61, 0x63, 0x68, 0x65, 0x5f, 0x70, 0x72, + 0x6f, 0x6d, 0x70, 0x74, 0x3a, 0x20, 0x74, 0x72, 0x75, 0x65, 0x2c, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x61, 0x70, 0x69, 0x5f, 0x6b, 0x65, + 0x79, 0x3a, 0x20, 0x27, 0x27, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x29, + 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x2f, 0x2a, 0x20, 0x53, 0x54, 0x41, + 0x52, 0x54, 0x3a, 0x20, 0x53, 0x75, 0x70, 0x70, 0x6f, 0x72, 0x74, 0x20, + 0x66, 0x6f, 0x72, 0x20, 0x73, 0x74, 0x6f, 0x72, 0x69, 0x6e, 0x67, 0x20, + 0x70, 0x72, 0x6f, 0x6d, 0x70, 0x74, 0x20, 0x74, 0x65, 0x6d, 0x70, 0x6c, + 0x61, 0x74, 0x65, 0x73, 0x20, 0x61, 0x6e, 0x64, 0x20, 0x70, 0x61, 0x72, + 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x73, 0x20, 0x69, 0x6e, 0x20, 0x62, + 0x72, 0x6f, 0x77, 0x73, 0x65, 0x72, 0x73, 0x20, 0x4c, 0x6f, 0x63, 0x61, + 0x6c, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x20, 0x2a, 0x2f, 0x0a, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x6c, + 0x6f, 0x63, 0x61, 0x6c, 0x5f, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, + 0x5f, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x4b, 0x65, 0x79, 0x20, + 0x3d, 0x20, 0x22, 0x6c, 0x6c, 0x61, 0x6d, 0x61, 0x63, 0x70, 0x70, 0x5f, + 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x6c, 0x6f, 0x63, 0x61, 0x6c, + 0x5f, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x22, 0x3b, 0x0a, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, + 0x20, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x5f, 0x73, 0x74, 0x6f, 0x72, 0x61, + 0x67, 0x65, 0x5f, 0x73, 0x65, 0x74, 0x44, 0x61, 0x74, 0x61, 0x46, 0x72, + 0x6f, 0x6d, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x28, 0x74, 0x61, 0x67, + 0x2c, 0x20, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x29, 0x20, 0x7b, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x6c, 0x6f, 0x63, 0x61, 0x6c, + 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x2e, 0x73, 0x65, 0x74, 0x49, + 0x74, 0x65, 0x6d, 0x28, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x5f, 0x73, 0x74, + 0x6f, 0x72, 0x61, 0x67, 0x65, 0x5f, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, + 0x65, 0x4b, 0x65, 0x79, 0x20, 0x2b, 0x20, 0x27, 0x2f, 0x27, 0x20, 0x2b, + 0x20, 0x74, 0x61, 0x67, 0x2c, 0x20, 0x4a, 0x53, 0x4f, 0x4e, 0x2e, 0x73, + 0x74, 0x72, 0x69, 0x6e, 0x67, 0x69, 0x66, 0x79, 0x28, 0x63, 0x6f, 0x6e, + 0x74, 0x65, 0x6e, 0x74, 0x29, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x7d, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x66, 0x75, 0x6e, 0x63, 0x74, + 0x69, 0x6f, 0x6e, 0x20, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x5f, 0x73, 0x74, + 0x6f, 0x72, 0x61, 0x67, 0x65, 0x5f, 0x73, 0x65, 0x74, 0x44, 0x61, 0x74, + 0x61, 0x46, 0x72, 0x6f, 0x6d, 0x52, 0x61, 0x77, 0x54, 0x65, 0x78, 0x74, + 0x28, 0x74, 0x61, 0x67, 0x2c, 0x20, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, + 0x74, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x6c, + 0x6f, 0x63, 0x61, 0x6c, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x2e, + 0x73, 0x65, 0x74, 0x49, 0x74, 0x65, 0x6d, 0x28, 0x6c, 0x6f, 0x63, 0x61, + 0x6c, 0x5f, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x5f, 0x73, 0x74, + 0x6f, 0x72, 0x61, 0x67, 0x65, 0x4b, 0x65, 0x79, 0x20, 0x2b, 0x20, 0x27, + 0x2f, 0x27, 0x20, 0x2b, 0x20, 0x74, 0x61, 0x67, 0x2c, 0x20, 0x63, 0x6f, + 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x7d, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x66, 0x75, 0x6e, 0x63, 0x74, + 0x69, 0x6f, 0x6e, 0x20, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x5f, 0x73, 0x74, + 0x6f, 0x72, 0x61, 0x67, 0x65, 0x5f, 0x67, 0x65, 0x74, 0x44, 0x61, 0x74, + 0x61, 0x41, 0x73, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x28, 0x74, 0x61, + 0x67, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, + 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x69, 0x74, 0x65, 0x6d, 0x20, 0x3d, 0x20, + 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, + 0x2e, 0x67, 0x65, 0x74, 0x49, 0x74, 0x65, 0x6d, 0x28, 0x6c, 0x6f, 0x63, + 0x61, 0x6c, 0x5f, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x5f, 0x73, + 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x4b, 0x65, 0x79, 0x20, 0x2b, 0x20, + 0x27, 0x2f, 0x27, 0x20, 0x2b, 0x20, 0x74, 0x61, 0x67, 0x29, 0x3b, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, 0x21, 0x69, + 0x74, 0x65, 0x6d, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x6e, 0x75, + 0x6c, 0x6c, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x20, + 0x65, 0x6c, 0x73, 0x65, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x4a, 0x53, + 0x4f, 0x4e, 0x2e, 0x70, 0x61, 0x72, 0x73, 0x65, 0x28, 0x69, 0x74, 0x65, + 0x6d, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x66, + 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x6c, 0x6f, 0x63, 0x61, + 0x6c, 0x5f, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x5f, 0x67, 0x65, + 0x74, 0x44, 0x61, 0x74, 0x61, 0x41, 0x73, 0x52, 0x61, 0x77, 0x54, 0x65, + 0x78, 0x74, 0x28, 0x74, 0x61, 0x67, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x69, 0x74, + 0x65, 0x6d, 0x20, 0x3d, 0x20, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x53, 0x74, + 0x6f, 0x72, 0x61, 0x67, 0x65, 0x2e, 0x67, 0x65, 0x74, 0x49, 0x74, 0x65, + 0x6d, 0x28, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x5f, 0x73, 0x74, 0x6f, 0x72, + 0x61, 0x67, 0x65, 0x5f, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x4b, + 0x65, 0x79, 0x20, 0x2b, 0x20, 0x27, 0x2f, 0x27, 0x20, 0x2b, 0x20, 0x74, + 0x61, 0x67, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, + 0x66, 0x20, 0x28, 0x21, 0x69, 0x74, 0x65, 0x6d, 0x29, 0x20, 0x7b, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, + 0x72, 0x6e, 0x20, 0x6e, 0x75, 0x6c, 0x6c, 0x3b, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x7d, 0x20, 0x65, 0x6c, 0x73, 0x65, 0x20, 0x7b, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, + 0x72, 0x6e, 0x20, 0x69, 0x74, 0x65, 0x6d, 0x3b, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x2f, 0x2f, 0x20, 0x63, 0x72, 0x65, 0x61, 0x74, + 0x65, 0x20, 0x61, 0x20, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, + 0x72, 0x20, 0x66, 0x6f, 0x72, 0x20, 0x75, 0x73, 0x65, 0x72, 0x20, 0x74, + 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x73, 0x20, 0x61, 0x6e, 0x64, + 0x20, 0x73, 0x65, 0x74, 0x74, 0x69, 0x6e, 0x67, 0x73, 0x0a, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x73, 0x61, 0x76, + 0x65, 0x64, 0x55, 0x73, 0x65, 0x72, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, + 0x74, 0x65, 0x73, 0x20, 0x3d, 0x20, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x6c, + 0x28, 0x7b, 0x7d, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, + 0x73, 0x74, 0x20, 0x73, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x65, 0x64, 0x55, + 0x73, 0x65, 0x72, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x20, + 0x3d, 0x20, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x28, 0x7b, 0x20, 0x6e, + 0x61, 0x6d, 0x65, 0x3a, 0x20, 0x27, 0x27, 0x2c, 0x20, 0x74, 0x65, 0x6d, + 0x70, 0x6c, 0x61, 0x74, 0x65, 0x3a, 0x20, 0x7b, 0x20, 0x73, 0x65, 0x73, + 0x73, 0x69, 0x6f, 0x6e, 0x3a, 0x20, 0x7b, 0x7d, 0x2c, 0x20, 0x70, 0x61, + 0x72, 0x61, 0x6d, 0x73, 0x3a, 0x20, 0x7b, 0x7d, 0x20, 0x7d, 0x20, 0x7d, + 0x29, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x2f, 0x2f, 0x20, 0x6c, 0x65, + 0x74, 0x27, 0x73, 0x20, 0x69, 0x6d, 0x70, 0x6f, 0x72, 0x74, 0x20, 0x6c, + 0x6f, 0x63, 0x61, 0x6c, 0x6c, 0x79, 0x20, 0x73, 0x61, 0x76, 0x65, 0x64, + 0x20, 0x74, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x73, 0x20, 0x61, + 0x6e, 0x64, 0x20, 0x73, 0x65, 0x74, 0x74, 0x69, 0x6e, 0x67, 0x73, 0x20, + 0x69, 0x66, 0x20, 0x74, 0x68, 0x65, 0x72, 0x65, 0x20, 0x61, 0x72, 0x65, + 0x20, 0x61, 0x6e, 0x79, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x2f, 0x2f, 0x20, + 0x75, 0x73, 0x65, 0x72, 0x20, 0x74, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, + 0x65, 0x73, 0x20, 0x61, 0x6e, 0x64, 0x20, 0x73, 0x65, 0x74, 0x74, 0x69, + 0x6e, 0x67, 0x73, 0x20, 0x61, 0x72, 0x65, 0x20, 0x73, 0x74, 0x6f, 0x72, + 0x65, 0x64, 0x20, 0x69, 0x6e, 0x20, 0x6f, 0x6e, 0x65, 0x20, 0x6f, 0x62, + 0x6a, 0x65, 0x63, 0x74, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x2f, 0x2f, 0x20, + 0x69, 0x6e, 0x20, 0x66, 0x6f, 0x72, 0x6d, 0x20, 0x6f, 0x66, 0x20, 0x7b, + 0x20, 0x22, 0x74, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x6e, 0x61, + 0x6d, 0x65, 0x22, 0x3a, 0x20, 0x22, 0x74, 0x65, 0x6d, 0x70, 0x6c, 0x61, + 0x74, 0x65, 0x64, 0x61, 0x74, 0x61, 0x22, 0x20, 0x7d, 0x20, 0x61, 0x6e, + 0x64, 0x20, 0x7b, 0x20, 0x22, 0x73, 0x65, 0x74, 0x74, 0x69, 0x6e, 0x67, + 0x73, 0x74, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x6e, 0x61, 0x6d, + 0x65, 0x22, 0x3a, 0x22, 0x73, 0x65, 0x74, 0x74, 0x69, 0x6e, 0x67, 0x73, + 0x64, 0x61, 0x74, 0x61, 0x22, 0x20, 0x7d, 0x0a, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x6f, 0x6c, 0x65, 0x2e, 0x6c, 0x6f, 0x67, + 0x28, 0x27, 0x49, 0x6d, 0x70, 0x6f, 0x72, 0x74, 0x69, 0x6e, 0x67, 0x20, + 0x73, 0x61, 0x76, 0x65, 0x64, 0x20, 0x74, 0x65, 0x6d, 0x70, 0x6c, 0x61, + 0x74, 0x65, 0x73, 0x27, 0x29, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x6c, + 0x65, 0x74, 0x20, 0x69, 0x6d, 0x70, 0x6f, 0x72, 0x74, 0x65, 0x64, 0x54, + 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x73, 0x20, 0x3d, 0x20, 0x6c, + 0x6f, 0x63, 0x61, 0x6c, 0x5f, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, + 0x5f, 0x67, 0x65, 0x74, 0x44, 0x61, 0x74, 0x61, 0x41, 0x73, 0x4f, 0x62, + 0x6a, 0x65, 0x63, 0x74, 0x28, 0x27, 0x75, 0x73, 0x65, 0x72, 0x5f, 0x74, + 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x73, 0x27, 0x29, 0x0a, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, 0x69, 0x6d, 0x70, 0x6f, + 0x72, 0x74, 0x65, 0x64, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, + 0x73, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x2f, + 0x2f, 0x20, 0x73, 0x61, 0x76, 0x65, 0x64, 0x20, 0x74, 0x65, 0x6d, 0x70, + 0x6c, 0x61, 0x74, 0x65, 0x73, 0x20, 0x77, 0x65, 0x72, 0x65, 0x20, 0x73, + 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x66, 0x75, 0x6c, 0x6c, 0x79, 0x20, + 0x69, 0x6d, 0x70, 0x6f, 0x72, 0x74, 0x65, 0x64, 0x2e, 0x0a, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x6f, 0x6c, 0x65, + 0x2e, 0x6c, 0x6f, 0x67, 0x28, 0x27, 0x50, 0x72, 0x6f, 0x63, 0x65, 0x73, + 0x73, 0x69, 0x6e, 0x67, 0x20, 0x73, 0x61, 0x76, 0x65, 0x64, 0x20, 0x74, + 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x73, 0x20, 0x61, 0x6e, 0x64, + 0x20, 0x75, 0x70, 0x64, 0x61, 0x74, 0x69, 0x6e, 0x67, 0x20, 0x64, 0x65, + 0x66, 0x61, 0x75, 0x6c, 0x74, 0x20, 0x74, 0x65, 0x6d, 0x70, 0x6c, 0x61, + 0x74, 0x65, 0x27, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x70, + 0x61, 0x72, 0x61, 0x6d, 0x73, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x20, + 0x3d, 0x20, 0x7b, 0x20, 0x2e, 0x2e, 0x2e, 0x70, 0x61, 0x72, 0x61, 0x6d, + 0x73, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x2c, 0x20, 0x69, 0x6d, 0x61, + 0x67, 0x65, 0x5f, 0x64, 0x61, 0x74, 0x61, 0x3a, 0x20, 0x5b, 0x5d, 0x20, + 0x7d, 0x3b, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x2f, 0x2f, + 0x63, 0x6f, 0x6e, 0x73, 0x6f, 0x6c, 0x65, 0x2e, 0x6c, 0x6f, 0x67, 0x28, + 0x69, 0x6d, 0x70, 0x6f, 0x72, 0x74, 0x65, 0x64, 0x54, 0x65, 0x6d, 0x70, + 0x6c, 0x61, 0x74, 0x65, 0x73, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x73, 0x61, 0x76, 0x65, 0x64, 0x55, 0x73, 0x65, 0x72, 0x54, + 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x73, 0x2e, 0x76, 0x61, 0x6c, + 0x75, 0x65, 0x20, 0x3d, 0x20, 0x69, 0x6d, 0x70, 0x6f, 0x72, 0x74, 0x65, + 0x64, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x73, 0x3b, 0x0a, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x2f, 0x2f, 0x6f, 0x76, 0x65, + 0x72, 0x72, 0x69, 0x64, 0x65, 0x20, 0x64, 0x65, 0x66, 0x61, 0x75, 0x6c, + 0x74, 0x20, 0x74, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x73, 0x61, 0x76, 0x65, 0x64, 0x55, 0x73, + 0x65, 0x72, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x73, 0x2e, + 0x76, 0x61, 0x6c, 0x75, 0x65, 0x2e, 0x64, 0x65, 0x66, 0x61, 0x75, 0x6c, + 0x74, 0x20, 0x3d, 0x20, 0x7b, 0x20, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, + 0x6e, 0x3a, 0x20, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x2e, 0x76, + 0x61, 0x6c, 0x75, 0x65, 0x2c, 0x20, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, + 0x3a, 0x20, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x2e, 0x76, 0x61, 0x6c, + 0x75, 0x65, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x6c, + 0x6f, 0x63, 0x61, 0x6c, 0x5f, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, + 0x5f, 0x73, 0x65, 0x74, 0x44, 0x61, 0x74, 0x61, 0x46, 0x72, 0x6f, 0x6d, + 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x28, 0x27, 0x75, 0x73, 0x65, 0x72, + 0x5f, 0x74, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x73, 0x27, 0x2c, + 0x20, 0x73, 0x61, 0x76, 0x65, 0x64, 0x55, 0x73, 0x65, 0x72, 0x54, 0x65, + 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x73, 0x2e, 0x76, 0x61, 0x6c, 0x75, + 0x65, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x20, 0x65, 0x6c, 0x73, + 0x65, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x2f, 0x2f, + 0x20, 0x6e, 0x6f, 0x20, 0x73, 0x61, 0x76, 0x65, 0x64, 0x20, 0x74, 0x65, + 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x73, 0x20, 0x64, 0x65, 0x74, 0x65, + 0x63, 0x74, 0x65, 0x64, 0x2e, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x6f, 0x6c, 0x65, 0x2e, 0x6c, 0x6f, 0x67, + 0x28, 0x27, 0x49, 0x6e, 0x69, 0x74, 0x69, 0x61, 0x6c, 0x69, 0x7a, 0x69, + 0x6e, 0x67, 0x20, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x53, 0x74, 0x6f, 0x72, + 0x61, 0x67, 0x65, 0x20, 0x61, 0x6e, 0x64, 0x20, 0x73, 0x61, 0x76, 0x69, + 0x6e, 0x67, 0x20, 0x64, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x20, 0x74, + 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x27, 0x29, 0x0a, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x73, 0x61, 0x76, 0x65, 0x64, 0x55, 0x73, + 0x65, 0x72, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x73, 0x2e, + 0x76, 0x61, 0x6c, 0x75, 0x65, 0x20, 0x3d, 0x20, 0x7b, 0x20, 0x22, 0x64, + 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x22, 0x3a, 0x20, 0x7b, 0x20, 0x73, + 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x3a, 0x20, 0x73, 0x65, 0x73, 0x73, + 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x2c, 0x20, 0x70, + 0x61, 0x72, 0x61, 0x6d, 0x73, 0x3a, 0x20, 0x70, 0x61, 0x72, 0x61, 0x6d, + 0x73, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x20, 0x7d, 0x20, 0x7d, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x5f, + 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x5f, 0x73, 0x65, 0x74, 0x44, + 0x61, 0x74, 0x61, 0x46, 0x72, 0x6f, 0x6d, 0x4f, 0x62, 0x6a, 0x65, 0x63, + 0x74, 0x28, 0x27, 0x75, 0x73, 0x65, 0x72, 0x5f, 0x74, 0x65, 0x6d, 0x70, + 0x6c, 0x61, 0x74, 0x65, 0x73, 0x27, 0x2c, 0x20, 0x73, 0x61, 0x76, 0x65, + 0x64, 0x55, 0x73, 0x65, 0x72, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, + 0x65, 0x73, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x29, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x7d, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x66, 0x75, 0x6e, + 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x75, 0x73, 0x65, 0x72, 0x54, 0x65, + 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x52, 0x65, 0x73, 0x65, 0x74, 0x54, + 0x6f, 0x44, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x28, 0x29, 0x20, 0x7b, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x6f, + 0x6c, 0x65, 0x2e, 0x6c, 0x6f, 0x67, 0x28, 0x27, 0x52, 0x65, 0x73, 0x65, + 0x74, 0x74, 0x69, 0x6e, 0x67, 0x20, 0x74, 0x65, 0x6d, 0x70, 0x6c, 0x61, + 0x74, 0x65, 0x20, 0x74, 0x6f, 0x20, 0x64, 0x65, 0x66, 0x61, 0x75, 0x6c, + 0x74, 0x27, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x73, 0x65, + 0x6c, 0x65, 0x63, 0x74, 0x65, 0x64, 0x55, 0x73, 0x65, 0x72, 0x54, 0x65, + 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, + 0x2e, 0x6e, 0x61, 0x6d, 0x65, 0x20, 0x3d, 0x20, 0x27, 0x64, 0x65, 0x66, + 0x61, 0x75, 0x6c, 0x74, 0x27, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x73, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x65, 0x64, 0x55, 0x73, 0x65, + 0x72, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x2e, 0x76, 0x61, + 0x6c, 0x75, 0x65, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x20, 0x3d, 0x20, 0x73, + 0x61, 0x76, 0x65, 0x64, 0x55, 0x73, 0x65, 0x72, 0x54, 0x65, 0x6d, 0x70, + 0x6c, 0x61, 0x74, 0x65, 0x73, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x5b, + 0x27, 0x64, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x27, 0x5d, 0x3b, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x66, + 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x75, 0x73, 0x65, 0x72, + 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x41, 0x70, 0x70, 0x6c, + 0x79, 0x28, 0x74, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x61, 0x6c, + 0x75, 0x65, 0x20, 0x3d, 0x20, 0x74, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x2e, + 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x3b, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x2e, 0x76, + 0x61, 0x6c, 0x75, 0x65, 0x20, 0x3d, 0x20, 0x7b, 0x20, 0x2e, 0x2e, 0x2e, + 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x61, 0x6c, 0x75, + 0x65, 0x2c, 0x20, 0x69, 0x6d, 0x61, 0x67, 0x65, 0x5f, 0x73, 0x65, 0x6c, + 0x65, 0x63, 0x74, 0x65, 0x64, 0x3a, 0x20, 0x27, 0x27, 0x20, 0x7d, 0x3b, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x70, 0x61, 0x72, 0x61, 0x6d, + 0x73, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x20, 0x3d, 0x20, 0x74, 0x2e, + 0x64, 0x61, 0x74, 0x61, 0x2e, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x3b, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x70, 0x61, 0x72, 0x61, 0x6d, + 0x73, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x20, 0x3d, 0x20, 0x7b, 0x20, + 0x2e, 0x2e, 0x2e, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x2e, 0x76, 0x61, + 0x6c, 0x75, 0x65, 0x2c, 0x20, 0x69, 0x6d, 0x61, 0x67, 0x65, 0x5f, 0x64, + 0x61, 0x74, 0x61, 0x3a, 0x20, 0x5b, 0x5d, 0x20, 0x7d, 0x3b, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x66, 0x75, + 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x75, 0x73, 0x65, 0x72, 0x54, + 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x52, 0x65, 0x73, 0x65, 0x74, + 0x54, 0x6f, 0x44, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x41, 0x6e, 0x64, + 0x41, 0x70, 0x70, 0x6c, 0x79, 0x28, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x75, 0x73, 0x65, 0x72, 0x54, 0x65, 0x6d, 0x70, + 0x6c, 0x61, 0x74, 0x65, 0x52, 0x65, 0x73, 0x65, 0x74, 0x54, 0x6f, 0x44, + 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x28, 0x29, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x75, 0x73, 0x65, 0x72, 0x54, 0x65, 0x6d, 0x70, 0x6c, + 0x61, 0x74, 0x65, 0x41, 0x70, 0x70, 0x6c, 0x79, 0x28, 0x73, 0x65, 0x6c, + 0x65, 0x63, 0x74, 0x65, 0x64, 0x55, 0x73, 0x65, 0x72, 0x54, 0x65, 0x6d, + 0x70, 0x6c, 0x61, 0x74, 0x65, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x29, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x75, 0x73, 0x65, + 0x72, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x4c, 0x6f, 0x61, + 0x64, 0x41, 0x6e, 0x64, 0x41, 0x70, 0x70, 0x6c, 0x79, 0x41, 0x75, 0x74, + 0x6f, 0x73, 0x61, 0x76, 0x65, 0x64, 0x28, 0x29, 0x20, 0x7b, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x2f, 0x2f, 0x20, 0x67, 0x65, 0x74, 0x20, + 0x61, 0x75, 0x74, 0x6f, 0x73, 0x61, 0x76, 0x65, 0x64, 0x20, 0x6c, 0x61, + 0x73, 0x74, 0x20, 0x75, 0x73, 0x65, 0x64, 0x20, 0x74, 0x65, 0x6d, 0x70, + 0x6c, 0x61, 0x74, 0x65, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x6c, + 0x65, 0x74, 0x20, 0x6c, 0x61, 0x73, 0x74, 0x55, 0x73, 0x65, 0x64, 0x54, + 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x20, 0x3d, 0x20, 0x6c, 0x6f, + 0x63, 0x61, 0x6c, 0x5f, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x5f, + 0x67, 0x65, 0x74, 0x44, 0x61, 0x74, 0x61, 0x41, 0x73, 0x4f, 0x62, 0x6a, + 0x65, 0x63, 0x74, 0x28, 0x27, 0x75, 0x73, 0x65, 0x72, 0x5f, 0x74, 0x65, + 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x73, 0x5f, 0x6c, 0x61, 0x73, 0x74, + 0x27, 0x29, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, + 0x20, 0x28, 0x6c, 0x61, 0x73, 0x74, 0x55, 0x73, 0x65, 0x64, 0x54, 0x65, + 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x29, 0x20, 0x7b, 0x0a, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x6f, + 0x6c, 0x65, 0x2e, 0x6c, 0x6f, 0x67, 0x28, 0x27, 0x41, 0x75, 0x74, 0x6f, + 0x73, 0x61, 0x76, 0x65, 0x64, 0x20, 0x74, 0x65, 0x6d, 0x70, 0x6c, 0x61, + 0x74, 0x65, 0x20, 0x66, 0x6f, 0x75, 0x6e, 0x64, 0x2c, 0x20, 0x72, 0x65, + 0x73, 0x74, 0x6f, 0x72, 0x69, 0x6e, 0x67, 0x27, 0x29, 0x0a, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x73, 0x65, 0x6c, 0x65, 0x63, + 0x74, 0x65, 0x64, 0x55, 0x73, 0x65, 0x72, 0x54, 0x65, 0x6d, 0x70, 0x6c, + 0x61, 0x74, 0x65, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x20, 0x3d, 0x20, + 0x6c, 0x61, 0x73, 0x74, 0x55, 0x73, 0x65, 0x64, 0x54, 0x65, 0x6d, 0x70, + 0x6c, 0x61, 0x74, 0x65, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x65, 0x6c, 0x73, 0x65, 0x20, + 0x7b, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, + 0x6f, 0x6e, 0x73, 0x6f, 0x6c, 0x65, 0x2e, 0x6c, 0x6f, 0x67, 0x28, 0x27, + 0x4e, 0x6f, 0x20, 0x61, 0x75, 0x74, 0x6f, 0x73, 0x61, 0x76, 0x65, 0x64, + 0x20, 0x74, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x20, 0x66, 0x6f, + 0x75, 0x6e, 0x64, 0x2c, 0x20, 0x75, 0x73, 0x69, 0x6e, 0x67, 0x20, 0x64, + 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x20, 0x74, 0x65, 0x6d, 0x70, 0x6c, + 0x61, 0x74, 0x65, 0x27, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x2f, 0x2f, 0x20, 0x6e, 0x6f, 0x20, 0x61, 0x75, 0x74, 0x6f, + 0x73, 0x61, 0x76, 0x65, 0x64, 0x20, 0x6c, 0x61, 0x73, 0x74, 0x20, 0x75, + 0x73, 0x65, 0x64, 0x20, 0x74, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, + 0x20, 0x77, 0x61, 0x73, 0x20, 0x66, 0x6f, 0x75, 0x6e, 0x64, 0x2c, 0x20, + 0x73, 0x6f, 0x20, 0x6c, 0x6f, 0x61, 0x64, 0x20, 0x66, 0x72, 0x6f, 0x6d, + 0x20, 0x64, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x2e, 0x0a, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x75, 0x73, 0x65, 0x72, 0x54, + 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x52, 0x65, 0x73, 0x65, 0x74, + 0x54, 0x6f, 0x44, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x28, 0x29, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x6f, 0x6c, 0x65, 0x2e, 0x6c, + 0x6f, 0x67, 0x28, 0x27, 0x41, 0x70, 0x70, 0x6c, 0x79, 0x69, 0x6e, 0x67, + 0x20, 0x74, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x27, 0x29, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x2f, 0x2f, 0x20, 0x61, 0x6e, 0x64, + 0x20, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x20, 0x69, 0x6e, 0x74, 0x65, + 0x72, 0x6e, 0x61, 0x6c, 0x20, 0x64, 0x61, 0x74, 0x61, 0x20, 0x66, 0x72, + 0x6f, 0x6d, 0x20, 0x74, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x73, + 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x75, 0x73, 0x65, 0x72, + 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x41, 0x70, 0x70, 0x6c, + 0x79, 0x28, 0x73, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x65, 0x64, 0x55, 0x73, + 0x65, 0x72, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x2e, 0x76, + 0x61, 0x6c, 0x75, 0x65, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x2f, 0x2f, 0x63, 0x6f, 0x6e, 0x73, 0x6f, + 0x6c, 0x65, 0x2e, 0x6c, 0x6f, 0x67, 0x28, 0x73, 0x61, 0x76, 0x65, 0x64, + 0x55, 0x73, 0x65, 0x72, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, + 0x73, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x29, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x2f, 0x2f, 0x63, 0x6f, 0x6e, 0x73, 0x6f, 0x6c, 0x65, 0x2e, 0x6c, + 0x6f, 0x67, 0x28, 0x73, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x65, 0x64, 0x55, + 0x73, 0x65, 0x72, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x2e, + 0x76, 0x61, 0x6c, 0x75, 0x65, 0x29, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x75, 0x73, 0x65, + 0x72, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x41, 0x75, 0x74, + 0x6f, 0x73, 0x61, 0x76, 0x65, 0x28, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x6f, 0x6c, 0x65, 0x2e, + 0x6c, 0x6f, 0x67, 0x28, 0x27, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, + 0x65, 0x20, 0x41, 0x75, 0x74, 0x6f, 0x73, 0x61, 0x76, 0x65, 0x2e, 0x2e, + 0x2e, 0x27, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, + 0x20, 0x28, 0x73, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x65, 0x64, 0x55, 0x73, + 0x65, 0x72, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x2e, 0x76, + 0x61, 0x6c, 0x75, 0x65, 0x2e, 0x6e, 0x61, 0x6d, 0x65, 0x20, 0x3d, 0x3d, + 0x20, 0x27, 0x64, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x27, 0x29, 0x20, + 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x2f, 0x2f, + 0x20, 0x77, 0x65, 0x20, 0x64, 0x6f, 0x6e, 0x27, 0x74, 0x20, 0x77, 0x61, + 0x6e, 0x74, 0x20, 0x74, 0x6f, 0x20, 0x73, 0x61, 0x76, 0x65, 0x20, 0x6f, + 0x76, 0x65, 0x72, 0x20, 0x64, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x20, + 0x74, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x2c, 0x20, 0x73, 0x6f, + 0x20, 0x6c, 0x65, 0x74, 0x27, 0x73, 0x20, 0x63, 0x72, 0x65, 0x61, 0x74, + 0x65, 0x20, 0x61, 0x20, 0x6e, 0x65, 0x77, 0x20, 0x6f, 0x6e, 0x65, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x6c, 0x65, 0x74, 0x20, + 0x6e, 0x65, 0x77, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x4e, + 0x61, 0x6d, 0x65, 0x20, 0x3d, 0x20, 0x27, 0x55, 0x73, 0x65, 0x72, 0x54, + 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x2d, 0x27, 0x20, 0x2b, 0x20, + 0x44, 0x61, 0x74, 0x65, 0x2e, 0x6e, 0x6f, 0x77, 0x28, 0x29, 0x2e, 0x74, + 0x6f, 0x53, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x28, 0x29, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x6c, 0x65, 0x74, 0x20, 0x6e, 0x65, + 0x77, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x20, 0x3d, 0x20, + 0x7b, 0x20, 0x27, 0x6e, 0x61, 0x6d, 0x65, 0x27, 0x3a, 0x20, 0x6e, 0x65, + 0x77, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x4e, 0x61, 0x6d, + 0x65, 0x2c, 0x20, 0x27, 0x64, 0x61, 0x74, 0x61, 0x27, 0x3a, 0x20, 0x7b, + 0x20, 0x27, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x27, 0x3a, 0x20, + 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x61, 0x6c, 0x75, + 0x65, 0x2c, 0x20, 0x27, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x27, 0x3a, + 0x20, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x2e, 0x76, 0x61, 0x6c, 0x75, + 0x65, 0x20, 0x7d, 0x20, 0x7d, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x6f, 0x6c, 0x65, 0x2e, 0x6c, + 0x6f, 0x67, 0x28, 0x27, 0x53, 0x61, 0x76, 0x69, 0x6e, 0x67, 0x20, 0x61, + 0x73, 0x20, 0x27, 0x20, 0x2b, 0x20, 0x6e, 0x65, 0x77, 0x54, 0x65, 0x6d, + 0x70, 0x6c, 0x61, 0x74, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x29, 0x0a, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x2f, 0x2f, 0x20, 0x73, + 0x61, 0x76, 0x65, 0x20, 0x69, 0x6e, 0x20, 0x74, 0x68, 0x65, 0x20, 0x61, + 0x75, 0x74, 0x6f, 0x73, 0x61, 0x76, 0x65, 0x20, 0x73, 0x6c, 0x6f, 0x74, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x6c, 0x6f, 0x63, + 0x61, 0x6c, 0x5f, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x5f, 0x73, + 0x65, 0x74, 0x44, 0x61, 0x74, 0x61, 0x46, 0x72, 0x6f, 0x6d, 0x4f, 0x62, + 0x6a, 0x65, 0x63, 0x74, 0x28, 0x27, 0x75, 0x73, 0x65, 0x72, 0x5f, 0x74, + 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x73, 0x5f, 0x6c, 0x61, 0x73, + 0x74, 0x27, 0x2c, 0x20, 0x6e, 0x65, 0x77, 0x54, 0x65, 0x6d, 0x70, 0x6c, + 0x61, 0x74, 0x65, 0x29, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x2f, 0x2f, 0x20, 0x61, 0x6e, 0x64, 0x20, 0x6c, 0x6f, 0x61, + 0x64, 0x20, 0x69, 0x74, 0x20, 0x62, 0x61, 0x63, 0x6b, 0x20, 0x61, 0x6e, + 0x64, 0x20, 0x61, 0x70, 0x70, 0x6c, 0x79, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x75, 0x73, 0x65, 0x72, 0x54, 0x65, 0x6d, 0x70, + 0x6c, 0x61, 0x74, 0x65, 0x4c, 0x6f, 0x61, 0x64, 0x41, 0x6e, 0x64, 0x41, + 0x70, 0x70, 0x6c, 0x79, 0x41, 0x75, 0x74, 0x6f, 0x73, 0x61, 0x76, 0x65, + 0x64, 0x28, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x20, + 0x65, 0x6c, 0x73, 0x65, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x5f, 0x73, 0x74, 0x6f, + 0x72, 0x61, 0x67, 0x65, 0x5f, 0x73, 0x65, 0x74, 0x44, 0x61, 0x74, 0x61, + 0x46, 0x72, 0x6f, 0x6d, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x28, 0x27, + 0x75, 0x73, 0x65, 0x72, 0x5f, 0x74, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, + 0x65, 0x73, 0x5f, 0x6c, 0x61, 0x73, 0x74, 0x27, 0x2c, 0x20, 0x7b, 0x20, + 0x27, 0x6e, 0x61, 0x6d, 0x65, 0x27, 0x3a, 0x20, 0x73, 0x65, 0x6c, 0x65, + 0x63, 0x74, 0x65, 0x64, 0x55, 0x73, 0x65, 0x72, 0x54, 0x65, 0x6d, 0x70, + 0x6c, 0x61, 0x74, 0x65, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x2e, 0x6e, + 0x61, 0x6d, 0x65, 0x2c, 0x20, 0x27, 0x64, 0x61, 0x74, 0x61, 0x27, 0x3a, + 0x20, 0x7b, 0x20, 0x27, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x27, + 0x3a, 0x20, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x61, + 0x6c, 0x75, 0x65, 0x2c, 0x20, 0x27, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, + 0x27, 0x3a, 0x20, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x2e, 0x76, 0x61, + 0x6c, 0x75, 0x65, 0x20, 0x7d, 0x20, 0x7d, 0x29, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x6f, 0x6c, 0x65, 0x2e, + 0x6c, 0x6f, 0x67, 0x28, 0x27, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x69, 0x6e, + 0x67, 0x20, 0x66, 0x6f, 0x72, 0x20, 0x61, 0x75, 0x74, 0x6f, 0x73, 0x61, + 0x76, 0x65, 0x64, 0x20, 0x6c, 0x61, 0x73, 0x74, 0x20, 0x75, 0x73, 0x65, + 0x64, 0x20, 0x74, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x27, 0x29, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x75, 0x73, 0x65, 0x72, 0x54, 0x65, 0x6d, + 0x70, 0x6c, 0x61, 0x74, 0x65, 0x4c, 0x6f, 0x61, 0x64, 0x41, 0x6e, 0x64, + 0x41, 0x70, 0x70, 0x6c, 0x79, 0x41, 0x75, 0x74, 0x6f, 0x73, 0x61, 0x76, + 0x65, 0x64, 0x28, 0x29, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x2f, 0x2a, + 0x20, 0x45, 0x4e, 0x44, 0x3a, 0x20, 0x53, 0x75, 0x70, 0x70, 0x6f, 0x72, + 0x74, 0x20, 0x66, 0x6f, 0x72, 0x20, 0x73, 0x74, 0x6f, 0x72, 0x69, 0x6e, + 0x67, 0x20, 0x70, 0x72, 0x6f, 0x6d, 0x70, 0x74, 0x20, 0x74, 0x65, 0x6d, + 0x70, 0x6c, 0x61, 0x74, 0x65, 0x73, 0x20, 0x61, 0x6e, 0x64, 0x20, 0x70, + 0x61, 0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x73, 0x20, 0x69, 0x6e, + 0x20, 0x62, 0x72, 0x6f, 0x77, 0x73, 0x65, 0x72, 0x73, 0x20, 0x4c, 0x6f, + 0x63, 0x61, 0x6c, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x20, 0x2a, + 0x2f, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, + 0x20, 0x6c, 0x6c, 0x61, 0x6d, 0x61, 0x53, 0x74, 0x61, 0x74, 0x73, 0x20, + 0x3d, 0x20, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x28, 0x6e, 0x75, 0x6c, + 0x6c, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, + 0x20, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x20, + 0x3d, 0x20, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x28, 0x6e, 0x75, 0x6c, + 0x6c, 0x29, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x2f, 0x2f, 0x20, 0x63, + 0x75, 0x72, 0x72, 0x65, 0x6e, 0x74, 0x6c, 0x79, 0x20, 0x67, 0x65, 0x6e, + 0x65, 0x72, 0x61, 0x74, 0x69, 0x6e, 0x67, 0x20, 0x61, 0x20, 0x63, 0x6f, + 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x69, 0x6f, 0x6e, 0x3f, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x67, 0x65, 0x6e, 0x65, + 0x72, 0x61, 0x74, 0x69, 0x6e, 0x67, 0x20, 0x3d, 0x20, 0x63, 0x6f, 0x6d, + 0x70, 0x75, 0x74, 0x65, 0x64, 0x28, 0x28, 0x29, 0x20, 0x3d, 0x3e, 0x20, + 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x2e, 0x76, + 0x61, 0x6c, 0x75, 0x65, 0x20, 0x21, 0x3d, 0x20, 0x6e, 0x75, 0x6c, 0x6c, + 0x29, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x2f, 0x2f, 0x20, 0x68, 0x61, + 0x73, 0x20, 0x74, 0x68, 0x65, 0x20, 0x75, 0x73, 0x65, 0x72, 0x20, 0x73, + 0x74, 0x61, 0x72, 0x74, 0x65, 0x64, 0x20, 0x61, 0x20, 0x63, 0x68, 0x61, + 0x74, 0x3f, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, + 0x20, 0x63, 0x68, 0x61, 0x74, 0x53, 0x74, 0x61, 0x72, 0x74, 0x65, 0x64, + 0x20, 0x3d, 0x20, 0x63, 0x6f, 0x6d, 0x70, 0x75, 0x74, 0x65, 0x64, 0x28, + 0x28, 0x29, 0x20, 0x3d, 0x3e, 0x20, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, + 0x6e, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x2e, 0x74, 0x72, 0x61, 0x6e, + 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x2e, 0x6c, 0x65, 0x6e, 0x67, 0x74, + 0x68, 0x20, 0x3e, 0x20, 0x30, 0x29, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x63, + 0x72, 0x69, 0x70, 0x74, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x20, 0x3d, + 0x20, 0x28, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, + 0x29, 0x20, 0x3d, 0x3e, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x61, 0x6c, + 0x75, 0x65, 0x20, 0x3d, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x2e, 0x2e, 0x2e, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, + 0x6e, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x2c, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x63, 0x72, + 0x69, 0x70, 0x74, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x2f, + 0x2f, 0x20, 0x73, 0x69, 0x6d, 0x70, 0x6c, 0x65, 0x20, 0x74, 0x65, 0x6d, + 0x70, 0x6c, 0x61, 0x74, 0x65, 0x20, 0x72, 0x65, 0x70, 0x6c, 0x61, 0x63, + 0x65, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, + 0x74, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x20, 0x3d, 0x20, 0x28, + 0x73, 0x74, 0x72, 0x2c, 0x20, 0x65, 0x78, 0x74, 0x72, 0x61, 0x53, 0x65, + 0x74, 0x74, 0x69, 0x6e, 0x67, 0x73, 0x29, 0x20, 0x3d, 0x3e, 0x20, 0x7b, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x6c, 0x65, 0x74, 0x20, 0x73, + 0x65, 0x74, 0x74, 0x69, 0x6e, 0x67, 0x73, 0x20, 0x3d, 0x20, 0x73, 0x65, + 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3b, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, 0x65, + 0x78, 0x74, 0x72, 0x61, 0x53, 0x65, 0x74, 0x74, 0x69, 0x6e, 0x67, 0x73, + 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x73, 0x65, 0x74, 0x74, 0x69, 0x6e, 0x67, 0x73, 0x20, 0x3d, 0x20, 0x7b, + 0x20, 0x2e, 0x2e, 0x2e, 0x73, 0x65, 0x74, 0x74, 0x69, 0x6e, 0x67, 0x73, + 0x2c, 0x20, 0x2e, 0x2e, 0x2e, 0x65, 0x78, 0x74, 0x72, 0x61, 0x53, 0x65, + 0x74, 0x74, 0x69, 0x6e, 0x67, 0x73, 0x20, 0x7d, 0x3b, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x53, 0x74, 0x72, 0x69, 0x6e, + 0x67, 0x28, 0x73, 0x74, 0x72, 0x29, 0x2e, 0x72, 0x65, 0x70, 0x6c, 0x61, + 0x63, 0x65, 0x41, 0x6c, 0x6c, 0x28, 0x2f, 0x5c, 0x7b, 0x5c, 0x7b, 0x28, + 0x2e, 0x2a, 0x3f, 0x29, 0x5c, 0x7d, 0x5c, 0x7d, 0x2f, 0x67, 0x2c, 0x20, + 0x28, 0x5f, 0x2c, 0x20, 0x6b, 0x65, 0x79, 0x29, 0x20, 0x3d, 0x3e, 0x20, + 0x74, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x28, 0x73, 0x65, 0x74, + 0x74, 0x69, 0x6e, 0x67, 0x73, 0x5b, 0x6b, 0x65, 0x79, 0x5d, 0x29, 0x29, + 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x61, 0x73, 0x79, 0x6e, 0x63, 0x20, 0x66, 0x75, 0x6e, 0x63, 0x74, + 0x69, 0x6f, 0x6e, 0x20, 0x72, 0x75, 0x6e, 0x4c, 0x6c, 0x61, 0x6d, 0x61, + 0x28, 0x70, 0x72, 0x6f, 0x6d, 0x70, 0x74, 0x2c, 0x20, 0x6c, 0x6c, 0x61, + 0x6d, 0x61, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x2c, 0x20, 0x63, 0x68, + 0x61, 0x72, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, + 0x74, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x20, 0x3d, 0x20, + 0x5b, 0x5d, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, + 0x6e, 0x73, 0x74, 0x20, 0x68, 0x69, 0x73, 0x74, 0x6f, 0x72, 0x79, 0x20, + 0x3d, 0x20, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x61, + 0x6c, 0x75, 0x65, 0x2e, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x63, 0x72, 0x69, + 0x70, 0x74, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, + 0x20, 0x28, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, + 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x74, 0x68, 0x72, 0x6f, 0x77, 0x20, + 0x6e, 0x65, 0x77, 0x20, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x28, 0x22, 0x61, + 0x6c, 0x72, 0x65, 0x61, 0x64, 0x79, 0x20, 0x72, 0x75, 0x6e, 0x6e, 0x69, + 0x6e, 0x67, 0x22, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x74, + 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, + 0x20, 0x3d, 0x20, 0x6e, 0x65, 0x77, 0x20, 0x41, 0x62, 0x6f, 0x72, 0x74, + 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x28, 0x29, + 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x66, 0x6f, 0x72, 0x20, + 0x61, 0x77, 0x61, 0x69, 0x74, 0x20, 0x28, 0x63, 0x6f, 0x6e, 0x73, 0x74, + 0x20, 0x63, 0x68, 0x75, 0x6e, 0x6b, 0x20, 0x6f, 0x66, 0x20, 0x6c, 0x6c, + 0x61, 0x6d, 0x61, 0x28, 0x70, 0x72, 0x6f, 0x6d, 0x70, 0x74, 0x2c, 0x20, + 0x6c, 0x6c, 0x61, 0x6d, 0x61, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x2c, + 0x20, 0x7b, 0x20, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, + 0x72, 0x3a, 0x20, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, + 0x72, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x20, 0x7d, 0x29, 0x29, 0x20, + 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, + 0x6e, 0x73, 0x74, 0x20, 0x64, 0x61, 0x74, 0x61, 0x20, 0x3d, 0x20, 0x63, + 0x68, 0x75, 0x6e, 0x6b, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x3b, 0x0a, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, + 0x64, 0x61, 0x74, 0x61, 0x2e, 0x73, 0x74, 0x6f, 0x70, 0x29, 0x20, 0x7b, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x77, + 0x68, 0x69, 0x6c, 0x65, 0x20, 0x28, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x75, 0x72, 0x72, 0x65, + 0x6e, 0x74, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x2e, 0x6c, + 0x65, 0x6e, 0x67, 0x74, 0x68, 0x20, 0x3e, 0x20, 0x30, 0x20, 0x26, 0x26, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x74, 0x4d, 0x65, 0x73, 0x73, + 0x61, 0x67, 0x65, 0x73, 0x5b, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x74, + 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x2e, 0x6c, 0x65, 0x6e, + 0x67, 0x74, 0x68, 0x20, 0x2d, 0x20, 0x31, 0x5d, 0x2e, 0x63, 0x6f, 0x6e, + 0x74, 0x65, 0x6e, 0x74, 0x2e, 0x6d, 0x61, 0x74, 0x63, 0x68, 0x28, 0x2f, + 0x5c, 0x6e, 0x24, 0x2f, 0x29, 0x20, 0x21, 0x3d, 0x20, 0x6e, 0x75, 0x6c, + 0x6c, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x74, 0x4d, + 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x2e, 0x70, 0x6f, 0x70, 0x28, + 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x55, + 0x70, 0x64, 0x61, 0x74, 0x65, 0x28, 0x5b, 0x2e, 0x2e, 0x2e, 0x68, 0x69, + 0x73, 0x74, 0x6f, 0x72, 0x79, 0x2c, 0x20, 0x5b, 0x63, 0x68, 0x61, 0x72, + 0x2c, 0x20, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x74, 0x4d, 0x65, 0x73, + 0x73, 0x61, 0x67, 0x65, 0x73, 0x5d, 0x5d, 0x29, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x6f, + 0x6c, 0x65, 0x2e, 0x6c, 0x6f, 0x67, 0x28, 0x22, 0x43, 0x6f, 0x6d, 0x70, + 0x6c, 0x65, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x66, 0x69, 0x6e, 0x69, 0x73, + 0x68, 0x65, 0x64, 0x3a, 0x20, 0x27, 0x22, 0x2c, 0x20, 0x63, 0x75, 0x72, + 0x72, 0x65, 0x6e, 0x74, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, + 0x2e, 0x6d, 0x61, 0x70, 0x28, 0x6d, 0x73, 0x67, 0x20, 0x3d, 0x3e, 0x20, + 0x6d, 0x73, 0x67, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x29, + 0x2e, 0x6a, 0x6f, 0x69, 0x6e, 0x28, 0x27, 0x27, 0x29, 0x2c, 0x20, 0x22, + 0x27, 0x2c, 0x20, 0x73, 0x75, 0x6d, 0x6d, 0x61, 0x72, 0x79, 0x3a, 0x20, + 0x22, 0x2c, 0x20, 0x64, 0x61, 0x74, 0x61, 0x29, 0x3b, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x20, 0x65, 0x6c, 0x73, 0x65, + 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x74, 0x4d, 0x65, 0x73, 0x73, + 0x61, 0x67, 0x65, 0x73, 0x2e, 0x70, 0x75, 0x73, 0x68, 0x28, 0x64, 0x61, + 0x74, 0x61, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x73, 0x6c, 0x6f, 0x74, 0x5f, 0x69, 0x64, 0x20, 0x3d, + 0x20, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x73, 0x6c, 0x6f, 0x74, 0x5f, 0x69, + 0x64, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x69, 0x66, 0x20, 0x28, 0x73, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x65, + 0x64, 0x5f, 0x69, 0x6d, 0x61, 0x67, 0x65, 0x20, 0x26, 0x26, 0x20, 0x21, + 0x64, 0x61, 0x74, 0x61, 0x2e, 0x6d, 0x75, 0x6c, 0x74, 0x69, 0x6d, 0x6f, + 0x64, 0x61, 0x6c, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x61, 0x6c, 0x65, 0x72, 0x74, + 0x28, 0x22, 0x54, 0x68, 0x65, 0x20, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, + 0x20, 0x77, 0x61, 0x73, 0x20, 0x6e, 0x6f, 0x74, 0x20, 0x63, 0x6f, 0x6d, + 0x70, 0x69, 0x6c, 0x65, 0x64, 0x20, 0x66, 0x6f, 0x72, 0x20, 0x6d, 0x75, + 0x6c, 0x74, 0x69, 0x6d, 0x6f, 0x64, 0x61, 0x6c, 0x20, 0x6f, 0x72, 0x20, + 0x74, 0x68, 0x65, 0x20, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x20, 0x70, 0x72, + 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x20, 0x63, 0x61, 0x6e, 0x27, + 0x74, 0x20, 0x62, 0x65, 0x20, 0x6c, 0x6f, 0x61, 0x64, 0x65, 0x64, 0x2e, + 0x22, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x3b, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x74, 0x72, + 0x61, 0x6e, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x55, 0x70, 0x64, 0x61, + 0x74, 0x65, 0x28, 0x5b, 0x2e, 0x2e, 0x2e, 0x68, 0x69, 0x73, 0x74, 0x6f, + 0x72, 0x79, 0x2c, 0x20, 0x5b, 0x63, 0x68, 0x61, 0x72, 0x2c, 0x20, 0x63, + 0x75, 0x72, 0x72, 0x65, 0x6e, 0x74, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, + 0x65, 0x73, 0x5d, 0x5d, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x7d, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x69, 0x66, 0x20, 0x28, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x74, 0x69, + 0x6d, 0x69, 0x6e, 0x67, 0x73, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x6c, 0x6c, 0x61, 0x6d, 0x61, + 0x53, 0x74, 0x61, 0x74, 0x73, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x20, + 0x3d, 0x20, 0x64, 0x61, 0x74, 0x61, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x7d, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, + 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x2e, 0x76, 0x61, 0x6c, 0x75, + 0x65, 0x20, 0x3d, 0x20, 0x6e, 0x75, 0x6c, 0x6c, 0x3b, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x7d, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x2f, 0x2f, 0x20, + 0x73, 0x65, 0x6e, 0x64, 0x20, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, + 0x20, 0x74, 0x6f, 0x20, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x63, 0x68, 0x61, + 0x74, 0x20, 0x3d, 0x20, 0x61, 0x73, 0x79, 0x6e, 0x63, 0x20, 0x28, 0x6d, + 0x73, 0x67, 0x29, 0x20, 0x3d, 0x3e, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, 0x63, 0x6f, 0x6e, 0x74, 0x72, + 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x29, + 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, + 0x6f, 0x6e, 0x73, 0x6f, 0x6c, 0x65, 0x2e, 0x6c, 0x6f, 0x67, 0x28, 0x27, + 0x61, 0x6c, 0x72, 0x65, 0x61, 0x64, 0x79, 0x20, 0x72, 0x75, 0x6e, 0x6e, + 0x69, 0x6e, 0x67, 0x2e, 0x2e, 0x2e, 0x27, 0x29, 0x3b, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, + 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x63, 0x72, + 0x69, 0x70, 0x74, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x28, 0x5b, 0x2e, + 0x2e, 0x2e, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x61, + 0x6c, 0x75, 0x65, 0x2e, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x63, 0x72, 0x69, + 0x70, 0x74, 0x2c, 0x20, 0x5b, 0x22, 0x7b, 0x7b, 0x75, 0x73, 0x65, 0x72, + 0x7d, 0x7d, 0x22, 0x2c, 0x20, 0x6d, 0x73, 0x67, 0x5d, 0x5d, 0x29, 0x0a, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x6c, 0x65, 0x74, 0x20, 0x70, + 0x72, 0x6f, 0x6d, 0x70, 0x74, 0x20, 0x3d, 0x20, 0x74, 0x65, 0x6d, 0x70, + 0x6c, 0x61, 0x74, 0x65, 0x28, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, + 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x2e, 0x74, 0x65, 0x6d, 0x70, 0x6c, + 0x61, 0x74, 0x65, 0x2c, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x3a, 0x20, + 0x6d, 0x73, 0x67, 0x2c, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x68, 0x69, 0x73, 0x74, 0x6f, 0x72, 0x79, 0x3a, 0x20, 0x73, 0x65, + 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x2e, + 0x74, 0x72, 0x61, 0x6e, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x2e, 0x66, + 0x6c, 0x61, 0x74, 0x4d, 0x61, 0x70, 0x28, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x28, 0x5b, 0x6e, 0x61, 0x6d, 0x65, + 0x2c, 0x20, 0x64, 0x61, 0x74, 0x61, 0x5d, 0x29, 0x20, 0x3d, 0x3e, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x74, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x28, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x61, 0x6c, 0x75, + 0x65, 0x2e, 0x68, 0x69, 0x73, 0x74, 0x6f, 0x72, 0x79, 0x54, 0x65, 0x6d, + 0x70, 0x6c, 0x61, 0x74, 0x65, 0x2c, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7b, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x6e, 0x61, 0x6d, 0x65, 0x2c, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x3a, 0x20, 0x41, 0x72, + 0x72, 0x61, 0x79, 0x2e, 0x69, 0x73, 0x41, 0x72, 0x72, 0x61, 0x79, 0x28, + 0x64, 0x61, 0x74, 0x61, 0x29, 0x20, 0x3f, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x6d, 0x61, 0x70, 0x28, 0x6d, + 0x73, 0x67, 0x20, 0x3d, 0x3e, 0x20, 0x6d, 0x73, 0x67, 0x2e, 0x63, 0x6f, + 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x29, 0x2e, 0x6a, 0x6f, 0x69, 0x6e, 0x28, + 0x27, 0x27, 0x29, 0x2e, 0x72, 0x65, 0x70, 0x6c, 0x61, 0x63, 0x65, 0x28, + 0x2f, 0x5e, 0x5c, 0x73, 0x2f, 0x2c, 0x20, 0x27, 0x27, 0x29, 0x20, 0x3a, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x64, 0x61, 0x74, 0x61, 0x2c, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x29, 0x2e, 0x6a, 0x6f, 0x69, 0x6e, 0x28, 0x22, 0x5c, + 0x6e, 0x22, 0x29, 0x2c, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, + 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, + 0x28, 0x73, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x65, 0x64, 0x5f, 0x69, 0x6d, + 0x61, 0x67, 0x65, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x70, 0x72, 0x6f, 0x6d, 0x70, 0x74, 0x20, 0x3d, 0x20, + 0x60, 0x41, 0x20, 0x63, 0x68, 0x61, 0x74, 0x20, 0x62, 0x65, 0x74, 0x77, + 0x65, 0x65, 0x6e, 0x20, 0x61, 0x20, 0x63, 0x75, 0x72, 0x69, 0x6f, 0x75, + 0x73, 0x20, 0x68, 0x75, 0x6d, 0x61, 0x6e, 0x20, 0x61, 0x6e, 0x64, 0x20, + 0x61, 0x6e, 0x20, 0x61, 0x72, 0x74, 0x69, 0x66, 0x69, 0x63, 0x69, 0x61, + 0x6c, 0x20, 0x69, 0x6e, 0x74, 0x65, 0x6c, 0x6c, 0x69, 0x67, 0x65, 0x6e, + 0x63, 0x65, 0x20, 0x61, 0x73, 0x73, 0x69, 0x73, 0x74, 0x61, 0x6e, 0x74, + 0x2e, 0x20, 0x54, 0x68, 0x65, 0x20, 0x61, 0x73, 0x73, 0x69, 0x73, 0x74, + 0x61, 0x6e, 0x74, 0x20, 0x67, 0x69, 0x76, 0x65, 0x73, 0x20, 0x68, 0x65, + 0x6c, 0x70, 0x66, 0x75, 0x6c, 0x2c, 0x20, 0x64, 0x65, 0x74, 0x61, 0x69, + 0x6c, 0x65, 0x64, 0x2c, 0x20, 0x61, 0x6e, 0x64, 0x20, 0x70, 0x6f, 0x6c, + 0x69, 0x74, 0x65, 0x20, 0x61, 0x6e, 0x73, 0x77, 0x65, 0x72, 0x73, 0x20, + 0x74, 0x6f, 0x20, 0x74, 0x68, 0x65, 0x20, 0x68, 0x75, 0x6d, 0x61, 0x6e, + 0x27, 0x73, 0x20, 0x71, 0x75, 0x65, 0x73, 0x74, 0x69, 0x6f, 0x6e, 0x73, + 0x2e, 0x5c, 0x6e, 0x55, 0x53, 0x45, 0x52, 0x3a, 0x5b, 0x69, 0x6d, 0x67, + 0x2d, 0x31, 0x30, 0x5d, 0x24, 0x7b, 0x6d, 0x73, 0x67, 0x7d, 0x5c, 0x6e, + 0x41, 0x53, 0x53, 0x49, 0x53, 0x54, 0x41, 0x4e, 0x54, 0x3a, 0x60, 0x3b, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x61, 0x77, 0x61, 0x69, 0x74, 0x20, 0x72, 0x75, 0x6e, + 0x4c, 0x6c, 0x61, 0x6d, 0x61, 0x28, 0x70, 0x72, 0x6f, 0x6d, 0x70, 0x74, + 0x2c, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x2e, 0x2e, 0x2e, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x2e, 0x76, 0x61, + 0x6c, 0x75, 0x65, 0x2c, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x73, 0x6c, 0x6f, 0x74, 0x5f, 0x69, 0x64, 0x3a, 0x20, 0x73, 0x6c, + 0x6f, 0x74, 0x5f, 0x69, 0x64, 0x2c, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x73, 0x74, 0x6f, 0x70, 0x3a, 0x20, 0x5b, 0x22, 0x3c, + 0x2f, 0x73, 0x3e, 0x22, 0x2c, 0x20, 0x74, 0x65, 0x6d, 0x70, 0x6c, 0x61, + 0x74, 0x65, 0x28, 0x22, 0x7b, 0x7b, 0x63, 0x68, 0x61, 0x72, 0x7d, 0x7d, + 0x3a, 0x22, 0x29, 0x2c, 0x20, 0x74, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, + 0x65, 0x28, 0x22, 0x7b, 0x7b, 0x75, 0x73, 0x65, 0x72, 0x7d, 0x7d, 0x3a, + 0x22, 0x29, 0x5d, 0x2c, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, + 0x2c, 0x20, 0x22, 0x7b, 0x7b, 0x63, 0x68, 0x61, 0x72, 0x7d, 0x7d, 0x22, + 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x72, 0x75, 0x6e, 0x43, + 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x3d, 0x20, + 0x28, 0x29, 0x20, 0x3d, 0x3e, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, + 0x6c, 0x6c, 0x65, 0x72, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x29, 0x20, + 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, + 0x6e, 0x73, 0x6f, 0x6c, 0x65, 0x2e, 0x6c, 0x6f, 0x67, 0x28, 0x27, 0x61, + 0x6c, 0x72, 0x65, 0x61, 0x64, 0x79, 0x20, 0x72, 0x75, 0x6e, 0x6e, 0x69, + 0x6e, 0x67, 0x2e, 0x2e, 0x2e, 0x27, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x3b, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x7b, 0x20, 0x70, + 0x72, 0x6f, 0x6d, 0x70, 0x74, 0x20, 0x7d, 0x20, 0x3d, 0x20, 0x73, 0x65, + 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3b, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x74, 0x72, 0x61, 0x6e, 0x73, + 0x63, 0x72, 0x69, 0x70, 0x74, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x28, + 0x5b, 0x2e, 0x2e, 0x2e, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x2e, + 0x76, 0x61, 0x6c, 0x75, 0x65, 0x2e, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x63, + 0x72, 0x69, 0x70, 0x74, 0x2c, 0x20, 0x5b, 0x22, 0x22, 0x2c, 0x20, 0x70, + 0x72, 0x6f, 0x6d, 0x70, 0x74, 0x5d, 0x5d, 0x29, 0x3b, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x72, 0x75, 0x6e, 0x4c, 0x6c, 0x61, 0x6d, 0x61, + 0x28, 0x70, 0x72, 0x6f, 0x6d, 0x70, 0x74, 0x2c, 0x20, 0x7b, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x2e, 0x2e, 0x2e, 0x70, 0x61, + 0x72, 0x61, 0x6d, 0x73, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x2c, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x73, 0x6c, 0x6f, 0x74, + 0x5f, 0x69, 0x64, 0x3a, 0x20, 0x73, 0x6c, 0x6f, 0x74, 0x5f, 0x69, 0x64, + 0x2c, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x73, 0x74, + 0x6f, 0x70, 0x3a, 0x20, 0x5b, 0x5d, 0x2c, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x7d, 0x2c, 0x20, 0x22, 0x22, 0x29, 0x2e, 0x66, 0x69, 0x6e, + 0x61, 0x6c, 0x6c, 0x79, 0x28, 0x28, 0x29, 0x20, 0x3d, 0x3e, 0x20, 0x7b, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x73, 0x65, 0x73, + 0x73, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x2e, 0x70, + 0x72, 0x6f, 0x6d, 0x70, 0x74, 0x20, 0x3d, 0x20, 0x73, 0x65, 0x73, 0x73, + 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x2e, 0x74, 0x72, + 0x61, 0x6e, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x2e, 0x6d, 0x61, 0x70, + 0x28, 0x28, 0x5b, 0x5f, 0x2c, 0x20, 0x64, 0x61, 0x74, 0x61, 0x5d, 0x29, + 0x20, 0x3d, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x41, 0x72, 0x72, 0x61, 0x79, 0x2e, 0x69, 0x73, 0x41, 0x72, + 0x72, 0x61, 0x79, 0x28, 0x64, 0x61, 0x74, 0x61, 0x29, 0x20, 0x3f, 0x20, + 0x64, 0x61, 0x74, 0x61, 0x2e, 0x6d, 0x61, 0x70, 0x28, 0x6d, 0x73, 0x67, + 0x20, 0x3d, 0x3e, 0x20, 0x6d, 0x73, 0x67, 0x2e, 0x63, 0x6f, 0x6e, 0x74, + 0x65, 0x6e, 0x74, 0x29, 0x2e, 0x6a, 0x6f, 0x69, 0x6e, 0x28, 0x27, 0x27, + 0x29, 0x20, 0x3a, 0x20, 0x64, 0x61, 0x74, 0x61, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x29, 0x2e, 0x6a, 0x6f, 0x69, 0x6e, 0x28, + 0x27, 0x27, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x61, 0x6c, + 0x75, 0x65, 0x2e, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x63, 0x72, 0x69, 0x70, + 0x74, 0x20, 0x3d, 0x20, 0x5b, 0x5d, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x7d, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x73, 0x74, + 0x6f, 0x70, 0x20, 0x3d, 0x20, 0x28, 0x65, 0x29, 0x20, 0x3d, 0x3e, 0x20, + 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x65, 0x2e, 0x70, 0x72, + 0x65, 0x76, 0x65, 0x6e, 0x74, 0x44, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, + 0x28, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, + 0x20, 0x28, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, + 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, + 0x6c, 0x6c, 0x65, 0x72, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x2e, 0x61, + 0x62, 0x6f, 0x72, 0x74, 0x28, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, + 0x65, 0x72, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x20, 0x3d, 0x20, 0x6e, + 0x75, 0x6c, 0x6c, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x72, 0x65, 0x73, 0x65, 0x74, 0x20, + 0x3d, 0x20, 0x28, 0x65, 0x29, 0x20, 0x3d, 0x3e, 0x20, 0x7b, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x73, 0x74, 0x6f, 0x70, 0x28, 0x65, 0x29, + 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x74, 0x72, 0x61, 0x6e, + 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, + 0x28, 0x5b, 0x5d, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x75, + 0x70, 0x6c, 0x6f, 0x61, 0x64, 0x49, 0x6d, 0x61, 0x67, 0x65, 0x20, 0x3d, + 0x20, 0x28, 0x65, 0x29, 0x20, 0x3d, 0x3e, 0x20, 0x7b, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x65, 0x2e, 0x70, 0x72, 0x65, 0x76, 0x65, 0x6e, + 0x74, 0x44, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x28, 0x29, 0x3b, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x64, 0x6f, 0x63, 0x75, 0x6d, 0x65, + 0x6e, 0x74, 0x2e, 0x67, 0x65, 0x74, 0x45, 0x6c, 0x65, 0x6d, 0x65, 0x6e, + 0x74, 0x42, 0x79, 0x49, 0x64, 0x28, 0x22, 0x66, 0x69, 0x6c, 0x65, 0x49, + 0x6e, 0x70, 0x75, 0x74, 0x22, 0x29, 0x2e, 0x63, 0x6c, 0x69, 0x63, 0x6b, + 0x28, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x64, 0x6f, + 0x63, 0x75, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x67, 0x65, 0x74, 0x45, 0x6c, + 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x42, 0x79, 0x49, 0x64, 0x28, 0x22, 0x66, + 0x69, 0x6c, 0x65, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x22, 0x29, 0x2e, 0x61, + 0x64, 0x64, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x4c, 0x69, 0x73, 0x74, 0x65, + 0x6e, 0x65, 0x72, 0x28, 0x22, 0x63, 0x68, 0x61, 0x6e, 0x67, 0x65, 0x22, + 0x2c, 0x20, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x28, + 0x65, 0x76, 0x65, 0x6e, 0x74, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x73, + 0x65, 0x6c, 0x65, 0x63, 0x74, 0x65, 0x64, 0x46, 0x69, 0x6c, 0x65, 0x20, + 0x3d, 0x20, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x2e, 0x74, 0x61, 0x72, 0x67, + 0x65, 0x74, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x73, 0x5b, 0x30, 0x5d, 0x3b, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, + 0x28, 0x73, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x65, 0x64, 0x46, 0x69, 0x6c, + 0x65, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x72, 0x65, 0x61, + 0x64, 0x65, 0x72, 0x20, 0x3d, 0x20, 0x6e, 0x65, 0x77, 0x20, 0x46, 0x69, + 0x6c, 0x65, 0x52, 0x65, 0x61, 0x64, 0x65, 0x72, 0x28, 0x29, 0x3b, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, + 0x61, 0x64, 0x65, 0x72, 0x2e, 0x6f, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x20, + 0x3d, 0x20, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x28, + 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x69, 0x6d, + 0x61, 0x67, 0x65, 0x5f, 0x64, 0x61, 0x74, 0x61, 0x20, 0x3d, 0x20, 0x72, + 0x65, 0x61, 0x64, 0x65, 0x72, 0x2e, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, + 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x61, + 0x6c, 0x75, 0x65, 0x20, 0x3d, 0x20, 0x7b, 0x20, 0x2e, 0x2e, 0x2e, 0x73, + 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, + 0x2c, 0x20, 0x69, 0x6d, 0x61, 0x67, 0x65, 0x5f, 0x73, 0x65, 0x6c, 0x65, + 0x63, 0x74, 0x65, 0x64, 0x3a, 0x20, 0x69, 0x6d, 0x61, 0x67, 0x65, 0x5f, + 0x64, 0x61, 0x74, 0x61, 0x20, 0x7d, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x70, 0x61, 0x72, 0x61, + 0x6d, 0x73, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x20, 0x3d, 0x20, 0x7b, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x2e, 0x2e, 0x2e, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, + 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x2c, 0x20, 0x69, 0x6d, 0x61, 0x67, + 0x65, 0x5f, 0x64, 0x61, 0x74, 0x61, 0x3a, 0x20, 0x5b, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x7b, 0x20, 0x64, 0x61, 0x74, 0x61, 0x3a, 0x20, 0x69, 0x6d, + 0x61, 0x67, 0x65, 0x5f, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x72, 0x65, 0x70, + 0x6c, 0x61, 0x63, 0x65, 0x28, 0x2f, 0x64, 0x61, 0x74, 0x61, 0x3a, 0x69, + 0x6d, 0x61, 0x67, 0x65, 0x5c, 0x2f, 0x5b, 0x5e, 0x3b, 0x5d, 0x2b, 0x3b, + 0x62, 0x61, 0x73, 0x65, 0x36, 0x34, 0x2c, 0x2f, 0x2c, 0x20, 0x27, 0x27, + 0x29, 0x2c, 0x20, 0x69, 0x64, 0x3a, 0x20, 0x31, 0x30, 0x20, 0x7d, 0x5d, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x7d, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x73, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x65, 0x64, 0x5f, 0x69, + 0x6d, 0x61, 0x67, 0x65, 0x20, 0x3d, 0x20, 0x74, 0x72, 0x75, 0x65, 0x3b, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, + 0x65, 0x61, 0x64, 0x65, 0x72, 0x2e, 0x72, 0x65, 0x61, 0x64, 0x41, 0x73, + 0x44, 0x61, 0x74, 0x61, 0x55, 0x52, 0x4c, 0x28, 0x73, 0x65, 0x6c, 0x65, + 0x63, 0x74, 0x65, 0x64, 0x46, 0x69, 0x6c, 0x65, 0x29, 0x3b, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x7d, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x7d, + 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, + 0x6f, 0x6e, 0x20, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x49, 0x6e, + 0x70, 0x75, 0x74, 0x28, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x6d, 0x65, 0x73, 0x73, + 0x61, 0x67, 0x65, 0x20, 0x3d, 0x20, 0x75, 0x73, 0x65, 0x53, 0x69, 0x67, + 0x6e, 0x61, 0x6c, 0x28, 0x22, 0x22, 0x29, 0x0a, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x73, 0x75, 0x62, + 0x6d, 0x69, 0x74, 0x20, 0x3d, 0x20, 0x28, 0x65, 0x29, 0x20, 0x3d, 0x3e, + 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x73, + 0x74, 0x6f, 0x70, 0x28, 0x65, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x63, 0x68, 0x61, 0x74, 0x28, 0x6d, 0x65, 0x73, + 0x73, 0x61, 0x67, 0x65, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x29, 0x3b, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x6d, 0x65, 0x73, + 0x73, 0x61, 0x67, 0x65, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x20, 0x3d, + 0x20, 0x22, 0x22, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, + 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, + 0x74, 0x20, 0x65, 0x6e, 0x74, 0x65, 0x72, 0x53, 0x75, 0x62, 0x6d, 0x69, + 0x74, 0x73, 0x20, 0x3d, 0x20, 0x28, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x29, + 0x20, 0x3d, 0x3e, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x2e, + 0x77, 0x68, 0x69, 0x63, 0x68, 0x20, 0x3d, 0x3d, 0x3d, 0x20, 0x31, 0x33, + 0x20, 0x26, 0x26, 0x20, 0x21, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x2e, 0x73, + 0x68, 0x69, 0x66, 0x74, 0x4b, 0x65, 0x79, 0x29, 0x20, 0x7b, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x73, 0x75, 0x62, + 0x6d, 0x69, 0x74, 0x28, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x29, 0x3b, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x68, 0x74, 0x6d, 0x6c, + 0x60, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x66, + 0x6f, 0x72, 0x6d, 0x20, 0x6f, 0x6e, 0x73, 0x75, 0x62, 0x6d, 0x69, 0x74, + 0x3d, 0x24, 0x7b, 0x73, 0x75, 0x62, 0x6d, 0x69, 0x74, 0x7d, 0x3e, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x64, + 0x69, 0x76, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x3c, 0x74, 0x65, 0x78, 0x74, 0x61, 0x72, 0x65, + 0x61, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6c, 0x61, 0x73, 0x73, 0x4e, 0x61, + 0x6d, 0x65, 0x3d, 0x24, 0x7b, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, + 0x69, 0x6e, 0x67, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x20, 0x3f, 0x20, + 0x22, 0x6c, 0x6f, 0x61, 0x64, 0x69, 0x6e, 0x67, 0x22, 0x20, 0x3a, 0x20, + 0x6e, 0x75, 0x6c, 0x6c, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x6f, 0x6e, 0x69, + 0x6e, 0x70, 0x75, 0x74, 0x3d, 0x24, 0x7b, 0x28, 0x65, 0x29, 0x20, 0x3d, + 0x3e, 0x20, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x2e, 0x76, 0x61, + 0x6c, 0x75, 0x65, 0x20, 0x3d, 0x20, 0x65, 0x2e, 0x74, 0x61, 0x72, 0x67, + 0x65, 0x74, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x7d, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x6f, 0x6e, 0x6b, 0x65, 0x79, 0x70, 0x72, 0x65, 0x73, 0x73, 0x3d, + 0x24, 0x7b, 0x65, 0x6e, 0x74, 0x65, 0x72, 0x53, 0x75, 0x62, 0x6d, 0x69, + 0x74, 0x73, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x70, 0x6c, 0x61, 0x63, 0x65, + 0x68, 0x6f, 0x6c, 0x64, 0x65, 0x72, 0x3d, 0x22, 0x53, 0x61, 0x79, 0x20, + 0x73, 0x6f, 0x6d, 0x65, 0x74, 0x68, 0x69, 0x6e, 0x67, 0x2e, 0x2e, 0x2e, + 0x22, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, 0x6f, 0x77, 0x73, 0x3d, 0x32, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x74, 0x79, 0x70, 0x65, 0x3d, 0x22, 0x74, 0x65, 0x78, + 0x74, 0x22, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3d, + 0x22, 0x24, 0x7b, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x7d, 0x22, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x2f, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x3c, 0x2f, 0x64, 0x69, 0x76, 0x3e, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x64, 0x69, 0x76, 0x20, + 0x63, 0x6c, 0x61, 0x73, 0x73, 0x3d, 0x22, 0x72, 0x69, 0x67, 0x68, 0x74, + 0x22, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x3c, 0x62, 0x75, 0x74, 0x74, 0x6f, 0x6e, 0x20, 0x74, + 0x79, 0x70, 0x65, 0x3d, 0x22, 0x73, 0x75, 0x62, 0x6d, 0x69, 0x74, 0x22, + 0x20, 0x64, 0x69, 0x73, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x3d, 0x24, 0x7b, + 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x76, + 0x61, 0x6c, 0x75, 0x65, 0x7d, 0x3e, 0x53, 0x65, 0x6e, 0x64, 0x3c, 0x2f, + 0x62, 0x75, 0x74, 0x74, 0x6f, 0x6e, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x62, 0x75, 0x74, + 0x74, 0x6f, 0x6e, 0x20, 0x6f, 0x6e, 0x63, 0x6c, 0x69, 0x63, 0x6b, 0x3d, + 0x24, 0x7b, 0x75, 0x70, 0x6c, 0x6f, 0x61, 0x64, 0x49, 0x6d, 0x61, 0x67, + 0x65, 0x7d, 0x3e, 0x55, 0x70, 0x6c, 0x6f, 0x61, 0x64, 0x20, 0x49, 0x6d, + 0x61, 0x67, 0x65, 0x3c, 0x2f, 0x62, 0x75, 0x74, 0x74, 0x6f, 0x6e, 0x3e, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x3c, 0x62, 0x75, 0x74, 0x74, 0x6f, 0x6e, 0x20, 0x6f, 0x6e, 0x63, + 0x6c, 0x69, 0x63, 0x6b, 0x3d, 0x24, 0x7b, 0x73, 0x74, 0x6f, 0x70, 0x7d, + 0x20, 0x64, 0x69, 0x73, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x3d, 0x24, 0x7b, + 0x21, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6e, 0x67, 0x2e, + 0x76, 0x61, 0x6c, 0x75, 0x65, 0x7d, 0x3e, 0x53, 0x74, 0x6f, 0x70, 0x3c, + 0x2f, 0x62, 0x75, 0x74, 0x74, 0x6f, 0x6e, 0x3e, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x62, 0x75, + 0x74, 0x74, 0x6f, 0x6e, 0x20, 0x6f, 0x6e, 0x63, 0x6c, 0x69, 0x63, 0x6b, + 0x3d, 0x24, 0x7b, 0x72, 0x65, 0x73, 0x65, 0x74, 0x7d, 0x3e, 0x52, 0x65, + 0x73, 0x65, 0x74, 0x3c, 0x2f, 0x62, 0x75, 0x74, 0x74, 0x6f, 0x6e, 0x3e, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, + 0x2f, 0x64, 0x69, 0x76, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x3c, 0x2f, 0x66, 0x6f, 0x72, 0x6d, 0x3e, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x60, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, + 0x6e, 0x20, 0x43, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x69, 0x6f, 0x6e, + 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x73, 0x28, 0x29, 0x20, 0x7b, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, + 0x20, 0x73, 0x75, 0x62, 0x6d, 0x69, 0x74, 0x20, 0x3d, 0x20, 0x28, 0x65, + 0x29, 0x20, 0x3d, 0x3e, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x73, 0x74, 0x6f, 0x70, 0x28, 0x65, 0x29, 0x3b, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, 0x75, 0x6e, 0x43, + 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x29, 0x3b, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x68, 0x74, + 0x6d, 0x6c, 0x60, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x3c, 0x64, 0x69, 0x76, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x3c, 0x62, 0x75, 0x74, 0x74, 0x6f, 0x6e, 0x20, + 0x6f, 0x6e, 0x63, 0x6c, 0x69, 0x63, 0x6b, 0x3d, 0x24, 0x7b, 0x73, 0x75, + 0x62, 0x6d, 0x69, 0x74, 0x7d, 0x20, 0x74, 0x79, 0x70, 0x65, 0x3d, 0x22, + 0x62, 0x75, 0x74, 0x74, 0x6f, 0x6e, 0x22, 0x20, 0x64, 0x69, 0x73, 0x61, + 0x62, 0x6c, 0x65, 0x64, 0x3d, 0x24, 0x7b, 0x67, 0x65, 0x6e, 0x65, 0x72, + 0x61, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x7d, + 0x3e, 0x53, 0x74, 0x61, 0x72, 0x74, 0x3c, 0x2f, 0x62, 0x75, 0x74, 0x74, + 0x6f, 0x6e, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x3c, 0x62, 0x75, 0x74, 0x74, 0x6f, 0x6e, 0x20, 0x6f, 0x6e, + 0x63, 0x6c, 0x69, 0x63, 0x6b, 0x3d, 0x24, 0x7b, 0x73, 0x74, 0x6f, 0x70, + 0x7d, 0x20, 0x64, 0x69, 0x73, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x3d, 0x24, + 0x7b, 0x21, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6e, 0x67, + 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x7d, 0x3e, 0x53, 0x74, 0x6f, 0x70, + 0x3c, 0x2f, 0x62, 0x75, 0x74, 0x74, 0x6f, 0x6e, 0x3e, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x62, 0x75, 0x74, + 0x74, 0x6f, 0x6e, 0x20, 0x6f, 0x6e, 0x63, 0x6c, 0x69, 0x63, 0x6b, 0x3d, + 0x24, 0x7b, 0x72, 0x65, 0x73, 0x65, 0x74, 0x7d, 0x3e, 0x52, 0x65, 0x73, + 0x65, 0x74, 0x3c, 0x2f, 0x62, 0x75, 0x74, 0x74, 0x6f, 0x6e, 0x3e, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x2f, 0x64, 0x69, + 0x76, 0x3e, 0x60, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x43, 0x68, + 0x61, 0x74, 0x4c, 0x6f, 0x67, 0x20, 0x3d, 0x20, 0x28, 0x70, 0x72, 0x6f, + 0x70, 0x73, 0x29, 0x20, 0x3d, 0x3e, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x6d, 0x65, 0x73, + 0x73, 0x61, 0x67, 0x65, 0x73, 0x20, 0x3d, 0x20, 0x73, 0x65, 0x73, 0x73, + 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x2e, 0x74, 0x72, + 0x61, 0x6e, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x3b, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x63, 0x6f, + 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x20, 0x3d, 0x20, 0x75, 0x73, + 0x65, 0x52, 0x65, 0x66, 0x28, 0x6e, 0x75, 0x6c, 0x6c, 0x29, 0x0a, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x75, 0x73, 0x65, 0x45, 0x66, 0x66, + 0x65, 0x63, 0x74, 0x28, 0x28, 0x29, 0x20, 0x3d, 0x3e, 0x20, 0x7b, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x2f, 0x2f, 0x20, 0x73, + 0x63, 0x72, 0x6f, 0x6c, 0x6c, 0x20, 0x74, 0x6f, 0x20, 0x62, 0x6f, 0x74, + 0x74, 0x6f, 0x6d, 0x20, 0x28, 0x69, 0x66, 0x20, 0x6e, 0x65, 0x65, 0x64, + 0x65, 0x64, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x70, 0x61, 0x72, 0x65, 0x6e, 0x74, + 0x20, 0x3d, 0x20, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, + 0x2e, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x74, 0x2e, 0x70, 0x61, 0x72, + 0x65, 0x6e, 0x74, 0x45, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x3b, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, + 0x70, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x20, 0x26, 0x26, 0x20, 0x70, 0x61, + 0x72, 0x65, 0x6e, 0x74, 0x2e, 0x73, 0x63, 0x72, 0x6f, 0x6c, 0x6c, 0x48, + 0x65, 0x69, 0x67, 0x68, 0x74, 0x20, 0x3c, 0x3d, 0x20, 0x70, 0x61, 0x72, + 0x65, 0x6e, 0x74, 0x2e, 0x73, 0x63, 0x72, 0x6f, 0x6c, 0x6c, 0x54, 0x6f, + 0x70, 0x20, 0x2b, 0x20, 0x70, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x2e, 0x6f, + 0x66, 0x66, 0x73, 0x65, 0x74, 0x48, 0x65, 0x69, 0x67, 0x68, 0x74, 0x20, + 0x2b, 0x20, 0x33, 0x30, 0x30, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x70, 0x61, 0x72, 0x65, 0x6e, + 0x74, 0x2e, 0x73, 0x63, 0x72, 0x6f, 0x6c, 0x6c, 0x54, 0x6f, 0x28, 0x30, + 0x2c, 0x20, 0x70, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x2e, 0x73, 0x63, 0x72, + 0x6f, 0x6c, 0x6c, 0x48, 0x65, 0x69, 0x67, 0x68, 0x74, 0x29, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x7d, 0x2c, 0x20, 0x5b, 0x6d, 0x65, 0x73, 0x73, 0x61, + 0x67, 0x65, 0x73, 0x5d, 0x29, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x69, 0x73, 0x43, 0x6f, 0x6d, + 0x70, 0x6c, 0x65, 0x74, 0x69, 0x6f, 0x6e, 0x4d, 0x6f, 0x64, 0x65, 0x20, + 0x3d, 0x20, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x61, + 0x6c, 0x75, 0x65, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x20, 0x3d, 0x3d, 0x3d, + 0x20, 0x27, 0x63, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x69, 0x6f, 0x6e, + 0x27, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, + 0x74, 0x20, 0x63, 0x68, 0x61, 0x74, 0x4c, 0x69, 0x6e, 0x65, 0x20, 0x3d, + 0x20, 0x28, 0x5b, 0x75, 0x73, 0x65, 0x72, 0x2c, 0x20, 0x64, 0x61, 0x74, + 0x61, 0x5d, 0x2c, 0x20, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x29, 0x20, 0x3d, + 0x3e, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x6c, 0x65, 0x74, 0x20, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, + 0x74, 0x20, 0x69, 0x73, 0x41, 0x72, 0x72, 0x61, 0x79, 0x4d, 0x65, 0x73, + 0x73, 0x61, 0x67, 0x65, 0x20, 0x3d, 0x20, 0x41, 0x72, 0x72, 0x61, 0x79, + 0x2e, 0x69, 0x73, 0x41, 0x72, 0x72, 0x61, 0x79, 0x28, 0x64, 0x61, 0x74, + 0x61, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, + 0x66, 0x20, 0x28, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x2e, 0x76, 0x61, + 0x6c, 0x75, 0x65, 0x2e, 0x6e, 0x5f, 0x70, 0x72, 0x6f, 0x62, 0x73, 0x20, + 0x3e, 0x20, 0x30, 0x20, 0x26, 0x26, 0x20, 0x69, 0x73, 0x41, 0x72, 0x72, + 0x61, 0x79, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x29, 0x20, 0x7b, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x6d, + 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x20, 0x3d, 0x20, 0x68, 0x74, 0x6d, + 0x6c, 0x60, 0x3c, 0x24, 0x7b, 0x50, 0x72, 0x6f, 0x62, 0x61, 0x62, 0x69, + 0x6c, 0x69, 0x74, 0x69, 0x65, 0x73, 0x7d, 0x20, 0x64, 0x61, 0x74, 0x61, + 0x3d, 0x24, 0x7b, 0x64, 0x61, 0x74, 0x61, 0x7d, 0x20, 0x2f, 0x3e, 0x60, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x20, 0x65, + 0x6c, 0x73, 0x65, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x74, 0x65, + 0x78, 0x74, 0x20, 0x3d, 0x20, 0x69, 0x73, 0x41, 0x72, 0x72, 0x61, 0x79, + 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x20, 0x3f, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x64, 0x61, + 0x74, 0x61, 0x2e, 0x6d, 0x61, 0x70, 0x28, 0x6d, 0x73, 0x67, 0x20, 0x3d, + 0x3e, 0x20, 0x6d, 0x73, 0x67, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, + 0x74, 0x29, 0x2e, 0x6a, 0x6f, 0x69, 0x6e, 0x28, 0x27, 0x27, 0x29, 0x2e, + 0x72, 0x65, 0x70, 0x6c, 0x61, 0x63, 0x65, 0x28, 0x2f, 0x5e, 0x5c, 0x73, + 0x2b, 0x2f, 0x2c, 0x20, 0x27, 0x27, 0x29, 0x20, 0x3a, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x64, 0x61, + 0x74, 0x61, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x20, 0x3d, 0x20, + 0x69, 0x73, 0x43, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x69, 0x6f, 0x6e, + 0x4d, 0x6f, 0x64, 0x65, 0x20, 0x3f, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x74, 0x65, 0x78, 0x74, 0x20, + 0x3a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x68, 0x74, 0x6d, 0x6c, 0x60, 0x3c, 0x24, 0x7b, 0x4d, 0x61, + 0x72, 0x6b, 0x64, 0x6f, 0x77, 0x6e, 0x69, 0x73, 0x68, 0x7d, 0x20, 0x74, + 0x65, 0x78, 0x74, 0x3d, 0x24, 0x7b, 0x74, 0x65, 0x6d, 0x70, 0x6c, 0x61, + 0x74, 0x65, 0x28, 0x74, 0x65, 0x78, 0x74, 0x29, 0x7d, 0x20, 0x2f, 0x3e, + 0x60, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, + 0x75, 0x73, 0x65, 0x72, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, + 0x20, 0x68, 0x74, 0x6d, 0x6c, 0x60, 0x3c, 0x70, 0x20, 0x6b, 0x65, 0x79, + 0x3d, 0x24, 0x7b, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x7d, 0x3e, 0x3c, 0x73, + 0x74, 0x72, 0x6f, 0x6e, 0x67, 0x3e, 0x24, 0x7b, 0x74, 0x65, 0x6d, 0x70, + 0x6c, 0x61, 0x74, 0x65, 0x28, 0x75, 0x73, 0x65, 0x72, 0x29, 0x7d, 0x3a, + 0x3c, 0x2f, 0x73, 0x74, 0x72, 0x6f, 0x6e, 0x67, 0x3e, 0x20, 0x24, 0x7b, + 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x7d, 0x3c, 0x2f, 0x70, 0x3e, + 0x60, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x20, + 0x65, 0x6c, 0x73, 0x65, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, + 0x69, 0x73, 0x43, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x69, 0x6f, 0x6e, + 0x4d, 0x6f, 0x64, 0x65, 0x20, 0x3f, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x68, 0x74, 0x6d, 0x6c, 0x60, + 0x3c, 0x73, 0x70, 0x61, 0x6e, 0x20, 0x6b, 0x65, 0x79, 0x3d, 0x24, 0x7b, + 0x69, 0x6e, 0x64, 0x65, 0x78, 0x7d, 0x3e, 0x24, 0x7b, 0x6d, 0x65, 0x73, + 0x73, 0x61, 0x67, 0x65, 0x7d, 0x3c, 0x2f, 0x73, 0x70, 0x61, 0x6e, 0x3e, + 0x60, 0x20, 0x3a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x68, 0x74, 0x6d, 0x6c, 0x60, 0x3c, 0x70, 0x20, + 0x6b, 0x65, 0x79, 0x3d, 0x24, 0x7b, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x7d, + 0x3e, 0x24, 0x7b, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x7d, 0x3c, + 0x2f, 0x70, 0x3e, 0x60, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x3b, 0x0a, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, + 0x20, 0x68, 0x61, 0x6e, 0x64, 0x6c, 0x65, 0x43, 0x6f, 0x6d, 0x70, 0x6c, + 0x65, 0x74, 0x69, 0x6f, 0x6e, 0x45, 0x64, 0x69, 0x74, 0x20, 0x3d, 0x20, + 0x28, 0x65, 0x29, 0x20, 0x3d, 0x3e, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, + 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x6d, 0x70, + 0x74, 0x20, 0x3d, 0x20, 0x65, 0x2e, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, + 0x2e, 0x69, 0x6e, 0x6e, 0x65, 0x72, 0x54, 0x65, 0x78, 0x74, 0x3b, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x73, 0x65, 0x73, 0x73, + 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x2e, 0x74, 0x72, + 0x61, 0x6e, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x20, 0x3d, 0x20, 0x5b, + 0x5d, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, + 0x20, 0x68, 0x74, 0x6d, 0x6c, 0x60, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x3c, 0x64, 0x69, 0x76, 0x20, 0x69, 0x64, 0x3d, 0x22, + 0x63, 0x68, 0x61, 0x74, 0x22, 0x20, 0x72, 0x65, 0x66, 0x3d, 0x24, 0x7b, + 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x7d, 0x20, 0x6b, + 0x65, 0x79, 0x3d, 0x24, 0x7b, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, + 0x73, 0x2e, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x7d, 0x3e, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x69, 0x6d, + 0x67, 0x20, 0x73, 0x74, 0x79, 0x6c, 0x65, 0x3d, 0x22, 0x77, 0x69, 0x64, + 0x74, 0x68, 0x3a, 0x20, 0x36, 0x30, 0x25, 0x3b, 0x24, 0x7b, 0x21, 0x73, + 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, + 0x2e, 0x69, 0x6d, 0x61, 0x67, 0x65, 0x5f, 0x73, 0x65, 0x6c, 0x65, 0x63, + 0x74, 0x65, 0x64, 0x20, 0x3f, 0x20, 0x60, 0x64, 0x69, 0x73, 0x70, 0x6c, + 0x61, 0x79, 0x3a, 0x20, 0x6e, 0x6f, 0x6e, 0x65, 0x3b, 0x60, 0x20, 0x3a, + 0x20, 0x60, 0x60, 0x7d, 0x22, 0x20, 0x73, 0x72, 0x63, 0x3d, 0x22, 0x24, + 0x7b, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x61, 0x6c, + 0x75, 0x65, 0x2e, 0x69, 0x6d, 0x61, 0x67, 0x65, 0x5f, 0x73, 0x65, 0x6c, + 0x65, 0x63, 0x74, 0x65, 0x64, 0x7d, 0x22, 0x2f, 0x3e, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x73, 0x70, 0x61, + 0x6e, 0x20, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x65, 0x64, 0x69, + 0x74, 0x61, 0x62, 0x6c, 0x65, 0x3d, 0x24, 0x7b, 0x69, 0x73, 0x43, 0x6f, + 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x69, 0x6f, 0x6e, 0x4d, 0x6f, 0x64, 0x65, + 0x7d, 0x20, 0x72, 0x65, 0x66, 0x3d, 0x24, 0x7b, 0x63, 0x6f, 0x6e, 0x74, + 0x61, 0x69, 0x6e, 0x65, 0x72, 0x7d, 0x20, 0x6f, 0x6e, 0x69, 0x6e, 0x70, + 0x75, 0x74, 0x3d, 0x24, 0x7b, 0x68, 0x61, 0x6e, 0x64, 0x6c, 0x65, 0x43, + 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x69, 0x6f, 0x6e, 0x45, 0x64, 0x69, + 0x74, 0x7d, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x24, 0x7b, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, + 0x65, 0x73, 0x2e, 0x66, 0x6c, 0x61, 0x74, 0x4d, 0x61, 0x70, 0x28, 0x63, + 0x68, 0x61, 0x74, 0x4c, 0x69, 0x6e, 0x65, 0x29, 0x7d, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x2f, 0x73, 0x70, + 0x61, 0x6e, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x3c, 0x2f, 0x64, 0x69, 0x76, 0x3e, 0x60, 0x3b, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x7d, 0x3b, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, + 0x73, 0x74, 0x20, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x46, 0x6f, 0x72, + 0x6d, 0x20, 0x3d, 0x20, 0x28, 0x70, 0x72, 0x6f, 0x70, 0x73, 0x29, 0x20, + 0x3d, 0x3e, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, + 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x53, + 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x20, 0x3d, 0x20, 0x28, 0x65, 0x6c, + 0x29, 0x20, 0x3d, 0x3e, 0x20, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, + 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x20, 0x3d, 0x20, 0x7b, 0x20, 0x2e, + 0x2e, 0x2e, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x61, + 0x6c, 0x75, 0x65, 0x2c, 0x20, 0x5b, 0x65, 0x6c, 0x2e, 0x74, 0x61, 0x72, + 0x67, 0x65, 0x74, 0x2e, 0x6e, 0x61, 0x6d, 0x65, 0x5d, 0x3a, 0x20, 0x65, + 0x6c, 0x2e, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x2e, 0x76, 0x61, 0x6c, + 0x75, 0x65, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, + 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x50, + 0x61, 0x72, 0x61, 0x6d, 0x73, 0x20, 0x3d, 0x20, 0x28, 0x65, 0x6c, 0x29, + 0x20, 0x3d, 0x3e, 0x20, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x2e, 0x76, + 0x61, 0x6c, 0x75, 0x65, 0x20, 0x3d, 0x20, 0x7b, 0x20, 0x2e, 0x2e, 0x2e, + 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, + 0x2c, 0x20, 0x5b, 0x65, 0x6c, 0x2e, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, + 0x2e, 0x6e, 0x61, 0x6d, 0x65, 0x5d, 0x3a, 0x20, 0x65, 0x6c, 0x2e, 0x74, + 0x61, 0x72, 0x67, 0x65, 0x74, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x20, + 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, + 0x74, 0x20, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x50, 0x61, 0x72, 0x61, + 0x6d, 0x73, 0x46, 0x6c, 0x6f, 0x61, 0x74, 0x20, 0x3d, 0x20, 0x28, 0x65, + 0x6c, 0x29, 0x20, 0x3d, 0x3e, 0x20, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, + 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x20, 0x3d, 0x20, 0x7b, 0x20, 0x2e, + 0x2e, 0x2e, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x2e, 0x76, 0x61, 0x6c, + 0x75, 0x65, 0x2c, 0x20, 0x5b, 0x65, 0x6c, 0x2e, 0x74, 0x61, 0x72, 0x67, + 0x65, 0x74, 0x2e, 0x6e, 0x61, 0x6d, 0x65, 0x5d, 0x3a, 0x20, 0x70, 0x61, + 0x72, 0x73, 0x65, 0x46, 0x6c, 0x6f, 0x61, 0x74, 0x28, 0x65, 0x6c, 0x2e, + 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, + 0x29, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, + 0x6e, 0x73, 0x74, 0x20, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x50, 0x61, + 0x72, 0x61, 0x6d, 0x73, 0x49, 0x6e, 0x74, 0x20, 0x3d, 0x20, 0x28, 0x65, + 0x6c, 0x29, 0x20, 0x3d, 0x3e, 0x20, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, + 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x20, 0x3d, 0x20, 0x7b, 0x20, 0x2e, + 0x2e, 0x2e, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x2e, 0x76, 0x61, 0x6c, + 0x75, 0x65, 0x2c, 0x20, 0x5b, 0x65, 0x6c, 0x2e, 0x74, 0x61, 0x72, 0x67, + 0x65, 0x74, 0x2e, 0x6e, 0x61, 0x6d, 0x65, 0x5d, 0x3a, 0x20, 0x4d, 0x61, + 0x74, 0x68, 0x2e, 0x66, 0x6c, 0x6f, 0x6f, 0x72, 0x28, 0x70, 0x61, 0x72, + 0x73, 0x65, 0x46, 0x6c, 0x6f, 0x61, 0x74, 0x28, 0x65, 0x6c, 0x2e, 0x74, + 0x61, 0x72, 0x67, 0x65, 0x74, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x29, + 0x29, 0x20, 0x7d, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, + 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x67, 0x72, 0x61, 0x6d, 0x6d, 0x61, 0x72, + 0x4a, 0x73, 0x6f, 0x6e, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x50, 0x72, + 0x6f, 0x70, 0x4f, 0x72, 0x64, 0x65, 0x72, 0x20, 0x3d, 0x20, 0x73, 0x69, + 0x67, 0x6e, 0x61, 0x6c, 0x28, 0x27, 0x27, 0x29, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x75, 0x70, 0x64, + 0x61, 0x74, 0x65, 0x47, 0x72, 0x61, 0x6d, 0x6d, 0x61, 0x72, 0x4a, 0x73, + 0x6f, 0x6e, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x50, 0x72, 0x6f, 0x70, + 0x4f, 0x72, 0x64, 0x65, 0x72, 0x20, 0x3d, 0x20, 0x28, 0x65, 0x6c, 0x29, + 0x20, 0x3d, 0x3e, 0x20, 0x67, 0x72, 0x61, 0x6d, 0x6d, 0x61, 0x72, 0x4a, + 0x73, 0x6f, 0x6e, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x50, 0x72, 0x6f, + 0x70, 0x4f, 0x72, 0x64, 0x65, 0x72, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, + 0x20, 0x3d, 0x20, 0x65, 0x6c, 0x2e, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, + 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x63, 0x6f, 0x6e, 0x76, 0x65, + 0x72, 0x74, 0x4a, 0x53, 0x4f, 0x4e, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, + 0x47, 0x72, 0x61, 0x6d, 0x6d, 0x61, 0x72, 0x20, 0x3d, 0x20, 0x28, 0x29, + 0x20, 0x3d, 0x3e, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x74, 0x72, 0x79, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, + 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x20, 0x3d, 0x20, 0x4a, 0x53, 0x4f, + 0x4e, 0x2e, 0x70, 0x61, 0x72, 0x73, 0x65, 0x28, 0x70, 0x61, 0x72, 0x61, + 0x6d, 0x73, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x2e, 0x67, 0x72, 0x61, + 0x6d, 0x6d, 0x61, 0x72, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x63, 0x6f, + 0x6e, 0x76, 0x65, 0x72, 0x74, 0x65, 0x72, 0x20, 0x3d, 0x20, 0x6e, 0x65, + 0x77, 0x20, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x43, 0x6f, 0x6e, 0x76, + 0x65, 0x72, 0x74, 0x65, 0x72, 0x28, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x67, 0x72, 0x61, 0x6d, 0x6d, + 0x61, 0x72, 0x4a, 0x73, 0x6f, 0x6e, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, + 0x50, 0x72, 0x6f, 0x70, 0x4f, 0x72, 0x64, 0x65, 0x72, 0x2e, 0x76, 0x61, + 0x6c, 0x75, 0x65, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x2e, 0x73, 0x70, 0x6c, 0x69, 0x74, + 0x28, 0x27, 0x2c, 0x27, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x2e, 0x72, 0x65, 0x64, + 0x75, 0x63, 0x65, 0x28, 0x28, 0x61, 0x63, 0x63, 0x2c, 0x20, 0x63, 0x75, + 0x72, 0x2c, 0x20, 0x69, 0x29, 0x20, 0x3d, 0x3e, 0x20, 0x28, 0x7b, 0x20, + 0x2e, 0x2e, 0x2e, 0x61, 0x63, 0x63, 0x2c, 0x20, 0x5b, 0x63, 0x75, 0x72, + 0x2e, 0x74, 0x72, 0x69, 0x6d, 0x28, 0x29, 0x5d, 0x3a, 0x20, 0x69, 0x20, + 0x7d, 0x29, 0x2c, 0x20, 0x7b, 0x7d, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x76, 0x65, 0x72, + 0x74, 0x65, 0x72, 0x2e, 0x76, 0x69, 0x73, 0x69, 0x74, 0x28, 0x73, 0x63, + 0x68, 0x65, 0x6d, 0x61, 0x2c, 0x20, 0x27, 0x27, 0x29, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x70, 0x61, 0x72, 0x61, + 0x6d, 0x73, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x20, 0x3d, 0x20, 0x7b, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x2e, 0x2e, 0x2e, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x2e, 0x76, + 0x61, 0x6c, 0x75, 0x65, 0x2c, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x67, 0x72, 0x61, 0x6d, 0x6d, 0x61, + 0x72, 0x3a, 0x20, 0x63, 0x6f, 0x6e, 0x76, 0x65, 0x72, 0x74, 0x65, 0x72, + 0x2e, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x47, 0x72, 0x61, 0x6d, 0x6d, + 0x61, 0x72, 0x28, 0x29, 0x2c, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x7d, 0x20, 0x63, 0x61, 0x74, 0x63, 0x68, 0x20, 0x28, 0x65, + 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x61, 0x6c, 0x65, 0x72, 0x74, 0x28, 0x60, 0x43, 0x6f, 0x6e, + 0x76, 0x65, 0x72, 0x74, 0x20, 0x66, 0x61, 0x69, 0x6c, 0x65, 0x64, 0x3a, + 0x20, 0x24, 0x7b, 0x65, 0x2e, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, + 0x7d, 0x60, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x46, + 0x6c, 0x6f, 0x61, 0x74, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x20, 0x3d, 0x20, + 0x28, 0x7b, 0x20, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x2c, 0x20, 0x6d, 0x61, + 0x78, 0x2c, 0x20, 0x6d, 0x69, 0x6e, 0x2c, 0x20, 0x6e, 0x61, 0x6d, 0x65, + 0x2c, 0x20, 0x73, 0x74, 0x65, 0x70, 0x2c, 0x20, 0x76, 0x61, 0x6c, 0x75, + 0x65, 0x20, 0x7d, 0x29, 0x20, 0x3d, 0x3e, 0x20, 0x7b, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, + 0x20, 0x68, 0x74, 0x6d, 0x6c, 0x60, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x64, 0x69, 0x76, 0x3e, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, + 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x20, 0x66, 0x6f, 0x72, 0x3d, 0x22, 0x24, + 0x7b, 0x6e, 0x61, 0x6d, 0x65, 0x7d, 0x22, 0x3e, 0x24, 0x7b, 0x6c, 0x61, + 0x62, 0x65, 0x6c, 0x7d, 0x3c, 0x2f, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x3e, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x3c, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x20, 0x74, 0x79, 0x70, 0x65, + 0x3d, 0x22, 0x72, 0x61, 0x6e, 0x67, 0x65, 0x22, 0x20, 0x69, 0x64, 0x3d, + 0x22, 0x24, 0x7b, 0x6e, 0x61, 0x6d, 0x65, 0x7d, 0x22, 0x20, 0x6d, 0x69, + 0x6e, 0x3d, 0x22, 0x24, 0x7b, 0x6d, 0x69, 0x6e, 0x7d, 0x22, 0x20, 0x6d, + 0x61, 0x78, 0x3d, 0x22, 0x24, 0x7b, 0x6d, 0x61, 0x78, 0x7d, 0x22, 0x20, + 0x73, 0x74, 0x65, 0x70, 0x3d, 0x22, 0x24, 0x7b, 0x73, 0x74, 0x65, 0x70, + 0x7d, 0x22, 0x20, 0x6e, 0x61, 0x6d, 0x65, 0x3d, 0x22, 0x24, 0x7b, 0x6e, + 0x61, 0x6d, 0x65, 0x7d, 0x22, 0x20, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3d, + 0x22, 0x24, 0x7b, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x7d, 0x22, 0x20, 0x6f, + 0x6e, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x3d, 0x24, 0x7b, 0x75, 0x70, 0x64, + 0x61, 0x74, 0x65, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x46, 0x6c, 0x6f, + 0x61, 0x74, 0x7d, 0x20, 0x2f, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x73, 0x70, 0x61, 0x6e, + 0x3e, 0x24, 0x7b, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x7d, 0x3c, 0x2f, 0x73, + 0x70, 0x61, 0x6e, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x3c, 0x2f, 0x64, 0x69, 0x76, 0x3e, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x60, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x7d, 0x3b, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x49, 0x6e, 0x74, 0x46, 0x69, 0x65, + 0x6c, 0x64, 0x20, 0x3d, 0x20, 0x28, 0x7b, 0x20, 0x6c, 0x61, 0x62, 0x65, + 0x6c, 0x2c, 0x20, 0x6d, 0x61, 0x78, 0x2c, 0x20, 0x6d, 0x69, 0x6e, 0x2c, + 0x20, 0x6e, 0x61, 0x6d, 0x65, 0x2c, 0x20, 0x76, 0x61, 0x6c, 0x75, 0x65, + 0x20, 0x7d, 0x29, 0x20, 0x3d, 0x3e, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, + 0x68, 0x74, 0x6d, 0x6c, 0x60, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x3c, 0x64, 0x69, 0x76, 0x3e, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x6c, + 0x61, 0x62, 0x65, 0x6c, 0x20, 0x66, 0x6f, 0x72, 0x3d, 0x22, 0x24, 0x7b, + 0x6e, 0x61, 0x6d, 0x65, 0x7d, 0x22, 0x3e, 0x24, 0x7b, 0x6c, 0x61, 0x62, + 0x65, 0x6c, 0x7d, 0x3c, 0x2f, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x3e, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x3c, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x20, 0x74, 0x79, 0x70, 0x65, 0x3d, + 0x22, 0x72, 0x61, 0x6e, 0x67, 0x65, 0x22, 0x20, 0x69, 0x64, 0x3d, 0x22, + 0x24, 0x7b, 0x6e, 0x61, 0x6d, 0x65, 0x7d, 0x22, 0x20, 0x6d, 0x69, 0x6e, + 0x3d, 0x22, 0x24, 0x7b, 0x6d, 0x69, 0x6e, 0x7d, 0x22, 0x20, 0x6d, 0x61, + 0x78, 0x3d, 0x22, 0x24, 0x7b, 0x6d, 0x61, 0x78, 0x7d, 0x22, 0x20, 0x6e, + 0x61, 0x6d, 0x65, 0x3d, 0x22, 0x24, 0x7b, 0x6e, 0x61, 0x6d, 0x65, 0x7d, + 0x22, 0x20, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3d, 0x22, 0x24, 0x7b, 0x76, + 0x61, 0x6c, 0x75, 0x65, 0x7d, 0x22, 0x20, 0x6f, 0x6e, 0x69, 0x6e, 0x70, + 0x75, 0x74, 0x3d, 0x24, 0x7b, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x50, + 0x61, 0x72, 0x61, 0x6d, 0x73, 0x49, 0x6e, 0x74, 0x7d, 0x20, 0x2f, 0x3e, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x3c, 0x73, 0x70, 0x61, 0x6e, 0x3e, 0x24, 0x7b, 0x76, 0x61, 0x6c, + 0x75, 0x65, 0x7d, 0x3c, 0x2f, 0x73, 0x70, 0x61, 0x6e, 0x3e, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x2f, 0x64, + 0x69, 0x76, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x60, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x3b, 0x0a, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, + 0x75, 0x73, 0x65, 0x72, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, + 0x52, 0x65, 0x73, 0x65, 0x74, 0x20, 0x3d, 0x20, 0x28, 0x65, 0x29, 0x20, + 0x3d, 0x3e, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x65, 0x2e, 0x70, 0x72, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x44, 0x65, + 0x66, 0x61, 0x75, 0x6c, 0x74, 0x28, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x75, 0x73, 0x65, 0x72, 0x54, 0x65, 0x6d, + 0x70, 0x6c, 0x61, 0x74, 0x65, 0x52, 0x65, 0x73, 0x65, 0x74, 0x54, 0x6f, + 0x44, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x41, 0x6e, 0x64, 0x41, 0x70, + 0x70, 0x6c, 0x79, 0x28, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x7d, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, + 0x73, 0x74, 0x20, 0x55, 0x73, 0x65, 0x72, 0x54, 0x65, 0x6d, 0x70, 0x6c, + 0x61, 0x74, 0x65, 0x52, 0x65, 0x73, 0x65, 0x74, 0x42, 0x75, 0x74, 0x74, + 0x6f, 0x6e, 0x20, 0x3d, 0x20, 0x28, 0x29, 0x20, 0x3d, 0x3e, 0x20, 0x7b, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, + 0x28, 0x73, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x65, 0x64, 0x55, 0x73, 0x65, + 0x72, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x2e, 0x76, 0x61, + 0x6c, 0x75, 0x65, 0x2e, 0x6e, 0x61, 0x6d, 0x65, 0x20, 0x3d, 0x3d, 0x20, + 0x27, 0x64, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x27, 0x29, 0x20, 0x7b, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, + 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x68, 0x74, 0x6d, 0x6c, 0x60, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x3c, 0x62, 0x75, 0x74, 0x74, 0x6f, 0x6e, 0x20, 0x64, 0x69, 0x73, 0x61, + 0x62, 0x6c, 0x65, 0x64, 0x3e, 0x55, 0x73, 0x69, 0x6e, 0x67, 0x20, 0x64, + 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x20, 0x74, 0x65, 0x6d, 0x70, 0x6c, + 0x61, 0x74, 0x65, 0x3c, 0x2f, 0x62, 0x75, 0x74, 0x74, 0x6f, 0x6e, 0x3e, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x60, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, + 0x72, 0x6e, 0x20, 0x68, 0x74, 0x6d, 0x6c, 0x60, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x62, 0x75, 0x74, 0x74, + 0x6f, 0x6e, 0x20, 0x6f, 0x6e, 0x63, 0x6c, 0x69, 0x63, 0x6b, 0x3d, 0x24, + 0x7b, 0x75, 0x73, 0x65, 0x72, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, + 0x65, 0x52, 0x65, 0x73, 0x65, 0x74, 0x7d, 0x3e, 0x52, 0x65, 0x73, 0x65, + 0x74, 0x20, 0x61, 0x6c, 0x6c, 0x20, 0x74, 0x6f, 0x20, 0x64, 0x65, 0x66, + 0x61, 0x75, 0x6c, 0x74, 0x3c, 0x2f, 0x62, 0x75, 0x74, 0x74, 0x6f, 0x6e, + 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x60, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x3b, 0x0a, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x75, 0x73, 0x65, 0x45, 0x66, 0x66, 0x65, 0x63, + 0x74, 0x28, 0x28, 0x29, 0x20, 0x3d, 0x3e, 0x20, 0x7b, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x2f, 0x2f, 0x20, 0x61, 0x75, 0x74, + 0x6f, 0x73, 0x61, 0x76, 0x65, 0x20, 0x74, 0x65, 0x6d, 0x70, 0x6c, 0x61, + 0x74, 0x65, 0x20, 0x6f, 0x6e, 0x20, 0x65, 0x76, 0x65, 0x72, 0x79, 0x20, + 0x63, 0x68, 0x61, 0x6e, 0x67, 0x65, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x75, 0x73, 0x65, 0x72, 0x54, 0x65, 0x6d, 0x70, 0x6c, + 0x61, 0x74, 0x65, 0x41, 0x75, 0x74, 0x6f, 0x73, 0x61, 0x76, 0x65, 0x28, + 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x2c, 0x20, 0x5b, + 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x61, 0x6c, 0x75, + 0x65, 0x2c, 0x20, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x2e, 0x76, 0x61, + 0x6c, 0x75, 0x65, 0x5d, 0x29, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x47, 0x72, 0x61, 0x6d, 0x6d, + 0x61, 0x72, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x20, 0x3d, 0x20, + 0x28, 0x29, 0x20, 0x3d, 0x3e, 0x20, 0x28, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x68, 0x74, 0x6d, 0x6c, 0x60, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x64, 0x69, 0x76, + 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x3c, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x20, 0x66, 0x6f, 0x72, + 0x3d, 0x22, 0x74, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x22, 0x3e, + 0x47, 0x72, 0x61, 0x6d, 0x6d, 0x61, 0x72, 0x3c, 0x2f, 0x6c, 0x61, 0x62, + 0x65, 0x6c, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x3c, 0x74, 0x65, 0x78, 0x74, 0x61, 0x72, 0x65, + 0x61, 0x20, 0x69, 0x64, 0x3d, 0x22, 0x67, 0x72, 0x61, 0x6d, 0x6d, 0x61, + 0x72, 0x22, 0x20, 0x6e, 0x61, 0x6d, 0x65, 0x3d, 0x22, 0x67, 0x72, 0x61, + 0x6d, 0x6d, 0x61, 0x72, 0x22, 0x20, 0x70, 0x6c, 0x61, 0x63, 0x65, 0x68, + 0x6f, 0x6c, 0x64, 0x65, 0x72, 0x3d, 0x22, 0x55, 0x73, 0x65, 0x20, 0x67, + 0x62, 0x6e, 0x66, 0x20, 0x6f, 0x72, 0x20, 0x4a, 0x53, 0x4f, 0x4e, 0x20, + 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2b, 0x63, 0x6f, 0x6e, 0x76, 0x65, + 0x72, 0x74, 0x22, 0x20, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3d, 0x22, 0x24, + 0x7b, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x2e, 0x76, 0x61, 0x6c, 0x75, + 0x65, 0x2e, 0x67, 0x72, 0x61, 0x6d, 0x6d, 0x61, 0x72, 0x7d, 0x22, 0x20, + 0x72, 0x6f, 0x77, 0x73, 0x3d, 0x34, 0x20, 0x6f, 0x6e, 0x69, 0x6e, 0x70, + 0x75, 0x74, 0x3d, 0x24, 0x7b, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x50, + 0x61, 0x72, 0x61, 0x6d, 0x73, 0x7d, 0x2f, 0x3e, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x69, 0x6e, + 0x70, 0x75, 0x74, 0x20, 0x74, 0x79, 0x70, 0x65, 0x3d, 0x22, 0x74, 0x65, + 0x78, 0x74, 0x22, 0x20, 0x6e, 0x61, 0x6d, 0x65, 0x3d, 0x22, 0x70, 0x72, + 0x6f, 0x70, 0x2d, 0x6f, 0x72, 0x64, 0x65, 0x72, 0x22, 0x20, 0x70, 0x6c, + 0x61, 0x63, 0x65, 0x68, 0x6f, 0x6c, 0x64, 0x65, 0x72, 0x3d, 0x22, 0x6f, + 0x72, 0x64, 0x65, 0x72, 0x3a, 0x20, 0x70, 0x72, 0x6f, 0x70, 0x31, 0x2c, + 0x70, 0x72, 0x6f, 0x70, 0x32, 0x2c, 0x70, 0x72, 0x6f, 0x70, 0x33, 0x22, + 0x20, 0x6f, 0x6e, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x3d, 0x24, 0x7b, 0x75, + 0x70, 0x64, 0x61, 0x74, 0x65, 0x47, 0x72, 0x61, 0x6d, 0x6d, 0x61, 0x72, + 0x4a, 0x73, 0x6f, 0x6e, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x50, 0x72, + 0x6f, 0x70, 0x4f, 0x72, 0x64, 0x65, 0x72, 0x7d, 0x20, 0x2f, 0x3e, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x3c, 0x62, 0x75, 0x74, 0x74, 0x6f, 0x6e, 0x20, 0x74, 0x79, 0x70, 0x65, + 0x3d, 0x22, 0x62, 0x75, 0x74, 0x74, 0x6f, 0x6e, 0x22, 0x20, 0x6f, 0x6e, + 0x63, 0x6c, 0x69, 0x63, 0x6b, 0x3d, 0x24, 0x7b, 0x63, 0x6f, 0x6e, 0x76, + 0x65, 0x72, 0x74, 0x4a, 0x53, 0x4f, 0x4e, 0x53, 0x63, 0x68, 0x65, 0x6d, + 0x61, 0x47, 0x72, 0x61, 0x6d, 0x6d, 0x61, 0x72, 0x7d, 0x3e, 0x43, 0x6f, + 0x6e, 0x76, 0x65, 0x72, 0x74, 0x20, 0x4a, 0x53, 0x4f, 0x4e, 0x20, 0x53, + 0x63, 0x68, 0x65, 0x6d, 0x61, 0x3c, 0x2f, 0x62, 0x75, 0x74, 0x74, 0x6f, + 0x6e, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x3c, 0x2f, 0x64, 0x69, 0x76, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x60, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x29, 0x3b, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x50, 0x72, 0x6f, 0x6d, 0x70, 0x74, + 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x46, 0x69, 0x65, 0x6c, 0x64, + 0x53, 0x65, 0x74, 0x20, 0x3d, 0x20, 0x28, 0x29, 0x20, 0x3d, 0x3e, 0x20, + 0x28, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x68, 0x74, + 0x6d, 0x6c, 0x60, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x3c, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x65, 0x74, 0x3e, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x64, 0x69, + 0x76, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x3c, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x20, 0x68, 0x74, + 0x6d, 0x6c, 0x46, 0x6f, 0x72, 0x3d, 0x22, 0x70, 0x72, 0x6f, 0x6d, 0x70, + 0x74, 0x22, 0x3e, 0x50, 0x72, 0x6f, 0x6d, 0x70, 0x74, 0x3c, 0x2f, 0x6c, + 0x61, 0x62, 0x65, 0x6c, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x74, 0x65, 0x78, 0x74, 0x61, + 0x72, 0x65, 0x61, 0x20, 0x74, 0x79, 0x70, 0x65, 0x3d, 0x22, 0x74, 0x65, + 0x78, 0x74, 0x22, 0x20, 0x6e, 0x61, 0x6d, 0x65, 0x3d, 0x22, 0x70, 0x72, + 0x6f, 0x6d, 0x70, 0x74, 0x22, 0x20, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3d, + 0x22, 0x24, 0x7b, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x2e, 0x76, + 0x61, 0x6c, 0x75, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x6d, 0x70, 0x74, 0x7d, + 0x22, 0x20, 0x6f, 0x6e, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x3d, 0x24, 0x7b, + 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, + 0x6e, 0x7d, 0x2f, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x3c, 0x2f, 0x64, 0x69, 0x76, 0x3e, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x2f, 0x66, 0x69, 0x65, 0x6c, + 0x64, 0x73, 0x65, 0x74, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x60, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x29, 0x3b, + 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, + 0x74, 0x20, 0x43, 0x68, 0x61, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, + 0x46, 0x6f, 0x72, 0x6d, 0x20, 0x3d, 0x20, 0x28, 0x29, 0x20, 0x3d, 0x3e, + 0x20, 0x28, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x68, + 0x74, 0x6d, 0x6c, 0x60, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x24, 0x7b, 0x50, 0x72, 0x6f, 0x6d, 0x70, 0x74, 0x43, + 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x53, + 0x65, 0x74, 0x28, 0x29, 0x7d, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x73, + 0x65, 0x74, 0x20, 0x63, 0x6c, 0x61, 0x73, 0x73, 0x3d, 0x22, 0x74, 0x77, + 0x6f, 0x22, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x3c, 0x64, 0x69, 0x76, 0x3e, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x3c, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x20, 0x66, 0x6f, 0x72, 0x3d, 0x22, + 0x75, 0x73, 0x65, 0x72, 0x22, 0x3e, 0x55, 0x73, 0x65, 0x72, 0x20, 0x6e, + 0x61, 0x6d, 0x65, 0x3c, 0x2f, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x3e, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x3c, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x20, 0x74, 0x79, 0x70, + 0x65, 0x3d, 0x22, 0x74, 0x65, 0x78, 0x74, 0x22, 0x20, 0x6e, 0x61, 0x6d, + 0x65, 0x3d, 0x22, 0x75, 0x73, 0x65, 0x72, 0x22, 0x20, 0x76, 0x61, 0x6c, + 0x75, 0x65, 0x3d, 0x22, 0x24, 0x7b, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, + 0x6e, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x2e, 0x75, 0x73, 0x65, 0x72, + 0x7d, 0x22, 0x20, 0x6f, 0x6e, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x3d, 0x24, + 0x7b, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x53, 0x65, 0x73, 0x73, 0x69, + 0x6f, 0x6e, 0x7d, 0x20, 0x2f, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x2f, 0x64, 0x69, 0x76, + 0x3e, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x3c, 0x64, 0x69, 0x76, 0x3e, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, + 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x20, 0x66, 0x6f, 0x72, 0x3d, 0x22, 0x62, + 0x6f, 0x74, 0x22, 0x3e, 0x42, 0x6f, 0x74, 0x20, 0x6e, 0x61, 0x6d, 0x65, + 0x3c, 0x2f, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x3e, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, + 0x69, 0x6e, 0x70, 0x75, 0x74, 0x20, 0x74, 0x79, 0x70, 0x65, 0x3d, 0x22, + 0x74, 0x65, 0x78, 0x74, 0x22, 0x20, 0x6e, 0x61, 0x6d, 0x65, 0x3d, 0x22, + 0x63, 0x68, 0x61, 0x72, 0x22, 0x20, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3d, + 0x22, 0x24, 0x7b, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x2e, 0x76, + 0x61, 0x6c, 0x75, 0x65, 0x2e, 0x63, 0x68, 0x61, 0x72, 0x7d, 0x22, 0x20, + 0x6f, 0x6e, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x3d, 0x24, 0x7b, 0x75, 0x70, + 0x64, 0x61, 0x74, 0x65, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x7d, + 0x20, 0x2f, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x3c, 0x2f, 0x64, 0x69, 0x76, 0x3e, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x2f, 0x66, + 0x69, 0x65, 0x6c, 0x64, 0x73, 0x65, 0x74, 0x3e, 0x0a, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x66, 0x69, 0x65, + 0x6c, 0x64, 0x73, 0x65, 0x74, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x64, 0x69, 0x76, 0x3e, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x3c, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x20, 0x66, 0x6f, + 0x72, 0x3d, 0x22, 0x74, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x22, + 0x3e, 0x50, 0x72, 0x6f, 0x6d, 0x70, 0x74, 0x20, 0x74, 0x65, 0x6d, 0x70, + 0x6c, 0x61, 0x74, 0x65, 0x3c, 0x2f, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x3e, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x3c, 0x74, 0x65, 0x78, 0x74, 0x61, 0x72, 0x65, 0x61, + 0x20, 0x69, 0x64, 0x3d, 0x22, 0x74, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, + 0x65, 0x22, 0x20, 0x6e, 0x61, 0x6d, 0x65, 0x3d, 0x22, 0x74, 0x65, 0x6d, + 0x70, 0x6c, 0x61, 0x74, 0x65, 0x22, 0x20, 0x76, 0x61, 0x6c, 0x75, 0x65, + 0x3d, 0x22, 0x24, 0x7b, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x2e, + 0x76, 0x61, 0x6c, 0x75, 0x65, 0x2e, 0x74, 0x65, 0x6d, 0x70, 0x6c, 0x61, + 0x74, 0x65, 0x7d, 0x22, 0x20, 0x72, 0x6f, 0x77, 0x73, 0x3d, 0x34, 0x20, + 0x6f, 0x6e, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x3d, 0x24, 0x7b, 0x75, 0x70, + 0x64, 0x61, 0x74, 0x65, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x7d, + 0x2f, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x3c, 0x2f, 0x64, 0x69, 0x76, 0x3e, 0x0a, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, + 0x64, 0x69, 0x76, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x6c, 0x61, 0x62, 0x65, + 0x6c, 0x20, 0x66, 0x6f, 0x72, 0x3d, 0x22, 0x74, 0x65, 0x6d, 0x70, 0x6c, + 0x61, 0x74, 0x65, 0x22, 0x3e, 0x43, 0x68, 0x61, 0x74, 0x20, 0x68, 0x69, + 0x73, 0x74, 0x6f, 0x72, 0x79, 0x20, 0x74, 0x65, 0x6d, 0x70, 0x6c, 0x61, + 0x74, 0x65, 0x3c, 0x2f, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x3e, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x3c, 0x74, 0x65, 0x78, 0x74, 0x61, 0x72, 0x65, 0x61, 0x20, 0x69, + 0x64, 0x3d, 0x22, 0x74, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x22, + 0x20, 0x6e, 0x61, 0x6d, 0x65, 0x3d, 0x22, 0x68, 0x69, 0x73, 0x74, 0x6f, + 0x72, 0x79, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x22, 0x20, + 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3d, 0x22, 0x24, 0x7b, 0x73, 0x65, 0x73, + 0x73, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x2e, 0x68, + 0x69, 0x73, 0x74, 0x6f, 0x72, 0x79, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, + 0x74, 0x65, 0x7d, 0x22, 0x20, 0x72, 0x6f, 0x77, 0x73, 0x3d, 0x31, 0x20, + 0x6f, 0x6e, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x3d, 0x24, 0x7b, 0x75, 0x70, + 0x64, 0x61, 0x74, 0x65, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x7d, + 0x2f, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x3c, 0x2f, 0x64, 0x69, 0x76, 0x3e, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x24, 0x7b, + 0x47, 0x72, 0x61, 0x6d, 0x6d, 0x61, 0x72, 0x43, 0x6f, 0x6e, 0x74, 0x72, + 0x6f, 0x6c, 0x28, 0x29, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x3c, 0x2f, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x73, + 0x65, 0x74, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x60, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x29, 0x3b, 0x0a, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x43, 0x6f, + 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, + 0x69, 0x67, 0x46, 0x6f, 0x72, 0x6d, 0x20, 0x3d, 0x20, 0x28, 0x29, 0x20, + 0x3d, 0x3e, 0x20, 0x28, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x68, 0x74, 0x6d, 0x6c, 0x60, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x24, 0x7b, 0x50, 0x72, 0x6f, 0x6d, 0x70, + 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x46, 0x69, 0x65, 0x6c, + 0x64, 0x53, 0x65, 0x74, 0x28, 0x29, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x66, 0x69, 0x65, 0x6c, 0x64, + 0x73, 0x65, 0x74, 0x3e, 0x24, 0x7b, 0x47, 0x72, 0x61, 0x6d, 0x6d, 0x61, + 0x72, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x28, 0x29, 0x7d, 0x3c, + 0x2f, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x65, 0x74, 0x3e, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x60, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x29, 0x3b, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x68, 0x74, 0x6d, 0x6c, + 0x60, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x66, + 0x6f, 0x72, 0x6d, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x3c, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x65, 0x74, + 0x20, 0x63, 0x6c, 0x61, 0x73, 0x73, 0x3d, 0x22, 0x74, 0x77, 0x6f, 0x22, + 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x3c, 0x24, 0x7b, 0x55, 0x73, 0x65, 0x72, 0x54, 0x65, 0x6d, + 0x70, 0x6c, 0x61, 0x74, 0x65, 0x52, 0x65, 0x73, 0x65, 0x74, 0x42, 0x75, + 0x74, 0x74, 0x6f, 0x6e, 0x7d, 0x2f, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x64, 0x69, 0x76, + 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x3c, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x20, 0x63, + 0x6c, 0x61, 0x73, 0x73, 0x3d, 0x22, 0x73, 0x6c, 0x69, 0x6d, 0x22, 0x3e, + 0x3c, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x20, 0x74, 0x79, 0x70, 0x65, 0x3d, + 0x22, 0x72, 0x61, 0x64, 0x69, 0x6f, 0x22, 0x20, 0x6e, 0x61, 0x6d, 0x65, + 0x3d, 0x22, 0x74, 0x79, 0x70, 0x65, 0x22, 0x20, 0x76, 0x61, 0x6c, 0x75, + 0x65, 0x3d, 0x22, 0x63, 0x68, 0x61, 0x74, 0x22, 0x20, 0x63, 0x68, 0x65, + 0x63, 0x6b, 0x65, 0x64, 0x3d, 0x24, 0x7b, 0x73, 0x65, 0x73, 0x73, 0x69, + 0x6f, 0x6e, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x2e, 0x74, 0x79, 0x70, + 0x65, 0x20, 0x3d, 0x3d, 0x3d, 0x20, 0x22, 0x63, 0x68, 0x61, 0x74, 0x22, + 0x7d, 0x20, 0x6f, 0x6e, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x3d, 0x24, 0x7b, + 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, + 0x6e, 0x7d, 0x20, 0x2f, 0x3e, 0x20, 0x43, 0x68, 0x61, 0x74, 0x3c, 0x2f, + 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x6c, 0x61, + 0x62, 0x65, 0x6c, 0x20, 0x63, 0x6c, 0x61, 0x73, 0x73, 0x3d, 0x22, 0x73, + 0x6c, 0x69, 0x6d, 0x22, 0x3e, 0x3c, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x20, + 0x74, 0x79, 0x70, 0x65, 0x3d, 0x22, 0x72, 0x61, 0x64, 0x69, 0x6f, 0x22, + 0x20, 0x6e, 0x61, 0x6d, 0x65, 0x3d, 0x22, 0x74, 0x79, 0x70, 0x65, 0x22, + 0x20, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3d, 0x22, 0x63, 0x6f, 0x6d, 0x70, + 0x6c, 0x65, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x20, 0x63, 0x68, 0x65, 0x63, + 0x6b, 0x65, 0x64, 0x3d, 0x24, 0x7b, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, + 0x6e, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x2e, 0x74, 0x79, 0x70, 0x65, + 0x20, 0x3d, 0x3d, 0x3d, 0x20, 0x22, 0x63, 0x6f, 0x6d, 0x70, 0x6c, 0x65, + 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x7d, 0x20, 0x6f, 0x6e, 0x69, 0x6e, 0x70, + 0x75, 0x74, 0x3d, 0x24, 0x7b, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x53, + 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x7d, 0x20, 0x2f, 0x3e, 0x20, 0x43, + 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x69, 0x6f, 0x6e, 0x3c, 0x2f, 0x6c, + 0x61, 0x62, 0x65, 0x6c, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x2f, 0x64, 0x69, 0x76, 0x3e, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, + 0x2f, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x65, 0x74, 0x3e, 0x0a, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x24, 0x7b, + 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x61, 0x6c, 0x75, + 0x65, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x20, 0x3d, 0x3d, 0x3d, 0x20, 0x27, + 0x63, 0x68, 0x61, 0x74, 0x27, 0x20, 0x3f, 0x20, 0x43, 0x68, 0x61, 0x74, + 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x46, 0x6f, 0x72, 0x6d, 0x28, 0x29, + 0x20, 0x3a, 0x20, 0x43, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x69, 0x6f, + 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x46, 0x6f, 0x72, 0x6d, 0x28, + 0x29, 0x7d, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x3c, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x65, 0x74, 0x20, + 0x63, 0x6c, 0x61, 0x73, 0x73, 0x3d, 0x22, 0x74, 0x77, 0x6f, 0x22, 0x3e, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x24, 0x7b, 0x49, 0x6e, 0x74, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x28, + 0x7b, 0x20, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x3a, 0x20, 0x22, 0x50, 0x72, + 0x65, 0x64, 0x69, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x22, 0x2c, 0x20, + 0x6d, 0x61, 0x78, 0x3a, 0x20, 0x32, 0x30, 0x34, 0x38, 0x2c, 0x20, 0x6d, + 0x69, 0x6e, 0x3a, 0x20, 0x2d, 0x31, 0x2c, 0x20, 0x6e, 0x61, 0x6d, 0x65, + 0x3a, 0x20, 0x22, 0x6e, 0x5f, 0x70, 0x72, 0x65, 0x64, 0x69, 0x63, 0x74, + 0x22, 0x2c, 0x20, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x20, 0x70, 0x61, + 0x72, 0x61, 0x6d, 0x73, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x2e, 0x6e, + 0x5f, 0x70, 0x72, 0x65, 0x64, 0x69, 0x63, 0x74, 0x20, 0x7d, 0x29, 0x7d, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x24, 0x7b, 0x46, 0x6c, 0x6f, 0x61, 0x74, 0x46, 0x69, 0x65, 0x6c, + 0x64, 0x28, 0x7b, 0x20, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x3a, 0x20, 0x22, + 0x54, 0x65, 0x6d, 0x70, 0x65, 0x72, 0x61, 0x74, 0x75, 0x72, 0x65, 0x22, + 0x2c, 0x20, 0x6d, 0x61, 0x78, 0x3a, 0x20, 0x32, 0x2e, 0x30, 0x2c, 0x20, + 0x6d, 0x69, 0x6e, 0x3a, 0x20, 0x30, 0x2e, 0x30, 0x2c, 0x20, 0x6e, 0x61, + 0x6d, 0x65, 0x3a, 0x20, 0x22, 0x74, 0x65, 0x6d, 0x70, 0x65, 0x72, 0x61, + 0x74, 0x75, 0x72, 0x65, 0x22, 0x2c, 0x20, 0x73, 0x74, 0x65, 0x70, 0x3a, + 0x20, 0x30, 0x2e, 0x30, 0x31, 0x2c, 0x20, 0x76, 0x61, 0x6c, 0x75, 0x65, + 0x3a, 0x20, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x2e, 0x76, 0x61, 0x6c, + 0x75, 0x65, 0x2e, 0x74, 0x65, 0x6d, 0x70, 0x65, 0x72, 0x61, 0x74, 0x75, + 0x72, 0x65, 0x20, 0x7d, 0x29, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x24, 0x7b, 0x46, 0x6c, 0x6f, + 0x61, 0x74, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x28, 0x7b, 0x20, 0x6c, 0x61, + 0x62, 0x65, 0x6c, 0x3a, 0x20, 0x22, 0x50, 0x65, 0x6e, 0x61, 0x6c, 0x69, + 0x7a, 0x65, 0x20, 0x72, 0x65, 0x70, 0x65, 0x61, 0x74, 0x20, 0x73, 0x65, + 0x71, 0x75, 0x65, 0x6e, 0x63, 0x65, 0x22, 0x2c, 0x20, 0x6d, 0x61, 0x78, + 0x3a, 0x20, 0x32, 0x2e, 0x30, 0x2c, 0x20, 0x6d, 0x69, 0x6e, 0x3a, 0x20, + 0x30, 0x2e, 0x30, 0x2c, 0x20, 0x6e, 0x61, 0x6d, 0x65, 0x3a, 0x20, 0x22, + 0x72, 0x65, 0x70, 0x65, 0x61, 0x74, 0x5f, 0x70, 0x65, 0x6e, 0x61, 0x6c, + 0x74, 0x79, 0x22, 0x2c, 0x20, 0x73, 0x74, 0x65, 0x70, 0x3a, 0x20, 0x30, + 0x2e, 0x30, 0x31, 0x2c, 0x20, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x20, + 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, + 0x2e, 0x72, 0x65, 0x70, 0x65, 0x61, 0x74, 0x5f, 0x70, 0x65, 0x6e, 0x61, + 0x6c, 0x74, 0x79, 0x20, 0x7d, 0x29, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x24, 0x7b, 0x49, 0x6e, + 0x74, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x28, 0x7b, 0x20, 0x6c, 0x61, 0x62, + 0x65, 0x6c, 0x3a, 0x20, 0x22, 0x43, 0x6f, 0x6e, 0x73, 0x69, 0x64, 0x65, + 0x72, 0x20, 0x4e, 0x20, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x73, 0x20, 0x66, + 0x6f, 0x72, 0x20, 0x70, 0x65, 0x6e, 0x61, 0x6c, 0x69, 0x7a, 0x65, 0x22, + 0x2c, 0x20, 0x6d, 0x61, 0x78, 0x3a, 0x20, 0x32, 0x30, 0x34, 0x38, 0x2c, + 0x20, 0x6d, 0x69, 0x6e, 0x3a, 0x20, 0x30, 0x2c, 0x20, 0x6e, 0x61, 0x6d, + 0x65, 0x3a, 0x20, 0x22, 0x72, 0x65, 0x70, 0x65, 0x61, 0x74, 0x5f, 0x6c, + 0x61, 0x73, 0x74, 0x5f, 0x6e, 0x22, 0x2c, 0x20, 0x76, 0x61, 0x6c, 0x75, + 0x65, 0x3a, 0x20, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x2e, 0x76, 0x61, + 0x6c, 0x75, 0x65, 0x2e, 0x72, 0x65, 0x70, 0x65, 0x61, 0x74, 0x5f, 0x6c, + 0x61, 0x73, 0x74, 0x5f, 0x6e, 0x20, 0x7d, 0x29, 0x7d, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x24, 0x7b, + 0x49, 0x6e, 0x74, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x28, 0x7b, 0x20, 0x6c, + 0x61, 0x62, 0x65, 0x6c, 0x3a, 0x20, 0x22, 0x54, 0x6f, 0x70, 0x2d, 0x4b, + 0x20, 0x73, 0x61, 0x6d, 0x70, 0x6c, 0x69, 0x6e, 0x67, 0x22, 0x2c, 0x20, + 0x6d, 0x61, 0x78, 0x3a, 0x20, 0x31, 0x30, 0x30, 0x2c, 0x20, 0x6d, 0x69, + 0x6e, 0x3a, 0x20, 0x2d, 0x31, 0x2c, 0x20, 0x6e, 0x61, 0x6d, 0x65, 0x3a, + 0x20, 0x22, 0x74, 0x6f, 0x70, 0x5f, 0x6b, 0x22, 0x2c, 0x20, 0x76, 0x61, + 0x6c, 0x75, 0x65, 0x3a, 0x20, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x2e, + 0x76, 0x61, 0x6c, 0x75, 0x65, 0x2e, 0x74, 0x6f, 0x70, 0x5f, 0x6b, 0x20, + 0x7d, 0x29, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x24, 0x7b, 0x46, 0x6c, 0x6f, 0x61, 0x74, 0x46, + 0x69, 0x65, 0x6c, 0x64, 0x28, 0x7b, 0x20, 0x6c, 0x61, 0x62, 0x65, 0x6c, + 0x3a, 0x20, 0x22, 0x54, 0x6f, 0x70, 0x2d, 0x50, 0x20, 0x73, 0x61, 0x6d, + 0x70, 0x6c, 0x69, 0x6e, 0x67, 0x22, 0x2c, 0x20, 0x6d, 0x61, 0x78, 0x3a, + 0x20, 0x31, 0x2e, 0x30, 0x2c, 0x20, 0x6d, 0x69, 0x6e, 0x3a, 0x20, 0x30, + 0x2e, 0x30, 0x2c, 0x20, 0x6e, 0x61, 0x6d, 0x65, 0x3a, 0x20, 0x22, 0x74, + 0x6f, 0x70, 0x5f, 0x70, 0x22, 0x2c, 0x20, 0x73, 0x74, 0x65, 0x70, 0x3a, + 0x20, 0x30, 0x2e, 0x30, 0x31, 0x2c, 0x20, 0x76, 0x61, 0x6c, 0x75, 0x65, + 0x3a, 0x20, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x2e, 0x76, 0x61, 0x6c, + 0x75, 0x65, 0x2e, 0x74, 0x6f, 0x70, 0x5f, 0x70, 0x20, 0x7d, 0x29, 0x7d, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x24, 0x7b, 0x46, 0x6c, 0x6f, 0x61, 0x74, 0x46, 0x69, 0x65, 0x6c, + 0x64, 0x28, 0x7b, 0x20, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x3a, 0x20, 0x22, + 0x4d, 0x69, 0x6e, 0x2d, 0x50, 0x20, 0x73, 0x61, 0x6d, 0x70, 0x6c, 0x69, + 0x6e, 0x67, 0x22, 0x2c, 0x20, 0x6d, 0x61, 0x78, 0x3a, 0x20, 0x31, 0x2e, + 0x30, 0x2c, 0x20, 0x6d, 0x69, 0x6e, 0x3a, 0x20, 0x30, 0x2e, 0x30, 0x2c, + 0x20, 0x6e, 0x61, 0x6d, 0x65, 0x3a, 0x20, 0x22, 0x6d, 0x69, 0x6e, 0x5f, + 0x70, 0x22, 0x2c, 0x20, 0x73, 0x74, 0x65, 0x70, 0x3a, 0x20, 0x30, 0x2e, + 0x30, 0x31, 0x2c, 0x20, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x20, 0x70, + 0x61, 0x72, 0x61, 0x6d, 0x73, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x2e, + 0x6d, 0x69, 0x6e, 0x5f, 0x70, 0x20, 0x7d, 0x29, 0x7d, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x2f, 0x66, 0x69, + 0x65, 0x6c, 0x64, 0x73, 0x65, 0x74, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x64, 0x65, 0x74, 0x61, 0x69, + 0x6c, 0x73, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x3c, 0x73, 0x75, 0x6d, 0x6d, 0x61, 0x72, 0x79, + 0x3e, 0x4d, 0x6f, 0x72, 0x65, 0x20, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, + 0x73, 0x3c, 0x2f, 0x73, 0x75, 0x6d, 0x6d, 0x61, 0x72, 0x79, 0x3e, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x3c, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x65, 0x74, 0x20, 0x63, 0x6c, + 0x61, 0x73, 0x73, 0x3d, 0x22, 0x74, 0x77, 0x6f, 0x22, 0x3e, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x24, 0x7b, 0x46, 0x6c, 0x6f, 0x61, 0x74, 0x46, 0x69, 0x65, 0x6c, + 0x64, 0x28, 0x7b, 0x20, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x3a, 0x20, 0x22, + 0x54, 0x46, 0x53, 0x2d, 0x5a, 0x22, 0x2c, 0x20, 0x6d, 0x61, 0x78, 0x3a, + 0x20, 0x31, 0x2e, 0x30, 0x2c, 0x20, 0x6d, 0x69, 0x6e, 0x3a, 0x20, 0x30, + 0x2e, 0x30, 0x2c, 0x20, 0x6e, 0x61, 0x6d, 0x65, 0x3a, 0x20, 0x22, 0x74, + 0x66, 0x73, 0x5f, 0x7a, 0x22, 0x2c, 0x20, 0x73, 0x74, 0x65, 0x70, 0x3a, + 0x20, 0x30, 0x2e, 0x30, 0x31, 0x2c, 0x20, 0x76, 0x61, 0x6c, 0x75, 0x65, + 0x3a, 0x20, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x2e, 0x76, 0x61, 0x6c, + 0x75, 0x65, 0x2e, 0x74, 0x66, 0x73, 0x5f, 0x7a, 0x20, 0x7d, 0x29, 0x7d, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x24, 0x7b, 0x46, 0x6c, 0x6f, 0x61, 0x74, 0x46, 0x69, + 0x65, 0x6c, 0x64, 0x28, 0x7b, 0x20, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x3a, + 0x20, 0x22, 0x54, 0x79, 0x70, 0x69, 0x63, 0x61, 0x6c, 0x20, 0x50, 0x22, + 0x2c, 0x20, 0x6d, 0x61, 0x78, 0x3a, 0x20, 0x31, 0x2e, 0x30, 0x2c, 0x20, + 0x6d, 0x69, 0x6e, 0x3a, 0x20, 0x30, 0x2e, 0x30, 0x2c, 0x20, 0x6e, 0x61, + 0x6d, 0x65, 0x3a, 0x20, 0x22, 0x74, 0x79, 0x70, 0x69, 0x63, 0x61, 0x6c, + 0x5f, 0x70, 0x22, 0x2c, 0x20, 0x73, 0x74, 0x65, 0x70, 0x3a, 0x20, 0x30, + 0x2e, 0x30, 0x31, 0x2c, 0x20, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x20, + 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, + 0x2e, 0x74, 0x79, 0x70, 0x69, 0x63, 0x61, 0x6c, 0x5f, 0x70, 0x20, 0x7d, + 0x29, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x24, 0x7b, 0x46, 0x6c, 0x6f, 0x61, 0x74, + 0x46, 0x69, 0x65, 0x6c, 0x64, 0x28, 0x7b, 0x20, 0x6c, 0x61, 0x62, 0x65, + 0x6c, 0x3a, 0x20, 0x22, 0x50, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, + 0x20, 0x70, 0x65, 0x6e, 0x61, 0x6c, 0x74, 0x79, 0x22, 0x2c, 0x20, 0x6d, + 0x61, 0x78, 0x3a, 0x20, 0x31, 0x2e, 0x30, 0x2c, 0x20, 0x6d, 0x69, 0x6e, + 0x3a, 0x20, 0x30, 0x2e, 0x30, 0x2c, 0x20, 0x6e, 0x61, 0x6d, 0x65, 0x3a, + 0x20, 0x22, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x5f, 0x70, + 0x65, 0x6e, 0x61, 0x6c, 0x74, 0x79, 0x22, 0x2c, 0x20, 0x73, 0x74, 0x65, + 0x70, 0x3a, 0x20, 0x30, 0x2e, 0x30, 0x31, 0x2c, 0x20, 0x76, 0x61, 0x6c, + 0x75, 0x65, 0x3a, 0x20, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x2e, 0x76, + 0x61, 0x6c, 0x75, 0x65, 0x2e, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, + 0x65, 0x5f, 0x70, 0x65, 0x6e, 0x61, 0x6c, 0x74, 0x79, 0x20, 0x7d, 0x29, + 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x24, 0x7b, 0x46, 0x6c, 0x6f, 0x61, 0x74, 0x46, + 0x69, 0x65, 0x6c, 0x64, 0x28, 0x7b, 0x20, 0x6c, 0x61, 0x62, 0x65, 0x6c, + 0x3a, 0x20, 0x22, 0x46, 0x72, 0x65, 0x71, 0x75, 0x65, 0x6e, 0x63, 0x79, + 0x20, 0x70, 0x65, 0x6e, 0x61, 0x6c, 0x74, 0x79, 0x22, 0x2c, 0x20, 0x6d, + 0x61, 0x78, 0x3a, 0x20, 0x31, 0x2e, 0x30, 0x2c, 0x20, 0x6d, 0x69, 0x6e, + 0x3a, 0x20, 0x30, 0x2e, 0x30, 0x2c, 0x20, 0x6e, 0x61, 0x6d, 0x65, 0x3a, + 0x20, 0x22, 0x66, 0x72, 0x65, 0x71, 0x75, 0x65, 0x6e, 0x63, 0x79, 0x5f, + 0x70, 0x65, 0x6e, 0x61, 0x6c, 0x74, 0x79, 0x22, 0x2c, 0x20, 0x73, 0x74, + 0x65, 0x70, 0x3a, 0x20, 0x30, 0x2e, 0x30, 0x31, 0x2c, 0x20, 0x76, 0x61, + 0x6c, 0x75, 0x65, 0x3a, 0x20, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x2e, + 0x76, 0x61, 0x6c, 0x75, 0x65, 0x2e, 0x66, 0x72, 0x65, 0x71, 0x75, 0x65, + 0x6e, 0x63, 0x79, 0x5f, 0x70, 0x65, 0x6e, 0x61, 0x6c, 0x74, 0x79, 0x20, + 0x7d, 0x29, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x3c, 0x2f, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x73, + 0x65, 0x74, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x3c, 0x68, 0x72, 0x20, 0x2f, 0x3e, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, + 0x66, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x65, 0x74, 0x20, 0x63, 0x6c, 0x61, + 0x73, 0x73, 0x3d, 0x22, 0x74, 0x68, 0x72, 0x65, 0x65, 0x22, 0x3e, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x3c, 0x64, 0x69, 0x76, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x3c, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x3e, 0x3c, 0x69, 0x6e, 0x70, 0x75, + 0x74, 0x20, 0x74, 0x79, 0x70, 0x65, 0x3d, 0x22, 0x72, 0x61, 0x64, 0x69, + 0x6f, 0x22, 0x20, 0x6e, 0x61, 0x6d, 0x65, 0x3d, 0x22, 0x6d, 0x69, 0x72, + 0x6f, 0x73, 0x74, 0x61, 0x74, 0x22, 0x20, 0x76, 0x61, 0x6c, 0x75, 0x65, + 0x3d, 0x22, 0x30, 0x22, 0x20, 0x63, 0x68, 0x65, 0x63, 0x6b, 0x65, 0x64, + 0x3d, 0x24, 0x7b, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x2e, 0x76, 0x61, + 0x6c, 0x75, 0x65, 0x2e, 0x6d, 0x69, 0x72, 0x6f, 0x73, 0x74, 0x61, 0x74, + 0x20, 0x3d, 0x3d, 0x20, 0x30, 0x7d, 0x20, 0x6f, 0x6e, 0x69, 0x6e, 0x70, + 0x75, 0x74, 0x3d, 0x24, 0x7b, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x50, + 0x61, 0x72, 0x61, 0x6d, 0x73, 0x49, 0x6e, 0x74, 0x7d, 0x20, 0x2f, 0x3e, + 0x20, 0x6e, 0x6f, 0x20, 0x4d, 0x69, 0x72, 0x6f, 0x73, 0x74, 0x61, 0x74, + 0x3c, 0x2f, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x3e, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x3c, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x3e, 0x3c, 0x69, 0x6e, 0x70, + 0x75, 0x74, 0x20, 0x74, 0x79, 0x70, 0x65, 0x3d, 0x22, 0x72, 0x61, 0x64, + 0x69, 0x6f, 0x22, 0x20, 0x6e, 0x61, 0x6d, 0x65, 0x3d, 0x22, 0x6d, 0x69, + 0x72, 0x6f, 0x73, 0x74, 0x61, 0x74, 0x22, 0x20, 0x76, 0x61, 0x6c, 0x75, + 0x65, 0x3d, 0x22, 0x31, 0x22, 0x20, 0x63, 0x68, 0x65, 0x63, 0x6b, 0x65, + 0x64, 0x3d, 0x24, 0x7b, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x2e, 0x76, + 0x61, 0x6c, 0x75, 0x65, 0x2e, 0x6d, 0x69, 0x72, 0x6f, 0x73, 0x74, 0x61, + 0x74, 0x20, 0x3d, 0x3d, 0x20, 0x31, 0x7d, 0x20, 0x6f, 0x6e, 0x69, 0x6e, + 0x70, 0x75, 0x74, 0x3d, 0x24, 0x7b, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, + 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x49, 0x6e, 0x74, 0x7d, 0x20, 0x2f, + 0x3e, 0x20, 0x4d, 0x69, 0x72, 0x6f, 0x73, 0x74, 0x61, 0x74, 0x20, 0x76, + 0x31, 0x3c, 0x2f, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x3e, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x3c, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x3e, 0x3c, 0x69, 0x6e, + 0x70, 0x75, 0x74, 0x20, 0x74, 0x79, 0x70, 0x65, 0x3d, 0x22, 0x72, 0x61, + 0x64, 0x69, 0x6f, 0x22, 0x20, 0x6e, 0x61, 0x6d, 0x65, 0x3d, 0x22, 0x6d, + 0x69, 0x72, 0x6f, 0x73, 0x74, 0x61, 0x74, 0x22, 0x20, 0x76, 0x61, 0x6c, + 0x75, 0x65, 0x3d, 0x22, 0x32, 0x22, 0x20, 0x63, 0x68, 0x65, 0x63, 0x6b, + 0x65, 0x64, 0x3d, 0x24, 0x7b, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x2e, + 0x76, 0x61, 0x6c, 0x75, 0x65, 0x2e, 0x6d, 0x69, 0x72, 0x6f, 0x73, 0x74, + 0x61, 0x74, 0x20, 0x3d, 0x3d, 0x20, 0x32, 0x7d, 0x20, 0x6f, 0x6e, 0x69, + 0x6e, 0x70, 0x75, 0x74, 0x3d, 0x24, 0x7b, 0x75, 0x70, 0x64, 0x61, 0x74, + 0x65, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x49, 0x6e, 0x74, 0x7d, 0x20, + 0x2f, 0x3e, 0x20, 0x4d, 0x69, 0x72, 0x6f, 0x73, 0x74, 0x61, 0x74, 0x20, + 0x76, 0x32, 0x3c, 0x2f, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x3e, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x3c, 0x2f, 0x64, 0x69, 0x76, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x24, 0x7b, + 0x46, 0x6c, 0x6f, 0x61, 0x74, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x28, 0x7b, + 0x20, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x3a, 0x20, 0x22, 0x4d, 0x69, 0x72, + 0x6f, 0x73, 0x74, 0x61, 0x74, 0x20, 0x74, 0x61, 0x75, 0x22, 0x2c, 0x20, + 0x6d, 0x61, 0x78, 0x3a, 0x20, 0x31, 0x30, 0x2e, 0x30, 0x2c, 0x20, 0x6d, + 0x69, 0x6e, 0x3a, 0x20, 0x30, 0x2e, 0x30, 0x2c, 0x20, 0x6e, 0x61, 0x6d, + 0x65, 0x3a, 0x20, 0x22, 0x6d, 0x69, 0x72, 0x6f, 0x73, 0x74, 0x61, 0x74, + 0x5f, 0x74, 0x61, 0x75, 0x22, 0x2c, 0x20, 0x73, 0x74, 0x65, 0x70, 0x3a, + 0x20, 0x30, 0x2e, 0x30, 0x31, 0x2c, 0x20, 0x76, 0x61, 0x6c, 0x75, 0x65, + 0x3a, 0x20, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x2e, 0x76, 0x61, 0x6c, + 0x75, 0x65, 0x2e, 0x6d, 0x69, 0x72, 0x6f, 0x73, 0x74, 0x61, 0x74, 0x5f, + 0x74, 0x61, 0x75, 0x20, 0x7d, 0x29, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x24, 0x7b, + 0x46, 0x6c, 0x6f, 0x61, 0x74, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x28, 0x7b, + 0x20, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x3a, 0x20, 0x22, 0x4d, 0x69, 0x72, + 0x6f, 0x73, 0x74, 0x61, 0x74, 0x20, 0x65, 0x74, 0x61, 0x22, 0x2c, 0x20, + 0x6d, 0x61, 0x78, 0x3a, 0x20, 0x31, 0x2e, 0x30, 0x2c, 0x20, 0x6d, 0x69, + 0x6e, 0x3a, 0x20, 0x30, 0x2e, 0x30, 0x2c, 0x20, 0x6e, 0x61, 0x6d, 0x65, + 0x3a, 0x20, 0x22, 0x6d, 0x69, 0x72, 0x6f, 0x73, 0x74, 0x61, 0x74, 0x5f, + 0x65, 0x74, 0x61, 0x22, 0x2c, 0x20, 0x73, 0x74, 0x65, 0x70, 0x3a, 0x20, + 0x30, 0x2e, 0x30, 0x31, 0x2c, 0x20, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, + 0x20, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x2e, 0x76, 0x61, 0x6c, 0x75, + 0x65, 0x2e, 0x6d, 0x69, 0x72, 0x6f, 0x73, 0x74, 0x61, 0x74, 0x5f, 0x65, + 0x74, 0x61, 0x20, 0x7d, 0x29, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x2f, 0x66, 0x69, 0x65, + 0x6c, 0x64, 0x73, 0x65, 0x74, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x66, 0x69, 0x65, 0x6c, + 0x64, 0x73, 0x65, 0x74, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x24, 0x7b, 0x49, 0x6e, + 0x74, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x28, 0x7b, 0x20, 0x6c, 0x61, 0x62, + 0x65, 0x6c, 0x3a, 0x20, 0x22, 0x53, 0x68, 0x6f, 0x77, 0x20, 0x50, 0x72, + 0x6f, 0x62, 0x61, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x69, 0x65, 0x73, 0x22, + 0x2c, 0x20, 0x6d, 0x61, 0x78, 0x3a, 0x20, 0x31, 0x30, 0x2c, 0x20, 0x6d, + 0x69, 0x6e, 0x3a, 0x20, 0x30, 0x2c, 0x20, 0x6e, 0x61, 0x6d, 0x65, 0x3a, + 0x20, 0x22, 0x6e, 0x5f, 0x70, 0x72, 0x6f, 0x62, 0x73, 0x22, 0x2c, 0x20, + 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x20, 0x70, 0x61, 0x72, 0x61, 0x6d, + 0x73, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x2e, 0x6e, 0x5f, 0x70, 0x72, + 0x6f, 0x62, 0x73, 0x20, 0x7d, 0x29, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x2f, 0x66, 0x69, + 0x65, 0x6c, 0x64, 0x73, 0x65, 0x74, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x66, 0x69, 0x65, + 0x6c, 0x64, 0x73, 0x65, 0x74, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x6c, 0x61, + 0x62, 0x65, 0x6c, 0x20, 0x66, 0x6f, 0x72, 0x3d, 0x22, 0x61, 0x70, 0x69, + 0x5f, 0x6b, 0x65, 0x79, 0x22, 0x3e, 0x41, 0x50, 0x49, 0x20, 0x4b, 0x65, + 0x79, 0x3c, 0x2f, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x3e, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x3c, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x20, 0x74, 0x79, 0x70, 0x65, 0x3d, + 0x22, 0x74, 0x65, 0x78, 0x74, 0x22, 0x20, 0x6e, 0x61, 0x6d, 0x65, 0x3d, + 0x22, 0x61, 0x70, 0x69, 0x5f, 0x6b, 0x65, 0x79, 0x22, 0x20, 0x76, 0x61, + 0x6c, 0x75, 0x65, 0x3d, 0x22, 0x24, 0x7b, 0x70, 0x61, 0x72, 0x61, 0x6d, + 0x73, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x2e, 0x61, 0x70, 0x69, 0x5f, + 0x6b, 0x65, 0x79, 0x7d, 0x22, 0x20, 0x70, 0x6c, 0x61, 0x63, 0x65, 0x68, + 0x6f, 0x6c, 0x64, 0x65, 0x72, 0x3d, 0x22, 0x45, 0x6e, 0x74, 0x65, 0x72, + 0x20, 0x41, 0x50, 0x49, 0x20, 0x6b, 0x65, 0x79, 0x22, 0x20, 0x6f, 0x6e, + 0x69, 0x6e, 0x70, 0x75, 0x74, 0x3d, 0x24, 0x7b, 0x75, 0x70, 0x64, 0x61, + 0x74, 0x65, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x7d, 0x20, 0x2f, 0x3e, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x3c, 0x2f, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x65, 0x74, 0x3e, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, + 0x2f, 0x64, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x73, 0x3e, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x2f, 0x66, 0x6f, 0x72, 0x6d, + 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x60, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x7d, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, + 0x73, 0x74, 0x20, 0x70, 0x72, 0x6f, 0x62, 0x43, 0x6f, 0x6c, 0x6f, 0x72, + 0x20, 0x3d, 0x20, 0x28, 0x70, 0x29, 0x20, 0x3d, 0x3e, 0x20, 0x7b, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, + 0x72, 0x20, 0x3d, 0x20, 0x4d, 0x61, 0x74, 0x68, 0x2e, 0x66, 0x6c, 0x6f, + 0x6f, 0x72, 0x28, 0x31, 0x39, 0x32, 0x20, 0x2a, 0x20, 0x28, 0x31, 0x20, + 0x2d, 0x20, 0x70, 0x29, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x67, 0x20, 0x3d, 0x20, 0x4d, + 0x61, 0x74, 0x68, 0x2e, 0x66, 0x6c, 0x6f, 0x6f, 0x72, 0x28, 0x31, 0x39, + 0x32, 0x20, 0x2a, 0x20, 0x70, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x60, 0x72, 0x67, + 0x62, 0x61, 0x28, 0x24, 0x7b, 0x72, 0x7d, 0x2c, 0x24, 0x7b, 0x67, 0x7d, + 0x2c, 0x30, 0x2c, 0x30, 0x2e, 0x33, 0x29, 0x60, 0x3b, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x7d, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, + 0x73, 0x74, 0x20, 0x50, 0x72, 0x6f, 0x62, 0x61, 0x62, 0x69, 0x6c, 0x69, + 0x74, 0x69, 0x65, 0x73, 0x20, 0x3d, 0x20, 0x28, 0x70, 0x61, 0x72, 0x61, + 0x6d, 0x73, 0x29, 0x20, 0x3d, 0x3e, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x70, 0x61, + 0x72, 0x61, 0x6d, 0x73, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x6d, 0x61, + 0x70, 0x28, 0x6d, 0x73, 0x67, 0x20, 0x3d, 0x3e, 0x20, 0x7b, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, + 0x20, 0x7b, 0x20, 0x63, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x69, 0x6f, + 0x6e, 0x5f, 0x70, 0x72, 0x6f, 0x62, 0x61, 0x62, 0x69, 0x6c, 0x69, 0x74, + 0x69, 0x65, 0x73, 0x20, 0x7d, 0x20, 0x3d, 0x20, 0x6d, 0x73, 0x67, 0x3b, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, + 0x28, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x21, 0x63, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x69, 0x6f, 0x6e, 0x5f, + 0x70, 0x72, 0x6f, 0x62, 0x61, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x69, 0x65, + 0x73, 0x20, 0x7c, 0x7c, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x69, 0x6f, + 0x6e, 0x5f, 0x70, 0x72, 0x6f, 0x62, 0x61, 0x62, 0x69, 0x6c, 0x69, 0x74, + 0x69, 0x65, 0x73, 0x2e, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x20, 0x3d, + 0x3d, 0x3d, 0x20, 0x30, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x29, 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x6d, 0x73, + 0x67, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x0a, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, 0x63, + 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x70, 0x72, + 0x6f, 0x62, 0x61, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x69, 0x65, 0x73, 0x2e, + 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x20, 0x3e, 0x20, 0x31, 0x29, 0x20, + 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x2f, 0x2f, 0x20, 0x4e, 0x6f, 0x74, 0x20, 0x66, 0x6f, 0x72, 0x20, 0x62, + 0x79, 0x74, 0x65, 0x20, 0x70, 0x61, 0x69, 0x72, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, 0x63, + 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x70, 0x72, + 0x6f, 0x62, 0x61, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x69, 0x65, 0x73, 0x5b, + 0x30, 0x5d, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x2e, 0x73, + 0x74, 0x61, 0x72, 0x74, 0x73, 0x57, 0x69, 0x74, 0x68, 0x28, 0x27, 0x62, + 0x79, 0x74, 0x65, 0x3a, 0x20, 0x5c, 0x5c, 0x27, 0x29, 0x29, 0x20, 0x72, + 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x6d, 0x73, 0x67, 0x2e, 0x63, 0x6f, + 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x73, + 0x70, 0x6c, 0x69, 0x74, 0x44, 0x61, 0x74, 0x61, 0x20, 0x3d, 0x20, 0x63, + 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x70, 0x72, + 0x6f, 0x62, 0x61, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x69, 0x65, 0x73, 0x2e, + 0x6d, 0x61, 0x70, 0x28, 0x70, 0x72, 0x6f, 0x62, 0x20, 0x3d, 0x3e, 0x20, + 0x28, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x3a, 0x20, + 0x70, 0x72, 0x6f, 0x62, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, + 0x2c, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x63, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x69, 0x6f, 0x6e, + 0x5f, 0x70, 0x72, 0x6f, 0x62, 0x61, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x69, + 0x65, 0x73, 0x3a, 0x20, 0x5b, 0x70, 0x72, 0x6f, 0x62, 0x5d, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x29, 0x29, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, + 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x68, 0x74, 0x6d, 0x6c, 0x60, 0x3c, + 0x24, 0x7b, 0x50, 0x72, 0x6f, 0x62, 0x61, 0x62, 0x69, 0x6c, 0x69, 0x74, + 0x69, 0x65, 0x73, 0x7d, 0x20, 0x64, 0x61, 0x74, 0x61, 0x3d, 0x24, 0x7b, + 0x73, 0x70, 0x6c, 0x69, 0x74, 0x44, 0x61, 0x74, 0x61, 0x7d, 0x20, 0x2f, + 0x3e, 0x60, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, + 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, + 0x6e, 0x73, 0x74, 0x20, 0x7b, 0x20, 0x70, 0x72, 0x6f, 0x62, 0x73, 0x2c, + 0x20, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x20, 0x7d, 0x20, 0x3d, + 0x20, 0x63, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x69, 0x6f, 0x6e, 0x5f, + 0x70, 0x72, 0x6f, 0x62, 0x61, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x69, 0x65, + 0x73, 0x5b, 0x30, 0x5d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x66, 0x6f, 0x75, 0x6e, 0x64, + 0x20, 0x3d, 0x20, 0x70, 0x72, 0x6f, 0x62, 0x73, 0x2e, 0x66, 0x69, 0x6e, + 0x64, 0x28, 0x70, 0x20, 0x3d, 0x3e, 0x20, 0x70, 0x2e, 0x74, 0x6f, 0x6b, + 0x5f, 0x73, 0x74, 0x72, 0x20, 0x3d, 0x3d, 0x3d, 0x20, 0x6d, 0x73, 0x67, + 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x29, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, + 0x70, 0x43, 0x6f, 0x6c, 0x6f, 0x72, 0x20, 0x3d, 0x20, 0x66, 0x6f, 0x75, + 0x6e, 0x64, 0x20, 0x3f, 0x20, 0x70, 0x72, 0x6f, 0x62, 0x43, 0x6f, 0x6c, + 0x6f, 0x72, 0x28, 0x66, 0x6f, 0x75, 0x6e, 0x64, 0x2e, 0x70, 0x72, 0x6f, + 0x62, 0x29, 0x20, 0x3a, 0x20, 0x27, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x70, + 0x61, 0x72, 0x65, 0x6e, 0x74, 0x27, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x70, 0x6f, + 0x70, 0x6f, 0x76, 0x65, 0x72, 0x43, 0x68, 0x69, 0x6c, 0x64, 0x72, 0x65, + 0x6e, 0x20, 0x3d, 0x20, 0x68, 0x74, 0x6d, 0x6c, 0x60, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x64, 0x69, 0x76, + 0x20, 0x63, 0x6c, 0x61, 0x73, 0x73, 0x3d, 0x22, 0x70, 0x72, 0x6f, 0x62, + 0x2d, 0x73, 0x65, 0x74, 0x22, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x24, 0x7b, 0x70, 0x72, 0x6f, + 0x62, 0x73, 0x2e, 0x6d, 0x61, 0x70, 0x28, 0x28, 0x70, 0x2c, 0x20, 0x69, + 0x6e, 0x64, 0x65, 0x78, 0x29, 0x20, 0x3d, 0x3e, 0x20, 0x7b, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, 0x74, + 0x75, 0x72, 0x6e, 0x20, 0x68, 0x74, 0x6d, 0x6c, 0x60, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x3c, 0x64, 0x69, 0x76, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x6b, 0x65, 0x79, 0x3d, 0x24, 0x7b, 0x69, 0x6e, 0x64, 0x65, 0x78, + 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x74, 0x69, 0x74, 0x6c, + 0x65, 0x3d, 0x24, 0x7b, 0x60, 0x70, 0x72, 0x6f, 0x62, 0x3a, 0x20, 0x24, + 0x7b, 0x70, 0x2e, 0x70, 0x72, 0x6f, 0x62, 0x7d, 0x60, 0x7d, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x73, 0x74, 0x79, 0x6c, 0x65, 0x3d, 0x24, + 0x7b, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x70, 0x61, 0x64, 0x64, 0x69, 0x6e, 0x67, + 0x3a, 0x20, 0x27, 0x30, 0x2e, 0x33, 0x65, 0x6d, 0x27, 0x2c, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x62, 0x61, 0x63, 0x6b, 0x67, 0x72, 0x6f, 0x75, 0x6e, 0x64, 0x43, + 0x6f, 0x6c, 0x6f, 0x72, 0x3a, 0x20, 0x70, 0x2e, 0x74, 0x6f, 0x6b, 0x5f, + 0x73, 0x74, 0x72, 0x20, 0x3d, 0x3d, 0x3d, 0x20, 0x63, 0x6f, 0x6e, 0x74, + 0x65, 0x6e, 0x74, 0x20, 0x3f, 0x20, 0x70, 0x72, 0x6f, 0x62, 0x43, 0x6f, + 0x6c, 0x6f, 0x72, 0x28, 0x70, 0x2e, 0x70, 0x72, 0x6f, 0x62, 0x29, 0x20, + 0x3a, 0x20, 0x27, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x70, 0x61, 0x72, 0x65, + 0x6e, 0x74, 0x27, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x7d, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3e, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x73, 0x70, 0x61, 0x6e, + 0x3e, 0x24, 0x7b, 0x70, 0x2e, 0x74, 0x6f, 0x6b, 0x5f, 0x73, 0x74, 0x72, + 0x7d, 0x3a, 0x20, 0x3c, 0x2f, 0x73, 0x70, 0x61, 0x6e, 0x3e, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x73, 0x70, 0x61, 0x6e, 0x3e, 0x24, + 0x7b, 0x4d, 0x61, 0x74, 0x68, 0x2e, 0x66, 0x6c, 0x6f, 0x6f, 0x72, 0x28, + 0x70, 0x2e, 0x70, 0x72, 0x6f, 0x62, 0x20, 0x2a, 0x20, 0x31, 0x30, 0x30, + 0x29, 0x7d, 0x25, 0x3c, 0x2f, 0x73, 0x70, 0x61, 0x6e, 0x3e, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x3c, 0x2f, 0x64, 0x69, 0x76, 0x3e, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x60, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x29, + 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x3c, 0x2f, 0x64, 0x69, 0x76, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x60, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x68, 0x74, 0x6d, + 0x6c, 0x60, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x3c, 0x24, 0x7b, 0x50, 0x6f, 0x70, 0x6f, 0x76, 0x65, 0x72, 0x7d, + 0x20, 0x73, 0x74, 0x79, 0x6c, 0x65, 0x3d, 0x24, 0x7b, 0x7b, 0x20, 0x62, + 0x61, 0x63, 0x6b, 0x67, 0x72, 0x6f, 0x75, 0x6e, 0x64, 0x43, 0x6f, 0x6c, + 0x6f, 0x72, 0x3a, 0x20, 0x70, 0x43, 0x6f, 0x6c, 0x6f, 0x72, 0x20, 0x7d, + 0x7d, 0x20, 0x70, 0x6f, 0x70, 0x6f, 0x76, 0x65, 0x72, 0x43, 0x68, 0x69, + 0x6c, 0x64, 0x72, 0x65, 0x6e, 0x3d, 0x24, 0x7b, 0x70, 0x6f, 0x70, 0x6f, + 0x76, 0x65, 0x72, 0x43, 0x68, 0x69, 0x6c, 0x64, 0x72, 0x65, 0x6e, 0x7d, + 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x24, 0x7b, 0x6d, 0x73, 0x67, 0x2e, 0x63, 0x6f, 0x6e, 0x74, + 0x65, 0x6e, 0x74, 0x2e, 0x6d, 0x61, 0x74, 0x63, 0x68, 0x28, 0x2f, 0x5c, + 0x6e, 0x2f, 0x67, 0x69, 0x6d, 0x29, 0x20, 0x3f, 0x20, 0x68, 0x74, 0x6d, + 0x6c, 0x60, 0x3c, 0x62, 0x72, 0x20, 0x2f, 0x3e, 0x60, 0x20, 0x3a, 0x20, + 0x6d, 0x73, 0x67, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x7d, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, + 0x2f, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x60, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x29, 0x3b, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x2f, 0x2f, + 0x20, 0x70, 0x6f, 0x6f, 0x72, 0x20, 0x6d, 0x61, 0x6e, 0x73, 0x20, 0x6d, + 0x61, 0x72, 0x6b, 0x64, 0x6f, 0x77, 0x6e, 0x20, 0x72, 0x65, 0x70, 0x6c, + 0x61, 0x63, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x4d, 0x61, 0x72, 0x6b, 0x64, 0x6f, + 0x77, 0x6e, 0x69, 0x73, 0x68, 0x20, 0x3d, 0x20, 0x28, 0x70, 0x61, 0x72, + 0x61, 0x6d, 0x73, 0x29, 0x20, 0x3d, 0x3e, 0x20, 0x7b, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x6d, 0x64, + 0x20, 0x3d, 0x20, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x2e, 0x74, 0x65, + 0x78, 0x74, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x2e, + 0x72, 0x65, 0x70, 0x6c, 0x61, 0x63, 0x65, 0x28, 0x2f, 0x26, 0x2f, 0x67, + 0x2c, 0x20, 0x27, 0x26, 0x61, 0x6d, 0x70, 0x3b, 0x27, 0x29, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x2e, 0x72, 0x65, 0x70, 0x6c, + 0x61, 0x63, 0x65, 0x28, 0x2f, 0x3c, 0x2f, 0x67, 0x2c, 0x20, 0x27, 0x26, + 0x6c, 0x74, 0x3b, 0x27, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x2e, 0x72, 0x65, 0x70, 0x6c, 0x61, 0x63, 0x65, 0x28, 0x2f, + 0x3e, 0x2f, 0x67, 0x2c, 0x20, 0x27, 0x26, 0x67, 0x74, 0x3b, 0x27, 0x29, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x2e, 0x72, 0x65, + 0x70, 0x6c, 0x61, 0x63, 0x65, 0x28, 0x2f, 0x5e, 0x23, 0x7b, 0x31, 0x2c, + 0x36, 0x7d, 0x20, 0x28, 0x2e, 0x2a, 0x29, 0x24, 0x2f, 0x67, 0x69, 0x6d, + 0x2c, 0x20, 0x27, 0x3c, 0x68, 0x33, 0x3e, 0x24, 0x31, 0x3c, 0x2f, 0x68, + 0x33, 0x3e, 0x27, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x2e, 0x72, 0x65, 0x70, 0x6c, 0x61, 0x63, 0x65, 0x28, 0x2f, 0x5c, + 0x2a, 0x5c, 0x2a, 0x28, 0x2e, 0x2a, 0x3f, 0x29, 0x5c, 0x2a, 0x5c, 0x2a, + 0x2f, 0x67, 0x2c, 0x20, 0x27, 0x3c, 0x73, 0x74, 0x72, 0x6f, 0x6e, 0x67, + 0x3e, 0x24, 0x31, 0x3c, 0x2f, 0x73, 0x74, 0x72, 0x6f, 0x6e, 0x67, 0x3e, + 0x27, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x2e, + 0x72, 0x65, 0x70, 0x6c, 0x61, 0x63, 0x65, 0x28, 0x2f, 0x5f, 0x5f, 0x28, + 0x2e, 0x2a, 0x3f, 0x29, 0x5f, 0x5f, 0x2f, 0x67, 0x2c, 0x20, 0x27, 0x3c, + 0x73, 0x74, 0x72, 0x6f, 0x6e, 0x67, 0x3e, 0x24, 0x31, 0x3c, 0x2f, 0x73, + 0x74, 0x72, 0x6f, 0x6e, 0x67, 0x3e, 0x27, 0x29, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x2e, 0x72, 0x65, 0x70, 0x6c, 0x61, 0x63, + 0x65, 0x28, 0x2f, 0x5c, 0x2a, 0x28, 0x2e, 0x2a, 0x3f, 0x29, 0x5c, 0x2a, + 0x2f, 0x67, 0x2c, 0x20, 0x27, 0x3c, 0x65, 0x6d, 0x3e, 0x24, 0x31, 0x3c, + 0x2f, 0x65, 0x6d, 0x3e, 0x27, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x2e, 0x72, 0x65, 0x70, 0x6c, 0x61, 0x63, 0x65, 0x28, + 0x2f, 0x5f, 0x28, 0x2e, 0x2a, 0x3f, 0x29, 0x5f, 0x2f, 0x67, 0x2c, 0x20, + 0x27, 0x3c, 0x65, 0x6d, 0x3e, 0x24, 0x31, 0x3c, 0x2f, 0x65, 0x6d, 0x3e, + 0x27, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x2e, + 0x72, 0x65, 0x70, 0x6c, 0x61, 0x63, 0x65, 0x28, 0x2f, 0x60, 0x60, 0x60, + 0x2e, 0x2a, 0x3f, 0x5c, 0x6e, 0x28, 0x5b, 0x5c, 0x73, 0x5c, 0x53, 0x5d, + 0x2a, 0x3f, 0x29, 0x60, 0x60, 0x60, 0x2f, 0x67, 0x2c, 0x20, 0x27, 0x3c, + 0x70, 0x72, 0x65, 0x3e, 0x3c, 0x63, 0x6f, 0x64, 0x65, 0x3e, 0x24, 0x31, + 0x3c, 0x2f, 0x63, 0x6f, 0x64, 0x65, 0x3e, 0x3c, 0x2f, 0x70, 0x72, 0x65, + 0x3e, 0x27, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x2e, 0x72, 0x65, 0x70, 0x6c, 0x61, 0x63, 0x65, 0x28, 0x2f, 0x60, 0x28, + 0x2e, 0x2a, 0x3f, 0x29, 0x60, 0x2f, 0x67, 0x2c, 0x20, 0x27, 0x3c, 0x63, + 0x6f, 0x64, 0x65, 0x3e, 0x24, 0x31, 0x3c, 0x2f, 0x63, 0x6f, 0x64, 0x65, + 0x3e, 0x27, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x2e, 0x72, 0x65, 0x70, 0x6c, 0x61, 0x63, 0x65, 0x28, 0x2f, 0x5c, 0x6e, + 0x2f, 0x67, 0x69, 0x6d, 0x2c, 0x20, 0x27, 0x3c, 0x62, 0x72, 0x20, 0x2f, + 0x3e, 0x27, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, + 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x68, 0x74, 0x6d, 0x6c, 0x60, 0x3c, + 0x73, 0x70, 0x61, 0x6e, 0x20, 0x64, 0x61, 0x6e, 0x67, 0x65, 0x72, 0x6f, + 0x75, 0x73, 0x6c, 0x79, 0x53, 0x65, 0x74, 0x49, 0x6e, 0x6e, 0x65, 0x72, + 0x48, 0x54, 0x4d, 0x4c, 0x3d, 0x24, 0x7b, 0x7b, 0x20, 0x5f, 0x5f, 0x68, + 0x74, 0x6d, 0x6c, 0x3a, 0x20, 0x6d, 0x64, 0x20, 0x7d, 0x7d, 0x20, 0x2f, + 0x3e, 0x60, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x3b, 0x0a, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x4d, 0x6f, + 0x64, 0x65, 0x6c, 0x47, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, + 0x6e, 0x49, 0x6e, 0x66, 0x6f, 0x20, 0x3d, 0x20, 0x28, 0x70, 0x61, 0x72, + 0x61, 0x6d, 0x73, 0x29, 0x20, 0x3d, 0x3e, 0x20, 0x7b, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, 0x21, 0x6c, 0x6c, 0x61, + 0x6d, 0x61, 0x53, 0x74, 0x61, 0x74, 0x73, 0x2e, 0x76, 0x61, 0x6c, 0x75, + 0x65, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x68, 0x74, 0x6d, 0x6c, + 0x60, 0x3c, 0x73, 0x70, 0x61, 0x6e, 0x2f, 0x3e, 0x60, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x68, 0x74, 0x6d, 0x6c, 0x60, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x73, 0x70, + 0x61, 0x6e, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x24, 0x7b, 0x6c, 0x6c, 0x61, 0x6d, 0x61, 0x53, 0x74, 0x61, + 0x74, 0x73, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x2e, 0x74, 0x6f, 0x6b, + 0x65, 0x6e, 0x73, 0x5f, 0x70, 0x72, 0x65, 0x64, 0x69, 0x63, 0x74, 0x65, + 0x64, 0x7d, 0x20, 0x70, 0x72, 0x65, 0x64, 0x69, 0x63, 0x74, 0x65, 0x64, + 0x2c, 0x20, 0x24, 0x7b, 0x6c, 0x6c, 0x61, 0x6d, 0x61, 0x53, 0x74, 0x61, + 0x74, 0x73, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x2e, 0x74, 0x6f, 0x6b, + 0x65, 0x6e, 0x73, 0x5f, 0x63, 0x61, 0x63, 0x68, 0x65, 0x64, 0x7d, 0x20, + 0x63, 0x61, 0x63, 0x68, 0x65, 0x64, 0x2c, 0x20, 0x24, 0x7b, 0x6c, 0x6c, + 0x61, 0x6d, 0x61, 0x53, 0x74, 0x61, 0x74, 0x73, 0x2e, 0x76, 0x61, 0x6c, + 0x75, 0x65, 0x2e, 0x74, 0x69, 0x6d, 0x69, 0x6e, 0x67, 0x73, 0x2e, 0x70, + 0x72, 0x65, 0x64, 0x69, 0x63, 0x74, 0x65, 0x64, 0x5f, 0x70, 0x65, 0x72, + 0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x5f, 0x6d, 0x73, 0x2e, 0x74, 0x6f, + 0x46, 0x69, 0x78, 0x65, 0x64, 0x28, 0x29, 0x7d, 0x6d, 0x73, 0x20, 0x70, + 0x65, 0x72, 0x20, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x2c, 0x20, 0x24, 0x7b, + 0x6c, 0x6c, 0x61, 0x6d, 0x61, 0x53, 0x74, 0x61, 0x74, 0x73, 0x2e, 0x76, + 0x61, 0x6c, 0x75, 0x65, 0x2e, 0x74, 0x69, 0x6d, 0x69, 0x6e, 0x67, 0x73, + 0x2e, 0x70, 0x72, 0x65, 0x64, 0x69, 0x63, 0x74, 0x65, 0x64, 0x5f, 0x70, + 0x65, 0x72, 0x5f, 0x73, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x2e, 0x74, 0x6f, + 0x46, 0x69, 0x78, 0x65, 0x64, 0x28, 0x32, 0x29, 0x7d, 0x20, 0x74, 0x6f, + 0x6b, 0x65, 0x6e, 0x73, 0x20, 0x70, 0x65, 0x72, 0x20, 0x73, 0x65, 0x63, + 0x6f, 0x6e, 0x64, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x3c, 0x2f, 0x73, 0x70, 0x61, 0x6e, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x60, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x2f, 0x2f, 0x20, 0x73, 0x69, 0x6d, 0x70, 0x6c, 0x65, + 0x20, 0x70, 0x6f, 0x70, 0x6f, 0x76, 0x65, 0x72, 0x20, 0x69, 0x6d, 0x70, + 0x6c, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, + 0x50, 0x6f, 0x70, 0x6f, 0x76, 0x65, 0x72, 0x20, 0x3d, 0x20, 0x28, 0x70, + 0x72, 0x6f, 0x70, 0x73, 0x29, 0x20, 0x3d, 0x3e, 0x20, 0x7b, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x69, + 0x73, 0x4f, 0x70, 0x65, 0x6e, 0x20, 0x3d, 0x20, 0x75, 0x73, 0x65, 0x53, + 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x28, 0x66, 0x61, 0x6c, 0x73, 0x65, 0x29, + 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, + 0x74, 0x20, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x3d, + 0x20, 0x75, 0x73, 0x65, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x28, 0x7b, + 0x20, 0x74, 0x6f, 0x70, 0x3a, 0x20, 0x27, 0x30, 0x70, 0x78, 0x27, 0x2c, + 0x20, 0x6c, 0x65, 0x66, 0x74, 0x3a, 0x20, 0x27, 0x30, 0x70, 0x78, 0x27, + 0x20, 0x7d, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, + 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x62, 0x75, 0x74, 0x74, 0x6f, 0x6e, 0x52, + 0x65, 0x66, 0x20, 0x3d, 0x20, 0x75, 0x73, 0x65, 0x52, 0x65, 0x66, 0x28, + 0x6e, 0x75, 0x6c, 0x6c, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x70, 0x6f, 0x70, 0x6f, 0x76, + 0x65, 0x72, 0x52, 0x65, 0x66, 0x20, 0x3d, 0x20, 0x75, 0x73, 0x65, 0x52, + 0x65, 0x66, 0x28, 0x6e, 0x75, 0x6c, 0x6c, 0x29, 0x3b, 0x0a, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x74, + 0x6f, 0x67, 0x67, 0x6c, 0x65, 0x50, 0x6f, 0x70, 0x6f, 0x76, 0x65, 0x72, + 0x20, 0x3d, 0x20, 0x28, 0x29, 0x20, 0x3d, 0x3e, 0x20, 0x7b, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, 0x62, + 0x75, 0x74, 0x74, 0x6f, 0x6e, 0x52, 0x65, 0x66, 0x2e, 0x63, 0x75, 0x72, + 0x72, 0x65, 0x6e, 0x74, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, + 0x72, 0x65, 0x63, 0x74, 0x20, 0x3d, 0x20, 0x62, 0x75, 0x74, 0x74, 0x6f, + 0x6e, 0x52, 0x65, 0x66, 0x2e, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x74, + 0x2e, 0x67, 0x65, 0x74, 0x42, 0x6f, 0x75, 0x6e, 0x64, 0x69, 0x6e, 0x67, + 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x63, 0x74, 0x28, 0x29, + 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x70, 0x6f, 0x73, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x61, 0x6c, + 0x75, 0x65, 0x20, 0x3d, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x74, 0x6f, 0x70, 0x3a, 0x20, + 0x60, 0x24, 0x7b, 0x72, 0x65, 0x63, 0x74, 0x2e, 0x62, 0x6f, 0x74, 0x74, + 0x6f, 0x6d, 0x20, 0x2b, 0x20, 0x77, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x2e, + 0x73, 0x63, 0x72, 0x6f, 0x6c, 0x6c, 0x59, 0x7d, 0x70, 0x78, 0x60, 0x2c, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x6c, 0x65, 0x66, 0x74, 0x3a, 0x20, 0x60, 0x24, 0x7b, 0x72, 0x65, + 0x63, 0x74, 0x2e, 0x6c, 0x65, 0x66, 0x74, 0x20, 0x2b, 0x20, 0x77, 0x69, + 0x6e, 0x64, 0x6f, 0x77, 0x2e, 0x73, 0x63, 0x72, 0x6f, 0x6c, 0x6c, 0x58, + 0x7d, 0x70, 0x78, 0x60, 0x2c, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x7d, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x69, 0x73, 0x4f, 0x70, 0x65, 0x6e, 0x2e, 0x76, 0x61, 0x6c, 0x75, + 0x65, 0x20, 0x3d, 0x20, 0x21, 0x69, 0x73, 0x4f, 0x70, 0x65, 0x6e, 0x2e, + 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x7d, 0x3b, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, + 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x68, 0x61, 0x6e, 0x64, 0x6c, 0x65, 0x43, + 0x6c, 0x69, 0x63, 0x6b, 0x4f, 0x75, 0x74, 0x73, 0x69, 0x64, 0x65, 0x20, + 0x3d, 0x20, 0x28, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x29, 0x20, 0x3d, 0x3e, + 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, + 0x66, 0x20, 0x28, 0x70, 0x6f, 0x70, 0x6f, 0x76, 0x65, 0x72, 0x52, 0x65, + 0x66, 0x2e, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x74, 0x20, 0x26, 0x26, + 0x20, 0x21, 0x70, 0x6f, 0x70, 0x6f, 0x76, 0x65, 0x72, 0x52, 0x65, 0x66, + 0x2e, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x74, 0x2e, 0x63, 0x6f, 0x6e, + 0x74, 0x61, 0x69, 0x6e, 0x73, 0x28, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x2e, + 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x29, 0x20, 0x26, 0x26, 0x20, 0x21, + 0x62, 0x75, 0x74, 0x74, 0x6f, 0x6e, 0x52, 0x65, 0x66, 0x2e, 0x63, 0x75, + 0x72, 0x72, 0x65, 0x6e, 0x74, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, + 0x6e, 0x73, 0x28, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x2e, 0x74, 0x61, 0x72, + 0x67, 0x65, 0x74, 0x29, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x73, 0x4f, 0x70, 0x65, 0x6e, + 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x20, 0x3d, 0x20, 0x66, 0x61, 0x6c, + 0x73, 0x65, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x3b, 0x0a, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x75, 0x73, 0x65, 0x45, 0x66, 0x66, + 0x65, 0x63, 0x74, 0x28, 0x28, 0x29, 0x20, 0x3d, 0x3e, 0x20, 0x7b, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x64, 0x6f, 0x63, 0x75, + 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x61, 0x64, 0x64, 0x45, 0x76, 0x65, 0x6e, + 0x74, 0x4c, 0x69, 0x73, 0x74, 0x65, 0x6e, 0x65, 0x72, 0x28, 0x27, 0x6d, + 0x6f, 0x75, 0x73, 0x65, 0x64, 0x6f, 0x77, 0x6e, 0x27, 0x2c, 0x20, 0x68, + 0x61, 0x6e, 0x64, 0x6c, 0x65, 0x43, 0x6c, 0x69, 0x63, 0x6b, 0x4f, 0x75, + 0x74, 0x73, 0x69, 0x64, 0x65, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x28, + 0x29, 0x20, 0x3d, 0x3e, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x64, 0x6f, 0x63, 0x75, 0x6d, 0x65, 0x6e, + 0x74, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x45, 0x76, 0x65, 0x6e, + 0x74, 0x4c, 0x69, 0x73, 0x74, 0x65, 0x6e, 0x65, 0x72, 0x28, 0x27, 0x6d, + 0x6f, 0x75, 0x73, 0x65, 0x64, 0x6f, 0x77, 0x6e, 0x27, 0x2c, 0x20, 0x68, + 0x61, 0x6e, 0x64, 0x6c, 0x65, 0x43, 0x6c, 0x69, 0x63, 0x6b, 0x4f, 0x75, + 0x74, 0x73, 0x69, 0x64, 0x65, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x7d, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x7d, 0x2c, 0x20, 0x5b, 0x5d, 0x29, 0x3b, 0x0a, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x68, + 0x74, 0x6d, 0x6c, 0x60, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x3c, 0x73, 0x70, 0x61, 0x6e, 0x20, 0x73, 0x74, 0x79, 0x6c, 0x65, + 0x3d, 0x24, 0x7b, 0x70, 0x72, 0x6f, 0x70, 0x73, 0x2e, 0x73, 0x74, 0x79, + 0x6c, 0x65, 0x7d, 0x20, 0x72, 0x65, 0x66, 0x3d, 0x24, 0x7b, 0x62, 0x75, + 0x74, 0x74, 0x6f, 0x6e, 0x52, 0x65, 0x66, 0x7d, 0x20, 0x6f, 0x6e, 0x43, + 0x6c, 0x69, 0x63, 0x6b, 0x3d, 0x24, 0x7b, 0x74, 0x6f, 0x67, 0x67, 0x6c, + 0x65, 0x50, 0x6f, 0x70, 0x6f, 0x76, 0x65, 0x72, 0x7d, 0x3e, 0x24, 0x7b, + 0x70, 0x72, 0x6f, 0x70, 0x73, 0x2e, 0x63, 0x68, 0x69, 0x6c, 0x64, 0x72, + 0x65, 0x6e, 0x7d, 0x3c, 0x2f, 0x73, 0x70, 0x61, 0x6e, 0x3e, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x24, 0x7b, 0x69, 0x73, 0x4f, + 0x70, 0x65, 0x6e, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x20, 0x26, 0x26, + 0x20, 0x68, 0x74, 0x6d, 0x6c, 0x60, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x24, 0x7b, 0x50, 0x6f, 0x72, 0x74, + 0x61, 0x6c, 0x7d, 0x20, 0x69, 0x6e, 0x74, 0x6f, 0x3d, 0x22, 0x23, 0x70, + 0x6f, 0x72, 0x74, 0x61, 0x6c, 0x22, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x64, 0x69, 0x76, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x72, 0x65, 0x66, 0x3d, 0x24, 0x7b, 0x70, 0x6f, 0x70, + 0x6f, 0x76, 0x65, 0x72, 0x52, 0x65, 0x66, 0x7d, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, + 0x6c, 0x61, 0x73, 0x73, 0x3d, 0x22, 0x70, 0x6f, 0x70, 0x6f, 0x76, 0x65, + 0x72, 0x2d, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x22, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x73, 0x74, 0x79, 0x6c, 0x65, 0x3d, 0x24, 0x7b, 0x7b, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x74, + 0x6f, 0x70, 0x3a, 0x20, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x69, 0x6f, 0x6e, + 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x2e, 0x74, 0x6f, 0x70, 0x2c, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x6c, 0x65, 0x66, 0x74, 0x3a, 0x20, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x69, + 0x6f, 0x6e, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x2e, 0x6c, 0x65, 0x66, + 0x74, 0x2c, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x7d, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x24, 0x7b, 0x70, 0x72, + 0x6f, 0x70, 0x73, 0x2e, 0x70, 0x6f, 0x70, 0x6f, 0x76, 0x65, 0x72, 0x43, + 0x68, 0x69, 0x6c, 0x64, 0x72, 0x65, 0x6e, 0x7d, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x2f, 0x64, + 0x69, 0x76, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x3c, 0x2f, 0x24, 0x7b, 0x50, 0x6f, 0x72, 0x74, 0x61, 0x6c, + 0x7d, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x60, + 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x60, 0x3b, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x7d, 0x3b, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x2f, + 0x2f, 0x20, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x3a, 0x20, 0x70, 0x72, + 0x65, 0x61, 0x63, 0x74, 0x2d, 0x70, 0x6f, 0x72, 0x74, 0x61, 0x6c, 0x20, + 0x28, 0x68, 0x74, 0x74, 0x70, 0x73, 0x3a, 0x2f, 0x2f, 0x67, 0x69, 0x74, + 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x64, 0x65, 0x76, 0x65, + 0x6c, 0x6f, 0x70, 0x69, 0x74, 0x2f, 0x70, 0x72, 0x65, 0x61, 0x63, 0x74, + 0x2d, 0x70, 0x6f, 0x72, 0x74, 0x61, 0x6c, 0x2f, 0x62, 0x6c, 0x6f, 0x62, + 0x2f, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x2f, 0x73, 0x72, 0x63, 0x2f, + 0x70, 0x72, 0x65, 0x61, 0x63, 0x74, 0x2d, 0x70, 0x6f, 0x72, 0x74, 0x61, + 0x6c, 0x2e, 0x6a, 0x73, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x2f, 0x2a, + 0x2a, 0x20, 0x52, 0x65, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x20, 0x72, + 0x65, 0x6e, 0x64, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x20, 0x6f, 0x66, 0x20, + 0x64, 0x65, 0x73, 0x63, 0x65, 0x6e, 0x64, 0x61, 0x6e, 0x74, 0x73, 0x20, + 0x69, 0x6e, 0x74, 0x6f, 0x20, 0x74, 0x68, 0x65, 0x20, 0x67, 0x69, 0x76, + 0x65, 0x6e, 0x20, 0x43, 0x53, 0x53, 0x20, 0x73, 0x65, 0x6c, 0x65, 0x63, + 0x74, 0x6f, 0x72, 0x20, 0x2a, 0x2f, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x63, + 0x6c, 0x61, 0x73, 0x73, 0x20, 0x50, 0x6f, 0x72, 0x74, 0x61, 0x6c, 0x20, + 0x65, 0x78, 0x74, 0x65, 0x6e, 0x64, 0x73, 0x20, 0x43, 0x6f, 0x6d, 0x70, + 0x6f, 0x6e, 0x65, 0x6e, 0x74, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x63, 0x6f, 0x6d, 0x70, 0x6f, 0x6e, 0x65, 0x6e, 0x74, 0x44, + 0x69, 0x64, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x28, 0x70, 0x72, 0x6f, + 0x70, 0x73, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x66, 0x6f, 0x72, 0x20, 0x28, 0x6c, 0x65, 0x74, 0x20, 0x69, + 0x20, 0x69, 0x6e, 0x20, 0x70, 0x72, 0x6f, 0x70, 0x73, 0x29, 0x20, 0x7b, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, + 0x66, 0x20, 0x28, 0x70, 0x72, 0x6f, 0x70, 0x73, 0x5b, 0x69, 0x5d, 0x20, + 0x21, 0x3d, 0x3d, 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x70, 0x72, 0x6f, + 0x70, 0x73, 0x5b, 0x69, 0x5d, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, 0x74, + 0x75, 0x72, 0x6e, 0x20, 0x73, 0x65, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x6f, + 0x75, 0x74, 0x28, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x72, 0x65, 0x6e, 0x64, + 0x65, 0x72, 0x4c, 0x61, 0x79, 0x65, 0x72, 0x29, 0x3b, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x7d, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, + 0x6f, 0x6d, 0x70, 0x6f, 0x6e, 0x65, 0x6e, 0x74, 0x44, 0x69, 0x64, 0x4d, + 0x6f, 0x75, 0x6e, 0x74, 0x28, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x69, 0x73, + 0x4d, 0x6f, 0x75, 0x6e, 0x74, 0x65, 0x64, 0x20, 0x3d, 0x20, 0x74, 0x72, + 0x75, 0x65, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x74, 0x68, 0x69, 0x73, 0x2e, 0x72, 0x65, 0x6e, 0x64, 0x65, 0x72, 0x4c, + 0x61, 0x79, 0x65, 0x72, 0x20, 0x3d, 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, + 0x72, 0x65, 0x6e, 0x64, 0x65, 0x72, 0x4c, 0x61, 0x79, 0x65, 0x72, 0x2e, + 0x62, 0x69, 0x6e, 0x64, 0x28, 0x74, 0x68, 0x69, 0x73, 0x29, 0x3b, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x74, 0x68, 0x69, 0x73, + 0x2e, 0x72, 0x65, 0x6e, 0x64, 0x65, 0x72, 0x4c, 0x61, 0x79, 0x65, 0x72, + 0x28, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6d, 0x70, 0x6f, + 0x6e, 0x65, 0x6e, 0x74, 0x57, 0x69, 0x6c, 0x6c, 0x55, 0x6e, 0x6d, 0x6f, + 0x75, 0x6e, 0x74, 0x28, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x72, 0x65, 0x6e, + 0x64, 0x65, 0x72, 0x4c, 0x61, 0x79, 0x65, 0x72, 0x28, 0x66, 0x61, 0x6c, + 0x73, 0x65, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x69, 0x73, 0x4d, 0x6f, 0x75, 0x6e, + 0x74, 0x65, 0x64, 0x20, 0x3d, 0x20, 0x66, 0x61, 0x6c, 0x73, 0x65, 0x3b, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, + 0x28, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, + 0x20, 0x26, 0x26, 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x72, 0x65, 0x6d, + 0x6f, 0x74, 0x65, 0x2e, 0x70, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x4e, 0x6f, + 0x64, 0x65, 0x29, 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x72, 0x65, 0x6d, + 0x6f, 0x74, 0x65, 0x2e, 0x70, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x4e, 0x6f, + 0x64, 0x65, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x68, 0x69, + 0x6c, 0x64, 0x28, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x72, 0x65, 0x6d, 0x6f, + 0x74, 0x65, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, + 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x66, 0x69, 0x6e, 0x64, + 0x4e, 0x6f, 0x64, 0x65, 0x28, 0x6e, 0x6f, 0x64, 0x65, 0x29, 0x20, 0x7b, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, 0x74, + 0x75, 0x72, 0x6e, 0x20, 0x74, 0x79, 0x70, 0x65, 0x6f, 0x66, 0x20, 0x6e, + 0x6f, 0x64, 0x65, 0x20, 0x3d, 0x3d, 0x3d, 0x20, 0x27, 0x73, 0x74, 0x72, + 0x69, 0x6e, 0x67, 0x27, 0x20, 0x3f, 0x20, 0x64, 0x6f, 0x63, 0x75, 0x6d, + 0x65, 0x6e, 0x74, 0x2e, 0x71, 0x75, 0x65, 0x72, 0x79, 0x53, 0x65, 0x6c, + 0x65, 0x63, 0x74, 0x6f, 0x72, 0x28, 0x6e, 0x6f, 0x64, 0x65, 0x29, 0x20, + 0x3a, 0x20, 0x6e, 0x6f, 0x64, 0x65, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x7d, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, + 0x65, 0x6e, 0x64, 0x65, 0x72, 0x4c, 0x61, 0x79, 0x65, 0x72, 0x28, 0x73, + 0x68, 0x6f, 0x77, 0x20, 0x3d, 0x20, 0x74, 0x72, 0x75, 0x65, 0x29, 0x20, + 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, + 0x20, 0x28, 0x21, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x69, 0x73, 0x4d, 0x6f, + 0x75, 0x6e, 0x74, 0x65, 0x64, 0x29, 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, + 0x6e, 0x3b, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x2f, 0x2f, 0x20, 0x63, 0x6c, 0x65, 0x61, 0x6e, 0x20, 0x75, 0x70, 0x20, + 0x6f, 0x6c, 0x64, 0x20, 0x6e, 0x6f, 0x64, 0x65, 0x20, 0x69, 0x66, 0x20, + 0x6d, 0x6f, 0x76, 0x69, 0x6e, 0x67, 0x20, 0x62, 0x61, 0x73, 0x65, 0x73, + 0x3a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, + 0x20, 0x28, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x70, 0x73, + 0x2e, 0x69, 0x6e, 0x74, 0x6f, 0x20, 0x21, 0x3d, 0x3d, 0x20, 0x74, 0x68, + 0x69, 0x73, 0x2e, 0x69, 0x6e, 0x74, 0x6f, 0x50, 0x6f, 0x69, 0x6e, 0x74, + 0x65, 0x72, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x69, 0x6e, 0x74, + 0x6f, 0x50, 0x6f, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x20, 0x3d, 0x20, 0x74, + 0x68, 0x69, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x70, 0x73, 0x2e, 0x69, 0x6e, + 0x74, 0x6f, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x69, + 0x6e, 0x74, 0x6f, 0x20, 0x26, 0x26, 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, + 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x74, 0x68, + 0x69, 0x73, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x20, 0x3d, 0x20, + 0x72, 0x65, 0x6e, 0x64, 0x65, 0x72, 0x28, 0x68, 0x74, 0x6d, 0x6c, 0x60, + 0x3c, 0x24, 0x7b, 0x50, 0x6f, 0x72, 0x74, 0x61, 0x6c, 0x50, 0x72, 0x6f, + 0x78, 0x79, 0x7d, 0x20, 0x2f, 0x3e, 0x60, 0x2c, 0x20, 0x74, 0x68, 0x69, + 0x73, 0x2e, 0x69, 0x6e, 0x74, 0x6f, 0x2c, 0x20, 0x74, 0x68, 0x69, 0x73, + 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x29, 0x3b, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x74, 0x68, 0x69, 0x73, + 0x2e, 0x69, 0x6e, 0x74, 0x6f, 0x20, 0x3d, 0x20, 0x74, 0x68, 0x69, 0x73, + 0x2e, 0x66, 0x69, 0x6e, 0x64, 0x4e, 0x6f, 0x64, 0x65, 0x28, 0x74, 0x68, + 0x69, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x70, 0x73, 0x2e, 0x69, 0x6e, 0x74, + 0x6f, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x7d, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x74, + 0x68, 0x69, 0x73, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x20, 0x3d, + 0x20, 0x72, 0x65, 0x6e, 0x64, 0x65, 0x72, 0x28, 0x68, 0x74, 0x6d, 0x6c, + 0x60, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x3c, 0x24, 0x7b, 0x50, 0x6f, 0x72, 0x74, 0x61, 0x6c, 0x50, 0x72, 0x6f, + 0x78, 0x79, 0x7d, 0x20, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x3d, + 0x24, 0x7b, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x65, + 0x78, 0x74, 0x7d, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x24, 0x7b, 0x73, 0x68, 0x6f, 0x77, 0x20, + 0x26, 0x26, 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x70, + 0x73, 0x2e, 0x63, 0x68, 0x69, 0x6c, 0x64, 0x72, 0x65, 0x6e, 0x20, 0x7c, + 0x7c, 0x20, 0x6e, 0x75, 0x6c, 0x6c, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x2f, 0x24, 0x7b, 0x50, 0x6f, + 0x72, 0x74, 0x61, 0x6c, 0x50, 0x72, 0x6f, 0x78, 0x79, 0x7d, 0x3e, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x60, 0x2c, 0x20, 0x74, + 0x68, 0x69, 0x73, 0x2e, 0x69, 0x6e, 0x74, 0x6f, 0x2c, 0x20, 0x74, 0x68, + 0x69, 0x73, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x29, 0x3b, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x72, 0x65, 0x6e, 0x64, 0x65, 0x72, 0x28, 0x29, 0x20, + 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, + 0x74, 0x75, 0x72, 0x6e, 0x20, 0x6e, 0x75, 0x6c, 0x6c, 0x3b, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x7d, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x2f, 0x2f, 0x20, 0x68, 0x69, 0x67, 0x68, + 0x2d, 0x6f, 0x72, 0x64, 0x65, 0x72, 0x20, 0x63, 0x6f, 0x6d, 0x70, 0x6f, + 0x6e, 0x65, 0x6e, 0x74, 0x20, 0x74, 0x68, 0x61, 0x74, 0x20, 0x72, 0x65, + 0x6e, 0x64, 0x65, 0x72, 0x73, 0x20, 0x69, 0x74, 0x73, 0x20, 0x66, 0x69, + 0x72, 0x73, 0x74, 0x20, 0x63, 0x68, 0x69, 0x6c, 0x64, 0x20, 0x69, 0x66, + 0x20, 0x69, 0x74, 0x20, 0x65, 0x78, 0x69, 0x73, 0x74, 0x73, 0x2e, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x2f, 0x2f, 0x20, 0x75, 0x73, 0x65, 0x64, 0x20, + 0x61, 0x73, 0x20, 0x61, 0x20, 0x63, 0x6f, 0x6e, 0x64, 0x69, 0x74, 0x69, + 0x6f, 0x6e, 0x61, 0x6c, 0x20, 0x72, 0x65, 0x6e, 0x64, 0x65, 0x72, 0x69, + 0x6e, 0x67, 0x20, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x2e, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x63, 0x6c, 0x61, 0x73, 0x73, 0x20, 0x50, 0x6f, 0x72, 0x74, + 0x61, 0x6c, 0x50, 0x72, 0x6f, 0x78, 0x79, 0x20, 0x65, 0x78, 0x74, 0x65, + 0x6e, 0x64, 0x73, 0x20, 0x43, 0x6f, 0x6d, 0x70, 0x6f, 0x6e, 0x65, 0x6e, + 0x74, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x67, 0x65, + 0x74, 0x43, 0x68, 0x69, 0x6c, 0x64, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, + 0x74, 0x28, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x74, 0x68, 0x69, + 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x70, 0x73, 0x2e, 0x63, 0x6f, 0x6e, 0x74, + 0x65, 0x78, 0x74, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, 0x6e, 0x64, 0x65, + 0x72, 0x28, 0x7b, 0x20, 0x63, 0x68, 0x69, 0x6c, 0x64, 0x72, 0x65, 0x6e, + 0x20, 0x7d, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x63, 0x68, 0x69, + 0x6c, 0x64, 0x72, 0x65, 0x6e, 0x20, 0x7c, 0x7c, 0x20, 0x6e, 0x75, 0x6c, + 0x6c, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x66, 0x75, + 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x41, 0x70, 0x70, 0x28, 0x70, + 0x72, 0x6f, 0x70, 0x73, 0x29, 0x20, 0x7b, 0x0a, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x68, 0x74, + 0x6d, 0x6c, 0x60, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x3c, 0x64, 0x69, 0x76, 0x20, 0x63, 0x6c, 0x61, 0x73, 0x73, 0x3d, 0x22, + 0x6d, 0x6f, 0x64, 0x65, 0x2d, 0x24, 0x7b, 0x73, 0x65, 0x73, 0x73, 0x69, + 0x6f, 0x6e, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x2e, 0x74, 0x79, 0x70, + 0x65, 0x7d, 0x22, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x3c, 0x68, 0x65, 0x61, 0x64, 0x65, 0x72, 0x3e, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x3c, 0x68, 0x31, 0x3e, 0x6c, 0x6c, 0x61, 0x6d, 0x61, 0x2e, 0x63, 0x70, + 0x70, 0x3c, 0x2f, 0x68, 0x31, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x2f, 0x68, 0x65, 0x61, 0x64, 0x65, + 0x72, 0x3e, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x3c, 0x6d, 0x61, 0x69, 0x6e, 0x20, 0x69, 0x64, 0x3d, 0x22, + 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x22, 0x3e, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x24, + 0x7b, 0x63, 0x68, 0x61, 0x74, 0x53, 0x74, 0x61, 0x72, 0x74, 0x65, 0x64, + 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x20, 0x3f, 0x20, 0x43, 0x68, 0x61, + 0x74, 0x4c, 0x6f, 0x67, 0x20, 0x3a, 0x20, 0x43, 0x6f, 0x6e, 0x66, 0x69, + 0x67, 0x46, 0x6f, 0x72, 0x6d, 0x7d, 0x20, 0x2f, 0x3e, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x2f, 0x6d, 0x61, + 0x69, 0x6e, 0x3e, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x3c, 0x73, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, + 0x69, 0x64, 0x3d, 0x22, 0x77, 0x72, 0x69, 0x74, 0x65, 0x22, 0x3e, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x3c, 0x24, 0x7b, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x2e, 0x76, + 0x61, 0x6c, 0x75, 0x65, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x20, 0x3d, 0x3d, + 0x3d, 0x20, 0x27, 0x63, 0x68, 0x61, 0x74, 0x27, 0x20, 0x3f, 0x20, 0x4d, + 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x20, + 0x3a, 0x20, 0x43, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x69, 0x6f, 0x6e, + 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x73, 0x7d, 0x20, 0x2f, 0x3e, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, + 0x2f, 0x73, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x3e, 0x0a, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x66, 0x6f, + 0x6f, 0x74, 0x65, 0x72, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x70, 0x3e, 0x3c, 0x24, 0x7b, + 0x4d, 0x6f, 0x64, 0x65, 0x6c, 0x47, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, + 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, 0x6f, 0x7d, 0x20, 0x2f, 0x3e, 0x3c, + 0x2f, 0x70, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x3c, 0x70, 0x3e, 0x50, 0x6f, 0x77, 0x65, 0x72, + 0x65, 0x64, 0x20, 0x62, 0x79, 0x20, 0x3c, 0x61, 0x20, 0x68, 0x72, 0x65, + 0x66, 0x3d, 0x22, 0x68, 0x74, 0x74, 0x70, 0x73, 0x3a, 0x2f, 0x2f, 0x67, + 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x67, 0x67, + 0x65, 0x72, 0x67, 0x61, 0x6e, 0x6f, 0x76, 0x2f, 0x6c, 0x6c, 0x61, 0x6d, + 0x61, 0x2e, 0x63, 0x70, 0x70, 0x22, 0x3e, 0x6c, 0x6c, 0x61, 0x6d, 0x61, + 0x2e, 0x63, 0x70, 0x70, 0x3c, 0x2f, 0x61, 0x3e, 0x20, 0x61, 0x6e, 0x64, + 0x20, 0x3c, 0x61, 0x20, 0x68, 0x72, 0x65, 0x66, 0x3d, 0x22, 0x68, 0x74, + 0x74, 0x70, 0x73, 0x3a, 0x2f, 0x2f, 0x67, 0x67, 0x6d, 0x6c, 0x2e, 0x61, + 0x69, 0x22, 0x3e, 0x67, 0x67, 0x6d, 0x6c, 0x2e, 0x61, 0x69, 0x3c, 0x2f, + 0x61, 0x3e, 0x2e, 0x3c, 0x2f, 0x70, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x2f, 0x66, 0x6f, 0x6f, 0x74, + 0x65, 0x72, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x3c, 0x2f, 0x64, 0x69, 0x76, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x60, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x72, 0x65, 0x6e, 0x64, 0x65, 0x72, 0x28, 0x68, 0x28, + 0x41, 0x70, 0x70, 0x29, 0x2c, 0x20, 0x64, 0x6f, 0x63, 0x75, 0x6d, 0x65, + 0x6e, 0x74, 0x2e, 0x71, 0x75, 0x65, 0x72, 0x79, 0x53, 0x65, 0x6c, 0x65, + 0x63, 0x74, 0x6f, 0x72, 0x28, 0x27, 0x23, 0x63, 0x6f, 0x6e, 0x74, 0x61, + 0x69, 0x6e, 0x65, 0x72, 0x27, 0x29, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x3c, + 0x2f, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x3e, 0x0a, 0x3c, 0x2f, 0x68, + 0x65, 0x61, 0x64, 0x3e, 0x0a, 0x0a, 0x3c, 0x62, 0x6f, 0x64, 0x79, 0x3e, + 0x0a, 0x20, 0x20, 0x3c, 0x64, 0x69, 0x76, 0x20, 0x69, 0x64, 0x3d, 0x22, + 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x22, 0x3e, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x3c, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x20, 0x74, + 0x79, 0x70, 0x65, 0x3d, 0x22, 0x66, 0x69, 0x6c, 0x65, 0x22, 0x20, 0x69, + 0x64, 0x3d, 0x22, 0x66, 0x69, 0x6c, 0x65, 0x49, 0x6e, 0x70, 0x75, 0x74, + 0x22, 0x20, 0x61, 0x63, 0x63, 0x65, 0x70, 0x74, 0x3d, 0x22, 0x69, 0x6d, + 0x61, 0x67, 0x65, 0x2f, 0x2a, 0x22, 0x20, 0x73, 0x74, 0x79, 0x6c, 0x65, + 0x3d, 0x22, 0x64, 0x69, 0x73, 0x70, 0x6c, 0x61, 0x79, 0x3a, 0x20, 0x6e, + 0x6f, 0x6e, 0x65, 0x3b, 0x22, 0x3e, 0x0a, 0x20, 0x20, 0x3c, 0x2f, 0x64, + 0x69, 0x76, 0x3e, 0x0a, 0x20, 0x20, 0x3c, 0x64, 0x69, 0x76, 0x20, 0x69, + 0x64, 0x3d, 0x22, 0x70, 0x6f, 0x72, 0x74, 0x61, 0x6c, 0x22, 0x3e, 0x3c, + 0x2f, 0x64, 0x69, 0x76, 0x3e, 0x0a, 0x3c, 0x2f, 0x62, 0x6f, 0x64, 0x79, + 0x3e, 0x0a, 0x0a, 0x3c, 0x2f, 0x68, 0x74, 0x6d, 0x6c, 0x3e, 0x0a, 0x0a +}; +unsigned int index_html_len = 33456; diff --git a/examples/server/index.js.hpp b/examples/server/index.js.hpp index 647abe116..e09b3c8c5 100644 --- a/examples/server/index.js.hpp +++ b/examples/server/index.js.hpp @@ -1,4 +1,1903 @@ -const char index_js[] = R"LITERAL( -function t(){throw new Error("Cycle detected")}const n=Symbol.for("preact-signals");function e(){if(f>1){f--;return}let t,n=!1;while(void 0!==o){let _=o;o=void 0;s++;while(void 0!==_){const i=_.o;_.o=void 0;_.f&=-3;if(!(8&_.f)&&p(_))try{_.c()}catch(e){if(!n){t=e;n=!0}}_=i}}s=0;f--;if(n)throw t}function _(t){if(f>0)return t();f++;try{return t()}finally{e()}}let i,o,r=0;function u(t){if(r>0)return t();const n=i;i=void 0;r++;try{return t()}finally{r--;i=n}}let f=0,s=0,l=0;function c(t){if(void 0===i)return;let n=t.n;if(void 0===n||n.t!==i){n={i:0,S:t,p:i.s,n:void 0,t:i,e:void 0,x:void 0,r:n};if(void 0!==i.s)i.s.n=n;i.s=n;t.n=n;if(32&i.f)t.S(n);return n}else if(-1===n.i){n.i=0;if(void 0!==n.n){n.n.p=n.p;if(void 0!==n.p)n.p.n=n.n;n.p=i.s;n.n=void 0;i.s.n=n;i.s=n}return n}}function h(t){this.v=t;this.i=0;this.n=void 0;this.t=void 0}h.prototype.brand=n;h.prototype.h=function(){return!0};h.prototype.S=function(t){if(this.t!==t&&void 0===t.e){t.x=this.t;if(void 0!==this.t)this.t.e=t;this.t=t}};h.prototype.U=function(t){if(void 0!==this.t){const n=t.e,e=t.x;if(void 0!==n){n.x=e;t.e=void 0}if(void 0!==e){e.e=n;t.x=void 0}if(t===this.t)this.t=e}};h.prototype.subscribe=function(t){const n=this;return w((function(){const e=n.value,_=32&this.f;this.f&=-33;try{t(e)}finally{this.f|=_}}))};h.prototype.valueOf=function(){return this.value};h.prototype.toString=function(){return this.value+""};h.prototype.toJSON=function(){return this.value};h.prototype.peek=function(){return this.v};Object.defineProperty(h.prototype,"value",{get(){const t=c(this);if(void 0!==t)t.i=this.i;return this.v},set(n){if(i instanceof y)!function(){throw new Error("Computed cannot have side-effects")}();if(n!==this.v){if(s>100)t();this.v=n;this.i++;l++;f++;try{for(let t=this.t;void 0!==t;t=t.x)t.t.N()}finally{e()}}}});function a(t){return new h(t)}function p(t){for(let n=t.s;void 0!==n;n=n.n)if(n.S.i!==n.i||!n.S.h()||n.S.i!==n.i)return!0;return!1}function d(t){for(let n=t.s;void 0!==n;n=n.n){const e=n.S.n;if(void 0!==e)n.r=e;n.S.n=n;n.i=-1;if(void 0===n.n){t.s=n;break}}}function v(t){let n,e=t.s;while(void 0!==e){const t=e.p;if(-1===e.i){e.S.U(e);if(void 0!==t)t.n=e.n;if(void 0!==e.n)e.n.p=t}else n=e;e.S.n=e.r;if(void 0!==e.r)e.r=void 0;e=t}t.s=n}function y(t){h.call(this,void 0);this.x=t;this.s=void 0;this.g=l-1;this.f=4}(y.prototype=new h).h=function(){this.f&=-3;if(1&this.f)return!1;if(32==(36&this.f))return!0;this.f&=-5;if(this.g===l)return!0;this.g=l;this.f|=1;if(this.i>0&&!p(this)){this.f&=-2;return!0}const t=i;try{d(this);i=this;const t=this.x();if(16&this.f||this.v!==t||0===this.i){this.v=t;this.f&=-17;this.i++}}catch(t){this.v=t;this.f|=16;this.i++}i=t;v(this);this.f&=-2;return!0};y.prototype.S=function(t){if(void 0===this.t){this.f|=36;for(let t=this.s;void 0!==t;t=t.n)t.S.S(t)}h.prototype.S.call(this,t)};y.prototype.U=function(t){if(void 0!==this.t){h.prototype.U.call(this,t);if(void 0===this.t){this.f&=-33;for(let t=this.s;void 0!==t;t=t.n)t.S.U(t)}}};y.prototype.N=function(){if(!(2&this.f)){this.f|=6;for(let t=this.t;void 0!==t;t=t.x)t.t.N()}};y.prototype.peek=function(){if(!this.h())t();if(16&this.f)throw this.v;return this.v};Object.defineProperty(y.prototype,"value",{get(){if(1&this.f)t();const n=c(this);this.h();if(void 0!==n)n.i=this.i;if(16&this.f)throw this.v;return this.v}});function m(t){return new y(t)}function g(t){const n=t.u;t.u=void 0;if("function"==typeof n){f++;const _=i;i=void 0;try{n()}catch(n){t.f&=-2;t.f|=8;b(t);throw n}finally{i=_;e()}}}function b(t){for(let n=t.s;void 0!==n;n=n.n)n.S.U(n);t.x=void 0;t.s=void 0;g(t)}function k(t){if(i!==this)throw new Error("Out-of-order effect");v(this);i=t;this.f&=-2;if(8&this.f)b(this);e()}function S(t){this.x=t;this.u=void 0;this.s=void 0;this.o=void 0;this.f=32}S.prototype.c=function(){const t=this.S();try{if(8&this.f)return;if(void 0===this.x)return;const n=this.x();if("function"==typeof n)this.u=n}finally{t()}};S.prototype.S=function(){if(1&this.f)t();this.f|=1;this.f&=-9;g(this);d(this);f++;const n=i;i=this;return k.bind(this,n)};S.prototype.N=function(){if(!(2&this.f)){this.f|=2;this.o=o;o=this}};S.prototype.d=function(){this.f|=8;if(!(1&this.f))b(this)};function w(t){const n=new S(t);try{n.c()}catch(t){n.d();throw t}return n.d.bind(n)}var x,C,E,U,H,P,N,$,D,T={},V=[],A=/acit|ex(?:s|g|n|p|$)|rph|grid|ows|mnc|ntw|ine[ch]|zoo|^ord|itera/i,F=Array.isArray;function M(t,n){for(var e in n)t[e]=n[e];return t}function W(t){var n=t.parentNode;n&&n.removeChild(t)}function L(t,n,e){var _,i,o,r={};for(o in n)"key"==o?_=n[o]:"ref"==o?i=n[o]:r[o]=n[o];if(arguments.length>2&&(r.children=arguments.length>3?x.call(arguments,2):e),"function"==typeof t&&null!=t.defaultProps)for(o in t.defaultProps)void 0===r[o]&&(r[o]=t.defaultProps[o]);return O(t,r,_,i,null)}function O(t,n,e,_,i){var o={type:t,props:n,key:e,ref:_,__k:null,__:null,__b:0,__e:null,__d:void 0,__c:null,constructor:void 0,__v:null==i?++E:i,__i:-1,__u:0};return null==i&&null!=C.vnode&&C.vnode(o),o}function R(){return{current:null}}function j(t){return t.children}function I(t,n){this.props=t,this.context=n}function q(t,n){if(null==n)return t.__?q(t.__,t.__i+1):null;for(var e;nn&&H.sort($));z.__r=0}function J(t,n,e,_,i,o,r,u,f,s,l){var c,h,a,p,d,v=_&&_.__k||V,y=n.length;for(e.__d=f,K(e,n,v),f=e.__d,c=0;c0?O(i.type,i.props,i.key,i.ref?i.ref:null,i.__v):i)?(i.__=t,i.__b=t.__b+1,u=Y(i,e,r=_+c,l),i.__i=u,o=null,-1!==u&&(l--,(o=e[u])&&(o.__u|=131072)),null==o||null===o.__v?(-1==u&&c--,"function"!=typeof i.type&&(i.__u|=65536)):u!==r&&(u===r+1?c++:u>r?l>f-r?c+=u-r:c--:c=u(null!=f&&0==(131072&f.__u)?1:0))for(;r>=0||u=0){if((f=n[r])&&0==(131072&f.__u)&&i==f.key&&o===f.type)return r;r--}if(u2&&(u.children=arguments.length>3?x.call(arguments,2):e),O(t.type,u,_||t.key,i||t.ref,null)}function ht(t,n){var e={__c:n="__cC"+D++,__:t,Consumer:function(t,n){return t.children(n)},Provider:function(t){var e,_;return this.getChildContext||(e=[],(_={})[n]=this,this.getChildContext=function(){return _},this.shouldComponentUpdate=function(t){this.props.value!==t.value&&e.some((function(t){t.__e=!0,G(t)}))},this.sub=function(t){e.push(t);var n=t.componentWillUnmount;t.componentWillUnmount=function(){e.splice(e.indexOf(t),1),n&&n.call(t)}}),t.children}};return e.Provider.__=e.Consumer.contextType=e}x=V.slice,C={__e:function(t,n,e,_){for(var i,o,r;n=n.__;)if((i=n.__c)&&!i.__)try{if((o=i.constructor)&&null!=o.getDerivedStateFromError&&(i.setState(o.getDerivedStateFromError(t)),r=i.__d),null!=i.componentDidCatch&&(i.componentDidCatch(t,_||{}),r=i.__d),r)return i.__E=i}catch(n){t=n}throw t}},E=0,U=function(t){return null!=t&&null==t.constructor},I.prototype.setState=function(t,n){var e;e=null!=this.__s&&this.__s!==this.state?this.__s:this.__s=M({},this.state),"function"==typeof t&&(t=t(M({},e),this.props)),t&&M(e,t),null!=t&&this.__v&&(n&&this._sb.push(n),G(this))},I.prototype.forceUpdate=function(t){this.__v&&(this.__e=!0,t&&this.__h.push(t),G(this))},I.prototype.render=j,H=[],N="function"==typeof Promise?Promise.prototype.then.bind(Promise.resolve()):setTimeout,$=function(t,n){return t.__v.__b-n.__v.__b},z.__r=0,D=0;var at,pt,dt,vt,yt=0,mt=[],gt=[],bt=C.__b,kt=C.__r,St=C.diffed,wt=C.__c,xt=C.unmount;function Ct(t,n){C.__h&&C.__h(pt,t,yt||n),yt=0;var e=pt.__H||(pt.__H={__:[],__h:[]});return t>=e.__.length&&e.__.push({__V:gt}),e.__[t]}function Et(t){return yt=1,Ut(qt,t)}function Ut(t,n,e){var _=Ct(at++,2);if(_.t=t,!_.__c&&(_.__=[e?e(n):qt(void 0,n),function(t){var n=_.__N?_.__N[0]:_.__[0],e=_.t(n,t);n!==e&&(_.__N=[e,_.__[1]],_.__c.setState({}))}],_.__c=pt,!pt.u)){var i=function(t,n,e){if(!_.__c.__H)return!0;var i=_.__c.__H.__.filter((function(t){return t.__c}));if(i.every((function(t){return!t.__N})))return!o||o.call(this,t,n,e);var r=!1;return i.forEach((function(t){if(t.__N){var n=t.__[0];t.__=t.__N,t.__N=void 0,n!==t.__[0]&&(r=!0)}})),!(!r&&_.__c.props===t)&&(!o||o.call(this,t,n,e))};pt.u=!0;var o=pt.shouldComponentUpdate,r=pt.componentWillUpdate;pt.componentWillUpdate=function(t,n,e){if(this.__e){var _=o;o=void 0,i(t,n,e),o=_}r&&r.call(this,t,n,e)},pt.shouldComponentUpdate=i}return _.__N||_.__}function Ht(t,n){var e=Ct(at++,3);!C.__s&&It(e.__H,n)&&(e.__=t,e.i=n,pt.__H.__h.push(e))}function Pt(t,n){var e=Ct(at++,4);!C.__s&&It(e.__H,n)&&(e.__=t,e.i=n,pt.__h.push(e))}function Nt(t){return yt=5,Dt((function(){return{current:t}}),[])}function $t(t,n,e){yt=6,Pt((function(){return"function"==typeof t?(t(n()),function(){return t(null)}):t?(t.current=n(),function(){return t.current=null}):void 0}),null==e?e:e.concat(t))}function Dt(t,n){var e=Ct(at++,7);return It(e.__H,n)?(e.__V=t(),e.i=n,e.__h=t,e.__V):e.__}function Tt(t,n){return yt=8,Dt((function(){return t}),n)}function Vt(t){var n=pt.context[t.__c],e=Ct(at++,9);return e.c=t,n?(null==e.__&&(e.__=!0,n.sub(pt)),n.props.value):t.__}function At(t,n){C.useDebugValue&&C.useDebugValue(n?n(t):t)}function Ft(t){var n=Ct(at++,10),e=Et();return n.__=t,pt.componentDidCatch||(pt.componentDidCatch=function(t,_){n.__&&n.__(t,_),e[1](t)}),[e[0],function(){e[1](void 0)}]}function Mt(){var t=Ct(at++,11);if(!t.__){for(var n=pt.__v;null!==n&&!n.__m&&null!==n.__;)n=n.__;var e=n.__m||(n.__m=[0,0]);t.__="P"+e[0]+"-"+e[1]++}return t.__}function Wt(){for(var t;t=mt.shift();)if(t.__P&&t.__H)try{t.__H.__h.forEach(Rt),t.__H.__h.forEach(jt),t.__H.__h=[]}catch(u){t.__H.__h=[],C.__e(u,t.__v)}}C.__b=function(t){pt=null,bt&&bt(t)},C.__r=function(t){kt&&kt(t),at=0;var n=(pt=t.__c).__H;n&&(dt===pt?(n.__h=[],pt.__h=[],n.__.forEach((function(t){t.__N&&(t.__=t.__N),t.__V=gt,t.__N=t.i=void 0}))):(n.__h.forEach(Rt),n.__h.forEach(jt),n.__h=[],at=0)),dt=pt},C.diffed=function(t){St&&St(t);var n=t.__c;n&&n.__H&&(n.__H.__h.length&&(1!==mt.push(n)&&vt===C.requestAnimationFrame||((vt=C.requestAnimationFrame)||Ot)(Wt)),n.__H.__.forEach((function(t){t.i&&(t.__H=t.i),t.__V!==gt&&(t.__=t.__V),t.i=void 0,t.__V=gt}))),dt=pt=null},C.__c=function(t,n){n.some((function(t){try{t.__h.forEach(Rt),t.__h=t.__h.filter((function(t){return!t.__||jt(t)}))}catch(l){n.some((function(t){t.__h&&(t.__h=[])})),n=[],C.__e(l,t.__v)}})),wt&&wt(t,n)},C.unmount=function(t){xt&&xt(t);var n,e=t.__c;e&&e.__H&&(e.__H.__.forEach((function(t){try{Rt(t)}catch(t){n=t}})),e.__H=void 0,n&&C.__e(n,e.__v))};var Lt="function"==typeof requestAnimationFrame;function Ot(t){var n,e=function(){clearTimeout(_),Lt&&cancelAnimationFrame(n),setTimeout(t)},_=setTimeout(e,100);Lt&&(n=requestAnimationFrame(e))}function Rt(t){var n=pt,e=t.__c;"function"==typeof e&&(t.__c=void 0,e()),pt=n}function jt(t){var n=pt;t.__c=t.__(),pt=n}function It(t,n){return!t||t.length!==n.length||n.some((function(n,e){return n!==t[e]}))}function qt(t,n){return"function"==typeof n?n(t):n}function Bt(t,n){C[t]=n.bind(null,C[t]||(()=>{}))}let Gt,zt;function Jt(t){if(zt)zt();zt=t&&t.S()}function Kt({data:t}){const n=Xt(t);n.value=t;const e=Dt(()=>{let t=this.__v;while(t=t.__)if(t.__c){t.__c.__$f|=4;break}this.__$u.c=()=>{var t;if(!U(e.peek())&&3===(null==(t=this.base)?void 0:t.nodeType))this.base.data=e.peek();else{this.__$f|=1;this.setState({})}};return m(()=>{let t=n.value.value;return 0===t?0:!0===t?"":t||""})},[]);return e.value}Kt.displayName="_st";Object.defineProperties(h.prototype,{constructor:{configurable:!0,value:void 0},type:{configurable:!0,value:Kt},props:{configurable:!0,get(){return{data:this}}},__b:{configurable:!0,value:1}});Bt("__b",(t,n)=>{if("string"==typeof n.type){let t,e=n.props;for(let _ in e){if("children"===_)continue;let i=e[_];if(i instanceof h){if(!t)n.__np=t={};t[_]=i;e[_]=i.peek()}}}t(n)});Bt("__r",(t,n)=>{Jt();let e,_=n.__c;if(_){_.__$f&=-2;e=_.__$u;if(void 0===e)_.__$u=e=function(t){let n;w((function(){n=this}));n.c=()=>{_.__$f|=1;_.setState({})};return n}()}Gt=_;Jt(e);t(n)});Bt("__e",(t,n,e,_)=>{Jt();Gt=void 0;t(n,e,_)});Bt("diffed",(t,n)=>{Jt();Gt=void 0;let e;if("string"==typeof n.type&&(e=n.__e)){let t=n.__np,_=n.props;if(t){let n=e.U;if(n)for(let e in n){let _=n[e];if(void 0!==_&&!(e in t)){_.d();n[e]=void 0}}else{n={};e.U=n}for(let i in t){let o=n[i],r=t[i];if(void 0===o){o=Qt(e,i,r,_);n[i]=o}else o.o(r,_)}}}t(n)});function Qt(t,n,e,_){const i=n in t&&void 0===t.ownerSVGElement,o=a(e);return{o:(t,n)=>{o.value=t;_=n},d:w(()=>{const e=o.value.value;if(_[n]!==e){_[n]=e;if(i)t[n]=e;else if(e)t.setAttribute(n,e);else t.removeAttribute(n)}})}}Bt("unmount",(t,n)=>{if("string"==typeof n.type){let t=n.__e;if(t){const n=t.U;if(n){t.U=void 0;for(let t in n){let e=n[t];if(e)e.d()}}}}else{let t=n.__c;if(t){const n=t.__$u;if(n){t.__$u=void 0;n.d()}}}t(n)});Bt("__h",(t,n,e,_)=>{if(_<3||9===_)n.__$f|=2;t(n,e,_)});I.prototype.shouldComponentUpdate=function(t,n){const e=this.__$u;if(!(e&&void 0!==e.s||4&this.__$f))return!0;if(3&this.__$f)return!0;for(let _ in n)return!0;for(let _ in t)if("__source"!==_&&t[_]!==this.props[_])return!0;for(let _ in this.props)if(!(_ in t))return!0;return!1};function Xt(t){return Dt(()=>a(t),[])}function Yt(t){const n=Nt(t);n.current=t;Gt.__$f|=4;return Dt(()=>m(()=>n.current()),[])}function Zt(t){const n=Nt(t);n.current=t;Ht(()=>w(()=>n.current()),[])}var tn=function(t,n,e,_){var i;n[0]=0;for(var o=1;o=5&&((i||!t&&5===_)&&(r.push(_,0,i,e),_=6),t&&(r.push(_,t,0,e),_=6)),i=""},f=0;f"===n?(_=1,i=""):i=n+i[0]:o?n===o?o="":i+=n:'"'===n||"'"===n?o=n:">"===n?(u(),_=1):_&&("="===n?(_=5,e=i,i=""):"/"===n&&(_<5||">"===t[f][s+1])?(u(),3===_&&(r=r[0]),_=r,(r=r[0]).push(2,0,_),_=0):" "===n||"\t"===n||"\n"===n||"\r"===n?(u(),_=2):i+=n),3===_&&"!--"===i&&(_=4,r=r[0])}return u(),r}(t)),n),arguments,[])).length>1?n:n[0]}var _n=en.bind(L);export{I as Component,j as Fragment,h as Signal,_ as batch,ct as cloneElement,m as computed,ht as createContext,L as createElement,R as createRef,w as effect,L as h,_n as html,lt as hydrate,U as isValidElement,C as options,st as render,a as signal,X as toChildArray,u as untracked,Tt as useCallback,Yt as useComputed,Vt as useContext,At as useDebugValue,Ht as useEffect,Ft as useErrorBoundary,Mt as useId,$t as useImperativeHandle,Pt as useLayoutEffect,Dt as useMemo,Ut as useReducer,Nt as useRef,Xt as useSignal,Zt as useSignalEffect,Et as useState}; -)LITERAL"; -unsigned int index_js_len = sizeof(index_js); +unsigned char index_js[] = { + 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x74, 0x28, 0x29, + 0x7b, 0x74, 0x68, 0x72, 0x6f, 0x77, 0x20, 0x6e, 0x65, 0x77, 0x20, 0x45, + 0x72, 0x72, 0x6f, 0x72, 0x28, 0x22, 0x43, 0x79, 0x63, 0x6c, 0x65, 0x20, + 0x64, 0x65, 0x74, 0x65, 0x63, 0x74, 0x65, 0x64, 0x22, 0x29, 0x7d, 0x63, + 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x6e, 0x3d, 0x53, 0x79, 0x6d, 0x62, 0x6f, + 0x6c, 0x2e, 0x66, 0x6f, 0x72, 0x28, 0x22, 0x70, 0x72, 0x65, 0x61, 0x63, + 0x74, 0x2d, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x73, 0x22, 0x29, 0x3b, + 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x65, 0x28, 0x29, + 0x7b, 0x69, 0x66, 0x28, 0x66, 0x3e, 0x31, 0x29, 0x7b, 0x66, 0x2d, 0x2d, + 0x3b, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x7d, 0x6c, 0x65, 0x74, 0x20, + 0x74, 0x2c, 0x6e, 0x3d, 0x21, 0x31, 0x3b, 0x77, 0x68, 0x69, 0x6c, 0x65, + 0x28, 0x76, 0x6f, 0x69, 0x64, 0x20, 0x30, 0x21, 0x3d, 0x3d, 0x6f, 0x29, + 0x7b, 0x6c, 0x65, 0x74, 0x20, 0x5f, 0x3d, 0x6f, 0x3b, 0x6f, 0x3d, 0x76, + 0x6f, 0x69, 0x64, 0x20, 0x30, 0x3b, 0x73, 0x2b, 0x2b, 0x3b, 0x77, 0x68, + 0x69, 0x6c, 0x65, 0x28, 0x76, 0x6f, 0x69, 0x64, 0x20, 0x30, 0x21, 0x3d, + 0x3d, 0x5f, 0x29, 0x7b, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x69, 0x3d, + 0x5f, 0x2e, 0x6f, 0x3b, 0x5f, 0x2e, 0x6f, 0x3d, 0x76, 0x6f, 0x69, 0x64, + 0x20, 0x30, 0x3b, 0x5f, 0x2e, 0x66, 0x26, 0x3d, 0x2d, 0x33, 0x3b, 0x69, + 0x66, 0x28, 0x21, 0x28, 0x38, 0x26, 0x5f, 0x2e, 0x66, 0x29, 0x26, 0x26, + 0x70, 0x28, 0x5f, 0x29, 0x29, 0x74, 0x72, 0x79, 0x7b, 0x5f, 0x2e, 0x63, + 0x28, 0x29, 0x7d, 0x63, 0x61, 0x74, 0x63, 0x68, 0x28, 0x65, 0x29, 0x7b, + 0x69, 0x66, 0x28, 0x21, 0x6e, 0x29, 0x7b, 0x74, 0x3d, 0x65, 0x3b, 0x6e, + 0x3d, 0x21, 0x30, 0x7d, 0x7d, 0x5f, 0x3d, 0x69, 0x7d, 0x7d, 0x73, 0x3d, + 0x30, 0x3b, 0x66, 0x2d, 0x2d, 0x3b, 0x69, 0x66, 0x28, 0x6e, 0x29, 0x74, + 0x68, 0x72, 0x6f, 0x77, 0x20, 0x74, 0x7d, 0x66, 0x75, 0x6e, 0x63, 0x74, + 0x69, 0x6f, 0x6e, 0x20, 0x5f, 0x28, 0x74, 0x29, 0x7b, 0x69, 0x66, 0x28, + 0x66, 0x3e, 0x30, 0x29, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x74, + 0x28, 0x29, 0x3b, 0x66, 0x2b, 0x2b, 0x3b, 0x74, 0x72, 0x79, 0x7b, 0x72, + 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x74, 0x28, 0x29, 0x7d, 0x66, 0x69, + 0x6e, 0x61, 0x6c, 0x6c, 0x79, 0x7b, 0x65, 0x28, 0x29, 0x7d, 0x7d, 0x6c, + 0x65, 0x74, 0x20, 0x69, 0x2c, 0x6f, 0x2c, 0x72, 0x3d, 0x30, 0x3b, 0x66, + 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x75, 0x28, 0x74, 0x29, + 0x7b, 0x69, 0x66, 0x28, 0x72, 0x3e, 0x30, 0x29, 0x72, 0x65, 0x74, 0x75, + 0x72, 0x6e, 0x20, 0x74, 0x28, 0x29, 0x3b, 0x63, 0x6f, 0x6e, 0x73, 0x74, + 0x20, 0x6e, 0x3d, 0x69, 0x3b, 0x69, 0x3d, 0x76, 0x6f, 0x69, 0x64, 0x20, + 0x30, 0x3b, 0x72, 0x2b, 0x2b, 0x3b, 0x74, 0x72, 0x79, 0x7b, 0x72, 0x65, + 0x74, 0x75, 0x72, 0x6e, 0x20, 0x74, 0x28, 0x29, 0x7d, 0x66, 0x69, 0x6e, + 0x61, 0x6c, 0x6c, 0x79, 0x7b, 0x72, 0x2d, 0x2d, 0x3b, 0x69, 0x3d, 0x6e, + 0x7d, 0x7d, 0x6c, 0x65, 0x74, 0x20, 0x66, 0x3d, 0x30, 0x2c, 0x73, 0x3d, + 0x30, 0x2c, 0x6c, 0x3d, 0x30, 0x3b, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, + 0x6f, 0x6e, 0x20, 0x63, 0x28, 0x74, 0x29, 0x7b, 0x69, 0x66, 0x28, 0x76, + 0x6f, 0x69, 0x64, 0x20, 0x30, 0x3d, 0x3d, 0x3d, 0x69, 0x29, 0x72, 0x65, + 0x74, 0x75, 0x72, 0x6e, 0x3b, 0x6c, 0x65, 0x74, 0x20, 0x6e, 0x3d, 0x74, + 0x2e, 0x6e, 0x3b, 0x69, 0x66, 0x28, 0x76, 0x6f, 0x69, 0x64, 0x20, 0x30, + 0x3d, 0x3d, 0x3d, 0x6e, 0x7c, 0x7c, 0x6e, 0x2e, 0x74, 0x21, 0x3d, 0x3d, + 0x69, 0x29, 0x7b, 0x6e, 0x3d, 0x7b, 0x69, 0x3a, 0x30, 0x2c, 0x53, 0x3a, + 0x74, 0x2c, 0x70, 0x3a, 0x69, 0x2e, 0x73, 0x2c, 0x6e, 0x3a, 0x76, 0x6f, + 0x69, 0x64, 0x20, 0x30, 0x2c, 0x74, 0x3a, 0x69, 0x2c, 0x65, 0x3a, 0x76, + 0x6f, 0x69, 0x64, 0x20, 0x30, 0x2c, 0x78, 0x3a, 0x76, 0x6f, 0x69, 0x64, + 0x20, 0x30, 0x2c, 0x72, 0x3a, 0x6e, 0x7d, 0x3b, 0x69, 0x66, 0x28, 0x76, + 0x6f, 0x69, 0x64, 0x20, 0x30, 0x21, 0x3d, 0x3d, 0x69, 0x2e, 0x73, 0x29, + 0x69, 0x2e, 0x73, 0x2e, 0x6e, 0x3d, 0x6e, 0x3b, 0x69, 0x2e, 0x73, 0x3d, + 0x6e, 0x3b, 0x74, 0x2e, 0x6e, 0x3d, 0x6e, 0x3b, 0x69, 0x66, 0x28, 0x33, + 0x32, 0x26, 0x69, 0x2e, 0x66, 0x29, 0x74, 0x2e, 0x53, 0x28, 0x6e, 0x29, + 0x3b, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x6e, 0x7d, 0x65, 0x6c, + 0x73, 0x65, 0x20, 0x69, 0x66, 0x28, 0x2d, 0x31, 0x3d, 0x3d, 0x3d, 0x6e, + 0x2e, 0x69, 0x29, 0x7b, 0x6e, 0x2e, 0x69, 0x3d, 0x30, 0x3b, 0x69, 0x66, + 0x28, 0x76, 0x6f, 0x69, 0x64, 0x20, 0x30, 0x21, 0x3d, 0x3d, 0x6e, 0x2e, + 0x6e, 0x29, 0x7b, 0x6e, 0x2e, 0x6e, 0x2e, 0x70, 0x3d, 0x6e, 0x2e, 0x70, + 0x3b, 0x69, 0x66, 0x28, 0x76, 0x6f, 0x69, 0x64, 0x20, 0x30, 0x21, 0x3d, + 0x3d, 0x6e, 0x2e, 0x70, 0x29, 0x6e, 0x2e, 0x70, 0x2e, 0x6e, 0x3d, 0x6e, + 0x2e, 0x6e, 0x3b, 0x6e, 0x2e, 0x70, 0x3d, 0x69, 0x2e, 0x73, 0x3b, 0x6e, + 0x2e, 0x6e, 0x3d, 0x76, 0x6f, 0x69, 0x64, 0x20, 0x30, 0x3b, 0x69, 0x2e, + 0x73, 0x2e, 0x6e, 0x3d, 0x6e, 0x3b, 0x69, 0x2e, 0x73, 0x3d, 0x6e, 0x7d, + 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x6e, 0x7d, 0x7d, 0x66, 0x75, + 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x68, 0x28, 0x74, 0x29, 0x7b, + 0x74, 0x68, 0x69, 0x73, 0x2e, 0x76, 0x3d, 0x74, 0x3b, 0x74, 0x68, 0x69, + 0x73, 0x2e, 0x69, 0x3d, 0x30, 0x3b, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x6e, + 0x3d, 0x76, 0x6f, 0x69, 0x64, 0x20, 0x30, 0x3b, 0x74, 0x68, 0x69, 0x73, + 0x2e, 0x74, 0x3d, 0x76, 0x6f, 0x69, 0x64, 0x20, 0x30, 0x7d, 0x68, 0x2e, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x74, 0x79, 0x70, 0x65, 0x2e, 0x62, 0x72, + 0x61, 0x6e, 0x64, 0x3d, 0x6e, 0x3b, 0x68, 0x2e, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x74, 0x79, 0x70, 0x65, 0x2e, 0x68, 0x3d, 0x66, 0x75, 0x6e, 0x63, + 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x29, 0x7b, 0x72, 0x65, 0x74, 0x75, 0x72, + 0x6e, 0x21, 0x30, 0x7d, 0x3b, 0x68, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x74, 0x79, 0x70, 0x65, 0x2e, 0x53, 0x3d, 0x66, 0x75, 0x6e, 0x63, 0x74, + 0x69, 0x6f, 0x6e, 0x28, 0x74, 0x29, 0x7b, 0x69, 0x66, 0x28, 0x74, 0x68, + 0x69, 0x73, 0x2e, 0x74, 0x21, 0x3d, 0x3d, 0x74, 0x26, 0x26, 0x76, 0x6f, + 0x69, 0x64, 0x20, 0x30, 0x3d, 0x3d, 0x3d, 0x74, 0x2e, 0x65, 0x29, 0x7b, + 0x74, 0x2e, 0x78, 0x3d, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x74, 0x3b, 0x69, + 0x66, 0x28, 0x76, 0x6f, 0x69, 0x64, 0x20, 0x30, 0x21, 0x3d, 0x3d, 0x74, + 0x68, 0x69, 0x73, 0x2e, 0x74, 0x29, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x74, + 0x2e, 0x65, 0x3d, 0x74, 0x3b, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x74, 0x3d, + 0x74, 0x7d, 0x7d, 0x3b, 0x68, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x74, + 0x79, 0x70, 0x65, 0x2e, 0x55, 0x3d, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, + 0x6f, 0x6e, 0x28, 0x74, 0x29, 0x7b, 0x69, 0x66, 0x28, 0x76, 0x6f, 0x69, + 0x64, 0x20, 0x30, 0x21, 0x3d, 0x3d, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x74, + 0x29, 0x7b, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x6e, 0x3d, 0x74, 0x2e, + 0x65, 0x2c, 0x65, 0x3d, 0x74, 0x2e, 0x78, 0x3b, 0x69, 0x66, 0x28, 0x76, + 0x6f, 0x69, 0x64, 0x20, 0x30, 0x21, 0x3d, 0x3d, 0x6e, 0x29, 0x7b, 0x6e, + 0x2e, 0x78, 0x3d, 0x65, 0x3b, 0x74, 0x2e, 0x65, 0x3d, 0x76, 0x6f, 0x69, + 0x64, 0x20, 0x30, 0x7d, 0x69, 0x66, 0x28, 0x76, 0x6f, 0x69, 0x64, 0x20, + 0x30, 0x21, 0x3d, 0x3d, 0x65, 0x29, 0x7b, 0x65, 0x2e, 0x65, 0x3d, 0x6e, + 0x3b, 0x74, 0x2e, 0x78, 0x3d, 0x76, 0x6f, 0x69, 0x64, 0x20, 0x30, 0x7d, + 0x69, 0x66, 0x28, 0x74, 0x3d, 0x3d, 0x3d, 0x74, 0x68, 0x69, 0x73, 0x2e, + 0x74, 0x29, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x74, 0x3d, 0x65, 0x7d, 0x7d, + 0x3b, 0x68, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x74, 0x79, 0x70, 0x65, + 0x2e, 0x73, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x62, 0x65, 0x3d, 0x66, + 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x74, 0x29, 0x7b, 0x63, + 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x6e, 0x3d, 0x74, 0x68, 0x69, 0x73, 0x3b, + 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x77, 0x28, 0x28, 0x66, 0x75, + 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x29, 0x7b, 0x63, 0x6f, 0x6e, + 0x73, 0x74, 0x20, 0x65, 0x3d, 0x6e, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, + 0x2c, 0x5f, 0x3d, 0x33, 0x32, 0x26, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x66, + 0x3b, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x66, 0x26, 0x3d, 0x2d, 0x33, 0x33, + 0x3b, 0x74, 0x72, 0x79, 0x7b, 0x74, 0x28, 0x65, 0x29, 0x7d, 0x66, 0x69, + 0x6e, 0x61, 0x6c, 0x6c, 0x79, 0x7b, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x66, + 0x7c, 0x3d, 0x5f, 0x7d, 0x7d, 0x29, 0x29, 0x7d, 0x3b, 0x68, 0x2e, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x74, 0x79, 0x70, 0x65, 0x2e, 0x76, 0x61, 0x6c, + 0x75, 0x65, 0x4f, 0x66, 0x3d, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, + 0x6e, 0x28, 0x29, 0x7b, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x74, + 0x68, 0x69, 0x73, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x7d, 0x3b, 0x68, + 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x74, 0x79, 0x70, 0x65, 0x2e, 0x74, + 0x6f, 0x53, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x3d, 0x66, 0x75, 0x6e, 0x63, + 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x29, 0x7b, 0x72, 0x65, 0x74, 0x75, 0x72, + 0x6e, 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, + 0x2b, 0x22, 0x22, 0x7d, 0x3b, 0x68, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x74, 0x79, 0x70, 0x65, 0x2e, 0x74, 0x6f, 0x4a, 0x53, 0x4f, 0x4e, 0x3d, + 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x29, 0x7b, 0x72, + 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x76, + 0x61, 0x6c, 0x75, 0x65, 0x7d, 0x3b, 0x68, 0x2e, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x74, 0x79, 0x70, 0x65, 0x2e, 0x70, 0x65, 0x65, 0x6b, 0x3d, 0x66, + 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x29, 0x7b, 0x72, 0x65, + 0x74, 0x75, 0x72, 0x6e, 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x76, 0x7d, + 0x3b, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x2e, 0x64, 0x65, 0x66, 0x69, + 0x6e, 0x65, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x79, 0x28, 0x68, + 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x74, 0x79, 0x70, 0x65, 0x2c, 0x22, + 0x76, 0x61, 0x6c, 0x75, 0x65, 0x22, 0x2c, 0x7b, 0x67, 0x65, 0x74, 0x28, + 0x29, 0x7b, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x74, 0x3d, 0x63, 0x28, + 0x74, 0x68, 0x69, 0x73, 0x29, 0x3b, 0x69, 0x66, 0x28, 0x76, 0x6f, 0x69, + 0x64, 0x20, 0x30, 0x21, 0x3d, 0x3d, 0x74, 0x29, 0x74, 0x2e, 0x69, 0x3d, + 0x74, 0x68, 0x69, 0x73, 0x2e, 0x69, 0x3b, 0x72, 0x65, 0x74, 0x75, 0x72, + 0x6e, 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x76, 0x7d, 0x2c, 0x73, 0x65, + 0x74, 0x28, 0x6e, 0x29, 0x7b, 0x69, 0x66, 0x28, 0x69, 0x20, 0x69, 0x6e, + 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x6f, 0x66, 0x20, 0x79, 0x29, 0x21, + 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x29, 0x7b, 0x74, + 0x68, 0x72, 0x6f, 0x77, 0x20, 0x6e, 0x65, 0x77, 0x20, 0x45, 0x72, 0x72, + 0x6f, 0x72, 0x28, 0x22, 0x43, 0x6f, 0x6d, 0x70, 0x75, 0x74, 0x65, 0x64, + 0x20, 0x63, 0x61, 0x6e, 0x6e, 0x6f, 0x74, 0x20, 0x68, 0x61, 0x76, 0x65, + 0x20, 0x73, 0x69, 0x64, 0x65, 0x2d, 0x65, 0x66, 0x66, 0x65, 0x63, 0x74, + 0x73, 0x22, 0x29, 0x7d, 0x28, 0x29, 0x3b, 0x69, 0x66, 0x28, 0x6e, 0x21, + 0x3d, 0x3d, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x76, 0x29, 0x7b, 0x69, 0x66, + 0x28, 0x73, 0x3e, 0x31, 0x30, 0x30, 0x29, 0x74, 0x28, 0x29, 0x3b, 0x74, + 0x68, 0x69, 0x73, 0x2e, 0x76, 0x3d, 0x6e, 0x3b, 0x74, 0x68, 0x69, 0x73, + 0x2e, 0x69, 0x2b, 0x2b, 0x3b, 0x6c, 0x2b, 0x2b, 0x3b, 0x66, 0x2b, 0x2b, + 0x3b, 0x74, 0x72, 0x79, 0x7b, 0x66, 0x6f, 0x72, 0x28, 0x6c, 0x65, 0x74, + 0x20, 0x74, 0x3d, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x74, 0x3b, 0x76, 0x6f, + 0x69, 0x64, 0x20, 0x30, 0x21, 0x3d, 0x3d, 0x74, 0x3b, 0x74, 0x3d, 0x74, + 0x2e, 0x78, 0x29, 0x74, 0x2e, 0x74, 0x2e, 0x4e, 0x28, 0x29, 0x7d, 0x66, + 0x69, 0x6e, 0x61, 0x6c, 0x6c, 0x79, 0x7b, 0x65, 0x28, 0x29, 0x7d, 0x7d, + 0x7d, 0x7d, 0x29, 0x3b, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, + 0x20, 0x61, 0x28, 0x74, 0x29, 0x7b, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, + 0x20, 0x6e, 0x65, 0x77, 0x20, 0x68, 0x28, 0x74, 0x29, 0x7d, 0x66, 0x75, + 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x70, 0x28, 0x74, 0x29, 0x7b, + 0x66, 0x6f, 0x72, 0x28, 0x6c, 0x65, 0x74, 0x20, 0x6e, 0x3d, 0x74, 0x2e, + 0x73, 0x3b, 0x76, 0x6f, 0x69, 0x64, 0x20, 0x30, 0x21, 0x3d, 0x3d, 0x6e, + 0x3b, 0x6e, 0x3d, 0x6e, 0x2e, 0x6e, 0x29, 0x69, 0x66, 0x28, 0x6e, 0x2e, + 0x53, 0x2e, 0x69, 0x21, 0x3d, 0x3d, 0x6e, 0x2e, 0x69, 0x7c, 0x7c, 0x21, + 0x6e, 0x2e, 0x53, 0x2e, 0x68, 0x28, 0x29, 0x7c, 0x7c, 0x6e, 0x2e, 0x53, + 0x2e, 0x69, 0x21, 0x3d, 0x3d, 0x6e, 0x2e, 0x69, 0x29, 0x72, 0x65, 0x74, + 0x75, 0x72, 0x6e, 0x21, 0x30, 0x3b, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, + 0x21, 0x31, 0x7d, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, + 0x64, 0x28, 0x74, 0x29, 0x7b, 0x66, 0x6f, 0x72, 0x28, 0x6c, 0x65, 0x74, + 0x20, 0x6e, 0x3d, 0x74, 0x2e, 0x73, 0x3b, 0x76, 0x6f, 0x69, 0x64, 0x20, + 0x30, 0x21, 0x3d, 0x3d, 0x6e, 0x3b, 0x6e, 0x3d, 0x6e, 0x2e, 0x6e, 0x29, + 0x7b, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x65, 0x3d, 0x6e, 0x2e, 0x53, + 0x2e, 0x6e, 0x3b, 0x69, 0x66, 0x28, 0x76, 0x6f, 0x69, 0x64, 0x20, 0x30, + 0x21, 0x3d, 0x3d, 0x65, 0x29, 0x6e, 0x2e, 0x72, 0x3d, 0x65, 0x3b, 0x6e, + 0x2e, 0x53, 0x2e, 0x6e, 0x3d, 0x6e, 0x3b, 0x6e, 0x2e, 0x69, 0x3d, 0x2d, + 0x31, 0x3b, 0x69, 0x66, 0x28, 0x76, 0x6f, 0x69, 0x64, 0x20, 0x30, 0x3d, + 0x3d, 0x3d, 0x6e, 0x2e, 0x6e, 0x29, 0x7b, 0x74, 0x2e, 0x73, 0x3d, 0x6e, + 0x3b, 0x62, 0x72, 0x65, 0x61, 0x6b, 0x7d, 0x7d, 0x7d, 0x66, 0x75, 0x6e, + 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x76, 0x28, 0x74, 0x29, 0x7b, 0x6c, + 0x65, 0x74, 0x20, 0x6e, 0x2c, 0x65, 0x3d, 0x74, 0x2e, 0x73, 0x3b, 0x77, + 0x68, 0x69, 0x6c, 0x65, 0x28, 0x76, 0x6f, 0x69, 0x64, 0x20, 0x30, 0x21, + 0x3d, 0x3d, 0x65, 0x29, 0x7b, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x74, + 0x3d, 0x65, 0x2e, 0x70, 0x3b, 0x69, 0x66, 0x28, 0x2d, 0x31, 0x3d, 0x3d, + 0x3d, 0x65, 0x2e, 0x69, 0x29, 0x7b, 0x65, 0x2e, 0x53, 0x2e, 0x55, 0x28, + 0x65, 0x29, 0x3b, 0x69, 0x66, 0x28, 0x76, 0x6f, 0x69, 0x64, 0x20, 0x30, + 0x21, 0x3d, 0x3d, 0x74, 0x29, 0x74, 0x2e, 0x6e, 0x3d, 0x65, 0x2e, 0x6e, + 0x3b, 0x69, 0x66, 0x28, 0x76, 0x6f, 0x69, 0x64, 0x20, 0x30, 0x21, 0x3d, + 0x3d, 0x65, 0x2e, 0x6e, 0x29, 0x65, 0x2e, 0x6e, 0x2e, 0x70, 0x3d, 0x74, + 0x7d, 0x65, 0x6c, 0x73, 0x65, 0x20, 0x6e, 0x3d, 0x65, 0x3b, 0x65, 0x2e, + 0x53, 0x2e, 0x6e, 0x3d, 0x65, 0x2e, 0x72, 0x3b, 0x69, 0x66, 0x28, 0x76, + 0x6f, 0x69, 0x64, 0x20, 0x30, 0x21, 0x3d, 0x3d, 0x65, 0x2e, 0x72, 0x29, + 0x65, 0x2e, 0x72, 0x3d, 0x76, 0x6f, 0x69, 0x64, 0x20, 0x30, 0x3b, 0x65, + 0x3d, 0x74, 0x7d, 0x74, 0x2e, 0x73, 0x3d, 0x6e, 0x7d, 0x66, 0x75, 0x6e, + 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x79, 0x28, 0x74, 0x29, 0x7b, 0x68, + 0x2e, 0x63, 0x61, 0x6c, 0x6c, 0x28, 0x74, 0x68, 0x69, 0x73, 0x2c, 0x76, + 0x6f, 0x69, 0x64, 0x20, 0x30, 0x29, 0x3b, 0x74, 0x68, 0x69, 0x73, 0x2e, + 0x78, 0x3d, 0x74, 0x3b, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x73, 0x3d, 0x76, + 0x6f, 0x69, 0x64, 0x20, 0x30, 0x3b, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x67, + 0x3d, 0x6c, 0x2d, 0x31, 0x3b, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x66, 0x3d, + 0x34, 0x7d, 0x28, 0x79, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x74, 0x79, + 0x70, 0x65, 0x3d, 0x6e, 0x65, 0x77, 0x20, 0x68, 0x29, 0x2e, 0x68, 0x3d, + 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x29, 0x7b, 0x74, + 0x68, 0x69, 0x73, 0x2e, 0x66, 0x26, 0x3d, 0x2d, 0x33, 0x3b, 0x69, 0x66, + 0x28, 0x31, 0x26, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x66, 0x29, 0x72, 0x65, + 0x74, 0x75, 0x72, 0x6e, 0x21, 0x31, 0x3b, 0x69, 0x66, 0x28, 0x33, 0x32, + 0x3d, 0x3d, 0x28, 0x33, 0x36, 0x26, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x66, + 0x29, 0x29, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x21, 0x30, 0x3b, 0x74, + 0x68, 0x69, 0x73, 0x2e, 0x66, 0x26, 0x3d, 0x2d, 0x35, 0x3b, 0x69, 0x66, + 0x28, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x67, 0x3d, 0x3d, 0x3d, 0x6c, 0x29, + 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x21, 0x30, 0x3b, 0x74, 0x68, 0x69, + 0x73, 0x2e, 0x67, 0x3d, 0x6c, 0x3b, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x66, + 0x7c, 0x3d, 0x31, 0x3b, 0x69, 0x66, 0x28, 0x74, 0x68, 0x69, 0x73, 0x2e, + 0x69, 0x3e, 0x30, 0x26, 0x26, 0x21, 0x70, 0x28, 0x74, 0x68, 0x69, 0x73, + 0x29, 0x29, 0x7b, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x66, 0x26, 0x3d, 0x2d, + 0x32, 0x3b, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x21, 0x30, 0x7d, 0x63, + 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x74, 0x3d, 0x69, 0x3b, 0x74, 0x72, 0x79, + 0x7b, 0x64, 0x28, 0x74, 0x68, 0x69, 0x73, 0x29, 0x3b, 0x69, 0x3d, 0x74, + 0x68, 0x69, 0x73, 0x3b, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x74, 0x3d, + 0x74, 0x68, 0x69, 0x73, 0x2e, 0x78, 0x28, 0x29, 0x3b, 0x69, 0x66, 0x28, + 0x31, 0x36, 0x26, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x66, 0x7c, 0x7c, 0x74, + 0x68, 0x69, 0x73, 0x2e, 0x76, 0x21, 0x3d, 0x3d, 0x74, 0x7c, 0x7c, 0x30, + 0x3d, 0x3d, 0x3d, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x69, 0x29, 0x7b, 0x74, + 0x68, 0x69, 0x73, 0x2e, 0x76, 0x3d, 0x74, 0x3b, 0x74, 0x68, 0x69, 0x73, + 0x2e, 0x66, 0x26, 0x3d, 0x2d, 0x31, 0x37, 0x3b, 0x74, 0x68, 0x69, 0x73, + 0x2e, 0x69, 0x2b, 0x2b, 0x7d, 0x7d, 0x63, 0x61, 0x74, 0x63, 0x68, 0x28, + 0x74, 0x29, 0x7b, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x76, 0x3d, 0x74, 0x3b, + 0x74, 0x68, 0x69, 0x73, 0x2e, 0x66, 0x7c, 0x3d, 0x31, 0x36, 0x3b, 0x74, + 0x68, 0x69, 0x73, 0x2e, 0x69, 0x2b, 0x2b, 0x7d, 0x69, 0x3d, 0x74, 0x3b, + 0x76, 0x28, 0x74, 0x68, 0x69, 0x73, 0x29, 0x3b, 0x74, 0x68, 0x69, 0x73, + 0x2e, 0x66, 0x26, 0x3d, 0x2d, 0x32, 0x3b, 0x72, 0x65, 0x74, 0x75, 0x72, + 0x6e, 0x21, 0x30, 0x7d, 0x3b, 0x79, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x74, 0x79, 0x70, 0x65, 0x2e, 0x53, 0x3d, 0x66, 0x75, 0x6e, 0x63, 0x74, + 0x69, 0x6f, 0x6e, 0x28, 0x74, 0x29, 0x7b, 0x69, 0x66, 0x28, 0x76, 0x6f, + 0x69, 0x64, 0x20, 0x30, 0x3d, 0x3d, 0x3d, 0x74, 0x68, 0x69, 0x73, 0x2e, + 0x74, 0x29, 0x7b, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x66, 0x7c, 0x3d, 0x33, + 0x36, 0x3b, 0x66, 0x6f, 0x72, 0x28, 0x6c, 0x65, 0x74, 0x20, 0x74, 0x3d, + 0x74, 0x68, 0x69, 0x73, 0x2e, 0x73, 0x3b, 0x76, 0x6f, 0x69, 0x64, 0x20, + 0x30, 0x21, 0x3d, 0x3d, 0x74, 0x3b, 0x74, 0x3d, 0x74, 0x2e, 0x6e, 0x29, + 0x74, 0x2e, 0x53, 0x2e, 0x53, 0x28, 0x74, 0x29, 0x7d, 0x68, 0x2e, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x74, 0x79, 0x70, 0x65, 0x2e, 0x53, 0x2e, 0x63, + 0x61, 0x6c, 0x6c, 0x28, 0x74, 0x68, 0x69, 0x73, 0x2c, 0x74, 0x29, 0x7d, + 0x3b, 0x79, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x74, 0x79, 0x70, 0x65, + 0x2e, 0x55, 0x3d, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x28, + 0x74, 0x29, 0x7b, 0x69, 0x66, 0x28, 0x76, 0x6f, 0x69, 0x64, 0x20, 0x30, + 0x21, 0x3d, 0x3d, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x74, 0x29, 0x7b, 0x68, + 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x74, 0x79, 0x70, 0x65, 0x2e, 0x55, + 0x2e, 0x63, 0x61, 0x6c, 0x6c, 0x28, 0x74, 0x68, 0x69, 0x73, 0x2c, 0x74, + 0x29, 0x3b, 0x69, 0x66, 0x28, 0x76, 0x6f, 0x69, 0x64, 0x20, 0x30, 0x3d, + 0x3d, 0x3d, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x74, 0x29, 0x7b, 0x74, 0x68, + 0x69, 0x73, 0x2e, 0x66, 0x26, 0x3d, 0x2d, 0x33, 0x33, 0x3b, 0x66, 0x6f, + 0x72, 0x28, 0x6c, 0x65, 0x74, 0x20, 0x74, 0x3d, 0x74, 0x68, 0x69, 0x73, + 0x2e, 0x73, 0x3b, 0x76, 0x6f, 0x69, 0x64, 0x20, 0x30, 0x21, 0x3d, 0x3d, + 0x74, 0x3b, 0x74, 0x3d, 0x74, 0x2e, 0x6e, 0x29, 0x74, 0x2e, 0x53, 0x2e, + 0x55, 0x28, 0x74, 0x29, 0x7d, 0x7d, 0x7d, 0x3b, 0x79, 0x2e, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x74, 0x79, 0x70, 0x65, 0x2e, 0x4e, 0x3d, 0x66, 0x75, + 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x29, 0x7b, 0x69, 0x66, 0x28, + 0x21, 0x28, 0x32, 0x26, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x66, 0x29, 0x29, + 0x7b, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x66, 0x7c, 0x3d, 0x36, 0x3b, 0x66, + 0x6f, 0x72, 0x28, 0x6c, 0x65, 0x74, 0x20, 0x74, 0x3d, 0x74, 0x68, 0x69, + 0x73, 0x2e, 0x74, 0x3b, 0x76, 0x6f, 0x69, 0x64, 0x20, 0x30, 0x21, 0x3d, + 0x3d, 0x74, 0x3b, 0x74, 0x3d, 0x74, 0x2e, 0x78, 0x29, 0x74, 0x2e, 0x74, + 0x2e, 0x4e, 0x28, 0x29, 0x7d, 0x7d, 0x3b, 0x79, 0x2e, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x74, 0x79, 0x70, 0x65, 0x2e, 0x70, 0x65, 0x65, 0x6b, 0x3d, + 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x29, 0x7b, 0x69, + 0x66, 0x28, 0x21, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x68, 0x28, 0x29, 0x29, + 0x74, 0x28, 0x29, 0x3b, 0x69, 0x66, 0x28, 0x31, 0x36, 0x26, 0x74, 0x68, + 0x69, 0x73, 0x2e, 0x66, 0x29, 0x74, 0x68, 0x72, 0x6f, 0x77, 0x20, 0x74, + 0x68, 0x69, 0x73, 0x2e, 0x76, 0x3b, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, + 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x76, 0x7d, 0x3b, 0x4f, 0x62, 0x6a, + 0x65, 0x63, 0x74, 0x2e, 0x64, 0x65, 0x66, 0x69, 0x6e, 0x65, 0x50, 0x72, + 0x6f, 0x70, 0x65, 0x72, 0x74, 0x79, 0x28, 0x79, 0x2e, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x74, 0x79, 0x70, 0x65, 0x2c, 0x22, 0x76, 0x61, 0x6c, 0x75, + 0x65, 0x22, 0x2c, 0x7b, 0x67, 0x65, 0x74, 0x28, 0x29, 0x7b, 0x69, 0x66, + 0x28, 0x31, 0x26, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x66, 0x29, 0x74, 0x28, + 0x29, 0x3b, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x6e, 0x3d, 0x63, 0x28, + 0x74, 0x68, 0x69, 0x73, 0x29, 0x3b, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x68, + 0x28, 0x29, 0x3b, 0x69, 0x66, 0x28, 0x76, 0x6f, 0x69, 0x64, 0x20, 0x30, + 0x21, 0x3d, 0x3d, 0x6e, 0x29, 0x6e, 0x2e, 0x69, 0x3d, 0x74, 0x68, 0x69, + 0x73, 0x2e, 0x69, 0x3b, 0x69, 0x66, 0x28, 0x31, 0x36, 0x26, 0x74, 0x68, + 0x69, 0x73, 0x2e, 0x66, 0x29, 0x74, 0x68, 0x72, 0x6f, 0x77, 0x20, 0x74, + 0x68, 0x69, 0x73, 0x2e, 0x76, 0x3b, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, + 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x76, 0x7d, 0x7d, 0x29, 0x3b, 0x66, + 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x6d, 0x28, 0x74, 0x29, + 0x7b, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x6e, 0x65, 0x77, 0x20, + 0x79, 0x28, 0x74, 0x29, 0x7d, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, + 0x6e, 0x20, 0x67, 0x28, 0x74, 0x29, 0x7b, 0x63, 0x6f, 0x6e, 0x73, 0x74, + 0x20, 0x6e, 0x3d, 0x74, 0x2e, 0x75, 0x3b, 0x74, 0x2e, 0x75, 0x3d, 0x76, + 0x6f, 0x69, 0x64, 0x20, 0x30, 0x3b, 0x69, 0x66, 0x28, 0x22, 0x66, 0x75, + 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x3d, 0x3d, 0x74, 0x79, 0x70, + 0x65, 0x6f, 0x66, 0x20, 0x6e, 0x29, 0x7b, 0x66, 0x2b, 0x2b, 0x3b, 0x63, + 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x5f, 0x3d, 0x69, 0x3b, 0x69, 0x3d, 0x76, + 0x6f, 0x69, 0x64, 0x20, 0x30, 0x3b, 0x74, 0x72, 0x79, 0x7b, 0x6e, 0x28, + 0x29, 0x7d, 0x63, 0x61, 0x74, 0x63, 0x68, 0x28, 0x6e, 0x29, 0x7b, 0x74, + 0x2e, 0x66, 0x26, 0x3d, 0x2d, 0x32, 0x3b, 0x74, 0x2e, 0x66, 0x7c, 0x3d, + 0x38, 0x3b, 0x62, 0x28, 0x74, 0x29, 0x3b, 0x74, 0x68, 0x72, 0x6f, 0x77, + 0x20, 0x6e, 0x7d, 0x66, 0x69, 0x6e, 0x61, 0x6c, 0x6c, 0x79, 0x7b, 0x69, + 0x3d, 0x5f, 0x3b, 0x65, 0x28, 0x29, 0x7d, 0x7d, 0x7d, 0x66, 0x75, 0x6e, + 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x62, 0x28, 0x74, 0x29, 0x7b, 0x66, + 0x6f, 0x72, 0x28, 0x6c, 0x65, 0x74, 0x20, 0x6e, 0x3d, 0x74, 0x2e, 0x73, + 0x3b, 0x76, 0x6f, 0x69, 0x64, 0x20, 0x30, 0x21, 0x3d, 0x3d, 0x6e, 0x3b, + 0x6e, 0x3d, 0x6e, 0x2e, 0x6e, 0x29, 0x6e, 0x2e, 0x53, 0x2e, 0x55, 0x28, + 0x6e, 0x29, 0x3b, 0x74, 0x2e, 0x78, 0x3d, 0x76, 0x6f, 0x69, 0x64, 0x20, + 0x30, 0x3b, 0x74, 0x2e, 0x73, 0x3d, 0x76, 0x6f, 0x69, 0x64, 0x20, 0x30, + 0x3b, 0x67, 0x28, 0x74, 0x29, 0x7d, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, + 0x6f, 0x6e, 0x20, 0x6b, 0x28, 0x74, 0x29, 0x7b, 0x69, 0x66, 0x28, 0x69, + 0x21, 0x3d, 0x3d, 0x74, 0x68, 0x69, 0x73, 0x29, 0x74, 0x68, 0x72, 0x6f, + 0x77, 0x20, 0x6e, 0x65, 0x77, 0x20, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x28, + 0x22, 0x4f, 0x75, 0x74, 0x2d, 0x6f, 0x66, 0x2d, 0x6f, 0x72, 0x64, 0x65, + 0x72, 0x20, 0x65, 0x66, 0x66, 0x65, 0x63, 0x74, 0x22, 0x29, 0x3b, 0x76, + 0x28, 0x74, 0x68, 0x69, 0x73, 0x29, 0x3b, 0x69, 0x3d, 0x74, 0x3b, 0x74, + 0x68, 0x69, 0x73, 0x2e, 0x66, 0x26, 0x3d, 0x2d, 0x32, 0x3b, 0x69, 0x66, + 0x28, 0x38, 0x26, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x66, 0x29, 0x62, 0x28, + 0x74, 0x68, 0x69, 0x73, 0x29, 0x3b, 0x65, 0x28, 0x29, 0x7d, 0x66, 0x75, + 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x53, 0x28, 0x74, 0x29, 0x7b, + 0x74, 0x68, 0x69, 0x73, 0x2e, 0x78, 0x3d, 0x74, 0x3b, 0x74, 0x68, 0x69, + 0x73, 0x2e, 0x75, 0x3d, 0x76, 0x6f, 0x69, 0x64, 0x20, 0x30, 0x3b, 0x74, + 0x68, 0x69, 0x73, 0x2e, 0x73, 0x3d, 0x76, 0x6f, 0x69, 0x64, 0x20, 0x30, + 0x3b, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x6f, 0x3d, 0x76, 0x6f, 0x69, 0x64, + 0x20, 0x30, 0x3b, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x66, 0x3d, 0x33, 0x32, + 0x7d, 0x53, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x74, 0x79, 0x70, 0x65, + 0x2e, 0x63, 0x3d, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x28, + 0x29, 0x7b, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x74, 0x3d, 0x74, 0x68, + 0x69, 0x73, 0x2e, 0x53, 0x28, 0x29, 0x3b, 0x74, 0x72, 0x79, 0x7b, 0x69, + 0x66, 0x28, 0x38, 0x26, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x66, 0x29, 0x72, + 0x65, 0x74, 0x75, 0x72, 0x6e, 0x3b, 0x69, 0x66, 0x28, 0x76, 0x6f, 0x69, + 0x64, 0x20, 0x30, 0x3d, 0x3d, 0x3d, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x78, + 0x29, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x3b, 0x63, 0x6f, 0x6e, 0x73, + 0x74, 0x20, 0x6e, 0x3d, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x78, 0x28, 0x29, + 0x3b, 0x69, 0x66, 0x28, 0x22, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, + 0x6e, 0x22, 0x3d, 0x3d, 0x74, 0x79, 0x70, 0x65, 0x6f, 0x66, 0x20, 0x6e, + 0x29, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x75, 0x3d, 0x6e, 0x7d, 0x66, 0x69, + 0x6e, 0x61, 0x6c, 0x6c, 0x79, 0x7b, 0x74, 0x28, 0x29, 0x7d, 0x7d, 0x3b, + 0x53, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x74, 0x79, 0x70, 0x65, 0x2e, + 0x53, 0x3d, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x29, + 0x7b, 0x69, 0x66, 0x28, 0x31, 0x26, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x66, + 0x29, 0x74, 0x28, 0x29, 0x3b, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x66, 0x7c, + 0x3d, 0x31, 0x3b, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x66, 0x26, 0x3d, 0x2d, + 0x39, 0x3b, 0x67, 0x28, 0x74, 0x68, 0x69, 0x73, 0x29, 0x3b, 0x64, 0x28, + 0x74, 0x68, 0x69, 0x73, 0x29, 0x3b, 0x66, 0x2b, 0x2b, 0x3b, 0x63, 0x6f, + 0x6e, 0x73, 0x74, 0x20, 0x6e, 0x3d, 0x69, 0x3b, 0x69, 0x3d, 0x74, 0x68, + 0x69, 0x73, 0x3b, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x6b, 0x2e, + 0x62, 0x69, 0x6e, 0x64, 0x28, 0x74, 0x68, 0x69, 0x73, 0x2c, 0x6e, 0x29, + 0x7d, 0x3b, 0x53, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x74, 0x79, 0x70, + 0x65, 0x2e, 0x4e, 0x3d, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, + 0x28, 0x29, 0x7b, 0x69, 0x66, 0x28, 0x21, 0x28, 0x32, 0x26, 0x74, 0x68, + 0x69, 0x73, 0x2e, 0x66, 0x29, 0x29, 0x7b, 0x74, 0x68, 0x69, 0x73, 0x2e, + 0x66, 0x7c, 0x3d, 0x32, 0x3b, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x6f, 0x3d, + 0x6f, 0x3b, 0x6f, 0x3d, 0x74, 0x68, 0x69, 0x73, 0x7d, 0x7d, 0x3b, 0x53, + 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x74, 0x79, 0x70, 0x65, 0x2e, 0x64, + 0x3d, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x29, 0x7b, + 0x74, 0x68, 0x69, 0x73, 0x2e, 0x66, 0x7c, 0x3d, 0x38, 0x3b, 0x69, 0x66, + 0x28, 0x21, 0x28, 0x31, 0x26, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x66, 0x29, + 0x29, 0x62, 0x28, 0x74, 0x68, 0x69, 0x73, 0x29, 0x7d, 0x3b, 0x66, 0x75, + 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x77, 0x28, 0x74, 0x29, 0x7b, + 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x6e, 0x3d, 0x6e, 0x65, 0x77, 0x20, + 0x53, 0x28, 0x74, 0x29, 0x3b, 0x74, 0x72, 0x79, 0x7b, 0x6e, 0x2e, 0x63, + 0x28, 0x29, 0x7d, 0x63, 0x61, 0x74, 0x63, 0x68, 0x28, 0x74, 0x29, 0x7b, + 0x6e, 0x2e, 0x64, 0x28, 0x29, 0x3b, 0x74, 0x68, 0x72, 0x6f, 0x77, 0x20, + 0x74, 0x7d, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x6e, 0x2e, 0x64, + 0x2e, 0x62, 0x69, 0x6e, 0x64, 0x28, 0x6e, 0x29, 0x7d, 0x76, 0x61, 0x72, + 0x20, 0x78, 0x2c, 0x43, 0x2c, 0x45, 0x2c, 0x55, 0x2c, 0x48, 0x2c, 0x50, + 0x2c, 0x4e, 0x2c, 0x24, 0x2c, 0x44, 0x2c, 0x54, 0x3d, 0x7b, 0x7d, 0x2c, + 0x56, 0x3d, 0x5b, 0x5d, 0x2c, 0x41, 0x3d, 0x2f, 0x61, 0x63, 0x69, 0x74, + 0x7c, 0x65, 0x78, 0x28, 0x3f, 0x3a, 0x73, 0x7c, 0x67, 0x7c, 0x6e, 0x7c, + 0x70, 0x7c, 0x24, 0x29, 0x7c, 0x72, 0x70, 0x68, 0x7c, 0x67, 0x72, 0x69, + 0x64, 0x7c, 0x6f, 0x77, 0x73, 0x7c, 0x6d, 0x6e, 0x63, 0x7c, 0x6e, 0x74, + 0x77, 0x7c, 0x69, 0x6e, 0x65, 0x5b, 0x63, 0x68, 0x5d, 0x7c, 0x7a, 0x6f, + 0x6f, 0x7c, 0x5e, 0x6f, 0x72, 0x64, 0x7c, 0x69, 0x74, 0x65, 0x72, 0x61, + 0x2f, 0x69, 0x2c, 0x46, 0x3d, 0x41, 0x72, 0x72, 0x61, 0x79, 0x2e, 0x69, + 0x73, 0x41, 0x72, 0x72, 0x61, 0x79, 0x3b, 0x66, 0x75, 0x6e, 0x63, 0x74, + 0x69, 0x6f, 0x6e, 0x20, 0x4d, 0x28, 0x74, 0x2c, 0x6e, 0x29, 0x7b, 0x66, + 0x6f, 0x72, 0x28, 0x76, 0x61, 0x72, 0x20, 0x65, 0x20, 0x69, 0x6e, 0x20, + 0x6e, 0x29, 0x74, 0x5b, 0x65, 0x5d, 0x3d, 0x6e, 0x5b, 0x65, 0x5d, 0x3b, + 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x74, 0x7d, 0x66, 0x75, 0x6e, + 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x57, 0x28, 0x74, 0x29, 0x7b, 0x76, + 0x61, 0x72, 0x20, 0x6e, 0x3d, 0x74, 0x2e, 0x70, 0x61, 0x72, 0x65, 0x6e, + 0x74, 0x4e, 0x6f, 0x64, 0x65, 0x3b, 0x6e, 0x26, 0x26, 0x6e, 0x2e, 0x72, + 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x68, 0x69, 0x6c, 0x64, 0x28, 0x74, + 0x29, 0x7d, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x4c, + 0x28, 0x74, 0x2c, 0x6e, 0x2c, 0x65, 0x29, 0x7b, 0x76, 0x61, 0x72, 0x20, + 0x5f, 0x2c, 0x69, 0x2c, 0x6f, 0x2c, 0x72, 0x3d, 0x7b, 0x7d, 0x3b, 0x66, + 0x6f, 0x72, 0x28, 0x6f, 0x20, 0x69, 0x6e, 0x20, 0x6e, 0x29, 0x22, 0x6b, + 0x65, 0x79, 0x22, 0x3d, 0x3d, 0x6f, 0x3f, 0x5f, 0x3d, 0x6e, 0x5b, 0x6f, + 0x5d, 0x3a, 0x22, 0x72, 0x65, 0x66, 0x22, 0x3d, 0x3d, 0x6f, 0x3f, 0x69, + 0x3d, 0x6e, 0x5b, 0x6f, 0x5d, 0x3a, 0x72, 0x5b, 0x6f, 0x5d, 0x3d, 0x6e, + 0x5b, 0x6f, 0x5d, 0x3b, 0x69, 0x66, 0x28, 0x61, 0x72, 0x67, 0x75, 0x6d, + 0x65, 0x6e, 0x74, 0x73, 0x2e, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x3e, + 0x32, 0x26, 0x26, 0x28, 0x72, 0x2e, 0x63, 0x68, 0x69, 0x6c, 0x64, 0x72, + 0x65, 0x6e, 0x3d, 0x61, 0x72, 0x67, 0x75, 0x6d, 0x65, 0x6e, 0x74, 0x73, + 0x2e, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x3e, 0x33, 0x3f, 0x78, 0x2e, + 0x63, 0x61, 0x6c, 0x6c, 0x28, 0x61, 0x72, 0x67, 0x75, 0x6d, 0x65, 0x6e, + 0x74, 0x73, 0x2c, 0x32, 0x29, 0x3a, 0x65, 0x29, 0x2c, 0x22, 0x66, 0x75, + 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x3d, 0x3d, 0x74, 0x79, 0x70, + 0x65, 0x6f, 0x66, 0x20, 0x74, 0x26, 0x26, 0x6e, 0x75, 0x6c, 0x6c, 0x21, + 0x3d, 0x74, 0x2e, 0x64, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x50, 0x72, + 0x6f, 0x70, 0x73, 0x29, 0x66, 0x6f, 0x72, 0x28, 0x6f, 0x20, 0x69, 0x6e, + 0x20, 0x74, 0x2e, 0x64, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x50, 0x72, + 0x6f, 0x70, 0x73, 0x29, 0x76, 0x6f, 0x69, 0x64, 0x20, 0x30, 0x3d, 0x3d, + 0x3d, 0x72, 0x5b, 0x6f, 0x5d, 0x26, 0x26, 0x28, 0x72, 0x5b, 0x6f, 0x5d, + 0x3d, 0x74, 0x2e, 0x64, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x50, 0x72, + 0x6f, 0x70, 0x73, 0x5b, 0x6f, 0x5d, 0x29, 0x3b, 0x72, 0x65, 0x74, 0x75, + 0x72, 0x6e, 0x20, 0x4f, 0x28, 0x74, 0x2c, 0x72, 0x2c, 0x5f, 0x2c, 0x69, + 0x2c, 0x6e, 0x75, 0x6c, 0x6c, 0x29, 0x7d, 0x66, 0x75, 0x6e, 0x63, 0x74, + 0x69, 0x6f, 0x6e, 0x20, 0x4f, 0x28, 0x74, 0x2c, 0x6e, 0x2c, 0x65, 0x2c, + 0x5f, 0x2c, 0x69, 0x29, 0x7b, 0x76, 0x61, 0x72, 0x20, 0x6f, 0x3d, 0x7b, + 0x74, 0x79, 0x70, 0x65, 0x3a, 0x74, 0x2c, 0x70, 0x72, 0x6f, 0x70, 0x73, + 0x3a, 0x6e, 0x2c, 0x6b, 0x65, 0x79, 0x3a, 0x65, 0x2c, 0x72, 0x65, 0x66, + 0x3a, 0x5f, 0x2c, 0x5f, 0x5f, 0x6b, 0x3a, 0x6e, 0x75, 0x6c, 0x6c, 0x2c, + 0x5f, 0x5f, 0x3a, 0x6e, 0x75, 0x6c, 0x6c, 0x2c, 0x5f, 0x5f, 0x62, 0x3a, + 0x30, 0x2c, 0x5f, 0x5f, 0x65, 0x3a, 0x6e, 0x75, 0x6c, 0x6c, 0x2c, 0x5f, + 0x5f, 0x64, 0x3a, 0x76, 0x6f, 0x69, 0x64, 0x20, 0x30, 0x2c, 0x5f, 0x5f, + 0x63, 0x3a, 0x6e, 0x75, 0x6c, 0x6c, 0x2c, 0x63, 0x6f, 0x6e, 0x73, 0x74, + 0x72, 0x75, 0x63, 0x74, 0x6f, 0x72, 0x3a, 0x76, 0x6f, 0x69, 0x64, 0x20, + 0x30, 0x2c, 0x5f, 0x5f, 0x76, 0x3a, 0x6e, 0x75, 0x6c, 0x6c, 0x3d, 0x3d, + 0x69, 0x3f, 0x2b, 0x2b, 0x45, 0x3a, 0x69, 0x2c, 0x5f, 0x5f, 0x69, 0x3a, + 0x2d, 0x31, 0x2c, 0x5f, 0x5f, 0x75, 0x3a, 0x30, 0x7d, 0x3b, 0x72, 0x65, + 0x74, 0x75, 0x72, 0x6e, 0x20, 0x6e, 0x75, 0x6c, 0x6c, 0x3d, 0x3d, 0x69, + 0x26, 0x26, 0x6e, 0x75, 0x6c, 0x6c, 0x21, 0x3d, 0x43, 0x2e, 0x76, 0x6e, + 0x6f, 0x64, 0x65, 0x26, 0x26, 0x43, 0x2e, 0x76, 0x6e, 0x6f, 0x64, 0x65, + 0x28, 0x6f, 0x29, 0x2c, 0x6f, 0x7d, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, + 0x6f, 0x6e, 0x20, 0x52, 0x28, 0x29, 0x7b, 0x72, 0x65, 0x74, 0x75, 0x72, + 0x6e, 0x7b, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x74, 0x3a, 0x6e, 0x75, + 0x6c, 0x6c, 0x7d, 0x7d, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, + 0x20, 0x6a, 0x28, 0x74, 0x29, 0x7b, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, + 0x20, 0x74, 0x2e, 0x63, 0x68, 0x69, 0x6c, 0x64, 0x72, 0x65, 0x6e, 0x7d, + 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x49, 0x28, 0x74, + 0x2c, 0x6e, 0x29, 0x7b, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x70, 0x72, 0x6f, + 0x70, 0x73, 0x3d, 0x74, 0x2c, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x63, 0x6f, + 0x6e, 0x74, 0x65, 0x78, 0x74, 0x3d, 0x6e, 0x7d, 0x66, 0x75, 0x6e, 0x63, + 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x71, 0x28, 0x74, 0x2c, 0x6e, 0x29, 0x7b, + 0x69, 0x66, 0x28, 0x6e, 0x75, 0x6c, 0x6c, 0x3d, 0x3d, 0x6e, 0x29, 0x72, + 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x74, 0x2e, 0x5f, 0x5f, 0x3f, 0x71, + 0x28, 0x74, 0x2e, 0x5f, 0x5f, 0x2c, 0x74, 0x2e, 0x5f, 0x5f, 0x69, 0x2b, + 0x31, 0x29, 0x3a, 0x6e, 0x75, 0x6c, 0x6c, 0x3b, 0x66, 0x6f, 0x72, 0x28, + 0x76, 0x61, 0x72, 0x20, 0x65, 0x3b, 0x6e, 0x3c, 0x74, 0x2e, 0x5f, 0x5f, + 0x6b, 0x2e, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x3b, 0x6e, 0x2b, 0x2b, + 0x29, 0x69, 0x66, 0x28, 0x6e, 0x75, 0x6c, 0x6c, 0x21, 0x3d, 0x28, 0x65, + 0x3d, 0x74, 0x2e, 0x5f, 0x5f, 0x6b, 0x5b, 0x6e, 0x5d, 0x29, 0x26, 0x26, + 0x6e, 0x75, 0x6c, 0x6c, 0x21, 0x3d, 0x65, 0x2e, 0x5f, 0x5f, 0x65, 0x29, + 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x65, 0x2e, 0x5f, 0x5f, 0x65, + 0x3b, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x22, 0x66, 0x75, 0x6e, 0x63, + 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x3d, 0x3d, 0x74, 0x79, 0x70, 0x65, 0x6f, + 0x66, 0x20, 0x74, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x3f, 0x71, 0x28, 0x74, + 0x29, 0x3a, 0x6e, 0x75, 0x6c, 0x6c, 0x7d, 0x66, 0x75, 0x6e, 0x63, 0x74, + 0x69, 0x6f, 0x6e, 0x20, 0x42, 0x28, 0x74, 0x29, 0x7b, 0x76, 0x61, 0x72, + 0x20, 0x6e, 0x2c, 0x65, 0x3b, 0x69, 0x66, 0x28, 0x6e, 0x75, 0x6c, 0x6c, + 0x21, 0x3d, 0x28, 0x74, 0x3d, 0x74, 0x2e, 0x5f, 0x5f, 0x29, 0x26, 0x26, + 0x6e, 0x75, 0x6c, 0x6c, 0x21, 0x3d, 0x74, 0x2e, 0x5f, 0x5f, 0x63, 0x29, + 0x7b, 0x66, 0x6f, 0x72, 0x28, 0x74, 0x2e, 0x5f, 0x5f, 0x65, 0x3d, 0x74, + 0x2e, 0x5f, 0x5f, 0x63, 0x2e, 0x62, 0x61, 0x73, 0x65, 0x3d, 0x6e, 0x75, + 0x6c, 0x6c, 0x2c, 0x6e, 0x3d, 0x30, 0x3b, 0x6e, 0x3c, 0x74, 0x2e, 0x5f, + 0x5f, 0x6b, 0x2e, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x3b, 0x6e, 0x2b, + 0x2b, 0x29, 0x69, 0x66, 0x28, 0x6e, 0x75, 0x6c, 0x6c, 0x21, 0x3d, 0x28, + 0x65, 0x3d, 0x74, 0x2e, 0x5f, 0x5f, 0x6b, 0x5b, 0x6e, 0x5d, 0x29, 0x26, + 0x26, 0x6e, 0x75, 0x6c, 0x6c, 0x21, 0x3d, 0x65, 0x2e, 0x5f, 0x5f, 0x65, + 0x29, 0x7b, 0x74, 0x2e, 0x5f, 0x5f, 0x65, 0x3d, 0x74, 0x2e, 0x5f, 0x5f, + 0x63, 0x2e, 0x62, 0x61, 0x73, 0x65, 0x3d, 0x65, 0x2e, 0x5f, 0x5f, 0x65, + 0x3b, 0x62, 0x72, 0x65, 0x61, 0x6b, 0x7d, 0x72, 0x65, 0x74, 0x75, 0x72, + 0x6e, 0x20, 0x42, 0x28, 0x74, 0x29, 0x7d, 0x7d, 0x66, 0x75, 0x6e, 0x63, + 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x47, 0x28, 0x74, 0x29, 0x7b, 0x28, 0x21, + 0x74, 0x2e, 0x5f, 0x5f, 0x64, 0x26, 0x26, 0x28, 0x74, 0x2e, 0x5f, 0x5f, + 0x64, 0x3d, 0x21, 0x30, 0x29, 0x26, 0x26, 0x48, 0x2e, 0x70, 0x75, 0x73, + 0x68, 0x28, 0x74, 0x29, 0x26, 0x26, 0x21, 0x7a, 0x2e, 0x5f, 0x5f, 0x72, + 0x2b, 0x2b, 0x7c, 0x7c, 0x50, 0x21, 0x3d, 0x3d, 0x43, 0x2e, 0x64, 0x65, + 0x62, 0x6f, 0x75, 0x6e, 0x63, 0x65, 0x52, 0x65, 0x6e, 0x64, 0x65, 0x72, + 0x69, 0x6e, 0x67, 0x29, 0x26, 0x26, 0x28, 0x28, 0x50, 0x3d, 0x43, 0x2e, + 0x64, 0x65, 0x62, 0x6f, 0x75, 0x6e, 0x63, 0x65, 0x52, 0x65, 0x6e, 0x64, + 0x65, 0x72, 0x69, 0x6e, 0x67, 0x29, 0x7c, 0x7c, 0x4e, 0x29, 0x28, 0x7a, + 0x29, 0x7d, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x7a, + 0x28, 0x29, 0x7b, 0x76, 0x61, 0x72, 0x20, 0x74, 0x2c, 0x6e, 0x2c, 0x65, + 0x2c, 0x5f, 0x2c, 0x69, 0x2c, 0x6f, 0x2c, 0x72, 0x2c, 0x75, 0x2c, 0x66, + 0x3b, 0x66, 0x6f, 0x72, 0x28, 0x48, 0x2e, 0x73, 0x6f, 0x72, 0x74, 0x28, + 0x24, 0x29, 0x3b, 0x74, 0x3d, 0x48, 0x2e, 0x73, 0x68, 0x69, 0x66, 0x74, + 0x28, 0x29, 0x3b, 0x29, 0x74, 0x2e, 0x5f, 0x5f, 0x64, 0x26, 0x26, 0x28, + 0x6e, 0x3d, 0x48, 0x2e, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x2c, 0x5f, + 0x3d, 0x76, 0x6f, 0x69, 0x64, 0x20, 0x30, 0x2c, 0x6f, 0x3d, 0x28, 0x69, + 0x3d, 0x28, 0x65, 0x3d, 0x74, 0x29, 0x2e, 0x5f, 0x5f, 0x76, 0x29, 0x2e, + 0x5f, 0x5f, 0x65, 0x2c, 0x75, 0x3d, 0x5b, 0x5d, 0x2c, 0x66, 0x3d, 0x5b, + 0x5d, 0x2c, 0x28, 0x72, 0x3d, 0x65, 0x2e, 0x5f, 0x5f, 0x50, 0x29, 0x26, + 0x26, 0x28, 0x28, 0x5f, 0x3d, 0x4d, 0x28, 0x7b, 0x7d, 0x2c, 0x69, 0x29, + 0x29, 0x2e, 0x5f, 0x5f, 0x76, 0x3d, 0x69, 0x2e, 0x5f, 0x5f, 0x76, 0x2b, + 0x31, 0x2c, 0x43, 0x2e, 0x76, 0x6e, 0x6f, 0x64, 0x65, 0x26, 0x26, 0x43, + 0x2e, 0x76, 0x6e, 0x6f, 0x64, 0x65, 0x28, 0x5f, 0x29, 0x2c, 0x5f, 0x74, + 0x28, 0x72, 0x2c, 0x5f, 0x2c, 0x69, 0x2c, 0x65, 0x2e, 0x5f, 0x5f, 0x6e, + 0x2c, 0x76, 0x6f, 0x69, 0x64, 0x20, 0x30, 0x21, 0x3d, 0x3d, 0x72, 0x2e, + 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x53, 0x56, 0x47, 0x45, 0x6c, 0x65, 0x6d, + 0x65, 0x6e, 0x74, 0x2c, 0x33, 0x32, 0x26, 0x69, 0x2e, 0x5f, 0x5f, 0x75, + 0x3f, 0x5b, 0x6f, 0x5d, 0x3a, 0x6e, 0x75, 0x6c, 0x6c, 0x2c, 0x75, 0x2c, + 0x6e, 0x75, 0x6c, 0x6c, 0x3d, 0x3d, 0x6f, 0x3f, 0x71, 0x28, 0x69, 0x29, + 0x3a, 0x6f, 0x2c, 0x21, 0x21, 0x28, 0x33, 0x32, 0x26, 0x69, 0x2e, 0x5f, + 0x5f, 0x75, 0x29, 0x2c, 0x66, 0x29, 0x2c, 0x5f, 0x2e, 0x5f, 0x5f, 0x2e, + 0x5f, 0x5f, 0x6b, 0x5b, 0x5f, 0x2e, 0x5f, 0x5f, 0x69, 0x5d, 0x3d, 0x5f, + 0x2c, 0x69, 0x74, 0x28, 0x75, 0x2c, 0x5f, 0x2c, 0x66, 0x29, 0x2c, 0x5f, + 0x2e, 0x5f, 0x5f, 0x65, 0x21, 0x3d, 0x6f, 0x26, 0x26, 0x42, 0x28, 0x5f, + 0x29, 0x29, 0x2c, 0x48, 0x2e, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x3e, + 0x6e, 0x26, 0x26, 0x48, 0x2e, 0x73, 0x6f, 0x72, 0x74, 0x28, 0x24, 0x29, + 0x29, 0x3b, 0x7a, 0x2e, 0x5f, 0x5f, 0x72, 0x3d, 0x30, 0x7d, 0x66, 0x75, + 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x4a, 0x28, 0x74, 0x2c, 0x6e, + 0x2c, 0x65, 0x2c, 0x5f, 0x2c, 0x69, 0x2c, 0x6f, 0x2c, 0x72, 0x2c, 0x75, + 0x2c, 0x66, 0x2c, 0x73, 0x2c, 0x6c, 0x29, 0x7b, 0x76, 0x61, 0x72, 0x20, + 0x63, 0x2c, 0x68, 0x2c, 0x61, 0x2c, 0x70, 0x2c, 0x64, 0x2c, 0x76, 0x3d, + 0x5f, 0x26, 0x26, 0x5f, 0x2e, 0x5f, 0x5f, 0x6b, 0x7c, 0x7c, 0x56, 0x2c, + 0x79, 0x3d, 0x6e, 0x2e, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x3b, 0x66, + 0x6f, 0x72, 0x28, 0x65, 0x2e, 0x5f, 0x5f, 0x64, 0x3d, 0x66, 0x2c, 0x4b, + 0x28, 0x65, 0x2c, 0x6e, 0x2c, 0x76, 0x29, 0x2c, 0x66, 0x3d, 0x65, 0x2e, + 0x5f, 0x5f, 0x64, 0x2c, 0x63, 0x3d, 0x30, 0x3b, 0x63, 0x3c, 0x79, 0x3b, + 0x63, 0x2b, 0x2b, 0x29, 0x6e, 0x75, 0x6c, 0x6c, 0x21, 0x3d, 0x28, 0x61, + 0x3d, 0x65, 0x2e, 0x5f, 0x5f, 0x6b, 0x5b, 0x63, 0x5d, 0x29, 0x26, 0x26, + 0x22, 0x62, 0x6f, 0x6f, 0x6c, 0x65, 0x61, 0x6e, 0x22, 0x21, 0x3d, 0x74, + 0x79, 0x70, 0x65, 0x6f, 0x66, 0x20, 0x61, 0x26, 0x26, 0x22, 0x66, 0x75, + 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x21, 0x3d, 0x74, 0x79, 0x70, + 0x65, 0x6f, 0x66, 0x20, 0x61, 0x26, 0x26, 0x28, 0x68, 0x3d, 0x2d, 0x31, + 0x3d, 0x3d, 0x3d, 0x61, 0x2e, 0x5f, 0x5f, 0x69, 0x3f, 0x54, 0x3a, 0x76, + 0x5b, 0x61, 0x2e, 0x5f, 0x5f, 0x69, 0x5d, 0x7c, 0x7c, 0x54, 0x2c, 0x61, + 0x2e, 0x5f, 0x5f, 0x69, 0x3d, 0x63, 0x2c, 0x5f, 0x74, 0x28, 0x74, 0x2c, + 0x61, 0x2c, 0x68, 0x2c, 0x69, 0x2c, 0x6f, 0x2c, 0x72, 0x2c, 0x75, 0x2c, + 0x66, 0x2c, 0x73, 0x2c, 0x6c, 0x29, 0x2c, 0x70, 0x3d, 0x61, 0x2e, 0x5f, + 0x5f, 0x65, 0x2c, 0x61, 0x2e, 0x72, 0x65, 0x66, 0x26, 0x26, 0x68, 0x2e, + 0x72, 0x65, 0x66, 0x21, 0x3d, 0x61, 0x2e, 0x72, 0x65, 0x66, 0x26, 0x26, + 0x28, 0x68, 0x2e, 0x72, 0x65, 0x66, 0x26, 0x26, 0x72, 0x74, 0x28, 0x68, + 0x2e, 0x72, 0x65, 0x66, 0x2c, 0x6e, 0x75, 0x6c, 0x6c, 0x2c, 0x61, 0x29, + 0x2c, 0x6c, 0x2e, 0x70, 0x75, 0x73, 0x68, 0x28, 0x61, 0x2e, 0x72, 0x65, + 0x66, 0x2c, 0x61, 0x2e, 0x5f, 0x5f, 0x63, 0x7c, 0x7c, 0x70, 0x2c, 0x61, + 0x29, 0x29, 0x2c, 0x6e, 0x75, 0x6c, 0x6c, 0x3d, 0x3d, 0x64, 0x26, 0x26, + 0x6e, 0x75, 0x6c, 0x6c, 0x21, 0x3d, 0x70, 0x26, 0x26, 0x28, 0x64, 0x3d, + 0x70, 0x29, 0x2c, 0x36, 0x35, 0x35, 0x33, 0x36, 0x26, 0x61, 0x2e, 0x5f, + 0x5f, 0x75, 0x7c, 0x7c, 0x68, 0x2e, 0x5f, 0x5f, 0x6b, 0x3d, 0x3d, 0x3d, + 0x61, 0x2e, 0x5f, 0x5f, 0x6b, 0x3f, 0x66, 0x3d, 0x51, 0x28, 0x61, 0x2c, + 0x66, 0x2c, 0x74, 0x29, 0x3a, 0x22, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, + 0x6f, 0x6e, 0x22, 0x3d, 0x3d, 0x74, 0x79, 0x70, 0x65, 0x6f, 0x66, 0x20, + 0x61, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x26, 0x26, 0x76, 0x6f, 0x69, 0x64, + 0x20, 0x30, 0x21, 0x3d, 0x3d, 0x61, 0x2e, 0x5f, 0x5f, 0x64, 0x3f, 0x66, + 0x3d, 0x61, 0x2e, 0x5f, 0x5f, 0x64, 0x3a, 0x70, 0x26, 0x26, 0x28, 0x66, + 0x3d, 0x70, 0x2e, 0x6e, 0x65, 0x78, 0x74, 0x53, 0x69, 0x62, 0x6c, 0x69, + 0x6e, 0x67, 0x29, 0x2c, 0x61, 0x2e, 0x5f, 0x5f, 0x64, 0x3d, 0x76, 0x6f, + 0x69, 0x64, 0x20, 0x30, 0x2c, 0x61, 0x2e, 0x5f, 0x5f, 0x75, 0x26, 0x3d, + 0x2d, 0x31, 0x39, 0x36, 0x36, 0x30, 0x39, 0x29, 0x3b, 0x65, 0x2e, 0x5f, + 0x5f, 0x64, 0x3d, 0x66, 0x2c, 0x65, 0x2e, 0x5f, 0x5f, 0x65, 0x3d, 0x64, + 0x7d, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x4b, 0x28, + 0x74, 0x2c, 0x6e, 0x2c, 0x65, 0x29, 0x7b, 0x76, 0x61, 0x72, 0x20, 0x5f, + 0x2c, 0x69, 0x2c, 0x6f, 0x2c, 0x72, 0x2c, 0x75, 0x2c, 0x66, 0x3d, 0x6e, + 0x2e, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x2c, 0x73, 0x3d, 0x65, 0x2e, + 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x2c, 0x6c, 0x3d, 0x73, 0x2c, 0x63, + 0x3d, 0x30, 0x3b, 0x66, 0x6f, 0x72, 0x28, 0x74, 0x2e, 0x5f, 0x5f, 0x6b, + 0x3d, 0x5b, 0x5d, 0x2c, 0x5f, 0x3d, 0x30, 0x3b, 0x5f, 0x3c, 0x66, 0x3b, + 0x5f, 0x2b, 0x2b, 0x29, 0x6e, 0x75, 0x6c, 0x6c, 0x21, 0x3d, 0x28, 0x69, + 0x3d, 0x74, 0x2e, 0x5f, 0x5f, 0x6b, 0x5b, 0x5f, 0x5d, 0x3d, 0x6e, 0x75, + 0x6c, 0x6c, 0x3d, 0x3d, 0x28, 0x69, 0x3d, 0x6e, 0x5b, 0x5f, 0x5d, 0x29, + 0x7c, 0x7c, 0x22, 0x62, 0x6f, 0x6f, 0x6c, 0x65, 0x61, 0x6e, 0x22, 0x3d, + 0x3d, 0x74, 0x79, 0x70, 0x65, 0x6f, 0x66, 0x20, 0x69, 0x7c, 0x7c, 0x22, + 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x3d, 0x3d, 0x74, + 0x79, 0x70, 0x65, 0x6f, 0x66, 0x20, 0x69, 0x3f, 0x6e, 0x75, 0x6c, 0x6c, + 0x3a, 0x22, 0x73, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x22, 0x3d, 0x3d, 0x74, + 0x79, 0x70, 0x65, 0x6f, 0x66, 0x20, 0x69, 0x7c, 0x7c, 0x22, 0x6e, 0x75, + 0x6d, 0x62, 0x65, 0x72, 0x22, 0x3d, 0x3d, 0x74, 0x79, 0x70, 0x65, 0x6f, + 0x66, 0x20, 0x69, 0x7c, 0x7c, 0x22, 0x62, 0x69, 0x67, 0x69, 0x6e, 0x74, + 0x22, 0x3d, 0x3d, 0x74, 0x79, 0x70, 0x65, 0x6f, 0x66, 0x20, 0x69, 0x7c, + 0x7c, 0x69, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x72, 0x75, 0x63, 0x74, + 0x6f, 0x72, 0x3d, 0x3d, 0x53, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x3f, 0x4f, + 0x28, 0x6e, 0x75, 0x6c, 0x6c, 0x2c, 0x69, 0x2c, 0x6e, 0x75, 0x6c, 0x6c, + 0x2c, 0x6e, 0x75, 0x6c, 0x6c, 0x2c, 0x69, 0x29, 0x3a, 0x46, 0x28, 0x69, + 0x29, 0x3f, 0x4f, 0x28, 0x6a, 0x2c, 0x7b, 0x63, 0x68, 0x69, 0x6c, 0x64, + 0x72, 0x65, 0x6e, 0x3a, 0x69, 0x7d, 0x2c, 0x6e, 0x75, 0x6c, 0x6c, 0x2c, + 0x6e, 0x75, 0x6c, 0x6c, 0x2c, 0x6e, 0x75, 0x6c, 0x6c, 0x29, 0x3a, 0x76, + 0x6f, 0x69, 0x64, 0x20, 0x30, 0x3d, 0x3d, 0x3d, 0x69, 0x2e, 0x63, 0x6f, + 0x6e, 0x73, 0x74, 0x72, 0x75, 0x63, 0x74, 0x6f, 0x72, 0x26, 0x26, 0x69, + 0x2e, 0x5f, 0x5f, 0x62, 0x3e, 0x30, 0x3f, 0x4f, 0x28, 0x69, 0x2e, 0x74, + 0x79, 0x70, 0x65, 0x2c, 0x69, 0x2e, 0x70, 0x72, 0x6f, 0x70, 0x73, 0x2c, + 0x69, 0x2e, 0x6b, 0x65, 0x79, 0x2c, 0x69, 0x2e, 0x72, 0x65, 0x66, 0x3f, + 0x69, 0x2e, 0x72, 0x65, 0x66, 0x3a, 0x6e, 0x75, 0x6c, 0x6c, 0x2c, 0x69, + 0x2e, 0x5f, 0x5f, 0x76, 0x29, 0x3a, 0x69, 0x29, 0x3f, 0x28, 0x69, 0x2e, + 0x5f, 0x5f, 0x3d, 0x74, 0x2c, 0x69, 0x2e, 0x5f, 0x5f, 0x62, 0x3d, 0x74, + 0x2e, 0x5f, 0x5f, 0x62, 0x2b, 0x31, 0x2c, 0x75, 0x3d, 0x59, 0x28, 0x69, + 0x2c, 0x65, 0x2c, 0x72, 0x3d, 0x5f, 0x2b, 0x63, 0x2c, 0x6c, 0x29, 0x2c, + 0x69, 0x2e, 0x5f, 0x5f, 0x69, 0x3d, 0x75, 0x2c, 0x6f, 0x3d, 0x6e, 0x75, + 0x6c, 0x6c, 0x2c, 0x2d, 0x31, 0x21, 0x3d, 0x3d, 0x75, 0x26, 0x26, 0x28, + 0x6c, 0x2d, 0x2d, 0x2c, 0x28, 0x6f, 0x3d, 0x65, 0x5b, 0x75, 0x5d, 0x29, + 0x26, 0x26, 0x28, 0x6f, 0x2e, 0x5f, 0x5f, 0x75, 0x7c, 0x3d, 0x31, 0x33, + 0x31, 0x30, 0x37, 0x32, 0x29, 0x29, 0x2c, 0x6e, 0x75, 0x6c, 0x6c, 0x3d, + 0x3d, 0x6f, 0x7c, 0x7c, 0x6e, 0x75, 0x6c, 0x6c, 0x3d, 0x3d, 0x3d, 0x6f, + 0x2e, 0x5f, 0x5f, 0x76, 0x3f, 0x28, 0x2d, 0x31, 0x3d, 0x3d, 0x75, 0x26, + 0x26, 0x63, 0x2d, 0x2d, 0x2c, 0x22, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, + 0x6f, 0x6e, 0x22, 0x21, 0x3d, 0x74, 0x79, 0x70, 0x65, 0x6f, 0x66, 0x20, + 0x69, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x26, 0x26, 0x28, 0x69, 0x2e, 0x5f, + 0x5f, 0x75, 0x7c, 0x3d, 0x36, 0x35, 0x35, 0x33, 0x36, 0x29, 0x29, 0x3a, + 0x75, 0x21, 0x3d, 0x3d, 0x72, 0x26, 0x26, 0x28, 0x75, 0x3d, 0x3d, 0x3d, + 0x72, 0x2b, 0x31, 0x3f, 0x63, 0x2b, 0x2b, 0x3a, 0x75, 0x3e, 0x72, 0x3f, + 0x6c, 0x3e, 0x66, 0x2d, 0x72, 0x3f, 0x63, 0x2b, 0x3d, 0x75, 0x2d, 0x72, + 0x3a, 0x63, 0x2d, 0x2d, 0x3a, 0x63, 0x3d, 0x75, 0x3c, 0x72, 0x26, 0x26, + 0x75, 0x3d, 0x3d, 0x72, 0x2d, 0x31, 0x3f, 0x75, 0x2d, 0x72, 0x3a, 0x30, + 0x2c, 0x75, 0x21, 0x3d, 0x3d, 0x5f, 0x2b, 0x63, 0x26, 0x26, 0x28, 0x69, + 0x2e, 0x5f, 0x5f, 0x75, 0x7c, 0x3d, 0x36, 0x35, 0x35, 0x33, 0x36, 0x29, + 0x29, 0x29, 0x3a, 0x28, 0x6f, 0x3d, 0x65, 0x5b, 0x5f, 0x5d, 0x29, 0x26, + 0x26, 0x6e, 0x75, 0x6c, 0x6c, 0x3d, 0x3d, 0x6f, 0x2e, 0x6b, 0x65, 0x79, + 0x26, 0x26, 0x6f, 0x2e, 0x5f, 0x5f, 0x65, 0x26, 0x26, 0x28, 0x6f, 0x2e, + 0x5f, 0x5f, 0x65, 0x3d, 0x3d, 0x74, 0x2e, 0x5f, 0x5f, 0x64, 0x26, 0x26, + 0x28, 0x74, 0x2e, 0x5f, 0x5f, 0x64, 0x3d, 0x71, 0x28, 0x6f, 0x29, 0x29, + 0x2c, 0x75, 0x74, 0x28, 0x6f, 0x2c, 0x6f, 0x2c, 0x21, 0x31, 0x29, 0x2c, + 0x65, 0x5b, 0x5f, 0x5d, 0x3d, 0x6e, 0x75, 0x6c, 0x6c, 0x2c, 0x6c, 0x2d, + 0x2d, 0x29, 0x3b, 0x69, 0x66, 0x28, 0x6c, 0x29, 0x66, 0x6f, 0x72, 0x28, + 0x5f, 0x3d, 0x30, 0x3b, 0x5f, 0x3c, 0x73, 0x3b, 0x5f, 0x2b, 0x2b, 0x29, + 0x6e, 0x75, 0x6c, 0x6c, 0x21, 0x3d, 0x28, 0x6f, 0x3d, 0x65, 0x5b, 0x5f, + 0x5d, 0x29, 0x26, 0x26, 0x30, 0x3d, 0x3d, 0x28, 0x31, 0x33, 0x31, 0x30, + 0x37, 0x32, 0x26, 0x6f, 0x2e, 0x5f, 0x5f, 0x75, 0x29, 0x26, 0x26, 0x28, + 0x6f, 0x2e, 0x5f, 0x5f, 0x65, 0x3d, 0x3d, 0x74, 0x2e, 0x5f, 0x5f, 0x64, + 0x26, 0x26, 0x28, 0x74, 0x2e, 0x5f, 0x5f, 0x64, 0x3d, 0x71, 0x28, 0x6f, + 0x29, 0x29, 0x2c, 0x75, 0x74, 0x28, 0x6f, 0x2c, 0x6f, 0x29, 0x29, 0x7d, + 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x51, 0x28, 0x74, + 0x2c, 0x6e, 0x2c, 0x65, 0x29, 0x7b, 0x76, 0x61, 0x72, 0x20, 0x5f, 0x2c, + 0x69, 0x3b, 0x69, 0x66, 0x28, 0x22, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, + 0x6f, 0x6e, 0x22, 0x3d, 0x3d, 0x74, 0x79, 0x70, 0x65, 0x6f, 0x66, 0x20, + 0x74, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x29, 0x7b, 0x66, 0x6f, 0x72, 0x28, + 0x5f, 0x3d, 0x74, 0x2e, 0x5f, 0x5f, 0x6b, 0x2c, 0x69, 0x3d, 0x30, 0x3b, + 0x5f, 0x26, 0x26, 0x69, 0x3c, 0x5f, 0x2e, 0x6c, 0x65, 0x6e, 0x67, 0x74, + 0x68, 0x3b, 0x69, 0x2b, 0x2b, 0x29, 0x5f, 0x5b, 0x69, 0x5d, 0x26, 0x26, + 0x28, 0x5f, 0x5b, 0x69, 0x5d, 0x2e, 0x5f, 0x5f, 0x3d, 0x74, 0x2c, 0x6e, + 0x3d, 0x51, 0x28, 0x5f, 0x5b, 0x69, 0x5d, 0x2c, 0x6e, 0x2c, 0x65, 0x29, + 0x29, 0x3b, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x6e, 0x7d, 0x72, + 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x74, 0x2e, 0x5f, 0x5f, 0x65, 0x21, + 0x3d, 0x6e, 0x26, 0x26, 0x28, 0x65, 0x2e, 0x69, 0x6e, 0x73, 0x65, 0x72, + 0x74, 0x42, 0x65, 0x66, 0x6f, 0x72, 0x65, 0x28, 0x74, 0x2e, 0x5f, 0x5f, + 0x65, 0x2c, 0x6e, 0x7c, 0x7c, 0x6e, 0x75, 0x6c, 0x6c, 0x29, 0x2c, 0x6e, + 0x3d, 0x74, 0x2e, 0x5f, 0x5f, 0x65, 0x29, 0x2c, 0x6e, 0x26, 0x26, 0x6e, + 0x2e, 0x6e, 0x65, 0x78, 0x74, 0x53, 0x69, 0x62, 0x6c, 0x69, 0x6e, 0x67, + 0x7d, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x58, 0x28, + 0x74, 0x2c, 0x6e, 0x29, 0x7b, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, + 0x6e, 0x3d, 0x6e, 0x7c, 0x7c, 0x5b, 0x5d, 0x2c, 0x6e, 0x75, 0x6c, 0x6c, + 0x3d, 0x3d, 0x74, 0x7c, 0x7c, 0x22, 0x62, 0x6f, 0x6f, 0x6c, 0x65, 0x61, + 0x6e, 0x22, 0x3d, 0x3d, 0x74, 0x79, 0x70, 0x65, 0x6f, 0x66, 0x20, 0x74, + 0x7c, 0x7c, 0x28, 0x46, 0x28, 0x74, 0x29, 0x3f, 0x74, 0x2e, 0x73, 0x6f, + 0x6d, 0x65, 0x28, 0x28, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, + 0x28, 0x74, 0x29, 0x7b, 0x58, 0x28, 0x74, 0x2c, 0x6e, 0x29, 0x7d, 0x29, + 0x29, 0x3a, 0x6e, 0x2e, 0x70, 0x75, 0x73, 0x68, 0x28, 0x74, 0x29, 0x29, + 0x2c, 0x6e, 0x7d, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, + 0x59, 0x28, 0x74, 0x2c, 0x6e, 0x2c, 0x65, 0x2c, 0x5f, 0x29, 0x7b, 0x76, + 0x61, 0x72, 0x20, 0x69, 0x3d, 0x74, 0x2e, 0x6b, 0x65, 0x79, 0x2c, 0x6f, + 0x3d, 0x74, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x2c, 0x72, 0x3d, 0x65, 0x2d, + 0x31, 0x2c, 0x75, 0x3d, 0x65, 0x2b, 0x31, 0x2c, 0x66, 0x3d, 0x6e, 0x5b, + 0x65, 0x5d, 0x3b, 0x69, 0x66, 0x28, 0x6e, 0x75, 0x6c, 0x6c, 0x3d, 0x3d, + 0x3d, 0x66, 0x7c, 0x7c, 0x66, 0x26, 0x26, 0x69, 0x3d, 0x3d, 0x66, 0x2e, + 0x6b, 0x65, 0x79, 0x26, 0x26, 0x6f, 0x3d, 0x3d, 0x3d, 0x66, 0x2e, 0x74, + 0x79, 0x70, 0x65, 0x29, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x65, + 0x3b, 0x69, 0x66, 0x28, 0x5f, 0x3e, 0x28, 0x6e, 0x75, 0x6c, 0x6c, 0x21, + 0x3d, 0x66, 0x26, 0x26, 0x30, 0x3d, 0x3d, 0x28, 0x31, 0x33, 0x31, 0x30, + 0x37, 0x32, 0x26, 0x66, 0x2e, 0x5f, 0x5f, 0x75, 0x29, 0x3f, 0x31, 0x3a, + 0x30, 0x29, 0x29, 0x66, 0x6f, 0x72, 0x28, 0x3b, 0x72, 0x3e, 0x3d, 0x30, + 0x7c, 0x7c, 0x75, 0x3c, 0x6e, 0x2e, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, + 0x3b, 0x29, 0x7b, 0x69, 0x66, 0x28, 0x72, 0x3e, 0x3d, 0x30, 0x29, 0x7b, + 0x69, 0x66, 0x28, 0x28, 0x66, 0x3d, 0x6e, 0x5b, 0x72, 0x5d, 0x29, 0x26, + 0x26, 0x30, 0x3d, 0x3d, 0x28, 0x31, 0x33, 0x31, 0x30, 0x37, 0x32, 0x26, + 0x66, 0x2e, 0x5f, 0x5f, 0x75, 0x29, 0x26, 0x26, 0x69, 0x3d, 0x3d, 0x66, + 0x2e, 0x6b, 0x65, 0x79, 0x26, 0x26, 0x6f, 0x3d, 0x3d, 0x3d, 0x66, 0x2e, + 0x74, 0x79, 0x70, 0x65, 0x29, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, + 0x72, 0x3b, 0x72, 0x2d, 0x2d, 0x7d, 0x69, 0x66, 0x28, 0x75, 0x3c, 0x6e, + 0x2e, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x29, 0x7b, 0x69, 0x66, 0x28, + 0x28, 0x66, 0x3d, 0x6e, 0x5b, 0x75, 0x5d, 0x29, 0x26, 0x26, 0x30, 0x3d, + 0x3d, 0x28, 0x31, 0x33, 0x31, 0x30, 0x37, 0x32, 0x26, 0x66, 0x2e, 0x5f, + 0x5f, 0x75, 0x29, 0x26, 0x26, 0x69, 0x3d, 0x3d, 0x66, 0x2e, 0x6b, 0x65, + 0x79, 0x26, 0x26, 0x6f, 0x3d, 0x3d, 0x3d, 0x66, 0x2e, 0x74, 0x79, 0x70, + 0x65, 0x29, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x75, 0x3b, 0x75, + 0x2b, 0x2b, 0x7d, 0x7d, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x2d, 0x31, + 0x7d, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x5a, 0x28, + 0x74, 0x2c, 0x6e, 0x2c, 0x65, 0x29, 0x7b, 0x22, 0x2d, 0x22, 0x3d, 0x3d, + 0x3d, 0x6e, 0x5b, 0x30, 0x5d, 0x3f, 0x74, 0x2e, 0x73, 0x65, 0x74, 0x50, + 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x79, 0x28, 0x6e, 0x2c, 0x6e, 0x75, + 0x6c, 0x6c, 0x3d, 0x3d, 0x65, 0x3f, 0x22, 0x22, 0x3a, 0x65, 0x29, 0x3a, + 0x74, 0x5b, 0x6e, 0x5d, 0x3d, 0x6e, 0x75, 0x6c, 0x6c, 0x3d, 0x3d, 0x65, + 0x3f, 0x22, 0x22, 0x3a, 0x22, 0x6e, 0x75, 0x6d, 0x62, 0x65, 0x72, 0x22, + 0x21, 0x3d, 0x74, 0x79, 0x70, 0x65, 0x6f, 0x66, 0x20, 0x65, 0x7c, 0x7c, + 0x41, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x28, 0x6e, 0x29, 0x3f, 0x65, 0x3a, + 0x65, 0x2b, 0x22, 0x70, 0x78, 0x22, 0x7d, 0x66, 0x75, 0x6e, 0x63, 0x74, + 0x69, 0x6f, 0x6e, 0x20, 0x74, 0x74, 0x28, 0x74, 0x2c, 0x6e, 0x2c, 0x65, + 0x2c, 0x5f, 0x2c, 0x69, 0x29, 0x7b, 0x76, 0x61, 0x72, 0x20, 0x6f, 0x3b, + 0x74, 0x3a, 0x69, 0x66, 0x28, 0x22, 0x73, 0x74, 0x79, 0x6c, 0x65, 0x22, + 0x3d, 0x3d, 0x3d, 0x6e, 0x29, 0x69, 0x66, 0x28, 0x22, 0x73, 0x74, 0x72, + 0x69, 0x6e, 0x67, 0x22, 0x3d, 0x3d, 0x74, 0x79, 0x70, 0x65, 0x6f, 0x66, + 0x20, 0x65, 0x29, 0x74, 0x2e, 0x73, 0x74, 0x79, 0x6c, 0x65, 0x2e, 0x63, + 0x73, 0x73, 0x54, 0x65, 0x78, 0x74, 0x3d, 0x65, 0x3b, 0x65, 0x6c, 0x73, + 0x65, 0x7b, 0x69, 0x66, 0x28, 0x22, 0x73, 0x74, 0x72, 0x69, 0x6e, 0x67, + 0x22, 0x3d, 0x3d, 0x74, 0x79, 0x70, 0x65, 0x6f, 0x66, 0x20, 0x5f, 0x26, + 0x26, 0x28, 0x74, 0x2e, 0x73, 0x74, 0x79, 0x6c, 0x65, 0x2e, 0x63, 0x73, + 0x73, 0x54, 0x65, 0x78, 0x74, 0x3d, 0x5f, 0x3d, 0x22, 0x22, 0x29, 0x2c, + 0x5f, 0x29, 0x66, 0x6f, 0x72, 0x28, 0x6e, 0x20, 0x69, 0x6e, 0x20, 0x5f, + 0x29, 0x65, 0x26, 0x26, 0x6e, 0x20, 0x69, 0x6e, 0x20, 0x65, 0x7c, 0x7c, + 0x5a, 0x28, 0x74, 0x2e, 0x73, 0x74, 0x79, 0x6c, 0x65, 0x2c, 0x6e, 0x2c, + 0x22, 0x22, 0x29, 0x3b, 0x69, 0x66, 0x28, 0x65, 0x29, 0x66, 0x6f, 0x72, + 0x28, 0x6e, 0x20, 0x69, 0x6e, 0x20, 0x65, 0x29, 0x5f, 0x26, 0x26, 0x65, + 0x5b, 0x6e, 0x5d, 0x3d, 0x3d, 0x3d, 0x5f, 0x5b, 0x6e, 0x5d, 0x7c, 0x7c, + 0x5a, 0x28, 0x74, 0x2e, 0x73, 0x74, 0x79, 0x6c, 0x65, 0x2c, 0x6e, 0x2c, + 0x65, 0x5b, 0x6e, 0x5d, 0x29, 0x7d, 0x65, 0x6c, 0x73, 0x65, 0x20, 0x69, + 0x66, 0x28, 0x22, 0x6f, 0x22, 0x3d, 0x3d, 0x3d, 0x6e, 0x5b, 0x30, 0x5d, + 0x26, 0x26, 0x22, 0x6e, 0x22, 0x3d, 0x3d, 0x3d, 0x6e, 0x5b, 0x31, 0x5d, + 0x29, 0x6f, 0x3d, 0x6e, 0x21, 0x3d, 0x3d, 0x28, 0x6e, 0x3d, 0x6e, 0x2e, + 0x72, 0x65, 0x70, 0x6c, 0x61, 0x63, 0x65, 0x28, 0x2f, 0x28, 0x50, 0x6f, + 0x69, 0x6e, 0x74, 0x65, 0x72, 0x43, 0x61, 0x70, 0x74, 0x75, 0x72, 0x65, + 0x29, 0x24, 0x7c, 0x43, 0x61, 0x70, 0x74, 0x75, 0x72, 0x65, 0x24, 0x2f, + 0x2c, 0x22, 0x24, 0x31, 0x22, 0x29, 0x29, 0x2c, 0x6e, 0x3d, 0x6e, 0x2e, + 0x74, 0x6f, 0x4c, 0x6f, 0x77, 0x65, 0x72, 0x43, 0x61, 0x73, 0x65, 0x28, + 0x29, 0x69, 0x6e, 0x20, 0x74, 0x3f, 0x6e, 0x2e, 0x74, 0x6f, 0x4c, 0x6f, + 0x77, 0x65, 0x72, 0x43, 0x61, 0x73, 0x65, 0x28, 0x29, 0x2e, 0x73, 0x6c, + 0x69, 0x63, 0x65, 0x28, 0x32, 0x29, 0x3a, 0x6e, 0x2e, 0x73, 0x6c, 0x69, + 0x63, 0x65, 0x28, 0x32, 0x29, 0x2c, 0x74, 0x2e, 0x6c, 0x7c, 0x7c, 0x28, + 0x74, 0x2e, 0x6c, 0x3d, 0x7b, 0x7d, 0x29, 0x2c, 0x74, 0x2e, 0x6c, 0x5b, + 0x6e, 0x2b, 0x6f, 0x5d, 0x3d, 0x65, 0x2c, 0x65, 0x3f, 0x5f, 0x3f, 0x65, + 0x2e, 0x75, 0x3d, 0x5f, 0x2e, 0x75, 0x3a, 0x28, 0x65, 0x2e, 0x75, 0x3d, + 0x44, 0x61, 0x74, 0x65, 0x2e, 0x6e, 0x6f, 0x77, 0x28, 0x29, 0x2c, 0x74, + 0x2e, 0x61, 0x64, 0x64, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x4c, 0x69, 0x73, + 0x74, 0x65, 0x6e, 0x65, 0x72, 0x28, 0x6e, 0x2c, 0x6f, 0x3f, 0x65, 0x74, + 0x3a, 0x6e, 0x74, 0x2c, 0x6f, 0x29, 0x29, 0x3a, 0x74, 0x2e, 0x72, 0x65, + 0x6d, 0x6f, 0x76, 0x65, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x4c, 0x69, 0x73, + 0x74, 0x65, 0x6e, 0x65, 0x72, 0x28, 0x6e, 0x2c, 0x6f, 0x3f, 0x65, 0x74, + 0x3a, 0x6e, 0x74, 0x2c, 0x6f, 0x29, 0x3b, 0x65, 0x6c, 0x73, 0x65, 0x7b, + 0x69, 0x66, 0x28, 0x69, 0x29, 0x6e, 0x3d, 0x6e, 0x2e, 0x72, 0x65, 0x70, + 0x6c, 0x61, 0x63, 0x65, 0x28, 0x2f, 0x78, 0x6c, 0x69, 0x6e, 0x6b, 0x28, + 0x48, 0x7c, 0x3a, 0x68, 0x29, 0x2f, 0x2c, 0x22, 0x68, 0x22, 0x29, 0x2e, + 0x72, 0x65, 0x70, 0x6c, 0x61, 0x63, 0x65, 0x28, 0x2f, 0x73, 0x4e, 0x61, + 0x6d, 0x65, 0x24, 0x2f, 0x2c, 0x22, 0x73, 0x22, 0x29, 0x3b, 0x65, 0x6c, + 0x73, 0x65, 0x20, 0x69, 0x66, 0x28, 0x22, 0x77, 0x69, 0x64, 0x74, 0x68, + 0x22, 0x21, 0x3d, 0x3d, 0x6e, 0x26, 0x26, 0x22, 0x68, 0x65, 0x69, 0x67, + 0x68, 0x74, 0x22, 0x21, 0x3d, 0x3d, 0x6e, 0x26, 0x26, 0x22, 0x68, 0x72, + 0x65, 0x66, 0x22, 0x21, 0x3d, 0x3d, 0x6e, 0x26, 0x26, 0x22, 0x6c, 0x69, + 0x73, 0x74, 0x22, 0x21, 0x3d, 0x3d, 0x6e, 0x26, 0x26, 0x22, 0x66, 0x6f, + 0x72, 0x6d, 0x22, 0x21, 0x3d, 0x3d, 0x6e, 0x26, 0x26, 0x22, 0x74, 0x61, + 0x62, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x22, 0x21, 0x3d, 0x3d, 0x6e, 0x26, + 0x26, 0x22, 0x64, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x22, 0x21, + 0x3d, 0x3d, 0x6e, 0x26, 0x26, 0x22, 0x72, 0x6f, 0x77, 0x53, 0x70, 0x61, + 0x6e, 0x22, 0x21, 0x3d, 0x3d, 0x6e, 0x26, 0x26, 0x22, 0x63, 0x6f, 0x6c, + 0x53, 0x70, 0x61, 0x6e, 0x22, 0x21, 0x3d, 0x3d, 0x6e, 0x26, 0x26, 0x22, + 0x72, 0x6f, 0x6c, 0x65, 0x22, 0x21, 0x3d, 0x3d, 0x6e, 0x26, 0x26, 0x6e, + 0x20, 0x69, 0x6e, 0x20, 0x74, 0x29, 0x74, 0x72, 0x79, 0x7b, 0x74, 0x5b, + 0x6e, 0x5d, 0x3d, 0x6e, 0x75, 0x6c, 0x6c, 0x3d, 0x3d, 0x65, 0x3f, 0x22, + 0x22, 0x3a, 0x65, 0x3b, 0x62, 0x72, 0x65, 0x61, 0x6b, 0x20, 0x74, 0x7d, + 0x63, 0x61, 0x74, 0x63, 0x68, 0x28, 0x74, 0x29, 0x7b, 0x7d, 0x22, 0x66, + 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x3d, 0x3d, 0x74, 0x79, + 0x70, 0x65, 0x6f, 0x66, 0x20, 0x65, 0x7c, 0x7c, 0x28, 0x6e, 0x75, 0x6c, + 0x6c, 0x3d, 0x3d, 0x65, 0x7c, 0x7c, 0x21, 0x31, 0x3d, 0x3d, 0x3d, 0x65, + 0x26, 0x26, 0x22, 0x2d, 0x22, 0x21, 0x3d, 0x3d, 0x6e, 0x5b, 0x34, 0x5d, + 0x3f, 0x74, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x41, 0x74, 0x74, + 0x72, 0x69, 0x62, 0x75, 0x74, 0x65, 0x28, 0x6e, 0x29, 0x3a, 0x74, 0x2e, + 0x73, 0x65, 0x74, 0x41, 0x74, 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, 0x65, + 0x28, 0x6e, 0x2c, 0x65, 0x29, 0x29, 0x7d, 0x7d, 0x66, 0x75, 0x6e, 0x63, + 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x6e, 0x74, 0x28, 0x74, 0x29, 0x7b, 0x76, + 0x61, 0x72, 0x20, 0x6e, 0x3d, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x6c, 0x5b, + 0x74, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x2b, 0x21, 0x31, 0x5d, 0x3b, 0x69, + 0x66, 0x28, 0x74, 0x2e, 0x74, 0x29, 0x7b, 0x69, 0x66, 0x28, 0x74, 0x2e, + 0x74, 0x3c, 0x3d, 0x6e, 0x2e, 0x75, 0x29, 0x72, 0x65, 0x74, 0x75, 0x72, + 0x6e, 0x7d, 0x65, 0x6c, 0x73, 0x65, 0x20, 0x74, 0x2e, 0x74, 0x3d, 0x44, + 0x61, 0x74, 0x65, 0x2e, 0x6e, 0x6f, 0x77, 0x28, 0x29, 0x3b, 0x72, 0x65, + 0x74, 0x75, 0x72, 0x6e, 0x20, 0x6e, 0x28, 0x43, 0x2e, 0x65, 0x76, 0x65, + 0x6e, 0x74, 0x3f, 0x43, 0x2e, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x28, 0x74, + 0x29, 0x3a, 0x74, 0x29, 0x7d, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, + 0x6e, 0x20, 0x65, 0x74, 0x28, 0x74, 0x29, 0x7b, 0x72, 0x65, 0x74, 0x75, + 0x72, 0x6e, 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x6c, 0x5b, 0x74, 0x2e, + 0x74, 0x79, 0x70, 0x65, 0x2b, 0x21, 0x30, 0x5d, 0x28, 0x43, 0x2e, 0x65, + 0x76, 0x65, 0x6e, 0x74, 0x3f, 0x43, 0x2e, 0x65, 0x76, 0x65, 0x6e, 0x74, + 0x28, 0x74, 0x29, 0x3a, 0x74, 0x29, 0x7d, 0x66, 0x75, 0x6e, 0x63, 0x74, + 0x69, 0x6f, 0x6e, 0x20, 0x5f, 0x74, 0x28, 0x74, 0x2c, 0x6e, 0x2c, 0x65, + 0x2c, 0x5f, 0x2c, 0x69, 0x2c, 0x6f, 0x2c, 0x72, 0x2c, 0x75, 0x2c, 0x66, + 0x2c, 0x73, 0x29, 0x7b, 0x76, 0x61, 0x72, 0x20, 0x6c, 0x2c, 0x63, 0x2c, + 0x68, 0x2c, 0x61, 0x2c, 0x70, 0x2c, 0x64, 0x2c, 0x76, 0x2c, 0x79, 0x2c, + 0x6d, 0x2c, 0x67, 0x2c, 0x62, 0x2c, 0x6b, 0x2c, 0x53, 0x2c, 0x77, 0x2c, + 0x78, 0x2c, 0x45, 0x3d, 0x6e, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x3b, 0x69, + 0x66, 0x28, 0x76, 0x6f, 0x69, 0x64, 0x20, 0x30, 0x21, 0x3d, 0x3d, 0x6e, + 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x72, 0x75, 0x63, 0x74, 0x6f, 0x72, + 0x29, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x6e, 0x75, 0x6c, 0x6c, + 0x3b, 0x31, 0x32, 0x38, 0x26, 0x65, 0x2e, 0x5f, 0x5f, 0x75, 0x26, 0x26, + 0x28, 0x66, 0x3d, 0x21, 0x21, 0x28, 0x33, 0x32, 0x26, 0x65, 0x2e, 0x5f, + 0x5f, 0x75, 0x29, 0x2c, 0x6f, 0x3d, 0x5b, 0x75, 0x3d, 0x6e, 0x2e, 0x5f, + 0x5f, 0x65, 0x3d, 0x65, 0x2e, 0x5f, 0x5f, 0x65, 0x5d, 0x29, 0x2c, 0x28, + 0x6c, 0x3d, 0x43, 0x2e, 0x5f, 0x5f, 0x62, 0x29, 0x26, 0x26, 0x6c, 0x28, + 0x6e, 0x29, 0x3b, 0x74, 0x3a, 0x69, 0x66, 0x28, 0x22, 0x66, 0x75, 0x6e, + 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x3d, 0x3d, 0x74, 0x79, 0x70, 0x65, + 0x6f, 0x66, 0x20, 0x45, 0x29, 0x74, 0x72, 0x79, 0x7b, 0x69, 0x66, 0x28, + 0x79, 0x3d, 0x6e, 0x2e, 0x70, 0x72, 0x6f, 0x70, 0x73, 0x2c, 0x6d, 0x3d, + 0x28, 0x6c, 0x3d, 0x45, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, + 0x54, 0x79, 0x70, 0x65, 0x29, 0x26, 0x26, 0x5f, 0x5b, 0x6c, 0x2e, 0x5f, + 0x5f, 0x63, 0x5d, 0x2c, 0x67, 0x3d, 0x6c, 0x3f, 0x6d, 0x3f, 0x6d, 0x2e, + 0x70, 0x72, 0x6f, 0x70, 0x73, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, + 0x6c, 0x2e, 0x5f, 0x5f, 0x3a, 0x5f, 0x2c, 0x65, 0x2e, 0x5f, 0x5f, 0x63, + 0x3f, 0x76, 0x3d, 0x28, 0x63, 0x3d, 0x6e, 0x2e, 0x5f, 0x5f, 0x63, 0x3d, + 0x65, 0x2e, 0x5f, 0x5f, 0x63, 0x29, 0x2e, 0x5f, 0x5f, 0x3d, 0x63, 0x2e, + 0x5f, 0x5f, 0x45, 0x3a, 0x28, 0x22, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x74, + 0x79, 0x70, 0x65, 0x22, 0x69, 0x6e, 0x20, 0x45, 0x26, 0x26, 0x45, 0x2e, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x74, 0x79, 0x70, 0x65, 0x2e, 0x72, 0x65, + 0x6e, 0x64, 0x65, 0x72, 0x3f, 0x6e, 0x2e, 0x5f, 0x5f, 0x63, 0x3d, 0x63, + 0x3d, 0x6e, 0x65, 0x77, 0x20, 0x45, 0x28, 0x79, 0x2c, 0x67, 0x29, 0x3a, + 0x28, 0x6e, 0x2e, 0x5f, 0x5f, 0x63, 0x3d, 0x63, 0x3d, 0x6e, 0x65, 0x77, + 0x20, 0x49, 0x28, 0x79, 0x2c, 0x67, 0x29, 0x2c, 0x63, 0x2e, 0x63, 0x6f, + 0x6e, 0x73, 0x74, 0x72, 0x75, 0x63, 0x74, 0x6f, 0x72, 0x3d, 0x45, 0x2c, + 0x63, 0x2e, 0x72, 0x65, 0x6e, 0x64, 0x65, 0x72, 0x3d, 0x66, 0x74, 0x29, + 0x2c, 0x6d, 0x26, 0x26, 0x6d, 0x2e, 0x73, 0x75, 0x62, 0x28, 0x63, 0x29, + 0x2c, 0x63, 0x2e, 0x70, 0x72, 0x6f, 0x70, 0x73, 0x3d, 0x79, 0x2c, 0x63, + 0x2e, 0x73, 0x74, 0x61, 0x74, 0x65, 0x7c, 0x7c, 0x28, 0x63, 0x2e, 0x73, + 0x74, 0x61, 0x74, 0x65, 0x3d, 0x7b, 0x7d, 0x29, 0x2c, 0x63, 0x2e, 0x63, + 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x3d, 0x67, 0x2c, 0x63, 0x2e, 0x5f, + 0x5f, 0x6e, 0x3d, 0x5f, 0x2c, 0x68, 0x3d, 0x63, 0x2e, 0x5f, 0x5f, 0x64, + 0x3d, 0x21, 0x30, 0x2c, 0x63, 0x2e, 0x5f, 0x5f, 0x68, 0x3d, 0x5b, 0x5d, + 0x2c, 0x63, 0x2e, 0x5f, 0x73, 0x62, 0x3d, 0x5b, 0x5d, 0x29, 0x2c, 0x6e, + 0x75, 0x6c, 0x6c, 0x3d, 0x3d, 0x63, 0x2e, 0x5f, 0x5f, 0x73, 0x26, 0x26, + 0x28, 0x63, 0x2e, 0x5f, 0x5f, 0x73, 0x3d, 0x63, 0x2e, 0x73, 0x74, 0x61, + 0x74, 0x65, 0x29, 0x2c, 0x6e, 0x75, 0x6c, 0x6c, 0x21, 0x3d, 0x45, 0x2e, + 0x67, 0x65, 0x74, 0x44, 0x65, 0x72, 0x69, 0x76, 0x65, 0x64, 0x53, 0x74, + 0x61, 0x74, 0x65, 0x46, 0x72, 0x6f, 0x6d, 0x50, 0x72, 0x6f, 0x70, 0x73, + 0x26, 0x26, 0x28, 0x63, 0x2e, 0x5f, 0x5f, 0x73, 0x3d, 0x3d, 0x63, 0x2e, + 0x73, 0x74, 0x61, 0x74, 0x65, 0x26, 0x26, 0x28, 0x63, 0x2e, 0x5f, 0x5f, + 0x73, 0x3d, 0x4d, 0x28, 0x7b, 0x7d, 0x2c, 0x63, 0x2e, 0x5f, 0x5f, 0x73, + 0x29, 0x29, 0x2c, 0x4d, 0x28, 0x63, 0x2e, 0x5f, 0x5f, 0x73, 0x2c, 0x45, + 0x2e, 0x67, 0x65, 0x74, 0x44, 0x65, 0x72, 0x69, 0x76, 0x65, 0x64, 0x53, + 0x74, 0x61, 0x74, 0x65, 0x46, 0x72, 0x6f, 0x6d, 0x50, 0x72, 0x6f, 0x70, + 0x73, 0x28, 0x79, 0x2c, 0x63, 0x2e, 0x5f, 0x5f, 0x73, 0x29, 0x29, 0x29, + 0x2c, 0x61, 0x3d, 0x63, 0x2e, 0x70, 0x72, 0x6f, 0x70, 0x73, 0x2c, 0x70, + 0x3d, 0x63, 0x2e, 0x73, 0x74, 0x61, 0x74, 0x65, 0x2c, 0x63, 0x2e, 0x5f, + 0x5f, 0x76, 0x3d, 0x6e, 0x2c, 0x68, 0x29, 0x6e, 0x75, 0x6c, 0x6c, 0x3d, + 0x3d, 0x45, 0x2e, 0x67, 0x65, 0x74, 0x44, 0x65, 0x72, 0x69, 0x76, 0x65, + 0x64, 0x53, 0x74, 0x61, 0x74, 0x65, 0x46, 0x72, 0x6f, 0x6d, 0x50, 0x72, + 0x6f, 0x70, 0x73, 0x26, 0x26, 0x6e, 0x75, 0x6c, 0x6c, 0x21, 0x3d, 0x63, + 0x2e, 0x63, 0x6f, 0x6d, 0x70, 0x6f, 0x6e, 0x65, 0x6e, 0x74, 0x57, 0x69, + 0x6c, 0x6c, 0x4d, 0x6f, 0x75, 0x6e, 0x74, 0x26, 0x26, 0x63, 0x2e, 0x63, + 0x6f, 0x6d, 0x70, 0x6f, 0x6e, 0x65, 0x6e, 0x74, 0x57, 0x69, 0x6c, 0x6c, + 0x4d, 0x6f, 0x75, 0x6e, 0x74, 0x28, 0x29, 0x2c, 0x6e, 0x75, 0x6c, 0x6c, + 0x21, 0x3d, 0x63, 0x2e, 0x63, 0x6f, 0x6d, 0x70, 0x6f, 0x6e, 0x65, 0x6e, + 0x74, 0x44, 0x69, 0x64, 0x4d, 0x6f, 0x75, 0x6e, 0x74, 0x26, 0x26, 0x63, + 0x2e, 0x5f, 0x5f, 0x68, 0x2e, 0x70, 0x75, 0x73, 0x68, 0x28, 0x63, 0x2e, + 0x63, 0x6f, 0x6d, 0x70, 0x6f, 0x6e, 0x65, 0x6e, 0x74, 0x44, 0x69, 0x64, + 0x4d, 0x6f, 0x75, 0x6e, 0x74, 0x29, 0x3b, 0x65, 0x6c, 0x73, 0x65, 0x7b, + 0x69, 0x66, 0x28, 0x6e, 0x75, 0x6c, 0x6c, 0x3d, 0x3d, 0x45, 0x2e, 0x67, + 0x65, 0x74, 0x44, 0x65, 0x72, 0x69, 0x76, 0x65, 0x64, 0x53, 0x74, 0x61, + 0x74, 0x65, 0x46, 0x72, 0x6f, 0x6d, 0x50, 0x72, 0x6f, 0x70, 0x73, 0x26, + 0x26, 0x79, 0x21, 0x3d, 0x3d, 0x61, 0x26, 0x26, 0x6e, 0x75, 0x6c, 0x6c, + 0x21, 0x3d, 0x63, 0x2e, 0x63, 0x6f, 0x6d, 0x70, 0x6f, 0x6e, 0x65, 0x6e, + 0x74, 0x57, 0x69, 0x6c, 0x6c, 0x52, 0x65, 0x63, 0x65, 0x69, 0x76, 0x65, + 0x50, 0x72, 0x6f, 0x70, 0x73, 0x26, 0x26, 0x63, 0x2e, 0x63, 0x6f, 0x6d, + 0x70, 0x6f, 0x6e, 0x65, 0x6e, 0x74, 0x57, 0x69, 0x6c, 0x6c, 0x52, 0x65, + 0x63, 0x65, 0x69, 0x76, 0x65, 0x50, 0x72, 0x6f, 0x70, 0x73, 0x28, 0x79, + 0x2c, 0x67, 0x29, 0x2c, 0x21, 0x63, 0x2e, 0x5f, 0x5f, 0x65, 0x26, 0x26, + 0x28, 0x6e, 0x75, 0x6c, 0x6c, 0x21, 0x3d, 0x63, 0x2e, 0x73, 0x68, 0x6f, + 0x75, 0x6c, 0x64, 0x43, 0x6f, 0x6d, 0x70, 0x6f, 0x6e, 0x65, 0x6e, 0x74, + 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x26, 0x26, 0x21, 0x31, 0x3d, 0x3d, + 0x3d, 0x63, 0x2e, 0x73, 0x68, 0x6f, 0x75, 0x6c, 0x64, 0x43, 0x6f, 0x6d, + 0x70, 0x6f, 0x6e, 0x65, 0x6e, 0x74, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, + 0x28, 0x79, 0x2c, 0x63, 0x2e, 0x5f, 0x5f, 0x73, 0x2c, 0x67, 0x29, 0x7c, + 0x7c, 0x6e, 0x2e, 0x5f, 0x5f, 0x76, 0x3d, 0x3d, 0x3d, 0x65, 0x2e, 0x5f, + 0x5f, 0x76, 0x29, 0x29, 0x7b, 0x66, 0x6f, 0x72, 0x28, 0x6e, 0x2e, 0x5f, + 0x5f, 0x76, 0x21, 0x3d, 0x3d, 0x65, 0x2e, 0x5f, 0x5f, 0x76, 0x26, 0x26, + 0x28, 0x63, 0x2e, 0x70, 0x72, 0x6f, 0x70, 0x73, 0x3d, 0x79, 0x2c, 0x63, + 0x2e, 0x73, 0x74, 0x61, 0x74, 0x65, 0x3d, 0x63, 0x2e, 0x5f, 0x5f, 0x73, + 0x2c, 0x63, 0x2e, 0x5f, 0x5f, 0x64, 0x3d, 0x21, 0x31, 0x29, 0x2c, 0x6e, + 0x2e, 0x5f, 0x5f, 0x65, 0x3d, 0x65, 0x2e, 0x5f, 0x5f, 0x65, 0x2c, 0x6e, + 0x2e, 0x5f, 0x5f, 0x6b, 0x3d, 0x65, 0x2e, 0x5f, 0x5f, 0x6b, 0x2c, 0x6e, + 0x2e, 0x5f, 0x5f, 0x6b, 0x2e, 0x66, 0x6f, 0x72, 0x45, 0x61, 0x63, 0x68, + 0x28, 0x28, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x74, + 0x29, 0x7b, 0x74, 0x26, 0x26, 0x28, 0x74, 0x2e, 0x5f, 0x5f, 0x3d, 0x6e, + 0x29, 0x7d, 0x29, 0x29, 0x2c, 0x62, 0x3d, 0x30, 0x3b, 0x62, 0x3c, 0x63, + 0x2e, 0x5f, 0x73, 0x62, 0x2e, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x3b, + 0x62, 0x2b, 0x2b, 0x29, 0x63, 0x2e, 0x5f, 0x5f, 0x68, 0x2e, 0x70, 0x75, + 0x73, 0x68, 0x28, 0x63, 0x2e, 0x5f, 0x73, 0x62, 0x5b, 0x62, 0x5d, 0x29, + 0x3b, 0x63, 0x2e, 0x5f, 0x73, 0x62, 0x3d, 0x5b, 0x5d, 0x2c, 0x63, 0x2e, + 0x5f, 0x5f, 0x68, 0x2e, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x26, 0x26, + 0x72, 0x2e, 0x70, 0x75, 0x73, 0x68, 0x28, 0x63, 0x29, 0x3b, 0x62, 0x72, + 0x65, 0x61, 0x6b, 0x20, 0x74, 0x7d, 0x6e, 0x75, 0x6c, 0x6c, 0x21, 0x3d, + 0x63, 0x2e, 0x63, 0x6f, 0x6d, 0x70, 0x6f, 0x6e, 0x65, 0x6e, 0x74, 0x57, + 0x69, 0x6c, 0x6c, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x26, 0x26, 0x63, + 0x2e, 0x63, 0x6f, 0x6d, 0x70, 0x6f, 0x6e, 0x65, 0x6e, 0x74, 0x57, 0x69, + 0x6c, 0x6c, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x28, 0x79, 0x2c, 0x63, + 0x2e, 0x5f, 0x5f, 0x73, 0x2c, 0x67, 0x29, 0x2c, 0x6e, 0x75, 0x6c, 0x6c, + 0x21, 0x3d, 0x63, 0x2e, 0x63, 0x6f, 0x6d, 0x70, 0x6f, 0x6e, 0x65, 0x6e, + 0x74, 0x44, 0x69, 0x64, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x26, 0x26, + 0x63, 0x2e, 0x5f, 0x5f, 0x68, 0x2e, 0x70, 0x75, 0x73, 0x68, 0x28, 0x28, + 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x29, 0x7b, 0x63, + 0x2e, 0x63, 0x6f, 0x6d, 0x70, 0x6f, 0x6e, 0x65, 0x6e, 0x74, 0x44, 0x69, + 0x64, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x28, 0x61, 0x2c, 0x70, 0x2c, + 0x64, 0x29, 0x7d, 0x29, 0x29, 0x7d, 0x69, 0x66, 0x28, 0x63, 0x2e, 0x63, + 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x3d, 0x67, 0x2c, 0x63, 0x2e, 0x70, + 0x72, 0x6f, 0x70, 0x73, 0x3d, 0x79, 0x2c, 0x63, 0x2e, 0x5f, 0x5f, 0x50, + 0x3d, 0x74, 0x2c, 0x63, 0x2e, 0x5f, 0x5f, 0x65, 0x3d, 0x21, 0x31, 0x2c, + 0x6b, 0x3d, 0x43, 0x2e, 0x5f, 0x5f, 0x72, 0x2c, 0x53, 0x3d, 0x30, 0x2c, + 0x22, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x74, 0x79, 0x70, 0x65, 0x22, 0x69, + 0x6e, 0x20, 0x45, 0x26, 0x26, 0x45, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x74, 0x79, 0x70, 0x65, 0x2e, 0x72, 0x65, 0x6e, 0x64, 0x65, 0x72, 0x29, + 0x7b, 0x66, 0x6f, 0x72, 0x28, 0x63, 0x2e, 0x73, 0x74, 0x61, 0x74, 0x65, + 0x3d, 0x63, 0x2e, 0x5f, 0x5f, 0x73, 0x2c, 0x63, 0x2e, 0x5f, 0x5f, 0x64, + 0x3d, 0x21, 0x31, 0x2c, 0x6b, 0x26, 0x26, 0x6b, 0x28, 0x6e, 0x29, 0x2c, + 0x6c, 0x3d, 0x63, 0x2e, 0x72, 0x65, 0x6e, 0x64, 0x65, 0x72, 0x28, 0x63, + 0x2e, 0x70, 0x72, 0x6f, 0x70, 0x73, 0x2c, 0x63, 0x2e, 0x73, 0x74, 0x61, + 0x74, 0x65, 0x2c, 0x63, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, + 0x29, 0x2c, 0x77, 0x3d, 0x30, 0x3b, 0x77, 0x3c, 0x63, 0x2e, 0x5f, 0x73, + 0x62, 0x2e, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x3b, 0x77, 0x2b, 0x2b, + 0x29, 0x63, 0x2e, 0x5f, 0x5f, 0x68, 0x2e, 0x70, 0x75, 0x73, 0x68, 0x28, + 0x63, 0x2e, 0x5f, 0x73, 0x62, 0x5b, 0x77, 0x5d, 0x29, 0x3b, 0x63, 0x2e, + 0x5f, 0x73, 0x62, 0x3d, 0x5b, 0x5d, 0x7d, 0x65, 0x6c, 0x73, 0x65, 0x20, + 0x64, 0x6f, 0x7b, 0x63, 0x2e, 0x5f, 0x5f, 0x64, 0x3d, 0x21, 0x31, 0x2c, + 0x6b, 0x26, 0x26, 0x6b, 0x28, 0x6e, 0x29, 0x2c, 0x6c, 0x3d, 0x63, 0x2e, + 0x72, 0x65, 0x6e, 0x64, 0x65, 0x72, 0x28, 0x63, 0x2e, 0x70, 0x72, 0x6f, + 0x70, 0x73, 0x2c, 0x63, 0x2e, 0x73, 0x74, 0x61, 0x74, 0x65, 0x2c, 0x63, + 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x29, 0x2c, 0x63, 0x2e, + 0x73, 0x74, 0x61, 0x74, 0x65, 0x3d, 0x63, 0x2e, 0x5f, 0x5f, 0x73, 0x7d, + 0x77, 0x68, 0x69, 0x6c, 0x65, 0x28, 0x63, 0x2e, 0x5f, 0x5f, 0x64, 0x26, + 0x26, 0x2b, 0x2b, 0x53, 0x3c, 0x32, 0x35, 0x29, 0x3b, 0x63, 0x2e, 0x73, + 0x74, 0x61, 0x74, 0x65, 0x3d, 0x63, 0x2e, 0x5f, 0x5f, 0x73, 0x2c, 0x6e, + 0x75, 0x6c, 0x6c, 0x21, 0x3d, 0x63, 0x2e, 0x67, 0x65, 0x74, 0x43, 0x68, + 0x69, 0x6c, 0x64, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x26, 0x26, + 0x28, 0x5f, 0x3d, 0x4d, 0x28, 0x4d, 0x28, 0x7b, 0x7d, 0x2c, 0x5f, 0x29, + 0x2c, 0x63, 0x2e, 0x67, 0x65, 0x74, 0x43, 0x68, 0x69, 0x6c, 0x64, 0x43, + 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x28, 0x29, 0x29, 0x29, 0x2c, 0x68, + 0x7c, 0x7c, 0x6e, 0x75, 0x6c, 0x6c, 0x3d, 0x3d, 0x63, 0x2e, 0x67, 0x65, + 0x74, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x42, 0x65, 0x66, + 0x6f, 0x72, 0x65, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x7c, 0x7c, 0x28, + 0x64, 0x3d, 0x63, 0x2e, 0x67, 0x65, 0x74, 0x53, 0x6e, 0x61, 0x70, 0x73, + 0x68, 0x6f, 0x74, 0x42, 0x65, 0x66, 0x6f, 0x72, 0x65, 0x55, 0x70, 0x64, + 0x61, 0x74, 0x65, 0x28, 0x61, 0x2c, 0x70, 0x29, 0x29, 0x2c, 0x4a, 0x28, + 0x74, 0x2c, 0x46, 0x28, 0x78, 0x3d, 0x6e, 0x75, 0x6c, 0x6c, 0x21, 0x3d, + 0x6c, 0x26, 0x26, 0x6c, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x3d, 0x3d, 0x3d, + 0x6a, 0x26, 0x26, 0x6e, 0x75, 0x6c, 0x6c, 0x3d, 0x3d, 0x6c, 0x2e, 0x6b, + 0x65, 0x79, 0x3f, 0x6c, 0x2e, 0x70, 0x72, 0x6f, 0x70, 0x73, 0x2e, 0x63, + 0x68, 0x69, 0x6c, 0x64, 0x72, 0x65, 0x6e, 0x3a, 0x6c, 0x29, 0x3f, 0x78, + 0x3a, 0x5b, 0x78, 0x5d, 0x2c, 0x6e, 0x2c, 0x65, 0x2c, 0x5f, 0x2c, 0x69, + 0x2c, 0x6f, 0x2c, 0x72, 0x2c, 0x75, 0x2c, 0x66, 0x2c, 0x73, 0x29, 0x2c, + 0x63, 0x2e, 0x62, 0x61, 0x73, 0x65, 0x3d, 0x6e, 0x2e, 0x5f, 0x5f, 0x65, + 0x2c, 0x6e, 0x2e, 0x5f, 0x5f, 0x75, 0x26, 0x3d, 0x2d, 0x31, 0x36, 0x31, + 0x2c, 0x63, 0x2e, 0x5f, 0x5f, 0x68, 0x2e, 0x6c, 0x65, 0x6e, 0x67, 0x74, + 0x68, 0x26, 0x26, 0x72, 0x2e, 0x70, 0x75, 0x73, 0x68, 0x28, 0x63, 0x29, + 0x2c, 0x76, 0x26, 0x26, 0x28, 0x63, 0x2e, 0x5f, 0x5f, 0x45, 0x3d, 0x63, + 0x2e, 0x5f, 0x5f, 0x3d, 0x6e, 0x75, 0x6c, 0x6c, 0x29, 0x7d, 0x63, 0x61, + 0x74, 0x63, 0x68, 0x28, 0x74, 0x29, 0x7b, 0x6e, 0x2e, 0x5f, 0x5f, 0x76, + 0x3d, 0x6e, 0x75, 0x6c, 0x6c, 0x2c, 0x66, 0x7c, 0x7c, 0x6e, 0x75, 0x6c, + 0x6c, 0x21, 0x3d, 0x6f, 0x3f, 0x28, 0x6e, 0x2e, 0x5f, 0x5f, 0x65, 0x3d, + 0x75, 0x2c, 0x6e, 0x2e, 0x5f, 0x5f, 0x75, 0x7c, 0x3d, 0x66, 0x3f, 0x31, + 0x36, 0x30, 0x3a, 0x33, 0x32, 0x2c, 0x6f, 0x5b, 0x6f, 0x2e, 0x69, 0x6e, + 0x64, 0x65, 0x78, 0x4f, 0x66, 0x28, 0x75, 0x29, 0x5d, 0x3d, 0x6e, 0x75, + 0x6c, 0x6c, 0x29, 0x3a, 0x28, 0x6e, 0x2e, 0x5f, 0x5f, 0x65, 0x3d, 0x65, + 0x2e, 0x5f, 0x5f, 0x65, 0x2c, 0x6e, 0x2e, 0x5f, 0x5f, 0x6b, 0x3d, 0x65, + 0x2e, 0x5f, 0x5f, 0x6b, 0x29, 0x2c, 0x43, 0x2e, 0x5f, 0x5f, 0x65, 0x28, + 0x74, 0x2c, 0x6e, 0x2c, 0x65, 0x29, 0x7d, 0x65, 0x6c, 0x73, 0x65, 0x20, + 0x6e, 0x75, 0x6c, 0x6c, 0x3d, 0x3d, 0x6f, 0x26, 0x26, 0x6e, 0x2e, 0x5f, + 0x5f, 0x76, 0x3d, 0x3d, 0x3d, 0x65, 0x2e, 0x5f, 0x5f, 0x76, 0x3f, 0x28, + 0x6e, 0x2e, 0x5f, 0x5f, 0x6b, 0x3d, 0x65, 0x2e, 0x5f, 0x5f, 0x6b, 0x2c, + 0x6e, 0x2e, 0x5f, 0x5f, 0x65, 0x3d, 0x65, 0x2e, 0x5f, 0x5f, 0x65, 0x29, + 0x3a, 0x6e, 0x2e, 0x5f, 0x5f, 0x65, 0x3d, 0x6f, 0x74, 0x28, 0x65, 0x2e, + 0x5f, 0x5f, 0x65, 0x2c, 0x6e, 0x2c, 0x65, 0x2c, 0x5f, 0x2c, 0x69, 0x2c, + 0x6f, 0x2c, 0x72, 0x2c, 0x66, 0x2c, 0x73, 0x29, 0x3b, 0x28, 0x6c, 0x3d, + 0x43, 0x2e, 0x64, 0x69, 0x66, 0x66, 0x65, 0x64, 0x29, 0x26, 0x26, 0x6c, + 0x28, 0x6e, 0x29, 0x7d, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, + 0x20, 0x69, 0x74, 0x28, 0x74, 0x2c, 0x6e, 0x2c, 0x65, 0x29, 0x7b, 0x6e, + 0x2e, 0x5f, 0x5f, 0x64, 0x3d, 0x76, 0x6f, 0x69, 0x64, 0x20, 0x30, 0x3b, + 0x66, 0x6f, 0x72, 0x28, 0x76, 0x61, 0x72, 0x20, 0x5f, 0x3d, 0x30, 0x3b, + 0x5f, 0x3c, 0x65, 0x2e, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x3b, 0x5f, + 0x2b, 0x2b, 0x29, 0x72, 0x74, 0x28, 0x65, 0x5b, 0x5f, 0x5d, 0x2c, 0x65, + 0x5b, 0x2b, 0x2b, 0x5f, 0x5d, 0x2c, 0x65, 0x5b, 0x2b, 0x2b, 0x5f, 0x5d, + 0x29, 0x3b, 0x43, 0x2e, 0x5f, 0x5f, 0x63, 0x26, 0x26, 0x43, 0x2e, 0x5f, + 0x5f, 0x63, 0x28, 0x6e, 0x2c, 0x74, 0x29, 0x2c, 0x74, 0x2e, 0x73, 0x6f, + 0x6d, 0x65, 0x28, 0x28, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, + 0x28, 0x6e, 0x29, 0x7b, 0x74, 0x72, 0x79, 0x7b, 0x74, 0x3d, 0x6e, 0x2e, + 0x5f, 0x5f, 0x68, 0x2c, 0x6e, 0x2e, 0x5f, 0x5f, 0x68, 0x3d, 0x5b, 0x5d, + 0x2c, 0x74, 0x2e, 0x73, 0x6f, 0x6d, 0x65, 0x28, 0x28, 0x66, 0x75, 0x6e, + 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x74, 0x29, 0x7b, 0x74, 0x2e, 0x63, + 0x61, 0x6c, 0x6c, 0x28, 0x6e, 0x29, 0x7d, 0x29, 0x29, 0x7d, 0x63, 0x61, + 0x74, 0x63, 0x68, 0x28, 0x74, 0x29, 0x7b, 0x43, 0x2e, 0x5f, 0x5f, 0x65, + 0x28, 0x74, 0x2c, 0x6e, 0x2e, 0x5f, 0x5f, 0x76, 0x29, 0x7d, 0x7d, 0x29, + 0x29, 0x7d, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x6f, + 0x74, 0x28, 0x74, 0x2c, 0x6e, 0x2c, 0x65, 0x2c, 0x5f, 0x2c, 0x69, 0x2c, + 0x6f, 0x2c, 0x72, 0x2c, 0x75, 0x2c, 0x66, 0x29, 0x7b, 0x76, 0x61, 0x72, + 0x20, 0x73, 0x2c, 0x6c, 0x2c, 0x63, 0x2c, 0x68, 0x2c, 0x61, 0x2c, 0x70, + 0x2c, 0x64, 0x2c, 0x76, 0x3d, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x70, 0x73, + 0x2c, 0x79, 0x3d, 0x6e, 0x2e, 0x70, 0x72, 0x6f, 0x70, 0x73, 0x2c, 0x6d, + 0x3d, 0x6e, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x3b, 0x69, 0x66, 0x28, 0x22, + 0x73, 0x76, 0x67, 0x22, 0x3d, 0x3d, 0x3d, 0x6d, 0x26, 0x26, 0x28, 0x69, + 0x3d, 0x21, 0x30, 0x29, 0x2c, 0x6e, 0x75, 0x6c, 0x6c, 0x21, 0x3d, 0x6f, + 0x29, 0x66, 0x6f, 0x72, 0x28, 0x73, 0x3d, 0x30, 0x3b, 0x73, 0x3c, 0x6f, + 0x2e, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x3b, 0x73, 0x2b, 0x2b, 0x29, + 0x69, 0x66, 0x28, 0x28, 0x61, 0x3d, 0x6f, 0x5b, 0x73, 0x5d, 0x29, 0x26, + 0x26, 0x22, 0x73, 0x65, 0x74, 0x41, 0x74, 0x74, 0x72, 0x69, 0x62, 0x75, + 0x74, 0x65, 0x22, 0x69, 0x6e, 0x20, 0x61, 0x3d, 0x3d, 0x21, 0x21, 0x6d, + 0x26, 0x26, 0x28, 0x6d, 0x3f, 0x61, 0x2e, 0x6c, 0x6f, 0x63, 0x61, 0x6c, + 0x4e, 0x61, 0x6d, 0x65, 0x3d, 0x3d, 0x3d, 0x6d, 0x3a, 0x33, 0x3d, 0x3d, + 0x3d, 0x61, 0x2e, 0x6e, 0x6f, 0x64, 0x65, 0x54, 0x79, 0x70, 0x65, 0x29, + 0x29, 0x7b, 0x74, 0x3d, 0x61, 0x2c, 0x6f, 0x5b, 0x73, 0x5d, 0x3d, 0x6e, + 0x75, 0x6c, 0x6c, 0x3b, 0x62, 0x72, 0x65, 0x61, 0x6b, 0x7d, 0x69, 0x66, + 0x28, 0x6e, 0x75, 0x6c, 0x6c, 0x3d, 0x3d, 0x74, 0x29, 0x7b, 0x69, 0x66, + 0x28, 0x6e, 0x75, 0x6c, 0x6c, 0x3d, 0x3d, 0x3d, 0x6d, 0x29, 0x72, 0x65, + 0x74, 0x75, 0x72, 0x6e, 0x20, 0x64, 0x6f, 0x63, 0x75, 0x6d, 0x65, 0x6e, + 0x74, 0x2e, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x54, 0x65, 0x78, 0x74, + 0x4e, 0x6f, 0x64, 0x65, 0x28, 0x79, 0x29, 0x3b, 0x74, 0x3d, 0x69, 0x3f, + 0x64, 0x6f, 0x63, 0x75, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x63, 0x72, 0x65, + 0x61, 0x74, 0x65, 0x45, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x4e, 0x53, + 0x28, 0x22, 0x68, 0x74, 0x74, 0x70, 0x3a, 0x2f, 0x2f, 0x77, 0x77, 0x77, + 0x2e, 0x77, 0x33, 0x2e, 0x6f, 0x72, 0x67, 0x2f, 0x32, 0x30, 0x30, 0x30, + 0x2f, 0x73, 0x76, 0x67, 0x22, 0x2c, 0x6d, 0x29, 0x3a, 0x64, 0x6f, 0x63, + 0x75, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, + 0x45, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x28, 0x6d, 0x2c, 0x79, 0x2e, + 0x69, 0x73, 0x26, 0x26, 0x79, 0x29, 0x2c, 0x6f, 0x3d, 0x6e, 0x75, 0x6c, + 0x6c, 0x2c, 0x75, 0x3d, 0x21, 0x31, 0x7d, 0x69, 0x66, 0x28, 0x6e, 0x75, + 0x6c, 0x6c, 0x3d, 0x3d, 0x3d, 0x6d, 0x29, 0x76, 0x3d, 0x3d, 0x3d, 0x79, + 0x7c, 0x7c, 0x75, 0x26, 0x26, 0x74, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x3d, + 0x3d, 0x3d, 0x79, 0x7c, 0x7c, 0x28, 0x74, 0x2e, 0x64, 0x61, 0x74, 0x61, + 0x3d, 0x79, 0x29, 0x3b, 0x65, 0x6c, 0x73, 0x65, 0x7b, 0x69, 0x66, 0x28, + 0x6f, 0x3d, 0x6f, 0x26, 0x26, 0x78, 0x2e, 0x63, 0x61, 0x6c, 0x6c, 0x28, + 0x74, 0x2e, 0x63, 0x68, 0x69, 0x6c, 0x64, 0x4e, 0x6f, 0x64, 0x65, 0x73, + 0x29, 0x2c, 0x76, 0x3d, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x70, 0x73, 0x7c, + 0x7c, 0x54, 0x2c, 0x21, 0x75, 0x26, 0x26, 0x6e, 0x75, 0x6c, 0x6c, 0x21, + 0x3d, 0x6f, 0x29, 0x66, 0x6f, 0x72, 0x28, 0x76, 0x3d, 0x7b, 0x7d, 0x2c, + 0x73, 0x3d, 0x30, 0x3b, 0x73, 0x3c, 0x74, 0x2e, 0x61, 0x74, 0x74, 0x72, + 0x69, 0x62, 0x75, 0x74, 0x65, 0x73, 0x2e, 0x6c, 0x65, 0x6e, 0x67, 0x74, + 0x68, 0x3b, 0x73, 0x2b, 0x2b, 0x29, 0x76, 0x5b, 0x28, 0x61, 0x3d, 0x74, + 0x2e, 0x61, 0x74, 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, 0x65, 0x73, 0x5b, + 0x73, 0x5d, 0x29, 0x2e, 0x6e, 0x61, 0x6d, 0x65, 0x5d, 0x3d, 0x61, 0x2e, + 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3b, 0x66, 0x6f, 0x72, 0x28, 0x73, 0x20, + 0x69, 0x6e, 0x20, 0x76, 0x29, 0x61, 0x3d, 0x76, 0x5b, 0x73, 0x5d, 0x2c, + 0x22, 0x63, 0x68, 0x69, 0x6c, 0x64, 0x72, 0x65, 0x6e, 0x22, 0x3d, 0x3d, + 0x73, 0x7c, 0x7c, 0x28, 0x22, 0x64, 0x61, 0x6e, 0x67, 0x65, 0x72, 0x6f, + 0x75, 0x73, 0x6c, 0x79, 0x53, 0x65, 0x74, 0x49, 0x6e, 0x6e, 0x65, 0x72, + 0x48, 0x54, 0x4d, 0x4c, 0x22, 0x3d, 0x3d, 0x73, 0x3f, 0x63, 0x3d, 0x61, + 0x3a, 0x22, 0x6b, 0x65, 0x79, 0x22, 0x3d, 0x3d, 0x3d, 0x73, 0x7c, 0x7c, + 0x73, 0x20, 0x69, 0x6e, 0x20, 0x79, 0x7c, 0x7c, 0x74, 0x74, 0x28, 0x74, + 0x2c, 0x73, 0x2c, 0x6e, 0x75, 0x6c, 0x6c, 0x2c, 0x61, 0x2c, 0x69, 0x29, + 0x29, 0x3b, 0x66, 0x6f, 0x72, 0x28, 0x73, 0x20, 0x69, 0x6e, 0x20, 0x79, + 0x29, 0x61, 0x3d, 0x79, 0x5b, 0x73, 0x5d, 0x2c, 0x22, 0x63, 0x68, 0x69, + 0x6c, 0x64, 0x72, 0x65, 0x6e, 0x22, 0x3d, 0x3d, 0x73, 0x3f, 0x68, 0x3d, + 0x61, 0x3a, 0x22, 0x64, 0x61, 0x6e, 0x67, 0x65, 0x72, 0x6f, 0x75, 0x73, + 0x6c, 0x79, 0x53, 0x65, 0x74, 0x49, 0x6e, 0x6e, 0x65, 0x72, 0x48, 0x54, + 0x4d, 0x4c, 0x22, 0x3d, 0x3d, 0x73, 0x3f, 0x6c, 0x3d, 0x61, 0x3a, 0x22, + 0x76, 0x61, 0x6c, 0x75, 0x65, 0x22, 0x3d, 0x3d, 0x73, 0x3f, 0x70, 0x3d, + 0x61, 0x3a, 0x22, 0x63, 0x68, 0x65, 0x63, 0x6b, 0x65, 0x64, 0x22, 0x3d, + 0x3d, 0x73, 0x3f, 0x64, 0x3d, 0x61, 0x3a, 0x22, 0x6b, 0x65, 0x79, 0x22, + 0x3d, 0x3d, 0x3d, 0x73, 0x7c, 0x7c, 0x75, 0x26, 0x26, 0x22, 0x66, 0x75, + 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x21, 0x3d, 0x74, 0x79, 0x70, + 0x65, 0x6f, 0x66, 0x20, 0x61, 0x7c, 0x7c, 0x76, 0x5b, 0x73, 0x5d, 0x3d, + 0x3d, 0x3d, 0x61, 0x7c, 0x7c, 0x74, 0x74, 0x28, 0x74, 0x2c, 0x73, 0x2c, + 0x61, 0x2c, 0x76, 0x5b, 0x73, 0x5d, 0x2c, 0x69, 0x29, 0x3b, 0x69, 0x66, + 0x28, 0x6c, 0x29, 0x75, 0x7c, 0x7c, 0x63, 0x26, 0x26, 0x28, 0x6c, 0x2e, + 0x5f, 0x5f, 0x68, 0x74, 0x6d, 0x6c, 0x3d, 0x3d, 0x3d, 0x63, 0x2e, 0x5f, + 0x5f, 0x68, 0x74, 0x6d, 0x6c, 0x7c, 0x7c, 0x6c, 0x2e, 0x5f, 0x5f, 0x68, + 0x74, 0x6d, 0x6c, 0x3d, 0x3d, 0x3d, 0x74, 0x2e, 0x69, 0x6e, 0x6e, 0x65, + 0x72, 0x48, 0x54, 0x4d, 0x4c, 0x29, 0x7c, 0x7c, 0x28, 0x74, 0x2e, 0x69, + 0x6e, 0x6e, 0x65, 0x72, 0x48, 0x54, 0x4d, 0x4c, 0x3d, 0x6c, 0x2e, 0x5f, + 0x5f, 0x68, 0x74, 0x6d, 0x6c, 0x29, 0x2c, 0x6e, 0x2e, 0x5f, 0x5f, 0x6b, + 0x3d, 0x5b, 0x5d, 0x3b, 0x65, 0x6c, 0x73, 0x65, 0x20, 0x69, 0x66, 0x28, + 0x63, 0x26, 0x26, 0x28, 0x74, 0x2e, 0x69, 0x6e, 0x6e, 0x65, 0x72, 0x48, + 0x54, 0x4d, 0x4c, 0x3d, 0x22, 0x22, 0x29, 0x2c, 0x4a, 0x28, 0x74, 0x2c, + 0x46, 0x28, 0x68, 0x29, 0x3f, 0x68, 0x3a, 0x5b, 0x68, 0x5d, 0x2c, 0x6e, + 0x2c, 0x65, 0x2c, 0x5f, 0x2c, 0x69, 0x26, 0x26, 0x22, 0x66, 0x6f, 0x72, + 0x65, 0x69, 0x67, 0x6e, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x22, 0x21, + 0x3d, 0x3d, 0x6d, 0x2c, 0x6f, 0x2c, 0x72, 0x2c, 0x6f, 0x3f, 0x6f, 0x5b, + 0x30, 0x5d, 0x3a, 0x65, 0x2e, 0x5f, 0x5f, 0x6b, 0x26, 0x26, 0x71, 0x28, + 0x65, 0x2c, 0x30, 0x29, 0x2c, 0x75, 0x2c, 0x66, 0x29, 0x2c, 0x6e, 0x75, + 0x6c, 0x6c, 0x21, 0x3d, 0x6f, 0x29, 0x66, 0x6f, 0x72, 0x28, 0x73, 0x3d, + 0x6f, 0x2e, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x3b, 0x73, 0x2d, 0x2d, + 0x3b, 0x29, 0x6e, 0x75, 0x6c, 0x6c, 0x21, 0x3d, 0x6f, 0x5b, 0x73, 0x5d, + 0x26, 0x26, 0x57, 0x28, 0x6f, 0x5b, 0x73, 0x5d, 0x29, 0x3b, 0x75, 0x7c, + 0x7c, 0x28, 0x73, 0x3d, 0x22, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x22, 0x2c, + 0x76, 0x6f, 0x69, 0x64, 0x20, 0x30, 0x21, 0x3d, 0x3d, 0x70, 0x26, 0x26, + 0x28, 0x70, 0x21, 0x3d, 0x3d, 0x74, 0x5b, 0x73, 0x5d, 0x7c, 0x7c, 0x22, + 0x70, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x22, 0x3d, 0x3d, 0x3d, + 0x6d, 0x26, 0x26, 0x21, 0x70, 0x7c, 0x7c, 0x22, 0x6f, 0x70, 0x74, 0x69, + 0x6f, 0x6e, 0x22, 0x3d, 0x3d, 0x3d, 0x6d, 0x26, 0x26, 0x70, 0x21, 0x3d, + 0x3d, 0x76, 0x5b, 0x73, 0x5d, 0x29, 0x26, 0x26, 0x74, 0x74, 0x28, 0x74, + 0x2c, 0x73, 0x2c, 0x70, 0x2c, 0x76, 0x5b, 0x73, 0x5d, 0x2c, 0x21, 0x31, + 0x29, 0x2c, 0x73, 0x3d, 0x22, 0x63, 0x68, 0x65, 0x63, 0x6b, 0x65, 0x64, + 0x22, 0x2c, 0x76, 0x6f, 0x69, 0x64, 0x20, 0x30, 0x21, 0x3d, 0x3d, 0x64, + 0x26, 0x26, 0x64, 0x21, 0x3d, 0x3d, 0x74, 0x5b, 0x73, 0x5d, 0x26, 0x26, + 0x74, 0x74, 0x28, 0x74, 0x2c, 0x73, 0x2c, 0x64, 0x2c, 0x76, 0x5b, 0x73, + 0x5d, 0x2c, 0x21, 0x31, 0x29, 0x29, 0x7d, 0x72, 0x65, 0x74, 0x75, 0x72, + 0x6e, 0x20, 0x74, 0x7d, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, + 0x20, 0x72, 0x74, 0x28, 0x74, 0x2c, 0x6e, 0x2c, 0x65, 0x29, 0x7b, 0x74, + 0x72, 0x79, 0x7b, 0x22, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, + 0x22, 0x3d, 0x3d, 0x74, 0x79, 0x70, 0x65, 0x6f, 0x66, 0x20, 0x74, 0x3f, + 0x74, 0x28, 0x6e, 0x29, 0x3a, 0x74, 0x2e, 0x63, 0x75, 0x72, 0x72, 0x65, + 0x6e, 0x74, 0x3d, 0x6e, 0x7d, 0x63, 0x61, 0x74, 0x63, 0x68, 0x28, 0x74, + 0x29, 0x7b, 0x43, 0x2e, 0x5f, 0x5f, 0x65, 0x28, 0x74, 0x2c, 0x65, 0x29, + 0x7d, 0x7d, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x75, + 0x74, 0x28, 0x74, 0x2c, 0x6e, 0x2c, 0x65, 0x29, 0x7b, 0x76, 0x61, 0x72, + 0x20, 0x5f, 0x2c, 0x69, 0x3b, 0x69, 0x66, 0x28, 0x43, 0x2e, 0x75, 0x6e, + 0x6d, 0x6f, 0x75, 0x6e, 0x74, 0x26, 0x26, 0x43, 0x2e, 0x75, 0x6e, 0x6d, + 0x6f, 0x75, 0x6e, 0x74, 0x28, 0x74, 0x29, 0x2c, 0x28, 0x5f, 0x3d, 0x74, + 0x2e, 0x72, 0x65, 0x66, 0x29, 0x26, 0x26, 0x28, 0x5f, 0x2e, 0x63, 0x75, + 0x72, 0x72, 0x65, 0x6e, 0x74, 0x26, 0x26, 0x5f, 0x2e, 0x63, 0x75, 0x72, + 0x72, 0x65, 0x6e, 0x74, 0x21, 0x3d, 0x3d, 0x74, 0x2e, 0x5f, 0x5f, 0x65, + 0x7c, 0x7c, 0x72, 0x74, 0x28, 0x5f, 0x2c, 0x6e, 0x75, 0x6c, 0x6c, 0x2c, + 0x6e, 0x29, 0x29, 0x2c, 0x6e, 0x75, 0x6c, 0x6c, 0x21, 0x3d, 0x28, 0x5f, + 0x3d, 0x74, 0x2e, 0x5f, 0x5f, 0x63, 0x29, 0x29, 0x7b, 0x69, 0x66, 0x28, + 0x5f, 0x2e, 0x63, 0x6f, 0x6d, 0x70, 0x6f, 0x6e, 0x65, 0x6e, 0x74, 0x57, + 0x69, 0x6c, 0x6c, 0x55, 0x6e, 0x6d, 0x6f, 0x75, 0x6e, 0x74, 0x29, 0x74, + 0x72, 0x79, 0x7b, 0x5f, 0x2e, 0x63, 0x6f, 0x6d, 0x70, 0x6f, 0x6e, 0x65, + 0x6e, 0x74, 0x57, 0x69, 0x6c, 0x6c, 0x55, 0x6e, 0x6d, 0x6f, 0x75, 0x6e, + 0x74, 0x28, 0x29, 0x7d, 0x63, 0x61, 0x74, 0x63, 0x68, 0x28, 0x74, 0x29, + 0x7b, 0x43, 0x2e, 0x5f, 0x5f, 0x65, 0x28, 0x74, 0x2c, 0x6e, 0x29, 0x7d, + 0x5f, 0x2e, 0x62, 0x61, 0x73, 0x65, 0x3d, 0x5f, 0x2e, 0x5f, 0x5f, 0x50, + 0x3d, 0x6e, 0x75, 0x6c, 0x6c, 0x2c, 0x74, 0x2e, 0x5f, 0x5f, 0x63, 0x3d, + 0x76, 0x6f, 0x69, 0x64, 0x20, 0x30, 0x7d, 0x69, 0x66, 0x28, 0x5f, 0x3d, + 0x74, 0x2e, 0x5f, 0x5f, 0x6b, 0x29, 0x66, 0x6f, 0x72, 0x28, 0x69, 0x3d, + 0x30, 0x3b, 0x69, 0x3c, 0x5f, 0x2e, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, + 0x3b, 0x69, 0x2b, 0x2b, 0x29, 0x5f, 0x5b, 0x69, 0x5d, 0x26, 0x26, 0x75, + 0x74, 0x28, 0x5f, 0x5b, 0x69, 0x5d, 0x2c, 0x6e, 0x2c, 0x65, 0x7c, 0x7c, + 0x22, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x21, 0x3d, + 0x74, 0x79, 0x70, 0x65, 0x6f, 0x66, 0x20, 0x74, 0x2e, 0x74, 0x79, 0x70, + 0x65, 0x29, 0x3b, 0x65, 0x7c, 0x7c, 0x6e, 0x75, 0x6c, 0x6c, 0x3d, 0x3d, + 0x74, 0x2e, 0x5f, 0x5f, 0x65, 0x7c, 0x7c, 0x57, 0x28, 0x74, 0x2e, 0x5f, + 0x5f, 0x65, 0x29, 0x2c, 0x74, 0x2e, 0x5f, 0x5f, 0x3d, 0x74, 0x2e, 0x5f, + 0x5f, 0x65, 0x3d, 0x74, 0x2e, 0x5f, 0x5f, 0x64, 0x3d, 0x76, 0x6f, 0x69, + 0x64, 0x20, 0x30, 0x7d, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, + 0x20, 0x66, 0x74, 0x28, 0x74, 0x2c, 0x6e, 0x2c, 0x65, 0x29, 0x7b, 0x72, + 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x63, + 0x6f, 0x6e, 0x73, 0x74, 0x72, 0x75, 0x63, 0x74, 0x6f, 0x72, 0x28, 0x74, + 0x2c, 0x65, 0x29, 0x7d, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, + 0x20, 0x73, 0x74, 0x28, 0x74, 0x2c, 0x6e, 0x2c, 0x65, 0x29, 0x7b, 0x76, + 0x61, 0x72, 0x20, 0x5f, 0x2c, 0x69, 0x2c, 0x6f, 0x2c, 0x72, 0x3b, 0x43, + 0x2e, 0x5f, 0x5f, 0x26, 0x26, 0x43, 0x2e, 0x5f, 0x5f, 0x28, 0x74, 0x2c, + 0x6e, 0x29, 0x2c, 0x69, 0x3d, 0x28, 0x5f, 0x3d, 0x22, 0x66, 0x75, 0x6e, + 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x3d, 0x3d, 0x74, 0x79, 0x70, 0x65, + 0x6f, 0x66, 0x20, 0x65, 0x29, 0x3f, 0x6e, 0x75, 0x6c, 0x6c, 0x3a, 0x65, + 0x26, 0x26, 0x65, 0x2e, 0x5f, 0x5f, 0x6b, 0x7c, 0x7c, 0x6e, 0x2e, 0x5f, + 0x5f, 0x6b, 0x2c, 0x6f, 0x3d, 0x5b, 0x5d, 0x2c, 0x72, 0x3d, 0x5b, 0x5d, + 0x2c, 0x5f, 0x74, 0x28, 0x6e, 0x2c, 0x74, 0x3d, 0x28, 0x21, 0x5f, 0x26, + 0x26, 0x65, 0x7c, 0x7c, 0x6e, 0x29, 0x2e, 0x5f, 0x5f, 0x6b, 0x3d, 0x4c, + 0x28, 0x6a, 0x2c, 0x6e, 0x75, 0x6c, 0x6c, 0x2c, 0x5b, 0x74, 0x5d, 0x29, + 0x2c, 0x69, 0x7c, 0x7c, 0x54, 0x2c, 0x54, 0x2c, 0x76, 0x6f, 0x69, 0x64, + 0x20, 0x30, 0x21, 0x3d, 0x3d, 0x6e, 0x2e, 0x6f, 0x77, 0x6e, 0x65, 0x72, + 0x53, 0x56, 0x47, 0x45, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2c, 0x21, + 0x5f, 0x26, 0x26, 0x65, 0x3f, 0x5b, 0x65, 0x5d, 0x3a, 0x69, 0x3f, 0x6e, + 0x75, 0x6c, 0x6c, 0x3a, 0x6e, 0x2e, 0x66, 0x69, 0x72, 0x73, 0x74, 0x43, + 0x68, 0x69, 0x6c, 0x64, 0x3f, 0x78, 0x2e, 0x63, 0x61, 0x6c, 0x6c, 0x28, + 0x6e, 0x2e, 0x63, 0x68, 0x69, 0x6c, 0x64, 0x4e, 0x6f, 0x64, 0x65, 0x73, + 0x29, 0x3a, 0x6e, 0x75, 0x6c, 0x6c, 0x2c, 0x6f, 0x2c, 0x21, 0x5f, 0x26, + 0x26, 0x65, 0x3f, 0x65, 0x3a, 0x69, 0x3f, 0x69, 0x2e, 0x5f, 0x5f, 0x65, + 0x3a, 0x6e, 0x2e, 0x66, 0x69, 0x72, 0x73, 0x74, 0x43, 0x68, 0x69, 0x6c, + 0x64, 0x2c, 0x5f, 0x2c, 0x72, 0x29, 0x2c, 0x69, 0x74, 0x28, 0x6f, 0x2c, + 0x74, 0x2c, 0x72, 0x29, 0x7d, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, + 0x6e, 0x20, 0x6c, 0x74, 0x28, 0x74, 0x2c, 0x6e, 0x29, 0x7b, 0x73, 0x74, + 0x28, 0x74, 0x2c, 0x6e, 0x2c, 0x6c, 0x74, 0x29, 0x7d, 0x66, 0x75, 0x6e, + 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x63, 0x74, 0x28, 0x74, 0x2c, 0x6e, + 0x2c, 0x65, 0x29, 0x7b, 0x76, 0x61, 0x72, 0x20, 0x5f, 0x2c, 0x69, 0x2c, + 0x6f, 0x2c, 0x72, 0x2c, 0x75, 0x3d, 0x4d, 0x28, 0x7b, 0x7d, 0x2c, 0x74, + 0x2e, 0x70, 0x72, 0x6f, 0x70, 0x73, 0x29, 0x3b, 0x66, 0x6f, 0x72, 0x28, + 0x6f, 0x20, 0x69, 0x6e, 0x20, 0x74, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x26, + 0x26, 0x74, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x2e, 0x64, 0x65, 0x66, 0x61, + 0x75, 0x6c, 0x74, 0x50, 0x72, 0x6f, 0x70, 0x73, 0x26, 0x26, 0x28, 0x72, + 0x3d, 0x74, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x2e, 0x64, 0x65, 0x66, 0x61, + 0x75, 0x6c, 0x74, 0x50, 0x72, 0x6f, 0x70, 0x73, 0x29, 0x2c, 0x6e, 0x29, + 0x22, 0x6b, 0x65, 0x79, 0x22, 0x3d, 0x3d, 0x6f, 0x3f, 0x5f, 0x3d, 0x6e, + 0x5b, 0x6f, 0x5d, 0x3a, 0x22, 0x72, 0x65, 0x66, 0x22, 0x3d, 0x3d, 0x6f, + 0x3f, 0x69, 0x3d, 0x6e, 0x5b, 0x6f, 0x5d, 0x3a, 0x75, 0x5b, 0x6f, 0x5d, + 0x3d, 0x76, 0x6f, 0x69, 0x64, 0x20, 0x30, 0x3d, 0x3d, 0x3d, 0x6e, 0x5b, + 0x6f, 0x5d, 0x26, 0x26, 0x76, 0x6f, 0x69, 0x64, 0x20, 0x30, 0x21, 0x3d, + 0x3d, 0x72, 0x3f, 0x72, 0x5b, 0x6f, 0x5d, 0x3a, 0x6e, 0x5b, 0x6f, 0x5d, + 0x3b, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x61, 0x72, 0x67, 0x75, + 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x2e, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, + 0x3e, 0x32, 0x26, 0x26, 0x28, 0x75, 0x2e, 0x63, 0x68, 0x69, 0x6c, 0x64, + 0x72, 0x65, 0x6e, 0x3d, 0x61, 0x72, 0x67, 0x75, 0x6d, 0x65, 0x6e, 0x74, + 0x73, 0x2e, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x3e, 0x33, 0x3f, 0x78, + 0x2e, 0x63, 0x61, 0x6c, 0x6c, 0x28, 0x61, 0x72, 0x67, 0x75, 0x6d, 0x65, + 0x6e, 0x74, 0x73, 0x2c, 0x32, 0x29, 0x3a, 0x65, 0x29, 0x2c, 0x4f, 0x28, + 0x74, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x2c, 0x75, 0x2c, 0x5f, 0x7c, 0x7c, + 0x74, 0x2e, 0x6b, 0x65, 0x79, 0x2c, 0x69, 0x7c, 0x7c, 0x74, 0x2e, 0x72, + 0x65, 0x66, 0x2c, 0x6e, 0x75, 0x6c, 0x6c, 0x29, 0x7d, 0x66, 0x75, 0x6e, + 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x68, 0x74, 0x28, 0x74, 0x2c, 0x6e, + 0x29, 0x7b, 0x76, 0x61, 0x72, 0x20, 0x65, 0x3d, 0x7b, 0x5f, 0x5f, 0x63, + 0x3a, 0x6e, 0x3d, 0x22, 0x5f, 0x5f, 0x63, 0x43, 0x22, 0x2b, 0x44, 0x2b, + 0x2b, 0x2c, 0x5f, 0x5f, 0x3a, 0x74, 0x2c, 0x43, 0x6f, 0x6e, 0x73, 0x75, + 0x6d, 0x65, 0x72, 0x3a, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, + 0x28, 0x74, 0x2c, 0x6e, 0x29, 0x7b, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, + 0x20, 0x74, 0x2e, 0x63, 0x68, 0x69, 0x6c, 0x64, 0x72, 0x65, 0x6e, 0x28, + 0x6e, 0x29, 0x7d, 0x2c, 0x50, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, + 0x3a, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x74, 0x29, + 0x7b, 0x76, 0x61, 0x72, 0x20, 0x65, 0x2c, 0x5f, 0x3b, 0x72, 0x65, 0x74, + 0x75, 0x72, 0x6e, 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x67, 0x65, 0x74, + 0x43, 0x68, 0x69, 0x6c, 0x64, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, + 0x7c, 0x7c, 0x28, 0x65, 0x3d, 0x5b, 0x5d, 0x2c, 0x28, 0x5f, 0x3d, 0x7b, + 0x7d, 0x29, 0x5b, 0x6e, 0x5d, 0x3d, 0x74, 0x68, 0x69, 0x73, 0x2c, 0x74, + 0x68, 0x69, 0x73, 0x2e, 0x67, 0x65, 0x74, 0x43, 0x68, 0x69, 0x6c, 0x64, + 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x3d, 0x66, 0x75, 0x6e, 0x63, + 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x29, 0x7b, 0x72, 0x65, 0x74, 0x75, 0x72, + 0x6e, 0x20, 0x5f, 0x7d, 0x2c, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x73, 0x68, + 0x6f, 0x75, 0x6c, 0x64, 0x43, 0x6f, 0x6d, 0x70, 0x6f, 0x6e, 0x65, 0x6e, + 0x74, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x3d, 0x66, 0x75, 0x6e, 0x63, + 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x74, 0x29, 0x7b, 0x74, 0x68, 0x69, 0x73, + 0x2e, 0x70, 0x72, 0x6f, 0x70, 0x73, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, + 0x21, 0x3d, 0x3d, 0x74, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x26, 0x26, + 0x65, 0x2e, 0x73, 0x6f, 0x6d, 0x65, 0x28, 0x28, 0x66, 0x75, 0x6e, 0x63, + 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x74, 0x29, 0x7b, 0x74, 0x2e, 0x5f, 0x5f, + 0x65, 0x3d, 0x21, 0x30, 0x2c, 0x47, 0x28, 0x74, 0x29, 0x7d, 0x29, 0x29, + 0x7d, 0x2c, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x73, 0x75, 0x62, 0x3d, 0x66, + 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x74, 0x29, 0x7b, 0x65, + 0x2e, 0x70, 0x75, 0x73, 0x68, 0x28, 0x74, 0x29, 0x3b, 0x76, 0x61, 0x72, + 0x20, 0x6e, 0x3d, 0x74, 0x2e, 0x63, 0x6f, 0x6d, 0x70, 0x6f, 0x6e, 0x65, + 0x6e, 0x74, 0x57, 0x69, 0x6c, 0x6c, 0x55, 0x6e, 0x6d, 0x6f, 0x75, 0x6e, + 0x74, 0x3b, 0x74, 0x2e, 0x63, 0x6f, 0x6d, 0x70, 0x6f, 0x6e, 0x65, 0x6e, + 0x74, 0x57, 0x69, 0x6c, 0x6c, 0x55, 0x6e, 0x6d, 0x6f, 0x75, 0x6e, 0x74, + 0x3d, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x29, 0x7b, + 0x65, 0x2e, 0x73, 0x70, 0x6c, 0x69, 0x63, 0x65, 0x28, 0x65, 0x2e, 0x69, + 0x6e, 0x64, 0x65, 0x78, 0x4f, 0x66, 0x28, 0x74, 0x29, 0x2c, 0x31, 0x29, + 0x2c, 0x6e, 0x26, 0x26, 0x6e, 0x2e, 0x63, 0x61, 0x6c, 0x6c, 0x28, 0x74, + 0x29, 0x7d, 0x7d, 0x29, 0x2c, 0x74, 0x2e, 0x63, 0x68, 0x69, 0x6c, 0x64, + 0x72, 0x65, 0x6e, 0x7d, 0x7d, 0x3b, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, + 0x20, 0x65, 0x2e, 0x50, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x2e, + 0x5f, 0x5f, 0x3d, 0x65, 0x2e, 0x43, 0x6f, 0x6e, 0x73, 0x75, 0x6d, 0x65, + 0x72, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x54, 0x79, 0x70, + 0x65, 0x3d, 0x65, 0x7d, 0x78, 0x3d, 0x56, 0x2e, 0x73, 0x6c, 0x69, 0x63, + 0x65, 0x2c, 0x43, 0x3d, 0x7b, 0x5f, 0x5f, 0x65, 0x3a, 0x66, 0x75, 0x6e, + 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x74, 0x2c, 0x6e, 0x2c, 0x65, 0x2c, + 0x5f, 0x29, 0x7b, 0x66, 0x6f, 0x72, 0x28, 0x76, 0x61, 0x72, 0x20, 0x69, + 0x2c, 0x6f, 0x2c, 0x72, 0x3b, 0x6e, 0x3d, 0x6e, 0x2e, 0x5f, 0x5f, 0x3b, + 0x29, 0x69, 0x66, 0x28, 0x28, 0x69, 0x3d, 0x6e, 0x2e, 0x5f, 0x5f, 0x63, + 0x29, 0x26, 0x26, 0x21, 0x69, 0x2e, 0x5f, 0x5f, 0x29, 0x74, 0x72, 0x79, + 0x7b, 0x69, 0x66, 0x28, 0x28, 0x6f, 0x3d, 0x69, 0x2e, 0x63, 0x6f, 0x6e, + 0x73, 0x74, 0x72, 0x75, 0x63, 0x74, 0x6f, 0x72, 0x29, 0x26, 0x26, 0x6e, + 0x75, 0x6c, 0x6c, 0x21, 0x3d, 0x6f, 0x2e, 0x67, 0x65, 0x74, 0x44, 0x65, + 0x72, 0x69, 0x76, 0x65, 0x64, 0x53, 0x74, 0x61, 0x74, 0x65, 0x46, 0x72, + 0x6f, 0x6d, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x26, 0x26, 0x28, 0x69, 0x2e, + 0x73, 0x65, 0x74, 0x53, 0x74, 0x61, 0x74, 0x65, 0x28, 0x6f, 0x2e, 0x67, + 0x65, 0x74, 0x44, 0x65, 0x72, 0x69, 0x76, 0x65, 0x64, 0x53, 0x74, 0x61, + 0x74, 0x65, 0x46, 0x72, 0x6f, 0x6d, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x28, + 0x74, 0x29, 0x29, 0x2c, 0x72, 0x3d, 0x69, 0x2e, 0x5f, 0x5f, 0x64, 0x29, + 0x2c, 0x6e, 0x75, 0x6c, 0x6c, 0x21, 0x3d, 0x69, 0x2e, 0x63, 0x6f, 0x6d, + 0x70, 0x6f, 0x6e, 0x65, 0x6e, 0x74, 0x44, 0x69, 0x64, 0x43, 0x61, 0x74, + 0x63, 0x68, 0x26, 0x26, 0x28, 0x69, 0x2e, 0x63, 0x6f, 0x6d, 0x70, 0x6f, + 0x6e, 0x65, 0x6e, 0x74, 0x44, 0x69, 0x64, 0x43, 0x61, 0x74, 0x63, 0x68, + 0x28, 0x74, 0x2c, 0x5f, 0x7c, 0x7c, 0x7b, 0x7d, 0x29, 0x2c, 0x72, 0x3d, + 0x69, 0x2e, 0x5f, 0x5f, 0x64, 0x29, 0x2c, 0x72, 0x29, 0x72, 0x65, 0x74, + 0x75, 0x72, 0x6e, 0x20, 0x69, 0x2e, 0x5f, 0x5f, 0x45, 0x3d, 0x69, 0x7d, + 0x63, 0x61, 0x74, 0x63, 0x68, 0x28, 0x6e, 0x29, 0x7b, 0x74, 0x3d, 0x6e, + 0x7d, 0x74, 0x68, 0x72, 0x6f, 0x77, 0x20, 0x74, 0x7d, 0x7d, 0x2c, 0x45, + 0x3d, 0x30, 0x2c, 0x55, 0x3d, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, + 0x6e, 0x28, 0x74, 0x29, 0x7b, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, + 0x6e, 0x75, 0x6c, 0x6c, 0x21, 0x3d, 0x74, 0x26, 0x26, 0x6e, 0x75, 0x6c, + 0x6c, 0x3d, 0x3d, 0x74, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x72, 0x75, + 0x63, 0x74, 0x6f, 0x72, 0x7d, 0x2c, 0x49, 0x2e, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x74, 0x79, 0x70, 0x65, 0x2e, 0x73, 0x65, 0x74, 0x53, 0x74, 0x61, + 0x74, 0x65, 0x3d, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x28, + 0x74, 0x2c, 0x6e, 0x29, 0x7b, 0x76, 0x61, 0x72, 0x20, 0x65, 0x3b, 0x65, + 0x3d, 0x6e, 0x75, 0x6c, 0x6c, 0x21, 0x3d, 0x74, 0x68, 0x69, 0x73, 0x2e, + 0x5f, 0x5f, 0x73, 0x26, 0x26, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x5f, 0x5f, + 0x73, 0x21, 0x3d, 0x3d, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x73, 0x74, 0x61, + 0x74, 0x65, 0x3f, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x5f, 0x5f, 0x73, 0x3a, + 0x74, 0x68, 0x69, 0x73, 0x2e, 0x5f, 0x5f, 0x73, 0x3d, 0x4d, 0x28, 0x7b, + 0x7d, 0x2c, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x73, 0x74, 0x61, 0x74, 0x65, + 0x29, 0x2c, 0x22, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x22, + 0x3d, 0x3d, 0x74, 0x79, 0x70, 0x65, 0x6f, 0x66, 0x20, 0x74, 0x26, 0x26, + 0x28, 0x74, 0x3d, 0x74, 0x28, 0x4d, 0x28, 0x7b, 0x7d, 0x2c, 0x65, 0x29, + 0x2c, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x70, 0x73, 0x29, + 0x29, 0x2c, 0x74, 0x26, 0x26, 0x4d, 0x28, 0x65, 0x2c, 0x74, 0x29, 0x2c, + 0x6e, 0x75, 0x6c, 0x6c, 0x21, 0x3d, 0x74, 0x26, 0x26, 0x74, 0x68, 0x69, + 0x73, 0x2e, 0x5f, 0x5f, 0x76, 0x26, 0x26, 0x28, 0x6e, 0x26, 0x26, 0x74, + 0x68, 0x69, 0x73, 0x2e, 0x5f, 0x73, 0x62, 0x2e, 0x70, 0x75, 0x73, 0x68, + 0x28, 0x6e, 0x29, 0x2c, 0x47, 0x28, 0x74, 0x68, 0x69, 0x73, 0x29, 0x29, + 0x7d, 0x2c, 0x49, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x74, 0x79, 0x70, + 0x65, 0x2e, 0x66, 0x6f, 0x72, 0x63, 0x65, 0x55, 0x70, 0x64, 0x61, 0x74, + 0x65, 0x3d, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x74, + 0x29, 0x7b, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x5f, 0x5f, 0x76, 0x26, 0x26, + 0x28, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x5f, 0x5f, 0x65, 0x3d, 0x21, 0x30, + 0x2c, 0x74, 0x26, 0x26, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x5f, 0x5f, 0x68, + 0x2e, 0x70, 0x75, 0x73, 0x68, 0x28, 0x74, 0x29, 0x2c, 0x47, 0x28, 0x74, + 0x68, 0x69, 0x73, 0x29, 0x29, 0x7d, 0x2c, 0x49, 0x2e, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x74, 0x79, 0x70, 0x65, 0x2e, 0x72, 0x65, 0x6e, 0x64, 0x65, + 0x72, 0x3d, 0x6a, 0x2c, 0x48, 0x3d, 0x5b, 0x5d, 0x2c, 0x4e, 0x3d, 0x22, + 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x3d, 0x3d, 0x74, + 0x79, 0x70, 0x65, 0x6f, 0x66, 0x20, 0x50, 0x72, 0x6f, 0x6d, 0x69, 0x73, + 0x65, 0x3f, 0x50, 0x72, 0x6f, 0x6d, 0x69, 0x73, 0x65, 0x2e, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x74, 0x79, 0x70, 0x65, 0x2e, 0x74, 0x68, 0x65, 0x6e, + 0x2e, 0x62, 0x69, 0x6e, 0x64, 0x28, 0x50, 0x72, 0x6f, 0x6d, 0x69, 0x73, + 0x65, 0x2e, 0x72, 0x65, 0x73, 0x6f, 0x6c, 0x76, 0x65, 0x28, 0x29, 0x29, + 0x3a, 0x73, 0x65, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x2c, + 0x24, 0x3d, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x74, + 0x2c, 0x6e, 0x29, 0x7b, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x74, + 0x2e, 0x5f, 0x5f, 0x76, 0x2e, 0x5f, 0x5f, 0x62, 0x2d, 0x6e, 0x2e, 0x5f, + 0x5f, 0x76, 0x2e, 0x5f, 0x5f, 0x62, 0x7d, 0x2c, 0x7a, 0x2e, 0x5f, 0x5f, + 0x72, 0x3d, 0x30, 0x2c, 0x44, 0x3d, 0x30, 0x3b, 0x76, 0x61, 0x72, 0x20, + 0x61, 0x74, 0x2c, 0x70, 0x74, 0x2c, 0x64, 0x74, 0x2c, 0x76, 0x74, 0x2c, + 0x79, 0x74, 0x3d, 0x30, 0x2c, 0x6d, 0x74, 0x3d, 0x5b, 0x5d, 0x2c, 0x67, + 0x74, 0x3d, 0x5b, 0x5d, 0x2c, 0x62, 0x74, 0x3d, 0x43, 0x2e, 0x5f, 0x5f, + 0x62, 0x2c, 0x6b, 0x74, 0x3d, 0x43, 0x2e, 0x5f, 0x5f, 0x72, 0x2c, 0x53, + 0x74, 0x3d, 0x43, 0x2e, 0x64, 0x69, 0x66, 0x66, 0x65, 0x64, 0x2c, 0x77, + 0x74, 0x3d, 0x43, 0x2e, 0x5f, 0x5f, 0x63, 0x2c, 0x78, 0x74, 0x3d, 0x43, + 0x2e, 0x75, 0x6e, 0x6d, 0x6f, 0x75, 0x6e, 0x74, 0x3b, 0x66, 0x75, 0x6e, + 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x43, 0x74, 0x28, 0x74, 0x2c, 0x6e, + 0x29, 0x7b, 0x43, 0x2e, 0x5f, 0x5f, 0x68, 0x26, 0x26, 0x43, 0x2e, 0x5f, + 0x5f, 0x68, 0x28, 0x70, 0x74, 0x2c, 0x74, 0x2c, 0x79, 0x74, 0x7c, 0x7c, + 0x6e, 0x29, 0x2c, 0x79, 0x74, 0x3d, 0x30, 0x3b, 0x76, 0x61, 0x72, 0x20, + 0x65, 0x3d, 0x70, 0x74, 0x2e, 0x5f, 0x5f, 0x48, 0x7c, 0x7c, 0x28, 0x70, + 0x74, 0x2e, 0x5f, 0x5f, 0x48, 0x3d, 0x7b, 0x5f, 0x5f, 0x3a, 0x5b, 0x5d, + 0x2c, 0x5f, 0x5f, 0x68, 0x3a, 0x5b, 0x5d, 0x7d, 0x29, 0x3b, 0x72, 0x65, + 0x74, 0x75, 0x72, 0x6e, 0x20, 0x74, 0x3e, 0x3d, 0x65, 0x2e, 0x5f, 0x5f, + 0x2e, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x26, 0x26, 0x65, 0x2e, 0x5f, + 0x5f, 0x2e, 0x70, 0x75, 0x73, 0x68, 0x28, 0x7b, 0x5f, 0x5f, 0x56, 0x3a, + 0x67, 0x74, 0x7d, 0x29, 0x2c, 0x65, 0x2e, 0x5f, 0x5f, 0x5b, 0x74, 0x5d, + 0x7d, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x45, 0x74, + 0x28, 0x74, 0x29, 0x7b, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x79, + 0x74, 0x3d, 0x31, 0x2c, 0x55, 0x74, 0x28, 0x71, 0x74, 0x2c, 0x74, 0x29, + 0x7d, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x55, 0x74, + 0x28, 0x74, 0x2c, 0x6e, 0x2c, 0x65, 0x29, 0x7b, 0x76, 0x61, 0x72, 0x20, + 0x5f, 0x3d, 0x43, 0x74, 0x28, 0x61, 0x74, 0x2b, 0x2b, 0x2c, 0x32, 0x29, + 0x3b, 0x69, 0x66, 0x28, 0x5f, 0x2e, 0x74, 0x3d, 0x74, 0x2c, 0x21, 0x5f, + 0x2e, 0x5f, 0x5f, 0x63, 0x26, 0x26, 0x28, 0x5f, 0x2e, 0x5f, 0x5f, 0x3d, + 0x5b, 0x65, 0x3f, 0x65, 0x28, 0x6e, 0x29, 0x3a, 0x71, 0x74, 0x28, 0x76, + 0x6f, 0x69, 0x64, 0x20, 0x30, 0x2c, 0x6e, 0x29, 0x2c, 0x66, 0x75, 0x6e, + 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x74, 0x29, 0x7b, 0x76, 0x61, 0x72, + 0x20, 0x6e, 0x3d, 0x5f, 0x2e, 0x5f, 0x5f, 0x4e, 0x3f, 0x5f, 0x2e, 0x5f, + 0x5f, 0x4e, 0x5b, 0x30, 0x5d, 0x3a, 0x5f, 0x2e, 0x5f, 0x5f, 0x5b, 0x30, + 0x5d, 0x2c, 0x65, 0x3d, 0x5f, 0x2e, 0x74, 0x28, 0x6e, 0x2c, 0x74, 0x29, + 0x3b, 0x6e, 0x21, 0x3d, 0x3d, 0x65, 0x26, 0x26, 0x28, 0x5f, 0x2e, 0x5f, + 0x5f, 0x4e, 0x3d, 0x5b, 0x65, 0x2c, 0x5f, 0x2e, 0x5f, 0x5f, 0x5b, 0x31, + 0x5d, 0x5d, 0x2c, 0x5f, 0x2e, 0x5f, 0x5f, 0x63, 0x2e, 0x73, 0x65, 0x74, + 0x53, 0x74, 0x61, 0x74, 0x65, 0x28, 0x7b, 0x7d, 0x29, 0x29, 0x7d, 0x5d, + 0x2c, 0x5f, 0x2e, 0x5f, 0x5f, 0x63, 0x3d, 0x70, 0x74, 0x2c, 0x21, 0x70, + 0x74, 0x2e, 0x75, 0x29, 0x29, 0x7b, 0x76, 0x61, 0x72, 0x20, 0x69, 0x3d, + 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x74, 0x2c, 0x6e, + 0x2c, 0x65, 0x29, 0x7b, 0x69, 0x66, 0x28, 0x21, 0x5f, 0x2e, 0x5f, 0x5f, + 0x63, 0x2e, 0x5f, 0x5f, 0x48, 0x29, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, + 0x21, 0x30, 0x3b, 0x76, 0x61, 0x72, 0x20, 0x69, 0x3d, 0x5f, 0x2e, 0x5f, + 0x5f, 0x63, 0x2e, 0x5f, 0x5f, 0x48, 0x2e, 0x5f, 0x5f, 0x2e, 0x66, 0x69, + 0x6c, 0x74, 0x65, 0x72, 0x28, 0x28, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, + 0x6f, 0x6e, 0x28, 0x74, 0x29, 0x7b, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, + 0x20, 0x74, 0x2e, 0x5f, 0x5f, 0x63, 0x7d, 0x29, 0x29, 0x3b, 0x69, 0x66, + 0x28, 0x69, 0x2e, 0x65, 0x76, 0x65, 0x72, 0x79, 0x28, 0x28, 0x66, 0x75, + 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x74, 0x29, 0x7b, 0x72, 0x65, + 0x74, 0x75, 0x72, 0x6e, 0x21, 0x74, 0x2e, 0x5f, 0x5f, 0x4e, 0x7d, 0x29, + 0x29, 0x29, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x21, 0x6f, 0x7c, 0x7c, + 0x6f, 0x2e, 0x63, 0x61, 0x6c, 0x6c, 0x28, 0x74, 0x68, 0x69, 0x73, 0x2c, + 0x74, 0x2c, 0x6e, 0x2c, 0x65, 0x29, 0x3b, 0x76, 0x61, 0x72, 0x20, 0x72, + 0x3d, 0x21, 0x31, 0x3b, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x69, + 0x2e, 0x66, 0x6f, 0x72, 0x45, 0x61, 0x63, 0x68, 0x28, 0x28, 0x66, 0x75, + 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x74, 0x29, 0x7b, 0x69, 0x66, + 0x28, 0x74, 0x2e, 0x5f, 0x5f, 0x4e, 0x29, 0x7b, 0x76, 0x61, 0x72, 0x20, + 0x6e, 0x3d, 0x74, 0x2e, 0x5f, 0x5f, 0x5b, 0x30, 0x5d, 0x3b, 0x74, 0x2e, + 0x5f, 0x5f, 0x3d, 0x74, 0x2e, 0x5f, 0x5f, 0x4e, 0x2c, 0x74, 0x2e, 0x5f, + 0x5f, 0x4e, 0x3d, 0x76, 0x6f, 0x69, 0x64, 0x20, 0x30, 0x2c, 0x6e, 0x21, + 0x3d, 0x3d, 0x74, 0x2e, 0x5f, 0x5f, 0x5b, 0x30, 0x5d, 0x26, 0x26, 0x28, + 0x72, 0x3d, 0x21, 0x30, 0x29, 0x7d, 0x7d, 0x29, 0x29, 0x2c, 0x21, 0x28, + 0x21, 0x72, 0x26, 0x26, 0x5f, 0x2e, 0x5f, 0x5f, 0x63, 0x2e, 0x70, 0x72, + 0x6f, 0x70, 0x73, 0x3d, 0x3d, 0x3d, 0x74, 0x29, 0x26, 0x26, 0x28, 0x21, + 0x6f, 0x7c, 0x7c, 0x6f, 0x2e, 0x63, 0x61, 0x6c, 0x6c, 0x28, 0x74, 0x68, + 0x69, 0x73, 0x2c, 0x74, 0x2c, 0x6e, 0x2c, 0x65, 0x29, 0x29, 0x7d, 0x3b, + 0x70, 0x74, 0x2e, 0x75, 0x3d, 0x21, 0x30, 0x3b, 0x76, 0x61, 0x72, 0x20, + 0x6f, 0x3d, 0x70, 0x74, 0x2e, 0x73, 0x68, 0x6f, 0x75, 0x6c, 0x64, 0x43, + 0x6f, 0x6d, 0x70, 0x6f, 0x6e, 0x65, 0x6e, 0x74, 0x55, 0x70, 0x64, 0x61, + 0x74, 0x65, 0x2c, 0x72, 0x3d, 0x70, 0x74, 0x2e, 0x63, 0x6f, 0x6d, 0x70, + 0x6f, 0x6e, 0x65, 0x6e, 0x74, 0x57, 0x69, 0x6c, 0x6c, 0x55, 0x70, 0x64, + 0x61, 0x74, 0x65, 0x3b, 0x70, 0x74, 0x2e, 0x63, 0x6f, 0x6d, 0x70, 0x6f, + 0x6e, 0x65, 0x6e, 0x74, 0x57, 0x69, 0x6c, 0x6c, 0x55, 0x70, 0x64, 0x61, + 0x74, 0x65, 0x3d, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x28, + 0x74, 0x2c, 0x6e, 0x2c, 0x65, 0x29, 0x7b, 0x69, 0x66, 0x28, 0x74, 0x68, + 0x69, 0x73, 0x2e, 0x5f, 0x5f, 0x65, 0x29, 0x7b, 0x76, 0x61, 0x72, 0x20, + 0x5f, 0x3d, 0x6f, 0x3b, 0x6f, 0x3d, 0x76, 0x6f, 0x69, 0x64, 0x20, 0x30, + 0x2c, 0x69, 0x28, 0x74, 0x2c, 0x6e, 0x2c, 0x65, 0x29, 0x2c, 0x6f, 0x3d, + 0x5f, 0x7d, 0x72, 0x26, 0x26, 0x72, 0x2e, 0x63, 0x61, 0x6c, 0x6c, 0x28, + 0x74, 0x68, 0x69, 0x73, 0x2c, 0x74, 0x2c, 0x6e, 0x2c, 0x65, 0x29, 0x7d, + 0x2c, 0x70, 0x74, 0x2e, 0x73, 0x68, 0x6f, 0x75, 0x6c, 0x64, 0x43, 0x6f, + 0x6d, 0x70, 0x6f, 0x6e, 0x65, 0x6e, 0x74, 0x55, 0x70, 0x64, 0x61, 0x74, + 0x65, 0x3d, 0x69, 0x7d, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x5f, + 0x2e, 0x5f, 0x5f, 0x4e, 0x7c, 0x7c, 0x5f, 0x2e, 0x5f, 0x5f, 0x7d, 0x66, + 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x48, 0x74, 0x28, 0x74, + 0x2c, 0x6e, 0x29, 0x7b, 0x76, 0x61, 0x72, 0x20, 0x65, 0x3d, 0x43, 0x74, + 0x28, 0x61, 0x74, 0x2b, 0x2b, 0x2c, 0x33, 0x29, 0x3b, 0x21, 0x43, 0x2e, + 0x5f, 0x5f, 0x73, 0x26, 0x26, 0x49, 0x74, 0x28, 0x65, 0x2e, 0x5f, 0x5f, + 0x48, 0x2c, 0x6e, 0x29, 0x26, 0x26, 0x28, 0x65, 0x2e, 0x5f, 0x5f, 0x3d, + 0x74, 0x2c, 0x65, 0x2e, 0x69, 0x3d, 0x6e, 0x2c, 0x70, 0x74, 0x2e, 0x5f, + 0x5f, 0x48, 0x2e, 0x5f, 0x5f, 0x68, 0x2e, 0x70, 0x75, 0x73, 0x68, 0x28, + 0x65, 0x29, 0x29, 0x7d, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, + 0x20, 0x50, 0x74, 0x28, 0x74, 0x2c, 0x6e, 0x29, 0x7b, 0x76, 0x61, 0x72, + 0x20, 0x65, 0x3d, 0x43, 0x74, 0x28, 0x61, 0x74, 0x2b, 0x2b, 0x2c, 0x34, + 0x29, 0x3b, 0x21, 0x43, 0x2e, 0x5f, 0x5f, 0x73, 0x26, 0x26, 0x49, 0x74, + 0x28, 0x65, 0x2e, 0x5f, 0x5f, 0x48, 0x2c, 0x6e, 0x29, 0x26, 0x26, 0x28, + 0x65, 0x2e, 0x5f, 0x5f, 0x3d, 0x74, 0x2c, 0x65, 0x2e, 0x69, 0x3d, 0x6e, + 0x2c, 0x70, 0x74, 0x2e, 0x5f, 0x5f, 0x68, 0x2e, 0x70, 0x75, 0x73, 0x68, + 0x28, 0x65, 0x29, 0x29, 0x7d, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, + 0x6e, 0x20, 0x4e, 0x74, 0x28, 0x74, 0x29, 0x7b, 0x72, 0x65, 0x74, 0x75, + 0x72, 0x6e, 0x20, 0x79, 0x74, 0x3d, 0x35, 0x2c, 0x44, 0x74, 0x28, 0x28, + 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x29, 0x7b, 0x72, + 0x65, 0x74, 0x75, 0x72, 0x6e, 0x7b, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, + 0x74, 0x3a, 0x74, 0x7d, 0x7d, 0x29, 0x2c, 0x5b, 0x5d, 0x29, 0x7d, 0x66, + 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x24, 0x74, 0x28, 0x74, + 0x2c, 0x6e, 0x2c, 0x65, 0x29, 0x7b, 0x79, 0x74, 0x3d, 0x36, 0x2c, 0x50, + 0x74, 0x28, 0x28, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x28, + 0x29, 0x7b, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x22, 0x66, 0x75, 0x6e, + 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x3d, 0x3d, 0x74, 0x79, 0x70, 0x65, + 0x6f, 0x66, 0x20, 0x74, 0x3f, 0x28, 0x74, 0x28, 0x6e, 0x28, 0x29, 0x29, + 0x2c, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x29, 0x7b, + 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x74, 0x28, 0x6e, 0x75, 0x6c, + 0x6c, 0x29, 0x7d, 0x29, 0x3a, 0x74, 0x3f, 0x28, 0x74, 0x2e, 0x63, 0x75, + 0x72, 0x72, 0x65, 0x6e, 0x74, 0x3d, 0x6e, 0x28, 0x29, 0x2c, 0x66, 0x75, + 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x29, 0x7b, 0x72, 0x65, 0x74, + 0x75, 0x72, 0x6e, 0x20, 0x74, 0x2e, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, + 0x74, 0x3d, 0x6e, 0x75, 0x6c, 0x6c, 0x7d, 0x29, 0x3a, 0x76, 0x6f, 0x69, + 0x64, 0x20, 0x30, 0x7d, 0x29, 0x2c, 0x6e, 0x75, 0x6c, 0x6c, 0x3d, 0x3d, + 0x65, 0x3f, 0x65, 0x3a, 0x65, 0x2e, 0x63, 0x6f, 0x6e, 0x63, 0x61, 0x74, + 0x28, 0x74, 0x29, 0x29, 0x7d, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, + 0x6e, 0x20, 0x44, 0x74, 0x28, 0x74, 0x2c, 0x6e, 0x29, 0x7b, 0x76, 0x61, + 0x72, 0x20, 0x65, 0x3d, 0x43, 0x74, 0x28, 0x61, 0x74, 0x2b, 0x2b, 0x2c, + 0x37, 0x29, 0x3b, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x49, 0x74, + 0x28, 0x65, 0x2e, 0x5f, 0x5f, 0x48, 0x2c, 0x6e, 0x29, 0x3f, 0x28, 0x65, + 0x2e, 0x5f, 0x5f, 0x56, 0x3d, 0x74, 0x28, 0x29, 0x2c, 0x65, 0x2e, 0x69, + 0x3d, 0x6e, 0x2c, 0x65, 0x2e, 0x5f, 0x5f, 0x68, 0x3d, 0x74, 0x2c, 0x65, + 0x2e, 0x5f, 0x5f, 0x56, 0x29, 0x3a, 0x65, 0x2e, 0x5f, 0x5f, 0x7d, 0x66, + 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x54, 0x74, 0x28, 0x74, + 0x2c, 0x6e, 0x29, 0x7b, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x79, + 0x74, 0x3d, 0x38, 0x2c, 0x44, 0x74, 0x28, 0x28, 0x66, 0x75, 0x6e, 0x63, + 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x29, 0x7b, 0x72, 0x65, 0x74, 0x75, 0x72, + 0x6e, 0x20, 0x74, 0x7d, 0x29, 0x2c, 0x6e, 0x29, 0x7d, 0x66, 0x75, 0x6e, + 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x56, 0x74, 0x28, 0x74, 0x29, 0x7b, + 0x76, 0x61, 0x72, 0x20, 0x6e, 0x3d, 0x70, 0x74, 0x2e, 0x63, 0x6f, 0x6e, + 0x74, 0x65, 0x78, 0x74, 0x5b, 0x74, 0x2e, 0x5f, 0x5f, 0x63, 0x5d, 0x2c, + 0x65, 0x3d, 0x43, 0x74, 0x28, 0x61, 0x74, 0x2b, 0x2b, 0x2c, 0x39, 0x29, + 0x3b, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x65, 0x2e, 0x63, 0x3d, + 0x74, 0x2c, 0x6e, 0x3f, 0x28, 0x6e, 0x75, 0x6c, 0x6c, 0x3d, 0x3d, 0x65, + 0x2e, 0x5f, 0x5f, 0x26, 0x26, 0x28, 0x65, 0x2e, 0x5f, 0x5f, 0x3d, 0x21, + 0x30, 0x2c, 0x6e, 0x2e, 0x73, 0x75, 0x62, 0x28, 0x70, 0x74, 0x29, 0x29, + 0x2c, 0x6e, 0x2e, 0x70, 0x72, 0x6f, 0x70, 0x73, 0x2e, 0x76, 0x61, 0x6c, + 0x75, 0x65, 0x29, 0x3a, 0x74, 0x2e, 0x5f, 0x5f, 0x7d, 0x66, 0x75, 0x6e, + 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x41, 0x74, 0x28, 0x74, 0x2c, 0x6e, + 0x29, 0x7b, 0x43, 0x2e, 0x75, 0x73, 0x65, 0x44, 0x65, 0x62, 0x75, 0x67, + 0x56, 0x61, 0x6c, 0x75, 0x65, 0x26, 0x26, 0x43, 0x2e, 0x75, 0x73, 0x65, + 0x44, 0x65, 0x62, 0x75, 0x67, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x28, 0x6e, + 0x3f, 0x6e, 0x28, 0x74, 0x29, 0x3a, 0x74, 0x29, 0x7d, 0x66, 0x75, 0x6e, + 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x46, 0x74, 0x28, 0x74, 0x29, 0x7b, + 0x76, 0x61, 0x72, 0x20, 0x6e, 0x3d, 0x43, 0x74, 0x28, 0x61, 0x74, 0x2b, + 0x2b, 0x2c, 0x31, 0x30, 0x29, 0x2c, 0x65, 0x3d, 0x45, 0x74, 0x28, 0x29, + 0x3b, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x6e, 0x2e, 0x5f, 0x5f, + 0x3d, 0x74, 0x2c, 0x70, 0x74, 0x2e, 0x63, 0x6f, 0x6d, 0x70, 0x6f, 0x6e, + 0x65, 0x6e, 0x74, 0x44, 0x69, 0x64, 0x43, 0x61, 0x74, 0x63, 0x68, 0x7c, + 0x7c, 0x28, 0x70, 0x74, 0x2e, 0x63, 0x6f, 0x6d, 0x70, 0x6f, 0x6e, 0x65, + 0x6e, 0x74, 0x44, 0x69, 0x64, 0x43, 0x61, 0x74, 0x63, 0x68, 0x3d, 0x66, + 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x74, 0x2c, 0x5f, 0x29, + 0x7b, 0x6e, 0x2e, 0x5f, 0x5f, 0x26, 0x26, 0x6e, 0x2e, 0x5f, 0x5f, 0x28, + 0x74, 0x2c, 0x5f, 0x29, 0x2c, 0x65, 0x5b, 0x31, 0x5d, 0x28, 0x74, 0x29, + 0x7d, 0x29, 0x2c, 0x5b, 0x65, 0x5b, 0x30, 0x5d, 0x2c, 0x66, 0x75, 0x6e, + 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x29, 0x7b, 0x65, 0x5b, 0x31, 0x5d, + 0x28, 0x76, 0x6f, 0x69, 0x64, 0x20, 0x30, 0x29, 0x7d, 0x5d, 0x7d, 0x66, + 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x4d, 0x74, 0x28, 0x29, + 0x7b, 0x76, 0x61, 0x72, 0x20, 0x74, 0x3d, 0x43, 0x74, 0x28, 0x61, 0x74, + 0x2b, 0x2b, 0x2c, 0x31, 0x31, 0x29, 0x3b, 0x69, 0x66, 0x28, 0x21, 0x74, + 0x2e, 0x5f, 0x5f, 0x29, 0x7b, 0x66, 0x6f, 0x72, 0x28, 0x76, 0x61, 0x72, + 0x20, 0x6e, 0x3d, 0x70, 0x74, 0x2e, 0x5f, 0x5f, 0x76, 0x3b, 0x6e, 0x75, + 0x6c, 0x6c, 0x21, 0x3d, 0x3d, 0x6e, 0x26, 0x26, 0x21, 0x6e, 0x2e, 0x5f, + 0x5f, 0x6d, 0x26, 0x26, 0x6e, 0x75, 0x6c, 0x6c, 0x21, 0x3d, 0x3d, 0x6e, + 0x2e, 0x5f, 0x5f, 0x3b, 0x29, 0x6e, 0x3d, 0x6e, 0x2e, 0x5f, 0x5f, 0x3b, + 0x76, 0x61, 0x72, 0x20, 0x65, 0x3d, 0x6e, 0x2e, 0x5f, 0x5f, 0x6d, 0x7c, + 0x7c, 0x28, 0x6e, 0x2e, 0x5f, 0x5f, 0x6d, 0x3d, 0x5b, 0x30, 0x2c, 0x30, + 0x5d, 0x29, 0x3b, 0x74, 0x2e, 0x5f, 0x5f, 0x3d, 0x22, 0x50, 0x22, 0x2b, + 0x65, 0x5b, 0x30, 0x5d, 0x2b, 0x22, 0x2d, 0x22, 0x2b, 0x65, 0x5b, 0x31, + 0x5d, 0x2b, 0x2b, 0x7d, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x74, + 0x2e, 0x5f, 0x5f, 0x7d, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, + 0x20, 0x57, 0x74, 0x28, 0x29, 0x7b, 0x66, 0x6f, 0x72, 0x28, 0x76, 0x61, + 0x72, 0x20, 0x74, 0x3b, 0x74, 0x3d, 0x6d, 0x74, 0x2e, 0x73, 0x68, 0x69, + 0x66, 0x74, 0x28, 0x29, 0x3b, 0x29, 0x69, 0x66, 0x28, 0x74, 0x2e, 0x5f, + 0x5f, 0x50, 0x26, 0x26, 0x74, 0x2e, 0x5f, 0x5f, 0x48, 0x29, 0x74, 0x72, + 0x79, 0x7b, 0x74, 0x2e, 0x5f, 0x5f, 0x48, 0x2e, 0x5f, 0x5f, 0x68, 0x2e, + 0x66, 0x6f, 0x72, 0x45, 0x61, 0x63, 0x68, 0x28, 0x52, 0x74, 0x29, 0x2c, + 0x74, 0x2e, 0x5f, 0x5f, 0x48, 0x2e, 0x5f, 0x5f, 0x68, 0x2e, 0x66, 0x6f, + 0x72, 0x45, 0x61, 0x63, 0x68, 0x28, 0x6a, 0x74, 0x29, 0x2c, 0x74, 0x2e, + 0x5f, 0x5f, 0x48, 0x2e, 0x5f, 0x5f, 0x68, 0x3d, 0x5b, 0x5d, 0x7d, 0x63, + 0x61, 0x74, 0x63, 0x68, 0x28, 0x75, 0x29, 0x7b, 0x74, 0x2e, 0x5f, 0x5f, + 0x48, 0x2e, 0x5f, 0x5f, 0x68, 0x3d, 0x5b, 0x5d, 0x2c, 0x43, 0x2e, 0x5f, + 0x5f, 0x65, 0x28, 0x75, 0x2c, 0x74, 0x2e, 0x5f, 0x5f, 0x76, 0x29, 0x7d, + 0x7d, 0x43, 0x2e, 0x5f, 0x5f, 0x62, 0x3d, 0x66, 0x75, 0x6e, 0x63, 0x74, + 0x69, 0x6f, 0x6e, 0x28, 0x74, 0x29, 0x7b, 0x70, 0x74, 0x3d, 0x6e, 0x75, + 0x6c, 0x6c, 0x2c, 0x62, 0x74, 0x26, 0x26, 0x62, 0x74, 0x28, 0x74, 0x29, + 0x7d, 0x2c, 0x43, 0x2e, 0x5f, 0x5f, 0x72, 0x3d, 0x66, 0x75, 0x6e, 0x63, + 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x74, 0x29, 0x7b, 0x6b, 0x74, 0x26, 0x26, + 0x6b, 0x74, 0x28, 0x74, 0x29, 0x2c, 0x61, 0x74, 0x3d, 0x30, 0x3b, 0x76, + 0x61, 0x72, 0x20, 0x6e, 0x3d, 0x28, 0x70, 0x74, 0x3d, 0x74, 0x2e, 0x5f, + 0x5f, 0x63, 0x29, 0x2e, 0x5f, 0x5f, 0x48, 0x3b, 0x6e, 0x26, 0x26, 0x28, + 0x64, 0x74, 0x3d, 0x3d, 0x3d, 0x70, 0x74, 0x3f, 0x28, 0x6e, 0x2e, 0x5f, + 0x5f, 0x68, 0x3d, 0x5b, 0x5d, 0x2c, 0x70, 0x74, 0x2e, 0x5f, 0x5f, 0x68, + 0x3d, 0x5b, 0x5d, 0x2c, 0x6e, 0x2e, 0x5f, 0x5f, 0x2e, 0x66, 0x6f, 0x72, + 0x45, 0x61, 0x63, 0x68, 0x28, 0x28, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, + 0x6f, 0x6e, 0x28, 0x74, 0x29, 0x7b, 0x74, 0x2e, 0x5f, 0x5f, 0x4e, 0x26, + 0x26, 0x28, 0x74, 0x2e, 0x5f, 0x5f, 0x3d, 0x74, 0x2e, 0x5f, 0x5f, 0x4e, + 0x29, 0x2c, 0x74, 0x2e, 0x5f, 0x5f, 0x56, 0x3d, 0x67, 0x74, 0x2c, 0x74, + 0x2e, 0x5f, 0x5f, 0x4e, 0x3d, 0x74, 0x2e, 0x69, 0x3d, 0x76, 0x6f, 0x69, + 0x64, 0x20, 0x30, 0x7d, 0x29, 0x29, 0x29, 0x3a, 0x28, 0x6e, 0x2e, 0x5f, + 0x5f, 0x68, 0x2e, 0x66, 0x6f, 0x72, 0x45, 0x61, 0x63, 0x68, 0x28, 0x52, + 0x74, 0x29, 0x2c, 0x6e, 0x2e, 0x5f, 0x5f, 0x68, 0x2e, 0x66, 0x6f, 0x72, + 0x45, 0x61, 0x63, 0x68, 0x28, 0x6a, 0x74, 0x29, 0x2c, 0x6e, 0x2e, 0x5f, + 0x5f, 0x68, 0x3d, 0x5b, 0x5d, 0x2c, 0x61, 0x74, 0x3d, 0x30, 0x29, 0x29, + 0x2c, 0x64, 0x74, 0x3d, 0x70, 0x74, 0x7d, 0x2c, 0x43, 0x2e, 0x64, 0x69, + 0x66, 0x66, 0x65, 0x64, 0x3d, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, + 0x6e, 0x28, 0x74, 0x29, 0x7b, 0x53, 0x74, 0x26, 0x26, 0x53, 0x74, 0x28, + 0x74, 0x29, 0x3b, 0x76, 0x61, 0x72, 0x20, 0x6e, 0x3d, 0x74, 0x2e, 0x5f, + 0x5f, 0x63, 0x3b, 0x6e, 0x26, 0x26, 0x6e, 0x2e, 0x5f, 0x5f, 0x48, 0x26, + 0x26, 0x28, 0x6e, 0x2e, 0x5f, 0x5f, 0x48, 0x2e, 0x5f, 0x5f, 0x68, 0x2e, + 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x26, 0x26, 0x28, 0x31, 0x21, 0x3d, + 0x3d, 0x6d, 0x74, 0x2e, 0x70, 0x75, 0x73, 0x68, 0x28, 0x6e, 0x29, 0x26, + 0x26, 0x76, 0x74, 0x3d, 0x3d, 0x3d, 0x43, 0x2e, 0x72, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x41, 0x6e, 0x69, 0x6d, 0x61, 0x74, 0x69, 0x6f, 0x6e, + 0x46, 0x72, 0x61, 0x6d, 0x65, 0x7c, 0x7c, 0x28, 0x28, 0x76, 0x74, 0x3d, + 0x43, 0x2e, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x41, 0x6e, 0x69, + 0x6d, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x46, 0x72, 0x61, 0x6d, 0x65, 0x29, + 0x7c, 0x7c, 0x4f, 0x74, 0x29, 0x28, 0x57, 0x74, 0x29, 0x29, 0x2c, 0x6e, + 0x2e, 0x5f, 0x5f, 0x48, 0x2e, 0x5f, 0x5f, 0x2e, 0x66, 0x6f, 0x72, 0x45, + 0x61, 0x63, 0x68, 0x28, 0x28, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, + 0x6e, 0x28, 0x74, 0x29, 0x7b, 0x74, 0x2e, 0x69, 0x26, 0x26, 0x28, 0x74, + 0x2e, 0x5f, 0x5f, 0x48, 0x3d, 0x74, 0x2e, 0x69, 0x29, 0x2c, 0x74, 0x2e, + 0x5f, 0x5f, 0x56, 0x21, 0x3d, 0x3d, 0x67, 0x74, 0x26, 0x26, 0x28, 0x74, + 0x2e, 0x5f, 0x5f, 0x3d, 0x74, 0x2e, 0x5f, 0x5f, 0x56, 0x29, 0x2c, 0x74, + 0x2e, 0x69, 0x3d, 0x76, 0x6f, 0x69, 0x64, 0x20, 0x30, 0x2c, 0x74, 0x2e, + 0x5f, 0x5f, 0x56, 0x3d, 0x67, 0x74, 0x7d, 0x29, 0x29, 0x29, 0x2c, 0x64, + 0x74, 0x3d, 0x70, 0x74, 0x3d, 0x6e, 0x75, 0x6c, 0x6c, 0x7d, 0x2c, 0x43, + 0x2e, 0x5f, 0x5f, 0x63, 0x3d, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, + 0x6e, 0x28, 0x74, 0x2c, 0x6e, 0x29, 0x7b, 0x6e, 0x2e, 0x73, 0x6f, 0x6d, + 0x65, 0x28, 0x28, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x28, + 0x74, 0x29, 0x7b, 0x74, 0x72, 0x79, 0x7b, 0x74, 0x2e, 0x5f, 0x5f, 0x68, + 0x2e, 0x66, 0x6f, 0x72, 0x45, 0x61, 0x63, 0x68, 0x28, 0x52, 0x74, 0x29, + 0x2c, 0x74, 0x2e, 0x5f, 0x5f, 0x68, 0x3d, 0x74, 0x2e, 0x5f, 0x5f, 0x68, + 0x2e, 0x66, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x28, 0x28, 0x66, 0x75, 0x6e, + 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x74, 0x29, 0x7b, 0x72, 0x65, 0x74, + 0x75, 0x72, 0x6e, 0x21, 0x74, 0x2e, 0x5f, 0x5f, 0x7c, 0x7c, 0x6a, 0x74, + 0x28, 0x74, 0x29, 0x7d, 0x29, 0x29, 0x7d, 0x63, 0x61, 0x74, 0x63, 0x68, + 0x28, 0x6c, 0x29, 0x7b, 0x6e, 0x2e, 0x73, 0x6f, 0x6d, 0x65, 0x28, 0x28, + 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x74, 0x29, 0x7b, + 0x74, 0x2e, 0x5f, 0x5f, 0x68, 0x26, 0x26, 0x28, 0x74, 0x2e, 0x5f, 0x5f, + 0x68, 0x3d, 0x5b, 0x5d, 0x29, 0x7d, 0x29, 0x29, 0x2c, 0x6e, 0x3d, 0x5b, + 0x5d, 0x2c, 0x43, 0x2e, 0x5f, 0x5f, 0x65, 0x28, 0x6c, 0x2c, 0x74, 0x2e, + 0x5f, 0x5f, 0x76, 0x29, 0x7d, 0x7d, 0x29, 0x29, 0x2c, 0x77, 0x74, 0x26, + 0x26, 0x77, 0x74, 0x28, 0x74, 0x2c, 0x6e, 0x29, 0x7d, 0x2c, 0x43, 0x2e, + 0x75, 0x6e, 0x6d, 0x6f, 0x75, 0x6e, 0x74, 0x3d, 0x66, 0x75, 0x6e, 0x63, + 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x74, 0x29, 0x7b, 0x78, 0x74, 0x26, 0x26, + 0x78, 0x74, 0x28, 0x74, 0x29, 0x3b, 0x76, 0x61, 0x72, 0x20, 0x6e, 0x2c, + 0x65, 0x3d, 0x74, 0x2e, 0x5f, 0x5f, 0x63, 0x3b, 0x65, 0x26, 0x26, 0x65, + 0x2e, 0x5f, 0x5f, 0x48, 0x26, 0x26, 0x28, 0x65, 0x2e, 0x5f, 0x5f, 0x48, + 0x2e, 0x5f, 0x5f, 0x2e, 0x66, 0x6f, 0x72, 0x45, 0x61, 0x63, 0x68, 0x28, + 0x28, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x74, 0x29, + 0x7b, 0x74, 0x72, 0x79, 0x7b, 0x52, 0x74, 0x28, 0x74, 0x29, 0x7d, 0x63, + 0x61, 0x74, 0x63, 0x68, 0x28, 0x74, 0x29, 0x7b, 0x6e, 0x3d, 0x74, 0x7d, + 0x7d, 0x29, 0x29, 0x2c, 0x65, 0x2e, 0x5f, 0x5f, 0x48, 0x3d, 0x76, 0x6f, + 0x69, 0x64, 0x20, 0x30, 0x2c, 0x6e, 0x26, 0x26, 0x43, 0x2e, 0x5f, 0x5f, + 0x65, 0x28, 0x6e, 0x2c, 0x65, 0x2e, 0x5f, 0x5f, 0x76, 0x29, 0x29, 0x7d, + 0x3b, 0x76, 0x61, 0x72, 0x20, 0x4c, 0x74, 0x3d, 0x22, 0x66, 0x75, 0x6e, + 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x3d, 0x3d, 0x74, 0x79, 0x70, 0x65, + 0x6f, 0x66, 0x20, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x41, 0x6e, + 0x69, 0x6d, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x46, 0x72, 0x61, 0x6d, 0x65, + 0x3b, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x4f, 0x74, + 0x28, 0x74, 0x29, 0x7b, 0x76, 0x61, 0x72, 0x20, 0x6e, 0x2c, 0x65, 0x3d, + 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x29, 0x7b, 0x63, + 0x6c, 0x65, 0x61, 0x72, 0x54, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x28, + 0x5f, 0x29, 0x2c, 0x4c, 0x74, 0x26, 0x26, 0x63, 0x61, 0x6e, 0x63, 0x65, + 0x6c, 0x41, 0x6e, 0x69, 0x6d, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x46, 0x72, + 0x61, 0x6d, 0x65, 0x28, 0x6e, 0x29, 0x2c, 0x73, 0x65, 0x74, 0x54, 0x69, + 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x28, 0x74, 0x29, 0x7d, 0x2c, 0x5f, 0x3d, + 0x73, 0x65, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x28, 0x65, + 0x2c, 0x31, 0x30, 0x30, 0x29, 0x3b, 0x4c, 0x74, 0x26, 0x26, 0x28, 0x6e, + 0x3d, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x41, 0x6e, 0x69, 0x6d, + 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x46, 0x72, 0x61, 0x6d, 0x65, 0x28, 0x65, + 0x29, 0x29, 0x7d, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, + 0x52, 0x74, 0x28, 0x74, 0x29, 0x7b, 0x76, 0x61, 0x72, 0x20, 0x6e, 0x3d, + 0x70, 0x74, 0x2c, 0x65, 0x3d, 0x74, 0x2e, 0x5f, 0x5f, 0x63, 0x3b, 0x22, + 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x3d, 0x3d, 0x74, + 0x79, 0x70, 0x65, 0x6f, 0x66, 0x20, 0x65, 0x26, 0x26, 0x28, 0x74, 0x2e, + 0x5f, 0x5f, 0x63, 0x3d, 0x76, 0x6f, 0x69, 0x64, 0x20, 0x30, 0x2c, 0x65, + 0x28, 0x29, 0x29, 0x2c, 0x70, 0x74, 0x3d, 0x6e, 0x7d, 0x66, 0x75, 0x6e, + 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x6a, 0x74, 0x28, 0x74, 0x29, 0x7b, + 0x76, 0x61, 0x72, 0x20, 0x6e, 0x3d, 0x70, 0x74, 0x3b, 0x74, 0x2e, 0x5f, + 0x5f, 0x63, 0x3d, 0x74, 0x2e, 0x5f, 0x5f, 0x28, 0x29, 0x2c, 0x70, 0x74, + 0x3d, 0x6e, 0x7d, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, + 0x49, 0x74, 0x28, 0x74, 0x2c, 0x6e, 0x29, 0x7b, 0x72, 0x65, 0x74, 0x75, + 0x72, 0x6e, 0x21, 0x74, 0x7c, 0x7c, 0x74, 0x2e, 0x6c, 0x65, 0x6e, 0x67, + 0x74, 0x68, 0x21, 0x3d, 0x3d, 0x6e, 0x2e, 0x6c, 0x65, 0x6e, 0x67, 0x74, + 0x68, 0x7c, 0x7c, 0x6e, 0x2e, 0x73, 0x6f, 0x6d, 0x65, 0x28, 0x28, 0x66, + 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x6e, 0x2c, 0x65, 0x29, + 0x7b, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x6e, 0x21, 0x3d, 0x3d, + 0x74, 0x5b, 0x65, 0x5d, 0x7d, 0x29, 0x29, 0x7d, 0x66, 0x75, 0x6e, 0x63, + 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x71, 0x74, 0x28, 0x74, 0x2c, 0x6e, 0x29, + 0x7b, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x22, 0x66, 0x75, 0x6e, 0x63, + 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x3d, 0x3d, 0x74, 0x79, 0x70, 0x65, 0x6f, + 0x66, 0x20, 0x6e, 0x3f, 0x6e, 0x28, 0x74, 0x29, 0x3a, 0x6e, 0x7d, 0x66, + 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x42, 0x74, 0x28, 0x74, + 0x2c, 0x6e, 0x29, 0x7b, 0x43, 0x5b, 0x74, 0x5d, 0x3d, 0x6e, 0x2e, 0x62, + 0x69, 0x6e, 0x64, 0x28, 0x6e, 0x75, 0x6c, 0x6c, 0x2c, 0x43, 0x5b, 0x74, + 0x5d, 0x7c, 0x7c, 0x28, 0x28, 0x29, 0x3d, 0x3e, 0x7b, 0x7d, 0x29, 0x29, + 0x7d, 0x6c, 0x65, 0x74, 0x20, 0x47, 0x74, 0x2c, 0x7a, 0x74, 0x3b, 0x66, + 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x4a, 0x74, 0x28, 0x74, + 0x29, 0x7b, 0x69, 0x66, 0x28, 0x7a, 0x74, 0x29, 0x7a, 0x74, 0x28, 0x29, + 0x3b, 0x7a, 0x74, 0x3d, 0x74, 0x26, 0x26, 0x74, 0x2e, 0x53, 0x28, 0x29, + 0x7d, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x4b, 0x74, + 0x28, 0x7b, 0x64, 0x61, 0x74, 0x61, 0x3a, 0x74, 0x7d, 0x29, 0x7b, 0x63, + 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x6e, 0x3d, 0x58, 0x74, 0x28, 0x74, 0x29, + 0x3b, 0x6e, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3d, 0x74, 0x3b, 0x63, + 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x65, 0x3d, 0x44, 0x74, 0x28, 0x28, 0x29, + 0x3d, 0x3e, 0x7b, 0x6c, 0x65, 0x74, 0x20, 0x74, 0x3d, 0x74, 0x68, 0x69, + 0x73, 0x2e, 0x5f, 0x5f, 0x76, 0x3b, 0x77, 0x68, 0x69, 0x6c, 0x65, 0x28, + 0x74, 0x3d, 0x74, 0x2e, 0x5f, 0x5f, 0x29, 0x69, 0x66, 0x28, 0x74, 0x2e, + 0x5f, 0x5f, 0x63, 0x29, 0x7b, 0x74, 0x2e, 0x5f, 0x5f, 0x63, 0x2e, 0x5f, + 0x5f, 0x24, 0x66, 0x7c, 0x3d, 0x34, 0x3b, 0x62, 0x72, 0x65, 0x61, 0x6b, + 0x7d, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x5f, 0x5f, 0x24, 0x75, 0x2e, 0x63, + 0x3d, 0x28, 0x29, 0x3d, 0x3e, 0x7b, 0x76, 0x61, 0x72, 0x20, 0x74, 0x3b, + 0x69, 0x66, 0x28, 0x21, 0x55, 0x28, 0x65, 0x2e, 0x70, 0x65, 0x65, 0x6b, + 0x28, 0x29, 0x29, 0x26, 0x26, 0x33, 0x3d, 0x3d, 0x3d, 0x28, 0x6e, 0x75, + 0x6c, 0x6c, 0x3d, 0x3d, 0x28, 0x74, 0x3d, 0x74, 0x68, 0x69, 0x73, 0x2e, + 0x62, 0x61, 0x73, 0x65, 0x29, 0x3f, 0x76, 0x6f, 0x69, 0x64, 0x20, 0x30, + 0x3a, 0x74, 0x2e, 0x6e, 0x6f, 0x64, 0x65, 0x54, 0x79, 0x70, 0x65, 0x29, + 0x29, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x62, 0x61, 0x73, 0x65, 0x2e, 0x64, + 0x61, 0x74, 0x61, 0x3d, 0x65, 0x2e, 0x70, 0x65, 0x65, 0x6b, 0x28, 0x29, + 0x3b, 0x65, 0x6c, 0x73, 0x65, 0x7b, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x5f, + 0x5f, 0x24, 0x66, 0x7c, 0x3d, 0x31, 0x3b, 0x74, 0x68, 0x69, 0x73, 0x2e, + 0x73, 0x65, 0x74, 0x53, 0x74, 0x61, 0x74, 0x65, 0x28, 0x7b, 0x7d, 0x29, + 0x7d, 0x7d, 0x3b, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x6d, 0x28, + 0x28, 0x29, 0x3d, 0x3e, 0x7b, 0x6c, 0x65, 0x74, 0x20, 0x74, 0x3d, 0x6e, + 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, + 0x3b, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x30, 0x3d, 0x3d, 0x3d, + 0x74, 0x3f, 0x30, 0x3a, 0x21, 0x30, 0x3d, 0x3d, 0x3d, 0x74, 0x3f, 0x22, + 0x22, 0x3a, 0x74, 0x7c, 0x7c, 0x22, 0x22, 0x7d, 0x29, 0x7d, 0x2c, 0x5b, + 0x5d, 0x29, 0x3b, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x65, 0x2e, + 0x76, 0x61, 0x6c, 0x75, 0x65, 0x7d, 0x4b, 0x74, 0x2e, 0x64, 0x69, 0x73, + 0x70, 0x6c, 0x61, 0x79, 0x4e, 0x61, 0x6d, 0x65, 0x3d, 0x22, 0x5f, 0x73, + 0x74, 0x22, 0x3b, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x2e, 0x64, 0x65, + 0x66, 0x69, 0x6e, 0x65, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, + 0x65, 0x73, 0x28, 0x68, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x74, 0x79, + 0x70, 0x65, 0x2c, 0x7b, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x72, 0x75, 0x63, + 0x74, 0x6f, 0x72, 0x3a, 0x7b, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, + 0x72, 0x61, 0x62, 0x6c, 0x65, 0x3a, 0x21, 0x30, 0x2c, 0x76, 0x61, 0x6c, + 0x75, 0x65, 0x3a, 0x76, 0x6f, 0x69, 0x64, 0x20, 0x30, 0x7d, 0x2c, 0x74, + 0x79, 0x70, 0x65, 0x3a, 0x7b, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, + 0x72, 0x61, 0x62, 0x6c, 0x65, 0x3a, 0x21, 0x30, 0x2c, 0x76, 0x61, 0x6c, + 0x75, 0x65, 0x3a, 0x4b, 0x74, 0x7d, 0x2c, 0x70, 0x72, 0x6f, 0x70, 0x73, + 0x3a, 0x7b, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x61, 0x62, + 0x6c, 0x65, 0x3a, 0x21, 0x30, 0x2c, 0x67, 0x65, 0x74, 0x28, 0x29, 0x7b, + 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x7b, 0x64, 0x61, 0x74, 0x61, 0x3a, + 0x74, 0x68, 0x69, 0x73, 0x7d, 0x7d, 0x7d, 0x2c, 0x5f, 0x5f, 0x62, 0x3a, + 0x7b, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x61, 0x62, 0x6c, + 0x65, 0x3a, 0x21, 0x30, 0x2c, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x31, + 0x7d, 0x7d, 0x29, 0x3b, 0x42, 0x74, 0x28, 0x22, 0x5f, 0x5f, 0x62, 0x22, + 0x2c, 0x28, 0x74, 0x2c, 0x6e, 0x29, 0x3d, 0x3e, 0x7b, 0x69, 0x66, 0x28, + 0x22, 0x73, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x22, 0x3d, 0x3d, 0x74, 0x79, + 0x70, 0x65, 0x6f, 0x66, 0x20, 0x6e, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x29, + 0x7b, 0x6c, 0x65, 0x74, 0x20, 0x74, 0x2c, 0x65, 0x3d, 0x6e, 0x2e, 0x70, + 0x72, 0x6f, 0x70, 0x73, 0x3b, 0x66, 0x6f, 0x72, 0x28, 0x6c, 0x65, 0x74, + 0x20, 0x5f, 0x20, 0x69, 0x6e, 0x20, 0x65, 0x29, 0x7b, 0x69, 0x66, 0x28, + 0x22, 0x63, 0x68, 0x69, 0x6c, 0x64, 0x72, 0x65, 0x6e, 0x22, 0x3d, 0x3d, + 0x3d, 0x5f, 0x29, 0x63, 0x6f, 0x6e, 0x74, 0x69, 0x6e, 0x75, 0x65, 0x3b, + 0x6c, 0x65, 0x74, 0x20, 0x69, 0x3d, 0x65, 0x5b, 0x5f, 0x5d, 0x3b, 0x69, + 0x66, 0x28, 0x69, 0x20, 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, + 0x6f, 0x66, 0x20, 0x68, 0x29, 0x7b, 0x69, 0x66, 0x28, 0x21, 0x74, 0x29, + 0x6e, 0x2e, 0x5f, 0x5f, 0x6e, 0x70, 0x3d, 0x74, 0x3d, 0x7b, 0x7d, 0x3b, + 0x74, 0x5b, 0x5f, 0x5d, 0x3d, 0x69, 0x3b, 0x65, 0x5b, 0x5f, 0x5d, 0x3d, + 0x69, 0x2e, 0x70, 0x65, 0x65, 0x6b, 0x28, 0x29, 0x7d, 0x7d, 0x7d, 0x74, + 0x28, 0x6e, 0x29, 0x7d, 0x29, 0x3b, 0x42, 0x74, 0x28, 0x22, 0x5f, 0x5f, + 0x72, 0x22, 0x2c, 0x28, 0x74, 0x2c, 0x6e, 0x29, 0x3d, 0x3e, 0x7b, 0x4a, + 0x74, 0x28, 0x29, 0x3b, 0x6c, 0x65, 0x74, 0x20, 0x65, 0x2c, 0x5f, 0x3d, + 0x6e, 0x2e, 0x5f, 0x5f, 0x63, 0x3b, 0x69, 0x66, 0x28, 0x5f, 0x29, 0x7b, + 0x5f, 0x2e, 0x5f, 0x5f, 0x24, 0x66, 0x26, 0x3d, 0x2d, 0x32, 0x3b, 0x65, + 0x3d, 0x5f, 0x2e, 0x5f, 0x5f, 0x24, 0x75, 0x3b, 0x69, 0x66, 0x28, 0x76, + 0x6f, 0x69, 0x64, 0x20, 0x30, 0x3d, 0x3d, 0x3d, 0x65, 0x29, 0x5f, 0x2e, + 0x5f, 0x5f, 0x24, 0x75, 0x3d, 0x65, 0x3d, 0x66, 0x75, 0x6e, 0x63, 0x74, + 0x69, 0x6f, 0x6e, 0x28, 0x74, 0x29, 0x7b, 0x6c, 0x65, 0x74, 0x20, 0x6e, + 0x3b, 0x77, 0x28, 0x28, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, + 0x28, 0x29, 0x7b, 0x6e, 0x3d, 0x74, 0x68, 0x69, 0x73, 0x7d, 0x29, 0x29, + 0x3b, 0x6e, 0x2e, 0x63, 0x3d, 0x28, 0x29, 0x3d, 0x3e, 0x7b, 0x5f, 0x2e, + 0x5f, 0x5f, 0x24, 0x66, 0x7c, 0x3d, 0x31, 0x3b, 0x5f, 0x2e, 0x73, 0x65, + 0x74, 0x53, 0x74, 0x61, 0x74, 0x65, 0x28, 0x7b, 0x7d, 0x29, 0x7d, 0x3b, + 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x6e, 0x7d, 0x28, 0x29, 0x7d, + 0x47, 0x74, 0x3d, 0x5f, 0x3b, 0x4a, 0x74, 0x28, 0x65, 0x29, 0x3b, 0x74, + 0x28, 0x6e, 0x29, 0x7d, 0x29, 0x3b, 0x42, 0x74, 0x28, 0x22, 0x5f, 0x5f, + 0x65, 0x22, 0x2c, 0x28, 0x74, 0x2c, 0x6e, 0x2c, 0x65, 0x2c, 0x5f, 0x29, + 0x3d, 0x3e, 0x7b, 0x4a, 0x74, 0x28, 0x29, 0x3b, 0x47, 0x74, 0x3d, 0x76, + 0x6f, 0x69, 0x64, 0x20, 0x30, 0x3b, 0x74, 0x28, 0x6e, 0x2c, 0x65, 0x2c, + 0x5f, 0x29, 0x7d, 0x29, 0x3b, 0x42, 0x74, 0x28, 0x22, 0x64, 0x69, 0x66, + 0x66, 0x65, 0x64, 0x22, 0x2c, 0x28, 0x74, 0x2c, 0x6e, 0x29, 0x3d, 0x3e, + 0x7b, 0x4a, 0x74, 0x28, 0x29, 0x3b, 0x47, 0x74, 0x3d, 0x76, 0x6f, 0x69, + 0x64, 0x20, 0x30, 0x3b, 0x6c, 0x65, 0x74, 0x20, 0x65, 0x3b, 0x69, 0x66, + 0x28, 0x22, 0x73, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x22, 0x3d, 0x3d, 0x74, + 0x79, 0x70, 0x65, 0x6f, 0x66, 0x20, 0x6e, 0x2e, 0x74, 0x79, 0x70, 0x65, + 0x26, 0x26, 0x28, 0x65, 0x3d, 0x6e, 0x2e, 0x5f, 0x5f, 0x65, 0x29, 0x29, + 0x7b, 0x6c, 0x65, 0x74, 0x20, 0x74, 0x3d, 0x6e, 0x2e, 0x5f, 0x5f, 0x6e, + 0x70, 0x2c, 0x5f, 0x3d, 0x6e, 0x2e, 0x70, 0x72, 0x6f, 0x70, 0x73, 0x3b, + 0x69, 0x66, 0x28, 0x74, 0x29, 0x7b, 0x6c, 0x65, 0x74, 0x20, 0x6e, 0x3d, + 0x65, 0x2e, 0x55, 0x3b, 0x69, 0x66, 0x28, 0x6e, 0x29, 0x66, 0x6f, 0x72, + 0x28, 0x6c, 0x65, 0x74, 0x20, 0x65, 0x20, 0x69, 0x6e, 0x20, 0x6e, 0x29, + 0x7b, 0x6c, 0x65, 0x74, 0x20, 0x5f, 0x3d, 0x6e, 0x5b, 0x65, 0x5d, 0x3b, + 0x69, 0x66, 0x28, 0x76, 0x6f, 0x69, 0x64, 0x20, 0x30, 0x21, 0x3d, 0x3d, + 0x5f, 0x26, 0x26, 0x21, 0x28, 0x65, 0x20, 0x69, 0x6e, 0x20, 0x74, 0x29, + 0x29, 0x7b, 0x5f, 0x2e, 0x64, 0x28, 0x29, 0x3b, 0x6e, 0x5b, 0x65, 0x5d, + 0x3d, 0x76, 0x6f, 0x69, 0x64, 0x20, 0x30, 0x7d, 0x7d, 0x65, 0x6c, 0x73, + 0x65, 0x7b, 0x6e, 0x3d, 0x7b, 0x7d, 0x3b, 0x65, 0x2e, 0x55, 0x3d, 0x6e, + 0x7d, 0x66, 0x6f, 0x72, 0x28, 0x6c, 0x65, 0x74, 0x20, 0x69, 0x20, 0x69, + 0x6e, 0x20, 0x74, 0x29, 0x7b, 0x6c, 0x65, 0x74, 0x20, 0x6f, 0x3d, 0x6e, + 0x5b, 0x69, 0x5d, 0x2c, 0x72, 0x3d, 0x74, 0x5b, 0x69, 0x5d, 0x3b, 0x69, + 0x66, 0x28, 0x76, 0x6f, 0x69, 0x64, 0x20, 0x30, 0x3d, 0x3d, 0x3d, 0x6f, + 0x29, 0x7b, 0x6f, 0x3d, 0x51, 0x74, 0x28, 0x65, 0x2c, 0x69, 0x2c, 0x72, + 0x2c, 0x5f, 0x29, 0x3b, 0x6e, 0x5b, 0x69, 0x5d, 0x3d, 0x6f, 0x7d, 0x65, + 0x6c, 0x73, 0x65, 0x20, 0x6f, 0x2e, 0x6f, 0x28, 0x72, 0x2c, 0x5f, 0x29, + 0x7d, 0x7d, 0x7d, 0x74, 0x28, 0x6e, 0x29, 0x7d, 0x29, 0x3b, 0x66, 0x75, + 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x51, 0x74, 0x28, 0x74, 0x2c, + 0x6e, 0x2c, 0x65, 0x2c, 0x5f, 0x29, 0x7b, 0x63, 0x6f, 0x6e, 0x73, 0x74, + 0x20, 0x69, 0x3d, 0x6e, 0x20, 0x69, 0x6e, 0x20, 0x74, 0x26, 0x26, 0x76, + 0x6f, 0x69, 0x64, 0x20, 0x30, 0x3d, 0x3d, 0x3d, 0x74, 0x2e, 0x6f, 0x77, + 0x6e, 0x65, 0x72, 0x53, 0x56, 0x47, 0x45, 0x6c, 0x65, 0x6d, 0x65, 0x6e, + 0x74, 0x2c, 0x6f, 0x3d, 0x61, 0x28, 0x65, 0x29, 0x3b, 0x72, 0x65, 0x74, + 0x75, 0x72, 0x6e, 0x7b, 0x6f, 0x3a, 0x28, 0x74, 0x2c, 0x6e, 0x29, 0x3d, + 0x3e, 0x7b, 0x6f, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3d, 0x74, 0x3b, + 0x5f, 0x3d, 0x6e, 0x7d, 0x2c, 0x64, 0x3a, 0x77, 0x28, 0x28, 0x29, 0x3d, + 0x3e, 0x7b, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x65, 0x3d, 0x6f, 0x2e, + 0x76, 0x61, 0x6c, 0x75, 0x65, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3b, + 0x69, 0x66, 0x28, 0x5f, 0x5b, 0x6e, 0x5d, 0x21, 0x3d, 0x3d, 0x65, 0x29, + 0x7b, 0x5f, 0x5b, 0x6e, 0x5d, 0x3d, 0x65, 0x3b, 0x69, 0x66, 0x28, 0x69, + 0x29, 0x74, 0x5b, 0x6e, 0x5d, 0x3d, 0x65, 0x3b, 0x65, 0x6c, 0x73, 0x65, + 0x20, 0x69, 0x66, 0x28, 0x65, 0x29, 0x74, 0x2e, 0x73, 0x65, 0x74, 0x41, + 0x74, 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, 0x65, 0x28, 0x6e, 0x2c, 0x65, + 0x29, 0x3b, 0x65, 0x6c, 0x73, 0x65, 0x20, 0x74, 0x2e, 0x72, 0x65, 0x6d, + 0x6f, 0x76, 0x65, 0x41, 0x74, 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, 0x65, + 0x28, 0x6e, 0x29, 0x7d, 0x7d, 0x29, 0x7d, 0x7d, 0x42, 0x74, 0x28, 0x22, + 0x75, 0x6e, 0x6d, 0x6f, 0x75, 0x6e, 0x74, 0x22, 0x2c, 0x28, 0x74, 0x2c, + 0x6e, 0x29, 0x3d, 0x3e, 0x7b, 0x69, 0x66, 0x28, 0x22, 0x73, 0x74, 0x72, + 0x69, 0x6e, 0x67, 0x22, 0x3d, 0x3d, 0x74, 0x79, 0x70, 0x65, 0x6f, 0x66, + 0x20, 0x6e, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x29, 0x7b, 0x6c, 0x65, 0x74, + 0x20, 0x74, 0x3d, 0x6e, 0x2e, 0x5f, 0x5f, 0x65, 0x3b, 0x69, 0x66, 0x28, + 0x74, 0x29, 0x7b, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x6e, 0x3d, 0x74, + 0x2e, 0x55, 0x3b, 0x69, 0x66, 0x28, 0x6e, 0x29, 0x7b, 0x74, 0x2e, 0x55, + 0x3d, 0x76, 0x6f, 0x69, 0x64, 0x20, 0x30, 0x3b, 0x66, 0x6f, 0x72, 0x28, + 0x6c, 0x65, 0x74, 0x20, 0x74, 0x20, 0x69, 0x6e, 0x20, 0x6e, 0x29, 0x7b, + 0x6c, 0x65, 0x74, 0x20, 0x65, 0x3d, 0x6e, 0x5b, 0x74, 0x5d, 0x3b, 0x69, + 0x66, 0x28, 0x65, 0x29, 0x65, 0x2e, 0x64, 0x28, 0x29, 0x7d, 0x7d, 0x7d, + 0x7d, 0x65, 0x6c, 0x73, 0x65, 0x7b, 0x6c, 0x65, 0x74, 0x20, 0x74, 0x3d, + 0x6e, 0x2e, 0x5f, 0x5f, 0x63, 0x3b, 0x69, 0x66, 0x28, 0x74, 0x29, 0x7b, + 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x6e, 0x3d, 0x74, 0x2e, 0x5f, 0x5f, + 0x24, 0x75, 0x3b, 0x69, 0x66, 0x28, 0x6e, 0x29, 0x7b, 0x74, 0x2e, 0x5f, + 0x5f, 0x24, 0x75, 0x3d, 0x76, 0x6f, 0x69, 0x64, 0x20, 0x30, 0x3b, 0x6e, + 0x2e, 0x64, 0x28, 0x29, 0x7d, 0x7d, 0x7d, 0x74, 0x28, 0x6e, 0x29, 0x7d, + 0x29, 0x3b, 0x42, 0x74, 0x28, 0x22, 0x5f, 0x5f, 0x68, 0x22, 0x2c, 0x28, + 0x74, 0x2c, 0x6e, 0x2c, 0x65, 0x2c, 0x5f, 0x29, 0x3d, 0x3e, 0x7b, 0x69, + 0x66, 0x28, 0x5f, 0x3c, 0x33, 0x7c, 0x7c, 0x39, 0x3d, 0x3d, 0x3d, 0x5f, + 0x29, 0x6e, 0x2e, 0x5f, 0x5f, 0x24, 0x66, 0x7c, 0x3d, 0x32, 0x3b, 0x74, + 0x28, 0x6e, 0x2c, 0x65, 0x2c, 0x5f, 0x29, 0x7d, 0x29, 0x3b, 0x49, 0x2e, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x74, 0x79, 0x70, 0x65, 0x2e, 0x73, 0x68, + 0x6f, 0x75, 0x6c, 0x64, 0x43, 0x6f, 0x6d, 0x70, 0x6f, 0x6e, 0x65, 0x6e, + 0x74, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x3d, 0x66, 0x75, 0x6e, 0x63, + 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x74, 0x2c, 0x6e, 0x29, 0x7b, 0x63, 0x6f, + 0x6e, 0x73, 0x74, 0x20, 0x65, 0x3d, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x5f, + 0x5f, 0x24, 0x75, 0x3b, 0x69, 0x66, 0x28, 0x21, 0x28, 0x65, 0x26, 0x26, + 0x76, 0x6f, 0x69, 0x64, 0x20, 0x30, 0x21, 0x3d, 0x3d, 0x65, 0x2e, 0x73, + 0x7c, 0x7c, 0x34, 0x26, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x5f, 0x5f, 0x24, + 0x66, 0x29, 0x29, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x21, 0x30, 0x3b, + 0x69, 0x66, 0x28, 0x33, 0x26, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x5f, 0x5f, + 0x24, 0x66, 0x29, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x21, 0x30, 0x3b, + 0x66, 0x6f, 0x72, 0x28, 0x6c, 0x65, 0x74, 0x20, 0x5f, 0x20, 0x69, 0x6e, + 0x20, 0x6e, 0x29, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x21, 0x30, 0x3b, + 0x66, 0x6f, 0x72, 0x28, 0x6c, 0x65, 0x74, 0x20, 0x5f, 0x20, 0x69, 0x6e, + 0x20, 0x74, 0x29, 0x69, 0x66, 0x28, 0x22, 0x5f, 0x5f, 0x73, 0x6f, 0x75, + 0x72, 0x63, 0x65, 0x22, 0x21, 0x3d, 0x3d, 0x5f, 0x26, 0x26, 0x74, 0x5b, + 0x5f, 0x5d, 0x21, 0x3d, 0x3d, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x70, 0x72, + 0x6f, 0x70, 0x73, 0x5b, 0x5f, 0x5d, 0x29, 0x72, 0x65, 0x74, 0x75, 0x72, + 0x6e, 0x21, 0x30, 0x3b, 0x66, 0x6f, 0x72, 0x28, 0x6c, 0x65, 0x74, 0x20, + 0x5f, 0x20, 0x69, 0x6e, 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x70, 0x72, + 0x6f, 0x70, 0x73, 0x29, 0x69, 0x66, 0x28, 0x21, 0x28, 0x5f, 0x20, 0x69, + 0x6e, 0x20, 0x74, 0x29, 0x29, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x21, + 0x30, 0x3b, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x21, 0x31, 0x7d, 0x3b, + 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x58, 0x74, 0x28, + 0x74, 0x29, 0x7b, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x44, 0x74, + 0x28, 0x28, 0x29, 0x3d, 0x3e, 0x61, 0x28, 0x74, 0x29, 0x2c, 0x5b, 0x5d, + 0x29, 0x7d, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x59, + 0x74, 0x28, 0x74, 0x29, 0x7b, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x6e, + 0x3d, 0x4e, 0x74, 0x28, 0x74, 0x29, 0x3b, 0x6e, 0x2e, 0x63, 0x75, 0x72, + 0x72, 0x65, 0x6e, 0x74, 0x3d, 0x74, 0x3b, 0x47, 0x74, 0x2e, 0x5f, 0x5f, + 0x24, 0x66, 0x7c, 0x3d, 0x34, 0x3b, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, + 0x20, 0x44, 0x74, 0x28, 0x28, 0x29, 0x3d, 0x3e, 0x6d, 0x28, 0x28, 0x29, + 0x3d, 0x3e, 0x6e, 0x2e, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x74, 0x28, + 0x29, 0x29, 0x2c, 0x5b, 0x5d, 0x29, 0x7d, 0x66, 0x75, 0x6e, 0x63, 0x74, + 0x69, 0x6f, 0x6e, 0x20, 0x5a, 0x74, 0x28, 0x74, 0x29, 0x7b, 0x63, 0x6f, + 0x6e, 0x73, 0x74, 0x20, 0x6e, 0x3d, 0x4e, 0x74, 0x28, 0x74, 0x29, 0x3b, + 0x6e, 0x2e, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x74, 0x3d, 0x74, 0x3b, + 0x48, 0x74, 0x28, 0x28, 0x29, 0x3d, 0x3e, 0x77, 0x28, 0x28, 0x29, 0x3d, + 0x3e, 0x6e, 0x2e, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x74, 0x28, 0x29, + 0x29, 0x2c, 0x5b, 0x5d, 0x29, 0x7d, 0x76, 0x61, 0x72, 0x20, 0x74, 0x6e, + 0x3d, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x74, 0x2c, + 0x6e, 0x2c, 0x65, 0x2c, 0x5f, 0x29, 0x7b, 0x76, 0x61, 0x72, 0x20, 0x69, + 0x3b, 0x6e, 0x5b, 0x30, 0x5d, 0x3d, 0x30, 0x3b, 0x66, 0x6f, 0x72, 0x28, + 0x76, 0x61, 0x72, 0x20, 0x6f, 0x3d, 0x31, 0x3b, 0x6f, 0x3c, 0x6e, 0x2e, + 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x3b, 0x6f, 0x2b, 0x2b, 0x29, 0x7b, + 0x76, 0x61, 0x72, 0x20, 0x72, 0x3d, 0x6e, 0x5b, 0x6f, 0x2b, 0x2b, 0x5d, + 0x2c, 0x75, 0x3d, 0x6e, 0x5b, 0x6f, 0x5d, 0x3f, 0x28, 0x6e, 0x5b, 0x30, + 0x5d, 0x7c, 0x3d, 0x72, 0x3f, 0x31, 0x3a, 0x32, 0x2c, 0x65, 0x5b, 0x6e, + 0x5b, 0x6f, 0x2b, 0x2b, 0x5d, 0x5d, 0x29, 0x3a, 0x6e, 0x5b, 0x2b, 0x2b, + 0x6f, 0x5d, 0x3b, 0x33, 0x3d, 0x3d, 0x3d, 0x72, 0x3f, 0x5f, 0x5b, 0x30, + 0x5d, 0x3d, 0x75, 0x3a, 0x34, 0x3d, 0x3d, 0x3d, 0x72, 0x3f, 0x5f, 0x5b, + 0x31, 0x5d, 0x3d, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x2e, 0x61, 0x73, + 0x73, 0x69, 0x67, 0x6e, 0x28, 0x5f, 0x5b, 0x31, 0x5d, 0x7c, 0x7c, 0x7b, + 0x7d, 0x2c, 0x75, 0x29, 0x3a, 0x35, 0x3d, 0x3d, 0x3d, 0x72, 0x3f, 0x28, + 0x5f, 0x5b, 0x31, 0x5d, 0x3d, 0x5f, 0x5b, 0x31, 0x5d, 0x7c, 0x7c, 0x7b, + 0x7d, 0x29, 0x5b, 0x6e, 0x5b, 0x2b, 0x2b, 0x6f, 0x5d, 0x5d, 0x3d, 0x75, + 0x3a, 0x36, 0x3d, 0x3d, 0x3d, 0x72, 0x3f, 0x5f, 0x5b, 0x31, 0x5d, 0x5b, + 0x6e, 0x5b, 0x2b, 0x2b, 0x6f, 0x5d, 0x5d, 0x2b, 0x3d, 0x75, 0x2b, 0x22, + 0x22, 0x3a, 0x72, 0x3f, 0x28, 0x69, 0x3d, 0x74, 0x2e, 0x61, 0x70, 0x70, + 0x6c, 0x79, 0x28, 0x75, 0x2c, 0x74, 0x6e, 0x28, 0x74, 0x2c, 0x75, 0x2c, + 0x65, 0x2c, 0x5b, 0x22, 0x22, 0x2c, 0x6e, 0x75, 0x6c, 0x6c, 0x5d, 0x29, + 0x29, 0x2c, 0x5f, 0x2e, 0x70, 0x75, 0x73, 0x68, 0x28, 0x69, 0x29, 0x2c, + 0x75, 0x5b, 0x30, 0x5d, 0x3f, 0x6e, 0x5b, 0x30, 0x5d, 0x7c, 0x3d, 0x32, + 0x3a, 0x28, 0x6e, 0x5b, 0x6f, 0x2d, 0x32, 0x5d, 0x3d, 0x30, 0x2c, 0x6e, + 0x5b, 0x6f, 0x5d, 0x3d, 0x69, 0x29, 0x29, 0x3a, 0x5f, 0x2e, 0x70, 0x75, + 0x73, 0x68, 0x28, 0x75, 0x29, 0x7d, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, + 0x20, 0x5f, 0x7d, 0x2c, 0x6e, 0x6e, 0x3d, 0x6e, 0x65, 0x77, 0x20, 0x4d, + 0x61, 0x70, 0x3b, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, + 0x65, 0x6e, 0x28, 0x74, 0x29, 0x7b, 0x76, 0x61, 0x72, 0x20, 0x6e, 0x3d, + 0x6e, 0x6e, 0x2e, 0x67, 0x65, 0x74, 0x28, 0x74, 0x68, 0x69, 0x73, 0x29, + 0x3b, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x6e, 0x7c, 0x7c, 0x28, + 0x6e, 0x3d, 0x6e, 0x65, 0x77, 0x20, 0x4d, 0x61, 0x70, 0x2c, 0x6e, 0x6e, + 0x2e, 0x73, 0x65, 0x74, 0x28, 0x74, 0x68, 0x69, 0x73, 0x2c, 0x6e, 0x29, + 0x29, 0x2c, 0x28, 0x6e, 0x3d, 0x74, 0x6e, 0x28, 0x74, 0x68, 0x69, 0x73, + 0x2c, 0x6e, 0x2e, 0x67, 0x65, 0x74, 0x28, 0x74, 0x29, 0x7c, 0x7c, 0x28, + 0x6e, 0x2e, 0x73, 0x65, 0x74, 0x28, 0x74, 0x2c, 0x6e, 0x3d, 0x66, 0x75, + 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x74, 0x29, 0x7b, 0x66, 0x6f, + 0x72, 0x28, 0x76, 0x61, 0x72, 0x20, 0x6e, 0x2c, 0x65, 0x2c, 0x5f, 0x3d, + 0x31, 0x2c, 0x69, 0x3d, 0x22, 0x22, 0x2c, 0x6f, 0x3d, 0x22, 0x22, 0x2c, + 0x72, 0x3d, 0x5b, 0x30, 0x5d, 0x2c, 0x75, 0x3d, 0x66, 0x75, 0x6e, 0x63, + 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x74, 0x29, 0x7b, 0x31, 0x3d, 0x3d, 0x3d, + 0x5f, 0x26, 0x26, 0x28, 0x74, 0x7c, 0x7c, 0x28, 0x69, 0x3d, 0x69, 0x2e, + 0x72, 0x65, 0x70, 0x6c, 0x61, 0x63, 0x65, 0x28, 0x2f, 0x5e, 0x5c, 0x73, + 0x2a, 0x5c, 0x6e, 0x5c, 0x73, 0x2a, 0x7c, 0x5c, 0x73, 0x2a, 0x5c, 0x6e, + 0x5c, 0x73, 0x2a, 0x24, 0x2f, 0x67, 0x2c, 0x22, 0x22, 0x29, 0x29, 0x29, + 0x3f, 0x72, 0x2e, 0x70, 0x75, 0x73, 0x68, 0x28, 0x30, 0x2c, 0x74, 0x2c, + 0x69, 0x29, 0x3a, 0x33, 0x3d, 0x3d, 0x3d, 0x5f, 0x26, 0x26, 0x28, 0x74, + 0x7c, 0x7c, 0x69, 0x29, 0x3f, 0x28, 0x72, 0x2e, 0x70, 0x75, 0x73, 0x68, + 0x28, 0x33, 0x2c, 0x74, 0x2c, 0x69, 0x29, 0x2c, 0x5f, 0x3d, 0x32, 0x29, + 0x3a, 0x32, 0x3d, 0x3d, 0x3d, 0x5f, 0x26, 0x26, 0x22, 0x2e, 0x2e, 0x2e, + 0x22, 0x3d, 0x3d, 0x3d, 0x69, 0x26, 0x26, 0x74, 0x3f, 0x72, 0x2e, 0x70, + 0x75, 0x73, 0x68, 0x28, 0x34, 0x2c, 0x74, 0x2c, 0x30, 0x29, 0x3a, 0x32, + 0x3d, 0x3d, 0x3d, 0x5f, 0x26, 0x26, 0x69, 0x26, 0x26, 0x21, 0x74, 0x3f, + 0x72, 0x2e, 0x70, 0x75, 0x73, 0x68, 0x28, 0x35, 0x2c, 0x30, 0x2c, 0x21, + 0x30, 0x2c, 0x69, 0x29, 0x3a, 0x5f, 0x3e, 0x3d, 0x35, 0x26, 0x26, 0x28, + 0x28, 0x69, 0x7c, 0x7c, 0x21, 0x74, 0x26, 0x26, 0x35, 0x3d, 0x3d, 0x3d, + 0x5f, 0x29, 0x26, 0x26, 0x28, 0x72, 0x2e, 0x70, 0x75, 0x73, 0x68, 0x28, + 0x5f, 0x2c, 0x30, 0x2c, 0x69, 0x2c, 0x65, 0x29, 0x2c, 0x5f, 0x3d, 0x36, + 0x29, 0x2c, 0x74, 0x26, 0x26, 0x28, 0x72, 0x2e, 0x70, 0x75, 0x73, 0x68, + 0x28, 0x5f, 0x2c, 0x74, 0x2c, 0x30, 0x2c, 0x65, 0x29, 0x2c, 0x5f, 0x3d, + 0x36, 0x29, 0x29, 0x2c, 0x69, 0x3d, 0x22, 0x22, 0x7d, 0x2c, 0x66, 0x3d, + 0x30, 0x3b, 0x66, 0x3c, 0x74, 0x2e, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, + 0x3b, 0x66, 0x2b, 0x2b, 0x29, 0x7b, 0x66, 0x26, 0x26, 0x28, 0x31, 0x3d, + 0x3d, 0x3d, 0x5f, 0x26, 0x26, 0x75, 0x28, 0x29, 0x2c, 0x75, 0x28, 0x66, + 0x29, 0x29, 0x3b, 0x66, 0x6f, 0x72, 0x28, 0x76, 0x61, 0x72, 0x20, 0x73, + 0x3d, 0x30, 0x3b, 0x73, 0x3c, 0x74, 0x5b, 0x66, 0x5d, 0x2e, 0x6c, 0x65, + 0x6e, 0x67, 0x74, 0x68, 0x3b, 0x73, 0x2b, 0x2b, 0x29, 0x6e, 0x3d, 0x74, + 0x5b, 0x66, 0x5d, 0x5b, 0x73, 0x5d, 0x2c, 0x31, 0x3d, 0x3d, 0x3d, 0x5f, + 0x3f, 0x22, 0x3c, 0x22, 0x3d, 0x3d, 0x3d, 0x6e, 0x3f, 0x28, 0x75, 0x28, + 0x29, 0x2c, 0x72, 0x3d, 0x5b, 0x72, 0x5d, 0x2c, 0x5f, 0x3d, 0x33, 0x29, + 0x3a, 0x69, 0x2b, 0x3d, 0x6e, 0x3a, 0x34, 0x3d, 0x3d, 0x3d, 0x5f, 0x3f, + 0x22, 0x2d, 0x2d, 0x22, 0x3d, 0x3d, 0x3d, 0x69, 0x26, 0x26, 0x22, 0x3e, + 0x22, 0x3d, 0x3d, 0x3d, 0x6e, 0x3f, 0x28, 0x5f, 0x3d, 0x31, 0x2c, 0x69, + 0x3d, 0x22, 0x22, 0x29, 0x3a, 0x69, 0x3d, 0x6e, 0x2b, 0x69, 0x5b, 0x30, + 0x5d, 0x3a, 0x6f, 0x3f, 0x6e, 0x3d, 0x3d, 0x3d, 0x6f, 0x3f, 0x6f, 0x3d, + 0x22, 0x22, 0x3a, 0x69, 0x2b, 0x3d, 0x6e, 0x3a, 0x27, 0x22, 0x27, 0x3d, + 0x3d, 0x3d, 0x6e, 0x7c, 0x7c, 0x22, 0x27, 0x22, 0x3d, 0x3d, 0x3d, 0x6e, + 0x3f, 0x6f, 0x3d, 0x6e, 0x3a, 0x22, 0x3e, 0x22, 0x3d, 0x3d, 0x3d, 0x6e, + 0x3f, 0x28, 0x75, 0x28, 0x29, 0x2c, 0x5f, 0x3d, 0x31, 0x29, 0x3a, 0x5f, + 0x26, 0x26, 0x28, 0x22, 0x3d, 0x22, 0x3d, 0x3d, 0x3d, 0x6e, 0x3f, 0x28, + 0x5f, 0x3d, 0x35, 0x2c, 0x65, 0x3d, 0x69, 0x2c, 0x69, 0x3d, 0x22, 0x22, + 0x29, 0x3a, 0x22, 0x2f, 0x22, 0x3d, 0x3d, 0x3d, 0x6e, 0x26, 0x26, 0x28, + 0x5f, 0x3c, 0x35, 0x7c, 0x7c, 0x22, 0x3e, 0x22, 0x3d, 0x3d, 0x3d, 0x74, + 0x5b, 0x66, 0x5d, 0x5b, 0x73, 0x2b, 0x31, 0x5d, 0x29, 0x3f, 0x28, 0x75, + 0x28, 0x29, 0x2c, 0x33, 0x3d, 0x3d, 0x3d, 0x5f, 0x26, 0x26, 0x28, 0x72, + 0x3d, 0x72, 0x5b, 0x30, 0x5d, 0x29, 0x2c, 0x5f, 0x3d, 0x72, 0x2c, 0x28, + 0x72, 0x3d, 0x72, 0x5b, 0x30, 0x5d, 0x29, 0x2e, 0x70, 0x75, 0x73, 0x68, + 0x28, 0x32, 0x2c, 0x30, 0x2c, 0x5f, 0x29, 0x2c, 0x5f, 0x3d, 0x30, 0x29, + 0x3a, 0x22, 0x20, 0x22, 0x3d, 0x3d, 0x3d, 0x6e, 0x7c, 0x7c, 0x22, 0x5c, + 0x74, 0x22, 0x3d, 0x3d, 0x3d, 0x6e, 0x7c, 0x7c, 0x22, 0x5c, 0x6e, 0x22, + 0x3d, 0x3d, 0x3d, 0x6e, 0x7c, 0x7c, 0x22, 0x5c, 0x72, 0x22, 0x3d, 0x3d, + 0x3d, 0x6e, 0x3f, 0x28, 0x75, 0x28, 0x29, 0x2c, 0x5f, 0x3d, 0x32, 0x29, + 0x3a, 0x69, 0x2b, 0x3d, 0x6e, 0x29, 0x2c, 0x33, 0x3d, 0x3d, 0x3d, 0x5f, + 0x26, 0x26, 0x22, 0x21, 0x2d, 0x2d, 0x22, 0x3d, 0x3d, 0x3d, 0x69, 0x26, + 0x26, 0x28, 0x5f, 0x3d, 0x34, 0x2c, 0x72, 0x3d, 0x72, 0x5b, 0x30, 0x5d, + 0x29, 0x7d, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x75, 0x28, 0x29, + 0x2c, 0x72, 0x7d, 0x28, 0x74, 0x29, 0x29, 0x2c, 0x6e, 0x29, 0x2c, 0x61, + 0x72, 0x67, 0x75, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x2c, 0x5b, 0x5d, 0x29, + 0x29, 0x2e, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x3e, 0x31, 0x3f, 0x6e, + 0x3a, 0x6e, 0x5b, 0x30, 0x5d, 0x7d, 0x76, 0x61, 0x72, 0x20, 0x5f, 0x6e, + 0x3d, 0x65, 0x6e, 0x2e, 0x62, 0x69, 0x6e, 0x64, 0x28, 0x4c, 0x29, 0x3b, + 0x65, 0x78, 0x70, 0x6f, 0x72, 0x74, 0x7b, 0x49, 0x20, 0x61, 0x73, 0x20, + 0x43, 0x6f, 0x6d, 0x70, 0x6f, 0x6e, 0x65, 0x6e, 0x74, 0x2c, 0x6a, 0x20, + 0x61, 0x73, 0x20, 0x46, 0x72, 0x61, 0x67, 0x6d, 0x65, 0x6e, 0x74, 0x2c, + 0x68, 0x20, 0x61, 0x73, 0x20, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x2c, + 0x5f, 0x20, 0x61, 0x73, 0x20, 0x62, 0x61, 0x74, 0x63, 0x68, 0x2c, 0x63, + 0x74, 0x20, 0x61, 0x73, 0x20, 0x63, 0x6c, 0x6f, 0x6e, 0x65, 0x45, 0x6c, + 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2c, 0x6d, 0x20, 0x61, 0x73, 0x20, 0x63, + 0x6f, 0x6d, 0x70, 0x75, 0x74, 0x65, 0x64, 0x2c, 0x68, 0x74, 0x20, 0x61, + 0x73, 0x20, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x43, 0x6f, 0x6e, 0x74, + 0x65, 0x78, 0x74, 0x2c, 0x4c, 0x20, 0x61, 0x73, 0x20, 0x63, 0x72, 0x65, + 0x61, 0x74, 0x65, 0x45, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2c, 0x52, + 0x20, 0x61, 0x73, 0x20, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x52, 0x65, + 0x66, 0x2c, 0x77, 0x20, 0x61, 0x73, 0x20, 0x65, 0x66, 0x66, 0x65, 0x63, + 0x74, 0x2c, 0x4c, 0x20, 0x61, 0x73, 0x20, 0x68, 0x2c, 0x5f, 0x6e, 0x20, + 0x61, 0x73, 0x20, 0x68, 0x74, 0x6d, 0x6c, 0x2c, 0x6c, 0x74, 0x20, 0x61, + 0x73, 0x20, 0x68, 0x79, 0x64, 0x72, 0x61, 0x74, 0x65, 0x2c, 0x55, 0x20, + 0x61, 0x73, 0x20, 0x69, 0x73, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x45, 0x6c, + 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2c, 0x43, 0x20, 0x61, 0x73, 0x20, 0x6f, + 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x2c, 0x73, 0x74, 0x20, 0x61, 0x73, + 0x20, 0x72, 0x65, 0x6e, 0x64, 0x65, 0x72, 0x2c, 0x61, 0x20, 0x61, 0x73, + 0x20, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x2c, 0x58, 0x20, 0x61, 0x73, + 0x20, 0x74, 0x6f, 0x43, 0x68, 0x69, 0x6c, 0x64, 0x41, 0x72, 0x72, 0x61, + 0x79, 0x2c, 0x75, 0x20, 0x61, 0x73, 0x20, 0x75, 0x6e, 0x74, 0x72, 0x61, + 0x63, 0x6b, 0x65, 0x64, 0x2c, 0x54, 0x74, 0x20, 0x61, 0x73, 0x20, 0x75, + 0x73, 0x65, 0x43, 0x61, 0x6c, 0x6c, 0x62, 0x61, 0x63, 0x6b, 0x2c, 0x59, + 0x74, 0x20, 0x61, 0x73, 0x20, 0x75, 0x73, 0x65, 0x43, 0x6f, 0x6d, 0x70, + 0x75, 0x74, 0x65, 0x64, 0x2c, 0x56, 0x74, 0x20, 0x61, 0x73, 0x20, 0x75, + 0x73, 0x65, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x2c, 0x41, 0x74, + 0x20, 0x61, 0x73, 0x20, 0x75, 0x73, 0x65, 0x44, 0x65, 0x62, 0x75, 0x67, + 0x56, 0x61, 0x6c, 0x75, 0x65, 0x2c, 0x48, 0x74, 0x20, 0x61, 0x73, 0x20, + 0x75, 0x73, 0x65, 0x45, 0x66, 0x66, 0x65, 0x63, 0x74, 0x2c, 0x46, 0x74, + 0x20, 0x61, 0x73, 0x20, 0x75, 0x73, 0x65, 0x45, 0x72, 0x72, 0x6f, 0x72, + 0x42, 0x6f, 0x75, 0x6e, 0x64, 0x61, 0x72, 0x79, 0x2c, 0x4d, 0x74, 0x20, + 0x61, 0x73, 0x20, 0x75, 0x73, 0x65, 0x49, 0x64, 0x2c, 0x24, 0x74, 0x20, + 0x61, 0x73, 0x20, 0x75, 0x73, 0x65, 0x49, 0x6d, 0x70, 0x65, 0x72, 0x61, + 0x74, 0x69, 0x76, 0x65, 0x48, 0x61, 0x6e, 0x64, 0x6c, 0x65, 0x2c, 0x50, + 0x74, 0x20, 0x61, 0x73, 0x20, 0x75, 0x73, 0x65, 0x4c, 0x61, 0x79, 0x6f, + 0x75, 0x74, 0x45, 0x66, 0x66, 0x65, 0x63, 0x74, 0x2c, 0x44, 0x74, 0x20, + 0x61, 0x73, 0x20, 0x75, 0x73, 0x65, 0x4d, 0x65, 0x6d, 0x6f, 0x2c, 0x55, + 0x74, 0x20, 0x61, 0x73, 0x20, 0x75, 0x73, 0x65, 0x52, 0x65, 0x64, 0x75, + 0x63, 0x65, 0x72, 0x2c, 0x4e, 0x74, 0x20, 0x61, 0x73, 0x20, 0x75, 0x73, + 0x65, 0x52, 0x65, 0x66, 0x2c, 0x58, 0x74, 0x20, 0x61, 0x73, 0x20, 0x75, + 0x73, 0x65, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x2c, 0x5a, 0x74, 0x20, + 0x61, 0x73, 0x20, 0x75, 0x73, 0x65, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x6c, + 0x45, 0x66, 0x66, 0x65, 0x63, 0x74, 0x2c, 0x45, 0x74, 0x20, 0x61, 0x73, + 0x20, 0x75, 0x73, 0x65, 0x53, 0x74, 0x61, 0x74, 0x65, 0x7d, 0x3b, 0x0a +}; +unsigned int index_js_len = 22800; diff --git a/examples/server/json-schema-to-grammar.mjs.hpp b/examples/server/json-schema-to-grammar.mjs.hpp index 83b22d670..0a05c369d 100644 --- a/examples/server/json-schema-to-grammar.mjs.hpp +++ b/examples/server/json-schema-to-grammar.mjs.hpp @@ -1,115 +1,311 @@ -const char json_schema_to_grammar_mjs[] = R"LITERAL( -const SPACE_RULE = '" "?'; - -const PRIMITIVE_RULES = { - boolean: '("true" | "false") space', - number: '("-"? ([0-9] | [1-9] [0-9]*)) ("." [0-9]+)? ([eE] [-+]? [0-9]+)? space', - integer: '("-"? ([0-9] | [1-9] [0-9]*)) space', - string: ` "\\"" ( - [^"\\\\] | - "\\\\" (["\\\\/bfnrt] | "u" [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F]) - )* "\\"" space`, - null: '"null" space', +unsigned char json_schema_to_grammar_mjs[] = { + 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x53, 0x50, 0x41, 0x43, 0x45, 0x5f, + 0x52, 0x55, 0x4c, 0x45, 0x20, 0x3d, 0x20, 0x27, 0x22, 0x20, 0x22, 0x3f, + 0x27, 0x3b, 0x0a, 0x0a, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x50, 0x52, + 0x49, 0x4d, 0x49, 0x54, 0x49, 0x56, 0x45, 0x5f, 0x52, 0x55, 0x4c, 0x45, + 0x53, 0x20, 0x3d, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x62, 0x6f, 0x6f, 0x6c, + 0x65, 0x61, 0x6e, 0x3a, 0x20, 0x27, 0x28, 0x22, 0x74, 0x72, 0x75, 0x65, + 0x22, 0x20, 0x7c, 0x20, 0x22, 0x66, 0x61, 0x6c, 0x73, 0x65, 0x22, 0x29, + 0x20, 0x73, 0x70, 0x61, 0x63, 0x65, 0x27, 0x2c, 0x0a, 0x20, 0x20, 0x6e, + 0x75, 0x6d, 0x62, 0x65, 0x72, 0x3a, 0x20, 0x27, 0x28, 0x22, 0x2d, 0x22, + 0x3f, 0x20, 0x28, 0x5b, 0x30, 0x2d, 0x39, 0x5d, 0x20, 0x7c, 0x20, 0x5b, + 0x31, 0x2d, 0x39, 0x5d, 0x20, 0x5b, 0x30, 0x2d, 0x39, 0x5d, 0x2a, 0x29, + 0x29, 0x20, 0x28, 0x22, 0x2e, 0x22, 0x20, 0x5b, 0x30, 0x2d, 0x39, 0x5d, + 0x2b, 0x29, 0x3f, 0x20, 0x28, 0x5b, 0x65, 0x45, 0x5d, 0x20, 0x5b, 0x2d, + 0x2b, 0x5d, 0x3f, 0x20, 0x5b, 0x30, 0x2d, 0x39, 0x5d, 0x2b, 0x29, 0x3f, + 0x20, 0x73, 0x70, 0x61, 0x63, 0x65, 0x27, 0x2c, 0x0a, 0x20, 0x20, 0x69, + 0x6e, 0x74, 0x65, 0x67, 0x65, 0x72, 0x3a, 0x20, 0x27, 0x28, 0x22, 0x2d, + 0x22, 0x3f, 0x20, 0x28, 0x5b, 0x30, 0x2d, 0x39, 0x5d, 0x20, 0x7c, 0x20, + 0x5b, 0x31, 0x2d, 0x39, 0x5d, 0x20, 0x5b, 0x30, 0x2d, 0x39, 0x5d, 0x2a, + 0x29, 0x29, 0x20, 0x73, 0x70, 0x61, 0x63, 0x65, 0x27, 0x2c, 0x0a, 0x20, + 0x20, 0x73, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x3a, 0x20, 0x60, 0x20, 0x22, + 0x5c, 0x5c, 0x22, 0x22, 0x20, 0x28, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x5b, 0x5e, 0x22, 0x5c, 0x5c, 0x5c, 0x5c, 0x5d, 0x20, + 0x7c, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x22, 0x5c, + 0x5c, 0x5c, 0x5c, 0x22, 0x20, 0x28, 0x5b, 0x22, 0x5c, 0x5c, 0x5c, 0x5c, + 0x2f, 0x62, 0x66, 0x6e, 0x72, 0x74, 0x5d, 0x20, 0x7c, 0x20, 0x22, 0x75, + 0x22, 0x20, 0x5b, 0x30, 0x2d, 0x39, 0x61, 0x2d, 0x66, 0x41, 0x2d, 0x46, + 0x5d, 0x20, 0x5b, 0x30, 0x2d, 0x39, 0x61, 0x2d, 0x66, 0x41, 0x2d, 0x46, + 0x5d, 0x20, 0x5b, 0x30, 0x2d, 0x39, 0x61, 0x2d, 0x66, 0x41, 0x2d, 0x46, + 0x5d, 0x20, 0x5b, 0x30, 0x2d, 0x39, 0x61, 0x2d, 0x66, 0x41, 0x2d, 0x46, + 0x5d, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x29, 0x2a, 0x20, + 0x22, 0x5c, 0x5c, 0x22, 0x22, 0x20, 0x73, 0x70, 0x61, 0x63, 0x65, 0x60, + 0x2c, 0x0a, 0x20, 0x20, 0x6e, 0x75, 0x6c, 0x6c, 0x3a, 0x20, 0x27, 0x22, + 0x6e, 0x75, 0x6c, 0x6c, 0x22, 0x20, 0x73, 0x70, 0x61, 0x63, 0x65, 0x27, + 0x2c, 0x0a, 0x7d, 0x3b, 0x0a, 0x0a, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, + 0x49, 0x4e, 0x56, 0x41, 0x4c, 0x49, 0x44, 0x5f, 0x52, 0x55, 0x4c, 0x45, + 0x5f, 0x43, 0x48, 0x41, 0x52, 0x53, 0x5f, 0x52, 0x45, 0x20, 0x3d, 0x20, + 0x2f, 0x5b, 0x5e, 0x5c, 0x64, 0x41, 0x2d, 0x5a, 0x61, 0x2d, 0x7a, 0x2d, + 0x5d, 0x2b, 0x2f, 0x67, 0x3b, 0x0a, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, + 0x47, 0x52, 0x41, 0x4d, 0x4d, 0x41, 0x52, 0x5f, 0x4c, 0x49, 0x54, 0x45, + 0x52, 0x41, 0x4c, 0x5f, 0x45, 0x53, 0x43, 0x41, 0x50, 0x45, 0x5f, 0x52, + 0x45, 0x20, 0x3d, 0x20, 0x2f, 0x5b, 0x5c, 0x6e, 0x5c, 0x72, 0x22, 0x5d, + 0x2f, 0x67, 0x3b, 0x0a, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x47, 0x52, + 0x41, 0x4d, 0x4d, 0x41, 0x52, 0x5f, 0x4c, 0x49, 0x54, 0x45, 0x52, 0x41, + 0x4c, 0x5f, 0x45, 0x53, 0x43, 0x41, 0x50, 0x45, 0x53, 0x20, 0x3d, 0x20, + 0x7b, 0x27, 0x5c, 0x72, 0x27, 0x3a, 0x20, 0x27, 0x5c, 0x5c, 0x72, 0x27, + 0x2c, 0x20, 0x27, 0x5c, 0x6e, 0x27, 0x3a, 0x20, 0x27, 0x5c, 0x5c, 0x6e, + 0x27, 0x2c, 0x20, 0x27, 0x22, 0x27, 0x3a, 0x20, 0x27, 0x5c, 0x5c, 0x22, + 0x27, 0x7d, 0x3b, 0x0a, 0x0a, 0x65, 0x78, 0x70, 0x6f, 0x72, 0x74, 0x20, + 0x63, 0x6c, 0x61, 0x73, 0x73, 0x20, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, + 0x43, 0x6f, 0x6e, 0x76, 0x65, 0x72, 0x74, 0x65, 0x72, 0x20, 0x7b, 0x0a, + 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x72, 0x75, 0x63, 0x74, 0x6f, + 0x72, 0x28, 0x70, 0x72, 0x6f, 0x70, 0x4f, 0x72, 0x64, 0x65, 0x72, 0x29, + 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, + 0x5f, 0x70, 0x72, 0x6f, 0x70, 0x4f, 0x72, 0x64, 0x65, 0x72, 0x20, 0x3d, + 0x20, 0x70, 0x72, 0x6f, 0x70, 0x4f, 0x72, 0x64, 0x65, 0x72, 0x20, 0x7c, + 0x7c, 0x20, 0x7b, 0x7d, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x74, 0x68, + 0x69, 0x73, 0x2e, 0x5f, 0x72, 0x75, 0x6c, 0x65, 0x73, 0x20, 0x3d, 0x20, + 0x6e, 0x65, 0x77, 0x20, 0x4d, 0x61, 0x70, 0x28, 0x29, 0x3b, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x5f, 0x72, 0x75, 0x6c, + 0x65, 0x73, 0x2e, 0x73, 0x65, 0x74, 0x28, 0x27, 0x73, 0x70, 0x61, 0x63, + 0x65, 0x27, 0x2c, 0x20, 0x53, 0x50, 0x41, 0x43, 0x45, 0x5f, 0x52, 0x55, + 0x4c, 0x45, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x7d, 0x0a, 0x0a, 0x20, 0x20, + 0x5f, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x4c, 0x69, 0x74, 0x65, 0x72, + 0x61, 0x6c, 0x28, 0x6c, 0x69, 0x74, 0x65, 0x72, 0x61, 0x6c, 0x29, 0x20, + 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, + 0x65, 0x73, 0x63, 0x61, 0x70, 0x65, 0x64, 0x20, 0x3d, 0x20, 0x4a, 0x53, + 0x4f, 0x4e, 0x2e, 0x73, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x69, 0x66, 0x79, + 0x28, 0x6c, 0x69, 0x74, 0x65, 0x72, 0x61, 0x6c, 0x29, 0x2e, 0x72, 0x65, + 0x70, 0x6c, 0x61, 0x63, 0x65, 0x28, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x47, 0x52, 0x41, 0x4d, 0x4d, 0x41, 0x52, 0x5f, 0x4c, 0x49, 0x54, + 0x45, 0x52, 0x41, 0x4c, 0x5f, 0x45, 0x53, 0x43, 0x41, 0x50, 0x45, 0x5f, + 0x52, 0x45, 0x2c, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x6d, 0x20, + 0x3d, 0x3e, 0x20, 0x47, 0x52, 0x41, 0x4d, 0x4d, 0x41, 0x52, 0x5f, 0x4c, + 0x49, 0x54, 0x45, 0x52, 0x41, 0x4c, 0x5f, 0x45, 0x53, 0x43, 0x41, 0x50, + 0x45, 0x53, 0x5b, 0x6d, 0x5d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x29, 0x3b, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, + 0x60, 0x22, 0x24, 0x7b, 0x65, 0x73, 0x63, 0x61, 0x70, 0x65, 0x64, 0x7d, + 0x22, 0x60, 0x3b, 0x0a, 0x20, 0x20, 0x7d, 0x0a, 0x0a, 0x20, 0x20, 0x5f, + 0x61, 0x64, 0x64, 0x52, 0x75, 0x6c, 0x65, 0x28, 0x6e, 0x61, 0x6d, 0x65, + 0x2c, 0x20, 0x72, 0x75, 0x6c, 0x65, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x6c, 0x65, 0x74, 0x20, 0x65, 0x73, 0x63, 0x4e, 0x61, 0x6d, + 0x65, 0x20, 0x3d, 0x20, 0x6e, 0x61, 0x6d, 0x65, 0x2e, 0x72, 0x65, 0x70, + 0x6c, 0x61, 0x63, 0x65, 0x28, 0x49, 0x4e, 0x56, 0x41, 0x4c, 0x49, 0x44, + 0x5f, 0x52, 0x55, 0x4c, 0x45, 0x5f, 0x43, 0x48, 0x41, 0x52, 0x53, 0x5f, + 0x52, 0x45, 0x2c, 0x20, 0x27, 0x2d, 0x27, 0x29, 0x3b, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x6c, 0x65, 0x74, 0x20, 0x6b, 0x65, 0x79, 0x20, 0x3d, 0x20, + 0x65, 0x73, 0x63, 0x4e, 0x61, 0x6d, 0x65, 0x3b, 0x0a, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x5f, + 0x72, 0x75, 0x6c, 0x65, 0x73, 0x2e, 0x68, 0x61, 0x73, 0x28, 0x65, 0x73, + 0x63, 0x4e, 0x61, 0x6d, 0x65, 0x29, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, 0x74, 0x68, 0x69, 0x73, + 0x2e, 0x5f, 0x72, 0x75, 0x6c, 0x65, 0x73, 0x2e, 0x67, 0x65, 0x74, 0x28, + 0x65, 0x73, 0x63, 0x4e, 0x61, 0x6d, 0x65, 0x29, 0x20, 0x3d, 0x3d, 0x3d, + 0x20, 0x72, 0x75, 0x6c, 0x65, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, + 0x6b, 0x65, 0x79, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, + 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x6c, 0x65, 0x74, 0x20, + 0x69, 0x20, 0x3d, 0x20, 0x30, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x77, 0x68, 0x69, 0x6c, 0x65, 0x20, 0x28, 0x74, 0x68, 0x69, 0x73, + 0x2e, 0x5f, 0x72, 0x75, 0x6c, 0x65, 0x73, 0x2e, 0x68, 0x61, 0x73, 0x28, + 0x60, 0x24, 0x7b, 0x65, 0x73, 0x63, 0x4e, 0x61, 0x6d, 0x65, 0x7d, 0x24, + 0x7b, 0x69, 0x7d, 0x60, 0x29, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x20, 0x2b, 0x3d, 0x20, 0x31, 0x3b, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x6b, 0x65, 0x79, 0x20, 0x3d, 0x20, 0x60, 0x24, 0x7b, + 0x65, 0x73, 0x63, 0x4e, 0x61, 0x6d, 0x65, 0x7d, 0x24, 0x7b, 0x69, 0x7d, + 0x60, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x5f, 0x72, 0x75, 0x6c, 0x65, + 0x73, 0x2e, 0x73, 0x65, 0x74, 0x28, 0x6b, 0x65, 0x79, 0x2c, 0x20, 0x72, + 0x75, 0x6c, 0x65, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, + 0x74, 0x75, 0x72, 0x6e, 0x20, 0x6b, 0x65, 0x79, 0x3b, 0x0a, 0x20, 0x20, + 0x7d, 0x0a, 0x0a, 0x20, 0x20, 0x76, 0x69, 0x73, 0x69, 0x74, 0x28, 0x73, + 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2c, 0x20, 0x6e, 0x61, 0x6d, 0x65, 0x29, + 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, + 0x20, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x54, 0x79, 0x70, 0x65, 0x20, + 0x3d, 0x20, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2e, 0x74, 0x79, 0x70, + 0x65, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, + 0x20, 0x72, 0x75, 0x6c, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x20, 0x3d, 0x20, + 0x6e, 0x61, 0x6d, 0x65, 0x20, 0x7c, 0x7c, 0x20, 0x27, 0x72, 0x6f, 0x6f, + 0x74, 0x27, 0x3b, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, + 0x28, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2e, 0x6f, 0x6e, 0x65, 0x4f, + 0x66, 0x20, 0x7c, 0x7c, 0x20, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2e, + 0x61, 0x6e, 0x79, 0x4f, 0x66, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x72, 0x75, 0x6c, + 0x65, 0x20, 0x3d, 0x20, 0x28, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2e, + 0x6f, 0x6e, 0x65, 0x4f, 0x66, 0x20, 0x7c, 0x7c, 0x20, 0x73, 0x63, 0x68, + 0x65, 0x6d, 0x61, 0x2e, 0x61, 0x6e, 0x79, 0x4f, 0x66, 0x29, 0x2e, 0x6d, + 0x61, 0x70, 0x28, 0x28, 0x61, 0x6c, 0x74, 0x53, 0x63, 0x68, 0x65, 0x6d, + 0x61, 0x2c, 0x20, 0x69, 0x29, 0x20, 0x3d, 0x3e, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x76, 0x69, + 0x73, 0x69, 0x74, 0x28, 0x61, 0x6c, 0x74, 0x53, 0x63, 0x68, 0x65, 0x6d, + 0x61, 0x2c, 0x20, 0x60, 0x24, 0x7b, 0x6e, 0x61, 0x6d, 0x65, 0x7d, 0x24, + 0x7b, 0x6e, 0x61, 0x6d, 0x65, 0x20, 0x3f, 0x20, 0x22, 0x2d, 0x22, 0x20, + 0x3a, 0x20, 0x22, 0x22, 0x7d, 0x24, 0x7b, 0x69, 0x7d, 0x60, 0x29, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x29, 0x2e, 0x6a, 0x6f, 0x69, 0x6e, + 0x28, 0x27, 0x20, 0x7c, 0x20, 0x27, 0x29, 0x3b, 0x0a, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x74, + 0x68, 0x69, 0x73, 0x2e, 0x5f, 0x61, 0x64, 0x64, 0x52, 0x75, 0x6c, 0x65, + 0x28, 0x72, 0x75, 0x6c, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x2c, 0x20, 0x72, + 0x75, 0x6c, 0x65, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x20, + 0x65, 0x6c, 0x73, 0x65, 0x20, 0x69, 0x66, 0x20, 0x28, 0x27, 0x63, 0x6f, + 0x6e, 0x73, 0x74, 0x27, 0x20, 0x69, 0x6e, 0x20, 0x73, 0x63, 0x68, 0x65, + 0x6d, 0x61, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, + 0x5f, 0x61, 0x64, 0x64, 0x52, 0x75, 0x6c, 0x65, 0x28, 0x72, 0x75, 0x6c, + 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x2c, 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, + 0x5f, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x4c, 0x69, 0x74, 0x65, 0x72, + 0x61, 0x6c, 0x28, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2e, 0x63, 0x6f, + 0x6e, 0x73, 0x74, 0x29, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x7d, + 0x20, 0x65, 0x6c, 0x73, 0x65, 0x20, 0x69, 0x66, 0x20, 0x28, 0x27, 0x65, + 0x6e, 0x75, 0x6d, 0x27, 0x20, 0x69, 0x6e, 0x20, 0x73, 0x63, 0x68, 0x65, + 0x6d, 0x61, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x72, 0x75, 0x6c, 0x65, 0x20, 0x3d, + 0x20, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2e, 0x65, 0x6e, 0x75, 0x6d, + 0x2e, 0x6d, 0x61, 0x70, 0x28, 0x76, 0x20, 0x3d, 0x3e, 0x20, 0x74, 0x68, + 0x69, 0x73, 0x2e, 0x5f, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x4c, 0x69, + 0x74, 0x65, 0x72, 0x61, 0x6c, 0x28, 0x76, 0x29, 0x29, 0x2e, 0x6a, 0x6f, + 0x69, 0x6e, 0x28, 0x27, 0x20, 0x7c, 0x20, 0x27, 0x29, 0x3b, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, + 0x74, 0x68, 0x69, 0x73, 0x2e, 0x5f, 0x61, 0x64, 0x64, 0x52, 0x75, 0x6c, + 0x65, 0x28, 0x72, 0x75, 0x6c, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x2c, 0x20, + 0x72, 0x75, 0x6c, 0x65, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x7d, + 0x20, 0x65, 0x6c, 0x73, 0x65, 0x20, 0x69, 0x66, 0x20, 0x28, 0x73, 0x63, + 0x68, 0x65, 0x6d, 0x61, 0x54, 0x79, 0x70, 0x65, 0x20, 0x3d, 0x3d, 0x3d, + 0x20, 0x27, 0x6f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x27, 0x20, 0x26, 0x26, + 0x20, 0x27, 0x70, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, + 0x27, 0x20, 0x69, 0x6e, 0x20, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x29, + 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x2f, 0x2f, 0x20, + 0x54, 0x4f, 0x44, 0x4f, 0x3a, 0x20, 0x60, 0x72, 0x65, 0x71, 0x75, 0x69, + 0x72, 0x65, 0x64, 0x60, 0x20, 0x6b, 0x65, 0x79, 0x77, 0x6f, 0x72, 0x64, + 0x20, 0x28, 0x66, 0x72, 0x6f, 0x6d, 0x20, 0x70, 0x79, 0x74, 0x68, 0x6f, + 0x6e, 0x20, 0x69, 0x6d, 0x70, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x61, + 0x74, 0x69, 0x6f, 0x6e, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x70, 0x72, 0x6f, 0x70, 0x4f, 0x72, + 0x64, 0x65, 0x72, 0x20, 0x3d, 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x5f, + 0x70, 0x72, 0x6f, 0x70, 0x4f, 0x72, 0x64, 0x65, 0x72, 0x3b, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x70, + 0x72, 0x6f, 0x70, 0x50, 0x61, 0x69, 0x72, 0x73, 0x20, 0x3d, 0x20, 0x4f, + 0x62, 0x6a, 0x65, 0x63, 0x74, 0x2e, 0x65, 0x6e, 0x74, 0x72, 0x69, 0x65, + 0x73, 0x28, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2e, 0x70, 0x72, 0x6f, + 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x29, 0x2e, 0x73, 0x6f, 0x72, + 0x74, 0x28, 0x28, 0x61, 0x2c, 0x20, 0x62, 0x29, 0x20, 0x3d, 0x3e, 0x20, + 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x2f, 0x2f, + 0x20, 0x73, 0x6f, 0x72, 0x74, 0x20, 0x62, 0x79, 0x20, 0x70, 0x6f, 0x73, + 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x69, 0x6e, 0x20, 0x70, 0x72, 0x6f, + 0x70, 0x5f, 0x6f, 0x72, 0x64, 0x65, 0x72, 0x20, 0x28, 0x69, 0x66, 0x20, + 0x73, 0x70, 0x65, 0x63, 0x69, 0x66, 0x69, 0x65, 0x64, 0x29, 0x20, 0x74, + 0x68, 0x65, 0x6e, 0x20, 0x62, 0x79, 0x20, 0x6b, 0x65, 0x79, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, + 0x20, 0x6f, 0x72, 0x64, 0x65, 0x72, 0x41, 0x20, 0x3d, 0x20, 0x74, 0x79, + 0x70, 0x65, 0x6f, 0x66, 0x20, 0x70, 0x72, 0x6f, 0x70, 0x4f, 0x72, 0x64, + 0x65, 0x72, 0x5b, 0x61, 0x5b, 0x30, 0x5d, 0x5d, 0x20, 0x3d, 0x3d, 0x3d, + 0x20, 0x27, 0x6e, 0x75, 0x6d, 0x62, 0x65, 0x72, 0x27, 0x20, 0x3f, 0x20, + 0x70, 0x72, 0x6f, 0x70, 0x4f, 0x72, 0x64, 0x65, 0x72, 0x5b, 0x61, 0x5b, + 0x30, 0x5d, 0x5d, 0x20, 0x3a, 0x20, 0x49, 0x6e, 0x66, 0x69, 0x6e, 0x69, + 0x74, 0x79, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x6f, 0x72, 0x64, 0x65, 0x72, 0x42, + 0x20, 0x3d, 0x20, 0x74, 0x79, 0x70, 0x65, 0x6f, 0x66, 0x20, 0x70, 0x72, + 0x6f, 0x70, 0x4f, 0x72, 0x64, 0x65, 0x72, 0x5b, 0x62, 0x5b, 0x30, 0x5d, + 0x5d, 0x20, 0x3d, 0x3d, 0x3d, 0x20, 0x27, 0x6e, 0x75, 0x6d, 0x62, 0x65, + 0x72, 0x27, 0x20, 0x3f, 0x20, 0x70, 0x72, 0x6f, 0x70, 0x4f, 0x72, 0x64, + 0x65, 0x72, 0x5b, 0x62, 0x5b, 0x30, 0x5d, 0x5d, 0x20, 0x3a, 0x20, 0x49, + 0x6e, 0x66, 0x69, 0x6e, 0x69, 0x74, 0x79, 0x3b, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, + 0x6f, 0x72, 0x64, 0x65, 0x72, 0x41, 0x20, 0x2d, 0x20, 0x6f, 0x72, 0x64, + 0x65, 0x72, 0x42, 0x20, 0x7c, 0x7c, 0x20, 0x61, 0x5b, 0x30, 0x5d, 0x2e, + 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x65, 0x43, 0x6f, 0x6d, 0x70, 0x61, 0x72, + 0x65, 0x28, 0x62, 0x5b, 0x30, 0x5d, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x7d, 0x29, 0x3b, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x6c, 0x65, 0x74, 0x20, 0x72, 0x75, 0x6c, 0x65, 0x20, 0x3d, + 0x20, 0x27, 0x22, 0x7b, 0x22, 0x20, 0x73, 0x70, 0x61, 0x63, 0x65, 0x27, + 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x70, 0x72, 0x6f, 0x70, + 0x50, 0x61, 0x69, 0x72, 0x73, 0x2e, 0x66, 0x6f, 0x72, 0x45, 0x61, 0x63, + 0x68, 0x28, 0x28, 0x5b, 0x70, 0x72, 0x6f, 0x70, 0x4e, 0x61, 0x6d, 0x65, + 0x2c, 0x20, 0x70, 0x72, 0x6f, 0x70, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, + 0x5d, 0x2c, 0x20, 0x69, 0x29, 0x20, 0x3d, 0x3e, 0x20, 0x7b, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, + 0x20, 0x70, 0x72, 0x6f, 0x70, 0x52, 0x75, 0x6c, 0x65, 0x4e, 0x61, 0x6d, + 0x65, 0x20, 0x3d, 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x76, 0x69, 0x73, + 0x69, 0x74, 0x28, 0x70, 0x72, 0x6f, 0x70, 0x53, 0x63, 0x68, 0x65, 0x6d, + 0x61, 0x2c, 0x20, 0x60, 0x24, 0x7b, 0x6e, 0x61, 0x6d, 0x65, 0x7d, 0x24, + 0x7b, 0x6e, 0x61, 0x6d, 0x65, 0x20, 0x3f, 0x20, 0x22, 0x2d, 0x22, 0x20, + 0x3a, 0x20, 0x22, 0x22, 0x7d, 0x24, 0x7b, 0x70, 0x72, 0x6f, 0x70, 0x4e, + 0x61, 0x6d, 0x65, 0x7d, 0x60, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, 0x69, 0x20, 0x3e, 0x20, + 0x30, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x72, 0x75, 0x6c, 0x65, 0x20, 0x2b, 0x3d, 0x20, 0x27, + 0x20, 0x22, 0x2c, 0x22, 0x20, 0x73, 0x70, 0x61, 0x63, 0x65, 0x27, 0x3b, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, 0x75, 0x6c, 0x65, 0x20, + 0x2b, 0x3d, 0x20, 0x60, 0x20, 0x24, 0x7b, 0x74, 0x68, 0x69, 0x73, 0x2e, + 0x5f, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x4c, 0x69, 0x74, 0x65, 0x72, + 0x61, 0x6c, 0x28, 0x70, 0x72, 0x6f, 0x70, 0x4e, 0x61, 0x6d, 0x65, 0x29, + 0x7d, 0x20, 0x73, 0x70, 0x61, 0x63, 0x65, 0x20, 0x22, 0x3a, 0x22, 0x20, + 0x73, 0x70, 0x61, 0x63, 0x65, 0x20, 0x24, 0x7b, 0x70, 0x72, 0x6f, 0x70, + 0x52, 0x75, 0x6c, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x7d, 0x60, 0x3b, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x29, 0x3b, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x72, 0x75, 0x6c, 0x65, 0x20, 0x2b, 0x3d, 0x20, + 0x27, 0x20, 0x22, 0x7d, 0x22, 0x20, 0x73, 0x70, 0x61, 0x63, 0x65, 0x27, + 0x3b, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, 0x74, + 0x75, 0x72, 0x6e, 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x5f, 0x61, 0x64, + 0x64, 0x52, 0x75, 0x6c, 0x65, 0x28, 0x72, 0x75, 0x6c, 0x65, 0x4e, 0x61, + 0x6d, 0x65, 0x2c, 0x20, 0x72, 0x75, 0x6c, 0x65, 0x29, 0x3b, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x7d, 0x20, 0x65, 0x6c, 0x73, 0x65, 0x20, 0x69, 0x66, + 0x20, 0x28, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x54, 0x79, 0x70, 0x65, + 0x20, 0x3d, 0x3d, 0x3d, 0x20, 0x27, 0x61, 0x72, 0x72, 0x61, 0x79, 0x27, + 0x20, 0x26, 0x26, 0x20, 0x27, 0x69, 0x74, 0x65, 0x6d, 0x73, 0x27, 0x20, + 0x69, 0x6e, 0x20, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x29, 0x20, 0x7b, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x2f, 0x2f, 0x20, 0x54, 0x4f, + 0x44, 0x4f, 0x20, 0x60, 0x70, 0x72, 0x65, 0x66, 0x69, 0x78, 0x49, 0x74, + 0x65, 0x6d, 0x73, 0x60, 0x20, 0x6b, 0x65, 0x79, 0x77, 0x6f, 0x72, 0x64, + 0x20, 0x28, 0x66, 0x72, 0x6f, 0x6d, 0x20, 0x70, 0x79, 0x74, 0x68, 0x6f, + 0x6e, 0x20, 0x69, 0x6d, 0x70, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x61, + 0x74, 0x69, 0x6f, 0x6e, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x69, 0x74, 0x65, 0x6d, 0x52, 0x75, + 0x6c, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x20, 0x3d, 0x20, 0x74, 0x68, 0x69, + 0x73, 0x2e, 0x76, 0x69, 0x73, 0x69, 0x74, 0x28, 0x73, 0x63, 0x68, 0x65, + 0x6d, 0x61, 0x2e, 0x69, 0x74, 0x65, 0x6d, 0x73, 0x2c, 0x20, 0x60, 0x24, + 0x7b, 0x6e, 0x61, 0x6d, 0x65, 0x7d, 0x24, 0x7b, 0x6e, 0x61, 0x6d, 0x65, + 0x20, 0x3f, 0x20, 0x22, 0x2d, 0x22, 0x20, 0x3a, 0x20, 0x22, 0x22, 0x7d, + 0x69, 0x74, 0x65, 0x6d, 0x60, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x72, 0x75, 0x6c, 0x65, + 0x20, 0x3d, 0x20, 0x60, 0x22, 0x5b, 0x22, 0x20, 0x73, 0x70, 0x61, 0x63, + 0x65, 0x20, 0x28, 0x24, 0x7b, 0x69, 0x74, 0x65, 0x6d, 0x52, 0x75, 0x6c, + 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x7d, 0x20, 0x28, 0x22, 0x2c, 0x22, 0x20, + 0x73, 0x70, 0x61, 0x63, 0x65, 0x20, 0x24, 0x7b, 0x69, 0x74, 0x65, 0x6d, + 0x52, 0x75, 0x6c, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x7d, 0x29, 0x2a, 0x29, + 0x3f, 0x20, 0x22, 0x5d, 0x22, 0x20, 0x73, 0x70, 0x61, 0x63, 0x65, 0x60, + 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, + 0x72, 0x6e, 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x5f, 0x61, 0x64, 0x64, + 0x52, 0x75, 0x6c, 0x65, 0x28, 0x72, 0x75, 0x6c, 0x65, 0x4e, 0x61, 0x6d, + 0x65, 0x2c, 0x20, 0x72, 0x75, 0x6c, 0x65, 0x29, 0x3b, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x7d, 0x20, 0x65, 0x6c, 0x73, 0x65, 0x20, 0x7b, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, 0x21, 0x50, 0x52, + 0x49, 0x4d, 0x49, 0x54, 0x49, 0x56, 0x45, 0x5f, 0x52, 0x55, 0x4c, 0x45, + 0x53, 0x5b, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x54, 0x79, 0x70, 0x65, + 0x5d, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x74, 0x68, 0x72, 0x6f, 0x77, 0x20, 0x6e, 0x65, 0x77, 0x20, 0x45, + 0x72, 0x72, 0x6f, 0x72, 0x28, 0x60, 0x55, 0x6e, 0x72, 0x65, 0x63, 0x6f, + 0x67, 0x6e, 0x69, 0x7a, 0x65, 0x64, 0x20, 0x73, 0x63, 0x68, 0x65, 0x6d, + 0x61, 0x3a, 0x20, 0x24, 0x7b, 0x4a, 0x53, 0x4f, 0x4e, 0x2e, 0x73, 0x74, + 0x72, 0x69, 0x6e, 0x67, 0x69, 0x66, 0x79, 0x28, 0x73, 0x63, 0x68, 0x65, + 0x6d, 0x61, 0x29, 0x7d, 0x60, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, + 0x74, 0x75, 0x72, 0x6e, 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x5f, 0x61, + 0x64, 0x64, 0x52, 0x75, 0x6c, 0x65, 0x28, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x72, 0x75, 0x6c, 0x65, 0x4e, 0x61, 0x6d, 0x65, + 0x20, 0x3d, 0x3d, 0x3d, 0x20, 0x27, 0x72, 0x6f, 0x6f, 0x74, 0x27, 0x20, + 0x3f, 0x20, 0x27, 0x72, 0x6f, 0x6f, 0x74, 0x27, 0x20, 0x3a, 0x20, 0x73, + 0x63, 0x68, 0x65, 0x6d, 0x61, 0x54, 0x79, 0x70, 0x65, 0x2c, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x50, 0x52, 0x49, 0x4d, 0x49, + 0x54, 0x49, 0x56, 0x45, 0x5f, 0x52, 0x55, 0x4c, 0x45, 0x53, 0x5b, 0x73, + 0x63, 0x68, 0x65, 0x6d, 0x61, 0x54, 0x79, 0x70, 0x65, 0x5d, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x7d, 0x0a, 0x20, 0x20, 0x7d, 0x0a, 0x0a, 0x20, 0x20, 0x66, 0x6f, 0x72, + 0x6d, 0x61, 0x74, 0x47, 0x72, 0x61, 0x6d, 0x6d, 0x61, 0x72, 0x28, 0x29, + 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x6c, 0x65, 0x74, 0x20, 0x67, + 0x72, 0x61, 0x6d, 0x6d, 0x61, 0x72, 0x20, 0x3d, 0x20, 0x27, 0x27, 0x3b, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x5f, 0x72, + 0x75, 0x6c, 0x65, 0x73, 0x2e, 0x66, 0x6f, 0x72, 0x45, 0x61, 0x63, 0x68, + 0x28, 0x28, 0x72, 0x75, 0x6c, 0x65, 0x2c, 0x20, 0x6e, 0x61, 0x6d, 0x65, + 0x29, 0x20, 0x3d, 0x3e, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x67, 0x72, 0x61, 0x6d, 0x6d, 0x61, 0x72, 0x20, 0x2b, 0x3d, 0x20, + 0x60, 0x24, 0x7b, 0x6e, 0x61, 0x6d, 0x65, 0x7d, 0x20, 0x3a, 0x3a, 0x3d, + 0x20, 0x24, 0x7b, 0x72, 0x75, 0x6c, 0x65, 0x7d, 0x5c, 0x6e, 0x60, 0x3b, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x67, 0x72, 0x61, 0x6d, + 0x6d, 0x61, 0x72, 0x3b, 0x0a, 0x20, 0x20, 0x7d, 0x0a, 0x7d, 0x0a }; - -const INVALID_RULE_CHARS_RE = /[^\dA-Za-z-]+/g; -const GRAMMAR_LITERAL_ESCAPE_RE = /[\n\r"]/g; -const GRAMMAR_LITERAL_ESCAPES = {'\r': '\\r', '\n': '\\n', '"': '\\"'}; - -export class SchemaConverter { - constructor(propOrder) { - this._propOrder = propOrder || {}; - this._rules = new Map(); - this._rules.set('space', SPACE_RULE); - } - - _formatLiteral(literal) { - const escaped = JSON.stringify(literal).replace( - GRAMMAR_LITERAL_ESCAPE_RE, - m => GRAMMAR_LITERAL_ESCAPES[m] - ); - return `"${escaped}"`; - } - - _addRule(name, rule) { - let escName = name.replace(INVALID_RULE_CHARS_RE, '-'); - let key = escName; - - if (this._rules.has(escName)) { - if (this._rules.get(escName) === rule) { - return key; - } - - let i = 0; - while (this._rules.has(`${escName}${i}`)) { - i += 1; - } - key = `${escName}${i}`; - } - - this._rules.set(key, rule); - return key; - } - - visit(schema, name) { - const schemaType = schema.type; - const ruleName = name || 'root'; - - if (schema.oneOf || schema.anyOf) { - const rule = (schema.oneOf || schema.anyOf).map((altSchema, i) => - this.visit(altSchema, `${name}${name ? "-" : ""}${i}`) - ).join(' | '); - - return this._addRule(ruleName, rule); - } else if ('const' in schema) { - return this._addRule(ruleName, this._formatLiteral(schema.const)); - } else if ('enum' in schema) { - const rule = schema.enum.map(v => this._formatLiteral(v)).join(' | '); - return this._addRule(ruleName, rule); - } else if (schemaType === 'object' && 'properties' in schema) { - // TODO: `required` keyword (from python implementation) - const propOrder = this._propOrder; - const propPairs = Object.entries(schema.properties).sort((a, b) => { - // sort by position in prop_order (if specified) then by key - const orderA = typeof propOrder[a[0]] === 'number' ? propOrder[a[0]] : Infinity; - const orderB = typeof propOrder[b[0]] === 'number' ? propOrder[b[0]] : Infinity; - return orderA - orderB || a[0].localeCompare(b[0]); - }); - - let rule = '"{" space'; - propPairs.forEach(([propName, propSchema], i) => { - const propRuleName = this.visit(propSchema, `${name}${name ? "-" : ""}${propName}`); - if (i > 0) { - rule += ' "," space'; - } - rule += ` ${this._formatLiteral(propName)} space ":" space ${propRuleName}`; - }); - rule += ' "}" space'; - - return this._addRule(ruleName, rule); - } else if (schemaType === 'array' && 'items' in schema) { - // TODO `prefixItems` keyword (from python implementation) - const itemRuleName = this.visit(schema.items, `${name}${name ? "-" : ""}item`); - const rule = `"[" space (${itemRuleName} ("," space ${itemRuleName})*)? "]" space`; - return this._addRule(ruleName, rule); - } else { - if (!PRIMITIVE_RULES[schemaType]) { - throw new Error(`Unrecognized schema: ${JSON.stringify(schema)}`); - } - return this._addRule( - ruleName === 'root' ? 'root' : schemaType, - PRIMITIVE_RULES[schemaType] - ); - } - } - - formatGrammar() { - let grammar = ''; - this._rules.forEach((rule, name) => { - grammar += `${name} ::= ${rule}\n`; - }); - return grammar; - } -} -)LITERAL"; -unsigned int json_schema_to_grammar_mjs_len = sizeof(json_schema_to_grammar_mjs); +unsigned int json_schema_to_grammar_mjs_len = 3695; From e8dc55d0065d076d4c20f3c4bfca562701b4edfe Mon Sep 17 00:00:00 2001 From: Jared Van Bortel Date: Tue, 30 Jan 2024 19:04:37 -0500 Subject: [PATCH 223/812] kompute : llama-bench support and ggml_cpu_has_kompute() (#5226) --- common/common.cpp | 1 + examples/llama-bench/llama-bench.cpp | 15 +++++++++++---- ggml.c | 11 ++++++++++- ggml.h | 1 + llama.cpp | 5 ----- 5 files changed, 23 insertions(+), 10 deletions(-) diff --git a/common/common.cpp b/common/common.cpp index 288013676..0dd1c50cf 100644 --- a/common/common.cpp +++ b/common/common.cpp @@ -1521,6 +1521,7 @@ void dump_non_result_info_yaml(FILE * stream, const gpt_params & params, const l fprintf(stream, "cpu_has_avx512_vnni: %s\n", ggml_cpu_has_avx512_vnni() ? "true" : "false"); fprintf(stream, "cpu_has_cublas: %s\n", ggml_cpu_has_cublas() ? "true" : "false"); fprintf(stream, "cpu_has_clblast: %s\n", ggml_cpu_has_clblast() ? "true" : "false"); + fprintf(stream, "cpu_has_kompute: %s\n", ggml_cpu_has_kompute() ? "true" : "false"); fprintf(stream, "cpu_has_fma: %s\n", ggml_cpu_has_fma() ? "true" : "false"); fprintf(stream, "cpu_has_gpublas: %s\n", ggml_cpu_has_gpublas() ? "true" : "false"); fprintf(stream, "cpu_has_neon: %s\n", ggml_cpu_has_neon() ? "true" : "false"); diff --git a/examples/llama-bench/llama-bench.cpp b/examples/llama-bench/llama-bench.cpp index f239415d3..542cc7bb8 100644 --- a/examples/llama-bench/llama-bench.cpp +++ b/examples/llama-bench/llama-bench.cpp @@ -563,6 +563,7 @@ struct test { static const bool cuda; static const bool opencl; static const bool vulkan; + static const bool kompute; static const bool metal; static const bool gpu_blas; static const bool blas; @@ -647,6 +648,9 @@ struct test { if (vulkan) { return "Vulkan"; } + if (kompute) { + return "Kompute"; + } if (metal) { return "Metal"; } @@ -662,7 +666,7 @@ struct test { static const std::vector & get_fields() { static const std::vector fields = { "build_commit", "build_number", - "cuda", "opencl", "vulkan", "metal", "gpu_blas", "blas", + "cuda", "opencl", "vulkan", "kompute", "metal", "gpu_blas", "blas", "cpu_info", "gpu_info", "model_filename", "model_type", "model_size", "model_n_params", "n_batch", "n_threads", "type_k", "type_v", @@ -686,8 +690,9 @@ struct test { field == "avg_ns" || field == "stddev_ns") { return INT; } - if (field == "cuda" || field == "opencl" || field == "vulkan"|| field == "metal" || field == "gpu_blas" || field == "blas" || - field == "f16_kv" || field == "no_kv_offload" || field == "mul_mat_q") { + if (field == "cuda" || field == "opencl" || field == "vulkan" || field == "kompute" || field == "metal" || + field == "gpu_blas" || field == "blas" || field == "f16_kv" || field == "no_kv_offload" || + field == "mul_mat_q") { return BOOL; } if (field == "avg_ts" || field == "stddev_ts") { @@ -714,7 +719,8 @@ struct test { } std::vector values = { build_commit, std::to_string(build_number), - std::to_string(cuda), std::to_string(opencl), std::to_string(vulkan), std::to_string(metal), std::to_string(gpu_blas), std::to_string(blas), + std::to_string(cuda), std::to_string(opencl), std::to_string(vulkan), std::to_string(vulkan), + std::to_string(metal), std::to_string(gpu_blas), std::to_string(blas), cpu_info, gpu_info, model_filename, model_type, std::to_string(model_size), std::to_string(model_n_params), std::to_string(n_batch), std::to_string(n_threads), ggml_type_name(type_k), ggml_type_name(type_v), @@ -743,6 +749,7 @@ const int test::build_number = LLAMA_BUILD_NUMBER; const bool test::cuda = !!ggml_cpu_has_cublas(); const bool test::opencl = !!ggml_cpu_has_clblast(); const bool test::vulkan = !!ggml_cpu_has_vulkan(); +const bool test::kompute = !!ggml_cpu_has_kompute(); const bool test::metal = !!ggml_cpu_has_metal(); const bool test::gpu_blas = !!ggml_cpu_has_gpublas(); const bool test::blas = !!ggml_cpu_has_blas(); diff --git a/ggml.c b/ggml.c index a7a9ea319..b2c8baaa8 100644 --- a/ggml.c +++ b/ggml.c @@ -20473,6 +20473,14 @@ int ggml_cpu_has_vulkan(void) { #endif } +int ggml_cpu_has_kompute(void) { +#if defined(GGML_USE_KOMPUTE) + return 1; +#else + return 0; +#endif +} + int ggml_cpu_has_sycl(void) { #if defined(GGML_USE_SYCL) return 1; @@ -20482,7 +20490,8 @@ int ggml_cpu_has_sycl(void) { } int ggml_cpu_has_gpublas(void) { - return ggml_cpu_has_cublas() || ggml_cpu_has_clblast() || ggml_cpu_has_vulkan() || ggml_cpu_has_sycl(); + return ggml_cpu_has_cublas() || ggml_cpu_has_clblast() || ggml_cpu_has_vulkan() || ggml_cpu_has_kompute() || + ggml_cpu_has_sycl(); } int ggml_cpu_has_sse3(void) { diff --git a/ggml.h b/ggml.h index bf782e6ad..afc87b843 100644 --- a/ggml.h +++ b/ggml.h @@ -2266,6 +2266,7 @@ extern "C" { GGML_API int ggml_cpu_has_cublas (void); GGML_API int ggml_cpu_has_clblast (void); GGML_API int ggml_cpu_has_vulkan (void); + GGML_API int ggml_cpu_has_kompute (void); GGML_API int ggml_cpu_has_gpublas (void); GGML_API int ggml_cpu_has_sse3 (void); GGML_API int ggml_cpu_has_ssse3 (void); diff --git a/llama.cpp b/llama.cpp index 7b9a5c079..a490eeab2 100644 --- a/llama.cpp +++ b/llama.cpp @@ -6878,11 +6878,6 @@ static int llama_decode_internal( n_threads = std::min(4, n_threads); } - const bool fully_offloaded = model.n_gpu_layers >= (int) hparams.n_layer + 1; - if ((ggml_cpu_has_cublas() || ggml_cpu_has_vulkan()) && fully_offloaded) { - n_threads = 1; - } - #ifdef GGML_USE_MPI const int64_t n_layer = hparams.n_layer; ggml_mpi_graph_compute_pre(lctx.ctx_mpi, gf, n_layer); From 01684139c352561840ae55ec627ab58abc3e06ab Mon Sep 17 00:00:00 2001 From: Neo Zhang Jianyu Date: Wed, 31 Jan 2024 10:38:07 +0800 Subject: [PATCH 224/812] support SYCL backend windows build (#5208) * support SYCL backend windows build * add windows build in CI * add for win build CI * correct install oneMKL * fix install issue * fix ci * fix install cmd * fix install cmd * fix install cmd * fix install cmd * fix install cmd * fix win build * fix win build * fix win build * restore other CI part * restore as base * rm no new line * fix no new line issue, add -j * fix grammer issue * allow to trigger manually, fix format issue * fix format * add newline * fix format * fix format * fix format issuse --------- Co-authored-by: Abhilash Majumder <30946547+abhilash1910@users.noreply.github.com> --- .github/workflows/build.yml | 25 ++++ .github/workflows/editorconfig.yml | 6 + .gitignore | 1 + CMakeLists.txt | 6 +- README_sycl.md => README-sycl.md | 198 +++++++++++++++++++++++++++-- README.md | 4 +- examples/sycl/win-build-sycl.bat | 23 ++++ examples/sycl/win-run-llama2.bat | 13 ++ scripts/install-oneapi.bat | 19 +++ 9 files changed, 281 insertions(+), 14 deletions(-) rename README_sycl.md => README-sycl.md (58%) create mode 100644 examples/sycl/win-build-sycl.bat create mode 100644 examples/sycl/win-run-llama2.bat create mode 100644 scripts/install-oneapi.bat diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index fb719a550..c6db1666e 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -565,6 +565,31 @@ jobs: path: | cudart-llama-bin-win-cu${{ matrix.cuda }}-x64.zip + windows-latest-cmake-sycl: + runs-on: windows-latest + defaults: + run: + shell: bash + + env: + WINDOWS_BASEKIT_URL: https://registrationcenter-download.intel.com/akdlm/IRC_NAS/62641e01-1e8d-4ace-91d6-ae03f7f8a71f/w_BaseKit_p_2024.0.0.49563_offline.exe + WINDOWS_DPCPP_MKL: intel.oneapi.win.cpp-dpcpp-common:intel.oneapi.win.mkl.devel + + + steps: + - name: Clone + id: checkout + uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Install + run: scripts/install-oneapi.bat $WINDOWS_BASEKIT_URL $WINDOWS_DPCPP_MKL + + - name: Build + id: cmake_build + run: examples/sycl/win-build-sycl.bat + ios-xcode-build: runs-on: macos-latest diff --git a/.github/workflows/editorconfig.yml b/.github/workflows/editorconfig.yml index b4e535acf..0e0993cd4 100644 --- a/.github/workflows/editorconfig.yml +++ b/.github/workflows/editorconfig.yml @@ -1,6 +1,12 @@ name: EditorConfig Checker on: + workflow_dispatch: # allows manual triggering + inputs: + create_release: + description: 'Create new release' + required: true + type: boolean push: branches: - master diff --git a/.gitignore b/.gitignore index cb0069bfb..b84459b92 100644 --- a/.gitignore +++ b/.gitignore @@ -89,3 +89,4 @@ examples/jeopardy/results.txt poetry.lock poetry.toml +nppBackup diff --git a/CMakeLists.txt b/CMakeLists.txt index 65a6f3971..15a1101aa 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -507,7 +507,11 @@ if (LLAMA_SYCL) set(GGML_HEADERS_SYCL ggml.h ggml-sycl.h) set(GGML_SOURCES_SYCL ggml-sycl.cpp) - set(LLAMA_EXTRA_LIBS ${LLAMA_EXTRA_LIBS} sycl OpenCL mkl_core pthread m dl mkl_sycl_blas mkl_intel_ilp64 mkl_tbb_thread) + if (WIN32) + set(LLAMA_EXTRA_LIBS ${LLAMA_EXTRA_LIBS} -fsycl sycl7 OpenCL mkl_sycl_blas_dll.lib mkl_intel_ilp64_dll.lib mkl_sequential_dll.lib mkl_core_dll.lib) + else() + set(LLAMA_EXTRA_LIBS ${LLAMA_EXTRA_LIBS} -fsycl OpenCL mkl_core pthread m dl mkl_sycl_blas mkl_intel_ilp64 mkl_tbb_thread) + endif() endif() if (LLAMA_KOMPUTE) diff --git a/README_sycl.md b/README-sycl.md similarity index 58% rename from README_sycl.md rename to README-sycl.md index d5a1818f5..2b2cfe03a 100644 --- a/README_sycl.md +++ b/README-sycl.md @@ -8,10 +8,14 @@ [Linux](#linux) +[Windows](#windows) + [Environment Variable](#environment-variable) [Known Issue](#known-issue) +[Q&A](#q&a) + [Todo](#todo) ## Background @@ -33,7 +37,7 @@ For Intel CPU, recommend to use llama.cpp for X86 (Intel MKL building). |OS|Status|Verified| |-|-|-| |Linux|Support|Ubuntu 22.04| -|Windows|Ongoing| | +|Windows|Support|Windows 11| ## Intel GPU @@ -42,7 +46,7 @@ For Intel CPU, recommend to use llama.cpp for X86 (Intel MKL building). |-|-|-| |Intel Data Center Max Series| Support| Max 1550| |Intel Data Center Flex Series| Support| Flex 170| -|Intel Arc Series| Support| Arc 770| +|Intel Arc Series| Support| Arc 770, 730M| |Intel built-in Arc GPU| Support| built-in Arc GPU in Meteor Lake| |Intel iGPU| Support| iGPU in i5-1250P, i7-1165G7| @@ -131,6 +135,7 @@ cmake .. -DLLAMA_SYCL=ON -DCMAKE_C_COMPILER=icx -DCMAKE_CXX_COMPILER=icpx #build all binary cmake --build . --config Release -v +cd .. ``` or @@ -195,7 +200,7 @@ GGML_SYCL_DEVICE=0 ./build/bin/main -m models/llama-2-7b.Q4_0.gguf -p "Building or run by script: ``` -./examples/sycl/run_llama2.sh +./examples/sycl/run-llama2.sh ``` Note: @@ -205,11 +210,175 @@ Note: 5. Check the device ID in output -Like: +Like: ``` Using device **0** (Intel(R) Arc(TM) A770 Graphics) as main device ``` +## Windows + +### Setup Environment + +1. Install Intel GPU driver. + +Please install Intel GPU driver by official guide: [Install GPU Drivers](https://www.intel.com/content/www/us/en/products/docs/discrete-gpus/arc/software/drivers.html). + +2. Install Intel® oneAPI Base toolkit. + +a. Please follow the procedure in [Get the Intel® oneAPI Base Toolkit ](https://www.intel.com/content/www/us/en/developer/tools/oneapi/base-toolkit.html). + +Recommend to install to default folder: **/opt/intel/oneapi**. + +Following guide uses the default folder as example. If you use other folder, please modify the following guide info with your folder. + +b. Enable oneAPI running environment: + +- In Search, input 'oneAPI'. + +Search & open "Intel oneAPI command prompt for Intel 64 for Visual Studio 2022" + +- In Run: + +In CMD: +``` +"C:\Program Files (x86)\Intel\oneAPI\setvars.bat" intel64 +``` + +c. Check GPU + +In oneAPI command line: + +``` +sycl-ls +``` + +There should be one or more level-zero devices. Like **[ext_oneapi_level_zero:gpu:0]**. + +Output (example): +``` +[opencl:acc:0] Intel(R) FPGA Emulation Platform for OpenCL(TM), Intel(R) FPGA Emulation Device OpenCL 1.2 [2023.16.10.0.17_160000] +[opencl:cpu:1] Intel(R) OpenCL, 11th Gen Intel(R) Core(TM) i7-1185G7 @ 3.00GHz OpenCL 3.0 (Build 0) [2023.16.10.0.17_160000] +[opencl:gpu:2] Intel(R) OpenCL Graphics, Intel(R) Iris(R) Xe Graphics OpenCL 3.0 NEO [31.0.101.5186] +[ext_oneapi_level_zero:gpu:0] Intel(R) Level-Zero, Intel(R) Iris(R) Xe Graphics 1.3 [1.3.28044] + +``` + +3. Install cmake & make + +a. Download & install cmake for windows: https://cmake.org/download/ + +b. Download & install make for windows provided by mingw-w64: https://www.mingw-w64.org/downloads/ + + +### Build locally: + +In oneAPI command line window: + +``` +mkdir -p build +cd build +@call "C:\Program Files (x86)\Intel\oneAPI\setvars.bat" intel64 --force + +:: for FP16 +:: faster for long-prompt inference +:: cmake -G "MinGW Makefiles" .. -DLLAMA_SYCL=ON -DCMAKE_C_COMPILER=icx -DCMAKE_CXX_COMPILER=icx -DCMAKE_BUILD_TYPE=Release -DLLAMA_SYCL_F16=ON + +:: for FP32 +cmake -G "MinGW Makefiles" .. -DLLAMA_SYCL=ON -DCMAKE_C_COMPILER=icx -DCMAKE_CXX_COMPILER=icx -DCMAKE_BUILD_TYPE=Release + + +:: build example/main only +:: make main + +:: build all binary +make -j +cd .. +``` + +or + +``` +.\examples\sycl\win-build-sycl.bat +``` + +Note: + +- By default, it will build for all binary files. It will take more time. To reduce the time, we recommend to build for **example/main** only. + +### Run + +1. Put model file to folder **models** + +2. Enable oneAPI running environment + +- In Search, input 'oneAPI'. + +Search & open "Intel oneAPI command prompt for Intel 64 for Visual Studio 2022" + +- In Run: + +In CMD: +``` +"C:\Program Files (x86)\Intel\oneAPI\setvars.bat" intel64 +``` + +3. List device ID + +Run without parameter: + +``` +build\bin\ls-sycl-device.exe + +or + +build\bin\main.exe +``` + +Check the ID in startup log, like: + +``` +found 4 SYCL devices: + Device 0: Intel(R) Arc(TM) A770 Graphics, compute capability 1.3, + max compute_units 512, max work group size 1024, max sub group size 32, global mem size 16225243136 + Device 1: Intel(R) FPGA Emulation Device, compute capability 1.2, + max compute_units 24, max work group size 67108864, max sub group size 64, global mem size 67065057280 + Device 2: 13th Gen Intel(R) Core(TM) i7-13700K, compute capability 3.0, + max compute_units 24, max work group size 8192, max sub group size 64, global mem size 67065057280 + Device 3: Intel(R) Arc(TM) A770 Graphics, compute capability 3.0, + max compute_units 512, max work group size 1024, max sub group size 32, global mem size 16225243136 + +``` + +|Attribute|Note| +|-|-| +|compute capability 1.3|Level-zero running time, recommended | +|compute capability 3.0|OpenCL running time, slower than level-zero in most cases| + +4. Set device ID and execute llama.cpp + +Set device ID = 0 by **set GGML_SYCL_DEVICE=0** + +``` +set GGML_SYCL_DEVICE=0 +build\bin\main.exe -m models\llama-2-7b.Q4_0.gguf -p "Building a website can be done in 10 simple steps:\nStep 1:" -n 400 -e -ngl 33 -s 0 +``` +or run by script: + +``` +.\examples\sycl\win-run-llama2.bat +``` + +Note: + +- By default, mmap is used to read model file. In some cases, it leads to the hang issue. Recommend to use parameter **--no-mmap** to disable mmap() to skip this issue. + + +5. Check the device ID in output + +Like: +``` +Using device **0** (Intel(R) Arc(TM) A770 Graphics) as main device +``` ## Environment Variable @@ -220,7 +389,7 @@ Using device **0** (Intel(R) Arc(TM) A770 Graphics) as main device |LLAMA_SYCL|ON (mandatory)|Enable build with SYCL code path.
For FP32/FP16, LLAMA_SYCL=ON is mandatory.| |LLAMA_SYCL_F16|ON (optional)|Enable FP16 build with SYCL code path. Faster for long-prompt inference.
For FP32, not set it.| |CMAKE_C_COMPILER|icx|Use icx compiler for SYCL code path| -|CMAKE_CXX_COMPILER|icpx|use icpx for SYCL code path| +|CMAKE_CXX_COMPILER|icpx (Linux), icx (Windows)|use icpx/icx for SYCL code path| #### Running @@ -232,19 +401,24 @@ Using device **0** (Intel(R) Arc(TM) A770 Graphics) as main device ## Known Issue -- Error: `error while loading shared libraries: libsycl.so.7: cannot open shared object file: No such file or directory`. - - Miss to enable oneAPI running environment. - - Install oneAPI base toolkit and enable it by: `source /opt/intel/oneapi/setvars.sh`. - - - Hang during startup llama.cpp use mmap as default way to read model file and copy to GPU. In some system, memcpy will be abnormal and block. Solution: add **--no-mmap**. +## Q&A + +- Error: `error while loading shared libraries: libsycl.so.7: cannot open shared object file: No such file or directory`. + + Miss to enable oneAPI running environment. + + Install oneAPI base toolkit and enable it by: `source /opt/intel/oneapi/setvars.sh`. + +- In Windows, no result, not error. + + Miss to enable oneAPI running environment. + ## Todo - Support to build in Windows. diff --git a/README.md b/README.md index b37348a74..7746cb510 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,8 @@ Inference of [LLaMA](https://arxiv.org/abs/2302.13971) model in pure C/C++ ### Hot topics +- ⚠️ Incoming backends: https://github.com/ggerganov/llama.cpp/discussions/5138 + - [SYCL backend](README-sycl.md) is ready (1/28/2024), support Linux/Windows in Intel GPUs (iGPU, Arc/Flex/Max series) - New SOTA quantized models, including pure 2-bits: https://huggingface.co/ikawrakow - Collecting Apple Silicon performance stats: - M-series: https://github.com/ggerganov/llama.cpp/discussions/4167 @@ -604,7 +606,7 @@ Building the program with BLAS support may lead to some performance improvements llama.cpp based on SYCL is used to support Intel GPU (Data Center Max series, Flex series, Arc series, Built-in GPU and iGPU). - For detailed info, please refer to [llama.cpp for SYCL](README_sycl.md). + For detailed info, please refer to [llama.cpp for SYCL](README-sycl.md). ### Prepare Data & Run diff --git a/examples/sycl/win-build-sycl.bat b/examples/sycl/win-build-sycl.bat new file mode 100644 index 000000000..f9d43f8ed --- /dev/null +++ b/examples/sycl/win-build-sycl.bat @@ -0,0 +1,23 @@ + +:: MIT license +:: Copyright (C) 2024 Intel Corporation +:: SPDX-License-Identifier: MIT + +mkdir -p build +cd build +@call "C:\Program Files (x86)\Intel\oneAPI\setvars.bat" intel64 --force + +:: for FP16 +:: faster for long-prompt inference +:: cmake -G "MinGW Makefiles" .. -DLLAMA_SYCL=ON -DCMAKE_C_COMPILER=icx -DCMAKE_CXX_COMPILER=icx -DCMAKE_BUILD_TYPE=Release -DLLAMA_SYCL_F16=ON + +:: for FP32 +cmake -G "MinGW Makefiles" .. -DLLAMA_SYCL=ON -DCMAKE_C_COMPILER=icx -DCMAKE_CXX_COMPILER=icx -DCMAKE_BUILD_TYPE=Release + + +:: build example/main only +:: make main + +:: build all binary +make -j +cd .. diff --git a/examples/sycl/win-run-llama2.bat b/examples/sycl/win-run-llama2.bat new file mode 100644 index 000000000..28d935541 --- /dev/null +++ b/examples/sycl/win-run-llama2.bat @@ -0,0 +1,13 @@ +:: MIT license +:: Copyright (C) 2024 Intel Corporation +:: SPDX-License-Identifier: MIT + +INPUT2="Building a website can be done in 10 simple steps:\nStep 1:" +@call "C:\Program Files (x86)\Intel\oneAPI\setvars.bat" intel64 --force + + +set GGML_SYCL_DEVICE=0 +rem set GGML_SYCL_DEBUG=1 +.\build\bin\main.exe -m models\llama-2-7b.Q4_0.gguf -p %INPUT2% -n 400 -e -ngl 33 -s 0 + + diff --git a/scripts/install-oneapi.bat b/scripts/install-oneapi.bat new file mode 100644 index 000000000..e99bef14a --- /dev/null +++ b/scripts/install-oneapi.bat @@ -0,0 +1,19 @@ +:: MIT license +:: Copyright (C) 2024 Intel Corporation +:: SPDX-License-Identifier: MIT + + +set URL=%1 +set COMPONENTS=%2 + +curl.exe --output %TEMP%\webimage.exe --url %URL% --retry 5 --retry-delay 5 +start /b /wait %TEMP%\webimage.exe -s -x -f webimage_extracted --log extract.log +del %TEMP%\webimage.exe +if "%COMPONENTS%"=="" ( + webimage_extracted\bootstrapper.exe -s --action install --eula=accept -p=NEED_VS2017_INTEGRATION=0 -p=NEED_VS2019_INTEGRATION=0 -p=NEED_VS2022_INTEGRATION=0 --log-dir=. +) else ( + webimage_extracted\bootstrapper.exe -s --action install --components=%COMPONENTS% --eula=accept -p=NEED_VS2017_INTEGRATION=0 -p=NEED_VS2019_INTEGRATION=0 -p=NEED_VS2022_INTEGRATION=0 --log-dir=. +) +set installer_exit_code=%ERRORLEVEL% +rd /s/q "webimage_extracted" +exit /b %installer_exit_code% From d62520eb2cc1d7168a30edec6110e1daefbd959f Mon Sep 17 00:00:00 2001 From: Yiming Cui Date: Wed, 31 Jan 2024 11:04:21 +0800 Subject: [PATCH 225/812] Fix typos of IQ2_XXS and IQ3_XXS in llama.cpp (#5231) --- llama.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/llama.cpp b/llama.cpp index a490eeab2..bb23689fa 100644 --- a/llama.cpp +++ b/llama.cpp @@ -2713,10 +2713,10 @@ static std::string llama_model_ftype_name(llama_ftype ftype) { case LLAMA_FTYPE_MOSTLY_Q5_K_S: return "Q5_K - Small"; case LLAMA_FTYPE_MOSTLY_Q5_K_M: return "Q5_K - Medium"; case LLAMA_FTYPE_MOSTLY_Q6_K: return "Q6_K"; - case LLAMA_FTYPE_MOSTLY_IQ2_XXS:return "IQ2_XSS - 2.0625 bpw"; + case LLAMA_FTYPE_MOSTLY_IQ2_XXS:return "IQ2_XXS - 2.0625 bpw"; case LLAMA_FTYPE_MOSTLY_IQ2_XS: return "IQ2_XS - 2.3125 bpw"; case LLAMA_FTYPE_MOSTLY_Q3_K_XS:return "Q3_K - Extra small"; - case LLAMA_FTYPE_MOSTLY_IQ3_XXS:return "IQ3_XSS - 3.0625 bpw"; + case LLAMA_FTYPE_MOSTLY_IQ3_XXS:return "IQ3_XXS - 3.0625 bpw"; default: return "unknown, may not work"; } From f8e9140cb46eebaa867e1184a9946e4840eec772 Mon Sep 17 00:00:00 2001 From: 0cc4m Date: Wed, 31 Jan 2024 11:44:19 +0100 Subject: [PATCH 226/812] Vulkan Fixes (#5223) * Fix Vulkan F16 models * Fix Vulkan context shift crash * Add Vulkan to common.cpp dump_non_result_info_yaml function * Fix bug in Vulkan CPY op * Fix small matrix multiplication errors in AMD GPUs on Windows or with amdvlk Co-authored-by: Engininja2 <139037756+Engininja2@users.noreply.github.com> --------- Co-authored-by: Engininja2 <139037756+Engininja2@users.noreply.github.com> --- common/common.cpp | 1 + ggml-vulkan-shaders.hpp | 1952 +++++++++++++---------------------- ggml-vulkan.cpp | 14 +- ggml_vk_generate_shaders.py | 4 +- 4 files changed, 704 insertions(+), 1267 deletions(-) diff --git a/common/common.cpp b/common/common.cpp index 0dd1c50cf..9d976c7c8 100644 --- a/common/common.cpp +++ b/common/common.cpp @@ -1520,6 +1520,7 @@ void dump_non_result_info_yaml(FILE * stream, const gpt_params & params, const l fprintf(stream, "cpu_has_avx512_vbmi: %s\n", ggml_cpu_has_avx512_vbmi() ? "true" : "false"); fprintf(stream, "cpu_has_avx512_vnni: %s\n", ggml_cpu_has_avx512_vnni() ? "true" : "false"); fprintf(stream, "cpu_has_cublas: %s\n", ggml_cpu_has_cublas() ? "true" : "false"); + fprintf(stream, "cpu_has_vulkan: %s\n", ggml_cpu_has_vulkan() ? "true" : "false"); fprintf(stream, "cpu_has_clblast: %s\n", ggml_cpu_has_clblast() ? "true" : "false"); fprintf(stream, "cpu_has_kompute: %s\n", ggml_cpu_has_kompute() ? "true" : "false"); fprintf(stream, "cpu_has_fma: %s\n", ggml_cpu_has_fma() ? "true" : "false"); diff --git a/ggml-vulkan-shaders.hpp b/ggml-vulkan-shaders.hpp index 321e36383..e2e9be22c 100644 --- a/ggml-vulkan-shaders.hpp +++ b/ggml-vulkan-shaders.hpp @@ -890,7 +890,7 @@ const uint64_t cpy_f32_f32_len = 2472; unsigned char dequant_f16_data[] = { 0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, -0x87,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, +0x81,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, 0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x09,0x00,0x00,0x00, 0x11,0x00,0x02,0x00,0x51,0x11,0x00,0x00,0x0b,0x00,0x06,0x00, 0x01,0x00,0x00,0x00,0x47,0x4c,0x53,0x4c,0x2e,0x73,0x74,0x64, @@ -898,7 +898,7 @@ unsigned char dequant_f16_data[] = { 0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x0f,0x00,0x09,0x00, 0x05,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x6d,0x61,0x69,0x6e, 0x00,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0x51,0x00,0x00,0x00,0x5f,0x00,0x00,0x00,0x10,0x00,0x06,0x00, +0x4f,0x00,0x00,0x00,0x5d,0x00,0x00,0x00,0x10,0x00,0x06,0x00, 0x04,0x00,0x00,0x00,0x11,0x00,0x00,0x00,0x00,0x01,0x00,0x00, 0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00, 0x0c,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x1c,0x00,0x00,0x00, @@ -910,23 +910,23 @@ unsigned char dequant_f16_data[] = { 0x48,0x00,0x05,0x00,0x14,0x00,0x00,0x00,0x03,0x00,0x00,0x00, 0x23,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x47,0x00,0x03,0x00, 0x14,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x4e,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x48,0x00,0x04,0x00,0x4f,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x4f,0x00,0x00,0x00, +0x4c,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x48,0x00,0x04,0x00,0x4d,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x4d,0x00,0x00,0x00, 0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x03,0x00,0x4f,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x51,0x00,0x00,0x00,0x22,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x51,0x00,0x00,0x00, +0x47,0x00,0x03,0x00,0x4d,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x4f,0x00,0x00,0x00,0x22,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x4f,0x00,0x00,0x00, 0x21,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x5c,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x48,0x00,0x04,0x00,0x5d,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x19,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x5d,0x00,0x00,0x00, +0x5a,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x48,0x00,0x04,0x00,0x5b,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x19,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x5b,0x00,0x00,0x00, 0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x03,0x00,0x5d,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x5f,0x00,0x00,0x00,0x22,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x5f,0x00,0x00,0x00, +0x47,0x00,0x03,0x00,0x5b,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x5d,0x00,0x00,0x00,0x22,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x5d,0x00,0x00,0x00, 0x21,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x80,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x19,0x00,0x00,0x00, +0x7e,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x19,0x00,0x00,0x00, 0x13,0x00,0x02,0x00,0x02,0x00,0x00,0x00,0x21,0x00,0x03,0x00, 0x03,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x15,0x00,0x04,0x00, 0x06,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x01,0x00,0x00,0x00, @@ -945,330 +945,109 @@ unsigned char dequant_f16_data[] = { 0x16,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, 0x06,0x00,0x00,0x00,0x17,0x00,0x00,0x00,0x01,0x00,0x00,0x00, 0x20,0x00,0x04,0x00,0x18,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x1b,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x14,0x00,0x02,0x00, -0x24,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x48,0x00,0x00,0x00, -0x10,0x00,0x00,0x00,0x16,0x00,0x03,0x00,0x4a,0x00,0x00,0x00, -0x10,0x00,0x00,0x00,0x1d,0x00,0x03,0x00,0x4e,0x00,0x00,0x00, -0x4a,0x00,0x00,0x00,0x1e,0x00,0x03,0x00,0x4f,0x00,0x00,0x00, -0x4e,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x50,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x50,0x00,0x00,0x00,0x51,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x54,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x4a,0x00,0x00,0x00,0x1d,0x00,0x03,0x00,0x5c,0x00,0x00,0x00, -0x4a,0x00,0x00,0x00,0x1e,0x00,0x03,0x00,0x5d,0x00,0x00,0x00, -0x5c,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x5e,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x5d,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x5e,0x00,0x00,0x00,0x5f,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x61,0x00,0x00,0x00, -0x03,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x79,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x7f,0x00,0x00,0x00,0x00,0x01,0x00,0x00, -0x2c,0x00,0x06,0x00,0x0a,0x00,0x00,0x00,0x80,0x00,0x00,0x00, -0x7f,0x00,0x00,0x00,0x79,0x00,0x00,0x00,0x79,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x6c,0x02,0x00,0x00, -0x11,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x6d,0x02,0x00,0x00,0x12,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x6e,0x02,0x00,0x00,0x13,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x6f,0x02,0x00,0x00, -0x04,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x70,0x02,0x00,0x00,0x14,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x71,0x02,0x00,0x00,0x05,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x72,0x02,0x00,0x00, -0x15,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x73,0x02,0x00,0x00,0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x74,0x02,0x00,0x00,0x16,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x75,0x02,0x00,0x00, -0x07,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x76,0x02,0x00,0x00,0x17,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x77,0x02,0x00,0x00,0x08,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x78,0x02,0x00,0x00, -0x18,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x79,0x02,0x00,0x00,0x09,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x7a,0x02,0x00,0x00,0x19,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x7b,0x02,0x00,0x00, -0x0a,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x7c,0x02,0x00,0x00,0x1a,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x7d,0x02,0x00,0x00,0x0b,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x7e,0x02,0x00,0x00, -0x1b,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x7f,0x02,0x00,0x00,0x0c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x80,0x02,0x00,0x00,0x1c,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x81,0x02,0x00,0x00, -0x0d,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x82,0x02,0x00,0x00,0x1d,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x83,0x02,0x00,0x00,0x0e,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x84,0x02,0x00,0x00, -0x1e,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x85,0x02,0x00,0x00,0x0f,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x86,0x02,0x00,0x00,0x1f,0x00,0x00,0x00, -0x36,0x00,0x05,0x00,0x02,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x05,0x00,0x00,0x00,0xf7,0x00,0x03,0x00,0x81,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0xfb,0x00,0x03,0x00,0x0d,0x00,0x00,0x00, -0x82,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x82,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x0e,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x0d,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x11,0x00,0x00,0x00, -0x10,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x18,0x00,0x00,0x00, -0x19,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x17,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x1a,0x00,0x00,0x00, -0x19,0x00,0x00,0x00,0x87,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x1c,0x00,0x00,0x00,0x1a,0x00,0x00,0x00,0x1b,0x00,0x00,0x00, -0x8b,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x1d,0x00,0x00,0x00, -0x11,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x87,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x11,0x00,0x00,0x00, -0x1c,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x26,0x00,0x00,0x00,0x1d,0x00,0x00,0x00,0x1b,0x00,0x00,0x00, -0xaf,0x00,0x05,0x00,0x24,0x00,0x00,0x00,0x29,0x00,0x00,0x00, -0x26,0x00,0x00,0x00,0x1a,0x00,0x00,0x00,0xa8,0x00,0x04,0x00, -0x24,0x00,0x00,0x00,0x2a,0x00,0x00,0x00,0x29,0x00,0x00,0x00, -0xf7,0x00,0x03,0x00,0x2c,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x2a,0x00,0x00,0x00,0x2b,0x00,0x00,0x00, -0x2c,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x2b,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x18,0x00,0x00,0x00,0x2f,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x30,0x00,0x00,0x00,0x2f,0x00,0x00,0x00, -0xaf,0x00,0x05,0x00,0x24,0x00,0x00,0x00,0x31,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x30,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x2c,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x2c,0x00,0x00,0x00, -0xf5,0x00,0x07,0x00,0x24,0x00,0x00,0x00,0x32,0x00,0x00,0x00, -0x29,0x00,0x00,0x00,0x82,0x00,0x00,0x00,0x31,0x00,0x00,0x00, -0x2b,0x00,0x00,0x00,0xf7,0x00,0x03,0x00,0x34,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x32,0x00,0x00,0x00, -0x33,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x33,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x81,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0x34,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x18,0x00,0x00,0x00,0x38,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0x37,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x38,0x00,0x00,0x00,0x87,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x1b,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x3e,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x3e,0x00,0x00,0x00,0x1d,0x00,0x00,0x00,0x41,0x00,0x06,0x00, -0x54,0x00,0x00,0x00,0x55,0x00,0x00,0x00,0x51,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4a,0x00,0x00,0x00,0x56,0x00,0x00,0x00,0x55,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x58,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0x17,0x00,0x00,0x00,0x41,0x00,0x06,0x00, -0x54,0x00,0x00,0x00,0x59,0x00,0x00,0x00,0x51,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x58,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4a,0x00,0x00,0x00,0x5a,0x00,0x00,0x00,0x59,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x18,0x00,0x00,0x00,0x62,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x61,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x63,0x00,0x00,0x00,0x62,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x64,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x63,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x67,0x00,0x00,0x00,0x64,0x00,0x00,0x00, -0x26,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x54,0x00,0x00,0x00, -0x6e,0x00,0x00,0x00,0x5f,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x67,0x00,0x00,0x00,0x3e,0x00,0x03,0x00,0x6e,0x00,0x00,0x00, -0x56,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x78,0x00,0x00,0x00,0x67,0x00,0x00,0x00,0x48,0x00,0x00,0x00, -0x41,0x00,0x06,0x00,0x54,0x00,0x00,0x00,0x7c,0x00,0x00,0x00, -0x5f,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x78,0x00,0x00,0x00, -0x3e,0x00,0x03,0x00,0x7c,0x00,0x00,0x00,0x5a,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0x92,0x00,0x00,0x00, -0x55,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0x95,0x00,0x00,0x00,0x59,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x9c,0x00,0x00,0x00,0x67,0x00,0x00,0x00, -0x17,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x54,0x00,0x00,0x00, -0x9f,0x00,0x00,0x00,0x5f,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x9c,0x00,0x00,0x00,0x3e,0x00,0x03,0x00,0x9f,0x00,0x00,0x00, -0x92,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xa6,0x00,0x00,0x00,0x67,0x00,0x00,0x00,0x6c,0x02,0x00,0x00, -0x41,0x00,0x06,0x00,0x54,0x00,0x00,0x00,0xa8,0x00,0x00,0x00, -0x5f,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0xa6,0x00,0x00,0x00, -0x3e,0x00,0x03,0x00,0xa8,0x00,0x00,0x00,0x95,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0xb2,0x00,0x00,0x00, -0x55,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0xb5,0x00,0x00,0x00,0x59,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xbc,0x00,0x00,0x00,0x67,0x00,0x00,0x00, -0x37,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x54,0x00,0x00,0x00, -0xbf,0x00,0x00,0x00,0x5f,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0xbc,0x00,0x00,0x00,0x3e,0x00,0x03,0x00,0xbf,0x00,0x00,0x00, -0xb2,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xc6,0x00,0x00,0x00,0x67,0x00,0x00,0x00,0x6d,0x02,0x00,0x00, -0x41,0x00,0x06,0x00,0x54,0x00,0x00,0x00,0xc8,0x00,0x00,0x00, -0x5f,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0xc6,0x00,0x00,0x00, -0x3e,0x00,0x03,0x00,0xc8,0x00,0x00,0x00,0xb5,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0xd2,0x00,0x00,0x00, -0x55,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0xd5,0x00,0x00,0x00,0x59,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xdc,0x00,0x00,0x00,0x67,0x00,0x00,0x00, -0x61,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x54,0x00,0x00,0x00, -0xdf,0x00,0x00,0x00,0x5f,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0xdc,0x00,0x00,0x00,0x3e,0x00,0x03,0x00,0xdf,0x00,0x00,0x00, -0xd2,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xe6,0x00,0x00,0x00,0x67,0x00,0x00,0x00,0x6e,0x02,0x00,0x00, -0x41,0x00,0x06,0x00,0x54,0x00,0x00,0x00,0xe8,0x00,0x00,0x00, -0x5f,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0xe6,0x00,0x00,0x00, -0x3e,0x00,0x03,0x00,0xe8,0x00,0x00,0x00,0xd5,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0xf2,0x00,0x00,0x00, -0x55,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0xf5,0x00,0x00,0x00,0x59,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xfc,0x00,0x00,0x00,0x67,0x00,0x00,0x00, -0x6f,0x02,0x00,0x00,0x41,0x00,0x06,0x00,0x54,0x00,0x00,0x00, -0xff,0x00,0x00,0x00,0x5f,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0xfc,0x00,0x00,0x00,0x3e,0x00,0x03,0x00,0xff,0x00,0x00,0x00, -0xf2,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x06,0x01,0x00,0x00,0x67,0x00,0x00,0x00,0x70,0x02,0x00,0x00, -0x41,0x00,0x06,0x00,0x54,0x00,0x00,0x00,0x08,0x01,0x00,0x00, -0x5f,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x06,0x01,0x00,0x00, -0x3e,0x00,0x03,0x00,0x08,0x01,0x00,0x00,0xf5,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0x12,0x01,0x00,0x00, -0x55,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0x15,0x01,0x00,0x00,0x59,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x1c,0x01,0x00,0x00,0x67,0x00,0x00,0x00, -0x71,0x02,0x00,0x00,0x41,0x00,0x06,0x00,0x54,0x00,0x00,0x00, -0x1f,0x01,0x00,0x00,0x5f,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x1c,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x1f,0x01,0x00,0x00, -0x12,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x26,0x01,0x00,0x00,0x67,0x00,0x00,0x00,0x72,0x02,0x00,0x00, -0x41,0x00,0x06,0x00,0x54,0x00,0x00,0x00,0x28,0x01,0x00,0x00, -0x5f,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x26,0x01,0x00,0x00, -0x3e,0x00,0x03,0x00,0x28,0x01,0x00,0x00,0x15,0x01,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0x32,0x01,0x00,0x00, -0x55,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0x35,0x01,0x00,0x00,0x59,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x3c,0x01,0x00,0x00,0x67,0x00,0x00,0x00, -0x73,0x02,0x00,0x00,0x41,0x00,0x06,0x00,0x54,0x00,0x00,0x00, -0x3f,0x01,0x00,0x00,0x5f,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x3c,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x3f,0x01,0x00,0x00, -0x32,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x46,0x01,0x00,0x00,0x67,0x00,0x00,0x00,0x74,0x02,0x00,0x00, -0x41,0x00,0x06,0x00,0x54,0x00,0x00,0x00,0x48,0x01,0x00,0x00, -0x5f,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x46,0x01,0x00,0x00, -0x3e,0x00,0x03,0x00,0x48,0x01,0x00,0x00,0x35,0x01,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0x52,0x01,0x00,0x00, -0x55,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0x55,0x01,0x00,0x00,0x59,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x5c,0x01,0x00,0x00,0x67,0x00,0x00,0x00, -0x75,0x02,0x00,0x00,0x41,0x00,0x06,0x00,0x54,0x00,0x00,0x00, -0x5f,0x01,0x00,0x00,0x5f,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x5c,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x5f,0x01,0x00,0x00, -0x52,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x66,0x01,0x00,0x00,0x67,0x00,0x00,0x00,0x76,0x02,0x00,0x00, -0x41,0x00,0x06,0x00,0x54,0x00,0x00,0x00,0x68,0x01,0x00,0x00, -0x5f,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x66,0x01,0x00,0x00, -0x3e,0x00,0x03,0x00,0x68,0x01,0x00,0x00,0x55,0x01,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0x72,0x01,0x00,0x00, -0x55,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0x75,0x01,0x00,0x00,0x59,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x7c,0x01,0x00,0x00,0x67,0x00,0x00,0x00, -0x77,0x02,0x00,0x00,0x41,0x00,0x06,0x00,0x54,0x00,0x00,0x00, -0x7f,0x01,0x00,0x00,0x5f,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x7c,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x7f,0x01,0x00,0x00, -0x72,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x86,0x01,0x00,0x00,0x67,0x00,0x00,0x00,0x78,0x02,0x00,0x00, -0x41,0x00,0x06,0x00,0x54,0x00,0x00,0x00,0x88,0x01,0x00,0x00, -0x5f,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x86,0x01,0x00,0x00, -0x3e,0x00,0x03,0x00,0x88,0x01,0x00,0x00,0x75,0x01,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0x92,0x01,0x00,0x00, -0x55,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0x95,0x01,0x00,0x00,0x59,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x9c,0x01,0x00,0x00,0x67,0x00,0x00,0x00, -0x79,0x02,0x00,0x00,0x41,0x00,0x06,0x00,0x54,0x00,0x00,0x00, -0x9f,0x01,0x00,0x00,0x5f,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x9c,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x9f,0x01,0x00,0x00, -0x92,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xa6,0x01,0x00,0x00,0x67,0x00,0x00,0x00,0x7a,0x02,0x00,0x00, -0x41,0x00,0x06,0x00,0x54,0x00,0x00,0x00,0xa8,0x01,0x00,0x00, -0x5f,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0xa6,0x01,0x00,0x00, -0x3e,0x00,0x03,0x00,0xa8,0x01,0x00,0x00,0x95,0x01,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0xb2,0x01,0x00,0x00, -0x55,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0xb5,0x01,0x00,0x00,0x59,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xbc,0x01,0x00,0x00,0x67,0x00,0x00,0x00, -0x7b,0x02,0x00,0x00,0x41,0x00,0x06,0x00,0x54,0x00,0x00,0x00, -0xbf,0x01,0x00,0x00,0x5f,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0xbc,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0xbf,0x01,0x00,0x00, -0xb2,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xc6,0x01,0x00,0x00,0x67,0x00,0x00,0x00,0x7c,0x02,0x00,0x00, -0x41,0x00,0x06,0x00,0x54,0x00,0x00,0x00,0xc8,0x01,0x00,0x00, -0x5f,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0xc6,0x01,0x00,0x00, -0x3e,0x00,0x03,0x00,0xc8,0x01,0x00,0x00,0xb5,0x01,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0xd2,0x01,0x00,0x00, -0x55,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0xd5,0x01,0x00,0x00,0x59,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xdc,0x01,0x00,0x00,0x67,0x00,0x00,0x00, -0x7d,0x02,0x00,0x00,0x41,0x00,0x06,0x00,0x54,0x00,0x00,0x00, -0xdf,0x01,0x00,0x00,0x5f,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0xdc,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0xdf,0x01,0x00,0x00, -0xd2,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xe6,0x01,0x00,0x00,0x67,0x00,0x00,0x00,0x7e,0x02,0x00,0x00, -0x41,0x00,0x06,0x00,0x54,0x00,0x00,0x00,0xe8,0x01,0x00,0x00, -0x5f,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0xe6,0x01,0x00,0x00, -0x3e,0x00,0x03,0x00,0xe8,0x01,0x00,0x00,0xd5,0x01,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0xf2,0x01,0x00,0x00, -0x55,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0xf5,0x01,0x00,0x00,0x59,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xfc,0x01,0x00,0x00,0x67,0x00,0x00,0x00, -0x7f,0x02,0x00,0x00,0x41,0x00,0x06,0x00,0x54,0x00,0x00,0x00, -0xff,0x01,0x00,0x00,0x5f,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0xfc,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0xff,0x01,0x00,0x00, -0xf2,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x06,0x02,0x00,0x00,0x67,0x00,0x00,0x00,0x80,0x02,0x00,0x00, -0x41,0x00,0x06,0x00,0x54,0x00,0x00,0x00,0x08,0x02,0x00,0x00, -0x5f,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x06,0x02,0x00,0x00, -0x3e,0x00,0x03,0x00,0x08,0x02,0x00,0x00,0xf5,0x01,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0x12,0x02,0x00,0x00, -0x55,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0x15,0x02,0x00,0x00,0x59,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x1c,0x02,0x00,0x00,0x67,0x00,0x00,0x00, -0x81,0x02,0x00,0x00,0x41,0x00,0x06,0x00,0x54,0x00,0x00,0x00, -0x1f,0x02,0x00,0x00,0x5f,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x1c,0x02,0x00,0x00,0x3e,0x00,0x03,0x00,0x1f,0x02,0x00,0x00, -0x12,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x26,0x02,0x00,0x00,0x67,0x00,0x00,0x00,0x82,0x02,0x00,0x00, -0x41,0x00,0x06,0x00,0x54,0x00,0x00,0x00,0x28,0x02,0x00,0x00, -0x5f,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x26,0x02,0x00,0x00, -0x3e,0x00,0x03,0x00,0x28,0x02,0x00,0x00,0x15,0x02,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0x32,0x02,0x00,0x00, -0x55,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0x35,0x02,0x00,0x00,0x59,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x3c,0x02,0x00,0x00,0x67,0x00,0x00,0x00, -0x83,0x02,0x00,0x00,0x41,0x00,0x06,0x00,0x54,0x00,0x00,0x00, -0x3f,0x02,0x00,0x00,0x5f,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x3c,0x02,0x00,0x00,0x3e,0x00,0x03,0x00,0x3f,0x02,0x00,0x00, -0x32,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x46,0x02,0x00,0x00,0x67,0x00,0x00,0x00,0x84,0x02,0x00,0x00, -0x41,0x00,0x06,0x00,0x54,0x00,0x00,0x00,0x48,0x02,0x00,0x00, -0x5f,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x46,0x02,0x00,0x00, -0x3e,0x00,0x03,0x00,0x48,0x02,0x00,0x00,0x35,0x02,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0x52,0x02,0x00,0x00, -0x55,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0x55,0x02,0x00,0x00,0x59,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x5c,0x02,0x00,0x00,0x67,0x00,0x00,0x00, -0x85,0x02,0x00,0x00,0x41,0x00,0x06,0x00,0x54,0x00,0x00,0x00, -0x5f,0x02,0x00,0x00,0x5f,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x5c,0x02,0x00,0x00,0x3e,0x00,0x03,0x00,0x5f,0x02,0x00,0x00, -0x52,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x66,0x02,0x00,0x00,0x67,0x00,0x00,0x00,0x86,0x02,0x00,0x00, -0x41,0x00,0x06,0x00,0x54,0x00,0x00,0x00,0x68,0x02,0x00,0x00, -0x5f,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x66,0x02,0x00,0x00, -0x3e,0x00,0x03,0x00,0x68,0x02,0x00,0x00,0x55,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0x81,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x81,0x00,0x00,0x00,0xfd,0x00,0x01,0x00,0x38,0x00,0x01,0x00, - +0x06,0x00,0x00,0x00,0x14,0x00,0x02,0x00,0x23,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x2d,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x36,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x16,0x00,0x03,0x00, +0x48,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, +0x4c,0x00,0x00,0x00,0x48,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, +0x4d,0x00,0x00,0x00,0x4c,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x4e,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x4d,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0x4e,0x00,0x00,0x00,0x4f,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x52,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x48,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, +0x5a,0x00,0x00,0x00,0x48,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, +0x5b,0x00,0x00,0x00,0x5a,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x5c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x5b,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0x5c,0x00,0x00,0x00,0x5d,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x5f,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x09,0x00,0x00,0x00,0x77,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x7d,0x00,0x00,0x00, +0x00,0x01,0x00,0x00,0x2c,0x00,0x06,0x00,0x0a,0x00,0x00,0x00, +0x7e,0x00,0x00,0x00,0x7d,0x00,0x00,0x00,0x77,0x00,0x00,0x00, +0x77,0x00,0x00,0x00,0x36,0x00,0x05,0x00,0x02,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x03,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0x05,0x00,0x00,0x00,0xf7,0x00,0x03,0x00, +0x7f,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xfb,0x00,0x03,0x00, +0x0d,0x00,0x00,0x00,0x80,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0x80,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x0e,0x00,0x00,0x00, +0x0f,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x0d,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x10,0x00,0x00,0x00, +0x0f,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x11,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x18,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x16,0x00,0x00,0x00, +0x17,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x1a,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x87,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x1b,0x00,0x00,0x00,0x1a,0x00,0x00,0x00, +0x17,0x00,0x00,0x00,0x8b,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x1c,0x00,0x00,0x00,0x11,0x00,0x00,0x00,0x1b,0x00,0x00,0x00, +0x87,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x22,0x00,0x00,0x00, +0x11,0x00,0x00,0x00,0x1b,0x00,0x00,0x00,0xaf,0x00,0x05,0x00, +0x23,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0x1c,0x00,0x00,0x00, +0x1a,0x00,0x00,0x00,0xa8,0x00,0x04,0x00,0x23,0x00,0x00,0x00, +0x29,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0xf7,0x00,0x03,0x00, +0x2b,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x29,0x00,0x00,0x00,0x2a,0x00,0x00,0x00,0x2b,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0x2a,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x18,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x16,0x00,0x00,0x00, +0x2d,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x2f,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0xaf,0x00,0x05,0x00, +0x23,0x00,0x00,0x00,0x30,0x00,0x00,0x00,0x22,0x00,0x00,0x00, +0x2f,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x2b,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0x2b,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, +0x23,0x00,0x00,0x00,0x31,0x00,0x00,0x00,0x28,0x00,0x00,0x00, +0x80,0x00,0x00,0x00,0x30,0x00,0x00,0x00,0x2a,0x00,0x00,0x00, +0xf7,0x00,0x03,0x00,0x33,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x31,0x00,0x00,0x00,0x32,0x00,0x00,0x00, +0x33,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x32,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x7f,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0x33,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x18,0x00,0x00,0x00, +0x37,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x36,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x38,0x00,0x00,0x00, +0x37,0x00,0x00,0x00,0x87,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x39,0x00,0x00,0x00,0x38,0x00,0x00,0x00,0x17,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3d,0x00,0x00,0x00, +0x22,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x3f,0x00,0x00,0x00,0x3d,0x00,0x00,0x00, +0x1c,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x52,0x00,0x00,0x00, +0x53,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x2d,0x00,0x00,0x00, +0x3f,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x48,0x00,0x00,0x00, +0x54,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x56,0x00,0x00,0x00,0x3f,0x00,0x00,0x00, +0x17,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x52,0x00,0x00,0x00, +0x57,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x2d,0x00,0x00,0x00, +0x56,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x48,0x00,0x00,0x00, +0x58,0x00,0x00,0x00,0x57,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x18,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x16,0x00,0x00,0x00, +0x5f,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x61,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x22,0x00,0x00,0x00, +0x61,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x65,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x1c,0x00,0x00,0x00, +0x41,0x00,0x06,0x00,0x52,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, +0x5d,0x00,0x00,0x00,0x2d,0x00,0x00,0x00,0x65,0x00,0x00,0x00, +0x3e,0x00,0x03,0x00,0x6c,0x00,0x00,0x00,0x54,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x76,0x00,0x00,0x00, +0x65,0x00,0x00,0x00,0x17,0x00,0x00,0x00,0x41,0x00,0x06,0x00, +0x52,0x00,0x00,0x00,0x7a,0x00,0x00,0x00,0x5d,0x00,0x00,0x00, +0x2d,0x00,0x00,0x00,0x76,0x00,0x00,0x00,0x3e,0x00,0x03,0x00, +0x7a,0x00,0x00,0x00,0x58,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x7f,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x7f,0x00,0x00,0x00, +0xfd,0x00,0x01,0x00,0x38,0x00,0x01,0x00, }; -const uint64_t dequant_f16_len = 4392; +const uint64_t dequant_f16_len = 1748; unsigned char dequant_f16_fp32_data[] = { 0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, -0xc8,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, +0x86,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, 0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x51,0x11,0x00,0x00, 0x0b,0x00,0x06,0x00,0x01,0x00,0x00,0x00,0x47,0x4c,0x53,0x4c, 0x2e,0x73,0x74,0x64,0x2e,0x34,0x35,0x30,0x00,0x00,0x00,0x00, 0x0e,0x00,0x03,0x00,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00, 0x0f,0x00,0x09,0x00,0x05,0x00,0x00,0x00,0x04,0x00,0x00,0x00, 0x6d,0x61,0x69,0x6e,0x00,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x52,0x00,0x00,0x00,0x62,0x00,0x00,0x00, +0x16,0x00,0x00,0x00,0x50,0x00,0x00,0x00,0x60,0x00,0x00,0x00, 0x10,0x00,0x06,0x00,0x04,0x00,0x00,0x00,0x11,0x00,0x00,0x00, 0x00,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00, 0x47,0x00,0x04,0x00,0x0c,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, @@ -1280,23 +1059,23 @@ unsigned char dequant_f16_fp32_data[] = { 0x08,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x14,0x00,0x00,0x00, 0x03,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, 0x47,0x00,0x03,0x00,0x14,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x4f,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0x50,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x4d,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0x4e,0x00,0x00,0x00, 0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x50,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x50,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x52,0x00,0x00,0x00, +0x4e,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x4e,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x50,0x00,0x00,0x00, 0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x52,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x5f,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0x60,0x00,0x00,0x00, +0x50,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x5d,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0x5e,0x00,0x00,0x00, 0x00,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x60,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x60,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x62,0x00,0x00,0x00, +0x5e,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x5e,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x60,0x00,0x00,0x00, 0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x62,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x85,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, +0x60,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x83,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, 0x19,0x00,0x00,0x00,0x13,0x00,0x02,0x00,0x02,0x00,0x00,0x00, 0x21,0x00,0x03,0x00,0x03,0x00,0x00,0x00,0x02,0x00,0x00,0x00, 0x15,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x20,0x00,0x00,0x00, @@ -1315,405 +1094,105 @@ unsigned char dequant_f16_fp32_data[] = { 0x15,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x09,0x00,0x00,0x00, 0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x17,0x00,0x00,0x00, 0x01,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x18,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x1b,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x14,0x00,0x02,0x00,0x24,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x37,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x48,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x16,0x00,0x03,0x00, -0x4a,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x16,0x00,0x03,0x00, -0x4e,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, -0x4f,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, -0x50,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x51,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x50,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x51,0x00,0x00,0x00,0x52,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x55,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, -0x5f,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, -0x60,0x00,0x00,0x00,0x5f,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x61,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x60,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x61,0x00,0x00,0x00,0x62,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x64,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x7d,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0x00,0x01,0x00,0x00,0x2c,0x00,0x06,0x00,0x0a,0x00,0x00,0x00, -0x85,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x7d,0x00,0x00,0x00, -0x7d,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xad,0x02,0x00,0x00,0x11,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xae,0x02,0x00,0x00,0x12,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xaf,0x02,0x00,0x00, -0x13,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xb0,0x02,0x00,0x00,0x04,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xb1,0x02,0x00,0x00,0x14,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xb2,0x02,0x00,0x00, -0x05,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xb3,0x02,0x00,0x00,0x15,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xb4,0x02,0x00,0x00,0x06,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xb5,0x02,0x00,0x00, -0x16,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xb6,0x02,0x00,0x00,0x07,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xb7,0x02,0x00,0x00,0x17,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xb8,0x02,0x00,0x00, -0x08,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xb9,0x02,0x00,0x00,0x18,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xba,0x02,0x00,0x00,0x09,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xbb,0x02,0x00,0x00, -0x19,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xbc,0x02,0x00,0x00,0x0a,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xbd,0x02,0x00,0x00,0x1a,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xbe,0x02,0x00,0x00, -0x0b,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xbf,0x02,0x00,0x00,0x1b,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xc0,0x02,0x00,0x00,0x0c,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xc1,0x02,0x00,0x00, -0x1c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xc2,0x02,0x00,0x00,0x0d,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xc3,0x02,0x00,0x00,0x1d,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xc4,0x02,0x00,0x00, -0x0e,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xc5,0x02,0x00,0x00,0x1e,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xc6,0x02,0x00,0x00,0x0f,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xc7,0x02,0x00,0x00, -0x1f,0x00,0x00,0x00,0x36,0x00,0x05,0x00,0x02,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x03,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0x05,0x00,0x00,0x00,0xf7,0x00,0x03,0x00, -0x86,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xfb,0x00,0x03,0x00, -0x0d,0x00,0x00,0x00,0x87,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x87,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x0e,0x00,0x00,0x00, -0x0f,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x0d,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x10,0x00,0x00,0x00, -0x0f,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x11,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x18,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0x17,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x1a,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x87,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x1a,0x00,0x00,0x00, -0x1b,0x00,0x00,0x00,0x8b,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x1d,0x00,0x00,0x00,0x11,0x00,0x00,0x00,0x1c,0x00,0x00,0x00, -0x87,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x11,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x26,0x00,0x00,0x00,0x1d,0x00,0x00,0x00, -0x1b,0x00,0x00,0x00,0xaf,0x00,0x05,0x00,0x24,0x00,0x00,0x00, -0x29,0x00,0x00,0x00,0x26,0x00,0x00,0x00,0x1a,0x00,0x00,0x00, -0xa8,0x00,0x04,0x00,0x24,0x00,0x00,0x00,0x2a,0x00,0x00,0x00, -0x29,0x00,0x00,0x00,0xf7,0x00,0x03,0x00,0x2c,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x2a,0x00,0x00,0x00, -0x2b,0x00,0x00,0x00,0x2c,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x2b,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x18,0x00,0x00,0x00, -0x2f,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x30,0x00,0x00,0x00, -0x2f,0x00,0x00,0x00,0xaf,0x00,0x05,0x00,0x24,0x00,0x00,0x00, -0x31,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x30,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x2c,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x2c,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x24,0x00,0x00,0x00, -0x32,0x00,0x00,0x00,0x29,0x00,0x00,0x00,0x87,0x00,0x00,0x00, -0x31,0x00,0x00,0x00,0x2b,0x00,0x00,0x00,0xf7,0x00,0x03,0x00, -0x34,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x32,0x00,0x00,0x00,0x33,0x00,0x00,0x00,0x34,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0x33,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x86,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x34,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x18,0x00,0x00,0x00,0x38,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x14,0x00,0x02,0x00, +0x23,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x2d,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x36,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x16,0x00,0x03,0x00,0x48,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0x16,0x00,0x03,0x00,0x4c,0x00,0x00,0x00,0x10,0x00,0x00,0x00, +0x1d,0x00,0x03,0x00,0x4d,0x00,0x00,0x00,0x4c,0x00,0x00,0x00, +0x1e,0x00,0x03,0x00,0x4e,0x00,0x00,0x00,0x4d,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x4f,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x4e,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x4f,0x00,0x00,0x00, +0x50,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x53,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x4c,0x00,0x00,0x00, +0x1d,0x00,0x03,0x00,0x5d,0x00,0x00,0x00,0x4c,0x00,0x00,0x00, +0x1e,0x00,0x03,0x00,0x5e,0x00,0x00,0x00,0x5d,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x5f,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x5e,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x5f,0x00,0x00,0x00, +0x60,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x03,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x7b,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x09,0x00,0x00,0x00, +0x82,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x2c,0x00,0x06,0x00, +0x0a,0x00,0x00,0x00,0x83,0x00,0x00,0x00,0x82,0x00,0x00,0x00, +0x7b,0x00,0x00,0x00,0x7b,0x00,0x00,0x00,0x36,0x00,0x05,0x00, +0x02,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x03,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x05,0x00,0x00,0x00, +0xf7,0x00,0x03,0x00,0x84,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0xfb,0x00,0x03,0x00,0x0d,0x00,0x00,0x00,0x85,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0x85,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x0e,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x0d,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x09,0x00,0x00,0x00, +0x10,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x11,0x00,0x00,0x00,0x10,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x18,0x00,0x00,0x00,0x19,0x00,0x00,0x00, +0x16,0x00,0x00,0x00,0x17,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x1a,0x00,0x00,0x00,0x19,0x00,0x00,0x00, +0x87,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x1b,0x00,0x00,0x00, +0x1a,0x00,0x00,0x00,0x17,0x00,0x00,0x00,0x8b,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x11,0x00,0x00,0x00, +0x1b,0x00,0x00,0x00,0x87,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x22,0x00,0x00,0x00,0x11,0x00,0x00,0x00,0x1b,0x00,0x00,0x00, +0xaf,0x00,0x05,0x00,0x23,0x00,0x00,0x00,0x28,0x00,0x00,0x00, +0x1c,0x00,0x00,0x00,0x1a,0x00,0x00,0x00,0xa8,0x00,0x04,0x00, +0x23,0x00,0x00,0x00,0x29,0x00,0x00,0x00,0x28,0x00,0x00,0x00, +0xf7,0x00,0x03,0x00,0x2b,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x29,0x00,0x00,0x00,0x2a,0x00,0x00,0x00, +0x2b,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x2a,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x18,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, +0x16,0x00,0x00,0x00,0x2d,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x2f,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, +0xaf,0x00,0x05,0x00,0x23,0x00,0x00,0x00,0x30,0x00,0x00,0x00, +0x22,0x00,0x00,0x00,0x2f,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x2b,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x2b,0x00,0x00,0x00, +0xf5,0x00,0x07,0x00,0x23,0x00,0x00,0x00,0x31,0x00,0x00,0x00, +0x28,0x00,0x00,0x00,0x85,0x00,0x00,0x00,0x30,0x00,0x00,0x00, +0x2a,0x00,0x00,0x00,0xf7,0x00,0x03,0x00,0x33,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x31,0x00,0x00,0x00, +0x32,0x00,0x00,0x00,0x33,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0x32,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x84,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0x33,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x18,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x16,0x00,0x00,0x00, +0x36,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x38,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x87,0x00,0x05,0x00, 0x06,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x38,0x00,0x00,0x00, -0x87,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x1b,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x3e,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x3a,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0x3e,0x00,0x00,0x00,0x1d,0x00,0x00,0x00, -0x41,0x00,0x06,0x00,0x55,0x00,0x00,0x00,0x56,0x00,0x00,0x00, -0x52,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4e,0x00,0x00,0x00,0x57,0x00,0x00,0x00, -0x56,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0x58,0x00,0x00,0x00,0x57,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x5a,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x17,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x55,0x00,0x00,0x00, -0x5b,0x00,0x00,0x00,0x52,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x5a,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x4e,0x00,0x00,0x00, -0x5c,0x00,0x00,0x00,0x5b,0x00,0x00,0x00,0x73,0x00,0x04,0x00, -0x4a,0x00,0x00,0x00,0x5d,0x00,0x00,0x00,0x5c,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x18,0x00,0x00,0x00,0x65,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x64,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x66,0x00,0x00,0x00,0x65,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x67,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x66,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x6a,0x00,0x00,0x00,0x67,0x00,0x00,0x00, -0x26,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x4e,0x00,0x00,0x00, -0x71,0x00,0x00,0x00,0x58,0x00,0x00,0x00,0x41,0x00,0x06,0x00, -0x55,0x00,0x00,0x00,0x72,0x00,0x00,0x00,0x62,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x6a,0x00,0x00,0x00,0x3e,0x00,0x03,0x00, -0x72,0x00,0x00,0x00,0x71,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x7c,0x00,0x00,0x00,0x6a,0x00,0x00,0x00, -0x48,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x4e,0x00,0x00,0x00, -0x80,0x00,0x00,0x00,0x5d,0x00,0x00,0x00,0x41,0x00,0x06,0x00, -0x55,0x00,0x00,0x00,0x81,0x00,0x00,0x00,0x62,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x7c,0x00,0x00,0x00,0x3e,0x00,0x03,0x00, -0x81,0x00,0x00,0x00,0x80,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4e,0x00,0x00,0x00,0x97,0x00,0x00,0x00,0x56,0x00,0x00,0x00, -0x73,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0x98,0x00,0x00,0x00, -0x97,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x4e,0x00,0x00,0x00, -0x9b,0x00,0x00,0x00,0x5b,0x00,0x00,0x00,0x73,0x00,0x04,0x00, -0x4a,0x00,0x00,0x00,0x9c,0x00,0x00,0x00,0x9b,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa3,0x00,0x00,0x00, -0x6a,0x00,0x00,0x00,0x17,0x00,0x00,0x00,0x73,0x00,0x04,0x00, -0x4e,0x00,0x00,0x00,0xa6,0x00,0x00,0x00,0x98,0x00,0x00,0x00, -0x41,0x00,0x06,0x00,0x55,0x00,0x00,0x00,0xa7,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0xa3,0x00,0x00,0x00, -0x3e,0x00,0x03,0x00,0xa7,0x00,0x00,0x00,0xa6,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xae,0x00,0x00,0x00, -0x6a,0x00,0x00,0x00,0xad,0x02,0x00,0x00,0x73,0x00,0x04,0x00, -0x4e,0x00,0x00,0x00,0xb0,0x00,0x00,0x00,0x9c,0x00,0x00,0x00, -0x41,0x00,0x06,0x00,0x55,0x00,0x00,0x00,0xb1,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0xae,0x00,0x00,0x00, -0x3e,0x00,0x03,0x00,0xb1,0x00,0x00,0x00,0xb0,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4e,0x00,0x00,0x00,0xbb,0x00,0x00,0x00, -0x56,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0xbc,0x00,0x00,0x00,0xbb,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4e,0x00,0x00,0x00,0xbf,0x00,0x00,0x00,0x5b,0x00,0x00,0x00, -0x73,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0xc0,0x00,0x00,0x00, -0xbf,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xc7,0x00,0x00,0x00,0x6a,0x00,0x00,0x00,0x37,0x00,0x00,0x00, -0x73,0x00,0x04,0x00,0x4e,0x00,0x00,0x00,0xca,0x00,0x00,0x00, -0xbc,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x55,0x00,0x00,0x00, -0xcb,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0xc7,0x00,0x00,0x00,0x3e,0x00,0x03,0x00,0xcb,0x00,0x00,0x00, -0xca,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xd2,0x00,0x00,0x00,0x6a,0x00,0x00,0x00,0xae,0x02,0x00,0x00, -0x73,0x00,0x04,0x00,0x4e,0x00,0x00,0x00,0xd4,0x00,0x00,0x00, -0xc0,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x55,0x00,0x00,0x00, -0xd5,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0xd2,0x00,0x00,0x00,0x3e,0x00,0x03,0x00,0xd5,0x00,0x00,0x00, -0xd4,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x4e,0x00,0x00,0x00, -0xdf,0x00,0x00,0x00,0x56,0x00,0x00,0x00,0x73,0x00,0x04,0x00, -0x4a,0x00,0x00,0x00,0xe0,0x00,0x00,0x00,0xdf,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4e,0x00,0x00,0x00,0xe3,0x00,0x00,0x00, -0x5b,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0xe4,0x00,0x00,0x00,0xe3,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xeb,0x00,0x00,0x00,0x6a,0x00,0x00,0x00, -0x64,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x4e,0x00,0x00,0x00, -0xee,0x00,0x00,0x00,0xe0,0x00,0x00,0x00,0x41,0x00,0x06,0x00, -0x55,0x00,0x00,0x00,0xef,0x00,0x00,0x00,0x62,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0xeb,0x00,0x00,0x00,0x3e,0x00,0x03,0x00, -0xef,0x00,0x00,0x00,0xee,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xf6,0x00,0x00,0x00,0x6a,0x00,0x00,0x00, -0xaf,0x02,0x00,0x00,0x73,0x00,0x04,0x00,0x4e,0x00,0x00,0x00, -0xf8,0x00,0x00,0x00,0xe4,0x00,0x00,0x00,0x41,0x00,0x06,0x00, -0x55,0x00,0x00,0x00,0xf9,0x00,0x00,0x00,0x62,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0xf6,0x00,0x00,0x00,0x3e,0x00,0x03,0x00, -0xf9,0x00,0x00,0x00,0xf8,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4e,0x00,0x00,0x00,0x03,0x01,0x00,0x00,0x56,0x00,0x00,0x00, -0x73,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0x04,0x01,0x00,0x00, -0x03,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0x4e,0x00,0x00,0x00, -0x07,0x01,0x00,0x00,0x5b,0x00,0x00,0x00,0x73,0x00,0x04,0x00, -0x4a,0x00,0x00,0x00,0x08,0x01,0x00,0x00,0x07,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x0f,0x01,0x00,0x00, -0x6a,0x00,0x00,0x00,0xb0,0x02,0x00,0x00,0x73,0x00,0x04,0x00, -0x4e,0x00,0x00,0x00,0x12,0x01,0x00,0x00,0x04,0x01,0x00,0x00, -0x41,0x00,0x06,0x00,0x55,0x00,0x00,0x00,0x13,0x01,0x00,0x00, -0x62,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x0f,0x01,0x00,0x00, -0x3e,0x00,0x03,0x00,0x13,0x01,0x00,0x00,0x12,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x1a,0x01,0x00,0x00, -0x6a,0x00,0x00,0x00,0xb1,0x02,0x00,0x00,0x73,0x00,0x04,0x00, -0x4e,0x00,0x00,0x00,0x1c,0x01,0x00,0x00,0x08,0x01,0x00,0x00, -0x41,0x00,0x06,0x00,0x55,0x00,0x00,0x00,0x1d,0x01,0x00,0x00, -0x62,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x1a,0x01,0x00,0x00, -0x3e,0x00,0x03,0x00,0x1d,0x01,0x00,0x00,0x1c,0x01,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4e,0x00,0x00,0x00,0x27,0x01,0x00,0x00, -0x56,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0x28,0x01,0x00,0x00,0x27,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4e,0x00,0x00,0x00,0x2b,0x01,0x00,0x00,0x5b,0x00,0x00,0x00, -0x73,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0x2c,0x01,0x00,0x00, -0x2b,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x33,0x01,0x00,0x00,0x6a,0x00,0x00,0x00,0xb2,0x02,0x00,0x00, -0x73,0x00,0x04,0x00,0x4e,0x00,0x00,0x00,0x36,0x01,0x00,0x00, -0x28,0x01,0x00,0x00,0x41,0x00,0x06,0x00,0x55,0x00,0x00,0x00, -0x37,0x01,0x00,0x00,0x62,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x33,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x37,0x01,0x00,0x00, -0x36,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x3e,0x01,0x00,0x00,0x6a,0x00,0x00,0x00,0xb3,0x02,0x00,0x00, -0x73,0x00,0x04,0x00,0x4e,0x00,0x00,0x00,0x40,0x01,0x00,0x00, -0x2c,0x01,0x00,0x00,0x41,0x00,0x06,0x00,0x55,0x00,0x00,0x00, -0x41,0x01,0x00,0x00,0x62,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x3e,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x41,0x01,0x00,0x00, -0x40,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0x4e,0x00,0x00,0x00, -0x4b,0x01,0x00,0x00,0x56,0x00,0x00,0x00,0x73,0x00,0x04,0x00, -0x4a,0x00,0x00,0x00,0x4c,0x01,0x00,0x00,0x4b,0x01,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4e,0x00,0x00,0x00,0x4f,0x01,0x00,0x00, -0x5b,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0x50,0x01,0x00,0x00,0x4f,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x57,0x01,0x00,0x00,0x6a,0x00,0x00,0x00, -0xb4,0x02,0x00,0x00,0x73,0x00,0x04,0x00,0x4e,0x00,0x00,0x00, -0x5a,0x01,0x00,0x00,0x4c,0x01,0x00,0x00,0x41,0x00,0x06,0x00, -0x55,0x00,0x00,0x00,0x5b,0x01,0x00,0x00,0x62,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x57,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0x5b,0x01,0x00,0x00,0x5a,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x62,0x01,0x00,0x00,0x6a,0x00,0x00,0x00, -0xb5,0x02,0x00,0x00,0x73,0x00,0x04,0x00,0x4e,0x00,0x00,0x00, -0x64,0x01,0x00,0x00,0x50,0x01,0x00,0x00,0x41,0x00,0x06,0x00, -0x55,0x00,0x00,0x00,0x65,0x01,0x00,0x00,0x62,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x62,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0x65,0x01,0x00,0x00,0x64,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4e,0x00,0x00,0x00,0x6f,0x01,0x00,0x00,0x56,0x00,0x00,0x00, -0x73,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0x70,0x01,0x00,0x00, -0x6f,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0x4e,0x00,0x00,0x00, -0x73,0x01,0x00,0x00,0x5b,0x00,0x00,0x00,0x73,0x00,0x04,0x00, -0x4a,0x00,0x00,0x00,0x74,0x01,0x00,0x00,0x73,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x7b,0x01,0x00,0x00, -0x6a,0x00,0x00,0x00,0xb6,0x02,0x00,0x00,0x73,0x00,0x04,0x00, -0x4e,0x00,0x00,0x00,0x7e,0x01,0x00,0x00,0x70,0x01,0x00,0x00, -0x41,0x00,0x06,0x00,0x55,0x00,0x00,0x00,0x7f,0x01,0x00,0x00, -0x62,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x7b,0x01,0x00,0x00, -0x3e,0x00,0x03,0x00,0x7f,0x01,0x00,0x00,0x7e,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x86,0x01,0x00,0x00, -0x6a,0x00,0x00,0x00,0xb7,0x02,0x00,0x00,0x73,0x00,0x04,0x00, -0x4e,0x00,0x00,0x00,0x88,0x01,0x00,0x00,0x74,0x01,0x00,0x00, -0x41,0x00,0x06,0x00,0x55,0x00,0x00,0x00,0x89,0x01,0x00,0x00, -0x62,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x86,0x01,0x00,0x00, -0x3e,0x00,0x03,0x00,0x89,0x01,0x00,0x00,0x88,0x01,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4e,0x00,0x00,0x00,0x93,0x01,0x00,0x00, -0x56,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0x94,0x01,0x00,0x00,0x93,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4e,0x00,0x00,0x00,0x97,0x01,0x00,0x00,0x5b,0x00,0x00,0x00, -0x73,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0x98,0x01,0x00,0x00, -0x97,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x9f,0x01,0x00,0x00,0x6a,0x00,0x00,0x00,0xb8,0x02,0x00,0x00, -0x73,0x00,0x04,0x00,0x4e,0x00,0x00,0x00,0xa2,0x01,0x00,0x00, -0x94,0x01,0x00,0x00,0x41,0x00,0x06,0x00,0x55,0x00,0x00,0x00, -0xa3,0x01,0x00,0x00,0x62,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x9f,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0xa3,0x01,0x00,0x00, -0xa2,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xaa,0x01,0x00,0x00,0x6a,0x00,0x00,0x00,0xb9,0x02,0x00,0x00, -0x73,0x00,0x04,0x00,0x4e,0x00,0x00,0x00,0xac,0x01,0x00,0x00, -0x98,0x01,0x00,0x00,0x41,0x00,0x06,0x00,0x55,0x00,0x00,0x00, -0xad,0x01,0x00,0x00,0x62,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0xaa,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0xad,0x01,0x00,0x00, -0xac,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0x4e,0x00,0x00,0x00, -0xb7,0x01,0x00,0x00,0x56,0x00,0x00,0x00,0x73,0x00,0x04,0x00, -0x4a,0x00,0x00,0x00,0xb8,0x01,0x00,0x00,0xb7,0x01,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4e,0x00,0x00,0x00,0xbb,0x01,0x00,0x00, -0x5b,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0xbc,0x01,0x00,0x00,0xbb,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xc3,0x01,0x00,0x00,0x6a,0x00,0x00,0x00, -0xba,0x02,0x00,0x00,0x73,0x00,0x04,0x00,0x4e,0x00,0x00,0x00, -0xc6,0x01,0x00,0x00,0xb8,0x01,0x00,0x00,0x41,0x00,0x06,0x00, -0x55,0x00,0x00,0x00,0xc7,0x01,0x00,0x00,0x62,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0xc3,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0xc7,0x01,0x00,0x00,0xc6,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xce,0x01,0x00,0x00,0x6a,0x00,0x00,0x00, -0xbb,0x02,0x00,0x00,0x73,0x00,0x04,0x00,0x4e,0x00,0x00,0x00, -0xd0,0x01,0x00,0x00,0xbc,0x01,0x00,0x00,0x41,0x00,0x06,0x00, -0x55,0x00,0x00,0x00,0xd1,0x01,0x00,0x00,0x62,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0xce,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0xd1,0x01,0x00,0x00,0xd0,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4e,0x00,0x00,0x00,0xdb,0x01,0x00,0x00,0x56,0x00,0x00,0x00, -0x73,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0xdc,0x01,0x00,0x00, -0xdb,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0x4e,0x00,0x00,0x00, -0xdf,0x01,0x00,0x00,0x5b,0x00,0x00,0x00,0x73,0x00,0x04,0x00, -0x4a,0x00,0x00,0x00,0xe0,0x01,0x00,0x00,0xdf,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xe7,0x01,0x00,0x00, -0x6a,0x00,0x00,0x00,0xbc,0x02,0x00,0x00,0x73,0x00,0x04,0x00, -0x4e,0x00,0x00,0x00,0xea,0x01,0x00,0x00,0xdc,0x01,0x00,0x00, -0x41,0x00,0x06,0x00,0x55,0x00,0x00,0x00,0xeb,0x01,0x00,0x00, -0x62,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0xe7,0x01,0x00,0x00, -0x3e,0x00,0x03,0x00,0xeb,0x01,0x00,0x00,0xea,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xf2,0x01,0x00,0x00, -0x6a,0x00,0x00,0x00,0xbd,0x02,0x00,0x00,0x73,0x00,0x04,0x00, -0x4e,0x00,0x00,0x00,0xf4,0x01,0x00,0x00,0xe0,0x01,0x00,0x00, -0x41,0x00,0x06,0x00,0x55,0x00,0x00,0x00,0xf5,0x01,0x00,0x00, -0x62,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0xf2,0x01,0x00,0x00, -0x3e,0x00,0x03,0x00,0xf5,0x01,0x00,0x00,0xf4,0x01,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4e,0x00,0x00,0x00,0xff,0x01,0x00,0x00, -0x56,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0x00,0x02,0x00,0x00,0xff,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4e,0x00,0x00,0x00,0x03,0x02,0x00,0x00,0x5b,0x00,0x00,0x00, -0x73,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0x04,0x02,0x00,0x00, -0x03,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x0b,0x02,0x00,0x00,0x6a,0x00,0x00,0x00,0xbe,0x02,0x00,0x00, -0x73,0x00,0x04,0x00,0x4e,0x00,0x00,0x00,0x0e,0x02,0x00,0x00, -0x00,0x02,0x00,0x00,0x41,0x00,0x06,0x00,0x55,0x00,0x00,0x00, -0x0f,0x02,0x00,0x00,0x62,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x0b,0x02,0x00,0x00,0x3e,0x00,0x03,0x00,0x0f,0x02,0x00,0x00, -0x0e,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x16,0x02,0x00,0x00,0x6a,0x00,0x00,0x00,0xbf,0x02,0x00,0x00, -0x73,0x00,0x04,0x00,0x4e,0x00,0x00,0x00,0x18,0x02,0x00,0x00, -0x04,0x02,0x00,0x00,0x41,0x00,0x06,0x00,0x55,0x00,0x00,0x00, -0x19,0x02,0x00,0x00,0x62,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x16,0x02,0x00,0x00,0x3e,0x00,0x03,0x00,0x19,0x02,0x00,0x00, -0x18,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0x4e,0x00,0x00,0x00, -0x23,0x02,0x00,0x00,0x56,0x00,0x00,0x00,0x73,0x00,0x04,0x00, -0x4a,0x00,0x00,0x00,0x24,0x02,0x00,0x00,0x23,0x02,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4e,0x00,0x00,0x00,0x27,0x02,0x00,0x00, -0x5b,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0x28,0x02,0x00,0x00,0x27,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x2f,0x02,0x00,0x00,0x6a,0x00,0x00,0x00, -0xc0,0x02,0x00,0x00,0x73,0x00,0x04,0x00,0x4e,0x00,0x00,0x00, -0x32,0x02,0x00,0x00,0x24,0x02,0x00,0x00,0x41,0x00,0x06,0x00, -0x55,0x00,0x00,0x00,0x33,0x02,0x00,0x00,0x62,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x2f,0x02,0x00,0x00,0x3e,0x00,0x03,0x00, -0x33,0x02,0x00,0x00,0x32,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x3a,0x02,0x00,0x00,0x6a,0x00,0x00,0x00, -0xc1,0x02,0x00,0x00,0x73,0x00,0x04,0x00,0x4e,0x00,0x00,0x00, -0x3c,0x02,0x00,0x00,0x28,0x02,0x00,0x00,0x41,0x00,0x06,0x00, -0x55,0x00,0x00,0x00,0x3d,0x02,0x00,0x00,0x62,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x3a,0x02,0x00,0x00,0x3e,0x00,0x03,0x00, -0x3d,0x02,0x00,0x00,0x3c,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4e,0x00,0x00,0x00,0x47,0x02,0x00,0x00,0x56,0x00,0x00,0x00, -0x73,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0x48,0x02,0x00,0x00, -0x47,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0x4e,0x00,0x00,0x00, -0x4b,0x02,0x00,0x00,0x5b,0x00,0x00,0x00,0x73,0x00,0x04,0x00, -0x4a,0x00,0x00,0x00,0x4c,0x02,0x00,0x00,0x4b,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x53,0x02,0x00,0x00, -0x6a,0x00,0x00,0x00,0xc2,0x02,0x00,0x00,0x73,0x00,0x04,0x00, -0x4e,0x00,0x00,0x00,0x56,0x02,0x00,0x00,0x48,0x02,0x00,0x00, -0x41,0x00,0x06,0x00,0x55,0x00,0x00,0x00,0x57,0x02,0x00,0x00, -0x62,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x53,0x02,0x00,0x00, -0x3e,0x00,0x03,0x00,0x57,0x02,0x00,0x00,0x56,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x5e,0x02,0x00,0x00, -0x6a,0x00,0x00,0x00,0xc3,0x02,0x00,0x00,0x73,0x00,0x04,0x00, -0x4e,0x00,0x00,0x00,0x60,0x02,0x00,0x00,0x4c,0x02,0x00,0x00, -0x41,0x00,0x06,0x00,0x55,0x00,0x00,0x00,0x61,0x02,0x00,0x00, -0x62,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x5e,0x02,0x00,0x00, -0x3e,0x00,0x03,0x00,0x61,0x02,0x00,0x00,0x60,0x02,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4e,0x00,0x00,0x00,0x6b,0x02,0x00,0x00, -0x56,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0x6c,0x02,0x00,0x00,0x6b,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4e,0x00,0x00,0x00,0x6f,0x02,0x00,0x00,0x5b,0x00,0x00,0x00, -0x73,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0x70,0x02,0x00,0x00, -0x6f,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x77,0x02,0x00,0x00,0x6a,0x00,0x00,0x00,0xc4,0x02,0x00,0x00, -0x73,0x00,0x04,0x00,0x4e,0x00,0x00,0x00,0x7a,0x02,0x00,0x00, -0x6c,0x02,0x00,0x00,0x41,0x00,0x06,0x00,0x55,0x00,0x00,0x00, -0x7b,0x02,0x00,0x00,0x62,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x77,0x02,0x00,0x00,0x3e,0x00,0x03,0x00,0x7b,0x02,0x00,0x00, -0x7a,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x82,0x02,0x00,0x00,0x6a,0x00,0x00,0x00,0xc5,0x02,0x00,0x00, -0x73,0x00,0x04,0x00,0x4e,0x00,0x00,0x00,0x84,0x02,0x00,0x00, -0x70,0x02,0x00,0x00,0x41,0x00,0x06,0x00,0x55,0x00,0x00,0x00, -0x85,0x02,0x00,0x00,0x62,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x82,0x02,0x00,0x00,0x3e,0x00,0x03,0x00,0x85,0x02,0x00,0x00, -0x84,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0x4e,0x00,0x00,0x00, -0x8f,0x02,0x00,0x00,0x56,0x00,0x00,0x00,0x73,0x00,0x04,0x00, -0x4a,0x00,0x00,0x00,0x90,0x02,0x00,0x00,0x8f,0x02,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4e,0x00,0x00,0x00,0x93,0x02,0x00,0x00, -0x5b,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0x94,0x02,0x00,0x00,0x93,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x9b,0x02,0x00,0x00,0x6a,0x00,0x00,0x00, -0xc6,0x02,0x00,0x00,0x73,0x00,0x04,0x00,0x4e,0x00,0x00,0x00, -0x9e,0x02,0x00,0x00,0x90,0x02,0x00,0x00,0x41,0x00,0x06,0x00, -0x55,0x00,0x00,0x00,0x9f,0x02,0x00,0x00,0x62,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x9b,0x02,0x00,0x00,0x3e,0x00,0x03,0x00, -0x9f,0x02,0x00,0x00,0x9e,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xa6,0x02,0x00,0x00,0x6a,0x00,0x00,0x00, -0xc7,0x02,0x00,0x00,0x73,0x00,0x04,0x00,0x4e,0x00,0x00,0x00, -0xa8,0x02,0x00,0x00,0x94,0x02,0x00,0x00,0x41,0x00,0x06,0x00, -0x55,0x00,0x00,0x00,0xa9,0x02,0x00,0x00,0x62,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0xa6,0x02,0x00,0x00,0x3e,0x00,0x03,0x00, -0xa9,0x02,0x00,0x00,0xa8,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x86,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x86,0x00,0x00,0x00, -0xfd,0x00,0x01,0x00,0x38,0x00,0x01,0x00, +0x17,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x3d,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3f,0x00,0x00,0x00, +0x3d,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x41,0x00,0x06,0x00, +0x53,0x00,0x00,0x00,0x54,0x00,0x00,0x00,0x50,0x00,0x00,0x00, +0x2d,0x00,0x00,0x00,0x3f,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x4c,0x00,0x00,0x00,0x55,0x00,0x00,0x00,0x54,0x00,0x00,0x00, +0x73,0x00,0x04,0x00,0x48,0x00,0x00,0x00,0x56,0x00,0x00,0x00, +0x55,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x58,0x00,0x00,0x00,0x3f,0x00,0x00,0x00,0x17,0x00,0x00,0x00, +0x41,0x00,0x06,0x00,0x53,0x00,0x00,0x00,0x59,0x00,0x00,0x00, +0x50,0x00,0x00,0x00,0x2d,0x00,0x00,0x00,0x58,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x4c,0x00,0x00,0x00,0x5a,0x00,0x00,0x00, +0x59,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x48,0x00,0x00,0x00, +0x5b,0x00,0x00,0x00,0x5a,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x18,0x00,0x00,0x00,0x63,0x00,0x00,0x00,0x16,0x00,0x00,0x00, +0x62,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x64,0x00,0x00,0x00,0x63,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x65,0x00,0x00,0x00,0x22,0x00,0x00,0x00, +0x64,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x68,0x00,0x00,0x00,0x65,0x00,0x00,0x00,0x1c,0x00,0x00,0x00, +0x73,0x00,0x04,0x00,0x4c,0x00,0x00,0x00,0x6f,0x00,0x00,0x00, +0x56,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x53,0x00,0x00,0x00, +0x70,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x2d,0x00,0x00,0x00, +0x68,0x00,0x00,0x00,0x3e,0x00,0x03,0x00,0x70,0x00,0x00,0x00, +0x6f,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x7a,0x00,0x00,0x00,0x68,0x00,0x00,0x00,0x17,0x00,0x00,0x00, +0x73,0x00,0x04,0x00,0x4c,0x00,0x00,0x00,0x7e,0x00,0x00,0x00, +0x5b,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x53,0x00,0x00,0x00, +0x7f,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x2d,0x00,0x00,0x00, +0x7a,0x00,0x00,0x00,0x3e,0x00,0x03,0x00,0x7f,0x00,0x00,0x00, +0x7e,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x84,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0x84,0x00,0x00,0x00,0xfd,0x00,0x01,0x00, +0x38,0x00,0x01,0x00, }; -const uint64_t dequant_f16_fp32_len = 5420; +const uint64_t dequant_f16_fp32_len = 1816; unsigned char dequant_q2_K_data[] = { 0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, @@ -15313,7 +14792,7 @@ const uint64_t gelu_f32_len = 1408; unsigned char get_rows_f16_data[] = { 0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, -0x7a,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, +0x77,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, 0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x09,0x00,0x00,0x00, 0x11,0x00,0x02,0x00,0x51,0x11,0x00,0x00,0x0b,0x00,0x06,0x00, 0x01,0x00,0x00,0x00,0x47,0x4c,0x53,0x4c,0x2e,0x73,0x74,0x64, @@ -15321,7 +14800,7 @@ unsigned char get_rows_f16_data[] = { 0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x0f,0x00,0x0a,0x00, 0x05,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x6d,0x61,0x69,0x6e, 0x00,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x1f,0x00,0x00,0x00, -0x2d,0x00,0x00,0x00,0x57,0x00,0x00,0x00,0x65,0x00,0x00,0x00, +0x2d,0x00,0x00,0x00,0x55,0x00,0x00,0x00,0x63,0x00,0x00,0x00, 0x10,0x00,0x06,0x00,0x04,0x00,0x00,0x00,0x11,0x00,0x00,0x00, 0x00,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00, 0x47,0x00,0x04,0x00,0x0b,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, @@ -15341,22 +14820,184 @@ unsigned char get_rows_f16_data[] = { 0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x2d,0x00,0x00,0x00, 0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, 0x2d,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x54,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0x55,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x52,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0x53,0x00,0x00,0x00, 0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x55,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x55,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x57,0x00,0x00,0x00, +0x53,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x53,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x55,0x00,0x00,0x00, 0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x57,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x62,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0x63,0x00,0x00,0x00, +0x55,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x60,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0x61,0x00,0x00,0x00, 0x00,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x63,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x63,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x65,0x00,0x00,0x00, +0x61,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x61,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x63,0x00,0x00,0x00, 0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x65,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x63,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x74,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, +0x19,0x00,0x00,0x00,0x13,0x00,0x02,0x00,0x02,0x00,0x00,0x00, +0x21,0x00,0x03,0x00,0x03,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x15,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x17,0x00,0x04,0x00,0x09,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x0a,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x09,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x0d,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x15,0x00,0x04,0x00,0x10,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x10,0x00,0x00,0x00, +0x12,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x16,0x00,0x03,0x00,0x1c,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0x1e,0x00,0x06,0x00,0x1d,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x1c,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x1e,0x00,0x00,0x00,0x09,0x00,0x00,0x00, +0x1d,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x1e,0x00,0x00,0x00, +0x1f,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x10,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x21,0x00,0x00,0x00,0x09,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x14,0x00,0x02,0x00,0x24,0x00,0x00,0x00, +0x1d,0x00,0x03,0x00,0x2a,0x00,0x00,0x00,0x10,0x00,0x00,0x00, +0x1e,0x00,0x03,0x00,0x2b,0x00,0x00,0x00,0x2a,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x2c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x2b,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x2c,0x00,0x00,0x00, +0x2d,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x10,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x30,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x10,0x00,0x00,0x00,0x16,0x00,0x03,0x00,0x4e,0x00,0x00,0x00, +0x10,0x00,0x00,0x00,0x1d,0x00,0x03,0x00,0x52,0x00,0x00,0x00, +0x4e,0x00,0x00,0x00,0x1e,0x00,0x03,0x00,0x53,0x00,0x00,0x00, +0x52,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x54,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0x54,0x00,0x00,0x00,0x55,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x58,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x4e,0x00,0x00,0x00,0x1d,0x00,0x03,0x00,0x60,0x00,0x00,0x00, +0x4e,0x00,0x00,0x00,0x1e,0x00,0x03,0x00,0x61,0x00,0x00,0x00, +0x60,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x62,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x61,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0x62,0x00,0x00,0x00,0x63,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x73,0x00,0x00,0x00, +0x00,0x02,0x00,0x00,0x2c,0x00,0x06,0x00,0x09,0x00,0x00,0x00, +0x74,0x00,0x00,0x00,0x73,0x00,0x00,0x00,0x16,0x00,0x00,0x00, +0x16,0x00,0x00,0x00,0x36,0x00,0x05,0x00,0x02,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x03,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0x05,0x00,0x00,0x00,0xf7,0x00,0x03,0x00, +0x75,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xfb,0x00,0x03,0x00, +0x0c,0x00,0x00,0x00,0x76,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0x76,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00, +0x0e,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, +0x0e,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x10,0x00,0x00,0x00, +0x11,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x10,0x00,0x00,0x00,0x13,0x00,0x00,0x00,0x11,0x00,0x00,0x00, +0x12,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x14,0x00,0x00,0x00,0x13,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x0d,0x00,0x00,0x00,0x17,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, +0x16,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x18,0x00,0x00,0x00,0x17,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, +0x10,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x18,0x00,0x00,0x00, +0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x1a,0x00,0x00,0x00, +0x19,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x21,0x00,0x00,0x00, +0x22,0x00,0x00,0x00,0x1f,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x22,0x00,0x00,0x00,0xae,0x00,0x05,0x00,0x24,0x00,0x00,0x00, +0x25,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0xf7,0x00,0x03,0x00,0x27,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x25,0x00,0x00,0x00,0x26,0x00,0x00,0x00, +0x27,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x26,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x75,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0x27,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x30,0x00,0x00,0x00, +0x31,0x00,0x00,0x00,0x2d,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, +0x1a,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x10,0x00,0x00,0x00, +0x32,0x00,0x00,0x00,0x31,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x33,0x00,0x00,0x00,0x32,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x38,0x00,0x00,0x00, +0x33,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x38,0x00,0x00,0x00, +0x14,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x3f,0x00,0x00,0x00,0x1a,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x41,0x00,0x00,0x00, +0x3f,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x44,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, +0x16,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x47,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x16,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x48,0x00,0x00,0x00, +0x47,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x89,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x4c,0x00,0x00,0x00,0x41,0x00,0x00,0x00, +0x16,0x00,0x00,0x00,0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x4d,0x00,0x00,0x00,0x41,0x00,0x00,0x00,0x4c,0x00,0x00,0x00, +0x41,0x00,0x06,0x00,0x58,0x00,0x00,0x00,0x59,0x00,0x00,0x00, +0x55,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x44,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x4e,0x00,0x00,0x00,0x5a,0x00,0x00,0x00, +0x59,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x5c,0x00,0x00,0x00,0x44,0x00,0x00,0x00,0x16,0x00,0x00,0x00, +0x41,0x00,0x06,0x00,0x58,0x00,0x00,0x00,0x5d,0x00,0x00,0x00, +0x55,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x5c,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x4e,0x00,0x00,0x00,0x5e,0x00,0x00,0x00, +0x5d,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x66,0x00,0x00,0x00,0x4d,0x00,0x00,0x00,0x48,0x00,0x00,0x00, +0x41,0x00,0x06,0x00,0x58,0x00,0x00,0x00,0x6b,0x00,0x00,0x00, +0x63,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x66,0x00,0x00,0x00, +0x3e,0x00,0x03,0x00,0x6b,0x00,0x00,0x00,0x5a,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x6f,0x00,0x00,0x00, +0x66,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x41,0x00,0x06,0x00, +0x58,0x00,0x00,0x00,0x72,0x00,0x00,0x00,0x63,0x00,0x00,0x00, +0x2e,0x00,0x00,0x00,0x6f,0x00,0x00,0x00,0x3e,0x00,0x03,0x00, +0x72,0x00,0x00,0x00,0x5e,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x75,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x75,0x00,0x00,0x00, +0xfd,0x00,0x01,0x00,0x38,0x00,0x01,0x00, +}; +const uint64_t get_rows_f16_len = 1892; + +unsigned char get_rows_f16_f32_data[] = { +0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, +0x7a,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, +0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x09,0x00,0x00,0x00, +0x11,0x00,0x02,0x00,0x51,0x11,0x00,0x00,0x0b,0x00,0x06,0x00, +0x01,0x00,0x00,0x00,0x47,0x4c,0x53,0x4c,0x2e,0x73,0x74,0x64, +0x2e,0x34,0x35,0x30,0x00,0x00,0x00,0x00,0x0e,0x00,0x03,0x00, +0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x0f,0x00,0x0a,0x00, +0x05,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x6d,0x61,0x69,0x6e, +0x00,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x1f,0x00,0x00,0x00, +0x2d,0x00,0x00,0x00,0x55,0x00,0x00,0x00,0x63,0x00,0x00,0x00, +0x10,0x00,0x06,0x00,0x04,0x00,0x00,0x00,0x11,0x00,0x00,0x00, +0x00,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x0b,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, +0x1c,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x1d,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x1d,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x1d,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x08,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x1d,0x00,0x00,0x00, +0x03,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x47,0x00,0x03,0x00,0x1d,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x2a,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0x2b,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x2b,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x2b,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x2d,0x00,0x00,0x00, +0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x2d,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x52,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0x53,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x53,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x53,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x55,0x00,0x00,0x00, +0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x55,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x60,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0x61,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x61,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x61,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x63,0x00,0x00,0x00, +0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x63,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x02,0x00,0x00,0x00, 0x47,0x00,0x04,0x00,0x77,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, 0x19,0x00,0x00,0x00,0x13,0x00,0x02,0x00,0x02,0x00,0x00,0x00, 0x21,0x00,0x03,0x00,0x03,0x00,0x00,0x00,0x02,0x00,0x00,0x00, @@ -15388,198 +15029,28 @@ unsigned char get_rows_f16_data[] = { 0x2d,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, 0x10,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x00,0x00,0x00,0x00, 0x20,0x00,0x04,0x00,0x30,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x10,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x44,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x49,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x16,0x00,0x03,0x00,0x50,0x00,0x00,0x00,0x10,0x00,0x00,0x00, -0x1d,0x00,0x03,0x00,0x54,0x00,0x00,0x00,0x50,0x00,0x00,0x00, -0x1e,0x00,0x03,0x00,0x55,0x00,0x00,0x00,0x54,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x56,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x55,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x56,0x00,0x00,0x00, -0x57,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x5a,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x50,0x00,0x00,0x00, -0x1d,0x00,0x03,0x00,0x62,0x00,0x00,0x00,0x50,0x00,0x00,0x00, -0x1e,0x00,0x03,0x00,0x63,0x00,0x00,0x00,0x62,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x64,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x63,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x64,0x00,0x00,0x00, -0x65,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x71,0x00,0x00,0x00,0x10,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x76,0x00,0x00,0x00, -0x00,0x02,0x00,0x00,0x2c,0x00,0x06,0x00,0x09,0x00,0x00,0x00, -0x77,0x00,0x00,0x00,0x76,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x36,0x00,0x05,0x00,0x02,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x03,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0x05,0x00,0x00,0x00,0xf7,0x00,0x03,0x00, -0x78,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xfb,0x00,0x03,0x00, -0x0c,0x00,0x00,0x00,0x79,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x79,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00, -0x0e,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, -0x0e,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x10,0x00,0x00,0x00, -0x11,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x10,0x00,0x00,0x00,0x13,0x00,0x00,0x00,0x11,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x14,0x00,0x00,0x00,0x13,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x0d,0x00,0x00,0x00,0x17,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x17,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x10,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x18,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x1a,0x00,0x00,0x00, -0x19,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x21,0x00,0x00,0x00, -0x22,0x00,0x00,0x00,0x1f,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x22,0x00,0x00,0x00,0xae,0x00,0x05,0x00,0x24,0x00,0x00,0x00, -0x25,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0xf7,0x00,0x03,0x00,0x27,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x25,0x00,0x00,0x00,0x26,0x00,0x00,0x00, -0x27,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x26,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x78,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x27,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x30,0x00,0x00,0x00, -0x31,0x00,0x00,0x00,0x2d,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x1a,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x10,0x00,0x00,0x00, -0x32,0x00,0x00,0x00,0x31,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x33,0x00,0x00,0x00,0x32,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x38,0x00,0x00,0x00, -0x33,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x38,0x00,0x00,0x00, -0x14,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x3f,0x00,0x00,0x00,0x1a,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x41,0x00,0x00,0x00, -0x3f,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x45,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, -0x44,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x48,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x44,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x4a,0x00,0x00,0x00, -0x48,0x00,0x00,0x00,0x49,0x00,0x00,0x00,0x89,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x41,0x00,0x00,0x00, -0x44,0x00,0x00,0x00,0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x4f,0x00,0x00,0x00,0x41,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, -0x41,0x00,0x06,0x00,0x5a,0x00,0x00,0x00,0x5b,0x00,0x00,0x00, -0x57,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x45,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x50,0x00,0x00,0x00,0x5c,0x00,0x00,0x00, -0x5b,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x5e,0x00,0x00,0x00,0x45,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0x41,0x00,0x06,0x00,0x5a,0x00,0x00,0x00,0x5f,0x00,0x00,0x00, -0x57,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x5e,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x50,0x00,0x00,0x00,0x60,0x00,0x00,0x00, -0x5f,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x68,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x4a,0x00,0x00,0x00, -0x41,0x00,0x06,0x00,0x5a,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, -0x65,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x68,0x00,0x00,0x00, -0x3e,0x00,0x03,0x00,0x6d,0x00,0x00,0x00,0x5c,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x72,0x00,0x00,0x00, -0x68,0x00,0x00,0x00,0x71,0x00,0x00,0x00,0x41,0x00,0x06,0x00, -0x5a,0x00,0x00,0x00,0x75,0x00,0x00,0x00,0x65,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x72,0x00,0x00,0x00,0x3e,0x00,0x03,0x00, -0x75,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x78,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x78,0x00,0x00,0x00, -0xfd,0x00,0x01,0x00,0x38,0x00,0x01,0x00, -}; -const uint64_t get_rows_f16_len = 1940; - -unsigned char get_rows_f16_f32_data[] = { -0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, -0x7d,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, -0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x09,0x00,0x00,0x00, -0x11,0x00,0x02,0x00,0x51,0x11,0x00,0x00,0x0b,0x00,0x06,0x00, -0x01,0x00,0x00,0x00,0x47,0x4c,0x53,0x4c,0x2e,0x73,0x74,0x64, -0x2e,0x34,0x35,0x30,0x00,0x00,0x00,0x00,0x0e,0x00,0x03,0x00, -0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x0f,0x00,0x0a,0x00, -0x05,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x6d,0x61,0x69,0x6e, -0x00,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x1f,0x00,0x00,0x00, -0x2d,0x00,0x00,0x00,0x57,0x00,0x00,0x00,0x65,0x00,0x00,0x00, -0x10,0x00,0x06,0x00,0x04,0x00,0x00,0x00,0x11,0x00,0x00,0x00, -0x00,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x0b,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x1c,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x1d,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x1d,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x1d,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x08,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x1d,0x00,0x00,0x00, -0x03,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x47,0x00,0x03,0x00,0x1d,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x2a,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0x2b,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x2b,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x2b,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x2d,0x00,0x00,0x00, -0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x2d,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x54,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0x55,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x55,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x55,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x57,0x00,0x00,0x00, -0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x57,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x62,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0x63,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x63,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x63,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x65,0x00,0x00,0x00, -0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x65,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x7a,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x19,0x00,0x00,0x00,0x13,0x00,0x02,0x00,0x02,0x00,0x00,0x00, -0x21,0x00,0x03,0x00,0x03,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x15,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x17,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x0a,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x0d,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x15,0x00,0x04,0x00,0x10,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x10,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x16,0x00,0x03,0x00,0x1c,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x1e,0x00,0x06,0x00,0x1d,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x1c,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x1e,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x1d,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x1e,0x00,0x00,0x00, -0x1f,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x10,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x21,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x14,0x00,0x02,0x00,0x24,0x00,0x00,0x00, -0x1d,0x00,0x03,0x00,0x2a,0x00,0x00,0x00,0x10,0x00,0x00,0x00, -0x1e,0x00,0x03,0x00,0x2b,0x00,0x00,0x00,0x2a,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x2c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x2b,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x2c,0x00,0x00,0x00, -0x2d,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x10,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x30,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x10,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x44,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x49,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x16,0x00,0x03,0x00,0x50,0x00,0x00,0x00,0x10,0x00,0x00,0x00, -0x1d,0x00,0x03,0x00,0x54,0x00,0x00,0x00,0x50,0x00,0x00,0x00, -0x1e,0x00,0x03,0x00,0x55,0x00,0x00,0x00,0x54,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x56,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x55,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x56,0x00,0x00,0x00, -0x57,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x5a,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x50,0x00,0x00,0x00, -0x1d,0x00,0x03,0x00,0x62,0x00,0x00,0x00,0x1c,0x00,0x00,0x00, -0x1e,0x00,0x03,0x00,0x63,0x00,0x00,0x00,0x62,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x64,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x63,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x64,0x00,0x00,0x00, -0x65,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x6e,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x1c,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x73,0x00,0x00,0x00, -0x10,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x79,0x00,0x00,0x00,0x00,0x02,0x00,0x00,0x2c,0x00,0x06,0x00, -0x09,0x00,0x00,0x00,0x7a,0x00,0x00,0x00,0x79,0x00,0x00,0x00, +0x10,0x00,0x00,0x00,0x16,0x00,0x03,0x00,0x4e,0x00,0x00,0x00, +0x10,0x00,0x00,0x00,0x1d,0x00,0x03,0x00,0x52,0x00,0x00,0x00, +0x4e,0x00,0x00,0x00,0x1e,0x00,0x03,0x00,0x53,0x00,0x00,0x00, +0x52,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x54,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0x54,0x00,0x00,0x00,0x55,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x58,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x4e,0x00,0x00,0x00,0x1d,0x00,0x03,0x00,0x60,0x00,0x00,0x00, +0x1c,0x00,0x00,0x00,0x1e,0x00,0x03,0x00,0x61,0x00,0x00,0x00, +0x60,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x62,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x61,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0x62,0x00,0x00,0x00,0x63,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x6c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x1c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x76,0x00,0x00,0x00,0x00,0x02,0x00,0x00,0x2c,0x00,0x06,0x00, +0x09,0x00,0x00,0x00,0x77,0x00,0x00,0x00,0x76,0x00,0x00,0x00, 0x16,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x36,0x00,0x05,0x00, 0x02,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x00, 0x03,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x05,0x00,0x00,0x00, -0xf7,0x00,0x03,0x00,0x7b,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0xfb,0x00,0x03,0x00,0x0c,0x00,0x00,0x00,0x7c,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0x7c,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0xf7,0x00,0x03,0x00,0x78,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0xfb,0x00,0x03,0x00,0x0c,0x00,0x00,0x00,0x79,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0x79,0x00,0x00,0x00,0x41,0x00,0x05,0x00, 0x0d,0x00,0x00,0x00,0x0e,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, 0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, 0x0f,0x00,0x00,0x00,0x0e,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, @@ -15600,7 +15071,7 @@ unsigned char get_rows_f16_f32_data[] = { 0x23,0x00,0x00,0x00,0xf7,0x00,0x03,0x00,0x27,0x00,0x00,0x00, 0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x25,0x00,0x00,0x00, 0x26,0x00,0x00,0x00,0x27,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x26,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x7b,0x00,0x00,0x00, +0x26,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x78,0x00,0x00,0x00, 0xf8,0x00,0x02,0x00,0x27,0x00,0x00,0x00,0x41,0x00,0x06,0x00, 0x30,0x00,0x00,0x00,0x31,0x00,0x00,0x00,0x2d,0x00,0x00,0x00, 0x2e,0x00,0x00,0x00,0x1a,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, @@ -15613,51 +15084,51 @@ unsigned char get_rows_f16_f32_data[] = { 0x06,0x00,0x00,0x00,0x3f,0x00,0x00,0x00,0x1a,0x00,0x00,0x00, 0x23,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, 0x41,0x00,0x00,0x00,0x3f,0x00,0x00,0x00,0x14,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x45,0x00,0x00,0x00, -0x3a,0x00,0x00,0x00,0x44,0x00,0x00,0x00,0x89,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x48,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, -0x44,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x4a,0x00,0x00,0x00,0x48,0x00,0x00,0x00,0x49,0x00,0x00,0x00, -0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, -0x41,0x00,0x00,0x00,0x44,0x00,0x00,0x00,0x82,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x41,0x00,0x00,0x00, -0x4e,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x5a,0x00,0x00,0x00, -0x5b,0x00,0x00,0x00,0x57,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x45,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x50,0x00,0x00,0x00, -0x5c,0x00,0x00,0x00,0x5b,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x5e,0x00,0x00,0x00,0x45,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x5a,0x00,0x00,0x00, -0x5f,0x00,0x00,0x00,0x57,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x5e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x50,0x00,0x00,0x00, -0x60,0x00,0x00,0x00,0x5f,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x68,0x00,0x00,0x00,0x4f,0x00,0x00,0x00, -0x4a,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x1c,0x00,0x00,0x00, -0x6d,0x00,0x00,0x00,0x5c,0x00,0x00,0x00,0x41,0x00,0x06,0x00, -0x6e,0x00,0x00,0x00,0x6f,0x00,0x00,0x00,0x65,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x68,0x00,0x00,0x00,0x3e,0x00,0x03,0x00, -0x6f,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x74,0x00,0x00,0x00,0x68,0x00,0x00,0x00, -0x73,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x1c,0x00,0x00,0x00, -0x77,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x41,0x00,0x06,0x00, -0x6e,0x00,0x00,0x00,0x78,0x00,0x00,0x00,0x65,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x74,0x00,0x00,0x00,0x3e,0x00,0x03,0x00, -0x78,0x00,0x00,0x00,0x77,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x7b,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x7b,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x44,0x00,0x00,0x00, +0x3a,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x89,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x47,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, +0x16,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x48,0x00,0x00,0x00,0x47,0x00,0x00,0x00,0x16,0x00,0x00,0x00, +0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x4c,0x00,0x00,0x00, +0x41,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x82,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x4d,0x00,0x00,0x00,0x41,0x00,0x00,0x00, +0x4c,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x58,0x00,0x00,0x00, +0x59,0x00,0x00,0x00,0x55,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, +0x44,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x4e,0x00,0x00,0x00, +0x5a,0x00,0x00,0x00,0x59,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x5c,0x00,0x00,0x00,0x44,0x00,0x00,0x00, +0x16,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x58,0x00,0x00,0x00, +0x5d,0x00,0x00,0x00,0x55,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, +0x5c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x4e,0x00,0x00,0x00, +0x5e,0x00,0x00,0x00,0x5d,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x66,0x00,0x00,0x00,0x4d,0x00,0x00,0x00, +0x48,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x1c,0x00,0x00,0x00, +0x6b,0x00,0x00,0x00,0x5a,0x00,0x00,0x00,0x41,0x00,0x06,0x00, +0x6c,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x63,0x00,0x00,0x00, +0x2e,0x00,0x00,0x00,0x66,0x00,0x00,0x00,0x3e,0x00,0x03,0x00, +0x6d,0x00,0x00,0x00,0x6b,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x71,0x00,0x00,0x00,0x66,0x00,0x00,0x00, +0x16,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x1c,0x00,0x00,0x00, +0x74,0x00,0x00,0x00,0x5e,0x00,0x00,0x00,0x41,0x00,0x06,0x00, +0x6c,0x00,0x00,0x00,0x75,0x00,0x00,0x00,0x63,0x00,0x00,0x00, +0x2e,0x00,0x00,0x00,0x71,0x00,0x00,0x00,0x3e,0x00,0x03,0x00, +0x75,0x00,0x00,0x00,0x74,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x78,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x78,0x00,0x00,0x00, 0xfd,0x00,0x01,0x00,0x38,0x00,0x01,0x00, }; -const uint64_t get_rows_f16_f32_len = 1988; +const uint64_t get_rows_f16_f32_len = 1940; unsigned char get_rows_f16_f32_fp32_data[] = { 0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, -0x7d,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, +0x7a,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, 0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x51,0x11,0x00,0x00, 0x0b,0x00,0x06,0x00,0x01,0x00,0x00,0x00,0x47,0x4c,0x53,0x4c, 0x2e,0x73,0x74,0x64,0x2e,0x34,0x35,0x30,0x00,0x00,0x00,0x00, 0x0e,0x00,0x03,0x00,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00, 0x0f,0x00,0x0a,0x00,0x05,0x00,0x00,0x00,0x04,0x00,0x00,0x00, 0x6d,0x61,0x69,0x6e,0x00,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x1f,0x00,0x00,0x00,0x2d,0x00,0x00,0x00,0x57,0x00,0x00,0x00, -0x67,0x00,0x00,0x00,0x10,0x00,0x06,0x00,0x04,0x00,0x00,0x00, +0x1f,0x00,0x00,0x00,0x2d,0x00,0x00,0x00,0x55,0x00,0x00,0x00, +0x65,0x00,0x00,0x00,0x10,0x00,0x06,0x00,0x04,0x00,0x00,0x00, 0x11,0x00,0x00,0x00,0x00,0x02,0x00,0x00,0x01,0x00,0x00,0x00, 0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x0b,0x00,0x00,0x00, 0x0b,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x48,0x00,0x05,0x00, @@ -15676,23 +15147,23 @@ unsigned char get_rows_f16_f32_fp32_data[] = { 0x2b,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, 0x2d,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, 0x47,0x00,0x04,0x00,0x2d,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x54,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x52,0x00,0x00,0x00, 0x06,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x48,0x00,0x04,0x00, -0x55,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x55,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x53,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x53,0x00,0x00,0x00,0x00,0x00,0x00,0x00, 0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, -0x55,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x57,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x57,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x64,0x00,0x00,0x00, +0x53,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x55,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x55,0x00,0x00,0x00,0x21,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x62,0x00,0x00,0x00, 0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00, -0x65,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x19,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x65,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x63,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x19,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x63,0x00,0x00,0x00,0x00,0x00,0x00,0x00, 0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, -0x65,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x67,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x67,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x7a,0x00,0x00,0x00, +0x63,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x65,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x65,0x00,0x00,0x00,0x21,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x77,0x00,0x00,0x00, 0x0b,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x13,0x00,0x02,0x00, 0x02,0x00,0x00,0x00,0x21,0x00,0x03,0x00,0x03,0x00,0x00,0x00, 0x02,0x00,0x00,0x00,0x15,0x00,0x04,0x00,0x06,0x00,0x00,0x00, @@ -15723,32 +15194,28 @@ unsigned char get_rows_f16_f32_fp32_data[] = { 0x2c,0x00,0x00,0x00,0x2d,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, 0x2b,0x00,0x04,0x00,0x10,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, 0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x30,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x44,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x49,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x16,0x00,0x03,0x00,0x53,0x00,0x00,0x00, -0x10,0x00,0x00,0x00,0x1d,0x00,0x03,0x00,0x54,0x00,0x00,0x00, -0x53,0x00,0x00,0x00,0x1e,0x00,0x03,0x00,0x55,0x00,0x00,0x00, -0x54,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x56,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x55,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x56,0x00,0x00,0x00,0x57,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x5a,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x53,0x00,0x00,0x00,0x1d,0x00,0x03,0x00,0x64,0x00,0x00,0x00, -0x1c,0x00,0x00,0x00,0x1e,0x00,0x03,0x00,0x65,0x00,0x00,0x00, -0x64,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x66,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x65,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x66,0x00,0x00,0x00,0x67,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x6f,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x1c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x74,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x79,0x00,0x00,0x00,0x00,0x02,0x00,0x00, -0x2c,0x00,0x06,0x00,0x09,0x00,0x00,0x00,0x7a,0x00,0x00,0x00, -0x79,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x16,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x16,0x00,0x03,0x00, +0x51,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, +0x52,0x00,0x00,0x00,0x51,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, +0x53,0x00,0x00,0x00,0x52,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x54,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x53,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0x54,0x00,0x00,0x00,0x55,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x58,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x51,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, +0x62,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, +0x63,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x64,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x63,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0x64,0x00,0x00,0x00,0x65,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x6d,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x76,0x00,0x00,0x00,0x00,0x02,0x00,0x00, +0x2c,0x00,0x06,0x00,0x09,0x00,0x00,0x00,0x77,0x00,0x00,0x00, +0x76,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x16,0x00,0x00,0x00, 0x36,0x00,0x05,0x00,0x02,0x00,0x00,0x00,0x04,0x00,0x00,0x00, 0x00,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x05,0x00,0x00,0x00,0xf7,0x00,0x03,0x00,0x7b,0x00,0x00,0x00, +0x05,0x00,0x00,0x00,0xf7,0x00,0x03,0x00,0x78,0x00,0x00,0x00, 0x00,0x00,0x00,0x00,0xfb,0x00,0x03,0x00,0x0c,0x00,0x00,0x00, -0x7c,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x7c,0x00,0x00,0x00, +0x79,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x79,0x00,0x00,0x00, 0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00,0x0e,0x00,0x00,0x00, 0x0b,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, 0x06,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x0e,0x00,0x00,0x00, @@ -15770,7 +15237,7 @@ unsigned char get_rows_f16_f32_fp32_data[] = { 0x27,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, 0x25,0x00,0x00,0x00,0x26,0x00,0x00,0x00,0x27,0x00,0x00,0x00, 0xf8,0x00,0x02,0x00,0x26,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x7b,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x27,0x00,0x00,0x00, +0x78,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x27,0x00,0x00,0x00, 0x41,0x00,0x06,0x00,0x30,0x00,0x00,0x00,0x31,0x00,0x00,0x00, 0x2d,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x1a,0x00,0x00,0x00, 0x3d,0x00,0x04,0x00,0x10,0x00,0x00,0x00,0x32,0x00,0x00,0x00, @@ -15783,51 +15250,51 @@ unsigned char get_rows_f16_f32_fp32_data[] = { 0x1a,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x80,0x00,0x05,0x00, 0x06,0x00,0x00,0x00,0x41,0x00,0x00,0x00,0x3f,0x00,0x00,0x00, 0x14,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x45,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x44,0x00,0x00,0x00, -0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x48,0x00,0x00,0x00, -0x3a,0x00,0x00,0x00,0x44,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x4a,0x00,0x00,0x00,0x48,0x00,0x00,0x00, -0x49,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x4e,0x00,0x00,0x00,0x41,0x00,0x00,0x00,0x44,0x00,0x00,0x00, -0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x4f,0x00,0x00,0x00, -0x41,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x41,0x00,0x06,0x00, -0x5a,0x00,0x00,0x00,0x5b,0x00,0x00,0x00,0x57,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x45,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x53,0x00,0x00,0x00,0x5c,0x00,0x00,0x00,0x5b,0x00,0x00,0x00, -0x73,0x00,0x04,0x00,0x1c,0x00,0x00,0x00,0x5d,0x00,0x00,0x00, -0x5c,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x5f,0x00,0x00,0x00,0x45,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0x41,0x00,0x06,0x00,0x5a,0x00,0x00,0x00,0x60,0x00,0x00,0x00, -0x57,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x5f,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x53,0x00,0x00,0x00,0x61,0x00,0x00,0x00, -0x60,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x1c,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x61,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x6a,0x00,0x00,0x00,0x4f,0x00,0x00,0x00, -0x4a,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x6f,0x00,0x00,0x00, -0x70,0x00,0x00,0x00,0x67,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x6a,0x00,0x00,0x00,0x3e,0x00,0x03,0x00,0x70,0x00,0x00,0x00, -0x5d,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x75,0x00,0x00,0x00,0x6a,0x00,0x00,0x00,0x74,0x00,0x00,0x00, -0x41,0x00,0x06,0x00,0x6f,0x00,0x00,0x00,0x78,0x00,0x00,0x00, -0x67,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x75,0x00,0x00,0x00, -0x3e,0x00,0x03,0x00,0x78,0x00,0x00,0x00,0x62,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x7b,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x7b,0x00,0x00,0x00,0xfd,0x00,0x01,0x00,0x38,0x00,0x01,0x00, +0x44,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x16,0x00,0x00,0x00, +0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x47,0x00,0x00,0x00, +0x3a,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x48,0x00,0x00,0x00,0x47,0x00,0x00,0x00, +0x16,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x4c,0x00,0x00,0x00,0x41,0x00,0x00,0x00,0x16,0x00,0x00,0x00, +0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x4d,0x00,0x00,0x00, +0x41,0x00,0x00,0x00,0x4c,0x00,0x00,0x00,0x41,0x00,0x06,0x00, +0x58,0x00,0x00,0x00,0x59,0x00,0x00,0x00,0x55,0x00,0x00,0x00, +0x2e,0x00,0x00,0x00,0x44,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x51,0x00,0x00,0x00,0x5a,0x00,0x00,0x00,0x59,0x00,0x00,0x00, +0x73,0x00,0x04,0x00,0x1c,0x00,0x00,0x00,0x5b,0x00,0x00,0x00, +0x5a,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x5d,0x00,0x00,0x00,0x44,0x00,0x00,0x00,0x16,0x00,0x00,0x00, +0x41,0x00,0x06,0x00,0x58,0x00,0x00,0x00,0x5e,0x00,0x00,0x00, +0x55,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x5d,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x51,0x00,0x00,0x00,0x5f,0x00,0x00,0x00, +0x5e,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x1c,0x00,0x00,0x00, +0x60,0x00,0x00,0x00,0x5f,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x68,0x00,0x00,0x00,0x4d,0x00,0x00,0x00, +0x48,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x6d,0x00,0x00,0x00, +0x6e,0x00,0x00,0x00,0x65,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, +0x68,0x00,0x00,0x00,0x3e,0x00,0x03,0x00,0x6e,0x00,0x00,0x00, +0x5b,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x72,0x00,0x00,0x00,0x68,0x00,0x00,0x00,0x16,0x00,0x00,0x00, +0x41,0x00,0x06,0x00,0x6d,0x00,0x00,0x00,0x75,0x00,0x00,0x00, +0x65,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x72,0x00,0x00,0x00, +0x3e,0x00,0x03,0x00,0x75,0x00,0x00,0x00,0x60,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x78,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0x78,0x00,0x00,0x00,0xfd,0x00,0x01,0x00,0x38,0x00,0x01,0x00, }; -const uint64_t get_rows_f16_f32_fp32_len = 1980; +const uint64_t get_rows_f16_f32_fp32_len = 1932; unsigned char get_rows_f16_fp32_data[] = { 0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, -0x7e,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, +0x7b,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, 0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x51,0x11,0x00,0x00, 0x0b,0x00,0x06,0x00,0x01,0x00,0x00,0x00,0x47,0x4c,0x53,0x4c, 0x2e,0x73,0x74,0x64,0x2e,0x34,0x35,0x30,0x00,0x00,0x00,0x00, 0x0e,0x00,0x03,0x00,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00, 0x0f,0x00,0x0a,0x00,0x05,0x00,0x00,0x00,0x04,0x00,0x00,0x00, 0x6d,0x61,0x69,0x6e,0x00,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x1f,0x00,0x00,0x00,0x2d,0x00,0x00,0x00,0x57,0x00,0x00,0x00, -0x67,0x00,0x00,0x00,0x10,0x00,0x06,0x00,0x04,0x00,0x00,0x00, +0x1f,0x00,0x00,0x00,0x2d,0x00,0x00,0x00,0x55,0x00,0x00,0x00, +0x65,0x00,0x00,0x00,0x10,0x00,0x06,0x00,0x04,0x00,0x00,0x00, 0x11,0x00,0x00,0x00,0x00,0x02,0x00,0x00,0x01,0x00,0x00,0x00, 0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x0b,0x00,0x00,0x00, 0x0b,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x48,0x00,0x05,0x00, @@ -15846,23 +15313,23 @@ unsigned char get_rows_f16_fp32_data[] = { 0x2b,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, 0x2d,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, 0x47,0x00,0x04,0x00,0x2d,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x54,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x52,0x00,0x00,0x00, 0x06,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x48,0x00,0x04,0x00, -0x55,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x55,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x53,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x53,0x00,0x00,0x00,0x00,0x00,0x00,0x00, 0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, -0x55,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x57,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x57,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x64,0x00,0x00,0x00, +0x53,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x55,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x55,0x00,0x00,0x00,0x21,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x62,0x00,0x00,0x00, 0x06,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x48,0x00,0x04,0x00, -0x65,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x19,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x65,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x63,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x19,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x63,0x00,0x00,0x00,0x00,0x00,0x00,0x00, 0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, -0x65,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x67,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x67,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x7b,0x00,0x00,0x00, +0x63,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x65,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x65,0x00,0x00,0x00,0x21,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x78,0x00,0x00,0x00, 0x0b,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x13,0x00,0x02,0x00, 0x02,0x00,0x00,0x00,0x21,0x00,0x03,0x00,0x03,0x00,0x00,0x00, 0x02,0x00,0x00,0x00,0x15,0x00,0x04,0x00,0x06,0x00,0x00,0x00, @@ -15893,31 +15360,27 @@ unsigned char get_rows_f16_fp32_data[] = { 0x2c,0x00,0x00,0x00,0x2d,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, 0x2b,0x00,0x04,0x00,0x10,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, 0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x30,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x44,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x49,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x16,0x00,0x03,0x00,0x53,0x00,0x00,0x00, -0x10,0x00,0x00,0x00,0x1d,0x00,0x03,0x00,0x54,0x00,0x00,0x00, -0x53,0x00,0x00,0x00,0x1e,0x00,0x03,0x00,0x55,0x00,0x00,0x00, -0x54,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x56,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x55,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x56,0x00,0x00,0x00,0x57,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x5a,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x53,0x00,0x00,0x00,0x1d,0x00,0x03,0x00,0x64,0x00,0x00,0x00, -0x53,0x00,0x00,0x00,0x1e,0x00,0x03,0x00,0x65,0x00,0x00,0x00, -0x64,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x66,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x65,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x66,0x00,0x00,0x00,0x67,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x74,0x00,0x00,0x00, -0x10,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x7a,0x00,0x00,0x00,0x00,0x02,0x00,0x00,0x2c,0x00,0x06,0x00, -0x09,0x00,0x00,0x00,0x7b,0x00,0x00,0x00,0x7a,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x16,0x00,0x03,0x00, +0x51,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, +0x52,0x00,0x00,0x00,0x51,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, +0x53,0x00,0x00,0x00,0x52,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x54,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x53,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0x54,0x00,0x00,0x00,0x55,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x58,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x51,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, +0x62,0x00,0x00,0x00,0x51,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, +0x63,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x64,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x63,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0x64,0x00,0x00,0x00,0x65,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x77,0x00,0x00,0x00,0x00,0x02,0x00,0x00,0x2c,0x00,0x06,0x00, +0x09,0x00,0x00,0x00,0x78,0x00,0x00,0x00,0x77,0x00,0x00,0x00, 0x16,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x36,0x00,0x05,0x00, 0x02,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x00, 0x03,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x05,0x00,0x00,0x00, -0xf7,0x00,0x03,0x00,0x7c,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0xfb,0x00,0x03,0x00,0x0c,0x00,0x00,0x00,0x7d,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0x7d,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0xf7,0x00,0x03,0x00,0x79,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0xfb,0x00,0x03,0x00,0x0c,0x00,0x00,0x00,0x7a,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0x7a,0x00,0x00,0x00,0x41,0x00,0x05,0x00, 0x0d,0x00,0x00,0x00,0x0e,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, 0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, 0x0f,0x00,0x00,0x00,0x0e,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, @@ -15938,7 +15401,7 @@ unsigned char get_rows_f16_fp32_data[] = { 0x23,0x00,0x00,0x00,0xf7,0x00,0x03,0x00,0x27,0x00,0x00,0x00, 0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x25,0x00,0x00,0x00, 0x26,0x00,0x00,0x00,0x27,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x26,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x7c,0x00,0x00,0x00, +0x26,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x79,0x00,0x00,0x00, 0xf8,0x00,0x02,0x00,0x27,0x00,0x00,0x00,0x41,0x00,0x06,0x00, 0x30,0x00,0x00,0x00,0x31,0x00,0x00,0x00,0x2d,0x00,0x00,0x00, 0x2e,0x00,0x00,0x00,0x1a,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, @@ -15951,42 +15414,42 @@ unsigned char get_rows_f16_fp32_data[] = { 0x06,0x00,0x00,0x00,0x3f,0x00,0x00,0x00,0x1a,0x00,0x00,0x00, 0x23,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, 0x41,0x00,0x00,0x00,0x3f,0x00,0x00,0x00,0x14,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x45,0x00,0x00,0x00, -0x3a,0x00,0x00,0x00,0x44,0x00,0x00,0x00,0x89,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x48,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, -0x44,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x4a,0x00,0x00,0x00,0x48,0x00,0x00,0x00,0x49,0x00,0x00,0x00, -0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, -0x41,0x00,0x00,0x00,0x44,0x00,0x00,0x00,0x82,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x41,0x00,0x00,0x00, -0x4e,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x5a,0x00,0x00,0x00, -0x5b,0x00,0x00,0x00,0x57,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x45,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x53,0x00,0x00,0x00, -0x5c,0x00,0x00,0x00,0x5b,0x00,0x00,0x00,0x73,0x00,0x04,0x00, -0x1c,0x00,0x00,0x00,0x5d,0x00,0x00,0x00,0x5c,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x5f,0x00,0x00,0x00, -0x45,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x41,0x00,0x06,0x00, -0x5a,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x57,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x5f,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x53,0x00,0x00,0x00,0x61,0x00,0x00,0x00,0x60,0x00,0x00,0x00, -0x73,0x00,0x04,0x00,0x1c,0x00,0x00,0x00,0x62,0x00,0x00,0x00, -0x61,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x6a,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x4a,0x00,0x00,0x00, -0x73,0x00,0x04,0x00,0x53,0x00,0x00,0x00,0x6f,0x00,0x00,0x00, -0x5d,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x5a,0x00,0x00,0x00, -0x70,0x00,0x00,0x00,0x67,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x6a,0x00,0x00,0x00,0x3e,0x00,0x03,0x00,0x70,0x00,0x00,0x00, -0x6f,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x75,0x00,0x00,0x00,0x6a,0x00,0x00,0x00,0x74,0x00,0x00,0x00, -0x73,0x00,0x04,0x00,0x53,0x00,0x00,0x00,0x78,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x5a,0x00,0x00,0x00, -0x79,0x00,0x00,0x00,0x67,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x75,0x00,0x00,0x00,0x3e,0x00,0x03,0x00,0x79,0x00,0x00,0x00, -0x78,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x7c,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0x7c,0x00,0x00,0x00,0xfd,0x00,0x01,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x44,0x00,0x00,0x00, +0x3a,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x89,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x47,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, +0x16,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x48,0x00,0x00,0x00,0x47,0x00,0x00,0x00,0x16,0x00,0x00,0x00, +0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x4c,0x00,0x00,0x00, +0x41,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x82,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x4d,0x00,0x00,0x00,0x41,0x00,0x00,0x00, +0x4c,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x58,0x00,0x00,0x00, +0x59,0x00,0x00,0x00,0x55,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, +0x44,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x51,0x00,0x00,0x00, +0x5a,0x00,0x00,0x00,0x59,0x00,0x00,0x00,0x73,0x00,0x04,0x00, +0x1c,0x00,0x00,0x00,0x5b,0x00,0x00,0x00,0x5a,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x5d,0x00,0x00,0x00, +0x44,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x41,0x00,0x06,0x00, +0x58,0x00,0x00,0x00,0x5e,0x00,0x00,0x00,0x55,0x00,0x00,0x00, +0x2e,0x00,0x00,0x00,0x5d,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x51,0x00,0x00,0x00,0x5f,0x00,0x00,0x00,0x5e,0x00,0x00,0x00, +0x73,0x00,0x04,0x00,0x1c,0x00,0x00,0x00,0x60,0x00,0x00,0x00, +0x5f,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x68,0x00,0x00,0x00,0x4d,0x00,0x00,0x00,0x48,0x00,0x00,0x00, +0x73,0x00,0x04,0x00,0x51,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, +0x5b,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x58,0x00,0x00,0x00, +0x6e,0x00,0x00,0x00,0x65,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, +0x68,0x00,0x00,0x00,0x3e,0x00,0x03,0x00,0x6e,0x00,0x00,0x00, +0x6d,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x72,0x00,0x00,0x00,0x68,0x00,0x00,0x00,0x16,0x00,0x00,0x00, +0x73,0x00,0x04,0x00,0x51,0x00,0x00,0x00,0x75,0x00,0x00,0x00, +0x60,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x58,0x00,0x00,0x00, +0x76,0x00,0x00,0x00,0x65,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, +0x72,0x00,0x00,0x00,0x3e,0x00,0x03,0x00,0x76,0x00,0x00,0x00, +0x75,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x79,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0x79,0x00,0x00,0x00,0xfd,0x00,0x01,0x00, 0x38,0x00,0x01,0x00, }; -const uint64_t get_rows_f16_fp32_len = 1996; +const uint64_t get_rows_f16_fp32_len = 1948; unsigned char get_rows_q4_0_data[] = { 0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, @@ -52701,7 +52164,7 @@ const uint64_t mul_f32_len = 1456; unsigned char mul_mat_vec_f16_f32_data[] = { 0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, -0xba,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, +0xb6,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, 0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x51,0x11,0x00,0x00, 0x0b,0x00,0x06,0x00,0x01,0x00,0x00,0x00,0x47,0x4c,0x53,0x4c, 0x2e,0x73,0x74,0x64,0x2e,0x34,0x35,0x30,0x00,0x00,0x00,0x00, @@ -52709,9 +52172,9 @@ unsigned char mul_mat_vec_f16_f32_data[] = { 0x0f,0x00,0x0c,0x00,0x05,0x00,0x00,0x00,0x04,0x00,0x00,0x00, 0x6d,0x61,0x69,0x6e,0x00,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, 0x13,0x00,0x00,0x00,0x1b,0x00,0x00,0x00,0x2a,0x00,0x00,0x00, -0x51,0x00,0x00,0x00,0x66,0x00,0x00,0x00,0xad,0x00,0x00,0x00, +0x51,0x00,0x00,0x00,0x65,0x00,0x00,0x00,0xaa,0x00,0x00,0x00, 0x10,0x00,0x06,0x00,0x04,0x00,0x00,0x00,0x11,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00, 0x47,0x00,0x04,0x00,0x0c,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, 0x1a,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x13,0x00,0x00,0x00, 0x0b,0x00,0x00,0x00,0x1b,0x00,0x00,0x00,0x48,0x00,0x05,0x00, @@ -52729,23 +52192,23 @@ unsigned char mul_mat_vec_f16_f32_data[] = { 0x47,0x00,0x04,0x00,0x51,0x00,0x00,0x00,0x22,0x00,0x00,0x00, 0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x51,0x00,0x00,0x00, 0x21,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x63,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x48,0x00,0x04,0x00,0x64,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x64,0x00,0x00,0x00, +0x62,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x48,0x00,0x04,0x00,0x63,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x63,0x00,0x00,0x00, 0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x03,0x00,0x64,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x66,0x00,0x00,0x00,0x22,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x66,0x00,0x00,0x00, +0x47,0x00,0x03,0x00,0x63,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x65,0x00,0x00,0x00,0x22,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x65,0x00,0x00,0x00, 0x21,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0xaa,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x48,0x00,0x04,0x00,0xab,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x19,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0xab,0x00,0x00,0x00, +0xa7,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x48,0x00,0x04,0x00,0xa8,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x19,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0xa8,0x00,0x00,0x00, 0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x03,0x00,0xab,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0xad,0x00,0x00,0x00,0x22,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xad,0x00,0x00,0x00, +0x47,0x00,0x03,0x00,0xa8,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0xaa,0x00,0x00,0x00,0x22,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xaa,0x00,0x00,0x00, 0x21,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0xb5,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x19,0x00,0x00,0x00, +0xb2,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x19,0x00,0x00,0x00, 0x13,0x00,0x02,0x00,0x02,0x00,0x00,0x00,0x21,0x00,0x03,0x00, 0x03,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x15,0x00,0x04,0x00, 0x06,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x01,0x00,0x00,0x00, @@ -52760,7 +52223,7 @@ unsigned char mul_mat_vec_f16_f32_data[] = { 0x3b,0x00,0x04,0x00,0x0b,0x00,0x00,0x00,0x13,0x00,0x00,0x00, 0x01,0x00,0x00,0x00,0x16,0x00,0x03,0x00,0x17,0x00,0x00,0x00, 0x20,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x1c,0x00,0x04,0x00, +0x18,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x1c,0x00,0x04,0x00, 0x19,0x00,0x00,0x00,0x17,0x00,0x00,0x00,0x18,0x00,0x00,0x00, 0x20,0x00,0x04,0x00,0x1a,0x00,0x00,0x00,0x04,0x00,0x00,0x00, 0x19,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x1a,0x00,0x00,0x00, @@ -52775,7 +52238,7 @@ unsigned char mul_mat_vec_f16_f32_data[] = { 0x29,0x00,0x00,0x00,0x2a,0x00,0x00,0x00,0x09,0x00,0x00,0x00, 0x20,0x00,0x04,0x00,0x2b,0x00,0x00,0x00,0x09,0x00,0x00,0x00, 0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x14,0x00,0x02,0x00, +0x2e,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x14,0x00,0x02,0x00, 0x30,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, 0x35,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x16,0x00,0x03,0x00, 0x4d,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, @@ -52784,26 +52247,22 @@ unsigned char mul_mat_vec_f16_f32_data[] = { 0x50,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x4f,0x00,0x00,0x00, 0x3b,0x00,0x04,0x00,0x50,0x00,0x00,0x00,0x51,0x00,0x00,0x00, 0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x54,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x4d,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x59,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x1d,0x00,0x03,0x00,0x63,0x00,0x00,0x00,0x17,0x00,0x00,0x00, -0x1e,0x00,0x03,0x00,0x64,0x00,0x00,0x00,0x63,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x65,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x64,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x65,0x00,0x00,0x00, -0x66,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x6e,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x17,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x77,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x80,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x8b,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x8c,0x00,0x00,0x00, -0x08,0x01,0x00,0x00,0x1d,0x00,0x03,0x00,0xaa,0x00,0x00,0x00, -0x17,0x00,0x00,0x00,0x1e,0x00,0x03,0x00,0xab,0x00,0x00,0x00, -0xaa,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0xac,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0xab,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0xac,0x00,0x00,0x00,0xad,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x2c,0x00,0x06,0x00,0x0a,0x00,0x00,0x00,0xb5,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x77,0x00,0x00,0x00,0x77,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x4d,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, +0x62,0x00,0x00,0x00,0x17,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, +0x63,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x64,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x63,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0x64,0x00,0x00,0x00,0x65,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x6d,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x17,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x09,0x00,0x00,0x00,0x88,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x89,0x00,0x00,0x00, +0x08,0x01,0x00,0x00,0x1d,0x00,0x03,0x00,0xa7,0x00,0x00,0x00, +0x17,0x00,0x00,0x00,0x1e,0x00,0x03,0x00,0xa8,0x00,0x00,0x00, +0xa7,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0xa9,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0xa8,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0xa9,0x00,0x00,0x00,0xaa,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x2c,0x00,0x06,0x00,0x0a,0x00,0x00,0x00,0xb2,0x00,0x00,0x00, +0x18,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x18,0x00,0x00,0x00, 0x36,0x00,0x05,0x00,0x02,0x00,0x00,0x00,0x04,0x00,0x00,0x00, 0x00,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, 0x05,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x0e,0x00,0x00,0x00, @@ -52819,122 +52278,91 @@ unsigned char mul_mat_vec_f16_f32_data[] = { 0x1b,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x3e,0x00,0x03,0x00, 0x1f,0x00,0x00,0x00,0x1d,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, 0x22,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x22,0x00,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xb8,0x00,0x00,0x00, -0x21,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0x8a,0x00,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xb5,0x00,0x00,0x00, +0x21,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0x87,0x00,0x00,0x00, 0x23,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x2b,0x00,0x00,0x00, 0x2c,0x00,0x00,0x00,0x2a,0x00,0x00,0x00,0x21,0x00,0x00,0x00, 0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x2d,0x00,0x00,0x00, 0x2c,0x00,0x00,0x00,0x87,0x00,0x05,0x00,0x06,0x00,0x00,0x00, 0x2f,0x00,0x00,0x00,0x2d,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, 0xb1,0x00,0x05,0x00,0x30,0x00,0x00,0x00,0x31,0x00,0x00,0x00, -0xb8,0x00,0x00,0x00,0x2f,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0xb5,0x00,0x00,0x00,0x2f,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, 0x24,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x01,0x00,0x00,0x00, 0xfa,0x00,0x04,0x00,0x31,0x00,0x00,0x00,0x23,0x00,0x00,0x00, 0x24,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x23,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x34,0x00,0x00,0x00, -0xb8,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x35,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x38,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0x37,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3d,0x00,0x00,0x00, -0x11,0x00,0x00,0x00,0x2d,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x3f,0x00,0x00,0x00,0x3d,0x00,0x00,0x00, -0x38,0x00,0x00,0x00,0x87,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0x3f,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x8b,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x43,0x00,0x00,0x00, -0x38,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x87,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x44,0x00,0x00,0x00,0x43,0x00,0x00,0x00, -0x35,0x00,0x00,0x00,0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x49,0x00,0x00,0x00,0x38,0x00,0x00,0x00,0x43,0x00,0x00,0x00, -0x41,0x00,0x06,0x00,0x54,0x00,0x00,0x00,0x55,0x00,0x00,0x00, -0x51,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00,0x56,0x00,0x00,0x00, -0x55,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x17,0x00,0x00,0x00, -0x57,0x00,0x00,0x00,0x56,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x5a,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x59,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x54,0x00,0x00,0x00, -0x5b,0x00,0x00,0x00,0x51,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x5a,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0x5c,0x00,0x00,0x00,0x5b,0x00,0x00,0x00,0x73,0x00,0x04,0x00, -0x17,0x00,0x00,0x00,0x5d,0x00,0x00,0x00,0x5c,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x2b,0x00,0x00,0x00,0x67,0x00,0x00,0x00, -0x2a,0x00,0x00,0x00,0x59,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x68,0x00,0x00,0x00,0x67,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x6a,0x00,0x00,0x00, -0x68,0x00,0x00,0x00,0x49,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x6a,0x00,0x00,0x00, -0x44,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x6e,0x00,0x00,0x00, -0x6f,0x00,0x00,0x00,0x66,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x17,0x00,0x00,0x00, -0x70,0x00,0x00,0x00,0x6f,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x17,0x00,0x00,0x00,0x73,0x00,0x00,0x00,0x1f,0x00,0x00,0x00, -0x0c,0x00,0x08,0x00,0x17,0x00,0x00,0x00,0x74,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x57,0x00,0x00,0x00, -0x70,0x00,0x00,0x00,0x73,0x00,0x00,0x00,0x3e,0x00,0x03,0x00, -0x1f,0x00,0x00,0x00,0x74,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x81,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x80,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x6e,0x00,0x00,0x00, -0x82,0x00,0x00,0x00,0x66,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x81,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x17,0x00,0x00,0x00, -0x83,0x00,0x00,0x00,0x82,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x17,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x1f,0x00,0x00,0x00, -0x0c,0x00,0x08,0x00,0x17,0x00,0x00,0x00,0x87,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x5d,0x00,0x00,0x00, -0x83,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x3e,0x00,0x03,0x00, -0x1f,0x00,0x00,0x00,0x87,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x8a,0x00,0x00,0x00,0xb8,0x00,0x00,0x00, -0x35,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x22,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0x24,0x00,0x00,0x00,0xe0,0x00,0x04,0x00, -0x8b,0x00,0x00,0x00,0x8b,0x00,0x00,0x00,0x8c,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x8e,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x8e,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xb9,0x00,0x00,0x00,0x80,0x00,0x00,0x00,0x24,0x00,0x00,0x00, -0xa5,0x00,0x00,0x00,0x91,0x00,0x00,0x00,0xad,0x00,0x05,0x00, -0x30,0x00,0x00,0x00,0x94,0x00,0x00,0x00,0xb9,0x00,0x00,0x00, -0x21,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x90,0x00,0x00,0x00, -0x91,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x94,0x00,0x00,0x00,0x8f,0x00,0x00,0x00,0x90,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0x8f,0x00,0x00,0x00,0xb1,0x00,0x05,0x00, -0x30,0x00,0x00,0x00,0x97,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0xb9,0x00,0x00,0x00,0xf7,0x00,0x03,0x00,0x99,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x97,0x00,0x00,0x00, -0x98,0x00,0x00,0x00,0x99,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x98,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x9d,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0xb9,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x1e,0x00,0x00,0x00,0x9e,0x00,0x00,0x00, -0x1b,0x00,0x00,0x00,0x9d,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x17,0x00,0x00,0x00,0x9f,0x00,0x00,0x00,0x9e,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x17,0x00,0x00,0x00,0xa1,0x00,0x00,0x00, -0x1f,0x00,0x00,0x00,0x81,0x00,0x05,0x00,0x17,0x00,0x00,0x00, -0xa2,0x00,0x00,0x00,0xa1,0x00,0x00,0x00,0x9f,0x00,0x00,0x00, -0x3e,0x00,0x03,0x00,0x1f,0x00,0x00,0x00,0xa2,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x99,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x99,0x00,0x00,0x00,0xe0,0x00,0x04,0x00,0x8b,0x00,0x00,0x00, -0x8b,0x00,0x00,0x00,0x8c,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x91,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x91,0x00,0x00,0x00, -0xc3,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa5,0x00,0x00,0x00, -0xb9,0x00,0x00,0x00,0x59,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x8e,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x90,0x00,0x00,0x00, -0xaa,0x00,0x05,0x00,0x30,0x00,0x00,0x00,0xa7,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0xf7,0x00,0x03,0x00, -0xa9,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0xa7,0x00,0x00,0x00,0xa8,0x00,0x00,0x00,0xa9,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xa8,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x2b,0x00,0x00,0x00,0xae,0x00,0x00,0x00,0x2a,0x00,0x00,0x00, -0x35,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xaf,0x00,0x00,0x00,0xae,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xb1,0x00,0x00,0x00,0xaf,0x00,0x00,0x00, -0x11,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x1e,0x00,0x00,0x00, -0xb2,0x00,0x00,0x00,0x1b,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x17,0x00,0x00,0x00,0xb3,0x00,0x00,0x00, -0xb2,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x6e,0x00,0x00,0x00, -0xb4,0x00,0x00,0x00,0xad,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0xb1,0x00,0x00,0x00,0x3e,0x00,0x03,0x00,0xb4,0x00,0x00,0x00, -0xb3,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xa9,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xa9,0x00,0x00,0x00,0xfd,0x00,0x01,0x00, -0x38,0x00,0x01,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x37,0x00,0x00,0x00, +0x35,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x38,0x00,0x00,0x00,0xb5,0x00,0x00,0x00, +0x37,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x3d,0x00,0x00,0x00,0x11,0x00,0x00,0x00,0x2d,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3f,0x00,0x00,0x00, +0x3d,0x00,0x00,0x00,0x38,0x00,0x00,0x00,0x87,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x3f,0x00,0x00,0x00, +0x2e,0x00,0x00,0x00,0x8b,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x43,0x00,0x00,0x00,0x38,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, +0x87,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x44,0x00,0x00,0x00, +0x43,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x82,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x49,0x00,0x00,0x00,0x38,0x00,0x00,0x00, +0x43,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x54,0x00,0x00,0x00, +0x55,0x00,0x00,0x00,0x51,0x00,0x00,0x00,0x21,0x00,0x00,0x00, +0x40,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, +0x56,0x00,0x00,0x00,0x55,0x00,0x00,0x00,0x73,0x00,0x04,0x00, +0x17,0x00,0x00,0x00,0x57,0x00,0x00,0x00,0x56,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x59,0x00,0x00,0x00, +0x40,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x41,0x00,0x06,0x00, +0x54,0x00,0x00,0x00,0x5a,0x00,0x00,0x00,0x51,0x00,0x00,0x00, +0x21,0x00,0x00,0x00,0x59,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x4d,0x00,0x00,0x00,0x5b,0x00,0x00,0x00,0x5a,0x00,0x00,0x00, +0x73,0x00,0x04,0x00,0x17,0x00,0x00,0x00,0x5c,0x00,0x00,0x00, +0x5b,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x2b,0x00,0x00,0x00, +0x66,0x00,0x00,0x00,0x2a,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x67,0x00,0x00,0x00, +0x66,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x69,0x00,0x00,0x00,0x67,0x00,0x00,0x00,0x49,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x6b,0x00,0x00,0x00, +0x69,0x00,0x00,0x00,0x44,0x00,0x00,0x00,0x41,0x00,0x06,0x00, +0x6d,0x00,0x00,0x00,0x6e,0x00,0x00,0x00,0x65,0x00,0x00,0x00, +0x21,0x00,0x00,0x00,0x6b,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x17,0x00,0x00,0x00,0x6f,0x00,0x00,0x00,0x6e,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x17,0x00,0x00,0x00,0x72,0x00,0x00,0x00, +0x1f,0x00,0x00,0x00,0x0c,0x00,0x08,0x00,0x17,0x00,0x00,0x00, +0x73,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00, +0x57,0x00,0x00,0x00,0x6f,0x00,0x00,0x00,0x72,0x00,0x00,0x00, +0x3e,0x00,0x03,0x00,0x1f,0x00,0x00,0x00,0x73,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x7e,0x00,0x00,0x00, +0x6b,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x41,0x00,0x06,0x00, +0x6d,0x00,0x00,0x00,0x7f,0x00,0x00,0x00,0x65,0x00,0x00,0x00, +0x21,0x00,0x00,0x00,0x7e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x17,0x00,0x00,0x00,0x80,0x00,0x00,0x00,0x7f,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x17,0x00,0x00,0x00,0x83,0x00,0x00,0x00, +0x1f,0x00,0x00,0x00,0x0c,0x00,0x08,0x00,0x17,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00, +0x5c,0x00,0x00,0x00,0x80,0x00,0x00,0x00,0x83,0x00,0x00,0x00, +0x3e,0x00,0x03,0x00,0x1f,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x87,0x00,0x00,0x00, +0xb5,0x00,0x00,0x00,0x35,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x22,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x24,0x00,0x00,0x00, +0xe0,0x00,0x04,0x00,0x88,0x00,0x00,0x00,0x88,0x00,0x00,0x00, +0x89,0x00,0x00,0x00,0xaa,0x00,0x05,0x00,0x30,0x00,0x00,0x00, +0xa4,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x21,0x00,0x00,0x00, +0xf7,0x00,0x03,0x00,0xa6,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0xa4,0x00,0x00,0x00,0xa5,0x00,0x00,0x00, +0xa6,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xa5,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x2b,0x00,0x00,0x00,0xab,0x00,0x00,0x00, +0x2a,0x00,0x00,0x00,0x35,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xac,0x00,0x00,0x00,0xab,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xae,0x00,0x00,0x00, +0xac,0x00,0x00,0x00,0x11,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x1e,0x00,0x00,0x00,0xaf,0x00,0x00,0x00,0x1b,0x00,0x00,0x00, +0x21,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x17,0x00,0x00,0x00, +0xb0,0x00,0x00,0x00,0xaf,0x00,0x00,0x00,0x41,0x00,0x06,0x00, +0x6d,0x00,0x00,0x00,0xb1,0x00,0x00,0x00,0xaa,0x00,0x00,0x00, +0x21,0x00,0x00,0x00,0xae,0x00,0x00,0x00,0x3e,0x00,0x03,0x00, +0xb1,0x00,0x00,0x00,0xb0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xa6,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xa6,0x00,0x00,0x00, +0xfd,0x00,0x01,0x00,0x38,0x00,0x01,0x00, }; -const uint64_t mul_mat_vec_f16_f32_len = 2788; +const uint64_t mul_mat_vec_f16_f32_len = 2372; unsigned char mul_mat_vec_nc_f16_f32_data[] = { 0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, diff --git a/ggml-vulkan.cpp b/ggml-vulkan.cpp index 1d93ec6bb..bccc40bf5 100644 --- a/ggml-vulkan.cpp +++ b/ggml-vulkan.cpp @@ -817,7 +817,7 @@ static void ggml_vk_load_shaders() { // mulmat std::initializer_list warptile_l = { 128, 128, 128, 16, vk_device.subgroup_size * 2, 64, 2, 4, 4, vk_device.subgroup_size }; std::initializer_list warptile_m = { 128, 64, 64, 16, vk_device.subgroup_size, 32, 2, 4, 2, vk_device.subgroup_size }; - std::initializer_list warptile_s = { vk_device.subgroup_size, 32, 32, 8, 32, 32, 2, 2, 2, vk_device.subgroup_size }; + std::initializer_list warptile_s = { vk_device.subgroup_size, 32, 32, 16, 32, 32, 2, 2, 2, vk_device.subgroup_size }; std::array l_wg_denoms = {128, 128, 1 }; std::array m_wg_denoms = { 64, 64, 1 }; @@ -2873,7 +2873,8 @@ static void ggml_vk_op_f32(vk_context * ctx, const ggml_tensor * src0, const ggm if (op == GGML_OP_CPY) { GGML_ASSERT(!transfer_src0); GGML_ASSERT(!transfer_src1); - d_sz = dst->ne[1] * dst->nb[1]; + x_sz = ggml_nbytes(src0); + d_sz = ggml_nbytes(dst); if (extra->offset + d_sz >= d_D->size) { d_sz = VK_WHOLE_SIZE; @@ -4556,8 +4557,15 @@ GGML_CALL static bool ggml_backend_vk_graph_compute(ggml_backend_t backend, ggml } ggml_vk_preallocate_buffers(); + int last_node = cgraph->n_nodes - 1; + + // If the last op in the cgraph isn't backend GPU, the command buffer doesn't get closed properly + while (last_node > 0 && cgraph->nodes[last_node]->backend != GGML_BACKEND_GPU) { + last_node -= 1; + } + for (int i = 0; i < cgraph->n_nodes; i++) { - ggml_vk_build_graph(cgraph->nodes[i], i == cgraph->n_nodes - 1); + ggml_vk_build_graph(cgraph->nodes[i], i == last_node); } ggml_compute_params params = {}; diff --git a/ggml_vk_generate_shaders.py b/ggml_vk_generate_shaders.py index d0861fde4..6b1b82bf3 100644 --- a/ggml_vk_generate_shaders.py +++ b/ggml_vk_generate_shaders.py @@ -19,8 +19,8 @@ shader_int8_ext = """ # Type-specific defines shader_f16_defines = """ -#define QUANT_K 32 -#define QUANT_R 2 +#define QUANT_K 1 +#define QUANT_R 1 #define A_TYPE float16_t """ From dabcc5b471348e4ae03ddacc41e19ad75fb2f041 Mon Sep 17 00:00:00 2001 From: slaren Date: Wed, 31 Jan 2024 13:43:03 +0100 Subject: [PATCH 227/812] ggml : limit n_threads to the max n_tasks (#5238) --- ggml.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/ggml.c b/ggml.c index b2c8baaa8..afd9c6c61 100644 --- a/ggml.c +++ b/ggml.c @@ -16985,12 +16985,16 @@ struct ggml_cplan ggml_graph_plan(const struct ggml_cgraph * cgraph, int n_threa struct ggml_cplan cplan; memset(&cplan, 0, sizeof(struct ggml_cplan)); + int max_tasks = 1; + // thread scheduling for the different operations + work buffer size estimation for (int i = 0; i < cgraph->n_nodes; i++) { struct ggml_tensor * node = cgraph->nodes[i]; const int n_tasks = ggml_get_n_tasks(node, n_threads); + max_tasks = MAX(max_tasks, n_tasks); + size_t cur = 0; switch (node->op) { @@ -17157,7 +17161,7 @@ struct ggml_cplan ggml_graph_plan(const struct ggml_cgraph * cgraph, int n_threa work_size += CACHE_LINE_SIZE*(n_threads - 1); } - cplan.n_threads = n_threads; + cplan.n_threads = MIN(max_tasks, n_threads); cplan.work_size = work_size; cplan.work_data = NULL; From b2b9f025e7821e78bd501d75d01838c26de07a57 Mon Sep 17 00:00:00 2001 From: Neo Zhang Jianyu Date: Wed, 31 Jan 2024 21:04:46 +0800 Subject: [PATCH 228/812] format license text, restore apache license by legal suggestion (#5233) --- examples/sycl/ls-sycl-device.cpp | 10 ++++++---- ggml-sycl.cpp | 15 +++++++++++---- ggml-sycl.h | 9 +++++---- 3 files changed, 22 insertions(+), 12 deletions(-) diff --git a/examples/sycl/ls-sycl-device.cpp b/examples/sycl/ls-sycl-device.cpp index 42847154a..52442e4ca 100644 --- a/examples/sycl/ls-sycl-device.cpp +++ b/examples/sycl/ls-sycl-device.cpp @@ -1,7 +1,9 @@ -/*MIT license - Copyright (C) 2024 Intel Corporation - SPDX-License-Identifier: MIT -*/ +// +// MIT license +// Copyright (C) 2024 Intel Corporation +// SPDX-License-Identifier: MIT +// + #include "ggml-sycl.h" diff --git a/ggml-sycl.cpp b/ggml-sycl.cpp index 3fc346975..1cc55ef52 100644 --- a/ggml-sycl.cpp +++ b/ggml-sycl.cpp @@ -1,7 +1,14 @@ -/*MIT license - Copyright (C) 2024 Intel Corporation - SPDX-License-Identifier: MIT -*/ +// +// MIT license +// Copyright (C) 2024 Intel Corporation +// SPDX-License-Identifier: MIT +// + +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// #include #include diff --git a/ggml-sycl.h b/ggml-sycl.h index 0eabb53cc..ba0c61473 100644 --- a/ggml-sycl.h +++ b/ggml-sycl.h @@ -1,7 +1,8 @@ -/*MIT license - Copyright (C) 2024 Intel Corporation - SPDX-License-Identifier: MIT -*/ +// +// MIT license +// Copyright (C) 2024 Intel Corporation +// SPDX-License-Identifier: MIT +// #pragma once From 15606309a05ccf7fadbaad5538cb7c32acb1e06b Mon Sep 17 00:00:00 2001 From: JidongZhang-THU <1119708529@qq.com> Date: Wed, 31 Jan 2024 21:10:15 +0800 Subject: [PATCH 229/812] llava : add MobileVLM support (#5132) * New Feature: 1. Sum_Rows: fix cuda kernel overflow fix block shape error when nrows too big 2. Im2Col: Support Batch in cuda Support f32 to f32 both in cpu && cuda 3. DepthWiseConv: Support by Im2Col && MulMat 4. Pool_2d: Supoort avg pooling in cuda 5. HardSigmoid: Imp in cuda 6. HardSwish: Imp in cuda * fix tabs instead of spaces * code clean * CUDA POOL2D * ADD POOL2D test case in test-backend-ops.cpp * code clean * fix pool2d_kernel nits * fix bug in pool2d kernel * fix avg pooling, count_include_pad nits * test-backend-ops : add more pool_2d tests * cuda : fix warnings and formatting * ggml : check types in release builds too in pool_2d * test-backend-ops : remove f16 pool_2d tests * cuda : more style fixes * Add assert in ggml_cuda_op_pool2d * pool2d float padding fallback * test-backend-ops : add dst_type to im2col --------- Co-authored-by: slaren --- examples/llava/MobileVLM-README.md | 58 +++++++- ggml-cuda.cu | 209 ++++++++++++++++++++++++++--- ggml.c | 118 +++++++++++++--- ggml.h | 3 +- tests/test-backend-ops.cpp | 74 +++++++++- 5 files changed, 421 insertions(+), 41 deletions(-) diff --git a/examples/llava/MobileVLM-README.md b/examples/llava/MobileVLM-README.md index c6258eba6..9eba791da 100644 --- a/examples/llava/MobileVLM-README.md +++ b/examples/llava/MobileVLM-README.md @@ -111,17 +111,71 @@ llama_print_timings: eval time = 1279.03 ms / 18 runs ( 71.06 m llama_print_timings: total time = 34570.79 ms ``` +## Orin compile and run +### compile +```sh +make LLAMA_CUBLAS=1 CUDA_DOCKER_ARCH=sm_87 LLAMA_CUDA_F16=1 -j 32 +``` + +### run on Orin +### case 1 +**input** +```sh +./llava-cli \ + -m /data/local/tmp/ggml-model-q4_k.gguf \ + --mmproj /data/local/tmp/mmproj-model-f16.gguf \ + --image /data/local/tmp/demo.jpeg \ + -p "A chat between a curious user and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the user's questions. USER: \nWho is the author of this book? \nAnswer the question using a single word or phrase. ASSISTANT:" \ + --n-gpu-layers 999 +``` +**output** +```sh + +encode_image_with_clip: image encoded in 296.62 ms by CLIP ( 2.06 ms per image patch) + + Susan Wise Bauer + +llama_print_timings: load time = 1067.64 ms +llama_print_timings: sample time = 1.53 ms / 6 runs ( 0.25 ms per token, 3934.43 tokens per second) +llama_print_timings: prompt eval time = 306.84 ms / 246 tokens ( 1.25 ms per token, 801.72 tokens per second) +llama_print_timings: eval time = 91.50 ms / 6 runs ( 15.25 ms per token, 65.58 tokens per second) +llama_print_timings: total time = 1352.63 ms / 252 tokens +``` + +### case 2 +**input** +```sh +./llava-cli \ + -m /data/local/tmp/ggml-model-q4_k.gguf \ + --mmproj /data/local/tmp/mmproj-model-f16.gguf \ + -p "A chat between a curious user and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the user's questions. USER: \nWhat is in the image? ASSISTANT:" \ + --n-gpu-layers 999 + +``` +**output** +```sh +encode_image_with_clip: image encoded in 302.15 ms by CLIP ( 2.10 ms per image patch) + + The image features a cat lying in the grass. + +llama_print_timings: load time = 1057.07 ms +llama_print_timings: sample time = 3.27 ms / 11 runs ( 0.30 ms per token, 3360.83 tokens per second) +llama_print_timings: prompt eval time = 213.60 ms / 232 tokens ( 0.92 ms per token, 1086.14 tokens per second) +llama_print_timings: eval time = 166.65 ms / 11 runs ( 15.15 ms per token, 66.01 tokens per second) +llama_print_timings: total time = 1365.47 ms / 243 tokens +``` + ## Minor shortcomings The `n_patch` of output in `ldp` is 1/4 of the input. In order to implement quickly, we uniformly modified `clip_n_patches` function to a quarter. when counting the time consumption, the calculated time will be 4 times bigger than the real cost. ## TODO -- [ ] Support non-CPU backend for the new operators, such as `depthwise`, `hardswish`, `hardsigmoid` +- [x] Support non-CPU backend for the new operators, such as `depthwise`, `hardswish`, `hardsigmoid` - [ ] Optimize LDP projector performance - Optimize the structure definition to avoid unnecessary memory rearrangements, to reduce the use of `ggml_permute_cpy`; - Optimize operator implementation (ARM CPU/NVIDIA GPU): such as depthwise conv, hardswish, hardsigmoid, etc. -- [ ] run MobileVLM on `Jetson Orin` +- [x] run MobileVLM on `Jetson Orin` - [ ] Support more model variants, such as `MobileVLM-3B`. diff --git a/ggml-cuda.cu b/ggml-cuda.cu index 949bc8a1c..e56595742 100644 --- a/ggml-cuda.cu +++ b/ggml-cuda.cu @@ -524,6 +524,8 @@ static_assert(sizeof(block_iq3_xxs) == sizeof(ggml_fp16_t) + 3*(QK_K/8), "wrong #define CUDA_SILU_BLOCK_SIZE 256 #define CUDA_TANH_BLOCK_SIZE 256 #define CUDA_RELU_BLOCK_SIZE 256 +#define CUDA_HARDSIGMOID_BLOCK_SIZE 256 +#define CUDA_HARDSWISH_BLOCK_SIZE 256 #define CUDA_SQR_BLOCK_SIZE 256 #define CUDA_CPY_BLOCK_SIZE 32 #define CUDA_SCALE_BLOCK_SIZE 256 @@ -540,6 +542,7 @@ static_assert(sizeof(block_iq3_xxs) == sizeof(ggml_fp16_t) + 3*(QK_K/8), "wrong #define CUDA_PAD_BLOCK_SIZE 256 #define CUDA_ACC_BLOCK_SIZE 256 #define CUDA_IM2COL_BLOCK_SIZE 256 +#define CUDA_POOL2D_BLOCK_SIZE 256 #define CUDA_Q8_0_NE_ALIGN 2048 @@ -823,6 +826,24 @@ static __global__ void relu_f32(const float * x, float * dst, const int k) { dst[i] = fmaxf(x[i], 0); } +static __global__ void hardsigmoid_f32(const float * x, float * dst, const int k) { + const int i = blockDim.x*blockIdx.x + threadIdx.x; + + if (i >= k) { + return; + } + dst[i] = fminf(1.0f, fmaxf(0.0f, (x[i] + 3.0f) / 6.0f)); +} + +static __global__ void hardswish_f32(const float * x, float * dst, const int k) { + const int i = blockDim.x*blockIdx.x + threadIdx.x; + + if (i >= k) { + return; + } + dst[i] = x[i] * fminf(1.0f, fmaxf(0.0f, (x[i] + 3.0f) / 6.0f)); +} + static __global__ void leaky_relu_f32(const float * x, float * dst, const int k, const float negative_slope) { const int i = blockDim.x*blockIdx.x + threadIdx.x; if (i >= k) { @@ -5823,7 +5844,7 @@ static __global__ void alibi_f32(const float * x, float * dst, const int ncols, } static __global__ void k_sum_rows_f32(const float * x, float * dst, const int ncols) { - const int row = blockIdx.y; + const int row = blockIdx.x; const int col = threadIdx.x; float sum = 0.0f; @@ -6145,9 +6166,10 @@ static __global__ void clamp_f32(const float * x, float * dst, const float min, dst[i] = x[i] < min ? min : (x[i] > max ? max : x[i]); } -static __global__ void im2col_f32_f16( - const float * x, half * dst, - int offset_delta, int IW, int IH, int OW, int KW, int KH, int pelements, int CHW, +template +static __global__ void im2col_kernel( + const float * x, T * dst, int batch_offset, + int offset_delta, int IC, int IW, int IH, int OH, int OW, int KW, int KH, int pelements, int CHW, int s0, int s1, int p0, int p1, int d0, int d1) { const int i = threadIdx.x + blockIdx.x * blockDim.x; if (i >= pelements) { @@ -6160,21 +6182,73 @@ static __global__ void im2col_f32_f16( const int ky = (i - kd) / OW; const int ix = i % OW; + const int oh = blockIdx.y; + const int batch = blockIdx.z / IC; + const int ic = blockIdx.z % IC; + const int64_t iiw = ix * s0 + kx * d0 - p0; - const int64_t iih = blockIdx.y * s1 + ky * d1 - p1; + const int64_t iih = oh * s1 + ky * d1 - p1; const int64_t offset_dst = - (blockIdx.y * OW + ix) * CHW + - (blockIdx.z * (KW * KH) + ky * KW + kx); + ((batch * OH + oh) * OW + ix) * CHW + + (ic * (KW * KH) + ky * KW + kx); if (iih < 0 || iih >= IH || iiw < 0 || iiw >= IW) { - dst[offset_dst] = __float2half(0.0f); + dst[offset_dst] = 0.0f; } else { - const int64_t offset_src = blockIdx.z * offset_delta; - dst[offset_dst] = __float2half(x[offset_src + iih * IW + iiw]); + const int64_t offset_src = ic * offset_delta + batch * batch_offset; + dst[offset_dst] = x[offset_src + iih * IW + iiw]; } } +template +static __global__ void pool2d_nchw_kernel( + const int ih, const int iw, const int oh, const int ow, + const int kh, const int kw, const int sh, const int sw, + const int ph, const int pw, const int parallel_elements, + const Ti* src, To* dst, const enum ggml_op_pool op) { + int idx = threadIdx.x + blockIdx.x * blockDim.x; + if (idx >= parallel_elements) { + return; + } + + const int I_HW = ih * iw; + const int O_HW = oh * ow; + const int nc = idx / O_HW; + const int cur_oh = idx % O_HW / ow; + const int cur_ow = idx % O_HW % ow; + const Ti* i_ptr = src + nc * I_HW; + To* o_ptr = dst + nc * O_HW; + const int start_h = cur_oh * sh - ph; + const int bh = max(0, start_h); + const int eh = min(ih, start_h + kh); + const int start_w = cur_ow * sw - pw; + const int bw = max(0, start_w); + const int ew = min(iw, start_w + kw); + const To scale = 1. / (kh * kw); + To res = 0; + + switch (op) { + case GGML_OP_POOL_AVG: res = 0; break; + case GGML_OP_POOL_MAX: res = -FLT_MAX; break; + } + + for (int i = bh; i < eh; i += 1) { + for (int j = bw; j < ew; j += 1) { + #if __CUDA_ARCH__ >= 350 + Ti cur = __ldg(i_ptr + i * iw + j); + #else + Ti cur = i_ptr[i * iw + j]; + #endif + switch (op) { + case GGML_OP_POOL_AVG: res += cur * scale; break; + case GGML_OP_POOL_MAX: res = max(res, (To)cur); break; + } + } + } + o_ptr[cur_oh * ow + cur_ow] = res; +} + template static void get_rows_cuda(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, const void * src0_dd, const int32_t * src1_dd, float * dst_dd, cudaStream_t stream) { @@ -6388,6 +6462,16 @@ static void relu_f32_cuda(const float * x, float * dst, const int k, cudaStream_ relu_f32<<>>(x, dst, k); } +static void hardsigmoid_f32_cuda(const float * x, float * dst, const int k, cudaStream_t stream) { + const int num_blocks = (k + CUDA_HARDSIGMOID_BLOCK_SIZE - 1) / CUDA_HARDSIGMOID_BLOCK_SIZE; + hardsigmoid_f32<<>>(x, dst, k); +} + +static void hardswish_f32_cuda(const float * x, float * dst, const int k, cudaStream_t stream) { + const int num_blocks = (k + CUDA_HARDSWISH_BLOCK_SIZE - 1) / CUDA_HARDSWISH_BLOCK_SIZE; + hardswish_f32<<>>(x, dst, k); +} + static void leaky_relu_f32_cuda(const float * x, float * dst, const int k, const float negative_slope, cudaStream_t stream) { const int num_blocks = (k + CUDA_RELU_BLOCK_SIZE - 1) / CUDA_RELU_BLOCK_SIZE; leaky_relu_f32<<>>(x, dst, k, negative_slope); @@ -7475,7 +7559,7 @@ static void alibi_f32_cuda(const float * x, float * dst, const int ncols, const static void sum_rows_f32_cuda(const float * x, float * dst, const int ncols, const int nrows, cudaStream_t stream) { const dim3 block_dims(WARP_SIZE, 1, 1); - const dim3 block_nums(1, nrows, 1); + const dim3 block_nums(nrows, 1, 1); k_sum_rows_f32<<>>(x, dst, ncols); } @@ -7587,14 +7671,15 @@ static void soft_max_f32_cuda(const float * x, const float * y, float * dst, con } } -static void im2col_f32_f16_cuda(const float* x, half* dst, +template +static void im2col_cuda(const float* x, T* dst, int IW, int IH, int OW, int OH, int KW, int KH, int IC, - int offset_delta, + int batch, int batch_offset, int offset_delta, int s0,int s1,int p0,int p1,int d0,int d1, cudaStream_t stream) { const int parallel_elements = OW * KW * KH; const int num_blocks = (parallel_elements + CUDA_IM2COL_BLOCK_SIZE - 1) / CUDA_IM2COL_BLOCK_SIZE; - dim3 block_nums(num_blocks, OH, IC); - im2col_f32_f16<<>>(x, dst, offset_delta, IW, IH, OW, KW, KH, parallel_elements, (IC * KH * KW), s0, s1, p0, p1, d0, d1); + dim3 block_nums(num_blocks, OH, batch * IC); + im2col_kernel<<>>(x, dst, batch_offset, offset_delta, IC, IW, IH, OH, OW, KW, KH, parallel_elements, (IC * KH * KW), s0, s1, p0, p1, d0, d1); } // buffer pool for cuda @@ -8179,6 +8264,34 @@ static void ggml_cuda_op_relu( (void) src1_dd; } +static void ggml_cuda_op_hardsigmoid( + const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, + const float * src0_dd, const float * src1_dd, float * dst_dd, cudaStream_t main_stream) { + + GGML_ASSERT(src0->type == GGML_TYPE_F32); + GGML_ASSERT( dst->type == GGML_TYPE_F32); + + hardsigmoid_f32_cuda(src0_dd, dst_dd, ggml_nelements(src0), main_stream); + + (void) src1; + (void) dst; + (void) src1_dd; +} + +static void ggml_cuda_op_hardswish( + const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, + const float * src0_dd, const float * src1_dd, float * dst_dd, cudaStream_t main_stream) { + + GGML_ASSERT(src0->type == GGML_TYPE_F32); + GGML_ASSERT( dst->type == GGML_TYPE_F32); + + hardswish_f32_cuda(src0_dd, dst_dd, ggml_nelements(src0), main_stream); + + (void) src1; + (void) dst; + (void) src1_dd; +} + static void ggml_cuda_op_leaky_relu( const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, const float * src0_dd, const float * src1_dd, float * dst_dd, cudaStream_t main_stream) { @@ -8810,13 +8923,46 @@ static void ggml_cuda_op_alibi( (void) src1_dd; } +static void ggml_cuda_op_pool2d( + const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, + const float * src0_dd, const float * src1_dd, float * dst_dd, cudaStream_t main_stream) { + + GGML_ASSERT(src0->type == GGML_TYPE_F32); + GGML_ASSERT( dst->type == GGML_TYPE_F32); + + const int32_t * opts = (const int32_t *)dst->op_params; + enum ggml_op_pool op = static_cast(opts[0]); + const int k0 = opts[1]; + const int k1 = opts[2]; + const int s0 = opts[3]; + const int s1 = opts[4]; + const int p0 = opts[5]; + const int p1 = opts[6]; + + const int64_t IH = src0->ne[1]; + const int64_t IW = src0->ne[0]; + + const int64_t N = dst->ne[3]; + const int64_t OC = dst->ne[2]; + const int64_t OH = dst->ne[1]; + const int64_t OW = dst->ne[0]; + + const int parallel_elements = N * OC * OH * OW; + const int num_blocks = (parallel_elements + CUDA_POOL2D_BLOCK_SIZE - 1) / CUDA_POOL2D_BLOCK_SIZE; + dim3 block_nums(num_blocks); + pool2d_nchw_kernel<<>>(IH, IW, OH, OW, k1, k0, s1, s0, p1, p0, parallel_elements, src0_dd, dst_dd, op); + + (void) src1; + (void) src1_dd; +} + static void ggml_cuda_op_im2col( const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, const float * src0_dd, const float * src1_dd, float * dst_dd, cudaStream_t main_stream) { GGML_ASSERT(src0->type == GGML_TYPE_F16); GGML_ASSERT(src1->type == GGML_TYPE_F32); - GGML_ASSERT( dst->type == GGML_TYPE_F16); + GGML_ASSERT( dst->type == GGML_TYPE_F16 || dst->type == GGML_TYPE_F32); const int32_t s0 = ((const int32_t*)(dst->op_params))[0]; const int32_t s1 = ((const int32_t*)(dst->op_params))[1]; @@ -8838,8 +8984,14 @@ static void ggml_cuda_op_im2col( const int64_t OW = dst->ne[1]; const size_t delta_offset = src1->nb[is_2D ? 2 : 1] / 4; // nb is byte offset, src is type float32 + const int64_t batch = src1->ne[3]; + const size_t batch_offset = src1->nb[3] / 4; // nb is byte offset, src is type float32 - im2col_f32_f16_cuda(src1_dd, (half*) dst_dd, IW, IH, OW, OH, KW, KH, IC, delta_offset, s0, s1, p0, p1, d0, d1, main_stream); + if(dst->type == GGML_TYPE_F16) { + im2col_cuda(src1_dd, (half*) dst_dd, IW, IH, OW, OH, KW, KH, IC, batch, batch_offset, delta_offset, s0, s1, p0, p1, d0, d1, main_stream); + } else { + im2col_cuda(src1_dd, (float*) dst_dd, IW, IH, OW, OH, KW, KH, IC, batch, batch_offset, delta_offset, s0, s1, p0, p1, d0, d1, main_stream); + } (void) src0; (void) src0_dd; @@ -9435,6 +9587,13 @@ static void ggml_cuda_relu(const ggml_tensor * src0, const ggml_tensor * src1, g ggml_cuda_op_flatten(src0, src1, dst, ggml_cuda_op_relu); } +static void ggml_cuda_hardsigmoid(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + ggml_cuda_op_flatten(src0, src1, dst, ggml_cuda_op_hardsigmoid); +} + +static void ggml_cuda_hardswish(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + ggml_cuda_op_flatten(src0, src1, dst, ggml_cuda_op_hardswish); +} static void ggml_cuda_leaky_relu(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { ggml_cuda_op_flatten(src0, src1, dst, ggml_cuda_op_leaky_relu); } @@ -10220,6 +10379,10 @@ static void ggml_cuda_alibi(const ggml_tensor * src0, const ggml_tensor * src1, ggml_cuda_op_flatten(src0, src1, dst, ggml_cuda_op_alibi); } +static void ggml_cuda_pool2d(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + ggml_cuda_op_flatten(src0, src1, dst, ggml_cuda_op_pool2d); +} + static void ggml_cuda_im2col(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { ggml_cuda_op_flatten(src0, src1, dst, ggml_cuda_op_im2col); } @@ -10321,6 +10484,12 @@ GGML_CALL bool ggml_cuda_compute_forward(struct ggml_compute_params * params, st case GGML_UNARY_OP_RELU: func = ggml_cuda_relu; break; + case GGML_UNARY_OP_HARDSIGMOID: + func = ggml_cuda_hardsigmoid; + break; + case GGML_UNARY_OP_HARDSWISH: + func = ggml_cuda_hardswish; + break; default: return false; } @@ -10395,6 +10564,9 @@ GGML_CALL bool ggml_cuda_compute_forward(struct ggml_compute_params * params, st case GGML_OP_IM2COL: func = ggml_cuda_im2col; break; + case GGML_OP_POOL_2D: + func = ggml_cuda_pool2d; + break; case GGML_OP_SUM_ROWS: func = ggml_cuda_sum_rows; break; @@ -11123,6 +11295,8 @@ GGML_CALL static bool ggml_backend_cuda_supports_op(ggml_backend_t backend, cons case GGML_UNARY_OP_GELU: case GGML_UNARY_OP_SILU: case GGML_UNARY_OP_RELU: + case GGML_UNARY_OP_HARDSIGMOID: + case GGML_UNARY_OP_HARDSWISH: case GGML_UNARY_OP_GELU_QUICK: case GGML_UNARY_OP_TANH: return true; @@ -11221,6 +11395,7 @@ GGML_CALL static bool ggml_backend_cuda_supports_op(ggml_backend_t backend, cons case GGML_OP_ROPE: case GGML_OP_ALIBI: case GGML_OP_IM2COL: + case GGML_OP_POOL_2D: case GGML_OP_SUM_ROWS: case GGML_OP_ARGSORT: case GGML_OP_ACC: diff --git a/ggml.c b/ggml.c index afd9c6c61..ee994c875 100644 --- a/ggml.c +++ b/ggml.c @@ -5349,7 +5349,7 @@ GGML_API struct ggml_tensor * ggml_conv_1d( int s0, int p0, int d0) { - struct ggml_tensor * im2col = ggml_im2col(ctx, a, b, s0, 0, p0, 0, d0, 0, false); // [N, OL, IC * K] + struct ggml_tensor * im2col = ggml_im2col(ctx, a, b, s0, 0, p0, 0, d0, 0, false, GGML_TYPE_F16); // [N, OL, IC * K] struct ggml_tensor * result = ggml_mul_mat(ctx, @@ -5427,16 +5427,15 @@ struct ggml_tensor * ggml_conv_depthwise_2d( int p1, int d0, int d1) { + struct ggml_tensor * new_a = ggml_reshape_4d(ctx, a, a->ne[0], a->ne[1], 1, a->ne[2] * a->ne[3]); struct ggml_tensor * im2col = ggml_im2col(ctx, new_a, ggml_reshape_4d(ctx, b, b->ne[0], b->ne[1], 1, b->ne[2] * b->ne[3]), - s0, s1, p0, p1, d0, d1, true); // [N * IC, OH, OW, KH * KW] - - struct ggml_tensor * result = - ggml_mul_mat(ctx, - ggml_reshape_4d(ctx, new_a, (new_a->ne[0] * new_a->ne[1]), new_a->ne[2], new_a->ne[3], 1), // [OC,1, KH, KW] => [1, OC, 1, KH * KW] - ggml_reshape_4d(ctx, im2col, im2col->ne[0], im2col->ne[2] * im2col->ne[1], b->ne[2], b->ne[3])); // [N * IC, OH, OW, KH * KW] => [N, IC, OH * OW, KH * KW] + s0, s1, p0, p1, d0, d1, true, GGML_TYPE_F16); // [N * IC, OH, OW, KH * KW] + struct ggml_tensor * new_b = ggml_reshape_4d(ctx, im2col, im2col->ne[0], im2col->ne[2] * im2col->ne[1], b->ne[2], b->ne[3]); // [N * IC, OH, OW, KH * KW] => [N, IC, OH * OW, KH * KW] + new_a = ggml_reshape_4d(ctx, new_a, (new_a->ne[0] * new_a->ne[1]), new_a->ne[2], new_a->ne[3], 1); // [OC,1, KH, KW] => [1, OC, 1, KH * KW] + struct ggml_tensor * result = ggml_mul_mat(ctx, new_a, new_b); result = ggml_reshape_4d(ctx, result, im2col->ne[1], im2col->ne[2], b->ne[2], b->ne[3]); // [N, OC, OH, OW] return result; @@ -5457,7 +5456,8 @@ struct ggml_tensor * ggml_im2col( int p1, int d0, int d1, - bool is_2D) { + bool is_2D, + enum ggml_type dst_type) { if(is_2D) { GGML_ASSERT(a->ne[2] == b->ne[2]); @@ -5481,7 +5481,7 @@ struct ggml_tensor * ggml_im2col( is_2D ? b->ne[3] : 1, }; - struct ggml_tensor * result = ggml_new_tensor(ctx, GGML_TYPE_F16, 4, ne); + struct ggml_tensor * result = ggml_new_tensor(ctx, dst_type, 4, ne); int32_t params[] = { s0, s1, p0, p1, d0, d1, (is_2D ? 1 : 0) }; ggml_set_op_params(result, params, sizeof(params)); @@ -5506,7 +5506,7 @@ struct ggml_tensor * ggml_conv_2d( int p1, int d0, int d1) { - struct ggml_tensor * im2col = ggml_im2col(ctx, a, b, s0, s1, p0, p1, d0, d1, true); // [N, OH, OW, IC * KH * KW] + struct ggml_tensor * im2col = ggml_im2col(ctx, a, b, s0, s1, p0, p1, d0, d1, true, GGML_TYPE_F16); // [N, OH, OW, IC * KH * KW] struct ggml_tensor * result = ggml_mul_mat(ctx, @@ -5632,12 +5632,13 @@ struct ggml_tensor * ggml_pool_2d( is_node = true; } + struct ggml_tensor * result; const int64_t ne[3] = { ggml_calc_pool_output_size(a->ne[0], k0, s0, p0), ggml_calc_pool_output_size(a->ne[1], k1, s1, p1), a->ne[2], }; - struct ggml_tensor * result = ggml_new_tensor(ctx, GGML_TYPE_F32, 3, ne); + result = ggml_new_tensor(ctx, GGML_TYPE_F32, 3, ne); int32_t params[] = { op, k0, k1, s0, s1, p0, p1 }; ggml_set_op_params(result, params, sizeof(params)); @@ -5645,7 +5646,6 @@ struct ggml_tensor * ggml_pool_2d( result->op = GGML_OP_POOL_2D; result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL; result->src[0] = a; - return result; } @@ -12493,6 +12493,92 @@ static void ggml_compute_forward_conv_transpose_1d( } } +// src0: kernel [OC, IC, KH, KW] +// src1: image [N, IC, IH, IW] +// dst: result [N, OH, OW, IC*KH*KW] +static void ggml_compute_forward_im2col_f32( + const struct ggml_compute_params * params, + const struct ggml_tensor * src0, + const struct ggml_tensor * src1, + struct ggml_tensor * dst) { + GGML_ASSERT(src0->type == GGML_TYPE_F16); + GGML_ASSERT(src1->type == GGML_TYPE_F32); + GGML_ASSERT( dst->type == GGML_TYPE_F32); + + int64_t t0 = ggml_perf_time_us(); + UNUSED(t0); + + GGML_TENSOR_BINARY_OP_LOCALS; + + const int32_t s0 = ((const int32_t *)(dst->op_params))[0]; + const int32_t s1 = ((const int32_t *)(dst->op_params))[1]; + const int32_t p0 = ((const int32_t *)(dst->op_params))[2]; + const int32_t p1 = ((const int32_t *)(dst->op_params))[3]; + const int32_t d0 = ((const int32_t *)(dst->op_params))[4]; + const int32_t d1 = ((const int32_t *)(dst->op_params))[5]; + const bool is_2D = ((const int32_t *)(dst->op_params))[6] == 1; + + const int ith = params->ith; + const int nth = params->nth; + + const int64_t N = is_2D ? ne13 : ne12; + const int64_t IC = is_2D ? ne12 : ne11; + const int64_t IH = is_2D ? ne11 : 1; + const int64_t IW = ne10; + + const int64_t KH = is_2D ? ne01 : 1; + const int64_t KW = ne00; + + const int64_t OH = is_2D ? ne2 : 1; + const int64_t OW = ne1; + + int ofs0 = is_2D ? nb13 : nb12; + int ofs1 = is_2D ? nb12 : nb11; + + GGML_ASSERT(nb00 == sizeof(ggml_fp16_t)); + GGML_ASSERT(nb10 == sizeof(float)); + + if (params->type == GGML_TASK_INIT) { + return; + } + + if (params->type == GGML_TASK_FINALIZE) { + return; + } + + // im2col: [N, IC, IH, IW] => [N, OH, OW, IC*KH*KW] + { + float * const wdata = (float *) dst->data; + + for (int64_t in = 0; in < N; in++) { + for (int64_t ioh = 0; ioh < OH; ioh++) { // 1 + for (int64_t iow = 0; iow < OW; iow++) { + for (int64_t iic = ith; iic < IC; iic += nth) { + + // micro kernel + float * dst_data = wdata + (in*OH*OW + ioh*OW + iow)*(IC*KH*KW); // [IC, KH, KW] + const float * const src_data = (float *)((char *) src1->data + in*ofs0 + iic*ofs1); // [IH, IW] + + for (int64_t ikh = 0; ikh < KH; ikh++) { // 1 + for (int64_t ikw = 0; ikw < KW; ikw++) { + const int64_t iiw = iow*s0 + ikw*d0 - p0; + const int64_t iih = ioh*s1 + ikh*d1 - p1; + + if (iih < 0 || iih >= IH || iiw < 0 || iiw >= IW) { + dst_data[iic*(KH*KW) + ikh*KW + ikw] = 0; + } else { + dst_data[iic*(KH*KW) + ikh*KW + ikw] = (src_data[iih*IW + iiw]); + } + } + } + } + } + } + } + } +} + + // src0: kernel [OC, IC, KH, KW] // src1: image [N, IC, IH, IW] // dst: result [N, OH, OW, IC*KH*KW] @@ -12583,14 +12669,14 @@ static void ggml_compute_forward_im2col( const struct ggml_tensor * src0, const struct ggml_tensor * src1, struct ggml_tensor * dst) { - switch (src0->type) { + switch (dst->type) { case GGML_TYPE_F16: { ggml_compute_forward_im2col_f16(params, src0, src1, dst); } break; case GGML_TYPE_F32: { - GGML_ASSERT(false); + ggml_compute_forward_im2col_f32(params, src0, src1, dst); } break; default: { @@ -12781,8 +12867,8 @@ static void ggml_compute_forward_pool_2d( const struct ggml_compute_params * params, const struct ggml_tensor * src, struct ggml_tensor * dst) { - assert(src->type == GGML_TYPE_F32); - assert(params->ith == 0); + GGML_ASSERT(src->type == GGML_TYPE_F32); + GGML_ASSERT(params->ith == 0); if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) { return; diff --git a/ggml.h b/ggml.h index afc87b843..e0a4799f3 100644 --- a/ggml.h +++ b/ggml.h @@ -1495,7 +1495,8 @@ extern "C" { int p1, int d0, int d1, - bool is_2D); + bool is_2D, + enum ggml_type dst_type); GGML_API struct ggml_tensor * ggml_conv_depthwise_2d( struct ggml_context * ctx, diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index 1d29070b6..eb06123d2 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -227,6 +227,14 @@ static std::string var_to_str(ggml_type type) { return ggml_type_name(type); } +static std::string var_to_str(ggml_op_pool pool) { + switch (pool) { + case GGML_OP_POOL_AVG: return "avg"; + case GGML_OP_POOL_MAX: return "max"; + default: return std::to_string(pool); + } +} + #define VARS_TO_STR1(a) VAR_TO_STR(a) #define VARS_TO_STR2(a, b) VAR_TO_STR(a) + "," + VAR_TO_STR(b) #define VARS_TO_STR3(a, b, c) VAR_TO_STR(a) + "," + VARS_TO_STR2(b, c) @@ -238,6 +246,7 @@ static std::string var_to_str(ggml_type type) { #define VARS_TO_STR9(a, b, c, d, e, f, g, h, i) VAR_TO_STR(a) + "," + VARS_TO_STR8(b, c, d, e, f, g, h, i) #define VARS_TO_STR10(a, b, c, d, e, f, g, h, i, j) VAR_TO_STR(a) + "," + VARS_TO_STR9(b, c, d, e, f, g, h, i, j) #define VARS_TO_STR11(a, b, c, d, e, f, g, h, i, j, k) VAR_TO_STR(a) + "," + VARS_TO_STR10(b, c, d, e, f, g, h, i, j, k) +#define VARS_TO_STR12(a, b, c, d, e, f, g, h, i, j, k, l) VAR_TO_STR(a) + "," + VARS_TO_STR11(b, c, d, e, f, g, h, i, j, k, l) #ifdef GGML_USE_SYCL static bool inline _isinf(float f) { @@ -1162,10 +1171,45 @@ struct test_alibi : public test_case { } }; +// GGML_OP_POOL2D +struct test_pool2d : public test_case { + enum ggml_op_pool pool_type; + const ggml_type type_input; + const std::array ne_input; + // kernel size + const int k0; + const int k1; + // stride + const int s0; + const int s1; + // padding + const int p0; + const int p1; + + std::string vars() override { + return VARS_TO_STR9(pool_type, type_input, ne_input, k0, k1, s0, s1, p0, p1); + } + + test_pool2d(ggml_op_pool pool_type = GGML_OP_POOL_AVG, + ggml_type type_input = GGML_TYPE_F32, + std::array ne_input = {10, 10, 3, 1}, // [input_width, input_height, input_channels, 1] + int k0 = 3, int k1 = 3, + int s0 = 1, int s1 = 1, + int p0 = 1, int p1 = 1) + : pool_type(pool_type), type_input(type_input), ne_input(ne_input), k0(k0), k1(k1), s0(s0), s1(s1), p0(p0), p1(p1) {} + + ggml_tensor * build_graph(ggml_context * ctx) override { + ggml_tensor * input = ggml_new_tensor(ctx, type_input, 4, ne_input.data()); + ggml_tensor * out = ggml_pool_2d(ctx, input, pool_type, k0, k1, s0, s1, p0, p1); + return out; + } +}; + // GGML_OP_IM2COL struct test_im2col : public test_case { const ggml_type type_input; const ggml_type type_kernel; + const ggml_type dst_type; const std::array ne_input; const std::array ne_kernel; // stride @@ -1181,22 +1225,22 @@ struct test_im2col : public test_case { const bool is_2D; std::string vars() override { - return VARS_TO_STR11(type_input, type_kernel, ne_input, ne_kernel, s0, s1, p0, p1, d0, d1, is_2D); + return VARS_TO_STR12(type_input, type_kernel, dst_type, ne_input, ne_kernel, s0, s1, p0, p1, d0, d1, is_2D); } - test_im2col(ggml_type type_input = GGML_TYPE_F32, ggml_type type_kernel = GGML_TYPE_F16, + test_im2col(ggml_type type_input = GGML_TYPE_F32, ggml_type type_kernel = GGML_TYPE_F16, ggml_type dst_type = GGML_TYPE_F32, std::array ne_input = {10, 10, 3, 1}, // [input_width, input_height, input_channels, 1] std::array ne_kernel = {3, 3, 3, 1}, // [kernel_width, kernel_height, input_channels, 1] int s0 = 1, int s1 = 1, int p0 = 1, int p1 = 1, int d0 = 1, int d1 = 1, bool is_2D = true) - : type_input(type_input), type_kernel(type_kernel), ne_input(ne_input), ne_kernel(ne_kernel), s0(s0), s1(s1), p0(p0), p1(p1), d0(d0), d1(d1), is_2D(is_2D) {} + : type_input(type_input), type_kernel(type_kernel), dst_type(dst_type), ne_input(ne_input), ne_kernel(ne_kernel), s0(s0), s1(s1), p0(p0), p1(p1), d0(d0), d1(d1), is_2D(is_2D) {} ggml_tensor * build_graph(ggml_context * ctx) override { ggml_tensor * input = ggml_new_tensor(ctx, type_input, 4, ne_input.data()); ggml_tensor * kernel = ggml_new_tensor(ctx, type_kernel, 4, ne_kernel.data()); - ggml_tensor * out = ggml_im2col(ctx, kernel, input, s0, s1, p0, p1, d0, d1, is_2D); + ggml_tensor * out = ggml_im2col(ctx, kernel, input, s0, s1, p0, p1, d0, d1, is_2D, dst_type); return out; } }; @@ -1912,6 +1956,27 @@ static bool test_backend(ggml_backend_t backend, test_mode mode, const char * op } } + for (ggml_type type_input : {GGML_TYPE_F32}) { + for (ggml_op_pool pool_type : {GGML_OP_POOL_AVG, GGML_OP_POOL_MAX}) { + for (int k0 : {1, 3}) { + for (int k1 : {1, 3}) { + for (int s0 : {1, 2}) { + for (int s1 : {1, 2}) { + for (int p0 : {0, 1}) { + for (int p1 : {0, 1}) { + test_cases.emplace_back(new test_pool2d(pool_type, type_input, {10, 10, 3, 1}, k0, k1, s0, s1, p0, p1)); + } + } + } + } + } + } + } + } + + test_cases.emplace_back(new test_im2col(GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_F32)); + test_cases.emplace_back(new test_im2col(GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_F16)); + test_cases.emplace_back(new test_repeat(GGML_TYPE_F32, {10, 10, 10, 10}, {1, 1, 1, 1})); test_cases.emplace_back(new test_repeat(GGML_TYPE_F32, {10, 10, 10, 10}, {2, 1, 1, 1})); test_cases.emplace_back(new test_repeat(GGML_TYPE_F32, {10, 10, 10, 10}, {1, 2, 1, 1})); @@ -2049,7 +2114,6 @@ static bool test_backend(ggml_backend_t backend, test_mode mode, const char * op } test_cases.emplace_back(new test_alibi()); - test_cases.emplace_back(new test_im2col()); test_cases.emplace_back(new test_concat(GGML_TYPE_F32)); test_cases.emplace_back(new test_concat(GGML_TYPE_I32)); From efb7bdbbd061d087c788598b97992c653f992ddd Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Wed, 31 Jan 2024 15:35:41 +0200 Subject: [PATCH 230/812] metal : add im2col F32 dst support (#5132) --- ggml-metal.m | 13 ++++++++++--- ggml-metal.metal | 33 +++++++++++++++++++++++++++++---- 2 files changed, 39 insertions(+), 7 deletions(-) diff --git a/ggml-metal.m b/ggml-metal.m index f87859552..5260ed827 100644 --- a/ggml-metal.m +++ b/ggml-metal.m @@ -135,6 +135,7 @@ enum ggml_metal_kernel_type { GGML_METAL_KERNEL_TYPE_ROPE_F16, GGML_METAL_KERNEL_TYPE_ALIBI_F32, GGML_METAL_KERNEL_TYPE_IM2COL_F16, + GGML_METAL_KERNEL_TYPE_IM2COL_F32, GGML_METAL_KERNEL_TYPE_UPSCALE_F32, GGML_METAL_KERNEL_TYPE_PAD_F32, GGML_METAL_KERNEL_TYPE_ARGSORT_F32_I32_ASC, @@ -506,6 +507,7 @@ static struct ggml_metal_context * ggml_metal_init(int n_cb) { GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_ROPE_F16, rope_f16, true); GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_ALIBI_F32, alibi_f32, true); GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_IM2COL_F16, im2col_f16, true); + GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_IM2COL_F32, im2col_f32, true); GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_UPSCALE_F32, upscale_f32, true); GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_PAD_F32, pad_f32, true); GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_ARGSORT_F32_I32_ASC, argsort_f32_i32_asc, true); @@ -630,6 +632,10 @@ static bool ggml_metal_supports_op(const struct ggml_metal_context * ctx, const case GGML_OP_ALIBI: case GGML_OP_ROPE: case GGML_OP_IM2COL: + return true; + case GGML_OP_POOL_1D: + case GGML_OP_POOL_2D: + return false; case GGML_OP_UPSCALE: case GGML_OP_PAD: case GGML_OP_ARGSORT: @@ -2015,7 +2021,7 @@ static bool ggml_metal_graph_compute( { GGML_ASSERT(src0->type == GGML_TYPE_F16); GGML_ASSERT(src1->type == GGML_TYPE_F32); - GGML_ASSERT( dst->type == GGML_TYPE_F16); + GGML_ASSERT( dst->type == GGML_TYPE_F16 || dst->type == GGML_TYPE_F32); const int32_t s0 = ((const int32_t *)(dst->op_params))[0]; const int32_t s1 = ((const int32_t *)(dst->op_params))[1]; @@ -2023,6 +2029,7 @@ static bool ggml_metal_graph_compute( const int32_t p1 = ((const int32_t *)(dst->op_params))[3]; const int32_t d0 = ((const int32_t *)(dst->op_params))[4]; const int32_t d1 = ((const int32_t *)(dst->op_params))[5]; + const bool is_2D = ((const int32_t *)(dst->op_params))[6] == 1; const int32_t N = src1->ne[is_2D ? 3 : 2]; @@ -2043,8 +2050,8 @@ static bool ggml_metal_graph_compute( id pipeline = nil; - switch (src0->type) { - case GGML_TYPE_F32: GGML_ASSERT(false && "not implemented"); break; + switch (dst->type) { + case GGML_TYPE_F32: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_IM2COL_F32].pipeline; break; case GGML_TYPE_F16: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_IM2COL_F16].pipeline; break; default: GGML_ASSERT(false); }; diff --git a/ggml-metal.metal b/ggml-metal.metal index 2614d82e8..efed6ad46 100644 --- a/ggml-metal.metal +++ b/ggml-metal.metal @@ -1775,9 +1775,29 @@ kernel void kernel_rope( template [[host_name("kernel_rope_f32")]] kernel rope_t kernel_rope; template [[host_name("kernel_rope_f16")]] kernel rope_t kernel_rope; -kernel void kernel_im2col_f16( +typedef void (im2col_t)( device const float * x, - device half * dst, + device char * dst, + constant int32_t & ofs0, + constant int32_t & ofs1, + constant int32_t & IW, + constant int32_t & IH, + constant int32_t & CHW, + constant int32_t & s0, + constant int32_t & s1, + constant int32_t & p0, + constant int32_t & p1, + constant int32_t & d0, + constant int32_t & d1, + uint3 tgpig[[threadgroup_position_in_grid]], + uint3 tgpg[[threadgroups_per_grid]], + uint3 tpitg[[thread_position_in_threadgroup]], + uint3 ntg[[threads_per_threadgroup]]); + +template +kernel void kernel_im2col( + device const float * x, + device char * dst, constant int32_t & ofs0, constant int32_t & ofs1, constant int32_t & IW, @@ -1800,14 +1820,19 @@ kernel void kernel_im2col_f16( (tpitg[0] * tgpg[1] * tgpg[2] + tgpig[1] * tgpg[2] + tgpig[2]) * CHW + (tgpig[0] * (ntg[1] * ntg[2]) + tpitg[1] * ntg[2] + tpitg[2]); + device T * pdst = (device T *) (dst); + if (iih < 0 || iih >= IH || iiw < 0 || iiw >= IW) { - dst[offset_dst] = 0.0f; + pdst[offset_dst] = 0.0f; } else { const int32_t offset_src = tpitg[0] * ofs0 + tgpig[0] * ofs1; - dst[offset_dst] = x[offset_src + iih * IW + iiw]; + pdst[offset_dst] = x[offset_src + iih * IW + iiw]; } } +template [[host_name("kernel_im2col_f32")]] kernel im2col_t kernel_im2col; +template [[host_name("kernel_im2col_f16")]] kernel im2col_t kernel_im2col; + kernel void kernel_upscale_f32( device const char * src0, device char * dst, From 5cb04dbc16d1da38c8fdcc0111b40e67d00dd1c3 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Wed, 31 Jan 2024 17:30:17 +0200 Subject: [PATCH 231/812] llama : remove LLAMA_MAX_DEVICES and LLAMA_SUPPORTS_GPU_OFFLOAD (#5240) * llama : remove LLAMA_MAX_DEVICES from llama.h ggml-ci * Update llama.cpp Co-authored-by: slaren * server : remove LLAMA_MAX_DEVICES ggml-ci * llama : remove LLAMA_SUPPORTS_GPU_OFFLOAD ggml-ci * train : remove LLAMA_SUPPORTS_GPU_OFFLOAD * readme : add deprecation notice * readme : change deprecation notice to "remove" and fix url * llama : remove gpu includes from llama.h ggml-ci --------- Co-authored-by: slaren --- README.md | 3 +- common/common.cpp | 56 ++++++++++---------- common/common.h | 66 ++++++++++++------------ common/train.cpp | 12 ++--- examples/batched-bench/batched-bench.cpp | 2 +- examples/llama-bench/llama-bench.cpp | 16 +++--- examples/server/server.cpp | 44 ++++++++-------- llama.cpp | 39 +++++++++++--- llama.h | 29 ++++------- 9 files changed, 143 insertions(+), 124 deletions(-) diff --git a/README.md b/README.md index 7746cb510..e6ed1d429 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,8 @@ Inference of [LLaMA](https://arxiv.org/abs/2302.13971) model in pure C/C++ ### Hot topics -- ⚠️ Incoming backends: https://github.com/ggerganov/llama.cpp/discussions/5138 +- Remove LLAMA_MAX_DEVICES and LLAMA_SUPPORTS_GPU_OFFLOAD: https://github.com/ggerganov/llama.cpp/pull/5240 +- Incoming backends: https://github.com/ggerganov/llama.cpp/discussions/5138 - [SYCL backend](README-sycl.md) is ready (1/28/2024), support Linux/Windows in Intel GPUs (iGPU, Arc/Flex/Max series) - New SOTA quantized models, including pure 2-bits: https://huggingface.co/ikawrakow - Collecting Apple Silicon performance stats: diff --git a/common/common.cpp b/common/common.cpp index 9d976c7c8..ce739b15c 100644 --- a/common/common.cpp +++ b/common/common.cpp @@ -583,20 +583,20 @@ bool gpt_params_parse_ex(int argc, char ** argv, gpt_params & params) { break; } params.n_gpu_layers = std::stoi(argv[i]); -#ifndef LLAMA_SUPPORTS_GPU_OFFLOAD - fprintf(stderr, "warning: not compiled with GPU offload support, --n-gpu-layers option will be ignored\n"); - fprintf(stderr, "warning: see main README.md for information on enabling GPU BLAS support\n"); -#endif + if (!llama_supports_gpu_offload()) { + fprintf(stderr, "warning: not compiled with GPU offload support, --n-gpu-layers option will be ignored\n"); + fprintf(stderr, "warning: see main README.md for information on enabling GPU BLAS support\n"); + } } else if (arg == "--gpu-layers-draft" || arg == "-ngld" || arg == "--n-gpu-layers-draft") { if (++i >= argc) { invalid_param = true; break; } params.n_gpu_layers_draft = std::stoi(argv[i]); -#ifndef LLAMA_SUPPORTS_GPU_OFFLOAD - fprintf(stderr, "warning: not compiled with GPU offload support, --n-gpu-layers-draft option will be ignored\n"); - fprintf(stderr, "warning: see main README.md for information on enabling GPU BLAS support\n"); -#endif + if (!llama_supports_gpu_offload()) { + fprintf(stderr, "warning: not compiled with GPU offload support, --n-gpu-layers-draft option will be ignored\n"); + fprintf(stderr, "warning: see main README.md for information on enabling GPU BLAS support\n"); + } } else if (arg == "--main-gpu" || arg == "-mg") { if (++i >= argc) { invalid_param = true; @@ -637,11 +637,11 @@ bool gpt_params_parse_ex(int argc, char ** argv, gpt_params & params) { const std::regex regex{R"([,/]+)"}; std::sregex_token_iterator it{arg_next.begin(), arg_next.end(), regex, -1}; std::vector split_arg{it, {}}; - if (split_arg.size() >= LLAMA_MAX_DEVICES) { + if (split_arg.size() >= llama_max_devices()) { invalid_param = true; break; } - for (size_t i = 0; i < LLAMA_MAX_DEVICES; ++i) { + for (size_t i = 0; i < llama_max_devices(); ++i) { if (i < split_arg.size()) { params.tensor_split[i] = std::stof(split_arg[i]); } else { @@ -989,30 +989,30 @@ void gpt_print_usage(int /*argc*/, char ** argv, const gpt_params & params) { printf(" -cb, --cont-batching enable continuous batching (a.k.a dynamic batching) (default: disabled)\n"); printf(" --mmproj MMPROJ_FILE path to a multimodal projector file for LLaVA. see examples/llava/README.md\n"); printf(" --image IMAGE_FILE path to an image file. use with multimodal models\n"); - if (llama_mlock_supported()) { + if (llama_supports_mlock()) { printf(" --mlock force system to keep model in RAM rather than swapping or compressing\n"); } - if (llama_mmap_supported()) { + if (llama_supports_mmap()) { printf(" --no-mmap do not memory-map model (slower load but may reduce pageouts if not using mlock)\n"); } printf(" --numa attempt optimizations that help on some NUMA systems\n"); printf(" if run without this previously, it is recommended to drop the system page cache before using this\n"); printf(" see https://github.com/ggerganov/llama.cpp/issues/1437\n"); -#ifdef LLAMA_SUPPORTS_GPU_OFFLOAD - printf(" -ngl N, --n-gpu-layers N\n"); - printf(" number of layers to store in VRAM\n"); - printf(" -ngld N, --n-gpu-layers-draft N\n"); - printf(" number of layers to store in VRAM for the draft model\n"); - printf(" -sm SPLIT_MODE, --split-mode SPLIT_MODE\n"); - printf(" how to split the model across multiple GPUs, one of:\n"); - printf(" - none: use one GPU only\n"); - printf(" - layer (default): split layers and KV across GPUs\n"); - printf(" - row: split rows across GPUs\n"); - printf(" -ts SPLIT, --tensor-split SPLIT\n"); - printf(" fraction of the model to offload to each GPU, comma-separated list of proportions, e.g. 3,1\n"); - printf(" -mg i, --main-gpu i the GPU to use for the model (with split-mode = none),\n"); - printf(" or for intermediate results and KV (with split-mode = row) (default: %d)\n", params.main_gpu); -#endif // LLAMA_SUPPORTS_GPU_OFFLOAD + if (llama_supports_gpu_offload()) { + printf(" -ngl N, --n-gpu-layers N\n"); + printf(" number of layers to store in VRAM\n"); + printf(" -ngld N, --n-gpu-layers-draft N\n"); + printf(" number of layers to store in VRAM for the draft model\n"); + printf(" -sm SPLIT_MODE, --split-mode SPLIT_MODE\n"); + printf(" how to split the model across multiple GPUs, one of:\n"); + printf(" - none: use one GPU only\n"); + printf(" - layer (default): split layers and KV across GPUs\n"); + printf(" - row: split rows across GPUs\n"); + printf(" -ts SPLIT, --tensor-split SPLIT\n"); + printf(" fraction of the model to offload to each GPU, comma-separated list of proportions, e.g. 3,1\n"); + printf(" -mg i, --main-gpu i the GPU to use for the model (with split-mode = none),\n"); + printf(" or for intermediate results and KV (with split-mode = row) (default: %d)\n", params.main_gpu); + } printf(" --verbose-prompt print a verbose prompt before generation (default: %s)\n", params.verbose_prompt ? "true" : "false"); printf(" --no-display-prompt don't print prompt at generation (default: %s)\n", !params.display_prompt ? "true" : "false"); printf(" -gan N, --grp-attn-n N\n"); @@ -1651,7 +1651,7 @@ void dump_non_result_info_yaml(FILE * stream, const gpt_params & params, const l fprintf(stream, "cont_batching: %s # default: false\n", params.cont_batching ? "true" : "false"); fprintf(stream, "temp: %f # default: 0.8\n", sparams.temp); - const std::vector tensor_split_vector(params.tensor_split, params.tensor_split + LLAMA_MAX_DEVICES); + const std::vector tensor_split_vector(params.tensor_split, params.tensor_split + llama_max_devices()); dump_vector_float_yaml(stream, "tensor_split", tensor_split_vector); fprintf(stream, "tfs: %f # default: 1.0\n", sparams.tfs_z); diff --git a/common/common.h b/common/common.h index 214a379b5..24a99d728 100644 --- a/common/common.h +++ b/common/common.h @@ -43,40 +43,40 @@ extern char const *LLAMA_BUILD_TARGET; int32_t get_num_physical_cores(); struct gpt_params { - uint32_t seed = -1; // RNG seed + uint32_t seed = -1; // RNG seed - int32_t n_threads = get_num_physical_cores(); - int32_t n_threads_draft = -1; - int32_t n_threads_batch = -1; // number of threads to use for batch processing (-1 = use n_threads) - int32_t n_threads_batch_draft = -1; - int32_t n_predict = -1; // new tokens to predict - int32_t n_ctx = 512; // context size - int32_t n_batch = 512; // batch size for prompt processing (must be >=32 to use BLAS) - int32_t n_keep = 0; // number of tokens to keep from initial prompt - int32_t n_draft = 8; // number of tokens to draft during speculative decoding - int32_t n_chunks = -1; // max number of chunks to process (-1 = unlimited) - int32_t n_parallel = 1; // number of parallel sequences to decode - int32_t n_sequences = 1; // number of sequences to decode - float p_accept = 0.5f; // speculative decoding accept probability - float p_split = 0.1f; // speculative decoding split probability - int32_t n_gpu_layers = -1; // number of layers to store in VRAM (-1 - use default) - int32_t n_gpu_layers_draft = -1; // number of layers to store in VRAM for the draft model (-1 - use default) - llama_split_mode split_mode = LLAMA_SPLIT_LAYER; // how to split the model across GPUs - int32_t main_gpu = 0; // the GPU that is used for scratch and small tensors - float tensor_split[LLAMA_MAX_DEVICES] = {0}; // how split tensors should be distributed across GPUs - int32_t n_beams = 0; // if non-zero then use beam search of given width. - int32_t grp_attn_n = 1; // group-attention factor - int32_t grp_attn_w = 512; // group-attention width - int32_t n_print = -1; // print token count every n tokens (-1 = disabled) - float rope_freq_base = 0.0f; // RoPE base frequency - float rope_freq_scale = 0.0f; // RoPE frequency scaling factor - float yarn_ext_factor = -1.0f; // YaRN extrapolation mix factor - float yarn_attn_factor = 1.0f; // YaRN magnitude scaling factor - float yarn_beta_fast = 32.0f; // YaRN low correction dim - float yarn_beta_slow = 1.0f; // YaRN high correction dim - int32_t yarn_orig_ctx = 0; // YaRN original context length - int8_t rope_scaling_type = LLAMA_ROPE_SCALING_UNSPECIFIED; // TODO: better to be int32_t for alignment - // pinging @cebtenzzre + int32_t n_threads = get_num_physical_cores(); + int32_t n_threads_draft = -1; + int32_t n_threads_batch = -1; // number of threads to use for batch processing (-1 = use n_threads) + int32_t n_threads_batch_draft = -1; + int32_t n_predict = -1; // new tokens to predict + int32_t n_ctx = 512; // context size + int32_t n_batch = 512; // batch size for prompt processing (must be >=32 to use BLAS) + int32_t n_keep = 0; // number of tokens to keep from initial prompt + int32_t n_draft = 8; // number of tokens to draft during speculative decoding + int32_t n_chunks = -1; // max number of chunks to process (-1 = unlimited) + int32_t n_parallel = 1; // number of parallel sequences to decode + int32_t n_sequences = 1; // number of sequences to decode + float p_accept = 0.5f; // speculative decoding accept probability + float p_split = 0.1f; // speculative decoding split probability + int32_t n_gpu_layers = -1; // number of layers to store in VRAM (-1 - use default) + int32_t n_gpu_layers_draft = -1; // number of layers to store in VRAM for the draft model (-1 - use default) + llama_split_mode split_mode = LLAMA_SPLIT_LAYER; // how to split the model across GPUs + int32_t main_gpu = 0; // the GPU that is used for scratch and small tensors + float tensor_split[128] = {0}; // how split tensors should be distributed across GPUs + int32_t n_beams = 0; // if non-zero then use beam search of given width. + int32_t grp_attn_n = 1; // group-attention factor + int32_t grp_attn_w = 512; // group-attention width + int32_t n_print = -1; // print token count every n tokens (-1 = disabled) + float rope_freq_base = 0.0f; // RoPE base frequency + float rope_freq_scale = 0.0f; // RoPE frequency scaling factor + float yarn_ext_factor = -1.0f; // YaRN extrapolation mix factor + float yarn_attn_factor = 1.0f; // YaRN magnitude scaling factor + float yarn_beta_fast = 32.0f; // YaRN low correction dim + float yarn_beta_slow = 1.0f; // YaRN high correction dim + int32_t yarn_orig_ctx = 0; // YaRN original context length + int8_t rope_scaling_type = LLAMA_ROPE_SCALING_UNSPECIFIED; // TODO: better to be int32_t for alignment + // pinging @cebtenzzre // // sampling parameters struct llama_sampling_params sparams; diff --git a/common/train.cpp b/common/train.cpp index e6f2f7a2f..e4c3d5df6 100644 --- a/common/train.cpp +++ b/common/train.cpp @@ -1363,12 +1363,12 @@ bool consume_common_train_arg( *invalid_param = true; return true; } -#ifdef LLAMA_SUPPORTS_GPU_OFFLOAD - params->n_gpu_layers = std::stoi(argv[i]); -#else - fprintf(stderr, "warning: not compiled with GPU offload support, --n-gpu-layers option will be ignored\n"); - fprintf(stderr, "warning: see main README.md for information on enabling GPU BLAS support\n"); -#endif + if (llama_supports_gpu_offload()) { + params->n_gpu_layers = std::stoi(argv[i]); + } else { + fprintf(stderr, "warning: not compiled with GPU offload support, --n-gpu-layers option will be ignored\n"); + fprintf(stderr, "warning: see main README.md for information on enabling GPU BLAS support\n"); + } } else if (arg == "-h" || arg == "--help") { params->print_usage = true; return true; diff --git a/examples/batched-bench/batched-bench.cpp b/examples/batched-bench/batched-bench.cpp index 7924db267..b52d68457 100644 --- a/examples/batched-bench/batched-bench.cpp +++ b/examples/batched-bench/batched-bench.cpp @@ -88,7 +88,7 @@ int main(int argc, char ** argv) { llama_model_params model_params = llama_model_default_params(); - const std::vector t_split (LLAMA_MAX_DEVICES, 0.0f); + const std::vector t_split(llama_max_devices(), 0.0f); model_params.n_gpu_layers = n_gpu_layers; model_params.tensor_split = t_split.data(); diff --git a/examples/llama-bench/llama-bench.cpp b/examples/llama-bench/llama-bench.cpp index 542cc7bb8..c5a6f744e 100644 --- a/examples/llama-bench/llama-bench.cpp +++ b/examples/llama-bench/llama-bench.cpp @@ -160,7 +160,7 @@ struct cmd_params { std::vector main_gpu; std::vector no_kv_offload; std::vector mul_mat_q; - std::vector> tensor_split; + std::vector> tensor_split; int reps; bool verbose; output_formats output_format; @@ -179,7 +179,7 @@ static const cmd_params cmd_params_defaults = { /* main_gpu */ {0}, /* no_kv_offload */ {false}, /* mul_mat_q */ {true}, - /* tensor_split */ {{}}, + /* tensor_split */ {std::vector(llama_max_devices(), 0.0f)}, /* reps */ 5, /* verbose */ false, /* output_format */ MARKDOWN @@ -380,10 +380,10 @@ static cmd_params parse_cmd_params(int argc, char ** argv) { const std::regex regex{R"([;/]+)"}; std::sregex_token_iterator it{ts.begin(), ts.end(), regex, -1}; std::vector split_arg{it, {}}; - GGML_ASSERT(split_arg.size() <= LLAMA_MAX_DEVICES); + GGML_ASSERT(split_arg.size() <= llama_max_devices()); - std::array tensor_split; - for (size_t i = 0; i < LLAMA_MAX_DEVICES; ++i) { + std::vector tensor_split(llama_max_devices()); + for (size_t i = 0; i < llama_max_devices(); ++i) { if (i < split_arg.size()) { tensor_split[i] = std::stof(split_arg[i]); } else { @@ -459,7 +459,7 @@ struct cmd_params_instance { int main_gpu; bool no_kv_offload; bool mul_mat_q; - std::array tensor_split; + std::vector tensor_split; llama_model_params to_llama_mparams() const { llama_model_params mparams = llama_model_default_params(); @@ -582,7 +582,7 @@ struct test { int main_gpu; bool no_kv_offload; bool mul_mat_q; - std::array tensor_split; + std::vector tensor_split; int n_prompt; int n_gen; std::string test_time; @@ -704,7 +704,7 @@ struct test { std::vector get_values() const { std::string tensor_split_str; int max_nonzero = 0; - for (int i = 0; i < LLAMA_MAX_DEVICES; i++) { + for (size_t i = 0; i < llama_max_devices(); i++) { if (tensor_split[i] > 0) { max_nonzero = i; } diff --git a/examples/server/server.cpp b/examples/server/server.cpp index 21bdce8ed..ea77125ea 100644 --- a/examples/server/server.cpp +++ b/examples/server/server.cpp @@ -1789,28 +1789,28 @@ static void server_print_usage(const char *argv0, const gpt_params ¶ms, printf(" -b N, --batch-size N batch size for prompt processing (default: %d)\n", params.n_batch); printf(" --memory-f32 use f32 instead of f16 for memory key+value (default: disabled)\n"); printf(" not recommended: doubles context memory required and no measurable increase in quality\n"); - if (llama_mlock_supported()) + if (llama_supports_mlock()) { printf(" --mlock force system to keep model in RAM rather than swapping or compressing\n"); } - if (llama_mmap_supported()) + if (llama_supports_mmap()) { printf(" --no-mmap do not memory-map model (slower load but may reduce pageouts if not using mlock)\n"); } printf(" --numa attempt optimizations that help on some NUMA systems\n"); -#ifdef LLAMA_SUPPORTS_GPU_OFFLOAD - printf(" -ngl N, --n-gpu-layers N\n"); - printf(" number of layers to store in VRAM\n"); - printf(" -sm SPLIT_MODE, --split-mode SPLIT_MODE\n"); - printf(" how to split the model across multiple GPUs, one of:\n"); - printf(" - none: use one GPU only\n"); - printf(" - layer (default): split layers and KV across GPUs\n"); - printf(" - row: split rows across GPUs\n"); - printf(" -ts SPLIT --tensor-split SPLIT\n"); - printf(" fraction of the model to offload to each GPU, comma-separated list of proportions, e.g. 3,1\n"); - printf(" -mg i, --main-gpu i the GPU to use for the model (with split-mode = none),\n"); - printf(" or for intermediate results and KV (with split-mode = row)\n"); -#endif + if (llama_supports_gpu_offload()) { + printf(" -ngl N, --n-gpu-layers N\n"); + printf(" number of layers to store in VRAM\n"); + printf(" -sm SPLIT_MODE, --split-mode SPLIT_MODE\n"); + printf(" how to split the model across multiple GPUs, one of:\n"); + printf(" - none: use one GPU only\n"); + printf(" - layer (default): split layers and KV across GPUs\n"); + printf(" - row: split rows across GPUs\n"); + printf(" -ts SPLIT --tensor-split SPLIT\n"); + printf(" fraction of the model to offload to each GPU, comma-separated list of proportions, e.g. 3,1\n"); + printf(" -mg i, --main-gpu i the GPU to use for the model (with split-mode = none),\n"); + printf(" or for intermediate results and KV (with split-mode = row)\n"); + } printf(" -m FNAME, --model FNAME\n"); printf(" model path (default: %s)\n", params.model.c_str()); printf(" -a ALIAS, --alias ALIAS\n"); @@ -2066,13 +2066,13 @@ static void server_params_parse(int argc, char **argv, server_params &sparams, invalid_param = true; break; } -#ifdef LLAMA_SUPPORTS_GPU_OFFLOAD - params.n_gpu_layers = std::stoi(argv[i]); -#else - LOG_WARNING("Not compiled with GPU offload support, --n-gpu-layers option will be ignored. " + if (llama_supports_gpu_offload()) { + params.n_gpu_layers = std::stoi(argv[i]); + } else { + LOG_WARNING("Not compiled with GPU offload support, --n-gpu-layers option will be ignored. " "See main README.md for information on enabling GPU BLAS support", {{"n_gpu_layers", params.n_gpu_layers}}); -#endif + } } else if (arg == "--split-mode" || arg == "-sm") { @@ -2115,9 +2115,9 @@ static void server_params_parse(int argc, char **argv, server_params &sparams, const std::regex regex{R"([,/]+)"}; std::sregex_token_iterator it{arg_next.begin(), arg_next.end(), regex, -1}; std::vector split_arg{it, {}}; - GGML_ASSERT(split_arg.size() <= LLAMA_MAX_DEVICES); + GGML_ASSERT(split_arg.size() <= llama_max_devices()); - for (size_t i_device = 0; i_device < LLAMA_MAX_DEVICES; ++i_device) + for (size_t i_device = 0; i_device < llama_max_devices(); ++i_device) { if (i_device < split_arg.size()) { diff --git a/llama.cpp b/llama.cpp index bb23689fa..9b249ba9c 100644 --- a/llama.cpp +++ b/llama.cpp @@ -10090,18 +10090,45 @@ struct llama_model_quantize_params llama_model_quantize_default_params() { return result; } -int32_t llama_max_devices(void) { - return LLAMA_MAX_DEVICES; +size_t llama_max_devices(void) { +#if defined(GGML_USE_METAL) + return 1; +#elif defined(GGML_USE_CUBLAS) + return GGML_CUDA_MAX_DEVICES; +#elif defined(GGML_USE_SYCL) + return GGML_SYCL_MAX_DEVICES; +#else + return 1; +#endif } -bool llama_mmap_supported(void) { +bool llama_supports_mmap(void) { return llama_mmap::SUPPORTED; } -bool llama_mlock_supported(void) { +bool llama_supports_mlock(void) { return llama_mlock::SUPPORTED; } +bool llama_supports_gpu_offload(void) { +#if defined(GGML_USE_CUBLAS) || defined(GGML_USE_CLBLAST) || defined(GGML_USE_METAL) || defined(GGML_USE_VULKAN) || \ + defined(GGML_USE_SYCL) || defined(GGML_USE_KOMPUTE) + // Defined when llama.cpp is compiled with support for offloading model layers to GPU. + return true; +#else + return false; +#endif +} + +// deprecated: +bool llama_mmap_supported(void) { + return llama_supports_mmap(); +} + +bool llama_mlock_supported(void) { + return llama_supports_mlock(); +} + void llama_backend_init(bool numa) { ggml_time_init(); @@ -10133,8 +10160,8 @@ int64_t llama_time_us(void) { } struct llama_model * llama_load_model_from_file( - const char * path_model, - struct llama_model_params params) { + const char * path_model, + struct llama_model_params params) { ggml_time_init(); llama_model * model = new llama_model; diff --git a/llama.h b/llama.h index 17d43d039..9a60e9bfb 100644 --- a/llama.h +++ b/llama.h @@ -3,15 +3,7 @@ #include "ggml.h" #include "ggml-backend.h" -#ifdef GGML_USE_CUBLAS -#include "ggml-cuda.h" -#define LLAMA_MAX_DEVICES GGML_CUDA_MAX_DEVICES -#elif defined(GGML_USE_SYCL) -#include "ggml-sycl.h" -#define LLAMA_MAX_DEVICES GGML_SYCL_MAX_DEVICES -#else -#define LLAMA_MAX_DEVICES 1 -#endif // GGML_USE_CUBLAS + #include #include #include @@ -49,12 +41,6 @@ #define LLAMA_SESSION_MAGIC LLAMA_FILE_MAGIC_GGSN #define LLAMA_SESSION_VERSION 4 -#if defined(GGML_USE_CUBLAS) || defined(GGML_USE_CLBLAST) || defined(GGML_USE_METAL) || defined(GGML_USE_VULKAN) || \ - defined(GGML_USE_SYCL) || defined(GGML_USE_KOMPUTE) -// Defined when llama.cpp is compiled with support for offloading model layers to GPU. -#define LLAMA_SUPPORTS_GPU_OFFLOAD -#endif - #ifdef __cplusplus extern "C" { #endif @@ -201,7 +187,7 @@ extern "C" { // LLAMA_SPLIT_LAYER: ignored int32_t main_gpu; - // proportion of the model (layers or rows) to offload to each GPU, size: LLAMA_MAX_DEVICES + // proportion of the model (layers or rows) to offload to each GPU, size: llama_max_devices() const float * tensor_split; // Called with a progress value between 0.0 and 1.0. Pass NULL to disable. @@ -338,9 +324,14 @@ extern "C" { LLAMA_API int64_t llama_time_us(void); - LLAMA_API int32_t llama_max_devices(void); - LLAMA_API bool llama_mmap_supported (void); - LLAMA_API bool llama_mlock_supported(void); + LLAMA_API size_t llama_max_devices(void); + + LLAMA_API bool llama_supports_mmap (void); + LLAMA_API bool llama_supports_mlock (void); + LLAMA_API bool llama_supports_gpu_offload(void); + + LLAMA_API DEPRECATED(bool llama_mmap_supported (void), "use llama_supports_mmap() instead"); + LLAMA_API DEPRECATED(bool llama_mlock_supported(void), "use llama_supports_mlock() instead"); LLAMA_API const struct llama_model * llama_get_model(const struct llama_context * ctx); From d3bac7d58408c602ec1f1e423695f1df8410bb03 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Wed, 31 Jan 2024 18:47:10 +0200 Subject: [PATCH 232/812] llama : reorder build_orion() at correct place (#5118) --- llama.cpp | 239 +++++++++++++++++++++++++++--------------------------- 1 file changed, 119 insertions(+), 120 deletions(-) diff --git a/llama.cpp b/llama.cpp index 9b249ba9c..02b0a485a 100644 --- a/llama.cpp +++ b/llama.cpp @@ -4666,126 +4666,6 @@ struct llm_build_context { ctx0 = nullptr; } } - struct ggml_cgraph * build_orion() { - struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, LLAMA_MAX_NODES, false); - - const int64_t n_embd_head = hparams.n_embd_head_v; - GGML_ASSERT(n_embd_head == hparams.n_embd_head_k); - GGML_ASSERT(n_embd_head == hparams.n_rot); - - struct ggml_tensor * cur; - struct ggml_tensor * inpL; - - inpL = llm_build_inp_embd(ctx0, hparams, batch, model.tok_embd, lctx.inp_tokens, lctx.inp_embd, cb); - cb(inpL, "inp_embd", -1); - - // inp_pos - contains the positions - struct ggml_tensor * inp_pos = ggml_view_1d(ctx0, lctx.inp_pos, n_tokens, 0); - cb(inp_pos, "inp_pos", -1); - - // KQ_mask (mask for 1 head, it will be broadcasted to all heads) - struct ggml_tensor * KQ_mask = ggml_view_2d(ctx0, lctx.inp_KQ_mask, n_kv, n_tokens, n_kv*ggml_type_size(lctx.inp_KQ_mask->type), 0); - cb(KQ_mask, "KQ_mask", -1); - - // shift the entire K-cache if needed - if (do_rope_shift) { - llm_build_k_shift(ctx0, hparams, cparams, kv_self, gf, lctx.inp_K_shift, LLM_ROPE, n_ctx, freq_base, freq_scale, cb); - } - - for (int il = 0; il < n_layer; ++il) { - struct ggml_tensor * inpSA = inpL; - - // norm - cur = llm_build_norm(ctx0, inpL, hparams, - model.layers[il].attn_norm, model.layers[il].attn_norm_b, - LLM_NORM, cb, il); - cb(cur, "attn_norm", il); - - // self-attention - { - // compute Q and K and RoPE them - struct ggml_tensor * Qcur = ggml_mul_mat(ctx0, model.layers[il].wq, cur); - cb(Qcur, "Qcur", il); - // if (model.layers[il].bq) { - // Qcur = ggml_add(ctx0, Qcur, model.layers[il].bq); - // cb(Qcur, "Qcur", il); - // } - - struct ggml_tensor * Kcur = ggml_mul_mat(ctx0, model.layers[il].wk, cur); - cb(Kcur, "Kcur", il); - // if (model.layers[il].bk) { - // Kcur = ggml_add(ctx0, Kcur, model.layers[il].bk); - // cb(Kcur, "Kcur", il); - // } - - struct ggml_tensor * Vcur = ggml_mul_mat(ctx0, model.layers[il].wv, cur); - cb(Vcur, "Vcur", il); - // if (model.layers[il].bv) { - // Vcur = ggml_add(ctx0, Vcur, model.layers[il].bv); - // cb(Vcur, "Vcur", il); - // } - - Qcur = ggml_rope_custom( - ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, - hparams.n_rot, 2, 0, n_orig_ctx, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - cb(Qcur, "Qcur", il); - - Kcur = ggml_rope_custom( - ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, - hparams.n_rot, 2, 0, n_orig_ctx, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - cb(Kcur, "Kcur", il); - - cur = llm_build_kv(ctx0, model, hparams, kv_self, gf, - model.layers[il].wo, NULL, - Kcur, Vcur, Qcur, KQ_mask, n_ctx, n_tokens, kv_head, n_kv, -1.0f, 1.0f/sqrtf(float(n_embd_head)), cb, il); - cb(cur, "kqv_out", il); - } - - struct ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA); - cb(ffn_inp, "ffn_inp", il); - - // feed-forward network - cur = llm_build_norm(ctx0, ffn_inp, hparams, - model.layers[il].ffn_norm, model.layers[il].ffn_norm_b, - LLM_NORM, cb, il); - cb(cur, "ffn_norm", il); - - cur = llm_build_ffn(ctx0, cur, - model.layers[il].ffn_up, NULL, - model.layers[il].ffn_gate, NULL, - model.layers[il].ffn_down, NULL, - NULL, - LLM_FFN_SILU, LLM_FFN_PAR, cb, il); - cb(cur, "ffn_out", il); - - cur = ggml_add(ctx0, cur, ffn_inp); - cb(cur, "l_out", il); - - // input for next layer - inpL = cur; - } - - cur = inpL; - - cur = llm_build_norm(ctx0, cur, hparams, - model.output_norm, model.output_norm_b, - LLM_NORM, cb, -1); - cb(cur, "result_norm", -1); - - // lm_head - cur = ggml_mul_mat(ctx0, model.output, cur); - cb(cur, "result_output", -1); - - ggml_build_forward_expand(gf, cur); - - return gf; - } - - struct ggml_cgraph * build_llama() { struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, LLAMA_MAX_NODES, false); @@ -6589,6 +6469,125 @@ struct llm_build_context { return gf; } + + struct ggml_cgraph * build_orion() { + struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, LLAMA_MAX_NODES, false); + + const int64_t n_embd_head = hparams.n_embd_head_v; + GGML_ASSERT(n_embd_head == hparams.n_embd_head_k); + GGML_ASSERT(n_embd_head == hparams.n_rot); + + struct ggml_tensor * cur; + struct ggml_tensor * inpL; + + inpL = llm_build_inp_embd(ctx0, hparams, batch, model.tok_embd, lctx.inp_tokens, lctx.inp_embd, cb); + cb(inpL, "inp_embd", -1); + + // inp_pos - contains the positions + struct ggml_tensor * inp_pos = ggml_view_1d(ctx0, lctx.inp_pos, n_tokens, 0); + cb(inp_pos, "inp_pos", -1); + + // KQ_mask (mask for 1 head, it will be broadcasted to all heads) + struct ggml_tensor * KQ_mask = ggml_view_2d(ctx0, lctx.inp_KQ_mask, n_kv, n_tokens, n_kv*ggml_type_size(lctx.inp_KQ_mask->type), 0); + cb(KQ_mask, "KQ_mask", -1); + + // shift the entire K-cache if needed + if (do_rope_shift) { + llm_build_k_shift(ctx0, hparams, cparams, kv_self, gf, lctx.inp_K_shift, LLM_ROPE, n_ctx, freq_base, freq_scale, cb); + } + + for (int il = 0; il < n_layer; ++il) { + struct ggml_tensor * inpSA = inpL; + + // norm + cur = llm_build_norm(ctx0, inpL, hparams, + model.layers[il].attn_norm, model.layers[il].attn_norm_b, + LLM_NORM, cb, il); + cb(cur, "attn_norm", il); + + // self-attention + { + // compute Q and K and RoPE them + struct ggml_tensor * Qcur = ggml_mul_mat(ctx0, model.layers[il].wq, cur); + cb(Qcur, "Qcur", il); + // if (model.layers[il].bq) { + // Qcur = ggml_add(ctx0, Qcur, model.layers[il].bq); + // cb(Qcur, "Qcur", il); + // } + + struct ggml_tensor * Kcur = ggml_mul_mat(ctx0, model.layers[il].wk, cur); + cb(Kcur, "Kcur", il); + // if (model.layers[il].bk) { + // Kcur = ggml_add(ctx0, Kcur, model.layers[il].bk); + // cb(Kcur, "Kcur", il); + // } + + struct ggml_tensor * Vcur = ggml_mul_mat(ctx0, model.layers[il].wv, cur); + cb(Vcur, "Vcur", il); + // if (model.layers[il].bv) { + // Vcur = ggml_add(ctx0, Vcur, model.layers[il].bv); + // cb(Vcur, "Vcur", il); + // } + + Qcur = ggml_rope_custom( + ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, + hparams.n_rot, 2, 0, n_orig_ctx, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow + ); + cb(Qcur, "Qcur", il); + + Kcur = ggml_rope_custom( + ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, + hparams.n_rot, 2, 0, n_orig_ctx, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow + ); + cb(Kcur, "Kcur", il); + + cur = llm_build_kv(ctx0, model, hparams, kv_self, gf, + model.layers[il].wo, NULL, + Kcur, Vcur, Qcur, KQ_mask, n_ctx, n_tokens, kv_head, n_kv, -1.0f, 1.0f/sqrtf(float(n_embd_head)), cb, il); + cb(cur, "kqv_out", il); + } + + struct ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA); + cb(ffn_inp, "ffn_inp", il); + + // feed-forward network + cur = llm_build_norm(ctx0, ffn_inp, hparams, + model.layers[il].ffn_norm, model.layers[il].ffn_norm_b, + LLM_NORM, cb, il); + cb(cur, "ffn_norm", il); + + cur = llm_build_ffn(ctx0, cur, + model.layers[il].ffn_up, NULL, + model.layers[il].ffn_gate, NULL, + model.layers[il].ffn_down, NULL, + NULL, + LLM_FFN_SILU, LLM_FFN_PAR, cb, il); + cb(cur, "ffn_out", il); + + cur = ggml_add(ctx0, cur, ffn_inp); + cb(cur, "l_out", il); + + // input for next layer + inpL = cur; + } + + cur = inpL; + + cur = llm_build_norm(ctx0, cur, hparams, + model.output_norm, model.output_norm_b, + LLM_NORM, cb, -1); + cb(cur, "result_norm", -1); + + // lm_head + cur = ggml_mul_mat(ctx0, model.output, cur); + cb(cur, "result_output", -1); + + ggml_build_forward_expand(gf, cur); + + return gf; + } }; static struct ggml_cgraph * llama_build_graph( From 1cfb5372cf5707c8ec6dde7c874f4a44a6c4c915 Mon Sep 17 00:00:00 2001 From: Eve <139727413+netrunnereve@users.noreply.github.com> Date: Wed, 31 Jan 2024 19:21:55 +0000 Subject: [PATCH 233/812] Fix broken Vulkan Cmake (properly) (#5230) * build vulkan as object * vulkan ci --- .github/workflows/build.yml | 6 ++++-- CMakeLists.txt | 8 ++------ 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c6db1666e..f4c374ce5 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -356,6 +356,8 @@ jobs: defines: '-DLLAMA_NATIVE=OFF -DLLAMA_BUILD_SERVER=ON -DLLAMA_BLAS=ON -DBUILD_SHARED_LIBS=ON -DLLAMA_BLAS_VENDOR=OpenBLAS -DBLAS_INCLUDE_DIRS="$env:RUNNER_TEMP/openblas/include" -DBLAS_LIBRARIES="$env:RUNNER_TEMP/openblas/lib/openblas.lib"' - build: 'kompute' defines: '-DLLAMA_NATIVE=OFF -DLLAMA_BUILD_SERVER=ON -DLLAMA_KOMPUTE=ON -DKOMPUTE_OPT_DISABLE_VULKAN_VERSION_CHECK=ON -DBUILD_SHARED_LIBS=ON' + - build: 'vulkan' + defines: '-DLLAMA_NATIVE=OFF -DLLAMA_BUILD_SERVER=ON -DLLAMA_VULKAN=ON -DBUILD_SHARED_LIBS=ON' steps: - name: Clone @@ -406,7 +408,7 @@ jobs: - name: Install Vulkan SDK id: get_vulkan - if: ${{ matrix.build == 'kompute' }} + if: ${{ matrix.build == 'kompute' || matrix.build == 'vulkan' }} run: | curl.exe -o $env:RUNNER_TEMP/VulkanSDK-Installer.exe -L "https://sdk.lunarg.com/sdk/download/${env:VULKAN_VERSION}/windows/VulkanSDK-${env:VULKAN_VERSION}-Installer.exe" & "$env:RUNNER_TEMP\VulkanSDK-Installer.exe" --accept-licenses --default-answer --confirm-command install @@ -451,7 +453,7 @@ jobs: - name: Test id: cmake_test # not all machines have native AVX-512 - if: ${{ matrix.build != 'clblast' && matrix.build != 'kompute' && (matrix.build != 'avx512' || env.HAS_AVX512F == '1') }} + if: ${{ matrix.build != 'clblast' && matrix.build != 'kompute' && matrix.build != 'vulkan' && (matrix.build != 'avx512' || env.HAS_AVX512F == '1') }} run: | cd build ctest -L main -C Release --verbose --timeout 900 diff --git a/CMakeLists.txt b/CMakeLists.txt index 15a1101aa..1ee455b3a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -423,10 +423,7 @@ if (LLAMA_VULKAN) if (Vulkan_FOUND) message(STATUS "Vulkan found") - set(GGML_HEADERS_VULKAN ggml-vulkan.h) - set(GGML_SOURCES_VULKAN ggml-vulkan.cpp) - - add_library(ggml-vulkan STATIC ggml-vulkan.cpp ggml-vulkan.h) + add_library(ggml-vulkan OBJECT ggml-vulkan.cpp ggml-vulkan.h) if (BUILD_SHARED_LIBS) set_target_properties(ggml-vulkan PROPERTIES POSITION_INDEPENDENT_CODE ON) endif() @@ -1012,7 +1009,6 @@ add_library(ggml OBJECT ggml-quants.h ${GGML_SOURCES_CUDA} ${GGML_HEADERS_CUDA} ${GGML_SOURCES_OPENCL} ${GGML_HEADERS_OPENCL} - ${GGML_SOURCES_VULKAN} ${GGML_HEADERS_VULKAN} ${GGML_SOURCES_METAL} ${GGML_HEADERS_METAL} ${GGML_SOURCES_MPI} ${GGML_HEADERS_MPI} ${GGML_SOURCES_EXTRA} ${GGML_HEADERS_EXTRA} @@ -1094,7 +1090,7 @@ install(FILES ${CMAKE_CURRENT_BINARY_DIR}/LlamaConfig.cmake DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/Llama) set(GGML_PUBLIC_HEADERS "ggml.h" "ggml-alloc.h" "ggml-backend.h" - "${GGML_HEADERS_CUDA}" "${GGML_HEADERS_OPENCL}" "${GGML_HEADERS_VULKAN}" + "${GGML_HEADERS_CUDA}" "${GGML_HEADERS_OPENCL}" "${GGML_HEADERS_METAL}" "${GGML_HEADERS_MPI}" "${GGML_HEADERS_EXTRA}") set_target_properties(ggml PROPERTIES PUBLIC_HEADER "${GGML_PUBLIC_HEADERS}") From ce32060198b7e2d6a13a9b8e1e1369e3c295ae2a Mon Sep 17 00:00:00 2001 From: Guoteng <32697156+SolenoidWGT@users.noreply.github.com> Date: Thu, 1 Feb 2024 17:19:51 +0800 Subject: [PATCH 234/812] llama : support InternLM2 (#5184) * support InternLM2 inference * add add_space_prefix KV pair --- convert-hf-to-gguf.py | 152 ++++++++++++++++++++++++ gguf-py/gguf/constants.py | 18 +++ gguf-py/gguf/gguf_writer.py | 3 + gguf-py/gguf/tensor_mapping.py | 14 ++- llama.cpp | 205 ++++++++++++++++++++++++++++++++- 5 files changed, 387 insertions(+), 5 deletions(-) diff --git a/convert-hf-to-gguf.py b/convert-hf-to-gguf.py index 6ab7f486e..4ebab07b3 100755 --- a/convert-hf-to-gguf.py +++ b/convert-hf-to-gguf.py @@ -203,6 +203,8 @@ class Model: return CodeShellModel if model_architecture == "OrionForCausalLM": return OrionModel + if model_architecture == "InternLM2ForCausalLM": + return InternLM2Model return Model def _is_model_safetensors(self) -> bool: @@ -254,6 +256,8 @@ class Model: return gguf.MODEL_ARCH.CODESHELL if arch == "OrionForCausalLM": return gguf.MODEL_ARCH.ORION + if arch == "InternLM2ForCausalLM": + return gguf.MODEL_ARCH.INTERNLM2 raise NotImplementedError(f'Architecture "{arch}" not supported!') @@ -1344,6 +1348,154 @@ class CodeShellModel(Model): self.gguf_writer.add_tensor("output.weight", data) print(name, f"=> output.weight, shape = {data.shape}, {old_dtype} --> {data.dtype}") + +class InternLM2Model(Model): + def set_vocab(self): + # (TODO): Is there a better way? + # Copy from _set_vocab_sentencepiece, The only difference is that we will treat the character + # \x00 specially and convert it into an emoji character to prevent it from being mistakenly + # recognized as an empty string in C++. + from sentencepiece import SentencePieceProcessor + from sentencepiece import sentencepiece_model_pb2 as model + + tokenizer_path = self.dir_model / 'tokenizer.model' + + tokens: list[bytes] = [] + scores: list[float] = [] + toktypes: list[int] = [] + + if not tokenizer_path.is_file(): + print(f'Error: Missing {tokenizer_path}', file=sys.stderr) + sys.exit(1) + + sentencepiece_model = model.ModelProto() + sentencepiece_model.ParseFromString(open(tokenizer_path, "rb").read()) + add_prefix = sentencepiece_model.normalizer_spec.add_dummy_prefix + + tokenizer = SentencePieceProcessor(str(tokenizer_path)) + vocab_size = self.hparams.get('vocab_size', tokenizer.vocab_size()) + + for token_id in range(vocab_size): + piece = tokenizer.id_to_piece(token_id) + text = piece.encode("utf-8") + score = tokenizer.get_score(token_id) + if text == b"\x00": + # (TODO): fixme + # Hack here and replace the \x00 characters. + print(f"InternLM2 convert token '{text}' to '🐉'!") + text = "🐉" + + toktype = SentencePieceTokenTypes.NORMAL + if tokenizer.is_unknown(token_id): + toktype = SentencePieceTokenTypes.UNKNOWN + elif tokenizer.is_control(token_id): + toktype = SentencePieceTokenTypes.CONTROL + elif tokenizer.is_unused(token_id): + toktype = SentencePieceTokenTypes.UNUSED + elif tokenizer.is_byte(token_id): + toktype = SentencePieceTokenTypes.BYTE + + tokens.append(text) + scores.append(score) + toktypes.append(toktype) + + added_tokens_file = self.dir_model / 'added_tokens.json' + if added_tokens_file.is_file(): + with open(added_tokens_file, "r", encoding="utf-8") as f: + added_tokens_json = json.load(f) + + for key in added_tokens_json: + tokens.append(key.encode("utf-8")) + scores.append(-1000.0) + toktypes.append(SentencePieceTokenTypes.USER_DEFINED) + + self.gguf_writer.add_tokenizer_model("llama") + self.gguf_writer.add_token_list(tokens) + self.gguf_writer.add_token_scores(scores) + self.gguf_writer.add_token_types(toktypes) + self.gguf_writer.add_add_space_prefix(add_prefix) + + special_vocab = gguf.SpecialVocab(self.dir_model, n_vocab=len(tokens)) + special_vocab.add_to_gguf(self.gguf_writer) + + def set_gguf_parameters(self): + self.gguf_writer.add_name("InternLM2") + self.gguf_writer.add_context_length(self.hparams["max_position_embeddings"]) + self.gguf_writer.add_block_count(self.hparams["num_hidden_layers"]) + self.gguf_writer.add_embedding_length(self.hparams["hidden_size"]) + self.gguf_writer.add_feed_forward_length(self.hparams["intermediate_size"]) + self.gguf_writer.add_rope_freq_base(self.hparams["rope_theta"]) + self.gguf_writer.add_head_count(self.hparams["num_attention_heads"]) + self.gguf_writer.add_layer_norm_rms_eps(self.hparams["rms_norm_eps"]) + self.gguf_writer.add_head_count_kv(self.hparams["num_key_value_heads"]) + + def post_write_tensors(self, tensor_map, name, data_torch): + old_dtype = data_torch.dtype + + # convert any unsupported data types to float32 + if data_torch.dtype not in (torch.float16, torch.float32): + data_torch = data_torch.to(torch.float32) + + data = data_torch.squeeze().numpy() + + # map tensor names + new_name = tensor_map.get_name(name, try_suffixes=(".weight", ".bias")) + if new_name is None: + print(f"Can not map tensor {name!r}") + sys.exit() + + n_dims = len(data.shape) + data_dtype = data.dtype + + # if f32 desired, convert any float16 to float32 + if self.ftype == 0 and data_dtype == np.float16: + data = data.astype(np.float32) + + # TODO: Why cant we use these float16 as-is? There should be not reason to store float16 as float32 + if self.ftype == 1 and data_dtype == np.float16 and n_dims == 1: + data = data.astype(np.float32) + + # if f16 desired, convert any float32 2-dim weight tensors to float16 + if self.ftype == 1 and data_dtype == np.float32 and name.endswith(".weight") and n_dims == 2: + data = data.astype(np.float16) + + print(f"{new_name}, n_dims = {n_dims}, {old_dtype} --> {data.dtype}") + self.gguf_writer.add_tensor(new_name, data) + + def write_tensors(self): + from einops import rearrange + + num_heads = self.hparams.get("num_attention_heads") + num_kv_heads = self.hparams.get("num_key_value_heads") + hidden_size = self.hparams.get("hidden_size") + q_per_kv = num_heads // num_kv_heads + head_dim = hidden_size // num_heads + num_groups = num_heads // q_per_kv + + block_count = self.hparams["num_hidden_layers"] + model_kv = dict(self.get_tensors()) + tensor_map = gguf.get_tensor_name_map(self.model_arch, block_count) + qkv_pattern = r"model\.layers\.(\d+)\.attention\.wqkv" + for name, data_torch in model_kv.items(): + # we don't need these + if name.endswith(".rotary_emb.inv_freq"): + continue + + if re.match(qkv_pattern, name): + bid = re.findall(qkv_pattern, name)[0] + qkv = data_torch + qkv = rearrange(qkv.T, " o (g n i) ->o g n i", g=num_groups, n=q_per_kv + 2, i=head_dim) + q, k, v = qkv[..., : q_per_kv, :], qkv[..., q_per_kv: q_per_kv + 1, :], qkv[..., q_per_kv + 1: q_per_kv + 2, :] + q = rearrange(q, " o g n i -> o (g n i)").T + k = rearrange(k, " o g n i -> o (g n i)").T + v = rearrange(v, " o g n i -> o (g n i)").T + self.post_write_tensors(tensor_map, f"model.layers.{bid}.attention.wq.weight", q) + self.post_write_tensors(tensor_map, f"model.layers.{bid}.attention.wk.weight", k) + self.post_write_tensors(tensor_map, f"model.layers.{bid}.attention.wv.weight", v) + else: + self.post_write_tensors(tensor_map, name, data_torch) + + ###### CONVERSION LOGIC ###### diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py index f5c933a41..ed8e26f83 100644 --- a/gguf-py/gguf/constants.py +++ b/gguf-py/gguf/constants.py @@ -72,6 +72,7 @@ class Keys: PAD_ID = "tokenizer.ggml.padding_token_id" ADD_BOS = "tokenizer.ggml.add_bos_token" ADD_EOS = "tokenizer.ggml.add_eos_token" + ADD_PREFIX = "tokenizer.ggml.add_space_prefix" HF_JSON = "tokenizer.huggingface.json" RWKV = "tokenizer.rwkv.world" CHAT_TEMPLATE = "tokenizer.chat_template" @@ -102,6 +103,7 @@ class MODEL_ARCH(IntEnum): PLAMO = auto() CODESHELL = auto() ORION = auto() + INTERNLM2 = auto() class MODEL_TENSOR(IntEnum): @@ -153,6 +155,7 @@ MODEL_ARCH_NAMES: dict[MODEL_ARCH, str] = { MODEL_ARCH.PLAMO: "plamo", MODEL_ARCH.CODESHELL: "codeshell", MODEL_ARCH.ORION: "orion", + MODEL_ARCH.INTERNLM2: "internlm2", } TENSOR_NAMES: dict[MODEL_TENSOR, str] = { @@ -446,6 +449,21 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = { MODEL_TENSOR.FFN_DOWN, MODEL_TENSOR.FFN_UP, ], + MODEL_ARCH.INTERNLM2: [ + MODEL_TENSOR.TOKEN_EMBD, + MODEL_TENSOR.OUTPUT_NORM, + MODEL_TENSOR.OUTPUT, + MODEL_TENSOR.ATTN_NORM, + MODEL_TENSOR.ATTN_Q, + MODEL_TENSOR.ATTN_K, + MODEL_TENSOR.ATTN_V, + MODEL_TENSOR.ATTN_OUT, + MODEL_TENSOR.ATTN_ROT_EMBD, + MODEL_TENSOR.FFN_NORM, + MODEL_TENSOR.FFN_GATE, + MODEL_TENSOR.FFN_DOWN, + MODEL_TENSOR.FFN_UP, + ], # TODO } diff --git a/gguf-py/gguf/gguf_writer.py b/gguf-py/gguf/gguf_writer.py index d93aaa877..16808196e 100644 --- a/gguf-py/gguf/gguf_writer.py +++ b/gguf-py/gguf/gguf_writer.py @@ -411,6 +411,9 @@ class GGUFWriter: def add_add_eos_token(self, value: bool) -> None: self.add_bool(Keys.Tokenizer.ADD_EOS, value) + def add_add_space_prefix(self, value: bool) -> None: + self.add_bool(Keys.Tokenizer.ADD_PREFIX, value) + def add_chat_template(self, value: str) -> None: self.add_string(Keys.Tokenizer.CHAT_TEMPLATE, value) diff --git a/gguf-py/gguf/tensor_mapping.py b/gguf-py/gguf/tensor_mapping.py index de177af13..4f16d8504 100644 --- a/gguf-py/gguf/tensor_mapping.py +++ b/gguf-py/gguf/tensor_mapping.py @@ -19,6 +19,7 @@ class TensorNameMap: "language_model.embedding.word_embeddings", # persimmon "wte", # gpt2 "transformer.embd.wte", # phi2 + "model.tok_embeddings", # internlm2 ), # Token type embeddings @@ -42,7 +43,7 @@ class TensorNameMap: MODEL_TENSOR.OUTPUT: ( "embed_out", # gptneox "lm_head", # gpt2 mpt falcon llama-hf baichuan qwen - "output", # llama-pth bloom + "output", # llama-pth bloom internlm2 "word_embeddings_for_head", # persimmon "lm_head.linear", # phi2 ), @@ -51,7 +52,7 @@ class TensorNameMap: MODEL_TENSOR.OUTPUT_NORM: ( "gpt_neox.final_layer_norm", # gptneox "transformer.ln_f", # gpt2 gpt-j falcon - "model.norm", # llama-hf baichuan + "model.norm", # llama-hf baichuan internlm2 "norm", # llama-pth "embeddings.LayerNorm", # bert "transformer.norm_f", # mpt @@ -84,6 +85,7 @@ class TensorNameMap: "h.{bid}.ln_1", # gpt2 "transformer.h.{bid}.ln", # phi2 "model.layers.layers.{bid}.norm", # plamo + "model.layers.{bid}.attention_norm", # internlm2 ), # Attention norm 2 @@ -111,6 +113,7 @@ class TensorNameMap: "encoder.layer.{bid}.attention.self.query", # bert "transformer.h.{bid}.attn.q_proj", # gpt-j "model.layers.layers.{bid}.self_attn.q_proj", # plamo + "model.layers.{bid}.attention.wq" # internlm2 ), # Attention key @@ -120,6 +123,7 @@ class TensorNameMap: "encoder.layer.{bid}.attention.self.key", # bert "transformer.h.{bid}.attn.k_proj", # gpt-j "model.layers.layers.{bid}.self_attn.k_proj", # plamo + "model.layers.{bid}.attention.wk" # internlm2 ), # Attention value @@ -129,6 +133,7 @@ class TensorNameMap: "encoder.layer.{bid}.attention.self.value", # bert "transformer.h.{bid}.attn.v_proj", # gpt-j "model.layers.layers.{bid}.self_attn.v_proj", # plamo + "model.layers.{bid}.attention.wv" # internlm2 ), # Attention output @@ -147,6 +152,7 @@ class TensorNameMap: "h.{bid}.attn.c_proj", # gpt2 "transformer.h.{bid}.mixer.out_proj", # phi2 "model.layers.layers.{bid}.self_attn.o_proj", # plamo + "model.layers.{bid}.attention.wo", # internlm2 ), # Rotary embeddings @@ -169,6 +175,7 @@ class TensorNameMap: "language_model.encoder.layers.{bid}.post_attention_layernorm", # persimmon "model.layers.{bid}.ln2", # yi "h.{bid}.ln_2", # gpt2 + "model.layers.{bid}.ffn_norm", # internlm2 ), MODEL_TENSOR.FFN_GATE_INP: ( @@ -194,6 +201,7 @@ class TensorNameMap: "transformer.h.{bid}.mlp.fc1", # phi2 "model.layers.{bid}.mlp.fc1", # phi2 "model.layers.layers.{bid}.mlp.up_proj", # plamo + "model.layers.{bid}.feed_forward.w3", # internlm2 ), MODEL_TENSOR.FFN_UP_EXP: ( @@ -212,6 +220,7 @@ class TensorNameMap: "layers.{bid}.feed_forward.w1", # llama-pth "transformer.h.{bid}.mlp.w2", # qwen "model.layers.layers.{bid}.mlp.gate_proj", # plamo + "model.layers.{bid}.feed_forward.w1", # internlm2 ), MODEL_TENSOR.FFN_GATE_EXP: ( @@ -236,6 +245,7 @@ class TensorNameMap: "transformer.h.{bid}.mlp.fc2", # phi2 "model.layers.{bid}.mlp.fc2", # phi2 "model.layers.layers.{bid}.mlp.down_proj", # plamo + "model.layers.{bid}.feed_forward.w2", # internlm2 ), MODEL_TENSOR.FFN_DOWN_EXP: ( diff --git a/llama.cpp b/llama.cpp index 02b0a485a..e8f44c2cb 100644 --- a/llama.cpp +++ b/llama.cpp @@ -204,6 +204,7 @@ enum llm_arch { LLM_ARCH_PLAMO, LLM_ARCH_CODESHELL, LLM_ARCH_ORION, + LLM_ARCH_INTERNLM2, LLM_ARCH_UNKNOWN, }; @@ -226,6 +227,7 @@ static std::map LLM_ARCH_NAMES = { { LLM_ARCH_PLAMO, "plamo" }, { LLM_ARCH_CODESHELL, "codeshell" }, { LLM_ARCH_ORION, "orion" }, + { LLM_ARCH_INTERNLM2, "internlm2" }, }; enum llm_kv { @@ -278,6 +280,7 @@ enum llm_kv { LLM_KV_TOKENIZER_PAD_ID, LLM_KV_TOKENIZER_ADD_BOS, LLM_KV_TOKENIZER_ADD_EOS, + LLM_KV_TOKENIZER_ADD_PREFIX, LLM_KV_TOKENIZER_HF_JSON, LLM_KV_TOKENIZER_RWKV, }; @@ -332,6 +335,7 @@ static std::map LLM_KV_NAMES = { { LLM_KV_TOKENIZER_PAD_ID, "tokenizer.ggml.padding_token_id" }, { LLM_KV_TOKENIZER_ADD_BOS, "tokenizer.ggml.add_bos_token" }, { LLM_KV_TOKENIZER_ADD_EOS, "tokenizer.ggml.add_eos_token" }, + { LLM_KV_TOKENIZER_ADD_PREFIX, "tokenizer.ggml.add_space_prefix" }, { LLM_KV_TOKENIZER_HF_JSON, "tokenizer.huggingface.json" }, { LLM_KV_TOKENIZER_RWKV, "tokenizer.rwkv.world" }, }; @@ -669,7 +673,23 @@ static std::map> LLM_TENSOR_NAMES = { LLM_TENSOR_FFN_UP, "blk.%d.ffn_up" }, }, }, - + { + LLM_ARCH_INTERNLM2, + { + { LLM_TENSOR_TOKEN_EMBD, "token_embd" }, + { LLM_TENSOR_OUTPUT_NORM, "output_norm" }, + { LLM_TENSOR_OUTPUT, "output" }, + { LLM_TENSOR_ATTN_NORM, "blk.%d.attn_norm" }, + { LLM_TENSOR_ATTN_Q, "blk.%d.attn_q" }, + { LLM_TENSOR_ATTN_K, "blk.%d.attn_k" }, + { LLM_TENSOR_ATTN_V, "blk.%d.attn_v" }, + { LLM_TENSOR_ATTN_OUT, "blk.%d.attn_output" }, + { LLM_TENSOR_FFN_NORM, "blk.%d.ffn_norm" }, + { LLM_TENSOR_FFN_GATE, "blk.%d.ffn_gate" }, + { LLM_TENSOR_FFN_DOWN, "blk.%d.ffn_down" }, + { LLM_TENSOR_FFN_UP, "blk.%d.ffn_up" }, + }, + }, { LLM_ARCH_UNKNOWN, { @@ -1377,6 +1397,7 @@ enum e_model { MODEL_13B, MODEL_14B, MODEL_15B, + MODEL_20B, MODEL_30B, MODEL_34B, MODEL_40B, @@ -1618,6 +1639,8 @@ struct llama_vocab { id special_suffix_id = 32008; id special_eot_id = 32010; + bool add_space_prefix = true; + int find_bpe_rank(const std::string & token_left, const std::string & token_right) const { GGML_ASSERT(token_left.find(' ') == std::string::npos); GGML_ASSERT(token_left.find('\n') == std::string::npos); @@ -2731,6 +2754,7 @@ static const char * llama_model_type_name(e_model type) { case MODEL_13B: return "13B"; case MODEL_14B: return "14B"; case MODEL_15B: return "15B"; + case MODEL_20B: return "20B"; case MODEL_30B: return "30B"; case MODEL_34B: return "34B"; case MODEL_40B: return "40B"; @@ -2743,6 +2767,14 @@ static const char * llama_model_type_name(e_model type) { default: return "?B"; } } +static const char * llama_model_vocab_type_name(enum llama_vocab_type type){ + switch (type) { + case LLAMA_VOCAB_TYPE_SPM: return "SPM"; + case LLAMA_VOCAB_TYPE_BPE: return "BPE"; + default: return "unknown"; + } +} + static void llm_load_arch(llama_model_loader & ml, llama_model & model) { model.arch = ml.get_arch(); @@ -3006,6 +3038,15 @@ static void llm_load_hparams( default: model.type = e_model::MODEL_UNKNOWN; } } break; + case LLM_ARCH_INTERNLM2: + { + ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps); + switch (hparams.n_layer) { + case 32: model.type = e_model::MODEL_7B; break; + case 48: model.type = e_model::MODEL_20B; break; + default: model.type = e_model::MODEL_UNKNOWN; + } + } break; default: (void)0; } @@ -3057,6 +3098,11 @@ static void llm_load_vocab( vocab.special_unk_id = 0; vocab.special_sep_id = -1; vocab.special_pad_id = -1; + + const int add_space_prefix_keyidx = gguf_find_key(ctx, kv(LLM_KV_TOKENIZER_ADD_PREFIX).c_str()); + if (add_space_prefix_keyidx != -1) { + vocab.add_space_prefix = gguf_get_val_bool(ctx, add_space_prefix_keyidx); + } // The default value of add_space_prefix is true. } else if (tokenizer_name == "gpt2") { vocab.type = LLAMA_VOCAB_TYPE_BPE; @@ -3269,7 +3315,7 @@ static void llm_load_print_meta(llama_model_loader & ml, llama_model & model) { // hparams LLAMA_LOG_INFO("%s: format = %s\n", __func__, llama_file_version_name(ml.fver)); LLAMA_LOG_INFO("%s: arch = %s\n", __func__, LLM_ARCH_NAMES.at(model.arch).c_str()); - LLAMA_LOG_INFO("%s: vocab type = %s\n", __func__, vocab.type == LLAMA_VOCAB_TYPE_SPM ? "SPM" : "BPE"); // TODO: fix + LLAMA_LOG_INFO("%s: vocab type = %s\n", __func__, llama_model_vocab_type_name(vocab.type)); LLAMA_LOG_INFO("%s: n_vocab = %u\n", __func__, hparams.n_vocab); LLAMA_LOG_INFO("%s: n_merges = %u\n", __func__, (int) vocab.bpe_ranks.size()); LLAMA_LOG_INFO("%s: n_ctx_train = %u\n", __func__, hparams.n_ctx_train); @@ -4018,8 +4064,35 @@ static bool llm_load_tensors( layer.ffn_up = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff}); } } break; + case LLM_ARCH_INTERNLM2: + { + model.tok_embd = ml.create_tensor(ctx_input, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}); + // output + { + model.output_norm = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}); + model.output = ml.create_tensor(ctx_output_split, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}); + } + for (int i = 0; i < n_layer; ++i) { + ggml_context * ctx_layer = ctx_for_layer(i); + ggml_context * ctx_split = ctx_for_layer_split(i); + + auto & layer = model.layers[i]; + + layer.attn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}); + // layer.wqkv = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_QKV, "weight", i), {n_embd, n_embd + 2*n_embd_gqa}); + layer.wq = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_embd}); + layer.wk = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_K, "weight", i), {n_embd, n_embd_gqa}); + layer.wv = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_V, "weight", i), {n_embd, n_embd_gqa}); + + layer.wo = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd}); + layer.ffn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}); + layer.ffn_gate = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff}); + layer.ffn_down = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_DOWN, "weight", i), { n_ff, n_embd}); + layer.ffn_up = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff}); + } + } break; default: throw std::runtime_error("unknown architecture"); } @@ -6588,6 +6661,126 @@ struct llm_build_context { return gf; } + + struct ggml_cgraph * build_internlm2() { + struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, LLAMA_MAX_NODES, false); + + const int64_t n_embd_head = hparams.n_embd_head_v; + GGML_ASSERT(n_embd_head == hparams.n_embd_head_k); + GGML_ASSERT(n_embd_head == hparams.n_rot); + + struct ggml_tensor * cur; + struct ggml_tensor * inpL; + + inpL = llm_build_inp_embd(ctx0, hparams, batch, model.tok_embd, lctx.inp_tokens, lctx.inp_embd, cb); + cb(inpL, "inp_embd", -1); + + // inp_pos - contains the positions + struct ggml_tensor * inp_pos = ggml_view_1d(ctx0, lctx.inp_pos, n_tokens, 0); + cb(inp_pos, "inp_pos", -1); + + // KQ_mask (mask for 1 head, it will be broadcasted to all heads) + struct ggml_tensor * KQ_mask = ggml_view_2d(ctx0, lctx.inp_KQ_mask, n_kv, n_tokens, n_kv*ggml_type_size(lctx.inp_KQ_mask->type), 0); + cb(KQ_mask, "KQ_mask", -1); + + // shift the entire K-cache if needed + if (do_rope_shift) { + llm_build_k_shift(ctx0, hparams, cparams, kv_self, gf, lctx.inp_K_shift, LLM_ROPE, n_ctx, freq_base, freq_scale, cb); + } + + for (int il = 0; il < n_layer; ++il) { + struct ggml_tensor * inpSA = inpL; + + // norm + cur = llm_build_norm(ctx0, inpL, hparams, + model.layers[il].attn_norm, NULL, + LLM_NORM_RMS, cb, il); + cb(cur, "attn_norm", il); + + // self-attention + { + // compute Q and K and RoPE them + struct ggml_tensor * Qcur = ggml_mul_mat(ctx0, model.layers[il].wq, cur); + cb(Qcur, "Qcur", il); + if (model.layers[il].bq) { + Qcur = ggml_add(ctx0, Qcur, model.layers[il].bq); + cb(Qcur, "Qcur", il); + } + + struct ggml_tensor * Kcur = ggml_mul_mat(ctx0, model.layers[il].wk, cur); + cb(Kcur, "Kcur", il); + if (model.layers[il].bk) { + Kcur = ggml_add(ctx0, Kcur, model.layers[il].bk); + cb(Kcur, "Kcur", il); + } + + struct ggml_tensor * Vcur = ggml_mul_mat(ctx0, model.layers[il].wv, cur); + cb(Vcur, "Vcur", il); + if (model.layers[il].bv) { + Vcur = ggml_add(ctx0, Vcur, model.layers[il].bv); + cb(Vcur, "Vcur", il); + } + + Qcur = ggml_rope_custom( + ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, + hparams.n_rot, 0, 0, n_orig_ctx, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow + ); + cb(Qcur, "Qcur", il); + + Kcur = ggml_rope_custom( + ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, + hparams.n_rot, 0, 0, n_orig_ctx, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow + ); + cb(Kcur, "Kcur", il); + + cur = llm_build_kv(ctx0, model, hparams, kv_self, gf, + model.layers[il].wo, model.layers[il].bo, + Kcur, Vcur, Qcur, KQ_mask, n_ctx, n_tokens, kv_head, n_kv, -1.0f, 1.0f/sqrtf(float(n_embd_head)), cb, il); + cb(cur, "kqv_out", il); + } + + struct ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA); + cb(ffn_inp, "ffn_inp", il); + + // feed-forward network + cur = llm_build_norm(ctx0, ffn_inp, hparams, + model.layers[il].ffn_norm, NULL, + LLM_NORM_RMS, cb, il); + cb(cur, "ffn_norm", il); + + cur = llm_build_ffn(ctx0, cur, + model.layers[il].ffn_up, NULL, + model.layers[il].ffn_gate, NULL, + model.layers[il].ffn_down, NULL, + NULL, + LLM_FFN_SILU, LLM_FFN_PAR, cb, il); + cb(cur, "ffn_out", il); + + cur = ggml_add(ctx0, cur, ffn_inp); + cb(cur, "l_out", il); + + // input for next layer + inpL = cur; + } + + cur = inpL; + + cur = llm_build_norm(ctx0, cur, hparams, + model.output_norm, NULL, + LLM_NORM_RMS, cb, -1); + cb(cur, "result_norm", -1); + + // lm_head + cur = ggml_mul_mat(ctx0, model.output, cur); + cb(cur, "result_output", -1); + + ggml_build_forward_expand(gf, cur); + + return gf; + } + }; static struct ggml_cgraph * llama_build_graph( @@ -6746,6 +6939,10 @@ static struct ggml_cgraph * llama_build_graph( { result = llm.build_orion(); } break; + case LLM_ARCH_INTERNLM2: + { + result = llm.build_internlm2(); + } break; default: GGML_ASSERT(false); } @@ -7688,7 +7885,9 @@ static std::vector llama_tokenize_internal(const llama_vocab & // auto raw_text = fragment.raw_text.substr(fragment.offset, fragment.length); if (&fragment == &fragment_buffer.front()) { - raw_text = " " + raw_text; // prefix with space if the first token is not special + if (vocab.add_space_prefix) { + raw_text = " " + raw_text; // prefix with space if the first token is not special + } } #ifdef PRETOKENIZERDEBUG From d71ac90985854b0905e1abba778e407e17f9f887 Mon Sep 17 00:00:00 2001 From: Ali Nehzat Date: Fri, 2 Feb 2024 02:18:53 +1100 Subject: [PATCH 235/812] make : generate .a library for static linking (#5205) --- Makefile | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 781f0bf8c..bf9e085de 100644 --- a/Makefile +++ b/Makefile @@ -586,8 +586,11 @@ train.o: common/train.cpp common/train.h libllama.so: llama.o ggml.o $(OBJS) $(CXX) $(CXXFLAGS) -shared -fPIC -o $@ $^ $(LDFLAGS) +libllama.a: llama.o ggml.o $(OBJS) $(COMMON_DEPS) + ar rcs libllama.a llama.o ggml.o $(OBJS) $(COMMON_DEPS) + clean: - rm -vrf *.o tests/*.o *.so *.dll benchmark-matmult common/build-info.cpp *.dot $(COV_TARGETS) $(BUILD_TARGETS) $(TEST_TARGETS) + rm -vrf *.o tests/*.o *.so *.a *.dll benchmark-matmult common/build-info.cpp *.dot $(COV_TARGETS) $(BUILD_TARGETS) $(TEST_TARGETS) # # Examples From 8ca511cadee2c67f0bd8c7034a2513778ee9a1b7 Mon Sep 17 00:00:00 2001 From: slaren Date: Thu, 1 Feb 2024 18:30:17 +0100 Subject: [PATCH 236/812] cuda : fix LLAMA_CUDA_F16 (#5262) --- ggml-cuda.cu | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ggml-cuda.cu b/ggml-cuda.cu index e56595742..3242a0b4a 100644 --- a/ggml-cuda.cu +++ b/ggml-cuda.cu @@ -8657,9 +8657,9 @@ static void ggml_cuda_op_dequantize_mul_mat_vec( if (src1_convert_f16) { src1_dfloat = src1_dfloat_a.alloc(ne00); - ggml_cpy_f32_f16_cuda((const char *) src1_ddf_i, (char *) src1_dfloat, ne00, - ne00, 1, sizeof(float), 0, 0, - ne00, 1, sizeof(half), 0, 0, stream); + const to_fp16_cuda_t to_fp16_cuda = ggml_get_to_fp16_cuda(src1->type); + GGML_ASSERT(to_fp16_cuda != nullptr); + to_fp16_cuda(src1_ddf_i, src1_dfloat, ne00, stream); } #else const dfloat * src1_dfloat = (const dfloat *) src1_ddf_i; // dfloat == float, no conversion From 4d0924a8902010d31bd737b6f1f594943d120d0f Mon Sep 17 00:00:00 2001 From: 0cc4m Date: Thu, 1 Feb 2024 19:25:24 +0100 Subject: [PATCH 237/812] Vulkan Phi Fix for AMD Proprietary Drivers (#5260) * Replace tanh to avoid NaN in gelu shader on AMD proprietary driver * Fix another Vulkan CPY buffer size bug --- ggml-vulkan-shaders.hpp | 132 +++++++++++++++++++----------------- ggml-vulkan.cpp | 17 +++-- ggml_vk_generate_shaders.py | 3 +- 3 files changed, 83 insertions(+), 69 deletions(-) diff --git a/ggml-vulkan-shaders.hpp b/ggml-vulkan-shaders.hpp index e2e9be22c..195410c02 100644 --- a/ggml-vulkan-shaders.hpp +++ b/ggml-vulkan-shaders.hpp @@ -14670,14 +14670,14 @@ const uint64_t f32_to_f16_fp32_len = 1596; unsigned char gelu_f32_data[] = { 0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, -0x45,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, +0x4b,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, 0x01,0x00,0x00,0x00,0x0b,0x00,0x06,0x00,0x01,0x00,0x00,0x00, 0x47,0x4c,0x53,0x4c,0x2e,0x73,0x74,0x64,0x2e,0x34,0x35,0x30, 0x00,0x00,0x00,0x00,0x0e,0x00,0x03,0x00,0x00,0x00,0x00,0x00, 0x01,0x00,0x00,0x00,0x0f,0x00,0x09,0x00,0x05,0x00,0x00,0x00, 0x04,0x00,0x00,0x00,0x6d,0x61,0x69,0x6e,0x00,0x00,0x00,0x00, 0x0b,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x24,0x00,0x00,0x00, -0x2c,0x00,0x00,0x00,0x10,0x00,0x06,0x00,0x04,0x00,0x00,0x00, +0x38,0x00,0x00,0x00,0x10,0x00,0x06,0x00,0x04,0x00,0x00,0x00, 0x11,0x00,0x00,0x00,0x00,0x02,0x00,0x00,0x01,0x00,0x00,0x00, 0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x0b,0x00,0x00,0x00, 0x0b,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x48,0x00,0x05,0x00, @@ -14696,15 +14696,15 @@ unsigned char gelu_f32_data[] = { 0x22,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, 0x24,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, 0x47,0x00,0x04,0x00,0x24,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x29,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x35,0x00,0x00,0x00, 0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00, -0x2a,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x19,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x2a,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x36,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x19,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x36,0x00,0x00,0x00,0x00,0x00,0x00,0x00, 0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, -0x2a,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x2c,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x2c,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x42,0x00,0x00,0x00, +0x36,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x38,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x38,0x00,0x00,0x00,0x21,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x48,0x00,0x00,0x00, 0x0b,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x13,0x00,0x02,0x00, 0x02,0x00,0x00,0x00,0x21,0x00,0x03,0x00,0x03,0x00,0x00,0x00, 0x02,0x00,0x00,0x00,0x15,0x00,0x04,0x00,0x06,0x00,0x00,0x00, @@ -14731,64 +14731,70 @@ unsigned char gelu_f32_data[] = { 0x23,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x22,0x00,0x00,0x00, 0x3b,0x00,0x04,0x00,0x23,0x00,0x00,0x00,0x24,0x00,0x00,0x00, 0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x26,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x11,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, -0x29,0x00,0x00,0x00,0x11,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, -0x2a,0x00,0x00,0x00,0x29,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x2b,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x2a,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x2b,0x00,0x00,0x00,0x2c,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x11,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x11,0x00,0x00,0x00,0x2a,0x00,0x00,0x00,0x2a,0x42,0x4c,0x3f, +0x2b,0x00,0x04,0x00,0x11,0x00,0x00,0x00,0x2d,0x00,0x00,0x00, +0x00,0x00,0x80,0x3f,0x2b,0x00,0x04,0x00,0x11,0x00,0x00,0x00, +0x2e,0x00,0x00,0x00,0x13,0x27,0x37,0x3d,0x1d,0x00,0x03,0x00, +0x35,0x00,0x00,0x00,0x11,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, +0x36,0x00,0x00,0x00,0x35,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x37,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x36,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0x37,0x00,0x00,0x00,0x38,0x00,0x00,0x00, 0x0c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x11,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x00,0x00,0x00,0x3f,0x2b,0x00,0x04,0x00, -0x11,0x00,0x00,0x00,0x31,0x00,0x00,0x00,0x00,0x00,0x80,0x3f, -0x2b,0x00,0x04,0x00,0x11,0x00,0x00,0x00,0x32,0x00,0x00,0x00, -0x2a,0x42,0x4c,0x3f,0x2b,0x00,0x04,0x00,0x11,0x00,0x00,0x00, -0x35,0x00,0x00,0x00,0x13,0x27,0x37,0x3d,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x00,0x02,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x41,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x2c,0x00,0x06,0x00,0x09,0x00,0x00,0x00, -0x42,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x41,0x00,0x00,0x00, -0x41,0x00,0x00,0x00,0x36,0x00,0x05,0x00,0x02,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x03,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0x05,0x00,0x00,0x00,0xf7,0x00,0x03,0x00, -0x43,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xfb,0x00,0x03,0x00, -0x0c,0x00,0x00,0x00,0x44,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x44,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00, -0x0e,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, -0x0e,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x19,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0xae,0x00,0x05,0x00,0x1a,0x00,0x00,0x00, -0x1b,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x19,0x00,0x00,0x00, -0xf7,0x00,0x03,0x00,0x1d,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x1b,0x00,0x00,0x00,0x1c,0x00,0x00,0x00, -0x1d,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x1c,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x43,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x1d,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x26,0x00,0x00,0x00, -0x27,0x00,0x00,0x00,0x24,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0x0f,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x11,0x00,0x00,0x00, -0x28,0x00,0x00,0x00,0x27,0x00,0x00,0x00,0x85,0x00,0x05,0x00, +0x3a,0x00,0x00,0x00,0x00,0x00,0x00,0x3f,0x2b,0x00,0x04,0x00, +0x11,0x00,0x00,0x00,0x3d,0x00,0x00,0x00,0x00,0x00,0x00,0x40, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x46,0x00,0x00,0x00, +0x00,0x02,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x47,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x2c,0x00,0x06,0x00, +0x09,0x00,0x00,0x00,0x48,0x00,0x00,0x00,0x46,0x00,0x00,0x00, +0x47,0x00,0x00,0x00,0x47,0x00,0x00,0x00,0x36,0x00,0x05,0x00, +0x02,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x03,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x05,0x00,0x00,0x00, +0xf7,0x00,0x03,0x00,0x49,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0xfb,0x00,0x03,0x00,0x0c,0x00,0x00,0x00,0x4a,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0x4a,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x0d,0x00,0x00,0x00,0x0e,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x0f,0x00,0x00,0x00,0x0e,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x17,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x14,0x00,0x00,0x00, +0x16,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x19,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0xae,0x00,0x05,0x00, +0x1a,0x00,0x00,0x00,0x1b,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, +0x19,0x00,0x00,0x00,0xf7,0x00,0x03,0x00,0x1d,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x1b,0x00,0x00,0x00, +0x1c,0x00,0x00,0x00,0x1d,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0x1c,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x49,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0x1d,0x00,0x00,0x00,0x41,0x00,0x06,0x00, +0x26,0x00,0x00,0x00,0x27,0x00,0x00,0x00,0x24,0x00,0x00,0x00, +0x16,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x11,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0x27,0x00,0x00,0x00, +0x85,0x00,0x05,0x00,0x11,0x00,0x00,0x00,0x2c,0x00,0x00,0x00, +0x2a,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0x85,0x00,0x05,0x00, 0x11,0x00,0x00,0x00,0x30,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, +0x28,0x00,0x00,0x00,0x0c,0x00,0x08,0x00,0x11,0x00,0x00,0x00, +0x33,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00, +0x30,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0x2d,0x00,0x00,0x00, +0x85,0x00,0x05,0x00,0x11,0x00,0x00,0x00,0x34,0x00,0x00,0x00, +0x2c,0x00,0x00,0x00,0x33,0x00,0x00,0x00,0x85,0x00,0x05,0x00, +0x11,0x00,0x00,0x00,0x3c,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, 0x28,0x00,0x00,0x00,0x85,0x00,0x05,0x00,0x11,0x00,0x00,0x00, -0x34,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x28,0x00,0x00,0x00, -0x85,0x00,0x05,0x00,0x11,0x00,0x00,0x00,0x37,0x00,0x00,0x00, -0x35,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0x0c,0x00,0x08,0x00, -0x11,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x32,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x28,0x00,0x00,0x00, -0x31,0x00,0x00,0x00,0x85,0x00,0x05,0x00,0x11,0x00,0x00,0x00, -0x3b,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, -0x0c,0x00,0x06,0x00,0x11,0x00,0x00,0x00,0x3c,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x15,0x00,0x00,0x00,0x3b,0x00,0x00,0x00, -0x81,0x00,0x05,0x00,0x11,0x00,0x00,0x00,0x3d,0x00,0x00,0x00, -0x31,0x00,0x00,0x00,0x3c,0x00,0x00,0x00,0x85,0x00,0x05,0x00, -0x11,0x00,0x00,0x00,0x3e,0x00,0x00,0x00,0x30,0x00,0x00,0x00, -0x3d,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x26,0x00,0x00,0x00, -0x3f,0x00,0x00,0x00,0x2c,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0x0f,0x00,0x00,0x00,0x3e,0x00,0x03,0x00,0x3f,0x00,0x00,0x00, -0x3e,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x43,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0x43,0x00,0x00,0x00,0xfd,0x00,0x01,0x00, -0x38,0x00,0x01,0x00, +0x3f,0x00,0x00,0x00,0x3d,0x00,0x00,0x00,0x34,0x00,0x00,0x00, +0x0c,0x00,0x06,0x00,0x11,0x00,0x00,0x00,0x40,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x1b,0x00,0x00,0x00,0x3f,0x00,0x00,0x00, +0x81,0x00,0x05,0x00,0x11,0x00,0x00,0x00,0x41,0x00,0x00,0x00, +0x40,0x00,0x00,0x00,0x2d,0x00,0x00,0x00,0x88,0x00,0x05,0x00, +0x11,0x00,0x00,0x00,0x42,0x00,0x00,0x00,0x3d,0x00,0x00,0x00, +0x41,0x00,0x00,0x00,0x83,0x00,0x05,0x00,0x11,0x00,0x00,0x00, +0x43,0x00,0x00,0x00,0x3d,0x00,0x00,0x00,0x42,0x00,0x00,0x00, +0x85,0x00,0x05,0x00,0x11,0x00,0x00,0x00,0x44,0x00,0x00,0x00, +0x3c,0x00,0x00,0x00,0x43,0x00,0x00,0x00,0x41,0x00,0x06,0x00, +0x26,0x00,0x00,0x00,0x45,0x00,0x00,0x00,0x38,0x00,0x00,0x00, +0x16,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x3e,0x00,0x03,0x00, +0x45,0x00,0x00,0x00,0x44,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x49,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x49,0x00,0x00,0x00, +0xfd,0x00,0x01,0x00,0x38,0x00,0x01,0x00, }; -const uint64_t gelu_f32_len = 1408; +const uint64_t gelu_f32_len = 1484; unsigned char get_rows_f16_data[] = { 0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, diff --git a/ggml-vulkan.cpp b/ggml-vulkan.cpp index bccc40bf5..b1e0006bb 100644 --- a/ggml-vulkan.cpp +++ b/ggml-vulkan.cpp @@ -2876,6 +2876,9 @@ static void ggml_vk_op_f32(vk_context * ctx, const ggml_tensor * src0, const ggm x_sz = ggml_nbytes(src0); d_sz = ggml_nbytes(dst); + if (extra_src0->offset + x_sz >= d_X->size) { + x_sz = VK_WHOLE_SIZE; + } if (extra->offset + d_sz >= d_D->size) { d_sz = VK_WHOLE_SIZE; } @@ -2911,12 +2914,16 @@ static void ggml_vk_op_f32(vk_context * ctx, const ggml_tensor * src0, const ggm break; } - x_sz *= ne02 * ne03; - if (y_sz != VK_WHOLE_SIZE) { - y_sz *= ne12 * ne13; - } if (op != GGML_OP_CPY) { - d_sz *= ne02 * ne03; + if (x_sz != VK_WHOLE_SIZE) { + x_sz *= ne02 * ne03; + } + if (y_sz != VK_WHOLE_SIZE) { + y_sz *= ne12 * ne13; + } + if (d_sz != VK_WHOLE_SIZE) { + d_sz *= ne02 * ne03; + } } if (!use_src1 && op == GGML_OP_SOFT_MAX) { diff --git a/ggml_vk_generate_shaders.py b/ggml_vk_generate_shaders.py index 6b1b82bf3..67981a751 100644 --- a/ggml_vk_generate_shaders.py +++ b/ggml_vk_generate_shaders.py @@ -1689,7 +1689,8 @@ void main() { } const float xi = float(data_a[i]); - data_d[i] = D_TYPE(0.5f*xi*(1.0f + tanh(SQRT_2_OVER_PI*xi*(1.0f + GELU_COEF_A*xi*xi)))); + const float val = SQRT_2_OVER_PI*xi*(1.0f + GELU_COEF_A*xi*xi); + data_d[i] = D_TYPE(0.5f*xi*(2.0f - 2.0f / (exp(2 * val) + 1))); } """ From 128dcbd3c9c4b12f42b560a4430427d7b2828628 Mon Sep 17 00:00:00 2001 From: Neo Zhang Jianyu Date: Fri, 2 Feb 2024 03:48:53 +0800 Subject: [PATCH 238/812] add --no-mmap in llama-bench (#5257) * add --no-mmap, show sycl backend * fix conflict * fix code format, change print for --no-mmap * ren no_mmap to mmap, show mmap when not default value in printer * update guide for mmap * mv position to reduce model reload --- README-sycl.md | 2 +- examples/llama-bench/llama-bench.cpp | 60 +++++++++++++++++++++++++--- ggml-sycl.cpp | 34 +++++++++++++++- ggml-sycl.h | 3 +- 4 files changed, 89 insertions(+), 10 deletions(-) diff --git a/README-sycl.md b/README-sycl.md index 2b2cfe03a..b8ee212b8 100644 --- a/README-sycl.md +++ b/README-sycl.md @@ -405,7 +405,7 @@ Using device **0** (Intel(R) Arc(TM) A770 Graphics) as main device llama.cpp use mmap as default way to read model file and copy to GPU. In some system, memcpy will be abnormal and block. - Solution: add **--no-mmap**. + Solution: add **--no-mmap** or **--mmap 0**. ## Q&A diff --git a/examples/llama-bench/llama-bench.cpp b/examples/llama-bench/llama-bench.cpp index c5a6f744e..e36c061a2 100644 --- a/examples/llama-bench/llama-bench.cpp +++ b/examples/llama-bench/llama-bench.cpp @@ -20,6 +20,7 @@ #include "llama.h" #include "common.h" #include "ggml-cuda.h" +#include "ggml-sycl.h" // utils static uint64_t get_time_ns() { @@ -120,6 +121,22 @@ static std::string get_gpu_info() { id += "/"; } } +#endif +#ifdef GGML_USE_SYCL + int device_list[GGML_SYCL_MAX_DEVICES]; + ggml_sycl_get_gpu_list(device_list, GGML_SYCL_MAX_DEVICES); + + for (int i = 0; i < GGML_SYCL_MAX_DEVICES; i++) { + if (device_list[i] >0 ){ + char buf[128]; + ggml_sycl_get_device_description(i, buf, sizeof(buf)); + id += buf; + id += "/"; + } + } + if (id.length() >2 ) { + id.pop_back(); + } #endif // TODO: other backends return id; @@ -161,6 +178,7 @@ struct cmd_params { std::vector no_kv_offload; std::vector mul_mat_q; std::vector> tensor_split; + std::vector use_mmap; int reps; bool verbose; output_formats output_format; @@ -180,6 +198,7 @@ static const cmd_params cmd_params_defaults = { /* no_kv_offload */ {false}, /* mul_mat_q */ {true}, /* tensor_split */ {std::vector(llama_max_devices(), 0.0f)}, + /* use_mmap */ {true}, /* reps */ 5, /* verbose */ false, /* output_format */ MARKDOWN @@ -201,6 +220,7 @@ static void print_usage(int /* argc */, char ** argv) { printf(" -sm, --split-mode (default: %s)\n", join(transform_to_str(cmd_params_defaults.split_mode, split_mode_str), ",").c_str()); printf(" -mg, --main-gpu (default: %s)\n", join(cmd_params_defaults.main_gpu, ",").c_str()); printf(" -nkvo, --no-kv-offload <0|1> (default: %s)\n", join(cmd_params_defaults.no_kv_offload, ",").c_str()); + printf(" -mmp, --mmap <0|1> (default: %s)\n", join(cmd_params_defaults.use_mmap, ",").c_str()); printf(" -mmq, --mul-mat-q <0|1> (default: %s)\n", join(cmd_params_defaults.mul_mat_q, ",").c_str()); printf(" -ts, --tensor_split (default: 0)\n"); printf(" -r, --repetitions (default: %d)\n", cmd_params_defaults.reps); @@ -370,6 +390,13 @@ static cmd_params parse_cmd_params(int argc, char ** argv) { } auto p = split(argv[i], split_delim); params.mul_mat_q.insert(params.mul_mat_q.end(), p.begin(), p.end()); + } else if (arg == "-mmp" || arg == "--mmap") { + if (++i >= argc) { + invalid_param = true; + break; + } + auto p = split(argv[i], split_delim); + params.use_mmap.insert(params.use_mmap.end(), p.begin(), p.end()); } else if (arg == "-ts" || arg == "--tensor-split") { if (++i >= argc) { invalid_param = true; @@ -441,6 +468,7 @@ static cmd_params parse_cmd_params(int argc, char ** argv) { if (params.no_kv_offload.empty()){ params.no_kv_offload = cmd_params_defaults.no_kv_offload; } if (params.mul_mat_q.empty()) { params.mul_mat_q = cmd_params_defaults.mul_mat_q; } if (params.tensor_split.empty()) { params.tensor_split = cmd_params_defaults.tensor_split; } + if (params.use_mmap.empty()) { params.use_mmap = cmd_params_defaults.use_mmap; } if (params.n_threads.empty()) { params.n_threads = cmd_params_defaults.n_threads; } return params; @@ -460,6 +488,7 @@ struct cmd_params_instance { bool no_kv_offload; bool mul_mat_q; std::vector tensor_split; + bool use_mmap; llama_model_params to_llama_mparams() const { llama_model_params mparams = llama_model_default_params(); @@ -468,6 +497,7 @@ struct cmd_params_instance { mparams.split_mode = split_mode; mparams.main_gpu = main_gpu; mparams.tensor_split = tensor_split.data(); + mparams.use_mmap = use_mmap; return mparams; } @@ -477,6 +507,7 @@ struct cmd_params_instance { n_gpu_layers == other.n_gpu_layers && split_mode == other.split_mode && main_gpu == other.main_gpu && + use_mmap == other.use_mmap && tensor_split == other.tensor_split; } @@ -503,6 +534,7 @@ static std::vector get_cmd_params_instances(const cmd_param for (const auto & sm : params.split_mode) for (const auto & mg : params.main_gpu) for (const auto & ts : params.tensor_split) + for (const auto & mmp : params.use_mmap) for (const auto & nb : params.n_batch) for (const auto & tk : params.type_k) for (const auto & tv : params.type_v) @@ -527,6 +559,7 @@ static std::vector get_cmd_params_instances(const cmd_param /* .no_kv_offload= */ nkvo, /* .mul_mat_q = */ mmq, /* .tensor_split = */ ts, + /* .use_mmap = */ mmp, }; instances.push_back(instance); } @@ -549,6 +582,7 @@ static std::vector get_cmd_params_instances(const cmd_param /* .no_kv_offload= */ nkvo, /* .mul_mat_q = */ mmq, /* .tensor_split = */ ts, + /* .use_mmap = */ mmp, }; instances.push_back(instance); } @@ -565,6 +599,7 @@ struct test { static const bool vulkan; static const bool kompute; static const bool metal; + static const bool sycl; static const bool gpu_blas; static const bool blas; static const std::string cpu_info; @@ -583,6 +618,7 @@ struct test { bool no_kv_offload; bool mul_mat_q; std::vector tensor_split; + bool use_mmap; int n_prompt; int n_gen; std::string test_time; @@ -605,6 +641,7 @@ struct test { no_kv_offload = inst.no_kv_offload; mul_mat_q = inst.mul_mat_q; tensor_split = inst.tensor_split; + use_mmap = inst.use_mmap; n_prompt = inst.n_prompt; n_gen = inst.n_gen; // RFC 3339 date-time format @@ -654,25 +691,29 @@ struct test { if (metal) { return "Metal"; } + if (sycl) { + return GGML_SYCL_NAME; + } if (gpu_blas) { return "GPU BLAS"; } if (blas) { return "BLAS"; } + return "CPU"; } static const std::vector & get_fields() { static const std::vector fields = { "build_commit", "build_number", - "cuda", "opencl", "vulkan", "kompute", "metal", "gpu_blas", "blas", + "cuda", "opencl", "vulkan", "kompute", "metal", "sycl", "gpu_blas", "blas", "cpu_info", "gpu_info", "model_filename", "model_type", "model_size", "model_n_params", "n_batch", "n_threads", "type_k", "type_v", "n_gpu_layers", "split_mode", "main_gpu", "no_kv_offload", - "mul_mat_q", "tensor_split", + "mul_mat_q", "tensor_split", "use_mmap", "n_prompt", "n_gen", "test_time", "avg_ns", "stddev_ns", "avg_ts", "stddev_ts" @@ -691,8 +732,8 @@ struct test { return INT; } if (field == "cuda" || field == "opencl" || field == "vulkan" || field == "kompute" || field == "metal" || - field == "gpu_blas" || field == "blas" || field == "f16_kv" || field == "no_kv_offload" || - field == "mul_mat_q") { + field == "gpu_blas" || field == "blas" || field == "sycl" ||field == "f16_kv" || field == "no_kv_offload" || + field == "mul_mat_q" || field == "use_mmap") { return BOOL; } if (field == "avg_ts" || field == "stddev_ts") { @@ -720,13 +761,13 @@ struct test { std::vector values = { build_commit, std::to_string(build_number), std::to_string(cuda), std::to_string(opencl), std::to_string(vulkan), std::to_string(vulkan), - std::to_string(metal), std::to_string(gpu_blas), std::to_string(blas), + std::to_string(metal), std::to_string(sycl), std::to_string(gpu_blas), std::to_string(blas), cpu_info, gpu_info, model_filename, model_type, std::to_string(model_size), std::to_string(model_n_params), std::to_string(n_batch), std::to_string(n_threads), ggml_type_name(type_k), ggml_type_name(type_v), std::to_string(n_gpu_layers), split_mode_str(split_mode), std::to_string(main_gpu), std::to_string(no_kv_offload), - std::to_string(mul_mat_q), tensor_split_str, + std::to_string(mul_mat_q), tensor_split_str, std::to_string(use_mmap), std::to_string(n_prompt), std::to_string(n_gen), test_time, std::to_string(avg_ns()), std::to_string(stdev_ns()), std::to_string(avg_ts()), std::to_string(stdev_ts()) @@ -753,6 +794,7 @@ const bool test::kompute = !!ggml_cpu_has_kompute(); const bool test::metal = !!ggml_cpu_has_metal(); const bool test::gpu_blas = !!ggml_cpu_has_gpublas(); const bool test::blas = !!ggml_cpu_has_blas(); +const bool test::sycl = !!ggml_cpu_has_sycl(); const std::string test::cpu_info = get_cpu_info(); const std::string test::gpu_info = get_gpu_info(); @@ -895,6 +937,9 @@ struct markdown_printer : public printer { if (field == "no_kv_offload") { return "nkvo"; } + if (field == "use_mmap") { + return "mmap"; + } if (field == "tensor_split") { return "ts"; } @@ -938,6 +983,9 @@ struct markdown_printer : public printer { if (params.tensor_split.size() > 1 || params.tensor_split != cmd_params_defaults.tensor_split) { fields.push_back("tensor_split"); } + if (params.use_mmap.size() > 1 || params.use_mmap != cmd_params_defaults.use_mmap) { + fields.push_back("use_mmap"); + } fields.push_back("test"); fields.push_back("t/s"); diff --git a/ggml-sycl.cpp b/ggml-sycl.cpp index 1cc55ef52..e8ba48353 100644 --- a/ggml-sycl.cpp +++ b/ggml-sycl.cpp @@ -2928,7 +2928,6 @@ void ggml_sycl_set_main_device(int main_device); void ggml_sycl_set_mul_mat_q(bool mul_mat_q); void ggml_sycl_set_scratch_size(size_t scratch_size); void ggml_sycl_free_scratch(void); -int ggml_sycl_get_device_count(void); void ggml_sycl_get_device_description(int device, char * description, size_t description_size); bool ggml_backend_is_sycl(ggml_backend_t backend); int ggml_backend_sycl_get_device(ggml_backend_t backend); @@ -14493,6 +14492,37 @@ bool ggml_sycl_compute_forward(struct ggml_compute_params * params, struct ggml_ return true; } +GGML_API GGML_CALL void ggml_sycl_get_gpu_list(int *id_list, int max_len) try { + int max_compute_units = -1; + for(int i=0;i Date: Thu, 1 Feb 2024 23:20:13 -0800 Subject: [PATCH 239/812] llama : fix memory leak in llama_batch_free (#5252) The llama_batch_init allocates memory for a fixed number of tokens. However, the llama_batch_free only frees memory for the number of tokens that were added to the batch. This change-set uses a null terminated array for the batch seq_id, and frees all the elements until the nullptr is reached. This change-set also changes the name of the first parameter from `n_tokens` to `n_tokens_alloc` to more clearly indicate that this value is the number of tokens allocated to the batch, not the number of tokens in the batch. --- llama.cpp | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/llama.cpp b/llama.cpp index e8f44c2cb..6bf7f9efb 100644 --- a/llama.cpp +++ b/llama.cpp @@ -11377,22 +11377,24 @@ struct llama_batch llama_batch_get_one( }; } -struct llama_batch llama_batch_init(int32_t n_tokens, int32_t embd, int32_t n_seq_max) { +struct llama_batch llama_batch_init(int32_t n_tokens_alloc, int32_t embd, int32_t n_seq_max) { llama_batch batch = { 0, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, 0, 0, 0, }; if (embd) { - batch.embd = (float *) malloc(sizeof(float) * n_tokens * embd); + batch.embd = (float *) malloc(sizeof(float) * n_tokens_alloc * embd); } else { - batch.token = (llama_token *) malloc(sizeof(llama_token) * n_tokens); + batch.token = (llama_token *) malloc(sizeof(llama_token) * n_tokens_alloc); } - batch.pos = (llama_pos *) malloc(sizeof(llama_pos) * n_tokens); - batch.n_seq_id = (int32_t *) malloc(sizeof(int32_t) * n_tokens); - batch.seq_id = (llama_seq_id **) malloc(sizeof(llama_seq_id *) * n_tokens); - for (int i = 0; i < n_tokens; ++i) { + batch.pos = (llama_pos *) malloc(sizeof(llama_pos) * n_tokens_alloc); + batch.n_seq_id = (int32_t *) malloc(sizeof(int32_t) * n_tokens_alloc); + batch.seq_id = (llama_seq_id **) malloc(sizeof(llama_seq_id *) * (n_tokens_alloc + 1)); + for (int i = 0; i < n_tokens_alloc; ++i) { batch.seq_id[i] = (llama_seq_id *) malloc(sizeof(llama_seq_id) * n_seq_max); } - batch.logits = (int8_t *) malloc(sizeof(int8_t) * n_tokens); + batch.seq_id[n_tokens_alloc] = nullptr; + + batch.logits = (int8_t *) malloc(sizeof(int8_t) * n_tokens_alloc); return batch; } @@ -11403,7 +11405,7 @@ void llama_batch_free(struct llama_batch batch) { if (batch.pos) free(batch.pos); if (batch.n_seq_id) free(batch.n_seq_id); if (batch.seq_id) { - for (int i = 0; i < batch.n_tokens; ++i) { + for (int i = 0; batch.seq_id[i] != nullptr; ++i) { free(batch.seq_id[i]); } free(batch.seq_id); From af3ba5d94627d337e32a95129e31a3064c459f6b Mon Sep 17 00:00:00 2001 From: Neo Zhang Jianyu Date: Fri, 2 Feb 2024 15:53:27 +0800 Subject: [PATCH 240/812] [SYCL] update guide of SYCL backend (#5254) * update guide for make installation, memory, gguf model link, rm todo for windows build * add vs install requirement * update for gpu device check * update help of llama-bench * fix grammer issues --- README-sycl.md | 64 +++++++++++++++++++++++++++----- examples/llama-bench/README.md | 34 ++++++++++------- examples/sycl/win-run-llama2.bat | 2 +- 3 files changed, 77 insertions(+), 23 deletions(-) diff --git a/README-sycl.md b/README-sycl.md index b8ee212b8..f7edc1c3e 100644 --- a/README-sycl.md +++ b/README-sycl.md @@ -42,6 +42,8 @@ For Intel CPU, recommend to use llama.cpp for X86 (Intel MKL building). ## Intel GPU +### Verified + |Intel GPU| Status | Verified Model| |-|-|-| |Intel Data Center Max Series| Support| Max 1550| @@ -50,6 +52,17 @@ For Intel CPU, recommend to use llama.cpp for X86 (Intel MKL building). |Intel built-in Arc GPU| Support| built-in Arc GPU in Meteor Lake| |Intel iGPU| Support| iGPU in i5-1250P, i7-1165G7| +Note: If the EUs (Execution Unit) in iGPU is less than 80, the inference speed will be too slow to use. + +### Memory + +The memory is a limitation to run LLM on GPUs. + +When run llama.cpp, there is print log to show the applied memory on GPU. You could know how much memory to be used in your case. Like `llm_load_tensors: buffer size = 3577.56 MiB`. + +For iGPU, please make sure the shared memory from host memory is enough. For llama-2-7b.Q4_0, recommend the host memory is 8GB+. + +For dGPU, please make sure the device memory is enough. For llama-2-7b.Q4_0, recommend the device memory is 4GB+. ## Linux @@ -105,7 +118,7 @@ source /opt/intel/oneapi/setvars.sh sycl-ls ``` -There should be one or more level-zero devices. Like **[ext_oneapi_level_zero:gpu:0]**. +There should be one or more level-zero devices. Please confirm that at least one GPU is present, like **[ext_oneapi_level_zero:gpu:0]**. Output (example): ``` @@ -152,6 +165,8 @@ Note: 1. Put model file to folder **models** +You could download [llama-2-7b.Q4_0.gguf](https://huggingface.co/TheBloke/Llama-2-7B-GGUF/blob/main/llama-2-7b.Q4_0.gguf) as example. + 2. Enable oneAPI running environment ``` @@ -223,7 +238,13 @@ Using device **0** (Intel(R) Arc(TM) A770 Graphics) as main device Please install Intel GPU driver by official guide: [Install GPU Drivers](https://www.intel.com/content/www/us/en/products/docs/discrete-gpus/arc/software/drivers.html). -2. Install Intel® oneAPI Base toolkit. +Note: **The driver is mandatory for compute function**. + +2. Install Visual Studio. + +Please install [Visual Studio](https://visualstudio.microsoft.com/) which impact oneAPI environment enabling in Windows. + +3. Install Intel® oneAPI Base toolkit. a. Please follow the procedure in [Get the Intel® oneAPI Base Toolkit ](https://www.intel.com/content/www/us/en/developer/tools/oneapi/base-toolkit.html). @@ -252,7 +273,7 @@ In oneAPI command line: sycl-ls ``` -There should be one or more level-zero devices. Like **[ext_oneapi_level_zero:gpu:0]**. +There should be one or more level-zero devices. Please confirm that at least one GPU is present, like **[ext_oneapi_level_zero:gpu:0]**. Output (example): ``` @@ -260,15 +281,21 @@ Output (example): [opencl:cpu:1] Intel(R) OpenCL, 11th Gen Intel(R) Core(TM) i7-1185G7 @ 3.00GHz OpenCL 3.0 (Build 0) [2023.16.10.0.17_160000] [opencl:gpu:2] Intel(R) OpenCL Graphics, Intel(R) Iris(R) Xe Graphics OpenCL 3.0 NEO [31.0.101.5186] [ext_oneapi_level_zero:gpu:0] Intel(R) Level-Zero, Intel(R) Iris(R) Xe Graphics 1.3 [1.3.28044] - ``` -3. Install cmake & make +4. Install cmake & make -a. Download & install cmake for windows: https://cmake.org/download/ +a. Download & install cmake for Windows: https://cmake.org/download/ -b. Download & install make for windows provided by mingw-w64: https://www.mingw-w64.org/downloads/ +b. Download & install make for Windows provided by mingw-w64 +- Download binary package for Windows in https://github.com/niXman/mingw-builds-binaries/releases. + + Like [x86_64-13.2.0-release-win32-seh-msvcrt-rt_v11-rev1.7z](https://github.com/niXman/mingw-builds-binaries/releases/download/13.2.0-rt_v11-rev1/x86_64-13.2.0-release-win32-seh-msvcrt-rt_v11-rev1.7z). + +- Unzip the binary package. In the **bin** sub-folder and rename **xxx-make.exe** to **make.exe**. + +- Add the **bin** folder path in the Windows system PATH environment. ### Build locally: @@ -309,6 +336,8 @@ Note: 1. Put model file to folder **models** +You could download [llama-2-7b.Q4_0.gguf](https://huggingface.co/TheBloke/Llama-2-7B-GGUF/blob/main/llama-2-7b.Q4_0.gguf) as example. + 2. Enable oneAPI running environment - In Search, input 'oneAPI'. @@ -419,8 +448,25 @@ Using device **0** (Intel(R) Arc(TM) A770 Graphics) as main device Miss to enable oneAPI running environment. +- Meet compile error. + + Remove folder **build** and try again. + +- I can **not** see **[ext_oneapi_level_zero:gpu:0]** afer install GPU driver in Linux. + + Please run **sudo sycl-ls**. + + If you see it in result, please add video/render group to your ID: + + ``` + sudo usermod -aG render username + sudo usermod -aG video username + ``` + + Then **relogin**. + + If you do not see it, please check the installation GPU steps again. + ## Todo -- Support to build in Windows. - - Support multiple cards. diff --git a/examples/llama-bench/README.md b/examples/llama-bench/README.md index d02824bfa..374e40a7d 100644 --- a/examples/llama-bench/README.md +++ b/examples/llama-bench/README.md @@ -23,19 +23,23 @@ usage: ./llama-bench [options] options: -h, --help - -m, --model (default: models/7B/ggml-model-q4_0.gguf) - -p, --n-prompt (default: 512) - -n, --n-gen (default: 128) - -b, --batch-size (default: 512) - --memory-f32 <0|1> (default: 0) - -t, --threads (default: 16) - -ngl N, --n-gpu-layers (default: 99) - -mg i, --main-gpu (default: 0) - -mmq, --mul-mat-q <0|1> (default: 1) - -ts, --tensor_split - -r, --repetitions (default: 5) - -o, --output (default: md) - -v, --verbose (default: 0) + -m, --model (default: models/7B/ggml-model-q4_0.gguf) + -p, --n-prompt (default: 512) + -n, --n-gen (default: 128) + -b, --batch-size (default: 512) + -ctk , --cache-type-k (default: f16) + -ctv , --cache-type-v (default: f16) + -t, --threads (default: 112) + -ngl, --n-gpu-layers (default: 99) + -sm, --split-mode (default: layer) + -mg, --main-gpu (default: 0) + -nkvo, --no-kv-offload <0|1> (default: 0) + -mmp, --mmap <0|1> (default: 1) + -mmq, --mul-mat-q <0|1> (default: 1) + -ts, --tensor_split (default: 0) + -r, --repetitions (default: 5) + -o, --output (default: md) + -v, --verbose (default: 0) Multiple values can be given for each parameter by separating them with ',' or by specifying the parameter multiple times. ``` @@ -51,6 +55,10 @@ Each test is repeated the number of times given by `-r`, and the results are ave For a description of the other options, see the [main example](../main/README.md). +Note: + +- When using SYCL backend, there would be hang issue in some cases. Please set `--mmp 0`. + ## Examples ### Text generation with different models diff --git a/examples/sycl/win-run-llama2.bat b/examples/sycl/win-run-llama2.bat index 28d935541..cf621c675 100644 --- a/examples/sycl/win-run-llama2.bat +++ b/examples/sycl/win-run-llama2.bat @@ -2,7 +2,7 @@ :: Copyright (C) 2024 Intel Corporation :: SPDX-License-Identifier: MIT -INPUT2="Building a website can be done in 10 simple steps:\nStep 1:" +set INPUT2="Building a website can be done in 10 simple steps:\nStep 1:" @call "C:\Program Files (x86)\Intel\oneAPI\setvars.bat" intel64 --force From e805f0fa9951081ce0a86378a7aa52b6f636b82d Mon Sep 17 00:00:00 2001 From: "Meng, Hengyu" Date: Fri, 2 Feb 2024 15:54:14 +0800 Subject: [PATCH 241/812] [SYCL] get MAX_MEM_ALLOC from device property (#5270) * get max alloc size from device prop * fix macro typo --- ggml-sycl.cpp | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/ggml-sycl.cpp b/ggml-sycl.cpp index e8ba48353..4ee2eed38 100644 --- a/ggml-sycl.cpp +++ b/ggml-sycl.cpp @@ -337,6 +337,7 @@ namespace dpct } size_t get_global_mem_size() const { return _global_mem_size; } size_t get_local_mem_size() const { return _local_mem_size; } + size_t get_max_mem_alloc_size() const { return _max_mem_alloc_size; } /// Returns the maximum clock rate of device's global memory in kHz. If /// compiler does not support this API then returns default value 3200000 kHz. unsigned int get_memory_clock_rate() const { return _memory_clock_rate; } @@ -398,6 +399,10 @@ namespace dpct { _local_mem_size = local_mem_size; } + void set_max_mem_alloc_size(size_t max_mem_alloc_size) + { + _max_mem_alloc_size = max_mem_alloc_size; + } void set_max_work_group_size(int max_work_group_size) { _max_work_group_size = max_work_group_size; @@ -465,6 +470,7 @@ namespace dpct int _max_register_size_per_work_group; size_t _global_mem_size; size_t _local_mem_size; + size_t _max_mem_alloc_size; size_t _max_nd_range_size[3]; int _max_nd_range_size_i[3]; uint32_t _device_id; @@ -516,6 +522,7 @@ namespace dpct dev.get_info()); prop.set_global_mem_size(dev.get_info()); prop.set_local_mem_size(dev.get_info()); + prop.set_max_mem_alloc_size(dev.get_info()); #if (defined(SYCL_EXT_INTEL_DEVICE_INFO) && SYCL_EXT_INTEL_DEVICE_INFO >= 6) if (dev.has(sycl::aspect::ext_intel_memory_clock_rate)) @@ -644,6 +651,11 @@ namespace dpct return get_device_info().get_global_mem_size(); } + size_t get_max_mem_alloc_size() const + { + return get_device_info().get_max_mem_alloc_size(); + } + /// Get the number of bytes of free and total memory on the SYCL device. /// \param [out] free_memory The number of bytes of free memory on the SYCL device. /// \param [out] total_memory The number of bytes of total memory on the SYCL device. @@ -11311,10 +11323,10 @@ void ggml_init_sycl() try { GGML_ASSERT(g_all_sycl_device_count <= GGML_SYCL_MAX_DEVICES); int64_t total_vram = 0; -#if defined(GGML_SYCL_FP16) - fprintf(stderr, "%s: GGML_SYCL_FP16: yes\n", __func__); +#if defined(GGML_SYCL_F16) + fprintf(stderr, "%s: GGML_SYCL_F16: yes\n", __func__); #else - fprintf(stderr, "%s: GGML_SYCL_FP16: no\n", __func__); + fprintf(stderr, "%s: GGML_SYCL_F16: no\n", __func__); #endif @@ -14788,6 +14800,12 @@ static size_t ggml_backend_sycl_buffer_type_get_alignment(ggml_backend_buffer_ty UNUSED(buft); } +static size_t ggml_backend_sycl_buffer_type_get_max_size(ggml_backend_buffer_type_t buft) { + return dpct::get_current_device().get_max_mem_alloc_size(); + + UNUSED(buft); +} + static size_t ggml_backend_sycl_buffer_type_get_alloc_size(ggml_backend_buffer_type_t buft, const ggml_tensor * tensor) { int64_t row_low = 0; int64_t row_high = ggml_nrows(tensor); @@ -14818,7 +14836,7 @@ static ggml_backend_buffer_type_i ggml_backend_sycl_buffer_type_interface = { /* .get_name = */ ggml_backend_sycl_buffer_type_name, /* .alloc_buffer = */ ggml_backend_sycl_buffer_type_alloc_buffer, /* .get_alignment = */ ggml_backend_sycl_buffer_type_get_alignment, - /* .get_max_size = */ NULL, // TODO: return device.maxBufferLength + /* .get_max_size = */ ggml_backend_sycl_buffer_type_get_max_size, /* .get_alloc_size = */ ggml_backend_sycl_buffer_type_get_alloc_size, /* .supports_backend = */ ggml_backend_sycl_buffer_type_supports_backend, /* .is_host = */ nullptr, From 6b91b1e0a92ac2e4e269eec6361ca53a61ced6c6 Mon Sep 17 00:00:00 2001 From: Xuan Son Nguyen Date: Fri, 2 Feb 2024 08:56:31 +0100 Subject: [PATCH 242/812] docker : add build for SYCL, Vulkan + update readme (#5228) * add vulkan dockerfile * intel dockerfile: compile sycl by default * fix vulkan dockerfile * add docs for vulkan * docs: sycl build in docker * docs: remove trailing spaces * docs: sycl: add docker section * docs: clarify install vulkan SDK outside docker * sycl: use intel/oneapi-basekit docker image * docs: correct TOC * docs: correct docker image for Intel oneMKL --- .devops/main-intel.Dockerfile | 16 ++--- .devops/main-vulkan.Dockerfile | 29 +++++++++ .devops/server-intel.Dockerfile | 15 +++-- .devops/server-vulkan.Dockerfile | 29 +++++++++ README-sycl.md | 102 +++++++++++++++++++------------ README.md | 64 ++++++++++++++----- 6 files changed, 188 insertions(+), 67 deletions(-) create mode 100644 .devops/main-vulkan.Dockerfile create mode 100644 .devops/server-vulkan.Dockerfile diff --git a/.devops/main-intel.Dockerfile b/.devops/main-intel.Dockerfile index e1e6acc24..572e5d8ea 100644 --- a/.devops/main-intel.Dockerfile +++ b/.devops/main-intel.Dockerfile @@ -1,8 +1,8 @@ ARG ONEAPI_VERSION=2024.0.1-devel-ubuntu22.04 -ARG UBUNTU_VERSION=22.04 -FROM intel/hpckit:$ONEAPI_VERSION as build +FROM intel/oneapi-basekit:$ONEAPI_VERSION as build +ARG LLAMA_SYCL_F16=OFF RUN apt-get update && \ apt-get install -y git @@ -10,16 +10,18 @@ WORKDIR /app COPY . . -# for some reasons, "-DLLAMA_BLAS=ON -DLLAMA_BLAS_VENDOR=Intel10_64lp -DLLAMA_NATIVE=ON" give worse performance RUN mkdir build && \ cd build && \ - cmake .. -DCMAKE_C_COMPILER=icx -DCMAKE_CXX_COMPILER=icpx && \ - cmake --build . --config Release --target main server + if [ "${LLAMA_SYCL_F16}" = "ON" ]; then \ + echo "LLAMA_SYCL_F16 is set" && \ + export OPT_SYCL_F16="-DLLAMA_SYCL_F16=ON"; \ + fi && \ + cmake .. -DLLAMA_SYCL=ON -DCMAKE_C_COMPILER=icx -DCMAKE_CXX_COMPILER=icpx ${OPT_SYCL_F16} && \ + cmake --build . --config Release --target main -FROM ubuntu:$UBUNTU_VERSION as runtime +FROM intel/oneapi-basekit:$ONEAPI_VERSION as runtime COPY --from=build /app/build/bin/main /main -COPY --from=build /app/build/bin/server /server ENV LC_ALL=C.utf8 diff --git a/.devops/main-vulkan.Dockerfile b/.devops/main-vulkan.Dockerfile new file mode 100644 index 000000000..bca460365 --- /dev/null +++ b/.devops/main-vulkan.Dockerfile @@ -0,0 +1,29 @@ +ARG UBUNTU_VERSION=jammy + +FROM ubuntu:$UBUNTU_VERSION as build + +# Install build tools +RUN apt update && apt install -y git build-essential cmake wget + +# Install Vulkan SDK +RUN wget -qO - https://packages.lunarg.com/lunarg-signing-key-pub.asc | apt-key add - && \ + wget -qO /etc/apt/sources.list.d/lunarg-vulkan-jammy.list https://packages.lunarg.com/vulkan/lunarg-vulkan-jammy.list && \ + apt update -y && \ + apt-get install -y vulkan-sdk + +# Build it +WORKDIR /app +COPY . . +RUN mkdir build && \ + cd build && \ + cmake .. -DLLAMA_VULKAN=1 && \ + cmake --build . --config Release --target main + +# Clean up +WORKDIR / +RUN cp /app/build/bin/main /main && \ + rm -rf /app + +ENV LC_ALL=C.utf8 + +ENTRYPOINT [ "/main" ] diff --git a/.devops/server-intel.Dockerfile b/.devops/server-intel.Dockerfile index e343d278c..312f2df80 100644 --- a/.devops/server-intel.Dockerfile +++ b/.devops/server-intel.Dockerfile @@ -1,8 +1,8 @@ ARG ONEAPI_VERSION=2024.0.1-devel-ubuntu22.04 -ARG UBUNTU_VERSION=22.04 -FROM intel/hpckit:$ONEAPI_VERSION as build +FROM intel/oneapi-basekit:$ONEAPI_VERSION as build +ARG LLAMA_SYCL_F16=OFF RUN apt-get update && \ apt-get install -y git @@ -10,13 +10,16 @@ WORKDIR /app COPY . . -# for some reasons, "-DLLAMA_BLAS=ON -DLLAMA_BLAS_VENDOR=Intel10_64lp -DLLAMA_NATIVE=ON" give worse performance RUN mkdir build && \ cd build && \ - cmake .. -DCMAKE_C_COMPILER=icx -DCMAKE_CXX_COMPILER=icpx && \ - cmake --build . --config Release --target main server + if [ "${LLAMA_SYCL_F16}" = "ON" ]; then \ + echo "LLAMA_SYCL_F16 is set" && \ + export OPT_SYCL_F16="-DLLAMA_SYCL_F16=ON"; \ + fi && \ + cmake .. -DLLAMA_SYCL=ON -DCMAKE_C_COMPILER=icx -DCMAKE_CXX_COMPILER=icpx ${OPT_SYCL_F16} && \ + cmake --build . --config Release --target server -FROM ubuntu:$UBUNTU_VERSION as runtime +FROM intel/oneapi-basekit:$ONEAPI_VERSION as runtime COPY --from=build /app/build/bin/server /server diff --git a/.devops/server-vulkan.Dockerfile b/.devops/server-vulkan.Dockerfile new file mode 100644 index 000000000..e0add6fc3 --- /dev/null +++ b/.devops/server-vulkan.Dockerfile @@ -0,0 +1,29 @@ +ARG UBUNTU_VERSION=jammy + +FROM ubuntu:$UBUNTU_VERSION as build + +# Install build tools +RUN apt update && apt install -y git build-essential cmake wget + +# Install Vulkan SDK +RUN wget -qO - https://packages.lunarg.com/lunarg-signing-key-pub.asc | apt-key add - && \ + wget -qO /etc/apt/sources.list.d/lunarg-vulkan-jammy.list https://packages.lunarg.com/vulkan/lunarg-vulkan-jammy.list && \ + apt update -y && \ + apt-get install -y vulkan-sdk + +# Build it +WORKDIR /app +COPY . . +RUN mkdir build && \ + cd build && \ + cmake .. -DLLAMA_VULKAN=1 && \ + cmake --build . --config Release --target server + +# Clean up +WORKDIR / +RUN cp /app/build/bin/server /server && \ + rm -rf /app + +ENV LC_ALL=C.utf8 + +ENTRYPOINT [ "/server" ] diff --git a/README-sycl.md b/README-sycl.md index f7edc1c3e..7aa4274a9 100644 --- a/README-sycl.md +++ b/README-sycl.md @@ -1,22 +1,15 @@ # llama.cpp for SYCL -[Background](#background) - -[OS](#os) - -[Intel GPU](#intel-gpu) - -[Linux](#linux) - -[Windows](#windows) - -[Environment Variable](#environment-variable) - -[Known Issue](#known-issue) - -[Q&A](#q&a) - -[Todo](#todo) +- [Background](#background) +- [OS](#os) +- [Intel GPU](#intel-gpu) +- [Docker](#docker) +- [Linux](#linux) +- [Windows](#windows) +- [Environment Variable](#environment-variable) +- [Known Issue](#known-issue) +- [Q&A](#q&a) +- [Todo](#todo) ## Background @@ -36,7 +29,7 @@ For Intel CPU, recommend to use llama.cpp for X86 (Intel MKL building). |OS|Status|Verified| |-|-|-| -|Linux|Support|Ubuntu 22.04| +|Linux|Support|Ubuntu 22.04, Fedora Silverblue 39| |Windows|Support|Windows 11| @@ -50,7 +43,7 @@ For Intel CPU, recommend to use llama.cpp for X86 (Intel MKL building). |Intel Data Center Flex Series| Support| Flex 170| |Intel Arc Series| Support| Arc 770, 730M| |Intel built-in Arc GPU| Support| built-in Arc GPU in Meteor Lake| -|Intel iGPU| Support| iGPU in i5-1250P, i7-1165G7| +|Intel iGPU| Support| iGPU in i5-1250P, i7-1260P, i7-1165G7| Note: If the EUs (Execution Unit) in iGPU is less than 80, the inference speed will be too slow to use. @@ -64,6 +57,38 @@ For iGPU, please make sure the shared memory from host memory is enough. For lla For dGPU, please make sure the device memory is enough. For llama-2-7b.Q4_0, recommend the device memory is 4GB+. +## Docker + +Note: +- Only docker on Linux is tested. Docker on WSL may not work. +- You may need to install Intel GPU driver on the host machine (See the [Linux](#linux) section to know how to do that) + +### Build the image + +You can choose between **F16** and **F32** build. F16 is faster for long-prompt inference. + + +```sh +# For F16: +#docker build -t llama-cpp-sycl --build-arg="LLAMA_SYCL_F16=ON" -f .devops/main-intel.Dockerfile . + +# Or, for F32: +docker build -t llama-cpp-sycl -f .devops/main-intel.Dockerfile . + +# Note: you can also use the ".devops/main-server.Dockerfile", which compiles the "server" example +``` + +### Run + +```sh +# Firstly, find all the DRI cards: +ls -la /dev/dri +# Then, pick the card that you want to use. + +# For example with "/dev/dri/card1" +docker run -it --rm -v "$(pwd):/app:Z" --device /dev/dri/renderD128:/dev/dri/renderD128 --device /dev/dri/card1:/dev/dri/card1 llama-cpp-sycl -m "/app/models/YOUR_MODEL_FILE" -p "Building a website can be done in 10 simple steps:" -n 400 -e -ngl 33 +``` + ## Linux ### Setup Environment @@ -76,7 +101,7 @@ Note: for iGPU, please install the client GPU driver. b. Add user to group: video, render. -``` +```sh sudo usermod -aG render username sudo usermod -aG video username ``` @@ -85,7 +110,7 @@ Note: re-login to enable it. c. Check -``` +```sh sudo apt install clinfo sudo clinfo -l ``` @@ -103,7 +128,6 @@ Platform #0: Intel(R) OpenCL HD Graphics 2. Install Intel® oneAPI Base toolkit. - a. Please follow the procedure in [Get the Intel® oneAPI Base Toolkit ](https://www.intel.com/content/www/us/en/developer/tools/oneapi/base-toolkit.html). Recommend to install to default folder: **/opt/intel/oneapi**. @@ -112,7 +136,7 @@ Following guide use the default folder as example. If you use other folder, plea b. Check -``` +```sh source /opt/intel/oneapi/setvars.sh sycl-ls @@ -131,21 +155,25 @@ Output (example): 2. Build locally: -``` +Note: +- You can choose between **F16** and **F32** build. F16 is faster for long-prompt inference. +- By default, it will build for all binary files. It will take more time. To reduce the time, we recommend to build for **example/main** only. + +```sh mkdir -p build cd build source /opt/intel/oneapi/setvars.sh -#for FP16 -#cmake .. -DLLAMA_SYCL=ON -DCMAKE_C_COMPILER=icx -DCMAKE_CXX_COMPILER=icpx -DLLAMA_SYCL_F16=ON # faster for long-prompt inference +# For FP16: +#cmake .. -DLLAMA_SYCL=ON -DCMAKE_C_COMPILER=icx -DCMAKE_CXX_COMPILER=icpx -DLLAMA_SYCL_F16=ON -#for FP32 +# Or, for FP32: cmake .. -DLLAMA_SYCL=ON -DCMAKE_C_COMPILER=icx -DCMAKE_CXX_COMPILER=icpx -#build example/main only +# Build example/main only #cmake --build . --config Release --target main -#build all binary +# Or, build all binary cmake --build . --config Release -v cd .. @@ -153,14 +181,10 @@ cd .. or -``` +```sh ./examples/sycl/build.sh ``` -Note: - -- By default, it will build for all binary files. It will take more time. To reduce the time, we recommend to build for **example/main** only. - ### Run 1. Put model file to folder **models** @@ -177,10 +201,10 @@ source /opt/intel/oneapi/setvars.sh Run without parameter: -``` +```sh ./build/bin/ls-sycl-device -or +# or running the "main" executable and look at the output log: ./build/bin/main ``` @@ -209,13 +233,13 @@ found 4 SYCL devices: Set device ID = 0 by **GGML_SYCL_DEVICE=0** -``` +```sh GGML_SYCL_DEVICE=0 ./build/bin/main -m models/llama-2-7b.Q4_0.gguf -p "Building a website can be done in 10 simple steps:" -n 400 -e -ngl 33 ``` or run by script: -``` -./examples/sycl/run-llama2.sh +```sh +./examples/sycl/run_llama2.sh ``` Note: diff --git a/README.md b/README.md index e6ed1d429..af1f09fa0 100644 --- a/README.md +++ b/README.md @@ -393,28 +393,28 @@ Building the program with BLAS support may lead to some performance improvements Check [BLIS.md](docs/BLIS.md) for more information. +- #### SYCL + SYCL is a higher-level programming model to improve programming productivity on various hardware accelerators. + + llama.cpp based on SYCL is used to **support Intel GPU** (Data Center Max series, Flex series, Arc series, Built-in GPU and iGPU). + + For detailed info, please refer to [llama.cpp for SYCL](README-sycl.md). + - #### Intel oneMKL + Building through oneAPI compilers will make avx_vnni instruction set available for intel processors that do not support avx512 and avx512_vnni. Please note that this build config **does not support Intel GPU**. For Intel GPU support, please refer to [llama.cpp for SYCL](./README-sycl.md). + - Using manual oneAPI installation: By default, `LLAMA_BLAS_VENDOR` is set to `Generic`, so if you already sourced intel environment script and assign `-DLLAMA_BLAS=ON` in cmake, the mkl version of Blas will automatically been selected. Otherwise please install oneAPI and follow the below steps: ```bash mkdir build cd build - source /opt/intel/oneapi/setvars.sh # You can skip this step if in oneapi-runtime docker image, only required for manual installation + source /opt/intel/oneapi/setvars.sh # You can skip this step if in oneapi-basekit docker image, only required for manual installation cmake .. -DLLAMA_BLAS=ON -DLLAMA_BLAS_VENDOR=Intel10_64lp -DCMAKE_C_COMPILER=icx -DCMAKE_CXX_COMPILER=icpx -DLLAMA_NATIVE=ON cmake --build . --config Release ``` - Using oneAPI docker image: - If you do not want to source the environment vars and install oneAPI manually, you can also build the code using intel docker container: [oneAPI-runtime](https://hub.docker.com/r/intel/oneapi-runtime) - - ```bash - mkdir build - cd build - cmake .. -DLLAMA_BLAS=ON -DLLAMA_BLAS_VENDOR=Intel10_64lp -DCMAKE_C_COMPILER=icx -DCMAKE_CXX_COMPILER=icpx -DLLAMA_NATIVE=ON - cmake --build . --config Release - ``` - - Building through oneAPI compilers will make avx_vnni instruction set available for intel processors that do not support avx512 and avx512_vnni. + If you do not want to source the environment vars and install oneAPI manually, you can also build the code using intel docker container: [oneAPI-basekit](https://hub.docker.com/r/intel/oneapi-basekit). Then, you can use the commands given above. Check [Optimizing and Running LLaMA2 on Intel® CPU](https://www.intel.com/content/www/us/en/content-details/791610/optimizing-and-running-llama2-on-intel-cpu.html) for more information. @@ -601,14 +601,48 @@ Building the program with BLAS support may lead to some performance improvements You can get a list of platforms and devices from the `clinfo -l` command, etc. -- #### SYCL +- #### Vulkan - SYCL is a higher-level programming model to improve programming productivity on various hardware accelerators. + **With docker**: - llama.cpp based on SYCL is used to support Intel GPU (Data Center Max series, Flex series, Arc series, Built-in GPU and iGPU). + You don't need to install Vulkan SDK. It will be installed inside the container. - For detailed info, please refer to [llama.cpp for SYCL](README-sycl.md). + ```sh + # Build the image + docker build -t llama-cpp-vulkan -f .devops/main-vulkan.Dockerfile . + # Then, use it: + docker run -it --rm -v "$(pwd):/app:Z" --device /dev/dri/renderD128:/dev/dri/renderD128 --device /dev/dri/card1:/dev/dri/card1 llama-cpp-vulkan -m "/app/models/YOUR_MODEL_FILE" -p "Building a website can be done in 10 simple steps:" -n 400 -e -ngl 33 + ``` + + **Without docker**: + + Firstly, you need to make sure you installed [Vulkan SDK](https://vulkan.lunarg.com/doc/view/latest/linux/getting_started_ubuntu.html) + + For example, on Ubuntu 22.04 (jammy), use the command below: + + ```bash + wget -qO - https://packages.lunarg.com/lunarg-signing-key-pub.asc | apt-key add - + wget -qO /etc/apt/sources.list.d/lunarg-vulkan-jammy.list https://packages.lunarg.com/vulkan/lunarg-vulkan-jammy.list + apt update -y + apt-get install -y vulkan-sdk + # To verify the installation, use the command below: + vulkaninfo + ``` + + Then, build llama.cpp using the cmake command below: + + ```bash + mkdir -p build + cd build + cmake .. -DLLAMA_VULKAN=1 + cmake --build . --config Release + # Test the output binary (with "-ngl 33" to offload all layers to GPU) + ./bin/main -m "PATH_TO_MODEL" -p "Hi you how are you" -n 50 -e -ngl 33 -t 4 + + # You should see in the output, ggml_vulkan detected your GPU. For example: + # ggml_vulkan: Using Intel(R) Graphics (ADL GT2) | uma: 1 | fp16: 1 | warp size: 32 + ``` ### Prepare Data & Run From b05102fe8cfa9893851c6bf6efd15cdc20b6afa2 Mon Sep 17 00:00:00 2001 From: AidanBeltonS <87009434+AidanBeltonS@users.noreply.github.com> Date: Fri, 2 Feb 2024 08:39:48 +0000 Subject: [PATCH 243/812] Tidy ggml-sycl (#5261) * Tidy some code in ggml-sycl * Remove blank space * Remove std::printf comments --------- Co-authored-by: Abhilash Majumder <30946547+abhilash1910@users.noreply.github.com> --- ggml-sycl.cpp | 47 ++++++++++++++++++++++++----------------------- 1 file changed, 24 insertions(+), 23 deletions(-) diff --git a/ggml-sycl.cpp b/ggml-sycl.cpp index 4ee2eed38..ac75f8e16 100644 --- a/ggml-sycl.cpp +++ b/ggml-sycl.cpp @@ -1366,6 +1366,7 @@ namespace dpct } #else return q.memcpy(to_ptr, from_ptr, size, dep_events); + GGML_UNUSED(direction); #endif // DPCT_USM_LEVEL_NONE } @@ -1667,7 +1668,7 @@ namespace dpct using Ty = typename DataType::T2; Ty s_h; if (get_pointer_attribute(q, s) == pointer_access_attribute::device_only) - detail::dpct_memcpy(q, (void *)&s_h, (void *)s, sizeof(T), device_to_host) + detail::dpct_memcpy(q, (void *)&s_h, (const void *)s, sizeof(T), device_to_host) .wait(); else s_h = *reinterpret_cast(s); @@ -1691,6 +1692,20 @@ namespace dpct int ldb, const void *beta, void *c, int ldc) { #ifndef __INTEL_MKL__ + GGML_UNUSED(q); + GGML_UNUSED(a_trans); + GGML_UNUSED(b_trans); + GGML_UNUSED(m); + GGML_UNUSED(n); + GGML_UNUSED(k); + GGML_UNUSED(alpha); + GGML_UNUSED(a); + GGML_UNUSED(lda); + GGML_UNUSED(b); + GGML_UNUSED(ldb); + GGML_UNUSED(beta); + GGML_UNUSED(c); + GGML_UNUSED(ldc); throw std::runtime_error("The oneAPI Math Kernel Library (oneMKL) Interfaces " "Project does not support this API."); #else @@ -1830,7 +1845,7 @@ namespace dpct template T permute_sub_group_by_xor(sycl::sub_group g, T x, unsigned int mask, - int logical_sub_group_size = 32) + unsigned int logical_sub_group_size = 32) { unsigned int id = g.get_local_linear_id(); unsigned int start_index = @@ -2160,6 +2175,7 @@ namespace dpct } #else return q.memcpy(to_ptr, from_ptr, size, dep_events); + GGML_UNUSED(direction); #endif // DPCT_USM_LEVEL_NONE } @@ -3302,7 +3318,7 @@ void log_ggml_var_device(const char*name, float *src, size_t total_elements, boo std::ofstream logfile; logfile.open(filename); // printf("local buf element %d\n", total_elements); - for(int i=0; ibackend == GGML_BACKEND_GPU && device_id == g_main_device ? ne0 : row_diff; - const int compute_capability = g_device_caps[id].cc; #ifdef GGML_SYCL_F16 bool use_fp16 = true; // TODO(Yu) SYCL capability check #else @@ -12691,7 +12700,7 @@ static void ggml_sycl_set_peer_access(const int n_tokens) { continue; } - int can_access_peer; + // int can_access_peer; // SYCL_CHECK(syclDeviceCanAccessPeer(&can_access_peer, id, id_other)); // if (can_access_peer) { // if (enable_peer_access) { @@ -12716,7 +12725,6 @@ static void ggml_sycl_op_mul_mat(const ggml_tensor *src0, const int64_t ne01 = src0->ne[1]; const int64_t ne02 = src0->ne[2]; const int64_t ne03 = src0->ne[3]; - const int64_t nrows0 = ggml_nrows(src0); const int64_t ne10 = src1->ne[0]; const int64_t ne11 = src1->ne[1]; @@ -13812,13 +13820,6 @@ static void ggml_sycl_mul_mat_id(const ggml_tensor *src0, src1_row_extra.data_device[g_main_device_index] = src1_contiguous.get(); dst_row_extra.data_device[g_main_device_index] = dst_contiguous.get(); - const dpct::memcpy_direction src1_kind = - src1->backend == GGML_BACKEND_CPU ? dpct::host_to_device - : dpct::device_to_device; - const dpct::memcpy_direction dst_kind = dst->backend == GGML_BACKEND_CPU - ? dpct::device_to_host - : dpct::device_to_device; - for (int32_t row_id = 0; row_id < n_as; ++row_id) { const struct ggml_tensor * src0_row = dst->src[row_id + 2]; From 2d40085c26794e29c434480b9e06738e89e5686f Mon Sep 17 00:00:00 2001 From: Mirror Azure <54669636+MirrorAzure@users.noreply.github.com> Date: Fri, 2 Feb 2024 14:39:09 +0300 Subject: [PATCH 244/812] py : add check for '.attn.masked_bias' layers to GPT2model (#5281) --- convert-hf-to-gguf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/convert-hf-to-gguf.py b/convert-hf-to-gguf.py index 4ebab07b3..a6ffd128b 100755 --- a/convert-hf-to-gguf.py +++ b/convert-hf-to-gguf.py @@ -1138,7 +1138,7 @@ class GPT2Model(Model): for name, data_torch in self.get_tensors(): # we don't need these - if name.endswith((".attention.masked_bias", ".attention.bias", ".attention.rotary_emb.inv_freq", ".attn.bias")): + if name.endswith((".attention.masked_bias", ".attention.bias", ".attention.rotary_emb.inv_freq", ".attn.bias", ".attn.masked_bias")): continue if name.endswith((".c_attn.weight", ".c_proj.weight", ".c_fc.weight", ".c_proj.weight")): From e437b37fd0b2b97e6c6ff1045ec7f901faa6498a Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Fri, 2 Feb 2024 14:23:40 +0200 Subject: [PATCH 245/812] scripts : parse wtype in server-llm.sh (#5167) * scripts : parse wtype in server-llm.sh * scripts : fix check for wfile --- scripts/server-llm.sh | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/scripts/server-llm.sh b/scripts/server-llm.sh index 7bf0929bb..0b83cdbbc 100644 --- a/scripts/server-llm.sh +++ b/scripts/server-llm.sh @@ -141,6 +141,28 @@ for wt in "${wtypes[@]}"; do wfiles+=("") done +# map wtype input to index +if [[ ! -z "$wtype" ]]; then + iw=-1 + is=0 + for wt in "${wtypes[@]}"; do + # uppercase + uwt=$(echo "$wt" | tr '[:lower:]' '[:upper:]') + if [[ "$uwt" == "$wtype" ]]; then + iw=$is + break + fi + is=$((is+1)) + done + + if [[ $iw -eq -1 ]]; then + printf "[-] Invalid weight type: %s\n" "$wtype" + exit 1 + fi + + wtype="$iw" +fi + # sample repos repos=( "https://huggingface.co/TheBloke/Llama-2-7B-GGUF" @@ -252,8 +274,10 @@ for file in $model_files; do printf " %2d) %s %s\n" $iw "$have" "$file" done +wfile="${wfiles[$wtype]}" + # ask for weights type until provided and available -while [[ -z "$wtype" ]]; do +while [[ -z "$wfile" ]]; do printf "\n" read -p "[+] Select weight type: " wtype wfile="${wfiles[$wtype]}" From 191221178f51b6e81122c5bda0fd79620e547d07 Mon Sep 17 00:00:00 2001 From: kalomaze <66376113+kalomaze@users.noreply.github.com> Date: Fri, 2 Feb 2024 08:15:30 -0600 Subject: [PATCH 246/812] perplexity : fix KL divergence calculations on Windows (#5273) --- examples/perplexity/perplexity.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/perplexity/perplexity.cpp b/examples/perplexity/perplexity.cpp index 8d2204969..4b08145cd 100644 --- a/examples/perplexity/perplexity.cpp +++ b/examples/perplexity/perplexity.cpp @@ -457,14 +457,14 @@ static results_perplexity perplexity(llama_context * ctx, const gpt_params & par std::ofstream logits_stream; if (!params.logits_file.empty()) { - logits_stream.open(params.logits_file.c_str()); + logits_stream.open(params.logits_file.c_str(), std::ios::binary); if (!logits_stream.is_open()) { fprintf(stderr, "%s: failed to open %s for writing\n", __func__, params.logits_file.c_str()); return {}; } fprintf(stderr, "%s: saving all logits to %s\n", __func__, params.logits_file.c_str()); logits_stream.write("_logits_", 8); - logits_stream.write((const char *)&n_ctx, sizeof(n_ctx)); + logits_stream.write(reinterpret_cast(&n_ctx), sizeof(n_ctx)); } auto tim1 = std::chrono::high_resolution_clock::now(); From a305dba8ff642e57f538f42010868fe0bc5262a1 Mon Sep 17 00:00:00 2001 From: AidanBeltonS <87009434+AidanBeltonS@users.noreply.github.com> Date: Sat, 3 Feb 2024 08:11:37 +0000 Subject: [PATCH 247/812] Fix im2col with 32fp (#5286) --- ggml-sycl.cpp | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/ggml-sycl.cpp b/ggml-sycl.cpp index ac75f8e16..51445b5e7 100644 --- a/ggml-sycl.cpp +++ b/ggml-sycl.cpp @@ -8247,7 +8247,8 @@ static void clamp_f32(const float * x, float * dst, const float min, const float dst[i] = x[i] < min ? min : (x[i] > max ? max : x[i]); } -static void im2col_f32_f16(const float *x, sycl::half *dst, int offset_delta, +template +static void im2col_kernel(const float *x, T *dst, int offset_delta, int IW, int IH, int OW, int KW, int KH, int pelements, int CHW, int s0, int s1, int p0, int p1, int d0, int d1, @@ -11019,7 +11020,8 @@ static void soft_max_f32_sycl(const float *x, const float *y, float *dst, }); } -static void im2col_f32_f16_sycl(const float *x, sycl::half *dst, int IW, int IH, +template +static void im2col_sycl(const float *x, T *dst, int IW, int IH, int OW, int OH, int KW, int KH, int IC, int offset_delta, int s0, int s1, int p0, int p1, int d0, int d1, @@ -11036,7 +11038,7 @@ static void im2col_f32_f16_sycl(const float *x, sycl::half *dst, int IW, int IH, sycl::range<3>(1, 1, SYCL_IM2COL_BLOCK_SIZE), sycl::range<3>(1, 1, SYCL_IM2COL_BLOCK_SIZE)), [=](sycl::nd_item<3> item_ct1) { - im2col_f32_f16(x, dst, offset_delta, IW, IH, OW, KW, KH, + im2col_kernel(x, dst, offset_delta, IW, IH, OW, KW, KH, parallel_elements, (IC * KH * KW), s0, s1, p0, p1, d0, d1, item_ct1); }); @@ -12424,7 +12426,7 @@ inline void ggml_sycl_op_im2col(const ggml_tensor *src0, GGML_ASSERT(src0->type == GGML_TYPE_F16); GGML_ASSERT(src1->type == GGML_TYPE_F32); - GGML_ASSERT( dst->type == GGML_TYPE_F16); + GGML_ASSERT( dst->type == GGML_TYPE_F16 || dst->type == GGML_TYPE_F32); const int32_t s0 = ((const int32_t*)(dst->op_params))[0]; const int32_t s1 = ((const int32_t*)(dst->op_params))[1]; @@ -12447,8 +12449,11 @@ inline void ggml_sycl_op_im2col(const ggml_tensor *src0, const size_t delta_offset = src1->nb[is_2D ? 2 : 1] / 4; // nb is byte offset, src is type float32 - im2col_f32_f16_sycl(src1_dd, (sycl::half *)dst_dd, IW, IH, OW, OH, KW, KH, - IC, delta_offset, s0, s1, p0, p1, d0, d1, main_stream); + if (dst->type == GGML_TYPE_F16) { + im2col_sycl(src1_dd, (sycl::half *)dst_dd, IW, IH, OW, OH, KW, KH, IC, delta_offset, s0, s1, p0, p1, d0, d1, main_stream); + } else { + im2col_sycl(src1_dd, (float *)dst_dd, IW, IH, OW, OH, KW, KH, IC, delta_offset, s0, s1, p0, p1, d0, d1, main_stream); + } (void) src0; (void) src0_dd; From 6a66c5071a74a96c4f52cf1015a092acd18c3714 Mon Sep 17 00:00:00 2001 From: BADR Date: Sat, 3 Feb 2024 12:20:26 +0100 Subject: [PATCH 248/812] readme : add tenere in the ui tools list (#5284) --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index af1f09fa0..4a9bdf314 100644 --- a/README.md +++ b/README.md @@ -143,6 +143,7 @@ as the main playground for developing new features for the [ggml](https://github - [psugihara/FreeChat](https://github.com/psugihara/FreeChat) - [ptsochantaris/emeltal](https://github.com/ptsochantaris/emeltal) - [iohub/collama](https://github.com/iohub/coLLaMA) +- [pythops/tenere](https://github.com/pythops/tenere) --- From 1ec3332ade60aeb1494ace2211cf1a966db6d770 Mon Sep 17 00:00:00 2001 From: Jared Van Bortel Date: Sat, 3 Feb 2024 06:22:06 -0500 Subject: [PATCH 249/812] YaRN : store rope scaling type as int32_t in memory (#5285) * YaRN : store rope scaling type as int32_t in memory * llama : store mapped names as const char * --- common/common.h | 3 +-- llama.cpp | 24 ++++++++++++------------ llama.h | 2 +- 3 files changed, 14 insertions(+), 15 deletions(-) diff --git a/common/common.h b/common/common.h index 24a99d728..62de25d6a 100644 --- a/common/common.h +++ b/common/common.h @@ -75,8 +75,7 @@ struct gpt_params { float yarn_beta_fast = 32.0f; // YaRN low correction dim float yarn_beta_slow = 1.0f; // YaRN high correction dim int32_t yarn_orig_ctx = 0; // YaRN original context length - int8_t rope_scaling_type = LLAMA_ROPE_SCALING_UNSPECIFIED; // TODO: better to be int32_t for alignment - // pinging @cebtenzzre + int32_t rope_scaling_type = LLAMA_ROPE_SCALING_UNSPECIFIED; // // sampling parameters struct llama_sampling_params sparams; diff --git a/llama.cpp b/llama.cpp index 6bf7f9efb..4787a92fe 100644 --- a/llama.cpp +++ b/llama.cpp @@ -208,7 +208,7 @@ enum llm_arch { LLM_ARCH_UNKNOWN, }; -static std::map LLM_ARCH_NAMES = { +static std::map LLM_ARCH_NAMES = { { LLM_ARCH_LLAMA, "llama" }, { LLM_ARCH_FALCON, "falcon" }, { LLM_ARCH_GPT2, "gpt2" }, @@ -285,7 +285,7 @@ enum llm_kv { LLM_KV_TOKENIZER_RWKV, }; -static std::map LLM_KV_NAMES = { +static std::map LLM_KV_NAMES = { { LLM_KV_GENERAL_ARCHITECTURE, "general.architecture" }, { LLM_KV_GENERAL_QUANTIZATION_VERSION, "general.quantization_version" }, { LLM_KV_GENERAL_ALIGNMENT, "general.alignment" }, @@ -346,7 +346,7 @@ struct LLM_KV { llm_arch arch; std::string operator()(llm_kv kv) const { - return ::format(LLM_KV_NAMES[kv].c_str(), LLM_ARCH_NAMES[arch].c_str()); + return ::format(LLM_KV_NAMES[kv], LLM_ARCH_NAMES[arch]); } }; @@ -747,13 +747,13 @@ struct LLM_TN { // gguf helpers // -static std::map LLAMA_ROPE_SCALING_TYPES = { +static std::map LLAMA_ROPE_SCALING_TYPES = { { LLAMA_ROPE_SCALING_NONE, "none" }, { LLAMA_ROPE_SCALING_LINEAR, "linear" }, { LLAMA_ROPE_SCALING_YARN, "yarn" }, }; -static int8_t llama_rope_scaling_type_from_string(const std::string & name) { +static int32_t llama_rope_scaling_type_from_string(const std::string & name) { for (const auto & kv : LLAMA_ROPE_SCALING_TYPES) { if (kv.second == name) { return kv.first; @@ -1415,6 +1415,7 @@ static const size_t GiB = 1024*MiB; struct llama_hparams { bool vocab_only; + bool rope_finetuned; uint32_t n_vocab; uint32_t n_ctx_train; // context size the model was trained on uint32_t n_embd; @@ -1434,8 +1435,7 @@ struct llama_hparams { float rope_freq_base_train; float rope_freq_scale_train; uint32_t n_yarn_orig_ctx; - int8_t rope_scaling_type_train : 3; - bool rope_finetuned : 1; + int32_t rope_scaling_type_train; float f_clamp_kqv; float f_max_alibi_bias; @@ -2701,7 +2701,7 @@ struct llama_model_loader { // load LLaMA models // -static std::string llama_model_arch_name(llm_arch arch) { +static const char * llama_model_arch_name(llm_arch arch) { auto it = LLM_ARCH_NAMES.find(arch); if (it == LLM_ARCH_NAMES.end()) { return "unknown"; @@ -3310,11 +3310,11 @@ static void llm_load_print_meta(llama_model_loader & ml, llama_model & model) { const auto & hparams = model.hparams; const auto & vocab = model.vocab; - const auto rope_scaling_type = LLAMA_ROPE_SCALING_TYPES.at(hparams.rope_scaling_type_train); + const char * rope_scaling_type = LLAMA_ROPE_SCALING_TYPES.at(hparams.rope_scaling_type_train); // hparams LLAMA_LOG_INFO("%s: format = %s\n", __func__, llama_file_version_name(ml.fver)); - LLAMA_LOG_INFO("%s: arch = %s\n", __func__, LLM_ARCH_NAMES.at(model.arch).c_str()); + LLAMA_LOG_INFO("%s: arch = %s\n", __func__, LLM_ARCH_NAMES.at(model.arch)); LLAMA_LOG_INFO("%s: vocab type = %s\n", __func__, llama_model_vocab_type_name(vocab.type)); LLAMA_LOG_INFO("%s: n_vocab = %u\n", __func__, hparams.n_vocab); LLAMA_LOG_INFO("%s: n_merges = %u\n", __func__, (int) vocab.bpe_ranks.size()); @@ -3336,7 +3336,7 @@ static void llm_load_print_meta(llama_model_loader & ml, llama_model & model) { LLAMA_LOG_INFO("%s: n_ff = %u\n", __func__, hparams.n_ff); LLAMA_LOG_INFO("%s: n_expert = %u\n", __func__, hparams.n_expert); LLAMA_LOG_INFO("%s: n_expert_used = %u\n", __func__, hparams.n_expert_used); - LLAMA_LOG_INFO("%s: rope scaling = %s\n", __func__, rope_scaling_type.c_str()); + LLAMA_LOG_INFO("%s: rope scaling = %s\n", __func__, rope_scaling_type); LLAMA_LOG_INFO("%s: freq_base_train = %.1f\n", __func__, hparams.rope_freq_base_train); LLAMA_LOG_INFO("%s: freq_scale_train = %g\n", __func__, hparams.rope_freq_scale_train); LLAMA_LOG_INFO("%s: n_yarn_orig_ctx = %u\n", __func__, hparams.n_yarn_orig_ctx); @@ -10735,7 +10735,7 @@ int32_t llama_model_meta_val_str_by_index(const struct llama_model * model, int3 int32_t llama_model_desc(const struct llama_model * model, char * buf, size_t buf_size) { return snprintf(buf, buf_size, "%s %s %s", - llama_model_arch_name(model->arch).c_str(), + llama_model_arch_name(model->arch), llama_model_type_name(model->type), llama_model_ftype_name(model->ftype).c_str()); } diff --git a/llama.h b/llama.h index 9a60e9bfb..cec4158bc 100644 --- a/llama.h +++ b/llama.h @@ -213,7 +213,7 @@ extern "C" { uint32_t n_batch; // prompt processing maximum batch size uint32_t n_threads; // number of threads to use for generation uint32_t n_threads_batch; // number of threads to use for batch processing - int8_t rope_scaling_type; // RoPE scaling type, from `enum llama_rope_scaling_type` + int32_t rope_scaling_type; // RoPE scaling type, from `enum llama_rope_scaling_type` // ref: https://github.com/ggerganov/llama.cpp/pull/2054 float rope_freq_base; // RoPE base frequency, 0 = from model From 52bb63c7082c859c3f1dfc527227e6a95b299c7c Mon Sep 17 00:00:00 2001 From: Michael Klimenko Date: Sat, 3 Feb 2024 12:23:37 +0100 Subject: [PATCH 250/812] refactor : switch to emplace_back to avoid extra object (#5291) --- common/common.cpp | 8 ++--- examples/llama-bench/llama-bench.cpp | 34 +++++++++++----------- examples/main/main.cpp | 4 +-- examples/perplexity/perplexity.cpp | 8 ++--- examples/quantize-stats/quantize-stats.cpp | 4 +-- examples/quantize/quantize.cpp | 4 +-- examples/server/server.cpp | 8 ++--- tests/test-llama-grammar.cpp | 2 +- 8 files changed, 36 insertions(+), 36 deletions(-) diff --git a/common/common.cpp b/common/common.cpp index ce739b15c..3302caa20 100644 --- a/common/common.cpp +++ b/common/common.cpp @@ -515,7 +515,7 @@ bool gpt_params_parse_ex(int argc, char ** argv, gpt_params & params) { invalid_param = true; break; } - params.lora_adapter.push_back(std::make_tuple(argv[i], 1.0f)); + params.lora_adapter.emplace_back(argv[i], 1.0f); params.use_mmap = false; } else if (arg == "--lora-scaled") { if (++i >= argc) { @@ -527,7 +527,7 @@ bool gpt_params_parse_ex(int argc, char ** argv, gpt_params & params) { invalid_param = true; break; } - params.lora_adapter.push_back(std::make_tuple(lora_adapter, std::stof(argv[i]))); + params.lora_adapter.emplace_back(lora_adapter, std::stof(argv[i])); params.use_mmap = false; } else if (arg == "--lora-base") { if (++i >= argc) { @@ -664,7 +664,7 @@ bool gpt_params_parse_ex(int argc, char ** argv, gpt_params & params) { invalid_param = true; break; } - params.antiprompt.push_back(argv[i]); + params.antiprompt.emplace_back(argv[i]); } else if (arg == "-ld" || arg == "--logdir") { if (++i >= argc) { invalid_param = true; @@ -880,7 +880,7 @@ bool gpt_params_parse_ex(int argc, char ** argv, gpt_params & params) { } if (!params.kv_overrides.empty()) { - params.kv_overrides.emplace_back(llama_model_kv_override()); + params.kv_overrides.emplace_back(); params.kv_overrides.back().key[0] = 0; } diff --git a/examples/llama-bench/llama-bench.cpp b/examples/llama-bench/llama-bench.cpp index e36c061a2..ddb0ba064 100644 --- a/examples/llama-bench/llama-bench.cpp +++ b/examples/llama-bench/llama-bench.cpp @@ -948,46 +948,46 @@ struct markdown_printer : public printer { void print_header(const cmd_params & params) override { // select fields to print - fields.push_back("model"); - fields.push_back("size"); - fields.push_back("params"); - fields.push_back("backend"); + fields.emplace_back("model"); + fields.emplace_back("size"); + fields.emplace_back("params"); + fields.emplace_back("backend"); bool is_cpu_backend = test::get_backend() == "CPU" || test::get_backend() == "BLAS"; if (!is_cpu_backend) { - fields.push_back("n_gpu_layers"); + fields.emplace_back("n_gpu_layers"); } if (params.n_threads.size() > 1 || params.n_threads != cmd_params_defaults.n_threads || is_cpu_backend) { - fields.push_back("n_threads"); + fields.emplace_back("n_threads"); } if (params.n_batch.size() > 1 || params.n_batch != cmd_params_defaults.n_batch) { - fields.push_back("n_batch"); + fields.emplace_back("n_batch"); } if (params.type_k.size() > 1 || params.type_k != cmd_params_defaults.type_k) { - fields.push_back("type_k"); + fields.emplace_back("type_k"); } if (params.type_v.size() > 1 || params.type_v != cmd_params_defaults.type_v) { - fields.push_back("type_v"); + fields.emplace_back("type_v"); } if (params.main_gpu.size() > 1 || params.main_gpu != cmd_params_defaults.main_gpu) { - fields.push_back("main_gpu"); + fields.emplace_back("main_gpu"); } if (params.split_mode.size() > 1 || params.split_mode != cmd_params_defaults.split_mode) { - fields.push_back("split_mode"); + fields.emplace_back("split_mode"); } if (params.mul_mat_q.size() > 1 || params.mul_mat_q != cmd_params_defaults.mul_mat_q) { - fields.push_back("mul_mat_q"); + fields.emplace_back("mul_mat_q"); } if (params.no_kv_offload.size() > 1 || params.no_kv_offload != cmd_params_defaults.no_kv_offload) { - fields.push_back("no_kv_offload"); + fields.emplace_back("no_kv_offload"); } if (params.tensor_split.size() > 1 || params.tensor_split != cmd_params_defaults.tensor_split) { - fields.push_back("tensor_split"); + fields.emplace_back("tensor_split"); } if (params.use_mmap.size() > 1 || params.use_mmap != cmd_params_defaults.use_mmap) { - fields.push_back("use_mmap"); + fields.emplace_back("use_mmap"); } - fields.push_back("test"); - fields.push_back("t/s"); + fields.emplace_back("test"); + fields.emplace_back("t/s"); fprintf(fout, "|"); for (const auto & field : fields) { diff --git a/examples/main/main.cpp b/examples/main/main.cpp index 1c6138d23..0ed4d79f9 100644 --- a/examples/main/main.cpp +++ b/examples/main/main.cpp @@ -352,12 +352,12 @@ int main(int argc, char ** argv) { // in instruct mode, we inject a prefix and a suffix to each input by the user if (params.instruct) { params.interactive_first = true; - params.antiprompt.push_back("### Instruction:\n\n"); + params.antiprompt.emplace_back("### Instruction:\n\n"); } // similar for chatml mode else if (params.chatml) { params.interactive_first = true; - params.antiprompt.push_back("<|im_start|>user\n"); + params.antiprompt.emplace_back("<|im_start|>user\n"); } // enable interactive mode if interactive start is specified diff --git a/examples/perplexity/perplexity.cpp b/examples/perplexity/perplexity.cpp index 4b08145cd..b2c131d4c 100644 --- a/examples/perplexity/perplexity.cpp +++ b/examples/perplexity/perplexity.cpp @@ -881,7 +881,7 @@ static void hellaswag_score(llama_context * ctx, const gpt_params & params) { size_t li = hs_cur.common_prefix; for (int s = 0; s < 4; ++s) { for (size_t j = hs_cur.common_prefix; j < hs_cur.seq_tokens[s].size() - 1; j++) { - eval_pairs.push_back(std::make_pair(hs_cur.i_batch + li++, hs_cur.seq_tokens[s][j + 1])); + eval_pairs.emplace_back(hs_cur.i_batch + li++, hs_cur.seq_tokens[s][j + 1]); } ++li; } @@ -1159,13 +1159,13 @@ static void winogrande_score(llama_context * ctx, const gpt_params & params) { const int last_1st = task.seq_tokens[0].size() - n_base1 > 1 ? 1 : 0; size_t li = n_base1 - 1; for (size_t j = n_base1-1; j < task.seq_tokens[0].size()-1-last_1st; ++j) { - eval_pairs.push_back(std::make_pair(task.i_batch + li++, task.seq_tokens[0][j+1])); + eval_pairs.emplace_back(task.i_batch + li++, task.seq_tokens[0][j+1]); } const auto& n_base2 = skip_choice ? task.n_base2 : task.common_prefix; const int last_2nd = task.seq_tokens[1].size() - n_base2 > 1 ? 1 : 0; li = task.seq_tokens[0].size() - task.common_prefix + n_base2 - 1; for (size_t j = n_base2-1; j < task.seq_tokens[1].size()-1-last_2nd; ++j) { - eval_pairs.push_back(std::make_pair(task.i_batch + li++, task.seq_tokens[1][j+1])); + eval_pairs.emplace_back(task.i_batch + li++, task.seq_tokens[1][j+1]); } } compute_logprobs(batch_logits.data(), n_vocab, workers, eval_pairs, eval_results); @@ -1524,7 +1524,7 @@ static void multiple_choice_score(llama_context * ctx, const gpt_params & params size_t li = cur_task.common_prefix; for (int s = 0; s < int(cur_task.seq_tokens.size()); ++s) { for (size_t j = cur_task.common_prefix; j < cur_task.seq_tokens[s].size() - 1; j++) { - eval_pairs.push_back(std::make_pair(cur_task.i_batch + li++, cur_task.seq_tokens[s][j + 1])); + eval_pairs.emplace_back(cur_task.i_batch + li++, cur_task.seq_tokens[s][j + 1]); } ++li; } diff --git a/examples/quantize-stats/quantize-stats.cpp b/examples/quantize-stats/quantize-stats.cpp index 6d5f213dc..1d05f1391 100644 --- a/examples/quantize-stats/quantize-stats.cpp +++ b/examples/quantize-stats/quantize-stats.cpp @@ -257,13 +257,13 @@ int main(int argc, char ** argv) { invalid_param = true; break; } - params.include_layers.push_back(argv[i]); + params.include_layers.emplace_back(argv[i]); } else if (arg == "-L" || arg == "--exclude-layer") { if (++i >= argc) { invalid_param = true; break; } - params.exclude_layers.push_back(argv[i]); + params.exclude_layers.emplace_back(argv[i]); } else if (arg == "-t" || arg == "--type") { if (++i >= argc) { invalid_param = true; diff --git a/examples/quantize/quantize.cpp b/examples/quantize/quantize.cpp index a9673f0d4..85f403ffc 100644 --- a/examples/quantize/quantize.cpp +++ b/examples/quantize/quantize.cpp @@ -208,13 +208,13 @@ int main(int argc, char ** argv) { } } else if (strcmp(argv[arg_idx], "--include-weights") == 0) { if (arg_idx < argc-1) { - included_weights.push_back(argv[++arg_idx]); + included_weights.emplace_back(argv[++arg_idx]); } else { usage(argv[0]); } } else if (strcmp(argv[arg_idx], "--exclude-weights") == 0) { if (arg_idx < argc-1) { - excluded_weights.push_back(argv[++arg_idx]); + excluded_weights.emplace_back(argv[++arg_idx]); } else { usage(argv[0]); } diff --git a/examples/server/server.cpp b/examples/server/server.cpp index ea77125ea..a9f8cb369 100644 --- a/examples/server/server.cpp +++ b/examples/server/server.cpp @@ -1884,7 +1884,7 @@ static void server_params_parse(int argc, char **argv, server_params &sparams, invalid_param = true; break; } - sparams.api_keys.push_back(argv[i]); + sparams.api_keys.emplace_back(argv[i]); } else if (arg == "--api-key-file") { @@ -2160,7 +2160,7 @@ static void server_params_parse(int argc, char **argv, server_params &sparams, invalid_param = true; break; } - params.lora_adapter.push_back(std::make_tuple(argv[i], 1.0f)); + params.lora_adapter.emplace_back(argv[i], 1.0f); params.use_mmap = false; } else if (arg == "--lora-scaled") @@ -2176,7 +2176,7 @@ static void server_params_parse(int argc, char **argv, server_params &sparams, invalid_param = true; break; } - params.lora_adapter.push_back(std::make_tuple(lora_adapter, std::stof(argv[i]))); + params.lora_adapter.emplace_back(lora_adapter, std::stof(argv[i])); params.use_mmap = false; } else if (arg == "--lora-base") @@ -2318,7 +2318,7 @@ static void server_params_parse(int argc, char **argv, server_params &sparams, } } if (!params.kv_overrides.empty()) { - params.kv_overrides.emplace_back(llama_model_kv_override()); + params.kv_overrides.emplace_back(); params.kv_overrides.back().key[0] = 0; } diff --git a/tests/test-llama-grammar.cpp b/tests/test-llama-grammar.cpp index 78fc41117..16ebe753f 100644 --- a/tests/test-llama-grammar.cpp +++ b/tests/test-llama-grammar.cpp @@ -105,7 +105,7 @@ int main() for (auto rule : expected_rules) { - parsed_grammar.rules.push_back({}); + parsed_grammar.rules.emplace_back(); for (auto element : rule) { parsed_grammar.rules.back().push_back(element); From e920ed393d989ed35625ddaf182ebb52cda07fcd Mon Sep 17 00:00:00 2001 From: 0cc4m Date: Sat, 3 Feb 2024 18:15:00 +0100 Subject: [PATCH 251/812] Vulkan Intel Fixes, Optimizations and Debugging Flags (#5301) * Fix Vulkan on Intel ARC Optimize matmul for Intel ARC Add Vulkan dequant test * Add Vulkan debug and validate flags to Make and CMakeLists.txt * Enable asynchronous transfers in Vulkan backend * Fix flake8 * Disable Vulkan async backend functions for now * Also add Vulkan run tests command to Makefile and CMakeLists.txt --- CMakeLists.txt | 20 + Makefile | 12 + ggml-vulkan-shaders.hpp | 10922 +++------------------------------- ggml-vulkan.cpp | 420 +- ggml_vk_generate_shaders.py | 213 +- 5 files changed, 1257 insertions(+), 10330 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 1ee455b3a..c156c4824 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -100,6 +100,10 @@ option(LLAMA_HIPBLAS "llama: use hipBLAS" option(LLAMA_HIP_UMA "llama: use HIP unified memory architecture" OFF) option(LLAMA_CLBLAST "llama: use CLBlast" OFF) option(LLAMA_VULKAN "llama: use Vulkan" OFF) +option(LLAMA_VULKAN_CHECK_RESULTS "llama: run Vulkan op checks" OFF) +option(LLAMA_VULKAN_DEBUG "llama: enable Vulkan debug output" OFF) +option(LLAMA_VULKAN_VALIDATE "llama: enable Vulkan validation" OFF) +option(LLAMA_VULKAN_RUN_TESTS "llama: run Vulkan tests" OFF) option(LLAMA_METAL "llama: use Metal" ${LLAMA_METAL_DEFAULT}) option(LLAMA_METAL_NDEBUG "llama: disable Metal debugging" OFF) option(LLAMA_METAL_SHADER_DEBUG "llama: compile Metal with -fno-fast-math" OFF) @@ -431,6 +435,22 @@ if (LLAMA_VULKAN) add_compile_definitions(GGML_USE_VULKAN) + if (LLAMA_VULKAN_CHECK_RESULTS) + target_compile_definitions(ggml-vulkan PRIVATE GGML_VULKAN_CHECK_RESULTS) + endif() + + if (LLAMA_VULKAN_DEBUG) + target_compile_definitions(ggml-vulkan PRIVATE GGML_VULKAN_DEBUG) + endif() + + if (LLAMA_VULKAN_VALIDATE) + target_compile_definitions(ggml-vulkan PRIVATE GGML_VULKAN_VALIDATE) + endif() + + if (LLAMA_VULKAN_RUN_TESTS) + target_compile_definitions(ggml-vulkan PRIVATE GGML_VULKAN_RUN_TESTS) + endif() + set(LLAMA_EXTRA_LIBS ${LLAMA_EXTRA_LIBS} ggml-vulkan) else() message(WARNING "Vulkan not found") diff --git a/Makefile b/Makefile index bf9e085de..a55d15888 100644 --- a/Makefile +++ b/Makefile @@ -457,6 +457,18 @@ ifdef LLAMA_VULKAN_CHECK_RESULTS MK_CPPFLAGS += -DGGML_VULKAN_CHECK_RESULTS endif +ifdef LLAMA_VULKAN_DEBUG + MK_CPPFLAGS += -DGGML_VULKAN_DEBUG +endif + +ifdef LLAMA_VULKAN_VALIDATE + MK_CPPFLAGS += -DGGML_VULKAN_VALIDATE +endif + +ifdef LLAMA_VULKAN_RUN_TESTS + MK_CPPFLAGS += -DGGML_VULKAN_RUN_TESTS +endif + ggml-vulkan.o: ggml-vulkan.cpp ggml-vulkan.h $(CXX) $(CXXFLAGS) -c $< -o $@ endif # LLAMA_VULKAN diff --git a/ggml-vulkan-shaders.hpp b/ggml-vulkan-shaders.hpp index 195410c02..e5e7a8414 100644 --- a/ggml-vulkan-shaders.hpp +++ b/ggml-vulkan-shaders.hpp @@ -890,156 +890,6 @@ const uint64_t cpy_f32_f32_len = 2472; unsigned char dequant_f16_data[] = { 0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, -0x81,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, -0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x09,0x00,0x00,0x00, -0x11,0x00,0x02,0x00,0x51,0x11,0x00,0x00,0x0b,0x00,0x06,0x00, -0x01,0x00,0x00,0x00,0x47,0x4c,0x53,0x4c,0x2e,0x73,0x74,0x64, -0x2e,0x34,0x35,0x30,0x00,0x00,0x00,0x00,0x0e,0x00,0x03,0x00, -0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x0f,0x00,0x09,0x00, -0x05,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x6d,0x61,0x69,0x6e, -0x00,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0x4f,0x00,0x00,0x00,0x5d,0x00,0x00,0x00,0x10,0x00,0x06,0x00, -0x04,0x00,0x00,0x00,0x11,0x00,0x00,0x00,0x00,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x0c,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x1c,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x14,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x14,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x14,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x08,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x14,0x00,0x00,0x00,0x03,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x47,0x00,0x03,0x00, -0x14,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x4c,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x48,0x00,0x04,0x00,0x4d,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x4d,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x03,0x00,0x4d,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x4f,0x00,0x00,0x00,0x22,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x4f,0x00,0x00,0x00, -0x21,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x5a,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x48,0x00,0x04,0x00,0x5b,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x19,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x5b,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x03,0x00,0x5b,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x5d,0x00,0x00,0x00,0x22,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x5d,0x00,0x00,0x00, -0x21,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x7e,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x19,0x00,0x00,0x00, -0x13,0x00,0x02,0x00,0x02,0x00,0x00,0x00,0x21,0x00,0x03,0x00, -0x03,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x15,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x15,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x17,0x00,0x04,0x00,0x0a,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x0b,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x0a,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x0b,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x0d,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x0e,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x1e,0x00,0x06,0x00,0x14,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x15,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x14,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x15,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x17,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x18,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x14,0x00,0x02,0x00,0x23,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x2d,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x36,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x16,0x00,0x03,0x00, -0x48,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, -0x4c,0x00,0x00,0x00,0x48,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, -0x4d,0x00,0x00,0x00,0x4c,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x4e,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x4d,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x4e,0x00,0x00,0x00,0x4f,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x52,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x48,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, -0x5a,0x00,0x00,0x00,0x48,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, -0x5b,0x00,0x00,0x00,0x5a,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x5c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x5b,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x5c,0x00,0x00,0x00,0x5d,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x5f,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x77,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x7d,0x00,0x00,0x00, -0x00,0x01,0x00,0x00,0x2c,0x00,0x06,0x00,0x0a,0x00,0x00,0x00, -0x7e,0x00,0x00,0x00,0x7d,0x00,0x00,0x00,0x77,0x00,0x00,0x00, -0x77,0x00,0x00,0x00,0x36,0x00,0x05,0x00,0x02,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x03,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0x05,0x00,0x00,0x00,0xf7,0x00,0x03,0x00, -0x7f,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xfb,0x00,0x03,0x00, -0x0d,0x00,0x00,0x00,0x80,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x80,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x0e,0x00,0x00,0x00, -0x0f,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x0d,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x10,0x00,0x00,0x00, -0x0f,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x11,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x18,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0x17,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x1a,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x87,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x1b,0x00,0x00,0x00,0x1a,0x00,0x00,0x00, -0x17,0x00,0x00,0x00,0x8b,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x1c,0x00,0x00,0x00,0x11,0x00,0x00,0x00,0x1b,0x00,0x00,0x00, -0x87,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x22,0x00,0x00,0x00, -0x11,0x00,0x00,0x00,0x1b,0x00,0x00,0x00,0xaf,0x00,0x05,0x00, -0x23,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0x1c,0x00,0x00,0x00, -0x1a,0x00,0x00,0x00,0xa8,0x00,0x04,0x00,0x23,0x00,0x00,0x00, -0x29,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0xf7,0x00,0x03,0x00, -0x2b,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x29,0x00,0x00,0x00,0x2a,0x00,0x00,0x00,0x2b,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0x2a,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x18,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0x2d,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x2f,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0xaf,0x00,0x05,0x00, -0x23,0x00,0x00,0x00,0x30,0x00,0x00,0x00,0x22,0x00,0x00,0x00, -0x2f,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x2b,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0x2b,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, -0x23,0x00,0x00,0x00,0x31,0x00,0x00,0x00,0x28,0x00,0x00,0x00, -0x80,0x00,0x00,0x00,0x30,0x00,0x00,0x00,0x2a,0x00,0x00,0x00, -0xf7,0x00,0x03,0x00,0x33,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x31,0x00,0x00,0x00,0x32,0x00,0x00,0x00, -0x33,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x32,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x7f,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x33,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x18,0x00,0x00,0x00, -0x37,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x36,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x38,0x00,0x00,0x00, -0x37,0x00,0x00,0x00,0x87,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x38,0x00,0x00,0x00,0x17,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3d,0x00,0x00,0x00, -0x22,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x3f,0x00,0x00,0x00,0x3d,0x00,0x00,0x00, -0x1c,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x52,0x00,0x00,0x00, -0x53,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x2d,0x00,0x00,0x00, -0x3f,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x48,0x00,0x00,0x00, -0x54,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x56,0x00,0x00,0x00,0x3f,0x00,0x00,0x00, -0x17,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x52,0x00,0x00,0x00, -0x57,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x2d,0x00,0x00,0x00, -0x56,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x48,0x00,0x00,0x00, -0x58,0x00,0x00,0x00,0x57,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x18,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0x5f,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x61,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x22,0x00,0x00,0x00, -0x61,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x65,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x1c,0x00,0x00,0x00, -0x41,0x00,0x06,0x00,0x52,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x5d,0x00,0x00,0x00,0x2d,0x00,0x00,0x00,0x65,0x00,0x00,0x00, -0x3e,0x00,0x03,0x00,0x6c,0x00,0x00,0x00,0x54,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x76,0x00,0x00,0x00, -0x65,0x00,0x00,0x00,0x17,0x00,0x00,0x00,0x41,0x00,0x06,0x00, -0x52,0x00,0x00,0x00,0x7a,0x00,0x00,0x00,0x5d,0x00,0x00,0x00, -0x2d,0x00,0x00,0x00,0x76,0x00,0x00,0x00,0x3e,0x00,0x03,0x00, -0x7a,0x00,0x00,0x00,0x58,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x7f,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x7f,0x00,0x00,0x00, -0xfd,0x00,0x01,0x00,0x38,0x00,0x01,0x00, -}; -const uint64_t dequant_f16_len = 1748; - -unsigned char dequant_f16_fp32_data[] = { -0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, 0x86,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, 0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x51,0x11,0x00,0x00, 0x0b,0x00,0x06,0x00,0x01,0x00,0x00,0x00,0x47,0x4c,0x53,0x4c, @@ -1192,344 +1042,10 @@ unsigned char dequant_f16_fp32_data[] = { 0xf8,0x00,0x02,0x00,0x84,0x00,0x00,0x00,0xfd,0x00,0x01,0x00, 0x38,0x00,0x01,0x00, }; -const uint64_t dequant_f16_fp32_len = 1816; +const uint64_t dequant_f16_len = 1816; unsigned char dequant_q2_K_data[] = { 0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, -0x0c,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, -0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x09,0x00,0x00,0x00, -0x11,0x00,0x02,0x00,0x27,0x00,0x00,0x00,0x11,0x00,0x02,0x00, -0x51,0x11,0x00,0x00,0x11,0x00,0x02,0x00,0x60,0x11,0x00,0x00, -0x0b,0x00,0x06,0x00,0x01,0x00,0x00,0x00,0x47,0x4c,0x53,0x4c, -0x2e,0x73,0x74,0x64,0x2e,0x34,0x35,0x30,0x00,0x00,0x00,0x00, -0x0e,0x00,0x03,0x00,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x0f,0x00,0x0a,0x00,0x05,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x6d,0x61,0x69,0x6e,0x00,0x00,0x00,0x00,0x17,0x00,0x00,0x00, -0x25,0x00,0x00,0x00,0x33,0x00,0x00,0x00,0x63,0x00,0x00,0x00, -0x7b,0x00,0x00,0x00,0x10,0x00,0x06,0x00,0x04,0x00,0x00,0x00, -0x11,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x17,0x00,0x00,0x00, -0x0b,0x00,0x00,0x00,0x1a,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x23,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x23,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x23,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x23,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x33,0x00,0x00,0x00, -0x0b,0x00,0x00,0x00,0x1b,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x5a,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x5c,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x5f,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x5f,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x5f,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x50,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x60,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x54,0x00,0x00,0x00,0x48,0x00,0x04,0x00, -0x61,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x61,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, -0x61,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x63,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x63,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x78,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x48,0x00,0x04,0x00, -0x79,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x19,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x79,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, -0x79,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x7b,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x7b,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xfc,0x00,0x00,0x00, -0x0b,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x13,0x00,0x02,0x00, -0x02,0x00,0x00,0x00,0x21,0x00,0x03,0x00,0x03,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x15,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x10,0x00,0x00,0x00, -0x00,0x01,0x00,0x00,0x14,0x00,0x02,0x00,0x11,0x00,0x00,0x00, -0x15,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x17,0x00,0x04,0x00,0x15,0x00,0x00,0x00, -0x14,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x16,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x15,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x16,0x00,0x00,0x00,0x17,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x14,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x19,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x14,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0x1c,0x00,0x00,0x00, -0x00,0x01,0x00,0x00,0x1e,0x00,0x06,0x00,0x23,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x24,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x24,0x00,0x00,0x00,0x25,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x26,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x29,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x16,0x00,0x00,0x00,0x33,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x41,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x45,0x00,0x00,0x00,0x10,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x4b,0x00,0x00,0x00, -0x80,0x00,0x00,0x00,0x15,0x00,0x04,0x00,0x56,0x00,0x00,0x00, -0x08,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x14,0x00,0x00,0x00,0x59,0x00,0x00,0x00,0x10,0x00,0x00,0x00, -0x1c,0x00,0x04,0x00,0x5a,0x00,0x00,0x00,0x56,0x00,0x00,0x00, -0x59,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x14,0x00,0x00,0x00, -0x5b,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x1c,0x00,0x04,0x00, -0x5c,0x00,0x00,0x00,0x56,0x00,0x00,0x00,0x5b,0x00,0x00,0x00, -0x16,0x00,0x03,0x00,0x5d,0x00,0x00,0x00,0x10,0x00,0x00,0x00, -0x17,0x00,0x04,0x00,0x5e,0x00,0x00,0x00,0x5d,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x1e,0x00,0x05,0x00,0x5f,0x00,0x00,0x00, -0x5a,0x00,0x00,0x00,0x5c,0x00,0x00,0x00,0x5e,0x00,0x00,0x00, -0x1d,0x00,0x03,0x00,0x60,0x00,0x00,0x00,0x5f,0x00,0x00,0x00, -0x1e,0x00,0x03,0x00,0x61,0x00,0x00,0x00,0x60,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x62,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x61,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x62,0x00,0x00,0x00, -0x63,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x69,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x56,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x6f,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x70,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x5d,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x14,0x00,0x00,0x00,0x75,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x1d,0x00,0x03,0x00,0x78,0x00,0x00,0x00,0x5d,0x00,0x00,0x00, -0x1e,0x00,0x03,0x00,0x79,0x00,0x00,0x00,0x78,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x7a,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x79,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x7a,0x00,0x00,0x00, -0x7b,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x8c,0x00,0x00,0x00, -0x03,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x97,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xbc,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xdb,0x00,0x00,0x00, -0x60,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xe0,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x2c,0x00,0x06,0x00, -0x15,0x00,0x00,0x00,0xfc,0x00,0x00,0x00,0x5b,0x00,0x00,0x00, -0x75,0x00,0x00,0x00,0x75,0x00,0x00,0x00,0x2a,0x00,0x03,0x00, -0x11,0x00,0x00,0x00,0xff,0x00,0x00,0x00,0x29,0x00,0x03,0x00, -0x11,0x00,0x00,0x00,0x02,0x01,0x00,0x00,0x36,0x00,0x05,0x00, -0x02,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x03,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x05,0x00,0x00,0x00, -0xf7,0x00,0x03,0x00,0xfd,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0xfb,0x00,0x03,0x00,0x18,0x00,0x00,0x00,0xfe,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xfe,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x0a,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x0a,0x00,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x05,0x01,0x00,0x00, -0x09,0x00,0x00,0x00,0xfe,0x00,0x00,0x00,0xfb,0x00,0x00,0x00, -0x0d,0x00,0x00,0x00,0xb1,0x00,0x05,0x00,0x11,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x05,0x01,0x00,0x00,0x10,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0x0c,0x00,0x00,0x00,0x0d,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x12,0x00,0x00,0x00, -0x0b,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x0b,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x19,0x00,0x00,0x00, -0x1a,0x00,0x00,0x00,0x17,0x00,0x00,0x00,0x18,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0x1b,0x00,0x00,0x00, -0x1a,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x14,0x00,0x00,0x00, -0x1d,0x00,0x00,0x00,0x1b,0x00,0x00,0x00,0x1c,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0x1f,0x00,0x00,0x00, -0x05,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x14,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x1d,0x00,0x00,0x00,0x1f,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x26,0x00,0x00,0x00, -0x27,0x00,0x00,0x00,0x25,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x28,0x00,0x00,0x00, -0x27,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x26,0x00,0x00,0x00, -0x2a,0x00,0x00,0x00,0x25,0x00,0x00,0x00,0x29,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x2b,0x00,0x00,0x00, -0x2a,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x2c,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0x2b,0x00,0x00,0x00, -0x87,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x2d,0x00,0x00,0x00, -0x2c,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0xaf,0x00,0x05,0x00, -0x11,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x2d,0x00,0x00,0x00,0xf7,0x00,0x03,0x00,0x30,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x2e,0x00,0x00,0x00, -0x2f,0x00,0x00,0x00,0x30,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x2f,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x0c,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0x30,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x19,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0x33,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x14,0x00,0x00,0x00, -0x35,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x36,0x00,0x00,0x00,0x35,0x00,0x00,0x00, -0x87,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, -0x36,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x3e,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x3a,0x00,0x00,0x00,0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x3f,0x00,0x00,0x00,0x36,0x00,0x00,0x00,0x3e,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x43,0x00,0x00,0x00, -0x41,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x87,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x46,0x00,0x00,0x00,0x3f,0x00,0x00,0x00, -0x45,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x47,0x00,0x00,0x00,0x43,0x00,0x00,0x00,0x46,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x4a,0x00,0x00,0x00, -0x21,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x4d,0x00,0x00,0x00,0x4b,0x00,0x00,0x00, -0x3a,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x4e,0x00,0x00,0x00,0x4a,0x00,0x00,0x00,0x4d,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x50,0x00,0x00,0x00, -0x4e,0x00,0x00,0x00,0x3f,0x00,0x00,0x00,0x41,0x00,0x08,0x00, -0x69,0x00,0x00,0x00,0x6a,0x00,0x00,0x00,0x63,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x29,0x00,0x00,0x00, -0x36,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x56,0x00,0x00,0x00, -0x6b,0x00,0x00,0x00,0x6a,0x00,0x00,0x00,0x41,0x00,0x08,0x00, -0x70,0x00,0x00,0x00,0x71,0x00,0x00,0x00,0x63,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x6f,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x5d,0x00,0x00,0x00, -0x72,0x00,0x00,0x00,0x71,0x00,0x00,0x00,0x41,0x00,0x08,0x00, -0x70,0x00,0x00,0x00,0x76,0x00,0x00,0x00,0x63,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x6f,0x00,0x00,0x00, -0x75,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x5d,0x00,0x00,0x00, -0x77,0x00,0x00,0x00,0x76,0x00,0x00,0x00,0x41,0x00,0x08,0x00, -0x69,0x00,0x00,0x00,0x82,0x00,0x00,0x00,0x63,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x47,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x56,0x00,0x00,0x00, -0x83,0x00,0x00,0x00,0x82,0x00,0x00,0x00,0x71,0x00,0x04,0x00, -0x14,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x83,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x85,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x87,0x00,0x00,0x00,0x85,0x00,0x00,0x00,0x86,0x00,0x00,0x00, -0xc2,0x00,0x05,0x00,0x56,0x00,0x00,0x00,0x89,0x00,0x00,0x00, -0x6b,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x71,0x00,0x04,0x00, -0x14,0x00,0x00,0x00,0x8a,0x00,0x00,0x00,0x89,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x8b,0x00,0x00,0x00, -0x8a,0x00,0x00,0x00,0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x8d,0x00,0x00,0x00,0x8b,0x00,0x00,0x00,0x8c,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8e,0x00,0x00,0x00, -0x87,0x00,0x00,0x00,0x8d,0x00,0x00,0x00,0x6f,0x00,0x04,0x00, -0x5d,0x00,0x00,0x00,0x8f,0x00,0x00,0x00,0x8e,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x56,0x00,0x00,0x00,0x96,0x00,0x00,0x00, -0x82,0x00,0x00,0x00,0xc2,0x00,0x05,0x00,0x56,0x00,0x00,0x00, -0x98,0x00,0x00,0x00,0x96,0x00,0x00,0x00,0x97,0x00,0x00,0x00, -0x70,0x00,0x04,0x00,0x5d,0x00,0x00,0x00,0x99,0x00,0x00,0x00, -0x98,0x00,0x00,0x00,0x85,0x00,0x05,0x00,0x5d,0x00,0x00,0x00, -0x9a,0x00,0x00,0x00,0x77,0x00,0x00,0x00,0x99,0x00,0x00,0x00, -0x7f,0x00,0x04,0x00,0x5d,0x00,0x00,0x00,0x08,0x01,0x00,0x00, -0x9a,0x00,0x00,0x00,0x0c,0x00,0x08,0x00,0x5d,0x00,0x00,0x00, -0x9b,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00, -0x72,0x00,0x00,0x00,0x8f,0x00,0x00,0x00,0x08,0x01,0x00,0x00, -0x41,0x00,0x06,0x00,0x70,0x00,0x00,0x00,0x9c,0x00,0x00,0x00, -0x7b,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x50,0x00,0x00,0x00, -0x3e,0x00,0x03,0x00,0x9c,0x00,0x00,0x00,0x9b,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9e,0x00,0x00,0x00, -0x50,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xa2,0x00,0x00,0x00,0x47,0x00,0x00,0x00, -0x6f,0x00,0x00,0x00,0x41,0x00,0x08,0x00,0x69,0x00,0x00,0x00, -0xa3,0x00,0x00,0x00,0x63,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x21,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0xa2,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x56,0x00,0x00,0x00,0xa4,0x00,0x00,0x00, -0xa3,0x00,0x00,0x00,0x71,0x00,0x04,0x00,0x14,0x00,0x00,0x00, -0xa5,0x00,0x00,0x00,0xa4,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xa6,0x00,0x00,0x00,0xa5,0x00,0x00,0x00, -0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa7,0x00,0x00,0x00, -0xa6,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0xc2,0x00,0x05,0x00, -0x56,0x00,0x00,0x00,0xa9,0x00,0x00,0x00,0x6b,0x00,0x00,0x00, -0x6f,0x00,0x00,0x00,0x71,0x00,0x04,0x00,0x14,0x00,0x00,0x00, -0xaa,0x00,0x00,0x00,0xa9,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xab,0x00,0x00,0x00,0xaa,0x00,0x00,0x00, -0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xac,0x00,0x00,0x00, -0xab,0x00,0x00,0x00,0x8c,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xad,0x00,0x00,0x00,0xa7,0x00,0x00,0x00, -0xac,0x00,0x00,0x00,0x6f,0x00,0x04,0x00,0x5d,0x00,0x00,0x00, -0xae,0x00,0x00,0x00,0xad,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x56,0x00,0x00,0x00,0xb5,0x00,0x00,0x00,0xa3,0x00,0x00,0x00, -0xc2,0x00,0x05,0x00,0x56,0x00,0x00,0x00,0xb6,0x00,0x00,0x00, -0xb5,0x00,0x00,0x00,0x97,0x00,0x00,0x00,0x70,0x00,0x04,0x00, -0x5d,0x00,0x00,0x00,0xb7,0x00,0x00,0x00,0xb6,0x00,0x00,0x00, -0x85,0x00,0x05,0x00,0x5d,0x00,0x00,0x00,0xb8,0x00,0x00,0x00, -0x77,0x00,0x00,0x00,0xb7,0x00,0x00,0x00,0x7f,0x00,0x04,0x00, -0x5d,0x00,0x00,0x00,0x09,0x01,0x00,0x00,0xb8,0x00,0x00,0x00, -0x0c,0x00,0x08,0x00,0x5d,0x00,0x00,0x00,0xb9,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x72,0x00,0x00,0x00, -0xae,0x00,0x00,0x00,0x09,0x01,0x00,0x00,0x41,0x00,0x06,0x00, -0x70,0x00,0x00,0x00,0xba,0x00,0x00,0x00,0x7b,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x9e,0x00,0x00,0x00,0x3e,0x00,0x03,0x00, -0xba,0x00,0x00,0x00,0xb9,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xbd,0x00,0x00,0x00,0x50,0x00,0x00,0x00, -0xbc,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xc1,0x00,0x00,0x00,0x47,0x00,0x00,0x00,0x97,0x00,0x00,0x00, -0x41,0x00,0x08,0x00,0x69,0x00,0x00,0x00,0xc2,0x00,0x00,0x00, -0x63,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0xc1,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x56,0x00,0x00,0x00,0xc3,0x00,0x00,0x00,0xc2,0x00,0x00,0x00, -0x71,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0xc4,0x00,0x00,0x00, -0xc3,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xc5,0x00,0x00,0x00,0xc4,0x00,0x00,0x00,0xc7,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xc6,0x00,0x00,0x00,0xc5,0x00,0x00,0x00, -0x86,0x00,0x00,0x00,0xc2,0x00,0x05,0x00,0x56,0x00,0x00,0x00, -0xc8,0x00,0x00,0x00,0x6b,0x00,0x00,0x00,0x97,0x00,0x00,0x00, -0x71,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0xc9,0x00,0x00,0x00, -0xc8,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xca,0x00,0x00,0x00,0xc9,0x00,0x00,0x00,0xc7,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xcb,0x00,0x00,0x00,0xca,0x00,0x00,0x00, -0x8c,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xcc,0x00,0x00,0x00,0xc6,0x00,0x00,0x00,0xcb,0x00,0x00,0x00, -0x6f,0x00,0x04,0x00,0x5d,0x00,0x00,0x00,0xcd,0x00,0x00,0x00, -0xcc,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x56,0x00,0x00,0x00, -0xd4,0x00,0x00,0x00,0xc2,0x00,0x00,0x00,0xc2,0x00,0x05,0x00, -0x56,0x00,0x00,0x00,0xd5,0x00,0x00,0x00,0xd4,0x00,0x00,0x00, -0x97,0x00,0x00,0x00,0x70,0x00,0x04,0x00,0x5d,0x00,0x00,0x00, -0xd6,0x00,0x00,0x00,0xd5,0x00,0x00,0x00,0x85,0x00,0x05,0x00, -0x5d,0x00,0x00,0x00,0xd7,0x00,0x00,0x00,0x77,0x00,0x00,0x00, -0xd6,0x00,0x00,0x00,0x7f,0x00,0x04,0x00,0x5d,0x00,0x00,0x00, -0x0a,0x01,0x00,0x00,0xd7,0x00,0x00,0x00,0x0c,0x00,0x08,0x00, -0x5d,0x00,0x00,0x00,0xd8,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x32,0x00,0x00,0x00,0x72,0x00,0x00,0x00,0xcd,0x00,0x00,0x00, -0x0a,0x01,0x00,0x00,0x41,0x00,0x06,0x00,0x70,0x00,0x00,0x00, -0xd9,0x00,0x00,0x00,0x7b,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0xbd,0x00,0x00,0x00,0x3e,0x00,0x03,0x00,0xd9,0x00,0x00,0x00, -0xd8,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xdc,0x00,0x00,0x00,0x50,0x00,0x00,0x00,0xdb,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xe1,0x00,0x00,0x00, -0x47,0x00,0x00,0x00,0xe0,0x00,0x00,0x00,0x41,0x00,0x08,0x00, -0x69,0x00,0x00,0x00,0xe2,0x00,0x00,0x00,0x63,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0xe1,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x56,0x00,0x00,0x00, -0xe3,0x00,0x00,0x00,0xe2,0x00,0x00,0x00,0x71,0x00,0x04,0x00, -0x14,0x00,0x00,0x00,0xe4,0x00,0x00,0x00,0xe3,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xe5,0x00,0x00,0x00, -0xe4,0x00,0x00,0x00,0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xe6,0x00,0x00,0x00,0xe5,0x00,0x00,0x00,0x86,0x00,0x00,0x00, -0xc2,0x00,0x05,0x00,0x56,0x00,0x00,0x00,0xe8,0x00,0x00,0x00, -0x6b,0x00,0x00,0x00,0xe0,0x00,0x00,0x00,0x71,0x00,0x04,0x00, -0x14,0x00,0x00,0x00,0xe9,0x00,0x00,0x00,0xe8,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xea,0x00,0x00,0x00, -0xe9,0x00,0x00,0x00,0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xeb,0x00,0x00,0x00,0xea,0x00,0x00,0x00,0x8c,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xec,0x00,0x00,0x00, -0xe6,0x00,0x00,0x00,0xeb,0x00,0x00,0x00,0x6f,0x00,0x04,0x00, -0x5d,0x00,0x00,0x00,0xed,0x00,0x00,0x00,0xec,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x56,0x00,0x00,0x00,0xf4,0x00,0x00,0x00, -0xe2,0x00,0x00,0x00,0xc2,0x00,0x05,0x00,0x56,0x00,0x00,0x00, -0xf5,0x00,0x00,0x00,0xf4,0x00,0x00,0x00,0x97,0x00,0x00,0x00, -0x70,0x00,0x04,0x00,0x5d,0x00,0x00,0x00,0xf6,0x00,0x00,0x00, -0xf5,0x00,0x00,0x00,0x85,0x00,0x05,0x00,0x5d,0x00,0x00,0x00, -0xf7,0x00,0x00,0x00,0x77,0x00,0x00,0x00,0xf6,0x00,0x00,0x00, -0x7f,0x00,0x04,0x00,0x5d,0x00,0x00,0x00,0x0b,0x01,0x00,0x00, -0xf7,0x00,0x00,0x00,0x0c,0x00,0x08,0x00,0x5d,0x00,0x00,0x00, -0xf8,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00, -0x72,0x00,0x00,0x00,0xed,0x00,0x00,0x00,0x0b,0x01,0x00,0x00, -0x41,0x00,0x06,0x00,0x70,0x00,0x00,0x00,0xf9,0x00,0x00,0x00, -0x7b,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0xdc,0x00,0x00,0x00, -0x3e,0x00,0x03,0x00,0xf9,0x00,0x00,0x00,0xf8,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x0d,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x0d,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xfb,0x00,0x00,0x00,0x05,0x01,0x00,0x00,0x29,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x0a,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x0c,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x11,0x00,0x00,0x00, -0x06,0x01,0x00,0x00,0xff,0x00,0x00,0x00,0x0a,0x00,0x00,0x00, -0x02,0x01,0x00,0x00,0x2f,0x00,0x00,0x00,0xf7,0x00,0x03,0x00, -0x03,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x06,0x01,0x00,0x00,0xfd,0x00,0x00,0x00,0x03,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x03,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0xfd,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xfd,0x00,0x00,0x00, -0xfd,0x00,0x01,0x00,0x38,0x00,0x01,0x00, -}; -const uint64_t dequant_q2_K_len = 3956; - -unsigned char dequant_q2_K_fp32_data[] = { -0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, 0x13,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, 0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x27,0x00,0x00,0x00, 0x11,0x00,0x02,0x00,0x51,0x11,0x00,0x00,0x11,0x00,0x02,0x00, @@ -1869,414 +1385,10 @@ unsigned char dequant_q2_K_fp32_data[] = { 0x04,0x01,0x00,0x00,0xfd,0x00,0x01,0x00,0x38,0x00,0x01,0x00, }; -const uint64_t dequant_q2_K_fp32_len = 4056; +const uint64_t dequant_q2_K_len = 4056; unsigned char dequant_q3_K_data[] = { 0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, -0x3f,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, -0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x09,0x00,0x00,0x00, -0x11,0x00,0x02,0x00,0x27,0x00,0x00,0x00,0x11,0x00,0x02,0x00, -0x51,0x11,0x00,0x00,0x11,0x00,0x02,0x00,0x60,0x11,0x00,0x00, -0x0b,0x00,0x06,0x00,0x01,0x00,0x00,0x00,0x47,0x4c,0x53,0x4c, -0x2e,0x73,0x74,0x64,0x2e,0x34,0x35,0x30,0x00,0x00,0x00,0x00, -0x0e,0x00,0x03,0x00,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x0f,0x00,0x0a,0x00,0x05,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x6d,0x61,0x69,0x6e,0x00,0x00,0x00,0x00,0x17,0x00,0x00,0x00, -0x25,0x00,0x00,0x00,0x33,0x00,0x00,0x00,0x7b,0x00,0x00,0x00, -0x07,0x01,0x00,0x00,0x10,0x00,0x06,0x00,0x04,0x00,0x00,0x00, -0x11,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x17,0x00,0x00,0x00, -0x0b,0x00,0x00,0x00,0x1a,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x23,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x23,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x23,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x23,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x33,0x00,0x00,0x00, -0x0b,0x00,0x00,0x00,0x1b,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x71,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x73,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x75,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x77,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x77,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x77,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x77,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x78,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x6e,0x00,0x00,0x00,0x48,0x00,0x04,0x00, -0x79,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x79,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, -0x79,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x7b,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x7b,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x04,0x01,0x00,0x00, -0x06,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x48,0x00,0x04,0x00, -0x05,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x19,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x05,0x01,0x00,0x00,0x00,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, -0x05,0x01,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x07,0x01,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x07,0x01,0x00,0x00,0x21,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x2c,0x01,0x00,0x00, -0x0b,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x13,0x00,0x02,0x00, -0x02,0x00,0x00,0x00,0x21,0x00,0x03,0x00,0x03,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x15,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x10,0x00,0x00,0x00, -0x00,0x01,0x00,0x00,0x14,0x00,0x02,0x00,0x11,0x00,0x00,0x00, -0x15,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x17,0x00,0x04,0x00,0x15,0x00,0x00,0x00, -0x14,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x16,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x15,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x16,0x00,0x00,0x00,0x17,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x14,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x19,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x14,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0x1c,0x00,0x00,0x00, -0x00,0x01,0x00,0x00,0x1e,0x00,0x06,0x00,0x23,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x24,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x24,0x00,0x00,0x00,0x25,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x26,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x29,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x16,0x00,0x00,0x00,0x33,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x37,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x3b,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x41,0x00,0x00,0x00,0x10,0x00,0x00,0x00, -0x15,0x00,0x04,0x00,0x52,0x00,0x00,0x00,0x08,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x15,0x00,0x04,0x00,0x5a,0x00,0x00,0x00, -0x08,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x5e,0x00,0x00,0x00,0x08,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0x70,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x1c,0x00,0x04,0x00,0x71,0x00,0x00,0x00, -0x52,0x00,0x00,0x00,0x70,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x14,0x00,0x00,0x00,0x72,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x1c,0x00,0x04,0x00,0x73,0x00,0x00,0x00,0x52,0x00,0x00,0x00, -0x72,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x14,0x00,0x00,0x00, -0x74,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x1c,0x00,0x04,0x00, -0x75,0x00,0x00,0x00,0x52,0x00,0x00,0x00,0x74,0x00,0x00,0x00, -0x16,0x00,0x03,0x00,0x76,0x00,0x00,0x00,0x10,0x00,0x00,0x00, -0x1e,0x00,0x06,0x00,0x77,0x00,0x00,0x00,0x71,0x00,0x00,0x00, -0x73,0x00,0x00,0x00,0x75,0x00,0x00,0x00,0x76,0x00,0x00,0x00, -0x1d,0x00,0x03,0x00,0x78,0x00,0x00,0x00,0x77,0x00,0x00,0x00, -0x1e,0x00,0x03,0x00,0x79,0x00,0x00,0x00,0x78,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x7a,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x79,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x7a,0x00,0x00,0x00, -0x7b,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x7f,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x52,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0x0f,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x8e,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xad,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xd3,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0xe1,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x76,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xe8,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xef,0x00,0x00,0x00, -0x80,0x00,0x00,0x00,0x1d,0x00,0x03,0x00,0x04,0x01,0x00,0x00, -0x76,0x00,0x00,0x00,0x1e,0x00,0x03,0x00,0x05,0x01,0x00,0x00, -0x04,0x01,0x00,0x00,0x20,0x00,0x04,0x00,0x06,0x01,0x00,0x00, -0x0c,0x00,0x00,0x00,0x05,0x01,0x00,0x00,0x3b,0x00,0x04,0x00, -0x06,0x01,0x00,0x00,0x07,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0x2b,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0x2c,0x00,0x06,0x00,0x15,0x00,0x00,0x00, -0x2c,0x01,0x00,0x00,0x72,0x00,0x00,0x00,0x2b,0x01,0x00,0x00, -0x2b,0x01,0x00,0x00,0x2a,0x00,0x03,0x00,0x11,0x00,0x00,0x00, -0x2f,0x01,0x00,0x00,0x29,0x00,0x03,0x00,0x11,0x00,0x00,0x00, -0x32,0x01,0x00,0x00,0x36,0x00,0x05,0x00,0x02,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x03,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0x05,0x00,0x00,0x00,0xf7,0x00,0x03,0x00, -0x2d,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0xfb,0x00,0x03,0x00, -0x18,0x00,0x00,0x00,0x2e,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x2e,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x0a,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0x0a,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0x35,0x01,0x00,0x00,0x09,0x00,0x00,0x00, -0x2e,0x01,0x00,0x00,0x2a,0x01,0x00,0x00,0x0d,0x00,0x00,0x00, -0xb1,0x00,0x05,0x00,0x11,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x35,0x01,0x00,0x00,0x10,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0x0c,0x00,0x00,0x00,0x0d,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x12,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x0b,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x19,0x00,0x00,0x00,0x1a,0x00,0x00,0x00, -0x17,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x14,0x00,0x00,0x00,0x1b,0x00,0x00,0x00,0x1a,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x14,0x00,0x00,0x00,0x1d,0x00,0x00,0x00, -0x1b,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x14,0x00,0x00,0x00,0x1f,0x00,0x00,0x00,0x35,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x14,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x1d,0x00,0x00,0x00,0x1f,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x26,0x00,0x00,0x00,0x27,0x00,0x00,0x00, -0x25,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0x27,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x26,0x00,0x00,0x00,0x2a,0x00,0x00,0x00, -0x25,0x00,0x00,0x00,0x29,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x2b,0x00,0x00,0x00,0x2a,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x2c,0x00,0x00,0x00, -0x28,0x00,0x00,0x00,0x2b,0x00,0x00,0x00,0x87,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x2d,0x00,0x00,0x00,0x2c,0x00,0x00,0x00, -0x10,0x00,0x00,0x00,0xaf,0x00,0x05,0x00,0x11,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x2d,0x00,0x00,0x00, -0xf7,0x00,0x03,0x00,0x30,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x2e,0x00,0x00,0x00,0x2f,0x00,0x00,0x00, -0x30,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x2f,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x0c,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x30,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x19,0x00,0x00,0x00, -0x34,0x00,0x00,0x00,0x33,0x00,0x00,0x00,0x18,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0x35,0x00,0x00,0x00, -0x34,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x36,0x00,0x00,0x00,0x35,0x00,0x00,0x00,0x87,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x38,0x00,0x00,0x00,0x36,0x00,0x00,0x00, -0x37,0x00,0x00,0x00,0x87,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x3c,0x00,0x00,0x00,0x38,0x00,0x00,0x00,0x3b,0x00,0x00,0x00, -0x8b,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3f,0x00,0x00,0x00, -0x38,0x00,0x00,0x00,0x3b,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x43,0x00,0x00,0x00,0x41,0x00,0x00,0x00, -0x3f,0x00,0x00,0x00,0x8b,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x47,0x00,0x00,0x00,0x36,0x00,0x00,0x00,0x37,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x48,0x00,0x00,0x00, -0x37,0x00,0x00,0x00,0x47,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x49,0x00,0x00,0x00,0x43,0x00,0x00,0x00, -0x48,0x00,0x00,0x00,0x87,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x4c,0x00,0x00,0x00,0x3c,0x00,0x00,0x00,0x37,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x50,0x00,0x00,0x00, -0x37,0x00,0x00,0x00,0x4c,0x00,0x00,0x00,0x82,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x51,0x00,0x00,0x00,0x3c,0x00,0x00,0x00, -0x50,0x00,0x00,0x00,0xc4,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x59,0x00,0x00,0x00,0x29,0x00,0x00,0x00,0x3c,0x00,0x00,0x00, -0x72,0x00,0x04,0x00,0x5a,0x00,0x00,0x00,0x5b,0x00,0x00,0x00, -0x59,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x52,0x00,0x00,0x00, -0x5c,0x00,0x00,0x00,0x5b,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x5e,0x00,0x00,0x00, -0x4c,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x3b,0x00,0x00,0x00,0x51,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x63,0x00,0x00,0x00, -0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x65,0x00,0x00,0x00,0x63,0x00,0x00,0x00, -0x3f,0x00,0x00,0x00,0xb1,0x00,0x05,0x00,0x11,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x65,0x00,0x00,0x00,0x37,0x00,0x00,0x00, -0xf7,0x00,0x03,0x00,0x6f,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x6c,0x00,0x00,0x00,0x6e,0x00,0x00,0x00, -0x92,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x6e,0x00,0x00,0x00, -0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x7e,0x00,0x00,0x00, -0x65,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x41,0x00,0x08,0x00, -0x7f,0x00,0x00,0x00,0x80,0x00,0x00,0x00,0x7b,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x3b,0x00,0x00,0x00, -0x7e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x52,0x00,0x00,0x00, -0x81,0x00,0x00,0x00,0x80,0x00,0x00,0x00,0x71,0x00,0x04,0x00, -0x14,0x00,0x00,0x00,0x82,0x00,0x00,0x00,0x81,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x83,0x00,0x00,0x00, -0x82,0x00,0x00,0x00,0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x85,0x00,0x00,0x00,0x83,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x88,0x00,0x00,0x00, -0x65,0x00,0x00,0x00,0x5e,0x00,0x00,0x00,0x41,0x00,0x08,0x00, -0x7f,0x00,0x00,0x00,0x89,0x00,0x00,0x00,0x7b,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x3b,0x00,0x00,0x00, -0x88,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x52,0x00,0x00,0x00, -0x8a,0x00,0x00,0x00,0x89,0x00,0x00,0x00,0xc2,0x00,0x05,0x00, -0x52,0x00,0x00,0x00,0x8b,0x00,0x00,0x00,0x8a,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x71,0x00,0x04,0x00,0x14,0x00,0x00,0x00, -0x8c,0x00,0x00,0x00,0x8b,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x8d,0x00,0x00,0x00,0x8c,0x00,0x00,0x00, -0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8f,0x00,0x00,0x00, -0x8d,0x00,0x00,0x00,0x8e,0x00,0x00,0x00,0xc4,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x90,0x00,0x00,0x00,0x8f,0x00,0x00,0x00, -0x37,0x00,0x00,0x00,0xc5,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x91,0x00,0x00,0x00,0x85,0x00,0x00,0x00,0x90,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x6f,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x92,0x00,0x00,0x00,0xb1,0x00,0x05,0x00,0x11,0x00,0x00,0x00, -0x94,0x00,0x00,0x00,0x65,0x00,0x00,0x00,0x5e,0x00,0x00,0x00, -0xf7,0x00,0x03,0x00,0x97,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x94,0x00,0x00,0x00,0x96,0x00,0x00,0x00, -0xab,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x96,0x00,0x00,0x00, -0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9a,0x00,0x00,0x00, -0x65,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x41,0x00,0x08,0x00, -0x7f,0x00,0x00,0x00,0x9b,0x00,0x00,0x00,0x7b,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x3b,0x00,0x00,0x00, -0x9a,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x52,0x00,0x00,0x00, -0x9c,0x00,0x00,0x00,0x9b,0x00,0x00,0x00,0x71,0x00,0x04,0x00, -0x14,0x00,0x00,0x00,0x9d,0x00,0x00,0x00,0x9c,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x9e,0x00,0x00,0x00, -0x9d,0x00,0x00,0x00,0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x9f,0x00,0x00,0x00,0x9e,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa2,0x00,0x00,0x00, -0x65,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x41,0x00,0x08,0x00, -0x7f,0x00,0x00,0x00,0xa3,0x00,0x00,0x00,0x7b,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x3b,0x00,0x00,0x00, -0xa2,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x52,0x00,0x00,0x00, -0xa4,0x00,0x00,0x00,0xa3,0x00,0x00,0x00,0xc2,0x00,0x05,0x00, -0x52,0x00,0x00,0x00,0xa5,0x00,0x00,0x00,0xa4,0x00,0x00,0x00, -0x3b,0x00,0x00,0x00,0x71,0x00,0x04,0x00,0x14,0x00,0x00,0x00, -0xa6,0x00,0x00,0x00,0xa5,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xa7,0x00,0x00,0x00,0xa6,0x00,0x00,0x00, -0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa8,0x00,0x00,0x00, -0xa7,0x00,0x00,0x00,0x8e,0x00,0x00,0x00,0xc4,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xa9,0x00,0x00,0x00,0xa8,0x00,0x00,0x00, -0x37,0x00,0x00,0x00,0xc5,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xaa,0x00,0x00,0x00,0x9f,0x00,0x00,0x00,0xa9,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x97,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xab,0x00,0x00,0x00,0xb1,0x00,0x05,0x00,0x11,0x00,0x00,0x00, -0xae,0x00,0x00,0x00,0x65,0x00,0x00,0x00,0xad,0x00,0x00,0x00, -0xf7,0x00,0x03,0x00,0xb1,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xae,0x00,0x00,0x00,0xb0,0x00,0x00,0x00, -0xc5,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xb0,0x00,0x00,0x00, -0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xb4,0x00,0x00,0x00, -0x65,0x00,0x00,0x00,0x5e,0x00,0x00,0x00,0x41,0x00,0x08,0x00, -0x7f,0x00,0x00,0x00,0xb5,0x00,0x00,0x00,0x7b,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x3b,0x00,0x00,0x00, -0xb4,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x52,0x00,0x00,0x00, -0xb6,0x00,0x00,0x00,0xb5,0x00,0x00,0x00,0xc2,0x00,0x05,0x00, -0x52,0x00,0x00,0x00,0xb7,0x00,0x00,0x00,0xb6,0x00,0x00,0x00, -0x37,0x00,0x00,0x00,0x71,0x00,0x04,0x00,0x14,0x00,0x00,0x00, -0xb8,0x00,0x00,0x00,0xb7,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xb9,0x00,0x00,0x00,0xb8,0x00,0x00,0x00, -0x41,0x00,0x08,0x00,0x7f,0x00,0x00,0x00,0xbd,0x00,0x00,0x00, -0x7b,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x3b,0x00,0x00,0x00,0x65,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x52,0x00,0x00,0x00,0xbe,0x00,0x00,0x00,0xbd,0x00,0x00,0x00, -0xc2,0x00,0x05,0x00,0x52,0x00,0x00,0x00,0xbf,0x00,0x00,0x00, -0xbe,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x71,0x00,0x04,0x00, -0x14,0x00,0x00,0x00,0xc0,0x00,0x00,0x00,0xbf,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xc1,0x00,0x00,0x00, -0xc0,0x00,0x00,0x00,0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xc2,0x00,0x00,0x00,0xc1,0x00,0x00,0x00,0x8e,0x00,0x00,0x00, -0xc4,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xc3,0x00,0x00,0x00, -0xc2,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0xc5,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xc4,0x00,0x00,0x00,0xb9,0x00,0x00,0x00, -0xc3,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xb1,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xc5,0x00,0x00,0x00,0x82,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xc8,0x00,0x00,0x00,0x65,0x00,0x00,0x00, -0x5e,0x00,0x00,0x00,0x41,0x00,0x08,0x00,0x7f,0x00,0x00,0x00, -0xc9,0x00,0x00,0x00,0x7b,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x21,0x00,0x00,0x00,0x3b,0x00,0x00,0x00,0xc8,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x52,0x00,0x00,0x00,0xca,0x00,0x00,0x00, -0xc9,0x00,0x00,0x00,0xc2,0x00,0x05,0x00,0x52,0x00,0x00,0x00, -0xcb,0x00,0x00,0x00,0xca,0x00,0x00,0x00,0x37,0x00,0x00,0x00, -0x71,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0xcc,0x00,0x00,0x00, -0xcb,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xcd,0x00,0x00,0x00,0xcc,0x00,0x00,0x00,0x82,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xd0,0x00,0x00,0x00,0x65,0x00,0x00,0x00, -0x37,0x00,0x00,0x00,0x41,0x00,0x08,0x00,0x7f,0x00,0x00,0x00, -0xd1,0x00,0x00,0x00,0x7b,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x21,0x00,0x00,0x00,0x3b,0x00,0x00,0x00,0xd0,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x52,0x00,0x00,0x00,0xd2,0x00,0x00,0x00, -0xd1,0x00,0x00,0x00,0xc2,0x00,0x05,0x00,0x52,0x00,0x00,0x00, -0xd4,0x00,0x00,0x00,0xd2,0x00,0x00,0x00,0xd3,0x00,0x00,0x00, -0x71,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0xd5,0x00,0x00,0x00, -0xd4,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xd6,0x00,0x00,0x00,0xd5,0x00,0x00,0x00,0xc7,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xd7,0x00,0x00,0x00,0xd6,0x00,0x00,0x00, -0x8e,0x00,0x00,0x00,0xc4,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xd8,0x00,0x00,0x00,0xd7,0x00,0x00,0x00,0x37,0x00,0x00,0x00, -0xc5,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xd9,0x00,0x00,0x00, -0xcd,0x00,0x00,0x00,0xd8,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xb1,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xb1,0x00,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x36,0x01,0x00,0x00, -0xc4,0x00,0x00,0x00,0xb0,0x00,0x00,0x00,0xd9,0x00,0x00,0x00, -0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x97,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0x97,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0x37,0x01,0x00,0x00,0xaa,0x00,0x00,0x00, -0x96,0x00,0x00,0x00,0x36,0x01,0x00,0x00,0xb1,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x6f,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x6f,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0x38,0x01,0x00,0x00,0x91,0x00,0x00,0x00,0x6e,0x00,0x00,0x00, -0x37,0x01,0x00,0x00,0x97,0x00,0x00,0x00,0x72,0x00,0x04,0x00, -0x5a,0x00,0x00,0x00,0xdd,0x00,0x00,0x00,0x38,0x01,0x00,0x00, -0x41,0x00,0x07,0x00,0xe1,0x00,0x00,0x00,0xe2,0x00,0x00,0x00, -0x7b,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x8e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x76,0x00,0x00,0x00, -0xe3,0x00,0x00,0x00,0xe2,0x00,0x00,0x00,0x72,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xe7,0x00,0x00,0x00,0xdd,0x00,0x00,0x00, -0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xe9,0x00,0x00,0x00, -0xe7,0x00,0x00,0x00,0xe8,0x00,0x00,0x00,0x6f,0x00,0x04,0x00, -0x76,0x00,0x00,0x00,0xea,0x00,0x00,0x00,0xe9,0x00,0x00,0x00, -0x85,0x00,0x05,0x00,0x76,0x00,0x00,0x00,0xeb,0x00,0x00,0x00, -0xe3,0x00,0x00,0x00,0xea,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xee,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x10,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xf1,0x00,0x00,0x00,0xef,0x00,0x00,0x00,0x4c,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xf2,0x00,0x00,0x00, -0xee,0x00,0x00,0x00,0xf1,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xf4,0x00,0x00,0x00,0xe8,0x00,0x00,0x00, -0x51,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xf5,0x00,0x00,0x00,0xf2,0x00,0x00,0x00,0xf4,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xf8,0x00,0x00,0x00, -0xe8,0x00,0x00,0x00,0x4c,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xfb,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xfb,0x00,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x39,0x01,0x00,0x00, -0x49,0x00,0x00,0x00,0x6f,0x00,0x00,0x00,0x28,0x01,0x00,0x00, -0xfc,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x02,0x01,0x00,0x00,0x49,0x00,0x00,0x00,0x37,0x00,0x00,0x00, -0xb1,0x00,0x05,0x00,0x11,0x00,0x00,0x00,0x03,0x01,0x00,0x00, -0x39,0x01,0x00,0x00,0x02,0x01,0x00,0x00,0xf6,0x00,0x04,0x00, -0xfd,0x00,0x00,0x00,0xfc,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x03,0x01,0x00,0x00,0xfc,0x00,0x00,0x00, -0xfd,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xfc,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x0a,0x01,0x00,0x00, -0xf5,0x00,0x00,0x00,0x39,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x0f,0x01,0x00,0x00,0xf8,0x00,0x00,0x00, -0x39,0x01,0x00,0x00,0x41,0x00,0x08,0x00,0x7f,0x00,0x00,0x00, -0x10,0x01,0x00,0x00,0x7b,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x21,0x00,0x00,0x00,0x29,0x00,0x00,0x00,0x0f,0x01,0x00,0x00, -0x3d,0x00,0x04,0x00,0x52,0x00,0x00,0x00,0x11,0x01,0x00,0x00, -0x10,0x01,0x00,0x00,0xc2,0x00,0x05,0x00,0x52,0x00,0x00,0x00, -0x13,0x01,0x00,0x00,0x11,0x01,0x00,0x00,0x62,0x00,0x00,0x00, -0x71,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0x14,0x01,0x00,0x00, -0x13,0x01,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x15,0x01,0x00,0x00,0x14,0x01,0x00,0x00,0xc7,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x16,0x01,0x00,0x00,0x15,0x01,0x00,0x00, -0x8e,0x00,0x00,0x00,0x72,0x00,0x04,0x00,0x5a,0x00,0x00,0x00, -0x17,0x01,0x00,0x00,0x16,0x01,0x00,0x00,0x72,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x18,0x01,0x00,0x00,0x17,0x01,0x00,0x00, -0x41,0x00,0x08,0x00,0x7f,0x00,0x00,0x00,0x1b,0x01,0x00,0x00, -0x7b,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x39,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, -0x52,0x00,0x00,0x00,0x1c,0x01,0x00,0x00,0x1b,0x01,0x00,0x00, -0xc7,0x00,0x05,0x00,0x52,0x00,0x00,0x00,0x1e,0x01,0x00,0x00, -0x1c,0x01,0x00,0x00,0x5c,0x00,0x00,0x00,0x71,0x00,0x04,0x00, -0x14,0x00,0x00,0x00,0x1f,0x01,0x00,0x00,0x1e,0x01,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x20,0x01,0x00,0x00, -0x1f,0x01,0x00,0x00,0xab,0x00,0x05,0x00,0x11,0x00,0x00,0x00, -0x21,0x01,0x00,0x00,0x20,0x01,0x00,0x00,0x09,0x00,0x00,0x00, -0xa9,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x22,0x01,0x00,0x00, -0x21,0x01,0x00,0x00,0x09,0x00,0x00,0x00,0x37,0x00,0x00,0x00, -0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x23,0x01,0x00,0x00, -0x18,0x01,0x00,0x00,0x22,0x01,0x00,0x00,0x6f,0x00,0x04,0x00, -0x76,0x00,0x00,0x00,0x24,0x01,0x00,0x00,0x23,0x01,0x00,0x00, -0x85,0x00,0x05,0x00,0x76,0x00,0x00,0x00,0x25,0x01,0x00,0x00, -0xeb,0x00,0x00,0x00,0x24,0x01,0x00,0x00,0x41,0x00,0x06,0x00, -0xe1,0x00,0x00,0x00,0x26,0x01,0x00,0x00,0x07,0x01,0x00,0x00, -0x09,0x00,0x00,0x00,0x0a,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0x26,0x01,0x00,0x00,0x25,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x28,0x01,0x00,0x00,0x39,0x01,0x00,0x00, -0x29,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xfb,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xfd,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x0d,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x0d,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x2a,0x01,0x00,0x00, -0x35,0x01,0x00,0x00,0x29,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x0a,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x0c,0x00,0x00,0x00, -0xf5,0x00,0x07,0x00,0x11,0x00,0x00,0x00,0x3e,0x01,0x00,0x00, -0x2f,0x01,0x00,0x00,0x0a,0x00,0x00,0x00,0x32,0x01,0x00,0x00, -0x2f,0x00,0x00,0x00,0xf7,0x00,0x03,0x00,0x33,0x01,0x00,0x00, -0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x3e,0x01,0x00,0x00, -0x2d,0x01,0x00,0x00,0x33,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x33,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x2d,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x2d,0x01,0x00,0x00,0xfd,0x00,0x01,0x00, -0x38,0x00,0x01,0x00, -}; -const uint64_t dequant_q3_K_len = 4792; - -unsigned char dequant_q3_K_fp32_data[] = { -0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, 0x42,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, 0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x27,0x00,0x00,0x00, 0x11,0x00,0x02,0x00,0x51,0x11,0x00,0x00,0x11,0x00,0x02,0x00, @@ -2680,709 +1792,10 @@ unsigned char dequant_q3_K_fp32_data[] = { 0xf8,0x00,0x02,0x00,0x30,0x01,0x00,0x00,0xfd,0x00,0x01,0x00, 0x38,0x00,0x01,0x00, }; -const uint64_t dequant_q3_K_fp32_len = 4828; +const uint64_t dequant_q3_K_len = 4828; unsigned char dequant_q4_0_data[] = { 0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, -0xf7,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, -0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x09,0x00,0x00,0x00, -0x11,0x00,0x02,0x00,0x27,0x00,0x00,0x00,0x11,0x00,0x02,0x00, -0x51,0x11,0x00,0x00,0x11,0x00,0x02,0x00,0x60,0x11,0x00,0x00, -0x0b,0x00,0x06,0x00,0x01,0x00,0x00,0x00,0x47,0x4c,0x53,0x4c, -0x2e,0x73,0x74,0x64,0x2e,0x34,0x35,0x30,0x00,0x00,0x00,0x00, -0x0e,0x00,0x03,0x00,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x0f,0x00,0x09,0x00,0x05,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x6d,0x61,0x69,0x6e,0x00,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x54,0x00,0x00,0x00,0x77,0x00,0x00,0x00, -0x10,0x00,0x06,0x00,0x04,0x00,0x00,0x00,0x11,0x00,0x00,0x00, -0x00,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x0c,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x1c,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x14,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x14,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x14,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x08,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x14,0x00,0x00,0x00, -0x03,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x47,0x00,0x03,0x00,0x14,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x4f,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x50,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x50,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x51,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x48,0x00,0x04,0x00,0x52,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x52,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x03,0x00,0x52,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x54,0x00,0x00,0x00,0x22,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x54,0x00,0x00,0x00, -0x21,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x74,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x48,0x00,0x04,0x00,0x75,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x19,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x75,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x03,0x00,0x75,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x77,0x00,0x00,0x00,0x22,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x77,0x00,0x00,0x00, -0x21,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x97,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x19,0x00,0x00,0x00, -0x13,0x00,0x02,0x00,0x02,0x00,0x00,0x00,0x21,0x00,0x03,0x00, -0x03,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x15,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x15,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x17,0x00,0x04,0x00,0x0a,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x0b,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x0a,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x0b,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x0d,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x0e,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x1e,0x00,0x06,0x00,0x14,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x15,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x14,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x15,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x17,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x18,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x1b,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x14,0x00,0x02,0x00, -0x24,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x48,0x00,0x00,0x00, -0x10,0x00,0x00,0x00,0x16,0x00,0x03,0x00,0x4a,0x00,0x00,0x00, -0x10,0x00,0x00,0x00,0x15,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0x08,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x10,0x00,0x00,0x00, -0x1c,0x00,0x04,0x00,0x4f,0x00,0x00,0x00,0x4d,0x00,0x00,0x00, -0x4e,0x00,0x00,0x00,0x1e,0x00,0x04,0x00,0x50,0x00,0x00,0x00, -0x4a,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, -0x51,0x00,0x00,0x00,0x50,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, -0x52,0x00,0x00,0x00,0x51,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x53,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x52,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x53,0x00,0x00,0x00,0x54,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x56,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x4a,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x5d,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x4d,0x00,0x00,0x00, -0x17,0x00,0x04,0x00,0x60,0x00,0x00,0x00,0x4a,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x66,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x6a,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0x6f,0x00,0x00,0x00, -0x00,0x48,0x00,0x00,0x1d,0x00,0x03,0x00,0x74,0x00,0x00,0x00, -0x4a,0x00,0x00,0x00,0x1e,0x00,0x03,0x00,0x75,0x00,0x00,0x00, -0x74,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x76,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x75,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x76,0x00,0x00,0x00,0x77,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x79,0x00,0x00,0x00, -0x03,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x90,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x96,0x00,0x00,0x00,0x00,0x01,0x00,0x00, -0x2c,0x00,0x06,0x00,0x0a,0x00,0x00,0x00,0x97,0x00,0x00,0x00, -0x96,0x00,0x00,0x00,0x90,0x00,0x00,0x00,0x90,0x00,0x00,0x00, -0x2c,0x00,0x05,0x00,0x60,0x00,0x00,0x00,0xa2,0x00,0x00,0x00, -0x6f,0x00,0x00,0x00,0x6f,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xde,0x02,0x00,0x00,0x11,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xdf,0x02,0x00,0x00, -0x12,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xe0,0x02,0x00,0x00,0x13,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xe1,0x02,0x00,0x00,0x14,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xe2,0x02,0x00,0x00, -0x05,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xe3,0x02,0x00,0x00,0x15,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xe4,0x02,0x00,0x00,0x06,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xe5,0x02,0x00,0x00, -0x16,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xe6,0x02,0x00,0x00,0x07,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xe7,0x02,0x00,0x00,0x17,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xe8,0x02,0x00,0x00, -0x08,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xe9,0x02,0x00,0x00,0x18,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xea,0x02,0x00,0x00,0x09,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xeb,0x02,0x00,0x00, -0x19,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xec,0x02,0x00,0x00,0x0a,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xed,0x02,0x00,0x00,0x1a,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xee,0x02,0x00,0x00, -0x0b,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xef,0x02,0x00,0x00,0x1b,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xf0,0x02,0x00,0x00,0x0c,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xf1,0x02,0x00,0x00, -0x1c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xf2,0x02,0x00,0x00,0x0d,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xf3,0x02,0x00,0x00,0x1d,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xf4,0x02,0x00,0x00, -0x0e,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xf5,0x02,0x00,0x00,0x1e,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xf6,0x02,0x00,0x00,0x1f,0x00,0x00,0x00, -0x36,0x00,0x05,0x00,0x02,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x05,0x00,0x00,0x00,0xf7,0x00,0x03,0x00,0x98,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0xfb,0x00,0x03,0x00,0x0d,0x00,0x00,0x00, -0x99,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x99,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x0e,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x0d,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x11,0x00,0x00,0x00, -0x10,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x18,0x00,0x00,0x00, -0x19,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x17,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x1a,0x00,0x00,0x00, -0x19,0x00,0x00,0x00,0x87,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x1c,0x00,0x00,0x00,0x1a,0x00,0x00,0x00,0x1b,0x00,0x00,0x00, -0x8b,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x1d,0x00,0x00,0x00, -0x11,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x87,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x11,0x00,0x00,0x00, -0x1c,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x26,0x00,0x00,0x00,0x1d,0x00,0x00,0x00,0x1b,0x00,0x00,0x00, -0xaf,0x00,0x05,0x00,0x24,0x00,0x00,0x00,0x29,0x00,0x00,0x00, -0x26,0x00,0x00,0x00,0x1a,0x00,0x00,0x00,0xa8,0x00,0x04,0x00, -0x24,0x00,0x00,0x00,0x2a,0x00,0x00,0x00,0x29,0x00,0x00,0x00, -0xf7,0x00,0x03,0x00,0x2c,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x2a,0x00,0x00,0x00,0x2b,0x00,0x00,0x00, -0x2c,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x2b,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x18,0x00,0x00,0x00,0x2f,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x30,0x00,0x00,0x00,0x2f,0x00,0x00,0x00, -0xaf,0x00,0x05,0x00,0x24,0x00,0x00,0x00,0x31,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x30,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x2c,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x2c,0x00,0x00,0x00, -0xf5,0x00,0x07,0x00,0x24,0x00,0x00,0x00,0x32,0x00,0x00,0x00, -0x29,0x00,0x00,0x00,0x99,0x00,0x00,0x00,0x31,0x00,0x00,0x00, -0x2b,0x00,0x00,0x00,0xf7,0x00,0x03,0x00,0x34,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x32,0x00,0x00,0x00, -0x33,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x33,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x98,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0x34,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x18,0x00,0x00,0x00,0x38,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0x37,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x38,0x00,0x00,0x00,0x87,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x1b,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x3e,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x3e,0x00,0x00,0x00,0x1d,0x00,0x00,0x00,0x41,0x00,0x07,0x00, -0x56,0x00,0x00,0x00,0x57,0x00,0x00,0x00,0x54,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0x58,0x00,0x00,0x00, -0x57,0x00,0x00,0x00,0x41,0x00,0x08,0x00,0x5d,0x00,0x00,0x00, -0x5e,0x00,0x00,0x00,0x54,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0x17,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00,0x5f,0x00,0x00,0x00, -0x5e,0x00,0x00,0x00,0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x64,0x00,0x00,0x00,0x5f,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x65,0x00,0x00,0x00,0x64,0x00,0x00,0x00, -0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x67,0x00,0x00,0x00, -0x65,0x00,0x00,0x00,0x66,0x00,0x00,0x00,0x6f,0x00,0x04,0x00, -0x4a,0x00,0x00,0x00,0x68,0x00,0x00,0x00,0x67,0x00,0x00,0x00, -0xc2,0x00,0x05,0x00,0x4d,0x00,0x00,0x00,0x6b,0x00,0x00,0x00, -0x5f,0x00,0x00,0x00,0x6a,0x00,0x00,0x00,0x70,0x00,0x04,0x00, -0x4a,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x6b,0x00,0x00,0x00, -0x50,0x00,0x05,0x00,0x60,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, -0x68,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x83,0x00,0x05,0x00, -0x60,0x00,0x00,0x00,0x71,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, -0xa2,0x00,0x00,0x00,0x8e,0x00,0x05,0x00,0x60,0x00,0x00,0x00, -0x73,0x00,0x00,0x00,0x71,0x00,0x00,0x00,0x58,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x18,0x00,0x00,0x00,0x7a,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x79,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x7b,0x00,0x00,0x00,0x7a,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x7c,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x7b,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x7f,0x00,0x00,0x00,0x7c,0x00,0x00,0x00, -0x26,0x00,0x00,0x00,0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0x73,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x41,0x00,0x06,0x00,0x56,0x00,0x00,0x00,0x85,0x00,0x00,0x00, -0x77,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x7f,0x00,0x00,0x00, -0x3e,0x00,0x03,0x00,0x85,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8f,0x00,0x00,0x00, -0x7f,0x00,0x00,0x00,0x48,0x00,0x00,0x00,0x51,0x00,0x05,0x00, -0x4a,0x00,0x00,0x00,0x92,0x00,0x00,0x00,0x73,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x56,0x00,0x00,0x00, -0x93,0x00,0x00,0x00,0x77,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x8f,0x00,0x00,0x00,0x3e,0x00,0x03,0x00,0x93,0x00,0x00,0x00, -0x92,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0xa9,0x00,0x00,0x00,0x57,0x00,0x00,0x00,0x41,0x00,0x08,0x00, -0x5d,0x00,0x00,0x00,0xaa,0x00,0x00,0x00,0x54,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x17,0x00,0x00,0x00, -0x17,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0xab,0x00,0x00,0x00,0xaa,0x00,0x00,0x00,0x71,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0xac,0x00,0x00,0x00,0xab,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xad,0x00,0x00,0x00, -0xac,0x00,0x00,0x00,0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xae,0x00,0x00,0x00,0xad,0x00,0x00,0x00,0x66,0x00,0x00,0x00, -0x6f,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0xaf,0x00,0x00,0x00, -0xae,0x00,0x00,0x00,0xc2,0x00,0x05,0x00,0x4d,0x00,0x00,0x00, -0xb0,0x00,0x00,0x00,0xab,0x00,0x00,0x00,0x6a,0x00,0x00,0x00, -0x70,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0xb1,0x00,0x00,0x00, -0xb0,0x00,0x00,0x00,0x50,0x00,0x05,0x00,0x60,0x00,0x00,0x00, -0xb2,0x00,0x00,0x00,0xaf,0x00,0x00,0x00,0xb1,0x00,0x00,0x00, -0x83,0x00,0x05,0x00,0x60,0x00,0x00,0x00,0xb3,0x00,0x00,0x00, -0xb2,0x00,0x00,0x00,0xa2,0x00,0x00,0x00,0x8e,0x00,0x05,0x00, -0x60,0x00,0x00,0x00,0xb4,0x00,0x00,0x00,0xb3,0x00,0x00,0x00, -0xa9,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xba,0x00,0x00,0x00,0x7f,0x00,0x00,0x00,0x17,0x00,0x00,0x00, -0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00,0xbc,0x00,0x00,0x00, -0xb4,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x41,0x00,0x06,0x00, -0x56,0x00,0x00,0x00,0xbd,0x00,0x00,0x00,0x77,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0xba,0x00,0x00,0x00,0x3e,0x00,0x03,0x00, -0xbd,0x00,0x00,0x00,0xbc,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xc4,0x00,0x00,0x00,0x7f,0x00,0x00,0x00, -0xde,0x02,0x00,0x00,0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00, -0xc5,0x00,0x00,0x00,0xb4,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x41,0x00,0x06,0x00,0x56,0x00,0x00,0x00,0xc6,0x00,0x00,0x00, -0x77,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0xc4,0x00,0x00,0x00, -0x3e,0x00,0x03,0x00,0xc6,0x00,0x00,0x00,0xc5,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0xcf,0x00,0x00,0x00, -0x57,0x00,0x00,0x00,0x41,0x00,0x08,0x00,0x5d,0x00,0x00,0x00, -0xd0,0x00,0x00,0x00,0x54,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0x17,0x00,0x00,0x00,0x37,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00,0xd1,0x00,0x00,0x00, -0xd0,0x00,0x00,0x00,0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0xd2,0x00,0x00,0x00,0xd1,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xd3,0x00,0x00,0x00,0xd2,0x00,0x00,0x00, -0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xd4,0x00,0x00,0x00, -0xd3,0x00,0x00,0x00,0x66,0x00,0x00,0x00,0x6f,0x00,0x04,0x00, -0x4a,0x00,0x00,0x00,0xd5,0x00,0x00,0x00,0xd4,0x00,0x00,0x00, -0xc2,0x00,0x05,0x00,0x4d,0x00,0x00,0x00,0xd6,0x00,0x00,0x00, -0xd1,0x00,0x00,0x00,0x6a,0x00,0x00,0x00,0x70,0x00,0x04,0x00, -0x4a,0x00,0x00,0x00,0xd7,0x00,0x00,0x00,0xd6,0x00,0x00,0x00, -0x50,0x00,0x05,0x00,0x60,0x00,0x00,0x00,0xd8,0x00,0x00,0x00, -0xd5,0x00,0x00,0x00,0xd7,0x00,0x00,0x00,0x83,0x00,0x05,0x00, -0x60,0x00,0x00,0x00,0xd9,0x00,0x00,0x00,0xd8,0x00,0x00,0x00, -0xa2,0x00,0x00,0x00,0x8e,0x00,0x05,0x00,0x60,0x00,0x00,0x00, -0xda,0x00,0x00,0x00,0xd9,0x00,0x00,0x00,0xcf,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xe0,0x00,0x00,0x00, -0x7f,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x51,0x00,0x05,0x00, -0x4a,0x00,0x00,0x00,0xe2,0x00,0x00,0x00,0xda,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x56,0x00,0x00,0x00, -0xe3,0x00,0x00,0x00,0x77,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0xe0,0x00,0x00,0x00,0x3e,0x00,0x03,0x00,0xe3,0x00,0x00,0x00, -0xe2,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xea,0x00,0x00,0x00,0x7f,0x00,0x00,0x00,0xdf,0x02,0x00,0x00, -0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00,0xeb,0x00,0x00,0x00, -0xda,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x41,0x00,0x06,0x00, -0x56,0x00,0x00,0x00,0xec,0x00,0x00,0x00,0x77,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0xea,0x00,0x00,0x00,0x3e,0x00,0x03,0x00, -0xec,0x00,0x00,0x00,0xeb,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4a,0x00,0x00,0x00,0xf5,0x00,0x00,0x00,0x57,0x00,0x00,0x00, -0x41,0x00,0x08,0x00,0x5d,0x00,0x00,0x00,0xf6,0x00,0x00,0x00, -0x54,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x17,0x00,0x00,0x00,0x79,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0xf7,0x00,0x00,0x00,0xf6,0x00,0x00,0x00, -0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0xf8,0x00,0x00,0x00, -0xf7,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xf9,0x00,0x00,0x00,0xf8,0x00,0x00,0x00,0xc7,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xfa,0x00,0x00,0x00,0xf9,0x00,0x00,0x00, -0x66,0x00,0x00,0x00,0x6f,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0xfb,0x00,0x00,0x00,0xfa,0x00,0x00,0x00,0xc2,0x00,0x05,0x00, -0x4d,0x00,0x00,0x00,0xfc,0x00,0x00,0x00,0xf7,0x00,0x00,0x00, -0x6a,0x00,0x00,0x00,0x70,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0xfd,0x00,0x00,0x00,0xfc,0x00,0x00,0x00,0x50,0x00,0x05,0x00, -0x60,0x00,0x00,0x00,0xfe,0x00,0x00,0x00,0xfb,0x00,0x00,0x00, -0xfd,0x00,0x00,0x00,0x83,0x00,0x05,0x00,0x60,0x00,0x00,0x00, -0xff,0x00,0x00,0x00,0xfe,0x00,0x00,0x00,0xa2,0x00,0x00,0x00, -0x8e,0x00,0x05,0x00,0x60,0x00,0x00,0x00,0x00,0x01,0x00,0x00, -0xff,0x00,0x00,0x00,0xf5,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x06,0x01,0x00,0x00,0x7f,0x00,0x00,0x00, -0x79,0x00,0x00,0x00,0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00, -0x08,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x00,0x00,0x00, -0x41,0x00,0x06,0x00,0x56,0x00,0x00,0x00,0x09,0x01,0x00,0x00, -0x77,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x06,0x01,0x00,0x00, -0x3e,0x00,0x03,0x00,0x09,0x01,0x00,0x00,0x08,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x10,0x01,0x00,0x00, -0x7f,0x00,0x00,0x00,0xe0,0x02,0x00,0x00,0x51,0x00,0x05,0x00, -0x4a,0x00,0x00,0x00,0x11,0x01,0x00,0x00,0x00,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x56,0x00,0x00,0x00, -0x12,0x01,0x00,0x00,0x77,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x10,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x12,0x01,0x00,0x00, -0x11,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0x1b,0x01,0x00,0x00,0x57,0x00,0x00,0x00,0x41,0x00,0x08,0x00, -0x5d,0x00,0x00,0x00,0x1c,0x01,0x00,0x00,0x54,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x17,0x00,0x00,0x00, -0x6a,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0x1d,0x01,0x00,0x00,0x1c,0x01,0x00,0x00,0x71,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x1e,0x01,0x00,0x00,0x1d,0x01,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x1f,0x01,0x00,0x00, -0x1e,0x01,0x00,0x00,0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x20,0x01,0x00,0x00,0x1f,0x01,0x00,0x00,0x66,0x00,0x00,0x00, -0x6f,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0x21,0x01,0x00,0x00, -0x20,0x01,0x00,0x00,0xc2,0x00,0x05,0x00,0x4d,0x00,0x00,0x00, -0x22,0x01,0x00,0x00,0x1d,0x01,0x00,0x00,0x6a,0x00,0x00,0x00, -0x70,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0x23,0x01,0x00,0x00, -0x22,0x01,0x00,0x00,0x50,0x00,0x05,0x00,0x60,0x00,0x00,0x00, -0x24,0x01,0x00,0x00,0x21,0x01,0x00,0x00,0x23,0x01,0x00,0x00, -0x83,0x00,0x05,0x00,0x60,0x00,0x00,0x00,0x25,0x01,0x00,0x00, -0x24,0x01,0x00,0x00,0xa2,0x00,0x00,0x00,0x8e,0x00,0x05,0x00, -0x60,0x00,0x00,0x00,0x26,0x01,0x00,0x00,0x25,0x01,0x00,0x00, -0x1b,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x2c,0x01,0x00,0x00,0x7f,0x00,0x00,0x00,0x6a,0x00,0x00,0x00, -0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00,0x2e,0x01,0x00,0x00, -0x26,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x41,0x00,0x06,0x00, -0x56,0x00,0x00,0x00,0x2f,0x01,0x00,0x00,0x77,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x2c,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0x2f,0x01,0x00,0x00,0x2e,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x36,0x01,0x00,0x00,0x7f,0x00,0x00,0x00, -0xe1,0x02,0x00,0x00,0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00, -0x37,0x01,0x00,0x00,0x26,0x01,0x00,0x00,0x01,0x00,0x00,0x00, -0x41,0x00,0x06,0x00,0x56,0x00,0x00,0x00,0x38,0x01,0x00,0x00, -0x77,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x36,0x01,0x00,0x00, -0x3e,0x00,0x03,0x00,0x38,0x01,0x00,0x00,0x37,0x01,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0x41,0x01,0x00,0x00, -0x57,0x00,0x00,0x00,0x41,0x00,0x08,0x00,0x5d,0x00,0x00,0x00, -0x42,0x01,0x00,0x00,0x54,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0x17,0x00,0x00,0x00,0xe2,0x02,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00,0x43,0x01,0x00,0x00, -0x42,0x01,0x00,0x00,0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x44,0x01,0x00,0x00,0x43,0x01,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x45,0x01,0x00,0x00,0x44,0x01,0x00,0x00, -0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x46,0x01,0x00,0x00, -0x45,0x01,0x00,0x00,0x66,0x00,0x00,0x00,0x6f,0x00,0x04,0x00, -0x4a,0x00,0x00,0x00,0x47,0x01,0x00,0x00,0x46,0x01,0x00,0x00, -0xc2,0x00,0x05,0x00,0x4d,0x00,0x00,0x00,0x48,0x01,0x00,0x00, -0x43,0x01,0x00,0x00,0x6a,0x00,0x00,0x00,0x70,0x00,0x04,0x00, -0x4a,0x00,0x00,0x00,0x49,0x01,0x00,0x00,0x48,0x01,0x00,0x00, -0x50,0x00,0x05,0x00,0x60,0x00,0x00,0x00,0x4a,0x01,0x00,0x00, -0x47,0x01,0x00,0x00,0x49,0x01,0x00,0x00,0x83,0x00,0x05,0x00, -0x60,0x00,0x00,0x00,0x4b,0x01,0x00,0x00,0x4a,0x01,0x00,0x00, -0xa2,0x00,0x00,0x00,0x8e,0x00,0x05,0x00,0x60,0x00,0x00,0x00, -0x4c,0x01,0x00,0x00,0x4b,0x01,0x00,0x00,0x41,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x52,0x01,0x00,0x00, -0x7f,0x00,0x00,0x00,0xe2,0x02,0x00,0x00,0x51,0x00,0x05,0x00, -0x4a,0x00,0x00,0x00,0x54,0x01,0x00,0x00,0x4c,0x01,0x00,0x00, -0x00,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x56,0x00,0x00,0x00, -0x55,0x01,0x00,0x00,0x77,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x52,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x55,0x01,0x00,0x00, -0x54,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x5c,0x01,0x00,0x00,0x7f,0x00,0x00,0x00,0xe3,0x02,0x00,0x00, -0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00,0x5d,0x01,0x00,0x00, -0x4c,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x41,0x00,0x06,0x00, -0x56,0x00,0x00,0x00,0x5e,0x01,0x00,0x00,0x77,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x5c,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0x5e,0x01,0x00,0x00,0x5d,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4a,0x00,0x00,0x00,0x67,0x01,0x00,0x00,0x57,0x00,0x00,0x00, -0x41,0x00,0x08,0x00,0x5d,0x00,0x00,0x00,0x68,0x01,0x00,0x00, -0x54,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x17,0x00,0x00,0x00,0xe4,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0x69,0x01,0x00,0x00,0x68,0x01,0x00,0x00, -0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x6a,0x01,0x00,0x00, -0x69,0x01,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x6b,0x01,0x00,0x00,0x6a,0x01,0x00,0x00,0xc7,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x6c,0x01,0x00,0x00,0x6b,0x01,0x00,0x00, -0x66,0x00,0x00,0x00,0x6f,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0x6d,0x01,0x00,0x00,0x6c,0x01,0x00,0x00,0xc2,0x00,0x05,0x00, -0x4d,0x00,0x00,0x00,0x6e,0x01,0x00,0x00,0x69,0x01,0x00,0x00, -0x6a,0x00,0x00,0x00,0x70,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0x6f,0x01,0x00,0x00,0x6e,0x01,0x00,0x00,0x50,0x00,0x05,0x00, -0x60,0x00,0x00,0x00,0x70,0x01,0x00,0x00,0x6d,0x01,0x00,0x00, -0x6f,0x01,0x00,0x00,0x83,0x00,0x05,0x00,0x60,0x00,0x00,0x00, -0x71,0x01,0x00,0x00,0x70,0x01,0x00,0x00,0xa2,0x00,0x00,0x00, -0x8e,0x00,0x05,0x00,0x60,0x00,0x00,0x00,0x72,0x01,0x00,0x00, -0x71,0x01,0x00,0x00,0x67,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x78,0x01,0x00,0x00,0x7f,0x00,0x00,0x00, -0xe4,0x02,0x00,0x00,0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00, -0x7a,0x01,0x00,0x00,0x72,0x01,0x00,0x00,0x00,0x00,0x00,0x00, -0x41,0x00,0x06,0x00,0x56,0x00,0x00,0x00,0x7b,0x01,0x00,0x00, -0x77,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x78,0x01,0x00,0x00, -0x3e,0x00,0x03,0x00,0x7b,0x01,0x00,0x00,0x7a,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x82,0x01,0x00,0x00, -0x7f,0x00,0x00,0x00,0xe5,0x02,0x00,0x00,0x51,0x00,0x05,0x00, -0x4a,0x00,0x00,0x00,0x83,0x01,0x00,0x00,0x72,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x56,0x00,0x00,0x00, -0x84,0x01,0x00,0x00,0x77,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x82,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x84,0x01,0x00,0x00, -0x83,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0x8d,0x01,0x00,0x00,0x57,0x00,0x00,0x00,0x41,0x00,0x08,0x00, -0x5d,0x00,0x00,0x00,0x8e,0x01,0x00,0x00,0x54,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x17,0x00,0x00,0x00, -0xe6,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0x8f,0x01,0x00,0x00,0x8e,0x01,0x00,0x00,0x71,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x90,0x01,0x00,0x00,0x8f,0x01,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x91,0x01,0x00,0x00, -0x90,0x01,0x00,0x00,0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x92,0x01,0x00,0x00,0x91,0x01,0x00,0x00,0x66,0x00,0x00,0x00, -0x6f,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0x93,0x01,0x00,0x00, -0x92,0x01,0x00,0x00,0xc2,0x00,0x05,0x00,0x4d,0x00,0x00,0x00, -0x94,0x01,0x00,0x00,0x8f,0x01,0x00,0x00,0x6a,0x00,0x00,0x00, -0x70,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0x95,0x01,0x00,0x00, -0x94,0x01,0x00,0x00,0x50,0x00,0x05,0x00,0x60,0x00,0x00,0x00, -0x96,0x01,0x00,0x00,0x93,0x01,0x00,0x00,0x95,0x01,0x00,0x00, -0x83,0x00,0x05,0x00,0x60,0x00,0x00,0x00,0x97,0x01,0x00,0x00, -0x96,0x01,0x00,0x00,0xa2,0x00,0x00,0x00,0x8e,0x00,0x05,0x00, -0x60,0x00,0x00,0x00,0x98,0x01,0x00,0x00,0x97,0x01,0x00,0x00, -0x8d,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x9e,0x01,0x00,0x00,0x7f,0x00,0x00,0x00,0xe6,0x02,0x00,0x00, -0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00,0xa0,0x01,0x00,0x00, -0x98,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x41,0x00,0x06,0x00, -0x56,0x00,0x00,0x00,0xa1,0x01,0x00,0x00,0x77,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x9e,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0xa1,0x01,0x00,0x00,0xa0,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xa8,0x01,0x00,0x00,0x7f,0x00,0x00,0x00, -0xe7,0x02,0x00,0x00,0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00, -0xa9,0x01,0x00,0x00,0x98,0x01,0x00,0x00,0x01,0x00,0x00,0x00, -0x41,0x00,0x06,0x00,0x56,0x00,0x00,0x00,0xaa,0x01,0x00,0x00, -0x77,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0xa8,0x01,0x00,0x00, -0x3e,0x00,0x03,0x00,0xaa,0x01,0x00,0x00,0xa9,0x01,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0xb3,0x01,0x00,0x00, -0x57,0x00,0x00,0x00,0x41,0x00,0x08,0x00,0x5d,0x00,0x00,0x00, -0xb4,0x01,0x00,0x00,0x54,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0x17,0x00,0x00,0x00,0xe8,0x02,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00,0xb5,0x01,0x00,0x00, -0xb4,0x01,0x00,0x00,0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0xb6,0x01,0x00,0x00,0xb5,0x01,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xb7,0x01,0x00,0x00,0xb6,0x01,0x00,0x00, -0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xb8,0x01,0x00,0x00, -0xb7,0x01,0x00,0x00,0x66,0x00,0x00,0x00,0x6f,0x00,0x04,0x00, -0x4a,0x00,0x00,0x00,0xb9,0x01,0x00,0x00,0xb8,0x01,0x00,0x00, -0xc2,0x00,0x05,0x00,0x4d,0x00,0x00,0x00,0xba,0x01,0x00,0x00, -0xb5,0x01,0x00,0x00,0x6a,0x00,0x00,0x00,0x70,0x00,0x04,0x00, -0x4a,0x00,0x00,0x00,0xbb,0x01,0x00,0x00,0xba,0x01,0x00,0x00, -0x50,0x00,0x05,0x00,0x60,0x00,0x00,0x00,0xbc,0x01,0x00,0x00, -0xb9,0x01,0x00,0x00,0xbb,0x01,0x00,0x00,0x83,0x00,0x05,0x00, -0x60,0x00,0x00,0x00,0xbd,0x01,0x00,0x00,0xbc,0x01,0x00,0x00, -0xa2,0x00,0x00,0x00,0x8e,0x00,0x05,0x00,0x60,0x00,0x00,0x00, -0xbe,0x01,0x00,0x00,0xbd,0x01,0x00,0x00,0xb3,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xc4,0x01,0x00,0x00, -0x7f,0x00,0x00,0x00,0xe8,0x02,0x00,0x00,0x51,0x00,0x05,0x00, -0x4a,0x00,0x00,0x00,0xc6,0x01,0x00,0x00,0xbe,0x01,0x00,0x00, -0x00,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x56,0x00,0x00,0x00, -0xc7,0x01,0x00,0x00,0x77,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0xc4,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0xc7,0x01,0x00,0x00, -0xc6,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xce,0x01,0x00,0x00,0x7f,0x00,0x00,0x00,0xe9,0x02,0x00,0x00, -0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00,0xcf,0x01,0x00,0x00, -0xbe,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x41,0x00,0x06,0x00, -0x56,0x00,0x00,0x00,0xd0,0x01,0x00,0x00,0x77,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0xce,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0xd0,0x01,0x00,0x00,0xcf,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4a,0x00,0x00,0x00,0xd9,0x01,0x00,0x00,0x57,0x00,0x00,0x00, -0x41,0x00,0x08,0x00,0x5d,0x00,0x00,0x00,0xda,0x01,0x00,0x00, -0x54,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x17,0x00,0x00,0x00,0xea,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0xdb,0x01,0x00,0x00,0xda,0x01,0x00,0x00, -0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0xdc,0x01,0x00,0x00, -0xdb,0x01,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xdd,0x01,0x00,0x00,0xdc,0x01,0x00,0x00,0xc7,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xde,0x01,0x00,0x00,0xdd,0x01,0x00,0x00, -0x66,0x00,0x00,0x00,0x6f,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0xdf,0x01,0x00,0x00,0xde,0x01,0x00,0x00,0xc2,0x00,0x05,0x00, -0x4d,0x00,0x00,0x00,0xe0,0x01,0x00,0x00,0xdb,0x01,0x00,0x00, -0x6a,0x00,0x00,0x00,0x70,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0xe1,0x01,0x00,0x00,0xe0,0x01,0x00,0x00,0x50,0x00,0x05,0x00, -0x60,0x00,0x00,0x00,0xe2,0x01,0x00,0x00,0xdf,0x01,0x00,0x00, -0xe1,0x01,0x00,0x00,0x83,0x00,0x05,0x00,0x60,0x00,0x00,0x00, -0xe3,0x01,0x00,0x00,0xe2,0x01,0x00,0x00,0xa2,0x00,0x00,0x00, -0x8e,0x00,0x05,0x00,0x60,0x00,0x00,0x00,0xe4,0x01,0x00,0x00, -0xe3,0x01,0x00,0x00,0xd9,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xea,0x01,0x00,0x00,0x7f,0x00,0x00,0x00, -0xea,0x02,0x00,0x00,0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00, -0xec,0x01,0x00,0x00,0xe4,0x01,0x00,0x00,0x00,0x00,0x00,0x00, -0x41,0x00,0x06,0x00,0x56,0x00,0x00,0x00,0xed,0x01,0x00,0x00, -0x77,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0xea,0x01,0x00,0x00, -0x3e,0x00,0x03,0x00,0xed,0x01,0x00,0x00,0xec,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xf4,0x01,0x00,0x00, -0x7f,0x00,0x00,0x00,0xeb,0x02,0x00,0x00,0x51,0x00,0x05,0x00, -0x4a,0x00,0x00,0x00,0xf5,0x01,0x00,0x00,0xe4,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x56,0x00,0x00,0x00, -0xf6,0x01,0x00,0x00,0x77,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0xf4,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0xf6,0x01,0x00,0x00, -0xf5,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0xff,0x01,0x00,0x00,0x57,0x00,0x00,0x00,0x41,0x00,0x08,0x00, -0x5d,0x00,0x00,0x00,0x00,0x02,0x00,0x00,0x54,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x17,0x00,0x00,0x00, -0xec,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0x01,0x02,0x00,0x00,0x00,0x02,0x00,0x00,0x71,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x02,0x02,0x00,0x00,0x01,0x02,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x03,0x02,0x00,0x00, -0x02,0x02,0x00,0x00,0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x04,0x02,0x00,0x00,0x03,0x02,0x00,0x00,0x66,0x00,0x00,0x00, -0x6f,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0x05,0x02,0x00,0x00, -0x04,0x02,0x00,0x00,0xc2,0x00,0x05,0x00,0x4d,0x00,0x00,0x00, -0x06,0x02,0x00,0x00,0x01,0x02,0x00,0x00,0x6a,0x00,0x00,0x00, -0x70,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0x07,0x02,0x00,0x00, -0x06,0x02,0x00,0x00,0x50,0x00,0x05,0x00,0x60,0x00,0x00,0x00, -0x08,0x02,0x00,0x00,0x05,0x02,0x00,0x00,0x07,0x02,0x00,0x00, -0x83,0x00,0x05,0x00,0x60,0x00,0x00,0x00,0x09,0x02,0x00,0x00, -0x08,0x02,0x00,0x00,0xa2,0x00,0x00,0x00,0x8e,0x00,0x05,0x00, -0x60,0x00,0x00,0x00,0x0a,0x02,0x00,0x00,0x09,0x02,0x00,0x00, -0xff,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x10,0x02,0x00,0x00,0x7f,0x00,0x00,0x00,0xec,0x02,0x00,0x00, -0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00,0x12,0x02,0x00,0x00, -0x0a,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x41,0x00,0x06,0x00, -0x56,0x00,0x00,0x00,0x13,0x02,0x00,0x00,0x77,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x10,0x02,0x00,0x00,0x3e,0x00,0x03,0x00, -0x13,0x02,0x00,0x00,0x12,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x1a,0x02,0x00,0x00,0x7f,0x00,0x00,0x00, -0xed,0x02,0x00,0x00,0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00, -0x1b,0x02,0x00,0x00,0x0a,0x02,0x00,0x00,0x01,0x00,0x00,0x00, -0x41,0x00,0x06,0x00,0x56,0x00,0x00,0x00,0x1c,0x02,0x00,0x00, -0x77,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x1a,0x02,0x00,0x00, -0x3e,0x00,0x03,0x00,0x1c,0x02,0x00,0x00,0x1b,0x02,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0x25,0x02,0x00,0x00, -0x57,0x00,0x00,0x00,0x41,0x00,0x08,0x00,0x5d,0x00,0x00,0x00, -0x26,0x02,0x00,0x00,0x54,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0x17,0x00,0x00,0x00,0xee,0x02,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00,0x27,0x02,0x00,0x00, -0x26,0x02,0x00,0x00,0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x28,0x02,0x00,0x00,0x27,0x02,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x29,0x02,0x00,0x00,0x28,0x02,0x00,0x00, -0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x2a,0x02,0x00,0x00, -0x29,0x02,0x00,0x00,0x66,0x00,0x00,0x00,0x6f,0x00,0x04,0x00, -0x4a,0x00,0x00,0x00,0x2b,0x02,0x00,0x00,0x2a,0x02,0x00,0x00, -0xc2,0x00,0x05,0x00,0x4d,0x00,0x00,0x00,0x2c,0x02,0x00,0x00, -0x27,0x02,0x00,0x00,0x6a,0x00,0x00,0x00,0x70,0x00,0x04,0x00, -0x4a,0x00,0x00,0x00,0x2d,0x02,0x00,0x00,0x2c,0x02,0x00,0x00, -0x50,0x00,0x05,0x00,0x60,0x00,0x00,0x00,0x2e,0x02,0x00,0x00, -0x2b,0x02,0x00,0x00,0x2d,0x02,0x00,0x00,0x83,0x00,0x05,0x00, -0x60,0x00,0x00,0x00,0x2f,0x02,0x00,0x00,0x2e,0x02,0x00,0x00, -0xa2,0x00,0x00,0x00,0x8e,0x00,0x05,0x00,0x60,0x00,0x00,0x00, -0x30,0x02,0x00,0x00,0x2f,0x02,0x00,0x00,0x25,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x36,0x02,0x00,0x00, -0x7f,0x00,0x00,0x00,0xee,0x02,0x00,0x00,0x51,0x00,0x05,0x00, -0x4a,0x00,0x00,0x00,0x38,0x02,0x00,0x00,0x30,0x02,0x00,0x00, -0x00,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x56,0x00,0x00,0x00, -0x39,0x02,0x00,0x00,0x77,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x36,0x02,0x00,0x00,0x3e,0x00,0x03,0x00,0x39,0x02,0x00,0x00, -0x38,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x40,0x02,0x00,0x00,0x7f,0x00,0x00,0x00,0xef,0x02,0x00,0x00, -0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00,0x41,0x02,0x00,0x00, -0x30,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0x41,0x00,0x06,0x00, -0x56,0x00,0x00,0x00,0x42,0x02,0x00,0x00,0x77,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x40,0x02,0x00,0x00,0x3e,0x00,0x03,0x00, -0x42,0x02,0x00,0x00,0x41,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4a,0x00,0x00,0x00,0x4b,0x02,0x00,0x00,0x57,0x00,0x00,0x00, -0x41,0x00,0x08,0x00,0x5d,0x00,0x00,0x00,0x4c,0x02,0x00,0x00, -0x54,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x17,0x00,0x00,0x00,0xf0,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0x4d,0x02,0x00,0x00,0x4c,0x02,0x00,0x00, -0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x4e,0x02,0x00,0x00, -0x4d,0x02,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x4f,0x02,0x00,0x00,0x4e,0x02,0x00,0x00,0xc7,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x50,0x02,0x00,0x00,0x4f,0x02,0x00,0x00, -0x66,0x00,0x00,0x00,0x6f,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0x51,0x02,0x00,0x00,0x50,0x02,0x00,0x00,0xc2,0x00,0x05,0x00, -0x4d,0x00,0x00,0x00,0x52,0x02,0x00,0x00,0x4d,0x02,0x00,0x00, -0x6a,0x00,0x00,0x00,0x70,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0x53,0x02,0x00,0x00,0x52,0x02,0x00,0x00,0x50,0x00,0x05,0x00, -0x60,0x00,0x00,0x00,0x54,0x02,0x00,0x00,0x51,0x02,0x00,0x00, -0x53,0x02,0x00,0x00,0x83,0x00,0x05,0x00,0x60,0x00,0x00,0x00, -0x55,0x02,0x00,0x00,0x54,0x02,0x00,0x00,0xa2,0x00,0x00,0x00, -0x8e,0x00,0x05,0x00,0x60,0x00,0x00,0x00,0x56,0x02,0x00,0x00, -0x55,0x02,0x00,0x00,0x4b,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x5c,0x02,0x00,0x00,0x7f,0x00,0x00,0x00, -0xf0,0x02,0x00,0x00,0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00, -0x5e,0x02,0x00,0x00,0x56,0x02,0x00,0x00,0x00,0x00,0x00,0x00, -0x41,0x00,0x06,0x00,0x56,0x00,0x00,0x00,0x5f,0x02,0x00,0x00, -0x77,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x5c,0x02,0x00,0x00, -0x3e,0x00,0x03,0x00,0x5f,0x02,0x00,0x00,0x5e,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x66,0x02,0x00,0x00, -0x7f,0x00,0x00,0x00,0xf1,0x02,0x00,0x00,0x51,0x00,0x05,0x00, -0x4a,0x00,0x00,0x00,0x67,0x02,0x00,0x00,0x56,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x56,0x00,0x00,0x00, -0x68,0x02,0x00,0x00,0x77,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x66,0x02,0x00,0x00,0x3e,0x00,0x03,0x00,0x68,0x02,0x00,0x00, -0x67,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0x71,0x02,0x00,0x00,0x57,0x00,0x00,0x00,0x41,0x00,0x08,0x00, -0x5d,0x00,0x00,0x00,0x72,0x02,0x00,0x00,0x54,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x17,0x00,0x00,0x00, -0xf2,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0x73,0x02,0x00,0x00,0x72,0x02,0x00,0x00,0x71,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x74,0x02,0x00,0x00,0x73,0x02,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x75,0x02,0x00,0x00, -0x74,0x02,0x00,0x00,0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x76,0x02,0x00,0x00,0x75,0x02,0x00,0x00,0x66,0x00,0x00,0x00, -0x6f,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0x77,0x02,0x00,0x00, -0x76,0x02,0x00,0x00,0xc2,0x00,0x05,0x00,0x4d,0x00,0x00,0x00, -0x78,0x02,0x00,0x00,0x73,0x02,0x00,0x00,0x6a,0x00,0x00,0x00, -0x70,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0x79,0x02,0x00,0x00, -0x78,0x02,0x00,0x00,0x50,0x00,0x05,0x00,0x60,0x00,0x00,0x00, -0x7a,0x02,0x00,0x00,0x77,0x02,0x00,0x00,0x79,0x02,0x00,0x00, -0x83,0x00,0x05,0x00,0x60,0x00,0x00,0x00,0x7b,0x02,0x00,0x00, -0x7a,0x02,0x00,0x00,0xa2,0x00,0x00,0x00,0x8e,0x00,0x05,0x00, -0x60,0x00,0x00,0x00,0x7c,0x02,0x00,0x00,0x7b,0x02,0x00,0x00, -0x71,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x82,0x02,0x00,0x00,0x7f,0x00,0x00,0x00,0xf2,0x02,0x00,0x00, -0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00,0x84,0x02,0x00,0x00, -0x7c,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x41,0x00,0x06,0x00, -0x56,0x00,0x00,0x00,0x85,0x02,0x00,0x00,0x77,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x82,0x02,0x00,0x00,0x3e,0x00,0x03,0x00, -0x85,0x02,0x00,0x00,0x84,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x8c,0x02,0x00,0x00,0x7f,0x00,0x00,0x00, -0xf3,0x02,0x00,0x00,0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00, -0x8d,0x02,0x00,0x00,0x7c,0x02,0x00,0x00,0x01,0x00,0x00,0x00, -0x41,0x00,0x06,0x00,0x56,0x00,0x00,0x00,0x8e,0x02,0x00,0x00, -0x77,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x8c,0x02,0x00,0x00, -0x3e,0x00,0x03,0x00,0x8e,0x02,0x00,0x00,0x8d,0x02,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0x97,0x02,0x00,0x00, -0x57,0x00,0x00,0x00,0x41,0x00,0x08,0x00,0x5d,0x00,0x00,0x00, -0x98,0x02,0x00,0x00,0x54,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0x17,0x00,0x00,0x00,0xf4,0x02,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00,0x99,0x02,0x00,0x00, -0x98,0x02,0x00,0x00,0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x9a,0x02,0x00,0x00,0x99,0x02,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x9b,0x02,0x00,0x00,0x9a,0x02,0x00,0x00, -0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9c,0x02,0x00,0x00, -0x9b,0x02,0x00,0x00,0x66,0x00,0x00,0x00,0x6f,0x00,0x04,0x00, -0x4a,0x00,0x00,0x00,0x9d,0x02,0x00,0x00,0x9c,0x02,0x00,0x00, -0xc2,0x00,0x05,0x00,0x4d,0x00,0x00,0x00,0x9e,0x02,0x00,0x00, -0x99,0x02,0x00,0x00,0x6a,0x00,0x00,0x00,0x70,0x00,0x04,0x00, -0x4a,0x00,0x00,0x00,0x9f,0x02,0x00,0x00,0x9e,0x02,0x00,0x00, -0x50,0x00,0x05,0x00,0x60,0x00,0x00,0x00,0xa0,0x02,0x00,0x00, -0x9d,0x02,0x00,0x00,0x9f,0x02,0x00,0x00,0x83,0x00,0x05,0x00, -0x60,0x00,0x00,0x00,0xa1,0x02,0x00,0x00,0xa0,0x02,0x00,0x00, -0xa2,0x00,0x00,0x00,0x8e,0x00,0x05,0x00,0x60,0x00,0x00,0x00, -0xa2,0x02,0x00,0x00,0xa1,0x02,0x00,0x00,0x97,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa8,0x02,0x00,0x00, -0x7f,0x00,0x00,0x00,0xf4,0x02,0x00,0x00,0x51,0x00,0x05,0x00, -0x4a,0x00,0x00,0x00,0xaa,0x02,0x00,0x00,0xa2,0x02,0x00,0x00, -0x00,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x56,0x00,0x00,0x00, -0xab,0x02,0x00,0x00,0x77,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0xa8,0x02,0x00,0x00,0x3e,0x00,0x03,0x00,0xab,0x02,0x00,0x00, -0xaa,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xb2,0x02,0x00,0x00,0x7f,0x00,0x00,0x00,0xf5,0x02,0x00,0x00, -0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00,0xb3,0x02,0x00,0x00, -0xa2,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0x41,0x00,0x06,0x00, -0x56,0x00,0x00,0x00,0xb4,0x02,0x00,0x00,0x77,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0xb2,0x02,0x00,0x00,0x3e,0x00,0x03,0x00, -0xb4,0x02,0x00,0x00,0xb3,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4a,0x00,0x00,0x00,0xbd,0x02,0x00,0x00,0x57,0x00,0x00,0x00, -0x41,0x00,0x08,0x00,0x5d,0x00,0x00,0x00,0xbe,0x02,0x00,0x00, -0x54,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x17,0x00,0x00,0x00,0x66,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0xbf,0x02,0x00,0x00,0xbe,0x02,0x00,0x00, -0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0xc0,0x02,0x00,0x00, -0xbf,0x02,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xc1,0x02,0x00,0x00,0xc0,0x02,0x00,0x00,0xc7,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xc2,0x02,0x00,0x00,0xc1,0x02,0x00,0x00, -0x66,0x00,0x00,0x00,0x6f,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0xc3,0x02,0x00,0x00,0xc2,0x02,0x00,0x00,0xc2,0x00,0x05,0x00, -0x4d,0x00,0x00,0x00,0xc4,0x02,0x00,0x00,0xbf,0x02,0x00,0x00, -0x6a,0x00,0x00,0x00,0x70,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0xc5,0x02,0x00,0x00,0xc4,0x02,0x00,0x00,0x50,0x00,0x05,0x00, -0x60,0x00,0x00,0x00,0xc6,0x02,0x00,0x00,0xc3,0x02,0x00,0x00, -0xc5,0x02,0x00,0x00,0x83,0x00,0x05,0x00,0x60,0x00,0x00,0x00, -0xc7,0x02,0x00,0x00,0xc6,0x02,0x00,0x00,0xa2,0x00,0x00,0x00, -0x8e,0x00,0x05,0x00,0x60,0x00,0x00,0x00,0xc8,0x02,0x00,0x00, -0xc7,0x02,0x00,0x00,0xbd,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xce,0x02,0x00,0x00,0x7f,0x00,0x00,0x00, -0x66,0x00,0x00,0x00,0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00, -0xd0,0x02,0x00,0x00,0xc8,0x02,0x00,0x00,0x00,0x00,0x00,0x00, -0x41,0x00,0x06,0x00,0x56,0x00,0x00,0x00,0xd1,0x02,0x00,0x00, -0x77,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0xce,0x02,0x00,0x00, -0x3e,0x00,0x03,0x00,0xd1,0x02,0x00,0x00,0xd0,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xd8,0x02,0x00,0x00, -0x7f,0x00,0x00,0x00,0xf6,0x02,0x00,0x00,0x51,0x00,0x05,0x00, -0x4a,0x00,0x00,0x00,0xd9,0x02,0x00,0x00,0xc8,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x56,0x00,0x00,0x00, -0xda,0x02,0x00,0x00,0x77,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0xd8,0x02,0x00,0x00,0x3e,0x00,0x03,0x00,0xda,0x02,0x00,0x00, -0xd9,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x98,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0x98,0x00,0x00,0x00,0xfd,0x00,0x01,0x00, -0x38,0x00,0x01,0x00, -}; -const uint64_t dequant_q4_0_len = 8332; - -unsigned char dequant_q4_0_fp32_data[] = { -0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, 0x19,0x03,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, 0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x51,0x11,0x00,0x00, 0x11,0x00,0x02,0x00,0x60,0x11,0x00,0x00,0x0b,0x00,0x06,0x00, @@ -4122,758 +2535,10 @@ unsigned char dequant_q4_0_fp32_data[] = { 0x9b,0x00,0x00,0x00,0xfd,0x00,0x01,0x00,0x38,0x00,0x01,0x00, }; -const uint64_t dequant_q4_0_fp32_len = 8856; +const uint64_t dequant_q4_0_len = 8856; unsigned char dequant_q4_1_data[] = { 0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, -0x27,0x03,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, -0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x09,0x00,0x00,0x00, -0x11,0x00,0x02,0x00,0x27,0x00,0x00,0x00,0x11,0x00,0x02,0x00, -0x51,0x11,0x00,0x00,0x11,0x00,0x02,0x00,0x60,0x11,0x00,0x00, -0x0b,0x00,0x06,0x00,0x01,0x00,0x00,0x00,0x47,0x4c,0x53,0x4c, -0x2e,0x73,0x74,0x64,0x2e,0x34,0x35,0x30,0x00,0x00,0x00,0x00, -0x0e,0x00,0x03,0x00,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x0f,0x00,0x09,0x00,0x05,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x6d,0x61,0x69,0x6e,0x00,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x54,0x00,0x00,0x00,0x7b,0x00,0x00,0x00, -0x10,0x00,0x06,0x00,0x04,0x00,0x00,0x00,0x11,0x00,0x00,0x00, -0x00,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x0c,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x1c,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x14,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x14,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x14,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x08,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x14,0x00,0x00,0x00, -0x03,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x47,0x00,0x03,0x00,0x14,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x4f,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x50,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x50,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x50,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x51,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x48,0x00,0x04,0x00, -0x52,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x52,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, -0x52,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x54,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x54,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x78,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x48,0x00,0x04,0x00, -0x79,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x19,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x79,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, -0x79,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x7b,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x7b,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x9b,0x00,0x00,0x00, -0x0b,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x13,0x00,0x02,0x00, -0x02,0x00,0x00,0x00,0x21,0x00,0x03,0x00,0x03,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x15,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x15,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x17,0x00,0x04,0x00,0x0a,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x03,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x0b,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x0a,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x0b,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x0d,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x0e,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x1e,0x00,0x06,0x00, -0x14,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x15,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x14,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x15,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x17,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x18,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x1b,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x14,0x00,0x02,0x00,0x24,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x37,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x48,0x00,0x00,0x00,0x10,0x00,0x00,0x00, -0x16,0x00,0x03,0x00,0x4a,0x00,0x00,0x00,0x10,0x00,0x00,0x00, -0x15,0x00,0x04,0x00,0x4d,0x00,0x00,0x00,0x08,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x4e,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x1c,0x00,0x04,0x00, -0x4f,0x00,0x00,0x00,0x4d,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, -0x1e,0x00,0x05,0x00,0x50,0x00,0x00,0x00,0x4a,0x00,0x00,0x00, -0x4a,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, -0x51,0x00,0x00,0x00,0x50,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, -0x52,0x00,0x00,0x00,0x51,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x53,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x52,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x53,0x00,0x00,0x00,0x54,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x56,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x4a,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x61,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x4d,0x00,0x00,0x00, -0x17,0x00,0x04,0x00,0x64,0x00,0x00,0x00,0x4a,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x6a,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x6e,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x1d,0x00,0x03,0x00,0x78,0x00,0x00,0x00,0x4a,0x00,0x00,0x00, -0x1e,0x00,0x03,0x00,0x79,0x00,0x00,0x00,0x78,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x7a,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x79,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x7a,0x00,0x00,0x00, -0x7b,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x7d,0x00,0x00,0x00,0x03,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x94,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x9a,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x2c,0x00,0x06,0x00, -0x0a,0x00,0x00,0x00,0x9b,0x00,0x00,0x00,0x9a,0x00,0x00,0x00, -0x94,0x00,0x00,0x00,0x94,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x0e,0x03,0x00,0x00,0x11,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x0f,0x03,0x00,0x00, -0x12,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x10,0x03,0x00,0x00,0x13,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x11,0x03,0x00,0x00,0x14,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x12,0x03,0x00,0x00, -0x05,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x13,0x03,0x00,0x00,0x15,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x14,0x03,0x00,0x00,0x06,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x15,0x03,0x00,0x00, -0x16,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x16,0x03,0x00,0x00,0x07,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x17,0x03,0x00,0x00,0x17,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x18,0x03,0x00,0x00, -0x08,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x19,0x03,0x00,0x00,0x18,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x1a,0x03,0x00,0x00,0x09,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x1b,0x03,0x00,0x00, -0x19,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x1c,0x03,0x00,0x00,0x0a,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x1d,0x03,0x00,0x00,0x1a,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x1e,0x03,0x00,0x00, -0x0b,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x1f,0x03,0x00,0x00,0x1b,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x20,0x03,0x00,0x00,0x0c,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x21,0x03,0x00,0x00, -0x1c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x22,0x03,0x00,0x00,0x0d,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x23,0x03,0x00,0x00,0x1d,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x24,0x03,0x00,0x00, -0x0e,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x25,0x03,0x00,0x00,0x1e,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x26,0x03,0x00,0x00,0x1f,0x00,0x00,0x00, -0x36,0x00,0x05,0x00,0x02,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x05,0x00,0x00,0x00,0xf7,0x00,0x03,0x00,0x9c,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0xfb,0x00,0x03,0x00,0x0d,0x00,0x00,0x00, -0x9d,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x9d,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x0e,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x0d,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x11,0x00,0x00,0x00, -0x10,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x18,0x00,0x00,0x00, -0x19,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x17,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x1a,0x00,0x00,0x00, -0x19,0x00,0x00,0x00,0x87,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x1c,0x00,0x00,0x00,0x1a,0x00,0x00,0x00,0x1b,0x00,0x00,0x00, -0x8b,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x1d,0x00,0x00,0x00, -0x11,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x87,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x11,0x00,0x00,0x00, -0x1c,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x26,0x00,0x00,0x00,0x1d,0x00,0x00,0x00,0x1b,0x00,0x00,0x00, -0xaf,0x00,0x05,0x00,0x24,0x00,0x00,0x00,0x29,0x00,0x00,0x00, -0x26,0x00,0x00,0x00,0x1a,0x00,0x00,0x00,0xa8,0x00,0x04,0x00, -0x24,0x00,0x00,0x00,0x2a,0x00,0x00,0x00,0x29,0x00,0x00,0x00, -0xf7,0x00,0x03,0x00,0x2c,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x2a,0x00,0x00,0x00,0x2b,0x00,0x00,0x00, -0x2c,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x2b,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x18,0x00,0x00,0x00,0x2f,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x30,0x00,0x00,0x00,0x2f,0x00,0x00,0x00, -0xaf,0x00,0x05,0x00,0x24,0x00,0x00,0x00,0x31,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x30,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x2c,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x2c,0x00,0x00,0x00, -0xf5,0x00,0x07,0x00,0x24,0x00,0x00,0x00,0x32,0x00,0x00,0x00, -0x29,0x00,0x00,0x00,0x9d,0x00,0x00,0x00,0x31,0x00,0x00,0x00, -0x2b,0x00,0x00,0x00,0xf7,0x00,0x03,0x00,0x34,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x32,0x00,0x00,0x00, -0x33,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x33,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x9c,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0x34,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x18,0x00,0x00,0x00,0x38,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0x37,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x38,0x00,0x00,0x00,0x87,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x1b,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x3e,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x3e,0x00,0x00,0x00,0x1d,0x00,0x00,0x00,0x41,0x00,0x07,0x00, -0x56,0x00,0x00,0x00,0x57,0x00,0x00,0x00,0x54,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0x58,0x00,0x00,0x00, -0x57,0x00,0x00,0x00,0x41,0x00,0x07,0x00,0x56,0x00,0x00,0x00, -0x5b,0x00,0x00,0x00,0x54,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0x17,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4a,0x00,0x00,0x00,0x5c,0x00,0x00,0x00,0x5b,0x00,0x00,0x00, -0x41,0x00,0x08,0x00,0x61,0x00,0x00,0x00,0x62,0x00,0x00,0x00, -0x54,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x37,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0x63,0x00,0x00,0x00,0x62,0x00,0x00,0x00, -0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x68,0x00,0x00,0x00, -0x63,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x69,0x00,0x00,0x00,0x68,0x00,0x00,0x00,0xc7,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x6b,0x00,0x00,0x00,0x69,0x00,0x00,0x00, -0x6a,0x00,0x00,0x00,0x6f,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x6b,0x00,0x00,0x00,0xc2,0x00,0x05,0x00, -0x4d,0x00,0x00,0x00,0x6f,0x00,0x00,0x00,0x63,0x00,0x00,0x00, -0x6e,0x00,0x00,0x00,0x70,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0x70,0x00,0x00,0x00,0x6f,0x00,0x00,0x00,0x50,0x00,0x05,0x00, -0x64,0x00,0x00,0x00,0x71,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x70,0x00,0x00,0x00,0x8e,0x00,0x05,0x00,0x64,0x00,0x00,0x00, -0x74,0x00,0x00,0x00,0x71,0x00,0x00,0x00,0x58,0x00,0x00,0x00, -0x50,0x00,0x05,0x00,0x64,0x00,0x00,0x00,0x76,0x00,0x00,0x00, -0x5c,0x00,0x00,0x00,0x5c,0x00,0x00,0x00,0x81,0x00,0x05,0x00, -0x64,0x00,0x00,0x00,0x77,0x00,0x00,0x00,0x74,0x00,0x00,0x00, -0x76,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x18,0x00,0x00,0x00, -0x7e,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x7d,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x7f,0x00,0x00,0x00, -0x7e,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x80,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x7f,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x83,0x00,0x00,0x00, -0x80,0x00,0x00,0x00,0x26,0x00,0x00,0x00,0x51,0x00,0x05,0x00, -0x4a,0x00,0x00,0x00,0x88,0x00,0x00,0x00,0x77,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x56,0x00,0x00,0x00, -0x89,0x00,0x00,0x00,0x7b,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x83,0x00,0x00,0x00,0x3e,0x00,0x03,0x00,0x89,0x00,0x00,0x00, -0x88,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x93,0x00,0x00,0x00,0x83,0x00,0x00,0x00,0x48,0x00,0x00,0x00, -0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00,0x96,0x00,0x00,0x00, -0x77,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x41,0x00,0x06,0x00, -0x56,0x00,0x00,0x00,0x97,0x00,0x00,0x00,0x7b,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x93,0x00,0x00,0x00,0x3e,0x00,0x03,0x00, -0x97,0x00,0x00,0x00,0x96,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4a,0x00,0x00,0x00,0xac,0x00,0x00,0x00,0x57,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0xae,0x00,0x00,0x00, -0x5b,0x00,0x00,0x00,0x41,0x00,0x08,0x00,0x61,0x00,0x00,0x00, -0xaf,0x00,0x00,0x00,0x54,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x17,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00,0xb0,0x00,0x00,0x00, -0xaf,0x00,0x00,0x00,0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0xb1,0x00,0x00,0x00,0xb0,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xb2,0x00,0x00,0x00,0xb1,0x00,0x00,0x00, -0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xb3,0x00,0x00,0x00, -0xb2,0x00,0x00,0x00,0x6a,0x00,0x00,0x00,0x6f,0x00,0x04,0x00, -0x4a,0x00,0x00,0x00,0xb4,0x00,0x00,0x00,0xb3,0x00,0x00,0x00, -0xc2,0x00,0x05,0x00,0x4d,0x00,0x00,0x00,0xb5,0x00,0x00,0x00, -0xb0,0x00,0x00,0x00,0x6e,0x00,0x00,0x00,0x70,0x00,0x04,0x00, -0x4a,0x00,0x00,0x00,0xb6,0x00,0x00,0x00,0xb5,0x00,0x00,0x00, -0x50,0x00,0x05,0x00,0x64,0x00,0x00,0x00,0xb7,0x00,0x00,0x00, -0xb4,0x00,0x00,0x00,0xb6,0x00,0x00,0x00,0x8e,0x00,0x05,0x00, -0x64,0x00,0x00,0x00,0xb8,0x00,0x00,0x00,0xb7,0x00,0x00,0x00, -0xac,0x00,0x00,0x00,0x50,0x00,0x05,0x00,0x64,0x00,0x00,0x00, -0xb9,0x00,0x00,0x00,0xae,0x00,0x00,0x00,0xae,0x00,0x00,0x00, -0x81,0x00,0x05,0x00,0x64,0x00,0x00,0x00,0xba,0x00,0x00,0x00, -0xb8,0x00,0x00,0x00,0xb9,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xc0,0x00,0x00,0x00,0x83,0x00,0x00,0x00, -0x17,0x00,0x00,0x00,0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00, -0xc2,0x00,0x00,0x00,0xba,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x41,0x00,0x06,0x00,0x56,0x00,0x00,0x00,0xc3,0x00,0x00,0x00, -0x7b,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0xc0,0x00,0x00,0x00, -0x3e,0x00,0x03,0x00,0xc3,0x00,0x00,0x00,0xc2,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xca,0x00,0x00,0x00, -0x83,0x00,0x00,0x00,0x0e,0x03,0x00,0x00,0x51,0x00,0x05,0x00, -0x4a,0x00,0x00,0x00,0xcb,0x00,0x00,0x00,0xba,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x56,0x00,0x00,0x00, -0xcc,0x00,0x00,0x00,0x7b,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0xca,0x00,0x00,0x00,0x3e,0x00,0x03,0x00,0xcc,0x00,0x00,0x00, -0xcb,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0xd5,0x00,0x00,0x00,0x57,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4a,0x00,0x00,0x00,0xd7,0x00,0x00,0x00,0x5b,0x00,0x00,0x00, -0x41,0x00,0x08,0x00,0x61,0x00,0x00,0x00,0xd8,0x00,0x00,0x00, -0x54,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x37,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0xd9,0x00,0x00,0x00,0xd8,0x00,0x00,0x00, -0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0xda,0x00,0x00,0x00, -0xd9,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xdb,0x00,0x00,0x00,0xda,0x00,0x00,0x00,0xc7,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xdc,0x00,0x00,0x00,0xdb,0x00,0x00,0x00, -0x6a,0x00,0x00,0x00,0x6f,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0xdd,0x00,0x00,0x00,0xdc,0x00,0x00,0x00,0xc2,0x00,0x05,0x00, -0x4d,0x00,0x00,0x00,0xde,0x00,0x00,0x00,0xd9,0x00,0x00,0x00, -0x6e,0x00,0x00,0x00,0x70,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0xdf,0x00,0x00,0x00,0xde,0x00,0x00,0x00,0x50,0x00,0x05,0x00, -0x64,0x00,0x00,0x00,0xe0,0x00,0x00,0x00,0xdd,0x00,0x00,0x00, -0xdf,0x00,0x00,0x00,0x8e,0x00,0x05,0x00,0x64,0x00,0x00,0x00, -0xe1,0x00,0x00,0x00,0xe0,0x00,0x00,0x00,0xd5,0x00,0x00,0x00, -0x50,0x00,0x05,0x00,0x64,0x00,0x00,0x00,0xe2,0x00,0x00,0x00, -0xd7,0x00,0x00,0x00,0xd7,0x00,0x00,0x00,0x81,0x00,0x05,0x00, -0x64,0x00,0x00,0x00,0xe3,0x00,0x00,0x00,0xe1,0x00,0x00,0x00, -0xe2,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xe9,0x00,0x00,0x00,0x83,0x00,0x00,0x00,0x37,0x00,0x00,0x00, -0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00,0xeb,0x00,0x00,0x00, -0xe3,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x41,0x00,0x06,0x00, -0x56,0x00,0x00,0x00,0xec,0x00,0x00,0x00,0x7b,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0xe9,0x00,0x00,0x00,0x3e,0x00,0x03,0x00, -0xec,0x00,0x00,0x00,0xeb,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xf3,0x00,0x00,0x00,0x83,0x00,0x00,0x00, -0x0f,0x03,0x00,0x00,0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00, -0xf4,0x00,0x00,0x00,0xe3,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x41,0x00,0x06,0x00,0x56,0x00,0x00,0x00,0xf5,0x00,0x00,0x00, -0x7b,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0xf3,0x00,0x00,0x00, -0x3e,0x00,0x03,0x00,0xf5,0x00,0x00,0x00,0xf4,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0xfe,0x00,0x00,0x00, -0x57,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0x00,0x01,0x00,0x00,0x5b,0x00,0x00,0x00,0x41,0x00,0x08,0x00, -0x61,0x00,0x00,0x00,0x01,0x01,0x00,0x00,0x54,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x37,0x00,0x00,0x00, -0x7d,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0x02,0x01,0x00,0x00,0x01,0x01,0x00,0x00,0x71,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x03,0x01,0x00,0x00,0x02,0x01,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x04,0x01,0x00,0x00, -0x03,0x01,0x00,0x00,0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x05,0x01,0x00,0x00,0x04,0x01,0x00,0x00,0x6a,0x00,0x00,0x00, -0x6f,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0x06,0x01,0x00,0x00, -0x05,0x01,0x00,0x00,0xc2,0x00,0x05,0x00,0x4d,0x00,0x00,0x00, -0x07,0x01,0x00,0x00,0x02,0x01,0x00,0x00,0x6e,0x00,0x00,0x00, -0x70,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0x08,0x01,0x00,0x00, -0x07,0x01,0x00,0x00,0x50,0x00,0x05,0x00,0x64,0x00,0x00,0x00, -0x09,0x01,0x00,0x00,0x06,0x01,0x00,0x00,0x08,0x01,0x00,0x00, -0x8e,0x00,0x05,0x00,0x64,0x00,0x00,0x00,0x0a,0x01,0x00,0x00, -0x09,0x01,0x00,0x00,0xfe,0x00,0x00,0x00,0x50,0x00,0x05,0x00, -0x64,0x00,0x00,0x00,0x0b,0x01,0x00,0x00,0x00,0x01,0x00,0x00, -0x00,0x01,0x00,0x00,0x81,0x00,0x05,0x00,0x64,0x00,0x00,0x00, -0x0c,0x01,0x00,0x00,0x0a,0x01,0x00,0x00,0x0b,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x12,0x01,0x00,0x00, -0x83,0x00,0x00,0x00,0x7d,0x00,0x00,0x00,0x51,0x00,0x05,0x00, -0x4a,0x00,0x00,0x00,0x14,0x01,0x00,0x00,0x0c,0x01,0x00,0x00, -0x00,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x56,0x00,0x00,0x00, -0x15,0x01,0x00,0x00,0x7b,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x12,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x15,0x01,0x00,0x00, -0x14,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x1c,0x01,0x00,0x00,0x83,0x00,0x00,0x00,0x10,0x03,0x00,0x00, -0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00,0x1d,0x01,0x00,0x00, -0x0c,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x41,0x00,0x06,0x00, -0x56,0x00,0x00,0x00,0x1e,0x01,0x00,0x00,0x7b,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x1c,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0x1e,0x01,0x00,0x00,0x1d,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4a,0x00,0x00,0x00,0x27,0x01,0x00,0x00,0x57,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0x29,0x01,0x00,0x00, -0x5b,0x00,0x00,0x00,0x41,0x00,0x08,0x00,0x61,0x00,0x00,0x00, -0x2a,0x01,0x00,0x00,0x54,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x6e,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00,0x2b,0x01,0x00,0x00, -0x2a,0x01,0x00,0x00,0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x2c,0x01,0x00,0x00,0x2b,0x01,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x2d,0x01,0x00,0x00,0x2c,0x01,0x00,0x00, -0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x2e,0x01,0x00,0x00, -0x2d,0x01,0x00,0x00,0x6a,0x00,0x00,0x00,0x6f,0x00,0x04,0x00, -0x4a,0x00,0x00,0x00,0x2f,0x01,0x00,0x00,0x2e,0x01,0x00,0x00, -0xc2,0x00,0x05,0x00,0x4d,0x00,0x00,0x00,0x30,0x01,0x00,0x00, -0x2b,0x01,0x00,0x00,0x6e,0x00,0x00,0x00,0x70,0x00,0x04,0x00, -0x4a,0x00,0x00,0x00,0x31,0x01,0x00,0x00,0x30,0x01,0x00,0x00, -0x50,0x00,0x05,0x00,0x64,0x00,0x00,0x00,0x32,0x01,0x00,0x00, -0x2f,0x01,0x00,0x00,0x31,0x01,0x00,0x00,0x8e,0x00,0x05,0x00, -0x64,0x00,0x00,0x00,0x33,0x01,0x00,0x00,0x32,0x01,0x00,0x00, -0x27,0x01,0x00,0x00,0x50,0x00,0x05,0x00,0x64,0x00,0x00,0x00, -0x34,0x01,0x00,0x00,0x29,0x01,0x00,0x00,0x29,0x01,0x00,0x00, -0x81,0x00,0x05,0x00,0x64,0x00,0x00,0x00,0x35,0x01,0x00,0x00, -0x33,0x01,0x00,0x00,0x34,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x3b,0x01,0x00,0x00,0x83,0x00,0x00,0x00, -0x6e,0x00,0x00,0x00,0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00, -0x3d,0x01,0x00,0x00,0x35,0x01,0x00,0x00,0x00,0x00,0x00,0x00, -0x41,0x00,0x06,0x00,0x56,0x00,0x00,0x00,0x3e,0x01,0x00,0x00, -0x7b,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x3b,0x01,0x00,0x00, -0x3e,0x00,0x03,0x00,0x3e,0x01,0x00,0x00,0x3d,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x45,0x01,0x00,0x00, -0x83,0x00,0x00,0x00,0x11,0x03,0x00,0x00,0x51,0x00,0x05,0x00, -0x4a,0x00,0x00,0x00,0x46,0x01,0x00,0x00,0x35,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x56,0x00,0x00,0x00, -0x47,0x01,0x00,0x00,0x7b,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x45,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x47,0x01,0x00,0x00, -0x46,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0x50,0x01,0x00,0x00,0x57,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4a,0x00,0x00,0x00,0x52,0x01,0x00,0x00,0x5b,0x00,0x00,0x00, -0x41,0x00,0x08,0x00,0x61,0x00,0x00,0x00,0x53,0x01,0x00,0x00, -0x54,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x37,0x00,0x00,0x00,0x12,0x03,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0x54,0x01,0x00,0x00,0x53,0x01,0x00,0x00, -0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x55,0x01,0x00,0x00, -0x54,0x01,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x56,0x01,0x00,0x00,0x55,0x01,0x00,0x00,0xc7,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x57,0x01,0x00,0x00,0x56,0x01,0x00,0x00, -0x6a,0x00,0x00,0x00,0x6f,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0x58,0x01,0x00,0x00,0x57,0x01,0x00,0x00,0xc2,0x00,0x05,0x00, -0x4d,0x00,0x00,0x00,0x59,0x01,0x00,0x00,0x54,0x01,0x00,0x00, -0x6e,0x00,0x00,0x00,0x70,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0x5a,0x01,0x00,0x00,0x59,0x01,0x00,0x00,0x50,0x00,0x05,0x00, -0x64,0x00,0x00,0x00,0x5b,0x01,0x00,0x00,0x58,0x01,0x00,0x00, -0x5a,0x01,0x00,0x00,0x8e,0x00,0x05,0x00,0x64,0x00,0x00,0x00, -0x5c,0x01,0x00,0x00,0x5b,0x01,0x00,0x00,0x50,0x01,0x00,0x00, -0x50,0x00,0x05,0x00,0x64,0x00,0x00,0x00,0x5d,0x01,0x00,0x00, -0x52,0x01,0x00,0x00,0x52,0x01,0x00,0x00,0x81,0x00,0x05,0x00, -0x64,0x00,0x00,0x00,0x5e,0x01,0x00,0x00,0x5c,0x01,0x00,0x00, -0x5d,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x64,0x01,0x00,0x00,0x83,0x00,0x00,0x00,0x12,0x03,0x00,0x00, -0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00,0x66,0x01,0x00,0x00, -0x5e,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x41,0x00,0x06,0x00, -0x56,0x00,0x00,0x00,0x67,0x01,0x00,0x00,0x7b,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x64,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0x67,0x01,0x00,0x00,0x66,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x6e,0x01,0x00,0x00,0x83,0x00,0x00,0x00, -0x13,0x03,0x00,0x00,0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00, -0x6f,0x01,0x00,0x00,0x5e,0x01,0x00,0x00,0x01,0x00,0x00,0x00, -0x41,0x00,0x06,0x00,0x56,0x00,0x00,0x00,0x70,0x01,0x00,0x00, -0x7b,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x6e,0x01,0x00,0x00, -0x3e,0x00,0x03,0x00,0x70,0x01,0x00,0x00,0x6f,0x01,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0x79,0x01,0x00,0x00, -0x57,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0x7b,0x01,0x00,0x00,0x5b,0x00,0x00,0x00,0x41,0x00,0x08,0x00, -0x61,0x00,0x00,0x00,0x7c,0x01,0x00,0x00,0x54,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x37,0x00,0x00,0x00, -0x14,0x03,0x00,0x00,0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0x7d,0x01,0x00,0x00,0x7c,0x01,0x00,0x00,0x71,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x7e,0x01,0x00,0x00,0x7d,0x01,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x7f,0x01,0x00,0x00, -0x7e,0x01,0x00,0x00,0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x80,0x01,0x00,0x00,0x7f,0x01,0x00,0x00,0x6a,0x00,0x00,0x00, -0x6f,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0x81,0x01,0x00,0x00, -0x80,0x01,0x00,0x00,0xc2,0x00,0x05,0x00,0x4d,0x00,0x00,0x00, -0x82,0x01,0x00,0x00,0x7d,0x01,0x00,0x00,0x6e,0x00,0x00,0x00, -0x70,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0x83,0x01,0x00,0x00, -0x82,0x01,0x00,0x00,0x50,0x00,0x05,0x00,0x64,0x00,0x00,0x00, -0x84,0x01,0x00,0x00,0x81,0x01,0x00,0x00,0x83,0x01,0x00,0x00, -0x8e,0x00,0x05,0x00,0x64,0x00,0x00,0x00,0x85,0x01,0x00,0x00, -0x84,0x01,0x00,0x00,0x79,0x01,0x00,0x00,0x50,0x00,0x05,0x00, -0x64,0x00,0x00,0x00,0x86,0x01,0x00,0x00,0x7b,0x01,0x00,0x00, -0x7b,0x01,0x00,0x00,0x81,0x00,0x05,0x00,0x64,0x00,0x00,0x00, -0x87,0x01,0x00,0x00,0x85,0x01,0x00,0x00,0x86,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8d,0x01,0x00,0x00, -0x83,0x00,0x00,0x00,0x14,0x03,0x00,0x00,0x51,0x00,0x05,0x00, -0x4a,0x00,0x00,0x00,0x8f,0x01,0x00,0x00,0x87,0x01,0x00,0x00, -0x00,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x56,0x00,0x00,0x00, -0x90,0x01,0x00,0x00,0x7b,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x8d,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x90,0x01,0x00,0x00, -0x8f,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x97,0x01,0x00,0x00,0x83,0x00,0x00,0x00,0x15,0x03,0x00,0x00, -0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00,0x98,0x01,0x00,0x00, -0x87,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x41,0x00,0x06,0x00, -0x56,0x00,0x00,0x00,0x99,0x01,0x00,0x00,0x7b,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x97,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0x99,0x01,0x00,0x00,0x98,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4a,0x00,0x00,0x00,0xa2,0x01,0x00,0x00,0x57,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0xa4,0x01,0x00,0x00, -0x5b,0x00,0x00,0x00,0x41,0x00,0x08,0x00,0x61,0x00,0x00,0x00, -0xa5,0x01,0x00,0x00,0x54,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x16,0x03,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00,0xa6,0x01,0x00,0x00, -0xa5,0x01,0x00,0x00,0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0xa7,0x01,0x00,0x00,0xa6,0x01,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xa8,0x01,0x00,0x00,0xa7,0x01,0x00,0x00, -0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa9,0x01,0x00,0x00, -0xa8,0x01,0x00,0x00,0x6a,0x00,0x00,0x00,0x6f,0x00,0x04,0x00, -0x4a,0x00,0x00,0x00,0xaa,0x01,0x00,0x00,0xa9,0x01,0x00,0x00, -0xc2,0x00,0x05,0x00,0x4d,0x00,0x00,0x00,0xab,0x01,0x00,0x00, -0xa6,0x01,0x00,0x00,0x6e,0x00,0x00,0x00,0x70,0x00,0x04,0x00, -0x4a,0x00,0x00,0x00,0xac,0x01,0x00,0x00,0xab,0x01,0x00,0x00, -0x50,0x00,0x05,0x00,0x64,0x00,0x00,0x00,0xad,0x01,0x00,0x00, -0xaa,0x01,0x00,0x00,0xac,0x01,0x00,0x00,0x8e,0x00,0x05,0x00, -0x64,0x00,0x00,0x00,0xae,0x01,0x00,0x00,0xad,0x01,0x00,0x00, -0xa2,0x01,0x00,0x00,0x50,0x00,0x05,0x00,0x64,0x00,0x00,0x00, -0xaf,0x01,0x00,0x00,0xa4,0x01,0x00,0x00,0xa4,0x01,0x00,0x00, -0x81,0x00,0x05,0x00,0x64,0x00,0x00,0x00,0xb0,0x01,0x00,0x00, -0xae,0x01,0x00,0x00,0xaf,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xb6,0x01,0x00,0x00,0x83,0x00,0x00,0x00, -0x16,0x03,0x00,0x00,0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00, -0xb8,0x01,0x00,0x00,0xb0,0x01,0x00,0x00,0x00,0x00,0x00,0x00, -0x41,0x00,0x06,0x00,0x56,0x00,0x00,0x00,0xb9,0x01,0x00,0x00, -0x7b,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0xb6,0x01,0x00,0x00, -0x3e,0x00,0x03,0x00,0xb9,0x01,0x00,0x00,0xb8,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xc0,0x01,0x00,0x00, -0x83,0x00,0x00,0x00,0x17,0x03,0x00,0x00,0x51,0x00,0x05,0x00, -0x4a,0x00,0x00,0x00,0xc1,0x01,0x00,0x00,0xb0,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x56,0x00,0x00,0x00, -0xc2,0x01,0x00,0x00,0x7b,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0xc0,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0xc2,0x01,0x00,0x00, -0xc1,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0xcb,0x01,0x00,0x00,0x57,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4a,0x00,0x00,0x00,0xcd,0x01,0x00,0x00,0x5b,0x00,0x00,0x00, -0x41,0x00,0x08,0x00,0x61,0x00,0x00,0x00,0xce,0x01,0x00,0x00, -0x54,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x37,0x00,0x00,0x00,0x18,0x03,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0xcf,0x01,0x00,0x00,0xce,0x01,0x00,0x00, -0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0xd0,0x01,0x00,0x00, -0xcf,0x01,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xd1,0x01,0x00,0x00,0xd0,0x01,0x00,0x00,0xc7,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xd2,0x01,0x00,0x00,0xd1,0x01,0x00,0x00, -0x6a,0x00,0x00,0x00,0x6f,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0xd3,0x01,0x00,0x00,0xd2,0x01,0x00,0x00,0xc2,0x00,0x05,0x00, -0x4d,0x00,0x00,0x00,0xd4,0x01,0x00,0x00,0xcf,0x01,0x00,0x00, -0x6e,0x00,0x00,0x00,0x70,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0xd5,0x01,0x00,0x00,0xd4,0x01,0x00,0x00,0x50,0x00,0x05,0x00, -0x64,0x00,0x00,0x00,0xd6,0x01,0x00,0x00,0xd3,0x01,0x00,0x00, -0xd5,0x01,0x00,0x00,0x8e,0x00,0x05,0x00,0x64,0x00,0x00,0x00, -0xd7,0x01,0x00,0x00,0xd6,0x01,0x00,0x00,0xcb,0x01,0x00,0x00, -0x50,0x00,0x05,0x00,0x64,0x00,0x00,0x00,0xd8,0x01,0x00,0x00, -0xcd,0x01,0x00,0x00,0xcd,0x01,0x00,0x00,0x81,0x00,0x05,0x00, -0x64,0x00,0x00,0x00,0xd9,0x01,0x00,0x00,0xd7,0x01,0x00,0x00, -0xd8,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xdf,0x01,0x00,0x00,0x83,0x00,0x00,0x00,0x18,0x03,0x00,0x00, -0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00,0xe1,0x01,0x00,0x00, -0xd9,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x41,0x00,0x06,0x00, -0x56,0x00,0x00,0x00,0xe2,0x01,0x00,0x00,0x7b,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0xdf,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0xe2,0x01,0x00,0x00,0xe1,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xe9,0x01,0x00,0x00,0x83,0x00,0x00,0x00, -0x19,0x03,0x00,0x00,0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00, -0xea,0x01,0x00,0x00,0xd9,0x01,0x00,0x00,0x01,0x00,0x00,0x00, -0x41,0x00,0x06,0x00,0x56,0x00,0x00,0x00,0xeb,0x01,0x00,0x00, -0x7b,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0xe9,0x01,0x00,0x00, -0x3e,0x00,0x03,0x00,0xeb,0x01,0x00,0x00,0xea,0x01,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0xf4,0x01,0x00,0x00, -0x57,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0xf6,0x01,0x00,0x00,0x5b,0x00,0x00,0x00,0x41,0x00,0x08,0x00, -0x61,0x00,0x00,0x00,0xf7,0x01,0x00,0x00,0x54,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x37,0x00,0x00,0x00, -0x1a,0x03,0x00,0x00,0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0xf8,0x01,0x00,0x00,0xf7,0x01,0x00,0x00,0x71,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0xf9,0x01,0x00,0x00,0xf8,0x01,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xfa,0x01,0x00,0x00, -0xf9,0x01,0x00,0x00,0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xfb,0x01,0x00,0x00,0xfa,0x01,0x00,0x00,0x6a,0x00,0x00,0x00, -0x6f,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0xfc,0x01,0x00,0x00, -0xfb,0x01,0x00,0x00,0xc2,0x00,0x05,0x00,0x4d,0x00,0x00,0x00, -0xfd,0x01,0x00,0x00,0xf8,0x01,0x00,0x00,0x6e,0x00,0x00,0x00, -0x70,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0xfe,0x01,0x00,0x00, -0xfd,0x01,0x00,0x00,0x50,0x00,0x05,0x00,0x64,0x00,0x00,0x00, -0xff,0x01,0x00,0x00,0xfc,0x01,0x00,0x00,0xfe,0x01,0x00,0x00, -0x8e,0x00,0x05,0x00,0x64,0x00,0x00,0x00,0x00,0x02,0x00,0x00, -0xff,0x01,0x00,0x00,0xf4,0x01,0x00,0x00,0x50,0x00,0x05,0x00, -0x64,0x00,0x00,0x00,0x01,0x02,0x00,0x00,0xf6,0x01,0x00,0x00, -0xf6,0x01,0x00,0x00,0x81,0x00,0x05,0x00,0x64,0x00,0x00,0x00, -0x02,0x02,0x00,0x00,0x00,0x02,0x00,0x00,0x01,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x08,0x02,0x00,0x00, -0x83,0x00,0x00,0x00,0x1a,0x03,0x00,0x00,0x51,0x00,0x05,0x00, -0x4a,0x00,0x00,0x00,0x0a,0x02,0x00,0x00,0x02,0x02,0x00,0x00, -0x00,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x56,0x00,0x00,0x00, -0x0b,0x02,0x00,0x00,0x7b,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x08,0x02,0x00,0x00,0x3e,0x00,0x03,0x00,0x0b,0x02,0x00,0x00, -0x0a,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x12,0x02,0x00,0x00,0x83,0x00,0x00,0x00,0x1b,0x03,0x00,0x00, -0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00,0x13,0x02,0x00,0x00, -0x02,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0x41,0x00,0x06,0x00, -0x56,0x00,0x00,0x00,0x14,0x02,0x00,0x00,0x7b,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x12,0x02,0x00,0x00,0x3e,0x00,0x03,0x00, -0x14,0x02,0x00,0x00,0x13,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4a,0x00,0x00,0x00,0x1d,0x02,0x00,0x00,0x57,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0x1f,0x02,0x00,0x00, -0x5b,0x00,0x00,0x00,0x41,0x00,0x08,0x00,0x61,0x00,0x00,0x00, -0x20,0x02,0x00,0x00,0x54,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x1c,0x03,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00,0x21,0x02,0x00,0x00, -0x20,0x02,0x00,0x00,0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x22,0x02,0x00,0x00,0x21,0x02,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x23,0x02,0x00,0x00,0x22,0x02,0x00,0x00, -0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x24,0x02,0x00,0x00, -0x23,0x02,0x00,0x00,0x6a,0x00,0x00,0x00,0x6f,0x00,0x04,0x00, -0x4a,0x00,0x00,0x00,0x25,0x02,0x00,0x00,0x24,0x02,0x00,0x00, -0xc2,0x00,0x05,0x00,0x4d,0x00,0x00,0x00,0x26,0x02,0x00,0x00, -0x21,0x02,0x00,0x00,0x6e,0x00,0x00,0x00,0x70,0x00,0x04,0x00, -0x4a,0x00,0x00,0x00,0x27,0x02,0x00,0x00,0x26,0x02,0x00,0x00, -0x50,0x00,0x05,0x00,0x64,0x00,0x00,0x00,0x28,0x02,0x00,0x00, -0x25,0x02,0x00,0x00,0x27,0x02,0x00,0x00,0x8e,0x00,0x05,0x00, -0x64,0x00,0x00,0x00,0x29,0x02,0x00,0x00,0x28,0x02,0x00,0x00, -0x1d,0x02,0x00,0x00,0x50,0x00,0x05,0x00,0x64,0x00,0x00,0x00, -0x2a,0x02,0x00,0x00,0x1f,0x02,0x00,0x00,0x1f,0x02,0x00,0x00, -0x81,0x00,0x05,0x00,0x64,0x00,0x00,0x00,0x2b,0x02,0x00,0x00, -0x29,0x02,0x00,0x00,0x2a,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x31,0x02,0x00,0x00,0x83,0x00,0x00,0x00, -0x1c,0x03,0x00,0x00,0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00, -0x33,0x02,0x00,0x00,0x2b,0x02,0x00,0x00,0x00,0x00,0x00,0x00, -0x41,0x00,0x06,0x00,0x56,0x00,0x00,0x00,0x34,0x02,0x00,0x00, -0x7b,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x31,0x02,0x00,0x00, -0x3e,0x00,0x03,0x00,0x34,0x02,0x00,0x00,0x33,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3b,0x02,0x00,0x00, -0x83,0x00,0x00,0x00,0x1d,0x03,0x00,0x00,0x51,0x00,0x05,0x00, -0x4a,0x00,0x00,0x00,0x3c,0x02,0x00,0x00,0x2b,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x56,0x00,0x00,0x00, -0x3d,0x02,0x00,0x00,0x7b,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x3b,0x02,0x00,0x00,0x3e,0x00,0x03,0x00,0x3d,0x02,0x00,0x00, -0x3c,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0x46,0x02,0x00,0x00,0x57,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4a,0x00,0x00,0x00,0x48,0x02,0x00,0x00,0x5b,0x00,0x00,0x00, -0x41,0x00,0x08,0x00,0x61,0x00,0x00,0x00,0x49,0x02,0x00,0x00, -0x54,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x37,0x00,0x00,0x00,0x1e,0x03,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0x4a,0x02,0x00,0x00,0x49,0x02,0x00,0x00, -0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x4b,0x02,0x00,0x00, -0x4a,0x02,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x4c,0x02,0x00,0x00,0x4b,0x02,0x00,0x00,0xc7,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x4d,0x02,0x00,0x00,0x4c,0x02,0x00,0x00, -0x6a,0x00,0x00,0x00,0x6f,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0x4e,0x02,0x00,0x00,0x4d,0x02,0x00,0x00,0xc2,0x00,0x05,0x00, -0x4d,0x00,0x00,0x00,0x4f,0x02,0x00,0x00,0x4a,0x02,0x00,0x00, -0x6e,0x00,0x00,0x00,0x70,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0x50,0x02,0x00,0x00,0x4f,0x02,0x00,0x00,0x50,0x00,0x05,0x00, -0x64,0x00,0x00,0x00,0x51,0x02,0x00,0x00,0x4e,0x02,0x00,0x00, -0x50,0x02,0x00,0x00,0x8e,0x00,0x05,0x00,0x64,0x00,0x00,0x00, -0x52,0x02,0x00,0x00,0x51,0x02,0x00,0x00,0x46,0x02,0x00,0x00, -0x50,0x00,0x05,0x00,0x64,0x00,0x00,0x00,0x53,0x02,0x00,0x00, -0x48,0x02,0x00,0x00,0x48,0x02,0x00,0x00,0x81,0x00,0x05,0x00, -0x64,0x00,0x00,0x00,0x54,0x02,0x00,0x00,0x52,0x02,0x00,0x00, -0x53,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x5a,0x02,0x00,0x00,0x83,0x00,0x00,0x00,0x1e,0x03,0x00,0x00, -0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00,0x5c,0x02,0x00,0x00, -0x54,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x41,0x00,0x06,0x00, -0x56,0x00,0x00,0x00,0x5d,0x02,0x00,0x00,0x7b,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x5a,0x02,0x00,0x00,0x3e,0x00,0x03,0x00, -0x5d,0x02,0x00,0x00,0x5c,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x64,0x02,0x00,0x00,0x83,0x00,0x00,0x00, -0x1f,0x03,0x00,0x00,0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00, -0x65,0x02,0x00,0x00,0x54,0x02,0x00,0x00,0x01,0x00,0x00,0x00, -0x41,0x00,0x06,0x00,0x56,0x00,0x00,0x00,0x66,0x02,0x00,0x00, -0x7b,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x64,0x02,0x00,0x00, -0x3e,0x00,0x03,0x00,0x66,0x02,0x00,0x00,0x65,0x02,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0x6f,0x02,0x00,0x00, -0x57,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0x71,0x02,0x00,0x00,0x5b,0x00,0x00,0x00,0x41,0x00,0x08,0x00, -0x61,0x00,0x00,0x00,0x72,0x02,0x00,0x00,0x54,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x37,0x00,0x00,0x00, -0x20,0x03,0x00,0x00,0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0x73,0x02,0x00,0x00,0x72,0x02,0x00,0x00,0x71,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x74,0x02,0x00,0x00,0x73,0x02,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x75,0x02,0x00,0x00, -0x74,0x02,0x00,0x00,0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x76,0x02,0x00,0x00,0x75,0x02,0x00,0x00,0x6a,0x00,0x00,0x00, -0x6f,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0x77,0x02,0x00,0x00, -0x76,0x02,0x00,0x00,0xc2,0x00,0x05,0x00,0x4d,0x00,0x00,0x00, -0x78,0x02,0x00,0x00,0x73,0x02,0x00,0x00,0x6e,0x00,0x00,0x00, -0x70,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0x79,0x02,0x00,0x00, -0x78,0x02,0x00,0x00,0x50,0x00,0x05,0x00,0x64,0x00,0x00,0x00, -0x7a,0x02,0x00,0x00,0x77,0x02,0x00,0x00,0x79,0x02,0x00,0x00, -0x8e,0x00,0x05,0x00,0x64,0x00,0x00,0x00,0x7b,0x02,0x00,0x00, -0x7a,0x02,0x00,0x00,0x6f,0x02,0x00,0x00,0x50,0x00,0x05,0x00, -0x64,0x00,0x00,0x00,0x7c,0x02,0x00,0x00,0x71,0x02,0x00,0x00, -0x71,0x02,0x00,0x00,0x81,0x00,0x05,0x00,0x64,0x00,0x00,0x00, -0x7d,0x02,0x00,0x00,0x7b,0x02,0x00,0x00,0x7c,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x83,0x02,0x00,0x00, -0x83,0x00,0x00,0x00,0x20,0x03,0x00,0x00,0x51,0x00,0x05,0x00, -0x4a,0x00,0x00,0x00,0x85,0x02,0x00,0x00,0x7d,0x02,0x00,0x00, -0x00,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x56,0x00,0x00,0x00, -0x86,0x02,0x00,0x00,0x7b,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x83,0x02,0x00,0x00,0x3e,0x00,0x03,0x00,0x86,0x02,0x00,0x00, -0x85,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x8d,0x02,0x00,0x00,0x83,0x00,0x00,0x00,0x21,0x03,0x00,0x00, -0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00,0x8e,0x02,0x00,0x00, -0x7d,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0x41,0x00,0x06,0x00, -0x56,0x00,0x00,0x00,0x8f,0x02,0x00,0x00,0x7b,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x8d,0x02,0x00,0x00,0x3e,0x00,0x03,0x00, -0x8f,0x02,0x00,0x00,0x8e,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4a,0x00,0x00,0x00,0x98,0x02,0x00,0x00,0x57,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0x9a,0x02,0x00,0x00, -0x5b,0x00,0x00,0x00,0x41,0x00,0x08,0x00,0x61,0x00,0x00,0x00, -0x9b,0x02,0x00,0x00,0x54,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x22,0x03,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00,0x9c,0x02,0x00,0x00, -0x9b,0x02,0x00,0x00,0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x9d,0x02,0x00,0x00,0x9c,0x02,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x9e,0x02,0x00,0x00,0x9d,0x02,0x00,0x00, -0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9f,0x02,0x00,0x00, -0x9e,0x02,0x00,0x00,0x6a,0x00,0x00,0x00,0x6f,0x00,0x04,0x00, -0x4a,0x00,0x00,0x00,0xa0,0x02,0x00,0x00,0x9f,0x02,0x00,0x00, -0xc2,0x00,0x05,0x00,0x4d,0x00,0x00,0x00,0xa1,0x02,0x00,0x00, -0x9c,0x02,0x00,0x00,0x6e,0x00,0x00,0x00,0x70,0x00,0x04,0x00, -0x4a,0x00,0x00,0x00,0xa2,0x02,0x00,0x00,0xa1,0x02,0x00,0x00, -0x50,0x00,0x05,0x00,0x64,0x00,0x00,0x00,0xa3,0x02,0x00,0x00, -0xa0,0x02,0x00,0x00,0xa2,0x02,0x00,0x00,0x8e,0x00,0x05,0x00, -0x64,0x00,0x00,0x00,0xa4,0x02,0x00,0x00,0xa3,0x02,0x00,0x00, -0x98,0x02,0x00,0x00,0x50,0x00,0x05,0x00,0x64,0x00,0x00,0x00, -0xa5,0x02,0x00,0x00,0x9a,0x02,0x00,0x00,0x9a,0x02,0x00,0x00, -0x81,0x00,0x05,0x00,0x64,0x00,0x00,0x00,0xa6,0x02,0x00,0x00, -0xa4,0x02,0x00,0x00,0xa5,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xac,0x02,0x00,0x00,0x83,0x00,0x00,0x00, -0x22,0x03,0x00,0x00,0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00, -0xae,0x02,0x00,0x00,0xa6,0x02,0x00,0x00,0x00,0x00,0x00,0x00, -0x41,0x00,0x06,0x00,0x56,0x00,0x00,0x00,0xaf,0x02,0x00,0x00, -0x7b,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0xac,0x02,0x00,0x00, -0x3e,0x00,0x03,0x00,0xaf,0x02,0x00,0x00,0xae,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xb6,0x02,0x00,0x00, -0x83,0x00,0x00,0x00,0x23,0x03,0x00,0x00,0x51,0x00,0x05,0x00, -0x4a,0x00,0x00,0x00,0xb7,0x02,0x00,0x00,0xa6,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x56,0x00,0x00,0x00, -0xb8,0x02,0x00,0x00,0x7b,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0xb6,0x02,0x00,0x00,0x3e,0x00,0x03,0x00,0xb8,0x02,0x00,0x00, -0xb7,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0xc1,0x02,0x00,0x00,0x57,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4a,0x00,0x00,0x00,0xc3,0x02,0x00,0x00,0x5b,0x00,0x00,0x00, -0x41,0x00,0x08,0x00,0x61,0x00,0x00,0x00,0xc4,0x02,0x00,0x00, -0x54,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x37,0x00,0x00,0x00,0x24,0x03,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0xc5,0x02,0x00,0x00,0xc4,0x02,0x00,0x00, -0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0xc6,0x02,0x00,0x00, -0xc5,0x02,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xc7,0x02,0x00,0x00,0xc6,0x02,0x00,0x00,0xc7,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xc8,0x02,0x00,0x00,0xc7,0x02,0x00,0x00, -0x6a,0x00,0x00,0x00,0x6f,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0xc9,0x02,0x00,0x00,0xc8,0x02,0x00,0x00,0xc2,0x00,0x05,0x00, -0x4d,0x00,0x00,0x00,0xca,0x02,0x00,0x00,0xc5,0x02,0x00,0x00, -0x6e,0x00,0x00,0x00,0x70,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0xcb,0x02,0x00,0x00,0xca,0x02,0x00,0x00,0x50,0x00,0x05,0x00, -0x64,0x00,0x00,0x00,0xcc,0x02,0x00,0x00,0xc9,0x02,0x00,0x00, -0xcb,0x02,0x00,0x00,0x8e,0x00,0x05,0x00,0x64,0x00,0x00,0x00, -0xcd,0x02,0x00,0x00,0xcc,0x02,0x00,0x00,0xc1,0x02,0x00,0x00, -0x50,0x00,0x05,0x00,0x64,0x00,0x00,0x00,0xce,0x02,0x00,0x00, -0xc3,0x02,0x00,0x00,0xc3,0x02,0x00,0x00,0x81,0x00,0x05,0x00, -0x64,0x00,0x00,0x00,0xcf,0x02,0x00,0x00,0xcd,0x02,0x00,0x00, -0xce,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xd5,0x02,0x00,0x00,0x83,0x00,0x00,0x00,0x24,0x03,0x00,0x00, -0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00,0xd7,0x02,0x00,0x00, -0xcf,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x41,0x00,0x06,0x00, -0x56,0x00,0x00,0x00,0xd8,0x02,0x00,0x00,0x7b,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0xd5,0x02,0x00,0x00,0x3e,0x00,0x03,0x00, -0xd8,0x02,0x00,0x00,0xd7,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xdf,0x02,0x00,0x00,0x83,0x00,0x00,0x00, -0x25,0x03,0x00,0x00,0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00, -0xe0,0x02,0x00,0x00,0xcf,0x02,0x00,0x00,0x01,0x00,0x00,0x00, -0x41,0x00,0x06,0x00,0x56,0x00,0x00,0x00,0xe1,0x02,0x00,0x00, -0x7b,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0xdf,0x02,0x00,0x00, -0x3e,0x00,0x03,0x00,0xe1,0x02,0x00,0x00,0xe0,0x02,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0xea,0x02,0x00,0x00, -0x57,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0xec,0x02,0x00,0x00,0x5b,0x00,0x00,0x00,0x41,0x00,0x08,0x00, -0x61,0x00,0x00,0x00,0xed,0x02,0x00,0x00,0x54,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x37,0x00,0x00,0x00, -0x6a,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0xee,0x02,0x00,0x00,0xed,0x02,0x00,0x00,0x71,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0xef,0x02,0x00,0x00,0xee,0x02,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xf0,0x02,0x00,0x00, -0xef,0x02,0x00,0x00,0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xf1,0x02,0x00,0x00,0xf0,0x02,0x00,0x00,0x6a,0x00,0x00,0x00, -0x6f,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0xf2,0x02,0x00,0x00, -0xf1,0x02,0x00,0x00,0xc2,0x00,0x05,0x00,0x4d,0x00,0x00,0x00, -0xf3,0x02,0x00,0x00,0xee,0x02,0x00,0x00,0x6e,0x00,0x00,0x00, -0x70,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0xf4,0x02,0x00,0x00, -0xf3,0x02,0x00,0x00,0x50,0x00,0x05,0x00,0x64,0x00,0x00,0x00, -0xf5,0x02,0x00,0x00,0xf2,0x02,0x00,0x00,0xf4,0x02,0x00,0x00, -0x8e,0x00,0x05,0x00,0x64,0x00,0x00,0x00,0xf6,0x02,0x00,0x00, -0xf5,0x02,0x00,0x00,0xea,0x02,0x00,0x00,0x50,0x00,0x05,0x00, -0x64,0x00,0x00,0x00,0xf7,0x02,0x00,0x00,0xec,0x02,0x00,0x00, -0xec,0x02,0x00,0x00,0x81,0x00,0x05,0x00,0x64,0x00,0x00,0x00, -0xf8,0x02,0x00,0x00,0xf6,0x02,0x00,0x00,0xf7,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xfe,0x02,0x00,0x00, -0x83,0x00,0x00,0x00,0x6a,0x00,0x00,0x00,0x51,0x00,0x05,0x00, -0x4a,0x00,0x00,0x00,0x00,0x03,0x00,0x00,0xf8,0x02,0x00,0x00, -0x00,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x56,0x00,0x00,0x00, -0x01,0x03,0x00,0x00,0x7b,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0xfe,0x02,0x00,0x00,0x3e,0x00,0x03,0x00,0x01,0x03,0x00,0x00, -0x00,0x03,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x08,0x03,0x00,0x00,0x83,0x00,0x00,0x00,0x26,0x03,0x00,0x00, -0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00,0x09,0x03,0x00,0x00, -0xf8,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0x41,0x00,0x06,0x00, -0x56,0x00,0x00,0x00,0x0a,0x03,0x00,0x00,0x7b,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x08,0x03,0x00,0x00,0x3e,0x00,0x03,0x00, -0x0a,0x03,0x00,0x00,0x09,0x03,0x00,0x00,0xf9,0x00,0x02,0x00, -0x9c,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x9c,0x00,0x00,0x00, -0xfd,0x00,0x01,0x00,0x38,0x00,0x01,0x00, -}; -const uint64_t dequant_q4_1_len = 8924; - -unsigned char dequant_q4_1_fp32_data[] = { -0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, 0x59,0x03,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, 0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x51,0x11,0x00,0x00, 0x11,0x00,0x02,0x00,0x60,0x11,0x00,0x00,0x0b,0x00,0x06,0x00, @@ -5683,496 +3348,10 @@ unsigned char dequant_q4_1_fp32_data[] = { 0xa0,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xa0,0x00,0x00,0x00, 0xfd,0x00,0x01,0x00,0x38,0x00,0x01,0x00, }; -const uint64_t dequant_q4_1_fp32_len = 9704; +const uint64_t dequant_q4_1_len = 9704; unsigned char dequant_q4_K_data[] = { 0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, -0xa6,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, -0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x09,0x00,0x00,0x00, -0x11,0x00,0x02,0x00,0x27,0x00,0x00,0x00,0x11,0x00,0x02,0x00, -0x51,0x11,0x00,0x00,0x11,0x00,0x02,0x00,0x60,0x11,0x00,0x00, -0x0b,0x00,0x06,0x00,0x01,0x00,0x00,0x00,0x47,0x4c,0x53,0x4c, -0x2e,0x73,0x74,0x64,0x2e,0x34,0x35,0x30,0x00,0x00,0x00,0x00, -0x0e,0x00,0x03,0x00,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x0f,0x00,0x0a,0x00,0x05,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x6d,0x61,0x69,0x6e,0x00,0x00,0x00,0x00,0x17,0x00,0x00,0x00, -0x25,0x00,0x00,0x00,0x33,0x00,0x00,0x00,0x4f,0x00,0x00,0x00, -0x08,0x01,0x00,0x00,0x10,0x00,0x06,0x00,0x04,0x00,0x00,0x00, -0x11,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x17,0x00,0x00,0x00, -0x0b,0x00,0x00,0x00,0x1a,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x23,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x23,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x23,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x23,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x33,0x00,0x00,0x00, -0x0b,0x00,0x00,0x00,0x1b,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x48,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x4b,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x4b,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x4b,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x10,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x4c,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x90,0x00,0x00,0x00,0x48,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x4d,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, -0x4d,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x4f,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x4f,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x05,0x01,0x00,0x00, -0x06,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x48,0x00,0x04,0x00, -0x06,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x19,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x06,0x01,0x00,0x00,0x00,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, -0x06,0x01,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x08,0x01,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x08,0x01,0x00,0x00,0x21,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x31,0x01,0x00,0x00, -0x0b,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x13,0x00,0x02,0x00, -0x02,0x00,0x00,0x00,0x21,0x00,0x03,0x00,0x03,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x15,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x10,0x00,0x00,0x00, -0x00,0x01,0x00,0x00,0x14,0x00,0x02,0x00,0x11,0x00,0x00,0x00, -0x15,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x17,0x00,0x04,0x00,0x15,0x00,0x00,0x00, -0x14,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x16,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x15,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x16,0x00,0x00,0x00,0x17,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x14,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x19,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x14,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0x1c,0x00,0x00,0x00, -0x00,0x01,0x00,0x00,0x1e,0x00,0x06,0x00,0x23,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x24,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x24,0x00,0x00,0x00,0x25,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x26,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x29,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x16,0x00,0x00,0x00,0x33,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x08,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x3f,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x16,0x00,0x03,0x00, -0x42,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x17,0x00,0x04,0x00, -0x45,0x00,0x00,0x00,0x42,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x15,0x00,0x04,0x00,0x46,0x00,0x00,0x00,0x08,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x14,0x00,0x00,0x00, -0x47,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x1c,0x00,0x04,0x00, -0x48,0x00,0x00,0x00,0x46,0x00,0x00,0x00,0x47,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0x49,0x00,0x00,0x00, -0x80,0x00,0x00,0x00,0x1c,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0x46,0x00,0x00,0x00,0x49,0x00,0x00,0x00,0x1e,0x00,0x05,0x00, -0x4b,0x00,0x00,0x00,0x45,0x00,0x00,0x00,0x48,0x00,0x00,0x00, -0x4a,0x00,0x00,0x00,0x1d,0x00,0x03,0x00,0x4c,0x00,0x00,0x00, -0x4b,0x00,0x00,0x00,0x1e,0x00,0x03,0x00,0x4d,0x00,0x00,0x00, -0x4c,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x4e,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x4d,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x4e,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x51,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x42,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x14,0x00,0x00,0x00, -0x56,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x5c,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x60,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x65,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x73,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x46,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x78,0x00,0x00,0x00, -0x3f,0x00,0x00,0x00,0x15,0x00,0x04,0x00,0x7a,0x00,0x00,0x00, -0x08,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x90,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x97,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xc6,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xda,0x00,0x00,0x00,0x03,0x00,0x00,0x00, -0x1d,0x00,0x03,0x00,0x05,0x01,0x00,0x00,0x42,0x00,0x00,0x00, -0x1e,0x00,0x03,0x00,0x06,0x01,0x00,0x00,0x05,0x01,0x00,0x00, -0x20,0x00,0x04,0x00,0x07,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, -0x06,0x01,0x00,0x00,0x3b,0x00,0x04,0x00,0x07,0x01,0x00,0x00, -0x08,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x14,0x00,0x00,0x00,0x30,0x01,0x00,0x00,0x20,0x00,0x00,0x00, -0x2c,0x00,0x06,0x00,0x15,0x00,0x00,0x00,0x31,0x01,0x00,0x00, -0x30,0x01,0x00,0x00,0x56,0x00,0x00,0x00,0x56,0x00,0x00,0x00, -0x2a,0x00,0x03,0x00,0x11,0x00,0x00,0x00,0x34,0x01,0x00,0x00, -0x29,0x00,0x03,0x00,0x11,0x00,0x00,0x00,0x37,0x01,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x9f,0x01,0x00,0x00, -0x21,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xa2,0x01,0x00,0x00,0x22,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xa5,0x01,0x00,0x00,0x23,0x00,0x00,0x00, -0x36,0x00,0x05,0x00,0x02,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x05,0x00,0x00,0x00,0xf7,0x00,0x03,0x00,0x32,0x01,0x00,0x00, -0x00,0x00,0x00,0x00,0xfb,0x00,0x03,0x00,0x18,0x00,0x00,0x00, -0x33,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x33,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0x0a,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x0a,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0x3a,0x01,0x00,0x00,0x09,0x00,0x00,0x00,0x33,0x01,0x00,0x00, -0x2f,0x01,0x00,0x00,0x0d,0x00,0x00,0x00,0xb1,0x00,0x05,0x00, -0x11,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x3a,0x01,0x00,0x00, -0x10,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x0c,0x00,0x00,0x00, -0x0d,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x12,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0x0b,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x19,0x00,0x00,0x00,0x1a,0x00,0x00,0x00,0x17,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x14,0x00,0x00,0x00, -0x1b,0x00,0x00,0x00,0x1a,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x14,0x00,0x00,0x00,0x1d,0x00,0x00,0x00,0x1b,0x00,0x00,0x00, -0x1c,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x14,0x00,0x00,0x00, -0x1f,0x00,0x00,0x00,0x3a,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x14,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x1d,0x00,0x00,0x00, -0x1f,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x21,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x26,0x00,0x00,0x00,0x27,0x00,0x00,0x00,0x25,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x28,0x00,0x00,0x00,0x27,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x26,0x00,0x00,0x00,0x2a,0x00,0x00,0x00,0x25,0x00,0x00,0x00, -0x29,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x2b,0x00,0x00,0x00,0x2a,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x2c,0x00,0x00,0x00,0x28,0x00,0x00,0x00, -0x2b,0x00,0x00,0x00,0x87,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x2d,0x00,0x00,0x00,0x2c,0x00,0x00,0x00,0x10,0x00,0x00,0x00, -0xaf,0x00,0x05,0x00,0x11,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x21,0x00,0x00,0x00,0x2d,0x00,0x00,0x00,0xf7,0x00,0x03,0x00, -0x30,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x2e,0x00,0x00,0x00,0x2f,0x00,0x00,0x00,0x30,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0x2f,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x0c,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x30,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x19,0x00,0x00,0x00,0x34,0x00,0x00,0x00, -0x33,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x14,0x00,0x00,0x00,0x35,0x00,0x00,0x00,0x34,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x36,0x00,0x00,0x00, -0x35,0x00,0x00,0x00,0x87,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x3a,0x00,0x00,0x00,0x36,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x8b,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3d,0x00,0x00,0x00, -0x36,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x41,0x00,0x00,0x00,0x3f,0x00,0x00,0x00, -0x3a,0x00,0x00,0x00,0x41,0x00,0x08,0x00,0x51,0x00,0x00,0x00, -0x52,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x21,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x18,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x42,0x00,0x00,0x00,0x53,0x00,0x00,0x00, -0x52,0x00,0x00,0x00,0x41,0x00,0x08,0x00,0x51,0x00,0x00,0x00, -0x57,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x21,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x56,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x42,0x00,0x00,0x00,0x58,0x00,0x00,0x00, -0x57,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x5b,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x10,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x5e,0x00,0x00,0x00, -0x5c,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x5f,0x00,0x00,0x00,0x5b,0x00,0x00,0x00, -0x5e,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x3d,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x63,0x00,0x00,0x00, -0x5f,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x67,0x00,0x00,0x00,0x65,0x00,0x00,0x00, -0x3a,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x6a,0x00,0x00,0x00,0x67,0x00,0x00,0x00,0x62,0x00,0x00,0x00, -0xb1,0x00,0x05,0x00,0x11,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x41,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0xf7,0x00,0x03,0x00, -0x6e,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x6c,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x88,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0x6d,0x00,0x00,0x00,0x41,0x00,0x08,0x00, -0x73,0x00,0x00,0x00,0x74,0x00,0x00,0x00,0x4f,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x29,0x00,0x00,0x00, -0x41,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x46,0x00,0x00,0x00, -0x75,0x00,0x00,0x00,0x74,0x00,0x00,0x00,0x71,0x00,0x04,0x00, -0x14,0x00,0x00,0x00,0x76,0x00,0x00,0x00,0x75,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x77,0x00,0x00,0x00, -0x76,0x00,0x00,0x00,0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x79,0x00,0x00,0x00,0x77,0x00,0x00,0x00,0x78,0x00,0x00,0x00, -0x72,0x00,0x04,0x00,0x7a,0x00,0x00,0x00,0x7b,0x00,0x00,0x00, -0x79,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x46,0x00,0x00,0x00, -0x7c,0x00,0x00,0x00,0x7b,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x80,0x00,0x00,0x00,0x41,0x00,0x00,0x00, -0x60,0x00,0x00,0x00,0x41,0x00,0x08,0x00,0x73,0x00,0x00,0x00, -0x81,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x21,0x00,0x00,0x00,0x29,0x00,0x00,0x00,0x80,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x46,0x00,0x00,0x00,0x82,0x00,0x00,0x00, -0x81,0x00,0x00,0x00,0x71,0x00,0x04,0x00,0x14,0x00,0x00,0x00, -0x83,0x00,0x00,0x00,0x82,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x83,0x00,0x00,0x00, -0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x85,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0x78,0x00,0x00,0x00,0x72,0x00,0x04,0x00, -0x7a,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x85,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x46,0x00,0x00,0x00,0x87,0x00,0x00,0x00, -0x86,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x6e,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0x88,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x8b,0x00,0x00,0x00,0x41,0x00,0x00,0x00, -0x60,0x00,0x00,0x00,0x41,0x00,0x08,0x00,0x73,0x00,0x00,0x00, -0x8c,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x21,0x00,0x00,0x00,0x29,0x00,0x00,0x00,0x8b,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x46,0x00,0x00,0x00,0x8d,0x00,0x00,0x00, -0x8c,0x00,0x00,0x00,0x71,0x00,0x04,0x00,0x14,0x00,0x00,0x00, -0x8e,0x00,0x00,0x00,0x8d,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x8f,0x00,0x00,0x00,0x8e,0x00,0x00,0x00, -0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x91,0x00,0x00,0x00, -0x8f,0x00,0x00,0x00,0x90,0x00,0x00,0x00,0x82,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x94,0x00,0x00,0x00,0x41,0x00,0x00,0x00, -0x60,0x00,0x00,0x00,0x41,0x00,0x08,0x00,0x73,0x00,0x00,0x00, -0x95,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x21,0x00,0x00,0x00,0x29,0x00,0x00,0x00,0x94,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x46,0x00,0x00,0x00,0x96,0x00,0x00,0x00, -0x95,0x00,0x00,0x00,0xc2,0x00,0x05,0x00,0x46,0x00,0x00,0x00, -0x98,0x00,0x00,0x00,0x96,0x00,0x00,0x00,0x97,0x00,0x00,0x00, -0xc4,0x00,0x05,0x00,0x46,0x00,0x00,0x00,0x99,0x00,0x00,0x00, -0x98,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x71,0x00,0x04,0x00, -0x14,0x00,0x00,0x00,0x9a,0x00,0x00,0x00,0x99,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x9b,0x00,0x00,0x00, -0x9a,0x00,0x00,0x00,0xc5,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x9c,0x00,0x00,0x00,0x91,0x00,0x00,0x00,0x9b,0x00,0x00,0x00, -0x72,0x00,0x04,0x00,0x7a,0x00,0x00,0x00,0x9d,0x00,0x00,0x00, -0x9c,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x46,0x00,0x00,0x00, -0x9e,0x00,0x00,0x00,0x9d,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x46,0x00,0x00,0x00,0xa3,0x00,0x00,0x00,0x8c,0x00,0x00,0x00, -0xc2,0x00,0x05,0x00,0x46,0x00,0x00,0x00,0xa4,0x00,0x00,0x00, -0xa3,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x41,0x00,0x08,0x00, -0x73,0x00,0x00,0x00,0xa7,0x00,0x00,0x00,0x4f,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x29,0x00,0x00,0x00, -0x41,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x46,0x00,0x00,0x00, -0xa8,0x00,0x00,0x00,0xa7,0x00,0x00,0x00,0xc2,0x00,0x05,0x00, -0x46,0x00,0x00,0x00,0xa9,0x00,0x00,0x00,0xa8,0x00,0x00,0x00, -0x97,0x00,0x00,0x00,0xc4,0x00,0x05,0x00,0x46,0x00,0x00,0x00, -0xaa,0x00,0x00,0x00,0xa9,0x00,0x00,0x00,0x60,0x00,0x00,0x00, -0xc5,0x00,0x05,0x00,0x46,0x00,0x00,0x00,0xab,0x00,0x00,0x00, -0xa4,0x00,0x00,0x00,0xaa,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x6e,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x6e,0x00,0x00,0x00, -0xf5,0x00,0x07,0x00,0x46,0x00,0x00,0x00,0x3c,0x01,0x00,0x00, -0x87,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0xab,0x00,0x00,0x00, -0x88,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x46,0x00,0x00,0x00, -0x3b,0x01,0x00,0x00,0x7c,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, -0x9e,0x00,0x00,0x00,0x88,0x00,0x00,0x00,0x70,0x00,0x04,0x00, -0x42,0x00,0x00,0x00,0xaf,0x00,0x00,0x00,0x3b,0x01,0x00,0x00, -0x85,0x00,0x05,0x00,0x42,0x00,0x00,0x00,0xb0,0x00,0x00,0x00, -0x53,0x00,0x00,0x00,0xaf,0x00,0x00,0x00,0x70,0x00,0x04,0x00, -0x42,0x00,0x00,0x00,0xb4,0x00,0x00,0x00,0x3c,0x01,0x00,0x00, -0x85,0x00,0x05,0x00,0x42,0x00,0x00,0x00,0xb5,0x00,0x00,0x00, -0x58,0x00,0x00,0x00,0xb4,0x00,0x00,0x00,0xf7,0x00,0x03,0x00, -0xb9,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x6c,0x00,0x00,0x00,0xb8,0x00,0x00,0x00,0xcf,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xb8,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xbc,0x00,0x00,0x00,0x41,0x00,0x00,0x00, -0x29,0x00,0x00,0x00,0x41,0x00,0x08,0x00,0x73,0x00,0x00,0x00, -0xbd,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x21,0x00,0x00,0x00,0x29,0x00,0x00,0x00,0xbc,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x46,0x00,0x00,0x00,0xbe,0x00,0x00,0x00, -0xbd,0x00,0x00,0x00,0x71,0x00,0x04,0x00,0x14,0x00,0x00,0x00, -0xbf,0x00,0x00,0x00,0xbe,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xc0,0x00,0x00,0x00,0xbf,0x00,0x00,0x00, -0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xc1,0x00,0x00,0x00, -0xc0,0x00,0x00,0x00,0x78,0x00,0x00,0x00,0x72,0x00,0x04,0x00, -0x7a,0x00,0x00,0x00,0xc2,0x00,0x00,0x00,0xc1,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x46,0x00,0x00,0x00,0xc3,0x00,0x00,0x00, -0xc2,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xc7,0x00,0x00,0x00,0x41,0x00,0x00,0x00,0xc6,0x00,0x00,0x00, -0x41,0x00,0x08,0x00,0x73,0x00,0x00,0x00,0xc8,0x00,0x00,0x00, -0x4f,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x29,0x00,0x00,0x00,0xc7,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x46,0x00,0x00,0x00,0xc9,0x00,0x00,0x00,0xc8,0x00,0x00,0x00, -0x71,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0xca,0x00,0x00,0x00, -0xc9,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xcb,0x00,0x00,0x00,0xca,0x00,0x00,0x00,0xc7,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xcc,0x00,0x00,0x00,0xcb,0x00,0x00,0x00, -0x78,0x00,0x00,0x00,0x72,0x00,0x04,0x00,0x7a,0x00,0x00,0x00, -0xcd,0x00,0x00,0x00,0xcc,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x46,0x00,0x00,0x00,0xce,0x00,0x00,0x00,0xcd,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xb9,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xcf,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xd2,0x00,0x00,0x00,0x41,0x00,0x00,0x00,0xc6,0x00,0x00,0x00, -0x41,0x00,0x08,0x00,0x73,0x00,0x00,0x00,0xd3,0x00,0x00,0x00, -0x4f,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x29,0x00,0x00,0x00,0xd2,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x46,0x00,0x00,0x00,0xd4,0x00,0x00,0x00,0xd3,0x00,0x00,0x00, -0x71,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0xd5,0x00,0x00,0x00, -0xd4,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xd6,0x00,0x00,0x00,0xd5,0x00,0x00,0x00,0xc7,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xd7,0x00,0x00,0x00,0xd6,0x00,0x00,0x00, -0x90,0x00,0x00,0x00,0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xdb,0x00,0x00,0x00,0x41,0x00,0x00,0x00,0xda,0x00,0x00,0x00, -0x41,0x00,0x08,0x00,0x73,0x00,0x00,0x00,0xdc,0x00,0x00,0x00, -0x4f,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x29,0x00,0x00,0x00,0xdb,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x46,0x00,0x00,0x00,0xdd,0x00,0x00,0x00,0xdc,0x00,0x00,0x00, -0xc2,0x00,0x05,0x00,0x46,0x00,0x00,0x00,0xde,0x00,0x00,0x00, -0xdd,0x00,0x00,0x00,0x97,0x00,0x00,0x00,0xc4,0x00,0x05,0x00, -0x46,0x00,0x00,0x00,0xdf,0x00,0x00,0x00,0xde,0x00,0x00,0x00, -0x60,0x00,0x00,0x00,0x71,0x00,0x04,0x00,0x14,0x00,0x00,0x00, -0xe0,0x00,0x00,0x00,0xdf,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xe1,0x00,0x00,0x00,0xe0,0x00,0x00,0x00, -0xc5,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xe2,0x00,0x00,0x00, -0xd7,0x00,0x00,0x00,0xe1,0x00,0x00,0x00,0x72,0x00,0x04,0x00, -0x7a,0x00,0x00,0x00,0xe3,0x00,0x00,0x00,0xe2,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x46,0x00,0x00,0x00,0xe4,0x00,0x00,0x00, -0xe3,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x46,0x00,0x00,0x00, -0xe9,0x00,0x00,0x00,0xd3,0x00,0x00,0x00,0xc2,0x00,0x05,0x00, -0x46,0x00,0x00,0x00,0xea,0x00,0x00,0x00,0xe9,0x00,0x00,0x00, -0x60,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xed,0x00,0x00,0x00,0x41,0x00,0x00,0x00,0x29,0x00,0x00,0x00, -0x41,0x00,0x08,0x00,0x73,0x00,0x00,0x00,0xee,0x00,0x00,0x00, -0x4f,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x29,0x00,0x00,0x00,0xed,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x46,0x00,0x00,0x00,0xef,0x00,0x00,0x00,0xee,0x00,0x00,0x00, -0xc2,0x00,0x05,0x00,0x46,0x00,0x00,0x00,0xf0,0x00,0x00,0x00, -0xef,0x00,0x00,0x00,0x97,0x00,0x00,0x00,0xc4,0x00,0x05,0x00, -0x46,0x00,0x00,0x00,0xf1,0x00,0x00,0x00,0xf0,0x00,0x00,0x00, -0x60,0x00,0x00,0x00,0xc5,0x00,0x05,0x00,0x46,0x00,0x00,0x00, -0xf2,0x00,0x00,0x00,0xea,0x00,0x00,0x00,0xf1,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xb9,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xb9,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x46,0x00,0x00,0x00, -0x3e,0x01,0x00,0x00,0xce,0x00,0x00,0x00,0xb8,0x00,0x00,0x00, -0xf2,0x00,0x00,0x00,0xcf,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, -0x46,0x00,0x00,0x00,0x3d,0x01,0x00,0x00,0xc3,0x00,0x00,0x00, -0xb8,0x00,0x00,0x00,0xe4,0x00,0x00,0x00,0xcf,0x00,0x00,0x00, -0x70,0x00,0x04,0x00,0x42,0x00,0x00,0x00,0xf6,0x00,0x00,0x00, -0x3d,0x01,0x00,0x00,0x85,0x00,0x05,0x00,0x42,0x00,0x00,0x00, -0xf7,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0xf6,0x00,0x00,0x00, -0x70,0x00,0x04,0x00,0x42,0x00,0x00,0x00,0xfb,0x00,0x00,0x00, -0x3e,0x01,0x00,0x00,0x85,0x00,0x05,0x00,0x42,0x00,0x00,0x00, -0xfc,0x00,0x00,0x00,0x58,0x00,0x00,0x00,0xfb,0x00,0x00,0x00, -0x41,0x00,0x08,0x00,0x73,0x00,0x00,0x00,0x11,0x01,0x00,0x00, -0x4f,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x3f,0x00,0x00,0x00,0x6a,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x46,0x00,0x00,0x00,0x12,0x01,0x00,0x00,0x11,0x01,0x00,0x00, -0x71,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0x13,0x01,0x00,0x00, -0x12,0x01,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x14,0x01,0x00,0x00,0x13,0x01,0x00,0x00,0xc7,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x15,0x01,0x00,0x00,0x14,0x01,0x00,0x00, -0x90,0x00,0x00,0x00,0x6f,0x00,0x04,0x00,0x42,0x00,0x00,0x00, -0x16,0x01,0x00,0x00,0x15,0x01,0x00,0x00,0x7f,0x00,0x04,0x00, -0x42,0x00,0x00,0x00,0x9c,0x01,0x00,0x00,0xb5,0x00,0x00,0x00, -0x0c,0x00,0x08,0x00,0x42,0x00,0x00,0x00,0x19,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0xb0,0x00,0x00,0x00, -0x16,0x01,0x00,0x00,0x9c,0x01,0x00,0x00,0x41,0x00,0x06,0x00, -0x51,0x00,0x00,0x00,0x1a,0x01,0x00,0x00,0x08,0x01,0x00,0x00, -0x09,0x00,0x00,0x00,0x63,0x00,0x00,0x00,0x3e,0x00,0x03,0x00, -0x1a,0x01,0x00,0x00,0x19,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x1e,0x01,0x00,0x00,0x63,0x00,0x00,0x00, -0x65,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x46,0x00,0x00,0x00, -0x25,0x01,0x00,0x00,0x11,0x01,0x00,0x00,0xc2,0x00,0x05,0x00, -0x46,0x00,0x00,0x00,0x26,0x01,0x00,0x00,0x25,0x01,0x00,0x00, -0x60,0x00,0x00,0x00,0x70,0x00,0x04,0x00,0x42,0x00,0x00,0x00, -0x27,0x01,0x00,0x00,0x26,0x01,0x00,0x00,0x7f,0x00,0x04,0x00, -0x42,0x00,0x00,0x00,0x9d,0x01,0x00,0x00,0xfc,0x00,0x00,0x00, -0x0c,0x00,0x08,0x00,0x42,0x00,0x00,0x00,0x2a,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0xf7,0x00,0x00,0x00, -0x27,0x01,0x00,0x00,0x9d,0x01,0x00,0x00,0x41,0x00,0x06,0x00, -0x51,0x00,0x00,0x00,0x2b,0x01,0x00,0x00,0x08,0x01,0x00,0x00, -0x09,0x00,0x00,0x00,0x1e,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0x2b,0x01,0x00,0x00,0x2a,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x4d,0x01,0x00,0x00,0x63,0x00,0x00,0x00, -0x29,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x4e,0x01,0x00,0x00,0x6a,0x00,0x00,0x00,0x29,0x00,0x00,0x00, -0x41,0x00,0x08,0x00,0x73,0x00,0x00,0x00,0x4f,0x01,0x00,0x00, -0x4f,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x3f,0x00,0x00,0x00,0x4e,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, -0x46,0x00,0x00,0x00,0x50,0x01,0x00,0x00,0x4f,0x01,0x00,0x00, -0x71,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0x51,0x01,0x00,0x00, -0x50,0x01,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x52,0x01,0x00,0x00,0x51,0x01,0x00,0x00,0xc7,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x53,0x01,0x00,0x00,0x52,0x01,0x00,0x00, -0x90,0x00,0x00,0x00,0x6f,0x00,0x04,0x00,0x42,0x00,0x00,0x00, -0x54,0x01,0x00,0x00,0x53,0x01,0x00,0x00,0x0c,0x00,0x08,0x00, -0x42,0x00,0x00,0x00,0x56,0x01,0x00,0x00,0x01,0x00,0x00,0x00, -0x32,0x00,0x00,0x00,0xb0,0x00,0x00,0x00,0x54,0x01,0x00,0x00, -0x9c,0x01,0x00,0x00,0x41,0x00,0x06,0x00,0x51,0x00,0x00,0x00, -0x57,0x01,0x00,0x00,0x08,0x01,0x00,0x00,0x09,0x00,0x00,0x00, -0x4d,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x57,0x01,0x00,0x00, -0x56,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x59,0x01,0x00,0x00,0x63,0x00,0x00,0x00,0x9f,0x01,0x00,0x00, -0x3d,0x00,0x04,0x00,0x46,0x00,0x00,0x00,0x5c,0x01,0x00,0x00, -0x4f,0x01,0x00,0x00,0xc2,0x00,0x05,0x00,0x46,0x00,0x00,0x00, -0x5d,0x01,0x00,0x00,0x5c,0x01,0x00,0x00,0x60,0x00,0x00,0x00, -0x70,0x00,0x04,0x00,0x42,0x00,0x00,0x00,0x5e,0x01,0x00,0x00, -0x5d,0x01,0x00,0x00,0x0c,0x00,0x08,0x00,0x42,0x00,0x00,0x00, -0x60,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00, -0xf7,0x00,0x00,0x00,0x5e,0x01,0x00,0x00,0x9d,0x01,0x00,0x00, -0x41,0x00,0x06,0x00,0x51,0x00,0x00,0x00,0x61,0x01,0x00,0x00, -0x08,0x01,0x00,0x00,0x09,0x00,0x00,0x00,0x59,0x01,0x00,0x00, -0x3e,0x00,0x03,0x00,0x61,0x01,0x00,0x00,0x60,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x69,0x01,0x00,0x00, -0x63,0x00,0x00,0x00,0x3f,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x6a,0x01,0x00,0x00,0x6a,0x00,0x00,0x00, -0x3f,0x00,0x00,0x00,0x41,0x00,0x08,0x00,0x73,0x00,0x00,0x00, -0x6b,0x01,0x00,0x00,0x4f,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x21,0x00,0x00,0x00,0x3f,0x00,0x00,0x00,0x6a,0x01,0x00,0x00, -0x3d,0x00,0x04,0x00,0x46,0x00,0x00,0x00,0x6c,0x01,0x00,0x00, -0x6b,0x01,0x00,0x00,0x71,0x00,0x04,0x00,0x14,0x00,0x00,0x00, -0x6d,0x01,0x00,0x00,0x6c,0x01,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x6e,0x01,0x00,0x00,0x6d,0x01,0x00,0x00, -0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x6f,0x01,0x00,0x00, -0x6e,0x01,0x00,0x00,0x90,0x00,0x00,0x00,0x6f,0x00,0x04,0x00, -0x42,0x00,0x00,0x00,0x70,0x01,0x00,0x00,0x6f,0x01,0x00,0x00, -0x0c,0x00,0x08,0x00,0x42,0x00,0x00,0x00,0x72,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0xb0,0x00,0x00,0x00, -0x70,0x01,0x00,0x00,0x9c,0x01,0x00,0x00,0x41,0x00,0x06,0x00, -0x51,0x00,0x00,0x00,0x73,0x01,0x00,0x00,0x08,0x01,0x00,0x00, -0x09,0x00,0x00,0x00,0x69,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0x73,0x01,0x00,0x00,0x72,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x75,0x01,0x00,0x00,0x63,0x00,0x00,0x00, -0xa2,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0x46,0x00,0x00,0x00, -0x78,0x01,0x00,0x00,0x6b,0x01,0x00,0x00,0xc2,0x00,0x05,0x00, -0x46,0x00,0x00,0x00,0x79,0x01,0x00,0x00,0x78,0x01,0x00,0x00, -0x60,0x00,0x00,0x00,0x70,0x00,0x04,0x00,0x42,0x00,0x00,0x00, -0x7a,0x01,0x00,0x00,0x79,0x01,0x00,0x00,0x0c,0x00,0x08,0x00, -0x42,0x00,0x00,0x00,0x7c,0x01,0x00,0x00,0x01,0x00,0x00,0x00, -0x32,0x00,0x00,0x00,0xf7,0x00,0x00,0x00,0x7a,0x01,0x00,0x00, -0x9d,0x01,0x00,0x00,0x41,0x00,0x06,0x00,0x51,0x00,0x00,0x00, -0x7d,0x01,0x00,0x00,0x08,0x01,0x00,0x00,0x09,0x00,0x00,0x00, -0x75,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x7d,0x01,0x00,0x00, -0x7c,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x85,0x01,0x00,0x00,0x63,0x00,0x00,0x00,0xda,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x86,0x01,0x00,0x00, -0x6a,0x00,0x00,0x00,0xda,0x00,0x00,0x00,0x41,0x00,0x08,0x00, -0x73,0x00,0x00,0x00,0x87,0x01,0x00,0x00,0x4f,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x3f,0x00,0x00,0x00, -0x86,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0x46,0x00,0x00,0x00, -0x88,0x01,0x00,0x00,0x87,0x01,0x00,0x00,0x71,0x00,0x04,0x00, -0x14,0x00,0x00,0x00,0x89,0x01,0x00,0x00,0x88,0x01,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x8a,0x01,0x00,0x00, -0x89,0x01,0x00,0x00,0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x8b,0x01,0x00,0x00,0x8a,0x01,0x00,0x00,0x90,0x00,0x00,0x00, -0x6f,0x00,0x04,0x00,0x42,0x00,0x00,0x00,0x8c,0x01,0x00,0x00, -0x8b,0x01,0x00,0x00,0x0c,0x00,0x08,0x00,0x42,0x00,0x00,0x00, -0x8e,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00, -0xb0,0x00,0x00,0x00,0x8c,0x01,0x00,0x00,0x9c,0x01,0x00,0x00, -0x41,0x00,0x06,0x00,0x51,0x00,0x00,0x00,0x8f,0x01,0x00,0x00, -0x08,0x01,0x00,0x00,0x09,0x00,0x00,0x00,0x85,0x01,0x00,0x00, -0x3e,0x00,0x03,0x00,0x8f,0x01,0x00,0x00,0x8e,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x91,0x01,0x00,0x00, -0x63,0x00,0x00,0x00,0xa5,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, -0x46,0x00,0x00,0x00,0x94,0x01,0x00,0x00,0x87,0x01,0x00,0x00, -0xc2,0x00,0x05,0x00,0x46,0x00,0x00,0x00,0x95,0x01,0x00,0x00, -0x94,0x01,0x00,0x00,0x60,0x00,0x00,0x00,0x70,0x00,0x04,0x00, -0x42,0x00,0x00,0x00,0x96,0x01,0x00,0x00,0x95,0x01,0x00,0x00, -0x0c,0x00,0x08,0x00,0x42,0x00,0x00,0x00,0x98,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0xf7,0x00,0x00,0x00, -0x96,0x01,0x00,0x00,0x9d,0x01,0x00,0x00,0x41,0x00,0x06,0x00, -0x51,0x00,0x00,0x00,0x99,0x01,0x00,0x00,0x08,0x01,0x00,0x00, -0x09,0x00,0x00,0x00,0x91,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0x99,0x01,0x00,0x00,0x98,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0x0d,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x0d,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x2f,0x01,0x00,0x00, -0x3a,0x01,0x00,0x00,0x29,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x0a,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x0c,0x00,0x00,0x00, -0xf5,0x00,0x07,0x00,0x11,0x00,0x00,0x00,0x43,0x01,0x00,0x00, -0x34,0x01,0x00,0x00,0x0a,0x00,0x00,0x00,0x37,0x01,0x00,0x00, -0x2f,0x00,0x00,0x00,0xf7,0x00,0x03,0x00,0x38,0x01,0x00,0x00, -0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x43,0x01,0x00,0x00, -0x32,0x01,0x00,0x00,0x38,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x38,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x32,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x32,0x01,0x00,0x00,0xfd,0x00,0x01,0x00, -0x38,0x00,0x01,0x00, -}; -const uint64_t dequant_q4_K_len = 5776; - -unsigned char dequant_q4_K_fp32_data[] = { -0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, 0xb1,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, 0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x27,0x00,0x00,0x00, 0x11,0x00,0x02,0x00,0x51,0x11,0x00,0x00,0x11,0x00,0x02,0x00, @@ -6669,1134 +3848,10 @@ unsigned char dequant_q4_K_fp32_data[] = { 0x37,0x01,0x00,0x00,0xfd,0x00,0x01,0x00,0x38,0x00,0x01,0x00, }; -const uint64_t dequant_q4_K_fp32_len = 5940; +const uint64_t dequant_q4_K_len = 5940; unsigned char dequant_q5_0_data[] = { 0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, -0x7a,0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, -0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x09,0x00,0x00,0x00, -0x11,0x00,0x02,0x00,0x27,0x00,0x00,0x00,0x11,0x00,0x02,0x00, -0x51,0x11,0x00,0x00,0x11,0x00,0x02,0x00,0x60,0x11,0x00,0x00, -0x0b,0x00,0x06,0x00,0x01,0x00,0x00,0x00,0x47,0x4c,0x53,0x4c, -0x2e,0x73,0x74,0x64,0x2e,0x34,0x35,0x30,0x00,0x00,0x00,0x00, -0x0e,0x00,0x03,0x00,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x0f,0x00,0x09,0x00,0x05,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x6d,0x61,0x69,0x6e,0x00,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x57,0x00,0x00,0x00,0xa1,0x00,0x00,0x00, -0x10,0x00,0x06,0x00,0x04,0x00,0x00,0x00,0x11,0x00,0x00,0x00, -0x00,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x0c,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x1c,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x14,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x14,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x14,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x08,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x14,0x00,0x00,0x00, -0x03,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x47,0x00,0x03,0x00,0x14,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x4f,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x52,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x53,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x53,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x53,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x54,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0x48,0x00,0x04,0x00,0x55,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x55,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x03,0x00,0x55,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x57,0x00,0x00,0x00,0x22,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x57,0x00,0x00,0x00, -0x21,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x9e,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x48,0x00,0x04,0x00,0x9f,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x19,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x9f,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x03,0x00,0x9f,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0xa1,0x00,0x00,0x00,0x22,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xa1,0x00,0x00,0x00, -0x21,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0xc0,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x19,0x00,0x00,0x00, -0x13,0x00,0x02,0x00,0x02,0x00,0x00,0x00,0x21,0x00,0x03,0x00, -0x03,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x15,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x15,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x17,0x00,0x04,0x00,0x0a,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x0b,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x0a,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x0b,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x0d,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x0e,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x1e,0x00,0x06,0x00,0x14,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x15,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x14,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x15,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x17,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x18,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x1b,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x14,0x00,0x02,0x00, -0x24,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x48,0x00,0x00,0x00, -0x10,0x00,0x00,0x00,0x16,0x00,0x03,0x00,0x4a,0x00,0x00,0x00, -0x10,0x00,0x00,0x00,0x15,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0x10,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x1c,0x00,0x04,0x00,0x4f,0x00,0x00,0x00,0x4d,0x00,0x00,0x00, -0x4e,0x00,0x00,0x00,0x15,0x00,0x04,0x00,0x50,0x00,0x00,0x00, -0x08,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x51,0x00,0x00,0x00,0x10,0x00,0x00,0x00, -0x1c,0x00,0x04,0x00,0x52,0x00,0x00,0x00,0x50,0x00,0x00,0x00, -0x51,0x00,0x00,0x00,0x1e,0x00,0x05,0x00,0x53,0x00,0x00,0x00, -0x4a,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x52,0x00,0x00,0x00, -0x1d,0x00,0x03,0x00,0x54,0x00,0x00,0x00,0x53,0x00,0x00,0x00, -0x1e,0x00,0x03,0x00,0x55,0x00,0x00,0x00,0x54,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x56,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x55,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x56,0x00,0x00,0x00, -0x57,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x59,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x4a,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x5f,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x4d,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x6f,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x75,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x7f,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x50,0x00,0x00,0x00,0x17,0x00,0x04,0x00,0x82,0x00,0x00,0x00, -0x4a,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x88,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x92,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0x99,0x00,0x00,0x00,0x00,0x4c,0x00,0x00,0x1d,0x00,0x03,0x00, -0x9e,0x00,0x00,0x00,0x4a,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, -0x9f,0x00,0x00,0x00,0x9e,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0xa0,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x9f,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0xa0,0x00,0x00,0x00,0xa1,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xa3,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0xbf,0x00,0x00,0x00,0x00,0x01,0x00,0x00, -0x2c,0x00,0x06,0x00,0x0a,0x00,0x00,0x00,0xc0,0x00,0x00,0x00, -0xbf,0x00,0x00,0x00,0x92,0x00,0x00,0x00,0x92,0x00,0x00,0x00, -0x2c,0x00,0x05,0x00,0x82,0x00,0x00,0x00,0xcd,0x00,0x00,0x00, -0x99,0x00,0x00,0x00,0x99,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x62,0x04,0x00,0x00,0x0d,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x63,0x04,0x00,0x00, -0x11,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x64,0x04,0x00,0x00,0x0e,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x65,0x04,0x00,0x00,0x12,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x66,0x04,0x00,0x00, -0x13,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x67,0x04,0x00,0x00,0x14,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x68,0x04,0x00,0x00,0x05,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x69,0x04,0x00,0x00, -0x15,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x6a,0x04,0x00,0x00,0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x6b,0x04,0x00,0x00,0x16,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x6c,0x04,0x00,0x00, -0x07,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x6d,0x04,0x00,0x00,0x17,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x6e,0x04,0x00,0x00,0x08,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x6f,0x04,0x00,0x00, -0x18,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x70,0x04,0x00,0x00,0x09,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x71,0x04,0x00,0x00,0x19,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x72,0x04,0x00,0x00, -0x0a,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x73,0x04,0x00,0x00,0x1a,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x74,0x04,0x00,0x00,0x0b,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x75,0x04,0x00,0x00, -0x1b,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x76,0x04,0x00,0x00,0x1c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x77,0x04,0x00,0x00,0x1d,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x78,0x04,0x00,0x00, -0x1e,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x79,0x04,0x00,0x00,0x1f,0x00,0x00,0x00,0x36,0x00,0x05,0x00, -0x02,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x03,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x05,0x00,0x00,0x00, -0xf7,0x00,0x03,0x00,0xc1,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0xfb,0x00,0x03,0x00,0x0d,0x00,0x00,0x00,0xc2,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xc2,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x0e,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x0d,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x10,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x11,0x00,0x00,0x00,0x10,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x18,0x00,0x00,0x00,0x19,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x17,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x1a,0x00,0x00,0x00,0x19,0x00,0x00,0x00, -0x87,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x1c,0x00,0x00,0x00, -0x1a,0x00,0x00,0x00,0x1b,0x00,0x00,0x00,0x8b,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x1d,0x00,0x00,0x00,0x11,0x00,0x00,0x00, -0x1c,0x00,0x00,0x00,0x87,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x11,0x00,0x00,0x00,0x1c,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x26,0x00,0x00,0x00, -0x1d,0x00,0x00,0x00,0x1b,0x00,0x00,0x00,0xaf,0x00,0x05,0x00, -0x24,0x00,0x00,0x00,0x29,0x00,0x00,0x00,0x26,0x00,0x00,0x00, -0x1a,0x00,0x00,0x00,0xa8,0x00,0x04,0x00,0x24,0x00,0x00,0x00, -0x2a,0x00,0x00,0x00,0x29,0x00,0x00,0x00,0xf7,0x00,0x03,0x00, -0x2c,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x2a,0x00,0x00,0x00,0x2b,0x00,0x00,0x00,0x2c,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0x2b,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x18,0x00,0x00,0x00,0x2f,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x30,0x00,0x00,0x00,0x2f,0x00,0x00,0x00,0xaf,0x00,0x05,0x00, -0x24,0x00,0x00,0x00,0x31,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x30,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x2c,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0x2c,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, -0x24,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x29,0x00,0x00,0x00, -0xc2,0x00,0x00,0x00,0x31,0x00,0x00,0x00,0x2b,0x00,0x00,0x00, -0xf7,0x00,0x03,0x00,0x34,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x32,0x00,0x00,0x00,0x33,0x00,0x00,0x00, -0x34,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x33,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xc1,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x34,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x18,0x00,0x00,0x00, -0x38,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x37,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x38,0x00,0x00,0x00,0x87,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x3a,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x1b,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3e,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x3e,0x00,0x00,0x00, -0x1d,0x00,0x00,0x00,0x41,0x00,0x07,0x00,0x59,0x00,0x00,0x00, -0x5a,0x00,0x00,0x00,0x57,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4a,0x00,0x00,0x00,0x5b,0x00,0x00,0x00,0x5a,0x00,0x00,0x00, -0x41,0x00,0x08,0x00,0x5f,0x00,0x00,0x00,0x60,0x00,0x00,0x00, -0x57,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x17,0x00,0x00,0x00,0x17,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0x61,0x00,0x00,0x00,0x60,0x00,0x00,0x00, -0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x62,0x00,0x00,0x00, -0x61,0x00,0x00,0x00,0xc4,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0x63,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x48,0x00,0x00,0x00, -0x41,0x00,0x08,0x00,0x5f,0x00,0x00,0x00,0x65,0x00,0x00,0x00, -0x57,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x17,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0x66,0x00,0x00,0x00,0x65,0x00,0x00,0x00, -0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x67,0x00,0x00,0x00, -0x66,0x00,0x00,0x00,0xc5,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0x68,0x00,0x00,0x00,0x63,0x00,0x00,0x00,0x67,0x00,0x00,0x00, -0xc2,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0x6e,0x00,0x00,0x00, -0x68,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0xc4,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0x70,0x00,0x00,0x00,0x6e,0x00,0x00,0x00, -0x6f,0x00,0x00,0x00,0xc7,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0x71,0x00,0x00,0x00,0x70,0x00,0x00,0x00,0x51,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x72,0x00,0x00,0x00, -0x71,0x00,0x00,0x00,0xc2,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0x77,0x00,0x00,0x00,0x68,0x00,0x00,0x00,0x75,0x00,0x00,0x00, -0xc7,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0x78,0x00,0x00,0x00, -0x77,0x00,0x00,0x00,0x51,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x79,0x00,0x00,0x00,0x78,0x00,0x00,0x00, -0x41,0x00,0x08,0x00,0x7f,0x00,0x00,0x00,0x80,0x00,0x00,0x00, -0x57,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x37,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x50,0x00,0x00,0x00,0x81,0x00,0x00,0x00,0x80,0x00,0x00,0x00, -0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x86,0x00,0x00,0x00, -0x81,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x87,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0xc7,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x89,0x00,0x00,0x00,0x87,0x00,0x00,0x00, -0x88,0x00,0x00,0x00,0xc5,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x8c,0x00,0x00,0x00,0x89,0x00,0x00,0x00,0x72,0x00,0x00,0x00, -0x6f,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0x8d,0x00,0x00,0x00, -0x8c,0x00,0x00,0x00,0xc2,0x00,0x05,0x00,0x50,0x00,0x00,0x00, -0x8f,0x00,0x00,0x00,0x81,0x00,0x00,0x00,0x6f,0x00,0x00,0x00, -0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x90,0x00,0x00,0x00, -0x8f,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x91,0x00,0x00,0x00,0x90,0x00,0x00,0x00,0xc5,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x95,0x00,0x00,0x00,0x91,0x00,0x00,0x00, -0x79,0x00,0x00,0x00,0x6f,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0x96,0x00,0x00,0x00,0x95,0x00,0x00,0x00,0x50,0x00,0x05,0x00, -0x82,0x00,0x00,0x00,0x97,0x00,0x00,0x00,0x8d,0x00,0x00,0x00, -0x96,0x00,0x00,0x00,0x83,0x00,0x05,0x00,0x82,0x00,0x00,0x00, -0x9b,0x00,0x00,0x00,0x97,0x00,0x00,0x00,0xcd,0x00,0x00,0x00, -0x8e,0x00,0x05,0x00,0x82,0x00,0x00,0x00,0x9d,0x00,0x00,0x00, -0x9b,0x00,0x00,0x00,0x5b,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x18,0x00,0x00,0x00,0xa4,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0xa3,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xa5,0x00,0x00,0x00,0xa4,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xa6,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0xa5,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xa9,0x00,0x00,0x00,0xa6,0x00,0x00,0x00,0x26,0x00,0x00,0x00, -0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00,0xae,0x00,0x00,0x00, -0x9d,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x41,0x00,0x06,0x00, -0x59,0x00,0x00,0x00,0xaf,0x00,0x00,0x00,0xa1,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0xa9,0x00,0x00,0x00,0x3e,0x00,0x03,0x00, -0xaf,0x00,0x00,0x00,0xae,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xb9,0x00,0x00,0x00,0xa9,0x00,0x00,0x00, -0x48,0x00,0x00,0x00,0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00, -0xbb,0x00,0x00,0x00,0x9d,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x41,0x00,0x06,0x00,0x59,0x00,0x00,0x00,0xbc,0x00,0x00,0x00, -0xa1,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0xb9,0x00,0x00,0x00, -0x3e,0x00,0x03,0x00,0xbc,0x00,0x00,0x00,0xbb,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0xd4,0x00,0x00,0x00, -0x5a,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0xd6,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x71,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0xd7,0x00,0x00,0x00,0xd6,0x00,0x00,0x00, -0xc4,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0xd8,0x00,0x00,0x00, -0xd7,0x00,0x00,0x00,0x48,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0xda,0x00,0x00,0x00,0x65,0x00,0x00,0x00, -0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0xdb,0x00,0x00,0x00, -0xda,0x00,0x00,0x00,0xc5,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0xdc,0x00,0x00,0x00,0xd8,0x00,0x00,0x00,0xdb,0x00,0x00,0x00, -0xc2,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0xdd,0x00,0x00,0x00, -0xdc,0x00,0x00,0x00,0x17,0x00,0x00,0x00,0xc4,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0xde,0x00,0x00,0x00,0xdd,0x00,0x00,0x00, -0x6f,0x00,0x00,0x00,0xc7,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0xdf,0x00,0x00,0x00,0xde,0x00,0x00,0x00,0x51,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xe0,0x00,0x00,0x00, -0xdf,0x00,0x00,0x00,0xc2,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0xe2,0x00,0x00,0x00,0xdc,0x00,0x00,0x00,0x62,0x04,0x00,0x00, -0xc7,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0xe3,0x00,0x00,0x00, -0xe2,0x00,0x00,0x00,0x51,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xe4,0x00,0x00,0x00,0xe3,0x00,0x00,0x00, -0x41,0x00,0x08,0x00,0x7f,0x00,0x00,0x00,0xe6,0x00,0x00,0x00, -0x57,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x37,0x00,0x00,0x00,0x17,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x50,0x00,0x00,0x00,0xe7,0x00,0x00,0x00,0xe6,0x00,0x00,0x00, -0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0xe8,0x00,0x00,0x00, -0xe7,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xe9,0x00,0x00,0x00,0xe8,0x00,0x00,0x00,0xc7,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xea,0x00,0x00,0x00,0xe9,0x00,0x00,0x00, -0x88,0x00,0x00,0x00,0xc5,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xec,0x00,0x00,0x00,0xea,0x00,0x00,0x00,0xe0,0x00,0x00,0x00, -0x6f,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0xed,0x00,0x00,0x00, -0xec,0x00,0x00,0x00,0xc2,0x00,0x05,0x00,0x50,0x00,0x00,0x00, -0xee,0x00,0x00,0x00,0xe7,0x00,0x00,0x00,0x6f,0x00,0x00,0x00, -0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0xef,0x00,0x00,0x00, -0xee,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xf0,0x00,0x00,0x00,0xef,0x00,0x00,0x00,0xc5,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xf2,0x00,0x00,0x00,0xf0,0x00,0x00,0x00, -0xe4,0x00,0x00,0x00,0x6f,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0xf3,0x00,0x00,0x00,0xf2,0x00,0x00,0x00,0x50,0x00,0x05,0x00, -0x82,0x00,0x00,0x00,0xf4,0x00,0x00,0x00,0xed,0x00,0x00,0x00, -0xf3,0x00,0x00,0x00,0x83,0x00,0x05,0x00,0x82,0x00,0x00,0x00, -0xf5,0x00,0x00,0x00,0xf4,0x00,0x00,0x00,0xcd,0x00,0x00,0x00, -0x8e,0x00,0x05,0x00,0x82,0x00,0x00,0x00,0xf6,0x00,0x00,0x00, -0xf5,0x00,0x00,0x00,0xd4,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xfc,0x00,0x00,0x00,0xa9,0x00,0x00,0x00, -0x17,0x00,0x00,0x00,0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00, -0xfe,0x00,0x00,0x00,0xf6,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x41,0x00,0x06,0x00,0x59,0x00,0x00,0x00,0xff,0x00,0x00,0x00, -0xa1,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0xfc,0x00,0x00,0x00, -0x3e,0x00,0x03,0x00,0xff,0x00,0x00,0x00,0xfe,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x06,0x01,0x00,0x00, -0xa9,0x00,0x00,0x00,0x63,0x04,0x00,0x00,0x51,0x00,0x05,0x00, -0x4a,0x00,0x00,0x00,0x07,0x01,0x00,0x00,0xf6,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x59,0x00,0x00,0x00, -0x08,0x01,0x00,0x00,0xa1,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x06,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x08,0x01,0x00,0x00, -0x07,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0x11,0x01,0x00,0x00,0x5a,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0x13,0x01,0x00,0x00,0x60,0x00,0x00,0x00, -0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x14,0x01,0x00,0x00, -0x13,0x01,0x00,0x00,0xc4,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0x15,0x01,0x00,0x00,0x14,0x01,0x00,0x00,0x48,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00,0x17,0x01,0x00,0x00, -0x65,0x00,0x00,0x00,0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x18,0x01,0x00,0x00,0x17,0x01,0x00,0x00,0xc5,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0x19,0x01,0x00,0x00,0x15,0x01,0x00,0x00, -0x18,0x01,0x00,0x00,0xc2,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0x1a,0x01,0x00,0x00,0x19,0x01,0x00,0x00,0x37,0x00,0x00,0x00, -0xc4,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0x1b,0x01,0x00,0x00, -0x1a,0x01,0x00,0x00,0x6f,0x00,0x00,0x00,0xc7,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0x1c,0x01,0x00,0x00,0x1b,0x01,0x00,0x00, -0x51,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x1d,0x01,0x00,0x00,0x1c,0x01,0x00,0x00,0xc2,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0x1f,0x01,0x00,0x00,0x19,0x01,0x00,0x00, -0x64,0x04,0x00,0x00,0xc7,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0x20,0x01,0x00,0x00,0x1f,0x01,0x00,0x00,0x51,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x21,0x01,0x00,0x00, -0x20,0x01,0x00,0x00,0x41,0x00,0x08,0x00,0x7f,0x00,0x00,0x00, -0x23,0x01,0x00,0x00,0x57,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x37,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x50,0x00,0x00,0x00,0x24,0x01,0x00,0x00, -0x23,0x01,0x00,0x00,0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x25,0x01,0x00,0x00,0x24,0x01,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x26,0x01,0x00,0x00,0x25,0x01,0x00,0x00, -0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x27,0x01,0x00,0x00, -0x26,0x01,0x00,0x00,0x88,0x00,0x00,0x00,0xc5,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x29,0x01,0x00,0x00,0x27,0x01,0x00,0x00, -0x1d,0x01,0x00,0x00,0x6f,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0x2a,0x01,0x00,0x00,0x29,0x01,0x00,0x00,0xc2,0x00,0x05,0x00, -0x50,0x00,0x00,0x00,0x2b,0x01,0x00,0x00,0x24,0x01,0x00,0x00, -0x6f,0x00,0x00,0x00,0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x2c,0x01,0x00,0x00,0x2b,0x01,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x2d,0x01,0x00,0x00,0x2c,0x01,0x00,0x00, -0xc5,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x2f,0x01,0x00,0x00, -0x2d,0x01,0x00,0x00,0x21,0x01,0x00,0x00,0x6f,0x00,0x04,0x00, -0x4a,0x00,0x00,0x00,0x30,0x01,0x00,0x00,0x2f,0x01,0x00,0x00, -0x50,0x00,0x05,0x00,0x82,0x00,0x00,0x00,0x31,0x01,0x00,0x00, -0x2a,0x01,0x00,0x00,0x30,0x01,0x00,0x00,0x83,0x00,0x05,0x00, -0x82,0x00,0x00,0x00,0x32,0x01,0x00,0x00,0x31,0x01,0x00,0x00, -0xcd,0x00,0x00,0x00,0x8e,0x00,0x05,0x00,0x82,0x00,0x00,0x00, -0x33,0x01,0x00,0x00,0x32,0x01,0x00,0x00,0x11,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x39,0x01,0x00,0x00, -0xa9,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x51,0x00,0x05,0x00, -0x4a,0x00,0x00,0x00,0x3b,0x01,0x00,0x00,0x33,0x01,0x00,0x00, -0x00,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x59,0x00,0x00,0x00, -0x3c,0x01,0x00,0x00,0xa1,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x39,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x3c,0x01,0x00,0x00, -0x3b,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x43,0x01,0x00,0x00,0xa9,0x00,0x00,0x00,0x65,0x04,0x00,0x00, -0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00,0x44,0x01,0x00,0x00, -0x33,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x41,0x00,0x06,0x00, -0x59,0x00,0x00,0x00,0x45,0x01,0x00,0x00,0xa1,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x43,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0x45,0x01,0x00,0x00,0x44,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4a,0x00,0x00,0x00,0x4e,0x01,0x00,0x00,0x5a,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00,0x50,0x01,0x00,0x00, -0x60,0x00,0x00,0x00,0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x51,0x01,0x00,0x00,0x50,0x01,0x00,0x00,0xc4,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0x52,0x01,0x00,0x00,0x51,0x01,0x00,0x00, -0x48,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0x54,0x01,0x00,0x00,0x65,0x00,0x00,0x00,0x71,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x55,0x01,0x00,0x00,0x54,0x01,0x00,0x00, -0xc5,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0x56,0x01,0x00,0x00, -0x52,0x01,0x00,0x00,0x55,0x01,0x00,0x00,0xc2,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0x57,0x01,0x00,0x00,0x56,0x01,0x00,0x00, -0xa3,0x00,0x00,0x00,0xc4,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0x58,0x01,0x00,0x00,0x57,0x01,0x00,0x00,0x6f,0x00,0x00,0x00, -0xc7,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0x59,0x01,0x00,0x00, -0x58,0x01,0x00,0x00,0x51,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x5a,0x01,0x00,0x00,0x59,0x01,0x00,0x00, -0xc2,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0x5c,0x01,0x00,0x00, -0x56,0x01,0x00,0x00,0x88,0x00,0x00,0x00,0xc7,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0x5d,0x01,0x00,0x00,0x5c,0x01,0x00,0x00, -0x51,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x5e,0x01,0x00,0x00,0x5d,0x01,0x00,0x00,0x41,0x00,0x08,0x00, -0x7f,0x00,0x00,0x00,0x60,0x01,0x00,0x00,0x57,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x37,0x00,0x00,0x00, -0xa3,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x50,0x00,0x00,0x00, -0x61,0x01,0x00,0x00,0x60,0x01,0x00,0x00,0x71,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x62,0x01,0x00,0x00,0x61,0x01,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x63,0x01,0x00,0x00, -0x62,0x01,0x00,0x00,0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x64,0x01,0x00,0x00,0x63,0x01,0x00,0x00,0x88,0x00,0x00,0x00, -0xc5,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x66,0x01,0x00,0x00, -0x64,0x01,0x00,0x00,0x5a,0x01,0x00,0x00,0x6f,0x00,0x04,0x00, -0x4a,0x00,0x00,0x00,0x67,0x01,0x00,0x00,0x66,0x01,0x00,0x00, -0xc2,0x00,0x05,0x00,0x50,0x00,0x00,0x00,0x68,0x01,0x00,0x00, -0x61,0x01,0x00,0x00,0x6f,0x00,0x00,0x00,0x71,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x69,0x01,0x00,0x00,0x68,0x01,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x6a,0x01,0x00,0x00, -0x69,0x01,0x00,0x00,0xc5,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x6c,0x01,0x00,0x00,0x6a,0x01,0x00,0x00,0x5e,0x01,0x00,0x00, -0x6f,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0x6d,0x01,0x00,0x00, -0x6c,0x01,0x00,0x00,0x50,0x00,0x05,0x00,0x82,0x00,0x00,0x00, -0x6e,0x01,0x00,0x00,0x67,0x01,0x00,0x00,0x6d,0x01,0x00,0x00, -0x83,0x00,0x05,0x00,0x82,0x00,0x00,0x00,0x6f,0x01,0x00,0x00, -0x6e,0x01,0x00,0x00,0xcd,0x00,0x00,0x00,0x8e,0x00,0x05,0x00, -0x82,0x00,0x00,0x00,0x70,0x01,0x00,0x00,0x6f,0x01,0x00,0x00, -0x4e,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x76,0x01,0x00,0x00,0xa9,0x00,0x00,0x00,0xa3,0x00,0x00,0x00, -0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00,0x78,0x01,0x00,0x00, -0x70,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x41,0x00,0x06,0x00, -0x59,0x00,0x00,0x00,0x79,0x01,0x00,0x00,0xa1,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x76,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0x79,0x01,0x00,0x00,0x78,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x80,0x01,0x00,0x00,0xa9,0x00,0x00,0x00, -0x66,0x04,0x00,0x00,0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00, -0x81,0x01,0x00,0x00,0x70,0x01,0x00,0x00,0x01,0x00,0x00,0x00, -0x41,0x00,0x06,0x00,0x59,0x00,0x00,0x00,0x82,0x01,0x00,0x00, -0xa1,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x80,0x01,0x00,0x00, -0x3e,0x00,0x03,0x00,0x82,0x01,0x00,0x00,0x81,0x01,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0x8b,0x01,0x00,0x00, -0x5a,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0x8d,0x01,0x00,0x00,0x60,0x00,0x00,0x00,0x71,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x8e,0x01,0x00,0x00,0x8d,0x01,0x00,0x00, -0xc4,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0x8f,0x01,0x00,0x00, -0x8e,0x01,0x00,0x00,0x48,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0x91,0x01,0x00,0x00,0x65,0x00,0x00,0x00, -0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x92,0x01,0x00,0x00, -0x91,0x01,0x00,0x00,0xc5,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0x93,0x01,0x00,0x00,0x8f,0x01,0x00,0x00,0x92,0x01,0x00,0x00, -0xc2,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0x94,0x01,0x00,0x00, -0x93,0x01,0x00,0x00,0x6f,0x00,0x00,0x00,0xc4,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0x95,0x01,0x00,0x00,0x94,0x01,0x00,0x00, -0x6f,0x00,0x00,0x00,0xc7,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0x96,0x01,0x00,0x00,0x95,0x01,0x00,0x00,0x51,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x97,0x01,0x00,0x00, -0x96,0x01,0x00,0x00,0xc2,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0x99,0x01,0x00,0x00,0x93,0x01,0x00,0x00,0x48,0x00,0x00,0x00, -0xc7,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0x9a,0x01,0x00,0x00, -0x99,0x01,0x00,0x00,0x51,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x9b,0x01,0x00,0x00,0x9a,0x01,0x00,0x00, -0x41,0x00,0x08,0x00,0x7f,0x00,0x00,0x00,0x9d,0x01,0x00,0x00, -0x57,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x37,0x00,0x00,0x00,0x6f,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x50,0x00,0x00,0x00,0x9e,0x01,0x00,0x00,0x9d,0x01,0x00,0x00, -0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x9f,0x01,0x00,0x00, -0x9e,0x01,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xa0,0x01,0x00,0x00,0x9f,0x01,0x00,0x00,0xc7,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xa1,0x01,0x00,0x00,0xa0,0x01,0x00,0x00, -0x88,0x00,0x00,0x00,0xc5,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xa3,0x01,0x00,0x00,0xa1,0x01,0x00,0x00,0x97,0x01,0x00,0x00, -0x6f,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0xa4,0x01,0x00,0x00, -0xa3,0x01,0x00,0x00,0xc2,0x00,0x05,0x00,0x50,0x00,0x00,0x00, -0xa5,0x01,0x00,0x00,0x9e,0x01,0x00,0x00,0x6f,0x00,0x00,0x00, -0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0xa6,0x01,0x00,0x00, -0xa5,0x01,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xa7,0x01,0x00,0x00,0xa6,0x01,0x00,0x00,0xc5,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xa9,0x01,0x00,0x00,0xa7,0x01,0x00,0x00, -0x9b,0x01,0x00,0x00,0x6f,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0xaa,0x01,0x00,0x00,0xa9,0x01,0x00,0x00,0x50,0x00,0x05,0x00, -0x82,0x00,0x00,0x00,0xab,0x01,0x00,0x00,0xa4,0x01,0x00,0x00, -0xaa,0x01,0x00,0x00,0x83,0x00,0x05,0x00,0x82,0x00,0x00,0x00, -0xac,0x01,0x00,0x00,0xab,0x01,0x00,0x00,0xcd,0x00,0x00,0x00, -0x8e,0x00,0x05,0x00,0x82,0x00,0x00,0x00,0xad,0x01,0x00,0x00, -0xac,0x01,0x00,0x00,0x8b,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xb3,0x01,0x00,0x00,0xa9,0x00,0x00,0x00, -0x6f,0x00,0x00,0x00,0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00, -0xb5,0x01,0x00,0x00,0xad,0x01,0x00,0x00,0x00,0x00,0x00,0x00, -0x41,0x00,0x06,0x00,0x59,0x00,0x00,0x00,0xb6,0x01,0x00,0x00, -0xa1,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0xb3,0x01,0x00,0x00, -0x3e,0x00,0x03,0x00,0xb6,0x01,0x00,0x00,0xb5,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xbd,0x01,0x00,0x00, -0xa9,0x00,0x00,0x00,0x67,0x04,0x00,0x00,0x51,0x00,0x05,0x00, -0x4a,0x00,0x00,0x00,0xbe,0x01,0x00,0x00,0xad,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x59,0x00,0x00,0x00, -0xbf,0x01,0x00,0x00,0xa1,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0xbd,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0xbf,0x01,0x00,0x00, -0xbe,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0xc8,0x01,0x00,0x00,0x5a,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0xca,0x01,0x00,0x00,0x60,0x00,0x00,0x00, -0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0xcb,0x01,0x00,0x00, -0xca,0x01,0x00,0x00,0xc4,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0xcc,0x01,0x00,0x00,0xcb,0x01,0x00,0x00,0x48,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00,0xce,0x01,0x00,0x00, -0x65,0x00,0x00,0x00,0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0xcf,0x01,0x00,0x00,0xce,0x01,0x00,0x00,0xc5,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0xd0,0x01,0x00,0x00,0xcc,0x01,0x00,0x00, -0xcf,0x01,0x00,0x00,0xc2,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0xd1,0x01,0x00,0x00,0xd0,0x01,0x00,0x00,0x68,0x04,0x00,0x00, -0xc4,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0xd2,0x01,0x00,0x00, -0xd1,0x01,0x00,0x00,0x6f,0x00,0x00,0x00,0xc7,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0xd3,0x01,0x00,0x00,0xd2,0x01,0x00,0x00, -0x51,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xd4,0x01,0x00,0x00,0xd3,0x01,0x00,0x00,0xc2,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0xd6,0x01,0x00,0x00,0xd0,0x01,0x00,0x00, -0x63,0x04,0x00,0x00,0xc7,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0xd7,0x01,0x00,0x00,0xd6,0x01,0x00,0x00,0x51,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xd8,0x01,0x00,0x00, -0xd7,0x01,0x00,0x00,0x41,0x00,0x08,0x00,0x7f,0x00,0x00,0x00, -0xda,0x01,0x00,0x00,0x57,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x68,0x04,0x00,0x00, -0x3d,0x00,0x04,0x00,0x50,0x00,0x00,0x00,0xdb,0x01,0x00,0x00, -0xda,0x01,0x00,0x00,0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0xdc,0x01,0x00,0x00,0xdb,0x01,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xdd,0x01,0x00,0x00,0xdc,0x01,0x00,0x00, -0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xde,0x01,0x00,0x00, -0xdd,0x01,0x00,0x00,0x88,0x00,0x00,0x00,0xc5,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xe0,0x01,0x00,0x00,0xde,0x01,0x00,0x00, -0xd4,0x01,0x00,0x00,0x6f,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0xe1,0x01,0x00,0x00,0xe0,0x01,0x00,0x00,0xc2,0x00,0x05,0x00, -0x50,0x00,0x00,0x00,0xe2,0x01,0x00,0x00,0xdb,0x01,0x00,0x00, -0x6f,0x00,0x00,0x00,0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0xe3,0x01,0x00,0x00,0xe2,0x01,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xe4,0x01,0x00,0x00,0xe3,0x01,0x00,0x00, -0xc5,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xe6,0x01,0x00,0x00, -0xe4,0x01,0x00,0x00,0xd8,0x01,0x00,0x00,0x6f,0x00,0x04,0x00, -0x4a,0x00,0x00,0x00,0xe7,0x01,0x00,0x00,0xe6,0x01,0x00,0x00, -0x50,0x00,0x05,0x00,0x82,0x00,0x00,0x00,0xe8,0x01,0x00,0x00, -0xe1,0x01,0x00,0x00,0xe7,0x01,0x00,0x00,0x83,0x00,0x05,0x00, -0x82,0x00,0x00,0x00,0xe9,0x01,0x00,0x00,0xe8,0x01,0x00,0x00, -0xcd,0x00,0x00,0x00,0x8e,0x00,0x05,0x00,0x82,0x00,0x00,0x00, -0xea,0x01,0x00,0x00,0xe9,0x01,0x00,0x00,0xc8,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xf0,0x01,0x00,0x00, -0xa9,0x00,0x00,0x00,0x68,0x04,0x00,0x00,0x51,0x00,0x05,0x00, -0x4a,0x00,0x00,0x00,0xf2,0x01,0x00,0x00,0xea,0x01,0x00,0x00, -0x00,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x59,0x00,0x00,0x00, -0xf3,0x01,0x00,0x00,0xa1,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0xf0,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0xf3,0x01,0x00,0x00, -0xf2,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xfa,0x01,0x00,0x00,0xa9,0x00,0x00,0x00,0x69,0x04,0x00,0x00, -0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00,0xfb,0x01,0x00,0x00, -0xea,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x41,0x00,0x06,0x00, -0x59,0x00,0x00,0x00,0xfc,0x01,0x00,0x00,0xa1,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0xfa,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0xfc,0x01,0x00,0x00,0xfb,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4a,0x00,0x00,0x00,0x05,0x02,0x00,0x00,0x5a,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00,0x07,0x02,0x00,0x00, -0x60,0x00,0x00,0x00,0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x08,0x02,0x00,0x00,0x07,0x02,0x00,0x00,0xc4,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0x09,0x02,0x00,0x00,0x08,0x02,0x00,0x00, -0x48,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0x0b,0x02,0x00,0x00,0x65,0x00,0x00,0x00,0x71,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x0c,0x02,0x00,0x00,0x0b,0x02,0x00,0x00, -0xc5,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0x0d,0x02,0x00,0x00, -0x09,0x02,0x00,0x00,0x0c,0x02,0x00,0x00,0xc2,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0x0e,0x02,0x00,0x00,0x0d,0x02,0x00,0x00, -0x6a,0x04,0x00,0x00,0xc4,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0x0f,0x02,0x00,0x00,0x0e,0x02,0x00,0x00,0x6f,0x00,0x00,0x00, -0xc7,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0x10,0x02,0x00,0x00, -0x0f,0x02,0x00,0x00,0x51,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x11,0x02,0x00,0x00,0x10,0x02,0x00,0x00, -0xc2,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0x13,0x02,0x00,0x00, -0x0d,0x02,0x00,0x00,0x65,0x04,0x00,0x00,0xc7,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0x14,0x02,0x00,0x00,0x13,0x02,0x00,0x00, -0x51,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x15,0x02,0x00,0x00,0x14,0x02,0x00,0x00,0x41,0x00,0x08,0x00, -0x7f,0x00,0x00,0x00,0x17,0x02,0x00,0x00,0x57,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x37,0x00,0x00,0x00, -0x6a,0x04,0x00,0x00,0x3d,0x00,0x04,0x00,0x50,0x00,0x00,0x00, -0x18,0x02,0x00,0x00,0x17,0x02,0x00,0x00,0x71,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x19,0x02,0x00,0x00,0x18,0x02,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x1a,0x02,0x00,0x00, -0x19,0x02,0x00,0x00,0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x1b,0x02,0x00,0x00,0x1a,0x02,0x00,0x00,0x88,0x00,0x00,0x00, -0xc5,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x1d,0x02,0x00,0x00, -0x1b,0x02,0x00,0x00,0x11,0x02,0x00,0x00,0x6f,0x00,0x04,0x00, -0x4a,0x00,0x00,0x00,0x1e,0x02,0x00,0x00,0x1d,0x02,0x00,0x00, -0xc2,0x00,0x05,0x00,0x50,0x00,0x00,0x00,0x1f,0x02,0x00,0x00, -0x18,0x02,0x00,0x00,0x6f,0x00,0x00,0x00,0x71,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x20,0x02,0x00,0x00,0x1f,0x02,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x21,0x02,0x00,0x00, -0x20,0x02,0x00,0x00,0xc5,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x23,0x02,0x00,0x00,0x21,0x02,0x00,0x00,0x15,0x02,0x00,0x00, -0x6f,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0x24,0x02,0x00,0x00, -0x23,0x02,0x00,0x00,0x50,0x00,0x05,0x00,0x82,0x00,0x00,0x00, -0x25,0x02,0x00,0x00,0x1e,0x02,0x00,0x00,0x24,0x02,0x00,0x00, -0x83,0x00,0x05,0x00,0x82,0x00,0x00,0x00,0x26,0x02,0x00,0x00, -0x25,0x02,0x00,0x00,0xcd,0x00,0x00,0x00,0x8e,0x00,0x05,0x00, -0x82,0x00,0x00,0x00,0x27,0x02,0x00,0x00,0x26,0x02,0x00,0x00, -0x05,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x2d,0x02,0x00,0x00,0xa9,0x00,0x00,0x00,0x6a,0x04,0x00,0x00, -0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00,0x2f,0x02,0x00,0x00, -0x27,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x41,0x00,0x06,0x00, -0x59,0x00,0x00,0x00,0x30,0x02,0x00,0x00,0xa1,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x2d,0x02,0x00,0x00,0x3e,0x00,0x03,0x00, -0x30,0x02,0x00,0x00,0x2f,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x37,0x02,0x00,0x00,0xa9,0x00,0x00,0x00, -0x6b,0x04,0x00,0x00,0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00, -0x38,0x02,0x00,0x00,0x27,0x02,0x00,0x00,0x01,0x00,0x00,0x00, -0x41,0x00,0x06,0x00,0x59,0x00,0x00,0x00,0x39,0x02,0x00,0x00, -0xa1,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x37,0x02,0x00,0x00, -0x3e,0x00,0x03,0x00,0x39,0x02,0x00,0x00,0x38,0x02,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0x42,0x02,0x00,0x00, -0x5a,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0x44,0x02,0x00,0x00,0x60,0x00,0x00,0x00,0x71,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x45,0x02,0x00,0x00,0x44,0x02,0x00,0x00, -0xc4,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0x46,0x02,0x00,0x00, -0x45,0x02,0x00,0x00,0x48,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0x48,0x02,0x00,0x00,0x65,0x00,0x00,0x00, -0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x49,0x02,0x00,0x00, -0x48,0x02,0x00,0x00,0xc5,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0x4a,0x02,0x00,0x00,0x46,0x02,0x00,0x00,0x49,0x02,0x00,0x00, -0xc2,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0x4b,0x02,0x00,0x00, -0x4a,0x02,0x00,0x00,0x6c,0x04,0x00,0x00,0xc4,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0x4c,0x02,0x00,0x00,0x4b,0x02,0x00,0x00, -0x6f,0x00,0x00,0x00,0xc7,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0x4d,0x02,0x00,0x00,0x4c,0x02,0x00,0x00,0x51,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x4e,0x02,0x00,0x00, -0x4d,0x02,0x00,0x00,0xc2,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0x50,0x02,0x00,0x00,0x4a,0x02,0x00,0x00,0x66,0x04,0x00,0x00, -0xc7,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0x51,0x02,0x00,0x00, -0x50,0x02,0x00,0x00,0x51,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x52,0x02,0x00,0x00,0x51,0x02,0x00,0x00, -0x41,0x00,0x08,0x00,0x7f,0x00,0x00,0x00,0x54,0x02,0x00,0x00, -0x57,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x37,0x00,0x00,0x00,0x6c,0x04,0x00,0x00,0x3d,0x00,0x04,0x00, -0x50,0x00,0x00,0x00,0x55,0x02,0x00,0x00,0x54,0x02,0x00,0x00, -0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x56,0x02,0x00,0x00, -0x55,0x02,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x57,0x02,0x00,0x00,0x56,0x02,0x00,0x00,0xc7,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x58,0x02,0x00,0x00,0x57,0x02,0x00,0x00, -0x88,0x00,0x00,0x00,0xc5,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x5a,0x02,0x00,0x00,0x58,0x02,0x00,0x00,0x4e,0x02,0x00,0x00, -0x6f,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0x5b,0x02,0x00,0x00, -0x5a,0x02,0x00,0x00,0xc2,0x00,0x05,0x00,0x50,0x00,0x00,0x00, -0x5c,0x02,0x00,0x00,0x55,0x02,0x00,0x00,0x6f,0x00,0x00,0x00, -0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x5d,0x02,0x00,0x00, -0x5c,0x02,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x5e,0x02,0x00,0x00,0x5d,0x02,0x00,0x00,0xc5,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x60,0x02,0x00,0x00,0x5e,0x02,0x00,0x00, -0x52,0x02,0x00,0x00,0x6f,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0x61,0x02,0x00,0x00,0x60,0x02,0x00,0x00,0x50,0x00,0x05,0x00, -0x82,0x00,0x00,0x00,0x62,0x02,0x00,0x00,0x5b,0x02,0x00,0x00, -0x61,0x02,0x00,0x00,0x83,0x00,0x05,0x00,0x82,0x00,0x00,0x00, -0x63,0x02,0x00,0x00,0x62,0x02,0x00,0x00,0xcd,0x00,0x00,0x00, -0x8e,0x00,0x05,0x00,0x82,0x00,0x00,0x00,0x64,0x02,0x00,0x00, -0x63,0x02,0x00,0x00,0x42,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x6a,0x02,0x00,0x00,0xa9,0x00,0x00,0x00, -0x6c,0x04,0x00,0x00,0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00, -0x6c,0x02,0x00,0x00,0x64,0x02,0x00,0x00,0x00,0x00,0x00,0x00, -0x41,0x00,0x06,0x00,0x59,0x00,0x00,0x00,0x6d,0x02,0x00,0x00, -0xa1,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x6a,0x02,0x00,0x00, -0x3e,0x00,0x03,0x00,0x6d,0x02,0x00,0x00,0x6c,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x74,0x02,0x00,0x00, -0xa9,0x00,0x00,0x00,0x6d,0x04,0x00,0x00,0x51,0x00,0x05,0x00, -0x4a,0x00,0x00,0x00,0x75,0x02,0x00,0x00,0x64,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x59,0x00,0x00,0x00, -0x76,0x02,0x00,0x00,0xa1,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x74,0x02,0x00,0x00,0x3e,0x00,0x03,0x00,0x76,0x02,0x00,0x00, -0x75,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0x7f,0x02,0x00,0x00,0x5a,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0x81,0x02,0x00,0x00,0x60,0x00,0x00,0x00, -0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x82,0x02,0x00,0x00, -0x81,0x02,0x00,0x00,0xc4,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0x83,0x02,0x00,0x00,0x82,0x02,0x00,0x00,0x48,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00,0x85,0x02,0x00,0x00, -0x65,0x00,0x00,0x00,0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x86,0x02,0x00,0x00,0x85,0x02,0x00,0x00,0xc5,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0x87,0x02,0x00,0x00,0x83,0x02,0x00,0x00, -0x86,0x02,0x00,0x00,0xc2,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0x88,0x02,0x00,0x00,0x87,0x02,0x00,0x00,0x6e,0x04,0x00,0x00, -0xc4,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0x89,0x02,0x00,0x00, -0x88,0x02,0x00,0x00,0x6f,0x00,0x00,0x00,0xc7,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0x8a,0x02,0x00,0x00,0x89,0x02,0x00,0x00, -0x51,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x8b,0x02,0x00,0x00,0x8a,0x02,0x00,0x00,0xc2,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0x8d,0x02,0x00,0x00,0x87,0x02,0x00,0x00, -0x67,0x04,0x00,0x00,0xc7,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0x8e,0x02,0x00,0x00,0x8d,0x02,0x00,0x00,0x51,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x8f,0x02,0x00,0x00, -0x8e,0x02,0x00,0x00,0x41,0x00,0x08,0x00,0x7f,0x00,0x00,0x00, -0x91,0x02,0x00,0x00,0x57,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x6e,0x04,0x00,0x00, -0x3d,0x00,0x04,0x00,0x50,0x00,0x00,0x00,0x92,0x02,0x00,0x00, -0x91,0x02,0x00,0x00,0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x93,0x02,0x00,0x00,0x92,0x02,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x94,0x02,0x00,0x00,0x93,0x02,0x00,0x00, -0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x95,0x02,0x00,0x00, -0x94,0x02,0x00,0x00,0x88,0x00,0x00,0x00,0xc5,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x97,0x02,0x00,0x00,0x95,0x02,0x00,0x00, -0x8b,0x02,0x00,0x00,0x6f,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0x98,0x02,0x00,0x00,0x97,0x02,0x00,0x00,0xc2,0x00,0x05,0x00, -0x50,0x00,0x00,0x00,0x99,0x02,0x00,0x00,0x92,0x02,0x00,0x00, -0x6f,0x00,0x00,0x00,0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x9a,0x02,0x00,0x00,0x99,0x02,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x9b,0x02,0x00,0x00,0x9a,0x02,0x00,0x00, -0xc5,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9d,0x02,0x00,0x00, -0x9b,0x02,0x00,0x00,0x8f,0x02,0x00,0x00,0x6f,0x00,0x04,0x00, -0x4a,0x00,0x00,0x00,0x9e,0x02,0x00,0x00,0x9d,0x02,0x00,0x00, -0x50,0x00,0x05,0x00,0x82,0x00,0x00,0x00,0x9f,0x02,0x00,0x00, -0x98,0x02,0x00,0x00,0x9e,0x02,0x00,0x00,0x83,0x00,0x05,0x00, -0x82,0x00,0x00,0x00,0xa0,0x02,0x00,0x00,0x9f,0x02,0x00,0x00, -0xcd,0x00,0x00,0x00,0x8e,0x00,0x05,0x00,0x82,0x00,0x00,0x00, -0xa1,0x02,0x00,0x00,0xa0,0x02,0x00,0x00,0x7f,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa7,0x02,0x00,0x00, -0xa9,0x00,0x00,0x00,0x6e,0x04,0x00,0x00,0x51,0x00,0x05,0x00, -0x4a,0x00,0x00,0x00,0xa9,0x02,0x00,0x00,0xa1,0x02,0x00,0x00, -0x00,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x59,0x00,0x00,0x00, -0xaa,0x02,0x00,0x00,0xa1,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0xa7,0x02,0x00,0x00,0x3e,0x00,0x03,0x00,0xaa,0x02,0x00,0x00, -0xa9,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xb1,0x02,0x00,0x00,0xa9,0x00,0x00,0x00,0x6f,0x04,0x00,0x00, -0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00,0xb2,0x02,0x00,0x00, -0xa1,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0x41,0x00,0x06,0x00, -0x59,0x00,0x00,0x00,0xb3,0x02,0x00,0x00,0xa1,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0xb1,0x02,0x00,0x00,0x3e,0x00,0x03,0x00, -0xb3,0x02,0x00,0x00,0xb2,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4a,0x00,0x00,0x00,0xbc,0x02,0x00,0x00,0x5a,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00,0xbe,0x02,0x00,0x00, -0x60,0x00,0x00,0x00,0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0xbf,0x02,0x00,0x00,0xbe,0x02,0x00,0x00,0xc4,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0xc0,0x02,0x00,0x00,0xbf,0x02,0x00,0x00, -0x48,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0xc2,0x02,0x00,0x00,0x65,0x00,0x00,0x00,0x71,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0xc3,0x02,0x00,0x00,0xc2,0x02,0x00,0x00, -0xc5,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0xc4,0x02,0x00,0x00, -0xc0,0x02,0x00,0x00,0xc3,0x02,0x00,0x00,0xc2,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0xc5,0x02,0x00,0x00,0xc4,0x02,0x00,0x00, -0x70,0x04,0x00,0x00,0xc4,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0xc6,0x02,0x00,0x00,0xc5,0x02,0x00,0x00,0x6f,0x00,0x00,0x00, -0xc7,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0xc7,0x02,0x00,0x00, -0xc6,0x02,0x00,0x00,0x51,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xc8,0x02,0x00,0x00,0xc7,0x02,0x00,0x00, -0xc2,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0xca,0x02,0x00,0x00, -0xc4,0x02,0x00,0x00,0x69,0x04,0x00,0x00,0xc7,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0xcb,0x02,0x00,0x00,0xca,0x02,0x00,0x00, -0x51,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xcc,0x02,0x00,0x00,0xcb,0x02,0x00,0x00,0x41,0x00,0x08,0x00, -0x7f,0x00,0x00,0x00,0xce,0x02,0x00,0x00,0x57,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x37,0x00,0x00,0x00, -0x70,0x04,0x00,0x00,0x3d,0x00,0x04,0x00,0x50,0x00,0x00,0x00, -0xcf,0x02,0x00,0x00,0xce,0x02,0x00,0x00,0x71,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0xd0,0x02,0x00,0x00,0xcf,0x02,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xd1,0x02,0x00,0x00, -0xd0,0x02,0x00,0x00,0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xd2,0x02,0x00,0x00,0xd1,0x02,0x00,0x00,0x88,0x00,0x00,0x00, -0xc5,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xd4,0x02,0x00,0x00, -0xd2,0x02,0x00,0x00,0xc8,0x02,0x00,0x00,0x6f,0x00,0x04,0x00, -0x4a,0x00,0x00,0x00,0xd5,0x02,0x00,0x00,0xd4,0x02,0x00,0x00, -0xc2,0x00,0x05,0x00,0x50,0x00,0x00,0x00,0xd6,0x02,0x00,0x00, -0xcf,0x02,0x00,0x00,0x6f,0x00,0x00,0x00,0x71,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0xd7,0x02,0x00,0x00,0xd6,0x02,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xd8,0x02,0x00,0x00, -0xd7,0x02,0x00,0x00,0xc5,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xda,0x02,0x00,0x00,0xd8,0x02,0x00,0x00,0xcc,0x02,0x00,0x00, -0x6f,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0xdb,0x02,0x00,0x00, -0xda,0x02,0x00,0x00,0x50,0x00,0x05,0x00,0x82,0x00,0x00,0x00, -0xdc,0x02,0x00,0x00,0xd5,0x02,0x00,0x00,0xdb,0x02,0x00,0x00, -0x83,0x00,0x05,0x00,0x82,0x00,0x00,0x00,0xdd,0x02,0x00,0x00, -0xdc,0x02,0x00,0x00,0xcd,0x00,0x00,0x00,0x8e,0x00,0x05,0x00, -0x82,0x00,0x00,0x00,0xde,0x02,0x00,0x00,0xdd,0x02,0x00,0x00, -0xbc,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xe4,0x02,0x00,0x00,0xa9,0x00,0x00,0x00,0x70,0x04,0x00,0x00, -0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00,0xe6,0x02,0x00,0x00, -0xde,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x41,0x00,0x06,0x00, -0x59,0x00,0x00,0x00,0xe7,0x02,0x00,0x00,0xa1,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0xe4,0x02,0x00,0x00,0x3e,0x00,0x03,0x00, -0xe7,0x02,0x00,0x00,0xe6,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xee,0x02,0x00,0x00,0xa9,0x00,0x00,0x00, -0x71,0x04,0x00,0x00,0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00, -0xef,0x02,0x00,0x00,0xde,0x02,0x00,0x00,0x01,0x00,0x00,0x00, -0x41,0x00,0x06,0x00,0x59,0x00,0x00,0x00,0xf0,0x02,0x00,0x00, -0xa1,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0xee,0x02,0x00,0x00, -0x3e,0x00,0x03,0x00,0xf0,0x02,0x00,0x00,0xef,0x02,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0xf9,0x02,0x00,0x00, -0x5a,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0xfb,0x02,0x00,0x00,0x60,0x00,0x00,0x00,0x71,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0xfc,0x02,0x00,0x00,0xfb,0x02,0x00,0x00, -0xc4,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0xfd,0x02,0x00,0x00, -0xfc,0x02,0x00,0x00,0x48,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0xff,0x02,0x00,0x00,0x65,0x00,0x00,0x00, -0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x00,0x03,0x00,0x00, -0xff,0x02,0x00,0x00,0xc5,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0x01,0x03,0x00,0x00,0xfd,0x02,0x00,0x00,0x00,0x03,0x00,0x00, -0xc2,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0x02,0x03,0x00,0x00, -0x01,0x03,0x00,0x00,0x72,0x04,0x00,0x00,0xc4,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0x03,0x03,0x00,0x00,0x02,0x03,0x00,0x00, -0x6f,0x00,0x00,0x00,0xc7,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0x04,0x03,0x00,0x00,0x03,0x03,0x00,0x00,0x51,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x05,0x03,0x00,0x00, -0x04,0x03,0x00,0x00,0xc2,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0x07,0x03,0x00,0x00,0x01,0x03,0x00,0x00,0x6b,0x04,0x00,0x00, -0xc7,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0x08,0x03,0x00,0x00, -0x07,0x03,0x00,0x00,0x51,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x09,0x03,0x00,0x00,0x08,0x03,0x00,0x00, -0x41,0x00,0x08,0x00,0x7f,0x00,0x00,0x00,0x0b,0x03,0x00,0x00, -0x57,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x37,0x00,0x00,0x00,0x72,0x04,0x00,0x00,0x3d,0x00,0x04,0x00, -0x50,0x00,0x00,0x00,0x0c,0x03,0x00,0x00,0x0b,0x03,0x00,0x00, -0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x0d,0x03,0x00,0x00, -0x0c,0x03,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x0e,0x03,0x00,0x00,0x0d,0x03,0x00,0x00,0xc7,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x0f,0x03,0x00,0x00,0x0e,0x03,0x00,0x00, -0x88,0x00,0x00,0x00,0xc5,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x11,0x03,0x00,0x00,0x0f,0x03,0x00,0x00,0x05,0x03,0x00,0x00, -0x6f,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0x12,0x03,0x00,0x00, -0x11,0x03,0x00,0x00,0xc2,0x00,0x05,0x00,0x50,0x00,0x00,0x00, -0x13,0x03,0x00,0x00,0x0c,0x03,0x00,0x00,0x6f,0x00,0x00,0x00, -0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x14,0x03,0x00,0x00, -0x13,0x03,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x15,0x03,0x00,0x00,0x14,0x03,0x00,0x00,0xc5,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x17,0x03,0x00,0x00,0x15,0x03,0x00,0x00, -0x09,0x03,0x00,0x00,0x6f,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0x18,0x03,0x00,0x00,0x17,0x03,0x00,0x00,0x50,0x00,0x05,0x00, -0x82,0x00,0x00,0x00,0x19,0x03,0x00,0x00,0x12,0x03,0x00,0x00, -0x18,0x03,0x00,0x00,0x83,0x00,0x05,0x00,0x82,0x00,0x00,0x00, -0x1a,0x03,0x00,0x00,0x19,0x03,0x00,0x00,0xcd,0x00,0x00,0x00, -0x8e,0x00,0x05,0x00,0x82,0x00,0x00,0x00,0x1b,0x03,0x00,0x00, -0x1a,0x03,0x00,0x00,0xf9,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x21,0x03,0x00,0x00,0xa9,0x00,0x00,0x00, -0x72,0x04,0x00,0x00,0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00, -0x23,0x03,0x00,0x00,0x1b,0x03,0x00,0x00,0x00,0x00,0x00,0x00, -0x41,0x00,0x06,0x00,0x59,0x00,0x00,0x00,0x24,0x03,0x00,0x00, -0xa1,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x21,0x03,0x00,0x00, -0x3e,0x00,0x03,0x00,0x24,0x03,0x00,0x00,0x23,0x03,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x2b,0x03,0x00,0x00, -0xa9,0x00,0x00,0x00,0x73,0x04,0x00,0x00,0x51,0x00,0x05,0x00, -0x4a,0x00,0x00,0x00,0x2c,0x03,0x00,0x00,0x1b,0x03,0x00,0x00, -0x01,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x59,0x00,0x00,0x00, -0x2d,0x03,0x00,0x00,0xa1,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x2b,0x03,0x00,0x00,0x3e,0x00,0x03,0x00,0x2d,0x03,0x00,0x00, -0x2c,0x03,0x00,0x00,0x3d,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0x36,0x03,0x00,0x00,0x5a,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0x38,0x03,0x00,0x00,0x60,0x00,0x00,0x00, -0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x39,0x03,0x00,0x00, -0x38,0x03,0x00,0x00,0xc4,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0x3a,0x03,0x00,0x00,0x39,0x03,0x00,0x00,0x48,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00,0x3c,0x03,0x00,0x00, -0x65,0x00,0x00,0x00,0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x3d,0x03,0x00,0x00,0x3c,0x03,0x00,0x00,0xc5,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0x3e,0x03,0x00,0x00,0x3a,0x03,0x00,0x00, -0x3d,0x03,0x00,0x00,0xc2,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0x3f,0x03,0x00,0x00,0x3e,0x03,0x00,0x00,0x74,0x04,0x00,0x00, -0xc4,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0x40,0x03,0x00,0x00, -0x3f,0x03,0x00,0x00,0x6f,0x00,0x00,0x00,0xc7,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0x41,0x03,0x00,0x00,0x40,0x03,0x00,0x00, -0x51,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x42,0x03,0x00,0x00,0x41,0x03,0x00,0x00,0xc2,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0x44,0x03,0x00,0x00,0x3e,0x03,0x00,0x00, -0x6d,0x04,0x00,0x00,0xc7,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0x45,0x03,0x00,0x00,0x44,0x03,0x00,0x00,0x51,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x46,0x03,0x00,0x00, -0x45,0x03,0x00,0x00,0x41,0x00,0x08,0x00,0x7f,0x00,0x00,0x00, -0x48,0x03,0x00,0x00,0x57,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x74,0x04,0x00,0x00, -0x3d,0x00,0x04,0x00,0x50,0x00,0x00,0x00,0x49,0x03,0x00,0x00, -0x48,0x03,0x00,0x00,0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x4a,0x03,0x00,0x00,0x49,0x03,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x4b,0x03,0x00,0x00,0x4a,0x03,0x00,0x00, -0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x4c,0x03,0x00,0x00, -0x4b,0x03,0x00,0x00,0x88,0x00,0x00,0x00,0xc5,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x4e,0x03,0x00,0x00,0x4c,0x03,0x00,0x00, -0x42,0x03,0x00,0x00,0x6f,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0x4f,0x03,0x00,0x00,0x4e,0x03,0x00,0x00,0xc2,0x00,0x05,0x00, -0x50,0x00,0x00,0x00,0x50,0x03,0x00,0x00,0x49,0x03,0x00,0x00, -0x6f,0x00,0x00,0x00,0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x51,0x03,0x00,0x00,0x50,0x03,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x52,0x03,0x00,0x00,0x51,0x03,0x00,0x00, -0xc5,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x54,0x03,0x00,0x00, -0x52,0x03,0x00,0x00,0x46,0x03,0x00,0x00,0x6f,0x00,0x04,0x00, -0x4a,0x00,0x00,0x00,0x55,0x03,0x00,0x00,0x54,0x03,0x00,0x00, -0x50,0x00,0x05,0x00,0x82,0x00,0x00,0x00,0x56,0x03,0x00,0x00, -0x4f,0x03,0x00,0x00,0x55,0x03,0x00,0x00,0x83,0x00,0x05,0x00, -0x82,0x00,0x00,0x00,0x57,0x03,0x00,0x00,0x56,0x03,0x00,0x00, -0xcd,0x00,0x00,0x00,0x8e,0x00,0x05,0x00,0x82,0x00,0x00,0x00, -0x58,0x03,0x00,0x00,0x57,0x03,0x00,0x00,0x36,0x03,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x5e,0x03,0x00,0x00, -0xa9,0x00,0x00,0x00,0x74,0x04,0x00,0x00,0x51,0x00,0x05,0x00, -0x4a,0x00,0x00,0x00,0x60,0x03,0x00,0x00,0x58,0x03,0x00,0x00, -0x00,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x59,0x00,0x00,0x00, -0x61,0x03,0x00,0x00,0xa1,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x5e,0x03,0x00,0x00,0x3e,0x00,0x03,0x00,0x61,0x03,0x00,0x00, -0x60,0x03,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x68,0x03,0x00,0x00,0xa9,0x00,0x00,0x00,0x75,0x04,0x00,0x00, -0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00,0x69,0x03,0x00,0x00, -0x58,0x03,0x00,0x00,0x01,0x00,0x00,0x00,0x41,0x00,0x06,0x00, -0x59,0x00,0x00,0x00,0x6a,0x03,0x00,0x00,0xa1,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x68,0x03,0x00,0x00,0x3e,0x00,0x03,0x00, -0x6a,0x03,0x00,0x00,0x69,0x03,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4a,0x00,0x00,0x00,0x73,0x03,0x00,0x00,0x5a,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00,0x75,0x03,0x00,0x00, -0x60,0x00,0x00,0x00,0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x76,0x03,0x00,0x00,0x75,0x03,0x00,0x00,0xc4,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0x77,0x03,0x00,0x00,0x76,0x03,0x00,0x00, -0x48,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0x79,0x03,0x00,0x00,0x65,0x00,0x00,0x00,0x71,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x7a,0x03,0x00,0x00,0x79,0x03,0x00,0x00, -0xc5,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0x7b,0x03,0x00,0x00, -0x77,0x03,0x00,0x00,0x7a,0x03,0x00,0x00,0xc2,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0x7c,0x03,0x00,0x00,0x7b,0x03,0x00,0x00, -0x75,0x00,0x00,0x00,0xc4,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0x7d,0x03,0x00,0x00,0x7c,0x03,0x00,0x00,0x6f,0x00,0x00,0x00, -0xc7,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0x7e,0x03,0x00,0x00, -0x7d,0x03,0x00,0x00,0x51,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x7f,0x03,0x00,0x00,0x7e,0x03,0x00,0x00, -0xc2,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0x81,0x03,0x00,0x00, -0x7b,0x03,0x00,0x00,0x6f,0x04,0x00,0x00,0xc7,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0x82,0x03,0x00,0x00,0x81,0x03,0x00,0x00, -0x51,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x83,0x03,0x00,0x00,0x82,0x03,0x00,0x00,0x41,0x00,0x08,0x00, -0x7f,0x00,0x00,0x00,0x85,0x03,0x00,0x00,0x57,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x37,0x00,0x00,0x00, -0x75,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x50,0x00,0x00,0x00, -0x86,0x03,0x00,0x00,0x85,0x03,0x00,0x00,0x71,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x87,0x03,0x00,0x00,0x86,0x03,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x88,0x03,0x00,0x00, -0x87,0x03,0x00,0x00,0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x89,0x03,0x00,0x00,0x88,0x03,0x00,0x00,0x88,0x00,0x00,0x00, -0xc5,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8b,0x03,0x00,0x00, -0x89,0x03,0x00,0x00,0x7f,0x03,0x00,0x00,0x6f,0x00,0x04,0x00, -0x4a,0x00,0x00,0x00,0x8c,0x03,0x00,0x00,0x8b,0x03,0x00,0x00, -0xc2,0x00,0x05,0x00,0x50,0x00,0x00,0x00,0x8d,0x03,0x00,0x00, -0x86,0x03,0x00,0x00,0x6f,0x00,0x00,0x00,0x71,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x8e,0x03,0x00,0x00,0x8d,0x03,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x8f,0x03,0x00,0x00, -0x8e,0x03,0x00,0x00,0xc5,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x91,0x03,0x00,0x00,0x8f,0x03,0x00,0x00,0x83,0x03,0x00,0x00, -0x6f,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0x92,0x03,0x00,0x00, -0x91,0x03,0x00,0x00,0x50,0x00,0x05,0x00,0x82,0x00,0x00,0x00, -0x93,0x03,0x00,0x00,0x8c,0x03,0x00,0x00,0x92,0x03,0x00,0x00, -0x83,0x00,0x05,0x00,0x82,0x00,0x00,0x00,0x94,0x03,0x00,0x00, -0x93,0x03,0x00,0x00,0xcd,0x00,0x00,0x00,0x8e,0x00,0x05,0x00, -0x82,0x00,0x00,0x00,0x95,0x03,0x00,0x00,0x94,0x03,0x00,0x00, -0x73,0x03,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x9b,0x03,0x00,0x00,0xa9,0x00,0x00,0x00,0x75,0x00,0x00,0x00, -0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00,0x9d,0x03,0x00,0x00, -0x95,0x03,0x00,0x00,0x00,0x00,0x00,0x00,0x41,0x00,0x06,0x00, -0x59,0x00,0x00,0x00,0x9e,0x03,0x00,0x00,0xa1,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x9b,0x03,0x00,0x00,0x3e,0x00,0x03,0x00, -0x9e,0x03,0x00,0x00,0x9d,0x03,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xa5,0x03,0x00,0x00,0xa9,0x00,0x00,0x00, -0x76,0x04,0x00,0x00,0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00, -0xa6,0x03,0x00,0x00,0x95,0x03,0x00,0x00,0x01,0x00,0x00,0x00, -0x41,0x00,0x06,0x00,0x59,0x00,0x00,0x00,0xa7,0x03,0x00,0x00, -0xa1,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0xa5,0x03,0x00,0x00, -0x3e,0x00,0x03,0x00,0xa7,0x03,0x00,0x00,0xa6,0x03,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0xb0,0x03,0x00,0x00, -0x5a,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0xb2,0x03,0x00,0x00,0x60,0x00,0x00,0x00,0x71,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0xb3,0x03,0x00,0x00,0xb2,0x03,0x00,0x00, -0xc4,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0xb4,0x03,0x00,0x00, -0xb3,0x03,0x00,0x00,0x48,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0xb6,0x03,0x00,0x00,0x65,0x00,0x00,0x00, -0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0xb7,0x03,0x00,0x00, -0xb6,0x03,0x00,0x00,0xc5,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0xb8,0x03,0x00,0x00,0xb4,0x03,0x00,0x00,0xb7,0x03,0x00,0x00, -0xc2,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0xb9,0x03,0x00,0x00, -0xb8,0x03,0x00,0x00,0x62,0x04,0x00,0x00,0xc4,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0xba,0x03,0x00,0x00,0xb9,0x03,0x00,0x00, -0x6f,0x00,0x00,0x00,0xc7,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0xbb,0x03,0x00,0x00,0xba,0x03,0x00,0x00,0x51,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xbc,0x03,0x00,0x00, -0xbb,0x03,0x00,0x00,0xc2,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0xbe,0x03,0x00,0x00,0xb8,0x03,0x00,0x00,0x71,0x04,0x00,0x00, -0xc7,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0xbf,0x03,0x00,0x00, -0xbe,0x03,0x00,0x00,0x51,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xc0,0x03,0x00,0x00,0xbf,0x03,0x00,0x00, -0x41,0x00,0x08,0x00,0x7f,0x00,0x00,0x00,0xc2,0x03,0x00,0x00, -0x57,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x37,0x00,0x00,0x00,0x62,0x04,0x00,0x00,0x3d,0x00,0x04,0x00, -0x50,0x00,0x00,0x00,0xc3,0x03,0x00,0x00,0xc2,0x03,0x00,0x00, -0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0xc4,0x03,0x00,0x00, -0xc3,0x03,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xc5,0x03,0x00,0x00,0xc4,0x03,0x00,0x00,0xc7,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xc6,0x03,0x00,0x00,0xc5,0x03,0x00,0x00, -0x88,0x00,0x00,0x00,0xc5,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xc8,0x03,0x00,0x00,0xc6,0x03,0x00,0x00,0xbc,0x03,0x00,0x00, -0x6f,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0xc9,0x03,0x00,0x00, -0xc8,0x03,0x00,0x00,0xc2,0x00,0x05,0x00,0x50,0x00,0x00,0x00, -0xca,0x03,0x00,0x00,0xc3,0x03,0x00,0x00,0x6f,0x00,0x00,0x00, -0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0xcb,0x03,0x00,0x00, -0xca,0x03,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xcc,0x03,0x00,0x00,0xcb,0x03,0x00,0x00,0xc5,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xce,0x03,0x00,0x00,0xcc,0x03,0x00,0x00, -0xc0,0x03,0x00,0x00,0x6f,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0xcf,0x03,0x00,0x00,0xce,0x03,0x00,0x00,0x50,0x00,0x05,0x00, -0x82,0x00,0x00,0x00,0xd0,0x03,0x00,0x00,0xc9,0x03,0x00,0x00, -0xcf,0x03,0x00,0x00,0x83,0x00,0x05,0x00,0x82,0x00,0x00,0x00, -0xd1,0x03,0x00,0x00,0xd0,0x03,0x00,0x00,0xcd,0x00,0x00,0x00, -0x8e,0x00,0x05,0x00,0x82,0x00,0x00,0x00,0xd2,0x03,0x00,0x00, -0xd1,0x03,0x00,0x00,0xb0,0x03,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xd8,0x03,0x00,0x00,0xa9,0x00,0x00,0x00, -0x62,0x04,0x00,0x00,0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00, -0xda,0x03,0x00,0x00,0xd2,0x03,0x00,0x00,0x00,0x00,0x00,0x00, -0x41,0x00,0x06,0x00,0x59,0x00,0x00,0x00,0xdb,0x03,0x00,0x00, -0xa1,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0xd8,0x03,0x00,0x00, -0x3e,0x00,0x03,0x00,0xdb,0x03,0x00,0x00,0xda,0x03,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xe2,0x03,0x00,0x00, -0xa9,0x00,0x00,0x00,0x77,0x04,0x00,0x00,0x51,0x00,0x05,0x00, -0x4a,0x00,0x00,0x00,0xe3,0x03,0x00,0x00,0xd2,0x03,0x00,0x00, -0x01,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x59,0x00,0x00,0x00, -0xe4,0x03,0x00,0x00,0xa1,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0xe2,0x03,0x00,0x00,0x3e,0x00,0x03,0x00,0xe4,0x03,0x00,0x00, -0xe3,0x03,0x00,0x00,0x3d,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0xed,0x03,0x00,0x00,0x5a,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0xef,0x03,0x00,0x00,0x60,0x00,0x00,0x00, -0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0xf0,0x03,0x00,0x00, -0xef,0x03,0x00,0x00,0xc4,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0xf1,0x03,0x00,0x00,0xf0,0x03,0x00,0x00,0x48,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00,0xf3,0x03,0x00,0x00, -0x65,0x00,0x00,0x00,0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0xf4,0x03,0x00,0x00,0xf3,0x03,0x00,0x00,0xc5,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0xf5,0x03,0x00,0x00,0xf1,0x03,0x00,0x00, -0xf4,0x03,0x00,0x00,0xc2,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0xf6,0x03,0x00,0x00,0xf5,0x03,0x00,0x00,0x64,0x04,0x00,0x00, -0xc4,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0xf7,0x03,0x00,0x00, -0xf6,0x03,0x00,0x00,0x6f,0x00,0x00,0x00,0xc7,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0xf8,0x03,0x00,0x00,0xf7,0x03,0x00,0x00, -0x51,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xf9,0x03,0x00,0x00,0xf8,0x03,0x00,0x00,0xc2,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0xfb,0x03,0x00,0x00,0xf5,0x03,0x00,0x00, -0x73,0x04,0x00,0x00,0xc7,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0xfc,0x03,0x00,0x00,0xfb,0x03,0x00,0x00,0x51,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xfd,0x03,0x00,0x00, -0xfc,0x03,0x00,0x00,0x41,0x00,0x08,0x00,0x7f,0x00,0x00,0x00, -0xff,0x03,0x00,0x00,0x57,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x64,0x04,0x00,0x00, -0x3d,0x00,0x04,0x00,0x50,0x00,0x00,0x00,0x00,0x04,0x00,0x00, -0xff,0x03,0x00,0x00,0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x01,0x04,0x00,0x00,0x00,0x04,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x02,0x04,0x00,0x00,0x01,0x04,0x00,0x00, -0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x03,0x04,0x00,0x00, -0x02,0x04,0x00,0x00,0x88,0x00,0x00,0x00,0xc5,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x05,0x04,0x00,0x00,0x03,0x04,0x00,0x00, -0xf9,0x03,0x00,0x00,0x6f,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0x06,0x04,0x00,0x00,0x05,0x04,0x00,0x00,0xc2,0x00,0x05,0x00, -0x50,0x00,0x00,0x00,0x07,0x04,0x00,0x00,0x00,0x04,0x00,0x00, -0x6f,0x00,0x00,0x00,0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x08,0x04,0x00,0x00,0x07,0x04,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x09,0x04,0x00,0x00,0x08,0x04,0x00,0x00, -0xc5,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x0b,0x04,0x00,0x00, -0x09,0x04,0x00,0x00,0xfd,0x03,0x00,0x00,0x6f,0x00,0x04,0x00, -0x4a,0x00,0x00,0x00,0x0c,0x04,0x00,0x00,0x0b,0x04,0x00,0x00, -0x50,0x00,0x05,0x00,0x82,0x00,0x00,0x00,0x0d,0x04,0x00,0x00, -0x06,0x04,0x00,0x00,0x0c,0x04,0x00,0x00,0x83,0x00,0x05,0x00, -0x82,0x00,0x00,0x00,0x0e,0x04,0x00,0x00,0x0d,0x04,0x00,0x00, -0xcd,0x00,0x00,0x00,0x8e,0x00,0x05,0x00,0x82,0x00,0x00,0x00, -0x0f,0x04,0x00,0x00,0x0e,0x04,0x00,0x00,0xed,0x03,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x15,0x04,0x00,0x00, -0xa9,0x00,0x00,0x00,0x64,0x04,0x00,0x00,0x51,0x00,0x05,0x00, -0x4a,0x00,0x00,0x00,0x17,0x04,0x00,0x00,0x0f,0x04,0x00,0x00, -0x00,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x59,0x00,0x00,0x00, -0x18,0x04,0x00,0x00,0xa1,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x15,0x04,0x00,0x00,0x3e,0x00,0x03,0x00,0x18,0x04,0x00,0x00, -0x17,0x04,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x1f,0x04,0x00,0x00,0xa9,0x00,0x00,0x00,0x78,0x04,0x00,0x00, -0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00,0x20,0x04,0x00,0x00, -0x0f,0x04,0x00,0x00,0x01,0x00,0x00,0x00,0x41,0x00,0x06,0x00, -0x59,0x00,0x00,0x00,0x21,0x04,0x00,0x00,0xa1,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x1f,0x04,0x00,0x00,0x3e,0x00,0x03,0x00, -0x21,0x04,0x00,0x00,0x20,0x04,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4a,0x00,0x00,0x00,0x2a,0x04,0x00,0x00,0x5a,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00,0x2c,0x04,0x00,0x00, -0x60,0x00,0x00,0x00,0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x2d,0x04,0x00,0x00,0x2c,0x04,0x00,0x00,0xc4,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0x2e,0x04,0x00,0x00,0x2d,0x04,0x00,0x00, -0x48,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0x30,0x04,0x00,0x00,0x65,0x00,0x00,0x00,0x71,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x31,0x04,0x00,0x00,0x30,0x04,0x00,0x00, -0xc5,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0x32,0x04,0x00,0x00, -0x2e,0x04,0x00,0x00,0x31,0x04,0x00,0x00,0xc2,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0x33,0x04,0x00,0x00,0x32,0x04,0x00,0x00, -0x88,0x00,0x00,0x00,0xc4,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0x34,0x04,0x00,0x00,0x33,0x04,0x00,0x00,0x6f,0x00,0x00,0x00, -0xc7,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0x35,0x04,0x00,0x00, -0x34,0x04,0x00,0x00,0x51,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x36,0x04,0x00,0x00,0x35,0x04,0x00,0x00, -0xc2,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0x38,0x04,0x00,0x00, -0x32,0x04,0x00,0x00,0x75,0x04,0x00,0x00,0xc7,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0x39,0x04,0x00,0x00,0x38,0x04,0x00,0x00, -0x51,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x3a,0x04,0x00,0x00,0x39,0x04,0x00,0x00,0x41,0x00,0x08,0x00, -0x7f,0x00,0x00,0x00,0x3c,0x04,0x00,0x00,0x57,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x37,0x00,0x00,0x00, -0x88,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x50,0x00,0x00,0x00, -0x3d,0x04,0x00,0x00,0x3c,0x04,0x00,0x00,0x71,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x3e,0x04,0x00,0x00,0x3d,0x04,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x3f,0x04,0x00,0x00, -0x3e,0x04,0x00,0x00,0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x40,0x04,0x00,0x00,0x3f,0x04,0x00,0x00,0x88,0x00,0x00,0x00, -0xc5,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x42,0x04,0x00,0x00, -0x40,0x04,0x00,0x00,0x36,0x04,0x00,0x00,0x6f,0x00,0x04,0x00, -0x4a,0x00,0x00,0x00,0x43,0x04,0x00,0x00,0x42,0x04,0x00,0x00, -0xc2,0x00,0x05,0x00,0x50,0x00,0x00,0x00,0x44,0x04,0x00,0x00, -0x3d,0x04,0x00,0x00,0x6f,0x00,0x00,0x00,0x71,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x45,0x04,0x00,0x00,0x44,0x04,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x46,0x04,0x00,0x00, -0x45,0x04,0x00,0x00,0xc5,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x48,0x04,0x00,0x00,0x46,0x04,0x00,0x00,0x3a,0x04,0x00,0x00, -0x6f,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0x49,0x04,0x00,0x00, -0x48,0x04,0x00,0x00,0x50,0x00,0x05,0x00,0x82,0x00,0x00,0x00, -0x4a,0x04,0x00,0x00,0x43,0x04,0x00,0x00,0x49,0x04,0x00,0x00, -0x83,0x00,0x05,0x00,0x82,0x00,0x00,0x00,0x4b,0x04,0x00,0x00, -0x4a,0x04,0x00,0x00,0xcd,0x00,0x00,0x00,0x8e,0x00,0x05,0x00, -0x82,0x00,0x00,0x00,0x4c,0x04,0x00,0x00,0x4b,0x04,0x00,0x00, -0x2a,0x04,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x52,0x04,0x00,0x00,0xa9,0x00,0x00,0x00,0x88,0x00,0x00,0x00, -0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00,0x54,0x04,0x00,0x00, -0x4c,0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x41,0x00,0x06,0x00, -0x59,0x00,0x00,0x00,0x55,0x04,0x00,0x00,0xa1,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x52,0x04,0x00,0x00,0x3e,0x00,0x03,0x00, -0x55,0x04,0x00,0x00,0x54,0x04,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x5c,0x04,0x00,0x00,0xa9,0x00,0x00,0x00, -0x79,0x04,0x00,0x00,0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00, -0x5d,0x04,0x00,0x00,0x4c,0x04,0x00,0x00,0x01,0x00,0x00,0x00, -0x41,0x00,0x06,0x00,0x59,0x00,0x00,0x00,0x5e,0x04,0x00,0x00, -0xa1,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x5c,0x04,0x00,0x00, -0x3e,0x00,0x03,0x00,0x5e,0x04,0x00,0x00,0x5d,0x04,0x00,0x00, -0xf9,0x00,0x02,0x00,0xc1,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xc1,0x00,0x00,0x00,0xfd,0x00,0x01,0x00,0x38,0x00,0x01,0x00, - -}; -const uint64_t dequant_q5_0_len = 13428; - -unsigned char dequant_q5_0_fp32_data[] = { -0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, 0x9b,0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, 0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x51,0x11,0x00,0x00, 0x11,0x00,0x02,0x00,0x60,0x11,0x00,0x00,0x0b,0x00,0x06,0x00, @@ -8960,1079 +5015,10 @@ unsigned char dequant_q5_0_fp32_data[] = { 0xc3,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xc3,0x00,0x00,0x00, 0xfd,0x00,0x01,0x00,0x38,0x00,0x01,0x00, }; -const uint64_t dequant_q5_0_fp32_len = 13952; +const uint64_t dequant_q5_0_len = 13952; unsigned char dequant_q5_1_data[] = { 0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, -0x63,0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, -0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x09,0x00,0x00,0x00, -0x11,0x00,0x02,0x00,0x27,0x00,0x00,0x00,0x11,0x00,0x02,0x00, -0x51,0x11,0x00,0x00,0x11,0x00,0x02,0x00,0x60,0x11,0x00,0x00, -0x0b,0x00,0x06,0x00,0x01,0x00,0x00,0x00,0x47,0x4c,0x53,0x4c, -0x2e,0x73,0x74,0x64,0x2e,0x34,0x35,0x30,0x00,0x00,0x00,0x00, -0x0e,0x00,0x03,0x00,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x0f,0x00,0x09,0x00,0x05,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x6d,0x61,0x69,0x6e,0x00,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x54,0x00,0x00,0x00,0x9b,0x00,0x00,0x00, -0x10,0x00,0x06,0x00,0x04,0x00,0x00,0x00,0x11,0x00,0x00,0x00, -0x00,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x0c,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x1c,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x14,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x14,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x14,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x08,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x14,0x00,0x00,0x00, -0x03,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x47,0x00,0x03,0x00,0x14,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x4f,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x50,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x50,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x50,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x50,0x00,0x00,0x00, -0x03,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x08,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x51,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0x52,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x52,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x52,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x54,0x00,0x00,0x00, -0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x54,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x98,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0x99,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x99,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x99,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x9b,0x00,0x00,0x00, -0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x9b,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0xb9,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x19,0x00,0x00,0x00,0x13,0x00,0x02,0x00,0x02,0x00,0x00,0x00, -0x21,0x00,0x03,0x00,0x03,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x15,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x15,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x17,0x00,0x04,0x00, -0x0a,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x03,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x0b,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x0a,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x0b,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x0e,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x1e,0x00,0x06,0x00,0x14,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x15,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x15,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x17,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x18,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x1b,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x14,0x00,0x02,0x00,0x24,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x37,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x48,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x16,0x00,0x03,0x00, -0x4a,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x15,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, -0x10,0x00,0x00,0x00,0x1c,0x00,0x04,0x00,0x4f,0x00,0x00,0x00, -0x4d,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x1e,0x00,0x06,0x00, -0x50,0x00,0x00,0x00,0x4a,0x00,0x00,0x00,0x4a,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, -0x51,0x00,0x00,0x00,0x50,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, -0x52,0x00,0x00,0x00,0x51,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x53,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x52,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x53,0x00,0x00,0x00,0x54,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x56,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x4a,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x61,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x66,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x6e,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x77,0x00,0x00,0x00,0x03,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x79,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x4d,0x00,0x00,0x00,0x17,0x00,0x04,0x00,0x7c,0x00,0x00,0x00, -0x4a,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x82,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x8c,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x1d,0x00,0x03,0x00,0x98,0x00,0x00,0x00, -0x4a,0x00,0x00,0x00,0x1e,0x00,0x03,0x00,0x99,0x00,0x00,0x00, -0x98,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x9a,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x99,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x9a,0x00,0x00,0x00,0x9b,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0xb8,0x00,0x00,0x00, -0x00,0x01,0x00,0x00,0x2c,0x00,0x06,0x00,0x0a,0x00,0x00,0x00, -0xb9,0x00,0x00,0x00,0xb8,0x00,0x00,0x00,0x8c,0x00,0x00,0x00, -0x8c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x4b,0x04,0x00,0x00,0x0d,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x4c,0x04,0x00,0x00,0x11,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x4d,0x04,0x00,0x00, -0x0e,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x4e,0x04,0x00,0x00,0x12,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x4f,0x04,0x00,0x00,0x13,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x50,0x04,0x00,0x00, -0x14,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x51,0x04,0x00,0x00,0x05,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x52,0x04,0x00,0x00,0x15,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x53,0x04,0x00,0x00, -0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x54,0x04,0x00,0x00,0x16,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x55,0x04,0x00,0x00,0x07,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x56,0x04,0x00,0x00, -0x17,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x57,0x04,0x00,0x00,0x08,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x58,0x04,0x00,0x00,0x18,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x59,0x04,0x00,0x00, -0x09,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x5a,0x04,0x00,0x00,0x19,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x5b,0x04,0x00,0x00,0x0a,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x5c,0x04,0x00,0x00, -0x1a,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x5d,0x04,0x00,0x00,0x0b,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x5e,0x04,0x00,0x00,0x1b,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x5f,0x04,0x00,0x00, -0x1c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x60,0x04,0x00,0x00,0x1d,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x61,0x04,0x00,0x00,0x1e,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x62,0x04,0x00,0x00, -0x1f,0x00,0x00,0x00,0x36,0x00,0x05,0x00,0x02,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x03,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0x05,0x00,0x00,0x00,0xf7,0x00,0x03,0x00, -0xba,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xfb,0x00,0x03,0x00, -0x0d,0x00,0x00,0x00,0xbb,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xbb,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x0e,0x00,0x00,0x00, -0x0f,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x0d,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x10,0x00,0x00,0x00, -0x0f,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x11,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x18,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0x17,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x1a,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x87,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x1a,0x00,0x00,0x00, -0x1b,0x00,0x00,0x00,0x8b,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x1d,0x00,0x00,0x00,0x11,0x00,0x00,0x00,0x1c,0x00,0x00,0x00, -0x87,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x11,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x26,0x00,0x00,0x00,0x1d,0x00,0x00,0x00, -0x1b,0x00,0x00,0x00,0xaf,0x00,0x05,0x00,0x24,0x00,0x00,0x00, -0x29,0x00,0x00,0x00,0x26,0x00,0x00,0x00,0x1a,0x00,0x00,0x00, -0xa8,0x00,0x04,0x00,0x24,0x00,0x00,0x00,0x2a,0x00,0x00,0x00, -0x29,0x00,0x00,0x00,0xf7,0x00,0x03,0x00,0x2c,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x2a,0x00,0x00,0x00, -0x2b,0x00,0x00,0x00,0x2c,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x2b,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x18,0x00,0x00,0x00, -0x2f,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x30,0x00,0x00,0x00, -0x2f,0x00,0x00,0x00,0xaf,0x00,0x05,0x00,0x24,0x00,0x00,0x00, -0x31,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x30,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x2c,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x2c,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x24,0x00,0x00,0x00, -0x32,0x00,0x00,0x00,0x29,0x00,0x00,0x00,0xbb,0x00,0x00,0x00, -0x31,0x00,0x00,0x00,0x2b,0x00,0x00,0x00,0xf7,0x00,0x03,0x00, -0x34,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x32,0x00,0x00,0x00,0x33,0x00,0x00,0x00,0x34,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0x33,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xba,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x34,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x18,0x00,0x00,0x00,0x38,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x38,0x00,0x00,0x00, -0x87,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x1b,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x3e,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x3a,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0x3e,0x00,0x00,0x00,0x1d,0x00,0x00,0x00, -0x41,0x00,0x07,0x00,0x56,0x00,0x00,0x00,0x57,0x00,0x00,0x00, -0x54,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0x58,0x00,0x00,0x00,0x57,0x00,0x00,0x00,0x41,0x00,0x07,0x00, -0x56,0x00,0x00,0x00,0x5b,0x00,0x00,0x00,0x54,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x17,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0x5c,0x00,0x00,0x00, -0x5b,0x00,0x00,0x00,0x41,0x00,0x07,0x00,0x61,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x54,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x63,0x00,0x00,0x00,0x62,0x00,0x00,0x00, -0xc2,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0x65,0x00,0x00,0x00, -0x63,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0xc4,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0x67,0x00,0x00,0x00,0x65,0x00,0x00,0x00, -0x66,0x00,0x00,0x00,0xc7,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0x68,0x00,0x00,0x00,0x67,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x69,0x00,0x00,0x00, -0x68,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0xc2,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0x70,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x6e,0x00,0x00,0x00,0xc7,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0x71,0x00,0x00,0x00,0x70,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x72,0x00,0x00,0x00, -0x71,0x00,0x00,0x00,0x41,0x00,0x08,0x00,0x79,0x00,0x00,0x00, -0x7a,0x00,0x00,0x00,0x54,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0x77,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00,0x7b,0x00,0x00,0x00, -0x7a,0x00,0x00,0x00,0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x80,0x00,0x00,0x00,0x7b,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x81,0x00,0x00,0x00,0x80,0x00,0x00,0x00, -0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x83,0x00,0x00,0x00, -0x81,0x00,0x00,0x00,0x82,0x00,0x00,0x00,0xc5,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x83,0x00,0x00,0x00, -0x69,0x00,0x00,0x00,0x6f,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0x87,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0xc2,0x00,0x05,0x00, -0x4d,0x00,0x00,0x00,0x89,0x00,0x00,0x00,0x7b,0x00,0x00,0x00, -0x66,0x00,0x00,0x00,0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x8a,0x00,0x00,0x00,0x89,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x8b,0x00,0x00,0x00,0x8a,0x00,0x00,0x00, -0xc5,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8f,0x00,0x00,0x00, -0x8b,0x00,0x00,0x00,0x72,0x00,0x00,0x00,0x6f,0x00,0x04,0x00, -0x4a,0x00,0x00,0x00,0x90,0x00,0x00,0x00,0x8f,0x00,0x00,0x00, -0x50,0x00,0x05,0x00,0x7c,0x00,0x00,0x00,0x91,0x00,0x00,0x00, -0x87,0x00,0x00,0x00,0x90,0x00,0x00,0x00,0x8e,0x00,0x05,0x00, -0x7c,0x00,0x00,0x00,0x94,0x00,0x00,0x00,0x91,0x00,0x00,0x00, -0x58,0x00,0x00,0x00,0x50,0x00,0x05,0x00,0x7c,0x00,0x00,0x00, -0x96,0x00,0x00,0x00,0x5c,0x00,0x00,0x00,0x5c,0x00,0x00,0x00, -0x81,0x00,0x05,0x00,0x7c,0x00,0x00,0x00,0x97,0x00,0x00,0x00, -0x94,0x00,0x00,0x00,0x96,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x18,0x00,0x00,0x00,0x9d,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0x77,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x9e,0x00,0x00,0x00,0x9d,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x9f,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x9e,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xa2,0x00,0x00,0x00,0x9f,0x00,0x00,0x00,0x26,0x00,0x00,0x00, -0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00,0xa7,0x00,0x00,0x00, -0x97,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x41,0x00,0x06,0x00, -0x56,0x00,0x00,0x00,0xa8,0x00,0x00,0x00,0x9b,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0xa2,0x00,0x00,0x00,0x3e,0x00,0x03,0x00, -0xa8,0x00,0x00,0x00,0xa7,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xb2,0x00,0x00,0x00,0xa2,0x00,0x00,0x00, -0x48,0x00,0x00,0x00,0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00, -0xb4,0x00,0x00,0x00,0x97,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x41,0x00,0x06,0x00,0x56,0x00,0x00,0x00,0xb5,0x00,0x00,0x00, -0x9b,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0xb2,0x00,0x00,0x00, -0x3e,0x00,0x03,0x00,0xb5,0x00,0x00,0x00,0xb4,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0xcc,0x00,0x00,0x00, -0x57,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0xce,0x00,0x00,0x00,0x5b,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0xd0,0x00,0x00,0x00,0x62,0x00,0x00,0x00, -0xc2,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0xd1,0x00,0x00,0x00, -0xd0,0x00,0x00,0x00,0x17,0x00,0x00,0x00,0xc4,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0xd2,0x00,0x00,0x00,0xd1,0x00,0x00,0x00, -0x66,0x00,0x00,0x00,0xc7,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0xd3,0x00,0x00,0x00,0xd2,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xd4,0x00,0x00,0x00, -0xd3,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0xd6,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0xc2,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0xd8,0x00,0x00,0x00,0xd6,0x00,0x00,0x00, -0x4b,0x04,0x00,0x00,0xc7,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0xd9,0x00,0x00,0x00,0xd8,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xda,0x00,0x00,0x00, -0xd9,0x00,0x00,0x00,0x41,0x00,0x08,0x00,0x79,0x00,0x00,0x00, -0xdc,0x00,0x00,0x00,0x54,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0x77,0x00,0x00,0x00,0x17,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00,0xdd,0x00,0x00,0x00, -0xdc,0x00,0x00,0x00,0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0xde,0x00,0x00,0x00,0xdd,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xdf,0x00,0x00,0x00,0xde,0x00,0x00,0x00, -0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xe0,0x00,0x00,0x00, -0xdf,0x00,0x00,0x00,0x82,0x00,0x00,0x00,0xc5,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xe2,0x00,0x00,0x00,0xe0,0x00,0x00,0x00, -0xd4,0x00,0x00,0x00,0x6f,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0xe3,0x00,0x00,0x00,0xe2,0x00,0x00,0x00,0xc2,0x00,0x05,0x00, -0x4d,0x00,0x00,0x00,0xe4,0x00,0x00,0x00,0xdd,0x00,0x00,0x00, -0x66,0x00,0x00,0x00,0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0xe5,0x00,0x00,0x00,0xe4,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xe6,0x00,0x00,0x00,0xe5,0x00,0x00,0x00, -0xc5,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xe8,0x00,0x00,0x00, -0xe6,0x00,0x00,0x00,0xda,0x00,0x00,0x00,0x6f,0x00,0x04,0x00, -0x4a,0x00,0x00,0x00,0xe9,0x00,0x00,0x00,0xe8,0x00,0x00,0x00, -0x50,0x00,0x05,0x00,0x7c,0x00,0x00,0x00,0xea,0x00,0x00,0x00, -0xe3,0x00,0x00,0x00,0xe9,0x00,0x00,0x00,0x8e,0x00,0x05,0x00, -0x7c,0x00,0x00,0x00,0xeb,0x00,0x00,0x00,0xea,0x00,0x00,0x00, -0xcc,0x00,0x00,0x00,0x50,0x00,0x05,0x00,0x7c,0x00,0x00,0x00, -0xec,0x00,0x00,0x00,0xce,0x00,0x00,0x00,0xce,0x00,0x00,0x00, -0x81,0x00,0x05,0x00,0x7c,0x00,0x00,0x00,0xed,0x00,0x00,0x00, -0xeb,0x00,0x00,0x00,0xec,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xf3,0x00,0x00,0x00,0xa2,0x00,0x00,0x00, -0x17,0x00,0x00,0x00,0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00, -0xf5,0x00,0x00,0x00,0xed,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x41,0x00,0x06,0x00,0x56,0x00,0x00,0x00,0xf6,0x00,0x00,0x00, -0x9b,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0xf3,0x00,0x00,0x00, -0x3e,0x00,0x03,0x00,0xf6,0x00,0x00,0x00,0xf5,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xfd,0x00,0x00,0x00, -0xa2,0x00,0x00,0x00,0x4c,0x04,0x00,0x00,0x51,0x00,0x05,0x00, -0x4a,0x00,0x00,0x00,0xfe,0x00,0x00,0x00,0xed,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x56,0x00,0x00,0x00, -0xff,0x00,0x00,0x00,0x9b,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0xfd,0x00,0x00,0x00,0x3e,0x00,0x03,0x00,0xff,0x00,0x00,0x00, -0xfe,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0x08,0x01,0x00,0x00,0x57,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4a,0x00,0x00,0x00,0x0a,0x01,0x00,0x00,0x5b,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x0c,0x01,0x00,0x00, -0x62,0x00,0x00,0x00,0xc2,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0x0d,0x01,0x00,0x00,0x0c,0x01,0x00,0x00,0x37,0x00,0x00,0x00, -0xc4,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0x0e,0x01,0x00,0x00, -0x0d,0x01,0x00,0x00,0x66,0x00,0x00,0x00,0xc7,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0x0f,0x01,0x00,0x00,0x0e,0x01,0x00,0x00, -0x4e,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x10,0x01,0x00,0x00,0x0f,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x12,0x01,0x00,0x00,0x62,0x00,0x00,0x00, -0xc2,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0x14,0x01,0x00,0x00, -0x12,0x01,0x00,0x00,0x4d,0x04,0x00,0x00,0xc7,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0x15,0x01,0x00,0x00,0x14,0x01,0x00,0x00, -0x4e,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x16,0x01,0x00,0x00,0x15,0x01,0x00,0x00,0x41,0x00,0x08,0x00, -0x79,0x00,0x00,0x00,0x18,0x01,0x00,0x00,0x54,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x77,0x00,0x00,0x00, -0x37,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0x19,0x01,0x00,0x00,0x18,0x01,0x00,0x00,0x71,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x1a,0x01,0x00,0x00,0x19,0x01,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x1b,0x01,0x00,0x00, -0x1a,0x01,0x00,0x00,0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x1c,0x01,0x00,0x00,0x1b,0x01,0x00,0x00,0x82,0x00,0x00,0x00, -0xc5,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x1e,0x01,0x00,0x00, -0x1c,0x01,0x00,0x00,0x10,0x01,0x00,0x00,0x6f,0x00,0x04,0x00, -0x4a,0x00,0x00,0x00,0x1f,0x01,0x00,0x00,0x1e,0x01,0x00,0x00, -0xc2,0x00,0x05,0x00,0x4d,0x00,0x00,0x00,0x20,0x01,0x00,0x00, -0x19,0x01,0x00,0x00,0x66,0x00,0x00,0x00,0x71,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x21,0x01,0x00,0x00,0x20,0x01,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x22,0x01,0x00,0x00, -0x21,0x01,0x00,0x00,0xc5,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x24,0x01,0x00,0x00,0x22,0x01,0x00,0x00,0x16,0x01,0x00,0x00, -0x6f,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0x25,0x01,0x00,0x00, -0x24,0x01,0x00,0x00,0x50,0x00,0x05,0x00,0x7c,0x00,0x00,0x00, -0x26,0x01,0x00,0x00,0x1f,0x01,0x00,0x00,0x25,0x01,0x00,0x00, -0x8e,0x00,0x05,0x00,0x7c,0x00,0x00,0x00,0x27,0x01,0x00,0x00, -0x26,0x01,0x00,0x00,0x08,0x01,0x00,0x00,0x50,0x00,0x05,0x00, -0x7c,0x00,0x00,0x00,0x28,0x01,0x00,0x00,0x0a,0x01,0x00,0x00, -0x0a,0x01,0x00,0x00,0x81,0x00,0x05,0x00,0x7c,0x00,0x00,0x00, -0x29,0x01,0x00,0x00,0x27,0x01,0x00,0x00,0x28,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x2f,0x01,0x00,0x00, -0xa2,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x51,0x00,0x05,0x00, -0x4a,0x00,0x00,0x00,0x31,0x01,0x00,0x00,0x29,0x01,0x00,0x00, -0x00,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x56,0x00,0x00,0x00, -0x32,0x01,0x00,0x00,0x9b,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x2f,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x32,0x01,0x00,0x00, -0x31,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x39,0x01,0x00,0x00,0xa2,0x00,0x00,0x00,0x4e,0x04,0x00,0x00, -0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00,0x3a,0x01,0x00,0x00, -0x29,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x41,0x00,0x06,0x00, -0x56,0x00,0x00,0x00,0x3b,0x01,0x00,0x00,0x9b,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x39,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0x3b,0x01,0x00,0x00,0x3a,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4a,0x00,0x00,0x00,0x44,0x01,0x00,0x00,0x57,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0x46,0x01,0x00,0x00, -0x5b,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x48,0x01,0x00,0x00,0x62,0x00,0x00,0x00,0xc2,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0x49,0x01,0x00,0x00,0x48,0x01,0x00,0x00, -0x77,0x00,0x00,0x00,0xc4,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0x4a,0x01,0x00,0x00,0x49,0x01,0x00,0x00,0x66,0x00,0x00,0x00, -0xc7,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0x4b,0x01,0x00,0x00, -0x4a,0x01,0x00,0x00,0x4e,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x4c,0x01,0x00,0x00,0x4b,0x01,0x00,0x00, -0x3d,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x4e,0x01,0x00,0x00, -0x62,0x00,0x00,0x00,0xc2,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0x50,0x01,0x00,0x00,0x4e,0x01,0x00,0x00,0x82,0x00,0x00,0x00, -0xc7,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0x51,0x01,0x00,0x00, -0x50,0x01,0x00,0x00,0x4e,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x52,0x01,0x00,0x00,0x51,0x01,0x00,0x00, -0x41,0x00,0x08,0x00,0x79,0x00,0x00,0x00,0x54,0x01,0x00,0x00, -0x54,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x77,0x00,0x00,0x00,0x77,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0x55,0x01,0x00,0x00,0x54,0x01,0x00,0x00, -0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x56,0x01,0x00,0x00, -0x55,0x01,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x57,0x01,0x00,0x00,0x56,0x01,0x00,0x00,0xc7,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x58,0x01,0x00,0x00,0x57,0x01,0x00,0x00, -0x82,0x00,0x00,0x00,0xc5,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x5a,0x01,0x00,0x00,0x58,0x01,0x00,0x00,0x4c,0x01,0x00,0x00, -0x6f,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0x5b,0x01,0x00,0x00, -0x5a,0x01,0x00,0x00,0xc2,0x00,0x05,0x00,0x4d,0x00,0x00,0x00, -0x5c,0x01,0x00,0x00,0x55,0x01,0x00,0x00,0x66,0x00,0x00,0x00, -0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x5d,0x01,0x00,0x00, -0x5c,0x01,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x5e,0x01,0x00,0x00,0x5d,0x01,0x00,0x00,0xc5,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x60,0x01,0x00,0x00,0x5e,0x01,0x00,0x00, -0x52,0x01,0x00,0x00,0x6f,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0x61,0x01,0x00,0x00,0x60,0x01,0x00,0x00,0x50,0x00,0x05,0x00, -0x7c,0x00,0x00,0x00,0x62,0x01,0x00,0x00,0x5b,0x01,0x00,0x00, -0x61,0x01,0x00,0x00,0x8e,0x00,0x05,0x00,0x7c,0x00,0x00,0x00, -0x63,0x01,0x00,0x00,0x62,0x01,0x00,0x00,0x44,0x01,0x00,0x00, -0x50,0x00,0x05,0x00,0x7c,0x00,0x00,0x00,0x64,0x01,0x00,0x00, -0x46,0x01,0x00,0x00,0x46,0x01,0x00,0x00,0x81,0x00,0x05,0x00, -0x7c,0x00,0x00,0x00,0x65,0x01,0x00,0x00,0x63,0x01,0x00,0x00, -0x64,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x6b,0x01,0x00,0x00,0xa2,0x00,0x00,0x00,0x77,0x00,0x00,0x00, -0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00,0x6d,0x01,0x00,0x00, -0x65,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x41,0x00,0x06,0x00, -0x56,0x00,0x00,0x00,0x6e,0x01,0x00,0x00,0x9b,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x6b,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0x6e,0x01,0x00,0x00,0x6d,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x75,0x01,0x00,0x00,0xa2,0x00,0x00,0x00, -0x4f,0x04,0x00,0x00,0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00, -0x76,0x01,0x00,0x00,0x65,0x01,0x00,0x00,0x01,0x00,0x00,0x00, -0x41,0x00,0x06,0x00,0x56,0x00,0x00,0x00,0x77,0x01,0x00,0x00, -0x9b,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x75,0x01,0x00,0x00, -0x3e,0x00,0x03,0x00,0x77,0x01,0x00,0x00,0x76,0x01,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0x80,0x01,0x00,0x00, -0x57,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0x82,0x01,0x00,0x00,0x5b,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x84,0x01,0x00,0x00,0x62,0x00,0x00,0x00, -0xc2,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0x85,0x01,0x00,0x00, -0x84,0x01,0x00,0x00,0x66,0x00,0x00,0x00,0xc4,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0x86,0x01,0x00,0x00,0x85,0x01,0x00,0x00, -0x66,0x00,0x00,0x00,0xc7,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0x87,0x01,0x00,0x00,0x86,0x01,0x00,0x00,0x4e,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x88,0x01,0x00,0x00, -0x87,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x8a,0x01,0x00,0x00,0x62,0x00,0x00,0x00,0xc2,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0x8c,0x01,0x00,0x00,0x8a,0x01,0x00,0x00, -0x48,0x00,0x00,0x00,0xc7,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0x8d,0x01,0x00,0x00,0x8c,0x01,0x00,0x00,0x4e,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x8e,0x01,0x00,0x00, -0x8d,0x01,0x00,0x00,0x41,0x00,0x08,0x00,0x79,0x00,0x00,0x00, -0x90,0x01,0x00,0x00,0x54,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0x77,0x00,0x00,0x00,0x66,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00,0x91,0x01,0x00,0x00, -0x90,0x01,0x00,0x00,0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x92,0x01,0x00,0x00,0x91,0x01,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x93,0x01,0x00,0x00,0x92,0x01,0x00,0x00, -0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x94,0x01,0x00,0x00, -0x93,0x01,0x00,0x00,0x82,0x00,0x00,0x00,0xc5,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x96,0x01,0x00,0x00,0x94,0x01,0x00,0x00, -0x88,0x01,0x00,0x00,0x6f,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0x97,0x01,0x00,0x00,0x96,0x01,0x00,0x00,0xc2,0x00,0x05,0x00, -0x4d,0x00,0x00,0x00,0x98,0x01,0x00,0x00,0x91,0x01,0x00,0x00, -0x66,0x00,0x00,0x00,0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x99,0x01,0x00,0x00,0x98,0x01,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x9a,0x01,0x00,0x00,0x99,0x01,0x00,0x00, -0xc5,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9c,0x01,0x00,0x00, -0x9a,0x01,0x00,0x00,0x8e,0x01,0x00,0x00,0x6f,0x00,0x04,0x00, -0x4a,0x00,0x00,0x00,0x9d,0x01,0x00,0x00,0x9c,0x01,0x00,0x00, -0x50,0x00,0x05,0x00,0x7c,0x00,0x00,0x00,0x9e,0x01,0x00,0x00, -0x97,0x01,0x00,0x00,0x9d,0x01,0x00,0x00,0x8e,0x00,0x05,0x00, -0x7c,0x00,0x00,0x00,0x9f,0x01,0x00,0x00,0x9e,0x01,0x00,0x00, -0x80,0x01,0x00,0x00,0x50,0x00,0x05,0x00,0x7c,0x00,0x00,0x00, -0xa0,0x01,0x00,0x00,0x82,0x01,0x00,0x00,0x82,0x01,0x00,0x00, -0x81,0x00,0x05,0x00,0x7c,0x00,0x00,0x00,0xa1,0x01,0x00,0x00, -0x9f,0x01,0x00,0x00,0xa0,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xa7,0x01,0x00,0x00,0xa2,0x00,0x00,0x00, -0x66,0x00,0x00,0x00,0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00, -0xa9,0x01,0x00,0x00,0xa1,0x01,0x00,0x00,0x00,0x00,0x00,0x00, -0x41,0x00,0x06,0x00,0x56,0x00,0x00,0x00,0xaa,0x01,0x00,0x00, -0x9b,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0xa7,0x01,0x00,0x00, -0x3e,0x00,0x03,0x00,0xaa,0x01,0x00,0x00,0xa9,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xb1,0x01,0x00,0x00, -0xa2,0x00,0x00,0x00,0x50,0x04,0x00,0x00,0x51,0x00,0x05,0x00, -0x4a,0x00,0x00,0x00,0xb2,0x01,0x00,0x00,0xa1,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x56,0x00,0x00,0x00, -0xb3,0x01,0x00,0x00,0x9b,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0xb1,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0xb3,0x01,0x00,0x00, -0xb2,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0xbc,0x01,0x00,0x00,0x57,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4a,0x00,0x00,0x00,0xbe,0x01,0x00,0x00,0x5b,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0xc0,0x01,0x00,0x00, -0x62,0x00,0x00,0x00,0xc2,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0xc1,0x01,0x00,0x00,0xc0,0x01,0x00,0x00,0x51,0x04,0x00,0x00, -0xc4,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0xc2,0x01,0x00,0x00, -0xc1,0x01,0x00,0x00,0x66,0x00,0x00,0x00,0xc7,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0xc3,0x01,0x00,0x00,0xc2,0x01,0x00,0x00, -0x4e,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xc4,0x01,0x00,0x00,0xc3,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0xc6,0x01,0x00,0x00,0x62,0x00,0x00,0x00, -0xc2,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0xc8,0x01,0x00,0x00, -0xc6,0x01,0x00,0x00,0x4c,0x04,0x00,0x00,0xc7,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0xc9,0x01,0x00,0x00,0xc8,0x01,0x00,0x00, -0x4e,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xca,0x01,0x00,0x00,0xc9,0x01,0x00,0x00,0x41,0x00,0x08,0x00, -0x79,0x00,0x00,0x00,0xcc,0x01,0x00,0x00,0x54,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x77,0x00,0x00,0x00, -0x51,0x04,0x00,0x00,0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0xcd,0x01,0x00,0x00,0xcc,0x01,0x00,0x00,0x71,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0xce,0x01,0x00,0x00,0xcd,0x01,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xcf,0x01,0x00,0x00, -0xce,0x01,0x00,0x00,0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xd0,0x01,0x00,0x00,0xcf,0x01,0x00,0x00,0x82,0x00,0x00,0x00, -0xc5,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xd2,0x01,0x00,0x00, -0xd0,0x01,0x00,0x00,0xc4,0x01,0x00,0x00,0x6f,0x00,0x04,0x00, -0x4a,0x00,0x00,0x00,0xd3,0x01,0x00,0x00,0xd2,0x01,0x00,0x00, -0xc2,0x00,0x05,0x00,0x4d,0x00,0x00,0x00,0xd4,0x01,0x00,0x00, -0xcd,0x01,0x00,0x00,0x66,0x00,0x00,0x00,0x71,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0xd5,0x01,0x00,0x00,0xd4,0x01,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xd6,0x01,0x00,0x00, -0xd5,0x01,0x00,0x00,0xc5,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xd8,0x01,0x00,0x00,0xd6,0x01,0x00,0x00,0xca,0x01,0x00,0x00, -0x6f,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0xd9,0x01,0x00,0x00, -0xd8,0x01,0x00,0x00,0x50,0x00,0x05,0x00,0x7c,0x00,0x00,0x00, -0xda,0x01,0x00,0x00,0xd3,0x01,0x00,0x00,0xd9,0x01,0x00,0x00, -0x8e,0x00,0x05,0x00,0x7c,0x00,0x00,0x00,0xdb,0x01,0x00,0x00, -0xda,0x01,0x00,0x00,0xbc,0x01,0x00,0x00,0x50,0x00,0x05,0x00, -0x7c,0x00,0x00,0x00,0xdc,0x01,0x00,0x00,0xbe,0x01,0x00,0x00, -0xbe,0x01,0x00,0x00,0x81,0x00,0x05,0x00,0x7c,0x00,0x00,0x00, -0xdd,0x01,0x00,0x00,0xdb,0x01,0x00,0x00,0xdc,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xe3,0x01,0x00,0x00, -0xa2,0x00,0x00,0x00,0x51,0x04,0x00,0x00,0x51,0x00,0x05,0x00, -0x4a,0x00,0x00,0x00,0xe5,0x01,0x00,0x00,0xdd,0x01,0x00,0x00, -0x00,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x56,0x00,0x00,0x00, -0xe6,0x01,0x00,0x00,0x9b,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0xe3,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0xe6,0x01,0x00,0x00, -0xe5,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xed,0x01,0x00,0x00,0xa2,0x00,0x00,0x00,0x52,0x04,0x00,0x00, -0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00,0xee,0x01,0x00,0x00, -0xdd,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x41,0x00,0x06,0x00, -0x56,0x00,0x00,0x00,0xef,0x01,0x00,0x00,0x9b,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0xed,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0xef,0x01,0x00,0x00,0xee,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4a,0x00,0x00,0x00,0xf8,0x01,0x00,0x00,0x57,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0xfa,0x01,0x00,0x00, -0x5b,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0xfc,0x01,0x00,0x00,0x62,0x00,0x00,0x00,0xc2,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0xfd,0x01,0x00,0x00,0xfc,0x01,0x00,0x00, -0x53,0x04,0x00,0x00,0xc4,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0xfe,0x01,0x00,0x00,0xfd,0x01,0x00,0x00,0x66,0x00,0x00,0x00, -0xc7,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0xff,0x01,0x00,0x00, -0xfe,0x01,0x00,0x00,0x4e,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x00,0x02,0x00,0x00,0xff,0x01,0x00,0x00, -0x3d,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x02,0x02,0x00,0x00, -0x62,0x00,0x00,0x00,0xc2,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0x04,0x02,0x00,0x00,0x02,0x02,0x00,0x00,0x4e,0x04,0x00,0x00, -0xc7,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0x05,0x02,0x00,0x00, -0x04,0x02,0x00,0x00,0x4e,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x06,0x02,0x00,0x00,0x05,0x02,0x00,0x00, -0x41,0x00,0x08,0x00,0x79,0x00,0x00,0x00,0x08,0x02,0x00,0x00, -0x54,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x77,0x00,0x00,0x00,0x53,0x04,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0x09,0x02,0x00,0x00,0x08,0x02,0x00,0x00, -0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x0a,0x02,0x00,0x00, -0x09,0x02,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x0b,0x02,0x00,0x00,0x0a,0x02,0x00,0x00,0xc7,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x0c,0x02,0x00,0x00,0x0b,0x02,0x00,0x00, -0x82,0x00,0x00,0x00,0xc5,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x0e,0x02,0x00,0x00,0x0c,0x02,0x00,0x00,0x00,0x02,0x00,0x00, -0x6f,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0x0f,0x02,0x00,0x00, -0x0e,0x02,0x00,0x00,0xc2,0x00,0x05,0x00,0x4d,0x00,0x00,0x00, -0x10,0x02,0x00,0x00,0x09,0x02,0x00,0x00,0x66,0x00,0x00,0x00, -0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x11,0x02,0x00,0x00, -0x10,0x02,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x12,0x02,0x00,0x00,0x11,0x02,0x00,0x00,0xc5,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x14,0x02,0x00,0x00,0x12,0x02,0x00,0x00, -0x06,0x02,0x00,0x00,0x6f,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0x15,0x02,0x00,0x00,0x14,0x02,0x00,0x00,0x50,0x00,0x05,0x00, -0x7c,0x00,0x00,0x00,0x16,0x02,0x00,0x00,0x0f,0x02,0x00,0x00, -0x15,0x02,0x00,0x00,0x8e,0x00,0x05,0x00,0x7c,0x00,0x00,0x00, -0x17,0x02,0x00,0x00,0x16,0x02,0x00,0x00,0xf8,0x01,0x00,0x00, -0x50,0x00,0x05,0x00,0x7c,0x00,0x00,0x00,0x18,0x02,0x00,0x00, -0xfa,0x01,0x00,0x00,0xfa,0x01,0x00,0x00,0x81,0x00,0x05,0x00, -0x7c,0x00,0x00,0x00,0x19,0x02,0x00,0x00,0x17,0x02,0x00,0x00, -0x18,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x1f,0x02,0x00,0x00,0xa2,0x00,0x00,0x00,0x53,0x04,0x00,0x00, -0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00,0x21,0x02,0x00,0x00, -0x19,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x41,0x00,0x06,0x00, -0x56,0x00,0x00,0x00,0x22,0x02,0x00,0x00,0x9b,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x1f,0x02,0x00,0x00,0x3e,0x00,0x03,0x00, -0x22,0x02,0x00,0x00,0x21,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x29,0x02,0x00,0x00,0xa2,0x00,0x00,0x00, -0x54,0x04,0x00,0x00,0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00, -0x2a,0x02,0x00,0x00,0x19,0x02,0x00,0x00,0x01,0x00,0x00,0x00, -0x41,0x00,0x06,0x00,0x56,0x00,0x00,0x00,0x2b,0x02,0x00,0x00, -0x9b,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x29,0x02,0x00,0x00, -0x3e,0x00,0x03,0x00,0x2b,0x02,0x00,0x00,0x2a,0x02,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0x34,0x02,0x00,0x00, -0x57,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0x36,0x02,0x00,0x00,0x5b,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x38,0x02,0x00,0x00,0x62,0x00,0x00,0x00, -0xc2,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0x39,0x02,0x00,0x00, -0x38,0x02,0x00,0x00,0x55,0x04,0x00,0x00,0xc4,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0x3a,0x02,0x00,0x00,0x39,0x02,0x00,0x00, -0x66,0x00,0x00,0x00,0xc7,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0x3b,0x02,0x00,0x00,0x3a,0x02,0x00,0x00,0x4e,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x3c,0x02,0x00,0x00, -0x3b,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x3e,0x02,0x00,0x00,0x62,0x00,0x00,0x00,0xc2,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0x40,0x02,0x00,0x00,0x3e,0x02,0x00,0x00, -0x4f,0x04,0x00,0x00,0xc7,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0x41,0x02,0x00,0x00,0x40,0x02,0x00,0x00,0x4e,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x42,0x02,0x00,0x00, -0x41,0x02,0x00,0x00,0x41,0x00,0x08,0x00,0x79,0x00,0x00,0x00, -0x44,0x02,0x00,0x00,0x54,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0x77,0x00,0x00,0x00,0x55,0x04,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00,0x45,0x02,0x00,0x00, -0x44,0x02,0x00,0x00,0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x46,0x02,0x00,0x00,0x45,0x02,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x47,0x02,0x00,0x00,0x46,0x02,0x00,0x00, -0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x48,0x02,0x00,0x00, -0x47,0x02,0x00,0x00,0x82,0x00,0x00,0x00,0xc5,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x4a,0x02,0x00,0x00,0x48,0x02,0x00,0x00, -0x3c,0x02,0x00,0x00,0x6f,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0x4b,0x02,0x00,0x00,0x4a,0x02,0x00,0x00,0xc2,0x00,0x05,0x00, -0x4d,0x00,0x00,0x00,0x4c,0x02,0x00,0x00,0x45,0x02,0x00,0x00, -0x66,0x00,0x00,0x00,0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x4d,0x02,0x00,0x00,0x4c,0x02,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x4e,0x02,0x00,0x00,0x4d,0x02,0x00,0x00, -0xc5,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x50,0x02,0x00,0x00, -0x4e,0x02,0x00,0x00,0x42,0x02,0x00,0x00,0x6f,0x00,0x04,0x00, -0x4a,0x00,0x00,0x00,0x51,0x02,0x00,0x00,0x50,0x02,0x00,0x00, -0x50,0x00,0x05,0x00,0x7c,0x00,0x00,0x00,0x52,0x02,0x00,0x00, -0x4b,0x02,0x00,0x00,0x51,0x02,0x00,0x00,0x8e,0x00,0x05,0x00, -0x7c,0x00,0x00,0x00,0x53,0x02,0x00,0x00,0x52,0x02,0x00,0x00, -0x34,0x02,0x00,0x00,0x50,0x00,0x05,0x00,0x7c,0x00,0x00,0x00, -0x54,0x02,0x00,0x00,0x36,0x02,0x00,0x00,0x36,0x02,0x00,0x00, -0x81,0x00,0x05,0x00,0x7c,0x00,0x00,0x00,0x55,0x02,0x00,0x00, -0x53,0x02,0x00,0x00,0x54,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x5b,0x02,0x00,0x00,0xa2,0x00,0x00,0x00, -0x55,0x04,0x00,0x00,0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00, -0x5d,0x02,0x00,0x00,0x55,0x02,0x00,0x00,0x00,0x00,0x00,0x00, -0x41,0x00,0x06,0x00,0x56,0x00,0x00,0x00,0x5e,0x02,0x00,0x00, -0x9b,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x5b,0x02,0x00,0x00, -0x3e,0x00,0x03,0x00,0x5e,0x02,0x00,0x00,0x5d,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x65,0x02,0x00,0x00, -0xa2,0x00,0x00,0x00,0x56,0x04,0x00,0x00,0x51,0x00,0x05,0x00, -0x4a,0x00,0x00,0x00,0x66,0x02,0x00,0x00,0x55,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x56,0x00,0x00,0x00, -0x67,0x02,0x00,0x00,0x9b,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x65,0x02,0x00,0x00,0x3e,0x00,0x03,0x00,0x67,0x02,0x00,0x00, -0x66,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0x70,0x02,0x00,0x00,0x57,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4a,0x00,0x00,0x00,0x72,0x02,0x00,0x00,0x5b,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x74,0x02,0x00,0x00, -0x62,0x00,0x00,0x00,0xc2,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0x75,0x02,0x00,0x00,0x74,0x02,0x00,0x00,0x57,0x04,0x00,0x00, -0xc4,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0x76,0x02,0x00,0x00, -0x75,0x02,0x00,0x00,0x66,0x00,0x00,0x00,0xc7,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0x77,0x02,0x00,0x00,0x76,0x02,0x00,0x00, -0x4e,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x78,0x02,0x00,0x00,0x77,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x7a,0x02,0x00,0x00,0x62,0x00,0x00,0x00, -0xc2,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0x7c,0x02,0x00,0x00, -0x7a,0x02,0x00,0x00,0x50,0x04,0x00,0x00,0xc7,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0x7d,0x02,0x00,0x00,0x7c,0x02,0x00,0x00, -0x4e,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x7e,0x02,0x00,0x00,0x7d,0x02,0x00,0x00,0x41,0x00,0x08,0x00, -0x79,0x00,0x00,0x00,0x80,0x02,0x00,0x00,0x54,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x77,0x00,0x00,0x00, -0x57,0x04,0x00,0x00,0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0x81,0x02,0x00,0x00,0x80,0x02,0x00,0x00,0x71,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x82,0x02,0x00,0x00,0x81,0x02,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x83,0x02,0x00,0x00, -0x82,0x02,0x00,0x00,0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x84,0x02,0x00,0x00,0x83,0x02,0x00,0x00,0x82,0x00,0x00,0x00, -0xc5,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x86,0x02,0x00,0x00, -0x84,0x02,0x00,0x00,0x78,0x02,0x00,0x00,0x6f,0x00,0x04,0x00, -0x4a,0x00,0x00,0x00,0x87,0x02,0x00,0x00,0x86,0x02,0x00,0x00, -0xc2,0x00,0x05,0x00,0x4d,0x00,0x00,0x00,0x88,0x02,0x00,0x00, -0x81,0x02,0x00,0x00,0x66,0x00,0x00,0x00,0x71,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x89,0x02,0x00,0x00,0x88,0x02,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x8a,0x02,0x00,0x00, -0x89,0x02,0x00,0x00,0xc5,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x8c,0x02,0x00,0x00,0x8a,0x02,0x00,0x00,0x7e,0x02,0x00,0x00, -0x6f,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0x8d,0x02,0x00,0x00, -0x8c,0x02,0x00,0x00,0x50,0x00,0x05,0x00,0x7c,0x00,0x00,0x00, -0x8e,0x02,0x00,0x00,0x87,0x02,0x00,0x00,0x8d,0x02,0x00,0x00, -0x8e,0x00,0x05,0x00,0x7c,0x00,0x00,0x00,0x8f,0x02,0x00,0x00, -0x8e,0x02,0x00,0x00,0x70,0x02,0x00,0x00,0x50,0x00,0x05,0x00, -0x7c,0x00,0x00,0x00,0x90,0x02,0x00,0x00,0x72,0x02,0x00,0x00, -0x72,0x02,0x00,0x00,0x81,0x00,0x05,0x00,0x7c,0x00,0x00,0x00, -0x91,0x02,0x00,0x00,0x8f,0x02,0x00,0x00,0x90,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x97,0x02,0x00,0x00, -0xa2,0x00,0x00,0x00,0x57,0x04,0x00,0x00,0x51,0x00,0x05,0x00, -0x4a,0x00,0x00,0x00,0x99,0x02,0x00,0x00,0x91,0x02,0x00,0x00, -0x00,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x56,0x00,0x00,0x00, -0x9a,0x02,0x00,0x00,0x9b,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x97,0x02,0x00,0x00,0x3e,0x00,0x03,0x00,0x9a,0x02,0x00,0x00, -0x99,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xa1,0x02,0x00,0x00,0xa2,0x00,0x00,0x00,0x58,0x04,0x00,0x00, -0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00,0xa2,0x02,0x00,0x00, -0x91,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0x41,0x00,0x06,0x00, -0x56,0x00,0x00,0x00,0xa3,0x02,0x00,0x00,0x9b,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0xa1,0x02,0x00,0x00,0x3e,0x00,0x03,0x00, -0xa3,0x02,0x00,0x00,0xa2,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4a,0x00,0x00,0x00,0xac,0x02,0x00,0x00,0x57,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0xae,0x02,0x00,0x00, -0x5b,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0xb0,0x02,0x00,0x00,0x62,0x00,0x00,0x00,0xc2,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0xb1,0x02,0x00,0x00,0xb0,0x02,0x00,0x00, -0x59,0x04,0x00,0x00,0xc4,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0xb2,0x02,0x00,0x00,0xb1,0x02,0x00,0x00,0x66,0x00,0x00,0x00, -0xc7,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0xb3,0x02,0x00,0x00, -0xb2,0x02,0x00,0x00,0x4e,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xb4,0x02,0x00,0x00,0xb3,0x02,0x00,0x00, -0x3d,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0xb6,0x02,0x00,0x00, -0x62,0x00,0x00,0x00,0xc2,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0xb8,0x02,0x00,0x00,0xb6,0x02,0x00,0x00,0x52,0x04,0x00,0x00, -0xc7,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0xb9,0x02,0x00,0x00, -0xb8,0x02,0x00,0x00,0x4e,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xba,0x02,0x00,0x00,0xb9,0x02,0x00,0x00, -0x41,0x00,0x08,0x00,0x79,0x00,0x00,0x00,0xbc,0x02,0x00,0x00, -0x54,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x77,0x00,0x00,0x00,0x59,0x04,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0xbd,0x02,0x00,0x00,0xbc,0x02,0x00,0x00, -0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0xbe,0x02,0x00,0x00, -0xbd,0x02,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xbf,0x02,0x00,0x00,0xbe,0x02,0x00,0x00,0xc7,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xc0,0x02,0x00,0x00,0xbf,0x02,0x00,0x00, -0x82,0x00,0x00,0x00,0xc5,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xc2,0x02,0x00,0x00,0xc0,0x02,0x00,0x00,0xb4,0x02,0x00,0x00, -0x6f,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0xc3,0x02,0x00,0x00, -0xc2,0x02,0x00,0x00,0xc2,0x00,0x05,0x00,0x4d,0x00,0x00,0x00, -0xc4,0x02,0x00,0x00,0xbd,0x02,0x00,0x00,0x66,0x00,0x00,0x00, -0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0xc5,0x02,0x00,0x00, -0xc4,0x02,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xc6,0x02,0x00,0x00,0xc5,0x02,0x00,0x00,0xc5,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xc8,0x02,0x00,0x00,0xc6,0x02,0x00,0x00, -0xba,0x02,0x00,0x00,0x6f,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0xc9,0x02,0x00,0x00,0xc8,0x02,0x00,0x00,0x50,0x00,0x05,0x00, -0x7c,0x00,0x00,0x00,0xca,0x02,0x00,0x00,0xc3,0x02,0x00,0x00, -0xc9,0x02,0x00,0x00,0x8e,0x00,0x05,0x00,0x7c,0x00,0x00,0x00, -0xcb,0x02,0x00,0x00,0xca,0x02,0x00,0x00,0xac,0x02,0x00,0x00, -0x50,0x00,0x05,0x00,0x7c,0x00,0x00,0x00,0xcc,0x02,0x00,0x00, -0xae,0x02,0x00,0x00,0xae,0x02,0x00,0x00,0x81,0x00,0x05,0x00, -0x7c,0x00,0x00,0x00,0xcd,0x02,0x00,0x00,0xcb,0x02,0x00,0x00, -0xcc,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xd3,0x02,0x00,0x00,0xa2,0x00,0x00,0x00,0x59,0x04,0x00,0x00, -0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00,0xd5,0x02,0x00,0x00, -0xcd,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x41,0x00,0x06,0x00, -0x56,0x00,0x00,0x00,0xd6,0x02,0x00,0x00,0x9b,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0xd3,0x02,0x00,0x00,0x3e,0x00,0x03,0x00, -0xd6,0x02,0x00,0x00,0xd5,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xdd,0x02,0x00,0x00,0xa2,0x00,0x00,0x00, -0x5a,0x04,0x00,0x00,0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00, -0xde,0x02,0x00,0x00,0xcd,0x02,0x00,0x00,0x01,0x00,0x00,0x00, -0x41,0x00,0x06,0x00,0x56,0x00,0x00,0x00,0xdf,0x02,0x00,0x00, -0x9b,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0xdd,0x02,0x00,0x00, -0x3e,0x00,0x03,0x00,0xdf,0x02,0x00,0x00,0xde,0x02,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0xe8,0x02,0x00,0x00, -0x57,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0xea,0x02,0x00,0x00,0x5b,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0xec,0x02,0x00,0x00,0x62,0x00,0x00,0x00, -0xc2,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0xed,0x02,0x00,0x00, -0xec,0x02,0x00,0x00,0x5b,0x04,0x00,0x00,0xc4,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0xee,0x02,0x00,0x00,0xed,0x02,0x00,0x00, -0x66,0x00,0x00,0x00,0xc7,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0xef,0x02,0x00,0x00,0xee,0x02,0x00,0x00,0x4e,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xf0,0x02,0x00,0x00, -0xef,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0xf2,0x02,0x00,0x00,0x62,0x00,0x00,0x00,0xc2,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0xf4,0x02,0x00,0x00,0xf2,0x02,0x00,0x00, -0x54,0x04,0x00,0x00,0xc7,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0xf5,0x02,0x00,0x00,0xf4,0x02,0x00,0x00,0x4e,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xf6,0x02,0x00,0x00, -0xf5,0x02,0x00,0x00,0x41,0x00,0x08,0x00,0x79,0x00,0x00,0x00, -0xf8,0x02,0x00,0x00,0x54,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0x77,0x00,0x00,0x00,0x5b,0x04,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00,0xf9,0x02,0x00,0x00, -0xf8,0x02,0x00,0x00,0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0xfa,0x02,0x00,0x00,0xf9,0x02,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xfb,0x02,0x00,0x00,0xfa,0x02,0x00,0x00, -0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xfc,0x02,0x00,0x00, -0xfb,0x02,0x00,0x00,0x82,0x00,0x00,0x00,0xc5,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xfe,0x02,0x00,0x00,0xfc,0x02,0x00,0x00, -0xf0,0x02,0x00,0x00,0x6f,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0xff,0x02,0x00,0x00,0xfe,0x02,0x00,0x00,0xc2,0x00,0x05,0x00, -0x4d,0x00,0x00,0x00,0x00,0x03,0x00,0x00,0xf9,0x02,0x00,0x00, -0x66,0x00,0x00,0x00,0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x01,0x03,0x00,0x00,0x00,0x03,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x02,0x03,0x00,0x00,0x01,0x03,0x00,0x00, -0xc5,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x04,0x03,0x00,0x00, -0x02,0x03,0x00,0x00,0xf6,0x02,0x00,0x00,0x6f,0x00,0x04,0x00, -0x4a,0x00,0x00,0x00,0x05,0x03,0x00,0x00,0x04,0x03,0x00,0x00, -0x50,0x00,0x05,0x00,0x7c,0x00,0x00,0x00,0x06,0x03,0x00,0x00, -0xff,0x02,0x00,0x00,0x05,0x03,0x00,0x00,0x8e,0x00,0x05,0x00, -0x7c,0x00,0x00,0x00,0x07,0x03,0x00,0x00,0x06,0x03,0x00,0x00, -0xe8,0x02,0x00,0x00,0x50,0x00,0x05,0x00,0x7c,0x00,0x00,0x00, -0x08,0x03,0x00,0x00,0xea,0x02,0x00,0x00,0xea,0x02,0x00,0x00, -0x81,0x00,0x05,0x00,0x7c,0x00,0x00,0x00,0x09,0x03,0x00,0x00, -0x07,0x03,0x00,0x00,0x08,0x03,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x0f,0x03,0x00,0x00,0xa2,0x00,0x00,0x00, -0x5b,0x04,0x00,0x00,0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00, -0x11,0x03,0x00,0x00,0x09,0x03,0x00,0x00,0x00,0x00,0x00,0x00, -0x41,0x00,0x06,0x00,0x56,0x00,0x00,0x00,0x12,0x03,0x00,0x00, -0x9b,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x0f,0x03,0x00,0x00, -0x3e,0x00,0x03,0x00,0x12,0x03,0x00,0x00,0x11,0x03,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x19,0x03,0x00,0x00, -0xa2,0x00,0x00,0x00,0x5c,0x04,0x00,0x00,0x51,0x00,0x05,0x00, -0x4a,0x00,0x00,0x00,0x1a,0x03,0x00,0x00,0x09,0x03,0x00,0x00, -0x01,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x56,0x00,0x00,0x00, -0x1b,0x03,0x00,0x00,0x9b,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x19,0x03,0x00,0x00,0x3e,0x00,0x03,0x00,0x1b,0x03,0x00,0x00, -0x1a,0x03,0x00,0x00,0x3d,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0x24,0x03,0x00,0x00,0x57,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4a,0x00,0x00,0x00,0x26,0x03,0x00,0x00,0x5b,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x28,0x03,0x00,0x00, -0x62,0x00,0x00,0x00,0xc2,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0x29,0x03,0x00,0x00,0x28,0x03,0x00,0x00,0x5d,0x04,0x00,0x00, -0xc4,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0x2a,0x03,0x00,0x00, -0x29,0x03,0x00,0x00,0x66,0x00,0x00,0x00,0xc7,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0x2b,0x03,0x00,0x00,0x2a,0x03,0x00,0x00, -0x4e,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x2c,0x03,0x00,0x00,0x2b,0x03,0x00,0x00,0x3d,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x2e,0x03,0x00,0x00,0x62,0x00,0x00,0x00, -0xc2,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0x30,0x03,0x00,0x00, -0x2e,0x03,0x00,0x00,0x56,0x04,0x00,0x00,0xc7,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0x31,0x03,0x00,0x00,0x30,0x03,0x00,0x00, -0x4e,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x32,0x03,0x00,0x00,0x31,0x03,0x00,0x00,0x41,0x00,0x08,0x00, -0x79,0x00,0x00,0x00,0x34,0x03,0x00,0x00,0x54,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x77,0x00,0x00,0x00, -0x5d,0x04,0x00,0x00,0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0x35,0x03,0x00,0x00,0x34,0x03,0x00,0x00,0x71,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x36,0x03,0x00,0x00,0x35,0x03,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x37,0x03,0x00,0x00, -0x36,0x03,0x00,0x00,0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x38,0x03,0x00,0x00,0x37,0x03,0x00,0x00,0x82,0x00,0x00,0x00, -0xc5,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3a,0x03,0x00,0x00, -0x38,0x03,0x00,0x00,0x2c,0x03,0x00,0x00,0x6f,0x00,0x04,0x00, -0x4a,0x00,0x00,0x00,0x3b,0x03,0x00,0x00,0x3a,0x03,0x00,0x00, -0xc2,0x00,0x05,0x00,0x4d,0x00,0x00,0x00,0x3c,0x03,0x00,0x00, -0x35,0x03,0x00,0x00,0x66,0x00,0x00,0x00,0x71,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x3d,0x03,0x00,0x00,0x3c,0x03,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x3e,0x03,0x00,0x00, -0x3d,0x03,0x00,0x00,0xc5,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x40,0x03,0x00,0x00,0x3e,0x03,0x00,0x00,0x32,0x03,0x00,0x00, -0x6f,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0x41,0x03,0x00,0x00, -0x40,0x03,0x00,0x00,0x50,0x00,0x05,0x00,0x7c,0x00,0x00,0x00, -0x42,0x03,0x00,0x00,0x3b,0x03,0x00,0x00,0x41,0x03,0x00,0x00, -0x8e,0x00,0x05,0x00,0x7c,0x00,0x00,0x00,0x43,0x03,0x00,0x00, -0x42,0x03,0x00,0x00,0x24,0x03,0x00,0x00,0x50,0x00,0x05,0x00, -0x7c,0x00,0x00,0x00,0x44,0x03,0x00,0x00,0x26,0x03,0x00,0x00, -0x26,0x03,0x00,0x00,0x81,0x00,0x05,0x00,0x7c,0x00,0x00,0x00, -0x45,0x03,0x00,0x00,0x43,0x03,0x00,0x00,0x44,0x03,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x4b,0x03,0x00,0x00, -0xa2,0x00,0x00,0x00,0x5d,0x04,0x00,0x00,0x51,0x00,0x05,0x00, -0x4a,0x00,0x00,0x00,0x4d,0x03,0x00,0x00,0x45,0x03,0x00,0x00, -0x00,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x56,0x00,0x00,0x00, -0x4e,0x03,0x00,0x00,0x9b,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x4b,0x03,0x00,0x00,0x3e,0x00,0x03,0x00,0x4e,0x03,0x00,0x00, -0x4d,0x03,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x55,0x03,0x00,0x00,0xa2,0x00,0x00,0x00,0x5e,0x04,0x00,0x00, -0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00,0x56,0x03,0x00,0x00, -0x45,0x03,0x00,0x00,0x01,0x00,0x00,0x00,0x41,0x00,0x06,0x00, -0x56,0x00,0x00,0x00,0x57,0x03,0x00,0x00,0x9b,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x55,0x03,0x00,0x00,0x3e,0x00,0x03,0x00, -0x57,0x03,0x00,0x00,0x56,0x03,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4a,0x00,0x00,0x00,0x60,0x03,0x00,0x00,0x57,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0x62,0x03,0x00,0x00, -0x5b,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x64,0x03,0x00,0x00,0x62,0x00,0x00,0x00,0xc2,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0x65,0x03,0x00,0x00,0x64,0x03,0x00,0x00, -0x6e,0x00,0x00,0x00,0xc4,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0x66,0x03,0x00,0x00,0x65,0x03,0x00,0x00,0x66,0x00,0x00,0x00, -0xc7,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0x67,0x03,0x00,0x00, -0x66,0x03,0x00,0x00,0x4e,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x68,0x03,0x00,0x00,0x67,0x03,0x00,0x00, -0x3d,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x6a,0x03,0x00,0x00, -0x62,0x00,0x00,0x00,0xc2,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0x6c,0x03,0x00,0x00,0x6a,0x03,0x00,0x00,0x58,0x04,0x00,0x00, -0xc7,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0x6d,0x03,0x00,0x00, -0x6c,0x03,0x00,0x00,0x4e,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x6e,0x03,0x00,0x00,0x6d,0x03,0x00,0x00, -0x41,0x00,0x08,0x00,0x79,0x00,0x00,0x00,0x70,0x03,0x00,0x00, -0x54,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x77,0x00,0x00,0x00,0x6e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0x71,0x03,0x00,0x00,0x70,0x03,0x00,0x00, -0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x72,0x03,0x00,0x00, -0x71,0x03,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x73,0x03,0x00,0x00,0x72,0x03,0x00,0x00,0xc7,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x74,0x03,0x00,0x00,0x73,0x03,0x00,0x00, -0x82,0x00,0x00,0x00,0xc5,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x76,0x03,0x00,0x00,0x74,0x03,0x00,0x00,0x68,0x03,0x00,0x00, -0x6f,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0x77,0x03,0x00,0x00, -0x76,0x03,0x00,0x00,0xc2,0x00,0x05,0x00,0x4d,0x00,0x00,0x00, -0x78,0x03,0x00,0x00,0x71,0x03,0x00,0x00,0x66,0x00,0x00,0x00, -0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x79,0x03,0x00,0x00, -0x78,0x03,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x7a,0x03,0x00,0x00,0x79,0x03,0x00,0x00,0xc5,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x7c,0x03,0x00,0x00,0x7a,0x03,0x00,0x00, -0x6e,0x03,0x00,0x00,0x6f,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0x7d,0x03,0x00,0x00,0x7c,0x03,0x00,0x00,0x50,0x00,0x05,0x00, -0x7c,0x00,0x00,0x00,0x7e,0x03,0x00,0x00,0x77,0x03,0x00,0x00, -0x7d,0x03,0x00,0x00,0x8e,0x00,0x05,0x00,0x7c,0x00,0x00,0x00, -0x7f,0x03,0x00,0x00,0x7e,0x03,0x00,0x00,0x60,0x03,0x00,0x00, -0x50,0x00,0x05,0x00,0x7c,0x00,0x00,0x00,0x80,0x03,0x00,0x00, -0x62,0x03,0x00,0x00,0x62,0x03,0x00,0x00,0x81,0x00,0x05,0x00, -0x7c,0x00,0x00,0x00,0x81,0x03,0x00,0x00,0x7f,0x03,0x00,0x00, -0x80,0x03,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x87,0x03,0x00,0x00,0xa2,0x00,0x00,0x00,0x6e,0x00,0x00,0x00, -0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00,0x89,0x03,0x00,0x00, -0x81,0x03,0x00,0x00,0x00,0x00,0x00,0x00,0x41,0x00,0x06,0x00, -0x56,0x00,0x00,0x00,0x8a,0x03,0x00,0x00,0x9b,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x87,0x03,0x00,0x00,0x3e,0x00,0x03,0x00, -0x8a,0x03,0x00,0x00,0x89,0x03,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x91,0x03,0x00,0x00,0xa2,0x00,0x00,0x00, -0x5f,0x04,0x00,0x00,0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00, -0x92,0x03,0x00,0x00,0x81,0x03,0x00,0x00,0x01,0x00,0x00,0x00, -0x41,0x00,0x06,0x00,0x56,0x00,0x00,0x00,0x93,0x03,0x00,0x00, -0x9b,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x91,0x03,0x00,0x00, -0x3e,0x00,0x03,0x00,0x93,0x03,0x00,0x00,0x92,0x03,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0x9c,0x03,0x00,0x00, -0x57,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0x9e,0x03,0x00,0x00,0x5b,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0xa0,0x03,0x00,0x00,0x62,0x00,0x00,0x00, -0xc2,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0xa1,0x03,0x00,0x00, -0xa0,0x03,0x00,0x00,0x4b,0x04,0x00,0x00,0xc4,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0xa2,0x03,0x00,0x00,0xa1,0x03,0x00,0x00, -0x66,0x00,0x00,0x00,0xc7,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0xa3,0x03,0x00,0x00,0xa2,0x03,0x00,0x00,0x4e,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xa4,0x03,0x00,0x00, -0xa3,0x03,0x00,0x00,0x3d,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0xa6,0x03,0x00,0x00,0x62,0x00,0x00,0x00,0xc2,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0xa8,0x03,0x00,0x00,0xa6,0x03,0x00,0x00, -0x5a,0x04,0x00,0x00,0xc7,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0xa9,0x03,0x00,0x00,0xa8,0x03,0x00,0x00,0x4e,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xaa,0x03,0x00,0x00, -0xa9,0x03,0x00,0x00,0x41,0x00,0x08,0x00,0x79,0x00,0x00,0x00, -0xac,0x03,0x00,0x00,0x54,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0x77,0x00,0x00,0x00,0x4b,0x04,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00,0xad,0x03,0x00,0x00, -0xac,0x03,0x00,0x00,0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0xae,0x03,0x00,0x00,0xad,0x03,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xaf,0x03,0x00,0x00,0xae,0x03,0x00,0x00, -0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xb0,0x03,0x00,0x00, -0xaf,0x03,0x00,0x00,0x82,0x00,0x00,0x00,0xc5,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xb2,0x03,0x00,0x00,0xb0,0x03,0x00,0x00, -0xa4,0x03,0x00,0x00,0x6f,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0xb3,0x03,0x00,0x00,0xb2,0x03,0x00,0x00,0xc2,0x00,0x05,0x00, -0x4d,0x00,0x00,0x00,0xb4,0x03,0x00,0x00,0xad,0x03,0x00,0x00, -0x66,0x00,0x00,0x00,0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0xb5,0x03,0x00,0x00,0xb4,0x03,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xb6,0x03,0x00,0x00,0xb5,0x03,0x00,0x00, -0xc5,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xb8,0x03,0x00,0x00, -0xb6,0x03,0x00,0x00,0xaa,0x03,0x00,0x00,0x6f,0x00,0x04,0x00, -0x4a,0x00,0x00,0x00,0xb9,0x03,0x00,0x00,0xb8,0x03,0x00,0x00, -0x50,0x00,0x05,0x00,0x7c,0x00,0x00,0x00,0xba,0x03,0x00,0x00, -0xb3,0x03,0x00,0x00,0xb9,0x03,0x00,0x00,0x8e,0x00,0x05,0x00, -0x7c,0x00,0x00,0x00,0xbb,0x03,0x00,0x00,0xba,0x03,0x00,0x00, -0x9c,0x03,0x00,0x00,0x50,0x00,0x05,0x00,0x7c,0x00,0x00,0x00, -0xbc,0x03,0x00,0x00,0x9e,0x03,0x00,0x00,0x9e,0x03,0x00,0x00, -0x81,0x00,0x05,0x00,0x7c,0x00,0x00,0x00,0xbd,0x03,0x00,0x00, -0xbb,0x03,0x00,0x00,0xbc,0x03,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xc3,0x03,0x00,0x00,0xa2,0x00,0x00,0x00, -0x4b,0x04,0x00,0x00,0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00, -0xc5,0x03,0x00,0x00,0xbd,0x03,0x00,0x00,0x00,0x00,0x00,0x00, -0x41,0x00,0x06,0x00,0x56,0x00,0x00,0x00,0xc6,0x03,0x00,0x00, -0x9b,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0xc3,0x03,0x00,0x00, -0x3e,0x00,0x03,0x00,0xc6,0x03,0x00,0x00,0xc5,0x03,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xcd,0x03,0x00,0x00, -0xa2,0x00,0x00,0x00,0x60,0x04,0x00,0x00,0x51,0x00,0x05,0x00, -0x4a,0x00,0x00,0x00,0xce,0x03,0x00,0x00,0xbd,0x03,0x00,0x00, -0x01,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x56,0x00,0x00,0x00, -0xcf,0x03,0x00,0x00,0x9b,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0xcd,0x03,0x00,0x00,0x3e,0x00,0x03,0x00,0xcf,0x03,0x00,0x00, -0xce,0x03,0x00,0x00,0x3d,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0xd8,0x03,0x00,0x00,0x57,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4a,0x00,0x00,0x00,0xda,0x03,0x00,0x00,0x5b,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0xdc,0x03,0x00,0x00, -0x62,0x00,0x00,0x00,0xc2,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0xdd,0x03,0x00,0x00,0xdc,0x03,0x00,0x00,0x4d,0x04,0x00,0x00, -0xc4,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0xde,0x03,0x00,0x00, -0xdd,0x03,0x00,0x00,0x66,0x00,0x00,0x00,0xc7,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0xdf,0x03,0x00,0x00,0xde,0x03,0x00,0x00, -0x4e,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xe0,0x03,0x00,0x00,0xdf,0x03,0x00,0x00,0x3d,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0xe2,0x03,0x00,0x00,0x62,0x00,0x00,0x00, -0xc2,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0xe4,0x03,0x00,0x00, -0xe2,0x03,0x00,0x00,0x5c,0x04,0x00,0x00,0xc7,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0xe5,0x03,0x00,0x00,0xe4,0x03,0x00,0x00, -0x4e,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xe6,0x03,0x00,0x00,0xe5,0x03,0x00,0x00,0x41,0x00,0x08,0x00, -0x79,0x00,0x00,0x00,0xe8,0x03,0x00,0x00,0x54,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x77,0x00,0x00,0x00, -0x4d,0x04,0x00,0x00,0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0xe9,0x03,0x00,0x00,0xe8,0x03,0x00,0x00,0x71,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0xea,0x03,0x00,0x00,0xe9,0x03,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xeb,0x03,0x00,0x00, -0xea,0x03,0x00,0x00,0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xec,0x03,0x00,0x00,0xeb,0x03,0x00,0x00,0x82,0x00,0x00,0x00, -0xc5,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xee,0x03,0x00,0x00, -0xec,0x03,0x00,0x00,0xe0,0x03,0x00,0x00,0x6f,0x00,0x04,0x00, -0x4a,0x00,0x00,0x00,0xef,0x03,0x00,0x00,0xee,0x03,0x00,0x00, -0xc2,0x00,0x05,0x00,0x4d,0x00,0x00,0x00,0xf0,0x03,0x00,0x00, -0xe9,0x03,0x00,0x00,0x66,0x00,0x00,0x00,0x71,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0xf1,0x03,0x00,0x00,0xf0,0x03,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xf2,0x03,0x00,0x00, -0xf1,0x03,0x00,0x00,0xc5,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xf4,0x03,0x00,0x00,0xf2,0x03,0x00,0x00,0xe6,0x03,0x00,0x00, -0x6f,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0xf5,0x03,0x00,0x00, -0xf4,0x03,0x00,0x00,0x50,0x00,0x05,0x00,0x7c,0x00,0x00,0x00, -0xf6,0x03,0x00,0x00,0xef,0x03,0x00,0x00,0xf5,0x03,0x00,0x00, -0x8e,0x00,0x05,0x00,0x7c,0x00,0x00,0x00,0xf7,0x03,0x00,0x00, -0xf6,0x03,0x00,0x00,0xd8,0x03,0x00,0x00,0x50,0x00,0x05,0x00, -0x7c,0x00,0x00,0x00,0xf8,0x03,0x00,0x00,0xda,0x03,0x00,0x00, -0xda,0x03,0x00,0x00,0x81,0x00,0x05,0x00,0x7c,0x00,0x00,0x00, -0xf9,0x03,0x00,0x00,0xf7,0x03,0x00,0x00,0xf8,0x03,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xff,0x03,0x00,0x00, -0xa2,0x00,0x00,0x00,0x4d,0x04,0x00,0x00,0x51,0x00,0x05,0x00, -0x4a,0x00,0x00,0x00,0x01,0x04,0x00,0x00,0xf9,0x03,0x00,0x00, -0x00,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x56,0x00,0x00,0x00, -0x02,0x04,0x00,0x00,0x9b,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0xff,0x03,0x00,0x00,0x3e,0x00,0x03,0x00,0x02,0x04,0x00,0x00, -0x01,0x04,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x09,0x04,0x00,0x00,0xa2,0x00,0x00,0x00,0x61,0x04,0x00,0x00, -0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00,0x0a,0x04,0x00,0x00, -0xf9,0x03,0x00,0x00,0x01,0x00,0x00,0x00,0x41,0x00,0x06,0x00, -0x56,0x00,0x00,0x00,0x0b,0x04,0x00,0x00,0x9b,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x09,0x04,0x00,0x00,0x3e,0x00,0x03,0x00, -0x0b,0x04,0x00,0x00,0x0a,0x04,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4a,0x00,0x00,0x00,0x14,0x04,0x00,0x00,0x57,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0x16,0x04,0x00,0x00, -0x5b,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x18,0x04,0x00,0x00,0x62,0x00,0x00,0x00,0xc2,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0x19,0x04,0x00,0x00,0x18,0x04,0x00,0x00, -0x82,0x00,0x00,0x00,0xc4,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0x1a,0x04,0x00,0x00,0x19,0x04,0x00,0x00,0x66,0x00,0x00,0x00, -0xc7,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0x1b,0x04,0x00,0x00, -0x1a,0x04,0x00,0x00,0x4e,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x1c,0x04,0x00,0x00,0x1b,0x04,0x00,0x00, -0x3d,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x1e,0x04,0x00,0x00, -0x62,0x00,0x00,0x00,0xc2,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0x20,0x04,0x00,0x00,0x1e,0x04,0x00,0x00,0x5e,0x04,0x00,0x00, -0xc7,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0x21,0x04,0x00,0x00, -0x20,0x04,0x00,0x00,0x4e,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x22,0x04,0x00,0x00,0x21,0x04,0x00,0x00, -0x41,0x00,0x08,0x00,0x79,0x00,0x00,0x00,0x24,0x04,0x00,0x00, -0x54,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x77,0x00,0x00,0x00,0x82,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0x25,0x04,0x00,0x00,0x24,0x04,0x00,0x00, -0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x26,0x04,0x00,0x00, -0x25,0x04,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x27,0x04,0x00,0x00,0x26,0x04,0x00,0x00,0xc7,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x28,0x04,0x00,0x00,0x27,0x04,0x00,0x00, -0x82,0x00,0x00,0x00,0xc5,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x2a,0x04,0x00,0x00,0x28,0x04,0x00,0x00,0x1c,0x04,0x00,0x00, -0x6f,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0x2b,0x04,0x00,0x00, -0x2a,0x04,0x00,0x00,0xc2,0x00,0x05,0x00,0x4d,0x00,0x00,0x00, -0x2c,0x04,0x00,0x00,0x25,0x04,0x00,0x00,0x66,0x00,0x00,0x00, -0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x2d,0x04,0x00,0x00, -0x2c,0x04,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x2e,0x04,0x00,0x00,0x2d,0x04,0x00,0x00,0xc5,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x30,0x04,0x00,0x00,0x2e,0x04,0x00,0x00, -0x22,0x04,0x00,0x00,0x6f,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0x31,0x04,0x00,0x00,0x30,0x04,0x00,0x00,0x50,0x00,0x05,0x00, -0x7c,0x00,0x00,0x00,0x32,0x04,0x00,0x00,0x2b,0x04,0x00,0x00, -0x31,0x04,0x00,0x00,0x8e,0x00,0x05,0x00,0x7c,0x00,0x00,0x00, -0x33,0x04,0x00,0x00,0x32,0x04,0x00,0x00,0x14,0x04,0x00,0x00, -0x50,0x00,0x05,0x00,0x7c,0x00,0x00,0x00,0x34,0x04,0x00,0x00, -0x16,0x04,0x00,0x00,0x16,0x04,0x00,0x00,0x81,0x00,0x05,0x00, -0x7c,0x00,0x00,0x00,0x35,0x04,0x00,0x00,0x33,0x04,0x00,0x00, -0x34,0x04,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x3b,0x04,0x00,0x00,0xa2,0x00,0x00,0x00,0x82,0x00,0x00,0x00, -0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00,0x3d,0x04,0x00,0x00, -0x35,0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x41,0x00,0x06,0x00, -0x56,0x00,0x00,0x00,0x3e,0x04,0x00,0x00,0x9b,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x3b,0x04,0x00,0x00,0x3e,0x00,0x03,0x00, -0x3e,0x04,0x00,0x00,0x3d,0x04,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x45,0x04,0x00,0x00,0xa2,0x00,0x00,0x00, -0x62,0x04,0x00,0x00,0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00, -0x46,0x04,0x00,0x00,0x35,0x04,0x00,0x00,0x01,0x00,0x00,0x00, -0x41,0x00,0x06,0x00,0x56,0x00,0x00,0x00,0x47,0x04,0x00,0x00, -0x9b,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x45,0x04,0x00,0x00, -0x3e,0x00,0x03,0x00,0x47,0x04,0x00,0x00,0x46,0x04,0x00,0x00, -0xf9,0x00,0x02,0x00,0xba,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xba,0x00,0x00,0x00,0xfd,0x00,0x01,0x00,0x38,0x00,0x01,0x00, - -}; -const uint64_t dequant_q5_1_len = 12768; - -unsigned char dequant_q5_1_fp32_data[] = { -0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, 0x95,0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, 0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x51,0x11,0x00,0x00, 0x11,0x00,0x02,0x00,0x60,0x11,0x00,0x00,0x0b,0x00,0x06,0x00, @@ -11163,505 +6149,10 @@ unsigned char dequant_q5_1_fp32_data[] = { 0xbe,0x00,0x00,0x00,0xfd,0x00,0x01,0x00,0x38,0x00,0x01,0x00, }; -const uint64_t dequant_q5_1_fp32_len = 13548; +const uint64_t dequant_q5_1_len = 13548; unsigned char dequant_q5_K_data[] = { 0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, -0x99,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, -0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x09,0x00,0x00,0x00, -0x11,0x00,0x02,0x00,0x27,0x00,0x00,0x00,0x11,0x00,0x02,0x00, -0x51,0x11,0x00,0x00,0x11,0x00,0x02,0x00,0x60,0x11,0x00,0x00, -0x0b,0x00,0x06,0x00,0x01,0x00,0x00,0x00,0x47,0x4c,0x53,0x4c, -0x2e,0x73,0x74,0x64,0x2e,0x34,0x35,0x30,0x00,0x00,0x00,0x00, -0x0e,0x00,0x03,0x00,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x0f,0x00,0x0a,0x00,0x05,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x6d,0x61,0x69,0x6e,0x00,0x00,0x00,0x00,0x17,0x00,0x00,0x00, -0x25,0x00,0x00,0x00,0x33,0x00,0x00,0x00,0x51,0x00,0x00,0x00, -0x12,0x01,0x00,0x00,0x10,0x00,0x06,0x00,0x04,0x00,0x00,0x00, -0x11,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x17,0x00,0x00,0x00, -0x0b,0x00,0x00,0x00,0x1a,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x23,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x23,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x23,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x23,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x33,0x00,0x00,0x00, -0x0b,0x00,0x00,0x00,0x1b,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x48,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x4c,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x4d,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x4d,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x4d,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x4d,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x30,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x4e,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0xb0,0x00,0x00,0x00,0x48,0x00,0x04,0x00, -0x4f,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x4f,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, -0x4f,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x51,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x51,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x0f,0x01,0x00,0x00, -0x06,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x48,0x00,0x04,0x00, -0x10,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x19,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x10,0x01,0x00,0x00,0x00,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, -0x10,0x01,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x12,0x01,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x12,0x01,0x00,0x00,0x21,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x82,0x01,0x00,0x00, -0x0b,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x13,0x00,0x02,0x00, -0x02,0x00,0x00,0x00,0x21,0x00,0x03,0x00,0x03,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x15,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x10,0x00,0x00,0x00, -0x00,0x01,0x00,0x00,0x14,0x00,0x02,0x00,0x11,0x00,0x00,0x00, -0x15,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x17,0x00,0x04,0x00,0x15,0x00,0x00,0x00, -0x14,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x16,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x15,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x16,0x00,0x00,0x00,0x17,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x14,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x19,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x14,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0x1c,0x00,0x00,0x00, -0x00,0x01,0x00,0x00,0x1e,0x00,0x06,0x00,0x23,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x24,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x24,0x00,0x00,0x00,0x25,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x26,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x29,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x16,0x00,0x00,0x00,0x33,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x10,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x3f,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x16,0x00,0x03,0x00, -0x42,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x17,0x00,0x04,0x00, -0x45,0x00,0x00,0x00,0x42,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x15,0x00,0x04,0x00,0x46,0x00,0x00,0x00,0x08,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x14,0x00,0x00,0x00, -0x47,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x1c,0x00,0x04,0x00, -0x48,0x00,0x00,0x00,0x46,0x00,0x00,0x00,0x47,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0x49,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x1c,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0x46,0x00,0x00,0x00,0x49,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x14,0x00,0x00,0x00,0x4b,0x00,0x00,0x00,0x80,0x00,0x00,0x00, -0x1c,0x00,0x04,0x00,0x4c,0x00,0x00,0x00,0x46,0x00,0x00,0x00, -0x4b,0x00,0x00,0x00,0x1e,0x00,0x06,0x00,0x4d,0x00,0x00,0x00, -0x45,0x00,0x00,0x00,0x48,0x00,0x00,0x00,0x4a,0x00,0x00,0x00, -0x4c,0x00,0x00,0x00,0x1d,0x00,0x03,0x00,0x4e,0x00,0x00,0x00, -0x4d,0x00,0x00,0x00,0x1e,0x00,0x03,0x00,0x4f,0x00,0x00,0x00, -0x4e,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x50,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x50,0x00,0x00,0x00,0x51,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x53,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x42,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x14,0x00,0x00,0x00, -0x58,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x5e,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x66,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x70,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x78,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x46,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x7d,0x00,0x00,0x00, -0x3f,0x00,0x00,0x00,0x15,0x00,0x04,0x00,0x7f,0x00,0x00,0x00, -0x08,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x95,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x9c,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xcb,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xdf,0x00,0x00,0x00,0x03,0x00,0x00,0x00, -0x1d,0x00,0x03,0x00,0x0f,0x01,0x00,0x00,0x42,0x00,0x00,0x00, -0x1e,0x00,0x03,0x00,0x10,0x01,0x00,0x00,0x0f,0x01,0x00,0x00, -0x20,0x00,0x04,0x00,0x11,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, -0x10,0x01,0x00,0x00,0x3b,0x00,0x04,0x00,0x11,0x01,0x00,0x00, -0x12,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x63,0x01,0x00,0x00,0x21,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0x81,0x01,0x00,0x00, -0x40,0x00,0x00,0x00,0x2c,0x00,0x06,0x00,0x15,0x00,0x00,0x00, -0x82,0x01,0x00,0x00,0x81,0x01,0x00,0x00,0x58,0x00,0x00,0x00, -0x58,0x00,0x00,0x00,0x2a,0x00,0x03,0x00,0x11,0x00,0x00,0x00, -0x85,0x01,0x00,0x00,0x29,0x00,0x03,0x00,0x11,0x00,0x00,0x00, -0x88,0x01,0x00,0x00,0x36,0x00,0x05,0x00,0x02,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x03,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0x05,0x00,0x00,0x00,0xf7,0x00,0x03,0x00, -0x83,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0xfb,0x00,0x03,0x00, -0x18,0x00,0x00,0x00,0x84,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x84,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x0a,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0x0a,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0x8b,0x01,0x00,0x00,0x09,0x00,0x00,0x00, -0x84,0x01,0x00,0x00,0x80,0x01,0x00,0x00,0x0d,0x00,0x00,0x00, -0xb1,0x00,0x05,0x00,0x11,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x8b,0x01,0x00,0x00,0x10,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0x0c,0x00,0x00,0x00,0x0d,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x12,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x0b,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x19,0x00,0x00,0x00,0x1a,0x00,0x00,0x00, -0x17,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x14,0x00,0x00,0x00,0x1b,0x00,0x00,0x00,0x1a,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x14,0x00,0x00,0x00,0x1d,0x00,0x00,0x00, -0x1b,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x14,0x00,0x00,0x00,0x1f,0x00,0x00,0x00,0x8b,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x14,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x1d,0x00,0x00,0x00,0x1f,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x26,0x00,0x00,0x00,0x27,0x00,0x00,0x00, -0x25,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0x27,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x26,0x00,0x00,0x00,0x2a,0x00,0x00,0x00, -0x25,0x00,0x00,0x00,0x29,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x2b,0x00,0x00,0x00,0x2a,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x2c,0x00,0x00,0x00, -0x28,0x00,0x00,0x00,0x2b,0x00,0x00,0x00,0x87,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x2d,0x00,0x00,0x00,0x2c,0x00,0x00,0x00, -0x10,0x00,0x00,0x00,0xaf,0x00,0x05,0x00,0x11,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x2d,0x00,0x00,0x00, -0xf7,0x00,0x03,0x00,0x30,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x2e,0x00,0x00,0x00,0x2f,0x00,0x00,0x00, -0x30,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x2f,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x0c,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x30,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x19,0x00,0x00,0x00, -0x34,0x00,0x00,0x00,0x33,0x00,0x00,0x00,0x18,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0x35,0x00,0x00,0x00, -0x34,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x36,0x00,0x00,0x00,0x35,0x00,0x00,0x00,0x87,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x36,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x8b,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x3d,0x00,0x00,0x00,0x36,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x41,0x00,0x00,0x00, -0x3f,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x41,0x00,0x08,0x00, -0x53,0x00,0x00,0x00,0x54,0x00,0x00,0x00,0x51,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x42,0x00,0x00,0x00, -0x55,0x00,0x00,0x00,0x54,0x00,0x00,0x00,0x41,0x00,0x08,0x00, -0x53,0x00,0x00,0x00,0x59,0x00,0x00,0x00,0x51,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x58,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x42,0x00,0x00,0x00, -0x5a,0x00,0x00,0x00,0x59,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x5d,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x10,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x60,0x00,0x00,0x00,0x5e,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x61,0x00,0x00,0x00, -0x5d,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x63,0x00,0x00,0x00,0x3f,0x00,0x00,0x00, -0x3d,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x64,0x00,0x00,0x00,0x61,0x00,0x00,0x00,0x63,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x68,0x00,0x00,0x00, -0x66,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x6b,0x00,0x00,0x00,0x68,0x00,0x00,0x00, -0x63,0x00,0x00,0x00,0xb1,0x00,0x05,0x00,0x11,0x00,0x00,0x00, -0x71,0x00,0x00,0x00,0x41,0x00,0x00,0x00,0x70,0x00,0x00,0x00, -0xf7,0x00,0x03,0x00,0x73,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x71,0x00,0x00,0x00,0x72,0x00,0x00,0x00, -0x8d,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x72,0x00,0x00,0x00, -0x41,0x00,0x08,0x00,0x78,0x00,0x00,0x00,0x79,0x00,0x00,0x00, -0x51,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x29,0x00,0x00,0x00,0x41,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x46,0x00,0x00,0x00,0x7a,0x00,0x00,0x00,0x79,0x00,0x00,0x00, -0x71,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0x7b,0x00,0x00,0x00, -0x7a,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x7c,0x00,0x00,0x00,0x7b,0x00,0x00,0x00,0xc7,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x7e,0x00,0x00,0x00,0x7c,0x00,0x00,0x00, -0x7d,0x00,0x00,0x00,0x72,0x00,0x04,0x00,0x7f,0x00,0x00,0x00, -0x80,0x00,0x00,0x00,0x7e,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x46,0x00,0x00,0x00,0x81,0x00,0x00,0x00,0x80,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x85,0x00,0x00,0x00, -0x41,0x00,0x00,0x00,0x70,0x00,0x00,0x00,0x41,0x00,0x08,0x00, -0x78,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x51,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x29,0x00,0x00,0x00, -0x85,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x46,0x00,0x00,0x00, -0x87,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x71,0x00,0x04,0x00, -0x14,0x00,0x00,0x00,0x88,0x00,0x00,0x00,0x87,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x89,0x00,0x00,0x00, -0x88,0x00,0x00,0x00,0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x8a,0x00,0x00,0x00,0x89,0x00,0x00,0x00,0x7d,0x00,0x00,0x00, -0x72,0x00,0x04,0x00,0x7f,0x00,0x00,0x00,0x8b,0x00,0x00,0x00, -0x8a,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x46,0x00,0x00,0x00, -0x8c,0x00,0x00,0x00,0x8b,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x73,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x8d,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x90,0x00,0x00,0x00, -0x41,0x00,0x00,0x00,0x70,0x00,0x00,0x00,0x41,0x00,0x08,0x00, -0x78,0x00,0x00,0x00,0x91,0x00,0x00,0x00,0x51,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x29,0x00,0x00,0x00, -0x90,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x46,0x00,0x00,0x00, -0x92,0x00,0x00,0x00,0x91,0x00,0x00,0x00,0x71,0x00,0x04,0x00, -0x14,0x00,0x00,0x00,0x93,0x00,0x00,0x00,0x92,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x94,0x00,0x00,0x00, -0x93,0x00,0x00,0x00,0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x96,0x00,0x00,0x00,0x94,0x00,0x00,0x00,0x95,0x00,0x00,0x00, -0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x99,0x00,0x00,0x00, -0x41,0x00,0x00,0x00,0x70,0x00,0x00,0x00,0x41,0x00,0x08,0x00, -0x78,0x00,0x00,0x00,0x9a,0x00,0x00,0x00,0x51,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x29,0x00,0x00,0x00, -0x99,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x46,0x00,0x00,0x00, -0x9b,0x00,0x00,0x00,0x9a,0x00,0x00,0x00,0xc2,0x00,0x05,0x00, -0x46,0x00,0x00,0x00,0x9d,0x00,0x00,0x00,0x9b,0x00,0x00,0x00, -0x9c,0x00,0x00,0x00,0xc4,0x00,0x05,0x00,0x46,0x00,0x00,0x00, -0x9e,0x00,0x00,0x00,0x9d,0x00,0x00,0x00,0x70,0x00,0x00,0x00, -0x71,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0x9f,0x00,0x00,0x00, -0x9e,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xa0,0x00,0x00,0x00,0x9f,0x00,0x00,0x00,0xc5,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xa1,0x00,0x00,0x00,0x96,0x00,0x00,0x00, -0xa0,0x00,0x00,0x00,0x72,0x00,0x04,0x00,0x7f,0x00,0x00,0x00, -0xa2,0x00,0x00,0x00,0xa1,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x46,0x00,0x00,0x00,0xa3,0x00,0x00,0x00,0xa2,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x46,0x00,0x00,0x00,0xa8,0x00,0x00,0x00, -0x91,0x00,0x00,0x00,0xc2,0x00,0x05,0x00,0x46,0x00,0x00,0x00, -0xa9,0x00,0x00,0x00,0xa8,0x00,0x00,0x00,0x70,0x00,0x00,0x00, -0x41,0x00,0x08,0x00,0x78,0x00,0x00,0x00,0xac,0x00,0x00,0x00, -0x51,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x29,0x00,0x00,0x00,0x41,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x46,0x00,0x00,0x00,0xad,0x00,0x00,0x00,0xac,0x00,0x00,0x00, -0xc2,0x00,0x05,0x00,0x46,0x00,0x00,0x00,0xae,0x00,0x00,0x00, -0xad,0x00,0x00,0x00,0x9c,0x00,0x00,0x00,0xc4,0x00,0x05,0x00, -0x46,0x00,0x00,0x00,0xaf,0x00,0x00,0x00,0xae,0x00,0x00,0x00, -0x70,0x00,0x00,0x00,0xc5,0x00,0x05,0x00,0x46,0x00,0x00,0x00, -0xb0,0x00,0x00,0x00,0xa9,0x00,0x00,0x00,0xaf,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x73,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x73,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x46,0x00,0x00,0x00, -0x8d,0x01,0x00,0x00,0x8c,0x00,0x00,0x00,0x72,0x00,0x00,0x00, -0xb0,0x00,0x00,0x00,0x8d,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, -0x46,0x00,0x00,0x00,0x8c,0x01,0x00,0x00,0x81,0x00,0x00,0x00, -0x72,0x00,0x00,0x00,0xa3,0x00,0x00,0x00,0x8d,0x00,0x00,0x00, -0x70,0x00,0x04,0x00,0x42,0x00,0x00,0x00,0xb4,0x00,0x00,0x00, -0x8c,0x01,0x00,0x00,0x85,0x00,0x05,0x00,0x42,0x00,0x00,0x00, -0xb5,0x00,0x00,0x00,0x55,0x00,0x00,0x00,0xb4,0x00,0x00,0x00, -0x70,0x00,0x04,0x00,0x42,0x00,0x00,0x00,0xb9,0x00,0x00,0x00, -0x8d,0x01,0x00,0x00,0x85,0x00,0x05,0x00,0x42,0x00,0x00,0x00, -0xba,0x00,0x00,0x00,0x5a,0x00,0x00,0x00,0xb9,0x00,0x00,0x00, -0xf7,0x00,0x03,0x00,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x71,0x00,0x00,0x00,0xbd,0x00,0x00,0x00, -0xd4,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xbd,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xc1,0x00,0x00,0x00, -0x41,0x00,0x00,0x00,0x29,0x00,0x00,0x00,0x41,0x00,0x08,0x00, -0x78,0x00,0x00,0x00,0xc2,0x00,0x00,0x00,0x51,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x29,0x00,0x00,0x00, -0xc1,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x46,0x00,0x00,0x00, -0xc3,0x00,0x00,0x00,0xc2,0x00,0x00,0x00,0x71,0x00,0x04,0x00, -0x14,0x00,0x00,0x00,0xc4,0x00,0x00,0x00,0xc3,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xc5,0x00,0x00,0x00, -0xc4,0x00,0x00,0x00,0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xc6,0x00,0x00,0x00,0xc5,0x00,0x00,0x00,0x7d,0x00,0x00,0x00, -0x72,0x00,0x04,0x00,0x7f,0x00,0x00,0x00,0xc7,0x00,0x00,0x00, -0xc6,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x46,0x00,0x00,0x00, -0xc8,0x00,0x00,0x00,0xc7,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xcc,0x00,0x00,0x00,0x41,0x00,0x00,0x00, -0xcb,0x00,0x00,0x00,0x41,0x00,0x08,0x00,0x78,0x00,0x00,0x00, -0xcd,0x00,0x00,0x00,0x51,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x21,0x00,0x00,0x00,0x29,0x00,0x00,0x00,0xcc,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x46,0x00,0x00,0x00,0xce,0x00,0x00,0x00, -0xcd,0x00,0x00,0x00,0x71,0x00,0x04,0x00,0x14,0x00,0x00,0x00, -0xcf,0x00,0x00,0x00,0xce,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xd0,0x00,0x00,0x00,0xcf,0x00,0x00,0x00, -0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xd1,0x00,0x00,0x00, -0xd0,0x00,0x00,0x00,0x7d,0x00,0x00,0x00,0x72,0x00,0x04,0x00, -0x7f,0x00,0x00,0x00,0xd2,0x00,0x00,0x00,0xd1,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x46,0x00,0x00,0x00,0xd3,0x00,0x00,0x00, -0xd2,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xbe,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xd4,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xd7,0x00,0x00,0x00,0x41,0x00,0x00,0x00, -0xcb,0x00,0x00,0x00,0x41,0x00,0x08,0x00,0x78,0x00,0x00,0x00, -0xd8,0x00,0x00,0x00,0x51,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x21,0x00,0x00,0x00,0x29,0x00,0x00,0x00,0xd7,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x46,0x00,0x00,0x00,0xd9,0x00,0x00,0x00, -0xd8,0x00,0x00,0x00,0x71,0x00,0x04,0x00,0x14,0x00,0x00,0x00, -0xda,0x00,0x00,0x00,0xd9,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xdb,0x00,0x00,0x00,0xda,0x00,0x00,0x00, -0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xdc,0x00,0x00,0x00, -0xdb,0x00,0x00,0x00,0x95,0x00,0x00,0x00,0x82,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xe0,0x00,0x00,0x00,0x41,0x00,0x00,0x00, -0xdf,0x00,0x00,0x00,0x41,0x00,0x08,0x00,0x78,0x00,0x00,0x00, -0xe1,0x00,0x00,0x00,0x51,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x21,0x00,0x00,0x00,0x29,0x00,0x00,0x00,0xe0,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x46,0x00,0x00,0x00,0xe2,0x00,0x00,0x00, -0xe1,0x00,0x00,0x00,0xc2,0x00,0x05,0x00,0x46,0x00,0x00,0x00, -0xe3,0x00,0x00,0x00,0xe2,0x00,0x00,0x00,0x9c,0x00,0x00,0x00, -0xc4,0x00,0x05,0x00,0x46,0x00,0x00,0x00,0xe4,0x00,0x00,0x00, -0xe3,0x00,0x00,0x00,0x70,0x00,0x00,0x00,0x71,0x00,0x04,0x00, -0x14,0x00,0x00,0x00,0xe5,0x00,0x00,0x00,0xe4,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xe6,0x00,0x00,0x00, -0xe5,0x00,0x00,0x00,0xc5,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xe7,0x00,0x00,0x00,0xdc,0x00,0x00,0x00,0xe6,0x00,0x00,0x00, -0x72,0x00,0x04,0x00,0x7f,0x00,0x00,0x00,0xe8,0x00,0x00,0x00, -0xe7,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x46,0x00,0x00,0x00, -0xe9,0x00,0x00,0x00,0xe8,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x46,0x00,0x00,0x00,0xee,0x00,0x00,0x00,0xd8,0x00,0x00,0x00, -0xc2,0x00,0x05,0x00,0x46,0x00,0x00,0x00,0xef,0x00,0x00,0x00, -0xee,0x00,0x00,0x00,0x70,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xf2,0x00,0x00,0x00,0x41,0x00,0x00,0x00, -0x29,0x00,0x00,0x00,0x41,0x00,0x08,0x00,0x78,0x00,0x00,0x00, -0xf3,0x00,0x00,0x00,0x51,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x21,0x00,0x00,0x00,0x29,0x00,0x00,0x00,0xf2,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x46,0x00,0x00,0x00,0xf4,0x00,0x00,0x00, -0xf3,0x00,0x00,0x00,0xc2,0x00,0x05,0x00,0x46,0x00,0x00,0x00, -0xf5,0x00,0x00,0x00,0xf4,0x00,0x00,0x00,0x9c,0x00,0x00,0x00, -0xc4,0x00,0x05,0x00,0x46,0x00,0x00,0x00,0xf6,0x00,0x00,0x00, -0xf5,0x00,0x00,0x00,0x70,0x00,0x00,0x00,0xc5,0x00,0x05,0x00, -0x46,0x00,0x00,0x00,0xf7,0x00,0x00,0x00,0xef,0x00,0x00,0x00, -0xf6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xbe,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xbe,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, -0x46,0x00,0x00,0x00,0x8f,0x01,0x00,0x00,0xd3,0x00,0x00,0x00, -0xbd,0x00,0x00,0x00,0xf7,0x00,0x00,0x00,0xd4,0x00,0x00,0x00, -0xf5,0x00,0x07,0x00,0x46,0x00,0x00,0x00,0x8e,0x01,0x00,0x00, -0xc8,0x00,0x00,0x00,0xbd,0x00,0x00,0x00,0xe9,0x00,0x00,0x00, -0xd4,0x00,0x00,0x00,0x70,0x00,0x04,0x00,0x42,0x00,0x00,0x00, -0xfb,0x00,0x00,0x00,0x8e,0x01,0x00,0x00,0x85,0x00,0x05,0x00, -0x42,0x00,0x00,0x00,0xfc,0x00,0x00,0x00,0x55,0x00,0x00,0x00, -0xfb,0x00,0x00,0x00,0x70,0x00,0x04,0x00,0x42,0x00,0x00,0x00, -0x00,0x01,0x00,0x00,0x8f,0x01,0x00,0x00,0x85,0x00,0x05,0x00, -0x42,0x00,0x00,0x00,0x01,0x01,0x00,0x00,0x5a,0x00,0x00,0x00, -0x00,0x01,0x00,0x00,0xc4,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x05,0x01,0x00,0x00,0x29,0x00,0x00,0x00,0x41,0x00,0x00,0x00, -0x72,0x00,0x04,0x00,0x7f,0x00,0x00,0x00,0x06,0x01,0x00,0x00, -0x05,0x01,0x00,0x00,0x7c,0x00,0x04,0x00,0x46,0x00,0x00,0x00, -0x07,0x01,0x00,0x00,0x06,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x0b,0x01,0x00,0x00,0x41,0x00,0x00,0x00, -0x29,0x00,0x00,0x00,0xc4,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x0c,0x01,0x00,0x00,0x29,0x00,0x00,0x00,0x0b,0x01,0x00,0x00, -0x72,0x00,0x04,0x00,0x7f,0x00,0x00,0x00,0x0d,0x01,0x00,0x00, -0x0c,0x01,0x00,0x00,0x7c,0x00,0x04,0x00,0x46,0x00,0x00,0x00, -0x0e,0x01,0x00,0x00,0x0d,0x01,0x00,0x00,0x41,0x00,0x08,0x00, -0x78,0x00,0x00,0x00,0x17,0x01,0x00,0x00,0x51,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0xdf,0x00,0x00,0x00, -0x6b,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x46,0x00,0x00,0x00, -0x18,0x01,0x00,0x00,0x17,0x01,0x00,0x00,0x71,0x00,0x04,0x00, -0x14,0x00,0x00,0x00,0x19,0x01,0x00,0x00,0x18,0x01,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x1a,0x01,0x00,0x00, -0x19,0x01,0x00,0x00,0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x1b,0x01,0x00,0x00,0x1a,0x01,0x00,0x00,0x95,0x00,0x00,0x00, -0x41,0x00,0x08,0x00,0x78,0x00,0x00,0x00,0x1e,0x01,0x00,0x00, -0x51,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x3f,0x00,0x00,0x00,0x63,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x46,0x00,0x00,0x00,0x1f,0x01,0x00,0x00,0x1e,0x01,0x00,0x00, -0xc7,0x00,0x05,0x00,0x46,0x00,0x00,0x00,0x21,0x01,0x00,0x00, -0x1f,0x01,0x00,0x00,0x07,0x01,0x00,0x00,0x71,0x00,0x04,0x00, -0x14,0x00,0x00,0x00,0x22,0x01,0x00,0x00,0x21,0x01,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x23,0x01,0x00,0x00, -0x22,0x01,0x00,0x00,0xab,0x00,0x05,0x00,0x11,0x00,0x00,0x00, -0x24,0x01,0x00,0x00,0x23,0x01,0x00,0x00,0x09,0x00,0x00,0x00, -0xa9,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x25,0x01,0x00,0x00, -0x24,0x01,0x00,0x00,0x39,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x26,0x01,0x00,0x00, -0x1b,0x01,0x00,0x00,0x25,0x01,0x00,0x00,0x6f,0x00,0x04,0x00, -0x42,0x00,0x00,0x00,0x27,0x01,0x00,0x00,0x26,0x01,0x00,0x00, -0x7f,0x00,0x04,0x00,0x42,0x00,0x00,0x00,0x96,0x01,0x00,0x00, -0xba,0x00,0x00,0x00,0x0c,0x00,0x08,0x00,0x42,0x00,0x00,0x00, -0x2a,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00, -0xb5,0x00,0x00,0x00,0x27,0x01,0x00,0x00,0x96,0x01,0x00,0x00, -0x41,0x00,0x06,0x00,0x53,0x00,0x00,0x00,0x2b,0x01,0x00,0x00, -0x12,0x01,0x00,0x00,0x09,0x00,0x00,0x00,0x64,0x00,0x00,0x00, -0x3e,0x00,0x03,0x00,0x2b,0x01,0x00,0x00,0x2a,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x2d,0x01,0x00,0x00, -0x64,0x00,0x00,0x00,0x29,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x31,0x01,0x00,0x00,0x6b,0x00,0x00,0x00, -0x29,0x00,0x00,0x00,0x41,0x00,0x08,0x00,0x78,0x00,0x00,0x00, -0x32,0x01,0x00,0x00,0x51,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x21,0x00,0x00,0x00,0xdf,0x00,0x00,0x00,0x31,0x01,0x00,0x00, -0x3d,0x00,0x04,0x00,0x46,0x00,0x00,0x00,0x33,0x01,0x00,0x00, -0x32,0x01,0x00,0x00,0x71,0x00,0x04,0x00,0x14,0x00,0x00,0x00, -0x34,0x01,0x00,0x00,0x33,0x01,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x35,0x01,0x00,0x00,0x34,0x01,0x00,0x00, -0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x36,0x01,0x00,0x00, -0x35,0x01,0x00,0x00,0x95,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x39,0x01,0x00,0x00,0x63,0x00,0x00,0x00, -0x29,0x00,0x00,0x00,0x41,0x00,0x08,0x00,0x78,0x00,0x00,0x00, -0x3a,0x01,0x00,0x00,0x51,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x21,0x00,0x00,0x00,0x3f,0x00,0x00,0x00,0x39,0x01,0x00,0x00, -0x3d,0x00,0x04,0x00,0x46,0x00,0x00,0x00,0x3b,0x01,0x00,0x00, -0x3a,0x01,0x00,0x00,0xc7,0x00,0x05,0x00,0x46,0x00,0x00,0x00, -0x3d,0x01,0x00,0x00,0x3b,0x01,0x00,0x00,0x07,0x01,0x00,0x00, -0x71,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0x3e,0x01,0x00,0x00, -0x3d,0x01,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x3f,0x01,0x00,0x00,0x3e,0x01,0x00,0x00,0xab,0x00,0x05,0x00, -0x11,0x00,0x00,0x00,0x40,0x01,0x00,0x00,0x3f,0x01,0x00,0x00, -0x09,0x00,0x00,0x00,0xa9,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x41,0x01,0x00,0x00,0x40,0x01,0x00,0x00,0x39,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x42,0x01,0x00,0x00,0x36,0x01,0x00,0x00,0x41,0x01,0x00,0x00, -0x6f,0x00,0x04,0x00,0x42,0x00,0x00,0x00,0x43,0x01,0x00,0x00, -0x42,0x01,0x00,0x00,0x0c,0x00,0x08,0x00,0x42,0x00,0x00,0x00, -0x46,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00, -0xb5,0x00,0x00,0x00,0x43,0x01,0x00,0x00,0x96,0x01,0x00,0x00, -0x41,0x00,0x06,0x00,0x53,0x00,0x00,0x00,0x47,0x01,0x00,0x00, -0x12,0x01,0x00,0x00,0x09,0x00,0x00,0x00,0x2d,0x01,0x00,0x00, -0x3e,0x00,0x03,0x00,0x47,0x01,0x00,0x00,0x46,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x49,0x01,0x00,0x00, -0x64,0x00,0x00,0x00,0x66,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x46,0x00,0x00,0x00,0x4e,0x01,0x00,0x00,0x17,0x01,0x00,0x00, -0xc2,0x00,0x05,0x00,0x46,0x00,0x00,0x00,0x4f,0x01,0x00,0x00, -0x4e,0x01,0x00,0x00,0x70,0x00,0x00,0x00,0x71,0x00,0x04,0x00, -0x14,0x00,0x00,0x00,0x50,0x01,0x00,0x00,0x4f,0x01,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x51,0x01,0x00,0x00, -0x50,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0x46,0x00,0x00,0x00, -0x55,0x01,0x00,0x00,0x1e,0x01,0x00,0x00,0xc7,0x00,0x05,0x00, -0x46,0x00,0x00,0x00,0x57,0x01,0x00,0x00,0x55,0x01,0x00,0x00, -0x0e,0x01,0x00,0x00,0x71,0x00,0x04,0x00,0x14,0x00,0x00,0x00, -0x58,0x01,0x00,0x00,0x57,0x01,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x59,0x01,0x00,0x00,0x58,0x01,0x00,0x00, -0xab,0x00,0x05,0x00,0x11,0x00,0x00,0x00,0x5a,0x01,0x00,0x00, -0x59,0x01,0x00,0x00,0x09,0x00,0x00,0x00,0xa9,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x5b,0x01,0x00,0x00,0x5a,0x01,0x00,0x00, -0x39,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x5c,0x01,0x00,0x00,0x51,0x01,0x00,0x00, -0x5b,0x01,0x00,0x00,0x6f,0x00,0x04,0x00,0x42,0x00,0x00,0x00, -0x5d,0x01,0x00,0x00,0x5c,0x01,0x00,0x00,0x7f,0x00,0x04,0x00, -0x42,0x00,0x00,0x00,0x98,0x01,0x00,0x00,0x01,0x01,0x00,0x00, -0x0c,0x00,0x08,0x00,0x42,0x00,0x00,0x00,0x60,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0xfc,0x00,0x00,0x00, -0x5d,0x01,0x00,0x00,0x98,0x01,0x00,0x00,0x41,0x00,0x06,0x00, -0x53,0x00,0x00,0x00,0x61,0x01,0x00,0x00,0x12,0x01,0x00,0x00, -0x09,0x00,0x00,0x00,0x49,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0x61,0x01,0x00,0x00,0x60,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x64,0x01,0x00,0x00,0x64,0x00,0x00,0x00, -0x63,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0x46,0x00,0x00,0x00, -0x6a,0x01,0x00,0x00,0x32,0x01,0x00,0x00,0xc2,0x00,0x05,0x00, -0x46,0x00,0x00,0x00,0x6b,0x01,0x00,0x00,0x6a,0x01,0x00,0x00, -0x70,0x00,0x00,0x00,0x71,0x00,0x04,0x00,0x14,0x00,0x00,0x00, -0x6c,0x01,0x00,0x00,0x6b,0x01,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x6d,0x01,0x00,0x00,0x6c,0x01,0x00,0x00, -0x3d,0x00,0x04,0x00,0x46,0x00,0x00,0x00,0x72,0x01,0x00,0x00, -0x3a,0x01,0x00,0x00,0xc7,0x00,0x05,0x00,0x46,0x00,0x00,0x00, -0x74,0x01,0x00,0x00,0x72,0x01,0x00,0x00,0x0e,0x01,0x00,0x00, -0x71,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0x75,0x01,0x00,0x00, -0x74,0x01,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x76,0x01,0x00,0x00,0x75,0x01,0x00,0x00,0xab,0x00,0x05,0x00, -0x11,0x00,0x00,0x00,0x77,0x01,0x00,0x00,0x76,0x01,0x00,0x00, -0x09,0x00,0x00,0x00,0xa9,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x78,0x01,0x00,0x00,0x77,0x01,0x00,0x00,0x39,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x79,0x01,0x00,0x00,0x6d,0x01,0x00,0x00,0x78,0x01,0x00,0x00, -0x6f,0x00,0x04,0x00,0x42,0x00,0x00,0x00,0x7a,0x01,0x00,0x00, -0x79,0x01,0x00,0x00,0x0c,0x00,0x08,0x00,0x42,0x00,0x00,0x00, -0x7d,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00, -0xfc,0x00,0x00,0x00,0x7a,0x01,0x00,0x00,0x98,0x01,0x00,0x00, -0x41,0x00,0x06,0x00,0x53,0x00,0x00,0x00,0x7e,0x01,0x00,0x00, -0x12,0x01,0x00,0x00,0x09,0x00,0x00,0x00,0x64,0x01,0x00,0x00, -0x3e,0x00,0x03,0x00,0x7e,0x01,0x00,0x00,0x7d,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0x0d,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x0d,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x80,0x01,0x00,0x00,0x8b,0x01,0x00,0x00,0x29,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x0a,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x0c,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x11,0x00,0x00,0x00, -0x92,0x01,0x00,0x00,0x85,0x01,0x00,0x00,0x0a,0x00,0x00,0x00, -0x88,0x01,0x00,0x00,0x2f,0x00,0x00,0x00,0xf7,0x00,0x03,0x00, -0x89,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x92,0x01,0x00,0x00,0x83,0x01,0x00,0x00,0x89,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x89,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0x83,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x83,0x01,0x00,0x00, -0xfd,0x00,0x01,0x00,0x38,0x00,0x01,0x00, -}; -const uint64_t dequant_q5_K_len = 5888; - -unsigned char dequant_q5_K_fp32_data[] = { -0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, 0xa0,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, 0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x27,0x00,0x00,0x00, 0x11,0x00,0x02,0x00,0x51,0x11,0x00,0x00,0x11,0x00,0x02,0x00, @@ -12162,366 +6653,10 @@ unsigned char dequant_q5_K_fp32_data[] = { 0x8a,0x01,0x00,0x00,0xfd,0x00,0x01,0x00,0x38,0x00,0x01,0x00, }; -const uint64_t dequant_q5_K_fp32_len = 5988; +const uint64_t dequant_q5_K_len = 5988; unsigned char dequant_q6_K_data[] = { 0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, -0x0a,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, -0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x09,0x00,0x00,0x00, -0x11,0x00,0x02,0x00,0x27,0x00,0x00,0x00,0x11,0x00,0x02,0x00, -0x51,0x11,0x00,0x00,0x11,0x00,0x02,0x00,0x60,0x11,0x00,0x00, -0x0b,0x00,0x06,0x00,0x01,0x00,0x00,0x00,0x47,0x4c,0x53,0x4c, -0x2e,0x73,0x74,0x64,0x2e,0x34,0x35,0x30,0x00,0x00,0x00,0x00, -0x0e,0x00,0x03,0x00,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x0f,0x00,0x0a,0x00,0x05,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x6d,0x61,0x69,0x6e,0x00,0x00,0x00,0x00,0x17,0x00,0x00,0x00, -0x25,0x00,0x00,0x00,0x33,0x00,0x00,0x00,0x66,0x00,0x00,0x00, -0x79,0x00,0x00,0x00,0x10,0x00,0x06,0x00,0x04,0x00,0x00,0x00, -0x11,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x17,0x00,0x00,0x00, -0x0b,0x00,0x00,0x00,0x1a,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x23,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x23,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x23,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x23,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x33,0x00,0x00,0x00, -0x0b,0x00,0x00,0x00,0x1b,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x5b,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x5d,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x60,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x62,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x62,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x80,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x62,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0xc0,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x62,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0xd0,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x63,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0xd2,0x00,0x00,0x00,0x48,0x00,0x04,0x00, -0x64,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x64,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, -0x64,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x66,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x66,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x76,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x48,0x00,0x04,0x00, -0x77,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x19,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x77,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, -0x77,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x79,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x79,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xff,0x00,0x00,0x00, -0x0b,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x13,0x00,0x02,0x00, -0x02,0x00,0x00,0x00,0x21,0x00,0x03,0x00,0x03,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x15,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x10,0x00,0x00,0x00, -0x00,0x01,0x00,0x00,0x14,0x00,0x02,0x00,0x11,0x00,0x00,0x00, -0x15,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x17,0x00,0x04,0x00,0x15,0x00,0x00,0x00, -0x14,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x16,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x15,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x16,0x00,0x00,0x00,0x17,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x14,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x19,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x14,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0x1c,0x00,0x00,0x00, -0x00,0x01,0x00,0x00,0x1e,0x00,0x06,0x00,0x23,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x24,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x24,0x00,0x00,0x00,0x25,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x26,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x29,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x16,0x00,0x00,0x00,0x33,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x41,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x45,0x00,0x00,0x00,0x10,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x4b,0x00,0x00,0x00, -0x80,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x52,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x15,0x00,0x04,0x00, -0x57,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0x5a,0x00,0x00,0x00, -0x80,0x00,0x00,0x00,0x1c,0x00,0x04,0x00,0x5b,0x00,0x00,0x00, -0x57,0x00,0x00,0x00,0x5a,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x14,0x00,0x00,0x00,0x5c,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x1c,0x00,0x04,0x00,0x5d,0x00,0x00,0x00,0x57,0x00,0x00,0x00, -0x5c,0x00,0x00,0x00,0x15,0x00,0x04,0x00,0x5e,0x00,0x00,0x00, -0x08,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x14,0x00,0x00,0x00,0x5f,0x00,0x00,0x00,0x10,0x00,0x00,0x00, -0x1c,0x00,0x04,0x00,0x60,0x00,0x00,0x00,0x5e,0x00,0x00,0x00, -0x5f,0x00,0x00,0x00,0x16,0x00,0x03,0x00,0x61,0x00,0x00,0x00, -0x10,0x00,0x00,0x00,0x1e,0x00,0x06,0x00,0x62,0x00,0x00,0x00, -0x5b,0x00,0x00,0x00,0x5d,0x00,0x00,0x00,0x60,0x00,0x00,0x00, -0x61,0x00,0x00,0x00,0x1d,0x00,0x03,0x00,0x63,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x1e,0x00,0x03,0x00,0x64,0x00,0x00,0x00, -0x63,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x65,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x64,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x65,0x00,0x00,0x00,0x66,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x6c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x57,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x72,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x73,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x61,0x00,0x00,0x00, -0x1d,0x00,0x03,0x00,0x76,0x00,0x00,0x00,0x61,0x00,0x00,0x00, -0x1e,0x00,0x03,0x00,0x77,0x00,0x00,0x00,0x76,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x78,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x77,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x78,0x00,0x00,0x00, -0x79,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x7e,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x81,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x5e,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x8c,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x93,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xdc,0x00,0x00,0x00, -0x60,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xe1,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x14,0x00,0x00,0x00,0xfe,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x2c,0x00,0x06,0x00,0x15,0x00,0x00,0x00,0xff,0x00,0x00,0x00, -0x5c,0x00,0x00,0x00,0xfe,0x00,0x00,0x00,0xfe,0x00,0x00,0x00, -0x2a,0x00,0x03,0x00,0x11,0x00,0x00,0x00,0x02,0x01,0x00,0x00, -0x29,0x00,0x03,0x00,0x11,0x00,0x00,0x00,0x05,0x01,0x00,0x00, -0x36,0x00,0x05,0x00,0x02,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x05,0x00,0x00,0x00,0xf7,0x00,0x03,0x00,0x00,0x01,0x00,0x00, -0x00,0x00,0x00,0x00,0xfb,0x00,0x03,0x00,0x18,0x00,0x00,0x00, -0x01,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x01,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0x0a,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x0a,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0x08,0x01,0x00,0x00,0x09,0x00,0x00,0x00,0x01,0x01,0x00,0x00, -0xfd,0x00,0x00,0x00,0x0d,0x00,0x00,0x00,0xb1,0x00,0x05,0x00, -0x11,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x08,0x01,0x00,0x00, -0x10,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x0c,0x00,0x00,0x00, -0x0d,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x12,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0x0b,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x19,0x00,0x00,0x00,0x1a,0x00,0x00,0x00,0x17,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x14,0x00,0x00,0x00, -0x1b,0x00,0x00,0x00,0x1a,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x14,0x00,0x00,0x00,0x1d,0x00,0x00,0x00,0x1b,0x00,0x00,0x00, -0x1c,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x14,0x00,0x00,0x00, -0x1f,0x00,0x00,0x00,0x08,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x14,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x1d,0x00,0x00,0x00, -0x1f,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x21,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x26,0x00,0x00,0x00,0x27,0x00,0x00,0x00,0x25,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x28,0x00,0x00,0x00,0x27,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x26,0x00,0x00,0x00,0x2a,0x00,0x00,0x00,0x25,0x00,0x00,0x00, -0x29,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x2b,0x00,0x00,0x00,0x2a,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x2c,0x00,0x00,0x00,0x28,0x00,0x00,0x00, -0x2b,0x00,0x00,0x00,0x87,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x2d,0x00,0x00,0x00,0x2c,0x00,0x00,0x00,0x10,0x00,0x00,0x00, -0xaf,0x00,0x05,0x00,0x11,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x21,0x00,0x00,0x00,0x2d,0x00,0x00,0x00,0xf7,0x00,0x03,0x00, -0x30,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x2e,0x00,0x00,0x00,0x2f,0x00,0x00,0x00,0x30,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0x2f,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x0c,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x30,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x19,0x00,0x00,0x00,0x34,0x00,0x00,0x00, -0x33,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x14,0x00,0x00,0x00,0x35,0x00,0x00,0x00,0x34,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x36,0x00,0x00,0x00, -0x35,0x00,0x00,0x00,0x87,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x3a,0x00,0x00,0x00,0x36,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3e,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x82,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x3f,0x00,0x00,0x00,0x36,0x00,0x00,0x00, -0x3e,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x43,0x00,0x00,0x00,0x41,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, -0x87,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x46,0x00,0x00,0x00, -0x3f,0x00,0x00,0x00,0x45,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x47,0x00,0x00,0x00,0x43,0x00,0x00,0x00, -0x46,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x4a,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x10,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x4d,0x00,0x00,0x00, -0x4b,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x4a,0x00,0x00,0x00, -0x4d,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x50,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x3f,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x54,0x00,0x00,0x00, -0x52,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x56,0x00,0x00,0x00,0x54,0x00,0x00,0x00, -0x3f,0x00,0x00,0x00,0x41,0x00,0x08,0x00,0x6c,0x00,0x00,0x00, -0x6d,0x00,0x00,0x00,0x66,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x21,0x00,0x00,0x00,0x29,0x00,0x00,0x00,0x36,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x57,0x00,0x00,0x00,0x6e,0x00,0x00,0x00, -0x6d,0x00,0x00,0x00,0x41,0x00,0x07,0x00,0x73,0x00,0x00,0x00, -0x74,0x00,0x00,0x00,0x66,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x21,0x00,0x00,0x00,0x72,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x61,0x00,0x00,0x00,0x75,0x00,0x00,0x00,0x74,0x00,0x00,0x00, -0x41,0x00,0x08,0x00,0x81,0x00,0x00,0x00,0x82,0x00,0x00,0x00, -0x66,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x7e,0x00,0x00,0x00,0x47,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x5e,0x00,0x00,0x00,0x83,0x00,0x00,0x00,0x82,0x00,0x00,0x00, -0x72,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0x83,0x00,0x00,0x00,0x41,0x00,0x08,0x00,0x6c,0x00,0x00,0x00, -0x88,0x00,0x00,0x00,0x66,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x21,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x56,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x57,0x00,0x00,0x00,0x89,0x00,0x00,0x00, -0x88,0x00,0x00,0x00,0x71,0x00,0x04,0x00,0x14,0x00,0x00,0x00, -0x8a,0x00,0x00,0x00,0x89,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x8b,0x00,0x00,0x00,0x8a,0x00,0x00,0x00, -0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8d,0x00,0x00,0x00, -0x8b,0x00,0x00,0x00,0x8c,0x00,0x00,0x00,0xc2,0x00,0x05,0x00, -0x57,0x00,0x00,0x00,0x8f,0x00,0x00,0x00,0x6e,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x71,0x00,0x04,0x00,0x14,0x00,0x00,0x00, -0x90,0x00,0x00,0x00,0x8f,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x91,0x00,0x00,0x00,0x90,0x00,0x00,0x00, -0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x92,0x00,0x00,0x00, -0x91,0x00,0x00,0x00,0x72,0x00,0x00,0x00,0xc4,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x94,0x00,0x00,0x00,0x92,0x00,0x00,0x00, -0x93,0x00,0x00,0x00,0xc5,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x95,0x00,0x00,0x00,0x8d,0x00,0x00,0x00,0x94,0x00,0x00,0x00, -0x72,0x00,0x04,0x00,0x5e,0x00,0x00,0x00,0x96,0x00,0x00,0x00, -0x95,0x00,0x00,0x00,0x72,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x97,0x00,0x00,0x00,0x96,0x00,0x00,0x00,0x82,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x98,0x00,0x00,0x00,0x97,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x99,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x98,0x00,0x00,0x00, -0x6f,0x00,0x04,0x00,0x61,0x00,0x00,0x00,0x9a,0x00,0x00,0x00, -0x99,0x00,0x00,0x00,0x85,0x00,0x05,0x00,0x61,0x00,0x00,0x00, -0x9b,0x00,0x00,0x00,0x75,0x00,0x00,0x00,0x9a,0x00,0x00,0x00, -0x41,0x00,0x06,0x00,0x73,0x00,0x00,0x00,0x9c,0x00,0x00,0x00, -0x79,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x50,0x00,0x00,0x00, -0x3e,0x00,0x03,0x00,0x9c,0x00,0x00,0x00,0x9b,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9e,0x00,0x00,0x00, -0x50,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xa2,0x00,0x00,0x00,0x47,0x00,0x00,0x00, -0x7e,0x00,0x00,0x00,0x41,0x00,0x08,0x00,0x81,0x00,0x00,0x00, -0xa3,0x00,0x00,0x00,0x66,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x21,0x00,0x00,0x00,0x7e,0x00,0x00,0x00,0xa2,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x5e,0x00,0x00,0x00,0xa4,0x00,0x00,0x00, -0xa3,0x00,0x00,0x00,0x72,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xa5,0x00,0x00,0x00,0xa4,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xa8,0x00,0x00,0x00,0x56,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x41,0x00,0x08,0x00,0x6c,0x00,0x00,0x00, -0xa9,0x00,0x00,0x00,0x66,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x21,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0xa8,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x57,0x00,0x00,0x00,0xaa,0x00,0x00,0x00, -0xa9,0x00,0x00,0x00,0x71,0x00,0x04,0x00,0x14,0x00,0x00,0x00, -0xab,0x00,0x00,0x00,0xaa,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xac,0x00,0x00,0x00,0xab,0x00,0x00,0x00, -0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xad,0x00,0x00,0x00, -0xac,0x00,0x00,0x00,0x8c,0x00,0x00,0x00,0xc2,0x00,0x05,0x00, -0x57,0x00,0x00,0x00,0xaf,0x00,0x00,0x00,0x6e,0x00,0x00,0x00, -0x7e,0x00,0x00,0x00,0x71,0x00,0x04,0x00,0x14,0x00,0x00,0x00, -0xb0,0x00,0x00,0x00,0xaf,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xb1,0x00,0x00,0x00,0xb0,0x00,0x00,0x00, -0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xb2,0x00,0x00,0x00, -0xb1,0x00,0x00,0x00,0x72,0x00,0x00,0x00,0xc4,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xb3,0x00,0x00,0x00,0xb2,0x00,0x00,0x00, -0x93,0x00,0x00,0x00,0xc5,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xb4,0x00,0x00,0x00,0xad,0x00,0x00,0x00,0xb3,0x00,0x00,0x00, -0x72,0x00,0x04,0x00,0x5e,0x00,0x00,0x00,0xb5,0x00,0x00,0x00, -0xb4,0x00,0x00,0x00,0x72,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xb6,0x00,0x00,0x00,0xb5,0x00,0x00,0x00,0x82,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xb7,0x00,0x00,0x00,0xb6,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xb8,0x00,0x00,0x00,0xa5,0x00,0x00,0x00,0xb7,0x00,0x00,0x00, -0x6f,0x00,0x04,0x00,0x61,0x00,0x00,0x00,0xb9,0x00,0x00,0x00, -0xb8,0x00,0x00,0x00,0x85,0x00,0x05,0x00,0x61,0x00,0x00,0x00, -0xba,0x00,0x00,0x00,0x75,0x00,0x00,0x00,0xb9,0x00,0x00,0x00, -0x41,0x00,0x06,0x00,0x73,0x00,0x00,0x00,0xbb,0x00,0x00,0x00, -0x79,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x9e,0x00,0x00,0x00, -0x3e,0x00,0x03,0x00,0xbb,0x00,0x00,0x00,0xba,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xbd,0x00,0x00,0x00, -0x50,0x00,0x00,0x00,0x52,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xc1,0x00,0x00,0x00,0x47,0x00,0x00,0x00, -0x93,0x00,0x00,0x00,0x41,0x00,0x08,0x00,0x81,0x00,0x00,0x00, -0xc2,0x00,0x00,0x00,0x66,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x21,0x00,0x00,0x00,0x7e,0x00,0x00,0x00,0xc1,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x5e,0x00,0x00,0x00,0xc3,0x00,0x00,0x00, -0xc2,0x00,0x00,0x00,0x72,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xc4,0x00,0x00,0x00,0xc3,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x57,0x00,0x00,0x00,0xc9,0x00,0x00,0x00,0x88,0x00,0x00,0x00, -0xc2,0x00,0x05,0x00,0x57,0x00,0x00,0x00,0xca,0x00,0x00,0x00, -0xc9,0x00,0x00,0x00,0x93,0x00,0x00,0x00,0x71,0x00,0x04,0x00, -0x14,0x00,0x00,0x00,0xcb,0x00,0x00,0x00,0xca,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xcc,0x00,0x00,0x00, -0xcb,0x00,0x00,0x00,0xc2,0x00,0x05,0x00,0x57,0x00,0x00,0x00, -0xce,0x00,0x00,0x00,0x6e,0x00,0x00,0x00,0x93,0x00,0x00,0x00, -0x71,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0xcf,0x00,0x00,0x00, -0xce,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xd0,0x00,0x00,0x00,0xcf,0x00,0x00,0x00,0xc7,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xd1,0x00,0x00,0x00,0xd0,0x00,0x00,0x00, -0x72,0x00,0x00,0x00,0xc4,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xd2,0x00,0x00,0x00,0xd1,0x00,0x00,0x00,0x93,0x00,0x00,0x00, -0xc5,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xd3,0x00,0x00,0x00, -0xcc,0x00,0x00,0x00,0xd2,0x00,0x00,0x00,0x72,0x00,0x04,0x00, -0x5e,0x00,0x00,0x00,0xd4,0x00,0x00,0x00,0xd3,0x00,0x00,0x00, -0x72,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xd5,0x00,0x00,0x00, -0xd4,0x00,0x00,0x00,0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xd6,0x00,0x00,0x00,0xd5,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xd7,0x00,0x00,0x00, -0xc4,0x00,0x00,0x00,0xd6,0x00,0x00,0x00,0x6f,0x00,0x04,0x00, -0x61,0x00,0x00,0x00,0xd8,0x00,0x00,0x00,0xd7,0x00,0x00,0x00, -0x85,0x00,0x05,0x00,0x61,0x00,0x00,0x00,0xd9,0x00,0x00,0x00, -0x75,0x00,0x00,0x00,0xd8,0x00,0x00,0x00,0x41,0x00,0x06,0x00, -0x73,0x00,0x00,0x00,0xda,0x00,0x00,0x00,0x79,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0xbd,0x00,0x00,0x00,0x3e,0x00,0x03,0x00, -0xda,0x00,0x00,0x00,0xd9,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xdd,0x00,0x00,0x00,0x50,0x00,0x00,0x00, -0xdc,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xe2,0x00,0x00,0x00,0x47,0x00,0x00,0x00,0xe1,0x00,0x00,0x00, -0x41,0x00,0x08,0x00,0x81,0x00,0x00,0x00,0xe3,0x00,0x00,0x00, -0x66,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x7e,0x00,0x00,0x00,0xe2,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x5e,0x00,0x00,0x00,0xe4,0x00,0x00,0x00,0xe3,0x00,0x00,0x00, -0x72,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xe5,0x00,0x00,0x00, -0xe4,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x57,0x00,0x00,0x00, -0xea,0x00,0x00,0x00,0xa9,0x00,0x00,0x00,0xc2,0x00,0x05,0x00, -0x57,0x00,0x00,0x00,0xeb,0x00,0x00,0x00,0xea,0x00,0x00,0x00, -0x93,0x00,0x00,0x00,0x71,0x00,0x04,0x00,0x14,0x00,0x00,0x00, -0xec,0x00,0x00,0x00,0xeb,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xed,0x00,0x00,0x00,0xec,0x00,0x00,0x00, -0xc2,0x00,0x05,0x00,0x57,0x00,0x00,0x00,0xef,0x00,0x00,0x00, -0x6e,0x00,0x00,0x00,0xe1,0x00,0x00,0x00,0x71,0x00,0x04,0x00, -0x14,0x00,0x00,0x00,0xf0,0x00,0x00,0x00,0xef,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xf1,0x00,0x00,0x00, -0xf0,0x00,0x00,0x00,0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xf2,0x00,0x00,0x00,0xf1,0x00,0x00,0x00,0x72,0x00,0x00,0x00, -0xc4,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xf3,0x00,0x00,0x00, -0xf2,0x00,0x00,0x00,0x93,0x00,0x00,0x00,0xc5,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xf4,0x00,0x00,0x00,0xed,0x00,0x00,0x00, -0xf3,0x00,0x00,0x00,0x72,0x00,0x04,0x00,0x5e,0x00,0x00,0x00, -0xf5,0x00,0x00,0x00,0xf4,0x00,0x00,0x00,0x72,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xf6,0x00,0x00,0x00,0xf5,0x00,0x00,0x00, -0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xf7,0x00,0x00,0x00, -0xf6,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xf8,0x00,0x00,0x00,0xe5,0x00,0x00,0x00, -0xf7,0x00,0x00,0x00,0x6f,0x00,0x04,0x00,0x61,0x00,0x00,0x00, -0xf9,0x00,0x00,0x00,0xf8,0x00,0x00,0x00,0x85,0x00,0x05,0x00, -0x61,0x00,0x00,0x00,0xfa,0x00,0x00,0x00,0x75,0x00,0x00,0x00, -0xf9,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x73,0x00,0x00,0x00, -0xfb,0x00,0x00,0x00,0x79,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0xdd,0x00,0x00,0x00,0x3e,0x00,0x03,0x00,0xfb,0x00,0x00,0x00, -0xfa,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x0d,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0x0d,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xfd,0x00,0x00,0x00,0x08,0x01,0x00,0x00, -0x29,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x0a,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0x0c,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, -0x11,0x00,0x00,0x00,0x09,0x01,0x00,0x00,0x02,0x01,0x00,0x00, -0x0a,0x00,0x00,0x00,0x05,0x01,0x00,0x00,0x2f,0x00,0x00,0x00, -0xf7,0x00,0x03,0x00,0x06,0x01,0x00,0x00,0x00,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x09,0x01,0x00,0x00,0x00,0x01,0x00,0x00, -0x06,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x06,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0x00,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x00,0x01,0x00,0x00,0xfd,0x00,0x01,0x00,0x38,0x00,0x01,0x00, - -}; -const uint64_t dequant_q6_K_len = 4212; - -unsigned char dequant_q6_K_fp32_data[] = { -0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, 0x10,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, 0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x27,0x00,0x00,0x00, 0x11,0x00,0x02,0x00,0x51,0x11,0x00,0x00,0x11,0x00,0x02,0x00, @@ -12881,647 +7016,10 @@ unsigned char dequant_q6_K_fp32_data[] = { 0x06,0x01,0x00,0x00,0xfd,0x00,0x01,0x00,0x38,0x00,0x01,0x00, }; -const uint64_t dequant_q6_K_fp32_len = 4296; +const uint64_t dequant_q6_K_len = 4296; unsigned char dequant_q8_0_data[] = { 0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, -0xd2,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, -0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x09,0x00,0x00,0x00, -0x11,0x00,0x02,0x00,0x27,0x00,0x00,0x00,0x11,0x00,0x02,0x00, -0x51,0x11,0x00,0x00,0x11,0x00,0x02,0x00,0x60,0x11,0x00,0x00, -0x0b,0x00,0x06,0x00,0x01,0x00,0x00,0x00,0x47,0x4c,0x53,0x4c, -0x2e,0x73,0x74,0x64,0x2e,0x34,0x35,0x30,0x00,0x00,0x00,0x00, -0x0e,0x00,0x03,0x00,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x0f,0x00,0x09,0x00,0x05,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x6d,0x61,0x69,0x6e,0x00,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x6e,0x00,0x00,0x00, -0x10,0x00,0x06,0x00,0x04,0x00,0x00,0x00,0x11,0x00,0x00,0x00, -0x00,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x0c,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x1c,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x14,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x14,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x14,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x08,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x14,0x00,0x00,0x00, -0x03,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x47,0x00,0x03,0x00,0x14,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x4e,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x4f,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x4f,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x50,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x22,0x00,0x00,0x00, -0x48,0x00,0x04,0x00,0x51,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x51,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x03,0x00,0x51,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x53,0x00,0x00,0x00,0x22,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x53,0x00,0x00,0x00, -0x21,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x6b,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x48,0x00,0x04,0x00,0x6c,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x19,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x6c,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x03,0x00,0x6c,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x6e,0x00,0x00,0x00,0x22,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x6e,0x00,0x00,0x00, -0x21,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x8e,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x19,0x00,0x00,0x00, -0x13,0x00,0x02,0x00,0x02,0x00,0x00,0x00,0x21,0x00,0x03,0x00, -0x03,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x15,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x15,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x17,0x00,0x04,0x00,0x0a,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x0b,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x0a,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x0b,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x0d,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x0e,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x1e,0x00,0x06,0x00,0x14,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x15,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x14,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x15,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x17,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x18,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x1b,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x14,0x00,0x02,0x00, -0x24,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x16,0x00,0x03,0x00,0x49,0x00,0x00,0x00,0x10,0x00,0x00,0x00, -0x15,0x00,0x04,0x00,0x4c,0x00,0x00,0x00,0x08,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x4d,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x1c,0x00,0x04,0x00, -0x4e,0x00,0x00,0x00,0x4c,0x00,0x00,0x00,0x4d,0x00,0x00,0x00, -0x1e,0x00,0x04,0x00,0x4f,0x00,0x00,0x00,0x49,0x00,0x00,0x00, -0x4e,0x00,0x00,0x00,0x1d,0x00,0x03,0x00,0x50,0x00,0x00,0x00, -0x4f,0x00,0x00,0x00,0x1e,0x00,0x03,0x00,0x51,0x00,0x00,0x00, -0x50,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x52,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x51,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x52,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x55,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x49,0x00,0x00,0x00,0x17,0x00,0x04,0x00,0x58,0x00,0x00,0x00, -0x49,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x5d,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x4c,0x00,0x00,0x00, -0x1d,0x00,0x03,0x00,0x6b,0x00,0x00,0x00,0x49,0x00,0x00,0x00, -0x1e,0x00,0x03,0x00,0x6c,0x00,0x00,0x00,0x6b,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x6d,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x6d,0x00,0x00,0x00, -0x6e,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x70,0x00,0x00,0x00,0x03,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x87,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x8d,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x2c,0x00,0x06,0x00, -0x0a,0x00,0x00,0x00,0x8e,0x00,0x00,0x00,0x8d,0x00,0x00,0x00, -0x87,0x00,0x00,0x00,0x87,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xb6,0x02,0x00,0x00,0x04,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xb7,0x02,0x00,0x00, -0x05,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xb8,0x02,0x00,0x00,0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xb9,0x02,0x00,0x00,0x07,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xba,0x02,0x00,0x00, -0x08,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xbb,0x02,0x00,0x00,0x09,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xbc,0x02,0x00,0x00,0x0a,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xbd,0x02,0x00,0x00, -0x0b,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xbe,0x02,0x00,0x00,0x0c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xbf,0x02,0x00,0x00,0x0d,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xc0,0x02,0x00,0x00, -0x0e,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xc1,0x02,0x00,0x00,0x0f,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xc2,0x02,0x00,0x00,0x10,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xc3,0x02,0x00,0x00, -0x11,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xc4,0x02,0x00,0x00,0x12,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xc5,0x02,0x00,0x00,0x13,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xc6,0x02,0x00,0x00, -0x14,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xc7,0x02,0x00,0x00,0x15,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xc8,0x02,0x00,0x00,0x16,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xc9,0x02,0x00,0x00, -0x17,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xca,0x02,0x00,0x00,0x18,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xcb,0x02,0x00,0x00,0x19,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xcc,0x02,0x00,0x00, -0x1a,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xcd,0x02,0x00,0x00,0x1b,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xce,0x02,0x00,0x00,0x1c,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xcf,0x02,0x00,0x00, -0x1d,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xd0,0x02,0x00,0x00,0x1e,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xd1,0x02,0x00,0x00,0x1f,0x00,0x00,0x00, -0x36,0x00,0x05,0x00,0x02,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x05,0x00,0x00,0x00,0xf7,0x00,0x03,0x00,0x8f,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0xfb,0x00,0x03,0x00,0x0d,0x00,0x00,0x00, -0x90,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x90,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x0e,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x0d,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x11,0x00,0x00,0x00, -0x10,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x18,0x00,0x00,0x00, -0x19,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x17,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x1a,0x00,0x00,0x00, -0x19,0x00,0x00,0x00,0x87,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x1c,0x00,0x00,0x00,0x1a,0x00,0x00,0x00,0x1b,0x00,0x00,0x00, -0x8b,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x1d,0x00,0x00,0x00, -0x11,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x87,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x11,0x00,0x00,0x00, -0x1c,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x26,0x00,0x00,0x00,0x1d,0x00,0x00,0x00,0x1b,0x00,0x00,0x00, -0xaf,0x00,0x05,0x00,0x24,0x00,0x00,0x00,0x29,0x00,0x00,0x00, -0x26,0x00,0x00,0x00,0x1a,0x00,0x00,0x00,0xa8,0x00,0x04,0x00, -0x24,0x00,0x00,0x00,0x2a,0x00,0x00,0x00,0x29,0x00,0x00,0x00, -0xf7,0x00,0x03,0x00,0x2c,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x2a,0x00,0x00,0x00,0x2b,0x00,0x00,0x00, -0x2c,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x2b,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x18,0x00,0x00,0x00,0x2f,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x30,0x00,0x00,0x00,0x2f,0x00,0x00,0x00, -0xaf,0x00,0x05,0x00,0x24,0x00,0x00,0x00,0x31,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x30,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x2c,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x2c,0x00,0x00,0x00, -0xf5,0x00,0x07,0x00,0x24,0x00,0x00,0x00,0x32,0x00,0x00,0x00, -0x29,0x00,0x00,0x00,0x90,0x00,0x00,0x00,0x31,0x00,0x00,0x00, -0x2b,0x00,0x00,0x00,0xf7,0x00,0x03,0x00,0x34,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x32,0x00,0x00,0x00, -0x33,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x33,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x8f,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0x34,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x18,0x00,0x00,0x00,0x38,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0x37,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x38,0x00,0x00,0x00,0x87,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x1b,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x3e,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x3e,0x00,0x00,0x00,0x1d,0x00,0x00,0x00,0x41,0x00,0x07,0x00, -0x55,0x00,0x00,0x00,0x56,0x00,0x00,0x00,0x53,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x49,0x00,0x00,0x00,0x57,0x00,0x00,0x00, -0x56,0x00,0x00,0x00,0x41,0x00,0x08,0x00,0x5d,0x00,0x00,0x00, -0x5e,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0x17,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4c,0x00,0x00,0x00,0x5f,0x00,0x00,0x00, -0x5e,0x00,0x00,0x00,0x6f,0x00,0x04,0x00,0x49,0x00,0x00,0x00, -0x60,0x00,0x00,0x00,0x5f,0x00,0x00,0x00,0x41,0x00,0x08,0x00, -0x5d,0x00,0x00,0x00,0x64,0x00,0x00,0x00,0x53,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x17,0x00,0x00,0x00, -0x17,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x4c,0x00,0x00,0x00, -0x65,0x00,0x00,0x00,0x64,0x00,0x00,0x00,0x6f,0x00,0x04,0x00, -0x49,0x00,0x00,0x00,0x66,0x00,0x00,0x00,0x65,0x00,0x00,0x00, -0x50,0x00,0x05,0x00,0x58,0x00,0x00,0x00,0x67,0x00,0x00,0x00, -0x60,0x00,0x00,0x00,0x66,0x00,0x00,0x00,0x8e,0x00,0x05,0x00, -0x58,0x00,0x00,0x00,0x6a,0x00,0x00,0x00,0x67,0x00,0x00,0x00, -0x57,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x18,0x00,0x00,0x00, -0x71,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x70,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x72,0x00,0x00,0x00, -0x71,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x73,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x72,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x76,0x00,0x00,0x00, -0x73,0x00,0x00,0x00,0x26,0x00,0x00,0x00,0x51,0x00,0x05,0x00, -0x49,0x00,0x00,0x00,0x7b,0x00,0x00,0x00,0x6a,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x55,0x00,0x00,0x00, -0x7c,0x00,0x00,0x00,0x6e,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x76,0x00,0x00,0x00,0x3e,0x00,0x03,0x00,0x7c,0x00,0x00,0x00, -0x7b,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x86,0x00,0x00,0x00,0x76,0x00,0x00,0x00,0x17,0x00,0x00,0x00, -0x51,0x00,0x05,0x00,0x49,0x00,0x00,0x00,0x89,0x00,0x00,0x00, -0x6a,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x41,0x00,0x06,0x00, -0x55,0x00,0x00,0x00,0x8a,0x00,0x00,0x00,0x6e,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x3e,0x00,0x03,0x00, -0x8a,0x00,0x00,0x00,0x89,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x49,0x00,0x00,0x00,0x9f,0x00,0x00,0x00,0x56,0x00,0x00,0x00, -0x41,0x00,0x08,0x00,0x5d,0x00,0x00,0x00,0xa0,0x00,0x00,0x00, -0x53,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x17,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4c,0x00,0x00,0x00,0xa1,0x00,0x00,0x00,0xa0,0x00,0x00,0x00, -0x6f,0x00,0x04,0x00,0x49,0x00,0x00,0x00,0xa2,0x00,0x00,0x00, -0xa1,0x00,0x00,0x00,0x41,0x00,0x08,0x00,0x5d,0x00,0x00,0x00, -0xa4,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0x17,0x00,0x00,0x00,0x70,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4c,0x00,0x00,0x00,0xa5,0x00,0x00,0x00, -0xa4,0x00,0x00,0x00,0x6f,0x00,0x04,0x00,0x49,0x00,0x00,0x00, -0xa6,0x00,0x00,0x00,0xa5,0x00,0x00,0x00,0x50,0x00,0x05,0x00, -0x58,0x00,0x00,0x00,0xa7,0x00,0x00,0x00,0xa2,0x00,0x00,0x00, -0xa6,0x00,0x00,0x00,0x8e,0x00,0x05,0x00,0x58,0x00,0x00,0x00, -0xa8,0x00,0x00,0x00,0xa7,0x00,0x00,0x00,0x9f,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xae,0x00,0x00,0x00, -0x76,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x51,0x00,0x05,0x00, -0x49,0x00,0x00,0x00,0xb0,0x00,0x00,0x00,0xa8,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x55,0x00,0x00,0x00, -0xb1,0x00,0x00,0x00,0x6e,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0xae,0x00,0x00,0x00,0x3e,0x00,0x03,0x00,0xb1,0x00,0x00,0x00, -0xb0,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xb8,0x00,0x00,0x00,0x76,0x00,0x00,0x00,0x70,0x00,0x00,0x00, -0x51,0x00,0x05,0x00,0x49,0x00,0x00,0x00,0xb9,0x00,0x00,0x00, -0xa8,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x41,0x00,0x06,0x00, -0x55,0x00,0x00,0x00,0xba,0x00,0x00,0x00,0x6e,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0xb8,0x00,0x00,0x00,0x3e,0x00,0x03,0x00, -0xba,0x00,0x00,0x00,0xb9,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x49,0x00,0x00,0x00,0xc3,0x00,0x00,0x00,0x56,0x00,0x00,0x00, -0x41,0x00,0x08,0x00,0x5d,0x00,0x00,0x00,0xc4,0x00,0x00,0x00, -0x53,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x17,0x00,0x00,0x00,0xb6,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4c,0x00,0x00,0x00,0xc5,0x00,0x00,0x00,0xc4,0x00,0x00,0x00, -0x6f,0x00,0x04,0x00,0x49,0x00,0x00,0x00,0xc6,0x00,0x00,0x00, -0xc5,0x00,0x00,0x00,0x41,0x00,0x08,0x00,0x5d,0x00,0x00,0x00, -0xc8,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0x17,0x00,0x00,0x00,0xb7,0x02,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4c,0x00,0x00,0x00,0xc9,0x00,0x00,0x00, -0xc8,0x00,0x00,0x00,0x6f,0x00,0x04,0x00,0x49,0x00,0x00,0x00, -0xca,0x00,0x00,0x00,0xc9,0x00,0x00,0x00,0x50,0x00,0x05,0x00, -0x58,0x00,0x00,0x00,0xcb,0x00,0x00,0x00,0xc6,0x00,0x00,0x00, -0xca,0x00,0x00,0x00,0x8e,0x00,0x05,0x00,0x58,0x00,0x00,0x00, -0xcc,0x00,0x00,0x00,0xcb,0x00,0x00,0x00,0xc3,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xd2,0x00,0x00,0x00, -0x76,0x00,0x00,0x00,0xb6,0x02,0x00,0x00,0x51,0x00,0x05,0x00, -0x49,0x00,0x00,0x00,0xd4,0x00,0x00,0x00,0xcc,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x55,0x00,0x00,0x00, -0xd5,0x00,0x00,0x00,0x6e,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0xd2,0x00,0x00,0x00,0x3e,0x00,0x03,0x00,0xd5,0x00,0x00,0x00, -0xd4,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xdc,0x00,0x00,0x00,0x76,0x00,0x00,0x00,0xb7,0x02,0x00,0x00, -0x51,0x00,0x05,0x00,0x49,0x00,0x00,0x00,0xdd,0x00,0x00,0x00, -0xcc,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x41,0x00,0x06,0x00, -0x55,0x00,0x00,0x00,0xde,0x00,0x00,0x00,0x6e,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0xdc,0x00,0x00,0x00,0x3e,0x00,0x03,0x00, -0xde,0x00,0x00,0x00,0xdd,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x49,0x00,0x00,0x00,0xe7,0x00,0x00,0x00,0x56,0x00,0x00,0x00, -0x41,0x00,0x08,0x00,0x5d,0x00,0x00,0x00,0xe8,0x00,0x00,0x00, -0x53,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x17,0x00,0x00,0x00,0xb8,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4c,0x00,0x00,0x00,0xe9,0x00,0x00,0x00,0xe8,0x00,0x00,0x00, -0x6f,0x00,0x04,0x00,0x49,0x00,0x00,0x00,0xea,0x00,0x00,0x00, -0xe9,0x00,0x00,0x00,0x41,0x00,0x08,0x00,0x5d,0x00,0x00,0x00, -0xec,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0x17,0x00,0x00,0x00,0xb9,0x02,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4c,0x00,0x00,0x00,0xed,0x00,0x00,0x00, -0xec,0x00,0x00,0x00,0x6f,0x00,0x04,0x00,0x49,0x00,0x00,0x00, -0xee,0x00,0x00,0x00,0xed,0x00,0x00,0x00,0x50,0x00,0x05,0x00, -0x58,0x00,0x00,0x00,0xef,0x00,0x00,0x00,0xea,0x00,0x00,0x00, -0xee,0x00,0x00,0x00,0x8e,0x00,0x05,0x00,0x58,0x00,0x00,0x00, -0xf0,0x00,0x00,0x00,0xef,0x00,0x00,0x00,0xe7,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xf6,0x00,0x00,0x00, -0x76,0x00,0x00,0x00,0xb8,0x02,0x00,0x00,0x51,0x00,0x05,0x00, -0x49,0x00,0x00,0x00,0xf8,0x00,0x00,0x00,0xf0,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x55,0x00,0x00,0x00, -0xf9,0x00,0x00,0x00,0x6e,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0xf6,0x00,0x00,0x00,0x3e,0x00,0x03,0x00,0xf9,0x00,0x00,0x00, -0xf8,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x00,0x01,0x00,0x00,0x76,0x00,0x00,0x00,0xb9,0x02,0x00,0x00, -0x51,0x00,0x05,0x00,0x49,0x00,0x00,0x00,0x01,0x01,0x00,0x00, -0xf0,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x41,0x00,0x06,0x00, -0x55,0x00,0x00,0x00,0x02,0x01,0x00,0x00,0x6e,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0x02,0x01,0x00,0x00,0x01,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, -0x49,0x00,0x00,0x00,0x0b,0x01,0x00,0x00,0x56,0x00,0x00,0x00, -0x41,0x00,0x08,0x00,0x5d,0x00,0x00,0x00,0x0c,0x01,0x00,0x00, -0x53,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x17,0x00,0x00,0x00,0xba,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4c,0x00,0x00,0x00,0x0d,0x01,0x00,0x00,0x0c,0x01,0x00,0x00, -0x6f,0x00,0x04,0x00,0x49,0x00,0x00,0x00,0x0e,0x01,0x00,0x00, -0x0d,0x01,0x00,0x00,0x41,0x00,0x08,0x00,0x5d,0x00,0x00,0x00, -0x10,0x01,0x00,0x00,0x53,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0x17,0x00,0x00,0x00,0xbb,0x02,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4c,0x00,0x00,0x00,0x11,0x01,0x00,0x00, -0x10,0x01,0x00,0x00,0x6f,0x00,0x04,0x00,0x49,0x00,0x00,0x00, -0x12,0x01,0x00,0x00,0x11,0x01,0x00,0x00,0x50,0x00,0x05,0x00, -0x58,0x00,0x00,0x00,0x13,0x01,0x00,0x00,0x0e,0x01,0x00,0x00, -0x12,0x01,0x00,0x00,0x8e,0x00,0x05,0x00,0x58,0x00,0x00,0x00, -0x14,0x01,0x00,0x00,0x13,0x01,0x00,0x00,0x0b,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x1a,0x01,0x00,0x00, -0x76,0x00,0x00,0x00,0xba,0x02,0x00,0x00,0x51,0x00,0x05,0x00, -0x49,0x00,0x00,0x00,0x1c,0x01,0x00,0x00,0x14,0x01,0x00,0x00, -0x00,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x55,0x00,0x00,0x00, -0x1d,0x01,0x00,0x00,0x6e,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x1a,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x1d,0x01,0x00,0x00, -0x1c,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x24,0x01,0x00,0x00,0x76,0x00,0x00,0x00,0xbb,0x02,0x00,0x00, -0x51,0x00,0x05,0x00,0x49,0x00,0x00,0x00,0x25,0x01,0x00,0x00, -0x14,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x41,0x00,0x06,0x00, -0x55,0x00,0x00,0x00,0x26,0x01,0x00,0x00,0x6e,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x24,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0x26,0x01,0x00,0x00,0x25,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, -0x49,0x00,0x00,0x00,0x2f,0x01,0x00,0x00,0x56,0x00,0x00,0x00, -0x41,0x00,0x08,0x00,0x5d,0x00,0x00,0x00,0x30,0x01,0x00,0x00, -0x53,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x17,0x00,0x00,0x00,0xbc,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4c,0x00,0x00,0x00,0x31,0x01,0x00,0x00,0x30,0x01,0x00,0x00, -0x6f,0x00,0x04,0x00,0x49,0x00,0x00,0x00,0x32,0x01,0x00,0x00, -0x31,0x01,0x00,0x00,0x41,0x00,0x08,0x00,0x5d,0x00,0x00,0x00, -0x34,0x01,0x00,0x00,0x53,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0x17,0x00,0x00,0x00,0xbd,0x02,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4c,0x00,0x00,0x00,0x35,0x01,0x00,0x00, -0x34,0x01,0x00,0x00,0x6f,0x00,0x04,0x00,0x49,0x00,0x00,0x00, -0x36,0x01,0x00,0x00,0x35,0x01,0x00,0x00,0x50,0x00,0x05,0x00, -0x58,0x00,0x00,0x00,0x37,0x01,0x00,0x00,0x32,0x01,0x00,0x00, -0x36,0x01,0x00,0x00,0x8e,0x00,0x05,0x00,0x58,0x00,0x00,0x00, -0x38,0x01,0x00,0x00,0x37,0x01,0x00,0x00,0x2f,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3e,0x01,0x00,0x00, -0x76,0x00,0x00,0x00,0xbc,0x02,0x00,0x00,0x51,0x00,0x05,0x00, -0x49,0x00,0x00,0x00,0x40,0x01,0x00,0x00,0x38,0x01,0x00,0x00, -0x00,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x55,0x00,0x00,0x00, -0x41,0x01,0x00,0x00,0x6e,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x3e,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x41,0x01,0x00,0x00, -0x40,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x48,0x01,0x00,0x00,0x76,0x00,0x00,0x00,0xbd,0x02,0x00,0x00, -0x51,0x00,0x05,0x00,0x49,0x00,0x00,0x00,0x49,0x01,0x00,0x00, -0x38,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x41,0x00,0x06,0x00, -0x55,0x00,0x00,0x00,0x4a,0x01,0x00,0x00,0x6e,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x48,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0x4a,0x01,0x00,0x00,0x49,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, -0x49,0x00,0x00,0x00,0x53,0x01,0x00,0x00,0x56,0x00,0x00,0x00, -0x41,0x00,0x08,0x00,0x5d,0x00,0x00,0x00,0x54,0x01,0x00,0x00, -0x53,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x17,0x00,0x00,0x00,0xbe,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4c,0x00,0x00,0x00,0x55,0x01,0x00,0x00,0x54,0x01,0x00,0x00, -0x6f,0x00,0x04,0x00,0x49,0x00,0x00,0x00,0x56,0x01,0x00,0x00, -0x55,0x01,0x00,0x00,0x41,0x00,0x08,0x00,0x5d,0x00,0x00,0x00, -0x58,0x01,0x00,0x00,0x53,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0x17,0x00,0x00,0x00,0xbf,0x02,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4c,0x00,0x00,0x00,0x59,0x01,0x00,0x00, -0x58,0x01,0x00,0x00,0x6f,0x00,0x04,0x00,0x49,0x00,0x00,0x00, -0x5a,0x01,0x00,0x00,0x59,0x01,0x00,0x00,0x50,0x00,0x05,0x00, -0x58,0x00,0x00,0x00,0x5b,0x01,0x00,0x00,0x56,0x01,0x00,0x00, -0x5a,0x01,0x00,0x00,0x8e,0x00,0x05,0x00,0x58,0x00,0x00,0x00, -0x5c,0x01,0x00,0x00,0x5b,0x01,0x00,0x00,0x53,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x62,0x01,0x00,0x00, -0x76,0x00,0x00,0x00,0xbe,0x02,0x00,0x00,0x51,0x00,0x05,0x00, -0x49,0x00,0x00,0x00,0x64,0x01,0x00,0x00,0x5c,0x01,0x00,0x00, -0x00,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x55,0x00,0x00,0x00, -0x65,0x01,0x00,0x00,0x6e,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x62,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x65,0x01,0x00,0x00, -0x64,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x6c,0x01,0x00,0x00,0x76,0x00,0x00,0x00,0xbf,0x02,0x00,0x00, -0x51,0x00,0x05,0x00,0x49,0x00,0x00,0x00,0x6d,0x01,0x00,0x00, -0x5c,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x41,0x00,0x06,0x00, -0x55,0x00,0x00,0x00,0x6e,0x01,0x00,0x00,0x6e,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x6c,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0x6e,0x01,0x00,0x00,0x6d,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, -0x49,0x00,0x00,0x00,0x77,0x01,0x00,0x00,0x56,0x00,0x00,0x00, -0x41,0x00,0x08,0x00,0x5d,0x00,0x00,0x00,0x78,0x01,0x00,0x00, -0x53,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x17,0x00,0x00,0x00,0xc0,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4c,0x00,0x00,0x00,0x79,0x01,0x00,0x00,0x78,0x01,0x00,0x00, -0x6f,0x00,0x04,0x00,0x49,0x00,0x00,0x00,0x7a,0x01,0x00,0x00, -0x79,0x01,0x00,0x00,0x41,0x00,0x08,0x00,0x5d,0x00,0x00,0x00, -0x7c,0x01,0x00,0x00,0x53,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0x17,0x00,0x00,0x00,0xc1,0x02,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4c,0x00,0x00,0x00,0x7d,0x01,0x00,0x00, -0x7c,0x01,0x00,0x00,0x6f,0x00,0x04,0x00,0x49,0x00,0x00,0x00, -0x7e,0x01,0x00,0x00,0x7d,0x01,0x00,0x00,0x50,0x00,0x05,0x00, -0x58,0x00,0x00,0x00,0x7f,0x01,0x00,0x00,0x7a,0x01,0x00,0x00, -0x7e,0x01,0x00,0x00,0x8e,0x00,0x05,0x00,0x58,0x00,0x00,0x00, -0x80,0x01,0x00,0x00,0x7f,0x01,0x00,0x00,0x77,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x86,0x01,0x00,0x00, -0x76,0x00,0x00,0x00,0xc0,0x02,0x00,0x00,0x51,0x00,0x05,0x00, -0x49,0x00,0x00,0x00,0x88,0x01,0x00,0x00,0x80,0x01,0x00,0x00, -0x00,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x55,0x00,0x00,0x00, -0x89,0x01,0x00,0x00,0x6e,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x86,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x89,0x01,0x00,0x00, -0x88,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x90,0x01,0x00,0x00,0x76,0x00,0x00,0x00,0xc1,0x02,0x00,0x00, -0x51,0x00,0x05,0x00,0x49,0x00,0x00,0x00,0x91,0x01,0x00,0x00, -0x80,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x41,0x00,0x06,0x00, -0x55,0x00,0x00,0x00,0x92,0x01,0x00,0x00,0x6e,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x90,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0x92,0x01,0x00,0x00,0x91,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, -0x49,0x00,0x00,0x00,0x9b,0x01,0x00,0x00,0x56,0x00,0x00,0x00, -0x41,0x00,0x08,0x00,0x5d,0x00,0x00,0x00,0x9c,0x01,0x00,0x00, -0x53,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x17,0x00,0x00,0x00,0xc2,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4c,0x00,0x00,0x00,0x9d,0x01,0x00,0x00,0x9c,0x01,0x00,0x00, -0x6f,0x00,0x04,0x00,0x49,0x00,0x00,0x00,0x9e,0x01,0x00,0x00, -0x9d,0x01,0x00,0x00,0x41,0x00,0x08,0x00,0x5d,0x00,0x00,0x00, -0xa0,0x01,0x00,0x00,0x53,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0x17,0x00,0x00,0x00,0xc3,0x02,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4c,0x00,0x00,0x00,0xa1,0x01,0x00,0x00, -0xa0,0x01,0x00,0x00,0x6f,0x00,0x04,0x00,0x49,0x00,0x00,0x00, -0xa2,0x01,0x00,0x00,0xa1,0x01,0x00,0x00,0x50,0x00,0x05,0x00, -0x58,0x00,0x00,0x00,0xa3,0x01,0x00,0x00,0x9e,0x01,0x00,0x00, -0xa2,0x01,0x00,0x00,0x8e,0x00,0x05,0x00,0x58,0x00,0x00,0x00, -0xa4,0x01,0x00,0x00,0xa3,0x01,0x00,0x00,0x9b,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xaa,0x01,0x00,0x00, -0x76,0x00,0x00,0x00,0xc2,0x02,0x00,0x00,0x51,0x00,0x05,0x00, -0x49,0x00,0x00,0x00,0xac,0x01,0x00,0x00,0xa4,0x01,0x00,0x00, -0x00,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x55,0x00,0x00,0x00, -0xad,0x01,0x00,0x00,0x6e,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0xaa,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0xad,0x01,0x00,0x00, -0xac,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xb4,0x01,0x00,0x00,0x76,0x00,0x00,0x00,0xc3,0x02,0x00,0x00, -0x51,0x00,0x05,0x00,0x49,0x00,0x00,0x00,0xb5,0x01,0x00,0x00, -0xa4,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x41,0x00,0x06,0x00, -0x55,0x00,0x00,0x00,0xb6,0x01,0x00,0x00,0x6e,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0xb4,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0xb6,0x01,0x00,0x00,0xb5,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, -0x49,0x00,0x00,0x00,0xbf,0x01,0x00,0x00,0x56,0x00,0x00,0x00, -0x41,0x00,0x08,0x00,0x5d,0x00,0x00,0x00,0xc0,0x01,0x00,0x00, -0x53,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x17,0x00,0x00,0x00,0xc4,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4c,0x00,0x00,0x00,0xc1,0x01,0x00,0x00,0xc0,0x01,0x00,0x00, -0x6f,0x00,0x04,0x00,0x49,0x00,0x00,0x00,0xc2,0x01,0x00,0x00, -0xc1,0x01,0x00,0x00,0x41,0x00,0x08,0x00,0x5d,0x00,0x00,0x00, -0xc4,0x01,0x00,0x00,0x53,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0x17,0x00,0x00,0x00,0xc5,0x02,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4c,0x00,0x00,0x00,0xc5,0x01,0x00,0x00, -0xc4,0x01,0x00,0x00,0x6f,0x00,0x04,0x00,0x49,0x00,0x00,0x00, -0xc6,0x01,0x00,0x00,0xc5,0x01,0x00,0x00,0x50,0x00,0x05,0x00, -0x58,0x00,0x00,0x00,0xc7,0x01,0x00,0x00,0xc2,0x01,0x00,0x00, -0xc6,0x01,0x00,0x00,0x8e,0x00,0x05,0x00,0x58,0x00,0x00,0x00, -0xc8,0x01,0x00,0x00,0xc7,0x01,0x00,0x00,0xbf,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xce,0x01,0x00,0x00, -0x76,0x00,0x00,0x00,0xc4,0x02,0x00,0x00,0x51,0x00,0x05,0x00, -0x49,0x00,0x00,0x00,0xd0,0x01,0x00,0x00,0xc8,0x01,0x00,0x00, -0x00,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x55,0x00,0x00,0x00, -0xd1,0x01,0x00,0x00,0x6e,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0xce,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0xd1,0x01,0x00,0x00, -0xd0,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xd8,0x01,0x00,0x00,0x76,0x00,0x00,0x00,0xc5,0x02,0x00,0x00, -0x51,0x00,0x05,0x00,0x49,0x00,0x00,0x00,0xd9,0x01,0x00,0x00, -0xc8,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x41,0x00,0x06,0x00, -0x55,0x00,0x00,0x00,0xda,0x01,0x00,0x00,0x6e,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0xd8,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0xda,0x01,0x00,0x00,0xd9,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, -0x49,0x00,0x00,0x00,0xe3,0x01,0x00,0x00,0x56,0x00,0x00,0x00, -0x41,0x00,0x08,0x00,0x5d,0x00,0x00,0x00,0xe4,0x01,0x00,0x00, -0x53,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x17,0x00,0x00,0x00,0xc6,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4c,0x00,0x00,0x00,0xe5,0x01,0x00,0x00,0xe4,0x01,0x00,0x00, -0x6f,0x00,0x04,0x00,0x49,0x00,0x00,0x00,0xe6,0x01,0x00,0x00, -0xe5,0x01,0x00,0x00,0x41,0x00,0x08,0x00,0x5d,0x00,0x00,0x00, -0xe8,0x01,0x00,0x00,0x53,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0x17,0x00,0x00,0x00,0xc7,0x02,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4c,0x00,0x00,0x00,0xe9,0x01,0x00,0x00, -0xe8,0x01,0x00,0x00,0x6f,0x00,0x04,0x00,0x49,0x00,0x00,0x00, -0xea,0x01,0x00,0x00,0xe9,0x01,0x00,0x00,0x50,0x00,0x05,0x00, -0x58,0x00,0x00,0x00,0xeb,0x01,0x00,0x00,0xe6,0x01,0x00,0x00, -0xea,0x01,0x00,0x00,0x8e,0x00,0x05,0x00,0x58,0x00,0x00,0x00, -0xec,0x01,0x00,0x00,0xeb,0x01,0x00,0x00,0xe3,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xf2,0x01,0x00,0x00, -0x76,0x00,0x00,0x00,0xc6,0x02,0x00,0x00,0x51,0x00,0x05,0x00, -0x49,0x00,0x00,0x00,0xf4,0x01,0x00,0x00,0xec,0x01,0x00,0x00, -0x00,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x55,0x00,0x00,0x00, -0xf5,0x01,0x00,0x00,0x6e,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0xf2,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0xf5,0x01,0x00,0x00, -0xf4,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xfc,0x01,0x00,0x00,0x76,0x00,0x00,0x00,0xc7,0x02,0x00,0x00, -0x51,0x00,0x05,0x00,0x49,0x00,0x00,0x00,0xfd,0x01,0x00,0x00, -0xec,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x41,0x00,0x06,0x00, -0x55,0x00,0x00,0x00,0xfe,0x01,0x00,0x00,0x6e,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0xfc,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0xfe,0x01,0x00,0x00,0xfd,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, -0x49,0x00,0x00,0x00,0x07,0x02,0x00,0x00,0x56,0x00,0x00,0x00, -0x41,0x00,0x08,0x00,0x5d,0x00,0x00,0x00,0x08,0x02,0x00,0x00, -0x53,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x17,0x00,0x00,0x00,0xc8,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4c,0x00,0x00,0x00,0x09,0x02,0x00,0x00,0x08,0x02,0x00,0x00, -0x6f,0x00,0x04,0x00,0x49,0x00,0x00,0x00,0x0a,0x02,0x00,0x00, -0x09,0x02,0x00,0x00,0x41,0x00,0x08,0x00,0x5d,0x00,0x00,0x00, -0x0c,0x02,0x00,0x00,0x53,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0x17,0x00,0x00,0x00,0xc9,0x02,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4c,0x00,0x00,0x00,0x0d,0x02,0x00,0x00, -0x0c,0x02,0x00,0x00,0x6f,0x00,0x04,0x00,0x49,0x00,0x00,0x00, -0x0e,0x02,0x00,0x00,0x0d,0x02,0x00,0x00,0x50,0x00,0x05,0x00, -0x58,0x00,0x00,0x00,0x0f,0x02,0x00,0x00,0x0a,0x02,0x00,0x00, -0x0e,0x02,0x00,0x00,0x8e,0x00,0x05,0x00,0x58,0x00,0x00,0x00, -0x10,0x02,0x00,0x00,0x0f,0x02,0x00,0x00,0x07,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x16,0x02,0x00,0x00, -0x76,0x00,0x00,0x00,0xc8,0x02,0x00,0x00,0x51,0x00,0x05,0x00, -0x49,0x00,0x00,0x00,0x18,0x02,0x00,0x00,0x10,0x02,0x00,0x00, -0x00,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x55,0x00,0x00,0x00, -0x19,0x02,0x00,0x00,0x6e,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x16,0x02,0x00,0x00,0x3e,0x00,0x03,0x00,0x19,0x02,0x00,0x00, -0x18,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x20,0x02,0x00,0x00,0x76,0x00,0x00,0x00,0xc9,0x02,0x00,0x00, -0x51,0x00,0x05,0x00,0x49,0x00,0x00,0x00,0x21,0x02,0x00,0x00, -0x10,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0x41,0x00,0x06,0x00, -0x55,0x00,0x00,0x00,0x22,0x02,0x00,0x00,0x6e,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x20,0x02,0x00,0x00,0x3e,0x00,0x03,0x00, -0x22,0x02,0x00,0x00,0x21,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, -0x49,0x00,0x00,0x00,0x2b,0x02,0x00,0x00,0x56,0x00,0x00,0x00, -0x41,0x00,0x08,0x00,0x5d,0x00,0x00,0x00,0x2c,0x02,0x00,0x00, -0x53,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x17,0x00,0x00,0x00,0xca,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4c,0x00,0x00,0x00,0x2d,0x02,0x00,0x00,0x2c,0x02,0x00,0x00, -0x6f,0x00,0x04,0x00,0x49,0x00,0x00,0x00,0x2e,0x02,0x00,0x00, -0x2d,0x02,0x00,0x00,0x41,0x00,0x08,0x00,0x5d,0x00,0x00,0x00, -0x30,0x02,0x00,0x00,0x53,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0x17,0x00,0x00,0x00,0xcb,0x02,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4c,0x00,0x00,0x00,0x31,0x02,0x00,0x00, -0x30,0x02,0x00,0x00,0x6f,0x00,0x04,0x00,0x49,0x00,0x00,0x00, -0x32,0x02,0x00,0x00,0x31,0x02,0x00,0x00,0x50,0x00,0x05,0x00, -0x58,0x00,0x00,0x00,0x33,0x02,0x00,0x00,0x2e,0x02,0x00,0x00, -0x32,0x02,0x00,0x00,0x8e,0x00,0x05,0x00,0x58,0x00,0x00,0x00, -0x34,0x02,0x00,0x00,0x33,0x02,0x00,0x00,0x2b,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3a,0x02,0x00,0x00, -0x76,0x00,0x00,0x00,0xca,0x02,0x00,0x00,0x51,0x00,0x05,0x00, -0x49,0x00,0x00,0x00,0x3c,0x02,0x00,0x00,0x34,0x02,0x00,0x00, -0x00,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x55,0x00,0x00,0x00, -0x3d,0x02,0x00,0x00,0x6e,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x3a,0x02,0x00,0x00,0x3e,0x00,0x03,0x00,0x3d,0x02,0x00,0x00, -0x3c,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x44,0x02,0x00,0x00,0x76,0x00,0x00,0x00,0xcb,0x02,0x00,0x00, -0x51,0x00,0x05,0x00,0x49,0x00,0x00,0x00,0x45,0x02,0x00,0x00, -0x34,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0x41,0x00,0x06,0x00, -0x55,0x00,0x00,0x00,0x46,0x02,0x00,0x00,0x6e,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x44,0x02,0x00,0x00,0x3e,0x00,0x03,0x00, -0x46,0x02,0x00,0x00,0x45,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, -0x49,0x00,0x00,0x00,0x4f,0x02,0x00,0x00,0x56,0x00,0x00,0x00, -0x41,0x00,0x08,0x00,0x5d,0x00,0x00,0x00,0x50,0x02,0x00,0x00, -0x53,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x17,0x00,0x00,0x00,0xcc,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4c,0x00,0x00,0x00,0x51,0x02,0x00,0x00,0x50,0x02,0x00,0x00, -0x6f,0x00,0x04,0x00,0x49,0x00,0x00,0x00,0x52,0x02,0x00,0x00, -0x51,0x02,0x00,0x00,0x41,0x00,0x08,0x00,0x5d,0x00,0x00,0x00, -0x54,0x02,0x00,0x00,0x53,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0x17,0x00,0x00,0x00,0xcd,0x02,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4c,0x00,0x00,0x00,0x55,0x02,0x00,0x00, -0x54,0x02,0x00,0x00,0x6f,0x00,0x04,0x00,0x49,0x00,0x00,0x00, -0x56,0x02,0x00,0x00,0x55,0x02,0x00,0x00,0x50,0x00,0x05,0x00, -0x58,0x00,0x00,0x00,0x57,0x02,0x00,0x00,0x52,0x02,0x00,0x00, -0x56,0x02,0x00,0x00,0x8e,0x00,0x05,0x00,0x58,0x00,0x00,0x00, -0x58,0x02,0x00,0x00,0x57,0x02,0x00,0x00,0x4f,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x5e,0x02,0x00,0x00, -0x76,0x00,0x00,0x00,0xcc,0x02,0x00,0x00,0x51,0x00,0x05,0x00, -0x49,0x00,0x00,0x00,0x60,0x02,0x00,0x00,0x58,0x02,0x00,0x00, -0x00,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x55,0x00,0x00,0x00, -0x61,0x02,0x00,0x00,0x6e,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x5e,0x02,0x00,0x00,0x3e,0x00,0x03,0x00,0x61,0x02,0x00,0x00, -0x60,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x68,0x02,0x00,0x00,0x76,0x00,0x00,0x00,0xcd,0x02,0x00,0x00, -0x51,0x00,0x05,0x00,0x49,0x00,0x00,0x00,0x69,0x02,0x00,0x00, -0x58,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0x41,0x00,0x06,0x00, -0x55,0x00,0x00,0x00,0x6a,0x02,0x00,0x00,0x6e,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x68,0x02,0x00,0x00,0x3e,0x00,0x03,0x00, -0x6a,0x02,0x00,0x00,0x69,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, -0x49,0x00,0x00,0x00,0x73,0x02,0x00,0x00,0x56,0x00,0x00,0x00, -0x41,0x00,0x08,0x00,0x5d,0x00,0x00,0x00,0x74,0x02,0x00,0x00, -0x53,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x17,0x00,0x00,0x00,0xce,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4c,0x00,0x00,0x00,0x75,0x02,0x00,0x00,0x74,0x02,0x00,0x00, -0x6f,0x00,0x04,0x00,0x49,0x00,0x00,0x00,0x76,0x02,0x00,0x00, -0x75,0x02,0x00,0x00,0x41,0x00,0x08,0x00,0x5d,0x00,0x00,0x00, -0x78,0x02,0x00,0x00,0x53,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0x17,0x00,0x00,0x00,0xcf,0x02,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4c,0x00,0x00,0x00,0x79,0x02,0x00,0x00, -0x78,0x02,0x00,0x00,0x6f,0x00,0x04,0x00,0x49,0x00,0x00,0x00, -0x7a,0x02,0x00,0x00,0x79,0x02,0x00,0x00,0x50,0x00,0x05,0x00, -0x58,0x00,0x00,0x00,0x7b,0x02,0x00,0x00,0x76,0x02,0x00,0x00, -0x7a,0x02,0x00,0x00,0x8e,0x00,0x05,0x00,0x58,0x00,0x00,0x00, -0x7c,0x02,0x00,0x00,0x7b,0x02,0x00,0x00,0x73,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x82,0x02,0x00,0x00, -0x76,0x00,0x00,0x00,0xce,0x02,0x00,0x00,0x51,0x00,0x05,0x00, -0x49,0x00,0x00,0x00,0x84,0x02,0x00,0x00,0x7c,0x02,0x00,0x00, -0x00,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x55,0x00,0x00,0x00, -0x85,0x02,0x00,0x00,0x6e,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x82,0x02,0x00,0x00,0x3e,0x00,0x03,0x00,0x85,0x02,0x00,0x00, -0x84,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x8c,0x02,0x00,0x00,0x76,0x00,0x00,0x00,0xcf,0x02,0x00,0x00, -0x51,0x00,0x05,0x00,0x49,0x00,0x00,0x00,0x8d,0x02,0x00,0x00, -0x7c,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0x41,0x00,0x06,0x00, -0x55,0x00,0x00,0x00,0x8e,0x02,0x00,0x00,0x6e,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x8c,0x02,0x00,0x00,0x3e,0x00,0x03,0x00, -0x8e,0x02,0x00,0x00,0x8d,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, -0x49,0x00,0x00,0x00,0x97,0x02,0x00,0x00,0x56,0x00,0x00,0x00, -0x41,0x00,0x08,0x00,0x5d,0x00,0x00,0x00,0x98,0x02,0x00,0x00, -0x53,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x17,0x00,0x00,0x00,0xd0,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4c,0x00,0x00,0x00,0x99,0x02,0x00,0x00,0x98,0x02,0x00,0x00, -0x6f,0x00,0x04,0x00,0x49,0x00,0x00,0x00,0x9a,0x02,0x00,0x00, -0x99,0x02,0x00,0x00,0x41,0x00,0x08,0x00,0x5d,0x00,0x00,0x00, -0x9c,0x02,0x00,0x00,0x53,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0x17,0x00,0x00,0x00,0xd1,0x02,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4c,0x00,0x00,0x00,0x9d,0x02,0x00,0x00, -0x9c,0x02,0x00,0x00,0x6f,0x00,0x04,0x00,0x49,0x00,0x00,0x00, -0x9e,0x02,0x00,0x00,0x9d,0x02,0x00,0x00,0x50,0x00,0x05,0x00, -0x58,0x00,0x00,0x00,0x9f,0x02,0x00,0x00,0x9a,0x02,0x00,0x00, -0x9e,0x02,0x00,0x00,0x8e,0x00,0x05,0x00,0x58,0x00,0x00,0x00, -0xa0,0x02,0x00,0x00,0x9f,0x02,0x00,0x00,0x97,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa6,0x02,0x00,0x00, -0x76,0x00,0x00,0x00,0xd0,0x02,0x00,0x00,0x51,0x00,0x05,0x00, -0x49,0x00,0x00,0x00,0xa8,0x02,0x00,0x00,0xa0,0x02,0x00,0x00, -0x00,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x55,0x00,0x00,0x00, -0xa9,0x02,0x00,0x00,0x6e,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0xa6,0x02,0x00,0x00,0x3e,0x00,0x03,0x00,0xa9,0x02,0x00,0x00, -0xa8,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xb0,0x02,0x00,0x00,0x76,0x00,0x00,0x00,0xd1,0x02,0x00,0x00, -0x51,0x00,0x05,0x00,0x49,0x00,0x00,0x00,0xb1,0x02,0x00,0x00, -0xa0,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0x41,0x00,0x06,0x00, -0x55,0x00,0x00,0x00,0xb2,0x02,0x00,0x00,0x6e,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0xb0,0x02,0x00,0x00,0x3e,0x00,0x03,0x00, -0xb2,0x02,0x00,0x00,0xb1,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x8f,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x8f,0x00,0x00,0x00, -0xfd,0x00,0x01,0x00,0x38,0x00,0x01,0x00, -}; -const uint64_t dequant_q8_0_len = 7592; - -unsigned char dequant_q8_0_fp32_data[] = { -0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, 0x23,0x03,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, 0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x51,0x11,0x00,0x00, 0x11,0x00,0x02,0x00,0x60,0x11,0x00,0x00,0x0b,0x00,0x06,0x00, @@ -14262,7 +7760,7 @@ unsigned char dequant_q8_0_fp32_data[] = { 0x95,0x00,0x00,0x00,0xfd,0x00,0x01,0x00,0x38,0x00,0x01,0x00, }; -const uint64_t dequant_q8_0_fp32_len = 8868; +const uint64_t dequant_q8_0_len = 8868; unsigned char diag_mask_inf_f32_data[] = { 0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, @@ -14530,144 +8028,6 @@ unsigned char f32_to_f16_data[] = { }; const uint64_t f32_to_f16_len = 1596; -unsigned char f32_to_f16_fp32_data[] = { -0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, -0x53,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, -0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x51,0x11,0x00,0x00, -0x0b,0x00,0x06,0x00,0x01,0x00,0x00,0x00,0x47,0x4c,0x53,0x4c, -0x2e,0x73,0x74,0x64,0x2e,0x34,0x35,0x30,0x00,0x00,0x00,0x00, -0x0e,0x00,0x03,0x00,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x0f,0x00,0x09,0x00,0x05,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x6d,0x61,0x69,0x6e,0x00,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x13,0x00,0x00,0x00,0x36,0x00,0x00,0x00,0x42,0x00,0x00,0x00, -0x10,0x00,0x06,0x00,0x04,0x00,0x00,0x00,0x11,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x0c,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x1c,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x11,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x11,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x11,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x08,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x11,0x00,0x00,0x00, -0x03,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x47,0x00,0x03,0x00,0x11,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x33,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0x34,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x34,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x34,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x36,0x00,0x00,0x00, -0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x36,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x3f,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0x40,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x40,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x40,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x42,0x00,0x00,0x00, -0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x42,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x52,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x19,0x00,0x00,0x00,0x13,0x00,0x02,0x00,0x02,0x00,0x00,0x00, -0x21,0x00,0x03,0x00,0x03,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x15,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x15,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x17,0x00,0x04,0x00, -0x0a,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x03,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x0b,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x0a,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x0b,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x0e,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x1e,0x00,0x06,0x00,0x11,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x12,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x11,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x12,0x00,0x00,0x00,0x13,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x14,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x15,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x14,0x00,0x02,0x00, -0x23,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x2b,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x16,0x00,0x03,0x00, -0x32,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, -0x33,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, -0x34,0x00,0x00,0x00,0x33,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x35,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x34,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x35,0x00,0x00,0x00,0x36,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x38,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x16,0x00,0x03,0x00, -0x3e,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, -0x3f,0x00,0x00,0x00,0x3e,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, -0x40,0x00,0x00,0x00,0x3f,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x41,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x41,0x00,0x00,0x00,0x42,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x44,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x4a,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x3e,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x4e,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x32,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x50,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x51,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x2c,0x00,0x06,0x00,0x0a,0x00,0x00,0x00,0x52,0x00,0x00,0x00, -0x50,0x00,0x00,0x00,0x51,0x00,0x00,0x00,0x51,0x00,0x00,0x00, -0x36,0x00,0x05,0x00,0x02,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x05,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x0e,0x00,0x00,0x00, -0x0f,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x0d,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x10,0x00,0x00,0x00, -0x0f,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x13,0x00,0x00,0x00,0x14,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x17,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x17,0x00,0x00,0x00,0x89,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x10,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x1a,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x10,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x22,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0xb1,0x00,0x05,0x00, -0x23,0x00,0x00,0x00,0x27,0x00,0x00,0x00,0x1a,0x00,0x00,0x00, -0x17,0x00,0x00,0x00,0xf7,0x00,0x03,0x00,0x29,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x27,0x00,0x00,0x00, -0x28,0x00,0x00,0x00,0x29,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x28,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0x2c,0x00,0x00,0x00,0x13,0x00,0x00,0x00,0x2b,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x2d,0x00,0x00,0x00, -0x2c,0x00,0x00,0x00,0xb1,0x00,0x05,0x00,0x23,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x2d,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x29,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x29,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x23,0x00,0x00,0x00, -0x2f,0x00,0x00,0x00,0x27,0x00,0x00,0x00,0x05,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0xf7,0x00,0x03,0x00, -0x31,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x2f,0x00,0x00,0x00,0x30,0x00,0x00,0x00,0x31,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0x30,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x13,0x00,0x00,0x00, -0x38,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x3a,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x3b,0x00,0x00,0x00,0x22,0x00,0x00,0x00, -0x3a,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x3d,0x00,0x00,0x00,0x3b,0x00,0x00,0x00,0x1a,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x45,0x00,0x00,0x00, -0x13,0x00,0x00,0x00,0x44,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x46,0x00,0x00,0x00,0x45,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x47,0x00,0x00,0x00, -0x22,0x00,0x00,0x00,0x46,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x49,0x00,0x00,0x00,0x47,0x00,0x00,0x00, -0x1a,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x4a,0x00,0x00,0x00, -0x4b,0x00,0x00,0x00,0x42,0x00,0x00,0x00,0x2b,0x00,0x00,0x00, -0x49,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x3e,0x00,0x00,0x00, -0x4c,0x00,0x00,0x00,0x4b,0x00,0x00,0x00,0x73,0x00,0x04,0x00, -0x32,0x00,0x00,0x00,0x4d,0x00,0x00,0x00,0x4c,0x00,0x00,0x00, -0x41,0x00,0x06,0x00,0x4e,0x00,0x00,0x00,0x4f,0x00,0x00,0x00, -0x36,0x00,0x00,0x00,0x2b,0x00,0x00,0x00,0x3d,0x00,0x00,0x00, -0x3e,0x00,0x03,0x00,0x4f,0x00,0x00,0x00,0x4d,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x31,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x31,0x00,0x00,0x00,0xfd,0x00,0x01,0x00,0x38,0x00,0x01,0x00, - -}; -const uint64_t f32_to_f16_fp32_len = 1596; - unsigned char gelu_f32_data[] = { 0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, 0x4b,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, @@ -14798,500 +8158,6 @@ const uint64_t gelu_f32_len = 1484; unsigned char get_rows_f16_data[] = { 0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, -0x77,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, -0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x09,0x00,0x00,0x00, -0x11,0x00,0x02,0x00,0x51,0x11,0x00,0x00,0x0b,0x00,0x06,0x00, -0x01,0x00,0x00,0x00,0x47,0x4c,0x53,0x4c,0x2e,0x73,0x74,0x64, -0x2e,0x34,0x35,0x30,0x00,0x00,0x00,0x00,0x0e,0x00,0x03,0x00, -0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x0f,0x00,0x0a,0x00, -0x05,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x6d,0x61,0x69,0x6e, -0x00,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x1f,0x00,0x00,0x00, -0x2d,0x00,0x00,0x00,0x55,0x00,0x00,0x00,0x63,0x00,0x00,0x00, -0x10,0x00,0x06,0x00,0x04,0x00,0x00,0x00,0x11,0x00,0x00,0x00, -0x00,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x0b,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x1c,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x1d,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x1d,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x1d,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x08,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x1d,0x00,0x00,0x00, -0x03,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x47,0x00,0x03,0x00,0x1d,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x2a,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0x2b,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x2b,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x2b,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x2d,0x00,0x00,0x00, -0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x2d,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x52,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0x53,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x53,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x53,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x55,0x00,0x00,0x00, -0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x55,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x60,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0x61,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x61,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x61,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x63,0x00,0x00,0x00, -0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x63,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x74,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x19,0x00,0x00,0x00,0x13,0x00,0x02,0x00,0x02,0x00,0x00,0x00, -0x21,0x00,0x03,0x00,0x03,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x15,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x17,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x0a,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x0d,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x15,0x00,0x04,0x00,0x10,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x10,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x16,0x00,0x03,0x00,0x1c,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x1e,0x00,0x06,0x00,0x1d,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x1c,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x1e,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x1d,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x1e,0x00,0x00,0x00, -0x1f,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x10,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x21,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x14,0x00,0x02,0x00,0x24,0x00,0x00,0x00, -0x1d,0x00,0x03,0x00,0x2a,0x00,0x00,0x00,0x10,0x00,0x00,0x00, -0x1e,0x00,0x03,0x00,0x2b,0x00,0x00,0x00,0x2a,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x2c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x2b,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x2c,0x00,0x00,0x00, -0x2d,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x10,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x30,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x10,0x00,0x00,0x00,0x16,0x00,0x03,0x00,0x4e,0x00,0x00,0x00, -0x10,0x00,0x00,0x00,0x1d,0x00,0x03,0x00,0x52,0x00,0x00,0x00, -0x4e,0x00,0x00,0x00,0x1e,0x00,0x03,0x00,0x53,0x00,0x00,0x00, -0x52,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x54,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x54,0x00,0x00,0x00,0x55,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x58,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x4e,0x00,0x00,0x00,0x1d,0x00,0x03,0x00,0x60,0x00,0x00,0x00, -0x4e,0x00,0x00,0x00,0x1e,0x00,0x03,0x00,0x61,0x00,0x00,0x00, -0x60,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x62,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x61,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x62,0x00,0x00,0x00,0x63,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x73,0x00,0x00,0x00, -0x00,0x02,0x00,0x00,0x2c,0x00,0x06,0x00,0x09,0x00,0x00,0x00, -0x74,0x00,0x00,0x00,0x73,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x36,0x00,0x05,0x00,0x02,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x03,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0x05,0x00,0x00,0x00,0xf7,0x00,0x03,0x00, -0x75,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xfb,0x00,0x03,0x00, -0x0c,0x00,0x00,0x00,0x76,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x76,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00, -0x0e,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, -0x0e,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x10,0x00,0x00,0x00, -0x11,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x10,0x00,0x00,0x00,0x13,0x00,0x00,0x00,0x11,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x14,0x00,0x00,0x00,0x13,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x0d,0x00,0x00,0x00,0x17,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x17,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x10,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x18,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x1a,0x00,0x00,0x00, -0x19,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x21,0x00,0x00,0x00, -0x22,0x00,0x00,0x00,0x1f,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x22,0x00,0x00,0x00,0xae,0x00,0x05,0x00,0x24,0x00,0x00,0x00, -0x25,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0xf7,0x00,0x03,0x00,0x27,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x25,0x00,0x00,0x00,0x26,0x00,0x00,0x00, -0x27,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x26,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x75,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x27,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x30,0x00,0x00,0x00, -0x31,0x00,0x00,0x00,0x2d,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x1a,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x10,0x00,0x00,0x00, -0x32,0x00,0x00,0x00,0x31,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x33,0x00,0x00,0x00,0x32,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x38,0x00,0x00,0x00, -0x33,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x38,0x00,0x00,0x00, -0x14,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x3f,0x00,0x00,0x00,0x1a,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x41,0x00,0x00,0x00, -0x3f,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x44,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x47,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x48,0x00,0x00,0x00, -0x47,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x89,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x4c,0x00,0x00,0x00,0x41,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x4d,0x00,0x00,0x00,0x41,0x00,0x00,0x00,0x4c,0x00,0x00,0x00, -0x41,0x00,0x06,0x00,0x58,0x00,0x00,0x00,0x59,0x00,0x00,0x00, -0x55,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x44,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4e,0x00,0x00,0x00,0x5a,0x00,0x00,0x00, -0x59,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x5c,0x00,0x00,0x00,0x44,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0x41,0x00,0x06,0x00,0x58,0x00,0x00,0x00,0x5d,0x00,0x00,0x00, -0x55,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x5c,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4e,0x00,0x00,0x00,0x5e,0x00,0x00,0x00, -0x5d,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x66,0x00,0x00,0x00,0x4d,0x00,0x00,0x00,0x48,0x00,0x00,0x00, -0x41,0x00,0x06,0x00,0x58,0x00,0x00,0x00,0x6b,0x00,0x00,0x00, -0x63,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x66,0x00,0x00,0x00, -0x3e,0x00,0x03,0x00,0x6b,0x00,0x00,0x00,0x5a,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x6f,0x00,0x00,0x00, -0x66,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x41,0x00,0x06,0x00, -0x58,0x00,0x00,0x00,0x72,0x00,0x00,0x00,0x63,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x6f,0x00,0x00,0x00,0x3e,0x00,0x03,0x00, -0x72,0x00,0x00,0x00,0x5e,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x75,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x75,0x00,0x00,0x00, -0xfd,0x00,0x01,0x00,0x38,0x00,0x01,0x00, -}; -const uint64_t get_rows_f16_len = 1892; - -unsigned char get_rows_f16_f32_data[] = { -0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, -0x7a,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, -0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x09,0x00,0x00,0x00, -0x11,0x00,0x02,0x00,0x51,0x11,0x00,0x00,0x0b,0x00,0x06,0x00, -0x01,0x00,0x00,0x00,0x47,0x4c,0x53,0x4c,0x2e,0x73,0x74,0x64, -0x2e,0x34,0x35,0x30,0x00,0x00,0x00,0x00,0x0e,0x00,0x03,0x00, -0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x0f,0x00,0x0a,0x00, -0x05,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x6d,0x61,0x69,0x6e, -0x00,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x1f,0x00,0x00,0x00, -0x2d,0x00,0x00,0x00,0x55,0x00,0x00,0x00,0x63,0x00,0x00,0x00, -0x10,0x00,0x06,0x00,0x04,0x00,0x00,0x00,0x11,0x00,0x00,0x00, -0x00,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x0b,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x1c,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x1d,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x1d,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x1d,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x08,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x1d,0x00,0x00,0x00, -0x03,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x47,0x00,0x03,0x00,0x1d,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x2a,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0x2b,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x2b,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x2b,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x2d,0x00,0x00,0x00, -0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x2d,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x52,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0x53,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x53,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x53,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x55,0x00,0x00,0x00, -0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x55,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x60,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0x61,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x61,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x61,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x63,0x00,0x00,0x00, -0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x63,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x77,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x19,0x00,0x00,0x00,0x13,0x00,0x02,0x00,0x02,0x00,0x00,0x00, -0x21,0x00,0x03,0x00,0x03,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x15,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x17,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x0a,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x0d,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x15,0x00,0x04,0x00,0x10,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x10,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x16,0x00,0x03,0x00,0x1c,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x1e,0x00,0x06,0x00,0x1d,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x1c,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x1e,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x1d,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x1e,0x00,0x00,0x00, -0x1f,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x10,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x21,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x14,0x00,0x02,0x00,0x24,0x00,0x00,0x00, -0x1d,0x00,0x03,0x00,0x2a,0x00,0x00,0x00,0x10,0x00,0x00,0x00, -0x1e,0x00,0x03,0x00,0x2b,0x00,0x00,0x00,0x2a,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x2c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x2b,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x2c,0x00,0x00,0x00, -0x2d,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x10,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x30,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x10,0x00,0x00,0x00,0x16,0x00,0x03,0x00,0x4e,0x00,0x00,0x00, -0x10,0x00,0x00,0x00,0x1d,0x00,0x03,0x00,0x52,0x00,0x00,0x00, -0x4e,0x00,0x00,0x00,0x1e,0x00,0x03,0x00,0x53,0x00,0x00,0x00, -0x52,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x54,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x54,0x00,0x00,0x00,0x55,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x58,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x4e,0x00,0x00,0x00,0x1d,0x00,0x03,0x00,0x60,0x00,0x00,0x00, -0x1c,0x00,0x00,0x00,0x1e,0x00,0x03,0x00,0x61,0x00,0x00,0x00, -0x60,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x62,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x61,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x62,0x00,0x00,0x00,0x63,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x6c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x1c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x76,0x00,0x00,0x00,0x00,0x02,0x00,0x00,0x2c,0x00,0x06,0x00, -0x09,0x00,0x00,0x00,0x77,0x00,0x00,0x00,0x76,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x36,0x00,0x05,0x00, -0x02,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x03,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x05,0x00,0x00,0x00, -0xf7,0x00,0x03,0x00,0x78,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0xfb,0x00,0x03,0x00,0x0c,0x00,0x00,0x00,0x79,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0x79,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x0d,0x00,0x00,0x00,0x0e,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x0f,0x00,0x00,0x00,0x0e,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x10,0x00,0x00,0x00,0x11,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x13,0x00,0x00,0x00, -0x11,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x13,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00,0x17,0x00,0x00,0x00, -0x0b,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x17,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x10,0x00,0x00,0x00,0x19,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x1a,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x21,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x1f,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0xae,0x00,0x05,0x00, -0x24,0x00,0x00,0x00,0x25,0x00,0x00,0x00,0x14,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0xf7,0x00,0x03,0x00,0x27,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x25,0x00,0x00,0x00, -0x26,0x00,0x00,0x00,0x27,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x26,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x78,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0x27,0x00,0x00,0x00,0x41,0x00,0x06,0x00, -0x30,0x00,0x00,0x00,0x31,0x00,0x00,0x00,0x2d,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x1a,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x10,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x31,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x33,0x00,0x00,0x00, -0x32,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x38,0x00,0x00,0x00,0x33,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, -0x38,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x3f,0x00,0x00,0x00,0x1a,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x41,0x00,0x00,0x00,0x3f,0x00,0x00,0x00,0x14,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x44,0x00,0x00,0x00, -0x3a,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x89,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x47,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x48,0x00,0x00,0x00,0x47,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x4c,0x00,0x00,0x00, -0x41,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x82,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x4d,0x00,0x00,0x00,0x41,0x00,0x00,0x00, -0x4c,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x58,0x00,0x00,0x00, -0x59,0x00,0x00,0x00,0x55,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x44,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x4e,0x00,0x00,0x00, -0x5a,0x00,0x00,0x00,0x59,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x5c,0x00,0x00,0x00,0x44,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x58,0x00,0x00,0x00, -0x5d,0x00,0x00,0x00,0x55,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x5c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x4e,0x00,0x00,0x00, -0x5e,0x00,0x00,0x00,0x5d,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x66,0x00,0x00,0x00,0x4d,0x00,0x00,0x00, -0x48,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x1c,0x00,0x00,0x00, -0x6b,0x00,0x00,0x00,0x5a,0x00,0x00,0x00,0x41,0x00,0x06,0x00, -0x6c,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x63,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x66,0x00,0x00,0x00,0x3e,0x00,0x03,0x00, -0x6d,0x00,0x00,0x00,0x6b,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x71,0x00,0x00,0x00,0x66,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x1c,0x00,0x00,0x00, -0x74,0x00,0x00,0x00,0x5e,0x00,0x00,0x00,0x41,0x00,0x06,0x00, -0x6c,0x00,0x00,0x00,0x75,0x00,0x00,0x00,0x63,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x71,0x00,0x00,0x00,0x3e,0x00,0x03,0x00, -0x75,0x00,0x00,0x00,0x74,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x78,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x78,0x00,0x00,0x00, -0xfd,0x00,0x01,0x00,0x38,0x00,0x01,0x00, -}; -const uint64_t get_rows_f16_f32_len = 1940; - -unsigned char get_rows_f16_f32_fp32_data[] = { -0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, -0x7a,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, -0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x51,0x11,0x00,0x00, -0x0b,0x00,0x06,0x00,0x01,0x00,0x00,0x00,0x47,0x4c,0x53,0x4c, -0x2e,0x73,0x74,0x64,0x2e,0x34,0x35,0x30,0x00,0x00,0x00,0x00, -0x0e,0x00,0x03,0x00,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x0f,0x00,0x0a,0x00,0x05,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x6d,0x61,0x69,0x6e,0x00,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x1f,0x00,0x00,0x00,0x2d,0x00,0x00,0x00,0x55,0x00,0x00,0x00, -0x65,0x00,0x00,0x00,0x10,0x00,0x06,0x00,0x04,0x00,0x00,0x00, -0x11,0x00,0x00,0x00,0x00,0x02,0x00,0x00,0x01,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x0b,0x00,0x00,0x00, -0x0b,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x1d,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x1d,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x1d,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x1d,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x1d,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x2a,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00, -0x2b,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x2b,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, -0x2b,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x2d,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x2d,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x52,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x48,0x00,0x04,0x00, -0x53,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x53,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, -0x53,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x55,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x55,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x62,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00, -0x63,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x19,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x63,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, -0x63,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x65,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x65,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x77,0x00,0x00,0x00, -0x0b,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x13,0x00,0x02,0x00, -0x02,0x00,0x00,0x00,0x21,0x00,0x03,0x00,0x03,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x15,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x17,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x03,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x0a,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00, -0x0b,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x0d,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x15,0x00,0x04,0x00,0x10,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x10,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x16,0x00,0x03,0x00,0x1c,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x1e,0x00,0x06,0x00,0x1d,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x1c,0x00,0x00,0x00, -0x1c,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x1e,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x1d,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x1e,0x00,0x00,0x00,0x1f,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x10,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x21,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x14,0x00,0x02,0x00, -0x24,0x00,0x00,0x00,0x1d,0x00,0x03,0x00,0x2a,0x00,0x00,0x00, -0x10,0x00,0x00,0x00,0x1e,0x00,0x03,0x00,0x2b,0x00,0x00,0x00, -0x2a,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x2c,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x2b,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x2c,0x00,0x00,0x00,0x2d,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x10,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x30,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x16,0x00,0x03,0x00, -0x51,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, -0x52,0x00,0x00,0x00,0x51,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, -0x53,0x00,0x00,0x00,0x52,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x54,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x53,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x54,0x00,0x00,0x00,0x55,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x58,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x51,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, -0x62,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, -0x63,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x64,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x63,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x64,0x00,0x00,0x00,0x65,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x6d,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x76,0x00,0x00,0x00,0x00,0x02,0x00,0x00, -0x2c,0x00,0x06,0x00,0x09,0x00,0x00,0x00,0x77,0x00,0x00,0x00, -0x76,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0x36,0x00,0x05,0x00,0x02,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x05,0x00,0x00,0x00,0xf7,0x00,0x03,0x00,0x78,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0xfb,0x00,0x03,0x00,0x0c,0x00,0x00,0x00, -0x79,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x79,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00,0x0e,0x00,0x00,0x00, -0x0b,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x0e,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x10,0x00,0x00,0x00,0x11,0x00,0x00,0x00, -0x0f,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x13,0x00,0x00,0x00,0x11,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x14,0x00,0x00,0x00, -0x13,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00, -0x17,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x18,0x00,0x00,0x00, -0x17,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x10,0x00,0x00,0x00, -0x19,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x1a,0x00,0x00,0x00,0x19,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x21,0x00,0x00,0x00,0x22,0x00,0x00,0x00, -0x1f,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x22,0x00,0x00,0x00, -0xae,0x00,0x05,0x00,0x24,0x00,0x00,0x00,0x25,0x00,0x00,0x00, -0x14,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0xf7,0x00,0x03,0x00, -0x27,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x25,0x00,0x00,0x00,0x26,0x00,0x00,0x00,0x27,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0x26,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x78,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x27,0x00,0x00,0x00, -0x41,0x00,0x06,0x00,0x30,0x00,0x00,0x00,0x31,0x00,0x00,0x00, -0x2d,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x1a,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x10,0x00,0x00,0x00,0x32,0x00,0x00,0x00, -0x31,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x33,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x38,0x00,0x00,0x00,0x33,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x3a,0x00,0x00,0x00,0x38,0x00,0x00,0x00,0x14,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3f,0x00,0x00,0x00, -0x1a,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x41,0x00,0x00,0x00,0x3f,0x00,0x00,0x00, -0x14,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x44,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x47,0x00,0x00,0x00, -0x3a,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x48,0x00,0x00,0x00,0x47,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x4c,0x00,0x00,0x00,0x41,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x4d,0x00,0x00,0x00, -0x41,0x00,0x00,0x00,0x4c,0x00,0x00,0x00,0x41,0x00,0x06,0x00, -0x58,0x00,0x00,0x00,0x59,0x00,0x00,0x00,0x55,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x44,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x51,0x00,0x00,0x00,0x5a,0x00,0x00,0x00,0x59,0x00,0x00,0x00, -0x73,0x00,0x04,0x00,0x1c,0x00,0x00,0x00,0x5b,0x00,0x00,0x00, -0x5a,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x5d,0x00,0x00,0x00,0x44,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0x41,0x00,0x06,0x00,0x58,0x00,0x00,0x00,0x5e,0x00,0x00,0x00, -0x55,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x5d,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x51,0x00,0x00,0x00,0x5f,0x00,0x00,0x00, -0x5e,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x1c,0x00,0x00,0x00, -0x60,0x00,0x00,0x00,0x5f,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x68,0x00,0x00,0x00,0x4d,0x00,0x00,0x00, -0x48,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x6d,0x00,0x00,0x00, -0x6e,0x00,0x00,0x00,0x65,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x68,0x00,0x00,0x00,0x3e,0x00,0x03,0x00,0x6e,0x00,0x00,0x00, -0x5b,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x72,0x00,0x00,0x00,0x68,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0x41,0x00,0x06,0x00,0x6d,0x00,0x00,0x00,0x75,0x00,0x00,0x00, -0x65,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x72,0x00,0x00,0x00, -0x3e,0x00,0x03,0x00,0x75,0x00,0x00,0x00,0x60,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x78,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x78,0x00,0x00,0x00,0xfd,0x00,0x01,0x00,0x38,0x00,0x01,0x00, - -}; -const uint64_t get_rows_f16_f32_fp32_len = 1932; - -unsigned char get_rows_f16_fp32_data[] = { -0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, 0x7b,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, 0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x51,0x11,0x00,0x00, 0x0b,0x00,0x06,0x00,0x01,0x00,0x00,0x00,0x47,0x4c,0x53,0x4c, @@ -15455,21 +8321,19 @@ unsigned char get_rows_f16_fp32_data[] = { 0xf8,0x00,0x02,0x00,0x79,0x00,0x00,0x00,0xfd,0x00,0x01,0x00, 0x38,0x00,0x01,0x00, }; -const uint64_t get_rows_f16_fp32_len = 1948; +const uint64_t get_rows_f16_len = 1948; -unsigned char get_rows_q4_0_data[] = { +unsigned char get_rows_f16_f32_data[] = { 0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, -0x97,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, -0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x09,0x00,0x00,0x00, -0x11,0x00,0x02,0x00,0x27,0x00,0x00,0x00,0x11,0x00,0x02,0x00, -0x51,0x11,0x00,0x00,0x11,0x00,0x02,0x00,0x60,0x11,0x00,0x00, +0x7a,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, +0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x51,0x11,0x00,0x00, 0x0b,0x00,0x06,0x00,0x01,0x00,0x00,0x00,0x47,0x4c,0x53,0x4c, 0x2e,0x73,0x74,0x64,0x2e,0x34,0x35,0x30,0x00,0x00,0x00,0x00, 0x0e,0x00,0x03,0x00,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00, 0x0f,0x00,0x0a,0x00,0x05,0x00,0x00,0x00,0x04,0x00,0x00,0x00, 0x6d,0x61,0x69,0x6e,0x00,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x1f,0x00,0x00,0x00,0x2d,0x00,0x00,0x00,0x5a,0x00,0x00,0x00, -0x7d,0x00,0x00,0x00,0x10,0x00,0x06,0x00,0x04,0x00,0x00,0x00, +0x1f,0x00,0x00,0x00,0x2d,0x00,0x00,0x00,0x55,0x00,0x00,0x00, +0x65,0x00,0x00,0x00,0x10,0x00,0x06,0x00,0x04,0x00,0x00,0x00, 0x11,0x00,0x00,0x00,0x00,0x02,0x00,0x00,0x01,0x00,0x00,0x00, 0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x0b,0x00,0x00,0x00, 0x0b,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x48,0x00,0x05,0x00, @@ -15488,297 +8352,75 @@ unsigned char get_rows_q4_0_data[] = { 0x2b,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, 0x2d,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, 0x47,0x00,0x04,0x00,0x2d,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x55,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x56,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x56,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x57,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0x58,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x58,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x58,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x5a,0x00,0x00,0x00, -0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x5a,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x7a,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0x7b,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x7b,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x7b,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x7d,0x00,0x00,0x00, -0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x7d,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x8d,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x19,0x00,0x00,0x00,0x13,0x00,0x02,0x00,0x02,0x00,0x00,0x00, -0x21,0x00,0x03,0x00,0x03,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x15,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x17,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x0a,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x0d,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x15,0x00,0x04,0x00,0x10,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x10,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x16,0x00,0x03,0x00,0x1c,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x1e,0x00,0x06,0x00,0x1d,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x1c,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x1e,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x1d,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x1e,0x00,0x00,0x00, -0x1f,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x10,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x21,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x14,0x00,0x02,0x00,0x24,0x00,0x00,0x00, -0x1d,0x00,0x03,0x00,0x2a,0x00,0x00,0x00,0x10,0x00,0x00,0x00, -0x1e,0x00,0x03,0x00,0x2b,0x00,0x00,0x00,0x2a,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x2c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x2b,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x2c,0x00,0x00,0x00, -0x2d,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x10,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x30,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x10,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x44,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x49,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x16,0x00,0x03,0x00,0x50,0x00,0x00,0x00,0x10,0x00,0x00,0x00, -0x15,0x00,0x04,0x00,0x53,0x00,0x00,0x00,0x08,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x54,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x1c,0x00,0x04,0x00, -0x55,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x54,0x00,0x00,0x00, -0x1e,0x00,0x04,0x00,0x56,0x00,0x00,0x00,0x50,0x00,0x00,0x00, -0x55,0x00,0x00,0x00,0x1d,0x00,0x03,0x00,0x57,0x00,0x00,0x00, -0x56,0x00,0x00,0x00,0x1e,0x00,0x03,0x00,0x58,0x00,0x00,0x00, -0x57,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x59,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x58,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x59,0x00,0x00,0x00,0x5a,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x5c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x50,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x63,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x17,0x00,0x04,0x00, -0x66,0x00,0x00,0x00,0x50,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x10,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x0f,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x10,0x00,0x00,0x00, -0x70,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x50,0x00,0x00,0x00,0x75,0x00,0x00,0x00,0x00,0x48,0x00,0x00, -0x1d,0x00,0x03,0x00,0x7a,0x00,0x00,0x00,0x50,0x00,0x00,0x00, -0x1e,0x00,0x03,0x00,0x7b,0x00,0x00,0x00,0x7a,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x7c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x7b,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x7c,0x00,0x00,0x00, -0x7d,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x8c,0x00,0x00,0x00,0x00,0x02,0x00,0x00, -0x2c,0x00,0x06,0x00,0x09,0x00,0x00,0x00,0x8d,0x00,0x00,0x00, -0x8c,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0x2c,0x00,0x05,0x00,0x66,0x00,0x00,0x00,0x96,0x00,0x00,0x00, -0x75,0x00,0x00,0x00,0x75,0x00,0x00,0x00,0x36,0x00,0x05,0x00, -0x02,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x03,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x05,0x00,0x00,0x00, -0xf7,0x00,0x03,0x00,0x8e,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0xfb,0x00,0x03,0x00,0x0c,0x00,0x00,0x00,0x8f,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0x8f,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x0d,0x00,0x00,0x00,0x0e,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x0f,0x00,0x00,0x00,0x0e,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x10,0x00,0x00,0x00,0x11,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x13,0x00,0x00,0x00, -0x11,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x13,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00,0x17,0x00,0x00,0x00, -0x0b,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x17,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x10,0x00,0x00,0x00,0x19,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x1a,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x21,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x1f,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0xae,0x00,0x05,0x00, -0x24,0x00,0x00,0x00,0x25,0x00,0x00,0x00,0x14,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0xf7,0x00,0x03,0x00,0x27,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x25,0x00,0x00,0x00, -0x26,0x00,0x00,0x00,0x27,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x26,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x8e,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0x27,0x00,0x00,0x00,0x41,0x00,0x06,0x00, -0x30,0x00,0x00,0x00,0x31,0x00,0x00,0x00,0x2d,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x1a,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x10,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x31,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x33,0x00,0x00,0x00, -0x32,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x38,0x00,0x00,0x00,0x33,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, -0x38,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x3f,0x00,0x00,0x00,0x1a,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x41,0x00,0x00,0x00,0x3f,0x00,0x00,0x00,0x14,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x45,0x00,0x00,0x00, -0x3a,0x00,0x00,0x00,0x44,0x00,0x00,0x00,0x89,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x48,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, -0x44,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x4a,0x00,0x00,0x00,0x48,0x00,0x00,0x00,0x49,0x00,0x00,0x00, -0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, -0x41,0x00,0x00,0x00,0x44,0x00,0x00,0x00,0x82,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x41,0x00,0x00,0x00, -0x4e,0x00,0x00,0x00,0x41,0x00,0x07,0x00,0x5c,0x00,0x00,0x00, -0x5d,0x00,0x00,0x00,0x5a,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x45,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x50,0x00,0x00,0x00,0x5e,0x00,0x00,0x00,0x5d,0x00,0x00,0x00, -0x41,0x00,0x08,0x00,0x63,0x00,0x00,0x00,0x64,0x00,0x00,0x00, -0x5a,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x45,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x4a,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x53,0x00,0x00,0x00,0x65,0x00,0x00,0x00,0x64,0x00,0x00,0x00, -0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x6a,0x00,0x00,0x00, -0x65,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x10,0x00,0x00,0x00, -0x6b,0x00,0x00,0x00,0x6a,0x00,0x00,0x00,0xc7,0x00,0x05,0x00, -0x10,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x6b,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x6f,0x00,0x04,0x00,0x50,0x00,0x00,0x00, -0x6e,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0xc2,0x00,0x05,0x00, -0x53,0x00,0x00,0x00,0x71,0x00,0x00,0x00,0x65,0x00,0x00,0x00, -0x70,0x00,0x00,0x00,0x70,0x00,0x04,0x00,0x50,0x00,0x00,0x00, -0x72,0x00,0x00,0x00,0x71,0x00,0x00,0x00,0x50,0x00,0x05,0x00, -0x66,0x00,0x00,0x00,0x73,0x00,0x00,0x00,0x6e,0x00,0x00,0x00, -0x72,0x00,0x00,0x00,0x83,0x00,0x05,0x00,0x66,0x00,0x00,0x00, -0x77,0x00,0x00,0x00,0x73,0x00,0x00,0x00,0x96,0x00,0x00,0x00, -0x8e,0x00,0x05,0x00,0x66,0x00,0x00,0x00,0x79,0x00,0x00,0x00, -0x77,0x00,0x00,0x00,0x5e,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x80,0x00,0x00,0x00,0x4f,0x00,0x00,0x00, -0x4a,0x00,0x00,0x00,0x51,0x00,0x05,0x00,0x50,0x00,0x00,0x00, -0x83,0x00,0x00,0x00,0x79,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x41,0x00,0x06,0x00,0x5c,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0x7d,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x80,0x00,0x00,0x00, -0x3e,0x00,0x03,0x00,0x84,0x00,0x00,0x00,0x83,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x88,0x00,0x00,0x00, -0x80,0x00,0x00,0x00,0x54,0x00,0x00,0x00,0x51,0x00,0x05,0x00, -0x50,0x00,0x00,0x00,0x8a,0x00,0x00,0x00,0x79,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x5c,0x00,0x00,0x00, -0x8b,0x00,0x00,0x00,0x7d,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x88,0x00,0x00,0x00,0x3e,0x00,0x03,0x00,0x8b,0x00,0x00,0x00, -0x8a,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x8e,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0x8e,0x00,0x00,0x00,0xfd,0x00,0x01,0x00, -0x38,0x00,0x01,0x00, -}; -const uint64_t get_rows_q4_0_len = 2356; - -unsigned char get_rows_q4_0_f32_data[] = { -0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, -0x9a,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, -0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x09,0x00,0x00,0x00, -0x11,0x00,0x02,0x00,0x27,0x00,0x00,0x00,0x11,0x00,0x02,0x00, -0x51,0x11,0x00,0x00,0x11,0x00,0x02,0x00,0x60,0x11,0x00,0x00, -0x0b,0x00,0x06,0x00,0x01,0x00,0x00,0x00,0x47,0x4c,0x53,0x4c, -0x2e,0x73,0x74,0x64,0x2e,0x34,0x35,0x30,0x00,0x00,0x00,0x00, -0x0e,0x00,0x03,0x00,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x0f,0x00,0x0a,0x00,0x05,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x6d,0x61,0x69,0x6e,0x00,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x1f,0x00,0x00,0x00,0x2d,0x00,0x00,0x00,0x5a,0x00,0x00,0x00, -0x7d,0x00,0x00,0x00,0x10,0x00,0x06,0x00,0x04,0x00,0x00,0x00, -0x11,0x00,0x00,0x00,0x00,0x02,0x00,0x00,0x01,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x0b,0x00,0x00,0x00, -0x0b,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x1d,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x1d,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x1d,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x1d,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x1d,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x2a,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00, -0x2b,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x2b,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x52,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x48,0x00,0x04,0x00, +0x53,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x53,0x00,0x00,0x00,0x00,0x00,0x00,0x00, 0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, -0x2b,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x2d,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x2d,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x55,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x56,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x56,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x57,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0x58,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x58,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x58,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x5a,0x00,0x00,0x00, -0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x5a,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x7a,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0x7b,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x7b,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x7b,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x7d,0x00,0x00,0x00, -0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x7d,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x90,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x19,0x00,0x00,0x00,0x13,0x00,0x02,0x00,0x02,0x00,0x00,0x00, -0x21,0x00,0x03,0x00,0x03,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x15,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x17,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x0a,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x0d,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x15,0x00,0x04,0x00,0x10,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x10,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x16,0x00,0x03,0x00,0x1c,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x1e,0x00,0x06,0x00,0x1d,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x1c,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x1e,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x1d,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x1e,0x00,0x00,0x00, -0x1f,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x10,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x21,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x14,0x00,0x02,0x00,0x24,0x00,0x00,0x00, -0x1d,0x00,0x03,0x00,0x2a,0x00,0x00,0x00,0x10,0x00,0x00,0x00, -0x1e,0x00,0x03,0x00,0x2b,0x00,0x00,0x00,0x2a,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x2c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x2b,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x2c,0x00,0x00,0x00, -0x2d,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x10,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x30,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x10,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x44,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x49,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x16,0x00,0x03,0x00,0x50,0x00,0x00,0x00,0x10,0x00,0x00,0x00, -0x15,0x00,0x04,0x00,0x53,0x00,0x00,0x00,0x08,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x54,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x1c,0x00,0x04,0x00, -0x55,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x54,0x00,0x00,0x00, -0x1e,0x00,0x04,0x00,0x56,0x00,0x00,0x00,0x50,0x00,0x00,0x00, -0x55,0x00,0x00,0x00,0x1d,0x00,0x03,0x00,0x57,0x00,0x00,0x00, -0x56,0x00,0x00,0x00,0x1e,0x00,0x03,0x00,0x58,0x00,0x00,0x00, -0x57,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x59,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x58,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x59,0x00,0x00,0x00,0x5a,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x5c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x50,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x63,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x17,0x00,0x04,0x00, -0x66,0x00,0x00,0x00,0x50,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x10,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x0f,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x10,0x00,0x00,0x00, -0x70,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x50,0x00,0x00,0x00,0x75,0x00,0x00,0x00,0x00,0x48,0x00,0x00, -0x1d,0x00,0x03,0x00,0x7a,0x00,0x00,0x00,0x1c,0x00,0x00,0x00, -0x1e,0x00,0x03,0x00,0x7b,0x00,0x00,0x00,0x7a,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x7c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x7b,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x7c,0x00,0x00,0x00, -0x7d,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x85,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x1c,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x8f,0x00,0x00,0x00, -0x00,0x02,0x00,0x00,0x2c,0x00,0x06,0x00,0x09,0x00,0x00,0x00, -0x90,0x00,0x00,0x00,0x8f,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x2c,0x00,0x05,0x00,0x66,0x00,0x00,0x00, -0x99,0x00,0x00,0x00,0x75,0x00,0x00,0x00,0x75,0x00,0x00,0x00, +0x53,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x55,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x55,0x00,0x00,0x00,0x21,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x62,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00, +0x63,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x19,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x63,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, +0x63,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x65,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x65,0x00,0x00,0x00,0x21,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x77,0x00,0x00,0x00, +0x0b,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x13,0x00,0x02,0x00, +0x02,0x00,0x00,0x00,0x21,0x00,0x03,0x00,0x03,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x15,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x17,0x00,0x04,0x00, +0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x03,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x0a,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x09,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00, +0x0b,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x0d,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x15,0x00,0x04,0x00,0x10,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x10,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x16,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x16,0x00,0x03,0x00,0x1c,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0x1e,0x00,0x06,0x00,0x1d,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x1c,0x00,0x00,0x00, +0x1c,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x1e,0x00,0x00,0x00, +0x09,0x00,0x00,0x00,0x1d,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0x1e,0x00,0x00,0x00,0x1f,0x00,0x00,0x00,0x09,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x10,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x21,0x00,0x00,0x00, +0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x14,0x00,0x02,0x00, +0x24,0x00,0x00,0x00,0x1d,0x00,0x03,0x00,0x2a,0x00,0x00,0x00, +0x10,0x00,0x00,0x00,0x1e,0x00,0x03,0x00,0x2b,0x00,0x00,0x00, +0x2a,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x2c,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x2b,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0x2c,0x00,0x00,0x00,0x2d,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x10,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x30,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x16,0x00,0x03,0x00, +0x51,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, +0x52,0x00,0x00,0x00,0x51,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, +0x53,0x00,0x00,0x00,0x52,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x54,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x53,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0x54,0x00,0x00,0x00,0x55,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x58,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x51,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, +0x62,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, +0x63,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x64,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x63,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0x64,0x00,0x00,0x00,0x65,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x6d,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x76,0x00,0x00,0x00,0x00,0x02,0x00,0x00, +0x2c,0x00,0x06,0x00,0x09,0x00,0x00,0x00,0x77,0x00,0x00,0x00, +0x76,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x16,0x00,0x00,0x00, 0x36,0x00,0x05,0x00,0x02,0x00,0x00,0x00,0x04,0x00,0x00,0x00, 0x00,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x05,0x00,0x00,0x00,0xf7,0x00,0x03,0x00,0x91,0x00,0x00,0x00, +0x05,0x00,0x00,0x00,0xf7,0x00,0x03,0x00,0x78,0x00,0x00,0x00, 0x00,0x00,0x00,0x00,0xfb,0x00,0x03,0x00,0x0c,0x00,0x00,0x00, -0x92,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x92,0x00,0x00,0x00, +0x79,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x79,0x00,0x00,0x00, 0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00,0x0e,0x00,0x00,0x00, 0x0b,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, 0x06,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x0e,0x00,0x00,0x00, @@ -15800,7 +8442,7 @@ unsigned char get_rows_q4_0_f32_data[] = { 0x27,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, 0x25,0x00,0x00,0x00,0x26,0x00,0x00,0x00,0x27,0x00,0x00,0x00, 0xf8,0x00,0x02,0x00,0x26,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x91,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x27,0x00,0x00,0x00, +0x78,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x27,0x00,0x00,0x00, 0x41,0x00,0x06,0x00,0x30,0x00,0x00,0x00,0x31,0x00,0x00,0x00, 0x2d,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x1a,0x00,0x00,0x00, 0x3d,0x00,0x04,0x00,0x10,0x00,0x00,0x00,0x32,0x00,0x00,0x00, @@ -15813,258 +8455,41 @@ unsigned char get_rows_q4_0_f32_data[] = { 0x1a,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x80,0x00,0x05,0x00, 0x06,0x00,0x00,0x00,0x41,0x00,0x00,0x00,0x3f,0x00,0x00,0x00, 0x14,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x45,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x44,0x00,0x00,0x00, -0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x48,0x00,0x00,0x00, -0x3a,0x00,0x00,0x00,0x44,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x4a,0x00,0x00,0x00,0x48,0x00,0x00,0x00, -0x49,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x4e,0x00,0x00,0x00,0x41,0x00,0x00,0x00,0x44,0x00,0x00,0x00, -0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x4f,0x00,0x00,0x00, -0x41,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x41,0x00,0x07,0x00, -0x5c,0x00,0x00,0x00,0x5d,0x00,0x00,0x00,0x5a,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x45,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x50,0x00,0x00,0x00,0x5e,0x00,0x00,0x00, -0x5d,0x00,0x00,0x00,0x41,0x00,0x08,0x00,0x63,0x00,0x00,0x00, -0x64,0x00,0x00,0x00,0x5a,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x45,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x4a,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x53,0x00,0x00,0x00,0x65,0x00,0x00,0x00, -0x64,0x00,0x00,0x00,0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x6a,0x00,0x00,0x00,0x65,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x10,0x00,0x00,0x00,0x6b,0x00,0x00,0x00,0x6a,0x00,0x00,0x00, -0xc7,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, -0x6b,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x6f,0x00,0x04,0x00, -0x50,0x00,0x00,0x00,0x6e,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, -0xc2,0x00,0x05,0x00,0x53,0x00,0x00,0x00,0x71,0x00,0x00,0x00, -0x65,0x00,0x00,0x00,0x70,0x00,0x00,0x00,0x70,0x00,0x04,0x00, -0x50,0x00,0x00,0x00,0x72,0x00,0x00,0x00,0x71,0x00,0x00,0x00, -0x50,0x00,0x05,0x00,0x66,0x00,0x00,0x00,0x73,0x00,0x00,0x00, -0x6e,0x00,0x00,0x00,0x72,0x00,0x00,0x00,0x83,0x00,0x05,0x00, -0x66,0x00,0x00,0x00,0x77,0x00,0x00,0x00,0x73,0x00,0x00,0x00, -0x99,0x00,0x00,0x00,0x8e,0x00,0x05,0x00,0x66,0x00,0x00,0x00, -0x79,0x00,0x00,0x00,0x77,0x00,0x00,0x00,0x5e,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x80,0x00,0x00,0x00, -0x4f,0x00,0x00,0x00,0x4a,0x00,0x00,0x00,0x51,0x00,0x05,0x00, -0x50,0x00,0x00,0x00,0x83,0x00,0x00,0x00,0x79,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x1c,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0x83,0x00,0x00,0x00,0x41,0x00,0x06,0x00, -0x85,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x7d,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x80,0x00,0x00,0x00,0x3e,0x00,0x03,0x00, -0x86,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x8a,0x00,0x00,0x00,0x80,0x00,0x00,0x00, -0x54,0x00,0x00,0x00,0x51,0x00,0x05,0x00,0x50,0x00,0x00,0x00, -0x8c,0x00,0x00,0x00,0x79,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x73,0x00,0x04,0x00,0x1c,0x00,0x00,0x00,0x8d,0x00,0x00,0x00, -0x8c,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x85,0x00,0x00,0x00, -0x8e,0x00,0x00,0x00,0x7d,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x8a,0x00,0x00,0x00,0x3e,0x00,0x03,0x00,0x8e,0x00,0x00,0x00, -0x8d,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x91,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0x91,0x00,0x00,0x00,0xfd,0x00,0x01,0x00, -0x38,0x00,0x01,0x00, -}; -const uint64_t get_rows_q4_0_f32_len = 2404; +0x44,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x16,0x00,0x00,0x00, +0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x47,0x00,0x00,0x00, +0x3a,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x48,0x00,0x00,0x00,0x47,0x00,0x00,0x00, +0x16,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x4c,0x00,0x00,0x00,0x41,0x00,0x00,0x00,0x16,0x00,0x00,0x00, +0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x4d,0x00,0x00,0x00, +0x41,0x00,0x00,0x00,0x4c,0x00,0x00,0x00,0x41,0x00,0x06,0x00, +0x58,0x00,0x00,0x00,0x59,0x00,0x00,0x00,0x55,0x00,0x00,0x00, +0x2e,0x00,0x00,0x00,0x44,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x51,0x00,0x00,0x00,0x5a,0x00,0x00,0x00,0x59,0x00,0x00,0x00, +0x73,0x00,0x04,0x00,0x1c,0x00,0x00,0x00,0x5b,0x00,0x00,0x00, +0x5a,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x5d,0x00,0x00,0x00,0x44,0x00,0x00,0x00,0x16,0x00,0x00,0x00, +0x41,0x00,0x06,0x00,0x58,0x00,0x00,0x00,0x5e,0x00,0x00,0x00, +0x55,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x5d,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x51,0x00,0x00,0x00,0x5f,0x00,0x00,0x00, +0x5e,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x1c,0x00,0x00,0x00, +0x60,0x00,0x00,0x00,0x5f,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x68,0x00,0x00,0x00,0x4d,0x00,0x00,0x00, +0x48,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x6d,0x00,0x00,0x00, +0x6e,0x00,0x00,0x00,0x65,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, +0x68,0x00,0x00,0x00,0x3e,0x00,0x03,0x00,0x6e,0x00,0x00,0x00, +0x5b,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x72,0x00,0x00,0x00,0x68,0x00,0x00,0x00,0x16,0x00,0x00,0x00, +0x41,0x00,0x06,0x00,0x6d,0x00,0x00,0x00,0x75,0x00,0x00,0x00, +0x65,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x72,0x00,0x00,0x00, +0x3e,0x00,0x03,0x00,0x75,0x00,0x00,0x00,0x60,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x78,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0x78,0x00,0x00,0x00,0xfd,0x00,0x01,0x00,0x38,0x00,0x01,0x00, -unsigned char get_rows_q4_0_f32_fp32_data[] = { -0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, -0x97,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, -0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x51,0x11,0x00,0x00, -0x11,0x00,0x02,0x00,0x60,0x11,0x00,0x00,0x0b,0x00,0x06,0x00, -0x01,0x00,0x00,0x00,0x47,0x4c,0x53,0x4c,0x2e,0x73,0x74,0x64, -0x2e,0x34,0x35,0x30,0x00,0x00,0x00,0x00,0x0e,0x00,0x03,0x00, -0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x0f,0x00,0x0a,0x00, -0x05,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x6d,0x61,0x69,0x6e, -0x00,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x1f,0x00,0x00,0x00, -0x2d,0x00,0x00,0x00,0x5a,0x00,0x00,0x00,0x7c,0x00,0x00,0x00, -0x10,0x00,0x06,0x00,0x04,0x00,0x00,0x00,0x11,0x00,0x00,0x00, -0x00,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x0b,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x1c,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x1d,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x1d,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x1d,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x08,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x1d,0x00,0x00,0x00, -0x03,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x47,0x00,0x03,0x00,0x1d,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x2a,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0x2b,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x2b,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x2b,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x2d,0x00,0x00,0x00, -0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x2d,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x55,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x56,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x56,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x57,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x48,0x00,0x04,0x00,0x58,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x58,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x03,0x00,0x58,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x5a,0x00,0x00,0x00,0x22,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x5a,0x00,0x00,0x00, -0x21,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x79,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x48,0x00,0x04,0x00,0x7a,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x19,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x7a,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x03,0x00,0x7a,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x7c,0x00,0x00,0x00,0x22,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x7c,0x00,0x00,0x00, -0x21,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x8d,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x19,0x00,0x00,0x00, -0x13,0x00,0x02,0x00,0x02,0x00,0x00,0x00,0x21,0x00,0x03,0x00, -0x03,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x15,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x17,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x03,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x0a,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x0a,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x0d,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x15,0x00,0x04,0x00, -0x10,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x10,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x16,0x00,0x03,0x00, -0x1c,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x1e,0x00,0x06,0x00, -0x1d,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x1c,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x1e,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x1d,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x1e,0x00,0x00,0x00,0x1f,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x10,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x21,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x14,0x00,0x02,0x00,0x24,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, -0x2a,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, -0x2b,0x00,0x00,0x00,0x2a,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x2c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x2b,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x2c,0x00,0x00,0x00,0x2d,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x10,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x30,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x10,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x44,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x49,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x16,0x00,0x03,0x00, -0x52,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x15,0x00,0x04,0x00, -0x53,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x54,0x00,0x00,0x00, -0x10,0x00,0x00,0x00,0x1c,0x00,0x04,0x00,0x55,0x00,0x00,0x00, -0x53,0x00,0x00,0x00,0x54,0x00,0x00,0x00,0x1e,0x00,0x04,0x00, -0x56,0x00,0x00,0x00,0x52,0x00,0x00,0x00,0x55,0x00,0x00,0x00, -0x1d,0x00,0x03,0x00,0x57,0x00,0x00,0x00,0x56,0x00,0x00,0x00, -0x1e,0x00,0x03,0x00,0x58,0x00,0x00,0x00,0x57,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x59,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x58,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x59,0x00,0x00,0x00, -0x5a,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x5c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x52,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x63,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x53,0x00,0x00,0x00,0x17,0x00,0x04,0x00,0x67,0x00,0x00,0x00, -0x1c,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x6b,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x10,0x00,0x00,0x00,0x6f,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x1c,0x00,0x00,0x00, -0x74,0x00,0x00,0x00,0x00,0x00,0x00,0x41,0x1d,0x00,0x03,0x00, -0x79,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, -0x7a,0x00,0x00,0x00,0x79,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x7b,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x7a,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x7b,0x00,0x00,0x00,0x7c,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x83,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x8c,0x00,0x00,0x00,0x00,0x02,0x00,0x00, -0x2c,0x00,0x06,0x00,0x09,0x00,0x00,0x00,0x8d,0x00,0x00,0x00, -0x8c,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0x2c,0x00,0x05,0x00,0x67,0x00,0x00,0x00,0x96,0x00,0x00,0x00, -0x74,0x00,0x00,0x00,0x74,0x00,0x00,0x00,0x36,0x00,0x05,0x00, -0x02,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x03,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x05,0x00,0x00,0x00, -0xf7,0x00,0x03,0x00,0x8e,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0xfb,0x00,0x03,0x00,0x0c,0x00,0x00,0x00,0x8f,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0x8f,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x0d,0x00,0x00,0x00,0x0e,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x0f,0x00,0x00,0x00,0x0e,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x10,0x00,0x00,0x00,0x11,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x13,0x00,0x00,0x00, -0x11,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x13,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00,0x17,0x00,0x00,0x00, -0x0b,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x17,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x10,0x00,0x00,0x00,0x19,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x1a,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x21,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x1f,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0xae,0x00,0x05,0x00, -0x24,0x00,0x00,0x00,0x25,0x00,0x00,0x00,0x14,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0xf7,0x00,0x03,0x00,0x27,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x25,0x00,0x00,0x00, -0x26,0x00,0x00,0x00,0x27,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x26,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x8e,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0x27,0x00,0x00,0x00,0x41,0x00,0x06,0x00, -0x30,0x00,0x00,0x00,0x31,0x00,0x00,0x00,0x2d,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x1a,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x10,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x31,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x33,0x00,0x00,0x00, -0x32,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x38,0x00,0x00,0x00,0x33,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, -0x38,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x3f,0x00,0x00,0x00,0x1a,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x41,0x00,0x00,0x00,0x3f,0x00,0x00,0x00,0x14,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x45,0x00,0x00,0x00, -0x3a,0x00,0x00,0x00,0x44,0x00,0x00,0x00,0x89,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x48,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, -0x44,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x4a,0x00,0x00,0x00,0x48,0x00,0x00,0x00,0x49,0x00,0x00,0x00, -0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, -0x41,0x00,0x00,0x00,0x44,0x00,0x00,0x00,0x82,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x41,0x00,0x00,0x00, -0x4e,0x00,0x00,0x00,0x41,0x00,0x07,0x00,0x5c,0x00,0x00,0x00, -0x5d,0x00,0x00,0x00,0x5a,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x45,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x52,0x00,0x00,0x00,0x5e,0x00,0x00,0x00,0x5d,0x00,0x00,0x00, -0x73,0x00,0x04,0x00,0x1c,0x00,0x00,0x00,0x5f,0x00,0x00,0x00, -0x5e,0x00,0x00,0x00,0x41,0x00,0x08,0x00,0x63,0x00,0x00,0x00, -0x64,0x00,0x00,0x00,0x5a,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x45,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x4a,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x53,0x00,0x00,0x00,0x65,0x00,0x00,0x00, -0x64,0x00,0x00,0x00,0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x66,0x00,0x00,0x00,0x65,0x00,0x00,0x00,0xc7,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x66,0x00,0x00,0x00, -0x6b,0x00,0x00,0x00,0x70,0x00,0x04,0x00,0x1c,0x00,0x00,0x00, -0x6d,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0xc2,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x70,0x00,0x00,0x00,0x66,0x00,0x00,0x00, -0x6f,0x00,0x00,0x00,0x70,0x00,0x04,0x00,0x1c,0x00,0x00,0x00, -0x71,0x00,0x00,0x00,0x70,0x00,0x00,0x00,0x50,0x00,0x05,0x00, -0x67,0x00,0x00,0x00,0x72,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, -0x71,0x00,0x00,0x00,0x83,0x00,0x05,0x00,0x67,0x00,0x00,0x00, -0x76,0x00,0x00,0x00,0x72,0x00,0x00,0x00,0x96,0x00,0x00,0x00, -0x8e,0x00,0x05,0x00,0x67,0x00,0x00,0x00,0x78,0x00,0x00,0x00, -0x76,0x00,0x00,0x00,0x5f,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x7f,0x00,0x00,0x00,0x4f,0x00,0x00,0x00, -0x4a,0x00,0x00,0x00,0x51,0x00,0x05,0x00,0x1c,0x00,0x00,0x00, -0x82,0x00,0x00,0x00,0x78,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x41,0x00,0x06,0x00,0x83,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0x7c,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x7f,0x00,0x00,0x00, -0x3e,0x00,0x03,0x00,0x84,0x00,0x00,0x00,0x82,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x88,0x00,0x00,0x00, -0x7f,0x00,0x00,0x00,0x54,0x00,0x00,0x00,0x51,0x00,0x05,0x00, -0x1c,0x00,0x00,0x00,0x8a,0x00,0x00,0x00,0x78,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x83,0x00,0x00,0x00, -0x8b,0x00,0x00,0x00,0x7c,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x88,0x00,0x00,0x00,0x3e,0x00,0x03,0x00,0x8b,0x00,0x00,0x00, -0x8a,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x8e,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0x8e,0x00,0x00,0x00,0xfd,0x00,0x01,0x00, -0x38,0x00,0x01,0x00, }; -const uint64_t get_rows_q4_0_f32_fp32_len = 2356; +const uint64_t get_rows_f16_f32_len = 1932; -unsigned char get_rows_q4_0_fp32_data[] = { +unsigned char get_rows_q4_0_data[] = { 0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, 0x98,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, 0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x51,0x11,0x00,0x00, @@ -16264,425 +8689,11 @@ unsigned char get_rows_q4_0_fp32_data[] = { 0x8f,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x8f,0x00,0x00,0x00, 0xfd,0x00,0x01,0x00,0x38,0x00,0x01,0x00, }; -const uint64_t get_rows_q4_0_fp32_len = 2372; +const uint64_t get_rows_q4_0_len = 2372; -unsigned char get_rows_q4_1_data[] = { -0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, -0x94,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, -0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x09,0x00,0x00,0x00, -0x11,0x00,0x02,0x00,0x27,0x00,0x00,0x00,0x11,0x00,0x02,0x00, -0x51,0x11,0x00,0x00,0x11,0x00,0x02,0x00,0x60,0x11,0x00,0x00, -0x0b,0x00,0x06,0x00,0x01,0x00,0x00,0x00,0x47,0x4c,0x53,0x4c, -0x2e,0x73,0x74,0x64,0x2e,0x34,0x35,0x30,0x00,0x00,0x00,0x00, -0x0e,0x00,0x03,0x00,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x0f,0x00,0x0a,0x00,0x05,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x6d,0x61,0x69,0x6e,0x00,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x1f,0x00,0x00,0x00,0x2d,0x00,0x00,0x00,0x5a,0x00,0x00,0x00, -0x81,0x00,0x00,0x00,0x10,0x00,0x06,0x00,0x04,0x00,0x00,0x00, -0x11,0x00,0x00,0x00,0x00,0x02,0x00,0x00,0x01,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x0b,0x00,0x00,0x00, -0x0b,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x1d,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x1d,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x1d,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x1d,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x1d,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x2a,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00, -0x2b,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x2b,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, -0x2b,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x2d,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x2d,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x55,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x56,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x56,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x56,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x57,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x14,0x00,0x00,0x00, -0x48,0x00,0x04,0x00,0x58,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x58,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x03,0x00,0x58,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x5a,0x00,0x00,0x00,0x22,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x5a,0x00,0x00,0x00, -0x21,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x7e,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x48,0x00,0x04,0x00,0x7f,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x19,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x7f,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x03,0x00,0x7f,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x81,0x00,0x00,0x00,0x22,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x81,0x00,0x00,0x00, -0x21,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x91,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x19,0x00,0x00,0x00, -0x13,0x00,0x02,0x00,0x02,0x00,0x00,0x00,0x21,0x00,0x03,0x00, -0x03,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x15,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x17,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x03,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x0a,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x0a,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x0d,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x15,0x00,0x04,0x00, -0x10,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x10,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x16,0x00,0x03,0x00, -0x1c,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x1e,0x00,0x06,0x00, -0x1d,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x1c,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x1e,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x1d,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x1e,0x00,0x00,0x00,0x1f,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x10,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x21,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x14,0x00,0x02,0x00,0x24,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, -0x2a,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, -0x2b,0x00,0x00,0x00,0x2a,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x2c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x2b,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x2c,0x00,0x00,0x00,0x2d,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x10,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x30,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x10,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x44,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x49,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x16,0x00,0x03,0x00, -0x50,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x15,0x00,0x04,0x00, -0x53,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x54,0x00,0x00,0x00, -0x10,0x00,0x00,0x00,0x1c,0x00,0x04,0x00,0x55,0x00,0x00,0x00, -0x53,0x00,0x00,0x00,0x54,0x00,0x00,0x00,0x1e,0x00,0x05,0x00, -0x56,0x00,0x00,0x00,0x50,0x00,0x00,0x00,0x50,0x00,0x00,0x00, -0x55,0x00,0x00,0x00,0x1d,0x00,0x03,0x00,0x57,0x00,0x00,0x00, -0x56,0x00,0x00,0x00,0x1e,0x00,0x03,0x00,0x58,0x00,0x00,0x00, -0x57,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x59,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x58,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x59,0x00,0x00,0x00,0x5a,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x5c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x50,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x67,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x17,0x00,0x04,0x00, -0x6a,0x00,0x00,0x00,0x50,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x10,0x00,0x00,0x00,0x70,0x00,0x00,0x00, -0x0f,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x10,0x00,0x00,0x00, -0x74,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, -0x7e,0x00,0x00,0x00,0x50,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, -0x7f,0x00,0x00,0x00,0x7e,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x80,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x7f,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x80,0x00,0x00,0x00,0x81,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x90,0x00,0x00,0x00,0x00,0x02,0x00,0x00,0x2c,0x00,0x06,0x00, -0x09,0x00,0x00,0x00,0x91,0x00,0x00,0x00,0x90,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x36,0x00,0x05,0x00, -0x02,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x03,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x05,0x00,0x00,0x00, -0xf7,0x00,0x03,0x00,0x92,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0xfb,0x00,0x03,0x00,0x0c,0x00,0x00,0x00,0x93,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0x93,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x0d,0x00,0x00,0x00,0x0e,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x0f,0x00,0x00,0x00,0x0e,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x10,0x00,0x00,0x00,0x11,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x13,0x00,0x00,0x00, -0x11,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x13,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00,0x17,0x00,0x00,0x00, -0x0b,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x17,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x10,0x00,0x00,0x00,0x19,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x1a,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x21,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x1f,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0xae,0x00,0x05,0x00, -0x24,0x00,0x00,0x00,0x25,0x00,0x00,0x00,0x14,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0xf7,0x00,0x03,0x00,0x27,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x25,0x00,0x00,0x00, -0x26,0x00,0x00,0x00,0x27,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x26,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x92,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0x27,0x00,0x00,0x00,0x41,0x00,0x06,0x00, -0x30,0x00,0x00,0x00,0x31,0x00,0x00,0x00,0x2d,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x1a,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x10,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x31,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x33,0x00,0x00,0x00, -0x32,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x38,0x00,0x00,0x00,0x33,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, -0x38,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x3f,0x00,0x00,0x00,0x1a,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x41,0x00,0x00,0x00,0x3f,0x00,0x00,0x00,0x14,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x45,0x00,0x00,0x00, -0x3a,0x00,0x00,0x00,0x44,0x00,0x00,0x00,0x89,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x48,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, -0x44,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x4a,0x00,0x00,0x00,0x48,0x00,0x00,0x00,0x49,0x00,0x00,0x00, -0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, -0x41,0x00,0x00,0x00,0x44,0x00,0x00,0x00,0x82,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x41,0x00,0x00,0x00, -0x4e,0x00,0x00,0x00,0x41,0x00,0x07,0x00,0x5c,0x00,0x00,0x00, -0x5d,0x00,0x00,0x00,0x5a,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x45,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x50,0x00,0x00,0x00,0x5e,0x00,0x00,0x00,0x5d,0x00,0x00,0x00, -0x41,0x00,0x07,0x00,0x5c,0x00,0x00,0x00,0x61,0x00,0x00,0x00, -0x5a,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x45,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x50,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x61,0x00,0x00,0x00,0x41,0x00,0x08,0x00, -0x67,0x00,0x00,0x00,0x68,0x00,0x00,0x00,0x5a,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x45,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x4a,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x53,0x00,0x00,0x00, -0x69,0x00,0x00,0x00,0x68,0x00,0x00,0x00,0x71,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x6e,0x00,0x00,0x00,0x69,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x10,0x00,0x00,0x00,0x6f,0x00,0x00,0x00, -0x6e,0x00,0x00,0x00,0xc7,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x71,0x00,0x00,0x00,0x6f,0x00,0x00,0x00,0x70,0x00,0x00,0x00, -0x6f,0x00,0x04,0x00,0x50,0x00,0x00,0x00,0x72,0x00,0x00,0x00, -0x71,0x00,0x00,0x00,0xc2,0x00,0x05,0x00,0x53,0x00,0x00,0x00, -0x75,0x00,0x00,0x00,0x69,0x00,0x00,0x00,0x74,0x00,0x00,0x00, -0x70,0x00,0x04,0x00,0x50,0x00,0x00,0x00,0x76,0x00,0x00,0x00, -0x75,0x00,0x00,0x00,0x50,0x00,0x05,0x00,0x6a,0x00,0x00,0x00, -0x77,0x00,0x00,0x00,0x72,0x00,0x00,0x00,0x76,0x00,0x00,0x00, -0x8e,0x00,0x05,0x00,0x6a,0x00,0x00,0x00,0x7a,0x00,0x00,0x00, -0x77,0x00,0x00,0x00,0x5e,0x00,0x00,0x00,0x50,0x00,0x05,0x00, -0x6a,0x00,0x00,0x00,0x7c,0x00,0x00,0x00,0x62,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x81,0x00,0x05,0x00,0x6a,0x00,0x00,0x00, -0x7d,0x00,0x00,0x00,0x7a,0x00,0x00,0x00,0x7c,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0x4f,0x00,0x00,0x00,0x4a,0x00,0x00,0x00,0x51,0x00,0x05,0x00, -0x50,0x00,0x00,0x00,0x87,0x00,0x00,0x00,0x7d,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x5c,0x00,0x00,0x00, -0x88,0x00,0x00,0x00,0x81,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0x3e,0x00,0x03,0x00,0x88,0x00,0x00,0x00, -0x87,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x8c,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x54,0x00,0x00,0x00, -0x51,0x00,0x05,0x00,0x50,0x00,0x00,0x00,0x8e,0x00,0x00,0x00, -0x7d,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x41,0x00,0x06,0x00, -0x5c,0x00,0x00,0x00,0x8f,0x00,0x00,0x00,0x81,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x8c,0x00,0x00,0x00,0x3e,0x00,0x03,0x00, -0x8f,0x00,0x00,0x00,0x8e,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x92,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x92,0x00,0x00,0x00, -0xfd,0x00,0x01,0x00,0x38,0x00,0x01,0x00, -}; -const uint64_t get_rows_q4_1_len = 2408; - -unsigned char get_rows_q4_1_f32_data[] = { +unsigned char get_rows_q4_0_f32_data[] = { 0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, 0x97,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, -0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x09,0x00,0x00,0x00, -0x11,0x00,0x02,0x00,0x27,0x00,0x00,0x00,0x11,0x00,0x02,0x00, -0x51,0x11,0x00,0x00,0x11,0x00,0x02,0x00,0x60,0x11,0x00,0x00, -0x0b,0x00,0x06,0x00,0x01,0x00,0x00,0x00,0x47,0x4c,0x53,0x4c, -0x2e,0x73,0x74,0x64,0x2e,0x34,0x35,0x30,0x00,0x00,0x00,0x00, -0x0e,0x00,0x03,0x00,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x0f,0x00,0x0a,0x00,0x05,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x6d,0x61,0x69,0x6e,0x00,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x1f,0x00,0x00,0x00,0x2d,0x00,0x00,0x00,0x5a,0x00,0x00,0x00, -0x81,0x00,0x00,0x00,0x10,0x00,0x06,0x00,0x04,0x00,0x00,0x00, -0x11,0x00,0x00,0x00,0x00,0x02,0x00,0x00,0x01,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x0b,0x00,0x00,0x00, -0x0b,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x1d,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x1d,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x1d,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x1d,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x1d,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x2a,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00, -0x2b,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x2b,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, -0x2b,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x2d,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x2d,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x55,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x56,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x56,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x56,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x57,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x14,0x00,0x00,0x00, -0x48,0x00,0x04,0x00,0x58,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x58,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x03,0x00,0x58,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x5a,0x00,0x00,0x00,0x22,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x5a,0x00,0x00,0x00, -0x21,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x7e,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x48,0x00,0x04,0x00,0x7f,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x19,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x7f,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x03,0x00,0x7f,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x81,0x00,0x00,0x00,0x22,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x81,0x00,0x00,0x00, -0x21,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x94,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x19,0x00,0x00,0x00, -0x13,0x00,0x02,0x00,0x02,0x00,0x00,0x00,0x21,0x00,0x03,0x00, -0x03,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x15,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x17,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x03,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x0a,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x0a,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x0d,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x15,0x00,0x04,0x00, -0x10,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x10,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x16,0x00,0x03,0x00, -0x1c,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x1e,0x00,0x06,0x00, -0x1d,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x1c,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x1e,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x1d,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x1e,0x00,0x00,0x00,0x1f,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x10,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x21,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x14,0x00,0x02,0x00,0x24,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, -0x2a,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, -0x2b,0x00,0x00,0x00,0x2a,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x2c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x2b,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x2c,0x00,0x00,0x00,0x2d,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x10,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x30,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x10,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x44,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x49,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x16,0x00,0x03,0x00, -0x50,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x15,0x00,0x04,0x00, -0x53,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x54,0x00,0x00,0x00, -0x10,0x00,0x00,0x00,0x1c,0x00,0x04,0x00,0x55,0x00,0x00,0x00, -0x53,0x00,0x00,0x00,0x54,0x00,0x00,0x00,0x1e,0x00,0x05,0x00, -0x56,0x00,0x00,0x00,0x50,0x00,0x00,0x00,0x50,0x00,0x00,0x00, -0x55,0x00,0x00,0x00,0x1d,0x00,0x03,0x00,0x57,0x00,0x00,0x00, -0x56,0x00,0x00,0x00,0x1e,0x00,0x03,0x00,0x58,0x00,0x00,0x00, -0x57,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x59,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x58,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x59,0x00,0x00,0x00,0x5a,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x5c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x50,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x67,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x17,0x00,0x04,0x00, -0x6a,0x00,0x00,0x00,0x50,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x10,0x00,0x00,0x00,0x70,0x00,0x00,0x00, -0x0f,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x10,0x00,0x00,0x00, -0x74,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, -0x7e,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, -0x7f,0x00,0x00,0x00,0x7e,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x80,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x7f,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x80,0x00,0x00,0x00,0x81,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x89,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x93,0x00,0x00,0x00,0x00,0x02,0x00,0x00, -0x2c,0x00,0x06,0x00,0x09,0x00,0x00,0x00,0x94,0x00,0x00,0x00, -0x93,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0x36,0x00,0x05,0x00,0x02,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x05,0x00,0x00,0x00,0xf7,0x00,0x03,0x00,0x95,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0xfb,0x00,0x03,0x00,0x0c,0x00,0x00,0x00, -0x96,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x96,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00,0x0e,0x00,0x00,0x00, -0x0b,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x0e,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x10,0x00,0x00,0x00,0x11,0x00,0x00,0x00, -0x0f,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x13,0x00,0x00,0x00,0x11,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x14,0x00,0x00,0x00, -0x13,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00, -0x17,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x18,0x00,0x00,0x00, -0x17,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x10,0x00,0x00,0x00, -0x19,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x1a,0x00,0x00,0x00,0x19,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x21,0x00,0x00,0x00,0x22,0x00,0x00,0x00, -0x1f,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x22,0x00,0x00,0x00, -0xae,0x00,0x05,0x00,0x24,0x00,0x00,0x00,0x25,0x00,0x00,0x00, -0x14,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0xf7,0x00,0x03,0x00, -0x27,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x25,0x00,0x00,0x00,0x26,0x00,0x00,0x00,0x27,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0x26,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x95,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x27,0x00,0x00,0x00, -0x41,0x00,0x06,0x00,0x30,0x00,0x00,0x00,0x31,0x00,0x00,0x00, -0x2d,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x1a,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x10,0x00,0x00,0x00,0x32,0x00,0x00,0x00, -0x31,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x33,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x38,0x00,0x00,0x00,0x33,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x3a,0x00,0x00,0x00,0x38,0x00,0x00,0x00,0x14,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3f,0x00,0x00,0x00, -0x1a,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x41,0x00,0x00,0x00,0x3f,0x00,0x00,0x00, -0x14,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x45,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x44,0x00,0x00,0x00, -0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x48,0x00,0x00,0x00, -0x3a,0x00,0x00,0x00,0x44,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x4a,0x00,0x00,0x00,0x48,0x00,0x00,0x00, -0x49,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x4e,0x00,0x00,0x00,0x41,0x00,0x00,0x00,0x44,0x00,0x00,0x00, -0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x4f,0x00,0x00,0x00, -0x41,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x41,0x00,0x07,0x00, -0x5c,0x00,0x00,0x00,0x5d,0x00,0x00,0x00,0x5a,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x45,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x50,0x00,0x00,0x00,0x5e,0x00,0x00,0x00, -0x5d,0x00,0x00,0x00,0x41,0x00,0x07,0x00,0x5c,0x00,0x00,0x00, -0x61,0x00,0x00,0x00,0x5a,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x45,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x50,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x61,0x00,0x00,0x00, -0x41,0x00,0x08,0x00,0x67,0x00,0x00,0x00,0x68,0x00,0x00,0x00, -0x5a,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x45,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x4a,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x53,0x00,0x00,0x00,0x69,0x00,0x00,0x00,0x68,0x00,0x00,0x00, -0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x6e,0x00,0x00,0x00, -0x69,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x10,0x00,0x00,0x00, -0x6f,0x00,0x00,0x00,0x6e,0x00,0x00,0x00,0xc7,0x00,0x05,0x00, -0x10,0x00,0x00,0x00,0x71,0x00,0x00,0x00,0x6f,0x00,0x00,0x00, -0x70,0x00,0x00,0x00,0x6f,0x00,0x04,0x00,0x50,0x00,0x00,0x00, -0x72,0x00,0x00,0x00,0x71,0x00,0x00,0x00,0xc2,0x00,0x05,0x00, -0x53,0x00,0x00,0x00,0x75,0x00,0x00,0x00,0x69,0x00,0x00,0x00, -0x74,0x00,0x00,0x00,0x70,0x00,0x04,0x00,0x50,0x00,0x00,0x00, -0x76,0x00,0x00,0x00,0x75,0x00,0x00,0x00,0x50,0x00,0x05,0x00, -0x6a,0x00,0x00,0x00,0x77,0x00,0x00,0x00,0x72,0x00,0x00,0x00, -0x76,0x00,0x00,0x00,0x8e,0x00,0x05,0x00,0x6a,0x00,0x00,0x00, -0x7a,0x00,0x00,0x00,0x77,0x00,0x00,0x00,0x5e,0x00,0x00,0x00, -0x50,0x00,0x05,0x00,0x6a,0x00,0x00,0x00,0x7c,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x81,0x00,0x05,0x00, -0x6a,0x00,0x00,0x00,0x7d,0x00,0x00,0x00,0x7a,0x00,0x00,0x00, -0x7c,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x4a,0x00,0x00,0x00, -0x51,0x00,0x05,0x00,0x50,0x00,0x00,0x00,0x87,0x00,0x00,0x00, -0x7d,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x73,0x00,0x04,0x00, -0x1c,0x00,0x00,0x00,0x88,0x00,0x00,0x00,0x87,0x00,0x00,0x00, -0x41,0x00,0x06,0x00,0x89,0x00,0x00,0x00,0x8a,0x00,0x00,0x00, -0x81,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0x3e,0x00,0x03,0x00,0x8a,0x00,0x00,0x00,0x88,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8e,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0x54,0x00,0x00,0x00,0x51,0x00,0x05,0x00, -0x50,0x00,0x00,0x00,0x90,0x00,0x00,0x00,0x7d,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x1c,0x00,0x00,0x00, -0x91,0x00,0x00,0x00,0x90,0x00,0x00,0x00,0x41,0x00,0x06,0x00, -0x89,0x00,0x00,0x00,0x92,0x00,0x00,0x00,0x81,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x8e,0x00,0x00,0x00,0x3e,0x00,0x03,0x00, -0x92,0x00,0x00,0x00,0x91,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x95,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x95,0x00,0x00,0x00, -0xfd,0x00,0x01,0x00,0x38,0x00,0x01,0x00, -}; -const uint64_t get_rows_q4_1_f32_len = 2456; - -unsigned char get_rows_q4_1_f32_fp32_data[] = { -0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, -0x95,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, 0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x51,0x11,0x00,0x00, 0x11,0x00,0x02,0x00,0x60,0x11,0x00,0x00,0x0b,0x00,0x06,0x00, 0x01,0x00,0x00,0x00,0x47,0x4c,0x53,0x4c,0x2e,0x73,0x74,0x64, @@ -16690,7 +8701,7 @@ unsigned char get_rows_q4_1_f32_fp32_data[] = { 0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x0f,0x00,0x0a,0x00, 0x05,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x6d,0x61,0x69,0x6e, 0x00,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x1f,0x00,0x00,0x00, -0x2d,0x00,0x00,0x00,0x5a,0x00,0x00,0x00,0x81,0x00,0x00,0x00, +0x2d,0x00,0x00,0x00,0x5a,0x00,0x00,0x00,0x7c,0x00,0x00,0x00, 0x10,0x00,0x06,0x00,0x04,0x00,0x00,0x00,0x11,0x00,0x00,0x00, 0x00,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00, 0x47,0x00,0x04,0x00,0x0b,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, @@ -16714,91 +8725,92 @@ unsigned char get_rows_q4_1_f32_fp32_data[] = { 0x01,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x56,0x00,0x00,0x00, 0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, 0x48,0x00,0x05,0x00,0x56,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x56,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x57,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x48,0x00,0x04,0x00, -0x58,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x58,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, -0x58,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x5a,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x5a,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x7e,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00, -0x7f,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x19,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x7f,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, -0x7f,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x81,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x81,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x92,0x00,0x00,0x00, -0x0b,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x13,0x00,0x02,0x00, -0x02,0x00,0x00,0x00,0x21,0x00,0x03,0x00,0x03,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x15,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x17,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x03,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x0a,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00, -0x0b,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x0d,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x15,0x00,0x04,0x00,0x10,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x10,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x16,0x00,0x03,0x00,0x1c,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x1e,0x00,0x06,0x00,0x1d,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x1c,0x00,0x00,0x00, -0x1c,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x1e,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x1d,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x1e,0x00,0x00,0x00,0x1f,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x10,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x21,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x14,0x00,0x02,0x00, -0x24,0x00,0x00,0x00,0x1d,0x00,0x03,0x00,0x2a,0x00,0x00,0x00, -0x10,0x00,0x00,0x00,0x1e,0x00,0x03,0x00,0x2b,0x00,0x00,0x00, -0x2a,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x2c,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x2b,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x2c,0x00,0x00,0x00,0x2d,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x10,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x30,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x44,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x49,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x16,0x00,0x03,0x00,0x52,0x00,0x00,0x00, -0x10,0x00,0x00,0x00,0x15,0x00,0x04,0x00,0x53,0x00,0x00,0x00, -0x08,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x54,0x00,0x00,0x00,0x10,0x00,0x00,0x00, -0x1c,0x00,0x04,0x00,0x55,0x00,0x00,0x00,0x53,0x00,0x00,0x00, -0x54,0x00,0x00,0x00,0x1e,0x00,0x05,0x00,0x56,0x00,0x00,0x00, -0x52,0x00,0x00,0x00,0x52,0x00,0x00,0x00,0x55,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x57,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0x48,0x00,0x04,0x00,0x58,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x58,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x03,0x00,0x58,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x5a,0x00,0x00,0x00,0x22,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x5a,0x00,0x00,0x00, +0x21,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x79,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x48,0x00,0x04,0x00,0x7a,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x19,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x7a,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x03,0x00,0x7a,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x7c,0x00,0x00,0x00,0x22,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x7c,0x00,0x00,0x00, +0x21,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x8d,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x19,0x00,0x00,0x00, +0x13,0x00,0x02,0x00,0x02,0x00,0x00,0x00,0x21,0x00,0x03,0x00, +0x03,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x15,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x17,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x03,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x0a,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0x0a,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x0d,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x15,0x00,0x04,0x00, +0x10,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x10,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x16,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x16,0x00,0x03,0x00, +0x1c,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x1e,0x00,0x06,0x00, +0x1d,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x1c,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x1e,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x1d,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0x1e,0x00,0x00,0x00,0x1f,0x00,0x00,0x00, +0x09,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x10,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x21,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x14,0x00,0x02,0x00,0x24,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, +0x2a,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, +0x2b,0x00,0x00,0x00,0x2a,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x2c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x2b,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0x2c,0x00,0x00,0x00,0x2d,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x10,0x00,0x00,0x00, +0x2e,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x30,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x10,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x44,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x49,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x16,0x00,0x03,0x00, +0x52,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x15,0x00,0x04,0x00, +0x53,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x54,0x00,0x00,0x00, +0x10,0x00,0x00,0x00,0x1c,0x00,0x04,0x00,0x55,0x00,0x00,0x00, +0x53,0x00,0x00,0x00,0x54,0x00,0x00,0x00,0x1e,0x00,0x04,0x00, +0x56,0x00,0x00,0x00,0x52,0x00,0x00,0x00,0x55,0x00,0x00,0x00, 0x1d,0x00,0x03,0x00,0x57,0x00,0x00,0x00,0x56,0x00,0x00,0x00, 0x1e,0x00,0x03,0x00,0x58,0x00,0x00,0x00,0x57,0x00,0x00,0x00, 0x20,0x00,0x04,0x00,0x59,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, 0x58,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x59,0x00,0x00,0x00, 0x5a,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00, 0x5c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x52,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x68,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x53,0x00,0x00,0x00,0x17,0x00,0x04,0x00,0x6c,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x63,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x53,0x00,0x00,0x00,0x17,0x00,0x04,0x00,0x67,0x00,0x00,0x00, 0x1c,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x70,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x10,0x00,0x00,0x00,0x74,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x1d,0x00,0x03,0x00,0x7e,0x00,0x00,0x00, -0x1c,0x00,0x00,0x00,0x1e,0x00,0x03,0x00,0x7f,0x00,0x00,0x00, -0x7e,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x80,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x7f,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x80,0x00,0x00,0x00,0x81,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x88,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x1c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x91,0x00,0x00,0x00,0x00,0x02,0x00,0x00,0x2c,0x00,0x06,0x00, -0x09,0x00,0x00,0x00,0x92,0x00,0x00,0x00,0x91,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x36,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x6b,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x10,0x00,0x00,0x00,0x6f,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x1c,0x00,0x00,0x00, +0x74,0x00,0x00,0x00,0x00,0x00,0x00,0x41,0x1d,0x00,0x03,0x00, +0x79,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, +0x7a,0x00,0x00,0x00,0x79,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x7b,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x7a,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0x7b,0x00,0x00,0x00,0x7c,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x83,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x8c,0x00,0x00,0x00,0x00,0x02,0x00,0x00, +0x2c,0x00,0x06,0x00,0x09,0x00,0x00,0x00,0x8d,0x00,0x00,0x00, +0x8c,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x16,0x00,0x00,0x00, +0x2c,0x00,0x05,0x00,0x67,0x00,0x00,0x00,0x96,0x00,0x00,0x00, +0x74,0x00,0x00,0x00,0x74,0x00,0x00,0x00,0x36,0x00,0x05,0x00, 0x02,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x00, 0x03,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x05,0x00,0x00,0x00, -0xf7,0x00,0x03,0x00,0x93,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0xfb,0x00,0x03,0x00,0x0c,0x00,0x00,0x00,0x94,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0x94,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0xf7,0x00,0x03,0x00,0x8e,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0xfb,0x00,0x03,0x00,0x0c,0x00,0x00,0x00,0x8f,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0x8f,0x00,0x00,0x00,0x41,0x00,0x05,0x00, 0x0d,0x00,0x00,0x00,0x0e,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, 0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, 0x0f,0x00,0x00,0x00,0x0e,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, @@ -16819,7 +8831,7 @@ unsigned char get_rows_q4_1_f32_fp32_data[] = { 0x23,0x00,0x00,0x00,0xf7,0x00,0x03,0x00,0x27,0x00,0x00,0x00, 0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x25,0x00,0x00,0x00, 0x26,0x00,0x00,0x00,0x27,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x26,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x93,0x00,0x00,0x00, +0x26,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x8e,0x00,0x00,0x00, 0xf8,0x00,0x02,0x00,0x27,0x00,0x00,0x00,0x41,0x00,0x06,0x00, 0x30,0x00,0x00,0x00,0x31,0x00,0x00,0x00,0x2d,0x00,0x00,0x00, 0x2e,0x00,0x00,0x00,0x1a,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, @@ -16845,49 +8857,42 @@ unsigned char get_rows_q4_1_f32_fp32_data[] = { 0x45,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, 0x52,0x00,0x00,0x00,0x5e,0x00,0x00,0x00,0x5d,0x00,0x00,0x00, 0x73,0x00,0x04,0x00,0x1c,0x00,0x00,0x00,0x5f,0x00,0x00,0x00, -0x5e,0x00,0x00,0x00,0x41,0x00,0x07,0x00,0x5c,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x5a,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x45,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x52,0x00,0x00,0x00,0x63,0x00,0x00,0x00,0x62,0x00,0x00,0x00, -0x73,0x00,0x04,0x00,0x1c,0x00,0x00,0x00,0x64,0x00,0x00,0x00, -0x63,0x00,0x00,0x00,0x41,0x00,0x08,0x00,0x68,0x00,0x00,0x00, -0x69,0x00,0x00,0x00,0x5a,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x45,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x4a,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x53,0x00,0x00,0x00,0x6a,0x00,0x00,0x00, -0x69,0x00,0x00,0x00,0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x6b,0x00,0x00,0x00,0x6a,0x00,0x00,0x00,0xc7,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x71,0x00,0x00,0x00,0x6b,0x00,0x00,0x00, -0x70,0x00,0x00,0x00,0x70,0x00,0x04,0x00,0x1c,0x00,0x00,0x00, -0x72,0x00,0x00,0x00,0x71,0x00,0x00,0x00,0xc2,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x75,0x00,0x00,0x00,0x6b,0x00,0x00,0x00, -0x74,0x00,0x00,0x00,0x70,0x00,0x04,0x00,0x1c,0x00,0x00,0x00, -0x76,0x00,0x00,0x00,0x75,0x00,0x00,0x00,0x50,0x00,0x05,0x00, -0x6c,0x00,0x00,0x00,0x77,0x00,0x00,0x00,0x72,0x00,0x00,0x00, -0x76,0x00,0x00,0x00,0x8e,0x00,0x05,0x00,0x6c,0x00,0x00,0x00, -0x7a,0x00,0x00,0x00,0x77,0x00,0x00,0x00,0x5f,0x00,0x00,0x00, -0x50,0x00,0x05,0x00,0x6c,0x00,0x00,0x00,0x7c,0x00,0x00,0x00, -0x64,0x00,0x00,0x00,0x64,0x00,0x00,0x00,0x81,0x00,0x05,0x00, -0x6c,0x00,0x00,0x00,0x7d,0x00,0x00,0x00,0x7a,0x00,0x00,0x00, -0x7c,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x4a,0x00,0x00,0x00, -0x51,0x00,0x05,0x00,0x1c,0x00,0x00,0x00,0x87,0x00,0x00,0x00, -0x7d,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x41,0x00,0x06,0x00, -0x88,0x00,0x00,0x00,0x89,0x00,0x00,0x00,0x81,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x3e,0x00,0x03,0x00, -0x89,0x00,0x00,0x00,0x87,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x8d,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0x54,0x00,0x00,0x00,0x51,0x00,0x05,0x00,0x1c,0x00,0x00,0x00, -0x8f,0x00,0x00,0x00,0x7d,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x41,0x00,0x06,0x00,0x88,0x00,0x00,0x00,0x90,0x00,0x00,0x00, -0x81,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x8d,0x00,0x00,0x00, -0x3e,0x00,0x03,0x00,0x90,0x00,0x00,0x00,0x8f,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x93,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x93,0x00,0x00,0x00,0xfd,0x00,0x01,0x00,0x38,0x00,0x01,0x00, - +0x5e,0x00,0x00,0x00,0x41,0x00,0x08,0x00,0x63,0x00,0x00,0x00, +0x64,0x00,0x00,0x00,0x5a,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, +0x45,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x4a,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x53,0x00,0x00,0x00,0x65,0x00,0x00,0x00, +0x64,0x00,0x00,0x00,0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x66,0x00,0x00,0x00,0x65,0x00,0x00,0x00,0xc7,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x66,0x00,0x00,0x00, +0x6b,0x00,0x00,0x00,0x70,0x00,0x04,0x00,0x1c,0x00,0x00,0x00, +0x6d,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0xc2,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x70,0x00,0x00,0x00,0x66,0x00,0x00,0x00, +0x6f,0x00,0x00,0x00,0x70,0x00,0x04,0x00,0x1c,0x00,0x00,0x00, +0x71,0x00,0x00,0x00,0x70,0x00,0x00,0x00,0x50,0x00,0x05,0x00, +0x67,0x00,0x00,0x00,0x72,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, +0x71,0x00,0x00,0x00,0x83,0x00,0x05,0x00,0x67,0x00,0x00,0x00, +0x76,0x00,0x00,0x00,0x72,0x00,0x00,0x00,0x96,0x00,0x00,0x00, +0x8e,0x00,0x05,0x00,0x67,0x00,0x00,0x00,0x78,0x00,0x00,0x00, +0x76,0x00,0x00,0x00,0x5f,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x7f,0x00,0x00,0x00,0x4f,0x00,0x00,0x00, +0x4a,0x00,0x00,0x00,0x51,0x00,0x05,0x00,0x1c,0x00,0x00,0x00, +0x82,0x00,0x00,0x00,0x78,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x41,0x00,0x06,0x00,0x83,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0x7c,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x7f,0x00,0x00,0x00, +0x3e,0x00,0x03,0x00,0x84,0x00,0x00,0x00,0x82,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x88,0x00,0x00,0x00, +0x7f,0x00,0x00,0x00,0x54,0x00,0x00,0x00,0x51,0x00,0x05,0x00, +0x1c,0x00,0x00,0x00,0x8a,0x00,0x00,0x00,0x78,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x83,0x00,0x00,0x00, +0x8b,0x00,0x00,0x00,0x7c,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, +0x88,0x00,0x00,0x00,0x3e,0x00,0x03,0x00,0x8b,0x00,0x00,0x00, +0x8a,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x8e,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0x8e,0x00,0x00,0x00,0xfd,0x00,0x01,0x00, +0x38,0x00,0x01,0x00, }; -const uint64_t get_rows_q4_1_f32_fp32_len = 2424; +const uint64_t get_rows_q4_0_f32_len = 2356; -unsigned char get_rows_q4_1_fp32_data[] = { +unsigned char get_rows_q4_1_data[] = { 0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, 0x96,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, 0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x51,0x11,0x00,0x00, @@ -17093,503 +9098,11 @@ unsigned char get_rows_q4_1_fp32_data[] = { 0xf8,0x00,0x02,0x00,0x94,0x00,0x00,0x00,0xfd,0x00,0x01,0x00, 0x38,0x00,0x01,0x00, }; -const uint64_t get_rows_q4_1_fp32_len = 2440; +const uint64_t get_rows_q4_1_len = 2440; -unsigned char get_rows_q5_0_data[] = { +unsigned char get_rows_q4_1_f32_data[] = { 0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, -0xc2,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, -0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x09,0x00,0x00,0x00, -0x11,0x00,0x02,0x00,0x27,0x00,0x00,0x00,0x11,0x00,0x02,0x00, -0x51,0x11,0x00,0x00,0x11,0x00,0x02,0x00,0x60,0x11,0x00,0x00, -0x0b,0x00,0x06,0x00,0x01,0x00,0x00,0x00,0x47,0x4c,0x53,0x4c, -0x2e,0x73,0x74,0x64,0x2e,0x34,0x35,0x30,0x00,0x00,0x00,0x00, -0x0e,0x00,0x03,0x00,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x0f,0x00,0x0a,0x00,0x05,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x6d,0x61,0x69,0x6e,0x00,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x1f,0x00,0x00,0x00,0x2d,0x00,0x00,0x00,0x5c,0x00,0x00,0x00, -0xa6,0x00,0x00,0x00,0x10,0x00,0x06,0x00,0x04,0x00,0x00,0x00, -0x11,0x00,0x00,0x00,0x00,0x02,0x00,0x00,0x01,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x0b,0x00,0x00,0x00, -0x0b,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x1d,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x1d,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x1d,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x1d,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x1d,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x2a,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00, -0x2b,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x2b,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, -0x2b,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x2d,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x2d,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x54,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x57,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x58,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x58,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x58,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x59,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0x5a,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x5a,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x5a,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x5c,0x00,0x00,0x00, -0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x5c,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0xa3,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0xa4,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0xa4,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0xa4,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xa6,0x00,0x00,0x00, -0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0xa6,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0xb6,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x19,0x00,0x00,0x00,0x13,0x00,0x02,0x00,0x02,0x00,0x00,0x00, -0x21,0x00,0x03,0x00,0x03,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x15,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x17,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x0a,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x0d,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x15,0x00,0x04,0x00,0x10,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x10,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x16,0x00,0x03,0x00,0x1c,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x1e,0x00,0x06,0x00,0x1d,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x1c,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x1e,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x1d,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x1e,0x00,0x00,0x00, -0x1f,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x10,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x21,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x14,0x00,0x02,0x00,0x24,0x00,0x00,0x00, -0x1d,0x00,0x03,0x00,0x2a,0x00,0x00,0x00,0x10,0x00,0x00,0x00, -0x1e,0x00,0x03,0x00,0x2b,0x00,0x00,0x00,0x2a,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x2c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x2b,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x2c,0x00,0x00,0x00, -0x2d,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x10,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x30,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x10,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x44,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x49,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x16,0x00,0x03,0x00,0x50,0x00,0x00,0x00,0x10,0x00,0x00,0x00, -0x15,0x00,0x04,0x00,0x53,0x00,0x00,0x00,0x10,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x1c,0x00,0x04,0x00,0x54,0x00,0x00,0x00, -0x53,0x00,0x00,0x00,0x49,0x00,0x00,0x00,0x15,0x00,0x04,0x00, -0x55,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x56,0x00,0x00,0x00, -0x10,0x00,0x00,0x00,0x1c,0x00,0x04,0x00,0x57,0x00,0x00,0x00, -0x55,0x00,0x00,0x00,0x56,0x00,0x00,0x00,0x1e,0x00,0x05,0x00, -0x58,0x00,0x00,0x00,0x50,0x00,0x00,0x00,0x54,0x00,0x00,0x00, -0x57,0x00,0x00,0x00,0x1d,0x00,0x03,0x00,0x59,0x00,0x00,0x00, -0x58,0x00,0x00,0x00,0x1e,0x00,0x03,0x00,0x5a,0x00,0x00,0x00, -0x59,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x5b,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x5a,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x5b,0x00,0x00,0x00,0x5c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x5e,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x50,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x63,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x10,0x00,0x00,0x00,0x67,0x00,0x00,0x00,0x10,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x10,0x00,0x00,0x00,0x74,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x7a,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x84,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x55,0x00,0x00,0x00, -0x17,0x00,0x04,0x00,0x87,0x00,0x00,0x00,0x50,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x10,0x00,0x00,0x00, -0x8d,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x50,0x00,0x00,0x00,0x9e,0x00,0x00,0x00,0x00,0x4c,0x00,0x00, -0x1d,0x00,0x03,0x00,0xa3,0x00,0x00,0x00,0x50,0x00,0x00,0x00, -0x1e,0x00,0x03,0x00,0xa4,0x00,0x00,0x00,0xa3,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0xa5,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0xa4,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0xa5,0x00,0x00,0x00, -0xa6,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xb5,0x00,0x00,0x00,0x00,0x02,0x00,0x00, -0x2c,0x00,0x06,0x00,0x09,0x00,0x00,0x00,0xb6,0x00,0x00,0x00, -0xb5,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0x2c,0x00,0x05,0x00,0x87,0x00,0x00,0x00,0xc1,0x00,0x00,0x00, -0x9e,0x00,0x00,0x00,0x9e,0x00,0x00,0x00,0x36,0x00,0x05,0x00, -0x02,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x03,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x05,0x00,0x00,0x00, -0xf7,0x00,0x03,0x00,0xb7,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0xfb,0x00,0x03,0x00,0x0c,0x00,0x00,0x00,0xb8,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xb8,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x0d,0x00,0x00,0x00,0x0e,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x0f,0x00,0x00,0x00,0x0e,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x10,0x00,0x00,0x00,0x11,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x13,0x00,0x00,0x00, -0x11,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x13,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00,0x17,0x00,0x00,0x00, -0x0b,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x17,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x10,0x00,0x00,0x00,0x19,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x1a,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x21,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x1f,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0xae,0x00,0x05,0x00, -0x24,0x00,0x00,0x00,0x25,0x00,0x00,0x00,0x14,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0xf7,0x00,0x03,0x00,0x27,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x25,0x00,0x00,0x00, -0x26,0x00,0x00,0x00,0x27,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x26,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xb7,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0x27,0x00,0x00,0x00,0x41,0x00,0x06,0x00, -0x30,0x00,0x00,0x00,0x31,0x00,0x00,0x00,0x2d,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x1a,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x10,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x31,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x33,0x00,0x00,0x00, -0x32,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x38,0x00,0x00,0x00,0x33,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, -0x38,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x3f,0x00,0x00,0x00,0x1a,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x41,0x00,0x00,0x00,0x3f,0x00,0x00,0x00,0x14,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x45,0x00,0x00,0x00, -0x3a,0x00,0x00,0x00,0x44,0x00,0x00,0x00,0x89,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x48,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, -0x44,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x4a,0x00,0x00,0x00,0x48,0x00,0x00,0x00,0x49,0x00,0x00,0x00, -0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, -0x41,0x00,0x00,0x00,0x44,0x00,0x00,0x00,0x82,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x41,0x00,0x00,0x00, -0x4e,0x00,0x00,0x00,0x41,0x00,0x07,0x00,0x5e,0x00,0x00,0x00, -0x5f,0x00,0x00,0x00,0x5c,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x45,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x50,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x5f,0x00,0x00,0x00, -0x41,0x00,0x08,0x00,0x63,0x00,0x00,0x00,0x64,0x00,0x00,0x00, -0x5c,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x45,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x53,0x00,0x00,0x00,0x65,0x00,0x00,0x00,0x64,0x00,0x00,0x00, -0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x66,0x00,0x00,0x00, -0x65,0x00,0x00,0x00,0xc4,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x68,0x00,0x00,0x00,0x66,0x00,0x00,0x00,0x67,0x00,0x00,0x00, -0x41,0x00,0x08,0x00,0x63,0x00,0x00,0x00,0x6a,0x00,0x00,0x00, -0x5c,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x45,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x53,0x00,0x00,0x00,0x6b,0x00,0x00,0x00,0x6a,0x00,0x00,0x00, -0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x6b,0x00,0x00,0x00,0xc5,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x6d,0x00,0x00,0x00,0x68,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0xc2,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x73,0x00,0x00,0x00, -0x6d,0x00,0x00,0x00,0x4a,0x00,0x00,0x00,0xc4,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x75,0x00,0x00,0x00,0x73,0x00,0x00,0x00, -0x74,0x00,0x00,0x00,0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x76,0x00,0x00,0x00,0x75,0x00,0x00,0x00,0x56,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x10,0x00,0x00,0x00,0x77,0x00,0x00,0x00, -0x76,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x7b,0x00,0x00,0x00,0x4a,0x00,0x00,0x00,0x7a,0x00,0x00,0x00, -0xc2,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x7c,0x00,0x00,0x00, -0x6d,0x00,0x00,0x00,0x7b,0x00,0x00,0x00,0xc7,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x7d,0x00,0x00,0x00,0x7c,0x00,0x00,0x00, -0x56,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x10,0x00,0x00,0x00, -0x7e,0x00,0x00,0x00,0x7d,0x00,0x00,0x00,0x41,0x00,0x08,0x00, -0x84,0x00,0x00,0x00,0x85,0x00,0x00,0x00,0x5c,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x45,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x4a,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x55,0x00,0x00,0x00, -0x86,0x00,0x00,0x00,0x85,0x00,0x00,0x00,0x71,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x8b,0x00,0x00,0x00,0x86,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x10,0x00,0x00,0x00,0x8c,0x00,0x00,0x00, -0x8b,0x00,0x00,0x00,0xc7,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x8e,0x00,0x00,0x00,0x8c,0x00,0x00,0x00,0x8d,0x00,0x00,0x00, -0xc5,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x92,0x00,0x00,0x00, -0x8e,0x00,0x00,0x00,0x77,0x00,0x00,0x00,0x6f,0x00,0x04,0x00, -0x50,0x00,0x00,0x00,0x93,0x00,0x00,0x00,0x92,0x00,0x00,0x00, -0xc2,0x00,0x05,0x00,0x55,0x00,0x00,0x00,0x95,0x00,0x00,0x00, -0x86,0x00,0x00,0x00,0x74,0x00,0x00,0x00,0x71,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x96,0x00,0x00,0x00,0x95,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x10,0x00,0x00,0x00,0x97,0x00,0x00,0x00, -0x96,0x00,0x00,0x00,0xc5,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x9a,0x00,0x00,0x00,0x97,0x00,0x00,0x00,0x7e,0x00,0x00,0x00, -0x6f,0x00,0x04,0x00,0x50,0x00,0x00,0x00,0x9b,0x00,0x00,0x00, -0x9a,0x00,0x00,0x00,0x50,0x00,0x05,0x00,0x87,0x00,0x00,0x00, -0x9c,0x00,0x00,0x00,0x93,0x00,0x00,0x00,0x9b,0x00,0x00,0x00, -0x83,0x00,0x05,0x00,0x87,0x00,0x00,0x00,0xa0,0x00,0x00,0x00, -0x9c,0x00,0x00,0x00,0xc1,0x00,0x00,0x00,0x8e,0x00,0x05,0x00, -0x87,0x00,0x00,0x00,0xa2,0x00,0x00,0x00,0xa0,0x00,0x00,0x00, -0x60,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xa9,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x4a,0x00,0x00,0x00, -0x51,0x00,0x05,0x00,0x50,0x00,0x00,0x00,0xac,0x00,0x00,0x00, -0xa2,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x41,0x00,0x06,0x00, -0x5e,0x00,0x00,0x00,0xad,0x00,0x00,0x00,0xa6,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0xa9,0x00,0x00,0x00,0x3e,0x00,0x03,0x00, -0xad,0x00,0x00,0x00,0xac,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xb1,0x00,0x00,0x00,0xa9,0x00,0x00,0x00, -0x56,0x00,0x00,0x00,0x51,0x00,0x05,0x00,0x50,0x00,0x00,0x00, -0xb3,0x00,0x00,0x00,0xa2,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x41,0x00,0x06,0x00,0x5e,0x00,0x00,0x00,0xb4,0x00,0x00,0x00, -0xa6,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0xb1,0x00,0x00,0x00, -0x3e,0x00,0x03,0x00,0xb4,0x00,0x00,0x00,0xb3,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xb7,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xb7,0x00,0x00,0x00,0xfd,0x00,0x01,0x00,0x38,0x00,0x01,0x00, - -}; -const uint64_t get_rows_q5_0_len = 2868; - -unsigned char get_rows_q5_0_f32_data[] = { -0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, -0xc5,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, -0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x09,0x00,0x00,0x00, -0x11,0x00,0x02,0x00,0x27,0x00,0x00,0x00,0x11,0x00,0x02,0x00, -0x51,0x11,0x00,0x00,0x11,0x00,0x02,0x00,0x60,0x11,0x00,0x00, -0x0b,0x00,0x06,0x00,0x01,0x00,0x00,0x00,0x47,0x4c,0x53,0x4c, -0x2e,0x73,0x74,0x64,0x2e,0x34,0x35,0x30,0x00,0x00,0x00,0x00, -0x0e,0x00,0x03,0x00,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x0f,0x00,0x0a,0x00,0x05,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x6d,0x61,0x69,0x6e,0x00,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x1f,0x00,0x00,0x00,0x2d,0x00,0x00,0x00,0x5c,0x00,0x00,0x00, -0xa6,0x00,0x00,0x00,0x10,0x00,0x06,0x00,0x04,0x00,0x00,0x00, -0x11,0x00,0x00,0x00,0x00,0x02,0x00,0x00,0x01,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x0b,0x00,0x00,0x00, -0x0b,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x1d,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x1d,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x1d,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x1d,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x1d,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x2a,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00, -0x2b,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x2b,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, -0x2b,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x2d,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x2d,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x54,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x57,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x58,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x58,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x58,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x59,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0x5a,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x5a,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x5a,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x5c,0x00,0x00,0x00, -0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x5c,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0xa3,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0xa4,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0xa4,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0xa4,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xa6,0x00,0x00,0x00, -0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0xa6,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0xb9,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x19,0x00,0x00,0x00,0x13,0x00,0x02,0x00,0x02,0x00,0x00,0x00, -0x21,0x00,0x03,0x00,0x03,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x15,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x17,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x0a,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x0d,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x15,0x00,0x04,0x00,0x10,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x10,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x16,0x00,0x03,0x00,0x1c,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x1e,0x00,0x06,0x00,0x1d,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x1c,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x1e,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x1d,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x1e,0x00,0x00,0x00, -0x1f,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x10,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x21,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x14,0x00,0x02,0x00,0x24,0x00,0x00,0x00, -0x1d,0x00,0x03,0x00,0x2a,0x00,0x00,0x00,0x10,0x00,0x00,0x00, -0x1e,0x00,0x03,0x00,0x2b,0x00,0x00,0x00,0x2a,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x2c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x2b,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x2c,0x00,0x00,0x00, -0x2d,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x10,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x30,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x10,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x44,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x49,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x16,0x00,0x03,0x00,0x50,0x00,0x00,0x00,0x10,0x00,0x00,0x00, -0x15,0x00,0x04,0x00,0x53,0x00,0x00,0x00,0x10,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x1c,0x00,0x04,0x00,0x54,0x00,0x00,0x00, -0x53,0x00,0x00,0x00,0x49,0x00,0x00,0x00,0x15,0x00,0x04,0x00, -0x55,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x56,0x00,0x00,0x00, -0x10,0x00,0x00,0x00,0x1c,0x00,0x04,0x00,0x57,0x00,0x00,0x00, -0x55,0x00,0x00,0x00,0x56,0x00,0x00,0x00,0x1e,0x00,0x05,0x00, -0x58,0x00,0x00,0x00,0x50,0x00,0x00,0x00,0x54,0x00,0x00,0x00, -0x57,0x00,0x00,0x00,0x1d,0x00,0x03,0x00,0x59,0x00,0x00,0x00, -0x58,0x00,0x00,0x00,0x1e,0x00,0x03,0x00,0x5a,0x00,0x00,0x00, -0x59,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x5b,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x5a,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x5b,0x00,0x00,0x00,0x5c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x5e,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x50,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x63,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x10,0x00,0x00,0x00,0x67,0x00,0x00,0x00,0x10,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x10,0x00,0x00,0x00,0x74,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x7a,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x84,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x55,0x00,0x00,0x00, -0x17,0x00,0x04,0x00,0x87,0x00,0x00,0x00,0x50,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x10,0x00,0x00,0x00, -0x8d,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x50,0x00,0x00,0x00,0x9e,0x00,0x00,0x00,0x00,0x4c,0x00,0x00, -0x1d,0x00,0x03,0x00,0xa3,0x00,0x00,0x00,0x1c,0x00,0x00,0x00, -0x1e,0x00,0x03,0x00,0xa4,0x00,0x00,0x00,0xa3,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0xa5,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0xa4,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0xa5,0x00,0x00,0x00, -0xa6,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0xae,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x1c,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xb8,0x00,0x00,0x00, -0x00,0x02,0x00,0x00,0x2c,0x00,0x06,0x00,0x09,0x00,0x00,0x00, -0xb9,0x00,0x00,0x00,0xb8,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x2c,0x00,0x05,0x00,0x87,0x00,0x00,0x00, -0xc4,0x00,0x00,0x00,0x9e,0x00,0x00,0x00,0x9e,0x00,0x00,0x00, -0x36,0x00,0x05,0x00,0x02,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x05,0x00,0x00,0x00,0xf7,0x00,0x03,0x00,0xba,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0xfb,0x00,0x03,0x00,0x0c,0x00,0x00,0x00, -0xbb,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xbb,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00,0x0e,0x00,0x00,0x00, -0x0b,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x0e,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x10,0x00,0x00,0x00,0x11,0x00,0x00,0x00, -0x0f,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x13,0x00,0x00,0x00,0x11,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x14,0x00,0x00,0x00, -0x13,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00, -0x17,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x18,0x00,0x00,0x00, -0x17,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x10,0x00,0x00,0x00, -0x19,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x1a,0x00,0x00,0x00,0x19,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x21,0x00,0x00,0x00,0x22,0x00,0x00,0x00, -0x1f,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x22,0x00,0x00,0x00, -0xae,0x00,0x05,0x00,0x24,0x00,0x00,0x00,0x25,0x00,0x00,0x00, -0x14,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0xf7,0x00,0x03,0x00, -0x27,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x25,0x00,0x00,0x00,0x26,0x00,0x00,0x00,0x27,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0x26,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xba,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x27,0x00,0x00,0x00, -0x41,0x00,0x06,0x00,0x30,0x00,0x00,0x00,0x31,0x00,0x00,0x00, -0x2d,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x1a,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x10,0x00,0x00,0x00,0x32,0x00,0x00,0x00, -0x31,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x33,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x38,0x00,0x00,0x00,0x33,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x3a,0x00,0x00,0x00,0x38,0x00,0x00,0x00,0x14,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3f,0x00,0x00,0x00, -0x1a,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x41,0x00,0x00,0x00,0x3f,0x00,0x00,0x00, -0x14,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x45,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x44,0x00,0x00,0x00, -0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x48,0x00,0x00,0x00, -0x3a,0x00,0x00,0x00,0x44,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x4a,0x00,0x00,0x00,0x48,0x00,0x00,0x00, -0x49,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x4e,0x00,0x00,0x00,0x41,0x00,0x00,0x00,0x44,0x00,0x00,0x00, -0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x4f,0x00,0x00,0x00, -0x41,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x41,0x00,0x07,0x00, -0x5e,0x00,0x00,0x00,0x5f,0x00,0x00,0x00,0x5c,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x45,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x50,0x00,0x00,0x00,0x60,0x00,0x00,0x00, -0x5f,0x00,0x00,0x00,0x41,0x00,0x08,0x00,0x63,0x00,0x00,0x00, -0x64,0x00,0x00,0x00,0x5c,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x45,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x53,0x00,0x00,0x00,0x65,0x00,0x00,0x00, -0x64,0x00,0x00,0x00,0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x66,0x00,0x00,0x00,0x65,0x00,0x00,0x00,0xc4,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x68,0x00,0x00,0x00,0x66,0x00,0x00,0x00, -0x67,0x00,0x00,0x00,0x41,0x00,0x08,0x00,0x63,0x00,0x00,0x00, -0x6a,0x00,0x00,0x00,0x5c,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x45,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x53,0x00,0x00,0x00,0x6b,0x00,0x00,0x00, -0x6a,0x00,0x00,0x00,0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x6b,0x00,0x00,0x00,0xc5,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x68,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0xc2,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x73,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x4a,0x00,0x00,0x00, -0xc4,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x75,0x00,0x00,0x00, -0x73,0x00,0x00,0x00,0x74,0x00,0x00,0x00,0xc7,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x76,0x00,0x00,0x00,0x75,0x00,0x00,0x00, -0x56,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x10,0x00,0x00,0x00, -0x77,0x00,0x00,0x00,0x76,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x7b,0x00,0x00,0x00,0x4a,0x00,0x00,0x00, -0x7a,0x00,0x00,0x00,0xc2,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x7c,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x7b,0x00,0x00,0x00, -0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x7d,0x00,0x00,0x00, -0x7c,0x00,0x00,0x00,0x56,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x10,0x00,0x00,0x00,0x7e,0x00,0x00,0x00,0x7d,0x00,0x00,0x00, -0x41,0x00,0x08,0x00,0x84,0x00,0x00,0x00,0x85,0x00,0x00,0x00, -0x5c,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x45,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x4a,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x55,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x85,0x00,0x00,0x00, -0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x8b,0x00,0x00,0x00, -0x86,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x10,0x00,0x00,0x00, -0x8c,0x00,0x00,0x00,0x8b,0x00,0x00,0x00,0xc7,0x00,0x05,0x00, -0x10,0x00,0x00,0x00,0x8e,0x00,0x00,0x00,0x8c,0x00,0x00,0x00, -0x8d,0x00,0x00,0x00,0xc5,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x92,0x00,0x00,0x00,0x8e,0x00,0x00,0x00,0x77,0x00,0x00,0x00, -0x6f,0x00,0x04,0x00,0x50,0x00,0x00,0x00,0x93,0x00,0x00,0x00, -0x92,0x00,0x00,0x00,0xc2,0x00,0x05,0x00,0x55,0x00,0x00,0x00, -0x95,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x74,0x00,0x00,0x00, -0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x96,0x00,0x00,0x00, -0x95,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x10,0x00,0x00,0x00, -0x97,0x00,0x00,0x00,0x96,0x00,0x00,0x00,0xc5,0x00,0x05,0x00, -0x10,0x00,0x00,0x00,0x9a,0x00,0x00,0x00,0x97,0x00,0x00,0x00, -0x7e,0x00,0x00,0x00,0x6f,0x00,0x04,0x00,0x50,0x00,0x00,0x00, -0x9b,0x00,0x00,0x00,0x9a,0x00,0x00,0x00,0x50,0x00,0x05,0x00, -0x87,0x00,0x00,0x00,0x9c,0x00,0x00,0x00,0x93,0x00,0x00,0x00, -0x9b,0x00,0x00,0x00,0x83,0x00,0x05,0x00,0x87,0x00,0x00,0x00, -0xa0,0x00,0x00,0x00,0x9c,0x00,0x00,0x00,0xc4,0x00,0x00,0x00, -0x8e,0x00,0x05,0x00,0x87,0x00,0x00,0x00,0xa2,0x00,0x00,0x00, -0xa0,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xa9,0x00,0x00,0x00,0x4f,0x00,0x00,0x00, -0x4a,0x00,0x00,0x00,0x51,0x00,0x05,0x00,0x50,0x00,0x00,0x00, -0xac,0x00,0x00,0x00,0xa2,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x73,0x00,0x04,0x00,0x1c,0x00,0x00,0x00,0xad,0x00,0x00,0x00, -0xac,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0xae,0x00,0x00,0x00, -0xaf,0x00,0x00,0x00,0xa6,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0xa9,0x00,0x00,0x00,0x3e,0x00,0x03,0x00,0xaf,0x00,0x00,0x00, -0xad,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xb3,0x00,0x00,0x00,0xa9,0x00,0x00,0x00,0x56,0x00,0x00,0x00, -0x51,0x00,0x05,0x00,0x50,0x00,0x00,0x00,0xb5,0x00,0x00,0x00, -0xa2,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x73,0x00,0x04,0x00, -0x1c,0x00,0x00,0x00,0xb6,0x00,0x00,0x00,0xb5,0x00,0x00,0x00, -0x41,0x00,0x06,0x00,0xae,0x00,0x00,0x00,0xb7,0x00,0x00,0x00, -0xa6,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0xb3,0x00,0x00,0x00, -0x3e,0x00,0x03,0x00,0xb7,0x00,0x00,0x00,0xb6,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xba,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xba,0x00,0x00,0x00,0xfd,0x00,0x01,0x00,0x38,0x00,0x01,0x00, - -}; -const uint64_t get_rows_q5_0_f32_len = 2916; - -unsigned char get_rows_q5_0_f32_fp32_data[] = { -0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, -0xc2,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, +0x95,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, 0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x51,0x11,0x00,0x00, 0x11,0x00,0x02,0x00,0x60,0x11,0x00,0x00,0x0b,0x00,0x06,0x00, 0x01,0x00,0x00,0x00,0x47,0x4c,0x53,0x4c,0x2e,0x73,0x74,0x64, @@ -17597,7 +9110,7 @@ unsigned char get_rows_q5_0_f32_fp32_data[] = { 0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x0f,0x00,0x0a,0x00, 0x05,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x6d,0x61,0x69,0x6e, 0x00,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x1f,0x00,0x00,0x00, -0x2d,0x00,0x00,0x00,0x5c,0x00,0x00,0x00,0xa5,0x00,0x00,0x00, +0x2d,0x00,0x00,0x00,0x5a,0x00,0x00,0x00,0x81,0x00,0x00,0x00, 0x10,0x00,0x06,0x00,0x04,0x00,0x00,0x00,0x11,0x00,0x00,0x00, 0x00,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00, 0x47,0x00,0x04,0x00,0x0b,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, @@ -17617,106 +9130,95 @@ unsigned char get_rows_q5_0_f32_fp32_data[] = { 0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x2d,0x00,0x00,0x00, 0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, 0x2d,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x54,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x57,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x58,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x58,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x58,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x59,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0x48,0x00,0x04,0x00,0x5a,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x5a,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x55,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x56,0x00,0x00,0x00, 0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x03,0x00,0x5a,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x5c,0x00,0x00,0x00,0x22,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x5c,0x00,0x00,0x00, -0x21,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0xa2,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x48,0x00,0x04,0x00,0xa3,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x19,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0xa3,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x03,0x00,0xa3,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0xa5,0x00,0x00,0x00,0x22,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xa5,0x00,0x00,0x00, -0x21,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0xb6,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x19,0x00,0x00,0x00, -0x13,0x00,0x02,0x00,0x02,0x00,0x00,0x00,0x21,0x00,0x03,0x00, -0x03,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x15,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x17,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x03,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x0a,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x0a,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x0d,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x15,0x00,0x04,0x00, -0x10,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x10,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x16,0x00,0x03,0x00, -0x1c,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x1e,0x00,0x06,0x00, -0x1d,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x1c,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x1e,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x1d,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x1e,0x00,0x00,0x00,0x1f,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x10,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x21,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x14,0x00,0x02,0x00,0x24,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, -0x2a,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, -0x2b,0x00,0x00,0x00,0x2a,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x2c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x2b,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x2c,0x00,0x00,0x00,0x2d,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x10,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x30,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x10,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x44,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x49,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x16,0x00,0x03,0x00, -0x52,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x15,0x00,0x04,0x00, -0x53,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x1c,0x00,0x04,0x00,0x54,0x00,0x00,0x00,0x53,0x00,0x00,0x00, -0x49,0x00,0x00,0x00,0x15,0x00,0x04,0x00,0x55,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x56,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x56,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x57,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x48,0x00,0x04,0x00, +0x58,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x58,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, +0x58,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x5a,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x5a,0x00,0x00,0x00,0x21,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x7e,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00, +0x7f,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x19,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x7f,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, +0x7f,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x81,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x81,0x00,0x00,0x00,0x21,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x92,0x00,0x00,0x00, +0x0b,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x13,0x00,0x02,0x00, +0x02,0x00,0x00,0x00,0x21,0x00,0x03,0x00,0x03,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x15,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x17,0x00,0x04,0x00, +0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x03,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x0a,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x09,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00, +0x0b,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x0d,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x15,0x00,0x04,0x00,0x10,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x10,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x16,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x16,0x00,0x03,0x00,0x1c,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0x1e,0x00,0x06,0x00,0x1d,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x1c,0x00,0x00,0x00, +0x1c,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x1e,0x00,0x00,0x00, +0x09,0x00,0x00,0x00,0x1d,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0x1e,0x00,0x00,0x00,0x1f,0x00,0x00,0x00,0x09,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x10,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x21,0x00,0x00,0x00, +0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x14,0x00,0x02,0x00, +0x24,0x00,0x00,0x00,0x1d,0x00,0x03,0x00,0x2a,0x00,0x00,0x00, +0x10,0x00,0x00,0x00,0x1e,0x00,0x03,0x00,0x2b,0x00,0x00,0x00, +0x2a,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x2c,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x2b,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0x2c,0x00,0x00,0x00,0x2d,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x10,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x30,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x44,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x49,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x16,0x00,0x03,0x00,0x52,0x00,0x00,0x00, +0x10,0x00,0x00,0x00,0x15,0x00,0x04,0x00,0x53,0x00,0x00,0x00, 0x08,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x56,0x00,0x00,0x00,0x10,0x00,0x00,0x00, -0x1c,0x00,0x04,0x00,0x57,0x00,0x00,0x00,0x55,0x00,0x00,0x00, -0x56,0x00,0x00,0x00,0x1e,0x00,0x05,0x00,0x58,0x00,0x00,0x00, -0x52,0x00,0x00,0x00,0x54,0x00,0x00,0x00,0x57,0x00,0x00,0x00, -0x1d,0x00,0x03,0x00,0x59,0x00,0x00,0x00,0x58,0x00,0x00,0x00, -0x1e,0x00,0x03,0x00,0x5a,0x00,0x00,0x00,0x59,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x5b,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x5a,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x5b,0x00,0x00,0x00, -0x5c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x5e,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x52,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x64,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x53,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x10,0x00,0x00,0x00, -0x68,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x10,0x00,0x00,0x00,0x75,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x7b,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x84,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x55,0x00,0x00,0x00,0x17,0x00,0x04,0x00, -0x88,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x8c,0x00,0x00,0x00, -0x0f,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x1c,0x00,0x00,0x00, -0x9d,0x00,0x00,0x00,0x00,0x00,0x80,0x41,0x1d,0x00,0x03,0x00, -0xa2,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, -0xa3,0x00,0x00,0x00,0xa2,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0xa4,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0xa3,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0xa4,0x00,0x00,0x00,0xa5,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0xac,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xb5,0x00,0x00,0x00,0x00,0x02,0x00,0x00, -0x2c,0x00,0x06,0x00,0x09,0x00,0x00,0x00,0xb6,0x00,0x00,0x00, -0xb5,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0x2c,0x00,0x05,0x00,0x88,0x00,0x00,0x00,0xc1,0x00,0x00,0x00, -0x9d,0x00,0x00,0x00,0x9d,0x00,0x00,0x00,0x36,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x54,0x00,0x00,0x00,0x10,0x00,0x00,0x00, +0x1c,0x00,0x04,0x00,0x55,0x00,0x00,0x00,0x53,0x00,0x00,0x00, +0x54,0x00,0x00,0x00,0x1e,0x00,0x05,0x00,0x56,0x00,0x00,0x00, +0x52,0x00,0x00,0x00,0x52,0x00,0x00,0x00,0x55,0x00,0x00,0x00, +0x1d,0x00,0x03,0x00,0x57,0x00,0x00,0x00,0x56,0x00,0x00,0x00, +0x1e,0x00,0x03,0x00,0x58,0x00,0x00,0x00,0x57,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x59,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x58,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x59,0x00,0x00,0x00, +0x5a,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x5c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x52,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x68,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x53,0x00,0x00,0x00,0x17,0x00,0x04,0x00,0x6c,0x00,0x00,0x00, +0x1c,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x70,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x10,0x00,0x00,0x00,0x74,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x1d,0x00,0x03,0x00,0x7e,0x00,0x00,0x00, +0x1c,0x00,0x00,0x00,0x1e,0x00,0x03,0x00,0x7f,0x00,0x00,0x00, +0x7e,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x80,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x7f,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0x80,0x00,0x00,0x00,0x81,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x88,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x1c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x91,0x00,0x00,0x00,0x00,0x02,0x00,0x00,0x2c,0x00,0x06,0x00, +0x09,0x00,0x00,0x00,0x92,0x00,0x00,0x00,0x91,0x00,0x00,0x00, +0x16,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x36,0x00,0x05,0x00, 0x02,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x00, 0x03,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x05,0x00,0x00,0x00, -0xf7,0x00,0x03,0x00,0xb7,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0xfb,0x00,0x03,0x00,0x0c,0x00,0x00,0x00,0xb8,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xb8,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0xf7,0x00,0x03,0x00,0x93,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0xfb,0x00,0x03,0x00,0x0c,0x00,0x00,0x00,0x94,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0x94,0x00,0x00,0x00,0x41,0x00,0x05,0x00, 0x0d,0x00,0x00,0x00,0x0e,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, 0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, 0x0f,0x00,0x00,0x00,0x0e,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, @@ -17737,7 +9239,7 @@ unsigned char get_rows_q5_0_f32_fp32_data[] = { 0x23,0x00,0x00,0x00,0xf7,0x00,0x03,0x00,0x27,0x00,0x00,0x00, 0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x25,0x00,0x00,0x00, 0x26,0x00,0x00,0x00,0x27,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x26,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xb7,0x00,0x00,0x00, +0x26,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x93,0x00,0x00,0x00, 0xf8,0x00,0x02,0x00,0x27,0x00,0x00,0x00,0x41,0x00,0x06,0x00, 0x30,0x00,0x00,0x00,0x31,0x00,0x00,0x00,0x2d,0x00,0x00,0x00, 0x2e,0x00,0x00,0x00,0x1a,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, @@ -17758,80 +9260,54 @@ unsigned char get_rows_q5_0_f32_fp32_data[] = { 0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, 0x41,0x00,0x00,0x00,0x44,0x00,0x00,0x00,0x82,0x00,0x05,0x00, 0x06,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x41,0x00,0x00,0x00, -0x4e,0x00,0x00,0x00,0x41,0x00,0x07,0x00,0x5e,0x00,0x00,0x00, -0x5f,0x00,0x00,0x00,0x5c,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, +0x4e,0x00,0x00,0x00,0x41,0x00,0x07,0x00,0x5c,0x00,0x00,0x00, +0x5d,0x00,0x00,0x00,0x5a,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, 0x45,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x52,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x5f,0x00,0x00,0x00, -0x73,0x00,0x04,0x00,0x1c,0x00,0x00,0x00,0x61,0x00,0x00,0x00, -0x60,0x00,0x00,0x00,0x41,0x00,0x08,0x00,0x64,0x00,0x00,0x00, -0x65,0x00,0x00,0x00,0x5c,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x45,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x53,0x00,0x00,0x00,0x66,0x00,0x00,0x00, -0x65,0x00,0x00,0x00,0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x67,0x00,0x00,0x00,0x66,0x00,0x00,0x00,0xc4,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x69,0x00,0x00,0x00,0x67,0x00,0x00,0x00, -0x68,0x00,0x00,0x00,0x41,0x00,0x08,0x00,0x64,0x00,0x00,0x00, -0x6b,0x00,0x00,0x00,0x5c,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x45,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x53,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x6b,0x00,0x00,0x00,0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x6d,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0xc5,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x6e,0x00,0x00,0x00,0x69,0x00,0x00,0x00, -0x6d,0x00,0x00,0x00,0xc2,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x74,0x00,0x00,0x00,0x6e,0x00,0x00,0x00,0x4a,0x00,0x00,0x00, -0xc4,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x76,0x00,0x00,0x00, -0x74,0x00,0x00,0x00,0x75,0x00,0x00,0x00,0xc7,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x77,0x00,0x00,0x00,0x76,0x00,0x00,0x00, -0x56,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x10,0x00,0x00,0x00, -0x78,0x00,0x00,0x00,0x77,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x7c,0x00,0x00,0x00,0x4a,0x00,0x00,0x00, -0x7b,0x00,0x00,0x00,0xc2,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x7d,0x00,0x00,0x00,0x6e,0x00,0x00,0x00,0x7c,0x00,0x00,0x00, -0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x7e,0x00,0x00,0x00, -0x7d,0x00,0x00,0x00,0x56,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x10,0x00,0x00,0x00,0x7f,0x00,0x00,0x00,0x7e,0x00,0x00,0x00, -0x41,0x00,0x08,0x00,0x84,0x00,0x00,0x00,0x85,0x00,0x00,0x00, -0x5c,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x45,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x4a,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x55,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x85,0x00,0x00,0x00, -0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x87,0x00,0x00,0x00, -0x86,0x00,0x00,0x00,0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x8d,0x00,0x00,0x00,0x87,0x00,0x00,0x00,0x8c,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x91,0x00,0x00,0x00, -0x78,0x00,0x00,0x00,0xc5,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x92,0x00,0x00,0x00,0x8d,0x00,0x00,0x00,0x91,0x00,0x00,0x00, -0x70,0x00,0x04,0x00,0x1c,0x00,0x00,0x00,0x93,0x00,0x00,0x00, -0x92,0x00,0x00,0x00,0xc2,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x95,0x00,0x00,0x00,0x87,0x00,0x00,0x00,0x75,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x98,0x00,0x00,0x00, -0x7f,0x00,0x00,0x00,0xc5,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x99,0x00,0x00,0x00,0x95,0x00,0x00,0x00,0x98,0x00,0x00,0x00, -0x70,0x00,0x04,0x00,0x1c,0x00,0x00,0x00,0x9a,0x00,0x00,0x00, -0x99,0x00,0x00,0x00,0x50,0x00,0x05,0x00,0x88,0x00,0x00,0x00, -0x9b,0x00,0x00,0x00,0x93,0x00,0x00,0x00,0x9a,0x00,0x00,0x00, -0x83,0x00,0x05,0x00,0x88,0x00,0x00,0x00,0x9f,0x00,0x00,0x00, -0x9b,0x00,0x00,0x00,0xc1,0x00,0x00,0x00,0x8e,0x00,0x05,0x00, -0x88,0x00,0x00,0x00,0xa1,0x00,0x00,0x00,0x9f,0x00,0x00,0x00, -0x61,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xa8,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x4a,0x00,0x00,0x00, -0x51,0x00,0x05,0x00,0x1c,0x00,0x00,0x00,0xab,0x00,0x00,0x00, -0xa1,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x41,0x00,0x06,0x00, -0xac,0x00,0x00,0x00,0xad,0x00,0x00,0x00,0xa5,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0xa8,0x00,0x00,0x00,0x3e,0x00,0x03,0x00, -0xad,0x00,0x00,0x00,0xab,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xb1,0x00,0x00,0x00,0xa8,0x00,0x00,0x00, -0x56,0x00,0x00,0x00,0x51,0x00,0x05,0x00,0x1c,0x00,0x00,0x00, -0xb3,0x00,0x00,0x00,0xa1,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x41,0x00,0x06,0x00,0xac,0x00,0x00,0x00,0xb4,0x00,0x00,0x00, -0xa5,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0xb1,0x00,0x00,0x00, -0x3e,0x00,0x03,0x00,0xb4,0x00,0x00,0x00,0xb3,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xb7,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xb7,0x00,0x00,0x00,0xfd,0x00,0x01,0x00,0x38,0x00,0x01,0x00, +0x52,0x00,0x00,0x00,0x5e,0x00,0x00,0x00,0x5d,0x00,0x00,0x00, +0x73,0x00,0x04,0x00,0x1c,0x00,0x00,0x00,0x5f,0x00,0x00,0x00, +0x5e,0x00,0x00,0x00,0x41,0x00,0x07,0x00,0x5c,0x00,0x00,0x00, +0x62,0x00,0x00,0x00,0x5a,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, +0x45,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x52,0x00,0x00,0x00,0x63,0x00,0x00,0x00,0x62,0x00,0x00,0x00, +0x73,0x00,0x04,0x00,0x1c,0x00,0x00,0x00,0x64,0x00,0x00,0x00, +0x63,0x00,0x00,0x00,0x41,0x00,0x08,0x00,0x68,0x00,0x00,0x00, +0x69,0x00,0x00,0x00,0x5a,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, +0x45,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x4a,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x53,0x00,0x00,0x00,0x6a,0x00,0x00,0x00, +0x69,0x00,0x00,0x00,0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x6b,0x00,0x00,0x00,0x6a,0x00,0x00,0x00,0xc7,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x71,0x00,0x00,0x00,0x6b,0x00,0x00,0x00, +0x70,0x00,0x00,0x00,0x70,0x00,0x04,0x00,0x1c,0x00,0x00,0x00, +0x72,0x00,0x00,0x00,0x71,0x00,0x00,0x00,0xc2,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x75,0x00,0x00,0x00,0x6b,0x00,0x00,0x00, +0x74,0x00,0x00,0x00,0x70,0x00,0x04,0x00,0x1c,0x00,0x00,0x00, +0x76,0x00,0x00,0x00,0x75,0x00,0x00,0x00,0x50,0x00,0x05,0x00, +0x6c,0x00,0x00,0x00,0x77,0x00,0x00,0x00,0x72,0x00,0x00,0x00, +0x76,0x00,0x00,0x00,0x8e,0x00,0x05,0x00,0x6c,0x00,0x00,0x00, +0x7a,0x00,0x00,0x00,0x77,0x00,0x00,0x00,0x5f,0x00,0x00,0x00, +0x50,0x00,0x05,0x00,0x6c,0x00,0x00,0x00,0x7c,0x00,0x00,0x00, +0x64,0x00,0x00,0x00,0x64,0x00,0x00,0x00,0x81,0x00,0x05,0x00, +0x6c,0x00,0x00,0x00,0x7d,0x00,0x00,0x00,0x7a,0x00,0x00,0x00, +0x7c,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x4a,0x00,0x00,0x00, +0x51,0x00,0x05,0x00,0x1c,0x00,0x00,0x00,0x87,0x00,0x00,0x00, +0x7d,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x41,0x00,0x06,0x00, +0x88,0x00,0x00,0x00,0x89,0x00,0x00,0x00,0x81,0x00,0x00,0x00, +0x2e,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x3e,0x00,0x03,0x00, +0x89,0x00,0x00,0x00,0x87,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x8d,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0x54,0x00,0x00,0x00,0x51,0x00,0x05,0x00,0x1c,0x00,0x00,0x00, +0x8f,0x00,0x00,0x00,0x7d,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x41,0x00,0x06,0x00,0x88,0x00,0x00,0x00,0x90,0x00,0x00,0x00, +0x81,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x8d,0x00,0x00,0x00, +0x3e,0x00,0x03,0x00,0x90,0x00,0x00,0x00,0x8f,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x93,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0x93,0x00,0x00,0x00,0xfd,0x00,0x01,0x00,0x38,0x00,0x01,0x00, }; -const uint64_t get_rows_q5_0_f32_fp32_len = 2868; +const uint64_t get_rows_q4_1_f32_len = 2424; -unsigned char get_rows_q5_0_fp32_data[] = { +unsigned char get_rows_q5_0_data[] = { 0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, 0xc3,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, 0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x51,0x11,0x00,0x00, @@ -18074,485 +9550,11 @@ unsigned char get_rows_q5_0_fp32_data[] = { 0xf8,0x00,0x02,0x00,0xb8,0x00,0x00,0x00,0xfd,0x00,0x01,0x00, 0x38,0x00,0x01,0x00, }; -const uint64_t get_rows_q5_0_fp32_len = 2884; +const uint64_t get_rows_q5_0_len = 2884; -unsigned char get_rows_q5_1_data[] = { +unsigned char get_rows_q5_0_f32_data[] = { 0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, -0xb4,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, -0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x09,0x00,0x00,0x00, -0x11,0x00,0x02,0x00,0x27,0x00,0x00,0x00,0x11,0x00,0x02,0x00, -0x51,0x11,0x00,0x00,0x11,0x00,0x02,0x00,0x60,0x11,0x00,0x00, -0x0b,0x00,0x06,0x00,0x01,0x00,0x00,0x00,0x47,0x4c,0x53,0x4c, -0x2e,0x73,0x74,0x64,0x2e,0x34,0x35,0x30,0x00,0x00,0x00,0x00, -0x0e,0x00,0x03,0x00,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x0f,0x00,0x0a,0x00,0x05,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x6d,0x61,0x69,0x6e,0x00,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x1f,0x00,0x00,0x00,0x2d,0x00,0x00,0x00,0x5a,0x00,0x00,0x00, -0xa1,0x00,0x00,0x00,0x10,0x00,0x06,0x00,0x04,0x00,0x00,0x00, -0x11,0x00,0x00,0x00,0x00,0x02,0x00,0x00,0x01,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x0b,0x00,0x00,0x00, -0x0b,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x1d,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x1d,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x1d,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x1d,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x1d,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x2a,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00, -0x2b,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x2b,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, -0x2b,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x2d,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x2d,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x55,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x56,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x56,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x56,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x56,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x08,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x57,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x48,0x00,0x04,0x00, -0x58,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x58,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, -0x58,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x5a,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x5a,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x9e,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x48,0x00,0x04,0x00, -0x9f,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x19,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x9f,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, -0x9f,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0xa1,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0xa1,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xb1,0x00,0x00,0x00, -0x0b,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x13,0x00,0x02,0x00, -0x02,0x00,0x00,0x00,0x21,0x00,0x03,0x00,0x03,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x15,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x17,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x03,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x0a,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00, -0x0b,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x0d,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x15,0x00,0x04,0x00,0x10,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x10,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x16,0x00,0x03,0x00,0x1c,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x1e,0x00,0x06,0x00,0x1d,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x1c,0x00,0x00,0x00, -0x1c,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x1e,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x1d,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x1e,0x00,0x00,0x00,0x1f,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x10,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x21,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x14,0x00,0x02,0x00, -0x24,0x00,0x00,0x00,0x1d,0x00,0x03,0x00,0x2a,0x00,0x00,0x00, -0x10,0x00,0x00,0x00,0x1e,0x00,0x03,0x00,0x2b,0x00,0x00,0x00, -0x2a,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x2c,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x2b,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x2c,0x00,0x00,0x00,0x2d,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x10,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x30,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x44,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x49,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x16,0x00,0x03,0x00,0x50,0x00,0x00,0x00, -0x10,0x00,0x00,0x00,0x15,0x00,0x04,0x00,0x53,0x00,0x00,0x00, -0x08,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x54,0x00,0x00,0x00,0x10,0x00,0x00,0x00, -0x1c,0x00,0x04,0x00,0x55,0x00,0x00,0x00,0x53,0x00,0x00,0x00, -0x54,0x00,0x00,0x00,0x1e,0x00,0x06,0x00,0x56,0x00,0x00,0x00, -0x50,0x00,0x00,0x00,0x50,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x55,0x00,0x00,0x00,0x1d,0x00,0x03,0x00,0x57,0x00,0x00,0x00, -0x56,0x00,0x00,0x00,0x1e,0x00,0x03,0x00,0x58,0x00,0x00,0x00, -0x57,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x59,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x58,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x59,0x00,0x00,0x00,0x5a,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x5c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x50,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x67,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x10,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x74,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x10,0x00,0x00,0x00, -0x7d,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x7f,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x53,0x00,0x00,0x00, -0x17,0x00,0x04,0x00,0x82,0x00,0x00,0x00,0x50,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x10,0x00,0x00,0x00, -0x88,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, -0x9e,0x00,0x00,0x00,0x50,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, -0x9f,0x00,0x00,0x00,0x9e,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0xa0,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x9f,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0xa0,0x00,0x00,0x00,0xa1,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xb0,0x00,0x00,0x00,0x00,0x02,0x00,0x00,0x2c,0x00,0x06,0x00, -0x09,0x00,0x00,0x00,0xb1,0x00,0x00,0x00,0xb0,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x36,0x00,0x05,0x00, -0x02,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x03,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x05,0x00,0x00,0x00, -0xf7,0x00,0x03,0x00,0xb2,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0xfb,0x00,0x03,0x00,0x0c,0x00,0x00,0x00,0xb3,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xb3,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x0d,0x00,0x00,0x00,0x0e,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x0f,0x00,0x00,0x00,0x0e,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x10,0x00,0x00,0x00,0x11,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x13,0x00,0x00,0x00, -0x11,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x13,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00,0x17,0x00,0x00,0x00, -0x0b,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x17,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x10,0x00,0x00,0x00,0x19,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x1a,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x21,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x1f,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0xae,0x00,0x05,0x00, -0x24,0x00,0x00,0x00,0x25,0x00,0x00,0x00,0x14,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0xf7,0x00,0x03,0x00,0x27,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x25,0x00,0x00,0x00, -0x26,0x00,0x00,0x00,0x27,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x26,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xb2,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0x27,0x00,0x00,0x00,0x41,0x00,0x06,0x00, -0x30,0x00,0x00,0x00,0x31,0x00,0x00,0x00,0x2d,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x1a,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x10,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x31,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x33,0x00,0x00,0x00, -0x32,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x38,0x00,0x00,0x00,0x33,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, -0x38,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x3f,0x00,0x00,0x00,0x1a,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x41,0x00,0x00,0x00,0x3f,0x00,0x00,0x00,0x14,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x45,0x00,0x00,0x00, -0x3a,0x00,0x00,0x00,0x44,0x00,0x00,0x00,0x89,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x48,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, -0x44,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x4a,0x00,0x00,0x00,0x48,0x00,0x00,0x00,0x49,0x00,0x00,0x00, -0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, -0x41,0x00,0x00,0x00,0x44,0x00,0x00,0x00,0x82,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x41,0x00,0x00,0x00, -0x4e,0x00,0x00,0x00,0x41,0x00,0x07,0x00,0x5c,0x00,0x00,0x00, -0x5d,0x00,0x00,0x00,0x5a,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x45,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x50,0x00,0x00,0x00,0x5e,0x00,0x00,0x00,0x5d,0x00,0x00,0x00, -0x41,0x00,0x07,0x00,0x5c,0x00,0x00,0x00,0x61,0x00,0x00,0x00, -0x5a,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x45,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x50,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x61,0x00,0x00,0x00,0x41,0x00,0x07,0x00, -0x67,0x00,0x00,0x00,0x68,0x00,0x00,0x00,0x5a,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x45,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x69,0x00,0x00,0x00, -0x68,0x00,0x00,0x00,0xc2,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x6b,0x00,0x00,0x00,0x69,0x00,0x00,0x00,0x4a,0x00,0x00,0x00, -0xc4,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, -0x6b,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0xc7,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x6e,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, -0x54,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x10,0x00,0x00,0x00, -0x6f,0x00,0x00,0x00,0x6e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x72,0x00,0x00,0x00,0x68,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x75,0x00,0x00,0x00, -0x4a,0x00,0x00,0x00,0x74,0x00,0x00,0x00,0xc2,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x76,0x00,0x00,0x00,0x72,0x00,0x00,0x00, -0x75,0x00,0x00,0x00,0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x77,0x00,0x00,0x00,0x76,0x00,0x00,0x00,0x54,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x10,0x00,0x00,0x00,0x78,0x00,0x00,0x00, -0x77,0x00,0x00,0x00,0x41,0x00,0x08,0x00,0x7f,0x00,0x00,0x00, -0x80,0x00,0x00,0x00,0x5a,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x45,0x00,0x00,0x00,0x7d,0x00,0x00,0x00,0x4a,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x53,0x00,0x00,0x00,0x81,0x00,0x00,0x00, -0x80,0x00,0x00,0x00,0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x86,0x00,0x00,0x00,0x81,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x10,0x00,0x00,0x00,0x87,0x00,0x00,0x00,0x86,0x00,0x00,0x00, -0xc7,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x89,0x00,0x00,0x00, -0x87,0x00,0x00,0x00,0x88,0x00,0x00,0x00,0xc5,0x00,0x05,0x00, -0x10,0x00,0x00,0x00,0x8d,0x00,0x00,0x00,0x89,0x00,0x00,0x00, -0x6f,0x00,0x00,0x00,0x6f,0x00,0x04,0x00,0x50,0x00,0x00,0x00, -0x8e,0x00,0x00,0x00,0x8d,0x00,0x00,0x00,0xc2,0x00,0x05,0x00, -0x53,0x00,0x00,0x00,0x90,0x00,0x00,0x00,0x81,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x91,0x00,0x00,0x00,0x90,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x10,0x00,0x00,0x00,0x92,0x00,0x00,0x00,0x91,0x00,0x00,0x00, -0xc5,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x95,0x00,0x00,0x00, -0x92,0x00,0x00,0x00,0x78,0x00,0x00,0x00,0x6f,0x00,0x04,0x00, -0x50,0x00,0x00,0x00,0x96,0x00,0x00,0x00,0x95,0x00,0x00,0x00, -0x50,0x00,0x05,0x00,0x82,0x00,0x00,0x00,0x97,0x00,0x00,0x00, -0x8e,0x00,0x00,0x00,0x96,0x00,0x00,0x00,0x8e,0x00,0x05,0x00, -0x82,0x00,0x00,0x00,0x9a,0x00,0x00,0x00,0x97,0x00,0x00,0x00, -0x5e,0x00,0x00,0x00,0x50,0x00,0x05,0x00,0x82,0x00,0x00,0x00, -0x9c,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x62,0x00,0x00,0x00, -0x81,0x00,0x05,0x00,0x82,0x00,0x00,0x00,0x9d,0x00,0x00,0x00, -0x9a,0x00,0x00,0x00,0x9c,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xa4,0x00,0x00,0x00,0x4f,0x00,0x00,0x00, -0x4a,0x00,0x00,0x00,0x51,0x00,0x05,0x00,0x50,0x00,0x00,0x00, -0xa7,0x00,0x00,0x00,0x9d,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x41,0x00,0x06,0x00,0x5c,0x00,0x00,0x00,0xa8,0x00,0x00,0x00, -0xa1,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0xa4,0x00,0x00,0x00, -0x3e,0x00,0x03,0x00,0xa8,0x00,0x00,0x00,0xa7,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xac,0x00,0x00,0x00, -0xa4,0x00,0x00,0x00,0x54,0x00,0x00,0x00,0x51,0x00,0x05,0x00, -0x50,0x00,0x00,0x00,0xae,0x00,0x00,0x00,0x9d,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x5c,0x00,0x00,0x00, -0xaf,0x00,0x00,0x00,0xa1,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0xac,0x00,0x00,0x00,0x3e,0x00,0x03,0x00,0xaf,0x00,0x00,0x00, -0xae,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xb2,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xb2,0x00,0x00,0x00,0xfd,0x00,0x01,0x00, -0x38,0x00,0x01,0x00, -}; -const uint64_t get_rows_q5_1_len = 2764; - -unsigned char get_rows_q5_1_f32_data[] = { -0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, -0xb7,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, -0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x09,0x00,0x00,0x00, -0x11,0x00,0x02,0x00,0x27,0x00,0x00,0x00,0x11,0x00,0x02,0x00, -0x51,0x11,0x00,0x00,0x11,0x00,0x02,0x00,0x60,0x11,0x00,0x00, -0x0b,0x00,0x06,0x00,0x01,0x00,0x00,0x00,0x47,0x4c,0x53,0x4c, -0x2e,0x73,0x74,0x64,0x2e,0x34,0x35,0x30,0x00,0x00,0x00,0x00, -0x0e,0x00,0x03,0x00,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x0f,0x00,0x0a,0x00,0x05,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x6d,0x61,0x69,0x6e,0x00,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x1f,0x00,0x00,0x00,0x2d,0x00,0x00,0x00,0x5a,0x00,0x00,0x00, -0xa1,0x00,0x00,0x00,0x10,0x00,0x06,0x00,0x04,0x00,0x00,0x00, -0x11,0x00,0x00,0x00,0x00,0x02,0x00,0x00,0x01,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x0b,0x00,0x00,0x00, -0x0b,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x1d,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x1d,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x1d,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x1d,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x1d,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x2a,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00, -0x2b,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x2b,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, -0x2b,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x2d,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x2d,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x55,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x56,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x56,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x56,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x56,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x08,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x57,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x48,0x00,0x04,0x00, -0x58,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x58,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, -0x58,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x5a,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x5a,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x9e,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00, -0x9f,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x19,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x9f,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, -0x9f,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0xa1,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0xa1,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xb4,0x00,0x00,0x00, -0x0b,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x13,0x00,0x02,0x00, -0x02,0x00,0x00,0x00,0x21,0x00,0x03,0x00,0x03,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x15,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x17,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x03,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x0a,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00, -0x0b,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x0d,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x15,0x00,0x04,0x00,0x10,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x10,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x16,0x00,0x03,0x00,0x1c,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x1e,0x00,0x06,0x00,0x1d,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x1c,0x00,0x00,0x00, -0x1c,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x1e,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x1d,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x1e,0x00,0x00,0x00,0x1f,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x10,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x21,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x14,0x00,0x02,0x00, -0x24,0x00,0x00,0x00,0x1d,0x00,0x03,0x00,0x2a,0x00,0x00,0x00, -0x10,0x00,0x00,0x00,0x1e,0x00,0x03,0x00,0x2b,0x00,0x00,0x00, -0x2a,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x2c,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x2b,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x2c,0x00,0x00,0x00,0x2d,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x10,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x30,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x44,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x49,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x16,0x00,0x03,0x00,0x50,0x00,0x00,0x00, -0x10,0x00,0x00,0x00,0x15,0x00,0x04,0x00,0x53,0x00,0x00,0x00, -0x08,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x54,0x00,0x00,0x00,0x10,0x00,0x00,0x00, -0x1c,0x00,0x04,0x00,0x55,0x00,0x00,0x00,0x53,0x00,0x00,0x00, -0x54,0x00,0x00,0x00,0x1e,0x00,0x06,0x00,0x56,0x00,0x00,0x00, -0x50,0x00,0x00,0x00,0x50,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x55,0x00,0x00,0x00,0x1d,0x00,0x03,0x00,0x57,0x00,0x00,0x00, -0x56,0x00,0x00,0x00,0x1e,0x00,0x03,0x00,0x58,0x00,0x00,0x00, -0x57,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x59,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x58,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x59,0x00,0x00,0x00,0x5a,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x5c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x50,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x67,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x10,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x74,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x10,0x00,0x00,0x00, -0x7d,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x7f,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x53,0x00,0x00,0x00, -0x17,0x00,0x04,0x00,0x82,0x00,0x00,0x00,0x50,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x10,0x00,0x00,0x00, -0x88,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, -0x9e,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, -0x9f,0x00,0x00,0x00,0x9e,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0xa0,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x9f,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0xa0,0x00,0x00,0x00,0xa1,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0xa9,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xb3,0x00,0x00,0x00,0x00,0x02,0x00,0x00, -0x2c,0x00,0x06,0x00,0x09,0x00,0x00,0x00,0xb4,0x00,0x00,0x00, -0xb3,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0x36,0x00,0x05,0x00,0x02,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x05,0x00,0x00,0x00,0xf7,0x00,0x03,0x00,0xb5,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0xfb,0x00,0x03,0x00,0x0c,0x00,0x00,0x00, -0xb6,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xb6,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00,0x0e,0x00,0x00,0x00, -0x0b,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x0e,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x10,0x00,0x00,0x00,0x11,0x00,0x00,0x00, -0x0f,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x13,0x00,0x00,0x00,0x11,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x14,0x00,0x00,0x00, -0x13,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00, -0x17,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x18,0x00,0x00,0x00, -0x17,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x10,0x00,0x00,0x00, -0x19,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x1a,0x00,0x00,0x00,0x19,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x21,0x00,0x00,0x00,0x22,0x00,0x00,0x00, -0x1f,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x22,0x00,0x00,0x00, -0xae,0x00,0x05,0x00,0x24,0x00,0x00,0x00,0x25,0x00,0x00,0x00, -0x14,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0xf7,0x00,0x03,0x00, -0x27,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x25,0x00,0x00,0x00,0x26,0x00,0x00,0x00,0x27,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0x26,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xb5,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x27,0x00,0x00,0x00, -0x41,0x00,0x06,0x00,0x30,0x00,0x00,0x00,0x31,0x00,0x00,0x00, -0x2d,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x1a,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x10,0x00,0x00,0x00,0x32,0x00,0x00,0x00, -0x31,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x33,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x38,0x00,0x00,0x00,0x33,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x3a,0x00,0x00,0x00,0x38,0x00,0x00,0x00,0x14,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3f,0x00,0x00,0x00, -0x1a,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x41,0x00,0x00,0x00,0x3f,0x00,0x00,0x00, -0x14,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x45,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x44,0x00,0x00,0x00, -0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x48,0x00,0x00,0x00, -0x3a,0x00,0x00,0x00,0x44,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x4a,0x00,0x00,0x00,0x48,0x00,0x00,0x00, -0x49,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x4e,0x00,0x00,0x00,0x41,0x00,0x00,0x00,0x44,0x00,0x00,0x00, -0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x4f,0x00,0x00,0x00, -0x41,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x41,0x00,0x07,0x00, -0x5c,0x00,0x00,0x00,0x5d,0x00,0x00,0x00,0x5a,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x45,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x50,0x00,0x00,0x00,0x5e,0x00,0x00,0x00, -0x5d,0x00,0x00,0x00,0x41,0x00,0x07,0x00,0x5c,0x00,0x00,0x00, -0x61,0x00,0x00,0x00,0x5a,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x45,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x50,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x61,0x00,0x00,0x00, -0x41,0x00,0x07,0x00,0x67,0x00,0x00,0x00,0x68,0x00,0x00,0x00, -0x5a,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x45,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x69,0x00,0x00,0x00,0x68,0x00,0x00,0x00,0xc2,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x6b,0x00,0x00,0x00,0x69,0x00,0x00,0x00, -0x4a,0x00,0x00,0x00,0xc4,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x6d,0x00,0x00,0x00,0x6b,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x6e,0x00,0x00,0x00, -0x6d,0x00,0x00,0x00,0x54,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x10,0x00,0x00,0x00,0x6f,0x00,0x00,0x00,0x6e,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x72,0x00,0x00,0x00, -0x68,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x75,0x00,0x00,0x00,0x4a,0x00,0x00,0x00,0x74,0x00,0x00,0x00, -0xc2,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x76,0x00,0x00,0x00, -0x72,0x00,0x00,0x00,0x75,0x00,0x00,0x00,0xc7,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x77,0x00,0x00,0x00,0x76,0x00,0x00,0x00, -0x54,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x10,0x00,0x00,0x00, -0x78,0x00,0x00,0x00,0x77,0x00,0x00,0x00,0x41,0x00,0x08,0x00, -0x7f,0x00,0x00,0x00,0x80,0x00,0x00,0x00,0x5a,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x45,0x00,0x00,0x00,0x7d,0x00,0x00,0x00, -0x4a,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x53,0x00,0x00,0x00, -0x81,0x00,0x00,0x00,0x80,0x00,0x00,0x00,0x71,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x81,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x10,0x00,0x00,0x00,0x87,0x00,0x00,0x00, -0x86,0x00,0x00,0x00,0xc7,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x89,0x00,0x00,0x00,0x87,0x00,0x00,0x00,0x88,0x00,0x00,0x00, -0xc5,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x8d,0x00,0x00,0x00, -0x89,0x00,0x00,0x00,0x6f,0x00,0x00,0x00,0x6f,0x00,0x04,0x00, -0x50,0x00,0x00,0x00,0x8e,0x00,0x00,0x00,0x8d,0x00,0x00,0x00, -0xc2,0x00,0x05,0x00,0x53,0x00,0x00,0x00,0x90,0x00,0x00,0x00, -0x81,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x71,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x91,0x00,0x00,0x00,0x90,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x10,0x00,0x00,0x00,0x92,0x00,0x00,0x00, -0x91,0x00,0x00,0x00,0xc5,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x95,0x00,0x00,0x00,0x92,0x00,0x00,0x00,0x78,0x00,0x00,0x00, -0x6f,0x00,0x04,0x00,0x50,0x00,0x00,0x00,0x96,0x00,0x00,0x00, -0x95,0x00,0x00,0x00,0x50,0x00,0x05,0x00,0x82,0x00,0x00,0x00, -0x97,0x00,0x00,0x00,0x8e,0x00,0x00,0x00,0x96,0x00,0x00,0x00, -0x8e,0x00,0x05,0x00,0x82,0x00,0x00,0x00,0x9a,0x00,0x00,0x00, -0x97,0x00,0x00,0x00,0x5e,0x00,0x00,0x00,0x50,0x00,0x05,0x00, -0x82,0x00,0x00,0x00,0x9c,0x00,0x00,0x00,0x62,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x81,0x00,0x05,0x00,0x82,0x00,0x00,0x00, -0x9d,0x00,0x00,0x00,0x9a,0x00,0x00,0x00,0x9c,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa4,0x00,0x00,0x00, -0x4f,0x00,0x00,0x00,0x4a,0x00,0x00,0x00,0x51,0x00,0x05,0x00, -0x50,0x00,0x00,0x00,0xa7,0x00,0x00,0x00,0x9d,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x1c,0x00,0x00,0x00, -0xa8,0x00,0x00,0x00,0xa7,0x00,0x00,0x00,0x41,0x00,0x06,0x00, -0xa9,0x00,0x00,0x00,0xaa,0x00,0x00,0x00,0xa1,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0xa4,0x00,0x00,0x00,0x3e,0x00,0x03,0x00, -0xaa,0x00,0x00,0x00,0xa8,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xae,0x00,0x00,0x00,0xa4,0x00,0x00,0x00, -0x54,0x00,0x00,0x00,0x51,0x00,0x05,0x00,0x50,0x00,0x00,0x00, -0xb0,0x00,0x00,0x00,0x9d,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x73,0x00,0x04,0x00,0x1c,0x00,0x00,0x00,0xb1,0x00,0x00,0x00, -0xb0,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0xa9,0x00,0x00,0x00, -0xb2,0x00,0x00,0x00,0xa1,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0xae,0x00,0x00,0x00,0x3e,0x00,0x03,0x00,0xb2,0x00,0x00,0x00, -0xb1,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xb5,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xb5,0x00,0x00,0x00,0xfd,0x00,0x01,0x00, -0x38,0x00,0x01,0x00, -}; -const uint64_t get_rows_q5_1_f32_len = 2812; - -unsigned char get_rows_q5_1_f32_fp32_data[] = { -0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, -0xb5,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, +0xc2,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, 0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x51,0x11,0x00,0x00, 0x11,0x00,0x02,0x00,0x60,0x11,0x00,0x00,0x0b,0x00,0x06,0x00, 0x01,0x00,0x00,0x00,0x47,0x4c,0x53,0x4c,0x2e,0x73,0x74,0x64, @@ -18560,7 +9562,7 @@ unsigned char get_rows_q5_1_f32_fp32_data[] = { 0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x0f,0x00,0x0a,0x00, 0x05,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x6d,0x61,0x69,0x6e, 0x00,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x1f,0x00,0x00,0x00, -0x2d,0x00,0x00,0x00,0x5a,0x00,0x00,0x00,0xa1,0x00,0x00,0x00, +0x2d,0x00,0x00,0x00,0x5c,0x00,0x00,0x00,0xa5,0x00,0x00,0x00, 0x10,0x00,0x06,0x00,0x04,0x00,0x00,0x00,0x11,0x00,0x00,0x00, 0x00,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00, 0x47,0x00,0x04,0x00,0x0b,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, @@ -18580,101 +9582,106 @@ unsigned char get_rows_q5_1_f32_fp32_data[] = { 0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x2d,0x00,0x00,0x00, 0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, 0x2d,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x55,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x56,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x56,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x56,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x56,0x00,0x00,0x00, -0x03,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x08,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x57,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0x58,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x47,0x00,0x04,0x00,0x54,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x57,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x48,0x00,0x05,0x00, 0x58,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x58,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x5a,0x00,0x00,0x00, -0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x5a,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x9e,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0x9f,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x9f,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x9f,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xa1,0x00,0x00,0x00, -0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0xa1,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0xb2,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x19,0x00,0x00,0x00,0x13,0x00,0x02,0x00,0x02,0x00,0x00,0x00, -0x21,0x00,0x03,0x00,0x03,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x15,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x17,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x0a,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x0d,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x15,0x00,0x04,0x00,0x10,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x10,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x16,0x00,0x03,0x00,0x1c,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x1e,0x00,0x06,0x00,0x1d,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x1c,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x1e,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x1d,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x1e,0x00,0x00,0x00, -0x1f,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x58,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x58,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x59,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x16,0x00,0x00,0x00, +0x48,0x00,0x04,0x00,0x5a,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x5a,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x03,0x00,0x5a,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x5c,0x00,0x00,0x00,0x22,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x5c,0x00,0x00,0x00, +0x21,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0xa2,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x48,0x00,0x04,0x00,0xa3,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x19,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0xa3,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x03,0x00,0xa3,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0xa5,0x00,0x00,0x00,0x22,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xa5,0x00,0x00,0x00, +0x21,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0xb6,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x19,0x00,0x00,0x00, +0x13,0x00,0x02,0x00,0x02,0x00,0x00,0x00,0x21,0x00,0x03,0x00, +0x03,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x15,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x17,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x03,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x0a,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0x0a,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x0d,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x15,0x00,0x04,0x00, 0x10,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x21,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x14,0x00,0x02,0x00,0x24,0x00,0x00,0x00, -0x1d,0x00,0x03,0x00,0x2a,0x00,0x00,0x00,0x10,0x00,0x00,0x00, -0x1e,0x00,0x03,0x00,0x2b,0x00,0x00,0x00,0x2a,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x2c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x2b,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x2c,0x00,0x00,0x00, -0x2d,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x10,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x30,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x10,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x44,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x49,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x16,0x00,0x03,0x00,0x52,0x00,0x00,0x00,0x10,0x00,0x00,0x00, -0x15,0x00,0x04,0x00,0x53,0x00,0x00,0x00,0x08,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x54,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x1c,0x00,0x04,0x00, -0x55,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x54,0x00,0x00,0x00, -0x1e,0x00,0x06,0x00,0x56,0x00,0x00,0x00,0x52,0x00,0x00,0x00, -0x52,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x55,0x00,0x00,0x00, -0x1d,0x00,0x03,0x00,0x57,0x00,0x00,0x00,0x56,0x00,0x00,0x00, -0x1e,0x00,0x03,0x00,0x58,0x00,0x00,0x00,0x57,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x59,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x58,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x59,0x00,0x00,0x00, -0x5a,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x5c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x52,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x69,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x10,0x00,0x00,0x00, -0x6e,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x76,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x10,0x00,0x00,0x00,0x7e,0x00,0x00,0x00, -0x03,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x80,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x17,0x00,0x04,0x00, -0x84,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x88,0x00,0x00,0x00, -0x0f,0x00,0x00,0x00,0x1d,0x00,0x03,0x00,0x9e,0x00,0x00,0x00, -0x1c,0x00,0x00,0x00,0x1e,0x00,0x03,0x00,0x9f,0x00,0x00,0x00, -0x9e,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0xa0,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x9f,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0xa0,0x00,0x00,0x00,0xa1,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0xa8,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x1c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xb1,0x00,0x00,0x00,0x00,0x02,0x00,0x00,0x2c,0x00,0x06,0x00, -0x09,0x00,0x00,0x00,0xb2,0x00,0x00,0x00,0xb1,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x36,0x00,0x05,0x00, +0x2b,0x00,0x04,0x00,0x10,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x16,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x16,0x00,0x03,0x00, +0x1c,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x1e,0x00,0x06,0x00, +0x1d,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x1c,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x1e,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x1d,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0x1e,0x00,0x00,0x00,0x1f,0x00,0x00,0x00, +0x09,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x10,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x21,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x14,0x00,0x02,0x00,0x24,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, +0x2a,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, +0x2b,0x00,0x00,0x00,0x2a,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x2c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x2b,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0x2c,0x00,0x00,0x00,0x2d,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x10,0x00,0x00,0x00, +0x2e,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x30,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x10,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x44,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x49,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x16,0x00,0x03,0x00, +0x52,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x15,0x00,0x04,0x00, +0x53,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x1c,0x00,0x04,0x00,0x54,0x00,0x00,0x00,0x53,0x00,0x00,0x00, +0x49,0x00,0x00,0x00,0x15,0x00,0x04,0x00,0x55,0x00,0x00,0x00, +0x08,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x56,0x00,0x00,0x00,0x10,0x00,0x00,0x00, +0x1c,0x00,0x04,0x00,0x57,0x00,0x00,0x00,0x55,0x00,0x00,0x00, +0x56,0x00,0x00,0x00,0x1e,0x00,0x05,0x00,0x58,0x00,0x00,0x00, +0x52,0x00,0x00,0x00,0x54,0x00,0x00,0x00,0x57,0x00,0x00,0x00, +0x1d,0x00,0x03,0x00,0x59,0x00,0x00,0x00,0x58,0x00,0x00,0x00, +0x1e,0x00,0x03,0x00,0x5a,0x00,0x00,0x00,0x59,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x5b,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x5a,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x5b,0x00,0x00,0x00, +0x5c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x5e,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x52,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x64,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x53,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x10,0x00,0x00,0x00, +0x68,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x10,0x00,0x00,0x00,0x75,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x7b,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x84,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x55,0x00,0x00,0x00,0x17,0x00,0x04,0x00, +0x88,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x8c,0x00,0x00,0x00, +0x0f,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x1c,0x00,0x00,0x00, +0x9d,0x00,0x00,0x00,0x00,0x00,0x80,0x41,0x1d,0x00,0x03,0x00, +0xa2,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, +0xa3,0x00,0x00,0x00,0xa2,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0xa4,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0xa3,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0xa4,0x00,0x00,0x00,0xa5,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0xac,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xb5,0x00,0x00,0x00,0x00,0x02,0x00,0x00, +0x2c,0x00,0x06,0x00,0x09,0x00,0x00,0x00,0xb6,0x00,0x00,0x00, +0xb5,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x16,0x00,0x00,0x00, +0x2c,0x00,0x05,0x00,0x88,0x00,0x00,0x00,0xc1,0x00,0x00,0x00, +0x9d,0x00,0x00,0x00,0x9d,0x00,0x00,0x00,0x36,0x00,0x05,0x00, 0x02,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x00, 0x03,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x05,0x00,0x00,0x00, -0xf7,0x00,0x03,0x00,0xb3,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0xfb,0x00,0x03,0x00,0x0c,0x00,0x00,0x00,0xb4,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xb4,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0xf7,0x00,0x03,0x00,0xb7,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0xfb,0x00,0x03,0x00,0x0c,0x00,0x00,0x00,0xb8,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xb8,0x00,0x00,0x00,0x41,0x00,0x05,0x00, 0x0d,0x00,0x00,0x00,0x0e,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, 0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, 0x0f,0x00,0x00,0x00,0x0e,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, @@ -18695,7 +9702,7 @@ unsigned char get_rows_q5_1_f32_fp32_data[] = { 0x23,0x00,0x00,0x00,0xf7,0x00,0x03,0x00,0x27,0x00,0x00,0x00, 0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x25,0x00,0x00,0x00, 0x26,0x00,0x00,0x00,0x27,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x26,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xb3,0x00,0x00,0x00, +0x26,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xb7,0x00,0x00,0x00, 0xf8,0x00,0x02,0x00,0x27,0x00,0x00,0x00,0x41,0x00,0x06,0x00, 0x30,0x00,0x00,0x00,0x31,0x00,0x00,0x00,0x2d,0x00,0x00,0x00, 0x2e,0x00,0x00,0x00,0x1a,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, @@ -18716,77 +9723,80 @@ unsigned char get_rows_q5_1_f32_fp32_data[] = { 0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, 0x41,0x00,0x00,0x00,0x44,0x00,0x00,0x00,0x82,0x00,0x05,0x00, 0x06,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x41,0x00,0x00,0x00, -0x4e,0x00,0x00,0x00,0x41,0x00,0x07,0x00,0x5c,0x00,0x00,0x00, -0x5d,0x00,0x00,0x00,0x5a,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, +0x4e,0x00,0x00,0x00,0x41,0x00,0x07,0x00,0x5e,0x00,0x00,0x00, +0x5f,0x00,0x00,0x00,0x5c,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, 0x45,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x52,0x00,0x00,0x00,0x5e,0x00,0x00,0x00,0x5d,0x00,0x00,0x00, -0x73,0x00,0x04,0x00,0x1c,0x00,0x00,0x00,0x5f,0x00,0x00,0x00, -0x5e,0x00,0x00,0x00,0x41,0x00,0x07,0x00,0x5c,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x5a,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x45,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x52,0x00,0x00,0x00,0x63,0x00,0x00,0x00,0x62,0x00,0x00,0x00, -0x73,0x00,0x04,0x00,0x1c,0x00,0x00,0x00,0x64,0x00,0x00,0x00, -0x63,0x00,0x00,0x00,0x41,0x00,0x07,0x00,0x69,0x00,0x00,0x00, -0x6a,0x00,0x00,0x00,0x5a,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x45,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x6b,0x00,0x00,0x00,0x6a,0x00,0x00,0x00, -0xc2,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, -0x6b,0x00,0x00,0x00,0x4a,0x00,0x00,0x00,0xc4,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x6f,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, -0x6e,0x00,0x00,0x00,0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x70,0x00,0x00,0x00,0x6f,0x00,0x00,0x00,0x54,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x10,0x00,0x00,0x00,0x71,0x00,0x00,0x00, -0x70,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x74,0x00,0x00,0x00,0x6a,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x77,0x00,0x00,0x00,0x4a,0x00,0x00,0x00, -0x76,0x00,0x00,0x00,0xc2,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x78,0x00,0x00,0x00,0x74,0x00,0x00,0x00,0x77,0x00,0x00,0x00, -0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x79,0x00,0x00,0x00, -0x78,0x00,0x00,0x00,0x54,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x10,0x00,0x00,0x00,0x7a,0x00,0x00,0x00,0x79,0x00,0x00,0x00, -0x41,0x00,0x08,0x00,0x80,0x00,0x00,0x00,0x81,0x00,0x00,0x00, -0x5a,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x45,0x00,0x00,0x00, -0x7e,0x00,0x00,0x00,0x4a,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x53,0x00,0x00,0x00,0x82,0x00,0x00,0x00,0x81,0x00,0x00,0x00, -0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x83,0x00,0x00,0x00, -0x82,0x00,0x00,0x00,0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x89,0x00,0x00,0x00,0x83,0x00,0x00,0x00,0x88,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x8d,0x00,0x00,0x00, -0x71,0x00,0x00,0x00,0xc5,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x8e,0x00,0x00,0x00,0x89,0x00,0x00,0x00,0x8d,0x00,0x00,0x00, -0x70,0x00,0x04,0x00,0x1c,0x00,0x00,0x00,0x8f,0x00,0x00,0x00, -0x8e,0x00,0x00,0x00,0xc2,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x91,0x00,0x00,0x00,0x83,0x00,0x00,0x00,0x6e,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x94,0x00,0x00,0x00, -0x7a,0x00,0x00,0x00,0xc5,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x95,0x00,0x00,0x00,0x91,0x00,0x00,0x00,0x94,0x00,0x00,0x00, -0x70,0x00,0x04,0x00,0x1c,0x00,0x00,0x00,0x96,0x00,0x00,0x00, -0x95,0x00,0x00,0x00,0x50,0x00,0x05,0x00,0x84,0x00,0x00,0x00, -0x97,0x00,0x00,0x00,0x8f,0x00,0x00,0x00,0x96,0x00,0x00,0x00, -0x8e,0x00,0x05,0x00,0x84,0x00,0x00,0x00,0x9a,0x00,0x00,0x00, -0x97,0x00,0x00,0x00,0x5f,0x00,0x00,0x00,0x50,0x00,0x05,0x00, -0x84,0x00,0x00,0x00,0x9c,0x00,0x00,0x00,0x64,0x00,0x00,0x00, -0x64,0x00,0x00,0x00,0x81,0x00,0x05,0x00,0x84,0x00,0x00,0x00, -0x9d,0x00,0x00,0x00,0x9a,0x00,0x00,0x00,0x9c,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa4,0x00,0x00,0x00, -0x4f,0x00,0x00,0x00,0x4a,0x00,0x00,0x00,0x51,0x00,0x05,0x00, -0x1c,0x00,0x00,0x00,0xa7,0x00,0x00,0x00,0x9d,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0xa8,0x00,0x00,0x00, -0xa9,0x00,0x00,0x00,0xa1,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0xa4,0x00,0x00,0x00,0x3e,0x00,0x03,0x00,0xa9,0x00,0x00,0x00, -0xa7,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xad,0x00,0x00,0x00,0xa4,0x00,0x00,0x00,0x54,0x00,0x00,0x00, -0x51,0x00,0x05,0x00,0x1c,0x00,0x00,0x00,0xaf,0x00,0x00,0x00, -0x9d,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x41,0x00,0x06,0x00, -0xa8,0x00,0x00,0x00,0xb0,0x00,0x00,0x00,0xa1,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0xad,0x00,0x00,0x00,0x3e,0x00,0x03,0x00, -0xb0,0x00,0x00,0x00,0xaf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xb3,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xb3,0x00,0x00,0x00, -0xfd,0x00,0x01,0x00,0x38,0x00,0x01,0x00, -}; -const uint64_t get_rows_q5_1_f32_fp32_len = 2780; +0x52,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x5f,0x00,0x00,0x00, +0x73,0x00,0x04,0x00,0x1c,0x00,0x00,0x00,0x61,0x00,0x00,0x00, +0x60,0x00,0x00,0x00,0x41,0x00,0x08,0x00,0x64,0x00,0x00,0x00, +0x65,0x00,0x00,0x00,0x5c,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, +0x45,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x53,0x00,0x00,0x00,0x66,0x00,0x00,0x00, +0x65,0x00,0x00,0x00,0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x67,0x00,0x00,0x00,0x66,0x00,0x00,0x00,0xc4,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x69,0x00,0x00,0x00,0x67,0x00,0x00,0x00, +0x68,0x00,0x00,0x00,0x41,0x00,0x08,0x00,0x64,0x00,0x00,0x00, +0x6b,0x00,0x00,0x00,0x5c,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, +0x45,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x53,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, +0x6b,0x00,0x00,0x00,0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x6d,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0xc5,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x6e,0x00,0x00,0x00,0x69,0x00,0x00,0x00, +0x6d,0x00,0x00,0x00,0xc2,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x74,0x00,0x00,0x00,0x6e,0x00,0x00,0x00,0x4a,0x00,0x00,0x00, +0xc4,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x76,0x00,0x00,0x00, +0x74,0x00,0x00,0x00,0x75,0x00,0x00,0x00,0xc7,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x77,0x00,0x00,0x00,0x76,0x00,0x00,0x00, +0x56,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x10,0x00,0x00,0x00, +0x78,0x00,0x00,0x00,0x77,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x7c,0x00,0x00,0x00,0x4a,0x00,0x00,0x00, +0x7b,0x00,0x00,0x00,0xc2,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x7d,0x00,0x00,0x00,0x6e,0x00,0x00,0x00,0x7c,0x00,0x00,0x00, +0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x7e,0x00,0x00,0x00, +0x7d,0x00,0x00,0x00,0x56,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, +0x10,0x00,0x00,0x00,0x7f,0x00,0x00,0x00,0x7e,0x00,0x00,0x00, +0x41,0x00,0x08,0x00,0x84,0x00,0x00,0x00,0x85,0x00,0x00,0x00, +0x5c,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x45,0x00,0x00,0x00, +0x12,0x00,0x00,0x00,0x4a,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x55,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x85,0x00,0x00,0x00, +0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x87,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x8d,0x00,0x00,0x00,0x87,0x00,0x00,0x00,0x8c,0x00,0x00,0x00, +0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x91,0x00,0x00,0x00, +0x78,0x00,0x00,0x00,0xc5,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x92,0x00,0x00,0x00,0x8d,0x00,0x00,0x00,0x91,0x00,0x00,0x00, +0x70,0x00,0x04,0x00,0x1c,0x00,0x00,0x00,0x93,0x00,0x00,0x00, +0x92,0x00,0x00,0x00,0xc2,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x95,0x00,0x00,0x00,0x87,0x00,0x00,0x00,0x75,0x00,0x00,0x00, +0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x98,0x00,0x00,0x00, +0x7f,0x00,0x00,0x00,0xc5,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x99,0x00,0x00,0x00,0x95,0x00,0x00,0x00,0x98,0x00,0x00,0x00, +0x70,0x00,0x04,0x00,0x1c,0x00,0x00,0x00,0x9a,0x00,0x00,0x00, +0x99,0x00,0x00,0x00,0x50,0x00,0x05,0x00,0x88,0x00,0x00,0x00, +0x9b,0x00,0x00,0x00,0x93,0x00,0x00,0x00,0x9a,0x00,0x00,0x00, +0x83,0x00,0x05,0x00,0x88,0x00,0x00,0x00,0x9f,0x00,0x00,0x00, +0x9b,0x00,0x00,0x00,0xc1,0x00,0x00,0x00,0x8e,0x00,0x05,0x00, +0x88,0x00,0x00,0x00,0xa1,0x00,0x00,0x00,0x9f,0x00,0x00,0x00, +0x61,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xa8,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x4a,0x00,0x00,0x00, +0x51,0x00,0x05,0x00,0x1c,0x00,0x00,0x00,0xab,0x00,0x00,0x00, +0xa1,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x41,0x00,0x06,0x00, +0xac,0x00,0x00,0x00,0xad,0x00,0x00,0x00,0xa5,0x00,0x00,0x00, +0x2e,0x00,0x00,0x00,0xa8,0x00,0x00,0x00,0x3e,0x00,0x03,0x00, +0xad,0x00,0x00,0x00,0xab,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xb1,0x00,0x00,0x00,0xa8,0x00,0x00,0x00, +0x56,0x00,0x00,0x00,0x51,0x00,0x05,0x00,0x1c,0x00,0x00,0x00, +0xb3,0x00,0x00,0x00,0xa1,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x41,0x00,0x06,0x00,0xac,0x00,0x00,0x00,0xb4,0x00,0x00,0x00, +0xa5,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0xb1,0x00,0x00,0x00, +0x3e,0x00,0x03,0x00,0xb4,0x00,0x00,0x00,0xb3,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xb7,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xb7,0x00,0x00,0x00,0xfd,0x00,0x01,0x00,0x38,0x00,0x01,0x00, -unsigned char get_rows_q5_1_fp32_data[] = { +}; +const uint64_t get_rows_q5_0_f32_len = 2868; + +unsigned char get_rows_q5_1_data[] = { 0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, 0xb6,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, 0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x51,0x11,0x00,0x00, @@ -19022,61 +10032,63 @@ unsigned char get_rows_q5_1_fp32_data[] = { 0xb4,0x00,0x00,0x00,0xfd,0x00,0x01,0x00,0x38,0x00,0x01,0x00, }; -const uint64_t get_rows_q5_1_fp32_len = 2796; +const uint64_t get_rows_q5_1_len = 2796; -unsigned char get_rows_q8_0_data[] = { +unsigned char get_rows_q5_1_f32_data[] = { 0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, -0x86,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, -0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x09,0x00,0x00,0x00, -0x11,0x00,0x02,0x00,0x27,0x00,0x00,0x00,0x11,0x00,0x02,0x00, -0x51,0x11,0x00,0x00,0x11,0x00,0x02,0x00,0x60,0x11,0x00,0x00, -0x0b,0x00,0x06,0x00,0x01,0x00,0x00,0x00,0x47,0x4c,0x53,0x4c, -0x2e,0x73,0x74,0x64,0x2e,0x34,0x35,0x30,0x00,0x00,0x00,0x00, -0x0e,0x00,0x03,0x00,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x0f,0x00,0x0a,0x00,0x05,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x6d,0x61,0x69,0x6e,0x00,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x1f,0x00,0x00,0x00,0x2d,0x00,0x00,0x00,0x58,0x00,0x00,0x00, -0x73,0x00,0x00,0x00,0x10,0x00,0x06,0x00,0x04,0x00,0x00,0x00, -0x11,0x00,0x00,0x00,0x00,0x02,0x00,0x00,0x01,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x0b,0x00,0x00,0x00, -0x0b,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x1d,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x1d,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x1d,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x1d,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x1d,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x2a,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00, -0x2b,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x2b,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, -0x2b,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x2d,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x2d,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x53,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x54,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x54,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x55,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x22,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0x56,0x00,0x00,0x00, +0xb5,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, +0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x51,0x11,0x00,0x00, +0x11,0x00,0x02,0x00,0x60,0x11,0x00,0x00,0x0b,0x00,0x06,0x00, +0x01,0x00,0x00,0x00,0x47,0x4c,0x53,0x4c,0x2e,0x73,0x74,0x64, +0x2e,0x34,0x35,0x30,0x00,0x00,0x00,0x00,0x0e,0x00,0x03,0x00, +0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x0f,0x00,0x0a,0x00, +0x05,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x6d,0x61,0x69,0x6e, +0x00,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x1f,0x00,0x00,0x00, +0x2d,0x00,0x00,0x00,0x5a,0x00,0x00,0x00,0xa1,0x00,0x00,0x00, +0x10,0x00,0x06,0x00,0x04,0x00,0x00,0x00,0x11,0x00,0x00,0x00, +0x00,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x0b,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, +0x1c,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x1d,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x1d,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x1d,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x08,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x1d,0x00,0x00,0x00, +0x03,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x47,0x00,0x03,0x00,0x1d,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x2a,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0x2b,0x00,0x00,0x00, 0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x56,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x56,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x58,0x00,0x00,0x00, +0x2b,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x2b,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x2d,0x00,0x00,0x00, 0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x58,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x70,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0x71,0x00,0x00,0x00, +0x2d,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x55,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x56,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x56,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x56,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x56,0x00,0x00,0x00, +0x03,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x08,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x57,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x18,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0x58,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x58,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x58,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x5a,0x00,0x00,0x00, +0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x5a,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x9e,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0x9f,0x00,0x00,0x00, 0x00,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x71,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x71,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x73,0x00,0x00,0x00, +0x9f,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x9f,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xa1,0x00,0x00,0x00, 0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x73,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x83,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, +0xa1,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0xb2,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, 0x19,0x00,0x00,0x00,0x13,0x00,0x02,0x00,0x02,0x00,0x00,0x00, 0x21,0x00,0x03,0x00,0x03,0x00,0x00,0x00,0x02,0x00,0x00,0x00, 0x15,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x20,0x00,0x00,0x00, @@ -19108,226 +10120,45 @@ unsigned char get_rows_q8_0_data[] = { 0x10,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x00,0x00,0x00,0x00, 0x20,0x00,0x04,0x00,0x30,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, 0x10,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x44,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x16,0x00,0x03,0x00, -0x4f,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x15,0x00,0x04,0x00, -0x52,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x1c,0x00,0x04,0x00,0x53,0x00,0x00,0x00,0x52,0x00,0x00,0x00, -0x44,0x00,0x00,0x00,0x1e,0x00,0x04,0x00,0x54,0x00,0x00,0x00, -0x4f,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, -0x55,0x00,0x00,0x00,0x54,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, -0x56,0x00,0x00,0x00,0x55,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x57,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x56,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x57,0x00,0x00,0x00,0x58,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x5a,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x17,0x00,0x04,0x00, -0x5d,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x62,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x52,0x00,0x00,0x00,0x1d,0x00,0x03,0x00,0x70,0x00,0x00,0x00, -0x4f,0x00,0x00,0x00,0x1e,0x00,0x03,0x00,0x71,0x00,0x00,0x00, -0x70,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x72,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x71,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x72,0x00,0x00,0x00,0x73,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x82,0x00,0x00,0x00, -0x00,0x02,0x00,0x00,0x2c,0x00,0x06,0x00,0x09,0x00,0x00,0x00, -0x83,0x00,0x00,0x00,0x82,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x36,0x00,0x05,0x00,0x02,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x03,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0x05,0x00,0x00,0x00,0xf7,0x00,0x03,0x00, -0x84,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xfb,0x00,0x03,0x00, -0x0c,0x00,0x00,0x00,0x85,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x85,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00, -0x0e,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, -0x0e,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x10,0x00,0x00,0x00, -0x11,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x10,0x00,0x00,0x00,0x13,0x00,0x00,0x00,0x11,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x14,0x00,0x00,0x00,0x13,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x0d,0x00,0x00,0x00,0x17,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x17,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x10,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x18,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x1a,0x00,0x00,0x00, -0x19,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x21,0x00,0x00,0x00, -0x22,0x00,0x00,0x00,0x1f,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x22,0x00,0x00,0x00,0xae,0x00,0x05,0x00,0x24,0x00,0x00,0x00, -0x25,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0xf7,0x00,0x03,0x00,0x27,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x25,0x00,0x00,0x00,0x26,0x00,0x00,0x00, -0x27,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x26,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x84,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x27,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x30,0x00,0x00,0x00, -0x31,0x00,0x00,0x00,0x2d,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x1a,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x10,0x00,0x00,0x00, -0x32,0x00,0x00,0x00,0x31,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x33,0x00,0x00,0x00,0x32,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x38,0x00,0x00,0x00, -0x33,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x38,0x00,0x00,0x00, -0x14,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x3f,0x00,0x00,0x00,0x1a,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x41,0x00,0x00,0x00, -0x3f,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x45,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, -0x44,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x48,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x44,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x49,0x00,0x00,0x00, -0x48,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x89,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x4d,0x00,0x00,0x00,0x41,0x00,0x00,0x00, -0x44,0x00,0x00,0x00,0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x4e,0x00,0x00,0x00,0x41,0x00,0x00,0x00,0x4d,0x00,0x00,0x00, -0x41,0x00,0x07,0x00,0x5a,0x00,0x00,0x00,0x5b,0x00,0x00,0x00, -0x58,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x45,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x4f,0x00,0x00,0x00, -0x5c,0x00,0x00,0x00,0x5b,0x00,0x00,0x00,0x41,0x00,0x08,0x00, -0x62,0x00,0x00,0x00,0x63,0x00,0x00,0x00,0x58,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x45,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x49,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x52,0x00,0x00,0x00, -0x64,0x00,0x00,0x00,0x63,0x00,0x00,0x00,0x6f,0x00,0x04,0x00, -0x4f,0x00,0x00,0x00,0x65,0x00,0x00,0x00,0x64,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x68,0x00,0x00,0x00, -0x49,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x41,0x00,0x08,0x00, -0x62,0x00,0x00,0x00,0x69,0x00,0x00,0x00,0x58,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x45,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x68,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x52,0x00,0x00,0x00, -0x6a,0x00,0x00,0x00,0x69,0x00,0x00,0x00,0x6f,0x00,0x04,0x00, -0x4f,0x00,0x00,0x00,0x6b,0x00,0x00,0x00,0x6a,0x00,0x00,0x00, -0x50,0x00,0x05,0x00,0x5d,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x65,0x00,0x00,0x00,0x6b,0x00,0x00,0x00,0x8e,0x00,0x05,0x00, -0x5d,0x00,0x00,0x00,0x6f,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x5c,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x76,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x49,0x00,0x00,0x00, -0x51,0x00,0x05,0x00,0x4f,0x00,0x00,0x00,0x79,0x00,0x00,0x00, -0x6f,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x41,0x00,0x06,0x00, -0x5a,0x00,0x00,0x00,0x7a,0x00,0x00,0x00,0x73,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x76,0x00,0x00,0x00,0x3e,0x00,0x03,0x00, -0x7a,0x00,0x00,0x00,0x79,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x7e,0x00,0x00,0x00,0x76,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x51,0x00,0x05,0x00,0x4f,0x00,0x00,0x00, -0x80,0x00,0x00,0x00,0x6f,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x41,0x00,0x06,0x00,0x5a,0x00,0x00,0x00,0x81,0x00,0x00,0x00, -0x73,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x7e,0x00,0x00,0x00, -0x3e,0x00,0x03,0x00,0x81,0x00,0x00,0x00,0x80,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x84,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x84,0x00,0x00,0x00,0xfd,0x00,0x01,0x00,0x38,0x00,0x01,0x00, - -}; -const uint64_t get_rows_q8_0_len = 2232; - -unsigned char get_rows_q8_0_f32_data[] = { -0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, -0x89,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, -0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x09,0x00,0x00,0x00, -0x11,0x00,0x02,0x00,0x27,0x00,0x00,0x00,0x11,0x00,0x02,0x00, -0x51,0x11,0x00,0x00,0x11,0x00,0x02,0x00,0x60,0x11,0x00,0x00, -0x0b,0x00,0x06,0x00,0x01,0x00,0x00,0x00,0x47,0x4c,0x53,0x4c, -0x2e,0x73,0x74,0x64,0x2e,0x34,0x35,0x30,0x00,0x00,0x00,0x00, -0x0e,0x00,0x03,0x00,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x0f,0x00,0x0a,0x00,0x05,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x6d,0x61,0x69,0x6e,0x00,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x1f,0x00,0x00,0x00,0x2d,0x00,0x00,0x00,0x58,0x00,0x00,0x00, -0x73,0x00,0x00,0x00,0x10,0x00,0x06,0x00,0x04,0x00,0x00,0x00, -0x11,0x00,0x00,0x00,0x00,0x02,0x00,0x00,0x01,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x0b,0x00,0x00,0x00, -0x0b,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x1d,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x1d,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x1d,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x1d,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x1d,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x2a,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00, -0x2b,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x2b,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, -0x2b,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x2d,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x2d,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x53,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x54,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x54,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x55,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x22,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0x56,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x56,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x56,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x58,0x00,0x00,0x00, -0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x58,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x70,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0x71,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x71,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x71,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x73,0x00,0x00,0x00, -0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x73,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x86,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x19,0x00,0x00,0x00,0x13,0x00,0x02,0x00,0x02,0x00,0x00,0x00, -0x21,0x00,0x03,0x00,0x03,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x15,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x17,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x0a,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x0d,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x15,0x00,0x04,0x00,0x10,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x10,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x16,0x00,0x03,0x00,0x1c,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x1e,0x00,0x06,0x00,0x1d,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x1c,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x1e,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x1d,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x1e,0x00,0x00,0x00, -0x1f,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x10,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x21,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x14,0x00,0x02,0x00,0x24,0x00,0x00,0x00, -0x1d,0x00,0x03,0x00,0x2a,0x00,0x00,0x00,0x10,0x00,0x00,0x00, -0x1e,0x00,0x03,0x00,0x2b,0x00,0x00,0x00,0x2a,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x2c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x2b,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x2c,0x00,0x00,0x00, -0x2d,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x10,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x30,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x10,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x44,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x16,0x00,0x03,0x00, -0x4f,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x15,0x00,0x04,0x00, -0x52,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x1c,0x00,0x04,0x00,0x53,0x00,0x00,0x00,0x52,0x00,0x00,0x00, -0x44,0x00,0x00,0x00,0x1e,0x00,0x04,0x00,0x54,0x00,0x00,0x00, -0x4f,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, -0x55,0x00,0x00,0x00,0x54,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, -0x56,0x00,0x00,0x00,0x55,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x57,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x56,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x57,0x00,0x00,0x00,0x58,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x5a,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x17,0x00,0x04,0x00, -0x5d,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x62,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x52,0x00,0x00,0x00,0x1d,0x00,0x03,0x00,0x70,0x00,0x00,0x00, -0x1c,0x00,0x00,0x00,0x1e,0x00,0x03,0x00,0x71,0x00,0x00,0x00, -0x70,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x72,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x71,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x72,0x00,0x00,0x00,0x73,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x7b,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x44,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x49,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x16,0x00,0x03,0x00,0x52,0x00,0x00,0x00,0x10,0x00,0x00,0x00, +0x15,0x00,0x04,0x00,0x53,0x00,0x00,0x00,0x08,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x54,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x1c,0x00,0x04,0x00, +0x55,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x54,0x00,0x00,0x00, +0x1e,0x00,0x06,0x00,0x56,0x00,0x00,0x00,0x52,0x00,0x00,0x00, +0x52,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x55,0x00,0x00,0x00, +0x1d,0x00,0x03,0x00,0x57,0x00,0x00,0x00,0x56,0x00,0x00,0x00, +0x1e,0x00,0x03,0x00,0x58,0x00,0x00,0x00,0x57,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x59,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x58,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x59,0x00,0x00,0x00, +0x5a,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x5c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x52,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x69,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x10,0x00,0x00,0x00, +0x6e,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x76,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x10,0x00,0x00,0x00,0x7e,0x00,0x00,0x00, +0x03,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x80,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x17,0x00,0x04,0x00, +0x84,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x88,0x00,0x00,0x00, +0x0f,0x00,0x00,0x00,0x1d,0x00,0x03,0x00,0x9e,0x00,0x00,0x00, +0x1c,0x00,0x00,0x00,0x1e,0x00,0x03,0x00,0x9f,0x00,0x00,0x00, +0x9e,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0xa0,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x9f,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0xa0,0x00,0x00,0x00,0xa1,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0xa8,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, 0x1c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x85,0x00,0x00,0x00,0x00,0x02,0x00,0x00,0x2c,0x00,0x06,0x00, -0x09,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x85,0x00,0x00,0x00, +0xb1,0x00,0x00,0x00,0x00,0x02,0x00,0x00,0x2c,0x00,0x06,0x00, +0x09,0x00,0x00,0x00,0xb2,0x00,0x00,0x00,0xb1,0x00,0x00,0x00, 0x16,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x36,0x00,0x05,0x00, 0x02,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x00, 0x03,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x05,0x00,0x00,0x00, -0xf7,0x00,0x03,0x00,0x87,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0xfb,0x00,0x03,0x00,0x0c,0x00,0x00,0x00,0x88,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0x88,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0xf7,0x00,0x03,0x00,0xb3,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0xfb,0x00,0x03,0x00,0x0c,0x00,0x00,0x00,0xb4,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xb4,0x00,0x00,0x00,0x41,0x00,0x05,0x00, 0x0d,0x00,0x00,0x00,0x0e,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, 0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, 0x0f,0x00,0x00,0x00,0x0e,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, @@ -19348,7 +10179,7 @@ unsigned char get_rows_q8_0_f32_data[] = { 0x23,0x00,0x00,0x00,0xf7,0x00,0x03,0x00,0x27,0x00,0x00,0x00, 0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x25,0x00,0x00,0x00, 0x26,0x00,0x00,0x00,0x27,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x26,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x87,0x00,0x00,0x00, +0x26,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xb3,0x00,0x00,0x00, 0xf8,0x00,0x02,0x00,0x27,0x00,0x00,0x00,0x41,0x00,0x06,0x00, 0x30,0x00,0x00,0x00,0x31,0x00,0x00,0x00,0x2d,0x00,0x00,0x00, 0x2e,0x00,0x00,0x00,0x1a,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, @@ -19365,247 +10196,81 @@ unsigned char get_rows_q8_0_f32_data[] = { 0x3a,0x00,0x00,0x00,0x44,0x00,0x00,0x00,0x89,0x00,0x05,0x00, 0x06,0x00,0x00,0x00,0x48,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, 0x44,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x49,0x00,0x00,0x00,0x48,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x4d,0x00,0x00,0x00, +0x4a,0x00,0x00,0x00,0x48,0x00,0x00,0x00,0x49,0x00,0x00,0x00, +0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, 0x41,0x00,0x00,0x00,0x44,0x00,0x00,0x00,0x82,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x41,0x00,0x00,0x00, -0x4d,0x00,0x00,0x00,0x41,0x00,0x07,0x00,0x5a,0x00,0x00,0x00, -0x5b,0x00,0x00,0x00,0x58,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x41,0x00,0x00,0x00, +0x4e,0x00,0x00,0x00,0x41,0x00,0x07,0x00,0x5c,0x00,0x00,0x00, +0x5d,0x00,0x00,0x00,0x5a,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, 0x45,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4f,0x00,0x00,0x00,0x5c,0x00,0x00,0x00,0x5b,0x00,0x00,0x00, -0x41,0x00,0x08,0x00,0x62,0x00,0x00,0x00,0x63,0x00,0x00,0x00, -0x58,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x45,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x49,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x52,0x00,0x00,0x00,0x64,0x00,0x00,0x00,0x63,0x00,0x00,0x00, -0x6f,0x00,0x04,0x00,0x4f,0x00,0x00,0x00,0x65,0x00,0x00,0x00, -0x64,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x68,0x00,0x00,0x00,0x49,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0x41,0x00,0x08,0x00,0x62,0x00,0x00,0x00,0x69,0x00,0x00,0x00, -0x58,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x45,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x68,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x52,0x00,0x00,0x00,0x6a,0x00,0x00,0x00,0x69,0x00,0x00,0x00, -0x6f,0x00,0x04,0x00,0x4f,0x00,0x00,0x00,0x6b,0x00,0x00,0x00, -0x6a,0x00,0x00,0x00,0x50,0x00,0x05,0x00,0x5d,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x65,0x00,0x00,0x00,0x6b,0x00,0x00,0x00, -0x8e,0x00,0x05,0x00,0x5d,0x00,0x00,0x00,0x6f,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x5c,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x76,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, -0x49,0x00,0x00,0x00,0x51,0x00,0x05,0x00,0x4f,0x00,0x00,0x00, -0x79,0x00,0x00,0x00,0x6f,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x73,0x00,0x04,0x00,0x1c,0x00,0x00,0x00,0x7a,0x00,0x00,0x00, -0x79,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x7b,0x00,0x00,0x00, -0x7c,0x00,0x00,0x00,0x73,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x76,0x00,0x00,0x00,0x3e,0x00,0x03,0x00,0x7c,0x00,0x00,0x00, -0x7a,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x80,0x00,0x00,0x00,0x76,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0x51,0x00,0x05,0x00,0x4f,0x00,0x00,0x00,0x82,0x00,0x00,0x00, -0x6f,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x73,0x00,0x04,0x00, -0x1c,0x00,0x00,0x00,0x83,0x00,0x00,0x00,0x82,0x00,0x00,0x00, -0x41,0x00,0x06,0x00,0x7b,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0x73,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x80,0x00,0x00,0x00, -0x3e,0x00,0x03,0x00,0x84,0x00,0x00,0x00,0x83,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x87,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x87,0x00,0x00,0x00,0xfd,0x00,0x01,0x00,0x38,0x00,0x01,0x00, - +0x52,0x00,0x00,0x00,0x5e,0x00,0x00,0x00,0x5d,0x00,0x00,0x00, +0x73,0x00,0x04,0x00,0x1c,0x00,0x00,0x00,0x5f,0x00,0x00,0x00, +0x5e,0x00,0x00,0x00,0x41,0x00,0x07,0x00,0x5c,0x00,0x00,0x00, +0x62,0x00,0x00,0x00,0x5a,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, +0x45,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x52,0x00,0x00,0x00,0x63,0x00,0x00,0x00,0x62,0x00,0x00,0x00, +0x73,0x00,0x04,0x00,0x1c,0x00,0x00,0x00,0x64,0x00,0x00,0x00, +0x63,0x00,0x00,0x00,0x41,0x00,0x07,0x00,0x69,0x00,0x00,0x00, +0x6a,0x00,0x00,0x00,0x5a,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, +0x45,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x6b,0x00,0x00,0x00,0x6a,0x00,0x00,0x00, +0xc2,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, +0x6b,0x00,0x00,0x00,0x4a,0x00,0x00,0x00,0xc4,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x6f,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, +0x6e,0x00,0x00,0x00,0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x70,0x00,0x00,0x00,0x6f,0x00,0x00,0x00,0x54,0x00,0x00,0x00, +0x7c,0x00,0x04,0x00,0x10,0x00,0x00,0x00,0x71,0x00,0x00,0x00, +0x70,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x74,0x00,0x00,0x00,0x6a,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x77,0x00,0x00,0x00,0x4a,0x00,0x00,0x00, +0x76,0x00,0x00,0x00,0xc2,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x78,0x00,0x00,0x00,0x74,0x00,0x00,0x00,0x77,0x00,0x00,0x00, +0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x79,0x00,0x00,0x00, +0x78,0x00,0x00,0x00,0x54,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, +0x10,0x00,0x00,0x00,0x7a,0x00,0x00,0x00,0x79,0x00,0x00,0x00, +0x41,0x00,0x08,0x00,0x80,0x00,0x00,0x00,0x81,0x00,0x00,0x00, +0x5a,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x45,0x00,0x00,0x00, +0x7e,0x00,0x00,0x00,0x4a,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x53,0x00,0x00,0x00,0x82,0x00,0x00,0x00,0x81,0x00,0x00,0x00, +0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x83,0x00,0x00,0x00, +0x82,0x00,0x00,0x00,0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x89,0x00,0x00,0x00,0x83,0x00,0x00,0x00,0x88,0x00,0x00,0x00, +0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x8d,0x00,0x00,0x00, +0x71,0x00,0x00,0x00,0xc5,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x8e,0x00,0x00,0x00,0x89,0x00,0x00,0x00,0x8d,0x00,0x00,0x00, +0x70,0x00,0x04,0x00,0x1c,0x00,0x00,0x00,0x8f,0x00,0x00,0x00, +0x8e,0x00,0x00,0x00,0xc2,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x91,0x00,0x00,0x00,0x83,0x00,0x00,0x00,0x6e,0x00,0x00,0x00, +0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x94,0x00,0x00,0x00, +0x7a,0x00,0x00,0x00,0xc5,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x95,0x00,0x00,0x00,0x91,0x00,0x00,0x00,0x94,0x00,0x00,0x00, +0x70,0x00,0x04,0x00,0x1c,0x00,0x00,0x00,0x96,0x00,0x00,0x00, +0x95,0x00,0x00,0x00,0x50,0x00,0x05,0x00,0x84,0x00,0x00,0x00, +0x97,0x00,0x00,0x00,0x8f,0x00,0x00,0x00,0x96,0x00,0x00,0x00, +0x8e,0x00,0x05,0x00,0x84,0x00,0x00,0x00,0x9a,0x00,0x00,0x00, +0x97,0x00,0x00,0x00,0x5f,0x00,0x00,0x00,0x50,0x00,0x05,0x00, +0x84,0x00,0x00,0x00,0x9c,0x00,0x00,0x00,0x64,0x00,0x00,0x00, +0x64,0x00,0x00,0x00,0x81,0x00,0x05,0x00,0x84,0x00,0x00,0x00, +0x9d,0x00,0x00,0x00,0x9a,0x00,0x00,0x00,0x9c,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa4,0x00,0x00,0x00, +0x4f,0x00,0x00,0x00,0x4a,0x00,0x00,0x00,0x51,0x00,0x05,0x00, +0x1c,0x00,0x00,0x00,0xa7,0x00,0x00,0x00,0x9d,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0xa8,0x00,0x00,0x00, +0xa9,0x00,0x00,0x00,0xa1,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, +0xa4,0x00,0x00,0x00,0x3e,0x00,0x03,0x00,0xa9,0x00,0x00,0x00, +0xa7,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xad,0x00,0x00,0x00,0xa4,0x00,0x00,0x00,0x54,0x00,0x00,0x00, +0x51,0x00,0x05,0x00,0x1c,0x00,0x00,0x00,0xaf,0x00,0x00,0x00, +0x9d,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x41,0x00,0x06,0x00, +0xa8,0x00,0x00,0x00,0xb0,0x00,0x00,0x00,0xa1,0x00,0x00,0x00, +0x2e,0x00,0x00,0x00,0xad,0x00,0x00,0x00,0x3e,0x00,0x03,0x00, +0xb0,0x00,0x00,0x00,0xaf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xb3,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xb3,0x00,0x00,0x00, +0xfd,0x00,0x01,0x00,0x38,0x00,0x01,0x00, }; -const uint64_t get_rows_q8_0_f32_len = 2280; +const uint64_t get_rows_q5_1_f32_len = 2780; -unsigned char get_rows_q8_0_f32_fp32_data[] = { -0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, -0x8a,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, -0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x51,0x11,0x00,0x00, -0x11,0x00,0x02,0x00,0x60,0x11,0x00,0x00,0x0b,0x00,0x06,0x00, -0x01,0x00,0x00,0x00,0x47,0x4c,0x53,0x4c,0x2e,0x73,0x74,0x64, -0x2e,0x34,0x35,0x30,0x00,0x00,0x00,0x00,0x0e,0x00,0x03,0x00, -0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x0f,0x00,0x0a,0x00, -0x05,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x6d,0x61,0x69,0x6e, -0x00,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x1f,0x00,0x00,0x00, -0x2d,0x00,0x00,0x00,0x58,0x00,0x00,0x00,0x76,0x00,0x00,0x00, -0x10,0x00,0x06,0x00,0x04,0x00,0x00,0x00,0x11,0x00,0x00,0x00, -0x00,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x0b,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x1c,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x1d,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x1d,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x1d,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x08,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x1d,0x00,0x00,0x00, -0x03,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x47,0x00,0x03,0x00,0x1d,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x2a,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0x2b,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x2b,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x2b,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x2d,0x00,0x00,0x00, -0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x2d,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x53,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x54,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x54,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x55,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x22,0x00,0x00,0x00, -0x48,0x00,0x04,0x00,0x56,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x56,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x03,0x00,0x56,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x58,0x00,0x00,0x00,0x22,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x58,0x00,0x00,0x00, -0x21,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x73,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x48,0x00,0x04,0x00,0x74,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x19,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x74,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x03,0x00,0x74,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x76,0x00,0x00,0x00,0x22,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x76,0x00,0x00,0x00, -0x21,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x87,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x19,0x00,0x00,0x00, -0x13,0x00,0x02,0x00,0x02,0x00,0x00,0x00,0x21,0x00,0x03,0x00, -0x03,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x15,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x17,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x03,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x0a,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x0a,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x0d,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x15,0x00,0x04,0x00, -0x10,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x10,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x16,0x00,0x03,0x00, -0x1c,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x1e,0x00,0x06,0x00, -0x1d,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x1c,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x1e,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x1d,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x1e,0x00,0x00,0x00,0x1f,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x10,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x21,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x14,0x00,0x02,0x00,0x24,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, -0x2a,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, -0x2b,0x00,0x00,0x00,0x2a,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x2c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x2b,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x2c,0x00,0x00,0x00,0x2d,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x10,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x30,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x10,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x44,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x16,0x00,0x03,0x00,0x51,0x00,0x00,0x00, -0x10,0x00,0x00,0x00,0x15,0x00,0x04,0x00,0x52,0x00,0x00,0x00, -0x08,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x1c,0x00,0x04,0x00, -0x53,0x00,0x00,0x00,0x52,0x00,0x00,0x00,0x44,0x00,0x00,0x00, -0x1e,0x00,0x04,0x00,0x54,0x00,0x00,0x00,0x51,0x00,0x00,0x00, -0x53,0x00,0x00,0x00,0x1d,0x00,0x03,0x00,0x55,0x00,0x00,0x00, -0x54,0x00,0x00,0x00,0x1e,0x00,0x03,0x00,0x56,0x00,0x00,0x00, -0x55,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x57,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x56,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x57,0x00,0x00,0x00,0x58,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x5a,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x51,0x00,0x00,0x00,0x17,0x00,0x04,0x00,0x5e,0x00,0x00,0x00, -0x1c,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x63,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x52,0x00,0x00,0x00, -0x1d,0x00,0x03,0x00,0x73,0x00,0x00,0x00,0x1c,0x00,0x00,0x00, -0x1e,0x00,0x03,0x00,0x74,0x00,0x00,0x00,0x73,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x75,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x74,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x75,0x00,0x00,0x00, -0x76,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x7d,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x1c,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x86,0x00,0x00,0x00, -0x00,0x02,0x00,0x00,0x2c,0x00,0x06,0x00,0x09,0x00,0x00,0x00, -0x87,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x36,0x00,0x05,0x00,0x02,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x03,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0x05,0x00,0x00,0x00,0xf7,0x00,0x03,0x00, -0x88,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xfb,0x00,0x03,0x00, -0x0c,0x00,0x00,0x00,0x89,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x89,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00, -0x0e,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, -0x0e,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x10,0x00,0x00,0x00, -0x11,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x10,0x00,0x00,0x00,0x13,0x00,0x00,0x00,0x11,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x14,0x00,0x00,0x00,0x13,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x0d,0x00,0x00,0x00,0x17,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x17,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x10,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x18,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x1a,0x00,0x00,0x00, -0x19,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x21,0x00,0x00,0x00, -0x22,0x00,0x00,0x00,0x1f,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x22,0x00,0x00,0x00,0xae,0x00,0x05,0x00,0x24,0x00,0x00,0x00, -0x25,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0xf7,0x00,0x03,0x00,0x27,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x25,0x00,0x00,0x00,0x26,0x00,0x00,0x00, -0x27,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x26,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x88,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x27,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x30,0x00,0x00,0x00, -0x31,0x00,0x00,0x00,0x2d,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x1a,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x10,0x00,0x00,0x00, -0x32,0x00,0x00,0x00,0x31,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x33,0x00,0x00,0x00,0x32,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x38,0x00,0x00,0x00, -0x33,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x38,0x00,0x00,0x00, -0x14,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x3f,0x00,0x00,0x00,0x1a,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x41,0x00,0x00,0x00, -0x3f,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x45,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, -0x44,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x48,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x44,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x49,0x00,0x00,0x00, -0x48,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x89,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x4d,0x00,0x00,0x00,0x41,0x00,0x00,0x00, -0x44,0x00,0x00,0x00,0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x4e,0x00,0x00,0x00,0x41,0x00,0x00,0x00,0x4d,0x00,0x00,0x00, -0x41,0x00,0x07,0x00,0x5a,0x00,0x00,0x00,0x5b,0x00,0x00,0x00, -0x58,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x45,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x51,0x00,0x00,0x00, -0x5c,0x00,0x00,0x00,0x5b,0x00,0x00,0x00,0x73,0x00,0x04,0x00, -0x1c,0x00,0x00,0x00,0x5d,0x00,0x00,0x00,0x5c,0x00,0x00,0x00, -0x41,0x00,0x08,0x00,0x63,0x00,0x00,0x00,0x64,0x00,0x00,0x00, -0x58,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x45,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x49,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x52,0x00,0x00,0x00,0x65,0x00,0x00,0x00,0x64,0x00,0x00,0x00, -0x72,0x00,0x04,0x00,0x10,0x00,0x00,0x00,0x66,0x00,0x00,0x00, -0x65,0x00,0x00,0x00,0x6f,0x00,0x04,0x00,0x1c,0x00,0x00,0x00, -0x67,0x00,0x00,0x00,0x66,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x6a,0x00,0x00,0x00,0x49,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x41,0x00,0x08,0x00,0x63,0x00,0x00,0x00, -0x6b,0x00,0x00,0x00,0x58,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x45,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x6a,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x52,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x6b,0x00,0x00,0x00,0x72,0x00,0x04,0x00,0x10,0x00,0x00,0x00, -0x6d,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x6f,0x00,0x04,0x00, -0x1c,0x00,0x00,0x00,0x6e,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, -0x50,0x00,0x05,0x00,0x5e,0x00,0x00,0x00,0x6f,0x00,0x00,0x00, -0x67,0x00,0x00,0x00,0x6e,0x00,0x00,0x00,0x8e,0x00,0x05,0x00, -0x5e,0x00,0x00,0x00,0x72,0x00,0x00,0x00,0x6f,0x00,0x00,0x00, -0x5d,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x79,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x49,0x00,0x00,0x00, -0x51,0x00,0x05,0x00,0x1c,0x00,0x00,0x00,0x7c,0x00,0x00,0x00, -0x72,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x41,0x00,0x06,0x00, -0x7d,0x00,0x00,0x00,0x7e,0x00,0x00,0x00,0x76,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x79,0x00,0x00,0x00,0x3e,0x00,0x03,0x00, -0x7e,0x00,0x00,0x00,0x7c,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x82,0x00,0x00,0x00,0x79,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x51,0x00,0x05,0x00,0x1c,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0x72,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x41,0x00,0x06,0x00,0x7d,0x00,0x00,0x00,0x85,0x00,0x00,0x00, -0x76,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x82,0x00,0x00,0x00, -0x3e,0x00,0x03,0x00,0x85,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x88,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x88,0x00,0x00,0x00,0xfd,0x00,0x01,0x00,0x38,0x00,0x01,0x00, - -}; -const uint64_t get_rows_q8_0_f32_fp32_len = 2280; - -unsigned char get_rows_q8_0_fp32_data[] = { +unsigned char get_rows_q8_0_data[] = { 0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, 0x8b,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, 0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x51,0x11,0x00,0x00, @@ -19799,7 +10464,202 @@ unsigned char get_rows_q8_0_fp32_data[] = { 0xf8,0x00,0x02,0x00,0x89,0x00,0x00,0x00,0xfd,0x00,0x01,0x00, 0x38,0x00,0x01,0x00, }; -const uint64_t get_rows_q8_0_fp32_len = 2296; +const uint64_t get_rows_q8_0_len = 2296; + +unsigned char get_rows_q8_0_f32_data[] = { +0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, +0x8a,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, +0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x51,0x11,0x00,0x00, +0x11,0x00,0x02,0x00,0x60,0x11,0x00,0x00,0x0b,0x00,0x06,0x00, +0x01,0x00,0x00,0x00,0x47,0x4c,0x53,0x4c,0x2e,0x73,0x74,0x64, +0x2e,0x34,0x35,0x30,0x00,0x00,0x00,0x00,0x0e,0x00,0x03,0x00, +0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x0f,0x00,0x0a,0x00, +0x05,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x6d,0x61,0x69,0x6e, +0x00,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x1f,0x00,0x00,0x00, +0x2d,0x00,0x00,0x00,0x58,0x00,0x00,0x00,0x76,0x00,0x00,0x00, +0x10,0x00,0x06,0x00,0x04,0x00,0x00,0x00,0x11,0x00,0x00,0x00, +0x00,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x0b,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, +0x1c,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x1d,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x1d,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x1d,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x08,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x1d,0x00,0x00,0x00, +0x03,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x47,0x00,0x03,0x00,0x1d,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x2a,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0x2b,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x2b,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x2b,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x2d,0x00,0x00,0x00, +0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x2d,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x53,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x54,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x54,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x55,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x22,0x00,0x00,0x00, +0x48,0x00,0x04,0x00,0x56,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x56,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x03,0x00,0x56,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x58,0x00,0x00,0x00,0x22,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x58,0x00,0x00,0x00, +0x21,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x73,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x48,0x00,0x04,0x00,0x74,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x19,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x74,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x03,0x00,0x74,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x76,0x00,0x00,0x00,0x22,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x76,0x00,0x00,0x00, +0x21,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x87,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x19,0x00,0x00,0x00, +0x13,0x00,0x02,0x00,0x02,0x00,0x00,0x00,0x21,0x00,0x03,0x00, +0x03,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x15,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x17,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x03,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x0a,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0x0a,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x0d,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x15,0x00,0x04,0x00, +0x10,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x10,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x16,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x16,0x00,0x03,0x00, +0x1c,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x1e,0x00,0x06,0x00, +0x1d,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x1c,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x1e,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x1d,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0x1e,0x00,0x00,0x00,0x1f,0x00,0x00,0x00, +0x09,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x10,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x21,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x14,0x00,0x02,0x00,0x24,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, +0x2a,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, +0x2b,0x00,0x00,0x00,0x2a,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x2c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x2b,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0x2c,0x00,0x00,0x00,0x2d,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x10,0x00,0x00,0x00, +0x2e,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x30,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x10,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x44,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0x16,0x00,0x03,0x00,0x51,0x00,0x00,0x00, +0x10,0x00,0x00,0x00,0x15,0x00,0x04,0x00,0x52,0x00,0x00,0x00, +0x08,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x1c,0x00,0x04,0x00, +0x53,0x00,0x00,0x00,0x52,0x00,0x00,0x00,0x44,0x00,0x00,0x00, +0x1e,0x00,0x04,0x00,0x54,0x00,0x00,0x00,0x51,0x00,0x00,0x00, +0x53,0x00,0x00,0x00,0x1d,0x00,0x03,0x00,0x55,0x00,0x00,0x00, +0x54,0x00,0x00,0x00,0x1e,0x00,0x03,0x00,0x56,0x00,0x00,0x00, +0x55,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x57,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x56,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0x57,0x00,0x00,0x00,0x58,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x5a,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x51,0x00,0x00,0x00,0x17,0x00,0x04,0x00,0x5e,0x00,0x00,0x00, +0x1c,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x63,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x52,0x00,0x00,0x00, +0x1d,0x00,0x03,0x00,0x73,0x00,0x00,0x00,0x1c,0x00,0x00,0x00, +0x1e,0x00,0x03,0x00,0x74,0x00,0x00,0x00,0x73,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x75,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x74,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x75,0x00,0x00,0x00, +0x76,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x7d,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x1c,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0x00,0x02,0x00,0x00,0x2c,0x00,0x06,0x00,0x09,0x00,0x00,0x00, +0x87,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x16,0x00,0x00,0x00, +0x16,0x00,0x00,0x00,0x36,0x00,0x05,0x00,0x02,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x03,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0x05,0x00,0x00,0x00,0xf7,0x00,0x03,0x00, +0x88,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xfb,0x00,0x03,0x00, +0x0c,0x00,0x00,0x00,0x89,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0x89,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00, +0x0e,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, +0x0e,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x10,0x00,0x00,0x00, +0x11,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x10,0x00,0x00,0x00,0x13,0x00,0x00,0x00,0x11,0x00,0x00,0x00, +0x12,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x14,0x00,0x00,0x00,0x13,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x0d,0x00,0x00,0x00,0x17,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, +0x16,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x18,0x00,0x00,0x00,0x17,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, +0x10,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x18,0x00,0x00,0x00, +0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x1a,0x00,0x00,0x00, +0x19,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x21,0x00,0x00,0x00, +0x22,0x00,0x00,0x00,0x1f,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x22,0x00,0x00,0x00,0xae,0x00,0x05,0x00,0x24,0x00,0x00,0x00, +0x25,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0xf7,0x00,0x03,0x00,0x27,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x25,0x00,0x00,0x00,0x26,0x00,0x00,0x00, +0x27,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x26,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x88,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0x27,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x30,0x00,0x00,0x00, +0x31,0x00,0x00,0x00,0x2d,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, +0x1a,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x10,0x00,0x00,0x00, +0x32,0x00,0x00,0x00,0x31,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x33,0x00,0x00,0x00,0x32,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x38,0x00,0x00,0x00, +0x33,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x38,0x00,0x00,0x00, +0x14,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x3f,0x00,0x00,0x00,0x1a,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x41,0x00,0x00,0x00, +0x3f,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x45,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, +0x44,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x48,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x44,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x49,0x00,0x00,0x00, +0x48,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x89,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x4d,0x00,0x00,0x00,0x41,0x00,0x00,0x00, +0x44,0x00,0x00,0x00,0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x4e,0x00,0x00,0x00,0x41,0x00,0x00,0x00,0x4d,0x00,0x00,0x00, +0x41,0x00,0x07,0x00,0x5a,0x00,0x00,0x00,0x5b,0x00,0x00,0x00, +0x58,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x45,0x00,0x00,0x00, +0x2e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x51,0x00,0x00,0x00, +0x5c,0x00,0x00,0x00,0x5b,0x00,0x00,0x00,0x73,0x00,0x04,0x00, +0x1c,0x00,0x00,0x00,0x5d,0x00,0x00,0x00,0x5c,0x00,0x00,0x00, +0x41,0x00,0x08,0x00,0x63,0x00,0x00,0x00,0x64,0x00,0x00,0x00, +0x58,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x45,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0x49,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x52,0x00,0x00,0x00,0x65,0x00,0x00,0x00,0x64,0x00,0x00,0x00, +0x72,0x00,0x04,0x00,0x10,0x00,0x00,0x00,0x66,0x00,0x00,0x00, +0x65,0x00,0x00,0x00,0x6f,0x00,0x04,0x00,0x1c,0x00,0x00,0x00, +0x67,0x00,0x00,0x00,0x66,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x6a,0x00,0x00,0x00,0x49,0x00,0x00,0x00, +0x16,0x00,0x00,0x00,0x41,0x00,0x08,0x00,0x63,0x00,0x00,0x00, +0x6b,0x00,0x00,0x00,0x58,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, +0x45,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x6a,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x52,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, +0x6b,0x00,0x00,0x00,0x72,0x00,0x04,0x00,0x10,0x00,0x00,0x00, +0x6d,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x6f,0x00,0x04,0x00, +0x1c,0x00,0x00,0x00,0x6e,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, +0x50,0x00,0x05,0x00,0x5e,0x00,0x00,0x00,0x6f,0x00,0x00,0x00, +0x67,0x00,0x00,0x00,0x6e,0x00,0x00,0x00,0x8e,0x00,0x05,0x00, +0x5e,0x00,0x00,0x00,0x72,0x00,0x00,0x00,0x6f,0x00,0x00,0x00, +0x5d,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x79,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x49,0x00,0x00,0x00, +0x51,0x00,0x05,0x00,0x1c,0x00,0x00,0x00,0x7c,0x00,0x00,0x00, +0x72,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x41,0x00,0x06,0x00, +0x7d,0x00,0x00,0x00,0x7e,0x00,0x00,0x00,0x76,0x00,0x00,0x00, +0x2e,0x00,0x00,0x00,0x79,0x00,0x00,0x00,0x3e,0x00,0x03,0x00, +0x7e,0x00,0x00,0x00,0x7c,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x82,0x00,0x00,0x00,0x79,0x00,0x00,0x00, +0x16,0x00,0x00,0x00,0x51,0x00,0x05,0x00,0x1c,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0x72,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x41,0x00,0x06,0x00,0x7d,0x00,0x00,0x00,0x85,0x00,0x00,0x00, +0x76,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x82,0x00,0x00,0x00, +0x3e,0x00,0x03,0x00,0x85,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x88,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0x88,0x00,0x00,0x00,0xfd,0x00,0x01,0x00,0x38,0x00,0x01,0x00, + +}; +const uint64_t get_rows_q8_0_f32_len = 2280; unsigned char matmul_f16_aligned_l_data[] = { 0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, diff --git a/ggml-vulkan.cpp b/ggml-vulkan.cpp index b1e0006bb..14fb89e09 100644 --- a/ggml-vulkan.cpp +++ b/ggml-vulkan.cpp @@ -1,6 +1,6 @@ #include "ggml-vulkan.h" -#ifdef VK_RUN_TESTS +#ifdef GGML_VULKAN_RUN_TESTS #include #endif @@ -255,6 +255,7 @@ static size_t vk_staging_offset; static vk_buffer vk_sync_staging; static vk_context * vk_ctx; +static vk_context * vk_transfer_ctx; static bool vk_disable; @@ -264,7 +265,7 @@ size_t vk_output_tensor; #endif static vk_pipeline ggml_vk_create_pipeline(const std::string& name, size_t spv_size, const void* spv_data, const std::string& entrypoint, uint32_t parameter_count, uint32_t push_constant_size, std::array wg_denoms, std::vector&& specialization_constants, uint32_t align) { -#ifdef VK_DEBUG +#ifdef GGML_VULKAN_DEBUG std::cerr << "ggml_vk_create_pipeline(" << name << ", " << entrypoint << ", " << parameter_count << ", " << push_constant_size << ", (" << wg_denoms[0] << "," << wg_denoms[1] << "," << wg_denoms[2] << "), specialization_constants, " << align << ")" << std::endl; #endif GGML_ASSERT(parameter_count > 0); @@ -368,7 +369,7 @@ static vk_pipeline ggml_vk_create_pipeline(const std::string& name, size_t spv_s } static void ggml_vk_pipeline_allocate_descriptor_sets(vk_pipeline& pipeline, uint32_t n) { -#ifdef VK_DEBUG +#ifdef GGML_VULKAN_DEBUG std::cerr << "ggml_vk_pipeline_allocate_descriptor_sets(" << pipeline.name << ", " << n << ")" << std::endl; #endif // Check if gc already contains pipeline before adding it @@ -413,14 +414,14 @@ static void ggml_vk_pipeline_allocate_descriptor_sets(vk_pipeline& pipeline, uin } static void ggml_vk_pipeline_cleanup(vk_pipeline& pipeline) { -#ifdef VK_DEBUG +#ifdef GGML_VULKAN_DEBUG std::cerr << "ggml_vk_pipeline_cleanup(" << pipeline.name << ")" << std::endl; #endif pipeline.descriptor_set_idx = 0; } static vk::CommandBuffer ggml_vk_create_cmd_buffer(vk_queue& q) { -#ifdef VK_DEBUG +#ifdef GGML_VULKAN_DEBUG std::cerr << "ggml_vk_create_cmd_buffer()" << std::endl; #endif if (q.cmd_buffers.size() > q.cmd_buffer_idx) { @@ -442,7 +443,7 @@ static vk::CommandBuffer ggml_vk_create_cmd_buffer(vk_queue& q) { } static vk_submission ggml_vk_create_submission(vk_queue& q, std::vector wait_semaphores, std::vector signal_semaphores) { -#ifdef VK_DEBUG +#ifdef GGML_VULKAN_DEBUG std::cerr << "ggml_vk_create_submission()" << std::endl; #endif vk_submission s; @@ -453,14 +454,14 @@ static vk_submission ggml_vk_create_submission(vk_queue& q, std::vector wait_semaphores, std::vector signal_semaphores) { -#ifdef VK_DEBUG +#ifdef GGML_VULKAN_DEBUG std::cerr << "ggml_vk_create_sequence_1()" << std::endl; #endif return { ggml_vk_create_submission(q, std::move(wait_semaphores), std::move(signal_semaphores)) }; } static void ggml_vk_submit(vk_context * ctx, vk::Fence fence) { -#ifdef VK_DEBUG +#ifdef GGML_VULKAN_DEBUG std::cerr << "ggml_vk_submit(" << ctx->seqs.size() << ", " << fence << ")" << std::endl; #endif if (ctx->seqs.empty()) { @@ -536,7 +537,7 @@ static void ggml_vk_submit(vk_context * ctx, vk::Fence fence) { } static uint32_t ggml_vk_find_queue_family_index(std::vector& queue_family_props, const vk::QueueFlags& required, const vk::QueueFlags& avoid, int32_t compute_index, uint32_t min_num_queues) { -#ifdef VK_DEBUG +#ifdef GGML_VULKAN_DEBUG std::cerr << "ggml_vk_find_queue_family_index()" << std::endl; #endif const uint32_t qfsize = queue_family_props.size(); @@ -578,7 +579,7 @@ static uint32_t ggml_vk_find_queue_family_index(std::vector= vk_gc.tl_semaphores.size()) { @@ -642,7 +643,7 @@ static vk::Event ggml_vk_create_event() { } static void ggml_vk_queue_cleanup(vk_queue& q) { -#ifdef VK_DEBUG +#ifdef GGML_VULKAN_DEBUG std::cerr << "ggml_vk_queue_cleanup()" << std::endl; #endif // Requires command buffers to be done @@ -652,7 +653,7 @@ static void ggml_vk_queue_cleanup(vk_queue& q) { } static vk_buffer ggml_vk_create_buffer(size_t size, vk::MemoryPropertyFlags req_flags) { -#ifdef VK_DEBUG +#ifdef GGML_VULKAN_DEBUG std::cerr << "ggml_vk_create_buffer(" << size << ", " << to_string(req_flags) << ")" << std::endl; #endif GGML_ASSERT(size > 0); @@ -743,7 +744,7 @@ static void ggml_vk_destroy_buffer(vk_buffer& buf) { if (buf.size == 0) { return; } -#ifdef VK_DEBUG +#ifdef GGML_VULKAN_DEBUG std::cerr << "ggml_vk_destroy_buffer(" << buf.size << ")" << std::endl; #endif @@ -757,7 +758,7 @@ static vk_subbuffer ggml_vk_subbuffer(vk_buffer& buf) { } static void ggml_vk_sync_buffers(vk_context * ctx) { -#ifdef VK_DEBUG +#ifdef GGML_VULKAN_DEBUG std::cerr << "ggml_vk_sync_buffers()" << std::endl; #endif const std::vector mem_barriers{ { { vk::AccessFlagBits::eMemoryRead | vk::AccessFlagBits::eMemoryWrite }, { vk::AccessFlagBits::eMemoryRead | vk::AccessFlagBits::eMemoryWrite } } }; @@ -773,7 +774,7 @@ static void ggml_vk_sync_buffers(vk_context * ctx) { } static void ggml_vk_wait_events(vk::CommandBuffer& cmd_buffer, std::vector&& events, vk::PipelineStageFlags src_stages, vk::PipelineStageFlags dst_stages) { -#ifdef VK_DEBUG +#ifdef GGML_VULKAN_DEBUG std::cerr << "ggml_vk_wait_events()" << std::endl; #endif if (events.empty()) { @@ -810,7 +811,7 @@ static bool ggml_vk_build_shader(ggml_type type) { } static void ggml_vk_load_shaders() { -#ifdef VK_DEBUG +#ifdef GGML_VULKAN_DEBUG std::cerr << "ggml_vk_load_shaders()" << std::endl; #endif @@ -849,36 +850,6 @@ static void ggml_vk_load_shaders() { vk_pipeline_matmul_f16_f32_aligned_l = ggml_vk_create_pipeline("matmul_f16_f32_aligned_l", matmul_f16_f32_aligned_l_len, matmul_f16_f32_aligned_l_data, "main", 3, 14 * sizeof(uint32_t), l_wg_denoms, warptile_l, l_align); vk_pipeline_matmul_f16_f32_aligned_m = ggml_vk_create_pipeline("matmul_f16_f32_aligned_m", matmul_f16_f32_aligned_m_len, matmul_f16_f32_aligned_m_data, "main", 3, 14 * sizeof(uint32_t), m_wg_denoms, warptile_m, m_align); vk_pipeline_matmul_f16_f32_aligned_s = ggml_vk_create_pipeline("matmul_f16_f32_aligned_s", matmul_f16_f32_aligned_s_len, matmul_f16_f32_aligned_s_data, "main", 3, 14 * sizeof(uint32_t), s_wg_denoms, warptile_s, s_align); - - // Build dequant shaders - vk_pipeline_dequant[GGML_TYPE_F32] = ggml_vk_create_pipeline("f32_to_f16", f32_to_f16_len, f32_to_f16_data, "main", 2, 4 * sizeof(int), {64, 1, 1}, {}, 1); - - vk_pipeline_dequant[GGML_TYPE_F16] = ggml_vk_create_pipeline("dequant_f16", dequant_f16_len, dequant_f16_data, "main", 2, 4 * sizeof(int), {256 * 32, 1, 1}, {}, 1); - vk_pipeline_dequant[GGML_TYPE_Q4_0] = ggml_vk_create_pipeline("dequant_q4_0", dequant_q4_0_len, dequant_q4_0_data, "main", 2, 4 * sizeof(int), {256 * 32, 1, 1}, {}, 1); - vk_pipeline_dequant[GGML_TYPE_Q4_1] = ggml_vk_create_pipeline("dequant_q4_1", dequant_q4_1_len, dequant_q4_1_data, "main", 2, 4 * sizeof(int), {256 * 32, 1, 1}, {}, 1); - vk_pipeline_dequant[GGML_TYPE_Q5_0] = ggml_vk_create_pipeline("dequant_q5_0", dequant_q5_0_len, dequant_q5_0_data, "main", 2, 4 * sizeof(int), {256 * 32, 1, 1}, {}, 1); - vk_pipeline_dequant[GGML_TYPE_Q5_1] = ggml_vk_create_pipeline("dequant_q5_1", dequant_q5_1_len, dequant_q5_1_data, "main", 2, 4 * sizeof(int), {256 * 32, 1, 1}, {}, 1); - vk_pipeline_dequant[GGML_TYPE_Q8_0] = ggml_vk_create_pipeline("dequant_q8_0", dequant_q8_0_len, dequant_q8_0_data, "main", 2, 4 * sizeof(int), {256 * 32, 1, 1}, {}, 1); - vk_pipeline_dequant[GGML_TYPE_Q2_K] = ggml_vk_create_pipeline("dequant_q2_K", dequant_q2_K_len, dequant_q2_K_data, "main", 2, 4 * sizeof(int), {256 * 64, 1, 1}, {}, 1); - vk_pipeline_dequant[GGML_TYPE_Q3_K] = ggml_vk_create_pipeline("dequant_q3_K", dequant_q3_K_len, dequant_q3_K_data, "main", 2, 4 * sizeof(int), {256 * 64, 1, 1}, {}, 1); - vk_pipeline_dequant[GGML_TYPE_Q4_K] = ggml_vk_create_pipeline("dequant_q4_K", dequant_q4_K_len, dequant_q4_K_data, "main", 2, 4 * sizeof(int), {256 * 32, 1, 1}, {}, 1); - vk_pipeline_dequant[GGML_TYPE_Q5_K] = ggml_vk_create_pipeline("dequant_q5_K", dequant_q5_K_len, dequant_q5_K_data, "main", 2, 4 * sizeof(int), {256 * 64, 1, 1}, {}, 1); - vk_pipeline_dequant[GGML_TYPE_Q6_K] = ggml_vk_create_pipeline("dequant_q6_K", dequant_q6_K_len, dequant_q6_K_data, "main", 2, 4 * sizeof(int), {256 * 64, 1, 1}, {}, 1); - - // get_rows - vk_pipeline_get_rows[GGML_TYPE_F16] = ggml_vk_create_pipeline("get_rows_f16", get_rows_f16_len, get_rows_f16_data, "main", 3, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1); - vk_pipeline_get_rows[GGML_TYPE_Q4_0] = ggml_vk_create_pipeline("get_rows_q4_0", get_rows_q4_0_len, get_rows_q4_0_data, "main", 3, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1); - vk_pipeline_get_rows[GGML_TYPE_Q4_1] = ggml_vk_create_pipeline("get_rows_q4_1", get_rows_q4_1_len, get_rows_q4_1_data, "main", 3, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1); - vk_pipeline_get_rows[GGML_TYPE_Q5_0] = ggml_vk_create_pipeline("get_rows_q5_0", get_rows_q5_0_len, get_rows_q5_0_data, "main", 3, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1); - vk_pipeline_get_rows[GGML_TYPE_Q5_1] = ggml_vk_create_pipeline("get_rows_q5_1", get_rows_q5_1_len, get_rows_q5_1_data, "main", 3, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1); - vk_pipeline_get_rows[GGML_TYPE_Q8_0] = ggml_vk_create_pipeline("get_rows_q8_0", get_rows_q8_0_len, get_rows_q8_0_data, "main", 3, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1); - - vk_pipeline_get_rows_f32[GGML_TYPE_F16] = ggml_vk_create_pipeline("get_rows_f16_f32", get_rows_f16_f32_len, get_rows_f16_f32_data, "main", 3, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1); - vk_pipeline_get_rows_f32[GGML_TYPE_Q4_0] = ggml_vk_create_pipeline("get_rows_q4_0_f32", get_rows_q4_0_f32_len, get_rows_q4_0_f32_data, "main", 3, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1); - vk_pipeline_get_rows_f32[GGML_TYPE_Q4_1] = ggml_vk_create_pipeline("get_rows_q4_1_f32", get_rows_q4_1_f32_len, get_rows_q4_1_f32_data, "main", 3, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1); - vk_pipeline_get_rows_f32[GGML_TYPE_Q5_0] = ggml_vk_create_pipeline("get_rows_q5_0_f32", get_rows_q5_0_f32_len, get_rows_q5_0_f32_data, "main", 3, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1); - vk_pipeline_get_rows_f32[GGML_TYPE_Q5_1] = ggml_vk_create_pipeline("get_rows_q5_1_f32", get_rows_q5_1_f32_len, get_rows_q5_1_f32_data, "main", 3, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1); - vk_pipeline_get_rows_f32[GGML_TYPE_Q8_0] = ggml_vk_create_pipeline("get_rows_q8_0_f32", get_rows_q8_0_f32_len, get_rows_q8_0_f32_data, "main", 3, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1); } else { vk_pipeline_matmul_f32_l = ggml_vk_create_pipeline("matmul_f32_l", matmul_f32_l_fp32_len, matmul_f32_l_fp32_data, "main", 3, 14 * sizeof(uint32_t), l_wg_denoms, warptile_l, 1); vk_pipeline_matmul_f32_m = ggml_vk_create_pipeline("matmul_f32_m", matmul_f32_m_fp32_len, matmul_f32_m_fp32_data, "main", 3, 14 * sizeof(uint32_t), m_wg_denoms, warptile_m, 1); @@ -901,36 +872,6 @@ static void ggml_vk_load_shaders() { vk_pipeline_matmul_f16_f32_aligned_l = ggml_vk_create_pipeline("matmul_f16_f32_aligned_l", matmul_f16_f32_aligned_l_fp32_len, matmul_f16_f32_aligned_l_fp32_data, "main", 3, 14 * sizeof(uint32_t), l_wg_denoms, warptile_l, l_align); vk_pipeline_matmul_f16_f32_aligned_m = ggml_vk_create_pipeline("matmul_f16_f32_aligned_m", matmul_f16_f32_aligned_m_fp32_len, matmul_f16_f32_aligned_m_fp32_data, "main", 3, 14 * sizeof(uint32_t), m_wg_denoms, warptile_m, m_align); vk_pipeline_matmul_f16_f32_aligned_s = ggml_vk_create_pipeline("matmul_f16_f32_aligned_s", matmul_f16_f32_aligned_s_fp32_len, matmul_f16_f32_aligned_s_fp32_data, "main", 3, 14 * sizeof(uint32_t), s_wg_denoms, warptile_s, s_align); - - // Build dequant shaders - vk_pipeline_dequant[GGML_TYPE_F32] = ggml_vk_create_pipeline("f32_to_f16", f32_to_f16_fp32_len, f32_to_f16_fp32_data, "main", 2, 4 * sizeof(int), {64, 1, 1}, {}, 1); - - vk_pipeline_dequant[GGML_TYPE_F16] = ggml_vk_create_pipeline("dequant_f16", dequant_f16_fp32_len, dequant_f16_fp32_data, "main", 2, 4 * sizeof(int), {256 * 32, 1, 1}, {}, 1); - vk_pipeline_dequant[GGML_TYPE_Q4_0] = ggml_vk_create_pipeline("dequant_q4_0", dequant_q4_0_fp32_len, dequant_q4_0_fp32_data, "main", 2, 4 * sizeof(int), {256 * 32, 1, 1}, {}, 1); - vk_pipeline_dequant[GGML_TYPE_Q4_1] = ggml_vk_create_pipeline("dequant_q4_1", dequant_q4_1_fp32_len, dequant_q4_1_fp32_data, "main", 2, 4 * sizeof(int), {256 * 32, 1, 1}, {}, 1); - vk_pipeline_dequant[GGML_TYPE_Q5_0] = ggml_vk_create_pipeline("dequant_q5_0", dequant_q5_0_fp32_len, dequant_q5_0_fp32_data, "main", 2, 4 * sizeof(int), {256 * 32, 1, 1}, {}, 1); - vk_pipeline_dequant[GGML_TYPE_Q5_1] = ggml_vk_create_pipeline("dequant_q5_1", dequant_q5_1_fp32_len, dequant_q5_1_fp32_data, "main", 2, 4 * sizeof(int), {256 * 32, 1, 1}, {}, 1); - vk_pipeline_dequant[GGML_TYPE_Q8_0] = ggml_vk_create_pipeline("dequant_q8_0", dequant_q8_0_fp32_len, dequant_q8_0_fp32_data, "main", 2, 4 * sizeof(int), {256 * 32, 1, 1}, {}, 1); - vk_pipeline_dequant[GGML_TYPE_Q2_K] = ggml_vk_create_pipeline("dequant_q2_K", dequant_q2_K_fp32_len, dequant_q2_K_fp32_data, "main", 2, 4 * sizeof(int), {256 * 32, 1, 1}, {}, 1); - vk_pipeline_dequant[GGML_TYPE_Q3_K] = ggml_vk_create_pipeline("dequant_q3_K", dequant_q3_K_fp32_len, dequant_q3_K_fp32_data, "main", 2, 4 * sizeof(int), {256 * 32, 1, 1}, {}, 1); - vk_pipeline_dequant[GGML_TYPE_Q4_K] = ggml_vk_create_pipeline("dequant_q4_K", dequant_q4_K_fp32_len, dequant_q4_K_fp32_data, "main", 2, 4 * sizeof(int), {256 * 32, 1, 1}, {}, 1); - vk_pipeline_dequant[GGML_TYPE_Q5_K] = ggml_vk_create_pipeline("dequant_q5_K", dequant_q5_K_fp32_len, dequant_q5_K_fp32_data, "main", 2, 4 * sizeof(int), {256 * 32, 1, 1}, {}, 1); - vk_pipeline_dequant[GGML_TYPE_Q6_K] = ggml_vk_create_pipeline("dequant_q6_K", dequant_q6_K_fp32_len, dequant_q6_K_fp32_data, "main", 2, 4 * sizeof(int), {256 * 32, 1, 1}, {}, 1); - - // get_rows - vk_pipeline_get_rows[GGML_TYPE_F16] = ggml_vk_create_pipeline("get_rows_f16", get_rows_f16_fp32_len, get_rows_f16_fp32_data, "main", 3, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1); - vk_pipeline_get_rows[GGML_TYPE_Q4_0] = ggml_vk_create_pipeline("get_rows_q4_0", get_rows_q4_0_fp32_len, get_rows_q4_0_fp32_data, "main", 3, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1); - vk_pipeline_get_rows[GGML_TYPE_Q4_1] = ggml_vk_create_pipeline("get_rows_q4_1", get_rows_q4_1_fp32_len, get_rows_q4_1_fp32_data, "main", 3, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1); - vk_pipeline_get_rows[GGML_TYPE_Q5_0] = ggml_vk_create_pipeline("get_rows_q5_0", get_rows_q5_0_fp32_len, get_rows_q5_0_fp32_data, "main", 3, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1); - vk_pipeline_get_rows[GGML_TYPE_Q5_1] = ggml_vk_create_pipeline("get_rows_q5_1", get_rows_q5_1_fp32_len, get_rows_q5_1_fp32_data, "main", 3, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1); - vk_pipeline_get_rows[GGML_TYPE_Q8_0] = ggml_vk_create_pipeline("get_rows_q8_0", get_rows_q8_0_fp32_len, get_rows_q8_0_fp32_data, "main", 3, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1); - - vk_pipeline_get_rows_f32[GGML_TYPE_F16] = ggml_vk_create_pipeline("get_rows_f16_f32", get_rows_f16_f32_fp32_len, get_rows_f16_f32_fp32_data, "main", 3, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1); - vk_pipeline_get_rows_f32[GGML_TYPE_Q4_0] = ggml_vk_create_pipeline("get_rows_q4_0_f32", get_rows_q4_0_f32_fp32_len, get_rows_q4_0_f32_fp32_data, "main", 3, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1); - vk_pipeline_get_rows_f32[GGML_TYPE_Q4_1] = ggml_vk_create_pipeline("get_rows_q4_1_f32", get_rows_q4_1_f32_fp32_len, get_rows_q4_1_f32_fp32_data, "main", 3, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1); - vk_pipeline_get_rows_f32[GGML_TYPE_Q5_0] = ggml_vk_create_pipeline("get_rows_q5_0_f32", get_rows_q5_0_f32_fp32_len, get_rows_q5_0_f32_fp32_data, "main", 3, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1); - vk_pipeline_get_rows_f32[GGML_TYPE_Q5_1] = ggml_vk_create_pipeline("get_rows_q5_1_f32", get_rows_q5_1_f32_fp32_len, get_rows_q5_1_f32_fp32_data, "main", 3, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1); - vk_pipeline_get_rows_f32[GGML_TYPE_Q8_0] = ggml_vk_create_pipeline("get_rows_q8_0_f32", get_rows_q8_0_f32_fp32_len, get_rows_q8_0_f32_fp32_data, "main", 3, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1); } vk_pipeline_dequant_mul_mat_vec_f32[GGML_TYPE_F16] = ggml_vk_create_pipeline("mul_mat_vec_f16_f32", mul_mat_vec_f16_f32_len, mul_mat_vec_f16_f32_data, "main", 3, 3 * sizeof(int), {1, 1, 1}, {}, 1); @@ -945,6 +886,36 @@ static void ggml_vk_load_shaders() { vk_pipeline_dequant_mul_mat_vec_f32[GGML_TYPE_Q5_K] = ggml_vk_create_pipeline("mul_mat_vec_q5_K_f32", mul_mat_vec_q5_K_f32_len, mul_mat_vec_q5_K_f32_data, "main", 3, 3 * sizeof(int), {1, 1, 1}, {}, 1); vk_pipeline_dequant_mul_mat_vec_f32[GGML_TYPE_Q6_K] = ggml_vk_create_pipeline("mul_mat_vec_q6_K_f32", mul_mat_vec_q6_K_f32_len, mul_mat_vec_q6_K_f32_data, "main", 3, 3 * sizeof(int), {1, 1, 1}, {}, 1); + // dequant shaders + vk_pipeline_dequant[GGML_TYPE_F32] = ggml_vk_create_pipeline("f32_to_f16", f32_to_f16_len, f32_to_f16_data, "main", 2, 4 * sizeof(int), {64, 1, 1}, {}, 1); + + vk_pipeline_dequant[GGML_TYPE_F16] = ggml_vk_create_pipeline("dequant_f16", dequant_f16_len, dequant_f16_data, "main", 2, 4 * sizeof(int), {256 * 32, 1, 1}, {}, 1); + vk_pipeline_dequant[GGML_TYPE_Q4_0] = ggml_vk_create_pipeline("dequant_q4_0", dequant_q4_0_len, dequant_q4_0_data, "main", 2, 4 * sizeof(int), {256 * 32, 1, 1}, {}, 1); + vk_pipeline_dequant[GGML_TYPE_Q4_1] = ggml_vk_create_pipeline("dequant_q4_1", dequant_q4_1_len, dequant_q4_1_data, "main", 2, 4 * sizeof(int), {256 * 32, 1, 1}, {}, 1); + vk_pipeline_dequant[GGML_TYPE_Q5_0] = ggml_vk_create_pipeline("dequant_q5_0", dequant_q5_0_len, dequant_q5_0_data, "main", 2, 4 * sizeof(int), {256 * 32, 1, 1}, {}, 1); + vk_pipeline_dequant[GGML_TYPE_Q5_1] = ggml_vk_create_pipeline("dequant_q5_1", dequant_q5_1_len, dequant_q5_1_data, "main", 2, 4 * sizeof(int), {256 * 32, 1, 1}, {}, 1); + vk_pipeline_dequant[GGML_TYPE_Q8_0] = ggml_vk_create_pipeline("dequant_q8_0", dequant_q8_0_len, dequant_q8_0_data, "main", 2, 4 * sizeof(int), {256 * 32, 1, 1}, {}, 1); + vk_pipeline_dequant[GGML_TYPE_Q2_K] = ggml_vk_create_pipeline("dequant_q2_K", dequant_q2_K_len, dequant_q2_K_data, "main", 2, 4 * sizeof(int), {256 * 64, 1, 1}, {}, 1); + vk_pipeline_dequant[GGML_TYPE_Q3_K] = ggml_vk_create_pipeline("dequant_q3_K", dequant_q3_K_len, dequant_q3_K_data, "main", 2, 4 * sizeof(int), {256 * 64, 1, 1}, {}, 1); + vk_pipeline_dequant[GGML_TYPE_Q4_K] = ggml_vk_create_pipeline("dequant_q4_K", dequant_q4_K_len, dequant_q4_K_data, "main", 2, 4 * sizeof(int), {256 * 32, 1, 1}, {}, 1); + vk_pipeline_dequant[GGML_TYPE_Q5_K] = ggml_vk_create_pipeline("dequant_q5_K", dequant_q5_K_len, dequant_q5_K_data, "main", 2, 4 * sizeof(int), {256 * 64, 1, 1}, {}, 1); + vk_pipeline_dequant[GGML_TYPE_Q6_K] = ggml_vk_create_pipeline("dequant_q6_K", dequant_q6_K_len, dequant_q6_K_data, "main", 2, 4 * sizeof(int), {256 * 64, 1, 1}, {}, 1); + + // get_rows + vk_pipeline_get_rows[GGML_TYPE_F16] = ggml_vk_create_pipeline("get_rows_f16", get_rows_f16_len, get_rows_f16_data, "main", 3, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1); + vk_pipeline_get_rows[GGML_TYPE_Q4_0] = ggml_vk_create_pipeline("get_rows_q4_0", get_rows_q4_0_len, get_rows_q4_0_data, "main", 3, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1); + vk_pipeline_get_rows[GGML_TYPE_Q4_1] = ggml_vk_create_pipeline("get_rows_q4_1", get_rows_q4_1_len, get_rows_q4_1_data, "main", 3, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1); + vk_pipeline_get_rows[GGML_TYPE_Q5_0] = ggml_vk_create_pipeline("get_rows_q5_0", get_rows_q5_0_len, get_rows_q5_0_data, "main", 3, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1); + vk_pipeline_get_rows[GGML_TYPE_Q5_1] = ggml_vk_create_pipeline("get_rows_q5_1", get_rows_q5_1_len, get_rows_q5_1_data, "main", 3, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1); + vk_pipeline_get_rows[GGML_TYPE_Q8_0] = ggml_vk_create_pipeline("get_rows_q8_0", get_rows_q8_0_len, get_rows_q8_0_data, "main", 3, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1); + + vk_pipeline_get_rows_f32[GGML_TYPE_F16] = ggml_vk_create_pipeline("get_rows_f16_f32", get_rows_f16_f32_len, get_rows_f16_f32_data, "main", 3, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1); + vk_pipeline_get_rows_f32[GGML_TYPE_Q4_0] = ggml_vk_create_pipeline("get_rows_q4_0_f32", get_rows_q4_0_f32_len, get_rows_q4_0_f32_data, "main", 3, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1); + vk_pipeline_get_rows_f32[GGML_TYPE_Q4_1] = ggml_vk_create_pipeline("get_rows_q4_1_f32", get_rows_q4_1_f32_len, get_rows_q4_1_f32_data, "main", 3, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1); + vk_pipeline_get_rows_f32[GGML_TYPE_Q5_0] = ggml_vk_create_pipeline("get_rows_q5_0_f32", get_rows_q5_0_f32_len, get_rows_q5_0_f32_data, "main", 3, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1); + vk_pipeline_get_rows_f32[GGML_TYPE_Q5_1] = ggml_vk_create_pipeline("get_rows_q5_1_f32", get_rows_q5_1_f32_len, get_rows_q5_1_f32_data, "main", 3, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1); + vk_pipeline_get_rows_f32[GGML_TYPE_Q8_0] = ggml_vk_create_pipeline("get_rows_q8_0_f32", get_rows_q8_0_f32_len, get_rows_q8_0_f32_data, "main", 3, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1); + vk_pipeline_matmul_split_k_reduce = ggml_vk_create_pipeline("split_k_reduce", split_k_reduce_len, split_k_reduce_data, "main", 2, 2 * sizeof(uint32_t), {256, 1, 1}, {}, 1); vk_pipeline_mul_mat_vec_p021_f16_f32 = ggml_vk_create_pipeline("mul_mat_vec_p021_f16_f32", mul_mat_vec_p021_f16_f32_len, mul_mat_vec_p021_f16_f32_data, "main", 3, 6 * sizeof(uint32_t), {1, 1, 1}, {}, 1); @@ -983,7 +954,7 @@ static void ggml_vk_load_shaders() { } void ggml_vk_init() { -#ifdef VK_DEBUG +#ifdef GGML_VULKAN_DEBUG std::cerr << "ggml_vk_init()" << std::endl; #endif static bool initialized = false; @@ -999,17 +970,17 @@ void ggml_vk_init() { vk::ApplicationInfo app_info{ "ggml-vulkan", 1, nullptr, 0, VK_API_VERSION }; const std::vector layers = { -#ifdef VK_VALIDATE +#ifdef GGML_VULKAN_VALIDATE "VK_LAYER_KHRONOS_validation", #endif }; const std::vector extensions = { -#ifdef VK_VALIDATE +#ifdef GGML_VULKAN_VALIDATE "VK_EXT_validation_features", #endif }; vk::InstanceCreateInfo instance_create_info(vk::InstanceCreateFlags(), &app_info, layers, extensions); -#ifdef VK_VALIDATE +#ifdef GGML_VULKAN_VALIDATE const std::vector features_enable = { vk::ValidationFeatureEnableEXT::eBestPractices }; vk::ValidationFeaturesEXT validation_features = { features_enable, @@ -1120,7 +1091,7 @@ std::cerr << "ggml_vulkan: Validation layers enabled" << std::endl; device_extensions.push_back("VK_KHR_16bit_storage"); -#ifdef VK_VALIDATE +#ifdef GGML_VULKAN_VALIDATE device_extensions.push_back("VK_KHR_shader_non_semantic_info"); #endif @@ -1154,6 +1125,7 @@ std::cerr << "ggml_vulkan: Validation layers enabled" << std::endl; vk_fence = vk_device.device.createFence({}); vk_ctx = nullptr; + vk_transfer_ctx = nullptr; vk_disable = false; @@ -1166,7 +1138,7 @@ std::cerr << "ggml_vulkan: Validation layers enabled" << std::endl; } static vk_pipeline* ggml_vk_get_to_fp16(ggml_type type) { -#ifdef VK_DEBUG +#ifdef GGML_VULKAN_DEBUG std::cerr << "ggml_vk_get_to_fp16()" << std::endl; #endif switch (type) { @@ -1190,7 +1162,7 @@ static vk_pipeline* ggml_vk_get_to_fp16(ggml_type type) { } static vk_pipeline* ggml_vk_get_dequantize_mul_mat_vec(ggml_type type) { -#ifdef VK_DEBUG +#ifdef GGML_VULKAN_DEBUG std::cerr << "ggml_vk_get_dequantize_mul_mat_vec()" << std::endl; #endif switch (type) { @@ -1219,7 +1191,7 @@ static vk_pipeline* ggml_vk_get_dequantize_mul_mat_vec(ggml_type type) { static vk_buffer g_vk_buffer_pool[MAX_VK_BUFFERS]; static vk_buffer ggml_vk_pool_malloc(size_t size) { -#ifdef VK_DEBUG +#ifdef GGML_VULKAN_DEBUG std::cerr << "ggml_vk_pool_malloc(" << size << ")" << std::endl; #endif int best_i = -1; @@ -1253,7 +1225,7 @@ static vk_buffer ggml_vk_pool_malloc(size_t size) { } static void ggml_vk_pool_free(vk_buffer& buffer) { -#ifdef VK_DEBUG +#ifdef GGML_VULKAN_DEBUG std::cerr << "ggml_vk_pool_free(" << buffer.size << ")" << std::endl; #endif for (int i = 0; i < MAX_VK_BUFFERS; ++i) { @@ -1286,7 +1258,7 @@ static vk_buffer ggml_vk_create_buffer_temp(size_t size) { } static void * ggml_vk_host_malloc(size_t size) { -#ifdef VK_DEBUG +#ifdef GGML_VULKAN_DEBUG std::cerr << "ggml_vk_host_malloc(" << size << ")" << std::endl; #endif vk_buffer buf = ggml_vk_create_buffer(size, vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent | vk::MemoryPropertyFlagBits::eHostCached); @@ -1309,7 +1281,7 @@ static void ggml_vk_host_free(void* ptr) { if (ptr == nullptr) { return; } -#ifdef VK_DEBUG +#ifdef GGML_VULKAN_DEBUG std::cerr << "ggml_vk_host_free(" << ptr << ")" << std::endl; #endif vk_buffer* buf = nullptr; @@ -1363,7 +1335,7 @@ static void ggml_vk_dispatch_pipeline(vk_context * ctx, vk_pipeline& pipeline, s const uint32_t wg0 = CEIL_DIV(elements[0], pipeline.wg_denoms[0]); const uint32_t wg1 = CEIL_DIV(elements[1], pipeline.wg_denoms[1]); const uint32_t wg2 = CEIL_DIV(elements[2], pipeline.wg_denoms[2]); -#ifdef VK_DEBUG +#ifdef GGML_VULKAN_DEBUG std::cerr << "ggml_vk_dispatch_pipeline(" << pipeline.name << ", (" << wg0 << "," << wg1 << "," << wg2 << "))" << std::endl; #endif std::vector descriptor_buffer_infos; @@ -1398,7 +1370,7 @@ static void ggml_vk_end_submission(vk_submission& s, std::vector w } static void ggml_vk_ctx_end(vk_context * ctx) { -#ifdef VK_DEBUG +#ifdef GGML_VULKAN_DEBUG std::cerr << "ggml_vk_ctx_end(" << ctx << ", " << ctx->seqs.size() << ")" << std::endl; #endif if (ctx->s == nullptr) { @@ -1410,7 +1382,7 @@ static void ggml_vk_ctx_end(vk_context * ctx) { } static void ggml_vk_ctx_begin(vk_context * ctx) { -#ifdef VK_DEBUG +#ifdef GGML_VULKAN_DEBUG std::cerr << "ggml_vk_ctx_begin(" << ctx << ")" << std::endl; #endif if (ctx->s != nullptr) { @@ -1441,7 +1413,7 @@ static void ensure_sync_staging_buffer(size_t size) { } static void ggml_vk_buffer_write_nc_async(vk_context * ctx, vk_buffer* dst, size_t offset, const ggml_tensor * tensor, bool sync_staging = false) { -#ifdef VK_DEBUG +#ifdef GGML_VULKAN_DEBUG std::cerr << "ggml_vk_buffer_write_nc_async(" << tensor << ")" << std::endl; #endif GGML_ASSERT(!ggml_is_contiguous(tensor)); @@ -1548,7 +1520,7 @@ static void ggml_vk_buffer_write_nc_async(vk_context * ctx, vk_buffer* dst, size } static void ggml_vk_buffer_write_2d_async(vk_context * ctx, vk_buffer* dst, size_t offset, const void * src, size_t spitch, size_t width, size_t height, bool sync_staging = false) { -#ifdef VK_DEBUG +#ifdef GGML_VULKAN_DEBUG std::cerr << "ggml_vk_buffer_write_2d_async(" << width << ", " << height << ")" << std::endl; #endif // Buffer is already mapped @@ -1582,7 +1554,7 @@ static void ggml_vk_buffer_write_2d_async(vk_context * ctx, vk_buffer* dst, size ctx->s->buffer.copyBuffer(buf->buffer, dst->buffer, slices); return; } -#ifdef VK_DEBUG +#ifdef GGML_VULKAN_DEBUG std::cerr << "STAGING" << std::endl; #endif @@ -1619,14 +1591,14 @@ static void ggml_vk_buffer_write_2d_async(vk_context * ctx, vk_buffer* dst, size } static void ggml_vk_buffer_write_async(vk_context * ctx, vk_buffer* dst, size_t offset, const void * src, size_t size, bool sync_staging = false) { -#ifdef VK_DEBUG +#ifdef GGML_VULKAN_DEBUG std::cerr << "ggml_vk_buffer_write_async(" << size << ")" << std::endl; #endif return ggml_vk_buffer_write_2d_async(ctx, dst, offset, src, size, size, 1, sync_staging); } static void ggml_vk_buffer_write_2d(vk_buffer* dst, size_t offset, const void * src, size_t spitch, size_t width, size_t height) { -#ifdef VK_DEBUG +#ifdef GGML_VULKAN_DEBUG std::cerr << "ggml_vk_buffer_write_2d(" << width << ", " << height << ")" << std::endl; #endif // Buffer is already mapped @@ -1653,14 +1625,14 @@ static void ggml_vk_buffer_write_2d(vk_buffer* dst, size_t offset, const void * } static void ggml_vk_buffer_write(vk_buffer* dst, size_t offset, const void * src, size_t size) { -#ifdef VK_DEBUG +#ifdef GGML_VULKAN_DEBUG std::cerr << "ggml_vk_buffer_write(" << size << ")" << std::endl; #endif ggml_vk_buffer_write_2d(dst, offset, src, 0, size, 1); } static void ggml_vk_buffer_read_2d_async(vk_context * ctx, vk_buffer* src, size_t offset, void * dst, size_t spitch, size_t dpitch, size_t width, size_t height, bool sync_staging = false) { -#ifdef VK_DEBUG +#ifdef GGML_VULKAN_DEBUG std::cerr << "ggml_vk_buffer_read_2d_async(offset=" << offset << ", width=" << width << ", height=" << height << ")" << std::endl; #endif GGML_ASSERT(width > 0); @@ -1693,7 +1665,7 @@ static void ggml_vk_buffer_read_2d_async(vk_context * ctx, vk_buffer* src, size_ return; } -#ifdef VK_DEBUG +#ifdef GGML_VULKAN_DEBUG std::cerr << "STAGING" << std::endl; #endif @@ -1722,7 +1694,7 @@ static void ggml_vk_buffer_read_async(vk_context * ctx, vk_buffer* src, size_t o } static void ggml_vk_buffer_read(vk_buffer* src, size_t offset, void * dst, size_t size) { -#ifdef VK_DEBUG +#ifdef GGML_VULKAN_DEBUG std::cerr << "ggml_vk_buffer_read(" << offset << ", " << size << ")" << std::endl; #endif if(src->memory_property_flags & vk::MemoryPropertyFlagBits::eHostVisible) { @@ -1746,7 +1718,7 @@ static void ggml_vk_buffer_read(vk_buffer* src, size_t offset, void * dst, size_ } static void ggml_vk_buffer_copy_async(vk_context * ctx, vk_buffer * dst, size_t dst_offset, vk_buffer * src, size_t src_offset, size_t size) { -#ifdef VK_DEBUG +#ifdef GGML_VULKAN_DEBUG std::cerr << "ggml_vk_buffer_copy_async(" << size << ")" << std::endl; #endif VkBufferCopy bc{ src_offset, dst_offset, size }; @@ -1755,7 +1727,7 @@ static void ggml_vk_buffer_copy_async(vk_context * ctx, vk_buffer * dst, size_t } static void ggml_vk_buffer_copy(vk_buffer * dst, size_t dst_offset, vk_buffer * src, size_t src_offset, size_t size) { -#ifdef VK_DEBUG +#ifdef GGML_VULKAN_DEBUG std::cerr << "ggml_vk_buffer_copy(" << size << ")" << std::endl; #endif VkBufferCopy bc{ src_offset, dst_offset, size }; @@ -1771,7 +1743,7 @@ static void ggml_vk_buffer_copy(vk_buffer * dst, size_t dst_offset, vk_buffer * } static void ggml_vk_buffer_memset(vk_buffer* dst, size_t offset, uint32_t c, size_t size) { -#ifdef VK_DEBUG +#ifdef GGML_VULKAN_DEBUG std::cerr << "ggml_vk_buffer_memset(" << offset << ", " << c << ", " << size << ")" << std::endl; #endif vk_context * ctx = ggml_vk_create_context(vk_device.transfer_queue); @@ -1785,7 +1757,7 @@ static void ggml_vk_buffer_memset(vk_buffer* dst, size_t offset, uint32_t c, siz } static void ggml_vk_h2d_tensor_2d(vk_context * ctx, vk_buffer * dst, size_t offset, const ggml_tensor * src, uint64_t i3, uint64_t i2, uint64_t i1) { -#ifdef VK_DEBUG +#ifdef GGML_VULKAN_DEBUG std::cerr << "ggml_vk_h2d_tensor_2d(dst=" << dst << ", offset=" << offset << ", src=" << src << ", i3=" << i3 << ", i2=" << i2 << ", i1=" << i1 << ")" << std::endl; #endif const uint64_t ne0 = src->ne[0]; @@ -1815,7 +1787,7 @@ static void ggml_vk_h2d_tensor_2d(vk_context * ctx, vk_buffer * dst, size_t offs } static void ggml_vk_d2h_tensor_2d(vk_context * ctx, vk_buffer * src, size_t offset, const ggml_tensor * dst) { -#ifdef VK_DEBUG +#ifdef GGML_VULKAN_DEBUG std::cerr << "ggml_vk_d2h_tensor_2d()" << std::endl; #endif const uint64_t ne0 = dst->ne[0]; @@ -1841,24 +1813,24 @@ static void ggml_vk_d2h_tensor_2d(vk_context * ctx, vk_buffer * src, size_t offs } static uint32_t ggml_vk_guess_split_k(int m, int n, int k) { -#ifdef VK_DEBUG +#ifdef GGML_VULKAN_DEBUG std::cerr << "ggml_vk_guess_split_k(" << m << ", " << n << ", " << k << ")"; #endif if (k > 128 && (m < 128 || n < 128) && m > 2 && n > 2) { -#ifdef VK_DEBUG +#ifdef GGML_VULKAN_DEBUG std::cerr << " = 4" << std::endl; #endif return 4; } -#ifdef VK_DEBUG +#ifdef GGML_VULKAN_DEBUG std::cerr << " = 1" << std::endl; #endif return 1; } static uint32_t ggml_vk_guess_matmul_pipeline_align(int m, int n) { -#ifdef VK_DEBUG +#ifdef GGML_VULKAN_DEBUG std::cerr << "ggml_vk_guess_matmul_pipeline_align(" << m << ", " << n << ")" << std::endl; #endif if (m <= 32 || n <= 32) { @@ -1871,41 +1843,41 @@ static uint32_t ggml_vk_guess_matmul_pipeline_align(int m, int n) { } static vk_pipeline* ggml_vk_guess_matmul_pipeline(bool bit16_x, bool bit16_y, int m, int n, bool aligned) { -#ifdef VK_DEBUG +#ifdef GGML_VULKAN_DEBUG std::cerr << "ggml_vk_guess_matmul_pipeline(" << bit16_x << ", " << bit16_y << ", " << m << ", " << n << ", " << aligned << ")"; #endif if (bit16_x && bit16_y) { - if (m <= 32 || n <= 32) { -#ifdef VK_DEBUG + if (vk_device.vendor_id == VK_VENDOR_ID_INTEL || m <= 32 || n <= 32) { +#ifdef GGML_VULKAN_DEBUG std::cerr << " S" << std::endl; #endif return aligned ? &vk_pipeline_matmul_f16_aligned_s : &vk_pipeline_matmul_f16_s; } if (vk_device.subgroup_size == 64 || m <= 64 || n <= 64) { -#ifdef VK_DEBUG +#ifdef GGML_VULKAN_DEBUG std::cerr << " M" << std::endl; #endif return aligned ? &vk_pipeline_matmul_f16_aligned_m : &vk_pipeline_matmul_f16_m; } -#ifdef VK_DEBUG +#ifdef GGML_VULKAN_DEBUG std::cerr << " L" << std::endl; #endif return aligned ? &vk_pipeline_matmul_f16_aligned_l : &vk_pipeline_matmul_f16_l; } if (bit16_x && !bit16_y) { - if (m <= 32 || n <= 32) { -#ifdef VK_DEBUG + if (vk_device.vendor_id == VK_VENDOR_ID_INTEL || m <= 32 || n <= 32) { +#ifdef GGML_VULKAN_DEBUG std::cerr << " S" << std::endl; #endif return aligned ? &vk_pipeline_matmul_f16_f32_aligned_s : &vk_pipeline_matmul_f16_f32_s; } if (vk_device.subgroup_size == 64 || m <= 64 || n <= 64) { -#ifdef VK_DEBUG +#ifdef GGML_VULKAN_DEBUG std::cerr << " M" << std::endl; #endif return aligned ? &vk_pipeline_matmul_f16_f32_aligned_m : &vk_pipeline_matmul_f16_f32_m; } -#ifdef VK_DEBUG +#ifdef GGML_VULKAN_DEBUG std::cerr << " L" << std::endl; #endif return aligned ? &vk_pipeline_matmul_f16_f32_aligned_l : &vk_pipeline_matmul_f16_f32_l; @@ -1914,30 +1886,30 @@ static vk_pipeline* ggml_vk_guess_matmul_pipeline(bool bit16_x, bool bit16_y, in GGML_ASSERT(false); } - if (m <= 32 || n <= 32) { -#ifdef VK_DEBUG + if (vk_device.vendor_id == VK_VENDOR_ID_INTEL || m <= 32 || n <= 32) { +#ifdef GGML_VULKAN_DEBUG std::cerr << " S" << std::endl; #endif return aligned ? &vk_pipeline_matmul_f32_aligned_s : &vk_pipeline_matmul_f32_s; } if (vk_device.subgroup_size == 64 || m <= 64 || n <= 64) { -#ifdef VK_DEBUG +#ifdef GGML_VULKAN_DEBUG std::cerr << " M" << std::endl; #endif return aligned ? &vk_pipeline_matmul_f32_aligned_m : &vk_pipeline_matmul_f32_m; } -#ifdef VK_DEBUG +#ifdef GGML_VULKAN_DEBUG std::cerr << " L" << std::endl; #endif return aligned ? &vk_pipeline_matmul_f32_aligned_l : &vk_pipeline_matmul_f32_l; } static void ggml_vk_matmul(vk_context * ctx, vk_pipeline& pipeline, vk_subbuffer&& a, vk_subbuffer&& b, vk_subbuffer&& d, vk_subbuffer&& split_k_buffer, uint32_t m, uint32_t n, uint32_t k, uint32_t stride_a, uint32_t stride_b, uint32_t stride_d, uint32_t split_k, uint32_t batch, uint32_t ne02, uint32_t ne12, uint32_t broadcast2, uint32_t broadcast3, uint32_t batch_stride_a, uint32_t batch_stride_b, uint32_t batch_stride_d) { -#ifdef VK_DEBUG +#ifdef GGML_VULKAN_DEBUG std::cerr << "ggml_vk_matmul(a: (" << a.buffer.buffer << ", " << a.offset << ", " << a.size << "), b: (" << b.buffer.buffer << ", " << b.offset << ", " << b.size << "), c: (" << d.buffer.buffer << ", " << d.offset << ", " << d.size << "), split_k: (" << split_k_buffer.buffer.buffer << ", " << split_k_buffer.offset << ", " << split_k_buffer.size << "), m: " << m << ", n: " << n << ", k: " << k << ", stride_a: " << stride_a << ", stride_b: " << stride_b << ", stride_d: " << stride_d << ", split_k: " << split_k << ", batch: " << batch << ", ne02: " << ne02 << ", ne12: " << ne12 << ", broadcast2: " << broadcast2 << ", broadcast3: " << broadcast3 << ", batch_stride_a: " << batch_stride_a << ", batch_stride_b: " << batch_stride_b << ", batch_stride_d: " << batch_stride_d << ")" << std::endl; #endif + ggml_vk_sync_buffers(ctx); if (split_k == 1) { - ggml_vk_sync_buffers(ctx); const std::array pc = { m, n, k, stride_a, stride_b, stride_d, k, ne02, ne12, broadcast2, broadcast3, batch_stride_a, batch_stride_b, batch_stride_d }; ggml_vk_dispatch_pipeline(ctx, pipeline, { a, b, d }, pc.size() * sizeof(uint32_t), pc.data(), { m, n, batch }); return; @@ -1945,10 +1917,6 @@ static void ggml_vk_matmul(vk_context * ctx, vk_pipeline& pipeline, vk_subbuffer GGML_ASSERT(batch_stride_d == m * n); - // Synchronize the two submissions - ggml_vk_sync_buffers(ctx); - ctx->s->buffer.fillBuffer(split_k_buffer.buffer.buffer, 0, split_k_buffer.size, 0); - ggml_vk_sync_buffers(ctx); const std::array pc1 = { m, n, k, stride_a, stride_b, stride_d, CEIL_DIV(k, split_k), ne02, ne12, broadcast2, broadcast3, batch_stride_a, batch_stride_b, batch_stride_d }; // Make sure enough workgroups get assigned for split k to work ggml_vk_dispatch_pipeline(ctx, pipeline, { a, b, split_k_buffer }, pc1.size() * sizeof(uint32_t), pc1.data(), { (CEIL_DIV(m, pipeline.wg_denoms[0]) * pipeline.wg_denoms[0]) * split_k, n, batch }); @@ -1980,7 +1948,7 @@ static vk_pipeline * ggml_vk_get_cpy_pipeline(ggml_type from, ggml_type to) { } static void ggml_vk_cpy_to_contiguous(vk_context * ctx, vk_pipeline * pipeline, const ggml_tensor * tensor, vk_subbuffer&& in, vk_subbuffer&& out, ggml_type buffer_type, bool aligned=true) { -#ifdef VK_DEBUG +#ifdef GGML_VULKAN_DEBUG std::cerr << "ggml_vk_cpy_to_contiguous((" << tensor << ", type=" << tensor->type << ", backend=" << tensor->backend << ", ne0=" << tensor->ne[0] << ", ne1=" << tensor->ne[1] << ", ne2=" << tensor->ne[2] << ", ne3=" << tensor->ne[3] << ", nb0=" << tensor->nb[0] << ", nb1=" << tensor->nb[1] << ", nb2=" << tensor->nb[2] << ", nb3=" << tensor->nb[3] << "), "; std::cerr << "buffer in size=" << in.buffer.size << ", buffer out size=" << out.buffer.size << ")" << std::endl; #endif @@ -2002,7 +1970,7 @@ static void ggml_vk_cpy_to_contiguous(vk_context * ctx, vk_pipeline * pipeline, } static void ggml_vk_mul_mat_q_f16(vk_context * ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { -#ifdef VK_DEBUG +#ifdef GGML_VULKAN_DEBUG std::cerr << "ggml_vk_mul_mat_q_f16((" << src0 << ", name=" << src0->name << ", type=" << src0->type << ", backend=" << src0->backend << ", ne0=" << src0->ne[0] << ", ne1=" << src0->ne[1] << ", ne2=" << src0->ne[2] << ", ne3=" << src0->ne[3] << ", nb0=" << src0->nb[0] << ", nb1=" << src0->nb[1] << ", nb2=" << src0->nb[2] << ", nb3=" << src0->nb[3]; std::cerr << "), (" << src1 << ", name=" << src1->name << ", type=" << src1->type << ", backend=" << src1->backend << ", ne0=" << src1->ne[0] << ", ne1=" << src1->ne[1] << ", ne2=" << src1->ne[2] << ", ne3=" << src1->ne[3] << ", nb0=" << src1->nb[0] << ", nb1=" << src1->nb[1] << ", nb2=" << src1->nb[2] << ", nb3=" << src1->nb[3]; std::cerr << "), (" << dst << ", name=" << dst->name << ", type=" << dst->type << ", backend=" << dst->backend << ", ne0=" << dst->ne[0] << ", ne1=" << dst->ne[1] << ", ne2=" << dst->ne[2] << ", ne3=" << dst->ne[3] << ", nb0=" << dst->nb[0] << ", nb1=" << dst->nb[1] << ", nb2=" << dst->nb[2] << ", nb3=" << dst->nb[3] << "),)" << std::endl; @@ -2186,7 +2154,7 @@ static void ggml_vk_mul_mat_q_f16(vk_context * ctx, const ggml_tensor * src0, co } static void ggml_vk_mul_mat_vec_q_f16(vk_context * ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { -#ifdef VK_DEBUG +#ifdef GGML_VULKAN_DEBUG std::cerr << "ggml_vk_mul_mat_vec_q_f16((" << src0 << ", name=" << src0->name << ", type=" << src0->type << ", backend=" << src0->backend << ", ne0=" << src0->ne[0] << ", ne1=" << src0->ne[1] << ", ne2=" << src0->ne[2] << ", ne3=" << src0->ne[3] << ", nb0=" << src0->nb[0] << ", nb1=" << src0->nb[1] << ", nb2=" << src0->nb[2] << ", nb3=" << src0->nb[3]; std::cerr << "), (" << src1 << ", name=" << src1->name << ", type=" << src1->type << ", backend=" << src1->backend << ", ne0=" << src1->ne[0] << ", ne1=" << src1->ne[1] << ", ne2=" << src1->ne[2] << ", ne3=" << src1->ne[3] << ", nb0=" << src1->nb[0] << ", nb1=" << src1->nb[1] << ", nb2=" << src1->nb[2] << ", nb3=" << src1->nb[3]; std::cerr << "), (" << dst << ", name=" << dst->name << ", type=" << dst->type << ", backend=" << dst->backend << ", ne0=" << dst->ne[0] << ", ne1=" << dst->ne[1] << ", ne2=" << dst->ne[2] << ", ne3=" << dst->ne[3] << ", nb0=" << dst->nb[0] << ", nb1=" << dst->nb[1] << ", nb2=" << dst->nb[2] << ", nb3=" << dst->nb[3] << "),)" << std::endl; @@ -2366,7 +2334,7 @@ static void ggml_vk_mul_mat_vec_q_f16(vk_context * ctx, const ggml_tensor * src0 } static void ggml_vk_mul_mat_vec_p021_f16_f32(vk_context * ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { -#ifdef VK_DEBUG +#ifdef GGML_VULKAN_DEBUG std::cerr << "ggml_vk_mul_mat_p021_f16_f32((" << src0 << ", name=" << src0->name << ", type=" << src0->type << ", backend=" << src0->backend << ", ne0=" << src0->ne[0] << ", ne1=" << src0->ne[1] << ", ne2=" << src0->ne[2] << ", ne3=" << src0->ne[3] << ", nb0=" << src0->nb[0] << ", nb1=" << src0->nb[1] << ", nb2=" << src0->nb[2] << ", nb3=" << src0->nb[3]; std::cerr << "), (" << src1 << ", name=" << src1->name << ", type=" << src1->type << ", backend=" << src1->backend << ", ne0=" << src1->ne[0] << ", ne1=" << src1->ne[1] << ", ne2=" << src1->ne[2] << ", ne3=" << src1->ne[3] << ", nb0=" << src1->nb[0] << ", nb1=" << src1->nb[1] << ", nb2=" << src1->nb[2] << ", nb3=" << src1->nb[3]; std::cerr << "), (" << dst << ", name=" << dst->name << ", type=" << dst->type << ", backend=" << dst->backend << ", ne0=" << dst->ne[0] << ", ne1=" << dst->ne[1] << ", ne2=" << dst->ne[2] << ", ne3=" << dst->ne[3] << ", nb0=" << dst->nb[0] << ", nb1=" << dst->nb[1] << ", nb2=" << dst->nb[2] << ", nb3=" << dst->nb[3] << "),)" << std::endl; @@ -2455,7 +2423,7 @@ static void ggml_vk_mul_mat_vec_p021_f16_f32(vk_context * ctx, const ggml_tensor } static void ggml_vk_mul_mat_vec_nc_f16_f32(vk_context * ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { -#ifdef VK_DEBUG +#ifdef GGML_VULKAN_DEBUG std::cerr << "ggml_vk_mul_mat_nc_f16_f32((" << src0 << ", name=" << src0->name << ", type=" << src0->type << ", backend=" << src0->backend << ", ne0=" << src0->ne[0] << ", ne1=" << src0->ne[1] << ", ne2=" << src0->ne[2] << ", ne3=" << src0->ne[3] << ", nb0=" << src0->nb[0] << ", nb1=" << src0->nb[1] << ", nb2=" << src0->nb[2] << ", nb3=" << src0->nb[3]; std::cerr << "), (" << src1 << ", name=" << src1->name << ", type=" << src1->type << ", backend=" << src1->backend << ", ne0=" << src1->ne[0] << ", ne1=" << src1->ne[1] << ", ne2=" << src1->ne[2] << ", ne3=" << src1->ne[3] << ", nb0=" << src1->nb[0] << ", nb1=" << src1->nb[1] << ", nb2=" << src1->nb[2] << ", nb3=" << src1->nb[3]; std::cerr << "), (" << dst << ", name=" << dst->name << ", type=" << dst->type << ", backend=" << dst->backend << ", ne0=" << dst->ne[0] << ", ne1=" << dst->ne[1] << ", ne2=" << dst->ne[2] << ", ne3=" << dst->ne[3] << ", nb0=" << dst->nb[0] << ", nb1=" << dst->nb[1] << ", nb2=" << dst->nb[2] << ", nb3=" << dst->nb[3] << "),)" << std::endl; @@ -2561,7 +2529,7 @@ static bool ggml_vk_can_mul_mat(const ggml_tensor * src0, const ggml_tensor * sr } static void ggml_vk_mul_mat(vk_context * ctx, const struct ggml_tensor * src0, const struct ggml_tensor * src1, struct ggml_tensor * dst) { -#ifdef VK_DEBUG +#ifdef GGML_VULKAN_DEBUG std::cerr << "ggml_vk_mul_mat(" << src0 << ", " << src1 << ", " << dst << ")" << std::endl; #endif if (src0->type == GGML_TYPE_F16 && ggml_is_permuted(src0) && ggml_is_permuted(src1) && src1->ne[1] == 1) { @@ -2774,7 +2742,7 @@ static void ggml_vk_check_results_0(ggml_compute_params * params, ggml_tensor * template static void ggml_vk_op_f32(vk_context * ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, ggml_op op, const PC&& pc) { -#ifdef VK_DEBUG +#ifdef GGML_VULKAN_DEBUG std::cerr << "ggml_vk_op_f32((" << src0 << ", name=" << src0->name << ", type=" << src0->type << ", backend=" << src0->backend << ", ne0=" << src0->ne[0] << ", ne1=" << src0->ne[1] << ", ne2=" << src0->ne[2] << ", ne3=" << src0->ne[3] << ", nb0=" << src0->nb[0] << ", nb1=" << src0->nb[1] << ", nb2=" << src0->nb[2] << ", nb3=" << src0->nb[3]; if (src1 != nullptr) { std::cerr << "), (" << src1 << ", name=" << src1->name << ", type=" << src1->type << ", backend=" << src1->backend << ", ne0=" << src1->ne[0] << ", ne1=" << src1->ne[1] << ", ne2=" << src1->ne[2] << ", ne3=" << src1->ne[3] << ", nb0=" << src1->nb[0] << ", nb1=" << src1->nb[1] << ", nb2=" << src1->nb[2] << ", nb3=" << src1->nb[3]; @@ -3095,7 +3063,7 @@ static void ggml_vk_nop(vk_context * ctx, const ggml_tensor * src0, ggml_tensor } } -#ifdef VK_RUN_TESTS +#ifdef GGML_VULKAN_RUN_TESTS static void ggml_vk_print_matrix_area(const void * data, ggml_type type, int ne0, int ne1, int i0, int i1, int i2) { if (type != GGML_TYPE_F32 && type != GGML_TYPE_F16) { return; @@ -3129,7 +3097,7 @@ static void ggml_vk_print_matrix_area(const void * data, ggml_type type, int ne0 template static void ggml_vk_test_matmul(size_t m, size_t n, size_t k, size_t batch, size_t num_it, int split_k, int shader_size) { -#ifdef VK_DEBUG +#ifdef GGML_VULKAN_DEBUG std::cerr << "ggml_vk_test_matmul(" << m << ", " << n << ", " << k << ", " << batch << ", " << num_it << ", " << split_k << ", " << shader_size << ")" << std::endl; #endif const size_t x_ne = m * k * batch; @@ -3520,7 +3488,7 @@ static void ggml_vk_test_h2d_nc(size_t ne0, size_t ne1, size_t ne2, size_t ne3) } static void ggml_vk_test_transfer(size_t ne, bool pinned) { -#ifdef VK_DEBUG +#ifdef GGML_VULKAN_DEBUG std::cerr << "ggml_vk_test_transfer(" << ne << ")" << std::endl; #endif // Check transfers are correct @@ -3600,10 +3568,103 @@ static void ggml_vk_test_transfer(size_t ne, bool pinned) { free(y); } } + +static void ggml_vk_test_dequant(size_t ne, ggml_type quant) { +#ifdef GGML_VULKAN_DEBUG + std::cerr << "ggml_vk_test_dequant(" << ne << ")" << std::endl; +#endif + const size_t x_sz = sizeof(float) * ne; + const size_t x_sz_f16 = sizeof(ggml_fp16_t) * ne; + const size_t qx_sz = ne * ggml_type_size(quant)/ggml_blck_size(quant); + float * x = (float *) malloc(x_sz); + void * qx = malloc(qx_sz); + vk_buffer qx_buf = ggml_vk_create_buffer_check(qx_sz, vk::MemoryPropertyFlagBits::eDeviceLocal); + vk_buffer x_buf = ggml_vk_create_buffer_check(x_sz_f16, vk::MemoryPropertyFlagBits::eDeviceLocal); + ggml_fp16_t * x_chk = (ggml_fp16_t *) malloc(x_sz_f16); + + for (size_t i = 0; i < ne; i++) { + x[i] = rand() / (float)RAND_MAX; + } + + std::vector hist_cur(1 << 4, 0); + + vk_pipeline& p = vk_pipeline_dequant[quant]; + + switch(quant) { + case GGML_TYPE_Q4_0: + ggml_quantize_q4_0(x, qx, ne, ne, hist_cur.data()); + break; + case GGML_TYPE_Q4_1: + ggml_quantize_q4_1(x, qx, ne, ne, hist_cur.data()); + break; + case GGML_TYPE_Q5_0: + ggml_quantize_q5_0(x, qx, ne, ne, hist_cur.data()); + break; + case GGML_TYPE_Q5_1: + ggml_quantize_q4_1(x, qx, ne, ne, hist_cur.data()); + break; + case GGML_TYPE_Q8_0: + ggml_quantize_q8_0(x, qx, ne, ne, hist_cur.data()); + break; + case GGML_TYPE_Q2_K: + ggml_quantize_q2_K(x, qx, ne, ne, hist_cur.data()); + break; + case GGML_TYPE_Q3_K: + ggml_quantize_q3_K(x, qx, ne, ne, hist_cur.data()); + break; + case GGML_TYPE_Q4_K: + ggml_quantize_q4_K(x, qx, ne, ne, hist_cur.data()); + break; + case GGML_TYPE_Q5_K: + ggml_quantize_q5_K(x, qx, ne, ne, hist_cur.data()); + break; + case GGML_TYPE_Q6_K: + ggml_quantize_q6_K(x, qx, ne, ne, hist_cur.data()); + break; + default: + GGML_ASSERT(false); + } + + ggml_vk_pipeline_allocate_descriptor_sets(p, 1); + + ggml_vk_buffer_write(&qx_buf, 0, qx, qx_sz); + + vk_context * ctx = ggml_vk_create_context(vk_device.compute_queue); + ggml_vk_ctx_begin(ctx); + const std::vector pc = { 1, (int)ne, (int)ne, (int)ne }; + ggml_vk_sync_buffers(ctx); + ggml_vk_dispatch_pipeline(ctx, p, { { qx_buf, 0, qx_sz }, { x_buf, 0, x_sz_f16 } }, pc.size() * sizeof(int), pc.data(), { (uint32_t)ne, 1, 1}); + ggml_vk_ctx_end(ctx); + + auto begin = std::chrono::high_resolution_clock::now(); + + ggml_vk_submit(ctx, vk_fence); + VK_CHECK(vk_device.device.waitForFences({ vk_fence }, true, UINT64_MAX), "ggml_vk_compute_forward waitForFences"); + vk_device.device.resetFences({ vk_fence }); + + auto end = std::chrono::high_resolution_clock::now(); + + double ms_dequant = std::chrono::duration_cast(end-begin).count() / 1000.0; + ggml_vk_buffer_read(&x_buf, 0, x_chk, x_sz_f16); + + double avg_err = 0.0; + for (size_t i = 0; i < ne; i++) { + avg_err += std::fabs(x[i] - ggml_fp16_to_fp32(x_chk[i])); + } + + std::cerr << "TEST DEQUANT " << ggml_type_name(quant) << " time=" << ms_dequant << "ms avg_err=" << avg_err / ne << std::endl; + + ggml_vk_destroy_buffer(x_buf); + ggml_vk_destroy_buffer(qx_buf); + + free(x); + free(qx); + free(x_chk); +} #endif static ggml_tensor_extra_gpu * ggml_vk_tensor_create_extra(ggml_tensor * tensor) { -#ifdef VK_DEBUG +#ifdef GGML_VULKAN_DEBUG std::cerr << "ggml_vk_create_extra(" << tensor << " (" << tensor->name << ", " << ggml_op_name(tensor->op) << "))" << std::endl; #endif ggml_tensor_extra_gpu * extra = new ggml_tensor_extra_gpu; @@ -3627,7 +3688,7 @@ static ggml_tensor * ggml_vk_find_last_use(const ggml_tensor * node, ggml_cgraph } void ggml_vk_preallocate_buffers_graph(ggml_tensor * node){ -#ifdef VK_DEBUG +#ifdef GGML_VULKAN_DEBUG std::cerr << "ggml_vk_preallocate_buffers_graph(" << node << ")" << std::endl; #endif const bool any_on_device = node->backend == GGML_BACKEND_GPU @@ -3746,15 +3807,26 @@ void ggml_vk_preallocate_buffers() { if (vk_disable) { return; } -#ifdef VK_DEBUG +#ifdef GGML_VULKAN_DEBUG std::cerr << "ggml_vk_preallocate_buffers()" << std::endl; std::cerr << "qx_size: " << vk_prealloc_size_qx << " qy_size: " << vk_prealloc_size_qy << " x_size: " << vk_prealloc_size_x << " y_size: " << vk_prealloc_size_y << " split_k_size: " << vk_prealloc_size_split_k << std::endl; #endif -#if defined(VK_RUN_TESTS) +#if defined(GGML_VULKAN_RUN_TESTS) vk_staging = ggml_vk_create_buffer_check(100ul * 1024ul * 1024ul, vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent | vk::MemoryPropertyFlagBits::eHostCached); ggml_vk_test_transfer(8192 * 1000, false); ggml_vk_test_transfer(8192 * 1000, true); + ggml_vk_test_dequant(2560 * 7680, GGML_TYPE_Q4_0); + ggml_vk_test_dequant(2560 * 7680, GGML_TYPE_Q4_1); + ggml_vk_test_dequant(2560 * 7680, GGML_TYPE_Q5_0); + ggml_vk_test_dequant(2560 * 7680, GGML_TYPE_Q5_1); + ggml_vk_test_dequant(2560 * 7680, GGML_TYPE_Q8_0); + ggml_vk_test_dequant(2560 * 7680, GGML_TYPE_Q2_K); + ggml_vk_test_dequant(2560 * 7680, GGML_TYPE_Q3_K); + ggml_vk_test_dequant(2560 * 7680, GGML_TYPE_Q4_K); + ggml_vk_test_dequant(2560 * 7680, GGML_TYPE_Q5_K); + ggml_vk_test_dequant(2560 * 7680, GGML_TYPE_Q6_K); + const std::vector vals { 8, 8, 8, 100, 46, 576, @@ -3845,7 +3917,7 @@ void ggml_vk_build_graph(ggml_tensor * node, bool last_node){ return; } -#ifdef VK_DEBUG +#ifdef GGML_VULKAN_DEBUG std::cerr << "ggml_vk_build_graph(" << node << ", " << ggml_op_name(node->op) << ")" << std::endl; #endif vk_semaphore_idx = 0; @@ -4068,7 +4140,7 @@ bool ggml_vk_compute_forward(ggml_compute_params * params, ggml_tensor * tensor) return true; } -#ifdef VK_DEBUG +#ifdef GGML_VULKAN_DEBUG std::cerr << "ggml_vk_compute_forward(" << tensor << ", name=" << tensor->name << ", op=" << ggml_op_name(tensor->op) << ", type=" << tensor->type << ", backend=" << tensor->backend << ", ne0=" << tensor->ne[0] << ", ne1=" << tensor->ne[1] << ", ne2=" << tensor->ne[2] << ", ne3=" << tensor->ne[3] << ", nb0=" << tensor->nb[0] << ", nb1=" << tensor->nb[1] << ", nb2=" << tensor->nb[2] << ", nb3=" << tensor->nb[3] << ", view_src=" << tensor->view_src << ", view_offs=" << tensor->view_offs << ")" << std::endl; #endif @@ -4111,7 +4183,7 @@ void ggml_vk_graph_cleanup() { if (vk_disable) { return; } -#ifdef VK_DEBUG +#ifdef GGML_VULKAN_DEBUG std::cerr << "ggml_vk_graph_cleanup()" << std::endl; #endif for (auto& buffer : vk_gc.temp_buffers) { @@ -4150,7 +4222,7 @@ void ggml_vk_graph_cleanup() { } static void ggml_vk_cleanup() { -#ifdef VK_DEBUG +#ifdef GGML_VULKAN_DEBUG std::cerr << "ggml_vk_cleanup()" << std::endl; #endif ggml_vk_destroy_buffer(vk_prealloc_x); @@ -4234,7 +4306,7 @@ GGML_CALL static void * ggml_backend_vk_buffer_get_base(ggml_backend_buffer_t bu } GGML_CALL static void ggml_backend_vk_buffer_init_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor) { -#ifdef VK_DEBUG +#ifdef GGML_VULKAN_DEBUG std::cerr << "ggml_backend_vk_buffer_init_tensor(" << buffer << " (" << buffer->context << "), " << tensor << ")" << std::endl; #endif ggml_backend_vk_buffer_context * ctx = (ggml_backend_vk_buffer_context *)buffer->context; @@ -4254,7 +4326,7 @@ GGML_CALL static void ggml_backend_vk_buffer_init_tensor(ggml_backend_buffer_t b } GGML_CALL static void ggml_backend_vk_buffer_set_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor, const void * data, size_t offset, size_t size) { -#ifdef VK_DEBUG +#ifdef GGML_VULKAN_DEBUG std::cerr << "ggml_backend_vk_buffer_set_tensor(" << buffer << ", " << tensor << ", " << data << ", " << offset << ", " << size << ")" << std::endl; #endif GGML_ASSERT(tensor->backend == GGML_BACKEND_GPU); @@ -4267,7 +4339,7 @@ GGML_CALL static void ggml_backend_vk_buffer_set_tensor(ggml_backend_buffer_t bu } GGML_CALL static void ggml_backend_vk_buffer_get_tensor(ggml_backend_buffer_t buffer, const ggml_tensor * tensor, void * data, size_t offset, size_t size) { -#ifdef VK_DEBUG +#ifdef GGML_VULKAN_DEBUG std::cerr << "ggml_backend_vk_buffer_get_tensor(" << buffer << ", " << tensor << ", " << data << ", " << offset << ", " << size << ")" << std::endl; #endif GGML_ASSERT(tensor->backend == GGML_BACKEND_GPU); @@ -4323,7 +4395,7 @@ GGML_CALL static const char * ggml_backend_vk_buffer_type_name(ggml_backend_buff } GGML_CALL static ggml_backend_buffer_t ggml_backend_vk_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size) { -#ifdef VK_DEBUG +#ifdef GGML_VULKAN_DEBUG std::cerr << "ggml_backend_vk_buffer_type_alloc_buffer(" << size << ")" << std::endl; #endif vk_buffer dev_buffer = ggml_vk_create_buffer_device(size); @@ -4467,7 +4539,7 @@ GGML_CALL static ggml_backend_buffer_type_t ggml_backend_vk_get_default_buffer_t } GGML_CALL static void ggml_backend_vk_set_tensor_async(ggml_backend_t backend, ggml_tensor * tensor, const void * data, size_t offset, size_t size) { -#ifdef VK_DEBUG +#ifdef GGML_VULKAN_DEBUG std::cerr << "ggml_backend_vk_set_tensor_async(" << size << ")" << std::endl; #endif GGML_ASSERT((tensor->buffer->buft == ggml_backend_vk_buffer_type() || tensor->buffer->buft == ggml_backend_vk_host_buffer_type()) && "unsupported buffer type"); @@ -4475,19 +4547,19 @@ GGML_CALL static void ggml_backend_vk_set_tensor_async(ggml_backend_t backend, g ggml_tensor_extra_gpu * extra = (ggml_tensor_extra_gpu *) tensor->extra; - if (vk_ctx == nullptr) { + if (vk_transfer_ctx == nullptr) { // Initialize new transfer context - vk_ctx = ggml_vk_create_context(vk_device.transfer_queue); - ggml_vk_ctx_begin(vk_ctx); + vk_transfer_ctx = ggml_vk_create_context(vk_device.transfer_queue); + ggml_vk_ctx_begin(vk_transfer_ctx); } - ggml_vk_buffer_write_async(vk_ctx, &extra->buffer_gpu, extra->offset + offset, data, size); + ggml_vk_buffer_write_async(vk_transfer_ctx, &extra->buffer_gpu, extra->offset + offset, data, size); UNUSED(backend); } GGML_CALL static void ggml_backend_vk_get_tensor_async(ggml_backend_t backend, const ggml_tensor * tensor, void * data, size_t offset, size_t size) { -#ifdef VK_DEBUG +#ifdef GGML_VULKAN_DEBUG std::cerr << "ggml_backend_vk_get_tensor_async(" << size << ")" << std::endl; #endif GGML_ASSERT((tensor->buffer->buft == ggml_backend_vk_buffer_type() || tensor->buffer->buft == ggml_backend_vk_host_buffer_type()) && "unsupported buffer type"); @@ -4495,32 +4567,32 @@ GGML_CALL static void ggml_backend_vk_get_tensor_async(ggml_backend_t backend, c ggml_tensor_extra_gpu * extra = (ggml_tensor_extra_gpu *) tensor->extra; - if (vk_ctx == nullptr) { + if (vk_transfer_ctx == nullptr) { // Initialize new transfer context - vk_ctx = ggml_vk_create_context(vk_device.transfer_queue); - ggml_vk_ctx_begin(vk_ctx); + vk_transfer_ctx = ggml_vk_create_context(vk_device.transfer_queue); + ggml_vk_ctx_begin(vk_transfer_ctx); } - ggml_vk_buffer_read_async(vk_ctx, &extra->buffer_gpu, extra->offset + offset, data, size); + ggml_vk_buffer_read_async(vk_transfer_ctx, &extra->buffer_gpu, extra->offset + offset, data, size); UNUSED(backend); } GGML_CALL static bool ggml_backend_vk_cpy_tensor_async(ggml_backend_t backend, const ggml_tensor * src, ggml_tensor * dst) { -#ifdef VK_DEBUG +#ifdef GGML_VULKAN_DEBUG std::cerr << "ggml_backend_vk_cpy_tensor_async()" << std::endl; #endif if ((dst->buffer->buft == ggml_backend_vk_buffer_type() || dst->buffer->buft == ggml_backend_vk_host_buffer_type()) && ggml_backend_buffer_is_vk(src->buffer)) { ggml_tensor_extra_gpu * src_extra = (ggml_tensor_extra_gpu *) src->extra; ggml_tensor_extra_gpu * dst_extra = (ggml_tensor_extra_gpu *) dst->extra; - if (vk_ctx == nullptr) { + if (vk_transfer_ctx == nullptr) { // Initialize new transfer context - vk_ctx = ggml_vk_create_context(vk_device.transfer_queue); - ggml_vk_ctx_begin(vk_ctx); + vk_transfer_ctx = ggml_vk_create_context(vk_device.transfer_queue); + ggml_vk_ctx_begin(vk_transfer_ctx); } - ggml_vk_buffer_copy_async(vk_ctx, &src_extra->buffer_gpu, src_extra->offset, &dst_extra->buffer_gpu, dst_extra->offset, ggml_nbytes(src)); + ggml_vk_buffer_copy_async(vk_transfer_ctx, &src_extra->buffer_gpu, src_extra->offset, &dst_extra->buffer_gpu, dst_extra->offset, ggml_nbytes(src)); return true; } @@ -4530,28 +4602,28 @@ GGML_CALL static bool ggml_backend_vk_cpy_tensor_async(ggml_backend_t backend, c } GGML_CALL static void ggml_backend_vk_synchronize(ggml_backend_t backend) { -#ifdef VK_DEBUG +#ifdef GGML_VULKAN_DEBUG std::cerr << "ggml_backend_vk_synchronize()" << std::endl; #endif - if(vk_ctx == nullptr) { + if(vk_transfer_ctx == nullptr) { return; } - ggml_vk_ctx_end(vk_ctx); + ggml_vk_ctx_end(vk_transfer_ctx); - for (auto& cpy : vk_ctx->in_memcpys) { + for (auto& cpy : vk_transfer_ctx->in_memcpys) { memcpy(cpy.dst, cpy.src, cpy.n); } - ggml_vk_submit(vk_ctx, vk_fence); + ggml_vk_submit(vk_transfer_ctx, vk_fence); VK_CHECK(vk_device.device.waitForFences({ vk_fence }, true, UINT64_MAX), "ggml_backend_vk_synchronize waitForFences"); vk_device.device.resetFences({ vk_fence }); - for (auto& cpy : vk_ctx->out_memcpys) { + for (auto& cpy : vk_transfer_ctx->out_memcpys) { memcpy(cpy.dst, cpy.src, cpy.n); } - vk_ctx = nullptr; + vk_transfer_ctx = nullptr; UNUSED(backend); } diff --git a/ggml_vk_generate_shaders.py b/ggml_vk_generate_shaders.py index 67981a751..4abb0383f 100644 --- a/ggml_vk_generate_shaders.py +++ b/ggml_vk_generate_shaders.py @@ -157,19 +157,10 @@ struct block_q6_K # Dequant functions shader_f16_dequant_func = """ -#define DEQUANT_FUNC f16vec2 v = f16vec2(data_a[ib + 0], data_a[ib + 1]); -""" -shader_f16_dequant_func_compat = """ #define DEQUANT_FUNC vec2 v = vec2(data_a[ib + 0], data_a[ib + 1]); """ shader_q4_0_dequant_func = """ -#define DEQUANT_FUNC const float16_t d = data_a[ib].d; \ -const uint8_t vui = data_a[ib].qs[iqs]; \ -f16vec2 v = f16vec2(vui & 0xF, vui >> 4); \ -v = (v - 8.0hf)*d; -""" -shader_q4_0_dequant_func_compat = """ #define DEQUANT_FUNC const float d = float(data_a[ib].d); \ const uint vui = uint(data_a[ib].qs[iqs]); \ vec2 v = vec2(vui & 0xF, vui >> 4); \ @@ -177,13 +168,6 @@ v = (v - 8.0f)*d; """ shader_q4_1_dequant_func = """ -#define DEQUANT_FUNC const float16_t d = data_a[ib].d; \ -const float16_t m = data_a[ib].m; \ -const uint8_t vui = data_a[ib].qs[iqs]; \ -f16vec2 v = f16vec2(vui & 0xF, vui >> 4); \ -v = v*d + m; -""" -shader_q4_1_dequant_func_compat = """ #define DEQUANT_FUNC const float d = float(data_a[ib].d); \ const float m = float(data_a[ib].m); \ const uint vui = uint(data_a[ib].qs[iqs]); \ @@ -192,14 +176,6 @@ v = v*d + m; """ shader_q5_0_dequant_func = """ -#define DEQUANT_FUNC const float16_t d = data_a[ib].d; \ -const uint uint_qh = uint(data_a[ib].qh[1]) << 16 | data_a[ib].qh[0]; \ -const ivec2 qh = ivec2(((uint_qh >> iqs) << 4) & 0x10, (uint_qh >> (iqs + 12)) & 0x10); \ -const uint8_t vui = data_a[ib].qs[iqs]; \ -f16vec2 v = f16vec2((vui & 0xF) | qh.x, (vui >> 4) | qh.y); \ -v = (v - 16.0hf) * d; -""" -shader_q5_0_dequant_func_compat = """ #define DEQUANT_FUNC const float d = float(data_a[ib].d); \ const uint uint_qh = uint(data_a[ib].qh[1]) << 16 | data_a[ib].qh[0]; \ const ivec2 qh = ivec2(((uint_qh >> iqs) << 4) & 0x10, (uint_qh >> (iqs + 12)) & 0x10); \ @@ -209,14 +185,6 @@ v = (v - 16.0f) * d; """ shader_q5_1_dequant_func = """ -#define DEQUANT_FUNC const float16_t d = data_a[ib].d; \ -const float16_t m = data_a[ib].m; \ -const ivec2 qh = ivec2(((data_a[ib].qh >> iqs) << 4) & 0x10, (data_a[ib].qh >> (iqs + 12)) & 0x10); \ -const uint8_t vui = data_a[ib].qs[iqs]; \ -f16vec2 v = f16vec2((vui & 0xF) | qh.x, (vui >> 4) | qh.y); \ -v = v*d + m; -""" -shader_q5_1_dequant_func_compat = """ #define DEQUANT_FUNC const float d = float(data_a[ib].d); \ const float m = float(data_a[ib].m); \ const ivec2 qh = ivec2(((data_a[ib].qh >> iqs) << 4) & 0x10, (data_a[ib].qh >> (iqs + 12)) & 0x10); \ @@ -226,11 +194,6 @@ v = v*d + m; """ shader_q8_0_dequant_func = """ -#define DEQUANT_FUNC const float16_t d = data_a[ib].d; \ -f16vec2 v = f16vec2(data_a[ib].qs[iqs], data_a[ib].qs[iqs + 1]); \ -v = v * d; -""" -shader_q8_0_dequant_func_compat = """ #define DEQUANT_FUNC const float d = float(data_a[ib].d); \ vec2 v = vec2(int(data_a[ib].qs[iqs]), int(data_a[ib].qs[iqs + 1])); \ v = v * d; @@ -2110,7 +2073,7 @@ lock = asyncio.Lock() shader_fnames = [] -async def string_to_spv(name, code, defines, fp16): +async def string_to_spv(name, code, defines, fp16=True): f = NamedTemporaryFile(mode="w", delete=False) f.write(code) f.flush() @@ -2200,64 +2163,6 @@ async def main(): tasks.append(string_to_spv("matmul_f16_f32_aligned_m", "".join(stream), {"LOAD_VEC": load_vec, "A_TYPE": vec_type_f16, "B_TYPE": vec_type, "D_TYPE": "float"}, fp16)) tasks.append(string_to_spv("matmul_f16_f32_aligned_s", "".join(stream), {"LOAD_VEC": load_vec, "A_TYPE": vec_type_f16, "B_TYPE": vec_type, "D_TYPE": "float"}, fp16)) - # Build dequant shaders - tasks.append(string_to_spv("f32_to_f16", f32_to_f16_src, {}, fp16)) - - for i in range(0, VK_NUM_TYPES): - stream.clear() - - stream.extend((dequant_head, shader_int8_ext, shader_float_type)) - - if i == GGML_TYPE_F16: - stream.extend((shader_f16_defines, shader_f16_dequant_func_compat if not fp16 else shader_f16_dequant_func, dequant_body)) - elif i == GGML_TYPE_Q4_0: - stream.extend((shader_q4_0_defines, shader_q4_0_dequant_func_compat if not fp16 else shader_q4_0_dequant_func, dequant_body)) - elif i == GGML_TYPE_Q4_1: - stream.extend((shader_q4_1_defines, shader_q4_1_dequant_func_compat if not fp16 else shader_q4_1_dequant_func, dequant_body)) - elif i == GGML_TYPE_Q5_0: - stream.extend((shader_q5_0_defines, shader_q5_0_dequant_func_compat if not fp16 else shader_q5_0_dequant_func, dequant_body)) - elif i == GGML_TYPE_Q5_1: - stream.extend((shader_q5_1_defines, shader_q5_1_dequant_func_compat if not fp16 else shader_q5_1_dequant_func, dequant_body)) - elif i == GGML_TYPE_Q8_0: - stream.extend((shader_q8_0_defines, shader_q8_0_dequant_func_compat if not fp16 else shader_q8_0_dequant_func, dequant_body)) - elif i == GGML_TYPE_Q2_K: - stream.extend((shader_q2_K_defines, dequant_q2_K_body)) - elif i == GGML_TYPE_Q3_K: - stream.extend((shader_q3_K_defines, dequant_q3_K_body)) - elif i == GGML_TYPE_Q4_K: - stream.extend((shader_q4_K_defines, dequant_q4_K_body)) - elif i == GGML_TYPE_Q5_K: - stream.extend((shader_q5_K_defines, dequant_q5_K_body)) - elif i == GGML_TYPE_Q6_K: - stream.extend((shader_q6_K_defines, dequant_q6_K_body)) - else: - continue - - tasks.append(string_to_spv(f"dequant_{type_names[i]}", "".join(stream), {"D_TYPE": "float16_t"}, fp16)) - - # get_rows - for i in range(0, VK_NUM_TYPES): - stream.clear() - stream.extend((generic_head, shader_int8_ext, shader_float_type)) - - if i == GGML_TYPE_F16: - stream.extend((shader_f16_defines, shader_f16_dequant_func_compat if not fp16 else shader_f16_dequant_func, get_rows_body)) - elif i == GGML_TYPE_Q4_0: - stream.extend((shader_q4_0_defines, shader_q4_0_dequant_func_compat if not fp16 else shader_q4_0_dequant_func, get_rows_body)) - elif i == GGML_TYPE_Q4_1: - stream.extend((shader_q4_1_defines, shader_q4_1_dequant_func_compat if not fp16 else shader_q4_1_dequant_func, get_rows_body)) - elif i == GGML_TYPE_Q5_0: - stream.extend((shader_q5_0_defines, shader_q5_0_dequant_func_compat if not fp16 else shader_q5_0_dequant_func, get_rows_body)) - elif i == GGML_TYPE_Q5_1: - stream.extend((shader_q5_1_defines, shader_q5_1_dequant_func_compat if not fp16 else shader_q5_1_dequant_func, get_rows_body)) - elif i == GGML_TYPE_Q8_0: - stream.extend((shader_q8_0_defines, shader_q8_0_dequant_func_compat if not fp16 else shader_q8_0_dequant_func, get_rows_body)) - else: - continue - - tasks.append(string_to_spv(f"get_rows_{type_names[i]}", "".join(stream), {"B_TYPE": "float", "D_TYPE": "float16_t"}, fp16)) - tasks.append(string_to_spv(f"get_rows_{type_names[i]}_f32", "".join(stream), {"B_TYPE": "float", "D_TYPE": "float"}, fp16)) - # Shaders where precision is needed, so no fp16 version # mul mat vec @@ -2266,17 +2171,17 @@ async def main(): stream.extend((mul_mat_vec_head, shader_int8_ext, shader_f32)) if i == GGML_TYPE_F16: - stream.extend((shader_f16_defines, shader_f16_dequant_func_compat, mul_mat_vec_body)) + stream.extend((shader_f16_defines, shader_f16_dequant_func, mul_mat_vec_body)) elif i == GGML_TYPE_Q4_0: - stream.extend((shader_q4_0_defines, shader_q4_0_dequant_func_compat, mul_mat_vec_body)) + stream.extend((shader_q4_0_defines, shader_q4_0_dequant_func, mul_mat_vec_body)) elif i == GGML_TYPE_Q4_1: - stream.extend((shader_q4_1_defines, shader_q4_1_dequant_func_compat, mul_mat_vec_body)) + stream.extend((shader_q4_1_defines, shader_q4_1_dequant_func, mul_mat_vec_body)) elif i == GGML_TYPE_Q5_0: - stream.extend((shader_q5_0_defines, shader_q5_0_dequant_func_compat, mul_mat_vec_body)) + stream.extend((shader_q5_0_defines, shader_q5_0_dequant_func, mul_mat_vec_body)) elif i == GGML_TYPE_Q5_1: - stream.extend((shader_q5_1_defines, shader_q5_1_dequant_func_compat, mul_mat_vec_body)) + stream.extend((shader_q5_1_defines, shader_q5_1_dequant_func, mul_mat_vec_body)) elif i == GGML_TYPE_Q8_0: - stream.extend((shader_q8_0_defines, shader_q8_0_dequant_func_compat, mul_mat_vec_body)) + stream.extend((shader_q8_0_defines, shader_q8_0_dequant_func, mul_mat_vec_body)) elif i == GGML_TYPE_Q2_K: stream.extend((shader_q2_K_defines, mul_mat_vec_q2_K_body)) elif i == GGML_TYPE_Q3_K: @@ -2290,43 +2195,101 @@ async def main(): else: continue - tasks.append(string_to_spv(f"mul_mat_vec_{type_names[i]}_f32", "".join(stream), {"B_TYPE": "float", "D_TYPE": "float", "K_QUANTS_PER_ITERATION": K_QUANTS_PER_ITERATION}, fp16)) + tasks.append(string_to_spv(f"mul_mat_vec_{type_names[i]}_f32", "".join(stream), {"B_TYPE": "float", "D_TYPE": "float", "K_QUANTS_PER_ITERATION": K_QUANTS_PER_ITERATION})) - tasks.append(string_to_spv("mul_mat_vec_p021_f16_f32", mul_mat_p021_src, {"A_TYPE": "float16_t", "B_TYPE": "float", "D_TYPE": "float"}, True)) - tasks.append(string_to_spv("mul_mat_vec_nc_f16_f32", mul_mat_nc_src, {"A_TYPE": "float16_t", "B_TYPE": "float", "D_TYPE": "float"}, True)) + # Dequant shaders + for i in range(0, VK_NUM_TYPES): + stream.clear() + + stream.extend((dequant_head, shader_int8_ext, shader_f32)) + + if i == GGML_TYPE_F16: + stream.extend((shader_f16_defines, shader_f16_dequant_func, dequant_body)) + elif i == GGML_TYPE_Q4_0: + stream.extend((shader_q4_0_defines, shader_q4_0_dequant_func, dequant_body)) + elif i == GGML_TYPE_Q4_1: + stream.extend((shader_q4_1_defines, shader_q4_1_dequant_func, dequant_body)) + elif i == GGML_TYPE_Q5_0: + stream.extend((shader_q5_0_defines, shader_q5_0_dequant_func, dequant_body)) + elif i == GGML_TYPE_Q5_1: + stream.extend((shader_q5_1_defines, shader_q5_1_dequant_func, dequant_body)) + elif i == GGML_TYPE_Q8_0: + stream.extend((shader_q8_0_defines, shader_q8_0_dequant_func, dequant_body)) + elif i == GGML_TYPE_Q2_K: + stream.extend((shader_q2_K_defines, dequant_q2_K_body)) + elif i == GGML_TYPE_Q3_K: + stream.extend((shader_q3_K_defines, dequant_q3_K_body)) + elif i == GGML_TYPE_Q4_K: + stream.extend((shader_q4_K_defines, dequant_q4_K_body)) + elif i == GGML_TYPE_Q5_K: + stream.extend((shader_q5_K_defines, dequant_q5_K_body)) + elif i == GGML_TYPE_Q6_K: + stream.extend((shader_q6_K_defines, dequant_q6_K_body)) + else: + continue + + tasks.append(string_to_spv(f"dequant_{type_names[i]}", "".join(stream), {"D_TYPE": "float16_t"})) + + tasks.append(string_to_spv("f32_to_f16", f32_to_f16_src, {})) + + # get_rows + for i in range(0, VK_NUM_TYPES): + stream.clear() + stream.extend((generic_head, shader_int8_ext, shader_f32)) + + if i == GGML_TYPE_F16: + stream.extend((shader_f16_defines, shader_f16_dequant_func, get_rows_body)) + elif i == GGML_TYPE_Q4_0: + stream.extend((shader_q4_0_defines, shader_q4_0_dequant_func, get_rows_body)) + elif i == GGML_TYPE_Q4_1: + stream.extend((shader_q4_1_defines, shader_q4_1_dequant_func, get_rows_body)) + elif i == GGML_TYPE_Q5_0: + stream.extend((shader_q5_0_defines, shader_q5_0_dequant_func, get_rows_body)) + elif i == GGML_TYPE_Q5_1: + stream.extend((shader_q5_1_defines, shader_q5_1_dequant_func, get_rows_body)) + elif i == GGML_TYPE_Q8_0: + stream.extend((shader_q8_0_defines, shader_q8_0_dequant_func, get_rows_body)) + else: + continue + + tasks.append(string_to_spv(f"get_rows_{type_names[i]}", "".join(stream), {"B_TYPE": "float", "D_TYPE": "float16_t"})) + tasks.append(string_to_spv(f"get_rows_{type_names[i]}_f32", "".join(stream), {"B_TYPE": "float", "D_TYPE": "float"})) + + tasks.append(string_to_spv("mul_mat_vec_p021_f16_f32", mul_mat_p021_src, {"A_TYPE": "float16_t", "B_TYPE": "float", "D_TYPE": "float"})) + tasks.append(string_to_spv("mul_mat_vec_nc_f16_f32", mul_mat_nc_src, {"A_TYPE": "float16_t", "B_TYPE": "float", "D_TYPE": "float"})) # Norms - tasks.append(string_to_spv("norm_f32", f"{generic_head}\n{shader_f32}\n{norm_body}", {"A_TYPE": "float", "D_TYPE": "float"}, True)) - tasks.append(string_to_spv("rms_norm_f32", f"{generic_head}\n{shader_f32}\n{rms_norm_body}", {"A_TYPE": "float", "D_TYPE": "float"}, True)) + tasks.append(string_to_spv("norm_f32", f"{generic_head}\n{shader_f32}\n{norm_body}", {"A_TYPE": "float", "D_TYPE": "float"})) + tasks.append(string_to_spv("rms_norm_f32", f"{generic_head}\n{shader_f32}\n{rms_norm_body}", {"A_TYPE": "float", "D_TYPE": "float"})) - tasks.append(string_to_spv("cpy_f32_f32", f"{cpy_src}\n{cpy_end}", {"A_TYPE": "float", "D_TYPE": "float"}, True)) - tasks.append(string_to_spv("cpy_f32_f16", f"{cpy_src}\n{cpy_end}", {"A_TYPE": "float", "D_TYPE": "float16_t"}, True)) - tasks.append(string_to_spv("cpy_f16_f16", f"{cpy_src}\n{cpy_f16_f16_end}", {"A_TYPE": "float16_t", "D_TYPE": "float16_t"}, True)) + tasks.append(string_to_spv("cpy_f32_f32", f"{cpy_src}\n{cpy_end}", {"A_TYPE": "float", "D_TYPE": "float"})) + tasks.append(string_to_spv("cpy_f32_f16", f"{cpy_src}\n{cpy_end}", {"A_TYPE": "float", "D_TYPE": "float16_t"})) + tasks.append(string_to_spv("cpy_f16_f16", f"{cpy_src}\n{cpy_f16_f16_end}", {"A_TYPE": "float16_t", "D_TYPE": "float16_t"})) - tasks.append(string_to_spv("add_f32", f"{generic_head}\n{shader_f32}\n{add_body}", {"A_TYPE": "float", "B_TYPE": "float", "D_TYPE": "float"}, True)) + tasks.append(string_to_spv("add_f32", f"{generic_head}\n{shader_f32}\n{add_body}", {"A_TYPE": "float", "B_TYPE": "float", "D_TYPE": "float"})) - tasks.append(string_to_spv("split_k_reduce", mulmat_split_k_reduce_src, {}, True)) - tasks.append(string_to_spv("mul_f32", f"{generic_head}\n{shader_f32}\n{mul_body}", {"A_TYPE": "float", "B_TYPE": "float", "D_TYPE": "float"}, True)) + tasks.append(string_to_spv("split_k_reduce", mulmat_split_k_reduce_src, {})) + tasks.append(string_to_spv("mul_f32", f"{generic_head}\n{shader_f32}\n{mul_body}", {"A_TYPE": "float", "B_TYPE": "float", "D_TYPE": "float"})) - tasks.append(string_to_spv("scale_f32", f"{generic_head}\n{shader_f32}\n{scale_body}", {"A_TYPE": "float", "D_TYPE": "float"}, True)) + tasks.append(string_to_spv("scale_f32", f"{generic_head}\n{shader_f32}\n{scale_body}", {"A_TYPE": "float", "D_TYPE": "float"})) - tasks.append(string_to_spv("sqr_f32", f"{generic_head}\n{shader_f32}\n{sqr_body}", {"A_TYPE": "float", "D_TYPE": "float"}, True)) + tasks.append(string_to_spv("sqr_f32", f"{generic_head}\n{shader_f32}\n{sqr_body}", {"A_TYPE": "float", "D_TYPE": "float"})) - tasks.append(string_to_spv("clamp_f32", f"{generic_head}\n{shader_f32}\n{clamp_body}", {"A_TYPE": "float", "D_TYPE": "float"}, True)) + tasks.append(string_to_spv("clamp_f32", f"{generic_head}\n{shader_f32}\n{clamp_body}", {"A_TYPE": "float", "D_TYPE": "float"})) - tasks.append(string_to_spv("gelu_f32", f"{generic_head}\n{shader_f32}\n{gelu_body}", {"A_TYPE": "float", "D_TYPE": "float"}, True)) - tasks.append(string_to_spv("silu_f32", f"{generic_head}\n{shader_f32}\n{silu_body}", {"A_TYPE": "float", "D_TYPE": "float"}, True)) - tasks.append(string_to_spv("relu_f32", f"{generic_head}\n{shader_f32}\n{relu_body}", {"A_TYPE": "float", "D_TYPE": "float"}, True)) + tasks.append(string_to_spv("gelu_f32", f"{generic_head}\n{shader_f32}\n{gelu_body}", {"A_TYPE": "float", "D_TYPE": "float"})) + tasks.append(string_to_spv("silu_f32", f"{generic_head}\n{shader_f32}\n{silu_body}", {"A_TYPE": "float", "D_TYPE": "float"})) + tasks.append(string_to_spv("relu_f32", f"{generic_head}\n{shader_f32}\n{relu_body}", {"A_TYPE": "float", "D_TYPE": "float"})) - tasks.append(string_to_spv("diag_mask_inf_f32", f"{diag_mask_inf_head}\n{shader_f32}\n{diag_mask_inf_body}", {"A_TYPE": "float", "D_TYPE": "float"}, True)) + tasks.append(string_to_spv("diag_mask_inf_f32", f"{diag_mask_inf_head}\n{shader_f32}\n{diag_mask_inf_body}", {"A_TYPE": "float", "D_TYPE": "float"})) - tasks.append(string_to_spv("soft_max_f32", f"{generic_head}\n{shader_f32}\n{soft_max_body}", {"A_TYPE": "float", "B_TYPE": "float", "D_TYPE": "float"}, True)) + tasks.append(string_to_spv("soft_max_f32", f"{generic_head}\n{shader_f32}\n{soft_max_body}", {"A_TYPE": "float", "B_TYPE": "float", "D_TYPE": "float"})) - tasks.append(string_to_spv("rope_f32", rope_src, {"A_TYPE": "float", "D_TYPE": "float"}, True)) - tasks.append(string_to_spv("rope_f16", rope_src, {"A_TYPE": "float16_t", "D_TYPE": "float16_t"}, True)) + tasks.append(string_to_spv("rope_f32", rope_src, {"A_TYPE": "float", "D_TYPE": "float"})) + tasks.append(string_to_spv("rope_f16", rope_src, {"A_TYPE": "float16_t", "D_TYPE": "float16_t"})) - tasks.append(string_to_spv("rope_neox_f32", rope_neox_src, {"A_TYPE": "float", "D_TYPE": "float"}, True)) - tasks.append(string_to_spv("rope_neox_f16", rope_neox_src, {"A_TYPE": "float16_t", "D_TYPE": "float16_t"}, True)) + tasks.append(string_to_spv("rope_neox_f32", rope_neox_src, {"A_TYPE": "float", "D_TYPE": "float"})) + tasks.append(string_to_spv("rope_neox_f16", rope_neox_src, {"A_TYPE": "float16_t", "D_TYPE": "float16_t"})) await asyncio.gather(*tasks) From 60ecf099eddfe70fec797ef6790572e452054add Mon Sep 17 00:00:00 2001 From: Martin Schwaighofer Date: Sun, 28 Jan 2024 12:59:43 +0100 Subject: [PATCH 252/812] add Vulkan support to Nix flake --- .devops/nix/package.nix | 21 +++++++++++++++++---- flake.nix | 1 + 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/.devops/nix/package.nix b/.devops/nix/package.nix index a868a9a61..ad23f7dd7 100644 --- a/.devops/nix/package.nix +++ b/.devops/nix/package.nix @@ -13,18 +13,22 @@ cudaPackages, darwin, rocmPackages, + vulkan-headers, + vulkan-loader, clblast, useBlas ? builtins.all (x: !x) [ useCuda useMetalKit useOpenCL useRocm + useVulkan ], useCuda ? config.cudaSupport, useMetalKit ? stdenv.isAarch64 && stdenv.isDarwin && !useOpenCL, useMpi ? false, # Increases the runtime closure size by ~700M useOpenCL ? false, useRocm ? config.rocmSupport, + useVulkan ? false, llamaVersion ? "0.0.0", # Arbitrary version, substituted by the flake }@inputs: @@ -48,7 +52,8 @@ let ++ lib.optionals useMetalKit [ "MetalKit" ] ++ lib.optionals useMpi [ "MPI" ] ++ lib.optionals useOpenCL [ "OpenCL" ] - ++ lib.optionals useRocm [ "ROCm" ]; + ++ lib.optionals useRocm [ "ROCm" ] + ++ lib.optionals useVulkan [ "Vulkan" ]; pnameSuffix = strings.optionalString (suffices != [ ]) @@ -108,6 +113,11 @@ let hipblas rocblas ]; + + vulkanBuildInputs = [ + vulkan-headers + vulkan-loader + ]; in effectiveStdenv.mkDerivation ( @@ -164,7 +174,8 @@ effectiveStdenv.mkDerivation ( ++ optionals useCuda cudaBuildInputs ++ optionals useMpi [ mpi ] ++ optionals useOpenCL [ clblast ] - ++ optionals useRocm rocmBuildInputs; + ++ optionals useRocm rocmBuildInputs + ++ optionals useVulkan vulkanBuildInputs; cmakeFlags = [ @@ -178,6 +189,7 @@ effectiveStdenv.mkDerivation ( (cmakeBool "LLAMA_HIPBLAS" useRocm) (cmakeBool "LLAMA_METAL" useMetalKit) (cmakeBool "LLAMA_MPI" useMpi) + (cmakeBool "LLAMA_VULKAN" useVulkan) ] ++ optionals useCuda [ ( @@ -218,6 +230,7 @@ effectiveStdenv.mkDerivation ( useMpi useOpenCL useRocm + useVulkan ; shell = mkShell { @@ -242,11 +255,11 @@ effectiveStdenv.mkDerivation ( # Configurations we don't want even the CI to evaluate. Results in the # "unsupported platform" messages. This is mostly a no-op, because # cudaPackages would've refused to evaluate anyway. - badPlatforms = optionals (useCuda || useOpenCL) lib.platforms.darwin; + badPlatforms = optionals (useCuda || useOpenCL || useVulkan) lib.platforms.darwin; # Configurations that are known to result in build failures. Can be # overridden by importing Nixpkgs with `allowBroken = true`. - broken = (useMetalKit && !effectiveStdenv.isDarwin); + broken = (useMetalKit && !effectiveStdenv.isDarwin) || (useVulkan && effectiveStdenv.isDarwin); description = "Inference of LLaMA model in pure C/C++${descriptionSuffix}"; homepage = "https://github.com/ggerganov/llama.cpp/"; diff --git a/flake.nix b/flake.nix index a776ba024..ad2f9b295 100644 --- a/flake.nix +++ b/flake.nix @@ -157,6 +157,7 @@ mpi-cpu = config.packages.default.override { useMpi = true; }; mpi-cuda = config.packages.default.override { useMpi = true; }; + vulkan = config.packages.default.override { useVulkan = true; }; } // lib.optionalAttrs (system == "x86_64-linux") { rocm = config.legacyPackages.llamaPackagesRocm.llama-cpp; From 3cc5ed353c07201d8d5b98b0a4713ab633da6d04 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20G=C3=A4=C3=9Fler?= Date: Sat, 3 Feb 2024 20:14:59 +0100 Subject: [PATCH 253/812] make: fix nvcc optimization flags for host code (#5309) --- Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index a55d15888..40b16e0ea 100644 --- a/Makefile +++ b/Makefile @@ -109,6 +109,7 @@ MK_NVCCFLAGS += -O3 else MK_CFLAGS += -O3 MK_CXXFLAGS += -O3 +MK_NVCCFLAGS += -O3 endif # clock_gettime came in POSIX.1b (1993) @@ -365,7 +366,7 @@ ifdef LLAMA_CUBLAS MK_CPPFLAGS += -DGGML_USE_CUBLAS -I/usr/local/cuda/include -I/opt/cuda/include -I$(CUDA_PATH)/targets/x86_64-linux/include -I/usr/local/cuda/targets/aarch64-linux/include MK_LDFLAGS += -lcuda -lcublas -lculibos -lcudart -lcublasLt -lpthread -ldl -lrt -L/usr/local/cuda/lib64 -L/opt/cuda/lib64 -L$(CUDA_PATH)/targets/x86_64-linux/lib -L/usr/local/cuda/targets/aarch64-linux/lib -L/usr/lib/wsl/lib OBJS += ggml-cuda.o - MK_NVCCFLAGS = -use_fast_math + MK_NVCCFLAGS += -use_fast_math ifndef JETSON_EOL_MODULE_DETECT MK_NVCCFLAGS += --forward-unknown-to-host-compiler endif # JETSON_EOL_MODULE_DETECT From 3c0d25c4756742ebf15ad44700fabc0700c638bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20G=C3=A4=C3=9Fler?= Date: Sat, 3 Feb 2024 20:15:13 +0100 Subject: [PATCH 254/812] make: add nvcc info print (#5310) --- Makefile | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 40b16e0ea..21d5e15ba 100644 --- a/Makefile +++ b/Makefile @@ -553,8 +553,11 @@ $(info I CFLAGS: $(CFLAGS)) $(info I CXXFLAGS: $(CXXFLAGS)) $(info I NVCCFLAGS: $(NVCCFLAGS)) $(info I LDFLAGS: $(LDFLAGS)) -$(info I CC: $(shell $(CC) --version | head -n 1)) -$(info I CXX: $(shell $(CXX) --version | head -n 1)) +$(info I CC: $(shell $(CC) --version | head -n 1)) +$(info I CXX: $(shell $(CXX) --version | head -n 1)) +ifdef LLAMA_CUBLAS +$(info I NVCC: $(shell $(NVCC) --version | tail -n 1)) +endif # LLAMA_CUBLAS $(info ) # From 277fad30c60ef3559dc2d01b19d05e659d40a824 Mon Sep 17 00:00:00 2001 From: Welby Seely Date: Sat, 3 Feb 2024 23:18:51 -0500 Subject: [PATCH 255/812] cmake : use set() for LLAMA_WIN_VER (#5298) option() is specifically for booleans. Fixes #5158 --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index c156c4824..8c04e4c19 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -79,7 +79,7 @@ if (NOT MSVC) endif() if (WIN32) - option(LLAMA_WIN_VER "llama: Windows Version" 0x602) + set(LLAMA_WIN_VER "0x602" CACHE STRING "llama: Windows Version") endif() # 3rd party libs From 5ed26e1fc9fab4ce96ecf2d84183fe45bdcab0d4 Mon Sep 17 00:00:00 2001 From: Kawrakow <48489457+ikawrakow@users.noreply.github.com> Date: Sun, 4 Feb 2024 10:39:58 +0200 Subject: [PATCH 256/812] Adding some imatrix tools (#5302) * imatrix: adding --combine and --continue-from * imatrix: be able to start from a specific chunk --------- Co-authored-by: Iwan Kawrakow --- examples/imatrix/imatrix.cpp | 116 +++++++++++++++++++++++++++++++++-- 1 file changed, 112 insertions(+), 4 deletions(-) diff --git a/examples/imatrix/imatrix.cpp b/examples/imatrix/imatrix.cpp index ea06fcdbf..bc9f6fa68 100644 --- a/examples/imatrix/imatrix.cpp +++ b/examples/imatrix/imatrix.cpp @@ -36,6 +36,8 @@ public: void set_parameters(StatParams&& params) { m_params = std::move(params); } bool collect_imatrix(struct ggml_tensor * t, bool ask, void * user_data); void save_imatrix() const; + bool load_imatrix(const char * file_name, bool add); + static bool load_imatrix(const char * file_name, std::unordered_map& imatrix); private: std::unordered_map m_stats; StatParams m_params; @@ -189,6 +191,57 @@ void IMatrixCollector::save_imatrix(const char * fname) const { } } +bool IMatrixCollector::load_imatrix(const char * imatrix_file, std::unordered_map& imatrix_data) { + std::ifstream in(imatrix_file, std::ios::binary); + if (!in) { + printf("%s: failed to open %s\n",__func__,imatrix_file); + return false; + } + int n_entries; + in.read((char*)&n_entries, sizeof(n_entries)); + if (in.fail() || n_entries < 1) { + printf("%s: no data in file %s\n", __func__, imatrix_file); + return false; + } + for (int i = 0; i < n_entries; ++i) { + int len; in.read((char *)&len, sizeof(len)); + std::vector name_as_vec(len+1); + in.read((char *)name_as_vec.data(), len); + if (in.fail()) { + printf("%s: failed reading name for entry %d from %s\n",__func__,i+1,imatrix_file); + return false; + } + name_as_vec[len] = 0; + std::string name{name_as_vec.data()}; + auto& e = imatrix_data[std::move(name)]; + int ncall; + in.read((char*)&ncall, sizeof(ncall)); + int nval; + in.read((char *)&nval, sizeof(nval)); + if (in.fail() || nval < 1) { + printf("%s: failed reading number of values for entry %d\n",__func__,i); + imatrix_data = {}; + return false; + } + e.values.resize(nval); + in.read((char*)e.values.data(), nval*sizeof(float)); + if (in.fail()) { + printf("%s: failed reading data for entry %d\n",__func__,i); + imatrix_data = {}; + return false; + } + e.ncall = ncall; + } + return true; +} + +bool IMatrixCollector::load_imatrix(const char * file_name, bool add) { + if (!add) { + m_stats.clear(); + } + return load_imatrix(file_name, m_stats); +} + static IMatrixCollector g_collector; static bool ik_collect_imatrix(struct ggml_tensor * t, bool ask, void * user_data) { @@ -269,7 +322,7 @@ static void process_logits( } } -static bool compute_imatrix(llama_context * ctx, const gpt_params & params, bool compute_ppl) { +static bool compute_imatrix(llama_context * ctx, const gpt_params & params, bool compute_ppl, int from_chunk) { const bool add_bos = llama_should_add_bos_token(llama_get_model(ctx)); const int n_ctx = llama_n_ctx(ctx); @@ -282,6 +335,15 @@ static bool compute_imatrix(llama_context * ctx, const gpt_params & params, bool auto tim2 = std::chrono::high_resolution_clock::now(); fprintf(stderr, "%s: tokenization took %g ms\n",__func__,1e-3*std::chrono::duration_cast(tim2-tim1).count()); + if (from_chunk > 0) { + if (size_t((from_chunk + 2)*n_ctx) >= tokens.size()) { + fprintf(stderr, "%s: there will be not enough tokens left after removing %d chunks\n", __func__, from_chunk); + return false; + } + fprintf(stderr, "%s: removing initial %d chunks (%d tokens)\n", __func__, from_chunk, from_chunk*n_ctx); + tokens.erase(tokens.begin(), tokens.begin() + from_chunk*n_ctx); + } + if (int(tokens.size()) < 2*n_ctx) { fprintf(stderr, "%s: you need at least %d tokens for a context of %d tokens\n",__func__,2*n_ctx, n_ctx); @@ -402,7 +464,10 @@ static bool compute_imatrix(llama_context * ctx, const gpt_params & params, bool int main(int argc, char ** argv) { StatParams sparams; + std::string prev_result_file; + std::string combine_files; bool compute_ppl = true; + int from_chunk = 0; std::vector args; args.push_back(argv[0]); int iarg = 1; @@ -423,6 +488,13 @@ int main(int argc, char ** argv) { compute_ppl = false; } else if (arg == "--keep-imatrix") { sparams.keep_every = std::stoi(argv[++iarg]); + } else if (arg == "--continue-from") { + prev_result_file = argv[++iarg]; + } else if (arg == "--combine") { + combine_files = argv[++iarg]; + } + else if (arg == "--from-chunk") { + from_chunk = std::stoi(argv[++iarg]); } else { args.push_back(argv[iarg]); } @@ -436,14 +508,50 @@ int main(int argc, char ** argv) { } } + g_collector.set_parameters(std::move(sparams)); + + if (!combine_files.empty()) { + std::vector files; + size_t pos = 0; + while (true) { + auto new_pos = combine_files.find(',', pos); + if (new_pos != std::string::npos) { + files.emplace_back(combine_files.substr(pos, new_pos - pos)); + pos = new_pos + 1; + } else { + files.emplace_back(combine_files.substr(pos)); + break; + } + } + if (files.size() < 2) { + fprintf(stderr, "You must provide at least two comma separated files to use --combine\n"); + return 1; + } + printf("Combining the following %d files\n", int(files.size())); + for (auto& file : files) { + printf(" %s\n", file.c_str()); + if (!g_collector.load_imatrix(file.c_str(), true)) { + fprintf(stderr, "Failed to load %s\n", file.c_str()); + return 1; + } + } + g_collector.save_imatrix(); + return 0; + } + + if (!prev_result_file.empty()) { + if (!g_collector.load_imatrix(prev_result_file.c_str(), false)) { + fprintf(stderr, "=============== Failed to load %s\n", prev_result_file.c_str()); + return 1; + } + } + gpt_params params; params.n_batch = 512; if (!gpt_params_parse(args.size(), args.data(), params)) { return 1; } - g_collector.set_parameters(std::move(sparams)); - params.logits_all = true; params.n_batch = std::min(params.n_batch, params.n_ctx); @@ -495,7 +603,7 @@ int main(int argc, char ** argv) { fprintf(stderr, "%s\n", get_system_info(params).c_str()); } - bool OK = compute_imatrix(ctx, params, compute_ppl); + bool OK = compute_imatrix(ctx, params, compute_ppl, from_chunk); if (!OK) { return 1; } From 9392ebd49ea5ae236a55b47cbf6a13247e8a3b8c Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Sun, 4 Feb 2024 00:17:24 +0000 Subject: [PATCH 257/812] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'flake-parts': 'github:hercules-ci/flake-parts/07f6395285469419cf9d078f59b5b49993198c00' (2024-01-11) → 'github:hercules-ci/flake-parts/b253292d9c0a5ead9bc98c4e9a26c6312e27d69f' (2024-02-01) • Updated input 'flake-parts/nixpkgs-lib': 'github:NixOS/nixpkgs/b0d36bd0a420ecee3bc916c91886caca87c894e9?dir=lib' (2023-12-30) → 'github:NixOS/nixpkgs/97b17f32362e475016f942bbdfda4a4a72a8a652?dir=lib' (2024-01-29) • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/ae5c332cbb5827f6b1f02572496b141021de335f' (2024-01-25) → 'github:NixOS/nixpkgs/b8b232ae7b8b144397fdb12d20f592e5e7c1a64d' (2024-01-31) --- flake.lock | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/flake.lock b/flake.lock index 95e41f333..8cfc78273 100644 --- a/flake.lock +++ b/flake.lock @@ -5,11 +5,11 @@ "nixpkgs-lib": "nixpkgs-lib" }, "locked": { - "lastModified": 1704982712, - "narHash": "sha256-2Ptt+9h8dczgle2Oo6z5ni5rt/uLMG47UFTR1ry/wgg=", + "lastModified": 1706830856, + "narHash": "sha256-a0NYyp+h9hlb7ddVz4LUn1vT/PLwqfrWYcHMvFB1xYg=", "owner": "hercules-ci", "repo": "flake-parts", - "rev": "07f6395285469419cf9d078f59b5b49993198c00", + "rev": "b253292d9c0a5ead9bc98c4e9a26c6312e27d69f", "type": "github" }, "original": { @@ -20,11 +20,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1706191920, - "narHash": "sha256-eLihrZAPZX0R6RyM5fYAWeKVNuQPYjAkCUBr+JNvtdE=", + "lastModified": 1706732774, + "narHash": "sha256-hqJlyJk4MRpcItGYMF+3uHe8HvxNETWvlGtLuVpqLU0=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "ae5c332cbb5827f6b1f02572496b141021de335f", + "rev": "b8b232ae7b8b144397fdb12d20f592e5e7c1a64d", "type": "github" }, "original": { @@ -37,11 +37,11 @@ "nixpkgs-lib": { "locked": { "dir": "lib", - "lastModified": 1703961334, - "narHash": "sha256-M1mV/Cq+pgjk0rt6VxoyyD+O8cOUiai8t9Q6Yyq4noY=", + "lastModified": 1706550542, + "narHash": "sha256-UcsnCG6wx++23yeER4Hg18CXWbgNpqNXcHIo5/1Y+hc=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "b0d36bd0a420ecee3bc916c91886caca87c894e9", + "rev": "97b17f32362e475016f942bbdfda4a4a72a8a652", "type": "github" }, "original": { From 4833ac209da6a427de64f97e8f403dcdc5de6bc3 Mon Sep 17 00:00:00 2001 From: AidanBeltonS <87009434+AidanBeltonS@users.noreply.github.com> Date: Mon, 5 Feb 2024 07:08:24 +0000 Subject: [PATCH 258/812] [SYCL] Fix cpy with dims of 3 (#5289) * Fix cpy with dims of 3 * rm asserts --------- Co-authored-by: Abhilash Majumder <30946547+abhilash1910@users.noreply.github.com> --- ggml-sycl.cpp | 194 +++++++++++++++++++++++++++++--------------------- 1 file changed, 114 insertions(+), 80 deletions(-) diff --git a/ggml-sycl.cpp b/ggml-sycl.cpp index 51445b5e7..a03df4c65 100644 --- a/ggml-sycl.cpp +++ b/ggml-sycl.cpp @@ -7693,6 +7693,13 @@ static void cpy_1_f16_f16(const char * cxi, char * cdsti) { *dsti = *xi; } +static void cpy_1_f16_f32(const char * cxi, char * cdsti) { + const sycl::half *xi = (const sycl::half *)cxi; + float *dsti = (float *)cdsti; + + *dsti = *xi; +} + static void cpy_1_i16_i16(const char * cxi, char * cdsti) { const int16_t *xi = (const int16_t *)cxi; int16_t *dsti = (int16_t *)cdsti; @@ -7709,9 +7716,9 @@ static void cpy_1_i32_i32(const char * cxi, char * cdsti) { template static void cpy_f32_f16(const char * cx, char * cdst, const int ne, - const int ne00, const int ne01, const int nb00, const int nb01, const int nb02, - const int ne10, const int ne11, const int nb10, const int nb11, const int nb12, - const sycl::nd_item<3> &item_ct1) { + const int ne00, const int ne01, const int ne02, const int nb00, const int nb01, const int nb02, + const int nb03, const int ne10, const int ne11, const int ne12, const int nb10, const int nb11, + const int nb12, const int nb13, const sycl::nd_item<3> &item_ct1) { const int i = item_ct1.get_local_range(2) * item_ct1.get_group(2) + item_ct1.get_local_id(2); @@ -7721,15 +7728,17 @@ static void cpy_f32_f16(const char * cx, char * cdst, const int ne, // determine indices i02/i12, i01/i11, i00/i10 as a function of index i of flattened tensor // then combine those indices with the corresponding byte offsets to get the total offsets - const int i02 = i / (ne00*ne01); - const int i01 = (i - i02*ne01*ne00) / ne00; - const int i00 = i - i02*ne01*ne00 - i01*ne00; - const int x_offset = i00*nb00 + i01*nb01 + i02*nb02; + const int i03 = i/(ne00 * ne01 * ne02); + const int i02 = (i - i03*ne00*ne01*ne02 )/ (ne00*ne01); + const int i01 = (i - i03*ne00*ne01*ne02 - i02*ne01*ne00) / ne00; + const int i00 = i - i03*ne00*ne01*ne02 - i02*ne01*ne00 - i01*ne00; + const int x_offset = i00*nb00 + i01*nb01 + i02*nb02 + i03 * nb03; - const int i12 = i / (ne10*ne11); - const int i11 = (i - i12*ne10*ne11) / ne10; - const int i10 = i - i12*ne10*ne11 - i11*ne10; - const int dst_offset = i10*nb10 + i11*nb11 + i12*nb12; + const int i13 = i/(ne10 * ne11 * ne12); + const int i12 = (i - i13*ne10*ne11*ne12) / (ne10*ne11); + const int i11 = (i - i13*ne10*ne11*ne12 - i12*ne10*ne11) / ne10; + const int i10 = i - i13*ne10*ne11*ne12 - i12*ne10*ne11 - i11*ne10; + const int dst_offset = i10*nb10 + i11*nb11 + i12*nb12 + i13 * nb13; cpy_1(cx + x_offset, cdst + dst_offset); } @@ -7823,9 +7832,9 @@ static void cpy_blck_f32_q4_1(const char * cxi, char * cdsti) { template static void cpy_f32_q(const char * cx, char * cdst, const int ne, - const int ne00, const int ne01, const int nb00, const int nb01, const int nb02, - const int ne10, const int ne11, const int nb10, const int nb11, const int nb12, - const sycl::nd_item<3> &item_ct1) { + const int ne00, const int ne01, const int ne02, const int nb00, const int nb01, const int nb02, + const int nb03, const int ne10, const int ne11, const int ne12, const int nb10, const int nb11, + const int nb12, const int nb13, const sycl::nd_item<3> &item_ct1) { const int i = (item_ct1.get_local_range(2) * item_ct1.get_group(2) + item_ct1.get_local_id(2)) * qk; @@ -7834,15 +7843,17 @@ static void cpy_f32_q(const char * cx, char * cdst, const int ne, return; } - const int i02 = i / (ne00*ne01); - const int i01 = (i - i02*ne01*ne00) / ne00; - const int i00 = (i - i02*ne01*ne00 - i01*ne00); - const int x_offset = i00*nb00 + i01*nb01 + i02*nb02; + const int i03 = i/(ne00 * ne01 * ne02); + const int i02 = (i - i03*ne00*ne01*ne02 )/ (ne00*ne01); + const int i01 = (i - i03*ne00*ne01*ne02 - i02*ne01*ne00) / ne00; + const int i00 = i - i03*ne00*ne01*ne02 - i02*ne01*ne00 - i01*ne00; + const int x_offset = i00*nb00 + i01*nb01 + i02*nb02 + i03 * nb03; - const int i12 = i / (ne10*ne11); - const int i11 = (i - i12*ne10*ne11) / ne10; - const int i10 = (i - i12*ne10*ne11 - i11*ne10)/qk; - const int dst_offset = i10*nb10 + i11*nb11 + i12*nb12; + const int i13 = i/(ne10 * ne11 * ne12); + const int i12 = (i - i13*ne10*ne11*ne12) / (ne10*ne11); + const int i11 = (i - i13*ne10*ne11*ne12 - i12*ne10*ne11) / ne10; + const int i10 = i - i13*ne10*ne11*ne12 - i12*ne10*ne11 - i11*ne10; + const int dst_offset = (i10/qk)*nb10 + i11*nb11 + i12*nb12 + i13*nb13; cpy_blck(cx + x_offset, cdst + dst_offset); } @@ -10599,10 +10610,12 @@ static void ggml_mul_mat_vec_nc_f16_f32_sycl( static void ggml_cpy_f32_f32_sycl(const char *cx, char *cdst, const int ne, const int ne00, const int ne01, - const int nb00, const int nb01, - const int nb02, const int ne10, - const int ne11, const int nb10, - const int nb11, const int nb12, + const int ne02, const int nb00, + const int nb01, const int nb02, + const int nb03, const int ne10, + const int ne11, const int ne12, + const int nb10, const int nb11, + const int nb12, const int nb13, dpct::queue_ptr stream) { const int num_blocks = (ne + SYCL_CPY_BLOCK_SIZE - 1) / SYCL_CPY_BLOCK_SIZE; @@ -10615,8 +10628,8 @@ static void ggml_cpy_f32_f32_sycl(const char *cx, char *cdst, const int ne, sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), [=](sycl::nd_item<3> item_ct1) { - cpy_f32_f16(cx, cdst, ne, ne00, ne01, nb00, nb01, - nb02, ne10, ne11, nb10, nb11, nb12, + cpy_f32_f16(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, + nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); }); } @@ -10624,10 +10637,12 @@ static void ggml_cpy_f32_f32_sycl(const char *cx, char *cdst, const int ne, static void ggml_cpy_f32_f16_sycl(const char *cx, char *cdst, const int ne, const int ne00, const int ne01, - const int nb00, const int nb01, - const int nb02, const int ne10, - const int ne11, const int nb10, - const int nb11, const int nb12, + const int ne02, const int nb00, + const int nb01, const int nb02, + const int nb03, const int ne10, + const int ne11, const int ne12, + const int nb10, const int nb11, + const int nb12, const int nb13, dpct::queue_ptr stream) { const int num_blocks = (ne + SYCL_CPY_BLOCK_SIZE - 1) / SYCL_CPY_BLOCK_SIZE; @@ -10640,8 +10655,8 @@ static void ggml_cpy_f32_f16_sycl(const char *cx, char *cdst, const int ne, sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), [=](sycl::nd_item<3> item_ct1) { - cpy_f32_f16(cx, cdst, ne, ne00, ne01, nb00, nb01, - nb02, ne10, ne11, nb10, nb11, nb12, + cpy_f32_f16(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, + nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); }); } @@ -10649,10 +10664,12 @@ static void ggml_cpy_f32_f16_sycl(const char *cx, char *cdst, const int ne, static void ggml_cpy_f32_q8_0_sycl(const char *cx, char *cdst, const int ne, const int ne00, const int ne01, - const int nb00, const int nb01, - const int nb02, const int ne10, - const int ne11, const int nb10, - const int nb11, const int nb12, + const int ne02, const int nb00, + const int nb01, const int nb02, + const int nb03, const int ne10, + const int ne11, const int ne12, + const int nb10, const int nb11, + const int nb12, const int nb13, dpct::queue_ptr stream) { GGML_ASSERT(ne % QK8_0 == 0); @@ -10661,17 +10678,20 @@ static void ggml_cpy_f32_q8_0_sycl(const char *cx, char *cdst, const int ne, sycl::range<3>(1, 1, 1)), [=](sycl::nd_item<3> item_ct1) { cpy_f32_q( - cx, cdst, ne, ne00, ne01, nb00, nb01, nb02, - ne10, ne11, nb10, nb11, nb12, item_ct1); + cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, + nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, + item_ct1); }); } static void ggml_cpy_f32_q4_0_sycl(const char *cx, char *cdst, const int ne, const int ne00, const int ne01, - const int nb00, const int nb01, - const int nb02, const int ne10, - const int ne11, const int nb10, - const int nb11, const int nb12, + const int ne02, const int nb00, + const int nb01, const int nb02, + const int nb03, const int ne10, + const int ne11, const int ne12, + const int nb10, const int nb11, + const int nb12, const int nb13, dpct::queue_ptr stream) { GGML_ASSERT(ne % QK4_0 == 0); @@ -10680,17 +10700,20 @@ static void ggml_cpy_f32_q4_0_sycl(const char *cx, char *cdst, const int ne, sycl::range<3>(1, 1, 1)), [=](sycl::nd_item<3> item_ct1) { cpy_f32_q( - cx, cdst, ne, ne00, ne01, nb00, nb01, nb02, - ne10, ne11, nb10, nb11, nb12, item_ct1); + cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, + nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, + item_ct1); }); } static void ggml_cpy_f32_q4_1_sycl(const char *cx, char *cdst, const int ne, const int ne00, const int ne01, - const int nb00, const int nb01, - const int nb02, const int ne10, - const int ne11, const int nb10, - const int nb11, const int nb12, + const int ne02, const int nb00, + const int nb01, const int nb02, + const int nb03, const int ne10, + const int ne11, const int ne12, + const int nb10, const int nb11, + const int nb12, const int nb13, dpct::queue_ptr stream) { GGML_ASSERT(ne % QK4_1 == 0); @@ -10699,17 +10722,20 @@ static void ggml_cpy_f32_q4_1_sycl(const char *cx, char *cdst, const int ne, sycl::range<3>(1, 1, 1)), [=](sycl::nd_item<3> item_ct1) { cpy_f32_q( - cx, cdst, ne, ne00, ne01, nb00, nb01, nb02, - ne10, ne11, nb10, nb11, nb12, item_ct1); + cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, + nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, + item_ct1); }); } static void ggml_cpy_f16_f16_sycl(const char *cx, char *cdst, const int ne, const int ne00, const int ne01, - const int nb00, const int nb01, - const int nb02, const int ne10, - const int ne11, const int nb10, - const int nb11, const int nb12, + const int ne02, const int nb00, + const int nb01, const int nb02, + const int nb03, const int ne10, + const int ne11, const int ne12, + const int nb10, const int nb11, + const int nb12, const int nb13, dpct::queue_ptr stream) { const int num_blocks = (ne + SYCL_CPY_BLOCK_SIZE - 1) / SYCL_CPY_BLOCK_SIZE; @@ -10722,8 +10748,8 @@ static void ggml_cpy_f16_f16_sycl(const char *cx, char *cdst, const int ne, sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), [=](sycl::nd_item<3> item_ct1) { - cpy_f32_f16(cx, cdst, ne, ne00, ne01, nb00, nb01, - nb02, ne10, ne11, nb10, nb11, nb12, + cpy_f32_f16(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, + nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); }); } @@ -10731,10 +10757,12 @@ static void ggml_cpy_f16_f16_sycl(const char *cx, char *cdst, const int ne, static void ggml_cpy_i16_i16_sycl(const char *cx, char *cdst, const int ne, const int ne00, const int ne01, - const int nb00, const int nb01, - const int nb02, const int ne10, - const int ne11, const int nb10, - const int nb11, const int nb12, + const int ne02, const int nb00, + const int nb01, const int nb02, + const int nb03, const int ne10, + const int ne11, const int ne12, + const int nb10, const int nb11, + const int nb12, const int nb13, dpct::queue_ptr stream) { const int num_blocks = (ne + SYCL_CPY_BLOCK_SIZE - 1) / SYCL_CPY_BLOCK_SIZE; @@ -10747,8 +10775,8 @@ static void ggml_cpy_i16_i16_sycl(const char *cx, char *cdst, const int ne, sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), [=](sycl::nd_item<3> item_ct1) { - cpy_f32_f16(cx, cdst, ne, ne00, ne01, nb00, nb01, - nb02, ne10, ne11, nb10, nb11, nb12, + cpy_f32_f16(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, + nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); }); } @@ -10756,10 +10784,12 @@ static void ggml_cpy_i16_i16_sycl(const char *cx, char *cdst, const int ne, static void ggml_cpy_i32_i32_sycl(const char *cx, char *cdst, const int ne, const int ne00, const int ne01, - const int nb00, const int nb01, - const int nb02, const int ne10, - const int ne11, const int nb10, - const int nb11, const int nb12, + const int ne02, const int nb00, + const int nb01, const int nb02, + const int nb03, const int ne10, + const int ne11, const int ne12, + const int nb10, const int nb11, + const int nb12, const int nb13, dpct::queue_ptr stream) { const int num_blocks = (ne + SYCL_CPY_BLOCK_SIZE - 1) / SYCL_CPY_BLOCK_SIZE; @@ -10772,8 +10802,8 @@ static void ggml_cpy_i32_i32_sycl(const char *cx, char *cdst, const int ne, sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), [=](sycl::nd_item<3> item_ct1) { - cpy_f32_f16(cx, cdst, ne, ne00, ne01, nb00, nb01, - nb02, ne10, ne11, nb10, nb11, nb12, + cpy_f32_f16(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, + nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); }); } @@ -13910,19 +13940,23 @@ static void ggml_sycl_cpy(const ggml_tensor *src0, const ggml_tensor *src1, const int64_t ne00 = src0->ne[0]; const int64_t ne01 = src0->ne[1]; - GGML_ASSERT(src0->ne[3] == 1); + const int64_t ne02 = src0->ne[2]; + const int64_t nb00 = src0->nb[0]; const int64_t nb01 = src0->nb[1]; const int64_t nb02 = src0->nb[2]; + const int64_t nb03 = src0->nb[3]; const int64_t ne10 = src1->ne[0]; const int64_t ne11 = src1->ne[1]; - GGML_ASSERT(src1->ne[3] == 1); + const int64_t ne12 = src1->ne[2]; + const int64_t nb10 = src1->nb[0]; const int64_t nb11 = src1->nb[1]; const int64_t nb12 = src1->nb[2]; + const int64_t nb13 = src1->nb[3]; SYCL_CHECK(ggml_sycl_set_device(g_main_device)); dpct::queue_ptr main_stream = g_syclStreams[g_main_device_index][0]; @@ -13934,21 +13968,21 @@ static void ggml_sycl_cpy(const ggml_tensor *src0, const ggml_tensor *src1, char * src1_ddc = (char *) src1_extra->data_device[g_main_device_index]; if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_F32) { - ggml_cpy_f32_f32_sycl (src0_ddc, src1_ddc, ne, ne00, ne01, nb00, nb01, nb02, ne10, ne11, nb10, nb11, nb12, main_stream); + ggml_cpy_f32_f32_sycl (src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream); } else if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_F16) { - ggml_cpy_f32_f16_sycl (src0_ddc, src1_ddc, ne, ne00, ne01, nb00, nb01, nb02, ne10, ne11, nb10, nb11, nb12, main_stream); + ggml_cpy_f32_f16_sycl (src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream); } else if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_Q8_0) { - ggml_cpy_f32_q8_0_sycl(src0_ddc, src1_ddc, ne, ne00, ne01, nb00, nb01, nb02, ne10, ne11, nb10, nb11, nb12, main_stream); + ggml_cpy_f32_q8_0_sycl(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream); } else if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_Q4_0) { - ggml_cpy_f32_q4_0_sycl(src0_ddc, src1_ddc, ne, ne00, ne01, nb00, nb01, nb02, ne10, ne11, nb10, nb11, nb12, main_stream); + ggml_cpy_f32_q4_0_sycl(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream); } else if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_Q4_1) { - ggml_cpy_f32_q4_1_sycl(src0_ddc, src1_ddc, ne, ne00, ne01, nb00, nb01, nb02, ne10, ne11, nb10, nb11, nb12, main_stream); + ggml_cpy_f32_q4_1_sycl(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream); } else if (src0->type == GGML_TYPE_F16 && src1->type == GGML_TYPE_F16) { - ggml_cpy_f16_f16_sycl (src0_ddc, src1_ddc, ne, ne00, ne01, nb00, nb01, nb02, ne10, ne11, nb10, nb11, nb12, main_stream); + ggml_cpy_f16_f16_sycl (src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream); } else if (src0->type == GGML_TYPE_I16 && src1->type == GGML_TYPE_I16) { - ggml_cpy_i16_i16_sycl (src0_ddc, src1_ddc, ne, ne00, ne01, nb00, nb01, nb02, ne10, ne11, nb10, nb11, nb12, main_stream); + ggml_cpy_i16_i16_sycl (src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream); } else if (src0->type == GGML_TYPE_I32 && src1->type == GGML_TYPE_I32) { - ggml_cpy_i32_i32_sycl (src0_ddc, src1_ddc, ne, ne00, ne01, nb00, nb01, nb02, ne10, ne11, nb10, nb11, nb12, main_stream); + ggml_cpy_i32_i32_sycl (src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream); } else { fprintf(stderr, "%s: unsupported type combination (%s to %s)\n", __func__, ggml_type_name(src0->type), ggml_type_name(src1->type)); From 5d55b0cd827bb0fcfedfa329a82bd5d6ef2c93ca Mon Sep 17 00:00:00 2001 From: chiranko <96988916+chiranko@users.noreply.github.com> Date: Mon, 5 Feb 2024 15:41:38 +0800 Subject: [PATCH 259/812] readme : add CodeShell models to the supported models list (#5330) --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 4a9bdf314..a6fe34629 100644 --- a/README.md +++ b/README.md @@ -107,6 +107,7 @@ as the main playground for developing new features for the [ggml](https://github - [x] [Mixtral MoE](https://huggingface.co/models?search=mistral-ai/Mixtral) - [x] [PLaMo-13B](https://github.com/ggerganov/llama.cpp/pull/3557) - [x] [GPT-2](https://huggingface.co/gpt2) +- [x] [CodeShell](https://github.com/WisdomShell/codeshell) **Multimodal models:** From 4be04c8965578edc09194fab769b4b922b8444f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9D=D0=B8=D1=8F=D0=B7=20=D0=93=D0=B0=D1=80=D0=B8=D1=84?= =?UTF-8?q?=D0=B7=D1=8F=D0=BD=D0=BE=D0=B2?= <112617865+garrnizon@users.noreply.github.com> Date: Mon, 5 Feb 2024 10:43:57 +0300 Subject: [PATCH 260/812] scripts : add non-interactive server-llm.sh (#5303) * Update server-llm.sh Add flag --non-interactive that allows run script without asking a permission * Update scripts/server-llm.sh --------- Co-authored-by: Georgi Gerganov --- scripts/server-llm.sh | 73 ++++++++++++++++++++++++------------------- 1 file changed, 40 insertions(+), 33 deletions(-) diff --git a/scripts/server-llm.sh b/scripts/server-llm.sh index 0b83cdbbc..062b70496 100644 --- a/scripts/server-llm.sh +++ b/scripts/server-llm.sh @@ -47,6 +47,7 @@ if ! command -v make &> /dev/null; then fi # parse arguments +is_interactive=1 port=8888 repo="" wtype="" @@ -66,15 +67,16 @@ verbose=0 function print_usage { printf "Usage:\n" - printf " ./server-llm.sh [--port] [--repo] [--wtype] [--backend] [--gpu-id] [--n-parallel] [--n-kv] [--verbose]\n\n" - printf " --port: port number, default is 8888\n" - printf " --repo: path to a repo containing GGUF model files\n" - printf " --wtype: weights type (f16, q8_0, q4_0, q4_1), default is user-input\n" - printf " --backend: cpu, cuda, metal, opencl, depends on the OS\n" - printf " --gpu-id: gpu id, default is 0\n" - printf " --n-parallel: number of parallel requests, default is 8\n" - printf " --n-kv: KV cache size, default is 4096\n" - printf " --verbose: verbose output\n\n" + printf " ./server-llm.sh [-interactive] [--port] [--repo] [--wtype] [--backend] [--gpu-id] [--n-parallel] [--n-kv] [--verbose]\n\n" + printf " --non-interactive: run without asking a permision to run\n" + printf " --port: port number, default is 8888\n" + printf " --repo: path to a repo containing GGUF model files\n" + printf " --wtype: weights type (f16, q8_0, q4_0, q4_1), default is user-input\n" + printf " --backend: cpu, cuda, metal, opencl, depends on the OS\n" + printf " --gpu-id: gpu id, default is 0\n" + printf " --n-parallel: number of parallel requests, default is 8\n" + printf " --n-kv: KV cache size, default is 4096\n" + printf " --verbose: verbose output\n\n" printf "Example:\n\n" printf ' bash -c "$(curl -s https://ggml.ai/server-llm.sh)"\n\n' } @@ -82,6 +84,10 @@ function print_usage { while [[ $# -gt 0 ]]; do key="$1" case $key in + --non-interactive) + is_interactive=0 + shift + ;; --port) port="$2" shift @@ -176,31 +182,32 @@ repos=( "https://huggingface.co/TheBloke/OpenHermes-2-Mistral-7B-GGUF" "https://huggingface.co/TheBloke/CausalLM-7B-GGUF" ) +if [ $is_interactive -eq 1 ]; then + printf "\n" + printf "[I] This is a helper script for deploying llama.cpp's server on this machine.\n\n" + printf " Based on the options that follow, the script might download a model file\n" + printf " from the internet, which can be a few GBs in size. The script will also\n" + printf " build the latest llama.cpp source code from GitHub, which can be unstable.\n" + printf "\n" + printf " Upon success, an HTTP server will be started and it will serve the selected\n" + printf " model using llama.cpp for demonstration purposes.\n" + printf "\n" + printf " Please note:\n" + printf "\n" + printf " - All new data will be stored in the current folder\n" + printf " - The server will be listening on all network interfaces\n" + printf " - The server will run with default settings which are not always optimal\n" + printf " - Do not judge the quality of a model based on the results from this script\n" + printf " - Do not use this script to benchmark llama.cpp\n" + printf " - Do not use this script in production\n" + printf " - This script is only for demonstration purposes\n" + printf "\n" + printf " If you don't know what you are doing, please press Ctrl-C to abort now\n" + printf "\n" + printf " Press Enter to continue ...\n\n" -printf "\n" -printf "[I] This is a helper script for deploying llama.cpp's server on this machine.\n\n" -printf " Based on the options that follow, the script might download a model file\n" -printf " from the internet, which can be a few GBs in size. The script will also\n" -printf " build the latest llama.cpp source code from GitHub, which can be unstable.\n" -printf "\n" -printf " Upon success, an HTTP server will be started and it will serve the selected\n" -printf " model using llama.cpp for demonstration purposes.\n" -printf "\n" -printf " Please note:\n" -printf "\n" -printf " - All new data will be stored in the current folder\n" -printf " - The server will be listening on all network interfaces\n" -printf " - The server will run with default settings which are not always optimal\n" -printf " - Do not judge the quality of a model based on the results from this script\n" -printf " - Do not use this script to benchmark llama.cpp\n" -printf " - Do not use this script in production\n" -printf " - This script is only for demonstration purposes\n" -printf "\n" -printf " If you don't know what you are doing, please press Ctrl-C to abort now\n" -printf "\n" -printf " Press Enter to continue ...\n\n" - -read + read +fi if [[ -z "$repo" ]]; then printf "[+] No repo provided from the command line\n" From 30679d438d5225b3aecf5cec6482cbc9f8f87ba5 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Mon, 5 Feb 2024 09:48:03 +0200 Subject: [PATCH 261/812] scripts : fix typos, cleanup (#5303) --- scripts/server-llm.sh | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/scripts/server-llm.sh b/scripts/server-llm.sh index 062b70496..30bbac321 100644 --- a/scripts/server-llm.sh +++ b/scripts/server-llm.sh @@ -14,16 +14,17 @@ # - Might be unstable! # # Usage: -# ./server-llm.sh [--port] [--repo] [--wtype] [--backend] [--gpu-id] [--n-parallel] [--n-kv] [--verbose] +# ./server-llm.sh [--port] [--repo] [--wtype] [--backend] [--gpu-id] [--n-parallel] [--n-kv] [--verbose] [-non-interactive] # -# --port: port number, default is 8888 -# --repo: path to a repo containing GGUF model files -# --wtype: weights type (f16, q8_0, q4_0, q4_1), default is user-input -# --backend: cpu, cuda, metal, opencl, depends on the OS -# --gpu-id: gpu id, default is 0 -# --n-parallel: number of parallel requests, default is 8 -# --n-kv: KV cache size, default is 4096 -# --verbose: verbose output +# --port: port number, default is 8888 +# --repo: path to a repo containing GGUF model files +# --wtype: weights type (f16, q8_0, q4_0, q4_1), default is user-input +# --backend: cpu, cuda, metal, opencl, depends on the OS +# --gpu-id: gpu id, default is 0 +# --n-parallel: number of parallel requests, default is 8 +# --n-kv: KV cache size, default is 4096 +# --verbose: verbose output +# --non-interactive: run without asking a permission to run # # Example: # @@ -67,8 +68,7 @@ verbose=0 function print_usage { printf "Usage:\n" - printf " ./server-llm.sh [-interactive] [--port] [--repo] [--wtype] [--backend] [--gpu-id] [--n-parallel] [--n-kv] [--verbose]\n\n" - printf " --non-interactive: run without asking a permision to run\n" + printf " ./server-llm.sh [--port] [--repo] [--wtype] [--backend] [--gpu-id] [--n-parallel] [--n-kv] [--verbose] [-non-interactive]\n\n" printf " --port: port number, default is 8888\n" printf " --repo: path to a repo containing GGUF model files\n" printf " --wtype: weights type (f16, q8_0, q4_0, q4_1), default is user-input\n" @@ -77,6 +77,7 @@ function print_usage { printf " --n-parallel: number of parallel requests, default is 8\n" printf " --n-kv: KV cache size, default is 4096\n" printf " --verbose: verbose output\n\n" + printf " --non-interactive: run without asking a permission to run\n" printf "Example:\n\n" printf ' bash -c "$(curl -s https://ggml.ai/server-llm.sh)"\n\n' } From e6f81775323f6f4e4a30abf022a6028fa86b79ac Mon Sep 17 00:00:00 2001 From: l3utterfly Date: Mon, 5 Feb 2024 17:00:47 +0900 Subject: [PATCH 262/812] common : add dynamic temperature parameters to main example cli (#5295) * added dynamic temp params in main * added help text --- common/common.cpp | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/common/common.cpp b/common/common.cpp index 3302caa20..8c1a60583 100644 --- a/common/common.cpp +++ b/common/common.cpp @@ -399,6 +399,18 @@ bool gpt_params_parse_ex(int argc, char ** argv, gpt_params & params) { break; } sparams.penalty_present = std::stof(argv[i]); + } else if (arg == "--dynatemp-range") { + if (++i >= argc) { + invalid_param = true; + break; + } + sparams.dynatemp_range = std::stof(argv[i]); + } else if (arg == "--dynatemp-exp") { + if (++i >= argc) { + invalid_param = true; + break; + } + sparams.dynatemp_exponent = std::stof(argv[i]); } else if (arg == "--mirostat") { if (++i >= argc) { invalid_param = true; @@ -942,6 +954,8 @@ void gpt_print_usage(int /*argc*/, char ** argv, const gpt_params & params) { printf(" --repeat-penalty N penalize repeat sequence of tokens (default: %.1f, 1.0 = disabled)\n", (double)sparams.penalty_repeat); printf(" --presence-penalty N repeat alpha presence penalty (default: %.1f, 0.0 = disabled)\n", (double)sparams.penalty_present); printf(" --frequency-penalty N repeat alpha frequency penalty (default: %.1f, 0.0 = disabled)\n", (double)sparams.penalty_freq); + printf(" --dynatemp-range N dynamic temperature range (default: %.1f, 0.0 = disabled)\n", (double)sparams.dynatemp_range); + printf(" --dynatemp-exp N dynamic temperature exponent (default: %.1f)\n", (double)sparams.dynatemp_exponent); printf(" --mirostat N use Mirostat sampling.\n"); printf(" Top K, Nucleus, Tail Free and Locally Typical samplers are ignored if used.\n"); printf(" (default: %d, 0 = disabled, 1 = Mirostat, 2 = Mirostat 2.0)\n", sparams.mirostat); From a2d60c9158435ae9a6f14632f07f1acf7a3becef Mon Sep 17 00:00:00 2001 From: Alexey Parfenov Date: Mon, 5 Feb 2024 08:10:22 +0000 Subject: [PATCH 263/812] server : allow to get default generation settings for completion (#5307) --- examples/server/README.md | 16 +++++++++++++++- examples/server/server.cpp | 7 ++++++- 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/examples/server/README.md b/examples/server/README.md index fe934dab1..d8e7c313e 100644 --- a/examples/server/README.md +++ b/examples/server/README.md @@ -264,7 +264,21 @@ Notice that each `probs` is an array of length `n_probs`. It also accepts all the options of `/completion` except `stream` and `prompt`. -- **GET** `/props`: Return the required assistant name and anti-prompt to generate the prompt in case you have specified a system prompt for all slots. +- **GET** `/props`: Return current server settings. + +### Result JSON + +```json +{ + "assistant_name": "", + "user_name": "", + "default_generation_settings": { ... } +} +``` + +- `assistant_name` - the required assistant name to generate the prompt in case you have specified a system prompt for all slots. +- `user_name` - the required anti-prompt to generate the prompt in case you have specified a system prompt for all slots. +- `default_generation_settings` - the default generation settings for the `/completion` endpoint, has the same fields as the `generation_settings` response object from the `/completion` endpoint. - **POST** `/v1/chat/completions`: OpenAI-compatible Chat Completions API. Given a ChatML-formatted json description in `messages`, it returns the predicted completion. Both synchronous and streaming mode are supported, so scripted and interactive applications work fine. While no strong claims of compatibility with OpenAI API spec is being made, in our experience it suffices to support many apps. Only ChatML-tuned models, such as Dolphin, OpenOrca, OpenHermes, OpenChat-3.5, etc can be used with this endpoint. Compared to `api_like_OAI.py` this API implementation does not require a wrapper to be served. diff --git a/examples/server/server.cpp b/examples/server/server.cpp index a9f8cb369..8000fee5c 100644 --- a/examples/server/server.cpp +++ b/examples/server/server.cpp @@ -334,6 +334,7 @@ struct llama_server_context // slots / clients std::vector slots; + json default_generation_settings_for_props; llama_server_queue queue_tasks; llama_server_response queue_results; @@ -430,6 +431,9 @@ struct llama_server_context slots.push_back(slot); } + default_generation_settings_for_props = get_formated_generation(slots.front()); + default_generation_settings_for_props["seed"] = -1; + batch = llama_batch_init(n_ctx, 0, params.n_parallel); // empty system prompt @@ -2614,7 +2618,8 @@ int main(int argc, char **argv) res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin")); json data = { { "user_name", llama.name_user.c_str() }, - { "assistant_name", llama.name_assistant.c_str() } + { "assistant_name", llama.name_assistant.c_str() }, + { "default_generation_settings", llama.default_generation_settings_for_props } }; res.set_content(data.dump(), "application/json; charset=utf-8"); }); From 6fdfa2ecc684000a25a4ad91823bc82a6652b645 Mon Sep 17 00:00:00 2001 From: Kawrakow <48489457+ikawrakow@users.noreply.github.com> Date: Mon, 5 Feb 2024 10:46:06 +0200 Subject: [PATCH 264/812] iq2_xxs: tune quantization (#5320) We get slightly better PPL, and we cut quantization time in nearly half. The trick is to 1st quantize without forcing points onto the E8-lattice. We can then use a narrower search range around the block scale that we got that way. Co-authored-by: Iwan Kawrakow --- ggml-quants.c | 58 ++++++--------------------------------------------- 1 file changed, 6 insertions(+), 52 deletions(-) diff --git a/ggml-quants.c b/ggml-quants.c index 8236385bc..014c0525a 100644 --- a/ggml-quants.c +++ b/ggml-quants.c @@ -9048,8 +9048,6 @@ static void quantize_row_iq2_xxs_impl(const float * restrict x, void * restrict int8_t L[32]; int8_t Laux[32]; float waux[32]; - bool is_on_grid[4]; - bool is_on_grid_aux[4]; uint8_t block_signs[4]; uint32_t q2[2*(QK_K/32)]; @@ -9099,10 +9097,11 @@ static void quantize_row_iq2_xxs_impl(const float * restrict x, void * restrict memset(L, 0, 32); continue; } + float scale = make_qp_quants(32, kMaxQ+1, xval, (uint8_t*)L, weight); + float eff_max = scale*kMaxQ; float best = 0; - float scale = max/(2*kMaxQ-1); - for (int is = -9; is <= 9; ++is) { - float id = (2*kMaxQ-1+is*0.1f)/max; + for (int is = -6; is <= 6; ++is) { + float id = (2*kMaxQ-1+is*0.1f)/eff_max; float this_scale = 1/id; for (int k = 0; k < 4; ++k) { for (int i = 0; i < 8; ++i) { @@ -9112,9 +9111,7 @@ static void quantize_row_iq2_xxs_impl(const float * restrict x, void * restrict uint16_t u = 0; for (int i = 0; i < 8; ++i) u |= (Laux[8*k+i] << 2*i); int grid_index = kmap_q2xs[u]; - is_on_grid_aux[k] = true; if (grid_index < 0) { - is_on_grid_aux[k] = false; const uint16_t * neighbours = kneighbors_q2xs - kmap_q2xs[u] - 1; grid_index = iq2_find_best_neighbour(neighbours, kgrid_q2xs, xval + 8*k, waux + 8*k, this_scale, Laux + 8*k); } @@ -9128,16 +9125,12 @@ static void quantize_row_iq2_xxs_impl(const float * restrict x, void * restrict } if (sumq2 > 0 && sumqx*sumqx > best*sumq2) { scale = sumqx/sumq2; best = scale*sumqx; - for (int i = 0; i < 32; ++i) L[i] = Laux[i]; - for (int k = 0; k < 4; ++k) is_on_grid[k] = is_on_grid_aux[k]; + memcpy(L, Laux, 32); } } - int n_not_ongrid = 0; - for (int k = 0; k < 4; ++k) if (!is_on_grid[k]) ++n_not_ongrid; - if (n_not_ongrid > 0 && scale > 0) { + if (scale > 0) { float id = 1/scale; for (int k = 0; k < 4; ++k) { - if (is_on_grid[k]) continue; uint16_t u = 0; for (int i = 0; i < 8; ++i) { int l = nearest_int(0.5f*(id*xval[8*k+i]-1)); @@ -9193,49 +9186,10 @@ static void quantize_row_iq2_xxs_impl(const float * restrict x, void * restrict float d = max_scale/31; y[ibl].d = GGML_FP32_TO_FP16(d); float id = 1/d; - float sumqx = 0, sumq2 = 0; for (int ib = 0; ib < QK_K/32; ++ib) { int l = nearest_int(0.5f*(id*scales[ib]-1)); l = MAX(0, MIN(15, l)); q2[2*ib+1] |= ((uint32_t)l << 28); - const float * xb = xbl + 32*ib; - const float * qw = quant_weights + QK_K*ibl + 32*ib; - for (int i = 0; i < 32; ++i) weight[i] = qw[i] * sqrtf(sigma2 + xb[i]*xb[i]); - const uint8_t * aux8 = (const uint8_t *)(q2 + 2*ib); - const float db = d * (1 + 2*l); - uint32_t u = 0; - for (int k = 0; k < 4; ++k) { - const int8_t * signs = keven_signs_q2xs + 8*((q2[2*ib+1] >> 7*k) & 127); - const float * xk = xb + 8*k; - const float * wk = weight + 8*k; - const uint8_t * grid = (const uint8_t *)(kgrid_q2xs + aux8[k]); - float best_mse = 0; int best_index = aux8[k]; - for (int j = 0; j < 8; ++j) { - float diff = db * grid[j] * signs[j] - xk[j]; - best_mse += wk[j] * diff * diff; - } - for (int idx = 0; idx < 256; ++idx) { - grid = (const uint8_t *)(kgrid_q2xs + idx); - float mse = 0; - for (int j = 0; j < 8; ++j) { - float diff = db * grid[j] * signs[j] - xk[j]; - mse += wk[j] * diff * diff; - } - if (mse < best_mse) { - best_mse = mse; best_index = idx; - } - } - u |= (best_index << 8*k); - grid = (const uint8_t *)(kgrid_q2xs + best_index); - //grid = (const uint8_t *)(kgrid_q2xs + aux8[k]); - for (int j = 0; j < 8; ++j) { - float q = db * grid[j] * signs[j]; - sumqx += wk[j] * q * xk[j]; - sumq2 += wk[j] * q * q; - } - } - q2[2*ib] = u; - if (sumq2 > 0) y[ibl].d = GGML_FP32_TO_FP16(d*sumqx/sumq2); } memcpy(y[ibl].qs, q2, QK_K/4); } From 7e1ae372f36d98fa66b1d778c5862904b4d80c88 Mon Sep 17 00:00:00 2001 From: Guoteng <32697156+SolenoidWGT@users.noreply.github.com> Date: Mon, 5 Feb 2024 17:04:06 +0800 Subject: [PATCH 265/812] py : fix internlm2-hf convert to gguf (#5305) * py : fix internlm2-hf convert to gguf * ggml-ci --- convert-hf-to-gguf.py | 29 +++++++++++++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/convert-hf-to-gguf.py b/convert-hf-to-gguf.py index a6ffd128b..5e343742d 100755 --- a/convert-hf-to-gguf.py +++ b/convert-hf-to-gguf.py @@ -1416,8 +1416,32 @@ class InternLM2Model(Model): self.gguf_writer.add_add_space_prefix(add_prefix) special_vocab = gguf.SpecialVocab(self.dir_model, n_vocab=len(tokens)) + old_eos = special_vocab.special_token_ids["eos"] + if "chat" in os.path.basename(self.dir_model.absolute()): + # For the chat model, we replace the eos with '<|im_end|>'. + special_vocab.special_token_ids["eos"] = self._try_get_sft_eos(tokenizer) + print(f"Replace eos:{old_eos} with a special token:{special_vocab.special_token_ids['eos']} \ +in chat mode so that the conversation can end normally.") + special_vocab.add_to_gguf(self.gguf_writer) + def _try_get_sft_eos(self, tokenizer): + unused_145_list = tokenizer.encode('[UNUSED_TOKEN_145]') + im_end_list = tokenizer.encode('<|im_end|>') + assert (len(unused_145_list) == 1) ^ (len(im_end_list) == 1) + if len(unused_145_list) == 1: + eos_token = unused_145_list[0] + if len(im_end_list) == 1: + eos_token = im_end_list[0] + return eos_token + + def _hf_permute_qk(self, weights, n_head: int, n_head_kv: int): + if n_head_kv is not None and n_head != n_head_kv: + n_head = n_head_kv + return (weights.reshape(n_head, 2, weights.shape[0] // n_head // 2, *weights.shape[1:]) + .swapaxes(1, 2) + .reshape(weights.shape)) + def set_gguf_parameters(self): self.gguf_writer.add_name("InternLM2") self.gguf_writer.add_context_length(self.hparams["max_position_embeddings"]) @@ -1486,8 +1510,9 @@ class InternLM2Model(Model): qkv = data_torch qkv = rearrange(qkv.T, " o (g n i) ->o g n i", g=num_groups, n=q_per_kv + 2, i=head_dim) q, k, v = qkv[..., : q_per_kv, :], qkv[..., q_per_kv: q_per_kv + 1, :], qkv[..., q_per_kv + 1: q_per_kv + 2, :] - q = rearrange(q, " o g n i -> o (g n i)").T - k = rearrange(k, " o g n i -> o (g n i)").T + # The model weights of q and k equire additional reshape. + q = self._hf_permute_qk(rearrange(q, " o g n i -> o (g n i)").T, num_heads, num_heads) + k = self._hf_permute_qk(rearrange(k, " o g n i -> o (g n i)").T, num_heads, num_kv_heads) v = rearrange(v, " o g n i -> o (g n i)").T self.post_write_tensors(tensor_map, f"model.layers.{bid}.attention.wq.weight", q) self.post_write_tensors(tensor_map, f"model.layers.{bid}.attention.wk.weight", k) From 89503dcb5f764a5cc7093db1f395f5121876a2cc Mon Sep 17 00:00:00 2001 From: Kawrakow <48489457+ikawrakow@users.noreply.github.com> Date: Mon, 5 Feb 2024 12:32:27 +0200 Subject: [PATCH 266/812] iq3_xxs: quards for the no-imatrix situation (#5334) Co-authored-by: Iwan Kawrakow --- llama.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/llama.cpp b/llama.cpp index 4787a92fe..65e399adc 100644 --- a/llama.cpp +++ b/llama.cpp @@ -9456,8 +9456,8 @@ static ggml_type get_k_quant_type(quantize_state_internal & qs, ggml_type new_ty else if (ftype == LLAMA_FTYPE_MOSTLY_Q2_K_S && qs.model.hparams.n_gqa() >= 4) { new_type = GGML_TYPE_Q4_K; } - else if (ftype == LLAMA_FTYPE_MOSTLY_IQ3_XXS && qs.model.hparams.n_gqa() >= 4) { - new_type = GGML_TYPE_Q4_K; + else if (ftype == LLAMA_FTYPE_MOSTLY_IQ3_XXS) { + new_type = qs.model.hparams.n_gqa() >= 4 ? GGML_TYPE_Q4_K : !qs.has_imatrix ? GGML_TYPE_Q3_K : GGML_TYPE_IQ3_XXS; } else if (ftype == LLAMA_FTYPE_MOSTLY_Q3_K_M) { new_type = qs.i_attention_wv < 2 ? GGML_TYPE_Q5_K : GGML_TYPE_Q4_K; @@ -9496,9 +9496,9 @@ static ggml_type get_k_quant_type(quantize_state_internal & qs, ggml_type new_ty else if (ftype == LLAMA_FTYPE_MOSTLY_Q2_K_S || ftype == LLAMA_FTYPE_MOSTLY_Q3_K_XS) { if (i_layer < n_layer/8) new_type = GGML_TYPE_Q4_K; } - //else if (ftype == LLAMA_FTYPE_MOSTLY_IQ3_XXS) { - // if (i_layer < n_layer/8) new_type = GGML_TYPE_Q5_K; - //} + else if (ftype == LLAMA_FTYPE_MOSTLY_IQ3_XXS && !qs.has_imatrix) { + new_type = i_layer < n_layer/8 ? GGML_TYPE_Q4_K : GGML_TYPE_Q3_K; + } else if (ftype == LLAMA_FTYPE_MOSTLY_Q3_K_M) { new_type = i_layer < n_layer/16 ? GGML_TYPE_Q5_K : arch != LLM_ARCH_FALCON || use_more_bits(i_layer, n_layer) ? GGML_TYPE_Q4_K From abb61944a5f64dec62c893ed0db10790169b672a Mon Sep 17 00:00:00 2001 From: "Dr. Tom Murphy VII Ph.D" <499244+tom7@users.noreply.github.com> Date: Mon, 5 Feb 2024 06:13:57 -0500 Subject: [PATCH 267/812] ggml : avoid duplicating function calls using MIN/MAX macros (#5325) * Avoid duplicating function calls when using MIN/MAX macros. Since these copy "a" and "b" they ask the compiler to evaluate one of them twice. The compiler doesn't have a problem with removing the duplication in something like MAX(0, x + 2), but in some cases we're calling functions, and those calls just happen twice. By explicitly evaluating at the expression we get smaller and faster code without duplicate calls. See ggml_rope_yarn_corr_dims in Compiler Explorer: https://godbolt.org/z/Ee4KMrvKh Code behaves exactly the same. * Update ggml.c --------- Co-authored-by: Georgi Gerganov --- ggml.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/ggml.c b/ggml.c index ee994c875..b9ec0c981 100644 --- a/ggml.c +++ b/ggml.c @@ -2470,7 +2470,8 @@ size_t ggml_get_max_tensor_size(const struct ggml_context * ctx) { size_t max_size = 0; for (struct ggml_tensor * tensor = ggml_get_first_tensor(ctx); tensor != NULL; tensor = ggml_get_next_tensor(ctx, tensor)) { - max_size = MAX(max_size, ggml_nbytes(tensor)); + size_t bytes = ggml_nbytes(tensor); + max_size = MAX(max_size, bytes); } return max_size; @@ -11887,8 +11888,10 @@ GGML_CALL void ggml_rope_yarn_corr_dims( int n_dims, int n_orig_ctx, float freq_base, float beta_fast, float beta_slow, float dims[2] ) { // start and end correction dims - dims[0] = MAX(0, floorf(ggml_rope_yarn_corr_dim(n_dims, n_orig_ctx, beta_fast, freq_base))); - dims[1] = MIN(n_dims - 1, ceilf(ggml_rope_yarn_corr_dim(n_dims, n_orig_ctx, beta_slow, freq_base))); + float start = floorf(ggml_rope_yarn_corr_dim(n_dims, n_orig_ctx, beta_fast, freq_base)); + float end = ceilf(ggml_rope_yarn_corr_dim(n_dims, n_orig_ctx, beta_slow, freq_base)); + dims[0] = MAX(0, start); + dims[1] = MIN(n_dims - 1, end); } static void ggml_compute_forward_rope_f32( From c6b395535a6874d749ef47c33eacd466cb252cd5 Mon Sep 17 00:00:00 2001 From: Kawrakow <48489457+ikawrakow@users.noreply.github.com> Date: Mon, 5 Feb 2024 14:09:47 +0200 Subject: [PATCH 268/812] ggml : make use of ggml-quants.h possible in C++ code (#5338) * Make use of ggml-quants.h possible in C++ code * One cannot possibly be defining static_assert in a C++ compilation --------- Co-authored-by: Iwan Kawrakow --- ggml-impl.h | 2 + ggml-quants.h | 117 +++++++++++++++++++++++++++----------------------- 2 files changed, 65 insertions(+), 54 deletions(-) diff --git a/ggml-impl.h b/ggml-impl.h index 2c58075ac..19df66bce 100644 --- a/ggml-impl.h +++ b/ggml-impl.h @@ -19,6 +19,7 @@ extern "C" { // fall back to the _Static_assert C11 keyword. // if C99 - static_assert is noop // ref: https://stackoverflow.com/a/53923785/4039976 +#ifndef __cplusplus #ifndef static_assert #if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201100L) #define static_assert(cond, msg) _Static_assert(cond, msg) @@ -26,6 +27,7 @@ extern "C" { #define static_assert(cond, msg) struct global_scope_noop_trick #endif #endif +#endif // __FMA__ and __F16C__ are not defined in MSVC, however they are implied with AVX2/AVX512 #if defined(_MSC_VER) && (defined(__AVX2__) || defined(__AVX512F__)) diff --git a/ggml-quants.h b/ggml-quants.h index 5c9f63bd9..bfdf3c997 100644 --- a/ggml-quants.h +++ b/ggml-quants.h @@ -191,70 +191,74 @@ typedef struct { } block_iq3_xxs; static_assert(sizeof(block_iq3_xxs) == sizeof(ggml_fp16_t) + 3*(QK_K/8), "wrong iq3_xxs block size/padding"); +#ifdef __cplusplus +extern "C" { +#endif + // Quantization -void quantize_row_q4_0_reference(const float * restrict x, block_q4_0 * restrict y, int k); -void quantize_row_q4_1_reference(const float * restrict x, block_q4_1 * restrict y, int k); -void quantize_row_q5_0_reference(const float * restrict x, block_q5_0 * restrict y, int k); -void quantize_row_q5_1_reference(const float * restrict x, block_q5_1 * restrict y, int k); -void quantize_row_q8_0_reference(const float * restrict x, block_q8_0 * restrict y, int k); -void quantize_row_q8_1_reference(const float * restrict x, block_q8_1 * restrict y, int k); +void quantize_row_q4_0_reference(const float * GGML_RESTRICT x, block_q4_0 * GGML_RESTRICT y, int k); +void quantize_row_q4_1_reference(const float * GGML_RESTRICT x, block_q4_1 * GGML_RESTRICT y, int k); +void quantize_row_q5_0_reference(const float * GGML_RESTRICT x, block_q5_0 * GGML_RESTRICT y, int k); +void quantize_row_q5_1_reference(const float * GGML_RESTRICT x, block_q5_1 * GGML_RESTRICT y, int k); +void quantize_row_q8_0_reference(const float * GGML_RESTRICT x, block_q8_0 * GGML_RESTRICT y, int k); +void quantize_row_q8_1_reference(const float * GGML_RESTRICT x, block_q8_1 * GGML_RESTRICT y, int k); -void quantize_row_q2_K_reference(const float * restrict x, block_q2_K * restrict y, int k); -void quantize_row_q3_K_reference(const float * restrict x, block_q3_K * restrict y, int k); -void quantize_row_q4_K_reference(const float * restrict x, block_q4_K * restrict y, int k); -void quantize_row_q5_K_reference(const float * restrict x, block_q5_K * restrict y, int k); -void quantize_row_q6_K_reference(const float * restrict x, block_q6_K * restrict y, int k); -void quantize_row_q8_K_reference(const float * restrict x, block_q8_K * restrict y, int k); -void quantize_row_iq3_xxs_reference(const float * restrict x, block_iq3_xxs * restrict y, int k); +void quantize_row_q2_K_reference(const float * GGML_RESTRICT x, block_q2_K * GGML_RESTRICT y, int k); +void quantize_row_q3_K_reference(const float * GGML_RESTRICT x, block_q3_K * GGML_RESTRICT y, int k); +void quantize_row_q4_K_reference(const float * GGML_RESTRICT x, block_q4_K * GGML_RESTRICT y, int k); +void quantize_row_q5_K_reference(const float * GGML_RESTRICT x, block_q5_K * GGML_RESTRICT y, int k); +void quantize_row_q6_K_reference(const float * GGML_RESTRICT x, block_q6_K * GGML_RESTRICT y, int k); +void quantize_row_q8_K_reference(const float * GGML_RESTRICT x, block_q8_K * GGML_RESTRICT y, int k); +void quantize_row_iq3_xxs_reference(const float * GGML_RESTRICT x, block_iq3_xxs * GGML_RESTRICT y, int k); -void quantize_row_q4_0(const float * restrict x, void * restrict y, int k); -void quantize_row_q4_1(const float * restrict x, void * restrict y, int k); -void quantize_row_q5_0(const float * restrict x, void * restrict y, int k); -void quantize_row_q5_1(const float * restrict x, void * restrict y, int k); -void quantize_row_q8_0(const float * restrict x, void * restrict y, int k); -void quantize_row_q8_1(const float * restrict x, void * restrict y, int k); +void quantize_row_q4_0(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int k); +void quantize_row_q4_1(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int k); +void quantize_row_q5_0(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int k); +void quantize_row_q5_1(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int k); +void quantize_row_q8_0(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int k); +void quantize_row_q8_1(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int k); -void quantize_row_q2_K(const float * restrict x, void * restrict y, int k); -void quantize_row_q3_K(const float * restrict x, void * restrict y, int k); -void quantize_row_q4_K(const float * restrict x, void * restrict y, int k); -void quantize_row_q5_K(const float * restrict x, void * restrict y, int k); -void quantize_row_q6_K(const float * restrict x, void * restrict y, int k); -void quantize_row_q8_K(const float * restrict x, void * restrict y, int k); -void quantize_row_iq3_xxs(const float * restrict x, void * restrict y, int k); +void quantize_row_q2_K(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int k); +void quantize_row_q3_K(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int k); +void quantize_row_q4_K(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int k); +void quantize_row_q5_K(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int k); +void quantize_row_q6_K(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int k); +void quantize_row_q8_K(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int k); +void quantize_row_iq3_xxs(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int k); // Dequantization -void dequantize_row_q4_0(const block_q4_0 * restrict x, float * restrict y, int k); -void dequantize_row_q4_1(const block_q4_1 * restrict x, float * restrict y, int k); -void dequantize_row_q5_0(const block_q5_0 * restrict x, float * restrict y, int k); -void dequantize_row_q5_1(const block_q5_1 * restrict x, float * restrict y, int k); -void dequantize_row_q8_0(const block_q8_0 * restrict x, float * restrict y, int k); -//void dequantize_row_q8_1(const block_q8_1 * restrict x, float * restrict y, int k); +void dequantize_row_q4_0(const block_q4_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int k); +void dequantize_row_q4_1(const block_q4_1 * GGML_RESTRICT x, float * GGML_RESTRICT y, int k); +void dequantize_row_q5_0(const block_q5_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int k); +void dequantize_row_q5_1(const block_q5_1 * GGML_RESTRICT x, float * GGML_RESTRICT y, int k); +void dequantize_row_q8_0(const block_q8_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int k); +//void dequantize_row_q8_1(const block_q8_1 * GGML_RESTRICT x, float * GGML_RESTRICT y, int k); -void dequantize_row_q2_K(const block_q2_K * restrict x, float * restrict y, int k); -void dequantize_row_q3_K(const block_q3_K * restrict x, float * restrict y, int k); -void dequantize_row_q4_K(const block_q4_K * restrict x, float * restrict y, int k); -void dequantize_row_q5_K(const block_q5_K * restrict x, float * restrict y, int k); -void dequantize_row_q6_K(const block_q6_K * restrict x, float * restrict y, int k); -void dequantize_row_q8_K(const block_q8_K * restrict x, float * restrict y, int k); -void dequantize_row_iq2_xxs(const block_iq2_xxs * restrict x, float * restrict y, int k); -void dequantize_row_iq2_xs (const block_iq2_xs * restrict x, float * restrict y, int k); -void dequantize_row_iq3_xxs(const block_iq3_xxs * restrict x, float * restrict y, int k); +void dequantize_row_q2_K(const block_q2_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int k); +void dequantize_row_q3_K(const block_q3_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int k); +void dequantize_row_q4_K(const block_q4_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int k); +void dequantize_row_q5_K(const block_q5_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int k); +void dequantize_row_q6_K(const block_q6_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int k); +void dequantize_row_q8_K(const block_q8_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int k); +void dequantize_row_iq2_xxs(const block_iq2_xxs * GGML_RESTRICT x, float * GGML_RESTRICT y, int k); +void dequantize_row_iq2_xs (const block_iq2_xs * GGML_RESTRICT x, float * GGML_RESTRICT y, int k); +void dequantize_row_iq3_xxs(const block_iq3_xxs * GGML_RESTRICT x, float * GGML_RESTRICT y, int k); // Dot product -void ggml_vec_dot_q4_0_q8_0(int n, float * restrict s, const void * restrict vx, const void * restrict vy); -void ggml_vec_dot_q4_1_q8_1(int n, float * restrict s, const void * restrict vx, const void * restrict vy); -void ggml_vec_dot_q5_0_q8_0(int n, float * restrict s, const void * restrict vx, const void * restrict vy); -void ggml_vec_dot_q5_1_q8_1(int n, float * restrict s, const void * restrict vx, const void * restrict vy); -void ggml_vec_dot_q8_0_q8_0(int n, float * restrict s, const void * restrict vx, const void * restrict vy); +void ggml_vec_dot_q4_0_q8_0(int n, float * GGML_RESTRICT s, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy); +void ggml_vec_dot_q4_1_q8_1(int n, float * GGML_RESTRICT s, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy); +void ggml_vec_dot_q5_0_q8_0(int n, float * GGML_RESTRICT s, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy); +void ggml_vec_dot_q5_1_q8_1(int n, float * GGML_RESTRICT s, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy); +void ggml_vec_dot_q8_0_q8_0(int n, float * GGML_RESTRICT s, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy); -void ggml_vec_dot_q2_K_q8_K(int n, float * restrict s, const void * restrict vx, const void * restrict vy); -void ggml_vec_dot_q3_K_q8_K(int n, float * restrict s, const void * restrict vx, const void * restrict vy); -void ggml_vec_dot_q4_K_q8_K(int n, float * restrict s, const void * restrict vx, const void * restrict vy); -void ggml_vec_dot_q5_K_q8_K(int n, float * restrict s, const void * restrict vx, const void * restrict vy); -void ggml_vec_dot_q6_K_q8_K(int n, float * restrict s, const void * restrict vx, const void * restrict vy); -void ggml_vec_dot_iq2_xxs_q8_K(int n, float * restrict s, const void * restrict vx, const void * restrict vy); -void ggml_vec_dot_iq2_xs_q8_K (int n, float * restrict s, const void * restrict vx, const void * restrict vy); -void ggml_vec_dot_iq3_xxs_q8_K(int n, float * restrict s, const void * restrict vx, const void * restrict vy); +void ggml_vec_dot_q2_K_q8_K(int n, float * GGML_RESTRICT s, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy); +void ggml_vec_dot_q3_K_q8_K(int n, float * GGML_RESTRICT s, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy); +void ggml_vec_dot_q4_K_q8_K(int n, float * GGML_RESTRICT s, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy); +void ggml_vec_dot_q5_K_q8_K(int n, float * GGML_RESTRICT s, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy); +void ggml_vec_dot_q6_K_q8_K(int n, float * GGML_RESTRICT s, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy); +void ggml_vec_dot_iq2_xxs_q8_K(int n, float * GGML_RESTRICT s, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy); +void ggml_vec_dot_iq2_xs_q8_K (int n, float * GGML_RESTRICT s, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy); +void ggml_vec_dot_iq3_xxs_q8_K(int n, float * GGML_RESTRICT s, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy); // // Quantization utilizing an importance matrix (a.k.a. "Activation aWare Quantization") @@ -276,3 +280,8 @@ void iq2xs_init_impl(int grid_size); void iq2xs_free_impl(int grid_size); void iq3xs_init_impl(int grid_size); void iq3xs_free_impl(int grid_size); + +#ifdef __cplusplus +} +#endif + From 78b00dda6c0d62c34f5371d47718defff6ed2b22 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20G=C3=A4=C3=9Fler?= Date: Mon, 5 Feb 2024 15:55:10 +0100 Subject: [PATCH 269/812] README: updated introduction (#5343) * README: updated introduction * readme : update --------- Co-authored-by: Georgi Gerganov --- README.md | 49 ++++++++++++++++++++++++++++++------------------- 1 file changed, 30 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index a6fe34629..bb6c49338 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ [Roadmap](https://github.com/users/ggerganov/projects/7) / [Project status](https://github.com/ggerganov/llama.cpp/discussions/3471) / [Manifesto](https://github.com/ggerganov/llama.cpp/discussions/205) / [ggml](https://github.com/ggerganov/ggml) -Inference of [LLaMA](https://arxiv.org/abs/2302.13971) model in pure C/C++ +Inference of Meta's [LLaMA](https://arxiv.org/abs/2302.13971) model (and others) in pure C/C++ ### Hot topics @@ -58,18 +58,20 @@ Inference of [LLaMA](https://arxiv.org/abs/2302.13971) model in pure C/C++ ## Description -The main goal of `llama.cpp` is to run the LLaMA model using 4-bit integer quantization on a MacBook +The main goal of `llama.cpp` is to enable LLM inference with minimal setup and state-of-the-art performance on a wide +variety of hardware - locally and in the cloud. -- Plain C/C++ implementation without dependencies -- Apple silicon first-class citizen - optimized via ARM NEON, Accelerate and Metal frameworks +- Plain C/C++ implementation without any dependencies +- Apple silicon is a first-class citizen - optimized via ARM NEON, Accelerate and Metal frameworks - AVX, AVX2 and AVX512 support for x86 architectures -- Mixed F16 / F32 precision -- 2-bit, 3-bit, 4-bit, 5-bit, 6-bit and 8-bit integer quantization support -- CUDA, Metal, OpenCL, SYCL GPU backend support +- 2-bit, 3-bit, 4-bit, 5-bit, 6-bit, and 8-bit integer quantization for faster inference and reduced memory use +- Custom CUDA kernels for running LLMs on NVIDIA GPUs (support for AMD GPUs via HIP) +- Vulkan, SYCL, and (partial) OpenCL backend support +- CPU+GPU hybrid inference to partially accelerate models larger than the total VRAM capacity -The original implementation of `llama.cpp` was [hacked in an evening](https://github.com/ggerganov/llama.cpp/issues/33#issuecomment-1465108022). -Since then, the project has improved significantly thanks to many contributions. This project is mainly for educational purposes and serves -as the main playground for developing new features for the [ggml](https://github.com/ggerganov/ggml) library. +Since its [inception](https://github.com/ggerganov/llama.cpp/issues/33#issuecomment-1465108022), the project has +improved significantly thanks to many contributions. It is the main playground for developing new features for the +[ggml](https://github.com/ggerganov/ggml) library. **Supported platforms:** @@ -77,11 +79,14 @@ as the main playground for developing new features for the [ggml](https://github - [X] Linux - [X] Windows (via CMake) - [X] Docker +- [X] FreeBSD **Supported models:** - [X] LLaMA 🦙 - [x] LLaMA 2 🦙🦙 +- [X] [Mistral AI v0.1](https://huggingface.co/mistralai/Mistral-7B-v0.1) +- [x] [Mixtral MoE](https://huggingface.co/models?search=mistral-ai/Mixtral) - [X] Falcon - [X] [Alpaca](https://github.com/ggerganov/llama.cpp#instruction-mode-with-alpaca) - [X] [GPT4All](https://github.com/ggerganov/llama.cpp#using-gpt4all) @@ -95,7 +100,6 @@ as the main playground for developing new features for the [ggml](https://github - [X] [Baichuan 1 & 2](https://huggingface.co/models?search=baichuan-inc/Baichuan) + [derivations](https://huggingface.co/hiyouga/baichuan-7b-sft) - [X] [Aquila 1 & 2](https://huggingface.co/models?search=BAAI/Aquila) - [X] [Starcoder models](https://github.com/ggerganov/llama.cpp/pull/3187) -- [X] [Mistral AI v0.1](https://huggingface.co/mistralai/Mistral-7B-v0.1) - [X] [Refact](https://huggingface.co/smallcloudai/Refact-1_6B-fim) - [X] [Persimmon 8B](https://github.com/ggerganov/llama.cpp/pull/3410) - [X] [MPT](https://github.com/ggerganov/llama.cpp/pull/3417) @@ -104,15 +108,14 @@ as the main playground for developing new features for the [ggml](https://github - [X] [StableLM-3b-4e1t](https://github.com/ggerganov/llama.cpp/pull/3586) - [x] [Deepseek models](https://huggingface.co/models?search=deepseek-ai/deepseek) - [x] [Qwen models](https://huggingface.co/models?search=Qwen/Qwen) -- [x] [Mixtral MoE](https://huggingface.co/models?search=mistral-ai/Mixtral) - [x] [PLaMo-13B](https://github.com/ggerganov/llama.cpp/pull/3557) - [x] [GPT-2](https://huggingface.co/gpt2) - [x] [CodeShell](https://github.com/WisdomShell/codeshell) **Multimodal models:** -- [x] [Llava 1.5 models](https://huggingface.co/collections/liuhaotian/llava-15-653aac15d994e992e2677a7e) -- [x] [Bakllava](https://huggingface.co/models?search=SkunkworksAI/Bakllava) +- [x] [LLaVA 1.5 models](https://huggingface.co/collections/liuhaotian/llava-15-653aac15d994e992e2677a7e) +- [x] [BakLLaVA](https://huggingface.co/models?search=SkunkworksAI/Bakllava) - [x] [Obsidian](https://huggingface.co/NousResearch/Obsidian-3B-V0.5) - [x] [ShareGPT4V](https://huggingface.co/models?search=Lin-Chen/ShareGPT4V) - [x] [MobileVLM 1.7B/3B models](https://huggingface.co/models?search=mobileVLM) @@ -137,14 +140,22 @@ as the main playground for developing new features for the [ggml](https://github **UI:** +Unless otherwise noted these projects are open-source with permissive licensing: + +- [iohub/collama](https://github.com/iohub/coLLaMA) +- [janhq/jan](https://github.com/janhq/jan) (AGPL) - [nat/openplayground](https://github.com/nat/openplayground) -- [oobabooga/text-generation-webui](https://github.com/oobabooga/text-generation-webui) -- [withcatai/catai](https://github.com/withcatai/catai) -- [semperai/amica](https://github.com/semperai/amica) +- [LMStudio](https://lmstudio.ai/) (proprietary) +- [LostRuins/koboldcpp](https://github.com/LostRuins/koboldcpp) (AGPL) +- [Mozilla-Ocho/llamafile](https://github.com/Mozilla-Ocho/llamafile) +- [nomic-ai/gpt4all](https://github.com/nomic-ai/gpt4all) +- [ollama/ollama](https://github.com/ollama/ollama) +- [oobabooga/text-generation-webui](https://github.com/oobabooga/text-generation-webui) (AGPL) - [psugihara/FreeChat](https://github.com/psugihara/FreeChat) - [ptsochantaris/emeltal](https://github.com/ptsochantaris/emeltal) -- [iohub/collama](https://github.com/iohub/coLLaMA) -- [pythops/tenere](https://github.com/pythops/tenere) +- [pythops/tenere](https://github.com/pythops/tenere) (AGPL) +- [semperai/amica](https://github.com/semperai/amica) +- [withcatai/catai](https://github.com/withcatai/catai) --- From 098f6d737b65134cf220d12b9b706e8cfc5e4610 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20G=C3=A4=C3=9Fler?= Date: Mon, 5 Feb 2024 19:33:00 +0100 Subject: [PATCH 270/812] make: Use ccache for faster compilation (#5318) * make: Use ccache for faster compilation --- CMakeLists.txt | 4 +- Makefile | 169 ++++++++++++++++++++++++++++++++++--------------- 2 files changed, 121 insertions(+), 52 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 8c04e4c19..427015be5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -809,9 +809,9 @@ if (LLAMA_CCACHE) if (LLAMA_CCACHE_FOUND) set_property(GLOBAL PROPERTY RULE_LAUNCH_COMPILE ccache) set(ENV{CCACHE_SLOPPINESS} time_macros) - message(STATUS "Using ccache") + message(STATUS "ccache found, compilation results will be cached. Disable with LLAMA_CCACHE=OFF.") else() - message(STATUS "Warning: ccache not found - consider installing it or use LLAMA_CCACHE=OFF") + message(STATUS "Warning: ccache not found - consider installing it for faster compilation or disable this warning with LLAMA_CCACHE=OFF") endif () endif() diff --git a/Makefile b/Makefile index 21d5e15ba..ba73f0637 100644 --- a/Makefile +++ b/Makefile @@ -112,6 +112,18 @@ MK_CXXFLAGS += -O3 MK_NVCCFLAGS += -O3 endif +ifndef LLAMA_NO_CCACHE +CCACHE := $(shell which ccache) +ifdef CCACHE +export CCACHE_SLOPPINESS = time_macros +$(info I ccache found, compilation results will be cached. Disable with LLAMA_NO_CCACHE.) +CC := $(CCACHE) $(CC) +CXX := $(CCACHE) $(CXX) +else +$(info I ccache not found. Consider installing it for faster compilation.) +endif # CCACHE +endif # LLAMA_NO_CCACHE + # clock_gettime came in POSIX.1b (1993) # CLOCK_MONOTONIC came in POSIX.1-2001 / SUSv3 as optional # posix_memalign came in POSIX.1-2001 / SUSv3 @@ -374,9 +386,9 @@ ifdef LLAMA_DEBUG MK_NVCCFLAGS += -lineinfo endif # LLAMA_DEBUG ifdef LLAMA_CUDA_NVCC - NVCC = $(LLAMA_CUDA_NVCC) + NVCC = $(CCACHE) $(LLAMA_CUDA_NVCC) else - NVCC = nvcc + NVCC = $(CCACHE) nvcc endif #LLAMA_CUDA_NVCC ifdef CUDA_DOCKER_ARCH MK_NVCCFLAGS += -Wno-deprecated-gpu-targets -arch=$(CUDA_DOCKER_ARCH) @@ -483,7 +495,7 @@ ifdef LLAMA_HIPBLAS ROCM_PATH ?= /opt/rocm GPU_TARGETS ?= $(shell $(ROCM_PATH)/llvm/bin/amdgpu-arch) endif - HIPCC ?= $(ROCM_PATH)/bin/hipcc + HIPCC ?= $(CCACHE) $(ROCM_PATH)/bin/hipcc LLAMA_CUDA_DMMV_X ?= 32 LLAMA_CUDA_MMV_Y ?= 1 LLAMA_CUDA_KQUANTS_ITER ?= 2 @@ -607,97 +619,135 @@ libllama.a: llama.o ggml.o $(OBJS) $(COMMON_DEPS) clean: rm -vrf *.o tests/*.o *.so *.a *.dll benchmark-matmult common/build-info.cpp *.dot $(COV_TARGETS) $(BUILD_TARGETS) $(TEST_TARGETS) + find examples pocs -type f -name "*.o" -delete # # Examples # +# $< is the first prerequisite, i.e. the source file. +# Explicitly compile this to an object file so that it can be cached with ccache. +# The source file is then filtered out from $^ (the list of all prerequisites) and the object file is added instead. + +# Helper function that replaces .c, .cpp, and .cu file endings with .o: +GET_OBJ_FILE = $(patsubst %.c,%.o,$(patsubst %.cpp,%.o,$(patsubst %.cu,%.o,$(1)))) + main: examples/main/main.cpp ggml.o llama.o $(COMMON_DEPS) console.o grammar-parser.o $(OBJS) - $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) + $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) + $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) @echo @echo '==== Run ./main -h for help. ====' @echo infill: examples/infill/infill.cpp ggml.o llama.o $(COMMON_DEPS) console.o grammar-parser.o $(OBJS) - $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) + $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) + $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) simple: examples/simple/simple.cpp ggml.o llama.o $(COMMON_DEPS) $(OBJS) - $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) + $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) + $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) tokenize: examples/tokenize/tokenize.cpp ggml.o llama.o $(COMMON_DEPS) $(OBJS) - $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) + $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) + $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) batched: examples/batched/batched.cpp ggml.o llama.o $(COMMON_DEPS) $(OBJS) - $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) + $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) + $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) batched-bench: examples/batched-bench/batched-bench.cpp build-info.o ggml.o llama.o common.o $(OBJS) - $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) + $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) + $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) quantize: examples/quantize/quantize.cpp build-info.o ggml.o llama.o $(OBJS) - $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) + $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) + $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) quantize-stats: examples/quantize-stats/quantize-stats.cpp build-info.o ggml.o llama.o $(OBJS) - $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) + $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) + $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) perplexity: examples/perplexity/perplexity.cpp ggml.o llama.o $(COMMON_DEPS) $(OBJS) - $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) + $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) + $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) imatrix: examples/imatrix/imatrix.cpp ggml.o llama.o $(COMMON_DEPS) $(OBJS) - $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) + $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) + $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) embedding: examples/embedding/embedding.cpp ggml.o llama.o $(COMMON_DEPS) $(OBJS) - $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) + $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) + $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) save-load-state: examples/save-load-state/save-load-state.cpp ggml.o llama.o $(COMMON_DEPS) $(OBJS) - $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) + $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) + $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) server: examples/server/server.cpp examples/server/oai.hpp examples/server/utils.hpp examples/server/httplib.h examples/server/json.hpp examples/server/index.html.hpp examples/server/index.js.hpp examples/server/completion.js.hpp examples/llava/clip.cpp examples/llava/clip.h common/stb_image.h ggml.o llama.o $(COMMON_DEPS) grammar-parser.o $(OBJS) - $(CXX) $(CXXFLAGS) -Iexamples/server $(filter-out %.h,$(filter-out %.hpp,$^)) -o $@ $(LDFLAGS) $(LWINSOCK2) -Wno-cast-qual + $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) + $(CXX) $(CXXFLAGS) -c examples/llava/clip.cpp -o $(call GET_OBJ_FILE, examples/llava/clip.cpp) -Wno-cast-qual + $(CXX) $(CXXFLAGS) -Iexamples/server $(filter-out %.h %.hpp $< examples/llava/clip.cpp,$^) $(call GET_OBJ_FILE, $<) $(call GET_OBJ_FILE, examples/llava/clip.cpp) -o $@ $(LDFLAGS) $(LWINSOCK2) gguf: examples/gguf/gguf.cpp ggml.o $(OBJS) - $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) + $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) + $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) train-text-from-scratch: examples/train-text-from-scratch/train-text-from-scratch.cpp ggml.o llama.o $(COMMON_DEPS) train.o $(OBJS) - $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) + $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) + $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) convert-llama2c-to-ggml: examples/convert-llama2c-to-ggml/convert-llama2c-to-ggml.cpp ggml.o llama.o $(OBJS) - $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) + $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) + $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) llama-bench: examples/llama-bench/llama-bench.cpp ggml.o llama.o $(COMMON_DEPS) $(OBJS) - $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) + $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) + $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) libllava.a: examples/llava/llava.cpp examples/llava/llava.h examples/llava/clip.cpp examples/llava/clip.h common/stb_image.h common/base64.hpp ggml.o llama.o $(COMMON_DEPS) $(OBJS) $(CXX) $(CXXFLAGS) -static -fPIC -c $< -o $@ -Wno-cast-qual llava-cli: examples/llava/llava-cli.cpp examples/llava/clip.h examples/llava/clip.cpp examples/llava/llava.h examples/llava/llava.cpp ggml.o llama.o $(COMMON_DEPS) $(OBJS) - $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) -Wno-cast-qual + $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) + $(CXX) $(CXXFLAGS) -c examples/llava/clip.cpp -o $(call GET_OBJ_FILE, examples/llava/clip.cpp) -Wno-cast-qual + $(CXX) $(CXXFLAGS) -c examples/llava/llava.cpp -o $(call GET_OBJ_FILE, examples/llava/llava.cpp) + $(CXX) $(CXXFLAGS) $(filter-out %.h $< examples/llava/clip.cpp examples/llava/llava.cpp,$^) $(call GET_OBJ_FILE, $<) $(call GET_OBJ_FILE, examples/llava/clip.cpp) $(call GET_OBJ_FILE, examples/llava/llava.cpp) -o $@ $(LDFLAGS) baby-llama: examples/baby-llama/baby-llama.cpp ggml.o llama.o $(COMMON_DEPS) train.o $(OBJS) - $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) + $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) + $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) beam-search: examples/beam-search/beam-search.cpp ggml.o llama.o $(COMMON_DEPS) $(OBJS) - $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) + $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) + $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) finetune: examples/finetune/finetune.cpp ggml.o llama.o $(COMMON_DEPS) train.o $(OBJS) - $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) + $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) + $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) export-lora: examples/export-lora/export-lora.cpp ggml.o common/common.h $(OBJS) - $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) + $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) + $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) speculative: examples/speculative/speculative.cpp ggml.o llama.o $(COMMON_DEPS) grammar-parser.o $(OBJS) - $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) + $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) + $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) parallel: examples/parallel/parallel.cpp ggml.o llama.o $(COMMON_DEPS) $(OBJS) - $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) + $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) + $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) lookahead: examples/lookahead/lookahead.cpp ggml.o llama.o $(COMMON_DEPS) $(OBJS) - $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) + $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) + $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) lookup: examples/lookup/lookup.cpp ggml.o llama.o $(COMMON_DEPS) $(OBJS) - $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) + $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) + $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) passkey: examples/passkey/passkey.cpp ggml.o llama.o $(COMMON_DEPS) $(OBJS) - $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) + $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) + $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) ifeq ($(UNAME_S),Darwin) swift: examples/batched.swift @@ -705,7 +755,7 @@ swift: examples/batched.swift endif common/build-info.cpp: $(wildcard .git/index) scripts/build-info.sh - @sh scripts/build-info.sh $(CC) > $@.tmp + @sh scripts/build-info.sh "$(CC)" > $@.tmp @if ! cmp -s $@.tmp $@; then \ mv $@.tmp $@; \ else \ @@ -722,7 +772,8 @@ build-info.o: common/build-info.cpp tests: $(TEST_TARGETS) benchmark-matmult: examples/benchmark/benchmark-matmult.cpp build-info.o ggml.o $(OBJS) - $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) + $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) + $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) run-benchmark-matmult: benchmark-matmult ./$@ @@ -730,58 +781,76 @@ run-benchmark-matmult: benchmark-matmult .PHONY: run-benchmark-matmult swift vdot: pocs/vdot/vdot.cpp ggml.o $(OBJS) - $(CXX) $(CXXFLAGS) $^ -o $@ $(LDFLAGS) + $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) + $(CXX) $(CXXFLAGS) $(filter-out $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) q8dot: pocs/vdot/q8dot.cpp ggml.o $(OBJS) - $(CXX) $(CXXFLAGS) $^ -o $@ $(LDFLAGS) + $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) + $(CXX) $(CXXFLAGS) $(filter-out $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) tests/test-llama-grammar: tests/test-llama-grammar.cpp ggml.o grammar-parser.o $(OBJS) - $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) + $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) + $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) tests/test-grammar-parser: tests/test-grammar-parser.cpp ggml.o llama.o grammar-parser.o $(OBJS) - $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) + $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) + $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) tests/test-double-float: tests/test-double-float.cpp ggml.o $(OBJS) - $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) + $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) + $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) tests/test-grad0: tests/test-grad0.cpp ggml.o $(OBJS) - $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) + $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) + $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) tests/test-opt: tests/test-opt.cpp ggml.o $(OBJS) - $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) + $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) + $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) tests/test-quantize-fns: tests/test-quantize-fns.cpp ggml.o $(OBJS) - $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) + $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) + $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) tests/test-quantize-perf: tests/test-quantize-perf.cpp ggml.o $(OBJS) - $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) + $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) + $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) tests/test-sampling: tests/test-sampling.cpp ggml.o llama.o $(OBJS) - $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) + $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) + $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) tests/test-tokenizer-0-falcon: tests/test-tokenizer-0-falcon.cpp ggml.o llama.o $(COMMON_DEPS) console.o $(OBJS) - $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) + $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) + $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) tests/test-tokenizer-0-llama: tests/test-tokenizer-0-llama.cpp ggml.o llama.o $(COMMON_DEPS) console.o $(OBJS) - $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) + $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) + $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) tests/test-tokenizer-1-bpe: tests/test-tokenizer-1-bpe.cpp ggml.o llama.o $(COMMON_DEPS) console.o $(OBJS) - $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) + $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) + $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) tests/test-tokenizer-1-llama: tests/test-tokenizer-1-llama.cpp ggml.o llama.o $(COMMON_DEPS) console.o $(OBJS) - $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) + $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) + $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) tests/test-rope: tests/test-rope.cpp ggml.o $(OBJS) - $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) + $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) + $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) tests/test-c.o: tests/test-c.c llama.h $(CC) $(CFLAGS) -c $(filter-out %.h,$^) -o $@ tests/test-backend-ops: tests/test-backend-ops.cpp ggml.o $(OBJS) - $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) + $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) + $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) tests/test-model-load-cancel: tests/test-model-load-cancel.cpp ggml.o llama.o tests/get-model.cpp $(COMMON_DEPS) $(OBJS) - $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) + $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) + $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) tests/test-autorelease: tests/test-autorelease.cpp ggml.o llama.o tests/get-model.cpp $(COMMON_DEPS) $(OBJS) - $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) + $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) + $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) From 906cff55c2848fda091d888a1585915ec0c9ea9e Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Tue, 6 Feb 2024 07:47:22 +0200 Subject: [PATCH 271/812] py : handle byte tokens in `get_token_type` (#5341) * py : handle byte tokens in `get_token_type` * py : fix empty bytes arg --- convert.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/convert.py b/convert.py index 75c100118..4a2847a27 100755 --- a/convert.py +++ b/convert.py @@ -515,10 +515,14 @@ class HfVocab: # Yield token text, score, and type yield token_text, self.get_token_score(token_id), self.get_token_type( - token_id, self.special_ids # Reuse already stored special IDs + token_id, token_text, self.special_ids # Reuse already stored special IDs ) - def get_token_type(self, token_id: int, special_ids: set[int]) -> gguf.TokenType: + def get_token_type(self, token_id: int, token_text: bytes, special_ids: set[int]) -> gguf.TokenType: + # Special case for byte tokens + if re.fullmatch(br"<0x[0-9A-Fa-f]{2}>", token_text): + return gguf.TokenType.BYTE + # Determine token type based on whether it's a special token return gguf.TokenType.CONTROL if token_id in special_ids else gguf.TokenType.NORMAL @@ -530,7 +534,7 @@ class HfVocab: def added_tokens(self) -> Iterable[tuple[bytes, float, gguf.TokenType]]: for text in self.added_tokens_list: if text in self.specials: - toktype = self.get_token_type(self.specials[text], self.special_ids) + toktype = self.get_token_type(self.specials[text], b'', self.special_ids) score = self.get_token_score(self.specials[text]) else: toktype = gguf.TokenType.USER_DEFINED From 4ffc7a17d4e80c5f3f905139cb570ed9b6934fcb Mon Sep 17 00:00:00 2001 From: Niall Coates <1349685+Niall-@users.noreply.github.com> Date: Tue, 6 Feb 2024 08:16:23 +0000 Subject: [PATCH 272/812] server : various fixes for the prompt field in /completion (#5300) server : fix deadlock when prompt array contains strings and numbers server : removed an unnecessary generation when generating multi-prompts server : removed an unnecessary assert --- examples/server/server.cpp | 34 +++++++++++++++++++++++++++------- 1 file changed, 27 insertions(+), 7 deletions(-) diff --git a/examples/server/server.cpp b/examples/server/server.cpp index 8000fee5c..fc7e723a1 100644 --- a/examples/server/server.cpp +++ b/examples/server/server.cpp @@ -1163,13 +1163,30 @@ struct llama_server_context task.multitask_id = multitask_id; // when a completion task's prompt array is not a singleton, we split it into multiple requests - if (task.data.count("prompt") && task.data.at("prompt").size() > 1) - { - split_multiprompt_task(task_id, task); - } - // otherwise, it's a single-prompt task, we actually queue it - queue_tasks.post(task); + // if there's numbers in the prompt array it will be treated as an array of tokens + if (task.data.count("prompt") != 0 && task.data.at("prompt").size() > 1) { + bool numbers = false; + for (const auto& e : task.data.at("prompt")) { + if (e.is_number()) { + numbers = true; + break; + } + } + + // NOTE: split_multiprompt_task() does not handle a mix of strings and numbers, + // it will completely stall the server. I don't know where the bug for this is. + // + // if there are numbers, it needs to be treated like a single prompt, + // queue_tasks handles a mix of strings and numbers just fine. + if (numbers) { + queue_tasks.post(task); + } else { + split_multiprompt_task(task_id, task); + } + } else { + queue_tasks.post(task); + } } // for multiple images processing @@ -1251,7 +1268,10 @@ struct llama_server_context void split_multiprompt_task(int multitask_id, task_server& multiprompt_task) { int prompt_count = multiprompt_task.data.at("prompt").size(); - assert(prompt_count > 1); + if (prompt_count <= 1) { + send_error(multiprompt_task, "error while handling multiple prompts"); + return; + } // generate all the ID for subtask std::vector subtask_ids(prompt_count); From 31e790322133a4b1d0684527ea446e765e8a96cf Mon Sep 17 00:00:00 2001 From: Michael Coppola Date: Tue, 6 Feb 2024 04:20:00 -0500 Subject: [PATCH 273/812] server : add `dynatemp_range` and `dynatemp_exponent` (#5352) * server: added `dynatemp_range` and `dynatemp_exponent` * Update README.md --------- Co-authored-by: Michael Coppola --- examples/server/README.md | 4 ++++ examples/server/server.cpp | 46 +++++++++++++++++++++----------------- 2 files changed, 29 insertions(+), 21 deletions(-) diff --git a/examples/server/README.md b/examples/server/README.md index d8e7c313e..46d8f85ae 100644 --- a/examples/server/README.md +++ b/examples/server/README.md @@ -137,6 +137,10 @@ node index.js `temperature`: Adjust the randomness of the generated text (default: 0.8). + `dynatemp_range`: Dynamic temperature range (default: 0.0, 0.0 = disabled). + + `dynatemp_exponent`: Dynamic temperature exponent (default: 1.0). + `top_k`: Limit the next token selection to the K most probable tokens (default: 40). `top_p`: Limit the next token selection to a subset of tokens with a cumulative probability above a threshold P (default: 0.95). diff --git a/examples/server/server.cpp b/examples/server/server.cpp index fc7e723a1..e48a1da75 100644 --- a/examples/server/server.cpp +++ b/examples/server/server.cpp @@ -524,27 +524,29 @@ struct llama_server_context slot->oaicompat_model = ""; } - slot->params.stream = json_value(data, "stream", false); - slot->params.cache_prompt = json_value(data, "cache_prompt", false); - slot->params.n_predict = json_value(data, "n_predict", default_params.n_predict); - slot->sparams.top_k = json_value(data, "top_k", default_sparams.top_k); - slot->sparams.top_p = json_value(data, "top_p", default_sparams.top_p); - slot->sparams.min_p = json_value(data, "min_p", default_sparams.min_p); - slot->sparams.tfs_z = json_value(data, "tfs_z", default_sparams.tfs_z); - slot->sparams.typical_p = json_value(data, "typical_p", default_sparams.typical_p); - slot->sparams.temp = json_value(data, "temperature", default_sparams.temp); - slot->sparams.penalty_last_n = json_value(data, "repeat_last_n", default_sparams.penalty_last_n); - slot->sparams.penalty_repeat = json_value(data, "repeat_penalty", default_sparams.penalty_repeat); - slot->sparams.penalty_freq = json_value(data, "frequency_penalty", default_sparams.penalty_freq); - slot->sparams.penalty_present = json_value(data, "presence_penalty", default_sparams.penalty_present); - slot->sparams.mirostat = json_value(data, "mirostat", default_sparams.mirostat); - slot->sparams.mirostat_tau = json_value(data, "mirostat_tau", default_sparams.mirostat_tau); - slot->sparams.mirostat_eta = json_value(data, "mirostat_eta", default_sparams.mirostat_eta); - slot->sparams.penalize_nl = json_value(data, "penalize_nl", default_sparams.penalize_nl); - slot->params.n_keep = json_value(data, "n_keep", slot->params.n_keep); - slot->params.seed = json_value(data, "seed", default_params.seed); - slot->sparams.grammar = json_value(data, "grammar", default_sparams.grammar); - slot->sparams.n_probs = json_value(data, "n_probs", default_sparams.n_probs); + slot->params.stream = json_value(data, "stream", false); + slot->params.cache_prompt = json_value(data, "cache_prompt", false); + slot->params.n_predict = json_value(data, "n_predict", default_params.n_predict); + slot->sparams.top_k = json_value(data, "top_k", default_sparams.top_k); + slot->sparams.top_p = json_value(data, "top_p", default_sparams.top_p); + slot->sparams.min_p = json_value(data, "min_p", default_sparams.min_p); + slot->sparams.tfs_z = json_value(data, "tfs_z", default_sparams.tfs_z); + slot->sparams.typical_p = json_value(data, "typical_p", default_sparams.typical_p); + slot->sparams.temp = json_value(data, "temperature", default_sparams.temp); + slot->sparams.dynatemp_range = json_value(data, "dynatemp_range", default_sparams.dynatemp_range); + slot->sparams.dynatemp_exponent = json_value(data, "dynatemp_exponent", default_sparams.dynatemp_exponent); + slot->sparams.penalty_last_n = json_value(data, "repeat_last_n", default_sparams.penalty_last_n); + slot->sparams.penalty_repeat = json_value(data, "repeat_penalty", default_sparams.penalty_repeat); + slot->sparams.penalty_freq = json_value(data, "frequency_penalty", default_sparams.penalty_freq); + slot->sparams.penalty_present = json_value(data, "presence_penalty", default_sparams.penalty_present); + slot->sparams.mirostat = json_value(data, "mirostat", default_sparams.mirostat); + slot->sparams.mirostat_tau = json_value(data, "mirostat_tau", default_sparams.mirostat_tau); + slot->sparams.mirostat_eta = json_value(data, "mirostat_eta", default_sparams.mirostat_eta); + slot->sparams.penalize_nl = json_value(data, "penalize_nl", default_sparams.penalize_nl); + slot->params.n_keep = json_value(data, "n_keep", slot->params.n_keep); + slot->params.seed = json_value(data, "seed", default_params.seed); + slot->sparams.grammar = json_value(data, "grammar", default_sparams.grammar); + slot->sparams.n_probs = json_value(data, "n_probs", default_sparams.n_probs); // infill if (data.count("input_prefix") != 0) @@ -1002,6 +1004,8 @@ struct llama_server_context {"model", params.model_alias}, {"seed", slot.params.seed}, {"temperature", slot.sparams.temp}, + {"dynatemp_range", slot.sparams.dynatemp_range}, + {"dynatemp_exponent", slot.sparams.dynatemp_exponent}, {"top_k", slot.sparams.top_k}, {"top_p", slot.sparams.top_p}, {"min_p", slot.sparams.min_p}, From 8a79c591de9b7ff3242a94f68b7fb5a17ed8c2be Mon Sep 17 00:00:00 2001 From: Justin Parker Date: Tue, 6 Feb 2024 04:20:59 -0500 Subject: [PATCH 274/812] server : include total "num_slots" in props endpoint (#5349) --- examples/server/server.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/examples/server/server.cpp b/examples/server/server.cpp index e48a1da75..d86d7e04a 100644 --- a/examples/server/server.cpp +++ b/examples/server/server.cpp @@ -432,6 +432,7 @@ struct llama_server_context } default_generation_settings_for_props = get_formated_generation(slots.front()); + default_generation_settings_for_props["num_slots"] = params.n_parallel; default_generation_settings_for_props["seed"] = -1; batch = llama_batch_init(n_ctx, 0, params.n_parallel); From 2c516611f1d0f1e5e9754f8ea1cf97cb1b17bf2c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20G=C3=A4=C3=9Fler?= Date: Tue, 6 Feb 2024 14:44:06 +0100 Subject: [PATCH 275/812] CUDA: mul_mat_vec_q for batch sizes > 1 (#5351) --- ggml-cuda.cu | 240 +++++++++++++++++++++------------------------------ 1 file changed, 98 insertions(+), 142 deletions(-) diff --git a/ggml-cuda.cu b/ggml-cuda.cu index 3242a0b4a..95161b3f4 100644 --- a/ggml-cuda.cu +++ b/ggml-cuda.cu @@ -5310,41 +5310,50 @@ template static __global__ void #endif // __CUDA_ARCH__ >= CC_VOLTA } -template -static __global__ void mul_mat_vec_q(const void * __restrict__ vx, const void * __restrict__ vy, float * __restrict__ dst, const int ncols, const int nrows) { +template +static __global__ void mul_mat_vec_q( + const void * __restrict__ vx, const void * __restrict__ vy, float * __restrict__ dst, + const int ncols_x, const int nrows_x, const int nrows_y, const int ncols_y_par) { + + const int ncols_y = ncols_y_template != 0 ? ncols_y_template : ncols_y_par; + const int row = blockIdx.x*blockDim.y + threadIdx.y; - if (row >= nrows) { + if (row >= nrows_x) { return; } - const int blocks_per_row = ncols / qk; + const int blocks_per_row_x = ncols_x / qk; + const int blocks_per_col_y = nrows_y / QK8_1; const int blocks_per_warp = vdr * WARP_SIZE / qi; // partial sum for each thread - float tmp = 0.0f; + float tmp[ncols_y_template != 0 ? ncols_y_template : 8] = {0.0f}; const block_q_t * x = (const block_q_t *) vx; const block_q8_1 * y = (const block_q8_1 *) vy; - for (int i = threadIdx.x / (qi/vdr); i < blocks_per_row; i += blocks_per_warp) { - const int ibx = row*blocks_per_row + i; // x block index + for (int i = threadIdx.x / (qi/vdr); i < blocks_per_row_x; i += blocks_per_warp) { + const int ibx = row*blocks_per_row_x + i; // x block index const int iby = i * (qk/QK8_1); // y block index that aligns with ibx const int iqs = vdr * (threadIdx.x % (qi/vdr)); // x block quant index when casting the quants to int - tmp += vec_dot_q_cuda(&x[ibx], &y[iby], iqs); +#pragma unroll + for (int j = 0; j < ncols_y; ++j) { + tmp[j] += vec_dot_q_cuda(&x[ibx], &y[j*blocks_per_col_y + iby], iqs); + } } // sum up partial sums and write back result #pragma unroll - for (int mask = 16; mask > 0; mask >>= 1) { - tmp += __shfl_xor_sync(0xffffffff, tmp, mask, 32); - } + for (int j = 0; j < ncols_y; ++j) { + tmp[j] = warp_reduce_sum(tmp[j]); - if (threadIdx.x == 0) { - dst[row] = tmp; + if (threadIdx.x == 0) { + dst[j*nrows_x + row] = tmp[j]; + } } } @@ -6816,121 +6825,56 @@ static void convert_mul_mat_vec_f16_cuda(const void * vx, const dfloat * y, floa <<>>(vx, y, dst, ncols, nrows); } -static void mul_mat_vec_q4_0_q8_1_cuda(const void * vx, const void * vy, float * dst, const int ncols, const int nrows, cudaStream_t stream) { - GGML_ASSERT(ncols % QK4_0 == 0); - const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y; - const dim3 block_nums(block_num_y, 1, 1); - const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1); - mul_mat_vec_q - <<>>(vx, vy, dst, ncols, nrows); -} +template +static void mul_mat_vec_q_cuda( + const void * vx, const void * vy, float * dst, + const int ncols_x, const int nrows_x, const int nrows_y, const int ncols_y, cudaStream_t stream) { -static void mul_mat_vec_q4_1_q8_1_cuda(const void * vx, const void * vy, float * dst, const int ncols, const int nrows, cudaStream_t stream) { - GGML_ASSERT(ncols % QK4_1 == 0); - const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y; - const dim3 block_nums(block_num_y, 1, 1); - const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1); - mul_mat_vec_q - <<>>(vx, vy, dst, ncols, nrows); -} + GGML_ASSERT(ncols_x % qk == 0); + GGML_ASSERT(ncols_y <= 8); -static void mul_mat_vec_q5_0_q8_1_cuda(const void * vx, const void * vy, float * dst, const int ncols, const int nrows, cudaStream_t stream) { - GGML_ASSERT(ncols % QK5_0 == 0); - const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y; + const int block_num_y = (nrows_x + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y; const dim3 block_nums(block_num_y, 1, 1); const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1); - mul_mat_vec_q - <<>>(vx, vy, dst, ncols, nrows); -} - -static void mul_mat_vec_q5_1_q8_1_cuda(const void * vx, const void * vy, float * dst, const int ncols, const int nrows, cudaStream_t stream) { - GGML_ASSERT(ncols % QK5_1 == 0); - const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y; - const dim3 block_nums(block_num_y, 1, 1); - const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1); - mul_mat_vec_q - <<>>(vx, vy, dst, ncols, nrows); -} - -static void mul_mat_vec_q8_0_q8_1_cuda(const void * vx, const void * vy, float * dst, const int ncols, const int nrows, cudaStream_t stream) { - GGML_ASSERT(ncols % QK8_0 == 0); - const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y; - const dim3 block_nums(block_num_y, 1, 1); - const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1); - mul_mat_vec_q - <<>>(vx, vy, dst, ncols, nrows); -} - -static void mul_mat_vec_q2_K_q8_1_cuda(const void * vx, const void * vy, float * dst, const int ncols, const int nrows, cudaStream_t stream) { - GGML_ASSERT(ncols % QK_K == 0); - const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y; - const dim3 block_nums(block_num_y, 1, 1); - const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1); - mul_mat_vec_q - <<>>(vx, vy, dst, ncols, nrows); -} - -static void mul_mat_vec_q3_K_q8_1_cuda(const void * vx, const void * vy, float * dst, const int ncols, const int nrows, cudaStream_t stream) { - GGML_ASSERT(ncols % QK_K == 0); - const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y; - const dim3 block_nums(block_num_y, 1, 1); - const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1); - mul_mat_vec_q - <<>>(vx, vy, dst, ncols, nrows); -} - -static void mul_mat_vec_q4_K_q8_1_cuda(const void * vx, const void * vy, float * dst, const int ncols, const int nrows, cudaStream_t stream) { - GGML_ASSERT(ncols % QK_K == 0); - const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y; - const dim3 block_nums(block_num_y, 1, 1); - const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1); - mul_mat_vec_q - <<>>(vx, vy, dst, ncols, nrows); -} - -static void mul_mat_vec_q5_K_q8_1_cuda(const void * vx, const void * vy, float * dst, const int ncols, const int nrows, cudaStream_t stream) { - GGML_ASSERT(ncols % QK_K == 0); - const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y; - const dim3 block_nums(block_num_y, 1, 1); - const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1); - mul_mat_vec_q - <<>>(vx, vy, dst, ncols, nrows); -} - -static void mul_mat_vec_q6_K_q8_1_cuda(const void * vx, const void * vy, float * dst, const int ncols, const int nrows, cudaStream_t stream) { - GGML_ASSERT(ncols % QK_K == 0); - const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y; - const dim3 block_nums(block_num_y, 1, 1); - const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1); - mul_mat_vec_q - <<>>(vx, vy, dst, ncols, nrows); -} - -static void mul_mat_vec_iq2_xxs_q8_1_cuda(const void * vx, const void * vy, float * dst, const int ncols, const int nrows, cudaStream_t stream) { - GGML_ASSERT(ncols % QK_K == 0); - const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y; - const dim3 block_nums(block_num_y, 1, 1); - const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1); - mul_mat_vec_q - <<>>(vx, vy, dst, ncols, nrows); -} - -static void mul_mat_vec_iq2_xs_q8_1_cuda(const void * vx, const void * vy, float * dst, const int ncols, const int nrows, cudaStream_t stream) { - GGML_ASSERT(ncols % QK_K == 0); - const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y; - const dim3 block_nums(block_num_y, 1, 1); - const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1); - mul_mat_vec_q - <<>>(vx, vy, dst, ncols, nrows); -} - -static void mul_mat_vec_iq3_xxs_q8_1_cuda(const void * vx, const void * vy, float * dst, const int ncols, const int nrows, cudaStream_t stream) { - GGML_ASSERT(ncols % QK_K == 0); - const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y; - const dim3 block_nums(block_num_y, 1, 1); - const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1); - mul_mat_vec_q - <<>>(vx, vy, dst, ncols, nrows); + switch (ncols_y) { + case 1: + mul_mat_vec_q<1, qk, qi, block_q_t, vdr, vec_dot> + <<>>(vx, vy, dst, ncols_x, nrows_x, nrows_y, ncols_y); + break; + case 2: + mul_mat_vec_q<2, qk, qi, block_q_t, vdr, vec_dot> + <<>>(vx, vy, dst, ncols_x, nrows_x, nrows_y, ncols_y); + break; + case 3: + mul_mat_vec_q<3, qk, qi, block_q_t, vdr, vec_dot> + <<>>(vx, vy, dst, ncols_x, nrows_x, nrows_y, ncols_y); + break; + case 4: + mul_mat_vec_q<4, qk, qi, block_q_t, vdr, vec_dot> + <<>>(vx, vy, dst, ncols_x, nrows_x, nrows_y, ncols_y); + break; + case 5: + mul_mat_vec_q<5, qk, qi, block_q_t, vdr, vec_dot> + <<>>(vx, vy, dst, ncols_x, nrows_x, nrows_y, ncols_y); + break; + case 6: + mul_mat_vec_q<6, qk, qi, block_q_t, vdr, vec_dot> + <<>>(vx, vy, dst, ncols_x, nrows_x, nrows_y, ncols_y); + break; + case 7: + mul_mat_vec_q<7, qk, qi, block_q_t, vdr, vec_dot> + <<>>(vx, vy, dst, ncols_x, nrows_x, nrows_y, ncols_y); + break; + case 8: + mul_mat_vec_q<8, qk, qi, block_q_t, vdr, vec_dot> + <<>>(vx, vy, dst, ncols_x, nrows_x, nrows_y, ncols_y); + break; + default: + GGML_ASSERT(false); + // mul_mat_vec_q<0, qk, qi, block_q_t, vdr, vec_dot> + // <<>>(vx, vy, dst, ncols_x, nrows_x, nrows_y, ncols_y); + break; + } } static void ggml_mul_mat_q4_0_q8_1_cuda( @@ -8578,50 +8522,61 @@ static void ggml_cuda_op_mul_mat_vec_q( const char * src1_ddq_i, float * dst_dd_i, const int64_t row_low, const int64_t row_high, const int64_t src1_ncols, const int64_t src1_padded_row_size, cudaStream_t stream) { - GGML_ASSERT(ggml_nrows(src1) == 1); - const int64_t ne00 = src0->ne[0]; const int64_t row_diff = row_high - row_low; switch (src0->type) { case GGML_TYPE_Q4_0: - mul_mat_vec_q4_0_q8_1_cuda(src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, stream); + mul_mat_vec_q_cuda + (src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_padded_row_size, src1_ncols, stream); break; case GGML_TYPE_Q4_1: - mul_mat_vec_q4_1_q8_1_cuda(src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, stream); + mul_mat_vec_q_cuda + (src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_padded_row_size, src1_ncols, stream); break; case GGML_TYPE_Q5_0: - mul_mat_vec_q5_0_q8_1_cuda(src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, stream); + mul_mat_vec_q_cuda + (src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_padded_row_size, src1_ncols, stream); break; case GGML_TYPE_Q5_1: - mul_mat_vec_q5_1_q8_1_cuda(src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, stream); + mul_mat_vec_q_cuda + (src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_padded_row_size, src1_ncols, stream); break; case GGML_TYPE_Q8_0: - mul_mat_vec_q8_0_q8_1_cuda(src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, stream); + mul_mat_vec_q_cuda + (src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_padded_row_size, src1_ncols, stream); break; case GGML_TYPE_Q2_K: - mul_mat_vec_q2_K_q8_1_cuda(src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, stream); + mul_mat_vec_q_cuda + (src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_padded_row_size, src1_ncols, stream); break; case GGML_TYPE_Q3_K: - mul_mat_vec_q3_K_q8_1_cuda(src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, stream); + mul_mat_vec_q_cuda + (src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_padded_row_size, src1_ncols, stream); break; case GGML_TYPE_Q4_K: - mul_mat_vec_q4_K_q8_1_cuda(src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, stream); + mul_mat_vec_q_cuda + (src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_padded_row_size, src1_ncols, stream); break; case GGML_TYPE_Q5_K: - mul_mat_vec_q5_K_q8_1_cuda(src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, stream); + mul_mat_vec_q_cuda + (src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_padded_row_size, src1_ncols, stream); break; case GGML_TYPE_Q6_K: - mul_mat_vec_q6_K_q8_1_cuda(src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, stream); + mul_mat_vec_q_cuda + (src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_padded_row_size, src1_ncols, stream); break; case GGML_TYPE_IQ2_XXS: - mul_mat_vec_iq2_xxs_q8_1_cuda(src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, stream); + mul_mat_vec_q_cuda + (src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_padded_row_size, src1_ncols, stream); break; case GGML_TYPE_IQ2_XS: - mul_mat_vec_iq2_xs_q8_1_cuda(src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, stream); + mul_mat_vec_q_cuda + (src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_padded_row_size, src1_ncols, stream); break; case GGML_TYPE_IQ3_XXS: - mul_mat_vec_iq3_xxs_q8_1_cuda(src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, stream); + mul_mat_vec_q_cuda + (src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_padded_row_size, src1_ncols, stream); break; default: GGML_ASSERT(false); @@ -9945,17 +9900,18 @@ static void ggml_cuda_mul_mat(const ggml_tensor * src0, const ggml_tensor * src1 #ifdef GGML_CUDA_FORCE_DMMV const bool use_mul_mat_vec_q = false; #else - const bool use_mul_mat_vec_q = min_compute_capability >= MIN_CC_DP4A && ggml_is_quantized(src0->type) && ggml_nrows(src1) == 1; + const bool use_mul_mat_vec_q = min_compute_capability >= MIN_CC_DP4A && ggml_is_quantized(src0->type); #endif // GGML_CUDA_FORCE_DMMV if (use_mul_mat_vec_q) { - // NOTE: this kernel does not support ggml_nrows(src1) > 1 ggml_cuda_op_mul_mat(src0, src1, dst, ggml_cuda_op_mul_mat_vec_q, true); } else { ggml_cuda_op_mul_mat(src0, src1, dst, ggml_cuda_op_dequantize_mul_mat_vec, false); } } else { - if (use_mul_mat_q) { + if (src1->ne[1] <= 8 && min_compute_capability >= MIN_CC_DP4A && ggml_is_quantized(src0->type)) { + ggml_cuda_op_mul_mat(src0, src1, dst, ggml_cuda_op_mul_mat_vec_q, true); + } else if (use_mul_mat_q) { ggml_cuda_op_mul_mat(src0, src1, dst, ggml_cuda_op_mul_mat_q, true); } else { ggml_cuda_op_mul_mat(src0, src1, dst, ggml_cuda_op_mul_mat_cublas, false); From 2e9c0bd6b301155ce749e162527fc55e9fb5b832 Mon Sep 17 00:00:00 2001 From: BarfingLemurs <128182951+BarfingLemurs@users.noreply.github.com> Date: Tue, 6 Feb 2024 09:06:48 -0500 Subject: [PATCH 276/812] readme : add phi, orion 14b, internlm2, and yi-VL to readme (#5362) --- README.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index bb6c49338..cc87ac797 100644 --- a/README.md +++ b/README.md @@ -105,11 +105,14 @@ improved significantly thanks to many contributions. It is the main playground f - [X] [MPT](https://github.com/ggerganov/llama.cpp/pull/3417) - [X] [Bloom](https://github.com/ggerganov/llama.cpp/pull/3553) - [x] [Yi models](https://huggingface.co/models?search=01-ai/Yi) -- [X] [StableLM-3b-4e1t](https://github.com/ggerganov/llama.cpp/pull/3586) +- [X] [StableLM models](https://huggingface.co/stabilityai) - [x] [Deepseek models](https://huggingface.co/models?search=deepseek-ai/deepseek) - [x] [Qwen models](https://huggingface.co/models?search=Qwen/Qwen) - [x] [PLaMo-13B](https://github.com/ggerganov/llama.cpp/pull/3557) +- [x] [Phi models](https://huggingface.co/models?search=microsoft/phi) - [x] [GPT-2](https://huggingface.co/gpt2) +- [x] [Orion 14B](https://github.com/ggerganov/llama.cpp/pull/5118) +- [x] [InternLM2](https://huggingface.co/models?search=internlm2) - [x] [CodeShell](https://github.com/WisdomShell/codeshell) **Multimodal models:** @@ -119,6 +122,7 @@ improved significantly thanks to many contributions. It is the main playground f - [x] [Obsidian](https://huggingface.co/NousResearch/Obsidian-3B-V0.5) - [x] [ShareGPT4V](https://huggingface.co/models?search=Lin-Chen/ShareGPT4V) - [x] [MobileVLM 1.7B/3B models](https://huggingface.co/models?search=mobileVLM) +- [x] [Yi-VL](https://huggingface.co/models?search=Yi-VL) **Bindings:** From f57fadc009cbff741a1961cb7896c47d73978d2c Mon Sep 17 00:00:00 2001 From: Kawrakow <48489457+ikawrakow@users.noreply.github.com> Date: Tue, 6 Feb 2024 17:28:02 +0200 Subject: [PATCH 277/812] Slight quantization improvement for Q4_K and Q5_K (#5361) * Q4_K: slightly better quantization * Q5_K: slightly better quantization --------- Co-authored-by: Iwan Kawrakow --- ggml-quants.c | 75 +++++++++++++++++++++++---------------------------- 1 file changed, 33 insertions(+), 42 deletions(-) diff --git a/ggml-quants.c b/ggml-quants.c index 014c0525a..101d3e783 100644 --- a/ggml-quants.c +++ b/ggml-quants.c @@ -2381,19 +2381,20 @@ static void quantize_row_q4_K_impl(const float * restrict x, block_q4_K * restri uint8_t L[QK_K]; uint8_t Laux[32]; + uint8_t Ls[QK_K/32]; + uint8_t Lm[QK_K/32]; float weights[32]; - float mins[QK_K/32]; - float scales[QK_K/32]; + float sw[QK_K/32]; + float mins[QK_K/32]; + float scales[QK_K/32]; for (int i = 0; i < nb; i++) { float sum_x2 = 0; for (int l = 0; l < QK_K; ++l) sum_x2 += x[l] * x[l]; - float sigma2 = sum_x2/QK_K; + float sigma2 = 2*sum_x2/QK_K; float av_x = sqrtf(sigma2); - float max_scale = 0; // as we are deducting the min, scales are always positive - float max_min = 0; for (int j = 0; j < QK_K/32; ++j) { if (quant_weights) { const float * qw = quant_weights + QK_K*i + 32*j; @@ -2401,25 +2402,17 @@ static void quantize_row_q4_K_impl(const float * restrict x, block_q4_K * restri } else { for (int l = 0; l < 32; ++l) weights[l] = av_x + fabsf(x[32*j + l]); } + float sumw = 0; + for (int l = 0; l < 32; ++l) sumw += weights[l]; + sw[j] = sumw; scales[j] = make_qkx3_quants(32, 15, x + 32*j, weights, L + 32*j, &mins[j], Laux, -0.9f, 0.05f, 36, false); - //scales[j] = make_qkx2_quants(32, 15, x + 32*j, weights, L + 32*j, &mins[j], Laux, -1.f, 0.1f, 20, false); - float scale = scales[j]; - if (scale > max_scale) { - max_scale = scale; - } - float min = mins[j]; - if (min > max_min) { - max_min = min; - } } - float inv_scale = max_scale > 0 ? 63.f/max_scale : 0.f; - float inv_min = max_min > 0 ? 63.f/max_min : 0.f; + float d_block = make_qp_quants(QK_K/32, 63, scales, Ls, sw); + float m_block = make_qp_quants(QK_K/32, 63, mins, Lm, sw); for (int j = 0; j < QK_K/32; ++j) { - uint8_t ls = nearest_int(inv_scale*scales[j]); - uint8_t lm = nearest_int(inv_min*mins[j]); - ls = MIN(63, ls); - lm = MIN(63, lm); + uint8_t ls = Ls[j]; + uint8_t lm = Lm[j]; if (j < 4) { y[i].scales[j] = ls; y[i].scales[j+4] = lm; @@ -2429,8 +2422,8 @@ static void quantize_row_q4_K_impl(const float * restrict x, block_q4_K * restri y[i].scales[j-0] |= ((lm >> 4) << 6); } } - y[i].d = GGML_FP32_TO_FP16(max_scale/63.f); - y[i].dmin = GGML_FP32_TO_FP16(max_min/63.f); + y[i].d = GGML_FP32_TO_FP16(d_block); + y[i].dmin = GGML_FP32_TO_FP16(m_block); uint8_t sc, m; for (int j = 0; j < QK_K/32; ++j) { @@ -2688,20 +2681,21 @@ static void quantize_row_q5_K_impl(const float * restrict x, block_q5_K * restri const int nb = n_per_row / QK_K; uint8_t L[QK_K]; - float mins[QK_K/32]; - float scales[QK_K/32]; - float weights[32]; uint8_t Laux[32]; + uint8_t Ls[QK_K/32]; + uint8_t Lm[QK_K/32]; + float mins[QK_K/32]; + float scales[QK_K/32]; + float sw[QK_K/32]; + float weights[32]; for (int i = 0; i < nb; i++) { float sum_x2 = 0; for (int l = 0; l < QK_K; ++l) sum_x2 += x[l] * x[l]; - float sigma2 = sum_x2/QK_K; + float sigma2 = 2*sum_x2/QK_K; float av_x = sqrtf(sigma2); - float max_scale = 0; // as we are deducting the min, scales are always positive - float max_min = 0; for (int j = 0; j < QK_K/32; ++j) { if (quant_weights) { const float * qw = quant_weights + QK_K*i + 32*j; @@ -2709,22 +2703,19 @@ static void quantize_row_q5_K_impl(const float * restrict x, block_q5_K * restri } else { for (int l = 0; l < 32; ++l) weights[l] = av_x + fabsf(x[32*j + l]); } + float sumw = 0; + for (int l = 0; l < 32; ++l) sumw += weights[l]; + sw[j] = sumw; + scales[j] = make_qkx3_quants(32, 31, x + 32*j, weights, L + 32*j, &mins[j], Laux, -0.9f, 0.05f, 36, false); - float scale = scales[j]; - if (scale > max_scale) { - max_scale = scale; - } - float min = mins[j]; - if (min > max_min) { - max_min = min; - } } - float inv_scale = max_scale > 0 ? 63.f/max_scale : 0.f; - float inv_min = max_min > 0 ? 63.f/max_min : 0.f; + float d_block = make_qp_quants(QK_K/32, 63, scales, Ls, sw); + float m_block = make_qp_quants(QK_K/32, 63, mins, Lm, sw); + for (int j = 0; j < QK_K/32; ++j) { - uint8_t ls = nearest_int(inv_scale*scales[j]); - uint8_t lm = nearest_int(inv_min*mins[j]); + uint8_t ls = Ls[j]; + uint8_t lm = Lm[j]; ls = MIN(63, ls); lm = MIN(63, lm); if (j < 4) { @@ -2736,8 +2727,8 @@ static void quantize_row_q5_K_impl(const float * restrict x, block_q5_K * restri y[i].scales[j-0] |= ((lm >> 4) << 6); } } - y[i].d = GGML_FP32_TO_FP16(max_scale/63.f); - y[i].dmin = GGML_FP32_TO_FP16(max_min/63.f); + y[i].d = GGML_FP32_TO_FP16(d_block); + y[i].dmin = GGML_FP32_TO_FP16(m_block); uint8_t sc, m; for (int j = 0; j < QK_K/32; ++j) { From b08f22c882a1443e6b97081f3ce718a4d1a741f8 Mon Sep 17 00:00:00 2001 From: Kawrakow <48489457+ikawrakow@users.noreply.github.com> Date: Tue, 6 Feb 2024 19:00:16 +0200 Subject: [PATCH 278/812] Update README.md (#5366) Add some links to quantization related PRs --- README.md | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index cc87ac797..34f2021f9 100644 --- a/README.md +++ b/README.md @@ -736,9 +736,21 @@ Several quantization methods are supported. They differ in the resulting model d | 13B | bits/weight | 16.0 | 4.5 | 5.0 | 5.5 | 6.0 | 8.5 | - [k-quants](https://github.com/ggerganov/llama.cpp/pull/1684) -- recent k-quants improvements +- recent k-quants improvements and new i-quants - [#2707](https://github.com/ggerganov/llama.cpp/pull/2707) - [#2807](https://github.com/ggerganov/llama.cpp/pull/2807) + - [#4773 - 2-bit i-quants (inference)](https://github.com/ggerganov/llama.cpp/pull/4773) + - [#4856 - 2-bit i-quants (inference)](https://github.com/ggerganov/llama.cpp/pull/4856) + - [#4861 - importance matrix](https://github.com/ggerganov/llama.cpp/pull/4861) + - [#4872 - MoE models](https://github.com/ggerganov/llama.cpp/pull/4872) + - [#4897 - 2-bit quantization](https://github.com/ggerganov/llama.cpp/pull/4897) + - [#4930 - imatrix for all k-quants](https://github.com/ggerganov/llama.cpp/pull/4930) + - [#4951 - imatrix on the GPU](https://github.com/ggerganov/llama.cpp/pull/4957) + - [#4969 - imatrix for legacy quants](https://github.com/ggerganov/llama.cpp/pull/4969) + - [#4996 - k-qunats tuning](https://github.com/ggerganov/llama.cpp/pull/4996) + - [#5060 - Q3_K_XS](https://github.com/ggerganov/llama.cpp/pull/5060) + - [#5196 - 3-bit i-quants](https://github.com/ggerganov/llama.cpp/pull/5196) + - [quantization tuning](https://github.com/ggerganov/llama.cpp/pull/5320), [another one](https://github.com/ggerganov/llama.cpp/pull/5334), and [another one](https://github.com/ggerganov/llama.cpp/pull/5361) ### Perplexity (measuring model quality) From 17c97fb0620448b37516a3f53fea6c482b0a30a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20G=C3=A4=C3=9Fler?= Date: Tue, 6 Feb 2024 18:43:06 +0100 Subject: [PATCH 279/812] CUDA: mul_mat_vec_q max. batch size 8 -> 4 (#5370) --- ggml-cuda.cu | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/ggml-cuda.cu b/ggml-cuda.cu index 95161b3f4..3b828375e 100644 --- a/ggml-cuda.cu +++ b/ggml-cuda.cu @@ -6831,7 +6831,7 @@ static void mul_mat_vec_q_cuda( const int ncols_x, const int nrows_x, const int nrows_y, const int ncols_y, cudaStream_t stream) { GGML_ASSERT(ncols_x % qk == 0); - GGML_ASSERT(ncols_y <= 8); + GGML_ASSERT(ncols_y <= 4); const int block_num_y = (nrows_x + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y; const dim3 block_nums(block_num_y, 1, 1); @@ -6853,22 +6853,22 @@ static void mul_mat_vec_q_cuda( mul_mat_vec_q<4, qk, qi, block_q_t, vdr, vec_dot> <<>>(vx, vy, dst, ncols_x, nrows_x, nrows_y, ncols_y); break; - case 5: - mul_mat_vec_q<5, qk, qi, block_q_t, vdr, vec_dot> - <<>>(vx, vy, dst, ncols_x, nrows_x, nrows_y, ncols_y); - break; - case 6: - mul_mat_vec_q<6, qk, qi, block_q_t, vdr, vec_dot> - <<>>(vx, vy, dst, ncols_x, nrows_x, nrows_y, ncols_y); - break; - case 7: - mul_mat_vec_q<7, qk, qi, block_q_t, vdr, vec_dot> - <<>>(vx, vy, dst, ncols_x, nrows_x, nrows_y, ncols_y); - break; - case 8: - mul_mat_vec_q<8, qk, qi, block_q_t, vdr, vec_dot> - <<>>(vx, vy, dst, ncols_x, nrows_x, nrows_y, ncols_y); - break; + // case 5: + // mul_mat_vec_q<5, qk, qi, block_q_t, vdr, vec_dot> + // <<>>(vx, vy, dst, ncols_x, nrows_x, nrows_y, ncols_y); + // break; + // case 6: + // mul_mat_vec_q<6, qk, qi, block_q_t, vdr, vec_dot> + // <<>>(vx, vy, dst, ncols_x, nrows_x, nrows_y, ncols_y); + // break; + // case 7: + // mul_mat_vec_q<7, qk, qi, block_q_t, vdr, vec_dot> + // <<>>(vx, vy, dst, ncols_x, nrows_x, nrows_y, ncols_y); + // break; + // case 8: + // mul_mat_vec_q<8, qk, qi, block_q_t, vdr, vec_dot> + // <<>>(vx, vy, dst, ncols_x, nrows_x, nrows_y, ncols_y); + // break; default: GGML_ASSERT(false); // mul_mat_vec_q<0, qk, qi, block_q_t, vdr, vec_dot> @@ -9909,7 +9909,7 @@ static void ggml_cuda_mul_mat(const ggml_tensor * src0, const ggml_tensor * src1 ggml_cuda_op_mul_mat(src0, src1, dst, ggml_cuda_op_dequantize_mul_mat_vec, false); } } else { - if (src1->ne[1] <= 8 && min_compute_capability >= MIN_CC_DP4A && ggml_is_quantized(src0->type)) { + if (src1->ne[1] <= 4 && min_compute_capability >= MIN_CC_DP4A && ggml_is_quantized(src0->type)) { ggml_cuda_op_mul_mat(src0, src1, dst, ggml_cuda_op_mul_mat_vec_q, true); } else if (use_mul_mat_q) { ggml_cuda_op_mul_mat(src0, src1, dst, ggml_cuda_op_mul_mat_q, true); From 213d1439fadefe182f69c5f7e8dd3b4b6572ebcb Mon Sep 17 00:00:00 2001 From: Alexey Parfenov Date: Tue, 6 Feb 2024 18:08:38 +0000 Subject: [PATCH 280/812] server : remove model.json endpoint (#5371) --- examples/server/completion.js.hpp | 448 +++++++++++++++------------ examples/server/public/completion.js | 3 +- examples/server/server.cpp | 11 - 3 files changed, 244 insertions(+), 218 deletions(-) diff --git a/examples/server/completion.js.hpp b/examples/server/completion.js.hpp index fe5f81228..f5e696e17 100644 --- a/examples/server/completion.js.hpp +++ b/examples/server/completion.js.hpp @@ -236,214 +236,250 @@ unsigned char completion_js[] = { 0x20, 0x4a, 0x53, 0x4f, 0x4e, 0x2e, 0x70, 0x61, 0x72, 0x73, 0x65, 0x28, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x2e, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x6f, 0x6c, 0x65, 0x2e, 0x65, - 0x72, 0x72, 0x6f, 0x72, 0x28, 0x60, 0x6c, 0x6c, 0x61, 0x6d, 0x61, 0x2e, - 0x63, 0x70, 0x70, 0x20, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x3a, 0x20, 0x24, - 0x7b, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x2e, 0x65, 0x72, 0x72, 0x6f, - 0x72, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x7d, 0x60, 0x29, - 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, + 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, 0x72, 0x65, 0x73, 0x75, 0x6c, + 0x74, 0x2e, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x2e, 0x63, 0x6f, 0x6e, 0x74, + 0x65, 0x6e, 0x74, 0x2e, 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x73, + 0x28, 0x27, 0x73, 0x6c, 0x6f, 0x74, 0x20, 0x75, 0x6e, 0x61, 0x76, 0x61, + 0x69, 0x6c, 0x61, 0x62, 0x6c, 0x65, 0x27, 0x29, 0x29, 0x20, 0x7b, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x2f, 0x2f, 0x20, 0x54, 0x68, 0x72, 0x6f, 0x77, 0x20, 0x61, + 0x6e, 0x20, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x20, 0x74, 0x6f, 0x20, 0x62, + 0x65, 0x20, 0x63, 0x61, 0x75, 0x67, 0x68, 0x74, 0x20, 0x62, 0x79, 0x20, + 0x75, 0x70, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x20, 0x63, 0x61, 0x6c, + 0x6c, 0x65, 0x72, 0x73, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x74, 0x68, 0x72, 0x6f, 0x77, + 0x20, 0x6e, 0x65, 0x77, 0x20, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x28, 0x27, + 0x73, 0x6c, 0x6f, 0x74, 0x20, 0x75, 0x6e, 0x61, 0x76, 0x61, 0x69, 0x6c, + 0x61, 0x62, 0x6c, 0x65, 0x27, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x20, 0x65, 0x6c, + 0x73, 0x65, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x6f, + 0x6c, 0x65, 0x2e, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x28, 0x60, 0x6c, 0x6c, + 0x61, 0x6d, 0x61, 0x2e, 0x63, 0x70, 0x70, 0x20, 0x65, 0x72, 0x72, 0x6f, + 0x72, 0x3a, 0x20, 0x24, 0x7b, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x2e, + 0x65, 0x72, 0x72, 0x6f, 0x72, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, + 0x74, 0x7d, 0x60, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, - 0x7d, 0x0a, 0x20, 0x20, 0x7d, 0x20, 0x63, 0x61, 0x74, 0x63, 0x68, 0x20, - 0x28, 0x65, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, - 0x20, 0x28, 0x65, 0x2e, 0x6e, 0x61, 0x6d, 0x65, 0x20, 0x21, 0x3d, 0x3d, - 0x20, 0x27, 0x41, 0x62, 0x6f, 0x72, 0x74, 0x45, 0x72, 0x72, 0x6f, 0x72, - 0x27, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, - 0x6f, 0x6e, 0x73, 0x6f, 0x6c, 0x65, 0x2e, 0x65, 0x72, 0x72, 0x6f, 0x72, - 0x28, 0x22, 0x6c, 0x6c, 0x61, 0x6d, 0x61, 0x20, 0x65, 0x72, 0x72, 0x6f, - 0x72, 0x3a, 0x20, 0x22, 0x2c, 0x20, 0x65, 0x29, 0x3b, 0x0a, 0x20, 0x20, - 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x74, 0x68, 0x72, 0x6f, - 0x77, 0x20, 0x65, 0x3b, 0x0a, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x66, - 0x69, 0x6e, 0x61, 0x6c, 0x6c, 0x79, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, - 0x20, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x2e, - 0x61, 0x62, 0x6f, 0x72, 0x74, 0x28, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x7d, - 0x0a, 0x0a, 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x63, - 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x3b, 0x0a, 0x7d, 0x0a, 0x0a, 0x2f, - 0x2f, 0x20, 0x43, 0x61, 0x6c, 0x6c, 0x20, 0x6c, 0x6c, 0x61, 0x6d, 0x61, - 0x2c, 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x61, 0x6e, 0x20, - 0x65, 0x76, 0x65, 0x6e, 0x74, 0x20, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, - 0x20, 0x74, 0x68, 0x61, 0x74, 0x20, 0x79, 0x6f, 0x75, 0x20, 0x63, 0x61, - 0x6e, 0x20, 0x73, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x62, 0x65, 0x20, - 0x74, 0x6f, 0x0a, 0x2f, 0x2f, 0x0a, 0x2f, 0x2f, 0x20, 0x45, 0x78, 0x61, - 0x6d, 0x70, 0x6c, 0x65, 0x3a, 0x0a, 0x2f, 0x2f, 0x0a, 0x2f, 0x2f, 0x20, - 0x20, 0x20, 0x20, 0x69, 0x6d, 0x70, 0x6f, 0x72, 0x74, 0x20, 0x7b, 0x20, - 0x6c, 0x6c, 0x61, 0x6d, 0x61, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x54, 0x61, - 0x72, 0x67, 0x65, 0x74, 0x20, 0x7d, 0x20, 0x66, 0x72, 0x6f, 0x6d, 0x20, - 0x27, 0x2f, 0x63, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x69, 0x6f, 0x6e, - 0x2e, 0x6a, 0x73, 0x27, 0x0a, 0x2f, 0x2f, 0x0a, 0x2f, 0x2f, 0x20, 0x20, - 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x63, 0x6f, 0x6e, 0x6e, - 0x20, 0x3d, 0x20, 0x6c, 0x6c, 0x61, 0x6d, 0x61, 0x45, 0x76, 0x65, 0x6e, - 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x28, 0x70, 0x72, 0x6f, 0x6d, - 0x70, 0x74, 0x29, 0x0a, 0x2f, 0x2f, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, - 0x6e, 0x6e, 0x2e, 0x61, 0x64, 0x64, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x4c, - 0x69, 0x73, 0x74, 0x65, 0x6e, 0x65, 0x72, 0x28, 0x22, 0x6d, 0x65, 0x73, - 0x73, 0x61, 0x67, 0x65, 0x22, 0x2c, 0x20, 0x28, 0x63, 0x68, 0x75, 0x6e, - 0x6b, 0x29, 0x20, 0x3d, 0x3e, 0x20, 0x7b, 0x0a, 0x2f, 0x2f, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x64, 0x6f, 0x63, 0x75, 0x6d, 0x65, 0x6e, 0x74, - 0x2e, 0x77, 0x72, 0x69, 0x74, 0x65, 0x28, 0x63, 0x68, 0x75, 0x6e, 0x6b, - 0x2e, 0x64, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x74, - 0x65, 0x6e, 0x74, 0x29, 0x0a, 0x2f, 0x2f, 0x20, 0x20, 0x20, 0x20, 0x7d, - 0x29, 0x0a, 0x2f, 0x2f, 0x0a, 0x65, 0x78, 0x70, 0x6f, 0x72, 0x74, 0x20, - 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x6c, 0x6c, 0x61, 0x6d, 0x61, 0x45, - 0x76, 0x65, 0x6e, 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x20, 0x3d, - 0x20, 0x28, 0x70, 0x72, 0x6f, 0x6d, 0x70, 0x74, 0x2c, 0x20, 0x70, 0x61, - 0x72, 0x61, 0x6d, 0x73, 0x20, 0x3d, 0x20, 0x7b, 0x7d, 0x2c, 0x20, 0x63, - 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x20, 0x3d, 0x20, 0x7b, 0x7d, 0x29, 0x20, - 0x3d, 0x3e, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, - 0x20, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, - 0x20, 0x3d, 0x20, 0x6e, 0x65, 0x77, 0x20, 0x45, 0x76, 0x65, 0x6e, 0x74, - 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x28, 0x29, 0x3b, 0x0a, 0x20, 0x20, - 0x28, 0x61, 0x73, 0x79, 0x6e, 0x63, 0x20, 0x28, 0x29, 0x20, 0x3d, 0x3e, - 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x6c, 0x65, 0x74, 0x20, 0x63, - 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x20, 0x3d, 0x20, 0x22, 0x22, 0x3b, - 0x0a, 0x20, 0x20, 0x20, 0x20, 0x66, 0x6f, 0x72, 0x20, 0x61, 0x77, 0x61, - 0x69, 0x74, 0x20, 0x28, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x63, 0x68, - 0x75, 0x6e, 0x6b, 0x20, 0x6f, 0x66, 0x20, 0x6c, 0x6c, 0x61, 0x6d, 0x61, - 0x28, 0x70, 0x72, 0x6f, 0x6d, 0x70, 0x74, 0x2c, 0x20, 0x70, 0x61, 0x72, - 0x61, 0x6d, 0x73, 0x2c, 0x20, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x29, - 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, - 0x20, 0x28, 0x63, 0x68, 0x75, 0x6e, 0x6b, 0x2e, 0x64, 0x61, 0x74, 0x61, - 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x20, 0x2b, 0x3d, 0x20, 0x63, - 0x68, 0x75, 0x6e, 0x6b, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x63, 0x6f, - 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x54, 0x61, 0x72, 0x67, - 0x65, 0x74, 0x2e, 0x64, 0x69, 0x73, 0x70, 0x61, 0x74, 0x63, 0x68, 0x45, - 0x76, 0x65, 0x6e, 0x74, 0x28, 0x6e, 0x65, 0x77, 0x20, 0x43, 0x75, 0x73, - 0x74, 0x6f, 0x6d, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x28, 0x22, 0x6d, 0x65, - 0x73, 0x73, 0x61, 0x67, 0x65, 0x22, 0x2c, 0x20, 0x7b, 0x20, 0x64, 0x65, - 0x74, 0x61, 0x69, 0x6c, 0x3a, 0x20, 0x63, 0x68, 0x75, 0x6e, 0x6b, 0x2e, - 0x64, 0x61, 0x74, 0x61, 0x20, 0x7d, 0x29, 0x29, 0x3b, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, 0x72, 0x65, + 0x73, 0x75, 0x6c, 0x74, 0x2e, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x29, 0x20, + 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x2e, 0x65, 0x72, 0x72, + 0x6f, 0x72, 0x20, 0x3d, 0x20, 0x4a, 0x53, 0x4f, 0x4e, 0x2e, 0x70, 0x61, + 0x72, 0x73, 0x65, 0x28, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x2e, 0x65, + 0x72, 0x72, 0x6f, 0x72, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x6f, + 0x6c, 0x65, 0x2e, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x28, 0x60, 0x6c, 0x6c, + 0x61, 0x6d, 0x61, 0x2e, 0x63, 0x70, 0x70, 0x20, 0x65, 0x72, 0x72, 0x6f, + 0x72, 0x3a, 0x20, 0x24, 0x7b, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x2e, + 0x65, 0x72, 0x72, 0x6f, 0x72, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, + 0x74, 0x7d, 0x60, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x69, 0x66, 0x20, 0x28, 0x63, 0x68, 0x75, 0x6e, 0x6b, 0x2e, 0x64, 0x61, - 0x74, 0x61, 0x2e, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, - 0x6e, 0x5f, 0x73, 0x65, 0x74, 0x74, 0x69, 0x6e, 0x67, 0x73, 0x29, 0x20, - 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x65, 0x76, - 0x65, 0x6e, 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x2e, 0x64, 0x69, - 0x73, 0x70, 0x61, 0x74, 0x63, 0x68, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x28, - 0x6e, 0x65, 0x77, 0x20, 0x43, 0x75, 0x73, 0x74, 0x6f, 0x6d, 0x45, 0x76, - 0x65, 0x6e, 0x74, 0x28, 0x22, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, - 0x69, 0x6f, 0x6e, 0x5f, 0x73, 0x65, 0x74, 0x74, 0x69, 0x6e, 0x67, 0x73, - 0x22, 0x2c, 0x20, 0x7b, 0x20, 0x64, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x3a, - 0x20, 0x63, 0x68, 0x75, 0x6e, 0x6b, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x2e, - 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x73, - 0x65, 0x74, 0x74, 0x69, 0x6e, 0x67, 0x73, 0x20, 0x7d, 0x29, 0x29, 0x3b, - 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, 0x63, 0x68, 0x75, 0x6e, 0x6b, - 0x2e, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x74, 0x69, 0x6d, 0x69, 0x6e, 0x67, - 0x73, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, - 0x2e, 0x64, 0x69, 0x73, 0x70, 0x61, 0x74, 0x63, 0x68, 0x45, 0x76, 0x65, - 0x6e, 0x74, 0x28, 0x6e, 0x65, 0x77, 0x20, 0x43, 0x75, 0x73, 0x74, 0x6f, - 0x6d, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x28, 0x22, 0x74, 0x69, 0x6d, 0x69, - 0x6e, 0x67, 0x73, 0x22, 0x2c, 0x20, 0x7b, 0x20, 0x64, 0x65, 0x74, 0x61, - 0x69, 0x6c, 0x3a, 0x20, 0x63, 0x68, 0x75, 0x6e, 0x6b, 0x2e, 0x64, 0x61, - 0x74, 0x61, 0x2e, 0x74, 0x69, 0x6d, 0x69, 0x6e, 0x67, 0x73, 0x20, 0x7d, - 0x29, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, - 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x65, 0x76, - 0x65, 0x6e, 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x2e, 0x64, 0x69, - 0x73, 0x70, 0x61, 0x74, 0x63, 0x68, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x28, - 0x6e, 0x65, 0x77, 0x20, 0x43, 0x75, 0x73, 0x74, 0x6f, 0x6d, 0x45, 0x76, - 0x65, 0x6e, 0x74, 0x28, 0x22, 0x64, 0x6f, 0x6e, 0x65, 0x22, 0x2c, 0x20, - 0x7b, 0x20, 0x64, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x3a, 0x20, 0x7b, 0x20, - 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x20, 0x7d, 0x20, 0x7d, 0x29, - 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x7d, 0x29, 0x28, 0x29, 0x3b, 0x0a, 0x20, - 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x65, 0x76, 0x65, 0x6e, - 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x3b, 0x0a, 0x7d, 0x0a, 0x0a, - 0x2f, 0x2f, 0x20, 0x43, 0x61, 0x6c, 0x6c, 0x20, 0x6c, 0x6c, 0x61, 0x6d, - 0x61, 0x2c, 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x61, 0x20, - 0x70, 0x72, 0x6f, 0x6d, 0x69, 0x73, 0x65, 0x20, 0x74, 0x68, 0x61, 0x74, - 0x20, 0x72, 0x65, 0x73, 0x6f, 0x6c, 0x76, 0x65, 0x73, 0x20, 0x74, 0x6f, - 0x20, 0x74, 0x68, 0x65, 0x20, 0x63, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, - 0x65, 0x64, 0x20, 0x74, 0x65, 0x78, 0x74, 0x2e, 0x20, 0x54, 0x68, 0x69, - 0x73, 0x20, 0x64, 0x6f, 0x65, 0x73, 0x20, 0x6e, 0x6f, 0x74, 0x20, 0x73, - 0x75, 0x70, 0x70, 0x6f, 0x72, 0x74, 0x20, 0x73, 0x74, 0x72, 0x65, 0x61, - 0x6d, 0x69, 0x6e, 0x67, 0x0a, 0x2f, 0x2f, 0x0a, 0x2f, 0x2f, 0x20, 0x45, - 0x78, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x3a, 0x0a, 0x2f, 0x2f, 0x0a, 0x2f, - 0x2f, 0x20, 0x20, 0x20, 0x20, 0x20, 0x6c, 0x6c, 0x61, 0x6d, 0x61, 0x50, - 0x72, 0x6f, 0x6d, 0x69, 0x73, 0x65, 0x28, 0x70, 0x72, 0x6f, 0x6d, 0x70, - 0x74, 0x29, 0x2e, 0x74, 0x68, 0x65, 0x6e, 0x28, 0x28, 0x63, 0x6f, 0x6e, - 0x74, 0x65, 0x6e, 0x74, 0x29, 0x20, 0x3d, 0x3e, 0x20, 0x7b, 0x0a, 0x2f, - 0x2f, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x64, 0x6f, 0x63, 0x75, + 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x7d, 0x20, 0x63, 0x61, + 0x74, 0x63, 0x68, 0x20, 0x28, 0x65, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, 0x65, 0x2e, 0x6e, 0x61, 0x6d, 0x65, + 0x20, 0x21, 0x3d, 0x3d, 0x20, 0x27, 0x41, 0x62, 0x6f, 0x72, 0x74, 0x45, + 0x72, 0x72, 0x6f, 0x72, 0x27, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x6f, 0x6c, 0x65, 0x2e, 0x65, + 0x72, 0x72, 0x6f, 0x72, 0x28, 0x22, 0x6c, 0x6c, 0x61, 0x6d, 0x61, 0x20, + 0x65, 0x72, 0x72, 0x6f, 0x72, 0x3a, 0x20, 0x22, 0x2c, 0x20, 0x65, 0x29, + 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x74, 0x68, 0x72, 0x6f, 0x77, 0x20, 0x65, 0x3b, 0x0a, 0x20, 0x20, 0x7d, + 0x0a, 0x20, 0x20, 0x66, 0x69, 0x6e, 0x61, 0x6c, 0x6c, 0x79, 0x20, 0x7b, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, + 0x6c, 0x65, 0x72, 0x2e, 0x61, 0x62, 0x6f, 0x72, 0x74, 0x28, 0x29, 0x3b, + 0x0a, 0x20, 0x20, 0x7d, 0x0a, 0x0a, 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, + 0x72, 0x6e, 0x20, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x3b, 0x0a, + 0x7d, 0x0a, 0x0a, 0x2f, 0x2f, 0x20, 0x43, 0x61, 0x6c, 0x6c, 0x20, 0x6c, + 0x6c, 0x61, 0x6d, 0x61, 0x2c, 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, + 0x20, 0x61, 0x6e, 0x20, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x20, 0x74, 0x61, + 0x72, 0x67, 0x65, 0x74, 0x20, 0x74, 0x68, 0x61, 0x74, 0x20, 0x79, 0x6f, + 0x75, 0x20, 0x63, 0x61, 0x6e, 0x20, 0x73, 0x75, 0x62, 0x73, 0x63, 0x72, + 0x69, 0x62, 0x65, 0x20, 0x74, 0x6f, 0x0a, 0x2f, 0x2f, 0x0a, 0x2f, 0x2f, + 0x20, 0x45, 0x78, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x3a, 0x0a, 0x2f, 0x2f, + 0x0a, 0x2f, 0x2f, 0x20, 0x20, 0x20, 0x20, 0x69, 0x6d, 0x70, 0x6f, 0x72, + 0x74, 0x20, 0x7b, 0x20, 0x6c, 0x6c, 0x61, 0x6d, 0x61, 0x45, 0x76, 0x65, + 0x6e, 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x20, 0x7d, 0x20, 0x66, + 0x72, 0x6f, 0x6d, 0x20, 0x27, 0x2f, 0x63, 0x6f, 0x6d, 0x70, 0x6c, 0x65, + 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x6a, 0x73, 0x27, 0x0a, 0x2f, 0x2f, 0x0a, + 0x2f, 0x2f, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, + 0x63, 0x6f, 0x6e, 0x6e, 0x20, 0x3d, 0x20, 0x6c, 0x6c, 0x61, 0x6d, 0x61, + 0x45, 0x76, 0x65, 0x6e, 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x28, + 0x70, 0x72, 0x6f, 0x6d, 0x70, 0x74, 0x29, 0x0a, 0x2f, 0x2f, 0x20, 0x20, + 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x6e, 0x2e, 0x61, 0x64, 0x64, 0x45, 0x76, + 0x65, 0x6e, 0x74, 0x4c, 0x69, 0x73, 0x74, 0x65, 0x6e, 0x65, 0x72, 0x28, + 0x22, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x22, 0x2c, 0x20, 0x28, + 0x63, 0x68, 0x75, 0x6e, 0x6b, 0x29, 0x20, 0x3d, 0x3e, 0x20, 0x7b, 0x0a, + 0x2f, 0x2f, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x64, 0x6f, 0x63, 0x75, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x77, 0x72, 0x69, 0x74, 0x65, 0x28, 0x63, - 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x29, 0x0a, 0x2f, 0x2f, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x7d, 0x29, 0x0a, 0x2f, 0x2f, 0x0a, 0x2f, 0x2f, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x6f, 0x72, 0x0a, 0x2f, 0x2f, 0x0a, 0x2f, 0x2f, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x63, - 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x20, 0x3d, 0x20, 0x61, 0x77, 0x61, - 0x69, 0x74, 0x20, 0x6c, 0x6c, 0x61, 0x6d, 0x61, 0x50, 0x72, 0x6f, 0x6d, - 0x69, 0x73, 0x65, 0x28, 0x70, 0x72, 0x6f, 0x6d, 0x70, 0x74, 0x29, 0x0a, - 0x2f, 0x2f, 0x20, 0x20, 0x20, 0x20, 0x20, 0x64, 0x6f, 0x63, 0x75, 0x6d, - 0x65, 0x6e, 0x74, 0x2e, 0x77, 0x72, 0x69, 0x74, 0x65, 0x28, 0x63, 0x6f, - 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x29, 0x0a, 0x2f, 0x2f, 0x0a, 0x65, 0x78, - 0x70, 0x6f, 0x72, 0x74, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x6c, - 0x6c, 0x61, 0x6d, 0x61, 0x50, 0x72, 0x6f, 0x6d, 0x69, 0x73, 0x65, 0x20, - 0x3d, 0x20, 0x28, 0x70, 0x72, 0x6f, 0x6d, 0x70, 0x74, 0x2c, 0x20, 0x70, - 0x61, 0x72, 0x61, 0x6d, 0x73, 0x20, 0x3d, 0x20, 0x7b, 0x7d, 0x2c, 0x20, - 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x20, 0x3d, 0x20, 0x7b, 0x7d, 0x29, - 0x20, 0x3d, 0x3e, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, - 0x72, 0x6e, 0x20, 0x6e, 0x65, 0x77, 0x20, 0x50, 0x72, 0x6f, 0x6d, 0x69, - 0x73, 0x65, 0x28, 0x61, 0x73, 0x79, 0x6e, 0x63, 0x20, 0x28, 0x72, 0x65, - 0x73, 0x6f, 0x6c, 0x76, 0x65, 0x2c, 0x20, 0x72, 0x65, 0x6a, 0x65, 0x63, - 0x74, 0x29, 0x20, 0x3d, 0x3e, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, - 0x6c, 0x65, 0x74, 0x20, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x20, - 0x3d, 0x20, 0x22, 0x22, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x74, 0x72, - 0x79, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x66, 0x6f, - 0x72, 0x20, 0x61, 0x77, 0x61, 0x69, 0x74, 0x20, 0x28, 0x63, 0x6f, 0x6e, - 0x73, 0x74, 0x20, 0x63, 0x68, 0x75, 0x6e, 0x6b, 0x20, 0x6f, 0x66, 0x20, - 0x6c, 0x6c, 0x61, 0x6d, 0x61, 0x28, 0x70, 0x72, 0x6f, 0x6d, 0x70, 0x74, - 0x2c, 0x20, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x2c, 0x20, 0x63, 0x6f, - 0x6e, 0x66, 0x69, 0x67, 0x29, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, - 0x20, 0x2b, 0x3d, 0x20, 0x63, 0x68, 0x75, 0x6e, 0x6b, 0x2e, 0x64, 0x61, - 0x74, 0x61, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x3b, 0x0a, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x72, 0x65, 0x73, 0x6f, 0x6c, 0x76, 0x65, 0x28, 0x63, 0x6f, - 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, - 0x7d, 0x20, 0x63, 0x61, 0x74, 0x63, 0x68, 0x20, 0x28, 0x65, 0x72, 0x72, - 0x6f, 0x72, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x72, 0x65, 0x6a, 0x65, 0x63, 0x74, 0x28, 0x65, 0x72, 0x72, 0x6f, 0x72, - 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x7d, - 0x29, 0x3b, 0x0a, 0x7d, 0x3b, 0x0a, 0x0a, 0x2f, 0x2a, 0x2a, 0x0a, 0x20, - 0x2a, 0x20, 0x28, 0x64, 0x65, 0x70, 0x72, 0x65, 0x63, 0x61, 0x74, 0x65, - 0x64, 0x29, 0x0a, 0x20, 0x2a, 0x2f, 0x0a, 0x65, 0x78, 0x70, 0x6f, 0x72, - 0x74, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x6c, 0x6c, 0x61, 0x6d, - 0x61, 0x43, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65, 0x20, 0x3d, 0x20, - 0x61, 0x73, 0x79, 0x6e, 0x63, 0x20, 0x28, 0x70, 0x61, 0x72, 0x61, 0x6d, - 0x73, 0x2c, 0x20, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, - 0x72, 0x2c, 0x20, 0x63, 0x61, 0x6c, 0x6c, 0x62, 0x61, 0x63, 0x6b, 0x29, - 0x20, 0x3d, 0x3e, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x66, 0x6f, 0x72, 0x20, - 0x61, 0x77, 0x61, 0x69, 0x74, 0x20, 0x28, 0x63, 0x6f, 0x6e, 0x73, 0x74, - 0x20, 0x63, 0x68, 0x75, 0x6e, 0x6b, 0x20, 0x6f, 0x66, 0x20, 0x6c, 0x6c, - 0x61, 0x6d, 0x61, 0x28, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x2e, 0x70, - 0x72, 0x6f, 0x6d, 0x70, 0x74, 0x2c, 0x20, 0x70, 0x61, 0x72, 0x61, 0x6d, - 0x73, 0x2c, 0x20, 0x7b, 0x20, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, - 0x6c, 0x65, 0x72, 0x20, 0x7d, 0x29, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, - 0x20, 0x20, 0x63, 0x61, 0x6c, 0x6c, 0x62, 0x61, 0x63, 0x6b, 0x28, 0x63, - 0x68, 0x75, 0x6e, 0x6b, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x7d, 0x0a, 0x7d, - 0x0a, 0x0a, 0x2f, 0x2f, 0x20, 0x47, 0x65, 0x74, 0x20, 0x74, 0x68, 0x65, - 0x20, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x20, 0x69, 0x6e, 0x66, 0x6f, 0x20, - 0x66, 0x72, 0x6f, 0x6d, 0x20, 0x74, 0x68, 0x65, 0x20, 0x73, 0x65, 0x72, - 0x76, 0x65, 0x72, 0x2e, 0x20, 0x54, 0x68, 0x69, 0x73, 0x20, 0x69, 0x73, - 0x20, 0x75, 0x73, 0x65, 0x66, 0x75, 0x6c, 0x20, 0x66, 0x6f, 0x72, 0x20, - 0x67, 0x65, 0x74, 0x74, 0x69, 0x6e, 0x67, 0x20, 0x74, 0x68, 0x65, 0x20, - 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x20, 0x77, 0x69, 0x6e, 0x64, - 0x6f, 0x77, 0x20, 0x61, 0x6e, 0x64, 0x20, 0x73, 0x6f, 0x20, 0x6f, 0x6e, - 0x2e, 0x0a, 0x65, 0x78, 0x70, 0x6f, 0x72, 0x74, 0x20, 0x63, 0x6f, 0x6e, - 0x73, 0x74, 0x20, 0x6c, 0x6c, 0x61, 0x6d, 0x61, 0x4d, 0x6f, 0x64, 0x65, - 0x6c, 0x49, 0x6e, 0x66, 0x6f, 0x20, 0x3d, 0x20, 0x61, 0x73, 0x79, 0x6e, - 0x63, 0x20, 0x28, 0x29, 0x20, 0x3d, 0x3e, 0x20, 0x7b, 0x0a, 0x20, 0x20, - 0x69, 0x66, 0x20, 0x28, 0x21, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, - 0x69, 0x6f, 0x6e, 0x5f, 0x73, 0x65, 0x74, 0x74, 0x69, 0x6e, 0x67, 0x73, - 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x67, 0x65, 0x6e, 0x65, - 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x73, 0x65, 0x74, 0x74, 0x69, - 0x6e, 0x67, 0x73, 0x20, 0x3d, 0x20, 0x61, 0x77, 0x61, 0x69, 0x74, 0x20, - 0x66, 0x65, 0x74, 0x63, 0x68, 0x28, 0x22, 0x2f, 0x6d, 0x6f, 0x64, 0x65, - 0x6c, 0x2e, 0x6a, 0x73, 0x6f, 0x6e, 0x22, 0x29, 0x2e, 0x74, 0x68, 0x65, - 0x6e, 0x28, 0x72, 0x20, 0x3d, 0x3e, 0x20, 0x72, 0x2e, 0x6a, 0x73, 0x6f, - 0x6e, 0x28, 0x29, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, - 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x67, 0x65, 0x6e, 0x65, 0x72, + 0x68, 0x75, 0x6e, 0x6b, 0x2e, 0x64, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x2e, + 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x29, 0x0a, 0x2f, 0x2f, 0x20, + 0x20, 0x20, 0x20, 0x7d, 0x29, 0x0a, 0x2f, 0x2f, 0x0a, 0x65, 0x78, 0x70, + 0x6f, 0x72, 0x74, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x6c, 0x6c, + 0x61, 0x6d, 0x61, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x54, 0x61, 0x72, 0x67, + 0x65, 0x74, 0x20, 0x3d, 0x20, 0x28, 0x70, 0x72, 0x6f, 0x6d, 0x70, 0x74, + 0x2c, 0x20, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x20, 0x3d, 0x20, 0x7b, + 0x7d, 0x2c, 0x20, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x20, 0x3d, 0x20, + 0x7b, 0x7d, 0x29, 0x20, 0x3d, 0x3e, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x63, + 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x54, 0x61, + 0x72, 0x67, 0x65, 0x74, 0x20, 0x3d, 0x20, 0x6e, 0x65, 0x77, 0x20, 0x45, + 0x76, 0x65, 0x6e, 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x28, 0x29, + 0x3b, 0x0a, 0x20, 0x20, 0x28, 0x61, 0x73, 0x79, 0x6e, 0x63, 0x20, 0x28, + 0x29, 0x20, 0x3d, 0x3e, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x6c, + 0x65, 0x74, 0x20, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x20, 0x3d, + 0x20, 0x22, 0x22, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x66, 0x6f, 0x72, + 0x20, 0x61, 0x77, 0x61, 0x69, 0x74, 0x20, 0x28, 0x63, 0x6f, 0x6e, 0x73, + 0x74, 0x20, 0x63, 0x68, 0x75, 0x6e, 0x6b, 0x20, 0x6f, 0x66, 0x20, 0x6c, + 0x6c, 0x61, 0x6d, 0x61, 0x28, 0x70, 0x72, 0x6f, 0x6d, 0x70, 0x74, 0x2c, + 0x20, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x2c, 0x20, 0x63, 0x6f, 0x6e, + 0x66, 0x69, 0x67, 0x29, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, 0x63, 0x68, 0x75, 0x6e, 0x6b, 0x2e, + 0x64, 0x61, 0x74, 0x61, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x20, + 0x2b, 0x3d, 0x20, 0x63, 0x68, 0x75, 0x6e, 0x6b, 0x2e, 0x64, 0x61, 0x74, + 0x61, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x3b, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x65, 0x76, 0x65, 0x6e, 0x74, + 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x2e, 0x64, 0x69, 0x73, 0x70, 0x61, + 0x74, 0x63, 0x68, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x28, 0x6e, 0x65, 0x77, + 0x20, 0x43, 0x75, 0x73, 0x74, 0x6f, 0x6d, 0x45, 0x76, 0x65, 0x6e, 0x74, + 0x28, 0x22, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x22, 0x2c, 0x20, + 0x7b, 0x20, 0x64, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x3a, 0x20, 0x63, 0x68, + 0x75, 0x6e, 0x6b, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x20, 0x7d, 0x29, 0x29, + 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, 0x63, 0x68, 0x75, 0x6e, + 0x6b, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x73, 0x65, 0x74, 0x74, 0x69, 0x6e, - 0x67, 0x73, 0x3b, 0x0a, 0x7d, 0x0a + 0x67, 0x73, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, + 0x74, 0x2e, 0x64, 0x69, 0x73, 0x70, 0x61, 0x74, 0x63, 0x68, 0x45, 0x76, + 0x65, 0x6e, 0x74, 0x28, 0x6e, 0x65, 0x77, 0x20, 0x43, 0x75, 0x73, 0x74, + 0x6f, 0x6d, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x28, 0x22, 0x67, 0x65, 0x6e, + 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x73, 0x65, 0x74, 0x74, + 0x69, 0x6e, 0x67, 0x73, 0x22, 0x2c, 0x20, 0x7b, 0x20, 0x64, 0x65, 0x74, + 0x61, 0x69, 0x6c, 0x3a, 0x20, 0x63, 0x68, 0x75, 0x6e, 0x6b, 0x2e, 0x64, + 0x61, 0x74, 0x61, 0x2e, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x69, + 0x6f, 0x6e, 0x5f, 0x73, 0x65, 0x74, 0x74, 0x69, 0x6e, 0x67, 0x73, 0x20, + 0x7d, 0x29, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, 0x63, + 0x68, 0x75, 0x6e, 0x6b, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x74, 0x69, + 0x6d, 0x69, 0x6e, 0x67, 0x73, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x54, 0x61, + 0x72, 0x67, 0x65, 0x74, 0x2e, 0x64, 0x69, 0x73, 0x70, 0x61, 0x74, 0x63, + 0x68, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x28, 0x6e, 0x65, 0x77, 0x20, 0x43, + 0x75, 0x73, 0x74, 0x6f, 0x6d, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x28, 0x22, + 0x74, 0x69, 0x6d, 0x69, 0x6e, 0x67, 0x73, 0x22, 0x2c, 0x20, 0x7b, 0x20, + 0x64, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x3a, 0x20, 0x63, 0x68, 0x75, 0x6e, + 0x6b, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x74, 0x69, 0x6d, 0x69, 0x6e, + 0x67, 0x73, 0x20, 0x7d, 0x29, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, + 0x74, 0x2e, 0x64, 0x69, 0x73, 0x70, 0x61, 0x74, 0x63, 0x68, 0x45, 0x76, + 0x65, 0x6e, 0x74, 0x28, 0x6e, 0x65, 0x77, 0x20, 0x43, 0x75, 0x73, 0x74, + 0x6f, 0x6d, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x28, 0x22, 0x64, 0x6f, 0x6e, + 0x65, 0x22, 0x2c, 0x20, 0x7b, 0x20, 0x64, 0x65, 0x74, 0x61, 0x69, 0x6c, + 0x3a, 0x20, 0x7b, 0x20, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x20, + 0x7d, 0x20, 0x7d, 0x29, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x7d, 0x29, 0x28, + 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, + 0x65, 0x76, 0x65, 0x6e, 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x3b, + 0x0a, 0x7d, 0x0a, 0x0a, 0x2f, 0x2f, 0x20, 0x43, 0x61, 0x6c, 0x6c, 0x20, + 0x6c, 0x6c, 0x61, 0x6d, 0x61, 0x2c, 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, + 0x6e, 0x20, 0x61, 0x20, 0x70, 0x72, 0x6f, 0x6d, 0x69, 0x73, 0x65, 0x20, + 0x74, 0x68, 0x61, 0x74, 0x20, 0x72, 0x65, 0x73, 0x6f, 0x6c, 0x76, 0x65, + 0x73, 0x20, 0x74, 0x6f, 0x20, 0x74, 0x68, 0x65, 0x20, 0x63, 0x6f, 0x6d, + 0x70, 0x6c, 0x65, 0x74, 0x65, 0x64, 0x20, 0x74, 0x65, 0x78, 0x74, 0x2e, + 0x20, 0x54, 0x68, 0x69, 0x73, 0x20, 0x64, 0x6f, 0x65, 0x73, 0x20, 0x6e, + 0x6f, 0x74, 0x20, 0x73, 0x75, 0x70, 0x70, 0x6f, 0x72, 0x74, 0x20, 0x73, + 0x74, 0x72, 0x65, 0x61, 0x6d, 0x69, 0x6e, 0x67, 0x0a, 0x2f, 0x2f, 0x0a, + 0x2f, 0x2f, 0x20, 0x45, 0x78, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x3a, 0x0a, + 0x2f, 0x2f, 0x0a, 0x2f, 0x2f, 0x20, 0x20, 0x20, 0x20, 0x20, 0x6c, 0x6c, + 0x61, 0x6d, 0x61, 0x50, 0x72, 0x6f, 0x6d, 0x69, 0x73, 0x65, 0x28, 0x70, + 0x72, 0x6f, 0x6d, 0x70, 0x74, 0x29, 0x2e, 0x74, 0x68, 0x65, 0x6e, 0x28, + 0x28, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x29, 0x20, 0x3d, 0x3e, + 0x20, 0x7b, 0x0a, 0x2f, 0x2f, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x64, 0x6f, 0x63, 0x75, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x77, 0x72, 0x69, + 0x74, 0x65, 0x28, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x29, 0x0a, + 0x2f, 0x2f, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x29, 0x0a, 0x2f, 0x2f, + 0x0a, 0x2f, 0x2f, 0x20, 0x20, 0x20, 0x20, 0x20, 0x6f, 0x72, 0x0a, 0x2f, + 0x2f, 0x0a, 0x2f, 0x2f, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, + 0x73, 0x74, 0x20, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x20, 0x3d, + 0x20, 0x61, 0x77, 0x61, 0x69, 0x74, 0x20, 0x6c, 0x6c, 0x61, 0x6d, 0x61, + 0x50, 0x72, 0x6f, 0x6d, 0x69, 0x73, 0x65, 0x28, 0x70, 0x72, 0x6f, 0x6d, + 0x70, 0x74, 0x29, 0x0a, 0x2f, 0x2f, 0x20, 0x20, 0x20, 0x20, 0x20, 0x64, + 0x6f, 0x63, 0x75, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x77, 0x72, 0x69, 0x74, + 0x65, 0x28, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x29, 0x0a, 0x2f, + 0x2f, 0x0a, 0x65, 0x78, 0x70, 0x6f, 0x72, 0x74, 0x20, 0x63, 0x6f, 0x6e, + 0x73, 0x74, 0x20, 0x6c, 0x6c, 0x61, 0x6d, 0x61, 0x50, 0x72, 0x6f, 0x6d, + 0x69, 0x73, 0x65, 0x20, 0x3d, 0x20, 0x28, 0x70, 0x72, 0x6f, 0x6d, 0x70, + 0x74, 0x2c, 0x20, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x20, 0x3d, 0x20, + 0x7b, 0x7d, 0x2c, 0x20, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x20, 0x3d, + 0x20, 0x7b, 0x7d, 0x29, 0x20, 0x3d, 0x3e, 0x20, 0x7b, 0x0a, 0x20, 0x20, + 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x6e, 0x65, 0x77, 0x20, 0x50, + 0x72, 0x6f, 0x6d, 0x69, 0x73, 0x65, 0x28, 0x61, 0x73, 0x79, 0x6e, 0x63, + 0x20, 0x28, 0x72, 0x65, 0x73, 0x6f, 0x6c, 0x76, 0x65, 0x2c, 0x20, 0x72, + 0x65, 0x6a, 0x65, 0x63, 0x74, 0x29, 0x20, 0x3d, 0x3e, 0x20, 0x7b, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x6c, 0x65, 0x74, 0x20, 0x63, 0x6f, 0x6e, 0x74, + 0x65, 0x6e, 0x74, 0x20, 0x3d, 0x20, 0x22, 0x22, 0x3b, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x74, 0x72, 0x79, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x66, 0x6f, 0x72, 0x20, 0x61, 0x77, 0x61, 0x69, 0x74, 0x20, + 0x28, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x63, 0x68, 0x75, 0x6e, 0x6b, + 0x20, 0x6f, 0x66, 0x20, 0x6c, 0x6c, 0x61, 0x6d, 0x61, 0x28, 0x70, 0x72, + 0x6f, 0x6d, 0x70, 0x74, 0x2c, 0x20, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, + 0x2c, 0x20, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x29, 0x29, 0x20, 0x7b, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, + 0x74, 0x65, 0x6e, 0x74, 0x20, 0x2b, 0x3d, 0x20, 0x63, 0x68, 0x75, 0x6e, + 0x6b, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x65, + 0x6e, 0x74, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, 0x73, 0x6f, 0x6c, 0x76, + 0x65, 0x28, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x29, 0x3b, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x7d, 0x20, 0x63, 0x61, 0x74, 0x63, 0x68, 0x20, + 0x28, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, 0x6a, 0x65, 0x63, 0x74, 0x28, 0x65, + 0x72, 0x72, 0x6f, 0x72, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x7d, + 0x0a, 0x20, 0x20, 0x7d, 0x29, 0x3b, 0x0a, 0x7d, 0x3b, 0x0a, 0x0a, 0x2f, + 0x2a, 0x2a, 0x0a, 0x20, 0x2a, 0x20, 0x28, 0x64, 0x65, 0x70, 0x72, 0x65, + 0x63, 0x61, 0x74, 0x65, 0x64, 0x29, 0x0a, 0x20, 0x2a, 0x2f, 0x0a, 0x65, + 0x78, 0x70, 0x6f, 0x72, 0x74, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, + 0x6c, 0x6c, 0x61, 0x6d, 0x61, 0x43, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, + 0x65, 0x20, 0x3d, 0x20, 0x61, 0x73, 0x79, 0x6e, 0x63, 0x20, 0x28, 0x70, + 0x61, 0x72, 0x61, 0x6d, 0x73, 0x2c, 0x20, 0x63, 0x6f, 0x6e, 0x74, 0x72, + 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x2c, 0x20, 0x63, 0x61, 0x6c, 0x6c, 0x62, + 0x61, 0x63, 0x6b, 0x29, 0x20, 0x3d, 0x3e, 0x20, 0x7b, 0x0a, 0x20, 0x20, + 0x66, 0x6f, 0x72, 0x20, 0x61, 0x77, 0x61, 0x69, 0x74, 0x20, 0x28, 0x63, + 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x63, 0x68, 0x75, 0x6e, 0x6b, 0x20, 0x6f, + 0x66, 0x20, 0x6c, 0x6c, 0x61, 0x6d, 0x61, 0x28, 0x70, 0x61, 0x72, 0x61, + 0x6d, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x6d, 0x70, 0x74, 0x2c, 0x20, 0x70, + 0x61, 0x72, 0x61, 0x6d, 0x73, 0x2c, 0x20, 0x7b, 0x20, 0x63, 0x6f, 0x6e, + 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x20, 0x7d, 0x29, 0x29, 0x20, + 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x63, 0x61, 0x6c, 0x6c, 0x62, 0x61, + 0x63, 0x6b, 0x28, 0x63, 0x68, 0x75, 0x6e, 0x6b, 0x29, 0x3b, 0x0a, 0x20, + 0x20, 0x7d, 0x0a, 0x7d, 0x0a, 0x0a, 0x2f, 0x2f, 0x20, 0x47, 0x65, 0x74, + 0x20, 0x74, 0x68, 0x65, 0x20, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x20, 0x69, + 0x6e, 0x66, 0x6f, 0x20, 0x66, 0x72, 0x6f, 0x6d, 0x20, 0x74, 0x68, 0x65, + 0x20, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x2e, 0x20, 0x54, 0x68, 0x69, + 0x73, 0x20, 0x69, 0x73, 0x20, 0x75, 0x73, 0x65, 0x66, 0x75, 0x6c, 0x20, + 0x66, 0x6f, 0x72, 0x20, 0x67, 0x65, 0x74, 0x74, 0x69, 0x6e, 0x67, 0x20, + 0x74, 0x68, 0x65, 0x20, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x20, + 0x77, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x20, 0x61, 0x6e, 0x64, 0x20, 0x73, + 0x6f, 0x20, 0x6f, 0x6e, 0x2e, 0x0a, 0x65, 0x78, 0x70, 0x6f, 0x72, 0x74, + 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x6c, 0x6c, 0x61, 0x6d, 0x61, + 0x4d, 0x6f, 0x64, 0x65, 0x6c, 0x49, 0x6e, 0x66, 0x6f, 0x20, 0x3d, 0x20, + 0x61, 0x73, 0x79, 0x6e, 0x63, 0x20, 0x28, 0x29, 0x20, 0x3d, 0x3e, 0x20, + 0x7b, 0x0a, 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, 0x21, 0x67, 0x65, 0x6e, + 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x73, 0x65, 0x74, 0x74, + 0x69, 0x6e, 0x67, 0x73, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x70, 0x72, 0x6f, 0x70, 0x73, 0x20, + 0x3d, 0x20, 0x61, 0x77, 0x61, 0x69, 0x74, 0x20, 0x66, 0x65, 0x74, 0x63, + 0x68, 0x28, 0x22, 0x2f, 0x70, 0x72, 0x6f, 0x70, 0x73, 0x22, 0x29, 0x2e, + 0x74, 0x68, 0x65, 0x6e, 0x28, 0x72, 0x20, 0x3d, 0x3e, 0x20, 0x72, 0x2e, + 0x6a, 0x73, 0x6f, 0x6e, 0x28, 0x29, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, + 0x73, 0x65, 0x74, 0x74, 0x69, 0x6e, 0x67, 0x73, 0x20, 0x3d, 0x20, 0x70, + 0x72, 0x6f, 0x70, 0x73, 0x2e, 0x64, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, + 0x5f, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, + 0x73, 0x65, 0x74, 0x74, 0x69, 0x6e, 0x67, 0x73, 0x3b, 0x0a, 0x20, 0x20, + 0x7d, 0x0a, 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x67, + 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x73, 0x65, + 0x74, 0x74, 0x69, 0x6e, 0x67, 0x73, 0x3b, 0x0a, 0x7d, 0x0a }; -unsigned int completion_js_len = 5346; +unsigned int completion_js_len = 5782; diff --git a/examples/server/public/completion.js b/examples/server/public/completion.js index baaec1d60..ab38a7b40 100644 --- a/examples/server/public/completion.js +++ b/examples/server/public/completion.js @@ -195,7 +195,8 @@ export const llamaComplete = async (params, controller, callback) => { // Get the model info from the server. This is useful for getting the context window and so on. export const llamaModelInfo = async () => { if (!generation_settings) { - generation_settings = await fetch("/model.json").then(r => r.json()); + const props = await fetch("/props").then(r => r.json()); + generation_settings = props.default_generation_settings; } return generation_settings; } diff --git a/examples/server/server.cpp b/examples/server/server.cpp index d86d7e04a..9481ce6b1 100644 --- a/examples/server/server.cpp +++ b/examples/server/server.cpp @@ -990,11 +990,6 @@ struct llama_server_context queue_results.send(res); } - json get_model_props() - { - return get_formated_generation(slots[0]); - } - json get_formated_generation(llama_client_slot &slot) { const auto eos_bias = slot.sparams.logit_bias.find(llama_token_eos(model)); @@ -2895,12 +2890,6 @@ int main(int argc, char **argv) } }); - svr.Get("/model.json", [&llama](const httplib::Request &, httplib::Response &res) - { - const json data = llama.get_model_props(); - return res.set_content(data.dump(), "application/json; charset=utf-8"); - }); - svr.Options(R"(/.*)", [](const httplib::Request &, httplib::Response &res) { return res.set_content("", "application/json; charset=utf-8"); }); From f68664ac241a6b5c233d8f1051eef20929b06008 Mon Sep 17 00:00:00 2001 From: Sang-Kil Park Date: Wed, 7 Feb 2024 13:28:00 +0900 Subject: [PATCH 281/812] convert : fix TypeError on GPT-2 vocab.json (#5288) --- convert.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/convert.py b/convert.py index 4a2847a27..323e8058d 100755 --- a/convert.py +++ b/convert.py @@ -334,9 +334,9 @@ class Params: class BpeVocab: def __init__(self, fname_tokenizer: Path, fname_added_tokens: Path | None) -> None: self.bpe_tokenizer = json.loads(open(str(fname_tokenizer), encoding="utf-8").read()) - try: + if isinstance(self.bpe_tokenizer.get('model'), dict): self.vocab = self.bpe_tokenizer["model"]["vocab"] - except KeyError: + else: self.vocab = self.bpe_tokenizer added_tokens: dict[str, int] if fname_added_tokens is not None: From f3e2b4fa3f81a410ecb7dec929c259ef8d8dbb7d Mon Sep 17 00:00:00 2001 From: Justin Parker Date: Wed, 7 Feb 2024 01:15:19 -0500 Subject: [PATCH 282/812] server : update `/props` with "total_slots" value (#5373) * include total "num_slots" in default_generation_settings_for_props * cleanup total_slots return value in /props endpoint * update /props endpoint docs with total_slots * remove num_slots from default_generation_settings_for_props * update /props endpoint section --- examples/server/README.md | 4 +++- examples/server/server.cpp | 4 ++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/examples/server/README.md b/examples/server/README.md index 46d8f85ae..1db7cdf21 100644 --- a/examples/server/README.md +++ b/examples/server/README.md @@ -276,13 +276,15 @@ Notice that each `probs` is an array of length `n_probs`. { "assistant_name": "", "user_name": "", - "default_generation_settings": { ... } + "default_generation_settings": { ... }, + "total_slots": 1 } ``` - `assistant_name` - the required assistant name to generate the prompt in case you have specified a system prompt for all slots. - `user_name` - the required anti-prompt to generate the prompt in case you have specified a system prompt for all slots. - `default_generation_settings` - the default generation settings for the `/completion` endpoint, has the same fields as the `generation_settings` response object from the `/completion` endpoint. +- `total_slots` - the total number of slots for process requests (defined by `--parallel` option) - **POST** `/v1/chat/completions`: OpenAI-compatible Chat Completions API. Given a ChatML-formatted json description in `messages`, it returns the predicted completion. Both synchronous and streaming mode are supported, so scripted and interactive applications work fine. While no strong claims of compatibility with OpenAI API spec is being made, in our experience it suffices to support many apps. Only ChatML-tuned models, such as Dolphin, OpenOrca, OpenHermes, OpenChat-3.5, etc can be used with this endpoint. Compared to `api_like_OAI.py` this API implementation does not require a wrapper to be served. diff --git a/examples/server/server.cpp b/examples/server/server.cpp index 9481ce6b1..eceda30d0 100644 --- a/examples/server/server.cpp +++ b/examples/server/server.cpp @@ -432,7 +432,6 @@ struct llama_server_context } default_generation_settings_for_props = get_formated_generation(slots.front()); - default_generation_settings_for_props["num_slots"] = params.n_parallel; default_generation_settings_for_props["seed"] = -1; batch = llama_batch_init(n_ctx, 0, params.n_parallel); @@ -2639,7 +2638,8 @@ int main(int argc, char **argv) json data = { { "user_name", llama.name_user.c_str() }, { "assistant_name", llama.name_assistant.c_str() }, - { "default_generation_settings", llama.default_generation_settings_for_props } + { "default_generation_settings", llama.default_generation_settings_for_props }, + { "total_slots", llama.params.n_parallel } }; res.set_content(data.dump(), "application/json; charset=utf-8"); }); From 316c7faf7740fa98ea68f1445f4505810f706b9e Mon Sep 17 00:00:00 2001 From: runfuture Date: Wed, 7 Feb 2024 14:15:56 +0800 Subject: [PATCH 283/812] llama : add MiniCPM support (#5346) * support minicpm arch. * fix tab/space typo. * convert minicpm model via convert-hf-gguf.py * try to make tokenizer work * fix bug for quantize minicpm * fix for flake8 lint * remove convert-minicpm.py * fix for editorconfig * correct minicpm model type (size) * constants expanded for minicpm * Minor change of the constant names for minicpm --- convert-hf-to-gguf.py | 49 ++++++++++ gguf-py/gguf/constants.py | 21 +++++ llama.cpp | 190 +++++++++++++++++++++++++++++++++++++- 3 files changed, 259 insertions(+), 1 deletion(-) diff --git a/convert-hf-to-gguf.py b/convert-hf-to-gguf.py index 5e343742d..829d68368 100755 --- a/convert-hf-to-gguf.py +++ b/convert-hf-to-gguf.py @@ -22,6 +22,8 @@ if 'NO_LOCAL_GGUF' not in os.environ: sys.path.insert(1, str(Path(__file__).parent / 'gguf-py')) import gguf +from convert import HfVocab + # check for any of the given keys in the dictionary and return the value of the first key found def get_key_opts(d, keys): @@ -205,6 +207,8 @@ class Model: return OrionModel if model_architecture == "InternLM2ForCausalLM": return InternLM2Model + if model_architecture == "MiniCPMForCausalLM": + return MiniCPMModel return Model def _is_model_safetensors(self) -> bool: @@ -258,6 +262,8 @@ class Model: return gguf.MODEL_ARCH.ORION if arch == "InternLM2ForCausalLM": return gguf.MODEL_ARCH.INTERNLM2 + if arch == "MiniCPMForCausalLM": + return gguf.MODEL_ARCH.MINICPM raise NotImplementedError(f'Architecture "{arch}" not supported!') @@ -402,6 +408,31 @@ class Model: special_vocab = gguf.SpecialVocab(self.dir_model, n_vocab=len(tokens)) special_vocab.add_to_gguf(self.gguf_writer) + def _set_vocab_hf(self): + path = self.dir_model + added_tokens_path = self.dir_model + vocab = HfVocab( + path, added_tokens_path if added_tokens_path.exists() else None + ) + tokens = [] + scores = [] + toktypes = [] + + for text, score, toktype in vocab.all_tokens(): + tokens.append(text) + scores.append(score) + toktypes.append(toktype) + + assert len(tokens) == vocab.vocab_size + + self.gguf_writer.add_tokenizer_model("llama") + self.gguf_writer.add_token_list(tokens) + self.gguf_writer.add_token_scores(scores) + self.gguf_writer.add_token_types(toktypes) + + special_vocab = gguf.SpecialVocab(self.dir_model, n_vocab=len(tokens)) + special_vocab.add_to_gguf(self.gguf_writer) + class GPTNeoXModel(Model): def set_gguf_parameters(self): @@ -1041,6 +1072,24 @@ class MixtralModel(Model): self._set_vocab_sentencepiece() +class MiniCPMModel(Model): + def set_gguf_parameters(self): + block_count = self.hparams["num_hidden_layers"] + self.gguf_writer.add_name("MiniCPM") + self.gguf_writer.add_context_length(self.hparams["max_position_embeddings"]) + self.gguf_writer.add_embedding_length(self.hparams["hidden_size"]) + self.gguf_writer.add_feed_forward_length(self.hparams["intermediate_size"]) + self.gguf_writer.add_block_count(block_count) + self.gguf_writer.add_head_count(self.hparams["num_attention_heads"]) + self.gguf_writer.add_head_count_kv(self.hparams["num_key_value_heads"]) + self.gguf_writer.add_layer_norm_rms_eps(self.hparams["rms_norm_eps"]) + self.gguf_writer.add_file_type(self.ftype) + self.gguf_writer.add_rope_dimension_count(self.hparams["hidden_size"] // self.hparams["num_attention_heads"]) + + def set_vocab(self): + self._set_vocab_hf() + + class QwenModel(Model): @staticmethod def token_bytes_to_string(b): diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py index ed8e26f83..1cfd41c0b 100644 --- a/gguf-py/gguf/constants.py +++ b/gguf-py/gguf/constants.py @@ -104,6 +104,7 @@ class MODEL_ARCH(IntEnum): CODESHELL = auto() ORION = auto() INTERNLM2 = auto() + MINICPM = auto() class MODEL_TENSOR(IntEnum): @@ -156,6 +157,7 @@ MODEL_ARCH_NAMES: dict[MODEL_ARCH, str] = { MODEL_ARCH.CODESHELL: "codeshell", MODEL_ARCH.ORION: "orion", MODEL_ARCH.INTERNLM2: "internlm2", + MODEL_ARCH.MINICPM: "minicpm", } TENSOR_NAMES: dict[MODEL_TENSOR, str] = { @@ -464,6 +466,25 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = { MODEL_TENSOR.FFN_DOWN, MODEL_TENSOR.FFN_UP, ], + MODEL_ARCH.MINICPM: [ + MODEL_TENSOR.TOKEN_EMBD, + MODEL_TENSOR.OUTPUT_NORM, + MODEL_TENSOR.ROPE_FREQS, + MODEL_TENSOR.ATTN_NORM, + MODEL_TENSOR.ATTN_Q, + MODEL_TENSOR.ATTN_K, + MODEL_TENSOR.ATTN_V, + MODEL_TENSOR.ATTN_OUT, + MODEL_TENSOR.ATTN_ROT_EMBD, + MODEL_TENSOR.FFN_GATE_INP, + MODEL_TENSOR.FFN_NORM, + MODEL_TENSOR.FFN_GATE, + MODEL_TENSOR.FFN_DOWN, + MODEL_TENSOR.FFN_UP, + MODEL_TENSOR.FFN_GATE_EXP, + MODEL_TENSOR.FFN_DOWN_EXP, + MODEL_TENSOR.FFN_UP_EXP, + ], # TODO } diff --git a/llama.cpp b/llama.cpp index 65e399adc..f3c5146d1 100644 --- a/llama.cpp +++ b/llama.cpp @@ -205,6 +205,7 @@ enum llm_arch { LLM_ARCH_CODESHELL, LLM_ARCH_ORION, LLM_ARCH_INTERNLM2, + LLM_ARCH_MINICPM, LLM_ARCH_UNKNOWN, }; @@ -228,6 +229,7 @@ static std::map LLM_ARCH_NAMES = { { LLM_ARCH_CODESHELL, "codeshell" }, { LLM_ARCH_ORION, "orion" }, { LLM_ARCH_INTERNLM2, "internlm2" }, + { LLM_ARCH_MINICPM, "minicpm" }, }; enum llm_kv { @@ -690,6 +692,29 @@ static std::map> LLM_TENSOR_NAMES = { LLM_TENSOR_FFN_UP, "blk.%d.ffn_up" }, }, }, + { + LLM_ARCH_MINICPM, + { + { LLM_TENSOR_TOKEN_EMBD, "token_embd" }, + { LLM_TENSOR_OUTPUT_NORM, "output_norm" }, + { LLM_TENSOR_OUTPUT, "output" }, + { LLM_TENSOR_ROPE_FREQS, "rope_freqs" }, + { LLM_TENSOR_ATTN_NORM, "blk.%d.attn_norm" }, + { LLM_TENSOR_ATTN_Q, "blk.%d.attn_q" }, + { LLM_TENSOR_ATTN_K, "blk.%d.attn_k" }, + { LLM_TENSOR_ATTN_V, "blk.%d.attn_v" }, + { LLM_TENSOR_ATTN_OUT, "blk.%d.attn_output" }, + { LLM_TENSOR_ATTN_ROT_EMBD, "blk.%d.attn_rot_embd" }, + { LLM_TENSOR_FFN_GATE_INP, "blk.%d.ffn_gate_inp" }, + { LLM_TENSOR_FFN_NORM, "blk.%d.ffn_norm" }, + { LLM_TENSOR_FFN_GATE, "blk.%d.ffn_gate" }, + { LLM_TENSOR_FFN_DOWN, "blk.%d.ffn_down" }, + { LLM_TENSOR_FFN_UP, "blk.%d.ffn_up" }, + { LLM_TENSOR_FFN_GATE_EXP, "blk.%d.ffn_gate.%d" }, + { LLM_TENSOR_FFN_DOWN_EXP, "blk.%d.ffn_down.%d" }, + { LLM_TENSOR_FFN_UP_EXP, "blk.%d.ffn_up.%d" }, + }, + }, { LLM_ARCH_UNKNOWN, { @@ -1390,6 +1415,7 @@ enum e_model { MODEL_UNKNOWN, MODEL_0_5B, MODEL_1B, + MODEL_2B, MODEL_3B, MODEL_4B, MODEL_7B, @@ -2748,6 +2774,7 @@ static std::string llama_model_ftype_name(llama_ftype ftype) { static const char * llama_model_type_name(e_model type) { switch (type) { case MODEL_1B: return "1B"; + case MODEL_2B: return "2B"; case MODEL_3B: return "3B"; case MODEL_7B: return "7B"; case MODEL_8B: return "8B"; @@ -2887,6 +2914,13 @@ static void llm_load_hparams( default: model.type = e_model::MODEL_UNKNOWN; } } break; + case LLM_ARCH_MINICPM: + { + switch (hparams.n_layer) { + case 40: model.type = e_model::MODEL_2B; break; + default: model.type = e_model::MODEL_UNKNOWN; + } + } break; case LLM_ARCH_FALCON: { ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps); @@ -3524,13 +3558,16 @@ static bool llm_load_tensors( switch (model.arch) { case LLM_ARCH_LLAMA: case LLM_ARCH_REFACT: + case LLM_ARCH_MINICPM: { model.tok_embd = ml.create_tensor(ctx_input, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}); // output { model.output_norm = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}); - model.output = ml.create_tensor(ctx_output_split, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}); + if (model.arch != LLM_ARCH_MINICPM){ + model.output = ml.create_tensor(ctx_output_split, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}); + } } for (int i = 0; i < n_layer; ++i) { @@ -6781,6 +6818,153 @@ struct llm_build_context { return gf; } + // ref: https://arxiv.org/abs/2203.03466 + // https://github.com/ggerganov/llama.cpp/issues/5276#issuecomment-1925774738 + // based on the original build_llama() function + struct ggml_cgraph * build_minicpm() { + struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, LLAMA_MAX_NODES, false); + + const int64_t n_embd_head = hparams.n_embd_head_v; + GGML_ASSERT(n_embd_head == hparams.n_embd_head_k); + GGML_ASSERT(n_embd_head == hparams.n_rot); + + const int64_t n_embd = hparams.n_embd; + //TODO: if the model varies, these parameters need to be read from the model + const int64_t n_embd_base = 256; + const float scale_embd = 12.0f; + const float scale_depth = 1.4f; + + struct ggml_tensor * cur; + struct ggml_tensor * inpL; + + inpL = llm_build_inp_embd(ctx0, hparams, batch, model.tok_embd, lctx.inp_tokens, lctx.inp_embd, cb); + cb(inpL, "inp_embd", -1); + + // scale the input embeddings + inpL = ggml_scale(ctx0, inpL, scale_embd); + cb(inpL, "inp_scaled", -1); + + // inp_pos - contains the positions + struct ggml_tensor * inp_pos = ggml_view_1d(ctx0, lctx.inp_pos, n_tokens, 0); + cb(inp_pos, "inp_pos", -1); + + // KQ_mask (mask for 1 head, it will be broadcasted to all heads) + struct ggml_tensor * KQ_mask = ggml_view_2d(ctx0, lctx.inp_KQ_mask, n_kv, n_tokens, n_kv*ggml_type_size(lctx.inp_KQ_mask->type), 0); + cb(KQ_mask, "KQ_mask", -1); + + // shift the entire K-cache if needed + if (do_rope_shift) { + llm_build_k_shift(ctx0, hparams, cparams, kv_self, gf, lctx.inp_K_shift, LLM_ROPE, n_ctx, freq_base, freq_scale, cb); + } + + for (int il = 0; il < n_layer; ++il) { + struct ggml_tensor * inpSA = inpL; + + // norm + cur = llm_build_norm(ctx0, inpL, hparams, + model.layers[il].attn_norm, NULL, + LLM_NORM_RMS, cb, il); + cb(cur, "attn_norm", il); + + // self-attention + { + // compute Q and K and RoPE them + struct ggml_tensor * Qcur = ggml_mul_mat(ctx0, model.layers[il].wq, cur); + cb(Qcur, "Qcur", il); + if (model.layers[il].bq) { + Qcur = ggml_add(ctx0, Qcur, model.layers[il].bq); + cb(Qcur, "Qcur", il); + } + + struct ggml_tensor * Kcur = ggml_mul_mat(ctx0, model.layers[il].wk, cur); + cb(Kcur, "Kcur", il); + if (model.layers[il].bk) { + Kcur = ggml_add(ctx0, Kcur, model.layers[il].bk); + cb(Kcur, "Kcur", il); + } + + struct ggml_tensor * Vcur = ggml_mul_mat(ctx0, model.layers[il].wv, cur); + cb(Vcur, "Vcur", il); + if (model.layers[il].bv) { + Vcur = ggml_add(ctx0, Vcur, model.layers[il].bv); + cb(Vcur, "Vcur", il); + } + + Qcur = ggml_rope_custom( + ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, + hparams.n_rot, 0, 0, n_orig_ctx, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow + ); + cb(Qcur, "Qcur", il); + + Kcur = ggml_rope_custom( + ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, + hparams.n_rot, 0, 0, n_orig_ctx, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow + ); + cb(Kcur, "Kcur", il); + + cur = llm_build_kv(ctx0, model, hparams, kv_self, gf, + model.layers[il].wo, model.layers[il].bo, + Kcur, Vcur, Qcur, KQ_mask, n_ctx, n_tokens, kv_head, n_kv, -1.0f, 1.0f/sqrtf(float(n_embd_head)), cb, il); + cb(cur, "kqv_out", il); + } + + // scale_res - scale the hidden states for residual connection + const float scale_res = scale_depth/sqrtf(float(n_layer)); + cur = ggml_scale(ctx0, cur, scale_res); + cb(cur, "hidden_scaled", -1); + + struct ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA); + cb(ffn_inp, "ffn_inp", il); + + // feed-forward network + { + cur = llm_build_norm(ctx0, ffn_inp, hparams, + model.layers[il].ffn_norm, NULL, + LLM_NORM_RMS, cb, il); + cb(cur, "ffn_norm", il); + + cur = llm_build_ffn(ctx0, cur, + model.layers[il].ffn_up, NULL, + model.layers[il].ffn_gate, NULL, + model.layers[il].ffn_down, NULL, + NULL, + LLM_FFN_SILU, LLM_FFN_PAR, cb, il); + cb(cur, "ffn_out", il); + } + + // scale the hidden states for residual connection + cur = ggml_scale(ctx0, cur, scale_res); + cb(cur, "hidden_scaled_ffn", -1); + + cur = ggml_add(ctx0, cur, ffn_inp); + cb(cur, "l_out", il); + + // input for next layer + inpL = cur; + } + + cur = inpL; + + cur = llm_build_norm(ctx0, cur, hparams, + model.output_norm, NULL, + LLM_NORM_RMS, cb, -1); + cb(cur, "result_norm", -1); + + // lm_head scaling + const float scale_lmhead = float(n_embd_base)/float(n_embd); + cur = ggml_scale(ctx0, cur, scale_lmhead); + cb(cur, "lmhead_scaling", -1); + + // lm_head + cur = ggml_mul_mat(ctx0, model.tok_embd, cur); + cb(cur, "result_output", -1); + + ggml_build_forward_expand(gf, cur); + + return gf; + } }; static struct ggml_cgraph * llama_build_graph( @@ -6943,6 +7127,10 @@ static struct ggml_cgraph * llama_build_graph( { result = llm.build_internlm2(); } break; + case LLM_ARCH_MINICPM: + { + result = llm.build_minicpm(); + } break; default: GGML_ASSERT(false); } From 9a697d842bc0cfce8268ebd2ba703ffc1c904f98 Mon Sep 17 00:00:00 2001 From: Ben Williams Date: Tue, 6 Feb 2024 22:16:48 -0800 Subject: [PATCH 284/812] readme : update ui list (#5354) --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 34f2021f9..672512d18 100644 --- a/README.md +++ b/README.md @@ -149,6 +149,7 @@ Unless otherwise noted these projects are open-source with permissive licensing: - [iohub/collama](https://github.com/iohub/coLLaMA) - [janhq/jan](https://github.com/janhq/jan) (AGPL) - [nat/openplayground](https://github.com/nat/openplayground) +- [Faraday](https://faraday.dev/) (proprietary) - [LMStudio](https://lmstudio.ai/) (proprietary) - [LostRuins/koboldcpp](https://github.com/LostRuins/koboldcpp) (AGPL) - [Mozilla-Ocho/llamafile](https://github.com/Mozilla-Ocho/llamafile) From ed0bf32290ee5b30ffad5becd99cbecef74aedd7 Mon Sep 17 00:00:00 2001 From: Eve <139727413+netrunnereve@users.noreply.github.com> Date: Wed, 7 Feb 2024 06:21:30 +0000 Subject: [PATCH 285/812] readme : modernize (#5379) * first cleanup, update everything to Llama 2 and remove outdated content * Delete SHA256SUMS * make build instructions generic * recommend Q4_K_M quantization method * Update README.md --- README.md | 127 +++++++++++++++-------------------------------------- SHA256SUMS | 40 ----------------- 2 files changed, 36 insertions(+), 131 deletions(-) delete mode 100644 SHA256SUMS diff --git a/README.md b/README.md index 672512d18..0509b0ba1 100644 --- a/README.md +++ b/README.md @@ -33,17 +33,14 @@ Inference of Meta's [LLaMA](https://arxiv.org/abs/2302.13971) model (and others)
  • Get the Code
  • Build
  • BLAS Build
  • -
  • Prepare Data & Run
  • +
  • Prepare and Quantize
  • +
  • Run the quantized model
  • Memory/Disk Requirements
  • Quantization
  • Interactive mode
  • Constrained output with grammars
  • -
  • Instruction mode with Alpaca
  • -
  • Using OpenLLaMA
  • -
  • Using GPT4All
  • -
  • Using Pygmalion 7B & Metharme 7B
  • -
  • Obtaining the Facebook LLaMA original model and Stanford Alpaca model data
  • -
  • Verifying the model files
  • +
  • Instruct mode
  • +
  • Obtaining and using the Facebook LLaMA 2 model
  • Seminal papers and background on the models
  • Perplexity (measuring model quality)
  • Android
  • @@ -83,20 +80,16 @@ improved significantly thanks to many contributions. It is the main playground f **Supported models:** +Typically finetunes of the base models below are supported as well. + - [X] LLaMA 🦙 - [x] LLaMA 2 🦙🦙 -- [X] [Mistral AI v0.1](https://huggingface.co/mistralai/Mistral-7B-v0.1) +- [X] [Mistral 7B](https://huggingface.co/mistralai/Mistral-7B-v0.1) - [x] [Mixtral MoE](https://huggingface.co/models?search=mistral-ai/Mixtral) - [X] Falcon -- [X] [Alpaca](https://github.com/ggerganov/llama.cpp#instruction-mode-with-alpaca) -- [X] [GPT4All](https://github.com/ggerganov/llama.cpp#using-gpt4all) - [X] [Chinese LLaMA / Alpaca](https://github.com/ymcui/Chinese-LLaMA-Alpaca) and [Chinese LLaMA-2 / Alpaca-2](https://github.com/ymcui/Chinese-LLaMA-Alpaca-2) - [X] [Vigogne (French)](https://github.com/bofenghuang/vigogne) -- [X] [Vicuna](https://github.com/ggerganov/llama.cpp/discussions/643#discussioncomment-5533894) - [X] [Koala](https://bair.berkeley.edu/blog/2023/04/03/koala/) -- [X] [OpenBuddy 🐶 (Multilingual)](https://github.com/OpenBuddy/OpenBuddy) -- [X] [Pygmalion/Metharme](#using-pygmalion-7b--metharme-7b) -- [X] [WizardLM](https://github.com/nlpxucan/WizardLM) - [X] [Baichuan 1 & 2](https://huggingface.co/models?search=baichuan-inc/Baichuan) + [derivations](https://huggingface.co/hiyouga/baichuan-7b-sft) - [X] [Aquila 1 & 2](https://huggingface.co/models?search=BAAI/Aquila) - [X] [Starcoder models](https://github.com/ggerganov/llama.cpp/pull/3187) @@ -166,7 +159,7 @@ Unless otherwise noted these projects are open-source with permissive licensing: Here is a typical run using LLaMA v2 13B on M2 Ultra: -```java +``` $ make -j && ./main -m models/llama-13b-v2/ggml-model-q4_0.gguf -p "Building a website can be done in 10 simple steps:\nStep 1:" -n 400 -e I llama.cpp build info: I UNAME_S: Darwin @@ -250,7 +243,7 @@ https://user-images.githubusercontent.com/1991296/224442907-7693d4be-acaa-4e01-8 ## Usage -Here are the end-to-end binary build and model conversion steps for the LLaMA-7B model. +Here are the end-to-end binary build and model conversion steps for most supported models. ### Get the Code @@ -635,7 +628,7 @@ Building the program with BLAS support may lead to some performance improvements **Without docker**: - Firstly, you need to make sure you installed [Vulkan SDK](https://vulkan.lunarg.com/doc/view/latest/linux/getting_started_ubuntu.html) + Firstly, you need to make sure you have installed [Vulkan SDK](https://vulkan.lunarg.com/doc/view/latest/linux/getting_started_ubuntu.html) For example, on Ubuntu 22.04 (jammy), use the command below: @@ -648,6 +641,8 @@ Building the program with BLAS support may lead to some performance improvements vulkaninfo ``` + Alternatively your package manager might be able to provide the appropiate libraries. For example for Ubuntu 22.04 you can install `libvulkan-dev` instead. + Then, build llama.cpp using the cmake command below: ```bash @@ -662,34 +657,42 @@ Building the program with BLAS support may lead to some performance improvements # ggml_vulkan: Using Intel(R) Graphics (ADL GT2) | uma: 1 | fp16: 1 | warp size: 32 ``` -### Prepare Data & Run +### Prepare and Quantize + +To obtain the official LLaMA 2 weights please see the Obtaining and using the Facebook LLaMA 2 model section. There is also a large selection of pre-quantized `gguf` models available on Hugging Face. ```bash -# obtain the original LLaMA model weights and place them in ./models +# obtain the official LLaMA model weights and place them in ./models ls ./models -65B 30B 13B 7B tokenizer_checklist.chk tokenizer.model +llama-2-7b tokenizer_checklist.chk tokenizer.model # [Optional] for models using BPE tokenizers ls ./models -65B 30B 13B 7B vocab.json + vocab.json +# [Optional] for PyTorch .bin models like Mistral-7B +ls ./models + # install Python dependencies python3 -m pip install -r requirements.txt -# convert the 7B model to ggml FP16 format -python3 convert.py models/7B/ +# convert the model to ggml FP16 format +python3 convert.py models/mymodel/ # [Optional] for models using BPE tokenizers -python convert.py models/7B/ --vocabtype bpe +python convert.py models/mymodel/ --vocabtype bpe -# quantize the model to 4-bits (using q4_0 method) -./quantize ./models/7B/ggml-model-f16.gguf ./models/7B/ggml-model-q4_0.gguf q4_0 +# quantize the model to 4-bits (using Q4_K_M method) +./quantize ./models/mymodel/ggml-model-f16.gguf ./models/mymodel/ggml-model-Q4_K_M.gguf Q4_K_M -# update the gguf filetype to current if older version is unsupported by another application -./quantize ./models/7B/ggml-model-q4_0.gguf ./models/7B/ggml-model-q4_0-v2.gguf COPY +# update the gguf filetype to current version if older version is now unsupported +./quantize ./models/mymodel/ggml-model-Q4_K_M.gguf ./models/mymodel/ggml-model-Q4_K_M-v2.gguf COPY +``` +### Run the quantized model -# run the inference -./main -m ./models/7B/ggml-model-q4_0.gguf -n 128 +```bash +# start inference on a gguf model +./main -m ./models/mymodel/ggml-model-Q4_K_M.gguf -n 128 ``` When running the larger models, make sure you have enough disk space to store all the intermediate files. @@ -710,7 +713,7 @@ From the unzipped folder, open a terminal/cmd window here and place a pre-conver As the models are currently fully loaded into memory, you will need adequate disk space to save them and sufficient RAM to load them. At the moment, memory and disk requirements are the same. -| Model | Original size | Quantized size (4-bit) | +| Model | Original size | Quantized size (Q4_0) | |------:|--------------:|-----------------------:| | 7B | 13 GB | 3.9 GB | | 13B | 24 GB | 7.8 GB | @@ -826,9 +829,9 @@ The `grammars/` folder contains a handful of sample grammars. To write your own, For authoring more complex JSON grammars, you can also check out https://grammar.intrinsiclabs.ai/, a browser app that lets you write TypeScript interfaces which it compiles to GBNF grammars that you can save for local use. Note that the app is built and maintained by members of the community, please file any issues or FRs on [its repo](http://github.com/intrinsiclabsai/gbnfgen) and not this one. -### Instruction mode with Alpaca +### Instruct mode -1. First, download the `ggml` Alpaca model into the `./models` folder +1. First, download and place the `ggml` model into the `./models` folder 2. Run the `main` tool like this: ``` @@ -854,50 +857,6 @@ cadaver, cauliflower, cabbage (vegetable), catalpa (tree) and Cailleach. > ``` -### Using [OpenLLaMA](https://github.com/openlm-research/open_llama) - -OpenLLaMA is an openly licensed reproduction of Meta's original LLaMA model. It uses the same architecture and is a drop-in replacement for the original LLaMA weights. - -- Download the [3B](https://huggingface.co/openlm-research/open_llama_3b), [7B](https://huggingface.co/openlm-research/open_llama_7b), or [13B](https://huggingface.co/openlm-research/open_llama_13b) model from Hugging Face. -- Convert the model to ggml FP16 format using `python convert.py ` - -### Using [GPT4All](https://github.com/nomic-ai/gpt4all) - -*Note: these instructions are likely obsoleted by the GGUF update* - -- Obtain the `tokenizer.model` file from LLaMA model and put it to `models` -- Obtain the `added_tokens.json` file from Alpaca model and put it to `models` -- Obtain the `gpt4all-lora-quantized.bin` file from GPT4All model and put it to `models/gpt4all-7B` -- It is distributed in the old `ggml` format which is now obsoleted -- You have to convert it to the new format using `convert.py`: - -```bash -python3 convert.py models/gpt4all-7B/gpt4all-lora-quantized.bin -``` - -- You can now use the newly generated `models/gpt4all-7B/ggml-model-q4_0.bin` model in exactly the same way as all other models - -- The newer GPT4All-J model is not yet supported! - -### Using Pygmalion 7B & Metharme 7B - -- Obtain the [LLaMA weights](#obtaining-the-facebook-llama-original-model-and-stanford-alpaca-model-data) -- Obtain the [Pygmalion 7B](https://huggingface.co/PygmalionAI/pygmalion-7b/) or [Metharme 7B](https://huggingface.co/PygmalionAI/metharme-7b) XOR encoded weights -- Convert the LLaMA model with [the latest HF convert script](https://github.com/huggingface/transformers/blob/main/src/transformers/models/llama/convert_llama_weights_to_hf.py) -- Merge the XOR files with the converted LLaMA weights by running the [xor_codec](https://huggingface.co/PygmalionAI/pygmalion-7b/blob/main/xor_codec.py) script -- Convert to `ggml` format using the `convert.py` script in this repo: -```bash -python3 convert.py pygmalion-7b/ --outtype q4_1 -``` -> The Pygmalion 7B & Metharme 7B weights are saved in [bfloat16](https://en.wikipedia.org/wiki/Bfloat16_floating-point_format) precision. If you wish to convert to `ggml` without quantizating, please specify the `--outtype` as `f32` instead of `f16`. - - -### Obtaining the Facebook LLaMA original model and Stanford Alpaca model data - -- **Under no circumstances should IPFS, magnet links, or any other links to model downloads be shared anywhere in this repository, including in issues, discussions, or pull requests. They will be immediately deleted.** -- The LLaMA models are officially distributed by Facebook and will **never** be provided through this repository. -- Refer to [Facebook's LLaMA repository](https://github.com/facebookresearch/llama/pull/73/files) if you need to request access to the model data. - ### Obtaining and using the Facebook LLaMA 2 model - Refer to [Facebook's LLaMA download page](https://ai.meta.com/resources/models-and-libraries/llama-downloads/) if you want to access the model data. @@ -909,20 +868,6 @@ python3 convert.py pygmalion-7b/ --outtype q4_1 - [LLaMA 2 13B chat](https://huggingface.co/TheBloke/Llama-2-13B-chat-GGUF) - [LLaMA 2 70B chat](https://huggingface.co/TheBloke/Llama-2-70B-chat-GGUF) -### Verifying the model files - -Please verify the [sha256 checksums](SHA256SUMS) of all downloaded model files to confirm that you have the correct model data files before creating an issue relating to your model files. -- The following python script will verify if you have all possible latest files in your self-installed `./models` subdirectory: - -```bash -# run the verification script -./scripts/verify-checksum-models.py -``` - -- On linux or macOS it is also possible to run the following commands to verify if you have all possible latest files in your self-installed `./models` subdirectory: - - On Linux: `sha256sum --ignore-missing -c SHA256SUMS` - - on macOS: `shasum -a 256 --ignore-missing -c SHA256SUMS` - ### Seminal papers and background on the models If your issue is with model generation quality, then please at least scan the following links and papers to understand the limitations of LLaMA models. This is especially important when choosing an appropriate model size and appreciating both the significant and subtle differences between LLaMA models and ChatGPT: diff --git a/SHA256SUMS b/SHA256SUMS deleted file mode 100644 index ca4d5a4a5..000000000 --- a/SHA256SUMS +++ /dev/null @@ -1,40 +0,0 @@ -700df0d3013b703a806d2ae7f1bfb8e59814e3d06ae78be0c66368a50059f33d models/7B/consolidated.00.pth -666a4bb533b303bdaf89e1b6a3b6f93535d868de31d903afdc20983dc526c847 models/7B/ggml-model-f16.bin -ec2f2d1f0dfb73b72a4cbac7fa121abbe04c37ab327125a38248f930c0f09ddf models/7B/ggml-model-q4_0.bin -ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff models/7B/ggml-model-q4_1.bin -ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff models/7B/ggml-model-q5_0.bin -ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff models/7B/ggml-model-q5_1.bin -7e89e242ddc0dd6f060b43ca219ce8b3e8f08959a72cb3c0855df8bb04d46265 models/7B/params.json -745bf4e29a4dd6f411e72976d92b452da1b49168a4f41c951cfcc8051823cf08 models/13B/consolidated.00.pth -d5ccbcc465c71c0de439a5aeffebe8344c68a519bce70bc7f9f92654ee567085 models/13B/consolidated.01.pth -2b206e9b21fb1076f11cafc624e2af97c9e48ea09312a0962153acc20d45f808 models/13B/ggml-model-f16.bin -fad169e6f0f575402cf75945961cb4a8ecd824ba4da6be2af831f320c4348fa5 models/13B/ggml-model-q4_0.bin -ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff models/13B/ggml-model-q4_1.bin -ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff models/13B/ggml-model-q5_0.bin -ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff models/13B/ggml-model-q5_1.bin -4ab77bec4d4405ccb66a97b282574c89a94417e3c32e5f68f37e2876fc21322f models/13B/params.json -e23294a58552d8cdec5b7e8abb87993b97ea6eced4178ff2697c02472539d067 models/30B/consolidated.00.pth -4e077b7136c7ae2302e954860cf64930458d3076fcde9443f4d0e939e95903ff models/30B/consolidated.01.pth -24a87f01028cbd3a12de551dcedb712346c0b5cbdeff1454e0ddf2df9b675378 models/30B/consolidated.02.pth -1adfcef71420886119544949767f6a56cb6339b4d5fcde755d80fe68b49de93b models/30B/consolidated.03.pth -7e1b524061a9f4b27c22a12d6d2a5bf13b8ebbea73e99f218809351ed9cf7d37 models/30B/ggml-model-f16.bin -d2a441403944819492ec8c2002cc36fa38468149bfb4b7b4c52afc7bd9a7166d models/30B/ggml-model-q4_0.bin -ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff models/30B/ggml-model-q4_1.bin -ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff models/30B/ggml-model-q5_0.bin -ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff models/30B/ggml-model-q5_1.bin -2c07118ea98d69dbe7810d88520e30288fa994751b337f8fca02b171955f44cb models/30B/params.json -135c563f6b3938114458183afb01adc9a63bef3d8ff7cccc3977e5d3664ecafe models/65B/consolidated.00.pth -9a600b37b19d38c7e43809485f70d17d1dc12206c07efa83bc72bb498a568bde models/65B/consolidated.01.pth -e7babf7c5606f165a3756f527cb0fedc4f83e67ef1290391e52fb1cce5f26770 models/65B/consolidated.02.pth -73176ffb426b40482f2aa67ae1217ef79fbbd1fff5482bae5060cdc5a24ab70e models/65B/consolidated.03.pth -882e6431d0b08a8bc66261a0d3607da21cbaeafa96a24e7e59777632dbdac225 models/65B/consolidated.04.pth -a287c0dfe49081626567c7fe87f74cce5831f58e459b427b5e05567641f47b78 models/65B/consolidated.05.pth -72b4eba67a1a3b18cb67a85b70f8f1640caae9b40033ea943fb166bd80a7b36b models/65B/consolidated.06.pth -d27f5b0677d7ff129ceacd73fd461c4d06910ad7787cf217b249948c3f3bc638 models/65B/consolidated.07.pth -60758f2384d74e423dffddfd020ffed9d3bb186ebc54506f9c4a787d0f5367b0 models/65B/ggml-model-f16.bin -cde053439fa4910ae454407e2717cc46cc2c2b4995c00c93297a2b52e790fa92 models/65B/ggml-model-q4_0.bin -ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff models/65B/ggml-model-q4_1.bin -ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff models/65B/ggml-model-q5_0.bin -ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff models/65B/ggml-model-q5_1.bin -999ed1659b469ccc2a941714c0a9656fa571d17c9f7c8c7589817ca90edef51b models/65B/params.json -9e556afd44213b6bd1be2b850ebbbd98f5481437a8021afaf58ee7fb1818d347 models/tokenizer.model From ee1628bdfea8b0079fed0140ac2f00ef1b465b57 Mon Sep 17 00:00:00 2001 From: 0cc4m Date: Wed, 7 Feb 2024 07:54:50 +0100 Subject: [PATCH 286/812] Basic Vulkan Multi-GPU implementation (#5321) * Initial Vulkan multi-gpu implementation Move most global variables into backend context * Add names to backend device functions * Add further missing cleanup code * Reduce code duplication in tensor split layer assignment * generalize LLAMA_SPLIT_LAYER for all backends, do not expose device count and memory in llama.h * Only do device info print in the beginning and initialize one backend for cpu assist Add missing cleanup code * Rework backend memory management to make sure devices and buffers get properly allocated and freed * Rename cpu assist free function --------- Co-authored-by: slaren --- common/common.cpp | 8 +- ggml-vulkan.cpp | 2639 ++++++++++++++++++++++++++------------------- ggml-vulkan.h | 23 +- ggml.c | 14 +- llama.cpp | 69 +- 5 files changed, 1587 insertions(+), 1166 deletions(-) diff --git a/common/common.cpp b/common/common.cpp index 8c1a60583..e0082a823 100644 --- a/common/common.cpp +++ b/common/common.cpp @@ -46,6 +46,10 @@ #define GGML_USE_CUBLAS_SYCL #endif +#if (defined(GGML_USE_CUBLAS) || defined(GGML_USE_SYCL)) || defined(GGML_USE_VULKAN) +#define GGML_USE_CUBLAS_SYCL_VULKAN +#endif + int32_t get_num_physical_cores() { #ifdef __linux__ // enumerate the set of thread siblings, num entries is num cores @@ -660,8 +664,8 @@ bool gpt_params_parse_ex(int argc, char ** argv, gpt_params & params) { params.tensor_split[i] = 0.0f; } } -#ifndef GGML_USE_CUBLAS_SYCL - fprintf(stderr, "warning: llama.cpp was compiled without cuBLAS/SYCL. Setting a tensor split has no effect.\n"); +#ifndef GGML_USE_CUBLAS_SYCL_VULKAN + fprintf(stderr, "warning: llama.cpp was compiled without cuBLAS/SYCL/Vulkan. Setting a tensor split has no effect.\n"); #endif // GGML_USE_CUBLAS_SYCL } else if (arg == "--no-mmap") { params.use_mmap = false; diff --git a/ggml-vulkan.cpp b/ggml-vulkan.cpp index 14fb89e09..9e2846ee4 100644 --- a/ggml-vulkan.cpp +++ b/ggml-vulkan.cpp @@ -15,6 +15,7 @@ #include #include #include +#include #include "ggml.h" #include "ggml-backend-impl.h" @@ -37,6 +38,8 @@ #define GGML_VK_MAX_NODES 8192 +#define MAX_VK_BUFFERS 256 + #ifndef K_QUANTS_PER_ITERATION #define K_QUANTS_PER_ITERATION 1 #else @@ -53,15 +56,68 @@ static_assert(K_QUANTS_PER_ITERATION == 1 || K_QUANTS_PER_ITERATION == 2, "K_QUA } \ } while (0) -struct vk_buffer { +struct ggml_backend_vk_context; + +struct vk_queue { + uint32_t queue_family_index; + vk::Queue queue; + vk::CommandPool pool; + uint32_t cmd_buffer_idx; + std::vector cmd_buffers; + + vk::PipelineStageFlags stage_flags; +}; + +struct vk_device { + vk::PhysicalDevice physical_device; + vk::PhysicalDeviceProperties properties; + std::string name; + uint64_t max_memory_allocation_size; + bool fp16; + vk::Device device; + uint32_t vendor_id; + vk_queue compute_queue; + vk_queue transfer_queue; + bool single_queue; + uint32_t descriptor_set_mode; + uint32_t subgroup_size; + bool uma; + + ~vk_device() { +#ifdef GGML_VULKAN_DEBUG + std::cerr << "destroy device " << name << std::endl; +#endif + device.destroy(); + } +}; + +struct vk_buffer_struct { vk::Buffer buffer; vk::DeviceMemory device_memory; vk::MemoryPropertyFlags memory_property_flags; void * ptr; size_t size = 0; - uint32_t qf_owner; + + ggml_backend_vk_context * ctx; + + std::shared_ptr device; + + ~vk_buffer_struct() { + if (size == 0) { + return; + } +#ifdef GGML_VULKAN_DEBUG + std::cerr << "~vk_buffer_struct(" << buffer << ", " << size << ")" << std::endl; +#endif + + device->device.freeMemory(device_memory); + device->device.destroyBuffer(buffer); + } }; +typedef std::shared_ptr vk_buffer; +typedef std::weak_ptr vk_buffer_ref; + struct vk_subbuffer { vk_buffer buffer; uint64_t offset; @@ -70,6 +126,7 @@ struct vk_subbuffer { struct vk_pipeline { std::string name; + vk::ShaderModule shader_module; vk::DescriptorSetLayout dsl; std::vector descriptor_pools; std::vector descriptor_sets; @@ -82,16 +139,6 @@ struct vk_pipeline { uint32_t align; }; -struct vk_queue { - uint32_t queue_family_index; - vk::Queue queue; - vk::CommandPool pool; - uint32_t cmd_buffer_idx; - std::vector cmd_buffers; - - vk::PipelineStageFlags stage_flags; -}; - struct vk_semaphore { vk::Semaphore s; uint64_t value; @@ -105,20 +152,6 @@ struct vk_submission { typedef std::vector vk_sequence; -struct vk_device { - vk::PhysicalDevice physical_device; - vk::PhysicalDeviceProperties properties; - uint64_t max_memory_allocation_size; - bool fp16; - vk::Device device; - uint32_t vendor_id; - vk_queue compute_queue; - vk_queue transfer_queue; - uint32_t descriptor_set_mode; - uint32_t subgroup_size; - bool uma; -}; - struct vk_op_push_constants { uint32_t KX; uint32_t KY; @@ -190,13 +223,13 @@ struct ggml_tensor_extra_gpu { size_t ctx_idx; - vk_buffer buffer_gpu; + vk_buffer_ref buffer_gpu; uint64_t offset; void reset() { ready = false; ctx_idx = 0; - buffer_gpu.size = 0; + buffer_gpu.reset(); offset = 0; } }; @@ -210,69 +243,96 @@ struct ggml_vk_garbage_collector { std::vector contexts; }; -typedef void (*ggml_vk_func_t)(vk_context * ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst); +struct ggml_backend_vk_context { + std::string name; -vk::Instance vk_instance; -vk_device vk_device; -vk_pipeline vk_pipeline_matmul_f32_l, vk_pipeline_matmul_f32_m, vk_pipeline_matmul_f32_s; -vk_pipeline vk_pipeline_matmul_f32_aligned_l, vk_pipeline_matmul_f32_aligned_m, vk_pipeline_matmul_f32_aligned_s; -vk_pipeline vk_pipeline_matmul_f16_l, vk_pipeline_matmul_f16_m, vk_pipeline_matmul_f16_s; -vk_pipeline vk_pipeline_matmul_f16_aligned_l, vk_pipeline_matmul_f16_aligned_m, vk_pipeline_matmul_f16_aligned_s; -vk_pipeline vk_pipeline_matmul_f16_f32_l, vk_pipeline_matmul_f16_f32_m, vk_pipeline_matmul_f16_f32_s; -vk_pipeline vk_pipeline_matmul_f16_f32_aligned_l, vk_pipeline_matmul_f16_f32_aligned_m, vk_pipeline_matmul_f16_f32_aligned_s; -vk_pipeline vk_pipeline_matmul_split_k_reduce; -vk_pipeline vk_pipeline_dequant[VK_NUM_TYPES]; -vk_pipeline vk_pipeline_dequant_mul_mat_vec_f32[VK_NUM_TYPES]; -vk_pipeline vk_pipeline_mul_mat_vec_p021_f16_f32; -vk_pipeline vk_pipeline_mul_mat_vec_nc_f16_f32; -vk_pipeline vk_pipeline_get_rows[VK_NUM_TYPES]; -vk_pipeline vk_pipeline_get_rows_f32[VK_NUM_TYPES]; -vk_pipeline vk_pipeline_mul_f32; -vk_pipeline vk_pipeline_add_f32; -vk_pipeline vk_pipeline_scale_f32; -vk_pipeline vk_pipeline_sqr_f32; -vk_pipeline vk_pipeline_clamp_f32; -vk_pipeline vk_pipeline_cpy_f32_f32, vk_pipeline_cpy_f32_f16, vk_pipeline_cpy_f16_f16; -vk_pipeline vk_pipeline_norm_f32; -vk_pipeline vk_pipeline_rms_norm_f32; -vk_pipeline vk_pipeline_gelu_f32; -vk_pipeline vk_pipeline_silu_f32; -vk_pipeline vk_pipeline_relu_f32; -vk_pipeline vk_pipeline_diag_mask_inf_f32; -vk_pipeline vk_pipeline_soft_max_f32; -vk_pipeline vk_pipeline_rope_f32, vk_pipeline_rope_f16; -vk_pipeline vk_pipeline_rope_neox_f32, vk_pipeline_rope_neox_f16; + std::weak_ptr device; + vk_pipeline pipeline_matmul_f32_l, pipeline_matmul_f32_m, pipeline_matmul_f32_s; + vk_pipeline pipeline_matmul_f32_aligned_l, pipeline_matmul_f32_aligned_m, pipeline_matmul_f32_aligned_s; + vk_pipeline pipeline_matmul_f16_l, pipeline_matmul_f16_m, pipeline_matmul_f16_s; + vk_pipeline pipeline_matmul_f16_aligned_l, pipeline_matmul_f16_aligned_m, pipeline_matmul_f16_aligned_s; + vk_pipeline pipeline_matmul_f16_f32_l, pipeline_matmul_f16_f32_m, pipeline_matmul_f16_f32_s; + vk_pipeline pipeline_matmul_f16_f32_aligned_l, pipeline_matmul_f16_f32_aligned_m, pipeline_matmul_f16_f32_aligned_s; + vk_pipeline pipeline_matmul_split_k_reduce; + vk_pipeline pipeline_dequant[VK_NUM_TYPES]; + vk_pipeline pipeline_dequant_mul_mat_vec_f32[VK_NUM_TYPES]; + vk_pipeline pipeline_mul_mat_vec_p021_f16_f32; + vk_pipeline pipeline_mul_mat_vec_nc_f16_f32; + vk_pipeline pipeline_get_rows[VK_NUM_TYPES]; + vk_pipeline pipeline_get_rows_f32[VK_NUM_TYPES]; + vk_pipeline pipeline_mul_f32; + vk_pipeline pipeline_add_f32; + vk_pipeline pipeline_scale_f32; + vk_pipeline pipeline_sqr_f32; + vk_pipeline pipeline_clamp_f32; + vk_pipeline pipeline_cpy_f32_f32, pipeline_cpy_f32_f16, pipeline_cpy_f16_f16; + vk_pipeline pipeline_norm_f32; + vk_pipeline pipeline_rms_norm_f32; + vk_pipeline pipeline_gelu_f32; + vk_pipeline pipeline_silu_f32; + vk_pipeline pipeline_relu_f32; + vk_pipeline pipeline_diag_mask_inf_f32; + vk_pipeline pipeline_soft_max_f32; + vk_pipeline pipeline_rope_f32, pipeline_rope_f16; + vk_pipeline pipeline_rope_neox_f32, pipeline_rope_neox_f16; -static size_t vk_semaphore_idx, vk_event_idx; -static ggml_vk_garbage_collector vk_gc; -static std::vector> vk_pinned_memory; -static size_t vk_prealloc_size_qx, vk_prealloc_size_qy, vk_prealloc_size_x, vk_prealloc_size_y, vk_prealloc_size_split_k; -static vk_buffer vk_prealloc_qx, vk_prealloc_qy, vk_prealloc_x, vk_prealloc_y, vk_prealloc_split_k; -static vk::Fence vk_fence; -static vk_buffer vk_staging; -static size_t vk_staging_size; -static size_t vk_staging_offset; -static vk_buffer vk_sync_staging; + size_t semaphore_idx, event_idx; + ggml_vk_garbage_collector gc; + std::vector> pinned_memory; + size_t prealloc_size_qx, prealloc_size_qy, prealloc_size_x, prealloc_size_y, prealloc_size_split_k; + vk_buffer prealloc_qx, prealloc_qy, prealloc_x, prealloc_y, prealloc_split_k; + vk::Fence fence; + vk_buffer staging; + size_t staging_size; + size_t staging_offset; + vk_buffer sync_staging; -static vk_context * vk_ctx; -static vk_context * vk_transfer_ctx; + vk_buffer buffer_pool[MAX_VK_BUFFERS]; -static bool vk_disable; + vk_context * compute_ctx; + vk_context * transfer_ctx; + + bool disable; + bool initialized; + + size_t idx; +}; + +struct vk_instance { + vk::Instance instance; + + std::vector device_indices; + + std::shared_ptr devices[GGML_VK_MAX_DEVICES]; + ggml_backend_t backends[GGML_VK_MAX_DEVICES]; + ggml_backend_vk_context contexts[GGML_VK_MAX_DEVICES]; + ggml_backend_buffer_type buffer_types[GGML_VK_MAX_DEVICES]; + bool initialized[GGML_VK_MAX_DEVICES]; +}; #ifdef GGML_VULKAN_CHECK_RESULTS -size_t vk_skip_checks; -size_t vk_output_tensor; +static size_t vk_skip_checks; +static size_t vk_output_tensor; + +static void ggml_vk_print_tensor(ggml_backend * ctx, const ggml_tensor * tensor, const char * name); +static void ggml_vk_check_results_0(ggml_backend_vk_context * ctx, ggml_compute_params * params, ggml_tensor * tensor); +static void ggml_vk_check_results_1(ggml_backend_vk_context * ctx, ggml_compute_params * params, ggml_tensor * tensor); #endif -static vk_pipeline ggml_vk_create_pipeline(const std::string& name, size_t spv_size, const void* spv_data, const std::string& entrypoint, uint32_t parameter_count, uint32_t push_constant_size, std::array wg_denoms, std::vector&& specialization_constants, uint32_t align) { +typedef void (*ggml_vk_func_t)(ggml_backend_vk_context * ctx, vk_context * subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst); + +static bool vk_instance_initialized = false; +static vk_instance vk_instance; + +GGML_CALL static void ggml_backend_vk_free(ggml_backend_t backend); + +static void ggml_vk_create_pipeline(ggml_backend_vk_context * ctx, vk_pipeline& pipeline, const std::string& name, size_t spv_size, const void* spv_data, const std::string& entrypoint, uint32_t parameter_count, uint32_t push_constant_size, std::array wg_denoms, std::vector&& specialization_constants, uint32_t align) { #ifdef GGML_VULKAN_DEBUG std::cerr << "ggml_vk_create_pipeline(" << name << ", " << entrypoint << ", " << parameter_count << ", " << push_constant_size << ", (" << wg_denoms[0] << "," << wg_denoms[1] << "," << wg_denoms[2] << "), specialization_constants, " << align << ")" << std::endl; #endif GGML_ASSERT(parameter_count > 0); GGML_ASSERT(wg_denoms[0] > 0 && wg_denoms[1] > 0 && wg_denoms[2] > 0); // NOLINT - vk_pipeline pipeline; - pipeline.name = name; pipeline.parameter_count = parameter_count; pipeline.push_constant_size = push_constant_size; @@ -280,7 +340,7 @@ static vk_pipeline ggml_vk_create_pipeline(const std::string& name, size_t spv_s pipeline.align = align; vk::ShaderModuleCreateInfo shader_module_create_info({}, spv_size, reinterpret_cast(spv_data)); - vk::ShaderModule shader_module = vk_device.device.createShaderModule(shader_module_create_info); + pipeline.shader_module = ctx->device.lock()->device.createShaderModule(shader_module_create_info); std::vector dsl_binding; std::vector dsl_binding_flags; @@ -301,17 +361,17 @@ static vk_pipeline ggml_vk_create_pipeline(const std::string& name, size_t spv_s {}, dsl_binding); descriptor_set_layout_create_info.setPNext(&dslbfci); - pipeline.dsl = vk_device.device.createDescriptorSetLayout(descriptor_set_layout_create_info); + pipeline.dsl = ctx->device.lock()->device.createDescriptorSetLayout(descriptor_set_layout_create_info); // Check if device supports multiple descriptors per pool - if (vk_device.descriptor_set_mode == VK_DEVICE_DESCRIPTOR_POOL_MODE_UNKNOWN) { + if (ctx->device.lock()->descriptor_set_mode == VK_DEVICE_DESCRIPTOR_POOL_MODE_UNKNOWN) { const uint32_t alloc_count = 2; // Try allocating multiple sets from one pool // This fails on AMD for some reason, so add a fall back to allocating one pool per set vk::DescriptorPoolSize descriptor_pool_size(vk::DescriptorType::eStorageBuffer, pipeline.parameter_count); vk::DescriptorPoolCreateInfo descriptor_pool_create_info({}, alloc_count, descriptor_pool_size); - vk::DescriptorPool pool = vk_device.device.createDescriptorPool(descriptor_pool_create_info); + vk::DescriptorPool pool = ctx->device.lock()->device.createDescriptorPool(descriptor_pool_create_info); std::vector layouts(alloc_count); for (uint32_t i = 0; i < alloc_count; i++) { @@ -319,24 +379,24 @@ static vk_pipeline ggml_vk_create_pipeline(const std::string& name, size_t spv_s } try { vk::DescriptorSetAllocateInfo descriptor_set_alloc_info(pool, alloc_count, layouts.data()); - std::vector sets = vk_device.device.allocateDescriptorSets(descriptor_set_alloc_info); + std::vector sets = ctx->device.lock()->device.allocateDescriptorSets(descriptor_set_alloc_info); } catch(vk::OutOfPoolMemoryError const&) { - vk_device.descriptor_set_mode = VK_DEVICE_DESCRIPTOR_POOL_MODE_SINGLE; + ctx->device.lock()->descriptor_set_mode = VK_DEVICE_DESCRIPTOR_POOL_MODE_SINGLE; } - vk_device.device.destroyDescriptorPool(pool); + ctx->device.lock()->device.destroyDescriptorPool(pool); } - if (vk_device.descriptor_set_mode == VK_DEVICE_DESCRIPTOR_POOL_MODE_MULTI) { + if (ctx->device.lock()->descriptor_set_mode == VK_DEVICE_DESCRIPTOR_POOL_MODE_MULTI) { vk::DescriptorPoolSize descriptor_pool_size(vk::DescriptorType::eStorageBuffer, pipeline.parameter_count); vk::DescriptorPoolCreateInfo descriptor_pool_create_info({}, 128, descriptor_pool_size); - pipeline.descriptor_pools.push_back(vk_device.device.createDescriptorPool(descriptor_pool_create_info)); + pipeline.descriptor_pools.push_back(ctx->device.lock()->device.createDescriptorPool(descriptor_pool_create_info)); } pipeline.descriptor_set_idx = 0; vk::PipelineLayoutCreateInfo pipeline_layout_create_info(vk::PipelineLayoutCreateFlags(), pipeline.dsl, pcr); - pipeline.layout = vk_device.device.createPipelineLayout(pipeline_layout_create_info); + pipeline.layout = ctx->device.lock()->device.createPipelineLayout(pipeline_layout_create_info); std::vector specialization_entries(specialization_constants.size()); @@ -356,41 +416,45 @@ static vk_pipeline ggml_vk_create_pipeline(const std::string& name, size_t spv_s vk::PipelineShaderStageCreateInfo pipeline_shader_create_info( vk::PipelineShaderStageCreateFlags(), vk::ShaderStageFlagBits::eCompute, - shader_module, + pipeline.shader_module, entrypoint.c_str(), &specialization_info); vk::ComputePipelineCreateInfo compute_pipeline_create_info( vk::PipelineCreateFlags(), pipeline_shader_create_info, pipeline.layout); - pipeline.pipeline = vk_device.device.createComputePipeline(VK_NULL_HANDLE, compute_pipeline_create_info).value; + pipeline.pipeline = ctx->device.lock()->device.createComputePipeline(VK_NULL_HANDLE, compute_pipeline_create_info).value; - return pipeline; + ctx->gc.pipelines.push_back(&pipeline); } -static void ggml_vk_pipeline_allocate_descriptor_sets(vk_pipeline& pipeline, uint32_t n) { +static void ggml_vk_destroy_pipeline(ggml_backend_vk_context * ctx, vk_pipeline * pipeline) { + for (auto& pool : pipeline->descriptor_pools) { + ctx->device.lock()->device.destroyDescriptorPool(pool); + } + pipeline->descriptor_pools.clear(); + pipeline->descriptor_sets.clear(); + pipeline->descriptor_set_idx = 0; + + ctx->device.lock()->device.destroyDescriptorSetLayout(pipeline->dsl); + + ctx->device.lock()->device.destroyPipelineLayout(pipeline->layout); + + ctx->device.lock()->device.destroyShaderModule(pipeline->shader_module); + + ctx->device.lock()->device.destroyPipeline(pipeline->pipeline); +} + +static void ggml_pipeline_allocate_descriptor_sets(ggml_backend_vk_context * ctx, vk_pipeline& pipeline, uint32_t n) { #ifdef GGML_VULKAN_DEBUG - std::cerr << "ggml_vk_pipeline_allocate_descriptor_sets(" << pipeline.name << ", " << n << ")" << std::endl; + std::cerr << "ggml_pipeline_allocate_descriptor_sets(" << pipeline.name << ", " << n << ")" << std::endl; #endif - // Check if gc already contains pipeline before adding it - bool gc_found = false; - for (auto * pl : vk_gc.pipelines) { - if (&pipeline == pl) { - gc_found = true; - break; - } - } - - if (!gc_found) { - vk_gc.pipelines.push_back(&pipeline); - } - if (pipeline.descriptor_sets.size() >= pipeline.descriptor_set_idx + n) { // Enough descriptors are available return; } - if (vk_device.descriptor_set_mode == VK_DEVICE_DESCRIPTOR_POOL_MODE_MULTI) { + if (ctx->device.lock()->descriptor_set_mode == VK_DEVICE_DESCRIPTOR_POOL_MODE_MULTI) { const uint32_t alloc_count = pipeline.descriptor_set_idx + n - pipeline.descriptor_sets.size(); std::vector layouts(alloc_count); @@ -398,29 +462,29 @@ static void ggml_vk_pipeline_allocate_descriptor_sets(vk_pipeline& pipeline, uin layouts[i] = pipeline.dsl; } vk::DescriptorSetAllocateInfo descriptor_set_alloc_info(pipeline.descriptor_pools[0], alloc_count, layouts.data()); - std::vector sets = vk_device.device.allocateDescriptorSets(descriptor_set_alloc_info); + std::vector sets = ctx->device.lock()->device.allocateDescriptorSets(descriptor_set_alloc_info); pipeline.descriptor_sets.insert(pipeline.descriptor_sets.end(), sets.begin(), sets.end()); } else { for (uint32_t i = pipeline.descriptor_sets.size(); i < pipeline.descriptor_set_idx + n; i++) { vk::DescriptorPoolSize descriptor_pool_size(vk::DescriptorType::eStorageBuffer, pipeline.parameter_count); vk::DescriptorPoolCreateInfo descriptor_pool_create_info({}, 1, descriptor_pool_size); - pipeline.descriptor_pools.push_back(vk_device.device.createDescriptorPool(descriptor_pool_create_info)); + pipeline.descriptor_pools.push_back(ctx->device.lock()->device.createDescriptorPool(descriptor_pool_create_info)); vk::DescriptorSetAllocateInfo descriptor_set_alloc_info(pipeline.descriptor_pools[i], 1, &pipeline.dsl); - std::vector sets = vk_device.device.allocateDescriptorSets(descriptor_set_alloc_info); + std::vector sets = ctx->device.lock()->device.allocateDescriptorSets(descriptor_set_alloc_info); pipeline.descriptor_sets.push_back(sets[0]); } } } -static void ggml_vk_pipeline_cleanup(vk_pipeline& pipeline) { +static void ggml_pipeline_cleanup(vk_pipeline& pipeline) { #ifdef GGML_VULKAN_DEBUG - std::cerr << "ggml_vk_pipeline_cleanup(" << pipeline.name << ")" << std::endl; + std::cerr << "ggml_pipeline_cleanup(" << pipeline.name << ")" << std::endl; #endif pipeline.descriptor_set_idx = 0; } -static vk::CommandBuffer ggml_vk_create_cmd_buffer(vk_queue& q) { +static vk::CommandBuffer ggml_vk_create_cmd_buffer(ggml_backend_vk_context * ctx, vk_queue& q) { #ifdef GGML_VULKAN_DEBUG std::cerr << "ggml_vk_create_cmd_buffer()" << std::endl; #endif @@ -433,7 +497,7 @@ static vk::CommandBuffer ggml_vk_create_cmd_buffer(vk_queue& q) { q.pool, vk::CommandBufferLevel::ePrimary, 1); - const std::vector cmd_buffers = vk_device.device.allocateCommandBuffers(command_buffer_alloc_info); + const std::vector cmd_buffers = ctx->device.lock()->device.allocateCommandBuffers(command_buffer_alloc_info); auto buf = cmd_buffers.front(); q.cmd_buffers.push_back(buf); @@ -442,24 +506,17 @@ static vk::CommandBuffer ggml_vk_create_cmd_buffer(vk_queue& q) { return buf; } -static vk_submission ggml_vk_create_submission(vk_queue& q, std::vector wait_semaphores, std::vector signal_semaphores) { +static vk_submission ggml_vk_create_submission(ggml_backend_vk_context * ctx, vk_queue& q, std::vector wait_semaphores, std::vector signal_semaphores) { #ifdef GGML_VULKAN_DEBUG std::cerr << "ggml_vk_create_submission()" << std::endl; #endif vk_submission s; - s.buffer = ggml_vk_create_cmd_buffer(q); + s.buffer = ggml_vk_create_cmd_buffer(ctx, q); s.wait_semaphores = std::move(wait_semaphores); s.signal_semaphores = std::move(signal_semaphores); return s; } -static vk_sequence ggml_vk_create_sequence_1(vk_queue& q, std::vector wait_semaphores, std::vector signal_semaphores) { -#ifdef GGML_VULKAN_DEBUG - std::cerr << "ggml_vk_create_sequence_1()" << std::endl; -#endif - return { ggml_vk_create_submission(q, std::move(wait_semaphores), std::move(signal_semaphores)) }; -} - static void ggml_vk_submit(vk_context * ctx, vk::Fence fence) { #ifdef GGML_VULKAN_DEBUG std::cerr << "ggml_vk_submit(" << ctx->seqs.size() << ", " << fence << ")" << std::endl; @@ -578,89 +635,89 @@ static uint32_t ggml_vk_find_queue_family_index(std::vectordevice.lock()->device.createCommandPool(command_pool_create_info_compute); q.cmd_buffer_idx = 0; - q.queue = vk_device.device.getQueue(queue_family_index, queue_index); + q.queue = ctx->device.lock()->device.getQueue(queue_family_index, queue_index); q.stage_flags = stage_flags; - - return q; } -static vk_context * ggml_vk_create_context(vk_queue& q) { +static vk_context * ggml_vk_create_context(ggml_backend_vk_context * ctx, vk_queue& q) { #ifdef GGML_VULKAN_DEBUG std::cerr << "ggml_vk_create_context()" << std::endl; #endif - vk_gc.contexts.emplace_back(); - vk_context * result = &vk_gc.contexts[vk_gc.contexts.size() - 1]; + ctx->gc.contexts.emplace_back(); + vk_context * result = &ctx->gc.contexts[ctx->gc.contexts.size() - 1]; memset((void *) result, 0, sizeof(vk_context)); - result->idx = vk_gc.contexts.size() - 1; + result->idx = ctx->gc.contexts.size() - 1; result->q = &q; return result; } -static vk_semaphore * ggml_vk_create_binary_semaphore() { +static vk_semaphore * ggml_vk_create_binary_semaphore(ggml_backend_vk_context * ctx) { #ifdef GGML_VULKAN_DEBUG std::cerr << "ggml_vk_create_timeline_semaphore()" << std::endl; #endif vk::SemaphoreTypeCreateInfo tci{ vk::SemaphoreType::eBinary, 0 }; vk::SemaphoreCreateInfo ci{}; ci.setPNext(&tci); - vk::Semaphore semaphore = vk_device.device.createSemaphore(ci); - vk_gc.semaphores.push_back({ semaphore, 0 }); - return &vk_gc.semaphores[vk_gc.semaphores.size() - 1]; + vk::Semaphore semaphore = ctx->device.lock()->device.createSemaphore(ci); + ctx->gc.semaphores.push_back({ semaphore, 0 }); + return &ctx->gc.semaphores[ctx->gc.semaphores.size() - 1]; } -static vk_semaphore * ggml_vk_create_timeline_semaphore() { +static vk_semaphore * ggml_vk_create_timeline_semaphore(ggml_backend_vk_context * ctx) { #ifdef GGML_VULKAN_DEBUG std::cerr << "ggml_vk_create_timeline_semaphore()" << std::endl; #endif - if (vk_semaphore_idx >= vk_gc.tl_semaphores.size()) { + if (ctx->semaphore_idx >= ctx->gc.tl_semaphores.size()) { vk::SemaphoreTypeCreateInfo tci{ vk::SemaphoreType::eTimeline, 0 }; vk::SemaphoreCreateInfo ci{}; ci.setPNext(&tci); - vk::Semaphore semaphore = vk_device.device.createSemaphore(ci); - vk_gc.tl_semaphores.push_back({ semaphore, 0 }); + vk::Semaphore semaphore = ctx->device.lock()->device.createSemaphore(ci); + ctx->gc.tl_semaphores.push_back({ semaphore, 0 }); } - return &vk_gc.tl_semaphores[vk_semaphore_idx++]; + return &ctx->gc.tl_semaphores[ctx->semaphore_idx++]; } -static vk::Event ggml_vk_create_event() { - if (vk_event_idx >= vk_gc.events.size()) { - vk_gc.events.push_back(vk_device.device.createEvent({})); +static vk::Event ggml_vk_create_event(ggml_backend_vk_context * ctx) { + if (ctx->event_idx >= ctx->gc.events.size()) { + ctx->gc.events.push_back(ctx->device.lock()->device.createEvent({})); } - return vk_gc.events[vk_event_idx++]; + return ctx->gc.events[ctx->event_idx++]; } -static void ggml_vk_queue_cleanup(vk_queue& q) { +static void ggml_vk_queue_cleanup(ggml_backend_vk_context * ctx, vk_queue& q) { #ifdef GGML_VULKAN_DEBUG std::cerr << "ggml_vk_queue_cleanup()" << std::endl; #endif // Requires command buffers to be done - vk_device.device.resetCommandPool(q.pool); + ctx->device.lock()->device.resetCommandPool(q.pool); q.cmd_buffer_idx = 0; } -static vk_buffer ggml_vk_create_buffer(size_t size, vk::MemoryPropertyFlags req_flags) { +static vk_buffer ggml_vk_create_buffer(ggml_backend_vk_context * ctx, size_t size, vk::MemoryPropertyFlags req_flags) { #ifdef GGML_VULKAN_DEBUG std::cerr << "ggml_vk_create_buffer(" << size << ", " << to_string(req_flags) << ")" << std::endl; #endif - GGML_ASSERT(size > 0); + vk_buffer buf = std::make_shared(); - vk_buffer buf; + if (size == 0) { + buf->size = 0; + return buf; + } - buf.size = size; + buf->size = size; vk::BufferCreateInfo buffer_create_info{ vk::BufferCreateFlags(), size, @@ -670,11 +727,11 @@ static vk_buffer ggml_vk_create_buffer(size_t size, vk::MemoryPropertyFlags req_ nullptr, }; - buf.buffer = vk_device.device.createBuffer(buffer_create_info); + buf->buffer = ctx->device.lock()->device.createBuffer(buffer_create_info); - vk::MemoryRequirements mem_req = vk_device.device.getBufferMemoryRequirements(buf.buffer); + vk::MemoryRequirements mem_req = ctx->device.lock()->device.getBufferMemoryRequirements(buf->buffer); - vk::PhysicalDeviceMemoryProperties mem_props = vk_device.physical_device.getMemoryProperties(); + vk::PhysicalDeviceMemoryProperties mem_props = ctx->device.lock()->physical_device.getMemoryProperties(); uint32_t memory_type_index = UINT32_MAX; @@ -691,30 +748,36 @@ static vk_buffer ggml_vk_create_buffer(size_t size, vk::MemoryPropertyFlags req_ } try { - buf.device_memory = vk_device.device.allocateMemory({ mem_req.size, memory_type_index }); + buf->device_memory = ctx->device.lock()->device.allocateMemory({ mem_req.size, memory_type_index }); } catch (const vk::SystemError& e) { // Out of Host/Device memory, clean up buffer - vk_device.device.destroyBuffer(buf.buffer); - buf.size = 0; + ctx->device.lock()->device.destroyBuffer(buf->buffer); + buf->size = 0; throw e; } - buf.memory_property_flags = req_flags; - buf.ptr = nullptr; + buf->memory_property_flags = req_flags; + buf->ptr = nullptr; if (req_flags & vk::MemoryPropertyFlagBits::eHostVisible) { - buf.ptr = vk_device.device.mapMemory(buf.device_memory, 0, VK_WHOLE_SIZE); + buf->ptr = ctx->device.lock()->device.mapMemory(buf->device_memory, 0, VK_WHOLE_SIZE); } - vk_device.device.bindBufferMemory(buf.buffer, buf.device_memory, 0); + ctx->device.lock()->device.bindBufferMemory(buf->buffer, buf->device_memory, 0); - buf.qf_owner = VK_QUEUE_FAMILY_IGNORED; + buf->ctx = ctx; + + buf->device = ctx->device.lock(); + +#ifdef GGML_VULKAN_DEBUG + std::cerr << "Created buffer " << buf->buffer << std::endl; +#endif return buf; } -static vk_buffer ggml_vk_create_buffer_check(size_t size, vk::MemoryPropertyFlags req_flags) { +static vk_buffer ggml_vk_create_buffer_check(ggml_backend_vk_context * ctx, size_t size, vk::MemoryPropertyFlags req_flags) { try { - return ggml_vk_create_buffer(size, req_flags); + return ggml_vk_create_buffer(ctx, size, req_flags); } catch (const vk::SystemError& e) { std::cerr << "ggml_vulkan: Memory allocation of size " << size << " failed." << std::endl; std::cerr << "ggml_vulkan: " << e.what() << std::endl; @@ -722,14 +785,14 @@ static vk_buffer ggml_vk_create_buffer_check(size_t size, vk::MemoryPropertyFlag } } -static vk_buffer ggml_vk_create_buffer_device(size_t size) { +static vk_buffer ggml_vk_create_buffer_device(ggml_backend_vk_context * ctx, size_t size) { vk_buffer buf; try { - buf = ggml_vk_create_buffer(size, vk::MemoryPropertyFlagBits::eDeviceLocal); + buf = ggml_vk_create_buffer(ctx, size, vk::MemoryPropertyFlagBits::eDeviceLocal); } catch (const vk::SystemError& e) { - if (vk_device.uma) { + if (ctx->device.lock()->uma) { // Fall back to host memory type - buf = ggml_vk_create_buffer_check(size, vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent); + buf = ggml_vk_create_buffer_check(ctx, size, vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent); } else { std::cerr << "ggml_vulkan: Device memory allocation of size " << size << " failed." << std::endl; std::cerr << "ggml_vulkan: " << e.what() << std::endl; @@ -741,16 +804,7 @@ static vk_buffer ggml_vk_create_buffer_device(size_t size) { } static void ggml_vk_destroy_buffer(vk_buffer& buf) { - if (buf.size == 0) { - return; - } -#ifdef GGML_VULKAN_DEBUG - std::cerr << "ggml_vk_destroy_buffer(" << buf.size << ")" << std::endl; -#endif - - buf.size = 0; - vk_device.device.freeMemory(buf.device_memory); - vk_device.device.destroyBuffer(buf.buffer); + buf.reset(); } static vk_subbuffer ggml_vk_subbuffer(vk_buffer& buf) { @@ -773,7 +827,7 @@ static void ggml_vk_sync_buffers(vk_context * ctx) { ); } -static void ggml_vk_wait_events(vk::CommandBuffer& cmd_buffer, std::vector&& events, vk::PipelineStageFlags src_stages, vk::PipelineStageFlags dst_stages) { +static void ggml_vk_wait_events(vk_context * ctx, std::vector&& events) { #ifdef GGML_VULKAN_DEBUG std::cerr << "ggml_vk_wait_events()" << std::endl; #endif @@ -781,10 +835,10 @@ static void ggml_vk_wait_events(vk::CommandBuffer& cmd_buffer, std::vectors->buffer.waitEvents( events, - src_stages, - dst_stages, + ctx->q->stage_flags, + ctx->q->stage_flags, {}, {}, {} @@ -810,15 +864,15 @@ static bool ggml_vk_build_shader(ggml_type type) { } } -static void ggml_vk_load_shaders() { +static void ggml_vk_load_shaders(ggml_backend_vk_context * ctx) { #ifdef GGML_VULKAN_DEBUG - std::cerr << "ggml_vk_load_shaders()" << std::endl; + std::cerr << "ggml_vk_load_shaders(" << ctx->name << ")" << std::endl; #endif // mulmat - std::initializer_list warptile_l = { 128, 128, 128, 16, vk_device.subgroup_size * 2, 64, 2, 4, 4, vk_device.subgroup_size }; - std::initializer_list warptile_m = { 128, 64, 64, 16, vk_device.subgroup_size, 32, 2, 4, 2, vk_device.subgroup_size }; - std::initializer_list warptile_s = { vk_device.subgroup_size, 32, 32, 16, 32, 32, 2, 2, 2, vk_device.subgroup_size }; + std::initializer_list warptile_l = { 128, 128, 128, 16, ctx->device.lock()->subgroup_size * 2, 64, 2, 4, 4, ctx->device.lock()->subgroup_size }; + std::initializer_list warptile_m = { 128, 64, 64, 16, ctx->device.lock()->subgroup_size, 32, 2, 4, 2, ctx->device.lock()->subgroup_size }; + std::initializer_list warptile_s = { ctx->device.lock()->subgroup_size, 32, 32, 16, 32, 32, 2, 2, 2, ctx->device.lock()->subgroup_size }; std::array l_wg_denoms = {128, 128, 1 }; std::array m_wg_denoms = { 64, 64, 1 }; @@ -828,145 +882,208 @@ static void ggml_vk_load_shaders() { uint32_t m_align = 64; uint32_t s_align = 32; - if (vk_device.fp16) { - vk_pipeline_matmul_f32_l = ggml_vk_create_pipeline("matmul_f32_l", matmul_f32_l_len, matmul_f32_l_data, "main", 3, 14 * sizeof(uint32_t), l_wg_denoms, warptile_l, 1); - vk_pipeline_matmul_f32_m = ggml_vk_create_pipeline("matmul_f32_m", matmul_f32_m_len, matmul_f32_m_data, "main", 3, 14 * sizeof(uint32_t), m_wg_denoms, warptile_m, 1); - vk_pipeline_matmul_f32_s = ggml_vk_create_pipeline("matmul_f32_s", matmul_f32_s_len, matmul_f32_s_data, "main", 3, 14 * sizeof(uint32_t), s_wg_denoms, warptile_s, 1); - vk_pipeline_matmul_f32_aligned_l = ggml_vk_create_pipeline("matmul_f32_aligned_l", matmul_f32_aligned_l_len, matmul_f32_aligned_l_data, "main", 3, 14 * sizeof(uint32_t), l_wg_denoms, warptile_l, l_align); - vk_pipeline_matmul_f32_aligned_m = ggml_vk_create_pipeline("matmul_f32_aligned_m", matmul_f32_aligned_m_len, matmul_f32_aligned_m_data, "main", 3, 14 * sizeof(uint32_t), m_wg_denoms, warptile_m, m_align); - vk_pipeline_matmul_f32_aligned_s = ggml_vk_create_pipeline("matmul_f32_aligned_s", matmul_f32_aligned_s_len, matmul_f32_aligned_s_data, "main", 3, 14 * sizeof(uint32_t), s_wg_denoms, warptile_s, s_align); + if (ctx->device.lock()->fp16) { + ggml_vk_create_pipeline(ctx, ctx->pipeline_matmul_f32_l, "matmul_f32_l", matmul_f32_l_len, matmul_f32_l_data, "main", 3, 14 * sizeof(uint32_t), l_wg_denoms, warptile_l, 1); + ggml_vk_create_pipeline(ctx, ctx->pipeline_matmul_f32_m, "matmul_f32_m", matmul_f32_m_len, matmul_f32_m_data, "main", 3, 14 * sizeof(uint32_t), m_wg_denoms, warptile_m, 1); + ggml_vk_create_pipeline(ctx, ctx->pipeline_matmul_f32_s, "matmul_f32_s", matmul_f32_s_len, matmul_f32_s_data, "main", 3, 14 * sizeof(uint32_t), s_wg_denoms, warptile_s, 1); + ggml_vk_create_pipeline(ctx, ctx->pipeline_matmul_f32_aligned_l, "matmul_f32_aligned_l", matmul_f32_aligned_l_len, matmul_f32_aligned_l_data, "main", 3, 14 * sizeof(uint32_t), l_wg_denoms, warptile_l, l_align); + ggml_vk_create_pipeline(ctx, ctx->pipeline_matmul_f32_aligned_m, "matmul_f32_aligned_m", matmul_f32_aligned_m_len, matmul_f32_aligned_m_data, "main", 3, 14 * sizeof(uint32_t), m_wg_denoms, warptile_m, m_align); + ggml_vk_create_pipeline(ctx, ctx->pipeline_matmul_f32_aligned_s, "matmul_f32_aligned_s", matmul_f32_aligned_s_len, matmul_f32_aligned_s_data, "main", 3, 14 * sizeof(uint32_t), s_wg_denoms, warptile_s, s_align); - vk_pipeline_matmul_f16_l = ggml_vk_create_pipeline("matmul_f16_l", matmul_f16_l_len, matmul_f16_l_data, "main", 3, 14 * sizeof(uint32_t), l_wg_denoms, warptile_l, 1); - vk_pipeline_matmul_f16_m = ggml_vk_create_pipeline("matmul_f16_m", matmul_f16_m_len, matmul_f16_m_data, "main", 3, 14 * sizeof(uint32_t), m_wg_denoms, warptile_m, 1); - vk_pipeline_matmul_f16_s = ggml_vk_create_pipeline("matmul_f16_s", matmul_f16_s_len, matmul_f16_s_data, "main", 3, 14 * sizeof(uint32_t), s_wg_denoms, warptile_s, 1); + ggml_vk_create_pipeline(ctx, ctx->pipeline_matmul_f16_l, "matmul_f16_l", matmul_f16_l_len, matmul_f16_l_data, "main", 3, 14 * sizeof(uint32_t), l_wg_denoms, warptile_l, 1); + ggml_vk_create_pipeline(ctx, ctx->pipeline_matmul_f16_m, "matmul_f16_m", matmul_f16_m_len, matmul_f16_m_data, "main", 3, 14 * sizeof(uint32_t), m_wg_denoms, warptile_m, 1); + ggml_vk_create_pipeline(ctx, ctx->pipeline_matmul_f16_s, "matmul_f16_s", matmul_f16_s_len, matmul_f16_s_data, "main", 3, 14 * sizeof(uint32_t), s_wg_denoms, warptile_s, 1); + ggml_vk_create_pipeline(ctx, ctx->pipeline_matmul_f16_aligned_l, "matmul_f16_aligned_l", matmul_f16_aligned_l_len, matmul_f16_aligned_l_data, "main", 3, 14 * sizeof(uint32_t), l_wg_denoms, warptile_l, l_align); + ggml_vk_create_pipeline(ctx, ctx->pipeline_matmul_f16_aligned_m, "matmul_f16_aligned_m", matmul_f16_aligned_m_len, matmul_f16_aligned_m_data, "main", 3, 14 * sizeof(uint32_t), m_wg_denoms, warptile_m, m_align); + ggml_vk_create_pipeline(ctx, ctx->pipeline_matmul_f16_aligned_s, "matmul_f16_aligned_s", matmul_f16_aligned_s_len, matmul_f16_aligned_s_data, "main", 3, 14 * sizeof(uint32_t), s_wg_denoms, warptile_s, s_align); - vk_pipeline_matmul_f16_aligned_l = ggml_vk_create_pipeline("matmul_f16_aligned_l", matmul_f16_aligned_l_len, matmul_f16_aligned_l_data, "main", 3, 14 * sizeof(uint32_t), l_wg_denoms, warptile_l, l_align); - vk_pipeline_matmul_f16_aligned_m = ggml_vk_create_pipeline("matmul_f16_aligned_m", matmul_f16_aligned_m_len, matmul_f16_aligned_m_data, "main", 3, 14 * sizeof(uint32_t), m_wg_denoms, warptile_m, m_align); - vk_pipeline_matmul_f16_aligned_s = ggml_vk_create_pipeline("matmul_f16_aligned_s", matmul_f16_aligned_s_len, matmul_f16_aligned_s_data, "main", 3, 14 * sizeof(uint32_t), s_wg_denoms, warptile_s, s_align); - - vk_pipeline_matmul_f16_f32_l = ggml_vk_create_pipeline("matmul_f16_f32_l", matmul_f16_f32_l_len, matmul_f16_f32_l_data, "main", 3, 14 * sizeof(uint32_t), l_wg_denoms, warptile_l, 1); - vk_pipeline_matmul_f16_f32_m = ggml_vk_create_pipeline("matmul_f16_f32_m", matmul_f16_f32_m_len, matmul_f16_f32_m_data, "main", 3, 14 * sizeof(uint32_t), m_wg_denoms, warptile_m, 1); - vk_pipeline_matmul_f16_f32_s = ggml_vk_create_pipeline("matmul_f16_f32_s", matmul_f16_f32_s_len, matmul_f16_f32_s_data, "main", 3, 14 * sizeof(uint32_t), s_wg_denoms, warptile_s, 1); - vk_pipeline_matmul_f16_f32_aligned_l = ggml_vk_create_pipeline("matmul_f16_f32_aligned_l", matmul_f16_f32_aligned_l_len, matmul_f16_f32_aligned_l_data, "main", 3, 14 * sizeof(uint32_t), l_wg_denoms, warptile_l, l_align); - vk_pipeline_matmul_f16_f32_aligned_m = ggml_vk_create_pipeline("matmul_f16_f32_aligned_m", matmul_f16_f32_aligned_m_len, matmul_f16_f32_aligned_m_data, "main", 3, 14 * sizeof(uint32_t), m_wg_denoms, warptile_m, m_align); - vk_pipeline_matmul_f16_f32_aligned_s = ggml_vk_create_pipeline("matmul_f16_f32_aligned_s", matmul_f16_f32_aligned_s_len, matmul_f16_f32_aligned_s_data, "main", 3, 14 * sizeof(uint32_t), s_wg_denoms, warptile_s, s_align); + ggml_vk_create_pipeline(ctx, ctx->pipeline_matmul_f16_f32_l, "matmul_f16_f32_l", matmul_f16_f32_l_len, matmul_f16_f32_l_data, "main", 3, 14 * sizeof(uint32_t), l_wg_denoms, warptile_l, 1); + ggml_vk_create_pipeline(ctx, ctx->pipeline_matmul_f16_f32_m, "matmul_f16_f32_m", matmul_f16_f32_m_len, matmul_f16_f32_m_data, "main", 3, 14 * sizeof(uint32_t), m_wg_denoms, warptile_m, 1); + ggml_vk_create_pipeline(ctx, ctx->pipeline_matmul_f16_f32_s, "matmul_f16_f32_s", matmul_f16_f32_s_len, matmul_f16_f32_s_data, "main", 3, 14 * sizeof(uint32_t), s_wg_denoms, warptile_s, 1); + ggml_vk_create_pipeline(ctx, ctx->pipeline_matmul_f16_f32_aligned_l, "matmul_f16_f32_aligned_l", matmul_f16_f32_aligned_l_len, matmul_f16_f32_aligned_l_data, "main", 3, 14 * sizeof(uint32_t), l_wg_denoms, warptile_l, l_align); + ggml_vk_create_pipeline(ctx, ctx->pipeline_matmul_f16_f32_aligned_m, "matmul_f16_f32_aligned_m", matmul_f16_f32_aligned_m_len, matmul_f16_f32_aligned_m_data, "main", 3, 14 * sizeof(uint32_t), m_wg_denoms, warptile_m, m_align); + ggml_vk_create_pipeline(ctx, ctx->pipeline_matmul_f16_f32_aligned_s, "matmul_f16_f32_aligned_s", matmul_f16_f32_aligned_s_len, matmul_f16_f32_aligned_s_data, "main", 3, 14 * sizeof(uint32_t), s_wg_denoms, warptile_s, s_align); } else { - vk_pipeline_matmul_f32_l = ggml_vk_create_pipeline("matmul_f32_l", matmul_f32_l_fp32_len, matmul_f32_l_fp32_data, "main", 3, 14 * sizeof(uint32_t), l_wg_denoms, warptile_l, 1); - vk_pipeline_matmul_f32_m = ggml_vk_create_pipeline("matmul_f32_m", matmul_f32_m_fp32_len, matmul_f32_m_fp32_data, "main", 3, 14 * sizeof(uint32_t), m_wg_denoms, warptile_m, 1); - vk_pipeline_matmul_f32_s = ggml_vk_create_pipeline("matmul_f32_s", matmul_f32_s_fp32_len, matmul_f32_s_fp32_data, "main", 3, 14 * sizeof(uint32_t), s_wg_denoms, warptile_s, 1); - vk_pipeline_matmul_f32_aligned_l = ggml_vk_create_pipeline("matmul_f32_aligned_l", matmul_f32_aligned_l_fp32_len, matmul_f32_aligned_l_fp32_data, "main", 3, 14 * sizeof(uint32_t), l_wg_denoms, warptile_l, l_align); - vk_pipeline_matmul_f32_aligned_m = ggml_vk_create_pipeline("matmul_f32_aligned_m", matmul_f32_aligned_m_fp32_len, matmul_f32_aligned_m_fp32_data, "main", 3, 14 * sizeof(uint32_t), m_wg_denoms, warptile_m, m_align); - vk_pipeline_matmul_f32_aligned_s = ggml_vk_create_pipeline("matmul_f32_aligned_s", matmul_f32_aligned_s_fp32_len, matmul_f32_aligned_s_fp32_data, "main", 3, 14 * sizeof(uint32_t), s_wg_denoms, warptile_s, s_align); + ggml_vk_create_pipeline(ctx, ctx->pipeline_matmul_f32_l, "matmul_f32_l", matmul_f32_l_fp32_len, matmul_f32_l_fp32_data, "main", 3, 14 * sizeof(uint32_t), l_wg_denoms, warptile_l, 1); + ggml_vk_create_pipeline(ctx, ctx->pipeline_matmul_f32_m, "matmul_f32_m", matmul_f32_m_fp32_len, matmul_f32_m_fp32_data, "main", 3, 14 * sizeof(uint32_t), m_wg_denoms, warptile_m, 1); + ggml_vk_create_pipeline(ctx, ctx->pipeline_matmul_f32_s, "matmul_f32_s", matmul_f32_s_fp32_len, matmul_f32_s_fp32_data, "main", 3, 14 * sizeof(uint32_t), s_wg_denoms, warptile_s, 1); + ggml_vk_create_pipeline(ctx, ctx->pipeline_matmul_f32_aligned_l, "matmul_f32_aligned_l", matmul_f32_aligned_l_fp32_len, matmul_f32_aligned_l_fp32_data, "main", 3, 14 * sizeof(uint32_t), l_wg_denoms, warptile_l, l_align); + ggml_vk_create_pipeline(ctx, ctx->pipeline_matmul_f32_aligned_m, "matmul_f32_aligned_m", matmul_f32_aligned_m_fp32_len, matmul_f32_aligned_m_fp32_data, "main", 3, 14 * sizeof(uint32_t), m_wg_denoms, warptile_m, m_align); + ggml_vk_create_pipeline(ctx, ctx->pipeline_matmul_f32_aligned_s, "matmul_f32_aligned_s", matmul_f32_aligned_s_fp32_len, matmul_f32_aligned_s_fp32_data, "main", 3, 14 * sizeof(uint32_t), s_wg_denoms, warptile_s, s_align); - vk_pipeline_matmul_f16_l = ggml_vk_create_pipeline("matmul_f16_l", matmul_f16_l_fp32_len, matmul_f16_l_fp32_data, "main", 3, 14 * sizeof(uint32_t), l_wg_denoms, warptile_l, 1); - vk_pipeline_matmul_f16_m = ggml_vk_create_pipeline("matmul_f16_m", matmul_f16_m_fp32_len, matmul_f16_m_fp32_data, "main", 3, 14 * sizeof(uint32_t), m_wg_denoms, warptile_m, 1); - vk_pipeline_matmul_f16_s = ggml_vk_create_pipeline("matmul_f16_s", matmul_f16_s_fp32_len, matmul_f16_s_fp32_data, "main", 3, 14 * sizeof(uint32_t), s_wg_denoms, warptile_s, 1); + ggml_vk_create_pipeline(ctx, ctx->pipeline_matmul_f16_l, "matmul_f16_l", matmul_f16_l_fp32_len, matmul_f16_l_fp32_data, "main", 3, 14 * sizeof(uint32_t), l_wg_denoms, warptile_l, 1); + ggml_vk_create_pipeline(ctx, ctx->pipeline_matmul_f16_m, "matmul_f16_m", matmul_f16_m_fp32_len, matmul_f16_m_fp32_data, "main", 3, 14 * sizeof(uint32_t), m_wg_denoms, warptile_m, 1); + ggml_vk_create_pipeline(ctx, ctx->pipeline_matmul_f16_s, "matmul_f16_s", matmul_f16_s_fp32_len, matmul_f16_s_fp32_data, "main", 3, 14 * sizeof(uint32_t), s_wg_denoms, warptile_s, 1); + ggml_vk_create_pipeline(ctx, ctx->pipeline_matmul_f16_aligned_l, "matmul_f16_aligned_l", matmul_f16_aligned_l_fp32_len, matmul_f16_aligned_l_fp32_data, "main", 3, 14 * sizeof(uint32_t), l_wg_denoms, warptile_l, l_align); + ggml_vk_create_pipeline(ctx, ctx->pipeline_matmul_f16_aligned_m, "matmul_f16_aligned_m", matmul_f16_aligned_m_fp32_len, matmul_f16_aligned_m_fp32_data, "main", 3, 14 * sizeof(uint32_t), m_wg_denoms, warptile_m, m_align); + ggml_vk_create_pipeline(ctx, ctx->pipeline_matmul_f16_aligned_s, "matmul_f16_aligned_s", matmul_f16_aligned_s_fp32_len, matmul_f16_aligned_s_fp32_data, "main", 3, 14 * sizeof(uint32_t), s_wg_denoms, warptile_s, s_align); - vk_pipeline_matmul_f16_aligned_l = ggml_vk_create_pipeline("matmul_f16_aligned_l", matmul_f16_aligned_l_fp32_len, matmul_f16_aligned_l_fp32_data, "main", 3, 14 * sizeof(uint32_t), l_wg_denoms, warptile_l, l_align); - vk_pipeline_matmul_f16_aligned_m = ggml_vk_create_pipeline("matmul_f16_aligned_m", matmul_f16_aligned_m_fp32_len, matmul_f16_aligned_m_fp32_data, "main", 3, 14 * sizeof(uint32_t), m_wg_denoms, warptile_m, m_align); - vk_pipeline_matmul_f16_aligned_s = ggml_vk_create_pipeline("matmul_f16_aligned_s", matmul_f16_aligned_s_fp32_len, matmul_f16_aligned_s_fp32_data, "main", 3, 14 * sizeof(uint32_t), s_wg_denoms, warptile_s, s_align); - - vk_pipeline_matmul_f16_f32_l = ggml_vk_create_pipeline("matmul_f16_f32_l", matmul_f16_f32_l_fp32_len, matmul_f16_f32_l_fp32_data, "main", 3, 14 * sizeof(uint32_t), l_wg_denoms, warptile_l, 1); - vk_pipeline_matmul_f16_f32_m = ggml_vk_create_pipeline("matmul_f16_f32_m", matmul_f16_f32_m_fp32_len, matmul_f16_f32_m_fp32_data, "main", 3, 14 * sizeof(uint32_t), m_wg_denoms, warptile_m, 1); - vk_pipeline_matmul_f16_f32_s = ggml_vk_create_pipeline("matmul_f16_f32_s", matmul_f16_f32_s_fp32_len, matmul_f16_f32_s_fp32_data, "main", 3, 14 * sizeof(uint32_t), s_wg_denoms, warptile_s, 1); - vk_pipeline_matmul_f16_f32_aligned_l = ggml_vk_create_pipeline("matmul_f16_f32_aligned_l", matmul_f16_f32_aligned_l_fp32_len, matmul_f16_f32_aligned_l_fp32_data, "main", 3, 14 * sizeof(uint32_t), l_wg_denoms, warptile_l, l_align); - vk_pipeline_matmul_f16_f32_aligned_m = ggml_vk_create_pipeline("matmul_f16_f32_aligned_m", matmul_f16_f32_aligned_m_fp32_len, matmul_f16_f32_aligned_m_fp32_data, "main", 3, 14 * sizeof(uint32_t), m_wg_denoms, warptile_m, m_align); - vk_pipeline_matmul_f16_f32_aligned_s = ggml_vk_create_pipeline("matmul_f16_f32_aligned_s", matmul_f16_f32_aligned_s_fp32_len, matmul_f16_f32_aligned_s_fp32_data, "main", 3, 14 * sizeof(uint32_t), s_wg_denoms, warptile_s, s_align); + ggml_vk_create_pipeline(ctx, ctx->pipeline_matmul_f16_f32_l, "matmul_f16_f32_l", matmul_f16_f32_l_fp32_len, matmul_f16_f32_l_fp32_data, "main", 3, 14 * sizeof(uint32_t), l_wg_denoms, warptile_l, 1); + ggml_vk_create_pipeline(ctx, ctx->pipeline_matmul_f16_f32_m, "matmul_f16_f32_m", matmul_f16_f32_m_fp32_len, matmul_f16_f32_m_fp32_data, "main", 3, 14 * sizeof(uint32_t), m_wg_denoms, warptile_m, 1); + ggml_vk_create_pipeline(ctx, ctx->pipeline_matmul_f16_f32_s, "matmul_f16_f32_s", matmul_f16_f32_s_fp32_len, matmul_f16_f32_s_fp32_data, "main", 3, 14 * sizeof(uint32_t), s_wg_denoms, warptile_s, 1); + ggml_vk_create_pipeline(ctx, ctx->pipeline_matmul_f16_f32_aligned_l, "matmul_f16_f32_aligned_l", matmul_f16_f32_aligned_l_fp32_len, matmul_f16_f32_aligned_l_fp32_data, "main", 3, 14 * sizeof(uint32_t), l_wg_denoms, warptile_l, l_align); + ggml_vk_create_pipeline(ctx, ctx->pipeline_matmul_f16_f32_aligned_m, "matmul_f16_f32_aligned_m", matmul_f16_f32_aligned_m_fp32_len, matmul_f16_f32_aligned_m_fp32_data, "main", 3, 14 * sizeof(uint32_t), m_wg_denoms, warptile_m, m_align); + ggml_vk_create_pipeline(ctx, ctx->pipeline_matmul_f16_f32_aligned_s, "matmul_f16_f32_aligned_s", matmul_f16_f32_aligned_s_fp32_len, matmul_f16_f32_aligned_s_fp32_data, "main", 3, 14 * sizeof(uint32_t), s_wg_denoms, warptile_s, s_align); } - vk_pipeline_dequant_mul_mat_vec_f32[GGML_TYPE_F16] = ggml_vk_create_pipeline("mul_mat_vec_f16_f32", mul_mat_vec_f16_f32_len, mul_mat_vec_f16_f32_data, "main", 3, 3 * sizeof(int), {1, 1, 1}, {}, 1); - vk_pipeline_dequant_mul_mat_vec_f32[GGML_TYPE_Q4_0] = ggml_vk_create_pipeline("mul_mat_vec_q4_0_f32", mul_mat_vec_q4_0_f32_len, mul_mat_vec_q4_0_f32_data, "main", 3, 3 * sizeof(int), {1, 1, 1}, {}, 1); - vk_pipeline_dequant_mul_mat_vec_f32[GGML_TYPE_Q4_1] = ggml_vk_create_pipeline("mul_mat_vec_q4_1_f32", mul_mat_vec_q4_1_f32_len, mul_mat_vec_q4_1_f32_data, "main", 3, 3 * sizeof(int), {1, 1, 1}, {}, 1); - vk_pipeline_dequant_mul_mat_vec_f32[GGML_TYPE_Q5_0] = ggml_vk_create_pipeline("mul_mat_vec_q5_0_f32", mul_mat_vec_q5_0_f32_len, mul_mat_vec_q5_0_f32_data, "main", 3, 3 * sizeof(int), {1, 1, 1}, {}, 1); - vk_pipeline_dequant_mul_mat_vec_f32[GGML_TYPE_Q5_1] = ggml_vk_create_pipeline("mul_mat_vec_q5_1_f32", mul_mat_vec_q5_1_f32_len, mul_mat_vec_q5_1_f32_data, "main", 3, 3 * sizeof(int), {1, 1, 1}, {}, 1); - vk_pipeline_dequant_mul_mat_vec_f32[GGML_TYPE_Q8_0] = ggml_vk_create_pipeline("mul_mat_vec_q8_0_f32", mul_mat_vec_q8_0_f32_len, mul_mat_vec_q8_0_f32_data, "main", 3, 3 * sizeof(int), {1, 1, 1}, {}, 1); - vk_pipeline_dequant_mul_mat_vec_f32[GGML_TYPE_Q2_K] = ggml_vk_create_pipeline("mul_mat_vec_q2_K_f32", mul_mat_vec_q2_K_f32_len, mul_mat_vec_q2_K_f32_data, "main", 3, 3 * sizeof(int), {1, 1, 1}, {}, 1); - vk_pipeline_dequant_mul_mat_vec_f32[GGML_TYPE_Q3_K] = ggml_vk_create_pipeline("mul_mat_vec_q3_K_f32", mul_mat_vec_q3_K_f32_len, mul_mat_vec_q3_K_f32_data, "main", 3, 3 * sizeof(int), {1, 1, 1}, {}, 1); - vk_pipeline_dequant_mul_mat_vec_f32[GGML_TYPE_Q4_K] = ggml_vk_create_pipeline("mul_mat_vec_q4_K_f32", mul_mat_vec_q4_K_f32_len, mul_mat_vec_q4_K_f32_data, "main", 3, 3 * sizeof(int), {1, 1, 1}, {}, 1); - vk_pipeline_dequant_mul_mat_vec_f32[GGML_TYPE_Q5_K] = ggml_vk_create_pipeline("mul_mat_vec_q5_K_f32", mul_mat_vec_q5_K_f32_len, mul_mat_vec_q5_K_f32_data, "main", 3, 3 * sizeof(int), {1, 1, 1}, {}, 1); - vk_pipeline_dequant_mul_mat_vec_f32[GGML_TYPE_Q6_K] = ggml_vk_create_pipeline("mul_mat_vec_q6_K_f32", mul_mat_vec_q6_K_f32_len, mul_mat_vec_q6_K_f32_data, "main", 3, 3 * sizeof(int), {1, 1, 1}, {}, 1); + ggml_vk_create_pipeline(ctx, ctx->pipeline_dequant_mul_mat_vec_f32[GGML_TYPE_F16 ], "mul_mat_vec_f16_f32", mul_mat_vec_f16_f32_len, mul_mat_vec_f16_f32_data, "main", 3, 3 * sizeof(int), {1, 1, 1}, {}, 1); + ggml_vk_create_pipeline(ctx, ctx->pipeline_dequant_mul_mat_vec_f32[GGML_TYPE_Q4_0], "mul_mat_vec_q4_0_f32", mul_mat_vec_q4_0_f32_len, mul_mat_vec_q4_0_f32_data, "main", 3, 3 * sizeof(int), {1, 1, 1}, {}, 1); + ggml_vk_create_pipeline(ctx, ctx->pipeline_dequant_mul_mat_vec_f32[GGML_TYPE_Q4_1], "mul_mat_vec_q4_1_f32", mul_mat_vec_q4_1_f32_len, mul_mat_vec_q4_1_f32_data, "main", 3, 3 * sizeof(int), {1, 1, 1}, {}, 1); + ggml_vk_create_pipeline(ctx, ctx->pipeline_dequant_mul_mat_vec_f32[GGML_TYPE_Q5_0], "mul_mat_vec_q5_0_f32", mul_mat_vec_q5_0_f32_len, mul_mat_vec_q5_0_f32_data, "main", 3, 3 * sizeof(int), {1, 1, 1}, {}, 1); + ggml_vk_create_pipeline(ctx, ctx->pipeline_dequant_mul_mat_vec_f32[GGML_TYPE_Q5_1], "mul_mat_vec_q5_1_f32", mul_mat_vec_q5_1_f32_len, mul_mat_vec_q5_1_f32_data, "main", 3, 3 * sizeof(int), {1, 1, 1}, {}, 1); + ggml_vk_create_pipeline(ctx, ctx->pipeline_dequant_mul_mat_vec_f32[GGML_TYPE_Q8_0], "mul_mat_vec_q8_0_f32", mul_mat_vec_q8_0_f32_len, mul_mat_vec_q8_0_f32_data, "main", 3, 3 * sizeof(int), {1, 1, 1}, {}, 1); + ggml_vk_create_pipeline(ctx, ctx->pipeline_dequant_mul_mat_vec_f32[GGML_TYPE_Q2_K], "mul_mat_vec_q2_K_f32", mul_mat_vec_q2_K_f32_len, mul_mat_vec_q2_K_f32_data, "main", 3, 3 * sizeof(int), {1, 1, 1}, {}, 1); + ggml_vk_create_pipeline(ctx, ctx->pipeline_dequant_mul_mat_vec_f32[GGML_TYPE_Q3_K], "mul_mat_vec_q3_K_f32", mul_mat_vec_q3_K_f32_len, mul_mat_vec_q3_K_f32_data, "main", 3, 3 * sizeof(int), {1, 1, 1}, {}, 1); + ggml_vk_create_pipeline(ctx, ctx->pipeline_dequant_mul_mat_vec_f32[GGML_TYPE_Q4_K], "mul_mat_vec_q4_K_f32", mul_mat_vec_q4_K_f32_len, mul_mat_vec_q4_K_f32_data, "main", 3, 3 * sizeof(int), {1, 1, 1}, {}, 1); + ggml_vk_create_pipeline(ctx, ctx->pipeline_dequant_mul_mat_vec_f32[GGML_TYPE_Q5_K], "mul_mat_vec_q5_K_f32", mul_mat_vec_q5_K_f32_len, mul_mat_vec_q5_K_f32_data, "main", 3, 3 * sizeof(int), {1, 1, 1}, {}, 1); + ggml_vk_create_pipeline(ctx, ctx->pipeline_dequant_mul_mat_vec_f32[GGML_TYPE_Q6_K], "mul_mat_vec_q6_K_f32", mul_mat_vec_q6_K_f32_len, mul_mat_vec_q6_K_f32_data, "main", 3, 3 * sizeof(int), {1, 1, 1}, {}, 1); // dequant shaders - vk_pipeline_dequant[GGML_TYPE_F32] = ggml_vk_create_pipeline("f32_to_f16", f32_to_f16_len, f32_to_f16_data, "main", 2, 4 * sizeof(int), {64, 1, 1}, {}, 1); - - vk_pipeline_dequant[GGML_TYPE_F16] = ggml_vk_create_pipeline("dequant_f16", dequant_f16_len, dequant_f16_data, "main", 2, 4 * sizeof(int), {256 * 32, 1, 1}, {}, 1); - vk_pipeline_dequant[GGML_TYPE_Q4_0] = ggml_vk_create_pipeline("dequant_q4_0", dequant_q4_0_len, dequant_q4_0_data, "main", 2, 4 * sizeof(int), {256 * 32, 1, 1}, {}, 1); - vk_pipeline_dequant[GGML_TYPE_Q4_1] = ggml_vk_create_pipeline("dequant_q4_1", dequant_q4_1_len, dequant_q4_1_data, "main", 2, 4 * sizeof(int), {256 * 32, 1, 1}, {}, 1); - vk_pipeline_dequant[GGML_TYPE_Q5_0] = ggml_vk_create_pipeline("dequant_q5_0", dequant_q5_0_len, dequant_q5_0_data, "main", 2, 4 * sizeof(int), {256 * 32, 1, 1}, {}, 1); - vk_pipeline_dequant[GGML_TYPE_Q5_1] = ggml_vk_create_pipeline("dequant_q5_1", dequant_q5_1_len, dequant_q5_1_data, "main", 2, 4 * sizeof(int), {256 * 32, 1, 1}, {}, 1); - vk_pipeline_dequant[GGML_TYPE_Q8_0] = ggml_vk_create_pipeline("dequant_q8_0", dequant_q8_0_len, dequant_q8_0_data, "main", 2, 4 * sizeof(int), {256 * 32, 1, 1}, {}, 1); - vk_pipeline_dequant[GGML_TYPE_Q2_K] = ggml_vk_create_pipeline("dequant_q2_K", dequant_q2_K_len, dequant_q2_K_data, "main", 2, 4 * sizeof(int), {256 * 64, 1, 1}, {}, 1); - vk_pipeline_dequant[GGML_TYPE_Q3_K] = ggml_vk_create_pipeline("dequant_q3_K", dequant_q3_K_len, dequant_q3_K_data, "main", 2, 4 * sizeof(int), {256 * 64, 1, 1}, {}, 1); - vk_pipeline_dequant[GGML_TYPE_Q4_K] = ggml_vk_create_pipeline("dequant_q4_K", dequant_q4_K_len, dequant_q4_K_data, "main", 2, 4 * sizeof(int), {256 * 32, 1, 1}, {}, 1); - vk_pipeline_dequant[GGML_TYPE_Q5_K] = ggml_vk_create_pipeline("dequant_q5_K", dequant_q5_K_len, dequant_q5_K_data, "main", 2, 4 * sizeof(int), {256 * 64, 1, 1}, {}, 1); - vk_pipeline_dequant[GGML_TYPE_Q6_K] = ggml_vk_create_pipeline("dequant_q6_K", dequant_q6_K_len, dequant_q6_K_data, "main", 2, 4 * sizeof(int), {256 * 64, 1, 1}, {}, 1); + ggml_vk_create_pipeline(ctx, ctx->pipeline_dequant[GGML_TYPE_F32 ], "f32_to_f16", f32_to_f16_len, f32_to_f16_data, "main", 2, 4 * sizeof(int), { 64, 1, 1}, {}, 1); + ggml_vk_create_pipeline(ctx, ctx->pipeline_dequant[GGML_TYPE_F16 ], "dequant_f16", dequant_f16_len, dequant_f16_data, "main", 2, 4 * sizeof(int), {256 * 32, 1, 1}, {}, 1); + ggml_vk_create_pipeline(ctx, ctx->pipeline_dequant[GGML_TYPE_Q4_0], "dequant_q4_0", dequant_q4_0_len, dequant_q4_0_data, "main", 2, 4 * sizeof(int), {256 * 32, 1, 1}, {}, 1); + ggml_vk_create_pipeline(ctx, ctx->pipeline_dequant[GGML_TYPE_Q4_1], "dequant_q4_1", dequant_q4_1_len, dequant_q4_1_data, "main", 2, 4 * sizeof(int), {256 * 32, 1, 1}, {}, 1); + ggml_vk_create_pipeline(ctx, ctx->pipeline_dequant[GGML_TYPE_Q5_0], "dequant_q5_0", dequant_q5_0_len, dequant_q5_0_data, "main", 2, 4 * sizeof(int), {256 * 32, 1, 1}, {}, 1); + ggml_vk_create_pipeline(ctx, ctx->pipeline_dequant[GGML_TYPE_Q5_1], "dequant_q5_1", dequant_q5_1_len, dequant_q5_1_data, "main", 2, 4 * sizeof(int), {256 * 32, 1, 1}, {}, 1); + ggml_vk_create_pipeline(ctx, ctx->pipeline_dequant[GGML_TYPE_Q8_0], "dequant_q8_0", dequant_q8_0_len, dequant_q8_0_data, "main", 2, 4 * sizeof(int), {256 * 32, 1, 1}, {}, 1); + ggml_vk_create_pipeline(ctx, ctx->pipeline_dequant[GGML_TYPE_Q2_K], "dequant_q2_K", dequant_q2_K_len, dequant_q2_K_data, "main", 2, 4 * sizeof(int), {256 * 64, 1, 1}, {}, 1); + ggml_vk_create_pipeline(ctx, ctx->pipeline_dequant[GGML_TYPE_Q3_K], "dequant_q3_K", dequant_q3_K_len, dequant_q3_K_data, "main", 2, 4 * sizeof(int), {256 * 64, 1, 1}, {}, 1); + ggml_vk_create_pipeline(ctx, ctx->pipeline_dequant[GGML_TYPE_Q4_K], "dequant_q4_K", dequant_q4_K_len, dequant_q4_K_data, "main", 2, 4 * sizeof(int), {256 * 32, 1, 1}, {}, 1); + ggml_vk_create_pipeline(ctx, ctx->pipeline_dequant[GGML_TYPE_Q5_K], "dequant_q5_K", dequant_q5_K_len, dequant_q5_K_data, "main", 2, 4 * sizeof(int), {256 * 64, 1, 1}, {}, 1); + ggml_vk_create_pipeline(ctx, ctx->pipeline_dequant[GGML_TYPE_Q6_K], "dequant_q6_K", dequant_q6_K_len, dequant_q6_K_data, "main", 2, 4 * sizeof(int), {256 * 64, 1, 1}, {}, 1); // get_rows - vk_pipeline_get_rows[GGML_TYPE_F16] = ggml_vk_create_pipeline("get_rows_f16", get_rows_f16_len, get_rows_f16_data, "main", 3, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1); - vk_pipeline_get_rows[GGML_TYPE_Q4_0] = ggml_vk_create_pipeline("get_rows_q4_0", get_rows_q4_0_len, get_rows_q4_0_data, "main", 3, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1); - vk_pipeline_get_rows[GGML_TYPE_Q4_1] = ggml_vk_create_pipeline("get_rows_q4_1", get_rows_q4_1_len, get_rows_q4_1_data, "main", 3, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1); - vk_pipeline_get_rows[GGML_TYPE_Q5_0] = ggml_vk_create_pipeline("get_rows_q5_0", get_rows_q5_0_len, get_rows_q5_0_data, "main", 3, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1); - vk_pipeline_get_rows[GGML_TYPE_Q5_1] = ggml_vk_create_pipeline("get_rows_q5_1", get_rows_q5_1_len, get_rows_q5_1_data, "main", 3, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1); - vk_pipeline_get_rows[GGML_TYPE_Q8_0] = ggml_vk_create_pipeline("get_rows_q8_0", get_rows_q8_0_len, get_rows_q8_0_data, "main", 3, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1); + ggml_vk_create_pipeline(ctx, ctx->pipeline_get_rows[GGML_TYPE_F16 ], "get_rows_f16", get_rows_f16_len, get_rows_f16_data, "main", 3, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1); + ggml_vk_create_pipeline(ctx, ctx->pipeline_get_rows[GGML_TYPE_Q4_0], "get_rows_q4_0", get_rows_q4_0_len, get_rows_q4_0_data, "main", 3, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1); + ggml_vk_create_pipeline(ctx, ctx->pipeline_get_rows[GGML_TYPE_Q4_1], "get_rows_q4_1", get_rows_q4_1_len, get_rows_q4_1_data, "main", 3, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1); + ggml_vk_create_pipeline(ctx, ctx->pipeline_get_rows[GGML_TYPE_Q5_0], "get_rows_q5_0", get_rows_q5_0_len, get_rows_q5_0_data, "main", 3, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1); + ggml_vk_create_pipeline(ctx, ctx->pipeline_get_rows[GGML_TYPE_Q5_1], "get_rows_q5_1", get_rows_q5_1_len, get_rows_q5_1_data, "main", 3, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1); + ggml_vk_create_pipeline(ctx, ctx->pipeline_get_rows[GGML_TYPE_Q8_0], "get_rows_q8_0", get_rows_q8_0_len, get_rows_q8_0_data, "main", 3, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1); - vk_pipeline_get_rows_f32[GGML_TYPE_F16] = ggml_vk_create_pipeline("get_rows_f16_f32", get_rows_f16_f32_len, get_rows_f16_f32_data, "main", 3, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1); - vk_pipeline_get_rows_f32[GGML_TYPE_Q4_0] = ggml_vk_create_pipeline("get_rows_q4_0_f32", get_rows_q4_0_f32_len, get_rows_q4_0_f32_data, "main", 3, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1); - vk_pipeline_get_rows_f32[GGML_TYPE_Q4_1] = ggml_vk_create_pipeline("get_rows_q4_1_f32", get_rows_q4_1_f32_len, get_rows_q4_1_f32_data, "main", 3, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1); - vk_pipeline_get_rows_f32[GGML_TYPE_Q5_0] = ggml_vk_create_pipeline("get_rows_q5_0_f32", get_rows_q5_0_f32_len, get_rows_q5_0_f32_data, "main", 3, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1); - vk_pipeline_get_rows_f32[GGML_TYPE_Q5_1] = ggml_vk_create_pipeline("get_rows_q5_1_f32", get_rows_q5_1_f32_len, get_rows_q5_1_f32_data, "main", 3, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1); - vk_pipeline_get_rows_f32[GGML_TYPE_Q8_0] = ggml_vk_create_pipeline("get_rows_q8_0_f32", get_rows_q8_0_f32_len, get_rows_q8_0_f32_data, "main", 3, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1); + ggml_vk_create_pipeline(ctx, ctx->pipeline_get_rows_f32[GGML_TYPE_F32 ], "get_rows_f16_f32", get_rows_f16_f32_len, get_rows_f16_f32_data, "main", 3, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1); + ggml_vk_create_pipeline(ctx, ctx->pipeline_get_rows_f32[GGML_TYPE_Q4_0], "get_rows_q4_0_f32", get_rows_q4_0_f32_len, get_rows_q4_0_f32_data, "main", 3, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1); + ggml_vk_create_pipeline(ctx, ctx->pipeline_get_rows_f32[GGML_TYPE_Q4_1], "get_rows_q4_1_f32", get_rows_q4_1_f32_len, get_rows_q4_1_f32_data, "main", 3, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1); + ggml_vk_create_pipeline(ctx, ctx->pipeline_get_rows_f32[GGML_TYPE_Q5_0], "get_rows_q5_0_f32", get_rows_q5_0_f32_len, get_rows_q5_0_f32_data, "main", 3, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1); + ggml_vk_create_pipeline(ctx, ctx->pipeline_get_rows_f32[GGML_TYPE_Q5_1], "get_rows_q5_1_f32", get_rows_q5_1_f32_len, get_rows_q5_1_f32_data, "main", 3, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1); + ggml_vk_create_pipeline(ctx, ctx->pipeline_get_rows_f32[GGML_TYPE_Q8_0], "get_rows_q8_0_f32", get_rows_q8_0_f32_len, get_rows_q8_0_f32_data, "main", 3, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1); - vk_pipeline_matmul_split_k_reduce = ggml_vk_create_pipeline("split_k_reduce", split_k_reduce_len, split_k_reduce_data, "main", 2, 2 * sizeof(uint32_t), {256, 1, 1}, {}, 1); + ggml_vk_create_pipeline(ctx, ctx->pipeline_matmul_split_k_reduce, "split_k_reduce", split_k_reduce_len, split_k_reduce_data, "main", 2, 2 * sizeof(uint32_t), {256, 1, 1}, {}, 1); - vk_pipeline_mul_mat_vec_p021_f16_f32 = ggml_vk_create_pipeline("mul_mat_vec_p021_f16_f32", mul_mat_vec_p021_f16_f32_len, mul_mat_vec_p021_f16_f32_data, "main", 3, 6 * sizeof(uint32_t), {1, 1, 1}, {}, 1); - vk_pipeline_mul_mat_vec_nc_f16_f32 = ggml_vk_create_pipeline("mul_mat_vec_nc_f16_f32", mul_mat_vec_nc_f16_f32_len, mul_mat_vec_nc_f16_f32_data, "main", 3, 7 * sizeof(uint32_t), {1, 1, 1}, {}, 1); + ggml_vk_create_pipeline(ctx, ctx->pipeline_mul_mat_vec_p021_f16_f32, "mul_mat_vec_p021_f16_f32", mul_mat_vec_p021_f16_f32_len, mul_mat_vec_p021_f16_f32_data, "main", 3, 6 * sizeof(uint32_t), {1, 1, 1}, {}, 1); + ggml_vk_create_pipeline(ctx, ctx->pipeline_mul_mat_vec_nc_f16_f32, "mul_mat_vec_nc_f16_f32", mul_mat_vec_nc_f16_f32_len, mul_mat_vec_nc_f16_f32_data, "main", 3, 7 * sizeof(uint32_t), {1, 1, 1}, {}, 1); - vk_pipeline_norm_f32 = ggml_vk_create_pipeline("norm_f32", norm_f32_len, norm_f32_data, "main", 2, sizeof(vk_op_push_constants), {1, 1, 1}, {}, 1); - vk_pipeline_rms_norm_f32 = ggml_vk_create_pipeline("rms_norm_f32", rms_norm_f32_len, rms_norm_f32_data, "main", 2, sizeof(vk_op_push_constants), {1, 1, 1}, {}, 1); + ggml_vk_create_pipeline(ctx, ctx->pipeline_norm_f32, "norm_f32", norm_f32_len, norm_f32_data, "main", 2, sizeof(vk_op_push_constants), {1, 1, 1}, {}, 1); + ggml_vk_create_pipeline(ctx, ctx->pipeline_rms_norm_f32, "rms_norm_f32", rms_norm_f32_len, rms_norm_f32_data, "main", 2, sizeof(vk_op_push_constants), {1, 1, 1}, {}, 1); - vk_pipeline_cpy_f32_f32 = ggml_vk_create_pipeline("cpy_f32_f32", cpy_f32_f32_len, cpy_f32_f32_data, "main", 2, sizeof(vk_op_cpy_push_constants), {512, 1, 1}, {}, 1); - vk_pipeline_cpy_f32_f16 = ggml_vk_create_pipeline("cpy_f32_f16", cpy_f32_f16_len, cpy_f32_f16_data, "main", 2, sizeof(vk_op_cpy_push_constants), {512, 1, 1}, {}, 1); - vk_pipeline_cpy_f16_f16 = ggml_vk_create_pipeline("cpy_f16_f16", cpy_f16_f16_len, cpy_f16_f16_data, "main", 2, sizeof(vk_op_cpy_push_constants), {512, 1, 1}, {}, 1); + ggml_vk_create_pipeline(ctx, ctx->pipeline_cpy_f32_f32, "cpy_f32_f32", cpy_f32_f32_len, cpy_f32_f32_data, "main", 2, sizeof(vk_op_cpy_push_constants), {512, 1, 1}, {}, 1); + ggml_vk_create_pipeline(ctx, ctx->pipeline_cpy_f32_f16, "cpy_f32_f16", cpy_f32_f16_len, cpy_f32_f16_data, "main", 2, sizeof(vk_op_cpy_push_constants), {512, 1, 1}, {}, 1); + ggml_vk_create_pipeline(ctx, ctx->pipeline_cpy_f16_f16, "cpy_f16_f16", cpy_f16_f16_len, cpy_f16_f16_data, "main", 2, sizeof(vk_op_cpy_push_constants), {512, 1, 1}, {}, 1); - vk_pipeline_add_f32 = ggml_vk_create_pipeline("add_f32", add_f32_len, add_f32_data, "main", 3, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1); + ggml_vk_create_pipeline(ctx, ctx->pipeline_add_f32, "add_f32", add_f32_len, add_f32_data, "main", 3, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1); - vk_pipeline_mul_f32 = ggml_vk_create_pipeline("mul_f32", mul_f32_len, mul_f32_data, "main", 3, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1); + ggml_vk_create_pipeline(ctx, ctx->pipeline_mul_f32, "mul_f32", mul_f32_len, mul_f32_data, "main", 3, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1); - vk_pipeline_scale_f32 = ggml_vk_create_pipeline("scale_f32", scale_f32_len, scale_f32_data, "main", 2, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1); + ggml_vk_create_pipeline(ctx, ctx->pipeline_scale_f32, "scale_f32", scale_f32_len, scale_f32_data, "main", 2, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1); - vk_pipeline_sqr_f32 = ggml_vk_create_pipeline("sqr_f32", sqr_f32_len, sqr_f32_data, "main", 2, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1); + ggml_vk_create_pipeline(ctx, ctx->pipeline_sqr_f32, "sqr_f32", sqr_f32_len, sqr_f32_data, "main", 2, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1); - vk_pipeline_clamp_f32 = ggml_vk_create_pipeline("clamp_f32", clamp_f32_len, clamp_f32_data, "main", 2, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1); + ggml_vk_create_pipeline(ctx, ctx->pipeline_clamp_f32, "clamp_f32", clamp_f32_len, clamp_f32_data, "main", 2, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1); - vk_pipeline_gelu_f32 = ggml_vk_create_pipeline("gelu_f32", gelu_f32_len, gelu_f32_data, "main", 2, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1); - vk_pipeline_silu_f32 = ggml_vk_create_pipeline("silu_f32", silu_f32_len, silu_f32_data, "main", 2, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1); - vk_pipeline_relu_f32 = ggml_vk_create_pipeline("relu_f32", relu_f32_len, relu_f32_data, "main", 2, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1); + ggml_vk_create_pipeline(ctx, ctx->pipeline_gelu_f32, "gelu_f32", gelu_f32_len, gelu_f32_data, "main", 2, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1); + ggml_vk_create_pipeline(ctx, ctx->pipeline_silu_f32, "silu_f32", silu_f32_len, silu_f32_data, "main", 2, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1); + ggml_vk_create_pipeline(ctx, ctx->pipeline_relu_f32, "relu_f32", relu_f32_len, relu_f32_data, "main", 2, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1); - vk_pipeline_diag_mask_inf_f32 = ggml_vk_create_pipeline("diag_mask_inf_f32", diag_mask_inf_f32_len, diag_mask_inf_f32_data, "main", 2, sizeof(vk_op_diag_mask_push_constants), {512, 1, 1}, {}, 1); + ggml_vk_create_pipeline(ctx, ctx->pipeline_diag_mask_inf_f32, "diag_mask_inf_f32", diag_mask_inf_f32_len, diag_mask_inf_f32_data, "main", 2, sizeof(vk_op_diag_mask_push_constants), {512, 1, 1}, {}, 1); - vk_pipeline_soft_max_f32 = ggml_vk_create_pipeline("soft_max_f32", soft_max_f32_len, soft_max_f32_data, "main", 3, sizeof(vk_op_push_constants), {1, 1, 1}, {}, 1); + ggml_vk_create_pipeline(ctx, ctx->pipeline_soft_max_f32, "soft_max_f32", soft_max_f32_len, soft_max_f32_data, "main", 3, sizeof(vk_op_push_constants), {1, 1, 1}, {}, 1); - vk_pipeline_rope_f32 = ggml_vk_create_pipeline("rope_f32", rope_f32_len, rope_f32_data, "main", 3, sizeof(vk_op_rope_push_constants), {1, 512, 1}, {}, 1); - vk_pipeline_rope_f16 = ggml_vk_create_pipeline("rope_f16", rope_f16_len, rope_f16_data, "main", 3, sizeof(vk_op_rope_push_constants), {1, 512, 1}, {}, 1); + ggml_vk_create_pipeline(ctx, ctx->pipeline_rope_f32, "rope_f32", rope_f32_len, rope_f32_data, "main", 3, sizeof(vk_op_rope_push_constants), {1, 512, 1}, {}, 1); + ggml_vk_create_pipeline(ctx, ctx->pipeline_rope_f16, "rope_f16", rope_f16_len, rope_f16_data, "main", 3, sizeof(vk_op_rope_push_constants), {1, 512, 1}, {}, 1); - vk_pipeline_rope_neox_f32 = ggml_vk_create_pipeline("rope_neox_f32", rope_neox_f32_len, rope_neox_f32_data, "main", 3, sizeof(vk_op_rope_neox_push_constants), {1, 512, 1}, {}, 1); - vk_pipeline_rope_neox_f16 = ggml_vk_create_pipeline("rope_neox_f16", rope_neox_f16_len, rope_neox_f16_data, "main", 3, sizeof(vk_op_rope_neox_push_constants), {1, 512, 1}, {}, 1); + ggml_vk_create_pipeline(ctx, ctx->pipeline_rope_neox_f32, "rope_neox_f32", rope_neox_f32_len, rope_neox_f32_data, "main", 3, sizeof(vk_op_rope_neox_push_constants), {1, 512, 1}, {}, 1); + ggml_vk_create_pipeline(ctx, ctx->pipeline_rope_neox_f16, "rope_neox_f16", rope_neox_f16_len, rope_neox_f16_data, "main", 3, sizeof(vk_op_rope_neox_push_constants), {1, 512, 1}, {}, 1); } -void ggml_vk_init() { +static void ggml_vk_print_gpu_info(size_t idx) { + GGML_ASSERT(idx < vk_instance.device_indices.size()); + size_t dev_num = vk_instance.device_indices[idx]; #ifdef GGML_VULKAN_DEBUG - std::cerr << "ggml_vk_init()" << std::endl; + std::cerr << "ggml_vk_print_gpu_info(" << dev_num << ")" << std::endl; #endif - static bool initialized = false; + GGML_ASSERT(vk_instance.initialized); - if (initialized) { - return; + std::vector devices = vk_instance.instance.enumeratePhysicalDevices(); + + if (dev_num >= devices.size()) { + std::cerr << "ggml_vulkan: Device with index " << dev_num << " does not exist." << std::endl; + throw std::runtime_error("Device not found"); } - initialized = true; + vk::PhysicalDevice physical_device = devices[dev_num]; + std::vector ext_props = physical_device.enumerateDeviceExtensionProperties(); - const char* GGML_VULKAN_DEVICE = getenv("GGML_VULKAN_DEVICE"); - int dev_num = (GGML_VULKAN_DEVICE == NULL ? 0 : atoi(GGML_VULKAN_DEVICE)); + vk::PhysicalDeviceProperties2 props2; + vk::PhysicalDeviceMaintenance3Properties props3; + vk::PhysicalDeviceSubgroupProperties subgroup_props; + props2.pNext = &props3; + props3.pNext = &subgroup_props; + physical_device.getProperties2(&props2); + + const size_t subgroup_size = subgroup_props.subgroupSize; + const bool uma = props2.properties.deviceType == vk::PhysicalDeviceType::eIntegratedGpu; + + bool fp16_storage = false; + bool fp16_compute = false; + + for (auto properties : ext_props) { + if (strcmp("VK_KHR_16bit_storage", properties.extensionName) == 0) { + fp16_storage = true; + } else if (strcmp("VK_KHR_shader_float16_int8", properties.extensionName) == 0) { + fp16_compute = true; + } + } + + const char* GGML_VULKAN_DISABLE_F16 = getenv("GGML_VULKAN_DISABLE_F16"); + bool force_disable_f16 = GGML_VULKAN_DISABLE_F16 != nullptr; + + bool fp16 = !force_disable_f16 && fp16_storage && fp16_compute; + + vk::PhysicalDeviceFeatures device_features = physical_device.getFeatures(); + + VkPhysicalDeviceFeatures2 device_features2; + device_features2.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FEATURES_2; + device_features2.pNext = nullptr; + device_features2.features = (VkPhysicalDeviceFeatures)device_features; + + VkPhysicalDeviceVulkan11Features vk11_features; + vk11_features.pNext = nullptr; + vk11_features.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_1_FEATURES; + device_features2.pNext = &vk11_features; + + VkPhysicalDeviceVulkan12Features vk12_features; + vk12_features.pNext = nullptr; + vk12_features.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_2_FEATURES; + vk11_features.pNext = &vk12_features; + + vkGetPhysicalDeviceFeatures2(physical_device, &device_features2); + + fp16 = fp16 && vk12_features.shaderFloat16; + + std::string device_name = props2.properties.deviceName.data(); + std::cerr << GGML_VK_NAME << idx << ": " << device_name << " | uma: " << uma << " | fp16: " << fp16 << " | warp size: " << subgroup_size << std::endl; + + if (props2.properties.deviceType == vk::PhysicalDeviceType::eCpu) { + std::cerr << "ggml_vulkan: Warning: Device type is CPU. This is probably not the device you want." << std::endl; + } +} + +void ggml_vk_instance_init() { + if (vk_instance_initialized) { + return; + } +#ifdef GGML_VULKAN_DEBUG + std::cerr << "ggml_vk_instance_init()" << std::endl; +#endif vk::ApplicationInfo app_info{ "ggml-vulkan", 1, nullptr, 0, VK_API_VERSION }; const std::vector layers = { @@ -989,12 +1106,55 @@ void ggml_vk_init() { validation_features.setPNext(nullptr); instance_create_info.setPNext(&validation_features); -std::cerr << "ggml_vulkan: Validation layers enabled" << std::endl; + std::cerr << "ggml_vulkan: Validation layers enabled" << std::endl; #endif - vk_instance = vk::createInstance(instance_create_info); + vk_instance.instance = vk::createInstance(instance_create_info); - vk_device.physical_device = vk_instance.enumeratePhysicalDevices()[dev_num]; - std::vector ext_props = vk_device.physical_device.enumerateDeviceExtensionProperties(); + memset(vk_instance.initialized, 0, sizeof(bool) * GGML_VK_MAX_DEVICES); + + size_t num_available_devices = vk_instance.instance.enumeratePhysicalDevices().size(); + + // Emulate behavior of CUDA_VISIBLE_DEVICES for Vulkan + char * devices_env = getenv("GGML_VK_VISIBLE_DEVICES"); + if (devices_env != nullptr) { + std::string devices(devices_env); + std::replace(devices.begin(), devices.end(), ',', ' '); + + std::stringstream ss(devices); + size_t tmp; + while (ss >> tmp) { + if(tmp >= num_available_devices) { + std::cerr << "ggml_vulkan: Invalid device index " << tmp << " in GGML_VK_VISIBLE_DEVICES." << std::endl; + throw std::runtime_error("Invalid Vulkan device index"); + } + vk_instance.device_indices.push_back(tmp); + } + } else { + vk_instance.device_indices.push_back(0); + } + + vk_instance_initialized = true; +} + +void ggml_vk_init(ggml_backend_vk_context * ctx, size_t idx) { + GGML_ASSERT(idx < vk_instance.device_indices.size()); + size_t dev_num = vk_instance.device_indices[idx]; +#ifdef GGML_VULKAN_DEBUG + std::cerr << "ggml_vk_init(" << ctx->name << ", " << dev_num << ")" << std::endl; +#endif + ggml_vk_instance_init(); + + std::vector devices = vk_instance.instance.enumeratePhysicalDevices(); + + if (dev_num >= devices.size()) { + std::cerr << "ggml_vulkan: Device with index " << dev_num << " does not exist." << std::endl; + throw std::runtime_error("Device not found"); + } + + vk_instance.devices[idx] = std::make_shared(); + ctx->device = vk_instance.devices[idx]; + ctx->device.lock()->physical_device = devices[dev_num]; + std::vector ext_props = ctx->device.lock()->physical_device.enumerateDeviceExtensionProperties(); bool maintenance4_support = false; @@ -1014,18 +1174,18 @@ std::cerr << "ggml_vulkan: Validation layers enabled" << std::endl; if (maintenance4_support) { subgroup_props.pNext = &props4; } - vk_device.physical_device.getProperties2(&props2); - vk_device.properties = props2.properties; + ctx->device.lock()->physical_device.getProperties2(&props2); + ctx->device.lock()->properties = props2.properties; if (maintenance4_support) { - vk_device.max_memory_allocation_size = std::min(props3.maxMemoryAllocationSize, props4.maxBufferSize); + ctx->device.lock()->max_memory_allocation_size = std::min(props3.maxMemoryAllocationSize, props4.maxBufferSize); } else { - vk_device.max_memory_allocation_size = props3.maxMemoryAllocationSize; + ctx->device.lock()->max_memory_allocation_size = props3.maxMemoryAllocationSize; } - vk_device.vendor_id = vk_device.properties.vendorID; - vk_device.subgroup_size = subgroup_props.subgroupSize; - vk_device.uma = vk_device.properties.deviceType == vk::PhysicalDeviceType::eIntegratedGpu; + ctx->device.lock()->vendor_id = ctx->device.lock()->properties.vendorID; + ctx->device.lock()->subgroup_size = subgroup_props.subgroupSize; + ctx->device.lock()->uma = ctx->device.lock()->properties.deviceType == vk::PhysicalDeviceType::eIntegratedGpu; bool fp16_storage = false; bool fp16_compute = false; @@ -1039,31 +1199,31 @@ std::cerr << "ggml_vulkan: Validation layers enabled" << std::endl; } const char* GGML_VULKAN_DISABLE_F16 = getenv("GGML_VULKAN_DISABLE_F16"); - bool force_disable_f16 = GGML_VULKAN_DISABLE_F16 != NULL; + bool force_disable_f16 = GGML_VULKAN_DISABLE_F16 != nullptr; - vk_device.fp16 = !force_disable_f16 && fp16_storage && fp16_compute; + ctx->device.lock()->fp16 = !force_disable_f16 && fp16_storage && fp16_compute; - std::vector queue_family_props = vk_device.physical_device.getQueueFamilyProperties(); + std::vector queue_family_props = ctx->device.lock()->physical_device.getQueueFamilyProperties(); // Try to find a non-graphics compute queue and transfer-focused queues const uint32_t compute_queue_family_index = ggml_vk_find_queue_family_index(queue_family_props, vk::QueueFlagBits::eCompute, vk::QueueFlagBits::eGraphics, -1, 1); const uint32_t transfer_queue_family_index = ggml_vk_find_queue_family_index(queue_family_props, vk::QueueFlagBits::eTransfer, vk::QueueFlagBits::eCompute | vk::QueueFlagBits::eGraphics, compute_queue_family_index, 1); const float priorities[] = { 1.0f, 1.0f }; - const bool single_queue = compute_queue_family_index == transfer_queue_family_index && queue_family_props[compute_queue_family_index].queueCount == 1; + ctx->device.lock()->single_queue = compute_queue_family_index == transfer_queue_family_index && queue_family_props[compute_queue_family_index].queueCount == 1; std::vector device_queue_create_infos; if (compute_queue_family_index != transfer_queue_family_index) { device_queue_create_infos.push_back({vk::DeviceQueueCreateFlags(), compute_queue_family_index, 1, priorities}); device_queue_create_infos.push_back({vk::DeviceQueueCreateFlags(), transfer_queue_family_index, 1, priorities + 1}); - } else if(!single_queue) { + } else if(!ctx->device.lock()->single_queue) { device_queue_create_infos.push_back({vk::DeviceQueueCreateFlags(), compute_queue_family_index, 2, priorities}); } else { device_queue_create_infos.push_back({vk::DeviceQueueCreateFlags(), compute_queue_family_index, 1, priorities}); } vk::DeviceCreateInfo device_create_info; std::vector device_extensions; - vk::PhysicalDeviceFeatures device_features = vk_device.physical_device.getFeatures(); + vk::PhysicalDeviceFeatures device_features = ctx->device.lock()->physical_device.getFeatures(); VkPhysicalDeviceFeatures2 device_features2; device_features2.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FEATURES_2; @@ -1080,13 +1240,13 @@ std::cerr << "ggml_vulkan: Validation layers enabled" << std::endl; vk12_features.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_2_FEATURES; vk11_features.pNext = &vk12_features; - vkGetPhysicalDeviceFeatures2(vk_device.physical_device, &device_features2); + vkGetPhysicalDeviceFeatures2(ctx->device.lock()->physical_device, &device_features2); - vk_device.fp16 = vk_device.fp16 && vk12_features.shaderFloat16; + ctx->device.lock()->fp16 = ctx->device.lock()->fp16 && vk12_features.shaderFloat16; if (!vk11_features.storageBuffer16BitAccess) { - std::cerr << "ggml_vulkan: device does not support 16-bit storage" << std::endl; - GGML_ASSERT(false); + std::cerr << "ggml_vulkan: device " << GGML_VK_NAME << idx << " does not support 16-bit storage." << std::endl; + throw std::runtime_error("Unsupported device"); } device_extensions.push_back("VK_KHR_16bit_storage"); @@ -1095,10 +1255,11 @@ std::cerr << "ggml_vulkan: Validation layers enabled" << std::endl; device_extensions.push_back("VK_KHR_shader_non_semantic_info"); #endif - if (vk_device.fp16) { + if (ctx->device.lock()->fp16) { device_extensions.push_back("VK_KHR_shader_float16_int8"); } - std::cerr << "ggml_vulkan: Using " << vk_device.properties.deviceName << " | uma: " << vk_device.uma << " | fp16: " << vk_device.fp16 << " | warp size: " << vk_device.subgroup_size << std::endl; + ctx->device.lock()->name = ctx->device.lock()->properties.deviceName.data(); + device_create_info = { vk::DeviceCreateFlags(), device_queue_create_infos, @@ -1106,28 +1267,32 @@ std::cerr << "ggml_vulkan: Validation layers enabled" << std::endl; device_extensions }; device_create_info.setPNext(&device_features2); - vk_device.device = vk_device.physical_device.createDevice(device_create_info); + ctx->device.lock()->device = ctx->device.lock()->physical_device.createDevice(device_create_info); - vk_device.descriptor_set_mode = VK_DEVICE_DESCRIPTOR_POOL_MODE_UNKNOWN; + ctx->device.lock()->descriptor_set_mode = VK_DEVICE_DESCRIPTOR_POOL_MODE_UNKNOWN; // Shaders - ggml_vk_load_shaders(); + ggml_vk_load_shaders(ctx); // Queues - vk_device.compute_queue = ggml_vk_create_queue(compute_queue_family_index, 0, { vk::PipelineStageFlagBits::eComputeShader | vk::PipelineStageFlagBits::eTransfer }); - if (!single_queue) { + ggml_vk_create_queue(ctx, ctx->device.lock()->compute_queue, compute_queue_family_index, 0, { vk::PipelineStageFlagBits::eComputeShader | vk::PipelineStageFlagBits::eTransfer }); + if (!ctx->device.lock()->single_queue) { const uint32_t transfer_queue_index = compute_queue_family_index == transfer_queue_family_index ? 1 : 0; - vk_device.transfer_queue = ggml_vk_create_queue(transfer_queue_family_index, transfer_queue_index, { vk::PipelineStageFlagBits::eTransfer }); + ggml_vk_create_queue(ctx, ctx->device.lock()->transfer_queue, transfer_queue_family_index, transfer_queue_index, { vk::PipelineStageFlagBits::eTransfer }); } else { - vk_device.transfer_queue = vk_device.compute_queue; + // TODO: Use pointer or reference to avoid copy + ctx->device.lock()->transfer_queue = ctx->device.lock()->compute_queue; } - vk_fence = vk_device.device.createFence({}); + ctx->fence = ctx->device.lock()->device.createFence({}); - vk_ctx = nullptr; - vk_transfer_ctx = nullptr; + ctx->compute_ctx = nullptr; + ctx->transfer_ctx = nullptr; - vk_disable = false; + ctx->disable = false; + ctx->initialized = true; + + ctx->idx = idx; #ifdef GGML_VULKAN_CHECK_RESULTS const char* skip_checks = getenv("GGML_VULKAN_SKIP_CHECKS"); @@ -1137,7 +1302,7 @@ std::cerr << "ggml_vulkan: Validation layers enabled" << std::endl; #endif } -static vk_pipeline* ggml_vk_get_to_fp16(ggml_type type) { +static vk_pipeline* ggml_vk_get_to_fp16(ggml_backend_vk_context * ctx, ggml_type type) { #ifdef GGML_VULKAN_DEBUG std::cerr << "ggml_vk_get_to_fp16()" << std::endl; #endif @@ -1158,10 +1323,10 @@ static vk_pipeline* ggml_vk_get_to_fp16(ggml_type type) { return nullptr; } - return &vk_pipeline_dequant[type]; + return &ctx->pipeline_dequant[type]; } -static vk_pipeline* ggml_vk_get_dequantize_mul_mat_vec(ggml_type type) { +static vk_pipeline* ggml_vk_get_dequantize_mul_mat_vec(ggml_backend_vk_context * ctx, ggml_type type) { #ifdef GGML_VULKAN_DEBUG std::cerr << "ggml_vk_get_dequantize_mul_mat_vec()" << std::endl; #endif @@ -1182,15 +1347,10 @@ static vk_pipeline* ggml_vk_get_dequantize_mul_mat_vec(ggml_type type) { return nullptr; } - return &vk_pipeline_dequant_mul_mat_vec_f32[type]; + return &ctx->pipeline_dequant_mul_mat_vec_f32[type]; } -// buffer pool for vulkan -#define MAX_VK_BUFFERS 256 - -static vk_buffer g_vk_buffer_pool[MAX_VK_BUFFERS]; - -static vk_buffer ggml_vk_pool_malloc(size_t size) { +static vk_buffer ggml_vk_pool_malloc(ggml_backend_vk_context * ctx, size_t size) { #ifdef GGML_VULKAN_DEBUG std::cerr << "ggml_vk_pool_malloc(" << size << ")" << std::endl; #endif @@ -1199,98 +1359,95 @@ static vk_buffer ggml_vk_pool_malloc(size_t size) { int worst_i = -1; size_t worst_size = 0; //largest unused buffer seen so far for (int i = 0; i < MAX_VK_BUFFERS; ++i) { - vk_buffer &b = g_vk_buffer_pool[i]; - if (b.size > 0 && b.size >= size && b.size < best_size) { + vk_buffer &b = ctx->buffer_pool[i]; + if (b != nullptr && b->size >= size && b->size < best_size) { best_i = i; - best_size = b.size; + best_size = b->size; } - if (b.size > 0 && b.size > worst_size) { + if (b != nullptr && b->size > worst_size) { worst_i = i; - worst_size = b.size; + worst_size = b->size; } } if(best_i != -1) { //found the smallest buffer that fits our needs - vk_buffer b = g_vk_buffer_pool[best_i]; - g_vk_buffer_pool[best_i].size = 0; + vk_buffer b = ctx->buffer_pool[best_i]; + ctx->buffer_pool[best_i].reset(); return b; } if(worst_i != -1) { //no buffer that fits our needs, resize largest one to save memory - vk_buffer& b = g_vk_buffer_pool[worst_i]; + vk_buffer& b = ctx->buffer_pool[worst_i]; ggml_vk_destroy_buffer(b); } - return ggml_vk_create_buffer_check(size, vk::MemoryPropertyFlagBits::eDeviceLocal); + return ggml_vk_create_buffer_check(ctx, size, vk::MemoryPropertyFlagBits::eDeviceLocal); } -static void ggml_vk_pool_free(vk_buffer& buffer) { +static void ggml_vk_pool_free(ggml_backend_vk_context * ctx, vk_buffer& buffer) { #ifdef GGML_VULKAN_DEBUG - std::cerr << "ggml_vk_pool_free(" << buffer.size << ")" << std::endl; + std::cerr << "ggml_vk_pool_free(" << buffer->size << ")" << std::endl; #endif for (int i = 0; i < MAX_VK_BUFFERS; ++i) { - vk_buffer& b = g_vk_buffer_pool[i]; - if (b.size == 0) { + vk_buffer& b = ctx->buffer_pool[i]; + if (b == nullptr) { b = buffer; - // Set owning queue family index to ignored to avoid synchronization on next use - b.qf_owner = VK_QUEUE_FAMILY_IGNORED; return; } } - fprintf(stderr, "WARNING: vk buffer pool full, increase MAX_VK_BUFFERS\n"); + std::cerr << "ggml_vulkan: WARNING: vk buffer pool full, increase MAX_VK_BUFFERS" << std::endl; ggml_vk_destroy_buffer(buffer); } // Returns an available temporary buffer that may only be used temporarily, it will be reused -static vk_buffer ggml_vk_create_buffer_temp(size_t size) { +static vk_buffer ggml_vk_create_buffer_temp(ggml_backend_vk_context * ctx, size_t size) { // Try to find existing temp buffer with enough capacity - for (auto& buffer : vk_gc.temp_buffers) { - if (buffer.size >= size) { + for (auto& buffer : ctx->gc.temp_buffers) { + if (buffer->size >= size) { return buffer; } } // Otherwise create new buffer - vk_buffer buf = ggml_vk_pool_malloc(size); - vk_gc.temp_buffers.push_back(buf); + vk_buffer buf = ggml_vk_pool_malloc(ctx, size); + ctx->gc.temp_buffers.push_back(buf); return buf; } -static void * ggml_vk_host_malloc(size_t size) { +static void * ggml_vk_host_malloc(ggml_backend_vk_context * ctx, size_t size) { #ifdef GGML_VULKAN_DEBUG std::cerr << "ggml_vk_host_malloc(" << size << ")" << std::endl; #endif - vk_buffer buf = ggml_vk_create_buffer(size, vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent | vk::MemoryPropertyFlagBits::eHostCached); + vk_buffer buf = ggml_vk_create_buffer(ctx, size, vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent | vk::MemoryPropertyFlagBits::eHostCached); - if(!(buf.memory_property_flags & vk::MemoryPropertyFlagBits::eHostVisible)) { + if(!(buf->memory_property_flags & vk::MemoryPropertyFlagBits::eHostVisible)) { fprintf(stderr, "WARNING: failed to allocate %.2f MB of pinned memory\n", size/1024.0/1024.0); - buf.size = 0; - vk_device.device.freeMemory(buf.device_memory); - vk_device.device.destroyBuffer(buf.buffer); + ctx->device.lock()->device.freeMemory(buf->device_memory); + ctx->device.lock()->device.destroyBuffer(buf->buffer); return nullptr; } - vk_pinned_memory.push_back(std::make_tuple(buf.ptr, size, buf)); + ctx->pinned_memory.push_back(std::make_tuple(buf->ptr, size, buf)); - return buf.ptr; + return buf->ptr; } -static void ggml_vk_host_free(void* ptr) { +static void ggml_vk_host_free(ggml_backend_vk_context * ctx, void* ptr) { if (ptr == nullptr) { return; } #ifdef GGML_VULKAN_DEBUG std::cerr << "ggml_vk_host_free(" << ptr << ")" << std::endl; #endif - vk_buffer* buf = nullptr; + vk_buffer buf; size_t index; - for (size_t i = 0; i < vk_pinned_memory.size(); i++) { - const uint8_t* addr = (const uint8_t*) std::get<0>(vk_pinned_memory[i]); - const uint8_t* endr = addr + std::get<1>(vk_pinned_memory[i]); + for (size_t i = 0; i < ctx->pinned_memory.size(); i++) { + const uint8_t* addr = (const uint8_t*) std::get<0>(ctx->pinned_memory[i]); + const uint8_t* endr = addr + std::get<1>(ctx->pinned_memory[i]); if (ptr >= addr && ptr < endr) { - buf = &std::get<2>(vk_pinned_memory[i]); + buf = std::get<2>(ctx->pinned_memory[i]); index = i; break; } @@ -1300,28 +1457,28 @@ static void ggml_vk_host_free(void* ptr) { return; } - ggml_vk_destroy_buffer(*buf); + ggml_vk_destroy_buffer(buf); - vk_pinned_memory.erase(vk_pinned_memory.begin() + index); + ctx->pinned_memory.erase(ctx->pinned_memory.begin() + index); } -static void ggml_vk_host_get(const void * ptr, vk_buffer *& buf, size_t& buf_offset) { +static void ggml_vk_host_get(ggml_backend_vk_context * ctx, const void * ptr, vk_buffer& buf, size_t& buf_offset) { buf = nullptr; buf_offset = 0; - for (size_t i = 0; i < vk_pinned_memory.size(); i++) { - const uint8_t* addr = (const uint8_t*) std::get<0>(vk_pinned_memory[i]); - const uint8_t* endr = addr + std::get<1>(vk_pinned_memory[i]); + for (size_t i = 0; i < ctx->pinned_memory.size(); i++) { + const uint8_t* addr = (const uint8_t*) std::get<0>(ctx->pinned_memory[i]); + const uint8_t* endr = addr + std::get<1>(ctx->pinned_memory[i]); if (ptr >= addr && ptr < endr) { - buf = &std::get<2>(vk_pinned_memory[i]); + buf = std::get<2>(ctx->pinned_memory[i]); buf_offset = ((const uint8_t *)ptr) - addr; break; } } } -static vk_submission ggml_vk_begin_submission(vk_queue& q, bool one_time = true) { +static vk_submission ggml_vk_begin_submission(ggml_backend_vk_context * ctx, vk_queue& q, bool one_time = true) { vk_submission s; - s.buffer = ggml_vk_create_cmd_buffer(q); + s.buffer = ggml_vk_create_cmd_buffer(ctx, q); if (one_time) { s.buffer.begin({ vk::CommandBufferUsageFlagBits::eOneTimeSubmit }); } else { @@ -1331,7 +1488,7 @@ static vk_submission ggml_vk_begin_submission(vk_queue& q, bool one_time = true) return s; } -static void ggml_vk_dispatch_pipeline(vk_context * ctx, vk_pipeline& pipeline, std::vector&& buffers, size_t push_constant_size, const void* push_constants, std::array elements) { +static void ggml_vk_dispatch_pipeline(ggml_backend_vk_context * ctx, vk_context * subctx, vk_pipeline& pipeline, std::vector&& buffers, size_t push_constant_size, const void* push_constants, std::array elements) { const uint32_t wg0 = CEIL_DIV(elements[0], pipeline.wg_denoms[0]); const uint32_t wg1 = CEIL_DIV(elements[1], pipeline.wg_denoms[1]); const uint32_t wg2 = CEIL_DIV(elements[2], pipeline.wg_denoms[2]); @@ -1344,22 +1501,22 @@ static void ggml_vk_dispatch_pipeline(vk_context * ctx, vk_pipeline& pipeline, s GGML_ASSERT(buffers.size() == pipeline.parameter_count); vk::DescriptorSet& descriptor_set = pipeline.descriptor_sets[pipeline.descriptor_set_idx++]; for (uint32_t i = 0; i < pipeline.parameter_count; i++) { - descriptor_buffer_infos.push_back({buffers[i].buffer.buffer, buffers[i].offset, buffers[i].size}); + descriptor_buffer_infos.push_back({buffers[i].buffer->buffer, buffers[i].offset, buffers[i].size}); } for (uint32_t i = 0; i < pipeline.parameter_count; i++) { write_descriptor_sets.push_back({descriptor_set, i, 0, 1, vk::DescriptorType::eStorageBuffer, nullptr, &descriptor_buffer_infos[i]}); } - vk_device.device.updateDescriptorSets(write_descriptor_sets, {}); + ctx->device.lock()->device.updateDescriptorSets(write_descriptor_sets, {}); - ctx->s->buffer.pushConstants(pipeline.layout, vk::ShaderStageFlagBits::eCompute, 0, push_constant_size, push_constants); - ctx->s->buffer.bindPipeline(vk::PipelineBindPoint::eCompute, pipeline.pipeline); - ctx->s->buffer.bindDescriptorSets(vk::PipelineBindPoint::eCompute, + subctx->s->buffer.pushConstants(pipeline.layout, vk::ShaderStageFlagBits::eCompute, 0, push_constant_size, push_constants); + subctx->s->buffer.bindPipeline(vk::PipelineBindPoint::eCompute, pipeline.pipeline); + subctx->s->buffer.bindDescriptorSets(vk::PipelineBindPoint::eCompute, pipeline.layout, 0, { descriptor_set }, {}); - ctx->s->buffer.dispatch(wg0, wg1, wg2); + subctx->s->buffer.dispatch(wg0, wg1, wg2); } static void ggml_vk_end_submission(vk_submission& s, std::vector wait_semaphores, std::vector signal_semaphores) { @@ -1381,16 +1538,16 @@ static void ggml_vk_ctx_end(vk_context * ctx) { ctx->s = nullptr; } -static void ggml_vk_ctx_begin(vk_context * ctx) { +static void ggml_vk_ctx_begin(ggml_backend_vk_context * ctx, vk_context * subctx) { #ifdef GGML_VULKAN_DEBUG std::cerr << "ggml_vk_ctx_begin(" << ctx << ")" << std::endl; #endif - if (ctx->s != nullptr) { - ggml_vk_ctx_end(ctx); + if (subctx->s != nullptr) { + ggml_vk_ctx_end(subctx); } - ctx->seqs.push_back({ ggml_vk_begin_submission(*ctx->q) }); - ctx->s = ctx->seqs[ctx->seqs.size() - 1].data(); + subctx->seqs.push_back({ ggml_vk_begin_submission(ctx, *subctx->q) }); + subctx->s = subctx->seqs[subctx->seqs.size() - 1].data(); } static size_t ggml_vk_align_size(size_t width, size_t align) { @@ -1405,14 +1562,14 @@ static void deferred_memcpy(void * dst, const void * src, size_t size, std::vect } } -static void ensure_sync_staging_buffer(size_t size) { - if (vk_sync_staging.size < size) { - ggml_vk_destroy_buffer(vk_sync_staging); - vk_sync_staging = ggml_vk_create_buffer_check(size, vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent | vk::MemoryPropertyFlagBits::eHostCached); +static void ggml_vk_ensure_sync_staging_buffer(ggml_backend_vk_context * ctx, size_t size) { + if (ctx->sync_staging == nullptr || ctx->sync_staging->size < size) { + ggml_vk_destroy_buffer(ctx->sync_staging); + ctx->sync_staging = ggml_vk_create_buffer_check(ctx, size, vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent | vk::MemoryPropertyFlagBits::eHostCached); } } -static void ggml_vk_buffer_write_nc_async(vk_context * ctx, vk_buffer* dst, size_t offset, const ggml_tensor * tensor, bool sync_staging = false) { +static void ggml_vk_buffer_write_nc_async(ggml_backend_vk_context * ctx, vk_context * subctx, vk_buffer& dst, size_t offset, const ggml_tensor * tensor, bool sync_staging = false) { #ifdef GGML_VULKAN_DEBUG std::cerr << "ggml_vk_buffer_write_nc_async(" << tensor << ")" << std::endl; #endif @@ -1423,9 +1580,9 @@ static void ggml_vk_buffer_write_nc_async(vk_context * ctx, vk_buffer* dst, size GGML_ASSERT(false); } // Check if src is pinned memory - vk_buffer * buf = nullptr; + vk_buffer buf; size_t buf_offset; - ggml_vk_host_get(tensor->data, buf, buf_offset); + ggml_vk_host_get(ctx, tensor->data, buf, buf_offset); const uint64_t ne0 = tensor->ne[0]; const uint64_t ne1 = tensor->ne[1]; @@ -1471,21 +1628,21 @@ static void ggml_vk_buffer_write_nc_async(vk_context * ctx, vk_buffer* dst, size } } - ggml_vk_sync_buffers(ctx); - ctx->s->buffer.copyBuffer(buf->buffer, dst->buffer, slices); + ggml_vk_sync_buffers(subctx); + subctx->s->buffer.copyBuffer(buf->buffer, dst->buffer, slices); return; } // Staging buffer required - vk_buffer * staging = &vk_staging; - size_t staging_offset = vk_staging_offset; + vk_buffer staging = ctx->staging; + size_t staging_offset = ctx->staging_offset; const size_t copy_size = ts*ne/bs; - if (vk_staging.size < vk_staging_offset + copy_size) { + if (ctx->staging->size < ctx->staging_offset + copy_size) { if (sync_staging) { // Create temporary larger buffer - ensure_sync_staging_buffer(copy_size); + ggml_vk_ensure_sync_staging_buffer(ctx, copy_size); - staging = &vk_sync_staging; + staging = ctx->sync_staging; staging_offset = 0; } else { GGML_ASSERT(false); @@ -1494,23 +1651,23 @@ static void ggml_vk_buffer_write_nc_async(vk_context * ctx, vk_buffer* dst, size VkBufferCopy buf_copy{ staging_offset, offset, copy_size }; - ggml_vk_sync_buffers(ctx); - vkCmdCopyBuffer(ctx->s->buffer, staging->buffer, dst->buffer, 1, &buf_copy); + ggml_vk_sync_buffers(subctx); + vkCmdCopyBuffer(subctx->s->buffer, staging->buffer, dst->buffer, 1, &buf_copy); for (uint64_t i3 = 0; i3 < ne3; i3++) { for (uint64_t i2 = 0; i2 < ne2; i2++) { // Find longest contiguous slice if (ne1*nb1 == dstnb2) { - deferred_memcpy((uint8_t *)staging->ptr + staging_offset + i3*dstnb3 + i2*dstnb2, (const uint8_t *) tensor->data + buf_offset + i3*nb3 + i2*nb2, dstnb2, &ctx->in_memcpys); + deferred_memcpy((uint8_t *)staging->ptr + staging_offset + i3*dstnb3 + i2*dstnb2, (const uint8_t *) tensor->data + buf_offset + i3*nb3 + i2*nb2, dstnb2, &subctx->in_memcpys); } else { for (uint64_t i1 = 0; i1 < ne1; i1++) { if (ne0*nb0/bs == dstnb1) { - deferred_memcpy((uint8_t *)staging->ptr + staging_offset + i3*dstnb3 + i2*dstnb2 + i1*dstnb1, (const uint8_t *) tensor->data + buf_offset + i3*nb3 + i2*nb2 + i1*nb1, dstnb1, &ctx->in_memcpys); + deferred_memcpy((uint8_t *)staging->ptr + staging_offset + i3*dstnb3 + i2*dstnb2 + i1*dstnb1, (const uint8_t *) tensor->data + buf_offset + i3*nb3 + i2*nb2 + i1*nb1, dstnb1, &subctx->in_memcpys); } else { const uint64_t s_off = buf_offset + i3*nb3 + i2*nb2 + i1*nb1; const uint64_t d_off = staging_offset + i3*dstnb3 + i2*dstnb2 + i1*dstnb1; for (uint64_t i0 = 0; i0 < ne0; i0++) { - deferred_memcpy((uint8_t *)staging->ptr + d_off + i0*dstnb0, (const uint8_t *) tensor->data + s_off + i0*nb0, dstnb0, &ctx->in_memcpys); + deferred_memcpy((uint8_t *)staging->ptr + d_off + i0*dstnb0, (const uint8_t *) tensor->data + s_off + i0*nb0, dstnb0, &subctx->in_memcpys); } } } @@ -1519,19 +1676,22 @@ static void ggml_vk_buffer_write_nc_async(vk_context * ctx, vk_buffer* dst, size } } -static void ggml_vk_buffer_write_2d_async(vk_context * ctx, vk_buffer* dst, size_t offset, const void * src, size_t spitch, size_t width, size_t height, bool sync_staging = false) { +static void ggml_vk_buffer_write_2d_async(ggml_backend_vk_context * ctx, vk_context * subctx, vk_buffer& dst, size_t offset, const void * src, size_t spitch, size_t width, size_t height, bool sync_staging = false) { #ifdef GGML_VULKAN_DEBUG std::cerr << "ggml_vk_buffer_write_2d_async(" << width << ", " << height << ")" << std::endl; #endif + // Make sure ctx owns the buffer + GGML_ASSERT(dst->ctx == ctx); + // Buffer is already mapped if(dst->memory_property_flags & vk::MemoryPropertyFlagBits::eHostVisible) { std::cerr << "ggml_vulkan: buffer_write_async dst buffer is host_visible. Use synchronous write." << std::endl; GGML_ASSERT(false); } // Check if src is pinned memory - vk_buffer * buf = nullptr; + vk_buffer buf = nullptr; size_t buf_offset; - ggml_vk_host_get(src, buf, buf_offset); + ggml_vk_host_get(ctx, src, buf, buf_offset); if (buf != nullptr) { // Memory is pinned, use as staging buffer @@ -1550,8 +1710,8 @@ static void ggml_vk_buffer_write_2d_async(vk_context * ctx, vk_buffer* dst, size } } - ggml_vk_sync_buffers(ctx); - ctx->s->buffer.copyBuffer(buf->buffer, dst->buffer, slices); + ggml_vk_sync_buffers(subctx); + subctx->s->buffer.copyBuffer(buf->buffer, dst->buffer, slices); return; } #ifdef GGML_VULKAN_DEBUG @@ -1559,14 +1719,14 @@ static void ggml_vk_buffer_write_2d_async(vk_context * ctx, vk_buffer* dst, size #endif // Staging buffer required - vk_buffer * staging = &vk_staging; - size_t staging_offset = vk_staging_offset; + vk_buffer staging = ctx->staging; + size_t staging_offset = ctx->staging_offset; const size_t copy_size = width*height; - if (vk_staging.size < vk_staging_offset + copy_size) { + if (ctx->staging == nullptr || ctx->staging->size < ctx->staging_offset + copy_size) { if (sync_staging) { - ensure_sync_staging_buffer(copy_size); + ggml_vk_ensure_sync_staging_buffer(ctx, copy_size); - staging = &vk_sync_staging; + staging = ctx->sync_staging; staging_offset = 0; } else { GGML_ASSERT(false); @@ -1578,26 +1738,26 @@ static void ggml_vk_buffer_write_2d_async(vk_context * ctx, vk_buffer* dst, size offset, copy_size}; - ggml_vk_sync_buffers(ctx); - vkCmdCopyBuffer(ctx->s->buffer, staging->buffer, dst->buffer, 1, &buf_copy); + ggml_vk_sync_buffers(subctx); + vkCmdCopyBuffer(subctx->s->buffer, staging->buffer, dst->buffer, 1, &buf_copy); if (width == spitch) { - deferred_memcpy((uint8_t *)staging->ptr + staging_offset, src, width * height, &ctx->in_memcpys); + deferred_memcpy((uint8_t *)staging->ptr + staging_offset, src, width * height, &subctx->in_memcpys); } else { for (size_t i = 0; i < height; i++) { - deferred_memcpy((uint8_t *)staging->ptr + staging_offset + i * width, (const uint8_t *) src + i * spitch, width, &ctx->in_memcpys); + deferred_memcpy((uint8_t *)staging->ptr + staging_offset + i * width, (const uint8_t *) src + i * spitch, width, &subctx->in_memcpys); } } } -static void ggml_vk_buffer_write_async(vk_context * ctx, vk_buffer* dst, size_t offset, const void * src, size_t size, bool sync_staging = false) { +static void ggml_vk_buffer_write_async(ggml_backend_vk_context * ctx, vk_context * subctx, vk_buffer& dst, size_t offset, const void * src, size_t size, bool sync_staging = false) { #ifdef GGML_VULKAN_DEBUG std::cerr << "ggml_vk_buffer_write_async(" << size << ")" << std::endl; #endif - return ggml_vk_buffer_write_2d_async(ctx, dst, offset, src, size, size, 1, sync_staging); + return ggml_vk_buffer_write_2d_async(ctx, subctx, dst, offset, src, size, size, 1, sync_staging); } -static void ggml_vk_buffer_write_2d(vk_buffer* dst, size_t offset, const void * src, size_t spitch, size_t width, size_t height) { +static void ggml_vk_buffer_write_2d(ggml_backend_vk_context * ctx, vk_buffer& dst, size_t offset, const void * src, size_t spitch, size_t width, size_t height) { #ifdef GGML_VULKAN_DEBUG std::cerr << "ggml_vk_buffer_write_2d(" << width << ", " << height << ")" << std::endl; #endif @@ -1609,39 +1769,42 @@ static void ggml_vk_buffer_write_2d(vk_buffer* dst, size_t offset, const void * memcpy((uint8_t *)dst->ptr + offset + i * width, (const uint8_t *) src + i * spitch, width); } } else { - vk_context * ctx = ggml_vk_create_context(vk_device.transfer_queue); - ggml_vk_ctx_begin(ctx); - ggml_vk_buffer_write_2d_async(ctx, dst, offset, src, spitch, width, height, true); - ggml_vk_ctx_end(ctx); + vk_context * subctx = ggml_vk_create_context(ctx, ctx->device.lock()->transfer_queue); + ggml_vk_ctx_begin(ctx, subctx); + ggml_vk_buffer_write_2d_async(ctx, subctx, dst, offset, src, spitch, width, height, true); + ggml_vk_ctx_end(subctx); - for (auto& cpy : ctx->in_memcpys) { + for (auto& cpy : subctx->in_memcpys) { memcpy(cpy.dst, cpy.src, cpy.n); } - ggml_vk_submit(ctx, vk_fence); - VK_CHECK(vk_device.device.waitForFences({ vk_fence }, true, UINT64_MAX), "vk_buffer_write_2d waitForFences"); - vk_device.device.resetFences({ vk_fence }); + ggml_vk_submit(subctx, ctx->fence); + VK_CHECK(ctx->device.lock()->device.waitForFences({ ctx->fence }, true, UINT64_MAX), "vk_buffer_write_2d waitForFences"); + ctx->device.lock()->device.resetFences({ ctx->fence }); } } -static void ggml_vk_buffer_write(vk_buffer* dst, size_t offset, const void * src, size_t size) { +static void ggml_vk_buffer_write(ggml_backend_vk_context * ctx, vk_buffer& dst, size_t offset, const void * src, size_t size) { #ifdef GGML_VULKAN_DEBUG std::cerr << "ggml_vk_buffer_write(" << size << ")" << std::endl; #endif - ggml_vk_buffer_write_2d(dst, offset, src, 0, size, 1); + ggml_vk_buffer_write_2d(ctx, dst, offset, src, 0, size, 1); } -static void ggml_vk_buffer_read_2d_async(vk_context * ctx, vk_buffer* src, size_t offset, void * dst, size_t spitch, size_t dpitch, size_t width, size_t height, bool sync_staging = false) { +static void ggml_vk_buffer_read_2d_async(ggml_backend_vk_context * ctx, vk_context * subctx, vk_buffer& src, size_t offset, void * dst, size_t spitch, size_t dpitch, size_t width, size_t height, bool sync_staging = false) { #ifdef GGML_VULKAN_DEBUG std::cerr << "ggml_vk_buffer_read_2d_async(offset=" << offset << ", width=" << width << ", height=" << height << ")" << std::endl; #endif GGML_ASSERT(width > 0); GGML_ASSERT(height > 0); - GGML_ASSERT(src->size > 0); + GGML_ASSERT(src != nullptr); + // Make sure ctx owns the buffer + GGML_ASSERT(src->ctx == ctx); + // Check if dst is pinned memory - vk_buffer * buf = nullptr; + vk_buffer buf = nullptr; size_t buf_offset; - ggml_vk_host_get(dst, buf, buf_offset); + ggml_vk_host_get(ctx, dst, buf, buf_offset); std::vector slices(1); if (width == spitch && width == dpitch) { @@ -1660,8 +1823,8 @@ static void ggml_vk_buffer_read_2d_async(vk_context * ctx, vk_buffer* src, size_ if (buf != nullptr) { // Memory is pinned, use as staging buffer - ggml_vk_sync_buffers(ctx); - ctx->s->buffer.copyBuffer(src->buffer, buf->buffer, slices); + ggml_vk_sync_buffers(subctx); + subctx->s->buffer.copyBuffer(src->buffer, buf->buffer, slices); return; } @@ -1670,30 +1833,30 @@ static void ggml_vk_buffer_read_2d_async(vk_context * ctx, vk_buffer* src, size_ #endif // Fall back to staging buffer - vk_buffer * staging = &vk_staging; + vk_buffer staging = ctx->staging; const size_t copy_size = dpitch * height; - if (vk_staging.size < vk_staging_offset + copy_size) { + if (ctx->staging == nullptr || ctx->staging->size < ctx->staging_offset + copy_size) { if (sync_staging) { // Create temporary larger buffer - ensure_sync_staging_buffer(copy_size); + ggml_vk_ensure_sync_staging_buffer(ctx, copy_size); - staging = &vk_sync_staging; + staging = ctx->sync_staging; } else { GGML_ASSERT(false); } } - ggml_vk_sync_buffers(ctx); - ctx->s->buffer.copyBuffer(src->buffer, staging->buffer, slices); + ggml_vk_sync_buffers(subctx); + subctx->s->buffer.copyBuffer(src->buffer, staging->buffer, slices); - deferred_memcpy(dst, staging->ptr, copy_size, &ctx->out_memcpys); + deferred_memcpy(dst, staging->ptr, copy_size, &subctx->out_memcpys); } -static void ggml_vk_buffer_read_async(vk_context * ctx, vk_buffer* src, size_t offset, void * dst, size_t size, bool sync_staging = false) { - return ggml_vk_buffer_read_2d_async(ctx, src, offset, dst, size, size, size, 1, sync_staging); +static void ggml_vk_buffer_read_async(ggml_backend_vk_context * ctx, vk_context * subctx, vk_buffer& src, size_t offset, void * dst, size_t size, bool sync_staging = false) { + return ggml_vk_buffer_read_2d_async(ctx, subctx, src, offset, dst, size, size, size, 1, sync_staging); } -static void ggml_vk_buffer_read(vk_buffer* src, size_t offset, void * dst, size_t size) { +static void ggml_vk_buffer_read(ggml_backend_vk_context * ctx, vk_buffer& src, size_t offset, void * dst, size_t size) { #ifdef GGML_VULKAN_DEBUG std::cerr << "ggml_vk_buffer_read(" << offset << ", " << size << ")" << std::endl; #endif @@ -1702,61 +1865,88 @@ static void ggml_vk_buffer_read(vk_buffer* src, size_t offset, void * dst, size_ memcpy(dst, (uint8_t *) src->ptr + offset, size); } else { - vk_context * ctx = ggml_vk_create_context(vk_device.transfer_queue); - ggml_vk_ctx_begin(ctx); - ggml_vk_buffer_read_async(ctx, src, offset, dst, size, true); - ggml_vk_ctx_end(ctx); + vk_context * subctx = ggml_vk_create_context(ctx, ctx->device.lock()->transfer_queue); + ggml_vk_ctx_begin(ctx, subctx); + ggml_vk_buffer_read_async(ctx, subctx, src, offset, dst, size, true); + ggml_vk_ctx_end(subctx); - ggml_vk_submit(ctx, vk_fence); - VK_CHECK(vk_device.device.waitForFences({ vk_fence }, true, UINT64_MAX), "vk_buffer_read waitForFences"); - vk_device.device.resetFences({ vk_fence }); + ggml_vk_submit(subctx, ctx->fence); + VK_CHECK(ctx->device.lock()->device.waitForFences({ ctx->fence }, true, UINT64_MAX), "vk_buffer_read waitForFences"); + ctx->device.lock()->device.resetFences({ ctx->fence }); - for (auto& cpy : ctx->out_memcpys) { + for (auto& cpy : subctx->out_memcpys) { memcpy(cpy.dst, cpy.src, cpy.n); } } } -static void ggml_vk_buffer_copy_async(vk_context * ctx, vk_buffer * dst, size_t dst_offset, vk_buffer * src, size_t src_offset, size_t size) { +static void ggml_vk_buffer_copy_async(vk_context * ctx, vk_buffer& dst, size_t dst_offset, vk_buffer& src, size_t src_offset, size_t size) { #ifdef GGML_VULKAN_DEBUG std::cerr << "ggml_vk_buffer_copy_async(" << size << ")" << std::endl; #endif + // Make sure both buffers are on same ctx + GGML_ASSERT(src->ctx == dst->ctx); + VkBufferCopy bc{ src_offset, dst_offset, size }; vkCmdCopyBuffer(ctx->s->buffer, src->buffer, dst->buffer, 1, &bc); } -static void ggml_vk_buffer_copy(vk_buffer * dst, size_t dst_offset, vk_buffer * src, size_t src_offset, size_t size) { +static void ggml_vk_buffer_copy(vk_buffer& dst, size_t dst_offset, vk_buffer& src, size_t src_offset, size_t size) { + if (src->ctx == dst->ctx) { #ifdef GGML_VULKAN_DEBUG - std::cerr << "ggml_vk_buffer_copy(" << size << ")" << std::endl; + std::cerr << "ggml_vk_buffer_copy(SINGLE_DEVICE, " << size << ")" << std::endl; #endif - VkBufferCopy bc{ src_offset, dst_offset, size }; + // Copy within the device + ggml_backend_vk_context * ctx = src->ctx; - vk_context * ctx = ggml_vk_create_context(vk_device.transfer_queue); - ggml_vk_ctx_begin(ctx); - vkCmdCopyBuffer(ctx->s->buffer, src->buffer, dst->buffer, 1, &bc); - ggml_vk_buffer_copy_async(ctx, dst, dst_offset, src, src_offset, size); - ggml_vk_ctx_end(ctx); - ggml_vk_submit(ctx, vk_fence); - VK_CHECK(vk_device.device.waitForFences({ vk_fence }, true, UINT64_MAX), "vk_buffer_copy waitForFences"); - vk_device.device.resetFences({ vk_fence }); + VkBufferCopy bc{ src_offset, dst_offset, size }; + + vk_context * subctx = ggml_vk_create_context(ctx, ctx->device.lock()->transfer_queue); + ggml_vk_ctx_begin(ctx, subctx); + ggml_vk_buffer_copy_async(subctx, dst, dst_offset, src, src_offset, size); + ggml_vk_ctx_end(subctx); + ggml_vk_submit(subctx, ctx->fence); + VK_CHECK(ctx->device.lock()->device.waitForFences({ ctx->fence }, true, UINT64_MAX), "vk_buffer_copy waitForFences"); + ctx->device.lock()->device.resetFences({ ctx->fence }); + } else { +#ifdef GGML_VULKAN_DEBUG + std::cerr << "ggml_vk_buffer_copy(MULTI_DEVICE, " << size << ")" << std::endl; +#endif + // Copy device to device + ggml_backend_vk_context * src_ctx = src->ctx; + ggml_backend_vk_context * dst_ctx = dst->ctx; + + ggml_vk_ensure_sync_staging_buffer(src_ctx, size); + ggml_vk_ensure_sync_staging_buffer(dst_ctx, size); + + // Copy to src staging buffer + ggml_vk_buffer_copy(src_ctx->sync_staging, 0, src, src_offset, size); + // memcpy to dst staging buffer + memcpy(dst_ctx->sync_staging->ptr, src_ctx->sync_staging->ptr, size); + // Copy to dst buffer + ggml_vk_buffer_copy(dst, dst_offset, dst_ctx->sync_staging, 0, size); + } } -static void ggml_vk_buffer_memset(vk_buffer* dst, size_t offset, uint32_t c, size_t size) { +static void ggml_vk_buffer_memset(ggml_backend_vk_context * ctx, vk_buffer& dst, size_t offset, uint32_t c, size_t size) { #ifdef GGML_VULKAN_DEBUG std::cerr << "ggml_vk_buffer_memset(" << offset << ", " << c << ", " << size << ")" << std::endl; #endif - vk_context * ctx = ggml_vk_create_context(vk_device.transfer_queue); - ggml_vk_ctx_begin(ctx); - ctx->s->buffer.fillBuffer(dst->buffer, offset, size, c); - ggml_vk_ctx_end(ctx); + // Make sure ctx owns the buffer + GGML_ASSERT(dst->ctx == ctx); - ggml_vk_submit(ctx, vk_fence); - VK_CHECK(vk_device.device.waitForFences({ vk_fence }, true, UINT64_MAX), "vk_memset waitForFences"); - vk_device.device.resetFences({ vk_fence }); + vk_context * subctx = ggml_vk_create_context(ctx, ctx->device.lock()->transfer_queue); + ggml_vk_ctx_begin(ctx, subctx); + subctx->s->buffer.fillBuffer(dst->buffer, offset, size, c); + ggml_vk_ctx_end(subctx); + + ggml_vk_submit(subctx, ctx->fence); + VK_CHECK(ctx->device.lock()->device.waitForFences({ ctx->fence }, true, UINT64_MAX), "vk_memset waitForFences"); + ctx->device.lock()->device.resetFences({ ctx->fence }); } -static void ggml_vk_h2d_tensor_2d(vk_context * ctx, vk_buffer * dst, size_t offset, const ggml_tensor * src, uint64_t i3, uint64_t i2, uint64_t i1) { +static void ggml_vk_h2d_tensor_2d(ggml_backend_vk_context * ctx, vk_context * subctx, vk_buffer& dst, size_t offset, const ggml_tensor * src, uint64_t i3, uint64_t i2, uint64_t i1) { #ifdef GGML_VULKAN_DEBUG std::cerr << "ggml_vk_h2d_tensor_2d(dst=" << dst << ", offset=" << offset << ", src=" << src << ", i3=" << i3 << ", i2=" << i2 << ", i1=" << i1 << ")" << std::endl; #endif @@ -1773,20 +1963,20 @@ static void ggml_vk_h2d_tensor_2d(vk_context * ctx, vk_buffer * dst, size_t offs const void * x = (const void *) ((const char *) src->data + i2*nb2 + i3*nb3); if (nb0 == ts && nb1 == row_length) { - return ggml_vk_buffer_write_async(ctx, dst, offset, x, i1*nb1); + return ggml_vk_buffer_write_async(ctx, subctx, dst, offset, x, i1*nb1); } if (nb0 == ts && (i1 == ne1 || !ggml_is_permuted(src))) { - return ggml_vk_buffer_write_2d_async(ctx, dst, offset, x, nb1, row_length, i1); + return ggml_vk_buffer_write_2d_async(ctx, subctx, dst, offset, x, nb1, row_length, i1); } GGML_ASSERT(i3 == 0); GGML_ASSERT(i2 == 0); GGML_ASSERT(i1 == (uint64_t) ggml_nrows(src)); - return ggml_vk_buffer_write_nc_async(ctx, dst, offset, src); + return ggml_vk_buffer_write_nc_async(ctx, subctx, dst, offset, src); } -static void ggml_vk_d2h_tensor_2d(vk_context * ctx, vk_buffer * src, size_t offset, const ggml_tensor * dst) { +static void ggml_vk_d2h_tensor_2d(ggml_backend_vk_context * ctx, vk_context * subctx, vk_buffer& src, size_t offset, const ggml_tensor * dst) { #ifdef GGML_VULKAN_DEBUG std::cerr << "ggml_vk_d2h_tensor_2d()" << std::endl; #endif @@ -1804,10 +1994,10 @@ static void ggml_vk_d2h_tensor_2d(vk_context * ctx, vk_buffer * src, size_t offs const size_t row_length = ts*ne0/bs; if (ggml_is_contiguous(dst)) { - return ggml_vk_buffer_read_async(ctx, src, offset, dst->data, ne1*nb1*ne2*ne3); + return ggml_vk_buffer_read_async(ctx, subctx, src, offset, dst->data, ne1*nb1*ne2*ne3); } if (nb0 == ts) { - return ggml_vk_buffer_read_2d_async(ctx, src, offset, dst->data, nb1, nb1, row_length, ne1*ne2*ne3); + return ggml_vk_buffer_read_2d_async(ctx, subctx, src, offset, dst->data, nb1, nb1, row_length, ne1*ne2*ne3); } GGML_ASSERT(false); } @@ -1829,89 +2019,89 @@ static uint32_t ggml_vk_guess_split_k(int m, int n, int k) { return 1; } -static uint32_t ggml_vk_guess_matmul_pipeline_align(int m, int n) { +static uint32_t ggml_vk_guess_matmul_pipeline_align(ggml_backend_vk_context * ctx, int m, int n) { #ifdef GGML_VULKAN_DEBUG std::cerr << "ggml_vk_guess_matmul_pipeline_align(" << m << ", " << n << ")" << std::endl; #endif if (m <= 32 || n <= 32) { - return vk_pipeline_matmul_f32_aligned_s.align; + return ctx->pipeline_matmul_f32_aligned_s.align; } - if (vk_device.subgroup_size == 64 || m <= 64 || n <= 64) { - return vk_pipeline_matmul_f32_aligned_m.align; + if (ctx->device.lock()->subgroup_size == 64 || m <= 64 || n <= 64) { + return ctx->pipeline_matmul_f32_aligned_m.align; } - return vk_pipeline_matmul_f32_aligned_l.align; + return ctx->pipeline_matmul_f32_aligned_l.align; } -static vk_pipeline* ggml_vk_guess_matmul_pipeline(bool bit16_x, bool bit16_y, int m, int n, bool aligned) { +static vk_pipeline* ggml_vk_guess_matmul_pipeline(ggml_backend_vk_context * ctx, bool bit16_x, bool bit16_y, int m, int n, bool aligned) { #ifdef GGML_VULKAN_DEBUG std::cerr << "ggml_vk_guess_matmul_pipeline(" << bit16_x << ", " << bit16_y << ", " << m << ", " << n << ", " << aligned << ")"; #endif if (bit16_x && bit16_y) { - if (vk_device.vendor_id == VK_VENDOR_ID_INTEL || m <= 32 || n <= 32) { + if (ctx->device.lock()->vendor_id == VK_VENDOR_ID_INTEL || m <= 32 || n <= 32) { #ifdef GGML_VULKAN_DEBUG std::cerr << " S" << std::endl; #endif - return aligned ? &vk_pipeline_matmul_f16_aligned_s : &vk_pipeline_matmul_f16_s; + return aligned ? &ctx->pipeline_matmul_f16_aligned_s : &ctx->pipeline_matmul_f16_s; } - if (vk_device.subgroup_size == 64 || m <= 64 || n <= 64) { + if (ctx->device.lock()->subgroup_size == 64 || m <= 64 || n <= 64) { #ifdef GGML_VULKAN_DEBUG std::cerr << " M" << std::endl; #endif - return aligned ? &vk_pipeline_matmul_f16_aligned_m : &vk_pipeline_matmul_f16_m; + return aligned ? &ctx->pipeline_matmul_f16_aligned_m : &ctx->pipeline_matmul_f16_m; } #ifdef GGML_VULKAN_DEBUG std::cerr << " L" << std::endl; #endif - return aligned ? &vk_pipeline_matmul_f16_aligned_l : &vk_pipeline_matmul_f16_l; + return aligned ? &ctx->pipeline_matmul_f16_aligned_l : &ctx->pipeline_matmul_f16_l; } if (bit16_x && !bit16_y) { - if (vk_device.vendor_id == VK_VENDOR_ID_INTEL || m <= 32 || n <= 32) { + if (ctx->device.lock()->vendor_id == VK_VENDOR_ID_INTEL || m <= 32 || n <= 32) { #ifdef GGML_VULKAN_DEBUG std::cerr << " S" << std::endl; #endif - return aligned ? &vk_pipeline_matmul_f16_f32_aligned_s : &vk_pipeline_matmul_f16_f32_s; + return aligned ? &ctx->pipeline_matmul_f16_f32_aligned_s : &ctx->pipeline_matmul_f16_f32_s; } - if (vk_device.subgroup_size == 64 || m <= 64 || n <= 64) { + if (ctx->device.lock()->subgroup_size == 64 || m <= 64 || n <= 64) { #ifdef GGML_VULKAN_DEBUG std::cerr << " M" << std::endl; #endif - return aligned ? &vk_pipeline_matmul_f16_f32_aligned_m : &vk_pipeline_matmul_f16_f32_m; + return aligned ? &ctx->pipeline_matmul_f16_f32_aligned_m : &ctx->pipeline_matmul_f16_f32_m; } #ifdef GGML_VULKAN_DEBUG std::cerr << " L" << std::endl; #endif - return aligned ? &vk_pipeline_matmul_f16_f32_aligned_l : &vk_pipeline_matmul_f16_f32_l; + return aligned ? &ctx->pipeline_matmul_f16_f32_aligned_l : &ctx->pipeline_matmul_f16_f32_l; } if (!bit16_x && bit16_y) { GGML_ASSERT(false); } - if (vk_device.vendor_id == VK_VENDOR_ID_INTEL || m <= 32 || n <= 32) { + if (ctx->device.lock()->vendor_id == VK_VENDOR_ID_INTEL || m <= 32 || n <= 32) { #ifdef GGML_VULKAN_DEBUG std::cerr << " S" << std::endl; #endif - return aligned ? &vk_pipeline_matmul_f32_aligned_s : &vk_pipeline_matmul_f32_s; + return aligned ? &ctx->pipeline_matmul_f32_aligned_s : &ctx->pipeline_matmul_f32_s; } - if (vk_device.subgroup_size == 64 || m <= 64 || n <= 64) { + if (ctx->device.lock()->subgroup_size == 64 || m <= 64 || n <= 64) { #ifdef GGML_VULKAN_DEBUG std::cerr << " M" << std::endl; #endif - return aligned ? &vk_pipeline_matmul_f32_aligned_m : &vk_pipeline_matmul_f32_m; + return aligned ? &ctx->pipeline_matmul_f32_aligned_m : &ctx->pipeline_matmul_f32_m; } #ifdef GGML_VULKAN_DEBUG std::cerr << " L" << std::endl; #endif - return aligned ? &vk_pipeline_matmul_f32_aligned_l : &vk_pipeline_matmul_f32_l; + return aligned ? &ctx->pipeline_matmul_f32_aligned_l : &ctx->pipeline_matmul_f32_l; } -static void ggml_vk_matmul(vk_context * ctx, vk_pipeline& pipeline, vk_subbuffer&& a, vk_subbuffer&& b, vk_subbuffer&& d, vk_subbuffer&& split_k_buffer, uint32_t m, uint32_t n, uint32_t k, uint32_t stride_a, uint32_t stride_b, uint32_t stride_d, uint32_t split_k, uint32_t batch, uint32_t ne02, uint32_t ne12, uint32_t broadcast2, uint32_t broadcast3, uint32_t batch_stride_a, uint32_t batch_stride_b, uint32_t batch_stride_d) { +static void ggml_vk_matmul(ggml_backend_vk_context * ctx, vk_context * subctx, vk_pipeline& pipeline, vk_subbuffer&& a, vk_subbuffer&& b, vk_subbuffer&& d, vk_subbuffer&& split_k_buffer, uint32_t m, uint32_t n, uint32_t k, uint32_t stride_a, uint32_t stride_b, uint32_t stride_d, uint32_t split_k, uint32_t batch, uint32_t ne02, uint32_t ne12, uint32_t broadcast2, uint32_t broadcast3, uint32_t batch_stride_a, uint32_t batch_stride_b, uint32_t batch_stride_d) { #ifdef GGML_VULKAN_DEBUG - std::cerr << "ggml_vk_matmul(a: (" << a.buffer.buffer << ", " << a.offset << ", " << a.size << "), b: (" << b.buffer.buffer << ", " << b.offset << ", " << b.size << "), c: (" << d.buffer.buffer << ", " << d.offset << ", " << d.size << "), split_k: (" << split_k_buffer.buffer.buffer << ", " << split_k_buffer.offset << ", " << split_k_buffer.size << "), m: " << m << ", n: " << n << ", k: " << k << ", stride_a: " << stride_a << ", stride_b: " << stride_b << ", stride_d: " << stride_d << ", split_k: " << split_k << ", batch: " << batch << ", ne02: " << ne02 << ", ne12: " << ne12 << ", broadcast2: " << broadcast2 << ", broadcast3: " << broadcast3 << ", batch_stride_a: " << batch_stride_a << ", batch_stride_b: " << batch_stride_b << ", batch_stride_d: " << batch_stride_d << ")" << std::endl; + std::cerr << "ggml_vk_matmul(a: (" << a.buffer->buffer << ", " << a.offset << ", " << a.size << "), b: (" << b.buffer->buffer << ", " << b.offset << ", " << b.size << "), c: (" << d.buffer->buffer << ", " << d.offset << ", " << d.size << "), split_k: (" << split_k_buffer.buffer->buffer << ", " << split_k_buffer.offset << ", " << split_k_buffer.size << "), m: " << m << ", n: " << n << ", k: " << k << ", stride_a: " << stride_a << ", stride_b: " << stride_b << ", stride_d: " << stride_d << ", split_k: " << split_k << ", batch: " << batch << ", ne02: " << ne02 << ", ne12: " << ne12 << ", broadcast2: " << broadcast2 << ", broadcast3: " << broadcast3 << ", batch_stride_a: " << batch_stride_a << ", batch_stride_b: " << batch_stride_b << ", batch_stride_d: " << batch_stride_d << ")" << std::endl; #endif - ggml_vk_sync_buffers(ctx); + ggml_vk_sync_buffers(subctx); if (split_k == 1) { const std::array pc = { m, n, k, stride_a, stride_b, stride_d, k, ne02, ne12, broadcast2, broadcast3, batch_stride_a, batch_stride_b, batch_stride_d }; - ggml_vk_dispatch_pipeline(ctx, pipeline, { a, b, d }, pc.size() * sizeof(uint32_t), pc.data(), { m, n, batch }); + ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, { a, b, d }, pc.size() * sizeof(uint32_t), pc.data(), { m, n, batch }); return; } @@ -1919,10 +2109,10 @@ static void ggml_vk_matmul(vk_context * ctx, vk_pipeline& pipeline, vk_subbuffer const std::array pc1 = { m, n, k, stride_a, stride_b, stride_d, CEIL_DIV(k, split_k), ne02, ne12, broadcast2, broadcast3, batch_stride_a, batch_stride_b, batch_stride_d }; // Make sure enough workgroups get assigned for split k to work - ggml_vk_dispatch_pipeline(ctx, pipeline, { a, b, split_k_buffer }, pc1.size() * sizeof(uint32_t), pc1.data(), { (CEIL_DIV(m, pipeline.wg_denoms[0]) * pipeline.wg_denoms[0]) * split_k, n, batch }); - ggml_vk_sync_buffers(ctx); + ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, { a, b, split_k_buffer }, pc1.size() * sizeof(uint32_t), pc1.data(), { (CEIL_DIV(m, pipeline.wg_denoms[0]) * pipeline.wg_denoms[0]) * split_k, n, batch }); + ggml_vk_sync_buffers(subctx); const std::array pc2 = { (uint32_t)(m * n * batch), split_k }; - ggml_vk_dispatch_pipeline(ctx, vk_pipeline_matmul_split_k_reduce, { split_k_buffer, d }, pc2.size() * sizeof(uint32_t), pc2.data(), { m * n * batch, 1, 1 }); + ggml_vk_dispatch_pipeline(ctx, subctx, ctx->pipeline_matmul_split_k_reduce, { split_k_buffer, d }, pc2.size() * sizeof(uint32_t), pc2.data(), { m * n * batch, 1, 1 }); } static bool ggml_vk_dim01_contiguous(const ggml_tensor * tensor) { @@ -1932,32 +2122,32 @@ static bool ggml_vk_dim01_contiguous(const ggml_tensor * tensor) { tensor->nb[3] == tensor->nb[2]*tensor->ne[2]; } -static vk_pipeline * ggml_vk_get_cpy_pipeline(ggml_type from, ggml_type to) { +static vk_pipeline * ggml_vk_get_cpy_pipeline(ggml_backend_vk_context * ctx, ggml_type from, ggml_type to) { if (from == GGML_TYPE_F32 && to == GGML_TYPE_F32) { - return &vk_pipeline_cpy_f32_f32; + return &ctx->pipeline_cpy_f32_f32; } if (from == GGML_TYPE_F32 && to == GGML_TYPE_F16) { - return &vk_pipeline_cpy_f32_f16; + return &ctx->pipeline_cpy_f32_f16; } if (from == GGML_TYPE_F16 && to == GGML_TYPE_F16) { - return &vk_pipeline_cpy_f16_f16; + return &ctx->pipeline_cpy_f16_f16; } std::cerr << "Missing CPY op for types: " << ggml_type_name(from) << " " << ggml_type_name(to) << std::endl; GGML_ASSERT(false); } -static void ggml_vk_cpy_to_contiguous(vk_context * ctx, vk_pipeline * pipeline, const ggml_tensor * tensor, vk_subbuffer&& in, vk_subbuffer&& out, ggml_type buffer_type, bool aligned=true) { +static void ggml_vk_cpy_to_contiguous(ggml_backend_vk_context * ctx, vk_context * subctx, vk_pipeline * pipeline, const ggml_tensor * tensor, vk_subbuffer&& in, vk_subbuffer&& out, ggml_type buffer_type, bool aligned=true) { #ifdef GGML_VULKAN_DEBUG std::cerr << "ggml_vk_cpy_to_contiguous((" << tensor << ", type=" << tensor->type << ", backend=" << tensor->backend << ", ne0=" << tensor->ne[0] << ", ne1=" << tensor->ne[1] << ", ne2=" << tensor->ne[2] << ", ne3=" << tensor->ne[3] << ", nb0=" << tensor->nb[0] << ", nb1=" << tensor->nb[1] << ", nb2=" << tensor->nb[2] << ", nb3=" << tensor->nb[3] << "), "; - std::cerr << "buffer in size=" << in.buffer.size << ", buffer out size=" << out.buffer.size << ")" << std::endl; + std::cerr << "buffer in size=" << in.buffer->size << ", buffer out size=" << out.buffer->size << ")" << std::endl; #endif const int tensor_type_size = ggml_type_size(tensor->type); const int dst_type_size = ggml_type_size(buffer_type); const uint32_t ne = tensor->ne[0] * tensor->ne[1] * tensor->ne[2]; - const uint32_t nb2 = aligned ? ggml_vk_align_size(dst_type_size * tensor->ne[0] * tensor->ne[1], vk_device.properties.limits.minStorageBufferOffsetAlignment) / dst_type_size : tensor->ne[0] * tensor->ne[1]; + const uint32_t nb2 = aligned ? ggml_vk_align_size(dst_type_size * tensor->ne[0] * tensor->ne[1], ctx->device.lock()->properties.limits.minStorageBufferOffsetAlignment) / dst_type_size : tensor->ne[0] * tensor->ne[1]; const vk_op_cpy_push_constants pc = { (uint32_t)ne, @@ -1965,11 +2155,11 @@ static void ggml_vk_cpy_to_contiguous(vk_context * ctx, vk_pipeline * pipeline, (uint32_t)tensor->ne[0], (uint32_t)tensor->ne[1], 1 , (uint32_t)tensor->ne[0] , nb2, 0, }; - ggml_vk_sync_buffers(ctx); - ggml_vk_dispatch_pipeline(ctx, *pipeline, { in, out }, sizeof(vk_op_cpy_push_constants), &pc, { ne, 1, 1 }); + ggml_vk_sync_buffers(subctx); + ggml_vk_dispatch_pipeline(ctx, subctx, *pipeline, { in, out }, sizeof(vk_op_cpy_push_constants), &pc, { ne, 1, 1 }); } -static void ggml_vk_mul_mat_q_f16(vk_context * ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { +static void ggml_vk_mul_mat_q_f16(ggml_backend_vk_context * ctx, vk_context * subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { #ifdef GGML_VULKAN_DEBUG std::cerr << "ggml_vk_mul_mat_q_f16((" << src0 << ", name=" << src0->name << ", type=" << src0->type << ", backend=" << src0->backend << ", ne0=" << src0->ne[0] << ", ne1=" << src0->ne[1] << ", ne2=" << src0->ne[2] << ", ne3=" << src0->ne[3] << ", nb0=" << src0->nb[0] << ", nb1=" << src0->nb[1] << ", nb2=" << src0->nb[2] << ", nb3=" << src0->nb[3]; std::cerr << "), (" << src1 << ", name=" << src1->name << ", type=" << src1->type << ", backend=" << src1->backend << ", ne0=" << src1->ne[0] << ", ne1=" << src1->ne[1] << ", ne2=" << src1->ne[2] << ", ne3=" << src1->ne[3] << ", nb0=" << src1->nb[0] << ", nb1=" << src1->nb[1] << ", nb2=" << src1->nb[2] << ", nb3=" << src1->nb[3]; @@ -1998,17 +2188,17 @@ static void ggml_vk_mul_mat_q_f16(vk_context * ctx, const ggml_tensor * src0, co ggml_tensor_extra_gpu * extra_src0 = (ggml_tensor_extra_gpu *) src0->extra; ggml_tensor_extra_gpu * extra_src1 = (ggml_tensor_extra_gpu *) src1->extra; - vk_buffer * d_Qx = nullptr; + vk_buffer d_Qx; size_t qx_buf_offset = 0; - vk_buffer * d_Qy = nullptr; + vk_buffer d_Qy; size_t qy_buf_offset = 0; bool src0_uma = false; bool src1_uma = false; - if (vk_device.uma) { - ggml_vk_host_get(src0->data, d_Qx, qx_buf_offset); - ggml_vk_host_get(src1->data, d_Qy, qy_buf_offset); + if (ctx->device.lock()->uma) { + ggml_vk_host_get(ctx, src0->data, d_Qx, qx_buf_offset); + ggml_vk_host_get(ctx, src1->data, d_Qy, qy_buf_offset); src0_uma = d_Qx != nullptr; src1_uma = d_Qy != nullptr; } @@ -2031,12 +2221,12 @@ static void ggml_vk_mul_mat_q_f16(vk_context * ctx, const ggml_tensor * src0, co const int y_ne = ne11 * ne10; const int d_ne = ne11 * ne01; - const uint32_t kpad = ggml_vk_align_size(ne10, ggml_vk_guess_matmul_pipeline_align(ne01, ne11)); + const uint32_t kpad = ggml_vk_align_size(ne10, ggml_vk_guess_matmul_pipeline_align(ctx, ne01, ne11)); const bool aligned = ne10 == kpad; const uint32_t split_k = ggml_vk_guess_split_k(ne01, ne11, ne10); - vk_pipeline * pipeline = ggml_vk_guess_matmul_pipeline(true, !f16_f32_kernel, ne01, ne11, aligned); + vk_pipeline * pipeline = ggml_vk_guess_matmul_pipeline(ctx, true, !f16_f32_kernel, ne01, ne11, aligned); const uint64_t qx_sz = ggml_type_size(src0->type) * x_ne / ggml_blck_size(src0->type); const uint64_t qy_sz = ggml_type_size(src1->type) * y_ne / ggml_blck_size(src1->type); @@ -2044,30 +2234,30 @@ static void ggml_vk_mul_mat_q_f16(vk_context * ctx, const ggml_tensor * src0, co const uint64_t y_sz = f16_f32_kernel ? sizeof(float) * y_ne : sizeof(ggml_fp16_t) * y_ne; const uint64_t d_sz = sizeof(float) * d_ne; - vk_buffer* d_D = &extra->buffer_gpu; + vk_buffer d_D = extra->buffer_gpu.lock(); const uint64_t d_buf_offset = extra->offset; GGML_ASSERT(d_D != nullptr); GGML_ASSERT(d_D->size >= d_buf_offset + d_sz * ne02 * ne03); - vk_buffer* d_X; + vk_buffer d_X; uint64_t x_buf_offset = 0; - vk_buffer* d_Y; + vk_buffer d_Y; uint64_t y_buf_offset = 0; if (load_x) { - d_Qx = &vk_prealloc_qx; + d_Qx = ctx->prealloc_qx; } else if (!src0_uma) { - d_Qx = &extra_src0->buffer_gpu; + d_Qx = extra_src0->buffer_gpu.lock(); qx_buf_offset = extra_src0->offset; GGML_ASSERT(d_Qx != nullptr); } if (load_y) { - d_Qy = &vk_prealloc_qy; + d_Qy = ctx->prealloc_qy; } else if (!src1_uma) { - d_Qy = &extra_src1->buffer_gpu; + d_Qy = extra_src1->buffer_gpu.lock(); qy_buf_offset = extra_src1->offset; GGML_ASSERT(d_Qy != nullptr); } if (qx_needs_dequant) { - d_X = &vk_prealloc_x; + d_X = ctx->prealloc_x; GGML_ASSERT(d_X->size >= x_sz * ne02 * ne03); } else { d_X = d_Qx; @@ -2075,7 +2265,7 @@ static void ggml_vk_mul_mat_q_f16(vk_context * ctx, const ggml_tensor * src0, co GGML_ASSERT(qx_sz == x_sz); // NOLINT } if (qy_needs_dequant) { - d_Y = &vk_prealloc_y; + d_Y = ctx->prealloc_y; GGML_ASSERT(d_Y->size >= y_sz * ne02 * ne03); } else { d_Y = d_Qy; @@ -2087,49 +2277,49 @@ static void ggml_vk_mul_mat_q_f16(vk_context * ctx, const ggml_tensor * src0, co vk_pipeline * to_fp16_vk_1 = nullptr; if (x_non_contig) { - to_fp16_vk_0 = ggml_vk_get_cpy_pipeline(src0->type, GGML_TYPE_F16); + to_fp16_vk_0 = ggml_vk_get_cpy_pipeline(ctx, src0->type, GGML_TYPE_F16); } else { - to_fp16_vk_0 = ggml_vk_get_to_fp16(src0->type); + to_fp16_vk_0 = ggml_vk_get_to_fp16(ctx, src0->type); } if (y_non_contig) { - to_fp16_vk_1 = ggml_vk_get_cpy_pipeline(src1->type, GGML_TYPE_F16); + to_fp16_vk_1 = ggml_vk_get_cpy_pipeline(ctx, src1->type, GGML_TYPE_F16); } else { - to_fp16_vk_1 = ggml_vk_get_to_fp16(src1->type); + to_fp16_vk_1 = ggml_vk_get_to_fp16(ctx, src1->type); } GGML_ASSERT(!qx_needs_dequant || to_fp16_vk_0 != nullptr); // NOLINT GGML_ASSERT(!qy_needs_dequant || to_fp16_vk_1 != nullptr); // NOLINT // Allocate descriptor sets - ggml_vk_pipeline_allocate_descriptor_sets(*pipeline, ne12 * ne13); + ggml_pipeline_allocate_descriptor_sets(ctx, *pipeline, ne12 * ne13); if (qx_needs_dequant) { - ggml_vk_pipeline_allocate_descriptor_sets(*to_fp16_vk_0, x_non_contig ? 1 : ne12 * ne13); + ggml_pipeline_allocate_descriptor_sets(ctx, *to_fp16_vk_0, x_non_contig ? 1 : ne12 * ne13); } if (qy_needs_dequant) { - ggml_vk_pipeline_allocate_descriptor_sets(*to_fp16_vk_1, y_non_contig ? 1 : ne12 * ne13); + ggml_pipeline_allocate_descriptor_sets(ctx, *to_fp16_vk_1, y_non_contig ? 1 : ne12 * ne13); } if (split_k > 1) { - ggml_vk_pipeline_allocate_descriptor_sets(vk_pipeline_matmul_split_k_reduce, ne12 * ne13); + ggml_pipeline_allocate_descriptor_sets(ctx, ctx->pipeline_matmul_split_k_reduce, ne12 * ne13); } if (x_non_contig) { - ggml_vk_cpy_to_contiguous(ctx, to_fp16_vk_0, src0, { *d_Qx, qx_buf_offset, VK_WHOLE_SIZE }, { *d_X, 0, VK_WHOLE_SIZE }, dst->type, false); + ggml_vk_cpy_to_contiguous(ctx, subctx, to_fp16_vk_0, src0, { d_Qx, qx_buf_offset, VK_WHOLE_SIZE }, { d_X, 0, VK_WHOLE_SIZE }, dst->type, false); } else if (load_x || qx_needs_dequant) { if (load_x) { // copy data to device - ggml_vk_h2d_tensor_2d(ctx, d_Qx, 0, src0, 0, 0, ggml_nrows(src0)); - vk_staging_offset = qx_sz * ne02 * ne03; + ggml_vk_h2d_tensor_2d(ctx, subctx, d_Qx, 0, src0, 0, 0, ggml_nrows(src0)); + ctx->staging_offset = qx_sz * ne02 * ne03; } if (qx_needs_dequant) { const std::vector pc = { (int)ne01, (int)ne10, (int)ne10, (int)ne10 }; - ggml_vk_sync_buffers(ctx); - ggml_vk_dispatch_pipeline(ctx, *to_fp16_vk_0, { { *d_Qx, qx_buf_offset, qx_sz * ne02 * ne03 }, { *d_X, 0, x_sz * ne02 * ne03 } }, pc.size() * sizeof(int), pc.data(), { (uint32_t)(x_ne * ne02 * ne03), 1, 1}); + ggml_vk_sync_buffers(subctx); + ggml_vk_dispatch_pipeline(ctx, subctx, *to_fp16_vk_0, { { d_Qx, qx_buf_offset, qx_sz * ne02 * ne03 }, { d_X, 0, x_sz * ne02 * ne03 } }, pc.size() * sizeof(int), pc.data(), { (uint32_t)(x_ne * ne02 * ne03), 1, 1}); } } if (y_non_contig) { - ggml_vk_cpy_to_contiguous(ctx, to_fp16_vk_1, src1, { *d_Qy, qy_buf_offset, VK_WHOLE_SIZE }, { *d_Y, 0, VK_WHOLE_SIZE }, dst->type); + ggml_vk_cpy_to_contiguous(ctx, subctx, to_fp16_vk_1, src1, { d_Qy, qy_buf_offset, VK_WHOLE_SIZE }, { d_Y, 0, VK_WHOLE_SIZE }, dst->type); } else if (load_y) { - ggml_vk_h2d_tensor_2d(ctx, d_Qy, 0, src1, 0, 0, ggml_nrows(src1)); + ggml_vk_h2d_tensor_2d(ctx, subctx, d_Qy, 0, src1, 0, 0, ggml_nrows(src1)); } uint32_t stride_batch_x = ne00*ne01; @@ -2144,16 +2334,16 @@ static void ggml_vk_mul_mat_q_f16(vk_context * ctx, const ggml_tensor * src0, co } // compute - ggml_vk_matmul(ctx, *pipeline, { *d_X, x_buf_offset, x_sz * ne02 * ne03 }, { *d_Y, y_buf_offset, y_sz * ne12 * ne13 }, { *d_D, d_buf_offset, d_sz * ne12 * ne13 }, { vk_prealloc_split_k, 0, d_sz * ne12 * ne13 * split_k }, ne01, ne11, ne10, ne10, ne10, ne01, split_k, ne12*ne13, ne02, ne12, r2, r3, stride_batch_x, stride_batch_y, ne20*ne21); // NOLINT + ggml_vk_matmul(ctx, subctx, *pipeline, { d_X, x_buf_offset, x_sz * ne02 * ne03 }, { d_Y, y_buf_offset, y_sz * ne12 * ne13 }, { d_D, d_buf_offset, d_sz * ne12 * ne13 }, { ctx->prealloc_split_k, 0, d_sz * ne12 * ne13 * split_k }, ne01, ne11, ne10, ne10, ne10, ne01, split_k, ne12*ne13, ne02, ne12, r2, r3, stride_batch_x, stride_batch_y, ne20*ne21); // NOLINT if (dst->backend == GGML_BACKEND_CPU) { // copy dst to host float * d = (float *) ((char *) dst->data); - ggml_vk_buffer_read_async(ctx, d_D, 0, d, sizeof(float) * d_ne * ne12 * ne13); + ggml_vk_buffer_read_async(ctx, subctx, d_D, 0, d, sizeof(float) * d_ne * ne12 * ne13); } } -static void ggml_vk_mul_mat_vec_q_f16(vk_context * ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { +static void ggml_vk_mul_mat_vec_q_f16(ggml_backend_vk_context * ctx, vk_context * subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { #ifdef GGML_VULKAN_DEBUG std::cerr << "ggml_vk_mul_mat_vec_q_f16((" << src0 << ", name=" << src0->name << ", type=" << src0->type << ", backend=" << src0->backend << ", ne0=" << src0->ne[0] << ", ne1=" << src0->ne[1] << ", ne2=" << src0->ne[2] << ", ne3=" << src0->ne[3] << ", nb0=" << src0->nb[0] << ", nb1=" << src0->nb[1] << ", nb2=" << src0->nb[2] << ", nb3=" << src0->nb[3]; std::cerr << "), (" << src1 << ", name=" << src1->name << ", type=" << src1->type << ", backend=" << src1->backend << ", ne0=" << src1->ne[0] << ", ne1=" << src1->ne[1] << ", ne2=" << src1->ne[2] << ", ne3=" << src1->ne[3] << ", nb0=" << src1->nb[0] << ", nb1=" << src1->nb[1] << ", nb2=" << src1->nb[2] << ", nb3=" << src1->nb[3]; @@ -2184,17 +2374,17 @@ static void ggml_vk_mul_mat_vec_q_f16(vk_context * ctx, const ggml_tensor * src0 ggml_tensor_extra_gpu * extra_src0 = (ggml_tensor_extra_gpu *) src0->extra; ggml_tensor_extra_gpu * extra_src1 = (ggml_tensor_extra_gpu *) src1->extra; - vk_buffer * d_Qx = nullptr; + vk_buffer d_Qx; size_t qx_buf_offset = 0; - vk_buffer * d_Qy = nullptr; + vk_buffer d_Qy; size_t qy_buf_offset = 0; bool src0_uma = false; bool src1_uma = false; - if (vk_device.uma) { - ggml_vk_host_get(src0->data, d_Qx, qx_buf_offset); - ggml_vk_host_get(src1->data, d_Qy, qy_buf_offset); + if (ctx->device.lock()->uma) { + ggml_vk_host_get(ctx, src0->data, d_Qx, qx_buf_offset); + ggml_vk_host_get(ctx, src1->data, d_Qy, qy_buf_offset); src0_uma = d_Qx != nullptr; src1_uma = d_Qy != nullptr; } @@ -2214,42 +2404,42 @@ static void ggml_vk_mul_mat_vec_q_f16(vk_context * ctx, const ggml_tensor * src0 const uint64_t y_ne = ne11 * ne10; const uint64_t d_ne = ne11 * ne01; - const uint64_t qx_sz = ggml_vk_align_size(ggml_type_size(src0->type) * x_ne / ggml_blck_size(src0->type), vk_device.properties.limits.minStorageBufferOffsetAlignment); + const uint64_t qx_sz = ggml_vk_align_size(ggml_type_size(src0->type) * x_ne / ggml_blck_size(src0->type), ctx->device.lock()->properties.limits.minStorageBufferOffsetAlignment); const uint64_t qy_sz = ggml_type_size(src1->type) * y_ne / ggml_blck_size(src1->type); - const uint64_t x_sz = x_non_contig ? ggml_vk_align_size(ggml_type_size(src0->type) * x_ne, vk_device.properties.limits.minStorageBufferOffsetAlignment) : qx_sz; + const uint64_t x_sz = x_non_contig ? ggml_vk_align_size(ggml_type_size(src0->type) * x_ne, ctx->device.lock()->properties.limits.minStorageBufferOffsetAlignment) : qx_sz; const uint64_t y_sz = f16_f32_kernel ? sizeof(float) * y_ne : sizeof(ggml_fp16_t) * y_ne; const uint64_t d_sz = sizeof(float) * d_ne; - vk_buffer* d_D = &extra->buffer_gpu; + vk_buffer d_D = extra->buffer_gpu.lock(); const uint64_t d_buf_offset = extra->offset; GGML_ASSERT(d_D != nullptr); - vk_buffer* d_X; + vk_buffer d_X; uint64_t x_buf_offset = 0; - vk_buffer* d_Y; + vk_buffer d_Y; uint64_t y_buf_offset = 0; if (load_x) { - d_Qx = &vk_prealloc_qx; + d_Qx = ctx->prealloc_qx; } else if(!src1_uma) { - d_Qx = &extra_src0->buffer_gpu; + d_Qx = extra_src0->buffer_gpu.lock(); qx_buf_offset = extra_src0->offset; GGML_ASSERT(d_Qx != nullptr); } if (load_y) { - d_Qy = &vk_prealloc_qy; + d_Qy = ctx->prealloc_qy; } else if(!src1_uma) { - d_Qy = &extra_src1->buffer_gpu; + d_Qy = extra_src1->buffer_gpu.lock(); qy_buf_offset = extra_src1->offset; GGML_ASSERT(d_Qy != nullptr); } if (qx_needs_dequant) { - d_X = &vk_prealloc_x; + d_X = ctx->prealloc_x; } else { d_X = d_Qx; x_buf_offset = qx_buf_offset; GGML_ASSERT(qx_sz == x_sz); } if (qy_needs_dequant) { - d_Y = &vk_prealloc_y; + d_Y = ctx->prealloc_y; } else { d_Y = d_Qy; y_buf_offset = qy_buf_offset; @@ -2259,39 +2449,39 @@ static void ggml_vk_mul_mat_vec_q_f16(vk_context * ctx, const ggml_tensor * src0 vk_pipeline * to_fp16_vk_0 = nullptr; vk_pipeline* to_fp16_vk_1 = nullptr; if (x_non_contig) { - to_fp16_vk_0 = ggml_vk_get_cpy_pipeline(src0->type, src0->type); + to_fp16_vk_0 = ggml_vk_get_cpy_pipeline(ctx, src0->type, src0->type); } if (y_non_contig) { - to_fp16_vk_1 = ggml_vk_get_cpy_pipeline(src1->type, src1->type); + to_fp16_vk_1 = ggml_vk_get_cpy_pipeline(ctx, src1->type, src1->type); } else { - to_fp16_vk_1 = ggml_vk_get_to_fp16(src1->type); + to_fp16_vk_1 = ggml_vk_get_to_fp16(ctx, src1->type); } - vk_pipeline* dmmv = ggml_vk_get_dequantize_mul_mat_vec(src0->type); + vk_pipeline* dmmv = ggml_vk_get_dequantize_mul_mat_vec(ctx, src0->type); GGML_ASSERT(!qx_needs_dequant || to_fp16_vk_0 != nullptr); // NOLINT GGML_ASSERT(!qy_needs_dequant || to_fp16_vk_1 != nullptr); // NOLINT GGML_ASSERT(dmmv != nullptr); // Allocate descriptor sets if (qx_needs_dequant) { - ggml_vk_pipeline_allocate_descriptor_sets(*to_fp16_vk_0, 1); + ggml_pipeline_allocate_descriptor_sets(ctx, *to_fp16_vk_0, 1); } if (qy_needs_dequant) { - ggml_vk_pipeline_allocate_descriptor_sets(*to_fp16_vk_1, y_non_contig ? 1 : ne12 * ne13); + ggml_pipeline_allocate_descriptor_sets(ctx, *to_fp16_vk_1, y_non_contig ? 1 : ne12 * ne13); } - ggml_vk_pipeline_allocate_descriptor_sets(*dmmv, ne12 * ne13); + ggml_pipeline_allocate_descriptor_sets(ctx, *dmmv, ne12 * ne13); if (x_non_contig) { - GGML_ASSERT(x_sz == ggml_vk_align_size(ggml_type_size(src0->type) * x_ne, vk_device.properties.limits.minStorageBufferOffsetAlignment)); - ggml_vk_cpy_to_contiguous(ctx, to_fp16_vk_0, src0, { *d_Qx, qx_buf_offset, VK_WHOLE_SIZE }, { *d_X, 0, VK_WHOLE_SIZE }, src0->type); + GGML_ASSERT(x_sz == ggml_vk_align_size(ggml_type_size(src0->type) * x_ne, ctx->device.lock()->properties.limits.minStorageBufferOffsetAlignment)); + ggml_vk_cpy_to_contiguous(ctx, subctx, to_fp16_vk_0, src0, { d_Qx, qx_buf_offset, VK_WHOLE_SIZE }, { d_X, 0, VK_WHOLE_SIZE }, src0->type); } else if (load_x) { // copy data to device - ggml_vk_h2d_tensor_2d(ctx, d_Qx, 0, src0, 0, 0, ggml_nrows(src0)); + ggml_vk_h2d_tensor_2d(ctx, subctx, d_Qx, 0, src0, 0, 0, ggml_nrows(src0)); } if (y_non_contig) { GGML_ASSERT(y_sz == ggml_type_size(src1->type) * y_ne); - ggml_vk_cpy_to_contiguous(ctx, to_fp16_vk_1, src1, { *d_Qy, qy_buf_offset, VK_WHOLE_SIZE }, { *d_Y, 0, VK_WHOLE_SIZE }, src1->type); + ggml_vk_cpy_to_contiguous(ctx, subctx, to_fp16_vk_1, src1, { d_Qy, qy_buf_offset, VK_WHOLE_SIZE }, { d_Y, 0, VK_WHOLE_SIZE }, src1->type); } else if (load_y) { - ggml_vk_h2d_tensor_2d(ctx, d_Qy, 0, src1, 0, 0, ggml_nrows(src1)); + ggml_vk_h2d_tensor_2d(ctx, subctx, d_Qy, 0, src1, 0, 0, ggml_nrows(src1)); } for (uint64_t i13 = 0; i13 < ne13; i13++) { @@ -2306,34 +2496,34 @@ static void ggml_vk_mul_mat_vec_q_f16(vk_context * ctx, const ggml_tensor * src0 const uint64_t y_offset = y_buf_offset + y_sz * it_idx1; const uint64_t d_offset = d_buf_offset + d_sz * it_idx1; - const uint64_t y_buffer_offset = (y_offset / vk_device.properties.limits.minStorageBufferOffsetAlignment) * vk_device.properties.limits.minStorageBufferOffsetAlignment; + const uint64_t y_buffer_offset = (y_offset / ctx->device.lock()->properties.limits.minStorageBufferOffsetAlignment) * ctx->device.lock()->properties.limits.minStorageBufferOffsetAlignment; const uint64_t y_shader_offset = y_offset - y_buffer_offset; - const uint64_t d_buffer_offset = (d_offset / vk_device.properties.limits.minStorageBufferOffsetAlignment) * vk_device.properties.limits.minStorageBufferOffsetAlignment; + const uint64_t d_buffer_offset = (d_offset / ctx->device.lock()->properties.limits.minStorageBufferOffsetAlignment) * ctx->device.lock()->properties.limits.minStorageBufferOffsetAlignment; const uint64_t d_shader_offset = d_offset - d_buffer_offset; if (!y_non_contig && qy_needs_dequant) { const std::vector pc = { (int)ne11, (int)ne10, (int)ne10, (int)ne10 }; - ggml_vk_sync_buffers(ctx); - ggml_vk_dispatch_pipeline(ctx, *to_fp16_vk_1, { { *d_Qy, qy_offset, qy_sz }, { *d_Y, y_offset, y_sz } }, pc.size() * sizeof(int), pc.data(), { (uint32_t)y_ne, 1, 1}); + ggml_vk_sync_buffers(subctx); + ggml_vk_dispatch_pipeline(ctx, subctx, *to_fp16_vk_1, { { d_Qy, qy_offset, qy_sz }, { d_Y, y_offset, y_sz } }, pc.size() * sizeof(int), pc.data(), { (uint32_t)y_ne, 1, 1}); } // compute const std::array pc = { (int)ne00, (int)(y_shader_offset / ggml_type_size(src1->type)), (int)(d_shader_offset / ggml_type_size(dst->type))}; - ggml_vk_sync_buffers(ctx); - ggml_vk_dispatch_pipeline(ctx, *dmmv, { { *d_X, x_offset, x_sz }, { *d_Y, y_buffer_offset, y_sz + y_shader_offset }, { *d_D, d_buffer_offset, d_sz + d_shader_offset } }, 3 * sizeof(int), &pc, { (uint32_t)ne01, 1, 1}); + ggml_vk_sync_buffers(subctx); + ggml_vk_dispatch_pipeline(ctx, subctx, *dmmv, { { d_X, x_offset, x_sz }, { d_Y, y_buffer_offset, y_sz + y_shader_offset }, { d_D, d_buffer_offset, d_sz + d_shader_offset } }, 3 * sizeof(int), &pc, { (uint32_t)ne01, 1, 1}); if (dst->backend == GGML_BACKEND_CPU) { // copy dst to host float * d = (float *) ((char *) dst->data + i12*nb2 + i13*nb3); - ggml_vk_sync_buffers(ctx); - ggml_vk_buffer_read_async(ctx, d_D, d_offset, d, sizeof(float) * d_ne); + ggml_vk_sync_buffers(subctx); + ggml_vk_buffer_read_async(ctx, subctx, d_D, d_offset, d, sizeof(float) * d_ne); } } } } -static void ggml_vk_mul_mat_vec_p021_f16_f32(vk_context * ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { +static void ggml_vk_mul_mat_vec_p021_f16_f32(ggml_backend_vk_context * ctx, vk_context * subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { #ifdef GGML_VULKAN_DEBUG std::cerr << "ggml_vk_mul_mat_p021_f16_f32((" << src0 << ", name=" << src0->name << ", type=" << src0->type << ", backend=" << src0->backend << ", ne0=" << src0->ne[0] << ", ne1=" << src0->ne[1] << ", ne2=" << src0->ne[2] << ", ne3=" << src0->ne[3] << ", nb0=" << src0->nb[0] << ", nb1=" << src0->nb[1] << ", nb2=" << src0->nb[2] << ", nb3=" << src0->nb[3]; std::cerr << "), (" << src1 << ", name=" << src1->name << ", type=" << src1->type << ", backend=" << src1->backend << ", ne0=" << src1->ne[0] << ", ne1=" << src1->ne[1] << ", ne2=" << src1->ne[2] << ", ne3=" << src1->ne[3] << ", nb0=" << src1->nb[0] << ", nb1=" << src1->nb[1] << ", nb2=" << src1->nb[2] << ", nb3=" << src1->nb[3]; @@ -2362,13 +2552,13 @@ static void ggml_vk_mul_mat_vec_p021_f16_f32(vk_context * ctx, const ggml_tensor ggml_tensor_extra_gpu * extra_src0 = (ggml_tensor_extra_gpu *) src0->extra; ggml_tensor_extra_gpu * extra_src1 = (ggml_tensor_extra_gpu *) src1->extra; - vk_buffer * d_Qy = nullptr; + vk_buffer d_Qy; size_t qy_buf_offset = 0; bool src1_uma = false; - if (vk_device.uma) { - ggml_vk_host_get(src1->data, d_Qy, qy_buf_offset); + if (ctx->device.lock()->uma) { + ggml_vk_host_get(ctx, src1->data, d_Qy, qy_buf_offset); src1_uma = d_Qy != nullptr; } @@ -2378,51 +2568,51 @@ static void ggml_vk_mul_mat_vec_p021_f16_f32(vk_context * ctx, const ggml_tensor const uint64_t y_ne = ne10 * ne11 * ne12; const uint64_t d_ne = ne01 * ne11 * ne12; - const uint64_t qx_sz = ggml_vk_align_size(ggml_type_size(src0->type) * x_ne / ggml_blck_size(src0->type), vk_device.properties.limits.minStorageBufferOffsetAlignment); + const uint64_t qx_sz = ggml_vk_align_size(ggml_type_size(src0->type) * x_ne / ggml_blck_size(src0->type), ctx->device.lock()->properties.limits.minStorageBufferOffsetAlignment); const uint64_t qy_sz = ggml_type_size(src1->type) * y_ne / ggml_blck_size(src1->type); const uint64_t d_sz = sizeof(float) * d_ne; - vk_buffer* d_D = &extra->buffer_gpu; + vk_buffer d_D = extra->buffer_gpu.lock(); const uint64_t d_buf_offset = extra->offset; GGML_ASSERT(d_D != nullptr); - vk_buffer* d_Qx = &extra_src0->buffer_gpu; + vk_buffer d_Qx = extra_src0->buffer_gpu.lock(); const uint64_t qx_buf_offset = extra_src0->offset; GGML_ASSERT(d_Qx != nullptr); if (load_y) { - d_Qy = &vk_prealloc_qy; + d_Qy = ctx->prealloc_qy; } else if (!src1_uma) { - d_Qy = &extra_src1->buffer_gpu; + d_Qy = extra_src1->buffer_gpu.lock(); qy_buf_offset = extra_src1->offset; GGML_ASSERT(d_Qx != nullptr); } // Allocate descriptor sets - ggml_vk_pipeline_allocate_descriptor_sets(vk_pipeline_mul_mat_vec_p021_f16_f32, 1); + ggml_pipeline_allocate_descriptor_sets(ctx, ctx->pipeline_mul_mat_vec_p021_f16_f32, 1); - const uint64_t qy_buffer_offset = (qy_buf_offset / vk_device.properties.limits.minStorageBufferOffsetAlignment) * vk_device.properties.limits.minStorageBufferOffsetAlignment; + const uint64_t qy_buffer_offset = (qy_buf_offset / ctx->device.lock()->properties.limits.minStorageBufferOffsetAlignment) * ctx->device.lock()->properties.limits.minStorageBufferOffsetAlignment; const uint64_t qy_shader_offset = qy_buf_offset - qy_buffer_offset; - const uint64_t d_buffer_offset = (d_buf_offset / vk_device.properties.limits.minStorageBufferOffsetAlignment) * vk_device.properties.limits.minStorageBufferOffsetAlignment; + const uint64_t d_buffer_offset = (d_buf_offset / ctx->device.lock()->properties.limits.minStorageBufferOffsetAlignment) * ctx->device.lock()->properties.limits.minStorageBufferOffsetAlignment; const uint64_t d_shader_offset = d_buf_offset - d_buffer_offset; if (load_y) { - ggml_vk_h2d_tensor_2d(ctx, d_Qy, qy_buf_offset, src1, 0, 0, ggml_nrows(src1)); + ggml_vk_h2d_tensor_2d(ctx, subctx, d_Qy, qy_buf_offset, src1, 0, 0, ggml_nrows(src1)); } // compute const std::array pc = { (uint32_t)ne00, (uint32_t)ne01, (uint32_t)ne02, (uint32_t)ne12, (uint32_t)(qy_shader_offset / ggml_type_size(src1->type)), (uint32_t)(d_shader_offset / ggml_type_size(dst->type)) }; - ggml_vk_sync_buffers(ctx); - ggml_vk_dispatch_pipeline(ctx, vk_pipeline_mul_mat_vec_p021_f16_f32, { { *d_Qx, qx_buf_offset, qx_sz }, { *d_Qy, qy_buffer_offset, qy_sz + qy_shader_offset }, { *d_D, d_buffer_offset, d_sz + d_shader_offset } }, 6 * sizeof(uint32_t), &pc, { 1, (uint32_t)ne01, (uint32_t)ne12 }); + ggml_vk_sync_buffers(subctx); + ggml_vk_dispatch_pipeline(ctx, subctx, ctx->pipeline_mul_mat_vec_p021_f16_f32, { { d_Qx, qx_buf_offset, qx_sz }, { d_Qy, qy_buffer_offset, qy_sz + qy_shader_offset }, { d_D, d_buffer_offset, d_sz + d_shader_offset } }, 6 * sizeof(uint32_t), &pc, { 1, (uint32_t)ne01, (uint32_t)ne12 }); if (dst->backend == GGML_BACKEND_CPU) { // copy dst to host float * d = (float *) dst->data; - ggml_vk_sync_buffers(ctx); - ggml_vk_buffer_read_async(ctx, d_D, d_buf_offset, d, sizeof(float) * d_ne); + ggml_vk_sync_buffers(subctx); + ggml_vk_buffer_read_async(ctx, subctx, d_D, d_buf_offset, d, sizeof(float) * d_ne); } } -static void ggml_vk_mul_mat_vec_nc_f16_f32(vk_context * ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { +static void ggml_vk_mul_mat_vec_nc_f16_f32(ggml_backend_vk_context * ctx, vk_context * subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { #ifdef GGML_VULKAN_DEBUG std::cerr << "ggml_vk_mul_mat_nc_f16_f32((" << src0 << ", name=" << src0->name << ", type=" << src0->type << ", backend=" << src0->backend << ", ne0=" << src0->ne[0] << ", ne1=" << src0->ne[1] << ", ne2=" << src0->ne[2] << ", ne3=" << src0->ne[3] << ", nb0=" << src0->nb[0] << ", nb1=" << src0->nb[1] << ", nb2=" << src0->nb[2] << ", nb3=" << src0->nb[3]; std::cerr << "), (" << src1 << ", name=" << src1->name << ", type=" << src1->type << ", backend=" << src1->backend << ", ne0=" << src1->ne[0] << ", ne1=" << src1->ne[1] << ", ne2=" << src1->ne[2] << ", ne3=" << src1->ne[3] << ", nb0=" << src1->nb[0] << ", nb1=" << src1->nb[1] << ", nb2=" << src1->nb[2] << ", nb3=" << src1->nb[3]; @@ -2454,13 +2644,13 @@ static void ggml_vk_mul_mat_vec_nc_f16_f32(vk_context * ctx, const ggml_tensor * ggml_tensor_extra_gpu * extra_src0 = (ggml_tensor_extra_gpu *) src0->extra; ggml_tensor_extra_gpu * extra_src1 = (ggml_tensor_extra_gpu *) src1->extra; - vk_buffer * d_Qy = nullptr; + vk_buffer d_Qy = nullptr; size_t qy_buf_offset = 0; bool src1_uma = false; - if (vk_device.uma) { - ggml_vk_host_get(src1->data, d_Qy, qy_buf_offset); + if (ctx->device.lock()->uma) { + ggml_vk_host_get(ctx, src1->data, d_Qy, qy_buf_offset); src1_uma = d_Qy != nullptr; } @@ -2475,43 +2665,43 @@ static void ggml_vk_mul_mat_vec_nc_f16_f32(vk_context * ctx, const ggml_tensor * const uint64_t qy_sz = ggml_nbytes(src1); const uint64_t d_sz = sizeof(float) * d_ne; - vk_buffer* d_D = &extra->buffer_gpu; + vk_buffer d_D = extra->buffer_gpu.lock(); const uint64_t d_buf_offset = extra->offset; GGML_ASSERT(d_D != nullptr); - vk_buffer* d_Qx = &extra_src0->buffer_gpu; + vk_buffer d_Qx = extra_src0->buffer_gpu.lock(); const uint64_t qx_buf_offset = extra_src0->offset; GGML_ASSERT(d_Qx != nullptr); if (load_y) { - d_Qy = &vk_prealloc_qy; + d_Qy = ctx->prealloc_qy; } else { - d_Qy = &extra_src1->buffer_gpu; + d_Qy = extra_src1->buffer_gpu.lock(); qy_buf_offset = extra_src1->offset; GGML_ASSERT(d_Qx != nullptr); } // Allocate descriptor sets - ggml_vk_pipeline_allocate_descriptor_sets(vk_pipeline_mul_mat_vec_nc_f16_f32, 1); + ggml_pipeline_allocate_descriptor_sets(ctx, ctx->pipeline_mul_mat_vec_nc_f16_f32, 1); - const uint64_t qy_buffer_offset = (qy_buf_offset / vk_device.properties.limits.minStorageBufferOffsetAlignment) * vk_device.properties.limits.minStorageBufferOffsetAlignment; + const uint64_t qy_buffer_offset = (qy_buf_offset / ctx->device.lock()->properties.limits.minStorageBufferOffsetAlignment) * ctx->device.lock()->properties.limits.minStorageBufferOffsetAlignment; const uint64_t qy_shader_offset = qy_buf_offset - qy_buffer_offset; - const uint64_t d_buffer_offset = (d_buf_offset / vk_device.properties.limits.minStorageBufferOffsetAlignment) * vk_device.properties.limits.minStorageBufferOffsetAlignment; + const uint64_t d_buffer_offset = (d_buf_offset / ctx->device.lock()->properties.limits.minStorageBufferOffsetAlignment) * ctx->device.lock()->properties.limits.minStorageBufferOffsetAlignment; const uint64_t d_shader_offset = d_buf_offset - d_buffer_offset; if (load_y) { - ggml_vk_h2d_tensor_2d(ctx, d_Qy, qy_buf_offset, src1, 0, 0, ggml_nrows(src1)); + ggml_vk_h2d_tensor_2d(ctx, subctx, d_Qy, qy_buf_offset, src1, 0, 0, ggml_nrows(src1)); } // compute const std::array pc = { (uint32_t)ne00, (uint32_t)ne01, row_stride_x, channel_stride_x, (uint32_t)(ne12 / ne02), (uint32_t)(qy_shader_offset / ggml_type_size(src1->type)), (uint32_t)(d_shader_offset / ggml_type_size(dst->type)) }; - ggml_vk_sync_buffers(ctx); - ggml_vk_dispatch_pipeline(ctx, vk_pipeline_mul_mat_vec_nc_f16_f32, { { *d_Qx, qx_buf_offset, qx_sz }, { *d_Qy, qy_buffer_offset, qy_sz + qy_shader_offset }, { *d_D, d_buffer_offset, d_sz + d_shader_offset } }, 7 * sizeof(uint32_t), &pc, { 1, (uint32_t)ne01, (uint32_t)ne12 }); + ggml_vk_sync_buffers(subctx); + ggml_vk_dispatch_pipeline(ctx, subctx, ctx->pipeline_mul_mat_vec_nc_f16_f32, { { d_Qx, qx_buf_offset, qx_sz }, { d_Qy, qy_buffer_offset, qy_sz + qy_shader_offset }, { d_D, d_buffer_offset, d_sz + d_shader_offset } }, 7 * sizeof(uint32_t), &pc, { 1, (uint32_t)ne01, (uint32_t)ne12 }); if (dst->backend == GGML_BACKEND_CPU) { // copy dst to host float * d = (float *) dst->data; - ggml_vk_sync_buffers(ctx); - ggml_vk_buffer_read_async(ctx, d_D, d_buf_offset, d, sizeof(float) * d_ne); + ggml_vk_sync_buffers(subctx); + ggml_vk_buffer_read_async(ctx, subctx, d_D, d_buf_offset, d, sizeof(float) * d_ne); } } @@ -2528,22 +2718,22 @@ static bool ggml_vk_can_mul_mat(const ggml_tensor * src0, const ggml_tensor * sr ((ne0 >= 32 && ne1 >= 32 && ne10 >= 32) || src0->backend == GGML_BACKEND_GPU); } -static void ggml_vk_mul_mat(vk_context * ctx, const struct ggml_tensor * src0, const struct ggml_tensor * src1, struct ggml_tensor * dst) { +static void ggml_vk_mul_mat(ggml_backend_vk_context * ctx, vk_context * subctx, const struct ggml_tensor * src0, const struct ggml_tensor * src1, struct ggml_tensor * dst) { #ifdef GGML_VULKAN_DEBUG std::cerr << "ggml_vk_mul_mat(" << src0 << ", " << src1 << ", " << dst << ")" << std::endl; #endif if (src0->type == GGML_TYPE_F16 && ggml_is_permuted(src0) && ggml_is_permuted(src1) && src1->ne[1] == 1) { - ggml_vk_mul_mat_vec_p021_f16_f32(ctx, src0, src1, dst); + ggml_vk_mul_mat_vec_p021_f16_f32(ctx, subctx, src0, src1, dst); } else if (src0->type == GGML_TYPE_F16 && !ggml_is_contiguous(src0) && !ggml_is_transposed(src1) && src1->ne[1] == 1) { - ggml_vk_mul_mat_vec_nc_f16_f32(ctx, src0, src1, dst); + ggml_vk_mul_mat_vec_nc_f16_f32(ctx, subctx, src0, src1, dst); } else if (src1->ne[1] == 1 && (src0->type == GGML_TYPE_F16 || ggml_is_quantized(src0->type))) { - ggml_vk_mul_mat_vec_q_f16(ctx, src0, src1, dst); + ggml_vk_mul_mat_vec_q_f16(ctx, subctx, src0, src1, dst); } else { - ggml_vk_mul_mat_q_f16(ctx, src0, src1, dst); + ggml_vk_mul_mat_q_f16(ctx, subctx, src0, src1, dst); } } -static void ggml_vk_op_repeat(vk_context * ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { +static void ggml_vk_op_repeat(ggml_backend_vk_context * ctx, vk_context * subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { // guaranteed to be an integer due to the check in ggml_can_repeat const uint64_t ne0 = dst->ne[0]; const uint64_t ne1 = dst->ne[1]; @@ -2579,9 +2769,9 @@ static void ggml_vk_op_repeat(vk_context * ctx, const ggml_tensor * src0, const ggml_tensor_extra_gpu * extra = (ggml_tensor_extra_gpu *) dst->extra; ggml_tensor_extra_gpu * extra_src0 = (ggml_tensor_extra_gpu *) src0->extra; - const vk_buffer* src_buf = &extra_src0->buffer_gpu; + const vk_buffer src_buf = extra_src0->buffer_gpu.lock(); const uint64_t src_offset = extra_src0->offset; - vk_buffer* dst_buf = &extra->buffer_gpu; + vk_buffer dst_buf = extra->buffer_gpu.lock(); const uint64_t dst_offset = extra->offset; std::vector copies; @@ -2606,78 +2796,79 @@ static void ggml_vk_op_repeat(vk_context * ctx, const ggml_tensor * src0, const } } - ggml_vk_sync_buffers(ctx); - ctx->s->buffer.copyBuffer(src_buf->buffer, dst_buf->buffer, copies); + ggml_vk_sync_buffers(subctx); + subctx->s->buffer.copyBuffer(src_buf->buffer, dst_buf->buffer, copies); - (void) src1; + GGML_UNUSED(ctx); + GGML_UNUSED(src1); } -static vk_pipeline* ggml_vk_op_get_pipeline(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, ggml_op op) { +static vk_pipeline* ggml_vk_op_get_pipeline(ggml_backend_vk_context * ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, ggml_op op) { switch (op) { case GGML_OP_ADD: if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { - return &vk_pipeline_add_f32; + return &ctx->pipeline_add_f32; } return nullptr; case GGML_OP_GET_ROWS: GGML_ASSERT(src1->type == GGML_TYPE_I32); if (dst->type == GGML_TYPE_F16) { - return &vk_pipeline_get_rows[src0->type]; + return &ctx->pipeline_get_rows[src0->type]; } if (dst->type == GGML_TYPE_F32) { - return &vk_pipeline_get_rows_f32[src0->type]; + return &ctx->pipeline_get_rows_f32[src0->type]; } return nullptr; case GGML_OP_MUL: if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { - return &vk_pipeline_mul_f32; + return &ctx->pipeline_mul_f32; } return nullptr; case GGML_OP_SCALE: if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { - return &vk_pipeline_scale_f32; + return &ctx->pipeline_scale_f32; } return nullptr; case GGML_OP_SQR: if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { - return &vk_pipeline_sqr_f32; + return &ctx->pipeline_sqr_f32; } return nullptr; case GGML_OP_CLAMP: if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { - return &vk_pipeline_clamp_f32; + return &ctx->pipeline_clamp_f32; } return nullptr; case GGML_OP_CPY: case GGML_OP_CONT: case GGML_OP_DUP: - return ggml_vk_get_cpy_pipeline(src0->type, dst->type); + return ggml_vk_get_cpy_pipeline(ctx, src0->type, dst->type); case GGML_OP_NORM: if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { - return &vk_pipeline_norm_f32; + return &ctx->pipeline_norm_f32; } return nullptr; case GGML_OP_RMS_NORM: if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { - return &vk_pipeline_rms_norm_f32; + return &ctx->pipeline_rms_norm_f32; } return nullptr; case GGML_OP_UNARY: switch (ggml_get_unary_op(dst)) { case GGML_UNARY_OP_SILU: if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { - return &vk_pipeline_silu_f32; + return &ctx->pipeline_silu_f32; } break; case GGML_UNARY_OP_GELU: if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { - return &vk_pipeline_gelu_f32; + return &ctx->pipeline_gelu_f32; } break; case GGML_UNARY_OP_RELU: if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { - return &vk_pipeline_relu_f32; + return &ctx->pipeline_relu_f32; } break; default: @@ -2686,12 +2877,12 @@ static vk_pipeline* ggml_vk_op_get_pipeline(const ggml_tensor * src0, const ggml return nullptr; case GGML_OP_DIAG_MASK_INF: if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { - return &vk_pipeline_diag_mask_inf_f32; + return &ctx->pipeline_diag_mask_inf_f32; } return nullptr; case GGML_OP_SOFT_MAX: if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { - return &vk_pipeline_soft_max_f32; + return &ctx->pipeline_soft_max_f32; } return nullptr; case GGML_OP_ROPE: @@ -2706,17 +2897,17 @@ static vk_pipeline* ggml_vk_op_get_pipeline(const ggml_tensor * src0, const ggml if (is_neox) { if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { - return &vk_pipeline_rope_neox_f32; + return &ctx->pipeline_rope_neox_f32; } if (src0->type == GGML_TYPE_F16 && dst->type == GGML_TYPE_F16) { - return &vk_pipeline_rope_neox_f16; + return &ctx->pipeline_rope_neox_f16; } } else { if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { - return &vk_pipeline_rope_f32; + return &ctx->pipeline_rope_f32; } if (src0->type == GGML_TYPE_F16 && dst->type == GGML_TYPE_F16) { - return &vk_pipeline_rope_f16; + return &ctx->pipeline_rope_f16; } } return nullptr; @@ -2735,13 +2926,8 @@ static ggml_vk_func_t ggml_vk_op_get_func(ggml_op op) { } } -#ifdef GGML_VULKAN_CHECK_RESULTS -static void ggml_vk_print_tensor(const ggml_tensor * tensor, const char * name); -static void ggml_vk_check_results_0(ggml_compute_params * params, ggml_tensor * tensor); -#endif - template -static void ggml_vk_op_f32(vk_context * ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, ggml_op op, const PC&& pc) { +static void ggml_vk_op_f32(ggml_backend_vk_context * ctx, vk_context * subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, ggml_op op, const PC&& pc) { #ifdef GGML_VULKAN_DEBUG std::cerr << "ggml_vk_op_f32((" << src0 << ", name=" << src0->name << ", type=" << src0->type << ", backend=" << src0->backend << ", ne0=" << src0->ne[0] << ", ne1=" << src0->ne[1] << ", ne2=" << src0->ne[2] << ", ne3=" << src0->ne[3] << ", nb0=" << src0->nb[0] << ", nb1=" << src0->nb[1] << ", nb2=" << src0->nb[2] << ", nb3=" << src0->nb[3]; if (src1 != nullptr) { @@ -2768,7 +2954,7 @@ static void ggml_vk_op_f32(vk_context * ctx, const ggml_tensor * src0, const ggm const uint64_t nb2 = dst->nb[2]; const uint64_t nb3 = dst->nb[3]; - vk_pipeline * pipeline = ggml_vk_op_get_pipeline(src0, src1, dst, op); + vk_pipeline * pipeline = ggml_vk_op_get_pipeline(ctx, src0, src1, dst, op); ggml_vk_func_t op_func; if (pipeline == nullptr) { @@ -2782,7 +2968,7 @@ static void ggml_vk_op_f32(vk_context * ctx, const ggml_tensor * src0, const ggm GGML_ASSERT(false); } - op_func(ctx, src0, src1, dst); + op_func(ctx, subctx, src0, src1, dst); return; } @@ -2790,19 +2976,19 @@ static void ggml_vk_op_f32(vk_context * ctx, const ggml_tensor * src0, const ggm ggml_tensor_extra_gpu * extra_src0 = (ggml_tensor_extra_gpu *) src0->extra; ggml_tensor_extra_gpu * extra_src1 = use_src1 ? (ggml_tensor_extra_gpu *) src1->extra : nullptr; - vk_buffer * d_X = nullptr; + vk_buffer d_X = nullptr; size_t x_buf_offset = 0; - vk_buffer * d_Y = nullptr; + vk_buffer d_Y = nullptr; size_t y_buf_offset = 0; bool src0_uma = false; bool src1_uma = false; - if (vk_device.uma) { - ggml_vk_host_get(src0->data, d_X, x_buf_offset); + if (ctx->device.lock()->uma) { + ggml_vk_host_get(ctx, src0->data, d_X, x_buf_offset); src0_uma = d_X != nullptr; if (use_src1) { - ggml_vk_host_get(src1->data, d_Y, y_buf_offset); + ggml_vk_host_get(ctx, src1->data, d_Y, y_buf_offset); src1_uma = d_Y != nullptr; } } @@ -2810,30 +2996,31 @@ static void ggml_vk_op_f32(vk_context * ctx, const ggml_tensor * src0, const ggm const bool transfer_src0 = src0->backend != GGML_BACKEND_GPU && !src0_uma; const bool transfer_src1 = use_src1 && src1->backend != GGML_BACKEND_GPU && !src1_uma; - uint64_t x_sz = ggml_vk_align_size(ggml_type_size(src0->type) * ne0, vk_device.properties.limits.minStorageBufferOffsetAlignment); - uint64_t y_sz = use_src1 ? ggml_vk_align_size(ggml_type_size(src1->type) * ne1, vk_device.properties.limits.minStorageBufferOffsetAlignment) : 0; + uint64_t x_sz = ggml_vk_align_size(ggml_type_size(src0->type) * ne0, ctx->device.lock()->properties.limits.minStorageBufferOffsetAlignment); + uint64_t y_sz = use_src1 ? ggml_vk_align_size(ggml_type_size(src1->type) * ne1, ctx->device.lock()->properties.limits.minStorageBufferOffsetAlignment) : 0; uint64_t d_sz = ggml_type_size(dst->type) * ne0; + vk_buffer d_D = extra->buffer_gpu.lock(); + // Workaround for tiny tensor inputs on ROPE - if (use_src1 && src1->backend == GGML_BACKEND_GPU && y_sz > extra_src1->buffer_gpu.size) { + if (use_src1 && src1->backend == GGML_BACKEND_GPU && y_sz > d_D->size) { y_sz = VK_WHOLE_SIZE; } - vk_buffer* d_D = &extra->buffer_gpu; GGML_ASSERT(d_D != nullptr); - uint64_t d_buf_offset = (extra->offset / vk_device.properties.limits.minStorageBufferOffsetAlignment) * vk_device.properties.limits.minStorageBufferOffsetAlignment; + uint64_t d_buf_offset = (extra->offset / ctx->device.lock()->properties.limits.minStorageBufferOffsetAlignment) * ctx->device.lock()->properties.limits.minStorageBufferOffsetAlignment; GGML_ASSERT(d_buf_offset == extra->offset || op == GGML_OP_CPY); // NOLINT if (transfer_src0) { - d_X = &vk_prealloc_qx; + d_X = ctx->prealloc_qx; } else if(!src0_uma) { - d_X = &extra_src0->buffer_gpu; + d_X = extra_src0->buffer_gpu.lock(); x_buf_offset = extra_src0->offset; GGML_ASSERT(d_X != nullptr); } if (transfer_src1) { - d_Y = &vk_prealloc_qy; + d_Y = ctx->prealloc_qy; } else if (use_src1 && !src1_uma) { - d_Y = &extra_src1->buffer_gpu; + d_Y = extra_src1->buffer_gpu.lock(); y_buf_offset = extra_src1->offset; GGML_ASSERT(d_Y != nullptr); } @@ -2856,16 +3043,16 @@ static void ggml_vk_op_f32(vk_context * ctx, const ggml_tensor * src0, const ggm // copy src0 to device if (transfer_src0) { - ggml_vk_h2d_tensor_2d(ctx, d_X, 0, src0, 0, 0, ggml_nrows(src0)); - vk_staging_offset = x_sz * ne02 * ne03; + ggml_vk_h2d_tensor_2d(ctx, subctx, d_X, 0, src0, 0, 0, ggml_nrows(src0)); + ctx->staging_offset = x_sz * ne02 * ne03; } if (transfer_src1) { - ggml_vk_h2d_tensor_2d(ctx, d_Y, 0, src1, 0, 0, ggml_nrows(src1)); + ggml_vk_h2d_tensor_2d(ctx, subctx, d_Y, 0, src1, 0, 0, ggml_nrows(src1)); } // Single call if dimension 2 is contiguous if (op == GGML_OP_CPY || (ggml_is_contiguous(src0) && (src1 == nullptr || ggml_is_contiguous(src1)))) { - ggml_vk_pipeline_allocate_descriptor_sets(*pipeline, 1); + ggml_pipeline_allocate_descriptor_sets(ctx, *pipeline, 1); switch (dst->op) { case GGML_OP_NORM: @@ -2896,24 +3083,24 @@ static void ggml_vk_op_f32(vk_context * ctx, const ggml_tensor * src0, const ggm if (!use_src1 && op == GGML_OP_SOFT_MAX) { // Empty src1 is possible on soft_max, but the shader needs a buffer - ggml_vk_sync_buffers(ctx); - ggml_vk_dispatch_pipeline(ctx, *pipeline, { { *d_X, x_buf_offset, x_sz }, { vk_prealloc_y, 0, vk_prealloc_y.size }, { *d_D, d_buf_offset, d_sz } }, sizeof(PC), &pc, elements); + ggml_vk_sync_buffers(subctx); + ggml_vk_dispatch_pipeline(ctx, subctx, *pipeline, { { d_X, x_buf_offset, x_sz }, { ctx->prealloc_y, 0, ctx->prealloc_y->size }, { d_D, d_buf_offset, d_sz } }, sizeof(PC), &pc, elements); } else if (use_src1) { - ggml_vk_sync_buffers(ctx); - ggml_vk_dispatch_pipeline(ctx, *pipeline, { { *d_X, x_buf_offset, x_sz }, { *d_Y, y_buf_offset, y_sz }, { *d_D, d_buf_offset, d_sz } }, sizeof(PC), &pc, elements); + ggml_vk_sync_buffers(subctx); + ggml_vk_dispatch_pipeline(ctx, subctx, *pipeline, { { d_X, x_buf_offset, x_sz }, { d_Y, y_buf_offset, y_sz }, { d_D, d_buf_offset, d_sz } }, sizeof(PC), &pc, elements); } else { - ggml_vk_sync_buffers(ctx); - ggml_vk_dispatch_pipeline(ctx, *pipeline, { { *d_X, x_buf_offset, x_sz }, { *d_D, d_buf_offset, d_sz } }, sizeof(PC), &pc, elements); + ggml_vk_sync_buffers(subctx); + ggml_vk_dispatch_pipeline(ctx, subctx, *pipeline, { { d_X, x_buf_offset, x_sz }, { d_D, d_buf_offset, d_sz } }, sizeof(PC), &pc, elements); } if (dst->backend == GGML_BACKEND_CPU && op == GGML_OP_CPY) { - ggml_vk_d2h_tensor_2d(ctx, d_D, 0, dst); + ggml_vk_d2h_tensor_2d(ctx, subctx, d_D, 0, dst); } else if(dst->backend == GGML_BACKEND_CPU) { // copy dst to host float * d = (float *) dst->data; - ggml_vk_buffer_read_async(ctx, d_D, 0, d, d_sz); + ggml_vk_buffer_read_async(ctx, subctx, d_D, 0, d, d_sz); } } else { - ggml_vk_pipeline_allocate_descriptor_sets(*pipeline, ne02 * ne03); + ggml_pipeline_allocate_descriptor_sets(ctx, *pipeline, ne02 * ne03); switch (dst->op) { case GGML_OP_NORM: @@ -2940,60 +3127,60 @@ static void ggml_vk_op_f32(vk_context * ctx, const ggml_tensor * src0, const ggm if (!use_src1 && op == GGML_OP_SOFT_MAX) { // Empty src1 is possible on soft_max, but the shader needs a buffer - ggml_vk_sync_buffers(ctx); - ggml_vk_dispatch_pipeline(ctx, *pipeline, { { *d_X, x_buf_offset, x_sz }, { vk_prealloc_y, 0, vk_prealloc_y.size }, { *d_D, d_buf_offset, d_sz } }, sizeof(PC), &pc, elements); + ggml_vk_sync_buffers(subctx); + ggml_vk_dispatch_pipeline(ctx, subctx, *pipeline, { { d_X, x_buf_offset, x_sz }, { ctx->prealloc_y, 0, ctx->prealloc_y->size }, { d_D, d_buf_offset, d_sz } }, sizeof(PC), &pc, elements); } else if (use_src1) { - ggml_vk_sync_buffers(ctx); - ggml_vk_dispatch_pipeline(ctx, *pipeline, { { *d_X, x_buf_offset + x_offset, x_sz }, { *d_Y, y_buf_offset + y_offset, y_sz }, { *d_D, d_buf_offset + d_offset, d_sz } }, sizeof(PC), &pc, elements); + ggml_vk_sync_buffers(subctx); + ggml_vk_dispatch_pipeline(ctx, subctx, *pipeline, { { d_X, x_buf_offset + x_offset, x_sz }, { d_Y, y_buf_offset + y_offset, y_sz }, { d_D, d_buf_offset + d_offset, d_sz } }, sizeof(PC), &pc, elements); } else { - ggml_vk_sync_buffers(ctx); - ggml_vk_dispatch_pipeline(ctx, *pipeline, { { *d_X, x_buf_offset + x_offset, x_sz }, { *d_D, d_buf_offset + d_offset, d_sz } }, sizeof(PC), &pc, elements); + ggml_vk_sync_buffers(subctx); + ggml_vk_dispatch_pipeline(ctx, subctx, *pipeline, { { d_X, x_buf_offset + x_offset, x_sz }, { d_D, d_buf_offset + d_offset, d_sz } }, sizeof(PC), &pc, elements); } if (dst->backend == GGML_BACKEND_CPU) { // copy dst to host - ggml_vk_buffer_read_async(ctx, d_D, d_buf_offset + d_offset, (char *) dst->data + i02*nb2 + i03*nb3, d_sz); + ggml_vk_buffer_read_async(ctx, subctx, d_D, d_buf_offset + d_offset, (char *) dst->data + i02*nb2 + i03*nb3, d_sz); } } } } } -static void ggml_vk_repeat(vk_context * ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { - ggml_vk_op_f32(ctx, src0, src1, dst, GGML_OP_REPEAT, { (uint32_t)ggml_nelements(src0), (uint32_t)ggml_nelements(src1), 0.0f, 0.0f }); +static void ggml_vk_repeat(ggml_backend_vk_context * ctx, vk_context * subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + ggml_vk_op_f32(ctx, subctx, src0, src1, dst, GGML_OP_REPEAT, { (uint32_t)ggml_nelements(src0), (uint32_t)ggml_nelements(src1), 0.0f, 0.0f }); } -static void ggml_vk_get_rows(vk_context * ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { - ggml_vk_op_f32(ctx, src0, src1, dst, GGML_OP_GET_ROWS, { (uint32_t)ggml_nelements(src0), (uint32_t)ggml_nelements(src1), 0.0f, 0.0f }); +static void ggml_vk_get_rows(ggml_backend_vk_context * ctx, vk_context * subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + ggml_vk_op_f32(ctx, subctx, src0, src1, dst, GGML_OP_GET_ROWS, { (uint32_t)ggml_nelements(src0), (uint32_t)ggml_nelements(src1), 0.0f, 0.0f }); } -static void ggml_vk_add(vk_context * ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { - ggml_vk_op_f32(ctx, src0, src1, dst, GGML_OP_ADD, { (uint32_t)ggml_nelements(src0), (uint32_t)ggml_nelements(src1), 0.0f, 0.0f }); +static void ggml_vk_add(ggml_backend_vk_context * ctx, vk_context * subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + ggml_vk_op_f32(ctx, subctx, src0, src1, dst, GGML_OP_ADD, { (uint32_t)ggml_nelements(src0), (uint32_t)ggml_nelements(src1), 0.0f, 0.0f }); } -static void ggml_vk_mul(vk_context * ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { - ggml_vk_op_f32(ctx, src0, src1, dst, GGML_OP_MUL, { (uint32_t)ggml_nelements(src0), (uint32_t)ggml_nelements(src1), 0.0f, 0.0f }); +static void ggml_vk_mul(ggml_backend_vk_context * ctx, vk_context * subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + ggml_vk_op_f32(ctx, subctx, src0, src1, dst, GGML_OP_MUL, { (uint32_t)ggml_nelements(src0), (uint32_t)ggml_nelements(src1), 0.0f, 0.0f }); } -static void ggml_vk_scale(vk_context * ctx, const ggml_tensor * src0, ggml_tensor * dst) { +static void ggml_vk_scale(ggml_backend_vk_context * ctx, vk_context * subctx, const ggml_tensor * src0, ggml_tensor * dst) { float * op_params = (float *)dst->op_params; - ggml_vk_op_f32(ctx, src0, nullptr, dst, GGML_OP_SCALE, { (uint32_t)ggml_nelements(src0), 0, op_params[0], 0.0f }); + ggml_vk_op_f32(ctx, subctx, src0, nullptr, dst, GGML_OP_SCALE, { (uint32_t)ggml_nelements(src0), 0, op_params[0], 0.0f }); } -static void ggml_vk_sqr(vk_context * ctx, const ggml_tensor * src0, ggml_tensor * dst) { - ggml_vk_op_f32(ctx, src0, nullptr, dst, GGML_OP_SQR, { (uint32_t)ggml_nelements(src0), 0, 0.0f, 0.0f }); +static void ggml_vk_sqr(ggml_backend_vk_context * ctx, vk_context * subctx, const ggml_tensor * src0, ggml_tensor * dst) { + ggml_vk_op_f32(ctx, subctx, src0, nullptr, dst, GGML_OP_SQR, { (uint32_t)ggml_nelements(src0), 0, 0.0f, 0.0f }); } -static void ggml_vk_clamp(vk_context * ctx, const ggml_tensor * src0, ggml_tensor * dst) { +static void ggml_vk_clamp(ggml_backend_vk_context * ctx, vk_context * subctx, const ggml_tensor * src0, ggml_tensor * dst) { float * op_params = (float *)dst->op_params; - ggml_vk_op_f32(ctx, src0, nullptr, dst, GGML_OP_CLAMP, { (uint32_t)ggml_nelements(src0), 0, op_params[0], op_params[1] }); + ggml_vk_op_f32(ctx, subctx, src0, nullptr, dst, GGML_OP_CLAMP, { (uint32_t)ggml_nelements(src0), 0, op_params[0], op_params[1] }); } -static void ggml_vk_cpy(vk_context * ctx, const ggml_tensor * src0, ggml_tensor * dst) { +static void ggml_vk_cpy(ggml_backend_vk_context * ctx, vk_context * subctx, const ggml_tensor * src0, ggml_tensor * dst) { ggml_tensor_extra_gpu * extra = (ggml_tensor_extra_gpu *) dst->extra; const int src0_type_size = ggml_type_size(src0->type); const int dst_type_size = ggml_type_size(dst->type); - const uint32_t d_offset = (extra->offset % vk_device.properties.limits.minStorageBufferOffsetAlignment) / dst_type_size; - ggml_vk_op_f32(ctx, src0, nullptr, dst, GGML_OP_CPY, { + const uint32_t d_offset = (extra->offset % ctx->device.lock()->properties.limits.minStorageBufferOffsetAlignment) / dst_type_size; + ggml_vk_op_f32(ctx, subctx, src0, nullptr, dst, GGML_OP_CPY, { (uint32_t)ggml_nelements(src0), (uint32_t)src0->ne[0], (uint32_t)src0->ne[1], (uint32_t)src0->nb[0] / src0_type_size, (uint32_t)src0->nb[1] / src0_type_size, (uint32_t)src0->nb[2] / src0_type_size, (uint32_t) dst->ne[0], (uint32_t) dst->ne[1], (uint32_t) dst->nb[0] / dst_type_size, (uint32_t) dst->nb[1] / dst_type_size, (uint32_t) dst->nb[2] / dst_type_size, @@ -3001,30 +3188,30 @@ static void ggml_vk_cpy(vk_context * ctx, const ggml_tensor * src0, ggml_tensor }); } -static void ggml_vk_norm(vk_context * ctx, const ggml_tensor * src0, ggml_tensor * dst) { - ggml_vk_op_f32(ctx, src0, nullptr, dst, GGML_OP_NORM, { (uint32_t)src0->ne[0], (uint32_t)src0->ne[1], 0.0f, 0.0f }); +static void ggml_vk_norm(ggml_backend_vk_context * ctx, vk_context * subctx, const ggml_tensor * src0, ggml_tensor * dst) { + ggml_vk_op_f32(ctx, subctx, src0, nullptr, dst, GGML_OP_NORM, { (uint32_t)src0->ne[0], (uint32_t)src0->ne[1], 0.0f, 0.0f }); } -static void ggml_vk_rms_norm(vk_context * ctx, const ggml_tensor * src0, ggml_tensor * dst) { +static void ggml_vk_rms_norm(ggml_backend_vk_context * ctx, vk_context * subctx, const ggml_tensor * src0, ggml_tensor * dst) { float * op_params = (float *)dst->op_params; - ggml_vk_op_f32(ctx, src0, nullptr, dst, GGML_OP_RMS_NORM, { (uint32_t)src0->ne[0], (uint32_t)src0->ne[1], op_params[0], 0.0f }); + ggml_vk_op_f32(ctx, subctx, src0, nullptr, dst, GGML_OP_RMS_NORM, { (uint32_t)src0->ne[0], (uint32_t)src0->ne[1], op_params[0], 0.0f }); } -static void ggml_vk_unary(vk_context * ctx, const ggml_tensor * src0, ggml_tensor * dst) { - ggml_vk_op_f32(ctx, src0, nullptr, dst, GGML_OP_UNARY, { (uint32_t)ggml_nelements(src0), 0, 0.0f, 0.0f }); +static void ggml_vk_unary(ggml_backend_vk_context * ctx, vk_context * subctx, const ggml_tensor * src0, ggml_tensor * dst) { + ggml_vk_op_f32(ctx, subctx, src0, nullptr, dst, GGML_OP_UNARY, { (uint32_t)ggml_nelements(src0), 0, 0.0f, 0.0f }); } -static void ggml_vk_diag_mask_inf(vk_context * ctx, const ggml_tensor * src0, ggml_tensor * dst) { +static void ggml_vk_diag_mask_inf(ggml_backend_vk_context * ctx, vk_context * subctx, const ggml_tensor * src0, ggml_tensor * dst) { int32_t * op_params = (int32_t *)dst->op_params; - ggml_vk_op_f32(ctx, src0, nullptr, dst, GGML_OP_DIAG_MASK_INF, { (uint32_t)src0->ne[0], (uint32_t)src0->ne[1], op_params[0] }); + ggml_vk_op_f32(ctx, subctx, src0, nullptr, dst, GGML_OP_DIAG_MASK_INF, { (uint32_t)src0->ne[0], (uint32_t)src0->ne[1], op_params[0] }); } -static void ggml_vk_soft_max(vk_context * ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { +static void ggml_vk_soft_max(ggml_backend_vk_context * ctx, vk_context * subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { float * op_params = (float *)dst->op_params; - ggml_vk_op_f32(ctx, src0, src1, dst, GGML_OP_SOFT_MAX, { (uint32_t)src0->ne[0], (uint32_t)(src1 != nullptr ? ggml_nrows(src1) : 0), op_params[0], 0.0f }); + ggml_vk_op_f32(ctx, subctx, src0, src1, dst, GGML_OP_SOFT_MAX, { (uint32_t)src0->ne[0], (uint32_t)(src1 != nullptr ? ggml_nrows(src1) : 0), op_params[0], 0.0f }); } -static void ggml_vk_rope(vk_context * ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { +static void ggml_vk_rope(ggml_backend_vk_context * ctx, vk_context * subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { const int n_dims = ((int32_t *) dst->op_params)[1]; const int mode = ((int32_t *) dst->op_params)[2]; // const int n_ctx = ((int32_t *) dst->op_params)[3]; @@ -3047,19 +3234,19 @@ static void ggml_vk_rope(vk_context * ctx, const ggml_tensor * src0, const ggml_ if (is_neox) { const float theta_scale = powf(freq_base, -2.0f/n_dims); const float inv_ndims = -1.0f / n_dims; - ggml_vk_op_f32(ctx, src0, src1, dst, GGML_OP_ROPE, { (uint32_t)src0->ne[0], (uint32_t)n_dims, freq_scale, (uint32_t)src0->ne[1], freq_base, ext_factor, attn_factor, corr_dims[0], corr_dims[1], 0.0f, 0.0f, theta_scale, inv_ndims }); + ggml_vk_op_f32(ctx, subctx, src0, src1, dst, GGML_OP_ROPE, { (uint32_t)src0->ne[0], (uint32_t)n_dims, freq_scale, (uint32_t)src0->ne[1], freq_base, ext_factor, attn_factor, corr_dims[0], corr_dims[1], 0.0f, 0.0f, theta_scale, inv_ndims }); } else { - ggml_vk_op_f32(ctx, src0, src1, dst, GGML_OP_ROPE, { (uint32_t)src0->ne[0], freq_scale, (uint32_t)src0->ne[1], freq_base, ext_factor, attn_factor, corr_dims[0], corr_dims[1], 0.0f, 0.0f }); + ggml_vk_op_f32(ctx, subctx, src0, src1, dst, GGML_OP_ROPE, { (uint32_t)src0->ne[0], freq_scale, (uint32_t)src0->ne[1], freq_base, ext_factor, attn_factor, corr_dims[0], corr_dims[1], 0.0f, 0.0f }); } } -static void ggml_vk_nop(vk_context * ctx, const ggml_tensor * src0, ggml_tensor * dst) { +static void ggml_vk_nop(ggml_backend_vk_context * ctx, vk_context * subctx, const ggml_tensor * src0, ggml_tensor * dst) { // If backend is CPU, data from src0 has to be copied off the device if (dst->backend == GGML_BACKEND_CPU) { ggml_tensor_extra_gpu * extra_src0 = (ggml_tensor_extra_gpu *) src0->extra; - vk_buffer * d_D = &extra_src0->buffer_gpu; - ggml_vk_sync_buffers(ctx); - ggml_vk_buffer_read_async(ctx, d_D, 0, dst->data, d_D->size); + vk_buffer d_D = extra_src0->buffer_gpu.lock(); + ggml_vk_sync_buffers(subctx); + ggml_vk_buffer_read_async(ctx, subctx, d_D, 0, dst->data, d_D->size); } } @@ -3096,7 +3283,7 @@ static void ggml_vk_print_matrix_area(const void * data, ggml_type type, int ne0 } template -static void ggml_vk_test_matmul(size_t m, size_t n, size_t k, size_t batch, size_t num_it, int split_k, int shader_size) { +static void ggml_vk_test_matmul(ggml_backend_vk_context * ctx, size_t m, size_t n, size_t k, size_t batch, size_t num_it, int split_k, int shader_size) { #ifdef GGML_VULKAN_DEBUG std::cerr << "ggml_vk_test_matmul(" << m << ", " << n << ", " << k << ", " << batch << ", " << num_it << ", " << split_k << ", " << shader_size << ")" << std::endl; #endif @@ -3108,39 +3295,39 @@ static void ggml_vk_test_matmul(size_t m, size_t n, size_t k, size_t batch, size std::string shname; if (shader_size == 0) { if (std::is_same() && std::is_same()) { - p = &vk_pipeline_matmul_f32_aligned_s; + p = &ctx->pipeline_matmul_f32_aligned_s; shname = "F32_ALIGNED_S"; } else if (std::is_same() && std::is_same()) { - p = &vk_pipeline_matmul_f16_f32_aligned_s; + p = &ctx->pipeline_matmul_f16_f32_aligned_s; shname = "F16_F32_ALIGNED_S"; } else if (std::is_same() && std::is_same()) { - p = &vk_pipeline_matmul_f16_aligned_s; + p = &ctx->pipeline_matmul_f16_aligned_s; shname = "F16_ALIGNED_S"; } else { GGML_ASSERT(false); } } else if (shader_size == 1) { if (std::is_same() && std::is_same()) { - p = &vk_pipeline_matmul_f32_aligned_m; + p = &ctx->pipeline_matmul_f32_aligned_m; shname = "F32_ALIGNED_M"; } else if (std::is_same() && std::is_same()) { - p = &vk_pipeline_matmul_f16_f32_aligned_m; + p = &ctx->pipeline_matmul_f16_f32_aligned_m; shname = "F16_F32_ALIGNED_M"; } else if (std::is_same() && std::is_same()) { - p = &vk_pipeline_matmul_f16_aligned_m; + p = &ctx->pipeline_matmul_f16_aligned_m; shname = "F16_ALIGNED_M"; } else { GGML_ASSERT(false); } } else if (shader_size == 2) { if (std::is_same() && std::is_same()) { - p = &vk_pipeline_matmul_f32_aligned_l; + p = &ctx->pipeline_matmul_f32_aligned_l; shname = "F32_ALIGNED_L"; } else if (std::is_same() && std::is_same()) { - p = &vk_pipeline_matmul_f16_f32_aligned_l; + p = &ctx->pipeline_matmul_f16_f32_aligned_l; shname = "F16_F32_ALIGNED_L"; } else if (std::is_same() && std::is_same()) { - p = &vk_pipeline_matmul_f16_aligned_l; + p = &ctx->pipeline_matmul_f16_aligned_l; shname = "F16_ALIGNED_L"; } else { GGML_ASSERT(false); @@ -3154,56 +3341,56 @@ static void ggml_vk_test_matmul(size_t m, size_t n, size_t k, size_t batch, size if (k != kpad) { if (shader_size == 0) { if (std::is_same() && std::is_same()) { - p = &vk_pipeline_matmul_f32_s; + p = &ctx->pipeline_matmul_f32_s; shname = "F32_S"; } else if (std::is_same() && std::is_same()) { - p = &vk_pipeline_matmul_f16_f32_s; + p = &ctx->pipeline_matmul_f16_f32_s; shname = "F16_F32_S"; } else if (std::is_same() && std::is_same()) { - p = &vk_pipeline_matmul_f16_s; + p = &ctx->pipeline_matmul_f16_s; shname = "F16_S"; } } else if (shader_size == 1) { if (std::is_same() && std::is_same()) { - p = &vk_pipeline_matmul_f32_m; + p = &ctx->pipeline_matmul_f32_m; shname = "F32_M"; } else if (std::is_same() && std::is_same()) { - p = &vk_pipeline_matmul_f16_f32_m; + p = &ctx->pipeline_matmul_f16_f32_m; shname = "F16_F32_M"; } else if (std::is_same() && std::is_same()) { - p = &vk_pipeline_matmul_f16_m; + p = &ctx->pipeline_matmul_f16_m; shname = "F16_M"; } } else if (shader_size == 2) { if (std::is_same() && std::is_same()) { - p = &vk_pipeline_matmul_f32_l; + p = &ctx->pipeline_matmul_f32_l; shname = "F32_L"; } else if (std::is_same() && std::is_same()) { - p = &vk_pipeline_matmul_f16_f32_l; + p = &ctx->pipeline_matmul_f16_f32_l; shname = "F16_F32_L"; } else if (std::is_same() && std::is_same()) { - p = &vk_pipeline_matmul_f16_l; + p = &ctx->pipeline_matmul_f16_l; shname = "F16_L"; } } } - ggml_vk_pipeline_allocate_descriptor_sets(*p, num_it); + ggml_pipeline_allocate_descriptor_sets(ctx, *p, num_it); if (split_k > 1) { - ggml_vk_pipeline_allocate_descriptor_sets(vk_pipeline_matmul_split_k_reduce, num_it); + ggml_pipeline_allocate_descriptor_sets(ctx, ctx->pipeline_matmul_split_k_reduce, num_it); - if (vk_prealloc_split_k.size < sizeof(float) * d_ne * split_k) { + if (ctx->prealloc_split_k == nullptr || ctx->prealloc_split_k->size < sizeof(float) * d_ne * split_k) { // Resize buffer - if (vk_prealloc_split_k.size > 0) { - ggml_vk_destroy_buffer(vk_prealloc_split_k); + if (ctx->prealloc_split_k != nullptr) { + ggml_vk_destroy_buffer(ctx->prealloc_split_k); } - vk_prealloc_split_k = ggml_vk_create_buffer_check(sizeof(float) * d_ne * split_k, vk::MemoryPropertyFlagBits::eDeviceLocal); + ctx->prealloc_split_k = ggml_vk_create_buffer_check(ctx, sizeof(float) * d_ne * split_k, vk::MemoryPropertyFlagBits::eDeviceLocal); } } - vk_buffer d_X = ggml_vk_create_buffer_check(sizeof(X_TYPE) * x_ne, vk::MemoryPropertyFlagBits::eDeviceLocal); - vk_buffer d_Y = ggml_vk_create_buffer_check(sizeof(Y_TYPE) * y_ne, vk::MemoryPropertyFlagBits::eDeviceLocal); - vk_buffer d_D = ggml_vk_create_buffer_check(sizeof(float) * d_ne, vk::MemoryPropertyFlagBits::eDeviceLocal); + vk_buffer d_X = ggml_vk_create_buffer_check(ctx, sizeof(X_TYPE) * x_ne, vk::MemoryPropertyFlagBits::eDeviceLocal); + vk_buffer d_Y = ggml_vk_create_buffer_check(ctx, sizeof(Y_TYPE) * y_ne, vk::MemoryPropertyFlagBits::eDeviceLocal); + vk_buffer d_D = ggml_vk_create_buffer_check(ctx, sizeof(float) * d_ne, vk::MemoryPropertyFlagBits::eDeviceLocal); X_TYPE* x = (X_TYPE *) malloc(sizeof(X_TYPE) * x_ne); Y_TYPE* y = (Y_TYPE *) malloc(sizeof(Y_TYPE) * y_ne); @@ -3228,26 +3415,26 @@ static void ggml_vk_test_matmul(size_t m, size_t n, size_t k, size_t batch, size } } - ggml_vk_buffer_write(&d_X, 0, x, sizeof(X_TYPE) * k * m * batch); - ggml_vk_buffer_write(&d_Y, 0, y, sizeof(Y_TYPE) * k * n * batch); + ggml_vk_buffer_write(ctx, d_X, 0, x, sizeof(X_TYPE) * k * m * batch); + ggml_vk_buffer_write(ctx, d_Y, 0, y, sizeof(Y_TYPE) * k * n * batch); - vk_context * ctx = ggml_vk_create_context(vk_device.compute_queue); + vk_context * subctx = ggml_vk_create_context(ctx, ctx->device.lock()->compute_queue); for (size_t i = 0; i < num_it; i++) { - ggml_vk_ctx_begin(ctx); - ggml_vk_matmul(ctx, *p, ggml_vk_subbuffer(d_X), ggml_vk_subbuffer(d_Y), ggml_vk_subbuffer(d_D), ggml_vk_subbuffer(vk_prealloc_split_k), m, n, k, k, k, m, split_k, batch, batch, batch, 1, 1, k*m, k*n, m*n); - ggml_vk_ctx_end(ctx); + ggml_vk_ctx_begin(ctx, subctx); + ggml_vk_matmul(ctx, subctx, *p, ggml_vk_subbuffer(d_X), ggml_vk_subbuffer(d_Y), ggml_vk_subbuffer(d_D), ggml_vk_subbuffer(ctx->prealloc_split_k), m, n, k, k, k, m, split_k, batch, batch, batch, 1, 1, k*m, k*n, m*n); + ggml_vk_ctx_end(subctx); } auto begin = std::chrono::high_resolution_clock::now(); - ggml_vk_submit(ctx, vk_fence); - VK_CHECK(vk_device.device.waitForFences({ vk_fence }, true, UINT64_MAX), "ggml_vk_test_matmul waitForFences"); - vk_device.device.resetFences({ vk_fence }); + ggml_vk_submit(subctx, ctx->fence); + VK_CHECK(ctx->device.lock()->device.waitForFences({ ctx->fence }, true, UINT64_MAX), "ggml_vk_test_matmul waitForFences"); + ctx->device.lock()->device.resetFences({ ctx->fence }); auto end = std::chrono::high_resolution_clock::now(); double time = std::chrono::duration_cast(end-begin).count() / 1000.0; // copy dst to host - ggml_vk_buffer_read(&d_D, 0, d, sizeof(float) * d_ne); + ggml_vk_buffer_read(ctx, d_D, 0, d, sizeof(float) * d_ne); float * d_chk = (float *) malloc(sizeof(float) * d_ne); @@ -3285,14 +3472,14 @@ static void ggml_vk_test_matmul(size_t m, size_t n, size_t k, size_t batch, size src1_ggml->data = y; tensor_ggml->data = d_chk; - vk_disable = true; + ctx->disable = true; ggml_cgraph * cgraph = ggml_new_graph(ggml_ctx); ggml_build_forward_expand(cgraph, tensor_ggml); ggml_graph_compute_with_ctx(ggml_ctx, cgraph, 1); - vk_disable = false; + ctx->disable = false; ggml_free(ggml_ctx); @@ -3325,7 +3512,7 @@ static void ggml_vk_test_matmul(size_t m, size_t n, size_t k, size_t batch, size if (split_k > 1) { float * split_k_buf = (float *) malloc(sizeof(float) * d_ne * split_k); - ggml_vk_buffer_read(&vk_prealloc_split_k, 0, split_k_buf, sizeof(float) * d_ne * split_k); + ggml_vk_buffer_read(ctx, ctx->prealloc_split_k, 0, split_k_buf, sizeof(float) * d_ne * split_k); std::cerr << "d_buf0: " << std::endl << std::endl; ggml_vk_print_matrix_area(split_k_buf, GGML_TYPE_F32, m, n, first_err_m, first_err_n, first_err_b); @@ -3345,15 +3532,15 @@ static void ggml_vk_test_matmul(size_t m, size_t n, size_t k, size_t batch, size free(d_chk); - ggml_vk_queue_cleanup(vk_device.transfer_queue); - ggml_vk_queue_cleanup(vk_device.compute_queue); + ggml_vk_queue_cleanup(ctx, ctx->device.lock()->transfer_queue); + ggml_vk_queue_cleanup(ctx, ctx->device.lock()->compute_queue); ggml_vk_destroy_buffer(d_X); ggml_vk_destroy_buffer(d_Y); ggml_vk_destroy_buffer(d_D); - ggml_vk_pipeline_cleanup(*p); - ggml_vk_pipeline_cleanup(vk_pipeline_matmul_split_k_reduce); + ggml_pipeline_cleanup(*p); + ggml_pipeline_cleanup(ctx->pipeline_matmul_split_k_reduce); free(x); free(y); @@ -3392,7 +3579,7 @@ static void ggml_vk_print_tensor_area(const ggml_tensor * tensor, int i0, int i1 } } -static void ggml_vk_test_h2d_nc(size_t ne0, size_t ne1, size_t ne2, size_t ne3) { +static void ggml_vk_test_h2d_nc(ggml_backend_vk_context * ctx, size_t ne0, size_t ne1, size_t ne2, size_t ne3) { const size_t ne = ne0 * ne1 * ne2 * ne3; ggml_init_params iparams = { @@ -3406,7 +3593,7 @@ static void ggml_vk_test_h2d_nc(size_t ne0, size_t ne1, size_t ne2, size_t ne3) ggml_tensor * tensor = ggml_new_tensor_4d(ggml_ctx, GGML_TYPE_F32, ne0, ne2, ne1, ne3); // NOLINT ggml_tensor * result_tensor = ggml_new_tensor_4d(ggml_ctx, GGML_TYPE_F32, ne0, ne1, ne2, ne3); - float * data = (float *) ggml_vk_host_malloc(ggml_nbytes(tensor)); + float * data = (float *) ggml_vk_host_malloc(ctx, ggml_nbytes(tensor)); tensor->data = data; float * result_data = (float *) malloc(ggml_nbytes(tensor)); @@ -3426,19 +3613,19 @@ static void ggml_vk_test_h2d_nc(size_t ne0, size_t ne1, size_t ne2, size_t ne3) data[i] = (rand() / (float)RAND_MAX) * 2.0f - 1.0f; } - vk_context * ctx = ggml_vk_create_context(vk_device.compute_queue); - ggml_vk_ctx_begin(ctx); + vk_context * subctx = ggml_vk_create_context(ctx, ctx->device.lock()->compute_queue); + ggml_vk_ctx_begin(ctx, subctx); - vk_buffer buffer = ggml_vk_create_buffer_check(ggml_nbytes(tensor), vk::MemoryPropertyFlagBits::eDeviceLocal); + vk_buffer buffer = ggml_vk_create_buffer_check(ctx, ggml_nbytes(tensor), vk::MemoryPropertyFlagBits::eDeviceLocal); - ggml_vk_h2d_tensor_2d(ctx, &buffer, 0, tensor, 0, 0, ggml_nrows(tensor)); + ggml_vk_h2d_tensor_2d(ctx, subctx, buffer, 0, tensor, 0, 0, ggml_nrows(tensor)); - ggml_vk_ctx_end(ctx); - ggml_vk_submit(ctx, vk_fence); - VK_CHECK(vk_device.device.waitForFences({ vk_fence }, true, UINT64_MAX), "ggml_vk_compute_forward waitForFences"); - vk_device.device.resetFences({ vk_fence }); + ggml_vk_ctx_end(subctx); + ggml_vk_submit(subctx, ctx->fence); + VK_CHECK(ctx->device.lock()->device.waitForFences({ ctx->fence }, true, UINT64_MAX), "ggml_vk_test_h2d_nc waitForFences"); + ctx->device.lock()->device.resetFences({ ctx->fence }); - ggml_vk_buffer_read(&buffer, 0, result_data, ggml_nbytes(tensor)); + ggml_vk_buffer_read(ctx, buffer, 0, result_data, ggml_nbytes(tensor)); double avg_err = 0.0; int first_err_i0 = -1; @@ -3483,22 +3670,22 @@ static void ggml_vk_test_h2d_nc(size_t ne0, size_t ne1, size_t ne2, size_t ne3) ggml_vk_destroy_buffer(buffer); - ggml_vk_host_free(data); + ggml_vk_host_free(ctx, data); free(result_data); } -static void ggml_vk_test_transfer(size_t ne, bool pinned) { +static void ggml_vk_test_transfer(ggml_backend_vk_context * ctx, size_t ne, bool pinned) { #ifdef GGML_VULKAN_DEBUG std::cerr << "ggml_vk_test_transfer(" << ne << ")" << std::endl; #endif // Check transfers are correct - vk_buffer buffer = ggml_vk_create_buffer_check(sizeof(float) * ne, vk::MemoryPropertyFlagBits::eDeviceLocal); + vk_buffer buffer = ggml_vk_create_buffer_check(ctx, sizeof(float) * ne, vk::MemoryPropertyFlagBits::eDeviceLocal); float * x; float * y; if (pinned) { - x = (float *) ggml_vk_host_malloc(sizeof(float) * ne); - y = (float *) ggml_vk_host_malloc(sizeof(float) * ne); + x = (float *) ggml_vk_host_malloc(ctx, sizeof(float) * ne); + y = (float *) ggml_vk_host_malloc(ctx, sizeof(float) * ne); } else { x = (float *) malloc(sizeof(float) * ne); y = (float *) malloc(sizeof(float) * ne); @@ -3508,42 +3695,42 @@ static void ggml_vk_test_transfer(size_t ne, bool pinned) { x[i] = rand() / (float)RAND_MAX; } - vk_context * ctx = ggml_vk_create_context(vk_device.compute_queue); - ggml_vk_ctx_begin(ctx); + vk_context * subctx = ggml_vk_create_context(ctx, ctx->device.lock()->compute_queue); + ggml_vk_ctx_begin(ctx, subctx); auto begin = std::chrono::high_resolution_clock::now(); - ggml_vk_buffer_write_async(ctx, &buffer, 0, x, sizeof(float) * ne); + ggml_vk_buffer_write_async(ctx, subctx, buffer, 0, x, sizeof(float) * ne); - for (auto& cpy : ctx->in_memcpys) { + for (auto& cpy : subctx->in_memcpys) { memcpy(cpy.dst, cpy.src, cpy.n); } - ctx->in_memcpys.clear(); + subctx->in_memcpys.clear(); - ggml_vk_ctx_end(ctx); - ggml_vk_submit(ctx, vk_fence); - VK_CHECK(vk_device.device.waitForFences({ vk_fence }, true, UINT64_MAX), "ggml_vk_compute_forward waitForFences"); - vk_device.device.resetFences({ vk_fence }); + ggml_vk_ctx_end(subctx); + ggml_vk_submit(subctx, ctx->fence); + VK_CHECK(ctx->device.lock()->device.waitForFences({ ctx->fence }, true, UINT64_MAX), "ggml_vk_test_transfer waitForFences"); + ctx->device.lock()->device.resetFences({ ctx->fence }); auto end = std::chrono::high_resolution_clock::now(); double ms_to_gpu = std::chrono::duration_cast(end-begin).count() / 1000.0; - ggml_vk_ctx_begin(ctx); + ggml_vk_ctx_begin(ctx, subctx); begin = std::chrono::high_resolution_clock::now(); - ggml_vk_buffer_read_async(ctx, &buffer, 0, y, sizeof(float) * ne); + ggml_vk_buffer_read_async(ctx, subctx, buffer, 0, y, sizeof(float) * ne); - ggml_vk_ctx_end(ctx); - ggml_vk_submit(ctx, vk_fence); - VK_CHECK(vk_device.device.waitForFences({ vk_fence }, true, UINT64_MAX), "ggml_vk_compute_forward waitForFences"); - vk_device.device.resetFences({ vk_fence }); + ggml_vk_ctx_end(subctx); + ggml_vk_submit(subctx, ctx->fence); + VK_CHECK(ctx->device.lock()->device.waitForFences({ ctx->fence }, true, UINT64_MAX), "ggml_vk_test_transfer waitForFences"); + ctx->device.lock()->device.resetFences({ ctx->fence }); - for (auto& cpy : ctx->out_memcpys) { + for (auto& cpy : subctx->out_memcpys) { memcpy(cpy.dst, cpy.src, cpy.n); } - ctx->out_memcpys.clear(); + subctx->out_memcpys.clear(); end = std::chrono::high_resolution_clock::now(); @@ -3561,15 +3748,15 @@ static void ggml_vk_test_transfer(size_t ne, bool pinned) { ggml_vk_destroy_buffer(buffer); if (pinned) { - ggml_vk_host_free(x); - ggml_vk_host_free(y); + ggml_vk_host_free(ctx, x); + ggml_vk_host_free(ctx, y); } else { free(x); free(y); } } -static void ggml_vk_test_dequant(size_t ne, ggml_type quant) { +static void ggml_vk_test_dequant(ggml_backend_vk_context * ctx, size_t ne, ggml_type quant) { #ifdef GGML_VULKAN_DEBUG std::cerr << "ggml_vk_test_dequant(" << ne << ")" << std::endl; #endif @@ -3578,8 +3765,8 @@ static void ggml_vk_test_dequant(size_t ne, ggml_type quant) { const size_t qx_sz = ne * ggml_type_size(quant)/ggml_blck_size(quant); float * x = (float *) malloc(x_sz); void * qx = malloc(qx_sz); - vk_buffer qx_buf = ggml_vk_create_buffer_check(qx_sz, vk::MemoryPropertyFlagBits::eDeviceLocal); - vk_buffer x_buf = ggml_vk_create_buffer_check(x_sz_f16, vk::MemoryPropertyFlagBits::eDeviceLocal); + vk_buffer qx_buf = ggml_vk_create_buffer_check(ctx, qx_sz, vk::MemoryPropertyFlagBits::eDeviceLocal); + vk_buffer x_buf = ggml_vk_create_buffer_check(ctx, x_sz_f16, vk::MemoryPropertyFlagBits::eDeviceLocal); ggml_fp16_t * x_chk = (ggml_fp16_t *) malloc(x_sz_f16); for (size_t i = 0; i < ne; i++) { @@ -3588,7 +3775,7 @@ static void ggml_vk_test_dequant(size_t ne, ggml_type quant) { std::vector hist_cur(1 << 4, 0); - vk_pipeline& p = vk_pipeline_dequant[quant]; + vk_pipeline& p = ctx->pipeline_dequant[quant]; switch(quant) { case GGML_TYPE_Q4_0: @@ -3625,27 +3812,26 @@ static void ggml_vk_test_dequant(size_t ne, ggml_type quant) { GGML_ASSERT(false); } - ggml_vk_pipeline_allocate_descriptor_sets(p, 1); + ggml_pipeline_allocate_descriptor_sets(ctx, p, 1); - ggml_vk_buffer_write(&qx_buf, 0, qx, qx_sz); + ggml_vk_buffer_write(ctx, qx_buf, 0, qx, qx_sz); - vk_context * ctx = ggml_vk_create_context(vk_device.compute_queue); - ggml_vk_ctx_begin(ctx); + vk_context * subctx = ggml_vk_create_context(ctx, ctx->device.lock()->compute_queue); + ggml_vk_ctx_begin(ctx, subctx); const std::vector pc = { 1, (int)ne, (int)ne, (int)ne }; - ggml_vk_sync_buffers(ctx); - ggml_vk_dispatch_pipeline(ctx, p, { { qx_buf, 0, qx_sz }, { x_buf, 0, x_sz_f16 } }, pc.size() * sizeof(int), pc.data(), { (uint32_t)ne, 1, 1}); - ggml_vk_ctx_end(ctx); + ggml_vk_dispatch_pipeline(ctx, subctx, p, { { qx_buf, 0, qx_sz }, { x_buf, 0, x_sz_f16 } }, pc.size() * sizeof(int), pc.data(), { (uint32_t)ne, 1, 1}); + ggml_vk_ctx_end(subctx); auto begin = std::chrono::high_resolution_clock::now(); - ggml_vk_submit(ctx, vk_fence); - VK_CHECK(vk_device.device.waitForFences({ vk_fence }, true, UINT64_MAX), "ggml_vk_compute_forward waitForFences"); - vk_device.device.resetFences({ vk_fence }); + ggml_vk_submit(subctx, ctx->fence); + VK_CHECK(ctx->device.lock()->device.waitForFences({ ctx->fence }, true, UINT64_MAX), "ggml_vk_test_dequant waitForFences"); + ctx->device.lock()->device.resetFences({ ctx->fence }); auto end = std::chrono::high_resolution_clock::now(); double ms_dequant = std::chrono::duration_cast(end-begin).count() / 1000.0; - ggml_vk_buffer_read(&x_buf, 0, x_chk, x_sz_f16); + ggml_vk_buffer_read(ctx, x_buf, 0, x_chk, x_sz_f16); double avg_err = 0.0; for (size_t i = 0; i < ne; i++) { @@ -3687,15 +3873,15 @@ static ggml_tensor * ggml_vk_find_last_use(const ggml_tensor * node, ggml_cgraph return nullptr; } -void ggml_vk_preallocate_buffers_graph(ggml_tensor * node){ +static void ggml_vk_preallocate_buffers_graph(ggml_backend_vk_context * ctx, ggml_tensor * node){ #ifdef GGML_VULKAN_DEBUG - std::cerr << "ggml_vk_preallocate_buffers_graph(" << node << ")" << std::endl; + std::cerr << "ggml_ctx->preallocate_buffers_graph(" << node << ")" << std::endl; #endif const bool any_on_device = node->backend == GGML_BACKEND_GPU || (node->src[0] != nullptr && (node->src[0]->backend == GGML_BACKEND_GPU || node->src[0]->backend == GGML_BACKEND_GPU_SPLIT)) || (node->src[1] != nullptr && (node->src[1]->backend == GGML_BACKEND_GPU)); - if (vk_disable || (!any_on_device && node->op != GGML_OP_MUL_MAT)) { + if (ctx->disable || (!any_on_device && node->op != GGML_OP_MUL_MAT)) { return; } @@ -3735,16 +3921,16 @@ void ggml_vk_preallocate_buffers_graph(ggml_tensor * node){ const uint32_t y_ne = ne10 * ne11; const uint32_t d_ne = ne20 * ne21; - const uint64_t qx_sz = use_src0 ? ggml_vk_align_size(ggml_type_size(src0->type) * x_ne / ggml_blck_size(src0->type), vk_device.properties.limits.minStorageBufferOffsetAlignment) * ne02 * ne03 : 0; - const uint64_t qy_sz = use_src1 ? ggml_vk_align_size(ggml_type_size(src1->type) * y_ne / ggml_blck_size(src1->type), vk_device.properties.limits.minStorageBufferOffsetAlignment) * ne12 * ne13 : 0; - const uint64_t x_sz = use_src0 ? ggml_vk_align_size(sizeof(ggml_fp16_t) * x_ne, vk_device.properties.limits.minStorageBufferOffsetAlignment) * ne02 * ne03 : 0; - const uint64_t y_sz = use_src1 ? ggml_vk_align_size(f16_f32_kernel ? sizeof(float) * y_ne : sizeof(ggml_fp16_t) * y_ne, vk_device.properties.limits.minStorageBufferOffsetAlignment) * ne12 * ne13 : 0; - uint64_t d_sz = ggml_vk_align_size(ggml_type_size(node->type) * d_ne, vk_device.properties.limits.minStorageBufferOffsetAlignment) * ne22 * ne23; + const uint64_t qx_sz = use_src0 ? ggml_vk_align_size(ggml_type_size(src0->type) * x_ne / ggml_blck_size(src0->type), ctx->device.lock()->properties.limits.minStorageBufferOffsetAlignment) * ne02 * ne03 : 0; + const uint64_t qy_sz = use_src1 ? ggml_vk_align_size(ggml_type_size(src1->type) * y_ne / ggml_blck_size(src1->type), ctx->device.lock()->properties.limits.minStorageBufferOffsetAlignment) * ne12 * ne13 : 0; + const uint64_t x_sz = use_src0 ? ggml_vk_align_size(sizeof(ggml_fp16_t) * x_ne, ctx->device.lock()->properties.limits.minStorageBufferOffsetAlignment) * ne02 * ne03 : 0; + const uint64_t y_sz = use_src1 ? ggml_vk_align_size(f16_f32_kernel ? sizeof(float) * y_ne : sizeof(ggml_fp16_t) * y_ne, ctx->device.lock()->properties.limits.minStorageBufferOffsetAlignment) * ne12 * ne13 : 0; + uint64_t d_sz = ggml_vk_align_size(ggml_type_size(node->type) * d_ne, ctx->device.lock()->properties.limits.minStorageBufferOffsetAlignment) * ne22 * ne23; const uint64_t split_k_size = split_k > 1 ? d_sz * 4 : 0; - if (extra->buffer_gpu.size == 0) { + if (extra->buffer_gpu.expired()) { // Workaround for CPU backend BLAS matmul calls - extra->buffer_gpu = ggml_vk_create_buffer_temp(d_sz); + extra->buffer_gpu = ggml_vk_create_buffer_temp(ctx, d_sz); } switch (node->op) { @@ -3779,23 +3965,23 @@ void ggml_vk_preallocate_buffers_graph(ggml_tensor * node){ } break; case GGML_OP_MUL_MAT: - if (vk_prealloc_size_qx < qx_sz) { - vk_prealloc_size_qx = qx_sz; + if (ctx->prealloc_size_qx < qx_sz) { + ctx->prealloc_size_qx = qx_sz; } - if (vk_prealloc_size_qy < qy_sz) { - vk_prealloc_size_qy = qy_sz; + if (ctx->prealloc_size_qy < qy_sz) { + ctx->prealloc_size_qy = qy_sz; } - if (vk_prealloc_size_x < x_sz) { - vk_prealloc_size_x = x_sz; + if (ctx->prealloc_size_x < x_sz) { + ctx->prealloc_size_x = x_sz; } - if (vk_prealloc_size_y < y_sz) { - vk_prealloc_size_y = y_sz; + if (ctx->prealloc_size_y < y_sz) { + ctx->prealloc_size_y = y_sz; } - if (vk_prealloc_size_split_k < split_k_size) { - vk_prealloc_size_split_k = split_k_size; + if (ctx->prealloc_size_split_k < split_k_size) { + ctx->prealloc_size_split_k = split_k_size; } - if (vk_staging_size < x_sz + y_sz) { - vk_staging_size = x_sz + y_sz; + if (ctx->staging_size < x_sz + y_sz) { + ctx->staging_size = x_sz + y_sz; } break; default: @@ -3803,29 +3989,29 @@ void ggml_vk_preallocate_buffers_graph(ggml_tensor * node){ } } -void ggml_vk_preallocate_buffers() { - if (vk_disable) { +static void ggml_vk_preallocate_buffers(ggml_backend_vk_context * ctx) { + if (ctx->disable) { return; } #ifdef GGML_VULKAN_DEBUG - std::cerr << "ggml_vk_preallocate_buffers()" << std::endl; - std::cerr << "qx_size: " << vk_prealloc_size_qx << " qy_size: " << vk_prealloc_size_qy << " x_size: " << vk_prealloc_size_x << " y_size: " << vk_prealloc_size_y << " split_k_size: " << vk_prealloc_size_split_k << std::endl; + std::cerr << "ggml_ctx->preallocate_buffers()" << std::endl; + std::cerr << "qx_size: " << ctx->prealloc_size_qx << " qy_size: " << ctx->prealloc_size_qy << " x_size: " << ctx->prealloc_size_x << " y_size: " << ctx->prealloc_size_y << " split_k_size: " << ctx->prealloc_size_split_k << std::endl; #endif #if defined(GGML_VULKAN_RUN_TESTS) - vk_staging = ggml_vk_create_buffer_check(100ul * 1024ul * 1024ul, vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent | vk::MemoryPropertyFlagBits::eHostCached); - ggml_vk_test_transfer(8192 * 1000, false); - ggml_vk_test_transfer(8192 * 1000, true); + ctx->staging = ggml_vk_create_buffer_check(ctx, 100ul * 1024ul * 1024ul, vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent | vk::MemoryPropertyFlagBits::eHostCached); + ggml_vk_test_transfer(ctx, 8192 * 1000, false); + ggml_vk_test_transfer(ctx, 8192 * 1000, true); - ggml_vk_test_dequant(2560 * 7680, GGML_TYPE_Q4_0); - ggml_vk_test_dequant(2560 * 7680, GGML_TYPE_Q4_1); - ggml_vk_test_dequant(2560 * 7680, GGML_TYPE_Q5_0); - ggml_vk_test_dequant(2560 * 7680, GGML_TYPE_Q5_1); - ggml_vk_test_dequant(2560 * 7680, GGML_TYPE_Q8_0); - ggml_vk_test_dequant(2560 * 7680, GGML_TYPE_Q2_K); - ggml_vk_test_dequant(2560 * 7680, GGML_TYPE_Q3_K); - ggml_vk_test_dequant(2560 * 7680, GGML_TYPE_Q4_K); - ggml_vk_test_dequant(2560 * 7680, GGML_TYPE_Q5_K); - ggml_vk_test_dequant(2560 * 7680, GGML_TYPE_Q6_K); + ggml_vk_test_dequant(ctx, 2560 * 7680, GGML_TYPE_Q4_0); + ggml_vk_test_dequant(ctx, 2560 * 7680, GGML_TYPE_Q4_1); + ggml_vk_test_dequant(ctx, 2560 * 7680, GGML_TYPE_Q5_0); + ggml_vk_test_dequant(ctx, 2560 * 7680, GGML_TYPE_Q5_1); + ggml_vk_test_dequant(ctx, 2560 * 7680, GGML_TYPE_Q8_0); + ggml_vk_test_dequant(ctx, 2560 * 7680, GGML_TYPE_Q2_K); + ggml_vk_test_dequant(ctx, 2560 * 7680, GGML_TYPE_Q3_K); + ggml_vk_test_dequant(ctx, 2560 * 7680, GGML_TYPE_Q4_K); + ggml_vk_test_dequant(ctx, 2560 * 7680, GGML_TYPE_Q5_K); + ggml_vk_test_dequant(ctx, 2560 * 7680, GGML_TYPE_Q6_K); const std::vector vals { 8, 8, 8, @@ -3852,76 +4038,76 @@ void ggml_vk_preallocate_buffers() { }; const size_t num_it = 1; for (size_t i = 0; i < vals.size(); i += 3) { - ggml_vk_test_matmul(vals[i], vals[i + 1], vals[i + 2], 2, num_it, 1, 0); - ggml_vk_test_matmul(vals[i], vals[i + 1], vals[i + 2], 2, num_it, 1, 1); - ggml_vk_test_matmul(vals[i], vals[i + 1], vals[i + 2], 2, num_it, 1, 2); - ggml_vk_test_matmul(vals[i], vals[i + 1], vals[i + 2], 2, num_it, 4, 0); - ggml_vk_test_matmul(vals[i], vals[i + 1], vals[i + 2], 2, num_it, 4, 1); - ggml_vk_test_matmul(vals[i], vals[i + 1], vals[i + 2], 2, num_it, 4, 2); + ggml_vk_test_matmul(ctx, vals[i], vals[i + 1], vals[i + 2], 2, num_it, 1, 0); + ggml_vk_test_matmul(ctx, vals[i], vals[i + 1], vals[i + 2], 2, num_it, 1, 1); + ggml_vk_test_matmul(ctx, vals[i], vals[i + 1], vals[i + 2], 2, num_it, 1, 2); + ggml_vk_test_matmul(ctx, vals[i], vals[i + 1], vals[i + 2], 2, num_it, 4, 0); + ggml_vk_test_matmul(ctx, vals[i], vals[i + 1], vals[i + 2], 2, num_it, 4, 1); + ggml_vk_test_matmul(ctx, vals[i], vals[i + 1], vals[i + 2], 2, num_it, 4, 2); std::cerr << std::endl; } GGML_ASSERT(false); #endif - if (vk_prealloc_size_qx > 0 && vk_prealloc_qx.size < vk_prealloc_size_qx) { + if (ctx->prealloc_qx == nullptr || (ctx->prealloc_size_qx > 0 && ctx->prealloc_qx->size < ctx->prealloc_size_qx)) { // Resize buffer - if (vk_prealloc_qx.size > 0) { - ggml_vk_destroy_buffer(vk_prealloc_qx); + if (ctx->prealloc_qx != nullptr) { + ggml_vk_destroy_buffer(ctx->prealloc_qx); } - vk_prealloc_qx = ggml_vk_create_buffer_device(vk_prealloc_size_qx); + ctx->prealloc_qx = ggml_vk_create_buffer_device(ctx, ctx->prealloc_size_qx); } - if (vk_prealloc_size_qy > 0 && vk_prealloc_qy.size < vk_prealloc_size_qy) { + if (ctx->prealloc_qy == nullptr || (ctx->prealloc_size_qy > 0 && ctx->prealloc_qy->size < ctx->prealloc_size_qy)) { // Resize buffer - if (vk_prealloc_qy.size > 0) { - ggml_vk_destroy_buffer(vk_prealloc_qy); + if (ctx->prealloc_qy != nullptr) { + ggml_vk_destroy_buffer(ctx->prealloc_qy); } - vk_prealloc_qy = ggml_vk_create_buffer_device(vk_prealloc_size_qy); + ctx->prealloc_qy = ggml_vk_create_buffer_device(ctx, ctx->prealloc_size_qy); } - if (vk_prealloc_size_x > 0 && vk_prealloc_x.size < vk_prealloc_size_x) { + if (ctx->prealloc_x == nullptr || (ctx->prealloc_size_x > 0 && ctx->prealloc_x->size < ctx->prealloc_size_x)) { // Resize buffer - if (vk_prealloc_x.size > 0) { - ggml_vk_destroy_buffer(vk_prealloc_x); + if (ctx->prealloc_x != nullptr) { + ggml_vk_destroy_buffer(ctx->prealloc_x); } - vk_prealloc_x = ggml_vk_create_buffer_device(vk_prealloc_size_x); + ctx->prealloc_x = ggml_vk_create_buffer_device(ctx, ctx->prealloc_size_x); } - if (vk_prealloc_size_y > 0 && vk_prealloc_y.size < vk_prealloc_size_y) { + if (ctx->prealloc_y == nullptr || (ctx->prealloc_size_y > 0 && ctx->prealloc_y->size < ctx->prealloc_size_y)) { // Resize buffer - if (vk_prealloc_y.size > 0) { - ggml_vk_destroy_buffer(vk_prealloc_y); + if (ctx->prealloc_y != nullptr) { + ggml_vk_destroy_buffer(ctx->prealloc_y); } - vk_prealloc_y = ggml_vk_create_buffer_device(vk_prealloc_size_y); + ctx->prealloc_y = ggml_vk_create_buffer_device(ctx, ctx->prealloc_size_y); } - if (vk_prealloc_size_split_k > 0 && vk_prealloc_split_k.size < vk_prealloc_size_split_k) { + if (ctx->prealloc_split_k == nullptr || (ctx->prealloc_size_split_k > 0 && ctx->prealloc_split_k->size < ctx->prealloc_size_split_k)) { // Resize buffer - if (vk_prealloc_split_k.size > 0) { - ggml_vk_destroy_buffer(vk_prealloc_split_k); + if (ctx->prealloc_split_k != nullptr) { + ggml_vk_destroy_buffer(ctx->prealloc_split_k); } - vk_prealloc_split_k = ggml_vk_create_buffer_device(vk_prealloc_size_split_k); + ctx->prealloc_split_k = ggml_vk_create_buffer_device(ctx, ctx->prealloc_size_split_k); } - if (vk_staging_size > 0 && vk_staging.size < vk_staging_size) { + if (ctx->staging == nullptr || (ctx->staging_size > 0 && ctx->staging->size < ctx->staging_size)) { // Resize buffer - if (vk_staging.size > 0) { - ggml_vk_destroy_buffer(vk_staging); + if (ctx->staging != nullptr) { + ggml_vk_destroy_buffer(ctx->staging); } - vk_staging = ggml_vk_create_buffer_check(vk_staging_size, vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent | vk::MemoryPropertyFlagBits::eHostCached); + ctx->staging = ggml_vk_create_buffer_check(ctx, ctx->staging_size, vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent | vk::MemoryPropertyFlagBits::eHostCached); } } -void ggml_vk_build_graph(ggml_tensor * node, bool last_node){ +static void ggml_vk_build_graph(ggml_backend_vk_context * ctx, ggml_tensor * node, bool last_node){ const bool any_on_device = node->backend == GGML_BACKEND_GPU || (node->src[0] != nullptr && (node->src[0]->backend == GGML_BACKEND_GPU || node->src[0]->backend == GGML_BACKEND_GPU_SPLIT)) || (node->src[1] != nullptr && node->src[1]->backend == GGML_BACKEND_GPU); - if (vk_disable || (!any_on_device && node->op != GGML_OP_MUL_MAT) || (node->op == GGML_OP_MUL_MAT && !any_on_device && !ggml_vk_can_mul_mat(node->src[0], node->src[1], node))) { + if (ctx->disable || (!any_on_device && node->op != GGML_OP_MUL_MAT) || (node->op == GGML_OP_MUL_MAT && !any_on_device && !ggml_vk_can_mul_mat(node->src[0], node->src[1], node))) { return; } #ifdef GGML_VULKAN_DEBUG std::cerr << "ggml_vk_build_graph(" << node << ", " << ggml_op_name(node->op) << ")" << std::endl; #endif - vk_semaphore_idx = 0; - vk_staging_offset = 0; + ctx->semaphore_idx = 0; + ctx->staging_offset = 0; const ggml_tensor * src0 = node->src[0]; const ggml_tensor * src1 = node->src[1]; @@ -3969,44 +4155,44 @@ void ggml_vk_build_graph(ggml_tensor * node, bool last_node){ return; } - if (vk_ctx == nullptr) { - vk_ctx = ggml_vk_create_context(vk_device.compute_queue); - ggml_vk_ctx_begin(vk_ctx); + if (ctx->compute_ctx == nullptr) { + ctx->compute_ctx = ggml_vk_create_context(ctx, ctx->device.lock()->compute_queue); + ggml_vk_ctx_begin(ctx, ctx->compute_ctx); } switch (node->op) { case GGML_OP_REPEAT: - ggml_vk_repeat(vk_ctx, src0, src1, node); + ggml_vk_repeat(ctx, ctx->compute_ctx, src0, src1, node); break; case GGML_OP_GET_ROWS: - ggml_vk_get_rows(vk_ctx, src0, src1, node); + ggml_vk_get_rows(ctx, ctx->compute_ctx, src0, src1, node); break; case GGML_OP_ADD: - ggml_vk_add(vk_ctx, src0, src1, node); + ggml_vk_add(ctx, ctx->compute_ctx, src0, src1, node); break; case GGML_OP_MUL: - ggml_vk_mul(vk_ctx, src0, src1, node); + ggml_vk_mul(ctx, ctx->compute_ctx, src0, src1, node); break; case GGML_OP_SCALE: - ggml_vk_scale(vk_ctx, src0, node); + ggml_vk_scale(ctx, ctx->compute_ctx, src0, node); break; case GGML_OP_SQR: - ggml_vk_sqr(vk_ctx, src0, node); + ggml_vk_sqr(ctx, ctx->compute_ctx, src0, node); break; case GGML_OP_CLAMP: - ggml_vk_clamp(vk_ctx, src0, node); + ggml_vk_clamp(ctx, ctx->compute_ctx, src0, node); break; case GGML_OP_CPY: case GGML_OP_CONT: case GGML_OP_DUP: - ggml_vk_cpy(vk_ctx, src0, node); + ggml_vk_cpy(ctx, ctx->compute_ctx, src0, node); break; case GGML_OP_RESHAPE: @@ -4014,15 +4200,15 @@ void ggml_vk_build_graph(ggml_tensor * node, bool last_node){ case GGML_OP_PERMUTE: case GGML_OP_TRANSPOSE: case GGML_OP_NONE: - ggml_vk_nop(vk_ctx, src0, node); + ggml_vk_nop(ctx, ctx->compute_ctx, src0, node); break; case GGML_OP_NORM: - ggml_vk_norm(vk_ctx, src0, node); + ggml_vk_norm(ctx, ctx->compute_ctx, src0, node); break; case GGML_OP_RMS_NORM: - ggml_vk_rms_norm(vk_ctx, src0, node); + ggml_vk_rms_norm(ctx, ctx->compute_ctx, src0, node); break; case GGML_OP_UNARY: @@ -4030,26 +4216,26 @@ void ggml_vk_build_graph(ggml_tensor * node, bool last_node){ case GGML_UNARY_OP_SILU: case GGML_UNARY_OP_GELU: case GGML_UNARY_OP_RELU: - ggml_vk_unary(vk_ctx, src0, node); + ggml_vk_unary(ctx, ctx->compute_ctx, src0, node); break; default: return; } break; case GGML_OP_DIAG_MASK_INF: - ggml_vk_diag_mask_inf(vk_ctx, src0, node); + ggml_vk_diag_mask_inf(ctx, ctx->compute_ctx, src0, node); break; case GGML_OP_SOFT_MAX: - ggml_vk_soft_max(vk_ctx, src0, src1, node); + ggml_vk_soft_max(ctx, ctx->compute_ctx, src0, src1, node); break; case GGML_OP_ROPE: - ggml_vk_rope(vk_ctx, src0, src1, node); + ggml_vk_rope(ctx, ctx->compute_ctx, src0, src1, node); break; case GGML_OP_MUL_MAT: - ggml_vk_mul_mat(vk_ctx, src0, src1, node); + ggml_vk_mul_mat(ctx, ctx->compute_ctx, src0, src1, node); break; default: @@ -4057,7 +4243,7 @@ void ggml_vk_build_graph(ggml_tensor * node, bool last_node){ } extra->ready = true; - extra->ctx_idx = vk_ctx->idx; + extra->ctx_idx = ctx->compute_ctx->idx; #ifdef GGML_VULKAN_CHECK_RESULTS // Force context reset on each node so that each tensor ends up in its own context @@ -4066,18 +4252,18 @@ void ggml_vk_build_graph(ggml_tensor * node, bool last_node){ #endif if (node->backend == GGML_BACKEND_CPU || last_node) { - ggml_vk_ctx_end(vk_ctx); - vk_ctx->exit_tensor = node; - vk_ctx = nullptr; + ggml_vk_ctx_end(ctx->compute_ctx); + ctx->compute_ctx->exit_tensor = node; + ctx->compute_ctx = nullptr; } } -bool ggml_vk_compute_forward(ggml_compute_params * params, ggml_tensor * tensor){ +static bool ggml_vk_compute_forward(ggml_backend_vk_context * ctx, ggml_compute_params * params, ggml_tensor * tensor){ const bool any_on_device = tensor->backend == GGML_BACKEND_GPU || (tensor->src[0] != nullptr && (tensor->src[0]->backend == GGML_BACKEND_GPU || tensor->src[0]->backend == GGML_BACKEND_GPU_SPLIT)) || (tensor->src[1] != nullptr && tensor->src[1]->backend == GGML_BACKEND_GPU); - if (vk_disable || (!any_on_device && tensor->op != GGML_OP_MUL_MAT)) { + if (ctx->disable || (!any_on_device && tensor->op != GGML_OP_MUL_MAT)) { return false; } @@ -4145,33 +4331,33 @@ bool ggml_vk_compute_forward(ggml_compute_params * params, ggml_tensor * tensor) #endif #ifdef GGML_VULKAN_CHECK_RESULTS - ggml_vk_check_results_0(params, tensor); + ggml_vk_check_results_0(ctx, params, tensor); #endif GGML_ASSERT(extra->ready); - vk_context& ctx = vk_gc.contexts[extra->ctx_idx]; + vk_context& subctx = ctx->gc.contexts[extra->ctx_idx]; // Only run if ctx hasn't been submitted yet - if (!ctx.seqs.empty()) { + if (!subctx.seqs.empty()) { // Do staging buffer copies - for (auto& cpy : ctx.in_memcpys) { + for (auto& cpy : subctx.in_memcpys) { memcpy(cpy.dst, cpy.src, cpy.n); } - ggml_vk_submit(&ctx, vk_fence); + ggml_vk_submit(&subctx, ctx->fence); } - if (tensor == ctx.exit_tensor) { - VK_CHECK(vk_device.device.waitForFences({ vk_fence }, true, UINT64_MAX), "ggml_vk_compute_forward waitForFences"); - vk_device.device.resetFences({ vk_fence }); + if (tensor == subctx.exit_tensor) { + VK_CHECK(ctx->device.lock()->device.waitForFences({ ctx->fence }, true, UINT64_MAX), "ggml_vk_compute_forward waitForFences"); + ctx->device.lock()->device.resetFences({ ctx->fence }); // Do staging buffer copies - for (auto& cpy : ctx.out_memcpys) { + for (auto& cpy : subctx.out_memcpys) { memcpy(cpy.dst, cpy.src, cpy.n); } - ctx.in_memcpys.clear(); - ctx.out_memcpys.clear(); + subctx.in_memcpys.clear(); + subctx.out_memcpys.clear(); } extra->ready = false; @@ -4179,90 +4365,204 @@ bool ggml_vk_compute_forward(ggml_compute_params * params, ggml_tensor * tensor) return true; } -void ggml_vk_graph_cleanup() { - if (vk_disable) { +// Clean up after graph processing is done +static void ggml_vk_graph_cleanup(ggml_backend_vk_context * ctx) { + if (ctx->disable) { return; } #ifdef GGML_VULKAN_DEBUG std::cerr << "ggml_vk_graph_cleanup()" << std::endl; #endif - for (auto& buffer : vk_gc.temp_buffers) { - ggml_vk_pool_free(buffer); + for (auto& buffer : ctx->gc.temp_buffers) { + ggml_vk_pool_free(ctx, buffer); } - vk_gc.temp_buffers.clear(); + ctx->gc.temp_buffers.clear(); - for (auto * pipeline : vk_gc.pipelines) { - ggml_vk_pipeline_cleanup(*pipeline); - } - vk_gc.pipelines.clear(); - - ggml_vk_queue_cleanup(vk_device.compute_queue); - ggml_vk_queue_cleanup(vk_device.transfer_queue); - - for (size_t i = 0; i < vk_gc.semaphores.size(); i++) { - vk_device.device.destroySemaphore({ vk_gc.semaphores[i].s }); - } - vk_gc.semaphores.clear(); - - for (size_t i = 0; i < vk_gc.tl_semaphores.size(); i++) { - vk_device.device.destroySemaphore({ vk_gc.tl_semaphores[i].s }); - } - vk_gc.tl_semaphores.clear(); - - vk_event_idx = 0; - - for (auto& event : vk_gc.events) { - vk_device.device.resetEvent(event); + for (auto * pipeline : ctx->gc.pipelines) { + ggml_pipeline_cleanup(*pipeline); } - vk_staging_offset = 0; + ggml_vk_queue_cleanup(ctx, ctx->device.lock()->compute_queue); + ggml_vk_queue_cleanup(ctx, ctx->device.lock()->transfer_queue); - vk_ctx = nullptr; - vk_gc.contexts.clear(); + for (size_t i = 0; i < ctx->gc.semaphores.size(); i++) { + ctx->device.lock()->device.destroySemaphore({ ctx->gc.semaphores[i].s }); + } + ctx->gc.semaphores.clear(); + + for (size_t i = 0; i < ctx->gc.tl_semaphores.size(); i++) { + ctx->device.lock()->device.destroySemaphore({ ctx->gc.tl_semaphores[i].s }); + } + ctx->gc.tl_semaphores.clear(); + ctx->semaphore_idx = 0; + + ctx->event_idx = 0; + + for (auto& event : ctx->gc.events) { + ctx->device.lock()->device.resetEvent(event); + } + + ctx->staging_offset = 0; + + ctx->compute_ctx = nullptr; + ctx->transfer_ctx = nullptr; + ctx->gc.contexts.clear(); } -static void ggml_vk_cleanup() { +// Clean up on backend free +static void ggml_vk_cleanup(ggml_backend_vk_context * ctx) { #ifdef GGML_VULKAN_DEBUG - std::cerr << "ggml_vk_cleanup()" << std::endl; + std::cerr << "ggml_vk_cleanup(" << ctx->idx << ")" << std::endl; #endif - ggml_vk_destroy_buffer(vk_prealloc_x); - ggml_vk_destroy_buffer(vk_prealloc_y); - ggml_vk_destroy_buffer(vk_prealloc_split_k); - ggml_vk_destroy_buffer(vk_staging); - ggml_vk_destroy_buffer(vk_sync_staging); + ggml_vk_graph_cleanup(ctx); - vk_prealloc_size_x = 0; - vk_prealloc_size_y = 0; - vk_prealloc_size_split_k = 0; - vk_staging_size = 0; + ggml_vk_destroy_buffer(ctx->prealloc_qx); + ggml_vk_destroy_buffer(ctx->prealloc_qy); + ggml_vk_destroy_buffer(ctx->prealloc_x); + ggml_vk_destroy_buffer(ctx->prealloc_y); + ggml_vk_destroy_buffer(ctx->prealloc_split_k); + ggml_vk_destroy_buffer(ctx->staging); + ggml_vk_destroy_buffer(ctx->sync_staging); - for (auto& event : vk_gc.events) { - vk_device.device.destroyEvent(event); + for (auto& buffer : ctx->buffer_pool) { + ggml_vk_destroy_buffer(buffer); } - vk_gc.events.clear(); + + ctx->prealloc_size_qx = 0; + ctx->prealloc_size_qy = 0; + ctx->prealloc_size_x = 0; + ctx->prealloc_size_y = 0; + ctx->prealloc_size_split_k = 0; + ctx->staging_size = 0; + + for (auto& event : ctx->gc.events) { + ctx->device.lock()->device.destroyEvent(event); + } + ctx->gc.events.clear(); + + for (auto* pipeline : ctx->gc.pipelines) { + ggml_vk_destroy_pipeline(ctx, pipeline); + } + ctx->gc.pipelines.clear(); + + ctx->device.lock()->device.destroyFence(ctx->fence); + + ctx->device.lock()->device.destroyCommandPool(ctx->device.lock()->compute_queue.pool); + if (!ctx->device.lock()->single_queue) { + ctx->device.lock()->device.destroyCommandPool(ctx->device.lock()->transfer_queue.pool); + } +} + +GGML_CALL int ggml_vk_get_device_count() { + ggml_vk_instance_init(); + + return vk_instance.device_indices.size(); +} + +GGML_CALL void ggml_vk_get_device_description(int device, char * description, size_t description_size) { + ggml_vk_instance_init(); + + std::vector devices = vk_instance.instance.enumeratePhysicalDevices(); + + vk::PhysicalDeviceProperties props; + devices[device].getProperties(&props); + + snprintf(description, description_size, "%s", props.deviceName.data()); +} + +// CPU assist interface + +void ggml_vk_init_cpu_assist() { + ggml_vk_instance_init(); + + std::cerr << "ggml_vulkan: Found " << ggml_vk_get_device_count() << " Vulkan devices:" << std::endl; + + for (size_t i = 0; i < ggml_vk_get_device_count(); i++) { + ggml_vk_print_gpu_info(i); + } + // Initialize the first backend to make sure CPU matrix multiplications can be offloaded. + ggml_backend_vk_init(0); +} + +void ggml_vk_preallocate_buffers_graph_cpu_assist(ggml_tensor * node) { + ggml_backend_vk_context * ctx = &vk_instance.contexts[0]; + + if (!ctx->initialized) { + return; + } + + ggml_vk_preallocate_buffers_graph(ctx, node); +} + +void ggml_vk_preallocate_buffers_cpu_assist() { + ggml_backend_vk_context * ctx = &vk_instance.contexts[0]; + + if (!ctx->initialized) { + return; + } + + ggml_vk_preallocate_buffers(ctx); +} + +void ggml_vk_build_graph_cpu_assist(ggml_tensor * node, bool last_node) { + ggml_backend_vk_context * ctx = &vk_instance.contexts[0]; + + if (!ctx->initialized) { + return; + } + + ggml_vk_build_graph(ctx, node, last_node); +} + +bool ggml_vk_compute_forward_cpu_assist(ggml_compute_params * params, ggml_tensor * tensor){ + ggml_backend_vk_context * ctx = &vk_instance.contexts[0]; + + if (!ctx->initialized) { + return false; + } + + return ggml_vk_compute_forward(ctx, params, tensor); +} + +void ggml_vk_graph_cleanup_cpu_assist() { + ggml_backend_vk_context * ctx = &vk_instance.contexts[0]; + + if (!ctx->initialized) { + return; + } + + ggml_vk_graph_cleanup(ctx); +} + +void ggml_vk_free_cpu_assist() { + ggml_backend_vk_context * ctx = &vk_instance.contexts[0]; + + if (!ctx->initialized || vk_instance.backends[0] == nullptr) { + return; + } + + ggml_backend_vk_free(vk_instance.backends[0]); } // backend interface #define UNUSED GGML_UNUSED -struct ggml_backend_vk_context { - std::string name; -}; - // device backend static void * const vk_ptr_base = (void *)(uintptr_t) 0x1000; // NOLINT struct ggml_backend_vk_buffer_context { + ggml_backend_vk_context * ctx; vk_buffer dev_buffer; ggml_tensor_extra_gpu * temp_tensor_extras = nullptr; size_t temp_tensor_extra_index = 0; std::string name; - ggml_backend_vk_buffer_context(vk_buffer dev_buffer) : + ggml_backend_vk_buffer_context(ggml_backend_vk_context * ctx, vk_buffer&& dev_buffer, std::string& name) : + ctx(ctx), dev_buffer(dev_buffer), - name(GGML_VK_NAME) { + name(name) { } ~ggml_backend_vk_buffer_context() { @@ -4294,6 +4594,9 @@ GGML_CALL static bool ggml_backend_buffer_is_vk(ggml_backend_buffer_t buffer) { } GGML_CALL static void ggml_backend_vk_buffer_free_buffer(ggml_backend_buffer_t buffer) { +#ifdef GGML_VULKAN_DEBUG + std::cerr << "ggml_backend_vk_buffer_free_buffer()" << std::endl; +#endif ggml_backend_vk_buffer_context * ctx = (ggml_backend_vk_buffer_context *)buffer->context; ggml_vk_destroy_buffer(ctx->dev_buffer); delete ctx; @@ -4313,6 +4616,7 @@ GGML_CALL static void ggml_backend_vk_buffer_init_tensor(ggml_backend_buffer_t b ggml_tensor_extra_gpu * extra = ctx->ggml_vk_alloc_temp_tensor_extra(); if (tensor->view_src != nullptr && tensor->view_src->extra != nullptr) { + GGML_ASSERT(tensor->view_src->buffer->buft == buffer->buft); ggml_tensor_extra_gpu * extra_view = (ggml_tensor_extra_gpu *) tensor->view_src->extra; extra->buffer_gpu = extra_view->buffer_gpu; extra->offset = extra_view->offset + tensor->view_offs; @@ -4331,11 +4635,13 @@ GGML_CALL static void ggml_backend_vk_buffer_set_tensor(ggml_backend_buffer_t bu #endif GGML_ASSERT(tensor->backend == GGML_BACKEND_GPU); + ggml_backend_vk_buffer_context * ctx = (ggml_backend_vk_buffer_context *)buffer->context; + ggml_tensor_extra_gpu * extra = (ggml_tensor_extra_gpu *) tensor->extra; - ggml_vk_buffer_write(&extra->buffer_gpu, extra->offset + offset, data, size); + vk_buffer buf = extra->buffer_gpu.lock(); - UNUSED(buffer); + ggml_vk_buffer_write(ctx->ctx, buf, extra->offset + offset, data, size); } GGML_CALL static void ggml_backend_vk_buffer_get_tensor(ggml_backend_buffer_t buffer, const ggml_tensor * tensor, void * data, size_t offset, size_t size) { @@ -4344,31 +4650,35 @@ GGML_CALL static void ggml_backend_vk_buffer_get_tensor(ggml_backend_buffer_t bu #endif GGML_ASSERT(tensor->backend == GGML_BACKEND_GPU); + ggml_backend_vk_buffer_context * ctx = (ggml_backend_vk_buffer_context *)buffer->context; + ggml_tensor_extra_gpu * extra = (ggml_tensor_extra_gpu *) tensor->extra; - ggml_vk_buffer_read(&extra->buffer_gpu, extra->offset + offset, data, size); + vk_buffer buf = extra->buffer_gpu.lock(); - UNUSED(buffer); + ggml_vk_buffer_read(ctx->ctx, buf, extra->offset + offset, data, size); } GGML_CALL static bool ggml_backend_vk_buffer_cpy_tensor(ggml_backend_buffer_t buffer, const ggml_tensor * src, ggml_tensor * dst) { if (ggml_backend_buffer_is_vk(src->buffer)) { + ggml_backend_vk_buffer_context * ctx = (ggml_backend_vk_buffer_context *)buffer->context; ggml_tensor_extra_gpu * src_extra = (ggml_tensor_extra_gpu *) src->extra; ggml_tensor_extra_gpu * dst_extra = (ggml_tensor_extra_gpu *) dst->extra; - ggml_vk_buffer_copy(&src_extra->buffer_gpu, src_extra->offset, &dst_extra->buffer_gpu, dst_extra->offset, ggml_nbytes(src)); + vk_buffer src_buf = src_extra->buffer_gpu.lock(); + vk_buffer dst_buf = dst_extra->buffer_gpu.lock(); + + ggml_vk_buffer_copy(dst_buf, dst_extra->offset, src_buf, src_extra->offset, ggml_nbytes(src)); return true; } return false; - - UNUSED(buffer); } GGML_CALL static void ggml_backend_vk_buffer_clear(ggml_backend_buffer_t buffer, uint8_t value) { ggml_backend_vk_buffer_context * ctx = (ggml_backend_vk_buffer_context *)buffer->context; - ggml_vk_buffer_memset(&ctx->dev_buffer, 0, value, buffer->size); + ggml_vk_buffer_memset(ctx->ctx, ctx->dev_buffer, 0, value, buffer->size); } static ggml_backend_buffer_i ggml_backend_vk_buffer_interface = { @@ -4386,6 +4696,7 @@ static ggml_backend_buffer_i ggml_backend_vk_buffer_interface = { // vk buffer type struct ggml_backend_vk_buffer_type_context { std::string name; + ggml_backend_vk_context * ctx; }; GGML_CALL static const char * ggml_backend_vk_buffer_type_name(ggml_backend_buffer_type_t buft) { @@ -4398,25 +4709,22 @@ GGML_CALL static ggml_backend_buffer_t ggml_backend_vk_buffer_type_alloc_buffer( #ifdef GGML_VULKAN_DEBUG std::cerr << "ggml_backend_vk_buffer_type_alloc_buffer(" << size << ")" << std::endl; #endif - vk_buffer dev_buffer = ggml_vk_create_buffer_device(size); + ggml_backend_vk_buffer_type_context * ctx = (ggml_backend_vk_buffer_type_context *) buft->context; + vk_buffer dev_buffer = ggml_vk_create_buffer_device(ctx->ctx, size); - ggml_backend_vk_buffer_context * ctx = new ggml_backend_vk_buffer_context(dev_buffer); + ggml_backend_vk_buffer_context * bufctx = new ggml_backend_vk_buffer_context(ctx->ctx, std::move(dev_buffer), ctx->name); - return ggml_backend_buffer_init(buft, ggml_backend_vk_buffer_interface, ctx, size); - - UNUSED(buft); + return ggml_backend_buffer_init(buft, ggml_backend_vk_buffer_interface, bufctx, size); } GGML_CALL static size_t ggml_backend_vk_buffer_type_get_alignment(ggml_backend_buffer_type_t buft) { - return vk_device.properties.limits.minStorageBufferOffsetAlignment; - - UNUSED(buft); + ggml_backend_vk_buffer_type_context * ctx = (ggml_backend_vk_buffer_type_context *) buft->context; + return ctx->ctx->device.lock()->properties.limits.minStorageBufferOffsetAlignment; } GGML_CALL static size_t ggml_backend_vk_buffer_type_get_max_size(ggml_backend_buffer_type_t buft) { - return vk_device.max_memory_allocation_size; - - UNUSED(buft); + ggml_backend_vk_buffer_type_context * ctx = (ggml_backend_vk_buffer_type_context *) buft->context; + return ctx->ctx->device.lock()->max_memory_allocation_size; } GGML_CALL static size_t ggml_backend_vk_buffer_type_get_alloc_size(ggml_backend_buffer_type_t buft, const ggml_tensor * tensor) { @@ -4426,9 +4734,14 @@ GGML_CALL static size_t ggml_backend_vk_buffer_type_get_alloc_size(ggml_backend_ } GGML_CALL static bool ggml_backend_vk_buffer_type_supports_backend(ggml_backend_buffer_type_t buft, ggml_backend_t backend) { - return ggml_backend_is_vk(backend); + if (!ggml_backend_is_vk(backend)) { + return false; + } - UNUSED(buft); + ggml_backend_vk_buffer_type_context * buft_ctx = (ggml_backend_vk_buffer_type_context *)buft->context; + ggml_backend_vk_context * ctx = (ggml_backend_vk_context *)backend->context; + + return buft_ctx->ctx->idx == ctx->idx; } static ggml_backend_buffer_type_i ggml_backend_vk_buffer_type_interface = { @@ -4441,20 +4754,16 @@ static ggml_backend_buffer_type_i ggml_backend_vk_buffer_type_interface = { /* .is_host = */ NULL, }; -GGML_CALL ggml_backend_buffer_type_t ggml_backend_vk_buffer_type() { - static ggml_backend_buffer_type ggml_backend_vk_buffer_type; +GGML_CALL ggml_backend_buffer_type_t ggml_backend_vk_buffer_type(size_t idx) { +#ifdef GGML_VULKAN_DEBUG + std::cerr << "ggml_backend_vk_buffer_type(" << idx << ")" << std::endl; +#endif - static bool ggml_backend_vk_buffer_type_initialized = false; + GGML_ASSERT(idx < vk_instance.device_indices.size()); - if (!ggml_backend_vk_buffer_type_initialized) { - ggml_backend_vk_buffer_type = { - /* .iface = */ ggml_backend_vk_buffer_type_interface, - /* .context = */ new ggml_backend_vk_buffer_type_context{GGML_VK_NAME}, - }; - ggml_backend_vk_buffer_type_initialized = true; - } + ggml_backend_vk_init(idx); - return &ggml_backend_vk_buffer_type; + return &vk_instance.buffer_types[idx]; } // host buffer type @@ -4472,13 +4781,19 @@ GGML_CALL static const char * ggml_backend_vk_host_buffer_name(ggml_backend_buff } GGML_CALL static void ggml_backend_vk_host_buffer_free_buffer(ggml_backend_buffer_t buffer) { - ggml_vk_host_free(buffer->context); +#ifdef GGML_VULKAN_DEBUG + std::cerr << "ggml_backend_vk_host_buffer_free_buffer()" << std::endl; +#endif + ggml_vk_host_free(&vk_instance.contexts[0], buffer->context); } GGML_CALL static ggml_backend_buffer_t ggml_backend_vk_host_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size) { +#ifdef GGML_VULKAN_DEBUG + std::cerr << "ggml_backend_vk_host_buffer_type_alloc_buffer(" << size << ")" << std::endl; +#endif void * ptr = nullptr; try { - ptr = ggml_vk_host_malloc(size); + ptr = ggml_vk_host_malloc(&vk_instance.contexts[0], size); } catch (vk::SystemError& e) { std::cerr << "ggml_vulkan: Failed to allocate pinned memory." << std::endl; std::cerr << "ggml_vulkan: " << e.what() << std::endl; @@ -4495,7 +4810,7 @@ GGML_CALL static ggml_backend_buffer_t ggml_backend_vk_host_buffer_type_alloc_bu } GGML_CALL static size_t ggml_backend_vk_host_buffer_type_get_alignment(ggml_backend_buffer_type_t buft) { - return vk_device.properties.limits.minMemoryMapAlignment; + return vk_instance.contexts[0].device.lock()->properties.limits.minMemoryMapAlignment; UNUSED(buft); } @@ -4514,127 +4829,150 @@ GGML_CALL ggml_backend_buffer_type_t ggml_backend_vk_host_buffer_type() { /* .context = */ nullptr, }; + if (!vk_instance.contexts[0].initialized) { + // Fall back to CPU + return ggml_backend_cpu_buffer_type(); + } + return &ggml_backend_vk_buffer_type_host; } // backend GGML_CALL static const char * ggml_backend_vk_name(ggml_backend_t backend) { - ggml_backend_vk_context * vk_ctx = (ggml_backend_vk_context *)backend->context; + ggml_backend_vk_context * ctx = (ggml_backend_vk_context *)backend->context; - return vk_ctx->name.c_str(); + return ctx->name.c_str(); } GGML_CALL static void ggml_backend_vk_free(ggml_backend_t backend) { - ggml_backend_vk_context * vk_ctx = (ggml_backend_vk_context *)backend->context; + ggml_backend_vk_context * ctx = (ggml_backend_vk_context *)backend->context; +#ifdef GGML_VULKAN_DEBUG + std::cerr << "ggml_backend_vk_free(" << ctx->name << ")" << std::endl; +#endif - delete vk_ctx; + size_t idx = ctx->idx; + + ggml_vk_cleanup(ctx); + + // Release device + vk_instance.devices[ctx->idx].reset(); + ctx->initialized = false; + + vk_instance.initialized[idx] = false; + vk_instance.backends[idx] = nullptr; + memset(&vk_instance.buffer_types[idx], 0, sizeof(ggml_backend_buffer_type)); delete backend; } GGML_CALL static ggml_backend_buffer_type_t ggml_backend_vk_get_default_buffer_type(ggml_backend_t backend) { - return ggml_backend_vk_buffer_type(); + ggml_backend_vk_context * ctx = (ggml_backend_vk_context *)backend->context; - UNUSED(backend); + GGML_ASSERT(ctx->initialized); + + return ggml_backend_vk_buffer_type(ctx->idx); } GGML_CALL static void ggml_backend_vk_set_tensor_async(ggml_backend_t backend, ggml_tensor * tensor, const void * data, size_t offset, size_t size) { #ifdef GGML_VULKAN_DEBUG std::cerr << "ggml_backend_vk_set_tensor_async(" << size << ")" << std::endl; #endif - GGML_ASSERT((tensor->buffer->buft == ggml_backend_vk_buffer_type() || tensor->buffer->buft == ggml_backend_vk_host_buffer_type()) && "unsupported buffer type"); + ggml_backend_vk_context * ctx = (ggml_backend_vk_context *)backend->context; + GGML_ASSERT((tensor->buffer->buft == ggml_backend_vk_buffer_type(ctx->idx) || tensor->buffer->buft == ggml_backend_vk_host_buffer_type()) && "unsupported buffer type"); GGML_ASSERT(tensor->backend == GGML_BACKEND_GPU); ggml_tensor_extra_gpu * extra = (ggml_tensor_extra_gpu *) tensor->extra; - if (vk_transfer_ctx == nullptr) { + if (ctx->transfer_ctx == nullptr) { // Initialize new transfer context - vk_transfer_ctx = ggml_vk_create_context(vk_device.transfer_queue); - ggml_vk_ctx_begin(vk_transfer_ctx); + ctx->transfer_ctx = ggml_vk_create_context(ctx, ctx->device.lock()->transfer_queue); + ggml_vk_ctx_begin(ctx, ctx->transfer_ctx); } - ggml_vk_buffer_write_async(vk_transfer_ctx, &extra->buffer_gpu, extra->offset + offset, data, size); + vk_buffer buf = extra->buffer_gpu.lock(); - UNUSED(backend); + ggml_vk_buffer_write_async(ctx, ctx->transfer_ctx, buf, extra->offset + offset, data, size); } GGML_CALL static void ggml_backend_vk_get_tensor_async(ggml_backend_t backend, const ggml_tensor * tensor, void * data, size_t offset, size_t size) { #ifdef GGML_VULKAN_DEBUG std::cerr << "ggml_backend_vk_get_tensor_async(" << size << ")" << std::endl; #endif - GGML_ASSERT((tensor->buffer->buft == ggml_backend_vk_buffer_type() || tensor->buffer->buft == ggml_backend_vk_host_buffer_type()) && "unsupported buffer type"); + ggml_backend_vk_context * ctx = (ggml_backend_vk_context *)backend->context; + GGML_ASSERT((tensor->buffer->buft == ggml_backend_vk_buffer_type(ctx->idx) || tensor->buffer->buft == ggml_backend_vk_host_buffer_type()) && "unsupported buffer type"); GGML_ASSERT(tensor->backend == GGML_BACKEND_GPU); ggml_tensor_extra_gpu * extra = (ggml_tensor_extra_gpu *) tensor->extra; - if (vk_transfer_ctx == nullptr) { + if (ctx->transfer_ctx == nullptr) { // Initialize new transfer context - vk_transfer_ctx = ggml_vk_create_context(vk_device.transfer_queue); - ggml_vk_ctx_begin(vk_transfer_ctx); + ctx->transfer_ctx = ggml_vk_create_context(ctx, ctx->device.lock()->transfer_queue); + ggml_vk_ctx_begin(ctx, ctx->transfer_ctx); } - ggml_vk_buffer_read_async(vk_transfer_ctx, &extra->buffer_gpu, extra->offset + offset, data, size); + vk_buffer buf = extra->buffer_gpu.lock(); - UNUSED(backend); + ggml_vk_buffer_read_async(ctx, ctx->transfer_ctx, buf, extra->offset + offset, data, size); } GGML_CALL static bool ggml_backend_vk_cpy_tensor_async(ggml_backend_t backend, const ggml_tensor * src, ggml_tensor * dst) { #ifdef GGML_VULKAN_DEBUG std::cerr << "ggml_backend_vk_cpy_tensor_async()" << std::endl; #endif - if ((dst->buffer->buft == ggml_backend_vk_buffer_type() || dst->buffer->buft == ggml_backend_vk_host_buffer_type()) && ggml_backend_buffer_is_vk(src->buffer)) { + ggml_backend_vk_context * ctx = (ggml_backend_vk_context *)backend->context; + if ((dst->buffer->buft == ggml_backend_vk_buffer_type(ctx->idx) || dst->buffer->buft == ggml_backend_vk_host_buffer_type()) && ggml_backend_buffer_is_vk(src->buffer)) { ggml_tensor_extra_gpu * src_extra = (ggml_tensor_extra_gpu *) src->extra; ggml_tensor_extra_gpu * dst_extra = (ggml_tensor_extra_gpu *) dst->extra; - if (vk_transfer_ctx == nullptr) { + if (ctx->transfer_ctx == nullptr) { // Initialize new transfer context - vk_transfer_ctx = ggml_vk_create_context(vk_device.transfer_queue); - ggml_vk_ctx_begin(vk_transfer_ctx); + ctx->transfer_ctx = ggml_vk_create_context(ctx, ctx->device.lock()->transfer_queue); + ggml_vk_ctx_begin(ctx, ctx->transfer_ctx); } - ggml_vk_buffer_copy_async(vk_transfer_ctx, &src_extra->buffer_gpu, src_extra->offset, &dst_extra->buffer_gpu, dst_extra->offset, ggml_nbytes(src)); + vk_buffer src_buf = src_extra->buffer_gpu.lock(); + vk_buffer dst_buf = dst_extra->buffer_gpu.lock(); + + ggml_vk_buffer_copy_async(ctx->transfer_ctx, src_buf, src_extra->offset, dst_buf, dst_extra->offset, ggml_nbytes(src)); return true; } return false; - - UNUSED(backend); } GGML_CALL static void ggml_backend_vk_synchronize(ggml_backend_t backend) { #ifdef GGML_VULKAN_DEBUG std::cerr << "ggml_backend_vk_synchronize()" << std::endl; #endif - if(vk_transfer_ctx == nullptr) { + ggml_backend_vk_context * ctx = (ggml_backend_vk_context *)backend->context; + if(ctx->transfer_ctx == nullptr) { return; } - ggml_vk_ctx_end(vk_transfer_ctx); + ggml_vk_ctx_end(ctx->transfer_ctx); - for (auto& cpy : vk_transfer_ctx->in_memcpys) { + for (auto& cpy : ctx->transfer_ctx->in_memcpys) { memcpy(cpy.dst, cpy.src, cpy.n); } - ggml_vk_submit(vk_transfer_ctx, vk_fence); - VK_CHECK(vk_device.device.waitForFences({ vk_fence }, true, UINT64_MAX), "ggml_backend_vk_synchronize waitForFences"); - vk_device.device.resetFences({ vk_fence }); + ggml_vk_submit(ctx->transfer_ctx, ctx->fence); + VK_CHECK(ctx->device.lock()->device.waitForFences({ ctx->fence }, true, UINT64_MAX), "ggml_backend_vk_synchronize waitForFences"); + ctx->device.lock()->device.resetFences({ ctx->fence }); - for (auto& cpy : vk_transfer_ctx->out_memcpys) { + for (auto& cpy : ctx->transfer_ctx->out_memcpys) { memcpy(cpy.dst, cpy.src, cpy.n); } - vk_transfer_ctx = nullptr; - - UNUSED(backend); + ctx->transfer_ctx = nullptr; } GGML_CALL static bool ggml_backend_vk_graph_compute(ggml_backend_t backend, ggml_cgraph * cgraph) { - // ggml_backend_vk_context * vk_ctx = (ggml_backend_vk_context *)backend->context; + ggml_backend_vk_context * ctx = (ggml_backend_vk_context *)backend->context; for (int i = 0; i < cgraph->n_nodes; i++) { - ggml_vk_preallocate_buffers_graph(cgraph->nodes[i]); + ggml_vk_preallocate_buffers_graph(ctx, cgraph->nodes[i]); } - ggml_vk_preallocate_buffers(); + ggml_vk_preallocate_buffers(ctx); int last_node = cgraph->n_nodes - 1; @@ -4644,7 +4982,7 @@ GGML_CALL static bool ggml_backend_vk_graph_compute(ggml_backend_t backend, ggml } for (int i = 0; i < cgraph->n_nodes; i++) { - ggml_vk_build_graph(cgraph->nodes[i], i == last_node); + ggml_vk_build_graph(ctx,cgraph->nodes[i], i == last_node); } ggml_compute_params params = {}; @@ -4657,19 +4995,19 @@ GGML_CALL static bool ggml_backend_vk_graph_compute(ggml_backend_t backend, ggml continue; } - bool ok = ggml_vk_compute_forward(¶ms, node); + bool ok = ggml_vk_compute_forward(ctx, ¶ms, node); if (!ok) { fprintf(stderr, "%s: error: op not supported %s (%s)\n", __func__, node->name, ggml_op_name(node->op)); } #ifdef GGML_VULKAN_CHECK_RESULTS else { - ggml_vk_check_results_1(¶ms, node); + ggml_vk_check_results_1(ctx, ¶ms, node); } #endif GGML_ASSERT(ok); } - ggml_vk_graph_cleanup(); + ggml_vk_graph_cleanup(ctx); return true; @@ -4734,7 +5072,7 @@ GGML_CALL static bool ggml_backend_vk_supports_op(ggml_backend_t backend, const } return false; } break; - // case GGML_OP_DUP: + case GGML_OP_DUP: // case GGML_OP_REPEAT: // { // ggml_type src0_type = op->src[0]->type; @@ -4786,18 +5124,30 @@ static ggml_backend_i ggml_backend_vk_interface = { /* .supports_op = */ ggml_backend_vk_supports_op, }; -GGML_CALL ggml_backend_t ggml_backend_vk_init() { - ggml_vk_init(); // TODO: remove from ggml.c +GGML_CALL ggml_backend_t ggml_backend_vk_init(size_t idx) { + if (vk_instance.initialized[idx]) { + return vk_instance.backends[idx]; + } +#ifdef GGML_VULKAN_DEBUG + std::cerr << "ggml_backend_vk_init(" << idx << ")" << std::endl; +#endif - ggml_backend_vk_context * ctx = new ggml_backend_vk_context { - /* .name = */ GGML_VK_NAME, + ggml_backend_vk_context * ctx = &vk_instance.contexts[idx]; + ggml_vk_init(ctx, idx); + ctx->name = GGML_VK_NAME + std::to_string(idx); + vk_instance.buffer_types[idx] = { + /* .iface = */ ggml_backend_vk_buffer_type_interface, + /* .context = */ new ggml_backend_vk_buffer_type_context{ ctx->name, ctx }, }; + vk_instance.initialized[idx] = true; ggml_backend_t vk_backend = new ggml_backend { /* .interface = */ ggml_backend_vk_interface, - /* .context = */ ctx + /* .context = */ &vk_instance.contexts[ctx->idx], }; + vk_instance.backends[idx] = vk_backend; + return vk_backend; } @@ -4805,20 +5155,47 @@ GGML_CALL bool ggml_backend_is_vk(ggml_backend_t backend) { return backend && backend->iface.get_name == ggml_backend_vk_name; } +GGML_CALL int ggml_backend_vk_get_device_count() { + return ggml_vk_get_device_count(); +} + +GGML_CALL void ggml_backend_vk_get_device_description(int device, char * description, size_t description_size) { + ggml_vk_get_device_description(device, description, description_size); +} + +GGML_CALL void ggml_backend_vk_get_device_memory(int device, size_t * free, size_t * total) { + GGML_ASSERT(device < vk_instance.device_indices.size()); + + vk::PhysicalDevice vkdev = vk_instance.instance.enumeratePhysicalDevices()[vk_instance.device_indices[device]]; + + vk::PhysicalDeviceMemoryProperties memprops = vkdev.getMemoryProperties(); + + for (const vk::MemoryHeap& heap : memprops.memoryHeaps) { + if (heap.flags & vk::MemoryHeapFlagBits::eDeviceLocal) { + *total = heap.size; + *free = heap.size; + break; + } + } +} + // backend registry GGML_CALL static ggml_backend_t ggml_backend_reg_vk_init(const char * params, void * user_data) { - ggml_backend_t vk_backend = ggml_backend_vk_init(); + ggml_backend_t vk_backend = ggml_backend_vk_init((int) (intptr_t) user_data); return vk_backend; UNUSED(params); - UNUSED(user_data); } extern "C" GGML_CALL int ggml_backend_vk_reg_devices(); GGML_CALL int ggml_backend_vk_reg_devices() { - ggml_backend_register(GGML_VK_NAME, ggml_backend_reg_vk_init, ggml_backend_vk_buffer_type(), nullptr); - return 1; + for (auto idx : vk_instance.device_indices) { + char name[128]; + snprintf(name, sizeof(name), "%s%ld", GGML_VK_NAME, idx); + ggml_backend_register(name, ggml_backend_reg_vk_init, ggml_backend_vk_buffer_type(idx), (void *) (intptr_t) idx); + } + return vk_instance.device_indices.size(); } // checks @@ -4874,7 +5251,7 @@ static void ggml_vk_print_tensor_area(const ggml_tensor * tensor, const void * d } } -static void ggml_vk_print_tensor(const ggml_tensor * tensor, const char * name) { +static void ggml_vk_print_tensor(ggml_backend_vk_context * ctx, const ggml_tensor * tensor, const char * name) { void * tensor_data = tensor->data; if (tensor->backend == GGML_BACKEND_GPU) { @@ -4883,7 +5260,7 @@ static void ggml_vk_print_tensor(const ggml_tensor * tensor, const char * name) ggml_tensor_extra_gpu * extra = (ggml_tensor_extra_gpu *) tensor->extra; - ggml_vk_buffer_read(&extra->buffer_gpu, extra->offset, tensor_data, tensor_size); + ggml_vk_buffer_read(ctx, extra->buffer_gpu, extra->offset, tensor_data, tensor_size); } std::cerr << "TENSOR CHECK " << name << " (" << tensor->name << "): " << ggml_op_name(tensor->op) << std::endl; @@ -4944,7 +5321,7 @@ void * comp_result; size_t comp_size; size_t comp_nb[GGML_MAX_DIMS]; size_t check_counter = 0; -static void ggml_vk_check_results_0(ggml_compute_params * params, ggml_tensor * tensor) { +static void ggml_vk_check_results_0(ggml_backend_vk_context * ctx, ggml_compute_params * params, ggml_tensor * tensor) { if (params->ith != 0) { return; } @@ -4966,7 +5343,7 @@ static void ggml_vk_check_results_0(ggml_compute_params * params, ggml_tensor * /*.no_alloc =*/ false, }; - struct ggml_context * ctx = ggml_init(iparams); + struct ggml_context * ggml_ctx = ggml_init(iparams); struct ggml_tensor * src0_clone = nullptr; struct ggml_tensor * src1_clone = nullptr; @@ -4979,7 +5356,7 @@ static void ggml_vk_check_results_0(ggml_compute_params * params, ggml_tensor * void * src1_buffer; if (src0 != nullptr) { - src0_clone = ggml_dup_tensor(ctx, src0); + src0_clone = ggml_dup_tensor(ggml_ctx, src0); src0_size = ggml_nbytes(src0); @@ -4995,7 +5372,7 @@ static void ggml_vk_check_results_0(ggml_compute_params * params, ggml_tensor * for (int i3 = 0; i3 < src0->ne[3]; i3++) { for (int i2 = 0; i2 < src0->ne[2]; i2++) { const int idx = i3*src0->ne[2] + i2; - ggml_vk_buffer_read(&extra->buffer_gpu, offset + idx * src0->nb[2], ((char *)src0_clone->data + idx * src0_clone->nb[2]), src0->ne[1] * src0->nb[1]); + ggml_vk_buffer_read(ctx, extra->buffer_gpu, offset + idx * src0->nb[2], ((char *)src0_clone->data + idx * src0_clone->nb[2]), src0->ne[1] * src0->nb[1]); } } @@ -5005,10 +5382,10 @@ static void ggml_vk_check_results_0(ggml_compute_params * params, ggml_tensor * src0_clone->nb[i] = src0_clone->nb[i - 1]*src0_clone->ne[i - 1]; } } else { - if (offset + src0_size >= extra->buffer_gpu.size) { - src0_size = extra->buffer_gpu.size - offset; + if (offset + src0_size >= extra->buffer_gpu->size) { + src0_size = extra->buffer_gpu->size - offset; } - ggml_vk_buffer_read(&extra->buffer_gpu, offset, src0_clone->data, src0_size); + ggml_vk_buffer_read(ctx, extra->buffer_gpu, offset, src0_clone->data, src0_size); memcpy(src0_clone->nb, src0->nb, sizeof(size_t) * GGML_MAX_DIMS); } } else { @@ -5016,13 +5393,13 @@ static void ggml_vk_check_results_0(ggml_compute_params * params, ggml_tensor * } if (vk_output_tensor > 0 && vk_output_tensor == check_counter) { - ggml_vk_print_tensor(src0, "src0"); + ggml_vk_print_tensor(ctx, src0, "src0"); } ggml_vk_check_tensor(std::string(ggml_op_name(tensor->op)) + "->src0", src0_clone); } if (src1 != nullptr) { - src1_clone = ggml_dup_tensor(ctx, src1); + src1_clone = ggml_dup_tensor(ggml_ctx, src1); src1_size = ggml_nbytes(src1); @@ -5038,7 +5415,7 @@ static void ggml_vk_check_results_0(ggml_compute_params * params, ggml_tensor * for (int i3 = 0; i3 < src1->ne[3]; i3++) { for (int i2 = 0; i2 < src1->ne[2]; i2++) { const int idx = i3*src1->ne[2] + i2; - ggml_vk_buffer_read(&extra->buffer_gpu, offset + idx * src1->nb[2], ((char *)src1_clone->data + idx * src1_clone->nb[2]), src1->ne[1] * src1->nb[1]); + ggml_vk_buffer_read(ctx, extra->buffer_gpu, offset + idx * src1->nb[2], ((char *)src1_clone->data + idx * src1_clone->nb[2]), src1->ne[1] * src1->nb[1]); } } @@ -5048,10 +5425,10 @@ static void ggml_vk_check_results_0(ggml_compute_params * params, ggml_tensor * src1_clone->nb[i] = src1_clone->nb[i - 1]*src1_clone->ne[i - 1]; } } else { - if (offset + src1_size >= extra->buffer_gpu.size) { - src1_size = extra->buffer_gpu.size - offset; + if (offset + src1_size >= extra->buffer_gpu->size) { + src1_size = extra->buffer_gpu->size - offset; } - ggml_vk_buffer_read(&extra->buffer_gpu, offset, src1_clone->data, src1_size); + ggml_vk_buffer_read(ctx, extra->buffer_gpu, offset, src1_clone->data, src1_size); memcpy(src1_clone->nb, src1->nb, sizeof(size_t) * GGML_MAX_DIMS); } } else { @@ -5059,7 +5436,7 @@ static void ggml_vk_check_results_0(ggml_compute_params * params, ggml_tensor * } if (vk_output_tensor > 0 && vk_output_tensor == check_counter) { - ggml_vk_print_tensor(src1, "src1"); + ggml_vk_print_tensor(ctx, src1, "src1"); std::cerr << "TENSOR CHECK: " << ggml_op_name(src1_clone->op) << " (check " << check_counter << ")" << std::endl; std::cerr << "src1_clone=" << tensor << " src1_clone->backend: " << src1_clone->backend << " src1_clone->type: " << ggml_type_name(src1_clone->type) << " ne0=" << src1_clone->ne[0] << " nb0=" << src1_clone->nb[0] << " ne1=" << src1_clone->ne[1] << " nb1=" << src1_clone->nb[1] << " ne2=" << src1_clone->ne[2] << " nb2=" << src1_clone->nb[2] << " ne3=" << src1_clone->ne[3] << " nb3=" << src1_clone->nb[3] << std::endl; if (src1->src[0] != nullptr) { @@ -5082,51 +5459,51 @@ static void ggml_vk_check_results_0(ggml_compute_params * params, ggml_tensor * } if (tensor->op == GGML_OP_MUL_MAT) { - tensor_clone = ggml_mul_mat(ctx, src0_clone, src1_clone); + tensor_clone = ggml_mul_mat(ggml_ctx, src0_clone, src1_clone); } else if (tensor->op == GGML_OP_MUL) { - tensor_clone = ggml_mul(ctx, src0_clone, src1_clone); + tensor_clone = ggml_mul(ggml_ctx, src0_clone, src1_clone); } else if (tensor->op == GGML_OP_SCALE) { - tensor_clone = ggml_scale(ctx, src0_clone, ((float *)tensor->op_params)[0]); + tensor_clone = ggml_scale(ggml_ctx, src0_clone, ((float *)tensor->op_params)[0]); } else if (tensor->op == GGML_OP_SQR) { - tensor_clone = ggml_sqr(ctx, src0_clone); + tensor_clone = ggml_sqr(ggml_ctx, src0_clone); } else if (tensor->op == GGML_OP_CLAMP) { - tensor_clone = ggml_clamp(ctx, src0_clone, ((float *)tensor->op_params)[0], ((float *)tensor->op_params)[1]); + tensor_clone = ggml_clamp(ggml_ctx, src0_clone, ((float *)tensor->op_params)[0], ((float *)tensor->op_params)[1]); } else if (tensor->op == GGML_OP_ADD) { - tensor_clone = ggml_add(ctx, src0_clone, src1_clone); + tensor_clone = ggml_add(ggml_ctx, src0_clone, src1_clone); } else if (tensor->op == GGML_OP_NORM) { - tensor_clone = ggml_norm(ctx, src0_clone, *(float *)tensor->op_params); + tensor_clone = ggml_norm(ggml_ctx, src0_clone, *(float *)tensor->op_params); } else if (tensor->op == GGML_OP_RMS_NORM) { - tensor_clone = ggml_rms_norm(ctx, src0_clone, *(float *)tensor->op_params); + tensor_clone = ggml_rms_norm(ggml_ctx, src0_clone, *(float *)tensor->op_params); } else if (tensor->op == GGML_OP_SOFT_MAX) { if (src1 != nullptr) { - tensor_clone = ggml_soft_max_ext(ctx, src0_clone, src1_clone, *(float *)tensor->op_params); + tensor_clone = ggml_soft_max_ext(ggml_ctx, src0_clone, src1_clone, *(float *)tensor->op_params); } else { - tensor_clone = ggml_soft_max(ctx, src0_clone); + tensor_clone = ggml_soft_max(ggml_ctx, src0_clone); } } else if (tensor->op == GGML_OP_DIAG_MASK_INF) { - tensor_clone = ggml_diag_mask_inf(ctx, src0_clone, *(float *)tensor->op_params); + tensor_clone = ggml_diag_mask_inf(ggml_ctx, src0_clone, *(float *)tensor->op_params); } else if (tensor->op == GGML_OP_ROPE) { const int n_dims = ((int32_t *) tensor->op_params)[1]; const int mode = ((int32_t *) tensor->op_params)[2]; - const int n_ctx = ((int32_t *) tensor->op_params)[3]; - const int n_orig_ctx = ((int32_t *) tensor->op_params)[4]; + const int n_ggml_ctx = ((int32_t *) tensor->op_params)[3]; + const int n_orig_ggml_ctx = ((int32_t *) tensor->op_params)[4]; float freq_base = ((float *) tensor->op_params)[5]; float freq_scale = ((float *) tensor->op_params)[6]; float ext_factor = ((float *) tensor->op_params)[7]; float attn_factor = ((float *) tensor->op_params)[8]; float beta_fast = ((float *) tensor->op_params)[9]; float beta_slow = ((float *) tensor->op_params)[10]; - tensor_clone = ggml_rope_custom(ctx, src0_clone, src1_clone, n_dims, mode, n_ctx, n_orig_ctx, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow); + tensor_clone = ggml_rope_custom(ggml_ctx, src0_clone, src1_clone, n_dims, mode, n_ggml_ctx, n_orig_ggml_ctx, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow); } else if (tensor->op == GGML_OP_UNARY) { switch (ggml_get_unary_op(tensor)) { case GGML_UNARY_OP_SILU: - tensor_clone = ggml_silu(ctx, src0_clone); + tensor_clone = ggml_silu(ggml_ctx, src0_clone); break; case GGML_UNARY_OP_GELU: - tensor_clone = ggml_gelu(ctx, src0_clone); + tensor_clone = ggml_gelu(ggml_ctx, src0_clone); break; case GGML_UNARY_OP_RELU: - tensor_clone = ggml_relu(ctx, src0_clone); + tensor_clone = ggml_relu(ggml_ctx, src0_clone); break; default: std::cerr << "Missing vk_check_results OP: " << ggml_op_name(tensor->op) << std::endl; @@ -5134,40 +5511,40 @@ static void ggml_vk_check_results_0(ggml_compute_params * params, ggml_tensor * } } else if (tensor->op == GGML_OP_CPY || tensor->op == GGML_OP_DUP) { if (src1 == nullptr) { - tensor_clone = ggml_dup(ctx, src0_clone); + tensor_clone = ggml_dup(ggml_ctx, src0_clone); tensor_clone->type = tensor->type; } else { - tensor_clone = ggml_cpy(ctx, src0_clone, src1_clone); + tensor_clone = ggml_cpy(ggml_ctx, src0_clone, src1_clone); } } else if (tensor->op == GGML_OP_CONT) { - tensor_clone = ggml_cont_4d(ctx, src0_clone, tensor->ne[0], tensor->ne[1], tensor->ne[2], tensor->ne[3]); + tensor_clone = ggml_cont_4d(ggml_ctx, src0_clone, tensor->ne[0], tensor->ne[1], tensor->ne[2], tensor->ne[3]); } else if (tensor->op == GGML_OP_RESHAPE) { - tensor_clone = ggml_reshape_4d(ctx, src0_clone, tensor->ne[0], tensor->ne[1], tensor->ne[2], tensor->ne[3]); + tensor_clone = ggml_reshape_4d(ggml_ctx, src0_clone, tensor->ne[0], tensor->ne[1], tensor->ne[2], tensor->ne[3]); } else if (tensor->op == GGML_OP_VIEW) { - tensor_clone = ggml_view_4d(ctx, src0_clone, tensor->ne[0], tensor->ne[1], tensor->ne[2], tensor->ne[3], tensor->nb[1], tensor->nb[2], tensor->nb[3], ((int32_t *) tensor->op_params)[0]); + tensor_clone = ggml_view_4d(ggml_ctx, src0_clone, tensor->ne[0], tensor->ne[1], tensor->ne[2], tensor->ne[3], tensor->nb[1], tensor->nb[2], tensor->nb[3], ((int32_t *) tensor->op_params)[0]); } else if (tensor->op == GGML_OP_PERMUTE) { int32_t * params = (int32_t *)tensor->op_params; - tensor_clone = ggml_permute(ctx, src0_clone, params[0], params[1], params[2], params[3]); + tensor_clone = ggml_permute(ggml_ctx, src0_clone, params[0], params[1], params[2], params[3]); } else if (tensor->op == GGML_OP_TRANSPOSE) { - tensor_clone = ggml_transpose(ctx, src0_clone); + tensor_clone = ggml_transpose(ggml_ctx, src0_clone); } else { std::cerr << "Missing vk_check_results OP: " << ggml_op_name(tensor->op) << std::endl; GGML_ASSERT(false); } // Disable vulkan here to avoid the hooks in ggml.c - vk_disable = true; + ctx->disable = true; - ggml_cgraph * cgraph = ggml_new_graph(ctx); + ggml_cgraph * cgraph = ggml_new_graph(ggml_ctx); ggml_build_forward_expand(cgraph, tensor_clone); - ggml_graph_compute_with_ctx(ctx, cgraph, 8); + ggml_graph_compute_with_ctx(ggml_ctx, cgraph, 8); - vk_disable = false; + ctx->disable = false; ggml_vk_check_tensor(ggml_op_name(tensor->op), tensor_clone); if (vk_output_tensor > 0 && vk_output_tensor == check_counter) { - ggml_vk_print_tensor(tensor_clone, "tensor_clone"); + ggml_vk_print_tensor(ctx, tensor_clone, "tensor_clone"); } comp_size = ggml_nbytes(tensor_clone); @@ -5183,10 +5560,10 @@ static void ggml_vk_check_results_0(ggml_compute_params * params, ggml_tensor * free(src1_buffer); } - ggml_free(ctx); + ggml_free(ggml_ctx); } -void ggml_vk_check_results_1(ggml_compute_params * params, ggml_tensor * tensor) { +static void ggml_vk_check_results_1(ggml_backend_vk_context * ctx, ggml_compute_params * params, ggml_tensor * tensor) { if (params->ith != 0) { return; } @@ -5208,11 +5585,11 @@ void ggml_vk_check_results_1(ggml_compute_params * params, ggml_tensor * tensor) ggml_tensor_extra_gpu * extra = (ggml_tensor_extra_gpu *) tensor->extra; - if (extra->offset + tensor_size >= extra->buffer_gpu.size) { - tensor_size = extra->buffer_gpu.size - (extra->offset); + if (extra->offset + tensor_size >= extra->buffer_gpu->size) { + tensor_size = extra->buffer_gpu->size - (extra->offset); } - ggml_vk_buffer_read(&extra->buffer_gpu, extra->offset, tensor_data, tensor_size); + ggml_vk_buffer_read(ctx, extra->buffer_gpu, extra->offset, tensor_data, tensor_size); } float first_error_result = -1.0f; @@ -5339,4 +5716,10 @@ void ggml_vk_check_results_1(ggml_compute_params * params, ggml_tensor * tensor) free(tensor_data); } } + +void ggml_vk_check_results_1_cpu_assist(struct ggml_compute_params * params, struct ggml_tensor * tensor) { + ggml_backend_vk_context * ctx = &vk_instance.contexts[0]; + + ggml_vk_check_results_0(ctx, params, tensor); +} #endif diff --git a/ggml-vulkan.h b/ggml-vulkan.h index eb8a148e2..9645126b4 100644 --- a/ggml-vulkan.h +++ b/ggml-vulkan.h @@ -8,24 +8,29 @@ extern "C" { #endif #define GGML_VK_NAME "Vulkan" +#define GGML_VK_MAX_DEVICES 16 -GGML_API void ggml_vk_init(void); +GGML_API void ggml_vk_init_cpu_assist(void); -GGML_API void ggml_vk_preallocate_buffers_graph(struct ggml_tensor * node); -GGML_API void ggml_vk_preallocate_buffers(void); -GGML_API void ggml_vk_build_graph(struct ggml_tensor * node, bool last_node); -GGML_API bool ggml_vk_compute_forward(struct ggml_compute_params * params, struct ggml_tensor * tensor); +GGML_API void ggml_vk_preallocate_buffers_graph_cpu_assist(struct ggml_tensor * node); +GGML_API void ggml_vk_preallocate_buffers_cpu_assist(void); +GGML_API void ggml_vk_build_graph_cpu_assist(struct ggml_tensor * node, bool last_node); +GGML_API bool ggml_vk_compute_forward_cpu_assist(struct ggml_compute_params * params, struct ggml_tensor * tensor); #ifdef GGML_VULKAN_CHECK_RESULTS -void ggml_vk_check_results_1(struct ggml_compute_params * params, struct ggml_tensor * tensor); +void ggml_vk_check_results_1_cpu_assist(struct ggml_compute_params * params, struct ggml_tensor * tensor); #endif -GGML_API void ggml_vk_graph_cleanup(void); +GGML_API void ggml_vk_graph_cleanup_cpu_assist(void); +GGML_API void ggml_vk_free_cpu_assist(void); // backend API -GGML_API GGML_CALL ggml_backend_t ggml_backend_vk_init(void); +GGML_API GGML_CALL ggml_backend_t ggml_backend_vk_init(size_t dev_num); GGML_API GGML_CALL bool ggml_backend_is_vk(ggml_backend_t backend); +GGML_API GGML_CALL int ggml_backend_vk_get_device_count(void); +GGML_API GGML_CALL void ggml_backend_vk_get_device_description(int device, char * description, size_t description_size); +GGML_API GGML_CALL void ggml_backend_vk_get_device_memory(int device, size_t * free, size_t * total); -GGML_API GGML_CALL ggml_backend_buffer_type_t ggml_backend_vk_buffer_type(void); +GGML_API GGML_CALL ggml_backend_buffer_type_t ggml_backend_vk_buffer_type(size_t dev_num); // pinned host buffer for use with the CPU backend for faster copies between CPU and GPU GGML_API GGML_CALL ggml_backend_buffer_type_t ggml_backend_vk_host_buffer_type(void); diff --git a/ggml.c b/ggml.c index b9ec0c981..f783a6fd3 100644 --- a/ggml.c +++ b/ggml.c @@ -2343,7 +2343,7 @@ struct ggml_context * ggml_init(struct ggml_init_params params) { #elif defined(GGML_USE_CLBLAST) ggml_cl_init(); #elif defined(GGML_USE_VULKAN) - ggml_vk_init(); + ggml_vk_init_cpu_assist(); #elif defined(GGML_USE_SYCL) ggml_init_sycl(); #endif @@ -14850,10 +14850,10 @@ static void ggml_compute_forward(struct ggml_compute_params * params, struct ggm GGML_ASSERT(tensor->src[0] == NULL || tensor->src[0]->backend == GGML_BACKEND_CPU); GGML_ASSERT(tensor->src[1] == NULL || tensor->src[1]->backend == GGML_BACKEND_CPU); #elif defined(GGML_USE_VULKAN) - const bool skip_cpu = ggml_vk_compute_forward(params, tensor); + const bool skip_cpu = ggml_vk_compute_forward_cpu_assist(params, tensor); #ifdef GGML_VULKAN_CHECK_RESULTS if (skip_cpu) { - ggml_vk_check_results_1(params, tensor); + ggml_vk_check_results_1_cpu_assist(params, tensor); } #endif if (skip_cpu) { @@ -17269,12 +17269,12 @@ int ggml_graph_compute(struct ggml_cgraph * cgraph, struct ggml_cplan * cplan) { #ifdef GGML_USE_VULKAN for (int i = 0; i < cgraph->n_nodes; i++) { - ggml_vk_preallocate_buffers_graph(cgraph->nodes[i]); + ggml_vk_preallocate_buffers_graph_cpu_assist(cgraph->nodes[i]); } - ggml_vk_preallocate_buffers(); + ggml_vk_preallocate_buffers_cpu_assist(); for (int i = 0; i < cgraph->n_nodes; i++) { - ggml_vk_build_graph(cgraph->nodes[i], i == cgraph->n_nodes - 1); + ggml_vk_build_graph_cpu_assist(cgraph->nodes[i], i == cgraph->n_nodes - 1); } #endif @@ -17330,7 +17330,7 @@ int ggml_graph_compute(struct ggml_cgraph * cgraph, struct ggml_cplan * cplan) { } #ifdef GGML_USE_VULKAN - ggml_vk_graph_cleanup(); + ggml_vk_graph_cleanup_cpu_assist(); #endif // performance stats (graph) diff --git a/llama.cpp b/llama.cpp index f3c5146d1..c45ae1d50 100644 --- a/llama.cpp +++ b/llama.cpp @@ -1355,7 +1355,7 @@ static ggml_backend_buffer_type_t llama_default_buffer_type_offload(int gpu) { #elif defined(GGML_USE_CUBLAS) buft = ggml_backend_cuda_buffer_type(gpu); #elif defined(GGML_USE_VULKAN) - buft = ggml_backend_vk_buffer_type(); + buft = ggml_backend_vk_buffer_type(gpu); #elif defined(GGML_USE_SYCL) buft = ggml_backend_sycl_buffer_type(gpu); #elif defined(GGML_USE_CLBLAST) @@ -1392,6 +1392,33 @@ static ggml_backend_buffer_type_t llama_default_buffer_type_split(int fallback_g GGML_UNUSED(tensor_split); } +static size_t llama_get_device_count() { +#if defined(GGML_USE_CUBLAS) + return ggml_backend_cuda_get_device_count(); +#elif defined(GGML_USE_VULKAN) + return ggml_backend_vk_get_device_count(); +#else + return 1; +#endif +} + +static size_t llama_get_device_memory(int device) { +#if defined(GGML_USE_CUBLAS) + size_t total; + size_t free; + ggml_backend_cuda_get_device_memory(device, &total, &free); + return free; +#elif defined(GGML_USE_VULKAN) + size_t total; + size_t free; + ggml_backend_vk_get_device_memory(device, &total, &free); + return free; +#else + return 1; + GGML_UNUSED(device); +#endif +} + // // globals // @@ -1763,6 +1790,10 @@ struct llama_context { ggml_backend_free(backend); } +#ifdef GGML_USE_VULKAN + ggml_vk_free_cpu_assist(); +#endif + ggml_backend_buffer_free(buf_input); ggml_free(ctx_input); } @@ -3436,22 +3467,18 @@ static bool llm_load_tensors( model.buft_layer[i] = llama_default_buffer_type_cpu(true); } -#ifdef GGML_USE_CUBLAS if (split_mode == LLAMA_SPLIT_LAYER) { // calculate the split points - int device_count = ggml_backend_cuda_get_device_count(); + int device_count = llama_get_device_count(); bool all_zero = tensor_split == nullptr || std::all_of(tensor_split, tensor_split + device_count, [](float x) { return x == 0.0f; }); - float splits[GGML_CUDA_MAX_DEVICES]; + std::vector splits(device_count); if (all_zero) { // default split, by free memory for (int i = 0; i < device_count; ++i) { - size_t total; - size_t free; - ggml_backend_cuda_get_device_memory(i, &total, &free); - splits[i] = free; + splits[i] = llama_get_device_memory(i); } } else { - std::copy(tensor_split, tensor_split + device_count, splits); + std::copy(tensor_split, tensor_split + device_count, splits.begin()); } // sum and normalize the splits to get the split points @@ -3467,19 +3494,17 @@ static bool llm_load_tensors( // assign the repeating layers to the devices according to the splits int act_gpu_layers = std::min(n_gpu_layers, (int)n_layer + 1); for (int64_t i = i_gpu_start; i < n_layer; ++i) { - int layer_gpu = std::upper_bound(splits, splits + device_count, float(i - i_gpu_start)/act_gpu_layers) - splits; + int layer_gpu = std::upper_bound(splits.begin(), splits.begin() + device_count, float(i - i_gpu_start)/act_gpu_layers) - splits.begin(); model.buft_layer[i] = llama_default_buffer_type_offload(layer_gpu); } // assign the output layer if (n_gpu_layers > n_layer) { - int layer_gpu = std::upper_bound(splits, splits + device_count, float(act_gpu_layers - 1)/act_gpu_layers) - splits; + int layer_gpu = std::upper_bound(splits.begin(), splits.begin() + device_count, float(act_gpu_layers - 1)/act_gpu_layers) - splits.begin(); model.buft_output = llama_default_buffer_type_offload(layer_gpu); } else { model.buft_output = llama_default_buffer_type_cpu(true); } - } else -#endif - { + } else { ggml_backend_buffer_type_t split_buft; if (split_mode == LLAMA_SPLIT_ROW) { split_buft = llama_default_buffer_type_split(main_gpu, tensor_split); @@ -10483,6 +10508,8 @@ size_t llama_max_devices(void) { return GGML_CUDA_MAX_DEVICES; #elif defined(GGML_USE_SYCL) return GGML_SYCL_MAX_DEVICES; +#elif defined(GGML_USE_VULKAN) + return GGML_VK_MAX_DEVICES; #else return 1; #endif @@ -10690,13 +10717,15 @@ struct llama_context * llama_new_context_with_model( } #elif defined(GGML_USE_VULKAN) if (model->n_gpu_layers > 0) { - ggml_backend_t backend = ggml_backend_vk_init(); - if (backend == nullptr) { - LLAMA_LOG_ERROR("%s: failed to initialize Vulkan backend\n", __func__); - llama_free(ctx); - return nullptr; + for (int device = 0; device < ggml_backend_vk_get_device_count(); ++device) { + ggml_backend_t backend = ggml_backend_vk_init(device); + if (backend == nullptr) { + LLAMA_LOG_ERROR("%s: failed to initialize Vulkan%d backend\n", __func__, device); + llama_free(ctx); + return nullptr; + } + ctx->backends.push_back(backend); } - ctx->backends.push_back(backend); } #elif defined(GGML_USE_SYCL) if (model->n_gpu_layers > 0) { From 0ef46da632c32faa1a538e5dc180994e8bbb46e1 Mon Sep 17 00:00:00 2001 From: Xiao-Yong Jin Date: Wed, 7 Feb 2024 02:17:25 -0600 Subject: [PATCH 287/812] llava-cli : always tokenize special tokens (#5382) * llava-cli: tokenize special tokens in prompt * llava-cli: use the escape CLI argument, remove incomplete separate escaping process --- examples/llava/llava-cli.cpp | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/examples/llava/llava-cli.cpp b/examples/llava/llava-cli.cpp index 6ac70ba69..031e9806d 100644 --- a/examples/llava/llava-cli.cpp +++ b/examples/llava/llava-cli.cpp @@ -34,7 +34,7 @@ static bool eval_id(struct llama_context * ctx_llama, int id, int * n_past) { static bool eval_string(struct llama_context * ctx_llama, const char* str, int n_batch, int * n_past, bool add_bos){ std::string str2 = str; - std::vector embd_inp = ::llama_tokenize(ctx_llama, str2, add_bos); + std::vector embd_inp = ::llama_tokenize(ctx_llama, str2, add_bos, true); eval_tokens(ctx_llama, embd_inp, n_batch, n_past); return true; } @@ -152,20 +152,8 @@ static void process_prompt(struct llava_context * ctx_llava, struct llava_image_ size_t image_pos = prompt.find(""); if (image_pos != std::string::npos) { // new templating mode: Provide the full prompt including system message and use as a placeholder for the image - system_prompt = prompt.substr(0, image_pos); user_prompt = prompt.substr(image_pos + std::string("").length()); - // We replace \n with actual newlines in user_prompt, just in case -e was not used in templating string - size_t pos = 0; - while ((pos = user_prompt.find("\\n", pos)) != std::string::npos) { - user_prompt.replace(pos, 2, "\n"); - pos += 1; // Advance past the replaced newline - } - while ((pos = system_prompt.find("\\n", pos)) != std::string::npos) { - system_prompt.replace(pos, 2, "\n"); - pos += 1; // Advance past the replaced newline - } - printf("system_prompt: %s\n", system_prompt.c_str()); printf("user_prompt: %s\n", user_prompt.c_str()); } else { From 10afa6f1d11ebc9fcc1085f468170002cbf6e2b5 Mon Sep 17 00:00:00 2001 From: Neo Zhang Jianyu Date: Wed, 7 Feb 2024 18:16:55 +0800 Subject: [PATCH 288/812] [SYCL] update install make by w64devkit (#5297) --- README-sycl.md | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/README-sycl.md b/README-sycl.md index 7aa4274a9..e3a8e726e 100644 --- a/README-sycl.md +++ b/README-sycl.md @@ -311,15 +311,13 @@ Output (example): a. Download & install cmake for Windows: https://cmake.org/download/ -b. Download & install make for Windows provided by mingw-w64 +b. Download & install mingw-w64 make for Windows provided by w64devkit -- Download binary package for Windows in https://github.com/niXman/mingw-builds-binaries/releases. +- Download the latest fortran version of [w64devkit](https://github.com/skeeto/w64devkit/releases). - Like [x86_64-13.2.0-release-win32-seh-msvcrt-rt_v11-rev1.7z](https://github.com/niXman/mingw-builds-binaries/releases/download/13.2.0-rt_v11-rev1/x86_64-13.2.0-release-win32-seh-msvcrt-rt_v11-rev1.7z). +- Extract `w64devkit` on your pc. -- Unzip the binary package. In the **bin** sub-folder and rename **xxx-make.exe** to **make.exe**. - -- Add the **bin** folder path in the Windows system PATH environment. +- Add the **bin** folder path in the Windows system PATH environment, like `C:\xxx\w64devkit\bin\`. ### Build locally: From aa7ab99be29b633263803f2e185265734c2d9427 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20G=C3=A4=C3=9Fler?= Date: Wed, 7 Feb 2024 12:40:26 +0100 Subject: [PATCH 289/812] CUDA: fixed mmvq kernel for bs 2,3,4 and -sm row (#5386) --- ggml-cuda.cu | 66 +++++++++++++++++++++++++++++++--------------------- 1 file changed, 39 insertions(+), 27 deletions(-) diff --git a/ggml-cuda.cu b/ggml-cuda.cu index 3b828375e..db9da2459 100644 --- a/ggml-cuda.cu +++ b/ggml-cuda.cu @@ -5313,7 +5313,7 @@ template static __global__ void template static __global__ void mul_mat_vec_q( const void * __restrict__ vx, const void * __restrict__ vy, float * __restrict__ dst, - const int ncols_x, const int nrows_x, const int nrows_y, const int ncols_y_par) { + const int ncols_x, const int nrows_x, const int nrows_y, const int ncols_y_par, const int nrows_dst) { const int ncols_y = ncols_y_template != 0 ? ncols_y_template : ncols_y_par; @@ -5352,7 +5352,7 @@ static __global__ void mul_mat_vec_q( tmp[j] = warp_reduce_sum(tmp[j]); if (threadIdx.x == 0) { - dst[j*nrows_x + row] = tmp[j]; + dst[j*nrows_dst + row] = tmp[j]; } } } @@ -6828,7 +6828,7 @@ static void convert_mul_mat_vec_f16_cuda(const void * vx, const dfloat * y, floa template static void mul_mat_vec_q_cuda( const void * vx, const void * vy, float * dst, - const int ncols_x, const int nrows_x, const int nrows_y, const int ncols_y, cudaStream_t stream) { + const int ncols_x, const int nrows_x, const int nrows_y, const int ncols_y, const int nrows_dst, cudaStream_t stream) { GGML_ASSERT(ncols_x % qk == 0); GGML_ASSERT(ncols_y <= 4); @@ -6839,40 +6839,40 @@ static void mul_mat_vec_q_cuda( switch (ncols_y) { case 1: mul_mat_vec_q<1, qk, qi, block_q_t, vdr, vec_dot> - <<>>(vx, vy, dst, ncols_x, nrows_x, nrows_y, ncols_y); + <<>>(vx, vy, dst, ncols_x, nrows_x, nrows_y, ncols_y, nrows_dst); break; case 2: mul_mat_vec_q<2, qk, qi, block_q_t, vdr, vec_dot> - <<>>(vx, vy, dst, ncols_x, nrows_x, nrows_y, ncols_y); + <<>>(vx, vy, dst, ncols_x, nrows_x, nrows_y, ncols_y, nrows_dst); break; case 3: mul_mat_vec_q<3, qk, qi, block_q_t, vdr, vec_dot> - <<>>(vx, vy, dst, ncols_x, nrows_x, nrows_y, ncols_y); + <<>>(vx, vy, dst, ncols_x, nrows_x, nrows_y, ncols_y, nrows_dst); break; case 4: mul_mat_vec_q<4, qk, qi, block_q_t, vdr, vec_dot> - <<>>(vx, vy, dst, ncols_x, nrows_x, nrows_y, ncols_y); + <<>>(vx, vy, dst, ncols_x, nrows_x, nrows_y, ncols_y, nrows_dst); break; // case 5: // mul_mat_vec_q<5, qk, qi, block_q_t, vdr, vec_dot> - // <<>>(vx, vy, dst, ncols_x, nrows_x, nrows_y, ncols_y); + // <<>>(vx, vy, dst, ncols_x, nrows_x, nrows_y, ncols_y, nrows_dst); // break; // case 6: // mul_mat_vec_q<6, qk, qi, block_q_t, vdr, vec_dot> - // <<>>(vx, vy, dst, ncols_x, nrows_x, nrows_y, ncols_y); + // <<>>(vx, vy, dst, ncols_x, nrows_x, nrows_y, ncols_y, nrows_dst); // break; // case 7: // mul_mat_vec_q<7, qk, qi, block_q_t, vdr, vec_dot> - // <<>>(vx, vy, dst, ncols_x, nrows_x, nrows_y, ncols_y); + // <<>>(vx, vy, dst, ncols_x, nrows_x, nrows_y, ncols_y, nrows_dst); // break; // case 8: // mul_mat_vec_q<8, qk, qi, block_q_t, vdr, vec_dot> - // <<>>(vx, vy, dst, ncols_x, nrows_x, nrows_y, ncols_y); + // <<>>(vx, vy, dst, ncols_x, nrows_x, nrows_y, ncols_y, nrows_dst); // break; default: GGML_ASSERT(false); // mul_mat_vec_q<0, qk, qi, block_q_t, vdr, vec_dot> - // <<>>(vx, vy, dst, ncols_x, nrows_x, nrows_y, ncols_y); + // <<>>(vx, vy, dst, ncols_x, nrows_x, nrows_y, ncols_y, nrows_dst); break; } } @@ -8391,7 +8391,7 @@ static void ggml_cuda_op_mul_mat_q( CUDA_CHECK(cudaGetDevice(&id)); // the main device has a larger memory buffer to hold the results from all GPUs - // nrows_dst == nrows of the matrix that the dequantize_mul_mat kernel writes into + // nrows_dst == nrows of the matrix that the kernel writes into const int64_t nrows_dst = dst->backend == GGML_BACKEND_GPU && id == g_main_device ? ne0 : row_diff; switch (src0->type) { @@ -8525,58 +8525,70 @@ static void ggml_cuda_op_mul_mat_vec_q( const int64_t ne00 = src0->ne[0]; const int64_t row_diff = row_high - row_low; + const int64_t ne10 = src1->ne[0]; + GGML_ASSERT(ne10 % QK8_1 == 0); + + const int64_t ne0 = dst->ne[0]; + + int id; + CUDA_CHECK(cudaGetDevice(&id)); + + // the main device has a larger memory buffer to hold the results from all GPUs + // nrows_dst == nrows of the matrix that the kernel writes into + const int64_t nrows_dst = dst->backend == GGML_BACKEND_GPU && id == g_main_device ? ne0 : row_diff; + switch (src0->type) { case GGML_TYPE_Q4_0: mul_mat_vec_q_cuda - (src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_padded_row_size, src1_ncols, stream); + (src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_padded_row_size, src1_ncols, nrows_dst, stream); break; case GGML_TYPE_Q4_1: mul_mat_vec_q_cuda - (src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_padded_row_size, src1_ncols, stream); + (src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_padded_row_size, src1_ncols, nrows_dst, stream); break; case GGML_TYPE_Q5_0: mul_mat_vec_q_cuda - (src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_padded_row_size, src1_ncols, stream); + (src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_padded_row_size, src1_ncols, nrows_dst, stream); break; case GGML_TYPE_Q5_1: mul_mat_vec_q_cuda - (src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_padded_row_size, src1_ncols, stream); + (src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_padded_row_size, src1_ncols, nrows_dst, stream); break; case GGML_TYPE_Q8_0: mul_mat_vec_q_cuda - (src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_padded_row_size, src1_ncols, stream); + (src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_padded_row_size, src1_ncols, nrows_dst, stream); break; case GGML_TYPE_Q2_K: mul_mat_vec_q_cuda - (src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_padded_row_size, src1_ncols, stream); + (src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_padded_row_size, src1_ncols, nrows_dst, stream); break; case GGML_TYPE_Q3_K: mul_mat_vec_q_cuda - (src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_padded_row_size, src1_ncols, stream); + (src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_padded_row_size, src1_ncols, nrows_dst, stream); break; case GGML_TYPE_Q4_K: mul_mat_vec_q_cuda - (src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_padded_row_size, src1_ncols, stream); + (src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_padded_row_size, src1_ncols, nrows_dst, stream); break; case GGML_TYPE_Q5_K: mul_mat_vec_q_cuda - (src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_padded_row_size, src1_ncols, stream); + (src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_padded_row_size, src1_ncols, nrows_dst, stream); break; case GGML_TYPE_Q6_K: mul_mat_vec_q_cuda - (src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_padded_row_size, src1_ncols, stream); + (src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_padded_row_size, src1_ncols, nrows_dst, stream); break; case GGML_TYPE_IQ2_XXS: mul_mat_vec_q_cuda - (src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_padded_row_size, src1_ncols, stream); + (src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_padded_row_size, src1_ncols, nrows_dst, stream); break; case GGML_TYPE_IQ2_XS: mul_mat_vec_q_cuda - (src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_padded_row_size, src1_ncols, stream); + (src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_padded_row_size, src1_ncols, nrows_dst, stream); break; case GGML_TYPE_IQ3_XXS: mul_mat_vec_q_cuda - (src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_padded_row_size, src1_ncols, stream); + (src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_padded_row_size, src1_ncols, nrows_dst, stream); break; default: GGML_ASSERT(false); @@ -9909,7 +9921,7 @@ static void ggml_cuda_mul_mat(const ggml_tensor * src0, const ggml_tensor * src1 ggml_cuda_op_mul_mat(src0, src1, dst, ggml_cuda_op_dequantize_mul_mat_vec, false); } } else { - if (src1->ne[1] <= 4 && min_compute_capability >= MIN_CC_DP4A && ggml_is_quantized(src0->type)) { + if (src1->ne[1] <= 4 && min_compute_capability >= MIN_CC_DP4A && ggml_is_quantized(src0->type) && src1->type == GGML_TYPE_F32) { ggml_cuda_op_mul_mat(src0, src1, dst, ggml_cuda_op_mul_mat_vec_q, true); } else if (use_mul_mat_q) { ggml_cuda_op_mul_mat(src0, src1, dst, ggml_cuda_op_mul_mat_q, true); From b906596bb775b17656c2e51d5ab1b347faab6860 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20Tom=C5=A1=C3=ADk?= Date: Wed, 7 Feb 2024 19:44:52 +0100 Subject: [PATCH 290/812] Add Ava in the list of llama.cpp UIs (#4362) --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 0509b0ba1..7e1187349 100644 --- a/README.md +++ b/README.md @@ -150,6 +150,7 @@ Unless otherwise noted these projects are open-source with permissive licensing: - [ollama/ollama](https://github.com/ollama/ollama) - [oobabooga/text-generation-webui](https://github.com/oobabooga/text-generation-webui) (AGPL) - [psugihara/FreeChat](https://github.com/psugihara/FreeChat) +- [cztomsik/ava](https://github.com/cztomsik/ava) (MIT) - [ptsochantaris/emeltal](https://github.com/ptsochantaris/emeltal) - [pythops/tenere](https://github.com/pythops/tenere) (AGPL) - [semperai/amica](https://github.com/semperai/amica) From 8c933b70c21e05b685d476d0a1f36b34cbda7365 Mon Sep 17 00:00:00 2001 From: Ebey Abraham Date: Wed, 7 Feb 2024 21:11:30 +0000 Subject: [PATCH 291/812] fix typo in readme (#5399) Co-authored-by: Ebey Abraham --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 7e1187349..66166c01b 100644 --- a/README.md +++ b/README.md @@ -680,7 +680,7 @@ python3 -m pip install -r requirements.txt python3 convert.py models/mymodel/ # [Optional] for models using BPE tokenizers -python convert.py models/mymodel/ --vocabtype bpe +python convert.py models/mymodel/ --vocab-type bpe # quantize the model to 4-bits (using Q4_K_M method) ./quantize ./models/mymodel/ggml-model-f16.gguf ./models/mymodel/ggml-model-Q4_K_M.gguf Q4_K_M From c4fbb6717c684196bd13b72d21747557130914e8 Mon Sep 17 00:00:00 2001 From: Michael Podvitskiy Date: Wed, 7 Feb 2024 22:39:23 +0100 Subject: [PATCH 292/812] CMAKE_OSX_ARCHITECTURES for MacOS cross compilation (#5393) Co-authored-by: Jared Van Bortel --- CMakeLists.txt | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 427015be5..a544f2da6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -850,7 +850,9 @@ endif() set(ARCH_FLAGS "") -if ((${CMAKE_SYSTEM_PROCESSOR} MATCHES "arm") OR (${CMAKE_SYSTEM_PROCESSOR} MATCHES "aarch64") OR ("${CMAKE_GENERATOR_PLATFORM_LWR}" MATCHES "arm64")) +if (CMAKE_OSX_ARCHITECTURES STREQUAL "arm64" OR CMAKE_GENERATOR_PLATFORM_LWR STREQUAL "arm64" OR + (NOT CMAKE_OSX_ARCHITECTURES AND NOT CMAKE_GENERATOR_PLATFORM_LWR AND + CMAKE_SYSTEM_PROCESSOR MATCHES "^(aarch64|arm.*|ARM64)$")) message(STATUS "ARM detected") if (MSVC) add_compile_definitions(__ARM_NEON) @@ -876,7 +878,9 @@ if ((${CMAKE_SYSTEM_PROCESSOR} MATCHES "arm") OR (${CMAKE_SYSTEM_PROCESSOR} MATC list(APPEND ARCH_FLAGS -mno-unaligned-access) endif() endif() -elseif (${CMAKE_SYSTEM_PROCESSOR} MATCHES "^(x86_64|i686|AMD64)$" OR "${CMAKE_GENERATOR_PLATFORM_LWR}" MATCHES "^(x86_64|i686|amd64|x64)$" ) +elseif (CMAKE_OSX_ARCHITECTURES STREQUAL "x86_64" OR CMAKE_GENERATOR_PLATFORM_LWR MATCHES "^(x86_64|i686|amd64|x64|win32)$" OR + (NOT CMAKE_OSX_ARCHITECTURES AND NOT CMAKE_GENERATOR_PLATFORM_LWR AND + CMAKE_SYSTEM_PROCESSOR MATCHES "^(x86_64|i686|AMD64)$")) message(STATUS "x86 detected") if (MSVC) # instruction set detection for MSVC only From 8504d2d0da8cc7a1f2eee0e9e56949f960510b75 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Thu, 8 Feb 2024 09:46:47 +0200 Subject: [PATCH 293/812] tests : .gitignore obj files --- tests/.gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/.gitignore b/tests/.gitignore index 092dce742..9427cf13d 100644 --- a/tests/.gitignore +++ b/tests/.gitignore @@ -1,3 +1,3 @@ * !*.* -test-c.o +*.o From 26d4efd11e48908e14e2ee9471a7fc4c57079a1d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20G=C3=A4=C3=9Fler?= Date: Thu, 8 Feb 2024 09:46:30 +0100 Subject: [PATCH 294/812] sampling: fix top_k <= 0 (#5388) * sampling: fix top_k <= 0 * Update llama.cpp Co-authored-by: Georgi Gerganov --------- Co-authored-by: Georgi Gerganov --- common/sampling.cpp | 2 +- llama.cpp | 4 ++++ tests/test-sampling.cpp | 2 ++ 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/common/sampling.cpp b/common/sampling.cpp index e8675a8c0..844ad7c53 100644 --- a/common/sampling.cpp +++ b/common/sampling.cpp @@ -132,7 +132,7 @@ static void sampler_queue( const float temp = params.temp; const float dynatemp_range = params.dynatemp_range; const float dynatemp_exponent = params.dynatemp_exponent; - const int32_t top_k = params.top_k <= 0 ? n_vocab : params.top_k; + const int32_t top_k = params.top_k; const float top_p = params.top_p; const float min_p = params.min_p; const float tfs_z = params.tfs_z; diff --git a/llama.cpp b/llama.cpp index c45ae1d50..f8f5796a4 100644 --- a/llama.cpp +++ b/llama.cpp @@ -8585,6 +8585,10 @@ void llama_sample_top_k(struct llama_context * ctx, llama_token_data_array * can // } const int64_t t_start_sample_us = ggml_time_us(); + + if (k <= 0) { + k = candidates->size; + } k = std::max(k, (int) min_keep); k = std::min(k, (int) candidates->size); diff --git a/tests/test-sampling.cpp b/tests/test-sampling.cpp index c3b3d6629..6374958fe 100644 --- a/tests/test-sampling.cpp +++ b/tests/test-sampling.cpp @@ -235,6 +235,8 @@ int main(void) { test_top_k({0.1f, 0.2f, 0.3f, 0.4f}, {0.4f}, 1); test_top_k({0.1f, 0.2f, 0.3f, 0.4f}, {0.4f, 0.3f, 0.2f}, 3); + test_top_k({0.1f, 0.2f, 0.3f, 0.4f}, {0.4f, 0.3f, 0.2f, 0.1f}, 4); + test_top_k({0.1f, 0.2f, 0.3f, 0.4f}, {0.4f, 0.3f, 0.2f, 0.1f}, 0); test_top_p({0.1f, 0.2f, 0.3f, 0.4f}, {0.4f}, 0); test_top_p({0.1f, 0.2f, 0.3f, 0.4f}, {0.4f, 0.3f}, 0.7f); From a6e514a85f0fda38ff78ec91782877ea3d19ed98 Mon Sep 17 00:00:00 2001 From: Daniel Bevenius Date: Thu, 8 Feb 2024 09:58:19 +0100 Subject: [PATCH 295/812] llava: fix typo/formatting in README.md (#5405) This commit fixes a typo in the README.md file for the llava example which is causing the formatting to look a little off: Clone llava-v15-7b`` and clip-vit-large-patch14-336`` locally Signed-off-by: Daniel Bevenius --- examples/llava/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/llava/README.md b/examples/llava/README.md index 323c5fdd0..295181a34 100644 --- a/examples/llava/README.md +++ b/examples/llava/README.md @@ -21,7 +21,7 @@ After building, run: `./llava-cli` to see the usage. For example: ## Model conversion -- Clone `llava-v15-7b`` and `clip-vit-large-patch14-336`` locally: +- Clone `llava-v15-7b` and `clip-vit-large-patch14-336` locally: ```sh git clone https://huggingface.co/liuhaotian/llava-v1.5-7b From 4aa43fab569215a13495a7f1a0f8afc541b16d03 Mon Sep 17 00:00:00 2001 From: runfuture Date: Thu, 8 Feb 2024 18:36:19 +0800 Subject: [PATCH 296/812] llama : fix MiniCPM (#5392) * fix bug for norm_rms_eps missing * to align with the same order as convert.py for model write * fix: undo HF models permute tensor * update for flake8 lint --- convert-hf-to-gguf.py | 63 +++++++++++++++++++++++++++++++++++++++++-- llama.cpp | 2 ++ 2 files changed, 63 insertions(+), 2 deletions(-) diff --git a/convert-hf-to-gguf.py b/convert-hf-to-gguf.py index 829d68368..0d4ea03b4 100755 --- a/convert-hf-to-gguf.py +++ b/convert-hf-to-gguf.py @@ -1078,17 +1078,76 @@ class MiniCPMModel(Model): self.gguf_writer.add_name("MiniCPM") self.gguf_writer.add_context_length(self.hparams["max_position_embeddings"]) self.gguf_writer.add_embedding_length(self.hparams["hidden_size"]) - self.gguf_writer.add_feed_forward_length(self.hparams["intermediate_size"]) self.gguf_writer.add_block_count(block_count) + self.gguf_writer.add_feed_forward_length(self.hparams["intermediate_size"]) + self.gguf_writer.add_rope_dimension_count(self.hparams["hidden_size"] // self.hparams["num_attention_heads"]) self.gguf_writer.add_head_count(self.hparams["num_attention_heads"]) self.gguf_writer.add_head_count_kv(self.hparams["num_key_value_heads"]) self.gguf_writer.add_layer_norm_rms_eps(self.hparams["rms_norm_eps"]) self.gguf_writer.add_file_type(self.ftype) - self.gguf_writer.add_rope_dimension_count(self.hparams["hidden_size"] // self.hparams["num_attention_heads"]) def set_vocab(self): self._set_vocab_hf() + def _reverse_hf_permute(self, weights: Tensor, n_head: int, n_kv_head: int | None = None) -> Tensor: + if n_kv_head is not None and n_head != n_kv_head: + n_head //= n_kv_head + + return ( + weights.reshape(n_head, 2, weights.shape[0] // n_head // 2, *weights.shape[1:]) + .swapaxes(1, 2) + .reshape(weights.shape) + ) + + def write_tensors(self): + block_count = self.hparams.get("n_layers", self.hparams.get("num_hidden_layers", self.hparams.get("n_layer"))) + tensor_map = gguf.get_tensor_name_map(self.model_arch, block_count) + n_head = self.hparams.get("num_attention_heads") + n_kv_head = self.hparams.get("num_key_value_heads") + for name, data_torch in self.get_tensors(): + # we don't need these + if name.endswith((".attention.masked_bias", ".attention.bias", ".attention.rotary_emb.inv_freq")): + continue + + old_dtype = data_torch.dtype + + # convert any unsupported data types to float32 + if data_torch.dtype not in (torch.float16, torch.float32): + data_torch = data_torch.to(torch.float32) + + # HF models permute some of the tensors, so we need to undo that + if name.endswith(("q_proj.weight")): + data_torch = self._reverse_hf_permute(data_torch, n_head, n_head) + if name.endswith(("k_proj.weight")): + data_torch = self._reverse_hf_permute(data_torch, n_head, n_kv_head) + + data = data_torch.squeeze().numpy() + + # map tensor names + new_name = tensor_map.get_name(name, try_suffixes=(".weight", ".bias")) + if new_name is None: + print(f"Can not map tensor {name!r}") + sys.exit() + + n_dims = len(data.shape) + data_dtype = data.dtype + + # if f32 desired, convert any float16 to float32 + if self.ftype == 0 and data_dtype == np.float16: + data = data.astype(np.float32) + + # TODO: Why cant we use these float16 as-is? There should be not reason to store float16 as float32 + if self.ftype == 1 and data_dtype == np.float16 and n_dims == 1: + data = data.astype(np.float32) + + # if f16 desired, convert any float32 2-dim weight tensors to float16 + if self.ftype == 1 and data_dtype == np.float32 and name.endswith(".weight") and n_dims == 2: + data = data.astype(np.float16) + + print(f"{new_name}, n_dims = {n_dims}, {old_dtype} --> {data.dtype}") + + self.gguf_writer.add_tensor(new_name, data) + class QwenModel(Model): @staticmethod diff --git a/llama.cpp b/llama.cpp index f8f5796a4..552e0d02e 100644 --- a/llama.cpp +++ b/llama.cpp @@ -2947,6 +2947,8 @@ static void llm_load_hparams( } break; case LLM_ARCH_MINICPM: { + ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps); + switch (hparams.n_layer) { case 40: model.type = e_model::MODEL_2B; break; default: model.type = e_model::MODEL_UNKNOWN; From b7b74cef36a93ae01e0b9af8986d131761742d0e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20G=C3=A4=C3=9Fler?= Date: Thu, 8 Feb 2024 11:36:54 +0100 Subject: [PATCH 297/812] fix trailing whitespace (#5407) --- llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llama.cpp b/llama.cpp index 552e0d02e..89acafbc3 100644 --- a/llama.cpp +++ b/llama.cpp @@ -8587,7 +8587,7 @@ void llama_sample_top_k(struct llama_context * ctx, llama_token_data_array * can // } const int64_t t_start_sample_us = ggml_time_us(); - + if (k <= 0) { k = candidates->size; } From ff4ff05c5ff4311c05a8ce1f984c7d8def4f07a5 Mon Sep 17 00:00:00 2001 From: Daniel Bevenius Date: Thu, 8 Feb 2024 15:20:03 +0100 Subject: [PATCH 298/812] llava : add missing .py, and fix paths in README.md (#5414) This commit adds the missing .py extension to the convert-image-encoder-to-gguf script. It also fixes the paths for the `model` and `mmproj` options in the example llava-cli command. Signed-off-by: Daniel Bevenius --- examples/llava/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/llava/README.md b/examples/llava/README.md index 295181a34..721d5e613 100644 --- a/examples/llava/README.md +++ b/examples/llava/README.md @@ -14,7 +14,7 @@ Build with cmake or run `make llava-cli` to build it. After building, run: `./llava-cli` to see the usage. For example: ```sh -./llava-cli -m llava-v1.5-7b/ggml-model-q5_k.gguf --mmproj llava-v1.5-7b/mmproj-model-f16.gguf --image path/to/an/image.jpg +./llava-cli -m ../llava-v1.5-7b/ggml-model-f16.gguf --mmproj ../llava-v1.5-7b/mmproj-model-f16.gguf --image path/to/an/image.jpg ``` **note**: A lower temperature like 0.1 is recommended for better quality. add `--temp 0.1` to the command to do so. @@ -38,7 +38,7 @@ python ./examples/llava/llava-surgery.py -m ../llava-v1.5-7b 3. Use `convert-image-encoder-to-gguf.py` to convert the LLaVA image encoder to GGUF: ```sh -python ./examples/llava/convert-image-encoder-to-gguf -m ../clip-vit-large-patch14-336 --llava-projector ../llava-v1.5-7b/llava.projector --output-dir ../llava-v1.5-7b +python ./examples/llava/convert-image-encoder-to-gguf.py -m ../clip-vit-large-patch14-336 --llava-projector ../llava-v1.5-7b/llava.projector --output-dir ../llava-v1.5-7b ``` 4. Use `convert.py` to convert the LLaMA part of LLaVA to GGUF: From 6e99f2a04f1871d637dd77eb4d81de31a5510253 Mon Sep 17 00:00:00 2001 From: Abhilash Majumder <30946547+abhilash1910@users.noreply.github.com> Date: Thu, 8 Feb 2024 22:39:10 +0530 Subject: [PATCH 299/812] Fix f16_sycl cpy call from Arc (#5411) * fix f16_sycl cpy call * rm old logic * add fp16 build CI * use macro * format fix --- .github/workflows/build.yml | 41 +++++++++++++++++++++++++++++++++++++ ggml-sycl.cpp | 8 +++++--- 2 files changed, 46 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index f4c374ce5..ed292d6b8 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -184,6 +184,47 @@ jobs: cmake -DLLAMA_SYCL=ON -DCMAKE_C_COMPILER=icx -DCMAKE_CXX_COMPILER=icpx .. cmake --build . --config Release -j $(nproc) + ubuntu-22-cmake-sycl-fp16: + runs-on: ubuntu-22.04 + + continue-on-error: true + + steps: + - uses: actions/checkout@v2 + + - name: add oneAPI to apt + shell: bash + run: | + cd /tmp + wget https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB + sudo apt-key add GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB + rm GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB + sudo add-apt-repository "deb https://apt.repos.intel.com/oneapi all main" + + - name: install oneAPI dpcpp compiler + shell: bash + run: | + sudo apt update + sudo apt install intel-oneapi-compiler-dpcpp-cpp + + - name: install oneAPI MKL library + shell: bash + run: | + sudo apt install intel-oneapi-mkl-devel + + - name: Clone + id: checkout + uses: actions/checkout@v3 + + - name: Build + id: cmake_build + run: | + source /opt/intel/oneapi/setvars.sh + mkdir build + cd build + cmake -DLLAMA_SYCL=ON -DCMAKE_C_COMPILER=icx -DCMAKE_CXX_COMPILER=icpx -DLLAMA_SYCL_F16=ON .. + cmake --build . --config Release -j $(nproc) + # TODO: build with LLAMA_NO_METAL because test-backend-ops fail on "Apple Paravirtual device" and I don't know # how to debug it. # ref: https://github.com/ggerganov/llama.cpp/actions/runs/7131777249/job/19420981052#step:5:1124 diff --git a/ggml-sycl.cpp b/ggml-sycl.cpp index a03df4c65..dd562a898 100644 --- a/ggml-sycl.cpp +++ b/ggml-sycl.cpp @@ -12148,7 +12148,8 @@ inline void ggml_sycl_op_dequantize_mul_mat_vec( const int64_t src1_ncols, const int64_t src1_padded_row_size, const dpct::queue_ptr &stream) { - const int64_t ne00 = src0->ne[0]; + GGML_TENSOR_BINARY_OP_LOCALS + const int64_t row_diff = row_high - row_low; // on some GPUs it is faster to convert src1 to half and to use half precision intrinsics @@ -12167,8 +12168,9 @@ inline void ggml_sycl_op_dequantize_mul_mat_vec( } else { src1_dfloat = src1_dfloat_a.alloc(ne00); ggml_cpy_f32_f16_sycl((const char *)src1_ddf_i, (char *)src1_dfloat, - ne00, ne00, 1, sizeof(float), 0, 0, ne00, 1, - sizeof(sycl::half), 0, 0, stream); + ne00, ne00, ne01, ne02, nb00, nb01, nb02, + nb03, ne10, ne11, ne12, nb10, nb11, nb12, + nb13, stream); } } #else From 41f308f58edc2a04bcf9e245100b0a9b10e9a0fb Mon Sep 17 00:00:00 2001 From: slaren Date: Thu, 8 Feb 2024 21:33:03 +0100 Subject: [PATCH 300/812] llama : do not print "offloading layers" message in CPU-only builds (#5416) --- llama.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/llama.cpp b/llama.cpp index 89acafbc3..db7d1c1cd 100644 --- a/llama.cpp +++ b/llama.cpp @@ -4209,8 +4209,7 @@ static bool llm_load_tensors( ctx_bufs.emplace_back(ctx, buf); } - // print memory requirements - { + if (llama_supports_gpu_offload()) { const int n_gpu = std::min(n_gpu_layers, int(hparams.n_layer)); LLAMA_LOG_INFO("%s: offloading %d repeating layers to GPU\n", __func__, n_gpu); @@ -4222,10 +4221,11 @@ static bool llm_load_tensors( const int max_offloadable_layers = hparams.n_layer + 1; LLAMA_LOG_INFO("%s: offloaded %d/%d layers to GPU\n", __func__, std::min(n_gpu_layers, max_offloadable_layers), max_backend_supported_layers); + } - for (ggml_backend_buffer_t buf : model.bufs) { - LLAMA_LOG_INFO("%s: %10s buffer size = %8.2f MiB\n", __func__, ggml_backend_buffer_name(buf), ggml_backend_buffer_get_size(buf) / 1024.0 / 1024.0); - } + // print memory requirements + for (ggml_backend_buffer_t buf : model.bufs) { + LLAMA_LOG_INFO("%s: %10s buffer size = %8.2f MiB\n", __func__, ggml_backend_buffer_name(buf), ggml_backend_buffer_get_size(buf) / 1024.0 / 1024.0); } // populate tensors_by_name From 8e6a9d2de0096af7120606c74ee2f26684e87b41 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20G=C3=A4=C3=9Fler?= Date: Thu, 8 Feb 2024 21:56:40 +0100 Subject: [PATCH 301/812] CUDA: more warps for mmvq on NVIDIA (#5394) --- ggml-cuda.cu | 133 +++++++++++++++++++++++++++++++++------------------ 1 file changed, 86 insertions(+), 47 deletions(-) diff --git a/ggml-cuda.cu b/ggml-cuda.cu index db9da2459..5053757e6 100644 --- a/ggml-cuda.cu +++ b/ggml-cuda.cu @@ -5310,22 +5310,26 @@ template static __global__ void #endif // __CUDA_ARCH__ >= CC_VOLTA } -template +#define MMVQ_NWARPS_NVIDIA 4 +#define MMVQ_NWARPS_AMD_RDNA2 1 +#define MMVQ_NWARPS_AMD_OLD 4 + +template +#if !(defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)) +__launch_bounds__(nwarps*WARP_SIZE, 1) // tells the compiler to use as many registers as it wants +#endif // !(defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)) static __global__ void mul_mat_vec_q( const void * __restrict__ vx, const void * __restrict__ vy, float * __restrict__ dst, const int ncols_x, const int nrows_x, const int nrows_y, const int ncols_y_par, const int nrows_dst) { const int ncols_y = ncols_y_template != 0 ? ncols_y_template : ncols_y_par; - const int row = blockIdx.x*blockDim.y + threadIdx.y; - - if (row >= nrows_x) { - return; - } + const int tid = WARP_SIZE*threadIdx.y + threadIdx.x; + const int row = blockIdx.x; const int blocks_per_row_x = ncols_x / qk; const int blocks_per_col_y = nrows_y / QK8_1; - const int blocks_per_warp = vdr * WARP_SIZE / qi; + const int blocks_per_iter = vdr * nwarps*WARP_SIZE / qi; // partial sum for each thread float tmp[ncols_y_template != 0 ? ncols_y_template : 8] = {0.0f}; @@ -5333,12 +5337,12 @@ static __global__ void mul_mat_vec_q( const block_q_t * x = (const block_q_t *) vx; const block_q8_1 * y = (const block_q8_1 *) vy; - for (int i = threadIdx.x / (qi/vdr); i < blocks_per_row_x; i += blocks_per_warp) { + for (int i = tid / (qi/vdr); i < blocks_per_row_x; i += blocks_per_iter) { const int ibx = row*blocks_per_row_x + i; // x block index const int iby = i * (qk/QK8_1); // y block index that aligns with ibx - const int iqs = vdr * (threadIdx.x % (qi/vdr)); // x block quant index when casting the quants to int + const int iqs = vdr * (tid % (qi/vdr)); // x block quant index when casting the quants to int #pragma unroll for (int j = 0; j < ncols_y; ++j) { @@ -5346,9 +5350,25 @@ static __global__ void mul_mat_vec_q( } } + __shared__ float tmp_shared[nwarps-1 > 0 ? nwarps-1 : 1][ncols_y_template != 0 ? ncols_y_template : 8][WARP_SIZE]; + if (threadIdx.y > 0) { +#pragma unroll + for (int j = 0; j < ncols_y; ++j) { + tmp_shared[threadIdx.y-1][j][threadIdx.x] = tmp[j]; + } + } + __syncthreads(); + if (threadIdx.y > 0) { + return; + } + // sum up partial sums and write back result #pragma unroll for (int j = 0; j < ncols_y; ++j) { +#pragma unroll + for (int i = 0; i < nwarps-1; ++i) { + tmp[j] += tmp_shared[i][j][threadIdx.x]; + } tmp[j] = warp_reduce_sum(tmp[j]); if (threadIdx.x == 0) { @@ -6833,46 +6853,65 @@ static void mul_mat_vec_q_cuda( GGML_ASSERT(ncols_x % qk == 0); GGML_ASSERT(ncols_y <= 4); - const int block_num_y = (nrows_x + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y; - const dim3 block_nums(block_num_y, 1, 1); - const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1); - switch (ncols_y) { - case 1: - mul_mat_vec_q<1, qk, qi, block_q_t, vdr, vec_dot> - <<>>(vx, vy, dst, ncols_x, nrows_x, nrows_y, ncols_y, nrows_dst); - break; - case 2: - mul_mat_vec_q<2, qk, qi, block_q_t, vdr, vec_dot> - <<>>(vx, vy, dst, ncols_x, nrows_x, nrows_y, ncols_y, nrows_dst); - break; - case 3: - mul_mat_vec_q<3, qk, qi, block_q_t, vdr, vec_dot> - <<>>(vx, vy, dst, ncols_x, nrows_x, nrows_y, ncols_y, nrows_dst); - break; - case 4: - mul_mat_vec_q<4, qk, qi, block_q_t, vdr, vec_dot> - <<>>(vx, vy, dst, ncols_x, nrows_x, nrows_y, ncols_y, nrows_dst); - break; - // case 5: - // mul_mat_vec_q<5, qk, qi, block_q_t, vdr, vec_dot> - // <<>>(vx, vy, dst, ncols_x, nrows_x, nrows_y, ncols_y, nrows_dst); - // break; - // case 6: - // mul_mat_vec_q<6, qk, qi, block_q_t, vdr, vec_dot> - // <<>>(vx, vy, dst, ncols_x, nrows_x, nrows_y, ncols_y, nrows_dst); - // break; - // case 7: - // mul_mat_vec_q<7, qk, qi, block_q_t, vdr, vec_dot> - // <<>>(vx, vy, dst, ncols_x, nrows_x, nrows_y, ncols_y, nrows_dst); - // break; - // case 8: - // mul_mat_vec_q<8, qk, qi, block_q_t, vdr, vec_dot> - // <<>>(vx, vy, dst, ncols_x, nrows_x, nrows_y, ncols_y, nrows_dst); - // break; + int id; + CUDA_CHECK(cudaGetDevice(&id)); + + int nwarps; + if (g_device_caps[id].cc >= CC_OFFSET_AMD) { + nwarps = g_device_caps[id].cc >= CC_RDNA2 ? MMVQ_NWARPS_AMD_RDNA2 : MMVQ_NWARPS_AMD_OLD; + } else { + nwarps = MMVQ_NWARPS_NVIDIA; + } + + const dim3 block_nums(nrows_x, 1, 1); + const dim3 block_dims(WARP_SIZE, nwarps, 1); + + switch (nwarps) { + case 1: switch(ncols_y) { + case 1: + mul_mat_vec_q<1, 1, qk, qi, block_q_t, vdr, vec_dot> + <<>>(vx, vy, dst, ncols_x, nrows_x, nrows_y, ncols_y, nrows_dst); + break; + case 2: + mul_mat_vec_q<1, 2, qk, qi, block_q_t, vdr, vec_dot> + <<>>(vx, vy, dst, ncols_x, nrows_x, nrows_y, ncols_y, nrows_dst); + break; + case 3: + mul_mat_vec_q<1, 3, qk, qi, block_q_t, vdr, vec_dot> + <<>>(vx, vy, dst, ncols_x, nrows_x, nrows_y, ncols_y, nrows_dst); + break; + case 4: + mul_mat_vec_q<1, 4, qk, qi, block_q_t, vdr, vec_dot> + <<>>(vx, vy, dst, ncols_x, nrows_x, nrows_y, ncols_y, nrows_dst); + break; + default: + GGML_ASSERT(false); + break; + } break; + case 4: switch(ncols_y) { + case 1: + mul_mat_vec_q<4, 1, qk, qi, block_q_t, vdr, vec_dot> + <<>>(vx, vy, dst, ncols_x, nrows_x, nrows_y, ncols_y, nrows_dst); + break; + case 2: + mul_mat_vec_q<4, 2, qk, qi, block_q_t, vdr, vec_dot> + <<>>(vx, vy, dst, ncols_x, nrows_x, nrows_y, ncols_y, nrows_dst); + break; + case 3: + mul_mat_vec_q<4, 3, qk, qi, block_q_t, vdr, vec_dot> + <<>>(vx, vy, dst, ncols_x, nrows_x, nrows_y, ncols_y, nrows_dst); + break; + case 4: + mul_mat_vec_q<4, 4, qk, qi, block_q_t, vdr, vec_dot> + <<>>(vx, vy, dst, ncols_x, nrows_x, nrows_y, ncols_y, nrows_dst); + break; + default: + GGML_ASSERT(false); + break; + } break; + default: GGML_ASSERT(false); - // mul_mat_vec_q<0, qk, qi, block_q_t, vdr, vec_dot> - // <<>>(vx, vy, dst, ncols_x, nrows_x, nrows_y, ncols_y, nrows_dst); break; } } From 44fbe34360dd760f9e68b4271f21533436397f84 Mon Sep 17 00:00:00 2001 From: 0cc4m Date: Fri, 9 Feb 2024 06:52:33 +0100 Subject: [PATCH 302/812] Fix Vulkan crash on APUs with very little device memory (#5424) * Fix Vulkan crash on APUs with very little device memory * Fix debug output function names --- ggml-vulkan.cpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/ggml-vulkan.cpp b/ggml-vulkan.cpp index 9e2846ee4..254f648a6 100644 --- a/ggml-vulkan.cpp +++ b/ggml-vulkan.cpp @@ -744,6 +744,8 @@ static vk_buffer ggml_vk_create_buffer(ggml_backend_vk_context * ctx, size_t siz } if (memory_type_index >= mem_props.memoryTypeCount) { + ctx->device.lock()->device.destroyBuffer(buf->buffer); + buf->size = 0; throw vk::OutOfDeviceMemoryError("No suitable memory type found"); } @@ -3875,7 +3877,7 @@ static ggml_tensor * ggml_vk_find_last_use(const ggml_tensor * node, ggml_cgraph static void ggml_vk_preallocate_buffers_graph(ggml_backend_vk_context * ctx, ggml_tensor * node){ #ifdef GGML_VULKAN_DEBUG - std::cerr << "ggml_ctx->preallocate_buffers_graph(" << node << ")" << std::endl; + std::cerr << "ggml_vk_preallocate_buffers_graph(" << node << ")" << std::endl; #endif const bool any_on_device = node->backend == GGML_BACKEND_GPU || (node->src[0] != nullptr && (node->src[0]->backend == GGML_BACKEND_GPU || node->src[0]->backend == GGML_BACKEND_GPU_SPLIT)) @@ -3994,8 +3996,7 @@ static void ggml_vk_preallocate_buffers(ggml_backend_vk_context * ctx) { return; } #ifdef GGML_VULKAN_DEBUG - std::cerr << "ggml_ctx->preallocate_buffers()" << std::endl; - std::cerr << "qx_size: " << ctx->prealloc_size_qx << " qy_size: " << ctx->prealloc_size_qy << " x_size: " << ctx->prealloc_size_x << " y_size: " << ctx->prealloc_size_y << " split_k_size: " << ctx->prealloc_size_split_k << std::endl; + std::cerr << "ggml_vk_preallocate_buffers(qx_size: " << ctx->prealloc_size_qx << " qy_size: " << ctx->prealloc_size_qy << " x_size: " << ctx->prealloc_size_x << " y_size: " << ctx->prealloc_size_y << " split_k_size: " << ctx->prealloc_size_split_k << ")" << std::endl; #endif #if defined(GGML_VULKAN_RUN_TESTS) ctx->staging = ggml_vk_create_buffer_check(ctx, 100ul * 1024ul * 1024ul, vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent | vk::MemoryPropertyFlagBits::eHostCached); From b2f87cb64db47d799b6f3656855c9caf9792ab2a Mon Sep 17 00:00:00 2001 From: Michael Podvitskiy Date: Fri, 9 Feb 2024 10:56:43 +0100 Subject: [PATCH 303/812] ggml : fix `error C2078: too many initializers` for MSVC ARM64 (#5404) --- ggml-quants.c | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/ggml-quants.c b/ggml-quants.c index 101d3e783..1031e3761 100644 --- a/ggml-quants.c +++ b/ggml-quants.c @@ -268,6 +268,17 @@ static inline float hsum_float_4x4(const __m128 a, const __m128 b, const __m128 #endif // defined(__AVX__) || defined(__AVX2__) || defined(__AVX512F__) || defined(__SSSE3__) #if defined(__ARM_NEON) + +#ifdef _MSC_VER + +#define ggml_vld1q_u32(w,x,y,z) { ((w) + ((uint64_t)(x) << 32)), ((y) + ((uint64_t)(z) << 32)) } + +#else + +#define ggml_vld1q_u32(w,x,y,z) { (w), (x), (y), (z) } + +#endif + #if !defined(__aarch64__) // 64-bit compatibility @@ -8698,10 +8709,10 @@ void ggml_vec_dot_iq3_xxs_q8_K(const int n, float * restrict s, const void * res for (int ib32 = 0; ib32 < QK_K/32; ib32 += 2) { q8b = ggml_vld1q_s8_x4(q8); q8 += 64; memcpy(aux32, gas, 2*sizeof(uint32_t)); gas += 2*sizeof(uint32_t); - const uint32x4_t aux32x4_0 = {iq3xxs_grid[q3[ 0]], iq3xxs_grid[q3[ 1]], iq3xxs_grid[q3[ 2]], iq3xxs_grid[q3[ 3]]}; - const uint32x4_t aux32x4_1 = {iq3xxs_grid[q3[ 4]], iq3xxs_grid[q3[ 5]], iq3xxs_grid[q3[ 6]], iq3xxs_grid[q3[ 7]]}; - const uint32x4_t aux32x4_2 = {iq3xxs_grid[q3[ 8]], iq3xxs_grid[q3[ 9]], iq3xxs_grid[q3[10]], iq3xxs_grid[q3[11]]}; - const uint32x4_t aux32x4_3 = {iq3xxs_grid[q3[12]], iq3xxs_grid[q3[13]], iq3xxs_grid[q3[14]], iq3xxs_grid[q3[15]]}; + const uint32x4_t aux32x4_0 = ggml_vld1q_u32(iq3xxs_grid[q3[ 0]], iq3xxs_grid[q3[ 1]], iq3xxs_grid[q3[ 2]], iq3xxs_grid[q3[ 3]]); + const uint32x4_t aux32x4_1 = ggml_vld1q_u32(iq3xxs_grid[q3[ 4]], iq3xxs_grid[q3[ 5]], iq3xxs_grid[q3[ 6]], iq3xxs_grid[q3[ 7]]); + const uint32x4_t aux32x4_2 = ggml_vld1q_u32(iq3xxs_grid[q3[ 8]], iq3xxs_grid[q3[ 9]], iq3xxs_grid[q3[10]], iq3xxs_grid[q3[11]]); + const uint32x4_t aux32x4_3 = ggml_vld1q_u32(iq3xxs_grid[q3[12]], iq3xxs_grid[q3[13]], iq3xxs_grid[q3[14]], iq3xxs_grid[q3[15]]); q3 += 16; q3s.val[0] = vcombine_s8(vld1_s8((const void *)(signs64 + ((aux32[0] >> 0) & 127))), vld1_s8((const void *)(signs64 + ((aux32[0] >> 7) & 127)))); q3s.val[1] = vcombine_s8(vld1_s8((const void *)(signs64 + ((aux32[0] >> 14) & 127))), vld1_s8((const void *)(signs64 + ((aux32[0] >> 21) & 127)))); From e4124c24775f2cb5b3d7acc93bf9dc5471c172ef Mon Sep 17 00:00:00 2001 From: Marko Tasic Date: Fri, 9 Feb 2024 11:17:00 +0100 Subject: [PATCH 304/812] readme : add JavaScript/Wasm repo (#5415) --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 66166c01b..0b4efdd33 100644 --- a/README.md +++ b/README.md @@ -124,6 +124,7 @@ Typically finetunes of the base models below are supported as well. - Go: [go-skynet/go-llama.cpp](https://github.com/go-skynet/go-llama.cpp) - Node.js: [withcatai/node-llama-cpp](https://github.com/withcatai/node-llama-cpp) - JS/TS (llama.cpp server client): [lgrammel/modelfusion](https://modelfusion.dev/integration/model-provider/llamacpp) +- JavaScript/Wasm (works in browser): [tangledgroup/llama-cpp-wasm](https://github.com/tangledgroup/llama-cpp-wasm) - Ruby: [yoshoku/llama_cpp.rb](https://github.com/yoshoku/llama_cpp.rb) - Rust (nicer API): [mdrokz/rust-llama.cpp](https://github.com/mdrokz/rust-llama.cpp) - Rust (more direct bindings): [utilityai/llama-cpp-rs](https://github.com/utilityai/llama-cpp-rs) From e5ca3937c685d6e012ac4db40555d6ec100ff03c Mon Sep 17 00:00:00 2001 From: Paul Tsochantaris Date: Fri, 9 Feb 2024 10:48:06 +0000 Subject: [PATCH 305/812] llama : do not cap thread count when MoE on CPU (#5419) * Not capping thread count when MoE inference is running on CPU * Whitespace --- llama.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/llama.cpp b/llama.cpp index db7d1c1cd..0566b087b 100644 --- a/llama.cpp +++ b/llama.cpp @@ -7285,7 +7285,9 @@ static int llama_decode_internal( // TODO: this is mostly important for Apple Silicon where CBLAS is still performing very well // we still need some threads to process all non-mul_mat ops, but not too much to avoid interfering // with the BLAS calls. need a better solution - if (n_tokens >= 32 && ggml_cpu_has_blas() && !ggml_cpu_has_gpublas()) { + // MoE Special Case: This logic applies when hparams.n_expert == 0, i.e. the model is NOT an MoE model. When an MoE is + // being processed then Accelerate/BLAS will not be involved, so capping would limit performance. + if (n_tokens >= 32 && hparams.n_expert == 0 && ggml_cpu_has_blas() && !ggml_cpu_has_gpublas()) { n_threads = std::min(4, n_threads); } From 7c777fcd5dd4af7079e33390cf6a19c328a2666f Mon Sep 17 00:00:00 2001 From: Riley Stewart Date: Fri, 9 Feb 2024 02:49:49 -0800 Subject: [PATCH 306/812] server : fix prompt caching for repeated prompts (#5420) --- examples/server/server.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/examples/server/server.cpp b/examples/server/server.cpp index eceda30d0..8d668f798 100644 --- a/examples/server/server.cpp +++ b/examples/server/server.cpp @@ -1592,10 +1592,6 @@ struct llama_server_context LOG_TEE("slot %d : in cache: %i tokens | to process: %i tokens\n", slot.id, slot.n_past, slot.num_prompt_tokens_processed); } - LOG_TEE("slot %d : kv cache rm - [%d, end)\n", slot.id, (int) system_tokens.size() + slot.n_past); - - llama_kv_cache_seq_rm(ctx, slot.id, system_tokens.size() + slot.n_past, -1); - slot.cache_tokens = prompt_tokens; if (slot.n_past == slot.num_prompt_tokens && slot.n_past > 0) @@ -1609,6 +1605,10 @@ struct llama_server_context } } + LOG_TEE("slot %d : kv cache rm - [%d, end)\n", slot.id, (int) system_tokens.size() + slot.n_past); + + llama_kv_cache_seq_rm(ctx, slot.id, system_tokens.size() + slot.n_past, -1); + LOG_VERBOSE("prompt ingested", { {"n_past", slot.n_past}, {"cached", tokens_to_str(ctx, slot.cache_tokens.cbegin(), slot.cache_tokens.cbegin() + slot.n_past)}, From e00d2a62dd1441e3b089570ec06d05c18800d368 Mon Sep 17 00:00:00 2001 From: Daniel Bevenius Date: Fri, 9 Feb 2024 14:00:59 +0100 Subject: [PATCH 307/812] llava : add requirements.txt and update README.md (#5428) * llava: add requirements.txt and update README.md This commit adds a `requirements.txt` file to the `examples/llava` directory. This file contains the required Python packages to run the scripts in the `examples/llava` directory. The motivation of this to make it easier for users to run the scripts in `examples/llava`. This will avoid users from having to possibly run into missing package issues if the packages are not installed on their system. Signed-off-by: Daniel Bevenius * llava: fix typo in llava-surgery.py output Signed-off-by: Daniel Bevenius --------- Signed-off-by: Daniel Bevenius --- examples/llava/README.md | 12 +++++++++--- examples/llava/llava-surgery.py | 2 +- examples/llava/requirements.txt | 3 +++ 3 files changed, 13 insertions(+), 4 deletions(-) create mode 100644 examples/llava/requirements.txt diff --git a/examples/llava/README.md b/examples/llava/README.md index 721d5e613..19f1a50a2 100644 --- a/examples/llava/README.md +++ b/examples/llava/README.md @@ -29,19 +29,25 @@ git clone https://huggingface.co/liuhaotian/llava-v1.5-7b git clone https://huggingface.co/openai/clip-vit-large-patch14-336 ``` -2. Use `llava-surgery.py` to split the LLaVA model to LLaMA and multimodel projector constituents: +2. Install the required Python packages: + +```sh +pip install -r examples/llava/requirements.txt +``` + +3. Use `llava-surgery.py` to split the LLaVA model to LLaMA and multimodel projector constituents: ```sh python ./examples/llava/llava-surgery.py -m ../llava-v1.5-7b ``` -3. Use `convert-image-encoder-to-gguf.py` to convert the LLaVA image encoder to GGUF: +4. Use `convert-image-encoder-to-gguf.py` to convert the LLaVA image encoder to GGUF: ```sh python ./examples/llava/convert-image-encoder-to-gguf.py -m ../clip-vit-large-patch14-336 --llava-projector ../llava-v1.5-7b/llava.projector --output-dir ../llava-v1.5-7b ``` -4. Use `convert.py` to convert the LLaMA part of LLaVA to GGUF: +5. Use `convert.py` to convert the LLaMA part of LLaVA to GGUF: ```sh python ./convert.py ../llava-v1.5-7b diff --git a/examples/llava/llava-surgery.py b/examples/llava/llava-surgery.py index 515f6b58d..0a61efdfe 100644 --- a/examples/llava/llava-surgery.py +++ b/examples/llava/llava-surgery.py @@ -42,5 +42,5 @@ if len(clip_tensors) > 0: torch.save(checkpoint, path) print("Done!") -print(f"Now you can convert {args.model} to a a regular LLaMA GGUF file.") +print(f"Now you can convert {args.model} to a regular LLaMA GGUF file.") print(f"Also, use {args.model}/llava.projector to prepare a llava-encoder.gguf file.") diff --git a/examples/llava/requirements.txt b/examples/llava/requirements.txt new file mode 100644 index 000000000..f80f727a7 --- /dev/null +++ b/examples/llava/requirements.txt @@ -0,0 +1,3 @@ +-r ../../requirements/requirements-convert.txt +pillow~=10.2.0 +torch~=2.1.1 From 4b7b38bef5addbd31f453871d79647fbae6bec8a Mon Sep 17 00:00:00 2001 From: Neuman Vong Date: Sat, 10 Feb 2024 05:30:19 +1100 Subject: [PATCH 308/812] vulkan: Set limit for task concurrency (#5427) A common default for the maximum number of open files is 256, which can lead to `asyncio.gather(*tasks)` failing with Too many open files. $ python ggml_vk_generate_shaders.py --glslc=$ANDROID_NDK_PATH/shader-tools/darwin-x86_64/glslc ggml_vulkan: Generating and compiling shaders to SPIR-V Traceback (most recent call last): File "/Users/neuman/Code.noindex/github/llama.cpp/ggml_vk_generate_shaders.py", line 2326, in asyncio.run(main()) File "/Users/neuman/Code.noindex/miniforge3/lib/python3.10/asyncio/runners.py", line 44, in run return loop.run_until_complete(main) File "/Users/neuman/Code.noindex/miniforge3/lib/python3.10/asyncio/base_events.py", line 649, in run_until_complete return future.result() File "/Users/neuman/Code.noindex/github/llama.cpp/ggml_vk_generate_shaders.py", line 2294, in main await asyncio.gather(*tasks) [...snip...] OSError: [Errno 24] Too many open files This change sets a reasonable concurrency limit for tasks (and therefore open files), without significant impact on run time. --- ggml_vk_generate_shaders.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/ggml_vk_generate_shaders.py b/ggml_vk_generate_shaders.py index 4abb0383f..b2e86e182 100644 --- a/ggml_vk_generate_shaders.py +++ b/ggml_vk_generate_shaders.py @@ -2067,6 +2067,8 @@ type_names = { K_QUANTS_PER_ITERATION = 2 +ASYNCIO_CONCURRENCY = 64 + output_dir = gettempdir() lock = asyncio.Lock() @@ -2291,7 +2293,14 @@ async def main(): tasks.append(string_to_spv("rope_neox_f32", rope_neox_src, {"A_TYPE": "float", "D_TYPE": "float"})) tasks.append(string_to_spv("rope_neox_f16", rope_neox_src, {"A_TYPE": "float16_t", "D_TYPE": "float16_t"})) - await asyncio.gather(*tasks) + # Helper to decorate tasks with semaphore acquisition. + async def withSemaphore(sem, task): + async with sem: + return await task + + # Run tasks concurrently guarded by a concurrency limit. + sem = asyncio.Semaphore(ASYNCIO_CONCURRENCY) + await asyncio.gather(*(withSemaphore(sem, task) for task in tasks)) with open("ggml-vulkan-shaders.hpp", "w") as f: f.write("#include \n\n") From 4633d93af08d890ecd00fa6e4f61d76f21cded4c Mon Sep 17 00:00:00 2001 From: Michael Podvitskiy Date: Fri, 9 Feb 2024 10:42:27 +0100 Subject: [PATCH 309/812] ggml : add abort_callback for cpu backend (ggml/725) * a way to use abort_callback with the cpu backend * whisper update --- ggml-backend.c | 26 ++++++++++++++++++++++---- ggml-backend.h | 5 +++-- ggml.c | 2 +- ggml.h | 9 +++++++-- 4 files changed, 33 insertions(+), 9 deletions(-) diff --git a/ggml-backend.c b/ggml-backend.c index 0764dfebc..532da8eda 100644 --- a/ggml-backend.c +++ b/ggml-backend.c @@ -653,6 +653,9 @@ struct ggml_backend_cpu_context { int n_threads; void * work_data; size_t work_size; + + ggml_abort_callback abort_callback; + void * abort_callback_data; }; GGML_CALL static const char * ggml_backend_cpu_name(ggml_backend_t backend) { @@ -691,6 +694,9 @@ GGML_CALL static ggml_backend_graph_plan_t ggml_backend_cpu_graph_plan_create(gg cpu_plan->cplan.work_data = malloc(cpu_plan->cplan.work_size); } + cpu_plan->cplan.abort_callback = cpu_ctx->abort_callback; + cpu_plan->cplan.abort_callback_data = cpu_ctx->abort_callback_data; + return cpu_plan; } @@ -721,9 +727,11 @@ GGML_CALL static bool ggml_backend_cpu_graph_compute(ggml_backend_t backend, str cpu_ctx->work_data = realloc(cpu_ctx->work_data, cplan.work_size); cpu_ctx->work_size = cplan.work_size; } - cplan.work_data = cpu_ctx->work_data; + cplan.abort_callback = cpu_ctx->abort_callback; + cplan.abort_callback_data = cpu_ctx->abort_callback_data; + ggml_graph_compute(cgraph, &cplan); return true; } @@ -759,9 +767,11 @@ static struct ggml_backend_i cpu_backend_i = { ggml_backend_t ggml_backend_cpu_init(void) { struct ggml_backend_cpu_context * ctx = malloc(sizeof(struct ggml_backend_cpu_context)); - ctx->n_threads = GGML_DEFAULT_N_THREADS; - ctx->work_data = NULL; - ctx->work_size = 0; + ctx->n_threads = GGML_DEFAULT_N_THREADS; + ctx->work_data = NULL; + ctx->work_size = 0; + ctx->abort_callback = NULL; + ctx->abort_callback_data = NULL; ggml_backend_t cpu_backend = malloc(sizeof(struct ggml_backend)); @@ -783,6 +793,14 @@ void ggml_backend_cpu_set_n_threads(ggml_backend_t backend_cpu, int n_threads) { ctx->n_threads = n_threads; } +void ggml_backend_cpu_set_abort_callback(ggml_backend_t backend_cpu, ggml_abort_callback abort_callback, void * abort_callback_data) { + GGML_ASSERT(ggml_backend_is_cpu(backend_cpu)); + + struct ggml_backend_cpu_context * ctx = (struct ggml_backend_cpu_context *)backend_cpu->context; + ctx->abort_callback = abort_callback; + ctx->abort_callback_data = abort_callback_data; +} + GGML_CALL ggml_backend_buffer_t ggml_backend_cpu_buffer_from_ptr(void * ptr, size_t size) { return ggml_backend_buffer_init(ggml_backend_cpu_buffer_type(), cpu_backend_buffer_i_from_ptr, ptr, size); } diff --git a/ggml-backend.h b/ggml-backend.h index 8b8160fcf..282b3a9b7 100644 --- a/ggml-backend.h +++ b/ggml-backend.h @@ -83,8 +83,9 @@ extern "C" { GGML_API ggml_backend_t ggml_backend_cpu_init(void); - GGML_API GGML_CALL bool ggml_backend_is_cpu (ggml_backend_t backend); - GGML_API void ggml_backend_cpu_set_n_threads(ggml_backend_t backend_cpu, int n_threads); + GGML_API GGML_CALL bool ggml_backend_is_cpu (ggml_backend_t backend); + GGML_API void ggml_backend_cpu_set_n_threads (ggml_backend_t backend_cpu, int n_threads); + GGML_API void ggml_backend_cpu_set_abort_callback(ggml_backend_t backend_cpu, ggml_abort_callback abort_callback, void * abort_callback_data); // Create a backend buffer from an existing pointer GGML_API GGML_CALL ggml_backend_buffer_t ggml_backend_cpu_buffer_from_ptr(void * ptr, size_t size); diff --git a/ggml.c b/ggml.c index f783a6fd3..86cd65862 100644 --- a/ggml.c +++ b/ggml.c @@ -16649,7 +16649,7 @@ struct ggml_compute_state_shared { atomic_int node_n; // active graph node atomic_int node_task; // active graph node task phase - bool (*abort_callback)(void * data); // abort ggml_graph_compute when true + ggml_abort_callback abort_callback; // abort ggml_graph_compute when true void * abort_callback_data; }; diff --git a/ggml.h b/ggml.h index e0a4799f3..1360cd8ee 100644 --- a/ggml.h +++ b/ggml.h @@ -567,6 +567,11 @@ extern "C" { static const size_t GGML_TENSOR_SIZE = sizeof(struct ggml_tensor); + // Abort callback + // If not NULL, called before ggml computation + // If it returns true, the computation is aborted + typedef bool (*ggml_abort_callback)(void * data); + // the compute plan that needs to be prepared for ggml_graph_compute() // since https://github.com/ggerganov/ggml/issues/287 struct ggml_cplan { @@ -576,8 +581,8 @@ extern "C" { int n_threads; // abort ggml_graph_compute when true - bool (*abort_callback)(void * data); - void * abort_callback_data; + ggml_abort_callback abort_callback; + void * abort_callback_data; }; enum ggml_cgraph_eval_order { From 43b65f5eb85e8741aba573a8f65bb8efad245d31 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Sat, 10 Feb 2024 09:30:36 +0200 Subject: [PATCH 310/812] sync : ggml --- scripts/sync-ggml.last | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/sync-ggml.last b/scripts/sync-ggml.last index 7b6c17915..6ae75bc31 100644 --- a/scripts/sync-ggml.last +++ b/scripts/sync-ggml.last @@ -1 +1 @@ -475cbad5c1c834e31e26a2283bc1413181644360 +2c7cf49810d523b9632da393a9e8270b60bf3b24 From cd9aea63b577a83def84dbd6dcd90a6fa02af745 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Sat, 10 Feb 2024 09:53:05 +0200 Subject: [PATCH 311/812] scripts : update sync scripts with new backends --- scripts/sync-ggml-am.sh | 12 ++++++++++++ scripts/sync-ggml.sh | 6 ++++++ 2 files changed, 18 insertions(+) diff --git a/scripts/sync-ggml-am.sh b/scripts/sync-ggml-am.sh index 6b2514a11..2c391e641 100755 --- a/scripts/sync-ggml-am.sh +++ b/scripts/sync-ggml-am.sh @@ -97,6 +97,8 @@ if [ -f $SRC_LLAMA/ggml-src.patch ]; then # src/ggml-cuda.cu -> ggml-cuda.cu # src/ggml-cuda.h -> ggml-cuda.h # src/ggml-impl.h -> ggml-impl.h + # src/ggml-kompute.cpp -> ggml-kompute.cpp + # src/ggml-kompute.h -> ggml-kompute.h # src/ggml-metal.h -> ggml-metal.h # src/ggml-metal.m -> ggml-metal.m # src/ggml-mpi.h -> ggml-mpi.h @@ -105,6 +107,10 @@ if [ -f $SRC_LLAMA/ggml-src.patch ]; then # src/ggml-opencl.h -> ggml-opencl.h # src/ggml-quants.c -> ggml-quants.c # src/ggml-quants.h -> ggml-quants.h + # src/ggml-sycl.cpp -> ggml-sycl.cpp + # src/ggml-sycl.h -> ggml-sycl.h + # src/ggml-vulkan.cpp -> ggml-vulkan.cpp + # src/ggml-vulkan.h -> ggml-vulkan.h # include/ggml/ggml.h -> ggml.h # include/ggml/ggml-alloc.h -> ggml-alloc.h # include/ggml/ggml-backend.h -> ggml-backend.h @@ -123,6 +129,8 @@ if [ -f $SRC_LLAMA/ggml-src.patch ]; then -e 's/src\/ggml-cuda\.cu/ggml-cuda.cu/g' \ -e 's/src\/ggml-cuda\.h/ggml-cuda.h/g' \ -e 's/src\/ggml-impl\.h/ggml-impl.h/g' \ + -e 's/src\/ggml-kompute\.cpp/ggml-kompute.cpp/g' \ + -e 's/src\/ggml-kompute\.h/ggml-kompute.h/g' \ -e 's/src\/ggml-metal\.h/ggml-metal.h/g' \ -e 's/src\/ggml-metal\.m/ggml-metal.m/g' \ -e 's/src\/ggml-mpi\.h/ggml-mpi.h/g' \ @@ -131,6 +139,10 @@ if [ -f $SRC_LLAMA/ggml-src.patch ]; then -e 's/src\/ggml-opencl\.h/ggml-opencl.h/g' \ -e 's/src\/ggml-quants\.c/ggml-quants.c/g' \ -e 's/src\/ggml-quants\.h/ggml-quants.h/g' \ + -e 's/src\/ggml-sycl\.cpp/ggml-sycl.cpp/g' \ + -e 's/src\/ggml-sycl\.h/ggml-sycl.h/g' \ + -e 's/src\/ggml-vulkan\.cpp/ggml-vulkan.cpp/g' \ + -e 's/src\/ggml-vulkan\.h/ggml-vulkan.h/g' \ -e 's/include\/ggml\/ggml\.h/ggml.h/g' \ -e 's/include\/ggml\/ggml-alloc\.h/ggml-alloc.h/g' \ -e 's/include\/ggml\/ggml-backend\.h/ggml-backend.h/g' \ diff --git a/scripts/sync-ggml.sh b/scripts/sync-ggml.sh index 0097db435..feb34bbc8 100755 --- a/scripts/sync-ggml.sh +++ b/scripts/sync-ggml.sh @@ -7,6 +7,8 @@ cp -rpv ../ggml/src/ggml-backend.c ./ggml-backend.c cp -rpv ../ggml/src/ggml-cuda.cu ./ggml-cuda.cu cp -rpv ../ggml/src/ggml-cuda.h ./ggml-cuda.h cp -rpv ../ggml/src/ggml-impl.h ./ggml-impl.h +cp -rpv ../ggml/src/ggml-kompute.cpp ./ggml-kompute.cpp +cp -rpv ../ggml/src/ggml-kompute.h ./ggml-kompute.h cp -rpv ../ggml/src/ggml-metal.h ./ggml-metal.h cp -rpv ../ggml/src/ggml-metal.m ./ggml-metal.m cp -rpv ../ggml/src/ggml-metal.metal ./ggml-metal.metal @@ -16,6 +18,10 @@ cp -rpv ../ggml/src/ggml-opencl.cpp ./ggml-opencl.cpp cp -rpv ../ggml/src/ggml-opencl.h ./ggml-opencl.h cp -rpv ../ggml/src/ggml-quants.c ./ggml-quants.c cp -rpv ../ggml/src/ggml-quants.h ./ggml-quants.h +cp -rpv ../ggml/src/ggml-sycl.cpp ./ggml-sycl.cpp +cp -rpv ../ggml/src/ggml-sycl.h ./ggml-sycl.h +cp -rpv ../ggml/src/ggml-vulkan.cpp ./ggml-vulkan.cpp +cp -rpv ../ggml/src/ggml-vulkan.h ./ggml-vulkan.h cp -rpv ../ggml/include/ggml/ggml.h ./ggml.h cp -rpv ../ggml/include/ggml/ggml-alloc.h ./ggml-alloc.h cp -rpv ../ggml/include/ggml/ggml-backend.h ./ggml-backend.h From f026f8120f97090d34a52b3dc023c82e0ede3f7d Mon Sep 17 00:00:00 2001 From: Ian Bull Date: Sat, 10 Feb 2024 02:53:28 -0800 Subject: [PATCH 312/812] metal : use autoreleasepool to avoid memory leaks (#5437) There appears to be a known memory leak when using the `MLTCommandBuffer`. It is suggested to use `@autoreleasepool` in [1,2] [1] https://developer.apple.com/forums/thread/662721 [2] https://forums.developer.apple.com/forums/thread/120931 This change-set wraps the `ggml_metal_graph_compute` in a `@autoreleasepool`. This commit addresses https://github.com/ggerganov/llama.cpp/issues/5436 --- ggml-metal.m | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ggml-metal.m b/ggml-metal.m index 5260ed827..c1d8e2de8 100644 --- a/ggml-metal.m +++ b/ggml-metal.m @@ -687,6 +687,7 @@ static bool ggml_metal_graph_compute( struct ggml_metal_context * ctx, struct ggml_cgraph * gf) { + @autoreleasepool { MTLComputePassDescriptor * edesc = MTLComputePassDescriptor.computePassDescriptor; edesc.dispatchType = MTLDispatchTypeSerial; @@ -2272,6 +2273,7 @@ static bool ggml_metal_graph_compute( [[MTLCaptureManager sharedCaptureManager] stopCapture]; } + } return true; } From 907e08c1109f498b01036367804cff3082c44524 Mon Sep 17 00:00:00 2001 From: Xuan Son Nguyen Date: Sun, 11 Feb 2024 11:16:22 +0100 Subject: [PATCH 313/812] server : add llama2 chat template (#5425) * server: add mistral chat template * server: fix typo * server: rename template mistral to llama2 * server: format_llama2: remove BOS * server: validate "--chat-template" argument * server: clean up using_chatml variable Co-authored-by: Jared Van Bortel --------- Co-authored-by: Jared Van Bortel --- examples/server/oai.hpp | 8 ++++++-- examples/server/server.cpp | 22 ++++++++++++++++++++-- examples/server/utils.hpp | 30 ++++++++++++++++++++++++++++++ 3 files changed, 56 insertions(+), 4 deletions(-) diff --git a/examples/server/oai.hpp b/examples/server/oai.hpp index 43410f803..2eca8a9fb 100644 --- a/examples/server/oai.hpp +++ b/examples/server/oai.hpp @@ -15,9 +15,13 @@ using json = nlohmann::json; inline static json oaicompat_completion_params_parse( - const json &body /* openai api json semantics */) + const json &body, /* openai api json semantics */ + const std::string &chat_template) { json llama_params; + std::string formatted_prompt = chat_template == "chatml" + ? format_chatml(body["messages"]) // OpenAI 'messages' to chatml (with <|im_start|>,...) + : format_llama2(body["messages"]); // OpenAI 'messages' to llama2 (with [INST],...) llama_params["__oaicompat"] = true; @@ -30,7 +34,7 @@ inline static json oaicompat_completion_params_parse( // https://platform.openai.com/docs/api-reference/chat/create llama_sampling_params default_sparams; llama_params["model"] = json_value(body, "model", std::string("unknown")); - llama_params["prompt"] = format_chatml(body["messages"]); // OpenAI 'messages' to llama.cpp 'prompt' + llama_params["prompt"] = formatted_prompt; llama_params["cache_prompt"] = json_value(body, "cache_prompt", false); llama_params["temperature"] = json_value(body, "temperature", 0.0); llama_params["top_k"] = json_value(body, "top_k", default_sparams.top_k); diff --git a/examples/server/server.cpp b/examples/server/server.cpp index 8d668f798..4d212f1f0 100644 --- a/examples/server/server.cpp +++ b/examples/server/server.cpp @@ -36,6 +36,7 @@ struct server_params std::string hostname = "127.0.0.1"; std::vector api_keys; std::string public_path = "examples/server/public"; + std::string chat_template = "chatml"; int32_t port = 8080; int32_t read_timeout = 600; int32_t write_timeout = 600; @@ -1859,6 +1860,8 @@ static void server_print_usage(const char *argv0, const gpt_params ¶ms, printf(" types: int, float, bool. example: --override-kv tokenizer.ggml.add_bos_token=bool:false\n"); printf(" -gan N, --grp-attn-n N set the group attention factor to extend context size through self-extend(default: 1=disabled), used together with group attention width `--grp-attn-w`"); printf(" -gaw N, --grp-attn-w N set the group attention width to extend context size through self-extend(default: 512), used together with group attention factor `--grp-attn-n`"); + printf(" --chat-template FORMAT_NAME"); + printf(" set chat template, possible valus is: llama2, chatml (default %s)", sparams.chat_template.c_str()); printf("\n"); } @@ -2290,6 +2293,21 @@ static void server_params_parse(int argc, char **argv, server_params &sparams, log_set_target(stdout); LOG_INFO("logging to file is disabled.", {}); } + else if (arg == "--chat-template") + { + if (++i >= argc) + { + invalid_param = true; + break; + } + std::string value(argv[i]); + if (value != "chatml" && value != "llama2") { + fprintf(stderr, "error: chat template can be \"llama2\" or \"chatml\", but got: %s\n", value.c_str()); + invalid_param = true; + break; + } + sparams.chat_template = value; + } else if (arg == "--override-kv") { if (++i >= argc) { @@ -2743,13 +2761,13 @@ int main(int argc, char **argv) // TODO: add mount point without "/v1" prefix -- how? - svr.Post("/v1/chat/completions", [&llama, &validate_api_key](const httplib::Request &req, httplib::Response &res) + svr.Post("/v1/chat/completions", [&llama, &validate_api_key, &sparams](const httplib::Request &req, httplib::Response &res) { res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin")); if (!validate_api_key(req, res)) { return; } - json data = oaicompat_completion_params_parse(json::parse(req.body)); + json data = oaicompat_completion_params_parse(json::parse(req.body), sparams.chat_template); const int task_id = llama.queue_tasks.get_new_id(); llama.queue_results.add_waiting_task_id(task_id); diff --git a/examples/server/utils.hpp b/examples/server/utils.hpp index 70cce0721..548548962 100644 --- a/examples/server/utils.hpp +++ b/examples/server/utils.hpp @@ -167,6 +167,34 @@ static T json_value(const json &body, const std::string &key, const T &default_v : default_value; } +inline std::string format_llama2(std::vector messages) +{ + std::ostringstream output; + bool is_inside_turn = false; + + for (auto it = messages.begin(); it != messages.end(); ++it) { + if (!is_inside_turn) { + output << "[INST] "; + } + std::string role = json_value(*it, "role", std::string("user")); + std::string content = json_value(*it, "content", std::string("")); + if (role == "system") { + output << "<>\n" << content << "\n<>\n\n"; + is_inside_turn = true; + } else if (role == "user") { + output << content << " [/INST]"; + is_inside_turn = true; + } else { + output << " " << content << " "; + is_inside_turn = false; + } + } + + LOG_VERBOSE("format_llama2", {{"text", output.str()}}); + + return output.str(); +} + inline std::string format_chatml(std::vector messages) { std::ostringstream chatml_msgs; @@ -180,6 +208,8 @@ inline std::string format_chatml(std::vector messages) chatml_msgs << "<|im_start|>assistant" << '\n'; + LOG_VERBOSE("format_chatml", {{"text", chatml_msgs.str()}}); + return chatml_msgs.str(); } From e4640d8fdf56f14a6db3d092bcd3d2d315cb5d04 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20G=C3=A4=C3=9Fler?= Date: Sun, 11 Feb 2024 12:44:51 +0100 Subject: [PATCH 314/812] lookup: add print for drafting performance (#5450) --- examples/lookup/lookup.cpp | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/examples/lookup/lookup.cpp b/examples/lookup/lookup.cpp index d8de7dd38..18235b8a1 100644 --- a/examples/lookup/lookup.cpp +++ b/examples/lookup/lookup.cpp @@ -1,7 +1,9 @@ #include "common.h" +#include "ggml.h" #include "llama.h" #include +#include #include #include #include @@ -73,6 +75,8 @@ int main(int argc, char ** argv){ int n_drafted = 0; int n_accept = 0; + int64_t t_draft_us = 0; + int n_past = inp.size(); bool has_eos = false; @@ -160,7 +164,7 @@ int main(int argc, char ** argv){ // generate n_pred tokens through prompt lookup auto prompt_lookup = [&]() -> void { - int inp_size = inp.size(); + const int inp_size = inp.size(); for (int ngram_size = ngram_max ; ngram_size > ngram_min; --ngram_size){ const llama_token * ngram = &inp[inp_size - ngram_size]; @@ -191,8 +195,12 @@ int main(int argc, char ** argv){ return; }; + const int64_t t_start_draft_us = ggml_time_us(); + prompt_lookup(); + t_draft_us += ggml_time_us() - t_start_draft_us; + llama_decode(ctx, batch_tgt); ++n_past; @@ -210,6 +218,8 @@ int main(int argc, char ** argv){ LOG_TEE("n_draft = %d\n", n_draft); LOG_TEE("n_predict = %d\n", n_predict); LOG_TEE("n_drafted = %d\n", n_drafted); + LOG_TEE("t_draft = %.2f ms, %.2f us per token, %.2f tokens per second\n", + t_draft_us*1e-3, 1.0f*t_draft_us/n_drafted, n_drafted/(1e-6*t_draft_us)); LOG_TEE("n_accept = %d\n", n_accept); LOG_TEE("accept = %.3f%%\n", 100.0f * n_accept / n_drafted); From a07d0fee1f05c5c1dc49948ae1a3293db017275f Mon Sep 17 00:00:00 2001 From: snadampal <87143774+snadampal@users.noreply.github.com> Date: Sun, 11 Feb 2024 07:22:33 -0600 Subject: [PATCH 315/812] ggml : add mmla kernels for quantized GEMM (#4966) * ggml: aarch64: implement smmla kernel for q8_0_q8_0 quantized gemm armv8.2-a and above supports MMLA instructions that have higher throughput than DOT. this commit adds mmla kernel for q8_0_q8_0 gemm. The feature is enabled if the platform supports "__ARM_FEATURE_MATMUL_INT8" On AWS Graviton3 processors this kernel resulted up to 1.5x improvement for prompt evaluation throughput compared to the default sdot kernel. * ggml: aarch64: implement smmla kernel for q4_0_q8_0 quantized gemm armv8.2-a and above supports MMLA instructions that have higher throughput than DOT. this commit adds mmla kernel for q4_0_q8_0 gemm. The feature is enabled if the platform supports "__ARM_FEATURE_MATMUL_INT8" On AWS Graviton3 processors this kernel resulted up to 1.5x improvement for prompt evaluation throughput compared to the default sdot kernel. * ggml: aarch64: implement smmla kernel for q4_1_q8_1 quantized gemm armv8.2-a and above supports MMLA instructions that have higher throughput than DOT. this commit adds mmla kernel for q4_1_q8_1 gemm. The feature is enabled if the platform supports "__ARM_FEATURE_MATMUL_INT8" On AWS Graviton3 processors this kernel resulted up to 1.5x improvement for prompt evaluation throughput compared to the default sdot kernel. * ggml: update unit tests for the new vec_dot interface * llama.cpp: add MATMUL_INT8 capability to system_info --- common/common.cpp | 1 + ggml-quants.c | 320 +++++++++++++++++++++++++++++++++-- ggml-quants.h | 26 +-- ggml.c | 164 ++++++++++++------ ggml.h | 5 +- llama.cpp | 1 + pocs/vdot/q8dot.cpp | 4 +- pocs/vdot/vdot.cpp | 4 +- tests/test-quantize-fns.cpp | 2 +- tests/test-quantize-perf.cpp | 2 +- 10 files changed, 441 insertions(+), 88 deletions(-) diff --git a/common/common.cpp b/common/common.cpp index e0082a823..9a489a553 100644 --- a/common/common.cpp +++ b/common/common.cpp @@ -1550,6 +1550,7 @@ void dump_non_result_info_yaml(FILE * stream, const gpt_params & params, const l fprintf(stream, "cpu_has_blas: %s\n", ggml_cpu_has_blas() ? "true" : "false"); fprintf(stream, "cpu_has_sse3: %s\n", ggml_cpu_has_sse3() ? "true" : "false"); fprintf(stream, "cpu_has_vsx: %s\n", ggml_cpu_has_vsx() ? "true" : "false"); + fprintf(stream, "cpu_has_matmul_int8: %s\n", ggml_cpu_has_matmul_int8() ? "true" : "false"); #ifdef NDEBUG fprintf(stream, "debug: false\n"); diff --git a/ggml-quants.c b/ggml-quants.c index 1031e3761..6c122dd2a 100644 --- a/ggml-quants.c +++ b/ggml-quants.c @@ -49,6 +49,8 @@ #define MIN(a, b) ((a) < (b) ? (a) : (b)) #define MAX(a, b) ((a) > (b) ? (a) : (b)) +#define UNUSED GGML_UNUSED + #define MM256_SET_M128I(a, b) _mm256_insertf128_si256(_mm256_castsi128_si256(b), (a), 1) #if defined(__AVX__) || defined(__AVX2__) || defined(__AVX512F__) || defined(__SSSE3__) @@ -3677,15 +3679,88 @@ static inline __m128i get_scale_shuffle(int i) { } #endif -void ggml_vec_dot_q4_0_q8_0(int n, float * restrict s, const void * restrict vx, const void * restrict vy) { +void ggml_vec_dot_q4_0_q8_0(int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) { const int qk = QK8_0; const int nb = n / qk; assert(n % qk == 0); +#if defined(__ARM_FEATURE_MATMUL_INT8) + assert((nrc == 2) || (nrc == 1)); +#else + assert(nrc == 1); +#endif const block_q4_0 * restrict x = vx; const block_q8_0 * restrict y = vy; +#if defined(__ARM_FEATURE_MATMUL_INT8) + if (nrc == 2) { + const block_q4_0 * restrict vx0 = vx; + const block_q4_0 * restrict vx1 = vx + bx; + + const block_q8_0 * restrict vy0 = vy; + const block_q8_0 * restrict vy1 = vy + by; + + float32x4_t sumv0 = vdupq_n_f32(0.0f); + + for (int i = 0; i < nb; i++) { + const block_q4_0 * restrict b_x0 = &vx0[i]; + const block_q4_0 * restrict b_x1 = &vx1[i]; + const block_q8_0 * restrict b_y0 = &vy0[i]; + const block_q8_0 * restrict b_y1 = &vy1[i]; + + const uint8x16_t m4b = vdupq_n_u8(0x0F); + const int8x16_t s8b = vdupq_n_s8(0x8); + + const uint8x16_t v0_0 = vld1q_u8(b_x0->qs); + const uint8x16_t v0_1 = vld1q_u8(b_x1->qs); + + // 4-bit -> 8-bit + const int8x16_t v0_0l = vreinterpretq_s8_u8(vandq_u8 (v0_0, m4b)); + const int8x16_t v0_0h = vreinterpretq_s8_u8(vshrq_n_u8(v0_0, 4)); + const int8x16_t v0_1l = vreinterpretq_s8_u8(vandq_u8 (v0_1, m4b)); + const int8x16_t v0_1h = vreinterpretq_s8_u8(vshrq_n_u8(v0_1, 4)); + + // sub 8 + const int8x16_t x0_l = vsubq_s8(v0_0l, s8b); + const int8x16_t x0_h = vsubq_s8(v0_0h, s8b); + const int8x16_t x1_l = vsubq_s8(v0_1l, s8b); + const int8x16_t x1_h = vsubq_s8(v0_1h, s8b); + + // load y + const int8x16_t y0_l = vld1q_s8(b_y0->qs); + const int8x16_t y0_h = vld1q_s8(b_y0->qs + 16); + const int8x16_t y1_l = vld1q_s8(b_y1->qs); + const int8x16_t y1_h = vld1q_s8(b_y1->qs + 16); + + float32x4_t scale = {GGML_FP16_TO_FP32(b_x0->d)*GGML_FP16_TO_FP32(b_y0->d), + GGML_FP16_TO_FP32(b_x0->d)*GGML_FP16_TO_FP32(b_y1->d), + GGML_FP16_TO_FP32(b_x1->d)*GGML_FP16_TO_FP32(b_y0->d), + GGML_FP16_TO_FP32(b_x1->d)*GGML_FP16_TO_FP32(b_y1->d)}; + + int8x16_t l0 = vreinterpretq_s8_s64(vzip1q_s64(vreinterpretq_s64_s8(x0_l), vreinterpretq_s64_s8(x1_l))); + int8x16_t l1 = vreinterpretq_s8_s64(vzip2q_s64(vreinterpretq_s64_s8(x0_l), vreinterpretq_s64_s8(x1_l))); + + int8x16_t l2 = vreinterpretq_s8_s64(vzip1q_s64(vreinterpretq_s64_s8(x0_h), vreinterpretq_s64_s8(x1_h))); + int8x16_t l3 = vreinterpretq_s8_s64(vzip2q_s64(vreinterpretq_s64_s8(x0_h), vreinterpretq_s64_s8(x1_h))); + + int8x16_t r0 = vreinterpretq_s8_s64(vzip1q_s64(vreinterpretq_s64_s8(y0_l), vreinterpretq_s64_s8(y1_l))); + int8x16_t r1 = vreinterpretq_s8_s64(vzip2q_s64(vreinterpretq_s64_s8(y0_l), vreinterpretq_s64_s8(y1_l))); + + int8x16_t r2 = vreinterpretq_s8_s64(vzip1q_s64(vreinterpretq_s64_s8(y0_h), vreinterpretq_s64_s8(y1_h))); + int8x16_t r3 = vreinterpretq_s8_s64(vzip2q_s64(vreinterpretq_s64_s8(y0_h), vreinterpretq_s64_s8(y1_h))); + + sumv0 = vmlaq_f32(sumv0,(vcvtq_f32_s32(vmmlaq_s32((vmmlaq_s32((vmmlaq_s32((vmmlaq_s32(vdupq_n_s32(0), l0, r0)), + l1, r1)), l2, r2)), l3, r3))), scale); + } + float32x4_t sumv1 = vextq_f32(sumv0, sumv0, 2); + float32x4_t sumv2 = vzip1q_f32(sumv0, sumv1); + + vst1_f32(s, vget_low_f32(sumv2)); + vst1_f32(s + bs, vget_high_f32(sumv2)); + return; + } +#endif #if defined(__ARM_NEON) float32x4_t sumv0 = vdupq_n_f32(0.0f); float32x4_t sumv1 = vdupq_n_f32(0.0f); @@ -3967,15 +4042,89 @@ void ggml_vec_dot_q4_0_q8_0(int n, float * restrict s, const void * restrict vx, #endif } -void ggml_vec_dot_q4_1_q8_1(const int n, float * restrict s, const void * restrict vx, const void * restrict vy) { +void ggml_vec_dot_q4_1_q8_1(int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) { const int qk = QK8_1; const int nb = n / qk; assert(n % qk == 0); +#if defined(__ARM_FEATURE_MATMUL_INT8) + assert((nrc == 2) || (nrc == 1)); +#else + assert(nrc == 1); +#endif const block_q4_1 * restrict x = vx; const block_q8_1 * restrict y = vy; +#if defined(__ARM_FEATURE_MATMUL_INT8) + if (nrc == 2) { + const block_q4_1 * restrict vx0 = vx; + const block_q4_1 * restrict vx1 = vx + bx; + const block_q8_1 * restrict vy0 = vy; + const block_q8_1 * restrict vy1 = vy + by; + + float32x4_t sumv0 = vdupq_n_f32(0.0f); + float32x4_t summs0 = vdupq_n_f32(0.0f); + + for (int i = 0; i < nb; i++) { + const block_q4_1 * restrict b_x0 = &vx0[i]; + const block_q4_1 * restrict b_x1 = &vx1[i]; + const block_q8_1 * restrict b_y0 = &vy0[i]; + const block_q8_1 * restrict b_y1 = &vy1[i]; + + float32x4_t summs_t = {GGML_FP16_TO_FP32(b_x0->m) * b_y0->s, + GGML_FP16_TO_FP32(b_x1->m) * b_y0->s, + GGML_FP16_TO_FP32(b_x0->m) * b_y1->s, + GGML_FP16_TO_FP32(b_x1->m) * b_y1->s}; + summs0 += summs_t; + + const uint8x16_t m4b = vdupq_n_u8(0x0F); + + const uint8x16_t v0_0 = vld1q_u8(b_x0->qs); + const uint8x16_t v0_1 = vld1q_u8(b_x1->qs); + + // 4-bit -> 8-bit + const int8x16_t x0_l = vreinterpretq_s8_u8(vandq_u8 (v0_0, m4b)); + const int8x16_t x0_h = vreinterpretq_s8_u8(vshrq_n_u8(v0_0, 4)); + const int8x16_t x1_l = vreinterpretq_s8_u8(vandq_u8 (v0_1, m4b)); + const int8x16_t x1_h = vreinterpretq_s8_u8(vshrq_n_u8(v0_1, 4)); + + // load y + const int8x16_t y0_l = vld1q_s8(b_y0->qs); + const int8x16_t y0_h = vld1q_s8(b_y0->qs + 16); + const int8x16_t y1_l = vld1q_s8(b_y1->qs); + const int8x16_t y1_h = vld1q_s8(b_y1->qs + 16); + + // mmla into int32x4_t + float32x4_t scale = {GGML_FP16_TO_FP32(b_x0->d)*GGML_FP16_TO_FP32(b_y0->d), + GGML_FP16_TO_FP32(b_x0->d)*GGML_FP16_TO_FP32(b_y1->d), + GGML_FP16_TO_FP32(b_x1->d)*GGML_FP16_TO_FP32(b_y0->d), + GGML_FP16_TO_FP32(b_x1->d)*GGML_FP16_TO_FP32(b_y1->d)}; + + int8x16_t l0 = vreinterpretq_s8_s64(vzip1q_s64(vreinterpretq_s64_s8(x0_l), vreinterpretq_s64_s8(x1_l))); + int8x16_t l1 = vreinterpretq_s8_s64(vzip2q_s64(vreinterpretq_s64_s8(x0_l), vreinterpretq_s64_s8(x1_l))); + + int8x16_t l2 = vreinterpretq_s8_s64(vzip1q_s64(vreinterpretq_s64_s8(x0_h), vreinterpretq_s64_s8(x1_h))); + int8x16_t l3 = vreinterpretq_s8_s64(vzip2q_s64(vreinterpretq_s64_s8(x0_h), vreinterpretq_s64_s8(x1_h))); + + int8x16_t r0 = vreinterpretq_s8_s64(vzip1q_s64(vreinterpretq_s64_s8(y0_l), vreinterpretq_s64_s8(y1_l))); + int8x16_t r1 = vreinterpretq_s8_s64(vzip2q_s64(vreinterpretq_s64_s8(y0_l), vreinterpretq_s64_s8(y1_l))); + + int8x16_t r2 = vreinterpretq_s8_s64(vzip1q_s64(vreinterpretq_s64_s8(y0_h), vreinterpretq_s64_s8(y1_h))); + int8x16_t r3 = vreinterpretq_s8_s64(vzip2q_s64(vreinterpretq_s64_s8(y0_h), vreinterpretq_s64_s8(y1_h))); + sumv0 = vmlaq_f32(sumv0,(vcvtq_f32_s32(vmmlaq_s32((vmmlaq_s32((vmmlaq_s32((vmmlaq_s32(vdupq_n_s32(0), l0, r0)), + l1, r1)), l2, r2)), l3, r3))), scale); + } + + float32x4_t sumv1 = vextq_f32(sumv0, sumv0, 2); + float32x4_t sumv2 = vzip1q_f32(sumv0, sumv1); + sumv2 = sumv2 + summs0; + + vst1_f32(s, vget_low_f32(sumv2)); + vst1_f32(s + bs, vget_high_f32(sumv2)); + return; + } +#endif // TODO: add WASM SIMD #if defined(__ARM_NEON) float32x4_t sumv0 = vdupq_n_f32(0.0f); @@ -4107,12 +4256,17 @@ void ggml_vec_dot_q4_1_q8_1(const int n, float * restrict s, const void * restri #endif } -void ggml_vec_dot_q5_0_q8_0(const int n, float * restrict s, const void * restrict vx, const void * restrict vy) { +void ggml_vec_dot_q5_0_q8_0(int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) { const int qk = QK8_0; const int nb = n / qk; assert(n % qk == 0); assert(qk == QK5_0); + assert(nrc == 1); + UNUSED(nrc); + UNUSED(bx); + UNUSED(by); + UNUSED(bs); const block_q5_0 * restrict x = vx; const block_q8_0 * restrict y = vy; @@ -4393,12 +4547,17 @@ void ggml_vec_dot_q5_0_q8_0(const int n, float * restrict s, const void * restri #endif } -void ggml_vec_dot_q5_1_q8_1(const int n, float * restrict s, const void * restrict vx, const void * restrict vy) { +void ggml_vec_dot_q5_1_q8_1(int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) { const int qk = QK8_1; const int nb = n / qk; assert(n % qk == 0); assert(qk == QK5_1); + assert(nrc == 1); + UNUSED(nrc); + UNUSED(bx); + UNUSED(by); + UNUSED(bs); const block_q5_1 * restrict x = vx; const block_q8_1 * restrict y = vy; @@ -4692,15 +4851,75 @@ void ggml_vec_dot_q5_1_q8_1(const int n, float * restrict s, const void * restri #endif } -void ggml_vec_dot_q8_0_q8_0(const int n, float * restrict s, const void * restrict vx, const void * restrict vy) { +void ggml_vec_dot_q8_0_q8_0(int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) { const int qk = QK8_0; const int nb = n / qk; assert(n % qk == 0); +#if defined(__ARM_FEATURE_MATMUL_INT8) + assert((nrc == 2) || (nrc == 1)); +#else + assert(nrc == 1); +#endif const block_q8_0 * restrict x = vx; const block_q8_0 * restrict y = vy; +#if defined(__ARM_FEATURE_MATMUL_INT8) + if (nrc == 2) { + const block_q8_0 * restrict vx0 = vx; + const block_q8_0 * restrict vx1 = vx + bx; + const block_q8_0 * restrict vy0 = vy; + const block_q8_0 * restrict vy1 = vy + by; + + float32x4_t sumv0 = vdupq_n_f32(0.0f); + + for (int i = 0; i < nb; i++) { + const block_q8_0 * restrict b_x0 = &vx0[i]; + const block_q8_0 * restrict b_y0 = &vy0[i]; + + const block_q8_0 * restrict b_x1 = &vx1[i]; + const block_q8_0 * restrict b_y1 = &vy1[i]; + + const int8x16_t x0_l = vld1q_s8(b_x0->qs); + const int8x16_t x0_h = vld1q_s8(b_x0->qs + 16); + const int8x16_t x1_l = vld1q_s8(b_x1->qs); + const int8x16_t x1_h = vld1q_s8(b_x1->qs + 16); + + // load y + const int8x16_t y0_l = vld1q_s8(b_y0->qs); + const int8x16_t y0_h = vld1q_s8(b_y0->qs + 16); + const int8x16_t y1_l = vld1q_s8(b_y1->qs); + const int8x16_t y1_h = vld1q_s8(b_y1->qs + 16); + + float32x4_t scale = {GGML_FP16_TO_FP32(b_x0->d)*GGML_FP16_TO_FP32(b_y0->d), + GGML_FP16_TO_FP32(b_x0->d)*GGML_FP16_TO_FP32(b_y1->d), + GGML_FP16_TO_FP32(b_x1->d)*GGML_FP16_TO_FP32(b_y0->d), + GGML_FP16_TO_FP32(b_x1->d)*GGML_FP16_TO_FP32(b_y1->d)}; + + int8x16_t l0 = vreinterpretq_s8_s64(vzip1q_s64(vreinterpretq_s64_s8(x0_l), vreinterpretq_s64_s8(x1_l))); + int8x16_t l1 = vreinterpretq_s8_s64(vzip2q_s64(vreinterpretq_s64_s8(x0_l), vreinterpretq_s64_s8(x1_l))); + + int8x16_t l2 = vreinterpretq_s8_s64(vzip1q_s64(vreinterpretq_s64_s8(x0_h), vreinterpretq_s64_s8(x1_h))); + int8x16_t l3 = vreinterpretq_s8_s64(vzip2q_s64(vreinterpretq_s64_s8(x0_h), vreinterpretq_s64_s8(x1_h))); + + int8x16_t r0 = vreinterpretq_s8_s64(vzip1q_s64(vreinterpretq_s64_s8(y0_l), vreinterpretq_s64_s8(y1_l))); + int8x16_t r1 = vreinterpretq_s8_s64(vzip2q_s64(vreinterpretq_s64_s8(y0_l), vreinterpretq_s64_s8(y1_l))); + + int8x16_t r2 = vreinterpretq_s8_s64(vzip1q_s64(vreinterpretq_s64_s8(y0_h), vreinterpretq_s64_s8(y1_h))); + int8x16_t r3 = vreinterpretq_s8_s64(vzip2q_s64(vreinterpretq_s64_s8(y0_h), vreinterpretq_s64_s8(y1_h))); + + sumv0 = vmlaq_f32(sumv0,(vcvtq_f32_s32(vmmlaq_s32((vmmlaq_s32((vmmlaq_s32((vmmlaq_s32(vdupq_n_s32(0), l0, r0)), + l1, r1)), l2, r2)), l3, r3))), scale); + } + float32x4_t sumv1 = vextq_f32(sumv0, sumv0, 2); + float32x4_t sumv2 = vzip1q_f32(sumv0, sumv1); + + vst1_f32(s, vget_low_f32(sumv2)); + vst1_f32(s + bs, vget_high_f32(sumv2)); + return; + } +#endif #if defined(__ARM_NEON) float32x4_t sumv0 = vdupq_n_f32(0.0f); float32x4_t sumv1 = vdupq_n_f32(0.0f); @@ -4795,7 +5014,12 @@ void ggml_vec_dot_q8_0_q8_0(const int n, float * restrict s, const void * restri } #if QK_K == 256 -void ggml_vec_dot_q2_K_q8_K(const int n, float * restrict s, const void * restrict vx, const void * restrict vy) { +void ggml_vec_dot_q2_K_q8_K(int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) { + assert(nrc == 1); + UNUSED(nrc); + UNUSED(bx); + UNUSED(by); + UNUSED(bs); const block_q2_K * restrict x = vx; const block_q8_K * restrict y = vy; @@ -5171,7 +5395,12 @@ void ggml_vec_dot_q2_K_q8_K(const int n, float * restrict s, const void * restri #else -void ggml_vec_dot_q2_K_q8_K(const int n, float * restrict s, const void * restrict vx, const void * restrict vy) { +void ggml_vec_dot_q2_K_q8_K(int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) { + assert(nrc == 1); + UNUSED(nrc); + UNUSED(bx); + UNUSED(by); + UNUSED(bs); const block_q2_K * restrict x = vx; const block_q8_K * restrict y = vy; @@ -5429,8 +5658,13 @@ void ggml_vec_dot_q2_K_q8_K(const int n, float * restrict s, const void * restri #endif #if QK_K == 256 -void ggml_vec_dot_q3_K_q8_K(const int n, float * restrict s, const void * restrict vx, const void * restrict vy) { +void ggml_vec_dot_q3_K_q8_K(int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) { assert(n % QK_K == 0); + assert(nrc == 1); + UNUSED(nrc); + UNUSED(bx); + UNUSED(by); + UNUSED(bs); const uint32_t kmask1 = 0x03030303; const uint32_t kmask2 = 0x0f0f0f0f; @@ -5949,8 +6183,13 @@ void ggml_vec_dot_q3_K_q8_K(const int n, float * restrict s, const void * restri #else -void ggml_vec_dot_q3_K_q8_K(const int n, float * restrict s, const void * restrict vx, const void * restrict vy) { +void ggml_vec_dot_q3_K_q8_K(int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) { assert(n % QK_K == 0); + assert(nrc == 1); + UNUSED(nrc); + UNUSED(bx); + UNUSED(by); + UNUSED(bs); const block_q3_K * restrict x = vx; const block_q8_K * restrict y = vy; @@ -6292,8 +6531,13 @@ void ggml_vec_dot_q3_K_q8_K(const int n, float * restrict s, const void * restri #endif #if QK_K == 256 -void ggml_vec_dot_q4_K_q8_K(const int n, float * restrict s, const void * restrict vx, const void * restrict vy) { +void ggml_vec_dot_q4_K_q8_K(int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) { assert(n % QK_K == 0); + assert(nrc == 1); + UNUSED(nrc); + UNUSED(bx); + UNUSED(by); + UNUSED(bs); const block_q4_K * restrict x = vx; const block_q8_K * restrict y = vy; @@ -6648,8 +6892,13 @@ void ggml_vec_dot_q4_K_q8_K(const int n, float * restrict s, const void * restri #endif } #else -void ggml_vec_dot_q4_K_q8_K(const int n, float * restrict s, const void * restrict vx, const void * restrict vy) { +void ggml_vec_dot_q4_K_q8_K(int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) { assert(n % QK_K == 0); + assert(nrc == 1); + UNUSED(nrc); + UNUSED(bx); + UNUSED(by); + UNUSED(bs); const block_q4_K * restrict x = vx; const block_q8_K * restrict y = vy; @@ -6891,8 +7140,13 @@ void ggml_vec_dot_q4_K_q8_K(const int n, float * restrict s, const void * restri #endif #if QK_K == 256 -void ggml_vec_dot_q5_K_q8_K(const int n, float * restrict s, const void * restrict vx, const void * restrict vy) { +void ggml_vec_dot_q5_K_q8_K(int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) { assert(n % QK_K == 0); + assert(nrc == 1); + UNUSED(nrc); + UNUSED(bx); + UNUSED(by); + UNUSED(bs); const block_q5_K * restrict x = vx; const block_q8_K * restrict y = vy; @@ -7311,8 +7565,13 @@ void ggml_vec_dot_q5_K_q8_K(const int n, float * restrict s, const void * restri #else -void ggml_vec_dot_q5_K_q8_K(const int n, float * restrict s, const void * restrict vx, const void * restrict vy) { +void ggml_vec_dot_q5_K_q8_K(int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) { assert(n % QK_K == 0); + assert(nrc == 1); + UNUSED(nrc); + UNUSED(bx); + UNUSED(by); + UNUSED(bs); const block_q5_K * restrict x = vx; const block_q8_K * restrict y = vy; @@ -7577,8 +7836,13 @@ void ggml_vec_dot_q5_K_q8_K(const int n, float * restrict s, const void * restri #if QK_K == 256 -void ggml_vec_dot_q6_K_q8_K(const int n, float * restrict s, const void * restrict vx, const void * restrict vy) { +void ggml_vec_dot_q6_K_q8_K(int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) { assert(n % QK_K == 0); + assert(nrc == 1); + UNUSED(nrc); + UNUSED(bx); + UNUSED(by); + UNUSED(bs); const block_q6_K * restrict x = vx; const block_q8_K * restrict y = vy; @@ -8009,8 +8273,13 @@ void ggml_vec_dot_q6_K_q8_K(const int n, float * restrict s, const void * restri #else -void ggml_vec_dot_q6_K_q8_K(const int n, float * restrict s, const void * restrict vx, const void * restrict vy) { +void ggml_vec_dot_q6_K_q8_K(int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) { assert(n % QK_K == 0); + assert(nrc == 1); + UNUSED(nrc); + UNUSED(bx); + UNUSED(by); + UNUSED(bs); const block_q6_K * restrict x = vx; const block_q8_K * restrict y = vy; @@ -8339,8 +8608,13 @@ static const int8_t keven_signs_q2xs[1024] = { 1, 1, -1, -1, -1, -1, -1, -1, -1, 1, -1, -1, -1, -1, -1, 1, 1, -1, -1, -1, -1, -1, -1, 1, -1, -1, -1, -1, -1, -1, -1, -1, }; -void ggml_vec_dot_iq2_xxs_q8_K(const int n, float * restrict s, const void * restrict vx, const void * restrict vy) { +void ggml_vec_dot_iq2_xxs_q8_K(int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) { assert(n % QK_K == 0); + assert(nrc == 1); + UNUSED(nrc); + UNUSED(bx); + UNUSED(by); + UNUSED(bs); const block_iq2_xxs * restrict x = vx; const block_q8_K * restrict y = vy; @@ -8462,8 +8736,13 @@ void ggml_vec_dot_iq2_xxs_q8_K(const int n, float * restrict s, const void * res #endif } -void ggml_vec_dot_iq2_xs_q8_K(const int n, float * restrict s, const void * restrict vx, const void * restrict vy) { +void ggml_vec_dot_iq2_xs_q8_K(int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) { assert(n % QK_K == 0); + assert(nrc == 1); + UNUSED(nrc); + UNUSED(bx); + UNUSED(by); + UNUSED(bs); const block_iq2_xs * restrict x = vx; const block_q8_K * restrict y = vy; @@ -8682,8 +8961,13 @@ void ggml_vec_dot_iq2_xs_q8_K(const int n, float * restrict s, const void * rest } // TODO -void ggml_vec_dot_iq3_xxs_q8_K(const int n, float * restrict s, const void * restrict vx, const void * restrict vy) { +void ggml_vec_dot_iq3_xxs_q8_K(int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) { assert(n % QK_K == 0); + assert(nrc == 1); + UNUSED(nrc); + UNUSED(bx); + UNUSED(by); + UNUSED(bs); const block_iq3_xxs * restrict x = vx; const block_q8_K * restrict y = vy; diff --git a/ggml-quants.h b/ggml-quants.h index bfdf3c997..68f09b1e1 100644 --- a/ggml-quants.h +++ b/ggml-quants.h @@ -245,20 +245,20 @@ void dequantize_row_iq2_xs (const block_iq2_xs * GGML_RESTRICT x, float * GGML_ void dequantize_row_iq3_xxs(const block_iq3_xxs * GGML_RESTRICT x, float * GGML_RESTRICT y, int k); // Dot product -void ggml_vec_dot_q4_0_q8_0(int n, float * GGML_RESTRICT s, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy); -void ggml_vec_dot_q4_1_q8_1(int n, float * GGML_RESTRICT s, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy); -void ggml_vec_dot_q5_0_q8_0(int n, float * GGML_RESTRICT s, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy); -void ggml_vec_dot_q5_1_q8_1(int n, float * GGML_RESTRICT s, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy); -void ggml_vec_dot_q8_0_q8_0(int n, float * GGML_RESTRICT s, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy); +void ggml_vec_dot_q4_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); +void ggml_vec_dot_q4_1_q8_1(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); +void ggml_vec_dot_q5_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); +void ggml_vec_dot_q5_1_q8_1(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); +void ggml_vec_dot_q8_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); -void ggml_vec_dot_q2_K_q8_K(int n, float * GGML_RESTRICT s, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy); -void ggml_vec_dot_q3_K_q8_K(int n, float * GGML_RESTRICT s, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy); -void ggml_vec_dot_q4_K_q8_K(int n, float * GGML_RESTRICT s, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy); -void ggml_vec_dot_q5_K_q8_K(int n, float * GGML_RESTRICT s, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy); -void ggml_vec_dot_q6_K_q8_K(int n, float * GGML_RESTRICT s, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy); -void ggml_vec_dot_iq2_xxs_q8_K(int n, float * GGML_RESTRICT s, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy); -void ggml_vec_dot_iq2_xs_q8_K (int n, float * GGML_RESTRICT s, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy); -void ggml_vec_dot_iq3_xxs_q8_K(int n, float * GGML_RESTRICT s, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy); +void ggml_vec_dot_q2_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); +void ggml_vec_dot_q3_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); +void ggml_vec_dot_q4_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); +void ggml_vec_dot_q5_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); +void ggml_vec_dot_q6_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); +void ggml_vec_dot_iq2_xxs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); +void ggml_vec_dot_iq2_xs_q8_K (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); +void ggml_vec_dot_iq3_xxs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); // // Quantization utilizing an importance matrix (a.k.a. "Activation aWare Quantization") diff --git a/ggml.c b/ggml.c index 86cd65862..e45b78d7e 100644 --- a/ggml.c +++ b/ggml.c @@ -428,8 +428,8 @@ int64_t ggml_cycles_per_ms(void) { static const size_t CACHE_LINE_SIZE_F32 = CACHE_LINE_SIZE/sizeof(float); -static void ggml_vec_dot_f32(const int n, float * restrict s, const float * restrict x, const float * restrict y); -static void ggml_vec_dot_f16(const int n, float * restrict s, ggml_fp16_t * restrict x, ggml_fp16_t * restrict y); +static void ggml_vec_dot_f32(int n, float * restrict s, size_t bs, const float * restrict x, size_t bx, const float * restrict y, size_t by, int nrc); +static void ggml_vec_dot_f16(int n, float * restrict s, size_t bs, ggml_fp16_t * restrict x, size_t bx, ggml_fp16_t * restrict y, size_t by, int nrc); static const ggml_type_traits_t type_traits[GGML_TYPE_COUNT] = { [GGML_TYPE_I8] = { @@ -457,6 +457,7 @@ static const ggml_type_traits_t type_traits[GGML_TYPE_COUNT] = { .is_quantized = false, .vec_dot = (ggml_vec_dot_t) ggml_vec_dot_f32, .vec_dot_type = GGML_TYPE_F32, + .nrows = 1, }, [GGML_TYPE_F16] = { .type_name = "f16", @@ -468,6 +469,7 @@ static const ggml_type_traits_t type_traits[GGML_TYPE_COUNT] = { .from_float_reference = (ggml_from_float_t) ggml_fp32_to_fp16_row, .vec_dot = (ggml_vec_dot_t) ggml_vec_dot_f16, .vec_dot_type = GGML_TYPE_F16, + .nrows = 1, }, [GGML_TYPE_Q4_0] = { .type_name = "q4_0", @@ -479,6 +481,11 @@ static const ggml_type_traits_t type_traits[GGML_TYPE_COUNT] = { .from_float_reference = (ggml_from_float_t) quantize_row_q4_0_reference, .vec_dot = ggml_vec_dot_q4_0_q8_0, .vec_dot_type = GGML_TYPE_Q8_0, +#if defined (__ARM_FEATURE_MATMUL_INT8) + .nrows = 2, +#else + .nrows = 1, +#endif }, [GGML_TYPE_Q4_1] = { .type_name = "q4_1", @@ -490,6 +497,11 @@ static const ggml_type_traits_t type_traits[GGML_TYPE_COUNT] = { .from_float_reference = (ggml_from_float_t) quantize_row_q4_1_reference, .vec_dot = ggml_vec_dot_q4_1_q8_1, .vec_dot_type = GGML_TYPE_Q8_1, +#if defined (__ARM_FEATURE_MATMUL_INT8) + .nrows = 2, +#else + .nrows = 1, +#endif }, [4] = { // GGML_TYPE_Q4_2 .type_name = "DEPRECATED", @@ -501,6 +513,7 @@ static const ggml_type_traits_t type_traits[GGML_TYPE_COUNT] = { .from_float_reference = NULL, .vec_dot = NULL, .vec_dot_type = GGML_TYPE_COUNT, + .nrows = 1, }, [5] = { // GGML_TYPE_Q4_3 .type_name = "DEPRECATED", @@ -512,6 +525,7 @@ static const ggml_type_traits_t type_traits[GGML_TYPE_COUNT] = { .from_float_reference = NULL, .vec_dot = NULL, .vec_dot_type = GGML_TYPE_COUNT, + .nrows = 1, }, [GGML_TYPE_Q5_0] = { .type_name = "q5_0", @@ -523,6 +537,7 @@ static const ggml_type_traits_t type_traits[GGML_TYPE_COUNT] = { .from_float_reference = (ggml_from_float_t) quantize_row_q5_0_reference, .vec_dot = ggml_vec_dot_q5_0_q8_0, .vec_dot_type = GGML_TYPE_Q8_0, + .nrows = 1, }, [GGML_TYPE_Q5_1] = { .type_name = "q5_1", @@ -534,6 +549,7 @@ static const ggml_type_traits_t type_traits[GGML_TYPE_COUNT] = { .from_float_reference = (ggml_from_float_t) quantize_row_q5_1_reference, .vec_dot = ggml_vec_dot_q5_1_q8_1, .vec_dot_type = GGML_TYPE_Q8_1, + .nrows = 1, }, [GGML_TYPE_Q8_0] = { .type_name = "q8_0", @@ -545,6 +561,11 @@ static const ggml_type_traits_t type_traits[GGML_TYPE_COUNT] = { .from_float_reference = (ggml_from_float_t) quantize_row_q8_0_reference, .vec_dot = ggml_vec_dot_q8_0_q8_0, .vec_dot_type = GGML_TYPE_Q8_0, +#if defined (__ARM_FEATURE_MATMUL_INT8) + .nrows = 2, +#else + .nrows = 1, +#endif }, [GGML_TYPE_Q8_1] = { .type_name = "q8_1", @@ -554,6 +575,7 @@ static const ggml_type_traits_t type_traits[GGML_TYPE_COUNT] = { .from_float = quantize_row_q8_1, .from_float_reference = (ggml_from_float_t) quantize_row_q8_1_reference, .vec_dot_type = GGML_TYPE_Q8_1, + .nrows = 1, }, [GGML_TYPE_Q2_K] = { .type_name = "q2_K", @@ -565,6 +587,7 @@ static const ggml_type_traits_t type_traits[GGML_TYPE_COUNT] = { .from_float_reference = (ggml_from_float_t) quantize_row_q2_K_reference, .vec_dot = ggml_vec_dot_q2_K_q8_K, .vec_dot_type = GGML_TYPE_Q8_K, + .nrows = 1, }, [GGML_TYPE_Q3_K] = { .type_name = "q3_K", @@ -576,6 +599,7 @@ static const ggml_type_traits_t type_traits[GGML_TYPE_COUNT] = { .from_float_reference = (ggml_from_float_t) quantize_row_q3_K_reference, .vec_dot = ggml_vec_dot_q3_K_q8_K, .vec_dot_type = GGML_TYPE_Q8_K, + .nrows = 1, }, [GGML_TYPE_Q4_K] = { .type_name = "q4_K", @@ -587,6 +611,7 @@ static const ggml_type_traits_t type_traits[GGML_TYPE_COUNT] = { .from_float_reference = (ggml_from_float_t) quantize_row_q4_K_reference, .vec_dot = ggml_vec_dot_q4_K_q8_K, .vec_dot_type = GGML_TYPE_Q8_K, + .nrows = 1, }, [GGML_TYPE_Q5_K] = { .type_name = "q5_K", @@ -598,6 +623,7 @@ static const ggml_type_traits_t type_traits[GGML_TYPE_COUNT] = { .from_float_reference = (ggml_from_float_t) quantize_row_q5_K_reference, .vec_dot = ggml_vec_dot_q5_K_q8_K, .vec_dot_type = GGML_TYPE_Q8_K, + .nrows = 1, }, [GGML_TYPE_Q6_K] = { .type_name = "q6_K", @@ -609,6 +635,7 @@ static const ggml_type_traits_t type_traits[GGML_TYPE_COUNT] = { .from_float_reference = (ggml_from_float_t) quantize_row_q6_K_reference, .vec_dot = ggml_vec_dot_q6_K_q8_K, .vec_dot_type = GGML_TYPE_Q8_K, + .nrows = 1, }, [GGML_TYPE_IQ2_XXS] = { .type_name = "iq2_xxs", @@ -620,6 +647,7 @@ static const ggml_type_traits_t type_traits[GGML_TYPE_COUNT] = { .from_float_reference = NULL, .vec_dot = ggml_vec_dot_iq2_xxs_q8_K, .vec_dot_type = GGML_TYPE_Q8_K, + .nrows = 1, }, [GGML_TYPE_IQ2_XS] = { .type_name = "iq2_xs", @@ -631,6 +659,7 @@ static const ggml_type_traits_t type_traits[GGML_TYPE_COUNT] = { .from_float_reference = NULL, .vec_dot = ggml_vec_dot_iq2_xs_q8_K, .vec_dot_type = GGML_TYPE_Q8_K, + .nrows = 1, }, [GGML_TYPE_IQ3_XXS] = { .type_name = "iq3_xxs", @@ -642,6 +671,7 @@ static const ggml_type_traits_t type_traits[GGML_TYPE_COUNT] = { .from_float_reference = (ggml_from_float_t)quantize_row_iq3_xxs_reference, .vec_dot = ggml_vec_dot_iq3_xxs_q8_K, .vec_dot_type = GGML_TYPE_Q8_K, + .nrows = 1, }, [GGML_TYPE_Q8_K] = { .type_name = "q8_K", @@ -1212,7 +1242,13 @@ inline static void ggml_vec_neg_f32 (const int n, float * y, const float * x) inline static void ggml_vec_mul_f32 (const int n, float * z, const float * x, const float * y) { for (int i = 0; i < n; ++i) z[i] = x[i]*y[i]; } inline static void ggml_vec_div_f32 (const int n, float * z, const float * x, const float * y) { for (int i = 0; i < n; ++i) z[i] = x[i]/y[i]; } -static void ggml_vec_dot_f32(const int n, float * restrict s, const float * restrict x, const float * restrict y) { +static void ggml_vec_dot_f32(int n, float * restrict s, size_t bs, const float * restrict x, size_t bx, const float * restrict y, size_t by, int nrc) { + assert(nrc == 1); + UNUSED(nrc); + UNUSED(bx); + UNUSED(by); + UNUSED(bs); + #ifdef GGML_SIMD float sumf = 0.0f; const int np = (n & ~(GGML_F32_STEP - 1)); @@ -1249,7 +1285,13 @@ static void ggml_vec_dot_f32(const int n, float * restrict s, const float * rest *s = sumf; } -static void ggml_vec_dot_f16(const int n, float * restrict s, ggml_fp16_t * restrict x, ggml_fp16_t * restrict y) { +static void ggml_vec_dot_f16(int n, float * restrict s, size_t bs, ggml_fp16_t * restrict x, size_t bx, ggml_fp16_t * restrict y, size_t by, int nrc) { + assert(nrc == 1); + UNUSED(nrc); + UNUSED(bx); + UNUSED(by); + UNUSED(bs); + ggml_float sumf = 0.0; #if defined(GGML_SIMD) @@ -1455,7 +1497,7 @@ inline static void ggml_vec_scale_f32(const int n, float * y, const float v) { #endif } -inline static void ggml_vec_norm_f32 (const int n, float * s, const float * x) { ggml_vec_dot_f32(n, s, x, x); *s = sqrtf(*s); } +inline static void ggml_vec_norm_f32 (const int n, float * s, const float * x) { ggml_vec_dot_f32(n, s, 0, x, 0, x, 0, 1); *s = sqrtf(*s); } inline static void ggml_vec_sqr_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = x[i]*x[i]; } inline static void ggml_vec_sqrt_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = sqrtf(x[i]); } inline static void ggml_vec_log_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = logf(x[i]); } @@ -9992,6 +10034,7 @@ static void ggml_compute_forward_mul_mat( ggml_vec_dot_t const vec_dot = type_traits[type].vec_dot; enum ggml_type const vec_dot_type = type_traits[type].vec_dot_type; ggml_from_float_t const from_float_to_vec_dot = type_traits[vec_dot_type].from_float; + int64_t const vec_dot_num_rows = type_traits[type].nrows; GGML_ASSERT(ne0 == ne01); GGML_ASSERT(ne1 == ne11); @@ -10159,12 +10202,23 @@ static void ggml_compute_forward_mul_mat( const int64_t blck_0 = 16; const int64_t blck_1 = 16; + // dot kernels can handle 1 row and col at a time, but mmla kernels can process 2 rows and cols + int64_t nrc = vec_dot_num_rows; + // TODO: currently the mmla kernels support only even numbered rows/cols. + // this check can be removed once they are extended to support odd numbered rows/cols too + if ((nr0 % 2 != 0) || (ne11 % 2 != 0)) { + nrc = 1; + } + + const size_t src1_col_stride = src1_cont || src1->type != vec_dot_type ? row_size : nb11; + // attempt to reduce false-sharing (does not seem to make a difference) - float tmp[16]; + // 16 * 2, accounting for mmla kernels + float tmp[32]; for (int64_t iir1 = ir110; iir1 < ir111; iir1 += blck_1) { for (int64_t iir0 = ir010; iir0 < ir011; iir0 += blck_0) { - for (int64_t ir1 = iir1; ir1 < iir1 + blck_1 && ir1 < ir111; ++ir1) { + for (int64_t ir1 = iir1; ir1 < iir1 + blck_1 && ir1 < ir111; ir1 += nrc) { const int64_t i13 = (ir1/(ne12*ne1)); const int64_t i12 = (ir1 - i13*ne12*ne1)/ne1; const int64_t i11 = (ir1 - i13*ne12*ne1 - i12*ne1); @@ -10187,17 +10241,19 @@ static void ggml_compute_forward_mul_mat( (src1_cont || src1->type != vec_dot_type ? (i11 + i12*ne11 + i13*ne12*ne11)*row_size : (i11*nb11 + i12*nb12 + i13*nb13)); - float * dst_col = (float *) ((char *) dst->data + (i1*nb1 + i2*nb2 + i3*nb3)); //for (int64_t ir0 = iir0; ir0 < iir0 + blck_0 && ir0 < ir011; ++ir0) { // vec_dot(ne00, &dst_col[ir0], src0_row + ir0*nb01, src1_col); //} - for (int64_t ir0 = iir0; ir0 < iir0 + blck_0 && ir0 < ir011; ++ir0) { - vec_dot(ne00, &tmp[ir0 - iir0], src0_row + ir0*nb01, src1_col); + for (int64_t ir0 = iir0; ir0 < iir0 + blck_0 && ir0 < ir011; ir0 += nrc) { + vec_dot(ne00, &tmp[ir0 - iir0], (nrc>1 ? 16 : 0), src0_row + ir0*nb01, (nrc>1 ? nb01 : 0), src1_col, (nrc>1 ? src1_col_stride : 0), nrc); + } + + for (int cn = 0; cn < nrc; ++cn) { + memcpy(&dst_col[iir0 + cn*nb1/nb0], tmp + (cn*16), (MIN(iir0 + blck_0, ir011) - iir0)*sizeof(float)); } - memcpy(&dst_col[iir0], tmp, (MIN(iir0 + blck_0, ir011) - iir0)*sizeof(float)); } } } @@ -10386,7 +10442,7 @@ static void ggml_compute_forward_mul_mat_id( //} for (int64_t ir0 = iir0; ir0 < iir0 + blck_0 && ir0 < ir011; ++ir0) { - vec_dot(ne00, &tmp[ir0 - iir0], src0_row + ir0*nb01, src1_col); + vec_dot(ne00, &tmp[ir0 - iir0], 0, src0_row + ir0*nb01, 0, src1_col, 0, 1); } memcpy(&dst_col[iir0], tmp, (MIN(iir0 + blck_0, ir011) - iir0)*sizeof(float)); } @@ -11568,7 +11624,7 @@ static void ggml_compute_forward_soft_max_back_f32( // linear runtime, no additional memory float dot_y_dy = 0; - ggml_vec_dot_f32 (nc, &dot_y_dy, y, dy); + ggml_vec_dot_f32 (nc, &dot_y_dy, 0, y, 0, dy, 0, 1); ggml_vec_cpy_f32 (nc, dx, dy); ggml_vec_acc1_f32(nc, dx, -dot_y_dy); ggml_vec_mul_f32 (nc, dx, dx, y); @@ -12369,9 +12425,9 @@ static void ggml_compute_forward_conv_transpose_1d_f16_f32( const int i1n = i10*ne11; for (int i00 = 0; i00 < ne00; i00++) { float v = 0; - ggml_vec_dot_f16(ne02, &v, - (ggml_fp16_t *) wdata_src + i1n, - (ggml_fp16_t *) wdata_kernel + i00*ne02); + ggml_vec_dot_f16(ne02, &v, 0, + (ggml_fp16_t *) wdata_src + i1n, 0, + (ggml_fp16_t *) wdata_kernel + i00*ne02, 0, 1); dst_data[i10*s0 + i00] += v; } } @@ -12466,9 +12522,9 @@ static void ggml_compute_forward_conv_transpose_1d_f32( const int i1n = i10*ne11; for (int i00 = 0; i00 < ne00; i00++) { float v = 0; - ggml_vec_dot_f32(ne02, &v, - wdata_src + i1n, - wdata_kernel + i00*ne02); + ggml_vec_dot_f32(ne02, &v, 0, + wdata_src + i1n, 0, + wdata_kernel + i00*ne02, 0, 1); dst_data[i10*s0 + i00] += v; } } @@ -12783,9 +12839,9 @@ static void ggml_compute_forward_conv_transpose_2d( for (int i01 = 0; i01 < ne01; i01++) { for (int i00 = 0; i00 < ne00; i00++) { float v = 0; - ggml_vec_dot_f16(ne03, &v, - wdata_src + i1n, - wdata_kernel + i01*ne00*ne03 + i00*ne03); + ggml_vec_dot_f16(ne03, &v, 0, + wdata_src + i1n, 0, + wdata_kernel + i01*ne00*ne03 + i00*ne03, 0, 1); dst_data[(i11*stride + i01)*ne0 + i10*stride + i00] += v; } } @@ -13214,9 +13270,9 @@ static void ggml_compute_forward_flash_attn_f32( const int i1 = ik1; ggml_vec_dot_f32(neq0, - S + i1, - (float *) ((char *) k->data + (ik1*nbk1 + ik2*nbk2 + ik3*nbk3)), - (float *) ((char *) q->data + (iq1*nbq1 + iq2*nbq2 + iq3*nbq3))); + S + i1, 0, + (float *) ((char *) k->data + (ik1*nbk1 + ik2*nbk2 + ik3*nbk3)), 0, + (float *) ((char *) q->data + (iq1*nbq1 + iq2*nbq2 + iq3*nbq3)), 0, 1); } // scale @@ -13299,9 +13355,9 @@ static void ggml_compute_forward_flash_attn_f32( const int iv3 = iq3; ggml_vec_dot_f32(masked_begin, - (float *) ((char *) dst->data + (ic*nb0 + i1*nb1 + i2*nb2 + i3*nb3)), - (float *) ((char *) v->data + ( ic*nbv1 + iv2*nbv2 + iv3*nbv3)), - S); + (float *) ((char *) dst->data + (ic*nb0 + i1*nb1 + i2*nb2 + i3*nb3)), 0, + (float *) ((char *) v->data + ( ic*nbv1 + iv2*nbv2 + iv3*nbv3)), 0, + S, 0, 1); } } } @@ -13404,9 +13460,9 @@ static void ggml_compute_forward_flash_attn_f16( const int i1 = ik1; ggml_vec_dot_f16(neq0, - S + i1, - (ggml_fp16_t *) ((char *) k->data + (ik1*nbk1 + ik2*nbk2 + ik3*nbk3)), - (ggml_fp16_t *) ((char *) q->data + (iq1*nbq1 + iq2*nbq2 + iq3*nbq3))); + S + i1, 0, + (ggml_fp16_t *) ((char *) k->data + (ik1*nbk1 + ik2*nbk2 + ik3*nbk3)), 0, + (ggml_fp16_t *) ((char *) q->data + (iq1*nbq1 + iq2*nbq2 + iq3*nbq3)), 0, 1); } } else { for (int64_t ic = 0; ic < nek1; ic += GGML_VEC_DOT_UNROLL) { @@ -13508,9 +13564,9 @@ static void ggml_compute_forward_flash_attn_f16( const int iv3 = iq3; ggml_vec_dot_f16(nev0, - (float *) ((char *) dst->data + (ic*nb0 + i1*nb1 + i2*nb2 + i3*nb3)), - (ggml_fp16_t *) ((char *) v->data + ( ic*nbv1 + iv2*nbv2 + iv3*nbv3)), - S16); + (float *) ((char *) dst->data + (ic*nb0 + i1*nb1 + i2*nb2 + i3*nb3)), 0, + (ggml_fp16_t *) ((char *) v->data + ( ic*nbv1 + iv2*nbv2 + iv3*nbv3)), 0, + S16, 0, 1); } } else { for (int64_t ic = 0; ic < nev1; ic += GGML_VEC_DOT_UNROLL) { @@ -13652,9 +13708,9 @@ static void ggml_compute_forward_flash_ff_f16( const int i1 = ib01; ggml_vec_dot_f16(nea0, - S + i1, - (ggml_fp16_t *) ((char *) b0->data + (ib01*nbb01 + ib02*nbb02 + ib03*nbb03)), - (ggml_fp16_t *) ((char *) a->data + ( ia1*nba1 + ia2*nba2 + ia3*nba3))); + S + i1, 0, + (ggml_fp16_t *) ((char *) b0->data + (ib01*nbb01 + ib02*nbb02 + ib03*nbb03)), 0, + (ggml_fp16_t *) ((char *) a->data + ( ia1*nba1 + ia2*nba2 + ia3*nba3)), 0, 1); } ggml_vec_add_f32(neb01, S, S, (float *) b1->data); @@ -13677,9 +13733,9 @@ static void ggml_compute_forward_flash_ff_f16( for (int64_t ic = 0; ic < nec01; ++ic) { ggml_vec_dot_f16(neb01, - (float *) ((char *) dst->data + (ic*nb0 + i1*nb1 + i2*nb2 + i3*nb3)), - (ggml_fp16_t *) ((char *) c0->data + ( ic*nbc01 + i2*nbc02 + i3*nbc03)), - S16); + (float *) ((char *) dst->data + (ic*nb0 + i1*nb1 + i2*nb2 + i3*nb3)), 0, + (ggml_fp16_t *) ((char *) c0->data + ( ic*nbc01 + i2*nbc02 + i3*nbc03)), 0, + S16, 0, 1); } ggml_vec_add_f32(nec01, @@ -13866,9 +13922,9 @@ static void ggml_compute_forward_flash_attn_back_f32( const int i1 = ik1; ggml_vec_dot_f32(neq0, - S + i1, - (float *) ((char *) k->data + (ik1*nbk1 + ik2*nbk2 + ik3*nbk3)), - (float *) ((char *) q->data + (iq1*nbq1 + iq2*nbq2 + iq3*nbq3))); + S + i1, 0, + (float *) ((char *) k->data + (ik1*nbk1 + ik2*nbk2 + ik3*nbk3)), 0, + (float *) ((char *) q->data + (iq1*nbq1 + iq2*nbq2 + iq3*nbq3)), 0, 1); } // scale @@ -14013,7 +14069,7 @@ static void ggml_compute_forward_flash_attn_back_f32( // S = SM * (S - dot(SM, S)) float dot_SM_gradSM = 0; - ggml_vec_dot_f32 (masked_begin, &dot_SM_gradSM, SM, S); + ggml_vec_dot_f32 (masked_begin, &dot_SM_gradSM, 0, SM, 0, S, 0, 1); ggml_vec_acc1_f32(M, S, -dot_SM_gradSM); ggml_vec_mul_f32 (masked_begin, S, S, SM); @@ -18382,7 +18438,7 @@ static enum ggml_opt_result linesearch_backtracking( } // compute the initial gradient in the search direction - ggml_vec_dot_f32(nx, &dginit, g, d); + ggml_vec_dot_f32(nx, &dginit, 0, g, 0, d, 0, 1); // make sure that d points to a descent direction if (0 < dginit) { @@ -18432,7 +18488,7 @@ static enum ggml_opt_result linesearch_backtracking( return count; } - ggml_vec_dot_f32(nx, &dg, g, d); + ggml_vec_dot_f32(nx, &dg, 0, g, 0, d, 0, 1); // check the Wolfe condition if (dg < params->lbfgs.wolfe * dginit) { @@ -18693,8 +18749,8 @@ static enum ggml_opt_result ggml_opt_lbfgs( // ys = y^t \cdot s -> 1 / \rho. // yy = y^t \cdot y. // - ggml_vec_dot_f32(nx, &ys, &lm_y[end[0]*nx], &lm_s[end[0]*nx]); - ggml_vec_dot_f32(nx, &yy, &lm_y[end[0]*nx], &lm_y[end[0]*nx]); + ggml_vec_dot_f32(nx, &ys, 0, &lm_y[end[0]*nx], 0, &lm_s[end[0]*nx], 0, 1); + ggml_vec_dot_f32(nx, &yy, 0, &lm_y[end[0]*nx], 0, &lm_y[end[0]*nx], 0, 1); lm_ys[end[0]] = ys; @@ -18713,7 +18769,7 @@ static enum ggml_opt_result ggml_opt_lbfgs( for (int i = 0; i < bound; ++i) { j[0] = (j[0] + m - 1) % m; // \alpha_{j} = \rho_{j} s^{t}_{j} \cdot q_{k+1} - ggml_vec_dot_f32(nx, &lm_alpha[j[0]], &lm_s[j[0]*nx], d); + ggml_vec_dot_f32(nx, &lm_alpha[j[0]], 0, &lm_s[j[0]*nx], 0, d, 0, 1); lm_alpha[j[0]] /= lm_ys[j[0]]; // q_{i} = q_{i+1} - \alpha_{i} y_{i} ggml_vec_mad_f32(nx, d, &lm_y[j[0]*nx], -lm_alpha[j[0]]); @@ -18723,7 +18779,7 @@ static enum ggml_opt_result ggml_opt_lbfgs( for (int i = 0; i < bound; ++i) { // \beta_{j} = \rho_{j} y^t_{j} \cdot \gamma_{i} - ggml_vec_dot_f32(nx, &beta, &lm_y[j[0]*nx], d); + ggml_vec_dot_f32(nx, &beta, 0, &lm_y[j[0]*nx], 0, d, 0, 1); beta /= lm_ys[j[0]]; // \gamma_{i+1} = \gamma_{i} + (\alpha_{j} - \beta_{j}) s_{j} ggml_vec_mad_f32(nx, d, &lm_s[j[0]*nx], lm_alpha[j[0]] - beta); @@ -20611,4 +20667,12 @@ int ggml_cpu_has_vsx(void) { #endif } +int ggml_cpu_has_matmul_int8(void) { +#if defined(__ARM_FEATURE_MATMUL_INT8) + return 1; +#else + return 0; +#endif +} + //////////////////////////////////////////////////////////////////////////////// diff --git a/ggml.h b/ggml.h index 1360cd8ee..9cfec5bac 100644 --- a/ggml.h +++ b/ggml.h @@ -2278,6 +2278,7 @@ extern "C" { GGML_API int ggml_cpu_has_ssse3 (void); GGML_API int ggml_cpu_has_sycl (void); GGML_API int ggml_cpu_has_vsx (void); + GGML_API int ggml_cpu_has_matmul_int8(void); // // Internal types and functions exposed for tests and benchmarks @@ -2291,7 +2292,8 @@ extern "C" { #endif typedef void (*ggml_to_float_t) (const void * GGML_RESTRICT x, float * GGML_RESTRICT y, int k); typedef void (*ggml_from_float_t)(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int k); - typedef void (*ggml_vec_dot_t) (const int n, float * GGML_RESTRICT s, const void * GGML_RESTRICT x, const void * GGML_RESTRICT y); + typedef void (*ggml_vec_dot_t) (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT x, size_t bx, + const void * GGML_RESTRICT y, size_t by, int nrc); typedef struct { const char * type_name; @@ -2303,6 +2305,7 @@ extern "C" { ggml_from_float_t from_float_reference; ggml_vec_dot_t vec_dot; enum ggml_type vec_dot_type; + int64_t nrows; // number of rows to process simultaneously; } ggml_type_traits_t; GGML_API ggml_type_traits_t ggml_internal_get_type_traits(enum ggml_type type); diff --git a/llama.cpp b/llama.cpp index 0566b087b..3f39a67fb 100644 --- a/llama.cpp +++ b/llama.cpp @@ -11869,6 +11869,7 @@ const char * llama_print_system_info(void) { s += "SSE3 = " + std::to_string(ggml_cpu_has_sse3()) + " | "; s += "SSSE3 = " + std::to_string(ggml_cpu_has_ssse3()) + " | "; s += "VSX = " + std::to_string(ggml_cpu_has_vsx()) + " | "; + s += "MATMUL_INT8 = " + std::to_string(ggml_cpu_has_matmul_int8()) + " | "; return s.c_str(); } diff --git a/pocs/vdot/q8dot.cpp b/pocs/vdot/q8dot.cpp index 111770d55..1a52ff5e9 100644 --- a/pocs/vdot/q8dot.cpp +++ b/pocs/vdot/q8dot.cpp @@ -156,8 +156,8 @@ int main(int argc, char** argv) { t1 = std::chrono::high_resolution_clock::now(); float fs; - if (type == 0) funcs.vec_dot(kVecSize * QK4_1, &fs, x40.data(), y.data()); - else funcs.vec_dot(kVecSize * QK4_1, &fs, x41.data(), y.data()); + if (type == 0) funcs.vec_dot(kVecSize * QK4_1, &fs, 0, x40.data(), 0, y.data(), 0, 1); + else funcs.vec_dot(kVecSize * QK4_1, &fs, 0, x41.data(), 0, y.data(), 0, 1); t2 = std::chrono::high_resolution_clock::now(); t = 1e-3*std::chrono::duration_cast(t2-t1).count(); if (iloop > 3) ggml.addResult(fs, t); diff --git a/pocs/vdot/vdot.cpp b/pocs/vdot/vdot.cpp index 73ffcd1ca..17e9e4482 100644 --- a/pocs/vdot/vdot.cpp +++ b/pocs/vdot/vdot.cpp @@ -284,8 +284,8 @@ int main(int argc, char** argv) { else { auto vdot = ggml_internal_get_type_traits(funcs.vec_dot_type); vdot.from_float(y1.data(), q8.data(), kVecSize); - if (useQ4_1) funcs.vec_dot(kVecSize, &result, q41.data(), q8.data()); - else funcs.vec_dot(kVecSize, &result, q40.data(), q8.data()); + if (useQ4_1) funcs.vec_dot(kVecSize, &result, 0, q41.data(), 0, q8.data(), 0, 1); + else funcs.vec_dot(kVecSize, &result, 0, q40.data(), 0, q8.data(), 0, 1); } sumq += result; t2 = std::chrono::high_resolution_clock::now(); diff --git a/tests/test-quantize-fns.cpp b/tests/test-quantize-fns.cpp index 43df8022d..5e92d5742 100644 --- a/tests/test-quantize-fns.cpp +++ b/tests/test-quantize-fns.cpp @@ -87,7 +87,7 @@ static float dot_product_error( vdot.from_float(test_data2, tmp_q2.data(), test_size); float result = INFINITY; - qfns.vec_dot(test_size, &result, tmp_q1.data(), tmp_q2.data()); + qfns.vec_dot(test_size, &result, 0, tmp_q1.data(), 0, tmp_q2.data(), 0, 1); const float dot_ref = dot_product(test_data1, test_data2, test_size); diff --git a/tests/test-quantize-perf.cpp b/tests/test-quantize-perf.cpp index 8ec817344..48d9fae3d 100644 --- a/tests/test-quantize-perf.cpp +++ b/tests/test-quantize-perf.cpp @@ -346,7 +346,7 @@ int main(int argc, char * argv[]) { printf(" %zu values (%.2f MB)\n", size, 4*size/(float)(1024*1024)); auto quantize_fn = [&](void) -> float { float result; - qfns.vec_dot(size, &result, test_q1, test_q2); + qfns.vec_dot(size, &result, 0, test_q1, 0, test_q2, 0, 1); return result; }; size_t quantized_size = ggml_row_size(type, size); From 0f2411f154db46780d3aaa3a0664691b2170c83f Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Sun, 11 Feb 2024 15:33:01 +0200 Subject: [PATCH 316/812] ggml : fix compile warnings (unused vars) (#4966) --- ggml-quants.c | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/ggml-quants.c b/ggml-quants.c index 6c122dd2a..b2a309bf8 100644 --- a/ggml-quants.c +++ b/ggml-quants.c @@ -3689,6 +3689,10 @@ void ggml_vec_dot_q4_0_q8_0(int n, float * restrict s, size_t bs, const void * r #else assert(nrc == 1); #endif + UNUSED(nrc); + UNUSED(bx); + UNUSED(by); + UNUSED(bs); const block_q4_0 * restrict x = vx; const block_q8_0 * restrict y = vy; @@ -4052,6 +4056,10 @@ void ggml_vec_dot_q4_1_q8_1(int n, float * restrict s, size_t bs, const void * r #else assert(nrc == 1); #endif + UNUSED(nrc); + UNUSED(bx); + UNUSED(by); + UNUSED(bs); const block_q4_1 * restrict x = vx; const block_q8_1 * restrict y = vy; @@ -4861,6 +4869,10 @@ void ggml_vec_dot_q8_0_q8_0(int n, float * restrict s, size_t bs, const void * r #else assert(nrc == 1); #endif + UNUSED(nrc); + UNUSED(bx); + UNUSED(by); + UNUSED(bs); const block_q8_0 * restrict x = vx; const block_q8_0 * restrict y = vy; From 139b62a839825ef20084ed75ed624db7a5ad554a Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Sun, 11 Feb 2024 15:33:43 +0200 Subject: [PATCH 317/812] common : fix compile warning --- common/sampling.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/common/sampling.cpp b/common/sampling.cpp index 844ad7c53..82cbdecea 100644 --- a/common/sampling.cpp +++ b/common/sampling.cpp @@ -127,8 +127,6 @@ static void sampler_queue( const llama_sampling_params & params, llama_token_data_array & cur_p, size_t & min_keep) { - const int n_vocab = llama_n_vocab(llama_get_model(ctx_main)); - const float temp = params.temp; const float dynatemp_range = params.dynatemp_range; const float dynatemp_exponent = params.dynatemp_exponent; From 85910c5b30f6e268321be8df044f5528a6efac52 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Sun, 11 Feb 2024 15:35:50 +0200 Subject: [PATCH 318/812] main : ctrl+C print timing in non-interactive mode (#3873) --- examples/main/main.cpp | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/examples/main/main.cpp b/examples/main/main.cpp index 0ed4d79f9..e8ab8cbae 100644 --- a/examples/main/main.cpp +++ b/examples/main/main.cpp @@ -98,7 +98,7 @@ static void write_logfile( #if defined (__unix__) || (defined (__APPLE__) && defined (__MACH__)) || defined (_WIN32) static void sigint_handler(int signo) { if (signo == SIGINT) { - if (!is_interacting) { + if (!is_interacting && g_params->interactive) { is_interacting = true; } else { console::cleanup(); @@ -392,7 +392,8 @@ int main(int argc, char ** argv) { LOG_TEE("\n"); } - if (params.interactive) { + // ctrl+C handling + { #if defined (__unix__) || (defined (__APPLE__) && defined (__MACH__)) struct sigaction sigint_action; sigint_action.sa_handler = sigint_handler; @@ -405,7 +406,9 @@ int main(int argc, char ** argv) { }; SetConsoleCtrlHandler(reinterpret_cast(console_ctrl_handler), true); #endif + } + if (params.interactive) { LOG_TEE("%s: interactive mode on.\n", __func__); if (!params.antiprompt.empty()) { From 684780141a08200ec98eba3e982dbafd1d0b5000 Mon Sep 17 00:00:00 2001 From: Alexey Parfenov Date: Sun, 11 Feb 2024 13:38:14 +0000 Subject: [PATCH 319/812] server : allow to specify tokens as strings in logit_bias (#5003) * server: allow to specify tokens as strings in logit_bias * Apply suggestions from code review Co-authored-by: Georgi Gerganov --------- Co-authored-by: Georgi Gerganov --- examples/server/README.md | 2 +- examples/server/server.cpp | 32 +++++++++++++++++++++++++------- 2 files changed, 26 insertions(+), 8 deletions(-) diff --git a/examples/server/README.md b/examples/server/README.md index 1db7cdf21..0f7373ae8 100644 --- a/examples/server/README.md +++ b/examples/server/README.md @@ -185,7 +185,7 @@ node index.js `ignore_eos`: Ignore end of stream token and continue generating (default: false). - `logit_bias`: Modify the likelihood of a token appearing in the generated text completion. For example, use `"logit_bias": [[15043,1.0]]` to increase the likelihood of the token 'Hello', or `"logit_bias": [[15043,-1.0]]` to decrease its likelihood. Setting the value to false, `"logit_bias": [[15043,false]]` ensures that the token `Hello` is never produced (default: []). + `logit_bias`: Modify the likelihood of a token appearing in the generated text completion. For example, use `"logit_bias": [[15043,1.0]]` to increase the likelihood of the token 'Hello', or `"logit_bias": [[15043,-1.0]]` to decrease its likelihood. Setting the value to false, `"logit_bias": [[15043,false]]` ensures that the token `Hello` is never produced. The tokens can also be represented as strings, e.g. `[["Hello, World!",-0.5]]` will reduce the likelihood of all the individual tokens that represent the string `Hello, World!`, just like the `presence_penalty` does. (default: []). `n_probs`: If greater than 0, the response also contains the probabilities of top N tokens for each generated token (default: 0) diff --git a/examples/server/server.cpp b/examples/server/server.cpp index 4d212f1f0..1699eb76b 100644 --- a/examples/server/server.cpp +++ b/examples/server/server.cpp @@ -626,18 +626,36 @@ struct llama_server_context const int n_vocab = llama_n_vocab(model); for (const auto &el : *logit_bias) { - if (el.is_array() && el.size() == 2 && el[0].is_number_integer()) + if (el.is_array() && el.size() == 2) { - llama_token tok = el[0].get(); - if (tok >= 0 && tok < n_vocab) + float bias; + if (el[1].is_number()) { - if (el[1].is_number()) + bias = el[1].get(); + } + else if (el[1].is_boolean() && !el[1].get()) + { + bias = -INFINITY; + } + else + { + continue; + } + + if (el[0].is_number_integer()) + { + llama_token tok = el[0].get(); + if (tok >= 0 && tok < n_vocab) { - slot->sparams.logit_bias[tok] = el[1].get(); + slot->sparams.logit_bias[tok] = bias; } - else if (el[1].is_boolean() && !el[1].get()) + } + else if (el[0].is_string()) + { + auto toks = llama_tokenize(model, el[0].get(), false); + for (auto tok : toks) { - slot->sparams.logit_bias[tok] = -INFINITY; + slot->sparams.logit_bias[tok] = bias; } } } From a803333a4e6fc534c93afe90d741bc2388bdec87 Mon Sep 17 00:00:00 2001 From: Alexey Parfenov Date: Sun, 11 Feb 2024 13:43:31 +0000 Subject: [PATCH 320/812] common : use enums for sampler types (#5418) * common: use enums for sampler types * Apply suggestions from code review Co-authored-by: Georgi Gerganov * minor : spaces --------- Co-authored-by: Georgi Gerganov --- common/common.cpp | 117 +++++++++++++++++++++++++++++++------------- common/common.h | 7 ++- common/sampling.cpp | 31 +++++------- common/sampling.h | 20 +++++++- 4 files changed, 120 insertions(+), 55 deletions(-) diff --git a/common/common.cpp b/common/common.cpp index 9a489a553..f64da2cb6 100644 --- a/common/common.cpp +++ b/common/common.cpp @@ -340,13 +340,14 @@ bool gpt_params_parse_ex(int argc, char ** argv, gpt_params & params) { invalid_param = true; break; } - sparams.samplers_sequence = parse_samplers_input(argv[i]); + const auto sampler_names = string_split(argv[i], ';'); + sparams.samplers_sequence = sampler_types_from_names(sampler_names); } else if (arg == "--sampling-seq") { if (++i >= argc) { invalid_param = true; break; } - sparams.samplers_sequence = argv[i]; + sparams.samplers_sequence = sampler_types_from_chars(argv[i]); } else if (arg == "--top-p") { if (++i >= argc) { invalid_param = true; @@ -906,6 +907,14 @@ bool gpt_params_parse_ex(int argc, char ** argv, gpt_params & params) { void gpt_print_usage(int /*argc*/, char ** argv, const gpt_params & params) { const llama_sampling_params & sparams = params.sparams; + std::string sampler_type_chars; + std::string sampler_type_names; + for (const auto sampler_type : sparams.samplers_sequence) { + sampler_type_chars += static_cast(sampler_type); + sampler_type_names += sampler_type_to_name_string(sampler_type) + ";"; + } + sampler_type_names.pop_back(); + printf("\n"); printf("usage: %s [options]\n", argv[0]); printf("\n"); @@ -947,8 +956,8 @@ void gpt_print_usage(int /*argc*/, char ** argv, const gpt_params & params) { printf(" -n N, --n-predict N number of tokens to predict (default: %d, -1 = infinity, -2 = until context filled)\n", params.n_predict); printf(" -c N, --ctx-size N size of the prompt context (default: %d, 0 = loaded from model)\n", params.n_ctx); printf(" -b N, --batch-size N batch size for prompt processing (default: %d)\n", params.n_batch); - printf(" --samplers samplers that will be used for generation in the order, separated by \';\', for example: \"top_k;tfs;typical;top_p;min_p;temp\"\n"); - printf(" --sampling-seq simplified sequence for samplers that will be used (default: %s)\n", sparams.samplers_sequence.c_str()); + printf(" --samplers samplers that will be used for generation in the order, separated by \';\' (default: %s)\n", sampler_type_names.c_str()); + printf(" --sampling-seq simplified sequence for samplers that will be used (default: %s)\n", sampler_type_chars.c_str()); printf(" --top-k N top-k sampling (default: %d, 0 = disabled)\n", sparams.top_k); printf(" --top-p N top-p sampling (default: %.1f, 1.0 = disabled)\n", (double)sparams.top_p); printf(" --min-p N min-p sampling (default: %.1f, 0.0 = disabled)\n", (double)sparams.min_p); @@ -1097,45 +1106,85 @@ std::string gpt_random_prompt(std::mt19937 & rng) { } // -// String parsing +// String utils // -std::string parse_samplers_input(std::string input) { - std::string output = ""; +std::vector string_split(std::string input, char separator) { + std::vector parts; + size_t separator_pos = input.find(separator); + while (separator_pos != std::string::npos) { + std::string part = input.substr(0, separator_pos); + parts.emplace_back(part); + input = input.substr(separator_pos + 1); + separator_pos = input.find(separator); + } + parts.emplace_back(input); + return parts; +} + +std::vector sampler_types_from_names(const std::vector & names) { // since samplers names are written multiple ways // make it ready for both system names and input names - std::unordered_map samplers_symbols { - {"top_k", 'k'}, - {"top-k", 'k'}, - {"top_p", 'p'}, - {"top-p", 'p'}, - {"nucleus", 'p'}, - {"typical_p", 'y'}, - {"typical-p", 'y'}, - {"typical", 'y'}, - {"min_p", 'm'}, - {"min-p", 'm'}, - {"tfs_z", 'f'}, - {"tfs-z", 'f'}, - {"tfs", 'f'}, - {"temp", 't'}, - {"temperature",'t'} + std::unordered_map sampler_name_map { + {"top_k", llama_sampler_type::TOP_K}, + {"top-k", llama_sampler_type::TOP_K}, + {"top_p", llama_sampler_type::TOP_P}, + {"top-p", llama_sampler_type::TOP_P}, + {"nucleus", llama_sampler_type::TOP_P}, + {"typical_p", llama_sampler_type::TYPICAL_P}, + {"typical-p", llama_sampler_type::TYPICAL_P}, + {"typical", llama_sampler_type::TYPICAL_P}, + {"min_p", llama_sampler_type::MIN_P}, + {"min-p", llama_sampler_type::MIN_P}, + {"tfs_z", llama_sampler_type::TFS_Z}, + {"tfs-z", llama_sampler_type::TFS_Z}, + {"tfs", llama_sampler_type::TFS_Z}, + {"temp", llama_sampler_type::TEMP}, + {"temperature", llama_sampler_type::TEMP} }; - // expected format example: "temp;top_k;tfs_z;typical_p;top_p;min_p" - size_t separator = input.find(';'); - while (separator != input.npos) { - std::string name = input.substr(0,separator); - input = input.substr(separator+1); - separator = input.find(';'); - if (samplers_symbols.find(name) != samplers_symbols.end()) { - output += samplers_symbols[name]; + std::vector sampler_types; + sampler_types.reserve(names.size()); + for (const auto& name : names) { + const auto sampler_item = sampler_name_map.find(name); + if (sampler_item != sampler_name_map.end()) { + sampler_types.push_back(sampler_item->second); } } - if (samplers_symbols.find(input) != samplers_symbols.end()) { - output += samplers_symbols[input]; + return sampler_types; +} + +std::vector sampler_types_from_chars(const std::string & names_string) { + std::unordered_map sampler_name_map { + {'k', llama_sampler_type::TOP_K}, + {'p', llama_sampler_type::TOP_P}, + {'y', llama_sampler_type::TYPICAL_P}, + {'m', llama_sampler_type::MIN_P}, + {'f', llama_sampler_type::TFS_Z}, + {'t', llama_sampler_type::TEMP} + }; + + std::vector sampler_types; + sampler_types.reserve(names_string.size()); + for (const auto & c : names_string) { + const auto sampler_item = sampler_name_map.find(c); + if (sampler_item != sampler_name_map.end()) { + sampler_types.push_back(sampler_item->second); + } + } + return sampler_types; +} + +std::string sampler_type_to_name_string(llama_sampler_type sampler_type) { + switch (sampler_type) { + case llama_sampler_type::TOP_K: return "top_k"; + case llama_sampler_type::TFS_Z: return "tfs_z"; + case llama_sampler_type::TYPICAL_P: return "typical_p"; + case llama_sampler_type::TOP_P: return "top_p"; + case llama_sampler_type::MIN_P: return "min_p"; + case llama_sampler_type::TEMP: return "temp"; + default : return ""; } - return output; } // diff --git a/common/common.h b/common/common.h index 62de25d6a..9bdd45cf9 100644 --- a/common/common.h +++ b/common/common.h @@ -162,10 +162,13 @@ std::string gpt_random_prompt(std::mt19937 & rng); void process_escapes(std::string& input); // -// String parsing +// String utils // -std::string parse_samplers_input(std::string input); +std::vector sampler_types_from_names(const std::vector & names); +std::vector sampler_types_from_chars(const std::string & names_string); +std::vector string_split(std::string input, char separator); +std::string sampler_type_to_name_string(llama_sampler_type sampler_type); // // Model utils diff --git a/common/sampling.cpp b/common/sampling.cpp index 82cbdecea..a001750da 100644 --- a/common/sampling.cpp +++ b/common/sampling.cpp @@ -103,15 +103,10 @@ std::string llama_sampling_print(const llama_sampling_params & params) { std::string llama_sampling_order_print(const llama_sampling_params & params) { std::string result = "CFG -> Penalties "; if (params.mirostat == 0) { - for (auto s : params.samplers_sequence) { - switch (s) { - case 'k': result += "-> top_k "; break; - case 'f': result += "-> tfs_z "; break; - case 'y': result += "-> typical_p "; break; - case 'p': result += "-> top_p "; break; - case 'm': result += "-> min_p "; break; - case 't': result += "-> temp "; break; - default : break; + for (auto sampler_type : params.samplers_sequence) { + const auto sampler_type_name = sampler_type_to_name_string(sampler_type); + if (!sampler_type_name.empty()) { + result += "-> " + sampler_type_name + " "; } } } else { @@ -135,16 +130,16 @@ static void sampler_queue( const float min_p = params.min_p; const float tfs_z = params.tfs_z; const float typical_p = params.typical_p; - const std::string & samplers_sequence = params.samplers_sequence; + const std::vector & samplers_sequence = params.samplers_sequence; - for (auto s : samplers_sequence) { - switch (s){ - case 'k': llama_sample_top_k (ctx_main, &cur_p, top_k, min_keep); break; - case 'f': llama_sample_tail_free(ctx_main, &cur_p, tfs_z, min_keep); break; - case 'y': llama_sample_typical (ctx_main, &cur_p, typical_p, min_keep); break; - case 'p': llama_sample_top_p (ctx_main, &cur_p, top_p, min_keep); break; - case 'm': llama_sample_min_p (ctx_main, &cur_p, min_p, min_keep); break; - case 't': + for (auto sampler_type : samplers_sequence) { + switch (sampler_type) { + case llama_sampler_type::TOP_K : llama_sample_top_k (ctx_main, &cur_p, top_k, min_keep); break; + case llama_sampler_type::TFS_Z : llama_sample_tail_free(ctx_main, &cur_p, tfs_z, min_keep); break; + case llama_sampler_type::TYPICAL_P: llama_sample_typical (ctx_main, &cur_p, typical_p, min_keep); break; + case llama_sampler_type::TOP_P : llama_sample_top_p (ctx_main, &cur_p, top_p, min_keep); break; + case llama_sampler_type::MIN_P : llama_sample_min_p (ctx_main, &cur_p, min_p, min_keep); break; + case llama_sampler_type::TEMP: if (dynatemp_range > 0) { float dynatemp_min = std::max(0.0f, temp - dynatemp_range); float dynatemp_max = std::max(0.0f, temp + dynatemp_range); diff --git a/common/sampling.h b/common/sampling.h index 88899c094..2bd6a75d2 100644 --- a/common/sampling.h +++ b/common/sampling.h @@ -8,6 +8,16 @@ #include #include +// sampler types +enum class llama_sampler_type : char { + TOP_K = 'k', + TOP_P = 'p', + MIN_P = 'm', + TFS_Z = 'f', + TYPICAL_P = 'y', + TEMP = 't' +}; + // sampling parameters typedef struct llama_sampling_params { int32_t n_prev = 64; // number of previous tokens to remember @@ -28,7 +38,15 @@ typedef struct llama_sampling_params { float mirostat_tau = 5.00f; // target entropy float mirostat_eta = 0.10f; // learning rate bool penalize_nl = true; // consider newlines as a repeatable token - std::string samplers_sequence = "kfypmt"; // top_k, tail_free, typical_p, top_p, min_p, temp + + std::vector samplers_sequence = { + llama_sampler_type::TOP_K, + llama_sampler_type::TFS_Z, + llama_sampler_type::TYPICAL_P, + llama_sampler_type::TOP_P, + llama_sampler_type::MIN_P, + llama_sampler_type::TEMP + }; std::string grammar; // optional BNF-like grammar to constrain sampling From c88c74f967028ae3d5ebade40ae586d20a961abc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20L=C3=B3pez?= Date: Sun, 11 Feb 2024 15:12:00 +0100 Subject: [PATCH 321/812] vulkan: only use M-sized matmul on Apple GPUs (#5412) * vulkan: refactor guess_matmul_pipeline for vendor Refactor ggml_vk_guess_matmul_pipeline to simplify adding per-vendor conditionals. Signed-off-by: Sergio Lopez * vulkan: only use M-sized matmul on Apple GPUs L-sized and S-sized matmuls are broken on Apple GPUs, force using M-size with this vendor. Signed-off-by: Sergio Lopez --------- Signed-off-by: Sergio Lopez --- ggml-vulkan.cpp | 103 +++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 93 insertions(+), 10 deletions(-) diff --git a/ggml-vulkan.cpp b/ggml-vulkan.cpp index 254f648a6..7834e635c 100644 --- a/ggml-vulkan.cpp +++ b/ggml-vulkan.cpp @@ -27,6 +27,7 @@ #define CEIL_DIV(M, N) (((M) + (N)-1) / (N)) #define VK_VENDOR_ID_AMD 0x1002 +#define VK_VENDOR_ID_APPLE 0x106b #define VK_VENDOR_ID_INTEL 0x8086 #define VK_VENDOR_ID_NVIDIA 0x10de @@ -2034,18 +2035,100 @@ static uint32_t ggml_vk_guess_matmul_pipeline_align(ggml_backend_vk_context * ct return ctx->pipeline_matmul_f32_aligned_l.align; } -static vk_pipeline* ggml_vk_guess_matmul_pipeline(ggml_backend_vk_context * ctx, bool bit16_x, bool bit16_y, int m, int n, bool aligned) { -#ifdef GGML_VULKAN_DEBUG - std::cerr << "ggml_vk_guess_matmul_pipeline(" << bit16_x << ", " << bit16_y << ", " << m << ", " << n << ", " << aligned << ")"; -#endif +static vk_pipeline* ggml_vk_guess_matmul_pipeline_amd(ggml_backend_vk_context * ctx, bool bit16_x, bool bit16_y, int m, int n, bool aligned) { if (bit16_x && bit16_y) { - if (ctx->device.lock()->vendor_id == VK_VENDOR_ID_INTEL || m <= 32 || n <= 32) { + if (m <= 32 || n <= 32) { #ifdef GGML_VULKAN_DEBUG std::cerr << " S" << std::endl; #endif return aligned ? &ctx->pipeline_matmul_f16_aligned_s : &ctx->pipeline_matmul_f16_s; } - if (ctx->device.lock()->subgroup_size == 64 || m <= 64 || n <= 64) { +#ifdef GGML_VULKAN_DEBUG + std::cerr << " M" << std::endl; +#endif + return aligned ? &ctx->pipeline_matmul_f16_aligned_m : &ctx->pipeline_matmul_f16_m; + } + if (bit16_x && !bit16_y) { + if (m <= 32 || n <= 32) { +#ifdef GGML_VULKAN_DEBUG + std::cerr << " S" << std::endl; +#endif + return aligned ? &ctx->pipeline_matmul_f16_f32_aligned_s : &ctx->pipeline_matmul_f16_f32_s; + } +#ifdef GGML_VULKAN_DEBUG + std::cerr << " M" << std::endl; +#endif + return aligned ? &ctx->pipeline_matmul_f16_f32_aligned_m : &ctx->pipeline_matmul_f16_f32_m; + } + if (!bit16_x && bit16_y) { + GGML_ASSERT(false); + } + + if (m <= 32 || n <= 32) { +#ifdef GGML_VULKAN_DEBUG + std::cerr << " S" << std::endl; +#endif + return aligned ? &ctx->pipeline_matmul_f32_aligned_s : &ctx->pipeline_matmul_f32_s; + } +#ifdef GGML_VULKAN_DEBUG + std::cerr << " M" << std::endl; +#endif + return aligned ? &ctx->pipeline_matmul_f32_aligned_m : &ctx->pipeline_matmul_f32_m; +} + +static vk_pipeline* ggml_vk_guess_matmul_pipeline_apple(ggml_backend_vk_context * ctx, bool bit16_x, bool bit16_y, bool aligned) { +#ifdef GGML_VULKAN_DEBUG + std::cerr << " M" << std::endl; +#endif + if (bit16_x && bit16_y) { + return aligned ? &ctx->pipeline_matmul_f16_aligned_m : &ctx->pipeline_matmul_f16_m; + } + if (bit16_x && !bit16_y) { + return aligned ? &ctx->pipeline_matmul_f16_f32_aligned_m : &ctx->pipeline_matmul_f16_f32_m; + } + if (!bit16_x && bit16_y) { + GGML_ASSERT(false); + } + return aligned ? &ctx->pipeline_matmul_f32_aligned_m : &ctx->pipeline_matmul_f32_m; +} + +static vk_pipeline* ggml_vk_guess_matmul_pipeline_intel(ggml_backend_vk_context * ctx, bool bit16_x, bool bit16_y, bool aligned) { +#ifdef GGML_VULKAN_DEBUG + std::cerr << " S" << std::endl; +#endif + if (bit16_x && bit16_y) { + return aligned ? &ctx->pipeline_matmul_f16_aligned_s : &ctx->pipeline_matmul_f16_s; + } + if (bit16_x && !bit16_y) { + return aligned ? &ctx->pipeline_matmul_f16_f32_aligned_s : &ctx->pipeline_matmul_f16_f32_s; + } + if (!bit16_x && bit16_y) { + GGML_ASSERT(false); + } + return aligned ? &ctx->pipeline_matmul_f32_aligned_s : &ctx->pipeline_matmul_f32_s; +} + +static vk_pipeline* ggml_vk_guess_matmul_pipeline(ggml_backend_vk_context * ctx, bool bit16_x, bool bit16_y, int m, int n, bool aligned) { +#ifdef GGML_VULKAN_DEBUG + std::cerr << "ggml_vk_guess_matmul_pipeline(" << bit16_x << ", " << bit16_y << ", " << m << ", " << n << ", " << aligned << ")"; +#endif + switch (ctx->device.lock()->vendor_id) { + case VK_VENDOR_ID_AMD: + return ggml_vk_guess_matmul_pipeline_amd(ctx, bit16_x, bit16_y, m, n, aligned); + case VK_VENDOR_ID_APPLE: + return ggml_vk_guess_matmul_pipeline_apple(ctx, bit16_x, bit16_y, aligned); + case VK_VENDOR_ID_INTEL: + return ggml_vk_guess_matmul_pipeline_intel(ctx, bit16_x, bit16_y, aligned); + } + + if (bit16_x && bit16_y) { + if (m <= 32 || n <= 32) { +#ifdef GGML_VULKAN_DEBUG + std::cerr << " S" << std::endl; +#endif + return aligned ? &ctx->pipeline_matmul_f16_aligned_s : &ctx->pipeline_matmul_f16_s; + } + if (m <= 64 || n <= 64) { #ifdef GGML_VULKAN_DEBUG std::cerr << " M" << std::endl; #endif @@ -2057,13 +2140,13 @@ static vk_pipeline* ggml_vk_guess_matmul_pipeline(ggml_backend_vk_context * ctx, return aligned ? &ctx->pipeline_matmul_f16_aligned_l : &ctx->pipeline_matmul_f16_l; } if (bit16_x && !bit16_y) { - if (ctx->device.lock()->vendor_id == VK_VENDOR_ID_INTEL || m <= 32 || n <= 32) { + if (m <= 32 || n <= 32) { #ifdef GGML_VULKAN_DEBUG std::cerr << " S" << std::endl; #endif return aligned ? &ctx->pipeline_matmul_f16_f32_aligned_s : &ctx->pipeline_matmul_f16_f32_s; } - if (ctx->device.lock()->subgroup_size == 64 || m <= 64 || n <= 64) { + if (m <= 64 || n <= 64) { #ifdef GGML_VULKAN_DEBUG std::cerr << " M" << std::endl; #endif @@ -2078,13 +2161,13 @@ static vk_pipeline* ggml_vk_guess_matmul_pipeline(ggml_backend_vk_context * ctx, GGML_ASSERT(false); } - if (ctx->device.lock()->vendor_id == VK_VENDOR_ID_INTEL || m <= 32 || n <= 32) { + if (m <= 32 || n <= 32) { #ifdef GGML_VULKAN_DEBUG std::cerr << " S" << std::endl; #endif return aligned ? &ctx->pipeline_matmul_f32_aligned_s : &ctx->pipeline_matmul_f32_s; } - if (ctx->device.lock()->subgroup_size == 64 || m <= 64 || n <= 64) { + if (m <= 64 || n <= 64) { #ifdef GGML_VULKAN_DEBUG std::cerr << " M" << std::endl; #endif From 97a336507ed9b971d72262bec7e2b8b7016a054a Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Sun, 11 Feb 2024 00:17:31 +0000 Subject: [PATCH 322/812] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/b8b232ae7b8b144397fdb12d20f592e5e7c1a64d' (2024-01-31) → 'github:NixOS/nixpkgs/f8e2ebd66d097614d51a56a755450d4ae1632df1' (2024-02-07) --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index 8cfc78273..239d0686c 100644 --- a/flake.lock +++ b/flake.lock @@ -20,11 +20,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1706732774, - "narHash": "sha256-hqJlyJk4MRpcItGYMF+3uHe8HvxNETWvlGtLuVpqLU0=", + "lastModified": 1707268954, + "narHash": "sha256-2en1kvde3cJVc3ZnTy8QeD2oKcseLFjYPLKhIGDanQ0=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "b8b232ae7b8b144397fdb12d20f592e5e7c1a64d", + "rev": "f8e2ebd66d097614d51a56a755450d4ae1632df1", "type": "github" }, "original": { From 2891c8aa9af17f4ff636ff3868bc34ff72b56e25 Mon Sep 17 00:00:00 2001 From: Douglas Hanley Date: Sun, 11 Feb 2024 10:21:38 -0600 Subject: [PATCH 323/812] Add support for BERT embedding models (#5423) * BERT model graph construction (build_bert) * WordPiece tokenizer (llm_tokenize_wpm) * Add flag for non-causal attention models * Allow for models that only output embeddings * Support conversion of BERT models to GGUF * Based on prior work by @xyzhang626 and @skeskinen --------- Co-authored-by: Jared Van Bortel Co-authored-by: Jared Van Bortel Co-authored-by: Georgi Gerganov --- .flake8 | 1 + convert-hf-to-gguf.py | 94 ++++++ examples/embedding/embedding.cpp | 12 +- gguf-py/gguf/constants.py | 43 +-- gguf-py/gguf/gguf_writer.py | 6 + gguf-py/gguf/tensor_mapping.py | 13 +- llama.cpp | 498 +++++++++++++++++++++++++++++-- llama.h | 1 + 8 files changed, 616 insertions(+), 52 deletions(-) diff --git a/.flake8 b/.flake8 index 113ca5fd3..18fba2c15 100644 --- a/.flake8 +++ b/.flake8 @@ -1,2 +1,3 @@ [flake8] max-line-length = 125 +ignore = W503 diff --git a/convert-hf-to-gguf.py b/convert-hf-to-gguf.py index 0d4ea03b4..cae1551a2 100755 --- a/convert-hf-to-gguf.py +++ b/convert-hf-to-gguf.py @@ -209,6 +209,8 @@ class Model: return InternLM2Model if model_architecture == "MiniCPMForCausalLM": return MiniCPMModel + if model_architecture == "BertModel": + return BertModel return Model def _is_model_safetensors(self) -> bool: @@ -264,6 +266,8 @@ class Model: return gguf.MODEL_ARCH.INTERNLM2 if arch == "MiniCPMForCausalLM": return gguf.MODEL_ARCH.MINICPM + if arch == "BertModel": + return gguf.MODEL_ARCH.BERT raise NotImplementedError(f'Architecture "{arch}" not supported!') @@ -1629,6 +1633,96 @@ in chat mode so that the conversation can end normally.") self.post_write_tensors(tensor_map, name, data_torch) +class BertModel(Model): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.block_count = self.hparams["num_hidden_layers"] + + def set_gguf_parameters(self): + # TODO(cebtenzzre): merge with parent class + self.gguf_writer.add_name(self.dir_model.name) + self.gguf_writer.add_context_length(self.hparams["max_position_embeddings"]) + self.gguf_writer.add_embedding_length(self.hparams["hidden_size"]) + self.gguf_writer.add_feed_forward_length(self.hparams["intermediate_size"]) + self.gguf_writer.add_block_count(self.block_count) + self.gguf_writer.add_head_count(self.hparams["num_attention_heads"]) + self.gguf_writer.add_layer_norm_eps(self.hparams["layer_norm_eps"]) + self.gguf_writer.add_causal_attention(False) + self.gguf_writer.add_file_type(self.ftype) + + def set_vocab(self): + path = self.dir_model + added_tokens_path = self.dir_model if self.dir_model.exists() else None + + # use huggingface vocab to get all tokens + vocab = HfVocab(path, added_tokens_path) + tokens, scores, toktypes = zip(*vocab.all_tokens()) + assert len(tokens) == vocab.vocab_size + + # we need this to validate the size of the token_type embeddings + # though currently we are passing all zeros to the token_type embeddings + n_token_types = len(set(toktypes)) + self.gguf_writer.add_token_type_count(n_token_types) + + # convert to phantom space vocab + def phantom(tok, typ): + if tok.startswith(b"[") and tok.endswith(b"]"): + return tok + if tok.startswith(b"##"): + return tok[2:] + return b"\xe2\x96\x81" + tok + tokens = [phantom(t, y) for t, y in zip(tokens, toktypes)] + + # set up bos and eos tokens (cls and sep) + self.gguf_writer.add_bos_token_id(vocab.tokenizer.cls_token_id) + self.gguf_writer.add_eos_token_id(vocab.tokenizer.sep_token_id) + + # add vocab to gguf + self.gguf_writer.add_tokenizer_model("bert") + self.gguf_writer.add_token_list(tokens) + self.gguf_writer.add_token_scores(scores) + self.gguf_writer.add_token_types(toktypes) + + # handle special tokens + special_vocab = gguf.SpecialVocab(self.dir_model, n_vocab=len(tokens)) + special_vocab.add_to_gguf(self.gguf_writer) + + def write_tensors(self): + tensor_map = gguf.get_tensor_name_map(self.model_arch, self.block_count) + tensors = dict(self.get_tensors()) + for name, data_torch in tensors.items(): + # we are only using BERT for embeddings so we don't need the pooling layer + if name in ("embeddings.position_ids", "pooler.dense.weight", "pooler.dense.bias"): + continue # we don't need these + + # map tensor names + new_name = tensor_map.get_name(name, try_suffixes=(".weight", ".bias")) + if new_name is None: + print(f"Can not map tensor {name!r}") + sys.exit() + + data = data_torch.squeeze().numpy() + n_dims = len(data.shape) + new_dtype: type[np.floating[Any]] + + if ( + self.ftype == 1 and name.endswith(".weight") and n_dims == 2 + and name != "embeddings.token_type_embeddings.weight" # not used with get_rows, must be F32 + ): + # if f16 desired, convert any float32 2-dim weight tensors to float16 + new_dtype = np.float16 + else: + # if f32 desired, convert any float16 to float32 + new_dtype = np.float32 + + print(f"{new_name}, n_dims = {n_dims}, {data_torch.dtype} --> {new_dtype}") + + if data.dtype != new_dtype: + data = data.astype(new_dtype) + + self.gguf_writer.add_tensor(new_name, data) + + ###### CONVERSION LOGIC ###### diff --git a/examples/embedding/embedding.cpp b/examples/embedding/embedding.cpp index 3295cd240..27376c8f0 100644 --- a/examples/embedding/embedding.cpp +++ b/examples/embedding/embedding.cpp @@ -87,7 +87,17 @@ int main(int argc, char ** argv) { } const int n_embd = llama_n_embd(model); - const auto * embeddings = llama_get_embeddings(ctx); + auto * embeddings = llama_get_embeddings(ctx); + + // l2-normalize embeddings + float norm = 0; + for (int i = 0; i < n_embd; i++) { + norm += embeddings[i] * embeddings[i]; + } + norm = sqrt(norm); + for (int i = 0; i < n_embd; i++) { + embeddings[i] /= norm; + } for (int i = 0; i < n_embd; i++) { printf("%f ", embeddings[i]); diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py index 1cfd41c0b..a9c13dd38 100644 --- a/gguf-py/gguf/constants.py +++ b/gguf-py/gguf/constants.py @@ -50,6 +50,7 @@ class Keys: VALUE_LENGTH = "{arch}.attention.value_length" LAYERNORM_EPS = "{arch}.attention.layer_norm_epsilon" LAYERNORM_RMS_EPS = "{arch}.attention.layer_norm_rms_epsilon" + CAUSAL = "{arch}.attention.causal" class Rope: DIMENSION_COUNT = "{arch}.rope.dimension_count" @@ -60,22 +61,23 @@ class Keys: SCALING_FINETUNED = "{arch}.rope.scaling.finetuned" class Tokenizer: - MODEL = "tokenizer.ggml.model" - LIST = "tokenizer.ggml.tokens" - TOKEN_TYPE = "tokenizer.ggml.token_type" - SCORES = "tokenizer.ggml.scores" - MERGES = "tokenizer.ggml.merges" - BOS_ID = "tokenizer.ggml.bos_token_id" - EOS_ID = "tokenizer.ggml.eos_token_id" - UNK_ID = "tokenizer.ggml.unknown_token_id" - SEP_ID = "tokenizer.ggml.seperator_token_id" - PAD_ID = "tokenizer.ggml.padding_token_id" - ADD_BOS = "tokenizer.ggml.add_bos_token" - ADD_EOS = "tokenizer.ggml.add_eos_token" - ADD_PREFIX = "tokenizer.ggml.add_space_prefix" - HF_JSON = "tokenizer.huggingface.json" - RWKV = "tokenizer.rwkv.world" - CHAT_TEMPLATE = "tokenizer.chat_template" + MODEL = "tokenizer.ggml.model" + LIST = "tokenizer.ggml.tokens" + TOKEN_TYPE = "tokenizer.ggml.token_type" + TOKEN_TYPE_COUNT = "tokenizer.ggml.token_type_count" # for BERT-style token types + SCORES = "tokenizer.ggml.scores" + MERGES = "tokenizer.ggml.merges" + BOS_ID = "tokenizer.ggml.bos_token_id" + EOS_ID = "tokenizer.ggml.eos_token_id" + UNK_ID = "tokenizer.ggml.unknown_token_id" + SEP_ID = "tokenizer.ggml.seperator_token_id" + PAD_ID = "tokenizer.ggml.padding_token_id" + ADD_BOS = "tokenizer.ggml.add_bos_token" + ADD_EOS = "tokenizer.ggml.add_eos_token" + ADD_PREFIX = "tokenizer.ggml.add_space_prefix" + HF_JSON = "tokenizer.huggingface.json" + RWKV = "tokenizer.rwkv.world" + CHAT_TEMPLATE = "tokenizer.chat_template" # @@ -122,6 +124,7 @@ class MODEL_TENSOR(IntEnum): ATTN_OUT = auto() ATTN_NORM = auto() ATTN_NORM_2 = auto() + ATTN_OUT_NORM = auto() ATTN_ROT_EMBD = auto() FFN_GATE_INP = auto() FFN_NORM = auto() @@ -134,6 +137,7 @@ class MODEL_TENSOR(IntEnum): FFN_UP_EXP = auto() ATTN_Q_NORM = auto() ATTN_K_NORM = auto() + LAYER_OUT_NORM = auto() MODEL_ARCH_NAMES: dict[MODEL_ARCH, str] = { @@ -178,6 +182,7 @@ TENSOR_NAMES: dict[MODEL_TENSOR, str] = { MODEL_TENSOR.ATTN_ROT_EMBD: "blk.{bid}.attn_rot_embd", MODEL_TENSOR.ATTN_Q_NORM: "blk.{bid}.attn_q_norm", MODEL_TENSOR.ATTN_K_NORM: "blk.{bid}.attn_k_norm", + MODEL_TENSOR.ATTN_OUT_NORM: "blk.{bid}.attn_output_norm", MODEL_TENSOR.FFN_GATE_INP: "blk.{bid}.ffn_gate_inp", MODEL_TENSOR.FFN_NORM: "blk.{bid}.ffn_norm", MODEL_TENSOR.FFN_GATE: "blk.{bid}.ffn_gate", @@ -187,6 +192,7 @@ TENSOR_NAMES: dict[MODEL_TENSOR, str] = { MODEL_TENSOR.FFN_GATE_EXP: "blk.{bid}.ffn_gate.{xid}", MODEL_TENSOR.FFN_DOWN_EXP: "blk.{bid}.ffn_down.{xid}", MODEL_TENSOR.FFN_UP_EXP: "blk.{bid}.ffn_up.{xid}", + MODEL_TENSOR.LAYER_OUT_NORM: "blk.{bid}.layer_output_norm", } MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = { @@ -262,17 +268,18 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = { ], MODEL_ARCH.BERT: [ MODEL_TENSOR.TOKEN_EMBD, + MODEL_TENSOR.TOKEN_EMBD_NORM, MODEL_TENSOR.TOKEN_TYPES, MODEL_TENSOR.POS_EMBD, MODEL_TENSOR.OUTPUT_NORM, - MODEL_TENSOR.ATTN_NORM, + MODEL_TENSOR.ATTN_OUT_NORM, MODEL_TENSOR.ATTN_Q, MODEL_TENSOR.ATTN_K, MODEL_TENSOR.ATTN_V, MODEL_TENSOR.ATTN_OUT, - MODEL_TENSOR.FFN_NORM, MODEL_TENSOR.FFN_DOWN, MODEL_TENSOR.FFN_UP, + MODEL_TENSOR.LAYER_OUT_NORM, ], MODEL_ARCH.MPT: [ MODEL_TENSOR.TOKEN_EMBD, diff --git a/gguf-py/gguf/gguf_writer.py b/gguf-py/gguf/gguf_writer.py index 16808196e..7af58a46c 100644 --- a/gguf-py/gguf/gguf_writer.py +++ b/gguf-py/gguf/gguf_writer.py @@ -357,6 +357,9 @@ class GGUFWriter: def add_layer_norm_rms_eps(self, value: float) -> None: self.add_float32(Keys.Attention.LAYERNORM_RMS_EPS.format(arch=self.arch), value) + def add_causal_attention(self, value: bool) -> None: + self.add_bool(Keys.Attention.CAUSAL.format(arch=self.arch), value) + def add_rope_dimension_count(self, count: int) -> None: self.add_uint32(Keys.Rope.DIMENSION_COUNT.format(arch=self.arch), count) @@ -387,6 +390,9 @@ class GGUFWriter: def add_token_types(self, types: Sequence[TokenType] | Sequence[int]) -> None: self.add_array(Keys.Tokenizer.TOKEN_TYPE, types) + def add_token_type_count(self, value: int) -> None: + self.add_uint32(Keys.Tokenizer.TOKEN_TYPE_COUNT, value) + def add_token_scores(self, scores: Sequence[float]) -> None: self.add_array(Keys.Tokenizer.SCORES, scores) diff --git a/gguf-py/gguf/tensor_mapping.py b/gguf-py/gguf/tensor_mapping.py index 4f16d8504..c7ba1420e 100644 --- a/gguf-py/gguf/tensor_mapping.py +++ b/gguf-py/gguf/tensor_mapping.py @@ -30,6 +30,7 @@ class TensorNameMap: # Normalization of token embeddings MODEL_TENSOR.TOKEN_EMBD_NORM: ( "word_embeddings_layernorm", # bloom + "embeddings.LayerNorm", # bert ), # Position embeddings @@ -54,7 +55,6 @@ class TensorNameMap: "transformer.ln_f", # gpt2 gpt-j falcon "model.norm", # llama-hf baichuan internlm2 "norm", # llama-pth - "embeddings.LayerNorm", # bert "transformer.norm_f", # mpt "ln_f", # refact bloom qwen gpt2 "language_model.encoder.final_layernorm", # persimmon @@ -79,7 +79,6 @@ class TensorNameMap: "transformer.h.{bid}.ln_mlp", # falcon40b "model.layers.{bid}.input_layernorm", # llama-hf "layers.{bid}.attention_norm", # llama-pth - "encoder.layer.{bid}.attention.output.LayerNorm", # bert "language_model.encoder.layers.{bid}.input_layernorm", # persimmon "model.layers.{bid}.ln1", # yi "h.{bid}.ln_1", # gpt2 @@ -155,6 +154,11 @@ class TensorNameMap: "model.layers.{bid}.attention.wo", # internlm2 ), + # Attention output norm + MODEL_TENSOR.ATTN_OUT_NORM: ( + "encoder.layer.{bid}.attention.output.LayerNorm", # bert + ), + # Rotary embeddings MODEL_TENSOR.ATTN_ROT_EMBD: ( "model.layers.{bid}.self_attn.rotary_emb.inv_freq", # llama-hf @@ -171,7 +175,6 @@ class TensorNameMap: "transformer.blocks.{bid}.norm_2", # mpt "model.layers.{bid}.post_attention_layernorm", # llama-hf "layers.{bid}.ffn_norm", # llama-pth - "encoder.layer.{bid}.output.LayerNorm", # bert "language_model.encoder.layers.{bid}.post_attention_layernorm", # persimmon "model.layers.{bid}.ln2", # yi "h.{bid}.ln_2", # gpt2 @@ -266,6 +269,10 @@ class TensorNameMap: MODEL_TENSOR.ROPE_FREQS: ( "language_model.encoder.layers.{bid}.self_attention.rotary_emb.inv_freq", # persimmon ), + + MODEL_TENSOR.LAYER_OUT_NORM: ( + "encoder.layer.{bid}.output.LayerNorm", # bert + ) } mapping: dict[str, tuple[MODEL_TENSOR, str]] diff --git a/llama.cpp b/llama.cpp index 3f39a67fb..d1ee26ce2 100644 --- a/llama.cpp +++ b/llama.cpp @@ -196,6 +196,7 @@ enum llm_arch { LLM_ARCH_STARCODER, LLM_ARCH_PERSIMMON, LLM_ARCH_REFACT, + LLM_ARCH_BERT, LLM_ARCH_BLOOM, LLM_ARCH_STABLELM, LLM_ARCH_QWEN, @@ -220,6 +221,7 @@ static std::map LLM_ARCH_NAMES = { { LLM_ARCH_STARCODER, "starcoder" }, { LLM_ARCH_PERSIMMON, "persimmon" }, { LLM_ARCH_REFACT, "refact" }, + { LLM_ARCH_BERT, "bert" }, { LLM_ARCH_BLOOM, "bloom" }, { LLM_ARCH_STABLELM, "stablelm" }, { LLM_ARCH_QWEN, "qwen" }, @@ -261,6 +263,7 @@ enum llm_kv { LLM_KV_ATTENTION_VALUE_LENGTH, LLM_KV_ATTENTION_LAYERNORM_EPS, LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, + LLM_KV_ATTENTION_CAUSAL, LLM_KV_ROPE_DIMENSION_COUNT, LLM_KV_ROPE_FREQ_BASE, @@ -273,6 +276,7 @@ enum llm_kv { LLM_KV_TOKENIZER_MODEL, LLM_KV_TOKENIZER_LIST, LLM_KV_TOKENIZER_TOKEN_TYPE, + LLM_KV_TOKENIZER_TOKEN_TYPE_COUNT, LLM_KV_TOKENIZER_SCORES, LLM_KV_TOKENIZER_MERGES, LLM_KV_TOKENIZER_BOS_ID, @@ -316,6 +320,7 @@ static std::map LLM_KV_NAMES = { { LLM_KV_ATTENTION_VALUE_LENGTH, "%s.attention.value_length" }, { LLM_KV_ATTENTION_LAYERNORM_EPS, "%s.attention.layer_norm_epsilon" }, { LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, "%s.attention.layer_norm_rms_epsilon" }, + { LLM_KV_ATTENTION_CAUSAL, "%s.attention.causal" }, { LLM_KV_ROPE_DIMENSION_COUNT, "%s.rope.dimension_count" }, { LLM_KV_ROPE_FREQ_BASE, "%s.rope.freq_base" }, @@ -328,6 +333,7 @@ static std::map LLM_KV_NAMES = { { LLM_KV_TOKENIZER_MODEL, "tokenizer.ggml.model" }, { LLM_KV_TOKENIZER_LIST, "tokenizer.ggml.tokens" }, { LLM_KV_TOKENIZER_TOKEN_TYPE, "tokenizer.ggml.token_type" }, + { LLM_KV_TOKENIZER_TOKEN_TYPE_COUNT, "tokenizer.ggml.token_type_count" }, { LLM_KV_TOKENIZER_SCORES, "tokenizer.ggml.scores" }, { LLM_KV_TOKENIZER_MERGES, "tokenizer.ggml.merges" }, { LLM_KV_TOKENIZER_BOS_ID, "tokenizer.ggml.bos_token_id" }, @@ -355,6 +361,7 @@ struct LLM_KV { enum llm_tensor { LLM_TENSOR_TOKEN_EMBD, LLM_TENSOR_TOKEN_EMBD_NORM, + LLM_TENSOR_TOKEN_TYPES, LLM_TENSOR_POS_EMBD, LLM_TENSOR_OUTPUT, LLM_TENSOR_OUTPUT_NORM, @@ -536,6 +543,23 @@ static std::map> LLM_TENSOR_NAMES = { LLM_TENSOR_FFN_UP, "blk.%d.ffn_up" }, }, }, + { + LLM_ARCH_BERT, + { + { LLM_TENSOR_TOKEN_EMBD, "token_embd" }, + { LLM_TENSOR_TOKEN_EMBD_NORM, "token_embd_norm" }, + { LLM_TENSOR_TOKEN_TYPES, "token_types" }, + { LLM_TENSOR_POS_EMBD, "position_embd" }, + { LLM_TENSOR_ATTN_NORM, "blk.%d.attn_output_norm" }, + { LLM_TENSOR_ATTN_Q, "blk.%d.attn_q" }, + { LLM_TENSOR_ATTN_K, "blk.%d.attn_k" }, + { LLM_TENSOR_ATTN_V, "blk.%d.attn_v" }, + { LLM_TENSOR_ATTN_OUT, "blk.%d.attn_output" }, + { LLM_TENSOR_FFN_NORM, "blk.%d.layer_output_norm" }, + { LLM_TENSOR_FFN_DOWN, "blk.%d.ffn_down" }, + { LLM_TENSOR_FFN_UP, "blk.%d.ffn_up" }, + }, + }, { LLM_ARCH_BLOOM, { @@ -1440,6 +1464,11 @@ static llama_state g_state; // available llama models enum e_model { MODEL_UNKNOWN, + MODEL_17M, + MODEL_22M, + MODEL_33M, + MODEL_109M, + MODEL_335M, MODEL_0_5B, MODEL_1B, MODEL_2B, @@ -1481,6 +1510,7 @@ struct llama_hparams { uint32_t n_ff; uint32_t n_expert = 0; uint32_t n_expert_used = 0; + uint32_t n_vocab_type = 0; // for BERT-style token types float f_norm_eps; float f_norm_rms_eps; @@ -1493,6 +1523,8 @@ struct llama_hparams { float f_clamp_kqv; float f_max_alibi_bias; + bool causal_attn = true; + bool operator!=(const llama_hparams & other) const { if (this->vocab_only != other.vocab_only) return true; @@ -1720,6 +1752,7 @@ struct llama_model { llama_vocab vocab; struct ggml_tensor * tok_embd; + struct ggml_tensor * type_embd; struct ggml_tensor * pos_embd; struct ggml_tensor * tok_norm; struct ggml_tensor * tok_norm_b; @@ -1850,6 +1883,7 @@ struct llama_context { struct ggml_tensor * inp_pos; // I32 [n_batch] struct ggml_tensor * inp_KQ_mask; // F32 [n_ctx, n_batch] struct ggml_tensor * inp_K_shift; // I32 [n_ctx] + struct ggml_tensor * inp_sum; // F32 [1, n_batch] #ifdef GGML_USE_MPI ggml_mpi_context * ctx_mpi = NULL; @@ -2829,6 +2863,7 @@ static const char * llama_model_vocab_type_name(enum llama_vocab_type type){ switch (type) { case LLAMA_VOCAB_TYPE_SPM: return "SPM"; case LLAMA_VOCAB_TYPE_BPE: return "BPE"; + case LLAMA_VOCAB_TYPE_WPM: return "WPM"; default: return "unknown"; } } @@ -3000,6 +3035,26 @@ static void llm_load_hparams( default: model.type = e_model::MODEL_UNKNOWN; } } break; + case LLM_ARCH_BERT: + { + ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps); + ml.get_key(LLM_KV_ATTENTION_CAUSAL, hparams.causal_attn); + ml.get_key(LLM_KV_TOKENIZER_TOKEN_TYPE_COUNT, hparams.n_vocab_type); + + switch (hparams.n_layer) { + case 3: + model.type = e_model::MODEL_17M; break; // bge-micro + case 6: + model.type = e_model::MODEL_22M; break; // MiniLM-L6 + case 12: + switch (hparams.n_embd) { + case 384: model.type = e_model::MODEL_33M; break; // MiniLM-L12, bge-small + case 768: model.type = e_model::MODEL_109M; break; // bge-base + } break; + case 24: + model.type = e_model::MODEL_335M; break; // bge-large + } + } break; case LLM_ARCH_BLOOM: { ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps); @@ -3204,6 +3259,16 @@ static void llm_load_vocab( vocab.special_unk_id = -1; vocab.special_sep_id = -1; vocab.special_pad_id = -1; + } else if (tokenizer_name == "bert") { + vocab.type = LLAMA_VOCAB_TYPE_WPM; + + // default special tokens + vocab.special_bos_id = 101; + vocab.special_eos_id = 102; + vocab.special_unk_id = 100; + vocab.special_sep_id = -1; + vocab.special_pad_id = -1; + vocab.add_space_prefix = false; } else { LLAMA_LOG_WARN("%s: unknown tokenizer: '%s'", __func__, tokenizer_name.c_str()); LLAMA_LOG_WARN("%s: using default tokenizer: 'llama'", __func__); @@ -3232,6 +3297,8 @@ static void llm_load_vocab( // determine the newline token: LLaMA "<0x0A>" == 10 == '\n', Falcon 193 == '\n' if (vocab.type == LLAMA_VOCAB_TYPE_SPM) { vocab.linefeed_id = llama_byte_to_token(vocab, '\n'); + } else if (vocab.type == LLAMA_VOCAB_TYPE_WPM) { + vocab.linefeed_id = vocab.special_pad_id; } else { const std::vector ids = llama_tokenize_internal(vocab, "\u010A", false); GGML_ASSERT(!ids.empty() && "model vocab missing newline token"); @@ -3569,6 +3636,7 @@ static bool llm_load_tensors( const int64_t n_embd_v_gqa = hparams.n_embd_v_gqa(); const int64_t n_embd_gqa = n_embd_v_gqa; const int64_t n_vocab = hparams.n_vocab; + const int64_t n_vocab_type = hparams.n_vocab_type; const int64_t n_ff = hparams.n_ff; GGML_ASSERT(n_embd_gqa == n_embd_k_gqa); @@ -3783,11 +3851,50 @@ static bool llm_load_tensors( layer.attn_k_norm_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_K_NORM, "bias", i), {64}); } } break; - case LLM_ARCH_BLOOM: + case LLM_ARCH_BERT: { model.tok_embd = ml.create_tensor(ctx_input, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}); - model.tok_norm = ml.create_tensor(ctx_input, tn(LLM_TENSOR_TOKEN_EMBD_NORM, "weight"), {n_embd}); - model.tok_norm_b = ml.create_tensor(ctx_input, tn(LLM_TENSOR_TOKEN_EMBD_NORM, "bias"), {n_embd}); + model.type_embd = ml.create_tensor(ctx_input, tn(LLM_TENSOR_TOKEN_TYPES, "weight"), {n_embd, n_vocab_type}); + model.pos_embd = ml.create_tensor(ctx_input, tn(LLM_TENSOR_POS_EMBD, "weight"), {n_embd, hparams.n_ctx_train}); + model.tok_norm = ml.create_tensor(ctx_output, tn(LLM_TENSOR_TOKEN_EMBD_NORM, "weight"), {n_embd}); + model.tok_norm_b = ml.create_tensor(ctx_output, tn(LLM_TENSOR_TOKEN_EMBD_NORM, "bias"), {n_embd}); + + for (int i = 0; i < n_layer; ++i) { + ggml_context * ctx_layer = ctx_for_layer(i); + ggml_context * ctx_split = ctx_for_layer_split(i); + + auto & layer = model.layers[i]; + + layer.attn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}); + layer.attn_norm_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM, "bias", i), {n_embd}); + + layer.ffn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}); + layer.ffn_norm_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_NORM, "bias", i), {n_embd}); + + layer.wq = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_embd}); + layer.bq = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_Q, "bias", i), {n_embd}); + + layer.wk = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_K, "weight", i), {n_embd, n_embd_gqa}); + layer.bk = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_K, "bias", i), {n_embd_gqa}); + + layer.wv = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_V, "weight", i), {n_embd, n_embd_gqa}); + layer.bv = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_V, "bias", i), {n_embd_gqa}); + + layer.wo = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd}); + layer.bo = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_OUT, "bias", i), {n_embd}); + + layer.ffn_up = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff}); + layer.ffn_up_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_UP, "bias", i), {n_ff}); + + layer.ffn_down = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_DOWN, "weight", i), {n_ff, n_embd}); + layer.ffn_down_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_DOWN, "bias", i), {n_embd}); + } + } break; + case LLM_ARCH_BLOOM: + { + model.tok_embd = ml.create_tensor(ctx_input, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}); + model.tok_norm = ml.create_tensor(ctx_output, tn(LLM_TENSOR_TOKEN_EMBD_NORM, "weight"), {n_embd}); + model.tok_norm_b = ml.create_tensor(ctx_output, tn(LLM_TENSOR_TOKEN_EMBD_NORM, "bias"), {n_embd}); // output { @@ -4739,6 +4846,7 @@ struct llm_build_context { const int32_t n_orig_ctx; const bool do_rope_shift; + const bool causal_attn; const llm_build_cb & cb; @@ -4782,6 +4890,7 @@ struct llm_build_context { kv_head (worst_case ? n_ctx - n_tokens : kv_self.head), n_orig_ctx (cparams.n_yarn_orig_ctx), do_rope_shift (worst_case || kv_self.has_shift), + causal_attn (hparams.causal_attn), cb (cb), buf_compute_meta (lctx.buf_compute_meta) { // all initializations should be done in init() @@ -5625,6 +5734,100 @@ struct llm_build_context { return gf; } + struct ggml_cgraph * build_bert() { + struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, LLAMA_MAX_NODES, false); + + const int64_t n_embd_head = hparams.n_embd_head_v; + GGML_ASSERT(n_embd_head == hparams.n_embd_head_k); + GGML_ASSERT(n_embd_head == hparams.n_rot); + + struct ggml_tensor * cur; + struct ggml_tensor * inpL; + + // get input vectors with right size + struct ggml_tensor * inp_pos = ggml_view_1d(ctx0, lctx.inp_pos, n_tokens, 0); + struct ggml_tensor * inp_sum = ggml_view_1d(ctx0, lctx.inp_sum, n_tokens, 0); + + // construct input embeddings (token, type, position) + inpL = llm_build_inp_embd(ctx0, hparams, batch, model.tok_embd, lctx.inp_tokens, lctx.inp_embd, cb); + // token types are hardcoded to zero ("Sentence A") + struct ggml_tensor * type_row0 = ggml_view_1d(ctx0, model.type_embd, n_embd, 0); + inpL = ggml_add(ctx0, inpL, type_row0); + inpL = ggml_add(ctx0, ggml_get_rows(ctx0, model.pos_embd, inp_pos), inpL); + cb(inpL, "inp_embd", -1); + + // embed layer norm + inpL = llm_build_norm(ctx0, inpL, hparams, model.tok_norm, model.tok_norm_b, LLM_NORM, cb, -1); + cb(inpL, "inp_norm", -1); + + // KQ_mask (mask for 1 head, it will be broadcasted to all heads) + struct ggml_tensor * KQ_mask = ggml_view_2d(ctx0, lctx.inp_KQ_mask, n_kv, n_tokens, n_kv*ggml_type_size(lctx.inp_KQ_mask->type), 0); + cb(KQ_mask, "KQ_mask", -1); // [n_kv, n_tokens] + + // iterate layers + for (int il = 0; il < n_layer; ++il) { + struct ggml_tensor * cur = inpL; + + // self-attention + { + struct ggml_tensor * Qcur = ggml_add(ctx0, ggml_mul_mat(ctx0, model.layers[il].wq, cur), model.layers[il].bq); + cb(Qcur, "Qcur", il); + + struct ggml_tensor * Kcur = ggml_add(ctx0, ggml_mul_mat(ctx0, model.layers[il].wk, cur), model.layers[il].bk); + cb(Kcur, "Kcur", il); + + struct ggml_tensor * Vcur = ggml_add(ctx0, ggml_mul_mat(ctx0, model.layers[il].wv, cur), model.layers[il].bv); + cb(Vcur, "Vcur", il); + + // seems like we just need to do this for Q? + Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); + + cur = llm_build_kv(ctx0, model, hparams, kv_self, gf, + model.layers[il].wo, model.layers[il].bo, + Kcur, Vcur, Qcur, KQ_mask, n_ctx, n_tokens, kv_head, n_kv, -1.0f, 1.0f/sqrtf(float(n_embd_head)), cb, il); + cb(cur, "kqv_out", il); + } + + // re-add the layer input + cur = ggml_add(ctx0, cur, inpL); + + // attention layer norm + cur = llm_build_norm(ctx0, cur, hparams, model.layers[il].attn_norm, model.layers[il].attn_norm_b, LLM_NORM, cb, il); + + struct ggml_tensor * ffn_inp = cur; + cb(ffn_inp, "ffn_inp", il); + + // feed-forward network + cur = llm_build_ffn(ctx0, cur, + model.layers[il].ffn_up, model.layers[il].ffn_up_b, + NULL, NULL, + model.layers[il].ffn_down, model.layers[il].ffn_down_b, + NULL, + LLM_FFN_GELU, LLM_FFN_SEQ, cb, il); + cb(cur, "ffn_out", il); + + // attentions bypass the intermediate layer + cur = ggml_add(ctx0, cur, ffn_inp); + + // output layer norm + cur = llm_build_norm(ctx0, cur, hparams, model.layers[il].ffn_norm, model.layers[il].ffn_norm_b, LLM_NORM, cb, il); + + // input for next layer + inpL = cur; + } + + // final output + cur = inpL; + + // pooling + cur = ggml_mul_mat(ctx0, inp_sum, ggml_cont(ctx0, ggml_transpose(ctx0, cur))); + cb(cur, "result_embed", -1); + + ggml_build_forward_expand(gf, cur); + + return gf; + } + struct ggml_cgraph * build_bloom() { struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, LLAMA_MAX_NODES, false); @@ -7060,7 +7263,8 @@ static struct ggml_cgraph * llama_build_graph( for (int i = 0; i < n_kv; ++i) { float f; - if (!lctx.kv_self.cells[i].has_seq_id(seq_id) || lctx.kv_self.cells[i].pos > pos) { + if (!lctx.kv_self.cells[i].has_seq_id(seq_id) || + (llm.causal_attn && lctx.kv_self.cells[i].pos > pos)) { f = -INFINITY; } else { f = 0; @@ -7081,6 +7285,15 @@ static struct ggml_cgraph * llama_build_graph( data[i] = lctx.kv_self.cells[i].delta; } } + + { + GGML_ASSERT(ggml_backend_buffer_is_host(lctx.inp_sum->buffer)); + float * data = (float *) lctx.inp_sum->data; + + for (int i = 0; i < batch.n_tokens; ++i) { + data[i] = 1.0f/float(batch.n_tokens); + } + } } llm.init(); @@ -7110,6 +7323,10 @@ static struct ggml_cgraph * llama_build_graph( { result = llm.build_refact(); } break; + case LLM_ARCH_BERT: + { + result = llm.build_bert(); + } break; case LLM_ARCH_BLOOM: { result = llm.build_bloom(); @@ -7269,13 +7486,18 @@ static int llama_decode_internal( // the output is always the last tensor in the graph struct ggml_tensor * res = gf->nodes[gf->n_nodes - 1]; - GGML_ASSERT(strcmp(res->name, "result_output") == 0); - - // the embeddings could be the second to last tensor, or the third to last tensor struct ggml_tensor * embeddings = gf->nodes[gf->n_nodes - 2]; - if (strcmp(embeddings->name, "result_norm") != 0) { - embeddings = gf->nodes[gf->n_nodes - 3]; - GGML_ASSERT(strcmp(embeddings->name, "result_norm") == 0); + if (strcmp(res->name, "result_output") == 0) { + // the embeddings could be the second to last tensor, or the third to last tensor + if (strcmp(embeddings->name, "result_norm") != 0) { + embeddings = gf->nodes[gf->n_nodes - 3]; + GGML_ASSERT(strcmp(embeddings->name, "result_norm") == 0); + } + } else if (strcmp(res->name, "result_embed") == 0) { + embeddings = res; + res = nullptr; + } else { + GGML_ASSERT(false); } // LLAMA_LOG_INFO("graph build time: %.3f ms (%d nodes, %d leafs)\n", (ggml_time_us() - t_start_us)/1000.0, gf->n_nodes, gf->n_leafs); @@ -7344,7 +7566,7 @@ static int llama_decode_internal( // extract logits // TODO: do not compute and extract logits if only embeddings are needed // need to update the graphs to skip "result_output" - { + if (res) { auto & logits_out = lctx.logits; #ifndef NDEBUG @@ -7388,9 +7610,11 @@ static int llama_decode_internal( if (!lctx.embedding.empty()) { auto & embedding_out = lctx.embedding; + const int64_t embed_pos = res ? n_embd * (n_tokens-1) : 0; + embedding_out.resize(n_embd); ggml_backend_t embeddings_backend = ggml_backend_sched_get_node_backend(lctx.sched, embeddings); - ggml_backend_tensor_get_async(embeddings_backend, embeddings, embedding_out.data(), (n_embd*(n_tokens - 1))*sizeof(float), n_embd*sizeof(float)); + ggml_backend_tensor_get_async(embeddings_backend, embeddings, embedding_out.data(), embed_pos*sizeof(float), n_embd*sizeof(float)); ggml_backend_synchronize(embeddings_backend); } @@ -7454,6 +7678,9 @@ static uint8_t llama_token_to_byte(const llama_vocab& vocab, llama_token id) { GGML_ASSERT(false); return unicode_to_bytes_bpe(token_data.text); } + case LLAMA_VOCAB_TYPE_WPM: { + GGML_ASSERT(false); + } default: GGML_ASSERT(false); } @@ -7466,6 +7693,7 @@ static llama_token llama_byte_to_token(const llama_vocab & vocab, uint8_t ch) { const char buf[7] = { '<', '0', 'x', hex[ch >> 4], hex[ch & 15], '>', 0 }; return vocab.token_to_id.at(buf); } + case LLAMA_VOCAB_TYPE_WPM: case LLAMA_VOCAB_TYPE_BPE: { return vocab.token_to_id.at(bytes_to_unicode_bpe(ch)); } @@ -7936,12 +8164,212 @@ private: llm_bigram_bpe::queue work_queue; }; -typedef enum FRAGMENT_BUFFER_VARIANT_TYPE{ +struct llm_tokenizer_wpm { + llm_tokenizer_wpm(const llama_vocab & vocab): vocab(vocab) {} + + void tokenize(const std::string & text, std::vector & output) { + auto * token_map = &vocab.token_to_id; + + // normalize and split by whitespace + std::vector words = preprocess(text); + + // bos token prepended already + + // find the longest tokens that form the words + for (const std::string &word : words) { + // skip empty words + if (word.size() == 0) { + continue; + } + + // prepend phantom space + std::string word1 = "\xe2\x96\x81" + word; + int n = word1.size(); + + // we're at the start of a new word + int i = 0; + bool match_any = false; + + // move through character position in word + while (i < n) { + // loop through possible match length + bool match = false; + for (int j = n; j > i; j--) { + auto it = token_map->find(word1.substr(i, j - i)); + if (it != token_map->end()) { + output.push_back(it->second); + match = true; + match_any = true; + i = j; + break; + } + } + + // must be an unknown character + if (!match) { + i++; + } + } + + // we didn't find any matches for this word + if (!match_any) { + output.push_back(vocab.special_unk_id); + } + } + + // append eos token + output.push_back(vocab.special_eos_id); + } + + std::vector preprocess(const std::string & text) { + std::string ori_str = normalize(text); + uint64_t ori_size = ori_str.size(); + + // single punct / single symbol / single digit + // baseline: add whitespace on the left and right of punct and chinese characters + std::vector words; + std::string new_str = ""; + uint64_t i = 0; + while (i < ori_size) { + int utf_char_len = utf8_len(ori_str[i]); + if ((utf_char_len == 1) && ispunct(ori_str[i])) { + new_str += " "; + new_str += ori_str[i]; + new_str += " "; + i += 1; + } + else if ((utf_char_len == 3) && is_chinese_char(ori_str.substr(i, 3))) { + new_str += " "; + new_str += ori_str.substr(i, 3); + new_str += " "; + i += 3; + } + else { + new_str += ori_str[i]; + i += 1; + } + } + + // split by whitespace + uint64_t l = 0; + uint64_t r = 0; + while (r < new_str.size()) { + // if is whitespace + if (isspace(new_str[r])) { + if (r > l) words.push_back(new_str.substr(l, (r - l))); + l = r + 1; + r = l; + } + else { + r += 1; + } + } + if (r > l) { + words.push_back(new_str.substr(l, (r - l))); + } + return words; + } + + std::string normalize(const std::string & text) { + // TODO: handle chinese characters? https://github.com/huggingface/tokenizers/blob/ef5f50605ddf9f8caef1598c0e4853862b9707a7/tokenizers/src/normalizers/bert.rs#L98 + std::string text2 = strip_accents(text); + for (size_t i = 0; i < text2.size(); i += utf8_len(text2[i])) { + char c = text2[i]; + if (c >= 'A' && c <= 'Z') { + text2[i] = c - 'A' + 'a'; + } + } + return text2; + } + + bool is_chinese_char(const std::string & str) { + int len = str.length(); + unsigned int codepoint = 0; + int num_bytes = 0; + int i = 0; + unsigned char ch = static_cast(str[i]); + if (ch <= 0x7f) { + codepoint = ch; + num_bytes = 1; + } else if ((ch >> 5) == 0x06) { + codepoint = ch & 0x1f; + num_bytes = 2; + } else if ((ch >> 4) == 0x0e) { + codepoint = ch & 0x0f; + num_bytes = 3; + } else if ((ch >> 3) == 0x1e) { + codepoint = ch & 0x07; + num_bytes = 4; + } + for (int j = 1; j < num_bytes; ++j) { + if (i + j >= len) { + return false; // incomplete UTF-8 character + } + unsigned char next_ch = static_cast(str[i + j]); + if ((next_ch >> 6) != 0x02) { + return false; // invalid trailing byte + } + codepoint = (codepoint << 6) | (next_ch & 0x3f); + } + if ((codepoint >= 0x4E00 && codepoint <= 0x9FFF) || + (codepoint >= 0x3400 && codepoint <= 0x4DBF) || + (codepoint >= 0x20000 && codepoint <= 0x2A6DF) || + (codepoint >= 0x2A700 && codepoint <= 0x2B73F) || + (codepoint >= 0x2B740 && codepoint <= 0x2B81F) || + (codepoint >= 0x2B920 && codepoint <= 0x2CEAF) || // this should be 0x2B820 but in hf rust code it is 0x2B920 + (codepoint >= 0xF900 && codepoint <= 0xFAFF) || + (codepoint >= 0x2F800 && codepoint <= 0x2FA1F) || + (codepoint >= 0x3000 && codepoint <= 0x303F) || + (codepoint >= 0xFF00 && codepoint <= 0xFFEF)) { + return true; // NOLINT + } + return false; + } + + std::string strip_accents(const std::string & input_string) { + std::string resultString; + std::map accent_map = { + {"À", 'A'}, {"Á", 'A'}, {"Â", 'A'}, {"Ã", 'A'}, {"Ä", 'A'}, {"Å", 'A'}, + {"à", 'a'}, {"á", 'a'}, {"â", 'a'}, {"ã", 'a'}, {"ä", 'a'}, {"å", 'a'}, + {"È", 'E'}, {"É", 'E'}, {"Ê", 'E'}, {"Ë", 'E'}, {"è", 'e'}, {"é", 'e'}, + {"ê", 'e'}, {"ë", 'e'}, {"Ì", 'I'}, {"Í", 'I'}, {"Î", 'I'}, {"Ï", 'I'}, + {"ì", 'i'}, {"í", 'i'}, {"î", 'i'}, {"ï", 'i'}, {"Ò", 'O'}, {"Ó", 'O'}, + {"Ô", 'O'}, {"Õ", 'O'}, {"Ö", 'O'}, {"ò", 'o'}, {"ó", 'o'}, {"ô", 'o'}, + {"õ", 'o'}, {"ö", 'o'}, {"Ù", 'U'}, {"Ú", 'U'}, {"Û", 'U'}, {"Ü", 'U'}, + {"ù", 'u'}, {"ú", 'u'}, {"û", 'u'}, {"ü", 'u'}, {"Ý", 'Y'}, {"ý", 'y'}, + {"Ç", 'C'}, {"ç", 'c'}, {"Ñ", 'N'}, {"ñ", 'n'}, + }; + + for (size_t i = 0; i < input_string.length();) { + int len = utf8_len(input_string[i]); + std::string curChar = input_string.substr(i, len); + auto iter = accent_map.find(curChar); + if (iter != accent_map.end()) { + resultString += iter->second; + } else { + resultString += curChar; + } + i += len; + } + + return resultString; + } + + static size_t utf8_len(char src) { + const size_t lookup[] = {1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 3, 4}; + uint8_t highbits = static_cast(src) >> 4; + return lookup[highbits]; + } + + const llama_vocab & vocab; +}; + +typedef enum FRAGMENT_BUFFER_VARIANT_TYPE { FRAGMENT_BUFFER_VARIANT_TYPE_TOKEN, FRAGMENT_BUFFER_VARIANT_TYPE_RAW_TEXT } FRAGMENT_BUFFER_VARIANT_TYPE; -struct fragment_buffer_variant{ +struct fragment_buffer_variant { fragment_buffer_variant(llama_vocab::id _token) : type(FRAGMENT_BUFFER_VARIANT_TYPE_TOKEN), @@ -7971,8 +8399,7 @@ struct fragment_buffer_variant{ // #define PRETOKENIZERDEBUG -static void tokenizer_st_partition(const llama_vocab & vocab, std::forward_list & buffer) -{ +static void tokenizer_st_partition(const llama_vocab & vocab, std::forward_list & buffer) { // for each special token for (const auto & st: vocab.special_tokens_cache) { const auto & special_token = st.first; @@ -8090,10 +8517,8 @@ static std::vector llama_tokenize_internal(const llama_vocab & switch (vocab.type) { case LLAMA_VOCAB_TYPE_SPM: { - for (const auto & fragment: fragment_buffer) - { - if (fragment.type == FRAGMENT_BUFFER_VARIANT_TYPE_RAW_TEXT) - { + for (const auto & fragment: fragment_buffer) { + if (fragment.type == FRAGMENT_BUFFER_VARIANT_TYPE_RAW_TEXT) { // without adding this leading whitespace, we do not get the same results as the original tokenizer // TODO: It's likely possible to get rid of this string copy entirely @@ -8113,19 +8538,15 @@ static std::vector llama_tokenize_internal(const llama_vocab & llm_tokenizer_spm tokenizer(vocab); llama_escape_whitespace(raw_text); tokenizer.tokenize(raw_text, output); - } - else // if (fragment.type == FRAGMENT_BUFFER_VARIANT_TYPE_TOKEN) - { + } else { // if (fragment.type == FRAGMENT_BUFFER_VARIANT_TYPE_TOKEN) output.push_back(fragment.token); } } } break; case LLAMA_VOCAB_TYPE_BPE: { - for (const auto & fragment: fragment_buffer) - { - if (fragment.type == FRAGMENT_BUFFER_VARIANT_TYPE_RAW_TEXT) - { + for (const auto & fragment: fragment_buffer) { + if (fragment.type == FRAGMENT_BUFFER_VARIANT_TYPE_RAW_TEXT) { auto raw_text = fragment.raw_text.substr(fragment.offset, fragment.length); #ifdef PRETOKENIZERDEBUG @@ -8133,9 +8554,23 @@ static std::vector llama_tokenize_internal(const llama_vocab & #endif llm_tokenizer_bpe tokenizer(vocab); tokenizer.tokenize(raw_text, output); + } else { // if (fragment.type == FRAGMENT_BUFFER_VARIANT_TYPE_TOKEN) + output.push_back(fragment.token); } - else // if (fragment.type == FRAGMENT_BUFFER_VARIANT_TYPE_TOKEN) - { + } + } break; + case LLAMA_VOCAB_TYPE_WPM: + { + for (const auto & fragment: fragment_buffer) { + if (fragment.type == FRAGMENT_BUFFER_VARIANT_TYPE_RAW_TEXT) { + auto raw_text = fragment.raw_text.substr(fragment.offset, fragment.length); + +#ifdef PRETOKENIZERDEBUG + LLAMA_LOG_WARN("TT: (%ld %ld %ld) '%s'\n", raw_text.length(), fragment.offset, fragment.length, raw_text.c_str()); +#endif + llm_tokenizer_wpm tokenizer(vocab); + tokenizer.tokenize(raw_text, output); + } else { // if (fragment.type == FRAGMENT_BUFFER_VARIANT_TYPE_TOKEN) output.push_back(fragment.token); } } @@ -10799,7 +11234,7 @@ struct llama_context * llama_new_context_with_model( // graph inputs { ggml_init_params init_params = { - /* .mem_size */ ggml_tensor_overhead()*5, + /* .mem_size */ ggml_tensor_overhead()*7, /* .mem_buffer */ nullptr, /* .no_alloc */ true, }; @@ -10810,12 +11245,14 @@ struct llama_context * llama_new_context_with_model( ctx->inp_pos = ggml_new_tensor_1d(ctx->ctx_input, GGML_TYPE_I32, cparams.n_batch); ctx->inp_KQ_mask = ggml_new_tensor_2d(ctx->ctx_input, GGML_TYPE_F32, cparams.n_ctx, cparams.n_batch); ctx->inp_K_shift = ggml_new_tensor_1d(ctx->ctx_input, GGML_TYPE_I32, cparams.n_ctx); + ctx->inp_sum = ggml_new_tensor_2d(ctx->ctx_input, GGML_TYPE_F32, 1, cparams.n_batch); ggml_set_name(ctx->inp_tokens, "inp_tokens"); ggml_set_name(ctx->inp_embd, "inp_embd"); ggml_set_name(ctx->inp_pos, "inp_pos"); ggml_set_name(ctx->inp_KQ_mask, "inp_KQ_mask"); ggml_set_name(ctx->inp_K_shift, "inp_K_shift"); + ggml_set_name(ctx->inp_sum, "inp_sum"); ctx->buf_input = ggml_backend_alloc_ctx_tensors_from_buft(ctx->ctx_input, llama_default_buffer_type_cpu(true)); @@ -11746,6 +12183,7 @@ static std::string llama_decode_text(const std::string & text) { int32_t llama_token_to_piece(const struct llama_model * model, llama_token token, char * buf, int32_t length) { if (0 <= token && token < llama_n_vocab(model)) { switch (llama_vocab_get_type(model->vocab)) { + case LLAMA_VOCAB_TYPE_WPM: case LLAMA_VOCAB_TYPE_SPM: { // NOTE: we accept all unsupported token types, // suppressing them like CONTROL tokens. diff --git a/llama.h b/llama.h index cec4158bc..367e8f1a1 100644 --- a/llama.h +++ b/llama.h @@ -61,6 +61,7 @@ extern "C" { enum llama_vocab_type { LLAMA_VOCAB_TYPE_SPM = 0, // SentencePiece LLAMA_VOCAB_TYPE_BPE = 1, // Byte Pair Encoding + LLAMA_VOCAB_TYPE_WPM = 2, // WordPiece }; enum llama_token_type { From 3bdc4cd0f595a6096cca4a64aa75ffa8a3503465 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20G=C3=A4=C3=9Fler?= Date: Sun, 11 Feb 2024 19:08:39 +0100 Subject: [PATCH 324/812] CUDA: mul_mat_vec_q tiling, refactor mul mat logic (#5434) * CUDA: mul_mat_vec_q tiling, refactor mul mat logic Co-authored-by: slaren --------- Co-authored-by: slaren --- ggml-cuda.cu | 265 +++++++++++++++++++++++++++++---------------------- 1 file changed, 149 insertions(+), 116 deletions(-) diff --git a/ggml-cuda.cu b/ggml-cuda.cu index 5053757e6..96976f248 100644 --- a/ggml-cuda.cu +++ b/ggml-cuda.cu @@ -150,8 +150,8 @@ #define CUDA_USE_TENSOR_CORES #endif -// max batch size to use MMQ kernels when tensor cores are available -#define MMQ_MAX_BATCH_SIZE 32 +#define MMVQ_MAX_BATCH_SIZE 8 // max batch size to use MMVQ kernels +#define MMQ_MAX_BATCH_SIZE 32 // max batch size to use MMQ kernels when tensor cores are available #if defined(GGML_USE_HIPBLAS) #define __CUDA_ARCH__ 1300 @@ -5310,51 +5310,59 @@ template static __global__ void #endif // __CUDA_ARCH__ >= CC_VOLTA } -#define MMVQ_NWARPS_NVIDIA 4 -#define MMVQ_NWARPS_AMD_RDNA2 1 -#define MMVQ_NWARPS_AMD_OLD 4 - -template +template #if !(defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)) -__launch_bounds__(nwarps*WARP_SIZE, 1) // tells the compiler to use as many registers as it wants +// tell the compiler to use as many registers as it wants, see nwarps definition below +__launch_bounds__((ncols_y <= 4 ? 4 : 2)*WARP_SIZE, 1) #endif // !(defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)) static __global__ void mul_mat_vec_q( const void * __restrict__ vx, const void * __restrict__ vy, float * __restrict__ dst, - const int ncols_x, const int nrows_x, const int nrows_y, const int ncols_y_par, const int nrows_dst) { + const int ncols_x, const int nrows_x, const int nrows_y, const int nrows_dst) { - const int ncols_y = ncols_y_template != 0 ? ncols_y_template : ncols_y_par; +#if defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__) && (defined(RDNA2) || defined(RDNA3)) + constexpr int nwarps = 1; + constexpr int rows_per_cuda_block = 1; +#else + constexpr int nwarps = ncols_y <= 4 ? 4 : 2; + constexpr int rows_per_cuda_block = ncols_y == 1 ? 1 : 2; +#endif // defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__) && !defined(RDNA2) && !defined(RDNA3) - const int tid = WARP_SIZE*threadIdx.y + threadIdx.x; - const int row = blockIdx.x; - - const int blocks_per_row_x = ncols_x / qk; - const int blocks_per_col_y = nrows_y / QK8_1; - const int blocks_per_iter = vdr * nwarps*WARP_SIZE / qi; + const int tid = WARP_SIZE*threadIdx.y + threadIdx.x; + const int row0 = rows_per_cuda_block*blockIdx.x; + const int blocks_per_row_x = ncols_x / qk; + const int blocks_per_col_y = nrows_y / QK8_1; + constexpr int blocks_per_iter = vdr * nwarps*WARP_SIZE / qi; // partial sum for each thread - float tmp[ncols_y_template != 0 ? ncols_y_template : 8] = {0.0f}; + float tmp[ncols_y][rows_per_cuda_block] = {0.0f}; const block_q_t * x = (const block_q_t *) vx; const block_q8_1 * y = (const block_q8_1 *) vy; - for (int i = tid / (qi/vdr); i < blocks_per_row_x; i += blocks_per_iter) { - const int ibx = row*blocks_per_row_x + i; // x block index + for (int kbx = tid / (qi/vdr); kbx < blocks_per_row_x; kbx += blocks_per_iter) { + const int kby = kbx * (qk/QK8_1); // y block index that aligns with kbx - const int iby = i * (qk/QK8_1); // y block index that aligns with ibx - - const int iqs = vdr * (tid % (qi/vdr)); // x block quant index when casting the quants to int + // x block quant index when casting the quants to int + const int kqs = vdr * (tid % (qi/vdr)); #pragma unroll for (int j = 0; j < ncols_y; ++j) { - tmp[j] += vec_dot_q_cuda(&x[ibx], &y[j*blocks_per_col_y + iby], iqs); +#pragma unroll + for (int i = 0; i < rows_per_cuda_block; ++i) { + tmp[j][i] += vec_dot_q_cuda( + &x[kbx + (row0 + i)*blocks_per_row_x], &y[j*blocks_per_col_y + kby], kqs); + } } } - __shared__ float tmp_shared[nwarps-1 > 0 ? nwarps-1 : 1][ncols_y_template != 0 ? ncols_y_template : 8][WARP_SIZE]; + __shared__ float tmp_shared[nwarps-1 > 0 ? nwarps-1 : 1][ncols_y][rows_per_cuda_block][WARP_SIZE]; if (threadIdx.y > 0) { #pragma unroll for (int j = 0; j < ncols_y; ++j) { - tmp_shared[threadIdx.y-1][j][threadIdx.x] = tmp[j]; +#pragma unroll + for (int i = 0; i < rows_per_cuda_block; ++i) { + tmp_shared[threadIdx.y-1][j][i][threadIdx.x] = tmp[j][i]; + } } } __syncthreads(); @@ -5366,13 +5374,16 @@ static __global__ void mul_mat_vec_q( #pragma unroll for (int j = 0; j < ncols_y; ++j) { #pragma unroll - for (int i = 0; i < nwarps-1; ++i) { - tmp[j] += tmp_shared[i][j][threadIdx.x]; + for (int i = 0; i < rows_per_cuda_block; ++i) { +#pragma unroll + for (int l = 0; l < nwarps-1; ++l) { + tmp[j][i] += tmp_shared[l][j][i][threadIdx.x]; + } + tmp[j][i] = warp_reduce_sum(tmp[j][i]); } - tmp[j] = warp_reduce_sum(tmp[j]); - if (threadIdx.x == 0) { - dst[j*nrows_dst + row] = tmp[j]; + if (threadIdx.x < rows_per_cuda_block) { + dst[j*nrows_dst + row0 + threadIdx.x] = tmp[j][threadIdx.x]; } } } @@ -6851,65 +6862,75 @@ static void mul_mat_vec_q_cuda( const int ncols_x, const int nrows_x, const int nrows_y, const int ncols_y, const int nrows_dst, cudaStream_t stream) { GGML_ASSERT(ncols_x % qk == 0); - GGML_ASSERT(ncols_y <= 4); + GGML_ASSERT(ncols_y <= MMVQ_MAX_BATCH_SIZE); int id; CUDA_CHECK(cudaGetDevice(&id)); - int nwarps; - if (g_device_caps[id].cc >= CC_OFFSET_AMD) { - nwarps = g_device_caps[id].cc >= CC_RDNA2 ? MMVQ_NWARPS_AMD_RDNA2 : MMVQ_NWARPS_AMD_OLD; - } else { - nwarps = MMVQ_NWARPS_NVIDIA; - } + int64_t nwarps = 1; + int64_t rows_per_cuda_block = 1; - const dim3 block_nums(nrows_x, 1, 1); + if (g_device_caps[id].cc < CC_RDNA2) { // NVIDIA and AMD older than RDNA2 + switch(ncols_y) { + case 1: + nwarps = 4; + rows_per_cuda_block = 1; + break; + case 2: + case 3: + case 4: + nwarps = 4; + rows_per_cuda_block = 2; + break; + case 5: + case 6: + case 7: + case 8: + nwarps = 2; + rows_per_cuda_block = 2; + break; + default: + GGML_ASSERT(false); + break; + } + } + const int64_t nblocks = (nrows_x + rows_per_cuda_block - 1) / rows_per_cuda_block; + const dim3 block_nums(nblocks, 1, 1); const dim3 block_dims(WARP_SIZE, nwarps, 1); - switch (nwarps) { - case 1: switch(ncols_y) { - case 1: - mul_mat_vec_q<1, 1, qk, qi, block_q_t, vdr, vec_dot> - <<>>(vx, vy, dst, ncols_x, nrows_x, nrows_y, ncols_y, nrows_dst); - break; - case 2: - mul_mat_vec_q<1, 2, qk, qi, block_q_t, vdr, vec_dot> - <<>>(vx, vy, dst, ncols_x, nrows_x, nrows_y, ncols_y, nrows_dst); - break; - case 3: - mul_mat_vec_q<1, 3, qk, qi, block_q_t, vdr, vec_dot> - <<>>(vx, vy, dst, ncols_x, nrows_x, nrows_y, ncols_y, nrows_dst); - break; - case 4: - mul_mat_vec_q<1, 4, qk, qi, block_q_t, vdr, vec_dot> - <<>>(vx, vy, dst, ncols_x, nrows_x, nrows_y, ncols_y, nrows_dst); - break; - default: - GGML_ASSERT(false); - break; - } break; - case 4: switch(ncols_y) { - case 1: - mul_mat_vec_q<4, 1, qk, qi, block_q_t, vdr, vec_dot> - <<>>(vx, vy, dst, ncols_x, nrows_x, nrows_y, ncols_y, nrows_dst); - break; - case 2: - mul_mat_vec_q<4, 2, qk, qi, block_q_t, vdr, vec_dot> - <<>>(vx, vy, dst, ncols_x, nrows_x, nrows_y, ncols_y, nrows_dst); - break; - case 3: - mul_mat_vec_q<4, 3, qk, qi, block_q_t, vdr, vec_dot> - <<>>(vx, vy, dst, ncols_x, nrows_x, nrows_y, ncols_y, nrows_dst); - break; - case 4: - mul_mat_vec_q<4, 4, qk, qi, block_q_t, vdr, vec_dot> - <<>>(vx, vy, dst, ncols_x, nrows_x, nrows_y, ncols_y, nrows_dst); - break; - default: - GGML_ASSERT(false); - break; - } break; - + switch (ncols_y) { + case 1: + mul_mat_vec_q<1, qk, qi, block_q_t, vdr, vec_dot> + <<>>(vx, vy, dst, ncols_x, nrows_x, nrows_y, nrows_dst); + break; + case 2: + mul_mat_vec_q<2, qk, qi, block_q_t, vdr, vec_dot> + <<>>(vx, vy, dst, ncols_x, nrows_x, nrows_y, nrows_dst); + break; + case 3: + mul_mat_vec_q<3, qk, qi, block_q_t, vdr, vec_dot> + <<>>(vx, vy, dst, ncols_x, nrows_x, nrows_y, nrows_dst); + break; + case 4: + mul_mat_vec_q<4, qk, qi, block_q_t, vdr, vec_dot> + <<>>(vx, vy, dst, ncols_x, nrows_x, nrows_y, nrows_dst); + break; + case 5: + mul_mat_vec_q<5, qk, qi, block_q_t, vdr, vec_dot> + <<>>(vx, vy, dst, ncols_x, nrows_x, nrows_y, nrows_dst); + break; + case 6: + mul_mat_vec_q<6, qk, qi, block_q_t, vdr, vec_dot> + <<>>(vx, vy, dst, ncols_x, nrows_x, nrows_y, nrows_dst); + break; + case 7: + mul_mat_vec_q<7, qk, qi, block_q_t, vdr, vec_dot> + <<>>(vx, vy, dst, ncols_x, nrows_x, nrows_y, nrows_dst); + break; + case 8: + mul_mat_vec_q<8, qk, qi, block_q_t, vdr, vec_dot> + <<>>(vx, vy, dst, ncols_x, nrows_x, nrows_y, nrows_dst); + break; default: GGML_ASSERT(false); break; @@ -9735,7 +9756,7 @@ static __global__ void k_compute_batched_ptrs( ptrs_dst[0*ne23 + i12 + i13*ne12] = ( char *) dst + i12*nbd2 + i13*nbd3; } -static void ggml_cuda_mul_mat_mat_batched_cublas(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { +static void ggml_cuda_mul_mat_batched_cublas(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { GGML_ASSERT(!ggml_is_transposed(src0)); GGML_ASSERT(!ggml_is_transposed(src1)); @@ -9893,39 +9914,69 @@ static void ggml_cuda_mul_mat(const ggml_tensor * src0, const ggml_tensor * src1 int64_t min_compute_capability = INT_MAX; + bool any_pascal_with_slow_fp16 = false; if (split) { ggml_backend_cuda_split_buffer_type_context * buft_ctx = (ggml_backend_cuda_split_buffer_type_context *) src0->buffer->buft->context; auto & tensor_split = buft_ctx->tensor_split; for (int id = 0; id < g_device_count; ++id) { - if (min_compute_capability > g_device_caps[id].cc && tensor_split[id] < (id + 1 < g_device_count ? tensor_split[id + 1] : 1.0f)) { + // skip devices that are not going to do any work: + if (tensor_split[id] >= (id + 1 < g_device_count ? tensor_split[id + 1] : 1.0f)) { + continue; + } + + if (min_compute_capability > g_device_caps[id].cc) { min_compute_capability = g_device_caps[id].cc; } + if (g_device_caps[id].cc == 610) { + any_pascal_with_slow_fp16 = true; + } } } else { - min_compute_capability = g_device_caps[g_main_device].cc; + min_compute_capability = g_device_caps[g_main_device].cc; + any_pascal_with_slow_fp16 = g_device_caps[g_main_device].cc == 610; } + // check data types and tensor shapes for custom matrix multiplication kernels: + bool use_dequantize_mul_mat_vec = (ggml_is_quantized(src0->type) || src0->type == GGML_TYPE_F16) + && src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32 + && src0->ne[0] % GGML_CUDA_DMMV_X == 0 && src1->ne[1] == 1; + + bool use_mul_mat_vec_q = ggml_is_quantized(src0->type) + && src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32 + && src1->ne[1] <= MMVQ_MAX_BATCH_SIZE; + + bool use_mul_mat_q = ggml_cuda_supports_mmq(src0->type) + && src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32; + #if defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__) const bool fp16_performance_good = min_compute_capability >= CC_RDNA1; - bool use_mul_mat_q = ggml_is_quantized(src0->type); + #ifdef CUDA_USE_TENSOR_CORES use_mul_mat_q = use_mul_mat_q && min_compute_capability < CC_RDNA3; #endif // CUDA_USE_TENSOR_CORES #else - const bool fp16_performance_good = min_compute_capability >= CC_VOLTA; - bool use_mul_mat_q = min_compute_capability >= MIN_CC_DP4A && ggml_is_quantized(src0->type); + // fp16 performance is good on Volta or newer and on P100 (compute capability 6.0) + const bool fp16_performance_good = min_compute_capability >= CC_PASCAL && !any_pascal_with_slow_fp16; + + // mmvq and mmq need the __dp4a instruction which on NVIDIA is only available for CC >= 6.1 + use_mul_mat_vec_q = use_mul_mat_vec_q && min_compute_capability >= MIN_CC_DP4A; + use_mul_mat_q = use_mul_mat_q && min_compute_capability >= MIN_CC_DP4A; + #ifdef CUDA_USE_TENSOR_CORES // when tensor cores are available, use them for large batch size // ref: https://github.com/ggerganov/llama.cpp/pull/3776 - use_mul_mat_q = use_mul_mat_q && !(fp16_performance_good && src1->ne[1] > MMQ_MAX_BATCH_SIZE); + use_mul_mat_q = use_mul_mat_q && (!fp16_performance_good || src1->ne[1] <= MMQ_MAX_BATCH_SIZE); #endif // CUDA_USE_TENSOR_CORES #endif // defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__) - use_mul_mat_q = use_mul_mat_q && ggml_cuda_supports_mmq(src0->type); + // if mmvq is available it's a better choice than dmmv: +#ifndef GGML_CUDA_FORCE_DMMV + use_dequantize_mul_mat_vec = use_dequantize_mul_mat_vec && !use_mul_mat_vec_q; +#endif // GGML_CUDA_FORCE_DMMV // debug helpers //printf("src0: %8d %8d %8d %8d\n", src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3]); @@ -9943,33 +9994,15 @@ static void ggml_cuda_mul_mat(const ggml_tensor * src0, const ggml_tensor * src1 ggml_cuda_mul_mat_vec_nc(src0, src1, dst); } else if (!split && all_on_device && fp16_performance_good && src0->type == GGML_TYPE_F16 && !ggml_is_transposed(src0) && !ggml_is_transposed(src1) && src1->ne[2]*src1->ne[3] > 1) { // KQ + KQV multi-batch - ggml_cuda_mul_mat_mat_batched_cublas(src0, src1, dst); - } else if (src0->type == GGML_TYPE_F32) { - ggml_cuda_op_mul_mat(src0, src1, dst, ggml_cuda_op_mul_mat_cublas, false); - } else if (ggml_is_quantized(src0->type) || src0->type == GGML_TYPE_F16) { - if (src1->ne[1] == 1 && src0->ne[0] % GGML_CUDA_DMMV_X == 0 && src1->type == GGML_TYPE_F32) { -#ifdef GGML_CUDA_FORCE_DMMV - const bool use_mul_mat_vec_q = false; -#else - const bool use_mul_mat_vec_q = min_compute_capability >= MIN_CC_DP4A && ggml_is_quantized(src0->type); -#endif // GGML_CUDA_FORCE_DMMV - - if (use_mul_mat_vec_q) { - ggml_cuda_op_mul_mat(src0, src1, dst, ggml_cuda_op_mul_mat_vec_q, true); - } else { - ggml_cuda_op_mul_mat(src0, src1, dst, ggml_cuda_op_dequantize_mul_mat_vec, false); - } - } else { - if (src1->ne[1] <= 4 && min_compute_capability >= MIN_CC_DP4A && ggml_is_quantized(src0->type) && src1->type == GGML_TYPE_F32) { - ggml_cuda_op_mul_mat(src0, src1, dst, ggml_cuda_op_mul_mat_vec_q, true); - } else if (use_mul_mat_q) { - ggml_cuda_op_mul_mat(src0, src1, dst, ggml_cuda_op_mul_mat_q, true); - } else { - ggml_cuda_op_mul_mat(src0, src1, dst, ggml_cuda_op_mul_mat_cublas, false); - } - } + ggml_cuda_mul_mat_batched_cublas(src0, src1, dst); + } else if (use_dequantize_mul_mat_vec) { + ggml_cuda_op_mul_mat(src0, src1, dst, ggml_cuda_op_dequantize_mul_mat_vec, false); + } else if (use_mul_mat_vec_q) { + ggml_cuda_op_mul_mat(src0, src1, dst, ggml_cuda_op_mul_mat_vec_q, true); + } else if (use_mul_mat_q) { + ggml_cuda_op_mul_mat(src0, src1, dst, ggml_cuda_op_mul_mat_q, true); } else { - GGML_ASSERT(false); + ggml_cuda_op_mul_mat(src0, src1, dst, ggml_cuda_op_mul_mat_cublas, false); } } From 3b169441dfe8e420f88d1592708cc2a871daadb9 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Mon, 12 Feb 2024 09:16:06 +0200 Subject: [PATCH 325/812] sync : ggml (#5452) * ggml-alloc : v3 (ggml/727) * ggml-alloc v3 ggml-ci * fix ci ggml-ci * whisper : check for backend buffer allocation failures * whisper : avoid leaks when initialization fails * cleanup ggml-ci * style fixes ggml-ci * sync : ggml * update llama.cpp, clip.cpp, export-lora.cpp * update finetune.cpp, train-text-from-scratch.cpp ggml-ci * ggml-backend : reduce alignment to 32 to match gguf and fix mmap --------- Co-authored-by: slaren --- examples/export-lora/export-lora.cpp | 19 +- examples/finetune/finetune.cpp | 147 +- examples/llava/clip.cpp | 152 +- .../train-text-from-scratch.cpp | 112 +- ggml-alloc.c | 1373 +++++++++-------- ggml-alloc.h | 110 +- ggml-backend.c | 492 +++--- ggml-backend.h | 15 +- ggml.c | 28 +- ggml.h | 18 +- llama.cpp | 181 +-- scripts/sync-ggml.last | 2 +- 12 files changed, 1287 insertions(+), 1362 deletions(-) diff --git a/examples/export-lora/export-lora.cpp b/examples/export-lora/export-lora.cpp index 4cd5d99bb..2f7be8a13 100644 --- a/examples/export-lora/export-lora.cpp +++ b/examples/export-lora/export-lora.cpp @@ -337,24 +337,14 @@ static bool apply_lora(struct ggml_tensor * tensor, struct lora_data * lora, int params.mem_buffer = NULL; params.no_alloc = true; struct ggml_context * ctx = NULL; - struct ggml_allocr * alloc = NULL; - struct ggml_cgraph * gf = NULL; + struct ggml_gallocr * alloc = NULL; + struct ggml_cgraph * gf = NULL; ctx = ggml_init(params); - alloc = ggml_allocr_new_measure(tensor_alignment); + alloc = ggml_gallocr_new(ggml_backend_cpu_buffer_type()); gf = build_graph_lora(ctx, tensor, lora_a, lora_b, scaling); - size_t alloc_size = ggml_allocr_alloc_graph(alloc, gf); - ggml_allocr_free(alloc); - ggml_free(ctx); - static std::vector data_compute; - data_compute.resize(alloc_size + tensor_alignment); - - ctx = ggml_init(params); - alloc = ggml_allocr_new(data_compute.data(), data_compute.size(), tensor_alignment); - gf = build_graph_lora(ctx, tensor, lora_a, lora_b, scaling); - ggml_allocr_alloc_graph(alloc, gf); - ggml_allocr_free(alloc); + ggml_gallocr_alloc_graph(alloc, gf); struct ggml_cplan cplan = ggml_graph_plan(gf, n_threads); static std::vector data_work; @@ -363,6 +353,7 @@ static bool apply_lora(struct ggml_tensor * tensor, struct lora_data * lora, int ggml_graph_compute(gf, &cplan); + ggml_gallocr_free(alloc); ggml_free(ctx); return true; } diff --git a/examples/finetune/finetune.cpp b/examples/finetune/finetune.cpp index b7e19c5fe..b11c56020 100644 --- a/examples/finetune/finetune.cpp +++ b/examples/finetune/finetune.cpp @@ -1,5 +1,6 @@ #include "ggml.h" #include "ggml-alloc.h" +#include "ggml-backend.h" #include "llama.h" #include "common.h" #include "train.h" @@ -13,8 +14,6 @@ #pragma warning(disable: 4244 4267) // possible loss of data #endif -static const size_t tensor_alignment = 32; - struct my_llama_hparams { uint32_t n_vocab = 32000; uint32_t n_ctx = 512; @@ -128,7 +127,7 @@ struct my_llama_lora_layer { struct my_llama_lora { struct ggml_context * ctx = NULL; - std::vector data; + ggml_backend_buffer_t data; my_llama_lora_hparams hparams; @@ -372,63 +371,6 @@ static void set_param_lora(struct my_llama_lora * lora) { } } -static void alloc_lora(struct ggml_allocr * alloc, struct my_llama_lora * lora) { - ggml_allocr_alloc(alloc, lora->tok_embeddings_a); - ggml_allocr_alloc(alloc, lora->tok_embeddings_b); - ggml_allocr_alloc(alloc, lora->norm_a); - ggml_allocr_alloc(alloc, lora->norm_b); - ggml_allocr_alloc(alloc, lora->output_a); - ggml_allocr_alloc(alloc, lora->output_b); - for (uint32_t i = 0; i < lora->layers.size(); ++i) { - auto & layer = lora->layers[i]; - ggml_allocr_alloc(alloc, layer.attention_norm_a); - ggml_allocr_alloc(alloc, layer.attention_norm_b); - ggml_allocr_alloc(alloc, layer.wq_a); - ggml_allocr_alloc(alloc, layer.wq_b); - ggml_allocr_alloc(alloc, layer.wk_a); - ggml_allocr_alloc(alloc, layer.wk_b); - ggml_allocr_alloc(alloc, layer.wv_a); - ggml_allocr_alloc(alloc, layer.wv_b); - ggml_allocr_alloc(alloc, layer.wo_a); - ggml_allocr_alloc(alloc, layer.wo_b); - ggml_allocr_alloc(alloc, layer.ffn_norm_a); - ggml_allocr_alloc(alloc, layer.ffn_norm_b); - ggml_allocr_alloc(alloc, layer.w1_a); - ggml_allocr_alloc(alloc, layer.w1_b); - ggml_allocr_alloc(alloc, layer.w2_a); - ggml_allocr_alloc(alloc, layer.w2_b); - ggml_allocr_alloc(alloc, layer.w3_a); - ggml_allocr_alloc(alloc, layer.w3_b); - } - ggml_allocr_alloc(alloc, lora->tok_embeddings_a->grad); - ggml_allocr_alloc(alloc, lora->tok_embeddings_b->grad); - ggml_allocr_alloc(alloc, lora->norm_a->grad); - ggml_allocr_alloc(alloc, lora->norm_b->grad); - ggml_allocr_alloc(alloc, lora->output_a->grad); - ggml_allocr_alloc(alloc, lora->output_b->grad); - for (uint32_t i = 0; i < lora->layers.size(); ++i) { - auto & layer = lora->layers[i]; - ggml_allocr_alloc(alloc, layer.attention_norm_a->grad); - ggml_allocr_alloc(alloc, layer.attention_norm_b->grad); - ggml_allocr_alloc(alloc, layer.wq_a->grad); - ggml_allocr_alloc(alloc, layer.wq_b->grad); - ggml_allocr_alloc(alloc, layer.wk_a->grad); - ggml_allocr_alloc(alloc, layer.wk_b->grad); - ggml_allocr_alloc(alloc, layer.wv_a->grad); - ggml_allocr_alloc(alloc, layer.wv_b->grad); - ggml_allocr_alloc(alloc, layer.wo_a->grad); - ggml_allocr_alloc(alloc, layer.wo_b->grad); - ggml_allocr_alloc(alloc, layer.ffn_norm_a->grad); - ggml_allocr_alloc(alloc, layer.ffn_norm_b->grad); - ggml_allocr_alloc(alloc, layer.w1_a->grad); - ggml_allocr_alloc(alloc, layer.w1_b->grad); - ggml_allocr_alloc(alloc, layer.w2_a->grad); - ggml_allocr_alloc(alloc, layer.w2_b->grad); - ggml_allocr_alloc(alloc, layer.w3_a->grad); - ggml_allocr_alloc(alloc, layer.w3_b->grad); - } -} - static void init_lora(const struct my_llama_model * model, struct my_llama_lora * lora) { const auto & lparams = lora->hparams; @@ -522,18 +464,8 @@ static void init_lora(const struct my_llama_model * model, struct my_llama_lora set_param_lora(lora); - // measure data size - size_t size = 0; - for (struct ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) { - size += GGML_PAD(ggml_nbytes(t), tensor_alignment); - } - - // allocate data - struct ggml_allocr * alloc = NULL; - lora->data.resize(size + tensor_alignment); - alloc = ggml_allocr_new(lora->data.data(), lora->data.size(), tensor_alignment); - alloc_lora(alloc, lora); - ggml_allocr_free(alloc); + // allocate data for lora tensors + lora->data = ggml_backend_alloc_ctx_tensors_from_buft(ctx, ggml_backend_cpu_buffer_type()); } static void randomize_lora(struct my_llama_lora * lora, int seed, float mean, float std, float min, float max) { @@ -579,7 +511,7 @@ static void randomize_lora(struct my_llama_lora * lora, int seed, float mean, fl static struct ggml_tensor * llama_build_lora_finetune_graphs( struct my_llama_model * model, struct my_llama_lora * lora, - struct ggml_allocr * alloc, + ggml_gallocr_t alloc, struct ggml_context * ctx, struct ggml_cgraph * gf, struct ggml_cgraph * gb, @@ -590,7 +522,8 @@ static struct ggml_tensor * llama_build_lora_finetune_graphs( const int n_tokens, const int n_batch, const bool enable_flash_attn, - const bool enable_checkpointing) { + const bool enable_checkpointing, + const bool measure_only) { ggml_set_scratch(ctx, { 0, 0, nullptr, }); const int n_past = 0; @@ -622,13 +555,7 @@ static struct ggml_tensor * llama_build_lora_finetune_graphs( // KQ_pos - contains the positions struct ggml_tensor * KQ_pos = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, N); - ggml_allocr_alloc(alloc, KQ_pos); - if (!ggml_allocr_is_measure(alloc)) { - int * data = (int *) KQ_pos->data; - for (int i = 0; i < N; ++i) { - data[i] = n_past + i; - } - } + ggml_set_input(KQ_pos); // rope has so much parameters that we make a custom function for it auto rope = [ctx, KQ_pos, n_rot, n_ctx, rope_freq_base, rope_freq_scale] @@ -780,7 +707,7 @@ static struct ggml_tensor * llama_build_lora_finetune_graphs( // input gradient ggml_build_forward_expand(gb, ggml_scale_inplace(ctx, t36->grad, 1.0f)); GGML_ASSERT(t36->grad->data == NULL && t36->grad->view_src == NULL); - ggml_allocr_alloc(alloc, t36->grad); + ggml_set_input(t36->grad); // KQ_pos ggml_build_forward_expand(gb, ggml_scale_inplace(ctx, KQ_pos, 1.0f)); @@ -805,11 +732,23 @@ static struct ggml_tensor * llama_build_lora_finetune_graphs( // note: they will be freed in reverse order for (unsigned int i = 0; i < checkpoints.size(); ++i) { if (checkpoints[i]->data == NULL && checkpoints[i]->view_src == NULL) { - ggml_allocr_alloc(alloc, checkpoints[i]); + ggml_set_input(checkpoints[i]); } } - ggml_allocr_alloc_graph(alloc, gb); + if (measure_only) { + ggml_gallocr_reserve(alloc, gb); + } else { + ggml_gallocr_alloc_graph(alloc, gb); + + // set KQ_pos + { + int * data = (int *) KQ_pos->data; + for (int i = 0; i < N; ++i) { + data[i] = n_past + i; + } + } + } // remove the additional nodes and leafs for (int i = n_leafs_before; i < gb->n_leafs; ++i) { @@ -1663,7 +1602,7 @@ int main(int argc, char ** argv) { printf("%s: seen train_samples %llu\n", __func__, (long long unsigned) train->train_samples); printf("%s: seen train_tokens %llu\n", __func__, (long long unsigned) train->train_tokens); printf("%s: completed train_epochs %llu\n", __func__, (long long unsigned) train->train_epochs); - printf("%s: lora_size = %zu bytes (%.1f MB)\n", __func__, (ggml_used_mem(lora.ctx) + lora.data.size()), (float) (ggml_used_mem(lora.ctx) + lora.data.size()) / (1024.0f*1024.0f)); + printf("%s: lora_size = %zu bytes (%.1f MB)\n", __func__, (ggml_used_mem(lora.ctx) + ggml_backend_buffer_get_size(lora.data)), (float) (ggml_used_mem(lora.ctx) + ggml_backend_buffer_get_size(lora.data)) / (1024.0f*1024.0f)); if (params.only_write_lora) { save_train_files_data save_data; @@ -1690,10 +1629,6 @@ int main(int argc, char ** argv) { int n_vocab = model.hparams.n_vocab; int n_batch = params.common.n_batch; - - std::vector mem_input_data; - std::vector mem_compute_data; - // context for input tensors without their data struct ggml_init_params ctx_input_params = { ggml_tensor_overhead() * 2, // mem_size @@ -1706,17 +1641,11 @@ int main(int argc, char ** argv) { struct ggml_tensor * tokens_input = ggml_new_tensor_2d(ctx_input, GGML_TYPE_I32, n_tokens, n_batch); struct ggml_tensor * target_probs = ggml_new_tensor_3d(ctx_input, GGML_TYPE_F32, n_vocab, n_tokens, n_batch); - // measure required memory for input tensors - size_t max_input_size = GGML_PAD(ggml_nbytes(tokens_input), tensor_alignment) + - GGML_PAD(ggml_nbytes(target_probs), tensor_alignment) + - tensor_alignment; - printf("%s: input_size = %zu bytes (%.1f MB)\n", __func__, max_input_size, (float) max_input_size / (1024.0f*1024.0f)); - // allocate input tensors - mem_input_data.resize(max_input_size); - ggml_allocr_t alloc_inps = ggml_allocr_new(mem_input_data.data(), mem_input_data.size(), tensor_alignment); - ggml_allocr_alloc(alloc_inps, tokens_input); - ggml_allocr_alloc(alloc_inps, target_probs); + // measure required memory for input tensors + ggml_backend_buffer_t input_data = ggml_backend_alloc_ctx_tensors_from_buft(ctx_input, ggml_backend_cpu_buffer_type()); + size_t max_input_size = ggml_backend_buffer_get_size(input_data); + printf("%s: input_size = %zu bytes (%.1f MB)\n", __func__, max_input_size, (float) max_input_size / (1024.0f*1024.0f)); // context for compute tensors without their data const size_t estimated_compute_size_wo_data = ( @@ -1743,7 +1672,7 @@ int main(int argc, char ** argv) { // find best evaluation order for (unsigned order = 0; order < (unsigned) GGML_CGRAPH_EVAL_ORDER_COUNT; ++order) { ctx_compute = ggml_init(ctx_compute_params); - ggml_allocr_t alloc = ggml_allocr_new_measure(tensor_alignment); + ggml_gallocr_t alloc = ggml_gallocr_new(ggml_backend_cpu_buffer_type()); gf = ggml_new_graph_custom(ctx_compute, LLAMA_TRAIN_MAX_NODES, true); gf->order = (enum ggml_cgraph_eval_order) order; gb = ggml_new_graph_custom(ctx_compute, LLAMA_TRAIN_MAX_NODES, true); @@ -1756,14 +1685,15 @@ int main(int argc, char ** argv) { &logits, tokens_input, target_probs, n_tokens, n_batch, params.common.use_flash, - params.common.use_checkpointing + params.common.use_checkpointing, + true ); - size_t max_compute_size = ggml_allocr_max_size(alloc) + tensor_alignment; + size_t max_compute_size = ggml_gallocr_get_buffer_size(alloc, 0); // FIXME: this will still allocate the buffer if (max_compute_size < best_compute_size) { best_compute_size = max_compute_size; best_order = gf->order; } - ggml_allocr_free(alloc); + ggml_gallocr_free(alloc); ggml_free(ctx_compute); } size_t max_compute_size = best_compute_size; @@ -1774,9 +1704,8 @@ int main(int argc, char ** argv) { "invalid"); // allocate compute tensors - mem_compute_data.resize(max_compute_size); ctx_compute = ggml_init(ctx_compute_params); - ggml_allocr_t alloc = ggml_allocr_new(mem_compute_data.data(), mem_compute_data.size(), tensor_alignment); + ggml_gallocr_t alloc = ggml_gallocr_new(ggml_backend_cpu_buffer_type()); gf = ggml_new_graph_custom(ctx_compute, LLAMA_TRAIN_MAX_NODES, true); gf->order = best_order; gb = ggml_new_graph_custom(ctx_compute, LLAMA_TRAIN_MAX_NODES, true); @@ -1789,11 +1718,9 @@ int main(int argc, char ** argv) { &logits, tokens_input, target_probs, n_tokens, n_batch, params.common.use_flash, - params.common.use_checkpointing + params.common.use_checkpointing, + false ); - ggml_allocr_free(alloc); - ggml_allocr_free(alloc_inps); - // tokenize data std::vector train_tokens; @@ -1908,6 +1835,8 @@ int main(int argc, char ** argv) { ggml_free(ctx_work); ggml_free(ctx_compute); ggml_free(ctx_input); + ggml_gallocr_free(alloc); + int64_t t1 = ggml_time_ms(); printf("%s: total training time: ", __func__); diff --git a/examples/llava/clip.cpp b/examples/llava/clip.cpp index 9129052a2..ccd0d85ad 100644 --- a/examples/llava/clip.cpp +++ b/examples/llava/clip.cpp @@ -367,7 +367,7 @@ struct clip_ctx { ggml_backend_buffer_t params_buffer = NULL; ggml_backend_buffer_t compute_buffer = NULL; ggml_backend_t backend = NULL; - ggml_allocr * compute_alloc = NULL; + ggml_gallocr_t compute_alloc = NULL; }; static ggml_cgraph * clip_image_build_graph(clip_ctx * ctx, const clip_image_f32_batch * imgs) { @@ -405,31 +405,8 @@ static ggml_cgraph * clip_image_build_graph(clip_ctx * ctx, const clip_image_f32 struct ggml_cgraph * gf = ggml_new_graph(ctx0); struct ggml_tensor * inp_raw = ggml_new_tensor_4d(ctx0, GGML_TYPE_F32, image_size, image_size, 3, batch_size); - ggml_allocr_alloc(ctx->compute_alloc, inp_raw); - - if (!ggml_allocr_is_measure(ctx->compute_alloc)) { - float * data = (float *)malloc(ggml_nbytes(inp_raw)); - - for (size_t i = 0; i < imgs->size; i++) { - const int nx = imgs->data[i].nx; - const int ny = imgs->data[i].ny; - GGML_ASSERT(nx == image_size && ny == image_size); - - const int n = nx * ny; - - for (int b = 0; b < batch_size; b++) { - for (int k = 0; k < 3; k++) { - for (int y = 0; y < ny; y++) { - for (int x = 0; x < nx; x++) { - data[(b * 3 * n) + k * n + y * nx + x] = imgs->data[b].buf[3 * (y * nx + x) + k]; - } - } - } - } - } - ggml_backend_tensor_set(inp_raw, data, 0, ggml_nbytes(inp_raw)); - free(data); - } + ggml_set_name(inp_raw, "inp_raw"); + ggml_set_input(inp_raw); struct ggml_tensor * inp = ggml_conv_2d(ctx0, model.patch_embeddings, inp_raw, patch_size, patch_size, 0, 0, 1, 1); @@ -438,13 +415,8 @@ static ggml_cgraph * clip_image_build_graph(clip_ctx * ctx, const clip_image_f32 // concat class_embeddings and patch_embeddings struct ggml_tensor * embeddings = ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, hidden_size, num_positions, batch_size); - ggml_allocr_alloc(ctx->compute_alloc, embeddings); - if (!ggml_allocr_is_measure(ctx->compute_alloc)) { - void* zero_mem = malloc(ggml_nbytes(embeddings)); - memset(zero_mem, 0, ggml_nbytes(embeddings)); - ggml_backend_tensor_set(embeddings, zero_mem, 0, ggml_nbytes(embeddings)); - free(zero_mem); - } + ggml_set_name(embeddings, "embeddings"); + ggml_set_input(embeddings); embeddings = ggml_acc(ctx0, embeddings, model.class_embedding, embeddings->nb[1], embeddings->nb[2], embeddings->nb[3], 0); @@ -453,15 +425,8 @@ static ggml_cgraph * clip_image_build_graph(clip_ctx * ctx, const clip_image_f32 embeddings->nb[1], embeddings->nb[2], embeddings->nb[3], model.class_embedding->nb[1]); struct ggml_tensor * positions = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, num_positions); - ggml_allocr_alloc(ctx->compute_alloc, positions); - if (!ggml_allocr_is_measure(ctx->compute_alloc)) { - int* positions_data = (int*)malloc(ggml_nbytes(positions)); - for (int i = 0; i < num_positions; i++) { - positions_data[i] = i; - } - ggml_backend_tensor_set(positions, positions_data, 0, ggml_nbytes(positions)); - free(positions_data); - } + ggml_set_name(positions, "positions"); + ggml_set_input(positions); embeddings = ggml_add(ctx0, embeddings, ggml_get_rows(ctx0, model.position_embeddings, positions)); @@ -560,15 +525,8 @@ static ggml_cgraph * clip_image_build_graph(clip_ctx * ctx, const clip_image_f32 embeddings = ggml_reshape_2d(ctx0, embeddings, embeddings->ne[0], embeddings->ne[1]); struct ggml_tensor * patches = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, num_patches); - ggml_allocr_alloc(ctx->compute_alloc, patches); - if (!ggml_allocr_is_measure(ctx->compute_alloc)) { - int* patches_data = (int*)malloc(ggml_nbytes(patches)); - for (int i = 0; i < num_patches; i++) { - patches_data[i] = i + 1; - } - ggml_backend_tensor_set(patches, patches_data, 0, ggml_nbytes(patches)); - free(patches_data); - } + ggml_set_name(patches, "patches"); + ggml_set_input(patches); // shape [1, 576, 1024] // ne is whcn, ne = [1024, 576, 1, 1] @@ -809,7 +767,7 @@ struct clip_ctx * clip_model_load(const char * fname, const int verbosity = 1) { } // data - size_t buffer_size = 0; + size_t model_size = 0; { for (int i = 0; i < n_tensors; ++i) { const char * name = gguf_get_tensor_name(ctx, i); @@ -817,7 +775,7 @@ struct clip_ctx * clip_model_load(const char * fname, const int verbosity = 1) { enum ggml_type type = gguf_get_tensor_type(ctx, i); struct ggml_tensor * cur = ggml_get_tensor(meta, name); size_t tensor_size = ggml_nbytes(cur); - buffer_size += tensor_size; + model_size += tensor_size; if (verbosity >= 3) { printf("%s: tensor[%d]: n_dims = %d, name = %s, tensor_size=%zu, offset=%zu, shape:[%" PRIu64 ", %" PRIu64 ", %" PRIu64 ", %" PRIu64 "], type = %s\n", __func__, i, ggml_n_dims(cur), cur->name, tensor_size, offset, cur->ne[0], cur->ne[1], cur->ne[2], cur->ne[3], ggml_type_name(type)); @@ -825,8 +783,6 @@ struct clip_ctx * clip_model_load(const char * fname, const int verbosity = 1) { } } - buffer_size += n_tensors * 128 /* CLIP PADDING */; - clip_ctx * new_clip = new clip_ctx; // update projector type @@ -886,12 +842,12 @@ struct clip_ctx * clip_model_load(const char * fname, const int verbosity = 1) { printf("%s: text_encoder: %d\n", __func__, new_clip->has_text_encoder); printf("%s: vision_encoder: %d\n", __func__, new_clip->has_vision_encoder); printf("%s: llava_projector: %d\n", __func__, new_clip->has_llava_projector); - printf("%s: model size: %.2f MB\n", __func__, buffer_size / 1024.0 / 1024.0); + printf("%s: model size: %.2f MB\n", __func__, model_size / 1024.0 / 1024.0); printf("%s: metadata size: %.2f MB\n", __func__, ggml_get_mem_size(meta) / 1024.0 / 1024.0); } } - printf("%s: params backend buffer size = % 6.2f MB (%i tensors)\n", __func__, buffer_size / (1024.0 * 1024.0), n_tensors); + printf("%s: params backend buffer size = % 6.2f MB (%i tensors)\n", __func__, model_size / (1024.0 * 1024.0), n_tensors); // load tensors { @@ -925,12 +881,10 @@ struct clip_ctx * clip_model_load(const char * fname, const int verbosity = 1) { } // alloc memory and offload data - new_clip->params_buffer = ggml_backend_alloc_buffer(new_clip->backend, buffer_size); - ggml_allocr* alloc = ggml_allocr_new_from_buffer(new_clip->params_buffer); + new_clip->params_buffer = ggml_backend_alloc_ctx_tensors(new_clip->ctx_data, new_clip->backend); for (int i = 0; i < n_tensors; ++i) { const char * name = gguf_get_tensor_name(ctx, i); struct ggml_tensor * cur = ggml_get_tensor(new_clip->ctx_data, name); - ggml_allocr_alloc(alloc, cur); const size_t offset = gguf_get_data_offset(ctx) + gguf_get_tensor_offset(ctx, i); fin.seekg(offset, std::ios::beg); if (!fin) { @@ -949,7 +903,6 @@ struct clip_ctx * clip_model_load(const char * fname, const int verbosity = 1) { ggml_backend_tensor_set(cur, read_buf.data(), 0, num_bytes); } } - ggml_allocr_free(alloc); fin.close(); } @@ -1077,15 +1030,12 @@ struct clip_ctx * clip_model_load(const char * fname, const int verbosity = 1) { // measure mem requirement and allocate { new_clip->buf_compute_meta.resize(GGML_DEFAULT_GRAPH_SIZE * ggml_tensor_overhead() + ggml_graph_overhead()); - new_clip->compute_alloc = ggml_allocr_new_measure_from_backend(new_clip->backend); + new_clip->compute_alloc = ggml_gallocr_new(ggml_backend_get_default_buffer_type(new_clip->backend)); clip_image_f32_batch batch; batch.size = 1; ggml_cgraph * gf = clip_image_build_graph(new_clip, &batch); - size_t compute_memory_buffer_size = ggml_allocr_alloc_graph(new_clip->compute_alloc, gf); - ggml_allocr_free(new_clip->compute_alloc); - new_clip->compute_buffer = ggml_backend_alloc_buffer(new_clip->backend, compute_memory_buffer_size); - new_clip->compute_alloc = ggml_allocr_new_from_buffer(new_clip->compute_buffer); - + ggml_gallocr_reserve(new_clip->compute_alloc, gf); + size_t compute_memory_buffer_size = ggml_gallocr_get_buffer_size(new_clip->compute_alloc, 0); printf("%s: compute allocated memory: %.2f MB\n", __func__, compute_memory_buffer_size /1024.0/1024.0); } @@ -1267,12 +1217,72 @@ bool clip_image_batch_encode(clip_ctx * ctx, const int n_threads, const clip_ima GGML_ASSERT(batch_size == 1); // TODO: support multiple images } - // reset alloc buffer to clean the memory from previous invocations - ggml_allocr_reset(ctx->compute_alloc); - // build the inference graph ggml_cgraph * gf = clip_image_build_graph(ctx, imgs); - ggml_allocr_alloc_graph(ctx->compute_alloc, gf); + ggml_gallocr_alloc_graph(ctx->compute_alloc, gf); + + // set inputs + const auto & model = ctx->vision_model; + const auto & hparams = model.hparams; + const int image_size = hparams.image_size; + const int patch_size = hparams.patch_size; + const int num_patches = ((image_size / patch_size) * (image_size / patch_size)); + const int num_positions = num_patches + 1; + + { + struct ggml_tensor * inp_raw = ggml_graph_get_tensor(gf, "inp_raw"); + float * data = (float *)malloc(ggml_nbytes(inp_raw)); + + for (size_t i = 0; i < imgs->size; i++) { + const int nx = imgs->data[i].nx; + const int ny = imgs->data[i].ny; + GGML_ASSERT(nx == image_size && ny == image_size); + + const int n = nx * ny; + + for (int b = 0; b < batch_size; b++) { + for (int k = 0; k < 3; k++) { + for (int y = 0; y < ny; y++) { + for (int x = 0; x < nx; x++) { + data[(b * 3 * n) + k * n + y * nx + x] = imgs->data[b].buf[3 * (y * nx + x) + k]; + } + } + } + } + } + ggml_backend_tensor_set(inp_raw, data, 0, ggml_nbytes(inp_raw)); + free(data); + } + + { + struct ggml_tensor * embeddings = ggml_graph_get_tensor(gf, "embeddings"); + + void* zero_mem = malloc(ggml_nbytes(embeddings)); + memset(zero_mem, 0, ggml_nbytes(embeddings)); + ggml_backend_tensor_set(embeddings, zero_mem, 0, ggml_nbytes(embeddings)); + free(zero_mem); + } + + { + struct ggml_tensor * positions = ggml_graph_get_tensor(gf, "positions"); + + int* positions_data = (int*)malloc(ggml_nbytes(positions)); + for (int i = 0; i < num_positions; i++) { + positions_data[i] = i; + } + ggml_backend_tensor_set(positions, positions_data, 0, ggml_nbytes(positions)); + free(positions_data); + } + + { + struct ggml_tensor * patches = ggml_graph_get_tensor(gf, "patches"); + int* patches_data = (int*)malloc(ggml_nbytes(patches)); + for (int i = 0; i < num_patches; i++) { + patches_data[i] = i + 1; + } + ggml_backend_tensor_set(patches, patches_data, 0, ggml_nbytes(patches)); + free(patches_data); + } if (ggml_backend_is_cpu(ctx->backend)) { ggml_backend_cpu_set_n_threads(ctx->backend, n_threads); diff --git a/examples/train-text-from-scratch/train-text-from-scratch.cpp b/examples/train-text-from-scratch/train-text-from-scratch.cpp index eee9d4de3..2e2a8ce08 100644 --- a/examples/train-text-from-scratch/train-text-from-scratch.cpp +++ b/examples/train-text-from-scratch/train-text-from-scratch.cpp @@ -1,5 +1,6 @@ #include "ggml.h" #include "ggml-alloc.h" +#include "ggml-backend.h" #include "common.h" #include "train.h" #include "llama.h" @@ -19,8 +20,6 @@ #pragma warning(disable: 4244 4267) // possible loss of data #endif -static const size_t tensor_alignment = 32; - struct my_llama_hparams { uint32_t n_vocab = 32000; uint32_t n_ctx = 512; @@ -58,7 +57,7 @@ struct my_llama_layer { struct my_llama_model { struct ggml_context * ctx = NULL; - std::vector data; + ggml_backend_buffer_t data = NULL; my_llama_hparams hparams; @@ -147,39 +146,6 @@ static void set_param_model(struct my_llama_model * model) { } } -static void alloc_model(struct ggml_allocr * alloc, struct my_llama_model * model) { - ggml_allocr_alloc(alloc, model->tok_embeddings); - ggml_allocr_alloc(alloc, model->norm); - ggml_allocr_alloc(alloc, model->output); - for (uint32_t i = 0; i < model->layers.size(); ++i) { - auto & layer = model->layers[i]; - ggml_allocr_alloc(alloc, layer.attention_norm); - ggml_allocr_alloc(alloc, layer.wq); - ggml_allocr_alloc(alloc, layer.wk); - ggml_allocr_alloc(alloc, layer.wv); - ggml_allocr_alloc(alloc, layer.wo); - ggml_allocr_alloc(alloc, layer.ffn_norm); - ggml_allocr_alloc(alloc, layer.w1); - ggml_allocr_alloc(alloc, layer.w2); - ggml_allocr_alloc(alloc, layer.w3); - } - ggml_allocr_alloc(alloc, model->tok_embeddings->grad); - ggml_allocr_alloc(alloc, model->norm->grad); - ggml_allocr_alloc(alloc, model->output->grad); - for (uint32_t i = 0; i < model->layers.size(); ++i) { - auto & layer = model->layers[i]; - ggml_allocr_alloc(alloc, layer.attention_norm->grad); - ggml_allocr_alloc(alloc, layer.wq->grad); - ggml_allocr_alloc(alloc, layer.wk->grad); - ggml_allocr_alloc(alloc, layer.wv->grad); - ggml_allocr_alloc(alloc, layer.wo->grad); - ggml_allocr_alloc(alloc, layer.ffn_norm->grad); - ggml_allocr_alloc(alloc, layer.w1->grad); - ggml_allocr_alloc(alloc, layer.w2->grad); - ggml_allocr_alloc(alloc, layer.w3->grad); - } -} - static void init_model(struct my_llama_model * model) { const auto & hparams = model->hparams; @@ -252,17 +218,8 @@ static void init_model(struct my_llama_model * model) { set_param_model(model); - // measure data size - size_t size = 0; - for (struct ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) { - size += GGML_PAD(ggml_nbytes(t), tensor_alignment); - } - // allocate data - struct ggml_allocr * alloc = NULL; - model->data.resize(size + tensor_alignment); - alloc = ggml_allocr_new(model->data.data(), model->data.size(), tensor_alignment); - alloc_model(alloc, model); + model->data = ggml_backend_alloc_ctx_tensors_from_buft(ctx, ggml_backend_cpu_buffer_type()); } static void randomize_model(struct my_llama_model * model, int seed, float mean, float std, float min, float max) { @@ -297,7 +254,7 @@ static void randomize_model(struct my_llama_model * model, int seed, float mean, static struct ggml_tensor * llama_build_train_graphs( struct my_llama_model * model, - struct ggml_allocr * alloc, + ggml_gallocr_t alloc, struct ggml_context * ctx, struct ggml_cgraph * gf, struct ggml_cgraph * gb, @@ -308,7 +265,8 @@ static struct ggml_tensor * llama_build_train_graphs( const int n_tokens, const int n_batch, const bool enable_flash_attn, - const bool enable_checkpointing) { + const bool enable_checkpointing, + const bool measure_only) { ggml_set_scratch(ctx, { 0, 0, nullptr, }); const int n_past = 0; @@ -334,13 +292,7 @@ static struct ggml_tensor * llama_build_train_graphs( // KQ_pos - contains the positions struct ggml_tensor * KQ_pos = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, N); - ggml_allocr_alloc(alloc, KQ_pos); - if (!ggml_allocr_is_measure(alloc)) { - int * data = (int *) KQ_pos->data; - for (int i = 0; i < N; ++i) { - data[i] = n_past + i; - } - } + ggml_set_input(KQ_pos); // rope has so much parameters that we make a custom function for it auto rope = [ctx, KQ_pos, n_rot, n_ctx, rope_freq_base, rope_freq_scale] @@ -448,21 +400,31 @@ static struct ggml_tensor * llama_build_train_graphs( // KQ_pos ggml_build_forward_expand(gb, ggml_scale_inplace(ctx, KQ_pos, 1.0f)); GGML_ASSERT(t36->grad->data == NULL && t36->grad->view_src == NULL); - - ggml_allocr_alloc(alloc, t36->grad); + ggml_set_input(t36->grad); // allocating checkpoints in one block to reduce memory fragmentation // note: they will be freed in reverse order for (int i = 0; i < (int) checkpoints.size(); ++i) { if (checkpoints[i]->data == NULL && checkpoints[i]->view_src == NULL) { - ggml_allocr_alloc(alloc, checkpoints[i]); + ggml_set_input(checkpoints[i]); } } //int n_leafs_after = gb->n_leafs; //int n_nodes_after = gb->n_nodes; + if (measure_only) { + // FIXME: will still allocate + ggml_gallocr_reserve(alloc, gb); + } else { + ggml_gallocr_alloc_graph(alloc, gb); - ggml_allocr_alloc_graph(alloc, gb); + if (!measure_only) { + int * data = (int *) KQ_pos->data; + for (int i = 0; i < N; ++i) { + data[i] = n_past + i; + } + } + } // remove the additional nodes and leafs for (int i = n_leafs_before; i < gb->n_leafs; ++i) { @@ -1046,7 +1008,7 @@ int main(int argc, char ** argv) { printf("%s: seen train_samples %llu\n", __func__, (long long unsigned) train->train_samples); printf("%s: seen train_tokens %llu\n", __func__, (long long unsigned) train->train_tokens); printf("%s: completed train_epochs %llu\n", __func__, (long long unsigned) train->train_epochs); - printf("%s: model_size = %zu bytes (%.1f MB)\n", __func__, (ggml_used_mem(model.ctx) + model.data.size()), (float) (ggml_used_mem(model.ctx) + model.data.size()) / (1024.0f*1024.0f)); + printf("%s: model_size = %zu bytes (%.1f MB)\n", __func__, (ggml_used_mem(model.ctx) + ggml_backend_buffer_get_size(model.data)), (float) (ggml_used_mem(model.ctx) + ggml_backend_buffer_get_size(model.data)) / (1024.0f*1024.0f)); if (params.only_write_model) { save_train_files_data save_data; @@ -1073,11 +1035,6 @@ int main(int argc, char ** argv) { int n_vocab = model.hparams.n_vocab; int n_batch = params.common.n_batch; - std::vector mem_input_data; - std::vector mem_compute_data; - - ggml_allocr * alloc = NULL; - // context for input tensors without their data struct ggml_init_params ctx_input_params = { ggml_tensor_overhead() * 2, // mem_size @@ -1091,16 +1048,10 @@ int main(int argc, char ** argv) { struct ggml_tensor * target_probs = ggml_new_tensor_3d(ctx_input, GGML_TYPE_F32, n_vocab, n_tokens, n_batch); // measure required memory for input tensors - size_t max_input_size = GGML_PAD(ggml_nbytes(tokens_input), tensor_alignment) + - GGML_PAD(ggml_nbytes(target_probs), tensor_alignment) + - tensor_alignment; - printf("%s: input_size = %zu bytes (%.1f MB)\n", __func__, max_input_size, (float) max_input_size / (1024.0f*1024.0f)); - // allocate input tensors - mem_input_data.resize(max_input_size); - alloc = ggml_allocr_new(mem_input_data.data(), mem_input_data.size(), tensor_alignment); - ggml_allocr_alloc(alloc, tokens_input); - ggml_allocr_alloc(alloc, target_probs); + ggml_backend_buffer_t input_data = ggml_backend_alloc_ctx_tensors_from_buft(ctx_input, ggml_backend_cpu_buffer_type()); + size_t max_input_size = ggml_backend_buffer_get_size(input_data); + printf("%s: input_size = %zu bytes (%.1f MB)\n", __func__, max_input_size, (float) max_input_size / (1024.0f*1024.0f)); // context for compute tensors without their data const size_t estimated_compute_size_wo_data = ( @@ -1127,7 +1078,7 @@ int main(int argc, char ** argv) { // find best evaluation order for (unsigned order = 0; order < (unsigned) GGML_CGRAPH_EVAL_ORDER_COUNT; ++order) { ctx_compute = ggml_init(ctx_compute_params); - alloc = ggml_allocr_new_measure(tensor_alignment); + ggml_gallocr_t alloc = ggml_gallocr_new(ggml_backend_cpu_buffer_type()); gf = ggml_new_graph_custom(ctx_compute, LLAMA_TRAIN_MAX_NODES, true); gf->order = (enum ggml_cgraph_eval_order) order; gb = ggml_new_graph_custom(ctx_compute, LLAMA_TRAIN_MAX_NODES, true); @@ -1140,9 +1091,10 @@ int main(int argc, char ** argv) { &logits, tokens_input, target_probs, n_tokens, n_batch, params.common.use_flash, - params.common.use_checkpointing + params.common.use_checkpointing, + true ); - size_t max_compute_size = ggml_allocr_max_size(alloc) + tensor_alignment; + size_t max_compute_size = ggml_gallocr_get_buffer_size(alloc, 0); // FIXME: this will still allocate the buffer if (max_compute_size < best_compute_size) { best_compute_size = max_compute_size; best_order = gf->order; @@ -1157,9 +1109,8 @@ int main(int argc, char ** argv) { "invalid"); // allocate compute tensors - mem_compute_data.resize(max_compute_size); ctx_compute = ggml_init(ctx_compute_params); - alloc = ggml_allocr_new(mem_compute_data.data(), mem_compute_data.size(), tensor_alignment); + ggml_gallocr_t alloc = ggml_gallocr_new(ggml_backend_cpu_buffer_type()); gf = ggml_new_graph_custom(ctx_compute, LLAMA_TRAIN_MAX_NODES, true); gf->order = best_order; gb = ggml_new_graph_custom(ctx_compute, LLAMA_TRAIN_MAX_NODES, true); @@ -1172,7 +1123,8 @@ int main(int argc, char ** argv) { &logits, tokens_input, target_probs, n_tokens, n_batch, params.common.use_flash, - params.common.use_checkpointing + params.common.use_checkpointing, + false ); std::vector train_tokens; diff --git a/ggml-alloc.c b/ggml-alloc.c index f9be6e1cb..c28c37c4f 100644 --- a/ggml-alloc.c +++ b/ggml-alloc.c @@ -17,397 +17,11 @@ //#define AT_PRINTF(...) fprintf(stderr, __VA_ARGS__) #define AT_PRINTF(...) -// TODO: GGML_PAD ? -static size_t aligned_offset(const void * buffer, size_t offset, size_t alignment) { - assert(alignment && !(alignment & (alignment - 1))); // power of 2 - size_t align = (alignment - (((uintptr_t)buffer + offset) % alignment)) % alignment; - return offset + align; -} -struct free_block { - void * addr; - size_t size; -}; - -struct ggml_tallocr { - struct ggml_backend_buffer * buffer; - bool buffer_owned; - void * base; - size_t alignment; - - int n_free_blocks; - struct free_block free_blocks[MAX_FREE_BLOCKS]; - - size_t max_size; - - bool measure; - -#ifdef GGML_ALLOCATOR_DEBUG - struct ggml_tensor * allocated_tensors[1024]; -#endif -}; - -#ifdef GGML_ALLOCATOR_DEBUG -static void add_allocated_tensor(ggml_tallocr_t alloc, struct ggml_tensor * tensor) { - for (int i = 0; i < 1024; i++) { - if (alloc->allocated_tensors[i] == NULL) { - alloc->allocated_tensors[i] = tensor; - return; - } - } - GGML_ASSERT(!"out of allocated_tensors"); -} -static void remove_allocated_tensor(ggml_tallocr_t alloc, struct ggml_tensor * tensor) { - for (int i = 0; i < 1024; i++) { - if (alloc->allocated_tensors[i] == tensor || - (alloc->allocated_tensors[i] != NULL && alloc->allocated_tensors[i]->data == tensor->data)) { - alloc->allocated_tensors[i] = NULL; - return; - } - } - printf("tried to free tensor %s not found\n", tensor->name); - GGML_ASSERT(!"tensor not found"); -} -#endif - -// check if a tensor is allocated by this buffer -static bool ggml_tallocr_is_own(ggml_tallocr_t alloc, const struct ggml_tensor * tensor) { - return tensor->buffer == alloc->buffer && (!tensor->view_src || tensor->view_src->buffer == alloc->buffer); -} - -static bool ggml_is_view(struct ggml_tensor * t) { +static bool ggml_is_view(const struct ggml_tensor * t) { return t->view_src != NULL; } -void ggml_tallocr_alloc(ggml_tallocr_t alloc, struct ggml_tensor * tensor) { - GGML_ASSERT(!ggml_is_view(tensor)); // views generally get data pointer from one of their sources - GGML_ASSERT(tensor->data == NULL); // avoid allocating tensor which already has memory allocated - - size_t size = ggml_backend_buffer_get_alloc_size(alloc->buffer, tensor); - size = aligned_offset(NULL, size, alloc->alignment); - - AT_PRINTF("%s: allocating %s (%zu bytes) - ", __func__, tensor->name, size); - - size_t max_avail = 0; - - // find the best fitting free block besides the last block - int best_fit_block = -1; - size_t best_fit_size = SIZE_MAX; - for (int i = 0; i < alloc->n_free_blocks - 1; i++) { - struct free_block * block = &alloc->free_blocks[i]; - max_avail = MAX(max_avail, block->size); - if (block->size >= size && block->size <= best_fit_size) { - best_fit_block = i; - best_fit_size = block->size; - } - } - - if (best_fit_block == -1) { - // the last block is our last resort - struct free_block * block = &alloc->free_blocks[alloc->n_free_blocks - 1]; - max_avail = MAX(max_avail, block->size); - if (block->size >= size) { - best_fit_block = alloc->n_free_blocks - 1; - } else { - fprintf(stderr, "%s: not enough space in the buffer to allocate %s (needed %zu, largest block available %zu)\n", - __func__, tensor->name, size, max_avail); - GGML_ASSERT(!"not enough space in the buffer"); - return; - } - } - - struct free_block * block = &alloc->free_blocks[best_fit_block]; - void * addr = block->addr; - block->addr = (char*)block->addr + size; - block->size -= size; - if (block->size == 0) { - // remove block if empty - alloc->n_free_blocks--; - for (int j = best_fit_block; j < alloc->n_free_blocks; j++) { - alloc->free_blocks[j] = alloc->free_blocks[j+1]; - } - } - - AT_PRINTF("block %d, addr %p\n", best_fit_block, addr); - - tensor->data = addr; - tensor->buffer = alloc->buffer; - if (!alloc->measure) { - ggml_backend_buffer_init_tensor(alloc->buffer, tensor); - } - -#ifdef GGML_ALLOCATOR_DEBUG - add_allocated_tensor(alloc, tensor); - size_t cur_max = (char*)addr - (char*)alloc->base + size; - if (cur_max > alloc->max_size) { - printf("max_size = %.2f MB: tensors: ", cur_max / 1024.0 / 1024.0); - for (int i = 0; i < 1024; i++) { - if (alloc->allocated_tensors[i]) { - printf("%s (%.2f MB) ", alloc->allocated_tensors[i]->name, ggml_nbytes(alloc->allocated_tensors[i]) / 1024.0 / 1024.0); - } - } - printf("\n"); - } -#endif - - alloc->max_size = MAX(alloc->max_size, (char*)addr - (char*)alloc->base + size); -} - -// this is a very naive implementation, but for our case the number of free blocks should be very small -static void ggml_tallocr_free_tensor(ggml_tallocr_t alloc, struct ggml_tensor * tensor) { - if (ggml_tallocr_is_own(alloc, tensor) == false) { - // the tensor was not allocated in this buffer - // this can happen because the graph allocator will try to free weights and other tensors from different buffers - // the easiest way to deal with this is just to ignore it - // AT_PRINTF("ignoring %s (their buffer: %p, our buffer: %p)\n", tensor->name, (void *)tensor->buffer, (void *)alloc->buffer); - return; - } - - void * ptr = tensor->data; - - size_t size = ggml_backend_buffer_get_alloc_size(alloc->buffer, tensor); - size = aligned_offset(NULL, size, alloc->alignment); - AT_PRINTF("%s: freeing %s at %p (%zu bytes) - n_free_blocks = %d\n", __func__, tensor->name, ptr, size, alloc->n_free_blocks); - -#ifdef GGML_ALLOCATOR_DEBUG - remove_allocated_tensor(alloc, tensor); -#endif - - // see if we can merge with an existing block - for (int i = 0; i < alloc->n_free_blocks; i++) { - struct free_block * block = &alloc->free_blocks[i]; - // check if ptr is at the end of the block - if ((char*)block->addr + block->size == ptr) { - block->size += size; - // check if we can merge with the next block - if (i < alloc->n_free_blocks - 1 && (char*)block->addr + block->size == alloc->free_blocks[i+1].addr) { - block->size += alloc->free_blocks[i+1].size; - alloc->n_free_blocks--; - for (int j = i+1; j < alloc->n_free_blocks; j++) { - alloc->free_blocks[j] = alloc->free_blocks[j+1]; - } - } - return; - } - // check if ptr is at the beginning of the block - if ((char*)ptr + size == block->addr) { - block->addr = ptr; - block->size += size; - // check if we can merge with the previous block - if (i > 0 && (char*)alloc->free_blocks[i-1].addr + alloc->free_blocks[i-1].size == block->addr) { - alloc->free_blocks[i-1].size += block->size; - alloc->n_free_blocks--; - for (int j = i; j < alloc->n_free_blocks; j++) { - alloc->free_blocks[j] = alloc->free_blocks[j+1]; - } - } - return; - } - } - // otherwise, add a new block - GGML_ASSERT(alloc->n_free_blocks < MAX_FREE_BLOCKS && "out of free blocks"); - // insert the new block in the correct position to keep the array sorted by address (to make merging blocks faster) - int insert_pos = 0; - while (insert_pos < alloc->n_free_blocks && alloc->free_blocks[insert_pos].addr < ptr) { - insert_pos++; - } - // shift all blocks from insert_pos onward to make room for the new block - for (int i = alloc->n_free_blocks; i > insert_pos; i--) { - alloc->free_blocks[i] = alloc->free_blocks[i-1]; - } - // insert the new block - alloc->free_blocks[insert_pos].addr = ptr; - alloc->free_blocks[insert_pos].size = size; - alloc->n_free_blocks++; -} - -void ggml_tallocr_reset(ggml_tallocr_t alloc) { - alloc->n_free_blocks = 1; - size_t align_offset = aligned_offset(alloc->base, 0, alloc->alignment); - alloc->free_blocks[0].addr = (char *)alloc->base + align_offset; - - if (alloc->measure) { - alloc->free_blocks[0].size = SIZE_MAX/2; // restrict maximum size of a measure allocator to half size_t max to avoid overflows - } else { - alloc->free_blocks[0].size = ggml_backend_buffer_get_size(alloc->buffer) - align_offset; - ggml_backend_buffer_reset(alloc->buffer); - } -} - -ggml_tallocr_t ggml_tallocr_new(void * data, size_t size, size_t alignment) { - struct ggml_backend_buffer * buffer = ggml_backend_cpu_buffer_from_ptr(data, size); - - ggml_tallocr_t alloc = (ggml_tallocr_t)malloc(sizeof(struct ggml_tallocr)); - - *alloc = (struct ggml_tallocr) { - /*.buffer = */ buffer, - /*.buffer_owned = */ true, - /*.base = */ ggml_backend_buffer_get_base(buffer), - /*.alignment = */ alignment, - /*.n_free_blocks = */ 0, - /*.free_blocks = */ {{0}}, - /*.max_size = */ 0, - /*.measure = */ false, -#ifdef GGML_ALLOCATOR_DEBUG - /*.allocated_tensors = */ {0}, -#endif - }; - - ggml_tallocr_reset(alloc); - - return alloc; -} - -ggml_tallocr_t ggml_tallocr_new_measure(size_t alignment) { - ggml_tallocr_t alloc = ggml_tallocr_new((void *)0x1000, SIZE_MAX/2, alignment); - alloc->measure = true; - - return alloc; -} - -ggml_tallocr_t ggml_tallocr_new_measure_from_buft(struct ggml_backend_buffer_type * buft) { - // create a backend buffer to get the correct tensor allocation sizes - ggml_backend_buffer_t buffer = ggml_backend_buft_alloc_buffer(buft, 1); - - // TODO: move alloc initialization to a common ggml_tallocr_new_impl function - ggml_tallocr_t alloc = ggml_tallocr_new_from_buffer(buffer); - alloc->buffer_owned = true; - alloc->measure = true; - ggml_tallocr_reset(alloc); - return alloc; -} - -ggml_tallocr_t ggml_tallocr_new_measure_from_backend(struct ggml_backend * backend) { - return ggml_tallocr_new_measure_from_buft(ggml_backend_get_default_buffer_type(backend)); -} - -ggml_tallocr_t ggml_tallocr_new_from_buft(struct ggml_backend_buffer_type * buft, size_t size) { - // create a backend buffer to get the correct tensor allocation sizes - ggml_backend_buffer_t buffer = ggml_backend_buft_alloc_buffer(buft, size); - ggml_tallocr_t alloc = ggml_tallocr_new_from_buffer(buffer); - alloc->buffer_owned = true; - return alloc; -} - -ggml_tallocr_t ggml_tallocr_new_from_backend(struct ggml_backend * backend, size_t size) { - return ggml_tallocr_new_from_buft(ggml_backend_get_default_buffer_type(backend), size); -} - -ggml_tallocr_t ggml_tallocr_new_from_buffer(struct ggml_backend_buffer * buffer) { - ggml_tallocr_t alloc = (ggml_tallocr_t)malloc(sizeof(struct ggml_tallocr)); - - *alloc = (struct ggml_tallocr) { - /*.buffer = */ buffer, - /*.buffer_owned = */ false, - /*.base = */ ggml_backend_buffer_get_base(buffer), - /*.alignment = */ ggml_backend_buffer_get_alignment(buffer), - /*.n_free_blocks = */ 0, - /*.free_blocks = */ {{0}}, - /*.max_size = */ 0, - /*.measure = */ false, -#ifdef GGML_ALLOCATOR_DEBUG - /*.allocated_tensors = */ {0}, -#endif - }; - - ggml_tallocr_reset(alloc); - - return alloc; -} - -struct ggml_backend_buffer * ggml_tallocr_get_buffer(ggml_tallocr_t alloc) { - return alloc->buffer; -} - -void ggml_tallocr_free(ggml_tallocr_t alloc) { - if (alloc == NULL) { - return; - } - - if (alloc->buffer_owned) { - ggml_backend_buffer_free(alloc->buffer); - } - free(alloc); -} - -bool ggml_tallocr_is_measure(ggml_tallocr_t alloc) { - return alloc->measure; -} - -size_t ggml_tallocr_max_size(ggml_tallocr_t alloc) { - // FIXME: changes in the tensor sizes compared to the measure graph may cause allocations to fail - // to avoid this, we add a 10% margin to the buffer size - return alloc->max_size + alloc->max_size/10; -} - -// graph allocator - -struct hash_node { - int n_children; - int n_views; -}; - -struct ggml_gallocr { - ggml_tallocr_t talloc; - struct ggml_hash_set hash_set; - struct hash_node * hash_values; - size_t hash_values_size; - ggml_tallocr_t * hash_allocs; - int * parse_seq; - int parse_seq_len; -}; - -ggml_gallocr_t ggml_gallocr_new(void) { - ggml_gallocr_t galloc = (ggml_gallocr_t)malloc(sizeof(struct ggml_gallocr)); - - *galloc = (struct ggml_gallocr) { - /*.talloc = */ NULL, - /*.hash_set = */ {0}, - /*.hash_values = */ NULL, - /*.hash_values_size = */ 0, - /*.hash_allocs = */ NULL, - /*.parse_seq = */ NULL, - /*.parse_seq_len = */ 0, - }; - - return galloc; -} - -void ggml_gallocr_free(ggml_gallocr_t galloc) { - if (galloc == NULL) { - return; - } - - if (galloc->hash_set.keys != NULL) { - free(galloc->hash_set.keys); - } - if (galloc->hash_values != NULL) { - free(galloc->hash_values); - } - if (galloc->hash_allocs != NULL) { - free(galloc->hash_allocs); - } - if (galloc->parse_seq != NULL) { - free(galloc->parse_seq); - } - free(galloc); -} - -void ggml_gallocr_set_parse_seq(ggml_gallocr_t galloc, const int * list, int n) { - free(galloc->parse_seq); - galloc->parse_seq = malloc(sizeof(int) * n); - - for (int i = 0; i < n; i++) { - galloc->parse_seq[i] = list[i]; - } - galloc->parse_seq_len = n; -} - -static struct hash_node * hash_get(ggml_gallocr_t galloc, struct ggml_tensor * t) { - size_t i = ggml_hash_find_or_insert(galloc->hash_set, t); - return &galloc->hash_values[i]; -} - static bool ggml_are_same_layout(const struct ggml_tensor * a, const struct ggml_tensor * b) { if (a->type != b->type) { return false; @@ -447,106 +61,511 @@ static bool ggml_op_can_inplace(enum ggml_op op) { } } -static ggml_tallocr_t node_tallocr(ggml_gallocr_t galloc, struct ggml_tensor * node) { - if (galloc->talloc != NULL) { - return galloc->talloc; - } - - return galloc->hash_allocs[ggml_hash_find_or_insert(galloc->hash_set, node)]; +// TODO: GGML_PAD ? +static size_t aligned_offset(const void * buffer, size_t offset, size_t alignment) { + assert(alignment && !(alignment & (alignment - 1))); // power of 2 + size_t align = (alignment - (((uintptr_t)buffer + offset) % alignment)) % alignment; + return offset + align; } -static void init_view(ggml_gallocr_t galloc, struct ggml_tensor * view, bool update_backend) { - ggml_tallocr_t alloc = node_tallocr(galloc, view); +// tallocr +struct ggml_tallocr { + ggml_backend_buffer_t buffer; + void * base; + size_t alignment; + size_t offset; +}; - GGML_ASSERT(view->view_src != NULL && view->view_src->data != NULL); - if (update_backend) { - view->backend = view->view_src->backend; +ggml_tallocr_t ggml_tallocr_new(ggml_backend_buffer_t buffer) { + ggml_tallocr_t talloc = malloc(sizeof(struct ggml_tallocr)); + if (talloc == NULL) { + return NULL; } - // views are initialized in the alloc buffer rather than the view_src buffer - view->buffer = alloc->buffer; - view->data = (char *)view->view_src->data + view->view_offs; - assert(ggml_tallocr_is_measure(alloc) || !view->buffer || view->buffer->buft == alloc->buffer->buft); + void * base = ggml_backend_buffer_get_base(buffer); + size_t align = ggml_backend_buffer_get_alignment(buffer); - if (!alloc->measure) { - ggml_backend_buffer_init_tensor(alloc->buffer, view); - } + assert(align && !(align & (align - 1))); // power of 2 + + *talloc = (struct ggml_tallocr) { + /*.buffer = */ buffer, + /*.base = */ base, + /*.alignment = */ align, + /*.offset = */ aligned_offset(base, 0, align), + }; + return talloc; } -static void allocate_node(ggml_gallocr_t galloc, struct ggml_tensor * node) { - ggml_tallocr_t alloc = node_tallocr(galloc, node); +void ggml_tallocr_free(ggml_tallocr_t talloc) { + free(talloc); +} - if (node->data == NULL) { - if (ggml_is_view(node)) { - init_view(galloc, node, true); +void ggml_tallocr_alloc(ggml_tallocr_t talloc, struct ggml_tensor * tensor) { + size_t size = ggml_backend_buffer_get_alloc_size(talloc->buffer, tensor); + size = GGML_PAD(size, talloc->alignment); + + if (talloc->offset + size > ggml_backend_buffer_get_size(talloc->buffer)) { + fprintf(stderr, "%s: not enough space in the buffer to allocate %s (needed %zu, available %zu)\n", + __func__, tensor->name, size, ggml_backend_buffer_get_size(talloc->buffer) - talloc->offset); + GGML_ASSERT(!"not enough space in the buffer"); + return; + } + + void * addr = (char *)ggml_backend_buffer_get_base(talloc->buffer) + talloc->offset; + talloc->offset += size; + + assert(((uintptr_t)addr % talloc->alignment) == 0); + + ggml_backend_tensor_alloc(talloc->buffer, tensor, addr); +} + +// dynamic tensor allocator + +struct free_block { + size_t offset; + size_t size; +}; + +struct ggml_dyn_tallocr { + size_t alignment; + int n_free_blocks; + struct free_block free_blocks[MAX_FREE_BLOCKS]; + size_t max_size; + +#ifdef GGML_ALLOCATOR_DEBUG + struct { + const struct ggml_tensor * tensor; + size_t offset; + } allocated_tensors[1024]; +#endif +}; + +#ifdef GGML_ALLOCATOR_DEBUG +static void add_allocated_tensor(struct ggml_dyn_tallocr * alloc, size_t offset, const struct ggml_tensor * tensor) { + for (int i = 0; i < 1024; i++) { + if (alloc->allocated_tensors[i].tensor == NULL) { + alloc->allocated_tensors[i].tensor = tensor; + alloc->allocated_tensors[i].offset = offset; + return; + } + } + GGML_ASSERT(!"out of allocated_tensors"); +} +static void remove_allocated_tensor(struct ggml_dyn_tallocr * alloc, size_t offset, const struct ggml_tensor * tensor) { + for (int i = 0; i < 1024; i++) { + if (alloc->allocated_tensors[i].offset == offset) { + alloc->allocated_tensors[i].tensor = NULL; + return; + } + } + fprintf(stderr, "tried to free tensor %s not found\n", tensor->name); + GGML_ASSERT(!"tensor not found"); +} +#endif + +static size_t ggml_dyn_tallocr_alloc(struct ggml_dyn_tallocr * alloc, size_t size, const struct ggml_tensor * tensor) { + size = aligned_offset(NULL, size, alloc->alignment); + + AT_PRINTF("%s: allocating %s (%zu bytes) - ", __func__, tensor->name, size); + + size_t max_avail = 0; + + // find the best fitting free block besides the last block + int best_fit_block = -1; + size_t best_fit_size = SIZE_MAX; + for (int i = 0; i < alloc->n_free_blocks - 1; i++) { + struct free_block * block = &alloc->free_blocks[i]; + max_avail = MAX(max_avail, block->size); + if (block->size >= size && block->size <= best_fit_size) { + best_fit_block = i; + best_fit_size = block->size; + } + } + + if (best_fit_block == -1) { + // the last block is our last resort + struct free_block * block = &alloc->free_blocks[alloc->n_free_blocks - 1]; + max_avail = MAX(max_avail, block->size); + if (block->size >= size) { + best_fit_block = alloc->n_free_blocks - 1; } else { - // see if we can reuse a parent's buffer (inplace) - if (ggml_op_can_inplace(node->op)) { - for (int i = 0; i < GGML_MAX_SRC; i++) { - struct ggml_tensor * parent = node->src[i]; - if (parent == NULL) { - break; - } + // this should never happen + fprintf(stderr, "%s: not enough space in the buffer to allocate %zu bytes, largest block available %zu bytes\n", + __func__, size, max_avail); + GGML_ASSERT(!"not enough space in the buffer"); + GGML_UNREACHABLE(); + } + } - // if the node's data is external, then we cannot re-use it - if (ggml_tallocr_is_own(alloc, parent) == false) { - AT_PRINTF("not reusing parent %s for %s as %p is external\n", parent->name, node->name, parent->data); - continue; - } + struct free_block * block = &alloc->free_blocks[best_fit_block]; + size_t offset = block->offset; + block->offset = offset + size; + block->size -= size; + if (block->size == 0) { + // remove block if empty + alloc->n_free_blocks--; + for (int j = best_fit_block; j < alloc->n_free_blocks; j++) { + alloc->free_blocks[j] = alloc->free_blocks[j+1]; + } + } - struct hash_node * p_hn = hash_get(galloc, parent); - if (parent->data != NULL && p_hn->n_children == 1 && p_hn->n_views == 0 && ggml_are_same_layout(node, parent)) { - if (ggml_is_view(parent)) { - struct ggml_tensor * view_src = parent->view_src; - struct hash_node * view_src_hn = hash_get(galloc, view_src); - if (view_src_hn->n_views == 1 && view_src_hn->n_children == 0 && view_src->data == parent->data) { - // TODO: the offset of the view parent must be kept to ensure that the op doesn't overwrite - // the parent's data that it will need later (same layout requirement). the problem is that then - // we cannot free the tensor because the original address of the allocation is lost. - // adding a view_src pointer to the tensor would solve this and simplify the code dealing with views - // for now, we only reuse the parent's data if the offset is zero (view_src->data == parent->data) - AT_PRINTF("reusing view parent %s (%s) for %s\n", parent->name, view_src->name, node->name); - node->view_src = view_src; - view_src_hn->n_views += 1; - init_view(galloc, node, false); - return; - } - } else { - AT_PRINTF("reusing parent %s for %s\n", parent->name, node->name); - node->view_src = parent; - p_hn->n_views += 1; - init_view(galloc, node, false); + AT_PRINTF("block %d, offset %zu\n", best_fit_block, offset); + +#ifdef GGML_ALLOCATOR_DEBUG + add_allocated_tensor(alloc, offset, tensor); + size_t cur_max = offset + size; + if (cur_max > alloc->max_size) { + // sort allocated_tensors by offset + for (int i = 0; i < 1024; i++) { + for (int j = i + 1; j < 1024; j++) { + if (alloc->allocated_tensors[i].offset > alloc->allocated_tensors[j].offset) { + const struct ggml_tensor * tmp_tensor = alloc->allocated_tensors[i].tensor; + size_t tmp_offset = alloc->allocated_tensors[i].offset; + alloc->allocated_tensors[i].tensor = alloc->allocated_tensors[j].tensor; + alloc->allocated_tensors[i].offset = alloc->allocated_tensors[j].offset; + alloc->allocated_tensors[j].tensor = tmp_tensor; + alloc->allocated_tensors[j].offset = tmp_offset; + } + } + } + fprintf(stderr, "max_size = %.2f MB: tensors: ", cur_max / 1024.0 / 1024.0); + for (int i = 0; i < 1024; i++) { + if (alloc->allocated_tensors[i].tensor) { + fprintf(stderr, "%s [%zx-%zx] (%.2f MB) ", alloc->allocated_tensors[i].tensor->name, + alloc->allocated_tensors[i].offset, + alloc->allocated_tensors[i].offset + ggml_nbytes(alloc->allocated_tensors[i].tensor), + ggml_nbytes(alloc->allocated_tensors[i].tensor) / 1024.0 / 1024.0); + } + } + fprintf(stderr, "\n"); + } +#endif + + alloc->max_size = MAX(alloc->max_size, offset + size); + + return offset; + + GGML_UNUSED(tensor); +} + +// this is a very naive implementation, but for our case the number of free blocks should be very small +static void ggml_dyn_tallocr_free_tensor(struct ggml_dyn_tallocr * alloc, size_t offset, size_t size, const struct ggml_tensor * tensor) { + size = aligned_offset(NULL, size, alloc->alignment); + + AT_PRINTF("%s: freeing %s at %zu (%zu bytes) - n_free_blocks = %d\n", __func__, tensor->name, offset, size, alloc->n_free_blocks); + +#ifdef GGML_ALLOCATOR_DEBUG + remove_allocated_tensor(alloc, offset, tensor); +#endif + + // see if we can merge with an existing block + for (int i = 0; i < alloc->n_free_blocks; i++) { + struct free_block * block = &alloc->free_blocks[i]; + // check if ptr is at the end of the block + if (block->offset + block->size == offset) { + block->size += size; + // check if we can merge with the next block + if (i < alloc->n_free_blocks - 1 && block->offset + block->size == alloc->free_blocks[i+1].offset) { + block->size += alloc->free_blocks[i+1].size; + alloc->n_free_blocks--; + for (int j = i+1; j < alloc->n_free_blocks; j++) { + alloc->free_blocks[j] = alloc->free_blocks[j+1]; + } + } + return; + } + // check if ptr is at the beginning of the block + if (offset + size == block->offset) { + block->offset = offset; + block->size += size; + // check if we can merge with the previous block + if (i > 0 && alloc->free_blocks[i-1].offset + alloc->free_blocks[i-1].size == block->offset) { + alloc->free_blocks[i-1].size += block->size; + alloc->n_free_blocks--; + for (int j = i; j < alloc->n_free_blocks; j++) { + alloc->free_blocks[j] = alloc->free_blocks[j+1]; + } + } + return; + } + } + // otherwise, add a new block + GGML_ASSERT(alloc->n_free_blocks < MAX_FREE_BLOCKS && "out of free blocks"); + // insert the new block in the correct position to keep the array sorted by address (to make merging blocks faster) + int insert_pos = 0; + while (insert_pos < alloc->n_free_blocks && alloc->free_blocks[insert_pos].offset < offset) { + insert_pos++; + } + // shift all blocks from insert_pos onward to make room for the new block + for (int i = alloc->n_free_blocks; i > insert_pos; i--) { + alloc->free_blocks[i] = alloc->free_blocks[i-1]; + } + // insert the new block + alloc->free_blocks[insert_pos].offset = offset; + alloc->free_blocks[insert_pos].size = size; + alloc->n_free_blocks++; + + GGML_UNUSED(tensor); +} + +static void ggml_dyn_tallocr_reset(struct ggml_dyn_tallocr * alloc) { + alloc->n_free_blocks = 1; + alloc->free_blocks[0].offset = 0; + alloc->free_blocks[0].size = SIZE_MAX/2; // restrict maximum size of a measure allocator to half size_t max to avoid overflows + alloc->max_size = 0; +} + +static struct ggml_dyn_tallocr * ggml_dyn_tallocr_new(size_t alignment) { + struct ggml_dyn_tallocr * alloc = (struct ggml_dyn_tallocr *)malloc(sizeof(struct ggml_dyn_tallocr)); + + *alloc = (struct ggml_dyn_tallocr) { + /*.alignment = */ alignment, + /*.n_free_blocks = */ 0, + /*.free_blocks = */ {{0}}, + /*.max_size = */ 0, +#ifdef GGML_ALLOCATOR_DEBUG + /*.allocated_tensors = */ {{0}}, +#endif + }; + + ggml_dyn_tallocr_reset(alloc); + + return alloc; +} + +static void ggml_dyn_tallocr_free(struct ggml_dyn_tallocr * alloc) { + free(alloc); +} + +static size_t ggml_dyn_tallocr_max_size(struct ggml_dyn_tallocr * alloc) { + return alloc->max_size; +} + + +///////////////////////////////////// + +// graph allocator + +struct hash_node { + int n_children; + int n_views; + int buffer_id; + size_t offset; // offset within the buffer + bool allocated; +}; + +// +struct tensor_alloc { + size_t offset; + size_t size_max; // 0 = pre-allocated, unused, or view +}; + +struct node_alloc { + int buffer_id; + struct tensor_alloc dst; + struct tensor_alloc src[GGML_MAX_SRC]; +}; + +struct ggml_gallocr { + ggml_backend_buffer_type_t * bufts; // [n_buffers] + ggml_backend_buffer_t * buffers; // [n_buffers] + struct ggml_dyn_tallocr ** buf_tallocs; // [n_buffers] + int n_buffers; + + struct ggml_hash_set hash_set; + struct hash_node * hash_values; // [hash_set.size] + + struct node_alloc * node_allocs; // [n_nodes] + int n_nodes; +}; + +ggml_gallocr_t ggml_gallocr_new_n(ggml_backend_buffer_type_t * bufts, int n_bufs) { + ggml_gallocr_t galloc = (ggml_gallocr_t)calloc(sizeof(struct ggml_gallocr), 1); + GGML_ASSERT(galloc != NULL); + + galloc->bufts = calloc(sizeof(ggml_backend_buffer_type_t) * n_bufs, 1); + GGML_ASSERT(galloc->bufts != NULL); + + galloc->buffers = calloc(sizeof(ggml_backend_buffer_t) * n_bufs, 1); + GGML_ASSERT(galloc->buffers != NULL); + + galloc->buf_tallocs = calloc(sizeof(struct ggml_dyn_tallocr *) * n_bufs, 1); + GGML_ASSERT(galloc->buf_tallocs != NULL); + + for (int i = 0; i < n_bufs; i++) { + galloc->bufts[i] = bufts[i]; + galloc->buffers[i] = NULL; + size_t alignment = ggml_backend_buft_get_alignment(bufts[i]); + galloc->buf_tallocs[i] = ggml_dyn_tallocr_new(alignment); + } + galloc->n_buffers = n_bufs; + + return galloc; +} + +ggml_gallocr_t ggml_gallocr_new(ggml_backend_buffer_type_t buft) { + return ggml_gallocr_new_n(&buft, 1); +} + +void ggml_gallocr_free(ggml_gallocr_t galloc) { + if (galloc == NULL) { + return; + } + + for (int i = 0; i < galloc->n_buffers; i++) { + if (galloc->buffers != NULL) { + ggml_backend_buffer_free(galloc->buffers[i]); + } + if (galloc->buf_tallocs != NULL) { + ggml_dyn_tallocr_free(galloc->buf_tallocs[i]); + } + } + + free(galloc->hash_set.keys); + free(galloc->hash_values); + free(galloc->bufts); + free(galloc->buffers); + free(galloc->buf_tallocs); + free(galloc->node_allocs); + free(galloc); +} + +typedef struct ggml_gallocr * ggml_gallocr_t; + +static struct hash_node * ggml_gallocr_hash_get(ggml_gallocr_t galloc, struct ggml_tensor * t) { + size_t i = ggml_hash_find_or_insert(galloc->hash_set, t); + return &galloc->hash_values[i]; +} + +static bool ggml_gallocr_is_own(ggml_gallocr_t galloc, struct ggml_tensor * t) { + return ggml_gallocr_hash_get(galloc, t)->allocated; +} + +static void ggml_gallocr_set_node_offset(ggml_gallocr_t galloc, struct ggml_tensor * node, int buffer_id, size_t offset) { + struct hash_node * hn = ggml_gallocr_hash_get(galloc, node); + hn->buffer_id = buffer_id; + hn->offset = offset; + hn->allocated = true; +} + +static bool ggml_gallocr_is_allocated(ggml_gallocr_t galloc, struct ggml_tensor * t) { + return t->data != NULL || ggml_gallocr_hash_get(galloc, t)->allocated; +} + +static void ggml_gallocr_allocate_node(ggml_gallocr_t galloc, struct ggml_tensor * node, int buffer_id) { + struct hash_node * hn = ggml_gallocr_hash_get(galloc, node); + + if (!ggml_gallocr_is_allocated(galloc, node) && !ggml_is_view(node)) { + hn->allocated = true; + assert(hn->offset == 0); + + // try to reuse a parent's buffer (inplace) + if (ggml_op_can_inplace(node->op)) { + for (int i = 0; i < GGML_MAX_SRC; i++) { + struct ggml_tensor * parent = node->src[i]; + if (parent == NULL) { + break; + } + + // if the node's data is external, then we cannot re-use it + if (!ggml_gallocr_is_own(galloc, parent)) { + AT_PRINTF("not reusing parent %s for %s as %p is external\n", parent->name, node->name, parent->data); + continue; + } + + // outputs cannot be reused + if (parent->flags & GGML_TENSOR_FLAG_OUTPUT || (parent->view_src != NULL && parent->view_src->flags & GGML_TENSOR_FLAG_OUTPUT)) { + AT_PRINTF("not reusing parent %s for %s as it is an output\n", parent->name, node->name); + continue; + } + + if (!ggml_are_same_layout(node, parent)) { + AT_PRINTF("not reusing parent %s for %s as layouts are different\n", parent->name, node->name); + continue; + } + + struct hash_node * p_hn = ggml_gallocr_hash_get(galloc, parent); + if (p_hn->n_children == 1 && p_hn->n_views == 0) { + if (ggml_is_view(parent)) { + struct ggml_tensor * view_src = parent->view_src; + struct hash_node * view_src_hn = ggml_gallocr_hash_get(galloc, view_src); + if (view_src_hn->n_views == 1 && view_src_hn->n_children == 0 && view_src->data == parent->data) { + AT_PRINTF("reusing view parent %s (%s) for %s\n", parent->name, view_src->name, node->name); + assert(view_src_hn->offset == p_hn->offset); + hn->buffer_id = p_hn->buffer_id; + hn->offset = p_hn->offset; + p_hn->allocated = false; // avoid freeing the parent + view_src_hn->allocated = false; return; } + } else { + AT_PRINTF("reusing parent %s for %s\n", parent->name, node->name); + hn->buffer_id = p_hn->buffer_id; + hn->offset = p_hn->offset; + p_hn->allocated = false; // avoid freeing the parent + return; } } } - ggml_tallocr_alloc(alloc, node); } + // allocate tensor from the buffer + struct ggml_dyn_tallocr * alloc = galloc->buf_tallocs[buffer_id]; + ggml_backend_buffer_type_t buft = galloc->bufts[buffer_id]; + size_t size = ggml_backend_buft_get_alloc_size(buft, node); + size_t offset = ggml_dyn_tallocr_alloc(alloc, size, node); + hn->buffer_id = buffer_id; + hn->offset = offset; + return; } } -static void free_node(ggml_gallocr_t galloc, struct ggml_tensor * node) { - ggml_tallocr_t alloc = node_tallocr(galloc, node); +static void ggml_gallocr_free_node(ggml_gallocr_t galloc, struct ggml_tensor * node, int buffer_id) { + // graph outputs are never freed + if (node->flags & GGML_TENSOR_FLAG_OUTPUT) { + AT_PRINTF("not freeing output %s\n", node->name); + return; + } - ggml_tallocr_free_tensor(alloc, node); + struct ggml_dyn_tallocr * alloc = galloc->buf_tallocs[buffer_id]; + ggml_backend_buffer_type_t buft = galloc->bufts[buffer_id]; + struct hash_node * hn = ggml_gallocr_hash_get(galloc, node); + size_t offset = hn->offset; + size_t size = ggml_backend_buft_get_alloc_size(buft, node); + ggml_dyn_tallocr_free_tensor(alloc, offset, size, node); + hn->allocated = false; } -static void ggml_tallocr_alloc_graph_impl(ggml_gallocr_t galloc, struct ggml_cgraph * gf) { - const int * parse_seq = galloc->parse_seq; - int parse_seq_len = galloc->parse_seq_len; +static int get_node_buffer_id(const int * node_buffer_ids, int i) { + return node_buffer_ids ? node_buffer_ids[i] : 0; +} + +static void ggml_gallocr_alloc_graph_impl(ggml_gallocr_t galloc, struct ggml_cgraph * graph, const int * node_buffer_ids) { + // clear hash tables + memset(galloc->hash_set.keys, 0, galloc->hash_set.size * sizeof(struct ggml_tensor *)); + memset(galloc->hash_values, 0, galloc->hash_set.size * sizeof(struct hash_node)); + + // allocate all graph inputs first to avoid overwriting them + for (int i = 0; i < graph->n_nodes; i++) { + if (graph->nodes[i]->flags & GGML_TENSOR_FLAG_INPUT) { + ggml_gallocr_allocate_node(galloc, graph->nodes[i], get_node_buffer_id(node_buffer_ids, i)); + } + for (int j = 0; j < GGML_MAX_SRC; j++) { + if (graph->nodes[i]->src[j] == NULL) { + break; + } + if (graph->nodes[i]->src[j]->flags & GGML_TENSOR_FLAG_INPUT) { + ggml_gallocr_allocate_node(galloc, graph->nodes[i]->src[j], get_node_buffer_id(node_buffer_ids, i)); + } + } + } // count number of children and views - for (int i = 0; i < gf->n_nodes; i++) { - struct ggml_tensor * node = gf->nodes[i]; + for (int i = 0; i < graph->n_nodes; i++) { + struct ggml_tensor * node = graph->nodes[i]; if (ggml_is_view(node)) { struct ggml_tensor * view_src = node->view_src; - hash_get(galloc, view_src)->n_views += 1; - if (node->buffer == NULL && node->data != NULL) { - // view of a pre-allocated tensor, didn't call init_view() yet - init_view(galloc, node, true); - } + ggml_gallocr_hash_get(galloc, view_src)->n_views += 1; } for (int j = 0; j < GGML_MAX_SRC; j++) { @@ -554,227 +573,283 @@ static void ggml_tallocr_alloc_graph_impl(ggml_gallocr_t galloc, struct ggml_cgr if (parent == NULL) { break; } - hash_get(galloc, parent)->n_children += 1; - if (ggml_is_view(parent) && parent->buffer == NULL && parent->data != NULL) { - init_view(galloc, parent, true); - } + ggml_gallocr_hash_get(galloc, parent)->n_children += 1; } } // allocate tensors - // if we have parse_seq then we allocate nodes following the list, and we only free nodes at barriers - int last_barrier_pos = 0; - int n_nodes = parse_seq_len ? parse_seq_len : gf->n_nodes; + for (int i = 0; i < graph->n_nodes; i++) { + struct ggml_tensor * node = graph->nodes[i]; + int buffer_id = get_node_buffer_id(node_buffer_ids, i); - for (int ind = 0; ind < n_nodes; ind++) { - // allocate a node if there is no parse_seq or this is not a barrier - if (parse_seq_len == 0 || parse_seq[ind] != -1) { - int i = parse_seq_len ? parse_seq[ind] : ind; - struct ggml_tensor * node = gf->nodes[i]; - - // allocate parents (leafs) - for (int j = 0; j < GGML_MAX_SRC; j++) { - struct ggml_tensor * parent = node->src[j]; - if (parent == NULL) { - break; - } - allocate_node(galloc, parent); + // allocate parents (only leafs need to be allocated at this point) + for (int j = 0; j < GGML_MAX_SRC; j++) { + struct ggml_tensor * parent = node->src[j]; + if (parent == NULL) { + break; } - - // allocate node - allocate_node(galloc, node); - - AT_PRINTF("exec: %s (%s) <= ", ggml_op_name(node->op), node->name); - for (int j = 0; j < GGML_MAX_SRC; j++) { - struct ggml_tensor * parent = node->src[j]; - if (parent == NULL) { - break; - } - AT_PRINTF("%s", parent->name); - if (j < GGML_MAX_SRC - 1 && node->src[j + 1] != NULL) { - AT_PRINTF(", "); - } - } - AT_PRINTF("\n"); + ggml_gallocr_allocate_node(galloc, parent, buffer_id); } + // allocate node + ggml_gallocr_allocate_node(galloc, node, buffer_id); + + AT_PRINTF("exec: %s (%s) <= ", ggml_op_desc(node), node->name); + for (int j = 0; j < GGML_MAX_SRC; j++) { + struct ggml_tensor * parent = node->src[j]; + if (parent == NULL) { + break; + } + AT_PRINTF("%s", parent->name); + if (j < GGML_MAX_SRC - 1 && node->src[j + 1] != NULL) { + AT_PRINTF(", "); + } + } + AT_PRINTF("\n"); + // update parents - // update immediately if there is no parse_seq - // update only at barriers if there is parse_seq - if ((parse_seq_len == 0) || parse_seq[ind] == -1) { - int update_start = parse_seq_len ? last_barrier_pos : ind; - int update_end = parse_seq_len ? ind : ind + 1; - for (int i = update_start; i < update_end; i++) { - int node_i = parse_seq_len ? parse_seq[i] : i; - struct ggml_tensor * node = gf->nodes[node_i]; + for (int j = 0; j < GGML_MAX_SRC; j++) { + struct ggml_tensor * parent = node->src[j]; + if (parent == NULL) { + break; + } + struct hash_node * p_hn = ggml_gallocr_hash_get(galloc, parent); + p_hn->n_children -= 1; - for (int j = 0; j < GGML_MAX_SRC; j++) { - struct ggml_tensor * parent = node->src[j]; - if (parent == NULL) { - break; - } - struct hash_node * p_hn = hash_get(galloc, parent); - p_hn->n_children -= 1; - - //AT_PRINTF("parent %s: %d children, %d views\n", parent->name, parent->n_children, parent->n_views); - - if (p_hn->n_children == 0 && p_hn->n_views == 0) { - if (ggml_is_view(parent)) { - struct ggml_tensor * view_src = parent->view_src; - struct hash_node * view_src_hn = hash_get(galloc, view_src); - view_src_hn->n_views -= 1; - AT_PRINTF("view_src %s: %d children, %d views\n", view_src->name, view_src_hn->n_children, view_src_hn->n_views); - if (view_src_hn->n_views == 0 && view_src_hn->n_children == 0) { - free_node(galloc, view_src); - } - } - else { - free_node(galloc, parent); - } + AT_PRINTF("parent %s: %d children, %d views, allocated: %d\n", + parent->name, p_hn->n_children, p_hn->n_views, p_hn->allocated); + + if (p_hn->n_children == 0 && p_hn->n_views == 0) { + if (ggml_is_view(parent)) { + struct ggml_tensor * view_src = parent->view_src; + struct hash_node * view_src_hn = ggml_gallocr_hash_get(galloc, view_src); + view_src_hn->n_views -= 1; + AT_PRINTF("view_src %s: %d children, %d views\n", + view_src->name, view_src_hn->n_children, view_src_hn->n_views); + if (view_src_hn->n_views == 0 && view_src_hn->n_children == 0 && view_src_hn->allocated) { + ggml_gallocr_free_node(galloc, view_src, buffer_id); } } + else if (p_hn->allocated) { + ggml_gallocr_free_node(galloc, parent, buffer_id); + } } AT_PRINTF("\n"); - if (parse_seq_len) { - last_barrier_pos = ind + 1; + } + } +} + +bool ggml_gallocr_reserve_n(ggml_gallocr_t galloc, struct ggml_cgraph * graph, const int * node_buffer_ids) { + size_t hash_size = graph->visited_hash_table.size; + + // initialize hash table + if (galloc->hash_set.size < hash_size) { + free(galloc->hash_set.keys); + free(galloc->hash_values); + galloc->hash_set.size = hash_size; + galloc->hash_set.keys = calloc(sizeof(struct ggml_tensor *), hash_size); + galloc->hash_values = calloc(sizeof(struct hash_node), hash_size); + GGML_ASSERT(galloc->hash_set.keys != NULL); + GGML_ASSERT(galloc->hash_values != NULL); + } else { + // reset hash table + memset(galloc->hash_set.keys, 0, sizeof(struct ggml_tensor *) * galloc->hash_set.size); + memset(galloc->hash_values, 0, sizeof(struct hash_node) * galloc->hash_set.size); + } + + // reset allocators + for (int i = 0; i < galloc->n_buffers; i++) { + ggml_dyn_tallocr_reset(galloc->buf_tallocs[i]); + } + + // allocate in hash table + ggml_gallocr_alloc_graph_impl(galloc, graph, node_buffer_ids); + + // set the node_allocs from the hash table + if (galloc->n_nodes < graph->n_nodes) { + free(galloc->node_allocs); + galloc->node_allocs = calloc(sizeof(struct node_alloc), graph->n_nodes); + GGML_ASSERT(galloc->node_allocs != NULL); + } + galloc->n_nodes = graph->n_nodes; + for (int i = 0; i < graph->n_nodes; i++) { + struct ggml_tensor * node = graph->nodes[i]; + struct node_alloc * node_alloc = &galloc->node_allocs[i]; + node_alloc->buffer_id = get_node_buffer_id(node_buffer_ids, i); + if (node->view_src || node->data) { + node_alloc->dst.offset = SIZE_MAX; + node_alloc->dst.size_max = 0; + } else { + struct hash_node * hn = ggml_gallocr_hash_get(galloc, node); + node_alloc->dst.offset = hn->offset; + node_alloc->dst.size_max = ggml_backend_buft_get_alloc_size(galloc->bufts[hn->buffer_id], node); + } + for (int j = 0; j < GGML_MAX_SRC; j++) { + struct ggml_tensor * src = node->src[j]; + if (!src || src->view_src || src->data) { + node_alloc->src[j].offset = SIZE_MAX; + node_alloc->src[j].size_max = 0; + } else { + struct hash_node * hn = ggml_gallocr_hash_get(galloc, src); + node_alloc->src[j].offset = hn->offset; + node_alloc->src[j].size_max = ggml_backend_buft_get_alloc_size(galloc->bufts[hn->buffer_id], src); } } } -} -size_t ggml_gallocr_alloc_graph(ggml_gallocr_t galloc, ggml_tallocr_t talloc, struct ggml_cgraph * graph) { - size_t hash_size = graph->visited_hash_table.size; + // reallocate buffers if needed + for (int i = 0; i < galloc->n_buffers; i++) { + size_t cur_size = galloc->buffers[i] ? ggml_backend_buffer_get_size(galloc->buffers[i]) : 0; + size_t new_size = ggml_dyn_tallocr_max_size(galloc->buf_tallocs[i]); - // check if the hash table is initialized and large enough - if (galloc->hash_set.size < hash_size) { - if (galloc->hash_set.keys != NULL) { - free(galloc->hash_set.keys); + if (new_size > cur_size) { +#ifndef NDEBUG + fprintf(stderr, "%s: reallocating %s buffer from size %.02f MiB to %.02f MiB\n", __func__, ggml_backend_buft_name(galloc->bufts[i]), cur_size / 1024.0 / 1024.0, new_size / 1024.0 / 1024.0); +#endif + ggml_backend_buffer_free(galloc->buffers[i]); + galloc->buffers[i] = ggml_backend_buft_alloc_buffer(galloc->bufts[i], new_size); + if (galloc->buffers[i] == NULL) { + fprintf(stderr, "%s: failed to allocate %s buffer of size %zu\n", __func__, ggml_backend_buft_name(galloc->bufts[i]), new_size); + return false; + } } - if (galloc->hash_values != NULL) { - free(galloc->hash_values); + } + + return true; +} + +bool ggml_gallocr_reserve(ggml_gallocr_t galloc, struct ggml_cgraph *graph) { + return ggml_gallocr_reserve_n(galloc, graph, NULL); +} + +static void ggml_gallocr_init_tensor(ggml_gallocr_t galloc, struct ggml_tensor * node, struct node_alloc * node_alloc, struct tensor_alloc * tensor_alloc) { + assert(node->data || node->view_src || ggml_backend_buffer_get_alloc_size(galloc->buffers[node_alloc->buffer_id], node) <= tensor_alloc->size_max); + + if (node->view_src != NULL) { + if (node->buffer == NULL) { + assert(tensor_alloc->offset == SIZE_MAX); + if (node->view_src->buffer == NULL) { + // this tensor was allocated without ggml-backend + return; + } + ggml_backend_view_init(galloc->buffers[node_alloc->buffer_id], node); } - galloc->hash_set.keys = malloc(sizeof(struct ggml_tensor *) * hash_size); - galloc->hash_set.size = hash_size; - galloc->hash_values = malloc(sizeof(struct hash_node) * hash_size); + } else { + if (node->data == NULL) { + assert(tensor_alloc->offset != SIZE_MAX); + assert(ggml_backend_buffer_get_alloc_size(galloc->buffers[node_alloc->buffer_id], node) <= tensor_alloc->size_max); + void * base = ggml_backend_buffer_get_base(galloc->buffers[node_alloc->buffer_id]); + void * addr = (char *)base + tensor_alloc->offset; + ggml_backend_tensor_alloc(galloc->buffers[node_alloc->buffer_id], node, addr); + } else { + if (node->buffer == NULL) { + // this tensor was allocated without ggml-backend + return; + } + +#ifndef NDEBUG + size_t offset = + (char *)node->data - + (char *)ggml_backend_buffer_get_base(node->buffer); + size_t size = ggml_backend_buffer_get_alloc_size(node->buffer, node); + assert(tensor_alloc->offset == SIZE_MAX || offset == tensor_alloc->offset); + assert(tensor_alloc->offset == SIZE_MAX || size <= tensor_alloc->size_max); +#endif + } + } +} + +static bool ggml_gallocr_node_needs_realloc(ggml_gallocr_t galloc, struct ggml_tensor * node, struct node_alloc * nalloc, struct tensor_alloc * talloc) { + ggml_backend_buffer_type_t buft = galloc->bufts[nalloc->buffer_id]; + size_t node_size = (node->data || node->view_src) ? 0 : ggml_backend_buft_get_alloc_size(buft, node); + return talloc->size_max >= node_size; +} + +static bool ggml_gallocr_needs_realloc(ggml_gallocr_t galloc, struct ggml_cgraph * graph) { + if (galloc->n_nodes != graph->n_nodes) { +#ifndef NDEBUG + fprintf(stderr, "%s: graph has different number of nodes\n", __func__); +#endif + return true; } - // reset hash table - memset(galloc->hash_set.keys, 0, sizeof(struct ggml_tensor *) * hash_size); - memset(galloc->hash_values, 0, sizeof(struct hash_node) * hash_size); + for (int i = 0; i < graph->n_nodes; i++) { + struct ggml_tensor * node = graph->nodes[i]; + struct node_alloc * node_alloc = &galloc->node_allocs[i]; - galloc->talloc = talloc; - ggml_tallocr_alloc_graph_impl(galloc, graph); - galloc->talloc = NULL; + if (!ggml_gallocr_node_needs_realloc(galloc, node, node_alloc, &node_alloc->dst)) { +#ifndef NDEBUG + fprintf(stderr, "%s: node %s is not valid\n", __func__, node->name); +#endif + return true; + } - size_t max_size = ggml_tallocr_max_size(talloc); - - return max_size; -} - -void ggml_gallocr_alloc_graph_n(ggml_gallocr_t galloc, struct ggml_cgraph * graph, struct ggml_hash_set hash_set, ggml_tallocr_t * hash_node_talloc) { - const size_t hash_size = hash_set.size; - - GGML_ASSERT(hash_size >= (size_t)(graph->n_nodes + graph->n_leafs)); - - galloc->talloc = NULL; - - // alloc hash_values if needed - if (galloc->hash_values == NULL || galloc->hash_values_size < hash_size) { - free(galloc->hash_values); - galloc->hash_values = malloc(sizeof(struct hash_node) * hash_size); - galloc->hash_values_size = hash_size; + for (int j = 0; j < GGML_MAX_SRC; j++) { + struct ggml_tensor * src = node->src[j]; + if (src == NULL) { + break; + } + if (!ggml_gallocr_node_needs_realloc(galloc, src, node_alloc, &node_alloc->src[j])) { +#ifndef NDEBUG + fprintf(stderr, "%s: src %d (%s) of node %s is not valid\n", __func__, j, src->name, node->name); +#endif + return true; + } + } } - // free hash_set.keys if needed - if (galloc->hash_set.keys != NULL) { - free(galloc->hash_set.keys); - } - galloc->hash_set = hash_set; - - // reset hash values - memset(galloc->hash_values, 0, sizeof(struct hash_node) * hash_size); - - galloc->hash_allocs = hash_node_talloc; - - ggml_tallocr_alloc_graph_impl(galloc, graph); - - // remove unowned resources - galloc->hash_set.keys = NULL; - galloc->hash_allocs = NULL; + return false; } -// legacy API wrapper - -struct ggml_allocr { - ggml_tallocr_t talloc; - ggml_gallocr_t galloc; -}; - -static ggml_allocr_t ggml_allocr_new_impl(ggml_tallocr_t talloc) { - ggml_allocr_t alloc = (ggml_allocr_t)malloc(sizeof(struct ggml_allocr)); - *alloc = (struct ggml_allocr) { - /*.talloc = */ talloc, - /*.galloc = */ ggml_gallocr_new(), - }; - return alloc; -} - -ggml_allocr_t ggml_allocr_new(void * data, size_t size, size_t alignment) { - return ggml_allocr_new_impl(ggml_tallocr_new(data, size, alignment)); -} - -ggml_allocr_t ggml_allocr_new_measure(size_t alignment) { - return ggml_allocr_new_impl(ggml_tallocr_new_measure(alignment)); -} - -ggml_allocr_t ggml_allocr_new_from_buffer(struct ggml_backend_buffer * buffer) { - return ggml_allocr_new_impl(ggml_tallocr_new_from_buffer(buffer)); -} - -ggml_allocr_t ggml_allocr_new_from_backend(struct ggml_backend * backend, size_t size) { - return ggml_allocr_new_impl(ggml_tallocr_new_from_backend(backend, size)); -} - -ggml_allocr_t ggml_allocr_new_measure_from_backend(struct ggml_backend * backend) { - return ggml_allocr_new_impl(ggml_tallocr_new_measure_from_backend(backend)); -} - -struct ggml_backend_buffer * ggml_allocr_get_buffer(ggml_allocr_t alloc) { - return ggml_tallocr_get_buffer(alloc->talloc); -} - -void ggml_allocr_set_parse_seq(ggml_allocr_t alloc, const int * list, int n) { - ggml_gallocr_set_parse_seq(alloc->galloc, list, n); -} - -void ggml_allocr_free(ggml_allocr_t alloc) { - if (alloc == NULL) { - return; +bool ggml_gallocr_alloc_graph(ggml_gallocr_t galloc, struct ggml_cgraph * graph) { + if (ggml_gallocr_needs_realloc(galloc, graph)) { + if (galloc->n_buffers == 1) { +#ifndef NDEBUG + fprintf(stderr, "%s: reallocating buffers automatically\n", __func__); +#endif + if (!ggml_gallocr_reserve(galloc, graph)) { + return false; + } + } else { +#ifndef NDEBUG + fprintf(stderr, "%s: cannot reallocate multi buffer graph automatically, call reserve\n", __func__); +#endif + return false; + } } - ggml_gallocr_free(alloc->galloc); - ggml_tallocr_free(alloc->talloc); - free(alloc); + // reset buffers + for (int i = 0; i < galloc->n_buffers; i++) { + // zero size buffers are not allocated + if (galloc->buffers[i] != NULL) { + ggml_backend_buffer_reset(galloc->buffers[i]); + } + } + + // allocate the graph tensors from the previous assignments + for (int i = 0; i < graph->n_nodes; i++) { + struct ggml_tensor * node = graph->nodes[i]; + struct node_alloc * node_alloc = &galloc->node_allocs[i]; + for (int j = 0; j < GGML_MAX_SRC; j++) { + struct ggml_tensor * src = node->src[j]; + if (src == NULL) { + break; + } + ggml_gallocr_init_tensor(galloc, src, node_alloc, &node_alloc->src[j]); + } + ggml_gallocr_init_tensor(galloc, node, node_alloc, &node_alloc->dst); + } + + return true; } -bool ggml_allocr_is_measure(ggml_allocr_t alloc) { - return ggml_tallocr_is_measure(alloc->talloc); -} +size_t ggml_gallocr_get_buffer_size(ggml_gallocr_t galloc, int buffer_id) { + GGML_ASSERT(buffer_id >= 0 && buffer_id < galloc->n_buffers); -void ggml_allocr_reset(ggml_allocr_t alloc) { - ggml_tallocr_reset(alloc->talloc); -} - -void ggml_allocr_alloc(ggml_allocr_t alloc, struct ggml_tensor * tensor) { - ggml_tallocr_alloc(alloc->talloc, tensor); -} - -size_t ggml_allocr_max_size(ggml_allocr_t alloc) { - return ggml_tallocr_max_size(alloc->talloc); -} - -size_t ggml_allocr_alloc_graph(ggml_allocr_t alloc, struct ggml_cgraph * graph) { - return ggml_gallocr_alloc_graph(alloc->galloc, alloc->talloc, graph); + if (galloc->buffers[buffer_id] == NULL) { + return 0; + } + return ggml_backend_buffer_get_size(galloc->buffers[buffer_id]); } // utils @@ -795,17 +870,17 @@ static bool alloc_tensor_range(struct ggml_context * ctx, return false; } - ggml_tallocr_t tallocr = ggml_tallocr_new_from_buffer(buffer); + struct ggml_tallocr * tallocr = ggml_tallocr_new(buffer); for (struct ggml_tensor * t = first; t != last; t = ggml_get_next_tensor(ctx, t)) { if (t->data == NULL) { if (t->view_src == NULL) { ggml_tallocr_alloc(tallocr, t); - } else { + } else if (t->buffer == NULL) { ggml_backend_view_init(buffer, t); } } else { - if (t->view_src != NULL) { + if (t->view_src != NULL && t->buffer == NULL) { // view of a pre-allocated tensor ggml_backend_view_init(buffer, t); } @@ -838,7 +913,6 @@ ggml_backend_buffer_t ggml_backend_alloc_ctx_tensors_from_buft(struct ggml_conte } if (this_size > max_size) { - // tensor is too large to fit in a single buffer fprintf(stderr, "%s: tensor %s is too large to fit in a %s buffer (tensor size: %zu, max buffer size: %zu)\n", __func__, t->name, ggml_backend_buft_name(buft), @@ -870,7 +944,6 @@ ggml_backend_buffer_t ggml_backend_alloc_ctx_tensors_from_buft(struct ggml_conte } if (n_buffers == 0) { - // all the tensors in the context are already allocated #ifndef NDEBUG fprintf(stderr, "%s: all tensors in the context are already allocated\n", __func__); #endif diff --git a/ggml-alloc.h b/ggml-alloc.h index 4e5997521..1d9085d15 100644 --- a/ggml-alloc.h +++ b/ggml-alloc.h @@ -6,88 +6,62 @@ extern "C" { #endif -struct ggml_backend; -struct ggml_backend_buffer; -struct ggml_backend_buffer_type; - -// -// Legacy API -// - -typedef struct ggml_allocr * ggml_allocr_t; - -// initialize allocator for use with CPU backend only -GGML_API ggml_allocr_t ggml_allocr_new(void * data, size_t size, size_t alignment); -GGML_API ggml_allocr_t ggml_allocr_new_measure(size_t alignment); - -// initialize allocator for use with ggml-backend -GGML_API ggml_allocr_t ggml_allocr_new_from_buffer(struct ggml_backend_buffer * buffer); -GGML_API ggml_allocr_t ggml_allocr_new_from_backend(struct ggml_backend * backend, size_t size); // allocates an owned buffer -GGML_API ggml_allocr_t ggml_allocr_new_measure_from_backend(struct ggml_backend * backend); - -GGML_API struct ggml_backend_buffer * ggml_allocr_get_buffer(ggml_allocr_t alloc); - -// tell the allocator to parse nodes following the order described in the list -// you should call this if your graph are optimized to execute out-of-order -GGML_API void ggml_allocr_set_parse_seq(ggml_allocr_t alloc, const int * list, int n); - -GGML_API void ggml_allocr_free (ggml_allocr_t alloc); -GGML_API bool ggml_allocr_is_measure (ggml_allocr_t alloc); -GGML_API void ggml_allocr_reset (ggml_allocr_t alloc); -GGML_API void ggml_allocr_alloc (ggml_allocr_t alloc, struct ggml_tensor * tensor); -GGML_API size_t ggml_allocr_max_size (ggml_allocr_t alloc); - -GGML_API size_t ggml_allocr_alloc_graph(ggml_allocr_t alloc, struct ggml_cgraph * graph); - -// -// ggml-backend v2 API -// - -// Separate tensor and graph allocator objects -// This is necessary for multi-backend allocation because the graph allocator needs to use multiple tensor allocators -// The original API is kept as a wrapper around the new API +typedef struct ggml_backend_buffer_type * ggml_backend_buffer_type_t; +typedef struct ggml_backend_buffer * ggml_backend_buffer_t; +typedef struct ggml_backend * ggml_backend_t; // Tensor allocator typedef struct ggml_tallocr * ggml_tallocr_t; -GGML_API ggml_tallocr_t ggml_tallocr_new(void * data, size_t size, size_t alignment); -GGML_API ggml_tallocr_t ggml_tallocr_new_measure(size_t alignment); -GGML_API ggml_tallocr_t ggml_tallocr_new_from_buft(struct ggml_backend_buffer_type * buft, size_t size); -GGML_API ggml_tallocr_t ggml_tallocr_new_from_backend(struct ggml_backend * backend, size_t size); // allocates an owned buffer -GGML_API ggml_tallocr_t ggml_tallocr_new_from_buffer(struct ggml_backend_buffer * buffer); -GGML_API ggml_tallocr_t ggml_tallocr_new_measure_from_buft(struct ggml_backend_buffer_type * buft); -GGML_API ggml_tallocr_t ggml_tallocr_new_measure_from_backend(struct ggml_backend * backend); - -GGML_API struct ggml_backend_buffer * ggml_tallocr_get_buffer(ggml_tallocr_t talloc); - -GGML_API void ggml_tallocr_free (ggml_tallocr_t talloc); -GGML_API bool ggml_tallocr_is_measure (ggml_tallocr_t talloc); -GGML_API void ggml_tallocr_reset (ggml_tallocr_t talloc); -GGML_API void ggml_tallocr_alloc (ggml_tallocr_t talloc, struct ggml_tensor * tensor); -GGML_API size_t ggml_tallocr_max_size (ggml_tallocr_t talloc); - +GGML_API ggml_tallocr_t ggml_tallocr_new(ggml_backend_buffer_t buffer); +GGML_API void ggml_tallocr_free(ggml_tallocr_t talloc); +GGML_API void ggml_tallocr_alloc(ggml_tallocr_t talloc, struct ggml_tensor * tensor); // Graph allocator +/* + Example usage: + ggml_gallocr_t galloc = ggml_gallocr_new(ggml_bacckend_cpu_buffer_type()); + + // optional: create a worst-case graph and reserve the buffers to avoid reallocations + ggml_gallocr_reserve(galloc, build_graph(max_batch)); + + // allocate the graph + struct ggml_cgraph * graph = build_graph(batch); + ggml_gallocr_alloc_graph(galloc, graph); + + printf("compute buffer size: %zu bytes\n", ggml_gallocr_get_buffer_size(galloc, 0)); + + // evaluate the graph + ggml_backend_graph_compute(backend, graph); +*/ + +// special tensor flags for use with the graph allocator: +// ggml_set_input(): all input tensors are allocated at the beginning of the graph in non-overlapping addresses +// ggml_set_output(): output tensors are never freed and never overwritten + typedef struct ggml_gallocr * ggml_gallocr_t; -GGML_API ggml_gallocr_t ggml_gallocr_new(void); -GGML_API void ggml_gallocr_free(ggml_gallocr_t galloc); +GGML_API ggml_gallocr_t ggml_gallocr_new(ggml_backend_buffer_type_t buft); +GGML_API ggml_gallocr_t ggml_gallocr_new_n(ggml_backend_buffer_type_t * bufts, int n_bufs); +GGML_API void ggml_gallocr_free(ggml_gallocr_t galloc); -GGML_API void ggml_gallocr_set_parse_seq(ggml_gallocr_t galloc, const int * list, int n); -GGML_API size_t ggml_gallocr_alloc_graph(ggml_gallocr_t galloc, ggml_tallocr_t talloc, struct ggml_cgraph * graph); +// pre-allocate buffers from a measure graph - does not allocate or modify the graph +// call with a worst-case graph to avoid buffer reallocations +// not strictly required for single buffer usage: ggml_gallocr_alloc_graph will reallocate the buffers automatically if needed +// returns false if the buffer allocation failed +GGML_API bool ggml_gallocr_reserve(ggml_gallocr_t galloc, struct ggml_cgraph * graph); +GGML_API bool ggml_gallocr_reserve_n(ggml_gallocr_t galloc, struct ggml_cgraph * graph, const int * node_buffer_ids); -// Allocate tensors from the allocators given by the hash table -GGML_API void ggml_gallocr_alloc_graph_n( - ggml_gallocr_t galloc, - struct ggml_cgraph * graph, - struct ggml_hash_set hash_set, - ggml_tallocr_t * hash_node_talloc); +// automatic reallocation if the topology changes when using a single buffer +// returns false if using multiple buffers and a re-allocation is needed (call ggml_gallocr_reserve_n first to set the node buffers) +GGML_API bool ggml_gallocr_alloc_graph(ggml_gallocr_t galloc, struct ggml_cgraph * graph); +GGML_API size_t ggml_gallocr_get_buffer_size(ggml_gallocr_t galloc, int buffer_id); // Utils // Create a buffer and allocate all the tensors in a ggml_context -GGML_API struct ggml_backend_buffer * ggml_backend_alloc_ctx_tensors_from_buft(struct ggml_context * ctx, struct ggml_backend_buffer_type * buft); -GGML_API struct ggml_backend_buffer * ggml_backend_alloc_ctx_tensors(struct ggml_context * ctx, struct ggml_backend * backend); +GGML_API struct ggml_backend_buffer * ggml_backend_alloc_ctx_tensors_from_buft(struct ggml_context * ctx, ggml_backend_buffer_type_t buft); +GGML_API struct ggml_backend_buffer * ggml_backend_alloc_ctx_tensors(struct ggml_context * ctx, ggml_backend_t backend); #ifdef __cplusplus } diff --git a/ggml-backend.c b/ggml-backend.c index 532da8eda..9ee81b766 100644 --- a/ggml-backend.c +++ b/ggml-backend.c @@ -475,6 +475,8 @@ ggml_backend_buffer_t ggml_backend_reg_alloc_buffer(size_t i, size_t size) { // backend CPU +static const size_t TENSOR_ALIGNMENT = 32; // required for mmap as gguf only guarantees 32-byte alignment + GGML_CALL static const char * ggml_backend_cpu_buffer_name(ggml_backend_buffer_t buffer) { return "CPU"; @@ -482,7 +484,14 @@ GGML_CALL static const char * ggml_backend_cpu_buffer_name(ggml_backend_buffer_t } GGML_CALL static void * ggml_backend_cpu_buffer_get_base(ggml_backend_buffer_t buffer) { - return (void *)buffer->context; + uintptr_t data = (uintptr_t)buffer->context; + + // align the buffer + if (data % TENSOR_ALIGNMENT != 0) { + data = GGML_PAD(data, TENSOR_ALIGNMENT); + } + + return (void *)data; } GGML_CALL static void ggml_backend_cpu_buffer_free_buffer(ggml_backend_buffer_t buffer) { @@ -540,8 +549,6 @@ static struct ggml_backend_buffer_i cpu_backend_buffer_i_from_ptr = { /* .reset = */ NULL, }; -static const size_t TENSOR_ALIGNMENT = 64; // should be enough for AVX 512 - GGML_CALL static const char * ggml_backend_cpu_buffer_type_get_name(ggml_backend_buffer_type_t buft) { return "CPU"; @@ -550,9 +557,11 @@ GGML_CALL static const char * ggml_backend_cpu_buffer_type_get_name(ggml_backend GGML_CALL static ggml_backend_buffer_t ggml_backend_cpu_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size) { size += TENSOR_ALIGNMENT; // malloc may return an address that is not aligned - void * data = malloc(size); // TODO: maybe use GGML_ALIGNED_MALLOC? - - GGML_ASSERT(data != NULL && "failed to allocate buffer"); + void * data = malloc(size); // TODO: use GGML_ALIGNED_MALLOC (move to ggml-impl.h) + if (data == NULL) { + fprintf(stderr, "%s: failed to allocate buffer of size %zu\n", __func__, size); + return NULL; + } return ggml_backend_buffer_init(buft, cpu_backend_buffer_i, data, size); } @@ -766,6 +775,9 @@ static struct ggml_backend_i cpu_backend_i = { ggml_backend_t ggml_backend_cpu_init(void) { struct ggml_backend_cpu_context * ctx = malloc(sizeof(struct ggml_backend_cpu_context)); + if (ctx == NULL) { + return NULL; + } ctx->n_threads = GGML_DEFAULT_N_THREADS; ctx->work_data = NULL; @@ -774,6 +786,10 @@ ggml_backend_t ggml_backend_cpu_init(void) { ctx->abort_callback_data = NULL; ggml_backend_t cpu_backend = malloc(sizeof(struct ggml_backend)); + if (cpu_backend == NULL) { + free(ctx); + return NULL; + } *cpu_backend = (struct ggml_backend) { /* .interface = */ cpu_backend_i, @@ -802,6 +818,7 @@ void ggml_backend_cpu_set_abort_callback(ggml_backend_t backend_cpu, ggml_abort_ } GGML_CALL ggml_backend_buffer_t ggml_backend_cpu_buffer_from_ptr(void * ptr, size_t size) { + GGML_ASSERT((uintptr_t)ptr % TENSOR_ALIGNMENT == 0 && "buffer pointer must be aligned"); return ggml_backend_buffer_init(ggml_backend_cpu_buffer_type(), cpu_backend_buffer_i_from_ptr, ptr, size); } @@ -865,6 +882,8 @@ GGML_CALL ggml_backend_buffer_t ggml_backend_multi_buffer_alloc_buffer(ggml_back ctx->n_buffers = n_buffers; ctx->buffers = (ggml_backend_buffer_t *) malloc(n_buffers * sizeof(ggml_backend_buffer_t)); + GGML_ASSERT(ctx->buffers != NULL); + size_t total_size = 0; for (size_t i = 0; i < n_buffers; i++) { ctx->buffers[i] = buffers[i]; @@ -886,6 +905,18 @@ GGML_CALL void ggml_backend_multi_buffer_set_usage(ggml_backend_buffer_t buffer, } } +// creates a copy of the tensor with the same memory layout +static struct ggml_tensor * ggml_dup_tensor_layout(struct ggml_context * ctx, const struct ggml_tensor * tensor) { + struct ggml_tensor * dup = ggml_dup_tensor(ctx, tensor); + for (int i = 0; i < GGML_MAX_DIMS; i++) { + dup->nb[i] = tensor->nb[i]; + } + return dup; +} + +static bool ggml_is_view_op(enum ggml_op op) { + return op == GGML_OP_VIEW || op == GGML_OP_RESHAPE || op == GGML_OP_PERMUTE || op == GGML_OP_TRANSPOSE; +} // scheduler @@ -894,7 +925,7 @@ GGML_CALL void ggml_backend_multi_buffer_set_usage(ggml_backend_buffer_t buffer, #define GGML_MAX_SPLIT_INPUTS 16 struct ggml_backend_sched_split { - ggml_tallocr_t tallocr; + int backend_id; int i_start; int i_end; struct ggml_tensor * inputs[GGML_MAX_SPLIT_INPUTS]; @@ -909,15 +940,17 @@ struct ggml_backend_sched { int n_backends; ggml_backend_t backends[GGML_MAX_BACKENDS]; ggml_backend_buffer_type_t bufts[GGML_MAX_BACKENDS]; - ggml_tallocr_t tallocs[GGML_MAX_BACKENDS]; ggml_gallocr_t galloc; // hash keys of the nodes in the graph struct ggml_hash_set hash_set; - // hash values (arrays of [hash_set.size]) - ggml_tallocr_t * node_talloc; // tallocr assigned to each node (indirectly this is the backend) - struct ggml_tensor * (* node_copies)[GGML_MAX_BACKENDS]; // copies of each node for each destination backend + // hash values + int * tensor_backend_id; + struct ggml_tensor * (* tensor_copies)[GGML_MAX_BACKENDS]; + + int * node_backend_ids; // [n_nodes] + int n_nodes; // copy of the graph with modified inputs struct ggml_cgraph * graph; @@ -927,77 +960,46 @@ struct ggml_backend_sched { struct ggml_context * ctx; + ggml_backend_sched_eval_callback callback_eval; + void * callback_eval_user_data; + // align context_buffer to GGML_MEM_ALIGN #ifdef _MSC_VER __declspec(align(GGML_MEM_ALIGN)) #else __attribute__((aligned(GGML_MEM_ALIGN))) #endif - char context_buffer[GGML_MAX_SPLITS*GGML_MAX_SPLIT_INPUTS*sizeof(struct ggml_tensor) + sizeof(struct ggml_cgraph)]; - - ggml_backend_sched_eval_callback callback_eval; - void * callback_eval_user_data; + char context_buffer[GGML_MAX_SPLITS*GGML_MAX_SPLIT_INPUTS*2*sizeof(struct ggml_tensor) + sizeof(struct ggml_cgraph)]; }; #define hash_id(node) ggml_hash_find_or_insert(sched->hash_set, node) -#define node_allocr(node) sched->node_talloc[hash_id(node)] +#define tensor_backend_id(node) sched->tensor_backend_id[hash_id(node)] +#define tensor_backend(node) (tensor_backend_id(node) == -1 ? NULL : sched->backends[tensor_backend_id(node)]) -static bool ggml_is_view_op(enum ggml_op op) { - return op == GGML_OP_VIEW || op == GGML_OP_RESHAPE || op == GGML_OP_PERMUTE || op == GGML_OP_TRANSPOSE; -} - -// returns the priority of the backend, lower is better -static int sched_backend_prio(ggml_backend_sched_t sched, ggml_backend_t backend) { +// returns the priority of the backend, lower id is higher priority +static int ggml_backend_sched_backend_id(ggml_backend_sched_t sched, ggml_backend_t backend) { for (int i = 0; i < sched->n_backends; i++) { if (sched->backends[i] == backend) { return i; } } - return INT_MAX; + return -1; } -static int sched_allocr_prio(ggml_backend_sched_t sched, ggml_tallocr_t allocr) { - for (int i = 0; i < sched->n_backends; i++) { - if (sched->tallocs[i] == allocr) { - return i; - } - } - return INT_MAX; -} - -static ggml_tallocr_t sched_allocr_from_buffer(ggml_backend_sched_t sched, ggml_backend_buffer_t buffer) { +static int ggml_backend_sched_backend_from_buffer(ggml_backend_sched_t sched, ggml_backend_buffer_t buffer) { if (buffer == NULL) { - return NULL; - } - - // check if this is already allocate in a allocr buffer (from user manual allocations) - for (int i = 0; i < sched->n_backends; i++) { - if (ggml_tallocr_get_buffer(sched->tallocs[i]) == buffer) { - return sched->tallocs[i]; - } + return -1; } // find highest prio backend that supports the buffer type for (int i = 0; i < sched->n_backends; i++) { if (ggml_backend_buft_supports_backend(buffer->buft, sched->backends[i])) { - return sched->tallocs[i]; + return i; } } GGML_ASSERT(false && "tensor buffer type not supported by any backend"); } -static ggml_backend_t get_allocr_backend(ggml_backend_sched_t sched, ggml_tallocr_t allocr) { - if (allocr == NULL) { - return NULL; - } - for (int i = 0; i < sched->n_backends; i++) { - if (sched->tallocs[i] == allocr) { - return sched->backends[i]; - } - } - GGML_UNREACHABLE(); -} - #if 0 static char causes[GGML_DEFAULT_GRAPH_SIZE*16 + GGML_MAX_SPLITS*GGML_MAX_SPLIT_INPUTS][128]; // debug only #define SET_CAUSE(node, ...) sprintf(causes[hash_id(node)], __VA_ARGS__) @@ -1008,37 +1010,39 @@ static char causes[GGML_DEFAULT_GRAPH_SIZE*16 + GGML_MAX_SPLITS*GGML_MAX_SPLIT_I #endif // returns the backend that should be used for the node based on the current locations -static ggml_tallocr_t sched_allocr_from_cur(ggml_backend_sched_t sched, struct ggml_tensor * node) { +static int ggml_backend_sched_backend_id_from_cur(ggml_backend_sched_t sched, struct ggml_tensor * tensor) { + // TODO: use supports_op to check if the backend supports the op + // assign pre-allocated nodes to their backend // dst - ggml_tallocr_t cur_allocr = sched_allocr_from_buffer(sched, node->buffer); - if (cur_allocr != NULL) { + int cur_backend = ggml_backend_sched_backend_from_buffer(sched, tensor->buffer); + if (cur_backend != -1) { SET_CAUSE(node, "1.dst"); - return cur_allocr; + return cur_backend; } // view_src - if (node->view_src != NULL) { - cur_allocr = sched_allocr_from_buffer(sched, node->view_src->buffer); - if (cur_allocr != NULL) { + if (tensor->view_src != NULL) { + cur_backend = ggml_backend_sched_backend_from_buffer(sched, tensor->view_src->buffer); + if (cur_backend != -1) { SET_CAUSE(node, "1.vsrc"); - return cur_allocr; + return cur_backend; } } // assign nodes that use weights to the backend of the weights for (int i = 0; i < GGML_MAX_SRC; i++) { - const struct ggml_tensor * src = node->src[i]; + const struct ggml_tensor * src = tensor->src[i]; if (src == NULL) { break; } if (src->buffer != NULL && src->buffer->usage == GGML_BACKEND_BUFFER_USAGE_WEIGHTS) { - ggml_tallocr_t src_allocr = sched_allocr_from_buffer(sched, src->buffer); + int src_backend = ggml_backend_sched_backend_from_buffer(sched, src->buffer); // operations with weights are always run on the same backend as the weights SET_CAUSE(node, "1.wgt%d", i); - return src_allocr; + return src_backend; } } - return NULL; + return -1; } static char * fmt_size(size_t size) { @@ -1051,11 +1055,11 @@ static char * fmt_size(size_t size) { return buffer; } -static void sched_print_assignments(ggml_backend_sched_t sched, struct ggml_cgraph * graph) { +static void ggml_backend_sched_print_assignments(ggml_backend_sched_t sched, struct ggml_cgraph * graph) { int cur_split = 0; for (int i = 0; i < graph->n_nodes; i++) { if (cur_split < sched->n_splits && i == sched->splits[cur_split].i_start) { - ggml_backend_t split_backend = get_allocr_backend(sched, sched->splits[cur_split].tallocr); + ggml_backend_t split_backend = sched->backends[sched->splits[cur_split].backend_id]; fprintf(stderr, "\n## SPLIT #%d: %s # %d inputs: ", cur_split, ggml_backend_name(split_backend), sched->splits[cur_split].n_inputs); for (int j = 0; j < sched->splits[cur_split].n_inputs; j++) { @@ -1069,17 +1073,15 @@ static void sched_print_assignments(ggml_backend_sched_t sched, struct ggml_cgra if (ggml_is_view_op(node->op)) { continue; } - ggml_tallocr_t node_allocr = node_allocr(node); - ggml_backend_t node_backend = node_allocr ? get_allocr_backend(sched, node_allocr) : NULL; // FIXME: + ggml_backend_t tensor_backend = tensor_backend(node); fprintf(stderr, "node #%3d (%10.10s): %20.20s (%5.5s) [%5.5s %8.8s]:", i, ggml_op_name(node->op), node->name, - fmt_size(ggml_nbytes(node)), node_allocr ? ggml_backend_name(node_backend) : "NULL", GET_CAUSE(node)); + fmt_size(ggml_nbytes(node)), tensor_backend ? ggml_backend_name(tensor_backend) : "NULL", GET_CAUSE(node)); for (int j = 0; j < GGML_MAX_SRC; j++) { struct ggml_tensor * src = node->src[j]; if (src == NULL) { break; } - ggml_tallocr_t src_allocr = node_allocr(src); - ggml_backend_t src_backend = src_allocr ? get_allocr_backend(sched, src_allocr) : NULL; + ggml_backend_t src_backend = tensor_backend(src); fprintf(stderr, " %20.20s (%5.5s) [%5.5s %8.8s]", src->name, fmt_size(ggml_nbytes(src)), src_backend ? ggml_backend_name(src_backend) : "NULL", GET_CAUSE(src)); } @@ -1087,23 +1089,13 @@ static void sched_print_assignments(ggml_backend_sched_t sched, struct ggml_cgra } } -// creates a copy of the tensor with the same memory layout -static struct ggml_tensor * ggml_dup_tensor_layout(struct ggml_context * ctx, const struct ggml_tensor * tensor) { - struct ggml_tensor * dup = ggml_dup_tensor(ctx, tensor); - for (int i = 0; i < GGML_MAX_DIMS; i++) { - dup->nb[i] = tensor->nb[i]; - } - return dup; -} - - //#define DEBUG_PASS1 //#define DEBUG_PASS2 //#define DEBUG_PASS3 //#define DEBUG_PASS4 // assigns backends to ops and splits the graph into subgraphs that can be computed on the same backend -static void sched_split_graph(ggml_backend_sched_t sched, struct ggml_cgraph * graph) { +static void ggml_backend_sched_split_graph(ggml_backend_sched_t sched, struct ggml_cgraph * graph) { // reset splits sched->n_splits = 0; sched->is_reset = false; @@ -1125,28 +1117,28 @@ static void sched_split_graph(ggml_backend_sched_t sched, struct ggml_cgraph * g // pass 1: assign backends to ops with pre-allocated inputs for (int i = 0; i < graph->n_leafs; i++) { struct ggml_tensor * leaf = graph->leafs[i]; - if (node_allocr(leaf) != NULL) { + if (tensor_backend_id(leaf) != -1) { // do not overwrite user assignments continue; } - node_allocr(leaf) = sched_allocr_from_cur(sched, leaf); + tensor_backend_id(leaf) = ggml_backend_sched_backend_id_from_cur(sched, leaf); } for (int i = 0; i < graph->n_nodes; i++) { struct ggml_tensor * node = graph->nodes[i]; - if (node_allocr(node) != NULL) { + if (tensor_backend_id(node) != -1) { // do not overwrite user assignments continue; } - node_allocr(node) = sched_allocr_from_cur(sched, node); + tensor_backend_id(node) = ggml_backend_sched_backend_id_from_cur(sched, node); // src for (int j = 0; j < GGML_MAX_SRC; j++) { struct ggml_tensor * src = node->src[j]; if (src == NULL) { break; } - if (node_allocr(src) == NULL) { - node_allocr(src) = sched_allocr_from_cur(sched, src); + if (tensor_backend_id(src) == -1) { + tensor_backend_id(src) = ggml_backend_sched_backend_id_from_cur(sched, src); } } } @@ -1161,22 +1153,22 @@ static void sched_split_graph(ggml_backend_sched_t sched, struct ggml_cgraph * g // pass 2.1 expand gpu up { - ggml_tallocr_t cur_allocr = NULL; + int cur_backend_id = -1; for (int i = graph->n_nodes - 1; i >= 0; i--) { struct ggml_tensor * node = graph->nodes[i]; if (ggml_is_view_op(node->op)) { continue; } - ggml_tallocr_t node_allocr = node_allocr(node); - if (node_allocr != NULL) { - if (sched_allocr_prio(sched, node_allocr) == sched->n_backends - 1) { + int tensor_backend_id = tensor_backend_id(node); + if (tensor_backend_id != -1) { + if (tensor_backend_id == sched->n_backends - 1) { // skip cpu (lowest prio backend) - cur_allocr = NULL; + cur_backend_id = -1; } else { - cur_allocr = node_allocr; + cur_backend_id = tensor_backend_id; } } else { - node_allocr(node) = cur_allocr; + tensor_backend_id(node) = cur_backend_id; SET_CAUSE(node, "2.1"); } } @@ -1184,22 +1176,22 @@ static void sched_split_graph(ggml_backend_sched_t sched, struct ggml_cgraph * g // pass 2.2 expand gpu down { - ggml_tallocr_t cur_allocr = NULL; + int cur_backend_id = -1; for (int i = 0; i < graph->n_nodes; i++) { struct ggml_tensor * node = graph->nodes[i]; if (ggml_is_view_op(node->op)) { continue; } - ggml_tallocr_t node_allocr = node_allocr(node); - if (node_allocr != NULL) { - if (sched_allocr_prio(sched, node_allocr) == sched->n_backends - 1) { + int tensor_backend_id = tensor_backend_id(node); + if (tensor_backend_id != -1) { + if (tensor_backend_id == sched->n_backends - 1) { // skip cpu (lowest prio backend) - cur_allocr = NULL; + cur_backend_id = -1; } else { - cur_allocr = node_allocr; + cur_backend_id = tensor_backend_id; } } else { - node_allocr(node) = cur_allocr; + tensor_backend_id(node) = cur_backend_id; SET_CAUSE(node, "2.2"); } } @@ -1207,17 +1199,17 @@ static void sched_split_graph(ggml_backend_sched_t sched, struct ggml_cgraph * g // pass 2.3 expand rest up { - ggml_tallocr_t cur_allocr = NULL; + int cur_backend_id = -1; for (int i = graph->n_nodes - 1; i >= 0; i--) { struct ggml_tensor * node = graph->nodes[i]; if (ggml_is_view_op(node->op)) { continue; } - ggml_tallocr_t node_allocr = node_allocr(node); - if (node_allocr != NULL) { - cur_allocr = node_allocr; + int tensor_backend_id = tensor_backend_id(node); + if (tensor_backend_id != -1) { + cur_backend_id = tensor_backend_id; } else { - node_allocr(node) = cur_allocr; + tensor_backend_id(node) = cur_backend_id; SET_CAUSE(node, "2.3"); } } @@ -1225,17 +1217,17 @@ static void sched_split_graph(ggml_backend_sched_t sched, struct ggml_cgraph * g // pass 2.4 expand rest down { - ggml_tallocr_t cur_allocr = NULL; + int cur_backend_id = -1; for (int i = 0; i < graph->n_nodes; i++) { struct ggml_tensor * node = graph->nodes[i]; if (ggml_is_view_op(node->op)) { continue; } - ggml_tallocr_t node_allocr = node_allocr(node); - if (node_allocr != NULL) { - cur_allocr = node_allocr; + int tensor_backend_id = tensor_backend_id(node); + if (tensor_backend_id != -1) { + cur_backend_id = tensor_backend_id; } else { - node_allocr(node) = cur_allocr; + tensor_backend_id(node) = cur_backend_id; SET_CAUSE(node, "2.4"); } } @@ -1247,9 +1239,9 @@ static void sched_split_graph(ggml_backend_sched_t sched, struct ggml_cgraph * g // pass 3: assign backends to remaining src from dst and view_src for (int i = 0; i < graph->n_nodes; i++) { struct ggml_tensor * node = graph->nodes[i]; - ggml_tallocr_t cur_allocr = node_allocr(node); - if (node->view_src != NULL && cur_allocr == NULL) { - cur_allocr = node_allocr(node) = node_allocr(node->view_src); + int cur_backend_id = tensor_backend_id(node); + if (node->view_src != NULL && cur_backend_id == -1) { + cur_backend_id = tensor_backend_id(node) = tensor_backend_id(node->view_src); SET_CAUSE(node, "3.vsrc"); } for (int j = 0; j < GGML_MAX_SRC; j++) { @@ -1257,14 +1249,14 @@ static void sched_split_graph(ggml_backend_sched_t sched, struct ggml_cgraph * g if (src == NULL) { break; } - ggml_tallocr_t src_allocr = node_allocr(src); - if (src_allocr == NULL) { + int src_backend_id = tensor_backend_id(src); + if (src_backend_id == -1) { if (src->view_src != NULL) { // views are always on the same backend as the source - node_allocr(src) = node_allocr(src->view_src); + tensor_backend_id(src) = tensor_backend_id(src->view_src); SET_CAUSE(src, "3.vsrc"); } else { - node_allocr(src) = cur_allocr; + tensor_backend_id(src) = cur_backend_id; SET_CAUSE(src, "3.cur"); } } @@ -1281,15 +1273,14 @@ static void sched_split_graph(ggml_backend_sched_t sched, struct ggml_cgraph * g for (int i = 0; i < graph->n_nodes; i++) { struct ggml_tensor * node = graph->nodes[i]; if (!ggml_is_view_op(node->op)) { - sched->splits[0].tallocr = node_allocr(node); + sched->splits[0].backend_id = tensor_backend_id(node); break; } } sched->splits[0].i_start = 0; sched->splits[0].n_inputs = 0; memset(sched->splits[0].inputs, 0, sizeof(sched->splits[0].inputs)); //HACK - ggml_tallocr_t cur_allocr = sched->splits[0].tallocr; - size_t cur_backend_id = sched_allocr_prio(sched, cur_allocr); + int cur_backend_id = sched->splits[0].backend_id; for (int i = 0; i < graph->n_nodes; i++) { struct ggml_tensor * node = graph->nodes[i]; @@ -1297,19 +1288,18 @@ static void sched_split_graph(ggml_backend_sched_t sched, struct ggml_cgraph * g continue; } - ggml_tallocr_t node_allocr = node_allocr(node); + int tensor_backend_id = tensor_backend_id(node); - GGML_ASSERT(node_allocr != NULL); // all nodes should be assigned by now + GGML_ASSERT(tensor_backend_id != -1); // all nodes should be assigned by now - if (node_allocr != cur_allocr) { + if (tensor_backend_id != cur_backend_id) { sched->splits[cur_split].i_end = i; cur_split++; GGML_ASSERT(cur_split < GGML_MAX_SPLITS); - sched->splits[cur_split].tallocr = node_allocr; + sched->splits[cur_split].backend_id = tensor_backend_id; sched->splits[cur_split].i_start = i; sched->splits[cur_split].n_inputs = 0; - cur_allocr = node_allocr; - cur_backend_id = sched_allocr_prio(sched, cur_allocr); + cur_backend_id = tensor_backend_id; } // find inputs that are not on the same backend @@ -1318,43 +1308,25 @@ static void sched_split_graph(ggml_backend_sched_t sched, struct ggml_cgraph * g if (src == NULL) { break; } - ggml_tallocr_t src_allocr = node_allocr(src); - GGML_ASSERT(src_allocr != NULL); // all inputs should be assigned by now - if (src_allocr != node_allocr) { + int src_backend_id = tensor_backend_id(src); + assert(src_backend_id != -1); // all inputs should be assigned by now + if (src_backend_id != tensor_backend_id) { // create a copy of the input in the split's backend size_t id = hash_id(src); - if (sched->node_copies[id][cur_backend_id] == NULL) { - ggml_backend_t backend = get_allocr_backend(sched, cur_allocr); + if (sched->tensor_copies[id][cur_backend_id] == NULL) { + ggml_backend_t backend = sched->backends[cur_backend_id]; struct ggml_tensor * tensor_copy = ggml_dup_tensor_layout(sched->ctx, src); ggml_format_name(tensor_copy, "%s#%s", ggml_backend_name(backend), src->name); - sched->node_copies[id][cur_backend_id] = tensor_copy; - node_allocr(tensor_copy) = cur_allocr; + sched->tensor_copies[id][cur_backend_id] = tensor_copy; + tensor_backend_id(tensor_copy) = cur_backend_id; SET_CAUSE(tensor_copy, "4.cpy"); int n_inputs = sched->splits[cur_split].n_inputs++; GGML_ASSERT(n_inputs < GGML_MAX_SPLIT_INPUTS); sched->splits[cur_split].inputs[n_inputs] = src; } - node->src[j] = sched->node_copies[id][cur_backend_id]; - -#if 0 - // check if the input is already in the split - bool found = false; - for (int k = 0; k < sched->splits[cur_split].n_inputs; k++) { - if (sched->splits[cur_split].inputs[k] == src) { - found = true; - break; - } - } - - if (!found) { - int n_inputs = sched->splits[cur_split].n_inputs++; - //printf("split %d input %d: %s (%s)\n", cur_split, n_inputs, src->name, ggml_backend_name(get_allocr_backend(sched, src_allocr))); - GGML_ASSERT(n_inputs < GGML_MAX_SPLIT_INPUTS); - sched->splits[cur_split].inputs[n_inputs] = src; - } -#endif + node->src[j] = sched->tensor_copies[id][cur_backend_id]; } } } @@ -1369,30 +1341,30 @@ static void sched_split_graph(ggml_backend_sched_t sched, struct ggml_cgraph * g // sanity check: all sources should have the same backend as the node for (int i = 0; i < graph->n_nodes; i++) { struct ggml_tensor * node = graph->nodes[i]; - ggml_tallocr_t node_allocr = node_allocr(node); - if (node_allocr == NULL) { + ggml_backend_t tensor_backend = tensor_backend(node); + if (tensor_backend == NULL) { fprintf(stderr, "!!!!!!! %s has no backend\n", node->name); } - if (node->view_src != NULL && node_allocr != node_allocr(node->view_src)) { + if (node->view_src != NULL && tensor_backend != tensor_backend(node->view_src)) { fprintf(stderr, "!!!!!!! %s has backend %s, view_src %s has backend %s\n", - node->name, node_allocr ? ggml_backend_name(get_allocr_backend(sched, node_allocr)) : "NULL", - node->view_src->name, node_allocr(node->view_src) ? ggml_backend_name(get_allocr_backend(sched, node_allocr(node->view_src))) : "NULL"); + node->name, tensor_backend ? ggml_backend_name(tensor_backend) : "NULL", + node->view_src->name, tensor_backend(node->view_src) ? ggml_backend_name(tensor_backend(node->view_src)) : "NULL"); } for (int j = 0; j < GGML_MAX_SRC; j++) { struct ggml_tensor * src = node->src[j]; if (src == NULL) { break; } - ggml_tallocr_t src_allocr = node_allocr(src); - if (src_allocr != node_allocr /* && src_backend != NULL */) { // ignore nulls for now + ggml_backend_t src_backend = tensor_backend(src); + if (src_backend != tensor_backend /* && src_backend != NULL */) { fprintf(stderr, "!!!! %s has backend %s, src %d (%s) has backend %s\n", - node->name, node_allocr ? ggml_backend_name(get_allocr_backend(sched, node_allocr)) : "NULL", - j, src->name, src_allocr ? ggml_backend_name(get_allocr_backend(sched, src_allocr)) : "NULL"); + node->name, tensor_backend ? ggml_backend_name(tensor_backend) : "NULL", + j, src->name, src_backend ? ggml_backend_name(src_backend) : "NULL"); } - if (src->view_src != NULL && src_allocr != node_allocr(src->view_src)) { + if (src->view_src != NULL && src_backend != tensor_backend(src->view_src)) { fprintf(stderr, "!!!!!!! [src] %s has backend %s, view_src %s has backend %s\n", - src->name, src_allocr ? ggml_backend_name(get_allocr_backend(sched, src_allocr)) : "NULL", - src->view_src->name, node_allocr(src->view_src) ? ggml_backend_name(get_allocr_backend(sched, node_allocr(src->view_src))) : "NULL"); + src->name, src_backend ? ggml_backend_name(src_backend) : "NULL", + src->view_src->name, tensor_backend(src->view_src) ? ggml_backend_name(tensor_backend(src->view_src)) : "NULL"); } } } @@ -1406,32 +1378,45 @@ static void sched_split_graph(ggml_backend_sched_t sched, struct ggml_cgraph * g struct ggml_backend_sched_split * split = &sched->splits[i]; split->graph = ggml_graph_view(graph, split->i_start, split->i_end); - // add inputs to the graph copy so that they are allocated by ggml-alloc at the start of the split for (int j = 0; j < split->n_inputs; j++) { struct ggml_tensor * input = split->inputs[j]; - struct ggml_tensor * input_cpy = sched->node_copies[hash_id(input)][sched_allocr_prio(sched, split->tallocr)]; + struct ggml_tensor * input_cpy = sched->tensor_copies[hash_id(input)][split->backend_id]; + // add a dependency to the input source so that it is not freed before the copy is done - GGML_ASSERT(input_cpy->src[0] == NULL || input_cpy->src[0] == input); - input_cpy->src[0] = input; + struct ggml_tensor * input_dep = ggml_view_tensor(sched->ctx, input); + sched->node_backend_ids[graph_copy->n_nodes] = tensor_backend_id(input); + graph_copy->nodes[graph_copy->n_nodes++] = input_dep; + + // add a dependency to the input copy so that it is allocated at the start of the split + sched->node_backend_ids[graph_copy->n_nodes] = split->backend_id; graph_copy->nodes[graph_copy->n_nodes++] = input_cpy; } for (int j = split->i_start; j < split->i_end; j++) { + sched->node_backend_ids[graph_copy->n_nodes] = tensor_backend_id(graph->nodes[j]); graph_copy->nodes[graph_copy->n_nodes++] = graph->nodes[j]; } } sched->graph = graph_copy; } -static void sched_alloc_splits(ggml_backend_sched_t sched) { - ggml_gallocr_alloc_graph_n( - sched->galloc, - sched->graph, - sched->hash_set, - sched->node_talloc); +static bool ggml_backend_sched_alloc_splits(ggml_backend_sched_t sched) { + // ggml_gallocr_reserve_n(sched->galloc, sched->graph, sched->node_backend_ids); + if (!ggml_gallocr_alloc_graph(sched->galloc, sched->graph)) { +#ifndef NDEBUG + fprintf(stderr, "ggml_backend_sched: failed to allocate graph, reserving\n"); +#endif + ggml_gallocr_reserve_n(sched->galloc, sched->graph, sched->node_backend_ids); + if (!ggml_gallocr_alloc_graph(sched->galloc, sched->graph)) { + fprintf(stderr, "ggml_backend_sched: failed to allocate graph\n"); + return false; + } + } + + return true; } -static void sched_compute_splits(ggml_backend_sched_t sched) { +static bool ggml_backend_sched_compute_splits(ggml_backend_sched_t sched) { uint64_t copy_us[GGML_MAX_BACKENDS] = {0}; uint64_t compute_us[GGML_MAX_BACKENDS] = {0}; @@ -1439,20 +1424,18 @@ static void sched_compute_splits(ggml_backend_sched_t sched) { for (int i = 0; i < sched->n_splits; i++) { struct ggml_backend_sched_split * split = &splits[i]; - ggml_backend_t split_backend = get_allocr_backend(sched, split->tallocr); - int split_backend_id = sched_backend_prio(sched, split_backend); + int split_backend_id = split->backend_id; + ggml_backend_t split_backend = sched->backends[split_backend_id]; // copy the input tensors to the split backend uint64_t copy_start_us = ggml_time_us(); for (int j = 0; j < split->n_inputs; j++) { struct ggml_tensor * input = split->inputs[j]; - struct ggml_tensor * input_cpy = sched->node_copies[hash_id(input)][split_backend_id]; + struct ggml_tensor * input_cpy = sched->tensor_copies[hash_id(input)][split_backend_id]; GGML_ASSERT(input->buffer != NULL); GGML_ASSERT(input_cpy->buffer != NULL); - // TODO: avoid this copy if it was already copied in a previous split, and the input didn't change - // this is important to avoid copying constants such as KQ_mask and inp_pos multiple times ggml_backend_tensor_copy_async(split_backend, input, input_cpy); } //ggml_backend_synchronize(split_backend); // necessary to measure copy time @@ -1468,7 +1451,9 @@ static void sched_compute_splits(ggml_backend_sched_t sched) { uint64_t compute_start_us = ggml_time_us(); if (!sched->callback_eval) { - ggml_backend_graph_compute(split_backend, &split->graph); + if (!ggml_backend_graph_compute(split_backend, &split->graph)) { + return false; + } //ggml_backend_synchronize(split_backend); // necessary to measure compute time } else { // similar to ggml_backend_compare_graph_backend @@ -1488,7 +1473,9 @@ static void sched_compute_splits(ggml_backend_sched_t sched) { struct ggml_cgraph gv = ggml_graph_view(&split->graph, j0, j1 + 1); - ggml_backend_graph_compute(split_backend, &gv); + if (!ggml_backend_graph_compute(split_backend, &gv)) { + return false; + } if (need && !sched->callback_eval(t, false, sched->callback_eval_user_data)) { break; @@ -1510,19 +1497,8 @@ static void sched_compute_splits(ggml_backend_sched_t sched) { } } #endif -} -static void sched_reset(ggml_backend_sched_t sched) { - for (int i = 0; i < sched->n_backends; i++) { - ggml_tallocr_reset(sched->tallocs[i]); - } - // reset state for the next run - size_t hash_size = sched->hash_set.size; - memset(sched->hash_set.keys, 0, sizeof(sched->hash_set.keys[0]) * hash_size); - memset(sched->node_talloc, 0, sizeof(sched->node_talloc[0]) * hash_size); - memset(sched->node_copies, 0, sizeof(sched->node_copies[0]) * hash_size); - - sched->is_reset = true; + return true; } ggml_backend_sched_t ggml_backend_sched_new(ggml_backend_t * backends, ggml_backend_buffer_type_t * bufts, int n_backends, size_t graph_size) { @@ -1532,9 +1508,10 @@ ggml_backend_sched_t ggml_backend_sched_new(ggml_backend_t * backends, ggml_back struct ggml_backend_sched * sched = calloc(sizeof(struct ggml_backend_sched), 1); // initialize hash table - sched->hash_set = ggml_hash_set_new(graph_size + GGML_MAX_SPLITS*GGML_MAX_SPLIT_INPUTS); - sched->node_talloc = calloc(sizeof(sched->node_talloc[0]) * sched->hash_set.size, 1); - sched->node_copies = calloc(sizeof(sched->node_copies[0]) * sched->hash_set.size, 1); + sched->hash_set = ggml_hash_set_new(graph_size + GGML_MAX_SPLITS*GGML_MAX_SPLIT_INPUTS); + sched->tensor_backend_id = calloc(sizeof(sched->tensor_backend_id[0]), sched->hash_set.size); + sched->tensor_copies = calloc(sizeof(sched->tensor_copies[0]), sched->hash_set.size); + sched->node_backend_ids = calloc(sizeof(sched->node_backend_ids[0]), graph_size); sched->n_backends = n_backends; for (int i = 0; i < n_backends; i++) { @@ -1542,14 +1519,9 @@ ggml_backend_sched_t ggml_backend_sched_new(ggml_backend_t * backends, ggml_back sched->bufts[i] = bufts ? bufts[i] : ggml_backend_get_default_buffer_type(backends[i]); } - sched->galloc = ggml_gallocr_new(); + sched->galloc = ggml_gallocr_new_n(sched->bufts, n_backends); - // init measure allocs for each backend - for (int i = 0; i < n_backends; i++) { - sched->tallocs[i] = ggml_tallocr_new_measure_from_buft(sched->bufts[i]); - } - - sched_reset(sched); + ggml_backend_sched_reset(sched); return sched; } @@ -1558,49 +1530,54 @@ void ggml_backend_sched_free(ggml_backend_sched_t sched) { if (sched == NULL) { return; } - for (int i = 0; i < sched->n_backends; i++) { - ggml_tallocr_free(sched->tallocs[i]); - } ggml_gallocr_free(sched->galloc); ggml_free(sched->ctx); free(sched->hash_set.keys); - free(sched->node_talloc); - free(sched->node_copies); + free(sched->tensor_backend_id); + free(sched->tensor_copies); + free(sched->node_backend_ids); free(sched); } -void ggml_backend_sched_init_measure(ggml_backend_sched_t sched, struct ggml_cgraph * measure_graph) { - GGML_ASSERT(ggml_tallocr_is_measure(sched->tallocs[0])); // can only be initialized once +void ggml_backend_sched_reset(ggml_backend_sched_t sched) { + // reset state for the next run + size_t hash_size = sched->hash_set.size; + memset(sched->hash_set.keys, 0, sizeof(sched->hash_set.keys[0]) * hash_size); // NOLINT + memset(sched->tensor_backend_id, -1, sizeof(sched->tensor_backend_id[0]) * hash_size); + memset(sched->tensor_copies, 0, sizeof(sched->tensor_copies[0]) * hash_size); - sched_split_graph(sched, measure_graph); - sched_alloc_splits(sched); - - // allocate buffers and reset allocators - for (int i = 0; i < sched->n_backends; i++) { - size_t size = ggml_tallocr_max_size(sched->tallocs[i]); - ggml_tallocr_free(sched->tallocs[i]); - sched->tallocs[i] = ggml_tallocr_new_from_buft(sched->bufts[i], size); - } - - sched_reset(sched); + sched->is_reset = true; } -void ggml_backend_sched_graph_compute(ggml_backend_sched_t sched, struct ggml_cgraph * graph) { +bool ggml_backend_sched_reserve(ggml_backend_sched_t sched, struct ggml_cgraph * measure_graph) { + ggml_backend_sched_split_graph(sched, measure_graph); + + if (!ggml_gallocr_reserve_n(sched->galloc, sched->graph, sched->node_backend_ids)) { + return false; + } + + ggml_backend_sched_reset(sched); + return true; +} + +bool ggml_backend_sched_graph_compute(ggml_backend_sched_t sched, struct ggml_cgraph * graph) { GGML_ASSERT((int)sched->hash_set.size >= graph->n_nodes + GGML_MAX_SPLITS*GGML_MAX_SPLIT_INPUTS); if (!sched->is_reset) { - sched_reset(sched); + ggml_backend_sched_reset(sched); } - sched_split_graph(sched, graph); - sched_alloc_splits(sched); - sched_compute_splits(sched); -} + ggml_backend_sched_split_graph(sched, graph); + if (!ggml_backend_sched_alloc_splits(sched)) { + return false; + } -void ggml_backend_sched_reset(ggml_backend_sched_t sched) { - sched_reset(sched); -} + if (!ggml_backend_sched_compute_splits(sched)) { + return false; + } + return true; +} void ggml_backend_sched_set_eval_callback(ggml_backend_sched_t sched, ggml_backend_sched_eval_callback callback, void * user_data) { sched->callback_eval = callback; @@ -1611,37 +1588,30 @@ int ggml_backend_sched_get_n_splits(ggml_backend_sched_t sched) { return sched->n_splits; } -ggml_tallocr_t ggml_backend_sched_get_tallocr(ggml_backend_sched_t sched, ggml_backend_t backend) { - int backend_index = sched_backend_prio(sched, backend); +size_t ggml_backend_sched_get_buffer_size(ggml_backend_sched_t sched, ggml_backend_t backend) { + int backend_index = ggml_backend_sched_backend_id(sched, backend); GGML_ASSERT(backend_index >= 0 && backend_index < sched->n_backends); - return sched->tallocs[backend_index]; -} - -ggml_backend_buffer_t ggml_backend_sched_get_buffer(ggml_backend_sched_t sched, ggml_backend_t backend) { - int backend_index = sched_backend_prio(sched, backend); - GGML_ASSERT(backend_index >= 0 && backend_index < sched->n_backends); - return ggml_tallocr_get_buffer(sched->tallocs[backend_index]); + return ggml_gallocr_get_buffer_size(sched->galloc, backend_index); } void ggml_backend_sched_set_node_backend(ggml_backend_sched_t sched, struct ggml_tensor * node, ggml_backend_t backend) { - int backend_index = sched_backend_prio(sched, backend); + int backend_index = ggml_backend_sched_backend_id(sched, backend); GGML_ASSERT(backend_index >= 0 && backend_index < sched->n_backends); - node_allocr(node) = sched->tallocs[backend_index]; + tensor_backend_id(node) = backend_index; } ggml_backend_t ggml_backend_sched_get_node_backend(ggml_backend_sched_t sched, struct ggml_tensor * node) { - ggml_tallocr_t allocr = node_allocr(node); - if (allocr == NULL) { + int backend_index = tensor_backend_id(node); + if (backend_index == -1) { return NULL; } - return get_allocr_backend(sched, allocr); + return sched->backends[backend_index]; } // utils void ggml_backend_view_init(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor) { GGML_ASSERT(tensor->buffer == NULL); - //GGML_ASSERT(tensor->data == NULL); // views of pre-allocated tensors may have the data set in ggml_new_tensor, but still need to be initialized by the backend GGML_ASSERT(tensor->view_src != NULL); GGML_ASSERT(tensor->view_src->buffer != NULL); GGML_ASSERT(tensor->view_src->data != NULL); @@ -1665,7 +1635,7 @@ void ggml_backend_tensor_alloc(ggml_backend_buffer_t buffer, struct ggml_tensor ggml_backend_buffer_init_tensor(buffer, tensor); } -static struct ggml_tensor * graph_dup_tensor(struct ggml_hash_set hash_set, struct ggml_tensor ** node_copies, +static struct ggml_tensor * graph_copy_dup_tensor(struct ggml_hash_set hash_set, struct ggml_tensor ** node_copies, struct ggml_context * ctx_allocated, struct ggml_context * ctx_unallocated, struct ggml_tensor * src) { GGML_ASSERT(src != NULL); @@ -1678,7 +1648,7 @@ static struct ggml_tensor * graph_dup_tensor(struct ggml_hash_set hash_set, stru struct ggml_tensor * dst = ggml_dup_tensor_layout(src->data && !src->view_src ? ctx_allocated : ctx_unallocated, src); if (src->view_src != NULL) { - dst->view_src = graph_dup_tensor(hash_set, node_copies, ctx_allocated, ctx_unallocated, src->view_src); + dst->view_src = graph_copy_dup_tensor(hash_set, node_copies, ctx_allocated, ctx_unallocated, src->view_src); dst->view_offs = src->view_offs; } dst->op = src->op; @@ -1691,14 +1661,14 @@ static struct ggml_tensor * graph_dup_tensor(struct ggml_hash_set hash_set, stru if (s == NULL) { break; } - dst->src[i] = graph_dup_tensor(hash_set, node_copies, ctx_allocated, ctx_unallocated, s); + dst->src[i] = graph_copy_dup_tensor(hash_set, node_copies, ctx_allocated, ctx_unallocated, s); } node_copies[id] = dst; return dst; } -static void graph_init_tensor(struct ggml_hash_set hash_set, struct ggml_tensor ** node_copies, bool * node_init, struct ggml_tensor * src) { +static void graph_copy_init_tensor(struct ggml_hash_set hash_set, struct ggml_tensor ** node_copies, bool * node_init, struct ggml_tensor * src) { size_t id = ggml_hash_find(hash_set, src); if (node_init[id]) { return; @@ -1707,7 +1677,7 @@ static void graph_init_tensor(struct ggml_hash_set hash_set, struct ggml_tensor struct ggml_tensor * dst = node_copies[id]; if (dst->view_src != NULL) { - graph_init_tensor(hash_set, node_copies, node_init, src->view_src); + graph_copy_init_tensor(hash_set, node_copies, node_init, src->view_src); ggml_backend_view_init(dst->view_src->buffer, dst); } else { @@ -1720,17 +1690,17 @@ static void graph_init_tensor(struct ggml_hash_set hash_set, struct ggml_tensor if (s == NULL) { break; } - graph_init_tensor(hash_set, node_copies, node_init, s); + graph_copy_init_tensor(hash_set, node_copies, node_init, s); } } struct ggml_backend_graph_copy ggml_backend_graph_copy(ggml_backend_t backend, struct ggml_cgraph * graph) { struct ggml_hash_set hash_set = { /* .size = */ graph->visited_hash_table.size, - /* .keys = */ calloc(sizeof(hash_set.keys[0]) * graph->visited_hash_table.size, 1) + /* .keys = */ calloc(sizeof(hash_set.keys[0]), graph->visited_hash_table.size) // NOLINT }; - struct ggml_tensor ** node_copies = calloc(sizeof(node_copies[0]) * hash_set.size, 1); - bool * node_init = calloc(sizeof(node_init[0]) * hash_set.size, 1); + struct ggml_tensor ** node_copies = calloc(sizeof(node_copies[0]), hash_set.size); // NOLINT + bool * node_init = calloc(sizeof(node_init[0]), hash_set.size); struct ggml_init_params params = { /* .mem_size = */ ggml_tensor_overhead()*hash_set.size + ggml_graph_overhead_custom(graph->size, false), @@ -1759,7 +1729,7 @@ struct ggml_backend_graph_copy ggml_backend_graph_copy(ggml_backend_t backend, s // dup nodes for (int i = 0; i < graph->n_nodes; i++) { struct ggml_tensor * node = graph->nodes[i]; - graph_dup_tensor(hash_set, node_copies, ctx_allocated, ctx_unallocated, node); + graph_copy_dup_tensor(hash_set, node_copies, ctx_allocated, ctx_unallocated, node); } // allocate nodes @@ -1784,7 +1754,7 @@ struct ggml_backend_graph_copy ggml_backend_graph_copy(ggml_backend_t backend, s // copy data and init views for (int i = 0; i < graph->n_nodes; i++) { struct ggml_tensor * node = graph->nodes[i]; - graph_init_tensor(hash_set, node_copies, node_init, node); + graph_copy_init_tensor(hash_set, node_copies, node_init, node); } // build graph copy diff --git a/ggml-backend.h b/ggml-backend.h index 282b3a9b7..f13c69bff 100644 --- a/ggml-backend.h +++ b/ggml-backend.h @@ -130,11 +130,7 @@ extern "C" { // in build_graph: build_graph(...) { - // allocating tensors in a specific backend (optional, recommended: pre-allocate inputs in a different buffer) - alloc_cpu = ggml_backend_sched_get_allocr(sched, backend_cpu); - ggml_allocr_alloc(alloc_cpu, tensor); - - // manually assigning nodes to a backend (optional, shouldn't be needed in most cases) + // manually assign nodes to a backend (optional, should not be needed in most cases) struct ggml_tensor * node = ggml_mul_mat(ctx, ...); ggml_backend_sched_set_node_backend(sched, node, backend_gpu); } @@ -164,20 +160,19 @@ extern "C" { GGML_API ggml_backend_sched_t ggml_backend_sched_new(ggml_backend_t * backends, ggml_backend_buffer_type_t * bufts, int n_backends, size_t graph_size); GGML_API void ggml_backend_sched_free(ggml_backend_sched_t sched); // Initialize backend buffers from a measure graph - GGML_API void ggml_backend_sched_init_measure(ggml_backend_sched_t sched, struct ggml_cgraph * measure_graph); + GGML_API bool ggml_backend_sched_reserve(ggml_backend_sched_t sched, struct ggml_cgraph * measure_graph); // Get the number of splits of the last graph GGML_API int ggml_backend_sched_get_n_splits(ggml_backend_sched_t sched); - GGML_API ggml_tallocr_t ggml_backend_sched_get_tallocr(ggml_backend_sched_t sched, ggml_backend_t backend); - GGML_API ggml_backend_buffer_t ggml_backend_sched_get_buffer (ggml_backend_sched_t sched, ggml_backend_t backend); + GGML_API size_t ggml_backend_sched_get_buffer_size(ggml_backend_sched_t sched, ggml_backend_t backend); GGML_API void ggml_backend_sched_set_node_backend(ggml_backend_sched_t sched, struct ggml_tensor * node, ggml_backend_t backend); GGML_API ggml_backend_t ggml_backend_sched_get_node_backend(ggml_backend_sched_t sched, struct ggml_tensor * node); // Allocate and compute graph on the backend scheduler - GGML_API void ggml_backend_sched_graph_compute(ggml_backend_sched_t sched, struct ggml_cgraph * graph); + GGML_API bool ggml_backend_sched_graph_compute(ggml_backend_sched_t sched, struct ggml_cgraph * graph); - // Reset all assignments and allocators - must be called before using the sched allocators to allocate inputs + // Reset all assignments and allocators - must be called before changing the node backends GGML_API void ggml_backend_sched_reset(ggml_backend_sched_t sched); // Set a callback to be called for each resulting node during graph compute diff --git a/ggml.c b/ggml.c index e45b78d7e..d921d82fe 100644 --- a/ggml.c +++ b/ggml.c @@ -2649,7 +2649,7 @@ static struct ggml_tensor * ggml_new_tensor_impl( /*.nb =*/ { 0, 0, 0, 0 }, /*.op =*/ GGML_OP_NONE, /*.op_params =*/ { 0 }, - /*.is_param =*/ false, + /*.flags =*/ 0, /*.grad =*/ NULL, /*.src =*/ { NULL }, /*.perf_runs =*/ 0, @@ -6551,7 +6551,7 @@ struct ggml_tensor * ggml_cross_entropy_loss_back( void ggml_set_param( struct ggml_context * ctx, struct ggml_tensor * tensor) { - tensor->is_param = true; + tensor->flags |= GGML_TENSOR_FLAG_PARAM; GGML_ASSERT(tensor->grad == NULL); tensor->grad = ggml_dup_tensor(ctx, tensor); @@ -15367,7 +15367,7 @@ static struct ggml_tensor * ggml_recompute_graph_node( return NULL; } - if (node->is_param) { + if (node->flags & GGML_TENSOR_FLAG_PARAM) { return node; } @@ -15401,7 +15401,7 @@ static struct ggml_tensor * ggml_recompute_graph_node( clone->op = node->op; clone->grad = node->grad; - clone->is_param = node->is_param; + clone->flags = node->flags; clone->extra = node->extra; for (int k = 0; k < GGML_MAX_DIMS; ++k) { clone->nb[k] = node->nb[k]; @@ -16433,7 +16433,7 @@ void ggml_build_backward_expand(struct ggml_context * ctx, struct ggml_cgraph * for (int i = 0; i < gf->n_nodes; i++) { struct ggml_tensor * node = gf->nodes[i]; - if (node->is_param) { + if (node->flags & GGML_TENSOR_FLAG_PARAM) { GGML_PRINT_DEBUG("%s: found root node %p\n", __func__, (void *) node); ggml_build_forward_expand(gb, node->grad); } @@ -17918,7 +17918,7 @@ void ggml_graph_print(const struct ggml_cgraph * cgraph) { GGML_PRINT(" - %3d: [ %5" PRId64 ", %5" PRId64 ", %5" PRId64 "] %16s %s (%3d) cpu = %7.3f / %7.3f ms, wall = %7.3f / %7.3f ms\n", i, node->ne[0], node->ne[1], node->ne[2], - ggml_op_name(node->op), node->is_param ? "x" : node->grad ? "g" : " ", node->perf_runs, + ggml_op_name(node->op), (node->flags & GGML_TENSOR_FLAG_PARAM) ? "x" : node->grad ? "g" : " ", node->perf_runs, (double) node->perf_cycles / (double) ggml_cycles_per_ms(), (double) node->perf_cycles / (double) ggml_cycles_per_ms() / (double) node->perf_runs, (double) node->perf_time_us / 1000.0, @@ -18011,7 +18011,7 @@ void ggml_graph_dump_dot(const struct ggml_cgraph * gb, const struct ggml_cgraph continue; } - if (node->is_param) { + if (node->flags & GGML_TENSOR_FLAG_PARAM) { snprintf(color, sizeof(color), "yellow"); } else if (node->grad) { if (ggml_graph_find(gf, node)) { @@ -18185,7 +18185,7 @@ static enum ggml_opt_result ggml_opt_adam( int np = 0; int64_t nx = 0; for (int i = 0; i < gf->n_nodes; ++i) { - if (gf->nodes[i]->is_param) { + if (gf->nodes[i]->flags & GGML_TENSOR_FLAG_PARAM) { GGML_PRINT_DEBUG("found param %d: grad->op = %d\n", np, gf->nodes[i]->grad->op); GGML_ASSERT(np < GGML_MAX_PARAMS); @@ -18548,7 +18548,7 @@ static enum ggml_opt_result ggml_opt_lbfgs( int np = 0; int nx = 0; for (int i = 0; i < gf->n_nodes; ++i) { - if (gf->nodes[i]->is_param) { + if (gf->nodes[i]->flags & GGML_TENSOR_FLAG_PARAM) { GGML_PRINT_DEBUG("found param %d: grad->op = %d\n", np, gf->nodes[i]->grad->op); GGML_ASSERT(np < GGML_MAX_PARAMS); @@ -19023,6 +19023,16 @@ enum ggml_opt_result ggml_opt_resume_g( //////////////////////////////////////////////////////////////////////////////// +void ggml_set_input(struct ggml_tensor * tensor) { + tensor->flags |= GGML_TENSOR_FLAG_INPUT; +} + +void ggml_set_output(struct ggml_tensor * tensor) { + tensor->flags |= GGML_TENSOR_FLAG_OUTPUT; +} + +//////////////////////////////////////////////////////////////////////////////// + void ggml_quantize_init(enum ggml_type type) { ggml_critical_section_start(); diff --git a/ggml.h b/ggml.h index 9cfec5bac..01cecc1e1 100644 --- a/ggml.h +++ b/ggml.h @@ -505,11 +505,17 @@ extern "C" { enum ggml_log_level { GGML_LOG_LEVEL_ERROR = 2, - GGML_LOG_LEVEL_WARN = 3, - GGML_LOG_LEVEL_INFO = 4, + GGML_LOG_LEVEL_WARN = 3, + GGML_LOG_LEVEL_INFO = 4, GGML_LOG_LEVEL_DEBUG = 5 }; + enum ggml_tensor_flag { + GGML_TENSOR_FLAG_INPUT = 1, + GGML_TENSOR_FLAG_OUTPUT = 2, + GGML_TENSOR_FLAG_PARAM = 4, + }; + // ggml object struct ggml_object { size_t offs; @@ -543,7 +549,7 @@ extern "C" { // op params - allocated as int32_t for alignment int32_t op_params[GGML_MAX_OP_PARAMS / sizeof(int32_t)]; - bool is_param; + int32_t flags; struct ggml_tensor * grad; struct ggml_tensor * src[GGML_MAX_SRC]; @@ -2092,6 +2098,12 @@ extern "C" { ggml_opt_callback callback, void * callback_data); + // + // tensor flags + // + GGML_API void ggml_set_input(struct ggml_tensor * tensor); + GGML_API void ggml_set_output(struct ggml_tensor * tensor); + // // quantization // diff --git a/llama.cpp b/llama.cpp index d1ee26ce2..a5b873a7b 100644 --- a/llama.cpp +++ b/llama.cpp @@ -1872,8 +1872,6 @@ struct llama_context { // memory buffers used to evaluate the model std::vector buf_compute_meta; ggml_backend_sched_t sched = nullptr; - // allocator for the input tensors - ggml_tallocr * alloc = nullptr; // input tensors ggml_backend_buffer_t buf_input = nullptr; @@ -7199,12 +7197,10 @@ struct llm_build_context { static struct ggml_cgraph * llama_build_graph( llama_context & lctx, - const llama_batch & batch) { + const llama_batch & batch, + bool worst_case) { const auto & model = lctx.model; - // check if we should build the worst-case graph (for memory measurement) - const bool worst_case = ggml_tallocr_is_measure(lctx.alloc); - // this callback allows us to apply custom logic to each tensor (e.g. ggml-alloc, offloading, etc.) llm_build_cb cb = [&](struct ggml_tensor * cur, const char * name, int il) { if (il >= 0) { @@ -7225,77 +7221,6 @@ static struct ggml_cgraph * llama_build_graph( struct llm_build_context llm(lctx, batch, cb, worst_case); - // - // set input data - // - - if (!ggml_tallocr_is_measure(lctx.alloc)) { - if (batch.token) { - const int64_t n_tokens = batch.n_tokens; - - ggml_backend_tensor_set(lctx.inp_tokens, batch.token, 0, n_tokens*ggml_element_size(lctx.inp_tokens)); - } - - if (batch.embd) { - const int64_t n_embd = llm.n_embd; - const int64_t n_tokens = batch.n_tokens; - - ggml_backend_tensor_set(lctx.inp_embd, batch.embd, 0, n_tokens*n_embd*ggml_element_size(lctx.inp_embd)); - } - - if (batch.pos) { - const int64_t n_tokens = batch.n_tokens; - - ggml_backend_tensor_set(lctx.inp_pos, batch.pos, 0, n_tokens*ggml_element_size(lctx.inp_pos)); - } - - { - const int64_t n_kv = llm.n_kv; - const int64_t n_tokens = batch.n_tokens; - - GGML_ASSERT(ggml_backend_buffer_is_host(lctx.inp_KQ_mask->buffer)); - float * data = (float *) lctx.inp_KQ_mask->data; - - for (int h = 0; h < 1; ++h) { - for (int j = 0; j < n_tokens; ++j) { - const llama_pos pos = batch.pos[j]; - const llama_seq_id seq_id = batch.seq_id[j][0]; - - for (int i = 0; i < n_kv; ++i) { - float f; - if (!lctx.kv_self.cells[i].has_seq_id(seq_id) || - (llm.causal_attn && lctx.kv_self.cells[i].pos > pos)) { - f = -INFINITY; - } else { - f = 0; - } - data[h*(n_kv*n_tokens) + j*n_kv + i] = f; - } - } - } - } - - if (llm.do_rope_shift) { - const int64_t n_ctx = llm.n_ctx; - - GGML_ASSERT(ggml_backend_buffer_is_host(lctx.inp_K_shift->buffer)); - int32_t * data = (int32_t *) lctx.inp_K_shift->data; - - for (int i = 0; i < n_ctx; ++i) { - data[i] = lctx.kv_self.cells[i].delta; - } - } - - { - GGML_ASSERT(ggml_backend_buffer_is_host(lctx.inp_sum->buffer)); - float * data = (float *) lctx.inp_sum->data; - - for (int i = 0; i < batch.n_tokens; ++i) { - data[i] = 1.0f/float(batch.n_tokens); - } - } - } - llm.init(); switch (model.arch) { @@ -7384,6 +7309,83 @@ static struct ggml_cgraph * llama_build_graph( return result; } +static void llama_set_inputs(llama_context & lctx, const llama_batch & batch) { + // + // set input data + // + + const auto & hparams = lctx.model.hparams; + const auto & cparams = lctx.cparams; + const auto & kv_self = lctx.kv_self; + + if (batch.token) { + const int64_t n_tokens = batch.n_tokens; + + ggml_backend_tensor_set(lctx.inp_tokens, batch.token, 0, n_tokens*ggml_element_size(lctx.inp_tokens)); + } + + if (batch.embd) { + const int64_t n_embd = hparams.n_embd; + const int64_t n_tokens = batch.n_tokens; + + ggml_backend_tensor_set(lctx.inp_embd, batch.embd, 0, n_tokens*n_embd*ggml_element_size(lctx.inp_embd)); + } + + if (batch.pos) { + const int64_t n_tokens = batch.n_tokens; + + ggml_backend_tensor_set(lctx.inp_pos, batch.pos, 0, n_tokens*ggml_element_size(lctx.inp_pos)); + } + + { + const int64_t n_kv = kv_self.n; + const int64_t n_tokens = batch.n_tokens; + + assert(ggml_backend_buffer_is_host(lctx.inp_KQ_mask->buffer)); + + float * data = (float *) lctx.inp_KQ_mask->data; + + for (int h = 0; h < 1; ++h) { + for (int j = 0; j < n_tokens; ++j) { + const llama_pos pos = batch.pos[j]; + const llama_seq_id seq_id = batch.seq_id[j][0]; + + for (int i = 0; i < n_kv; ++i) { + float f; + if (!lctx.kv_self.cells[i].has_seq_id(seq_id) || lctx.kv_self.cells[i].pos > pos) { + f = -INFINITY; + } else { + f = 0; + } + data[h*(n_kv*n_tokens) + j*n_kv + i] = f; + } + } + } + } + + + { + assert(ggml_backend_buffer_is_host(lctx.inp_sum->buffer)); + float * data = (float *) lctx.inp_sum->data; + + for (int i = 0; i < batch.n_tokens; ++i) { + data[i] = 1.0f/float(batch.n_tokens); + } + } + + if (kv_self.has_shift) { + const int64_t n_ctx = cparams.n_ctx; + + assert(ggml_backend_buffer_is_host(lctx.inp_K_shift->buffer)); + + int32_t * data = (int32_t *) lctx.inp_K_shift->data; + + for (int i = 0; i < n_ctx; ++i) { + data[i] = lctx.kv_self.cells[i].delta; + } + } +} + // decode a batch of tokens by evaluating the transformer // // - lctx: llama context @@ -7482,7 +7484,7 @@ static int llama_decode_internal( ggml_backend_sched_reset(lctx.sched); ggml_backend_sched_set_eval_callback(lctx.sched, lctx.cparams.cb_eval, lctx.cparams.cb_eval_user_data); - ggml_cgraph * gf = llama_build_graph(lctx, batch); + ggml_cgraph * gf = llama_build_graph(lctx, batch, false); // the output is always the last tensor in the graph struct ggml_tensor * res = gf->nodes[gf->n_nodes - 1]; @@ -7527,6 +7529,9 @@ static int llama_decode_internal( if (lctx.backend_cpu != nullptr) { ggml_backend_cpu_set_n_threads(lctx.backend_cpu, n_threads); } + + llama_set_inputs(lctx, batch); + ggml_backend_sched_graph_compute(lctx.sched, gf); // fprintf(stderr, "splits: %d\n", ggml_backend_sched_get_n_splits(lctx.sched)); @@ -11278,23 +11283,27 @@ struct llama_context * llama_new_context_with_model( ctx->buf_compute_meta.resize(ggml_tensor_overhead()*LLAMA_MAX_NODES + ggml_graph_overhead()); ctx->sched = ggml_backend_sched_new(ctx->backends.data(), backend_buft.data(), ctx->backends.size(), LLAMA_MAX_NODES); - ctx->alloc = ggml_backend_sched_get_tallocr(ctx->sched, ctx->backend_cpu); // build worst-case graph int n_tokens = (int)std::min(cparams.n_ctx, cparams.n_batch); int n_past = cparams.n_ctx - n_tokens; llama_token token = llama_token_bos(&ctx->model); // not actually used by llama_build_graph, but required to choose between token and embedding inputs graph - ggml_cgraph * gf = llama_build_graph(*ctx, llama_batch_get_one(&token, n_tokens, n_past, 0)); + ggml_cgraph * gf = llama_build_graph(*ctx, llama_batch_get_one(&token, n_tokens, n_past, 0), true); // initialize scheduler with the worst-case graph - ggml_backend_sched_init_measure(ctx->sched, gf); - ctx->alloc = ggml_backend_sched_get_tallocr(ctx->sched, ctx->backend_cpu); + if (!ggml_backend_sched_reserve(ctx->sched, gf)) { + LLAMA_LOG_ERROR("%s: failed to allocate compute buffers\n", __func__); + llama_free(ctx); + return nullptr; + } - for (ggml_backend_t backend : ctx->backends) { - ggml_backend_buffer_t buf = ggml_backend_sched_get_buffer(ctx->sched, backend); + for (size_t i = 0; i < ctx->backends.size(); i++) { + ggml_backend_t backend = ctx->backends[i]; + ggml_backend_buffer_type_t buft = backend_buft[i]; + size_t size = ggml_backend_sched_get_buffer_size(ctx->sched, backend); LLAMA_LOG_INFO("%s: %10s compute buffer size = %8.2f MiB\n", __func__, - ggml_backend_buffer_name(buf), - ggml_backend_buffer_get_size(buf) / 1024.0 / 1024.0); + ggml_backend_buft_name(buft), + size / 1024.0 / 1024.0); } // note: the number of splits during measure is higher than during inference due to the kv shift diff --git a/scripts/sync-ggml.last b/scripts/sync-ggml.last index 6ae75bc31..7a23ab162 100644 --- a/scripts/sync-ggml.last +++ b/scripts/sync-ggml.last @@ -1 +1 @@ -2c7cf49810d523b9632da393a9e8270b60bf3b24 +5070f078a67c18c11736e78316ab715ca9afde16 From 4a46d2b7923be83d6019251671ee63aa1fa0d6bc Mon Sep 17 00:00:00 2001 From: Daniel Bevenius Date: Mon, 12 Feb 2024 09:38:44 +0100 Subject: [PATCH 326/812] llava : remove prog parameter from ArgumentParser (#5457) * llava: remove prog parameter from ArgumentParser This commit removes the `prog` parameter from `ArgumentParser` so that it uses the default value which is the name of the script. The motivation for this change is that currently the usage output looks like this: ```console $ python examples/llava/convert-image-encoder-to-gguf.py --help usage: convert_hf_to_gguf.py [-h] ... ``` And with this change it will look like this: ```console $ python examples/llava/convert-image-encoder-to-gguf.py --help usage: convert-image-encoder-to-gguf.py [-h] ... ``` Signed-off-by: Daniel Bevenius * ci: add W503 to flake8 ignore list This commit adds W503 to the ignore list for flake8. This is done to avoid the following error: W503 line break before binary operator Signed-off-by: Daniel Bevenius --------- Signed-off-by: Daniel Bevenius --- .github/workflows/python-lint.yml | 2 +- examples/llava/convert-image-encoder-to-gguf.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/python-lint.yml b/.github/workflows/python-lint.yml index 56d17b66c..ea0a05ea1 100644 --- a/.github/workflows/python-lint.yml +++ b/.github/workflows/python-lint.yml @@ -16,5 +16,5 @@ jobs: - name: flake8 Lint uses: py-actions/flake8@v2 with: - ignore: "E203,E211,E221,E225,E231,E241,E251,E261,E266,E501,E701,E704" + ignore: "E203,E211,E221,E225,E231,E241,E251,E261,E266,E501,E701,E704,W503" exclude: "examples/*,examples/*/**,*/**/__init__.py" diff --git a/examples/llava/convert-image-encoder-to-gguf.py b/examples/llava/convert-image-encoder-to-gguf.py index f5a3c9b46..e204b56be 100644 --- a/examples/llava/convert-image-encoder-to-gguf.py +++ b/examples/llava/convert-image-encoder-to-gguf.py @@ -71,7 +71,7 @@ def bytes_to_unicode(): return dict(zip(bs, cs)) -ap = argparse.ArgumentParser(prog="convert_hf_to_gguf.py") +ap = argparse.ArgumentParser() ap.add_argument("-m", "--model-dir", help="Path to model directory cloned from HF Hub", required=True) ap.add_argument("--use-f32", action="store_true", default=False, help="Use f32 instead of f16") ap.add_argument("--text-only", action="store_true", required=False, From 43fe07c1a4f3a58612e1d9543f7c6b556710f5d0 Mon Sep 17 00:00:00 2001 From: Abhilash Majumder <30946547+abhilash1910@users.noreply.github.com> Date: Mon, 12 Feb 2024 20:22:05 +0530 Subject: [PATCH 327/812] ggml-sycl: Replace 3d ops with macro (#5458) * use macro * use macro * fix format --- ggml-sycl.cpp | 75 ++++++++++----------------------------------------- 1 file changed, 14 insertions(+), 61 deletions(-) diff --git a/ggml-sycl.cpp b/ggml-sycl.cpp index dd562a898..cd4b3a1e1 100644 --- a/ggml-sycl.cpp +++ b/ggml-sycl.cpp @@ -11578,11 +11578,8 @@ static dpct::err0 ggml_sycl_cpy_tensor_2d(void *dst, } char * dst_ptr = (char *) dst; - const int64_t ne0 = src->ne[0]; - const int64_t nb0 = src->nb[0]; - const int64_t nb1 = src->nb[1]; - const int64_t nb2 = src->nb[2]; - const int64_t nb3 = src->nb[3]; + GGML_TENSOR_LOCALS_1(int64_t, ne, src, ne); + GGML_TENSOR_LOCALS(int64_t, nb, src, nb); const enum ggml_type type = src->type; const int64_t ts = ggml_type_size(type); const int64_t bs = ggml_blck_size(type); @@ -12426,9 +12423,7 @@ inline void ggml_sycl_op_alibi(const ggml_tensor *src0, const ggml_tensor *src1, GGML_ASSERT(src0->type == GGML_TYPE_F32); GGML_ASSERT( dst->type == GGML_TYPE_F32); - const int64_t ne00 = src0->ne[0]; - const int64_t ne01 = src0->ne[1]; - const int64_t ne02 = src0->ne[2]; + GGML_TENSOR_LOCALS_3(int64_t, ne0, src0, ne); const int64_t nrows = ggml_nrows(src0); //const int n_past = ((int32_t *) dst->op_params)[0]; @@ -12758,15 +12753,9 @@ static void ggml_sycl_op_mul_mat(const ggml_tensor *src0, ggml_sycl_op_mul_mat_t op, const bool convert_src1_to_q8_1) try { - const int64_t ne00 = src0->ne[0]; - const int64_t ne01 = src0->ne[1]; - const int64_t ne02 = src0->ne[2]; - const int64_t ne03 = src0->ne[3]; + GGML_TENSOR_LOCALS(int64_t, ne0, src0, ne); - const int64_t ne10 = src1->ne[0]; - const int64_t ne11 = src1->ne[1]; - const int64_t ne12 = src1->ne[2]; - const int64_t ne13 = src1->ne[3]; + GGML_TENSOR_LOCALS(int64_t, ne1, src1, ne); const int64_t nrows1 = ggml_nrows(src1); GGML_ASSERT(ne03 == ne13); @@ -13337,23 +13326,13 @@ static void ggml_sycl_mul_mat_mat_batched_sycl(const ggml_tensor *src0, GGML_ASSERT(src0->type == GGML_TYPE_F16); GGML_ASSERT(src1->type == GGML_TYPE_F32); - const int64_t ne00 = src0->ne[0]; GGML_UNUSED(ne00); - const int64_t ne01 = src0->ne[1]; - const int64_t ne02 = src0->ne[2]; - const int64_t ne03 = src0->ne[3]; + GGML_TENSOR_LOCALS(int64_t, ne0, src0, ne); - const int64_t nb01 = src0->nb[1]; - const int64_t nb02 = src0->nb[2]; GGML_UNUSED(nb02); - const int64_t nb03 = src0->nb[3]; GGML_UNUSED(nb03); + GGML_TENSOR_LOCALS(int64_t, nb0, src0, nb); - const int64_t ne10 = src1->ne[0]; - const int64_t ne11 = src1->ne[1]; - const int64_t ne12 = src1->ne[2]; - const int64_t ne13 = src1->ne[3]; + GGML_TENSOR_LOCALS(int64_t, ne1, src1, ne); - const int64_t nb11 = src1->nb[1]; - const int64_t nb12 = src1->nb[2]; GGML_UNUSED(nb12); - const int64_t nb13 = src1->nb[3]; GGML_UNUSED(nb13); + GGML_TENSOR_LOCALS(int64_t, nb1, src1, nb); const int64_t ne1 = ggml_nelements(src1); const int64_t ne = ggml_nelements(dst); @@ -13655,23 +13634,15 @@ static void ggml_sycl_mul_mat_id_sycl(ggml_tensor * dst) { GGML_ASSERT(src00->backend != GGML_BACKEND_GPU_SPLIT); GGML_ASSERT(src1->type == GGML_TYPE_F32); - const int64_t ne00 = src00->ne[0]; GGML_UNUSED(ne00); - const int64_t ne01 = src00->ne[1]; - const int64_t ne02 = src00->ne[2]; - const int64_t ne03 = src00->ne[3]; + GGML_TENSOR_LOCALS(int64_t, ne0, src00, ne); //const int64_t nb01 = src00->nb[1]; - const int64_t nb02 = src00->nb[2]; GGML_UNUSED(nb02); - const int64_t nb03 = src00->nb[3]; GGML_UNUSED(nb03); + GGML_TENSOR_LOCALS(int64_t, nb0, src00, nb); - const int64_t ne10 = src1->ne[0]; - const int64_t ne11 = src1->ne[1]; - const int64_t ne12 = src1->ne[2]; - const int64_t ne13 = src1->ne[3]; + GGML_TENSOR_LOCALS(int64_t, ne1, src1, ne); + GGML_TENSOR_LOCALS(int64_t, nb1, src1, nb); //const int64_t nb11 = src1->nb[1]; - const int64_t nb12 = src1->nb[2]; GGML_UNUSED(nb12); - const int64_t nb13 = src1->nb[3]; GGML_UNUSED(nb13); const int64_t ne1 = ggml_nelements(src1); const int64_t ne = ggml_nelements(dst); @@ -13940,25 +13911,7 @@ static void ggml_sycl_cpy(const ggml_tensor *src0, const ggml_tensor *src1, GGML_ASSERT(ggml_nbytes(src0) <= INT_MAX); GGML_ASSERT(ggml_nbytes(src1) <= INT_MAX); - const int64_t ne00 = src0->ne[0]; - const int64_t ne01 = src0->ne[1]; - const int64_t ne02 = src0->ne[2]; - - - const int64_t nb00 = src0->nb[0]; - const int64_t nb01 = src0->nb[1]; - const int64_t nb02 = src0->nb[2]; - const int64_t nb03 = src0->nb[3]; - - const int64_t ne10 = src1->ne[0]; - const int64_t ne11 = src1->ne[1]; - const int64_t ne12 = src1->ne[2]; - - - const int64_t nb10 = src1->nb[0]; - const int64_t nb11 = src1->nb[1]; - const int64_t nb12 = src1->nb[2]; - const int64_t nb13 = src1->nb[3]; + GGML_TENSOR_BINARY_OP_LOCALS; SYCL_CHECK(ggml_sycl_set_device(g_main_device)); dpct::queue_ptr main_stream = g_syclStreams[g_main_device_index][0]; From dbd8828eb03b9aa8d0af7e4c533d3c2f5b38aba6 Mon Sep 17 00:00:00 2001 From: Lee <44310445+lx200916@users.noreply.github.com> Date: Tue, 13 Feb 2024 01:29:57 +0800 Subject: [PATCH 328/812] py : fix persimmon `n_rot` conversion (#5460) * convert : fix persimmon offical weight conversion to write correct n_rot. * Update convert-persimmon-to-gguf.py --------- Co-authored-by: Georgi Gerganov --- convert-persimmon-to-gguf.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/convert-persimmon-to-gguf.py b/convert-persimmon-to-gguf.py index d2be805d1..def210531 100755 --- a/convert-persimmon-to-gguf.py +++ b/convert-persimmon-to-gguf.py @@ -88,7 +88,8 @@ def main(): gguf_writer.add_embedding_length(hidden_size) gguf_writer.add_block_count(block_count) gguf_writer.add_feed_forward_length(hparams.ffn_hidden_size) - gguf_writer.add_rope_dimension_count(hidden_size // head_count) + # ref: https://github.com/ggerganov/llama.cpp/pull/4889/commits/eea19039fc52ea2dbd1aab45b59ab4e3e29a3443 + gguf_writer.add_rope_dimension_count(hidden_size // head_count // 2) gguf_writer.add_head_count(head_count) gguf_writer.add_head_count_kv(head_count_kv) gguf_writer.add_rope_freq_base(hparams.rotary_emb_base) From df334a11251b81fd0b6a0e51e7146e0ba9e973f2 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Mon, 12 Feb 2024 19:54:29 +0200 Subject: [PATCH 329/812] swift : package no longer use ggml dependency (#5465) * Revert "swift : update Package.swift to use ggml as dependency (#4691)" This reverts commit ece9a45e8ffb73ad461c792720c2fec28b0137bc. * spm : add ggml headers --- Package.swift | 24 +++++++++++++++++++----- spm-headers/ggml-alloc.h | 1 + spm-headers/ggml-backend.h | 1 + spm-headers/ggml.h | 1 + 4 files changed, 22 insertions(+), 5 deletions(-) create mode 120000 spm-headers/ggml-alloc.h create mode 120000 spm-headers/ggml-backend.h create mode 120000 spm-headers/ggml.h diff --git a/Package.swift b/Package.swift index 37524edee..b24c9204a 100644 --- a/Package.swift +++ b/Package.swift @@ -13,17 +13,31 @@ let package = Package( products: [ .library(name: "llama", targets: ["llama"]), ], - dependencies: [ - .package(url: "https://github.com/ggerganov/ggml.git", .branch("release")) - ], targets: [ .target( name: "llama", - dependencies: ["ggml"], path: ".", - exclude: ["ggml-metal.metal"], + exclude: [ + "cmake", + "examples", + "scripts", + "models", + "tests", + "CMakeLists.txt", + "ggml-cuda.cu", + "ggml-cuda.h", + "Makefile" + ], sources: [ + "ggml.c", "llama.cpp", + "ggml-alloc.c", + "ggml-backend.c", + "ggml-quants.c", + "ggml-metal.m", + ], + resources: [ + .process("ggml-metal.metal") ], publicHeadersPath: "spm-headers", cSettings: [ diff --git a/spm-headers/ggml-alloc.h b/spm-headers/ggml-alloc.h new file mode 120000 index 000000000..a49d385a1 --- /dev/null +++ b/spm-headers/ggml-alloc.h @@ -0,0 +1 @@ +../ggml-alloc.h \ No newline at end of file diff --git a/spm-headers/ggml-backend.h b/spm-headers/ggml-backend.h new file mode 120000 index 000000000..17c2cf14f --- /dev/null +++ b/spm-headers/ggml-backend.h @@ -0,0 +1 @@ +../ggml-backend.h \ No newline at end of file diff --git a/spm-headers/ggml.h b/spm-headers/ggml.h new file mode 120000 index 000000000..39215298f --- /dev/null +++ b/spm-headers/ggml.h @@ -0,0 +1 @@ +../ggml.h \ No newline at end of file From 099afc6274c859ca67146e725839f2d97a5ef313 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Mon, 12 Feb 2024 20:14:39 +0200 Subject: [PATCH 330/812] llama : fix quantization when tensors are missing (#5423) --- llama.cpp | 32 ++++++++++++++++++++++++-------- 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/llama.cpp b/llama.cpp index a5b873a7b..d316d067b 100644 --- a/llama.cpp +++ b/llama.cpp @@ -772,22 +772,37 @@ struct LLM_TN { llm_arch arch; std::string operator()(llm_tensor tensor) const { + if (LLM_TENSOR_NAMES[arch].find(tensor) == LLM_TENSOR_NAMES[arch].end()) { + return "__missing__"; + } return LLM_TENSOR_NAMES[arch].at(tensor); } std::string operator()(llm_tensor tensor, const std::string & suffix) const { + if (LLM_TENSOR_NAMES[arch].find(tensor) == LLM_TENSOR_NAMES[arch].end()) { + return "__missing__"; + } return LLM_TENSOR_NAMES[arch].at(tensor) + "." + suffix; } std::string operator()(llm_tensor tensor, int bid) const { + if (LLM_TENSOR_NAMES[arch].find(tensor) == LLM_TENSOR_NAMES[arch].end()) { + return "__missing__"; + } return ::format(LLM_TENSOR_NAMES[arch].at(tensor).c_str(), bid); } std::string operator()(llm_tensor tensor, const std::string & suffix, int bid) const { + if (LLM_TENSOR_NAMES[arch].find(tensor) == LLM_TENSOR_NAMES[arch].end()) { + return "__missing__"; + } return ::format(LLM_TENSOR_NAMES[arch].at(tensor).c_str(), bid) + "." + suffix; } std::string operator()(llm_tensor tensor, const std::string & suffix, int bid, int xid) const { + if (LLM_TENSOR_NAMES[arch].find(tensor) == LLM_TENSOR_NAMES[arch].end()) { + return "__missing__"; + } return ::format(LLM_TENSOR_NAMES[arch].at(tensor).c_str(), bid, xid) + "." + suffix; } }; @@ -10227,6 +10242,7 @@ static ggml_type get_k_quant_type(quantize_state_internal & qs, ggml_type new_ty } ++qs.i_ffn_up; } + // if (ftype == LLAMA_FTYPE_MOSTLY_Q2_K) new_type = GGML_TYPE_Q3_K; //} // IK: let's remove this, else Q2_K is almost the same as Q3_K_S @@ -10286,19 +10302,19 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s // K-quants case LLAMA_FTYPE_MOSTLY_Q2_K_S: - case LLAMA_FTYPE_MOSTLY_Q2_K: quantized_type = GGML_TYPE_Q2_K; break; + case LLAMA_FTYPE_MOSTLY_Q2_K: quantized_type = GGML_TYPE_Q2_K; break; case LLAMA_FTYPE_MOSTLY_Q3_K_XS: case LLAMA_FTYPE_MOSTLY_Q3_K_S: case LLAMA_FTYPE_MOSTLY_Q3_K_M: - case LLAMA_FTYPE_MOSTLY_Q3_K_L: quantized_type = GGML_TYPE_Q3_K; break; + case LLAMA_FTYPE_MOSTLY_Q3_K_L: quantized_type = GGML_TYPE_Q3_K; break; case LLAMA_FTYPE_MOSTLY_Q4_K_S: - case LLAMA_FTYPE_MOSTLY_Q4_K_M: quantized_type = GGML_TYPE_Q4_K; break; + case LLAMA_FTYPE_MOSTLY_Q4_K_M: quantized_type = GGML_TYPE_Q4_K; break; case LLAMA_FTYPE_MOSTLY_Q5_K_S: - case LLAMA_FTYPE_MOSTLY_Q5_K_M: quantized_type = GGML_TYPE_Q5_K; break; - case LLAMA_FTYPE_MOSTLY_Q6_K: quantized_type = GGML_TYPE_Q6_K; break; - case LLAMA_FTYPE_MOSTLY_IQ2_XXS:quantized_type = GGML_TYPE_IQ2_XXS; break; - case LLAMA_FTYPE_MOSTLY_IQ2_XS :quantized_type = GGML_TYPE_IQ2_XS; break; - case LLAMA_FTYPE_MOSTLY_IQ3_XXS:quantized_type = GGML_TYPE_IQ3_XXS; break; + case LLAMA_FTYPE_MOSTLY_Q5_K_M: quantized_type = GGML_TYPE_Q5_K; break; + case LLAMA_FTYPE_MOSTLY_Q6_K: quantized_type = GGML_TYPE_Q6_K; break; + case LLAMA_FTYPE_MOSTLY_IQ2_XXS: quantized_type = GGML_TYPE_IQ2_XXS; break; + case LLAMA_FTYPE_MOSTLY_IQ2_XS: quantized_type = GGML_TYPE_IQ2_XS; break; + case LLAMA_FTYPE_MOSTLY_IQ3_XXS: quantized_type = GGML_TYPE_IQ3_XXS; break; default: throw std::runtime_error(format("invalid output file type %d\n", ftype)); } From 895407f31b358e3d9335e847d13f033491ec8a5b Mon Sep 17 00:00:00 2001 From: Kawrakow <48489457+ikawrakow@users.noreply.github.com> Date: Tue, 13 Feb 2024 09:07:57 +0200 Subject: [PATCH 331/812] ggml-quants : fix compiler warnings (shadow variable) (#5472) Co-authored-by: Iwan Kawrakow --- ggml-quants.c | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/ggml-quants.c b/ggml-quants.c index b2a309bf8..f44377f45 100644 --- a/ggml-quants.c +++ b/ggml-quants.c @@ -3819,15 +3819,15 @@ void ggml_vec_dot_q4_0_q8_0(int n, float * restrict s, size_t bs, const void * r /* Compute combined scale for the block */ const __m256 d = _mm256_set1_ps( GGML_FP16_TO_FP32(x[i].d) * GGML_FP16_TO_FP32(y[i].d) ); - __m256i bx = bytes_from_nibbles_32(x[i].qs); + __m256i qx = bytes_from_nibbles_32(x[i].qs); // Now we have a vector with bytes in [ 0 .. 15 ] interval. Offset them into [ -8 .. +7 ] interval. const __m256i off = _mm256_set1_epi8( 8 ); - bx = _mm256_sub_epi8( bx, off ); + qx = _mm256_sub_epi8( qx, off ); - __m256i by = _mm256_loadu_si256((const __m256i *)y[i].qs); + __m256i qy = _mm256_loadu_si256((const __m256i *)y[i].qs); - const __m256 q = mul_sum_i8_pairs_float(bx, by); + const __m256 q = mul_sum_i8_pairs_float(qx, qy); /* Multiply q with scale and accumulate */ acc = _mm256_fmadd_ps( d, q, acc ); @@ -4196,10 +4196,10 @@ void ggml_vec_dot_q4_1_q8_1(int n, float * restrict s, size_t bs, const void * r const __m256 d0d1 = _mm256_mul_ps( d0v, d1v ); // Load 16 bytes, and unpack 4 bit fields into bytes, making 32 bytes - const __m256i bx = bytes_from_nibbles_32(x[i].qs); - const __m256i by = _mm256_loadu_si256( (const __m256i *)y[i].qs ); + const __m256i qx = bytes_from_nibbles_32(x[i].qs); + const __m256i qy = _mm256_loadu_si256( (const __m256i *)y[i].qs ); - const __m256 xy = mul_sum_us8_pairs_float(bx, by); + const __m256 xy = mul_sum_us8_pairs_float(qx, qy); // Accumulate d0*d1*x*y #if defined(__AVX2__) @@ -4418,14 +4418,14 @@ void ggml_vec_dot_q5_0_q8_0(int n, float * restrict s, size_t bs, const void * r /* Compute combined scale for the block */ const __m256 d = _mm256_set1_ps(GGML_FP16_TO_FP32(x[i].d) * GGML_FP16_TO_FP32(y[i].d)); - __m256i bx = bytes_from_nibbles_32(x[i].qs); + __m256i qx = bytes_from_nibbles_32(x[i].qs); __m256i bxhi = bytes_from_bits_32(x[i].qh); bxhi = _mm256_andnot_si256(bxhi, _mm256_set1_epi8((char)0xF0)); - bx = _mm256_or_si256(bx, bxhi); + qx = _mm256_or_si256(qx, bxhi); - __m256i by = _mm256_loadu_si256((const __m256i *)y[i].qs); + __m256i qy = _mm256_loadu_si256((const __m256i *)y[i].qs); - const __m256 q = mul_sum_i8_pairs_float(bx, by); + const __m256 q = mul_sum_i8_pairs_float(qx, qy); /* Multiply q with scale and accumulate */ acc = _mm256_fmadd_ps(d, q, acc); @@ -4722,15 +4722,15 @@ void ggml_vec_dot_q5_1_q8_1(int n, float * restrict s, size_t bs, const void * r summs += GGML_FP16_TO_FP32(x[i].m) * y[i].s; - __m256i bx = bytes_from_nibbles_32(x[i].qs); + __m256i qx = bytes_from_nibbles_32(x[i].qs); __m256i bxhi = bytes_from_bits_32(x[i].qh); bxhi = _mm256_and_si256(bxhi, _mm256_set1_epi8(0x10)); - bx = _mm256_or_si256(bx, bxhi); + qx = _mm256_or_si256(qx, bxhi); const __m256 dy = _mm256_set1_ps(y[i].d); - const __m256i by = _mm256_loadu_si256((const __m256i *)y[i].qs); + const __m256i qy = _mm256_loadu_si256((const __m256i *)y[i].qs); - const __m256 q = mul_sum_us8_pairs_float(bx, by); + const __m256 q = mul_sum_us8_pairs_float(qx, qy); acc = _mm256_fmadd_ps(q, _mm256_mul_ps(dx, dy), acc); } @@ -4973,10 +4973,10 @@ void ggml_vec_dot_q8_0_q8_0(int n, float * restrict s, size_t bs, const void * r for (int i = 0; i < nb; ++i) { // Compute combined scale for the block const __m256 d = _mm256_set1_ps(GGML_FP16_TO_FP32(x[i].d) * GGML_FP16_TO_FP32(y[i].d)); - __m256i bx = _mm256_loadu_si256((const __m256i *)x[i].qs); - __m256i by = _mm256_loadu_si256((const __m256i *)y[i].qs); + __m256i qx = _mm256_loadu_si256((const __m256i *)x[i].qs); + __m256i qy = _mm256_loadu_si256((const __m256i *)y[i].qs); - const __m256 q = mul_sum_i8_pairs_float(bx, by); + const __m256 q = mul_sum_i8_pairs_float(qx, qy); // Multiply q with scale and accumulate #if defined(__AVX2__) From 99b8b43d7b185a6483f28cf798a2d968b2e16ca7 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Tue, 13 Feb 2024 11:20:24 +0200 Subject: [PATCH 332/812] tests : disable moe test (#5473) --- tests/test-backend-ops.cpp | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index eb06123d2..9af8517d9 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -2129,14 +2129,13 @@ static bool test_backend(ggml_backend_t backend, test_mode mode, const char * op test_cases.emplace_back(new test_pad()); test_cases.emplace_back(new test_leaky_relu()); + // these tests are disabled to save execution time, but they can be handy for debugging +#if 0 #if !defined(__SANITIZE_THREAD__) // FIXME: these tests use too much memory with thread sanitizer test_cases.emplace_back(new test_moe(8, 2, 1, 4096, 8*1024)); //test_cases.emplace_back(new test_moe(8, 2, 8, 4096, 14336)); #endif - - // these tests are disabled to save execution time, but they can be handy for debugging -#if 0 test_cases.emplace_back(new test_llama(1)); test_cases.emplace_back(new test_llama(2)); test_cases.emplace_back(new test_falcon(1)); From 49cc1f7d67de2da99f3ac185f9ff1319b7bf35f8 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Tue, 13 Feb 2024 13:01:29 +0200 Subject: [PATCH 333/812] bert : add tests + fix quantization (#5475) * llama : do not quantize pos embd and token type tensors * ci : add BERT tests ggml-ci * ci : do not do BERT tests on low-perf nodes ggml-ci --- ci/run.sh | 46 ++++++++++++++++++++++++++++++++++++++++++++++ llama.cpp | 6 +++++- 2 files changed, 51 insertions(+), 1 deletion(-) diff --git a/ci/run.sh b/ci/run.sh index 82fe247a5..a4264d775 100755 --- a/ci/run.sh +++ b/ci/run.sh @@ -568,6 +568,50 @@ function gg_sum_open_llama_7b_v2 { #gg_printf '- shakespeare (q8_0 / f16 base lora):\n```\n%s\n```\n' "$(cat $OUT/${ci}-ppl-shakespeare-lora-q8_0-f16.log)" } +# bge-small + +function gg_run_embd_bge_small { + cd ${SRC} + + gg_wget models-mnt/bge-small/ https://huggingface.co/BAAI/bge-small-en-v1.5/raw/main/config.json + gg_wget models-mnt/bge-small/ https://huggingface.co/BAAI/bge-small-en-v1.5/resolve/main/tokenizer.model + gg_wget models-mnt/bge-small/ https://huggingface.co/BAAI/bge-small-en-v1.5/raw/main/tokenizer_config.json + gg_wget models-mnt/bge-small/ https://huggingface.co/BAAI/bge-small-en-v1.5/raw/main/special_tokens_map.json + gg_wget models-mnt/bge-small/ https://huggingface.co/BAAI/bge-small-en-v1.5/resolve/main/pytorch_model.bin + gg_wget models-mnt/bge-small/ https://huggingface.co/BAAI/bge-small-en-v1.5/raw/main/sentence_bert_config.json + gg_wget models-mnt/bge-small/ https://huggingface.co/BAAI/bge-small-en-v1.5/raw/main/vocab.txt + + path_models="../models-mnt/bge-small" + + rm -rf build-ci-release && mkdir build-ci-release && cd build-ci-release + + set -e + + (time cmake -DCMAKE_BUILD_TYPE=Release ${CMAKE_EXTRA} .. ) 2>&1 | tee -a $OUT/${ci}-cmake.log + (time make -j ) 2>&1 | tee -a $OUT/${ci}-make.log + + python3 ../convert-hf-to-gguf.py ${path_models} + + model_f16="${path_models}/ggml-model-f16.gguf" + model_q8_0="${path_models}/ggml-model-q8_0.gguf" + + ./bin/quantize ${model_f16} ${model_q8_0} q8_0 + + (time ./bin/embedding --model ${model_f16} -p "I believe the meaning of life is" ) 2>&1 | tee -a $OUT/${ci}-tg-f16.log + (time ./bin/embedding --model ${model_q8_0} -p "I believe the meaning of life is" ) 2>&1 | tee -a $OUT/${ci}-tg-q8_0.log + + set +e +} + +function gg_sum_embd_bge_small { + gg_printf '### %s\n\n' "${ci}" + + gg_printf 'BGE Small (BERT):\n' + gg_printf '- status: %s\n' "$(cat $OUT/${ci}.exit)" + gg_printf '- f16: \n```\n%s\n```\n' "$(cat $OUT/${ci}-tg-f16.log)" + gg_printf '- q8_0:\n```\n%s\n```\n' "$(cat $OUT/${ci}-tg-q8_0.log)" +} + ## main if [ -z ${GG_BUILD_LOW_PERF} ]; then @@ -591,6 +635,8 @@ test $ret -eq 0 && gg_run ctest_debug test $ret -eq 0 && gg_run ctest_release if [ -z ${GG_BUILD_LOW_PERF} ]; then + test $ret -eq 0 && gg_run embd_bge_small + if [ -z ${GG_BUILD_VRAM_GB} ] || [ ${GG_BUILD_VRAM_GB} -ge 8 ]; then if [ -z ${GG_BUILD_CUDA} ]; then test $ret -eq 0 && gg_run open_llama_3b_v2 diff --git a/llama.cpp b/llama.cpp index d316d067b..6dce392df 100644 --- a/llama.cpp +++ b/llama.cpp @@ -10444,7 +10444,11 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s quantize &= !params->only_copy; // do not quantize expert gating tensors - quantize &= name.find("ffn_gate_inp.weight") == std::string::npos; + quantize &= name != LLM_TN(model.arch)(LLM_TENSOR_FFN_GATE_INP, "weight"); + + // do not quantize positional embeddings and token types (BERT) + quantize &= name != LLM_TN(model.arch)(LLM_TENSOR_POS_EMBD, "weight"); + quantize &= name != LLM_TN(model.arch)(LLM_TENSOR_TOKEN_TYPES, "weight"); enum ggml_type new_type; void * new_data; From ad014bba97ef6ef6c3e2f78b2fc463e91ae94579 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20G=C3=A4=C3=9Fler?= Date: Tue, 13 Feb 2024 12:38:37 +0100 Subject: [PATCH 334/812] make: add error message for bad CUDA version (#5444) * make: add error message for bad CUDA version * Update Makefile Co-authored-by: Jared Van Bortel --------- Co-authored-by: Jared Van Bortel --- Makefile | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/Makefile b/Makefile index ba73f0637..0a2070b53 100644 --- a/Makefile +++ b/Makefile @@ -569,6 +569,14 @@ $(info I CC: $(shell $(CC) --version | head -n 1)) $(info I CXX: $(shell $(CXX) --version | head -n 1)) ifdef LLAMA_CUBLAS $(info I NVCC: $(shell $(NVCC) --version | tail -n 1)) +CUDA_VERSION := $(shell nvcc --version | grep -oP 'release (\K[0-9]+\.[0-9])') +ifeq ($(shell awk -v "v=$(CUDA_VERSION)" 'BEGIN { print (v < 11.7) }'),1) +ifndef CUDA_DOCKER_ARCH +ifndef CUDA_POWER_ARCH +$(error I ERROR: For CUDA versions < 11.7 a target CUDA architecture must be explicitly provided via CUDA_DOCKER_ARCH) +endif # CUDA_POWER_ARCH +endif # CUDA_DOCKER_ARCH +endif # eq ($(shell echo "$(CUDA_VERSION) < 11.7" | bc),1) endif # LLAMA_CUBLAS $(info ) From 03bf161eb6dea6400ee49c6dc6b69bdcfa9fd3fc Mon Sep 17 00:00:00 2001 From: Douglas Hanley Date: Tue, 13 Feb 2024 06:06:58 -0600 Subject: [PATCH 335/812] llama : support batched embeddings (#5466) * batched embedding: pool outputs by sequence id. updated embedding example * bring back non-causal attention * embd : minor improvements * llama : minor --------- Co-authored-by: Georgi Gerganov --- convert-hf-to-gguf.py | 1 + examples/embedding/embedding.cpp | 146 +++++++++++++++++++++++-------- gguf-py/gguf/constants.py | 1 + gguf-py/gguf/gguf_writer.py | 3 + llama.cpp | 61 +++++++++---- llama.h | 5 ++ 6 files changed, 163 insertions(+), 54 deletions(-) diff --git a/convert-hf-to-gguf.py b/convert-hf-to-gguf.py index cae1551a2..5adfdc143 100755 --- a/convert-hf-to-gguf.py +++ b/convert-hf-to-gguf.py @@ -1648,6 +1648,7 @@ class BertModel(Model): self.gguf_writer.add_head_count(self.hparams["num_attention_heads"]) self.gguf_writer.add_layer_norm_eps(self.hparams["layer_norm_eps"]) self.gguf_writer.add_causal_attention(False) + self.gguf_writer.add_pooling_layer(True) self.gguf_writer.add_file_type(self.ftype) def set_vocab(self): diff --git a/examples/embedding/embedding.cpp b/examples/embedding/embedding.cpp index 27376c8f0..b4688cf51 100644 --- a/examples/embedding/embedding.cpp +++ b/examples/embedding/embedding.cpp @@ -7,6 +7,51 @@ #pragma warning(disable: 4244 4267) // possible loss of data #endif +static std::vector split_lines(const std::string & s) { + std::string line; + std::vector lines; + std::stringstream ss(s); + while (std::getline(ss, line)) { + lines.push_back(line); + } + return lines; +} + +static void batch_add_seq(llama_batch & batch, const std::vector & tokens, int seq_id) { + for (size_t i = 0; i < tokens.size(); i++) { + llama_batch_add(batch, tokens[i], i, { seq_id }, false); + } +} + +static void normalize(float * vec, float * out, int n) { + float norm = 0; + for (int i = 0; i < n; i++) { + norm += vec[i] * vec[i]; + } + norm = sqrt(norm); + for (int i = 0; i < n; i++) { + out[i] = vec[i] / norm; + } +} + +static void batch_decode(llama_context * ctx, llama_batch & batch, float * output, int n_seq, int n_embd) { + // clear previous kv_cache values (irrelevant for embeddings) + llama_kv_cache_clear(ctx); + + // run model + fprintf(stderr, "%s: n_tokens = %d, n_seq = %d\n", __func__, batch.n_tokens, n_seq); + if (llama_decode(ctx, batch) < 0) { + fprintf(stderr, "%s : failed to decode\n", __func__); + } + + // normalize on copy + for (int k = 0; k < n_seq; k++) { + float * emb = llama_get_embeddings_ith(ctx, k); + float * out = output + k * n_embd; + normalize(emb, out, n_embd); + } +} + int main(int argc, char ** argv) { gpt_params params; @@ -55,59 +100,84 @@ int main(int argc, char ** argv) { fprintf(stderr, "%s\n", get_system_info(params).c_str()); } - int n_past = 0; + // split the prompt into lines + std::vector prompts = split_lines(params.prompt); - // tokenize the prompt - auto embd_inp = ::llama_tokenize(ctx, params.prompt, true); + // max batch size + const uint64_t n_batch = params.n_batch; + GGML_ASSERT(params.n_batch == params.n_ctx); + // tokenize the prompts and trim + std::vector> inputs; + for (const auto & prompt : prompts) { + auto inp = ::llama_tokenize(ctx, prompt, true); + if (inp.size() > n_batch) { + inp.resize(n_batch); + } + inputs.push_back(inp); + } + + // tokenization stats if (params.verbose_prompt) { - fprintf(stderr, "\n"); - fprintf(stderr, "%s: prompt: '%s'\n", __func__, params.prompt.c_str()); - fprintf(stderr, "%s: number of tokens in prompt = %zu\n", __func__, embd_inp.size()); - for (int i = 0; i < (int) embd_inp.size(); i++) { - fprintf(stderr, "%6d -> '%s'\n", embd_inp[i], llama_token_to_piece(ctx, embd_inp[i]).c_str()); + for (int i = 0; i < (int) inputs.size(); i++) { + fprintf(stderr, "%s: prompt %d: '%s'\n", __func__, i, prompts[i].c_str()); + fprintf(stderr, "%s: number of tokens in prompt = %zu\n", __func__, inputs[i].size()); + for (int j = 0; j < (int) inputs[i].size(); j++) { + fprintf(stderr, "%6d -> '%s'\n", inputs[i][j], llama_token_to_piece(ctx, inputs[i][j]).c_str()); + } + fprintf(stderr, "\n\n"); } - fprintf(stderr, "\n"); } - if (embd_inp.size() > (size_t)n_ctx) { - fprintf(stderr, "%s: error: prompt is longer than the context window (%zu tokens, n_ctx = %d)\n", - __func__, embd_inp.size(), n_ctx); - return 1; - } - - while (!embd_inp.empty()) { - int n_tokens = std::min(params.n_batch, (int) embd_inp.size()); - if (llama_decode(ctx, llama_batch_get_one(embd_inp.data(), n_tokens, n_past, 0))) { - fprintf(stderr, "%s : failed to eval\n", __func__); - return 1; - } - n_past += n_tokens; - embd_inp.erase(embd_inp.begin(), embd_inp.begin() + n_tokens); - } + // initialize batch + const int n_prompts = prompts.size(); + struct llama_batch batch = llama_batch_init(n_batch, 0, n_prompts); + // allocate output const int n_embd = llama_n_embd(model); - auto * embeddings = llama_get_embeddings(ctx); + std::vector embeddings(n_prompts * n_embd, 0); + float * emb = embeddings.data(); - // l2-normalize embeddings - float norm = 0; - for (int i = 0; i < n_embd; i++) { - norm += embeddings[i] * embeddings[i]; - } - norm = sqrt(norm); - for (int i = 0; i < n_embd; i++) { - embeddings[i] /= norm; + // break into batches + int p = 0; // number of prompts processed already + int s = 0; // number of prompts in current batch + for (int k = 0; k < n_prompts; k++) { + // clamp to n_batch tokens + auto & inp = inputs[k]; + const uint64_t n_toks = inp.size(); + + // encode if at capacity + if (batch.n_tokens + n_toks > n_batch) { + float * out = emb + p * n_embd; + batch_decode(ctx, batch, out, s, n_embd); + llama_batch_clear(batch); + p += s; + s = 0; + } + + // add to batch + batch_add_seq(batch, inp, s); + s += 1; } - for (int i = 0; i < n_embd; i++) { - printf("%f ", embeddings[i]); - } - printf("\n"); + // final batch + float * out = emb + p * n_embd; + batch_decode(ctx, batch, out, s, n_embd); + // print first 3 embeddings + for (int j = 0; j < std::min(3, n_prompts); j++) { + fprintf(stderr, "embedding %d: ", j); + for (int i = 0; i < n_embd; i++) { + fprintf(stderr, "%f ", emb[j * n_embd + i]); + } + fprintf(stderr, "\n\n"); + } + fprintf(stderr, "\n"); + + // clean up llama_print_timings(ctx); llama_free(ctx); llama_free_model(model); - llama_backend_free(); return 0; diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py index a9c13dd38..644e1589c 100644 --- a/gguf-py/gguf/constants.py +++ b/gguf-py/gguf/constants.py @@ -40,6 +40,7 @@ class Keys: TENSOR_DATA_LAYOUT = "{arch}.tensor_data_layout" EXPERT_COUNT = "{arch}.expert_count" EXPERT_USED_COUNT = "{arch}.expert_used_count" + POOLING_LAYER = "{arch}.pooling_layer" class Attention: HEAD_COUNT = "{arch}.attention.head_count" diff --git a/gguf-py/gguf/gguf_writer.py b/gguf-py/gguf/gguf_writer.py index 7af58a46c..d87bd8e88 100644 --- a/gguf-py/gguf/gguf_writer.py +++ b/gguf-py/gguf/gguf_writer.py @@ -360,6 +360,9 @@ class GGUFWriter: def add_causal_attention(self, value: bool) -> None: self.add_bool(Keys.Attention.CAUSAL.format(arch=self.arch), value) + def add_pooling_layer(self, value: bool) -> None: + self.add_bool(Keys.LLM.POOLING_LAYER.format(arch=self.arch), value) + def add_rope_dimension_count(self, count: int) -> None: self.add_uint32(Keys.Rope.DIMENSION_COUNT.format(arch=self.arch), count) diff --git a/llama.cpp b/llama.cpp index 6dce392df..eb6c46f36 100644 --- a/llama.cpp +++ b/llama.cpp @@ -254,6 +254,7 @@ enum llm_kv { LLM_KV_TENSOR_DATA_LAYOUT, LLM_KV_EXPERT_COUNT, LLM_KV_EXPERT_USED_COUNT, + LLM_KV_POOLING_LAYER, LLM_KV_ATTENTION_HEAD_COUNT, LLM_KV_ATTENTION_HEAD_COUNT_KV, @@ -311,6 +312,7 @@ static std::map LLM_KV_NAMES = { { LLM_KV_TENSOR_DATA_LAYOUT, "%s.tensor_data_layout" }, { LLM_KV_EXPERT_COUNT, "%s.expert_count" }, { LLM_KV_EXPERT_USED_COUNT, "%s.expert_used_count" }, + { LLM_KV_POOLING_LAYER, "%s.pooling_layer" }, { LLM_KV_ATTENTION_HEAD_COUNT, "%s.attention.head_count" }, { LLM_KV_ATTENTION_HEAD_COUNT_KV, "%s.attention.head_count_kv" }, @@ -1539,6 +1541,7 @@ struct llama_hparams { float f_max_alibi_bias; bool causal_attn = true; + bool pooling_layer = false; bool operator!=(const llama_hparams & other) const { @@ -1601,6 +1604,7 @@ struct llama_cparams { bool mul_mat_q; bool offload_kqv; + bool do_pooling; ggml_backend_sched_eval_callback cb_eval; void * cb_eval_user_data; @@ -1896,7 +1900,7 @@ struct llama_context { struct ggml_tensor * inp_pos; // I32 [n_batch] struct ggml_tensor * inp_KQ_mask; // F32 [n_ctx, n_batch] struct ggml_tensor * inp_K_shift; // I32 [n_ctx] - struct ggml_tensor * inp_sum; // F32 [1, n_batch] + struct ggml_tensor * inp_sum; // F32 [n_batch, n_batch] #ifdef GGML_USE_MPI ggml_mpi_context * ctx_mpi = NULL; @@ -3053,6 +3057,7 @@ static void llm_load_hparams( ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps); ml.get_key(LLM_KV_ATTENTION_CAUSAL, hparams.causal_attn); ml.get_key(LLM_KV_TOKENIZER_TOKEN_TYPE_COUNT, hparams.n_vocab_type); + ml.get_key(LLM_KV_POOLING_LAYER, hparams.pooling_layer); switch (hparams.n_layer) { case 3: @@ -4859,7 +4864,7 @@ struct llm_build_context { const int32_t n_orig_ctx; const bool do_rope_shift; - const bool causal_attn; + const bool do_pooling; const llm_build_cb & cb; @@ -4903,7 +4908,7 @@ struct llm_build_context { kv_head (worst_case ? n_ctx - n_tokens : kv_self.head), n_orig_ctx (cparams.n_yarn_orig_ctx), do_rope_shift (worst_case || kv_self.has_shift), - causal_attn (hparams.causal_attn), + do_pooling (hparams.pooling_layer && cparams.do_pooling), cb (cb), buf_compute_meta (lctx.buf_compute_meta) { // all initializations should be done in init() @@ -5752,17 +5757,18 @@ struct llm_build_context { const int64_t n_embd_head = hparams.n_embd_head_v; GGML_ASSERT(n_embd_head == hparams.n_embd_head_k); - GGML_ASSERT(n_embd_head == hparams.n_rot); struct ggml_tensor * cur; struct ggml_tensor * inpL; // get input vectors with right size + const size_t stride1 = n_tokens * ggml_type_size(lctx.inp_tokens->type); struct ggml_tensor * inp_pos = ggml_view_1d(ctx0, lctx.inp_pos, n_tokens, 0); - struct ggml_tensor * inp_sum = ggml_view_1d(ctx0, lctx.inp_sum, n_tokens, 0); + struct ggml_tensor * inp_sum = ggml_view_2d(ctx0, lctx.inp_sum, n_tokens, n_tokens, stride1, 0); // construct input embeddings (token, type, position) inpL = llm_build_inp_embd(ctx0, hparams, batch, model.tok_embd, lctx.inp_tokens, lctx.inp_embd, cb); + // token types are hardcoded to zero ("Sentence A") struct ggml_tensor * type_row0 = ggml_view_1d(ctx0, model.type_embd, n_embd, 0); inpL = ggml_add(ctx0, inpL, type_row0); @@ -5832,9 +5838,11 @@ struct llm_build_context { // final output cur = inpL; - // pooling - cur = ggml_mul_mat(ctx0, inp_sum, ggml_cont(ctx0, ggml_transpose(ctx0, cur))); - cb(cur, "result_embed", -1); + // pooling layer + if (do_pooling) { + cur = ggml_mul_mat(ctx0, ggml_cont(ctx0, ggml_transpose(ctx0, cur)), inp_sum); + } + cb(cur, "result_embd", -1); ggml_build_forward_expand(gf, cur); @@ -7367,7 +7375,8 @@ static void llama_set_inputs(llama_context & lctx, const llama_batch & batch) { for (int i = 0; i < n_kv; ++i) { float f; - if (!lctx.kv_self.cells[i].has_seq_id(seq_id) || lctx.kv_self.cells[i].pos > pos) { + if (!lctx.kv_self.cells[i].has_seq_id(seq_id) || + (hparams.causal_attn && lctx.kv_self.cells[i].pos > pos)) { f = -INFINITY; } else { f = 0; @@ -7378,7 +7387,6 @@ static void llama_set_inputs(llama_context & lctx, const llama_batch & batch) { } } - { assert(ggml_backend_buffer_is_host(lctx.inp_sum->buffer)); float * data = (float *) lctx.inp_sum->data; @@ -7399,6 +7407,20 @@ static void llama_set_inputs(llama_context & lctx, const llama_batch & batch) { data[i] = lctx.kv_self.cells[i].delta; } } + + if (hparams.pooling_layer && cparams.do_pooling) { + const int64_t n_tokens = batch.n_tokens; + + GGML_ASSERT(ggml_backend_buffer_is_host(lctx.inp_sum->buffer)); + float * data = (float *) lctx.inp_sum->data; + + memset(lctx.inp_sum->data, 0, batch.n_tokens * batch.n_tokens * ggml_element_size(lctx.inp_sum)); + + for (int i = 0; i < n_tokens; ++i) { + const llama_seq_id seq_id = batch.seq_id[i][0]; + data[seq_id*n_tokens + i] = 1.0f; + } + } } // decode a batch of tokens by evaluating the transformer @@ -7510,7 +7532,7 @@ static int llama_decode_internal( embeddings = gf->nodes[gf->n_nodes - 3]; GGML_ASSERT(strcmp(embeddings->name, "result_norm") == 0); } - } else if (strcmp(res->name, "result_embed") == 0) { + } else if (strcmp(res->name, "result_embd") == 0) { embeddings = res; res = nullptr; } else { @@ -7630,11 +7652,12 @@ static int llama_decode_internal( if (!lctx.embedding.empty()) { auto & embedding_out = lctx.embedding; - const int64_t embed_pos = res ? n_embd * (n_tokens-1) : 0; + const int64_t embd_pos = res ? n_embd * (n_tokens-1) : 0; + const int64_t embd_size = res ? n_embd : n_embd * n_tokens; - embedding_out.resize(n_embd); + embedding_out.resize(embd_size); ggml_backend_t embeddings_backend = ggml_backend_sched_get_node_backend(lctx.sched, embeddings); - ggml_backend_tensor_get_async(embeddings_backend, embeddings, embedding_out.data(), embed_pos*sizeof(float), n_embd*sizeof(float)); + ggml_backend_tensor_get_async(embeddings_backend, embeddings, embedding_out.data(), embd_pos*sizeof(float), embd_size*sizeof(float)); ggml_backend_synchronize(embeddings_backend); } @@ -10950,6 +10973,7 @@ struct llama_context_params llama_context_default_params() { /*.logits_all =*/ false, /*.embedding =*/ false, /*.offload_kqv =*/ true, + /*.do_pooling =*/ true, }; return result; @@ -11105,6 +11129,7 @@ struct llama_context * llama_new_context_with_model( cparams.yarn_beta_slow = params.yarn_beta_slow; cparams.mul_mat_q = params.mul_mat_q; cparams.offload_kqv = params.offload_kqv; + cparams.do_pooling = params.do_pooling; cparams.n_ctx = params.n_ctx == 0 ? hparams.n_ctx_train : params.n_ctx; cparams.rope_freq_base = params.rope_freq_base == 0.0f ? hparams.rope_freq_base_train : params.rope_freq_base; @@ -11252,7 +11277,7 @@ struct llama_context * llama_new_context_with_model( // resized during inference, reserve maximum ctx->logits.reserve(hparams.n_vocab*cparams.n_batch); - if (params.embedding){ + if (params.embedding) { ctx->embedding.resize(hparams.n_embd); } @@ -11270,7 +11295,7 @@ struct llama_context * llama_new_context_with_model( ctx->inp_pos = ggml_new_tensor_1d(ctx->ctx_input, GGML_TYPE_I32, cparams.n_batch); ctx->inp_KQ_mask = ggml_new_tensor_2d(ctx->ctx_input, GGML_TYPE_F32, cparams.n_ctx, cparams.n_batch); ctx->inp_K_shift = ggml_new_tensor_1d(ctx->ctx_input, GGML_TYPE_I32, cparams.n_ctx); - ctx->inp_sum = ggml_new_tensor_2d(ctx->ctx_input, GGML_TYPE_F32, 1, cparams.n_batch); + ctx->inp_sum = ggml_new_tensor_2d(ctx->ctx_input, GGML_TYPE_F32, cparams.n_batch, cparams.n_batch); ggml_set_name(ctx->inp_tokens, "inp_tokens"); ggml_set_name(ctx->inp_embd, "inp_embd"); @@ -12128,6 +12153,10 @@ float * llama_get_embeddings(struct llama_context * ctx) { return ctx->embedding.data(); } +float * llama_get_embeddings_ith(struct llama_context * ctx, int32_t i) { + return ctx->embedding.data() + i*ctx->model.hparams.n_embd; +} + const char * llama_token_get_text(const struct llama_model * model, llama_token token) { return model->vocab.id_to_token[token].text.c_str(); } diff --git a/llama.h b/llama.h index 367e8f1a1..5ef78ec96 100644 --- a/llama.h +++ b/llama.h @@ -236,6 +236,7 @@ extern "C" { bool logits_all; // the llama_eval() call computes all logits, not just the last one (DEPRECATED - set llama_batch.logits instead) bool embedding; // embedding mode only bool offload_kqv; // whether to offload the KQV ops (including the KV cache) to GPU + bool do_pooling; // whether to pool (sum) embedding results by sequence id (ignored if no pooling layer) }; // model quantization parameters @@ -628,6 +629,10 @@ extern "C" { // shape: [n_embd] (1-dimensional) LLAMA_API float * llama_get_embeddings(struct llama_context * ctx); + // Get the embeddings for the ith sequence + // llama_get_embeddings(ctx) + i*n_embd + LLAMA_API float * llama_get_embeddings_ith(struct llama_context * ctx, int32_t i); + // // Vocab // From cf45252a7cfcb998bade46a886e20477cecc538a Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Tue, 13 Feb 2024 15:14:22 +0200 Subject: [PATCH 336/812] tests : multi-thread the tokenizer tests (#5474) * tests : multi-thread the tokenizer tests ggml-ci * unicode : fix data race for unidentified codepoints ggml-ci * unicode : minor style fixes ggml-ci --- llama.cpp | 24 +++++----- tests/test-tokenizer-1-bpe.cpp | 77 ++++++++++++++++---------------- tests/test-tokenizer-1-llama.cpp | 53 ++++++++++++---------- unicode.h | 72 ++++++++++++++++------------- 4 files changed, 124 insertions(+), 102 deletions(-) diff --git a/llama.cpp b/llama.cpp index eb6c46f36..381a03068 100644 --- a/llama.cpp +++ b/llama.cpp @@ -7782,7 +7782,7 @@ struct llm_bigram_spm { }; struct llm_tokenizer_spm { - llm_tokenizer_spm(const llama_vocab & vocab): vocab(vocab) {} + llm_tokenizer_spm(const llama_vocab & vocab) : vocab(vocab) {} void tokenize(const std::string & text, std::vector & output) { // split string into utf8 chars @@ -7857,6 +7857,7 @@ private: if (p == rev_merge.end()) { // output any symbols that did not form tokens as bytes. + output.reserve(output.size() + symbol.n); for (int j = 0; j < (int)symbol.n; ++j) { llama_vocab::id token_id = llama_byte_to_token(vocab, symbol.text[j]); output.push_back(token_id); @@ -8419,17 +8420,18 @@ struct fragment_buffer_variant { token(_token), raw_text(_dummy), offset(0), - length(0){} + length(0) {} + fragment_buffer_variant(const std::string & _raw_text, int64_t _offset, int64_t _length) : type(FRAGMENT_BUFFER_VARIANT_TYPE_RAW_TEXT), - token((llama_vocab::id)-1), + token((llama_vocab::id) - 1), raw_text(_raw_text), offset(_offset), length(_length){ - GGML_ASSERT( _offset >= 0 ); - GGML_ASSERT( _length >= 1 ); - GGML_ASSERT( offset + length <= raw_text.length() ); + GGML_ASSERT(_offset >= 0); + GGML_ASSERT(_length >= 1); + GGML_ASSERT(offset + length <= raw_text.length()); } const FRAGMENT_BUFFER_VARIANT_TYPE type; @@ -8553,14 +8555,14 @@ static std::vector llama_tokenize_internal(const llama_vocab & } std::forward_list fragment_buffer; - fragment_buffer.emplace_front( raw_text, 0, raw_text.length() ); + fragment_buffer.emplace_front(raw_text, 0, raw_text.length()); - if (special) tokenizer_st_partition( vocab, fragment_buffer ); + if (special) tokenizer_st_partition(vocab, fragment_buffer); switch (vocab.type) { case LLAMA_VOCAB_TYPE_SPM: { - for (const auto & fragment: fragment_buffer) { + for (const auto & fragment : fragment_buffer) { if (fragment.type == FRAGMENT_BUFFER_VARIANT_TYPE_RAW_TEXT) { // without adding this leading whitespace, we do not get the same results as the original tokenizer @@ -8588,7 +8590,7 @@ static std::vector llama_tokenize_internal(const llama_vocab & } break; case LLAMA_VOCAB_TYPE_BPE: { - for (const auto & fragment: fragment_buffer) { + for (const auto & fragment : fragment_buffer) { if (fragment.type == FRAGMENT_BUFFER_VARIANT_TYPE_RAW_TEXT) { auto raw_text = fragment.raw_text.substr(fragment.offset, fragment.length); @@ -8604,7 +8606,7 @@ static std::vector llama_tokenize_internal(const llama_vocab & } break; case LLAMA_VOCAB_TYPE_WPM: { - for (const auto & fragment: fragment_buffer) { + for (const auto & fragment : fragment_buffer) { if (fragment.type == FRAGMENT_BUFFER_VARIANT_TYPE_RAW_TEXT) { auto raw_text = fragment.raw_text.substr(fragment.offset, fragment.length); diff --git a/tests/test-tokenizer-1-bpe.cpp b/tests/test-tokenizer-1-bpe.cpp index 386530f23..3bb629561 100644 --- a/tests/test-tokenizer-1-bpe.cpp +++ b/tests/test-tokenizer-1-bpe.cpp @@ -4,13 +4,13 @@ #include "console.h" #include +#include #include #include -#include -#include -#include -#include #include +#include +#include +#include int main(int argc, char **argv) { if (argc < 2) { @@ -74,45 +74,46 @@ int main(int argc, char **argv) { } } catch (const std::invalid_argument &) { - fprintf(stderr, "%s : info: utf8 conversion %d '%s'\n", __func__, i, str.c_str()); + //fprintf(stderr, "%s : info: utf8 conversion %d '%s'\n", __func__, i, str.c_str()); } } - for (uint32_t cp = 0x0000; cp < 0xffff; ++cp) { - // NOTE: these exceptions seem to be necessary, because the GPT2 tokenizer doesn't want to interfere with some ASCII control characters - if ((cp < 0x03 || cp > 0x05) && cp != 0x0b && cp != 0x11 && (cp < 0x13 || cp > 0x17) && cp != 0x19 && (cp < 0x1c || cp > 0x1e) && (cp < 0xd800 || cp > 0xdfff)) { - std::string str = " " + codepoint_to_utf8(cp); - std::vector tokens = llama_tokenize(ctx, str, false); - std::string check = llama_detokenize_bpe(ctx, tokens); - if (str != check) { - fprintf(stderr, "%s : error: codepoint %x detokenizes to '%s'(%zu) instead of '%s'(%zu)\n", - __func__, cp, check.c_str(), check.length(), str.c_str(), str.length()); - return 3; - } - } - } - // Restrict to assigned unicode planes - // for (uint32_t cp = 0x10000; cp < 0x0010ffff; ++cp) { - for (uint32_t cp = 0x10000; cp < 0x00040000; ++cp) { - std::string str = codepoint_to_utf8(cp); - std::vector tokens = llama_tokenize(ctx, str, false); - std::string check = llama_detokenize_bpe(ctx, tokens); - if (str != check) { - fprintf(stderr, "%s : error: codepoint %x detokenizes to '%s'(%zu) instead of '%s'(%zu)\n", - __func__, cp, check.c_str(), check.length(), str.c_str(), str.length()); - return 4; - } - } - for (uint32_t cp = 0x000e0000; cp < 0x0010ffff; ++cp) { - std::string str = codepoint_to_utf8(cp); - std::vector tokens = llama_tokenize(ctx, str, false); - std::string check = llama_detokenize_bpe(ctx, tokens); - if (str != check) { - fprintf(stderr, "%s : error: codepoint %x detokenizes to '%s'(%zu) instead of '%s'(%zu)\n", - __func__, cp, check.c_str(), check.length(), str.c_str(), str.length()); - return 4; + // unicode + { + const int nthread = std::thread::hardware_concurrency(); + + std::vector threads(nthread); + + for (int i = 0; i < nthread; ++i) { + threads[i] = std::thread([i, nthread, ctx]() { + for (uint32_t cp = i; cp < 0x0010ffff; cp += nthread) { + if (!( // NOLINT + (cp < 0x03 || cp > 0x05) && cp != 0x0b && cp != 0x11 && + (cp < 0x13 || cp > 0x17) && cp != 0x19 && + (cp < 0x1c || cp > 0x1e) && + (cp < 0xd800 || cp > 0xdfff) && + (cp < 0x00040000 || cp >= 0x000e0000) + )) { + continue; + } + + std::string str = codepoint_to_utf8(cp); + std::vector tokens = llama_tokenize(ctx, str, false); + std::string check = llama_detokenize_bpe(ctx, tokens); + if (cp != 9601 && str != check) { + fprintf(stderr, "error: codepoint %x detokenizes to '%s'(%zu) instead of '%s'(%zu)\n", + cp, check.c_str(), check.length(), str.c_str(), str.length()); + std::exit(3); + } + } + }); + } + + for (auto & t : threads) { + t.join(); } } + llama_free_model(model); llama_free(ctx); diff --git a/tests/test-tokenizer-1-llama.cpp b/tests/test-tokenizer-1-llama.cpp index 4b58fe495..b0d814a41 100644 --- a/tests/test-tokenizer-1-llama.cpp +++ b/tests/test-tokenizer-1-llama.cpp @@ -4,13 +4,13 @@ #include "console.h" #include +#include #include #include -#include -#include -#include -#include #include +#include +#include +#include int main(int argc, char **argv) { if (argc < 2) { @@ -72,26 +72,33 @@ int main(int argc, char **argv) { } } - for (uint32_t cp = 0x0000; cp < 0xffff; ++cp) { - if (cp < 0xd800 || cp > 0xdfff) { - std::string str = codepoint_to_utf8(cp); - std::vector tokens = llama_tokenize(ctx, str, false); - std::string check = llama_detokenize_spm(ctx, tokens); - if (cp != 9601 && str != check) { - fprintf(stderr, "%s : error: codepoint %d detokenizes to '%s'(%zu) instead of '%s'(%zu)\n", - __func__, cp, check.c_str(), check.length(), str.c_str(), str.length()); - return 3; - } + // unicode + { + const int nthread = std::thread::hardware_concurrency(); + + std::vector threads(nthread); + + for (int i = 0; i < nthread; ++i) { + threads[i] = std::thread([i, nthread, ctx]() { + for (uint32_t cp = i; cp < 0x0010ffff; cp += nthread) { + if (cp >= 0xd800 && cp <= 0xdfff) { + continue; + } + + std::string str = codepoint_to_utf8(cp); + std::vector tokens = llama_tokenize(ctx, str, false); + std::string check = llama_detokenize_spm(ctx, tokens); + if (cp != 9601 && str != check) { + fprintf(stderr, "error: codepoint %x detokenizes to '%s'(%zu) instead of '%s'(%zu)\n", + cp, check.c_str(), check.length(), str.c_str(), str.length()); + std::exit(3); + } + } + }); } - } - for (uint32_t cp = 0x10000; cp < 0x0010ffff; ++cp) { - std::string str = codepoint_to_utf8(cp); - std::vector tokens = llama_tokenize(ctx, str, false); - std::string check = llama_detokenize_spm(ctx, tokens); - if (str != check) { - fprintf(stderr, "%s : error: codepoint %d detokenizes to '%s'(%zu) instead of '%s'(%zu)\n", - __func__, cp, check.c_str(), check.length(), str.c_str(), str.length()); - return 4; + + for (auto & t : threads) { + t.join(); } } diff --git a/unicode.h b/unicode.h index 844eff3da..263260702 100644 --- a/unicode.h +++ b/unicode.h @@ -264,26 +264,29 @@ static uint32_t codepoint_from_utf8(const std::string & utf8, size_t & offset) { offset += 1; return result; } - else if (!(utf8[offset + 0] & 0x40)) { + if (!(utf8[offset + 0] & 0x40)) { throw std::invalid_argument("invalid character"); } - else if (!(utf8[offset + 0] & 0x20)) { - if (offset + 1 >= utf8.size() || ! ((utf8[offset + 1] & 0xc0) == 0x80)) + if (!(utf8[offset + 0] & 0x20)) { + if (offset + 1 >= utf8.size() || ! ((utf8[offset + 1] & 0xc0) == 0x80)) { throw std::invalid_argument("invalid character"); + } auto result = ((utf8[offset + 0] & 0x1f) << 6) | (utf8[offset + 1] & 0x3f); offset += 2; return result; } - else if (!(utf8[offset + 0] & 0x10)) { - if (offset + 2 >= utf8.size() || ! ((utf8[offset + 1] & 0xc0) == 0x80) || ! ((utf8[offset + 2] & 0xc0) == 0x80)) + if (!(utf8[offset + 0] & 0x10)) { + if (offset + 2 >= utf8.size() || ! ((utf8[offset + 1] & 0xc0) == 0x80) || ! ((utf8[offset + 2] & 0xc0) == 0x80)) { throw std::invalid_argument("invalid character"); + } auto result = ((utf8[offset + 0] & 0x0f) << 12) | ((utf8[offset + 1] & 0x3f) << 6) | (utf8[offset + 2] & 0x3f); offset += 3; return result; } - else if (!(utf8[offset + 0] & 0x08)) { - if (offset + 3 >= utf8.size() || ! ((utf8[offset + 1] & 0xc0) == 0x80) || ! ((utf8[offset + 2] & 0xc0) == 0x80) || !((utf8[offset + 3] & 0xc0) == 0x80)) + if (!(utf8[offset + 0] & 0x08)) { + if (offset + 3 >= utf8.size() || ! ((utf8[offset + 1] & 0xc0) == 0x80) || ! ((utf8[offset + 2] & 0xc0) == 0x80) || !((utf8[offset + 3] & 0xc0) == 0x80)) { throw std::invalid_argument("invalid character"); + } auto result = ((utf8[offset + 0] & 0x07) << 18) | ((utf8[offset + 1] & 0x3f) << 12) | ((utf8[offset + 2] & 0x3f) << 6) | (utf8[offset + 3] & 0x3f); offset += 4; return result; @@ -331,21 +334,22 @@ static uint32_t codepoint_from_utf16(const std::vector & utf16, size_t offset += 1; return result; } - else { - if (offset + 1 >= utf16.size() || !((utf16[1] & 0xdc00) == 0xdc00)) - throw std::invalid_argument("invalid character"); - auto result = 0x10000 + (((utf16[0] & 0x03ff) << 10) | (utf16[1] & 0x03ff)); - offset += 2; - return result; + + if (offset + 1 >= utf16.size() || !((utf16[1] & 0xdc00) == 0xdc00)) { + throw std::invalid_argument("invalid character"); } - throw std::invalid_argument("invalid string"); + + auto result = 0x10000 + (((utf16[0] & 0x03ff) << 10) | (utf16[1] & 0x03ff)); + offset += 2; + return result; } static std::vector codepoints_from_utf16(const std::vector & utf16) { std::vector result; size_t offset = 0; - while (offset < utf16.size()) + while (offset < utf16.size()) { result.push_back(codepoint_from_utf16(utf16, offset)); + } return result; } @@ -361,44 +365,52 @@ static std::vector codepoints_from_utf16(const std::vector & static std::unordered_map codepoint_type_map() { std::unordered_map codepoint_types; for (auto p : digit_ranges) { - for(auto i = p.first; i <= p.second; ++ i) + for (auto i = p.first; i <= p.second; ++ i) { codepoint_types[i] = CODEPOINT_TYPE_DIGIT; + } } - for(auto p : letter_ranges) { - for(auto i = p.first; i <= p.second; ++ i) + for (auto p : letter_ranges) { + for (auto i = p.first; i <= p.second; ++ i) { codepoint_types[i] = CODEPOINT_TYPE_LETTER; + } } - for(auto p : whitespace_ranges) { - for(auto i = p.first; i <= p.second; ++ i) + for (auto p : whitespace_ranges) { + for (auto i = p.first; i <= p.second; ++ i) { codepoint_types[i] = CODEPOINT_TYPE_WHITESPACE; + } } - for(auto p : accent_mark_ranges) { - for(auto i = p.first; i <= p.second; ++ i) + for (auto p : accent_mark_ranges) { + for (auto i = p.first; i <= p.second; ++ i) { codepoint_types[i] = CODEPOINT_TYPE_ACCENT_MARK; + } } - for(auto p : punctuation_ranges) { - for(auto i = p.first; i <= p.second; ++ i) + for (auto p : punctuation_ranges) { + for (auto i = p.first; i <= p.second; ++ i) { codepoint_types[i] = CODEPOINT_TYPE_PUNCTUATION; + } } - for (auto p : symbol_ranges) { - for (auto i = p.first; i <= p.second; ++i) + for (auto p : symbol_ranges) { + for (auto i = p.first; i <= p.second; ++i) { codepoint_types[i] = CODEPOINT_TYPE_SYMBOL; + } } - for(auto p : control_ranges) { - for(auto i = p.first; i <= p.second; ++ i) + for (auto p : control_ranges) { + for (auto i = p.first; i <= p.second; ++ i) { codepoint_types[i] = CODEPOINT_TYPE_CONTROL; + } } return codepoint_types; } static int codepoint_type(uint32_t cp) { static std::unordered_map codepoint_types = codepoint_type_map(); - return codepoint_types[cp]; + return codepoint_types.find(cp) == codepoint_types.end() ? CODEPOINT_TYPE_UNIDENTIFIED : codepoint_types.at(cp); } static int codepoint_type(const std::string & utf8) { - if (utf8.length() == 0) + if (utf8.length() == 0) { return CODEPOINT_TYPE_UNIDENTIFIED; + } size_t offset = 0; return codepoint_type(codepoint_from_utf8(utf8, offset)); } From 263978904c7472db1865409a7ff1129599f6a40b Mon Sep 17 00:00:00 2001 From: Daniel Bevenius Date: Tue, 13 Feb 2024 14:15:42 +0100 Subject: [PATCH 337/812] finetune : rename feed-forward tensors (w1/w2/w3) (#4839) * finetune: rename feed-forward tensors (w1/w2/w3) This commit renames the feed-forward tensors w1, w2 and w3 to ffn_gate, ffn_down and ffn_up respectively. The motivation for this change is to make it easier to understand the purpose of the tensors. This also seems to be inline with the names used in the llama_layer struct in llama.cpp. Signed-off-by: Daniel Bevenius * train-text-from-scratch: rename ff tensors This commit renames the feed-forward tensors w1, w2 and w3 to ffn_gate, ffn_down and ffn_up respectively. The motivation for this change is to make it easier to understand the purpose of the tensors. This also seems to be inline with the names used in the llama_layer struct in llama.cpp Signed-off-by: Daniel Bevenius --------- Signed-off-by: Daniel Bevenius --- examples/finetune/README.md | 6 +- examples/finetune/finetune.cpp | 242 +++++++++--------- .../train-text-from-scratch.cpp | 54 ++-- 3 files changed, 151 insertions(+), 151 deletions(-) diff --git a/examples/finetune/README.md b/examples/finetune/README.md index a884706c5..2fafd505e 100644 --- a/examples/finetune/README.md +++ b/examples/finetune/README.md @@ -80,9 +80,9 @@ The LORA rank can be configured for each model tensor type separately with these --rank-wk N LORA rank for wk tensor (default 4) --rank-wv N LORA rank for wv tensor (default 4) --rank-wo N LORA rank for wo tensor (default 4) - --rank-w1 N LORA rank for w1 tensor (default 4) - --rank-w2 N LORA rank for w2 tensor (default 4) - --rank-w3 N LORA rank for w3 tensor (default 4) + --rank-ffn_gate N LORA rank for ffn_gate tensor (default 4) + --rank-ffn_down N LORA rank for ffn_down tensor (default 4) + --rank-ffn_up N LORA rank for ffn_up tensor (default 4) ``` The LORA rank of 'norm' tensors should always be 1. diff --git a/examples/finetune/finetune.cpp b/examples/finetune/finetune.cpp index b11c56020..98bf5a07a 100644 --- a/examples/finetune/finetune.cpp +++ b/examples/finetune/finetune.cpp @@ -60,9 +60,9 @@ struct my_llama_layer { struct ggml_tensor * ffn_norm; // ff - struct ggml_tensor * w1; - struct ggml_tensor * w2; - struct ggml_tensor * w3; + struct ggml_tensor * ffn_gate; // w1 + struct ggml_tensor * ffn_down; // w2 + struct ggml_tensor * ffn_up; // w3 }; struct my_llama_model { @@ -85,9 +85,9 @@ struct my_llama_lora_hparams { uint32_t n_rank_wv = 4; uint32_t n_rank_wo = 4; uint32_t n_rank_ffn_norm = 1; - uint32_t n_rank_w1 = 4; - uint32_t n_rank_w2 = 4; - uint32_t n_rank_w3 = 4; + uint32_t n_rank_ffn_gate = 4; + uint32_t n_rank_ffn_down = 4; + uint32_t n_rank_ffn_up = 4; uint32_t n_rank_tok_embeddings = 4; uint32_t n_rank_norm = 1; uint32_t n_rank_output = 4; @@ -117,12 +117,12 @@ struct my_llama_lora_layer { struct ggml_tensor * ffn_norm_b; // ff - struct ggml_tensor * w1_a; - struct ggml_tensor * w1_b; - struct ggml_tensor * w2_a; - struct ggml_tensor * w2_b; - struct ggml_tensor * w3_a; - struct ggml_tensor * w3_b; + struct ggml_tensor * ffn_gate_a; + struct ggml_tensor * ffn_gate_b; + struct ggml_tensor * ffn_down_a; + struct ggml_tensor * ffn_down_b; + struct ggml_tensor * ffn_up_a; + struct ggml_tensor * ffn_up_b; }; struct my_llama_lora { @@ -208,9 +208,9 @@ static void print_lora_params(struct my_llama_lora_hparams * params) { printf("%s: n_rank_wv : %u\n", __func__, params->n_rank_wv); printf("%s: n_rank_wo : %u\n", __func__, params->n_rank_wo); printf("%s: n_rank_ffn_norm : %u\n", __func__, params->n_rank_ffn_norm); - printf("%s: n_rank_w1 : %u\n", __func__, params->n_rank_w1); - printf("%s: n_rank_w2 : %u\n", __func__, params->n_rank_w2); - printf("%s: n_rank_w3 : %u\n", __func__, params->n_rank_w3); + printf("%s: n_rank_ffn_gate : %u\n", __func__, params->n_rank_ffn_gate); + printf("%s: n_rank_ffn_down : %u\n", __func__, params->n_rank_ffn_down); + printf("%s: n_rank_ffn_up : %u\n", __func__, params->n_rank_ffn_up); printf("%s: n_rank_tok_embeddings : %u\n", __func__, params->n_rank_tok_embeddings); printf("%s: n_rank_norm : %u\n", __func__, params->n_rank_norm); printf("%s: n_rank_output : %u\n", __func__, params->n_rank_output); @@ -319,9 +319,9 @@ static void init_model(struct llama_model * input, struct my_llama_model * model layer.wv = llama_get_model_tensor(input, tni(LLM_TENSOR_ATTN_V, i)); layer.wo = llama_get_model_tensor(input, tni(LLM_TENSOR_ATTN_OUT, i)); layer.ffn_norm = llama_get_model_tensor(input, tni(LLM_TENSOR_FFN_NORM, i)); - layer.w1 = llama_get_model_tensor(input, tni(LLM_TENSOR_FFN_GATE, i)); - layer.w2 = llama_get_model_tensor(input, tni(LLM_TENSOR_FFN_DOWN, i)); - layer.w3 = llama_get_model_tensor(input, tni(LLM_TENSOR_FFN_UP, i)); + layer.ffn_gate = llama_get_model_tensor(input, tni(LLM_TENSOR_FFN_GATE, i)); + layer.ffn_down = llama_get_model_tensor(input, tni(LLM_TENSOR_FFN_DOWN, i)); + layer.ffn_up = llama_get_model_tensor(input, tni(LLM_TENSOR_FFN_UP, i)); assert_shape_1d(layer.attention_norm, hparams.n_embd); assert_shape_2d(layer.wq, hparams.n_embd, hparams.n_embd); @@ -329,9 +329,9 @@ static void init_model(struct llama_model * input, struct my_llama_model * model assert_shape_2d(layer.wv, hparams.n_embd, hparams.n_embd_gqa()); assert_shape_2d(layer.wo, hparams.n_embd, hparams.n_embd); assert_shape_1d(layer.ffn_norm, hparams.n_embd); - assert_shape_2d(layer.w1, hparams.n_embd, hparams.n_ff); - assert_shape_2d(layer.w2, hparams.n_ff, hparams.n_embd); - assert_shape_2d(layer.w3, hparams.n_embd, hparams.n_ff); + assert_shape_2d(layer.ffn_gate, hparams.n_embd, hparams.n_ff); + assert_shape_2d(layer.ffn_down, hparams.n_ff, hparams.n_embd); + assert_shape_2d(layer.ffn_up, hparams.n_embd, hparams.n_ff); } } @@ -362,12 +362,12 @@ static void set_param_lora(struct my_llama_lora * lora) { ggml_set_param(ctx, layer.wo_b); ggml_set_param(ctx, layer.ffn_norm_a); ggml_set_param(ctx, layer.ffn_norm_b); - ggml_set_param(ctx, layer.w1_a); - ggml_set_param(ctx, layer.w1_b); - ggml_set_param(ctx, layer.w2_a); - ggml_set_param(ctx, layer.w2_b); - ggml_set_param(ctx, layer.w3_a); - ggml_set_param(ctx, layer.w3_b); + ggml_set_param(ctx, layer.ffn_gate_a); + ggml_set_param(ctx, layer.ffn_gate_b); + ggml_set_param(ctx, layer.ffn_down_a); + ggml_set_param(ctx, layer.ffn_down_b); + ggml_set_param(ctx, layer.ffn_up_a); + ggml_set_param(ctx, layer.ffn_up_b); } } @@ -435,12 +435,12 @@ static void init_lora(const struct my_llama_model * model, struct my_llama_lora layer.ffn_norm_a = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, lparams.n_rank_ffn_norm, n_embd); layer.ffn_norm_b = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, lparams.n_rank_ffn_norm, 1); - layer.w1_a = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, lparams.n_rank_w1, n_embd); - layer.w1_b = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, lparams.n_rank_w1, n_ff); - layer.w2_a = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, lparams.n_rank_w2, n_ff); - layer.w2_b = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, lparams.n_rank_w2, n_embd); - layer.w3_a = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, lparams.n_rank_w3, n_embd); - layer.w3_b = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, lparams.n_rank_w3, n_ff); + layer.ffn_gate_a = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, lparams.n_rank_ffn_gate, n_embd); + layer.ffn_gate_b = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, lparams.n_rank_ffn_gate, n_ff); + layer.ffn_down_a = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, lparams.n_rank_ffn_down, n_ff); + layer.ffn_down_b = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, lparams.n_rank_ffn_down, n_embd); + layer.ffn_up_a = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, lparams.n_rank_ffn_up, n_embd); + layer.ffn_up_b = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, lparams.n_rank_ffn_up, n_ff); ggml_set_name(layer.attention_norm_a, tni(LLM_TENSOR_ATTN_NORM, ".weight.lora_a", i)); ggml_set_name(layer.attention_norm_b, tni(LLM_TENSOR_ATTN_NORM, ".weight.lora_b", i)); @@ -454,12 +454,12 @@ static void init_lora(const struct my_llama_model * model, struct my_llama_lora ggml_set_name(layer.wo_b, tni(LLM_TENSOR_ATTN_OUT, ".weight.lora_b", i)); ggml_set_name(layer.ffn_norm_a, tni(LLM_TENSOR_FFN_NORM, ".weight.lora_a", i)); ggml_set_name(layer.ffn_norm_b, tni(LLM_TENSOR_FFN_NORM, ".weight.lora_b", i)); - ggml_set_name(layer.w1_a, tni(LLM_TENSOR_FFN_GATE, ".weight.lora_a", i)); - ggml_set_name(layer.w1_b, tni(LLM_TENSOR_FFN_GATE, ".weight.lora_b", i)); - ggml_set_name(layer.w2_a, tni(LLM_TENSOR_FFN_DOWN, ".weight.lora_a", i)); - ggml_set_name(layer.w2_b, tni(LLM_TENSOR_FFN_DOWN, ".weight.lora_b", i)); - ggml_set_name(layer.w3_a, tni(LLM_TENSOR_FFN_UP, ".weight.lora_a", i)); - ggml_set_name(layer.w3_b, tni(LLM_TENSOR_FFN_UP, ".weight.lora_b", i)); + ggml_set_name(layer.ffn_gate_a, tni(LLM_TENSOR_FFN_GATE, ".weight.lora_a", i)); + ggml_set_name(layer.ffn_gate_b, tni(LLM_TENSOR_FFN_GATE, ".weight.lora_b", i)); + ggml_set_name(layer.ffn_down_a, tni(LLM_TENSOR_FFN_DOWN, ".weight.lora_a", i)); + ggml_set_name(layer.ffn_down_b, tni(LLM_TENSOR_FFN_DOWN, ".weight.lora_b", i)); + ggml_set_name(layer.ffn_up_a, tni(LLM_TENSOR_FFN_UP, ".weight.lora_a", i)); + ggml_set_name(layer.ffn_up_b, tni(LLM_TENSOR_FFN_UP, ".weight.lora_b", i)); } set_param_lora(lora); @@ -497,12 +497,12 @@ static void randomize_lora(struct my_llama_lora * lora, int seed, float mean, fl randomize_tensor_normal(layer.ffn_norm_a, rnd); ggml_set_zero(layer.ffn_norm_b); - randomize_tensor_normal(layer.w1_a, rnd); - ggml_set_zero(layer.w1_b); - randomize_tensor_normal(layer.w2_a, rnd); - ggml_set_zero(layer.w2_b); - randomize_tensor_normal(layer.w3_a, rnd); - ggml_set_zero(layer.w3_b); + randomize_tensor_normal(layer.ffn_gate_a, rnd); + ggml_set_zero(layer.ffn_gate_b); + randomize_tensor_normal(layer.ffn_down_a, rnd); + ggml_set_zero(layer.ffn_down_b); + randomize_tensor_normal(layer.ffn_up_a, rnd); + ggml_set_zero(layer.ffn_up_b); } free_random_normal_distribution(rnd); @@ -610,13 +610,13 @@ static struct ggml_tensor * llama_build_lora_finetune_graphs( struct ggml_tensor * attention_norm = add_to_f32(ctx, layer.attention_norm, ggml_mul_mat(ctx, llayer.attention_norm_a, llayer.attention_norm_b)); struct ggml_tensor * ffn_norm = add_to_f32(ctx, layer.ffn_norm, ggml_mul_mat(ctx, llayer.ffn_norm_a, llayer.ffn_norm_b)); - struct ggml_tensor * wq = add_to_f32(ctx, layer.wq, ggml_mul_mat(ctx, llayer.wq_a, llayer.wq_b)); - struct ggml_tensor * wk = add_to_f32(ctx, layer.wk, ggml_mul_mat(ctx, llayer.wk_a, llayer.wk_b)); - struct ggml_tensor * wv = add_to_f32(ctx, layer.wv, ggml_mul_mat(ctx, llayer.wv_a, llayer.wv_b)); - struct ggml_tensor * wo = add_to_f32(ctx, layer.wo, ggml_mul_mat(ctx, llayer.wo_a, llayer.wo_b)); - struct ggml_tensor * w1 = add_to_f32(ctx, layer.w1, ggml_mul_mat(ctx, llayer.w1_a, llayer.w1_b)); - struct ggml_tensor * w2 = add_to_f32(ctx, layer.w2, ggml_mul_mat(ctx, llayer.w2_a, llayer.w2_b)); - struct ggml_tensor * w3 = add_to_f32(ctx, layer.w3, ggml_mul_mat(ctx, llayer.w3_a, llayer.w3_b)); + struct ggml_tensor * wq = add_to_f32(ctx, layer.wq, ggml_mul_mat(ctx, llayer.wq_a, llayer.wq_b)); + struct ggml_tensor * wk = add_to_f32(ctx, layer.wk, ggml_mul_mat(ctx, llayer.wk_a, llayer.wk_b)); + struct ggml_tensor * wv = add_to_f32(ctx, layer.wv, ggml_mul_mat(ctx, llayer.wv_a, llayer.wv_b)); + struct ggml_tensor * wo = add_to_f32(ctx, layer.wo, ggml_mul_mat(ctx, llayer.wo_a, llayer.wo_b)); + struct ggml_tensor * ffn_gate = add_to_f32(ctx, layer.ffn_gate, ggml_mul_mat(ctx, llayer.ffn_gate_a, llayer.ffn_gate_b)); + struct ggml_tensor * ffn_down = add_to_f32(ctx, layer.ffn_down, ggml_mul_mat(ctx, llayer.ffn_down_a, llayer.ffn_down_b)); + struct ggml_tensor * ffn_up = add_to_f32(ctx, layer.ffn_up, ggml_mul_mat(ctx, llayer.ffn_up_a, llayer.ffn_up_b)); struct ggml_tensor * t02 = ggml_rms_norm (ctx, cur, rms_norm_eps); set_name(t02, "t02"); assert_shape_2d(t02, n_embd, N*n_batch); struct ggml_tensor * t03 = ggml_repeat (ctx, attention_norm, t02); set_name(t03, "t03"); assert_shape_2d(t03, n_embd, N*n_batch); @@ -659,11 +659,11 @@ static struct ggml_tensor * llama_build_lora_finetune_graphs( struct ggml_tensor * t22 = ggml_rms_norm (ctx, t21, rms_norm_eps); set_name(t22, "t22"); assert_shape_2d(t22, n_embd, N*n_batch); struct ggml_tensor * t23 = ggml_repeat (ctx, ffn_norm, t22); set_name(t23, "t23"); assert_shape_2d(t23, n_embd, N*n_batch); struct ggml_tensor * t24 = ggml_mul (ctx, t23, t22); set_name(t24, "t24"); assert_shape_2d(t24, n_embd, N*n_batch); - struct ggml_tensor * t25 = ggml_mul_mat (ctx, w3, t24); set_name(t25, "t25"); assert_shape_2d(t25, n_ff, N*n_batch); - struct ggml_tensor * t26 = ggml_mul_mat (ctx, w1, t24); set_name(t26, "t26"); assert_shape_2d(t26, n_ff, N*n_batch); + struct ggml_tensor * t25 = ggml_mul_mat (ctx, ffn_up, t24); set_name(t25, "t25"); assert_shape_2d(t25, n_ff, N*n_batch); + struct ggml_tensor * t26 = ggml_mul_mat (ctx, ffn_gate, t24); set_name(t26, "t26"); assert_shape_2d(t26, n_ff, N*n_batch); struct ggml_tensor * t27 = ggml_silu (ctx, t26); set_name(t27, "t27"); assert_shape_2d(t27, n_ff, N*n_batch); struct ggml_tensor * t28 = ggml_mul (ctx, t27, t25); set_name(t28, "t28"); assert_shape_2d(t28, n_ff, N*n_batch); - struct ggml_tensor * t29 = ggml_mul_mat (ctx, w2, t28); set_name(t29, "t29"); assert_shape_2d(t29, n_embd, N*n_batch); + struct ggml_tensor * t29 = ggml_mul_mat (ctx, ffn_down, t28); set_name(t29, "t29"); assert_shape_2d(t29, n_embd, N*n_batch); struct ggml_tensor * t30 = ggml_add (ctx, t29, t21); set_name(t30, "t30"); assert_shape_2d(t30, n_embd, N*n_batch); cur = t30; if (enable_checkpointing) { @@ -723,9 +723,9 @@ static struct ggml_tensor * llama_build_lora_finetune_graphs( ggml_build_forward_expand(gb, ggml_scale_inplace(ctx, layer.wk, 1.0f)); ggml_build_forward_expand(gb, ggml_scale_inplace(ctx, layer.wv, 1.0f)); ggml_build_forward_expand(gb, ggml_scale_inplace(ctx, layer.wo, 1.0f)); - ggml_build_forward_expand(gb, ggml_scale_inplace(ctx, layer.w1, 1.0f)); - ggml_build_forward_expand(gb, ggml_scale_inplace(ctx, layer.w2, 1.0f)); - ggml_build_forward_expand(gb, ggml_scale_inplace(ctx, layer.w3, 1.0f)); + ggml_build_forward_expand(gb, ggml_scale_inplace(ctx, layer.ffn_gate, 1.0f)); + ggml_build_forward_expand(gb, ggml_scale_inplace(ctx, layer.ffn_down, 1.0f)); + ggml_build_forward_expand(gb, ggml_scale_inplace(ctx, layer.ffn_up, 1.0f)); } // allocating checkpoints in one block to reduce memory fragmentation @@ -798,9 +798,9 @@ static void load_llama_lora_gguf(struct gguf_context * fctx, struct ggml_context GGUF_GET_KEY(fctx, lora->hparams.n_rank_wv, gguf_get_val_u32, GGUF_TYPE_UINT32, true, LLM_KV_TRAINING_LORA_RANK_ATTN_V); GGUF_GET_KEY(fctx, lora->hparams.n_rank_wo, gguf_get_val_u32, GGUF_TYPE_UINT32, true, LLM_KV_TRAINING_LORA_RANK_ATTN_OUT); GGUF_GET_KEY(fctx, lora->hparams.n_rank_ffn_norm, gguf_get_val_u32, GGUF_TYPE_UINT32, true, LLM_KV_TRAINING_LORA_RANK_FFN_NORM); - GGUF_GET_KEY(fctx, lora->hparams.n_rank_w1, gguf_get_val_u32, GGUF_TYPE_UINT32, true, LLM_KV_TRAINING_LORA_RANK_FFN_GATE); - GGUF_GET_KEY(fctx, lora->hparams.n_rank_w2, gguf_get_val_u32, GGUF_TYPE_UINT32, true, LLM_KV_TRAINING_LORA_RANK_FFN_DOWN); - GGUF_GET_KEY(fctx, lora->hparams.n_rank_w3, gguf_get_val_u32, GGUF_TYPE_UINT32, true, LLM_KV_TRAINING_LORA_RANK_FFN_UP); + GGUF_GET_KEY(fctx, lora->hparams.n_rank_ffn_gate, gguf_get_val_u32, GGUF_TYPE_UINT32, true, LLM_KV_TRAINING_LORA_RANK_FFN_GATE); + GGUF_GET_KEY(fctx, lora->hparams.n_rank_ffn_down, gguf_get_val_u32, GGUF_TYPE_UINT32, true, LLM_KV_TRAINING_LORA_RANK_FFN_DOWN); + GGUF_GET_KEY(fctx, lora->hparams.n_rank_ffn_up, gguf_get_val_u32, GGUF_TYPE_UINT32, true, LLM_KV_TRAINING_LORA_RANK_FFN_UP); init_lora(model, lora); @@ -825,12 +825,12 @@ static void load_llama_lora_gguf(struct gguf_context * fctx, struct ggml_context copy_tensor_by_name(layer.wo_b, f_ggml_ctx, ggml_get_name(layer.wo_b)); copy_tensor_by_name(layer.ffn_norm_a, f_ggml_ctx, ggml_get_name(layer.ffn_norm_a)); copy_tensor_by_name(layer.ffn_norm_b, f_ggml_ctx, ggml_get_name(layer.ffn_norm_b)); - copy_tensor_by_name(layer.w1_a, f_ggml_ctx, ggml_get_name(layer.w1_a)); - copy_tensor_by_name(layer.w1_b, f_ggml_ctx, ggml_get_name(layer.w1_b)); - copy_tensor_by_name(layer.w2_a, f_ggml_ctx, ggml_get_name(layer.w2_a)); - copy_tensor_by_name(layer.w2_b, f_ggml_ctx, ggml_get_name(layer.w2_b)); - copy_tensor_by_name(layer.w3_a, f_ggml_ctx, ggml_get_name(layer.w3_a)); - copy_tensor_by_name(layer.w3_b, f_ggml_ctx, ggml_get_name(layer.w3_b)); + copy_tensor_by_name(layer.ffn_gate_a, f_ggml_ctx, ggml_get_name(layer.ffn_gate_a)); + copy_tensor_by_name(layer.ffn_gate_b, f_ggml_ctx, ggml_get_name(layer.ffn_gate_b)); + copy_tensor_by_name(layer.ffn_down_a, f_ggml_ctx, ggml_get_name(layer.ffn_down_a)); + copy_tensor_by_name(layer.ffn_down_b, f_ggml_ctx, ggml_get_name(layer.ffn_down_b)); + copy_tensor_by_name(layer.ffn_up_a, f_ggml_ctx, ggml_get_name(layer.ffn_up_a)); + copy_tensor_by_name(layer.ffn_up_b, f_ggml_ctx, ggml_get_name(layer.ffn_up_b)); } } @@ -868,9 +868,9 @@ static void save_llama_lora_gguf(struct gguf_context * fctx, struct my_llama_mod gguf_set_val_u32(fctx, LLM_KV_TRAINING_LORA_RANK_ATTN_V, lora->hparams.n_rank_wv); gguf_set_val_u32(fctx, LLM_KV_TRAINING_LORA_RANK_ATTN_OUT, lora->hparams.n_rank_wo); gguf_set_val_u32(fctx, LLM_KV_TRAINING_LORA_RANK_FFN_NORM, lora->hparams.n_rank_ffn_norm); - gguf_set_val_u32(fctx, LLM_KV_TRAINING_LORA_RANK_FFN_GATE, lora->hparams.n_rank_w1); - gguf_set_val_u32(fctx, LLM_KV_TRAINING_LORA_RANK_FFN_DOWN, lora->hparams.n_rank_w2); - gguf_set_val_u32(fctx, LLM_KV_TRAINING_LORA_RANK_FFN_UP, lora->hparams.n_rank_w3); + gguf_set_val_u32(fctx, LLM_KV_TRAINING_LORA_RANK_FFN_GATE, lora->hparams.n_rank_ffn_gate); + gguf_set_val_u32(fctx, LLM_KV_TRAINING_LORA_RANK_FFN_DOWN, lora->hparams.n_rank_ffn_down); + gguf_set_val_u32(fctx, LLM_KV_TRAINING_LORA_RANK_FFN_UP, lora->hparams.n_rank_ffn_up); gguf_add_tensor(fctx, lora->tok_embeddings_a); gguf_add_tensor(fctx, lora->tok_embeddings_b); @@ -894,12 +894,12 @@ static void save_llama_lora_gguf(struct gguf_context * fctx, struct my_llama_mod gguf_add_tensor(fctx, layer.wo_b); gguf_add_tensor(fctx, layer.ffn_norm_a); gguf_add_tensor(fctx, layer.ffn_norm_b); - gguf_add_tensor(fctx, layer.w1_a); - gguf_add_tensor(fctx, layer.w1_b); - gguf_add_tensor(fctx, layer.w2_a); - gguf_add_tensor(fctx, layer.w2_b); - gguf_add_tensor(fctx, layer.w3_a); - gguf_add_tensor(fctx, layer.w3_b); + gguf_add_tensor(fctx, layer.ffn_gate_a); + gguf_add_tensor(fctx, layer.ffn_gate_b); + gguf_add_tensor(fctx, layer.ffn_down_a); + gguf_add_tensor(fctx, layer.ffn_down_b); + gguf_add_tensor(fctx, layer.ffn_up_a); + gguf_add_tensor(fctx, layer.ffn_up_b); } } @@ -1104,12 +1104,12 @@ static void save_as_llama_lora(const char * filename, struct my_llama_lora * lor write_tensor(&file, layer.wo_b, tni(LLM_TENSOR_ATTN_OUT, i, ".weight.loraB")); write_tensor(&file, layer.ffn_norm_a, tni(LLM_TENSOR_FFN_NORM, i, ".weight.loraA")); write_tensor(&file, layer.ffn_norm_b, tni(LLM_TENSOR_FFN_NORM, i, ".weight.loraB")); - write_tensor(&file, layer.w1_a, tni(LLM_TENSOR_FFN_GATE, i, ".weight.loraA")); - write_tensor(&file, layer.w1_b, tni(LLM_TENSOR_FFN_GATE, i, ".weight.loraB")); - write_tensor(&file, layer.w2_a, tni(LLM_TENSOR_FFN_DOWN, i, ".weight.loraA")); - write_tensor(&file, layer.w2_b, tni(LLM_TENSOR_FFN_DOWN, i, ".weight.loraB")); - write_tensor(&file, layer.w3_a, tni(LLM_TENSOR_FFN_UP, i, ".weight.loraA")); - write_tensor(&file, layer.w3_b, tni(LLM_TENSOR_FFN_UP, i, ".weight.loraB")); + write_tensor(&file, layer.ffn_gate_a, tni(LLM_TENSOR_FFN_GATE, i, ".weight.loraA")); + write_tensor(&file, layer.ffn_gate_b, tni(LLM_TENSOR_FFN_GATE, i, ".weight.loraB")); + write_tensor(&file, layer.ffn_down_a, tni(LLM_TENSOR_FFN_DOWN, i, ".weight.loraA")); + write_tensor(&file, layer.ffn_down_b, tni(LLM_TENSOR_FFN_DOWN, i, ".weight.loraB")); + write_tensor(&file, layer.ffn_up_a, tni(LLM_TENSOR_FFN_UP, i, ".weight.loraA")); + write_tensor(&file, layer.ffn_up_b, tni(LLM_TENSOR_FFN_UP, i, ".weight.loraB")); } } @@ -1139,9 +1139,9 @@ struct train_params { uint32_t n_rank_wv; uint32_t n_rank_wo; uint32_t n_rank_ffn_norm; - uint32_t n_rank_w1; - uint32_t n_rank_w2; - uint32_t n_rank_w3; + uint32_t n_rank_ffn_gate; + uint32_t n_rank_ffn_down; + uint32_t n_rank_ffn_up; uint32_t n_rank_tok_embeddings; uint32_t n_rank_norm; uint32_t n_rank_output; @@ -1152,9 +1152,9 @@ struct train_params { bool custom_n_rank_wv; bool custom_n_rank_wo; bool custom_n_rank_ffn_norm; - bool custom_n_rank_w1; - bool custom_n_rank_w2; - bool custom_n_rank_w3; + bool custom_n_rank_ffn_gate; + bool custom_n_rank_ffn_down; + bool custom_n_rank_ffn_up; bool custom_n_rank_tok_embeddings; bool custom_n_rank_norm; bool custom_n_rank_output; @@ -1186,9 +1186,9 @@ static struct train_params get_default_train_params() { params.n_rank_wv = 4; params.n_rank_wo = 4; params.n_rank_ffn_norm = 1; - params.n_rank_w1 = 4; - params.n_rank_w2 = 4; - params.n_rank_w3 = 4; + params.n_rank_ffn_gate = 4; + params.n_rank_ffn_down = 4; + params.n_rank_ffn_up = 4; params.n_rank_tok_embeddings = 4; params.n_rank_norm = 1; params.n_rank_output = 4; @@ -1199,9 +1199,9 @@ static struct train_params get_default_train_params() { params.custom_n_rank_wv = false; params.custom_n_rank_wo = false; params.custom_n_rank_ffn_norm = false; - params.custom_n_rank_w1 = false; - params.custom_n_rank_w2 = false; - params.custom_n_rank_w3 = false; + params.custom_n_rank_ffn_gate = false; + params.custom_n_rank_ffn_down = false; + params.custom_n_rank_ffn_up = false; params.custom_n_rank_tok_embeddings = false; params.custom_n_rank_norm = false; params.custom_n_rank_output = false; @@ -1232,9 +1232,9 @@ static void train_print_usage(int argc, char ** argv, const struct train_params fprintf(stderr, " --rank-wk N LORA rank for wk tensor, overrides default rank.\n"); fprintf(stderr, " --rank-wv N LORA rank for wv tensor, overrides default rank.\n"); fprintf(stderr, " --rank-wo N LORA rank for wo tensor, overrides default rank.\n"); - fprintf(stderr, " --rank-w1 N LORA rank for w1 tensor, overrides default rank.\n"); - fprintf(stderr, " --rank-w2 N LORA rank for w2 tensor, overrides default rank.\n"); - fprintf(stderr, " --rank-w3 N LORA rank for w3 tensor, overrides default rank.\n"); + fprintf(stderr, " --rank-ffn_gate N LORA rank for ffn_gate tensor, overrides default rank.\n"); + fprintf(stderr, " --rank-ffn_down N LORA rank for ffn_down tensor, overrides default rank.\n"); + fprintf(stderr, " --rank-ffn_up N LORA rank for ffn_up tensor, overrides default rank.\n"); print_common_train_usage(argc, argv, ¶ms->common); } @@ -1369,27 +1369,27 @@ static bool train_params_parse(int argc, char ** argv, struct train_params * par } params->n_rank_wo = std::stoi(argv[i]); params->custom_n_rank_wo = true; - } else if (arg == "--rank-w1") { + } else if (arg == "--rank-ffn_gate") { if (++i >= argc) { invalid_param = true; break; } - params->n_rank_w1 = std::stoi(argv[i]); - params->custom_n_rank_w1 = true; - } else if (arg == "--rank-w2") { + params->n_rank_ffn_gate = std::stoi(argv[i]); + params->custom_n_rank_ffn_gate = true; + } else if (arg == "--rank-ffn_down") { if (++i >= argc) { invalid_param = true; break; } - params->n_rank_w2 = std::stoi(argv[i]); - params->custom_n_rank_w2 = true; - } else if (arg == "--rank-w3") { + params->n_rank_ffn_down = std::stoi(argv[i]); + params->custom_n_rank_ffn_down = true; + } else if (arg == "--rank-ffn_up") { if (++i >= argc) { invalid_param = true; break; } - params->n_rank_w3 = std::stoi(argv[i]); - params->custom_n_rank_w3 = true; + params->n_rank_ffn_up = std::stoi(argv[i]); + params->custom_n_rank_ffn_up = true; } else { fprintf(stderr, "error: unknown argument: %s\n", arg.c_str()); train_print_usage(argc, argv, &default_params); @@ -1452,12 +1452,12 @@ static int64_t get_parameter_count(struct my_llama_lora* lora) { nx += ggml_nelements(layer.wo_b); nx += ggml_nelements(layer.ffn_norm_a); nx += ggml_nelements(layer.ffn_norm_b); - nx += ggml_nelements(layer.w1_a); - nx += ggml_nelements(layer.w1_b); - nx += ggml_nelements(layer.w2_a); - nx += ggml_nelements(layer.w2_b); - nx += ggml_nelements(layer.w3_a); - nx += ggml_nelements(layer.w3_b); + nx += ggml_nelements(layer.ffn_gate_a); + nx += ggml_nelements(layer.ffn_gate_b); + nx += ggml_nelements(layer.ffn_down_a); + nx += ggml_nelements(layer.ffn_down_b); + nx += ggml_nelements(layer.ffn_up_a); + nx += ggml_nelements(layer.ffn_up_b); } return nx; } @@ -1511,9 +1511,9 @@ int main(int argc, char ** argv) { uint32_t n_rank_wv = params.custom_n_rank_wv ? params.n_rank_wv : params.lora_r; uint32_t n_rank_wo = params.custom_n_rank_wo ? params.n_rank_wo : params.lora_r; uint32_t n_rank_ffn_norm = params.custom_n_rank_ffn_norm ? params.n_rank_ffn_norm : 1; - uint32_t n_rank_w1 = params.custom_n_rank_w1 ? params.n_rank_w1 : params.lora_r; - uint32_t n_rank_w2 = params.custom_n_rank_w2 ? params.n_rank_w2 : params.lora_r; - uint32_t n_rank_w3 = params.custom_n_rank_w3 ? params.n_rank_w3 : params.lora_r; + uint32_t n_rank_ffn_gate = params.custom_n_rank_ffn_gate ? params.n_rank_ffn_gate : params.lora_r; + uint32_t n_rank_ffn_down = params.custom_n_rank_ffn_down ? params.n_rank_ffn_down : params.lora_r; + uint32_t n_rank_ffn_up = params.custom_n_rank_ffn_up ? params.n_rank_ffn_up : params.lora_r; uint32_t n_rank_tok_embeddings = params.custom_n_rank_tok_embeddings ? params.n_rank_tok_embeddings : params.lora_r; uint32_t n_rank_norm = params.custom_n_rank_norm ? params.n_rank_norm : 1; uint32_t n_rank_output = params.custom_n_rank_output ? params.n_rank_output : params.lora_r; @@ -1523,9 +1523,9 @@ int main(int argc, char ** argv) { lora.hparams.n_rank_wv = n_rank_wv; lora.hparams.n_rank_wo = n_rank_wo; lora.hparams.n_rank_ffn_norm = n_rank_ffn_norm; - lora.hparams.n_rank_w1 = n_rank_w1; - lora.hparams.n_rank_w2 = n_rank_w2; - lora.hparams.n_rank_w3 = n_rank_w3; + lora.hparams.n_rank_ffn_gate = n_rank_ffn_gate; + lora.hparams.n_rank_ffn_down = n_rank_ffn_down; + lora.hparams.n_rank_ffn_up = n_rank_ffn_up; lora.hparams.n_rank_tok_embeddings = n_rank_tok_embeddings; lora.hparams.n_rank_norm = n_rank_norm; lora.hparams.n_rank_output = n_rank_output; @@ -1566,9 +1566,9 @@ int main(int argc, char ** argv) { || (lora.hparams.n_rank_wv != n_rank_wv) || (lora.hparams.n_rank_wo != n_rank_wo) || (lora.hparams.n_rank_ffn_norm != n_rank_ffn_norm) - || (lora.hparams.n_rank_w1 != n_rank_w1) - || (lora.hparams.n_rank_w2 != n_rank_w2) - || (lora.hparams.n_rank_w3 != n_rank_w3) + || (lora.hparams.n_rank_ffn_gate != n_rank_ffn_gate) + || (lora.hparams.n_rank_ffn_down != n_rank_ffn_down) + || (lora.hparams.n_rank_ffn_up != n_rank_ffn_up) || (lora.hparams.n_rank_tok_embeddings != n_rank_tok_embeddings) || (lora.hparams.n_rank_norm != n_rank_norm) || (lora.hparams.n_rank_output != n_rank_output) diff --git a/examples/train-text-from-scratch/train-text-from-scratch.cpp b/examples/train-text-from-scratch/train-text-from-scratch.cpp index 2e2a8ce08..bfdf124d7 100644 --- a/examples/train-text-from-scratch/train-text-from-scratch.cpp +++ b/examples/train-text-from-scratch/train-text-from-scratch.cpp @@ -50,9 +50,9 @@ struct my_llama_layer { struct ggml_tensor * ffn_norm; // ff - struct ggml_tensor * w1; - struct ggml_tensor * w2; - struct ggml_tensor * w3; + struct ggml_tensor * ffn_gate; // w1 + struct ggml_tensor * ffn_down; // w2 + struct ggml_tensor * ffn_up; // w3 }; struct my_llama_model { @@ -140,9 +140,9 @@ static void set_param_model(struct my_llama_model * model) { ggml_set_param(ctx, layer.wv); ggml_set_param(ctx, layer.wo); ggml_set_param(ctx, layer.ffn_norm); - ggml_set_param(ctx, layer.w1); - ggml_set_param(ctx, layer.w2); - ggml_set_param(ctx, layer.w3); + ggml_set_param(ctx, layer.ffn_gate); + ggml_set_param(ctx, layer.ffn_down); + ggml_set_param(ctx, layer.ffn_up); } } @@ -198,9 +198,9 @@ static void init_model(struct my_llama_model * model) { layer.ffn_norm = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd); - layer.w1 = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, n_embd, n_ff); - layer.w2 = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, n_ff, n_embd); - layer.w3 = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, n_embd, n_ff); + layer.ffn_gate = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, n_embd, n_ff); + layer.ffn_down = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, n_ff, n_embd); + layer.ffn_up = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, n_embd, n_ff); ggml_set_name(layer.attention_norm, tni(LLM_TENSOR_ATTN_NORM, i)); @@ -211,9 +211,9 @@ static void init_model(struct my_llama_model * model) { ggml_set_name(layer.ffn_norm, tni(LLM_TENSOR_FFN_NORM, i)); - ggml_set_name(layer.w1, tni(LLM_TENSOR_FFN_GATE, i)); - ggml_set_name(layer.w2, tni(LLM_TENSOR_FFN_DOWN, i)); - ggml_set_name(layer.w3, tni(LLM_TENSOR_FFN_UP, i)); + ggml_set_name(layer.ffn_gate, tni(LLM_TENSOR_FFN_GATE, i)); + ggml_set_name(layer.ffn_down, tni(LLM_TENSOR_FFN_DOWN, i)); + ggml_set_name(layer.ffn_up, tni(LLM_TENSOR_FFN_UP, i)); } set_param_model(model); @@ -244,9 +244,9 @@ static void randomize_model(struct my_llama_model * model, int seed, float mean, randomize_tensor_normal(layer.ffn_norm, rnd); - randomize_tensor_normal(layer.w1, rnd); - randomize_tensor_normal(layer.w2, rnd); - randomize_tensor_normal(layer.w3, rnd); + randomize_tensor_normal(layer.ffn_gate, rnd); + randomize_tensor_normal(layer.ffn_down, rnd); + randomize_tensor_normal(layer.ffn_up, rnd); } free_random_normal_distribution(rnd); @@ -356,11 +356,11 @@ static struct ggml_tensor * llama_build_train_graphs( struct ggml_tensor * t22 = ggml_rms_norm (ctx, t21, f_norm_rms_eps); set_name(t22, "t22"); assert_shape_2d(t22, n_embd, N*n_batch); struct ggml_tensor * t23 = ggml_repeat (ctx, layer.ffn_norm, t22); set_name(t23, "t23"); assert_shape_2d(t23, n_embd, N*n_batch); struct ggml_tensor * t24 = ggml_mul (ctx, t23, t22); set_name(t24, "t24"); assert_shape_2d(t24, n_embd, N*n_batch); - struct ggml_tensor * t25 = ggml_mul_mat (ctx, layer.w3, t24); set_name(t25, "t25"); assert_shape_2d(t25, n_ff, N*n_batch); - struct ggml_tensor * t26 = ggml_mul_mat (ctx, layer.w1, t24); set_name(t26, "t26"); assert_shape_2d(t26, n_ff, N*n_batch); + struct ggml_tensor * t25 = ggml_mul_mat (ctx, layer.ffn_up, t24); set_name(t25, "t25"); assert_shape_2d(t25, n_ff, N*n_batch); + struct ggml_tensor * t26 = ggml_mul_mat (ctx, layer.ffn_gate, t24); set_name(t26, "t26"); assert_shape_2d(t26, n_ff, N*n_batch); struct ggml_tensor * t27 = ggml_silu (ctx, t26); set_name(t27, "t27"); assert_shape_2d(t27, n_ff, N*n_batch); struct ggml_tensor * t28 = ggml_mul (ctx, t27, t25); set_name(t28, "t28"); assert_shape_2d(t28, n_ff, N*n_batch); - struct ggml_tensor * t29 = ggml_mul_mat (ctx, layer.w2, t28); set_name(t29, "t29"); assert_shape_2d(t29, n_embd, N*n_batch); + struct ggml_tensor * t29 = ggml_mul_mat (ctx, layer.ffn_down, t28); set_name(t29, "t29"); assert_shape_2d(t29, n_embd, N*n_batch); struct ggml_tensor * t30 = ggml_add (ctx, t29, t21); set_name(t30, "t30"); assert_shape_2d(t30, n_embd, N*n_batch); cur = t30; checkpoints.push_back(cur); @@ -521,9 +521,9 @@ static void load_llama_model_gguf(struct gguf_context * fctx, struct ggml_contex copy_tensor_by_name(layer.wv, f_ggml_ctx, tni(LLM_TENSOR_ATTN_V, i)); copy_tensor_by_name(layer.wo, f_ggml_ctx, tni(LLM_TENSOR_ATTN_OUT, i)); copy_tensor_by_name(layer.ffn_norm, f_ggml_ctx, tni(LLM_TENSOR_FFN_NORM, i)); - copy_tensor_by_name(layer.w1, f_ggml_ctx, tni(LLM_TENSOR_FFN_GATE, i)); - copy_tensor_by_name(layer.w2, f_ggml_ctx, tni(LLM_TENSOR_FFN_DOWN, i)); - copy_tensor_by_name(layer.w3, f_ggml_ctx, tni(LLM_TENSOR_FFN_UP, i)); + copy_tensor_by_name(layer.ffn_gate, f_ggml_ctx, tni(LLM_TENSOR_FFN_GATE, i)); + copy_tensor_by_name(layer.ffn_down, f_ggml_ctx, tni(LLM_TENSOR_FFN_DOWN, i)); + copy_tensor_by_name(layer.ffn_up, f_ggml_ctx, tni(LLM_TENSOR_FFN_UP, i)); } } @@ -664,9 +664,9 @@ static void save_llama_model_gguf(struct gguf_context * fctx, const char * fn_vo gguf_add_tensor(fctx, layer.wv); gguf_add_tensor(fctx, layer.wo); gguf_add_tensor(fctx, layer.ffn_norm); - gguf_add_tensor(fctx, layer.w1); - gguf_add_tensor(fctx, layer.w2); - gguf_add_tensor(fctx, layer.w3); + gguf_add_tensor(fctx, layer.ffn_gate); + gguf_add_tensor(fctx, layer.ffn_down); + gguf_add_tensor(fctx, layer.ffn_up); } } @@ -915,9 +915,9 @@ static int64_t get_parameter_count(struct my_llama_model* model) { nx += ggml_nelements(layer.wv); nx += ggml_nelements(layer.wo); nx += ggml_nelements(layer.ffn_norm); - nx += ggml_nelements(layer.w1); - nx += ggml_nelements(layer.w2); - nx += ggml_nelements(layer.w3); + nx += ggml_nelements(layer.ffn_gate); + nx += ggml_nelements(layer.ffn_down); + nx += ggml_nelements(layer.ffn_up); } return nx; } From 037259be689353081e7bae3c1ab4ab18e7fbe8c9 Mon Sep 17 00:00:00 2001 From: Aarni Koskela Date: Tue, 13 Feb 2024 15:24:50 +0200 Subject: [PATCH 338/812] llama : make load error reporting more granular (#5477) Makes it easier to pinpoint where e.g. `unordered_map::at: key not found` comes from. --- llama.cpp | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/llama.cpp b/llama.cpp index 381a03068..61c695187 100644 --- a/llama.cpp +++ b/llama.cpp @@ -4384,9 +4384,21 @@ static int llama_model_load(const std::string & fname, llama_model & model, llam model.hparams.vocab_only = params.vocab_only; - llm_load_arch (ml, model); - llm_load_hparams(ml, model); - llm_load_vocab (ml, model); + try { + llm_load_arch(ml, model); + } catch(const std::exception & e) { + throw std::runtime_error("error loading model architecture: " + std::string(e.what())); + } + try { + llm_load_hparams(ml, model); + } catch(const std::exception & e) { + throw std::runtime_error("error loading model hyperparameters: " + std::string(e.what())); + } + try { + llm_load_vocab(ml, model); + } catch(const std::exception & e) { + throw std::runtime_error("error loading model vocabulary: " + std::string(e.what())); + } llm_load_print_meta(ml, model); From c4e6dd59e45ef7b14f7763fb073b517395dc176c Mon Sep 17 00:00:00 2001 From: Aarni Koskela Date: Tue, 13 Feb 2024 18:18:16 +0200 Subject: [PATCH 339/812] llama : allow raw byte in SPM vocabs; don't crash on nl 404 (#5478) * common : don't crash if newline token is not found * common : llama_byte_to_token: allow falling back to finding just the token byte in SPM vocabs --- llama.cpp | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/llama.cpp b/llama.cpp index 61c695187..8ebbf7628 100644 --- a/llama.cpp +++ b/llama.cpp @@ -3314,7 +3314,12 @@ static void llm_load_vocab( // determine the newline token: LLaMA "<0x0A>" == 10 == '\n', Falcon 193 == '\n' if (vocab.type == LLAMA_VOCAB_TYPE_SPM) { - vocab.linefeed_id = llama_byte_to_token(vocab, '\n'); + try { + vocab.linefeed_id = llama_byte_to_token(vocab, '\n'); + } catch (const std::exception & e) { + LLAMA_LOG_WARN("%s: SPM vocabulary, but newline token not found: %s! Using special_pad_id instead.", __func__, e.what()); + vocab.linefeed_id = vocab.special_pad_id; + } } else if (vocab.type == LLAMA_VOCAB_TYPE_WPM) { vocab.linefeed_id = vocab.special_pad_id; } else { @@ -7746,7 +7751,13 @@ static llama_token llama_byte_to_token(const llama_vocab & vocab, uint8_t ch) { switch (llama_vocab_get_type(vocab)) { case LLAMA_VOCAB_TYPE_SPM: { const char buf[7] = { '<', '0', 'x', hex[ch >> 4], hex[ch & 15], '>', 0 }; - return vocab.token_to_id.at(buf); + auto token = vocab.token_to_id.find(buf); + if (token != vocab.token_to_id.end()) { + return (*token).second; + } + // Try to fall back to just the byte as a string + const char buf2[2] = { (char)ch, 0 }; + return vocab.token_to_id.at(buf2); } case LLAMA_VOCAB_TYPE_WPM: case LLAMA_VOCAB_TYPE_BPE: { From ea9c8e11436ad50719987fa23a289c74b7b40d40 Mon Sep 17 00:00:00 2001 From: Jared Van Bortel Date: Tue, 13 Feb 2024 12:03:53 -0500 Subject: [PATCH 340/812] llama : add support for Nomic Embed (#5468) --- convert-hf-to-gguf.py | 117 ++++++++++++------- gguf-py/gguf/constants.py | 56 +++++---- gguf-py/gguf/tensor_mapping.py | 12 +- llama.cpp | 201 ++++++++++++++++++++++++--------- 4 files changed, 273 insertions(+), 113 deletions(-) diff --git a/convert-hf-to-gguf.py b/convert-hf-to-gguf.py index 5adfdc143..ae471481d 100755 --- a/convert-hf-to-gguf.py +++ b/convert-hf-to-gguf.py @@ -10,7 +10,7 @@ import re import sys from enum import IntEnum from pathlib import Path -from typing import TYPE_CHECKING, Any, ContextManager, Iterator, cast +from typing import TYPE_CHECKING, Any, ContextManager, Iterator, Sequence, cast import numpy as np import torch @@ -25,15 +25,6 @@ import gguf from convert import HfVocab -# check for any of the given keys in the dictionary and return the value of the first key found -def get_key_opts(d, keys): - for k in keys: - if k in d: - return d[k] - print(f"Could not find any of {keys}") - sys.exit() - - ###### MODEL DEFINITIONS ###### class SentencePieceTokenTypes(IntEnum): @@ -58,6 +49,15 @@ class Model: self.hparams = Model.load_hparams(self.dir_model) self.model_arch = self._get_model_architecture() self.gguf_writer = gguf.GGUFWriter(fname_out, gguf.MODEL_ARCH_NAMES[self.model_arch], endianess=self.endianess, use_temp_file=False) + self.block_count = self.find_hparam(["n_layers", "num_hidden_layers", "n_layer"]) + + def find_hparam(self, keys: Sequence[str], optional: bool = False) -> Any: + key = next((k for k in keys if k in self.hparams), None) + if key is not None: + return self.hparams[key] + if optional: + return None + raise KeyError(f"could not find any of: {keys}") def set_vocab(self): self._set_vocab_gpt2() @@ -79,28 +79,33 @@ class Model: def set_gguf_parameters(self): self.gguf_writer.add_name(self.dir_model.name) - self.gguf_writer.add_block_count(self.hparams.get( - "n_layers", self.hparams.get("num_hidden_layers", self.hparams.get("n_layer")), - )) - if (n_ctx := self.hparams.get("max_position_embeddings")) is not None: + self.gguf_writer.add_block_count(self.block_count) + + if (n_ctx := self.find_hparam(["max_position_embeddings", "n_ctx"], optional=True)) is not None: self.gguf_writer.add_context_length(n_ctx) - if (n_embd := self.hparams.get("hidden_size")) is not None: - self.gguf_writer.add_embedding_length(n_embd) - if (n_ff := self.hparams.get("intermediate_size")) is not None: + + n_embd = self.find_hparam(["hidden_size", "n_embd"]) + self.gguf_writer.add_embedding_length(n_embd) + + if (n_ff := self.find_hparam(["intermediate_size", "n_inner"], optional=True)) is not None: self.gguf_writer.add_feed_forward_length(n_ff) - if (n_head := self.hparams.get("num_attention_heads")) is not None: - self.gguf_writer.add_head_count(n_head) + + n_head = self.find_hparam(["num_attention_heads", "n_head"]) + self.gguf_writer.add_head_count(n_head) + if (n_head_kv := self.hparams.get("num_key_value_heads")) is not None: self.gguf_writer.add_head_count_kv(n_head_kv) - if (n_rms_eps := self.hparams.get("rms_norm_eps")) is not None: - self.gguf_writer.add_layer_norm_rms_eps(n_rms_eps) + if (f_rms_eps := self.hparams.get("rms_norm_eps")) is not None: + self.gguf_writer.add_layer_norm_rms_eps(f_rms_eps) + if (f_norm_eps := self.find_hparam(["layer_norm_eps", "layer_norm_epsilon"], optional=True)) is not None: + self.gguf_writer.add_layer_norm_eps(f_norm_eps) if (n_experts := self.hparams.get("num_local_experts")) is not None: self.gguf_writer.add_expert_count(n_experts) if (n_experts_used := self.hparams.get("num_experts_per_tok")) is not None: self.gguf_writer.add_expert_used_count(n_experts_used) - self.gguf_writer.add_parallel_residual(self.hparams.get("use_parallel_residual", True)) + self.gguf_writer.add_file_type(self.ftype) def write_tensors(self): block_count = self.hparams.get("n_layers", self.hparams.get("num_hidden_layers", self.hparams.get("n_layer"))) @@ -211,6 +216,8 @@ class Model: return MiniCPMModel if model_architecture == "BertModel": return BertModel + if model_architecture == "NomicBertModel": + return NomicBertModel return Model def _is_model_safetensors(self) -> bool: @@ -268,6 +275,8 @@ class Model: return gguf.MODEL_ARCH.MINICPM if arch == "BertModel": return gguf.MODEL_ARCH.BERT + if arch == "NomicBertModel": + return gguf.MODEL_ARCH.NOMIC_BERT raise NotImplementedError(f'Architecture "{arch}" not supported!') @@ -1297,21 +1306,21 @@ class GPT2Model(Model): class Phi2Model(Model): def set_gguf_parameters(self): - block_count = get_key_opts(self.hparams, ["num_hidden_layers", "n_layer"]) + block_count = self.find_hparam(["num_hidden_layers", "n_layer"]) - rot_pct = get_key_opts(self.hparams, ["partial_rotary_factor"]) - n_embd = get_key_opts(self.hparams, ["hidden_size", "n_embd"]) - n_head = get_key_opts(self.hparams, ["num_attention_heads", "n_head"]) + rot_pct = self.find_hparam(["partial_rotary_factor"]) + n_embd = self.find_hparam(["hidden_size", "n_embd"]) + n_head = self.find_hparam(["num_attention_heads", "n_head"]) self.gguf_writer.add_name("Phi2") - self.gguf_writer.add_context_length(get_key_opts(self.hparams, ["n_positions", "max_position_embeddings"])) + self.gguf_writer.add_context_length(self.find_hparam(["n_positions", "max_position_embeddings"])) self.gguf_writer.add_embedding_length(n_embd) self.gguf_writer.add_feed_forward_length(4 * n_embd) self.gguf_writer.add_block_count(block_count) self.gguf_writer.add_head_count(n_head) self.gguf_writer.add_head_count_kv(n_head) - self.gguf_writer.add_layer_norm_eps(get_key_opts(self.hparams, ["layer_norm_epsilon", "layer_norm_eps"])) + self.gguf_writer.add_layer_norm_eps(self.find_hparam(["layer_norm_epsilon", "layer_norm_eps"])) self.gguf_writer.add_rope_dimension_count(int(rot_pct * n_embd) // n_head) self.gguf_writer.add_file_type(self.ftype) self.gguf_writer.add_add_bos_token(False) @@ -1636,20 +1645,12 @@ in chat mode so that the conversation can end normally.") class BertModel(Model): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - self.block_count = self.hparams["num_hidden_layers"] + self.vocab_size = None def set_gguf_parameters(self): - # TODO(cebtenzzre): merge with parent class - self.gguf_writer.add_name(self.dir_model.name) - self.gguf_writer.add_context_length(self.hparams["max_position_embeddings"]) - self.gguf_writer.add_embedding_length(self.hparams["hidden_size"]) - self.gguf_writer.add_feed_forward_length(self.hparams["intermediate_size"]) - self.gguf_writer.add_block_count(self.block_count) - self.gguf_writer.add_head_count(self.hparams["num_attention_heads"]) - self.gguf_writer.add_layer_norm_eps(self.hparams["layer_norm_eps"]) + super().set_gguf_parameters() self.gguf_writer.add_causal_attention(False) self.gguf_writer.add_pooling_layer(True) - self.gguf_writer.add_file_type(self.ftype) def set_vocab(self): path = self.dir_model @@ -1659,6 +1660,7 @@ class BertModel(Model): vocab = HfVocab(path, added_tokens_path) tokens, scores, toktypes = zip(*vocab.all_tokens()) assert len(tokens) == vocab.vocab_size + self.vocab_size = vocab.vocab_size # we need this to validate the size of the token_type embeddings # though currently we are passing all zeros to the token_type embeddings @@ -1672,7 +1674,7 @@ class BertModel(Model): if tok.startswith(b"##"): return tok[2:] return b"\xe2\x96\x81" + tok - tokens = [phantom(t, y) for t, y in zip(tokens, toktypes)] + tokens = tuple(phantom(t, y) for t, y in zip(tokens, toktypes)) # set up bos and eos tokens (cls and sep) self.gguf_writer.add_bos_token_id(vocab.tokenizer.cls_token_id) @@ -1724,6 +1726,43 @@ class BertModel(Model): self.gguf_writer.add_tensor(new_name, data) +class NomicBertModel(BertModel): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + # the HF config claims n_ctx=8192, but it uses RoPE scaling + self.hparams["n_ctx"] = 2048 + + # SwigLU activation + assert self.hparams["activation_function"] == "swiglu" + # this doesn't do anything in the HF version + assert self.hparams["causal"] is False + # no bias tensors + assert self.hparams["qkv_proj_bias"] is False + assert self.hparams["mlp_fc1_bias"] is False + assert self.hparams["mlp_fc2_bias"] is False + # norm at end of layer + assert self.hparams["prenorm"] is False + # standard RoPE + assert self.hparams["rotary_emb_fraction"] == 1.0 + assert self.hparams["rotary_emb_interleaved"] is False + assert self.hparams["rotary_emb_scale_base"] is None + + def set_gguf_parameters(self): + super().set_gguf_parameters() + self.gguf_writer.add_rope_freq_base(self.hparams["rotary_emb_base"]) + + def get_tensors(self): + assert self.vocab_size is not None + for name, data in super().get_tensors(): + # Nomic Embed's token embeddings tensor is padded, but llama.cpp wants tensor sizes to match exactly. + if name == 'embeddings.word_embeddings.weight' and data.shape[1] != self.vocab_size: + rounded_vocab_size = (self.vocab_size + 63) // 64 * 64 + assert data.shape == (rounded_vocab_size, self.hparams["n_embd"]) + data = data[:self.vocab_size, :] + yield name, data + + ###### CONVERSION LOGIC ###### diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py index 644e1589c..5fba01714 100644 --- a/gguf-py/gguf/constants.py +++ b/gguf-py/gguf/constants.py @@ -87,27 +87,28 @@ class Keys: class MODEL_ARCH(IntEnum): - LLAMA = auto() - FALCON = auto() - BAICHUAN = auto() - GPT2 = auto() - GPTJ = auto() - GPTNEOX = auto() - MPT = auto() - STARCODER = auto() - PERSIMMON = auto() - REFACT = auto() - BERT = auto() - BLOOM = auto() - STABLELM = auto() - QWEN = auto() - QWEN2 = auto() - PHI2 = auto() - PLAMO = auto() - CODESHELL = auto() - ORION = auto() + LLAMA = auto() + FALCON = auto() + BAICHUAN = auto() + GPT2 = auto() + GPTJ = auto() + GPTNEOX = auto() + MPT = auto() + STARCODER = auto() + PERSIMMON = auto() + REFACT = auto() + BERT = auto() + NOMIC_BERT = auto() + BLOOM = auto() + STABLELM = auto() + QWEN = auto() + QWEN2 = auto() + PHI2 = auto() + PLAMO = auto() + CODESHELL = auto() + ORION = auto() INTERNLM2 = auto() - MINICPM = auto() + MINICPM = auto() class MODEL_TENSOR(IntEnum): @@ -153,6 +154,7 @@ MODEL_ARCH_NAMES: dict[MODEL_ARCH, str] = { MODEL_ARCH.PERSIMMON: "persimmon", MODEL_ARCH.REFACT: "refact", MODEL_ARCH.BERT: "bert", + MODEL_ARCH.NOMIC_BERT: "nomic-bert", MODEL_ARCH.BLOOM: "bloom", MODEL_ARCH.STABLELM: "stablelm", MODEL_ARCH.QWEN: "qwen", @@ -282,6 +284,20 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = { MODEL_TENSOR.FFN_UP, MODEL_TENSOR.LAYER_OUT_NORM, ], + MODEL_ARCH.NOMIC_BERT: [ + MODEL_TENSOR.TOKEN_EMBD, + MODEL_TENSOR.TOKEN_EMBD_NORM, + MODEL_TENSOR.TOKEN_TYPES, + MODEL_TENSOR.POS_EMBD, + MODEL_TENSOR.OUTPUT_NORM, + MODEL_TENSOR.ATTN_OUT_NORM, + MODEL_TENSOR.ATTN_QKV, + MODEL_TENSOR.ATTN_OUT, + MODEL_TENSOR.FFN_GATE, + MODEL_TENSOR.FFN_DOWN, + MODEL_TENSOR.FFN_UP, + MODEL_TENSOR.LAYER_OUT_NORM, + ], MODEL_ARCH.MPT: [ MODEL_TENSOR.TOKEN_EMBD, MODEL_TENSOR.OUTPUT_NORM, diff --git a/gguf-py/gguf/tensor_mapping.py b/gguf-py/gguf/tensor_mapping.py index c7ba1420e..861003776 100644 --- a/gguf-py/gguf/tensor_mapping.py +++ b/gguf-py/gguf/tensor_mapping.py @@ -15,7 +15,7 @@ class TensorNameMap: "word_embeddings", # bloom "model.embed_tokens", # llama-hf "tok_embeddings", # llama-pth - "embeddings.word_embeddings", # bert + "embeddings.word_embeddings", # bert nomic-bert "language_model.embedding.word_embeddings", # persimmon "wte", # gpt2 "transformer.embd.wte", # phi2 @@ -24,13 +24,14 @@ class TensorNameMap: # Token type embeddings MODEL_TENSOR.TOKEN_TYPES: ( - "embeddings.token_type_embeddings", # bert + "embeddings.token_type_embeddings", # bert nomic-bert ), # Normalization of token embeddings MODEL_TENSOR.TOKEN_EMBD_NORM: ( "word_embeddings_layernorm", # bloom "embeddings.LayerNorm", # bert + "emb_ln", # nomic-bert ), # Position embeddings @@ -103,6 +104,7 @@ class TensorNameMap: "model.layers.{bid}.self_attn.query_key_value", # persimmon "h.{bid}.attn.c_attn", # gpt2 "transformer.h.{bid}.mixer.Wqkv", # phi2 + "encoder.layers.{bid}.attn.Wqkv", # nomic-bert ), # Attention query @@ -152,11 +154,13 @@ class TensorNameMap: "transformer.h.{bid}.mixer.out_proj", # phi2 "model.layers.layers.{bid}.self_attn.o_proj", # plamo "model.layers.{bid}.attention.wo", # internlm2 + "encoder.layers.{bid}.attn.out_proj", # nomic-bert ), # Attention output norm MODEL_TENSOR.ATTN_OUT_NORM: ( "encoder.layer.{bid}.attention.output.LayerNorm", # bert + "encoder.layers.{bid}.norm1", # nomic-bert ), # Rotary embeddings @@ -205,6 +209,7 @@ class TensorNameMap: "model.layers.{bid}.mlp.fc1", # phi2 "model.layers.layers.{bid}.mlp.up_proj", # plamo "model.layers.{bid}.feed_forward.w3", # internlm2 + "encoder.layers.{bid}.mlp.fc11", # nomic-bert ), MODEL_TENSOR.FFN_UP_EXP: ( @@ -224,6 +229,7 @@ class TensorNameMap: "transformer.h.{bid}.mlp.w2", # qwen "model.layers.layers.{bid}.mlp.gate_proj", # plamo "model.layers.{bid}.feed_forward.w1", # internlm2 + "encoder.layers.{bid}.mlp.fc12", # nomic-bert ), MODEL_TENSOR.FFN_GATE_EXP: ( @@ -249,6 +255,7 @@ class TensorNameMap: "model.layers.{bid}.mlp.fc2", # phi2 "model.layers.layers.{bid}.mlp.down_proj", # plamo "model.layers.{bid}.feed_forward.w2", # internlm2 + "encoder.layers.{bid}.mlp.fc2", # nomic-bert ), MODEL_TENSOR.FFN_DOWN_EXP: ( @@ -272,6 +279,7 @@ class TensorNameMap: MODEL_TENSOR.LAYER_OUT_NORM: ( "encoder.layer.{bid}.output.LayerNorm", # bert + "encoder.layers.{bid}.norm2", # nomic-bert ) } diff --git a/llama.cpp b/llama.cpp index 8ebbf7628..14e8821cd 100644 --- a/llama.cpp +++ b/llama.cpp @@ -197,6 +197,7 @@ enum llm_arch { LLM_ARCH_PERSIMMON, LLM_ARCH_REFACT, LLM_ARCH_BERT, + LLM_ARCH_NOMIC_BERT, LLM_ARCH_BLOOM, LLM_ARCH_STABLELM, LLM_ARCH_QWEN, @@ -211,27 +212,28 @@ enum llm_arch { }; static std::map LLM_ARCH_NAMES = { - { LLM_ARCH_LLAMA, "llama" }, - { LLM_ARCH_FALCON, "falcon" }, - { LLM_ARCH_GPT2, "gpt2" }, - { LLM_ARCH_GPTJ, "gptj" }, - { LLM_ARCH_GPTNEOX, "gptneox" }, - { LLM_ARCH_MPT, "mpt" }, - { LLM_ARCH_BAICHUAN, "baichuan" }, - { LLM_ARCH_STARCODER, "starcoder" }, - { LLM_ARCH_PERSIMMON, "persimmon" }, - { LLM_ARCH_REFACT, "refact" }, - { LLM_ARCH_BERT, "bert" }, - { LLM_ARCH_BLOOM, "bloom" }, - { LLM_ARCH_STABLELM, "stablelm" }, - { LLM_ARCH_QWEN, "qwen" }, - { LLM_ARCH_QWEN2, "qwen2" }, - { LLM_ARCH_PHI2, "phi2" }, - { LLM_ARCH_PLAMO, "plamo" }, - { LLM_ARCH_CODESHELL, "codeshell" }, - { LLM_ARCH_ORION, "orion" }, - { LLM_ARCH_INTERNLM2, "internlm2" }, - { LLM_ARCH_MINICPM, "minicpm" }, + { LLM_ARCH_LLAMA, "llama" }, + { LLM_ARCH_FALCON, "falcon" }, + { LLM_ARCH_GPT2, "gpt2" }, + { LLM_ARCH_GPTJ, "gptj" }, + { LLM_ARCH_GPTNEOX, "gptneox" }, + { LLM_ARCH_MPT, "mpt" }, + { LLM_ARCH_BAICHUAN, "baichuan" }, + { LLM_ARCH_STARCODER, "starcoder" }, + { LLM_ARCH_PERSIMMON, "persimmon" }, + { LLM_ARCH_REFACT, "refact" }, + { LLM_ARCH_BERT, "bert" }, + { LLM_ARCH_NOMIC_BERT, "nomic-bert" }, + { LLM_ARCH_BLOOM, "bloom" }, + { LLM_ARCH_STABLELM, "stablelm" }, + { LLM_ARCH_QWEN, "qwen" }, + { LLM_ARCH_QWEN2, "qwen2" }, + { LLM_ARCH_PHI2, "phi2" }, + { LLM_ARCH_PLAMO, "plamo" }, + { LLM_ARCH_CODESHELL, "codeshell" }, + { LLM_ARCH_ORION, "orion" }, + { LLM_ARCH_INTERNLM2, "internlm2" }, + { LLM_ARCH_MINICPM, "minicpm" }, }; enum llm_kv { @@ -375,6 +377,7 @@ enum llm_tensor { LLM_TENSOR_ATTN_OUT, LLM_TENSOR_ATTN_NORM, LLM_TENSOR_ATTN_NORM_2, + LLM_TENSOR_ATTN_OUT_NORM, LLM_TENSOR_ATTN_ROT_EMBD, LLM_TENSOR_FFN_GATE_INP, LLM_TENSOR_FFN_NORM, @@ -387,6 +390,7 @@ enum llm_tensor { LLM_TENSOR_FFN_UP_EXP, LLM_TENSOR_ATTN_Q_NORM, LLM_TENSOR_ATTN_K_NORM, + LLM_TENSOR_LAYER_OUT_NORM, }; static std::map> LLM_TENSOR_NAMES = { @@ -552,12 +556,27 @@ static std::map> LLM_TENSOR_NAMES = { LLM_TENSOR_TOKEN_EMBD_NORM, "token_embd_norm" }, { LLM_TENSOR_TOKEN_TYPES, "token_types" }, { LLM_TENSOR_POS_EMBD, "position_embd" }, - { LLM_TENSOR_ATTN_NORM, "blk.%d.attn_output_norm" }, + { LLM_TENSOR_ATTN_OUT_NORM, "blk.%d.attn_output_norm" }, { LLM_TENSOR_ATTN_Q, "blk.%d.attn_q" }, { LLM_TENSOR_ATTN_K, "blk.%d.attn_k" }, { LLM_TENSOR_ATTN_V, "blk.%d.attn_v" }, { LLM_TENSOR_ATTN_OUT, "blk.%d.attn_output" }, - { LLM_TENSOR_FFN_NORM, "blk.%d.layer_output_norm" }, + { LLM_TENSOR_LAYER_OUT_NORM, "blk.%d.layer_output_norm" }, + { LLM_TENSOR_FFN_DOWN, "blk.%d.ffn_down" }, + { LLM_TENSOR_FFN_UP, "blk.%d.ffn_up" }, + }, + }, + { + LLM_ARCH_NOMIC_BERT, + { + { LLM_TENSOR_TOKEN_EMBD, "token_embd" }, + { LLM_TENSOR_TOKEN_EMBD_NORM, "token_embd_norm" }, + { LLM_TENSOR_TOKEN_TYPES, "token_types" }, + { LLM_TENSOR_ATTN_OUT_NORM, "blk.%d.attn_output_norm" }, + { LLM_TENSOR_ATTN_QKV, "blk.%d.attn_qkv" }, + { LLM_TENSOR_ATTN_OUT, "blk.%d.attn_output" }, + { LLM_TENSOR_LAYER_OUT_NORM, "blk.%d.layer_output_norm" }, + { LLM_TENSOR_FFN_GATE, "blk.%d.ffn_gate" }, { LLM_TENSOR_FFN_DOWN, "blk.%d.ffn_down" }, { LLM_TENSOR_FFN_UP, "blk.%d.ffn_up" }, }, @@ -1485,6 +1504,7 @@ enum e_model { MODEL_22M, MODEL_33M, MODEL_109M, + MODEL_137M, MODEL_335M, MODEL_0_5B, MODEL_1B, @@ -1620,6 +1640,8 @@ struct llama_layer { struct ggml_tensor * attn_q_norm_b; struct ggml_tensor * attn_k_norm; struct ggml_tensor * attn_k_norm_b; + struct ggml_tensor * attn_out_norm; + struct ggml_tensor * attn_out_norm_b; // attention struct ggml_tensor * wq; @@ -1638,6 +1660,8 @@ struct llama_layer { // normalization struct ggml_tensor * ffn_norm; struct ggml_tensor * ffn_norm_b; + struct ggml_tensor * layer_out_norm; + struct ggml_tensor * layer_out_norm_b; // ff struct ggml_tensor * ffn_gate; // w1 @@ -2855,6 +2879,11 @@ static std::string llama_model_ftype_name(llama_ftype ftype) { static const char * llama_model_type_name(e_model type) { switch (type) { + case MODEL_22M: return "22M"; + case MODEL_33M: return "33M"; + case MODEL_109M: return "109M"; + case MODEL_137M: return "137M"; + case MODEL_0_5B: return "0.5B"; case MODEL_1B: return "1B"; case MODEL_2B: return "2B"; case MODEL_3B: return "3B"; @@ -3073,6 +3102,17 @@ static void llm_load_hparams( model.type = e_model::MODEL_335M; break; // bge-large } } break; + case LLM_ARCH_NOMIC_BERT: + { + ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps); + ml.get_key(LLM_KV_ATTENTION_CAUSAL, hparams.causal_attn); + ml.get_key(LLM_KV_TOKENIZER_TOKEN_TYPE_COUNT, hparams.n_vocab_type); + ml.get_key(LLM_KV_POOLING_LAYER, hparams.pooling_layer); + + if (hparams.n_layer == 12 && hparams.n_embd == 768) { + model.type = e_model::MODEL_137M; + } + } break; case LLM_ARCH_BLOOM: { ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps); @@ -3875,10 +3915,14 @@ static bool llm_load_tensors( } } break; case LLM_ARCH_BERT: + case LLM_ARCH_NOMIC_BERT: { - model.tok_embd = ml.create_tensor(ctx_input, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}); - model.type_embd = ml.create_tensor(ctx_input, tn(LLM_TENSOR_TOKEN_TYPES, "weight"), {n_embd, n_vocab_type}); - model.pos_embd = ml.create_tensor(ctx_input, tn(LLM_TENSOR_POS_EMBD, "weight"), {n_embd, hparams.n_ctx_train}); + model.tok_embd = ml.create_tensor(ctx_input, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}); + model.type_embd = ml.create_tensor(ctx_input, tn(LLM_TENSOR_TOKEN_TYPES, "weight"), {n_embd, n_vocab_type}); + if (model.arch == LLM_ARCH_BERT) { + model.pos_embd = ml.create_tensor(ctx_input, tn(LLM_TENSOR_POS_EMBD, "weight"), {n_embd, hparams.n_ctx_train}); + } + model.tok_norm = ml.create_tensor(ctx_output, tn(LLM_TENSOR_TOKEN_EMBD_NORM, "weight"), {n_embd}); model.tok_norm_b = ml.create_tensor(ctx_output, tn(LLM_TENSOR_TOKEN_EMBD_NORM, "bias"), {n_embd}); @@ -3888,29 +3932,38 @@ static bool llm_load_tensors( auto & layer = model.layers[i]; - layer.attn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}); - layer.attn_norm_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM, "bias", i), {n_embd}); + if (model.arch == LLM_ARCH_BERT) { + layer.wq = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_embd}); + layer.bq = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_Q, "bias", i), {n_embd}); - layer.ffn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}); - layer.ffn_norm_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_NORM, "bias", i), {n_embd}); + layer.wk = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_K, "weight", i), {n_embd, n_embd_gqa}); + layer.bk = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_K, "bias", i), {n_embd_gqa}); - layer.wq = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_embd}); - layer.bq = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_Q, "bias", i), {n_embd}); + layer.wv = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_V, "weight", i), {n_embd, n_embd_gqa}); + layer.bv = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_V, "bias", i), {n_embd_gqa}); + } else { + layer.wqkv = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_QKV, "weight", i), {n_embd, n_embd + 2*n_embd_gqa}); + } - layer.wk = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_K, "weight", i), {n_embd, n_embd_gqa}); - layer.bk = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_K, "bias", i), {n_embd_gqa}); + layer.wo = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd}); - layer.wv = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_V, "weight", i), {n_embd, n_embd_gqa}); - layer.bv = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_V, "bias", i), {n_embd_gqa}); + layer.attn_out_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_OUT_NORM, "weight", i), {n_embd}); + layer.attn_out_norm_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_OUT_NORM, "bias", i), {n_embd}); - layer.wo = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd}); - layer.bo = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_OUT, "bias", i), {n_embd}); + layer.ffn_up = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff}); + layer.ffn_down = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_DOWN, "weight", i), {n_ff, n_embd}); - layer.ffn_up = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff}); - layer.ffn_up_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_UP, "bias", i), {n_ff}); + if (model.arch == LLM_ARCH_BERT) { + layer.bo = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_OUT, "bias", i), {n_embd}); + layer.ffn_up_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_UP, "bias", i), {n_ff}); - layer.ffn_down = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_DOWN, "weight", i), {n_ff, n_embd}); - layer.ffn_down_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_DOWN, "bias", i), {n_embd}); + layer.ffn_down_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_DOWN, "bias", i), {n_embd}); + } else { + layer.ffn_gate = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff}); + } + + layer.layer_out_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_LAYER_OUT_NORM, "weight", i), {n_embd}); + layer.layer_out_norm_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_LAYER_OUT_NORM, "bias", i), {n_embd}); } } break; case LLM_ARCH_BLOOM: @@ -5773,6 +5826,7 @@ struct llm_build_context { struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, LLAMA_MAX_NODES, false); const int64_t n_embd_head = hparams.n_embd_head_v; + const int64_t n_embd_gqa = hparams.n_embd_v_gqa(); GGML_ASSERT(n_embd_head == hparams.n_embd_head_k); struct ggml_tensor * cur; @@ -5789,7 +5843,9 @@ struct llm_build_context { // token types are hardcoded to zero ("Sentence A") struct ggml_tensor * type_row0 = ggml_view_1d(ctx0, model.type_embd, n_embd, 0); inpL = ggml_add(ctx0, inpL, type_row0); - inpL = ggml_add(ctx0, ggml_get_rows(ctx0, model.pos_embd, inp_pos), inpL); + if (model.arch == LLM_ARCH_BERT) { + inpL = ggml_add(ctx0, ggml_get_rows(ctx0, model.pos_embd, inp_pos), inpL); + } cb(inpL, "inp_embd", -1); // embed layer norm @@ -5805,7 +5861,7 @@ struct llm_build_context { struct ggml_tensor * cur = inpL; // self-attention - { + if (model.arch == LLM_ARCH_BERT) { struct ggml_tensor * Qcur = ggml_add(ctx0, ggml_mul_mat(ctx0, model.layers[il].wq, cur), model.layers[il].bq); cb(Qcur, "Qcur", il); @@ -5818,6 +5874,37 @@ struct llm_build_context { // seems like we just need to do this for Q? Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); + cur = llm_build_kv(ctx0, model, hparams, kv_self, gf, + model.layers[il].wo, model.layers[il].bo, + Kcur, Vcur, Qcur, KQ_mask, n_ctx, n_tokens, kv_head, n_kv, -1.0f, 1.0f/sqrtf(float(n_embd_head)), cb, il); + cb(cur, "kqv_out", il); + } else { + // compute Q and K and RoPE them + cur = ggml_mul_mat(ctx0, model.layers[il].wqkv, cur); + cb(cur, "wqkv", il); + + struct ggml_tensor * Qcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd, n_tokens, cur->nb[1], 0*sizeof(float)*(n_embd))); + struct ggml_tensor * Kcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd_gqa, n_tokens, cur->nb[1], 1*sizeof(float)*(n_embd))); + struct ggml_tensor * Vcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd_gqa, n_tokens, cur->nb[1], 1*sizeof(float)*(n_embd + n_embd_gqa))); + + cb(Qcur, "Qcur", il); + cb(Kcur, "Kcur", il); + cb(Vcur, "Vcur", il); + + Qcur = ggml_rope_custom( + ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, + hparams.n_rot, 2, 0, n_orig_ctx, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow + ); + cb(Qcur, "Qcur", il); + + Kcur = ggml_rope_custom( + ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, + hparams.n_rot, 2, 0, n_orig_ctx, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow + ); + cb(Kcur, "Kcur", il); + cur = llm_build_kv(ctx0, model, hparams, kv_self, gf, model.layers[il].wo, model.layers[il].bo, Kcur, Vcur, Qcur, KQ_mask, n_ctx, n_tokens, kv_head, n_kv, -1.0f, 1.0f/sqrtf(float(n_embd_head)), cb, il); @@ -5828,25 +5915,34 @@ struct llm_build_context { cur = ggml_add(ctx0, cur, inpL); // attention layer norm - cur = llm_build_norm(ctx0, cur, hparams, model.layers[il].attn_norm, model.layers[il].attn_norm_b, LLM_NORM, cb, il); + cur = llm_build_norm(ctx0, cur, hparams, model.layers[il].attn_out_norm, model.layers[il].attn_out_norm_b, LLM_NORM, cb, il); struct ggml_tensor * ffn_inp = cur; cb(ffn_inp, "ffn_inp", il); // feed-forward network - cur = llm_build_ffn(ctx0, cur, - model.layers[il].ffn_up, model.layers[il].ffn_up_b, - NULL, NULL, - model.layers[il].ffn_down, model.layers[il].ffn_down_b, - NULL, - LLM_FFN_GELU, LLM_FFN_SEQ, cb, il); + if (model.arch == LLM_ARCH_BERT) { + cur = llm_build_ffn(ctx0, cur, + model.layers[il].ffn_up, model.layers[il].ffn_up_b, + NULL, NULL, + model.layers[il].ffn_down, model.layers[il].ffn_down_b, + NULL, + LLM_FFN_GELU, LLM_FFN_SEQ, cb, il); + } else { + cur = llm_build_ffn(ctx0, cur, + model.layers[il].ffn_up, NULL, + model.layers[il].ffn_gate, NULL, + model.layers[il].ffn_down, NULL, + NULL, + LLM_FFN_SILU, LLM_FFN_PAR, cb, il); + } cb(cur, "ffn_out", il); // attentions bypass the intermediate layer cur = ggml_add(ctx0, cur, ffn_inp); // output layer norm - cur = llm_build_norm(ctx0, cur, hparams, model.layers[il].ffn_norm, model.layers[il].ffn_norm_b, LLM_NORM, cb, il); + cur = llm_build_norm(ctx0, cur, hparams, model.layers[il].layer_out_norm, model.layers[il].layer_out_norm_b, LLM_NORM, cb, il); // input for next layer inpL = cur; @@ -7289,6 +7385,7 @@ static struct ggml_cgraph * llama_build_graph( result = llm.build_refact(); } break; case LLM_ARCH_BERT: + case LLM_ARCH_NOMIC_BERT: { result = llm.build_bert(); } break; From 6c00a066928b0475b865a2e3e709e2166e02d548 Mon Sep 17 00:00:00 2001 From: John <78893154+cmp-nct@users.noreply.github.com> Date: Tue, 13 Feb 2024 18:56:38 +0100 Subject: [PATCH 341/812] gguf : add python reader example (#5216) * Update CMakeLists.txt * Create reader.py * Update reader.py * Update reader.py another whitespace :| * Update reader.py * lintlintlint --- examples/CMakeLists.txt | 1 + gguf-py/examples/reader.py | 45 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+) create mode 100644 gguf-py/examples/reader.py diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index 68ad89964..653abc73a 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -38,6 +38,7 @@ else() add_subdirectory(speculative) add_subdirectory(lookahead) add_subdirectory(lookup) + add_subdirectory(gguf) add_subdirectory(train-text-from-scratch) add_subdirectory(imatrix) if (LLAMA_BUILD_SERVER) diff --git a/gguf-py/examples/reader.py b/gguf-py/examples/reader.py new file mode 100644 index 000000000..62e0769da --- /dev/null +++ b/gguf-py/examples/reader.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python3 +import sys +from pathlib import Path +from gguf.gguf_reader import GGUFReader + + +sys.path.insert(0, str(Path(__file__).parent.parent)) + + +def read_gguf_file(gguf_file_path): + """ + Reads and prints key-value pairs and tensor information from a GGUF file in an improved format. + + Parameters: + - gguf_file_path: Path to the GGUF file. + """ + + reader = GGUFReader(gguf_file_path) + + # List all key-value pairs in a columnized format + print("Key-Value Pairs:") + max_key_length = max(len(key) for key in reader.fields.keys()) + for key, field in reader.fields.items(): + value = field.parts[field.data[0]] + print(f"{key:{max_key_length}} : {value}") + print("----") + + # List all tensors + print("Tensors:") + tensor_info_format = "{:<30} | Shape: {:<15} | Size: {:<12} | Quantization: {}" + print(tensor_info_format.format("Tensor Name", "Shape", "Size", "Quantization")) + print("-" * 80) + for tensor in reader.tensors: + shape_str = "x".join(map(str, tensor.shape)) + size_str = str(tensor.n_elements) + quantization_str = tensor.tensor_type.name + print(tensor_info_format.format(tensor.name, shape_str, size_str, quantization_str)) + + +if __name__ == '__main__': + if len(sys.argv) < 2: + print("Usage: reader.py ") + sys.exit(1) + gguf_file_path = sys.argv[1] + read_gguf_file(gguf_file_path) From f5ca054855dea83f424003162f26de376e5643f6 Mon Sep 17 00:00:00 2001 From: AT Date: Tue, 13 Feb 2024 15:44:25 -0600 Subject: [PATCH 342/812] Early return for zero size calls to get_tensor. (#5482) * Early return for zero size calls to get_tensor. Signed-off-by: Adam Treat * Update ggml-kompute.cpp Co-authored-by: Georgi Gerganov * Update ggml-kompute.cpp Co-authored-by: Georgi Gerganov * Add an early return to the get/set tensor when the size is null. Signed-off-by: Adam Treat * Early return after the assertions. Signed-off-by: Adam Treat * Since we do the early return in the generic backend now no reason to do so here as well. Signed-off-by: Adam Treat --------- Signed-off-by: Adam Treat Co-authored-by: Georgi Gerganov --- ggml-backend.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/ggml-backend.c b/ggml-backend.c index 9ee81b766..87eea8440 100644 --- a/ggml-backend.c +++ b/ggml-backend.c @@ -219,6 +219,10 @@ GGML_CALL void ggml_backend_tensor_set(struct ggml_tensor * tensor, const void * GGML_ASSERT(buf != NULL && "tensor buffer not set"); GGML_ASSERT(offset + size <= ggml_nbytes(tensor) && "tensor write out of bounds"); + if (!size) { + return; + } + tensor->buffer->iface.set_tensor(buf, tensor, data, offset, size); } @@ -229,6 +233,10 @@ GGML_CALL void ggml_backend_tensor_get(const struct ggml_tensor * tensor, void * GGML_ASSERT(tensor->buffer != NULL && "tensor buffer not set"); GGML_ASSERT(offset + size <= ggml_nbytes(tensor) && "tensor read out of bounds"); + if (!size) { + return; + } + tensor->buffer->iface.get_tensor(buf, tensor, data, offset, size); } From aa2341298924ac89778252015efcb792f2df1e20 Mon Sep 17 00:00:00 2001 From: John <78893154+cmp-nct@users.noreply.github.com> Date: Wed, 14 Feb 2024 08:38:35 +0100 Subject: [PATCH 343/812] llava : support v1.6 (#5267) * Create llava-survery-v2.py * Update convert-image-encoder-to-gguf.py * Update convert-image-encoder-to-gguf.py * Rename llava-survery-v2.py to llava-surgery-v2.py * Update convert-image-encoder-to-gguf.py will now search for projector * Update convert-image-encoder-to-gguf.py whoops * Update llava-surgery-v2.py * Clip: Bugfix for normalization (it did not loat the 3 std and mean values) Clip: bicubic resize function Clip: added save-to-bmp/pil for debugging and conversion from/to 32/8 images Clip: added normalization with FP16 precision simulation (image tensors match HF implementation, can be switched off, only used for llava-1.6) Clip: added newline tensor, mergetype kv, image-grid kv, new resize-pad function with resolution from gridpoints Clip: clip_image_preprocess now returns a float * vector instead of float, this way llava 1.5 and 1.6 is supported llava: added ggml cpu graph for embedding patching, added spatial_unpad preliminary support, added a lot of comments that need to be cleaned when all is final convert-image-encoder: fixed image-grid flattening * whitespace corrections * ws * Tensors are now properly permuted. Before the embeddings were inserted 1:1, now they are split into the 24x24 patches as in reference. * ws * added verbose_prompt support into cli added stopwords for llava-1.6 into cli * moved llava functions to llava.cpp, made clip.h C compatible API, replaced vector style functions with pointers, added a debug define to remove functions from compilation while not needed * ws * convert : skip unknown tensors (need for LLaVA) * llava : update readme * llava : fix compile warnings * llava : style * convert : add --skip-unknown CLI arg * server : remove clip structs * bugfix for non llava-1.6 It should now work with llava-1.5 as well * clip : minor code rearrange * llava : update readme a bit --------- Co-authored-by: John Co-authored-by: Georgi Gerganov --- convert.py | 37 +- examples/llava/README.md | 12 +- examples/llava/clip.cpp | 766 +++++++++++++++--- examples/llava/clip.h | 47 +- .../llava/convert-image-encoder-to-gguf.py | 66 +- examples/llava/llava-cli.cpp | 26 +- examples/llava/llava-surgery-v2.py | 167 ++++ examples/llava/llava.cpp | 296 ++++++- examples/llava/llava.h | 2 - examples/server/server.cpp | 15 +- 10 files changed, 1229 insertions(+), 205 deletions(-) create mode 100644 examples/llava/llava-surgery-v2.py diff --git a/convert.py b/convert.py index 323e8058d..63a0a5d78 100755 --- a/convert.py +++ b/convert.py @@ -1173,7 +1173,7 @@ def convert_to_output_type(model: LazyModel, output_type: GGMLFileType) -> LazyM for (name, tensor) in model.items()} -def convert_model_names(model: LazyModel, params: Params) -> LazyModel: +def convert_model_names(model: LazyModel, params: Params, skip_unknown: bool) -> LazyModel: tmap = gguf.TensorNameMap(ARCH, params.n_layer) should_skip: set[gguf.MODEL_TENSOR] = set(gguf.MODEL_TENSOR_SKIP.get(ARCH, [])) @@ -1199,7 +1199,11 @@ def convert_model_names(model: LazyModel, params: Params) -> LazyModel: for name, lazy_tensor in model.items(): tensor_type, name_new = tmap.get_type_and_name(name, try_suffixes = (".weight", ".bias")) or (None, None) if name_new is None: - raise Exception(f"Unexpected tensor name: {name}") + if skip_unknown: + print(f"Unexpected tensor name: {name} - skipping") + continue + else: + raise Exception(f"Unexpected tensor name: {name}. Use --skip-unknown to ignore it (e.g. LLaVA)") if tensor_type in should_skip: print(f"skipping tensor {name_new}") @@ -1377,19 +1381,20 @@ def main(args_in: list[str] | None = None) -> None: output_choices.append("q8_0") vocab_types = ["spm", "bpe", "hfft"] parser = argparse.ArgumentParser(description="Convert a LLaMa model to a GGML compatible file") - parser.add_argument("--awq-path", type=Path, help="Path to scale awq cache file", default=None) - parser.add_argument("--dump", action="store_true", help="don't convert, just show what's in the model") - parser.add_argument("--dump-single", action="store_true", help="don't convert, just show what's in a single model file") - parser.add_argument("--vocab-only", action="store_true", help="extract only the vocab") - parser.add_argument("--outtype", choices=output_choices, help="output format - note: q8_0 may be very slow (default: f16 or f32 based on input)") - parser.add_argument("--vocab-dir", type=Path, help="directory containing tokenizer.model, if separate from model file") - parser.add_argument("--vocab-type", choices=vocab_types, help="The vocabulary format used to define the tokenizer model (default: spm)", default="spm") - parser.add_argument("--outfile", type=Path, help="path to write to; default: based on input") - parser.add_argument("model", type=Path, help="directory containing model file, or model file itself (*.pth, *.pt, *.bin)") - parser.add_argument("--ctx", type=int, help="model training context (default: based on input)") - parser.add_argument("--concurrency", type=int, help=f"concurrency used for conversion (default: {DEFAULT_CONCURRENCY})", default=DEFAULT_CONCURRENCY) - parser.add_argument("--big-endian", action="store_true", help="model is executed on big endian machine") - parser.add_argument("--pad-vocab", action="store_true", help="add pad tokens when model vocab expects more than tokenizer metadata provides") + parser.add_argument("--awq-path", type=Path, help="Path to scale awq cache file", default=None) + parser.add_argument("--dump", action="store_true", help="don't convert, just show what's in the model") + parser.add_argument("--dump-single", action="store_true", help="don't convert, just show what's in a single model file") + parser.add_argument("--vocab-only", action="store_true", help="extract only the vocab") + parser.add_argument("--outtype", choices=output_choices, help="output format - note: q8_0 may be very slow (default: f16 or f32 based on input)") + parser.add_argument("--vocab-dir", type=Path, help="directory containing tokenizer.model, if separate from model file") + parser.add_argument("--vocab-type", choices=vocab_types, help="The vocabulary format used to define the tokenizer model (default: spm)", default="spm") + parser.add_argument("--outfile", type=Path, help="path to write to; default: based on input") + parser.add_argument("model", type=Path, help="directory containing model file, or model file itself (*.pth, *.pt, *.bin)") + parser.add_argument("--ctx", type=int, help="model training context (default: based on input)") + parser.add_argument("--concurrency", type=int, help=f"concurrency used for conversion (default: {DEFAULT_CONCURRENCY})", default=DEFAULT_CONCURRENCY) + parser.add_argument("--big-endian", action="store_true", help="model is executed on big endian machine") + parser.add_argument("--pad-vocab", action="store_true", help="add pad tokens when model vocab expects more than tokenizer metadata provides") + parser.add_argument("--skip-unknown", action="store_true", help="skip unknown tensor names instead of failing") args = parser.parse_args(args_in) if args.awq_path: @@ -1461,7 +1466,7 @@ def main(args_in: list[str] | None = None) -> None: print(f"Special vocab info: {special_vocab}") model = model_plus.model - model = convert_model_names(model, params) + model = convert_model_names(model, params, args.skip_unknown) ftype = pick_output_type(model, args.outtype) model = convert_to_output_type(model, ftype) outfile = args.outfile or default_outfile(model_plus.paths, ftype) diff --git a/examples/llava/README.md b/examples/llava/README.md index 19f1a50a2..e2ef0eff1 100644 --- a/examples/llava/README.md +++ b/examples/llava/README.md @@ -19,9 +19,9 @@ After building, run: `./llava-cli` to see the usage. For example: **note**: A lower temperature like 0.1 is recommended for better quality. add `--temp 0.1` to the command to do so. -## Model conversion +## LLaVA 1.5 -- Clone `llava-v15-7b` and `clip-vit-large-patch14-336` locally: +- Clone a LLaVA and a CLIP model ([available options](https://github.com/haotian-liu/LLaVA/blob/main/docs/MODEL_ZOO.md)). For example: ```sh git clone https://huggingface.co/liuhaotian/llava-v1.5-7b @@ -55,8 +55,14 @@ python ./convert.py ../llava-v1.5-7b Now both the LLaMA part and the image encoder is in the `llava-v1.5-7b` directory. +## LLaVA 1.6 + +- Use `llava-surgery-v2.py` + +- TODO: add detailed instructions + ## TODO -- [ ] Support non-CPU backend for the image encoding part. +- [x] Support non-CPU backend for the image encoding part. - [ ] Support different sampling methods. - [ ] Support more model variants. diff --git a/examples/llava/clip.cpp b/examples/llava/clip.cpp index ccd0d85ad..9c5091e61 100644 --- a/examples/llava/clip.cpp +++ b/examples/llava/clip.cpp @@ -1,7 +1,7 @@ // NOTE: This is modified from clip.cpp only for LLaVA, // so there might be still unnecessary artifacts hanging around // I'll gradually clean and extend it - +// Note: Even when using identical normalized image inputs (see normalize_image_u8_to_f32()) we have a significant difference in resulting embeddings compared to pytorch #include "clip.h" #include "ggml.h" #include "ggml-alloc.h" @@ -30,6 +30,26 @@ #include #include #include +#include + +//#define CLIP_DEBUG_FUNCTIONS + +// RGB uint8 image +struct clip_image_u8 { + int nx; + int ny; + + std::vector buf; +}; + +// RGB float32 image (NHWC) +// Memory layout: RGBRGBRGB... +struct clip_image_f32 { + int nx; + int ny; + + std::vector buf; +}; static std::string format(const char * fmt, ...) { va_list ap; @@ -50,50 +70,56 @@ static std::string format(const char * fmt, ...) { // key constants // -#define KEY_FTYPE "general.file_type" -#define KEY_NAME "general.name" -#define KEY_DESCRIPTION "general.description" -#define KEY_HAS_TEXT_ENC "clip.has_text_encoder" -#define KEY_HAS_VIS_ENC "clip.has_vision_encoder" +#define KEY_FTYPE "general.file_type" +#define KEY_NAME "general.name" +#define KEY_DESCRIPTION "general.description" +#define KEY_HAS_TEXT_ENC "clip.has_text_encoder" +#define KEY_HAS_VIS_ENC "clip.has_vision_encoder" #define KEY_HAS_LLAVA_PROJ "clip.has_llava_projector" -#define KEY_USE_GELU "clip.use_gelu" -#define KEY_N_EMBD "clip.%s.embedding_length" -#define KEY_N_FF "clip.%s.feed_forward_length" -#define KEY_N_BLOCK "clip.%s.block_count" -#define KEY_N_HEAD "clip.%s.attention.head_count" +#define KEY_USE_GELU "clip.use_gelu" +#define KEY_N_EMBD "clip.%s.embedding_length" +#define KEY_N_FF "clip.%s.feed_forward_length" +#define KEY_N_BLOCK "clip.%s.block_count" +#define KEY_N_HEAD "clip.%s.attention.head_count" #define KEY_LAYER_NORM_EPS "clip.%s.attention.layer_norm_epsilon" -#define KEY_PROJ_DIM "clip.%s.projection_dim" -#define KEY_TOKENS "tokenizer.ggml.tokens" -#define KEY_N_POSITIONS "clip.text.context_length" -#define KEY_IMAGE_SIZE "clip.vision.image_size" -#define KEY_PATCH_SIZE "clip.vision.patch_size" -#define KEY_IMAGE_MEAN "clip.vision.image_mean" -#define KEY_IMAGE_STD "clip.vision.image_std" -#define KEY_PROJ_TYPE "clip.projector_type" +#define KEY_PROJ_DIM "clip.%s.projection_dim" +#define KEY_TOKENS "tokenizer.ggml.tokens" +#define KEY_N_POSITIONS "clip.text.context_length" +#define KEY_IMAGE_SIZE "clip.vision.image_size" +#define KEY_PATCH_SIZE "clip.vision.patch_size" +#define KEY_IMAGE_MEAN "clip.vision.image_mean" +#define KEY_IMAGE_STD "clip.vision.image_std" +#define KEY_PROJ_TYPE "clip.projector_type" + +#define KEY_MM_PATCH_MERGE_TYPE "clip.vision.mm_patch_merge_type" +#define KEY_IMAGE_GRID_PINPOINTS "clip.vision.image_grid_pinpoints" +#define KEY_IMAGE_CROP_RESOLUTION "clip.vision.image_crop_resolution" + // // tensor name constants // -#define TN_TOKEN_EMBD "%s.token_embd.weight" -#define TN_POS_EMBD "%s.position_embd.weight" -#define TN_CLASS_EMBD "v.class_embd" -#define TN_PATCH_EMBD "v.patch_embd.weight" -#define TN_ATTN_K "%s.blk.%d.attn_k.%s" -#define TN_ATTN_Q "%s.blk.%d.attn_q.%s" -#define TN_ATTN_V "%s.blk.%d.attn_v.%s" -#define TN_ATTN_OUTPUT "%s.blk.%d.attn_out.%s" -#define TN_FFN_DOWN "%s.blk.%d.ffn_down.%s" -#define TN_FFN_UP "%s.blk.%d.ffn_up.%s" -#define TN_LN_1 "%s.blk.%d.ln1.%s" -#define TN_LN_2 "%s.blk.%d.ln2.%s" -#define TN_LN_PRE "%s.pre_ln.%s" -#define TN_LN_POST "%s.post_ln.%s" -#define TN_TEXT_PROJ "text_projection.weight" -#define TN_VIS_PROJ "visual_projection.weight" -#define TN_LLAVA_PROJ "mm.%d.%s" -#define TN_MVLM_PROJ_MLP "mm.model.mlp.%d.%s" +#define TN_TOKEN_EMBD "%s.token_embd.weight" +#define TN_POS_EMBD "%s.position_embd.weight" +#define TN_CLASS_EMBD "v.class_embd" +#define TN_PATCH_EMBD "v.patch_embd.weight" +#define TN_ATTN_K "%s.blk.%d.attn_k.%s" +#define TN_ATTN_Q "%s.blk.%d.attn_q.%s" +#define TN_ATTN_V "%s.blk.%d.attn_v.%s" +#define TN_ATTN_OUTPUT "%s.blk.%d.attn_out.%s" +#define TN_FFN_DOWN "%s.blk.%d.ffn_down.%s" +#define TN_FFN_UP "%s.blk.%d.ffn_up.%s" +#define TN_LN_1 "%s.blk.%d.ln1.%s" +#define TN_LN_2 "%s.blk.%d.ln2.%s" +#define TN_LN_PRE "%s.pre_ln.%s" +#define TN_LN_POST "%s.post_ln.%s" +#define TN_TEXT_PROJ "text_projection.weight" +#define TN_VIS_PROJ "visual_projection.weight" +#define TN_LLAVA_PROJ "mm.%d.%s" +#define TN_MVLM_PROJ_MLP "mm.model.mlp.%d.%s" #define TN_MVLM_PROJ_BLOCK "mm.model.mb_block.%d.block.%d.%s" +#define TN_IMAGE_NEWLINE "model.image_newline" enum projector_type { @@ -104,8 +130,8 @@ enum projector_type { }; static std::map PROJECTOR_TYPE_NAMES = { - { PROJECTOR_TYPE_MLP, "mlp" }, - { PROJECTOR_TYPE_LDP, "ldp" }, + { PROJECTOR_TYPE_MLP, "mlp" }, + { PROJECTOR_TYPE_LDP, "ldp" }, }; @@ -165,7 +191,6 @@ static std::string gguf_data_to_str(enum gguf_type type, const void * data, int } } - static void replace_all(std::string & s, const std::string & search, const std::string & replace) { std::string result; for (size_t pos = 0; ; pos += search.length()) { @@ -217,7 +242,7 @@ static std::string gguf_kv_to_str(const struct gguf_context * ctx_gguf, int i) { } } -static void print_tensor_info(const ggml_tensor* tensor, const char* prefix = "") { +static void print_tensor_info(const ggml_tensor * tensor, const char * prefix = "") { size_t tensor_size = ggml_nbytes(tensor); printf("%s: n_dims = %d, name = %s, tensor_size=%zu, shape:[%" PRId64 ", %" PRId64 ", %" PRId64 ", %" PRId64 "], type = %s\n", prefix, ggml_n_dims(tensor), tensor->name, tensor_size, @@ -233,31 +258,136 @@ static projector_type clip_projector_type_from_string(const std::string & name) return PROJECTOR_TYPE_UNKNOWN; } -// -// image data -// +#ifdef CLIP_DEBUG_FUNCTIONS +static void clip_image_write_image_to_ppm(const clip_image_u8& img, const std::string& filename) { + std::ofstream file(filename, std::ios::binary); + if (!file.is_open()) { + std::cerr << "Failed to open file for writing: " << filename << std::endl; + return; + } -// RGB uint8 image -struct clip_image_u8 { - int nx; - int ny; + // PPM header: P6 format, width, height, and max color value + file << "P6\n" << img.nx << " " << img.ny << "\n255\n"; - std::vector buf; -}; + // Write pixel data + for (size_t i = 0; i < img.buf.size(); i += 3) { + // PPM expects binary data in RGB format, which matches our image buffer + file.write(reinterpret_cast(&img.buf[i]), 3); + } -// RGB float32 image (NHWC) -// Memory layout: RGBRGBRGB... -struct clip_image_f32 { - int nx; - int ny; + file.close(); +} + +static void clip_image_save_to_bmp(const clip_image_u8& img, const std::string& filename) { + std::ofstream file(filename, std::ios::binary); + if (!file.is_open()) { + std::cerr << "Failed to open file for writing: " << filename << std::endl; + return; + } + + int fileSize = 54 + 3 * img.nx * img.ny; // File header + info header + pixel data + int bytesPerPixel = 3; + int widthInBytes = img.nx * bytesPerPixel; + int paddingAmount = (4 - (widthInBytes % 4)) % 4; + int stride = widthInBytes + paddingAmount; + + // Bitmap file header + unsigned char fileHeader[14] = { + 'B','M', // Signature + 0,0,0,0, // Image file size in bytes + 0,0,0,0, // Reserved + 54,0,0,0 // Start of pixel array + }; + + // Total file size + fileSize = 54 + (stride * img.ny); + fileHeader[2] = (unsigned char)(fileSize); + fileHeader[3] = (unsigned char)(fileSize >> 8); + fileHeader[4] = (unsigned char)(fileSize >> 16); + fileHeader[5] = (unsigned char)(fileSize >> 24); + + // Bitmap information header (BITMAPINFOHEADER) + unsigned char infoHeader[40] = { + 40,0,0,0, // Size of this header (40 bytes) + 0,0,0,0, // Image width + 0,0,0,0, // Image height + 1,0, // Number of color planes + 24,0, // Bits per pixel + 0,0,0,0, // No compression + 0,0,0,0, // Image size (can be 0 for no compression) + 0,0,0,0, // X pixels per meter (not specified) + 0,0,0,0, // Y pixels per meter (not specified) + 0,0,0,0, // Total colors (color table not used) + 0,0,0,0 // Important colors (all are important) + }; + + // Width and height in the information header + infoHeader[4] = (unsigned char)(img.nx); + infoHeader[5] = (unsigned char)(img.nx >> 8); + infoHeader[6] = (unsigned char)(img.nx >> 16); + infoHeader[7] = (unsigned char)(img.nx >> 24); + infoHeader[8] = (unsigned char)(img.ny); + infoHeader[9] = (unsigned char)(img.ny >> 8); + infoHeader[10] = (unsigned char)(img.ny >> 16); + infoHeader[11] = (unsigned char)(img.ny >> 24); + + // Write file headers + file.write(reinterpret_cast(fileHeader), sizeof(fileHeader)); + file.write(reinterpret_cast(infoHeader), sizeof(infoHeader)); + + // Pixel data + std::vector padding(3, 0); // Max padding size to be added to each row + for (int y = img.ny - 1; y >= 0; --y) { // BMP files are stored bottom-to-top + for (int x = 0; x < img.nx; ++x) { + // Each pixel + size_t pixelIndex = (y * img.nx + x) * 3; + unsigned char pixel[3] = { + img.buf[pixelIndex + 2], // BMP stores pixels in BGR format + img.buf[pixelIndex + 1], + img.buf[pixelIndex] + }; + file.write(reinterpret_cast(pixel), 3); + } + // Write padding for the row + file.write(reinterpret_cast(padding.data()), paddingAmount); + } + + file.close(); +} + +// debug function to convert f32 to u8 +static void clip_image_convert_f32_to_u8(const clip_image_f32& src, clip_image_u8& dst) { + dst.nx = src.nx; + dst.ny = src.ny; + dst.buf.resize(3 * src.nx * src.ny); + for (size_t i = 0; i < src.buf.size(); ++i) { + dst.buf[i] = static_cast(std::min(std::max(int(src.buf[i] * 255.0f), 0), 255)); + } +} +#endif - std::vector buf; -}; // // clip layers // +struct clip_hparams { + int32_t image_size; + int32_t patch_size; + int32_t hidden_size; + int32_t n_intermediate; + int32_t projection_dim; + int32_t n_head; + int32_t n_layer; + + float eps; + + char mm_patch_merge_type[32] = "flat"; // spatial_unpad or flat (default) + + int32_t image_grid_pinpoints[32]; + int32_t image_crop_resolution; +}; + struct clip_layer { // attention struct ggml_tensor * k_w; @@ -287,7 +417,7 @@ struct clip_layer { }; struct clip_vision_model { - struct clip_vision_hparams hparams; + struct clip_hparams hparams; // embeddings struct ggml_tensor * class_embedding; @@ -310,6 +440,8 @@ struct clip_vision_model { struct ggml_tensor * mm_2_w = NULL; struct ggml_tensor * mm_2_b = NULL; + struct ggml_tensor * image_newline = NULL; + // Yi type models with mlp+normalization projection struct ggml_tensor * mm_1_w = NULL; // Yi type models have 0, 1, 3, 4 struct ggml_tensor * mm_1_b = NULL; @@ -364,9 +496,10 @@ struct clip_ctx { std::vector buf_compute_meta; // memory buffers to evaluate the model - ggml_backend_buffer_t params_buffer = NULL; + ggml_backend_buffer_t params_buffer = NULL; ggml_backend_buffer_t compute_buffer = NULL; - ggml_backend_t backend = NULL; + + ggml_backend_t backend = NULL; ggml_gallocr_t compute_alloc = NULL; }; @@ -379,18 +512,19 @@ static ggml_cgraph * clip_image_build_graph(clip_ctx * ctx, const clip_image_f32 const auto & model = ctx->vision_model; const auto & hparams = model.hparams; - const int image_size = hparams.image_size; - const int patch_size = hparams.patch_size; - const int num_patches = ((image_size / patch_size) * (image_size / patch_size)); - const int num_positions = num_patches + 1; - const int hidden_size = hparams.hidden_size; - const int n_head = hparams.n_head; - const int d_head = hidden_size / n_head; - const int n_layer = hparams.n_layer; - //const int n_intermediate = hparams.n_intermediate; - //const int projection_dim = hparams.projection_dim; - const float eps = hparams.eps; - int batch_size = imgs->size; + const int image_size = hparams.image_size; + const int patch_size = hparams.patch_size; + const int num_patches = ((image_size / patch_size) * (image_size / patch_size)); + const int num_patches_per_side = image_size / patch_size; GGML_UNUSED(num_patches_per_side); + const int num_positions = num_patches + 1; + const int hidden_size = hparams.hidden_size; + const int n_head = hparams.n_head; + const int d_head = hidden_size / n_head; + const int n_layer = hparams.n_layer; + const float eps = hparams.eps; + + const int batch_size = imgs->size; + if (ctx->has_llava_projector) { GGML_ASSERT(batch_size == 1); } @@ -540,7 +674,6 @@ static ggml_cgraph * clip_image_build_graph(clip_ctx * ctx, const clip_image_f32 embeddings = ggml_add(ctx0, embeddings, model.mm_0_b); embeddings = ggml_gelu(ctx0, embeddings); - embeddings = ggml_mul_mat(ctx0, model.mm_2_w, embeddings); embeddings = ggml_add(ctx0, embeddings, model.mm_2_b); @@ -791,10 +924,10 @@ struct clip_ctx * clip_model_load(const char * fname, const int verbosity = 1) { if (idx != -1) { const std::string proj_type = gguf_get_val_str(ctx, idx); new_clip->proj_type = clip_projector_type_from_string(proj_type); - } - else { + } else { new_clip->proj_type = PROJECTOR_TYPE_MLP; } + if (new_clip->proj_type == PROJECTOR_TYPE_MLP) { if (gguf_find_tensor(ctx, format(TN_LLAVA_PROJ, 3, "weight").c_str()) != -1) { new_clip->proj_type = PROJECTOR_TYPE_MLP_NORM; @@ -920,11 +1053,41 @@ struct clip_ctx * clip_model_load(const char * fname, const int verbosity = 1) { hparams.projection_dim = get_u32(ctx, format(KEY_PROJ_DIM, "vision")); hparams.eps = get_f32(ctx, format(KEY_LAYER_NORM_EPS, "vision")); + try { + int idx = get_key_idx(ctx, KEY_IMAGE_GRID_PINPOINTS); + int n = gguf_get_arr_n(ctx, idx); + const int32_t * pinpoints = (const int32_t *)gguf_get_arr_data(ctx, idx); + for (int i = 0; i < 32 && i < n && pinpoints[i] != 0; ++i) { + hparams.image_grid_pinpoints[i] = pinpoints[i]; + } + if (n < 32) + hparams.image_grid_pinpoints[n] = 0; + } catch (std::runtime_error & e) { + hparams.image_grid_pinpoints[0]=0; + } + + try { + int idx = get_key_idx(ctx, KEY_MM_PATCH_MERGE_TYPE); + strcpy(hparams.mm_patch_merge_type, gguf_get_val_str(ctx, idx)); + } catch (std::runtime_error & e) { + strcpy(hparams.mm_patch_merge_type, "flat"); + } + + try { + hparams.image_crop_resolution = get_u32(ctx, KEY_IMAGE_CROP_RESOLUTION); // llava-1.6 + } catch(const std::exception& e) { + hparams.image_crop_resolution = hparams.image_size; + } + int idx_mean = get_key_idx(ctx, KEY_IMAGE_MEAN); int idx_std = get_key_idx(ctx, KEY_IMAGE_STD); + + const float * mean_data = (const float *)gguf_get_arr_data(ctx, idx_mean); + const float * std_data = (const float *)gguf_get_arr_data(ctx, idx_std); + for (int i = 0; i < 3; ++i) { - new_clip->image_mean[i] = *((const float *)gguf_get_arr_data(ctx, idx_mean)); - new_clip->image_std[i] = *((const float *)gguf_get_arr_data(ctx, idx_std)); + new_clip->image_mean[i] = mean_data[i]; + new_clip->image_std[i] = std_data[i]; } if (verbosity >= 2) { @@ -936,13 +1099,27 @@ struct clip_ctx * clip_model_load(const char * fname, const int verbosity = 1) { printf("v_projection_dim %d\n", hparams.projection_dim); printf("v_n_head %d\n", hparams.n_head); printf("v_n_layer %d\n", hparams.n_layer); + printf("v_eps %f\n", hparams.eps); + printf("v_image_mean %f %f %f\n", new_clip->image_mean[0], new_clip->image_mean[1], new_clip->image_mean[2]); + printf("v_image_std %f %f %f\n", new_clip->image_std[0], new_clip->image_std[1], new_clip->image_std[2]); + printf("v_image_grid_pinpoints: "); + for (int i = 0; i < 32 & hparams.image_grid_pinpoints[i]!=0; ++i) { + printf("%d ", hparams.image_grid_pinpoints[i]); + } + printf("\n"); + printf("v_mm_patch_merge_type: %s\n", hparams.mm_patch_merge_type); + } - vision_model.patch_embeddings = get_tensor(new_clip->ctx_data, TN_PATCH_EMBD); - vision_model.class_embedding = get_tensor(new_clip->ctx_data, TN_CLASS_EMBD); - vision_model.position_embeddings = get_tensor(new_clip->ctx_data, format(TN_POS_EMBD, "v")); - vision_model.pre_ln_w = get_tensor(new_clip->ctx_data, format(TN_LN_PRE, "v", "weight")); - vision_model.pre_ln_b = get_tensor(new_clip->ctx_data, format(TN_LN_PRE, "v", "bias")); + try { + vision_model.patch_embeddings = get_tensor(new_clip->ctx_data, TN_PATCH_EMBD); + vision_model.class_embedding = get_tensor(new_clip->ctx_data, TN_CLASS_EMBD); + vision_model.position_embeddings = get_tensor(new_clip->ctx_data, format(TN_POS_EMBD, "v")); + vision_model.pre_ln_w = get_tensor(new_clip->ctx_data, format(TN_LN_PRE, "v", "weight")); + vision_model.pre_ln_b = get_tensor(new_clip->ctx_data, format(TN_LN_PRE, "v", "bias")); + } catch(const std::exception& e) { + fprintf(stderr, "%s: failed to load vision model tensors\n", __func__); + } // LLaVA projection if (new_clip->proj_type == PROJECTOR_TYPE_MLP || new_clip->proj_type == PROJECTOR_TYPE_MLP_NORM) { @@ -968,40 +1145,43 @@ struct clip_ctx * clip_model_load(const char * fname, const int verbosity = 1) { vision_model.mm_4_w = get_tensor(new_clip->ctx_data, format(TN_LLAVA_PROJ, 4, "weight")); vision_model.mm_4_b = get_tensor(new_clip->ctx_data, format(TN_LLAVA_PROJ, 4, "bias")); } catch (std::runtime_error & e) { } - } - else if (new_clip->proj_type == PROJECTOR_TYPE_LDP) { + try { + vision_model.image_newline = get_tensor(new_clip->ctx_data, TN_IMAGE_NEWLINE); + // fprintf(stderr, "%s: image_newline tensor (llava-1.6) found\n", __func__); + } catch (std::runtime_error & e) { } + } else if (new_clip->proj_type == PROJECTOR_TYPE_LDP) { // MobileVLM projection - vision_model.mm_model_mlp_1_w = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_MLP, 1, "weight")); - vision_model.mm_model_mlp_1_b = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_MLP, 1, "bias")); - vision_model.mm_model_mlp_3_w = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_MLP, 3, "weight")); - vision_model.mm_model_mlp_3_b = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_MLP, 3, "bias")); - vision_model.mm_model_block_1_block_0_0_w = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_BLOCK, 1, 0, "0.weight")); - vision_model.mm_model_block_1_block_0_1_w = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_BLOCK, 1, 0, "1.weight")); - vision_model.mm_model_block_1_block_0_1_b = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_BLOCK, 1, 0, "1.bias")); + vision_model.mm_model_mlp_1_w = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_MLP, 1, "weight")); + vision_model.mm_model_mlp_1_b = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_MLP, 1, "bias")); + vision_model.mm_model_mlp_3_w = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_MLP, 3, "weight")); + vision_model.mm_model_mlp_3_b = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_MLP, 3, "bias")); + vision_model.mm_model_block_1_block_0_0_w = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_BLOCK, 1, 0, "0.weight")); + vision_model.mm_model_block_1_block_0_1_w = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_BLOCK, 1, 0, "1.weight")); + vision_model.mm_model_block_1_block_0_1_b = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_BLOCK, 1, 0, "1.bias")); vision_model.mm_model_block_1_block_1_fc1_w = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_BLOCK, 1, 1, "fc1.weight")); vision_model.mm_model_block_1_block_1_fc1_b = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_BLOCK, 1, 1, "fc1.bias")); vision_model.mm_model_block_1_block_1_fc2_w = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_BLOCK, 1, 1, "fc2.weight")); vision_model.mm_model_block_1_block_1_fc2_b = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_BLOCK, 1, 1, "fc2.bias")); - vision_model.mm_model_block_1_block_2_0_w = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_BLOCK, 1, 2, "0.weight")); - vision_model.mm_model_block_1_block_2_1_w = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_BLOCK, 1, 2, "1.weight")); - vision_model.mm_model_block_1_block_2_1_b = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_BLOCK, 1, 2, "1.bias")); - vision_model.mm_model_block_2_block_0_0_w = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_BLOCK, 2, 0, "0.weight")); - vision_model.mm_model_block_2_block_0_1_w = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_BLOCK, 2, 0, "1.weight")); - vision_model.mm_model_block_2_block_0_1_b = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_BLOCK, 2, 0, "1.bias")); + vision_model.mm_model_block_1_block_2_0_w = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_BLOCK, 1, 2, "0.weight")); + vision_model.mm_model_block_1_block_2_1_w = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_BLOCK, 1, 2, "1.weight")); + vision_model.mm_model_block_1_block_2_1_b = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_BLOCK, 1, 2, "1.bias")); + vision_model.mm_model_block_2_block_0_0_w = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_BLOCK, 2, 0, "0.weight")); + vision_model.mm_model_block_2_block_0_1_w = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_BLOCK, 2, 0, "1.weight")); + vision_model.mm_model_block_2_block_0_1_b = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_BLOCK, 2, 0, "1.bias")); vision_model.mm_model_block_2_block_1_fc1_w = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_BLOCK, 2, 1, "fc1.weight")); vision_model.mm_model_block_2_block_1_fc1_b = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_BLOCK, 2, 1, "fc1.bias")); vision_model.mm_model_block_2_block_1_fc2_w = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_BLOCK, 2, 1, "fc2.weight")); vision_model.mm_model_block_2_block_1_fc2_b = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_BLOCK, 2, 1, "fc2.bias")); - vision_model.mm_model_block_2_block_2_0_w = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_BLOCK, 2, 2, "0.weight")); - vision_model.mm_model_block_2_block_2_1_w = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_BLOCK, 2, 2, "1.weight")); - vision_model.mm_model_block_2_block_2_1_b = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_BLOCK, 2, 2, "1.bias")); - } - else { + vision_model.mm_model_block_2_block_2_0_w = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_BLOCK, 2, 2, "0.weight")); + vision_model.mm_model_block_2_block_2_1_w = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_BLOCK, 2, 2, "1.weight")); + vision_model.mm_model_block_2_block_2_1_b = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_BLOCK, 2, 2, "1.bias")); + } else { std::string proj_type = PROJECTOR_TYPE_NAMES[new_clip->proj_type]; throw std::runtime_error(format("%s: don't support projector with: %s currently\n", __func__, proj_type.c_str())); } vision_model.layers.resize(hparams.n_layer); + for (int il = 0; il < hparams.n_layer; ++il) { auto & layer = vision_model.layers[il]; layer.k_w = get_tensor(new_clip->ctx_data, format(TN_ATTN_K, "v", il, "weight")); @@ -1084,24 +1264,255 @@ bool clip_image_load_from_bytes(const unsigned char * bytes, size_t bytes_length return true; } -// normalize: x = (x - mean) / std -// TODO: implement bicubic interpolation instead of linear. -bool clip_image_preprocess(struct clip_ctx * ctx, const clip_image_u8 * img, clip_image_f32 * res, const bool pad2square) { +// Linear interpolation between two points +inline float lerp(float s, float e, float t) { + return s + (e - s) * t; +} +// Bilinear resize function +static void bilinear_resize(const clip_image_u8& src, clip_image_u8& dst, int target_width, int target_height) { + dst.nx = target_width; + dst.ny = target_height; + dst.buf.resize(3 * target_width * target_height); + + float x_ratio = static_cast(src.nx - 1) / target_width; + float y_ratio = static_cast(src.ny - 1) / target_height; + + for (int y = 0; y < target_height; y++) { + for (int x = 0; x < target_width; x++) { + float px = x_ratio * x; + float py = y_ratio * y; + int x_floor = static_cast(px); + int y_floor = static_cast(py); + float x_lerp = px - x_floor; + float y_lerp = py - y_floor; + + for (int c = 0; c < 3; c++) { + float top = lerp( + static_cast(src.buf[3 * (y_floor * src.nx + x_floor) + c]), + static_cast(src.buf[3 * (y_floor * src.nx + (x_floor + 1)) + c]), + x_lerp + ); + float bottom = lerp( + static_cast(src.buf[3 * ((y_floor + 1) * src.nx + x_floor) + c]), + static_cast(src.buf[3 * ((y_floor + 1) * src.nx + (x_floor + 1)) + c]), + x_lerp + ); + dst.buf[3 * (y * target_width + x) + c] = static_cast(lerp(top, bottom, y_lerp)); + } + } + } +} + +// Normalize image to float32 - careful with pytorch .to(model.device, dtype=torch.float16) - this sometimes reduces precision (32>16>32), sometimes not +static void normalize_image_u8_to_f32(const clip_image_u8* src, clip_image_f32* dst, const float mean[3], const float std[3]) { + dst->nx = src->nx; + dst->ny = src->ny; + dst->buf.resize(src->buf.size()); + + for (size_t i = 0; i < src->buf.size(); ++i) { + int c = i % 3; // rgb + dst->buf[i] = (static_cast(src->buf[i]) / 255.0f - mean[c]) / std[c]; + } +} + +inline float clip(float x, float lower, float upper) { + return std::max(lower, std::min(x, upper)); +} + +static bool bicubic_resize(const clip_image_u8 &img, clip_image_u8 &dst, int target_width, int target_height) { + const int nx = img.nx; + const int ny = img.ny; + + dst.nx = target_width; + dst.ny = target_height; + dst.buf.resize(3 * target_width * target_height); + + float Cc; + float C[5]; + float d0, d2, d3, a0, a1, a2, a3; + int i, j, k, jj; + int x, y; + float dx, dy; + float tx, ty; + + tx = (float)nx / (float)target_width; + ty = (float)ny / (float)target_height; + + // Bicubic interpolation; adapted from ViT.cpp, inspired from : + // -> https://github.com/yglukhov/bicubic-interpolation-image-processing/blob/master/libimage.c#L36 + // -> https://en.wikipedia.org/wiki/Bicubic_interpolation + + for (i = 0; i < target_height; i++) { + for (j = 0; j < target_width; j++) { + x = (int)(tx * j); + y = (int)(ty * i); + + dx = tx * j - x; + dy = ty * i - y; + + for (k = 0; k < 3; k++) { + for (jj = 0; jj <= 3; jj++) { + d0 = img.buf[(clip(y - 1 + jj, 0, ny - 1) * nx + clip(x - 1, 0, nx - 1)) * 3 + k] - img.buf[(clip(y - 1 + jj, 0, ny - 1) * nx + clip(x, 0, nx - 1)) * 3 + k]; + d2 = img.buf[(clip(y - 1 + jj, 0, ny - 1) * nx + clip(x + 1, 0, nx - 1)) * 3 + k] - img.buf[(clip(y - 1 + jj, 0, ny - 1) * nx + clip(x, 0, nx - 1)) * 3 + k]; + d3 = img.buf[(clip(y - 1 + jj, 0, ny - 1) * nx + clip(x + 2, 0, nx - 1)) * 3 + k] - img.buf[(clip(y - 1 + jj, 0, ny - 1) * nx + clip(x, 0, nx - 1)) * 3 + k]; + a0 = img.buf[(clip(y - 1 + jj, 0, ny - 1) * nx + clip(x, 0, nx - 1)) * 3 + k]; + + a1 = -1.0 / 3 * d0 + d2 - 1.0 / 6 * d3; + a2 = 1.0 / 2 * d0 + 1.0 / 2 * d2; + a3 = -1.0 / 6 * d0 - 1.0 / 2 * d2 + 1.0 / 6 * d3; + + C[jj] = a0 + a1 * dx + a2 * dx * dx + a3 * dx * dx * dx; + + d0 = C[0] - C[1]; + d2 = C[2] - C[1]; + d3 = C[3] - C[1]; + a0 = C[1]; + a1 = -1.0 / 3 * d0 + d2 - 1.0 / 6 * d3; + a2 = 1.0 / 2 * d0 + 1.0 / 2 * d2; + a3 = -1.0 / 6 * d0 - 1.0 / 2 * d2 + 1.0 / 6 * d3; + Cc = a0 + a1 * dy + a2 * dy * dy + a3 * dy * dy * dy; + + const uint8_t Cc2 = std::min(std::max(std::round(Cc), 0.0f), 255.0f); + dst.buf[(i * target_width + j) * 3 + k] = float(Cc2); + } + } + } + } + + return true; +} + +// llava-1.6 type of resize_and_pad (black) +static void resize_and_pad_image(const clip_image_u8& image, clip_image_u8 &image_output, const std::pair& target_resolution) { + int target_width = target_resolution.first; + int target_height = target_resolution.second; + + float scale_w = static_cast(target_width) / image.nx; + float scale_h = static_cast(target_height) / image.ny; + + int new_width, new_height; + + if (scale_w < scale_h) { + new_width = target_width; + new_height = std::min(static_cast(std::ceil(image.ny * scale_w)), target_height); + } else { + new_height = target_height; + new_width = std::min(static_cast(std::ceil(image.nx * scale_h)), target_width); + } + + clip_image_u8 resized_image; + // bilinear_resize(image, resized_image, new_width, new_height); + bicubic_resize(image, resized_image, new_width, new_height); + + clip_image_u8 padded_image; + padded_image.nx = target_width; + padded_image.ny = target_height; + padded_image.buf.resize(3 * target_width * target_height, 0); // Initialize with black + + // Calculate padding offsets + int pad_x = (target_width - new_width) / 2; + int pad_y = (target_height - new_height) / 2; + + // Copy the resized image into the center of the padded buffer + for (int y = 0; y < new_height; ++y) { + for (int x = 0; x < new_width; ++x) { + for (int c = 0; c < 3; ++c) { + padded_image.buf[3 * ((y + pad_y) * target_width + (x + pad_x)) + c] = resized_image.buf[3 * (y * new_width + x) + c]; + } + } + } + image_output = std::move(padded_image); +} + +/** + * Selects the best resolution from a list of possible resolutions based on the original size. + * + * @param original_size The original size of the image in the format (width, height). + * @param possible_resolutions A list of possible resolutions in the format [(width1, height1), (width2, height2), ...]. + * @return The best fit resolution in the format (width, height). + */ +static std::pair select_best_resolution(const std::pair & original_size, const std::vector> & possible_resolutions) { + int original_width = original_size.first; + int original_height = original_size.second; + std::pair best_fit; + int max_effective_resolution = 0; + int min_wasted_resolution = std::numeric_limits::max(); + + for (const auto& resolution : possible_resolutions) { + int width = resolution.first; + int height = resolution.second; + float scale = std::min(static_cast(width) / original_width, static_cast(height) / original_height); + int downscaled_width = static_cast(original_width * scale); + int downscaled_height = static_cast(original_height * scale); + int effective_resolution = std::min(downscaled_width * downscaled_height, original_width * original_height); + int wasted_resolution = (width * height) - effective_resolution; + // fprintf(stderr, "resolution: %d %d, scale: %f, downscaled: %d %d, effective: %d, wasted: %d\n", width, height, scale, downscaled_width, downscaled_height, effective_resolution, wasted_resolution); + if (effective_resolution > max_effective_resolution || (effective_resolution == max_effective_resolution && wasted_resolution < min_wasted_resolution)) { + max_effective_resolution = effective_resolution; + min_wasted_resolution = wasted_resolution; + best_fit = resolution; + } + } + + return best_fit; +} + +static std::vector divide_to_patches_u8(const clip_image_u8 & image, int patch_size) { + std::vector patches; + int width = image.nx; + int height = image.ny; + for (int i = 0; i < height; i += patch_size) { + for (int j = 0; j < width; j += patch_size) { + clip_image_u8 *patch = clip_image_u8_init(); + patch->nx = std::min(patch_size, width - j); + patch->ny = std::min(patch_size, height - i); + patch->buf.resize(3 * patch->nx * patch->ny); + for (int y = 0; y < patch->ny; ++y) { + for (int x = 0; x < patch->nx; ++x) { + for (int c = 0; c < 3; ++c) { + patch->buf[3 * (y * patch->nx + x) + c] = image.buf[3 * ((i + y) * width + (j + x)) + c]; + } + } + } + patches.push_back(patch); + } + } + return patches; +} + +// returns the normalized float tensor for llava-1.5, for spatial_unpad with anyres processing for llava-1.6 it returns the normalized image patch tensors as a vector +// res_imgs memory is being allocated here, previous allocations will be freed if found +bool clip_image_preprocess(struct clip_ctx * ctx, const clip_image_u8 * img, clip_image_f32_batch & res_imgs) { + bool pad_to_square = true; if (!ctx->has_vision_encoder) { printf("This gguf file seems to have no vision encoder\n"); return false; } + auto & params = ctx->vision_model.hparams; + // The model config actually contains all we need to decide on how to preprocess, here we automatically switch to the new llava-1.6 preprocessing + if (strcmp(params.mm_patch_merge_type, "spatial_unpad") == 0) { + pad_to_square = false; + } + // free the previous res_imgs if any set + if (res_imgs.size > 0 && res_imgs.size < 100) { + for (size_t i = 0; i < res_imgs.size; i++) { + clip_image_f32_free(&(res_imgs.data[i])); + } + delete[] res_imgs.data; + } + res_imgs.data = nullptr; + res_imgs.size = 0; // the logic below is to pad the shorter side to the longer side with a background color: rgb(122, 116, 104) // see https://github.com/haotian-liu/LLaVA/blob/e854a2bf85118c504f6f16bf5c3c7c92f8fa8c6b/llava/conversation.py#L113-L156 clip_image_u8 * temp = clip_image_u8_init(); // we will keep the input image data here temporarily - if (pad2square && img->nx != img->ny) { + if (pad_to_square && img->nx != img->ny) { int longer_side = std::max(img->nx, img->ny); temp->nx = longer_side; temp->ny = longer_side; temp->buf.resize(3 * longer_side * longer_side); - const uint8_t bc[3] = {122, 116, 104}; // background color in RGB from LLaVA + const uint8_t bc[3] = {122, 116, 104}; // background color in RGB from LLaVA (this is the mean rgb color * 255) // fill with background color for (size_t i = 0; i < temp->buf.size(); i++) { @@ -1119,18 +1530,63 @@ bool clip_image_preprocess(struct clip_ctx * ctx, const clip_image_u8 * img, cli } } } else { - temp->nx = img->nx; - temp->ny = img->ny; - temp->buf.resize(img->buf.size()); - memcpy(temp->buf.data(), img->buf.data(), temp->buf.size()); + if (params.image_grid_pinpoints[0] != 0) { + // "spatial_unpad" with "anyres" processing for llava-1.6 + std::vector> possible_resolutions; + for (int i = 0; i < 32 && params.image_grid_pinpoints[i] != 0; i+=2) { + possible_resolutions.push_back({params.image_grid_pinpoints[i], params.image_grid_pinpoints[i+1]}); + } + std::pair best_resolution = select_best_resolution({img->nx, img->ny}, possible_resolutions); + // clip_image_save_to_bmp(*img, "input.bmp"); + resize_and_pad_image(*img, *temp, best_resolution); // we do not pad with mean-bg color anymore in llava-1.6 + // clip_image_save_to_bmp(*temp, "resized.bmp"); + // visually verify normalized image: + // normalize_image_u8_to_f32(*temp, *res, ctx->image_mean, ctx->image_std); + // { + // clip_image_u8 * temp2 = clip_image_u8_init(); + // clip_image_convert_f32_to_u8(*res, *temp2); + // clip_image_save_to_bmp(*temp2, "resized_normalized_f32.bmp"); + // clip_image_u8_free(temp2); + // } + + std::vector patches = divide_to_patches_u8(*temp, params.image_size); // prepare spatial sorted main patches of image_size each (336 in llava-1.6) + + clip_image_u8 *image_original_resize = clip_image_u8_init(); + // bilinear_resize(*img, *image_original_resize, params.image_size, params.image_size); // in python this is "shortest_edge", but all CLIP are square + bicubic_resize(*img, *image_original_resize, params.image_size, params.image_size); // in python this is "shortest_edge", but all CLIP are square + patches.insert(patches.begin(), image_original_resize); + // clip_image_f32_batch_init(patches.size()); + res_imgs.size = patches.size(); + res_imgs.data = new clip_image_f32[res_imgs.size]; + int num=0; + for (auto& patch : patches) { + normalize_image_u8_to_f32(patch, &res_imgs.data[num], ctx->image_mean, ctx->image_std); + num++; + } + + for (size_t i = 0; i < patches.size(); i++) { + // printf("patch %d: %d %d\n", i, patches[i]->nx, patches[i]->ny); + clip_image_u8_free(patches[i]); + } + + clip_image_u8_free(temp); + + return true; + } else { + temp->nx = img->nx; + temp->ny = img->ny; + temp->buf.resize(img->buf.size()); + memcpy(temp->buf.data(), img->buf.data(), temp->buf.size()); + } } const int nx = temp->nx; const int ny = temp->ny; + // clip_image_save_to_bmp(*temp, "resized_vanilla.bmp"); const int nx2 = ctx->vision_model.hparams.image_size; const int ny2 = ctx->vision_model.hparams.image_size; - + clip_image_f32 * res = clip_image_f32_init(); res->nx = nx2; res->ny = ny2; res->buf.resize(3 * nx2 * ny2); @@ -1184,9 +1640,25 @@ bool clip_image_preprocess(struct clip_ctx * ctx, const clip_image_u8 * img, cli } clip_image_u8_free(temp); + // { + // clip_image_u8 * temp2 = clip_image_u8_init(); + // clip_image_convert_f32_to_u8(*res, *temp2); + // clip_image_save_to_bmp(*temp2, "resized_normalized_f32_vanilla.bmp"); + // clip_image_u8_free(temp2); + // } + // res_imgs.push_back(res); + + res_imgs.size = 1; + res_imgs.data = new clip_image_f32[res_imgs.size]; + res_imgs.data[0] = std::move(*res); + return true; } +ggml_tensor * clip_get_newline_tensor(const struct clip_ctx * ctx) { + return ctx->vision_model.image_newline; +} + void clip_free(clip_ctx * ctx) { ggml_free(ctx->ctx_data); gguf_free(ctx->ctx_gguf); @@ -1194,6 +1666,42 @@ void clip_free(clip_ctx * ctx) { delete ctx; } +size_t clip_embd_nbytes(const struct clip_ctx * ctx) { + return clip_n_patches(ctx) * clip_n_mmproj_embd(ctx) * sizeof(float); +} + +int32_t clip_image_size(const struct clip_ctx * ctx) { + return ctx->vision_model.hparams.image_size; +} + +int32_t clip_patch_size(const struct clip_ctx * ctx) { + return ctx->vision_model.hparams.patch_size; +} + +int32_t clip_hidden_size(const struct clip_ctx * ctx) { + return ctx->vision_model.hparams.hidden_size; +} + +const char * clip_patch_merge_type(const struct clip_ctx * ctx) { + return ctx->vision_model.hparams.mm_patch_merge_type; +} + +const int32_t * clip_image_grid(const struct clip_ctx * ctx) { + return ctx->vision_model.hparams.image_grid_pinpoints; +} + +int clip_n_patches(const struct clip_ctx * ctx) { + const auto & params = ctx->vision_model.hparams; + + int n_patches = (params.image_size / params.patch_size) * (params.image_size / params.patch_size); + + if (ctx->proj_type == PROJECTOR_TYPE_LDP) { + n_patches /= 4; + } + + return n_patches; +} + bool clip_image_encode(struct clip_ctx * ctx, const int n_threads, clip_image_f32 * img, float * vec) { if (!ctx->has_vision_encoder) { printf("This gguf file seems to have no vision encoder\n"); @@ -1213,7 +1721,7 @@ bool clip_image_batch_encode(clip_ctx * ctx, const int n_threads, const clip_ima } int batch_size = imgs->size; - if(ctx->has_llava_projector) { + if (ctx->has_llava_projector) { GGML_ASSERT(batch_size == 1); // TODO: support multiple images } @@ -1224,9 +1732,10 @@ bool clip_image_batch_encode(clip_ctx * ctx, const int n_threads, const clip_ima // set inputs const auto & model = ctx->vision_model; const auto & hparams = model.hparams; - const int image_size = hparams.image_size; - const int patch_size = hparams.patch_size; - const int num_patches = ((image_size / patch_size) * (image_size / patch_size)); + + const int image_size = hparams.image_size; + const int patch_size = hparams.patch_size; + const int num_patches = ((image_size / patch_size) * (image_size / patch_size)); const int num_positions = num_patches + 1; { @@ -1301,11 +1810,11 @@ bool clip_image_batch_encode(clip_ctx * ctx, const int n_threads, const clip_ima // copy the embeddings to the location passed by the user ggml_backend_tensor_get(embeddings, vec, 0, ggml_nbytes(embeddings)); + return true; } bool clip_model_quantize(const char * fname_inp, const char * fname_out, const int itype) { - ggml_type type = GGML_TYPE_Q4_1; assert(itype < GGML_TYPE_COUNT); @@ -1494,26 +2003,13 @@ int clip_n_mmproj_embd(const struct clip_ctx * ctx) { if (ctx->proj_type == PROJECTOR_TYPE_LDP) { return ctx->vision_model.mm_model_block_1_block_2_1_b->ne[0]; } - else if (ctx->proj_type == PROJECTOR_TYPE_MLP) { + if (ctx->proj_type == PROJECTOR_TYPE_MLP) { return ctx->vision_model.mm_2_b->ne[0]; - } else if (ctx->proj_type == PROJECTOR_TYPE_MLP_NORM) { + } + if (ctx->proj_type == PROJECTOR_TYPE_MLP_NORM) { return ctx->vision_model.mm_3_b->ne[0]; } - else { - std::string proj_type = PROJECTOR_TYPE_NAMES[ctx->proj_type]; - throw std::runtime_error(format("%s: don't support projector with: %s currently\n", __func__, proj_type.c_str())); - } -} -int clip_n_patches(const struct clip_ctx * ctx) { - auto & params = ctx->vision_model.hparams; - int n_patches = (params.image_size / params.patch_size) * (params.image_size / params.patch_size); - if (ctx->proj_type == PROJECTOR_TYPE_LDP) { - n_patches /= 4; - } - return n_patches; -} - -size_t clip_embd_nbytes(const struct clip_ctx * ctx) { - return clip_n_patches(ctx) * clip_n_mmproj_embd(ctx) * sizeof(float); + std::string proj_type = PROJECTOR_TYPE_NAMES[ctx->proj_type]; + throw std::runtime_error(format("%s: don't support projector with: %s currently\n", __func__, proj_type.c_str())); } diff --git a/examples/llava/clip.h b/examples/llava/clip.h index 458a256a1..cd9a4022f 100644 --- a/examples/llava/clip.h +++ b/examples/llava/clip.h @@ -24,25 +24,7 @@ struct clip_ctx; extern "C" { #endif -struct clip_vision_hparams { - int32_t image_size; - int32_t patch_size; - int32_t hidden_size; - int32_t n_intermediate; - int32_t projection_dim; - int32_t n_head; - int32_t n_layer; - float eps; -}; - -CLIP_API struct clip_ctx * clip_model_load(const char * fname, int verbosity); - -CLIP_API void clip_free(struct clip_ctx * ctx); - -CLIP_API size_t clip_embd_nbytes(const struct clip_ctx * ctx); - -CLIP_API int clip_n_patches (const struct clip_ctx * ctx); -CLIP_API int clip_n_mmproj_embd(const struct clip_ctx * ctx); +struct clip_ctx; struct clip_image_u8_batch { struct clip_image_u8 * data; @@ -54,10 +36,29 @@ struct clip_image_f32_batch { size_t size; }; +CLIP_API struct clip_ctx * clip_model_load (const char * fname, int verbosity); +CLIP_API struct clip_ctx * clip_model_load_cpu(const char * fname, int verbosity); + +CLIP_API void clip_free(struct clip_ctx * ctx); + +CLIP_API size_t clip_embd_nbytes(const struct clip_ctx * ctx); + +CLIP_API int32_t clip_image_size (const struct clip_ctx * ctx); +CLIP_API int32_t clip_patch_size (const struct clip_ctx * ctx); +CLIP_API int32_t clip_hidden_size(const struct clip_ctx * ctx); + +// TODO: should be enum, not string +CLIP_API const char * clip_patch_merge_type(const struct clip_ctx * ctx); + +CLIP_API const int32_t * clip_image_grid(const struct clip_ctx * ctx); + +CLIP_API int clip_n_patches (const struct clip_ctx * ctx); +CLIP_API int clip_n_mmproj_embd(const struct clip_ctx * ctx); + CLIP_API struct clip_image_u8 * clip_image_u8_init (); CLIP_API struct clip_image_f32 * clip_image_f32_init(); -CLIP_API void clip_image_u8_free (struct clip_image_u8 * img); +CLIP_API void clip_image_u8_free (struct clip_image_u8 * img); CLIP_API void clip_image_f32_free(struct clip_image_f32 * img); CLIP_API bool clip_image_load_from_file(const char * fname, struct clip_image_u8 * img); @@ -65,7 +66,11 @@ CLIP_API bool clip_image_load_from_file(const char * fname, struct clip_image_u8 /** interpret bytes as an image file with length bytes_length, and use the result to populate img */ CLIP_API bool clip_image_load_from_bytes(const unsigned char * bytes, size_t bytes_length, struct clip_image_u8 * img); -CLIP_API bool clip_image_preprocess (struct clip_ctx * ctx, const struct clip_image_u8 * img, struct clip_image_f32 * res, bool pad2square); +/** preprocess img and store the result in res_imgs, pad_to_square may be overriden to false depending on model configuration */ +CLIP_API bool clip_image_preprocess(struct clip_ctx * ctx, const clip_image_u8 * img, clip_image_f32_batch & res_imgs ); + +CLIP_API struct ggml_tensor * clip_get_newline_tensor(const struct clip_ctx * ctx); + CLIP_API bool clip_image_encode (struct clip_ctx * ctx, int n_threads, struct clip_image_f32 * img, float * vec); CLIP_API bool clip_image_batch_encode(struct clip_ctx * ctx, int n_threads, const struct clip_image_f32_batch * imgs, float * vec); diff --git a/examples/llava/convert-image-encoder-to-gguf.py b/examples/llava/convert-image-encoder-to-gguf.py index e204b56be..c69f89ac2 100644 --- a/examples/llava/convert-image-encoder-to-gguf.py +++ b/examples/llava/convert-image-encoder-to-gguf.py @@ -78,18 +78,19 @@ ap.add_argument("--text-only", action="store_true", required=False, help="Save a text-only model. It can't be used to encode images") ap.add_argument("--vision-only", action="store_true", required=False, help="Save a vision-only model. It can't be used to encode texts") -ap.add_argument("--clip_model_is_vision", action="store_true", required=False, +ap.add_argument("--clip-model-is-vision", action="store_true", required=False, help="The clip model is a pure vision model (ShareGPT4V vision extract for example)") +ap.add_argument("--clip-model-is-openclip", action="store_true", required=False, + help="The clip model is from openclip (for ViT-SO400M type))") ap.add_argument("--llava-projector", help="Path to llava.projector file. If specified, save an image encoder for LLaVA models.") ap.add_argument("--projector-type", help="Type of projector. Possible values: mlp, ldp", choices=["mlp", "ldp"], default="mlp") -ap.add_argument("--image-mean", nargs=3, type=float, required=False, help="Override image mean values") -ap.add_argument("--image-std", nargs=3, type=float, required=False, help="Override image std values") ap.add_argument("-o", "--output-dir", help="Directory to save GGUF files. Default is the original model directory", default=None) # Example --image_mean 0.48145466 0.4578275 0.40821073 --image_std 0.26862954 0.26130258 0.27577711 +# Example --image_mean 0.5 0.5 0.5 --image_std 0.5 0.5 0.5 default_image_mean = [0.48145466, 0.4578275, 0.40821073] default_image_std = [0.26862954, 0.26130258, 0.27577711] -ap.add_argument('--image_mean', type=float, nargs='+', help='Mean of the images for normalization (overrides processor) ', default=None) -ap.add_argument('--image_std', type=float, nargs='+', help='Standard deviation of the images for normalization (overrides processor)', default=None) +ap.add_argument('--image-mean', type=float, nargs='+', help='Mean of the images for normalization (overrides processor) ', default=None) +ap.add_argument('--image-std', type=float, nargs='+', help='Standard deviation of the images for normalization (overrides processor)', default=None) # with proper args = ap.parse_args() @@ -105,7 +106,7 @@ if args.use_f32: # output in the same directory as the model if output_dir is None dir_model = args.model_dir -if args.clip_model_is_vision: +if args.clip_model_is_vision or not os.path.exists(dir_model + "/vocab.json") or args.clip_model_is_openclip: vocab = None tokens = None else: @@ -133,7 +134,7 @@ ftype = 1 if args.use_f32: ftype = 0 -if args.clip_model_is_vision: +if args.clip_model_is_vision or args.clip_model_is_openclip: model = CLIPVisionModel.from_pretrained(dir_model) processor = None else: @@ -202,6 +203,57 @@ if has_vision_encoder: fout.add_float32(k(KEY_ATTENTION_LAYERNORM_EPS, VISION), v_hparams["layer_norm_eps"]) block_count = v_hparams["num_hidden_layers"] - 1 if has_llava_projector else v_hparams["num_hidden_layers"] fout.add_uint32(k(KEY_BLOCK_COUNT, VISION), block_count) + # /** + # "image_grid_pinpoints": [ + # [ + # 336, + # 672 + # ], + # [ + # 672, + # 336 + # ], + # [ + # 672, + # 672 + # ], + # [ + # 1008, + # 336 + # ], + # [ + # 336, + # 1008 + # ] + # ], + # Flattened: + # [ + # 336, 672, + # 672, 336, + # 672, 672, + # 1008, 336, + # 336, 1008 + # ] + # * + # */ + if "image_grid_pinpoints" in v_hparams: + # flatten it + image_grid_pinpoints = [] + for pinpoint in v_hparams["image_grid_pinpoints"]: + for p in pinpoint: + image_grid_pinpoints.append(p) + fout.add_array("clip.vision.image_grid_pinpoints", image_grid_pinpoints) + if "image_crop_resolution" in v_hparams: + fout.add_uint32("clip.vision.image_crop_resolution", v_hparams["image_crop_resolution"]) + if "image_aspect_ratio" in v_hparams: + fout.add_string("clip.vision.image_aspect_ratio", v_hparams["image_aspect_ratio"]) + if "image_split_resolution" in v_hparams: + fout.add_uint32("clip.vision.image_split_resolution", v_hparams["image_split_resolution"]) + if "mm_patch_merge_type" in v_hparams: + fout.add_string("clip.vision.mm_patch_merge_type", v_hparams["mm_patch_merge_type"]) + if "mm_projector_type" in v_hparams: + fout.add_string("clip.vision.mm_projector_type", v_hparams["mm_projector_type"]) + if processor is not None: image_mean = processor.image_processor.image_mean if args.image_mean is None or args.image_mean == default_image_mean else args.image_mean diff --git a/examples/llava/llava-cli.cpp b/examples/llava/llava-cli.cpp index 031e9806d..bef7f7c95 100644 --- a/examples/llava/llava-cli.cpp +++ b/examples/llava/llava-cli.cpp @@ -155,11 +155,29 @@ static void process_prompt(struct llava_context * ctx_llava, struct llava_image_ system_prompt = prompt.substr(0, image_pos); user_prompt = prompt.substr(image_pos + std::string("").length()); printf("system_prompt: %s\n", system_prompt.c_str()); + if (params->verbose_prompt) { + auto tmp = ::llama_tokenize(ctx_llava->ctx_llama, system_prompt, true, true); + for (int i = 0; i < (int) tmp.size(); i++) { + printf("%6d -> '%s'\n", tmp[i], llama_token_to_piece(ctx_llava->ctx_llama, tmp[i]).c_str()); + } + } printf("user_prompt: %s\n", user_prompt.c_str()); + if (params->verbose_prompt) { + auto tmp = ::llama_tokenize(ctx_llava->ctx_llama, user_prompt, true, true); + for (int i = 0; i < (int) tmp.size(); i++) { + printf("%6d -> '%s'\n", tmp[i], llama_token_to_piece(ctx_llava->ctx_llama, tmp[i]).c_str()); + } + } } else { // llava-1.5 native mode system_prompt = "A chat between a curious human and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the human's questions.\nUSER:"; user_prompt = prompt + "\nASSISTANT:"; + if (params->verbose_prompt) { + auto tmp = ::llama_tokenize(ctx_llava->ctx_llama, user_prompt, true, true); + for (int i = 0; i < (int) tmp.size(); i++) { + printf("%6d -> '%s'\n", tmp[i], llama_token_to_piece(ctx_llava->ctx_llama, tmp[i]).c_str()); + } + } } eval_string(ctx_llava->ctx_llama, system_prompt.c_str(), params->n_batch, &n_past, add_bos); @@ -171,13 +189,17 @@ static void process_prompt(struct llava_context * ctx_llava, struct llava_image_ fprintf(stderr, "\n"); struct llama_sampling_context * ctx_sampling = llama_sampling_init(params->sparams); - + std::string response = ""; for (int i = 0; i < max_tgt_len; i++) { const char * tmp = sample(ctx_sampling, ctx_llava->ctx_llama, &n_past); + response += tmp; if (strcmp(tmp, "") == 0) break; if (strstr(tmp, "###")) break; // Yi-VL behavior - printf("%s", tmp); + if (strstr(response.c_str(), "<|im_end|>")) break; // Yi-34B llava-1.6 - for some reason those decode not as the correct token (tokenizer works) + if (strstr(response.c_str(), "<|im_start|>")) break; // Yi-34B llava-1.6 + if (strstr(response.c_str(), "USER:")) break; // mistral llava-1.6 + fflush(stdout); } diff --git a/examples/llava/llava-surgery-v2.py b/examples/llava/llava-surgery-v2.py new file mode 100644 index 000000000..5bc5bc513 --- /dev/null +++ b/examples/llava/llava-surgery-v2.py @@ -0,0 +1,167 @@ +import argparse +import glob +import os +import torch +from safetensors.torch import load as safe_load, save as safe_save, safe_open, save_file + +# Function to determine if file is a SafeTensor file +def is_safetensor_file(file_path): + return file_path.endswith('.safetensors') + + +# Unified loading function +def load_model(file_path): + if is_safetensor_file(file_path): + tensors = {} + with safe_open(file_path, framework="pt", device="cpu") as f: + for key in f.keys(): + tensors[key] = f.get_tensor(key).clone() + # output shape + print(f"{key} : {tensors[key].shape}") + return tensors, 'safetensor' + else: + return torch.load(file_path, map_location=torch.device('cpu')), 'pytorch' + + +# Unified saving function +def save_model(model, file_path, file_type): + if file_type == 'safetensor': + # safe_save(model, file_path) + save_file(model, file_path) + else: + torch.save(model, file_path) + + +# Adapted function to clean vision tower from checkpoint +def clean_vision_tower_from_checkpoint(checkpoint_path): + checkpoint, file_type = load_model(checkpoint_path) + # file_type = 'pytorch' + model_path = os.path.dirname(checkpoint_path) + print(f"Searching for vision tower tensors in {checkpoint_path}") + clip_tensors = [k for k, v in checkpoint.items() if (k.startswith("model.vision_tower") or k.startswith("vit."))] + + if len(clip_tensors) > 0: + print(f"Found {len(clip_tensors)} tensors to extract from {checkpoint_path}") + # Adapted for file type + clip_path = os.path.join(model_path, "llava.clip") + + if os.path.exists(clip_path): + print(f"Loading existing llava.clip from {clip_path}") + existing_clip, _ = load_model(clip_path) + else: + print(f"Creating new llava.clip at {clip_path}") + existing_clip = {} + # Update existing_clip with new tensors, avoid duplicates + for name in clip_tensors: + simple_name = name[name.index('vision_model.'):] if 'vision_model.' in name else name + print(f"Adding {simple_name} to llava.clip") + if simple_name not in existing_clip: + existing_clip[simple_name] = checkpoint[name] + + # Save the updated clip tensors back to llava.clip + save_model(existing_clip, clip_path, 'pytorch') + + # Remove the tensors from the original checkpoint + for name in clip_tensors: + del checkpoint[name] + + # Save the updated checkpoint + checkpoint_path = checkpoint_path + save_model(checkpoint, checkpoint_path, file_type) + return True + return False + +def find_relevant_checkpoints(checkpoint_paths, newline_criteria, projector): + newline_checkpoint_path = None + projector_checkpoint_path = None + + for path in checkpoint_paths: + checkpoint, _ = load_model(path) + if newline_criteria(checkpoint) and newline_checkpoint_path is None: + newline_checkpoint_path = path + if projector(checkpoint): + projector_checkpoint_path = path + + return newline_checkpoint_path, projector_checkpoint_path + +def newline_criteria(checkpoint): + return any(k.startswith("model.image_newline") for k in checkpoint.keys()) + +def proj_criteria(checkpoint): + return any(k.startswith("model.mm_projector") or k.startswith("vision_proj.") for k in checkpoint.keys()) + + +# Command-line interface setup +ap = argparse.ArgumentParser() +ap.add_argument("-m", "--model", required=True, help="Path to LLaVA v1.5+ model") +ap.add_argument("-C", "--clean-vision-tower", action="store_true", help="Remove any vision tower from the model files") +args = ap.parse_args() + +if args.clean_vision_tower: + # Generalized to handle both PyTorch and SafeTensors models + model_files = sorted(glob.glob(f"{args.model}/*"), key=os.path.getmtime, reverse=True) + # checkpoint_paths = [path for path in model_files if (path.endswith('.bin') and path.startswith('pytorch')) or (path.endswith('.safetensors') and path.startswith('model'))] + checkpoint_paths = [path for path in model_files if (path.endswith('.bin') and 'pytorch' in path.split('/')[-1].split('\\')[-1]) or (path.endswith('.safetensors') and 'model' in path.split('/')[-1].split('\\')[-1])] + for projector_checkpoint_path in checkpoint_paths: + print(f"Cleaning {projector_checkpoint_path}") + if not clean_vision_tower_from_checkpoint(projector_checkpoint_path): + print(f"No vision tower found in {projector_checkpoint_path}") + # we break once none is found, so far all models append them at the end + # break + print("Done! All vision tower tensors are removed from the model files and stored in llava.clip file.") + +# Now we look for the projector in the last checkpoint +model_files = sorted(glob.glob(f"{args.model}/*"), key=os.path.getmtime, reverse=True) +checkpoint_paths = [path for path in model_files if (path.endswith('.bin') and 'pytorch' in path.split('/')[-1].split('\\')[-1]) or (path.endswith('.safetensors') and 'model' in path.split('/')[-1].split('\\')[-1])] +# last_checkpoint_path = checkpoint_paths[0] +# first_checkpoint_path = checkpoint_paths[-1] +newline_checkpoint_path, projector_checkpoint_path = find_relevant_checkpoints(checkpoint_paths, newline_criteria, proj_criteria) + +print(f"Taking projector from {projector_checkpoint_path}") +first_mm_tensors = [] +first_checkpoint = None +if newline_checkpoint_path is not None: + print(f"Taking newline from {newline_checkpoint_path}") + first_checkpoint, file_type = load_model(newline_checkpoint_path) + first_mm_tensors = [k for k, v in first_checkpoint.items() if k.startswith("model.image_newline")] + +# Load the checkpoint +mm_tensors = [] +last_checkpoint = None +if projector_checkpoint_path is not None: + last_checkpoint, file_type = load_model(projector_checkpoint_path) + mm_tensors = [k for k, v in last_checkpoint.items() if k.startswith("model.mm_projector") or k.startswith("vision_proj.")] + +if len(mm_tensors) == 0: + if last_checkpoint is not None: + for k, v in last_checkpoint.items(): + print(k) + print(f"Found {len(mm_tensors)} tensors to extract out of {len(last_checkpoint)} tensors.") + print("No tensors found. Is this a LLaVA model?") + exit() + +print(f"Found {len(mm_tensors)} tensors to extract.") +print(f"Found additional {len(first_mm_tensors)} tensors to extract.") +# projector = {name: checkpoint.[name].float() for name in mm_tensors} +projector = {} +for name in mm_tensors: + projector[name] = last_checkpoint[name].float() +for name in first_mm_tensors: + projector[name] = first_checkpoint[name].float() + +if len(projector) > 0: + save_model(projector, f"{args.model}/llava.projector", 'pytorch') + +for name in mm_tensors: + del last_checkpoint[name] +for name in first_mm_tensors: + del first_checkpoint[name] + +if len(mm_tensors) > 0: + save_model(last_checkpoint, projector_checkpoint_path, file_type) +if len(first_mm_tensors) > 0: + save_model(first_checkpoint, newline_checkpoint_path, file_type) + +print("Done!") +print(f"Now you can convert {args.model} to a a regular LLaMA GGUF file.") +print(f"Also, use {args.model}/llava.projector to prepare a llava-encoder.gguf file.") diff --git a/examples/llava/llava.cpp b/examples/llava/llava.cpp index d42e7582e..22953417f 100644 --- a/examples/llava/llava.cpp +++ b/examples/llava/llava.cpp @@ -2,32 +2,296 @@ #include "common.h" #include "llama.h" #include "llava.h" +#include "base64.hpp" #include #include #include +#include + +// RGB uint8 image +struct clip_image_u8 { + int nx; + int ny; + + std::vector buf; +}; + +// RGB float32 image (NHWC) +// Memory layout: RGBRGBRGB... +struct clip_image_f32 { + int nx; + int ny; + + std::vector buf; +}; + +struct clip_image_grid_shape { + int first; + int second; +}; + +/** + * Selects the best resolution from a list of possible resolutions based on the original size. + * + * @param original_size The original size of the image in the format (width, height). + * @param possible_resolutions A list of possible resolutions in the format [(width1, height1), (width2, height2), ...]. + * @return The best fit resolution in the format (width, height). + */ +static std::pair select_best_resolution(const std::pair& original_size, const std::vector>& possible_resolutions) { + int original_width = original_size.first; + int original_height = original_size.second; + + std::pair best_fit; + int max_effective_resolution = 0; + int min_wasted_resolution = std::numeric_limits::max(); + + for (const auto& resolution : possible_resolutions) { + int width = resolution.first; + int height = resolution.second; + float scale = std::min(static_cast(width) / original_width, static_cast(height) / original_height); + int downscaled_width = static_cast(original_width * scale); + int downscaled_height = static_cast(original_height * scale); + int effective_resolution = std::min(downscaled_width * downscaled_height, original_width * original_height); + int wasted_resolution = (width * height) - effective_resolution; + // fprintf(stderr, "resolution: %d %d, scale: %f, downscaled: %d %d, effective: %d, wasted: %d\n", width, height, scale, downscaled_width, downscaled_height, effective_resolution, wasted_resolution); + if (effective_resolution > max_effective_resolution || (effective_resolution == max_effective_resolution && wasted_resolution < min_wasted_resolution)) { + max_effective_resolution = effective_resolution; + min_wasted_resolution = wasted_resolution; + best_fit = resolution; + } + } + + return best_fit; +} + +/** + * @brief Get the anyres image grid shape object + * + * @param image_size + * @param grid_pinpoints + * @param image_patch_size + * @return + */ +static struct clip_image_grid_shape get_anyres_image_grid_shape(const std::pair & image_size, const std::vector> & grid_pinpoints, int image_patch_size) { + /** + Conversion from gguf flat array to vector: + std::vector> possible_resolutions; + for (int i = 0; i < 32 && params.image_grid_pinpoints[i] != 0; i+=2) { + possible_resolutions.push_back({params.image_grid_pinpoints[i], params.image_grid_pinpoints[i+1]}); + } + */ + auto best_resolution = select_best_resolution(image_size, grid_pinpoints); + return {best_resolution.first / image_patch_size, best_resolution.second / image_patch_size}; +} + +// Take the image segments in a grid configuration and return the embeddings and the number of embeddings into preallocated memory (image_embd_out) +static bool clip_llava_handle_patches(clip_ctx * ctx_clip, std::vector & image_embd_v, struct clip_image_grid_shape grid_shape, float * image_embd_out, int * n_img_pos_out) { + struct { + struct ggml_tensor * newline; + struct ggml_context * ctx; + } model; + + const int32_t image_size = clip_image_size(ctx_clip); + const int32_t patch_size = clip_patch_size(ctx_clip); + + int32_t num_patches_per_side = image_size / patch_size; // 336 / 14 = 24 - used for embedding-patching boxes (24*24 = 576 patches) + + int num_patches_width = grid_shape.first; // grid 1-4 + int num_patches_height = grid_shape.second; // grid 1-4 + + const size_t num_images = num_patches_width + num_patches_height + 1; + + // TODO: size calculation is not calculated - it's only tens of MB + size_t ctx_size = 0; + + { + ctx_size += clip_embd_nbytes(ctx_clip) * num_images * 8; // image_features + ctx_size += 1024*1024 * ggml_type_size(GGML_TYPE_F32); + } + + struct ggml_init_params params { + /*.mem_size =*/ ctx_size, + /*.mem_buffer =*/ NULL, + /*.no_alloc =*/ false, // NOTE: this should be false when using the legacy API + }; + + // Python reference code for full unpad: + /* + base_image_feature = image_feature[0] + image_feature = image_feature[1:] + image_feature = image_feature.permute(4, 0, 2, 1, 3).contiguous() + image_feature = image_feature.flatten(1, 2).flatten(2, 3) + image_feature = unpad_image(image_feature, image_sizes[image_idx]) + image_feature = torch.cat(( + image_feature, + self.model.image_newline[:, None, None].expand(*image_feature.shape[:-1], 1) + ), dim=-1) + image_feature = image_feature.flatten(1, 2).transpose(0, 1) + image_feature = torch.cat((base_image_feature, image_feature), dim=0) + */ + // We now have two options: unpad or no unpad. Unpad removes tokens for faster llm eval. + // In terms of result quality it appears to make no difference, so we'll start with the easier approach given 5D tensors are not supported in ggml yet. + // Without unpad we have to split the sub-image embeddings into patches of 24 features each and permute them. + // Once all images are processed to prepended the base_image_features without any changes. + + // Pytorch reference simplified, modified for ggml compatibility - confirmed identical output in python (for a 2x2 grid image (676x676 scaling)) + /* + image_feature = image_feature.view(2, 2, 24, 24, 4096) + image_feature = image_feature.permute(0, 2, 1, 3, 4).contiguous() + image_feature = image_feature.view(2, 24, 2, 24, 4096) + image_feature = image_feature.flatten(0, 3) + + // Reshape to 4D tensor by merging the last two dimensions + image_feature = image_feature.view(2, 2, 24, 24*4096) + image_feature = image_feature.permute(0, 2, 1, 3).contiguous() + image_feature = image_feature.view(-1, 4096) + */ + + model.ctx = ggml_init(params); + + ggml_tensor * newline_tmp = clip_get_newline_tensor(ctx_clip); + model.newline = ggml_new_tensor_1d(model.ctx, GGML_TYPE_F32, newline_tmp->ne[0]); + if (newline_tmp->backend != GGML_BACKEND_CPU) { + if (newline_tmp->buffer == NULL) { + printf("newline_tmp tensor buffer is NULL\n"); + } + ggml_backend_tensor_get(newline_tmp, model.newline->data, 0, ggml_nbytes(newline_tmp)); + } else { + model.newline->data = newline_tmp->data; + if (model.newline->data == NULL) { + printf("newline_tmp tensor data is NULL\n"); + } + } + + struct ggml_tensor * image_features = ggml_new_tensor_3d(model.ctx, GGML_TYPE_F32, clip_n_mmproj_embd(ctx_clip), clip_n_patches(ctx_clip), num_images - 1); // example: 4096 x 576 x 4 + // ggml_tensor_printf(image_features,"image_features",__LINE__,false,false); + // fill it with the image embeddings, ignoring the base + for (size_t i = 1; i < num_images; i++) { + size_t offset = (i-1) * clip_embd_nbytes(ctx_clip); + memcpy((uint8_t *)(image_features->data) + offset, image_embd_v[i], clip_embd_nbytes(ctx_clip)); + } + + struct ggml_cgraph * gf = ggml_new_graph(model.ctx); + size_t size_ele = ggml_type_size(GGML_TYPE_F32); + + struct ggml_tensor *image_features_patchview = ggml_view_4d(model.ctx, image_features, + num_patches_per_side * clip_n_mmproj_embd(ctx_clip), + num_patches_per_side, + num_patches_width, + num_patches_height, + size_ele * num_patches_per_side * clip_n_mmproj_embd(ctx_clip), + size_ele * num_patches_per_side * clip_n_mmproj_embd(ctx_clip) * num_patches_per_side, + size_ele * num_patches_per_side * clip_n_mmproj_embd(ctx_clip) * num_patches_per_side * num_patches_width, 0); + // ggml_tensor_printf(image_features_patchview,"image_features_patchview",__LINE__,false,false); + struct ggml_tensor *permuted_cont = ggml_cont(model.ctx, ggml_permute(model.ctx, image_features_patchview, 0, 2, 1, 3)); + /** + At the end of each row we have to add the row_end embeddings, which are the same as the newline embeddings + image_feature = torch.cat(( + image_feature, + self.model.image_newline[:, None, None].expand(*image_feature.shape[:-1], 1).to(image_feature.device) + ), dim=-1) + * + */ + + // ggml_tensor_printf(permuted_cont,"permuted_cont",__LINE__,false,false); + struct ggml_tensor *flatten = ggml_view_2d(model.ctx, permuted_cont, clip_n_mmproj_embd(ctx_clip), num_patches_height * num_patches_width * num_patches_per_side * num_patches_per_side, size_ele * clip_n_mmproj_embd(ctx_clip), 0); + // ggml_tensor_printf(flatten,"flatten",__LINE__,false,false); + ggml_build_forward_expand(gf, flatten); + ggml_graph_compute_with_ctx(model.ctx, gf, 1); + struct ggml_tensor* result = gf->nodes[gf->n_nodes - 1]; + + memcpy(image_embd_out, image_embd_v[0], clip_embd_nbytes(ctx_clip)); // main image as global context + // append without newline tokens (default behavior in llava_arch when not using unpad ): + memcpy(image_embd_out + clip_n_patches(ctx_clip) * clip_n_mmproj_embd(ctx_clip), (float*)result->data, clip_embd_nbytes(ctx_clip) * (num_images-1)); // grid patches + *n_img_pos_out = static_cast(result->ne[1]+clip_n_patches(ctx_clip)); + + // Debug: Test single segments + // Current findings: sending base image, sending a segment embedding all works similar to python + // However, permuted embeddings do not work yet (stride issue?) + // memcpy(image_embd_out, image_embd_v[0], clip_embd_nbytes(ctx_clip)); // main image as context + // memcpy(image_embd_out, (float*)prepared_cont->data, clip_embd_nbytes(ctx_clip)); // main image as context + // *n_img_pos_out=576; + + ggml_free(model.ctx); + return true; +} -#include "base64.hpp" static bool encode_image_with_clip(clip_ctx * ctx_clip, int n_threads, const clip_image_u8 * img, float * image_embd, int * n_img_pos) { - clip_image_f32 * img_res = clip_image_f32_init(); - if (!clip_image_preprocess(ctx_clip, img, img_res, /*pad2square =*/ true)) { + // std::vector img_res_v; // format VectN x H x W x RGB (N x 336 x 336 x 3), so interleaved RGB - different to the python implementation which is N x 3 x 336 x 336 + clip_image_f32_batch img_res_v; + img_res_v.size = 0; + img_res_v.data = nullptr; + if (!clip_image_preprocess(ctx_clip, img, img_res_v)) { fprintf(stderr, "%s: unable to preprocess image\n", __func__); - clip_image_f32_free(img_res); + delete[] img_res_v.data; return false; } - *n_img_pos = clip_n_patches(ctx_clip); - const int64_t t_img_enc_start_us = ggml_time_us(); - bool encoded = clip_image_encode(ctx_clip, n_threads, img_res, image_embd); - clip_image_f32_free(img_res); - if (!encoded) { - fprintf(stderr, "Unable to encode image\n"); - return false; + const char * mm_patch_merge_type = clip_patch_merge_type(ctx_clip); + + if (strcmp(mm_patch_merge_type, "spatial_unpad") != 0) { + // flat / default llava-1.5 type embedding + *n_img_pos = clip_n_patches(ctx_clip); + bool encoded = clip_image_encode(ctx_clip, n_threads, &img_res_v.data[0], image_embd); // image_embd shape is 576 x 4096 + delete[] img_res_v.data; + if (!encoded) { + fprintf(stderr, "Unable to encode image\n"); + + return false; + } + } else { + // spatial_unpad llava-1.6 type embedding + // TODO: CLIP needs batching support - in HF the llm projection is separate after encoding, which might be a solution to quickly get batching working + std::vector image_embd_v; + image_embd_v.resize(img_res_v.size); + for (size_t i = 0; i < img_res_v.size; i++) { + image_embd_v[i] = (float *)malloc(clip_embd_nbytes(ctx_clip)); // 576 patches * 4096 embeddings * 4 bytes = 9437184 + const bool encoded = clip_image_encode(ctx_clip, n_threads, &img_res_v.data[i], image_embd_v[i]); // image data is in 3x336x336 format and will be converted to 336x336x3 inside + if (!encoded) { + fprintf(stderr, "Unable to encode image - spatial_unpad - subimage %d of %d\n", (int) i+1, (int) img_res_v.size); + return false; + } + } + const int64_t t_img_enc_batch_us = ggml_time_us(); + printf("%s: %d segments encoded in %8.2f ms\n", __func__, (int)img_res_v.size, (t_img_enc_batch_us - t_img_enc_start_us) / 1000.0); + + const int32_t * image_grid = clip_image_grid(ctx_clip); + + std::vector> grid_pinpoints; + for (int i = 0; i < 32 && image_grid[i] != 0; i += 2) { + grid_pinpoints.push_back({image_grid[i], image_grid[i+1]}); + } + + // free all img_res_v - not needed anymore + delete[] img_res_v.data; + img_res_v.size = 0; + img_res_v.data = nullptr; + + const int32_t image_size = clip_image_size(ctx_clip); + + struct clip_image_grid_shape grid_shape = get_anyres_image_grid_shape({img->nx,img->ny}, grid_pinpoints, image_size); + + int n_img_pos_out; + clip_llava_handle_patches(ctx_clip, image_embd_v, grid_shape, image_embd, &n_img_pos_out); + *n_img_pos = n_img_pos_out; + + for (size_t i = 0; i < image_embd_v.size(); i++) { + free(image_embd_v[i]); + } + image_embd_v.clear(); + + // debug image/segment/normalization content: + // clip_image_u8 * tmp = clip_image_u8_init(); + // clip_image_convert_f32_to_u8(*image_feature, *tmp); + // clip_image_save_to_bmp(*tmp, "image_feature.bmp"); } + printf("%s: image embedding created: %d tokens\n", __func__, *n_img_pos); + const int64_t t_img_enc_end_us = ggml_time_us(); float t_img_enc_ms = (t_img_enc_end_us - t_img_enc_start_us) / 1000.0; @@ -48,7 +312,7 @@ bool llava_validate_embed_size(const llama_context * ctx_llama, const clip_ctx * } static bool llava_image_embed_make_with_clip_img(clip_ctx * ctx_clip, int n_threads, const clip_image_u8 * img, float ** image_embd_out, int * n_img_pos_out) { - float * image_embd = (float *)malloc(clip_embd_nbytes(ctx_clip)); + float * image_embd = (float *)malloc(clip_embd_nbytes(ctx_clip)*6); // TODO: base on gridsize/llava model if (!image_embd) { fprintf(stderr, "Unable to allocate memory for image embeddings\n"); free(image_embd); @@ -85,7 +349,7 @@ bool llava_eval_image_embed(llama_context * ctx_llama, const struct llava_image_ return true; } -LLAVA_API struct llava_image_embed * llava_image_embed_make_with_bytes(struct clip_ctx * ctx_clip, int n_threads, const unsigned char * image_bytes, int image_bytes_length) { +struct llava_image_embed * llava_image_embed_make_with_bytes(struct clip_ctx * ctx_clip, int n_threads, const unsigned char * image_bytes, int image_bytes_length) { clip_image_u8 * img = clip_image_u8_init(); if (!clip_image_load_from_bytes(image_bytes, image_bytes_length, img)) { clip_image_u8_free(img); @@ -142,7 +406,7 @@ static bool load_file_to_bytes(const char* path, unsigned char** bytesOut, long return true; } -LLAVA_API struct llava_image_embed * llava_image_embed_make_with_filename(struct clip_ctx * ctx_clip, int n_threads, const char * image_path) { +struct llava_image_embed * llava_image_embed_make_with_filename(struct clip_ctx * ctx_clip, int n_threads, const char * image_path) { unsigned char* image_bytes; long image_bytes_length; auto loaded = load_file_to_bytes(image_path, &image_bytes, &image_bytes_length); @@ -151,13 +415,13 @@ LLAVA_API struct llava_image_embed * llava_image_embed_make_with_filename(struct return NULL; } - auto embed = llava_image_embed_make_with_bytes(ctx_clip, n_threads, image_bytes, image_bytes_length); + llava_image_embed *embed = llava_image_embed_make_with_bytes(ctx_clip, n_threads, image_bytes, image_bytes_length); free(image_bytes); return embed; } -LLAVA_API void llava_image_embed_free(struct llava_image_embed * embed) { +void llava_image_embed_free(struct llava_image_embed * embed) { free(embed->embed); free(embed); } diff --git a/examples/llava/llava.h b/examples/llava/llava.h index e08ce7883..9e9466a5d 100644 --- a/examples/llava/llava.h +++ b/examples/llava/llava.h @@ -3,7 +3,6 @@ #include "ggml.h" - #ifdef LLAMA_SHARED # if defined(_WIN32) && !defined(__MINGW32__) # ifdef LLAMA_BUILD @@ -42,7 +41,6 @@ LLAVA_API void llava_image_embed_free(struct llava_image_embed * embed); /** write the image represented by embed into the llama context with batch size n_batch, starting at context pos n_past. on completion, n_past points to the next position in the context after the image embed. */ LLAVA_API bool llava_eval_image_embed(struct llama_context * ctx_llama, const struct llava_image_embed * embed, int n_batch, int * n_past); - #ifdef __cplusplus } #endif diff --git a/examples/server/server.cpp b/examples/server/server.cpp index 1699eb76b..6e3434030 100644 --- a/examples/server/server.cpp +++ b/examples/server/server.cpp @@ -968,13 +968,20 @@ struct llama_server_context { continue; } - clip_image_f32 * img_res = clip_image_f32_init(); - if (!clip_image_preprocess(clp_ctx, img.img_data, img_res, /*pad2square =*/ true)) + clip_image_f32_batch img_res_v; + img_res_v.size = 0; + img_res_v.data = nullptr; + if (!clip_image_preprocess(clp_ctx, img.img_data, img_res_v)) { LOG_TEE("Error processing the given image"); clip_free(clp_ctx); + clip_image_f32_free(img_res_v.data); return false; } + + // note: assumes only one image was returned by clip_image_preprocess + clip_image_f32 * img_res = img_res_v.data; + img.image_tokens = clip_n_patches(clp_ctx); img.image_embedding = (float *)malloc(clip_embd_nbytes(clp_ctx)); if (!img.image_embedding) @@ -989,7 +996,9 @@ struct llama_server_context LOG_TEE("Unable to encode image\n"); return false; } - clip_image_f32_free(img_res); + + clip_image_f32_free(img_res_v.data); + img.request_encode_image = false; } From 8084d554406b767d36b3250b3b787462d5dd626f Mon Sep 17 00:00:00 2001 From: Michael Podvitskiy Date: Wed, 14 Feb 2024 11:49:01 +0300 Subject: [PATCH 344/812] cmake : ARM intrinsics detection for MSVC (#5401) --- CMakeLists.txt | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index a544f2da6..f8c7f9978 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -855,11 +855,21 @@ if (CMAKE_OSX_ARCHITECTURES STREQUAL "arm64" OR CMAKE_GENERATOR_PLATFORM_LWR STR CMAKE_SYSTEM_PROCESSOR MATCHES "^(aarch64|arm.*|ARM64)$")) message(STATUS "ARM detected") if (MSVC) + add_compile_definitions(__aarch64__) # MSVC defines _M_ARM64 instead add_compile_definitions(__ARM_NEON) add_compile_definitions(__ARM_FEATURE_FMA) - add_compile_definitions(__ARM_FEATURE_DOTPROD) - # add_compile_definitions(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) # MSVC doesn't support vdupq_n_f16, vld1q_f16, vst1q_f16 - add_compile_definitions(__aarch64__) # MSVC defines _M_ARM64 instead + + set(CMAKE_REQUIRED_FLAGS_PREV ${CMAKE_REQUIRED_FLAGS}) + string(JOIN " " CMAKE_REQUIRED_FLAGS ${CMAKE_REQUIRED_FLAGS} "/arch:armv8.2") + check_cxx_source_compiles("#include \nint main() { int8x16_t _a, _b; int32x4_t _s = vdotq_s32(_s, _a, _b); return 0; }" GGML_COMPILER_SUPPORT_DOTPROD) + if (GGML_COMPILER_SUPPORT_DOTPROD) + add_compile_definitions(__ARM_FEATURE_DOTPROD) + endif () + check_cxx_source_compiles("#include \nint main() { float16_t _a; float16x8_t _s = vdupq_n_f16(_a); return 0; }" GGML_COMPILER_SUPPORT_FP16_VECTOR_ARITHMETIC) + if (GGML_COMPILER_SUPPORT_FP16_VECTOR_ARITHMETIC) + add_compile_definitions(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) + endif () + set(CMAKE_REQUIRED_FLAGS ${CMAKE_REQUIRED_FLAGS_PREV}) else() check_cxx_compiler_flag(-mfp16-format=ieee COMPILER_SUPPORTS_FP16_FORMAT_I3E) if (NOT "${COMPILER_SUPPORTS_FP16_FORMAT_I3E}" STREQUAL "") From ccbb277f4642fc0d84c72dbc0d51ed2df418d6ce Mon Sep 17 00:00:00 2001 From: John <78893154+cmp-nct@users.noreply.github.com> Date: Wed, 14 Feb 2024 15:49:42 +0100 Subject: [PATCH 345/812] llava : update README.md (#5489) * Update README.md * Update README.md * Update examples/llava/README.md --------- Co-authored-by: Georgi Gerganov --- examples/llava/README.md | 46 ++++++++++++++++++++++++++++++++++++---- 1 file changed, 42 insertions(+), 4 deletions(-) diff --git a/examples/llava/README.md b/examples/llava/README.md index e2ef0eff1..1d5374f2a 100644 --- a/examples/llava/README.md +++ b/examples/llava/README.md @@ -1,10 +1,12 @@ # LLaVA -Currently this implementation supports [llava-v1.5](https://huggingface.co/liuhaotian/llava-v1.5-7b) variants. +Currently this implementation supports [llava-v1.5](https://huggingface.co/liuhaotian/llava-v1.5-7b) variants, +as well as llava-1.6 [llava-v1.6](https://huggingface.co/collections/liuhaotian/llava-16-65b9e40155f60fd046a5ccf2) variants. The pre-converted [7b](https://huggingface.co/mys/ggml_llava-v1.5-7b) and [13b](https://huggingface.co/mys/ggml_llava-v1.5-13b) models are available. +For llava-1.6 a variety of prepared gguf models are available as well [7b-34b](https://huggingface.co/cmp-nct/llava-1.6-gguf) After API is confirmed, more models will be supported / uploaded. @@ -18,6 +20,7 @@ After building, run: `./llava-cli` to see the usage. For example: ``` **note**: A lower temperature like 0.1 is recommended for better quality. add `--temp 0.1` to the command to do so. +**note**: For GPU offloading ensure to use the `-ngl` flag just like usual ## LLaVA 1.5 @@ -55,11 +58,46 @@ python ./convert.py ../llava-v1.5-7b Now both the LLaMA part and the image encoder is in the `llava-v1.5-7b` directory. -## LLaVA 1.6 +## LLaVA 1.6 gguf conversion + +1) Backup your pth/safetensor model files as llava-surgery modifies them +2) Use `python llava-surgery-v2.py -C -m /path/to/hf-model` which also supports llava-1.5 variants pytorch as well as safetensor models: +- you will find a llava.projector and a llava.clip file in your model directory +3) Copy the llava.clip file into a subdirectory (like vit), rename it to pytorch_model.bin and add a fitting vit configuration to the directory (https://huggingface.co/cmp-nct/llava-1.6-gguf/blob/main/config.json) +4) Create the visual gguf model: `python ./examples/llava/convert-image-encoder-to-gguf.py -m ../path/to/vit --llava-projector ../path/to/llava.projector --output-dir ../path/to/output --clip_model_is_vision` +- This is similar to llava-1.5, the difference is that we tell the encoder that we are working with the pure vision model part of CLIP +5) Everything else as usual: convert.py the hf model, quantize as needed +**note** llava-1.6 needs more context than llava-1.5, at least 3000 is needed (just run it at -c 4096) +**note** llava-1.6 greatly benefits from batched prompt processing (defaults work) + +## llava-cli templating and llava-1.6 prompting + +llava-1.5 models all use the same vicuna prompt, here you can just add your image question like `-p "Provide a full description."` +For llava-1.5 models which are not vicuna (mistral and Yi) you need to adapt system prompt as well as user prompt, for this purpose llava-cli has a basic templating system: + +**For Mistral and using llava-cli binary:** +Add this: `-p "\nUSER:\nProvide a full description.\nASSISTANT:\n"` +The mistral template for llava-1.6 seems to be no system print and a USER/ASSISTANT role + +**For the 34B this should work:** +Add this: `-e -p <|im_start|>system\nAnswer the questions.<|im_end|><|im_start|>user\n\nProvide a full description.<|im_end|><|im_start|>assistant\n` + + +## How to know if you are running in llava-1.5 or llava-1.6 mode + +When running llava-cli you will see a visual information right before the prompt is being processed: + +**Llava-1.5:** +`encode_image_with_clip: image embedding created: 576 tokens` + +**Llava-1.6 (anything above 576):** +`encode_image_with_clip: image embedding created: 2880 tokens` + + +Alternatively just pay notice to how many "tokens" have been used for your prompt, it will also show 1000+ tokens for llava-1.6 + -- Use `llava-surgery-v2.py` -- TODO: add detailed instructions ## TODO From 594fca3fefe27b8e95cfb1656eb0e160ad15a793 Mon Sep 17 00:00:00 2001 From: Rune <43761327+Rune-AI@users.noreply.github.com> Date: Wed, 14 Feb 2024 16:15:49 +0100 Subject: [PATCH 346/812] readme : fix typo (#5490) executabhle -> executable --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 0b4efdd33..0c4ee5a27 100644 --- a/README.md +++ b/README.md @@ -958,7 +958,7 @@ We have three Docker images available for this project: 1. `ghcr.io/ggerganov/llama.cpp:full`: This image includes both the main executable file and the tools to convert LLaMA models into ggml and convert into 4-bit quantization. (platforms: `linux/amd64`, `linux/arm64`) 2. `ghcr.io/ggerganov/llama.cpp:light`: This image only includes the main executable file. (platforms: `linux/amd64`, `linux/arm64`) -3. `ghcr.io/ggerganov/llama.cpp:server`: This image only includes the server executabhle file. (platforms: `linux/amd64`, `linux/arm64`) +3. `ghcr.io/ggerganov/llama.cpp:server`: This image only includes the server executable file. (platforms: `linux/amd64`, `linux/arm64`) Additionally, there the following images, similar to the above: From 704359e29985a06a389337a2617b7f3fa8eff908 Mon Sep 17 00:00:00 2001 From: Neuman Vong Date: Thu, 15 Feb 2024 17:11:15 +1100 Subject: [PATCH 347/812] vulkan: Find optimal memory type but with fallback (#5381) * @0cc4m feedback * More feedback @0cc4m --- ggml-vulkan.cpp | 65 ++++++++++++++++++++++++++++++++----------------- 1 file changed, 42 insertions(+), 23 deletions(-) diff --git a/ggml-vulkan.cpp b/ggml-vulkan.cpp index 7834e635c..1fad24fd1 100644 --- a/ggml-vulkan.cpp +++ b/ggml-vulkan.cpp @@ -707,9 +707,21 @@ static void ggml_vk_queue_cleanup(ggml_backend_vk_context * ctx, vk_queue& q) { q.cmd_buffer_idx = 0; } -static vk_buffer ggml_vk_create_buffer(ggml_backend_vk_context * ctx, size_t size, vk::MemoryPropertyFlags req_flags) { +static uint32_t find_properties(const vk::PhysicalDeviceMemoryProperties* mem_props, vk::MemoryRequirements* mem_req, vk::MemoryPropertyFlags flags) { + for (uint32_t i = 0; i < mem_props->memoryTypeCount; ++i) { + vk::MemoryType memory_type = mem_props->memoryTypes[i]; + if ((mem_req->memoryTypeBits & ((uint64_t)1 << i)) && + (flags & memory_type.propertyFlags) == flags && + mem_props->memoryHeaps[memory_type.heapIndex].size >= mem_req->size) { + return static_cast(i); + } + } + return UINT32_MAX; +} + +static vk_buffer ggml_vk_create_buffer(ggml_backend_vk_context * ctx, size_t size, vk::MemoryPropertyFlags req_flags, vk::MemoryPropertyFlags fallback_flags = vk::MemoryPropertyFlags(0)) { #ifdef GGML_VULKAN_DEBUG - std::cerr << "ggml_vk_create_buffer(" << size << ", " << to_string(req_flags) << ")" << std::endl; + std::cerr << "ggml_vk_create_buffer(" << size << ", " << to_string(req_flags) << ", " << to_string(fallback_flags) << ")" << std::endl; #endif vk_buffer buf = std::make_shared(); @@ -736,15 +748,15 @@ static vk_buffer ggml_vk_create_buffer(ggml_backend_vk_context * ctx, size_t siz uint32_t memory_type_index = UINT32_MAX; - for (uint32_t i = 0; i < mem_props.memoryTypeCount; ++i) { - vk::MemoryType memory_type = mem_props.memoryTypes[i]; - if ((mem_req.memoryTypeBits & ((uint64_t)1 << i)) && (req_flags & memory_type.propertyFlags) == req_flags && mem_props.memoryHeaps[memory_type.heapIndex].size >= mem_req.size) { - memory_type_index = i; - break; - } + memory_type_index = find_properties(&mem_props, &mem_req, req_flags); + buf->memory_property_flags = req_flags; + + if (memory_type_index == UINT32_MAX && fallback_flags) { + memory_type_index = find_properties(&mem_props, &mem_req, fallback_flags); + buf->memory_property_flags = fallback_flags; } - if (memory_type_index >= mem_props.memoryTypeCount) { + if (memory_type_index == UINT32_MAX) { ctx->device.lock()->device.destroyBuffer(buf->buffer); buf->size = 0; throw vk::OutOfDeviceMemoryError("No suitable memory type found"); @@ -758,10 +770,9 @@ static vk_buffer ggml_vk_create_buffer(ggml_backend_vk_context * ctx, size_t siz buf->size = 0; throw e; } - buf->memory_property_flags = req_flags; buf->ptr = nullptr; - if (req_flags & vk::MemoryPropertyFlagBits::eHostVisible) { + if (buf->memory_property_flags & vk::MemoryPropertyFlagBits::eHostVisible) { buf->ptr = ctx->device.lock()->device.mapMemory(buf->device_memory, 0, VK_WHOLE_SIZE); } @@ -778,9 +789,9 @@ static vk_buffer ggml_vk_create_buffer(ggml_backend_vk_context * ctx, size_t siz return buf; } -static vk_buffer ggml_vk_create_buffer_check(ggml_backend_vk_context * ctx, size_t size, vk::MemoryPropertyFlags req_flags) { +static vk_buffer ggml_vk_create_buffer_check(ggml_backend_vk_context * ctx, size_t size, vk::MemoryPropertyFlags req_flags, vk::MemoryPropertyFlags fallback_flags = vk::MemoryPropertyFlags(0)) { try { - return ggml_vk_create_buffer(ctx, size, req_flags); + return ggml_vk_create_buffer(ctx, size, req_flags, fallback_flags); } catch (const vk::SystemError& e) { std::cerr << "ggml_vulkan: Memory allocation of size " << size << " failed." << std::endl; std::cerr << "ggml_vulkan: " << e.what() << std::endl; @@ -791,16 +802,16 @@ static vk_buffer ggml_vk_create_buffer_check(ggml_backend_vk_context * ctx, size static vk_buffer ggml_vk_create_buffer_device(ggml_backend_vk_context * ctx, size_t size) { vk_buffer buf; try { - buf = ggml_vk_create_buffer(ctx, size, vk::MemoryPropertyFlagBits::eDeviceLocal); - } catch (const vk::SystemError& e) { if (ctx->device.lock()->uma) { // Fall back to host memory type - buf = ggml_vk_create_buffer_check(ctx, size, vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent); + buf = ggml_vk_create_buffer(ctx, size, vk::MemoryPropertyFlagBits::eDeviceLocal, vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent); } else { - std::cerr << "ggml_vulkan: Device memory allocation of size " << size << " failed." << std::endl; - std::cerr << "ggml_vulkan: " << e.what() << std::endl; - throw e; + buf = ggml_vk_create_buffer(ctx, size, vk::MemoryPropertyFlagBits::eDeviceLocal); } + } catch (const vk::SystemError& e) { + std::cerr << "ggml_vulkan: Device memory allocation of size " << size << " failed." << std::endl; + std::cerr << "ggml_vulkan: " << e.what() << std::endl; + throw e; } return buf; @@ -1422,7 +1433,9 @@ static void * ggml_vk_host_malloc(ggml_backend_vk_context * ctx, size_t size) { #ifdef GGML_VULKAN_DEBUG std::cerr << "ggml_vk_host_malloc(" << size << ")" << std::endl; #endif - vk_buffer buf = ggml_vk_create_buffer(ctx, size, vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent | vk::MemoryPropertyFlagBits::eHostCached); + vk_buffer buf = ggml_vk_create_buffer(ctx, size, + vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent | vk::MemoryPropertyFlagBits::eHostCached, + vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent); if(!(buf->memory_property_flags & vk::MemoryPropertyFlagBits::eHostVisible)) { fprintf(stderr, "WARNING: failed to allocate %.2f MB of pinned memory\n", @@ -1568,7 +1581,9 @@ static void deferred_memcpy(void * dst, const void * src, size_t size, std::vect static void ggml_vk_ensure_sync_staging_buffer(ggml_backend_vk_context * ctx, size_t size) { if (ctx->sync_staging == nullptr || ctx->sync_staging->size < size) { ggml_vk_destroy_buffer(ctx->sync_staging); - ctx->sync_staging = ggml_vk_create_buffer_check(ctx, size, vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent | vk::MemoryPropertyFlagBits::eHostCached); + ctx->sync_staging = ggml_vk_create_buffer_check(ctx, size, + vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent | vk::MemoryPropertyFlagBits::eHostCached, + vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent); } } @@ -4082,7 +4097,9 @@ static void ggml_vk_preallocate_buffers(ggml_backend_vk_context * ctx) { std::cerr << "ggml_vk_preallocate_buffers(qx_size: " << ctx->prealloc_size_qx << " qy_size: " << ctx->prealloc_size_qy << " x_size: " << ctx->prealloc_size_x << " y_size: " << ctx->prealloc_size_y << " split_k_size: " << ctx->prealloc_size_split_k << ")" << std::endl; #endif #if defined(GGML_VULKAN_RUN_TESTS) - ctx->staging = ggml_vk_create_buffer_check(ctx, 100ul * 1024ul * 1024ul, vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent | vk::MemoryPropertyFlagBits::eHostCached); + ctx->staging = ggml_vk_create_buffer_check(ctx, 100ul * 1024ul * 1024ul, + vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent | vk::MemoryPropertyFlagBits::eHostCached + vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent); ggml_vk_test_transfer(ctx, 8192 * 1000, false); ggml_vk_test_transfer(ctx, 8192 * 1000, true); @@ -4174,7 +4191,9 @@ static void ggml_vk_preallocate_buffers(ggml_backend_vk_context * ctx) { if (ctx->staging != nullptr) { ggml_vk_destroy_buffer(ctx->staging); } - ctx->staging = ggml_vk_create_buffer_check(ctx, ctx->staging_size, vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent | vk::MemoryPropertyFlagBits::eHostCached); + ctx->staging = ggml_vk_create_buffer_check(ctx, ctx->staging_size, + vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent | vk::MemoryPropertyFlagBits::eHostCached, + vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent); } } From 7930a8a6e89a04c77c51e3ae5dc1cd8e845b6b8f Mon Sep 17 00:00:00 2001 From: John <78893154+cmp-nct@users.noreply.github.com> Date: Thu, 15 Feb 2024 08:59:18 +0100 Subject: [PATCH 348/812] llaba : hotfix for llava-1.6 image number (#5495) Co-authored-by: John --- examples/llava/llava.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/llava/llava.cpp b/examples/llava/llava.cpp index 22953417f..4ed310a0e 100644 --- a/examples/llava/llava.cpp +++ b/examples/llava/llava.cpp @@ -100,7 +100,7 @@ static bool clip_llava_handle_patches(clip_ctx * ctx_clip, std::vector int num_patches_width = grid_shape.first; // grid 1-4 int num_patches_height = grid_shape.second; // grid 1-4 - const size_t num_images = num_patches_width + num_patches_height + 1; + const size_t num_images = num_patches_width * num_patches_height + 1; // TODO: size calculation is not calculated - it's only tens of MB size_t ctx_size = 0; From 0d4177126b0556e202efb85bf3f768be81076400 Mon Sep 17 00:00:00 2001 From: Elbios <141279586+Elbios@users.noreply.github.com> Date: Thu, 15 Feb 2024 09:01:57 +0100 Subject: [PATCH 349/812] llava : fix memory management bug (#5491) * Fix memory management in llava and server code Fixes this error: llama_new_context_with_model: graph splits (measure): 3 Available slots: -> Slot 0 - max context: 6000 {"timestamp":1707926446,"level":"INFO","function":"main","line":2623,"message":"model loaded"} all slots are idle and system prompt is empty, clear the KV cache slot 0 - loaded image slot 0 is processing [task id: 0] slot 0 : kv cache rm - [0, end) slot 0 - encoding image [id: 1] munmap_chunk(): invalid pointer Aborted * Make it cleaner by checking size in batch free wrapper --- examples/llava/clip.cpp | 24 +++++++++++++++++------- examples/llava/clip.h | 2 ++ examples/server/server.cpp | 11 +++++++++-- 3 files changed, 28 insertions(+), 9 deletions(-) diff --git a/examples/llava/clip.cpp b/examples/llava/clip.cpp index 9c5091e61..2cad27e82 100644 --- a/examples/llava/clip.cpp +++ b/examples/llava/clip.cpp @@ -1230,8 +1230,20 @@ struct clip_image_f32 * clip_image_f32_init() { return new clip_image_f32(); } -void clip_image_u8_free (struct clip_image_u8 * img) { delete img; } +void clip_image_u8_free(struct clip_image_u8 * img) { delete img; } void clip_image_f32_free(struct clip_image_f32 * img) { delete img; } +void clip_image_u8_batch_free(struct clip_image_u8_batch & batch) { + if (batch.size > 0) { + delete[] batch.data; + batch.size = 0; + } +} +void clip_image_f32_batch_free(struct clip_image_f32_batch & batch) { + if (batch.size > 0) { + delete[] batch.data; + batch.size = 0; + } +} static void build_clip_img_from_data(const stbi_uc * data, int nx, int ny, clip_image_u8 * img) { img->nx = nx; @@ -1494,11 +1506,8 @@ bool clip_image_preprocess(struct clip_ctx * ctx, const clip_image_u8 * img, cli pad_to_square = false; } // free the previous res_imgs if any set - if (res_imgs.size > 0 && res_imgs.size < 100) { - for (size_t i = 0; i < res_imgs.size; i++) { - clip_image_f32_free(&(res_imgs.data[i])); - } - delete[] res_imgs.data; + if (res_imgs.size > 0) { + clip_image_f32_batch_free(res_imgs); } res_imgs.data = nullptr; res_imgs.size = 0; @@ -1650,7 +1659,8 @@ bool clip_image_preprocess(struct clip_ctx * ctx, const clip_image_u8 * img, cli res_imgs.size = 1; res_imgs.data = new clip_image_f32[res_imgs.size]; - res_imgs.data[0] = std::move(*res); + res_imgs.data[0] = *res; + clip_image_f32_free(res); return true; } diff --git a/examples/llava/clip.h b/examples/llava/clip.h index cd9a4022f..e5bd54924 100644 --- a/examples/llava/clip.h +++ b/examples/llava/clip.h @@ -60,6 +60,8 @@ CLIP_API struct clip_image_f32 * clip_image_f32_init(); CLIP_API void clip_image_u8_free (struct clip_image_u8 * img); CLIP_API void clip_image_f32_free(struct clip_image_f32 * img); +CLIP_API void clip_image_u8_batch_free (struct clip_image_u8_batch & batch); +CLIP_API void clip_image_f32_batch_free(struct clip_image_f32_batch & batch); CLIP_API bool clip_image_load_from_file(const char * fname, struct clip_image_u8 * img); diff --git a/examples/server/server.cpp b/examples/server/server.cpp index 6e3434030..2decd7762 100644 --- a/examples/server/server.cpp +++ b/examples/server/server.cpp @@ -975,7 +975,12 @@ struct llama_server_context { LOG_TEE("Error processing the given image"); clip_free(clp_ctx); - clip_image_f32_free(img_res_v.data); + clip_image_f32_batch_free(img_res_v); + return false; + } + if (img_res_v.size == 0) + { + LOG_TEE("Error processing the given image"); return false; } @@ -987,6 +992,7 @@ struct llama_server_context if (!img.image_embedding) { LOG_TEE("Unable to allocate memory for image embeddings\n"); + clip_image_f32_batch_free(img_res_v); clip_free(clp_ctx); return false; } @@ -994,10 +1000,11 @@ struct llama_server_context if (!clip_image_encode(clp_ctx, params.n_threads, img_res, img.image_embedding)) { LOG_TEE("Unable to encode image\n"); + clip_image_f32_batch_free(img_res_v); return false; } - clip_image_f32_free(img_res_v.data); + clip_image_f32_batch_free(img_res_v); img.request_encode_image = false; } From 73122473ffd73030146276dbb85da7c8021a3ee4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20de=20Vries?= Date: Thu, 15 Feb 2024 14:14:37 +0100 Subject: [PATCH 350/812] fix(gguf-py): special tokens are no longer skipped when add__token is set to false (#5487) * fix(gguf-py): special tokens are no longer skipped when add__token is set to false * fix(gguf-py): added missing cls and mask token ids to the gguf metadata --- gguf-py/gguf/constants.py | 4 ++++ gguf-py/gguf/gguf_writer.py | 6 ++++++ gguf-py/gguf/vocab.py | 6 +----- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py index 5fba01714..9986ce9de 100644 --- a/gguf-py/gguf/constants.py +++ b/gguf-py/gguf/constants.py @@ -73,6 +73,8 @@ class Keys: UNK_ID = "tokenizer.ggml.unknown_token_id" SEP_ID = "tokenizer.ggml.seperator_token_id" PAD_ID = "tokenizer.ggml.padding_token_id" + CLS_ID = "tokenizer.ggml.cls_token_id" + MASK_ID = "tokenizer.ggml.mask_token_id" ADD_BOS = "tokenizer.ggml.add_bos_token" ADD_EOS = "tokenizer.ggml.add_eos_token" ADD_PREFIX = "tokenizer.ggml.add_space_prefix" @@ -685,5 +687,7 @@ KEY_TOKENIZER_EOS_ID = Keys.Tokenizer.EOS_ID KEY_TOKENIZER_UNK_ID = Keys.Tokenizer.UNK_ID KEY_TOKENIZER_SEP_ID = Keys.Tokenizer.SEP_ID KEY_TOKENIZER_PAD_ID = Keys.Tokenizer.PAD_ID +KEY_TOKENIZER_CLS_ID = Keys.Tokenizer.CLS_ID +KEY_TOKENIZER_MASK_ID = Keys.Tokenizer.MASK_ID KEY_TOKENIZER_HF_JSON = Keys.Tokenizer.HF_JSON KEY_TOKENIZER_RWKV = Keys.Tokenizer.RWKV diff --git a/gguf-py/gguf/gguf_writer.py b/gguf-py/gguf/gguf_writer.py index d87bd8e88..26724bf94 100644 --- a/gguf-py/gguf/gguf_writer.py +++ b/gguf-py/gguf/gguf_writer.py @@ -414,6 +414,12 @@ class GGUFWriter: def add_pad_token_id(self, id: int) -> None: self.add_uint32(Keys.Tokenizer.PAD_ID, id) + def add_cls_token_id(self, id: int) -> None: + self.add_uint32(Keys.Tokenizer.CLS_ID, id) + + def add_mask_token_id(self, id: int) -> None: + self.add_uint32(Keys.Tokenizer.MASK_ID, id) + def add_add_bos_token(self, value: bool) -> None: self.add_bool(Keys.Tokenizer.ADD_BOS, value) diff --git a/gguf-py/gguf/vocab.py b/gguf-py/gguf/vocab.py index cd1942975..a23136b18 100644 --- a/gguf-py/gguf/vocab.py +++ b/gguf-py/gguf/vocab.py @@ -29,7 +29,7 @@ class SpecialVocab: if special_token_types is not None: self.special_token_types = special_token_types else: - self.special_token_types = ('bos', 'eos', 'unk', 'sep', 'pad') + self.special_token_types = ('bos', 'eos', 'unk', 'sep', 'pad', 'cls', 'mask') self._load(Path(path)) def __repr__(self) -> str: @@ -152,10 +152,6 @@ class SpecialVocab: add_entry = tokenizer_config.get(f'add_{typ}_token') if isinstance(add_entry, bool): self.add_special_token[typ] = add_entry - if not added_tokens: - # We will need this to get the content for the token, so if it's empty - # may as well just give up. - continue entry = tokenizer_config.get(f'{typ}_token') if isinstance(entry, str): tc_content = entry From 9350a1cf21b1492c69b20175b73a419b897d6a3a Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Thu, 15 Feb 2024 15:41:15 +0200 Subject: [PATCH 351/812] scripts : add hf.sh helper script (#5501) * scripts : add hf.sh helper scripts * hf : add error logs * hf : add support for --repo and --file --- scripts/hf.sh | 107 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 107 insertions(+) create mode 100755 scripts/hf.sh diff --git a/scripts/hf.sh b/scripts/hf.sh new file mode 100755 index 000000000..1e9e5a6ea --- /dev/null +++ b/scripts/hf.sh @@ -0,0 +1,107 @@ +#!/bin/bash +# +# Shortcut for downloading HF models +# +# Usage: +# ./main -m $(./examples/hf.sh https://huggingface.co/TheBloke/Mixtral-8x7B-v0.1-GGUF/resolve/main/mixtral-8x7b-v0.1.Q4_K_M.gguf) +# ./main -m $(./examples/hf.sh --url https://huggingface.co/TheBloke/Mixtral-8x7B-v0.1-GGUF/blob/main/mixtral-8x7b-v0.1.Q4_K_M.gguf) +# ./main -m $(./examples/hf.sh --repo TheBloke/Mixtral-8x7B-v0.1-GGUF --file mixtral-8x7b-v0.1.Q4_K_M.gguf) +# + +# all logs go to stderr +function log { + echo "$@" 1>&2 +} + +function usage { + log "Usage: $0 [[--url] ] [--repo ] [--file ] [-h|--help]" + exit 1 +} + +# check for curl or wget +function has_cmd { + if ! [ -x "$(command -v $1)" ]; then + return 1 + fi +} + +if has_cmd wget; then + cmd="wget -q --show-progress -c -O %s %s" +elif has_cmd curl; then + cmd="curl -C - -f -o %s -L %s" +else + log "[E] curl or wget not found" + exit 1 +fi + +url="" +repo="" +file="" + +# parse args +while [[ $# -gt 0 ]]; do + case "$1" in + --url) + url="$2" + shift 2 + ;; + --repo) + repo="$2" + shift 2 + ;; + --file) + file="$2" + shift 2 + ;; + -h|--help) + usage + ;; + *) + url="$1" + shift + ;; + esac +done + +if [ -n "$repo" ] && [ -n "$file" ]; then + url="https://huggingface.co/$repo/resolve/main/$file" +fi + +if [ -z "$url" ]; then + log "[E] missing --url" + usage +fi + +# check if the URL is a HuggingFace model, and if so, try to download it +is_url=false + +if [[ ${#url} -gt 22 ]]; then + if [[ ${url:0:22} == "https://huggingface.co" ]]; then + is_url=true + fi +fi + +if [ "$is_url" = false ]; then + log "[E] invalid URL, must start with https://huggingface.co" + exit 0 +fi + +# replace "blob/main" with "resolve/main" +url=${url/blob\/main/resolve\/main} + +basename=$(basename $url) + +log "[+] attempting to download $basename" + +if [ -n "$cmd" ]; then + cmd=$(printf "$cmd" "$basename" "$url") + log "[+] $cmd" + if $cmd; then + echo $basename + exit 0 + fi +fi + +log "[-] failed to download" + +exit 1 From 9060a1e9dfca6038906e819be5fa42217f49028c Mon Sep 17 00:00:00 2001 From: slaren Date: Thu, 15 Feb 2024 16:49:01 +0100 Subject: [PATCH 352/812] cuda : print message when initialization fails (#5512) * cuda : print message when initialization fails * use CUDA_NAME both times --- ggml-cuda.cu | 1 + 1 file changed, 1 insertion(+) diff --git a/ggml-cuda.cu b/ggml-cuda.cu index 96976f248..b35fcb7fd 100644 --- a/ggml-cuda.cu +++ b/ggml-cuda.cu @@ -7943,6 +7943,7 @@ GGML_CALL void ggml_init_cublas() { if (cudaGetDeviceCount(&g_device_count) != cudaSuccess) { initialized = true; g_cublas_loaded = false; + fprintf(stderr, "%s: no " GGML_CUDA_NAME " devices found, " GGML_CUDA_NAME " will be disabled\n", __func__); return; } From c06e45d72983d9ace7b1535f7e7ea258d212169e Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Thu, 15 Feb 2024 18:49:08 +0200 Subject: [PATCH 353/812] clip : fix wrong loop condition --- examples/llava/clip.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/llava/clip.cpp b/examples/llava/clip.cpp index 2cad27e82..98d512f67 100644 --- a/examples/llava/clip.cpp +++ b/examples/llava/clip.cpp @@ -1103,7 +1103,7 @@ struct clip_ctx * clip_model_load(const char * fname, const int verbosity = 1) { printf("v_image_mean %f %f %f\n", new_clip->image_mean[0], new_clip->image_mean[1], new_clip->image_mean[2]); printf("v_image_std %f %f %f\n", new_clip->image_std[0], new_clip->image_std[1], new_clip->image_std[2]); printf("v_image_grid_pinpoints: "); - for (int i = 0; i < 32 & hparams.image_grid_pinpoints[i]!=0; ++i) { + for (int i = 0; i < 32 && (hparams.image_grid_pinpoints[i] != 0); ++i) { printf("%d ", hparams.image_grid_pinpoints[i]); } printf("\n"); From 4524290e87b8e107cc2b56e1251751546f4b9051 Mon Sep 17 00:00:00 2001 From: Douglas Hanley Date: Thu, 15 Feb 2024 11:21:49 -0600 Subject: [PATCH 354/812] Use correct type of pooling for embedding models (#5500) Use correct type of pooling for embedding models --- convert-hf-to-gguf.py | 24 ++++++++++- gguf-py/gguf/constants.py | 8 +++- gguf-py/gguf/gguf_writer.py | 5 ++- llama.cpp | 82 +++++++++++++++++++++++++------------ llama.h | 6 +++ 5 files changed, 94 insertions(+), 31 deletions(-) diff --git a/convert-hf-to-gguf.py b/convert-hf-to-gguf.py index ae471481d..9771fccf9 100755 --- a/convert-hf-to-gguf.py +++ b/convert-hf-to-gguf.py @@ -1650,7 +1650,29 @@ class BertModel(Model): def set_gguf_parameters(self): super().set_gguf_parameters() self.gguf_writer.add_causal_attention(False) - self.gguf_writer.add_pooling_layer(True) + + # get pooling path + with open(self.dir_model / "modules.json", encoding="utf-8") as f: + modules = json.load(f) + pooling_path = None + for mod in modules: + if mod["type"] == "sentence_transformers.models.Pooling": + pooling_path = mod["path"] + break + + # get pooling type + pooling_type = gguf.PoolingType.NONE + if pooling_path is not None: + with open(self.dir_model / pooling_path / "config.json", encoding="utf-8") as f: + pooling = json.load(f) + if pooling["pooling_mode_mean_tokens"]: + pooling_type = gguf.PoolingType.MEAN + elif pooling["pooling_mode_cls_token"]: + pooling_type = gguf.PoolingType.CLS + else: + raise NotImplementedError("Only MEAN and CLS pooling types supported") + + self.gguf_writer.add_pooling_type(pooling_type.value) def set_vocab(self): path = self.dir_model diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py index 9986ce9de..114a9a974 100644 --- a/gguf-py/gguf/constants.py +++ b/gguf-py/gguf/constants.py @@ -40,7 +40,7 @@ class Keys: TENSOR_DATA_LAYOUT = "{arch}.tensor_data_layout" EXPERT_COUNT = "{arch}.expert_count" EXPERT_USED_COUNT = "{arch}.expert_used_count" - POOLING_LAYER = "{arch}.pooling_layer" + POOLING_TYPE = "{arch}.pooling_type" class Attention: HEAD_COUNT = "{arch}.attention.head_count" @@ -561,6 +561,12 @@ class RopeScalingType(Enum): YARN = 'yarn' +class PoolingType(IntEnum): + NONE = 0 + MEAN = 1 + CLS = 2 + + class GGMLQuantizationType(IntEnum): F32 = 0 F16 = 1 diff --git a/gguf-py/gguf/gguf_writer.py b/gguf-py/gguf/gguf_writer.py index 26724bf94..e4681475c 100644 --- a/gguf-py/gguf/gguf_writer.py +++ b/gguf-py/gguf/gguf_writer.py @@ -19,6 +19,7 @@ from .constants import ( GGUFValueType, Keys, RopeScalingType, + PoolingType, TokenType, ) @@ -360,8 +361,8 @@ class GGUFWriter: def add_causal_attention(self, value: bool) -> None: self.add_bool(Keys.Attention.CAUSAL.format(arch=self.arch), value) - def add_pooling_layer(self, value: bool) -> None: - self.add_bool(Keys.LLM.POOLING_LAYER.format(arch=self.arch), value) + def add_pooling_type(self, value: PoolingType) -> None: + self.add_uint32(Keys.LLM.POOLING_TYPE.format(arch=self.arch), value) def add_rope_dimension_count(self, count: int) -> None: self.add_uint32(Keys.Rope.DIMENSION_COUNT.format(arch=self.arch), count) diff --git a/llama.cpp b/llama.cpp index 14e8821cd..aceb9c25a 100644 --- a/llama.cpp +++ b/llama.cpp @@ -256,7 +256,7 @@ enum llm_kv { LLM_KV_TENSOR_DATA_LAYOUT, LLM_KV_EXPERT_COUNT, LLM_KV_EXPERT_USED_COUNT, - LLM_KV_POOLING_LAYER, + LLM_KV_POOLING_TYPE, LLM_KV_ATTENTION_HEAD_COUNT, LLM_KV_ATTENTION_HEAD_COUNT_KV, @@ -314,7 +314,7 @@ static std::map LLM_KV_NAMES = { { LLM_KV_TENSOR_DATA_LAYOUT, "%s.tensor_data_layout" }, { LLM_KV_EXPERT_COUNT, "%s.expert_count" }, { LLM_KV_EXPERT_USED_COUNT, "%s.expert_used_count" }, - { LLM_KV_POOLING_LAYER, "%s.pooling_layer" }, + { LLM_KV_POOLING_TYPE , "%s.pooling_type" }, { LLM_KV_ATTENTION_HEAD_COUNT, "%s.attention.head_count" }, { LLM_KV_ATTENTION_HEAD_COUNT_KV, "%s.attention.head_count_kv" }, @@ -1561,7 +1561,7 @@ struct llama_hparams { float f_max_alibi_bias; bool causal_attn = true; - bool pooling_layer = false; + uint32_t pooling_type = LLAMA_POOLING_NONE; bool operator!=(const llama_hparams & other) const { @@ -1924,7 +1924,8 @@ struct llama_context { struct ggml_tensor * inp_pos; // I32 [n_batch] struct ggml_tensor * inp_KQ_mask; // F32 [n_ctx, n_batch] struct ggml_tensor * inp_K_shift; // I32 [n_ctx] - struct ggml_tensor * inp_sum; // F32 [n_batch, n_batch] + struct ggml_tensor * inp_mean; // F32 [n_batch, n_batch] + struct ggml_tensor * inp_cls; // I32 [n_batch] #ifdef GGML_USE_MPI ggml_mpi_context * ctx_mpi = NULL; @@ -3086,7 +3087,7 @@ static void llm_load_hparams( ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps); ml.get_key(LLM_KV_ATTENTION_CAUSAL, hparams.causal_attn); ml.get_key(LLM_KV_TOKENIZER_TOKEN_TYPE_COUNT, hparams.n_vocab_type); - ml.get_key(LLM_KV_POOLING_LAYER, hparams.pooling_layer); + ml.get_key(LLM_KV_POOLING_TYPE, hparams.pooling_type); switch (hparams.n_layer) { case 3: @@ -3107,7 +3108,7 @@ static void llm_load_hparams( ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps); ml.get_key(LLM_KV_ATTENTION_CAUSAL, hparams.causal_attn); ml.get_key(LLM_KV_TOKENIZER_TOKEN_TYPE_COUNT, hparams.n_vocab_type); - ml.get_key(LLM_KV_POOLING_LAYER, hparams.pooling_layer); + ml.get_key(LLM_KV_POOLING_TYPE, hparams.pooling_type); if (hparams.n_layer == 12 && hparams.n_embd == 768) { model.type = e_model::MODEL_137M; @@ -4934,7 +4935,7 @@ struct llm_build_context { const int32_t n_orig_ctx; const bool do_rope_shift; - const bool do_pooling; + const uint32_t pooling_type; const llm_build_cb & cb; @@ -4978,7 +4979,7 @@ struct llm_build_context { kv_head (worst_case ? n_ctx - n_tokens : kv_self.head), n_orig_ctx (cparams.n_yarn_orig_ctx), do_rope_shift (worst_case || kv_self.has_shift), - do_pooling (hparams.pooling_layer && cparams.do_pooling), + pooling_type (cparams.do_pooling ? hparams.pooling_type : (uint32_t)LLAMA_POOLING_NONE), cb (cb), buf_compute_meta (lctx.buf_compute_meta) { // all initializations should be done in init() @@ -5835,7 +5836,8 @@ struct llm_build_context { // get input vectors with right size const size_t stride1 = n_tokens * ggml_type_size(lctx.inp_tokens->type); struct ggml_tensor * inp_pos = ggml_view_1d(ctx0, lctx.inp_pos, n_tokens, 0); - struct ggml_tensor * inp_sum = ggml_view_2d(ctx0, lctx.inp_sum, n_tokens, n_tokens, stride1, 0); + struct ggml_tensor * inp_mean = ggml_view_2d(ctx0, lctx.inp_mean, n_tokens, n_tokens, stride1, 0); + struct ggml_tensor * inp_cls = ggml_view_1d(ctx0, lctx.inp_cls, n_tokens, 0); // construct input embeddings (token, type, position) inpL = llm_build_inp_embd(ctx0, hparams, batch, model.tok_embd, lctx.inp_tokens, lctx.inp_embd, cb); @@ -5952,8 +5954,12 @@ struct llm_build_context { cur = inpL; // pooling layer - if (do_pooling) { - cur = ggml_mul_mat(ctx0, ggml_cont(ctx0, ggml_transpose(ctx0, cur)), inp_sum); + if (pooling_type == LLAMA_POOLING_MEAN) { + cur = ggml_mul_mat(ctx0, ggml_cont(ctx0, ggml_transpose(ctx0, cur)), inp_mean); + } else if (pooling_type == LLAMA_POOLING_CLS) { + cur = ggml_get_rows(ctx0, cur, inp_cls); + } else { + GGML_ASSERT(pooling_type == LLAMA_POOLING_NONE && "Invalid pooling type"); } cb(cur, "result_embd", -1); @@ -7501,15 +7507,6 @@ static void llama_set_inputs(llama_context & lctx, const llama_batch & batch) { } } - { - assert(ggml_backend_buffer_is_host(lctx.inp_sum->buffer)); - float * data = (float *) lctx.inp_sum->data; - - for (int i = 0; i < batch.n_tokens; ++i) { - data[i] = 1.0f/float(batch.n_tokens); - } - } - if (kv_self.has_shift) { const int64_t n_ctx = cparams.n_ctx; @@ -7522,17 +7519,46 @@ static void llama_set_inputs(llama_context & lctx, const llama_batch & batch) { } } - if (hparams.pooling_layer && cparams.do_pooling) { + if (cparams.do_pooling && hparams.pooling_type == LLAMA_POOLING_MEAN) { const int64_t n_tokens = batch.n_tokens; - GGML_ASSERT(ggml_backend_buffer_is_host(lctx.inp_sum->buffer)); - float * data = (float *) lctx.inp_sum->data; + GGML_ASSERT(ggml_backend_buffer_is_host(lctx.inp_mean->buffer)); + float * data = (float *) lctx.inp_mean->data; - memset(lctx.inp_sum->data, 0, batch.n_tokens * batch.n_tokens * ggml_element_size(lctx.inp_sum)); + memset(lctx.inp_mean->data, 0, n_tokens * n_tokens * ggml_element_size(lctx.inp_mean)); + + std::vector sum(n_tokens, 0); + for (int i = 0; i < n_tokens; ++i) { + const llama_seq_id seq_id = batch.seq_id[i][0]; + sum[seq_id] += 1; + } + + std::vector div(n_tokens, 0.0f); + for (int i = 0; i < n_tokens; ++i) { + const uint64_t s = sum[i]; + if (s > 0) { + div[i] = 1.0f/float(s); + } + } for (int i = 0; i < n_tokens; ++i) { const llama_seq_id seq_id = batch.seq_id[i][0]; - data[seq_id*n_tokens + i] = 1.0f; + data[seq_id*n_tokens + i] = div[seq_id]; + } + } + + if (cparams.do_pooling && hparams.pooling_type == LLAMA_POOLING_CLS) { + const int64_t n_tokens = batch.n_tokens; + + GGML_ASSERT(ggml_backend_buffer_is_host(lctx.inp_cls->buffer)); + uint32_t * data = (uint32_t *) lctx.inp_cls->data; + + for (int i = 0; i < n_tokens; ++i) { + const llama_seq_id seq_id = batch.seq_id[i][0]; + const llama_pos pos = batch.pos[i]; + if (pos == 0) { + data[seq_id] = i; + } } } } @@ -11417,14 +11443,16 @@ struct llama_context * llama_new_context_with_model( ctx->inp_pos = ggml_new_tensor_1d(ctx->ctx_input, GGML_TYPE_I32, cparams.n_batch); ctx->inp_KQ_mask = ggml_new_tensor_2d(ctx->ctx_input, GGML_TYPE_F32, cparams.n_ctx, cparams.n_batch); ctx->inp_K_shift = ggml_new_tensor_1d(ctx->ctx_input, GGML_TYPE_I32, cparams.n_ctx); - ctx->inp_sum = ggml_new_tensor_2d(ctx->ctx_input, GGML_TYPE_F32, cparams.n_batch, cparams.n_batch); + ctx->inp_mean = ggml_new_tensor_2d(ctx->ctx_input, GGML_TYPE_F32, cparams.n_batch, cparams.n_batch); + ctx->inp_cls = ggml_new_tensor_1d(ctx->ctx_input, GGML_TYPE_I32, cparams.n_batch); ggml_set_name(ctx->inp_tokens, "inp_tokens"); ggml_set_name(ctx->inp_embd, "inp_embd"); ggml_set_name(ctx->inp_pos, "inp_pos"); ggml_set_name(ctx->inp_KQ_mask, "inp_KQ_mask"); ggml_set_name(ctx->inp_K_shift, "inp_K_shift"); - ggml_set_name(ctx->inp_sum, "inp_sum"); + ggml_set_name(ctx->inp_mean, "inp_mean"); + ggml_set_name(ctx->inp_cls, "inp_cls"); ctx->buf_input = ggml_backend_alloc_ctx_tensors_from_buft(ctx->ctx_input, llama_default_buffer_type_cpu(true)); diff --git a/llama.h b/llama.h index 5ef78ec96..4a26bd619 100644 --- a/llama.h +++ b/llama.h @@ -112,6 +112,12 @@ extern "C" { LLAMA_ROPE_SCALING_MAX_VALUE = LLAMA_ROPE_SCALING_YARN, }; + enum llama_pooling_type { + LLAMA_POOLING_NONE = 0, + LLAMA_POOLING_MEAN = 1, + LLAMA_POOLING_CLS = 2, + }; + enum llama_split_mode { LLAMA_SPLIT_NONE = 0, // single GPU LLAMA_SPLIT_LAYER = 1, // split layers and KV across GPUs From 594845aab1c6775877f6d9545a51dc0f8d0b3d77 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Fri, 16 Feb 2024 09:57:55 +0200 Subject: [PATCH 355/812] ci : fix BERT model download and convert --- ci/run.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ci/run.sh b/ci/run.sh index a4264d775..979b4a793 100755 --- a/ci/run.sh +++ b/ci/run.sh @@ -580,6 +580,10 @@ function gg_run_embd_bge_small { gg_wget models-mnt/bge-small/ https://huggingface.co/BAAI/bge-small-en-v1.5/resolve/main/pytorch_model.bin gg_wget models-mnt/bge-small/ https://huggingface.co/BAAI/bge-small-en-v1.5/raw/main/sentence_bert_config.json gg_wget models-mnt/bge-small/ https://huggingface.co/BAAI/bge-small-en-v1.5/raw/main/vocab.txt + gg_wget models-mnt/bge-small/ https://huggingface.co/BAAI/bge-small-en-v1.5/raw/main/modules.json + gg_wget models-mnt/bge-small/ https://huggingface.co/BAAI/bge-small-en-v1.5/raw/main/config.json + + gg_wget models-mnt/bge-small/1_Pooling https://huggingface.co/BAAI/bge-small-en-v1.5/raw/main/1_Pooling/config.json path_models="../models-mnt/bge-small" From 60ed04cf82dc91ade725dd7ad53f0ee81f76eccf Mon Sep 17 00:00:00 2001 From: Daniel Bevenius Date: Fri, 16 Feb 2024 10:24:39 +0100 Subject: [PATCH 356/812] llava : fix clip-model-is-vision flag in README.md (#5509) * llava: fix clip-model-is-vision flag in README.md This commit fixes the flag `--clip_model_is_vision` in README.md which is does not match the actual flag: ```console $ python convert-image-encoder-to-gguf.py --help ... --clip-model-is-vision The clip model is a pure vision model (ShareGPT4V vision extract for example) ``` Signed-off-by: Daniel Bevenius * llava: update link to vit config in README.md Signed-off-by: Daniel Bevenius --------- Signed-off-by: Daniel Bevenius --- examples/llava/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/llava/README.md b/examples/llava/README.md index 1d5374f2a..57eb42932 100644 --- a/examples/llava/README.md +++ b/examples/llava/README.md @@ -63,8 +63,8 @@ Now both the LLaMA part and the image encoder is in the `llava-v1.5-7b` director 1) Backup your pth/safetensor model files as llava-surgery modifies them 2) Use `python llava-surgery-v2.py -C -m /path/to/hf-model` which also supports llava-1.5 variants pytorch as well as safetensor models: - you will find a llava.projector and a llava.clip file in your model directory -3) Copy the llava.clip file into a subdirectory (like vit), rename it to pytorch_model.bin and add a fitting vit configuration to the directory (https://huggingface.co/cmp-nct/llava-1.6-gguf/blob/main/config.json) -4) Create the visual gguf model: `python ./examples/llava/convert-image-encoder-to-gguf.py -m ../path/to/vit --llava-projector ../path/to/llava.projector --output-dir ../path/to/output --clip_model_is_vision` +3) Copy the llava.clip file into a subdirectory (like vit), rename it to pytorch_model.bin and add a fitting vit configuration to the directory (https://huggingface.co/cmp-nct/llava-1.6-gguf/blob/main/config_vit.json) and rename it to config.json. +4) Create the visual gguf model: `python ./examples/llava/convert-image-encoder-to-gguf.py -m ../path/to/vit --llava-projector ../path/to/llava.projector --output-dir ../path/to/output --clip-model-is-vision` - This is similar to llava-1.5, the difference is that we tell the encoder that we are working with the pure vision model part of CLIP 5) Everything else as usual: convert.py the hf model, quantize as needed **note** llava-1.6 needs more context than llava-1.5, at least 3000 is needed (just run it at -c 4096) From f486f6e1e5e9d01603d9325ab3e05f1edb362a95 Mon Sep 17 00:00:00 2001 From: bmwl Date: Fri, 16 Feb 2024 01:31:07 -0800 Subject: [PATCH 357/812] ggml : add numa options (#5377) * Added numa options to allow finer grained control as well as plumbing for a new mirror mode that will require numa.h * Reverted Makefile * Fixed include * Removed sched.h from ggml.h, moved ggml_get_numa_affinity into ggml.c, removed trailing whitespace and fixed up a few inconsistent variables * removed trailing whitespace * Added numa options to allow finer grained control as well as plumbing for a new mirror mode that will require numa.h * Reverting Makefile * Fixed a number of issues with the move from BOOL to ggml_numa_strategies. Added a note about mirror mode note being implemented yet * Removing MIRROR_MODE code for this PR * Removing last bit of MIRROR_MODE code for this PR * Removing unneeded branch in server.cpp example and moving get_numa_affinity and making it static * Fixed lingering init_llama_backend() bool calls in tests and examples * Remote enum llama_numa_strategies * Revert bad merge with dynatemp flags * add missing enum ggml_numa_strategies declaration and revert sync problem with master * add missing enum ggml_numa_strategies declaration * fixed ggml_init_numa variable * Update ggml.h Co-authored-by: Jared Van Bortel * Update READMEs with info about numa flags, change INTERLEAVE strategy name to DISTRIBUTE everywhere, implement the improved distribution strategy from @rankaiyx, fix a spelling mistake and un-merge some bad merges * split numa init out from llama_backend_init and created llama_numa_init. Updated all code paths and samples * Fix up some boolean vs enum comparisons * Added #ifdefs for non-Linux OS that don't have cpu_set_t datatype * Update ggml.h Align enum values Co-authored-by: Georgi Gerganov * Update ggml.c Remove whitespace Co-authored-by: Georgi Gerganov * Update ggml.c align paremeters Co-authored-by: Georgi Gerganov * Update examples/server/server.cpp remove whitespace and align brace Co-authored-by: Georgi Gerganov * Update common/common.cpp Remove whitespace and align brace Co-authored-by: Georgi Gerganov * unified ggml_numa_strategy enum and fixed text alignment in server.cpp example * Update ggml.c simplified return for platforms without NUMA support Co-authored-by: Jared Van Bortel * removed redundant else from cli argument processing of --numa * whitespace --------- Co-authored-by: root Co-authored-by: Jared Van Bortel Co-authored-by: Georgi Gerganov Co-authored-by: Jared Van Bortel --- common/common.cpp | 20 +++-- common/common.h | 2 +- examples/batched-bench/batched-bench.cpp | 3 +- examples/batched.swift/Sources/main.swift | 2 +- examples/batched/batched.cpp | 3 +- examples/beam-search/beam-search.cpp | 3 +- examples/embedding/embedding.cpp | 3 +- examples/imatrix/imatrix.cpp | 3 +- examples/infill/infill.cpp | 3 +- examples/llama-bench/llama-bench.cpp | 3 +- .../app/src/main/cpp/llama-android.cpp | 4 +- .../llama.cpp.swift/LibLlama.swift | 2 +- examples/llava/llava-cli.cpp | 3 +- examples/lookahead/lookahead.cpp | 3 +- examples/lookup/lookup.cpp | 3 +- examples/main/README.md | 6 +- examples/main/main.cpp | 3 +- examples/parallel/parallel.cpp | 3 +- examples/passkey/passkey.cpp | 3 +- examples/perplexity/perplexity.cpp | 3 +- examples/quantize/quantize.cpp | 2 +- examples/server/README.md | 7 ++ examples/server/server.cpp | 22 +++-- examples/simple/simple.cpp | 3 +- examples/speculative/speculative.cpp | 3 +- examples/tokenize/tokenize.cpp | 2 +- ggml.c | 80 ++++++++++++++++--- ggml.h | 12 ++- llama.cpp | 14 ++-- llama.h | 5 +- tests/test-autorelease.cpp | 2 +- tests/test-model-load-cancel.cpp | 2 +- tests/test-tokenizer-0-falcon.cpp | 2 +- tests/test-tokenizer-0-llama.cpp | 2 +- tests/test-tokenizer-1-bpe.cpp | 2 +- tests/test-tokenizer-1-llama.cpp | 2 +- 36 files changed, 178 insertions(+), 62 deletions(-) diff --git a/common/common.cpp b/common/common.cpp index f64da2cb6..c5e83cc2a 100644 --- a/common/common.cpp +++ b/common/common.cpp @@ -671,7 +671,15 @@ bool gpt_params_parse_ex(int argc, char ** argv, gpt_params & params) { } else if (arg == "--no-mmap") { params.use_mmap = false; } else if (arg == "--numa") { - params.numa = true; + if (++i >= argc) { + invalid_param = true; + break; + } + std::string value(argv[i]); + /**/ if (value == "distribute" || value == "") { params.numa = GGML_NUMA_STRATEGY_DISTRIBUTE; } + else if (value == "isolate") { params.numa = GGML_NUMA_STRATEGY_ISOLATE; } + else if (value == "numactl") { params.numa = GGML_NUMA_STRATEGY_NUMACTL; } + else { invalid_param = true; break; } } else if (arg == "--verbose-prompt") { params.verbose_prompt = true; } else if (arg == "--no-display-prompt") { @@ -935,7 +943,7 @@ void gpt_print_usage(int /*argc*/, char ** argv, const gpt_params & params) { printf(" -tb N, --threads-batch N\n"); printf(" number of threads to use during batch and prompt processing (default: same as --threads)\n"); printf(" -td N, --threads-draft N"); - printf(" number of threads to use during generation (default: same as --threads)"); + printf(" number of threads to use during generation (default: same as --threads)\n"); printf(" -tbd N, --threads-batch-draft N\n"); printf(" number of threads to use during batch and prompt processing (default: same as --threads-draft)\n"); printf(" -p PROMPT, --prompt PROMPT\n"); @@ -1005,7 +1013,7 @@ void gpt_print_usage(int /*argc*/, char ** argv, const gpt_params & params) { printf(" --winogrande-tasks N number of tasks to use when computing the Winogrande score (default: %zu)\n", params.winogrande_tasks); printf(" --multiple-choice compute multiple choice score over random tasks from datafile supplied with -f\n"); printf(" --multiple-choice-tasks N number of tasks to use when computing the multiple choice score (default: %zu)\n", params.winogrande_tasks); - printf(" --kl-divergence computes KL-divergence to logits provided via --kl-divergence-base"); + printf(" --kl-divergence computes KL-divergence to logits provided via --kl-divergence-base\n"); printf(" --keep N number of tokens to keep from the initial prompt (default: %d, -1 = all)\n", params.n_keep); printf(" --draft N number of tokens to draft for speculative decoding (default: %d)\n", params.n_draft); printf(" --chunks N max number of chunks to process (default: %d, -1 = all)\n", params.n_chunks); @@ -1022,7 +1030,10 @@ void gpt_print_usage(int /*argc*/, char ** argv, const gpt_params & params) { if (llama_supports_mmap()) { printf(" --no-mmap do not memory-map model (slower load but may reduce pageouts if not using mlock)\n"); } - printf(" --numa attempt optimizations that help on some NUMA systems\n"); + printf(" --numa TYPE attempt optimizations that help on some NUMA systems\n"); + printf(" - distribute: spread execution evenly over all nodes\n"); + printf(" - isolate: only spawn threads on CPUs on the node that execution started on\n"); + printf(" - numactl: use the CPU map provided by numactl\n"); printf(" if run without this previously, it is recommended to drop the system page cache before using this\n"); printf(" see https://github.com/ggerganov/llama.cpp/issues/1437\n"); if (llama_supports_gpu_offload()) { @@ -1689,7 +1700,6 @@ void dump_non_result_info_yaml(FILE * stream, const gpt_params & params, const l fprintf(stream, "no_mmap: %s # default: false\n", !params.use_mmap ? "true" : "false"); fprintf(stream, "no_mul_mat_q: %s # default: false\n", !params.mul_mat_q ? "true" : "false"); fprintf(stream, "no_penalize_nl: %s # default: false\n", !sparams.penalize_nl ? "true" : "false"); - fprintf(stream, "numa: %s # default: false\n", params.numa ? "true" : "false"); fprintf(stream, "ppl_output_type: %d # default: 0\n", params.ppl_output_type); fprintf(stream, "ppl_stride: %d # default: 0\n", params.ppl_stride); fprintf(stream, "presence_penalty: %f # default: 0.0\n", sparams.penalty_present); diff --git a/common/common.h b/common/common.h index 9bdd45cf9..74c136995 100644 --- a/common/common.h +++ b/common/common.h @@ -76,6 +76,7 @@ struct gpt_params { float yarn_beta_slow = 1.0f; // YaRN high correction dim int32_t yarn_orig_ctx = 0; // YaRN original context length int32_t rope_scaling_type = LLAMA_ROPE_SCALING_UNSPECIFIED; + ggml_numa_strategy numa = GGML_NUMA_STRATEGY_DISABLED; // // sampling parameters struct llama_sampling_params sparams; @@ -134,7 +135,6 @@ struct gpt_params { bool logits_all = false; // return logits for all tokens in the batch bool use_mmap = true; // use mmap for faster loads bool use_mlock = false; // use mlock to keep model in memory - bool numa = false; // attempt optimizations that help on some NUMA systems bool verbose_prompt = false; // print prompt tokens before generation bool display_prompt = true; // print prompt before generation bool infill = false; // use infill mode diff --git a/examples/batched-bench/batched-bench.cpp b/examples/batched-bench/batched-bench.cpp index b52d68457..55dfd9784 100644 --- a/examples/batched-bench/batched-bench.cpp +++ b/examples/batched-bench/batched-bench.cpp @@ -82,7 +82,8 @@ int main(int argc, char ** argv) { // init LLM - llama_backend_init(params.numa); + llama_backend_init(); + llama_numa_init(params.numa); // initialize the model diff --git a/examples/batched.swift/Sources/main.swift b/examples/batched.swift/Sources/main.swift index 4d0005349..d75c503d5 100644 --- a/examples/batched.swift/Sources/main.swift +++ b/examples/batched.swift/Sources/main.swift @@ -17,7 +17,7 @@ let n_parallel: Int = arguments.count > 3 && Int(arguments[3]) != nil ? Int(argu let n_len: Int = 32 // init LLM -llama_backend_init(false) +llama_backend_init() defer { llama_backend_free() } diff --git a/examples/batched/batched.cpp b/examples/batched/batched.cpp index b1775e0b0..eab636692 100644 --- a/examples/batched/batched.cpp +++ b/examples/batched/batched.cpp @@ -50,7 +50,8 @@ int main(int argc, char ** argv) { // init LLM - llama_backend_init(params.numa); + llama_backend_init(); + llama_numa_init(params.numa); // initialize the model diff --git a/examples/beam-search/beam-search.cpp b/examples/beam-search/beam-search.cpp index 679b382e1..866c6d7a6 100644 --- a/examples/beam-search/beam-search.cpp +++ b/examples/beam-search/beam-search.cpp @@ -119,7 +119,8 @@ int main(int argc, char ** argv) // Init LLM : //--------------------------------- - llama_backend_init(params.numa); + llama_backend_init(); + llama_numa_init(params.numa); llama_model * model; llama_context * ctx; diff --git a/examples/embedding/embedding.cpp b/examples/embedding/embedding.cpp index b4688cf51..acff715e9 100644 --- a/examples/embedding/embedding.cpp +++ b/examples/embedding/embedding.cpp @@ -74,7 +74,8 @@ int main(int argc, char ** argv) { params.prompt = gpt_random_prompt(rng); } - llama_backend_init(params.numa); + llama_backend_init(); + llama_numa_init(params.numa); llama_model * model; llama_context * ctx; diff --git a/examples/imatrix/imatrix.cpp b/examples/imatrix/imatrix.cpp index bc9f6fa68..f21bc48f3 100644 --- a/examples/imatrix/imatrix.cpp +++ b/examples/imatrix/imatrix.cpp @@ -568,7 +568,8 @@ int main(int argc, char ** argv) { params.prompt = gpt_random_prompt(rng); } - llama_backend_init(params.numa); + llama_backend_init(); + llama_numa_init(params.numa); llama_model_params mparams = llama_model_params_from_gpt_params(params); diff --git a/examples/infill/infill.cpp b/examples/infill/infill.cpp index 72fb133b4..92c67b7cf 100644 --- a/examples/infill/infill.cpp +++ b/examples/infill/infill.cpp @@ -202,7 +202,8 @@ int main(int argc, char ** argv) { std::mt19937 rng(params.seed); LOG("%s: llama backend init\n", __func__); - llama_backend_init(params.numa); + llama_backend_init(); + llama_numa_init(params.numa); llama_model * model; llama_context * ctx; diff --git a/examples/llama-bench/llama-bench.cpp b/examples/llama-bench/llama-bench.cpp index ddb0ba064..11410f8ae 100644 --- a/examples/llama-bench/llama-bench.cpp +++ b/examples/llama-bench/llama-bench.cpp @@ -1151,8 +1151,7 @@ int main(int argc, char ** argv) { if (!params.verbose) { llama_log_set(llama_null_log_callback, NULL); } - bool numa = false; - llama_backend_init(numa); + llama_backend_init(); // initialize printer std::unique_ptr p; diff --git a/examples/llama.android/app/src/main/cpp/llama-android.cpp b/examples/llama.android/app/src/main/cpp/llama-android.cpp index d5e705dce..2beb1e0d5 100644 --- a/examples/llama.android/app/src/main/cpp/llama-android.cpp +++ b/examples/llama.android/app/src/main/cpp/llama-android.cpp @@ -274,8 +274,8 @@ Java_com_example_llama_Llm_new_1batch(JNIEnv *, jobject, jint n_tokens, jint emb extern "C" JNIEXPORT void JNICALL -Java_com_example_llama_Llm_backend_1init(JNIEnv *, jobject, jboolean numa) { - llama_backend_init(numa); +Java_com_example_llama_Llm_backend_1init(JNIEnv *, jobject) { + llama_backend_init(); } extern "C" diff --git a/examples/llama.swiftui/llama.cpp.swift/LibLlama.swift b/examples/llama.swiftui/llama.cpp.swift/LibLlama.swift index fc79fd346..58fcf40c6 100644 --- a/examples/llama.swiftui/llama.cpp.swift/LibLlama.swift +++ b/examples/llama.swiftui/llama.cpp.swift/LibLlama.swift @@ -51,7 +51,7 @@ actor LlamaContext { } static func create_context(path: String) throws -> LlamaContext { - llama_backend_init(false) + llama_backend_init() var model_params = llama_model_default_params() #if targetEnvironment(simulator) diff --git a/examples/llava/llava-cli.cpp b/examples/llava/llava-cli.cpp index bef7f7c95..e29da6cb2 100644 --- a/examples/llava/llava-cli.cpp +++ b/examples/llava/llava-cli.cpp @@ -218,7 +218,8 @@ static struct llava_context * llava_init(gpt_params * params) { auto ctx_clip = clip_model_load(clip_path, /*verbosity=*/ 1); - llama_backend_init(params->numa); + llama_backend_init(); + llama_numa_init(params->numa); llama_model_params model_params = llama_model_params_from_gpt_params(*params); diff --git a/examples/lookahead/lookahead.cpp b/examples/lookahead/lookahead.cpp index e55a15a1b..e2551e7a4 100644 --- a/examples/lookahead/lookahead.cpp +++ b/examples/lookahead/lookahead.cpp @@ -54,7 +54,8 @@ int main(int argc, char ** argv) { #endif // LOG_DISABLE_LOGS // init llama.cpp - llama_backend_init(params.numa); + llama_backend_init(); + llama_numa_init(params.numa); llama_model * model = NULL; llama_context * ctx = NULL; diff --git a/examples/lookup/lookup.cpp b/examples/lookup/lookup.cpp index 18235b8a1..b53fae110 100644 --- a/examples/lookup/lookup.cpp +++ b/examples/lookup/lookup.cpp @@ -31,7 +31,8 @@ int main(int argc, char ** argv){ #endif // LOG_DISABLE_LOGS // init llama.cpp - llama_backend_init(params.numa); + llama_backend_init(); + llama_numa_init(params.numa); llama_model * model = NULL; llama_context * ctx = NULL; diff --git a/examples/main/README.md b/examples/main/README.md index c7997f665..7f84e4262 100644 --- a/examples/main/README.md +++ b/examples/main/README.md @@ -283,7 +283,11 @@ These options help improve the performance and memory usage of the LLaMA models. ### NUMA support -- `--numa`: Attempt optimizations that help on some systems with non-uniform memory access. This currently consists of pinning an equal proportion of the threads to the cores on each NUMA node, and disabling prefetch and readahead for mmap. The latter causes mapped pages to be faulted in on first access instead of all at once, and in combination with pinning threads to NUMA nodes, more of the pages end up on the NUMA node where they are used. Note that if the model is already in the system page cache, for example because of a previous run without this option, this will have little effect unless you drop the page cache first. This can be done by rebooting the system or on Linux by writing '3' to '/proc/sys/vm/drop_caches' as root. +- `--numa distribute`: Pin an equal proportion of the threads to the cores on each NUMA node. This will spread the load amongst all cores on the system, utilitizing all memory channels at the expense of potentially requiring memory to travel over the slow links between nodes. +- `--numa isolate`: Pin all threads to the NUMA node that the program starts on. This limits the number of cores and amount of memory that can be used, but guarantees all memory access remains local to the NUMA node. +- `--numa numactl`: Pin threads to the CPUMAP that is passed to the program by starting it with the numactl utility. This is the most flexible mode, and allow arbitraty core usage patterns, for example a map that uses all the cores on one NUMA nodes, and just enough cores on a second node to saturate the inter-node memory bus. + + These flags attempt optimizations that help on some systems with non-uniform memory access. This currently consists of one of the above strategies, and disabling prefetch and readahead for mmap. The latter causes mapped pages to be faulted in on first access instead of all at once, and in combination with pinning threads to NUMA nodes, more of the pages end up on the NUMA node where they are used. Note that if the model is already in the system page cache, for example because of a previous run without this option, this will have little effect unless you drop the page cache first. This can be done by rebooting the system or on Linux by writing '3' to '/proc/sys/vm/drop_caches' as root. ### Memory Float 32 diff --git a/examples/main/main.cpp b/examples/main/main.cpp index e8ab8cbae..f5d2f4893 100644 --- a/examples/main/main.cpp +++ b/examples/main/main.cpp @@ -185,7 +185,8 @@ int main(int argc, char ** argv) { } LOG("%s: llama backend init\n", __func__); - llama_backend_init(params.numa); + llama_backend_init(); + llama_numa_init(params.numa); llama_model * model; llama_context * ctx; diff --git a/examples/parallel/parallel.cpp b/examples/parallel/parallel.cpp index d2e074d9e..7d11fcd59 100644 --- a/examples/parallel/parallel.cpp +++ b/examples/parallel/parallel.cpp @@ -122,7 +122,8 @@ int main(int argc, char ** argv) { #endif // LOG_DISABLE_LOGS // init llama.cpp - llama_backend_init(params.numa); + llama_backend_init(); + llama_numa_init(params.numa); llama_model * model = NULL; llama_context * ctx = NULL; diff --git a/examples/passkey/passkey.cpp b/examples/passkey/passkey.cpp index 5c0022832..e12a1cdf1 100644 --- a/examples/passkey/passkey.cpp +++ b/examples/passkey/passkey.cpp @@ -71,7 +71,8 @@ int main(int argc, char ** argv) { // init LLM - llama_backend_init(params.numa); + llama_backend_init(); + llama_numa_init(params.numa); // initialize the model diff --git a/examples/perplexity/perplexity.cpp b/examples/perplexity/perplexity.cpp index b2c131d4c..67d2d3293 100644 --- a/examples/perplexity/perplexity.cpp +++ b/examples/perplexity/perplexity.cpp @@ -1809,7 +1809,8 @@ int main(int argc, char ** argv) { params.prompt = gpt_random_prompt(rng); } - llama_backend_init(params.numa); + llama_backend_init(); + llama_numa_init(params.numa); llama_model * model; llama_context * ctx; diff --git a/examples/quantize/quantize.cpp b/examples/quantize/quantize.cpp index 85f403ffc..4a5c504e3 100644 --- a/examples/quantize/quantize.cpp +++ b/examples/quantize/quantize.cpp @@ -237,7 +237,7 @@ int main(int argc, char ** argv) { params.imatrix = &imatrix_data; } - llama_backend_init(false); + llama_backend_init(); // parse command line arguments const std::string fname_inp = argv[arg_idx]; diff --git a/examples/server/README.md b/examples/server/README.md index 0f7373ae8..8e141d22d 100644 --- a/examples/server/README.md +++ b/examples/server/README.md @@ -16,6 +16,13 @@ Command line options: - `--memory-f32`: Use 32-bit floats instead of 16-bit floats for memory key+value. Not recommended. - `--mlock`: Lock the model in memory, preventing it from being swapped out when memory-mapped. - `--no-mmap`: Do not memory-map the model. By default, models are mapped into memory, which allows the system to load only the necessary parts of the model as needed. +- `--numa STRATEGY`: Attempt one of the below optimization strategies that help on some NUMA systems +- `--numa distribute`: Spread execution evenly over all nodes +- `--numa isolate`: Only spawn threads on CPUs on the node that execution started on +- `--numa numactl`: Use the CPU map provided by numactl +if run without this previously, it is recommended to drop the system page cache before using this +see https://github.com/ggerganov/llama.cpp/issues/1437 + - `--numa`: Attempt optimizations that help on some NUMA systems. - `--lora FNAME`: Apply a LoRA (Low-Rank Adaptation) adapter to the model (implies --no-mmap). This allows you to adapt the pretrained model to specific tasks or domains. - `--lora-base FNAME`: Optional model to use as a base for the layers modified by the LoRA adapter. This flag is used in conjunction with the `--lora` flag, and specifies the base model for the adaptation. diff --git a/examples/server/server.cpp b/examples/server/server.cpp index 2decd7762..912c750cc 100644 --- a/examples/server/server.cpp +++ b/examples/server/server.cpp @@ -1855,7 +1855,10 @@ static void server_print_usage(const char *argv0, const gpt_params ¶ms, { printf(" --no-mmap do not memory-map model (slower load but may reduce pageouts if not using mlock)\n"); } - printf(" --numa attempt optimizations that help on some NUMA systems\n"); + printf(" --numa TYPE attempt optimizations that help on some NUMA systems\n"); + printf(" - distribute: spread execution evenly over all nodes\n"); + printf(" - isolate: only spawn threads on CPUs on the node that execution started on\n"); + printf(" - numactl: use the CPU map provided my numactl\n"); if (llama_supports_gpu_offload()) { printf(" -ngl N, --n-gpu-layers N\n"); printf(" number of layers to store in VRAM\n"); @@ -2264,9 +2267,17 @@ static void server_params_parse(int argc, char **argv, server_params &sparams, { params.use_mmap = false; } - else if (arg == "--numa") - { - params.numa = true; + else if (arg == "--numa") { + if (++i >= argc) { + invalid_param = true; + break; + } else { + std::string value(argv[i]); + /**/ if (value == "distribute" || value == "" ) { params.numa = GGML_NUMA_STRATEGY_DISTRIBUTE; } + else if (value == "isolate") { params.numa = GGML_NUMA_STRATEGY_ISOLATE; } + else if (value == "numactl") { params.numa = GGML_NUMA_STRATEGY_NUMACTL; } + else { invalid_param = true; break; } + } } else if (arg == "--embedding") { @@ -2497,7 +2508,8 @@ int main(int argc, char **argv) params.model_alias = params.model; } - llama_backend_init(params.numa); + llama_backend_init(); + llama_numa_init(params.numa); LOG_INFO("build info", {{"build", LLAMA_BUILD_NUMBER}, {"commit", LLAMA_COMMIT}}); diff --git a/examples/simple/simple.cpp b/examples/simple/simple.cpp index 9cfde8308..39e2d8ea4 100644 --- a/examples/simple/simple.cpp +++ b/examples/simple/simple.cpp @@ -31,7 +31,8 @@ int main(int argc, char ** argv) { // init LLM - llama_backend_init(params.numa); + llama_backend_init(); + llama_numa_init(params.numa); // initialize the model diff --git a/examples/speculative/speculative.cpp b/examples/speculative/speculative.cpp index 7b3af01f3..3848791d4 100644 --- a/examples/speculative/speculative.cpp +++ b/examples/speculative/speculative.cpp @@ -50,7 +50,8 @@ int main(int argc, char ** argv) { #endif // LOG_DISABLE_LOGS // init llama.cpp - llama_backend_init(params.numa); + llama_backend_init(); + llama_numa_init(params.numa); llama_model * model_tgt = NULL; llama_model * model_dft = NULL; diff --git a/examples/tokenize/tokenize.cpp b/examples/tokenize/tokenize.cpp index 4ff8e3fa7..d95a92475 100644 --- a/examples/tokenize/tokenize.cpp +++ b/examples/tokenize/tokenize.cpp @@ -17,7 +17,7 @@ int main(int argc, char ** argv) { const bool printing_ids = argc > 3 && std::string(argv[3]) == "--ids"; - llama_backend_init(false); + llama_backend_init(); llama_model_params model_params = llama_model_default_params(); model_params.vocab_only = true; diff --git a/ggml.c b/ggml.c index d921d82fe..4e302fb7d 100644 --- a/ggml.c +++ b/ggml.c @@ -1954,9 +1954,16 @@ struct ggml_numa_node { }; struct ggml_numa_nodes { + enum ggml_numa_strategy numa_strategy; struct ggml_numa_node nodes[GGML_NUMA_MAX_NODES]; uint32_t n_nodes; uint32_t total_cpus; // hardware threads on system + uint32_t current_node; // node on which main process is execting +#ifdef __linux__ + cpu_set_t cpuset; // cpuset from numactl +#else + uint32_t cpuset; // no NUMA support outside of Linux at this time. Use a portable datatype +#endif }; // @@ -1990,7 +1997,22 @@ inline static void ggml_critical_section_end(void) { atomic_fetch_sub(&g_state_barrier, 1); } -void ggml_numa_init(void) { +#ifdef __linux__ +static cpu_set_t ggml_get_numa_affinity(void) { + cpu_set_t cpuset; + pthread_t thread; + thread = pthread_self(); + CPU_ZERO(&cpuset); + pthread_getaffinity_np(thread, sizeof(cpu_set_t), &cpuset); + return cpuset; +} +#else +static uint32_t ggml_get_numa_affinity(void) { + return 0; // no NUMA support +} +#endif + +void ggml_numa_init(enum ggml_numa_strategy numa_flag) { if (g_state.numa.n_nodes > 0) { fprintf(stderr, "ggml_numa_init: NUMA already initialized\n"); @@ -2002,6 +2024,13 @@ void ggml_numa_init(void) { char path[256]; int rv; + // set numa scheme + g_state.numa.numa_strategy = numa_flag; + + GGML_PRINT_DEBUG("numa strategy %u\n",g_state.numa.numa_strategy); + + g_state.numa.cpuset = ggml_get_numa_affinity(); + // enumerate nodes while (g_state.numa.n_nodes < GGML_NUMA_MAX_NODES) { rv = snprintf(path, sizeof(path), "/sys/devices/system/node/node%u", g_state.numa.n_nodes); @@ -2020,11 +2049,17 @@ void ggml_numa_init(void) { GGML_PRINT_DEBUG("found %u numa nodes, %u CPUs\n", g_state.numa.n_nodes, g_state.numa.total_cpus); - if (g_state.numa.n_nodes < 1 || g_state.numa.total_cpus < 1) { + // figure out which node we're on + uint current_cpu; + int getcpu_ret = getcpu(¤t_cpu, &g_state.numa.current_node); + + if (g_state.numa.n_nodes < 1 || g_state.numa.total_cpus < 1 || getcpu_ret != 0) { g_state.numa.n_nodes = 0; return; } + GGML_PRINT_DEBUG("found our process on numa node %u, CPU %u\n", g_state.numa.current_node, current_cpu); + for (uint32_t n = 0; n < g_state.numa.n_nodes; ++n) { struct ggml_numa_node * node = &g_state.numa.nodes[n]; GGML_PRINT_DEBUG("CPUs on node %u:", n); @@ -16638,26 +16673,46 @@ typedef pthread_t ggml_thread_t; // Android's libc implementation "bionic" does not support setting affinity #if defined(__linux__) && !defined(__BIONIC__) -static void set_numa_thread_affinity(int thread_n, int n_threads) { +static void set_numa_thread_affinity(int thread_n) { if (!ggml_is_numa()) { return; } - // run thread on node_num thread_n / (threads per node) - const int node_num = thread_n / ((n_threads + g_state.numa.n_nodes - 1) / g_state.numa.n_nodes); - struct ggml_numa_node * node = &g_state.numa.nodes[node_num]; + int node_num; + int rv; size_t setsize = CPU_ALLOC_SIZE(g_state.numa.total_cpus); + switch(g_state.numa.numa_strategy) { + case GGML_NUMA_STRATEGY_DISTRIBUTE: + // run thread on node_num thread_n / (threads per node) + node_num = thread_n % g_state.numa.n_nodes; + break; + case GGML_NUMA_STRATEGY_ISOLATE: + // run thread on current_node + node_num = g_state.numa.current_node; + break; + case GGML_NUMA_STRATEGY_NUMACTL: + // use the cpuset that numactl gave us + rv = pthread_setaffinity_np(pthread_self(), setsize, &g_state.numa.cpuset); + if (rv) { + fprintf(stderr, "warning: pthread_setaffinity_np() failed: %s\n",strerror(rv)); + } + return; + default: + return; + } + + struct ggml_numa_node * node = &g_state.numa.nodes[node_num]; + cpu_set_t * cpus = CPU_ALLOC(g_state.numa.total_cpus); CPU_ZERO_S(setsize, cpus); for (size_t i = 0; i < node->n_cpus; ++i) { CPU_SET_S(node->cpus[i], setsize, cpus); } - int rv = pthread_setaffinity_np(pthread_self(), setsize, cpus); + rv = pthread_setaffinity_np(pthread_self(), setsize, cpus); if (rv) { - fprintf(stderr, "warning: pthread_setaffinity_np() failed: %s\n", - strerror(rv)); + fprintf(stderr, "warning: pthread_setaffinity_np() failed: %s\n", strerror(rv)); } CPU_FREE(cpus); @@ -16678,8 +16733,7 @@ static void clear_numa_thread_affinity(void) { int rv = pthread_setaffinity_np(pthread_self(), setsize, cpus); if (rv) { - fprintf(stderr, "warning: pthread_setaffinity_np() failed: %s\n", - strerror(rv)); + fprintf(stderr, "warning: pthread_setaffinity_np() failed: %s\n", strerror(rv)); } CPU_FREE(cpus); @@ -16687,7 +16741,7 @@ static void clear_numa_thread_affinity(void) { #else // TODO: Windows etc. // (the linux implementation may also work on BSD, someone should test) -static void set_numa_thread_affinity(int thread_n, int n_threads) { UNUSED(thread_n); UNUSED(n_threads); } +static void set_numa_thread_affinity(int thread_n) { UNUSED(thread_n); } static void clear_numa_thread_affinity(void) {} #endif @@ -16987,7 +17041,7 @@ static thread_ret_t ggml_graph_compute_thread(void * data) { const int n_threads = state->shared->n_threads; - set_numa_thread_affinity(state->ith, n_threads); + set_numa_thread_affinity(state->ith); int node_n = -1; int task_phase = GGML_TASK_FINALIZE; diff --git a/ggml.h b/ggml.h index 01cecc1e1..270018185 100644 --- a/ggml.h +++ b/ggml.h @@ -658,6 +658,16 @@ extern "C" { void * wdata; }; + // numa strategies + enum ggml_numa_strategy { + GGML_NUMA_STRATEGY_DISABLED = 0, + GGML_NUMA_STRATEGY_DISTRIBUTE = 1, + GGML_NUMA_STRATEGY_ISOLATE = 2, + GGML_NUMA_STRATEGY_NUMACTL = 3, + GGML_NUMA_STRATEGY_MIRROR = 4, + GGML_NUMA_STRATEGY_COUNT + }; + // misc GGML_API void ggml_time_init(void); // call this once at the beginning of the program @@ -668,7 +678,7 @@ extern "C" { GGML_API void ggml_print_backtrace(void); - GGML_API void ggml_numa_init(void); // call once for better performance on NUMA systems + GGML_API void ggml_numa_init(enum ggml_numa_strategy numa); // call once for better performance on NUMA systems GGML_API bool ggml_is_numa(void); // true if init detected that system has >1 NUMA node GGML_API void ggml_print_object (const struct ggml_object * obj); diff --git a/llama.cpp b/llama.cpp index aceb9c25a..08e7b02b4 100644 --- a/llama.cpp +++ b/llama.cpp @@ -1034,7 +1034,7 @@ struct llama_mmap { int fd = fileno(file->fp); int flags = MAP_SHARED; // prefetch/readahead impairs performance on NUMA systems - if (numa) { prefetch = 0; } + if (numa) { prefetch = 0; } #ifdef __linux__ // advise the kernel to read the file sequentially (increases readahead) if (posix_fadvise(fd, 0, 0, POSIX_FADV_SEQUENTIAL)) { @@ -11182,7 +11182,7 @@ bool llama_mlock_supported(void) { return llama_supports_mlock(); } -void llama_backend_init(bool numa) { +void llama_backend_init(void) { ggml_time_init(); // needed to initialize f16 tables @@ -11192,15 +11192,17 @@ void llama_backend_init(bool numa) { ggml_free(ctx); } - if (numa) { - ggml_numa_init(); - } - #ifdef GGML_USE_MPI ggml_mpi_backend_init(); #endif } +void llama_numa_init(enum ggml_numa_strategy numa) { + if (numa != GGML_NUMA_STRATEGY_DISABLED) { + ggml_numa_init(numa); + } +} + void llama_backend_free(void) { #ifdef GGML_USE_MPI ggml_mpi_backend_free(); diff --git a/llama.h b/llama.h index 4a26bd619..f4ec6ea63 100644 --- a/llama.h +++ b/llama.h @@ -312,7 +312,10 @@ extern "C" { // Initialize the llama + ggml backend // If numa is true, use NUMA optimizations // Call once at the start of the program - LLAMA_API void llama_backend_init(bool numa); + LLAMA_API void llama_backend_init(void); + + //optional: + LLAMA_API void llama_numa_init(enum ggml_numa_strategy numa); // Call once at the end of the program - currently only used for MPI LLAMA_API void llama_backend_free(void); diff --git a/tests/test-autorelease.cpp b/tests/test-autorelease.cpp index 36a23c0bb..57fa00011 100644 --- a/tests/test-autorelease.cpp +++ b/tests/test-autorelease.cpp @@ -12,7 +12,7 @@ int main(int argc, char ** argv) { auto * model_path = get_model_or_exit(argc, argv); std::thread([&model_path]() { - llama_backend_init(false); + llama_backend_init(); auto * model = llama_load_model_from_file(model_path, llama_model_default_params()); auto * ctx = llama_new_context_with_model(model, llama_context_default_params()); llama_free(ctx); diff --git a/tests/test-model-load-cancel.cpp b/tests/test-model-load-cancel.cpp index 7ea4bbacc..858535c3c 100644 --- a/tests/test-model-load-cancel.cpp +++ b/tests/test-model-load-cancel.cpp @@ -14,7 +14,7 @@ int main(int argc, char *argv[] ) { fprintf(stderr, "using '%s'\n", model_path); fclose(file); - llama_backend_init(false); + llama_backend_init(); auto params = llama_model_params{}; params.use_mmap = false; params.progress_callback = [](float progress, void * ctx){ diff --git a/tests/test-tokenizer-0-falcon.cpp b/tests/test-tokenizer-0-falcon.cpp index a4e9d2b91..472b0b3a8 100644 --- a/tests/test-tokenizer-0-falcon.cpp +++ b/tests/test-tokenizer-0-falcon.cpp @@ -61,7 +61,7 @@ int main(int argc, char **argv) { llama_model * model; llama_context * ctx; - llama_backend_init(false); + llama_backend_init(); // load the vocab { diff --git a/tests/test-tokenizer-0-llama.cpp b/tests/test-tokenizer-0-llama.cpp index 39c8d188c..0a16cd7eb 100644 --- a/tests/test-tokenizer-0-llama.cpp +++ b/tests/test-tokenizer-0-llama.cpp @@ -60,7 +60,7 @@ int main(int argc, char **argv) { llama_model * model; llama_context * ctx; - llama_backend_init(false); + llama_backend_init(); // load the vocab { diff --git a/tests/test-tokenizer-1-bpe.cpp b/tests/test-tokenizer-1-bpe.cpp index 3bb629561..3596ce55a 100644 --- a/tests/test-tokenizer-1-bpe.cpp +++ b/tests/test-tokenizer-1-bpe.cpp @@ -25,7 +25,7 @@ int main(int argc, char **argv) { llama_model * model; llama_context * ctx; - llama_backend_init(false); + llama_backend_init(); // load the vocab { diff --git a/tests/test-tokenizer-1-llama.cpp b/tests/test-tokenizer-1-llama.cpp index b0d814a41..9333f8686 100644 --- a/tests/test-tokenizer-1-llama.cpp +++ b/tests/test-tokenizer-1-llama.cpp @@ -25,7 +25,7 @@ int main(int argc, char **argv) { llama_model * model; llama_context * ctx; - llama_backend_init(false); + llama_backend_init(); // load the vocab { From 5f5808ca7b7f23a1fa7a77241842bb84a0e55108 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C5=91czey=20Barnab=C3=A1s?= <31726601+An0nie@users.noreply.github.com> Date: Fri, 16 Feb 2024 11:00:56 +0100 Subject: [PATCH 358/812] server : fix system prompt cli (#5516) --- examples/server/server.cpp | 47 ++++++++++++++++++-------------------- 1 file changed, 22 insertions(+), 25 deletions(-) diff --git a/examples/server/server.cpp b/examples/server/server.cpp index 912c750cc..0cb802ce8 100644 --- a/examples/server/server.cpp +++ b/examples/server/server.cpp @@ -436,10 +436,6 @@ struct llama_server_context default_generation_settings_for_props["seed"] = -1; batch = llama_batch_init(n_ctx, 0, params.n_parallel); - - // empty system prompt - system_prompt = ""; - system_tokens.clear(); } std::vector tokenize(const json & json_prompt, bool add_bos) const @@ -765,27 +761,30 @@ struct llama_server_context } void update_system_prompt() { - system_tokens = ::llama_tokenize(ctx, system_prompt, add_bos_token); - - llama_batch_clear(batch); - kv_cache_clear(); + system_tokens.clear(); - for (int i = 0; i < (int) system_tokens.size(); ++i) - { - llama_batch_add(batch, system_tokens[i], i, { 0 }, false); - } + if (!system_prompt.empty()) { + system_tokens = ::llama_tokenize(ctx, system_prompt, add_bos_token); - if (llama_decode(ctx, batch) != 0) - { - LOG_TEE("%s: llama_decode() failed\n", __func__); - return; - } + llama_batch_clear(batch); - // assign the system KV cache to all parallel sequences - for (int32_t i = 1; i < params.n_parallel; ++i) - { - llama_kv_cache_seq_cp(ctx, 0, i, 0, system_tokens.size()); + for (int i = 0; i < (int)system_tokens.size(); ++i) + { + llama_batch_add(batch, system_tokens[i], i, { 0 }, false); + } + + if (llama_decode(ctx, batch) != 0) + { + LOG_TEE("%s: llama_decode() failed\n", __func__); + return; + } + + // assign the system KV cache to all parallel sequences + for (int32_t i = 1; i < params.n_parallel; ++i) + { + llama_kv_cache_seq_cp(ctx, 0, i, 0, system_tokens.size()); + } } LOG_TEE("system prompt updated\n"); @@ -807,10 +806,8 @@ struct llama_server_context name_user = sys_props.value("anti_prompt", ""); name_assistant = sys_props.value("assistant_name", ""); - if (slots.size() > 0) - { - notify_system_prompt_changed(); - } + + notify_system_prompt_changed(); } static size_t find_stopping_strings(const std::string &text, const size_t last_token_size, From 6dcc02d2444c779c18d49c364c5d5c5728b6b484 Mon Sep 17 00:00:00 2001 From: Alexey Parfenov Date: Fri, 16 Feb 2024 11:33:25 +0000 Subject: [PATCH 359/812] server : add "samplers" param to control the samplers order (#5494) --- common/common.cpp | 59 ++++++++++++++++++++++++-------------- common/common.h | 2 +- common/sampling.cpp | 2 +- common/sampling.h | 14 ++++----- examples/server/README.md | 2 ++ examples/server/server.cpp | 25 ++++++++++++++++ 6 files changed, 74 insertions(+), 30 deletions(-) diff --git a/common/common.cpp b/common/common.cpp index c5e83cc2a..3a92d3797 100644 --- a/common/common.cpp +++ b/common/common.cpp @@ -341,7 +341,7 @@ bool gpt_params_parse_ex(int argc, char ** argv, gpt_params & params) { break; } const auto sampler_names = string_split(argv[i], ';'); - sparams.samplers_sequence = sampler_types_from_names(sampler_names); + sparams.samplers_sequence = sampler_types_from_names(sampler_names, true); } else if (arg == "--sampling-seq") { if (++i >= argc) { invalid_param = true; @@ -964,7 +964,8 @@ void gpt_print_usage(int /*argc*/, char ** argv, const gpt_params & params) { printf(" -n N, --n-predict N number of tokens to predict (default: %d, -1 = infinity, -2 = until context filled)\n", params.n_predict); printf(" -c N, --ctx-size N size of the prompt context (default: %d, 0 = loaded from model)\n", params.n_ctx); printf(" -b N, --batch-size N batch size for prompt processing (default: %d)\n", params.n_batch); - printf(" --samplers samplers that will be used for generation in the order, separated by \';\' (default: %s)\n", sampler_type_names.c_str()); + printf(" --samplers samplers that will be used for generation in the order, separated by \';\'\n"); + printf(" (default: %s)\n", sampler_type_names.c_str()); printf(" --sampling-seq simplified sequence for samplers that will be used (default: %s)\n", sampler_type_chars.c_str()); printf(" --top-k N top-k sampling (default: %d, 0 = disabled)\n", sparams.top_k); printf(" --top-p N top-p sampling (default: %.1f, 1.0 = disabled)\n", (double)sparams.top_p); @@ -1133,34 +1134,50 @@ std::vector string_split(std::string input, char separator) { return parts; } -std::vector sampler_types_from_names(const std::vector & names) { +std::vector sampler_types_from_names(const std::vector & names, bool allow_alt_names) { + std::unordered_map sampler_canonical_name_map { + {"top_k", llama_sampler_type::TOP_K}, + {"top_p", llama_sampler_type::TOP_P}, + {"typical_p", llama_sampler_type::TYPICAL_P}, + {"min_p", llama_sampler_type::MIN_P}, + {"tfs_z", llama_sampler_type::TFS_Z}, + {"temperature", llama_sampler_type::TEMPERATURE} + }; + // since samplers names are written multiple ways // make it ready for both system names and input names - std::unordered_map sampler_name_map { - {"top_k", llama_sampler_type::TOP_K}, + std::unordered_map sampler_alt_name_map { {"top-k", llama_sampler_type::TOP_K}, - {"top_p", llama_sampler_type::TOP_P}, {"top-p", llama_sampler_type::TOP_P}, {"nucleus", llama_sampler_type::TOP_P}, - {"typical_p", llama_sampler_type::TYPICAL_P}, {"typical-p", llama_sampler_type::TYPICAL_P}, {"typical", llama_sampler_type::TYPICAL_P}, - {"min_p", llama_sampler_type::MIN_P}, {"min-p", llama_sampler_type::MIN_P}, - {"tfs_z", llama_sampler_type::TFS_Z}, {"tfs-z", llama_sampler_type::TFS_Z}, {"tfs", llama_sampler_type::TFS_Z}, - {"temp", llama_sampler_type::TEMP}, - {"temperature", llama_sampler_type::TEMP} + {"temp", llama_sampler_type::TEMPERATURE} }; std::vector sampler_types; sampler_types.reserve(names.size()); - for (const auto& name : names) { - const auto sampler_item = sampler_name_map.find(name); - if (sampler_item != sampler_name_map.end()) { + for (const auto & name : names) + { + auto sampler_item = sampler_canonical_name_map.find(name); + if (sampler_item != sampler_canonical_name_map.end()) + { sampler_types.push_back(sampler_item->second); } + else + { + if (allow_alt_names) + { + sampler_item = sampler_alt_name_map.find(name); + if (sampler_item != sampler_alt_name_map.end()) + { + sampler_types.push_back(sampler_item->second); + } + } + } } return sampler_types; } @@ -1172,7 +1189,7 @@ std::vector sampler_types_from_chars(const std::string & nam {'y', llama_sampler_type::TYPICAL_P}, {'m', llama_sampler_type::MIN_P}, {'f', llama_sampler_type::TFS_Z}, - {'t', llama_sampler_type::TEMP} + {'t', llama_sampler_type::TEMPERATURE} }; std::vector sampler_types; @@ -1188,12 +1205,12 @@ std::vector sampler_types_from_chars(const std::string & nam std::string sampler_type_to_name_string(llama_sampler_type sampler_type) { switch (sampler_type) { - case llama_sampler_type::TOP_K: return "top_k"; - case llama_sampler_type::TFS_Z: return "tfs_z"; - case llama_sampler_type::TYPICAL_P: return "typical_p"; - case llama_sampler_type::TOP_P: return "top_p"; - case llama_sampler_type::MIN_P: return "min_p"; - case llama_sampler_type::TEMP: return "temp"; + case llama_sampler_type::TOP_K: return "top_k"; + case llama_sampler_type::TFS_Z: return "tfs_z"; + case llama_sampler_type::TYPICAL_P: return "typical_p"; + case llama_sampler_type::TOP_P: return "top_p"; + case llama_sampler_type::MIN_P: return "min_p"; + case llama_sampler_type::TEMPERATURE: return "temperature"; default : return ""; } } diff --git a/common/common.h b/common/common.h index 74c136995..935771d44 100644 --- a/common/common.h +++ b/common/common.h @@ -165,7 +165,7 @@ void process_escapes(std::string& input); // String utils // -std::vector sampler_types_from_names(const std::vector & names); +std::vector sampler_types_from_names(const std::vector & names, bool allow_alt_names); std::vector sampler_types_from_chars(const std::string & names_string); std::vector string_split(std::string input, char separator); std::string sampler_type_to_name_string(llama_sampler_type sampler_type); diff --git a/common/sampling.cpp b/common/sampling.cpp index a001750da..53013138a 100644 --- a/common/sampling.cpp +++ b/common/sampling.cpp @@ -139,7 +139,7 @@ static void sampler_queue( case llama_sampler_type::TYPICAL_P: llama_sample_typical (ctx_main, &cur_p, typical_p, min_keep); break; case llama_sampler_type::TOP_P : llama_sample_top_p (ctx_main, &cur_p, top_p, min_keep); break; case llama_sampler_type::MIN_P : llama_sample_min_p (ctx_main, &cur_p, min_p, min_keep); break; - case llama_sampler_type::TEMP: + case llama_sampler_type::TEMPERATURE: if (dynatemp_range > 0) { float dynatemp_min = std::max(0.0f, temp - dynatemp_range); float dynatemp_max = std::max(0.0f, temp + dynatemp_range); diff --git a/common/sampling.h b/common/sampling.h index 2bd6a75d2..e1279a894 100644 --- a/common/sampling.h +++ b/common/sampling.h @@ -10,12 +10,12 @@ // sampler types enum class llama_sampler_type : char { - TOP_K = 'k', - TOP_P = 'p', - MIN_P = 'm', - TFS_Z = 'f', - TYPICAL_P = 'y', - TEMP = 't' + TOP_K = 'k', + TOP_P = 'p', + MIN_P = 'm', + TFS_Z = 'f', + TYPICAL_P = 'y', + TEMPERATURE = 't' }; // sampling parameters @@ -45,7 +45,7 @@ typedef struct llama_sampling_params { llama_sampler_type::TYPICAL_P, llama_sampler_type::TOP_P, llama_sampler_type::MIN_P, - llama_sampler_type::TEMP + llama_sampler_type::TEMPERATURE }; std::string grammar; // optional BNF-like grammar to constrain sampling diff --git a/examples/server/README.md b/examples/server/README.md index 8e141d22d..249368749 100644 --- a/examples/server/README.md +++ b/examples/server/README.md @@ -204,6 +204,8 @@ node index.js `system_prompt`: Change the system prompt (initial prompt of all slots), this is useful for chat applications. [See more](#change-system-prompt-on-runtime) + `samplers`: The order the samplers should be applied in. An array of strings representing sampler type names. If a sampler is not set, it will not be used. If a sampler is specified more than once, it will be applied multiple times. (default: `["top_k", "tfs_z", "typical_p", "top_p", "min_p", "temperature"]` - these are all the available values) + ### Result JSON - Note: When using streaming mode (`stream`) only `content` and `stop` will be returned until end of completion. diff --git a/examples/server/server.cpp b/examples/server/server.cpp index 0cb802ce8..a0b46970b 100644 --- a/examples/server/server.cpp +++ b/examples/server/server.cpp @@ -672,6 +672,24 @@ struct llama_server_context } } + const auto &samplers_sequence = data.find("samplers"); + if (samplers_sequence != data.end() && samplers_sequence->is_array()) + { + std::vector sampler_names; + for (const auto &sampler_name : *samplers_sequence) + { + if (sampler_name.is_string()) + { + sampler_names.emplace_back(sampler_name); + } + } + slot->sparams.samplers_sequence = sampler_types_from_names(sampler_names, false); + } + else + { + slot->sparams.samplers_sequence = default_sparams.samplers_sequence; + } + if (multimodal) { const auto &images_data = data.find("image_data"); @@ -1026,6 +1044,12 @@ struct llama_server_context const auto eos_bias = slot.sparams.logit_bias.find(llama_token_eos(model)); const bool ignore_eos = eos_bias != slot.sparams.logit_bias.end() && eos_bias->second < 0.0f && std::isinf(eos_bias->second); + std::vector samplers_sequence; + for (const auto &sampler_type : slot.sparams.samplers_sequence) + { + samplers_sequence.emplace_back(sampler_type_to_name_string(sampler_type)); + } + return json { {"n_ctx", slot.n_ctx}, {"model", params.model_alias}, @@ -1056,6 +1080,7 @@ struct llama_server_context {"logit_bias", slot.sparams.logit_bias}, {"n_probs", slot.sparams.n_probs}, {"grammar", slot.sparams.grammar}, + {"samplers", samplers_sequence} }; } From 65085c713e14f78cdda6abc275b1a5d8c2b8ca15 Mon Sep 17 00:00:00 2001 From: Herman Semenov Date: Fri, 16 Feb 2024 11:45:48 +0000 Subject: [PATCH 360/812] llama : minor fixed return int value (#5529) --- llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llama.cpp b/llama.cpp index 08e7b02b4..8966c3e66 100644 --- a/llama.cpp +++ b/llama.cpp @@ -10893,7 +10893,7 @@ static int llama_apply_lora_from_file_internal( { LLAMA_LOG_ERROR("%s: invalid tensor data type '%d'\n", __func__, ftype); - return false; + return 1; } } From 4cb072769804c77ab466bc8351c76ede9d5ba49d Mon Sep 17 00:00:00 2001 From: Herman Semenov Date: Fri, 16 Feb 2024 12:43:23 +0000 Subject: [PATCH 361/812] llava : removed excess free(NULL) operation (#5531) --- examples/llava/llava.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/examples/llava/llava.cpp b/examples/llava/llava.cpp index 4ed310a0e..4cb65a07b 100644 --- a/examples/llava/llava.cpp +++ b/examples/llava/llava.cpp @@ -315,7 +315,6 @@ static bool llava_image_embed_make_with_clip_img(clip_ctx * ctx_clip, int n_thre float * image_embd = (float *)malloc(clip_embd_nbytes(ctx_clip)*6); // TODO: base on gridsize/llava model if (!image_embd) { fprintf(stderr, "Unable to allocate memory for image embeddings\n"); - free(image_embd); return false; } From d2819d5577b35507be83d0c3f4d2d3c0ab1488ca Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Fri, 16 Feb 2024 15:14:40 +0200 Subject: [PATCH 362/812] scripts : add helpers script for bench comparing commits (#5521) * scripts : add helpers script for bench comparing commits * scripts : detect CUDA * set flags after checking the command line * fix make flags --------- Co-authored-by: slaren --- scripts/compare-commits.sh | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100755 scripts/compare-commits.sh diff --git a/scripts/compare-commits.sh b/scripts/compare-commits.sh new file mode 100755 index 000000000..331c4b9ce --- /dev/null +++ b/scripts/compare-commits.sh @@ -0,0 +1,37 @@ +#!/bin/bash + +if [ $# -lt 2 ]; then + echo "usage: ./scripts/compare-commits.sh [additional llama-bench arguments]" + exit 1 +fi + +set -e +set -x + +bench_args="${@:3}" + +rm -f llama-bench.sqlite + +backend="cpu" + +if [[ "$OSTYPE" == "darwin"* ]]; then + backend="metal" +elif command -v nvcc &> /dev/null; then + backend="cuda" +fi + +make_opts="" + +if [[ "$backend" == "cuda" ]]; then + make_opts="LLAMA_CUBLAS=1" +fi + +git checkout $1 +make clean && make -j32 $make_opts llama-bench +./llama-bench -o sql $bench_args | tee /dev/tty | sqlite3 llama-bench.sqlite + +git checkout $2 +make clean && make -j32 $make_opts llama-bench +./llama-bench -o sql $bench_args | tee /dev/tty | sqlite3 llama-bench.sqlite + +./scripts/compare-llama-bench.py -b $1 -c $2 From 5bf2b94dd4fb74378b78604023b31512fec55f8f Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Fri, 16 Feb 2024 19:05:56 +0200 Subject: [PATCH 363/812] cmake : fix VULKAN and ROCm builds (#5525) * cmake : fix VULKAN and ROCm builds * cmake : fix (cont) * vulkan : fix compile warnings ggml-ci * cmake : fix ggml-ci * cmake : minor ggml-ci --- CMakeLists.txt | 379 +++++++++++++++++++++++++----------------------- ggml-vulkan.cpp | 12 +- 2 files changed, 205 insertions(+), 186 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index f8c7f9978..2a922fdb3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -112,17 +112,14 @@ option(LLAMA_MPI "llama: use MPI" option(LLAMA_QKK_64 "llama: use super-block size of 64 for k-quants" OFF) option(LLAMA_SYCL "llama: use SYCL" OFF) option(LLAMA_SYCL_F16 "llama: use 16 bit floats for sycl calculations" OFF) +option(LLAMA_CPU_HBM "llama: use memkind for CPU HBM" OFF) option(LLAMA_BUILD_TESTS "llama: build tests" ${LLAMA_STANDALONE}) option(LLAMA_BUILD_EXAMPLES "llama: build examples" ${LLAMA_STANDALONE}) option(LLAMA_BUILD_SERVER "llama: build server example" ON) - # add perf arguments option(LLAMA_PERF "llama: enable perf" OFF) -if (LLAMA_PERF) - add_definitions(-DGGML_PERF) -endif() # Required for relocatable CMake package include(${CMAKE_CURRENT_SOURCE_DIR}/scripts/build-info.cmake) @@ -130,6 +127,7 @@ include(${CMAKE_CURRENT_SOURCE_DIR}/scripts/build-info.cmake) # # Compile flags # + if (LLAMA_SYCL) set(CMAKE_CXX_STANDARD 17) else() @@ -140,6 +138,7 @@ set(CMAKE_CXX_STANDARD_REQUIRED true) set(CMAKE_C_STANDARD 11) set(CMAKE_C_STANDARD_REQUIRED true) set(THREADS_PREFER_PTHREAD_FLAG ON) + find_package(Threads REQUIRED) include(CheckCXXCompilerFlag) @@ -151,17 +150,17 @@ endif() if (NOT MSVC) if (LLAMA_SANITIZE_THREAD) add_compile_options(-fsanitize=thread) - link_libraries(-fsanitize=thread) + link_libraries (-fsanitize=thread) endif() if (LLAMA_SANITIZE_ADDRESS) add_compile_options(-fsanitize=address -fno-omit-frame-pointer) - link_libraries(-fsanitize=address) + link_libraries (-fsanitize=address) endif() if (LLAMA_SANITIZE_UNDEFINED) add_compile_options(-fsanitize=undefined) - link_libraries(-fsanitize=undefined) + link_libraries (-fsanitize=undefined) endif() endif() @@ -298,14 +297,17 @@ if (LLAMA_BLAS) endif() message(STATUS "BLAS found, Includes: ${BLAS_INCLUDE_DIRS}") + add_compile_options(${BLAS_LINKER_FLAGS}) + add_compile_definitions(GGML_USE_OPENBLAS) + if (${BLAS_INCLUDE_DIRS} MATCHES "mkl" AND (${LLAMA_BLAS_VENDOR} MATCHES "Generic" OR ${LLAMA_BLAS_VENDOR} MATCHES "Intel")) add_compile_definitions(GGML_BLAS_USE_MKL) endif() - set(LLAMA_EXTRA_LIBS ${LLAMA_EXTRA_LIBS} ${BLAS_LIBRARIES}) - set(LLAMA_EXTRA_INCLUDES ${LLAMA_EXTRA_INCLUDES} ${BLAS_INCLUDE_DIRS}) + set(LLAMA_EXTRA_LIBS ${LLAMA_EXTRA_LIBS} ${BLAS_LIBRARIES}) + set(LLAMA_EXTRA_INCLUDES ${LLAMA_EXTRA_INCLUDES} ${BLAS_INCLUDE_DIRS}) else() message(WARNING "BLAS not found, please refer to " "https://cmake.org/cmake/help/latest/module/FindBLAS.html#blas-lapack-vendors" @@ -330,9 +332,6 @@ if (LLAMA_CUBLAS) set(GGML_SOURCES_CUDA ggml-cuda.cu) add_compile_definitions(GGML_USE_CUBLAS) -# if (LLAMA_CUDA_CUBLAS) -# add_compile_definitions(GGML_CUDA_CUBLAS) -# endif() if (LLAMA_CUDA_FORCE_DMMV) add_compile_definitions(GGML_CUDA_FORCE_DMMV) endif() @@ -387,15 +386,20 @@ if (LLAMA_MPI) find_package(MPI) if (MPI_C_FOUND) message(STATUS "MPI found") + set(GGML_HEADERS_MPI ggml-mpi.h) - set(GGML_SOURCES_MPI ggml-mpi.c ggml-mpi.h) + set(GGML_SOURCES_MPI ggml-mpi.c) + add_compile_definitions(GGML_USE_MPI) add_compile_definitions(${MPI_C_COMPILE_DEFINITIONS}) + if (NOT MSVC) add_compile_options(-Wno-cast-qual) endif() + set(LLAMA_EXTRA_LIBS ${LLAMA_EXTRA_LIBS} ${MPI_C_LIBRARIES}) set(LLAMA_EXTRA_INCLUDES ${LLAMA_EXTRA_INCLUDES} ${MPI_C_INCLUDE_DIRS}) + # Even if you're only using the C header, C++ programs may bring in MPI # C++ functions, so more linkage is needed if (MPI_CXX_FOUND) @@ -427,31 +431,28 @@ if (LLAMA_VULKAN) if (Vulkan_FOUND) message(STATUS "Vulkan found") - add_library(ggml-vulkan OBJECT ggml-vulkan.cpp ggml-vulkan.h) - if (BUILD_SHARED_LIBS) - set_target_properties(ggml-vulkan PROPERTIES POSITION_INDEPENDENT_CODE ON) - endif() - target_link_libraries(ggml-vulkan PRIVATE Vulkan::Vulkan) + set(GGML_HEADERS_VULKAN ggml-vulkan.h) + set(GGML_SOURCES_VULKAN ggml-vulkan.cpp) add_compile_definitions(GGML_USE_VULKAN) if (LLAMA_VULKAN_CHECK_RESULTS) - target_compile_definitions(ggml-vulkan PRIVATE GGML_VULKAN_CHECK_RESULTS) + add_compile_definitions(GGML_VULKAN_CHECK_RESULTS) endif() if (LLAMA_VULKAN_DEBUG) - target_compile_definitions(ggml-vulkan PRIVATE GGML_VULKAN_DEBUG) + add_compile_definitions(GGML_VULKAN_DEBUG) endif() if (LLAMA_VULKAN_VALIDATE) - target_compile_definitions(ggml-vulkan PRIVATE GGML_VULKAN_VALIDATE) + add_compile_definitions(GGML_VULKAN_VALIDATE) endif() if (LLAMA_VULKAN_RUN_TESTS) - target_compile_definitions(ggml-vulkan PRIVATE GGML_VULKAN_RUN_TESTS) + add_compile_definitions(GGML_VULKAN_RUN_TESTS) endif() - set(LLAMA_EXTRA_LIBS ${LLAMA_EXTRA_LIBS} ggml-vulkan) + set(LLAMA_EXTRA_LIBS ${LLAMA_EXTRA_LIBS} Vulkan::Vulkan) else() message(WARNING "Vulkan not found") endif() @@ -463,43 +464,45 @@ if (LLAMA_HIPBLAS) if (NOT ${CMAKE_C_COMPILER_ID} MATCHES "Clang") message(WARNING "Only LLVM is supported for HIP, hint: CC=/opt/rocm/llvm/bin/clang") endif() + if (NOT ${CMAKE_CXX_COMPILER_ID} MATCHES "Clang") message(WARNING "Only LLVM is supported for HIP, hint: CXX=/opt/rocm/llvm/bin/clang++") endif() - find_package(hip) - find_package(hipblas) - find_package(rocblas) + find_package(hip REQUIRED) + find_package(hipblas REQUIRED) + find_package(rocblas REQUIRED) - if (${hipblas_FOUND} AND ${hip_FOUND}) - message(STATUS "HIP and hipBLAS found") - add_compile_definitions(GGML_USE_HIPBLAS GGML_USE_CUBLAS) - if (LLAMA_HIP_UMA) - add_compile_definitions(GGML_HIP_UMA) - endif() - add_library(ggml-rocm OBJECT ggml-cuda.cu ggml-cuda.h) - if (BUILD_SHARED_LIBS) - set_target_properties(ggml-rocm PROPERTIES POSITION_INDEPENDENT_CODE ON) - endif() - if (LLAMA_CUDA_FORCE_DMMV) - target_compile_definitions(ggml-rocm PRIVATE GGML_CUDA_FORCE_DMMV) - endif() - if (LLAMA_CUDA_FORCE_MMQ) - target_compile_definitions(ggml-rocm PRIVATE GGML_CUDA_FORCE_MMQ) - endif() - target_compile_definitions(ggml-rocm PRIVATE GGML_CUDA_DMMV_X=${LLAMA_CUDA_DMMV_X}) - target_compile_definitions(ggml-rocm PRIVATE GGML_CUDA_MMV_Y=${LLAMA_CUDA_MMV_Y}) - target_compile_definitions(ggml-rocm PRIVATE K_QUANTS_PER_ITERATION=${LLAMA_CUDA_KQUANTS_ITER}) - set_source_files_properties(ggml-cuda.cu PROPERTIES LANGUAGE CXX) - target_link_libraries(ggml-rocm PRIVATE hip::device PUBLIC hip::host roc::rocblas roc::hipblas) + message(STATUS "HIP and hipBLAS found") - if (LLAMA_STATIC) - message(FATAL_ERROR "Static linking not supported for HIP/ROCm") - endif() - set(LLAMA_EXTRA_LIBS ${LLAMA_EXTRA_LIBS} ggml-rocm) - else() - message(WARNING "hipBLAS or HIP not found. Try setting CMAKE_PREFIX_PATH=/opt/rocm") + set(GGML_HEADERS_ROCM ggml-cuda.h) + set(GGML_SOURCES_ROCM ggml-cuda.cu) + + add_compile_definitions(GGML_USE_HIPBLAS GGML_USE_CUBLAS) + + if (LLAMA_HIP_UMA) + add_compile_definitions(GGML_HIP_UMA) endif() + + if (LLAMA_CUDA_FORCE_DMMV) + add_compile_definitions(GGML_CUDA_FORCE_DMMV) + endif() + + if (LLAMA_CUDA_FORCE_MMQ) + add_compile_definitions(GGML_CUDA_FORCE_MMQ) + endif() + + add_compile_definitions(GGML_CUDA_DMMV_X=${LLAMA_CUDA_DMMV_X}) + add_compile_definitions(GGML_CUDA_MMV_Y=${LLAMA_CUDA_MMV_Y}) + add_compile_definitions(K_QUANTS_PER_ITERATION=${LLAMA_CUDA_KQUANTS_ITER}) + + set_source_files_properties(ggml-cuda.cu PROPERTIES LANGUAGE CXX) + + if (LLAMA_STATIC) + message(FATAL_ERROR "Static linking not supported for HIP/ROCm") + endif() + + set(LLAMA_EXTRA_LIBS ${LLAMA_EXTRA_LIBS} hip::device PUBLIC hip::host roc::rocblas roc::hipblas) endif() if (LLAMA_SYCL) @@ -509,10 +512,14 @@ if (LLAMA_SYCL) #todo: AOT find_package(IntelSYCL REQUIRED) + + message(STATUS "SYCL found") + + add_compile_definitions(GML_USE_SYCL) + if (LLAMA_SYCL_F16) add_compile_definitions(GGML_SYCL_F16) endif() - add_compile_definitions(GGML_USE_SYCL) add_compile_options(-I./) #include DPCT add_compile_options(-I/${SYCL_INCLUDE_DIR}) @@ -521,7 +528,7 @@ if (LLAMA_SYCL) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O3") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsycl -L${MKLROOT}/lib") - set(GGML_HEADERS_SYCL ggml.h ggml-sycl.h) + set(GGML_HEADERS_SYCL ggml-sycl.h) set(GGML_SOURCES_SYCL ggml-sycl.cpp) if (WIN32) @@ -540,61 +547,61 @@ if (LLAMA_KOMPUTE) endif() function(compile_shader) - set(options) - set(oneValueArgs) - set(multiValueArgs SOURCES) - cmake_parse_arguments(compile_shader "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) - foreach(source ${compile_shader_SOURCES}) - get_filename_component(filename ${source} NAME) - set(spv_file ${filename}.spv) - add_custom_command( - OUTPUT ${spv_file} - DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/${source} - ${CMAKE_CURRENT_SOURCE_DIR}/kompute-shaders/common.comp - ${CMAKE_CURRENT_SOURCE_DIR}/kompute-shaders/op_getrows.comp - ${CMAKE_CURRENT_SOURCE_DIR}/kompute-shaders/op_mul_mv_q_n_pre.comp - ${CMAKE_CURRENT_SOURCE_DIR}/kompute-shaders/op_mul_mv_q_n.comp - COMMAND ${glslc_executable} --target-env=vulkan1.2 -o ${spv_file} ${CMAKE_CURRENT_SOURCE_DIR}/${source} - COMMENT "Compiling ${source} to ${spv_file}" - ) + set(options) + set(oneValueArgs) + set(multiValueArgs SOURCES) + cmake_parse_arguments(compile_shader "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) + foreach(source ${compile_shader_SOURCES}) + get_filename_component(filename ${source} NAME) + set(spv_file ${filename}.spv) + add_custom_command( + OUTPUT ${spv_file} + DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/${source} + ${CMAKE_CURRENT_SOURCE_DIR}/kompute-shaders/common.comp + ${CMAKE_CURRENT_SOURCE_DIR}/kompute-shaders/op_getrows.comp + ${CMAKE_CURRENT_SOURCE_DIR}/kompute-shaders/op_mul_mv_q_n_pre.comp + ${CMAKE_CURRENT_SOURCE_DIR}/kompute-shaders/op_mul_mv_q_n.comp + COMMAND ${glslc_executable} --target-env=vulkan1.2 -o ${spv_file} ${CMAKE_CURRENT_SOURCE_DIR}/${source} + COMMENT "Compiling ${source} to ${spv_file}" + ) - get_filename_component(RAW_FILE_NAME ${spv_file} NAME) - set(FILE_NAME "shader${RAW_FILE_NAME}") - string(REPLACE ".comp.spv" ".h" HEADER_FILE ${FILE_NAME}) - string(TOUPPER ${HEADER_FILE} HEADER_FILE_DEFINE) - string(REPLACE "." "_" HEADER_FILE_DEFINE "${HEADER_FILE_DEFINE}") - set(OUTPUT_HEADER_FILE "${HEADER_FILE}") - message(STATUS "${HEADER_FILE} generating ${HEADER_FILE_DEFINE}") - if(CMAKE_GENERATOR MATCHES "Visual Studio") - add_custom_command( - OUTPUT ${OUTPUT_HEADER_FILE} - COMMAND ${CMAKE_COMMAND} -E echo "/*THIS FILE HAS BEEN AUTOMATICALLY GENERATED - DO NOT EDIT*/" > ${OUTPUT_HEADER_FILE} - COMMAND ${CMAKE_COMMAND} -E echo \"\#ifndef ${HEADER_FILE_DEFINE}\" >> ${OUTPUT_HEADER_FILE} - COMMAND ${CMAKE_COMMAND} -E echo \"\#define ${HEADER_FILE_DEFINE}\" >> ${OUTPUT_HEADER_FILE} - COMMAND ${CMAKE_COMMAND} -E echo "namespace kp {" >> ${OUTPUT_HEADER_FILE} - COMMAND ${CMAKE_COMMAND} -E echo "namespace shader_data {" >> ${OUTPUT_HEADER_FILE} - COMMAND ${CMAKE_BINARY_DIR}/bin/$/xxd -i ${RAW_FILE_NAME} >> ${OUTPUT_HEADER_FILE} - COMMAND ${CMAKE_COMMAND} -E echo "}}" >> ${OUTPUT_HEADER_FILE} - COMMAND ${CMAKE_COMMAND} -E echo \"\#endif // define ${HEADER_FILE_DEFINE}\" >> ${OUTPUT_HEADER_FILE} - DEPENDS ${spv_file} xxd - COMMENT "Converting to hpp: ${FILE_NAME} ${CMAKE_BINARY_DIR}/bin/$/xxd" - ) - else() - add_custom_command( - OUTPUT ${OUTPUT_HEADER_FILE} - COMMAND ${CMAKE_COMMAND} -E echo "/*THIS FILE HAS BEEN AUTOMATICALLY GENERATED - DO NOT EDIT*/" > ${OUTPUT_HEADER_FILE} - COMMAND ${CMAKE_COMMAND} -E echo \"\#ifndef ${HEADER_FILE_DEFINE}\" >> ${OUTPUT_HEADER_FILE} - COMMAND ${CMAKE_COMMAND} -E echo \"\#define ${HEADER_FILE_DEFINE}\" >> ${OUTPUT_HEADER_FILE} - COMMAND ${CMAKE_COMMAND} -E echo "namespace kp {" >> ${OUTPUT_HEADER_FILE} - COMMAND ${CMAKE_COMMAND} -E echo "namespace shader_data {" >> ${OUTPUT_HEADER_FILE} - COMMAND ${CMAKE_BINARY_DIR}/bin/xxd -i ${RAW_FILE_NAME} >> ${OUTPUT_HEADER_FILE} - COMMAND ${CMAKE_COMMAND} -E echo "}}" >> ${OUTPUT_HEADER_FILE} - COMMAND ${CMAKE_COMMAND} -E echo \"\#endif // define ${HEADER_FILE_DEFINE}\" >> ${OUTPUT_HEADER_FILE} - DEPENDS ${spv_file} xxd - COMMENT "Converting to hpp: ${FILE_NAME} ${CMAKE_BINARY_DIR}/bin/xxd" - ) - endif() - endforeach() + get_filename_component(RAW_FILE_NAME ${spv_file} NAME) + set(FILE_NAME "shader${RAW_FILE_NAME}") + string(REPLACE ".comp.spv" ".h" HEADER_FILE ${FILE_NAME}) + string(TOUPPER ${HEADER_FILE} HEADER_FILE_DEFINE) + string(REPLACE "." "_" HEADER_FILE_DEFINE "${HEADER_FILE_DEFINE}") + set(OUTPUT_HEADER_FILE "${HEADER_FILE}") + message(STATUS "${HEADER_FILE} generating ${HEADER_FILE_DEFINE}") + if(CMAKE_GENERATOR MATCHES "Visual Studio") + add_custom_command( + OUTPUT ${OUTPUT_HEADER_FILE} + COMMAND ${CMAKE_COMMAND} -E echo "/*THIS FILE HAS BEEN AUTOMATICALLY GENERATED - DO NOT EDIT*/" > ${OUTPUT_HEADER_FILE} + COMMAND ${CMAKE_COMMAND} -E echo \"\#ifndef ${HEADER_FILE_DEFINE}\" >> ${OUTPUT_HEADER_FILE} + COMMAND ${CMAKE_COMMAND} -E echo \"\#define ${HEADER_FILE_DEFINE}\" >> ${OUTPUT_HEADER_FILE} + COMMAND ${CMAKE_COMMAND} -E echo "namespace kp {" >> ${OUTPUT_HEADER_FILE} + COMMAND ${CMAKE_COMMAND} -E echo "namespace shader_data {" >> ${OUTPUT_HEADER_FILE} + COMMAND ${CMAKE_BINARY_DIR}/bin/$/xxd -i ${RAW_FILE_NAME} >> ${OUTPUT_HEADER_FILE} + COMMAND ${CMAKE_COMMAND} -E echo "}}" >> ${OUTPUT_HEADER_FILE} + COMMAND ${CMAKE_COMMAND} -E echo \"\#endif // define ${HEADER_FILE_DEFINE}\" >> ${OUTPUT_HEADER_FILE} + DEPENDS ${spv_file} xxd + COMMENT "Converting to hpp: ${FILE_NAME} ${CMAKE_BINARY_DIR}/bin/$/xxd" + ) + else() + add_custom_command( + OUTPUT ${OUTPUT_HEADER_FILE} + COMMAND ${CMAKE_COMMAND} -E echo "/*THIS FILE HAS BEEN AUTOMATICALLY GENERATED - DO NOT EDIT*/" > ${OUTPUT_HEADER_FILE} + COMMAND ${CMAKE_COMMAND} -E echo \"\#ifndef ${HEADER_FILE_DEFINE}\" >> ${OUTPUT_HEADER_FILE} + COMMAND ${CMAKE_COMMAND} -E echo \"\#define ${HEADER_FILE_DEFINE}\" >> ${OUTPUT_HEADER_FILE} + COMMAND ${CMAKE_COMMAND} -E echo "namespace kp {" >> ${OUTPUT_HEADER_FILE} + COMMAND ${CMAKE_COMMAND} -E echo "namespace shader_data {" >> ${OUTPUT_HEADER_FILE} + COMMAND ${CMAKE_BINARY_DIR}/bin/xxd -i ${RAW_FILE_NAME} >> ${OUTPUT_HEADER_FILE} + COMMAND ${CMAKE_COMMAND} -E echo "}}" >> ${OUTPUT_HEADER_FILE} + COMMAND ${CMAKE_COMMAND} -E echo \"\#endif // define ${HEADER_FILE_DEFINE}\" >> ${OUTPUT_HEADER_FILE} + DEPENDS ${spv_file} xxd + COMMENT "Converting to hpp: ${FILE_NAME} ${CMAKE_BINARY_DIR}/bin/xxd" + ) + endif() + endforeach() endfunction() if (EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/kompute/CMakeLists.txt") @@ -604,66 +611,66 @@ if (LLAMA_KOMPUTE) # Compile our shaders compile_shader(SOURCES - kompute-shaders/op_scale.comp - kompute-shaders/op_scale_8.comp - kompute-shaders/op_add.comp - kompute-shaders/op_addrow.comp - kompute-shaders/op_mul.comp - kompute-shaders/op_silu.comp - kompute-shaders/op_relu.comp - kompute-shaders/op_gelu.comp - kompute-shaders/op_softmax.comp - kompute-shaders/op_norm.comp - kompute-shaders/op_rmsnorm.comp - kompute-shaders/op_diagmask.comp - kompute-shaders/op_mul_mat_mat_f32.comp - kompute-shaders/op_mul_mat_f16.comp - kompute-shaders/op_mul_mat_q8_0.comp - kompute-shaders/op_mul_mat_q4_0.comp - kompute-shaders/op_mul_mat_q4_1.comp - kompute-shaders/op_mul_mat_q6_k.comp - kompute-shaders/op_getrows_f16.comp - kompute-shaders/op_getrows_q4_0.comp - kompute-shaders/op_getrows_q4_1.comp - kompute-shaders/op_getrows_q6_k.comp - kompute-shaders/op_rope_f16.comp - kompute-shaders/op_rope_f32.comp - kompute-shaders/op_cpy_f16_f16.comp - kompute-shaders/op_cpy_f16_f32.comp - kompute-shaders/op_cpy_f32_f16.comp - kompute-shaders/op_cpy_f32_f32.comp + kompute-shaders/op_scale.comp + kompute-shaders/op_scale_8.comp + kompute-shaders/op_add.comp + kompute-shaders/op_addrow.comp + kompute-shaders/op_mul.comp + kompute-shaders/op_silu.comp + kompute-shaders/op_relu.comp + kompute-shaders/op_gelu.comp + kompute-shaders/op_softmax.comp + kompute-shaders/op_norm.comp + kompute-shaders/op_rmsnorm.comp + kompute-shaders/op_diagmask.comp + kompute-shaders/op_mul_mat_mat_f32.comp + kompute-shaders/op_mul_mat_f16.comp + kompute-shaders/op_mul_mat_q8_0.comp + kompute-shaders/op_mul_mat_q4_0.comp + kompute-shaders/op_mul_mat_q4_1.comp + kompute-shaders/op_mul_mat_q6_k.comp + kompute-shaders/op_getrows_f16.comp + kompute-shaders/op_getrows_q4_0.comp + kompute-shaders/op_getrows_q4_1.comp + kompute-shaders/op_getrows_q6_k.comp + kompute-shaders/op_rope_f16.comp + kompute-shaders/op_rope_f32.comp + kompute-shaders/op_cpy_f16_f16.comp + kompute-shaders/op_cpy_f16_f32.comp + kompute-shaders/op_cpy_f32_f16.comp + kompute-shaders/op_cpy_f32_f32.comp ) # Create a custom target for our generated shaders add_custom_target(generated_shaders DEPENDS - shaderop_scale.h - shaderop_scale_8.h - shaderop_add.h - shaderop_addrow.h - shaderop_mul.h - shaderop_silu.h - shaderop_relu.h - shaderop_gelu.h - shaderop_softmax.h - shaderop_norm.h - shaderop_rmsnorm.h - shaderop_diagmask.h - shaderop_mul_mat_mat_f32.h - shaderop_mul_mat_f16.h - shaderop_mul_mat_q8_0.h - shaderop_mul_mat_q4_0.h - shaderop_mul_mat_q4_1.h - shaderop_mul_mat_q6_k.h - shaderop_getrows_f16.h - shaderop_getrows_q4_0.h - shaderop_getrows_q4_1.h - shaderop_getrows_q6_k.h - shaderop_rope_f16.h - shaderop_rope_f32.h - shaderop_cpy_f16_f16.h - shaderop_cpy_f16_f32.h - shaderop_cpy_f32_f16.h - shaderop_cpy_f32_f32.h + shaderop_scale.h + shaderop_scale_8.h + shaderop_add.h + shaderop_addrow.h + shaderop_mul.h + shaderop_silu.h + shaderop_relu.h + shaderop_gelu.h + shaderop_softmax.h + shaderop_norm.h + shaderop_rmsnorm.h + shaderop_diagmask.h + shaderop_mul_mat_mat_f32.h + shaderop_mul_mat_f16.h + shaderop_mul_mat_q8_0.h + shaderop_mul_mat_q4_0.h + shaderop_mul_mat_q4_1.h + shaderop_mul_mat_q6_k.h + shaderop_getrows_f16.h + shaderop_getrows_q4_0.h + shaderop_getrows_q4_1.h + shaderop_getrows_q6_k.h + shaderop_rope_f16.h + shaderop_rope_f32.h + shaderop_cpy_f16_f16.h + shaderop_cpy_f16_f32.h + shaderop_cpy_f32_f16.h + shaderop_cpy_f32_f32.h ) # Create a custom command that depends on the generated_shaders @@ -676,8 +683,10 @@ if (LLAMA_KOMPUTE) # Add the stamp to the main sources to ensure dependency tracking set(GGML_SOURCES_KOMPUTE ggml-kompute.cpp ${CMAKE_CURRENT_BINARY_DIR}/ggml-kompute.stamp) - set(GGML_HEADERS_KOMPUTE ggml-kompute.h ${CMAKE_CURRENT_BINARY_DIR}/ggml-kompute.stamp) + set(GGML_HEADERS_KOMPUTE ggml-kompute.h ${CMAKE_CURRENT_BINARY_DIR}/ggml-kompute.stamp) + add_compile_definitions(GGML_USE_KOMPUTE) + set(LLAMA_EXTRA_LIBS ${LLAMA_EXTRA_LIBS} kompute) set(LLAMA_EXTRA_INCLUDES ${LLAMA_EXTRA_INCLUDES} ${CMAKE_BINARY_DIR}) else() @@ -685,6 +694,18 @@ if (LLAMA_KOMPUTE) endif() endif() +if (LLAMA_CPU_HBM) + find_library(memkind memkind REQUIRED) + + add_compile_definitions(GGML_USE_CPU_HBM) + + target_link_libraries(ggml PUBLIC memkind) +endif() + +if (LLAMA_PERF) + add_compile_definitions(GGML_PERF) +endif() + function(get_flags CCID CCVER) set(C_FLAGS "") set(CXX_FLAGS "") @@ -821,6 +842,7 @@ execute_process( ERROR_VARIABLE output OUTPUT_QUIET ) + if (output MATCHES "dyld-1015\.7") add_compile_definitions(HAVE_BUGGY_APPLE_LINKER) endif() @@ -830,10 +852,10 @@ endif() # feel free to update the Makefile for your architecture and send a pull request or issue message(STATUS "CMAKE_SYSTEM_PROCESSOR: ${CMAKE_SYSTEM_PROCESSOR}") if (MSVC) - string(TOLOWER "${CMAKE_GENERATOR_PLATFORM}" CMAKE_GENERATOR_PLATFORM_LWR) - message(STATUS "CMAKE_GENERATOR_PLATFORM: ${CMAKE_GENERATOR_PLATFORM}") + string(TOLOWER "${CMAKE_GENERATOR_PLATFORM}" CMAKE_GENERATOR_PLATFORM_LWR) + message(STATUS "CMAKE_GENERATOR_PLATFORM: ${CMAKE_GENERATOR_PLATFORM}") else () - set(CMAKE_GENERATOR_PLATFORM_LWR "") + set(CMAKE_GENERATOR_PLATFORM_LWR "") endif () if (NOT MSVC) @@ -1027,11 +1049,6 @@ endif() # ggml -if (GGML_USE_CPU_HBM) - add_definitions(-DGGML_USE_CPU_HBM) - find_library(memkind memkind REQUIRED) -endif() - add_library(ggml OBJECT ggml.c ggml.h @@ -1048,16 +1065,17 @@ add_library(ggml OBJECT ${GGML_SOURCES_EXTRA} ${GGML_HEADERS_EXTRA} ${GGML_SOURCES_SYCL} ${GGML_HEADERS_SYCL} ${GGML_SOURCES_KOMPUTE} ${GGML_HEADERS_KOMPUTE} + ${GGML_SOURCES_VULKAN} ${GGML_HEADERS_VULKAN} + ${GGML_SOURCES_ROCM} ${GGML_HEADERS_ROCM} ) target_include_directories(ggml PUBLIC . ${LLAMA_EXTRA_INCLUDES}) -target_compile_features(ggml PUBLIC c_std_11) # don't bump +target_compile_features (ggml PUBLIC c_std_11) # don't bump + target_link_libraries(ggml PUBLIC Threads::Threads ${LLAMA_EXTRA_LIBS}) -if (GGML_USE_CPU_HBM) - target_link_libraries(ggml PUBLIC memkind) -endif() add_library(ggml_static STATIC $) + if (BUILD_SHARED_LIBS) set_target_properties(ggml PROPERTIES POSITION_INDEPENDENT_CODE ON) add_library(ggml_shared SHARED $) @@ -1073,7 +1091,8 @@ add_library(llama ) target_include_directories(llama PUBLIC .) -target_compile_features(llama PUBLIC cxx_std_11) # don't bump +target_compile_features (llama PUBLIC cxx_std_11) # don't bump + target_link_libraries(llama PRIVATE ggml ${LLAMA_EXTRA_LIBS} @@ -1124,7 +1143,7 @@ install(FILES ${CMAKE_CURRENT_BINARY_DIR}/LlamaConfig.cmake DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/Llama) set(GGML_PUBLIC_HEADERS "ggml.h" "ggml-alloc.h" "ggml-backend.h" - "${GGML_HEADERS_CUDA}" "${GGML_HEADERS_OPENCL}" + "${GGML_HEADERS_CUDA}" "${GGML_HEADERS_OPENCL}" "${GGML_HEADERS_METAL}" "${GGML_HEADERS_MPI}" "${GGML_HEADERS_EXTRA}") set_target_properties(ggml PROPERTIES PUBLIC_HEADER "${GGML_PUBLIC_HEADERS}") diff --git a/ggml-vulkan.cpp b/ggml-vulkan.cpp index 1fad24fd1..4a30414df 100644 --- a/ggml-vulkan.cpp +++ b/ggml-vulkan.cpp @@ -1091,7 +1091,7 @@ static void ggml_vk_print_gpu_info(size_t idx) { } } -void ggml_vk_instance_init() { +static void ggml_vk_instance_init() { if (vk_instance_initialized) { return; } @@ -1150,7 +1150,7 @@ void ggml_vk_instance_init() { vk_instance_initialized = true; } -void ggml_vk_init(ggml_backend_vk_context * ctx, size_t idx) { +static void ggml_vk_init(ggml_backend_vk_context * ctx, size_t idx) { GGML_ASSERT(idx < vk_instance.device_indices.size()); size_t dev_num = vk_instance.device_indices[idx]; #ifdef GGML_VULKAN_DEBUG @@ -4556,13 +4556,13 @@ static void ggml_vk_cleanup(ggml_backend_vk_context * ctx) { } } -GGML_CALL int ggml_vk_get_device_count() { +GGML_CALL static int ggml_vk_get_device_count() { ggml_vk_instance_init(); return vk_instance.device_indices.size(); } -GGML_CALL void ggml_vk_get_device_description(int device, char * description, size_t description_size) { +GGML_CALL static void ggml_vk_get_device_description(int device, char * description, size_t description_size) { ggml_vk_instance_init(); std::vector devices = vk_instance.instance.enumeratePhysicalDevices(); @@ -4580,7 +4580,7 @@ void ggml_vk_init_cpu_assist() { std::cerr << "ggml_vulkan: Found " << ggml_vk_get_device_count() << " Vulkan devices:" << std::endl; - for (size_t i = 0; i < ggml_vk_get_device_count(); i++) { + for (int i = 0; i < ggml_vk_get_device_count(); i++) { ggml_vk_print_gpu_info(i); } // Initialize the first backend to make sure CPU matrix multiplications can be offloaded. @@ -5267,7 +5267,7 @@ GGML_CALL void ggml_backend_vk_get_device_description(int device, char * descrip } GGML_CALL void ggml_backend_vk_get_device_memory(int device, size_t * free, size_t * total) { - GGML_ASSERT(device < vk_instance.device_indices.size()); + GGML_ASSERT(device < (int) vk_instance.device_indices.size()); vk::PhysicalDevice vkdev = vk_instance.instance.enumeratePhysicalDevices()[vk_instance.device_indices[device]]; From d250c9d61d4d9f7346930814cc4aef3f3673dc3e Mon Sep 17 00:00:00 2001 From: clibdev <52199778+clibdev@users.noreply.github.com> Date: Sat, 17 Feb 2024 18:28:37 +0200 Subject: [PATCH 364/812] gitignore : update for CLion IDE (#5544) --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index b84459b92..62b6b8b1a 100644 --- a/.gitignore +++ b/.gitignore @@ -23,11 +23,13 @@ .clang-tidy .vs/ .vscode/ +.idea/ lcov-report/ gcovr-report/ build* +cmake-build-* out/ tmp/ From 6e4e973b2615f8d390b1c4f4a7e05a119078bb0f Mon Sep 17 00:00:00 2001 From: Ananta Bastola Date: Sat, 17 Feb 2024 16:03:14 -0500 Subject: [PATCH 365/812] ci : add an option to fail on compile warning (#3952) * feat(ci): add an option to fail on compile warning * Update CMakeLists.txt * minor : fix compile warnings ggml-ci * ggml : fix unreachable code warnings ggml-ci * ci : disable fatal warnings for windows, ios and tvos * ggml : fix strncpy warning * ci : disable fatal warnings for MPI build * ci : add fatal warnings to ggml-ci ggml-ci --------- Co-authored-by: Georgi Gerganov --- .github/workflows/build.yml | 10 +++++++--- CMakeLists.txt | 11 +++++++++++ Makefile | 29 ++++++++++++++++++++++++++++ ci/run.sh | 2 +- examples/export-lora/export-lora.cpp | 2 -- ggml-backend.c | 1 + ggml-metal.m | 2 +- ggml.c | 15 +++++++++----- 8 files changed, 60 insertions(+), 12 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index ed292d6b8..03d76d455 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -37,6 +37,8 @@ jobs: - name: Build id: make_build + env: + LLAMA_FATAL_WARNINGS: 1 run: | CC=gcc-8 make -j $(nproc) @@ -65,7 +67,7 @@ jobs: run: | mkdir build cd build - cmake .. + cmake .. -DLLAMA_FATAL_WARNINGS=ON cmake --build . --config Release -j $(nproc) - name: Test @@ -100,7 +102,7 @@ jobs: run: | mkdir build cd build - cmake .. -DLLAMA_SANITIZE_${{ matrix.sanitizer }}=ON -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} + cmake .. -DLLAMA_FATAL_WARNINGS=ON -DLLAMA_SANITIZE_${{ matrix.sanitizer }}=ON -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} cmake --build . --config ${{ matrix.build_type }} -j $(nproc) - name: Test @@ -244,6 +246,8 @@ jobs: - name: Build id: make_build + env: + LLAMA_FATAL_WARNINGS: 1 run: | LLAMA_NO_METAL=1 make -j $(sysctl -n hw.logicalcpu) @@ -277,7 +281,7 @@ jobs: sysctl -a mkdir build cd build - cmake -DLLAMA_METAL=OFF .. + cmake -DLLAMA_FATAL_WARNINGS=ON -DLLAMA_METAL=OFF .. cmake --build . --config Release -j $(sysctl -n hw.logicalcpu) - name: Test diff --git a/CMakeLists.txt b/CMakeLists.txt index 2a922fdb3..5ea4d4f19 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -55,6 +55,9 @@ option(LLAMA_ALL_WARNINGS "llama: enable all compiler warnings" option(LLAMA_ALL_WARNINGS_3RD_PARTY "llama: enable all compiler warnings in 3rd party libs" OFF) option(LLAMA_GPROF "llama: enable gprof" OFF) +# build +option(LLAMA_FATAL_WARNINGS "llama: enable -Werror flag" OFF) + # sanitizers option(LLAMA_SANITIZE_THREAD "llama: enable thread sanitizer" OFF) option(LLAMA_SANITIZE_ADDRESS "llama: enable address sanitizer" OFF) @@ -142,6 +145,14 @@ set(THREADS_PREFER_PTHREAD_FLAG ON) find_package(Threads REQUIRED) include(CheckCXXCompilerFlag) +if (LLAMA_FATAL_WARNINGS) + if (CMAKE_CXX_COMPILER_ID MATCHES "GNU" OR CMAKE_CXX_COMPILER_ID MATCHES "Clang") + add_compile_options(-Werror) + elseif (CMAKE_CXX_COMPILER_ID STREQUAL "MSVC") + add_compile_options(/WX) + endif() +endif() + # enable libstdc++ assertions for debug builds if (CMAKE_SYSTEM_NAME MATCHES "Linux") add_compile_definitions($<$:_GLIBCXX_ASSERTIONS>) diff --git a/Makefile b/Makefile index 0a2070b53..901798606 100644 --- a/Makefile +++ b/Makefile @@ -215,6 +215,35 @@ MK_CFLAGS += $(WARN_FLAGS) -Wshadow -Wstrict-prototypes -Wpointer-arith -Wmis -Werror=implicit-function-declaration MK_CXXFLAGS += $(WARN_FLAGS) -Wmissing-declarations -Wmissing-noreturn +ifeq ($(LLAMA_FATAL_WARNINGS),1) + MK_CFLAGS += -Werror + MK_CXXFLAGS += -Werror +endif + +ifeq ($(CC_IS_CLANG), 1) + # clang options + MK_CFLAGS += -Wunreachable-code-break -Wunreachable-code-return + MK_HOST_CXXFLAGS += -Wunreachable-code-break -Wunreachable-code-return -Wmissing-prototypes -Wextra-semi + + ifneq '' '$(and $(CC_IS_LLVM_CLANG),$(filter 1,$(shell expr $(CC_VER) \>= 030800)))' + MK_CFLAGS += -Wdouble-promotion + endif + ifneq '' '$(and $(CC_IS_APPLE_CLANG),$(filter 1,$(shell expr $(CC_VER) \>= 070300)))' + MK_CFLAGS += -Wdouble-promotion + endif +else + # gcc options + MK_CFLAGS += -Wdouble-promotion + MK_HOST_CXXFLAGS += -Wno-array-bounds + + ifeq ($(shell expr $(CC_VER) \>= 070100), 1) + MK_HOST_CXXFLAGS += -Wno-format-truncation + endif + ifeq ($(shell expr $(CC_VER) \>= 080100), 1) + MK_HOST_CXXFLAGS += -Wextra-semi + endif +endif + # this version of Apple ld64 is buggy ifneq '' '$(findstring dyld-1015.7,$(shell $(CC) $(LDFLAGS) -Wl,-v 2>&1))' MK_CPPFLAGS += -DHAVE_BUGGY_APPLE_LINKER diff --git a/ci/run.sh b/ci/run.sh index 979b4a793..b94658c96 100755 --- a/ci/run.sh +++ b/ci/run.sh @@ -33,7 +33,7 @@ sd=`dirname $0` cd $sd/../ SRC=`pwd` -CMAKE_EXTRA="" +CMAKE_EXTRA="-DLLAMA_FATAL_WARNINGS=ON" if [ ! -z ${GG_BUILD_METAL} ]; then CMAKE_EXTRA="${CMAKE_EXTRA} -DLLAMA_METAL_SHADER_DEBUG=ON" diff --git a/examples/export-lora/export-lora.cpp b/examples/export-lora/export-lora.cpp index 2f7be8a13..08413f57e 100644 --- a/examples/export-lora/export-lora.cpp +++ b/examples/export-lora/export-lora.cpp @@ -7,8 +7,6 @@ #include #include -static const size_t tensor_alignment = 32; - struct lora_info { std::string filename; float scale; diff --git a/ggml-backend.c b/ggml-backend.c index 87eea8440..d019d813a 100644 --- a/ggml-backend.c +++ b/ggml-backend.c @@ -1006,6 +1006,7 @@ static int ggml_backend_sched_backend_from_buffer(ggml_backend_sched_t sched, gg } } GGML_ASSERT(false && "tensor buffer type not supported by any backend"); + return -1; // silence warning } #if 0 diff --git a/ggml-metal.m b/ggml-metal.m index c1d8e2de8..6e76f8bed 100644 --- a/ggml-metal.m +++ b/ggml-metal.m @@ -176,7 +176,7 @@ struct ggml_metal_context { // MSL code // TODO: move the contents here when ready // for now it is easier to work in a separate file -//static NSString * const msl_library_source = @"see metal.metal"; +// static NSString * const msl_library_source = @"see metal.metal"; // Here to assist with NSBundle Path Hack @interface GGMLMetalClass : NSObject diff --git a/ggml.c b/ggml.c index 4e302fb7d..264cfd705 100644 --- a/ggml.c +++ b/ggml.c @@ -868,7 +868,7 @@ do { \ const __m128 t0 = _mm_add_ps(_mm256_castps256_ps128(x[0]), \ _mm256_extractf128_ps(x[0], 1)); \ const __m128 t1 = _mm_hadd_ps(t0, t0); \ - res = _mm_cvtss_f32(_mm_hadd_ps(t1, t1)); \ + res = (ggml_float) _mm_cvtss_f32(_mm_hadd_ps(t1, t1)); \ } while (0) // TODO: is this optimal ? @@ -1149,7 +1149,7 @@ inline static void __wasm_f16x4_store(ggml_fp16_t * p, v128_t x) { x[i] = _mm_add_ps(x[i], x[offset+i]); \ } \ const __m128 t0 = _mm_hadd_ps(x[0], x[0]); \ - res = _mm_cvtss_f32(_mm_hadd_ps(t0, t0)); \ + res = (ggml_float) _mm_cvtss_f32(_mm_hadd_ps(t0, t0)); \ } // TODO: is this optimal ? @@ -2086,6 +2086,7 @@ void ggml_numa_init(enum ggml_numa_strategy numa_flag) { } } #else + GGML_UNUSED(numa_flag); // TODO #endif } @@ -3219,7 +3220,7 @@ const char * ggml_get_name(const struct ggml_tensor * tensor) { } struct ggml_tensor * ggml_set_name(struct ggml_tensor * tensor, const char * name) { - strncpy(tensor->name, name, sizeof(tensor->name)); + strncpy(tensor->name, name, sizeof(tensor->name) - 1); tensor->name[sizeof(tensor->name) - 1] = '\0'; return tensor; } @@ -18575,7 +18576,9 @@ static enum ggml_opt_result linesearch_backtracking( (*step) *= width; } - GGML_UNREACHABLE(); + GGML_ASSERT(false && "line search failed"); + + return GGML_LINESEARCH_FAIL; } static enum ggml_opt_result ggml_opt_lbfgs( @@ -18843,7 +18846,9 @@ static enum ggml_opt_result ggml_opt_lbfgs( step[0] = 1.0; } - GGML_UNREACHABLE(); + GGML_ASSERT(false && "lbfgs failed"); + + return GGML_OPT_DID_NOT_CONVERGE; } struct ggml_opt_params ggml_opt_default_params(enum ggml_opt_type type) { From 8f1be0d42f23016cb6819dbae01126699c4bd9bc Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Sat, 17 Feb 2024 23:04:16 +0200 Subject: [PATCH 366/812] ggml : add ALiBi support for ggml_soft_max_ext (#5488) * ggml : avoid recomputing alibi slopes (CPU) * llama : reuse hparams.f_max_alibi_bias in all cases ggml-ci * ggml : support alibi bias in ggml_soft_max_ext (CPU + Metal) ggml-ci * ggml : handle all SRCs (do not break on first null) ggml-ci * tests : do not use slope for large soft_max accumulates too much error ggml-ci * ggml : alternative ALiBi without extra tensor We compute the slopes in the kernel ggml-ci * cuda : add ALiBi support in ggml_soft_max_ext ggml-ci * ggml : deprecate ggml_alibi * ggml : support multi-sequence ALiBi (Metal) ggml-ci * cuda : add multi-seq ALiBi + remote F16 soft_max ggml-ci * ggml : update deprecation message * ggml : fix pos ptr when no ALiBi ggml-ci * cuda : fix performance (pow -> powf) * cuda : precompute ALiBi constants * metal : pre-compute ALiBi slopes ggml-ci * llama : init kq_pos only if needed ggml-ci * test-backend-ops : add null pos test to soft_max test-backend-ops : replace soft_max tests ggml-ci --------- Co-authored-by: slaren --- ggml-alloc.c | 6 +- ggml-backend.c | 16 +-- ggml-cuda.cu | 263 ++++++++----------------------------- ggml-metal.m | 35 +++-- ggml-metal.metal | 47 ++++++- ggml.c | 118 +++++++++++------ ggml.h | 13 +- llama.cpp | 133 ++++++++++++------- tests/test-backend-ops.cpp | 74 +++++------ 9 files changed, 348 insertions(+), 357 deletions(-) diff --git a/ggml-alloc.c b/ggml-alloc.c index c28c37c4f..d4123564f 100644 --- a/ggml-alloc.c +++ b/ggml-alloc.c @@ -551,7 +551,7 @@ static void ggml_gallocr_alloc_graph_impl(ggml_gallocr_t galloc, struct ggml_cgr } for (int j = 0; j < GGML_MAX_SRC; j++) { if (graph->nodes[i]->src[j] == NULL) { - break; + continue; } if (graph->nodes[i]->src[j]->flags & GGML_TENSOR_FLAG_INPUT) { ggml_gallocr_allocate_node(galloc, graph->nodes[i]->src[j], get_node_buffer_id(node_buffer_ids, i)); @@ -787,7 +787,7 @@ static bool ggml_gallocr_needs_realloc(ggml_gallocr_t galloc, struct ggml_cgraph for (int j = 0; j < GGML_MAX_SRC; j++) { struct ggml_tensor * src = node->src[j]; if (src == NULL) { - break; + continue; } if (!ggml_gallocr_node_needs_realloc(galloc, src, node_alloc, &node_alloc->src[j])) { #ifndef NDEBUG @@ -833,7 +833,7 @@ bool ggml_gallocr_alloc_graph(ggml_gallocr_t galloc, struct ggml_cgraph * graph) for (int j = 0; j < GGML_MAX_SRC; j++) { struct ggml_tensor * src = node->src[j]; if (src == NULL) { - break; + continue; } ggml_gallocr_init_tensor(galloc, src, node_alloc, &node_alloc->src[j]); } diff --git a/ggml-backend.c b/ggml-backend.c index d019d813a..66e8c293a 100644 --- a/ggml-backend.c +++ b/ggml-backend.c @@ -1041,7 +1041,7 @@ static int ggml_backend_sched_backend_id_from_cur(ggml_backend_sched_t sched, st for (int i = 0; i < GGML_MAX_SRC; i++) { const struct ggml_tensor * src = tensor->src[i]; if (src == NULL) { - break; + continue; } if (src->buffer != NULL && src->buffer->usage == GGML_BACKEND_BUFFER_USAGE_WEIGHTS) { int src_backend = ggml_backend_sched_backend_from_buffer(sched, src->buffer); @@ -1088,7 +1088,7 @@ static void ggml_backend_sched_print_assignments(ggml_backend_sched_t sched, str for (int j = 0; j < GGML_MAX_SRC; j++) { struct ggml_tensor * src = node->src[j]; if (src == NULL) { - break; + continue; } ggml_backend_t src_backend = tensor_backend(src); fprintf(stderr, " %20.20s (%5.5s) [%5.5s %8.8s]", src->name, @@ -1144,7 +1144,7 @@ static void ggml_backend_sched_split_graph(ggml_backend_sched_t sched, struct gg for (int j = 0; j < GGML_MAX_SRC; j++) { struct ggml_tensor * src = node->src[j]; if (src == NULL) { - break; + continue; } if (tensor_backend_id(src) == -1) { tensor_backend_id(src) = ggml_backend_sched_backend_id_from_cur(sched, src); @@ -1256,7 +1256,7 @@ static void ggml_backend_sched_split_graph(ggml_backend_sched_t sched, struct gg for (int j = 0; j < GGML_MAX_SRC; j++) { struct ggml_tensor * src = node->src[j]; if (src == NULL) { - break; + continue; } int src_backend_id = tensor_backend_id(src); if (src_backend_id == -1) { @@ -1315,7 +1315,7 @@ static void ggml_backend_sched_split_graph(ggml_backend_sched_t sched, struct gg for (int j = 0; j < GGML_MAX_SRC; j++) { struct ggml_tensor * src = node->src[j]; if (src == NULL) { - break; + continue; } int src_backend_id = tensor_backend_id(src); assert(src_backend_id != -1); // all inputs should be assigned by now @@ -1362,7 +1362,7 @@ static void ggml_backend_sched_split_graph(ggml_backend_sched_t sched, struct gg for (int j = 0; j < GGML_MAX_SRC; j++) { struct ggml_tensor * src = node->src[j]; if (src == NULL) { - break; + continue; } ggml_backend_t src_backend = tensor_backend(src); if (src_backend != tensor_backend /* && src_backend != NULL */) { @@ -1668,7 +1668,7 @@ static struct ggml_tensor * graph_copy_dup_tensor(struct ggml_hash_set hash_set, for (int i = 0; i < GGML_MAX_SRC; i++) { struct ggml_tensor * s = src->src[i]; if (s == NULL) { - break; + continue; } dst->src[i] = graph_copy_dup_tensor(hash_set, node_copies, ctx_allocated, ctx_unallocated, s); } @@ -1697,7 +1697,7 @@ static void graph_copy_init_tensor(struct ggml_hash_set hash_set, struct ggml_te for (int i = 0; i < GGML_MAX_SRC; i++) { struct ggml_tensor * s = src->src[i]; if (s == NULL) { - break; + continue; } graph_copy_init_tensor(hash_set, node_copies, node_init, s); } diff --git a/ggml-cuda.cu b/ggml-cuda.cu index b35fcb7fd..5fd8a87e4 100644 --- a/ggml-cuda.cu +++ b/ggml-cuda.cu @@ -5956,149 +5956,31 @@ static __global__ void diag_mask_inf_f32(const float * x, float * dst, const int dst[i] = x[i] - (col > n_past + row % rows_per_channel) * FLT_MAX; } -template -static __global__ void soft_max_f16(const float * x, const float * y, float * dst, const int ncols_par, const int nrows_y, const float scale) { -#if !(defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)) && __CUDA_ARCH__ >= CC_PASCAL && CUDART_VERSION >= CUDART_HMAX - const int ncols_data = ncols_template == 0 ? ncols_par : ncols_template; - const int ncols_smem = GGML_PAD(ncols_data, 2*WARP_SIZE)/2; - - const int tid = threadIdx.x; - const int rowx = blockIdx.x; - const int rowy = rowx % nrows_y; // broadcast the mask (y) in the row dimension - - const int block_size = block_size_template == 0 ? blockDim.x : block_size_template; - - const int warp_id = threadIdx.x / WARP_SIZE; - const int lane_id = threadIdx.x % WARP_SIZE; - - extern __shared__ half data_soft_max_f16[]; - half * buf_iw = data_soft_max_f16 + 0; // shared memory buffer for inter-warp communication - // (shared memory) buffer to cache values between iterations: - half2 * vals = vals_smem ? (half2 *) (buf_iw + WARP_SIZE) : (half2 *) (dst + rowx*ncols_data); - // if the buffer is larger than max. shared memory per block, use dst as temp. buffer instead - // in that case col_smem == col_data must be enforced to avoid race conditions - - half2 max_val = make_half2(-INFINITY, -INFINITY); - -#pragma unroll - for (int col0 = 0; col0 < ncols_smem; col0 += block_size) { - const int col_data = 2*col0 + 2*WARP_SIZE*warp_id + lane_id; - const int col_smem = vals_smem ? col0 + tid : col_data; - - const int ix = rowx*ncols_data + col_data; - const int iy = rowy*ncols_data + col_data; - - half2 val; - if (need_check && col_data + 0 >= ncols_data) { - val.x = -INFINITY; - } else { - val.x = x[ix + 0]*scale + (y ? y[iy + 0] : 0.0f); - } - if (need_check && col_data + WARP_SIZE >= ncols_data) { - val.y = -INFINITY; - } else { - val.y = x[ix + WARP_SIZE]*scale + (y ? y[iy + WARP_SIZE] : 0.0f); - } - if (!need_check || col_smem < (vals_smem ? ncols_smem : ncols_data)) { - vals[col_smem] = val; - } - max_val = __hmax2(max_val, val); - } - - // find the max value in the block - max_val = warp_reduce_max(max_val); - if (block_size > WARP_SIZE) { - if (warp_id == 0) { - buf_iw[lane_id] = -INFINITY; - } - __syncthreads(); - - if (lane_id == 0) { - buf_iw[warp_id] = __hmax(max_val.x, max_val.y); - } - __syncthreads(); - - max_val = __half2half2(buf_iw[lane_id]); - max_val = warp_reduce_max(max_val); - } else { - max_val = __half2half2(__hmax(max_val.x, max_val.y)); - } - - half2 tmp = make_half2(0.0f, 0.0f); // partial sums - -#pragma unroll - for (int col0 = 0; col0 < ncols_smem; col0 += block_size) { - const int col_smem = vals_smem ? col0 + tid : 2*col0 + 2*warp_id*WARP_SIZE + lane_id; - - if (ncols_template == 0 && col_smem >= (vals_smem ? ncols_smem : ncols_data)) { - break; - } - - const half2 val = h2exp(vals[col_smem] - max_val); - - tmp += val; - vals[col_smem] = val; - } - - // find the sum of exps in the block - tmp = warp_reduce_sum(tmp); - if (block_size > WARP_SIZE) { - if (warp_id == 0) { - buf_iw[lane_id] = 0.0f; - } - __syncthreads(); - - if (lane_id == 0) { - buf_iw[warp_id] = tmp.x + tmp.y; - } - __syncthreads(); - - tmp = __half2half2(buf_iw[lane_id]); - tmp = warp_reduce_sum(tmp); - } else { - tmp = __half2half2(tmp.x + tmp.y); - } - - const half2 inv_sum = make_half2(1.0f, 1.0f) / tmp; - -#pragma unroll - for (int col0 = 0; col0 < ncols_smem; col0 += block_size) { - const int col_data = 2*col0 + 2*WARP_SIZE*warp_id + lane_id; - const int col_smem = vals_smem ? col0 + tid : col_data; - - const int idst = rowx*ncols_data + col_data; - const half2 result = vals[col_smem] * inv_sum; - - if (need_check && col_data + 0 >= ncols_data) { - return; - } - dst[idst] = result.x; - - if (need_check && col_data + WARP_SIZE >= ncols_data) { - return; - } - - dst[idst + WARP_SIZE] = result.y; - } -#else - (void) x; (void) y; (void) dst; (void) ncols_par; (void) nrows_y; (void) scale; - NO_DEVICE_CODE; -#endif // !(defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)) && __CUDA_ARCH__ >= CC_PASCAL && CUDART_VERSION >= CUDART_HMAX -} - template -static __global__ void soft_max_f32(const float * x, const float * y, float * dst, const int ncols_par, const int nrows_y, const float scale) { +static __global__ void soft_max_f32(const float * x, const float * mask, const float * pos, float * dst, const int ncols_par, const int nrows_y, const float scale, const float max_bias, const float m0, const float m1, uint32_t n_head_log2) { const int ncols = ncols_template == 0 ? ncols_par : ncols_template; const int tid = threadIdx.x; const int rowx = blockIdx.x; - const int rowy = rowx % nrows_y; // broadcast the mask (y) in the row dimension + const int rowy = rowx % nrows_y; // broadcast the mask in the row dimension const int block_size = block_size_template == 0 ? blockDim.x : block_size_template; const int warp_id = threadIdx.x / WARP_SIZE; const int lane_id = threadIdx.x % WARP_SIZE; + float slope = 0.0f; + + // ALiBi + if (max_bias > 0.0f) { + const int h = rowx/nrows_y; // head index + + const float base = h < n_head_log2 ? m0 : m1; + const int exp = h < n_head_log2 ? h + 1 : 2*(h - n_head_log2) + 1; + + slope = powf(base, exp); + } + extern __shared__ float data_soft_max_f32[]; float * buf_iw = data_soft_max_f32; // shared memory buffer for inter-warp communication // shared memory buffer to cache values between iterations: @@ -6117,7 +5999,8 @@ static __global__ void soft_max_f32(const float * x, const float * y, float * ds const int ix = rowx*ncols + col; const int iy = rowy*ncols + col; - const float val = x[ix]*scale + (y ? y[iy] : 0.0f); + const float val = x[ix]*scale + (mask ? mask[iy] : 0.0f) + slope*pos[col]; + vals[col] = val; max_val = max(max_val, val); } @@ -7589,89 +7472,53 @@ static void diag_mask_inf_f32_cuda(const float * x, float * dst, const int ncols diag_mask_inf_f32<<>>(x, dst, ncols_x, rows_per_channel, n_past); } -static void soft_max_f16_cuda(const float * x, const float * y, float * dst, const int ncols_x, const int nrows_x, const int nrows_y, const float scale, cudaStream_t stream) { - int nth = WARP_SIZE; - while (nth < ncols_x/2 && nth < CUDA_SOFT_MAX_BLOCK_SIZE) nth *= 2; - const dim3 block_dims(nth, 1, 1); - const dim3 block_nums(nrows_x, 1, 1); - const size_t shmem = (GGML_PAD(ncols_x, 2*WARP_SIZE) + WARP_SIZE)*sizeof(half); - static_assert(CUDA_SOFT_MAX_BLOCK_SIZE == 1024, "These values need to be adjusted."); - if (shmem <= g_device_caps[g_main_device].smpb) { - switch (ncols_x) { - case 32: - soft_max_f16<<>>(x, y, dst, ncols_x, nrows_y, scale); - break; - case 64: - soft_max_f16<<>>(x, y, dst, ncols_x, nrows_y, scale); - break; - case 128: - soft_max_f16<<>>(x, y, dst, ncols_x, nrows_y, scale); - break; - case 256: - soft_max_f16<<>>(x, y, dst, ncols_x, nrows_y, scale); - break; - case 512: - soft_max_f16<<>>(x, y, dst, ncols_x, nrows_y, scale); - break; - case 1024: - soft_max_f16<<>>(x, y, dst, ncols_x, nrows_y, scale); - break; - case 2048: - soft_max_f16<<>>(x, y, dst, ncols_x, nrows_y, scale); - break; - case 4096: - soft_max_f16<<>>(x, y, dst, ncols_x, nrows_y, scale); - break; - default: - soft_max_f16<<>>(x, y, dst, ncols_x, nrows_y, scale); - break; - } - } else { - const size_t shmem_low = WARP_SIZE*sizeof(half); - soft_max_f16<<>>(x, y, dst, ncols_x, nrows_y, scale); - } -} - -static void soft_max_f32_cuda(const float * x, const float * y, float * dst, const int ncols_x, const int nrows_x, const int nrows_y, const float scale, cudaStream_t stream) { +static void soft_max_f32_cuda(const float * x, const float * mask, const float * pos, float * dst, const int ncols_x, const int nrows_x, const int nrows_y, const float scale, const float max_bias, cudaStream_t stream) { int nth = WARP_SIZE; while (nth < ncols_x && nth < CUDA_SOFT_MAX_BLOCK_SIZE) nth *= 2; const dim3 block_dims(nth, 1, 1); const dim3 block_nums(nrows_x, 1, 1); const size_t shmem = (GGML_PAD(ncols_x, WARP_SIZE) + WARP_SIZE)*sizeof(float); static_assert(CUDA_SOFT_MAX_BLOCK_SIZE == 1024, "These values need to be adjusted."); + + const uint32_t n_head_kv = nrows_x/nrows_y; + const uint32_t n_head_log2 = 1u << (uint32_t) floorf(log2f((float) n_head_kv)); + + const float m0 = powf(2.0f, -(max_bias ) / n_head_log2); + const float m1 = powf(2.0f, -(max_bias / 2.0f) / n_head_log2); + if (shmem < g_device_caps[g_main_device].smpb) { switch (ncols_x) { case 32: - soft_max_f32<<>>(x, y, dst, ncols_x, nrows_y, scale); + soft_max_f32<<>>(x, mask, pos, dst, ncols_x, nrows_y, scale, max_bias, m0, m1, n_head_log2); break; case 64: - soft_max_f32<<>>(x, y, dst, ncols_x, nrows_y, scale); + soft_max_f32<<>>(x, mask, pos, dst, ncols_x, nrows_y, scale, max_bias, m0, m1, n_head_log2); break; case 128: - soft_max_f32<<>>(x, y, dst, ncols_x, nrows_y, scale); + soft_max_f32<<>>(x, mask, pos, dst, ncols_x, nrows_y, scale, max_bias, m0, m1, n_head_log2); break; case 256: - soft_max_f32<<>>(x, y, dst, ncols_x, nrows_y, scale); + soft_max_f32<<>>(x, mask, pos, dst, ncols_x, nrows_y, scale, max_bias, m0, m1, n_head_log2); break; case 512: - soft_max_f32<<>>(x, y, dst, ncols_x, nrows_y, scale); + soft_max_f32<<>>(x, mask, pos, dst, ncols_x, nrows_y, scale, max_bias, m0, m1, n_head_log2); break; case 1024: - soft_max_f32<<>>(x, y, dst, ncols_x, nrows_y, scale); + soft_max_f32<<>>(x, mask, pos, dst, ncols_x, nrows_y, scale, max_bias, m0, m1, n_head_log2); break; case 2048: - soft_max_f32<<>>(x, y, dst, ncols_x, nrows_y, scale); + soft_max_f32<<>>(x, mask, pos, dst, ncols_x, nrows_y, scale, max_bias, m0, m1, n_head_log2); break; case 4096: - soft_max_f32<<>>(x, y, dst, ncols_x, nrows_y, scale); + soft_max_f32<<>>(x, mask, pos, dst, ncols_x, nrows_y, scale, max_bias, m0, m1, n_head_log2); break; default: - soft_max_f32<<>>(x, y, dst, ncols_x, nrows_y, scale); + soft_max_f32<<>>(x, mask, pos, dst, ncols_x, nrows_y, scale, max_bias, m0, m1, n_head_log2); break; } } else { const size_t shmem_low = WARP_SIZE*sizeof(float); - soft_max_f32<<>>(x, y, dst, ncols_x, nrows_y, scale); + soft_max_f32<<>>(x, mask, pos, dst, ncols_x, nrows_y, scale, max_bias, m0, m1, n_head_log2); } } @@ -9090,30 +8937,36 @@ static void ggml_cuda_op_soft_max( GGML_ASSERT(!src1 || src1->type == GGML_TYPE_F32); // src1 contains mask and it is optional - const int64_t ne00 = src0->ne[0]; + const int64_t ne00 = src0->ne[0]; const int64_t nrows_x = ggml_nrows(src0); - const int64_t nrows_y = src1 ? ggml_nrows(src1) : 1; + const int64_t nrows_y = src0->ne[1]; - float scale = 1.0f; - memcpy(&scale, dst->op_params, sizeof(float)); + float scale = 1.0f; + float max_bias = 0.0f; -#if !(defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)) && CUDART_VERSION >= CUDART_HMAX -#ifdef GGML_CUDA_F16 - const bool use_f16_soft_max = true; -#else - const bool use_f16_soft_max = false; -#endif // GGML_CUDA_F16 -#else - const bool use_f16_soft_max = false; -#endif // defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__) && CUDART_VERSION >= CUDART_HMAX + memcpy(&scale, (float *) dst->op_params + 0, sizeof(float)); + memcpy(&max_bias, (float *) dst->op_params + 1, sizeof(float)); - if (use_f16_soft_max) { - soft_max_f16_cuda(src0_dd, src1 ? src1_dd : nullptr, dst_dd, ne00, nrows_x, nrows_y, scale, main_stream); - } else { - soft_max_f32_cuda(src0_dd, src1 ? src1_dd : nullptr, dst_dd, ne00, nrows_x, nrows_y, scale, main_stream); + // positions tensor + float * src2_dd = dst_dd; // default to avoid null checks in the kernel + cuda_pool_alloc src2_f; + + ggml_tensor * src2 = dst->src[2]; + const bool use_src2 = src2 != nullptr; + + if (use_src2) { + const bool src2_on_device = use_src2 && src2->backend == GGML_BACKEND_GPU; + ggml_tensor_extra_gpu * src2_extra = use_src2 ? (ggml_tensor_extra_gpu *) src2->extra : nullptr; + + if (src2_on_device) { + src2_dd = (float *) src2_extra->data_device[g_main_device]; + } else { + src2_dd = src2_f.alloc(ggml_nelements(src2)); + CUDA_CHECK(ggml_cuda_cpy_tensor_2d(src2_dd, src2, 0, 0, 0, 1, main_stream)); + } } - (void) dst; + soft_max_f32_cuda(src0_dd, src1 ? src1_dd : nullptr, src2_dd, dst_dd, ne00, nrows_x, nrows_y, scale, max_bias, main_stream); } static void ggml_cuda_op_scale( diff --git a/ggml-metal.m b/ggml-metal.m index 6e76f8bed..c0848a293 100644 --- a/ggml-metal.m +++ b/ggml-metal.m @@ -728,6 +728,7 @@ static bool ggml_metal_graph_compute( size_t offs_src0 = 0; size_t offs_src1 = 0; + size_t offs_src2 = 0; size_t offs_dst = 0; id command_buffer = command_buffers[cb_idx]; @@ -746,6 +747,7 @@ static bool ggml_metal_graph_compute( struct ggml_tensor * src0 = gf->nodes[i]->src[0]; struct ggml_tensor * src1 = gf->nodes[i]->src[1]; + struct ggml_tensor * src2 = gf->nodes[i]->src[2]; struct ggml_tensor * dst = gf->nodes[i]; switch (dst->op) { @@ -807,6 +809,7 @@ static bool ggml_metal_graph_compute( id id_src0 = src0 ? ggml_metal_get_buffer(src0, &offs_src0) : nil; id id_src1 = src1 ? ggml_metal_get_buffer(src1, &offs_src1) : nil; + id id_src2 = src2 ? ggml_metal_get_buffer(src2, &offs_src2) : nil; id id_dst = dst ? ggml_metal_get_buffer(dst, &offs_dst) : nil; //GGML_METAL_LOG_INFO("%s: op - %s\n", __func__, ggml_op_name(dst->op)); @@ -1188,7 +1191,16 @@ static bool ggml_metal_graph_compute( pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_SOFT_MAX].pipeline; } - const float scale = ((float *) dst->op_params)[0]; + const float scale = ((float *) dst->op_params)[0]; + const float max_bias = ((float *) dst->op_params)[1]; + + const int64_t nrows_x = ggml_nrows(src0); + const int64_t nrows_y = src0->ne[1]; + const uint32_t n_head_kv = nrows_x/nrows_y; + const uint32_t n_head_log2 = 1u << (uint32_t) floorf(log2f((float) n_head_kv)); + + const float m0 = powf(2.0f, -(max_bias ) / n_head_log2); + const float m1 = powf(2.0f, -(max_bias / 2.0f) / n_head_log2); [encoder setComputePipelineState:pipeline]; [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0]; @@ -1197,11 +1209,20 @@ static bool ggml_metal_graph_compute( } else { [encoder setBuffer:id_src0 offset:offs_src0 atIndex:1]; } - [encoder setBuffer:id_dst offset:offs_dst atIndex:2]; - [encoder setBytes:&ne00 length:sizeof(ne00) atIndex:3]; - [encoder setBytes:&ne01 length:sizeof(ne01) atIndex:4]; - [encoder setBytes:&ne02 length:sizeof(ne02) atIndex:5]; - [encoder setBytes:&scale length:sizeof(scale) atIndex:6]; + if (id_src2) { + [encoder setBuffer:id_src2 offset:offs_src2 atIndex:2]; + } else { + [encoder setBuffer:id_src0 offset:offs_src0 atIndex:2]; + } + [encoder setBuffer:id_dst offset:offs_dst atIndex:3]; + [encoder setBytes:&ne00 length:sizeof(ne00) atIndex:4]; + [encoder setBytes:&ne01 length:sizeof(ne01) atIndex:5]; + [encoder setBytes:&ne02 length:sizeof(ne02) atIndex:6]; + [encoder setBytes:&scale length:sizeof(scale) atIndex:7]; + [encoder setBytes:&max_bias length:sizeof(max_bias) atIndex:8]; + [encoder setBytes:&m0 length:sizeof(m0) atIndex:9]; + [encoder setBytes:&m1 length:sizeof(m1) atIndex:10]; + [encoder setBytes:&n_head_log2 length:sizeof(n_head_log2) atIndex:11]; [encoder setThreadgroupMemoryLength:32*sizeof(float) atIndex:0]; [encoder dispatchThreadgroups:MTLSizeMake(ne01*ne02*ne03, 1, 1) threadsPerThreadgroup:MTLSizeMake(nth, 1, 1)]; @@ -1514,8 +1535,6 @@ static bool ggml_metal_graph_compute( // max size of the src1ids array in the kernel stack GGML_ASSERT(ne11 <= 512); - struct ggml_tensor * src2 = gf->nodes[i]->src[2]; - const int64_t ne20 = src2 ? src2->ne[0] : 0; const int64_t ne21 = src2 ? src2->ne[1] : 0; const int64_t ne22 = src2 ? src2->ne[2] : 0; diff --git a/ggml-metal.metal b/ggml-metal.metal index efed6ad46..09ebcc9e3 100644 --- a/ggml-metal.metal +++ b/ggml-metal.metal @@ -351,12 +351,17 @@ kernel void kernel_sum_rows( kernel void kernel_soft_max( device const float * src0, device const float * src1, + device const float * src2, device float * dst, constant int64_t & ne00, constant int64_t & ne01, constant int64_t & ne02, constant float & scale, - threadgroup float * buf [[threadgroup(0)]], + constant float & max_bias, + constant float & m0, + constant float & m1, + constant uint32_t & n_head_log2, + threadgroup float * buf [[threadgroup(0)]], uint tgpig[[threadgroup_position_in_grid]], uint tpitg[[thread_position_in_threadgroup]], uint sgitg[[simdgroup_index_in_threadgroup]], @@ -368,13 +373,26 @@ kernel void kernel_soft_max( device const float * psrc0 = src0 + i03*ne02*ne01*ne00 + i02*ne01*ne00 + i01*ne00; device const float * pmask = src1 != src0 ? src1 + i01*ne00 : nullptr; + device const float * ppos = src2 != src0 ? src2 : nullptr; device float * pdst = dst + i03*ne02*ne01*ne00 + i02*ne01*ne00 + i01*ne00; + float slope = 0.0f; + + // ALiBi + if (max_bias > 0.0f) { + const int64_t h = i02; + + const float base = h < n_head_log2 ? m0 : m1; + const int exp = h < n_head_log2 ? h + 1 : 2*(h - n_head_log2) + 1; + + slope = pow(base, exp); + } + // parallel max float lmax = -INFINITY; for (int i00 = tpitg; i00 < ne00; i00 += ntg) { - lmax = MAX(lmax, psrc0[i00]*scale + (pmask ? pmask[i00] : 0.0f)); + lmax = MAX(lmax, psrc0[i00]*scale + (pmask ? pmask[i00] : 0.0f) + slope*ppos[i00]); } // find the max value in the block @@ -399,7 +417,7 @@ kernel void kernel_soft_max( // parallel sum float lsum = 0.0f; for (int i00 = tpitg; i00 < ne00; i00 += ntg) { - const float exp_psrc0 = exp((psrc0[i00]*scale + (pmask ? pmask[i00] : 0.0f)) - max_val); + const float exp_psrc0 = exp((psrc0[i00]*scale + (pmask ? pmask[i00] : 0.0f) + slope*ppos[i00]) - max_val); lsum += exp_psrc0; pdst[i00] = exp_psrc0; } @@ -437,12 +455,17 @@ kernel void kernel_soft_max( kernel void kernel_soft_max_4( device const float * src0, device const float * src1, + device const float * src2, device float * dst, constant int64_t & ne00, constant int64_t & ne01, constant int64_t & ne02, constant float & scale, - threadgroup float * buf [[threadgroup(0)]], + constant float & max_bias, + constant float & m0, + constant float & m1, + constant uint32_t & n_head_log2, + threadgroup float * buf [[threadgroup(0)]], uint tgpig[[threadgroup_position_in_grid]], uint tpitg[[thread_position_in_threadgroup]], uint sgitg[[simdgroup_index_in_threadgroup]], @@ -454,13 +477,25 @@ kernel void kernel_soft_max_4( device const float4 * psrc4 = (device const float4 *)(src0 + i03*ne02*ne01*ne00 + i02*ne01*ne00 + i01*ne00); device const float4 * pmask = src1 != src0 ? (device const float4 *)(src1 + i01*ne00) : nullptr; + device const float4 * ppos = src2 != src0 ? (device const float4 *)(src2) : nullptr; device float4 * pdst4 = (device float4 *)(dst + i03*ne02*ne01*ne00 + i02*ne01*ne00 + i01*ne00); + float slope = 0.0f; + + if (max_bias > 0.0f) { + const int64_t h = i02; + + const float base = h < n_head_log2 ? m0 : m1; + const int exp = h < n_head_log2 ? h + 1 : 2*(h - n_head_log2) + 1; + + slope = pow(base, exp); + } + // parallel max float4 lmax4 = -INFINITY; for (int i00 = tpitg; i00 < ne00/4; i00 += ntg) { - lmax4 = fmax(lmax4, psrc4[i00]*scale + (pmask ? pmask[i00] : 0.0f)); + lmax4 = fmax(lmax4, psrc4[i00]*scale + (pmask ? pmask[i00] : 0.0f) + slope*ppos[i00]); } const float lmax = MAX(MAX(lmax4[0], lmax4[1]), MAX(lmax4[2], lmax4[3])); @@ -486,7 +521,7 @@ kernel void kernel_soft_max_4( // parallel sum float4 lsum4 = 0.0f; for (int i00 = tpitg; i00 < ne00/4; i00 += ntg) { - const float4 exp_psrc4 = exp((psrc4[i00]*scale + (pmask ? pmask[i00] : 0.0f)) - max_val); + const float4 exp_psrc4 = exp((psrc4[i00]*scale + (pmask ? pmask[i00] : 0.0f) + slope*ppos[i00]) - max_val); lsum4 += exp_psrc4; pdst4[i00] = exp_psrc4; } diff --git a/ggml.c b/ggml.c index 264cfd705..e94024c62 100644 --- a/ggml.c +++ b/ggml.c @@ -5096,16 +5096,28 @@ static struct ggml_tensor * ggml_soft_max_impl( struct ggml_context * ctx, struct ggml_tensor * a, struct ggml_tensor * mask, + struct ggml_tensor * pos, float scale, + float max_bias, bool inplace) { GGML_ASSERT(ggml_is_contiguous(a)); + if (mask) { GGML_ASSERT(ggml_is_contiguous(mask)); - GGML_ASSERT(mask->ne[2] == 1); - GGML_ASSERT(mask->ne[3] == 1); + GGML_ASSERT(ggml_is_matrix(mask)); GGML_ASSERT(ggml_can_repeat_rows(mask, a)); } + if (pos) { + GGML_ASSERT(ggml_is_vector(pos)); + GGML_ASSERT(pos->type == GGML_TYPE_F32); + GGML_ASSERT(pos->ne[0] == a->ne[0]); + } + + if (max_bias > 0.0f) { + GGML_ASSERT(pos); + } + bool is_node = false; if (a->grad) { @@ -5114,13 +5126,14 @@ static struct ggml_tensor * ggml_soft_max_impl( struct ggml_tensor * result = inplace ? ggml_view_tensor(ctx, a) : ggml_dup_tensor(ctx, a); - float params[] = { scale }; + float params[] = { scale, max_bias }; ggml_set_op_params(result, params, sizeof(params)); result->op = GGML_OP_SOFT_MAX; result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL; result->src[0] = a; result->src[1] = mask; + result->src[2] = pos; return result; } @@ -5128,21 +5141,23 @@ static struct ggml_tensor * ggml_soft_max_impl( struct ggml_tensor * ggml_soft_max( struct ggml_context * ctx, struct ggml_tensor * a) { - return ggml_soft_max_impl(ctx, a, NULL, 1.0f, false); + return ggml_soft_max_impl(ctx, a, NULL, NULL, 1.0f, 0.0f, false); } struct ggml_tensor * ggml_soft_max_inplace( struct ggml_context * ctx, struct ggml_tensor * a) { - return ggml_soft_max_impl(ctx, a, NULL, 1.0f, true); + return ggml_soft_max_impl(ctx, a, NULL, NULL, 1.0f, 0.0f, true); } struct ggml_tensor * ggml_soft_max_ext( struct ggml_context * ctx, struct ggml_tensor * a, struct ggml_tensor * mask, - float scale) { - return ggml_soft_max_impl(ctx, a, mask, scale, false); + struct ggml_tensor * pos, + float scale, + float max_bias) { + return ggml_soft_max_impl(ctx, a, mask, pos, scale, max_bias, false); } // ggml_soft_max_back @@ -11495,6 +11510,7 @@ static void ggml_compute_forward_soft_max_f32( const struct ggml_compute_params * params, const struct ggml_tensor * src0, const struct ggml_tensor * src1, + const struct ggml_tensor * src2, struct ggml_tensor * dst) { assert(ggml_is_contiguous(dst)); assert(ggml_are_same_shape(src0, dst)); @@ -11503,16 +11519,29 @@ static void ggml_compute_forward_soft_max_f32( return; } - float scale = 1.0f; - memcpy(&scale, (float *) dst->op_params + 0, sizeof(float)); + float scale = 1.0f; + float max_bias = 0.0f; + + memcpy(&scale, (float *) dst->op_params + 0, sizeof(float)); + memcpy(&max_bias, (float *) dst->op_params + 1, sizeof(float)); // TODO: handle transposed/permuted matrices const int ith = params->ith; const int nth = params->nth; + GGML_TENSOR_UNARY_OP_LOCALS + const int64_t ne11 = src1 ? src1->ne[1] : 1; + // TODO: is this supposed to be ceil instead of floor? + // https://huggingface.co/mosaicml/mpt-7b/blob/main/attention.py#L370 + const uint32_t n_head_kv = ne02; + const uint32_t n_head_log2 = 1u << (uint32_t) floor(log2(n_head_kv)); + + const float m0 = powf(2.0f, -(max_bias ) / n_head_log2); + const float m1 = powf(2.0f, -(max_bias / 2.0f) / n_head_log2); + const int nc = src0->ne[0]; const int nr = ggml_nrows(src0); @@ -11525,6 +11554,9 @@ static void ggml_compute_forward_soft_max_f32( float * wp = (float *) params->wdata + (nc + CACHE_LINE_SIZE_F32) * ith; + // when max_bias <= 0.0f, src2 is not used and we default it to src0 to avoid branching + float * pos = src2 ? (float *) src2->data : src0->data; + for (int i1 = ir0; i1 < ir1; i1++) { float * sp = (float *)((char *) src0->data + i1*src0->nb[1]); float * dp = (float *)((char *) dst->data + i1*dst->nb[1]); @@ -11538,6 +11570,16 @@ static void ggml_compute_forward_soft_max_f32( ggml_vec_acc_f32(nc, wp, mp); } + // ALiBi bias + if (max_bias > 0.0f) { + const uint32_t h = (i1/ne01)%ne02; // head + const float slope = h < n_head_log2 ? powf(m0, h + 1) : powf(m1, 2*(h - n_head_log2) + 1); + + for (int i = 0; i < nc; i++) { + wp[i] = wp[i] + slope*pos[i]; + } + } + #ifndef NDEBUG for (int i = 0; i < nc; ++i) { //printf("p[%d] = %f\n", i, p[i]); @@ -11582,11 +11624,12 @@ static void ggml_compute_forward_soft_max( const struct ggml_compute_params * params, const struct ggml_tensor * src0, const struct ggml_tensor * src1, + const struct ggml_tensor * src2, struct ggml_tensor * dst) { switch (src0->type) { case GGML_TYPE_F32: { - ggml_compute_forward_soft_max_f32(params, src0, src1, dst); + ggml_compute_forward_soft_max_f32(params, src0, src1, src2, dst); } break; default: { @@ -11730,22 +11773,20 @@ static void ggml_compute_forward_alibi_f32( const float m0 = powf(2.0f, -(max_bias) / n_heads_log2_floor); const float m1 = powf(2.0f, -(max_bias / 2.0f) / n_heads_log2_floor); - for (int64_t i = 0; i < ne0; i++) { - for (int64_t j = 0; j < ne1; j++) { - for (int64_t k = 0; k < ne2_ne3; k++) { + for (int64_t k = 0; k < ne2_ne3; k++) { + // TODO: k*nb2 or k*nb3 + float m_k; + + if (k < n_heads_log2_floor) { + m_k = powf(m0, k + 1); + } else { + m_k = powf(m1, 2 * (k - n_heads_log2_floor) + 1); + } + + for (int64_t i = 0; i < ne0; i++) { + for (int64_t j = 0; j < ne1; j++) { float * const src = (float *)((char *) src0->data + i*nb0 + j*nb1 + k*nb2); float * pdst = (float *)((char *) dst->data + i*nb0 + j*nb1 + k*nb2); - - // TODO: k*nb2 or k*nb3 - - float m_k; - - if (k < n_heads_log2_floor) { - m_k = powf(m0, k + 1); - } else { - m_k = powf(m1, 2 * (k - n_heads_log2_floor) + 1); - } - pdst[0] = i * m_k + src[0]; } } @@ -11790,21 +11831,20 @@ static void ggml_compute_forward_alibi_f16( const float m0 = powf(2.0f, -(max_bias) / n_heads_log2_floor); const float m1 = powf(2.0f, -(max_bias / 2.0f) / n_heads_log2_floor); - for (int i = 0; i < ne0; i++) { - for (int j = 0; j < ne1; j++) { - for (int k = 0; k < ne2_ne3; k++) { + for (int k = 0; k < ne2_ne3; k++) { + // TODO: k*nb2 or k*nb3 + float m_k; + + if (k < n_heads_log2_floor) { + m_k = powf(m0, k + 1); + } else { + m_k = powf(m1, 2 * (k - n_heads_log2_floor) + 1); + } + + for (int i = 0; i < ne0; i++) { + for (int j = 0; j < ne1; j++) { ggml_fp16_t * const src = (ggml_fp16_t *)((char *) src0->data + i*nb0 + j*nb1 + k*nb2); - float * pdst = (float *)((char *) dst->data + i*nb0 + j*nb1 + k*nb2); - - // TODO: k*nb2 or k*nb3 - - float m_k; - - if (k < n_heads_log2_floor) { - m_k = powf(m0, k + 1); - } else { - m_k = powf(m1, 2 * (k - n_heads_log2_floor) + 1); - } + float * pdst = (float *)((char *) dst->data + i*nb0 + j*nb1 + k*nb2); // we return F32 pdst[0] = i * m_k + GGML_FP16_TO_FP32(src[0]); @@ -15116,7 +15156,7 @@ static void ggml_compute_forward(struct ggml_compute_params * params, struct ggm } break; case GGML_OP_SOFT_MAX: { - ggml_compute_forward_soft_max(params, tensor->src[0], tensor->src[1], tensor); + ggml_compute_forward_soft_max(params, tensor->src[0], tensor->src[1], tensor->src[2], tensor); } break; case GGML_OP_SOFT_MAX_BACK: { diff --git a/ggml.h b/ggml.h index 270018185..6c1956772 100644 --- a/ggml.h +++ b/ggml.h @@ -1383,13 +1383,17 @@ extern "C" { struct ggml_context * ctx, struct ggml_tensor * a); - // fused soft_max(a*scale + mask) + // fused soft_max(a*scale + mask + pos[i]*(ALiBi slope)) // mask is optional + // pos is required when max_bias > 0.0f + // max_bias = 0.0f for no ALiBi GGML_API struct ggml_tensor * ggml_soft_max_ext( struct ggml_context * ctx, struct ggml_tensor * a, struct ggml_tensor * mask, - float scale); + struct ggml_tensor * pos, + float scale, + float max_bias); GGML_API struct ggml_tensor * ggml_soft_max_back( struct ggml_context * ctx, @@ -1491,12 +1495,13 @@ extern "C" { // alibi position embedding // in-place, returns view(a) - GGML_API struct ggml_tensor * ggml_alibi( + GGML_DEPRECATED(GGML_API struct ggml_tensor * ggml_alibi( struct ggml_context * ctx, struct ggml_tensor * a, int n_past, int n_head, - float bias_max); + float bias_max), + "use ggml_soft_max_ext instead (will be removed in Mar 2024)"); // clamp // in-place, returns view(a) diff --git a/llama.cpp b/llama.cpp index 8966c3e66..6ac9caa95 100644 --- a/llama.cpp +++ b/llama.cpp @@ -1557,12 +1557,13 @@ struct llama_hparams { uint32_t n_yarn_orig_ctx; int32_t rope_scaling_type_train; - float f_clamp_kqv; - float f_max_alibi_bias; + float f_clamp_kqv = 0.0f; + float f_max_alibi_bias = 0.0f; bool causal_attn = true; - uint32_t pooling_type = LLAMA_POOLING_NONE; + bool need_kq_pos = false; + uint32_t pooling_type = LLAMA_POOLING_NONE; bool operator!=(const llama_hparams & other) const { if (this->vocab_only != other.vocab_only) return true; @@ -1923,6 +1924,7 @@ struct llama_context { struct ggml_tensor * inp_embd; // F32 [n_embd, n_batch] struct ggml_tensor * inp_pos; // I32 [n_batch] struct ggml_tensor * inp_KQ_mask; // F32 [n_ctx, n_batch] + struct ggml_tensor * inp_KQ_pos; // F32 [n_ctx] struct ggml_tensor * inp_K_shift; // I32 [n_ctx] struct ggml_tensor * inp_mean; // F32 [n_batch, n_batch] struct ggml_tensor * inp_cls; // I32 [n_batch] @@ -3054,6 +3056,11 @@ static void llm_load_hparams( case 40: model.type = e_model::MODEL_13B; break; default: model.type = e_model::MODEL_UNKNOWN; } + + if (model.type == e_model::MODEL_13B) { + // TODO: become GGUF KV parameter + hparams.f_max_alibi_bias = 8.0f; + } } break; case LLM_ARCH_STARCODER: { @@ -3081,6 +3088,9 @@ static void llm_load_hparams( case 32: model.type = e_model::MODEL_1B; break; default: model.type = e_model::MODEL_UNKNOWN; } + + // TODO: become GGUF KV parameter + hparams.f_max_alibi_bias = 8.0f; } break; case LLM_ARCH_BERT: { @@ -3126,11 +3136,12 @@ static void llm_load_hparams( case 4096: model.type = e_model::MODEL_7B; break; } break; } + + // TODO: become GGUF KV parameter + hparams.f_max_alibi_bias = 8.0f; } break; case LLM_ARCH_MPT: { - hparams.f_clamp_kqv = 0.0f; - ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps); ml.get_key(LLM_KV_ATTENTION_CLAMP_KQV, hparams.f_clamp_kqv, false); ml.get_key(LLM_KV_ATTENTION_MAX_ALIBI_BIAS, hparams.f_max_alibi_bias); @@ -3232,6 +3243,10 @@ static void llm_load_hparams( } model.ftype = ml.ftype; + + if (hparams.f_max_alibi_bias > 0.0f) { + hparams.need_kq_pos = true; + } } // TODO: This should probably be in llama.h @@ -4774,10 +4789,10 @@ static struct ggml_tensor * llm_build_kqv( struct ggml_tensor * wo_b, struct ggml_tensor * q_cur, struct ggml_tensor * kq_mask, + struct ggml_tensor * kq_pos, int64_t n_ctx, int32_t n_tokens, int32_t n_kv, - float max_alibi_bias, float kq_scale, const llm_build_cb & cb, int il) { @@ -4807,26 +4822,26 @@ static struct ggml_tensor * llm_build_kqv( ggml_mul_mat_set_prec(kq, GGML_PREC_F32); } - if (max_alibi_bias > 0.0f) { - // temporary branch until we figure out how to handle ggml_alibi through ggml_add +#if defined(GGML_USE_VULKAN) || defined(GGML_USE_KOMPUTE) || defined(GGML_USE_SYCL) +#pragma message("TODO: ALiBi support in ggml_soft_max_ext is not implemented for Vulkan, Kompute, and SYCL") +#pragma message(" Falling back to ggml_alibi(). Will become an error in Mar 2024") +#pragma message("ref: https://github.com/ggerganov/llama.cpp/pull/5488") + if (hparams.f_max_alibi_bias > 0.0f) { kq = ggml_scale(ctx, kq, kq_scale); cb(kq, "kq_scaled", il); - if (max_alibi_bias > 0.0f) { - // TODO: n_head or n_head_kv - // TODO: K-shift is likely not working - // TODO: change to ggml_add - kq = ggml_alibi(ctx, kq, /*n_past*/ 0, n_head, max_alibi_bias); - cb(kq, "kq_scaled_alibi", il); - } + kq = ggml_alibi(ctx, kq, /*n_past*/ 0, n_head, hparams.f_max_alibi_bias); + cb(kq, "kq_scaled_alibi", il); kq = ggml_add(ctx, kq, kq_mask); cb(kq, "kq_masked", il); kq = ggml_soft_max(ctx, kq); cb(kq, "kq_soft_max", il); - } else { - kq = ggml_soft_max_ext(ctx, kq, kq_mask, kq_scale); + } else +#endif + { + kq = ggml_soft_max_ext(ctx, kq, kq_mask, kq_pos, kq_scale, hparams.f_max_alibi_bias); cb(kq, "kq_soft_max_ext", il); } @@ -4874,11 +4889,11 @@ static struct ggml_tensor * llm_build_kv( struct ggml_tensor * v_cur, struct ggml_tensor * q_cur, struct ggml_tensor * kq_mask, + struct ggml_tensor * kq_pos, int64_t n_ctx, int32_t n_tokens, int32_t kv_head, int32_t n_kv, - float max_alibi_bias, float kq_scale, const llm_build_cb & cb, int il) { @@ -4892,9 +4907,8 @@ static struct ggml_tensor * llm_build_kv( llm_build_kv_store(ctx, hparams, kv, graph, k_cur, v_cur, n_ctx, n_tokens, kv_head, cb, il); struct ggml_tensor * cur; - cur = llm_build_kqv(ctx, model, hparams, kv, graph, - wo, wo_b, - q_cur, kq_mask, n_ctx, n_tokens, n_kv, max_alibi_bias, kq_scale, cb, il); + cur = llm_build_kqv(ctx, model, hparams, kv, graph, wo, wo_b, + q_cur, kq_mask, kq_pos, n_ctx, n_tokens, n_kv, kq_scale, cb, il); cb(cur, "kqv_out", il); return cur; @@ -5062,7 +5076,7 @@ struct llm_build_context { } Qcur = ggml_rope_custom( - ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, + ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, hparams.n_rot, 0, 0, n_orig_ctx, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); @@ -5077,7 +5091,7 @@ struct llm_build_context { cur = llm_build_kv(ctx0, model, hparams, kv_self, gf, model.layers[il].wo, model.layers[il].bo, - Kcur, Vcur, Qcur, KQ_mask, n_ctx, n_tokens, kv_head, n_kv, -1.0f, 1.0f/sqrtf(float(n_embd_head)), cb, il); + Kcur, Vcur, Qcur, KQ_mask, nullptr, n_ctx, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il); cb(cur, "kqv_out", il); } @@ -5207,6 +5221,10 @@ struct llm_build_context { struct ggml_tensor * KQ_mask = ggml_view_2d(ctx0, lctx.inp_KQ_mask, n_kv, n_tokens, n_kv*ggml_type_size(lctx.inp_KQ_mask->type), 0); cb(KQ_mask, "KQ_mask", -1); + // positions of the tokens in the KV cache + struct ggml_tensor * KQ_pos = ggml_view_1d(ctx0, lctx.inp_KQ_pos, n_kv, 0); + cb(KQ_pos, "KQ_pos", -1); + // shift the entire K-cache if needed if (do_rope_shift) { llm_build_k_shift(ctx0, hparams, cparams, kv_self, gf, lctx.inp_K_shift, LLM_ROPE, n_ctx, freq_base, freq_scale, cb); @@ -5255,12 +5273,9 @@ struct llm_build_context { cb(Kcur, "Kcur", il); - // apply ALiBi for 13B model - const float max_alibi_bias = model.type == MODEL_13B ? 8.0f : -1.0f; - cur = llm_build_kv(ctx0, model, hparams, kv_self, gf, model.layers[il].wo, NULL, - Kcur, Vcur, Qcur, KQ_mask, n_ctx, n_tokens, kv_head, n_kv, max_alibi_bias, 1.0f/sqrtf(float(n_embd_head)), cb, il); + Kcur, Vcur, Qcur, KQ_mask, KQ_pos, n_ctx, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il); cb(cur, "kqv_out", il); } @@ -5384,7 +5399,7 @@ struct llm_build_context { cur = llm_build_kv(ctx0, model, hparams, kv_self, gf, model.layers[il].wo, NULL, - Kcur, Vcur, Qcur, KQ_mask, n_ctx, n_tokens, kv_head, n_kv, -1.0f, 1.0f/sqrtf(float(n_embd_head)), cb, il); + Kcur, Vcur, Qcur, KQ_mask, nullptr, n_ctx, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il); cb(cur, "kqv_out", il); } @@ -5483,7 +5498,7 @@ struct llm_build_context { cur = llm_build_kv(ctx0, model, hparams, kv_self, gf, model.layers[il].wo, model.layers[il].bo, - Kcur, Vcur, Qcur, KQ_mask, n_ctx, n_tokens, kv_head, n_kv, -1.0f, 1.0f/sqrtf(float(n_embd_head)), cb, il); + Kcur, Vcur, Qcur, KQ_mask, nullptr, n_ctx, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il); cb(cur, "kqv_out", il); } @@ -5688,7 +5703,7 @@ struct llm_build_context { cur = llm_build_kv(ctx0, model, hparams, kv_self, gf, model.layers[il].wo, model.layers[il].bo, - Kcur, Vcur, Q, KQ_mask, n_ctx, n_tokens, kv_head, n_kv, -1.0f, 1.0f/sqrtf(float(n_embd_head)), cb, il); + Kcur, Vcur, Q, KQ_mask, nullptr, n_ctx, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il); cb(cur, "kqv_out", il); } @@ -5750,6 +5765,10 @@ struct llm_build_context { struct ggml_tensor * KQ_mask = ggml_view_2d(ctx0, lctx.inp_KQ_mask, n_kv, n_tokens, n_kv*ggml_type_size(lctx.inp_KQ_mask->type), 0); cb(KQ_mask, "KQ_mask", -1); + // positions of the tokens in the KV cache + struct ggml_tensor * KQ_pos = ggml_view_1d(ctx0, lctx.inp_KQ_pos, n_kv, 0); + cb(KQ_pos, "KQ_pos", -1); + for (int il = 0; il < n_layer; ++il) { struct ggml_tensor * inpSA = inpL; @@ -5777,7 +5796,7 @@ struct llm_build_context { cur = llm_build_kv(ctx0, model, hparams, kv_self, gf, model.layers[il].wo, NULL, - Kcur, Vcur, Qcur, KQ_mask, n_ctx, n_tokens, kv_head, n_kv, 8.0f, 1.0f/sqrtf(float(n_embd_head)), cb, il); + Kcur, Vcur, Qcur, KQ_mask, KQ_pos, n_ctx, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il); cb(cur, "kqv_out", il); } @@ -5878,7 +5897,7 @@ struct llm_build_context { cur = llm_build_kv(ctx0, model, hparams, kv_self, gf, model.layers[il].wo, model.layers[il].bo, - Kcur, Vcur, Qcur, KQ_mask, n_ctx, n_tokens, kv_head, n_kv, -1.0f, 1.0f/sqrtf(float(n_embd_head)), cb, il); + Kcur, Vcur, Qcur, KQ_mask, nullptr, n_ctx, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il); cb(cur, "kqv_out", il); } else { // compute Q and K and RoPE them @@ -5909,7 +5928,7 @@ struct llm_build_context { cur = llm_build_kv(ctx0, model, hparams, kv_self, gf, model.layers[il].wo, model.layers[il].bo, - Kcur, Vcur, Qcur, KQ_mask, n_ctx, n_tokens, kv_head, n_kv, -1.0f, 1.0f/sqrtf(float(n_embd_head)), cb, il); + Kcur, Vcur, Qcur, KQ_mask, nullptr, n_ctx, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il); cb(cur, "kqv_out", il); } @@ -5985,6 +6004,10 @@ struct llm_build_context { struct ggml_tensor * KQ_mask = ggml_view_2d(ctx0, lctx.inp_KQ_mask, n_kv, n_tokens, n_kv*ggml_type_size(lctx.inp_KQ_mask->type), 0); cb(KQ_mask, "KQ_mask", -1); + // positions of the tokens in the KV cache + struct ggml_tensor * KQ_pos = ggml_view_1d(ctx0, lctx.inp_KQ_pos, n_kv, 0); + cb(KQ_pos, "KQ_pos", -1); + inpL = llm_build_norm(ctx0, inpL, hparams, model.tok_norm, model.tok_norm_b, @@ -6018,7 +6041,7 @@ struct llm_build_context { cur = llm_build_kv(ctx0, model, hparams, kv_self, gf, model.layers[il].wo, model.layers[il].bo, - Kcur, Vcur, Qcur, KQ_mask, n_ctx, n_tokens, kv_head, n_kv, 8.0f, 1.0f/sqrtf(float(n_embd_head)), cb, il); + Kcur, Vcur, Qcur, KQ_mask, KQ_pos, n_ctx, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il); cb(cur, "kqv_out", il); } @@ -6078,6 +6101,10 @@ struct llm_build_context { struct ggml_tensor * KQ_mask = ggml_view_2d(ctx0, lctx.inp_KQ_mask, n_kv, n_tokens, n_kv*ggml_type_size(lctx.inp_KQ_mask->type), 0); cb(KQ_mask, "KQ_mask", -1); + // positions of the tokens in the KV cache + struct ggml_tensor * KQ_pos = ggml_view_1d(ctx0, lctx.inp_KQ_pos, n_kv, 0); + cb(KQ_pos, "KQ_pos", -1); + for (int il = 0; il < n_layer; ++il) { struct ggml_tensor * attn_norm; @@ -6111,7 +6138,7 @@ struct llm_build_context { cur = llm_build_kv(ctx0, model, hparams, kv_self, gf, model.layers[il].wo, NULL, - Kcur, Vcur, Qcur, KQ_mask, n_ctx, n_tokens, kv_head, n_kv, hparams.f_max_alibi_bias, 1.0f/sqrtf(float(n_embd_head)), cb, il); + Kcur, Vcur, Qcur, KQ_mask, KQ_pos, n_ctx, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il); cb(cur, "kqv_out", il); } @@ -6233,7 +6260,7 @@ struct llm_build_context { cur = llm_build_kv(ctx0, model, hparams, kv_self, gf, model.layers[il].wo, NULL, - Kcur, Vcur, Qcur, KQ_mask, n_ctx, n_tokens, kv_head, n_kv, -1.0f, 1.0f/sqrtf(float(n_embd_head)), cb, il); + Kcur, Vcur, Qcur, KQ_mask, nullptr, n_ctx, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il); cb(cur, "kqv_out", il); } @@ -6348,7 +6375,7 @@ struct llm_build_context { cur = llm_build_kv(ctx0, model, hparams, kv_self, gf, model.layers[il].wo, NULL, - Kcur, Vcur, Qcur, KQ_mask, n_ctx, n_tokens, kv_head, n_kv, -1.0f, 1.0f/sqrtf(float(n_embd_head)), cb, il); + Kcur, Vcur, Qcur, KQ_mask, nullptr, n_ctx, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il); cb(cur, "kqv_out", il); } @@ -6469,7 +6496,7 @@ struct llm_build_context { cur = llm_build_kv(ctx0, model, hparams, kv_self, gf, model.layers[il].wo, model.layers[il].bo, - Kcur, Vcur, Qcur, KQ_mask, n_ctx, n_tokens, kv_head, n_kv, -1.0f, 1.0f/sqrtf(float(n_embd_head)), cb, il); + Kcur, Vcur, Qcur, KQ_mask, nullptr, n_ctx, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il); cb(cur, "kqv_out", il); } @@ -6596,7 +6623,7 @@ struct llm_build_context { cur = llm_build_kv(ctx0, model, hparams, kv_self, gf, model.layers[il].wo, model.layers[il].bo, - Kcur, Vcur, Qcur, KQ_mask, n_ctx, n_tokens, kv_head, n_kv, -1.0f, 1.0f, cb, il); + Kcur, Vcur, Qcur, KQ_mask, nullptr, n_ctx, n_tokens, kv_head, n_kv, 1.0f, cb, il); cb(cur, "kqv_out", il); } @@ -6699,7 +6726,7 @@ struct llm_build_context { cur = llm_build_kv(ctx0, model, hparams, kv_self, gf, model.layers[il].wo, NULL, - Kcur, Vcur, Qcur, KQ_mask, n_ctx, n_tokens, kv_head, n_kv, -1.0f, 1.0f/sqrtf(float(n_embd_head)), cb, il); + Kcur, Vcur, Qcur, KQ_mask, nullptr, n_ctx, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il); cb(cur, "kqv_out", il); } struct ggml_tensor * sa_out = cur; @@ -6798,7 +6825,7 @@ struct llm_build_context { cur = llm_build_kv(ctx0, model, hparams, kv_self, gf, model.layers[il].wo, model.layers[il].bo, - Kcur, Vcur, Qcur, KQ_mask, n_ctx, n_tokens, kv_head, n_kv, -1.0f, 1.0f/sqrtf(float(n_embd_head)), cb, il); + Kcur, Vcur, Qcur, KQ_mask, nullptr, n_ctx, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il); cb(cur, "kqv_out", il); } @@ -6907,7 +6934,7 @@ struct llm_build_context { cur = llm_build_kv(ctx0, model, hparams, kv_self, gf, model.layers[il].wo, model.layers[il].bo, - Kcur, Vcur, Qcur, KQ_mask, n_ctx, n_tokens, kv_head, n_kv, -1.0f, 1.0f/sqrtf(float(n_embd_head)), cb, il); + Kcur, Vcur, Qcur, KQ_mask, nullptr, n_ctx, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il); cb(cur, "kqv_out", il); } @@ -7025,7 +7052,7 @@ struct llm_build_context { cur = llm_build_kv(ctx0, model, hparams, kv_self, gf, model.layers[il].wo, NULL, - Kcur, Vcur, Qcur, KQ_mask, n_ctx, n_tokens, kv_head, n_kv, -1.0f, 1.0f/sqrtf(float(n_embd_head)), cb, il); + Kcur, Vcur, Qcur, KQ_mask, nullptr, n_ctx, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il); cb(cur, "kqv_out", il); } @@ -7144,7 +7171,7 @@ struct llm_build_context { cur = llm_build_kv(ctx0, model, hparams, kv_self, gf, model.layers[il].wo, model.layers[il].bo, - Kcur, Vcur, Qcur, KQ_mask, n_ctx, n_tokens, kv_head, n_kv, -1.0f, 1.0f/sqrtf(float(n_embd_head)), cb, il); + Kcur, Vcur, Qcur, KQ_mask, nullptr, n_ctx, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il); cb(cur, "kqv_out", il); } @@ -7276,7 +7303,7 @@ struct llm_build_context { cur = llm_build_kv(ctx0, model, hparams, kv_self, gf, model.layers[il].wo, model.layers[il].bo, - Kcur, Vcur, Qcur, KQ_mask, n_ctx, n_tokens, kv_head, n_kv, -1.0f, 1.0f/sqrtf(float(n_embd_head)), cb, il); + Kcur, Vcur, Qcur, KQ_mask, nullptr, n_ctx, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il); cb(cur, "kqv_out", il); } @@ -7507,6 +7534,18 @@ static void llama_set_inputs(llama_context & lctx, const llama_batch & batch) { } } + if (hparams.need_kq_pos) { + const int64_t n_kv = kv_self.n; + + assert(ggml_backend_buffer_is_host(lctx.inp_KQ_pos->buffer)); + + float * data = (float *) lctx.inp_KQ_pos->data; + + for (int i = 0; i < n_kv; ++i) { + data[i] = float(lctx.kv_self.cells[i].pos); + } + } + if (kv_self.has_shift) { const int64_t n_ctx = cparams.n_ctx; @@ -11434,7 +11473,7 @@ struct llama_context * llama_new_context_with_model( // graph inputs { ggml_init_params init_params = { - /* .mem_size */ ggml_tensor_overhead()*7, + /* .mem_size */ ggml_tensor_overhead()*8, /* .mem_buffer */ nullptr, /* .no_alloc */ true, }; @@ -11444,6 +11483,7 @@ struct llama_context * llama_new_context_with_model( ctx->inp_embd = ggml_new_tensor_2d(ctx->ctx_input, GGML_TYPE_F32, hparams.n_embd, cparams.n_batch); ctx->inp_pos = ggml_new_tensor_1d(ctx->ctx_input, GGML_TYPE_I32, cparams.n_batch); ctx->inp_KQ_mask = ggml_new_tensor_2d(ctx->ctx_input, GGML_TYPE_F32, cparams.n_ctx, cparams.n_batch); + ctx->inp_KQ_pos = ggml_new_tensor_1d(ctx->ctx_input, GGML_TYPE_F32, cparams.n_ctx); ctx->inp_K_shift = ggml_new_tensor_1d(ctx->ctx_input, GGML_TYPE_I32, cparams.n_ctx); ctx->inp_mean = ggml_new_tensor_2d(ctx->ctx_input, GGML_TYPE_F32, cparams.n_batch, cparams.n_batch); ctx->inp_cls = ggml_new_tensor_1d(ctx->ctx_input, GGML_TYPE_I32, cparams.n_batch); @@ -11452,6 +11492,7 @@ struct llama_context * llama_new_context_with_model( ggml_set_name(ctx->inp_embd, "inp_embd"); ggml_set_name(ctx->inp_pos, "inp_pos"); ggml_set_name(ctx->inp_KQ_mask, "inp_KQ_mask"); + ggml_set_name(ctx->inp_KQ_pos, "inp_KQ_pos"); ggml_set_name(ctx->inp_K_shift, "inp_K_shift"); ggml_set_name(ctx->inp_mean, "inp_mean"); ggml_set_name(ctx->inp_cls, "inp_cls"); diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index 9af8517d9..30a7d1f5a 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -1085,24 +1085,32 @@ struct test_diag_mask_inf : public test_case { struct test_soft_max : public test_case { const ggml_type type; const std::array ne; - const float scale; const bool mask; + const float scale; + const float max_bias; std::string vars() override { - return VARS_TO_STR4(type, ne, scale, mask); + return VARS_TO_STR5(type, ne, mask, scale, max_bias); } test_soft_max(ggml_type type = GGML_TYPE_F32, std::array ne = {10, 10, 10, 10}, + bool mask = false, float scale = 1.0f, - bool mask = false) - : type(type), ne(ne), scale(scale), mask(mask) {} + float max_bias = 0.0f) + : type(type), ne(ne), mask(mask), scale(scale), max_bias(max_bias) {} ggml_tensor * build_graph(ggml_context * ctx) override { ggml_tensor * a = ggml_new_tensor(ctx, type, 4, ne.data()); - ggml_tensor * b = nullptr; - if (mask) { b = ggml_new_tensor_2d(ctx, type, ne[0], ne[1]); } - ggml_tensor * out = ggml_soft_max_ext(ctx, a, b, scale); + ggml_tensor * mask = nullptr; + if (this->mask) { + mask = ggml_new_tensor_2d(ctx, type, ne[0], ne[1]); + } + ggml_tensor * pos = nullptr; + if (max_bias > 0.0f) { + pos = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, ne[0]); + } + ggml_tensor * out = ggml_soft_max_ext(ctx, a, mask, pos, scale, max_bias); return out; } }; @@ -1147,30 +1155,6 @@ struct test_rope : public test_case { } }; -// GGML_OP_ALIBI -struct test_alibi : public test_case { - const ggml_type type; - const std::array ne; - int n_past; - int n_head; - float bias_max; - - std::string vars() override { - return VARS_TO_STR5(type, ne, n_past, n_head, bias_max); - } - - test_alibi(ggml_type type = GGML_TYPE_F32, - std::array ne = {10, 10, 10, 10}, - int n_past = 512, int n_head = 10, float bias_max = 0.5f) - : type(type), ne(ne), n_past(n_past), n_head(n_head), bias_max(bias_max) {} - - ggml_tensor * build_graph(ggml_context * ctx) override { - ggml_tensor * a = ggml_new_tensor(ctx, type, 4, ne.data()); - ggml_tensor * out = ggml_alibi(ctx, a, n_past, n_head, bias_max); - return out; - } -}; - // GGML_OP_POOL2D struct test_pool2d : public test_case { enum ggml_op_pool pool_type; @@ -1488,7 +1472,7 @@ struct test_moe : public test_case { ggml_tensor * cur = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, n_embd, n_tokens); ggml_tensor * logits = ggml_mul_mat(ctx, ffn_gate_inp, cur); - ggml_tensor * probs = ggml_soft_max_ext(ctx, logits, nullptr, 1.0f/sqrtf(n_embd)); + ggml_tensor * probs = ggml_soft_max_ext(ctx, logits, nullptr, nullptr, 1.0f/sqrtf(n_embd), 0.0f); // select experts ggml_tensor * selected_experts = ggml_top_k(ctx, probs, n_experts_per_tok); @@ -1617,7 +1601,6 @@ public: ggml_cpy(ctx, v_cur_t, v_cache_view); } - // if max_alibi_bias > 0 then apply ALiBi struct ggml_tensor * llm_build_kqv( struct ggml_context * ctx, struct ggml_tensor * k_l, @@ -1636,7 +1619,7 @@ public: struct ggml_tensor * kq = ggml_mul_mat(ctx, k, q); - kq = ggml_soft_max_ext(ctx, kq, kq_mask, kq_scale); + kq = ggml_soft_max_ext(ctx, kq, kq_mask, nullptr, kq_scale, 0.0f); // split cached v into n_head heads struct ggml_tensor * v = @@ -2083,6 +2066,7 @@ static bool test_backend(ggml_backend_t backend, test_mode mode, const char * op test_cases.emplace_back(new test_diag_mask_inf(GGML_TYPE_F32, {10, 10, 10, 1}, 5)); test_cases.emplace_back(new test_diag_mask_inf(GGML_TYPE_F32, {10, 10, 10, 10}, 5)); +#if 0 std::uniform_int_distribution<> dist_ne1(1, 50); int exponent = 1; while (exponent < (1 << 17)) { @@ -2091,14 +2075,29 @@ static bool test_backend(ggml_backend_t backend, test_mode mode, const char * op for (int n = 0; n < 10; ++n) { int64_t ne0 = dist_ne0(rng); int64_t ne1 = dist_ne1(rng); - test_cases.emplace_back(new test_soft_max(GGML_TYPE_F32, {ne0, ne1, 1, 1})); + test_cases.emplace_back(new test_soft_max(GGML_TYPE_F32, {ne0, ne1, 1, 1}, n/2 == 0, 0.1f, ne0 < 1000 ? 4.0f : 0.0f)); } exponent <<= 1; } +#endif + for (bool mask : {false, true}) { + for (float max_bias : {0.0f, 8.0f}) { + for (float scale : {1.0f, 0.1f}) { + for (int64_t ne0 : {16, 1024}) { + for (int64_t ne1 : {16, 1024}) { + test_cases.emplace_back(new test_soft_max(GGML_TYPE_F32, {ne0, ne1, 1, 1}, mask, scale, max_bias)); + test_cases.emplace_back(new test_soft_max(GGML_TYPE_F32, {ne0-1, ne1-1, 1, 1}, mask, scale, max_bias)); + } + } + } + } + } - test_cases.emplace_back(new test_soft_max(GGML_TYPE_F32, {16, 2, 32, 1}, 0.1f)); - test_cases.emplace_back(new test_soft_max(GGML_TYPE_F32, {32, 2, 32, 1}, 0.1f, true)); + test_cases.emplace_back(new test_soft_max(GGML_TYPE_F32, {16, 2, 32, 1}, false, 0.1f, 0.0f)); + test_cases.emplace_back(new test_soft_max(GGML_TYPE_F32, {32, 2, 32, 1}, true, 0.1f, 0.0f)); + test_cases.emplace_back(new test_soft_max(GGML_TYPE_F32, {16, 2, 32, 1}, false, 0.1f, 8.0f)); + test_cases.emplace_back(new test_soft_max(GGML_TYPE_F32, {32, 2, 32, 1}, true, 0.1f, 8.0f)); for (ggml_type type : {GGML_TYPE_F32, GGML_TYPE_F16}) { test_cases.emplace_back(new test_rope(type, {128, 32, 10, 1}, 128, 0, 512)); // llama 7B @@ -2113,7 +2112,6 @@ static bool test_backend(ggml_backend_t backend, test_mode mode, const char * op test_cases.emplace_back(new test_rope(type, { 80, 32, 10, 1}, 32, 2, 512)); // neox (phi-2) } - test_cases.emplace_back(new test_alibi()); test_cases.emplace_back(new test_concat(GGML_TYPE_F32)); test_cases.emplace_back(new test_concat(GGML_TYPE_I32)); From c8e0d7efeb7634ecc2e9832e879ab9fca4510e71 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Sun, 18 Feb 2024 00:17:07 +0000 Subject: [PATCH 367/812] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/f8e2ebd66d097614d51a56a755450d4ae1632df1' (2024-02-07) → 'github:NixOS/nixpkgs/5863c27340ba4de8f83e7e3c023b9599c3cb3c80' (2024-02-16) --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index 239d0686c..47d6448b5 100644 --- a/flake.lock +++ b/flake.lock @@ -20,11 +20,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1707268954, - "narHash": "sha256-2en1kvde3cJVc3ZnTy8QeD2oKcseLFjYPLKhIGDanQ0=", + "lastModified": 1708118438, + "narHash": "sha256-kk9/0nuVgA220FcqH/D2xaN6uGyHp/zoxPNUmPCMmEE=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "f8e2ebd66d097614d51a56a755450d4ae1632df1", + "rev": "5863c27340ba4de8f83e7e3c023b9599c3cb3c80", "type": "github" }, "original": { From bd2d4e393b2b7d2a1b2e201058e26017c9728ead Mon Sep 17 00:00:00 2001 From: Kawrakow <48489457+ikawrakow@users.noreply.github.com> Date: Sun, 18 Feb 2024 18:16:55 +0200 Subject: [PATCH 368/812] 1.5 bit quantization (#5453) * iq1_s: WIP basics * iq1_s: CUDA is working * iq1_s: scalar CPU dot product * iq1_s: WIP AVX2 dot product - something is not right * Fix tests * Fix shadow warnings * Fix after merge with latest master * iq1_s: AVX2 finally works * iq1_s: ARM_NEON dot product. Works, but not very fast * iq1_s: better grid * iq1_s: use IQ2_XXS for attn_output At a cost of 0.04 extra bpw this gives a big improvement in PPL. * iq1_s: Metal basics Dequantize works, but not dot product * iq1_s: Metal works, but quite slow As usual, Apple Silicon does not like the code I write. * iq1_s: Tests * iq1_s: slightly faster dot product --------- Co-authored-by: Iwan Kawrakow --- examples/quantize/quantize.cpp | 6 +- ggml-backend.c | 2 +- ggml-cuda.cu | 224 ++++++++++- ggml-metal.m | 29 +- ggml-metal.metal | 337 +++++++++++++++++ ggml-quants.c | 657 +++++++++++++++++++++++++++++++-- ggml-quants.h | 14 +- ggml.c | 44 ++- ggml.h | 2 + llama.cpp | 16 +- llama.h | 1 + tests/test-backend-ops.cpp | 2 +- 12 files changed, 1286 insertions(+), 48 deletions(-) diff --git a/examples/quantize/quantize.cpp b/examples/quantize/quantize.cpp index 4a5c504e3..ea7ba50c9 100644 --- a/examples/quantize/quantize.cpp +++ b/examples/quantize/quantize.cpp @@ -23,6 +23,7 @@ static const std::vector QUANT_OPTIONS = { { "Q5_1", LLAMA_FTYPE_MOSTLY_Q5_1, " 4.70G, +0.0349 ppl @ LLaMA-v1-7B", }, { "IQ2_XXS",LLAMA_FTYPE_MOSTLY_IQ2_XXS," 2.06 bpw quantization", }, { "IQ2_XS", LLAMA_FTYPE_MOSTLY_IQ2_XS, " 2.31 bpw quantization", }, + { "IQ1_S", LLAMA_FTYPE_MOSTLY_IQ1_S, " 1.56 bpw quantization", }, { "Q2_K", LLAMA_FTYPE_MOSTLY_Q2_K, " 2.63G, +0.6717 ppl @ LLaMA-v1-7B", }, { "Q2_K_S", LLAMA_FTYPE_MOSTLY_Q2_K_S, " 2.16G, +9.0634 ppl @ LLaMA-v1-7B", }, { "IQ3_XXS",LLAMA_FTYPE_MOSTLY_IQ3_XXS," 3.06 bpw quantization", }, @@ -287,9 +288,10 @@ int main(int argc, char ** argv) { } } - if ((params.ftype == LLAMA_FTYPE_MOSTLY_IQ2_XS || params.ftype == LLAMA_FTYPE_MOSTLY_IQ2_XXS || params.ftype == LLAMA_FTYPE_MOSTLY_Q2_K_S) && imatrix_data.empty()) { + if ((params.ftype == LLAMA_FTYPE_MOSTLY_IQ2_XS || params.ftype == LLAMA_FTYPE_MOSTLY_IQ2_XXS || + params.ftype == LLAMA_FTYPE_MOSTLY_Q2_K_S || params.ftype == LLAMA_FTYPE_MOSTLY_IQ1_S) && imatrix_data.empty()) { fprintf(stderr, "\n===============================================================================================\n"); - fprintf(stderr, "Please do not use IQ2_XXS, IQ2_XS or Q2_K_S quantization without an importance matrix\n"); + fprintf(stderr, "Please do not use IQ1_S, IQ2_XXS, IQ2_XS or Q2_K_S quantization without an importance matrix\n"); fprintf(stderr, "===============================================================================================\n\n\n"); return 1; } diff --git a/ggml-backend.c b/ggml-backend.c index 66e8c293a..5076d9e5e 100644 --- a/ggml-backend.c +++ b/ggml-backend.c @@ -756,7 +756,7 @@ GGML_CALL static bool ggml_backend_cpu_graph_compute(ggml_backend_t backend, str GGML_CALL static bool ggml_backend_cpu_supports_op(ggml_backend_t backend, const struct ggml_tensor * op) { switch (op->op) { case GGML_OP_CPY: - return op->type != GGML_TYPE_IQ2_XXS && op->type != GGML_TYPE_IQ2_XS; // missing type_traits.from_float + return op->type != GGML_TYPE_IQ2_XXS && op->type != GGML_TYPE_IQ2_XS && op->type != GGML_TYPE_IQ1_S; // missing type_traits.from_float case GGML_OP_MUL_MAT: return op->src[1]->type == GGML_TYPE_F32 || op->src[1]->type == ggml_internal_get_type_traits(op->src[0]->type).vec_dot_type; default: diff --git a/ggml-cuda.cu b/ggml-cuda.cu index 5fd8a87e4..933ebbc4e 100644 --- a/ggml-cuda.cu +++ b/ggml-cuda.cu @@ -517,6 +517,15 @@ typedef struct { } block_iq3_xxs; static_assert(sizeof(block_iq3_xxs) == sizeof(ggml_fp16_t) + 3*(QK_K/8), "wrong iq3_xxs block size/padding"); +#define QR1_S 8 +#define QI1_S (QK_K / (4*QR1_S)) +typedef struct { + half d; + uint8_t qs[QK_K/8]; + uint8_t scales[QK_K/16]; +} block_iq1_s; +static_assert(sizeof(block_iq1_s) == sizeof(ggml_fp16_t) + QK_K/8 + QK_K/16, "wrong iq1_s block size/padding"); + #define WARP_SIZE 32 #define MATRIX_ROW_PADDING 512 // last row of quant. matrices is a multiple of this to avoid out-of-bounds memory accesses @@ -1681,6 +1690,137 @@ static const __device__ uint32_t iq3xxs_grid[256] = { 0x3e1c1c1c, 0x3e1c3404, 0x3e24140c, 0x3e24240c, 0x3e2c0404, 0x3e2c0414, 0x3e2c1424, 0x3e341c04, }; +static const __device__ uint64_t iq1s_grid[512] = { + 0xffffffffffff0101, 0xffffffffff01ff00, 0xffffffffff010100, 0xffffffff00000000, + 0xffffffff01ff00ff, 0xffffffff01ff0001, 0xffffffff0101ffff, 0xffffffff0101ff01, + 0xffffff00ff000000, 0xffffff000000ff00, 0xffffff00000000ff, 0xffffff0000000100, + 0xffffff0000010000, 0xffffff0001000000, 0xffffff01ffff00ff, 0xffffff01ff01ff00, + 0xffffff01ff010100, 0xffffff0100000001, 0xffffff0101ffff00, 0xffffff0101ff0101, + 0xffffff0101010100, 0xffff00ffff00ff01, 0xffff00ffff0000ff, 0xffff00ff00ff0100, + 0xffff00ff0100ff00, 0xffff00ff010001ff, 0xffff0000ff0101ff, 0xffff000000ffff00, + 0xffff000000000000, 0xffff00000001ff01, 0xffff000001000101, 0xffff0000010100ff, + 0xffff0001ffff0100, 0xffff00010000ff00, 0xffff000100010101, 0xffff000101000000, + 0xffff01ffffff0000, 0xffff01ffff01ffff, 0xffff01ffff010100, 0xffff01ff00000000, + 0xffff01ff01ffffff, 0xffff01ff01ff0001, 0xffff01ff0101ffff, 0xffff01ff01010001, + 0xffff0100ffffff01, 0xffff01000000ffff, 0xffff010000000100, 0xffff010001ff01ff, + 0xffff010001000000, 0xffff0101ff000000, 0xffff0101000101ff, 0xffff010101ffff01, + 0xffff01010101ff00, 0xff00ffffff000000, 0xff00ffff00ffff00, 0xff00ffff00000001, + 0xff00ffff000001ff, 0xff00ffff01010000, 0xff00ff00ffff0000, 0xff00ff00ff00ff00, + 0xff00ff00ff0000ff, 0xff00ff00ff000100, 0xff00ff00ff010001, 0xff00ff0000ff0001, + 0xff00ff000000ffff, 0xff00ff0000000000, 0xff00ff000001ff00, 0xff00ff0000010100, + 0xff00ff0001ff0000, 0xff00ff000100ff00, 0xff00ff0001000100, 0xff00ff01ff000000, + 0xff00ff0100ff0000, 0xff00ff01000001ff, 0xff00ff0101010001, 0xff0000ff00000000, + 0xff0000ff0001ff00, 0xff0000ff00010100, 0xff000000ffff0101, 0xff000000ff000000, + 0xff000000ff01ff00, 0xff00000000ff0000, 0xff0000000000ff00, 0xff000000000000ff, + 0xff00000000000000, 0xff00000000000001, 0xff00000000000100, 0xff0000000001ffff, + 0xff00000000010000, 0xff00000001000000, 0xff00000001010100, 0xff000001ff00ff01, + 0xff000001ff0100ff, 0xff00000100000000, 0xff0000010001ff00, 0xff00000101ff0100, + 0xff0000010100ff00, 0xff0001ff00ff00ff, 0xff0001ff00000101, 0xff0001ff000100ff, + 0xff0001ff01000000, 0xff000100ff0001ff, 0xff0001000000ff01, 0xff00010000000000, + 0xff00010000010001, 0xff00010000010100, 0xff00010001ffff00, 0xff00010001ff0101, + 0xff00010001010000, 0xff000101ffffffff, 0xff000101ff000101, 0xff00010101ff00ff, + 0xff00010101000001, 0xff000101010100ff, 0xff01ffffff000101, 0xff01ffffff01ffff, + 0xff01ffffff01ff01, 0xff01ffffff0101ff, 0xff01ffff00000000, 0xff01ffff01ff0001, + 0xff01ffff0101ff01, 0xff01ff00ff000000, 0xff01ff0000ff0100, 0xff01ff000000ff01, + 0xff01ff0000010000, 0xff01ff00010000ff, 0xff01ff01ff01ff00, 0xff01ff0100000101, + 0xff0100ffffff0000, 0xff0100ffff010000, 0xff0100ff01ff00ff, 0xff0100ff01000100, + 0xff0100ff010100ff, 0xff010000ffffff01, 0xff01000000000000, 0xff0100000101ff00, + 0xff010001ffff00ff, 0xff010001ff000100, 0xff01000100ffff00, 0xff01000100010001, + 0xff01000101ff0001, 0xff010001010001ff, 0xff0101ffffffffff, 0xff0101ffff01ffff, + 0xff0101ffff010101, 0xff0101ff0000ff00, 0xff0101ff01010001, 0xff010100ff000000, + 0xff010100ff01ff01, 0xff01010000ff0001, 0xff01010000000100, 0xff01010001000000, + 0xff0101010100ffff, 0x00ffffff0000ff01, 0x00ffffff000000ff, 0x00ffffff00000100, + 0x00ffffff00010000, 0x00ffff00ffff0001, 0x00ffff00ff0000ff, 0x00ffff00ff000100, + 0x00ffff0000000000, 0x00ffff0001000100, 0x00ffff0001010001, 0x00ffff01ff00ff01, + 0x00ffff0100ff0100, 0x00ffff010000ff00, 0x00ffff01000100ff, 0x00ffff0101ff00ff, + 0x00ffff010101ff00, 0x00ff00ffffffffff, 0x00ff00ffffff01ff, 0x00ff00ffff000101, + 0x00ff00ff00000000, 0x00ff00ff000101ff, 0x00ff00ff01010101, 0x00ff0000ff000000, + 0x00ff0000ff01ffff, 0x00ff000000ff0000, 0x00ff00000000ff00, 0x00ff0000000000ff, + 0x00ff000000000000, 0x00ff000000000001, 0x00ff000000000100, 0x00ff000000010000, + 0x00ff000001ffff01, 0x00ff000001000000, 0x00ff0001ff000101, 0x00ff000100ffffff, + 0x00ff000100000000, 0x00ff0001010001ff, 0x00ff01ffff000000, 0x00ff01ff0001ff00, + 0x00ff01ff01ff0100, 0x00ff0100ff01ff01, 0x00ff010000ff00ff, 0x00ff010000ff0101, + 0x00ff010000000000, 0x00ff010000010101, 0x00ff01000100ff00, 0x00ff010001010000, + 0x00ff0101ffffff00, 0x00ff01010000ff01, 0x00ff010100000100, 0x00ff010101ff0000, + 0x0000ffffffff0100, 0x0000ffffff00ff00, 0x0000ffffff0000ff, 0x0000ffffff010000, + 0x0000ffff00000000, 0x0000ffff00010101, 0x0000ffff01ffff01, 0x0000ffff01000100, + 0x0000ff00ff000000, 0x0000ff00ff01ff00, 0x0000ff00ff0101ff, 0x0000ff0000ff0000, + 0x0000ff000000ff00, 0x0000ff00000000ff, 0x0000ff0000000000, 0x0000ff0000000001, + 0x0000ff0000000100, 0x0000ff0000010000, 0x0000ff0001ffffff, 0x0000ff0001ff01ff, + 0x0000ff0001000000, 0x0000ff000101ffff, 0x0000ff01ffff0101, 0x0000ff01ff010000, + 0x0000ff0100000000, 0x0000ff0101000101, 0x000000ffffff0001, 0x000000ffff000000, + 0x000000ff00ff0000, 0x000000ff0000ff00, 0x000000ff000000ff, 0x000000ff00000000, + 0x000000ff00000001, 0x000000ff00000100, 0x000000ff00010000, 0x000000ff01000000, + 0x000000ff0101ff00, 0x00000000ffff0000, 0x00000000ff00ff00, 0x00000000ff0000ff, + 0x00000000ff000000, 0x00000000ff000001, 0x00000000ff000100, 0x00000000ff010000, + 0x0000000000ffff00, 0x0000000000ff00ff, 0x0000000000ff0000, 0x0000000000ff0001, + 0x0000000000ff0100, 0x000000000000ffff, 0x000000000000ff00, 0x000000000000ff01, + 0x00000000000000ff, 0x0000000000000001, 0x00000000000001ff, 0x0000000000000100, + 0x0000000000000101, 0x000000000001ff00, 0x00000000000100ff, 0x0000000000010000, + 0x0000000000010001, 0x0000000000010100, 0x0000000001ff0000, 0x000000000100ff00, + 0x00000000010000ff, 0x0000000001000000, 0x0000000001000001, 0x0000000001000100, + 0x0000000001010000, 0x00000001ffff01ff, 0x00000001ff000000, 0x0000000100ff0000, + 0x000000010000ff00, 0x00000001000000ff, 0x0000000100000000, 0x0000000100000001, + 0x0000000100000100, 0x0000000100010000, 0x0000000101000000, 0x000001ffff00ff00, + 0x000001ffff010001, 0x000001ffff0101ff, 0x000001ff00ffff01, 0x000001ff0000ffff, + 0x000001ff00000000, 0x000001ff010000ff, 0x000001ff01010100, 0x00000100ffff0100, + 0x00000100ff000000, 0x0000010000ff0000, 0x000001000000ff00, 0x00000100000000ff, + 0x0000010000000000, 0x0000010000000001, 0x0000010000000100, 0x0000010000010000, + 0x0000010001000000, 0x000001000101ff01, 0x00000101ffff0001, 0x00000101ff01ffff, + 0x0000010100000000, 0x0000010101010100, 0x0001ffffff000000, 0x0001ffff00ffffff, + 0x0001ffff00000100, 0x0001ffff0001ff00, 0x0001ffff01000000, 0x0001ff00ffffff00, + 0x0001ff00ffff01ff, 0x0001ff00ff010000, 0x0001ff0000000000, 0x0001ff0000010001, + 0x0001ff0001ff0000, 0x0001ff0001010100, 0x0001ff01ff0000ff, 0x0001ff01ff000001, + 0x0001ff0100ffffff, 0x0001ff010001ffff, 0x0001ff01000101ff, 0x0001ff010100ff01, + 0x000100ffff00ffff, 0x000100ffff00ff01, 0x000100ffff000100, 0x000100ff00000000, + 0x000100ff000101ff, 0x000100ff01ff0101, 0x000100ff0100ffff, 0x000100ff01010101, + 0x00010000ff000000, 0x00010000ff010100, 0x0001000000ff0000, 0x000100000000ff00, + 0x00010000000000ff, 0x0001000000000000, 0x0001000000000001, 0x0001000000000100, + 0x0001000000010000, 0x0001000001ffff01, 0x0001000001000000, 0x0001000100ff0101, + 0x0001000100000000, 0x00010001010100ff, 0x000101ffffff01ff, 0x000101ffffff0101, + 0x000101ff00010000, 0x000101ff01ff0000, 0x000101ff0100ff01, 0x00010100ffff0000, + 0x0001010000000000, 0x000101000001ffff, 0x0001010000010101, 0x00010100010001ff, + 0x00010101ff00ff00, 0x00010101ff010001, 0x0001010100ffffff, 0x0001010100ff01ff, + 0x00010101000101ff, 0x0001010101ff0000, 0x000101010100ff01, 0x0001010101000101, + 0x01ffffffffff0101, 0x01ffffffff01ffff, 0x01ffffffff01ff01, 0x01ffffffff0101ff, + 0x01ffffffff010101, 0x01ffffff00000000, 0x01ffffff01ff01ff, 0x01ffffff01000101, + 0x01ffffff0101ff01, 0x01ffffff010100ff, 0x01ffff000000ff00, 0x01ffff0000000001, + 0x01ffff00000001ff, 0x01ffff0000010000, 0x01ffff0001ff0000, 0x01ffff01ffffffff, + 0x01ffff01ffff01ff, 0x01ffff01ff000000, 0x01ffff01ff01ffff, 0x01ffff01ff0101ff, + 0x01ffff010100ffff, 0x01ff00ffffff0000, 0x01ff00ffff010000, 0x01ff00ff00ffff01, + 0x01ff0000ff0000ff, 0x01ff000000000000, 0x01ff00000001ff01, 0x01ff000001ffffff, + 0x01ff000001010100, 0x01ff0001ffffff01, 0x01ff0001ff010001, 0x01ff000101ff0100, + 0x01ff000101000001, 0x01ff0001010100ff, 0x01ff01ffff00ffff, 0x01ff01ff00010001, + 0x01ff01ff01000000, 0x01ff01ff010101ff, 0x01ff0100ff000001, 0x01ff010000ffff00, + 0x01ff010000000100, 0x01ff010001ff01ff, 0x01ff01000101ffff, 0x01ff0101ffff00ff, + 0x01ff0101ffff0101, 0x01ff0101ff0101ff, 0x01ff010100010000, 0x0100ffff00ff00ff, + 0x0100ffff00ff0001, 0x0100ffff00000100, 0x0100ffff0100ff00, 0x0100ff00ffff0000, + 0x0100ff00ff00ffff, 0x0100ff00ff00ff01, 0x0100ff00ff000100, 0x0100ff00ff010000, + 0x0100ff0000000000, 0x0100ff00000100ff, 0x0100ff0001ff0101, 0x0100ff0001010101, + 0x0100ff0100ff00ff, 0x0100ff0100ff0001, 0x0100ff0100000100, 0x0100ff0100010001, + 0x0100ff0101000000, 0x010000ffff00ff00, 0x010000ff0000ffff, 0x010000ff00000000, + 0x010000ff010001ff, 0x010000ff01010001, 0x01000000ffffff00, 0x01000000ffff0101, + 0x01000000ff000000, 0x01000000ff0100ff, 0x01000000ff010101, 0x0100000000ff0000, + 0x010000000000ff00, 0x01000000000000ff, 0x0100000000000000, 0x0100000000000001, + 0x0100000000000100, 0x0100000000010000, 0x0100000001000000, 0x0100000100000000, + 0x01000001000101ff, 0x0100000101ffff01, 0x010001ffff000101, 0x010001ff00ff0100, + 0x010001ff0000ff00, 0x010001ff000100ff, 0x010001ff01ffffff, 0x01000100ffff0000, + 0x01000100ff0001ff, 0x0100010000000000, 0x010001000001ff00, 0x0100010001ff0000, + 0x01000100010000ff, 0x0100010001000101, 0x01000101ff00ff01, 0x0100010100ff0100, + 0x010001010000ffff, 0x0100010101010001, 0x0101ffffffff0101, 0x0101ffffff0001ff, + 0x0101ffffff01ffff, 0x0101ffffff010101, 0x0101ffff00000000, 0x0101ffff0101ffff, + 0x0101ffff010101ff, 0x0101ff00ff000000, 0x0101ff0000ff0100, 0x0101ff000000ff00, + 0x0101ff0000010000, 0x0101ff00010000ff, 0x0101ff0001000001, 0x0101ff01ff010101, + 0x0101ff0100000000, 0x0101ff010101ff00, 0x010100ffffff0000, 0x010100ffff010000, + 0x010100ff00ff01ff, 0x010100ff000000ff, 0x010100ff00000101, 0x010100ff01ffff00, + 0x01010000ffffff01, 0x01010000ff000100, 0x01010000ff01ff01, 0x0101000000000000, + 0x01010000000100ff, 0x010100000101ff01, 0x01010001ffff0000, 0x01010001ff00ffff, + 0x01010001ff010000, 0x0101000101ffffff, 0x0101000101ff01ff, 0x0101000101010101, + 0x010101ffff01ffff, 0x010101ff00000000, 0x010101ff0001ff01, 0x010101ff0101ffff, + 0x010101ff010101ff, 0x01010100ffffffff, 0x01010100ff000001, 0x010101000000ff00, + 0x0101010001010000, 0x0101010100ff0001, 0x010101010001ff01, 0x010101010101ffff, +}; + static const __device__ uint8_t ksigns_iq2xs[128] = { 0, 129, 130, 3, 132, 5, 6, 135, 136, 9, 10, 139, 12, 141, 142, 15, 144, 17, 18, 147, 20, 149, 150, 23, 24, 153, 154, 27, 156, 29, 30, 159, @@ -1823,6 +1963,29 @@ static __global__ void dequantize_block_iq3_xxs(const void * __restrict__ vx, ds } +template +static __global__ void dequantize_block_iq1_s(const void * __restrict__ vx, dst_t * __restrict__ yy) { + + const int i = blockIdx.x; + const block_iq1_s * x = (const block_iq1_s *) vx; + + const int tid = threadIdx.x; +#if QK_K == 256 + const int il = tid/8; // 0...3 + const int ib = tid%8; // 0...7 + dst_t * y = yy + i*QK_K + 32*ib + 8*il; + const int i8 = 4*ib+il; + uint8_t h = x[i].scales[i8/2] >> 4*(i8%2); + const int8_t * grid = (const int8_t *)(iq1s_grid + (x[i].qs[i8] | ((h & 8) << 5))); + const float d = (float)x[i].d * (2*(h & 7) + 1); + for (int j = 0; j < 8; ++j) y[j] = d * grid[j]; +#else + assert(false); +#endif + +} + + static __global__ void dequantize_mul_mat_vec_q2_k(const void * __restrict__ vx, const float * __restrict__ yy, float * __restrict__ dst, const int ncols, int nrows) { static_assert(16%K_QUANTS_PER_ITERATION == 0, "16 must be divisible by K_QUANTS_PER_ITERATION"); @@ -4522,6 +4685,49 @@ static __device__ __forceinline__ float vec_dot_iq3_xxs_q8_1( #endif } +static __device__ __forceinline__ float vec_dot_iq1_s_q8_1( + const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) { +#if QK_K == 256 + const block_iq1_s * bq1 = (const block_iq1_s *) vbq; + + const int ib32 = iqs; + int sumi1 = 0, sumi2 = 0, sumi3 = 0, sumi4 = 0; + const uint8_t h1 = bq1->scales[2*ib32+0]; + const uint8_t h2 = bq1->scales[2*ib32+1]; +#if __CUDA_ARCH__ >= MIN_CC_DP4A // lowest compute capability for integer intrinsics + const int * q8 = (const int *)bq8_1[ib32].qs; + const int * grid1 = (const int *)(iq1s_grid + (bq1->qs[4*ib32+0] | ((h1 & 0x08) << 5))); + const int * grid2 = (const int *)(iq1s_grid + (bq1->qs[4*ib32+1] | ((h1 & 0x80) << 1))); + const int * grid3 = (const int *)(iq1s_grid + (bq1->qs[4*ib32+2] | ((h2 & 0x08) << 5))); + const int * grid4 = (const int *)(iq1s_grid + (bq1->qs[4*ib32+3] | ((h2 & 0x80) << 1))); + for (int j = 0; j < 2; ++j) { + sumi1 = __dp4a(q8[j+0], grid1[j], sumi1); + sumi2 = __dp4a(q8[j+2], grid2[j], sumi2); + sumi3 = __dp4a(q8[j+4], grid3[j], sumi3); + sumi4 = __dp4a(q8[j+6], grid4[j], sumi4); + } +#else + const int8_t * q8 = bq8_1[ib32].qs; + const int8_t * grid1 = (const int8_t *)(iq1s_grid + (bq1->qs[4*ib32+0] | ((h1 & 0x08) << 5))); + const int8_t * grid2 = (const int8_t *)(iq1s_grid + (bq1->qs[4*ib32+1] | ((h1 & 0x80) << 1))); + const int8_t * grid3 = (const int8_t *)(iq1s_grid + (bq1->qs[4*ib32+2] | ((h2 & 0x08) << 5))); + const int8_t * grid4 = (const int8_t *)(iq1s_grid + (bq1->qs[4*ib32+3] | ((h2 & 0x80) << 1))); + for (int j = 0; j < 8; ++j) { + sumi1 += q8[j+ 0] * grid1[j]; + sumi2 += q8[j+ 8] * grid2[j]; + sumi3 += q8[j+16] * grid3[j]; + sumi4 += q8[j+24] * grid4[j]; + } +#endif + const float d = (float)bq1->d * __low2float(bq8_1[ib32].ds); + return d * (sumi1 * (2*(h1 & 7) + 1) + sumi2 * (2*((h1 >> 4) & 7) + 1) + + sumi3 * (2*(h2 & 7) + 1) + sumi4 * (2*((h2 >> 4) & 7) + 1)); +#else + assert(false); + return 0.f; +#endif +} + template static __device__ __forceinline__ void mul_mat_q( @@ -6561,6 +6767,12 @@ static void dequantize_row_iq3_xxs_cuda(const void * vx, dst_t * y, const int k, dequantize_block_iq3_xxs<<>>(vx, y); } +template +static void dequantize_row_iq1_s_cuda(const void * vx, dst_t * y, const int k, cudaStream_t stream) { + const int nb = k / QK_K; + dequantize_block_iq1_s<<>>(vx, y); +} + template static void convert_unary_cuda(const void * __restrict__ vx, dst_t * __restrict__ y, const int k, cudaStream_t stream) { const int num_blocks = (k + CUDA_DEQUANTIZE_BLOCK_SIZE - 1) / CUDA_DEQUANTIZE_BLOCK_SIZE; @@ -6600,6 +6812,8 @@ static to_fp16_cuda_t ggml_get_to_fp16_cuda(ggml_type type) { return dequantize_row_iq2_xs_cuda; case GGML_TYPE_IQ3_XXS: return dequantize_row_iq3_xxs_cuda; + case GGML_TYPE_IQ1_S: + return dequantize_row_iq1_s_cuda; case GGML_TYPE_F32: return convert_unary_cuda; default: @@ -6635,6 +6849,8 @@ static to_fp32_cuda_t ggml_get_to_fp32_cuda(ggml_type type) { return dequantize_row_iq2_xs_cuda; case GGML_TYPE_IQ3_XXS: return dequantize_row_iq3_xxs_cuda; + case GGML_TYPE_IQ1_S: + return dequantize_row_iq1_s_cuda; case GGML_TYPE_F16: return convert_unary_cuda; default: @@ -8378,6 +8594,7 @@ static int64_t get_row_rounding(ggml_type type, const std::array= CC_RDNA2 ? 128 : 64; default: GGML_ASSERT(false); @@ -8401,6 +8618,7 @@ static int64_t get_row_rounding(ggml_type type, const std::array= CC_VOLTA ? 128 : 64; case GGML_TYPE_Q6_K: return 64; @@ -8498,6 +8716,10 @@ static void ggml_cuda_op_mul_mat_vec_q( mul_mat_vec_q_cuda (src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_padded_row_size, src1_ncols, nrows_dst, stream); break; + case GGML_TYPE_IQ1_S: + mul_mat_vec_q_cuda + (src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_padded_row_size, src1_ncols, nrows_dst, stream); + break; default: GGML_ASSERT(false); break; @@ -11214,7 +11436,7 @@ GGML_CALL static bool ggml_backend_cuda_supports_op(ggml_backend_t backend, cons return false; } ggml_type a_type = a->type; - if (a_type == GGML_TYPE_IQ2_XXS || a_type == GGML_TYPE_IQ2_XS || a_type == GGML_TYPE_IQ3_XXS) { + if (a_type == GGML_TYPE_IQ2_XXS || a_type == GGML_TYPE_IQ2_XS || a_type == GGML_TYPE_IQ3_XXS || a_type == GGML_TYPE_IQ1_S) { if (b->ne[1] == 1 && ggml_nrows(b) > 1) { return false; } diff --git a/ggml-metal.m b/ggml-metal.m index c0848a293..f3c1fff8f 100644 --- a/ggml-metal.m +++ b/ggml-metal.m @@ -61,6 +61,7 @@ enum ggml_metal_kernel_type { GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ2_XXS, GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ2_XS, GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ3_XXS, + GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ1_S, GGML_METAL_KERNEL_TYPE_GET_ROWS_I32, GGML_METAL_KERNEL_TYPE_RMS_NORM, GGML_METAL_KERNEL_TYPE_GROUP_NORM, @@ -83,6 +84,7 @@ enum ggml_metal_kernel_type { GGML_METAL_KERNEL_TYPE_MUL_MV_IQ2_XXS_F32, GGML_METAL_KERNEL_TYPE_MUL_MV_IQ2_XS_F32, GGML_METAL_KERNEL_TYPE_MUL_MV_IQ3_XXS_F32, + GGML_METAL_KERNEL_TYPE_MUL_MV_IQ1_S_F32, GGML_METAL_KERNEL_TYPE_MUL_MV_ID_F32_F32, //GGML_METAL_KERNEL_TYPE_MUL_MV_ID_F16_F16, GGML_METAL_KERNEL_TYPE_MUL_MV_ID_F16_F32, @@ -101,6 +103,7 @@ enum ggml_metal_kernel_type { GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ2_XXS_F32, GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ2_XS_F32, GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ3_XXS_F32, + GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ1_S_F32, GGML_METAL_KERNEL_TYPE_MUL_MM_F32_F32, GGML_METAL_KERNEL_TYPE_MUL_MM_F16_F32, GGML_METAL_KERNEL_TYPE_MUL_MM_Q4_0_F32, @@ -116,6 +119,7 @@ enum ggml_metal_kernel_type { GGML_METAL_KERNEL_TYPE_MUL_MM_IQ2_XXS_F32, GGML_METAL_KERNEL_TYPE_MUL_MM_IQ2_XS_F32, GGML_METAL_KERNEL_TYPE_MUL_MM_IQ3_XXS_F32, + GGML_METAL_KERNEL_TYPE_MUL_MM_IQ1_S_F32, GGML_METAL_KERNEL_TYPE_MUL_MM_ID_F32_F32, GGML_METAL_KERNEL_TYPE_MUL_MM_ID_F16_F32, GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q4_0_F32, @@ -131,6 +135,7 @@ enum ggml_metal_kernel_type { GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ2_XXS_F32, GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ2_XS_F32, GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ3_XXS_F32, + GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ1_S_F32, GGML_METAL_KERNEL_TYPE_ROPE_F32, GGML_METAL_KERNEL_TYPE_ROPE_F16, GGML_METAL_KERNEL_TYPE_ALIBI_F32, @@ -433,6 +438,7 @@ static struct ggml_metal_context * ggml_metal_init(int n_cb) { GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ2_XXS, get_rows_iq2_xxs, true); GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ2_XS, get_rows_iq2_xs, true); GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ3_XXS, get_rows_iq3_xxs, true); + GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ1_S, get_rows_iq1_s, true); GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_I32, get_rows_i32, true); GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_RMS_NORM, rms_norm, ctx->support_simdgroup_reduction); GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GROUP_NORM, group_norm, ctx->support_simdgroup_reduction); @@ -455,6 +461,7 @@ static struct ggml_metal_context * ggml_metal_init(int n_cb) { GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_IQ2_XXS_F32, mul_mv_iq2_xxs_f32, ctx->support_simdgroup_reduction); GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_IQ2_XS_F32, mul_mv_iq2_xs_f32, ctx->support_simdgroup_reduction); GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_IQ3_XXS_F32, mul_mv_iq3_xxs_f32, ctx->support_simdgroup_reduction); + GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_IQ1_S_F32, mul_mv_iq1_s_f32, ctx->support_simdgroup_reduction); GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_F32_F32, mul_mv_id_f32_f32, ctx->support_simdgroup_reduction); //GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_F16_F16, mul_mv_id_f16_f16, ctx->support_simdgroup_reduction); GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_F16_F32, mul_mv_id_f16_f32, ctx->support_simdgroup_reduction); @@ -473,6 +480,7 @@ static struct ggml_metal_context * ggml_metal_init(int n_cb) { GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ2_XXS_F32, mul_mv_id_iq2_xxs_f32, ctx->support_simdgroup_reduction); GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ2_XS_F32, mul_mv_id_iq2_xs_f32, ctx->support_simdgroup_reduction); GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ3_XXS_F32, mul_mv_id_iq3_xxs_f32, ctx->support_simdgroup_reduction); + GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ1_S_F32, mul_mv_id_iq1_s_f32, ctx->support_simdgroup_reduction); GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_F32_F32, mul_mm_f32_f32, ctx->support_simdgroup_mm); GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_F16_F32, mul_mm_f16_f32, ctx->support_simdgroup_mm); GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_Q4_0_F32, mul_mm_q4_0_f32, ctx->support_simdgroup_mm); @@ -488,6 +496,7 @@ static struct ggml_metal_context * ggml_metal_init(int n_cb) { GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_IQ2_XXS_F32, mul_mm_iq2_xxs_f32, ctx->support_simdgroup_mm); GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_IQ2_XS_F32, mul_mm_iq2_xs_f32, ctx->support_simdgroup_mm); GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_IQ3_XXS_F32, mul_mm_iq3_xxs_f32, ctx->support_simdgroup_mm); + GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_IQ1_S_F32, mul_mm_iq1_s_f32, ctx->support_simdgroup_mm); GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_F32_F32, mul_mm_id_f32_f32, ctx->support_simdgroup_mm); GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_F16_F32, mul_mm_id_f16_f32, ctx->support_simdgroup_mm); GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q4_0_F32, mul_mm_id_q4_0_f32, ctx->support_simdgroup_mm); @@ -503,6 +512,7 @@ static struct ggml_metal_context * ggml_metal_init(int n_cb) { GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ2_XXS_F32, mul_mm_id_iq2_xxs_f32, ctx->support_simdgroup_mm); GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ2_XS_F32, mul_mm_id_iq2_xs_f32, ctx->support_simdgroup_mm); GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ3_XXS_F32, mul_mm_id_iq3_xxs_f32, ctx->support_simdgroup_mm); + GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ1_S_F32, mul_mm_id_iq1_s_f32, ctx->support_simdgroup_mm); GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_ROPE_F32, rope_f32, true); GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_ROPE_F16, rope_f16, true); GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_ALIBI_F32, alibi_f32, true); @@ -1318,6 +1328,7 @@ static bool ggml_metal_graph_compute( case GGML_TYPE_IQ2_XXS: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_IQ2_XXS_F32].pipeline; break; case GGML_TYPE_IQ2_XS: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_IQ2_XS_F32 ].pipeline; break; case GGML_TYPE_IQ3_XXS: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_IQ3_XXS_F32].pipeline; break; + case GGML_TYPE_IQ1_S: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_IQ1_S_F32 ].pipeline; break; default: GGML_ASSERT(false && "MUL MAT-MAT not implemented"); } @@ -1452,6 +1463,12 @@ static bool ggml_metal_graph_compute( nth1 = 16; pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_IQ3_XXS_F32].pipeline; } break; + case GGML_TYPE_IQ1_S: + { + nth0 = 4; + nth1 = 16; + pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_IQ1_S_F32].pipeline; + } break; default: { GGML_METAL_LOG_ERROR("Asserting on type %d\n", (int)src0t); @@ -1486,7 +1503,7 @@ static bool ggml_metal_graph_compute( if (src0t == GGML_TYPE_Q4_0 || src0t == GGML_TYPE_Q4_1 || src0t == GGML_TYPE_Q5_0 || src0t == GGML_TYPE_Q5_1 || src0t == GGML_TYPE_Q8_0 || - src0t == GGML_TYPE_Q2_K) { // || src0t == GGML_TYPE_Q4_K) { + src0t == GGML_TYPE_Q2_K || src0t == GGML_TYPE_IQ1_S) { // || src0t == GGML_TYPE_Q4_K) { [encoder dispatchThreadgroups:MTLSizeMake((ne01 + 7)/8, ne11, ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)]; } else if (src0t == GGML_TYPE_IQ2_XXS || src0t == GGML_TYPE_IQ2_XS) { @@ -1592,6 +1609,7 @@ static bool ggml_metal_graph_compute( case GGML_TYPE_IQ2_XXS: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ2_XXS_F32].pipeline; break; case GGML_TYPE_IQ2_XS: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ2_XS_F32 ].pipeline; break; case GGML_TYPE_IQ3_XXS: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ3_XXS_F32].pipeline; break; + case GGML_TYPE_IQ1_S: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ1_S_F32 ].pipeline; break; default: GGML_ASSERT(false && "MUL_MAT_ID not implemented"); } @@ -1729,6 +1747,12 @@ static bool ggml_metal_graph_compute( nth1 = 16; pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ3_XXS_F32].pipeline; } break; + case GGML_TYPE_IQ1_S: + { + nth0 = 4; + nth1 = 16; + pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ1_S_F32].pipeline; + } break; default: { GGML_METAL_LOG_ERROR("Asserting on type %d\n", (int)src2t); @@ -1779,7 +1803,7 @@ static bool ggml_metal_graph_compute( if (src2t == GGML_TYPE_Q4_0 || src2t == GGML_TYPE_Q4_1 || src2t == GGML_TYPE_Q5_0 || src2t == GGML_TYPE_Q5_1 || src2t == GGML_TYPE_Q8_0 || - src2t == GGML_TYPE_Q2_K) { // || src2t == GGML_TYPE_Q4_K) { + src2t == GGML_TYPE_Q2_K || src2t == GGML_TYPE_IQ1_S) { // || src2t == GGML_TYPE_Q4_K) { [encoder dispatchThreadgroups:MTLSizeMake((ne21 + 7)/8, _ne1, ne01*ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)]; } else if (src2t == GGML_TYPE_IQ2_XXS || src2t == GGML_TYPE_IQ2_XS) { @@ -1833,6 +1857,7 @@ static bool ggml_metal_graph_compute( case GGML_TYPE_IQ2_XXS: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ2_XXS].pipeline; break; case GGML_TYPE_IQ2_XS: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ2_XS ].pipeline; break; case GGML_TYPE_IQ3_XXS: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ3_XXS].pipeline; break; + case GGML_TYPE_IQ1_S: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ1_S ].pipeline; break; case GGML_TYPE_I32: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GET_ROWS_I32 ].pipeline; break; default: GGML_ASSERT(false && "not implemented"); } diff --git a/ggml-metal.metal b/ggml-metal.metal index 09ebcc9e3..a00962111 100644 --- a/ggml-metal.metal +++ b/ggml-metal.metal @@ -2525,6 +2525,13 @@ typedef struct { } block_iq3_xxs; // 98 bytes / block for QK_K = 256, so 3.0625 bpw +typedef struct { + half d; + uint8_t qs[QK_K/8]; + uint8_t scales[QK_K/16]; +} block_iq1_s; + + //====================================== dot products ========================= void kernel_mul_mv_q2_K_f32_impl( @@ -3782,6 +3789,137 @@ constexpr constant static uint32_t iq3xxs_grid[256] = { 0x3e1c1c1c, 0x3e1c3404, 0x3e24140c, 0x3e24240c, 0x3e2c0404, 0x3e2c0414, 0x3e2c1424, 0x3e341c04, }; +#define NGRID_IQ1S 512 +constexpr constant static uint64_t iq1s_grid[NGRID_IQ1S] = { + 0xffffffffffff0101, 0xffffffffff01ff00, 0xffffffffff010100, 0xffffffff00000000, + 0xffffffff01ff00ff, 0xffffffff01ff0001, 0xffffffff0101ffff, 0xffffffff0101ff01, + 0xffffff00ff000000, 0xffffff000000ff00, 0xffffff00000000ff, 0xffffff0000000100, + 0xffffff0000010000, 0xffffff0001000000, 0xffffff01ffff00ff, 0xffffff01ff01ff00, + 0xffffff01ff010100, 0xffffff0100000001, 0xffffff0101ffff00, 0xffffff0101ff0101, + 0xffffff0101010100, 0xffff00ffff00ff01, 0xffff00ffff0000ff, 0xffff00ff00ff0100, + 0xffff00ff0100ff00, 0xffff00ff010001ff, 0xffff0000ff0101ff, 0xffff000000ffff00, + 0xffff000000000000, 0xffff00000001ff01, 0xffff000001000101, 0xffff0000010100ff, + 0xffff0001ffff0100, 0xffff00010000ff00, 0xffff000100010101, 0xffff000101000000, + 0xffff01ffffff0000, 0xffff01ffff01ffff, 0xffff01ffff010100, 0xffff01ff00000000, + 0xffff01ff01ffffff, 0xffff01ff01ff0001, 0xffff01ff0101ffff, 0xffff01ff01010001, + 0xffff0100ffffff01, 0xffff01000000ffff, 0xffff010000000100, 0xffff010001ff01ff, + 0xffff010001000000, 0xffff0101ff000000, 0xffff0101000101ff, 0xffff010101ffff01, + 0xffff01010101ff00, 0xff00ffffff000000, 0xff00ffff00ffff00, 0xff00ffff00000001, + 0xff00ffff000001ff, 0xff00ffff01010000, 0xff00ff00ffff0000, 0xff00ff00ff00ff00, + 0xff00ff00ff0000ff, 0xff00ff00ff000100, 0xff00ff00ff010001, 0xff00ff0000ff0001, + 0xff00ff000000ffff, 0xff00ff0000000000, 0xff00ff000001ff00, 0xff00ff0000010100, + 0xff00ff0001ff0000, 0xff00ff000100ff00, 0xff00ff0001000100, 0xff00ff01ff000000, + 0xff00ff0100ff0000, 0xff00ff01000001ff, 0xff00ff0101010001, 0xff0000ff00000000, + 0xff0000ff0001ff00, 0xff0000ff00010100, 0xff000000ffff0101, 0xff000000ff000000, + 0xff000000ff01ff00, 0xff00000000ff0000, 0xff0000000000ff00, 0xff000000000000ff, + 0xff00000000000000, 0xff00000000000001, 0xff00000000000100, 0xff0000000001ffff, + 0xff00000000010000, 0xff00000001000000, 0xff00000001010100, 0xff000001ff00ff01, + 0xff000001ff0100ff, 0xff00000100000000, 0xff0000010001ff00, 0xff00000101ff0100, + 0xff0000010100ff00, 0xff0001ff00ff00ff, 0xff0001ff00000101, 0xff0001ff000100ff, + 0xff0001ff01000000, 0xff000100ff0001ff, 0xff0001000000ff01, 0xff00010000000000, + 0xff00010000010001, 0xff00010000010100, 0xff00010001ffff00, 0xff00010001ff0101, + 0xff00010001010000, 0xff000101ffffffff, 0xff000101ff000101, 0xff00010101ff00ff, + 0xff00010101000001, 0xff000101010100ff, 0xff01ffffff000101, 0xff01ffffff01ffff, + 0xff01ffffff01ff01, 0xff01ffffff0101ff, 0xff01ffff00000000, 0xff01ffff01ff0001, + 0xff01ffff0101ff01, 0xff01ff00ff000000, 0xff01ff0000ff0100, 0xff01ff000000ff01, + 0xff01ff0000010000, 0xff01ff00010000ff, 0xff01ff01ff01ff00, 0xff01ff0100000101, + 0xff0100ffffff0000, 0xff0100ffff010000, 0xff0100ff01ff00ff, 0xff0100ff01000100, + 0xff0100ff010100ff, 0xff010000ffffff01, 0xff01000000000000, 0xff0100000101ff00, + 0xff010001ffff00ff, 0xff010001ff000100, 0xff01000100ffff00, 0xff01000100010001, + 0xff01000101ff0001, 0xff010001010001ff, 0xff0101ffffffffff, 0xff0101ffff01ffff, + 0xff0101ffff010101, 0xff0101ff0000ff00, 0xff0101ff01010001, 0xff010100ff000000, + 0xff010100ff01ff01, 0xff01010000ff0001, 0xff01010000000100, 0xff01010001000000, + 0xff0101010100ffff, 0x00ffffff0000ff01, 0x00ffffff000000ff, 0x00ffffff00000100, + 0x00ffffff00010000, 0x00ffff00ffff0001, 0x00ffff00ff0000ff, 0x00ffff00ff000100, + 0x00ffff0000000000, 0x00ffff0001000100, 0x00ffff0001010001, 0x00ffff01ff00ff01, + 0x00ffff0100ff0100, 0x00ffff010000ff00, 0x00ffff01000100ff, 0x00ffff0101ff00ff, + 0x00ffff010101ff00, 0x00ff00ffffffffff, 0x00ff00ffffff01ff, 0x00ff00ffff000101, + 0x00ff00ff00000000, 0x00ff00ff000101ff, 0x00ff00ff01010101, 0x00ff0000ff000000, + 0x00ff0000ff01ffff, 0x00ff000000ff0000, 0x00ff00000000ff00, 0x00ff0000000000ff, + 0x00ff000000000000, 0x00ff000000000001, 0x00ff000000000100, 0x00ff000000010000, + 0x00ff000001ffff01, 0x00ff000001000000, 0x00ff0001ff000101, 0x00ff000100ffffff, + 0x00ff000100000000, 0x00ff0001010001ff, 0x00ff01ffff000000, 0x00ff01ff0001ff00, + 0x00ff01ff01ff0100, 0x00ff0100ff01ff01, 0x00ff010000ff00ff, 0x00ff010000ff0101, + 0x00ff010000000000, 0x00ff010000010101, 0x00ff01000100ff00, 0x00ff010001010000, + 0x00ff0101ffffff00, 0x00ff01010000ff01, 0x00ff010100000100, 0x00ff010101ff0000, + 0x0000ffffffff0100, 0x0000ffffff00ff00, 0x0000ffffff0000ff, 0x0000ffffff010000, + 0x0000ffff00000000, 0x0000ffff00010101, 0x0000ffff01ffff01, 0x0000ffff01000100, + 0x0000ff00ff000000, 0x0000ff00ff01ff00, 0x0000ff00ff0101ff, 0x0000ff0000ff0000, + 0x0000ff000000ff00, 0x0000ff00000000ff, 0x0000ff0000000000, 0x0000ff0000000001, + 0x0000ff0000000100, 0x0000ff0000010000, 0x0000ff0001ffffff, 0x0000ff0001ff01ff, + 0x0000ff0001000000, 0x0000ff000101ffff, 0x0000ff01ffff0101, 0x0000ff01ff010000, + 0x0000ff0100000000, 0x0000ff0101000101, 0x000000ffffff0001, 0x000000ffff000000, + 0x000000ff00ff0000, 0x000000ff0000ff00, 0x000000ff000000ff, 0x000000ff00000000, + 0x000000ff00000001, 0x000000ff00000100, 0x000000ff00010000, 0x000000ff01000000, + 0x000000ff0101ff00, 0x00000000ffff0000, 0x00000000ff00ff00, 0x00000000ff0000ff, + 0x00000000ff000000, 0x00000000ff000001, 0x00000000ff000100, 0x00000000ff010000, + 0x0000000000ffff00, 0x0000000000ff00ff, 0x0000000000ff0000, 0x0000000000ff0001, + 0x0000000000ff0100, 0x000000000000ffff, 0x000000000000ff00, 0x000000000000ff01, + 0x00000000000000ff, 0x0000000000000001, 0x00000000000001ff, 0x0000000000000100, + 0x0000000000000101, 0x000000000001ff00, 0x00000000000100ff, 0x0000000000010000, + 0x0000000000010001, 0x0000000000010100, 0x0000000001ff0000, 0x000000000100ff00, + 0x00000000010000ff, 0x0000000001000000, 0x0000000001000001, 0x0000000001000100, + 0x0000000001010000, 0x00000001ffff01ff, 0x00000001ff000000, 0x0000000100ff0000, + 0x000000010000ff00, 0x00000001000000ff, 0x0000000100000000, 0x0000000100000001, + 0x0000000100000100, 0x0000000100010000, 0x0000000101000000, 0x000001ffff00ff00, + 0x000001ffff010001, 0x000001ffff0101ff, 0x000001ff00ffff01, 0x000001ff0000ffff, + 0x000001ff00000000, 0x000001ff010000ff, 0x000001ff01010100, 0x00000100ffff0100, + 0x00000100ff000000, 0x0000010000ff0000, 0x000001000000ff00, 0x00000100000000ff, + 0x0000010000000000, 0x0000010000000001, 0x0000010000000100, 0x0000010000010000, + 0x0000010001000000, 0x000001000101ff01, 0x00000101ffff0001, 0x00000101ff01ffff, + 0x0000010100000000, 0x0000010101010100, 0x0001ffffff000000, 0x0001ffff00ffffff, + 0x0001ffff00000100, 0x0001ffff0001ff00, 0x0001ffff01000000, 0x0001ff00ffffff00, + 0x0001ff00ffff01ff, 0x0001ff00ff010000, 0x0001ff0000000000, 0x0001ff0000010001, + 0x0001ff0001ff0000, 0x0001ff0001010100, 0x0001ff01ff0000ff, 0x0001ff01ff000001, + 0x0001ff0100ffffff, 0x0001ff010001ffff, 0x0001ff01000101ff, 0x0001ff010100ff01, + 0x000100ffff00ffff, 0x000100ffff00ff01, 0x000100ffff000100, 0x000100ff00000000, + 0x000100ff000101ff, 0x000100ff01ff0101, 0x000100ff0100ffff, 0x000100ff01010101, + 0x00010000ff000000, 0x00010000ff010100, 0x0001000000ff0000, 0x000100000000ff00, + 0x00010000000000ff, 0x0001000000000000, 0x0001000000000001, 0x0001000000000100, + 0x0001000000010000, 0x0001000001ffff01, 0x0001000001000000, 0x0001000100ff0101, + 0x0001000100000000, 0x00010001010100ff, 0x000101ffffff01ff, 0x000101ffffff0101, + 0x000101ff00010000, 0x000101ff01ff0000, 0x000101ff0100ff01, 0x00010100ffff0000, + 0x0001010000000000, 0x000101000001ffff, 0x0001010000010101, 0x00010100010001ff, + 0x00010101ff00ff00, 0x00010101ff010001, 0x0001010100ffffff, 0x0001010100ff01ff, + 0x00010101000101ff, 0x0001010101ff0000, 0x000101010100ff01, 0x0001010101000101, + 0x01ffffffffff0101, 0x01ffffffff01ffff, 0x01ffffffff01ff01, 0x01ffffffff0101ff, + 0x01ffffffff010101, 0x01ffffff00000000, 0x01ffffff01ff01ff, 0x01ffffff01000101, + 0x01ffffff0101ff01, 0x01ffffff010100ff, 0x01ffff000000ff00, 0x01ffff0000000001, + 0x01ffff00000001ff, 0x01ffff0000010000, 0x01ffff0001ff0000, 0x01ffff01ffffffff, + 0x01ffff01ffff01ff, 0x01ffff01ff000000, 0x01ffff01ff01ffff, 0x01ffff01ff0101ff, + 0x01ffff010100ffff, 0x01ff00ffffff0000, 0x01ff00ffff010000, 0x01ff00ff00ffff01, + 0x01ff0000ff0000ff, 0x01ff000000000000, 0x01ff00000001ff01, 0x01ff000001ffffff, + 0x01ff000001010100, 0x01ff0001ffffff01, 0x01ff0001ff010001, 0x01ff000101ff0100, + 0x01ff000101000001, 0x01ff0001010100ff, 0x01ff01ffff00ffff, 0x01ff01ff00010001, + 0x01ff01ff01000000, 0x01ff01ff010101ff, 0x01ff0100ff000001, 0x01ff010000ffff00, + 0x01ff010000000100, 0x01ff010001ff01ff, 0x01ff01000101ffff, 0x01ff0101ffff00ff, + 0x01ff0101ffff0101, 0x01ff0101ff0101ff, 0x01ff010100010000, 0x0100ffff00ff00ff, + 0x0100ffff00ff0001, 0x0100ffff00000100, 0x0100ffff0100ff00, 0x0100ff00ffff0000, + 0x0100ff00ff00ffff, 0x0100ff00ff00ff01, 0x0100ff00ff000100, 0x0100ff00ff010000, + 0x0100ff0000000000, 0x0100ff00000100ff, 0x0100ff0001ff0101, 0x0100ff0001010101, + 0x0100ff0100ff00ff, 0x0100ff0100ff0001, 0x0100ff0100000100, 0x0100ff0100010001, + 0x0100ff0101000000, 0x010000ffff00ff00, 0x010000ff0000ffff, 0x010000ff00000000, + 0x010000ff010001ff, 0x010000ff01010001, 0x01000000ffffff00, 0x01000000ffff0101, + 0x01000000ff000000, 0x01000000ff0100ff, 0x01000000ff010101, 0x0100000000ff0000, + 0x010000000000ff00, 0x01000000000000ff, 0x0100000000000000, 0x0100000000000001, + 0x0100000000000100, 0x0100000000010000, 0x0100000001000000, 0x0100000100000000, + 0x01000001000101ff, 0x0100000101ffff01, 0x010001ffff000101, 0x010001ff00ff0100, + 0x010001ff0000ff00, 0x010001ff000100ff, 0x010001ff01ffffff, 0x01000100ffff0000, + 0x01000100ff0001ff, 0x0100010000000000, 0x010001000001ff00, 0x0100010001ff0000, + 0x01000100010000ff, 0x0100010001000101, 0x01000101ff00ff01, 0x0100010100ff0100, + 0x010001010000ffff, 0x0100010101010001, 0x0101ffffffff0101, 0x0101ffffff0001ff, + 0x0101ffffff01ffff, 0x0101ffffff010101, 0x0101ffff00000000, 0x0101ffff0101ffff, + 0x0101ffff010101ff, 0x0101ff00ff000000, 0x0101ff0000ff0100, 0x0101ff000000ff00, + 0x0101ff0000010000, 0x0101ff00010000ff, 0x0101ff0001000001, 0x0101ff01ff010101, + 0x0101ff0100000000, 0x0101ff010101ff00, 0x010100ffffff0000, 0x010100ffff010000, + 0x010100ff00ff01ff, 0x010100ff000000ff, 0x010100ff00000101, 0x010100ff01ffff00, + 0x01010000ffffff01, 0x01010000ff000100, 0x01010000ff01ff01, 0x0101000000000000, + 0x01010000000100ff, 0x010100000101ff01, 0x01010001ffff0000, 0x01010001ff00ffff, + 0x01010001ff010000, 0x0101000101ffffff, 0x0101000101ff01ff, 0x0101000101010101, + 0x010101ffff01ffff, 0x010101ff00000000, 0x010101ff0001ff01, 0x010101ff0101ffff, + 0x010101ff010101ff, 0x01010100ffffffff, 0x01010100ff000001, 0x010101000000ff00, + 0x0101010001010000, 0x0101010100ff0001, 0x010101010001ff01, 0x010101010101ffff, +}; constexpr constant static uint8_t ksigns_iq2xs[128] = { 0, 129, 130, 3, 132, 5, 6, 135, 136, 9, 10, 139, 12, 141, 142, 15, @@ -4208,6 +4346,123 @@ kernel void kernel_mul_mv_iq3_xxs_f32( kernel_mul_mv_iq3_xxs_f32_impl(src0, src1, dst, ne00, ne01, ne02, ne10, ne12, ne0, ne1, r2, r3, shared_values, tgpig, tiisg, sgitg); } +void kernel_mul_mv_iq1_s_f32_impl( + device const void * src0, + device const float * src1, + device float * dst, + constant int64_t & ne00, + constant int64_t & ne01, + constant int64_t & ne02, + constant int64_t & ne10, + constant int64_t & ne12, + constant int64_t & ne0, + constant int64_t & ne1, + constant uint & r2, + constant uint & r3, + uint3 tgpig[[threadgroup_position_in_grid]], + uint tiisg[[thread_index_in_simdgroup]], + uint sgitg[[simdgroup_index_in_threadgroup]]) { + + const int nb = ne00/QK_K; + const int r0 = tgpig.x; + const int r1 = tgpig.y; + const int im = tgpig.z; + + const int first_row = (r0 * N_SIMDGROUP + sgitg) * N_DST; + const int ib_row = first_row * nb; + + const uint i12 = im%ne12; + const uint i13 = im/ne12; + + const uint offset0 = (i12/r2)*(nb*ne01) + (i13/r3)*(nb*ne01*ne02); + + device const block_iq1_s * x = (device const block_iq1_s *) src0 + ib_row + offset0; + device const float * y = (device const float *) src1 + r1*ne10 + im*ne00*ne1; + + float yl[16]; + float sumf[N_DST]={0.f}, all_sum; + + const int nb32 = nb * (QK_K / 32); + +#if QK_K == 256 + const int ix = tiisg/2; + const int il = tiisg%2; + + device const float * y4 = y + 32 * ix + 16 * il; + + for (int ib32 = ix; ib32 < nb32; ib32 += 16) { + + for (int i = 0; i < 16; ++i) { + yl[i] = y4[i]; + } + + const int ibl = ib32 / (QK_K / 32); + const int ib = ib32 % (QK_K / 32); + + device const block_iq1_s * xr = x + ibl; + device const uint8_t * qs = xr->qs + 4 * ib + 2 * il; + device const uint8_t * sc = xr->scales + 2 * ib + il; + device const half * dh = &xr->d; + + for (int row = 0; row < N_DST; row++) { + + constant int8_t * grid1 = (constant int8_t *)(iq1s_grid + (qs[0] | ((sc[0] & 0x08) << 5))); + constant int8_t * grid2 = (constant int8_t *)(iq1s_grid + (qs[1] | ((sc[0] & 0x80) << 1))); + + float2 sum = {0}; + for (int j = 0; j < 8; ++j) { + sum[0] += yl[j+ 0] * grid1[j]; + sum[1] += yl[j+ 8] * grid2[j]; + } + sumf[row] += (float)dh[0] * (sum[0] * (2*(sc[0] & 7) + 1) + sum[1] * (2*((sc[0] >> 4) & 7) + 1)); + + dh += nb*sizeof(block_iq1_s)/2; + qs += nb*sizeof(block_iq1_s); + sc += nb*sizeof(block_iq1_s); + } + + y4 += 16 * 32; + } +#else + // TODO +#endif + + for (int row = 0; row < N_DST; ++row) { + all_sum = simd_sum(sumf[row]); + if (tiisg == 0) { + dst[r1*ne0 + im*ne0*ne1 + first_row + row] = all_sum; + } + } +} + +[[host_name("kernel_mul_mv_iq1_s_f32")]] +kernel void kernel_mul_mv_iq1_s_f32( + device const void * src0, + device const float * src1, + device float * dst, + constant int64_t & ne00, + constant int64_t & ne01, + constant int64_t & ne02, + constant uint64_t & nb00, + constant uint64_t & nb01, + constant uint64_t & nb02, + constant int64_t & ne10, + constant int64_t & ne11, + constant int64_t & ne12, + constant uint64_t & nb10, + constant uint64_t & nb11, + constant uint64_t & nb12, + constant int64_t & ne0, + constant int64_t & ne1, + constant uint & r2, + constant uint & r3, + uint3 tgpig[[threadgroup_position_in_grid]], + uint tiisg[[thread_index_in_simdgroup]], + uint sgitg[[simdgroup_index_in_threadgroup]]) { + + kernel_mul_mv_iq1_s_f32_impl(src0, src1, dst, ne00, ne01, ne02, ne10, ne12, ne0, ne1, r2, r3, tgpig, tiisg, sgitg); +} + //============================= templates and their specializations ============================= @@ -4553,6 +4808,22 @@ void dequantize_iq3_xxs(device const block_iq3_xxs * xb, short il, thread type4x } } +template +void dequantize_iq1_s(device const block_iq1_s * xb, short il, thread type4x4 & reg) { + // il is 0...15 for QK_K = 256 => index of block of 32 is il/2 + const float d = xb->d; + device const uint8_t * qs = xb->qs + 2*il; + device const uint8_t * sc = xb->scales + il; + const float dl1 = d * (2*(sc[0] & 7) + 1); + const float dl2 = d * (2*((sc[0] >> 4) & 7) + 1); + constant int8_t * grid1 = (constant int8_t *)(iq1s_grid + (qs[0] | ((sc[0] & 0x08) << 5))); + constant int8_t * grid2 = (constant int8_t *)(iq1s_grid + (qs[1] | ((sc[0] & 0x80) << 1))); + for (int i = 0; i < 8; ++i) { + reg[i/4+0][i%4] = dl1 * grid1[i]; + reg[i/4+2][i%4] = dl2 * grid2[i]; + } +} + template kernel void kernel_get_rows( device const void * src0, @@ -5095,6 +5366,7 @@ template [[host_name("kernel_get_rows_q6_K")]] kernel get_rows_t kernel_get_rows template [[host_name("kernel_get_rows_iq2_xxs")]] kernel get_rows_t kernel_get_rows; template [[host_name("kernel_get_rows_iq2_xs")]] kernel get_rows_t kernel_get_rows; template [[host_name("kernel_get_rows_iq3_xxs")]] kernel get_rows_t kernel_get_rows; +template [[host_name("kernel_get_rows_iq1_s")]] kernel get_rows_t kernel_get_rows; // // matrix-matrix multiplication @@ -5134,6 +5406,7 @@ template [[host_name("kernel_mul_mm_q6_K_f32")]] kernel mat_mm_t kernel_mul_mm; template [[host_name("kernel_mul_mm_iq2_xs_f32")]] kernel mat_mm_t kernel_mul_mm; template [[host_name("kernel_mul_mm_iq3_xxs_f32")]] kernel mat_mm_t kernel_mul_mm; +template [[host_name("kernel_mul_mm_iq1_s_f32")]] kernel mat_mm_t kernel_mul_mm; // // indirect matrix-matrix multiplication @@ -5185,6 +5458,7 @@ template [[host_name("kernel_mul_mm_id_q6_K_f32")]] kernel mat_mm_id_t kernel_mu template [[host_name("kernel_mul_mm_id_iq2_xxs_f32")]] kernel mat_mm_id_t kernel_mul_mm_id; template [[host_name("kernel_mul_mm_id_iq2_xs_f32")]] kernel mat_mm_id_t kernel_mul_mm_id; template [[host_name("kernel_mul_mm_id_iq3_xxs_f32")]] kernel mat_mm_id_t kernel_mul_mm_id; +template [[host_name("kernel_mul_mm_id_iq1_s_f32")]] kernel mat_mm_id_t kernel_mul_mm_id; // // matrix-vector multiplication @@ -6152,3 +6426,66 @@ kernel void kernel_mul_mv_id_iq3_xxs_f32( tiisg, sgitg); } + +[[host_name("kernel_mul_mv_id_iq1_s_f32")]] +kernel void kernel_mul_mv_id_iq1_s_f32( + device const char * ids, + device const char * src1, + device float * dst, + constant uint64_t & nbi1, + constant int64_t & ne00, + constant int64_t & ne01, + constant int64_t & ne02, + constant uint64_t & nb00, + constant uint64_t & nb01, + constant uint64_t & nb02, + constant int64_t & ne10, + constant int64_t & ne11, + constant int64_t & ne12, + constant int64_t & ne13, + constant uint64_t & nb10, + constant uint64_t & nb11, + constant uint64_t & nb12, + constant int64_t & ne0, + constant int64_t & ne1, + constant uint64_t & nb1, + constant uint & r2, + constant uint & r3, + constant int & idx, + device const char * src00, + device const char * src01, + device const char * src02, + device const char * src03, + device const char * src04, + device const char * src05, + device const char * src06, + device const char * src07, + uint3 tgpig[[threadgroup_position_in_grid]], + uint tiitg[[thread_index_in_threadgroup]], + uint tiisg[[thread_index_in_simdgroup]], + uint sgitg[[simdgroup_index_in_threadgroup]]) { + device const char * src0[8] = {src00, src01, src02, src03, src04, src05, src06, src07}; + + const int64_t bid = tgpig.z/(ne12*ne13); + + tgpig.z = tgpig.z%(ne12*ne13); + + const int32_t id = ((device int32_t *) (ids + bid*nbi1))[idx]; + + kernel_mul_mv_iq1_s_f32_impl( + src0[id], + (device const float *) (src1 + bid*nb11), + dst + bid*ne0, + ne00, + ne01, + ne02, + ne10, + ne12, + ne0, + ne1, + r2, + r3, + tgpig, + tiisg, + sgitg); +} diff --git a/ggml-quants.c b/ggml-quants.c index f44377f45..48f5294e1 100644 --- a/ggml-quants.c +++ b/ggml-quants.c @@ -3480,6 +3480,139 @@ static const uint32_t iq3xxs_grid[256] = { 0x3e1c1c1c, 0x3e1c3404, 0x3e24140c, 0x3e24240c, 0x3e2c0404, 0x3e2c0414, 0x3e2c1424, 0x3e341c04, }; +#define NGRID_IQ2XXS 512 +static const uint64_t iq1s_grid[NGRID_IQ2XXS] = { + 0xffffffffffff0101, 0xffffffffff01ff00, 0xffffffffff010100, 0xffffffff00000000, + 0xffffffff01ff00ff, 0xffffffff01ff0001, 0xffffffff0101ffff, 0xffffffff0101ff01, + 0xffffff00ff000000, 0xffffff000000ff00, 0xffffff00000000ff, 0xffffff0000000100, + 0xffffff0000010000, 0xffffff0001000000, 0xffffff01ffff00ff, 0xffffff01ff01ff00, + 0xffffff01ff010100, 0xffffff0100000001, 0xffffff0101ffff00, 0xffffff0101ff0101, + 0xffffff0101010100, 0xffff00ffff00ff01, 0xffff00ffff0000ff, 0xffff00ff00ff0100, + 0xffff00ff0100ff00, 0xffff00ff010001ff, 0xffff0000ff0101ff, 0xffff000000ffff00, + 0xffff000000000000, 0xffff00000001ff01, 0xffff000001000101, 0xffff0000010100ff, + 0xffff0001ffff0100, 0xffff00010000ff00, 0xffff000100010101, 0xffff000101000000, + 0xffff01ffffff0000, 0xffff01ffff01ffff, 0xffff01ffff010100, 0xffff01ff00000000, + 0xffff01ff01ffffff, 0xffff01ff01ff0001, 0xffff01ff0101ffff, 0xffff01ff01010001, + 0xffff0100ffffff01, 0xffff01000000ffff, 0xffff010000000100, 0xffff010001ff01ff, + 0xffff010001000000, 0xffff0101ff000000, 0xffff0101000101ff, 0xffff010101ffff01, + 0xffff01010101ff00, 0xff00ffffff000000, 0xff00ffff00ffff00, 0xff00ffff00000001, + 0xff00ffff000001ff, 0xff00ffff01010000, 0xff00ff00ffff0000, 0xff00ff00ff00ff00, + 0xff00ff00ff0000ff, 0xff00ff00ff000100, 0xff00ff00ff010001, 0xff00ff0000ff0001, + 0xff00ff000000ffff, 0xff00ff0000000000, 0xff00ff000001ff00, 0xff00ff0000010100, + 0xff00ff0001ff0000, 0xff00ff000100ff00, 0xff00ff0001000100, 0xff00ff01ff000000, + 0xff00ff0100ff0000, 0xff00ff01000001ff, 0xff00ff0101010001, 0xff0000ff00000000, + 0xff0000ff0001ff00, 0xff0000ff00010100, 0xff000000ffff0101, 0xff000000ff000000, + 0xff000000ff01ff00, 0xff00000000ff0000, 0xff0000000000ff00, 0xff000000000000ff, + 0xff00000000000000, 0xff00000000000001, 0xff00000000000100, 0xff0000000001ffff, + 0xff00000000010000, 0xff00000001000000, 0xff00000001010100, 0xff000001ff00ff01, + 0xff000001ff0100ff, 0xff00000100000000, 0xff0000010001ff00, 0xff00000101ff0100, + 0xff0000010100ff00, 0xff0001ff00ff00ff, 0xff0001ff00000101, 0xff0001ff000100ff, + 0xff0001ff01000000, 0xff000100ff0001ff, 0xff0001000000ff01, 0xff00010000000000, + 0xff00010000010001, 0xff00010000010100, 0xff00010001ffff00, 0xff00010001ff0101, + 0xff00010001010000, 0xff000101ffffffff, 0xff000101ff000101, 0xff00010101ff00ff, + 0xff00010101000001, 0xff000101010100ff, 0xff01ffffff000101, 0xff01ffffff01ffff, + 0xff01ffffff01ff01, 0xff01ffffff0101ff, 0xff01ffff00000000, 0xff01ffff01ff0001, + 0xff01ffff0101ff01, 0xff01ff00ff000000, 0xff01ff0000ff0100, 0xff01ff000000ff01, + 0xff01ff0000010000, 0xff01ff00010000ff, 0xff01ff01ff01ff00, 0xff01ff0100000101, + 0xff0100ffffff0000, 0xff0100ffff010000, 0xff0100ff01ff00ff, 0xff0100ff01000100, + 0xff0100ff010100ff, 0xff010000ffffff01, 0xff01000000000000, 0xff0100000101ff00, + 0xff010001ffff00ff, 0xff010001ff000100, 0xff01000100ffff00, 0xff01000100010001, + 0xff01000101ff0001, 0xff010001010001ff, 0xff0101ffffffffff, 0xff0101ffff01ffff, + 0xff0101ffff010101, 0xff0101ff0000ff00, 0xff0101ff01010001, 0xff010100ff000000, + 0xff010100ff01ff01, 0xff01010000ff0001, 0xff01010000000100, 0xff01010001000000, + 0xff0101010100ffff, 0x00ffffff0000ff01, 0x00ffffff000000ff, 0x00ffffff00000100, + 0x00ffffff00010000, 0x00ffff00ffff0001, 0x00ffff00ff0000ff, 0x00ffff00ff000100, + 0x00ffff0000000000, 0x00ffff0001000100, 0x00ffff0001010001, 0x00ffff01ff00ff01, + 0x00ffff0100ff0100, 0x00ffff010000ff00, 0x00ffff01000100ff, 0x00ffff0101ff00ff, + 0x00ffff010101ff00, 0x00ff00ffffffffff, 0x00ff00ffffff01ff, 0x00ff00ffff000101, + 0x00ff00ff00000000, 0x00ff00ff000101ff, 0x00ff00ff01010101, 0x00ff0000ff000000, + 0x00ff0000ff01ffff, 0x00ff000000ff0000, 0x00ff00000000ff00, 0x00ff0000000000ff, + 0x00ff000000000000, 0x00ff000000000001, 0x00ff000000000100, 0x00ff000000010000, + 0x00ff000001ffff01, 0x00ff000001000000, 0x00ff0001ff000101, 0x00ff000100ffffff, + 0x00ff000100000000, 0x00ff0001010001ff, 0x00ff01ffff000000, 0x00ff01ff0001ff00, + 0x00ff01ff01ff0100, 0x00ff0100ff01ff01, 0x00ff010000ff00ff, 0x00ff010000ff0101, + 0x00ff010000000000, 0x00ff010000010101, 0x00ff01000100ff00, 0x00ff010001010000, + 0x00ff0101ffffff00, 0x00ff01010000ff01, 0x00ff010100000100, 0x00ff010101ff0000, + 0x0000ffffffff0100, 0x0000ffffff00ff00, 0x0000ffffff0000ff, 0x0000ffffff010000, + 0x0000ffff00000000, 0x0000ffff00010101, 0x0000ffff01ffff01, 0x0000ffff01000100, + 0x0000ff00ff000000, 0x0000ff00ff01ff00, 0x0000ff00ff0101ff, 0x0000ff0000ff0000, + 0x0000ff000000ff00, 0x0000ff00000000ff, 0x0000ff0000000000, 0x0000ff0000000001, + 0x0000ff0000000100, 0x0000ff0000010000, 0x0000ff0001ffffff, 0x0000ff0001ff01ff, + 0x0000ff0001000000, 0x0000ff000101ffff, 0x0000ff01ffff0101, 0x0000ff01ff010000, + 0x0000ff0100000000, 0x0000ff0101000101, 0x000000ffffff0001, 0x000000ffff000000, + 0x000000ff00ff0000, 0x000000ff0000ff00, 0x000000ff000000ff, 0x000000ff00000000, + 0x000000ff00000001, 0x000000ff00000100, 0x000000ff00010000, 0x000000ff01000000, + 0x000000ff0101ff00, 0x00000000ffff0000, 0x00000000ff00ff00, 0x00000000ff0000ff, + 0x00000000ff000000, 0x00000000ff000001, 0x00000000ff000100, 0x00000000ff010000, + 0x0000000000ffff00, 0x0000000000ff00ff, 0x0000000000ff0000, 0x0000000000ff0001, + 0x0000000000ff0100, 0x000000000000ffff, 0x000000000000ff00, 0x000000000000ff01, + 0x00000000000000ff, 0x0000000000000001, 0x00000000000001ff, 0x0000000000000100, + 0x0000000000000101, 0x000000000001ff00, 0x00000000000100ff, 0x0000000000010000, + 0x0000000000010001, 0x0000000000010100, 0x0000000001ff0000, 0x000000000100ff00, + 0x00000000010000ff, 0x0000000001000000, 0x0000000001000001, 0x0000000001000100, + 0x0000000001010000, 0x00000001ffff01ff, 0x00000001ff000000, 0x0000000100ff0000, + 0x000000010000ff00, 0x00000001000000ff, 0x0000000100000000, 0x0000000100000001, + 0x0000000100000100, 0x0000000100010000, 0x0000000101000000, 0x000001ffff00ff00, + 0x000001ffff010001, 0x000001ffff0101ff, 0x000001ff00ffff01, 0x000001ff0000ffff, + 0x000001ff00000000, 0x000001ff010000ff, 0x000001ff01010100, 0x00000100ffff0100, + 0x00000100ff000000, 0x0000010000ff0000, 0x000001000000ff00, 0x00000100000000ff, + 0x0000010000000000, 0x0000010000000001, 0x0000010000000100, 0x0000010000010000, + 0x0000010001000000, 0x000001000101ff01, 0x00000101ffff0001, 0x00000101ff01ffff, + 0x0000010100000000, 0x0000010101010100, 0x0001ffffff000000, 0x0001ffff00ffffff, + 0x0001ffff00000100, 0x0001ffff0001ff00, 0x0001ffff01000000, 0x0001ff00ffffff00, + 0x0001ff00ffff01ff, 0x0001ff00ff010000, 0x0001ff0000000000, 0x0001ff0000010001, + 0x0001ff0001ff0000, 0x0001ff0001010100, 0x0001ff01ff0000ff, 0x0001ff01ff000001, + 0x0001ff0100ffffff, 0x0001ff010001ffff, 0x0001ff01000101ff, 0x0001ff010100ff01, + 0x000100ffff00ffff, 0x000100ffff00ff01, 0x000100ffff000100, 0x000100ff00000000, + 0x000100ff000101ff, 0x000100ff01ff0101, 0x000100ff0100ffff, 0x000100ff01010101, + 0x00010000ff000000, 0x00010000ff010100, 0x0001000000ff0000, 0x000100000000ff00, + 0x00010000000000ff, 0x0001000000000000, 0x0001000000000001, 0x0001000000000100, + 0x0001000000010000, 0x0001000001ffff01, 0x0001000001000000, 0x0001000100ff0101, + 0x0001000100000000, 0x00010001010100ff, 0x000101ffffff01ff, 0x000101ffffff0101, + 0x000101ff00010000, 0x000101ff01ff0000, 0x000101ff0100ff01, 0x00010100ffff0000, + 0x0001010000000000, 0x000101000001ffff, 0x0001010000010101, 0x00010100010001ff, + 0x00010101ff00ff00, 0x00010101ff010001, 0x0001010100ffffff, 0x0001010100ff01ff, + 0x00010101000101ff, 0x0001010101ff0000, 0x000101010100ff01, 0x0001010101000101, + 0x01ffffffffff0101, 0x01ffffffff01ffff, 0x01ffffffff01ff01, 0x01ffffffff0101ff, + 0x01ffffffff010101, 0x01ffffff00000000, 0x01ffffff01ff01ff, 0x01ffffff01000101, + 0x01ffffff0101ff01, 0x01ffffff010100ff, 0x01ffff000000ff00, 0x01ffff0000000001, + 0x01ffff00000001ff, 0x01ffff0000010000, 0x01ffff0001ff0000, 0x01ffff01ffffffff, + 0x01ffff01ffff01ff, 0x01ffff01ff000000, 0x01ffff01ff01ffff, 0x01ffff01ff0101ff, + 0x01ffff010100ffff, 0x01ff00ffffff0000, 0x01ff00ffff010000, 0x01ff00ff00ffff01, + 0x01ff0000ff0000ff, 0x01ff000000000000, 0x01ff00000001ff01, 0x01ff000001ffffff, + 0x01ff000001010100, 0x01ff0001ffffff01, 0x01ff0001ff010001, 0x01ff000101ff0100, + 0x01ff000101000001, 0x01ff0001010100ff, 0x01ff01ffff00ffff, 0x01ff01ff00010001, + 0x01ff01ff01000000, 0x01ff01ff010101ff, 0x01ff0100ff000001, 0x01ff010000ffff00, + 0x01ff010000000100, 0x01ff010001ff01ff, 0x01ff01000101ffff, 0x01ff0101ffff00ff, + 0x01ff0101ffff0101, 0x01ff0101ff0101ff, 0x01ff010100010000, 0x0100ffff00ff00ff, + 0x0100ffff00ff0001, 0x0100ffff00000100, 0x0100ffff0100ff00, 0x0100ff00ffff0000, + 0x0100ff00ff00ffff, 0x0100ff00ff00ff01, 0x0100ff00ff000100, 0x0100ff00ff010000, + 0x0100ff0000000000, 0x0100ff00000100ff, 0x0100ff0001ff0101, 0x0100ff0001010101, + 0x0100ff0100ff00ff, 0x0100ff0100ff0001, 0x0100ff0100000100, 0x0100ff0100010001, + 0x0100ff0101000000, 0x010000ffff00ff00, 0x010000ff0000ffff, 0x010000ff00000000, + 0x010000ff010001ff, 0x010000ff01010001, 0x01000000ffffff00, 0x01000000ffff0101, + 0x01000000ff000000, 0x01000000ff0100ff, 0x01000000ff010101, 0x0100000000ff0000, + 0x010000000000ff00, 0x01000000000000ff, 0x0100000000000000, 0x0100000000000001, + 0x0100000000000100, 0x0100000000010000, 0x0100000001000000, 0x0100000100000000, + 0x01000001000101ff, 0x0100000101ffff01, 0x010001ffff000101, 0x010001ff00ff0100, + 0x010001ff0000ff00, 0x010001ff000100ff, 0x010001ff01ffffff, 0x01000100ffff0000, + 0x01000100ff0001ff, 0x0100010000000000, 0x010001000001ff00, 0x0100010001ff0000, + 0x01000100010000ff, 0x0100010001000101, 0x01000101ff00ff01, 0x0100010100ff0100, + 0x010001010000ffff, 0x0100010101010001, 0x0101ffffffff0101, 0x0101ffffff0001ff, + 0x0101ffffff01ffff, 0x0101ffffff010101, 0x0101ffff00000000, 0x0101ffff0101ffff, + 0x0101ffff010101ff, 0x0101ff00ff000000, 0x0101ff0000ff0100, 0x0101ff000000ff00, + 0x0101ff0000010000, 0x0101ff00010000ff, 0x0101ff0001000001, 0x0101ff01ff010101, + 0x0101ff0100000000, 0x0101ff010101ff00, 0x010100ffffff0000, 0x010100ffff010000, + 0x010100ff00ff01ff, 0x010100ff000000ff, 0x010100ff00000101, 0x010100ff01ffff00, + 0x01010000ffffff01, 0x01010000ff000100, 0x01010000ff01ff01, 0x0101000000000000, + 0x01010000000100ff, 0x010100000101ff01, 0x01010001ffff0000, 0x01010001ff00ffff, + 0x01010001ff010000, 0x0101000101ffffff, 0x0101000101ff01ff, 0x0101000101010101, + 0x010101ffff01ffff, 0x010101ff00000000, 0x010101ff0001ff01, 0x010101ff0101ffff, + 0x010101ff010101ff, 0x01010100ffffffff, 0x01010100ff000001, 0x010101000000ff00, + 0x0101010001010000, 0x0101010100ff0001, 0x010101010001ff01, 0x010101010101ffff, + +}; + static const uint8_t ksigns_iq2xs[128] = { 0, 129, 130, 3, 132, 5, 6, 135, 136, 9, 10, 139, 12, 141, 142, 15, 144, 17, 18, 147, 20, 149, 150, 23, 24, 153, 154, 27, 156, 29, 30, 159, @@ -3578,6 +3711,49 @@ void dequantize_row_iq3_xxs(const block_iq3_xxs * restrict x, float * restrict y } } +// ====================== 1.5625 bpw (de)-quantization + +void dequantize_row_iq1_s(const block_iq1_s * restrict x, float * restrict y, int k) { + assert(k % QK_K == 0); + const int nb = k / QK_K; + + float db[4]; + uint16_t idx[4]; + //const int8_t * grid[4]; + + for (int i = 0; i < nb; i++) { + + const float d = GGML_FP16_TO_FP32(x[i].d); + const uint8_t * sc = x[i].scales; + const uint8_t * qs = x[i].qs; + + for (int i8 = 0; i8 < QK_K/8; i8 += 4) { + idx[0] = qs[0] | ((sc[0] & 0x08) << 5); + idx[1] = qs[1] | ((sc[0] & 0x80) << 1); + idx[2] = qs[2] | ((sc[1] & 0x08) << 5); + idx[3] = qs[3] | ((sc[1] & 0x80) << 1); + //grid[0] = (const int8_t *)(iq1s_grid + (qs[0] | ((sc[0] & 0x08) << 5))); + //grid[1] = (const int8_t *)(iq1s_grid + (qs[1] | ((sc[0] & 0x80) << 1))); + //grid[2] = (const int8_t *)(iq1s_grid + (qs[2] | ((sc[1] & 0x08) << 5))); + //grid[3] = (const int8_t *)(iq1s_grid + (qs[3] | ((sc[1] & 0x80) << 1))); + db[0] = d * (2*(sc[0] & 7) + 1); + db[1] = d * (2*((sc[0] >> 4) & 7) + 1); + db[2] = d * (2*(sc[1] & 7) + 1); + db[3] = d * (2*((sc[1] >> 4) & 7) + 1); + for (int l = 0; l < 4; ++l) { + const int8_t * grid = (const int8_t *)(iq1s_grid + idx[l]); + for (int j = 0; j < 8; ++j) { + //y[j] = db[l] * grid[l][j]; + y[j] = db[l] * grid[j]; + } + y += 8; + } + qs += 4; + sc += 2; + } + } +} + //===================================== Q8_K ============================================== void quantize_row_q8_K_reference(const float * restrict x, block_q8_K * restrict y, int k) { @@ -3679,7 +3855,7 @@ static inline __m128i get_scale_shuffle(int i) { } #endif -void ggml_vec_dot_q4_0_q8_0(int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) { +void ggml_vec_dot_q4_0_q8_0(int n, float * restrict s, size_t bs, const void * restrict vx, size_t bbx, const void * restrict vy, size_t bby, int nrc) { const int qk = QK8_0; const int nb = n / qk; @@ -3690,8 +3866,8 @@ void ggml_vec_dot_q4_0_q8_0(int n, float * restrict s, size_t bs, const void * r assert(nrc == 1); #endif UNUSED(nrc); - UNUSED(bx); - UNUSED(by); + UNUSED(bbx); + UNUSED(bby); UNUSED(bs); const block_q4_0 * restrict x = vx; @@ -4046,7 +4222,7 @@ void ggml_vec_dot_q4_0_q8_0(int n, float * restrict s, size_t bs, const void * r #endif } -void ggml_vec_dot_q4_1_q8_1(int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) { +void ggml_vec_dot_q4_1_q8_1(int n, float * restrict s, size_t bs, const void * restrict vx, size_t bbx, const void * restrict vy, size_t bby, int nrc) { const int qk = QK8_1; const int nb = n / qk; @@ -4057,8 +4233,8 @@ void ggml_vec_dot_q4_1_q8_1(int n, float * restrict s, size_t bs, const void * r assert(nrc == 1); #endif UNUSED(nrc); - UNUSED(bx); - UNUSED(by); + UNUSED(bbx); + UNUSED(bby); UNUSED(bs); const block_q4_1 * restrict x = vx; @@ -4264,7 +4440,7 @@ void ggml_vec_dot_q4_1_q8_1(int n, float * restrict s, size_t bs, const void * r #endif } -void ggml_vec_dot_q5_0_q8_0(int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) { +void ggml_vec_dot_q5_0_q8_0(int n, float * restrict s, size_t bs, const void * restrict vx, size_t bbx, const void * restrict vy, size_t bby, int nrc) { const int qk = QK8_0; const int nb = n / qk; @@ -4272,8 +4448,8 @@ void ggml_vec_dot_q5_0_q8_0(int n, float * restrict s, size_t bs, const void * r assert(qk == QK5_0); assert(nrc == 1); UNUSED(nrc); - UNUSED(bx); - UNUSED(by); + UNUSED(bbx); + UNUSED(bby); UNUSED(bs); const block_q5_0 * restrict x = vx; @@ -4555,7 +4731,7 @@ void ggml_vec_dot_q5_0_q8_0(int n, float * restrict s, size_t bs, const void * r #endif } -void ggml_vec_dot_q5_1_q8_1(int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) { +void ggml_vec_dot_q5_1_q8_1(int n, float * restrict s, size_t bs, const void * restrict vx, size_t bbx, const void * restrict vy, size_t bby, int nrc) { const int qk = QK8_1; const int nb = n / qk; @@ -4563,8 +4739,8 @@ void ggml_vec_dot_q5_1_q8_1(int n, float * restrict s, size_t bs, const void * r assert(qk == QK5_1); assert(nrc == 1); UNUSED(nrc); - UNUSED(bx); - UNUSED(by); + UNUSED(bbx); + UNUSED(bby); UNUSED(bs); const block_q5_1 * restrict x = vx; @@ -4859,7 +5035,7 @@ void ggml_vec_dot_q5_1_q8_1(int n, float * restrict s, size_t bs, const void * r #endif } -void ggml_vec_dot_q8_0_q8_0(int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) { +void ggml_vec_dot_q8_0_q8_0(int n, float * restrict s, size_t bs, const void * restrict vx, size_t bbx, const void * restrict vy, size_t bby, int nrc) { const int qk = QK8_0; const int nb = n / qk; @@ -4870,8 +5046,8 @@ void ggml_vec_dot_q8_0_q8_0(int n, float * restrict s, size_t bs, const void * r assert(nrc == 1); #endif UNUSED(nrc); - UNUSED(bx); - UNUSED(by); + UNUSED(bbx); + UNUSED(bby); UNUSED(bs); const block_q8_0 * restrict x = vx; @@ -9107,6 +9283,178 @@ void ggml_vec_dot_iq3_xxs_q8_K(int n, float * restrict s, size_t bs, const void #endif } +#ifdef __AVX2__ +static inline __m256i mul_add_epi8(const __m256i x, const __m256i y) { + const __m256i ax = _mm256_sign_epi8(x, x); + const __m256i sy = _mm256_sign_epi8(y, x); + return _mm256_maddubs_epi16(ax, sy); +} +#endif + +void ggml_vec_dot_iq1_s_q8_K (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { + assert(n % QK_K == 0); + assert(nrc == 1); + UNUSED(nrc); + UNUSED(bx); + UNUSED(by); + UNUSED(bs); + + const block_iq1_s * restrict x = vx; + const block_q8_K * restrict y = vy; + + const int nb = n / QK_K; + +#if defined __ARM_NEON + + const uint8x16_t m8 = vdupq_n_u8(0x08); + const uint8x16_t m7 = vdupq_n_u8(0x07); + const uint8x16_t m1 = vdupq_n_u8(0x01); + const int32x4_t vzero = vdupq_n_s32(0); + + uint16_t gindex[8]; + uint16x8x2_t vindex; + int8x16x4_t q1b; + int8x16x4_t q8b; + uint16x8x4_t scales; + int32x4x2_t sumi; + int32x4x2_t dotq; + + float sumf = 0; + for (int i = 0; i < nb; ++i) { + + const int8_t * q8 = y[i].qs; + const uint8_t * qs = x[i].qs; + const uint8_t * sc = x[i].scales; + + sumi.val[0] = sumi.val[1] = vzero; + + for (int i128 = 0; i128 < QK_K/128; ++i128) { + const uint8x16_t ql = vld1q_u8(qs); qs += 16; + const uint8x8_t tm1 = vld1_u8 (sc); sc += 8; + const uint8x8_t tm2 = vshr_n_u8(tm1, 4); + const uint8x16_t qh = vcombine_u8(vzip1_u8(tm1, tm2), vzip2_u8(tm1, tm2)); + const uint8x16_t hbit = vandq_u8(qh, m8); + vindex.val[0] = vorrq_u16(vmovl_u8(vget_low_u8 (ql)), vshlq_n_u16(vmovl_u8(vget_low_u8 (hbit)), 5)); + vindex.val[1] = vorrq_u16(vmovl_u8(vget_high_u8(ql)), vshlq_n_u16(vmovl_u8(vget_high_u8(hbit)), 5)); + const uint8x16_t scales8 = vorrq_u8(vshlq_n_u8(vandq_u8(qh, m7), 1), m1); + scales.val[0] = vmovl_u8(vget_low_u8 (scales8)); + scales.val[1] = vmovl_u8(vget_high_u8 (scales8)); + + for (int l = 0; l < 2; ++l) { + vst1q_u16(gindex+0, vindex.val[l]); + q1b.val[0] = vcombine_s8(vld1_s8((const void *)(iq1s_grid+gindex[0])), vld1_s8((const void *)(iq1s_grid+gindex[1]))); + q1b.val[1] = vcombine_s8(vld1_s8((const void *)(iq1s_grid+gindex[2])), vld1_s8((const void *)(iq1s_grid+gindex[3]))); + q1b.val[2] = vcombine_s8(vld1_s8((const void *)(iq1s_grid+gindex[4])), vld1_s8((const void *)(iq1s_grid+gindex[5]))); + q1b.val[3] = vcombine_s8(vld1_s8((const void *)(iq1s_grid+gindex[6])), vld1_s8((const void *)(iq1s_grid+gindex[7]))); + q8b = ggml_vld1q_s8_x4(q8); q8 += 64; + + dotq.val[0] = vpaddq_s32(ggml_vdotq_s32(vzero, q1b.val[0], q8b.val[0]), ggml_vdotq_s32(vzero, q1b.val[1], q8b.val[1])); + dotq.val[1] = vpaddq_s32(ggml_vdotq_s32(vzero, q1b.val[2], q8b.val[2]), ggml_vdotq_s32(vzero, q1b.val[3], q8b.val[3])); + + sumi.val[0] = vmlaq_s32(sumi.val[0], dotq.val[0], vreinterpretq_s32_u32(vmovl_u16(vget_low_u16 (scales.val[l])))); + sumi.val[1] = vmlaq_s32(sumi.val[1], dotq.val[1], vreinterpretq_s32_u32(vmovl_u16(vget_high_u16(scales.val[l])))); + } + } + + sumf += y[i].d * GGML_FP16_TO_FP32(x[i].d) * vaddvq_s32(vaddq_s32(sumi.val[0], sumi.val[1])); + } + + *s = sumf; + +#elif defined __AVX2__ + + const __m128i m8 = _mm_set1_epi8(0x08); + const __m128i m7 = _mm_set1_epi8(0x07); + const __m128i m1 = _mm_set1_epi8(0x01); + const __m128i shuffle_h = _mm_set_epi8(15, 7, 14, 6, 13, 5, 12, 4, 11, 3, 10, 2, 9, 1, 8, 0); + const __m128i shuffle_s[4] = { + _mm_set_epi32(0x03030303, 0x02020202, 0x01010101, 0x00000000), + _mm_set_epi32(0x07070707, 0x06060606, 0x05050505, 0x04040404), + _mm_set_epi32(0x0b0b0b0b, 0x0a0a0a0a, 0x09090909, 0x08080808), + _mm_set_epi32(0x0f0f0f0f, 0x0e0e0e0e, 0x0d0d0d0d, 0x0c0c0c0c) + }; + + uint64_t aux64; + + __m256i v_gindex; + const uint16_t * gindex = (const uint16_t *)&v_gindex; + + __m256 accum = _mm256_setzero_ps(); + for (int i = 0; i < nb; ++i) { + + const int8_t * q8 = y[i].qs; + const uint8_t * qs = x[i].qs; + const uint8_t * sc = x[i].scales; + + __m256i sumi = _mm256_setzero_si256(); + for (int i128 = 0; i128 < QK_K/128; ++i128) { + const __m128i ql = _mm_loadu_si128((const __m128i*)qs); qs += 16; + memcpy(&aux64, sc, 8); sc += 8; + const __m128i qh = _mm_shuffle_epi8(_mm_set_epi64x(aux64 >> 4, aux64), shuffle_h); + const __m256i hbit = _mm256_cvtepu8_epi16(_mm_and_si128(qh, m8)); + v_gindex = _mm256_or_si256(_mm256_cvtepu8_epi16(ql), _mm256_slli_epi16(hbit, 5)); + const __m128i scales = _mm_or_si128(_mm_slli_epi16(_mm_and_si128(qh, m7), 1), m1); + + for (int i32 = 0; i32 < 4; ++i32) { + const __m256i q8b = _mm256_loadu_si256((const __m256i*)q8); q8 += 32; + const __m256i q1b = _mm256_set_epi64x(iq1s_grid[gindex[4*i32+3]], iq1s_grid[gindex[4*i32+2]], + iq1s_grid[gindex[4*i32+1]], iq1s_grid[gindex[4*i32+0]]); + const __m256i dot = mul_add_epi8(q1b, q8b); + const __m256i s16 = _mm256_cvtepi8_epi16(_mm_shuffle_epi8(scales, shuffle_s[i32])); + const __m256i p = _mm256_madd_epi16(s16, dot); + sumi = _mm256_add_epi32(sumi, p); + } + + } + + accum = _mm256_fmadd_ps(_mm256_set1_ps(y[i].d * GGML_FP16_TO_FP32(x[i].d)), _mm256_cvtepi32_ps(sumi), accum); + + } + + *s = hsum_float_8(accum); + +#else + + int db[4]; + uint16_t idx[4]; + + float sumf = 0; + for (int i = 0; i < nb; ++i) { + + const int8_t * q8 = y[i].qs; + const uint8_t * qs = x[i].qs; + const uint8_t * sc = x[i].scales; + + int sumi = 0; + for (int i32 = 0; i32 < QK_K/32; ++i32) { + idx[0] = qs[0] | ((sc[0] & 0x08) << 5); + idx[1] = qs[1] | ((sc[0] & 0x80) << 1); + idx[2] = qs[2] | ((sc[1] & 0x08) << 5); + idx[3] = qs[3] | ((sc[1] & 0x80) << 1); + db[0] = (2*(sc[0] & 7) + 1); + db[1] = (2*((sc[0] >> 4) & 7) + 1); + db[2] = (2*(sc[1] & 7) + 1); + db[3] = (2*((sc[1] >> 4) & 7) + 1); + for (int l = 0; l < 4; ++l) { + const int8_t * grid = (const int8_t *)(iq1s_grid + idx[l]); + int suml = 0; + for (int j = 0; j < 8; ++j) suml += q8[j] * grid[j]; + sumi += db[l] * suml; + q8 += 8; + } + qs += 4; + sc += 2; + } + + sumf += GGML_FP16_TO_FP32(x[i].d) * y[i].d * sumi; + } + + *s = sumf; + +#endif + +} + // ================================ IQ2 quantization ============================================= typedef struct { @@ -9115,14 +9463,22 @@ typedef struct { uint16_t * neighbours; } iq2_entry_t; -static iq2_entry_t iq2_data[2] = { +static iq2_entry_t iq2_data[3] = { + {NULL, NULL, NULL}, {NULL, NULL, NULL}, {NULL, NULL, NULL}, }; -static inline int iq2_data_index(int grid_size) { - GGML_ASSERT(grid_size == 256 || grid_size == 512); - return grid_size == 256 ? 0 : 1; +static inline int iq2_data_index(enum ggml_type type) { + GGML_ASSERT(type == GGML_TYPE_IQ2_XXS || type == GGML_TYPE_IQ2_XS || type == GGML_TYPE_IQ1_S); + return type == GGML_TYPE_IQ2_XXS ? 0 : + type == GGML_TYPE_IQ2_XS ? 1 : 2; +} + +static inline int iq2_grid_size(enum ggml_type type) { + GGML_ASSERT(type == GGML_TYPE_IQ2_XXS || type == GGML_TYPE_IQ2_XS || type == GGML_TYPE_IQ1_S); + return type == GGML_TYPE_IQ2_XXS ? 256 : + type == GGML_TYPE_IQ2_XS ? 512 : 512; } static int iq2_compare_func(const void * left, const void * right) { @@ -9131,12 +9487,13 @@ static int iq2_compare_func(const void * left, const void * right) { return l[0] < r[0] ? -1 : l[0] > r[0] ? 1 : l[1] < r[1] ? -1 : l[1] > r[1] ? 1 : 0; } -void iq2xs_init_impl(int grid_size) { - const int gindex = iq2_data_index(grid_size); +void iq2xs_init_impl(enum ggml_type type) { + const int gindex = iq2_data_index(type); + const int grid_size = iq2_grid_size(type); if (iq2_data[gindex].grid) { return; } - static const uint16_t kgrid_256[256] = { + static const uint16_t kgrid_2bit_256[256] = { 0, 2, 5, 8, 10, 17, 20, 32, 34, 40, 42, 65, 68, 80, 88, 97, 100, 128, 130, 138, 162, 257, 260, 272, 277, 320, 388, 408, 512, 514, 546, 642, 1025, 1028, 1040, 1057, 1060, 1088, 1090, 1096, 1120, 1153, 1156, 1168, 1188, 1280, 1282, 1288, @@ -9154,7 +9511,7 @@ void iq2xs_init_impl(int grid_size) { 33888, 34048, 34118, 34196, 34313, 34368, 34400, 34818, 35076, 35345, 36868, 36880, 36900, 36928, 37025, 37142, 37248, 37445, 37888, 37922, 37956, 38225, 39041, 39200, 40962, 41040, 41093, 41225, 41472, 42008, 43088, 43268, }; - static const uint16_t kgrid_512[512] = { + static const uint16_t kgrid_2bit_512[512] = { 0, 2, 5, 8, 10, 17, 20, 22, 25, 32, 34, 37, 40, 65, 68, 70, 73, 80, 82, 85, 88, 97, 100, 128, 130, 133, 136, 145, 148, 153, 160, 257, 260, 262, 265, 272, 274, 277, 280, 282, 289, 292, 320, 322, 325, 328, 337, 340, @@ -9188,9 +9545,45 @@ void iq2xs_init_impl(int grid_size) { 40962, 40968, 40970, 40992, 41002, 41120, 41297, 41305, 41382, 41472, 41474, 41480, 41514, 41600, 41632, 42048, 42133, 42597, 42648, 43018, 43040, 43042, 43048, 43168, 43176, 43268, 43396, 43398, 43560, 43562, 43665, 43690, }; + static const uint16_t kgrid_1bit_512[512] = { + 10, 33, 41, 85, 132, 134, 160, 162, 277, 337, 340, 345, 357, 405, 516, 545, + 553, 598, 641, 650, 681, 1042, 1044, 1097, 1169, 1176, 1320, 1345, 1365, 1378, 1434, 1444, + 1545, 1617, 1642, 1685, 2053, 2080, 2089, 2133, 2176, 2182, 2208, 2214, 2306, 2384, 2393, 2440, + 2453, 2581, 2664, 2690, 2721, 4117, 4161, 4182, 4184, 4261, 4357, 4369, 4372, 4377, 4390, 4422, + 4432, 4437, 4449, 4457, 4485, 4497, 4505, 4629, 4677, 4696, 4774, 5205, 5217, 5225, 5386, 5397, + 5409, 5445, 5457, 5460, 5461, 5462, 5465, 5472, 5477, 5525, 5545, 5650, 5668, 5717, 5729, 5769, + 5777, 6212, 6234, 6244, 6293, 6424, 6482, 6485, 6502, 6505, 6529, 6538, 6565, 6656, 6682, 6788, + 6806, 6820, 8218, 8224, 8226, 8232, 8277, 8326, 8354, 8469, 8521, 8530, 8549, 8596, 8737, 8794, + 9221, 9253, 9348, 9369, 9380, 9474, 9557, 9633, 9732, 9753, 9793, 9830, 9862, 9880, 10240, 10272, + 10282, 10321, 10406, 10517, 10530, 10566, 10585, 10645, 10896, 16466, 16468, 16473, 16485, 16646, 16660, 16665, + 16725, 16793, 16806, 16914, 16969, 16977, 16996, 17028, 17057, 17408, 17416, 17434, 17493, 17512, 17578, 17685, + 17696, 17733, 17745, 17748, 17749, 17750, 17753, 17765, 17794, 17813, 17946, 17984, 18005, 18072, 18453, 18529, + 18569, 18722, 18756, 18762, 18773, 18794, 18833, 18853, 18945, 19026, 19033, 19077, 20489, 20497, 20500, 20517, + 20565, 20586, 20610, 20633, 20757, 20769, 20776, 20805, 20817, 20820, 20821, 20822, 20825, 20837, 20864, 20872, + 20885, 20896, 21002, 21029, 21077, 21146, 21510, 21525, 21573, 21585, 21588, 21589, 21590, 21593, 21605, 21653, + 21665, 21765, 21777, 21780, 21781, 21782, 21785, 21797, 21825, 21828, 21829, 21830, 21833, 21840, 21841, 21842, + 21844, 21846, 21848, 21849, 21850, 21857, 21860, 21861, 21862, 21865, 21893, 21905, 21908, 21909, 21910, 21913, + 21925, 22024, 22037, 22085, 22097, 22100, 22101, 22102, 22105, 22117, 22165, 22545, 22566, 22568, 22594, 22608, + 22613, 22676, 22697, 22793, 22805, 22853, 22865, 22868, 22869, 22870, 22873, 22885, 22933, 22946, 23046, 23072, + 23125, 23209, 24597, 24640, 24665, 24673, 24725, 24833, 24840, 24869, 24917, 24934, 24965, 25001, 25108, 25110, + 25152, 25184, 25192, 25234, 25616, 25618, 25625, 25685, 25704, 25738, 25744, 25770, 25877, 25897, 25925, 25937, + 25940, 25941, 25942, 25945, 25957, 25986, 26005, 26186, 26197, 26276, 26632, 26634, 26725, 26757, 26770, 26885, + 26965, 26976, 26986, 27032, 27153, 27174, 27200, 27208, 27240, 27269, 27282, 27290, 32778, 32800, 32802, 32808, + 32810, 32853, 32904, 32922, 32930, 32932, 33105, 33110, 33112, 33125, 33157, 33280, 33288, 33301, 33312, 33320, + 33424, 33797, 33829, 33858, 34068, 34133, 34146, 34176, 34217, 34306, 34342, 34441, 34454, 34468, 34832, 34918, + 34965, 34984, 35094, 35137, 35161, 35208, 35232, 35332, 35338, 35368, 35429, 36932, 36934, 36953, 37009, 37125, + 37136, 37138, 37145, 37157, 37205, 37220, 37258, 37290, 37444, 37446, 37465, 37478, 37525, 37905, 37968, 37973, + 38040, 38054, 38145, 38154, 38165, 38180, 38186, 38213, 38225, 38228, 38229, 38230, 38233, 38245, 38293, 38485, + 38504, 38530, 38938, 38985, 38993, 39012, 39040, 39173, 39192, 39253, 39265, 39301, 39316, 39322, 39442, 39497, + 39504, 39590, 40970, 40984, 40992, 41002, 41045, 41120, 41128, 41237, 41289, 41297, 41317, 41364, 41366, 41514, + 41557, 41633, 41989, 42021, 42056, 42068, 42074, 42113, 42242, 42265, 42274, 42325, 42340, 42402, 42501, 42512, + 42533, 42624, 42632, 42666, 43040, 43093, 43106, 43168, 43176, 43264, 43286, 43345, 43429, 43590, 43618, 43680, + }; + const int kmap_size = 43692; - const int nwant = 2; - const uint16_t * kgrid = grid_size == 256 ? kgrid_256 : kgrid_512; + const int nwant = type == GGML_TYPE_IQ1_S ? 3 : 2; + const uint16_t * kgrid = type == GGML_TYPE_IQ2_XXS ? kgrid_2bit_256 : + type == GGML_TYPE_IQ2_XS ? kgrid_2bit_512 : kgrid_1bit_512; uint64_t * kgrid_q2xs; int * kmap_q2xs; uint16_t * kneighbors_q2xs; @@ -9286,9 +9679,9 @@ void iq2xs_init_impl(int grid_size) { free(dist2); } -void iq2xs_free_impl(int grid_size) { - GGML_ASSERT(grid_size == 256 || grid_size == 512 || grid_size == 1024); - const int gindex = iq2_data_index(grid_size); +void iq2xs_free_impl(enum ggml_type type) { + GGML_ASSERT(type == GGML_TYPE_IQ2_XXS || type == GGML_TYPE_IQ2_XS || type == GGML_TYPE_IQ1_S); + const int gindex = iq2_data_index(type); if (iq2_data[gindex].grid) { free(iq2_data[gindex].grid); iq2_data[gindex].grid = NULL; free(iq2_data[gindex].map); iq2_data[gindex].map = NULL; @@ -9322,7 +9715,7 @@ static int iq2_find_best_neighbour(const uint16_t * restrict neighbours, const u static void quantize_row_iq2_xxs_impl(const float * restrict x, void * restrict vy, int n, const float * restrict quant_weights) { - const int gindex = iq2_data_index(256); + const int gindex = iq2_data_index(GGML_TYPE_IQ2_XXS); const uint64_t * kgrid_q2xs = iq2_data[gindex].grid; const int * kmap_q2xs = iq2_data[gindex].map; @@ -9495,7 +9888,7 @@ static void quantize_row_iq2_xxs_impl(const float * restrict x, void * restrict static void quantize_row_iq2_xs_impl(const float * restrict x, void * restrict vy, int n, const float * restrict quant_weights) { - const int gindex = iq2_data_index(512); + const int gindex = iq2_data_index(GGML_TYPE_IQ2_XS); const uint64_t * kgrid_q2xs = iq2_data[gindex].grid; const int * kmap_q2xs = iq2_data[gindex].map; @@ -10132,3 +10525,207 @@ void quantize_row_iq3_xxs_reference(const float * restrict x, block_iq3_xxs * re assert(k % QK_K == 0); quantize_row_iq3_xxs_impl(x, y, k, NULL); } + +// =================================== 1.5 bpw =================================================== + +static int iq1_find_best_neighbour(const uint16_t * restrict neighbours, const uint64_t * restrict grid, + const float * restrict xval, const float * restrict weight, float * scale, int8_t * restrict L, int ngrid) { + int num_neighbors = neighbours[0]; + GGML_ASSERT(num_neighbors > 0); + float best_score = 0; + int grid_index = -1; + for (int j = 1; j <= num_neighbors; ++j) { + const int8_t * pg = (const int8_t *)(grid + neighbours[j]); + float sumqx = 0, sumq2 = 0; + for (int i = 0; i < 8; ++i) { + float q = (pg[i] - 3)/2; + float w = weight[i]; + sumqx += w*q*xval[i]; + sumq2 += w*q*q; + } + if (sumqx > 0 && sumq2 > 0 && sumqx*sumqx > best_score*sumq2) { + *scale = sumqx/sumq2; best_score = *scale * sumqx; + grid_index = neighbours[j]; + } + } + if (grid_index < 0) { + for (int i = 0; i < ngrid; ++i) { + const int8_t * grid_i = (const int8_t *)(grid + i); + float sumqx = 0, sumq2 = 0; + for (int j = 0; j < 8; ++j) { + float w = weight[j]; + float q = (grid_i[j] - 3)/2; + sumqx += w*q*xval[j]; + sumq2 += w*q*q; + } + if (sumqx > 0 && sumq2 > 0 && sumqx*sumqx > best_score*sumq2) { + *scale = sumqx/sumq2; best_score = *scale*sumqx; + grid_index = i; + } + } + } + if (grid_index < 0) { + printf("Oops, did not find grid point\n"); + printf("Have %d neighbours\n", num_neighbors); + for (int j = 1; j <= num_neighbors; ++j) { + const int8_t * pg = (const int8_t *)(grid + neighbours[j]); + float sumqx = 0, sumq2 = 0; + for (int i = 0; i < 8; ++i) { + float q = (pg[i] - 3)/2; + float w = weight[i]; + sumqx += w*q*xval[i]; + sumq2 += w*q*q; + } + printf(" neighbour %d: sumqx = %g sumq2 = %g\n", j, (double)sumqx, (double)sumq2); + } + } + GGML_ASSERT(grid_index >= 0); + //!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + *scale *= 1.05f; // This is a fudge factor. Don't ask me why it improves the result. + //!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + const int8_t * pg = (const int8_t *)(grid + grid_index); + for (int i = 0; i < 8; ++i) L[i] = (pg[i] - 1)/2; + return grid_index; +} + +static int iq1_sort_helper(const void * left, const void * right) { + const float * l = left; + const float * r = right; + return *l < *r ? -1 : *l > *r ? 1 : 0; +} + +static void quantize_row_iq1_s_impl(const float * restrict x, void * restrict vy, int n, const float * restrict quant_weights) { + + const int gindex = iq2_data_index(GGML_TYPE_IQ1_S); + + const uint64_t * kgrid_q2xs = iq2_data[gindex].grid; + const int * kmap_q2xs = iq2_data[gindex].map; + const uint16_t * kneighbors_q2xs = iq2_data[gindex].neighbours; + + GGML_ASSERT(quant_weights && "missing quantization weights"); + GGML_ASSERT(kgrid_q2xs && "forgot to call ggml_quantize_init()?"); + GGML_ASSERT(kmap_q2xs && "forgot to call ggml_quantize_init()?"); + GGML_ASSERT(kneighbors_q2xs && "forgot to call ggml_quantize_init()?"); + GGML_ASSERT(n%QK_K == 0); + + const int nbl = n/256; + + block_iq1_s * y = vy; + + float scales[QK_K/8]; + float weight[8]; + int8_t L[8]; + float sumx[9]; + float sumw[9]; + float pairs[16]; + int * idx = (int *)(pairs + 1); + uint8_t hbit[QK_K/8]; + + for (int ibl = 0; ibl < nbl; ++ibl) { + + y[ibl].d = GGML_FP32_TO_FP16(0.f); + memset(y[ibl].qs, 0, QK_K/8); + memset(y[ibl].scales, 0, QK_K/16); + + float max_scale = 0; + + const float * xbl = x + QK_K*ibl; + float sumx2 = 0; + for (int i = 0; i < QK_K; ++i) sumx2 += xbl[i]*xbl[i]; + float sigma2 = sumx2/QK_K; + + for (int ib = 0; ib < QK_K/8; ++ib) { + const float * xb = xbl + 8*ib; + const float * qw = quant_weights + QK_K*ibl + 8*ib; + for (int i = 0; i < 8; ++i) weight[i] = qw[i] * sqrtf(sigma2 + xb[i]*xb[i]); + float max = fabsf(xb[0]); + for (int i = 1; i < 8; ++i) max = MAX(max, fabsf(xb[i])); + if (!max) { + scales[ib] = 0; + memset(L, 1, 8); + continue; + } + // Here we solve exactly the sum of squared difference (SSD) weighted minimization problem. + // With just 3 allowed quant values (-1, 0, 1), we can search exhaustively for the two + // boundaries that split the weights xb[i] into 3 groups. To do so, we sort the weights + // in ascending order, compute Si = sum[weight[j] xb[j], j = 0...i] and + // Wi = sum[weight[j], j = 0...i], and use these to quckly get get the optimum scale + // for each possible and score for each split. + for (int j = 0; j < 8; ++j) { + pairs[2*j] = xb[j]; + idx[2*j] = j; + } + qsort(pairs, 8, 2*sizeof(float), iq1_sort_helper); + { + sumx[0] = sumw[0] = 0; + for (int j = 0; j < 8; ++j) { + int i = idx[2*j]; + sumx[j+1] = sumx[j] + weight[i]*xb[i]; + sumw[j+1] = sumw[j] + weight[i]; + } + } + float best_score = 0, scale = max; + int besti1 = 0, besti2 = 0; + for (int i1 = 0; i1 <= 8; ++i1) { + for (int i2 = i1; i2 <= 8; ++i2) { + float sumqx = -(sumx[i1] - sumx[0]) + (sumx[8] - sumx[i2]); + float sumq2 = (sumw[i1] - sumw[0]) + (sumw[8] - sumw[i2]); + if (sumq2 > 0 && sumqx*sumqx > best_score*sumq2) { + scale = sumqx/sumq2; best_score = scale*sumqx; + besti1 = i1; besti2 = i2; + } + } + } + for (int j = 0; j < besti1; ++j) L[idx[2*j]] = 0; + for (int j = besti1; j < besti2; ++j) L[idx[2*j]] = 1; + for (int j = besti2; j < 8; ++j) L[idx[2*j]] = 2; + if (scale < 0) { + for (int j = 0; j < 8; ++j) L[j] = 2 - L[j]; + scale = -scale; + } + // Now we check if the solution found above corresponds to a grid point and, if not, use a neighbouring + // grid point that minimizes SSD. + uint16_t u = 0; + for (int j = 0; j < 8; ++j) u |= (L[j] << 2*j); + int grid_index = kmap_q2xs[u]; + if (grid_index < 0) { + const uint16_t * neighbours = kneighbors_q2xs - kmap_q2xs[u] - 1; + grid_index = iq1_find_best_neighbour(neighbours, kgrid_q2xs, xb, weight, &scale, L, NGRID_IQ2XXS); + GGML_ASSERT(grid_index >= 0); + } + y[ibl].qs[ib] = grid_index & 255; + hbit[ib] = grid_index >> 8; + GGML_ASSERT(scale >= 0); + scales[ib] = scale; + max_scale = MAX(max_scale, scale); + } + + if (!max_scale) { + memset(y[ibl].qs, 0, QK_K/8); + continue; + } + + float d = max_scale/15; + y[ibl].d = GGML_FP32_TO_FP16(d*1.085f); // 1.085f is another fudge factor. Don't ask me why it is needed. + float id = 1/d; + for (int ib = 0; ib < QK_K/8; ++ib) { + int l = nearest_int(0.5f*(id*scales[ib]-1)); + l = MAX(0, MIN(7, l)); + if (hbit[ib]) l |= 8; + y[ibl].scales[ib/2] |= (l << 4*(ib%2)); + } + } +} + +size_t quantize_iq1_s(const float * src, void * dst, int nrow, int n_per_row, int64_t * hist, const float * quant_weights) { + (void)hist; + GGML_ASSERT(n_per_row%QK_K == 0); + int nblock = n_per_row/QK_K; + char * qrow = (char *)dst; + for (int row = 0; row < nrow; ++row) { + quantize_row_iq1_s_impl(src, qrow, n_per_row, quant_weights); + src += n_per_row; + qrow += nblock*sizeof(block_iq1_s); + } + return nrow * nblock * sizeof(block_iq1_s); +} diff --git a/ggml-quants.h b/ggml-quants.h index 68f09b1e1..ad381cfab 100644 --- a/ggml-quants.h +++ b/ggml-quants.h @@ -191,6 +191,13 @@ typedef struct { } block_iq3_xxs; static_assert(sizeof(block_iq3_xxs) == sizeof(ggml_fp16_t) + 3*(QK_K/8), "wrong iq3_xxs block size/padding"); +typedef struct { + ggml_fp16_t d; + uint8_t qs[QK_K/8]; + uint8_t scales[QK_K/16]; +} block_iq1_s; +static_assert(sizeof(block_iq1_s) == sizeof(ggml_fp16_t) + QK_K/8 + QK_K/16, "wrong iq1_s block size/padding"); + #ifdef __cplusplus extern "C" { #endif @@ -243,6 +250,7 @@ void dequantize_row_q8_K(const block_q8_K * GGML_RESTRICT x, float * GGML_RESTRI void dequantize_row_iq2_xxs(const block_iq2_xxs * GGML_RESTRICT x, float * GGML_RESTRICT y, int k); void dequantize_row_iq2_xs (const block_iq2_xs * GGML_RESTRICT x, float * GGML_RESTRICT y, int k); void dequantize_row_iq3_xxs(const block_iq3_xxs * GGML_RESTRICT x, float * GGML_RESTRICT y, int k); +void dequantize_row_iq1_s (const block_iq1_s * GGML_RESTRICT x, float * GGML_RESTRICT y, int k); // Dot product void ggml_vec_dot_q4_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); @@ -259,6 +267,7 @@ void ggml_vec_dot_q6_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voi void ggml_vec_dot_iq2_xxs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); void ggml_vec_dot_iq2_xs_q8_K (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); void ggml_vec_dot_iq3_xxs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); +void ggml_vec_dot_iq1_s_q8_K (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); // // Quantization utilizing an importance matrix (a.k.a. "Activation aWare Quantization") @@ -266,6 +275,7 @@ void ggml_vec_dot_iq3_xxs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const size_t quantize_iq2_xxs(const float * src, void * dst, int nrows, int n_per_row, int64_t * hist, const float * imatrix); size_t quantize_iq2_xs (const float * src, void * dst, int nrows, int n_per_row, int64_t * hist, const float * imatrix); size_t quantize_iq3_xxs(const float * src, void * dst, int nrows, int n_per_row, int64_t * hist, const float * imatrix); +size_t quantize_iq1_s (const float * src, void * dst, int nrows, int n_per_row, int64_t * hist, const float * imatrix); size_t quantize_q2_K (const float * src, void * dst, int nrows, int n_per_row, int64_t * hist, const float * imatrix); size_t quantize_q3_K (const float * src, void * dst, int nrows, int n_per_row, int64_t * hist, const float * imatrix); size_t quantize_q4_K (const float * src, void * dst, int nrows, int n_per_row, int64_t * hist, const float * imatrix); @@ -276,8 +286,8 @@ size_t quantize_q4_1 (const float * src, void * dst, int nrows, int n_per_row, size_t quantize_q5_0 (const float * src, void * dst, int nrows, int n_per_row, int64_t * hist, const float * imatrix); size_t quantize_q5_1 (const float * src, void * dst, int nrows, int n_per_row, int64_t * hist, const float * imatrix); -void iq2xs_init_impl(int grid_size); -void iq2xs_free_impl(int grid_size); +void iq2xs_init_impl(enum ggml_type type); +void iq2xs_free_impl(enum ggml_type type); void iq3xs_init_impl(int grid_size); void iq3xs_free_impl(int grid_size); diff --git a/ggml.c b/ggml.c index e94024c62..aefcda6d4 100644 --- a/ggml.c +++ b/ggml.c @@ -673,6 +673,18 @@ static const ggml_type_traits_t type_traits[GGML_TYPE_COUNT] = { .vec_dot_type = GGML_TYPE_Q8_K, .nrows = 1, }, + [GGML_TYPE_IQ1_S] = { + .type_name = "iq1_s", + .blck_size = QK_K, + .type_size = sizeof(block_iq1_s), + .is_quantized = true, + .to_float = (ggml_to_float_t) dequantize_row_iq1_s, + .from_float = NULL, + .from_float_reference = NULL, + .vec_dot = ggml_vec_dot_iq1_s_q8_K, + .vec_dot_type = GGML_TYPE_Q8_K, + .nrows = 1, + }, [GGML_TYPE_Q8_K] = { .type_name = "q8_K", .blck_size = QK_K, @@ -2267,6 +2279,7 @@ enum ggml_type ggml_ftype_to_ggml_type(enum ggml_ftype ftype) { case GGML_FTYPE_MOSTLY_IQ2_XXS: wtype = GGML_TYPE_IQ2_XXS; break; case GGML_FTYPE_MOSTLY_IQ2_XS: wtype = GGML_TYPE_IQ2_XS; break; case GGML_FTYPE_MOSTLY_IQ3_XXS: wtype = GGML_TYPE_IQ3_XXS; break; + case GGML_FTYPE_MOSTLY_IQ1_S: wtype = GGML_TYPE_IQ1_S; break; case GGML_FTYPE_UNKNOWN: wtype = GGML_TYPE_COUNT; break; case GGML_FTYPE_MOSTLY_Q4_1_SOME_F16: wtype = GGML_TYPE_COUNT; break; } @@ -7677,6 +7690,7 @@ static void ggml_compute_forward_add( case GGML_TYPE_IQ2_XXS: case GGML_TYPE_IQ2_XS: case GGML_TYPE_IQ3_XXS: + case GGML_TYPE_IQ1_S: { ggml_compute_forward_add_q_f32(params, src0, src1, dst); } break; @@ -7944,6 +7958,7 @@ static void ggml_compute_forward_add1( case GGML_TYPE_IQ2_XXS: case GGML_TYPE_IQ2_XS: case GGML_TYPE_IQ3_XXS: + case GGML_TYPE_IQ1_S: { ggml_compute_forward_add1_q_f32(params, src0, src1, dst); } break; @@ -8064,6 +8079,7 @@ static void ggml_compute_forward_acc( case GGML_TYPE_IQ2_XXS: case GGML_TYPE_IQ2_XS: case GGML_TYPE_IQ3_XXS: + case GGML_TYPE_IQ1_S: default: { GGML_ASSERT(false); @@ -10830,6 +10846,7 @@ static void ggml_compute_forward_out_prod( case GGML_TYPE_IQ2_XXS: case GGML_TYPE_IQ2_XS: case GGML_TYPE_IQ3_XXS: + case GGML_TYPE_IQ1_S: { ggml_compute_forward_out_prod_q_f32(params, src0, src1, dst); } break; @@ -11010,6 +11027,7 @@ static void ggml_compute_forward_set( case GGML_TYPE_IQ2_XXS: case GGML_TYPE_IQ2_XS: case GGML_TYPE_IQ3_XXS: + case GGML_TYPE_IQ1_S: default: { GGML_ASSERT(false); @@ -11207,6 +11225,7 @@ static void ggml_compute_forward_get_rows( case GGML_TYPE_IQ2_XXS: case GGML_TYPE_IQ2_XS: case GGML_TYPE_IQ3_XXS: + case GGML_TYPE_IQ1_S: { ggml_compute_forward_get_rows_q(params, src0, src1, dst); } break; @@ -11880,6 +11899,7 @@ static void ggml_compute_forward_alibi( case GGML_TYPE_IQ2_XXS: case GGML_TYPE_IQ2_XS: case GGML_TYPE_IQ3_XXS: + case GGML_TYPE_IQ1_S: case GGML_TYPE_Q8_K: case GGML_TYPE_I8: case GGML_TYPE_I16: @@ -11957,6 +11977,7 @@ static void ggml_compute_forward_clamp( case GGML_TYPE_IQ2_XXS: case GGML_TYPE_IQ2_XS: case GGML_TYPE_IQ3_XXS: + case GGML_TYPE_IQ1_S: case GGML_TYPE_Q8_K: case GGML_TYPE_I8: case GGML_TYPE_I16: @@ -19136,8 +19157,9 @@ void ggml_quantize_init(enum ggml_type type) { ggml_critical_section_start(); switch (type) { - case GGML_TYPE_IQ2_XXS: iq2xs_init_impl(256); break; - case GGML_TYPE_IQ2_XS: iq2xs_init_impl(512); break; + case GGML_TYPE_IQ2_XXS: + case GGML_TYPE_IQ2_XS: + case GGML_TYPE_IQ1_S: iq2xs_init_impl(type); break; case GGML_TYPE_IQ3_XXS: iq3xs_init_impl(256); break; default: // nothing break; @@ -19149,8 +19171,10 @@ void ggml_quantize_init(enum ggml_type type) { void ggml_quantize_free(void) { ggml_critical_section_start(); - iq2xs_free_impl(256); - iq2xs_free_impl(512); + iq2xs_free_impl(GGML_TYPE_IQ2_XXS); + iq2xs_free_impl(GGML_TYPE_IQ2_XS); + iq2xs_free_impl(GGML_TYPE_IQ1_S); + iq3xs_free_impl(256); ggml_critical_section_end(); } @@ -19285,7 +19309,8 @@ size_t ggml_quantize_q8_0(const float * src, void * dst, int n, int k, int64_t * bool ggml_quantize_requires_imatrix(enum ggml_type type) { return type == GGML_TYPE_IQ2_XXS || - type == GGML_TYPE_IQ2_XS; + type == GGML_TYPE_IQ2_XS || + type == GGML_TYPE_IQ1_S; } size_t ggml_quantize_chunk(enum ggml_type type, const float * src, void * dst, int start, @@ -19410,6 +19435,15 @@ size_t ggml_quantize_chunk(enum ggml_type type, const float * src, void * dst, i result = quantize_iq3_xxs(src + start, (char *)dst + start_row * row_size, nrows, n_per_row, hist, imatrix); GGML_ASSERT(result == row_size * nrows); } break; + case GGML_TYPE_IQ1_S: + { + GGML_ASSERT(start % QK_K == 0); + GGML_ASSERT(start % n_per_row == 0); + size_t start_row = start / n_per_row; + size_t row_size = ggml_row_size(type, n_per_row); + result = quantize_iq1_s(src + start, (char *)dst + start_row * row_size, nrows, n_per_row, hist, imatrix); + GGML_ASSERT(result == row_size * nrows); + } break; case GGML_TYPE_F16: { size_t elemsize = sizeof(ggml_fp16_t); diff --git a/ggml.h b/ggml.h index 6c1956772..004d09c70 100644 --- a/ggml.h +++ b/ggml.h @@ -354,6 +354,7 @@ extern "C" { GGML_TYPE_IQ2_XXS = 16, GGML_TYPE_IQ2_XS = 17, GGML_TYPE_IQ3_XXS = 18, + GGML_TYPE_IQ1_S = 19, GGML_TYPE_I8, GGML_TYPE_I16, GGML_TYPE_I32, @@ -391,6 +392,7 @@ extern "C" { GGML_FTYPE_MOSTLY_IQ2_XXS = 15, // except 1d tensors GGML_FTYPE_MOSTLY_IQ2_XS = 16, // except 1d tensors GGML_FTYPE_MOSTLY_IQ3_XXS = 17, // except 1d tensors + GGML_FTYPE_MOSTLY_IQ1_S = 18, // except 1d tensors }; // available tensor operations: diff --git a/llama.cpp b/llama.cpp index 6ac9caa95..5cfebb3b1 100644 --- a/llama.cpp +++ b/llama.cpp @@ -2526,6 +2526,7 @@ struct llama_model_loader { case GGML_TYPE_IQ2_XXS: ftype = LLAMA_FTYPE_MOSTLY_IQ2_XXS; break; case GGML_TYPE_IQ2_XS: ftype = LLAMA_FTYPE_MOSTLY_IQ2_XS; break; case GGML_TYPE_IQ3_XXS: ftype = LLAMA_FTYPE_MOSTLY_IQ3_XXS; break; + case GGML_TYPE_IQ1_S: ftype = LLAMA_FTYPE_MOSTLY_IQ1_S; break; default: { LLAMA_LOG_WARN("%s: unknown type %s\n", __func__, ggml_type_name(type_max)); @@ -2875,6 +2876,7 @@ static std::string llama_model_ftype_name(llama_ftype ftype) { case LLAMA_FTYPE_MOSTLY_IQ2_XS: return "IQ2_XS - 2.3125 bpw"; case LLAMA_FTYPE_MOSTLY_Q3_K_XS:return "Q3_K - Extra small"; case LLAMA_FTYPE_MOSTLY_IQ3_XXS:return "IQ3_XXS - 3.0625 bpw"; + case LLAMA_FTYPE_MOSTLY_IQ1_S :return "IQ1_S - 1.5625 bpw"; default: return "unknown, may not work"; } @@ -10312,20 +10314,20 @@ static ggml_type get_k_quant_type(quantize_state_internal & qs, ggml_type new_ty if (arch == LLM_ARCH_FALCON || nx % QK_K != 0) { new_type = GGML_TYPE_Q8_0; } - else if (ftype == LLAMA_FTYPE_MOSTLY_IQ2_XXS || ftype == LLAMA_FTYPE_MOSTLY_IQ2_XS) { + else if (ftype == LLAMA_FTYPE_MOSTLY_IQ2_XXS || ftype == LLAMA_FTYPE_MOSTLY_IQ2_XS || ftype == LLAMA_FTYPE_MOSTLY_IQ1_S) { new_type = GGML_TYPE_Q5_K; } else if (new_type != GGML_TYPE_Q8_0) { new_type = GGML_TYPE_Q6_K; } } else if (name == "token_embd.weight") { - if (ftype == LLAMA_FTYPE_MOSTLY_IQ2_XXS || ftype == LLAMA_FTYPE_MOSTLY_IQ2_XS) { + if (ftype == LLAMA_FTYPE_MOSTLY_IQ2_XXS || ftype == LLAMA_FTYPE_MOSTLY_IQ2_XS || ftype == LLAMA_FTYPE_MOSTLY_IQ1_S) { new_type = GGML_TYPE_Q2_K; } else if (ftype == LLAMA_FTYPE_MOSTLY_IQ3_XXS) { new_type = GGML_TYPE_Q4_K; } - } else if (ftype == LLAMA_FTYPE_MOSTLY_IQ2_XXS || ftype == LLAMA_FTYPE_MOSTLY_IQ2_XS) { + } else if (ftype == LLAMA_FTYPE_MOSTLY_IQ2_XXS || ftype == LLAMA_FTYPE_MOSTLY_IQ2_XS || ftype == LLAMA_FTYPE_MOSTLY_IQ1_S) { if (name.find("attn_v.weight") != std::string::npos) { if (qs.model.hparams.n_gqa() >= 4 || qs.model.hparams.n_expert >= 4) new_type = GGML_TYPE_Q4_K; else new_type = GGML_TYPE_Q2_K; @@ -10335,6 +10337,9 @@ static ggml_type get_k_quant_type(quantize_state_internal & qs, ggml_type new_ty if (qs.i_ffn_down < qs.n_ffn_down/8) new_type = GGML_TYPE_Q2_K; ++qs.i_ffn_down; } + else if (name.find("attn_output.weight") != std::string::npos) { + if (ftype == LLAMA_FTYPE_MOSTLY_IQ1_S) new_type = GGML_TYPE_IQ2_XXS; + } } else if (name.find("attn_v.weight") != std::string::npos) { if (ftype == LLAMA_FTYPE_MOSTLY_Q2_K) { new_type = qs.model.hparams.n_gqa() >= 4 ? GGML_TYPE_Q4_K : GGML_TYPE_Q3_K; @@ -10468,7 +10473,7 @@ static ggml_type get_k_quant_type(quantize_state_internal & qs, ggml_type new_ty if (new_type == GGML_TYPE_Q2_K || new_type == GGML_TYPE_Q3_K || new_type == GGML_TYPE_Q4_K || new_type == GGML_TYPE_Q5_K || new_type == GGML_TYPE_Q6_K || new_type == GGML_TYPE_IQ2_XS || new_type == GGML_TYPE_IQ2_XXS || - new_type == GGML_TYPE_IQ3_XXS) { + new_type == GGML_TYPE_IQ3_XXS || ftype == LLAMA_FTYPE_MOSTLY_IQ1_S) { int nx = tensor->ne[0]; int ny = tensor->ne[1]; if (nx % QK_K != 0) { @@ -10483,6 +10488,7 @@ static ggml_type get_k_quant_type(quantize_state_internal & qs, ggml_type new_ty case GGML_TYPE_IQ2_XXS: case GGML_TYPE_IQ2_XS: case GGML_TYPE_IQ3_XXS: + case GGML_TYPE_IQ1_S: case GGML_TYPE_Q2_K: new_type = GGML_TYPE_Q4_0; break; case GGML_TYPE_Q3_K: new_type = GGML_TYPE_Q4_1; break; case GGML_TYPE_Q4_K: new_type = GGML_TYPE_Q5_0; break; @@ -10525,6 +10531,7 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s case LLAMA_FTYPE_MOSTLY_IQ2_XXS: quantized_type = GGML_TYPE_IQ2_XXS; break; case LLAMA_FTYPE_MOSTLY_IQ2_XS: quantized_type = GGML_TYPE_IQ2_XS; break; case LLAMA_FTYPE_MOSTLY_IQ3_XXS: quantized_type = GGML_TYPE_IQ3_XXS; break; + case LLAMA_FTYPE_MOSTLY_IQ1_S: quantized_type = GGML_TYPE_IQ1_S ; break; default: throw std::runtime_error(format("invalid output file type %d\n", ftype)); } @@ -10698,6 +10705,7 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s } if ((new_type == GGML_TYPE_IQ2_XXS || new_type == GGML_TYPE_IQ2_XS || + new_type == GGML_TYPE_IQ1_S || (new_type == GGML_TYPE_Q2_K && params->ftype == LLAMA_FTYPE_MOSTLY_Q2_K_S && strcmp(tensor->name, "token_embd.weight") != 0)) && !imatrix) { LLAMA_LOG_ERROR("\n\n============================================================\n"); LLAMA_LOG_ERROR("Missing importance matrix for tensor %s in a very low-bit quantization\n", tensor->name); diff --git a/llama.h b/llama.h index f4ec6ea63..5a97abcc9 100644 --- a/llama.h +++ b/llama.h @@ -100,6 +100,7 @@ extern "C" { LLAMA_FTYPE_MOSTLY_Q2_K_S = 21, // except 1d tensors LLAMA_FTYPE_MOSTLY_Q3_K_XS = 22, // except 1d tensors LLAMA_FTYPE_MOSTLY_IQ3_XXS = 23, // except 1d tensors + LLAMA_FTYPE_MOSTLY_IQ1_S = 24, // except 1d tensors LLAMA_FTYPE_GUESSED = 1024, // not specified in the model file }; diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index 30a7d1f5a..ef37c5af2 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -1917,7 +1917,7 @@ static bool test_backend(ggml_backend_t backend, test_mode mode, const char * op GGML_TYPE_Q4_K, GGML_TYPE_Q5_K, GGML_TYPE_Q6_K, GGML_TYPE_IQ2_XXS, GGML_TYPE_IQ2_XS, - GGML_TYPE_IQ3_XXS, + GGML_TYPE_IQ3_XXS, GGML_TYPE_IQ1_S, }; // unary ops From fc0c8d286a533363a9a663510b62af85ffad58b3 Mon Sep 17 00:00:00 2001 From: Daniel Bevenius Date: Sun, 18 Feb 2024 17:19:23 +0100 Subject: [PATCH 369/812] llava : update surgery script to not remove tensors (#5536) This commit updates the surgery script to not remove the tensors from the model file. For this to work the `--skip-unknown` flag is added as an argument to the convert.py script in README.md. The motivation for this change is that the surgery script currently removes the projector tensors from the model file. If the model was checked out from a repository, the model file will have been updated and have to be checked out again to reset this effect. If this can be avoided I think it would be preferable. I did not perform this change for BakLLaVA models as I am not sure how that part works. --- examples/llava/README.md | 2 +- examples/llava/llava-surgery.py | 6 +----- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/examples/llava/README.md b/examples/llava/README.md index 57eb42932..e42db6e5a 100644 --- a/examples/llava/README.md +++ b/examples/llava/README.md @@ -53,7 +53,7 @@ python ./examples/llava/convert-image-encoder-to-gguf.py -m ../clip-vit-large-pa 5. Use `convert.py` to convert the LLaMA part of LLaVA to GGUF: ```sh -python ./convert.py ../llava-v1.5-7b +python ./convert.py ../llava-v1.5-7b --skip-unknown ``` Now both the LLaMA part and the image encoder is in the `llava-v1.5-7b` directory. diff --git a/examples/llava/llava-surgery.py b/examples/llava/llava-surgery.py index 0a61efdfe..8b7a62fba 100644 --- a/examples/llava/llava-surgery.py +++ b/examples/llava/llava-surgery.py @@ -19,10 +19,6 @@ mm_tensors = [k for k, v in checkpoint.items() if k.startswith("model.mm_project projector = {name: checkpoint[name].float() for name in mm_tensors} torch.save(projector, f"{args.model}/llava.projector") -# remove these tensors from the checkpoint and save it again -for name in mm_tensors: - del checkpoint[name] - # BakLLaVA models contain CLIP tensors in it clip_tensors = [k for k, v in checkpoint.items() if k.startswith("model.vision_tower")] if len(clip_tensors) > 0: @@ -39,7 +35,7 @@ if len(clip_tensors) > 0: f.write("{}\n") -torch.save(checkpoint, path) + torch.save(checkpoint, path) print("Done!") print(f"Now you can convert {args.model} to a regular LLaMA GGUF file.") From 5d3de51f972055702a1859186fe7acb8f0b43dc4 Mon Sep 17 00:00:00 2001 From: Herman Semenov Date: Sun, 18 Feb 2024 16:20:12 +0000 Subject: [PATCH 370/812] ggml, common, examples, tests : fixed type arguments in printf (#5528) --- common/common.cpp | 4 +- examples/batched-bench/batched-bench.cpp | 2 +- examples/batched/batched.cpp | 2 +- .../convert-llama2c-to-ggml.cpp | 38 +++++++++---------- examples/perplexity/perplexity.cpp | 2 +- .../train-text-from-scratch.cpp | 14 +++---- ggml.c | 4 +- tests/test-grammar-parser.cpp | 20 +++++----- tests/test-llama-grammar.cpp | 4 +- 9 files changed, 45 insertions(+), 45 deletions(-) diff --git a/common/common.cpp b/common/common.cpp index 3a92d3797..9ffc3951f 100644 --- a/common/common.cpp +++ b/common/common.cpp @@ -1741,7 +1741,7 @@ void dump_non_result_info_yaml(FILE * stream, const gpt_params & params, const l fprintf(stream, "rope_freq_base: %f # default: 10000.0\n", params.rope_freq_base); fprintf(stream, "rope_freq_scale: %f # default: 1.0\n", params.rope_freq_scale); - fprintf(stream, "seed: %d # default: -1 (random seed)\n", params.seed); + fprintf(stream, "seed: %u # default: -1 (random seed)\n", params.seed); fprintf(stream, "simple_io: %s # default: false\n", params.simple_io ? "true" : "false"); fprintf(stream, "cont_batching: %s # default: false\n", params.cont_batching ? "true" : "false"); fprintf(stream, "temp: %f # default: 0.8\n", sparams.temp); @@ -1750,7 +1750,7 @@ void dump_non_result_info_yaml(FILE * stream, const gpt_params & params, const l dump_vector_float_yaml(stream, "tensor_split", tensor_split_vector); fprintf(stream, "tfs: %f # default: 1.0\n", sparams.tfs_z); - fprintf(stream, "threads: %d # default: %d\n", params.n_threads, std::thread::hardware_concurrency()); + fprintf(stream, "threads: %d # default: %u\n", params.n_threads, std::thread::hardware_concurrency()); fprintf(stream, "top_k: %d # default: 40\n", sparams.top_k); fprintf(stream, "top_p: %f # default: 0.95\n", sparams.top_p); fprintf(stream, "min_p: %f # default: 0.0\n", sparams.min_p); diff --git a/examples/batched-bench/batched-bench.cpp b/examples/batched-bench/batched-bench.cpp index 55dfd9784..b4b8a38e1 100644 --- a/examples/batched-bench/batched-bench.cpp +++ b/examples/batched-bench/batched-bench.cpp @@ -159,7 +159,7 @@ int main(int argc, char ** argv) { } LOG_TEE("\n"); - LOG_TEE("%s: n_kv_max = %d, is_pp_shared = %d, n_gpu_layers = %d, mmq = %d, n_threads = %d, n_threads_batch = %d\n", __func__, n_kv_max, is_pp_shared, n_gpu_layers, mmq, ctx_params.n_threads, ctx_params.n_threads_batch); + LOG_TEE("%s: n_kv_max = %d, is_pp_shared = %d, n_gpu_layers = %d, mmq = %d, n_threads = %u, n_threads_batch = %u\n", __func__, n_kv_max, is_pp_shared, n_gpu_layers, mmq, ctx_params.n_threads, ctx_params.n_threads_batch); LOG_TEE("\n"); LOG_TEE("|%6s | %6s | %4s | %6s | %8s | %8s | %8s | %8s | %8s | %8s |\n", "PP", "TG", "B", "N_KV", "T_PP s", "S_PP t/s", "T_TG s", "S_TG t/s", "T s", "S t/s"); diff --git a/examples/batched/batched.cpp b/examples/batched/batched.cpp index eab636692..9be7eb56b 100644 --- a/examples/batched/batched.cpp +++ b/examples/batched/batched.cpp @@ -92,7 +92,7 @@ int main(int argc, char ** argv) { const int n_ctx = llama_n_ctx(ctx); - LOG_TEE("\n%s: n_len = %d, n_ctx = %d, n_batch = %d, n_parallel = %d, n_kv_req = %d\n", __func__, n_len, n_ctx, ctx_params.n_batch, n_parallel, n_kv_req); + LOG_TEE("\n%s: n_len = %d, n_ctx = %d, n_batch = %u, n_parallel = %d, n_kv_req = %d\n", __func__, n_len, n_ctx, ctx_params.n_batch, n_parallel, n_kv_req); // make sure the KV cache is big enough to hold all the prompt and generated tokens if (n_kv_req > n_ctx) { diff --git a/examples/convert-llama2c-to-ggml/convert-llama2c-to-ggml.cpp b/examples/convert-llama2c-to-ggml/convert-llama2c-to-ggml.cpp index 4d41e1779..8209dcb64 100644 --- a/examples/convert-llama2c-to-ggml/convert-llama2c-to-ggml.cpp +++ b/examples/convert-llama2c-to-ggml/convert-llama2c-to-ggml.cpp @@ -325,14 +325,14 @@ struct train_params { }; static void print_params(struct my_llama_hparams * params) { - printf("%s: n_vocab: %d\n", __func__, params->n_vocab); - printf("%s: n_ctx: %d\n", __func__, params->n_ctx); - printf("%s: n_embd: %d\n", __func__, params->n_embd); - printf("%s: n_mult: %d\n", __func__, params->n_mult); - printf("%s: n_head: %d\n", __func__, params->n_head); - printf("%s: n_ff: %d\n", __func__, params->n_ff); - printf("%s: n_layer: %d\n", __func__, params->n_layer); - printf("%s: n_rot: %d\n", __func__, params->n_rot); + printf("%s: n_vocab: %u\n", __func__, params->n_vocab); + printf("%s: n_ctx: %u\n", __func__, params->n_ctx); + printf("%s: n_embd: %u\n", __func__, params->n_embd); + printf("%s: n_mult: %u\n", __func__, params->n_mult); + printf("%s: n_head: %u\n", __func__, params->n_head); + printf("%s: n_ff: %u\n", __func__, params->n_ff); + printf("%s: n_layer: %u\n", __func__, params->n_layer); + printf("%s: n_rot: %u\n", __func__, params->n_rot); } static void init_model(struct my_llama_model * model) { @@ -350,25 +350,25 @@ static void init_model(struct my_llama_model * model) { model->train_tokens = 0; model->tok_embeddings = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, n_embd, n_vocab); - printf("[%s:GG] Allocating [%d] x [%d] = [%d] float space for model->tok_embeddings\n",__func__,n_embd , n_vocab, n_embd * n_vocab); + printf("[%s:GG] Allocating [%u] x [%u] = [%u] float space for model->tok_embeddings\n",__func__,n_embd , n_vocab, n_embd * n_vocab); model->norm = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd); - printf("[%s:GG] Allocating [%d] float space for model->norm\n",__func__,n_embd); + printf("[%s:GG] Allocating [%u] float space for model->norm\n",__func__,n_embd); model->output = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, n_embd, n_vocab); - printf("[%s:GG] Allocating [%d] x[%d] = [%d] float space for model->output\n",__func__,n_embd, n_vocab, n_embd * n_vocab); + printf("[%s:GG] Allocating [%u] x[%u] = [%u] float space for model->output\n",__func__,n_embd, n_vocab, n_embd * n_vocab); // printing the per-layer allocations here so we dont print in the for loop. - printf("[%s:GG] Allocating [%d] x[%d] = [%d] float space for layer.wq for [%d] layers\n",__func__, n_embd, n_embd, n_embd * n_embd, n_layer); - printf("[%s:GG] Allocating [%d] x[%d] = [%d] float space for layer.wk for [%d] layers\n",__func__, n_embd, n_embd, n_embd * n_embd, n_layer); - printf("[%s:GG] Allocating [%d] x[%d] = [%d] float space for layer.wv for [%d] layers\n",__func__, n_embd, n_embd, n_embd * n_embd, n_layer); - printf("[%s:GG] Allocating [%d] x[%d] = [%d] float space for layer.wo for [%d] layers\n",__func__, n_embd, n_embd, n_embd * n_embd, n_layer); + printf("[%s:GG] Allocating [%u] x[%u] = [%u] float space for layer.wq for [%u] layers\n",__func__, n_embd, n_embd, n_embd * n_embd, n_layer); + printf("[%s:GG] Allocating [%u] x[%u] = [%u] float space for layer.wk for [%u] layers\n",__func__, n_embd, n_embd, n_embd * n_embd, n_layer); + printf("[%s:GG] Allocating [%u] x[%u] = [%u] float space for layer.wv for [%u] layers\n",__func__, n_embd, n_embd, n_embd * n_embd, n_layer); + printf("[%s:GG] Allocating [%u] x[%u] = [%u] float space for layer.wo for [%u] layers\n",__func__, n_embd, n_embd, n_embd * n_embd, n_layer); - printf("[%s:GG] Allocating [%d] float space for layer.ffn_norm for [%d] layers\n",__func__,n_embd, n_layer); + printf("[%s:GG] Allocating [%u] float space for layer.ffn_norm for [%u] layers\n",__func__,n_embd, n_layer); - printf("[%s:GG] Allocating [%d] x[%d] = [%d] float space for layer.w1 for [%d] layers\n",__func__, n_ff, n_embd, n_embd * n_ff, n_layer); - printf("[%s:GG] Allocating [%d] x[%d] = [%d] float space for layer.w2 for [%d] layers\n",__func__, n_embd, n_ff, n_ff * n_embd, n_layer); - printf("[%s:GG] Allocating [%d] x[%d] = [%d] float space for layer.w3 for [%d] layers\n",__func__, n_ff, n_embd, n_embd * n_ff, n_layer); + printf("[%s:GG] Allocating [%u] x[%u] = [%u] float space for layer.w1 for [%u] layers\n",__func__, n_ff, n_embd, n_embd * n_ff, n_layer); + printf("[%s:GG] Allocating [%u] x[%u] = [%u] float space for layer.w2 for [%u] layers\n",__func__, n_embd, n_ff, n_ff * n_embd, n_layer); + printf("[%s:GG] Allocating [%u] x[%u] = [%u] float space for layer.w3 for [%u] layers\n",__func__, n_ff, n_embd, n_embd * n_ff, n_layer); ggml_set_name(model->tok_embeddings, "tok_embeddings.weight"); ggml_set_name(model->norm, "norm.weight"); diff --git a/examples/perplexity/perplexity.cpp b/examples/perplexity/perplexity.cpp index 67d2d3293..74dcc642a 100644 --- a/examples/perplexity/perplexity.cpp +++ b/examples/perplexity/perplexity.cpp @@ -1623,7 +1623,7 @@ static void kl_divergence(llama_context * ctx, const gpt_params & params) { uint32_t n_ctx; in.read((char *)&n_ctx, sizeof(n_ctx)); if (n_ctx > llama_n_ctx(ctx)) { - fprintf(stderr, "%s: %s has been computed with %d, while the current context is %d. Increase it with -c and retry\n", + fprintf(stderr, "%s: %s has been computed with %u, while the current context is %d. Increase it with -c and retry\n", __func__, params.logits_file.c_str(), n_ctx, params.n_ctx); } diff --git a/examples/train-text-from-scratch/train-text-from-scratch.cpp b/examples/train-text-from-scratch/train-text-from-scratch.cpp index bfdf124d7..e78ab185d 100644 --- a/examples/train-text-from-scratch/train-text-from-scratch.cpp +++ b/examples/train-text-from-scratch/train-text-from-scratch.cpp @@ -111,13 +111,13 @@ static const char * LLM_TENSOR_FFN_DOWN = "blk.%d.ffn_down"; static const char * LLM_TENSOR_FFN_UP = "blk.%d.ffn_up"; static void print_params(struct my_llama_hparams * params) { - printf("%s: n_vocab: %d\n", __func__, params->n_vocab); - printf("%s: n_ctx: %d\n", __func__, params->n_ctx); - printf("%s: n_embd: %d\n", __func__, params->n_embd); - printf("%s: n_head: %d\n", __func__, params->n_head); - printf("%s: n_ff: %d\n", __func__, params->n_ff); - printf("%s: n_layer: %d\n", __func__, params->n_layer); - printf("%s: n_rot: %d\n", __func__, params->n_rot); + printf("%s: n_vocab: %u\n", __func__, params->n_vocab); + printf("%s: n_ctx: %u\n", __func__, params->n_ctx); + printf("%s: n_embd: %u\n", __func__, params->n_embd); + printf("%s: n_head: %u\n", __func__, params->n_head); + printf("%s: n_ff: %u\n", __func__, params->n_ff); + printf("%s: n_layer: %u\n", __func__, params->n_layer); + printf("%s: n_rot: %u\n", __func__, params->n_rot); } static void set_param_model(struct my_llama_model * model) { diff --git a/ggml.c b/ggml.c index aefcda6d4..8224652a9 100644 --- a/ggml.c +++ b/ggml.c @@ -17909,7 +17909,7 @@ struct ggml_cgraph * ggml_graph_import(const char * fname, struct ggml_context * ptr += ggml_nbytes(tensor); - fprintf(stderr, "%s: loaded leaf %d: '%16s', %9zu bytes\n", __func__, i, tensor->name, ggml_nbytes(tensor)); + fprintf(stderr, "%s: loaded leaf %u: '%16s', %9zu bytes\n", __func__, i, tensor->name, ggml_nbytes(tensor)); } } @@ -18012,7 +18012,7 @@ struct ggml_cgraph * ggml_graph_import(const char * fname, struct ggml_context * result->nodes[i] = tensor; - fprintf(stderr, "%s: loaded node %d: '%16s', %9zu bytes\n", __func__, i, tensor->name, ggml_nbytes(tensor)); + fprintf(stderr, "%s: loaded node %u: '%16s', %9zu bytes\n", __func__, i, tensor->name, ggml_nbytes(tensor)); } } } diff --git a/tests/test-grammar-parser.cpp b/tests/test-grammar-parser.cpp index a0b5b043d..91939e276 100644 --- a/tests/test-grammar-parser.cpp +++ b/tests/test-grammar-parser.cpp @@ -38,8 +38,8 @@ term ::= [0-9]+)"""; // pretty print error message before asserting if (expected_pair.first != key || expected_pair.second != value) { - fprintf(stderr, "expected_pair: %s, %d\n", expected_pair.first.c_str(), expected_pair.second); - fprintf(stderr, "actual_pair: %s, %d\n", key.c_str(), value); + fprintf(stderr, "expected_pair: %s, %u\n", expected_pair.first.c_str(), expected_pair.second); + fprintf(stderr, "actual_pair: %s, %u\n", key.c_str(), value); fprintf(stderr, "expected_pair != actual_pair\n"); } @@ -96,9 +96,9 @@ term ::= [0-9]+)"""; // pretty print error message before asserting if (expected_element.type != element.type || expected_element.value != element.value) { - fprintf(stderr, "index: %d\n", index); - fprintf(stderr, "expected_element: %d, %d\n", expected_element.type, expected_element.value); - fprintf(stderr, "actual_element: %d, %d\n", element.type, element.value); + fprintf(stderr, "index: %u\n", index); + fprintf(stderr, "expected_element: %d, %u\n", expected_element.type, expected_element.value); + fprintf(stderr, "actual_element: %d, %u\n", element.type, element.value); fprintf(stderr, "expected_element != actual_element\n"); } @@ -144,8 +144,8 @@ term ::= [0-9]+)"""; // pretty print error message before asserting if (expected_pair.first != key || expected_pair.second != value) { - fprintf(stderr, "expected_pair: %s, %d\n", expected_pair.first.c_str(), expected_pair.second); - fprintf(stderr, "actual_pair: %s, %d\n", key.c_str(), value); + fprintf(stderr, "expected_pair: %s, %u\n", expected_pair.first.c_str(), expected_pair.second); + fprintf(stderr, "actual_pair: %s, %u\n", key.c_str(), value); fprintf(stderr, "expected_pair != actual_pair\n"); } @@ -235,9 +235,9 @@ term ::= [0-9]+)"""; // pretty print error message before asserting if (expected_element.type != element.type || expected_element.value != element.value) { - fprintf(stderr, "index: %d\n", index); - fprintf(stderr, "expected_element: %d, %d\n", expected_element.type, expected_element.value); - fprintf(stderr, "actual_element: %d, %d\n", element.type, element.value); + fprintf(stderr, "index: %u\n", index); + fprintf(stderr, "expected_element: %d, %u\n", expected_element.type, expected_element.value); + fprintf(stderr, "actual_element: %d, %u\n", element.type, element.value); fprintf(stderr, "expected_element != actual_element\n"); } diff --git a/tests/test-llama-grammar.cpp b/tests/test-llama-grammar.cpp index 16ebe753f..27ca4d265 100644 --- a/tests/test-llama-grammar.cpp +++ b/tests/test-llama-grammar.cpp @@ -180,8 +180,8 @@ int main() if (expected_element.type != element->type || expected_element.value != element->value) { fprintf(stderr, "index: %d\n", index); - fprintf(stderr, "expected_element: %d, %d\n", expected_element.type, expected_element.value); - fprintf(stderr, "actual_element: %d, %d\n", element->type, element->value); + fprintf(stderr, "expected_element: %d, %u\n", expected_element.type, expected_element.value); + fprintf(stderr, "actual_element: %d, %u\n", element->type, element->value); fprintf(stderr, "expected_element != actual_element\n"); } From 1dcc3fde004787e6fc4d84c9de0bb34cd2901a3e Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Sun, 18 Feb 2024 18:21:52 +0200 Subject: [PATCH 371/812] common : fix ub (#5530) --- common/common.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/common/common.cpp b/common/common.cpp index 9ffc3951f..489462b5a 100644 --- a/common/common.cpp +++ b/common/common.cpp @@ -1801,7 +1801,8 @@ void dump_kv_cache_view_seqs(const llama_kv_cache_view & view, int row_size) { if (cs_curr[j] < 0) { continue; } if (seqs.find(cs_curr[j]) == seqs.end()) { if (seqs.size() + 1 >= sizeof(slot_chars)) { break; } - seqs[cs_curr[j]] = seqs.size(); + const size_t sz = seqs.size(); + seqs[cs_curr[j]] = sz; } } if (seqs.size() + 1 >= sizeof(slot_chars)) { break; } From 66c1968f7a2e895675425e875b6589f1233a1b52 Mon Sep 17 00:00:00 2001 From: Daniel Hiltgen Date: Sun, 18 Feb 2024 08:23:16 -0800 Subject: [PATCH 372/812] server : graceful server shutdown (#5244) This updates the server queue to support graceful shutdown of the server on signals. --- examples/server/server.cpp | 23 ++++++++++++++++++++++- examples/server/utils.hpp | 20 +++++++++++++++++--- 2 files changed, 39 insertions(+), 4 deletions(-) diff --git a/examples/server/server.cpp b/examples/server/server.cpp index a0b46970b..7800c6e7e 100644 --- a/examples/server/server.cpp +++ b/examples/server/server.cpp @@ -28,6 +28,7 @@ #include #include #include +#include using json = nlohmann::json; @@ -2511,6 +2512,9 @@ static void append_to_generated_text_from_generated_token_probs(llama_server_con } } +std::function shutdown_handler; +inline void signal_handler(int signal) { shutdown_handler(signal); } + int main(int argc, char **argv) { #if SERVER_VERBOSE != 1 @@ -3128,8 +3132,25 @@ int main(int argc, char **argv) std::placeholders::_2, std::placeholders::_3 )); - llama.queue_tasks.start_loop(); + shutdown_handler = [&](int) { + llama.queue_tasks.terminate(); + }; + +#if defined (__unix__) || (defined (__APPLE__) && defined (__MACH__)) + struct sigaction sigint_action; + sigint_action.sa_handler = signal_handler; + sigemptyset (&sigint_action.sa_mask); + sigint_action.sa_flags = 0; + sigaction(SIGINT, &sigint_action, NULL); +#elif defined (_WIN32) + auto console_ctrl_handler = +[](DWORD ctrl_type) -> BOOL { + return (ctrl_type == CTRL_C_EVENT) ? (signal_handler(SIGINT), true) : false; + }; + SetConsoleCtrlHandler(reinterpret_cast(console_ctrl_handler), true); +#endif + llama.queue_tasks.start_loop(); + svr.stop(); t.join(); llama_backend_free(); diff --git a/examples/server/utils.hpp b/examples/server/utils.hpp index 548548962..0ee670dba 100644 --- a/examples/server/utils.hpp +++ b/examples/server/utils.hpp @@ -220,6 +220,7 @@ inline std::string format_chatml(std::vector messages) struct llama_server_queue { int id = 0; std::mutex mutex_tasks; + bool running; // queues std::vector queue_tasks; std::vector queue_tasks_deferred; @@ -278,9 +279,18 @@ struct llama_server_queue { queue_tasks_deferred.clear(); } - // Start the main loop. This call is blocking - [[noreturn]] + // end the start_loop routine + void terminate() { + { + std::unique_lock lock(mutex_tasks); + running = false; + } + condition_tasks.notify_all(); + } + + // Start the main loop. void start_loop() { + running = true; while (true) { // new task arrived LOG_VERBOSE("have new task", {}); @@ -324,8 +334,12 @@ struct llama_server_queue { { std::unique_lock lock(mutex_tasks); if (queue_tasks.empty()) { + if (!running) { + LOG_VERBOSE("ending start_loop", {}); + return; + } condition_tasks.wait(lock, [&]{ - return !queue_tasks.empty(); + return (!queue_tasks.empty() || !running); }); } } From 36376abe05a12a8cb3af548a4af9b8d0e2e69597 Mon Sep 17 00:00:00 2001 From: Pierrick Hymbert Date: Sun, 18 Feb 2024 17:30:09 +0100 Subject: [PATCH 373/812] server : --n-predict option document and cap to max value (#5549) * server: document --n-predict * server: ensure client request cannot override n_predict if set * server: fix print usage LF in new --n-predict option --- examples/server/README.md | 1 + examples/server/server.cpp | 15 ++++++++++++++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/examples/server/README.md b/examples/server/README.md index 249368749..fe5cd8d5d 100644 --- a/examples/server/README.md +++ b/examples/server/README.md @@ -39,6 +39,7 @@ see https://github.com/ggerganov/llama.cpp/issues/1437 - `--mmproj MMPROJ_FILE`: Path to a multimodal projector file for LLaVA. - `--grp-attn-n`: Set the group attention factor to extend context size through self-extend(default: 1=disabled), used together with group attention width `--grp-attn-w` - `--grp-attn-w`: Set the group attention width to extend context size through self-extend(default: 512), used together with group attention factor `--grp-attn-n` +- `-n, --n-predict`: Set the maximum tokens to predict (default: -1) ## Build diff --git a/examples/server/server.cpp b/examples/server/server.cpp index 7800c6e7e..7aa706e95 100644 --- a/examples/server/server.cpp +++ b/examples/server/server.cpp @@ -159,6 +159,7 @@ struct llama_client_slot int32_t n_decoded = 0; int32_t n_remaining = -1; int32_t i_batch = -1; + int32_t n_predict = -1; int32_t num_prompt_tokens = 0; int32_t num_prompt_tokens_processed = 0; @@ -410,6 +411,7 @@ struct llama_server_context slot.id = i; slot.n_ctx = n_ctx_slot; + slot.n_predict = params.n_predict; LOG_TEE(" -> Slot %i - max context: %i\n", slot.id, n_ctx_slot); @@ -546,6 +548,15 @@ struct llama_server_context slot->sparams.grammar = json_value(data, "grammar", default_sparams.grammar); slot->sparams.n_probs = json_value(data, "n_probs", default_sparams.n_probs); + if (slot->n_predict > 0 && slot->params.n_predict > slot->n_predict) { + // Might be better to reject the request with a 400 ? + LOG_WARNING("Max tokens to predict exceeds server configuration", { + {"params.n_predict", slot->params.n_predict}, + {"slot.n_predict", slot->n_predict}, + }); + slot->params.n_predict = slot->n_predict; + } + // infill if (data.count("input_prefix") != 0) { @@ -1053,6 +1064,7 @@ struct llama_server_context return json { {"n_ctx", slot.n_ctx}, + {"n_predict", slot.n_predict}, {"model", params.model_alias}, {"seed", slot.params.seed}, {"temperature", slot.sparams.temp}, @@ -1915,13 +1927,14 @@ static void server_print_usage(const char *argv0, const gpt_params ¶ms, printf(" --mmproj MMPROJ_FILE path to a multimodal projector file for LLaVA.\n"); printf(" --log-disable disables logging to a file.\n"); printf("\n"); + printf(" -n, --n-predict maximum tokens to predict (default: %d)\n", params.n_predict); printf(" --override-kv KEY=TYPE:VALUE\n"); printf(" advanced option to override model metadata by key. may be specified multiple times.\n"); printf(" types: int, float, bool. example: --override-kv tokenizer.ggml.add_bos_token=bool:false\n"); printf(" -gan N, --grp-attn-n N set the group attention factor to extend context size through self-extend(default: 1=disabled), used together with group attention width `--grp-attn-w`"); printf(" -gaw N, --grp-attn-w N set the group attention width to extend context size through self-extend(default: 512), used together with group attention factor `--grp-attn-n`"); printf(" --chat-template FORMAT_NAME"); - printf(" set chat template, possible valus is: llama2, chatml (default %s)", sparams.chat_template.c_str()); + printf(" set chat template, possible value is: llama2, chatml (default %s)", sparams.chat_template.c_str()); printf("\n"); } From e75c6279d1c8e7abb82a331f5de7124eed402de2 Mon Sep 17 00:00:00 2001 From: Pierrick Hymbert Date: Sun, 18 Feb 2024 17:31:28 +0100 Subject: [PATCH 374/812] server : enhanced health endpoint (#5548) * server: enrich health endpoint with available slots, return 503 if not slots are available * server: document new status no slot available in the README.md --- examples/server/README.md | 1 + examples/server/server.cpp | 31 +++++++++++++++++++++++++++++-- 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/examples/server/README.md b/examples/server/README.md index fe5cd8d5d..5e3ae833b 100644 --- a/examples/server/README.md +++ b/examples/server/README.md @@ -136,6 +136,7 @@ node index.js - `{"status": "loading model"}` if the model is still being loaded. - `{"status": "error"}` if the model failed to load. - `{"status": "ok"}` if the model is successfully loaded and the server is ready for further requests mentioned below. + - `{"status": "no slot available", "slots_idle": 0, "slots_processing": 32}` if no slot are currently available - **POST** `/completion`: Given a `prompt`, it returns the predicted completion. diff --git a/examples/server/server.cpp b/examples/server/server.cpp index 7aa706e95..8145af867 100644 --- a/examples/server/server.cpp +++ b/examples/server/server.cpp @@ -2578,8 +2578,35 @@ int main(int argc, char **argv) server_state current_state = state.load(); switch(current_state) { case SERVER_STATE_READY: - res.set_content(R"({"status": "ok"})", "application/json"); - res.status = 200; // HTTP OK + if (llama.all_slots_are_idle) { + res.set_content(R"({"status": "ok"})", "application/json"); + res.status = 200; // HTTP OK + } else { + int available_slots = 0; + int processing_slots = 0; + for (llama_client_slot & slot : llama.slots) { + if (slot.available()) { + available_slots++; + } else { + processing_slots++; + } + } + if (available_slots > 0) { + json health = { + {"status", "ok"}, + {"slots_idle", available_slots}, + {"slots_processing", processing_slots}}; + res.set_content(health.dump(), "application/json"); + res.status = 200; // HTTP OK + } else { + json health = { + {"status", "no slot available"}, + {"slots_idle", available_slots}, + {"slots_processing", processing_slots}}; + res.set_content(health.dump(), "application/json"); + res.status = 503; // HTTP Service Unavailable + } + } break; case SERVER_STATE_LOADING_MODEL: res.set_content(R"({"status": "loading model"})", "application/json"); From f3f28c5395cd25b371617981b341616dbdd31e85 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Sun, 18 Feb 2024 19:17:00 +0200 Subject: [PATCH 375/812] cmake : fix GGML_USE_SYCL typo (#5555) --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 5ea4d4f19..0c29b5d09 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -526,7 +526,7 @@ if (LLAMA_SYCL) message(STATUS "SYCL found") - add_compile_definitions(GML_USE_SYCL) + add_compile_definitions(GGML_USE_SYCL) if (LLAMA_SYCL_F16) add_compile_definitions(GGML_SYCL_F16) From 689a091bbe0537ee9abff3e15a1d74f5f3561165 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Sun, 18 Feb 2024 19:38:06 +0200 Subject: [PATCH 376/812] sampling : do not set min_keep to n_probs (#5564) --- common/sampling.cpp | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/common/sampling.cpp b/common/sampling.cpp index 53013138a..611c327bb 100644 --- a/common/sampling.cpp +++ b/common/sampling.cpp @@ -121,7 +121,7 @@ static void sampler_queue( struct llama_context * ctx_main, const llama_sampling_params & params, llama_token_data_array & cur_p, - size_t & min_keep) { + size_t min_keep) { const float temp = params.temp; const float dynatemp_range = params.dynatemp_range; const float dynatemp_exponent = params.dynatemp_exponent; @@ -248,10 +248,7 @@ static llama_token llama_sampling_sample_impl( llama_sample_temp(ctx_main, &cur_p, temp); id = llama_sample_token_mirostat_v2(ctx_main, &cur_p, mirostat_tau, mirostat_eta, &ctx_sampling->mirostat_mu); } else { - // temperature sampling - size_t min_keep = std::max(1, params.n_probs); - - sampler_queue(ctx_main, params, cur_p, min_keep); + sampler_queue(ctx_main, params, cur_p, 1); id = llama_sample_token(ctx_main, &cur_p); From c145f8a132b2fe1d1e65987faddbd9a40bef7a12 Mon Sep 17 00:00:00 2001 From: Pierrick Hymbert Date: Sun, 18 Feb 2024 18:39:57 +0100 Subject: [PATCH 377/812] server : slots monitoring endpoint (#5550) --- examples/server/README.md | 64 ++++++++++++++++++++++++++++++++++++++ examples/server/server.cpp | 32 +++++++++++++++++++ 2 files changed, 96 insertions(+) diff --git a/examples/server/README.md b/examples/server/README.md index 5e3ae833b..ac5133d24 100644 --- a/examples/server/README.md +++ b/examples/server/README.md @@ -40,6 +40,7 @@ see https://github.com/ggerganov/llama.cpp/issues/1437 - `--grp-attn-n`: Set the group attention factor to extend context size through self-extend(default: 1=disabled), used together with group attention width `--grp-attn-w` - `--grp-attn-w`: Set the group attention width to extend context size through self-extend(default: 512), used together with group attention factor `--grp-attn-n` - `-n, --n-predict`: Set the maximum tokens to predict (default: -1) +- `--slots-endpoint-disable`: To disable slots state monitoring endpoint. Slots state may contain user data, prompts included. ## Build @@ -381,6 +382,69 @@ Notice that each `probs` is an array of length `n_probs`. }' ``` +- **GET** `/slots`: Returns the current slots processing state. Can be disabled with `--slots-endpoint-disable`. + +### Result JSON + +```json +[ + { + "dynatemp_exponent": 1.0, + "dynatemp_range": 0.0, + "frequency_penalty": 0.0, + "grammar": "", + "id": 0, + "ignore_eos": false, + "logit_bias": [], + "min_p": 0.05000000074505806, + "mirostat": 0, + "mirostat_eta": 0.10000000149011612, + "mirostat_tau": 5.0, + "model": "llama-2-7b-32k-instruct.Q2_K.gguf", + "n_ctx": 2048, + "n_keep": 0, + "n_predict": 100000, + "n_probs": 0, + "next_token": { + "has_next_token": true, + "n_remain": -1, + "num_tokens_predicted": 0, + "stopped_eos": false, + "stopped_limit": false, + "stopped_word": false, + "stopping_word": "" + }, + "penalize_nl": true, + "penalty_prompt_tokens": [], + "presence_penalty": 0.0, + "prompt": "Say hello to llama.cpp", + "repeat_last_n": 64, + "repeat_penalty": 1.100000023841858, + "samplers": [ + "top_k", + "tfs_z", + "typical_p", + "top_p", + "min_p", + "temperature" + ], + "seed": 42, + "state": 1, + "stop": [ + "\n" + ], + "stream": false, + "task_id": 0, + "temperature": 0.0, + "tfs_z": 1.0, + "top_k": 40, + "top_p": 0.949999988079071, + "typical_p": 1.0, + "use_penalty_prompt_tokens": false + } +] +``` + ## More examples ### Change system prompt on runtime diff --git a/examples/server/server.cpp b/examples/server/server.cpp index 8145af867..4f2e9c898 100644 --- a/examples/server/server.cpp +++ b/examples/server/server.cpp @@ -41,6 +41,7 @@ struct server_params int32_t port = 8080; int32_t read_timeout = 600; int32_t write_timeout = 600; + bool slots_endpoint = true; }; bool server_verbose = false; @@ -1926,6 +1927,7 @@ static void server_print_usage(const char *argv0, const gpt_params ¶ms, printf(" set a file to load a system prompt (initial prompt of all slots), this is useful for chat applications.\n"); printf(" --mmproj MMPROJ_FILE path to a multimodal projector file for LLaVA.\n"); printf(" --log-disable disables logging to a file.\n"); + printf(" --slots-endpoint-disable disables slots monitoring endpoint.\n"); printf("\n"); printf(" -n, --n-predict maximum tokens to predict (default: %d)\n", params.n_predict); printf(" --override-kv KEY=TYPE:VALUE\n"); @@ -2374,6 +2376,10 @@ static void server_params_parse(int argc, char **argv, server_params &sparams, log_set_target(stdout); LOG_INFO("logging to file is disabled.", {}); } + else if (arg == "--slots-endpoint-disable") + { + sparams.slots_endpoint = false; + } else if (arg == "--chat-template") { if (++i >= argc) @@ -2619,6 +2625,32 @@ int main(int argc, char **argv) } }); + if (sparams.slots_endpoint) { + svr.Get("/slots", [&](const httplib::Request&, httplib::Response& res) { + json slots; + for (llama_client_slot & slot : llama.slots) { + json slot_data = llama.get_formated_generation(slot); + slot_data["id"] = slot.id; + slot_data["task_id"] = slot.task_id; + slot_data["state"] = slot.state; + slot_data["prompt"] = slot.prompt; + slot_data["next_token"] = { + {"has_next_token", slot.has_next_token}, + {"n_remain", slot.n_remaining}, + {"num_tokens_predicted", slot.n_decoded}, + {"stopped_eos", slot.stopped_eos}, + {"stopped_word", slot.stopped_word}, + {"stopped_limit", slot.stopped_limit}, + {"stopping_word", slot.stopping_word}, + }; + + slots.push_back(slot_data); + } + res.set_content(slots.dump(), "application/json"); + res.status = 200; // HTTP OK + }); + } + svr.set_logger(log_server_request); svr.set_exception_handler([](const httplib::Request &, httplib::Response &res, std::exception_ptr ep) From 5ee99c32f5e47c8d32634eff9a47fb32a24c276b Mon Sep 17 00:00:00 2001 From: Robey Holderith Date: Sun, 18 Feb 2024 11:11:16 -0800 Subject: [PATCH 378/812] common, server : surface min_keep as its own parameter (#5567) * Feature - surface min_keep as its own parameter * Updated README with min_keep param --- common/common.cpp | 1 + common/sampling.cpp | 5 ++++- common/sampling.h | 1 + examples/server/README.md | 2 ++ examples/server/public/index.html | 4 ++++ examples/server/server.cpp | 2 ++ 6 files changed, 14 insertions(+), 1 deletion(-) diff --git a/common/common.cpp b/common/common.cpp index 489462b5a..10ef11829 100644 --- a/common/common.cpp +++ b/common/common.cpp @@ -1704,6 +1704,7 @@ void dump_non_result_info_yaml(FILE * stream, const gpt_params & params, const l } fprintf(stream, "lora_base: %s\n", params.lora_base.c_str()); fprintf(stream, "main_gpu: %d # default: 0\n", params.main_gpu); + fprintf(stream, "min_keep: %d # default: 0 (disabled)\n", sparams.min_keep); fprintf(stream, "mirostat: %d # default: 0 (disabled)\n", sparams.mirostat); fprintf(stream, "mirostat_ent: %f # default: 5.0\n", sparams.mirostat_tau); fprintf(stream, "mirostat_lr: %f # default: 0.1\n", sparams.mirostat_eta); diff --git a/common/sampling.cpp b/common/sampling.cpp index 611c327bb..de4331a11 100644 --- a/common/sampling.cpp +++ b/common/sampling.cpp @@ -248,7 +248,10 @@ static llama_token llama_sampling_sample_impl( llama_sample_temp(ctx_main, &cur_p, temp); id = llama_sample_token_mirostat_v2(ctx_main, &cur_p, mirostat_tau, mirostat_eta, &ctx_sampling->mirostat_mu); } else { - sampler_queue(ctx_main, params, cur_p, 1); + // temperature sampling + size_t min_keep = std::max(1, params.min_keep); + + sampler_queue(ctx_main, params, cur_p, min_keep); id = llama_sample_token(ctx_main, &cur_p); diff --git a/common/sampling.h b/common/sampling.h index e1279a894..95d875394 100644 --- a/common/sampling.h +++ b/common/sampling.h @@ -22,6 +22,7 @@ enum class llama_sampler_type : char { typedef struct llama_sampling_params { int32_t n_prev = 64; // number of previous tokens to remember int32_t n_probs = 0; // if greater than 0, output the probabilities of top n_probs tokens. + int32_t min_keep = 0; // 0 = disabled, otherwise samplers should return at least min_keep tokens int32_t top_k = 40; // <= 0 to use vocab size float top_p = 0.95f; // 1.0 = disabled float min_p = 0.05f; // 0.0 = disabled diff --git a/examples/server/README.md b/examples/server/README.md index ac5133d24..809e2d37c 100644 --- a/examples/server/README.md +++ b/examples/server/README.md @@ -199,6 +199,8 @@ node index.js `n_probs`: If greater than 0, the response also contains the probabilities of top N tokens for each generated token (default: 0) + `min_keep`: If greater than 0, force samplers to return N possible tokens at minimum (default: 0) + `image_data`: An array of objects to hold base64-encoded image `data` and its `id`s to be reference in `prompt`. You can determine the place of the image in the prompt as in the following: `USER:[img-12]Describe the image in detail.\nASSISTANT:`. In this case, `[img-12]` will be replaced by the embeddings of the image with id `12` in the following `image_data` array: `{..., "image_data": [{"data": "", "id": 12}]}`. Use `image_data` only with multimodal models, e.g., LLaVA. `slot_id`: Assign the completion task to an specific slot. If is -1 the task will be assigned to a Idle slot (default: -1) diff --git a/examples/server/public/index.html b/examples/server/public/index.html index b059c75f2..84038ddce 100644 --- a/examples/server/public/index.html +++ b/examples/server/public/index.html @@ -234,6 +234,7 @@ mirostat_eta: 0.1, // learning rate grammar: '', n_probs: 0, // no completion_probabilities, + min_keep: 0, // min probs from each sampler, image_data: [], cache_prompt: true, api_key: '' @@ -791,6 +792,9 @@
    ${IntField({ label: "Show Probabilities", max: 10, min: 0, name: "n_probs", value: params.value.n_probs })}
    +
    + ${IntField({ label: "Min Probabilities from each Sampler", max: 10, min: 0, name: "min_keep", value: params.value.min_keep })} +
    diff --git a/examples/server/server.cpp b/examples/server/server.cpp index 4f2e9c898..22c344dd4 100644 --- a/examples/server/server.cpp +++ b/examples/server/server.cpp @@ -548,6 +548,7 @@ struct llama_server_context slot->params.seed = json_value(data, "seed", default_params.seed); slot->sparams.grammar = json_value(data, "grammar", default_sparams.grammar); slot->sparams.n_probs = json_value(data, "n_probs", default_sparams.n_probs); + slot->sparams.min_keep = json_value(data, "min_keep", default_sparams.min_keep); if (slot->n_predict > 0 && slot->params.n_predict > slot->n_predict) { // Might be better to reject the request with a 400 ? @@ -1093,6 +1094,7 @@ struct llama_server_context {"stream", slot.params.stream}, {"logit_bias", slot.sparams.logit_bias}, {"n_probs", slot.sparams.n_probs}, + {"min_keep", slot.sparams.min_keep}, {"grammar", slot.sparams.grammar}, {"samplers", samplers_sequence} }; From 7ad554f90e735cf2a0f612ce44f9aa4fad6ae46a Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Sun, 18 Feb 2024 21:39:58 +0200 Subject: [PATCH 379/812] metal : fix unused warnings (#0) --- ggml-metal.metal | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/ggml-metal.metal b/ggml-metal.metal index a00962111..d0a85a192 100644 --- a/ggml-metal.metal +++ b/ggml-metal.metal @@ -4027,7 +4027,10 @@ void kernel_mul_mv_iq2_xxs_f32_impl( y4 += 32 * 32; } #else - // TODO + (void) x; + (void) y; + (void) yl; + (void) nb32; #endif for (int row = 0; row < N_DST; ++row) { @@ -4170,7 +4173,10 @@ void kernel_mul_mv_iq2_xs_f32_impl( y4 += 32 * 32; } #else - // TODO + (void) x; + (void) y; + (void) yl; + (void) nb32; #endif for (int row = 0; row < N_DST; ++row) { @@ -4306,7 +4312,10 @@ void kernel_mul_mv_iq3_xxs_f32_impl( y4 += 32 * 32; } #else - // TODO + (void) x; + (void) y; + (void) yl; + (void) nb32; #endif for (int row = 0; row < N_DST; ++row) { @@ -4424,7 +4433,10 @@ void kernel_mul_mv_iq1_s_f32_impl( y4 += 16 * 32; } #else - // TODO + (void) x; + (void) y; + (void) yl; + (void) nb32; #endif for (int row = 0; row < N_DST; ++row) { @@ -4659,6 +4671,8 @@ void dequantize_q4_K(device const block_q4_K *xb, short il, thread type4x4 & reg const float dl = d * sc[0]; const float ml = min * sc[1]; #else + (void) get_scale_min_k4_just2; + q = q + 16 * (il&1); device const uint8_t * s = xb->scales; device const half2 * dh = (device const half2 *)xb->d; From b1de96824bdbeb91ea458abcb3e5478690ad0727 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Sun, 18 Feb 2024 22:39:30 +0200 Subject: [PATCH 380/812] ci : fix wikitext url + compile warnings (#5569) ggml-ci --- README.md | 2 +- ci/run.sh | 4 ++-- examples/perplexity/perplexity.cpp | 4 ++-- ggml-quants.c | 6 +++--- scripts/get-wikitext-2.sh | 2 +- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 0c4ee5a27..8c7bc2689 100644 --- a/README.md +++ b/README.md @@ -768,7 +768,7 @@ The time per token is measured on a MacBook M1 Pro 32GB RAM using 4 and 8 thread #### How to run -1. Download/extract: https://s3.amazonaws.com/research.metamind.io/wikitext/wikitext-2-raw-v1.zip?ref=salesforce-research +1. Download/extract: https://huggingface.co/datasets/ggml-org/ci/resolve/main/wikitext-2-raw-v1.zip 2. Run `./perplexity -m models/7B/ggml-model-q4_0.gguf -f wiki.test.raw` 3. Output: ``` diff --git a/ci/run.sh b/ci/run.sh index b94658c96..f3a29c2e9 100755 --- a/ci/run.sh +++ b/ci/run.sh @@ -219,7 +219,7 @@ function gg_run_open_llama_3b_v2 { gg_wget models-mnt/open-llama/3B-v2/ https://huggingface.co/openlm-research/open_llama_3b_v2/resolve/main/pytorch_model.bin gg_wget models-mnt/open-llama/3B-v2/ https://huggingface.co/openlm-research/open_llama_3b_v2/raw/main/generation_config.json - gg_wget models-mnt/wikitext/ https://s3.amazonaws.com/research.metamind.io/wikitext/wikitext-2-raw-v1.zip + gg_wget models-mnt/wikitext/ https://huggingface.co/datasets/ggml-org/ci/resolve/main/wikitext-2-raw-v1.zip unzip -o models-mnt/wikitext/wikitext-2-raw-v1.zip -d models-mnt/wikitext/ head -n 60 models-mnt/wikitext/wikitext-2-raw/wiki.test.raw > models-mnt/wikitext/wikitext-2-raw/wiki.test-60.raw @@ -401,7 +401,7 @@ function gg_run_open_llama_7b_v2 { gg_wget models-mnt/open-llama/7B-v2/ https://huggingface.co/openlm-research/open_llama_7b_v2/resolve/main/pytorch_model-00002-of-00002.bin gg_wget models-mnt/open-llama/7B-v2/ https://huggingface.co/openlm-research/open_llama_7b_v2/raw/main/generation_config.json - gg_wget models-mnt/wikitext/ https://s3.amazonaws.com/research.metamind.io/wikitext/wikitext-2-raw-v1.zip + gg_wget models-mnt/wikitext/ https://huggingface.co/datasets/ggml-org/ci/resolve/main/wikitext-2-raw-v1.zip unzip -o models-mnt/wikitext/wikitext-2-raw-v1.zip -d models-mnt/wikitext/ path_models="../models-mnt/open-llama/7B-v2" diff --git a/examples/perplexity/perplexity.cpp b/examples/perplexity/perplexity.cpp index 74dcc642a..9ec989389 100644 --- a/examples/perplexity/perplexity.cpp +++ b/examples/perplexity/perplexity.cpp @@ -309,7 +309,7 @@ static void process_logits(int n_vocab, const float * logits, const int * tokens } static results_perplexity perplexity_v2(llama_context * ctx, const gpt_params & params) { - // Download: https://s3.amazonaws.com/research.metamind.io/wikitext/wikitext-2-raw-v1.zip?ref=salesforce-research + // Download: https://huggingface.co/datasets/ggml-org/ci/resolve/main/wikitext-2-raw-v1.zip // Run `./perplexity -m models/7B/ggml-model-q4_0.bin -f wiki.test.raw` // Output: `perplexity: 13.5106 [114/114]` // BOS tokens will be added for each chunk before eval @@ -447,7 +447,7 @@ static results_perplexity perplexity(llama_context * ctx, const gpt_params & par return perplexity_v2(ctx, params); } - // Download: https://s3.amazonaws.com/research.metamind.io/wikitext/wikitext-2-raw-v1.zip?ref=salesforce-research + // Download: https://huggingface.co/datasets/ggml-org/ci/resolve/main/wikitext-2-raw-v1.zip // Run `./perplexity -m models/7B/ggml-model-q4_0.bin -f wiki.test.raw` // Output: `perplexity: 13.5106 [114/114]` // BOS tokens will be added for each chunk before eval diff --git a/ggml-quants.c b/ggml-quants.c index 48f5294e1..43a8f1de4 100644 --- a/ggml-quants.c +++ b/ggml-quants.c @@ -1837,9 +1837,9 @@ static void quantize_row_q2_K_impl(const float * restrict x, block_q2_K * restri float sigma2 = sumx2/QK_K; for (int j = 0; j < QK_K/16; ++j) { const float * restrict qw = quant_weights + QK_K * i + 16*j; - for (int l = 0; l < 16; ++l) weight[l] = qw[l] * sqrtf(sigma2 + x[16*j + l]*x[16*j + l]); - for (int l = 0; l < 16; ++l) sw[j] += weight[l]; - scales[j] = make_qkx3_quants(16, 3, x + 16*j, weight, L + 16*j, &mins[j], Laux, -0.9f, 0.05f, 36, false); + for (int l = 0; l < QK_K/16; ++l) weight[l] = qw[l] * sqrtf(sigma2 + x[16*j + l]*x[16*j + l]); + for (int l = 0; l < QK_K/16; ++l) sw[j] += weight[l]; + scales[j] = make_qkx3_quants(QK_K/16, 3, x + 16*j, weight, L + 16*j, &mins[j], Laux, -0.9f, 0.05f, 36, false); } float dm = make_qp_quants(QK_K/16, 15, scales, Ls, sw); diff --git a/scripts/get-wikitext-2.sh b/scripts/get-wikitext-2.sh index ff96f331e..7ca760fa6 100755 --- a/scripts/get-wikitext-2.sh +++ b/scripts/get-wikitext-2.sh @@ -1,6 +1,6 @@ #!/bin/bash -wget https://s3.amazonaws.com/research.metamind.io/wikitext/wikitext-2-raw-v1.zip +wget https://huggingface.co/datasets/ggml-org/ci/resolve/main/wikitext-2-raw-v1.zip echo "Usage:" echo "" From 14278f55d2e2c6a53022075c7f2719b71e1cd61d Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Sun, 18 Feb 2024 22:58:57 +0200 Subject: [PATCH 381/812] ggml : restore vec dot stride arg names (#5453) --- ggml-quants.c | 76 +++++++++++++++++++++++++-------------------------- 1 file changed, 38 insertions(+), 38 deletions(-) diff --git a/ggml-quants.c b/ggml-quants.c index 43a8f1de4..3319d2ccf 100644 --- a/ggml-quants.c +++ b/ggml-quants.c @@ -3855,7 +3855,7 @@ static inline __m128i get_scale_shuffle(int i) { } #endif -void ggml_vec_dot_q4_0_q8_0(int n, float * restrict s, size_t bs, const void * restrict vx, size_t bbx, const void * restrict vy, size_t bby, int nrc) { +void ggml_vec_dot_q4_0_q8_0(int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) { const int qk = QK8_0; const int nb = n / qk; @@ -3866,8 +3866,8 @@ void ggml_vec_dot_q4_0_q8_0(int n, float * restrict s, size_t bs, const void * r assert(nrc == 1); #endif UNUSED(nrc); - UNUSED(bbx); - UNUSED(bby); + UNUSED(bx); + UNUSED(by); UNUSED(bs); const block_q4_0 * restrict x = vx; @@ -4024,15 +4024,15 @@ void ggml_vec_dot_q4_0_q8_0(int n, float * restrict s, size_t bs, const void * r const __m128i tmp = _mm_loadu_si128((const __m128i *)x[i].qs); - __m128i bx = _mm_and_si128(lowMask, tmp); - __m128i by = _mm_loadu_si128((const __m128i *)y[i].qs); - bx = _mm_sub_epi8(bx, off); - const __m128i i32_0 = mul_sum_i8_pairs(bx, by); + __m128i bx_0 = _mm_and_si128(lowMask, tmp); + __m128i by_0 = _mm_loadu_si128((const __m128i *)y[i].qs); + bx_0 = _mm_sub_epi8(bx_0, off); + const __m128i i32_0 = mul_sum_i8_pairs(bx_0, by_0); - bx = _mm_and_si128(lowMask, _mm_srli_epi64(tmp, 4)); - by = _mm_loadu_si128((const __m128i *)(y[i].qs + 16)); - bx = _mm_sub_epi8(bx, off); - const __m128i i32_1 = mul_sum_i8_pairs(bx, by); + bx_0 = _mm_and_si128(lowMask, _mm_srli_epi64(tmp, 4)); + by_0 = _mm_loadu_si128((const __m128i *)(y[i].qs + 16)); + bx_0 = _mm_sub_epi8(bx_0, off); + const __m128i i32_1 = mul_sum_i8_pairs(bx_0, by_0); // Convert int32_t to float __m256 p = _mm256_cvtepi32_ps(MM256_SET_M128I(i32_0, i32_1)); @@ -4222,7 +4222,7 @@ void ggml_vec_dot_q4_0_q8_0(int n, float * restrict s, size_t bs, const void * r #endif } -void ggml_vec_dot_q4_1_q8_1(int n, float * restrict s, size_t bs, const void * restrict vx, size_t bbx, const void * restrict vy, size_t bby, int nrc) { +void ggml_vec_dot_q4_1_q8_1(int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) { const int qk = QK8_1; const int nb = n / qk; @@ -4233,8 +4233,8 @@ void ggml_vec_dot_q4_1_q8_1(int n, float * restrict s, size_t bs, const void * r assert(nrc == 1); #endif UNUSED(nrc); - UNUSED(bbx); - UNUSED(bby); + UNUSED(bx); + UNUSED(by); UNUSED(bs); const block_q4_1 * restrict x = vx; @@ -4440,7 +4440,7 @@ void ggml_vec_dot_q4_1_q8_1(int n, float * restrict s, size_t bs, const void * r #endif } -void ggml_vec_dot_q5_0_q8_0(int n, float * restrict s, size_t bs, const void * restrict vx, size_t bbx, const void * restrict vy, size_t bby, int nrc) { +void ggml_vec_dot_q5_0_q8_0(int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) { const int qk = QK8_0; const int nb = n / qk; @@ -4448,8 +4448,8 @@ void ggml_vec_dot_q5_0_q8_0(int n, float * restrict s, size_t bs, const void * r assert(qk == QK5_0); assert(nrc == 1); UNUSED(nrc); - UNUSED(bbx); - UNUSED(bby); + UNUSED(bx); + UNUSED(by); UNUSED(bs); const block_q5_0 * restrict x = vx; @@ -4618,21 +4618,21 @@ void ggml_vec_dot_q5_0_q8_0(int n, float * restrict s, size_t bs, const void * r /* Compute combined scale for the block */ const __m256 d = _mm256_set1_ps(GGML_FP16_TO_FP32(x[i].d) * GGML_FP16_TO_FP32(y[i].d)); - __m256i bx = bytes_from_nibbles_32(x[i].qs); + __m256i bx_0 = bytes_from_nibbles_32(x[i].qs); const __m256i bxhi = bytes_from_bits_32(x[i].qh); __m128i bxhil = _mm256_castsi256_si128(bxhi); __m128i bxhih = _mm256_extractf128_si256(bxhi, 1); bxhil = _mm_andnot_si128(bxhil, mask); bxhih = _mm_andnot_si128(bxhih, mask); - __m128i bxl = _mm256_castsi256_si128(bx); - __m128i bxh = _mm256_extractf128_si256(bx, 1); + __m128i bxl = _mm256_castsi256_si128(bx_0); + __m128i bxh = _mm256_extractf128_si256(bx_0, 1); bxl = _mm_or_si128(bxl, bxhil); bxh = _mm_or_si128(bxh, bxhih); - bx = MM256_SET_M128I(bxh, bxl); + bx_0 = MM256_SET_M128I(bxh, bxl); - const __m256i by = _mm256_loadu_si256((const __m256i *)y[i].qs); + const __m256i by_0 = _mm256_loadu_si256((const __m256i *)y[i].qs); - const __m256 q = mul_sum_i8_pairs_float(bx, by); + const __m256 q = mul_sum_i8_pairs_float(bx_0, by_0); /* Multiply q with scale and accumulate */ acc = _mm256_add_ps(_mm256_mul_ps(d, q), acc); @@ -4731,7 +4731,7 @@ void ggml_vec_dot_q5_0_q8_0(int n, float * restrict s, size_t bs, const void * r #endif } -void ggml_vec_dot_q5_1_q8_1(int n, float * restrict s, size_t bs, const void * restrict vx, size_t bbx, const void * restrict vy, size_t bby, int nrc) { +void ggml_vec_dot_q5_1_q8_1(int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) { const int qk = QK8_1; const int nb = n / qk; @@ -4739,8 +4739,8 @@ void ggml_vec_dot_q5_1_q8_1(int n, float * restrict s, size_t bs, const void * r assert(qk == QK5_1); assert(nrc == 1); UNUSED(nrc); - UNUSED(bbx); - UNUSED(bby); + UNUSED(bx); + UNUSED(by); UNUSED(bs); const block_q5_1 * restrict x = vx; @@ -4925,22 +4925,22 @@ void ggml_vec_dot_q5_1_q8_1(int n, float * restrict s, size_t bs, const void * r summs += GGML_FP16_TO_FP32(x[i].m) * y[i].s; - __m256i bx = bytes_from_nibbles_32(x[i].qs); + __m256i bx_0 = bytes_from_nibbles_32(x[i].qs); const __m256i bxhi = bytes_from_bits_32(x[i].qh); __m128i bxhil = _mm256_castsi256_si128(bxhi); __m128i bxhih = _mm256_extractf128_si256(bxhi, 1); bxhil = _mm_and_si128(bxhil, mask); bxhih = _mm_and_si128(bxhih, mask); - __m128i bxl = _mm256_castsi256_si128(bx); - __m128i bxh = _mm256_extractf128_si256(bx, 1); + __m128i bxl = _mm256_castsi256_si128(bx_0); + __m128i bxh = _mm256_extractf128_si256(bx_0, 1); bxl = _mm_or_si128(bxl, bxhil); bxh = _mm_or_si128(bxh, bxhih); - bx = MM256_SET_M128I(bxh, bxl); + bx_0 = MM256_SET_M128I(bxh, bxl); const __m256 dy = _mm256_set1_ps(y[i].d); - const __m256i by = _mm256_loadu_si256((const __m256i *)y[i].qs); + const __m256i by_0 = _mm256_loadu_si256((const __m256i *)y[i].qs); - const __m256 q = mul_sum_us8_pairs_float(bx, by); + const __m256 q = mul_sum_us8_pairs_float(bx_0, by_0); acc = _mm256_add_ps(_mm256_mul_ps(q, _mm256_mul_ps(dx, dy)), acc); } @@ -5035,7 +5035,7 @@ void ggml_vec_dot_q5_1_q8_1(int n, float * restrict s, size_t bs, const void * r #endif } -void ggml_vec_dot_q8_0_q8_0(int n, float * restrict s, size_t bs, const void * restrict vx, size_t bbx, const void * restrict vy, size_t bby, int nrc) { +void ggml_vec_dot_q8_0_q8_0(int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) { const int qk = QK8_0; const int nb = n / qk; @@ -5046,8 +5046,8 @@ void ggml_vec_dot_q8_0_q8_0(int n, float * restrict s, size_t bs, const void * r assert(nrc == 1); #endif UNUSED(nrc); - UNUSED(bbx); - UNUSED(bby); + UNUSED(bx); + UNUSED(by); UNUSED(bs); const block_q8_0 * restrict x = vx; @@ -5169,10 +5169,10 @@ void ggml_vec_dot_q8_0_q8_0(int n, float * restrict s, size_t bs, const void * r for (int i = 0; i < nb; i++) { // load elements - vint8m1_t bx = __riscv_vle8_v_i8m1(x[i].qs, vl); - vint8m1_t by = __riscv_vle8_v_i8m1(y[i].qs, vl); + vint8m1_t bx_0 = __riscv_vle8_v_i8m1(x[i].qs, vl); + vint8m1_t by_0 = __riscv_vle8_v_i8m1(y[i].qs, vl); - vint16m2_t vw_mul = __riscv_vwmul_vv_i16m2(bx, by, vl); + vint16m2_t vw_mul = __riscv_vwmul_vv_i16m2(bx_0, by_0, vl); vint32m1_t v_zero = __riscv_vmv_v_x_i32m1(0, vl); vint32m1_t v_sum = __riscv_vwredsum_vs_i16m2_i32m1(vw_mul, v_zero, vl); From a0c2dad9d43456c677e205c6240a5f8afb0121ac Mon Sep 17 00:00:00 2001 From: Jared Van Bortel Date: Sun, 18 Feb 2024 16:21:52 -0500 Subject: [PATCH 382/812] build : pass all warning flags to nvcc via -Xcompiler (#5570) * build : pass all warning flags to nvcc via -Xcompiler * make : fix apparent mis-merge from #3952 * make : fix incorrect GF_CC_VER for CUDA host compiler --- CMakeLists.txt | 11 ++++++----- Makefile | 28 ++-------------------------- scripts/get-flags.mk | 2 +- 3 files changed, 9 insertions(+), 32 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 0c29b5d09..f5e7f2980 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -778,10 +778,7 @@ endif() set(CUDA_CXX_FLAGS "") if (LLAMA_CUBLAS) - set(CUDA_FLAGS ${CXX_FLAGS} -use_fast_math) - if (NOT MSVC) - list(APPEND CUDA_FLAGS -Wno-pedantic) - endif() + set(CUDA_FLAGS -use_fast_math) if (LLAMA_ALL_WARNINGS AND NOT MSVC) set(NVCC_CMD ${CMAKE_CUDA_COMPILER} .c) @@ -814,7 +811,11 @@ if (LLAMA_CUBLAS) message("-- CUDA host compiler is ${CUDA_CCID} ${CUDA_CCVER}") get_flags(${CUDA_CCID} ${CUDA_CCVER}) - list(APPEND CUDA_CXX_FLAGS ${GF_CXX_FLAGS}) # This is passed to -Xcompiler later + list(APPEND CUDA_CXX_FLAGS ${CXX_FLAGS} ${GF_CXX_FLAGS}) # This is passed to -Xcompiler later + endif() + + if (NOT MSVC) + list(APPEND CUDA_CXX_FLAGS -Wno-pedantic) endif() endif() diff --git a/Makefile b/Makefile index 901798606..f5f6d32a7 100644 --- a/Makefile +++ b/Makefile @@ -220,30 +220,6 @@ ifeq ($(LLAMA_FATAL_WARNINGS),1) MK_CXXFLAGS += -Werror endif -ifeq ($(CC_IS_CLANG), 1) - # clang options - MK_CFLAGS += -Wunreachable-code-break -Wunreachable-code-return - MK_HOST_CXXFLAGS += -Wunreachable-code-break -Wunreachable-code-return -Wmissing-prototypes -Wextra-semi - - ifneq '' '$(and $(CC_IS_LLVM_CLANG),$(filter 1,$(shell expr $(CC_VER) \>= 030800)))' - MK_CFLAGS += -Wdouble-promotion - endif - ifneq '' '$(and $(CC_IS_APPLE_CLANG),$(filter 1,$(shell expr $(CC_VER) \>= 070300)))' - MK_CFLAGS += -Wdouble-promotion - endif -else - # gcc options - MK_CFLAGS += -Wdouble-promotion - MK_HOST_CXXFLAGS += -Wno-array-bounds - - ifeq ($(shell expr $(CC_VER) \>= 070100), 1) - MK_HOST_CXXFLAGS += -Wno-format-truncation - endif - ifeq ($(shell expr $(CC_VER) \>= 080100), 1) - MK_HOST_CXXFLAGS += -Wextra-semi - endif -endif - # this version of Apple ld64 is buggy ifneq '' '$(findstring dyld-1015.7,$(shell $(CC) $(LDFLAGS) -Wl,-v 2>&1))' MK_CPPFLAGS += -DHAVE_BUGGY_APPLE_LINKER @@ -468,7 +444,7 @@ ggml-cuda.o: ggml-cuda.cu ggml-cuda.h ifdef JETSON_EOL_MODULE_DETECT $(NVCC) -I. -Icommon -D_XOPEN_SOURCE=600 -D_GNU_SOURCE -DNDEBUG -DGGML_USE_CUBLAS -I/usr/local/cuda/include -I/opt/cuda/include -I/usr/local/cuda/targets/aarch64-linux/include -std=c++11 -O3 $(NVCCFLAGS) -Xcompiler "$(CUDA_CXXFLAGS)" -c $< -o $@ else - $(NVCC) $(BASE_CXXFLAGS) $(NVCCFLAGS) -Wno-pedantic -Xcompiler "$(CUDA_CXXFLAGS)" -c $< -o $@ + $(NVCC) $(NVCCFLAGS) -Xcompiler "$(CUDA_CXXFLAGS)" -c $< -o $@ endif # JETSON_EOL_MODULE_DETECT endif # LLAMA_CUBLAS @@ -579,7 +555,7 @@ override LDFLAGS := $(MK_LDFLAGS) $(LDFLAGS) ifdef LLAMA_CUBLAS GF_CC := $(NVCC) $(NVCCFLAGS) 2>/dev/null .c -Xcompiler include scripts/get-flags.mk -CUDA_CXXFLAGS := $(GF_CXXFLAGS) +CUDA_CXXFLAGS := $(BASE_CXXFLAGS) $(GF_CXXFLAGS) -Wno-pedantic endif # diff --git a/scripts/get-flags.mk b/scripts/get-flags.mk index 596d7ead1..a742766d1 100644 --- a/scripts/get-flags.mk +++ b/scripts/get-flags.mk @@ -1,6 +1,6 @@ ifeq '' '$(findstring clang,$(shell $(GF_CC) --version))' GF_CC_IS_GCC = 1 - GF_CC_VER := $(shell { $(GF_CC) -dumpfullversion 2>/dev/null || $(GF_CC) -dumpversion; } | awk -F. '{ printf("%02d%02d%02d", $$1, $$2, $$3) }') + GF_CC_VER := $(shell { $(GF_CC) -dumpfullversion 2>/dev/null; echo; $(GF_CC) -dumpversion; } | awk -F. '/./ { printf("%02d%02d%02d", $$1, $$2, $$3); exit }') else GF_CC_IS_CLANG = 1 ifeq '' '$(findstring Apple,$(shell $(GF_CC) --version))' From f0d1fafc029a056cd765bdae58dcaa12312e9879 Mon Sep 17 00:00:00 2001 From: bmwl Date: Sun, 18 Feb 2024 23:38:32 -0800 Subject: [PATCH 383/812] ggml : android and old glibc NUMA incompatibility bugfixes (#5557) * #ifdef out some code NUMA blocks for Android due to lack of support * added in some __ANDROID__ if def gates around numa code and forced GLIBC prior to 2.29 to use a syscall for getcpu instead of the wrapper * Changed gates on numa platform specific stuff to __gnu_linux__ to skip any platforms without glibc * harmonizing #if defined blocks for numa code to __gnu_linux__ since that's the only model that's being followed anyways --------- Co-authored-by: root --- ggml.c | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/ggml.c b/ggml.c index 8224652a9..4ee2c5e11 100644 --- a/ggml.c +++ b/ggml.c @@ -23,6 +23,9 @@ #include #include #include +#if defined(__gnu_linux__) +#include +#endif #ifdef GGML_USE_METAL #include @@ -1971,7 +1974,7 @@ struct ggml_numa_nodes { uint32_t n_nodes; uint32_t total_cpus; // hardware threads on system uint32_t current_node; // node on which main process is execting -#ifdef __linux__ +#if defined(__gnu_linux__) cpu_set_t cpuset; // cpuset from numactl #else uint32_t cpuset; // no NUMA support outside of Linux at this time. Use a portable datatype @@ -2009,7 +2012,7 @@ inline static void ggml_critical_section_end(void) { atomic_fetch_sub(&g_state_barrier, 1); } -#ifdef __linux__ +#if defined(__gnu_linux__) static cpu_set_t ggml_get_numa_affinity(void) { cpu_set_t cpuset; pthread_t thread; @@ -2031,7 +2034,7 @@ void ggml_numa_init(enum ggml_numa_strategy numa_flag) { return; } -#ifdef __linux__ +#if defined(__gnu_linux__) struct stat st; char path[256]; int rv; @@ -2063,7 +2066,13 @@ void ggml_numa_init(enum ggml_numa_strategy numa_flag) { // figure out which node we're on uint current_cpu; - int getcpu_ret = getcpu(¤t_cpu, &g_state.numa.current_node); + int getcpu_ret = 0; +#if __GLIBC__ > 2 || (__GLIBC__ == 2 && __GLIBC_MINOR__ > 28) + getcpu_ret = getcpu(¤t_cpu, &g_state.numa.current_node); +#else + // old glibc doesn't have a wrapper for this call. Fall back on direct syscall + getcpu_ret = syscall(SYS_getcpu,¤t_cpu,&g_state.numa.current_node); +#endif if (g_state.numa.n_nodes < 1 || g_state.numa.total_cpus < 1 || getcpu_ret != 0) { g_state.numa.n_nodes = 0; @@ -16734,7 +16743,7 @@ typedef pthread_t ggml_thread_t; #endif // Android's libc implementation "bionic" does not support setting affinity -#if defined(__linux__) && !defined(__BIONIC__) +#if defined(__gnu_linux__) static void set_numa_thread_affinity(int thread_n) { if (!ggml_is_numa()) { return; From 769a716e30ba1da46f709df1c00727d6869d30e7 Mon Sep 17 00:00:00 2001 From: Mirko185 Date: Mon, 19 Feb 2024 08:39:31 +0100 Subject: [PATCH 384/812] readme : update (#5572) Added 1.5-bit on README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 8c7bc2689..70866e249 100644 --- a/README.md +++ b/README.md @@ -61,7 +61,7 @@ variety of hardware - locally and in the cloud. - Plain C/C++ implementation without any dependencies - Apple silicon is a first-class citizen - optimized via ARM NEON, Accelerate and Metal frameworks - AVX, AVX2 and AVX512 support for x86 architectures -- 2-bit, 3-bit, 4-bit, 5-bit, 6-bit, and 8-bit integer quantization for faster inference and reduced memory use +- 1.5-bit, 2-bit, 3-bit, 4-bit, 5-bit, 6-bit, and 8-bit integer quantization for faster inference and reduced memory use - Custom CUDA kernels for running LLMs on NVIDIA GPUs (support for AMD GPUs via HIP) - Vulkan, SYCL, and (partial) OpenCL backend support - CPU+GPU hybrid inference to partially accelerate models larger than the total VRAM capacity From 3a9cb4ca6408c29423373dd6cd7aa78a58286c00 Mon Sep 17 00:00:00 2001 From: slaren Date: Mon, 19 Feb 2024 09:04:45 +0100 Subject: [PATCH 385/812] cuda, metal : fix nans in soft_max (#5574) * cuda : fix nans in soft_max * metal : fix nans in soft_max --------- Co-authored-by: Georgi Gerganov --- ggml-cuda.cu | 8 ++++---- ggml-metal.metal | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/ggml-cuda.cu b/ggml-cuda.cu index 933ebbc4e..eef213509 100644 --- a/ggml-cuda.cu +++ b/ggml-cuda.cu @@ -6205,7 +6205,7 @@ static __global__ void soft_max_f32(const float * x, const float * mask, const f const int ix = rowx*ncols + col; const int iy = rowy*ncols + col; - const float val = x[ix]*scale + (mask ? mask[iy] : 0.0f) + slope*pos[col]; + const float val = x[ix]*scale + (mask ? mask[iy] : 0.0f) + (pos ? slope*pos[col] : 0.0f); vals[col] = val; max_val = max(max_val, val); @@ -9170,17 +9170,17 @@ static void ggml_cuda_op_soft_max( memcpy(&max_bias, (float *) dst->op_params + 1, sizeof(float)); // positions tensor - float * src2_dd = dst_dd; // default to avoid null checks in the kernel + float * src2_dd = nullptr; cuda_pool_alloc src2_f; ggml_tensor * src2 = dst->src[2]; const bool use_src2 = src2 != nullptr; if (use_src2) { - const bool src2_on_device = use_src2 && src2->backend == GGML_BACKEND_GPU; - ggml_tensor_extra_gpu * src2_extra = use_src2 ? (ggml_tensor_extra_gpu *) src2->extra : nullptr; + const bool src2_on_device = src2->backend == GGML_BACKEND_GPU; if (src2_on_device) { + ggml_tensor_extra_gpu * src2_extra = (ggml_tensor_extra_gpu *) src2->extra; src2_dd = (float *) src2_extra->data_device[g_main_device]; } else { src2_dd = src2_f.alloc(ggml_nelements(src2)); diff --git a/ggml-metal.metal b/ggml-metal.metal index d0a85a192..f0d77d446 100644 --- a/ggml-metal.metal +++ b/ggml-metal.metal @@ -392,7 +392,7 @@ kernel void kernel_soft_max( float lmax = -INFINITY; for (int i00 = tpitg; i00 < ne00; i00 += ntg) { - lmax = MAX(lmax, psrc0[i00]*scale + (pmask ? pmask[i00] : 0.0f) + slope*ppos[i00]); + lmax = MAX(lmax, psrc0[i00]*scale + (pmask ? pmask[i00] : 0.0f) + (ppos ? slope*ppos[i00] : 0.0f)); } // find the max value in the block @@ -417,7 +417,7 @@ kernel void kernel_soft_max( // parallel sum float lsum = 0.0f; for (int i00 = tpitg; i00 < ne00; i00 += ntg) { - const float exp_psrc0 = exp((psrc0[i00]*scale + (pmask ? pmask[i00] : 0.0f) + slope*ppos[i00]) - max_val); + const float exp_psrc0 = exp((psrc0[i00]*scale + (pmask ? pmask[i00] : 0.0f) + (ppos ? slope*ppos[i00] : 0.0f)) - max_val); lsum += exp_psrc0; pdst[i00] = exp_psrc0; } @@ -495,7 +495,7 @@ kernel void kernel_soft_max_4( float4 lmax4 = -INFINITY; for (int i00 = tpitg; i00 < ne00/4; i00 += ntg) { - lmax4 = fmax(lmax4, psrc4[i00]*scale + (pmask ? pmask[i00] : 0.0f) + slope*ppos[i00]); + lmax4 = fmax(lmax4, psrc4[i00]*scale + (pmask ? pmask[i00] : 0.0f) + (ppos ? slope*ppos[i00] : 0.0f)); } const float lmax = MAX(MAX(lmax4[0], lmax4[1]), MAX(lmax4[2], lmax4[3])); @@ -521,7 +521,7 @@ kernel void kernel_soft_max_4( // parallel sum float4 lsum4 = 0.0f; for (int i00 = tpitg; i00 < ne00/4; i00 += ntg) { - const float4 exp_psrc4 = exp((psrc4[i00]*scale + (pmask ? pmask[i00] : 0.0f) + slope*ppos[i00]) - max_val); + const float4 exp_psrc4 = exp((psrc4[i00]*scale + (pmask ? pmask[i00] : 0.0f) + (ppos ? slope*ppos[i00] : 0.0f)) - max_val); lsum4 += exp_psrc4; pdst4[i00] = exp_psrc4; } From 11b12de39bd787c0494da0cd405958fdfedc29c4 Mon Sep 17 00:00:00 2001 From: Xuan Son Nguyen Date: Mon, 19 Feb 2024 09:23:37 +0100 Subject: [PATCH 386/812] llama : add llama_chat_apply_template() (#5538) * llama: add llama_chat_apply_template * test-chat-template: remove dedundant vector * chat_template: do not use std::string for buffer * add clarification for llama_chat_apply_template * llama_chat_apply_template: add zephyr template * llama_chat_apply_template: correct docs * llama_chat_apply_template: use term "chat" everywhere * llama_chat_apply_template: change variable name to "tmpl" --- Makefile | 4 ++ llama.cpp | 117 +++++++++++++++++++++++++++++++++++ llama.h | 25 ++++++++ tests/CMakeLists.txt | 1 + tests/test-chat-template.cpp | 64 +++++++++++++++++++ 5 files changed, 211 insertions(+) create mode 100644 tests/test-chat-template.cpp diff --git a/Makefile b/Makefile index f5f6d32a7..59352eb53 100644 --- a/Makefile +++ b/Makefile @@ -867,3 +867,7 @@ tests/test-model-load-cancel: tests/test-model-load-cancel.cpp ggml.o llama.o te tests/test-autorelease: tests/test-autorelease.cpp ggml.o llama.o tests/get-model.cpp $(COMMON_DEPS) $(OBJS) $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) + +tests/test-chat-template: tests/test-chat-template.cpp ggml.o llama.o $(COMMON_DEPS) $(OBJS) + $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) + $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) diff --git a/llama.cpp b/llama.cpp index 5cfebb3b1..143870645 100644 --- a/llama.cpp +++ b/llama.cpp @@ -12508,6 +12508,123 @@ int32_t llama_token_to_piece(const struct llama_model * model, llama_token token return 0; } +// trim whitespace from the beginning and end of a string +static std::string trim(const std::string & str) { + size_t start = 0; + size_t end = str.size(); + while (start < end && isspace(str[start])) { + start += 1; + } + while (end > start && isspace(str[end - 1])) { + end -= 1; + } + return str.substr(start, end - start); +} + +// Simple version of "llama_apply_chat_template" that only works with strings +// This function uses heuristic checks to determine commonly used template. It is not a jinja parser. +static int32_t llama_chat_apply_template_internal( + const std::string & tmpl, + const std::vector & chat, + std::string & dest, bool add_ass) { + // Taken from the research: https://github.com/ggerganov/llama.cpp/issues/5527 + std::stringstream ss; + if (tmpl.find("<|im_start|>") != std::string::npos) { + // chatml template + for (auto message : chat) { + ss << "<|im_start|>" << message->role << "\n" << message->content << "<|im_end|>\n"; + } + if (add_ass) { + ss << "<|im_start|>assistant\n"; + } + } else if (tmpl.find("[INST]") != std::string::npos) { + // llama2 template and its variants + // [variant] support system message + bool support_system_message = tmpl.find("<>") != std::string::npos; + // [variant] space before + after response + bool space_around_response = tmpl.find("' ' + eos_token") != std::string::npos; + // [variant] add BOS inside history + bool add_bos_inside_history = tmpl.find("bos_token + '[INST]") != std::string::npos; + // [variant] trim spaces from the input message + bool strip_message = tmpl.find("content.strip()") != std::string::npos; + // construct the prompt + bool is_inside_turn = true; // skip BOS at the beginning + ss << "[INST] "; + for (auto message : chat) { + std::string content = strip_message ? trim(message->content) : message->content; + std::string role(message->role); + if (!is_inside_turn) { + is_inside_turn = true; + ss << (add_bos_inside_history ? "[INST] " : "[INST] "); + } + if (role == "system") { + if (support_system_message) { + ss << "<>\n" << content << "\n<>\n\n"; + } else { + // if the model does not support system message, we still include it in the first message, but without <> + ss << content << "\n"; + } + } else if (role == "user") { + ss << content << " [/INST]"; + } else { + ss << (space_around_response ? " " : "") << content << (space_around_response ? " " : "") << ""; + is_inside_turn = false; + } + } + // llama2 templates seem to not care about "add_generation_prompt" + } else if (tmpl.find("<|user|>") != std::string::npos) { + // zephyr template + for (auto message : chat) { + ss << "<|" << message->role << "|>" << "\n" << message->content << "<|endoftext|>\n"; + } + if (add_ass) { + ss << "<|assistant|>\n"; + } + } else { + // template not supported + return -1; + } + dest = ss.str(); + return dest.size(); +} + +LLAMA_API int32_t llama_chat_apply_template( + const struct llama_model * model, + const char * tmpl, + const struct llama_chat_message * chat, + size_t n_msg, + bool add_ass, + char * buf, + int32_t length) { + std::string curr_tmpl(tmpl == nullptr ? "" : tmpl); + if (tmpl == nullptr) { + GGML_ASSERT(model != nullptr); + // load template from model + std::vector model_template(2048, 0); // longest known template is about 1200 bytes + std::string template_key = "tokenizer.chat_template"; + int32_t res = llama_model_meta_val_str(model, template_key.c_str(), model_template.data(), curr_tmpl.size()); + if (res < 0) { + // worst case: there is no information about template, we will use chatml by default + curr_tmpl = "<|im_start|>"; // see llama_chat_apply_template_internal + } else { + curr_tmpl = std::string(model_template.data(), model_template.size()); + } + } + // format the chat to string + std::vector chat_vec; + chat_vec.resize(n_msg); + for (size_t i = 0; i < n_msg; i++) { + chat_vec[i] = &chat[i]; + } + std::string formatted_chat; + int32_t res = llama_chat_apply_template_internal(curr_tmpl, chat_vec, formatted_chat, add_ass); + if (res < 0) { + return res; + } + strncpy(buf, formatted_chat.c_str(), length); + return res; +} + struct llama_timings llama_get_timings(struct llama_context * ctx) { struct llama_timings result = { /*.t_start_ms =*/ 1e-3 * ctx->t_start_us, diff --git a/llama.h b/llama.h index 5a97abcc9..77a84c18a 100644 --- a/llama.h +++ b/llama.h @@ -305,6 +305,12 @@ extern "C" { int32_t n_eval; }; + // used in chat template + typedef struct llama_chat_message { + const char * role; + const char * content; + } llama_chat_message; + // Helpers for getting default parameters LLAMA_API struct llama_model_params llama_model_default_params(void); LLAMA_API struct llama_context_params llama_context_default_params(void); @@ -699,6 +705,25 @@ extern "C" { char * buf, int32_t length); + /// Apply chat template. Inspired by hf apply_chat_template() on python. + /// Both "model" and "custom_template" are optional, but at least one is required. "custom_template" has higher precedence than "model" + /// NOTE: This function only support some known jinja templates. It is not a jinja parser. + /// @param tmpl A Jinja template to use for this chat. If this is nullptr, the model’s default chat template will be used instead. + /// @param chat Pointer to a list of multiple llama_chat_message + /// @param n_msg Number of llama_chat_message in this chat + /// @param add_ass Whether to end the prompt with the token(s) that indicate the start of an assistant message. + /// @param buf A buffer to hold the output formatted prompt. The recommended alloc size is 2 * (total number of characters of all messages) + /// @param length The size of the allocated buffer + /// @return The total number of bytes of the formatted prompt. If is it larger than the size of buffer, you may need to re-alloc it and then re-apply the template. + LLAMA_API int32_t llama_chat_apply_template( + const struct llama_model * model, + const char * tmpl, + const struct llama_chat_message * chat, + size_t n_msg, + bool add_ass, + char * buf, + int32_t length); + // // Grammar // diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 3e40a78cd..10326d531 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -28,6 +28,7 @@ endfunction() llama_build_and_test_executable(test-quantize-fns.cpp) llama_build_and_test_executable(test-quantize-perf.cpp) llama_build_and_test_executable(test-sampling.cpp) +llama_build_and_test_executable(test-chat-template.cpp) llama_build_executable(test-tokenizer-0-llama.cpp) llama_test_executable (test-tokenizer-0-llama test-tokenizer-0-llama.cpp ${CMAKE_CURRENT_SOURCE_DIR}/../models/ggml-vocab-llama.gguf) diff --git a/tests/test-chat-template.cpp b/tests/test-chat-template.cpp new file mode 100644 index 000000000..9830650d4 --- /dev/null +++ b/tests/test-chat-template.cpp @@ -0,0 +1,64 @@ +#include +#include +#include +#include + +#undef NDEBUG +#include + +#include "llama.h" + +int main(void) { + llama_chat_message conversation[] = { + {"system", "You are a helpful assistant"}, + {"user", "Hello"}, + {"assistant", "Hi there"}, + {"user", "Who are you"}, + {"assistant", " I am an assistant "}, + {"user", "Another question"}, + }; + size_t message_count = 6; + std::vector templates = { + // teknium/OpenHermes-2.5-Mistral-7B + "{% for message in messages %}{{'<|im_start|>' + message['role'] + '\\n' + message['content'] + '<|im_end|>' + '\\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\\n' }}{% endif %}", + // mistralai/Mistral-7B-Instruct-v0.2 + "{{ bos_token }}{% for message in messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if message['role'] == 'user' %}{{ '[INST] ' + message['content'] + ' [/INST]' }}{% elif message['role'] == 'assistant' %}{{ message['content'] + eos_token}}{% else %}{{ raise_exception('Only user and assistant roles are supported!') }}{% endif %}{% endfor %}", + // TheBloke/FusionNet_34Bx2_MoE-AWQ + "{%- for idx in range(0, messages|length) -%}\\n{%- if messages[idx]['role'] == 'user' -%}\\n{%- if idx > 1 -%}\\n{{- bos_token + '[INST] ' + messages[idx]['content'] + ' [/INST]' -}}\\n{%- else -%}\\n{{- messages[idx]['content'] + ' [/INST]' -}}\\n{%- endif -%}\\n{% elif messages[idx]['role'] == 'system' %}\\n{{- '[INST] <>\\\\n' + messages[idx]['content'] + '\\\\n<>\\\\n\\\\n' -}}\\n{%- elif messages[idx]['role'] == 'assistant' -%}\\n{{- ' ' + messages[idx]['content'] + ' ' + eos_token -}}\\n{% endif %}\\n{% endfor %}", + // bofenghuang/vigogne-2-70b-chat + "{{ bos_token }}{% if messages[0]['role'] == 'system' %}{% set loop_messages = messages[1:] %}{% set system_message = messages[0]['content'] %}{% elif true == true and not '<>' in messages[0]['content'] %}{% set loop_messages = messages %}{% set system_message = 'Vous êtes Vigogne, un assistant IA créé par Zaion Lab. Vous suivez extrêmement bien les instructions. Aidez autant que vous le pouvez.' %}{% else %}{% set loop_messages = messages %}{% set system_message = false %}{% endif %}{% for message in loop_messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if loop.index0 == 0 and system_message != false %}{% set content = '<>\\\\n' + system_message + '\\\\n<>\\\\n\\\\n' + message['content'] %}{% else %}{% set content = message['content'] %}{% endif %}{% if message['role'] == 'user' %}{{ '[INST] ' + content.strip() + ' [/INST]' }}{% elif message['role'] == 'system' %}{{ '<>\\\\n' + content.strip() + '\\\\n<>\\\\n\\\\n' }}{% elif message['role'] == 'assistant' %}{{ ' ' + content.strip() + ' ' + eos_token }}{% endif %}{% endfor %}", + }; + std::vector expected_substr = { + "<|im_start|>assistant\n I am an assistant <|im_end|>\n<|im_start|>user\nAnother question<|im_end|>\n<|im_start|>assistant", + "[/INST]Hi there[INST] Who are you [/INST] I am an assistant [INST] Another question [/INST]", + "[INST] Who are you [/INST] I am an assistant [INST] Another question [/INST]", + "[/INST] Hi there [INST] Who are you [/INST] I am an assistant [INST] Another question [/INST]", + }; + std::vector formatted_chat(1024); + int32_t res; + + // test invalid chat template + res = llama_chat_apply_template(nullptr, "INVALID TEMPLATE", conversation, message_count, true, formatted_chat.data(), formatted_chat.size()); + assert(res < 0); + + for (size_t i = 0; i < templates.size(); i++) { + std::string custom_template = templates[i]; + std::string substr = expected_substr[i]; + formatted_chat.resize(1024); + res = llama_chat_apply_template( + nullptr, + custom_template.c_str(), + conversation, + message_count, + true, + formatted_chat.data(), + formatted_chat.size() + ); + formatted_chat.resize(res); + std::string output(formatted_chat.data(), formatted_chat.size()); + std::cout << output << "\n-------------------------\n"; + // expect the "formatted_chat" to contain pre-defined strings + assert(output.find(substr) != std::string::npos); + } + return 0; +} From 4480542b2271ba1438f0daff8e5f3a74b1dc8609 Mon Sep 17 00:00:00 2001 From: NawafAlansari <72708095+NawafAlansari@users.noreply.github.com> Date: Mon, 19 Feb 2024 03:25:38 -0500 Subject: [PATCH 387/812] baby-llama : allocate graphs in ggml_context (#5573) * Fixed the baby-llama issue (see issue #4830) * minor : fix whitespaces --------- Co-authored-by: Georgi Gerganov --- examples/baby-llama/baby-llama.cpp | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/examples/baby-llama/baby-llama.cpp b/examples/baby-llama/baby-llama.cpp index e7d2ad592..65bb238a0 100644 --- a/examples/baby-llama/baby-llama.cpp +++ b/examples/baby-llama/baby-llama.cpp @@ -1533,16 +1533,17 @@ int main(int argc, char ** argv) { int n_past = 0; - ggml_cgraph gf = {}; + struct ggml_cgraph * gf = NULL; + gf = ggml_new_graph_custom(ctx0, LLAMA_TRAIN_MAX_NODES, true); get_example_targets_batch(ctx0, 64*ex+0, tokens_input, targets); - struct ggml_tensor * logits = forward_batch(&model, &kv_self, ctx0, &gf, tokens_input, n_tokens, n_past, n_batch); + struct ggml_tensor * logits = forward_batch(&model, &kv_self, ctx0, gf, tokens_input, n_tokens, n_past, n_batch); // struct ggml_tensor * e = cross_entropy_loss(ctx0, targets, logits); struct ggml_tensor * e = square_error_loss(ctx0, targets, logits); - ggml_build_forward_expand(&gf, e); - ggml_graph_compute_helper(work_buffer, &gf, /*n_threads*/ 1); + ggml_build_forward_expand(gf, e); + ggml_graph_compute_helper(work_buffer, gf, /*n_threads*/ 1); float error_before_opt = ggml_get_f32_1d(e, 0); @@ -1552,8 +1553,8 @@ int main(int argc, char ** argv) { opt_params_lbfgs.lbfgs.n_iter = 16; ggml_opt(ctx0, opt_params_lbfgs, e); // - ggml_build_forward_expand(&gf, e); - ggml_graph_compute_helper(work_buffer, &gf, /*n_threads*/ 1); + ggml_build_forward_expand(gf, e); + ggml_graph_compute_helper(work_buffer, gf, /*n_threads*/ 1); float error_after_opt = ggml_get_f32_1d(e, 0); @@ -1600,13 +1601,14 @@ int main(int argc, char ** argv) { }; struct ggml_context * ctx0 = ggml_init(params); - ggml_cgraph gf = {}; + struct ggml_cgraph * gf = NULL; + gf = ggml_new_graph_custom(ctx0, LLAMA_TRAIN_MAX_NODES, true); int n_past = 0; - struct ggml_tensor * logits = forward(&model, &kv_self, ctx0, &gf, tokens_input, sample_ctx, n_past); + struct ggml_tensor * logits = forward(&model, &kv_self, ctx0, gf, tokens_input, sample_ctx, n_past); - ggml_build_forward_expand(&gf, logits); - ggml_graph_compute_helper(work_buffer, &gf, /*n_threads*/ 1); + ggml_build_forward_expand(gf, logits); + ggml_graph_compute_helper(work_buffer, gf, /*n_threads*/ 1); struct ggml_tensor * best_samples = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, sample_ctx); struct ggml_tensor * probs = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_vocab, sample_ctx); From 70847553963c85e86051d06df848236829f5f951 Mon Sep 17 00:00:00 2001 From: Daniel Bevenius Date: Mon, 19 Feb 2024 09:31:59 +0100 Subject: [PATCH 388/812] llava : avoid changing the original BakLLaVA model (#5577) This is a follup of Commit fc0c8d286a533363a9a663510b62af85ffad58b3 ("llava : update surgery script to not remove tensors") but this time the change is to the BakLLaVA specific part of the surgery script. I've been able to test this using SkunkworksAI/BakLLaVA-1 and it works as expected using the instructions in README.md. Signed-off-by: Daniel Bevenius --- examples/llava/llava-surgery.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/examples/llava/llava-surgery.py b/examples/llava/llava-surgery.py index 8b7a62fba..4f2da3bee 100644 --- a/examples/llava/llava-surgery.py +++ b/examples/llava/llava-surgery.py @@ -25,9 +25,6 @@ if len(clip_tensors) > 0: clip = {name.replace("vision_tower.vision_tower.", ""): checkpoint[name].float() for name in clip_tensors} torch.save(clip, f"{args.model}/llava.clip") - # remove these tensors - for name in clip_tensors: - del checkpoint[name] # added tokens should be removed to be able to convert Mistral models if os.path.exists(f"{args.model}/added_tokens.json"): @@ -35,7 +32,6 @@ if len(clip_tensors) > 0: f.write("{}\n") - torch.save(checkpoint, path) print("Done!") print(f"Now you can convert {args.model} to a regular LLaMA GGUF file.") From f53119cec4f073b6d214195ecbe1fad3abdf2b34 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Mon, 19 Feb 2024 10:34:10 +0200 Subject: [PATCH 389/812] minor : fix trailing whitespace (#5538) --- llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llama.cpp b/llama.cpp index 143870645..5de07dfa9 100644 --- a/llama.cpp +++ b/llama.cpp @@ -12525,7 +12525,7 @@ static std::string trim(const std::string & str) { // This function uses heuristic checks to determine commonly used template. It is not a jinja parser. static int32_t llama_chat_apply_template_internal( const std::string & tmpl, - const std::vector & chat, + const std::vector & chat, std::string & dest, bool add_ass) { // Taken from the research: https://github.com/ggerganov/llama.cpp/issues/5527 std::stringstream ss; From 13e2c771aa4212cd5405cf310203848d50f7f859 Mon Sep 17 00:00:00 2001 From: Abhilash Majumder <30946547+abhilash1910@users.noreply.github.com> Date: Mon, 19 Feb 2024 14:45:18 +0530 Subject: [PATCH 390/812] cmake : remove obsolete sycl compile flags (#5581) * rm unwanted sycl compile options * fix bug * fix bug * format fix --- CMakeLists.txt | 7 ------- 1 file changed, 7 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index f5e7f2980..40a098d01 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -741,13 +741,6 @@ function(get_flags CCID CCVER) if (CCVER VERSION_GREATER_EQUAL 8.1.0) list(APPEND CXX_FLAGS -Wextra-semi) endif() - elseif (CCID MATCHES "Intel") - if (NOT LLAMA_SYCL) - # enable max optimization level when using Intel compiler - set(C_FLAGS -ipo -O3 -static -fp-model=fast -flto -fno-stack-protector) - set(CXX_FLAGS -ipo -O3 -static -fp-model=fast -flto -fno-stack-protector) - add_link_options(-fuse-ld=lld -static-intel) - endif() endif() set(GF_C_FLAGS ${C_FLAGS} PARENT_SCOPE) From 70d45af0efce9ed360e1858b827989d971dd9caf Mon Sep 17 00:00:00 2001 From: valiray <133289098+valiray@users.noreply.github.com> Date: Mon, 19 Feb 2024 02:37:10 -0800 Subject: [PATCH 391/812] readme : fix typo in README-sycl.md (#5353) --- README-sycl.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README-sycl.md b/README-sycl.md index e3a8e726e..dd5bf9dea 100644 --- a/README-sycl.md +++ b/README-sycl.md @@ -272,7 +272,7 @@ Please install [Visual Studio](https://visualstudio.microsoft.com/) which impact a. Please follow the procedure in [Get the Intel® oneAPI Base Toolkit ](https://www.intel.com/content/www/us/en/developer/tools/oneapi/base-toolkit.html). -Recommend to install to default folder: **/opt/intel/oneapi**. +Recommend to install to default folder: **C:\Program Files (x86)\Intel\oneAPI**. Following guide uses the default folder as example. If you use other folder, please modify the following guide info with your folder. From 68a6b98b3c8af7e5baade3ee45fe1d2c7b9323a9 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Mon, 19 Feb 2024 13:41:51 +0200 Subject: [PATCH 392/812] make : fix CUDA build (#5580) --- Makefile | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 59352eb53..29fd2ca9c 100644 --- a/Makefile +++ b/Makefile @@ -97,9 +97,10 @@ endif # # keep standard at C11 and C++11 -MK_CPPFLAGS = -I. -Icommon -MK_CFLAGS = -std=c11 -fPIC -MK_CXXFLAGS = -std=c++11 -fPIC +MK_CPPFLAGS = -I. -Icommon +MK_CFLAGS = -std=c11 -fPIC +MK_CXXFLAGS = -std=c++11 -fPIC +MK_NVCCFLAGS = -std=c++11 # -Ofast tends to produce faster code, but may not be available for some compilers. ifdef LLAMA_FAST From d0e3ce51f45bd6a646da1952d7e5d143a087db3e Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Mon, 19 Feb 2024 14:45:41 +0200 Subject: [PATCH 393/812] ci : enable -Werror for CUDA builds (#5579) * cmake : pass -Werror through -Xcompiler ggml-ci * make, cmake : enable CUDA errors on warnings ggml-ci --- CMakeLists.txt | 33 +++++++++++++++++++-------------- Makefile | 5 ++++- ggml-cuda.cu | 50 ++++++++++++++++++++++++++------------------------ 3 files changed, 49 insertions(+), 39 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 40a098d01..168b133f4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -145,14 +145,6 @@ set(THREADS_PREFER_PTHREAD_FLAG ON) find_package(Threads REQUIRED) include(CheckCXXCompilerFlag) -if (LLAMA_FATAL_WARNINGS) - if (CMAKE_CXX_COMPILER_ID MATCHES "GNU" OR CMAKE_CXX_COMPILER_ID MATCHES "Clang") - add_compile_options(-Werror) - elseif (CMAKE_CXX_COMPILER_ID STREQUAL "MSVC") - add_compile_options(/WX) - endif() -endif() - # enable libstdc++ assertions for debug builds if (CMAKE_SYSTEM_NAME MATCHES "Linux") add_compile_definitions($<$:_GLIBCXX_ASSERTIONS>) @@ -747,15 +739,24 @@ function(get_flags CCID CCVER) set(GF_CXX_FLAGS ${CXX_FLAGS} PARENT_SCOPE) endfunction() +if (LLAMA_FATAL_WARNINGS) + if (CMAKE_CXX_COMPILER_ID MATCHES "GNU" OR CMAKE_CXX_COMPILER_ID MATCHES "Clang") + list(APPEND C_FLAGS -Werror) + list(APPEND CXX_FLAGS -Werror) + elseif (CMAKE_CXX_COMPILER_ID STREQUAL "MSVC") + add_compile_options(/WX) + endif() +endif() + if (LLAMA_ALL_WARNINGS) if (NOT MSVC) - set(WARNING_FLAGS -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function) - set(C_FLAGS -Wshadow -Wstrict-prototypes -Wpointer-arith -Wmissing-prototypes - -Werror=implicit-int -Werror=implicit-function-declaration) - set(CXX_FLAGS -Wmissing-declarations -Wmissing-noreturn) + list(APPEND WARNING_FLAGS -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function) + list(APPEND C_FLAGS -Wshadow -Wstrict-prototypes -Wpointer-arith -Wmissing-prototypes + -Werror=implicit-int -Werror=implicit-function-declaration) + list(APPEND CXX_FLAGS -Wmissing-declarations -Wmissing-noreturn) - set(C_FLAGS ${WARNING_FLAGS} ${C_FLAGS}) - set(CXX_FLAGS ${WARNING_FLAGS} ${CXX_FLAGS}) + list(APPEND C_FLAGS ${WARNING_FLAGS}) + list(APPEND CXX_FLAGS ${WARNING_FLAGS}) get_flags(${CMAKE_CXX_COMPILER_ID} ${CMAKE_CXX_COMPILER_VERSION}) @@ -773,6 +774,10 @@ set(CUDA_CXX_FLAGS "") if (LLAMA_CUBLAS) set(CUDA_FLAGS -use_fast_math) + if (LLAMA_FATAL_WARNINGS) + list(APPEND CUDA_FLAGS -Werror all-warnings) + endif() + if (LLAMA_ALL_WARNINGS AND NOT MSVC) set(NVCC_CMD ${CMAKE_CUDA_COMPILER} .c) if (NOT CMAKE_CUDA_HOST_COMPILER STREQUAL "") diff --git a/Makefile b/Makefile index 29fd2ca9c..63b4af9ba 100644 --- a/Makefile +++ b/Makefile @@ -217,7 +217,7 @@ MK_CFLAGS += $(WARN_FLAGS) -Wshadow -Wstrict-prototypes -Wpointer-arith -Wmis MK_CXXFLAGS += $(WARN_FLAGS) -Wmissing-declarations -Wmissing-noreturn ifeq ($(LLAMA_FATAL_WARNINGS),1) - MK_CFLAGS += -Werror + MK_CFLAGS += -Werror MK_CXXFLAGS += -Werror endif @@ -385,6 +385,9 @@ ifdef LLAMA_CUBLAS MK_LDFLAGS += -lcuda -lcublas -lculibos -lcudart -lcublasLt -lpthread -ldl -lrt -L/usr/local/cuda/lib64 -L/opt/cuda/lib64 -L$(CUDA_PATH)/targets/x86_64-linux/lib -L/usr/local/cuda/targets/aarch64-linux/lib -L/usr/lib/wsl/lib OBJS += ggml-cuda.o MK_NVCCFLAGS += -use_fast_math +ifdef LLAMA_FATAL_WARNINGS + MK_NVCCFLAGS += -Werror all-warnings +endif # LLAMA_FATAL_WARNINGS ifndef JETSON_EOL_MODULE_DETECT MK_NVCCFLAGS += --forward-unknown-to-host-compiler endif # JETSON_EOL_MODULE_DETECT diff --git a/ggml-cuda.cu b/ggml-cuda.cu index eef213509..e091dbdc1 100644 --- a/ggml-cuda.cu +++ b/ggml-cuda.cu @@ -651,18 +651,18 @@ static __device__ __forceinline__ float2 warp_reduce_sum(float2 a) { return a; } -static __device__ __forceinline__ half2 warp_reduce_sum(half2 a) { -#if !(defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)) && __CUDA_ARCH__ >= CC_PASCAL -#pragma unroll - for (int mask = 16; mask > 0; mask >>= 1) { - a = __hadd2(a, __shfl_xor_sync(0xffffffff, a, mask, 32)); - } - return a; -#else - (void) a; - NO_DEVICE_CODE; -#endif // !(defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)) && __CUDA_ARCH__ >= CC_PASCAL -} +//static __device__ __forceinline__ half2 warp_reduce_sum(half2 a) { +//#if !(defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)) && __CUDA_ARCH__ >= CC_PASCAL +//#pragma unroll +// for (int mask = 16; mask > 0; mask >>= 1) { +// a = __hadd2(a, __shfl_xor_sync(0xffffffff, a, mask, 32)); +// } +// return a; +//#else +// (void) a; +// NO_DEVICE_CODE; +//#endif // !(defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)) && __CUDA_ARCH__ >= CC_PASCAL +//} static __device__ __forceinline__ float warp_reduce_max(float x) { #pragma unroll @@ -672,18 +672,18 @@ static __device__ __forceinline__ float warp_reduce_max(float x) { return x; } -static __device__ __forceinline__ half2 warp_reduce_max(half2 x) { -#if !(defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)) && __CUDA_ARCH__ >= CC_PASCAL && CUDART_VERSION >= CUDART_HMAX -#pragma unroll - for (int mask = 16; mask > 0; mask >>= 1) { - x = __hmax2(x, __shfl_xor_sync(0xffffffff, x, mask, 32)); - } - return x; -#else - (void) x; - NO_DEVICE_CODE; -#endif // !(defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)) && __CUDA_ARCH__ >= CC_PASCAL && CUDART_VERSION >= CUDART_HMAX -} +//static __device__ __forceinline__ half2 warp_reduce_max(half2 x) { +//#if !(defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)) && __CUDA_ARCH__ >= CC_PASCAL && CUDART_VERSION >= CUDART_HMAX +//#pragma unroll +// for (int mask = 16; mask > 0; mask >>= 1) { +// x = __hmax2(x, __shfl_xor_sync(0xffffffff, x, mask, 32)); +// } +// return x; +//#else +// (void) x; +// NO_DEVICE_CODE; +//#endif // !(defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)) && __CUDA_ARCH__ >= CC_PASCAL && CUDART_VERSION >= CUDART_HMAX +//} static __device__ __forceinline__ float op_repeat(const float a, const float b) { return b; @@ -4641,10 +4641,12 @@ static __device__ __forceinline__ float vec_dot_iq2_xs_q8_1( const float d = (float)bq2->d * __low2float(bq8_1[ib32].ds) * 0.25f; return d * ((0.5f + ls1) * sumi1 + (0.5f + ls2) * sumi2); #else + (void) ksigns64; assert(false); return 0.f; #endif #else + (void) ksigns64; assert(false); return 0.f; #endif From 890559ab28e354052e16e770155ad007fd0856e8 Mon Sep 17 00:00:00 2001 From: Didzis Gosko Date: Sun, 11 Feb 2024 16:41:41 +0200 Subject: [PATCH 394/812] metal : option to embed MSL source into compiled binary (whisper/1842) * ggml : embed Metal library source (ggml-metal.metal) into binary enable by setting WHISPER_EMBED_METAL_LIBRARY * rename the build option * rename the preprocessor directive * generate Metal library embedding assembly on-fly during build process --- ggml-metal.m | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/ggml-metal.m b/ggml-metal.m index f3c1fff8f..956e323a0 100644 --- a/ggml-metal.m +++ b/ggml-metal.m @@ -277,6 +277,14 @@ static struct ggml_metal_context * ggml_metal_init(int n_cb) { return NULL; } } else { +#if GGML_METAL_EMBED_LIBRARY + GGML_METAL_LOG_INFO("%s: using embedded metal library\n", __func__); + + extern const char ggml_metallib_start[]; + extern const char ggml_metallib_end[]; + + NSString * src = [[NSString alloc] initWithBytes:ggml_metallib_start length:(ggml_metallib_end-ggml_metallib_start) encoding:NSUTF8StringEncoding]; +#else GGML_METAL_LOG_INFO("%s: default.metallib not found, loading from source\n", __func__); NSString * sourcePath; @@ -299,6 +307,7 @@ static struct ggml_metal_context * ggml_metal_init(int n_cb) { GGML_METAL_LOG_ERROR("%s: error: %s\n", __func__, [[error description] UTF8String]); return NULL; } +#endif @autoreleasepool { // dictionary of preprocessor macros From a3145bdc305422973e25f0b066da6f469ed5dc45 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Mon, 19 Feb 2024 14:53:48 +0200 Subject: [PATCH 395/812] ggml-alloc : apply ggml/731 --- ggml-alloc.c | 116 ++++++++++++++++++++++++++++++++------------------- 1 file changed, 73 insertions(+), 43 deletions(-) diff --git a/ggml-alloc.c b/ggml-alloc.c index d4123564f..e675306c8 100644 --- a/ggml-alloc.c +++ b/ggml-alloc.c @@ -377,6 +377,9 @@ struct ggml_gallocr { struct node_alloc * node_allocs; // [n_nodes] int n_nodes; + + struct tensor_alloc * leaf_allocs; // [n_leafs] + int n_leafs; }; ggml_gallocr_t ggml_gallocr_new_n(ggml_backend_buffer_type_t * bufts, int n_bufs) { @@ -427,6 +430,7 @@ void ggml_gallocr_free(ggml_gallocr_t galloc) { free(galloc->buffers); free(galloc->buf_tallocs); free(galloc->node_allocs); + free(galloc->leaf_allocs); free(galloc); } @@ -464,7 +468,7 @@ static void ggml_gallocr_allocate_node(ggml_gallocr_t galloc, struct ggml_tensor for (int i = 0; i < GGML_MAX_SRC; i++) { struct ggml_tensor * parent = node->src[i]; if (parent == NULL) { - break; + continue; } // if the node's data is external, then we cannot re-use it @@ -544,22 +548,8 @@ static void ggml_gallocr_alloc_graph_impl(ggml_gallocr_t galloc, struct ggml_cgr memset(galloc->hash_set.keys, 0, galloc->hash_set.size * sizeof(struct ggml_tensor *)); memset(galloc->hash_values, 0, galloc->hash_set.size * sizeof(struct hash_node)); - // allocate all graph inputs first to avoid overwriting them - for (int i = 0; i < graph->n_nodes; i++) { - if (graph->nodes[i]->flags & GGML_TENSOR_FLAG_INPUT) { - ggml_gallocr_allocate_node(galloc, graph->nodes[i], get_node_buffer_id(node_buffer_ids, i)); - } - for (int j = 0; j < GGML_MAX_SRC; j++) { - if (graph->nodes[i]->src[j] == NULL) { - continue; - } - if (graph->nodes[i]->src[j]->flags & GGML_TENSOR_FLAG_INPUT) { - ggml_gallocr_allocate_node(galloc, graph->nodes[i]->src[j], get_node_buffer_id(node_buffer_ids, i)); - } - } - } - // count number of children and views + // allocate all graph inputs and leafs first to avoid overwriting them for (int i = 0; i < graph->n_nodes; i++) { struct ggml_tensor * node = graph->nodes[i]; @@ -568,14 +558,37 @@ static void ggml_gallocr_alloc_graph_impl(ggml_gallocr_t galloc, struct ggml_cgr ggml_gallocr_hash_get(galloc, view_src)->n_views += 1; } - for (int j = 0; j < GGML_MAX_SRC; j++) { - struct ggml_tensor * parent = node->src[j]; - if (parent == NULL) { - break; - } - ggml_gallocr_hash_get(galloc, parent)->n_children += 1; + if (node->flags & GGML_TENSOR_FLAG_INPUT) { + ggml_gallocr_allocate_node(galloc, graph->nodes[i], get_node_buffer_id(node_buffer_ids, i)); } - } + + for (int j = 0; j < GGML_MAX_SRC; j++) { + struct ggml_tensor * src = node->src[j]; + if (src == NULL) { + continue; + } + + ggml_gallocr_hash_get(galloc, src)->n_children += 1; + + // allocate explicit inputs and leafs + if (src->flags & GGML_TENSOR_FLAG_INPUT || src->op == GGML_OP_NONE) { + ggml_gallocr_allocate_node(galloc, src, get_node_buffer_id(node_buffer_ids, i)); + } + } + } + + // allocate the remaining leafs that are unused on the graph + // these are effectively static tensors that the application is not using in the graph, but may still want to allocate for other purposes + for (int i = 0; i < graph->n_leafs; i++) { + struct ggml_tensor * leaf = graph->leafs[i]; + struct hash_node * hn = ggml_gallocr_hash_get(galloc, leaf); + + if (hn->n_children == 0) { + assert(!hn->allocated); + // since buffer ids are only given for nodes, these leafs are always allocated in the first buffer + ggml_gallocr_allocate_node(galloc, leaf, 0); + } + } // allocate tensors for (int i = 0; i < graph->n_nodes; i++) { @@ -586,7 +599,7 @@ static void ggml_gallocr_alloc_graph_impl(ggml_gallocr_t galloc, struct ggml_cgr for (int j = 0; j < GGML_MAX_SRC; j++) { struct ggml_tensor * parent = node->src[j]; if (parent == NULL) { - break; + continue; } ggml_gallocr_allocate_node(galloc, parent, buffer_id); } @@ -598,7 +611,7 @@ static void ggml_gallocr_alloc_graph_impl(ggml_gallocr_t galloc, struct ggml_cgr for (int j = 0; j < GGML_MAX_SRC; j++) { struct ggml_tensor * parent = node->src[j]; if (parent == NULL) { - break; + continue; } AT_PRINTF("%s", parent->name); if (j < GGML_MAX_SRC - 1 && node->src[j + 1] != NULL) { @@ -611,7 +624,7 @@ static void ggml_gallocr_alloc_graph_impl(ggml_gallocr_t galloc, struct ggml_cgr for (int j = 0; j < GGML_MAX_SRC; j++) { struct ggml_tensor * parent = node->src[j]; if (parent == NULL) { - break; + continue; } struct hash_node * p_hn = ggml_gallocr_hash_get(galloc, parent); p_hn->n_children -= 1; @@ -696,6 +709,18 @@ bool ggml_gallocr_reserve_n(ggml_gallocr_t galloc, struct ggml_cgraph * graph, c } } } + if (galloc->n_leafs < graph->n_leafs) { + free(galloc->leaf_allocs); + galloc->leaf_allocs = calloc(sizeof(struct tensor_alloc), graph->n_leafs); + GGML_ASSERT(galloc->leaf_allocs != NULL); + } + galloc->n_leafs = graph->n_leafs; + for (int i = 0; i < graph->n_leafs; i++) { + struct ggml_tensor * leaf = graph->leafs[i]; + struct hash_node * hn = ggml_gallocr_hash_get(galloc, leaf); + galloc->leaf_allocs[i].offset = hn->offset; + galloc->leaf_allocs[i].size_max = ggml_backend_buft_get_alloc_size(galloc->bufts[hn->buffer_id], leaf); + } // reallocate buffers if needed for (int i = 0; i < galloc->n_buffers; i++) { @@ -722,8 +747,8 @@ bool ggml_gallocr_reserve(ggml_gallocr_t galloc, struct ggml_cgraph *graph) { return ggml_gallocr_reserve_n(galloc, graph, NULL); } -static void ggml_gallocr_init_tensor(ggml_gallocr_t galloc, struct ggml_tensor * node, struct node_alloc * node_alloc, struct tensor_alloc * tensor_alloc) { - assert(node->data || node->view_src || ggml_backend_buffer_get_alloc_size(galloc->buffers[node_alloc->buffer_id], node) <= tensor_alloc->size_max); +static void ggml_gallocr_init_tensor(ggml_gallocr_t galloc, struct ggml_tensor * node, int buffer_id, struct tensor_alloc * tensor_alloc) { + assert(node->data || node->view_src || ggml_backend_buffer_get_alloc_size(galloc->buffers[buffer_id], node) <= tensor_alloc->size_max); if (node->view_src != NULL) { if (node->buffer == NULL) { @@ -732,29 +757,20 @@ static void ggml_gallocr_init_tensor(ggml_gallocr_t galloc, struct ggml_tensor * // this tensor was allocated without ggml-backend return; } - ggml_backend_view_init(galloc->buffers[node_alloc->buffer_id], node); + ggml_backend_view_init(galloc->buffers[buffer_id], node); } } else { if (node->data == NULL) { assert(tensor_alloc->offset != SIZE_MAX); - assert(ggml_backend_buffer_get_alloc_size(galloc->buffers[node_alloc->buffer_id], node) <= tensor_alloc->size_max); - void * base = ggml_backend_buffer_get_base(galloc->buffers[node_alloc->buffer_id]); + assert(ggml_backend_buffer_get_alloc_size(galloc->buffers[buffer_id], node) <= tensor_alloc->size_max); + void * base = ggml_backend_buffer_get_base(galloc->buffers[buffer_id]); void * addr = (char *)base + tensor_alloc->offset; - ggml_backend_tensor_alloc(galloc->buffers[node_alloc->buffer_id], node, addr); + ggml_backend_tensor_alloc(galloc->buffers[buffer_id], node, addr); } else { if (node->buffer == NULL) { // this tensor was allocated without ggml-backend return; } - -#ifndef NDEBUG - size_t offset = - (char *)node->data - - (char *)ggml_backend_buffer_get_base(node->buffer); - size_t size = ggml_backend_buffer_get_alloc_size(node->buffer, node); - assert(tensor_alloc->offset == SIZE_MAX || offset == tensor_alloc->offset); - assert(tensor_alloc->offset == SIZE_MAX || size <= tensor_alloc->size_max); -#endif } } } @@ -773,6 +789,13 @@ static bool ggml_gallocr_needs_realloc(ggml_gallocr_t galloc, struct ggml_cgraph return true; } + if (galloc->n_leafs != graph->n_leafs) { +#ifndef NDEBUG + fprintf(stderr, "%s: graph has different number of leafs\n", __func__); +#endif + return true; + } + for (int i = 0; i < graph->n_nodes; i++) { struct ggml_tensor * node = graph->nodes[i]; struct node_alloc * node_alloc = &galloc->node_allocs[i]; @@ -827,6 +850,7 @@ bool ggml_gallocr_alloc_graph(ggml_gallocr_t galloc, struct ggml_cgraph * graph) } // allocate the graph tensors from the previous assignments + // nodes for (int i = 0; i < graph->n_nodes; i++) { struct ggml_tensor * node = graph->nodes[i]; struct node_alloc * node_alloc = &galloc->node_allocs[i]; @@ -835,9 +859,15 @@ bool ggml_gallocr_alloc_graph(ggml_gallocr_t galloc, struct ggml_cgraph * graph) if (src == NULL) { continue; } - ggml_gallocr_init_tensor(galloc, src, node_alloc, &node_alloc->src[j]); + ggml_gallocr_init_tensor(galloc, src, node_alloc->buffer_id, &node_alloc->src[j]); } - ggml_gallocr_init_tensor(galloc, node, node_alloc, &node_alloc->dst); + ggml_gallocr_init_tensor(galloc, node, node_alloc->buffer_id, &node_alloc->dst); + } + // leafs + for (int i = 0; i < graph->n_leafs; i++) { + struct ggml_tensor * leaf = graph->leafs[i]; + struct tensor_alloc * leaf_alloc = &galloc->leaf_allocs[i]; + ggml_gallocr_init_tensor(galloc, leaf, 0, leaf_alloc); } return true; From 337c9cbd52918ae5fb9a9d9e25d7fae4e238c9f1 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Mon, 19 Feb 2024 14:54:21 +0200 Subject: [PATCH 396/812] sync : ggml ggml-ci --- scripts/sync-ggml.last | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/sync-ggml.last b/scripts/sync-ggml.last index 7a23ab162..733d8f95b 100644 --- a/scripts/sync-ggml.last +++ b/scripts/sync-ggml.last @@ -1 +1 @@ -5070f078a67c18c11736e78316ab715ca9afde16 +818eeb8a3be99125746a90ec63af8f51516a2ec6 From 6fd413791a754598a54a366145960f2e27eec015 Mon Sep 17 00:00:00 2001 From: slaren Date: Mon, 19 Feb 2024 14:02:36 +0100 Subject: [PATCH 397/812] llava : replace ggml_cpy with ggml_cont --- examples/llava/clip.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/llava/clip.cpp b/examples/llava/clip.cpp index 98d512f67..1a02fde32 100644 --- a/examples/llava/clip.cpp +++ b/examples/llava/clip.cpp @@ -618,7 +618,7 @@ static ggml_cgraph * clip_image_build_graph(clip_ctx * ctx, const clip_image_f32 KQV = ggml_reshape_4d(ctx0, KQV, d_head, num_positions, n_head, batch_size); KQV = ggml_cont(ctx0, ggml_permute(ctx0, KQV, 0, 2, 1, 3)); - cur = ggml_cpy(ctx0, KQV, ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, hidden_size, num_positions, batch_size)); + cur = ggml_cont_3d(ctx0, KQV, hidden_size, num_positions, batch_size); } // attention output From 1387cf60f758efb218fa06b670182c38ff149b7b Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Mon, 19 Feb 2024 15:23:17 +0200 Subject: [PATCH 398/812] llava : remove extra cont (#5587) --- examples/llava/clip.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/llava/clip.cpp b/examples/llava/clip.cpp index 1a02fde32..ef9e4ba7a 100644 --- a/examples/llava/clip.cpp +++ b/examples/llava/clip.cpp @@ -616,7 +616,7 @@ static ggml_cgraph * clip_image_build_graph(clip_ctx * ctx, const clip_image_f32 KQ = ggml_soft_max_inplace(ctx0, KQ); struct ggml_tensor * KQV = ggml_mul_mat(ctx0, V, KQ); KQV = ggml_reshape_4d(ctx0, KQV, d_head, num_positions, n_head, batch_size); - KQV = ggml_cont(ctx0, ggml_permute(ctx0, KQV, 0, 2, 1, 3)); + KQV = ggml_permute(ctx0, KQV, 0, 2, 1, 3); cur = ggml_cont_3d(ctx0, KQV, hidden_size, num_positions, batch_size); } From 9d679f0fccd4030779ed3c7684a40122fe41806c Mon Sep 17 00:00:00 2001 From: nopperl <54780682+nopperl@users.noreply.github.com> Date: Mon, 19 Feb 2024 14:14:07 +0000 Subject: [PATCH 399/812] examples : support minItems/maxItems in JSON grammar converter (#5039) * support minLength and maxLength in JSON schema grammar converter * Update examples/json-schema-to-grammar.py --------- Co-authored-by: Georgi Gerganov --- examples/json-schema-to-grammar.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/examples/json-schema-to-grammar.py b/examples/json-schema-to-grammar.py index 2a4cb65bc..6a977f031 100755 --- a/examples/json-schema-to-grammar.py +++ b/examples/json-schema-to-grammar.py @@ -87,7 +87,21 @@ class SchemaConverter: elif schema_type == 'array' and 'items' in schema: # TODO `prefixItems` keyword item_rule_name = self.visit(schema['items'], f'{name}{"-" if name else ""}item') - rule = f'"[" space ({item_rule_name} ("," space {item_rule_name})*)? "]" space' + list_item_operator = f'("," space {item_rule_name})' + successive_items = "" + min_items = schema.get("minItems", 0) + if min_items > 0: + first_item = f"({item_rule_name})" + successive_items = list_item_operator * (min_items - 1) + min_items -= 1 + else: + first_item = f"({item_rule_name})?" + max_items = schema.get("maxItems") + if max_items is not None and max_items > min_items: + successive_items += (list_item_operator + "?") * (max_items - min_items - 1) + else: + successive_items += list_item_operator + "*" + rule = f'"[" space {first_item} {successive_items} "]" space' return self._add_rule(rule_name, rule) else: From f24ed14ee0ce28dfe98115c378b37da144912016 Mon Sep 17 00:00:00 2001 From: Jared Van Bortel Date: Mon, 19 Feb 2024 15:54:12 -0500 Subject: [PATCH 400/812] make : pass CPPFLAGS directly to nvcc, not via -Xcompiler (#5598) --- Makefile | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index 63b4af9ba..db5df1b32 100644 --- a/Makefile +++ b/Makefile @@ -446,9 +446,9 @@ ifdef LLAMA_CUDA_CCBIN endif ggml-cuda.o: ggml-cuda.cu ggml-cuda.h ifdef JETSON_EOL_MODULE_DETECT - $(NVCC) -I. -Icommon -D_XOPEN_SOURCE=600 -D_GNU_SOURCE -DNDEBUG -DGGML_USE_CUBLAS -I/usr/local/cuda/include -I/opt/cuda/include -I/usr/local/cuda/targets/aarch64-linux/include -std=c++11 -O3 $(NVCCFLAGS) -Xcompiler "$(CUDA_CXXFLAGS)" -c $< -o $@ + $(NVCC) -I. -Icommon -D_XOPEN_SOURCE=600 -D_GNU_SOURCE -DNDEBUG -DGGML_USE_CUBLAS -I/usr/local/cuda/include -I/opt/cuda/include -I/usr/local/cuda/targets/aarch64-linux/include -std=c++11 -O3 $(NVCCFLAGS) $(CPPFLAGS) -Xcompiler "$(CUDA_CXXFLAGS)" -c $< -o $@ else - $(NVCC) $(NVCCFLAGS) -Xcompiler "$(CUDA_CXXFLAGS)" -c $< -o $@ + $(NVCC) $(NVCCFLAGS) $(CPPFLAGS) -Xcompiler "$(CUDA_CXXFLAGS)" -c $< -o $@ endif # JETSON_EOL_MODULE_DETECT endif # LLAMA_CUBLAS @@ -549,9 +549,10 @@ GF_CC := $(CC) include scripts/get-flags.mk # combine build flags with cmdline overrides -override CFLAGS := $(MK_CPPFLAGS) $(CPPFLAGS) $(MK_CFLAGS) $(GF_CFLAGS) $(CFLAGS) -BASE_CXXFLAGS := $(MK_CPPFLAGS) $(CPPFLAGS) $(MK_CXXFLAGS) $(CXXFLAGS) -override CXXFLAGS := $(BASE_CXXFLAGS) $(HOST_CXXFLAGS) $(GF_CXXFLAGS) +override CPPFLAGS := $(MK_CPPFLAGS) $(CPPFLAGS) +override CFLAGS := $(CPPFLAGS) $(MK_CFLAGS) $(GF_CFLAGS) $(CFLAGS) +BASE_CXXFLAGS := $(MK_CXXFLAGS) $(CXXFLAGS) +override CXXFLAGS := $(BASE_CXXFLAGS) $(HOST_CXXFLAGS) $(GF_CXXFLAGS) $(CPPFLAGS) override NVCCFLAGS := $(MK_NVCCFLAGS) $(NVCCFLAGS) override LDFLAGS := $(MK_LDFLAGS) $(LDFLAGS) From 40c3a6c1e11040088b4a1ce0abc4651cb3011dd4 Mon Sep 17 00:00:00 2001 From: slaren Date: Mon, 19 Feb 2024 23:40:26 +0100 Subject: [PATCH 401/812] cuda : ignore peer access already enabled errors (#5597) * cuda : ignore peer access already enabled errors * fix hip --- ggml-cuda.cu | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/ggml-cuda.cu b/ggml-cuda.cu index e091dbdc1..6caae56b0 100644 --- a/ggml-cuda.cu +++ b/ggml-cuda.cu @@ -54,6 +54,8 @@ #define cudaDeviceProp hipDeviceProp_t #define cudaDeviceSynchronize hipDeviceSynchronize #define cudaError_t hipError_t +#define cudaErrorPeerAccessAlreadyEnabled hipErrorPeerAccessAlreadyEnabled +#define cudaErrorPeerAccessNotEnabled hipErrorPeerAccessNotEnabled #define cudaEventCreateWithFlags hipEventCreateWithFlags #define cudaEventDisableTiming hipEventDisableTiming #define cudaEventRecord hipEventRecord @@ -9325,9 +9327,15 @@ static void ggml_cuda_set_peer_access(const int n_tokens) { CUDA_CHECK(cudaDeviceCanAccessPeer(&can_access_peer, id, id_other)); if (can_access_peer) { if (enable_peer_access) { - CUDA_CHECK(cudaDeviceEnablePeerAccess(id_other, 0)); + cudaError_t err = cudaDeviceEnablePeerAccess(id_other, 0); + if (err != cudaErrorPeerAccessAlreadyEnabled) { + CUDA_CHECK(err); + } } else { - CUDA_CHECK(cudaDeviceDisablePeerAccess(id_other)); + cudaError_t err = cudaDeviceDisablePeerAccess(id_other); + if (err != cudaErrorPeerAccessNotEnabled) { + CUDA_CHECK(err); + } } } } @@ -10999,10 +11007,10 @@ GGML_CALL static const char * ggml_backend_cuda_split_buffer_get_name(ggml_backe UNUSED(buffer); } -// unused at the moment -//static bool ggml_backend_buffer_is_cuda_split(ggml_backend_buffer_t buffer) { -// return buffer->iface.get_name == ggml_backend_cuda_split_buffer_get_name; -//} +static bool ggml_backend_buffer_is_cuda_split(ggml_backend_buffer_t buffer) { + return buffer->iface.get_name == ggml_backend_cuda_split_buffer_get_name; + UNUSED(ggml_backend_buffer_is_cuda_split); // only used in debug builds currently, avoid unused function warning in release builds +} GGML_CALL static void ggml_backend_cuda_split_buffer_free_buffer(ggml_backend_buffer_t buffer) { ggml_backend_cuda_split_buffer_context * ctx = (ggml_backend_cuda_split_buffer_context *)buffer->context; @@ -11390,7 +11398,7 @@ GGML_CALL static bool ggml_backend_cuda_graph_compute(ggml_backend_t backend, gg for (int j = 0; j < GGML_MAX_SRC; j++) { if (node->src[j] != nullptr) { assert(node->src[j]->backend == GGML_BACKEND_GPU || node->src[j]->backend == GGML_BACKEND_GPU_SPLIT); - assert(node->src[j]->buffer->buft == ggml_backend_cuda_buffer_type(cuda_ctx->device)); + assert(node->src[j]->buffer->buft == ggml_backend_cuda_buffer_type(cuda_ctx->device) || ggml_backend_buffer_is_cuda_split(node->src[j]->buffer)); assert(node->src[j]->extra != nullptr); } } From 5dde5408978eda22242b87e22e306d1c2d1a5834 Mon Sep 17 00:00:00 2001 From: Mathijs de Bruin Date: Sat, 3 Feb 2024 17:56:46 +0000 Subject: [PATCH 402/812] Allow for Vulkan build with Accelerate. Closes #5304 --- ggml.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ggml.c b/ggml.c index 4ee2c5e11..d129df505 100644 --- a/ggml.c +++ b/ggml.c @@ -273,6 +273,8 @@ inline static void * ggml_calloc(size_t num, size_t size) { #include #if defined(GGML_USE_CLBLAST) // allow usage of CLBlast alongside Accelerate functions #include "ggml-opencl.h" +#elif defined(GGML_USE_VULKAN) +#include "ggml-vulkan.h" #endif #elif defined(GGML_USE_OPENBLAS) #if defined(GGML_BLAS_USE_MKL) From 42f664a3825dfde13a32c3577ab66d10c56f3aa6 Mon Sep 17 00:00:00 2001 From: Mathijs de Bruin Date: Sat, 3 Feb 2024 18:00:11 +0000 Subject: [PATCH 403/812] Resolve ErrorIncompatibleDriver with Vulkan on MacOS. Refs: - https://chat.openai.com/share/7020ce72-65fc-45ec-b7be-9d9d798a5f3f - https://github.com/SaschaWillems/Vulkan/issues/954 - https://github.com/haasn/libplacebo/issues/128 - https://github.com/KhronosGroup/Vulkan-Samples/issues/476 --- ggml-vulkan.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ggml-vulkan.cpp b/ggml-vulkan.cpp index 4a30414df..e9e966dbf 100644 --- a/ggml-vulkan.cpp +++ b/ggml-vulkan.cpp @@ -1109,8 +1109,10 @@ static void ggml_vk_instance_init() { #ifdef GGML_VULKAN_VALIDATE "VK_EXT_validation_features", #endif + "VK_KHR_portability_enumeration", }; - vk::InstanceCreateInfo instance_create_info(vk::InstanceCreateFlags(), &app_info, layers, extensions); + + vk::InstanceCreateInfo instance_create_info(vk::InstanceCreateFlags(vk::InstanceCreateFlagBits::eEnumeratePortabilityKHR), &app_info, layers, extensions); #ifdef GGML_VULKAN_VALIDATE const std::vector features_enable = { vk::ValidationFeatureEnableEXT::eBestPractices }; vk::ValidationFeaturesEXT validation_features = { From d8c054517dc24f1316f3be12a98fff383e1e93e3 Mon Sep 17 00:00:00 2001 From: Mathijs de Bruin Date: Tue, 6 Feb 2024 14:39:22 +0000 Subject: [PATCH 404/812] Add preprocessor checks for Apple devices. Based on work by @rbourgeat in https://github.com/ggerganov/llama.cpp/pull/5322/files --- ggml-vulkan.cpp | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/ggml-vulkan.cpp b/ggml-vulkan.cpp index e9e966dbf..33b8a9061 100644 --- a/ggml-vulkan.cpp +++ b/ggml-vulkan.cpp @@ -1109,10 +1109,15 @@ static void ggml_vk_instance_init() { #ifdef GGML_VULKAN_VALIDATE "VK_EXT_validation_features", #endif +#ifdef __APPLE__ "VK_KHR_portability_enumeration", +#endif }; + vk::InstanceCreateInfo instance_create_info(vk::InstanceCreateFlags(), &app_info, layers, extensions); +#ifdef __APPLE__ + instance_create_info.flags = vk::InstanceCreateFlagBits::eEnumeratePortabilityKHR; +#endif - vk::InstanceCreateInfo instance_create_info(vk::InstanceCreateFlags(vk::InstanceCreateFlagBits::eEnumeratePortabilityKHR), &app_info, layers, extensions); #ifdef GGML_VULKAN_VALIDATE const std::vector features_enable = { vk::ValidationFeatureEnableEXT::eBestPractices }; vk::ValidationFeaturesEXT validation_features = { From f50db6ae0bdcb5f8593ca6ca46dfa03b177faa2f Mon Sep 17 00:00:00 2001 From: 0cc4m Date: Sat, 10 Feb 2024 22:14:52 +0100 Subject: [PATCH 405/812] Add check for VK_KHR_portability_enumeration for MoltenVK support --- ggml-vulkan.cpp | 41 +++++++++++++++++++++++++++++++---------- 1 file changed, 31 insertions(+), 10 deletions(-) diff --git a/ggml-vulkan.cpp b/ggml-vulkan.cpp index 33b8a9061..37123ac8f 100644 --- a/ggml-vulkan.cpp +++ b/ggml-vulkan.cpp @@ -1100,23 +1100,44 @@ static void ggml_vk_instance_init() { #endif vk::ApplicationInfo app_info{ "ggml-vulkan", 1, nullptr, 0, VK_API_VERSION }; - const std::vector layers = { + + const std::vector instance_extensions = vk::enumerateInstanceExtensionProperties(); +#ifdef __APPLE__ + bool portability_enumeration_ext = false; + // Check for portability enumeration extension for MoltenVK support + for (const auto& properties : instance_extensions) { + if (strcmp("VK_KHR_portability_enumeration", properties.extensionName) == 0) { + portability_enumeration_ext = true; + break; + } + } + if (!portability_enumeration_ext) { + std::cerr << "ggml_vulkan: WARNING: Instance extension VK_KHR_portability_enumeration not found." << std::endl; + } +#endif + + std::vector layers = { #ifdef GGML_VULKAN_VALIDATE "VK_LAYER_KHRONOS_validation", #endif }; - const std::vector extensions = { + std::vector extensions = { #ifdef GGML_VULKAN_VALIDATE "VK_EXT_validation_features", -#endif -#ifdef __APPLE__ - "VK_KHR_portability_enumeration", #endif }; - vk::InstanceCreateInfo instance_create_info(vk::InstanceCreateFlags(), &app_info, layers, extensions); #ifdef __APPLE__ - instance_create_info.flags = vk::InstanceCreateFlagBits::eEnumeratePortabilityKHR; + if (portability_enumeration_ext) { + extensions.push_back("VK_KHR_portability_enumeration"); + } #endif + vk::InstanceCreateInfo instance_create_info(vk::InstanceCreateFlags{}, &app_info, layers, extensions); +#ifdef __APPLE__ + if (portability_enumeration_ext) { + instance_create_info.flags |= vk::InstanceCreateFlagBits::eEnumeratePortabilityKHR; + } +#endif + #ifdef GGML_VULKAN_VALIDATE const std::vector features_enable = { vk::ValidationFeatureEnableEXT::eBestPractices }; @@ -1175,12 +1196,12 @@ static void ggml_vk_init(ggml_backend_vk_context * ctx, size_t idx) { vk_instance.devices[idx] = std::make_shared(); ctx->device = vk_instance.devices[idx]; ctx->device.lock()->physical_device = devices[dev_num]; - std::vector ext_props = ctx->device.lock()->physical_device.enumerateDeviceExtensionProperties(); + const std::vector ext_props = ctx->device.lock()->physical_device.enumerateDeviceExtensionProperties(); bool maintenance4_support = false; // Check if maintenance4 is supported - for (auto properties : ext_props) { + for (const auto& properties : ext_props) { if (strcmp("VK_KHR_maintenance4", properties.extensionName) == 0) { maintenance4_support = true; } @@ -1211,7 +1232,7 @@ static void ggml_vk_init(ggml_backend_vk_context * ctx, size_t idx) { bool fp16_storage = false; bool fp16_compute = false; - for (auto properties : ext_props) { + for (const auto& properties : ext_props) { if (strcmp("VK_KHR_16bit_storage", properties.extensionName) == 0) { fp16_storage = true; } else if (strcmp("VK_KHR_shader_float16_int8", properties.extensionName) == 0) { From bb9dcd560a7e81265398b0d463c40f3e467daf19 Mon Sep 17 00:00:00 2001 From: 0cc4m Date: Wed, 14 Feb 2024 20:57:17 +0100 Subject: [PATCH 406/812] Refactor validation and enumeration platform checks into functions to clean up ggml_vk_instance_init() --- ggml-vulkan.cpp | 101 ++++++++++++++++++++++++++++++------------------ 1 file changed, 63 insertions(+), 38 deletions(-) diff --git a/ggml-vulkan.cpp b/ggml-vulkan.cpp index 37123ac8f..4e5eaff15 100644 --- a/ggml-vulkan.cpp +++ b/ggml-vulkan.cpp @@ -1091,7 +1091,10 @@ static void ggml_vk_print_gpu_info(size_t idx) { } } -static void ggml_vk_instance_init() { +static bool ggml_vk_instance_validation_ext_available(const std::vector& instance_extensions); +static bool ggml_vk_instance_portability_enumeration_ext_available(const std::vector& instance_extensions); + +void ggml_vk_instance_init() { if (vk_instance_initialized) { return; } @@ -1102,54 +1105,40 @@ static void ggml_vk_instance_init() { vk::ApplicationInfo app_info{ "ggml-vulkan", 1, nullptr, 0, VK_API_VERSION }; const std::vector instance_extensions = vk::enumerateInstanceExtensionProperties(); -#ifdef __APPLE__ - bool portability_enumeration_ext = false; - // Check for portability enumeration extension for MoltenVK support - for (const auto& properties : instance_extensions) { - if (strcmp("VK_KHR_portability_enumeration", properties.extensionName) == 0) { - portability_enumeration_ext = true; - break; - } - } - if (!portability_enumeration_ext) { - std::cerr << "ggml_vulkan: WARNING: Instance extension VK_KHR_portability_enumeration not found." << std::endl; - } -#endif + const bool validation_ext = ggml_vk_instance_validation_ext_available(instance_extensions); + const bool portability_enumeration_ext = ggml_vk_instance_portability_enumeration_ext_available(instance_extensions); - std::vector layers = { -#ifdef GGML_VULKAN_VALIDATE - "VK_LAYER_KHRONOS_validation", -#endif - }; - std::vector extensions = { -#ifdef GGML_VULKAN_VALIDATE - "VK_EXT_validation_features", -#endif - }; -#ifdef __APPLE__ + std::vector layers; + + if (validation_ext) { + layers.push_back("VK_LAYER_KHRONOS_validation"); + } + std::vector extensions; + if (validation_ext) { + extensions.push_back("VK_EXT_validation_features"); + } if (portability_enumeration_ext) { extensions.push_back("VK_KHR_portability_enumeration"); } -#endif vk::InstanceCreateInfo instance_create_info(vk::InstanceCreateFlags{}, &app_info, layers, extensions); -#ifdef __APPLE__ if (portability_enumeration_ext) { instance_create_info.flags |= vk::InstanceCreateFlagBits::eEnumeratePortabilityKHR; } -#endif + std::vector features_enable; + vk::ValidationFeaturesEXT validation_features; -#ifdef GGML_VULKAN_VALIDATE - const std::vector features_enable = { vk::ValidationFeatureEnableEXT::eBestPractices }; - vk::ValidationFeaturesEXT validation_features = { - features_enable, - {}, - }; - validation_features.setPNext(nullptr); - instance_create_info.setPNext(&validation_features); + if (validation_ext) { + features_enable = { vk::ValidationFeatureEnableEXT::eBestPractices }; + validation_features = { + features_enable, + {}, + }; + validation_features.setPNext(nullptr); + instance_create_info.setPNext(&validation_features); - std::cerr << "ggml_vulkan: Validation layers enabled" << std::endl; -#endif + std::cerr << "ggml_vulkan: Validation layers enabled" << std::endl; + } vk_instance.instance = vk::createInstance(instance_create_info); memset(vk_instance.initialized, 0, sizeof(bool) * GGML_VK_MAX_DEVICES); @@ -5329,6 +5318,42 @@ GGML_CALL int ggml_backend_vk_reg_devices() { return vk_instance.device_indices.size(); } +// Extension availability +static bool ggml_vk_instance_validation_ext_available(const std::vector& instance_extensions) { +#ifdef GGML_VULKAN_VALIDATE + bool portability_enumeration_ext = false; + // Check for portability enumeration extension for MoltenVK support + for (const auto& properties : instance_extensions) { + if (strcmp("VK_KHR_portability_enumeration", properties.extensionName) == 0) { + return true; + } + } + if (!portability_enumeration_ext) { + std::cerr << "ggml_vulkan: WARNING: Instance extension VK_KHR_portability_enumeration not found." << std::endl; + } +#endif + return false; + + UNUSED(instance_extensions); +} +static bool ggml_vk_instance_portability_enumeration_ext_available(const std::vector& instance_extensions) { +#ifdef __APPLE__ + bool portability_enumeration_ext = false; + // Check for portability enumeration extension for MoltenVK support + for (const auto& properties : instance_extensions) { + if (strcmp("VK_KHR_portability_enumeration", properties.extensionName) == 0) { + return true; + } + } + if (!portability_enumeration_ext) { + std::cerr << "ggml_vulkan: WARNING: Instance extension VK_KHR_portability_enumeration not found." << std::endl; + } +#endif + return false; + + UNUSED(instance_extensions); +} + // checks #ifdef GGML_VULKAN_CHECK_RESULTS From 22f83f0c383e12106692b8afc224d61b8993a52c Mon Sep 17 00:00:00 2001 From: 0cc4m Date: Sat, 10 Feb 2024 22:18:33 +0100 Subject: [PATCH 407/812] Enable Vulkan MacOS CI --- .devops/nix/package.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.devops/nix/package.nix b/.devops/nix/package.nix index ad23f7dd7..815db6a2d 100644 --- a/.devops/nix/package.nix +++ b/.devops/nix/package.nix @@ -255,11 +255,11 @@ effectiveStdenv.mkDerivation ( # Configurations we don't want even the CI to evaluate. Results in the # "unsupported platform" messages. This is mostly a no-op, because # cudaPackages would've refused to evaluate anyway. - badPlatforms = optionals (useCuda || useOpenCL || useVulkan) lib.platforms.darwin; + badPlatforms = optionals (useCuda || useOpenCL) lib.platforms.darwin; # Configurations that are known to result in build failures. Can be # overridden by importing Nixpkgs with `allowBroken = true`. - broken = (useMetalKit && !effectiveStdenv.isDarwin) || (useVulkan && effectiveStdenv.isDarwin); + broken = (useMetalKit && !effectiveStdenv.isDarwin); description = "Inference of LLaMA model in pure C/C++${descriptionSuffix}"; homepage = "https://github.com/ggerganov/llama.cpp/"; From 633782b8d949f24b619e6c68ee37b5cc79167173 Mon Sep 17 00:00:00 2001 From: Mathijs de Bruin Date: Tue, 13 Feb 2024 20:28:02 +0000 Subject: [PATCH 408/812] nix: now that we can do so, allow MacOS to build Vulkan binaries Author: Philip Taron Date: Tue Feb 13 20:28:02 2024 +0000 --- flake.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flake.nix b/flake.nix index ad2f9b295..dc4e503c3 100644 --- a/flake.nix +++ b/flake.nix @@ -150,6 +150,7 @@ packages = { default = config.legacyPackages.llamaPackages.llama-cpp; + vulkan = config.packages.default.override { useVulkan = true; }; } // lib.optionalAttrs pkgs.stdenv.isLinux { opencl = config.packages.default.override { useOpenCL = true; }; @@ -157,7 +158,6 @@ mpi-cpu = config.packages.default.override { useMpi = true; }; mpi-cuda = config.packages.default.override { useMpi = true; }; - vulkan = config.packages.default.override { useVulkan = true; }; } // lib.optionalAttrs (system == "x86_64-linux") { rocm = config.legacyPackages.llamaPackagesRocm.llama-cpp; From b9111bd209c7b11b0592450a6ed2e0ca545b2c84 Mon Sep 17 00:00:00 2001 From: AidanBeltonS <87009434+AidanBeltonS@users.noreply.github.com> Date: Tue, 20 Feb 2024 07:01:25 +0000 Subject: [PATCH 409/812] Update ggml_sycl_op_mul_mat_vec_q (#5502) * Update ggml_sycl_op_mul_mat_vec_q * Apply suggestions from code review Co-authored-by: Abhilash Majumder <30946547+abhilash1910@users.noreply.github.com> * revert suggestion on macro * fix bug * Add quant type GGML_TYPE_IQ1_S to unsupported * fix format --------- Co-authored-by: Abhilash Majumder <30946547+abhilash1910@users.noreply.github.com> --- ggml-sycl.cpp | 258 ++++++++++++++------------------------------------ 1 file changed, 69 insertions(+), 189 deletions(-) diff --git a/ggml-sycl.cpp b/ggml-sycl.cpp index cd4b3a1e1..df1826112 100644 --- a/ggml-sycl.cpp +++ b/ggml-sycl.cpp @@ -9188,174 +9188,22 @@ static void convert_mul_mat_vec_f16_sycl(const void *vx, const dfloat *y, } } -static void mul_mat_vec_q4_0_q8_1_sycl(const void *vx, const void *vy, - float *dst, const int ncols, - const int nrows, - dpct::queue_ptr stream) { - GGML_ASSERT(ncols % QK4_0 == 0); - const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y; - const sycl::range<3> block_nums(1, 1, block_num_y); - const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE); - stream->parallel_for( - sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) [[intel::reqd_sub_group_size(32)]] { - mul_mat_vec_q(vx, vy, dst, ncols, nrows, - item_ct1); - }); -} - -static void mul_mat_vec_q4_1_q8_1_sycl(const void *vx, const void *vy, - float *dst, const int ncols, - const int nrows, - dpct::queue_ptr stream) { - GGML_ASSERT(ncols % QK4_1 == 0); - const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y; - const sycl::range<3> block_nums(1, 1, block_num_y); - const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE); - stream->parallel_for( - sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) [[intel::reqd_sub_group_size(32)]] { - mul_mat_vec_q(vx, vy, dst, ncols, nrows, - item_ct1); - }); -} - -static void mul_mat_vec_q5_0_q8_1_sycl(const void *vx, const void *vy, - float *dst, const int ncols, - const int nrows, - dpct::queue_ptr stream) { - GGML_ASSERT(ncols % QK5_0 == 0); - const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y; - const sycl::range<3> block_nums(1, 1, block_num_y); - const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE); - stream->parallel_for( - sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) [[intel::reqd_sub_group_size(32)]] { - mul_mat_vec_q(vx, vy, dst, ncols, nrows, - item_ct1); - }); -} - -static void mul_mat_vec_q5_1_q8_1_sycl(const void *vx, const void *vy, - float *dst, const int ncols, - const int nrows, - dpct::queue_ptr stream) { - GGML_ASSERT(ncols % QK5_1 == 0); - const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y; - const sycl::range<3> block_nums(1, 1, block_num_y); - const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE); - stream->parallel_for( - sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) [[intel::reqd_sub_group_size(32)]] { - mul_mat_vec_q(vx, vy, dst, ncols, nrows, - item_ct1); - }); -} - -static void mul_mat_vec_q8_0_q8_1_sycl(const void *vx, const void *vy, - float *dst, const int ncols, - const int nrows, - dpct::queue_ptr stream) { - GGML_ASSERT(ncols % QK8_0 == 0); - const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y; - const sycl::range<3> block_nums(1, 1, block_num_y); - const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE); - stream->parallel_for( - sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) [[intel::reqd_sub_group_size(32)]] { - mul_mat_vec_q(vx, vy, dst, ncols, nrows, - item_ct1); - }); -} - -static void mul_mat_vec_q2_K_q8_1_sycl(const void *vx, const void *vy, - float *dst, const int ncols, - const int nrows, - dpct::queue_ptr stream) { - GGML_ASSERT(ncols % QK_K == 0); - const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y; - const sycl::range<3> block_nums(1, 1, block_num_y); - const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE); - stream->parallel_for( - sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) [[intel::reqd_sub_group_size(32)]] { - mul_mat_vec_q(vx, vy, dst, ncols, nrows, - item_ct1); - }); -} - -static void mul_mat_vec_q3_K_q8_1_sycl(const void *vx, const void *vy, - float *dst, const int ncols, - const int nrows, - dpct::queue_ptr stream) { - GGML_ASSERT(ncols % QK_K == 0); - const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y; - const sycl::range<3> block_nums(1, 1, block_num_y); - const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE); - stream->parallel_for( - sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) [[intel::reqd_sub_group_size(32)]] { - mul_mat_vec_q(vx, vy, dst, ncols, nrows, - item_ct1); - }); -} - -static void mul_mat_vec_q4_K_q8_1_sycl(const void *vx, const void *vy, - float *dst, const int ncols, - const int nrows, - dpct::queue_ptr stream) { - GGML_ASSERT(ncols % QK_K == 0); - const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y; - const sycl::range<3> block_nums(1, 1, block_num_y); - const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE); - stream->parallel_for( - sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) [[intel::reqd_sub_group_size(32)]] { - mul_mat_vec_q(vx, vy, dst, ncols, nrows, - item_ct1); - }); -} - -static void mul_mat_vec_q5_K_q8_1_sycl(const void *vx, const void *vy, - float *dst, const int ncols, - const int nrows, - dpct::queue_ptr stream) { - GGML_ASSERT(ncols % QK_K == 0); - const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y; - const sycl::range<3> block_nums(1, 1, block_num_y); - const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE); - stream->parallel_for( - sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) [[intel::reqd_sub_group_size(32)]] { - mul_mat_vec_q(vx, vy, dst, ncols, nrows, - item_ct1); - }); -} - -static void mul_mat_vec_q6_K_q8_1_sycl(const void *vx, const void *vy, - float *dst, const int ncols, - const int nrows, - dpct::queue_ptr stream) { - GGML_ASSERT(ncols % QK_K == 0); - const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y; - const sycl::range<3> block_nums(1, 1, block_num_y); - const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE); - stream->parallel_for( - sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) [[intel::reqd_sub_group_size(32)]] { - mul_mat_vec_q(vx, vy, dst, ncols, nrows, - item_ct1); - }); +template +static void mul_mat_vec_q_sycl_submitter(const void *vx, const void *vy, + float *dst, const int ncols, + const int nrows, + dpct::queue_ptr stream) { + GGML_ASSERT(ncols % QK4_0 == 0); + const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y; + const sycl::range<3> block_nums(1, 1, block_num_y); + const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE); + stream->parallel_for( + sycl::nd_range<3>(block_nums * block_dims, block_dims), [= + ](sycl::nd_item<3> item_ct1) [[intel::reqd_sub_group_size(32)]] { + mul_mat_vec_q( + vx, vy, dst, ncols, nrows, item_ct1); + }); } int get_device_index_by_id(int id){ @@ -12095,37 +11943,63 @@ inline void ggml_sycl_op_mul_mat_vec_q( const int64_t ne00 = src0->ne[0]; const int64_t row_diff = row_high - row_low; + // TODO: support these quantization types + GGML_ASSERT(!(src0->type == GGML_TYPE_IQ2_XXS || + src0->type == GGML_TYPE_IQ2_XS || + src0->type == GGML_TYPE_IQ3_XXS || + src0->type == GGML_TYPE_IQ1_S)); + switch (src0->type) { case GGML_TYPE_Q4_0: - mul_mat_vec_q4_0_q8_1_sycl(src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, stream); - break; + mul_mat_vec_q_sycl_submitter( + src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, stream); + break; case GGML_TYPE_Q4_1: - mul_mat_vec_q4_1_q8_1_sycl(src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, stream); - break; + mul_mat_vec_q_sycl_submitter( + src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, stream); + break; case GGML_TYPE_Q5_0: - mul_mat_vec_q5_0_q8_1_sycl(src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, stream); - break; + mul_mat_vec_q_sycl_submitter( + src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, stream); + break; case GGML_TYPE_Q5_1: - mul_mat_vec_q5_1_q8_1_sycl(src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, stream); - break; + mul_mat_vec_q_sycl_submitter( + src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, stream); + break; case GGML_TYPE_Q8_0: - mul_mat_vec_q8_0_q8_1_sycl(src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, stream); - break; + mul_mat_vec_q_sycl_submitter( + src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, stream); + break; case GGML_TYPE_Q2_K: - mul_mat_vec_q2_K_q8_1_sycl(src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, stream); - break; + mul_mat_vec_q_sycl_submitter( + src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, stream); + break; case GGML_TYPE_Q3_K: - mul_mat_vec_q3_K_q8_1_sycl(src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, stream); - break; + mul_mat_vec_q_sycl_submitter( + src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, stream); + break; case GGML_TYPE_Q4_K: - mul_mat_vec_q4_K_q8_1_sycl(src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, stream); - break; + mul_mat_vec_q_sycl_submitter( + src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, stream); + break; case GGML_TYPE_Q5_K: - mul_mat_vec_q5_K_q8_1_sycl(src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, stream); - break; + mul_mat_vec_q_sycl_submitter( + src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, stream); + break; case GGML_TYPE_Q6_K: - mul_mat_vec_q6_K_q8_1_sycl(src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, stream); - break; + mul_mat_vec_q_sycl_submitter( + src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, stream); + break; default: GGML_ASSERT(false); break; @@ -12145,7 +12019,7 @@ inline void ggml_sycl_op_dequantize_mul_mat_vec( const int64_t src1_ncols, const int64_t src1_padded_row_size, const dpct::queue_ptr &stream) { - GGML_TENSOR_BINARY_OP_LOCALS + GGML_TENSOR_BINARY_OP_LOCALS; const int64_t row_diff = row_high - row_low; @@ -15093,6 +14967,12 @@ static bool ggml_backend_sycl_supports_op(ggml_backend_t backend, const ggml_ten return false; } + if (a->type == GGML_TYPE_IQ1_S) { + return false; + } + if (a->type == GGML_TYPE_IQ3_XXS) { + return false; + } if (a->type == GGML_TYPE_IQ2_XXS) { return false; } From c0a8c6db371cb3e4379900867b948879f5842201 Mon Sep 17 00:00:00 2001 From: Pierrick Hymbert Date: Tue, 20 Feb 2024 08:48:19 +0100 Subject: [PATCH 410/812] server : health endpoint configurable failure on no slot (#5594) --- examples/server/README.md | 9 ++++--- examples/server/server.cpp | 52 +++++++++++++++++++------------------- 2 files changed, 31 insertions(+), 30 deletions(-) diff --git a/examples/server/README.md b/examples/server/README.md index 809e2d37c..f6b9c7402 100644 --- a/examples/server/README.md +++ b/examples/server/README.md @@ -134,10 +134,11 @@ node index.js ## API Endpoints - **GET** `/health`: Returns the current state of the server: - - `{"status": "loading model"}` if the model is still being loaded. - - `{"status": "error"}` if the model failed to load. - - `{"status": "ok"}` if the model is successfully loaded and the server is ready for further requests mentioned below. - - `{"status": "no slot available", "slots_idle": 0, "slots_processing": 32}` if no slot are currently available + - 503 -> `{"status": "loading model"}` if the model is still being loaded. + - 500 -> `{"status": "error"}` if the model failed to load. + - 200 -> `{"status": "ok", "slots_idle": 1, "slots_processing": 2 }` if the model is successfully loaded and the server is ready for further requests mentioned below. + - 200 -> `{"status": "no slot available", "slots_idle": 0, "slots_processing": 32}` if no slot are currently available. + - 503 -> `{"status": "no slot available", "slots_idle": 0, "slots_processing": 32}` if the query parameter `fail_on_no_slot` is provided and no slot are currently available. - **POST** `/completion`: Given a `prompt`, it returns the predicted completion. diff --git a/examples/server/server.cpp b/examples/server/server.cpp index 22c344dd4..23482ed95 100644 --- a/examples/server/server.cpp +++ b/examples/server/server.cpp @@ -2582,40 +2582,40 @@ int main(int argc, char **argv) res.set_header("Access-Control-Allow-Headers", "*"); }); - svr.Get("/health", [&](const httplib::Request&, httplib::Response& res) { + svr.Get("/health", [&](const httplib::Request& req, httplib::Response& res) { server_state current_state = state.load(); switch(current_state) { - case SERVER_STATE_READY: - if (llama.all_slots_are_idle) { - res.set_content(R"({"status": "ok"})", "application/json"); + case SERVER_STATE_READY: { + int available_slots = 0; + int processing_slots = 0; + for (llama_client_slot &slot: llama.slots) { + if (slot.available()) { + available_slots++; + } else { + processing_slots++; + } + } + if (available_slots > 0) { + json health = { + {"status", "ok"}, + {"slots_idle", available_slots}, + {"slots_processing", processing_slots}}; + res.set_content(health.dump(), "application/json"); res.status = 200; // HTTP OK } else { - int available_slots = 0; - int processing_slots = 0; - for (llama_client_slot & slot : llama.slots) { - if (slot.available()) { - available_slots++; - } else { - processing_slots++; - } - } - if (available_slots > 0) { - json health = { - {"status", "ok"}, - {"slots_idle", available_slots}, - {"slots_processing", processing_slots}}; - res.set_content(health.dump(), "application/json"); - res.status = 200; // HTTP OK - } else { - json health = { - {"status", "no slot available"}, - {"slots_idle", available_slots}, - {"slots_processing", processing_slots}}; - res.set_content(health.dump(), "application/json"); + json health = { + {"status", "no slot available"}, + {"slots_idle", available_slots}, + {"slots_processing", processing_slots}}; + res.set_content(health.dump(), "application/json"); + if (req.has_param("fail_on_no_slot")) { res.status = 503; // HTTP Service Unavailable + } else { + res.status = 200; // HTTP OK } } break; + } case SERVER_STATE_LOADING_MODEL: res.set_content(R"({"status": "loading model"})", "application/json"); res.status = 503; // HTTP Service Unavailable From 8dbbd75754d43ec7b4bbe42fb287cc2553fdf0e9 Mon Sep 17 00:00:00 2001 From: Haoxiang Fei Date: Mon, 19 Feb 2024 22:58:36 -1100 Subject: [PATCH 411/812] metal : add build system support for embedded metal library (#5604) * add build support for embedded metal library * Update Makefile --------- Co-authored-by: Haoxiang Fei Co-authored-by: Georgi Gerganov --- CMakeLists.txt | 24 ++++++++++++++++++++++++ Makefile | 18 ++++++++++++++++++ 2 files changed, 42 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 168b133f4..3c4629001 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -110,6 +110,7 @@ option(LLAMA_VULKAN_RUN_TESTS "llama: run Vulkan tests" option(LLAMA_METAL "llama: use Metal" ${LLAMA_METAL_DEFAULT}) option(LLAMA_METAL_NDEBUG "llama: disable Metal debugging" OFF) option(LLAMA_METAL_SHADER_DEBUG "llama: compile Metal with -fno-fast-math" OFF) +option(LLAMA_METAL_EMBED_LIBRARY "llama: embed Metal library" OFF) option(LLAMA_KOMPUTE "llama: use Kompute" OFF) option(LLAMA_MPI "llama: use MPI" OFF) option(LLAMA_QKK_64 "llama: use super-block size of 64 for k-quants" OFF) @@ -201,6 +202,29 @@ if (LLAMA_METAL) # copy ggml-metal.metal to bin directory configure_file(ggml-metal.metal ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/ggml-metal.metal COPYONLY) + if (LLAMA_METAL_EMBED_LIBRARY) + enable_language(ASM) + add_compile_definitions(GGML_METAL_EMBED_LIBRARY) + + set(METALLIB_SOURCE "${CMAKE_SOURCE_DIR}/ggml-metal.metal") + file(MAKE_DIRECTORY "${CMAKE_BINARY_DIR}/autogenerated") + set(EMBED_METALLIB_ASSEMBLY "${CMAKE_BINARY_DIR}/autogenerated/ggml-embed-metallib.s") + + add_custom_command( + OUTPUT ${EMBED_METALLIB_ASSEMBLY} + COMMAND echo ".section __DATA,__ggml_metallib" > ${EMBED_METALLIB_ASSEMBLY} + COMMAND echo ".globl _ggml_metallib_start" >> ${EMBED_METALLIB_ASSEMBLY} + COMMAND echo "_ggml_metallib_start:" >> ${EMBED_METALLIB_ASSEMBLY} + COMMAND echo ".incbin \\\"${METALLIB_SOURCE}\\\"" >> ${EMBED_METALLIB_ASSEMBLY} + COMMAND echo ".globl _ggml_metallib_end" >> ${EMBED_METALLIB_ASSEMBLY} + COMMAND echo "_ggml_metallib_end:" >> ${EMBED_METALLIB_ASSEMBLY} + DEPENDS ${METALLIB_SOURCE} + COMMENT "Generate assembly for embedded Metal library" + ) + + set(GGML_SOURCES_METAL ${GGML_SOURCES_METAL} ${EMBED_METALLIB_ASSEMBLY}) + endif() + if (LLAMA_METAL_SHADER_DEBUG) # custom command to do the following: # xcrun -sdk macosx metal -fno-fast-math -c ggml-metal.metal -o ggml-metal.air diff --git a/Makefile b/Makefile index db5df1b32..211a08d7f 100644 --- a/Makefile +++ b/Makefile @@ -533,11 +533,29 @@ ifdef LLAMA_METAL ifdef LLAMA_METAL_NDEBUG MK_CPPFLAGS += -DGGML_METAL_NDEBUG endif +ifdef LLAMA_METAL_EMBED_LIBRARY + MK_CPPFLAGS += -DGGML_METAL_EMBED_LIBRARY + OBJS += ggml-metal-embed.o +endif endif # LLAMA_METAL ifdef LLAMA_METAL ggml-metal.o: ggml-metal.m ggml-metal.h $(CC) $(CFLAGS) -c $< -o $@ + +ifdef LLAMA_METAL_EMBED_LIBRARY +ggml-metal-embed.o: ggml-metal.metal + @echo "Embedding Metal library" + $(eval TEMP_ASSEMBLY=$(shell mktemp)) + @echo ".section __DATA, __ggml_metallib" > $(TEMP_ASSEMBLY) + @echo ".globl _ggml_metallib_start" >> $(TEMP_ASSEMBLY) + @echo "_ggml_metallib_start:" >> $(TEMP_ASSEMBLY) + @echo ".incbin \"$<\"" >> $(TEMP_ASSEMBLY) + @echo ".globl _ggml_metallib_end" >> $(TEMP_ASSEMBLY) + @echo "_ggml_metallib_end:" >> $(TEMP_ASSEMBLY) + @$(AS) $(TEMP_ASSEMBLY) -o $@ + @rm -f ${TEMP_ASSEMBLY} +endif endif # LLAMA_METAL ifdef LLAMA_MPI From 5207b3fbc500f89dfe528693e96540956dbaed96 Mon Sep 17 00:00:00 2001 From: Dane Madsen Date: Tue, 20 Feb 2024 21:00:23 +1100 Subject: [PATCH 412/812] readme : update UI list (#5605) * Add maid to ui list * Specify licence --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 70866e249..747d2e98b 100644 --- a/README.md +++ b/README.md @@ -156,6 +156,7 @@ Unless otherwise noted these projects are open-source with permissive licensing: - [pythops/tenere](https://github.com/pythops/tenere) (AGPL) - [semperai/amica](https://github.com/semperai/amica) - [withcatai/catai](https://github.com/withcatai/catai) +- [Mobile-Artificial-Intelligence/maid](https://github.com/Mobile-Artificial-Intelligence/maid) (MIT) --- From 9c405c9f9a7cfd23511fd6b2de05dc72481119b4 Mon Sep 17 00:00:00 2001 From: Xuan Son Nguyen Date: Tue, 20 Feb 2024 15:58:27 +0100 Subject: [PATCH 413/812] Server: use llama_chat_apply_template (#5593) * server: use llama_chat_apply_template * server: remove trailing space * server: fix format_chat * server: fix help message Co-authored-by: Georgi Gerganov * server: fix formatted_chat --------- Co-authored-by: Georgi Gerganov --- examples/server/oai.hpp | 6 ++-- examples/server/server.cpp | 17 +++++----- examples/server/utils.hpp | 69 ++++++++++++++++++-------------------- llama.cpp | 2 +- 4 files changed, 45 insertions(+), 49 deletions(-) diff --git a/examples/server/oai.hpp b/examples/server/oai.hpp index 2eca8a9fb..ff4ad6994 100644 --- a/examples/server/oai.hpp +++ b/examples/server/oai.hpp @@ -15,13 +15,11 @@ using json = nlohmann::json; inline static json oaicompat_completion_params_parse( + const struct llama_model * model, const json &body, /* openai api json semantics */ const std::string &chat_template) { json llama_params; - std::string formatted_prompt = chat_template == "chatml" - ? format_chatml(body["messages"]) // OpenAI 'messages' to chatml (with <|im_start|>,...) - : format_llama2(body["messages"]); // OpenAI 'messages' to llama2 (with [INST],...) llama_params["__oaicompat"] = true; @@ -34,7 +32,7 @@ inline static json oaicompat_completion_params_parse( // https://platform.openai.com/docs/api-reference/chat/create llama_sampling_params default_sparams; llama_params["model"] = json_value(body, "model", std::string("unknown")); - llama_params["prompt"] = formatted_prompt; + llama_params["prompt"] = format_chat(model, chat_template, body["messages"]); llama_params["cache_prompt"] = json_value(body, "cache_prompt", false); llama_params["temperature"] = json_value(body, "temperature", 0.0); llama_params["top_k"] = json_value(body, "top_k", default_sparams.top_k); diff --git a/examples/server/server.cpp b/examples/server/server.cpp index 23482ed95..c7821eca6 100644 --- a/examples/server/server.cpp +++ b/examples/server/server.cpp @@ -37,7 +37,7 @@ struct server_params std::string hostname = "127.0.0.1"; std::vector api_keys; std::string public_path = "examples/server/public"; - std::string chat_template = "chatml"; + std::string chat_template = ""; int32_t port = 8080; int32_t read_timeout = 600; int32_t write_timeout = 600; @@ -1937,8 +1937,9 @@ static void server_print_usage(const char *argv0, const gpt_params ¶ms, printf(" types: int, float, bool. example: --override-kv tokenizer.ggml.add_bos_token=bool:false\n"); printf(" -gan N, --grp-attn-n N set the group attention factor to extend context size through self-extend(default: 1=disabled), used together with group attention width `--grp-attn-w`"); printf(" -gaw N, --grp-attn-w N set the group attention width to extend context size through self-extend(default: 512), used together with group attention factor `--grp-attn-n`"); - printf(" --chat-template FORMAT_NAME"); - printf(" set chat template, possible value is: llama2, chatml (default %s)", sparams.chat_template.c_str()); + printf(" --chat-template JINJA_TEMPLATE\n"); + printf(" set custom jinja chat template (default: template taken from model's metadata)\n"); + printf(" Note: only commonly used templates are accepted, since we don't have jinja parser\n"); printf("\n"); } @@ -2389,13 +2390,13 @@ static void server_params_parse(int argc, char **argv, server_params &sparams, invalid_param = true; break; } - std::string value(argv[i]); - if (value != "chatml" && value != "llama2") { - fprintf(stderr, "error: chat template can be \"llama2\" or \"chatml\", but got: %s\n", value.c_str()); + if (!verify_custom_template(argv[i])) { + fprintf(stderr, "error: the supplied chat template is not supported: %s\n", argv[i]); + fprintf(stderr, "note: llama.cpp does not use jinja parser, we only support commonly used templates\n"); invalid_param = true; break; } - sparams.chat_template = value; + sparams.chat_template = argv[i]; } else if (arg == "--override-kv") { @@ -2913,7 +2914,7 @@ int main(int argc, char **argv) if (!validate_api_key(req, res)) { return; } - json data = oaicompat_completion_params_parse(json::parse(req.body), sparams.chat_template); + json data = oaicompat_completion_params_parse(llama.model, json::parse(req.body), sparams.chat_template); const int task_id = llama.queue_tasks.get_new_id(); llama.queue_results.add_waiting_task_id(task_id); diff --git a/examples/server/utils.hpp b/examples/server/utils.hpp index 0ee670dba..e954fb0ef 100644 --- a/examples/server/utils.hpp +++ b/examples/server/utils.hpp @@ -167,50 +167,47 @@ static T json_value(const json &body, const std::string &key, const T &default_v : default_value; } -inline std::string format_llama2(std::vector messages) -{ - std::ostringstream output; - bool is_inside_turn = false; - - for (auto it = messages.begin(); it != messages.end(); ++it) { - if (!is_inside_turn) { - output << "[INST] "; - } - std::string role = json_value(*it, "role", std::string("user")); - std::string content = json_value(*it, "content", std::string("")); - if (role == "system") { - output << "<>\n" << content << "\n<>\n\n"; - is_inside_turn = true; - } else if (role == "user") { - output << content << " [/INST]"; - is_inside_turn = true; - } else { - output << " " << content << " "; - is_inside_turn = false; - } - } - - LOG_VERBOSE("format_llama2", {{"text", output.str()}}); - - return output.str(); +// Check if the template supplied via "--chat-template" is supported or not. Returns true if it's valid +inline bool verify_custom_template(const std::string & tmpl) { + llama_chat_message chat[] = {{"user", "test"}}; + std::vector buf(1); + int res = llama_chat_apply_template(nullptr, tmpl.c_str(), chat, 1, true, buf.data(), buf.size()); + return res >= 0; } -inline std::string format_chatml(std::vector messages) +// Format given chat. If tmpl is empty, we take the template from model metadata +inline std::string format_chat(const struct llama_model * model, const std::string & tmpl, const std::vector & messages) { - std::ostringstream chatml_msgs; + size_t alloc_size = 0; + // vector holding all allocated string to be passed to llama_chat_apply_template + std::vector str(messages.size() * 2); + std::vector chat(messages.size()); - for (auto it = messages.begin(); it != messages.end(); ++it) { - chatml_msgs << "<|im_start|>" - << json_value(*it, "role", std::string("user")) << '\n'; - chatml_msgs << json_value(*it, "content", std::string("")) - << "<|im_end|>\n"; + for (size_t i = 0; i < messages.size(); ++i) { + auto &curr_msg = messages[i]; + str[i*2 + 0] = json_value(curr_msg, "role", std::string("")); + str[i*2 + 1] = json_value(curr_msg, "content", std::string("")); + alloc_size += str[i*2 + 1].length(); + chat[i].role = str[i*2 + 0].c_str(); + chat[i].content = str[i*2 + 1].c_str(); } - chatml_msgs << "<|im_start|>assistant" << '\n'; + const char * ptr_tmpl = tmpl.empty() ? nullptr : tmpl.c_str(); + std::vector buf(alloc_size * 2); - LOG_VERBOSE("format_chatml", {{"text", chatml_msgs.str()}}); + // run the first time to get the total output length + int32_t res = llama_chat_apply_template(model, ptr_tmpl, chat.data(), chat.size(), true, buf.data(), buf.size()); - return chatml_msgs.str(); + // if it turns out that our buffer is too small, we resize it + if ((size_t) res > buf.size()) { + buf.resize(res); + res = llama_chat_apply_template(model, ptr_tmpl, chat.data(), chat.size(), true, buf.data(), buf.size()); + } + + std::string formatted_chat(buf.data(), res); + LOG_VERBOSE("formatted_chat", {{"text", formatted_chat.c_str()}}); + + return formatted_chat; } // diff --git a/llama.cpp b/llama.cpp index 5de07dfa9..4296eca32 100644 --- a/llama.cpp +++ b/llama.cpp @@ -12602,7 +12602,7 @@ LLAMA_API int32_t llama_chat_apply_template( // load template from model std::vector model_template(2048, 0); // longest known template is about 1200 bytes std::string template_key = "tokenizer.chat_template"; - int32_t res = llama_model_meta_val_str(model, template_key.c_str(), model_template.data(), curr_tmpl.size()); + int32_t res = llama_model_meta_val_str(model, template_key.c_str(), model_template.data(), model_template.size()); if (res < 0) { // worst case: there is no information about template, we will use chatml by default curr_tmpl = "<|im_start|>"; // see llama_chat_apply_template_internal From 4ed8e4fbef6a15afd993bfcd9ffa279841e18ef1 Mon Sep 17 00:00:00 2001 From: Daniel Bevenius Date: Tue, 20 Feb 2024 18:30:27 +0100 Subject: [PATCH 414/812] llava : add explicit instructions for llava-1.6 (#5611) This commit contains a suggestion for the README.md in the llava example. The suggestion adds explicit instructions for how to convert a llava-1.6 model and run it using llava-cli. The motivation for this is that having explicit instructions similar to the 1.5 instructions will make it easier for users to try this out. Signed-off-by: Daniel Bevenius --- examples/llava/README.md | 38 ++++++++++++++++++++++++++++++++------ 1 file changed, 32 insertions(+), 6 deletions(-) diff --git a/examples/llava/README.md b/examples/llava/README.md index e42db6e5a..25ea96715 100644 --- a/examples/llava/README.md +++ b/examples/llava/README.md @@ -59,14 +59,40 @@ python ./convert.py ../llava-v1.5-7b --skip-unknown Now both the LLaMA part and the image encoder is in the `llava-v1.5-7b` directory. ## LLaVA 1.6 gguf conversion - -1) Backup your pth/safetensor model files as llava-surgery modifies them -2) Use `python llava-surgery-v2.py -C -m /path/to/hf-model` which also supports llava-1.5 variants pytorch as well as safetensor models: +1) First clone a LLaVA 1.6 model: +```console +git clone https://huggingface.co/liuhaotian/llava-v1.6-vicuna-7b +``` +2) Backup your pth/safetensor model files as llava-surgery modifies them +3) Use `llava-surgery-v2.py` which also supports llava-1.5 variants pytorch as well as safetensor models: +```console +python examples/llava/llava-surgery-v2.py -C -m ../llava-v1.6-vicuna-7b/ +``` - you will find a llava.projector and a llava.clip file in your model directory -3) Copy the llava.clip file into a subdirectory (like vit), rename it to pytorch_model.bin and add a fitting vit configuration to the directory (https://huggingface.co/cmp-nct/llava-1.6-gguf/blob/main/config_vit.json) and rename it to config.json. -4) Create the visual gguf model: `python ./examples/llava/convert-image-encoder-to-gguf.py -m ../path/to/vit --llava-projector ../path/to/llava.projector --output-dir ../path/to/output --clip-model-is-vision` +4) Copy the llava.clip file into a subdirectory (like vit), rename it to pytorch_model.bin and add a fitting vit configuration to the directory: +```console +mkdir vit +cp ../llava-v1.6-vicuna-7b/llava.clip vit/pytorch_model.bin +cp ../llava-v1.6-vicuna-7b/llava.projector vit/ +curl -s -q https://huggingface.co/cmp-nct/llava-1.6-gguf/raw/main/config_vit.json -o vit/config.json +``` + +5) Create the visual gguf model: +```console +python ./examples/llava/convert-image-encoder-to-gguf.py -m vit --llava-projector vit/llava.projector --output-dir vit --clip-model-is-vision +``` - This is similar to llava-1.5, the difference is that we tell the encoder that we are working with the pure vision model part of CLIP -5) Everything else as usual: convert.py the hf model, quantize as needed + +6) Then convert the model to gguf format: +```console +python ./convert.py ../llava-v1.6-vicuna-7b/ +``` + +7) And finally we can run the llava-cli using the 1.6 model version: +```console +./llava-cli -m ../llava-v1.6-vicuna-7b/ggml-model-f16.gguf --mmproj vit/mmproj-model-f16.gguf --image some-image.jpg -c 4096 +``` + **note** llava-1.6 needs more context than llava-1.5, at least 3000 is needed (just run it at -c 4096) **note** llava-1.6 greatly benefits from batched prompt processing (defaults work) From 06bf2cf8c406e6b70dbf9b431a02fa0ad845b9df Mon Sep 17 00:00:00 2001 From: slaren Date: Tue, 20 Feb 2024 20:06:17 +0100 Subject: [PATCH 415/812] make : fix debug build with CUDA (#5616) --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 211a08d7f..41c79c135 100644 --- a/Makefile +++ b/Makefile @@ -173,7 +173,7 @@ ifdef LLAMA_DEBUG MK_LDFLAGS += -g ifeq ($(UNAME_S),Linux) - MK_CXXFLAGS += -Wp,-D_GLIBCXX_ASSERTIONS + MK_CPPFLAGS += -D_GLIBCXX_ASSERTIONS endif else MK_CPPFLAGS += -DNDEBUG From 6560bed3f066c876682464762cad90f1e28e3f1b Mon Sep 17 00:00:00 2001 From: CJ Pais Date: Tue, 20 Feb 2024 11:07:22 -0800 Subject: [PATCH 416/812] server : support llava 1.6 (#5553) * server: init working 1.6 * move clip_image to header * remove commented code * remove c++ style from header * remove todo * expose llava_image_embed_make_with_clip_img * fix zig build --- Makefile | 2 +- build.zig | 3 ++- examples/llava/llava.cpp | 2 +- examples/llava/llava.h | 2 ++ examples/server/server.cpp | 36 +++--------------------------------- 5 files changed, 9 insertions(+), 36 deletions(-) diff --git a/Makefile b/Makefile index 41c79c135..f03faf6ed 100644 --- a/Makefile +++ b/Makefile @@ -719,7 +719,7 @@ save-load-state: examples/save-load-state/save-load-state.cpp ggml.o llama.o $(C $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) -server: examples/server/server.cpp examples/server/oai.hpp examples/server/utils.hpp examples/server/httplib.h examples/server/json.hpp examples/server/index.html.hpp examples/server/index.js.hpp examples/server/completion.js.hpp examples/llava/clip.cpp examples/llava/clip.h common/stb_image.h ggml.o llama.o $(COMMON_DEPS) grammar-parser.o $(OBJS) +server: examples/server/server.cpp examples/server/oai.hpp examples/server/utils.hpp examples/server/httplib.h examples/server/json.hpp examples/server/index.html.hpp examples/server/index.js.hpp examples/server/completion.js.hpp examples/llava/clip.cpp examples/llava/clip.h examples/llava/llava.h examples/llava/llava.cpp common/stb_image.h ggml.o llama.o $(COMMON_DEPS) grammar-parser.o $(OBJS) $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) $(CXX) $(CXXFLAGS) -c examples/llava/clip.cpp -o $(call GET_OBJ_FILE, examples/llava/clip.cpp) -Wno-cast-qual $(CXX) $(CXXFLAGS) -Iexamples/server $(filter-out %.h %.hpp $< examples/llava/clip.cpp,$^) $(call GET_OBJ_FILE, $<) $(call GET_OBJ_FILE, examples/llava/clip.cpp) -o $@ $(LDFLAGS) $(LWINSOCK2) diff --git a/build.zig b/build.zig index 699738f3d..c0af454dc 100644 --- a/build.zig +++ b/build.zig @@ -123,6 +123,7 @@ pub fn build(b: *std.build.Builder) !void { const grammar_parser = make.obj("grammar-parser", "common/grammar-parser.cpp"); const train = make.obj("train", "common/train.cpp"); const clip = make.obj("clip", "examples/llava/clip.cpp"); + const llava = make.obj("llava", "examples/llava/llava.cpp"); _ = make.exe("main", "examples/main/main.cpp", &.{ ggml, ggml_alloc, ggml_backend, ggml_quants, llama, common, buildinfo, sampling, console, grammar_parser }); _ = make.exe("quantize", "examples/quantize/quantize.cpp", &.{ ggml, ggml_alloc, ggml_backend, ggml_quants, llama, common, buildinfo }); @@ -131,7 +132,7 @@ pub fn build(b: *std.build.Builder) !void { _ = make.exe("finetune", "examples/finetune/finetune.cpp", &.{ ggml, ggml_alloc, ggml_backend, ggml_quants, llama, common, buildinfo, train }); _ = make.exe("train-text-from-scratch", "examples/train-text-from-scratch/train-text-from-scratch.cpp", &.{ ggml, ggml_alloc, ggml_backend, ggml_quants, llama, common, buildinfo, train }); - const server = make.exe("server", "examples/server/server.cpp", &.{ ggml, ggml_alloc, ggml_backend, ggml_quants, llama, common, buildinfo, sampling, grammar_parser, clip }); + const server = make.exe("server", "examples/server/server.cpp", &.{ ggml, ggml_alloc, ggml_backend, ggml_quants, llama, common, buildinfo, sampling, grammar_parser, clip, llava }); if (server.target.isWindows()) { server.linkSystemLibrary("ws2_32"); } diff --git a/examples/llava/llava.cpp b/examples/llava/llava.cpp index 4cb65a07b..1a1cf7c78 100644 --- a/examples/llava/llava.cpp +++ b/examples/llava/llava.cpp @@ -311,7 +311,7 @@ bool llava_validate_embed_size(const llama_context * ctx_llama, const clip_ctx * return true; } -static bool llava_image_embed_make_with_clip_img(clip_ctx * ctx_clip, int n_threads, const clip_image_u8 * img, float ** image_embd_out, int * n_img_pos_out) { +bool llava_image_embed_make_with_clip_img(clip_ctx * ctx_clip, int n_threads, const clip_image_u8 * img, float ** image_embd_out, int * n_img_pos_out) { float * image_embd = (float *)malloc(clip_embd_nbytes(ctx_clip)*6); // TODO: base on gridsize/llava model if (!image_embd) { fprintf(stderr, "Unable to allocate memory for image embeddings\n"); diff --git a/examples/llava/llava.h b/examples/llava/llava.h index 9e9466a5d..2d40f3f1d 100644 --- a/examples/llava/llava.h +++ b/examples/llava/llava.h @@ -31,6 +31,8 @@ struct llava_image_embed { /** sanity check for clip <-> llava embed size match */ LLAVA_API bool llava_validate_embed_size(const llama_context * ctx_llama, const clip_ctx * ctx_clip); +LLAVA_API bool llava_image_embed_make_with_clip_img(clip_ctx * ctx_clip, int n_threads, const clip_image_u8 * img, float ** image_embd_out, int * n_img_pos_out); + /** build an image embed from image file bytes */ LLAVA_API struct llava_image_embed * llava_image_embed_make_with_bytes(struct clip_ctx * ctx_clip, int n_threads, const unsigned char * image_bytes, int image_bytes_length); /** build an image embed from a path to an image filename */ diff --git a/examples/server/server.cpp b/examples/server/server.cpp index c7821eca6..eb01729fa 100644 --- a/examples/server/server.cpp +++ b/examples/server/server.cpp @@ -5,6 +5,7 @@ #include "oai.hpp" #include "../llava/clip.h" +#include "../llava/llava.h" #include "stb_image.h" @@ -997,43 +998,12 @@ struct llama_server_context { continue; } - clip_image_f32_batch img_res_v; - img_res_v.size = 0; - img_res_v.data = nullptr; - if (!clip_image_preprocess(clp_ctx, img.img_data, img_res_v)) - { - LOG_TEE("Error processing the given image"); - clip_free(clp_ctx); - clip_image_f32_batch_free(img_res_v); - return false; - } - if (img_res_v.size == 0) - { + + if (!llava_image_embed_make_with_clip_img(clp_ctx, params.n_threads, img.img_data, &img.image_embedding, &img.image_tokens)) { LOG_TEE("Error processing the given image"); return false; } - // note: assumes only one image was returned by clip_image_preprocess - clip_image_f32 * img_res = img_res_v.data; - - img.image_tokens = clip_n_patches(clp_ctx); - img.image_embedding = (float *)malloc(clip_embd_nbytes(clp_ctx)); - if (!img.image_embedding) - { - LOG_TEE("Unable to allocate memory for image embeddings\n"); - clip_image_f32_batch_free(img_res_v); - clip_free(clp_ctx); - return false; - } - LOG_TEE("slot %i - encoding image [id: %i]\n", slot.id, img.id); - if (!clip_image_encode(clp_ctx, params.n_threads, img_res, img.image_embedding)) - { - LOG_TEE("Unable to encode image\n"); - clip_image_f32_batch_free(img_res_v); - return false; - } - - clip_image_f32_batch_free(img_res_v); img.request_encode_image = false; } From a14679cc30c785e75d38028bae6ec39c6209ddef Mon Sep 17 00:00:00 2001 From: Kawrakow <48489457+ikawrakow@users.noreply.github.com> Date: Wed, 21 Feb 2024 11:39:52 +0200 Subject: [PATCH 417/812] IQ4_NL: 4-bit non-linear quants with blocks of 32 (#5590) * iq4_nl: squash commits for easier rebase * Basics (quantize, dequantize) * CUDA dequantize and dot product * Slightly faster CUDA dot product (120 t/s) * Switch to 6-bit scales * Scalar dot product * AVX2 dot product * ARM_NEON dot product * Works on metal, but still slow * Slightly better Metal dot product * Another small Metal improvement * Metal dot product is getting there * Faster CUDA dot product * Add 1/8 ffn_down layers as Q5_K when no imatrix has been provided * Report the actual bpw * Add _xs mix that is 4.05 bpw for non-MoE models * Remove IQ4_XS for now, slightly adjust kvalues_iq4nl * AVX2 dot product uses Q8_0 instead of Q8_K * Add to test-backend-ops * Minor fix * Also use use Q5_K for attn_output in MoE models * Fixes after merging latest master * Switching to blocks of 32 * AVX2 for blocks of 32 * Scaler dot product for blocks of 32 * ARM_NEON dot product for blocks of 32 * Metal kernels for blocks of 32 * Slightly faster Metal kernels * iq4_nl: Fix after merging with master * iq4_nl: another fix after merging with master * Use IQ4_NL instead of Q4_K when using k-quants is not possible * Fix typo that makes several tests fail * It was the ggml_vdotq thing missed inside the brackets --------- Co-authored-by: Iwan Kawrakow --- examples/quantize/quantize.cpp | 1 + ggml-cuda.cu | 98 +++++++++++++- ggml-metal.m | 35 +++++ ggml-metal.metal | 215 +++++++++++++++++++++++++++++- ggml-quants.c | 234 ++++++++++++++++++++++++++++++++- ggml-quants.h | 13 ++ ggml.c | 30 +++++ ggml.h | 2 + llama.cpp | 17 ++- llama.h | 1 + tests/test-backend-ops.cpp | 1 + 11 files changed, 640 insertions(+), 7 deletions(-) diff --git a/examples/quantize/quantize.cpp b/examples/quantize/quantize.cpp index ea7ba50c9..37520857f 100644 --- a/examples/quantize/quantize.cpp +++ b/examples/quantize/quantize.cpp @@ -32,6 +32,7 @@ static const std::vector QUANT_OPTIONS = { { "Q3_K_S", LLAMA_FTYPE_MOSTLY_Q3_K_S, " 2.75G, +0.5551 ppl @ LLaMA-v1-7B", }, { "Q3_K_M", LLAMA_FTYPE_MOSTLY_Q3_K_M, " 3.07G, +0.2496 ppl @ LLaMA-v1-7B", }, { "Q3_K_L", LLAMA_FTYPE_MOSTLY_Q3_K_L, " 3.35G, +0.1764 ppl @ LLaMA-v1-7B", }, + { "IQ4_NL", LLAMA_FTYPE_MOSTLY_IQ4_NL, " 4.25 bpw non-linear quantization", }, { "Q4_K", LLAMA_FTYPE_MOSTLY_Q4_K_M, "alias for Q4_K_M", }, { "Q4_K_S", LLAMA_FTYPE_MOSTLY_Q4_K_S, " 3.59G, +0.0992 ppl @ LLaMA-v1-7B", }, { "Q4_K_M", LLAMA_FTYPE_MOSTLY_Q4_K_M, " 3.80G, +0.0532 ppl @ LLaMA-v1-7B", }, diff --git a/ggml-cuda.cu b/ggml-cuda.cu index 6caae56b0..e7c211d7d 100644 --- a/ggml-cuda.cu +++ b/ggml-cuda.cu @@ -528,6 +528,15 @@ typedef struct { } block_iq1_s; static_assert(sizeof(block_iq1_s) == sizeof(ggml_fp16_t) + QK_K/8 + QK_K/16, "wrong iq1_s block size/padding"); +#define QK4_NL 32 +#define QR4_NL 2 +#define QI4_NL (QK4_NL / (4*QR4_NL)) +typedef struct { + half d; + uint8_t qs[QK4_NL/2]; +} block_iq4_nl; +static_assert(sizeof(block_iq4_nl) == sizeof(ggml_fp16_t) + QK4_NL/2, "wrong iq4_nl block size/padding"); + #define WARP_SIZE 32 #define MATRIX_ROW_PADDING 512 // last row of quant. matrices is a multiple of this to avoid out-of-bounds memory accesses @@ -1987,6 +1996,26 @@ static __global__ void dequantize_block_iq1_s(const void * __restrict__ vx, dst_ } +static const __device__ int8_t kvalues_iq4nl[16] = {-127, -104, -83, -65, -49, -35, -22, -10, 1, 13, 25, 38, 53, 69, 89, 113}; + +template +static __global__ void dequantize_block_iq4_nl(const void * __restrict__ vx, dst_t * __restrict__ yy) { + + const int i = blockIdx.x; + const block_iq4_nl * x = (const block_iq4_nl *) vx + i*(QK_K/QK4_NL); + + const int tid = threadIdx.x; + const int il = tid/8; // 0...3 + const int ib = tid%8; // 0...7 + dst_t * y = yy + i*QK_K + 32*ib + 4*il; + const uint8_t * q4 = x[ib].qs + 4*il; + const float d = (float)x[ib].d; + for (int j = 0; j < 4; ++j) { + y[j+ 0] = d * kvalues_iq4nl[q4[j] & 0xf]; + y[j+16] = d * kvalues_iq4nl[q4[j] >> 4]; + } + +} static __global__ void dequantize_mul_mat_vec_q2_k(const void * __restrict__ vx, const float * __restrict__ yy, float * __restrict__ dst, const int ncols, int nrows) { @@ -4732,6 +4761,56 @@ static __device__ __forceinline__ float vec_dot_iq1_s_q8_1( #endif } +#if __CUDA_ARCH__ >= MIN_CC_DP4A // lowest compute capability for integer intrinsics +static __device__ __forceinline__ void get_int_from_table_16(const uint32_t & q4, const uint8_t * values, + int & val1, int & val2) { + + uint32_t aux32; const uint8_t * q8 = (const uint8_t *)&aux32; + aux32 = q4 & 0x0f0f0f0f; + uint16_t v1 = values[q8[0]] | (values[q8[1]] << 8); + uint16_t v2 = values[q8[2]] | (values[q8[3]] << 8); + val1 = v1 | (v2 << 16); + aux32 = (q4 >> 4) & 0x0f0f0f0f; + v1 = values[q8[0]] | (values[q8[1]] << 8); + v2 = values[q8[2]] | (values[q8[3]] << 8); + val2 = v1 | (v2 << 16); +} +#endif + +static __device__ __forceinline__ float vec_dot_iq4_nl_q8_1( + const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) { + + const block_iq4_nl * bq = (const block_iq4_nl *) vbq; + +#if __CUDA_ARCH__ >= MIN_CC_DP4A // lowest compute capability for integer intrinsics + const uint16_t * q4 = (const uint16_t *)bq->qs + 2*iqs; + const int32_t * q8 = (const int32_t *)bq8_1->qs + iqs; + + const uint8_t * values = (const uint8_t *)kvalues_iq4nl; + + int v1, v2; + int sumi1 = 0, sumi2 = 0; + for (int l = 0; l < VDR_Q4_0_Q8_1_MMVQ; ++l) { + const uint32_t aux = q4[2*l] | (q4[2*l+1] << 16); + get_int_from_table_16(aux, values, v1, v2); + sumi1 = __dp4a(v1, q8[l+0], sumi1); + sumi2 = __dp4a(v2, q8[l+4], sumi2); + } + +#else + const uint8_t * q4 = bq->qs + 4*iqs; + const int8_t * q8 = bq8_1->qs + 4*iqs; + + int sumi1 = 0, sumi2 = 0; + for (int l = 0; l < 4*VDR_Q4_0_Q8_1_MMVQ; ++l) { + sumi1 += q8[l+ 0] * kvalues_iq4nl[q4[l] & 0xf]; + sumi2 += q8[l+16] * kvalues_iq4nl[q4[l] >> 4]; + } +#endif + const float d = (float)bq->d * __low2float(bq8_1->ds); + return d * (sumi1 + sumi2); +} + template static __device__ __forceinline__ void mul_mat_q( @@ -6777,6 +6856,12 @@ static void dequantize_row_iq1_s_cuda(const void * vx, dst_t * y, const int k, c dequantize_block_iq1_s<<>>(vx, y); } +template +static void dequantize_row_iq4_nl_cuda(const void * vx, dst_t * y, const int k, cudaStream_t stream) { + const int nb = (k + QK_K - 1) / QK_K; + dequantize_block_iq4_nl<<>>(vx, y); +} + template static void convert_unary_cuda(const void * __restrict__ vx, dst_t * __restrict__ y, const int k, cudaStream_t stream) { const int num_blocks = (k + CUDA_DEQUANTIZE_BLOCK_SIZE - 1) / CUDA_DEQUANTIZE_BLOCK_SIZE; @@ -6818,6 +6903,8 @@ static to_fp16_cuda_t ggml_get_to_fp16_cuda(ggml_type type) { return dequantize_row_iq3_xxs_cuda; case GGML_TYPE_IQ1_S: return dequantize_row_iq1_s_cuda; + case GGML_TYPE_IQ4_NL: + return dequantize_row_iq4_nl_cuda; case GGML_TYPE_F32: return convert_unary_cuda; default: @@ -6855,6 +6942,8 @@ static to_fp32_cuda_t ggml_get_to_fp32_cuda(ggml_type type) { return dequantize_row_iq3_xxs_cuda; case GGML_TYPE_IQ1_S: return dequantize_row_iq1_s_cuda; + case GGML_TYPE_IQ4_NL: + return dequantize_row_iq4_nl_cuda; case GGML_TYPE_F16: return convert_unary_cuda; default: @@ -8599,6 +8688,7 @@ static int64_t get_row_rounding(ggml_type type, const std::array= CC_RDNA2 ? 128 : 64; default: GGML_ASSERT(false); @@ -8623,6 +8713,7 @@ static int64_t get_row_rounding(ggml_type type, const std::array= CC_VOLTA ? 128 : 64; case GGML_TYPE_Q6_K: return 64; @@ -8724,6 +8815,10 @@ static void ggml_cuda_op_mul_mat_vec_q( mul_mat_vec_q_cuda (src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_padded_row_size, src1_ncols, nrows_dst, stream); break; + case GGML_TYPE_IQ4_NL: + mul_mat_vec_q_cuda + (src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_padded_row_size, src1_ncols, nrows_dst, stream); + break; default: GGML_ASSERT(false); break; @@ -11446,7 +11541,8 @@ GGML_CALL static bool ggml_backend_cuda_supports_op(ggml_backend_t backend, cons return false; } ggml_type a_type = a->type; - if (a_type == GGML_TYPE_IQ2_XXS || a_type == GGML_TYPE_IQ2_XS || a_type == GGML_TYPE_IQ3_XXS || a_type == GGML_TYPE_IQ1_S) { + if (a_type == GGML_TYPE_IQ2_XXS || a_type == GGML_TYPE_IQ2_XS || a_type == GGML_TYPE_IQ3_XXS || + a_type == GGML_TYPE_IQ1_S || a_type == GGML_TYPE_IQ4_NL) { if (b->ne[1] == 1 && ggml_nrows(b) > 1) { return false; } diff --git a/ggml-metal.m b/ggml-metal.m index 956e323a0..0d4aa4309 100644 --- a/ggml-metal.m +++ b/ggml-metal.m @@ -62,6 +62,7 @@ enum ggml_metal_kernel_type { GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ2_XS, GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ3_XXS, GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ1_S, + GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ4_NL, GGML_METAL_KERNEL_TYPE_GET_ROWS_I32, GGML_METAL_KERNEL_TYPE_RMS_NORM, GGML_METAL_KERNEL_TYPE_GROUP_NORM, @@ -85,6 +86,7 @@ enum ggml_metal_kernel_type { GGML_METAL_KERNEL_TYPE_MUL_MV_IQ2_XS_F32, GGML_METAL_KERNEL_TYPE_MUL_MV_IQ3_XXS_F32, GGML_METAL_KERNEL_TYPE_MUL_MV_IQ1_S_F32, + GGML_METAL_KERNEL_TYPE_MUL_MV_IQ4_NL_F32, GGML_METAL_KERNEL_TYPE_MUL_MV_ID_F32_F32, //GGML_METAL_KERNEL_TYPE_MUL_MV_ID_F16_F16, GGML_METAL_KERNEL_TYPE_MUL_MV_ID_F16_F32, @@ -104,6 +106,7 @@ enum ggml_metal_kernel_type { GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ2_XS_F32, GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ3_XXS_F32, GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ1_S_F32, + GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ4_NL_F32, GGML_METAL_KERNEL_TYPE_MUL_MM_F32_F32, GGML_METAL_KERNEL_TYPE_MUL_MM_F16_F32, GGML_METAL_KERNEL_TYPE_MUL_MM_Q4_0_F32, @@ -120,6 +123,7 @@ enum ggml_metal_kernel_type { GGML_METAL_KERNEL_TYPE_MUL_MM_IQ2_XS_F32, GGML_METAL_KERNEL_TYPE_MUL_MM_IQ3_XXS_F32, GGML_METAL_KERNEL_TYPE_MUL_MM_IQ1_S_F32, + GGML_METAL_KERNEL_TYPE_MUL_MM_IQ4_NL_F32, GGML_METAL_KERNEL_TYPE_MUL_MM_ID_F32_F32, GGML_METAL_KERNEL_TYPE_MUL_MM_ID_F16_F32, GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q4_0_F32, @@ -136,6 +140,7 @@ enum ggml_metal_kernel_type { GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ2_XS_F32, GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ3_XXS_F32, GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ1_S_F32, + GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ4_NL_F32, GGML_METAL_KERNEL_TYPE_ROPE_F32, GGML_METAL_KERNEL_TYPE_ROPE_F16, GGML_METAL_KERNEL_TYPE_ALIBI_F32, @@ -448,6 +453,7 @@ static struct ggml_metal_context * ggml_metal_init(int n_cb) { GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ2_XS, get_rows_iq2_xs, true); GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ3_XXS, get_rows_iq3_xxs, true); GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ1_S, get_rows_iq1_s, true); + GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ4_NL, get_rows_iq4_nl, true); GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_I32, get_rows_i32, true); GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_RMS_NORM, rms_norm, ctx->support_simdgroup_reduction); GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GROUP_NORM, group_norm, ctx->support_simdgroup_reduction); @@ -471,6 +477,7 @@ static struct ggml_metal_context * ggml_metal_init(int n_cb) { GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_IQ2_XS_F32, mul_mv_iq2_xs_f32, ctx->support_simdgroup_reduction); GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_IQ3_XXS_F32, mul_mv_iq3_xxs_f32, ctx->support_simdgroup_reduction); GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_IQ1_S_F32, mul_mv_iq1_s_f32, ctx->support_simdgroup_reduction); + GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_IQ4_NL_F32, mul_mv_iq4_nl_f32, ctx->support_simdgroup_reduction); GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_F32_F32, mul_mv_id_f32_f32, ctx->support_simdgroup_reduction); //GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_F16_F16, mul_mv_id_f16_f16, ctx->support_simdgroup_reduction); GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_F16_F32, mul_mv_id_f16_f32, ctx->support_simdgroup_reduction); @@ -490,6 +497,7 @@ static struct ggml_metal_context * ggml_metal_init(int n_cb) { GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ2_XS_F32, mul_mv_id_iq2_xs_f32, ctx->support_simdgroup_reduction); GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ3_XXS_F32, mul_mv_id_iq3_xxs_f32, ctx->support_simdgroup_reduction); GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ1_S_F32, mul_mv_id_iq1_s_f32, ctx->support_simdgroup_reduction); + GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ4_NL_F32, mul_mv_id_iq4_nl_f32, ctx->support_simdgroup_reduction); GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_F32_F32, mul_mm_f32_f32, ctx->support_simdgroup_mm); GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_F16_F32, mul_mm_f16_f32, ctx->support_simdgroup_mm); GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_Q4_0_F32, mul_mm_q4_0_f32, ctx->support_simdgroup_mm); @@ -506,6 +514,7 @@ static struct ggml_metal_context * ggml_metal_init(int n_cb) { GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_IQ2_XS_F32, mul_mm_iq2_xs_f32, ctx->support_simdgroup_mm); GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_IQ3_XXS_F32, mul_mm_iq3_xxs_f32, ctx->support_simdgroup_mm); GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_IQ1_S_F32, mul_mm_iq1_s_f32, ctx->support_simdgroup_mm); + GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_IQ4_NL_F32, mul_mm_iq4_nl_f32, ctx->support_simdgroup_mm); GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_F32_F32, mul_mm_id_f32_f32, ctx->support_simdgroup_mm); GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_F16_F32, mul_mm_id_f16_f32, ctx->support_simdgroup_mm); GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q4_0_F32, mul_mm_id_q4_0_f32, ctx->support_simdgroup_mm); @@ -522,6 +531,7 @@ static struct ggml_metal_context * ggml_metal_init(int n_cb) { GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ2_XS_F32, mul_mm_id_iq2_xs_f32, ctx->support_simdgroup_mm); GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ3_XXS_F32, mul_mm_id_iq3_xxs_f32, ctx->support_simdgroup_mm); GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ1_S_F32, mul_mm_id_iq1_s_f32, ctx->support_simdgroup_mm); + GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ4_NL_F32, mul_mm_id_iq4_nl_f32, ctx->support_simdgroup_mm); GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_ROPE_F32, rope_f32, true); GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_ROPE_F16, rope_f16, true); GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_ALIBI_F32, alibi_f32, true); @@ -1338,6 +1348,7 @@ static bool ggml_metal_graph_compute( case GGML_TYPE_IQ2_XS: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_IQ2_XS_F32 ].pipeline; break; case GGML_TYPE_IQ3_XXS: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_IQ3_XXS_F32].pipeline; break; case GGML_TYPE_IQ1_S: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_IQ1_S_F32 ].pipeline; break; + case GGML_TYPE_IQ4_NL: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_IQ4_NL_F32 ].pipeline; break; default: GGML_ASSERT(false && "MUL MAT-MAT not implemented"); } @@ -1478,6 +1489,12 @@ static bool ggml_metal_graph_compute( nth1 = 16; pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_IQ1_S_F32].pipeline; } break; + case GGML_TYPE_IQ4_NL: + { + nth0 = 4; + nth1 = 16; + pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_IQ4_NL_F32].pipeline; + } break; default: { GGML_METAL_LOG_ERROR("Asserting on type %d\n", (int)src0t); @@ -1525,6 +1542,11 @@ static bool ggml_metal_graph_compute( [encoder setThreadgroupMemoryLength:mem_size atIndex:0]; [encoder dispatchThreadgroups:MTLSizeMake((ne01 + 7)/8, ne11, ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)]; } + else if (src0t == GGML_TYPE_IQ4_NL) { + const int mem_size = 32*sizeof(float); + [encoder setThreadgroupMemoryLength:mem_size atIndex:0]; + [encoder dispatchThreadgroups:MTLSizeMake((ne01 + 3)/4, ne11, ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)]; + } else if (src0t == GGML_TYPE_Q4_K) { [encoder dispatchThreadgroups:MTLSizeMake((ne01 + 3)/4, ne11, ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)]; } @@ -1619,6 +1641,7 @@ static bool ggml_metal_graph_compute( case GGML_TYPE_IQ2_XS: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ2_XS_F32 ].pipeline; break; case GGML_TYPE_IQ3_XXS: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ3_XXS_F32].pipeline; break; case GGML_TYPE_IQ1_S: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ1_S_F32 ].pipeline; break; + case GGML_TYPE_IQ4_NL: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ4_NL_F32 ].pipeline; break; default: GGML_ASSERT(false && "MUL_MAT_ID not implemented"); } @@ -1762,6 +1785,12 @@ static bool ggml_metal_graph_compute( nth1 = 16; pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ1_S_F32].pipeline; } break; + case GGML_TYPE_IQ4_NL: + { + nth0 = 4; + nth1 = 16; + pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ4_NL_F32].pipeline; + } break; default: { GGML_METAL_LOG_ERROR("Asserting on type %d\n", (int)src2t); @@ -1825,6 +1854,11 @@ static bool ggml_metal_graph_compute( [encoder setThreadgroupMemoryLength:mem_size atIndex:0]; [encoder dispatchThreadgroups:MTLSizeMake((ne21 + 7)/8, _ne1, ne01*ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)]; } + else if (src2t == GGML_TYPE_IQ4_NL) { + const int mem_size = 32*sizeof(float); + [encoder setThreadgroupMemoryLength:mem_size atIndex:0]; + [encoder dispatchThreadgroups:MTLSizeMake((ne21 + 3)/4, _ne1, ne01*ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)]; + } else if (src2t == GGML_TYPE_Q4_K) { [encoder dispatchThreadgroups:MTLSizeMake((ne21 + 3)/4, _ne1, ne01*ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)]; } @@ -1867,6 +1901,7 @@ static bool ggml_metal_graph_compute( case GGML_TYPE_IQ2_XS: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ2_XS ].pipeline; break; case GGML_TYPE_IQ3_XXS: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ3_XXS].pipeline; break; case GGML_TYPE_IQ1_S: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ1_S ].pipeline; break; + case GGML_TYPE_IQ4_NL: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ4_NL ].pipeline; break; case GGML_TYPE_I32: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GET_ROWS_I32 ].pipeline; break; default: GGML_ASSERT(false && "not implemented"); } diff --git a/ggml-metal.metal b/ggml-metal.metal index f0d77d446..c223a981c 100644 --- a/ggml-metal.metal +++ b/ggml-metal.metal @@ -2531,6 +2531,12 @@ typedef struct { uint8_t scales[QK_K/16]; } block_iq1_s; +// Non-linear quants +#define QK4_NL 32 +typedef struct { + half d; + uint8_t qs[QK4_NL/2]; +} block_iq4_nl; //====================================== dot products ========================= @@ -4384,7 +4390,6 @@ void kernel_mul_mv_iq1_s_f32_impl( const uint i13 = im/ne12; const uint offset0 = (i12/r2)*(nb*ne01) + (i13/r3)*(nb*ne01*ne02); - device const block_iq1_s * x = (device const block_iq1_s *) src0 + ib_row + offset0; device const float * y = (device const float *) src1 + r1*ne10 + im*ne00*ne1; @@ -4447,6 +4452,103 @@ void kernel_mul_mv_iq1_s_f32_impl( } } +constexpr constant static float kvalues_iq4nl_f[16] = { + -127.f, -104.f, -83.f, -65.f, -49.f, -35.f, -22.f, -10.f, 1.f, 13.f, 25.f, 38.f, 53.f, 69.f, 89.f, 113.f +}; + +void kernel_mul_mv_iq4_nl_f32_impl( + device const void * src0, + device const float * src1, + device float * dst, + constant int64_t & ne00, + constant int64_t & ne01, + constant int64_t & ne02, + constant int64_t & ne10, + constant int64_t & ne12, + constant int64_t & ne0, + constant int64_t & ne1, + constant uint & r2, + constant uint & r3, + threadgroup float * shared_values [[threadgroup(0)]], + uint3 tgpig[[threadgroup_position_in_grid]], + uint tiisg[[thread_index_in_simdgroup]], + uint sgitg[[simdgroup_index_in_threadgroup]]) { + + const int nb = ne00/QK4_NL; + const int r0 = tgpig.x; + const int r1 = tgpig.y; + const int im = tgpig.z; + const int first_row = (r0 * 2 + sgitg) * 2; + const int ib_row = first_row * nb; + + const uint i12 = im%ne12; + const uint i13 = im/ne12; + + const uint offset0 = (i12/r2)*(nb*ne01) + (i13/r3)*(nb*ne01*ne02); + device const block_iq4_nl * x = (device const block_iq4_nl *) src0 + ib_row + offset0; + device const float * y = (device const float *) src1 + r1*ne10 + im*ne00*ne1; + + const int ix = tiisg/2; // 0...15 + const int it = tiisg%2; // 0 or 1 + + shared_values[tiisg] = kvalues_iq4nl_f[tiisg%16]; + threadgroup_barrier(mem_flags::mem_threadgroup); + + float4 yl[4]; + float sumf[2]={0.f}, all_sum; + + device const float * yb = y + ix * QK4_NL + it * 8; + + uint32_t aux32[2]; + thread const uint8_t * q8 = (thread const uint8_t *)aux32; + + float4 qf1, qf2; + + for (int ib = ix; ib < nb; ib += 16) { + + device const float4 * y4 = (device const float4 *)yb; + yl[0] = y4[0]; yl[1] = y4[4]; yl[2] = y4[1]; yl[3] = y4[5]; + + for (int row = 0; row < 2; ++row) { + + device const block_iq4_nl & xb = x[row*nb + ib]; + device const uint16_t * q4 = (device const uint16_t *)(xb.qs + 8*it); + + float4 acc1 = {0.f}, acc2 = {0.f}; + + aux32[0] = q4[0] | (q4[1] << 16); + aux32[1] = (aux32[0] >> 4) & 0x0f0f0f0f; + aux32[0] &= 0x0f0f0f0f; + qf1 = {shared_values[q8[0]], shared_values[q8[1]], shared_values[q8[2]], shared_values[q8[3]]}; + qf2 = {shared_values[q8[4]], shared_values[q8[5]], shared_values[q8[6]], shared_values[q8[7]]}; + acc1 += yl[0] * qf1; + acc2 += yl[1] * qf2; + + aux32[0] = q4[2] | (q4[3] << 16); + aux32[1] = (aux32[0] >> 4) & 0x0f0f0f0f; + aux32[0] &= 0x0f0f0f0f; + qf1 = {shared_values[q8[0]], shared_values[q8[1]], shared_values[q8[2]], shared_values[q8[3]]}; + qf2 = {shared_values[q8[4]], shared_values[q8[5]], shared_values[q8[6]], shared_values[q8[7]]}; + acc1 += yl[2] * qf1; + acc2 += yl[3] * qf2; + + acc1 += acc2; + + sumf[row] += (float)xb.d * (acc1[0] + acc1[1] + acc1[2] + acc1[3]); + + } + + yb += 16 * QK4_NL; + } + + for (int row = 0; row < 2; ++row) { + all_sum = simd_sum(sumf[row]); + if (tiisg == 0) { + dst[r1*ne0 + im*ne0*ne1 + first_row + row] = all_sum; + } + } +} + [[host_name("kernel_mul_mv_iq1_s_f32")]] kernel void kernel_mul_mv_iq1_s_f32( device const void * src0, @@ -4475,6 +4577,34 @@ kernel void kernel_mul_mv_iq1_s_f32( kernel_mul_mv_iq1_s_f32_impl(src0, src1, dst, ne00, ne01, ne02, ne10, ne12, ne0, ne1, r2, r3, tgpig, tiisg, sgitg); } +[[host_name("kernel_mul_mv_iq4_nl_f32")]] +kernel void kernel_mul_mv_iq4_nl_f32( + device const void * src0, + device const float * src1, + device float * dst, + constant int64_t & ne00, + constant int64_t & ne01, + constant int64_t & ne02, + constant uint64_t & nb00, + constant uint64_t & nb01, + constant uint64_t & nb02, + constant int64_t & ne10, + constant int64_t & ne11, + constant int64_t & ne12, + constant uint64_t & nb10, + constant uint64_t & nb11, + constant uint64_t & nb12, + constant int64_t & ne0, + constant int64_t & ne1, + constant uint & r2, + constant uint & r3, + threadgroup float * shared_values [[threadgroup(0)]], + uint3 tgpig[[threadgroup_position_in_grid]], + uint tiisg[[thread_index_in_simdgroup]], + uint sgitg[[simdgroup_index_in_threadgroup]]) { + + kernel_mul_mv_iq4_nl_f32_impl(src0, src1, dst, ne00, ne01, ne02, ne10, ne12, ne0, ne1, r2, r3, shared_values, tgpig, tiisg, sgitg); +} //============================= templates and their specializations ============================= @@ -4838,6 +4968,21 @@ void dequantize_iq1_s(device const block_iq1_s * xb, short il, thread type4x4 & } } +template +void dequantize_iq4_nl(device const block_iq4_nl * xb, short il, thread type4x4 & reg) { + device const uint16_t * q4 = (device const uint16_t *)xb->qs; + const float d = xb->d; + uint32_t aux32; + thread const uint8_t * q8 = (thread const uint8_t *)&aux32; + for (int i = 0; i < 4; ++i) { + aux32 = ((q4[2*i] | (q4[2*i+1] << 16)) >> 4*il) & 0x0f0f0f0f; + reg[i][0] = d * kvalues_iq4nl_f[q8[0]]; + reg[i][1] = d * kvalues_iq4nl_f[q8[1]]; + reg[i][2] = d * kvalues_iq4nl_f[q8[2]]; + reg[i][3] = d * kvalues_iq4nl_f[q8[3]]; + } +} + template kernel void kernel_get_rows( device const void * src0, @@ -5381,6 +5526,7 @@ template [[host_name("kernel_get_rows_iq2_xxs")]] kernel get_rows_t kernel_get_r template [[host_name("kernel_get_rows_iq2_xs")]] kernel get_rows_t kernel_get_rows; template [[host_name("kernel_get_rows_iq3_xxs")]] kernel get_rows_t kernel_get_rows; template [[host_name("kernel_get_rows_iq1_s")]] kernel get_rows_t kernel_get_rows; +template [[host_name("kernel_get_rows_iq4_nl")]] kernel get_rows_t kernel_get_rows; // // matrix-matrix multiplication @@ -5421,6 +5567,7 @@ template [[host_name("kernel_mul_mm_iq2_xxs_f32")]] kernel mat_mm_t kernel_mul_m template [[host_name("kernel_mul_mm_iq2_xs_f32")]] kernel mat_mm_t kernel_mul_mm; template [[host_name("kernel_mul_mm_iq3_xxs_f32")]] kernel mat_mm_t kernel_mul_mm; template [[host_name("kernel_mul_mm_iq1_s_f32")]] kernel mat_mm_t kernel_mul_mm; +template [[host_name("kernel_mul_mm_iq4_nl_f32")]] kernel mat_mm_t kernel_mul_mm; // // indirect matrix-matrix multiplication @@ -5473,6 +5620,7 @@ template [[host_name("kernel_mul_mm_id_iq2_xxs_f32")]] kernel mat_mm_id_t kernel template [[host_name("kernel_mul_mm_id_iq2_xs_f32")]] kernel mat_mm_id_t kernel_mul_mm_id; template [[host_name("kernel_mul_mm_id_iq3_xxs_f32")]] kernel mat_mm_id_t kernel_mul_mm_id; template [[host_name("kernel_mul_mm_id_iq1_s_f32")]] kernel mat_mm_id_t kernel_mul_mm_id; +template [[host_name("kernel_mul_mm_id_iq4_nl_f32")]] kernel mat_mm_id_t kernel_mul_mm_id; // // matrix-vector multiplication @@ -6503,3 +6651,68 @@ kernel void kernel_mul_mv_id_iq1_s_f32( tiisg, sgitg); } + +[[host_name("kernel_mul_mv_id_iq4_nl_f32")]] +kernel void kernel_mul_mv_id_iq4_nl_f32( + device const char * ids, + device const char * src1, + device float * dst, + constant uint64_t & nbi1, + constant int64_t & ne00, + constant int64_t & ne01, + constant int64_t & ne02, + constant uint64_t & nb00, + constant uint64_t & nb01, + constant uint64_t & nb02, + constant int64_t & ne10, + constant int64_t & ne11, + constant int64_t & ne12, + constant int64_t & ne13, + constant uint64_t & nb10, + constant uint64_t & nb11, + constant uint64_t & nb12, + constant int64_t & ne0, + constant int64_t & ne1, + constant uint64_t & nb1, + constant uint & r2, + constant uint & r3, + constant int & idx, + device const char * src00, + device const char * src01, + device const char * src02, + device const char * src03, + device const char * src04, + device const char * src05, + device const char * src06, + device const char * src07, + threadgroup float * shared_values [[threadgroup(0)]], + uint3 tgpig[[threadgroup_position_in_grid]], + uint tiitg[[thread_index_in_threadgroup]], + uint tiisg[[thread_index_in_simdgroup]], + uint sgitg[[simdgroup_index_in_threadgroup]]) { + device const char * src0[8] = {src00, src01, src02, src03, src04, src05, src06, src07}; + + const int64_t bid = tgpig.z/(ne12*ne13); + + tgpig.z = tgpig.z%(ne12*ne13); + + const int32_t id = ((device int32_t *) (ids + bid*nbi1))[idx]; + + kernel_mul_mv_iq4_nl_f32_impl( + src0[id], + (device const float *) (src1 + bid*nb11), + dst + bid*ne0, + ne00, + ne01, + ne02, + ne10, + ne12, + ne0, + ne1, + r2, + r3, + shared_values, + tgpig, + tiisg, + sgitg); +} diff --git a/ggml-quants.c b/ggml-quants.c index 3319d2ccf..6336538f0 100644 --- a/ggml-quants.c +++ b/ggml-quants.c @@ -3754,6 +3754,26 @@ void dequantize_row_iq1_s(const block_iq1_s * restrict x, float * restrict y, in } } +static const int8_t kvalues_iq4nl[16] = {-127, -104, -83, -65, -49, -35, -22, -10, 1, 13, 25, 38, 53, 69, 89, 113}; + +void dequantize_row_iq4_nl(const block_iq4_nl * restrict x, float * restrict y, int k) { + assert(k % QK4_NL == 0); + const int nb = k / QK4_NL; + + for (int i = 0; i < nb; i++) { + + const uint8_t * qs = x[i].qs; + + const float d = GGML_FP16_TO_FP32(x[i].d); + for (int j = 0; j < QK4_NL/2; ++j) { + y[j+ 0] = d * kvalues_iq4nl[qs[j] & 0xf]; + y[j+QK4_NL/2] = d * kvalues_iq4nl[qs[j] >> 4]; + } + y += QK4_NL; + qs += QK4_NL/2; + } +} + //===================================== Q8_K ============================================== void quantize_row_q8_K_reference(const float * restrict x, block_q8_K * restrict y, int k) { @@ -9148,7 +9168,6 @@ void ggml_vec_dot_iq2_xs_q8_K(int n, float * restrict s, size_t bs, const void * #endif } -// TODO void ggml_vec_dot_iq3_xxs_q8_K(int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) { assert(n % QK_K == 0); assert(nrc == 1); @@ -9452,7 +9471,100 @@ void ggml_vec_dot_iq1_s_q8_K (int n, float * GGML_RESTRICT s, size_t bs, const *s = sumf; #endif +} +void ggml_vec_dot_iq4_nl_q8_0(int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) { + assert(nrc == 1); + UNUSED(nrc); + UNUSED(bx); + UNUSED(by); + UNUSED(bs); + assert(n % QK4_NL == 0); + static_assert(QK4_NL == QK8_0, "QK4_NL and QK8_0 must be the same"); + + const block_iq4_nl * restrict x = vx; + const block_q8_0 * restrict y = vy; + + const int nb = n / QK4_NL; + +#if defined __ARM_NEON + const int8x16_t values = vld1q_s8(kvalues_iq4nl); + const uint8x16_t m4b = vdupq_n_u8(0x0f); + uint8x16x2_t q4bits; + int8x16x4_t q4b; + int8x16x4_t q8b; + int32x4_t prod_1, prod_2; + + float sumf = 0; + + for (int ib = 0; ib < nb; ib += 2) { + + q4bits.val[0] = vld1q_u8(x[ib+0].qs); + q4bits.val[1] = vld1q_u8(x[ib+1].qs); + q8b.val[0] = vld1q_s8(y[ib+0].qs); + q8b.val[1] = vld1q_s8(y[ib+0].qs + 16); + q8b.val[2] = vld1q_s8(y[ib+1].qs); + q8b.val[3] = vld1q_s8(y[ib+1].qs + 16); + + q4b.val[0] = vqtbl1q_s8(values, vandq_u8(q4bits.val[0], m4b)); + q4b.val[1] = vqtbl1q_s8(values, vshrq_n_u8(q4bits.val[0], 4)); + q4b.val[2] = vqtbl1q_s8(values, vandq_u8(q4bits.val[1], m4b)); + q4b.val[3] = vqtbl1q_s8(values, vshrq_n_u8(q4bits.val[1], 4)); + + prod_1 = ggml_vdotq_s32(ggml_vdotq_s32(vdupq_n_s32(0), q4b.val[0], q8b.val[0]), q4b.val[1], q8b.val[1]); + prod_2 = ggml_vdotq_s32(ggml_vdotq_s32(vdupq_n_s32(0), q4b.val[2], q8b.val[2]), q4b.val[3], q8b.val[3]); + + sumf += (float)x[ib+0].d * (float)y[ib+0].d * vaddvq_s32(prod_1) + (float)x[ib+1].d * (float)y[ib+1].d * vaddvq_s32(prod_2); + + } + + *s = sumf; + +#elif defined __AVX2__ + + const __m128i values128 = _mm_loadu_si128((const __m128i*)kvalues_iq4nl); + const __m128i m4b = _mm_set1_epi8(0x0f); + const __m256i mone = _mm256_set1_epi16(1); + + __m256 accum1 = _mm256_setzero_ps(); + __m256 accum2 = _mm256_setzero_ps(); + for (int ib = 0; ib < nb; ib += 2) { + const __m128i q4bits_1 = _mm_loadu_si128((const __m128i*)x[0].qs); + const __m128i q4bits_2 = _mm_loadu_si128((const __m128i*)x[1].qs); + const __m256i q8b_1 = _mm256_loadu_si256((const __m256i *)y[0].qs); + const __m256i q8b_2 = _mm256_loadu_si256((const __m256i *)y[1].qs); + const __m256i q4b_1 = _mm256_set_m128i(_mm_shuffle_epi8(values128, _mm_and_si128(_mm_srli_epi16(q4bits_1, 4), m4b)), + _mm_shuffle_epi8(values128, _mm_and_si128(q4bits_1, m4b))); + const __m256i q4b_2 = _mm256_set_m128i(_mm_shuffle_epi8(values128, _mm_and_si128(_mm_srli_epi16(q4bits_2, 4), m4b)), + _mm_shuffle_epi8(values128, _mm_and_si128(q4bits_2, m4b))); + const __m256i p16_1 = mul_add_epi8(q4b_1, q8b_1); + const __m256i p16_2 = mul_add_epi8(q4b_2, q8b_2); + const __m256i p_1 = _mm256_madd_epi16(p16_1, mone); + const __m256i p_2 = _mm256_madd_epi16(p16_2, mone); + accum1 = _mm256_fmadd_ps(_mm256_set1_ps(GGML_FP16_TO_FP32(y[0].d)*GGML_FP16_TO_FP32(x[0].d)), + _mm256_cvtepi32_ps(p_1), accum1); + accum2 = _mm256_fmadd_ps(_mm256_set1_ps(GGML_FP16_TO_FP32(y[1].d)*GGML_FP16_TO_FP32(x[1].d)), + _mm256_cvtepi32_ps(p_2), accum2); + + y += 2; + x += 2; + } + + *s = hsum_float_8(_mm256_add_ps(accum1, accum2)); + +#else + float sumf = 0; + for (int ib = 0; ib < nb; ++ib) { + const float d = GGML_FP16_TO_FP32(y[ib].d)*GGML_FP16_TO_FP32(x[ib].d); + int sumi1 = 0, sumi2 = 0; + for (int j = 0; j < QK4_NL/2; ++j) { + sumi1 += y[ib].qs[j+ 0] * kvalues_iq4nl[x[ib].qs[j] & 0xf]; + sumi2 += y[ib].qs[j+QK4_NL/2] * kvalues_iq4nl[x[ib].qs[j] >> 4]; + } + sumf += d * (sumi1 + sumi2); + } + *s = sumf; +#endif } // ================================ IQ2 quantization ============================================= @@ -10729,3 +10841,123 @@ size_t quantize_iq1_s(const float * src, void * dst, int nrow, int n_per_row, in } return nrow * nblock * sizeof(block_iq1_s); } + +// ============================ 4-bit non-linear quants + +static inline int best_index_int8(int n, const int8_t * val, float x) { + if (x <= val[0]) return 0; + if (x >= val[n-1]) return n-1; + int ml = 0, mu = n-1; + while (mu-ml > 1) { + int mav = (ml+mu)/2; + if (x < val[mav]) mu = mav; else ml = mav; + } + return x - val[mu-1] < val[mu] - x ? mu-1 : mu; +} + +static void quantize_row_iq4_nl_impl(const int block_size, const float * GGML_RESTRICT x, + ggml_fp16_t * dh, uint8_t * q4, + float * weight, uint8_t * L, + const int8_t * values, + const float * quant_weights) { + + const int ntry = 7; + + float sigma2 = 0; + for (int j = 0; j < QK4_NL; ++j) sigma2 += x[j]*x[j]; + sigma2 *= 2.f/QK4_NL; + + const int nb = QK4_NL/block_size; + + memset(q4, 0, QK4_NL/2); + for (int ib = 0; ib < nb; ++ib) { + dh[ib] = GGML_FP32_TO_FP16(0.f); + const float * xb = x + ib*block_size; + if (quant_weights) { + const float * qw = quant_weights + ib*block_size; + for (int j = 0; j < block_size; ++j) weight[j] = qw[j] * sqrtf(sigma2 + xb[j]*xb[j]); + } else { + for (int j = 0; j < block_size; ++j) weight[j] = xb[j]*xb[j]; + } + float amax = 0, max = 0; + for (int j = 0; j < block_size; ++j) { + float ax = fabsf(xb[j]); + if (ax > amax) { + amax = ax; max = xb[j]; + } + } + if (!amax) { + continue; + } + float d = -max/values[0]; + float id = 1/d; + float sumqx = 0, sumq2 = 0; + for (int j = 0; j < block_size; ++j) { + float al = id*xb[j]; + int l = best_index_int8(16, values, al); + float q = values[l]; + float w = weight[j]; + sumqx += w*q*xb[j]; + sumq2 += w*q*q; + } + float best_id = id; + d = sumqx/sumq2; + float best = d*sumqx; + for (int itry = -ntry; itry <= ntry; ++itry) { + id = (itry + values[0])/max; + sumqx = sumq2 = 0; + for (int j = 0; j < block_size; ++j) { + float al = id*xb[j]; + int l = best_index_int8(16, values, al); + float q = values[l]; + float w = weight[j]; + sumqx += w*q*xb[j]; + sumq2 += w*q*q; + } + if (sumq2 > 0 && sumqx*sumqx > best*sumq2) { + d = sumqx/sumq2; best = d * sumqx; + best_id = id; + } + } + dh[ib] = GGML_FP32_TO_FP16(d); + for (int j = 0; j < block_size; ++j) { + L[ib*block_size + j] = best_index_int8(16, values, best_id*xb[j]); + } + } + for (int i = 0; i < QK4_NL/32; ++i) { + for (int j = 0; j < 16; ++j) { + q4[16*i + j] = L[32*i + j] | (L[32*i + 16 + j] << 4); + } + } +} + +size_t quantize_iq4_nl(const float * src, void * dst, int nrow, int n_per_row, int64_t * hist, const float * quant_weights) { + (void)hist; + GGML_ASSERT(n_per_row%QK4_NL == 0); + int nblock = n_per_row/QK4_NL; + char * qrow = (char *)dst; + uint8_t L[QK4_NL]; + float weight[32]; + for (int row = 0; row < nrow; ++row) { + block_iq4_nl * iq4 = (block_iq4_nl *)qrow; + for (int ibl = 0; ibl < nblock; ++ibl) { + const float * qw = quant_weights ? quant_weights + QK4_NL*ibl : NULL; + quantize_row_iq4_nl_impl(32, src + QK4_NL*ibl, &iq4[ibl].d, iq4[ibl].qs, weight, L, kvalues_iq4nl, qw); + } + src += n_per_row; + qrow += nblock*sizeof(block_iq4_nl); + } + return nrow * nblock * sizeof(block_iq4_nl); +} + +void quantize_row_iq4_nl(const float * restrict x, void * restrict vy, int k) { + assert(k % QK4_NL == 0); + block_iq4_nl * restrict y = vy; + quantize_row_iq4_nl_reference(x, y, k); +} + +void quantize_row_iq4_nl_reference(const float * restrict x, block_iq4_nl * restrict y, int k) { + assert(k % QK4_NL == 0); + quantize_iq4_nl(x, y, 1, k, NULL, NULL); +} + diff --git a/ggml-quants.h b/ggml-quants.h index ad381cfab..113623b62 100644 --- a/ggml-quants.h +++ b/ggml-quants.h @@ -198,6 +198,14 @@ typedef struct { } block_iq1_s; static_assert(sizeof(block_iq1_s) == sizeof(ggml_fp16_t) + QK_K/8 + QK_K/16, "wrong iq1_s block size/padding"); +// Non-linear quants +#define QK4_NL 32 +typedef struct { + ggml_fp16_t d; + uint8_t qs[QK4_NL/2]; +} block_iq4_nl; +static_assert(sizeof(block_iq4_nl) == sizeof(ggml_fp16_t) + QK4_NL/2, "wrong iq4_nl block size/padding"); + #ifdef __cplusplus extern "C" { #endif @@ -217,6 +225,7 @@ void quantize_row_q5_K_reference(const float * GGML_RESTRICT x, block_q5_K * GGM void quantize_row_q6_K_reference(const float * GGML_RESTRICT x, block_q6_K * GGML_RESTRICT y, int k); void quantize_row_q8_K_reference(const float * GGML_RESTRICT x, block_q8_K * GGML_RESTRICT y, int k); void quantize_row_iq3_xxs_reference(const float * GGML_RESTRICT x, block_iq3_xxs * GGML_RESTRICT y, int k); +void quantize_row_iq4_nl_reference (const float * GGML_RESTRICT x, block_iq4_nl * GGML_RESTRICT y, int k); void quantize_row_q4_0(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int k); void quantize_row_q4_1(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int k); @@ -232,6 +241,7 @@ void quantize_row_q5_K(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, in void quantize_row_q6_K(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int k); void quantize_row_q8_K(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int k); void quantize_row_iq3_xxs(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int k); +void quantize_row_iq4_nl (const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int k); // Dequantization void dequantize_row_q4_0(const block_q4_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int k); @@ -251,6 +261,7 @@ void dequantize_row_iq2_xxs(const block_iq2_xxs * GGML_RESTRICT x, float * GGML_ void dequantize_row_iq2_xs (const block_iq2_xs * GGML_RESTRICT x, float * GGML_RESTRICT y, int k); void dequantize_row_iq3_xxs(const block_iq3_xxs * GGML_RESTRICT x, float * GGML_RESTRICT y, int k); void dequantize_row_iq1_s (const block_iq1_s * GGML_RESTRICT x, float * GGML_RESTRICT y, int k); +void dequantize_row_iq4_nl (const block_iq4_nl * GGML_RESTRICT x, float * GGML_RESTRICT y, int k); // Dot product void ggml_vec_dot_q4_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); @@ -268,6 +279,7 @@ void ggml_vec_dot_iq2_xxs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void ggml_vec_dot_iq2_xs_q8_K (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); void ggml_vec_dot_iq3_xxs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); void ggml_vec_dot_iq1_s_q8_K (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); +void ggml_vec_dot_iq4_nl_q8_0 (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); // // Quantization utilizing an importance matrix (a.k.a. "Activation aWare Quantization") @@ -276,6 +288,7 @@ size_t quantize_iq2_xxs(const float * src, void * dst, int nrows, int n_per_row, size_t quantize_iq2_xs (const float * src, void * dst, int nrows, int n_per_row, int64_t * hist, const float * imatrix); size_t quantize_iq3_xxs(const float * src, void * dst, int nrows, int n_per_row, int64_t * hist, const float * imatrix); size_t quantize_iq1_s (const float * src, void * dst, int nrows, int n_per_row, int64_t * hist, const float * imatrix); +size_t quantize_iq4_nl (const float * src, void * dst, int nrows, int n_per_row, int64_t * hist, const float * imatrix); size_t quantize_q2_K (const float * src, void * dst, int nrows, int n_per_row, int64_t * hist, const float * imatrix); size_t quantize_q3_K (const float * src, void * dst, int nrows, int n_per_row, int64_t * hist, const float * imatrix); size_t quantize_q4_K (const float * src, void * dst, int nrows, int n_per_row, int64_t * hist, const float * imatrix); diff --git a/ggml.c b/ggml.c index d129df505..91adbb0ae 100644 --- a/ggml.c +++ b/ggml.c @@ -690,6 +690,18 @@ static const ggml_type_traits_t type_traits[GGML_TYPE_COUNT] = { .vec_dot_type = GGML_TYPE_Q8_K, .nrows = 1, }, + [GGML_TYPE_IQ4_NL] = { + .type_name = "iq4_nl", + .blck_size = QK4_NL, + .type_size = sizeof(block_iq4_nl), + .is_quantized = true, + .to_float = (ggml_to_float_t) dequantize_row_iq4_nl, + .from_float = quantize_row_iq4_nl, + .from_float_reference = (ggml_from_float_t)quantize_row_iq4_nl_reference, + .vec_dot = ggml_vec_dot_iq4_nl_q8_0, + .vec_dot_type = GGML_TYPE_Q8_0, + .nrows = 1, + }, [GGML_TYPE_Q8_K] = { .type_name = "q8_K", .blck_size = QK_K, @@ -2291,6 +2303,7 @@ enum ggml_type ggml_ftype_to_ggml_type(enum ggml_ftype ftype) { case GGML_FTYPE_MOSTLY_IQ2_XS: wtype = GGML_TYPE_IQ2_XS; break; case GGML_FTYPE_MOSTLY_IQ3_XXS: wtype = GGML_TYPE_IQ3_XXS; break; case GGML_FTYPE_MOSTLY_IQ1_S: wtype = GGML_TYPE_IQ1_S; break; + case GGML_FTYPE_MOSTLY_IQ4_NL: wtype = GGML_TYPE_IQ4_NL; break; case GGML_FTYPE_UNKNOWN: wtype = GGML_TYPE_COUNT; break; case GGML_FTYPE_MOSTLY_Q4_1_SOME_F16: wtype = GGML_TYPE_COUNT; break; } @@ -7702,6 +7715,7 @@ static void ggml_compute_forward_add( case GGML_TYPE_IQ2_XS: case GGML_TYPE_IQ3_XXS: case GGML_TYPE_IQ1_S: + case GGML_TYPE_IQ4_NL: { ggml_compute_forward_add_q_f32(params, src0, src1, dst); } break; @@ -7970,6 +7984,7 @@ static void ggml_compute_forward_add1( case GGML_TYPE_IQ2_XS: case GGML_TYPE_IQ3_XXS: case GGML_TYPE_IQ1_S: + case GGML_TYPE_IQ4_NL: { ggml_compute_forward_add1_q_f32(params, src0, src1, dst); } break; @@ -8091,6 +8106,7 @@ static void ggml_compute_forward_acc( case GGML_TYPE_IQ2_XS: case GGML_TYPE_IQ3_XXS: case GGML_TYPE_IQ1_S: + case GGML_TYPE_IQ4_NL: default: { GGML_ASSERT(false); @@ -10858,6 +10874,7 @@ static void ggml_compute_forward_out_prod( case GGML_TYPE_IQ2_XS: case GGML_TYPE_IQ3_XXS: case GGML_TYPE_IQ1_S: + case GGML_TYPE_IQ4_NL: { ggml_compute_forward_out_prod_q_f32(params, src0, src1, dst); } break; @@ -11039,6 +11056,7 @@ static void ggml_compute_forward_set( case GGML_TYPE_IQ2_XS: case GGML_TYPE_IQ3_XXS: case GGML_TYPE_IQ1_S: + case GGML_TYPE_IQ4_NL: default: { GGML_ASSERT(false); @@ -11237,6 +11255,7 @@ static void ggml_compute_forward_get_rows( case GGML_TYPE_IQ2_XS: case GGML_TYPE_IQ3_XXS: case GGML_TYPE_IQ1_S: + case GGML_TYPE_IQ4_NL: { ggml_compute_forward_get_rows_q(params, src0, src1, dst); } break; @@ -11911,6 +11930,7 @@ static void ggml_compute_forward_alibi( case GGML_TYPE_IQ2_XS: case GGML_TYPE_IQ3_XXS: case GGML_TYPE_IQ1_S: + case GGML_TYPE_IQ4_NL: case GGML_TYPE_Q8_K: case GGML_TYPE_I8: case GGML_TYPE_I16: @@ -11989,6 +12009,7 @@ static void ggml_compute_forward_clamp( case GGML_TYPE_IQ2_XS: case GGML_TYPE_IQ3_XXS: case GGML_TYPE_IQ1_S: + case GGML_TYPE_IQ4_NL: case GGML_TYPE_Q8_K: case GGML_TYPE_I8: case GGML_TYPE_I16: @@ -19455,6 +19476,15 @@ size_t ggml_quantize_chunk(enum ggml_type type, const float * src, void * dst, i result = quantize_iq1_s(src + start, (char *)dst + start_row * row_size, nrows, n_per_row, hist, imatrix); GGML_ASSERT(result == row_size * nrows); } break; + case GGML_TYPE_IQ4_NL: + { + GGML_ASSERT(start % QK4_NL == 0); + GGML_ASSERT(start % n_per_row == 0); + size_t start_row = start / n_per_row; + size_t row_size = ggml_row_size(type, n_per_row); + result = quantize_iq4_nl(src + start, (char *)dst + start_row * row_size, nrows, n_per_row, hist, imatrix); + GGML_ASSERT(result == row_size * nrows); + } break; case GGML_TYPE_F16: { size_t elemsize = sizeof(ggml_fp16_t); diff --git a/ggml.h b/ggml.h index 004d09c70..bed7a36a0 100644 --- a/ggml.h +++ b/ggml.h @@ -355,6 +355,7 @@ extern "C" { GGML_TYPE_IQ2_XS = 17, GGML_TYPE_IQ3_XXS = 18, GGML_TYPE_IQ1_S = 19, + GGML_TYPE_IQ4_NL = 20, GGML_TYPE_I8, GGML_TYPE_I16, GGML_TYPE_I32, @@ -393,6 +394,7 @@ extern "C" { GGML_FTYPE_MOSTLY_IQ2_XS = 16, // except 1d tensors GGML_FTYPE_MOSTLY_IQ3_XXS = 17, // except 1d tensors GGML_FTYPE_MOSTLY_IQ1_S = 18, // except 1d tensors + GGML_FTYPE_MOSTLY_IQ4_NL = 19, // except 1d tensors }; // available tensor operations: diff --git a/llama.cpp b/llama.cpp index 4296eca32..3748d5eac 100644 --- a/llama.cpp +++ b/llama.cpp @@ -2527,6 +2527,7 @@ struct llama_model_loader { case GGML_TYPE_IQ2_XS: ftype = LLAMA_FTYPE_MOSTLY_IQ2_XS; break; case GGML_TYPE_IQ3_XXS: ftype = LLAMA_FTYPE_MOSTLY_IQ3_XXS; break; case GGML_TYPE_IQ1_S: ftype = LLAMA_FTYPE_MOSTLY_IQ1_S; break; + case GGML_TYPE_IQ4_NL: ftype = LLAMA_FTYPE_MOSTLY_IQ4_NL; break; default: { LLAMA_LOG_WARN("%s: unknown type %s\n", __func__, ggml_type_name(type_max)); @@ -2877,6 +2878,7 @@ static std::string llama_model_ftype_name(llama_ftype ftype) { case LLAMA_FTYPE_MOSTLY_Q3_K_XS:return "Q3_K - Extra small"; case LLAMA_FTYPE_MOSTLY_IQ3_XXS:return "IQ3_XXS - 3.0625 bpw"; case LLAMA_FTYPE_MOSTLY_IQ1_S :return "IQ1_S - 1.5625 bpw"; + case LLAMA_FTYPE_MOSTLY_IQ4_NL: return "IQ4_NL - 4.5 bpw"; default: return "unknown, may not work"; } @@ -10354,6 +10356,9 @@ static ggml_type get_k_quant_type(quantize_state_internal & qs, ggml_type new_ty new_type = qs.i_attention_wv < 2 ? GGML_TYPE_Q5_K : GGML_TYPE_Q4_K; } else if (ftype == LLAMA_FTYPE_MOSTLY_Q3_K_L) new_type = GGML_TYPE_Q5_K; + else if (ftype == LLAMA_FTYPE_MOSTLY_IQ4_NL && qs.model.hparams.n_gqa() >= 4) { + new_type = GGML_TYPE_Q5_K; + } else if ((ftype == LLAMA_FTYPE_MOSTLY_Q4_K_M || ftype == LLAMA_FTYPE_MOSTLY_Q5_K_M) && use_more_bits(qs.i_attention_wv, qs.n_attention_wv)) new_type = GGML_TYPE_Q6_K; else if (ftype == LLAMA_FTYPE_MOSTLY_Q4_K_S && qs.i_attention_wv < 4) new_type = GGML_TYPE_Q5_K; @@ -10406,6 +10411,9 @@ static ggml_type get_k_quant_type(quantize_state_internal & qs, ggml_type new_ty if (use_more_bits(i_layer, n_layer)) new_type = GGML_TYPE_Q6_K; } } + else if (ftype == LLAMA_FTYPE_MOSTLY_IQ4_NL && !qs.has_imatrix) { + if (i_layer < n_layer/8) new_type = GGML_TYPE_Q5_K; + } else if (ftype == LLAMA_FTYPE_MOSTLY_Q5_K_M && use_more_bits(i_layer, n_layer)) new_type = GGML_TYPE_Q6_K; else if (ftype == LLAMA_FTYPE_MOSTLY_Q4_K_S && arch != LLM_ARCH_FALCON && i_layer < n_layer/8) { new_type = GGML_TYPE_Q5_K; @@ -10422,7 +10430,7 @@ static ggml_type get_k_quant_type(quantize_state_internal & qs, ggml_type new_ty if (arch != LLM_ARCH_FALCON) { if (qs.model.hparams.n_expert == 8) { if (ftype == LLAMA_FTYPE_MOSTLY_Q2_K || ftype == LLAMA_FTYPE_MOSTLY_Q3_K_XS || ftype == LLAMA_FTYPE_MOSTLY_IQ3_XXS || - ftype == LLAMA_FTYPE_MOSTLY_Q3_K_S || ftype == LLAMA_FTYPE_MOSTLY_Q3_K_M || + ftype == LLAMA_FTYPE_MOSTLY_Q3_K_S || ftype == LLAMA_FTYPE_MOSTLY_Q3_K_M || ftype == LLAMA_FTYPE_MOSTLY_IQ4_NL || ftype == LLAMA_FTYPE_MOSTLY_Q4_K_S || ftype == LLAMA_FTYPE_MOSTLY_Q4_K_M) { new_type = GGML_TYPE_Q5_K; } @@ -10489,8 +10497,8 @@ static ggml_type get_k_quant_type(quantize_state_internal & qs, ggml_type new_ty case GGML_TYPE_IQ2_XS: case GGML_TYPE_IQ3_XXS: case GGML_TYPE_IQ1_S: - case GGML_TYPE_Q2_K: new_type = GGML_TYPE_Q4_0; break; - case GGML_TYPE_Q3_K: new_type = GGML_TYPE_Q4_1; break; + case GGML_TYPE_Q2_K: + case GGML_TYPE_Q3_K: new_type = GGML_TYPE_IQ4_NL; break; case GGML_TYPE_Q4_K: new_type = GGML_TYPE_Q5_0; break; case GGML_TYPE_Q5_K: new_type = GGML_TYPE_Q5_1; break; case GGML_TYPE_Q6_K: new_type = GGML_TYPE_Q8_0; break; @@ -10531,7 +10539,8 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s case LLAMA_FTYPE_MOSTLY_IQ2_XXS: quantized_type = GGML_TYPE_IQ2_XXS; break; case LLAMA_FTYPE_MOSTLY_IQ2_XS: quantized_type = GGML_TYPE_IQ2_XS; break; case LLAMA_FTYPE_MOSTLY_IQ3_XXS: quantized_type = GGML_TYPE_IQ3_XXS; break; - case LLAMA_FTYPE_MOSTLY_IQ1_S: quantized_type = GGML_TYPE_IQ1_S ; break; + case LLAMA_FTYPE_MOSTLY_IQ1_S: quantized_type = GGML_TYPE_IQ1_S; break; + case LLAMA_FTYPE_MOSTLY_IQ4_NL: quantized_type = GGML_TYPE_IQ4_NL; break; default: throw std::runtime_error(format("invalid output file type %d\n", ftype)); } diff --git a/llama.h b/llama.h index 77a84c18a..8ba20696f 100644 --- a/llama.h +++ b/llama.h @@ -101,6 +101,7 @@ extern "C" { LLAMA_FTYPE_MOSTLY_Q3_K_XS = 22, // except 1d tensors LLAMA_FTYPE_MOSTLY_IQ3_XXS = 23, // except 1d tensors LLAMA_FTYPE_MOSTLY_IQ1_S = 24, // except 1d tensors + LLAMA_FTYPE_MOSTLY_IQ4_NL = 25, // except 1d tensors LLAMA_FTYPE_GUESSED = 1024, // not specified in the model file }; diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index ef37c5af2..55db42bf6 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -1918,6 +1918,7 @@ static bool test_backend(ggml_backend_t backend, test_mode mode, const char * op GGML_TYPE_Q6_K, GGML_TYPE_IQ2_XXS, GGML_TYPE_IQ2_XS, GGML_TYPE_IQ3_XXS, GGML_TYPE_IQ1_S, + GGML_TYPE_IQ4_NL, }; // unary ops From 88c46cbdac05cebd936511b1d3c74112e721615f Mon Sep 17 00:00:00 2001 From: "Meng, Hengyu" Date: Wed, 21 Feb 2024 17:52:06 +0800 Subject: [PATCH 418/812] [SYCL] conext add name (#5624) * [SYCL] conext add name * name should start with SYCL* --- ggml-sycl.cpp | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/ggml-sycl.cpp b/ggml-sycl.cpp index df1826112..b897828f9 100644 --- a/ggml-sycl.cpp +++ b/ggml-sycl.cpp @@ -14642,7 +14642,8 @@ GGML_CALL static const char * ggml_backend_sycl_buffer_type_name(ggml_backend_bu static ggml_backend_buffer_t ggml_backend_sycl_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size) try { - int device = (int) (intptr_t) buft->context; + ggml_backend_sycl_buffer_type_context * buft_ctx = (ggml_backend_sycl_buffer_type_context *)buft->context; + int device = (int) buft_ctx->device; ggml_sycl_set_device(device); int device_index = get_device_index_by_id(device); @@ -14720,7 +14721,7 @@ ggml_backend_buffer_type_t ggml_backend_sycl_buffer_type(int device) { for (int i = 0; i < GGML_SYCL_MAX_DEVICES; i++) { ggml_backend_sycl_buffer_types[i] = { /* .iface = */ ggml_backend_sycl_buffer_type_interface, - /* .context = */ (ggml_backend_buffer_type_context_t) (intptr_t) i, + /* .context = */ new ggml_backend_sycl_buffer_type_context{i, GGML_SYCL_NAME + std::to_string(i)}, }; } ggml_backend_sycl_buffer_type_initialized = true; @@ -14782,10 +14783,6 @@ ggml_backend_buffer_type_t ggml_backend_sycl_host_buffer_type() { // backend -struct ggml_backend_context_sycl { - int device; -}; - static const char * ggml_backend_sycl_name(ggml_backend_t backend) { return GGML_SYCL_NAME; @@ -14793,14 +14790,14 @@ static const char * ggml_backend_sycl_name(ggml_backend_t backend) { } static void ggml_backend_sycl_free(ggml_backend_t backend) { - ggml_backend_context_sycl * sycl_ctx = (ggml_backend_context_sycl *)backend->context; + ggml_backend_sycl_context * sycl_ctx = (ggml_backend_sycl_context *)backend->context; delete sycl_ctx; delete backend; } static ggml_backend_buffer_type_t ggml_backend_sycl_get_default_buffer_type(ggml_backend_t backend) { - ggml_backend_context_sycl * sycl_ctx = (ggml_backend_context_sycl *)backend->context; + ggml_backend_sycl_context * sycl_ctx = (ggml_backend_sycl_context *)backend->context; return ggml_backend_sycl_buffer_type(sycl_ctx->device); } @@ -14809,7 +14806,7 @@ static void ggml_backend_sycl_set_tensor_async(ggml_backend_t backend, ggml_tensor *tensor, const void *data, size_t offset, size_t size) try { - ggml_backend_context_sycl * sycl_ctx = (ggml_backend_context_sycl *)backend->context; + ggml_backend_sycl_context * sycl_ctx = (ggml_backend_sycl_context *)backend->context; GGML_ASSERT(tensor->buffer->buft == ggml_backend_sycl_buffer_type(sycl_ctx->device) && "unsupported buffer type"); GGML_ASSERT(tensor->backend == GGML_BACKEND_GPU); @@ -14827,7 +14824,7 @@ static void ggml_backend_sycl_get_tensor_async(ggml_backend_t backend, const ggml_tensor *tensor, void *data, size_t offset, size_t size) try { - ggml_backend_context_sycl * sycl_ctx = (ggml_backend_context_sycl *)backend->context; + ggml_backend_sycl_context * sycl_ctx = (ggml_backend_sycl_context *)backend->context; GGML_ASSERT(tensor->buffer->buft == ggml_backend_sycl_buffer_type(sycl_ctx->device) && "unsupported buffer type"); GGML_ASSERT(tensor->backend == GGML_BACKEND_GPU); @@ -14842,7 +14839,7 @@ catch (sycl::exception const &exc) { } static void ggml_backend_sycl_synchronize(ggml_backend_t backend) try { - ggml_backend_context_sycl * sycl_ctx = (ggml_backend_context_sycl *)backend->context; + ggml_backend_sycl_context * sycl_ctx = (ggml_backend_sycl_context *)backend->context; SYCL_CHECK(CHECK_TRY_ERROR(g_syclStreams[sycl_ctx->device][0]->wait())); @@ -14878,7 +14875,7 @@ static void ggml_backend_sycl_graph_plan_compute(ggml_backend_t backend, ggml_ba } static bool ggml_backend_sycl_graph_compute(ggml_backend_t backend, ggml_cgraph * cgraph) { - ggml_backend_context_sycl * sycl_ctx = (ggml_backend_context_sycl *)backend->context; + ggml_backend_sycl_context * sycl_ctx = (ggml_backend_sycl_context *)backend->context; ggml_sycl_set_main_device(sycl_ctx->device); @@ -15092,8 +15089,9 @@ ggml_backend_t ggml_backend_sycl_init(int device) { // not strictly necessary, but it may reduce the overhead of the first graph_compute ggml_sycl_set_main_device(device); - ggml_backend_context_sycl * ctx = new ggml_backend_context_sycl { - /* .device = */ device + ggml_backend_sycl_context * ctx = new ggml_backend_sycl_context { + /* .device = */ device, + /* .name = */ GGML_SYCL_NAME + std::to_string(device), }; ggml_backend_t sycl_backend = new ggml_backend { From 580111d42b3b6ad0a390bfb267d6e3077506eb31 Mon Sep 17 00:00:00 2001 From: postmasters Date: Wed, 21 Feb 2024 05:08:22 -0800 Subject: [PATCH 419/812] llama : add `gemma` model (#5631) There are couple things in this architecture: 1. Shared input and output embedding parameters. 2. Key length and value length are not derived from `n_embd`. More information about the models can be found at https://ai.google.dev/gemma. GGUFs can be downloaded from https://huggingface.co/google. --- README.md | 1 + gguf-py/gguf/constants.py | 15 ++++ llama.cpp | 170 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 186 insertions(+) diff --git a/README.md b/README.md index 747d2e98b..225db8e49 100644 --- a/README.md +++ b/README.md @@ -107,6 +107,7 @@ Typically finetunes of the base models below are supported as well. - [x] [Orion 14B](https://github.com/ggerganov/llama.cpp/pull/5118) - [x] [InternLM2](https://huggingface.co/models?search=internlm2) - [x] [CodeShell](https://github.com/WisdomShell/codeshell) +- [x] [Gemma](https://ai.google.dev/gemma) **Multimodal models:** diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py index 114a9a974..8f9139d1b 100644 --- a/gguf-py/gguf/constants.py +++ b/gguf-py/gguf/constants.py @@ -111,6 +111,7 @@ class MODEL_ARCH(IntEnum): ORION = auto() INTERNLM2 = auto() MINICPM = auto() + GEMMA = auto() class MODEL_TENSOR(IntEnum): @@ -167,6 +168,7 @@ MODEL_ARCH_NAMES: dict[MODEL_ARCH, str] = { MODEL_ARCH.ORION: "orion", MODEL_ARCH.INTERNLM2: "internlm2", MODEL_ARCH.MINICPM: "minicpm", + MODEL_ARCH.GEMMA: "gemma", } TENSOR_NAMES: dict[MODEL_TENSOR, str] = { @@ -511,6 +513,19 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = { MODEL_TENSOR.FFN_DOWN_EXP, MODEL_TENSOR.FFN_UP_EXP, ], + MODEL_ARCH.GEMMA: [ + MODEL_TENSOR.TOKEN_EMBD, + MODEL_TENSOR.OUTPUT_NORM, + MODEL_TENSOR.ATTN_NORM, + MODEL_TENSOR.ATTN_Q, + MODEL_TENSOR.ATTN_K, + MODEL_TENSOR.ATTN_V, + MODEL_TENSOR.ATTN_OUT, + MODEL_TENSOR.FFN_GATE, + MODEL_TENSOR.FFN_DOWN, + MODEL_TENSOR.FFN_UP, + MODEL_TENSOR.FFN_NORM, + ], # TODO } diff --git a/llama.cpp b/llama.cpp index 3748d5eac..3a226c426 100644 --- a/llama.cpp +++ b/llama.cpp @@ -208,6 +208,7 @@ enum llm_arch { LLM_ARCH_ORION, LLM_ARCH_INTERNLM2, LLM_ARCH_MINICPM, + LLM_ARCH_GEMMA, LLM_ARCH_UNKNOWN, }; @@ -234,6 +235,7 @@ static std::map LLM_ARCH_NAMES = { { LLM_ARCH_ORION, "orion" }, { LLM_ARCH_INTERNLM2, "internlm2" }, { LLM_ARCH_MINICPM, "minicpm" }, + { LLM_ARCH_GEMMA, "gemma" }, }; enum llm_kv { @@ -760,6 +762,22 @@ static std::map> LLM_TENSOR_NAMES = { LLM_TENSOR_FFN_UP_EXP, "blk.%d.ffn_up.%d" }, }, }, + { + LLM_ARCH_GEMMA, + { + { LLM_TENSOR_TOKEN_EMBD, "token_embd" }, + { LLM_TENSOR_OUTPUT_NORM, "output_norm" }, + { LLM_TENSOR_ATTN_NORM, "blk.%d.attn_norm" }, + { LLM_TENSOR_ATTN_Q, "blk.%d.attn_q" }, + { LLM_TENSOR_ATTN_K, "blk.%d.attn_k" }, + { LLM_TENSOR_ATTN_V, "blk.%d.attn_v" }, + { LLM_TENSOR_ATTN_OUT, "blk.%d.attn_output" }, + { LLM_TENSOR_FFN_NORM, "blk.%d.ffn_norm" }, + { LLM_TENSOR_FFN_GATE, "blk.%d.ffn_gate" }, + { LLM_TENSOR_FFN_DOWN, "blk.%d.ffn_down" }, + { LLM_TENSOR_FFN_UP, "blk.%d.ffn_up" }, + }, + }, { LLM_ARCH_UNKNOWN, { @@ -3243,6 +3261,16 @@ static void llm_load_hparams( default: model.type = e_model::MODEL_UNKNOWN; } } break; + case LLM_ARCH_GEMMA: + { + ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps); + + switch (hparams.n_layer) { + case 18: model.type = e_model::MODEL_2B; break; + case 28: model.type = e_model::MODEL_7B; break; + default: model.type = e_model::MODEL_UNKNOWN; + } + } break; default: (void)0; } @@ -4360,6 +4388,37 @@ static bool llm_load_tensors( layer.ffn_up = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff}); } } break; + case LLM_ARCH_GEMMA: + { + model.tok_embd = ml.create_tensor(ctx_input, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}); + + // output + model.output_norm = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}); + + const int64_t n_ff = hparams.n_ff; + const int64_t n_embd_head_k = hparams.n_embd_head_k; + const int64_t n_embd_k_gqa = hparams.n_embd_k_gqa(); + const int64_t n_embd_v_gqa = hparams.n_embd_v_gqa(); + + for (uint32_t i = 0; i < n_layer; ++i) { + ggml_context * ctx_layer = ctx_for_layer(i); + ggml_context * ctx_split = ctx_for_layer_split(i); + + auto & layer = model.layers[i]; + + layer.attn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}); + + layer.wq = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_embd_head_k * hparams.n_head}); + layer.wk = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_K, "weight", i), {n_embd, n_embd_k_gqa}); + layer.wv = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_V, "weight", i), {n_embd, n_embd_v_gqa}); + layer.wo = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd_head_k * hparams.n_head, n_embd}); + + layer.ffn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}); + layer.ffn_gate = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff}); + layer.ffn_up = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff}); + layer.ffn_down = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_DOWN, "weight", i), { n_ff, n_embd}); + } + } break; default: throw std::runtime_error("unknown architecture"); } @@ -7366,6 +7425,113 @@ struct llm_build_context { return gf; } + + struct ggml_cgraph * build_gemma() { + struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, LLAMA_MAX_NODES, false); + + const int64_t n_embd_head_k = hparams.n_embd_head_k; + + struct ggml_tensor * cur; + struct ggml_tensor * inpL; + + inpL = llm_build_inp_embd(ctx0, hparams, batch, model.tok_embd, lctx.inp_tokens, lctx.inp_embd, cb); + cb(inpL, "inp_embd", -1); + inpL = ggml_scale(ctx0, inpL, sqrtf(n_embd)); + cb(inpL, "inp_scaled", -1); + + // inp_pos - contains the positions + struct ggml_tensor * inp_pos = ggml_view_1d(ctx0, lctx.inp_pos, n_tokens, 0); + cb(inp_pos, "inp_pos", -1); + + // KQ_mask (mask for 1 head, it will be broadcasted to all heads) + struct ggml_tensor * KQ_mask = ggml_view_2d(ctx0, lctx.inp_KQ_mask, n_kv, n_tokens, n_kv*ggml_type_size(lctx.inp_KQ_mask->type), 0); + cb(KQ_mask, "KQ_mask", -1); + + // shift the entire K-cache if needed + if (do_rope_shift) { + llm_build_k_shift(ctx0, hparams, cparams, kv_self, gf, lctx.inp_K_shift, LLM_ROPE, n_ctx, freq_base, freq_scale, cb); + } + + for (int il = 0; il < n_layer; ++il) { + + // norm + cur = llm_build_norm(ctx0, inpL, hparams, + model.layers[il].attn_norm, NULL, + LLM_NORM_RMS, cb, il); + cb(cur, "attn_norm", il); + + // self-attention + { + // compute Q and K and RoPE them + struct ggml_tensor * Qcur = ggml_mul_mat(ctx0, model.layers[il].wq, cur); + cb(Qcur, "Qcur", il); + + struct ggml_tensor * Kcur = ggml_mul_mat(ctx0, model.layers[il].wk, cur); + cb(Kcur, "Kcur", il); + + struct ggml_tensor * Vcur = ggml_mul_mat(ctx0, model.layers[il].wv, cur); + cb(Vcur, "Vcur", il); + + Qcur = ggml_rope_custom( + ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head_k, n_head, n_tokens), inp_pos, + n_embd_head_k, 2, 0, n_orig_ctx, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow); + cb(Qcur, "Qcur", il); + Qcur = ggml_scale(ctx0, Qcur, 1.0f / sqrtf(float(n_embd_head_k))); + cb(Qcur, "Qcur_scaled", il); + + Kcur = ggml_rope_custom( + ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head_k, n_head_kv, n_tokens), inp_pos, + n_embd_head_k, 2, 0, n_orig_ctx, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow); + cb(Kcur, "Kcur", il); + + cur = llm_build_kv(ctx0, model, hparams, kv_self, gf, + model.layers[il].wo, NULL, + Kcur, Vcur, Qcur, KQ_mask, nullptr, n_ctx, n_tokens, kv_head, n_kv, 1.0f, cb, il); + cb(cur, "kqv_out", il); + } + struct ggml_tensor * sa_out = ggml_add(ctx0, cur, inpL); + cb(sa_out, "sa_out", il); + + cur = llm_build_norm(ctx0, sa_out, hparams, + model.layers[il].ffn_norm, NULL, + LLM_NORM_RMS, cb, il); + cb(cur, "ffn_norm", il); + + // feed-forward network + { + cur = llm_build_ffn(ctx0, cur, + model.layers[il].ffn_up, NULL, + model.layers[il].ffn_gate, NULL, + model.layers[il].ffn_down, NULL, + NULL, + LLM_FFN_GELU, LLM_FFN_PAR, cb, il); + cb(cur, "ffn_out", il); + } + + cur = ggml_add(ctx0, cur, sa_out); + cb(cur, "l_out", il); + + // input for next layer + inpL = cur; + } + + cur = inpL; + + cur = llm_build_norm(ctx0, cur, hparams, + model.output_norm, NULL, + LLM_NORM_RMS, cb, -1); + cb(cur, "result_norm", -1); + + // lm_head + cur = ggml_mul_mat(ctx0, model.tok_embd, cur); + cb(cur, "result_output", -1); + + ggml_build_forward_expand(gf, cur); + + return gf; + } }; static struct ggml_cgraph * llama_build_graph( @@ -7474,6 +7640,10 @@ static struct ggml_cgraph * llama_build_graph( { result = llm.build_minicpm(); } break; + case LLM_ARCH_GEMMA: + { + result = llm.build_gemma(); + } break; default: GGML_ASSERT(false); } From cc6cac08e38e32bf40bbe07e9e8f8f0130b5fd94 Mon Sep 17 00:00:00 2001 From: Daniel Bevenius Date: Wed, 21 Feb 2024 14:36:57 +0100 Subject: [PATCH 420/812] llava : add --skip-unknown to 1.6 convert.py (#5632) This commit adds the `--skip-unknown` option to the convert.py script and removes the saving of the updated checkpoints to avoid updating possibly checked out files. The motivation for this change is that this was done for 1.5 in Commit fc0c8d286a533363a9a663510b62af85ffad58b3 ("llava : update surgery script to not remove tensors") and makes the examples more consistent. Signed-off-by: Daniel Bevenius --- examples/llava/README.md | 13 ++++++------- examples/llava/llava-surgery-v2.py | 12 ------------ 2 files changed, 6 insertions(+), 19 deletions(-) diff --git a/examples/llava/README.md b/examples/llava/README.md index 25ea96715..35e6d9e5d 100644 --- a/examples/llava/README.md +++ b/examples/llava/README.md @@ -63,13 +63,12 @@ Now both the LLaMA part and the image encoder is in the `llava-v1.5-7b` director ```console git clone https://huggingface.co/liuhaotian/llava-v1.6-vicuna-7b ``` -2) Backup your pth/safetensor model files as llava-surgery modifies them -3) Use `llava-surgery-v2.py` which also supports llava-1.5 variants pytorch as well as safetensor models: +2) Use `llava-surgery-v2.py` which also supports llava-1.5 variants pytorch as well as safetensor models: ```console python examples/llava/llava-surgery-v2.py -C -m ../llava-v1.6-vicuna-7b/ ``` - you will find a llava.projector and a llava.clip file in your model directory -4) Copy the llava.clip file into a subdirectory (like vit), rename it to pytorch_model.bin and add a fitting vit configuration to the directory: +3) Copy the llava.clip file into a subdirectory (like vit), rename it to pytorch_model.bin and add a fitting vit configuration to the directory: ```console mkdir vit cp ../llava-v1.6-vicuna-7b/llava.clip vit/pytorch_model.bin @@ -77,18 +76,18 @@ cp ../llava-v1.6-vicuna-7b/llava.projector vit/ curl -s -q https://huggingface.co/cmp-nct/llava-1.6-gguf/raw/main/config_vit.json -o vit/config.json ``` -5) Create the visual gguf model: +4) Create the visual gguf model: ```console python ./examples/llava/convert-image-encoder-to-gguf.py -m vit --llava-projector vit/llava.projector --output-dir vit --clip-model-is-vision ``` - This is similar to llava-1.5, the difference is that we tell the encoder that we are working with the pure vision model part of CLIP -6) Then convert the model to gguf format: +5) Then convert the model to gguf format: ```console -python ./convert.py ../llava-v1.6-vicuna-7b/ +python ./convert.py ../llava-v1.6-vicuna-7b/ --skip-unknown ``` -7) And finally we can run the llava-cli using the 1.6 model version: +6) And finally we can run the llava-cli using the 1.6 model version: ```console ./llava-cli -m ../llava-v1.6-vicuna-7b/ggml-model-f16.gguf --mmproj vit/mmproj-model-f16.gguf --image some-image.jpg -c 4096 ``` diff --git a/examples/llava/llava-surgery-v2.py b/examples/llava/llava-surgery-v2.py index 5bc5bc513..eb56d6988 100644 --- a/examples/llava/llava-surgery-v2.py +++ b/examples/llava/llava-surgery-v2.py @@ -65,9 +65,7 @@ def clean_vision_tower_from_checkpoint(checkpoint_path): for name in clip_tensors: del checkpoint[name] - # Save the updated checkpoint checkpoint_path = checkpoint_path - save_model(checkpoint, checkpoint_path, file_type) return True return False @@ -152,16 +150,6 @@ for name in first_mm_tensors: if len(projector) > 0: save_model(projector, f"{args.model}/llava.projector", 'pytorch') -for name in mm_tensors: - del last_checkpoint[name] -for name in first_mm_tensors: - del first_checkpoint[name] - -if len(mm_tensors) > 0: - save_model(last_checkpoint, projector_checkpoint_path, file_type) -if len(first_mm_tensors) > 0: - save_model(first_checkpoint, newline_checkpoint_path, file_type) - print("Done!") print(f"Now you can convert {args.model} to a a regular LLaMA GGUF file.") print(f"Also, use {args.model}/llava.projector to prepare a llava-encoder.gguf file.") From c14f72db9c62d71d35eb1c141745c0bd0cb27b49 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Wed, 21 Feb 2024 15:39:54 +0200 Subject: [PATCH 421/812] readme : update hot topics --- README.md | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 225db8e49..ce5dec7ca 100644 --- a/README.md +++ b/README.md @@ -10,13 +10,8 @@ Inference of Meta's [LLaMA](https://arxiv.org/abs/2302.13971) model (and others) ### Hot topics -- Remove LLAMA_MAX_DEVICES and LLAMA_SUPPORTS_GPU_OFFLOAD: https://github.com/ggerganov/llama.cpp/pull/5240 -- Incoming backends: https://github.com/ggerganov/llama.cpp/discussions/5138 - - [SYCL backend](README-sycl.md) is ready (1/28/2024), support Linux/Windows in Intel GPUs (iGPU, Arc/Flex/Max series) -- New SOTA quantized models, including pure 2-bits: https://huggingface.co/ikawrakow -- Collecting Apple Silicon performance stats: - - M-series: https://github.com/ggerganov/llama.cpp/discussions/4167 - - A-series: https://github.com/ggerganov/llama.cpp/discussions/4508 +- Support for Gemma models: https://github.com/ggerganov/llama.cpp/pull/5631 +- Non-linear quantization IQ4_NL: https://github.com/ggerganov/llama.cpp/pull/5590 - Looking for contributions to improve and maintain the `server` example: https://github.com/ggerganov/llama.cpp/issues/4216 ---- From eccd7a26ddbff19e4b8805648f5f14c501957859 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Wed, 21 Feb 2024 16:17:10 +0200 Subject: [PATCH 422/812] sync : ggml (#5633) * ggml : fix conv_2d batch mode (ggml/737) Co-authored-by: bssrdf * ggml : compute forward no longer pass src tensors (ggml/729) * sync : ggml ggml-ci --------- Co-authored-by: bssrdf Co-authored-by: bssrdf --- ggml.c | 1150 +++++++++++++++++++++++++--------------- scripts/sync-ggml.last | 2 +- 2 files changed, 711 insertions(+), 441 deletions(-) diff --git a/ggml.c b/ggml.c index 91adbb0ae..5b9fa741a 100644 --- a/ggml.c +++ b/ggml.c @@ -5644,7 +5644,9 @@ struct ggml_tensor * ggml_conv_2d( ggml_reshape_2d(ctx, im2col, im2col->ne[0], im2col->ne[3] * im2col->ne[2] * im2col->ne[1]), // [N, OH, OW, IC * KH * KW] => [N*OH*OW, IC * KH * KW] ggml_reshape_2d(ctx, a, (a->ne[0] * a->ne[1] * a->ne[2]), a->ne[3])); // [OC,IC, KH, KW] => [OC, IC * KH * KW] - result = ggml_reshape_4d(ctx, result, im2col->ne[1], im2col->ne[2], a->ne[3], im2col->ne[3]); // [N, OC, OH, OW] + result = ggml_reshape_4d(ctx, result, im2col->ne[1], im2col->ne[2], im2col->ne[3], a->ne[3]); // [OC, N, OH, OW] + result = ggml_cont(ctx, ggml_permute(ctx, result, 0, 1, 3, 2)); // [N, OC, OH, OW] + return result; } @@ -6650,8 +6652,10 @@ void ggml_set_param( static void ggml_compute_forward_dup_same_cont( const struct ggml_compute_params * params, - const struct ggml_tensor * src0, struct ggml_tensor * dst) { + + const struct ggml_tensor * src0 = dst->src[0]; + GGML_ASSERT(ggml_nelements(dst) == ggml_nelements(src0)); GGML_ASSERT(ggml_is_contiguous(dst) && ggml_is_contiguous(src0)); GGML_ASSERT(src0->type == dst->type); @@ -6682,8 +6686,10 @@ static void ggml_compute_forward_dup_same_cont( } static void ggml_compute_forward_dup_f16( const struct ggml_compute_params * params, - const struct ggml_tensor * src0, struct ggml_tensor * dst) { + + const struct ggml_tensor * src0 = dst->src[0]; + GGML_ASSERT(ggml_nelements(dst) == ggml_nelements(src0)); if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) { @@ -6696,7 +6702,7 @@ static void ggml_compute_forward_dup_f16( const int nth = params->nth; // number of threads if (ggml_is_contiguous(src0) && ggml_is_contiguous(dst) && src0->type == dst->type) { - ggml_compute_forward_dup_same_cont(params, src0, dst); + ggml_compute_forward_dup_same_cont(params, dst); return; } @@ -6953,8 +6959,10 @@ static void ggml_compute_forward_dup_f16( static void ggml_compute_forward_dup_f32( const struct ggml_compute_params * params, - const struct ggml_tensor * src0, struct ggml_tensor * dst) { + + const struct ggml_tensor * src0 = dst->src[0]; + GGML_ASSERT(ggml_nelements(dst) == ggml_nelements(src0)); if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) { @@ -6967,7 +6975,7 @@ static void ggml_compute_forward_dup_f32( const int nth = params->nth; // number of threads if (ggml_is_contiguous(src0) && ggml_is_contiguous(dst) && src0->type == dst->type) { - ggml_compute_forward_dup_same_cont(params, src0, dst); + ggml_compute_forward_dup_same_cont(params, dst); return; } @@ -7203,8 +7211,10 @@ static void ggml_compute_forward_dup_f32( // A simplified version of ggml_compute_forward_dup that doesn't do float upcasting, and just plain old memcpy. static void ggml_compute_forward_dup_bytes( const struct ggml_compute_params * params, - const struct ggml_tensor * src0, struct ggml_tensor * dst) { + + const struct ggml_tensor * src0 = dst->src[0]; + GGML_ASSERT(ggml_nelements(dst) == ggml_nelements(src0)); GGML_ASSERT(src0->type == dst->type); @@ -7213,7 +7223,7 @@ static void ggml_compute_forward_dup_bytes( } if (ggml_is_contiguous(src0) && ggml_is_contiguous(dst)) { - ggml_compute_forward_dup_same_cont(params, src0, dst); + ggml_compute_forward_dup_same_cont(params, dst); return; } @@ -7352,21 +7362,23 @@ static void ggml_compute_forward_dup_bytes( static void ggml_compute_forward_dup( const struct ggml_compute_params * params, - const struct ggml_tensor * src0, struct ggml_tensor * dst) { + + const struct ggml_tensor * src0 = dst->src[0]; + if (src0->type == dst->type) { - ggml_compute_forward_dup_bytes(params, src0, dst); + ggml_compute_forward_dup_bytes(params, dst); return; } switch (src0->type) { case GGML_TYPE_F16: { - ggml_compute_forward_dup_f16(params, src0, dst); + ggml_compute_forward_dup_f16(params, dst); } break; case GGML_TYPE_F32: { - ggml_compute_forward_dup_f32(params, src0, dst); + ggml_compute_forward_dup_f32(params, dst); } break; default: { @@ -7379,9 +7391,11 @@ static void ggml_compute_forward_dup( static void ggml_compute_forward_add_f32( const struct ggml_compute_params * params, - const struct ggml_tensor * src0, - const struct ggml_tensor * src1, struct ggml_tensor * dst) { + + const struct ggml_tensor * src0 = dst->src[0]; + const struct ggml_tensor * src1 = dst->src[1]; + GGML_ASSERT(ggml_can_repeat(src1, src0) && ggml_are_same_shape(src0, dst)); if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) { @@ -7467,9 +7481,11 @@ static void ggml_compute_forward_add_f32( static void ggml_compute_forward_add_f16_f32( const struct ggml_compute_params * params, - const struct ggml_tensor * src0, - const struct ggml_tensor * src1, struct ggml_tensor * dst) { + + const struct ggml_tensor * src0 = dst->src[0]; + const struct ggml_tensor * src1 = dst->src[1]; + GGML_ASSERT(ggml_are_same_shape(src0, src1) && ggml_are_same_shape(src0, dst)); if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) { @@ -7544,9 +7560,11 @@ static void ggml_compute_forward_add_f16_f32( static void ggml_compute_forward_add_f16_f16( const struct ggml_compute_params * params, - const struct ggml_tensor * src0, - const struct ggml_tensor * src1, struct ggml_tensor * dst) { + + const struct ggml_tensor * src0 = dst->src[0]; + const struct ggml_tensor * src1 = dst->src[1]; + GGML_ASSERT(ggml_are_same_shape(src0, src1) && ggml_are_same_shape(src0, dst)); if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) { @@ -7598,9 +7616,11 @@ static void ggml_compute_forward_add_f16_f16( static void ggml_compute_forward_add_q_f32( const struct ggml_compute_params * params, - const struct ggml_tensor * src0, - const struct ggml_tensor * src1, struct ggml_tensor * dst) { + + const struct ggml_tensor * src0 = dst->src[0]; + const struct ggml_tensor * src1 = dst->src[1]; + GGML_ASSERT(ggml_are_same_shape(src0, src1) && ggml_are_same_shape(src0, dst)); if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) { @@ -7676,14 +7696,16 @@ static void ggml_compute_forward_add_q_f32( static void ggml_compute_forward_add( const struct ggml_compute_params * params, - const struct ggml_tensor * src0, - const struct ggml_tensor * src1, struct ggml_tensor * dst) { + + const struct ggml_tensor * src0 = dst->src[0]; + const struct ggml_tensor * src1 = dst->src[1]; + switch (src0->type) { case GGML_TYPE_F32: { if (src1->type == GGML_TYPE_F32) { - ggml_compute_forward_add_f32(params, src0, src1, dst); + ggml_compute_forward_add_f32(params, dst); } else { GGML_ASSERT(false); @@ -7692,10 +7714,10 @@ static void ggml_compute_forward_add( case GGML_TYPE_F16: { if (src1->type == GGML_TYPE_F16) { - ggml_compute_forward_add_f16_f16(params, src0, src1, dst); + ggml_compute_forward_add_f16_f16(params, dst); } else if (src1->type == GGML_TYPE_F32) { - ggml_compute_forward_add_f16_f32(params, src0, src1, dst); + ggml_compute_forward_add_f16_f32(params, dst); } else { GGML_ASSERT(false); @@ -7717,7 +7739,7 @@ static void ggml_compute_forward_add( case GGML_TYPE_IQ1_S: case GGML_TYPE_IQ4_NL: { - ggml_compute_forward_add_q_f32(params, src0, src1, dst); + ggml_compute_forward_add_q_f32(params, dst); } break; default: { @@ -7730,9 +7752,11 @@ static void ggml_compute_forward_add( static void ggml_compute_forward_add1_f32( const struct ggml_compute_params * params, - const struct ggml_tensor * src0, - const struct ggml_tensor * src1, struct ggml_tensor * dst) { + + const struct ggml_tensor * src0 = dst->src[0]; + const struct ggml_tensor * src1 = dst->src[1]; + GGML_ASSERT(ggml_are_same_shape(src0, dst)); GGML_ASSERT(ggml_is_scalar(src1)); @@ -7782,9 +7806,11 @@ static void ggml_compute_forward_add1_f32( static void ggml_compute_forward_add1_f16_f32( const struct ggml_compute_params * params, - const struct ggml_tensor * src0, - const struct ggml_tensor * src1, struct ggml_tensor * dst) { + + const struct ggml_tensor * src0 = dst->src[0]; + const struct ggml_tensor * src1 = dst->src[1]; + GGML_ASSERT(ggml_are_same_shape(src0, dst)); GGML_ASSERT(ggml_is_scalar(src1)); @@ -7832,9 +7858,11 @@ static void ggml_compute_forward_add1_f16_f32( static void ggml_compute_forward_add1_f16_f16( const struct ggml_compute_params * params, - const struct ggml_tensor * src0, - const struct ggml_tensor * src1, struct ggml_tensor * dst) { + + const struct ggml_tensor * src0 = dst->src[0]; + const struct ggml_tensor * src1 = dst->src[1]; + GGML_ASSERT(ggml_are_same_shape(src0, dst)); GGML_ASSERT(ggml_is_scalar(src1)); @@ -7882,9 +7910,11 @@ static void ggml_compute_forward_add1_f16_f16( static void ggml_compute_forward_add1_q_f32( const struct ggml_compute_params * params, - const struct ggml_tensor * src0, - const struct ggml_tensor * src1, struct ggml_tensor * dst) { + + const struct ggml_tensor * src0 = dst->src[0]; + const struct ggml_tensor * src1 = dst->src[1]; + GGML_ASSERT(ggml_are_same_shape(src0, dst)); GGML_ASSERT(ggml_is_scalar(src1)); @@ -7949,21 +7979,23 @@ static void ggml_compute_forward_add1_q_f32( static void ggml_compute_forward_add1( const struct ggml_compute_params * params, - const struct ggml_tensor * src0, - const struct ggml_tensor * src1, struct ggml_tensor * dst) { + + const struct ggml_tensor * src0 = dst->src[0]; + const struct ggml_tensor * src1 = dst->src[1]; + switch (src0->type) { case GGML_TYPE_F32: { - ggml_compute_forward_add1_f32(params, src0, src1, dst); + ggml_compute_forward_add1_f32(params, dst); } break; case GGML_TYPE_F16: { if (src1->type == GGML_TYPE_F16) { - ggml_compute_forward_add1_f16_f16(params, src0, src1, dst); + ggml_compute_forward_add1_f16_f16(params, dst); } else if (src1->type == GGML_TYPE_F32) { - ggml_compute_forward_add1_f16_f32(params, src0, src1, dst); + ggml_compute_forward_add1_f16_f32(params, dst); } else { GGML_ASSERT(false); @@ -7986,7 +8018,7 @@ static void ggml_compute_forward_add1( case GGML_TYPE_IQ1_S: case GGML_TYPE_IQ4_NL: { - ggml_compute_forward_add1_q_f32(params, src0, src1, dst); + ggml_compute_forward_add1_q_f32(params, dst); } break; default: { @@ -7999,9 +8031,11 @@ static void ggml_compute_forward_add1( static void ggml_compute_forward_acc_f32( const struct ggml_compute_params * params, - const struct ggml_tensor * src0, - const struct ggml_tensor * src1, struct ggml_tensor * dst) { + + const struct ggml_tensor * src0 = dst->src[0]; + const struct ggml_tensor * src1 = dst->src[1]; + GGML_ASSERT(ggml_are_same_shape(src0, dst)); GGML_ASSERT(ggml_is_contiguous(dst) && ggml_is_contiguous(src0)); @@ -8081,14 +8115,14 @@ static void ggml_compute_forward_acc_f32( static void ggml_compute_forward_acc( const struct ggml_compute_params * params, - const struct ggml_tensor * src0, - const struct ggml_tensor * src1, struct ggml_tensor * dst) { + const struct ggml_tensor * src0 = dst->src[0]; + switch (src0->type) { case GGML_TYPE_F32: { - ggml_compute_forward_acc_f32(params, src0, src1, dst); + ggml_compute_forward_acc_f32(params, dst); } break; case GGML_TYPE_F16: case GGML_TYPE_Q4_0: @@ -8118,9 +8152,11 @@ static void ggml_compute_forward_acc( static void ggml_compute_forward_sub_f32( const struct ggml_compute_params * params, - const struct ggml_tensor * src0, - const struct ggml_tensor * src1, struct ggml_tensor * dst) { + + const struct ggml_tensor * src0 = dst->src[0]; + const struct ggml_tensor * src1 = dst->src[1]; + assert(params->ith == 0); assert(ggml_are_same_shape(src0, src1) && ggml_are_same_shape(src0, dst)); @@ -8178,13 +8214,14 @@ static void ggml_compute_forward_sub_f32( static void ggml_compute_forward_sub( const struct ggml_compute_params * params, - const struct ggml_tensor * src0, - const struct ggml_tensor * src1, struct ggml_tensor * dst) { + + const struct ggml_tensor * src0 = dst->src[0]; + switch (src0->type) { case GGML_TYPE_F32: { - ggml_compute_forward_sub_f32(params, src0, src1, dst); + ggml_compute_forward_sub_f32(params, dst); } break; default: { @@ -8197,9 +8234,11 @@ static void ggml_compute_forward_sub( static void ggml_compute_forward_mul_f32( const struct ggml_compute_params * params, - const struct ggml_tensor * src0, - const struct ggml_tensor * src1, struct ggml_tensor * dst) { + + const struct ggml_tensor * src0 = dst->src[0]; + const struct ggml_tensor * src1 = dst->src[1]; + GGML_ASSERT(ggml_can_repeat(src1, src0) && ggml_are_same_shape(src0, dst)); if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) { @@ -8280,15 +8319,17 @@ static void ggml_compute_forward_mul_f32( static void ggml_compute_forward_mul( const struct ggml_compute_params * params, - const struct ggml_tensor * src0, - const struct ggml_tensor * src1, struct ggml_tensor * dst) { + + const struct ggml_tensor * src0 = dst->src[0]; + const struct ggml_tensor * src1 = dst->src[1]; + GGML_ASSERT(src1->type == GGML_TYPE_F32 && "only f32 src1 supported for now"); switch (src0->type) { case GGML_TYPE_F32: { - ggml_compute_forward_mul_f32(params, src0, src1, dst); + ggml_compute_forward_mul_f32(params, dst); } break; default: { @@ -8301,9 +8342,11 @@ static void ggml_compute_forward_mul( static void ggml_compute_forward_div_f32( const struct ggml_compute_params * params, - const struct ggml_tensor * src0, - const struct ggml_tensor * src1, struct ggml_tensor * dst) { + + const struct ggml_tensor * src0 = dst->src[0]; + const struct ggml_tensor * src1 = dst->src[1]; + GGML_ASSERT(ggml_can_repeat(src1, src0) && ggml_are_same_shape(src0, dst)); if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) { @@ -8374,13 +8417,14 @@ static void ggml_compute_forward_div_f32( static void ggml_compute_forward_div( const struct ggml_compute_params * params, - const struct ggml_tensor * src0, - const struct ggml_tensor * src1, struct ggml_tensor * dst) { + + const struct ggml_tensor * src0 = dst->src[0]; + switch (src0->type) { case GGML_TYPE_F32: { - ggml_compute_forward_div_f32(params, src0, src1, dst); + ggml_compute_forward_div_f32(params, dst); } break; default: { @@ -8393,8 +8437,10 @@ static void ggml_compute_forward_div( static void ggml_compute_forward_sqr_f32( const struct ggml_compute_params * params, - const struct ggml_tensor * src0, struct ggml_tensor * dst) { + + const struct ggml_tensor * src0 = dst->src[0]; + assert(params->ith == 0); assert(ggml_are_same_shape(src0, dst)); @@ -8417,12 +8463,14 @@ static void ggml_compute_forward_sqr_f32( static void ggml_compute_forward_sqr( const struct ggml_compute_params * params, - const struct ggml_tensor * src0, struct ggml_tensor * dst) { + + const struct ggml_tensor * src0 = dst->src[0]; + switch (src0->type) { case GGML_TYPE_F32: { - ggml_compute_forward_sqr_f32(params, src0, dst); + ggml_compute_forward_sqr_f32(params, dst); } break; default: { @@ -8435,8 +8483,10 @@ static void ggml_compute_forward_sqr( static void ggml_compute_forward_sqrt_f32( const struct ggml_compute_params * params, - const struct ggml_tensor * src0, struct ggml_tensor * dst) { + + const struct ggml_tensor * src0 = dst->src[0]; + assert(params->ith == 0); assert(ggml_are_same_shape(src0, dst)); @@ -8459,12 +8509,14 @@ static void ggml_compute_forward_sqrt_f32( static void ggml_compute_forward_sqrt( const struct ggml_compute_params * params, - const struct ggml_tensor * src0, struct ggml_tensor * dst) { + + const struct ggml_tensor * src0 = dst->src[0]; + switch (src0->type) { case GGML_TYPE_F32: { - ggml_compute_forward_sqrt_f32(params, src0, dst); + ggml_compute_forward_sqrt_f32(params, dst); } break; default: { @@ -8477,8 +8529,10 @@ static void ggml_compute_forward_sqrt( static void ggml_compute_forward_log_f32( const struct ggml_compute_params * params, - const struct ggml_tensor * src0, struct ggml_tensor * dst) { + + const struct ggml_tensor * src0 = dst->src[0]; + GGML_ASSERT(params->ith == 0); GGML_ASSERT(ggml_are_same_shape(src0, dst)); @@ -8501,12 +8555,14 @@ static void ggml_compute_forward_log_f32( static void ggml_compute_forward_log( const struct ggml_compute_params * params, - const struct ggml_tensor * src0, struct ggml_tensor * dst) { + + const struct ggml_tensor * src0 = dst->src[0]; + switch (src0->type) { case GGML_TYPE_F32: { - ggml_compute_forward_log_f32(params, src0, dst); + ggml_compute_forward_log_f32(params, dst); } break; default: { @@ -8519,8 +8575,10 @@ static void ggml_compute_forward_log( static void ggml_compute_forward_sum_f32( const struct ggml_compute_params * params, - const struct ggml_tensor * src0, struct ggml_tensor * dst) { + + const struct ggml_tensor * src0 = dst->src[0]; + assert(params->ith == 0); assert(ggml_is_scalar(dst)); @@ -8552,8 +8610,10 @@ static void ggml_compute_forward_sum_f32( static void ggml_compute_forward_sum_f16( const struct ggml_compute_params * params, - const struct ggml_tensor * src0, struct ggml_tensor * dst) { + + const struct ggml_tensor * src0 = dst->src[0]; + assert(params->ith == 0); assert(ggml_is_scalar(dst)); @@ -8584,16 +8644,18 @@ static void ggml_compute_forward_sum_f16( static void ggml_compute_forward_sum( const struct ggml_compute_params * params, - const struct ggml_tensor * src0, struct ggml_tensor * dst) { + + const struct ggml_tensor * src0 = dst->src[0]; + switch (src0->type) { case GGML_TYPE_F32: { - ggml_compute_forward_sum_f32(params, src0, dst); + ggml_compute_forward_sum_f32(params, dst); } break; case GGML_TYPE_F16: { - ggml_compute_forward_sum_f16(params, src0, dst); + ggml_compute_forward_sum_f16(params, dst); } break; default: { @@ -8606,8 +8668,10 @@ static void ggml_compute_forward_sum( static void ggml_compute_forward_sum_rows_f32( const struct ggml_compute_params * params, - const struct ggml_tensor * src0, struct ggml_tensor * dst) { + + const struct ggml_tensor * src0 = dst->src[0]; + GGML_ASSERT(params->ith == 0); if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) { @@ -8639,12 +8703,14 @@ static void ggml_compute_forward_sum_rows_f32( static void ggml_compute_forward_sum_rows( const struct ggml_compute_params * params, - const struct ggml_tensor * src0, struct ggml_tensor * dst) { + + const struct ggml_tensor * src0 = dst->src[0]; + switch (src0->type) { case GGML_TYPE_F32: { - ggml_compute_forward_sum_rows_f32(params, src0, dst); + ggml_compute_forward_sum_rows_f32(params, dst); } break; default: { @@ -8657,8 +8723,10 @@ static void ggml_compute_forward_sum_rows( static void ggml_compute_forward_mean_f32( const struct ggml_compute_params * params, - const struct ggml_tensor * src0, struct ggml_tensor * dst) { + + const struct ggml_tensor * src0 = dst->src[0]; + assert(params->ith == 0); if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) { @@ -8694,12 +8762,14 @@ static void ggml_compute_forward_mean_f32( static void ggml_compute_forward_mean( const struct ggml_compute_params * params, - const struct ggml_tensor * src0, struct ggml_tensor * dst) { + + const struct ggml_tensor * src0 = dst->src[0]; + switch (src0->type) { case GGML_TYPE_F32: { - ggml_compute_forward_mean_f32(params, src0, dst); + ggml_compute_forward_mean_f32(params, dst); } break; default: { @@ -8712,8 +8782,10 @@ static void ggml_compute_forward_mean( static void ggml_compute_forward_argmax_f32( const struct ggml_compute_params * params, - const struct ggml_tensor * src0, struct ggml_tensor * dst) { + + const struct ggml_tensor * src0 = dst->src[0]; + assert(params->ith == 0); if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) { @@ -8740,12 +8812,14 @@ static void ggml_compute_forward_argmax_f32( static void ggml_compute_forward_argmax( const struct ggml_compute_params * params, - const struct ggml_tensor * src0, struct ggml_tensor * dst) { + + const struct ggml_tensor * src0 = dst->src[0]; + switch (src0->type) { case GGML_TYPE_F32: { - ggml_compute_forward_argmax_f32(params, src0, dst); + ggml_compute_forward_argmax_f32(params, dst); } break; default: { @@ -8758,8 +8832,10 @@ static void ggml_compute_forward_argmax( static void ggml_compute_forward_repeat_f32( const struct ggml_compute_params * params, - const struct ggml_tensor * src0, struct ggml_tensor * dst) { + + const struct ggml_tensor * src0 = dst->src[0]; + GGML_ASSERT(params->ith == 0); GGML_ASSERT(ggml_can_repeat(src0, dst)); @@ -8801,8 +8877,10 @@ static void ggml_compute_forward_repeat_f32( static void ggml_compute_forward_repeat_f16( const struct ggml_compute_params * params, - const struct ggml_tensor * src0, struct ggml_tensor * dst) { + + const struct ggml_tensor * src0 = dst->src[0]; + GGML_ASSERT(params->ith == 0); GGML_ASSERT(ggml_can_repeat(src0, dst)); @@ -8847,18 +8925,20 @@ static void ggml_compute_forward_repeat_f16( static void ggml_compute_forward_repeat( const struct ggml_compute_params * params, - const struct ggml_tensor * src0, struct ggml_tensor * dst) { + + const struct ggml_tensor * src0 = dst->src[0]; + switch (src0->type) { case GGML_TYPE_F16: case GGML_TYPE_I16: { - ggml_compute_forward_repeat_f16(params, src0, dst); + ggml_compute_forward_repeat_f16(params, dst); } break; case GGML_TYPE_F32: case GGML_TYPE_I32: { - ggml_compute_forward_repeat_f32(params, src0, dst); + ggml_compute_forward_repeat_f32(params, dst); } break; default: { @@ -8871,8 +8951,10 @@ static void ggml_compute_forward_repeat( static void ggml_compute_forward_repeat_back_f32( const struct ggml_compute_params * params, - const struct ggml_tensor * src0, struct ggml_tensor * dst) { + + const struct ggml_tensor * src0 = dst->src[0]; + GGML_ASSERT(params->ith == 0); GGML_ASSERT(ggml_can_repeat(dst, src0)); @@ -8928,12 +9010,14 @@ static void ggml_compute_forward_repeat_back_f32( static void ggml_compute_forward_repeat_back( const struct ggml_compute_params * params, - const struct ggml_tensor * src0, struct ggml_tensor * dst) { + + const struct ggml_tensor * src0 = dst->src[0]; + switch (src0->type) { case GGML_TYPE_F32: { - ggml_compute_forward_repeat_back_f32(params, src0, dst); + ggml_compute_forward_repeat_back_f32(params, dst); } break; default: { @@ -8946,10 +9030,11 @@ static void ggml_compute_forward_repeat_back( static void ggml_compute_forward_concat_f32( const struct ggml_compute_params * params, - const struct ggml_tensor * src0, - const struct ggml_tensor * src1, struct ggml_tensor * dst) { + const struct ggml_tensor * src0 = dst->src[0]; + const struct ggml_tensor * src1 = dst->src[1]; + if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) { return; } @@ -8994,14 +9079,15 @@ static void ggml_compute_forward_concat_f32( static void ggml_compute_forward_concat( const struct ggml_compute_params* params, - const struct ggml_tensor* src0, - const struct ggml_tensor* src1, struct ggml_tensor* dst) { + + const struct ggml_tensor * src0 = dst->src[0]; + switch (src0->type) { case GGML_TYPE_F32: case GGML_TYPE_I32: { - ggml_compute_forward_concat_f32(params, src0, src1, dst); + ggml_compute_forward_concat_f32(params, dst); } break; default: { @@ -9014,8 +9100,10 @@ static void ggml_compute_forward_concat( static void ggml_compute_forward_abs_f32( const struct ggml_compute_params * params, - const struct ggml_tensor * src0, struct ggml_tensor * dst) { + + const struct ggml_tensor * src0 = dst->src[0]; + assert(params->ith == 0); assert(ggml_are_same_shape(src0, dst)); @@ -9038,12 +9126,14 @@ static void ggml_compute_forward_abs_f32( static void ggml_compute_forward_abs( const struct ggml_compute_params * params, - const struct ggml_tensor * src0, struct ggml_tensor * dst) { + + const struct ggml_tensor * src0 = dst->src[0]; + switch (src0->type) { case GGML_TYPE_F32: { - ggml_compute_forward_abs_f32(params, src0, dst); + ggml_compute_forward_abs_f32(params, dst); } break; default: { @@ -9056,8 +9146,10 @@ static void ggml_compute_forward_abs( static void ggml_compute_forward_sgn_f32( const struct ggml_compute_params * params, - const struct ggml_tensor * src0, struct ggml_tensor * dst) { + + const struct ggml_tensor * src0 = dst->src[0]; + assert(params->ith == 0); assert(ggml_are_same_shape(src0, dst)); @@ -9080,12 +9172,14 @@ static void ggml_compute_forward_sgn_f32( static void ggml_compute_forward_sgn( const struct ggml_compute_params * params, - const struct ggml_tensor * src0, struct ggml_tensor * dst) { + + const struct ggml_tensor * src0 = dst->src[0]; + switch (src0->type) { case GGML_TYPE_F32: { - ggml_compute_forward_sgn_f32(params, src0, dst); + ggml_compute_forward_sgn_f32(params, dst); } break; default: { @@ -9098,8 +9192,10 @@ static void ggml_compute_forward_sgn( static void ggml_compute_forward_neg_f32( const struct ggml_compute_params * params, - const struct ggml_tensor * src0, struct ggml_tensor * dst) { + + const struct ggml_tensor * src0 = dst->src[0]; + assert(params->ith == 0); assert(ggml_are_same_shape(src0, dst)); @@ -9122,12 +9218,14 @@ static void ggml_compute_forward_neg_f32( static void ggml_compute_forward_neg( const struct ggml_compute_params * params, - const struct ggml_tensor * src0, struct ggml_tensor * dst) { + + const struct ggml_tensor * src0 = dst->src[0]; + switch (src0->type) { case GGML_TYPE_F32: { - ggml_compute_forward_neg_f32(params, src0, dst); + ggml_compute_forward_neg_f32(params, dst); } break; default: { @@ -9140,8 +9238,10 @@ static void ggml_compute_forward_neg( static void ggml_compute_forward_step_f32( const struct ggml_compute_params * params, - const struct ggml_tensor * src0, struct ggml_tensor * dst) { + + const struct ggml_tensor * src0 = dst->src[0]; + assert(params->ith == 0); assert(ggml_are_same_shape(src0, dst)); @@ -9164,12 +9264,14 @@ static void ggml_compute_forward_step_f32( static void ggml_compute_forward_step( const struct ggml_compute_params * params, - const struct ggml_tensor * src0, struct ggml_tensor * dst) { + + const struct ggml_tensor * src0 = dst->src[0]; + switch (src0->type) { case GGML_TYPE_F32: { - ggml_compute_forward_step_f32(params, src0, dst); + ggml_compute_forward_step_f32(params, dst); } break; default: { @@ -9182,8 +9284,10 @@ static void ggml_compute_forward_step( static void ggml_compute_forward_tanh_f32( const struct ggml_compute_params * params, - const struct ggml_tensor * src0, struct ggml_tensor * dst) { + + const struct ggml_tensor * src0 = dst->src[0]; + assert(params->ith == 0); assert(ggml_are_same_shape(src0, dst)); @@ -9206,12 +9310,14 @@ static void ggml_compute_forward_tanh_f32( static void ggml_compute_forward_tanh( const struct ggml_compute_params * params, - const struct ggml_tensor * src0, struct ggml_tensor * dst) { + + const struct ggml_tensor * src0 = dst->src[0]; + switch (src0->type) { case GGML_TYPE_F32: { - ggml_compute_forward_tanh_f32(params, src0, dst); + ggml_compute_forward_tanh_f32(params, dst); } break; default: { @@ -9224,8 +9330,10 @@ static void ggml_compute_forward_tanh( static void ggml_compute_forward_elu_f32( const struct ggml_compute_params * params, - const struct ggml_tensor * src0, struct ggml_tensor * dst) { + + const struct ggml_tensor * src0 = dst->src[0]; + assert(params->ith == 0); assert(ggml_are_same_shape(src0, dst)); @@ -9248,12 +9356,14 @@ static void ggml_compute_forward_elu_f32( static void ggml_compute_forward_elu( const struct ggml_compute_params * params, - const struct ggml_tensor * src0, struct ggml_tensor * dst) { + + const struct ggml_tensor * src0 = dst->src[0]; + switch (src0->type) { case GGML_TYPE_F32: { - ggml_compute_forward_elu_f32(params, src0, dst); + ggml_compute_forward_elu_f32(params, dst); } break; default: { @@ -9266,8 +9376,10 @@ static void ggml_compute_forward_elu( static void ggml_compute_forward_relu_f32( const struct ggml_compute_params * params, - const struct ggml_tensor * src0, struct ggml_tensor * dst) { + + const struct ggml_tensor * src0 = dst->src[0]; + assert(params->ith == 0); assert(ggml_are_same_shape(src0, dst)); @@ -9290,12 +9402,14 @@ static void ggml_compute_forward_relu_f32( static void ggml_compute_forward_relu( const struct ggml_compute_params * params, - const struct ggml_tensor * src0, struct ggml_tensor * dst) { + + const struct ggml_tensor * src0 = dst->src[0]; + switch (src0->type) { case GGML_TYPE_F32: { - ggml_compute_forward_relu_f32(params, src0, dst); + ggml_compute_forward_relu_f32(params, dst); } break; default: { @@ -9308,8 +9422,10 @@ static void ggml_compute_forward_relu( static void ggml_compute_forward_gelu_f32( const struct ggml_compute_params * params, - const struct ggml_tensor * src0, struct ggml_tensor * dst) { + + const struct ggml_tensor * src0 = dst->src[0]; + GGML_ASSERT(ggml_is_contiguous_except_dim_1(src0)); GGML_ASSERT(ggml_is_contiguous_except_dim_1(dst)); GGML_ASSERT(ggml_are_same_shape(src0, dst)); @@ -9349,12 +9465,14 @@ static void ggml_compute_forward_gelu_f32( static void ggml_compute_forward_gelu( const struct ggml_compute_params * params, - const struct ggml_tensor * src0, struct ggml_tensor * dst) { + + const struct ggml_tensor * src0 = dst->src[0]; + switch (src0->type) { case GGML_TYPE_F32: { - ggml_compute_forward_gelu_f32(params, src0, dst); + ggml_compute_forward_gelu_f32(params, dst); } break; default: { @@ -9367,8 +9485,10 @@ static void ggml_compute_forward_gelu( static void ggml_compute_forward_gelu_quick_f32( const struct ggml_compute_params * params, - const struct ggml_tensor * src0, struct ggml_tensor * dst) { + + const struct ggml_tensor * src0 = dst->src[0]; + GGML_ASSERT(ggml_is_contiguous_except_dim_1(src0)); GGML_ASSERT(ggml_is_contiguous_except_dim_1(dst)); GGML_ASSERT(ggml_are_same_shape(src0, dst)); @@ -9408,12 +9528,14 @@ static void ggml_compute_forward_gelu_quick_f32( static void ggml_compute_forward_gelu_quick( const struct ggml_compute_params * params, - const struct ggml_tensor * src0, struct ggml_tensor * dst) { + + const struct ggml_tensor * src0 = dst->src[0]; + switch (src0->type) { case GGML_TYPE_F32: { - ggml_compute_forward_gelu_quick_f32(params, src0, dst); + ggml_compute_forward_gelu_quick_f32(params, dst); } break; default: { @@ -9426,8 +9548,10 @@ static void ggml_compute_forward_gelu_quick( static void ggml_compute_forward_silu_f32( const struct ggml_compute_params * params, - const struct ggml_tensor * src0, struct ggml_tensor * dst) { + + const struct ggml_tensor * src0 = dst->src[0]; + GGML_ASSERT(ggml_is_contiguous_except_dim_1(src0)); GGML_ASSERT(ggml_is_contiguous_except_dim_1(dst)); GGML_ASSERT(ggml_are_same_shape(src0, dst)); @@ -9467,12 +9591,14 @@ static void ggml_compute_forward_silu_f32( static void ggml_compute_forward_silu( const struct ggml_compute_params * params, - const struct ggml_tensor * src0, struct ggml_tensor * dst) { + + const struct ggml_tensor * src0 = dst->src[0]; + switch (src0->type) { case GGML_TYPE_F32: { - ggml_compute_forward_silu_f32(params, src0, dst); + ggml_compute_forward_silu_f32(params, dst); } break; default: { @@ -9484,8 +9610,10 @@ static void ggml_compute_forward_silu( static void ggml_compute_forward_leaky_relu_f32( const struct ggml_compute_params * params, - const struct ggml_tensor * src0, struct ggml_tensor * dst) { + + const struct ggml_tensor * src0 = dst->src[0]; + assert(params->ith == 0); assert(ggml_are_same_shape(src0, dst)); @@ -9511,12 +9639,14 @@ static void ggml_compute_forward_leaky_relu_f32( static void ggml_compute_forward_leaky_relu( const struct ggml_compute_params * params, - const struct ggml_tensor * src0, struct ggml_tensor * dst) { + + const struct ggml_tensor * src0 = dst->src[0]; + switch (src0->type) { case GGML_TYPE_F32: { - ggml_compute_forward_leaky_relu_f32(params, src0, dst); + ggml_compute_forward_leaky_relu_f32(params, dst); } break; default: { @@ -9529,9 +9659,11 @@ static void ggml_compute_forward_leaky_relu( static void ggml_compute_forward_silu_back_f32( const struct ggml_compute_params * params, - const struct ggml_tensor * src0, - const struct ggml_tensor * grad, struct ggml_tensor * dst) { + + const struct ggml_tensor * src0 = dst->src[0]; + const struct ggml_tensor * grad = dst->src[1]; + GGML_ASSERT(ggml_is_contiguous_except_dim_1(grad)); GGML_ASSERT(ggml_is_contiguous_except_dim_1(src0)); GGML_ASSERT(ggml_is_contiguous_except_dim_1(dst)); @@ -9574,13 +9706,14 @@ static void ggml_compute_forward_silu_back_f32( static void ggml_compute_forward_silu_back( const struct ggml_compute_params * params, - const struct ggml_tensor * src0, - const struct ggml_tensor * grad, struct ggml_tensor * dst) { + + const struct ggml_tensor * src0 = dst->src[0]; + switch (src0->type) { case GGML_TYPE_F32: { - ggml_compute_forward_silu_back_f32(params, src0, grad, dst); + ggml_compute_forward_silu_back_f32(params, dst); } break; default: { @@ -9592,8 +9725,10 @@ static void ggml_compute_forward_silu_back( static void ggml_compute_forward_hardswish_f32( const struct ggml_compute_params * params, - const struct ggml_tensor * src0, struct ggml_tensor * dst) { + + const struct ggml_tensor * src0 = dst->src[0]; + assert(params->ith == 0); assert(ggml_are_same_shape(src0, dst)); @@ -9615,12 +9750,14 @@ static void ggml_compute_forward_hardswish_f32( } static void ggml_compute_forward_hardswish( const struct ggml_compute_params * params, - const struct ggml_tensor * src0, struct ggml_tensor * dst) { + + const struct ggml_tensor * src0 = dst->src[0]; + switch (src0->type) { case GGML_TYPE_F32: { - ggml_compute_forward_hardswish_f32(params, src0, dst); + ggml_compute_forward_hardswish_f32(params, dst); } break; default: { @@ -9631,8 +9768,10 @@ static void ggml_compute_forward_hardswish( static void ggml_compute_forward_hardsigmoid_f32( const struct ggml_compute_params * params, - const struct ggml_tensor * src0, struct ggml_tensor * dst) { + + const struct ggml_tensor * src0 = dst->src[0]; + assert(params->ith == 0); assert(ggml_are_same_shape(src0, dst)); @@ -9655,12 +9794,14 @@ static void ggml_compute_forward_hardsigmoid_f32( static void ggml_compute_forward_hardsigmoid( const struct ggml_compute_params * params, - const struct ggml_tensor * src0, struct ggml_tensor * dst) { + + const struct ggml_tensor * src0 = dst->src[0]; + switch (src0->type) { case GGML_TYPE_F32: { - ggml_compute_forward_hardsigmoid_f32(params, src0, dst); + ggml_compute_forward_hardsigmoid_f32(params, dst); } break; default: { @@ -9674,8 +9815,10 @@ static void ggml_compute_forward_hardsigmoid( static void ggml_compute_forward_norm_f32( const struct ggml_compute_params * params, - const struct ggml_tensor * src0, struct ggml_tensor * dst) { + + const struct ggml_tensor * src0 = dst->src[0]; + GGML_ASSERT(ggml_are_same_shape(src0, dst)); if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) { @@ -9727,12 +9870,14 @@ static void ggml_compute_forward_norm_f32( static void ggml_compute_forward_norm( const struct ggml_compute_params * params, - const struct ggml_tensor * src0, struct ggml_tensor * dst) { + + const struct ggml_tensor * src0 = dst->src[0]; + switch (src0->type) { case GGML_TYPE_F32: { - ggml_compute_forward_norm_f32(params, src0, dst); + ggml_compute_forward_norm_f32(params, dst); } break; default: { @@ -9745,8 +9890,10 @@ static void ggml_compute_forward_norm( static void ggml_compute_forward_rms_norm_f32( const struct ggml_compute_params * params, - const struct ggml_tensor * src0, struct ggml_tensor * dst) { + + const struct ggml_tensor * src0 = dst->src[0]; + GGML_ASSERT(ggml_are_same_shape(src0, dst)); if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) { @@ -9795,12 +9942,14 @@ static void ggml_compute_forward_rms_norm_f32( static void ggml_compute_forward_rms_norm( const struct ggml_compute_params * params, - const struct ggml_tensor * src0, struct ggml_tensor * dst) { + + const struct ggml_tensor * src0 = dst->src[0]; + switch (src0->type) { case GGML_TYPE_F32: { - ggml_compute_forward_rms_norm_f32(params, src0, dst); + ggml_compute_forward_rms_norm_f32(params, dst); } break; default: { @@ -9811,9 +9960,11 @@ static void ggml_compute_forward_rms_norm( static void ggml_compute_forward_rms_norm_back_f32( const struct ggml_compute_params * params, - const struct ggml_tensor * src0, - const struct ggml_tensor * src1, struct ggml_tensor * dst) { + + const struct ggml_tensor * src0 = dst->src[0]; + const struct ggml_tensor * src1 = dst->src[1]; + GGML_ASSERT(ggml_are_same_shape(src0, dst) && ggml_are_same_shape(src0, src1)); if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) { @@ -9968,13 +10119,14 @@ static void ggml_compute_forward_rms_norm_back_f32( static void ggml_compute_forward_rms_norm_back( const struct ggml_compute_params * params, - const struct ggml_tensor * src0, - const struct ggml_tensor * src1, struct ggml_tensor * dst) { + + const struct ggml_tensor * src0 = dst->src[0]; + switch (src0->type) { case GGML_TYPE_F32: { - ggml_compute_forward_rms_norm_back_f32(params, src0, src1, dst); + ggml_compute_forward_rms_norm_back_f32(params, dst); } break; default: { @@ -9987,8 +10139,10 @@ static void ggml_compute_forward_rms_norm_back( static void ggml_compute_forward_group_norm_f32( const struct ggml_compute_params * params, - const struct ggml_tensor * src0, struct ggml_tensor * dst) { + + const struct ggml_tensor * src0 = dst->src[0]; + GGML_ASSERT(ggml_are_same_shape(src0, dst)); if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) { @@ -10059,12 +10213,14 @@ static void ggml_compute_forward_group_norm_f32( static void ggml_compute_forward_group_norm( const struct ggml_compute_params * params, - const struct ggml_tensor * src0, struct ggml_tensor * dst) { + + const struct ggml_tensor * src0 = dst->src[0]; + switch (src0->type) { case GGML_TYPE_F32: { - ggml_compute_forward_group_norm_f32(params, src0, dst); + ggml_compute_forward_group_norm_f32(params, dst); } break; default: { @@ -10110,9 +10266,11 @@ static bool ggml_compute_forward_mul_mat_use_blas(struct ggml_tensor * dst) { static void ggml_compute_forward_mul_mat( const struct ggml_compute_params * params, - const struct ggml_tensor * src0, - const struct ggml_tensor * src1, struct ggml_tensor * dst) { + + const struct ggml_tensor * src0 = dst->src[0]; + const struct ggml_tensor * src1 = dst->src[1]; + int64_t t0 = ggml_perf_time_us(); UNUSED(t0); @@ -10357,10 +10515,11 @@ static void ggml_compute_forward_mul_mat( static void ggml_compute_forward_mul_mat_id( const struct ggml_compute_params * params, - const struct ggml_tensor * ids, - const struct ggml_tensor * src1, struct ggml_tensor * dst) { + const struct ggml_tensor * ids = dst->src[0]; + const struct ggml_tensor * src1 = dst->src[1]; + const struct ggml_tensor * src0 = dst->src[2]; // only for GGML_TENSOR_BINARY_OP_LOCALS GGML_TENSOR_BINARY_OP_LOCALS @@ -10551,9 +10710,11 @@ static void ggml_compute_forward_mul_mat_id( static void ggml_compute_forward_out_prod_f32( const struct ggml_compute_params * params, - const struct ggml_tensor * src0, - const struct ggml_tensor * src1, struct ggml_tensor * dst) { + + const struct ggml_tensor * src0 = dst->src[0]; + const struct ggml_tensor * src1 = dst->src[1]; + // int64_t t0 = ggml_perf_time_us(); // UNUSED(t0); @@ -10743,9 +10904,11 @@ static void ggml_compute_forward_out_prod_f32( static void ggml_compute_forward_out_prod_q_f32( const struct ggml_compute_params * params, - const struct ggml_tensor * src0, - const struct ggml_tensor * src1, struct ggml_tensor * dst) { + + const struct ggml_tensor * src0 = dst->src[0]; + const struct ggml_tensor * src1 = dst->src[1]; + // int64_t t0 = ggml_perf_time_us(); // UNUSED(t0); @@ -10856,9 +11019,10 @@ static void ggml_compute_forward_out_prod_q_f32( static void ggml_compute_forward_out_prod( const struct ggml_compute_params * params, - const struct ggml_tensor * src0, - const struct ggml_tensor * src1, struct ggml_tensor * dst) { + + const struct ggml_tensor * src0 = dst->src[0]; + switch (src0->type) { case GGML_TYPE_Q4_0: case GGML_TYPE_Q4_1: @@ -10876,16 +11040,16 @@ static void ggml_compute_forward_out_prod( case GGML_TYPE_IQ1_S: case GGML_TYPE_IQ4_NL: { - ggml_compute_forward_out_prod_q_f32(params, src0, src1, dst); + ggml_compute_forward_out_prod_q_f32(params, dst); } break; case GGML_TYPE_F16: { GGML_ASSERT(false); // todo - // ggml_compute_forward_out_prod_f16_f32(params, src0, src1, dst); + // ggml_compute_forward_out_prod_f16_f32(params, dst); } break; case GGML_TYPE_F32: { - ggml_compute_forward_out_prod_f32(params, src0, src1, dst); + ggml_compute_forward_out_prod_f32(params, dst); } break; default: { @@ -10898,8 +11062,10 @@ static void ggml_compute_forward_out_prod( static void ggml_compute_forward_scale_f32( const struct ggml_compute_params * params, - const struct ggml_tensor * src0, struct ggml_tensor * dst) { + + const struct ggml_tensor * src0 = dst->src[0]; + GGML_ASSERT(ggml_is_contiguous(src0)); GGML_ASSERT(ggml_is_contiguous(dst)); GGML_ASSERT(ggml_are_same_shape(src0, dst)); @@ -10940,12 +11106,14 @@ static void ggml_compute_forward_scale_f32( static void ggml_compute_forward_scale( const struct ggml_compute_params * params, - const struct ggml_tensor * src0, struct ggml_tensor * dst) { + + const struct ggml_tensor * src0 = dst->src[0]; + switch (src0->type) { case GGML_TYPE_F32: { - ggml_compute_forward_scale_f32(params, src0, dst); + ggml_compute_forward_scale_f32(params, dst); } break; default: { @@ -10958,9 +11126,11 @@ static void ggml_compute_forward_scale( static void ggml_compute_forward_set_f32( const struct ggml_compute_params * params, - const struct ggml_tensor * src0, - const struct ggml_tensor * src1, struct ggml_tensor * dst) { + + const struct ggml_tensor * src0 = dst->src[0]; + const struct ggml_tensor * src1 = dst->src[1]; + GGML_ASSERT(ggml_are_same_shape(src0, dst)); GGML_ASSERT(ggml_is_contiguous(dst) && ggml_is_contiguous(src0)); @@ -11031,14 +11201,14 @@ static void ggml_compute_forward_set_f32( static void ggml_compute_forward_set( const struct ggml_compute_params * params, - const struct ggml_tensor * src0, - const struct ggml_tensor * src1, struct ggml_tensor * dst) { + const struct ggml_tensor * src0 = dst->src[0]; + switch (src0->type) { case GGML_TYPE_F32: { - ggml_compute_forward_set_f32(params, src0, src1, dst); + ggml_compute_forward_set_f32(params, dst); } break; case GGML_TYPE_F16: case GGML_TYPE_Q4_0: @@ -11068,29 +11238,25 @@ static void ggml_compute_forward_set( static void ggml_compute_forward_cpy( const struct ggml_compute_params * params, - const struct ggml_tensor * src0, struct ggml_tensor * dst) { - ggml_compute_forward_dup(params, src0, dst); + ggml_compute_forward_dup(params, dst); } // ggml_compute_forward_cont static void ggml_compute_forward_cont( const struct ggml_compute_params * params, - const struct ggml_tensor * src0, struct ggml_tensor * dst) { - ggml_compute_forward_dup(params, src0, dst); + ggml_compute_forward_dup(params, dst); } // ggml_compute_forward_reshape static void ggml_compute_forward_reshape( const struct ggml_compute_params * params, - const struct ggml_tensor * src0, struct ggml_tensor * dst) { // NOP UNUSED(params); - UNUSED(src0); UNUSED(dst); } @@ -11098,39 +11264,41 @@ static void ggml_compute_forward_reshape( static void ggml_compute_forward_view( const struct ggml_compute_params * params, - const struct ggml_tensor * src0) { + const struct ggml_tensor * dst) { // NOP UNUSED(params); - UNUSED(src0); + UNUSED(dst); } // ggml_compute_forward_permute static void ggml_compute_forward_permute( const struct ggml_compute_params * params, - const struct ggml_tensor * src0) { + const struct ggml_tensor * dst) { // NOP UNUSED(params); - UNUSED(src0); + UNUSED(dst); } // ggml_compute_forward_transpose static void ggml_compute_forward_transpose( const struct ggml_compute_params * params, - const struct ggml_tensor * src0) { + const struct ggml_tensor * dst) { // NOP UNUSED(params); - UNUSED(src0); + UNUSED(dst); } // ggml_compute_forward_get_rows static void ggml_compute_forward_get_rows_q( const struct ggml_compute_params * params, - const struct ggml_tensor * src0, - const struct ggml_tensor * src1, struct ggml_tensor * dst) { + + const struct ggml_tensor * src0 = dst->src[0]; + const struct ggml_tensor * src1 = dst->src[1]; + assert(params->ith == 0); if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) { @@ -11166,9 +11334,11 @@ static void ggml_compute_forward_get_rows_q( static void ggml_compute_forward_get_rows_f16( const struct ggml_compute_params * params, - const struct ggml_tensor * src0, - const struct ggml_tensor * src1, struct ggml_tensor * dst) { + + const struct ggml_tensor * src0 = dst->src[0]; + const struct ggml_tensor * src1 = dst->src[1]; + assert(params->ith == 0); if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) { @@ -11201,9 +11371,11 @@ static void ggml_compute_forward_get_rows_f16( static void ggml_compute_forward_get_rows_f32( const struct ggml_compute_params * params, - const struct ggml_tensor * src0, - const struct ggml_tensor * src1, struct ggml_tensor * dst) { + + const struct ggml_tensor * src0 = dst->src[0]; + const struct ggml_tensor * src1 = dst->src[1]; + assert(params->ith == 0); if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) { @@ -11236,9 +11408,10 @@ static void ggml_compute_forward_get_rows_f32( static void ggml_compute_forward_get_rows( const struct ggml_compute_params * params, - const struct ggml_tensor * src0, - const struct ggml_tensor * src1, struct ggml_tensor * dst) { + + const struct ggml_tensor * src0 = dst->src[0]; + switch (src0->type) { case GGML_TYPE_Q4_0: case GGML_TYPE_Q4_1: @@ -11257,16 +11430,16 @@ static void ggml_compute_forward_get_rows( case GGML_TYPE_IQ1_S: case GGML_TYPE_IQ4_NL: { - ggml_compute_forward_get_rows_q(params, src0, src1, dst); + ggml_compute_forward_get_rows_q(params, dst); } break; case GGML_TYPE_F16: { - ggml_compute_forward_get_rows_f16(params, src0, src1, dst); + ggml_compute_forward_get_rows_f16(params, dst); } break; case GGML_TYPE_F32: case GGML_TYPE_I32: { - ggml_compute_forward_get_rows_f32(params, src0, src1, dst); + ggml_compute_forward_get_rows_f32(params, dst); } break; default: { @@ -11297,9 +11470,11 @@ static void ggml_compute_forward_get_rows( static void ggml_compute_forward_get_rows_back_f32_f16( const struct ggml_compute_params * params, - const struct ggml_tensor * src0, - const struct ggml_tensor * src1, struct ggml_tensor * dst) { + + const struct ggml_tensor * src0 = dst->src[0]; + const struct ggml_tensor * src1 = dst->src[1]; + GGML_ASSERT(params->ith == 0); GGML_ASSERT(ggml_is_contiguous(dst)); @@ -11334,9 +11509,11 @@ static void ggml_compute_forward_get_rows_back_f32_f16( static void ggml_compute_forward_get_rows_back_f32( const struct ggml_compute_params * params, - const struct ggml_tensor * src0, - const struct ggml_tensor * src1, struct ggml_tensor * dst) { + + const struct ggml_tensor * src0 = dst->src[0]; + const struct ggml_tensor * src1 = dst->src[1]; + GGML_ASSERT(params->ith == 0); GGML_ASSERT(ggml_is_contiguous(dst)); @@ -11371,17 +11548,18 @@ static void ggml_compute_forward_get_rows_back_f32( static void ggml_compute_forward_get_rows_back( const struct ggml_compute_params * params, - const struct ggml_tensor * src0, - const struct ggml_tensor * src1, struct ggml_tensor * dst) { + + const struct ggml_tensor * src0 = dst->src[0]; + switch (src0->type) { case GGML_TYPE_F16: { - ggml_compute_forward_get_rows_back_f32_f16(params, src0, src1, dst); + ggml_compute_forward_get_rows_back_f32_f16(params, dst); } break; case GGML_TYPE_F32: { - ggml_compute_forward_get_rows_back_f32(params, src0, src1, dst); + ggml_compute_forward_get_rows_back_f32(params, dst); } break; default: { @@ -11412,8 +11590,10 @@ static void ggml_compute_forward_get_rows_back( static void ggml_compute_forward_diag_f32( const struct ggml_compute_params * params, - const struct ggml_tensor * src0, struct ggml_tensor * dst) { + + const struct ggml_tensor * src0 = dst->src[0]; + GGML_ASSERT(params->ith == 0); if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) { @@ -11452,12 +11632,14 @@ static void ggml_compute_forward_diag_f32( static void ggml_compute_forward_diag( const struct ggml_compute_params * params, - const struct ggml_tensor * src0, struct ggml_tensor * dst) { + + const struct ggml_tensor * src0 = dst->src[0]; + switch (src0->type) { case GGML_TYPE_F32: { - ggml_compute_forward_diag_f32(params, src0, dst); + ggml_compute_forward_diag_f32(params, dst); } break; default: { @@ -11470,10 +11652,11 @@ static void ggml_compute_forward_diag( static void ggml_compute_forward_diag_mask_f32( const struct ggml_compute_params * params, - const struct ggml_tensor * src0, struct ggml_tensor * dst, const float value) { + const struct ggml_tensor * src0 = dst->src[0]; + const int ith = params->ith; const int nth = params->nth; @@ -11523,12 +11706,14 @@ static void ggml_compute_forward_diag_mask_f32( static void ggml_compute_forward_diag_mask_inf( const struct ggml_compute_params * params, - const struct ggml_tensor * src0, struct ggml_tensor * dst) { + + const struct ggml_tensor * src0 = dst->src[0]; + switch (src0->type) { case GGML_TYPE_F32: { - ggml_compute_forward_diag_mask_f32(params, src0, dst, -INFINITY); + ggml_compute_forward_diag_mask_f32(params, dst, -INFINITY); } break; default: { @@ -11539,12 +11724,14 @@ static void ggml_compute_forward_diag_mask_inf( static void ggml_compute_forward_diag_mask_zero( const struct ggml_compute_params * params, - const struct ggml_tensor * src0, struct ggml_tensor * dst) { + + const struct ggml_tensor * src0 = dst->src[0]; + switch (src0->type) { case GGML_TYPE_F32: { - ggml_compute_forward_diag_mask_f32(params, src0, dst, 0); + ggml_compute_forward_diag_mask_f32(params, dst, 0); } break; default: { @@ -11557,10 +11744,12 @@ static void ggml_compute_forward_diag_mask_zero( static void ggml_compute_forward_soft_max_f32( const struct ggml_compute_params * params, - const struct ggml_tensor * src0, - const struct ggml_tensor * src1, - const struct ggml_tensor * src2, struct ggml_tensor * dst) { + + const struct ggml_tensor * src0 = dst->src[0]; + const struct ggml_tensor * src1 = dst->src[1]; + const struct ggml_tensor * src2 = dst->src[2]; + assert(ggml_is_contiguous(dst)); assert(ggml_are_same_shape(src0, dst)); @@ -11671,14 +11860,14 @@ static void ggml_compute_forward_soft_max_f32( static void ggml_compute_forward_soft_max( const struct ggml_compute_params * params, - const struct ggml_tensor * src0, - const struct ggml_tensor * src1, - const struct ggml_tensor * src2, struct ggml_tensor * dst) { + + const struct ggml_tensor * src0 = dst->src[0]; + switch (src0->type) { case GGML_TYPE_F32: { - ggml_compute_forward_soft_max_f32(params, src0, src1, src2, dst); + ggml_compute_forward_soft_max_f32(params, dst); } break; default: { @@ -11691,9 +11880,11 @@ static void ggml_compute_forward_soft_max( static void ggml_compute_forward_soft_max_back_f32( const struct ggml_compute_params * params, - const struct ggml_tensor * src0, - const struct ggml_tensor * src1, struct ggml_tensor * dst) { + + const struct ggml_tensor * src0 = dst->src[0]; + const struct ggml_tensor * src1 = dst->src[1]; + GGML_ASSERT(ggml_is_contiguous(src0)); GGML_ASSERT(ggml_is_contiguous(src1)); GGML_ASSERT(ggml_is_contiguous(dst)); @@ -11768,13 +11959,14 @@ static void ggml_compute_forward_soft_max_back_f32( static void ggml_compute_forward_soft_max_back( const struct ggml_compute_params * params, - const struct ggml_tensor * src0, - const struct ggml_tensor * src1, struct ggml_tensor * dst) { + + const struct ggml_tensor * src0 = dst->src[0]; + switch (src0->type) { case GGML_TYPE_F32: { - ggml_compute_forward_soft_max_back_f32(params, src0, src1, dst); + ggml_compute_forward_soft_max_back_f32(params, dst); } break; default: { @@ -11787,8 +11979,10 @@ static void ggml_compute_forward_soft_max_back( static void ggml_compute_forward_alibi_f32( const struct ggml_compute_params * params, - const struct ggml_tensor * src0, struct ggml_tensor * dst) { + + const struct ggml_tensor * src0 = dst->src[0]; + assert(params->ith == 0); if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) { @@ -11844,8 +12038,10 @@ static void ggml_compute_forward_alibi_f32( static void ggml_compute_forward_alibi_f16( const struct ggml_compute_params * params, - const struct ggml_tensor * src0, struct ggml_tensor * dst) { + + const struct ggml_tensor * src0 = dst->src[0]; + assert(params->ith == 0); if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) { @@ -11904,16 +12100,18 @@ static void ggml_compute_forward_alibi_f16( static void ggml_compute_forward_alibi( const struct ggml_compute_params * params, - const struct ggml_tensor * src0, struct ggml_tensor * dst) { + + const struct ggml_tensor * src0 = dst->src[0]; + switch (src0->type) { case GGML_TYPE_F16: { - ggml_compute_forward_alibi_f16(params, src0, dst); + ggml_compute_forward_alibi_f16(params, dst); } break; case GGML_TYPE_F32: { - ggml_compute_forward_alibi_f32(params, src0, dst); + ggml_compute_forward_alibi_f32(params, dst); } break; case GGML_TYPE_Q4_0: case GGML_TYPE_Q4_1: @@ -11946,8 +12144,10 @@ static void ggml_compute_forward_alibi( static void ggml_compute_forward_clamp_f32( const struct ggml_compute_params * params, - const struct ggml_tensor * src0, struct ggml_tensor * dst) { + + const struct ggml_tensor * src0 = dst->src[0]; + assert(params->ith == 0); if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) { @@ -11986,12 +12186,14 @@ static void ggml_compute_forward_clamp_f32( static void ggml_compute_forward_clamp( const struct ggml_compute_params * params, - const struct ggml_tensor * src0, struct ggml_tensor * dst) { + + const struct ggml_tensor * src0 = dst->src[0]; + switch (src0->type) { case GGML_TYPE_F32: { - ggml_compute_forward_clamp_f32(params, src0, dst); + ggml_compute_forward_clamp_f32(params, dst); } break; case GGML_TYPE_F16: case GGML_TYPE_Q4_0: @@ -12081,10 +12283,12 @@ GGML_CALL void ggml_rope_yarn_corr_dims( static void ggml_compute_forward_rope_f32( const struct ggml_compute_params * params, - const struct ggml_tensor * src0, - const struct ggml_tensor * src1, struct ggml_tensor * dst, const bool forward) { + + const struct ggml_tensor * src0 = dst->src[0]; + const struct ggml_tensor * src1 = dst->src[1]; + if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) { return; } @@ -12257,10 +12461,12 @@ static void ggml_compute_forward_rope_f32( static void ggml_compute_forward_rope_f16( const struct ggml_compute_params * params, - const struct ggml_tensor * src0, - const struct ggml_tensor * src1, struct ggml_tensor * dst, const bool forward) { + + const struct ggml_tensor * src0 = dst->src[0]; + const struct ggml_tensor * src1 = dst->src[1]; + if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) { return; } @@ -12422,17 +12628,18 @@ static void ggml_compute_forward_rope_f16( static void ggml_compute_forward_rope( const struct ggml_compute_params * params, - const struct ggml_tensor * src0, - const struct ggml_tensor * src1, struct ggml_tensor * dst) { + + const struct ggml_tensor * src0 = dst->src[0]; + switch (src0->type) { case GGML_TYPE_F16: { - ggml_compute_forward_rope_f16(params, src0, src1, dst, true); + ggml_compute_forward_rope_f16(params, dst, true); } break; case GGML_TYPE_F32: { - ggml_compute_forward_rope_f32(params, src0, src1, dst, true); + ggml_compute_forward_rope_f32(params, dst, true); } break; default: { @@ -12445,17 +12652,18 @@ static void ggml_compute_forward_rope( static void ggml_compute_forward_rope_back( const struct ggml_compute_params * params, - const struct ggml_tensor * src0, - const struct ggml_tensor * src1, struct ggml_tensor * dst) { + + const struct ggml_tensor * src0 = dst->src[0]; + switch (src0->type) { case GGML_TYPE_F16: { - ggml_compute_forward_rope_f16(params, src0, src1, dst, false); + ggml_compute_forward_rope_f16(params, dst, false); } break; case GGML_TYPE_F32: { - ggml_compute_forward_rope_f32(params, src0, src1, dst, false); + ggml_compute_forward_rope_f32(params, dst, false); } break; default: { @@ -12468,9 +12676,11 @@ static void ggml_compute_forward_rope_back( static void ggml_compute_forward_conv_transpose_1d_f16_f32( const struct ggml_compute_params * params, - const struct ggml_tensor * src0, - const struct ggml_tensor * src1, struct ggml_tensor * dst) { + + const struct ggml_tensor * src0 = dst->src[0]; + const struct ggml_tensor * src1 = dst->src[1]; + GGML_ASSERT(src0->type == GGML_TYPE_F16); GGML_ASSERT(src1->type == GGML_TYPE_F32); GGML_ASSERT( dst->type == GGML_TYPE_F32); @@ -12565,9 +12775,11 @@ static void ggml_compute_forward_conv_transpose_1d_f16_f32( static void ggml_compute_forward_conv_transpose_1d_f32( const struct ggml_compute_params * params, - const struct ggml_tensor * src0, - const struct ggml_tensor * src1, struct ggml_tensor * dst) { + + const struct ggml_tensor * src0 = dst->src[0]; + const struct ggml_tensor * src1 = dst->src[1]; + GGML_ASSERT(src0->type == GGML_TYPE_F32); GGML_ASSERT(src1->type == GGML_TYPE_F32); GGML_ASSERT( dst->type == GGML_TYPE_F32); @@ -12662,17 +12874,18 @@ static void ggml_compute_forward_conv_transpose_1d_f32( static void ggml_compute_forward_conv_transpose_1d( const struct ggml_compute_params * params, - const struct ggml_tensor * src0, - const struct ggml_tensor * src1, struct ggml_tensor * dst) { + + const struct ggml_tensor * src0 = dst->src[0]; + switch (src0->type) { case GGML_TYPE_F16: { - ggml_compute_forward_conv_transpose_1d_f16_f32(params, src0, src1, dst); + ggml_compute_forward_conv_transpose_1d_f16_f32(params, dst); } break; case GGML_TYPE_F32: { - ggml_compute_forward_conv_transpose_1d_f32(params, src0, src1, dst); + ggml_compute_forward_conv_transpose_1d_f32(params, dst); } break; default: { @@ -12686,9 +12899,11 @@ static void ggml_compute_forward_conv_transpose_1d( // dst: result [N, OH, OW, IC*KH*KW] static void ggml_compute_forward_im2col_f32( const struct ggml_compute_params * params, - const struct ggml_tensor * src0, - const struct ggml_tensor * src1, struct ggml_tensor * dst) { + + const struct ggml_tensor * src0 = dst->src[0]; + const struct ggml_tensor * src1 = dst->src[1]; + GGML_ASSERT(src0->type == GGML_TYPE_F16); GGML_ASSERT(src1->type == GGML_TYPE_F32); GGML_ASSERT( dst->type == GGML_TYPE_F32); @@ -12772,9 +12987,11 @@ static void ggml_compute_forward_im2col_f32( // dst: result [N, OH, OW, IC*KH*KW] static void ggml_compute_forward_im2col_f16( const struct ggml_compute_params * params, - const struct ggml_tensor * src0, - const struct ggml_tensor * src1, struct ggml_tensor * dst) { + + const struct ggml_tensor * src0 = dst->src[0]; + const struct ggml_tensor * src1 = dst->src[1]; + GGML_ASSERT(src0->type == GGML_TYPE_F16); GGML_ASSERT(src1->type == GGML_TYPE_F32); GGML_ASSERT( dst->type == GGML_TYPE_F16); @@ -12854,17 +13071,15 @@ static void ggml_compute_forward_im2col_f16( static void ggml_compute_forward_im2col( const struct ggml_compute_params * params, - const struct ggml_tensor * src0, - const struct ggml_tensor * src1, struct ggml_tensor * dst) { switch (dst->type) { case GGML_TYPE_F16: { - ggml_compute_forward_im2col_f16(params, src0, src1, dst); + ggml_compute_forward_im2col_f16(params, dst); } break; case GGML_TYPE_F32: { - ggml_compute_forward_im2col_f32(params, src0, src1, dst); + ggml_compute_forward_im2col_f32(params, dst); } break; default: { @@ -12878,9 +13093,11 @@ static void ggml_compute_forward_im2col( static void ggml_compute_forward_conv_transpose_2d( const struct ggml_compute_params * params, - const struct ggml_tensor * src0, - const struct ggml_tensor * src1, struct ggml_tensor * dst) { + + const struct ggml_tensor * src0 = dst->src[0]; + const struct ggml_tensor * src1 = dst->src[1]; + GGML_ASSERT(src0->type == GGML_TYPE_F16); GGML_ASSERT(src1->type == GGML_TYPE_F32); GGML_ASSERT( dst->type == GGML_TYPE_F32); @@ -12984,9 +13201,11 @@ static void ggml_compute_forward_conv_transpose_2d( static void ggml_compute_forward_pool_1d_sk_p0( const struct ggml_compute_params * params, const enum ggml_op_pool op, - const struct ggml_tensor * src, const int k, struct ggml_tensor * dst) { + + const struct ggml_tensor * src = dst->src[0]; + assert(src->type == GGML_TYPE_F32); assert(params->ith == 0); @@ -13035,7 +13254,6 @@ static void ggml_compute_forward_pool_1d_sk_p0( static void ggml_compute_forward_pool_1d( const struct ggml_compute_params * params, - const struct ggml_tensor * src0, struct ggml_tensor * dst) { const int32_t * opts = (const int32_t *)dst->op_params; @@ -13046,15 +13264,17 @@ static void ggml_compute_forward_pool_1d( GGML_ASSERT(p0 == 0); // padding not supported GGML_ASSERT(k0 == s0); // only s = k supported - ggml_compute_forward_pool_1d_sk_p0(params, op, src0, k0, dst); + ggml_compute_forward_pool_1d_sk_p0(params, op, k0, dst); } // ggml_compute_forward_pool_2d static void ggml_compute_forward_pool_2d( const struct ggml_compute_params * params, - const struct ggml_tensor * src, struct ggml_tensor * dst) { + + const struct ggml_tensor * src = dst->src[0]; + GGML_ASSERT(src->type == GGML_TYPE_F32); GGML_ASSERT(params->ith == 0); @@ -13127,9 +13347,10 @@ static void ggml_compute_forward_pool_2d( static void ggml_compute_forward_upscale_f32( const struct ggml_compute_params * params, - const struct ggml_tensor * src0, struct ggml_tensor * dst) { + const struct ggml_tensor * src0 = dst->src[0]; + if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) { return; } @@ -13166,12 +13387,14 @@ static void ggml_compute_forward_upscale_f32( static void ggml_compute_forward_upscale( const struct ggml_compute_params * params, - const struct ggml_tensor * src0, struct ggml_tensor * dst) { + + const struct ggml_tensor * src0 = dst->src[0]; + switch (src0->type) { case GGML_TYPE_F32: { - ggml_compute_forward_upscale_f32(params, src0, dst); + ggml_compute_forward_upscale_f32(params, dst); } break; default: { @@ -13184,9 +13407,10 @@ static void ggml_compute_forward_upscale( static void ggml_compute_forward_pad_f32( const struct ggml_compute_params * params, - const struct ggml_tensor * src0, struct ggml_tensor * dst) { + const struct ggml_tensor * src0 = dst->src[0]; + if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) { return; } @@ -13224,12 +13448,14 @@ static void ggml_compute_forward_pad_f32( static void ggml_compute_forward_pad( const struct ggml_compute_params * params, - const struct ggml_tensor * src0, struct ggml_tensor * dst) { + + const struct ggml_tensor * src0 = dst->src[0]; + switch (src0->type) { case GGML_TYPE_F32: { - ggml_compute_forward_pad_f32(params, src0, dst); + ggml_compute_forward_pad_f32(params, dst); } break; default: { @@ -13242,9 +13468,10 @@ static void ggml_compute_forward_pad( static void ggml_compute_forward_argsort_f32( const struct ggml_compute_params * params, - const struct ggml_tensor * src0, struct ggml_tensor * dst) { + const struct ggml_tensor * src0 = dst->src[0]; + if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) { return; } @@ -13284,13 +13511,14 @@ static void ggml_compute_forward_argsort_f32( static void ggml_compute_forward_argsort( const struct ggml_compute_params * params, - const struct ggml_tensor * src0, struct ggml_tensor * dst) { + const struct ggml_tensor * src0 = dst->src[0]; + switch (src0->type) { case GGML_TYPE_F32: { - ggml_compute_forward_argsort_f32(params, src0, dst); + ggml_compute_forward_argsort_f32(params, dst); } break; default: { @@ -13303,11 +13531,13 @@ static void ggml_compute_forward_argsort( static void ggml_compute_forward_flash_attn_f32( const struct ggml_compute_params * params, - const struct ggml_tensor * q, - const struct ggml_tensor * k, - const struct ggml_tensor * v, const bool masked, struct ggml_tensor * dst) { + + const struct ggml_tensor * q = dst->src[0]; + const struct ggml_tensor * k = dst->src[1]; + const struct ggml_tensor * v = dst->src[2]; + int64_t t0 = ggml_perf_time_us(); UNUSED(t0); @@ -13493,11 +13723,13 @@ static void ggml_compute_forward_flash_attn_f32( static void ggml_compute_forward_flash_attn_f16( const struct ggml_compute_params * params, - const struct ggml_tensor * q, - const struct ggml_tensor * k, - const struct ggml_tensor * v, const bool masked, struct ggml_tensor * dst) { + + const struct ggml_tensor * q = dst->src[0]; + const struct ggml_tensor * k = dst->src[1]; + const struct ggml_tensor * v = dst->src[2]; + int64_t t0 = ggml_perf_time_us(); UNUSED(t0); @@ -13719,19 +13951,19 @@ static void ggml_compute_forward_flash_attn_f16( static void ggml_compute_forward_flash_attn( const struct ggml_compute_params * params, - const struct ggml_tensor * q, - const struct ggml_tensor * k, - const struct ggml_tensor * v, const bool masked, struct ggml_tensor * dst) { + + const struct ggml_tensor * q = dst->src[0]; + switch (q->type) { case GGML_TYPE_F16: { - ggml_compute_forward_flash_attn_f16(params, q, k, v, masked, dst); + ggml_compute_forward_flash_attn_f16(params, masked, dst); } break; case GGML_TYPE_F32: { - ggml_compute_forward_flash_attn_f32(params, q, k, v, masked, dst); + ggml_compute_forward_flash_attn_f32(params, masked, dst); } break; default: { @@ -13744,12 +13976,14 @@ static void ggml_compute_forward_flash_attn( static void ggml_compute_forward_flash_ff_f16( const struct ggml_compute_params * params, - const struct ggml_tensor * a, // F16 - const struct ggml_tensor * b0, // F16 fc_w - const struct ggml_tensor * b1, // F32 fc_b - const struct ggml_tensor * c0, // F16 proj_w - const struct ggml_tensor * c1, // F32 proj_b struct ggml_tensor * dst) { + + const struct ggml_tensor * a = dst->src[0]; // F16 + const struct ggml_tensor * b0 = dst->src[1]; // F16 fc_w + const struct ggml_tensor * b1 = dst->src[2]; // F32 fc_b + const struct ggml_tensor * c0 = dst->src[3]; // F16 proj_w + const struct ggml_tensor * c1 = dst->src[4]; // F32 proj_b + int64_t t0 = ggml_perf_time_us(); UNUSED(t0); @@ -13877,16 +14111,14 @@ static void ggml_compute_forward_flash_ff_f16( static void ggml_compute_forward_flash_ff( const struct ggml_compute_params * params, - const struct ggml_tensor * a, - const struct ggml_tensor * b0, - const struct ggml_tensor * b1, - const struct ggml_tensor * c0, - const struct ggml_tensor * c1, struct ggml_tensor * dst) { + + const struct ggml_tensor * b0 = dst->src[1]; + switch (b0->type) { case GGML_TYPE_F16: { - ggml_compute_forward_flash_ff_f16(params, a, b0, b1, c0, c1, dst); + ggml_compute_forward_flash_ff_f16(params, dst); } break; case GGML_TYPE_F32: { @@ -13903,12 +14135,14 @@ static void ggml_compute_forward_flash_ff( static void ggml_compute_forward_flash_attn_back_f32( const struct ggml_compute_params * params, - const struct ggml_tensor * q, - const struct ggml_tensor * k, - const struct ggml_tensor * v, - const struct ggml_tensor * d, const bool masked, struct ggml_tensor * dst) { + + const struct ggml_tensor * q = dst->src[0]; + const struct ggml_tensor * k = dst->src[1]; + const struct ggml_tensor * v = dst->src[2]; + const struct ggml_tensor * d = dst->src[3]; + int64_t t0 = ggml_perf_time_us(); UNUSED(t0); @@ -14256,16 +14490,15 @@ static void ggml_compute_forward_flash_attn_back_f32( static void ggml_compute_forward_flash_attn_back( const struct ggml_compute_params * params, - const struct ggml_tensor * q, - const struct ggml_tensor * k, - const struct ggml_tensor * v, - const struct ggml_tensor * d, const bool masked, struct ggml_tensor * dst) { + + const struct ggml_tensor * q = dst->src[0]; + switch (q->type) { case GGML_TYPE_F32: { - ggml_compute_forward_flash_attn_back_f32(params, q, k, v, d, masked, dst); + ggml_compute_forward_flash_attn_back_f32(params, masked, dst); } break; default: { @@ -14278,8 +14511,10 @@ static void ggml_compute_forward_flash_attn_back( static void ggml_compute_forward_win_part_f32( const struct ggml_compute_params * params, - const struct ggml_tensor * src0, struct ggml_tensor * dst) { + + const struct ggml_tensor * src0 = dst->src[0]; + if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) { return; } @@ -14322,12 +14557,14 @@ static void ggml_compute_forward_win_part_f32( static void ggml_compute_forward_win_part( const struct ggml_compute_params * params, - const struct ggml_tensor * src0, struct ggml_tensor * dst) { + + const struct ggml_tensor * src0 = dst->src[0]; + switch (src0->type) { case GGML_TYPE_F32: { - ggml_compute_forward_win_part_f32(params, src0, dst); + ggml_compute_forward_win_part_f32(params, dst); } break; default: { @@ -14340,8 +14577,10 @@ static void ggml_compute_forward_win_part( static void ggml_compute_forward_win_unpart_f32( const struct ggml_compute_params * params, - const struct ggml_tensor * src0, struct ggml_tensor * dst) { + + const struct ggml_tensor * src0 = dst->src[0]; + if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) { return; } @@ -14382,12 +14621,14 @@ static void ggml_compute_forward_win_unpart_f32( static void ggml_compute_forward_win_unpart( const struct ggml_compute_params * params, - const struct ggml_tensor * src0, struct ggml_tensor * dst) { + + const struct ggml_tensor * src0 = dst->src[0]; + switch (src0->type) { case GGML_TYPE_F32: { - ggml_compute_forward_win_unpart_f32(params, src0, dst); + ggml_compute_forward_win_unpart_f32(params, dst); } break; default: { @@ -14400,58 +14641,58 @@ static void ggml_compute_forward_win_unpart( static void ggml_compute_forward_unary( const struct ggml_compute_params * params, - const struct ggml_tensor * src0, struct ggml_tensor * dst) { + const enum ggml_unary_op op = ggml_get_unary_op(dst); switch (op) { case GGML_UNARY_OP_ABS: { - ggml_compute_forward_abs(params, src0, dst); + ggml_compute_forward_abs(params, dst); } break; case GGML_UNARY_OP_SGN: { - ggml_compute_forward_sgn(params, src0, dst); + ggml_compute_forward_sgn(params, dst); } break; case GGML_UNARY_OP_NEG: { - ggml_compute_forward_neg(params, src0, dst); + ggml_compute_forward_neg(params, dst); } break; case GGML_UNARY_OP_STEP: { - ggml_compute_forward_step(params, src0, dst); + ggml_compute_forward_step(params, dst); } break; case GGML_UNARY_OP_TANH: { - ggml_compute_forward_tanh(params, src0, dst); + ggml_compute_forward_tanh(params, dst); } break; case GGML_UNARY_OP_ELU: { - ggml_compute_forward_elu(params, src0, dst); + ggml_compute_forward_elu(params, dst); } break; case GGML_UNARY_OP_RELU: { - ggml_compute_forward_relu(params, src0, dst); + ggml_compute_forward_relu(params, dst); } break; case GGML_UNARY_OP_GELU: { - ggml_compute_forward_gelu(params, src0, dst); + ggml_compute_forward_gelu(params, dst); } break; case GGML_UNARY_OP_GELU_QUICK: { - ggml_compute_forward_gelu_quick(params, src0, dst); + ggml_compute_forward_gelu_quick(params, dst); } break; case GGML_UNARY_OP_SILU: { - ggml_compute_forward_silu(params, src0, dst); + ggml_compute_forward_silu(params, dst); } break; case GGML_UNARY_OP_HARDSWISH: { - ggml_compute_forward_hardswish(params, src0, dst); + ggml_compute_forward_hardswish(params, dst); } break; case GGML_UNARY_OP_HARDSIGMOID: { - ggml_compute_forward_hardsigmoid(params, src0, dst); + ggml_compute_forward_hardsigmoid(params, dst); } break; default: { @@ -14464,8 +14705,10 @@ static void ggml_compute_forward_unary( static void ggml_compute_forward_get_rel_pos_f16( const struct ggml_compute_params * params, - const struct ggml_tensor * src0, struct ggml_tensor * dst) { + + const struct ggml_tensor * src0 = dst->src[0]; + if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) { return; } @@ -14491,12 +14734,14 @@ static void ggml_compute_forward_get_rel_pos_f16( static void ggml_compute_forward_get_rel_pos( const struct ggml_compute_params * params, - const struct ggml_tensor * src0, struct ggml_tensor * dst) { + + const struct ggml_tensor * src0 = dst->src[0]; + switch (src0->type) { case GGML_TYPE_F16: { - ggml_compute_forward_get_rel_pos_f16(params, src0, dst); + ggml_compute_forward_get_rel_pos_f16(params, dst); } break; default: { @@ -14509,11 +14754,12 @@ static void ggml_compute_forward_get_rel_pos( static void ggml_compute_forward_add_rel_pos_f32( const struct ggml_compute_params * params, - const struct ggml_tensor * src0, - const struct ggml_tensor * src1, - const struct ggml_tensor * src2, struct ggml_tensor * dst) { + const struct ggml_tensor * src0 = dst->src[0]; + const struct ggml_tensor * src1 = dst->src[1]; + const struct ggml_tensor * src2 = dst->src[2]; + const bool inplace = (bool) ((int32_t *) dst->op_params)[0]; if (!inplace && params->type == GGML_TASK_INIT) { if (params->ith != 0) { @@ -14577,14 +14823,14 @@ static void ggml_compute_forward_add_rel_pos_f32( static void ggml_compute_forward_add_rel_pos( const struct ggml_compute_params * params, - const struct ggml_tensor * src0, - const struct ggml_tensor * src1, - const struct ggml_tensor * src2, struct ggml_tensor * dst) { + + const struct ggml_tensor * src0 = dst->src[0]; + switch (src0->type) { case GGML_TYPE_F32: { - ggml_compute_forward_add_rel_pos_f32(params, src0, src1, src2, dst); + ggml_compute_forward_add_rel_pos_f32(params, dst); } break; default: { @@ -14597,9 +14843,11 @@ static void ggml_compute_forward_add_rel_pos( static void ggml_compute_forward_map_unary_f32( const struct ggml_compute_params * params, - const struct ggml_tensor * src0, struct ggml_tensor * dst, const ggml_unary_op_f32_t fun) { + + const struct ggml_tensor * src0 = dst->src[0]; + GGML_ASSERT(ggml_are_same_shape(src0, dst)); if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) { @@ -14621,13 +14869,15 @@ static void ggml_compute_forward_map_unary_f32( static void ggml_compute_forward_map_unary( const struct ggml_compute_params * params, - const struct ggml_tensor * src0, struct ggml_tensor * dst, const ggml_unary_op_f32_t fun) { + + const struct ggml_tensor * src0 = dst->src[0]; + switch (src0->type) { case GGML_TYPE_F32: { - ggml_compute_forward_map_unary_f32(params, src0, dst, fun); + ggml_compute_forward_map_unary_f32(params, dst, fun); } break; default: { @@ -14640,10 +14890,12 @@ static void ggml_compute_forward_map_unary( static void ggml_compute_forward_map_binary_f32( const struct ggml_compute_params * params, - const struct ggml_tensor * src0, - const struct ggml_tensor * src1, struct ggml_tensor * dst, const ggml_binary_op_f32_t fun) { + + const struct ggml_tensor * src0 = dst->src[0]; + const struct ggml_tensor * src1 = dst->src[1]; + assert(params->ith == 0); assert(ggml_are_same_shape(src0, src1) && ggml_are_same_shape(src0, dst)); @@ -14668,14 +14920,15 @@ static void ggml_compute_forward_map_binary_f32( static void ggml_compute_forward_map_binary( const struct ggml_compute_params * params, - const struct ggml_tensor * src0, - const struct ggml_tensor * src1, struct ggml_tensor * dst, const ggml_binary_op_f32_t fun) { + + const struct ggml_tensor * src0 = dst->src[0]; + switch (src0->type) { case GGML_TYPE_F32: { - ggml_compute_forward_map_binary_f32(params, src0, src1, dst, fun); + ggml_compute_forward_map_binary_f32(params, dst, fun); } break; default: { @@ -14688,9 +14941,11 @@ static void ggml_compute_forward_map_binary( static void ggml_compute_forward_map_custom1_f32( const struct ggml_compute_params * params, - const struct ggml_tensor * a, struct ggml_tensor * dst, const ggml_custom1_op_f32_t fun) { + + const struct ggml_tensor * a = dst->src[0]; + assert(params->ith == 0); if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) { @@ -14704,10 +14959,12 @@ static void ggml_compute_forward_map_custom1_f32( static void ggml_compute_forward_map_custom2_f32( const struct ggml_compute_params * params, - const struct ggml_tensor * a, - const struct ggml_tensor * b, struct ggml_tensor * dst, const ggml_custom2_op_f32_t fun) { + + const struct ggml_tensor * a = dst->src[0]; + const struct ggml_tensor * b = dst->src[1]; + assert(params->ith == 0); if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) { @@ -14721,11 +14978,13 @@ static void ggml_compute_forward_map_custom2_f32( static void ggml_compute_forward_map_custom3_f32( const struct ggml_compute_params * params, - const struct ggml_tensor * a, - const struct ggml_tensor * b, - const struct ggml_tensor * c, struct ggml_tensor * dst, const ggml_custom3_op_f32_t fun) { + + const struct ggml_tensor * a = dst->src[0]; + const struct ggml_tensor * b = dst->src[1]; + const struct ggml_tensor * c = dst->src[1]; + assert(params->ith == 0); if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) { @@ -14739,8 +14998,10 @@ static void ggml_compute_forward_map_custom3_f32( static void ggml_compute_forward_map_custom1( const struct ggml_compute_params * params, - const struct ggml_tensor * a, struct ggml_tensor * dst) { + + const struct ggml_tensor * a = dst->src[0]; + if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) { return; } @@ -14754,9 +15015,11 @@ static void ggml_compute_forward_map_custom1( static void ggml_compute_forward_map_custom2( const struct ggml_compute_params * params, - const struct ggml_tensor * a, - const struct ggml_tensor * b, struct ggml_tensor * dst) { + + const struct ggml_tensor * a = dst->src[0]; + const struct ggml_tensor * b = dst->src[1]; + if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) { return; } @@ -14770,10 +15033,12 @@ static void ggml_compute_forward_map_custom2( static void ggml_compute_forward_map_custom3( const struct ggml_compute_params * params, - const struct ggml_tensor * a, - const struct ggml_tensor * b, - const struct ggml_tensor * c, struct ggml_tensor * dst) { + + const struct ggml_tensor * a = dst->src[0]; + const struct ggml_tensor * b = dst->src[1]; + const struct ggml_tensor * c = dst->src[2]; + if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) { return; } @@ -14787,9 +15052,11 @@ static void ggml_compute_forward_map_custom3( static void ggml_compute_forward_cross_entropy_loss_f32( const struct ggml_compute_params * params, - const struct ggml_tensor * src0, - const struct ggml_tensor * src1, struct ggml_tensor * dst) { + + const struct ggml_tensor * src0 = dst->src[0]; + const struct ggml_tensor * src1 = dst->src[1]; + GGML_ASSERT(ggml_is_contiguous(src0)); GGML_ASSERT(ggml_is_contiguous(src1)); GGML_ASSERT(ggml_is_scalar(dst)); @@ -14893,13 +15160,14 @@ static void ggml_compute_forward_cross_entropy_loss_f32( static void ggml_compute_forward_cross_entropy_loss( const struct ggml_compute_params * params, - const struct ggml_tensor * src0, - const struct ggml_tensor * src1, struct ggml_tensor * dst) { + + const struct ggml_tensor * src0 = dst->src[0]; + switch (src0->type) { case GGML_TYPE_F32: { - ggml_compute_forward_cross_entropy_loss_f32(params, src0, src1, dst); + ggml_compute_forward_cross_entropy_loss_f32(params, dst); } break; default: { @@ -14912,10 +15180,12 @@ static void ggml_compute_forward_cross_entropy_loss( static void ggml_compute_forward_cross_entropy_loss_back_f32( const struct ggml_compute_params * params, - const struct ggml_tensor * src0, - const struct ggml_tensor * src1, - const struct ggml_tensor * opt0, struct ggml_tensor * dst) { + + const struct ggml_tensor * src0 = dst->src[0]; + const struct ggml_tensor * src1 = dst->src[1]; + const struct ggml_tensor * opt0 = dst->src[2]; + GGML_ASSERT(ggml_is_contiguous(dst)); GGML_ASSERT(ggml_is_contiguous(src0)); GGML_ASSERT(ggml_is_contiguous(src1)); @@ -15002,14 +15272,14 @@ static void ggml_compute_forward_cross_entropy_loss_back_f32( static void ggml_compute_forward_cross_entropy_loss_back( const struct ggml_compute_params * params, - const struct ggml_tensor * src0, - const struct ggml_tensor * src1, - const struct ggml_tensor * opt0, struct ggml_tensor * dst) { + + const struct ggml_tensor * src0 = dst->src[0]; + switch (src0->type) { case GGML_TYPE_F32: { - ggml_compute_forward_cross_entropy_loss_back_f32(params, src0, src1, opt0, dst); + ggml_compute_forward_cross_entropy_loss_back_f32(params, dst); } break; default: { @@ -15057,312 +15327,312 @@ static void ggml_compute_forward(struct ggml_compute_params * params, struct ggm switch (tensor->op) { case GGML_OP_DUP: { - ggml_compute_forward_dup(params, tensor->src[0], tensor); + ggml_compute_forward_dup(params, tensor); } break; case GGML_OP_ADD: { - ggml_compute_forward_add(params, tensor->src[0], tensor->src[1], tensor); + ggml_compute_forward_add(params, tensor); } break; case GGML_OP_ADD1: { - ggml_compute_forward_add1(params, tensor->src[0], tensor->src[1], tensor); + ggml_compute_forward_add1(params, tensor); } break; case GGML_OP_ACC: { - ggml_compute_forward_acc(params, tensor->src[0], tensor->src[1], tensor); + ggml_compute_forward_acc(params, tensor); } break; case GGML_OP_SUB: { - ggml_compute_forward_sub(params, tensor->src[0], tensor->src[1], tensor); + ggml_compute_forward_sub(params, tensor); } break; case GGML_OP_MUL: { - ggml_compute_forward_mul(params, tensor->src[0], tensor->src[1], tensor); + ggml_compute_forward_mul(params, tensor); } break; case GGML_OP_DIV: { - ggml_compute_forward_div(params, tensor->src[0], tensor->src[1], tensor); + ggml_compute_forward_div(params, tensor); } break; case GGML_OP_SQR: { - ggml_compute_forward_sqr(params, tensor->src[0], tensor); + ggml_compute_forward_sqr(params, tensor); } break; case GGML_OP_SQRT: { - ggml_compute_forward_sqrt(params, tensor->src[0], tensor); + ggml_compute_forward_sqrt(params, tensor); } break; case GGML_OP_LOG: { - ggml_compute_forward_log(params, tensor->src[0], tensor); + ggml_compute_forward_log(params, tensor); } break; case GGML_OP_SUM: { - ggml_compute_forward_sum(params, tensor->src[0], tensor); + ggml_compute_forward_sum(params, tensor); } break; case GGML_OP_SUM_ROWS: { - ggml_compute_forward_sum_rows(params, tensor->src[0], tensor); + ggml_compute_forward_sum_rows(params, tensor); } break; case GGML_OP_MEAN: { - ggml_compute_forward_mean(params, tensor->src[0], tensor); + ggml_compute_forward_mean(params, tensor); } break; case GGML_OP_ARGMAX: { - ggml_compute_forward_argmax(params, tensor->src[0], tensor); + ggml_compute_forward_argmax(params, tensor); } break; case GGML_OP_REPEAT: { - ggml_compute_forward_repeat(params, tensor->src[0], tensor); + ggml_compute_forward_repeat(params, tensor); } break; case GGML_OP_REPEAT_BACK: { - ggml_compute_forward_repeat_back(params, tensor->src[0], tensor); + ggml_compute_forward_repeat_back(params, tensor); } break; case GGML_OP_CONCAT: { - ggml_compute_forward_concat(params, tensor->src[0], tensor->src[1], tensor); + ggml_compute_forward_concat(params, tensor); } break; case GGML_OP_SILU_BACK: { - ggml_compute_forward_silu_back(params, tensor->src[0], tensor->src[1], tensor); + ggml_compute_forward_silu_back(params, tensor); } break; case GGML_OP_NORM: { - ggml_compute_forward_norm(params, tensor->src[0], tensor); + ggml_compute_forward_norm(params, tensor); } break; case GGML_OP_RMS_NORM: { - ggml_compute_forward_rms_norm(params, tensor->src[0], tensor); + ggml_compute_forward_rms_norm(params, tensor); } break; case GGML_OP_RMS_NORM_BACK: { - ggml_compute_forward_rms_norm_back(params, tensor->src[0], tensor->src[1], tensor); + ggml_compute_forward_rms_norm_back(params, tensor); } break; case GGML_OP_GROUP_NORM: { - ggml_compute_forward_group_norm(params, tensor->src[0], tensor); + ggml_compute_forward_group_norm(params, tensor); } break; case GGML_OP_MUL_MAT: { - ggml_compute_forward_mul_mat(params, tensor->src[0], tensor->src[1], tensor); + ggml_compute_forward_mul_mat(params, tensor); } break; case GGML_OP_MUL_MAT_ID: { - ggml_compute_forward_mul_mat_id(params, tensor->src[0], tensor->src[1], tensor); + ggml_compute_forward_mul_mat_id(params, tensor); } break; case GGML_OP_OUT_PROD: { - ggml_compute_forward_out_prod(params, tensor->src[0], tensor->src[1], tensor); + ggml_compute_forward_out_prod(params, tensor); } break; case GGML_OP_SCALE: { - ggml_compute_forward_scale(params, tensor->src[0], tensor); + ggml_compute_forward_scale(params, tensor); } break; case GGML_OP_SET: { - ggml_compute_forward_set(params, tensor->src[0], tensor->src[1], tensor); + ggml_compute_forward_set(params, tensor); } break; case GGML_OP_CPY: { - ggml_compute_forward_cpy(params, tensor->src[0], tensor); + ggml_compute_forward_cpy(params, tensor); } break; case GGML_OP_CONT: { - ggml_compute_forward_cont(params, tensor->src[0], tensor); + ggml_compute_forward_cont(params, tensor); } break; case GGML_OP_RESHAPE: { - ggml_compute_forward_reshape(params, tensor->src[0], tensor); + ggml_compute_forward_reshape(params, tensor); } break; case GGML_OP_VIEW: { - ggml_compute_forward_view(params, tensor->src[0]); + ggml_compute_forward_view(params, tensor); } break; case GGML_OP_PERMUTE: { - ggml_compute_forward_permute(params, tensor->src[0]); + ggml_compute_forward_permute(params, tensor); } break; case GGML_OP_TRANSPOSE: { - ggml_compute_forward_transpose(params, tensor->src[0]); + ggml_compute_forward_transpose(params, tensor); } break; case GGML_OP_GET_ROWS: { - ggml_compute_forward_get_rows(params, tensor->src[0], tensor->src[1], tensor); + ggml_compute_forward_get_rows(params, tensor); } break; case GGML_OP_GET_ROWS_BACK: { - ggml_compute_forward_get_rows_back(params, tensor->src[0], tensor->src[1], tensor); + ggml_compute_forward_get_rows_back(params, tensor); } break; case GGML_OP_DIAG: { - ggml_compute_forward_diag(params, tensor->src[0], tensor); + ggml_compute_forward_diag(params, tensor); } break; case GGML_OP_DIAG_MASK_INF: { - ggml_compute_forward_diag_mask_inf(params, tensor->src[0], tensor); + ggml_compute_forward_diag_mask_inf(params, tensor); } break; case GGML_OP_DIAG_MASK_ZERO: { - ggml_compute_forward_diag_mask_zero(params, tensor->src[0], tensor); + ggml_compute_forward_diag_mask_zero(params, tensor); } break; case GGML_OP_SOFT_MAX: { - ggml_compute_forward_soft_max(params, tensor->src[0], tensor->src[1], tensor->src[2], tensor); + ggml_compute_forward_soft_max(params, tensor); } break; case GGML_OP_SOFT_MAX_BACK: { - ggml_compute_forward_soft_max_back(params, tensor->src[0], tensor->src[1], tensor); + ggml_compute_forward_soft_max_back(params, tensor); } break; case GGML_OP_ROPE: { - ggml_compute_forward_rope(params, tensor->src[0], tensor->src[1], tensor); + ggml_compute_forward_rope(params, tensor); } break; case GGML_OP_ROPE_BACK: { - ggml_compute_forward_rope_back(params, tensor->src[0], tensor->src[1], tensor); + ggml_compute_forward_rope_back(params, tensor); } break; case GGML_OP_ALIBI: { - ggml_compute_forward_alibi(params, tensor->src[0], tensor); + ggml_compute_forward_alibi(params, tensor); } break; case GGML_OP_CLAMP: { - ggml_compute_forward_clamp(params, tensor->src[0], tensor); + ggml_compute_forward_clamp(params, tensor); } break; case GGML_OP_CONV_TRANSPOSE_1D: { - ggml_compute_forward_conv_transpose_1d(params, tensor->src[0], tensor->src[1], tensor); + ggml_compute_forward_conv_transpose_1d(params, tensor); } break; case GGML_OP_IM2COL: { - ggml_compute_forward_im2col(params, tensor->src[0], tensor->src[1], tensor); + ggml_compute_forward_im2col(params, tensor); } break; case GGML_OP_CONV_TRANSPOSE_2D: { - ggml_compute_forward_conv_transpose_2d(params, tensor->src[0], tensor->src[1], tensor); + ggml_compute_forward_conv_transpose_2d(params, tensor); } break; case GGML_OP_POOL_1D: { - ggml_compute_forward_pool_1d(params, tensor->src[0], tensor); + ggml_compute_forward_pool_1d(params, tensor); } break; case GGML_OP_POOL_2D: { - ggml_compute_forward_pool_2d(params, tensor->src[0], tensor); + ggml_compute_forward_pool_2d(params, tensor); } break; case GGML_OP_UPSCALE: { - ggml_compute_forward_upscale(params, tensor->src[0], tensor); + ggml_compute_forward_upscale(params, tensor); } break; case GGML_OP_PAD: { - ggml_compute_forward_pad(params, tensor->src[0], tensor); + ggml_compute_forward_pad(params, tensor); } break; case GGML_OP_ARGSORT: { - ggml_compute_forward_argsort(params, tensor->src[0], tensor); + ggml_compute_forward_argsort(params, tensor); } break; case GGML_OP_LEAKY_RELU: { - ggml_compute_forward_leaky_relu(params, tensor->src[0], tensor); + ggml_compute_forward_leaky_relu(params, tensor); } break; case GGML_OP_FLASH_ATTN: { const int32_t t = ggml_get_op_params_i32(tensor, 0); GGML_ASSERT(t == 0 || t == 1); const bool masked = t != 0; - ggml_compute_forward_flash_attn(params, tensor->src[0], tensor->src[1], tensor->src[2], masked, tensor); + ggml_compute_forward_flash_attn(params, masked, tensor); } break; case GGML_OP_FLASH_FF: { - ggml_compute_forward_flash_ff(params, tensor->src[0], tensor->src[1], tensor->src[2], tensor->src[3], tensor->src[4], tensor); + ggml_compute_forward_flash_ff(params, tensor); } break; case GGML_OP_FLASH_ATTN_BACK: { int32_t t = ggml_get_op_params_i32(tensor, 0); GGML_ASSERT(t == 0 || t == 1); bool masked = t != 0; - ggml_compute_forward_flash_attn_back(params, tensor->src[0], tensor->src[1], tensor->src[2], tensor->src[3], masked, tensor); + ggml_compute_forward_flash_attn_back(params, masked, tensor); } break; case GGML_OP_WIN_PART: { - ggml_compute_forward_win_part(params, tensor->src[0], tensor); + ggml_compute_forward_win_part(params, tensor); } break; case GGML_OP_WIN_UNPART: { - ggml_compute_forward_win_unpart(params, tensor->src[0], tensor); + ggml_compute_forward_win_unpart(params, tensor); } break; case GGML_OP_UNARY: { - ggml_compute_forward_unary(params, tensor->src[0], tensor); + ggml_compute_forward_unary(params, tensor); } break; case GGML_OP_GET_REL_POS: { - ggml_compute_forward_get_rel_pos(params, tensor->src[0], tensor); + ggml_compute_forward_get_rel_pos(params, tensor); } break; case GGML_OP_ADD_REL_POS: { - ggml_compute_forward_add_rel_pos(params, tensor->src[0], tensor->src[1], tensor->src[2], tensor); + ggml_compute_forward_add_rel_pos(params, tensor); } break; case GGML_OP_MAP_UNARY: { ggml_unary_op_f32_t fun; memcpy(&fun, tensor->op_params, sizeof(fun)); - ggml_compute_forward_map_unary(params, tensor->src[0], tensor, fun); + ggml_compute_forward_map_unary(params, tensor, fun); } break; case GGML_OP_MAP_BINARY: { ggml_binary_op_f32_t fun; memcpy(&fun, tensor->op_params, sizeof(fun)); - ggml_compute_forward_map_binary(params, tensor->src[0], tensor->src[1], tensor, fun); + ggml_compute_forward_map_binary(params, tensor, fun); } break; case GGML_OP_MAP_CUSTOM1_F32: { ggml_custom1_op_f32_t fun; memcpy(&fun, tensor->op_params, sizeof(fun)); - ggml_compute_forward_map_custom1_f32(params, tensor->src[0], tensor, fun); + ggml_compute_forward_map_custom1_f32(params, tensor, fun); } break; case GGML_OP_MAP_CUSTOM2_F32: { ggml_custom2_op_f32_t fun; memcpy(&fun, tensor->op_params, sizeof(fun)); - ggml_compute_forward_map_custom2_f32(params, tensor->src[0], tensor->src[1], tensor, fun); + ggml_compute_forward_map_custom2_f32(params, tensor, fun); } break; case GGML_OP_MAP_CUSTOM3_F32: { ggml_custom3_op_f32_t fun; memcpy(&fun, tensor->op_params, sizeof(fun)); - ggml_compute_forward_map_custom3_f32(params, tensor->src[0], tensor->src[1], tensor->src[2], tensor, fun); + ggml_compute_forward_map_custom3_f32(params, tensor, fun); } break; case GGML_OP_MAP_CUSTOM1: { - ggml_compute_forward_map_custom1(params, tensor->src[0], tensor); + ggml_compute_forward_map_custom1(params, tensor); } break; case GGML_OP_MAP_CUSTOM2: { - ggml_compute_forward_map_custom2(params, tensor->src[0], tensor->src[1], tensor); + ggml_compute_forward_map_custom2(params, tensor); } break; case GGML_OP_MAP_CUSTOM3: { - ggml_compute_forward_map_custom3(params, tensor->src[0], tensor->src[1], tensor->src[2], tensor); + ggml_compute_forward_map_custom3(params, tensor); } break; case GGML_OP_CROSS_ENTROPY_LOSS: { - ggml_compute_forward_cross_entropy_loss(params, tensor->src[0], tensor->src[1], tensor); + ggml_compute_forward_cross_entropy_loss(params, tensor); } break; case GGML_OP_CROSS_ENTROPY_LOSS_BACK: { - ggml_compute_forward_cross_entropy_loss_back(params, tensor->src[0], tensor->src[1], tensor->src[2], tensor); + ggml_compute_forward_cross_entropy_loss_back(params, tensor); } break; case GGML_OP_NONE: diff --git a/scripts/sync-ggml.last b/scripts/sync-ggml.last index 733d8f95b..97f34ac85 100644 --- a/scripts/sync-ggml.last +++ b/scripts/sync-ggml.last @@ -1 +1 @@ -818eeb8a3be99125746a90ec63af8f51516a2ec6 +4712fd12d7acb9971f850b1b98588f934cb39444 From a00a35cef93e057eace8351a667d14d152a91ebc Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Wed, 21 Feb 2024 15:39:10 +0100 Subject: [PATCH 423/812] readme : add LocalAI to the availables UI (#5629) --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index ce5dec7ca..c1624b9f9 100644 --- a/README.md +++ b/README.md @@ -141,6 +141,7 @@ Unless otherwise noted these projects are open-source with permissive licensing: - [nat/openplayground](https://github.com/nat/openplayground) - [Faraday](https://faraday.dev/) (proprietary) - [LMStudio](https://lmstudio.ai/) (proprietary) +- [LocalAI](https://github.com/mudler/LocalAI) (MIT) - [LostRuins/koboldcpp](https://github.com/LostRuins/koboldcpp) (AGPL) - [Mozilla-Ocho/llamafile](https://github.com/Mozilla-Ocho/llamafile) - [nomic-ai/gpt4all](https://github.com/nomic-ai/gpt4all) From 1ecea255ebb70750b52688393f37a63606b90e3f Mon Sep 17 00:00:00 2001 From: Pierrick Hymbert Date: Wed, 21 Feb 2024 15:47:48 +0100 Subject: [PATCH 424/812] server: health: fix race condition on slots data using tasks queue (#5634) * server: health: fix race condition on slots data using tasks queue * server: health: * include_slots only if slots_endpoint * fix compile warning task.target_id not initialized. --- examples/server/README.md | 2 + examples/server/server.cpp | 122 ++++++++++++++++++++++++------------- examples/server/utils.hpp | 3 +- 3 files changed, 84 insertions(+), 43 deletions(-) diff --git a/examples/server/README.md b/examples/server/README.md index f6b9c7402..6d9f96cd4 100644 --- a/examples/server/README.md +++ b/examples/server/README.md @@ -140,6 +140,8 @@ node index.js - 200 -> `{"status": "no slot available", "slots_idle": 0, "slots_processing": 32}` if no slot are currently available. - 503 -> `{"status": "no slot available", "slots_idle": 0, "slots_processing": 32}` if the query parameter `fail_on_no_slot` is provided and no slot are currently available. + If the query parameter `include_slots` is passed, `slots` field will contain internal slots data except if `--slots-endpoint-disable` is set. + - **POST** `/completion`: Given a `prompt`, it returns the predicted completion. *Options:* diff --git a/examples/server/server.cpp b/examples/server/server.cpp index eb01729fa..1c4479512 100644 --- a/examples/server/server.cpp +++ b/examples/server/server.cpp @@ -1394,6 +1394,46 @@ struct llama_server_context case TASK_TYPE_NEXT_RESPONSE: { // do nothing } break; + case TASK_TYPE_SLOTS_DATA: { + json slots_data = json::array(); + int n_idle_slots = 0; + int n_processing_slots = 0; + + for (llama_client_slot &slot: slots) { + if (slot.available()) { + n_idle_slots++; + } else { + n_processing_slots++; + } + json slot_data = get_formated_generation(slot); + slot_data["id"] = slot.id; + slot_data["task_id"] = slot.task_id; + slot_data["state"] = slot.state; + slot_data["prompt"] = slot.prompt; + slot_data["next_token"] = { + {"has_next_token", slot.has_next_token}, + {"n_remain", slot.n_remaining}, + {"num_tokens_predicted", slot.n_decoded}, + {"stopped_eos", slot.stopped_eos}, + {"stopped_word", slot.stopped_word}, + {"stopped_limit", slot.stopped_limit}, + {"stopping_word", slot.stopping_word}, + }; + slots_data.push_back(slot_data); + } + LOG_TEE("task %i - slots data: idle=%i processing=%i\n", task.id, n_idle_slots, n_processing_slots); + task_result res; + res.id = task.id; + res.multitask_id = task.multitask_id; + res.stop = true; + res.error = false; + res.result_json = { + { "idle", n_idle_slots }, + { "processing", n_processing_slots }, + { "slots", slots_data } + }; + queue_results.send(res); + } break; } } @@ -2557,34 +2597,38 @@ int main(int argc, char **argv) server_state current_state = state.load(); switch(current_state) { case SERVER_STATE_READY: { - int available_slots = 0; - int processing_slots = 0; - for (llama_client_slot &slot: llama.slots) { - if (slot.available()) { - available_slots++; - } else { - processing_slots++; - } + // request slots data using task queue + task_server task; + task.id = llama.queue_tasks.get_new_id(); + task.type = TASK_TYPE_SLOTS_DATA; + task.target_id = -1; + + llama.queue_results.add_waiting_task_id(task.id); + llama.queue_tasks.post(task); + + // get the result + task_result result = llama.queue_results.recv(task.id); + llama.queue_results.remove_waiting_task_id(task.id); + + int n_idle_slots = result.result_json["idle"]; + int n_processing_slots = result.result_json["processing"]; + + json health = { + {"status", "ok"}, + {"slots_idle", n_idle_slots}, + {"slots_processing", n_processing_slots}}; + res.status = 200; // HTTP OK + if (sparams.slots_endpoint && req.has_param("include_slots")) { + health["slots"] = result.result_json["slots"]; } - if (available_slots > 0) { - json health = { - {"status", "ok"}, - {"slots_idle", available_slots}, - {"slots_processing", processing_slots}}; - res.set_content(health.dump(), "application/json"); - res.status = 200; // HTTP OK - } else { - json health = { - {"status", "no slot available"}, - {"slots_idle", available_slots}, - {"slots_processing", processing_slots}}; - res.set_content(health.dump(), "application/json"); + + if (n_idle_slots == 0) { + health["status"] = "no slot available"; if (req.has_param("fail_on_no_slot")) { res.status = 503; // HTTP Service Unavailable - } else { - res.status = 200; // HTTP OK } } + res.set_content(health.dump(), "application/json"); break; } case SERVER_STATE_LOADING_MODEL: @@ -2600,26 +2644,20 @@ int main(int argc, char **argv) if (sparams.slots_endpoint) { svr.Get("/slots", [&](const httplib::Request&, httplib::Response& res) { - json slots; - for (llama_client_slot & slot : llama.slots) { - json slot_data = llama.get_formated_generation(slot); - slot_data["id"] = slot.id; - slot_data["task_id"] = slot.task_id; - slot_data["state"] = slot.state; - slot_data["prompt"] = slot.prompt; - slot_data["next_token"] = { - {"has_next_token", slot.has_next_token}, - {"n_remain", slot.n_remaining}, - {"num_tokens_predicted", slot.n_decoded}, - {"stopped_eos", slot.stopped_eos}, - {"stopped_word", slot.stopped_word}, - {"stopped_limit", slot.stopped_limit}, - {"stopping_word", slot.stopping_word}, - }; + // request slots data using task queue + task_server task; + task.id = llama.queue_tasks.get_new_id(); + task.type = TASK_TYPE_SLOTS_DATA; + task.target_id = -1; - slots.push_back(slot_data); - } - res.set_content(slots.dump(), "application/json"); + llama.queue_results.add_waiting_task_id(task.id); + llama.queue_tasks.post(task); + + // get the result + task_result result = llama.queue_results.recv(task.id); + llama.queue_results.remove_waiting_task_id(task.id); + + res.set_content(result.result_json["slots"].dump(), "application/json"); res.status = 200; // HTTP OK }); } diff --git a/examples/server/utils.hpp b/examples/server/utils.hpp index e954fb0ef..88545eb69 100644 --- a/examples/server/utils.hpp +++ b/examples/server/utils.hpp @@ -49,7 +49,8 @@ enum server_state { enum task_type { TASK_TYPE_COMPLETION, TASK_TYPE_CANCEL, - TASK_TYPE_NEXT_RESPONSE + TASK_TYPE_NEXT_RESPONSE, + TASK_TYPE_SLOTS_DATA }; struct task_server { From 5022cf242d689e15defd133f96c4345ad30c5d19 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Wed, 21 Feb 2024 16:52:39 +0200 Subject: [PATCH 425/812] sync : ggml --- scripts/sync-ggml.last | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/sync-ggml.last b/scripts/sync-ggml.last index 97f34ac85..bbbf88d9d 100644 --- a/scripts/sync-ggml.last +++ b/scripts/sync-ggml.last @@ -1 +1 @@ -4712fd12d7acb9971f850b1b98588f934cb39444 +30805514e1bf389a59d30a54a0525cbdc30d5bd1 From 89febfed9322c8849520dc63c93ee4f5fd72556e Mon Sep 17 00:00:00 2001 From: Jared Van Bortel Date: Wed, 21 Feb 2024 10:33:54 -0500 Subject: [PATCH 426/812] examples : do not assume BOS when shifting context (#5622) --- examples/main/main.cpp | 12 +++++++----- examples/server/server.cpp | 13 +++++++------ 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/examples/main/main.cpp b/examples/main/main.cpp index f5d2f4893..7555dffe4 100644 --- a/examples/main/main.cpp +++ b/examples/main/main.cpp @@ -334,6 +334,8 @@ int main(int argc, char ** argv) { // number of tokens to keep when resetting context if (params.n_keep < 0 || params.n_keep > (int) embd_inp.size() || params.instruct || params.chatml) { params.n_keep = (int)embd_inp.size(); + } else { + params.n_keep += add_bos; // always keep the BOS token } // prefix & suffix for instruct mode @@ -383,8 +385,8 @@ int main(int argc, char ** argv) { } } - if (params.n_keep > 0) { - LOG_TEE("%s: static prompt based on n_keep: '", __func__); + if (params.n_keep > add_bos) { + LOG_TEE("%s: static prompt based on n_keep: '", __func__); for (int i = 0; i < params.n_keep; i++) { LOG_TEE("%s", llama_token_to_piece(ctx, embd_inp[i]).c_str()); } @@ -540,14 +542,14 @@ int main(int argc, char ** argv) { break; } - const int n_left = n_past - params.n_keep - 1; + const int n_left = n_past - params.n_keep; const int n_discard = n_left/2; LOG("context full, swapping: n_past = %d, n_left = %d, n_ctx = %d, n_keep = %d, n_discard = %d\n", n_past, n_left, n_ctx, params.n_keep, n_discard); - llama_kv_cache_seq_rm (ctx, 0, params.n_keep + 1 , params.n_keep + n_discard + 1); - llama_kv_cache_seq_shift(ctx, 0, params.n_keep + 1 + n_discard, n_past, -n_discard); + llama_kv_cache_seq_rm (ctx, 0, params.n_keep , params.n_keep + n_discard); + llama_kv_cache_seq_shift(ctx, 0, params.n_keep + n_discard, n_past, -n_discard); n_past -= n_discard; diff --git a/examples/server/server.cpp b/examples/server/server.cpp index 1c4479512..c84719a0d 100644 --- a/examples/server/server.cpp +++ b/examples/server/server.cpp @@ -1487,14 +1487,15 @@ struct llama_server_context if (slot.is_processing() && system_tokens.size() + slot.cache_tokens.size() >= (size_t) slot.n_ctx) { // Shift context - const int n_left = system_tokens.size() + slot.n_past - slot.params.n_keep - 1; + const int n_keep = slot.params.n_keep + add_bos_token; + const int n_left = system_tokens.size() + slot.n_past - n_keep; const int n_discard = n_left / 2; - LOG_TEE("slot %d: context shift - n_keep = %d, n_left = %d, n_discard = %d\n", slot.id, slot.params.n_keep, n_left, n_discard); - llama_kv_cache_seq_rm (ctx, slot.id, slot.params.n_keep + 1 , slot.params.n_keep + n_discard + 1); - llama_kv_cache_seq_shift(ctx, slot.id, slot.params.n_keep + 1 + n_discard, system_tokens.size() + slot.n_past, -n_discard); + LOG_TEE("slot %d: context shift - n_keep = %d, n_left = %d, n_discard = %d\n", slot.id, n_keep, n_left, n_discard); + llama_kv_cache_seq_rm (ctx, slot.id, n_keep , n_keep + n_discard); + llama_kv_cache_seq_shift(ctx, slot.id, n_keep + n_discard, system_tokens.size() + slot.n_past, -n_discard); - for (size_t i = slot.params.n_keep + 1 + n_discard; i < slot.cache_tokens.size(); i++) + for (size_t i = n_keep + n_discard; i < slot.cache_tokens.size(); i++) { slot.cache_tokens[i - n_discard] = slot.cache_tokens[i]; } @@ -1507,7 +1508,7 @@ struct llama_server_context LOG_VERBOSE("context shift", { { "n_ctx", n_ctx }, - { "n_keep", params.n_keep }, + { "n_keep", n_keep }, { "n_left", n_left }, }); } From ba2135ccae7462470b3865c6e41d2e1d734eac05 Mon Sep 17 00:00:00 2001 From: slaren Date: Wed, 21 Feb 2024 22:18:23 +0100 Subject: [PATCH 427/812] gemma : allow offloading the output tensor (#5646) --- llama.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/llama.cpp b/llama.cpp index 3a226c426..4054d5da6 100644 --- a/llama.cpp +++ b/llama.cpp @@ -4394,6 +4394,8 @@ static bool llm_load_tensors( // output model.output_norm = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}); + model.output = ml.create_tensor(ctx_output, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}); // same as tok_embd, duplicated to allow offloading + ml.n_created--; // artificial tensor const int64_t n_ff = hparams.n_ff; const int64_t n_embd_head_k = hparams.n_embd_head_k; @@ -7525,7 +7527,7 @@ struct llm_build_context { cb(cur, "result_norm", -1); // lm_head - cur = ggml_mul_mat(ctx0, model.tok_embd, cur); + cur = ggml_mul_mat(ctx0, model.output, cur); cb(cur, "result_output", -1); ggml_build_forward_expand(gf, cur); From 7fe4678b0244ba7b03eae66ebeaa947e2770bb1a Mon Sep 17 00:00:00 2001 From: slaren Date: Wed, 21 Feb 2024 22:52:39 +0100 Subject: [PATCH 428/812] llama : fix session save/load with quantized KV (#5649) --- llama.cpp | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/llama.cpp b/llama.cpp index 4054d5da6..d763cc80c 100644 --- a/llama.cpp +++ b/llama.cpp @@ -12176,18 +12176,19 @@ static void llama_copy_state_data_internal(struct llama_context * ctx, llama_dat data_ctx->write(&kv_used, sizeof(kv_used)); if (kv_buf_size) { - const size_t elt_size = ggml_element_size(kv_self.k_l[0]); - std::vector tmp_buf; for (int il = 0; il < (int) n_layer; ++il) { - tmp_buf.resize(elt_size*n_embd_k_gqa*kv_head); + size_t k_size = ggml_row_size(kv_self.k_l[il]->type, n_embd_k_gqa*kv_head); + tmp_buf.resize(k_size); ggml_backend_tensor_get(kv_self.k_l[il], tmp_buf.data(), 0, tmp_buf.size()); data_ctx->write(tmp_buf.data(), tmp_buf.size()); // v is not contiguous, copy row by row - tmp_buf.resize(elt_size*kv_head); + size_t v_row_size = ggml_row_size(kv_self.v_l[il]->type, kv_head); + size_t v_row_stride = ggml_row_size(kv_self.v_l[il]->type, n_ctx); + tmp_buf.resize(v_row_size); for (int ir = 0; ir < (int) n_embd_v_gqa; ++ir) { - ggml_backend_tensor_get(kv_self.v_l[il], tmp_buf.data(), ir*elt_size*n_ctx, tmp_buf.size()); + ggml_backend_tensor_get(kv_self.v_l[il], tmp_buf.data(), ir*v_row_stride, tmp_buf.size()); data_ctx->write(tmp_buf.data(), tmp_buf.size()); } } @@ -12289,17 +12290,16 @@ size_t llama_set_state_data(struct llama_context * ctx, uint8_t * src) { if (kv_buf_size) { GGML_ASSERT(kv_self.total_size() == kv_buf_size); - const size_t elt_size = ggml_element_size(kv_self.k_l[0]); - for (int il = 0; il < (int) n_layer; ++il) { - size_t k_size = elt_size*n_embd_k_gqa*kv_head; + size_t k_size = ggml_row_size(kv_self.k_l[il]->type, n_embd_k_gqa*kv_head); ggml_backend_tensor_set(kv_self.k_l[il], inp, 0, k_size); inp += k_size; // v is not contiguous, copy row by row - size_t v_row_size = elt_size*kv_head; + size_t v_row_size = ggml_row_size(kv_self.v_l[il]->type, kv_head); + size_t v_row_stride = ggml_row_size(kv_self.v_l[il]->type, n_ctx); for (int ir = 0; ir < (int) n_embd_v_gqa; ++ir) { - ggml_backend_tensor_set(kv_self.v_l[il], inp, ir*elt_size*n_ctx, v_row_size); + ggml_backend_tensor_set(kv_self.v_l[il], inp, ir*v_row_stride, v_row_size); inp += v_row_size; } } From 7c8bcc11dc61cf5930b70cd0168b84afcebe12a9 Mon Sep 17 00:00:00 2001 From: Xuan Son Nguyen Date: Thu, 22 Feb 2024 00:31:00 +0100 Subject: [PATCH 429/812] Add docs for llama_chat_apply_template (#5645) * add docs for llama_chat_apply_template * fix typo --- examples/server/README.md | 1 + llama.h | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/examples/server/README.md b/examples/server/README.md index 6d9f96cd4..4b24ee5dc 100644 --- a/examples/server/README.md +++ b/examples/server/README.md @@ -41,6 +41,7 @@ see https://github.com/ggerganov/llama.cpp/issues/1437 - `--grp-attn-w`: Set the group attention width to extend context size through self-extend(default: 512), used together with group attention factor `--grp-attn-n` - `-n, --n-predict`: Set the maximum tokens to predict (default: -1) - `--slots-endpoint-disable`: To disable slots state monitoring endpoint. Slots state may contain user data, prompts included. +- `--chat-template JINJA_TEMPLATE`: Set custom jinja chat template. This parameter accepts a string, not a file name (default: template taken from model's metadata). We only support [some pre-defined templates](https://github.com/ggerganov/llama.cpp/wiki/Templates-supported-by-llama_chat_apply_template) ## Build diff --git a/llama.h b/llama.h index 8ba20696f..84f196b3b 100644 --- a/llama.h +++ b/llama.h @@ -708,7 +708,7 @@ extern "C" { /// Apply chat template. Inspired by hf apply_chat_template() on python. /// Both "model" and "custom_template" are optional, but at least one is required. "custom_template" has higher precedence than "model" - /// NOTE: This function only support some known jinja templates. It is not a jinja parser. + /// NOTE: This function does not use a jinja parser. It only support a pre-defined list of template. See more: https://github.com/ggerganov/llama.cpp/wiki/Templates-supported-by-llama_chat_apply_template /// @param tmpl A Jinja template to use for this chat. If this is nullptr, the model’s default chat template will be used instead. /// @param chat Pointer to a list of multiple llama_chat_message /// @param n_msg Number of llama_chat_message in this chat From 973053d8b0d04809836b3339a50f68d9c842de90 Mon Sep 17 00:00:00 2001 From: slaren Date: Thu, 22 Feb 2024 00:42:09 +0100 Subject: [PATCH 430/812] llama : fix loading models with shared tok_embd and output (#5651) ggml-ci --- llama.cpp | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/llama.cpp b/llama.cpp index d763cc80c..259f2a3a3 100644 --- a/llama.cpp +++ b/llama.cpp @@ -2791,13 +2791,7 @@ struct llama_model_loader { std::vector> read_buf; - for (int i = 0; i < gguf_get_n_tensors(ctx_gguf); i++) { - struct ggml_tensor * cur = ggml_get_tensor(ctx, gguf_get_tensor_name(ctx_gguf, i)); - if (!cur) { - // some tensors may be allocated in a different context - continue; - } - + for (struct ggml_tensor * cur = ggml_get_first_tensor(ctx); cur != NULL; cur = ggml_get_next_tensor(ctx, cur)) { if (progress_callback) { if (!progress_callback((float) size_done / size_data, progress_callback_user_data)) { return false; @@ -3722,7 +3716,7 @@ static bool llm_load_tensors( } // create one context per buffer type - size_t ctx_size = ggml_tensor_overhead()*ml.n_tensors; + size_t ctx_size = ggml_tensor_overhead()*(ml.n_tensors + 1); // +1 for models where tok_embd is duplicated as output std::map ctx_map; for (auto & it : buft_layer_count) { struct ggml_init_params params = { @@ -3860,6 +3854,7 @@ static bool llm_load_tensors( } else { model.output = ml.create_tensor(ctx_output_split, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}); // needs to be on GPU ml.n_created--; // artificial tensor + ml.size_data += ggml_nbytes(model.output); } } @@ -4396,6 +4391,7 @@ static bool llm_load_tensors( model.output_norm = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}); model.output = ml.create_tensor(ctx_output, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}); // same as tok_embd, duplicated to allow offloading ml.n_created--; // artificial tensor + ml.size_data += ggml_nbytes(model.output); const int64_t n_ff = hparams.n_ff; const int64_t n_embd_head_k = hparams.n_embd_head_k; From 4ef245a92a968ba0f18a5adfd41e51980ce4fdf5 Mon Sep 17 00:00:00 2001 From: Dat Quoc Nguyen <2412555+datquocnguyen@users.noreply.github.com> Date: Thu, 22 Feb 2024 18:15:13 +1000 Subject: [PATCH 431/812] mpt : add optional bias tensors (#5638) Update for MPT with optional bias parameters: to work with PhoGPT and SEA-LION models that were pre-trained with 'bias'. --- llama.cpp | 38 +++++++++++++++++++++++++++----------- 1 file changed, 27 insertions(+), 11 deletions(-) diff --git a/llama.cpp b/llama.cpp index 259f2a3a3..9cae8c761 100644 --- a/llama.cpp +++ b/llama.cpp @@ -4054,6 +4054,8 @@ static bool llm_load_tensors( // output { model.output_norm = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}); + model.output_norm_b = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "bias"), {n_embd}, false); + model.output = ml.create_tensor(ctx_output_split, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}); } @@ -4063,14 +4065,23 @@ static bool llm_load_tensors( auto & layer = model.layers[i]; - layer.attn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}); + layer.attn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}); + layer.attn_norm_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM, "bias", i), {n_embd}, false); layer.wqkv = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_QKV, "weight", i), {n_embd, n_embd + 2*n_embd_gqa}); - layer.wo = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd}); + layer.bqkv = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_QKV, "bias", i), {n_embd + 2*n_embd_gqa}, false); - layer.ffn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}); - layer.ffn_down = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_DOWN, "weight", i), { n_ff, n_embd}); - layer.ffn_up = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff}); + layer.wo = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd}); + layer.bo = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_OUT, "bias", i), {n_embd}, false); + + layer.ffn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}); + layer.ffn_norm_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_NORM, "bias", i), {n_embd}, false); + + layer.ffn_down = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_DOWN, "weight", i), {n_ff, n_embd}); + layer.ffn_down_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_DOWN, "bias", i), {n_embd}, false); + + layer.ffn_up = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff}); + layer.ffn_up_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_UP, "bias", i), {n_ff}, false); // AWQ ScaleActivation layer layer.ffn_act = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_ACT, "scales", i), {n_ff}, false); @@ -6171,7 +6182,7 @@ struct llm_build_context { attn_norm = llm_build_norm(ctx0, inpL, hparams, model.layers[il].attn_norm, - NULL, + model.layers[il].attn_norm_b, LLM_NORM, cb, il); cb(attn_norm, "attn_norm", il); @@ -6181,6 +6192,11 @@ struct llm_build_context { cur = ggml_mul_mat(ctx0, model.layers[il].wqkv, cur); cb(cur, "wqkv", il); + + if (model.layers[il].bqkv){ + cur = ggml_add(ctx0, cur, model.layers[il].bqkv); + cb(cur, "bqkv", il); + } if (hparams.f_clamp_kqv > 0.0f) { cur = ggml_clamp(ctx0, cur, -hparams.f_clamp_kqv, hparams.f_clamp_kqv); @@ -6198,7 +6214,7 @@ struct llm_build_context { Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); cur = llm_build_kv(ctx0, model, hparams, kv_self, gf, - model.layers[il].wo, NULL, + model.layers[il].wo, model.layers[il].bo, Kcur, Vcur, Qcur, KQ_mask, KQ_pos, n_ctx, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il); cb(cur, "kqv_out", il); } @@ -6211,13 +6227,13 @@ struct llm_build_context { { cur = llm_build_norm(ctx0, ffn_inp, hparams, model.layers[il].ffn_norm, - NULL, + model.layers[il].ffn_norm_b, LLM_NORM, cb, il); cb(cur, "ffn_norm", il); cur = llm_build_ffn(ctx0, cur, - model.layers[il].ffn_up, NULL, + model.layers[il].ffn_up, model.layers[il].ffn_up_b, NULL, NULL, - model.layers[il].ffn_down, NULL, + model.layers[il].ffn_down, model.layers[il].ffn_down_b, model.layers[il].ffn_act, LLM_FFN_GELU, LLM_FFN_SEQ, cb, il); cb(cur, "ffn_out", il); @@ -6234,7 +6250,7 @@ struct llm_build_context { cur = llm_build_norm(ctx0, cur, hparams, model.output_norm, - NULL, + model.output_norm_b, LLM_NORM, cb, -1); cb(cur, "result_norm", -1); From c5688c6250430d2b8e0259efcf26c16dfa4c1f46 Mon Sep 17 00:00:00 2001 From: Alexey Parfenov Date: Thu, 22 Feb 2024 08:27:32 +0000 Subject: [PATCH 432/812] server : clarify some params in the docs (#5640) --- examples/server/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/server/README.md b/examples/server/README.md index 4b24ee5dc..4b6cd8326 100644 --- a/examples/server/README.md +++ b/examples/server/README.md @@ -151,7 +151,7 @@ node index.js `temperature`: Adjust the randomness of the generated text (default: 0.8). - `dynatemp_range`: Dynamic temperature range (default: 0.0, 0.0 = disabled). + `dynatemp_range`: Dynamic temperature range. The final temperature will be in the range of `[temperature - dynatemp_range; temperature + dynatemp_range]` (default: 0.0, 0.0 = disabled). `dynatemp_exponent`: Dynamic temperature exponent (default: 1.0). @@ -209,7 +209,7 @@ node index.js `slot_id`: Assign the completion task to an specific slot. If is -1 the task will be assigned to a Idle slot (default: -1) - `cache_prompt`: Save the prompt and generation for avoid reprocess entire prompt if a part of this isn't change (default: false) + `cache_prompt`: Re-use previously cached prompt from the last request if possible. This may prevent re-caching the prompt from scratch. (default: false) `system_prompt`: Change the system prompt (initial prompt of all slots), this is useful for chat applications. [See more](#change-system-prompt-on-runtime) @@ -242,7 +242,7 @@ Notice that each `probs` is an array of length `n_probs`. - `content`: Completion result as a string (excluding `stopping_word` if any). In case of streaming mode, will contain the next token as a string. - `stop`: Boolean for use with `stream` to check whether the generation has stopped (Note: This is not related to stopping words array `stop` from input options) -- `generation_settings`: The provided options above excluding `prompt` but including `n_ctx`, `model` +- `generation_settings`: The provided options above excluding `prompt` but including `n_ctx`, `model`. These options may differ from the original ones in some way (e.g. bad values filtered out, strings converted to tokens, etc.). - `model`: The path to the model loaded with `-m` - `prompt`: The provided `prompt` - `stopped_eos`: Indicating whether the completion has stopped because it encountered the EOS token From a46f50747b2028f7f9c9883b26bfba12bf92556e Mon Sep 17 00:00:00 2001 From: Xuan Son Nguyen Date: Thu, 22 Feb 2024 09:33:24 +0100 Subject: [PATCH 433/812] server : fallback to chatml, add AlphaMonarch chat template (#5628) * server: fallback to chatml * add new chat template * server: add AlphaMonarch to test chat template * server: only check model template if there is no custom tmpl * remove TODO --- examples/server/server.cpp | 15 +++++++++++++++ llama.cpp | 9 +++++++++ tests/test-chat-template.cpp | 23 +++++++++++++++-------- 3 files changed, 39 insertions(+), 8 deletions(-) diff --git a/examples/server/server.cpp b/examples/server/server.cpp index c84719a0d..369121e88 100644 --- a/examples/server/server.cpp +++ b/examples/server/server.cpp @@ -400,6 +400,16 @@ struct llama_server_context return true; } + void validate_model_chat_template(server_params & sparams) { + llama_chat_message chat[] = {{"user", "test"}}; + std::vector buf(1); + int res = llama_chat_apply_template(model, nullptr, chat, 1, true, buf.data(), buf.size()); + if (res < 0) { + LOG_ERROR("The chat template comes with this model is not yet supported, falling back to chatml. This may cause the model to output suboptimal responses", {}); + sparams.chat_template = "<|im_start|>"; // llama_chat_apply_template only checks if <|im_start|> exist in the template + } + } + void initialize() { // create slots all_slots_are_idle = true; @@ -2752,6 +2762,11 @@ int main(int argc, char **argv) LOG_INFO("model loaded", {}); } + if (sparams.chat_template.empty()) { // custom chat template is not supplied + // check if the template comes with the model is supported by us + llama.validate_model_chat_template(sparams); + } + // Middleware for API key validation auto validate_api_key = [&sparams](const httplib::Request &req, httplib::Response &res) -> bool { // If API key is not set, skip validation diff --git a/llama.cpp b/llama.cpp index 9cae8c761..055b57e31 100644 --- a/llama.cpp +++ b/llama.cpp @@ -12773,6 +12773,15 @@ static int32_t llama_chat_apply_template_internal( if (add_ass) { ss << "<|assistant|>\n"; } + } else if (tmpl.find("bos_token + message['role']") != std::string::npos) { + // mlabonne/AlphaMonarch-7B template (the is included inside history) + for (auto message : chat) { + std::string bos = (message == chat.front()) ? "" : ""; // skip BOS for first message + ss << bos << message->role << "\n" << message->content << "\n"; + } + if (add_ass) { + ss << "assistant\n"; + } } else { // template not supported return -1; diff --git a/tests/test-chat-template.cpp b/tests/test-chat-template.cpp index 9830650d4..d02b39e14 100644 --- a/tests/test-chat-template.cpp +++ b/tests/test-chat-template.cpp @@ -27,12 +27,20 @@ int main(void) { "{%- for idx in range(0, messages|length) -%}\\n{%- if messages[idx]['role'] == 'user' -%}\\n{%- if idx > 1 -%}\\n{{- bos_token + '[INST] ' + messages[idx]['content'] + ' [/INST]' -}}\\n{%- else -%}\\n{{- messages[idx]['content'] + ' [/INST]' -}}\\n{%- endif -%}\\n{% elif messages[idx]['role'] == 'system' %}\\n{{- '[INST] <>\\\\n' + messages[idx]['content'] + '\\\\n<>\\\\n\\\\n' -}}\\n{%- elif messages[idx]['role'] == 'assistant' -%}\\n{{- ' ' + messages[idx]['content'] + ' ' + eos_token -}}\\n{% endif %}\\n{% endfor %}", // bofenghuang/vigogne-2-70b-chat "{{ bos_token }}{% if messages[0]['role'] == 'system' %}{% set loop_messages = messages[1:] %}{% set system_message = messages[0]['content'] %}{% elif true == true and not '<>' in messages[0]['content'] %}{% set loop_messages = messages %}{% set system_message = 'Vous êtes Vigogne, un assistant IA créé par Zaion Lab. Vous suivez extrêmement bien les instructions. Aidez autant que vous le pouvez.' %}{% else %}{% set loop_messages = messages %}{% set system_message = false %}{% endif %}{% for message in loop_messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if loop.index0 == 0 and system_message != false %}{% set content = '<>\\\\n' + system_message + '\\\\n<>\\\\n\\\\n' + message['content'] %}{% else %}{% set content = message['content'] %}{% endif %}{% if message['role'] == 'user' %}{{ '[INST] ' + content.strip() + ' [/INST]' }}{% elif message['role'] == 'system' %}{{ '<>\\\\n' + content.strip() + '\\\\n<>\\\\n\\\\n' }}{% elif message['role'] == 'assistant' %}{{ ' ' + content.strip() + ' ' + eos_token }}{% endif %}{% endfor %}", + // mlabonne/AlphaMonarch-7B + "{% for message in messages %}{{bos_token + message['role'] + '\\n' + message['content'] + eos_token + '\\n'}}{% endfor %}{% if add_generation_prompt %}{{ bos_token + 'assistant\\n' }}{% endif %}", }; - std::vector expected_substr = { - "<|im_start|>assistant\n I am an assistant <|im_end|>\n<|im_start|>user\nAnother question<|im_end|>\n<|im_start|>assistant", - "[/INST]Hi there[INST] Who are you [/INST] I am an assistant [INST] Another question [/INST]", - "[INST] Who are you [/INST] I am an assistant [INST] Another question [/INST]", - "[/INST] Hi there [INST] Who are you [/INST] I am an assistant [INST] Another question [/INST]", + std::vector expected_output = { + // teknium/OpenHermes-2.5-Mistral-7B + "<|im_start|>system\nYou are a helpful assistant<|im_end|>\n<|im_start|>user\nHello<|im_end|>\n<|im_start|>assistant\nHi there<|im_end|>\n<|im_start|>user\nWho are you<|im_end|>\n<|im_start|>assistant\n I am an assistant <|im_end|>\n<|im_start|>user\nAnother question<|im_end|>\n<|im_start|>assistant\n", + // mistralai/Mistral-7B-Instruct-v0.2 + "[INST] You are a helpful assistant\nHello [/INST]Hi there[INST] Who are you [/INST] I am an assistant [INST] Another question [/INST]", + // TheBloke/FusionNet_34Bx2_MoE-AWQ + "[INST] <>\nYou are a helpful assistant\n<>\n\nHello [/INST] Hi there [INST] Who are you [/INST] I am an assistant [INST] Another question [/INST]", + // bofenghuang/vigogne-2-70b-chat + "[INST] <>\nYou are a helpful assistant\n<>\n\nHello [/INST] Hi there [INST] Who are you [/INST] I am an assistant [INST] Another question [/INST]", + // mlabonne/AlphaMonarch-7B + "system\nYou are a helpful assistant\nuser\nHello\nassistant\nHi there\nuser\nWho are you\nassistant\n I am an assistant \nuser\nAnother question\nassistant\n", }; std::vector formatted_chat(1024); int32_t res; @@ -43,7 +51,7 @@ int main(void) { for (size_t i = 0; i < templates.size(); i++) { std::string custom_template = templates[i]; - std::string substr = expected_substr[i]; + std::string expected = expected_output[i]; formatted_chat.resize(1024); res = llama_chat_apply_template( nullptr, @@ -57,8 +65,7 @@ int main(void) { formatted_chat.resize(res); std::string output(formatted_chat.data(), formatted_chat.size()); std::cout << output << "\n-------------------------\n"; - // expect the "formatted_chat" to contain pre-defined strings - assert(output.find(substr) != std::string::npos); + assert(output == expected); } return 0; } From 56d03d92be57f5880b9ed94542d87bb6effae31f Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Thu, 22 Feb 2024 10:35:54 +0200 Subject: [PATCH 434/812] readme : update hot topics --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index c1624b9f9..3bc512af0 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,7 @@ Inference of Meta's [LLaMA](https://arxiv.org/abs/2302.13971) model (and others) ### Hot topics +- Support for chat templates: [Wiki (contributions welcome)](https://github.com/ggerganov/llama.cpp/wiki/Templates-supported-by-llama_chat_apply_template) - Support for Gemma models: https://github.com/ggerganov/llama.cpp/pull/5631 - Non-linear quantization IQ4_NL: https://github.com/ggerganov/llama.cpp/pull/5590 - Looking for contributions to improve and maintain the `server` example: https://github.com/ggerganov/llama.cpp/issues/4216 From 3a03541cedea474fa9d41214484cc3fbcf468a9e Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Thu, 22 Feb 2024 13:54:03 +0200 Subject: [PATCH 435/812] minor : fix trailing whitespace (#5638) --- llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llama.cpp b/llama.cpp index 055b57e31..6ab5e1bf4 100644 --- a/llama.cpp +++ b/llama.cpp @@ -6192,7 +6192,7 @@ struct llm_build_context { cur = ggml_mul_mat(ctx0, model.layers[il].wqkv, cur); cb(cur, "wqkv", il); - + if (model.layers[il].bqkv){ cur = ggml_add(ctx0, cur, model.layers[il].bqkv); cb(cur, "bqkv", il); From 4cb4d8b22d4fda971621a68c570ce84d66897c37 Mon Sep 17 00:00:00 2001 From: Someone Date: Thu, 22 Feb 2024 16:32:09 +0000 Subject: [PATCH 436/812] workflows: nix: hardcode cachix ids, build unconditionally (#5663) GitHub does not expose environment and repository variables to PRs coming from forks implies that we've been disabling the Nix CI actions for most PRs. The `if:` also didn't make much sense, because we can always pull from cachix, and there's no point (albeit no risk either) in pushing cache for the untrusted code. --- .github/workflows/nix-ci-aarch64.yml | 7 +++---- .github/workflows/nix-ci.yml | 11 +++++------ 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/.github/workflows/nix-ci-aarch64.yml b/.github/workflows/nix-ci-aarch64.yml index 0c6cf5f09..8d0a3fd7f 100644 --- a/.github/workflows/nix-ci-aarch64.yml +++ b/.github/workflows/nix-ci-aarch64.yml @@ -19,7 +19,6 @@ on: jobs: nix-build-aarch64: - if: ${{ vars.CACHIX_NAME != '' }} runs-on: ubuntu-latest steps: - name: Checkout repository @@ -37,8 +36,8 @@ jobs: extra-conf: | extra-platforms = aarch64-linux extra-system-features = nixos-test kvm - extra-substituters = https://${{ vars.CACHIX_NAME }}.cachix.org https://cuda-maintainers.cachix.org - extra-trusted-public-keys = ${{ vars.CACHIX_PUBLIC_KEY }} cuda-maintainers.cachix.org-1:0dq3bujKpuEPMCX6U4WylrUDZ9JyUG0VpVZa7CNfq5E= + extra-substituters = https://llama-cpp.cachix.org https://cuda-maintainers.cachix.org + extra-trusted-public-keys = llama-cpp.cachix.org-1:H75X+w83wUKTIPSO1KWy9ADUrzThyGs8P5tmAbkWhQc= cuda-maintainers.cachix.org-1:0dq3bujKpuEPMCX6U4WylrUDZ9JyUG0VpVZa7CNfq5E= - uses: DeterminateSystems/magic-nix-cache-action@v2 with: upstream-cache: https://${{ matrix.cachixName }}.cachix.org @@ -46,7 +45,7 @@ jobs: uses: cachix/cachix-action@v13 with: authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}' - name: ${{ vars.CACHIX_NAME }} + name: llama-cpp - name: Show all output paths run: > nix run github:nix-community/nix-eval-jobs diff --git a/.github/workflows/nix-ci.yml b/.github/workflows/nix-ci.yml index d19c7a576..01c5a9d5a 100644 --- a/.github/workflows/nix-ci.yml +++ b/.github/workflows/nix-ci.yml @@ -23,8 +23,8 @@ jobs: with: github-token: ${{ secrets.GITHUB_TOKEN }} extra-conf: | - extra-substituters = https://${{ vars.CACHIX_NAME }}.cachix.org https://cuda-maintainers.cachix.org - extra-trusted-public-keys = ${{ vars.CACHIX_PUBLIC_KEY }} cuda-maintainers.cachix.org-1:0dq3bujKpuEPMCX6U4WylrUDZ9JyUG0VpVZa7CNfq5E= + extra-substituters = https://llama-cpp.cachix.org https://cuda-maintainers.cachix.org + extra-trusted-public-keys = llama-cpp.cachix.org-1:H75X+w83wUKTIPSO1KWy9ADUrzThyGs8P5tmAbkWhQc= cuda-maintainers.cachix.org-1:0dq3bujKpuEPMCX6U4WylrUDZ9JyUG0VpVZa7CNfq5E= - uses: DeterminateSystems/magic-nix-cache-action@v2 with: upstream-cache: https://${{ matrix.cachixName }}.cachix.org @@ -37,7 +37,6 @@ jobs: --flake ".#packages.$(nix eval --raw --impure --expr builtins.currentSystem)" nix-build: - if: ${{ vars.CACHIX_NAME != '' }} strategy: fail-fast: false matrix: @@ -51,8 +50,8 @@ jobs: with: github-token: ${{ secrets.GITHUB_TOKEN }} extra-conf: | - extra-substituters = https://${{ vars.CACHIX_NAME }}.cachix.org https://cuda-maintainers.cachix.org - extra-trusted-public-keys = ${{ vars.CACHIX_PUBLIC_KEY }} cuda-maintainers.cachix.org-1:0dq3bujKpuEPMCX6U4WylrUDZ9JyUG0VpVZa7CNfq5E= + extra-substituters = https://llama-cpp.cachix.org https://cuda-maintainers.cachix.org + extra-trusted-public-keys = llama-cpp.cachix.org-1:H75X+w83wUKTIPSO1KWy9ADUrzThyGs8P5tmAbkWhQc= cuda-maintainers.cachix.org-1:0dq3bujKpuEPMCX6U4WylrUDZ9JyUG0VpVZa7CNfq5E= - uses: DeterminateSystems/magic-nix-cache-action@v2 with: upstream-cache: https://${{ matrix.cachixName }}.cachix.org @@ -60,7 +59,7 @@ jobs: uses: cachix/cachix-action@v13 with: authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}' - name: ${{ vars.CACHIX_NAME }} + name: llama-cpp - name: Build run: > nix run github:Mic92/nix-fast-build From 373ee3fbbabc4c1508eed4f5c3795b23a20939a3 Mon Sep 17 00:00:00 2001 From: Xuan Son Nguyen Date: Thu, 22 Feb 2024 19:10:21 +0100 Subject: [PATCH 437/812] Add Gemma chat template (#5665) * add gemma chat template * gemma: only apply system_prompt on non-model message --- llama.cpp | 22 ++++++++++++++++++++++ tests/test-chat-template.cpp | 4 ++++ 2 files changed, 26 insertions(+) diff --git a/llama.cpp b/llama.cpp index 6ab5e1bf4..40dda265c 100644 --- a/llama.cpp +++ b/llama.cpp @@ -12782,6 +12782,28 @@ static int32_t llama_chat_apply_template_internal( if (add_ass) { ss << "assistant\n"; } + } else if (tmpl.find("") != std::string::npos) { + // google/gemma-7b-it + std::string system_prompt = ""; + for (auto message : chat) { + std::string role(message->role); + if (role == "system") { + // there is no system message for gemma, but we will merge it with user prompt, so nothing is broken + system_prompt = trim(message->content); + continue; + } + // in gemma, "assistant" is "model" + role = role == "assistant" ? "model" : message->role; + ss << "" << role << "\n"; + if (!system_prompt.empty() && role != "model") { + ss << system_prompt << "\n\n"; + system_prompt = ""; + } + ss << trim(message->content) << "\n"; + } + if (add_ass) { + ss << "model\n"; + } } else { // template not supported return -1; diff --git a/tests/test-chat-template.cpp b/tests/test-chat-template.cpp index d02b39e14..fa2eb577b 100644 --- a/tests/test-chat-template.cpp +++ b/tests/test-chat-template.cpp @@ -29,6 +29,8 @@ int main(void) { "{{ bos_token }}{% if messages[0]['role'] == 'system' %}{% set loop_messages = messages[1:] %}{% set system_message = messages[0]['content'] %}{% elif true == true and not '<>' in messages[0]['content'] %}{% set loop_messages = messages %}{% set system_message = 'Vous êtes Vigogne, un assistant IA créé par Zaion Lab. Vous suivez extrêmement bien les instructions. Aidez autant que vous le pouvez.' %}{% else %}{% set loop_messages = messages %}{% set system_message = false %}{% endif %}{% for message in loop_messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if loop.index0 == 0 and system_message != false %}{% set content = '<>\\\\n' + system_message + '\\\\n<>\\\\n\\\\n' + message['content'] %}{% else %}{% set content = message['content'] %}{% endif %}{% if message['role'] == 'user' %}{{ '[INST] ' + content.strip() + ' [/INST]' }}{% elif message['role'] == 'system' %}{{ '<>\\\\n' + content.strip() + '\\\\n<>\\\\n\\\\n' }}{% elif message['role'] == 'assistant' %}{{ ' ' + content.strip() + ' ' + eos_token }}{% endif %}{% endfor %}", // mlabonne/AlphaMonarch-7B "{% for message in messages %}{{bos_token + message['role'] + '\\n' + message['content'] + eos_token + '\\n'}}{% endfor %}{% if add_generation_prompt %}{{ bos_token + 'assistant\\n' }}{% endif %}", + // google/gemma-7b-it + "{% if messages[0]['role'] == 'system' %}{{ raise_exception('System role not supported') }}{% endif %}{% for message in messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if (message['role'] == 'assistant') %}{% set role = 'model' %}{% else %}{% set role = message['role'] %}{% endif %}{{ '' + role + '\\n' + message['content'] | trim + '\\n' }}{% endfor %}{% if add_generation_prompt %}{{'model\\n'}}{% endif %}", }; std::vector expected_output = { // teknium/OpenHermes-2.5-Mistral-7B @@ -41,6 +43,8 @@ int main(void) { "[INST] <>\nYou are a helpful assistant\n<>\n\nHello [/INST] Hi there [INST] Who are you [/INST] I am an assistant [INST] Another question [/INST]", // mlabonne/AlphaMonarch-7B "system\nYou are a helpful assistant\nuser\nHello\nassistant\nHi there\nuser\nWho are you\nassistant\n I am an assistant \nuser\nAnother question\nassistant\n", + // google/gemma-7b-it + "user\nYou are a helpful assistant\n\nHello\nmodel\nHi there\nuser\nWho are you\nmodel\nI am an assistant\nuser\nAnother question\nmodel\n", }; std::vector formatted_chat(1024); int32_t res; From 5a9e2f60ba3d8362ba17c77ac3092906d49b813f Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Thu, 22 Feb 2024 20:13:25 +0200 Subject: [PATCH 438/812] py : minor fixes (#5668) --- convert-hf-to-gguf.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/convert-hf-to-gguf.py b/convert-hf-to-gguf.py index 9771fccf9..8630bbf29 100755 --- a/convert-hf-to-gguf.py +++ b/convert-hf-to-gguf.py @@ -655,6 +655,8 @@ class OrionModel(Model): self.gguf_writer.add_feed_forward_length(self.hparams["intermediate_size"]) self.gguf_writer.add_head_count(head_count) self.gguf_writer.add_head_count_kv(head_count_kv) + # note: config provides rms norm but it is actually layer norm + # ref: https://huggingface.co/OrionStarAI/Orion-14B-Chat/blob/276a17221ce42beb45f66fac657a41540e71f4f5/modeling_orion.py#L570-L571 self.gguf_writer.add_layer_norm_eps(self.hparams["rms_norm_eps"]) def write_tensors(self): @@ -1031,7 +1033,6 @@ class PersimmonModel(Model): self.gguf_writer.add_head_count_kv(head_count_kv) self.gguf_writer.add_rope_freq_base(self.hparams["rope_theta"]) self.gguf_writer.add_layer_norm_eps(self.hparams["layer_norm_eps"]) - self.gguf_writer.add_layer_norm_rms_eps(self.hparams["rms_norm_eps"]) def set_vocab(self): self._set_vocab_sentencepiece() From 201294ae177b308fb3a99dc504dd6d27e8afa907 Mon Sep 17 00:00:00 2001 From: Someone Date: Thu, 22 Feb 2024 19:44:10 +0000 Subject: [PATCH 439/812] nix: init singularity and docker images (#5056) Exposes a few attributes demonstrating how to build [singularity](https://docs.sylabs.io/guides/latest/user-guide/)/[apptainer](https://apptainer.org/) and Docker images re-using llama.cpp's Nix expression. Built locally on `x86_64-linux` with `nix build github:someoneserge/llama.cpp/feat/nix/images#llamaPackages.{docker,docker-min,sif,llama-cpp}` and it's fast and effective. --- .devops/nix/docker.nix | 37 +++++++++++++++++++++++++++++++++++++ .devops/nix/scope.nix | 3 +++ .devops/nix/sif.nix | 27 +++++++++++++++++++++++++++ 3 files changed, 67 insertions(+) create mode 100644 .devops/nix/docker.nix create mode 100644 .devops/nix/sif.nix diff --git a/.devops/nix/docker.nix b/.devops/nix/docker.nix new file mode 100644 index 000000000..d607b4575 --- /dev/null +++ b/.devops/nix/docker.nix @@ -0,0 +1,37 @@ +{ + lib, + dockerTools, + buildEnv, + llama-cpp, + interactive ? true, + coreutils, +}: + +# A tar that can be fed into `docker load`: +# +# $ nix build .#llamaPackages.docker +# $ docker load < result + +# For details and variations cf. +# - https://nixos.org/manual/nixpkgs/unstable/#ssec-pkgs-dockerTools-buildLayeredImage +# - https://discourse.nixos.org/t/a-faster-dockertools-buildimage-prototype/16922 +# - https://nixery.dev/ + +# Approximate (compressed) sizes, at the time of writing, are: +# +# .#llamaPackages.docker: 125M; +# .#llamaPackagesCuda.docker: 537M; +# .#legacyPackages.aarch64-linux.llamaPackagesXavier.docker: 415M. + +dockerTools.buildLayeredImage { + name = llama-cpp.pname; + tag = "latest"; + + contents = + [ llama-cpp ] + ++ lib.optionals interactive [ + coreutils + dockerTools.binSh + dockerTools.caCertificates + ]; +} diff --git a/.devops/nix/scope.nix b/.devops/nix/scope.nix index d295995a4..78530c9e8 100644 --- a/.devops/nix/scope.nix +++ b/.devops/nix/scope.nix @@ -12,5 +12,8 @@ lib.makeScope newScope ( self: { inherit llamaVersion; llama-cpp = self.callPackage ./package.nix { }; + docker = self.callPackage ./docker.nix { }; + docker-min = self.callPackage ./docker.nix { interactive = false; }; + sif = self.callPackage ./sif.nix { }; } ) diff --git a/.devops/nix/sif.nix b/.devops/nix/sif.nix new file mode 100644 index 000000000..7535ca0f3 --- /dev/null +++ b/.devops/nix/sif.nix @@ -0,0 +1,27 @@ +{ + lib, + singularity-tools, + llama-cpp, + bashInteractive, + interactive ? false, +}: + +let + optionalInt = cond: x: if cond then x else 0; +in +singularity-tools.buildImage rec { + inherit (llama-cpp) name; + contents = [ llama-cpp ] ++ lib.optionals interactive [ bashInteractive ]; + + # These are excessive (but safe) for most variants. Building singularity + # images requires superuser privileges, so we build them inside a VM in a + # writable image of pre-determined size. + # + # ROCm is currently affected by https://github.com/NixOS/nixpkgs/issues/276846 + # + # Expected image sizes: + # - cpu/blas: 150M, + # - cuda, all gencodes: 560M, + diskSize = 4096 + optionalInt llama-cpp.useRocm 16384; + memSize = diskSize; +} From efd56b1c2139d50b9b4381a212feb75d69598fda Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Thu, 22 Feb 2024 18:31:40 +0200 Subject: [PATCH 440/812] ggml : 32-bit arm compat (whisper/1891) * ggml : 32-bit arm compat * ggml : add ggml_vqtbl1q_s8 impl * ggml : cont --- ggml-quants.c | 35 ++++++++++++++++++++++++++++++----- 1 file changed, 30 insertions(+), 5 deletions(-) diff --git a/ggml-quants.c b/ggml-quants.c index 6336538f0..8917c8af1 100644 --- a/ggml-quants.c +++ b/ggml-quants.c @@ -438,6 +438,30 @@ inline static ggml_int8x16x4_t ggml_vld1q_s8_x4(const int8_t * ptr) { return res; } +// NOTE: not tested +inline static int8x16_t ggml_vqtbl1q_s8(int8x16_t a, uint8x16_t b) { + int8x16_t res; + + res[ 0] = a[b[ 0]]; + res[ 1] = a[b[ 1]]; + res[ 2] = a[b[ 2]]; + res[ 3] = a[b[ 3]]; + res[ 4] = a[b[ 4]]; + res[ 5] = a[b[ 5]]; + res[ 6] = a[b[ 6]]; + res[ 7] = a[b[ 7]]; + res[ 8] = a[b[ 8]]; + res[ 9] = a[b[ 9]]; + res[10] = a[b[10]]; + res[11] = a[b[11]]; + res[12] = a[b[12]]; + res[13] = a[b[13]]; + res[14] = a[b[14]]; + res[15] = a[b[15]]; + + return res; +} + #else #define ggml_int16x8x2_t int16x8x2_t @@ -451,6 +475,7 @@ inline static ggml_int8x16x4_t ggml_vld1q_s8_x4(const int8_t * ptr) { #define ggml_vld1q_u8_x4 vld1q_u8_x4 #define ggml_vld1q_s8_x2 vld1q_s8_x2 #define ggml_vld1q_s8_x4 vld1q_s8_x4 +#define ggml_vqtbl1q_s8 vqtbl1q_s8 #endif @@ -9333,7 +9358,7 @@ void ggml_vec_dot_iq1_s_q8_K (int n, float * GGML_RESTRICT s, size_t bs, const uint16_t gindex[8]; uint16x8x2_t vindex; int8x16x4_t q1b; - int8x16x4_t q8b; + ggml_int8x16x4_t q8b; uint16x8x4_t scales; int32x4x2_t sumi; int32x4x2_t dotq; @@ -9506,10 +9531,10 @@ void ggml_vec_dot_iq4_nl_q8_0(int n, float * restrict s, size_t bs, const void * q8b.val[2] = vld1q_s8(y[ib+1].qs); q8b.val[3] = vld1q_s8(y[ib+1].qs + 16); - q4b.val[0] = vqtbl1q_s8(values, vandq_u8(q4bits.val[0], m4b)); - q4b.val[1] = vqtbl1q_s8(values, vshrq_n_u8(q4bits.val[0], 4)); - q4b.val[2] = vqtbl1q_s8(values, vandq_u8(q4bits.val[1], m4b)); - q4b.val[3] = vqtbl1q_s8(values, vshrq_n_u8(q4bits.val[1], 4)); + q4b.val[0] = ggml_vqtbl1q_s8(values, vandq_u8 (q4bits.val[0], m4b)); + q4b.val[1] = ggml_vqtbl1q_s8(values, vshrq_n_u8(q4bits.val[0], 4)); + q4b.val[2] = ggml_vqtbl1q_s8(values, vandq_u8 (q4bits.val[1], m4b)); + q4b.val[3] = ggml_vqtbl1q_s8(values, vshrq_n_u8(q4bits.val[1], 4)); prod_1 = ggml_vdotq_s32(ggml_vdotq_s32(vdupq_n_s32(0), q4b.val[0], q8b.val[0]), q4b.val[1], q8b.val[1]); prod_2 = ggml_vdotq_s32(ggml_vdotq_s32(vdupq_n_s32(0), q4b.val[2], q8b.val[2]), q4b.val[3], q8b.val[3]); From 334f76fa385ed81095165e5ae068756214893901 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Thu, 22 Feb 2024 23:21:05 +0200 Subject: [PATCH 441/812] sync : ggml --- scripts/sync-ggml.last | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/sync-ggml.last b/scripts/sync-ggml.last index bbbf88d9d..59de34370 100644 --- a/scripts/sync-ggml.last +++ b/scripts/sync-ggml.last @@ -1 +1 @@ -30805514e1bf389a59d30a54a0525cbdc30d5bd1 +8cdf783f288a98eddf521b0ab1b4d405be9e18ba From 7e4f339c404dbe029d4a117c03b37a9bf646cf0e Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Thu, 22 Feb 2024 23:21:39 +0200 Subject: [PATCH 442/812] ggml : always define ggml_fp16_t as uint16_t (#5666) * ggml : always define ggml_fp16_t as uint16_t ggml-ci * ggml : cont ggml-ci * ggml : cont * ggml : cont ggml-ci * ggml : cont ggml-ci * cuda : no longer ggml headers last ggml-ci * ggml : fix q6_K FP16 -> FP32 conversion ggml-ci * ggml : more FP16 -> FP32 conversion fixes ggml-ci --- ggml-cuda.cu | 9 ++++----- ggml-impl.h | 27 ++++++++++++++++++++------- ggml-quants.c | 30 +++++++++++++++--------------- ggml.c | 6 +++--- ggml.h | 6 ------ 5 files changed, 42 insertions(+), 36 deletions(-) diff --git a/ggml-cuda.cu b/ggml-cuda.cu index e7c211d7d..b0e454e02 100644 --- a/ggml-cuda.cu +++ b/ggml-cuda.cu @@ -1,3 +1,7 @@ +#include "ggml-cuda.h" +#include "ggml.h" +#include "ggml-backend-impl.h" + #include #include #include @@ -121,11 +125,6 @@ #endif // defined(GGML_USE_HIPBLAS) -// ggml-cuda need half type so keep ggml headers include at last -#include "ggml-cuda.h" -#include "ggml.h" -#include "ggml-backend-impl.h" - #define CUDART_HMAX 11070 // CUDA 11.7, min. ver. for which __hmax and __hmax2 are known to work (may be higher than needed) #define CC_PASCAL 600 diff --git a/ggml-impl.h b/ggml-impl.h index 19df66bce..c5637e4d4 100644 --- a/ggml-impl.h +++ b/ggml-impl.h @@ -53,11 +53,23 @@ extern "C" { // #include -#define GGML_COMPUTE_FP16_TO_FP32(x) ((float) (x)) -#define GGML_COMPUTE_FP32_TO_FP16(x) (x) +#define GGML_COMPUTE_FP16_TO_FP32(x) ggml_compute_fp16_to_fp32(x) +#define GGML_COMPUTE_FP32_TO_FP16(x) ggml_compute_fp32_to_fp16(x) -#define GGML_FP16_TO_FP32(x) ((float) (x)) -#define GGML_FP32_TO_FP16(x) (x) +#define GGML_FP16_TO_FP32(x) ggml_compute_fp16_to_fp32(x) + +static inline float ggml_compute_fp16_to_fp32(ggml_fp16_t h) { + __fp16 tmp; + memcpy(&tmp, &h, sizeof(ggml_fp16_t)); + return (float)tmp; +} + +static inline ggml_fp16_t ggml_compute_fp32_to_fp16(float f) { + ggml_fp16_t res; + __fp16 tmp = f; + memcpy(&res, &tmp, sizeof(ggml_fp16_t)); + return res; +} #else @@ -214,8 +226,7 @@ extern float ggml_table_f32_f16[1 << 16]; // On ARM NEON, it's quicker to directly convert x -> x instead of calling into ggml_lookup_fp16_to_fp32, // so we define GGML_FP16_TO_FP32 and GGML_FP32_TO_FP16 elsewhere for NEON. // This is also true for POWER9. -#if !defined(GGML_FP16_TO_FP32) || !defined(GGML_FP32_TO_FP16) - +#if !defined(GGML_FP16_TO_FP32) inline static float ggml_lookup_fp16_to_fp32(ggml_fp16_t f) { uint16_t s; memcpy(&s, &f, sizeof(uint16_t)); @@ -223,8 +234,10 @@ inline static float ggml_lookup_fp16_to_fp32(ggml_fp16_t f) { } #define GGML_FP16_TO_FP32(x) ggml_lookup_fp16_to_fp32(x) -#define GGML_FP32_TO_FP16(x) GGML_COMPUTE_FP32_TO_FP16(x) +#endif +#if !defined(GGML_FP32_TO_FP16) +#define GGML_FP32_TO_FP16(x) GGML_COMPUTE_FP32_TO_FP16(x) #endif #define GGML_HASHTABLE_FULL ((size_t)-1) diff --git a/ggml-quants.c b/ggml-quants.c index 8917c8af1..b15977f53 100644 --- a/ggml-quants.c +++ b/ggml-quants.c @@ -5654,8 +5654,8 @@ void ggml_vec_dot_q2_K_q8_K(int n, float * restrict s, size_t bs, const void * r for (int i = 0; i < nb; ++i) { - const float d = y[i].d * (float)x[i].d; - const float dmin = -y[i].d * (float)x[i].dmin; + const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d); + const float dmin = -y[i].d * GGML_FP16_TO_FP32(x[i].dmin); const uint8_t * restrict q2 = x[i].qs; const int8_t * restrict q8 = y[i].qs; @@ -5804,8 +5804,8 @@ void ggml_vec_dot_q2_K_q8_K(int n, float * restrict s, size_t bs, const void * r for (int i = 0; i < nb; ++i) { - const float d = y[i].d * (float)x[i].d; - const float dmin = -y[i].d * (float)x[i].dmin; + const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d); + const float dmin = -y[i].d * GGML_FP16_TO_FP32(x[i].dmin); const uint8_t * restrict q2 = x[i].qs; const int8_t * restrict q8 = y[i].qs; @@ -6458,7 +6458,7 @@ void ggml_vec_dot_q3_K_q8_K(int n, float * restrict s, size_t bs, const void * r int32_t isum = -4*(scales[0] * y[i].bsums[0] + scales[2] * y[i].bsums[1] + scales[1] * y[i].bsums[2] + scales[3] * y[i].bsums[3]); - const float d = y[i].d * (float)x[i].d; + const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d); const uint8x16_t htmp = vcombine_u8(hbits, vshr_n_u8(hbits, 1)); q3h.val[0] = vandq_u8(mh, vshlq_n_u8(htmp, 2)); @@ -6660,7 +6660,7 @@ void ggml_vec_dot_q3_K_q8_K(int n, float * restrict s, size_t bs, const void * r int32_t isum = -4*(scales[0] * y[i].bsums[0] + scales[2] * y[i].bsums[1] + scales[1] * y[i].bsums[2] + scales[3] * y[i].bsums[3]); - const float d = y[i].d * (float)x[i].d; + const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d); vint32m1_t vzero = __riscv_vmv_v_x_i32m1(0, 1); @@ -7163,9 +7163,9 @@ void ggml_vec_dot_q4_K_q8_K(int n, float * restrict s, size_t bs, const void * r aux16[1] = (a[0] >> 4) & 0x0f0f; const int32_t summi = scales[2] * (y[i].bsums[0] + y[i].bsums[1]) + scales[3] * (y[i].bsums[2] + y[i].bsums[3]); - sum_mins += y[i].d * (float)x[i].d[1] * summi; + sum_mins += y[i].d * GGML_FP16_TO_FP32(x[i].d[1]) * summi; - const float d = y[i].d * (float)x[i].d[0]; + const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d[0]); const ggml_uint8x16x2_t q4bits = ggml_vld1q_u8_x2(q4); @@ -7823,7 +7823,7 @@ void ggml_vec_dot_q5_K_q8_K(int n, float * restrict s, size_t bs, const void * r for (int i = 0; i < nb; ++i) { - const float d = y[i].d * (float)x[i].d; + const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d); const int8_t * sc = x[i].scales; const uint8_t * restrict q5 = x[i].qs; @@ -7965,7 +7965,7 @@ void ggml_vec_dot_q5_K_q8_K(int n, float * restrict s, size_t bs, const void * r for (int i = 0; i < nb; ++i) { - const float d = y[i].d * (float)x[i].d; + const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d); const int8_t * sc = x[i].scales; const uint8_t * restrict q5 = x[i].qs; @@ -8533,7 +8533,7 @@ void ggml_vec_dot_q6_K_q8_K(int n, float * restrict s, size_t bs, const void * r for (int i = 0; i < nb; ++i) { - const float d_all = (float)x[i].d; + const float d_all = GGML_FP16_TO_FP32(x[i].d); const uint8_t * restrict q6 = x[i].ql; const uint8_t * restrict qh = x[i].qh; @@ -8704,7 +8704,7 @@ void ggml_vec_dot_q6_K_q8_K(int n, float * restrict s, size_t bs, const void * r for (int i = 0; i < nb; ++i) { - const float d_all = (float)x[i].d; + const float d_all = GGML_FP16_TO_FP32(x[i].d); const uint8_t * restrict q6 = x[i].ql; const uint8_t * restrict qh = x[i].qh; @@ -9523,7 +9523,6 @@ void ggml_vec_dot_iq4_nl_q8_0(int n, float * restrict s, size_t bs, const void * float sumf = 0; for (int ib = 0; ib < nb; ib += 2) { - q4bits.val[0] = vld1q_u8(x[ib+0].qs); q4bits.val[1] = vld1q_u8(x[ib+1].qs); q8b.val[0] = vld1q_s8(y[ib+0].qs); @@ -9539,8 +9538,9 @@ void ggml_vec_dot_iq4_nl_q8_0(int n, float * restrict s, size_t bs, const void * prod_1 = ggml_vdotq_s32(ggml_vdotq_s32(vdupq_n_s32(0), q4b.val[0], q8b.val[0]), q4b.val[1], q8b.val[1]); prod_2 = ggml_vdotq_s32(ggml_vdotq_s32(vdupq_n_s32(0), q4b.val[2], q8b.val[2]), q4b.val[3], q8b.val[3]); - sumf += (float)x[ib+0].d * (float)y[ib+0].d * vaddvq_s32(prod_1) + (float)x[ib+1].d * (float)y[ib+1].d * vaddvq_s32(prod_2); - + sumf += + GGML_FP16_TO_FP32(x[ib+0].d) * GGML_FP16_TO_FP32(y[ib+0].d) * vaddvq_s32(prod_1) + + GGML_FP16_TO_FP32(x[ib+1].d) * GGML_FP16_TO_FP32(y[ib+1].d) * vaddvq_s32(prod_2); } *s = sumf; diff --git a/ggml.c b/ggml.c index 5b9fa741a..d710fe702 100644 --- a/ggml.c +++ b/ggml.c @@ -323,7 +323,7 @@ float ggml_table_f32_f16[1 << 16]; // note: do not use these inside ggml.c // these are meant to be used via the ggml.h API float ggml_fp16_to_fp32(ggml_fp16_t x) { - return (float) GGML_FP16_TO_FP32(x); + return GGML_FP16_TO_FP32(x); } ggml_fp16_t ggml_fp32_to_fp16(float x) { @@ -798,7 +798,7 @@ inline static float vaddvq_f32(float32x4_t v) { #define GGML_F16x8 float16x8_t #define GGML_F16x8_ZERO vdupq_n_f16(0.0f) #define GGML_F16x8_SET1(x) vdupq_n_f16(x) - #define GGML_F16x8_LOAD vld1q_f16 + #define GGML_F16x8_LOAD(x) vld1q_f16((const __fp16 *)(x)) #define GGML_F16x8_STORE vst1q_f16 #define GGML_F16x8_FMA(a, b, c) vfmaq_f16(a, b, c) #define GGML_F16x8_ADD vaddq_f16 @@ -841,7 +841,7 @@ inline static float vaddvq_f32(float32x4_t v) { #define GGML_F32Cx4 float32x4_t #define GGML_F32Cx4_ZERO vdupq_n_f32(0.0f) #define GGML_F32Cx4_SET1(x) vdupq_n_f32(x) - #define GGML_F32Cx4_LOAD(x) vcvt_f32_f16(vld1_f16(x)) + #define GGML_F32Cx4_LOAD(x) vcvt_f32_f16(vld1_f16((const __fp16 *)(x))) #define GGML_F32Cx4_STORE(x, y) vst1_f16(x, vcvt_f16_f32(y)) #define GGML_F32Cx4_FMA(a, b, c) vfmaq_f32(a, b, c) #define GGML_F32Cx4_ADD vaddq_f32 diff --git a/ggml.h b/ggml.h index bed7a36a0..37eff6279 100644 --- a/ggml.h +++ b/ggml.h @@ -315,13 +315,7 @@ extern "C" { #endif -#if defined(__ARM_NEON) && defined(__CUDACC__) - typedef half ggml_fp16_t; -#elif defined(__ARM_NEON) && !defined(_MSC_VER) - typedef __fp16 ggml_fp16_t; -#else typedef uint16_t ggml_fp16_t; -#endif // convert FP16 <-> FP32 GGML_API float ggml_fp16_to_fp32(ggml_fp16_t x); From 847eedbdb2d1ebf14ef56eb507d4b4b975510908 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Thu, 22 Feb 2024 23:22:48 +0200 Subject: [PATCH 443/812] py : add Gemma conversion from HF models (#5647) * py : add gemma conversion from HF models * Update convert-hf-to-gguf.py Co-authored-by: Aarni Koskela * Update convert-hf-to-gguf.py Co-authored-by: Aarni Koskela * Update convert-hf-to-gguf.py Co-authored-by: Jared Van Bortel --------- Co-authored-by: Aarni Koskela Co-authored-by: Jared Van Bortel --- convert-hf-to-gguf.py | 60 +++++++++++++++++++++++++++++++++++++++++++ llama.cpp | 3 +++ 2 files changed, 63 insertions(+) diff --git a/convert-hf-to-gguf.py b/convert-hf-to-gguf.py index 8630bbf29..481198dad 100755 --- a/convert-hf-to-gguf.py +++ b/convert-hf-to-gguf.py @@ -218,6 +218,8 @@ class Model: return BertModel if model_architecture == "NomicBertModel": return NomicBertModel + if model_architecture == "GemmaForCausalLM": + return GemmaModel return Model def _is_model_safetensors(self) -> bool: @@ -277,6 +279,8 @@ class Model: return gguf.MODEL_ARCH.BERT if arch == "NomicBertModel": return gguf.MODEL_ARCH.NOMIC_BERT + if arch == "GemmaForCausalLM": + return gguf.MODEL_ARCH.GEMMA raise NotImplementedError(f'Architecture "{arch}" not supported!') @@ -1786,6 +1790,62 @@ class NomicBertModel(BertModel): yield name, data +class GemmaModel(Model): + def set_vocab(self): + self._set_vocab_sentencepiece() + + def set_gguf_parameters(self): + hparams = self.hparams + block_count = hparams["num_hidden_layers"] + + self.gguf_writer.add_name(self.dir_model.name) + self.gguf_writer.add_context_length(hparams["max_position_embeddings"]) + self.gguf_writer.add_embedding_length(hparams["hidden_size"]) + self.gguf_writer.add_block_count(block_count) + self.gguf_writer.add_feed_forward_length(hparams["intermediate_size"]) + self.gguf_writer.add_head_count(hparams["num_attention_heads"]) + self.gguf_writer.add_head_count_kv(self.hparams["num_key_value_heads"] if "num_key_value_heads" in hparams else hparams["num_attention_heads"]) + self.gguf_writer.add_layer_norm_rms_eps(self.hparams["rms_norm_eps"]) + self.gguf_writer.add_key_length(hparams["head_dim"]) + self.gguf_writer.add_value_length(hparams["head_dim"]) + + def write_tensors(self): + block_count = self.hparams.get("n_layers", self.hparams.get("num_hidden_layers", self.hparams.get("n_layer"))) + tensor_map = gguf.get_tensor_name_map(self.model_arch, block_count) + + for name, data_torch in self.get_tensors(): + # ref: https://github.com/huggingface/transformers/blob/fc37f38915372c15992b540dfcbbe00a916d4fc6/src/transformers/models/gemma/modeling_gemma.py#L89 + if name.endswith("norm.weight"): + data_torch = data_torch + 1 + + old_dtype = data_torch.dtype + + # convert any unsupported data types to float32 + if data_torch.dtype not in (torch.float16, torch.float32): + data_torch = data_torch.to(torch.float32) + + data = data_torch.squeeze().numpy() + + # map tensor names + new_name = tensor_map.get_name(name, try_suffixes=(".weight", ".bias")) + if new_name is None: + print(f"Can not map tensor {name!r}") + sys.exit() + + n_dims = len(data.shape) + data_dtype = data.dtype + + data = data.astype(np.float32) + + # if f16 desired, convert any float32 2-dim weight tensors to float16 + if self.ftype == 1 and data_dtype == np.float32 and name.endswith(".weight") and n_dims == 2: + data = data.astype(np.float16) + + print(f"{new_name}, n_dims = {n_dims}, {old_dtype} --> {data.dtype}") + + self.gguf_writer.add_tensor(new_name, data) + + ###### CONVERSION LOGIC ###### diff --git a/llama.cpp b/llama.cpp index 40dda265c..7770fa0e8 100644 --- a/llama.cpp +++ b/llama.cpp @@ -7450,6 +7450,7 @@ struct llm_build_context { inpL = llm_build_inp_embd(ctx0, hparams, batch, model.tok_embd, lctx.inp_tokens, lctx.inp_embd, cb); cb(inpL, "inp_embd", -1); + inpL = ggml_scale(ctx0, inpL, sqrtf(n_embd)); cb(inpL, "inp_scaled", -1); @@ -7491,6 +7492,7 @@ struct llm_build_context { n_embd_head_k, 2, 0, n_orig_ctx, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow); cb(Qcur, "Qcur", il); + Qcur = ggml_scale(ctx0, Qcur, 1.0f / sqrtf(float(n_embd_head_k))); cb(Qcur, "Qcur_scaled", il); @@ -7505,6 +7507,7 @@ struct llm_build_context { Kcur, Vcur, Qcur, KQ_mask, nullptr, n_ctx, n_tokens, kv_head, n_kv, 1.0f, cb, il); cb(cur, "kqv_out", il); } + struct ggml_tensor * sa_out = ggml_add(ctx0, cur, inpL); cb(sa_out, "sa_out", il); From 96633eeca1265ed03e57230de54032041c58f9cd Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Thu, 22 Feb 2024 23:23:46 +0200 Subject: [PATCH 444/812] gemma : use more bits for the token_embd.weight tensor (#5650) * gemma : use Q8_0 for the token_embd.weight tensor * llama : quantize token_embd.weight using output type --- llama.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/llama.cpp b/llama.cpp index 7770fa0e8..2ebd40df2 100644 --- a/llama.cpp +++ b/llama.cpp @@ -10498,7 +10498,10 @@ static ggml_type get_k_quant_type(quantize_state_internal & qs, ggml_type new_ty return std::make_pair(i_layer, n_layer); }; - if (name == tn(LLM_TENSOR_OUTPUT, "weight")) { + // for arches that share the same tensor between the token embeddings and the output, we quantize the token embeddings + // with the quantization of the output tensor + if (name == tn(LLM_TENSOR_OUTPUT, "weight") || + (LLM_TENSOR_NAMES.at(arch).find(LLM_TENSOR_OUTPUT) == LLM_TENSOR_NAMES.at(arch).end() && name == "token_embd.weight")) { int nx = tensor->ne[0]; if (arch == LLM_ARCH_FALCON || nx % QK_K != 0) { new_type = GGML_TYPE_Q8_0; From 15499eb94227401bdc8875da6eb85c15d37068f7 Mon Sep 17 00:00:00 2001 From: Jared Van Bortel Date: Thu, 22 Feb 2024 17:05:23 -0500 Subject: [PATCH 445/812] mpt : do not duplicate token_embd.weight on disk (#5670) --- convert-hf-to-gguf.py | 5 ----- llama.cpp | 6 ++++-- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/convert-hf-to-gguf.py b/convert-hf-to-gguf.py index 481198dad..9bdfce07a 100755 --- a/convert-hf-to-gguf.py +++ b/convert-hf-to-gguf.py @@ -622,11 +622,6 @@ class MPTModel(Model): self.gguf_writer.add_tensor(new_name, data) - # note: MPT output is tied to (same as) wte in original model; - # for easier implementation in llama.cpp it's duplicated in GGUF, though :/ - if new_name == "token_embd.weight": - self.gguf_writer.add_tensor("output.weight", data) - class OrionModel(Model): def set_vocab(self): diff --git a/llama.cpp b/llama.cpp index 2ebd40df2..37477e6ef 100644 --- a/llama.cpp +++ b/llama.cpp @@ -509,7 +509,6 @@ static std::map> LLM_TENSOR_NAMES = { { LLM_TENSOR_TOKEN_EMBD, "token_embd" }, { LLM_TENSOR_OUTPUT_NORM, "output_norm" }, - { LLM_TENSOR_OUTPUT, "output" }, { LLM_TENSOR_ATTN_NORM, "blk.%d.attn_norm" }, { LLM_TENSOR_FFN_NORM, "blk.%d.ffn_norm" }, { LLM_TENSOR_ATTN_QKV, "blk.%d.attn_qkv" }, @@ -4056,7 +4055,10 @@ static bool llm_load_tensors( model.output_norm = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}); model.output_norm_b = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "bias"), {n_embd}, false); - model.output = ml.create_tensor(ctx_output_split, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}); + // same as tok_embd, duplicated to allow offloading + model.output = ml.create_tensor(ctx_output_split, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}); + ml.n_created--; // artificial tensor + ml.size_data += ggml_nbytes(model.output); } for (int i = 0; i < n_layer; ++i) { From 54fbcd2ce6c48c9e22eca6fbf9e53fb68c3e72ea Mon Sep 17 00:00:00 2001 From: Jared Van Bortel Date: Fri, 23 Feb 2024 13:39:14 -0500 Subject: [PATCH 446/812] convert : fix missing ftype for gemma (#5690) --- convert-hf-to-gguf.py | 1 + 1 file changed, 1 insertion(+) diff --git a/convert-hf-to-gguf.py b/convert-hf-to-gguf.py index 9bdfce07a..32d54b45f 100755 --- a/convert-hf-to-gguf.py +++ b/convert-hf-to-gguf.py @@ -1803,6 +1803,7 @@ class GemmaModel(Model): self.gguf_writer.add_layer_norm_rms_eps(self.hparams["rms_norm_eps"]) self.gguf_writer.add_key_length(hparams["head_dim"]) self.gguf_writer.add_value_length(hparams["head_dim"]) + self.gguf_writer.add_file_type(self.ftype) def write_tensors(self): block_count = self.hparams.get("n_layers", self.hparams.get("num_hidden_layers", self.hparams.get("n_layer"))) From fd43d66f46ee3b5345fb8a74a252d86ccd34a409 Mon Sep 17 00:00:00 2001 From: AlpinDale <52078762+AlpinDale@users.noreply.github.com> Date: Fri, 23 Feb 2024 19:31:54 +0000 Subject: [PATCH 447/812] server : add KV cache quantization options (#5684) --- examples/server/server.cpp | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/examples/server/server.cpp b/examples/server/server.cpp index 369121e88..524d0ada3 100644 --- a/examples/server/server.cpp +++ b/examples/server/server.cpp @@ -1948,6 +1948,10 @@ static void server_print_usage(const char *argv0, const gpt_params ¶ms, printf(" -cb, --cont-batching enable continuous batching (a.k.a dynamic batching) (default: disabled)\n"); printf(" -spf FNAME, --system-prompt-file FNAME\n"); printf(" set a file to load a system prompt (initial prompt of all slots), this is useful for chat applications.\n"); + printf(" -ctk TYPE, --cache-type-k TYPE\n"); + printf(" KV cache data type for K (default: f16)\n"); + printf(" -ctv TYPE, --cache-type-v TYPE\n"); + printf(" KV cache data type for V (default: f16)\n"); printf(" --mmproj MMPROJ_FILE path to a multimodal projector file for LLaVA.\n"); printf(" --log-disable disables logging to a file.\n"); printf(" --slots-endpoint-disable disables slots monitoring endpoint.\n"); @@ -2386,6 +2390,12 @@ static void server_params_parse(int argc, char **argv, server_params &sparams, ); llama.process_system_prompt_data(json::parse(systm_content)); } + else if (arg == "-ctk" || arg == "--cache-type-k") { + params.cache_type_k = argv[++i]; + } + else if (arg == "-ctv" || arg == "--cache-type-v") { + params.cache_type_v = argv[++i]; + } else if(arg == "--mmproj") { if (++i >= argc) From 525213d2f5da1eaf4b922b6b792cb52b2c613368 Mon Sep 17 00:00:00 2001 From: Pierrick Hymbert Date: Sat, 24 Feb 2024 12:28:55 +0100 Subject: [PATCH 448/812] server: init functional tests (#5566) * server: tests: init scenarios - health and slots endpoints - completion endpoint - OAI compatible chat completion requests w/ and without streaming - completion multi users scenario - multi users scenario on OAI compatible endpoint with streaming - multi users with total number of tokens to predict exceeds the KV Cache size - server wrong usage scenario, like in Infinite loop of "context shift" #3969 - slots shifting - continuous batching - embeddings endpoint - multi users embedding endpoint: Segmentation fault #5655 - OpenAI-compatible embeddings API - tokenize endpoint - CORS and api key scenario * server: CI GitHub workflow --------- Co-authored-by: Georgi Gerganov --- .github/ISSUE_TEMPLATE/bug.md | 2 + .github/workflows/server.yml | 127 ++++ examples/server/README.md | 6 + examples/server/server.cpp | 36 +- examples/server/tests/README.md | 46 ++ examples/server/tests/features/environment.py | 67 ++ examples/server/tests/features/issues.feature | 36 + .../server/tests/features/parallel.feature | 77 ++ .../server/tests/features/security.feature | 50 ++ examples/server/tests/features/server.feature | 69 ++ examples/server/tests/features/steps/steps.py | 709 ++++++++++++++++++ .../tests/features/wrong_usages.feature | 21 + examples/server/tests/requirements.txt | 3 + examples/server/tests/tests.sh | 12 + 14 files changed, 1243 insertions(+), 18 deletions(-) create mode 100644 .github/workflows/server.yml create mode 100644 examples/server/tests/README.md create mode 100644 examples/server/tests/features/environment.py create mode 100644 examples/server/tests/features/issues.feature create mode 100644 examples/server/tests/features/parallel.feature create mode 100644 examples/server/tests/features/security.feature create mode 100644 examples/server/tests/features/server.feature create mode 100644 examples/server/tests/features/steps/steps.py create mode 100644 examples/server/tests/features/wrong_usages.feature create mode 100644 examples/server/tests/requirements.txt create mode 100755 examples/server/tests/tests.sh diff --git a/.github/ISSUE_TEMPLATE/bug.md b/.github/ISSUE_TEMPLATE/bug.md index ce69e6395..49812832c 100644 --- a/.github/ISSUE_TEMPLATE/bug.md +++ b/.github/ISSUE_TEMPLATE/bug.md @@ -7,3 +7,5 @@ assignees: '' --- Please include information about your system, the steps to reproduce the bug, and the version of llama.cpp that you are using. If possible, please provide a minimal code example that reproduces the bug. + +If the bug concerns the server, please try to reproduce it first using the [server test scenario framework](https://github.com/ggerganov/llama.cpp/tree/master/examples/server/tests). diff --git a/.github/workflows/server.yml b/.github/workflows/server.yml new file mode 100644 index 000000000..ed27dc528 --- /dev/null +++ b/.github/workflows/server.yml @@ -0,0 +1,127 @@ +# Server build and tests +name: Server + +on: + workflow_dispatch: # allows manual triggering + push: + branches: + - master + - test/server-add-ci-test # FIXME remove + paths: ['.github/workflows/**', '**/CMakeLists.txt', '**/Makefile', '**/*.h', '**/*.hpp', '**/*.c', '**/*.cpp', '**/*.cu', '**/*.swift', '**/*.m', 'examples/server/**.*'] + pull_request: + types: [opened, synchronize, reopened] + paths: ['**/CMakeLists.txt', '**/Makefile', '**/*.h', '**/*.hpp', '**/*.c', '**/*.cpp', '**/*.cu', '**/*.swift', '**/*.m', 'examples/server/**.*'] + +jobs: + server: + runs-on: ubuntu-latest + + strategy: + matrix: + build: [noavx, avx2, avx, avx512, cublas, clblast, openblas, kompute, vulkan] + sanitizer: [ADDRESS, THREAD, UNDEFINED] + build_type: [Debug, Release] + include: + - build: 'noavx' + defines: '-DLLAMA_NATIVE=OFF -DLLAMA_BUILD_SERVER=ON -DLLAMA_AVX=OFF -DLLAMA_AVX2=OFF -DLLAMA_FMA=OFF' + image: ubuntu:latest + - build: 'avx2' + defines: '-DLLAMA_NATIVE=OFF -DLLAMA_BUILD_SERVER=ON' + image: ubuntu:latest + - build: 'avx' + defines: '-DLLAMA_NATIVE=OFF -DLLAMA_BUILD_SERVER=ON -DLLAMA_AVX2=OFF' + image: ubuntu:latest + - build: 'avx512' + defines: '-DLLAMA_NATIVE=OFF -DLLAMA_BUILD_SERVER=ON -DLLAMA_AVX512=ON' + image: ubuntu:latest + experimental: true + - build: 'cublas' + defines: '-DLLAMA_NATIVE=OFF -DLLAMA_BUILD_SERVER=ON -DLLAMA_CUBLAS=ON' + image: nvidia/cuda:12.3.1-devel-ubuntu22.04 + arch_not_available: true # require nvidia docker engine + - build: 'clblast' + defines: '-DLLAMA_NATIVE=OFF -DLLAMA_BUILD_SERVER=ON -DLLAMA_CLBLAST=ON' + image: ubuntu:latest + arch_not_available: true + - build: 'openblas' + defines: '-DLLAMA_NATIVE=OFF -DLLAMA_BUILD_SERVER=ON -DLLAMA_BLAS=ON -DLLAMA_BLAS_VENDOR=OpenBLAS' + image: ubuntu:latest + - build: 'kompute' + defines: '-DLLAMA_NATIVE=OFF -DLLAMA_BUILD_SERVER=ON -DLLAMA_KOMPUTE=ON -DKOMPUTE_OPT_DISABLE_VULKAN_VERSION_CHECK=ON' + image: ubuntu:latest + arch_not_available: true + - build: 'vulkan' + defines: '-DLLAMA_NATIVE=OFF -DLLAMA_BUILD_SERVER=ON -DLLAMA_VULKAN=ON' + image: ubuntu:latest + arch_not_available: true + + container: + image: ${{ matrix.image }} + ports: + - 8888 + options: --cpus 4 + + steps: + - name: Clone + id: checkout + uses: actions/checkout@v3 + + - name: Dependencies + id: depends + run: | + apt-get update + apt-get -y install \ + build-essential \ + pkg-config \ + git \ + cmake \ + python3-pip \ + wget \ + psmisc + + - name: Download CLBlast + id: get_clblast + if: ${{ matrix.build == 'clblast' }} + run: | + apt install -y libclblast-dev + + - name: Download OpenBLAS + id: get_openblas + if: ${{ matrix.build == 'openblas' }} + run: | + apt-get -y install libopenblas-dev + + - name: Install Vulkan SDK + id: get_vulkan + if: ${{ matrix.build == 'kompute' || matrix.build == 'vulkan' }} + run: | + wget -qO- https://packages.lunarg.com/lunarg-signing-key-pub.asc | tee /etc/apt/trusted.gpg.d/lunarg.asc + wget -qO /etc/apt/sources.list.d/lunarg-vulkan-jammy.list http://packages.lunarg.com/vulkan/lunarg-vulkan-jammy.list + apt-get update + apt-get -y install vulkan-sdk + + - name: Build + id: cmake_build + run: | + mkdir build + cd build + cmake .. -DLLAMA_SANITIZE_${{ matrix.sanitizer }}=ON -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} ${{ matrix.defines }} + cmake --build . --config ${{ matrix.build_type }} -j $(nproc) --target server + + - name: Tests dependencies + id: test_dependencies + run: | + pip install -r examples/server/tests/requirements.txt + + - name: Download models + id: download_models + run: | + cd examples/server/tests + ../../../scripts/hf.sh --repo ggml-org/models --file tinyllamas/stories260K.gguf + + - name: Tests + id: server_integration_test + continue-on-error: ${{ matrix.experimental || matrix.arch_not_available }} + run: | + cd examples/server/tests + PORT=8888 ./tests.sh diff --git a/examples/server/README.md b/examples/server/README.md index 4b6cd8326..0c43ac4c9 100644 --- a/examples/server/README.md +++ b/examples/server/README.md @@ -98,6 +98,12 @@ curl --request POST \ --data '{"prompt": "Building a website can be done in 10 simple steps:","n_predict": 128}' ``` +## Advanced testing + +We implemented a [server test framework](./tests/README.md) using human-readable scenario. + +*Before submitting an issue, please try to reproduce it with this format.* + ## Node JS Test You need to have [Node.js](https://nodejs.org/en) installed. diff --git a/examples/server/server.cpp b/examples/server/server.cpp index 524d0ada3..9fb436c2a 100644 --- a/examples/server/server.cpp +++ b/examples/server/server.cpp @@ -1410,11 +1410,6 @@ struct llama_server_context int n_processing_slots = 0; for (llama_client_slot &slot: slots) { - if (slot.available()) { - n_idle_slots++; - } else { - n_processing_slots++; - } json slot_data = get_formated_generation(slot); slot_data["id"] = slot.id; slot_data["task_id"] = slot.task_id; @@ -1429,6 +1424,11 @@ struct llama_server_context {"stopped_limit", slot.stopped_limit}, {"stopping_word", slot.stopping_word}, }; + if (slot_data["state"] == IDLE) { + n_idle_slots++; + } else { + n_processing_slots++; + } slots_data.push_back(slot_data); } LOG_TEE("task %i - slots data: idle=%i processing=%i\n", task.id, n_idle_slots, n_processing_slots); @@ -2748,19 +2748,6 @@ int main(int argc, char **argv) log_data["api_key"] = "api_key: " + std::to_string(sparams.api_keys.size()) + " keys loaded"; } - LOG_INFO("HTTP server listening", log_data); - // run the HTTP server in a thread - see comment below - std::thread t([&]() - { - if (!svr.listen_after_bind()) - { - state.store(SERVER_STATE_ERROR); - return 1; - } - - return 0; - }); - // load the model if (!llama.load_model(params)) { @@ -3228,6 +3215,19 @@ int main(int argc, char **argv) }*/ //); + LOG_INFO("HTTP server listening", log_data); + // run the HTTP server in a thread - see comment below + std::thread t([&]() + { + if (!svr.listen_after_bind()) + { + state.store(SERVER_STATE_ERROR); + return 1; + } + + return 0; + }); + llama.queue_tasks.on_new_task(std::bind( &llama_server_context::process_single_task, &llama, std::placeholders::_1)); llama.queue_tasks.on_finish_multitask(std::bind( diff --git a/examples/server/tests/README.md b/examples/server/tests/README.md new file mode 100644 index 000000000..e44c5c286 --- /dev/null +++ b/examples/server/tests/README.md @@ -0,0 +1,46 @@ +# Server tests + +Python based server tests scenario using [BDD](https://en.wikipedia.org/wiki/Behavior-driven_development) and [behave](https://behave.readthedocs.io/en/latest/): + * [issues.feature](./features/issues.feature) Pending issues scenario + * [parallel.feature](./features/parallel.feature) Scenario involving multi slots and concurrent requests + * [security.feature](./features/security.feature) Security, CORS and API Key + * [server.feature](./features/server.feature) Server base scenario: completion, embedding, tokenization, etc... + +Tests target GitHub workflows job runners with 4 vCPU. + +Requests are using [aiohttp](https://docs.aiohttp.org/en/stable/client_reference.html), [asyncio](https://docs.python.org/fr/3/library/asyncio.html) based http client. + +Note: If the host architecture inference speed is faster than GitHub runners one, parallel scenario may randomly fail. To mitigate it, you can increase values in `n_predict`, `kv_size`. + +### Install dependencies +`pip install -r requirements.txt` + +### Run tests +1. Build the server +```shell +cd ../../.. +mkdir build +cd build +cmake ../ +cmake --build . --target server +``` +2. download required models: + 1. `../../../scripts/hf.sh --repo ggml-org/models --file tinyllamas/stories260K.gguf` +3. Start the test: `./tests.sh` + +It's possible to override some scenario steps values with environment variables: + - `PORT` -> `context.server_port` to set the listening port of the server during scenario, default: `8080` + - `LLAMA_SERVER_BIN_PATH` -> to change the server binary path, default: `../../../build/bin/server` + - `DEBUG` -> "ON" to enable steps and server verbose mode `--verbose` + +### Run @bug, @wip or @wrong_usage annotated scenario + +Feature or Scenario must be annotated with `@llama.cpp` to be included in the default scope. +- `@bug` annotation aims to link a scenario with a GitHub issue. +- `@wrong_usage` are meant to show user issue that are actually an expected behavior +- `@wip` to focus on a scenario working in progress + +To run a scenario annotated with `@bug`, start: +`DEBUG=ON ./tests.sh --no-skipped --tags bug` + +After changing logic in `steps.py`, ensure that `@bug` and `@wrong_usage` scenario are updated. diff --git a/examples/server/tests/features/environment.py b/examples/server/tests/features/environment.py new file mode 100644 index 000000000..13cc84101 --- /dev/null +++ b/examples/server/tests/features/environment.py @@ -0,0 +1,67 @@ +import os +import socket +import subprocess +import time +from contextlib import closing +from signal import SIGKILL + + +def before_scenario(context, scenario): + print(f"\x1b[33;42mStarting new scenario: {scenario.name}!\x1b[0m") + port = 8080 + if 'PORT' in os.environ: + port = int(os.environ['PORT']) + if is_server_listening("localhost", port): + assert False, "Server already started" + + +def after_scenario(context, scenario): + if scenario.status == "failed": + if 'GITHUB_ACTIONS' in os.environ: + print(f"\x1b[33;101mSCENARIO FAILED: {scenario.name} server logs:\x1b[0m\n\n") + if os.path.isfile('llama.log'): + with closing(open('llama.log', 'r')) as f: + for line in f: + print(line) + if not is_server_listening(context.server_fqdn, context.server_port): + print("\x1b[33;101mERROR: Server stopped listening\x1b[0m") + + if not pid_exists(context.server_process.pid): + assert False, f"Server not running pid={context.server_process.pid} ..." + + print(f"stopping server pid={context.server_process.pid} ...") + context.server_process.kill() + # Wait few for socket to free up + time.sleep(0.05) + + attempts = 0 + while is_server_listening(context.server_fqdn, context.server_port): + print(f"stopping server pid={context.server_process.pid} ...") + os.kill(context.server_process.pid, SIGKILL) + time.sleep(0.1) + attempts += 1 + if attempts > 5: + print(f"Server dangling exits, killing all {context.server_path} ...") + process = subprocess.run(['killall', '-9', context.server_path], + stderr=subprocess.PIPE, + universal_newlines=True) + print(process) + + +def is_server_listening(server_fqdn, server_port): + with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as sock: + result = sock.connect_ex((server_fqdn, server_port)) + return result == 0 + + +def pid_exists(pid): + """Check whether pid exists in the current process table.""" + import errno + if pid < 0: + return False + try: + os.kill(pid, 0) + except OSError as e: + return e.errno == errno.EPERM + else: + return True diff --git a/examples/server/tests/features/issues.feature b/examples/server/tests/features/issues.feature new file mode 100644 index 000000000..542006d9a --- /dev/null +++ b/examples/server/tests/features/issues.feature @@ -0,0 +1,36 @@ +# List of ongoing issues +@bug +Feature: Issues + # Issue #5655 + Scenario: Multi users embeddings + Given a server listening on localhost:8080 + And a model file stories260K.gguf + And a model alias tinyllama-2 + And 42 as server seed + And 64 KV cache size + And 2 slots + And continuous batching + And embeddings extraction + Then the server is starting + Then the server is healthy + + Given a prompt: + """ + Write a very long story about AI. + """ + And a prompt: + """ + Write another very long music lyrics. + """ + And a prompt: + """ + Write a very long poem. + """ + And a prompt: + """ + Write a very long joke. + """ + Given concurrent embedding requests + Then the server is busy + Then the server is idle + Then all embeddings are generated diff --git a/examples/server/tests/features/parallel.feature b/examples/server/tests/features/parallel.feature new file mode 100644 index 000000000..802d624ff --- /dev/null +++ b/examples/server/tests/features/parallel.feature @@ -0,0 +1,77 @@ +@llama.cpp +Feature: Parallel + + Background: Server startup + Given a server listening on localhost:8080 + And a model file stories260K.gguf + And a model alias tinyllama-2 + And 42 as server seed + And 64 KV cache size + And 2 slots + And continuous batching + Then the server is starting + Then the server is healthy + + Scenario Outline: Multi users completion + Given a prompt: + """ + Write a very long story about AI. + """ + And a prompt: + """ + Write another very long music lyrics. + """ + And max tokens to predict + Given concurrent completion requests + Then the server is busy + Then the server is idle + And all slots are idle + Then all prompts are predicted with tokens + Examples: + | n_predict | + | 128 | + + Scenario Outline: Multi users OAI completions compatibility + Given a system prompt You are a writer. + And a model tinyllama-2 + Given a prompt: + """ + Write a very long book. + """ + And a prompt: + """ + Write another a poem. + """ + And max tokens to predict + And streaming is + Given concurrent OAI completions requests + Then the server is busy + Then the server is idle + Then all prompts are predicted with tokens + Examples: + | streaming | n_predict | + | disabled | 128 | + | enabled | 64 | + + Scenario: Multi users with total number of tokens to predict exceeds the KV Cache size #3969 + Given a prompt: + """ + Write a very long story about AI. + """ + And a prompt: + """ + Write another very long music lyrics. + """ + And a prompt: + """ + Write a very long poem. + """ + And a prompt: + """ + Write a very long joke. + """ + And 128 max tokens to predict + Given concurrent completion requests + Then the server is busy + Then the server is idle + Then all prompts are predicted diff --git a/examples/server/tests/features/security.feature b/examples/server/tests/features/security.feature new file mode 100644 index 000000000..db06d3977 --- /dev/null +++ b/examples/server/tests/features/security.feature @@ -0,0 +1,50 @@ +@llama.cpp +Feature: Security + + Background: Server startup with an api key defined + Given a server listening on localhost:8080 + And a model file stories260K.gguf + And a server api key llama.cpp + Then the server is starting + Then the server is healthy + + Scenario Outline: Completion with some user api key + Given a prompt test + And a user api key + And 4 max tokens to predict + And a completion request with api error + + Examples: Prompts + | api_key | api_error | + | llama.cpp | no | + | llama.cpp | no | + | hackeme | raised | + | | raised | + + Scenario Outline: OAI Compatibility + Given a system prompt test + And a user prompt test + And a model test + And 2 max tokens to predict + And streaming is disabled + And a user api key + Given an OAI compatible chat completions request with api error + + Examples: Prompts + | api_key | api_error | + | llama.cpp | no | + | llama.cpp | no | + | hackme | raised | + + + Scenario Outline: CORS Options + When an OPTIONS request is sent from + Then CORS header is set to + + Examples: Headers + | origin | cors_header | cors_header_value | + | localhost | Access-Control-Allow-Origin | localhost | + | web.mydomain.fr | Access-Control-Allow-Origin | web.mydomain.fr | + | origin | Access-Control-Allow-Credentials | true | + | web.mydomain.fr | Access-Control-Allow-Methods | POST | + | web.mydomain.fr | Access-Control-Allow-Headers | * | diff --git a/examples/server/tests/features/server.feature b/examples/server/tests/features/server.feature new file mode 100644 index 000000000..fedcfe5ae --- /dev/null +++ b/examples/server/tests/features/server.feature @@ -0,0 +1,69 @@ +@llama.cpp +Feature: llama.cpp server + + Background: Server startup + Given a server listening on localhost:8080 + And a model file stories260K.gguf + And a model alias tinyllama-2 + And 42 as server seed + # KV Cache corresponds to the total amount of tokens + # that can be stored across all independent sequences: #4130 + # see --ctx-size and #5568 + And 32 KV cache size + And 1 slots + And embeddings extraction + And 32 server max tokens to predict + Then the server is starting + Then the server is healthy + + Scenario: Health + Then the server is ready + And all slots are idle + + Scenario Outline: Completion + Given a prompt + And max tokens to predict + And a completion request with no api error + Then tokens are predicted matching + + Examples: Prompts + | prompt | n_predict | re_content | n_predicted | + | I believe the meaning of life is | 8 | read | 8 | + | Write a joke about AI | 64 | (parkfriendsscared)+ | 32 | + + Scenario Outline: OAI Compatibility + Given a model + And a system prompt + And a user prompt + And max tokens to predict + And streaming is + Given an OAI compatible chat completions request with no api error + Then tokens are predicted matching + + Examples: Prompts + | model | system_prompt | user_prompt | max_tokens | re_content | n_predicted | enable_streaming | + | llama-2 | Book | What is the best book | 8 | (Momwhat)+ | 8 | disabled | + | codellama70b | You are a coding assistant. | Write the fibonacci function in c++. | 64 | (thankshappybird)+ | 32 | enabled | + + Scenario: Embedding + When embeddings are computed for: + """ + What is the capital of Bulgaria ? + """ + Then embeddings are generated + + Scenario: OAI Embeddings compatibility + Given a model tinyllama-2 + When an OAI compatible embeddings computation request for: + """ + What is the capital of Spain ? + """ + Then embeddings are generated + + + Scenario: Tokenize / Detokenize + When tokenizing: + """ + What is the capital of France ? + """ + Then tokens can be detokenize diff --git a/examples/server/tests/features/steps/steps.py b/examples/server/tests/features/steps/steps.py new file mode 100644 index 000000000..50f2b641e --- /dev/null +++ b/examples/server/tests/features/steps/steps.py @@ -0,0 +1,709 @@ +import asyncio +import json +import os +import re +import socket +import subprocess +import time +from contextlib import closing +from re import RegexFlag + +import aiohttp +import openai +from behave import step +from behave.api.async_step import async_run_until_complete + + +@step(u"a server listening on {server_fqdn}:{server_port}") +def step_server_config(context, server_fqdn, server_port): + context.server_fqdn = server_fqdn + context.server_port = int(server_port) + if 'PORT' in os.environ: + context.server_port = int(os.environ['PORT']) + print(f"$PORT set, overriding server port with to {context.server_port}") + + context.base_url = f'http://{context.server_fqdn}:{context.server_port}' + + context.debug = 'DEBUG' in os.environ and os.environ['DEBUG'] == 'ON' + context.model_alias = None + context.n_ctx = None + context.n_predict = None + context.n_server_predict = None + context.n_slots = None + context.server_api_key = None + context.server_continuous_batching = False + context.server_embeddings = False + context.server_seed = None + context.user_api_key = None + + context.tasks_result = [] + context.concurrent_tasks = [] + context.prompts = [] + + +@step(u'a model file {model_file}') +def step_model_file(context, model_file): + context.model_file = model_file + + +@step(u'a model alias {model_alias}') +def step_model_alias(context, model_alias): + context.model_alias = model_alias + + +@step(u'{seed} as server seed') +def step_seed(context, seed): + context.server_seed = int(seed) + + +@step(u'{n_ctx} KV cache size') +def step_n_ctx(context, n_ctx): + context.n_ctx = int(n_ctx) + + +@step(u'{n_slots} slots') +def step_n_slots(context, n_slots): + context.n_slots = int(n_slots) + + +@step(u'{n_predict} server max tokens to predict') +def step_server_n_predict(context, n_predict): + context.n_server_predict = int(n_predict) + + +@step(u'continuous batching') +def step_server_continuous_batching(context): + context.server_continuous_batching = True + + +@step(u'embeddings extraction') +def step_server_embeddings(context): + context.server_embeddings = True + + +@step(u"the server is starting") +def step_start_server(context): + start_server_background(context) + attempts = 0 + while True: + with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as sock: + result = sock.connect_ex((context.server_fqdn, context.server_port)) + if result == 0: + print("\x1b[33;46mserver started!\x1b[0m") + return + attempts += 1 + if attempts > 20: + assert False, "server not started" + print(f"waiting for server to start, connect error code = {result}...") + time.sleep(0.1) + + +@step(u"the server is {expecting_status}") +@async_run_until_complete +async def step_wait_for_the_server_to_be_started(context, expecting_status): + match expecting_status: + case 'healthy': + await wait_for_health_status(context, context.base_url, 200, 'ok') + + case 'ready' | 'idle': + await wait_for_health_status(context, context.base_url, 200, 'ok', + params={'fail_on_no_slot': 0, 'include_slots': 0}, + slots_idle=context.n_slots, + slots_processing=0, + expected_slots=[{'id': slot_id, 'state': 0} + for slot_id in range(context.n_slots)]) + case 'busy': + await wait_for_health_status(context, context.base_url, 503, + 'no slot available', + params={'fail_on_no_slot': 0, 'include_slots': 0}, + slots_idle=0, + slots_processing=context.n_slots, + expected_slots=[{'id': slot_id, 'state': 1} + for slot_id in range(context.n_slots)]) + case _: + assert False, "unknown status" + + +@step(u'all slots are {expected_slot_status_string}') +@async_run_until_complete +async def step_all_slots_status(context, expected_slot_status_string): + match expected_slot_status_string: + case 'idle': + expected_slot_status = 0 + case 'busy': + expected_slot_status = 1 + case _: + assert False, "unknown status" + + expected_slots = [{'id': slot_id, 'state': expected_slot_status} + for slot_id in range(context.n_slots)] + await request_slots_status(context, expected_slots) + + +@step(u'a completion request with {api_error} api error') +@async_run_until_complete +async def step_request_completion(context, api_error): + expect_api_error = api_error == 'raised' + completion = await request_completion(context.prompts.pop(), + context.base_url, + debug=context.debug, + n_predict=context.n_predict, + server_seed=context.server_seed, + expect_api_error=expect_api_error, + user_api_key=context.user_api_key) + context.tasks_result.append(completion) + if context.debug: + print(f"Completion response: {completion}") + if expect_api_error: + assert completion == 401, f"completion must be an 401 status code: {completion}" + + +@step(u'{predicted_n} tokens are predicted matching {re_content}') +def step_n_tokens_predicted_with_content(context, predicted_n, re_content): + assert_n_tokens_predicted(context.tasks_result.pop(), int(predicted_n), re_content) + + +@step(u'{predicted_n} tokens are predicted') +def step_n_tokens_predicted(context, predicted_n): + assert_n_tokens_predicted(context.tasks_result.pop(), int(predicted_n)) + + +@step(u'a user prompt {user_prompt}') +def step_user_prompt(context, user_prompt): + context.prompts.append(user_prompt) + + +@step(u'a system prompt {system_prompt}') +def step_system_prompt(context, system_prompt): + context.system_prompt = system_prompt + + +@step(u'a model {model}') +def step_model(context, model): + context.model = model + + +@step(u'{max_tokens} max tokens to predict') +def step_max_tokens(context, max_tokens): + context.n_predict = int(max_tokens) + + +@step(u'streaming is {enable_streaming}') +def step_streaming(context, enable_streaming): + context.enable_streaming = enable_streaming == 'enabled' + + +@step(u'a user api key {user_api_key}') +def step_user_api_key(context, user_api_key): + context.user_api_key = user_api_key + + +@step(u'no user api key') +def step_no_user_api_key(context): + context.user_api_key = None + + +@step(u'a user api key ') +def step_no_user_api_key_space(context): + context.user_api_key = None + + +@step(u'a server api key {server_api_key}') +def step_server_api_key(context, server_api_key): + context.server_api_key = server_api_key + + +@step(u'an OAI compatible chat completions request with {api_error} api error') +@async_run_until_complete +async def step_oai_chat_completions(context, api_error): + if context.debug: + print(f"Submitting OAI compatible completions request...") + expect_api_error = api_error == 'raised' + completion = await oai_chat_completions(context.prompts.pop(), + context.system_prompt, + context.base_url, + False, + model=context.model if hasattr(context, 'model') else None, + + n_predict=context.n_predict + if hasattr(context, 'n_predict') else None, + + enable_streaming=context.enable_streaming + if hasattr(context, 'enable_streaming') else None, + + server_seed=context.server_seed + if hasattr(context, 'server_seed') else None, + + user_api_key=context.user_api_key + if hasattr(context, 'user_api_key') else None, + + expect_api_error=expect_api_error) + context.tasks_result.append(completion) + if context.debug: + print(f"Completion response: {completion}") + if expect_api_error: + assert completion == 401, f"completion must be an 401 status code: {completion}" + + if context.debug: + print(f"Completion response: {completion}") + + +@step(u'a prompt') +def step_a_prompt(context): + context.prompts.append(context.text) + + +@step(u'a prompt {prompt}') +def step_a_prompt_prompt(context, prompt): + context.prompts.append(prompt) + + +@step(u'concurrent completion requests') +@async_run_until_complete() +async def step_concurrent_completion_requests(context): + await concurrent_completion_requests(context, + request_completion, + # prompt is inserted automatically + context.base_url, + debug=context.debug, + n_predict=context.n_predict if hasattr(context, 'n_predict') else None, + server_seed=context.server_seed if hasattr(context, 'server_seed') else None, + user_api_key=context.user_api_key if hasattr(context, + 'user_api_key') else None) + + +@step(u'concurrent OAI completions requests') +@async_run_until_complete +async def step_oai_chat_completions(context): + await concurrent_completion_requests(context, oai_chat_completions, + # user_prompt is inserted automatically + context.system_prompt, + context.base_url, + True, # async_client + model=context.model + if hasattr(context, 'model') else None, + n_predict=context.n_predict + if hasattr(context, 'n_predict') else None, + enable_streaming=context.enable_streaming + if hasattr(context, 'enable_streaming') else None, + server_seed=context.server_seed + if hasattr(context, 'server_seed') else None, + user_api_key=context.user_api_key + if hasattr(context, 'user_api_key') else None) + + +@step(u'all prompts are predicted') +@async_run_until_complete +async def step_all_prompts_are_predicted(context): + await all_prompts_are_predicted(context) + + +@step(u'all prompts are predicted with {n_predict} tokens') +@async_run_until_complete +async def step_all_prompts_are_predicted_with_n_tokens(context, n_predict): + expected_predicted_n = int(n_predict) + await all_prompts_are_predicted(context, expected_predicted_n) + + +async def all_prompts_are_predicted(context, expected_predicted_n=None): + n_completions = await gather_tasks_results(context) + assert n_completions > 0 + for i in range(n_completions): + assert_n_tokens_predicted(context.tasks_result.pop(), expected_predicted_n=expected_predicted_n) + assert len(context.concurrent_tasks) == 0, f"{len(context.concurrent_tasks)} pending requests" + + +@step(u'embeddings are computed for') +@async_run_until_complete +async def step_compute_embedding(context): + content = context.text + base_url = context.base_url + context.embeddings = await request_embedding(content, base_url) + + +@step(u'embeddings are generated') +def step_assert_embeddings(context): + assert_embeddings(context.embeddings) + + +@step(u'an OAI compatible embeddings computation request for') +def step_oai_compute_embedding(context): + openai.api_key = 'nope' # openai client always expects an api_keu + if context.user_api_key is not None: + openai.api_key = context.user_api_key + openai.api_base = f'{context.base_url}/v1' + embeddings = openai.Embedding.create( + model=context.model, + input=context.text, + ) + context.embeddings = embeddings + + +@step(u'concurrent embedding requests') +@async_run_until_complete() +async def step_concurrent_embedding_requests(context): + await concurrent_completion_requests(context, + request_embedding, + # prompt is inserted automatically + context.base_url) + + +@step(u'all embeddings are generated') +@async_run_until_complete() +async def all_embeddings_are_generated(context): + n_embedding_requests = await gather_tasks_results(context) + assert n_embedding_requests > 0 + for i in range(n_embedding_requests): + assert_embeddings(context.tasks_result.pop()) + + +@step(u'tokenizing') +@async_run_until_complete +async def step_tokenize(context): + context.tokenized_text = context.text + async with aiohttp.ClientSession() as session: + async with session.post(f'{context.base_url}/tokenize', + json={ + "content": context.tokenized_text, + }) as response: + assert response.status == 200 + tokenize_json = await response.json() + context.tokens = tokenize_json['tokens'] + + +@step(u'tokens can be detokenize') +@async_run_until_complete +async def step_detokenize(context): + assert len(context.tokens) > 0 + async with aiohttp.ClientSession() as session: + async with session.post(f'{context.base_url}/detokenize', + json={ + "tokens": context.tokens, + }) as response: + assert response.status == 200 + detokenize_json = await response.json() + # SPM tokenizer adds a whitespace prefix: https://github.com/google/sentencepiece/issues/15 + assert context.tokenized_text == detokenize_json['content'].strip() + + +@step(u'an OPTIONS request is sent from {origin}') +@async_run_until_complete +async def step_options_request(context, origin): + async with aiohttp.ClientSession() as session: + async with session.options(f'{context.base_url}/v1/chat/completions', + headers={"Origin": origin}) as response: + assert response.status == 200 + context.options_response = response + + +@step(u'CORS header {cors_header} is set to {cors_header_value}') +def step_check_options_header_value(context, cors_header, cors_header_value): + assert context.options_response.headers[cors_header] == cors_header_value + + +async def concurrent_completion_requests(context, f_completion, *args, **kwargs): + n_prompts = len(context.prompts) + if context.debug: + print(f"starting {n_prompts} concurrent completion requests...") + assert n_prompts > 0 + for prompt_no in range(n_prompts): + shifted_args = [context.prompts.pop(), *args] + context.concurrent_tasks.append(asyncio.create_task(f_completion(*shifted_args, **kwargs))) + await asyncio.sleep(0.1) + + +async def request_completion(prompt, + base_url, + debug=False, + n_predict=None, + server_seed=None, + expect_api_error=None, + user_api_key=None): + if debug: + print(f"Sending completion request: {prompt}") + origin = "my.super.domain" + headers = { + 'Origin': origin + } + if user_api_key is not None: + if debug: + print(f"Set user_api_key: {user_api_key}") + headers['Authorization'] = f'Bearer {user_api_key}' + + async with aiohttp.ClientSession() as session: + async with session.post(f'{base_url}/completion', + json={ + "prompt": prompt, + "n_predict": int(n_predict) if n_predict is not None else -1, + "seed": server_seed if server_seed is not None else 42 + }, + headers=headers) as response: + if expect_api_error is None or not expect_api_error: + assert response.status == 200 + assert response.headers['Access-Control-Allow-Origin'] == origin + return await response.json() + else: + return response.status + + +async def oai_chat_completions(user_prompt, + system_prompt, + base_url, + async_client, + debug=False, + model=None, + n_predict=None, + enable_streaming=None, + server_seed=None, + user_api_key=None, + expect_api_error=None): + if debug: + print(f"Sending OAI Chat completions request: {user_prompt}") + # openai client always expects an api key + user_api_key = user_api_key if user_api_key is not None else 'nope' + seed = server_seed if server_seed is not None else 42 + enable_streaming = enable_streaming if enable_streaming is not None else False + payload = { + "messages": [ + { + "role": "system", + "content": system_prompt, + }, + { + "role": "user", + "content": user_prompt, + } + ], + "model": model, + "max_tokens": n_predict, + "stream": enable_streaming, + "seed": seed + } + completion_response = { + 'content': '', + 'timings': { + 'predicted_n': 0 + } + } + if async_client: + origin = 'llama.cpp' + headers = {'Authorization': f'Bearer {user_api_key}', 'Origin': origin} + async with aiohttp.ClientSession() as session: + async with session.post(f'{base_url}/v1/chat/completions', + json=payload, + headers=headers) as response: + if enable_streaming: + assert response.status == 200 + assert response.headers['Access-Control-Allow-Origin'] == origin + assert response.headers['Content-Type'] == "text/event-stream" + event_received = True + while event_received: + event_received = False + async for line_in_bytes in response.content: + line = line_in_bytes.decode('utf8') + line = line.rstrip('\n').rstrip('\r') + if line == '': + continue + event_data = line.split(': ', 1) + assert event_data[0] == 'data', f'Bad event code received: ```{event_data}```' + chunk_raw = event_data[1] + + chunk = json.loads(chunk_raw) + assert len(chunk['choices']) == 1, f"no choices provided, line ```{line}```" + delta = chunk['choices'][0]['delta'] + if 'content' in delta: + completion_response['content'] += delta['content'] + completion_response['timings']['predicted_n'] += 1 + else: + if expect_api_error is None or not expect_api_error: + assert response.status == 200 + assert response.headers['Access-Control-Allow-Origin'] == origin + assert response.headers['Content-Type'] == "application/json; charset=utf-8" + chat_completion_raw = await response.json() + completion_response = { + 'content': chat_completion_raw['choices'][0]['message'], + 'timings': { + 'predicted_n': chat_completion_raw['usage']['completion_tokens'] + } + } + else: + return response.status + else: + try: + openai.api_key = user_api_key + openai.api_base = f'{base_url}/v1/chat' + chat_completion = openai.Completion.create( + messages=payload['messages'], + model=model, + max_tokens=n_predict, + stream=enable_streaming, + seed=seed + ) + except openai.error.APIError as e: + if expect_api_error is not None and expect_api_error: + return 401 + else: + assert False, f'error raised: {e}' + + if enable_streaming: + for chunk in chat_completion: + assert len(chunk.choices) == 1 + delta = chunk.choices[0].delta + if 'content' in delta: + completion_response['content'] += delta['content'] + completion_response['timings']['predicted_n'] += 1 + else: + assert len(chat_completion.choices) == 1 + completion_response = { + 'content': chat_completion.choices[0].message.content, + 'timings': { + 'predicted_n': chat_completion.usage.completion_tokens + } + } + if debug: + print("OAI response formatted to llama.cpp:", completion_response) + return completion_response + + +async def request_embedding(content, base_url): + async with aiohttp.ClientSession() as session: + async with session.post(f'{base_url}/embedding', + json={ + "content": content, + }) as response: + assert response.status == 200 + response_json = await response.json() + return response_json['embedding'] + + +def assert_n_tokens_predicted(completion_response, expected_predicted_n=None, re_content=None): + content = completion_response['content'] + n_predicted = completion_response['timings']['predicted_n'] + assert len(content) > 0, "no token predicted" + if expected_predicted_n is not None: + assert n_predicted == expected_predicted_n, (f'invalid number of tokens predicted:' + f' {n_predicted} <> {expected_predicted_n}') + if re_content is not None: + re_content = '^.*' + re_content.replace('', '|') + '.*$' + assert re.match(re_content, content, flags=RegexFlag.IGNORECASE | RegexFlag.MULTILINE | RegexFlag.DOTALL), ( + f'invalid tokens predicted:' + f' ```\n{content}\n``` do not match /{re_content}/') + + +async def gather_tasks_results(context): + n_tasks = len(context.concurrent_tasks) + if context.debug: + print(f"Waiting for all {n_tasks} tasks results...") + for task_no in range(n_tasks): + context.tasks_result.append(await context.concurrent_tasks.pop()) + n_completions = len(context.tasks_result) + return n_completions + + +async def wait_for_health_status(context, + base_url, + expected_http_status_code, + expected_health_status, + params=None, + slots_idle=None, + slots_processing=None, + expected_slots=None): + if context.debug: + print(f"Starting checking for health for expected_health_status={expected_health_status}") + timeout = 3 # seconds + interval = 0.5 + counter = 0 + async with aiohttp.ClientSession() as session: + while True: + async with await session.get(f'{base_url}/health', params=params) as health_response: + status_code = health_response.status + health = await health_response.json() + if context.debug: + print(f"HEALTH - response for expected health status='{expected_health_status}' on " + f"'{base_url}/health'?{params} is {health}") + if (status_code == expected_http_status_code + and health['status'] == expected_health_status + and (slots_idle is None or health['slots_idle'] == slots_idle) + and (slots_processing is None or health['slots_processing'] == slots_processing)): + if expected_slots is not None: + assert_slots_status(health['slots'], expected_slots) + return + if (status_code == expected_http_status_code + and health['status'] == expected_health_status + and (slots_idle is None or health['slots_idle'] == slots_idle) + and (slots_processing is None or health['slots_processing'] == slots_processing)): + if expected_slots is not None: + assert_slots_status(health['slots'], expected_slots) + return + await asyncio.sleep(interval) + + counter += interval + if counter >= timeout: + # Sometimes health requests are triggered after completions are predicted + if expected_http_status_code == 503: + if len(context.tasks_result) == 0: + print("\x1b[5;37;43mWARNING: forcing concurrent tasks," + " busy health check missed, probably too fast inference\x1b[0m") + n_completions = await gather_tasks_results(context) + if n_completions > 0: + return + + assert False, 'timeout exceeded' + + +def assert_embeddings(embeddings): + assert len(embeddings) > 0 + embeddings_computed = False + for emb in embeddings: + if emb != 0: + embeddings_computed = True + assert embeddings_computed, f"Embeddings: {embeddings}" + + +async def request_slots_status(context, expected_slots): + async with aiohttp.ClientSession() as session: + async with await session.get(f'{context.base_url}/slots') as slots_response: + assert slots_response.status == 200 + slots = await slots_response.json() + assert_slots_status(slots, expected_slots) + + +def assert_slots_status(slots, expected_slots): + assert len(slots) == len(expected_slots) + for slot_id, (expected, slot) in enumerate(zip(expected_slots, slots)): + for key in expected: + assert expected[key] == slot[key], (f"invalid slot {slot_id}" + f" expected[{key}] != slot[{key}]" + f" = {expected[key]} != {slot[key]}") + + +def start_server_background(context): + context.server_path = '../../../build/bin/server' + if 'LLAMA_SERVER_BIN_PATH' in os.environ: + context.server_path = os.environ['LLAMA_SERVER_BIN_PATH'] + server_args = [ + '--host', context.server_fqdn, + '--port', context.server_port, + '--model', context.model_file + ] + if context.server_continuous_batching: + server_args.append('--cont-batching') + if context.server_embeddings: + server_args.append('--embedding') + if context.model_alias is not None: + server_args.extend(['--alias', context.model_alias]) + if context.n_ctx is not None: + server_args.extend(['--ctx-size', context.n_ctx]) + if context.n_slots is not None: + server_args.extend(['--parallel', context.n_slots]) + if context.n_server_predict is not None: + server_args.extend(['--n-predict', context.n_server_predict]) + if context.server_api_key is not None: + server_args.extend(['--api-key', context.server_api_key]) + if context.debug: + server_args.append('--verbose') + print(f"starting server with: {context.server_path}", *server_args) + context.server_process = subprocess.Popen( + [str(arg) for arg in [context.server_path, *server_args]], + close_fds=True) + print(f"server pid={context.server_process.pid}") diff --git a/examples/server/tests/features/wrong_usages.feature b/examples/server/tests/features/wrong_usages.feature new file mode 100644 index 000000000..e228b2371 --- /dev/null +++ b/examples/server/tests/features/wrong_usages.feature @@ -0,0 +1,21 @@ +# run with ./test.sh --tags wrong_usage +@wrong_usage +Feature: Wrong usage of llama.cpp server + + #3969 The user must always set --n-predict option + # to cap the number of tokens any completion request can generate + # or pass n_predict/max_tokens in the request. + Scenario: Infinite loop + Given a server listening on localhost:8080 + And a model file stories260K.gguf + # Uncomment below to fix the issue + #And 64 server max tokens to predict + Then the server is starting + Given a prompt: + """ + Go to: infinite loop + """ + # Uncomment below to fix the issue + #And 128 max tokens to predict + Given concurrent completion requests + Then all prompts are predicted diff --git a/examples/server/tests/requirements.txt b/examples/server/tests/requirements.txt new file mode 100644 index 000000000..3e51b12dc --- /dev/null +++ b/examples/server/tests/requirements.txt @@ -0,0 +1,3 @@ +aiohttp~=3.9.3 +behave~=1.2.6 +openai~=0.25.0 diff --git a/examples/server/tests/tests.sh b/examples/server/tests/tests.sh new file mode 100755 index 000000000..17a4e6fc6 --- /dev/null +++ b/examples/server/tests/tests.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +set -eu + +if [ $# -lt 1 ] +then + # Start @llama.cpp scenario + behave --summary --stop --no-capture --exclude 'issues|wrong_usages' --tags llama.cpp +else + behave "$@" +fi + From 4c4cb30736582cacb1a164a9d4bc8e17b1014be7 Mon Sep 17 00:00:00 2001 From: Kawrakow <48489457+ikawrakow@users.noreply.github.com> Date: Sat, 24 Feb 2024 16:23:52 +0200 Subject: [PATCH 449/812] IQ3_S: a much better alternative to Q3_K (#5676) * iq4_nl: squash commits for easier rebase * Basics (quantize, dequantize) * CUDA dequantize and dot product * Slightly faster CUDA dot product (120 t/s) * Switch to 6-bit scales * Scalar dot product * AVX2 dot product * ARM_NEON dot product * Works on metal, but still slow * Slightly better Metal dot product * Another small Metal improvement * Metal dot product is getting there * Faster CUDA dot product * Add 1/8 ffn_down layers as Q5_K when no imatrix has been provided * Report the actual bpw * Add _xs mix that is 4.05 bpw for non-MoE models * Remove IQ4_XS for now, slightly adjust kvalues_iq4nl * AVX2 dot product uses Q8_0 instead of Q8_K * Add to test-backend-ops * Minor fix * Also use use Q5_K for attn_output in MoE models * Fixes after merging latest master * Switching to blocks of 32 * AVX2 for blocks of 32 * Scaler dot product for blocks of 32 * ARM_NEON dot product for blocks of 32 * Metal kernels for blocks of 32 * Slightly faster Metal kernels * Resurrecting iq3_xs After all the experimentation, nothing was better than this. * Minor PPL improvement via a block scale fudge factor * Minor improvement via 3 neighbours * iq3_xs: working scalar and AVX2 dot products * iq3_xs: ARM_NEON dot product - works but extremely slow (10 t/s) * iq3_xs: working Metal implementation * Adding IQ3_M - IQ3_XS mix with mostly Q4_K * iiq3_xs: a 3.4375 bpw variant * iq3_xs: make CUDA work for new version * iq3_xs: make scalar and AVX2 work for new version * iq3_s: make ARM_NEON work with new version * iq3_xs: make new version work on metal Performance is very similar to Q3_K_S * iq3_xs: tiny Metal speed improvement * iq3_xs: tiny Metal speed improvement * Fix stupid warning * Q3_K_XS now uses a mix of IQ3_XS and IQ3_XXS * iq3_xs: rename to iq3_s * iq3_s: make tests pass * Move Q3_K_XS mix to 3.25 bpw * Attempt to fix failing tests * Another attempt to fix the Windows builds * Attempt to fix ROCm * ROCm again * iq3_s: partial fix for QK_K = 64 * iq3_s: make it work on metal for QK_K = 64 Pleasent surprise: the coding was super-block size independent, so all it took was to delete some QK_K == 256 guards. * Will this fix ROCm? --------- Co-authored-by: Iwan Kawrakow --- examples/quantize/quantize.cpp | 2 + ggml-cuda.cu | 171 ++++++++- ggml-metal.m | 33 +- ggml-metal.metal | 304 +++++++++++++++ ggml-quants.c | 674 +++++++++++++++++++++++++++++---- ggml-quants.h | 20 + ggml.c | 31 ++ ggml.h | 2 + llama.cpp | 50 ++- llama.h | 2 + tests/test-backend-ops.cpp | 2 +- tests/test-quantize-fns.cpp | 4 +- 12 files changed, 1211 insertions(+), 84 deletions(-) diff --git a/examples/quantize/quantize.cpp b/examples/quantize/quantize.cpp index 37520857f..ab7e72aaf 100644 --- a/examples/quantize/quantize.cpp +++ b/examples/quantize/quantize.cpp @@ -27,6 +27,8 @@ static const std::vector QUANT_OPTIONS = { { "Q2_K", LLAMA_FTYPE_MOSTLY_Q2_K, " 2.63G, +0.6717 ppl @ LLaMA-v1-7B", }, { "Q2_K_S", LLAMA_FTYPE_MOSTLY_Q2_K_S, " 2.16G, +9.0634 ppl @ LLaMA-v1-7B", }, { "IQ3_XXS",LLAMA_FTYPE_MOSTLY_IQ3_XXS," 3.06 bpw quantization", }, + { "IQ3_S", LLAMA_FTYPE_MOSTLY_IQ3_S, " 3.44 bpw quantization", }, + { "IQ3_M", LLAMA_FTYPE_MOSTLY_IQ3_M, " 3.66 bpw quantization mix", }, { "Q3_K", LLAMA_FTYPE_MOSTLY_Q3_K_M, "alias for Q3_K_M" }, { "Q3_K_XS",LLAMA_FTYPE_MOSTLY_Q3_K_XS,"3-bit extra small quantization" , }, { "Q3_K_S", LLAMA_FTYPE_MOSTLY_Q3_K_S, " 2.75G, +0.5551 ppl @ LLaMA-v1-7B", }, diff --git a/ggml-cuda.cu b/ggml-cuda.cu index b0e454e02..21c612cb7 100644 --- a/ggml-cuda.cu +++ b/ggml-cuda.cu @@ -172,6 +172,7 @@ #endif typedef int8_t int8x4_t __attribute__((ext_vector_type(4))); +typedef uint8_t uint8x4_t __attribute__((ext_vector_type(4))); static __device__ __forceinline__ int __vsubss4(const int a, const int b) { const int8x4_t va = reinterpret_cast(a); const int8x4_t vb = reinterpret_cast(b); @@ -196,6 +197,18 @@ static __device__ __forceinline__ int __vsub4(const int a, const int b) { return __vsubss4(a, b); } +static __device__ __forceinline__ unsigned int __vcmpeq4(unsigned int a, unsigned int b) { + const uint8x4_t& va = reinterpret_cast(a); + const uint8x4_t& vb = reinterpret_cast(b); + unsigned int c; + uint8x4_t& vc = reinterpret_cast(c); +#pragma unroll + for (int i = 0; i < 4; ++i) { + vc[i] = va[i] == vb[i] ? 0xff : 0x00; + } + return c; +} + static __device__ __forceinline__ int __dp4a(const int a, const int b, int c) { #if defined(__gfx906__) || defined(__gfx908__) || defined(__gfx90a__) || defined(__gfx1030__) c = __builtin_amdgcn_sdot4(a, b, c, false); @@ -518,6 +531,17 @@ typedef struct { } block_iq3_xxs; static_assert(sizeof(block_iq3_xxs) == sizeof(ggml_fp16_t) + 3*(QK_K/8), "wrong iq3_xxs block size/padding"); +#define QR3_XS 8 +#define QI3_XS (QK_K / (4*QR3_XS)) +typedef struct { + half d; + uint8_t qs[QK_K/4]; + uint8_t qh[QK_K/32]; + uint8_t signs[QK_K/8]; + uint8_t scales[QK_K/64]; +} block_iq3_s; +static_assert(sizeof(block_iq3_s) == sizeof(ggml_fp16_t) + 27*(QK_K/64), "wrong iq3_s block size/padding"); + #define QR1_S 8 #define QI1_S (QK_K / (4*QR1_S)) typedef struct { @@ -1700,6 +1724,74 @@ static const __device__ uint32_t iq3xxs_grid[256] = { 0x3e1c1c1c, 0x3e1c3404, 0x3e24140c, 0x3e24240c, 0x3e2c0404, 0x3e2c0414, 0x3e2c1424, 0x3e341c04, }; +static const __device__ uint32_t iq3xs_grid[512] = { + 0x04040404, 0x0404040c, 0x04040414, 0x0404042c, 0x0404043e, 0x04040c04, 0x04040c0c, 0x04040c14, + 0x04040c24, 0x04040c34, 0x04041404, 0x0404140c, 0x0404142c, 0x04041c1c, 0x04042404, 0x04042414, + 0x0404242c, 0x0404243e, 0x04042c0c, 0x04042c1c, 0x04043404, 0x04043414, 0x04043e0c, 0x04043e24, + 0x04043e3e, 0x040c0404, 0x040c040c, 0x040c0414, 0x040c0424, 0x040c0c04, 0x040c0c0c, 0x040c0c2c, + 0x040c1404, 0x040c141c, 0x040c143e, 0x040c1c0c, 0x040c1c2c, 0x040c2424, 0x040c340c, 0x040c342c, + 0x040c3e14, 0x04140404, 0x0414040c, 0x0414042c, 0x0414043e, 0x04140c04, 0x04140c1c, 0x04140c34, + 0x0414140c, 0x0414142c, 0x04141c04, 0x04141c24, 0x04142414, 0x0414242c, 0x0414243e, 0x04142c0c, + 0x04142c1c, 0x04143e04, 0x04143e1c, 0x041c041c, 0x041c0c0c, 0x041c0c2c, 0x041c1404, 0x041c1414, + 0x041c1c0c, 0x041c1c1c, 0x041c1c34, 0x041c2424, 0x041c2c04, 0x041c2c14, 0x041c343e, 0x041c3e0c, + 0x041c3e2c, 0x04240404, 0x04240c1c, 0x04240c3e, 0x0424140c, 0x04241424, 0x04241c14, 0x04242404, + 0x0424241c, 0x04242c0c, 0x04243e04, 0x042c0414, 0x042c0424, 0x042c1404, 0x042c1414, 0x042c1434, + 0x042c1c1c, 0x042c240c, 0x042c242c, 0x042c243e, 0x042c3434, 0x042c3e1c, 0x04340434, 0x04340c0c, + 0x04340c1c, 0x04341c0c, 0x04342c14, 0x04343e0c, 0x043e0404, 0x043e0414, 0x043e0424, 0x043e1404, + 0x043e1414, 0x043e1434, 0x043e1c1c, 0x043e2c04, 0x043e2c24, 0x0c040404, 0x0c04040c, 0x0c040414, + 0x0c040424, 0x0c040c04, 0x0c040c0c, 0x0c040c1c, 0x0c040c2c, 0x0c040c3e, 0x0c041404, 0x0c041414, + 0x0c041c0c, 0x0c041c24, 0x0c041c34, 0x0c042c24, 0x0c042c34, 0x0c04340c, 0x0c043e14, 0x0c0c0404, + 0x0c0c040c, 0x0c0c041c, 0x0c0c0434, 0x0c0c0c04, 0x0c0c0c24, 0x0c0c140c, 0x0c0c1c04, 0x0c0c1c1c, + 0x0c0c240c, 0x0c0c2c04, 0x0c0c2c14, 0x0c0c3e04, 0x0c0c3e34, 0x0c140404, 0x0c140c14, 0x0c140c2c, + 0x0c140c3e, 0x0c141404, 0x0c141424, 0x0c141c14, 0x0c142404, 0x0c14241c, 0x0c142c2c, 0x0c143404, + 0x0c143e14, 0x0c1c040c, 0x0c1c0424, 0x0c1c043e, 0x0c1c0c04, 0x0c1c0c1c, 0x0c1c140c, 0x0c1c143e, + 0x0c1c1c04, 0x0c1c1c24, 0x0c1c240c, 0x0c1c3414, 0x0c1c3e04, 0x0c24041c, 0x0c24042c, 0x0c240c14, + 0x0c240c24, 0x0c241c0c, 0x0c241c1c, 0x0c242414, 0x0c242434, 0x0c242c04, 0x0c242c24, 0x0c2c040c, + 0x0c2c0c04, 0x0c2c0c1c, 0x0c2c140c, 0x0c2c1c04, 0x0c2c1c14, 0x0c2c2c0c, 0x0c341404, 0x0c341424, + 0x0c34143e, 0x0c342424, 0x0c342434, 0x0c3e040c, 0x0c3e041c, 0x0c3e0c04, 0x0c3e0c14, 0x0c3e140c, + 0x0c3e1c2c, 0x0c3e240c, 0x0c3e3414, 0x0c3e3e04, 0x14040404, 0x1404040c, 0x1404041c, 0x1404042c, + 0x1404043e, 0x14040c04, 0x14040c14, 0x14040c24, 0x14040c34, 0x1404140c, 0x1404141c, 0x1404143e, + 0x14041c04, 0x14041c14, 0x1404240c, 0x1404241c, 0x1404242c, 0x14042c04, 0x14042c14, 0x1404343e, + 0x14043e04, 0x14043e1c, 0x14043e2c, 0x140c0404, 0x140c0414, 0x140c0c04, 0x140c0c1c, 0x140c0c3e, + 0x140c1414, 0x140c142c, 0x140c1c0c, 0x140c1c24, 0x140c2414, 0x140c2c0c, 0x1414040c, 0x14140424, + 0x1414043e, 0x1414140c, 0x1414141c, 0x14141c04, 0x14141c3e, 0x1414240c, 0x14142c1c, 0x14142c3e, + 0x14143e0c, 0x14143e24, 0x141c0404, 0x141c0414, 0x141c042c, 0x141c0c0c, 0x141c1414, 0x141c1424, + 0x141c1c0c, 0x141c1c1c, 0x141c2414, 0x141c2c04, 0x141c3434, 0x1424040c, 0x1424043e, 0x14241404, + 0x1424141c, 0x14241c14, 0x14241c2c, 0x1424240c, 0x14243e14, 0x14243e2c, 0x142c0424, 0x142c0c0c, + 0x142c1414, 0x142c1c3e, 0x142c2404, 0x142c2c1c, 0x142c3e04, 0x14340404, 0x14340414, 0x1434043e, + 0x1434140c, 0x14342c2c, 0x1434340c, 0x143e042c, 0x143e0c0c, 0x143e1434, 0x143e1c04, 0x143e241c, + 0x143e2c04, 0x1c040414, 0x1c040c0c, 0x1c040c1c, 0x1c040c2c, 0x1c040c3e, 0x1c041414, 0x1c041c0c, + 0x1c041c1c, 0x1c041c2c, 0x1c042414, 0x1c042424, 0x1c04243e, 0x1c042c0c, 0x1c04341c, 0x1c043e0c, + 0x1c0c040c, 0x1c0c041c, 0x1c0c042c, 0x1c0c0c24, 0x1c0c140c, 0x1c0c141c, 0x1c0c2404, 0x1c0c3404, + 0x1c0c3e14, 0x1c0c3e34, 0x1c140404, 0x1c140c14, 0x1c141404, 0x1c141c14, 0x1c141c24, 0x1c142c04, + 0x1c1c040c, 0x1c1c0c04, 0x1c1c0c24, 0x1c1c140c, 0x1c1c141c, 0x1c1c143e, 0x1c1c1c04, 0x1c1c240c, + 0x1c1c241c, 0x1c1c243e, 0x1c1c2c2c, 0x1c1c3e1c, 0x1c24041c, 0x1c240c0c, 0x1c240c34, 0x1c241414, + 0x1c241c0c, 0x1c242c14, 0x1c243404, 0x1c243424, 0x1c2c040c, 0x1c2c0c04, 0x1c2c0c14, 0x1c2c142c, + 0x1c2c1c14, 0x1c2c2424, 0x1c2c2c34, 0x1c2c3e1c, 0x1c340c34, 0x1c34240c, 0x1c3e040c, 0x1c3e041c, + 0x1c3e1404, 0x1c3e1414, 0x1c3e1c2c, 0x24040404, 0x24040424, 0x24040c14, 0x24041404, 0x24041424, + 0x2404143e, 0x24041c14, 0x2404240c, 0x24042c04, 0x24043e04, 0x240c0414, 0x240c043e, 0x240c0c0c, + 0x240c0c1c, 0x240c1414, 0x240c1c04, 0x240c1c2c, 0x240c241c, 0x240c2c0c, 0x240c2c2c, 0x2414040c, + 0x2414041c, 0x24140c04, 0x24140c2c, 0x2414140c, 0x24141c1c, 0x24142404, 0x24142c3e, 0x24143414, + 0x24143e04, 0x241c0424, 0x241c0c0c, 0x241c0c1c, 0x241c1404, 0x241c1414, 0x241c1c0c, 0x241c1c2c, + 0x24240404, 0x24240414, 0x24241424, 0x24241c3e, 0x24242404, 0x24243e0c, 0x242c042c, 0x242c043e, + 0x242c140c, 0x242c3414, 0x24340c1c, 0x24341c24, 0x24343404, 0x243e0c04, 0x243e0c2c, 0x243e1c04, + 0x243e241c, 0x243e2c0c, 0x2c040414, 0x2c040c04, 0x2c040c24, 0x2c041414, 0x2c042404, 0x2c042424, + 0x2c04243e, 0x2c042c14, 0x2c043434, 0x2c043e24, 0x2c0c040c, 0x2c0c041c, 0x2c0c042c, 0x2c0c0c14, + 0x2c0c140c, 0x2c0c1c14, 0x2c0c3e14, 0x2c140404, 0x2c140c0c, 0x2c14141c, 0x2c141c04, 0x2c141c34, + 0x2c142c1c, 0x2c1c0414, 0x2c1c043e, 0x2c1c0c04, 0x2c1c143e, 0x2c1c2424, 0x2c1c2c0c, 0x2c1c342c, + 0x2c1c3e1c, 0x2c24040c, 0x2c240424, 0x2c241404, 0x2c241c14, 0x2c242434, 0x2c2c0c14, 0x2c2c1434, + 0x2c2c2c0c, 0x2c2c2c1c, 0x2c342414, 0x2c3e0414, 0x2c3e0424, 0x2c3e1414, 0x34040c0c, 0x34040c1c, + 0x34040c2c, 0x34041c0c, 0x34041c1c, 0x34043404, 0x340c0404, 0x340c1404, 0x340c143e, 0x340c3424, + 0x34140c14, 0x34141c24, 0x34142414, 0x34142c2c, 0x34143414, 0x34143e04, 0x341c0404, 0x341c0c24, + 0x341c140c, 0x341c2404, 0x3424142c, 0x3424241c, 0x34243414, 0x342c0404, 0x342c041c, 0x342c1c24, + 0x342c3404, 0x3434042c, 0x34342404, 0x343e0c0c, 0x343e0c1c, 0x3e040404, 0x3e040424, 0x3e04043e, + 0x3e041404, 0x3e041414, 0x3e041c34, 0x3e042404, 0x3e042c24, 0x3e043414, 0x3e0c0414, 0x3e0c0c0c, + 0x3e0c1424, 0x3e0c241c, 0x3e0c242c, 0x3e14040c, 0x3e140424, 0x3e140c04, 0x3e140c34, 0x3e14140c, + 0x3e141c04, 0x3e142c0c, 0x3e1c0414, 0x3e1c1c14, 0x3e1c1c2c, 0x3e1c2c1c, 0x3e24040c, 0x3e24042c, + 0x3e240c1c, 0x3e241404, 0x3e242c04, 0x3e2c1414, 0x3e2c2414, 0x3e340414, 0x3e341c0c, 0x3e3e0404, +}; + + static const __device__ uint64_t iq1s_grid[512] = { 0xffffffffffff0101, 0xffffffffff01ff00, 0xffffffffff010100, 0xffffffff00000000, 0xffffffff01ff00ff, 0xffffffff01ff0001, 0xffffffff0101ffff, 0xffffffff0101ff01, @@ -1973,6 +2065,32 @@ static __global__ void dequantize_block_iq3_xxs(const void * __restrict__ vx, ds } +template +static __global__ void dequantize_block_iq3_s(const void * __restrict__ vx, dst_t * __restrict__ yy) { + + const int i = blockIdx.x; + const block_iq3_s * x = (const block_iq3_s *) vx; + + const int tid = threadIdx.x; +#if QK_K == 256 + const int il = tid/8; // 0...3 + const int ib = tid%8; // 0...7 + dst_t * y = yy + i*QK_K + 32*ib + 8*il; + const uint8_t * qs = x[i].qs + 8*ib; + const uint8_t * grid1 = (const uint8_t *)(iq3xs_grid + (qs[2*il+0] | ((x[i].qh[ib] << (8-2*il)) & 256))); + const uint8_t * grid2 = (const uint8_t *)(iq3xs_grid + (qs[2*il+1] | ((x[i].qh[ib] << (7-2*il)) & 256))); + const float d = (float)x[i].d * (0.5f + ((x[i].scales[ib/2] >> 4*(ib%2)) & 0xf)) * 0.5f; + const uint8_t signs = x[i].signs[4*ib + il]; + for (int j = 0; j < 4; ++j) { + y[j+0] = d * grid1[j] * (signs & kmask_iq2xs[j+0] ? -1.f : 1.f); + y[j+4] = d * grid2[j] * (signs & kmask_iq2xs[j+4] ? -1.f : 1.f); + } +#else + assert(false); +#endif + +} + template static __global__ void dequantize_block_iq1_s(const void * __restrict__ vx, dst_t * __restrict__ yy) { @@ -4717,6 +4835,41 @@ static __device__ __forceinline__ float vec_dot_iq3_xxs_q8_1( #endif } +// TODO: don't use lookup table for signs +static __device__ __forceinline__ float vec_dot_iq3_s_q8_1( + const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) { +#if __CUDA_ARCH__ >= MIN_CC_DP4A // lowest compute capability for integer intrinsics +#if QK_K == 256 + const block_iq3_s * bq2 = (const block_iq3_s *) vbq; + + const int ib32 = iqs; + const uint8_t * qs = bq2->qs + 8*ib32; + const int8_t * q8 = bq8_1[ib32].qs; + int sumi = 0; + for (int l = 0; l < 4; ++l) { + const uint32_t * grid1 = iq3xs_grid + (qs[2*l+0] | ((bq2->qh[ib32] << (8 - 2*l)) & 256)); + const uint32_t * grid2 = iq3xs_grid + (qs[2*l+1] | ((bq2->qh[ib32] << (7 - 2*l)) & 256)); + uint32_t signs0 = __vcmpeq4(((bq2->signs[4*ib32+l] & 0xf) * 0x01010101) & 0x08040201, 0x08040201); + uint32_t signs1 = __vcmpeq4(((bq2->signs[4*ib32+l] >> 4) * 0x01010101) & 0x08040201, 0x08040201); + const int grid_l = __vsub4(grid1[0] ^ signs0, signs0); + const int grid_h = __vsub4(grid2[0] ^ signs1, signs1); + sumi = __dp4a(grid_l, *((int *)q8+0), sumi); + sumi = __dp4a(grid_h, *((int *)q8+1), sumi); + q8 += 8; + } + const float d = (float)bq2->d * (0.5f + ((bq2->scales[ib32/2] >> 4*(ib32%2)) & 0xf)) * __low2float(bq8_1[ib32].ds) * 0.5f; + return d * sumi; +#else + assert(false); + return 0.f; +#endif +#else + assert(false); + return 0.f; +#endif +} + + static __device__ __forceinline__ float vec_dot_iq1_s_q8_1( const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) { #if QK_K == 256 @@ -6849,6 +7002,12 @@ static void dequantize_row_iq3_xxs_cuda(const void * vx, dst_t * y, const int k, dequantize_block_iq3_xxs<<>>(vx, y); } +template +static void dequantize_row_iq3_s_cuda(const void * vx, dst_t * y, const int k, cudaStream_t stream) { + const int nb = k / QK_K; + dequantize_block_iq3_s<<>>(vx, y); +} + template static void dequantize_row_iq1_s_cuda(const void * vx, dst_t * y, const int k, cudaStream_t stream) { const int nb = k / QK_K; @@ -6904,6 +7063,8 @@ static to_fp16_cuda_t ggml_get_to_fp16_cuda(ggml_type type) { return dequantize_row_iq1_s_cuda; case GGML_TYPE_IQ4_NL: return dequantize_row_iq4_nl_cuda; + case GGML_TYPE_IQ3_S: + return dequantize_row_iq3_s_cuda; case GGML_TYPE_F32: return convert_unary_cuda; default: @@ -6943,6 +7104,8 @@ static to_fp32_cuda_t ggml_get_to_fp32_cuda(ggml_type type) { return dequantize_row_iq1_s_cuda; case GGML_TYPE_IQ4_NL: return dequantize_row_iq4_nl_cuda; + case GGML_TYPE_IQ3_S: + return dequantize_row_iq3_s_cuda; case GGML_TYPE_F16: return convert_unary_cuda; default: @@ -8688,6 +8851,7 @@ static int64_t get_row_rounding(ggml_type type, const std::array= CC_RDNA2 ? 128 : 64; default: GGML_ASSERT(false); @@ -8713,6 +8877,7 @@ static int64_t get_row_rounding(ggml_type type, const std::array= CC_VOLTA ? 128 : 64; case GGML_TYPE_Q6_K: return 64; @@ -8818,6 +8983,10 @@ static void ggml_cuda_op_mul_mat_vec_q( mul_mat_vec_q_cuda (src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_padded_row_size, src1_ncols, nrows_dst, stream); break; + case GGML_TYPE_IQ3_S: + mul_mat_vec_q_cuda + (src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_padded_row_size, src1_ncols, nrows_dst, stream); + break; default: GGML_ASSERT(false); break; @@ -11541,7 +11710,7 @@ GGML_CALL static bool ggml_backend_cuda_supports_op(ggml_backend_t backend, cons } ggml_type a_type = a->type; if (a_type == GGML_TYPE_IQ2_XXS || a_type == GGML_TYPE_IQ2_XS || a_type == GGML_TYPE_IQ3_XXS || - a_type == GGML_TYPE_IQ1_S || a_type == GGML_TYPE_IQ4_NL) { + a_type == GGML_TYPE_IQ1_S || a_type == GGML_TYPE_IQ4_NL || a_type == GGML_TYPE_IQ3_S) { if (b->ne[1] == 1 && ggml_nrows(b) > 1) { return false; } diff --git a/ggml-metal.m b/ggml-metal.m index 0d4aa4309..ee584cfa7 100644 --- a/ggml-metal.m +++ b/ggml-metal.m @@ -61,6 +61,7 @@ enum ggml_metal_kernel_type { GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ2_XXS, GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ2_XS, GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ3_XXS, + GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ3_S, GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ1_S, GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ4_NL, GGML_METAL_KERNEL_TYPE_GET_ROWS_I32, @@ -85,6 +86,7 @@ enum ggml_metal_kernel_type { GGML_METAL_KERNEL_TYPE_MUL_MV_IQ2_XXS_F32, GGML_METAL_KERNEL_TYPE_MUL_MV_IQ2_XS_F32, GGML_METAL_KERNEL_TYPE_MUL_MV_IQ3_XXS_F32, + GGML_METAL_KERNEL_TYPE_MUL_MV_IQ3_S_F32, GGML_METAL_KERNEL_TYPE_MUL_MV_IQ1_S_F32, GGML_METAL_KERNEL_TYPE_MUL_MV_IQ4_NL_F32, GGML_METAL_KERNEL_TYPE_MUL_MV_ID_F32_F32, @@ -105,6 +107,7 @@ enum ggml_metal_kernel_type { GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ2_XXS_F32, GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ2_XS_F32, GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ3_XXS_F32, + GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ3_S_F32, GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ1_S_F32, GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ4_NL_F32, GGML_METAL_KERNEL_TYPE_MUL_MM_F32_F32, @@ -122,6 +125,7 @@ enum ggml_metal_kernel_type { GGML_METAL_KERNEL_TYPE_MUL_MM_IQ2_XXS_F32, GGML_METAL_KERNEL_TYPE_MUL_MM_IQ2_XS_F32, GGML_METAL_KERNEL_TYPE_MUL_MM_IQ3_XXS_F32, + GGML_METAL_KERNEL_TYPE_MUL_MM_IQ3_S_F32, GGML_METAL_KERNEL_TYPE_MUL_MM_IQ1_S_F32, GGML_METAL_KERNEL_TYPE_MUL_MM_IQ4_NL_F32, GGML_METAL_KERNEL_TYPE_MUL_MM_ID_F32_F32, @@ -139,6 +143,7 @@ enum ggml_metal_kernel_type { GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ2_XXS_F32, GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ2_XS_F32, GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ3_XXS_F32, + GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ3_S_F32, GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ1_S_F32, GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ4_NL_F32, GGML_METAL_KERNEL_TYPE_ROPE_F32, @@ -452,6 +457,7 @@ static struct ggml_metal_context * ggml_metal_init(int n_cb) { GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ2_XXS, get_rows_iq2_xxs, true); GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ2_XS, get_rows_iq2_xs, true); GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ3_XXS, get_rows_iq3_xxs, true); + GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ3_S, get_rows_iq3_s, true); GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ1_S, get_rows_iq1_s, true); GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ4_NL, get_rows_iq4_nl, true); GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_I32, get_rows_i32, true); @@ -476,6 +482,7 @@ static struct ggml_metal_context * ggml_metal_init(int n_cb) { GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_IQ2_XXS_F32, mul_mv_iq2_xxs_f32, ctx->support_simdgroup_reduction); GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_IQ2_XS_F32, mul_mv_iq2_xs_f32, ctx->support_simdgroup_reduction); GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_IQ3_XXS_F32, mul_mv_iq3_xxs_f32, ctx->support_simdgroup_reduction); + GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_IQ3_S_F32, mul_mv_iq3_s_f32, ctx->support_simdgroup_reduction); GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_IQ1_S_F32, mul_mv_iq1_s_f32, ctx->support_simdgroup_reduction); GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_IQ4_NL_F32, mul_mv_iq4_nl_f32, ctx->support_simdgroup_reduction); GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_F32_F32, mul_mv_id_f32_f32, ctx->support_simdgroup_reduction); @@ -496,6 +503,7 @@ static struct ggml_metal_context * ggml_metal_init(int n_cb) { GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ2_XXS_F32, mul_mv_id_iq2_xxs_f32, ctx->support_simdgroup_reduction); GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ2_XS_F32, mul_mv_id_iq2_xs_f32, ctx->support_simdgroup_reduction); GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ3_XXS_F32, mul_mv_id_iq3_xxs_f32, ctx->support_simdgroup_reduction); + GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ3_S_F32, mul_mv_id_iq3_s_f32, ctx->support_simdgroup_reduction); GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ1_S_F32, mul_mv_id_iq1_s_f32, ctx->support_simdgroup_reduction); GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ4_NL_F32, mul_mv_id_iq4_nl_f32, ctx->support_simdgroup_reduction); GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_F32_F32, mul_mm_f32_f32, ctx->support_simdgroup_mm); @@ -513,6 +521,7 @@ static struct ggml_metal_context * ggml_metal_init(int n_cb) { GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_IQ2_XXS_F32, mul_mm_iq2_xxs_f32, ctx->support_simdgroup_mm); GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_IQ2_XS_F32, mul_mm_iq2_xs_f32, ctx->support_simdgroup_mm); GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_IQ3_XXS_F32, mul_mm_iq3_xxs_f32, ctx->support_simdgroup_mm); + GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_IQ3_S_F32, mul_mm_iq3_s_f32, ctx->support_simdgroup_mm); GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_IQ1_S_F32, mul_mm_iq1_s_f32, ctx->support_simdgroup_mm); GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_IQ4_NL_F32, mul_mm_iq4_nl_f32, ctx->support_simdgroup_mm); GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_F32_F32, mul_mm_id_f32_f32, ctx->support_simdgroup_mm); @@ -530,6 +539,7 @@ static struct ggml_metal_context * ggml_metal_init(int n_cb) { GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ2_XXS_F32, mul_mm_id_iq2_xxs_f32, ctx->support_simdgroup_mm); GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ2_XS_F32, mul_mm_id_iq2_xs_f32, ctx->support_simdgroup_mm); GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ3_XXS_F32, mul_mm_id_iq3_xxs_f32, ctx->support_simdgroup_mm); + GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ3_S_F32, mul_mm_id_iq3_s_f32, ctx->support_simdgroup_mm); GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ1_S_F32, mul_mm_id_iq1_s_f32, ctx->support_simdgroup_mm); GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ4_NL_F32, mul_mm_id_iq4_nl_f32, ctx->support_simdgroup_mm); GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_ROPE_F32, rope_f32, true); @@ -1347,6 +1357,7 @@ static bool ggml_metal_graph_compute( case GGML_TYPE_IQ2_XXS: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_IQ2_XXS_F32].pipeline; break; case GGML_TYPE_IQ2_XS: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_IQ2_XS_F32 ].pipeline; break; case GGML_TYPE_IQ3_XXS: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_IQ3_XXS_F32].pipeline; break; + case GGML_TYPE_IQ3_S: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_IQ3_S_F32 ].pipeline; break; case GGML_TYPE_IQ1_S: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_IQ1_S_F32 ].pipeline; break; case GGML_TYPE_IQ4_NL: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_IQ4_NL_F32 ].pipeline; break; default: GGML_ASSERT(false && "MUL MAT-MAT not implemented"); @@ -1483,6 +1494,12 @@ static bool ggml_metal_graph_compute( nth1 = 16; pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_IQ3_XXS_F32].pipeline; } break; + case GGML_TYPE_IQ3_S: + { + nth0 = 4; + nth1 = 16; + pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_IQ3_S_F32].pipeline; + } break; case GGML_TYPE_IQ1_S: { nth0 = 4; @@ -1537,8 +1554,8 @@ static bool ggml_metal_graph_compute( [encoder setThreadgroupMemoryLength:mem_size atIndex:0]; [encoder dispatchThreadgroups:MTLSizeMake((ne01 + 7)/8, ne11, ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)]; } - else if (src0t == GGML_TYPE_IQ3_XXS) { - const int mem_size = 256*4+128; + else if (src0t == GGML_TYPE_IQ3_XXS || src0t == GGML_TYPE_IQ3_S) { + const int mem_size = src0t == GGML_TYPE_IQ3_XXS ? 256*4+128 : 512*4; [encoder setThreadgroupMemoryLength:mem_size atIndex:0]; [encoder dispatchThreadgroups:MTLSizeMake((ne01 + 7)/8, ne11, ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)]; } @@ -1640,6 +1657,7 @@ static bool ggml_metal_graph_compute( case GGML_TYPE_IQ2_XXS: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ2_XXS_F32].pipeline; break; case GGML_TYPE_IQ2_XS: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ2_XS_F32 ].pipeline; break; case GGML_TYPE_IQ3_XXS: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ3_XXS_F32].pipeline; break; + case GGML_TYPE_IQ3_S: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ3_S_F32 ].pipeline; break; case GGML_TYPE_IQ1_S: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ1_S_F32 ].pipeline; break; case GGML_TYPE_IQ4_NL: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ4_NL_F32 ].pipeline; break; default: GGML_ASSERT(false && "MUL_MAT_ID not implemented"); @@ -1779,6 +1797,12 @@ static bool ggml_metal_graph_compute( nth1 = 16; pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ3_XXS_F32].pipeline; } break; + case GGML_TYPE_IQ3_S: + { + nth0 = 4; + nth1 = 16; + pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ3_S_F32].pipeline; + } break; case GGML_TYPE_IQ1_S: { nth0 = 4; @@ -1849,8 +1873,8 @@ static bool ggml_metal_graph_compute( [encoder setThreadgroupMemoryLength:mem_size atIndex:0]; [encoder dispatchThreadgroups:MTLSizeMake((ne21 + 7)/8, _ne1, ne01*ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)]; } - else if (src2t == GGML_TYPE_IQ3_XXS) { - const int mem_size = 256*4+128; + else if (src2t == GGML_TYPE_IQ3_XXS || src2t == GGML_TYPE_IQ3_S) { + const int mem_size = src2t == GGML_TYPE_IQ3_XXS ? 256*4+128 : 512*4; [encoder setThreadgroupMemoryLength:mem_size atIndex:0]; [encoder dispatchThreadgroups:MTLSizeMake((ne21 + 7)/8, _ne1, ne01*ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)]; } @@ -1900,6 +1924,7 @@ static bool ggml_metal_graph_compute( case GGML_TYPE_IQ2_XXS: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ2_XXS].pipeline; break; case GGML_TYPE_IQ2_XS: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ2_XS ].pipeline; break; case GGML_TYPE_IQ3_XXS: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ3_XXS].pipeline; break; + case GGML_TYPE_IQ3_S: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ3_S ].pipeline; break; case GGML_TYPE_IQ1_S: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ1_S ].pipeline; break; case GGML_TYPE_IQ4_NL: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ4_NL ].pipeline; break; case GGML_TYPE_I32: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GET_ROWS_I32 ].pipeline; break; diff --git a/ggml-metal.metal b/ggml-metal.metal index c223a981c..b3bf40539 100644 --- a/ggml-metal.metal +++ b/ggml-metal.metal @@ -2525,6 +2525,20 @@ typedef struct { } block_iq3_xxs; // 98 bytes / block for QK_K = 256, so 3.0625 bpw +// 3.4375 bpw +#if QK_K == 64 +#define IQ3S_N_SCALE 2 +#else +#define IQ3S_N_SCALE QK_K/64 +#endif +typedef struct { + half d; + uint8_t qs[QK_K/4]; + uint8_t qh[QK_K/32]; + uint8_t signs[QK_K/8]; + uint8_t scales[IQ3S_N_SCALE]; +} block_iq3_s; + typedef struct { half d; uint8_t qs[QK_K/8]; @@ -3795,6 +3809,73 @@ constexpr constant static uint32_t iq3xxs_grid[256] = { 0x3e1c1c1c, 0x3e1c3404, 0x3e24140c, 0x3e24240c, 0x3e2c0404, 0x3e2c0414, 0x3e2c1424, 0x3e341c04, }; +constexpr constant static uint32_t iq3xs_grid[512] = { + 0x04040404, 0x0404040c, 0x04040414, 0x0404042c, 0x0404043e, 0x04040c04, 0x04040c0c, 0x04040c14, + 0x04040c24, 0x04040c34, 0x04041404, 0x0404140c, 0x0404142c, 0x04041c1c, 0x04042404, 0x04042414, + 0x0404242c, 0x0404243e, 0x04042c0c, 0x04042c1c, 0x04043404, 0x04043414, 0x04043e0c, 0x04043e24, + 0x04043e3e, 0x040c0404, 0x040c040c, 0x040c0414, 0x040c0424, 0x040c0c04, 0x040c0c0c, 0x040c0c2c, + 0x040c1404, 0x040c141c, 0x040c143e, 0x040c1c0c, 0x040c1c2c, 0x040c2424, 0x040c340c, 0x040c342c, + 0x040c3e14, 0x04140404, 0x0414040c, 0x0414042c, 0x0414043e, 0x04140c04, 0x04140c1c, 0x04140c34, + 0x0414140c, 0x0414142c, 0x04141c04, 0x04141c24, 0x04142414, 0x0414242c, 0x0414243e, 0x04142c0c, + 0x04142c1c, 0x04143e04, 0x04143e1c, 0x041c041c, 0x041c0c0c, 0x041c0c2c, 0x041c1404, 0x041c1414, + 0x041c1c0c, 0x041c1c1c, 0x041c1c34, 0x041c2424, 0x041c2c04, 0x041c2c14, 0x041c343e, 0x041c3e0c, + 0x041c3e2c, 0x04240404, 0x04240c1c, 0x04240c3e, 0x0424140c, 0x04241424, 0x04241c14, 0x04242404, + 0x0424241c, 0x04242c0c, 0x04243e04, 0x042c0414, 0x042c0424, 0x042c1404, 0x042c1414, 0x042c1434, + 0x042c1c1c, 0x042c240c, 0x042c242c, 0x042c243e, 0x042c3434, 0x042c3e1c, 0x04340434, 0x04340c0c, + 0x04340c1c, 0x04341c0c, 0x04342c14, 0x04343e0c, 0x043e0404, 0x043e0414, 0x043e0424, 0x043e1404, + 0x043e1414, 0x043e1434, 0x043e1c1c, 0x043e2c04, 0x043e2c24, 0x0c040404, 0x0c04040c, 0x0c040414, + 0x0c040424, 0x0c040c04, 0x0c040c0c, 0x0c040c1c, 0x0c040c2c, 0x0c040c3e, 0x0c041404, 0x0c041414, + 0x0c041c0c, 0x0c041c24, 0x0c041c34, 0x0c042c24, 0x0c042c34, 0x0c04340c, 0x0c043e14, 0x0c0c0404, + 0x0c0c040c, 0x0c0c041c, 0x0c0c0434, 0x0c0c0c04, 0x0c0c0c24, 0x0c0c140c, 0x0c0c1c04, 0x0c0c1c1c, + 0x0c0c240c, 0x0c0c2c04, 0x0c0c2c14, 0x0c0c3e04, 0x0c0c3e34, 0x0c140404, 0x0c140c14, 0x0c140c2c, + 0x0c140c3e, 0x0c141404, 0x0c141424, 0x0c141c14, 0x0c142404, 0x0c14241c, 0x0c142c2c, 0x0c143404, + 0x0c143e14, 0x0c1c040c, 0x0c1c0424, 0x0c1c043e, 0x0c1c0c04, 0x0c1c0c1c, 0x0c1c140c, 0x0c1c143e, + 0x0c1c1c04, 0x0c1c1c24, 0x0c1c240c, 0x0c1c3414, 0x0c1c3e04, 0x0c24041c, 0x0c24042c, 0x0c240c14, + 0x0c240c24, 0x0c241c0c, 0x0c241c1c, 0x0c242414, 0x0c242434, 0x0c242c04, 0x0c242c24, 0x0c2c040c, + 0x0c2c0c04, 0x0c2c0c1c, 0x0c2c140c, 0x0c2c1c04, 0x0c2c1c14, 0x0c2c2c0c, 0x0c341404, 0x0c341424, + 0x0c34143e, 0x0c342424, 0x0c342434, 0x0c3e040c, 0x0c3e041c, 0x0c3e0c04, 0x0c3e0c14, 0x0c3e140c, + 0x0c3e1c2c, 0x0c3e240c, 0x0c3e3414, 0x0c3e3e04, 0x14040404, 0x1404040c, 0x1404041c, 0x1404042c, + 0x1404043e, 0x14040c04, 0x14040c14, 0x14040c24, 0x14040c34, 0x1404140c, 0x1404141c, 0x1404143e, + 0x14041c04, 0x14041c14, 0x1404240c, 0x1404241c, 0x1404242c, 0x14042c04, 0x14042c14, 0x1404343e, + 0x14043e04, 0x14043e1c, 0x14043e2c, 0x140c0404, 0x140c0414, 0x140c0c04, 0x140c0c1c, 0x140c0c3e, + 0x140c1414, 0x140c142c, 0x140c1c0c, 0x140c1c24, 0x140c2414, 0x140c2c0c, 0x1414040c, 0x14140424, + 0x1414043e, 0x1414140c, 0x1414141c, 0x14141c04, 0x14141c3e, 0x1414240c, 0x14142c1c, 0x14142c3e, + 0x14143e0c, 0x14143e24, 0x141c0404, 0x141c0414, 0x141c042c, 0x141c0c0c, 0x141c1414, 0x141c1424, + 0x141c1c0c, 0x141c1c1c, 0x141c2414, 0x141c2c04, 0x141c3434, 0x1424040c, 0x1424043e, 0x14241404, + 0x1424141c, 0x14241c14, 0x14241c2c, 0x1424240c, 0x14243e14, 0x14243e2c, 0x142c0424, 0x142c0c0c, + 0x142c1414, 0x142c1c3e, 0x142c2404, 0x142c2c1c, 0x142c3e04, 0x14340404, 0x14340414, 0x1434043e, + 0x1434140c, 0x14342c2c, 0x1434340c, 0x143e042c, 0x143e0c0c, 0x143e1434, 0x143e1c04, 0x143e241c, + 0x143e2c04, 0x1c040414, 0x1c040c0c, 0x1c040c1c, 0x1c040c2c, 0x1c040c3e, 0x1c041414, 0x1c041c0c, + 0x1c041c1c, 0x1c041c2c, 0x1c042414, 0x1c042424, 0x1c04243e, 0x1c042c0c, 0x1c04341c, 0x1c043e0c, + 0x1c0c040c, 0x1c0c041c, 0x1c0c042c, 0x1c0c0c24, 0x1c0c140c, 0x1c0c141c, 0x1c0c2404, 0x1c0c3404, + 0x1c0c3e14, 0x1c0c3e34, 0x1c140404, 0x1c140c14, 0x1c141404, 0x1c141c14, 0x1c141c24, 0x1c142c04, + 0x1c1c040c, 0x1c1c0c04, 0x1c1c0c24, 0x1c1c140c, 0x1c1c141c, 0x1c1c143e, 0x1c1c1c04, 0x1c1c240c, + 0x1c1c241c, 0x1c1c243e, 0x1c1c2c2c, 0x1c1c3e1c, 0x1c24041c, 0x1c240c0c, 0x1c240c34, 0x1c241414, + 0x1c241c0c, 0x1c242c14, 0x1c243404, 0x1c243424, 0x1c2c040c, 0x1c2c0c04, 0x1c2c0c14, 0x1c2c142c, + 0x1c2c1c14, 0x1c2c2424, 0x1c2c2c34, 0x1c2c3e1c, 0x1c340c34, 0x1c34240c, 0x1c3e040c, 0x1c3e041c, + 0x1c3e1404, 0x1c3e1414, 0x1c3e1c2c, 0x24040404, 0x24040424, 0x24040c14, 0x24041404, 0x24041424, + 0x2404143e, 0x24041c14, 0x2404240c, 0x24042c04, 0x24043e04, 0x240c0414, 0x240c043e, 0x240c0c0c, + 0x240c0c1c, 0x240c1414, 0x240c1c04, 0x240c1c2c, 0x240c241c, 0x240c2c0c, 0x240c2c2c, 0x2414040c, + 0x2414041c, 0x24140c04, 0x24140c2c, 0x2414140c, 0x24141c1c, 0x24142404, 0x24142c3e, 0x24143414, + 0x24143e04, 0x241c0424, 0x241c0c0c, 0x241c0c1c, 0x241c1404, 0x241c1414, 0x241c1c0c, 0x241c1c2c, + 0x24240404, 0x24240414, 0x24241424, 0x24241c3e, 0x24242404, 0x24243e0c, 0x242c042c, 0x242c043e, + 0x242c140c, 0x242c3414, 0x24340c1c, 0x24341c24, 0x24343404, 0x243e0c04, 0x243e0c2c, 0x243e1c04, + 0x243e241c, 0x243e2c0c, 0x2c040414, 0x2c040c04, 0x2c040c24, 0x2c041414, 0x2c042404, 0x2c042424, + 0x2c04243e, 0x2c042c14, 0x2c043434, 0x2c043e24, 0x2c0c040c, 0x2c0c041c, 0x2c0c042c, 0x2c0c0c14, + 0x2c0c140c, 0x2c0c1c14, 0x2c0c3e14, 0x2c140404, 0x2c140c0c, 0x2c14141c, 0x2c141c04, 0x2c141c34, + 0x2c142c1c, 0x2c1c0414, 0x2c1c043e, 0x2c1c0c04, 0x2c1c143e, 0x2c1c2424, 0x2c1c2c0c, 0x2c1c342c, + 0x2c1c3e1c, 0x2c24040c, 0x2c240424, 0x2c241404, 0x2c241c14, 0x2c242434, 0x2c2c0c14, 0x2c2c1434, + 0x2c2c2c0c, 0x2c2c2c1c, 0x2c342414, 0x2c3e0414, 0x2c3e0424, 0x2c3e1414, 0x34040c0c, 0x34040c1c, + 0x34040c2c, 0x34041c0c, 0x34041c1c, 0x34043404, 0x340c0404, 0x340c1404, 0x340c143e, 0x340c3424, + 0x34140c14, 0x34141c24, 0x34142414, 0x34142c2c, 0x34143414, 0x34143e04, 0x341c0404, 0x341c0c24, + 0x341c140c, 0x341c2404, 0x3424142c, 0x3424241c, 0x34243414, 0x342c0404, 0x342c041c, 0x342c1c24, + 0x342c3404, 0x3434042c, 0x34342404, 0x343e0c0c, 0x343e0c1c, 0x3e040404, 0x3e040424, 0x3e04043e, + 0x3e041404, 0x3e041414, 0x3e041c34, 0x3e042404, 0x3e042c24, 0x3e043414, 0x3e0c0414, 0x3e0c0c0c, + 0x3e0c1424, 0x3e0c241c, 0x3e0c242c, 0x3e14040c, 0x3e140424, 0x3e140c04, 0x3e140c34, 0x3e14140c, + 0x3e141c04, 0x3e142c0c, 0x3e1c0414, 0x3e1c1c14, 0x3e1c1c2c, 0x3e1c2c1c, 0x3e24040c, 0x3e24042c, + 0x3e240c1c, 0x3e241404, 0x3e242c04, 0x3e2c1414, 0x3e2c2414, 0x3e340414, 0x3e341c0c, 0x3e3e0404, +}; + #define NGRID_IQ1S 512 constexpr constant static uint64_t iq1s_grid[NGRID_IQ1S] = { 0xffffffffffff0101, 0xffffffffff01ff00, 0xffffffffff010100, 0xffffffff00000000, @@ -4361,6 +4442,136 @@ kernel void kernel_mul_mv_iq3_xxs_f32( kernel_mul_mv_iq3_xxs_f32_impl(src0, src1, dst, ne00, ne01, ne02, ne10, ne12, ne0, ne1, r2, r3, shared_values, tgpig, tiisg, sgitg); } +void kernel_mul_mv_iq3_s_f32_impl( + device const void * src0, + device const float * src1, + device float * dst, + constant int64_t & ne00, + constant int64_t & ne01, + constant int64_t & ne02, + constant int64_t & ne10, + constant int64_t & ne12, + constant int64_t & ne0, + constant int64_t & ne1, + constant uint & r2, + constant uint & r3, + threadgroup int8_t * shared_values [[threadgroup(0)]], + uint3 tgpig[[threadgroup_position_in_grid]], + uint tiisg[[thread_index_in_simdgroup]], + uint sgitg[[simdgroup_index_in_threadgroup]]) { + + const int nb = ne00/QK_K; + const int r0 = tgpig.x; + const int r1 = tgpig.y; + const int im = tgpig.z; + + const int first_row = (r0 * N_SIMDGROUP + sgitg) * N_DST; + const int ib_row = first_row * nb; + + const uint i12 = im%ne12; + const uint i13 = im/ne12; + + const uint offset0 = (i12/r2)*(nb*ne01) + (i13/r3)*(nb*ne01*ne02); + + device const block_iq3_s * x = (device const block_iq3_s *) src0 + ib_row + offset0; + device const float * y = (device const float *) src1 + r1*ne10 + im*ne00*ne1; + + float yl[32]; + float sumf[N_DST]={0.f}, all_sum; + + const int nb32 = nb * (QK_K / 32); + + threadgroup uint32_t * values = (threadgroup uint32_t *)shared_values; + { + int nval = 8; + int pos = (32*sgitg + tiisg)*nval; + for (int i = 0; i < nval; ++i) values[pos + i] = iq3xs_grid[pos + i]; + threadgroup_barrier(mem_flags::mem_threadgroup); + } + + const int ix = tiisg; + + device const float * y4 = y + 32 * ix; + + for (int ib32 = ix; ib32 < nb32; ib32 += 32) { + + for (int i = 0; i < 32; ++i) { + yl[i] = y4[i]; + } + + const int ibl = ib32 / (QK_K / 32); + const int ib = ib32 % (QK_K / 32); + + device const block_iq3_s * xr = x + ibl; + device const uint8_t * qs = xr->qs + 8 * ib; + device const uint8_t * qh = xr->qh + ib; + device const uint8_t * sc = xr->scales + (ib/2); + device const uint8_t * signs = xr->signs + 4 * ib; + device const half * dh = &xr->d; + + for (int row = 0; row < N_DST; row++) { + + const float db = dh[0]; + const float d = db * (0.5f + ((sc[0] >> 4*(ib%2)) & 0xf)); + + float2 sum = {0}; + for (int l = 0; l < 4; ++l) { + const threadgroup uint8_t * grid1 = (const threadgroup uint8_t *)(values + (qs[2*l+0] | ((qh[0] << (8-2*l)) & 256))); + const threadgroup uint8_t * grid2 = (const threadgroup uint8_t *)(values + (qs[2*l+1] | ((qh[0] << (7-2*l)) & 256))); + for (int j = 0; j < 4; ++j) { + sum[0] += yl[8*l + j + 0] * grid1[j] * select(1, -1, signs[l] & kmask_iq2xs[j+0]); + sum[1] += yl[8*l + j + 4] * grid2[j] * select(1, -1, signs[l] & kmask_iq2xs[j+4]); + } + } + sumf[row] += d * (sum[0] + sum[1]); + + dh += nb*sizeof(block_iq3_s)/2; + qs += nb*sizeof(block_iq3_s); + qh += nb*sizeof(block_iq3_s); + sc += nb*sizeof(block_iq3_s); + signs += nb*sizeof(block_iq3_s); + } + + y4 += 32 * 32; + } + + for (int row = 0; row < N_DST; ++row) { + all_sum = simd_sum(sumf[row]); + if (tiisg == 0) { + dst[r1*ne0 + im*ne0*ne1 + first_row + row] = all_sum * 0.5f; + } + } +} + +[[host_name("kernel_mul_mv_iq3_s_f32")]] +kernel void kernel_mul_mv_iq3_s_f32( + device const void * src0, + device const float * src1, + device float * dst, + constant int64_t & ne00, + constant int64_t & ne01, + constant int64_t & ne02, + constant uint64_t & nb00, + constant uint64_t & nb01, + constant uint64_t & nb02, + constant int64_t & ne10, + constant int64_t & ne11, + constant int64_t & ne12, + constant uint64_t & nb10, + constant uint64_t & nb11, + constant uint64_t & nb12, + constant int64_t & ne0, + constant int64_t & ne1, + constant uint & r2, + constant uint & r3, + threadgroup int8_t * shared_values [[threadgroup(0)]], + uint3 tgpig[[threadgroup_position_in_grid]], + uint tiisg[[thread_index_in_simdgroup]], + uint sgitg[[simdgroup_index_in_threadgroup]]) { + + kernel_mul_mv_iq3_s_f32_impl(src0, src1, dst, ne00, ne01, ne02, ne10, ne12, ne0, ne1, r2, r3, shared_values, tgpig, tiisg, sgitg); +} + void kernel_mul_mv_iq1_s_f32_impl( device const void * src0, device const float * src1, @@ -4952,6 +5163,31 @@ void dequantize_iq3_xxs(device const block_iq3_xxs * xb, short il, thread type4x } } +template +void dequantize_iq3_s(device const block_iq3_s * xb, short il, thread type4x4 & reg) { + // il is 0...15 for QK_K = 256 => index of block of 32 is il/2 + const float d = xb->d; + const int ib32 = il/2; + il = il%2; + // il = 0 or 1. il = 0 processes the first 16 quants in a block of 32, il = 1 the second 16 + device const uint8_t * qs = xb->qs + 8*ib32; + device const uint8_t * signs = xb->signs + 4*ib32 + 2*il; + const uint8_t qh = xb->qh[ib32] >> 4*il; + const float dl = d * (0.5f + ((xb->scales[ib32/2] >> 4*(ib32%2)) & 0xf)) * 0.5f; + constant uint8_t * grid1 = (constant uint8_t *)(iq3xs_grid + (qs[4*il+0] | ((qh << 8) & 256))); + constant uint8_t * grid2 = (constant uint8_t *)(iq3xs_grid + (qs[4*il+1] | ((qh << 7) & 256))); + for (int i = 0; i < 4; ++i) { + reg[0][i] = dl * grid1[i] * select(1, -1, signs[0] & kmask_iq2xs[i+0]); + reg[1][i] = dl * grid2[i] * select(1, -1, signs[0] & kmask_iq2xs[i+4]); + } + grid1 = (constant uint8_t *)(iq3xs_grid + (qs[4*il+2] | ((qh << 6) & 256))); + grid2 = (constant uint8_t *)(iq3xs_grid + (qs[4*il+3] | ((qh << 5) & 256))); + for (int i = 0; i < 4; ++i) { + reg[2][i] = dl * grid1[i] * select(1, -1, signs[1] & kmask_iq2xs[i+0]); + reg[3][i] = dl * grid2[i] * select(1, -1, signs[1] & kmask_iq2xs[i+4]); + } +} + template void dequantize_iq1_s(device const block_iq1_s * xb, short il, thread type4x4 & reg) { // il is 0...15 for QK_K = 256 => index of block of 32 is il/2 @@ -5525,6 +5761,7 @@ template [[host_name("kernel_get_rows_q6_K")]] kernel get_rows_t kernel_get_rows template [[host_name("kernel_get_rows_iq2_xxs")]] kernel get_rows_t kernel_get_rows; template [[host_name("kernel_get_rows_iq2_xs")]] kernel get_rows_t kernel_get_rows; template [[host_name("kernel_get_rows_iq3_xxs")]] kernel get_rows_t kernel_get_rows; +template [[host_name("kernel_get_rows_iq3_s")]] kernel get_rows_t kernel_get_rows; template [[host_name("kernel_get_rows_iq1_s")]] kernel get_rows_t kernel_get_rows; template [[host_name("kernel_get_rows_iq4_nl")]] kernel get_rows_t kernel_get_rows; @@ -5566,6 +5803,7 @@ template [[host_name("kernel_mul_mm_q6_K_f32")]] kernel mat_mm_t kernel_mul_mm; template [[host_name("kernel_mul_mm_iq2_xs_f32")]] kernel mat_mm_t kernel_mul_mm; template [[host_name("kernel_mul_mm_iq3_xxs_f32")]] kernel mat_mm_t kernel_mul_mm; +template [[host_name("kernel_mul_mm_iq3_s_f32")]] kernel mat_mm_t kernel_mul_mm; template [[host_name("kernel_mul_mm_iq1_s_f32")]] kernel mat_mm_t kernel_mul_mm; template [[host_name("kernel_mul_mm_iq4_nl_f32")]] kernel mat_mm_t kernel_mul_mm; @@ -5619,6 +5857,7 @@ template [[host_name("kernel_mul_mm_id_q6_K_f32")]] kernel mat_mm_id_t kernel_mu template [[host_name("kernel_mul_mm_id_iq2_xxs_f32")]] kernel mat_mm_id_t kernel_mul_mm_id; template [[host_name("kernel_mul_mm_id_iq2_xs_f32")]] kernel mat_mm_id_t kernel_mul_mm_id; template [[host_name("kernel_mul_mm_id_iq3_xxs_f32")]] kernel mat_mm_id_t kernel_mul_mm_id; +template [[host_name("kernel_mul_mm_id_iq3_s_f32")]] kernel mat_mm_id_t kernel_mul_mm_id; template [[host_name("kernel_mul_mm_id_iq1_s_f32")]] kernel mat_mm_id_t kernel_mul_mm_id; template [[host_name("kernel_mul_mm_id_iq4_nl_f32")]] kernel mat_mm_id_t kernel_mul_mm_id; @@ -6589,6 +6828,71 @@ kernel void kernel_mul_mv_id_iq3_xxs_f32( sgitg); } +[[host_name("kernel_mul_mv_id_iq3_s_f32")]] +kernel void kernel_mul_mv_id_iq3_s_f32( + device const char * ids, + device const char * src1, + device float * dst, + constant uint64_t & nbi1, + constant int64_t & ne00, + constant int64_t & ne01, + constant int64_t & ne02, + constant uint64_t & nb00, + constant uint64_t & nb01, + constant uint64_t & nb02, + constant int64_t & ne10, + constant int64_t & ne11, + constant int64_t & ne12, + constant int64_t & ne13, + constant uint64_t & nb10, + constant uint64_t & nb11, + constant uint64_t & nb12, + constant int64_t & ne0, + constant int64_t & ne1, + constant uint64_t & nb1, + constant uint & r2, + constant uint & r3, + constant int & idx, + device const char * src00, + device const char * src01, + device const char * src02, + device const char * src03, + device const char * src04, + device const char * src05, + device const char * src06, + device const char * src07, + threadgroup int8_t * shared_values [[threadgroup(0)]], + uint3 tgpig[[threadgroup_position_in_grid]], + uint tiitg[[thread_index_in_threadgroup]], + uint tiisg[[thread_index_in_simdgroup]], + uint sgitg[[simdgroup_index_in_threadgroup]]) { + device const char * src0[8] = {src00, src01, src02, src03, src04, src05, src06, src07}; + + const int64_t bid = tgpig.z/(ne12*ne13); + + tgpig.z = tgpig.z%(ne12*ne13); + + const int32_t id = ((device int32_t *) (ids + bid*nbi1))[idx]; + + kernel_mul_mv_iq3_s_f32_impl( + src0[id], + (device const float *) (src1 + bid*nb11), + dst + bid*ne0, + ne00, + ne01, + ne02, + ne10, + ne12, + ne0, + ne1, + r2, + r3, + shared_values, + tgpig, + tiisg, + sgitg); +} + [[host_name("kernel_mul_mv_id_iq1_s_f32")]] kernel void kernel_mul_mv_id_iq1_s_f32( device const char * ids, diff --git a/ggml-quants.c b/ggml-quants.c index b15977f53..5c5f2ce1b 100644 --- a/ggml-quants.c +++ b/ggml-quants.c @@ -3505,6 +3505,73 @@ static const uint32_t iq3xxs_grid[256] = { 0x3e1c1c1c, 0x3e1c3404, 0x3e24140c, 0x3e24240c, 0x3e2c0404, 0x3e2c0414, 0x3e2c1424, 0x3e341c04, }; +static const uint32_t iq3xs_grid[512] = { + 0x04040404, 0x0404040c, 0x04040414, 0x0404042c, 0x0404043e, 0x04040c04, 0x04040c0c, 0x04040c14, + 0x04040c24, 0x04040c34, 0x04041404, 0x0404140c, 0x0404142c, 0x04041c1c, 0x04042404, 0x04042414, + 0x0404242c, 0x0404243e, 0x04042c0c, 0x04042c1c, 0x04043404, 0x04043414, 0x04043e0c, 0x04043e24, + 0x04043e3e, 0x040c0404, 0x040c040c, 0x040c0414, 0x040c0424, 0x040c0c04, 0x040c0c0c, 0x040c0c2c, + 0x040c1404, 0x040c141c, 0x040c143e, 0x040c1c0c, 0x040c1c2c, 0x040c2424, 0x040c340c, 0x040c342c, + 0x040c3e14, 0x04140404, 0x0414040c, 0x0414042c, 0x0414043e, 0x04140c04, 0x04140c1c, 0x04140c34, + 0x0414140c, 0x0414142c, 0x04141c04, 0x04141c24, 0x04142414, 0x0414242c, 0x0414243e, 0x04142c0c, + 0x04142c1c, 0x04143e04, 0x04143e1c, 0x041c041c, 0x041c0c0c, 0x041c0c2c, 0x041c1404, 0x041c1414, + 0x041c1c0c, 0x041c1c1c, 0x041c1c34, 0x041c2424, 0x041c2c04, 0x041c2c14, 0x041c343e, 0x041c3e0c, + 0x041c3e2c, 0x04240404, 0x04240c1c, 0x04240c3e, 0x0424140c, 0x04241424, 0x04241c14, 0x04242404, + 0x0424241c, 0x04242c0c, 0x04243e04, 0x042c0414, 0x042c0424, 0x042c1404, 0x042c1414, 0x042c1434, + 0x042c1c1c, 0x042c240c, 0x042c242c, 0x042c243e, 0x042c3434, 0x042c3e1c, 0x04340434, 0x04340c0c, + 0x04340c1c, 0x04341c0c, 0x04342c14, 0x04343e0c, 0x043e0404, 0x043e0414, 0x043e0424, 0x043e1404, + 0x043e1414, 0x043e1434, 0x043e1c1c, 0x043e2c04, 0x043e2c24, 0x0c040404, 0x0c04040c, 0x0c040414, + 0x0c040424, 0x0c040c04, 0x0c040c0c, 0x0c040c1c, 0x0c040c2c, 0x0c040c3e, 0x0c041404, 0x0c041414, + 0x0c041c0c, 0x0c041c24, 0x0c041c34, 0x0c042c24, 0x0c042c34, 0x0c04340c, 0x0c043e14, 0x0c0c0404, + 0x0c0c040c, 0x0c0c041c, 0x0c0c0434, 0x0c0c0c04, 0x0c0c0c24, 0x0c0c140c, 0x0c0c1c04, 0x0c0c1c1c, + 0x0c0c240c, 0x0c0c2c04, 0x0c0c2c14, 0x0c0c3e04, 0x0c0c3e34, 0x0c140404, 0x0c140c14, 0x0c140c2c, + 0x0c140c3e, 0x0c141404, 0x0c141424, 0x0c141c14, 0x0c142404, 0x0c14241c, 0x0c142c2c, 0x0c143404, + 0x0c143e14, 0x0c1c040c, 0x0c1c0424, 0x0c1c043e, 0x0c1c0c04, 0x0c1c0c1c, 0x0c1c140c, 0x0c1c143e, + 0x0c1c1c04, 0x0c1c1c24, 0x0c1c240c, 0x0c1c3414, 0x0c1c3e04, 0x0c24041c, 0x0c24042c, 0x0c240c14, + 0x0c240c24, 0x0c241c0c, 0x0c241c1c, 0x0c242414, 0x0c242434, 0x0c242c04, 0x0c242c24, 0x0c2c040c, + 0x0c2c0c04, 0x0c2c0c1c, 0x0c2c140c, 0x0c2c1c04, 0x0c2c1c14, 0x0c2c2c0c, 0x0c341404, 0x0c341424, + 0x0c34143e, 0x0c342424, 0x0c342434, 0x0c3e040c, 0x0c3e041c, 0x0c3e0c04, 0x0c3e0c14, 0x0c3e140c, + 0x0c3e1c2c, 0x0c3e240c, 0x0c3e3414, 0x0c3e3e04, 0x14040404, 0x1404040c, 0x1404041c, 0x1404042c, + 0x1404043e, 0x14040c04, 0x14040c14, 0x14040c24, 0x14040c34, 0x1404140c, 0x1404141c, 0x1404143e, + 0x14041c04, 0x14041c14, 0x1404240c, 0x1404241c, 0x1404242c, 0x14042c04, 0x14042c14, 0x1404343e, + 0x14043e04, 0x14043e1c, 0x14043e2c, 0x140c0404, 0x140c0414, 0x140c0c04, 0x140c0c1c, 0x140c0c3e, + 0x140c1414, 0x140c142c, 0x140c1c0c, 0x140c1c24, 0x140c2414, 0x140c2c0c, 0x1414040c, 0x14140424, + 0x1414043e, 0x1414140c, 0x1414141c, 0x14141c04, 0x14141c3e, 0x1414240c, 0x14142c1c, 0x14142c3e, + 0x14143e0c, 0x14143e24, 0x141c0404, 0x141c0414, 0x141c042c, 0x141c0c0c, 0x141c1414, 0x141c1424, + 0x141c1c0c, 0x141c1c1c, 0x141c2414, 0x141c2c04, 0x141c3434, 0x1424040c, 0x1424043e, 0x14241404, + 0x1424141c, 0x14241c14, 0x14241c2c, 0x1424240c, 0x14243e14, 0x14243e2c, 0x142c0424, 0x142c0c0c, + 0x142c1414, 0x142c1c3e, 0x142c2404, 0x142c2c1c, 0x142c3e04, 0x14340404, 0x14340414, 0x1434043e, + 0x1434140c, 0x14342c2c, 0x1434340c, 0x143e042c, 0x143e0c0c, 0x143e1434, 0x143e1c04, 0x143e241c, + 0x143e2c04, 0x1c040414, 0x1c040c0c, 0x1c040c1c, 0x1c040c2c, 0x1c040c3e, 0x1c041414, 0x1c041c0c, + 0x1c041c1c, 0x1c041c2c, 0x1c042414, 0x1c042424, 0x1c04243e, 0x1c042c0c, 0x1c04341c, 0x1c043e0c, + 0x1c0c040c, 0x1c0c041c, 0x1c0c042c, 0x1c0c0c24, 0x1c0c140c, 0x1c0c141c, 0x1c0c2404, 0x1c0c3404, + 0x1c0c3e14, 0x1c0c3e34, 0x1c140404, 0x1c140c14, 0x1c141404, 0x1c141c14, 0x1c141c24, 0x1c142c04, + 0x1c1c040c, 0x1c1c0c04, 0x1c1c0c24, 0x1c1c140c, 0x1c1c141c, 0x1c1c143e, 0x1c1c1c04, 0x1c1c240c, + 0x1c1c241c, 0x1c1c243e, 0x1c1c2c2c, 0x1c1c3e1c, 0x1c24041c, 0x1c240c0c, 0x1c240c34, 0x1c241414, + 0x1c241c0c, 0x1c242c14, 0x1c243404, 0x1c243424, 0x1c2c040c, 0x1c2c0c04, 0x1c2c0c14, 0x1c2c142c, + 0x1c2c1c14, 0x1c2c2424, 0x1c2c2c34, 0x1c2c3e1c, 0x1c340c34, 0x1c34240c, 0x1c3e040c, 0x1c3e041c, + 0x1c3e1404, 0x1c3e1414, 0x1c3e1c2c, 0x24040404, 0x24040424, 0x24040c14, 0x24041404, 0x24041424, + 0x2404143e, 0x24041c14, 0x2404240c, 0x24042c04, 0x24043e04, 0x240c0414, 0x240c043e, 0x240c0c0c, + 0x240c0c1c, 0x240c1414, 0x240c1c04, 0x240c1c2c, 0x240c241c, 0x240c2c0c, 0x240c2c2c, 0x2414040c, + 0x2414041c, 0x24140c04, 0x24140c2c, 0x2414140c, 0x24141c1c, 0x24142404, 0x24142c3e, 0x24143414, + 0x24143e04, 0x241c0424, 0x241c0c0c, 0x241c0c1c, 0x241c1404, 0x241c1414, 0x241c1c0c, 0x241c1c2c, + 0x24240404, 0x24240414, 0x24241424, 0x24241c3e, 0x24242404, 0x24243e0c, 0x242c042c, 0x242c043e, + 0x242c140c, 0x242c3414, 0x24340c1c, 0x24341c24, 0x24343404, 0x243e0c04, 0x243e0c2c, 0x243e1c04, + 0x243e241c, 0x243e2c0c, 0x2c040414, 0x2c040c04, 0x2c040c24, 0x2c041414, 0x2c042404, 0x2c042424, + 0x2c04243e, 0x2c042c14, 0x2c043434, 0x2c043e24, 0x2c0c040c, 0x2c0c041c, 0x2c0c042c, 0x2c0c0c14, + 0x2c0c140c, 0x2c0c1c14, 0x2c0c3e14, 0x2c140404, 0x2c140c0c, 0x2c14141c, 0x2c141c04, 0x2c141c34, + 0x2c142c1c, 0x2c1c0414, 0x2c1c043e, 0x2c1c0c04, 0x2c1c143e, 0x2c1c2424, 0x2c1c2c0c, 0x2c1c342c, + 0x2c1c3e1c, 0x2c24040c, 0x2c240424, 0x2c241404, 0x2c241c14, 0x2c242434, 0x2c2c0c14, 0x2c2c1434, + 0x2c2c2c0c, 0x2c2c2c1c, 0x2c342414, 0x2c3e0414, 0x2c3e0424, 0x2c3e1414, 0x34040c0c, 0x34040c1c, + 0x34040c2c, 0x34041c0c, 0x34041c1c, 0x34043404, 0x340c0404, 0x340c1404, 0x340c143e, 0x340c3424, + 0x34140c14, 0x34141c24, 0x34142414, 0x34142c2c, 0x34143414, 0x34143e04, 0x341c0404, 0x341c0c24, + 0x341c140c, 0x341c2404, 0x3424142c, 0x3424241c, 0x34243414, 0x342c0404, 0x342c041c, 0x342c1c24, + 0x342c3404, 0x3434042c, 0x34342404, 0x343e0c0c, 0x343e0c1c, 0x3e040404, 0x3e040424, 0x3e04043e, + 0x3e041404, 0x3e041414, 0x3e041c34, 0x3e042404, 0x3e042c24, 0x3e043414, 0x3e0c0414, 0x3e0c0c0c, + 0x3e0c1424, 0x3e0c241c, 0x3e0c242c, 0x3e14040c, 0x3e140424, 0x3e140c04, 0x3e140c34, 0x3e14140c, + 0x3e141c04, 0x3e142c0c, 0x3e1c0414, 0x3e1c1c14, 0x3e1c1c2c, 0x3e1c2c1c, 0x3e24040c, 0x3e24042c, + 0x3e240c1c, 0x3e241404, 0x3e242c04, 0x3e2c1414, 0x3e2c2414, 0x3e340414, 0x3e341c0c, 0x3e3e0404, +}; + #define NGRID_IQ2XXS 512 static const uint64_t iq1s_grid[NGRID_IQ2XXS] = { 0xffffffffffff0101, 0xffffffffff01ff00, 0xffffffffff010100, 0xffffffff00000000, @@ -3736,6 +3803,49 @@ void dequantize_row_iq3_xxs(const block_iq3_xxs * restrict x, float * restrict y } } +// ====================== 3.3125 bpw (de)-quantization + +void dequantize_row_iq3_s(const block_iq3_s * restrict x, float * restrict y, int k) { + assert(k % QK_K == 0); + const int nb = k / QK_K; + + for (int i = 0; i < nb; i++) { + + const float d = GGML_FP16_TO_FP32(x[i].d); + const uint8_t * qs = x[i].qs; + const uint8_t * qh = x[i].qh; + const uint8_t * signs = x[i].signs; + + for (int ib32 = 0; ib32 < QK_K/32; ib32 += 2) { + const float db1 = d * (0.5f + (x[i].scales[ib32/2] & 0xf)) * 0.5f; + const float db2 = d * (0.5f + (x[i].scales[ib32/2] >> 4)) * 0.5f; + for (int l = 0; l < 4; ++l) { + const uint8_t * grid1 = (const uint8_t *)(iq3xs_grid + (qs[2*l+0] | ((qh[0] << (8-2*l)) & 256))); + const uint8_t * grid2 = (const uint8_t *)(iq3xs_grid + (qs[2*l+1] | ((qh[0] << (7-2*l)) & 256))); + for (int j = 0; j < 4; ++j) { + y[j+0] = db1 * grid1[j] * (signs[l] & kmask_iq2xs[j+0] ? -1.f : 1.f); + y[j+4] = db1 * grid2[j] * (signs[l] & kmask_iq2xs[j+4] ? -1.f : 1.f); + } + y += 8; + } + qs += 8; + signs += 4; + for (int l = 0; l < 4; ++l) { + const uint8_t * grid1 = (const uint8_t *)(iq3xs_grid + (qs[2*l+0] | ((qh[1] << (8-2*l)) & 256))); + const uint8_t * grid2 = (const uint8_t *)(iq3xs_grid + (qs[2*l+1] | ((qh[1] << (7-2*l)) & 256))); + for (int j = 0; j < 4; ++j) { + y[j+0] = db2 * grid1[j] * (signs[l] & kmask_iq2xs[j+0] ? -1.f : 1.f); + y[j+4] = db2 * grid2[j] * (signs[l] & kmask_iq2xs[j+4] ? -1.f : 1.f); + } + y += 8; + } + qh += 2; + qs += 8; + signs += 4; + } + } +} + // ====================== 1.5625 bpw (de)-quantization void dequantize_row_iq1_s(const block_iq1_s * restrict x, float * restrict y, int k) { @@ -8806,6 +8916,7 @@ void ggml_vec_dot_q6_K_q8_K(int n, float * restrict s, size_t bs, const void * r #endif +#if defined (__AVX2__) || defined (__ARM_NEON) static const int8_t keven_signs_q2xs[1024] = { 1, 1, 1, 1, 1, 1, 1, 1, -1, 1, 1, 1, 1, 1, 1, -1, 1, -1, 1, 1, 1, 1, 1, -1, -1, -1, 1, 1, 1, 1, 1, 1, 1, 1, -1, 1, 1, 1, 1, -1, -1, 1, -1, 1, 1, 1, 1, 1, 1, -1, -1, 1, 1, 1, 1, 1, -1, -1, -1, 1, 1, 1, 1, -1, @@ -8840,6 +8951,7 @@ static const int8_t keven_signs_q2xs[1024] = { 1, 1, 1, -1, -1, -1, -1, 1, -1, 1, 1, -1, -1, -1, -1, -1, 1, -1, 1, -1, -1, -1, -1, -1, -1, -1, 1, -1, -1, -1, -1, 1, 1, 1, -1, -1, -1, -1, -1, -1, -1, 1, -1, -1, -1, -1, -1, 1, 1, -1, -1, -1, -1, -1, -1, 1, -1, -1, -1, -1, -1, -1, -1, -1, }; +#endif void ggml_vec_dot_iq2_xxs_q8_K(int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) { assert(n % QK_K == 0); @@ -9327,6 +9439,202 @@ void ggml_vec_dot_iq3_xxs_q8_K(int n, float * restrict s, size_t bs, const void #endif } +void ggml_vec_dot_iq3_s_q8_K (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { + assert(n % QK_K == 0); + assert(nrc == 1); + UNUSED(nrc); + UNUSED(bx); + UNUSED(by); + UNUSED(bs); + + const block_iq3_s * restrict x = vx; + const block_q8_K * restrict y = vy; + + const int nb = n / QK_K; + +#if defined(__ARM_NEON) + + static const uint8_t k_mask1[32] = {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03 + }; + + static const uint8_t k_mask2[16] = {0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80,}; + + const uint8x16x2_t mask1 = vld1q_u8_x2(k_mask1); + const uint8x16_t mask2 = vld1q_u8(k_mask2); + + uint8x16x2_t vs; + ggml_int8x16x4_t q3s; + ggml_int8x16x4_t q8b; + + float sumf = 0; + for (int i = 0; i < nb; ++i) { + const float d = GGML_FP16_TO_FP32(x[i].d) * y[i].d; + const uint8_t * restrict qs = x[i].qs; + const uint8_t * restrict qh = x[i].qh; + const uint16_t * restrict signs = (const uint16_t *)x[i].signs; + const int8_t * restrict q8 = y[i].qs; + int sumi1 = 0, sumi2 = 0; + for (int ib32 = 0; ib32 < QK_K/32; ib32 += 2) { + q8b = ggml_vld1q_s8_x4(q8); q8 += 64; + const uint32x4_t aux32x4_0 = {iq3xs_grid[qs[ 0] | ((qh[ib32+0] << 8) & 256)], iq3xs_grid[qs[ 1] | ((qh[ib32+0] << 7) & 256)], + iq3xs_grid[qs[ 2] | ((qh[ib32+0] << 6) & 256)], iq3xs_grid[qs[ 3] | ((qh[ib32+0] << 5) & 256)]}; + const uint32x4_t aux32x4_1 = {iq3xs_grid[qs[ 4] | ((qh[ib32+0] << 4) & 256)], iq3xs_grid[qs[ 5] | ((qh[ib32+0] << 3) & 256)], + iq3xs_grid[qs[ 6] | ((qh[ib32+0] << 2) & 256)], iq3xs_grid[qs[ 7] | ((qh[ib32+0] << 1) & 256)]}; + const uint32x4_t aux32x4_2 = {iq3xs_grid[qs[ 8] | ((qh[ib32+1] << 8) & 256)], iq3xs_grid[qs[ 9] | ((qh[ib32+1] << 7) & 256)], + iq3xs_grid[qs[10] | ((qh[ib32+1] << 6) & 256)], iq3xs_grid[qs[11] | ((qh[ib32+1] << 5) & 256)]}; + const uint32x4_t aux32x4_3 = {iq3xs_grid[qs[12] | ((qh[ib32+1] << 4) & 256)], iq3xs_grid[qs[13] | ((qh[ib32+1] << 3) & 256)], + iq3xs_grid[qs[14] | ((qh[ib32+1] << 2) & 256)], iq3xs_grid[qs[15] | ((qh[ib32+1] << 1) & 256)]}; + qs += 16; + + vs.val[0] = vreinterpretq_u8_u32(vdupq_n_u32(signs[0] | (signs[1] << 16))); + vs.val[1] = vandq_u8(vqtbl1q_u8(vs.val[0], mask1.val[1]), mask2); + vs.val[0] = vandq_u8(vqtbl1q_u8(vs.val[0], mask1.val[0]), mask2); + vs.val[0] = vceqq_u8(vs.val[0], mask2); + vs.val[1] = vceqq_u8(vs.val[1], mask2); + + q3s.val[0] = vsubq_s8(vreinterpretq_s8_u8(veorq_u8(vs.val[0], vreinterpretq_u8_u32(aux32x4_0))), vreinterpretq_s8_u8(vs.val[0])); + q3s.val[1] = vsubq_s8(vreinterpretq_s8_u8(veorq_u8(vs.val[1], vreinterpretq_u8_u32(aux32x4_1))), vreinterpretq_s8_u8(vs.val[1])); + + vs.val[0] = vreinterpretq_u8_u32(vdupq_n_u32(signs[2] | (signs[3] << 16))); + vs.val[1] = vandq_u8(vqtbl1q_u8(vs.val[0], mask1.val[1]), mask2); + vs.val[0] = vandq_u8(vqtbl1q_u8(vs.val[0], mask1.val[0]), mask2); + vs.val[0] = vceqq_u8(vs.val[0], mask2); + vs.val[1] = vceqq_u8(vs.val[1], mask2); + + signs += 4; + + q3s.val[2] = vsubq_s8(vreinterpretq_s8_u8(veorq_u8(vs.val[0], vreinterpretq_u8_u32(aux32x4_2))), vreinterpretq_s8_u8(vs.val[0])); + q3s.val[3] = vsubq_s8(vreinterpretq_s8_u8(veorq_u8(vs.val[1], vreinterpretq_u8_u32(aux32x4_3))), vreinterpretq_s8_u8(vs.val[1])); + + const int32x4_t p1 = ggml_vdotq_s32(ggml_vdotq_s32(vdupq_n_s32(0), q3s.val[0], q8b.val[0]), q3s.val[1], q8b.val[1]); + const int32x4_t p2 = ggml_vdotq_s32(ggml_vdotq_s32(vdupq_n_s32(0), q3s.val[2], q8b.val[2]), q3s.val[3], q8b.val[3]); + sumi1 += vaddvq_s32(p1) * (1 + 2*(x[i].scales[ib32/2] & 0xf)); + sumi2 += vaddvq_s32(p2) * (1 + 2*(x[i].scales[ib32/2] >> 4)); + } + sumf += d*(sumi1 + sumi2); + } + *s = 0.25f * sumf; + +#elif defined(__AVX2__) + + static const uint8_t k_mask1[32] = {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03 + }; + + static const uint8_t k_mask2[32] = {0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, + 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, + }; + + const __m256i mask1 = _mm256_loadu_si256((const __m256i*)k_mask1); + const __m256i mask2 = _mm256_loadu_si256((const __m256i*)k_mask2); + + __m256 accumf = _mm256_setzero_ps(); + for (int i = 0; i < nb; ++i) { + const float d = GGML_FP16_TO_FP32(x[i].d) * y[i].d; + const uint8_t * restrict qs = x[i].qs; + const uint8_t * restrict qh = x[i].qh; + const uint16_t * restrict signs = (const uint16_t *)x[i].signs; + const int8_t * restrict q8 = y[i].qs; + __m256i sumi1 = _mm256_setzero_si256(); + __m256i sumi2 = _mm256_setzero_si256(); + for (int ib32 = 0; ib32 < QK_K/32; ib32 += 2) { + const __m256i q8_1 = _mm256_loadu_si256((const __m256i *)q8); q8 += 32; + const __m256i q8_2 = _mm256_loadu_si256((const __m256i *)q8); q8 += 32; + const __m256i q2_1 = _mm256_set_epi32(iq3xs_grid[qs[7] | ((qh[ib32+0] << 1) & 256)], + iq3xs_grid[qs[6] | ((qh[ib32+0] << 2) & 256)], + iq3xs_grid[qs[5] | ((qh[ib32+0] << 3) & 256)], + iq3xs_grid[qs[4] | ((qh[ib32+0] << 4) & 256)], + iq3xs_grid[qs[3] | ((qh[ib32+0] << 5) & 256)], + iq3xs_grid[qs[2] | ((qh[ib32+0] << 6) & 256)], + iq3xs_grid[qs[1] | ((qh[ib32+0] << 7) & 256)], + iq3xs_grid[qs[0] | ((qh[ib32+0] << 8) & 256)]); + qs += 8; + const __m256i q2_2 = _mm256_set_epi32(iq3xs_grid[qs[7] | ((qh[ib32+1] << 1) & 256)], + iq3xs_grid[qs[6] | ((qh[ib32+1] << 2) & 256)], + iq3xs_grid[qs[5] | ((qh[ib32+1] << 3) & 256)], + iq3xs_grid[qs[4] | ((qh[ib32+1] << 4) & 256)], + iq3xs_grid[qs[3] | ((qh[ib32+1] << 5) & 256)], + iq3xs_grid[qs[2] | ((qh[ib32+1] << 6) & 256)], + iq3xs_grid[qs[1] | ((qh[ib32+1] << 7) & 256)], + iq3xs_grid[qs[0] | ((qh[ib32+1] << 8) & 256)]); + qs += 8; + + __m256i aux256 = _mm256_set1_epi32(signs[0] | (signs[1] << 16)); + aux256 = _mm256_and_si256(_mm256_shuffle_epi8(aux256,mask1), mask2); + const __m256i s2_1 = _mm256_cmpeq_epi8(aux256, mask2); + const __m256i q8s_1 = _mm256_sub_epi8(_mm256_xor_si256(s2_1, q8_1), s2_1); + + aux256 = _mm256_set1_epi32(signs[2] | (signs[3] << 16)); + aux256 = _mm256_and_si256(_mm256_shuffle_epi8(aux256,mask1), mask2); + const __m256i s2_2 = _mm256_cmpeq_epi8(aux256, mask2); + const __m256i q8s_2 = _mm256_sub_epi8(_mm256_xor_si256(s2_2, q8_2), s2_2); + + signs += 4; + + const __m256i dot1 = _mm256_maddubs_epi16(q2_1, q8s_1); + const __m256i dot2 = _mm256_maddubs_epi16(q2_2, q8s_2); + const uint16_t ls1 = x[i].scales[ib32/2] & 0xf; + const uint16_t ls2 = x[i].scales[ib32/2] >> 4; + const __m256i p1 = _mm256_madd_epi16(dot1, _mm256_set1_epi16(2*ls1+1)); + const __m256i p2 = _mm256_madd_epi16(dot2, _mm256_set1_epi16(2*ls2+1)); + sumi1 = _mm256_add_epi32(sumi1, p1); + sumi2 = _mm256_add_epi32(sumi2, p2); + } + + accumf = _mm256_fmadd_ps(_mm256_set1_ps(d), _mm256_cvtepi32_ps(_mm256_add_epi32(sumi1, sumi2)), accumf); + + } + + *s = 0.25f * hsum_float_8(accumf); + +#else + + float sumf = 0.f; + for (int i = 0; i < nb; ++i) { + const float d = GGML_FP16_TO_FP32(x[i].d) * y[i].d; + const uint8_t * restrict qs = x[i].qs; + const uint8_t * restrict qh = x[i].qh; + const uint8_t * restrict signs = x[i].signs; + const int8_t * restrict q8 = y[i].qs; + int32_t bsum = 0; + for (int ib32 = 0; ib32 < QK_K/32; ib32 += 2) { + const uint32_t ls1 = 2*(x[i].scales[ib32/2] & 0xf) + 1; + const uint32_t ls2 = 2*(x[i].scales[ib32/2] >> 4) + 1; + int32_t sumi = 0; + for (int l = 0; l < 4; ++l) { + const uint8_t * grid1 = (const uint8_t *)(iq3xs_grid + (qs[2*l+0] | ((qh[ib32+0] << (8-2*l)) & 256))); + const uint8_t * grid2 = (const uint8_t *)(iq3xs_grid + (qs[2*l+1] | ((qh[ib32+0] << (7-2*l)) & 256))); + for (int j = 0; j < 4; ++j) { + sumi += grid1[j] * q8[j+0] * (signs[l] & kmask_iq2xs[j+0] ? -1 : 1); + sumi += grid2[j] * q8[j+4] * (signs[l] & kmask_iq2xs[j+4] ? -1 : 1); + } + q8 += 8; + } + qs += 8; + signs += 4; + bsum += sumi * ls1; + sumi = 0; + for (int l = 0; l < 4; ++l) { + const uint8_t * grid1 = (const uint8_t *)(iq3xs_grid + (qs[2*l+0] | ((qh[ib32+1] << (8-2*l)) & 256))); + const uint8_t * grid2 = (const uint8_t *)(iq3xs_grid + (qs[2*l+1] | ((qh[ib32+1] << (7-2*l)) & 256))); + for (int j = 0; j < 4; ++j) { + sumi += grid1[j] * q8[j+0] * (signs[l] & kmask_iq2xs[j+0] ? -1 : 1); + sumi += grid2[j] * q8[j+4] * (signs[l] & kmask_iq2xs[j+4] ? -1 : 1); + } + q8 += 8; + } + qs += 8; + signs += 4; + bsum += sumi * ls2; + } + sumf += d * bsum; + } + *s = 0.25f * sumf; +#endif +} + + #ifdef __AVX2__ static inline __m256i mul_add_epi8(const __m256i x, const __m256i y) { const __m256i ax = _mm256_sign_epi8(x, x); @@ -9523,6 +9831,7 @@ void ggml_vec_dot_iq4_nl_q8_0(int n, float * restrict s, size_t bs, const void * float sumf = 0; for (int ib = 0; ib < nb; ib += 2) { + q4bits.val[0] = vld1q_u8(x[ib+0].qs); q4bits.val[1] = vld1q_u8(x[ib+1].qs); q8b.val[0] = vld1q_s8(y[ib+0].qs); @@ -10239,14 +10548,15 @@ typedef struct { uint16_t * neighbours; } iq3_entry_t; -static iq3_entry_t iq3_data[1] = { +static iq3_entry_t iq3_data[2] = { + {NULL, NULL, NULL}, {NULL, NULL, NULL}, }; static inline int iq3_data_index(int grid_size) { (void)grid_size; - GGML_ASSERT(grid_size == 256); - return 0; + GGML_ASSERT(grid_size == 256 || grid_size == 512); + return grid_size == 256 ? 0 : 1; } static int iq3_compare_func(const void * left, const void * right) { @@ -10278,9 +10588,44 @@ void iq3xs_init_impl(int grid_size) { 3185, 3215, 3252, 3288, 3294, 3364, 3397, 3434, 3483, 3523, 3537, 3587, 3589, 3591, 3592, 3610, 3626, 3670, 3680, 3722, 3749, 3754, 3776, 3789, 3803, 3824, 3857, 3873, 3904, 3906, 3924, 3992, }; + static const uint16_t kgrid_512[512] = { + 0, 1, 2, 5, 7, 8, 9, 10, 12, 14, 16, 17, 21, 27, 32, 34, + 37, 39, 41, 43, 48, 50, 57, 60, 63, 64, 65, 66, 68, 72, 73, 77, + 80, 83, 87, 89, 93, 100, 113, 117, 122, 128, 129, 133, 135, 136, 139, 142, + 145, 149, 152, 156, 162, 165, 167, 169, 171, 184, 187, 195, 201, 205, 208, 210, + 217, 219, 222, 228, 232, 234, 247, 249, 253, 256, 267, 271, 273, 276, 282, 288, + 291, 297, 312, 322, 324, 336, 338, 342, 347, 353, 357, 359, 374, 379, 390, 393, + 395, 409, 426, 441, 448, 450, 452, 464, 466, 470, 475, 488, 492, 512, 513, 514, + 516, 520, 521, 523, 525, 527, 528, 530, 537, 540, 542, 556, 558, 561, 570, 576, + 577, 579, 582, 584, 588, 593, 600, 603, 609, 616, 618, 632, 638, 640, 650, 653, + 655, 656, 660, 666, 672, 675, 685, 688, 698, 705, 708, 711, 712, 715, 721, 727, + 728, 732, 737, 754, 760, 771, 773, 778, 780, 793, 795, 802, 806, 808, 812, 833, + 840, 843, 849, 856, 858, 873, 912, 916, 919, 932, 934, 961, 963, 968, 970, 977, + 989, 993, 1010, 1016, 1024, 1025, 1027, 1029, 1031, 1032, 1034, 1036, 1038, 1041, 1043, 1047, + 1048, 1050, 1057, 1059, 1061, 1064, 1066, 1079, 1080, 1083, 1085, 1088, 1090, 1096, 1099, 1103, + 1106, 1109, 1113, 1116, 1122, 1129, 1153, 1156, 1159, 1169, 1171, 1176, 1183, 1185, 1195, 1199, + 1209, 1212, 1216, 1218, 1221, 1225, 1234, 1236, 1241, 1243, 1250, 1256, 1270, 1281, 1287, 1296, + 1299, 1306, 1309, 1313, 1338, 1341, 1348, 1353, 1362, 1375, 1376, 1387, 1400, 1408, 1410, 1415, + 1425, 1453, 1457, 1477, 1481, 1494, 1496, 1507, 1512, 1538, 1545, 1547, 1549, 1551, 1554, 1561, + 1563, 1565, 1570, 1572, 1575, 1577, 1587, 1593, 1601, 1603, 1605, 1612, 1617, 1619, 1632, 1648, + 1658, 1662, 1664, 1674, 1680, 1690, 1692, 1704, 1729, 1736, 1740, 1745, 1747, 1751, 1752, 1761, + 1763, 1767, 1773, 1787, 1795, 1801, 1806, 1810, 1817, 1834, 1840, 1844, 1857, 1864, 1866, 1877, + 1882, 1892, 1902, 1915, 1934, 1953, 1985, 1987, 2000, 2002, 2013, 2048, 2052, 2058, 2064, 2068, + 2071, 2074, 2081, 2088, 2104, 2114, 2119, 2121, 2123, 2130, 2136, 2141, 2147, 2153, 2157, 2177, + 2179, 2184, 2189, 2193, 2203, 2208, 2223, 2226, 2232, 2244, 2249, 2251, 2256, 2258, 2265, 2269, + 2304, 2306, 2324, 2335, 2336, 2361, 2373, 2375, 2385, 2418, 2443, 2460, 2480, 2504, 2509, 2520, + 2531, 2537, 2562, 2568, 2572, 2578, 2592, 2596, 2599, 2602, 2614, 2620, 2625, 2627, 2629, 2634, + 2641, 2650, 2682, 2688, 2697, 2707, 2712, 2718, 2731, 2754, 2759, 2760, 2775, 2788, 2793, 2805, + 2811, 2817, 2820, 2832, 2842, 2854, 2890, 2902, 2921, 2923, 2978, 3010, 3012, 3026, 3081, 3083, + 3085, 3097, 3099, 3120, 3136, 3152, 3159, 3188, 3210, 3228, 3234, 3245, 3250, 3256, 3264, 3276, + 3281, 3296, 3349, 3363, 3378, 3392, 3395, 3420, 3440, 3461, 3488, 3529, 3531, 3584, 3588, 3591, + 3600, 3602, 3614, 3616, 3628, 3634, 3650, 3657, 3668, 3683, 3685, 3713, 3716, 3720, 3726, 3729, + 3736, 3753, 3778, 3802, 3805, 3819, 3841, 3845, 3851, 3856, 3880, 3922, 3938, 3970, 3993, 4032, + }; + const int kmap_size = 4096; - const int nwant = 2; - const uint16_t * kgrid = kgrid_256; + const int nwant = grid_size == 256 ? 2 : 3; + const uint16_t * kgrid = grid_size == 256 ? kgrid_256 : kgrid_512; uint32_t * kgrid_q3xs; int * kmap_q3xs; uint16_t * kneighbors_q3xs; @@ -10377,7 +10722,7 @@ void iq3xs_init_impl(int grid_size) { } void iq3xs_free_impl(int grid_size) { - GGML_ASSERT(grid_size == 256); + GGML_ASSERT(grid_size == 256 || grid_size == 512); const int gindex = iq3_data_index(grid_size); if (iq3_data[gindex].grid) { free(iq3_data[gindex].grid); iq3_data[gindex].grid = NULL; @@ -10410,9 +10755,10 @@ static int iq3_find_best_neighbour(const uint16_t * restrict neighbours, const u return grid_index; } -static void quantize_row_iq3_xxs_impl(const float * restrict x, void * restrict vy, int n, const float * restrict quant_weights) { +static void quantize_row_iq3_xxs_impl(int grid_size, const float * restrict x, void * restrict vy, int n, + const float * restrict quant_weights) { - const int gindex = iq3_data_index(256); + const int gindex = iq3_data_index(grid_size); const uint32_t * kgrid_q3xs = iq3_data[gindex].grid; const int * kmap_q3xs = iq3_data[gindex].map; @@ -10426,9 +10772,23 @@ static void quantize_row_iq3_xxs_impl(const float * restrict x, void * restrict const int kMaxQ = 8; - const int nbl = n/256; + const int nbl = n/QK_K; - block_iq3_xxs * y = vy; + ggml_fp16_t * dh; + uint8_t * qs; + int block_size; + if (grid_size == 256) { + block_iq3_xxs * y = vy; + dh = &y->d; + qs = y->qs; + block_size = sizeof(block_iq3_xxs); + } else { + block_iq3_s * y = vy; + dh = &y->d; + qs = y->qs; + block_size = sizeof(block_iq3_s); + } + int quant_size = block_size - sizeof(ggml_fp16_t); float scales[QK_K/32]; float weight[32]; @@ -10439,20 +10799,21 @@ static void quantize_row_iq3_xxs_impl(const float * restrict x, void * restrict bool is_on_grid[8]; bool is_on_grid_aux[8]; uint8_t block_signs[8]; - uint8_t q3[3*(QK_K/8)]; + uint8_t q3[3*(QK_K/8)+QK_K/32]; uint32_t * scales_and_signs = (uint32_t *)(q3 + QK_K/4); + uint8_t * qh = q3 + 3*(QK_K/8); for (int ibl = 0; ibl < nbl; ++ibl) { - y[ibl].d = GGML_FP32_TO_FP16(0.f); - memset(q3, 0, 3*QK_K/8); + dh[0] = GGML_FP32_TO_FP16(0.f); + memset(q3, 0, 3*QK_K/8+QK_K/32); float max_scale = 0; const float * xbl = x + QK_K*ibl; float sumx2 = 0; for (int i = 0; i < QK_K; ++i) sumx2 += xbl[i]*xbl[i]; - float sigma2 = sumx2/QK_K; + float sigma2 = 2*sumx2/QK_K; for (int ib = 0; ib < QK_K/32; ++ib) { const float * xb = xbl + 32*ib; @@ -10570,7 +10931,13 @@ static void quantize_row_iq3_xxs_impl(const float * restrict x, void * restrict printf("\n"); GGML_ASSERT(false); } - q3[8*ib+k] = grid_index; + if (grid_size == 256) { + q3[8*ib+k] = grid_index; + } else { + q3[8*ib+k] = grid_index & 255; + qh[ib] |= ((grid_index >> 8) << k); + } + } scales_and_signs[ib] = block_signs[0] | (block_signs[1] << 7) | (block_signs[2] << 14) | (block_signs[3] << 21); GGML_ASSERT(scale >= 0); @@ -10579,63 +10946,25 @@ static void quantize_row_iq3_xxs_impl(const float * restrict x, void * restrict } if (!max_scale) { - memset(y[ibl].qs, 0, 3*QK_K/8); + memset(qs, 0, quant_size); + dh += block_size/sizeof(ggml_fp16_t); + qs += block_size; continue; } float d = max_scale/31; - y[ibl].d = GGML_FP32_TO_FP16(d); + dh[0] = GGML_FP32_TO_FP16(d * 1.0125f); // small improvement via this fudge factor float id = 1/d; - float sumqx = 0, sumq2 = 0; for (int ib = 0; ib < QK_K/32; ++ib) { int l = nearest_int(0.5f*(id*scales[ib]-1)); l = MAX(0, MIN(15, l)); scales_and_signs[ib] |= ((uint32_t)l << 28); - if (false) { - const float * xb = xbl + 32*ib; - if (quant_weights) { - const float * qw = quant_weights + QK_K*ibl + 32*ib; - for (int i = 0; i < 32; ++i) weight[i] = qw[i] * sqrtf(sigma2 + xb[i]*xb[i]); - } else { - for (int i = 0; i < 32; ++i) weight[i] = xb[i]*xb[i]; - } - const float db = 0.25f * d * (1 + 2*l); - for (int k = 0; k < 8; ++k) { - const int8_t * signs = keven_signs_q2xs + 8*((scales_and_signs[ib] >> 7*(k/2)) & 127) + 4*(k%2); - const float * xk = xb + 4*k; - const float * wk = weight + 4*k; - //const uint8_t * grid = (const uint8_t *)(kgrid_q3xs + q3[8*ib+k]); - const uint8_t * grid = (const uint8_t *)(iq3xxs_grid + q3[8*ib+k]); - float best_mse = 0; int best_index = q3[8*ib+k]; - for (int j = 0; j < 4; ++j) { - float diff = db * grid[j] * signs[j] - xk[j]; - best_mse += wk[j] * diff * diff; - } - for (int idx = 0; idx < 256; ++idx) { - //grid = (const uint8_t *)(kgrid_q3xs + idx); - grid = (const uint8_t *)(iq3xxs_grid + idx); - float mse = 0; - for (int j = 0; j < 4; ++j) { - float diff = db * grid[j] * signs[j] - xk[j]; - mse += wk[j] * diff * diff; - } - if (mse < best_mse) { - best_mse = mse; best_index = idx; - } - } - q3[8*ib+k] = best_index; - //grid = (const uint8_t *)(kgrid_q3xs + best_index); - grid = (const uint8_t *)(iq3xxs_grid + best_index); - for (int j = 0; j < 4; ++j) { - float q = db * grid[j] * signs[j]; - sumqx += wk[j] * q * xk[j]; - sumq2 += wk[j] * q * q; - } - } - if (sumq2 > 0) y[ibl].d = GGML_FP32_TO_FP16(d*sumqx/sumq2); - } } - memcpy(y[ibl].qs, q3, 3*QK_K/8); + memcpy(qs, q3, quant_size); + + dh += block_size/sizeof(ggml_fp16_t); + qs += block_size; + } } @@ -10645,7 +10974,7 @@ size_t quantize_iq3_xxs(const float * src, void * dst, int nrow, int n_per_row, int nblock = n_per_row/QK_K; char * qrow = (char *)dst; for (int row = 0; row < nrow; ++row) { - quantize_row_iq3_xxs_impl(src, qrow, n_per_row, quant_weights); + quantize_row_iq3_xxs_impl(256, src, qrow, n_per_row, quant_weights); src += n_per_row; qrow += nblock*sizeof(block_iq3_xxs); } @@ -10660,9 +10989,226 @@ void quantize_row_iq3_xxs(const float * restrict x, void * restrict vy, int k) { void quantize_row_iq3_xxs_reference(const float * restrict x, block_iq3_xxs * restrict y, int k) { assert(k % QK_K == 0); - quantize_row_iq3_xxs_impl(x, y, k, NULL); + quantize_row_iq3_xxs_impl(256, x, y, k, NULL); } +static void quantize_row_iq3_s_impl(int block_size, const float * restrict x, void * restrict vy, int n, + const float * restrict quant_weights, + float * scales, + float * weight, + float * xval, + int8_t * L, + int8_t * Laux, + float * waux, + bool * is_on_grid, + bool * is_on_grid_aux, + uint8_t * block_signs) { + + const int gindex = iq3_data_index(512); + + const uint32_t * kgrid_q3xs = iq3_data[gindex].grid; + const int * kmap_q3xs = iq3_data[gindex].map; + const uint16_t * kneighbors_q3xs = iq3_data[gindex].neighbours; + + //GGML_ASSERT(quant_weights && "missing quantization weights"); + GGML_ASSERT(kgrid_q3xs && "forgot to call ggml_quantize_init()?"); + GGML_ASSERT(kmap_q3xs && "forgot to call ggml_quantize_init()?"); + GGML_ASSERT(kneighbors_q3xs && "forgot to call ggml_quantize_init()?"); + GGML_ASSERT(n%QK_K == 0); + + const int kMaxQ = 8; + + const int nbl = n/QK_K; + + block_iq3_s * y = vy; + + const int bs4 = block_size/4; + const int bs8 = block_size/8; + + for (int ibl = 0; ibl < nbl; ++ibl) { + + memset(&y[ibl], 0, sizeof(block_iq3_s)); + y[ibl].d = GGML_FP32_TO_FP16(0.f); + + uint8_t * qs = y[ibl].qs; + uint8_t * qh = y[ibl].qh; + uint8_t * signs = y[ibl].signs; + + float max_scale = 0; + + const float * xbl = x + QK_K*ibl; + float sumx2 = 0; + for (int i = 0; i < QK_K; ++i) sumx2 += xbl[i]*xbl[i]; + float sigma2 = 2*sumx2/QK_K; + + for (int ib = 0; ib < QK_K/block_size; ++ib) { + const float * xb = xbl + block_size*ib; + if (quant_weights) { + const float * qw = quant_weights + QK_K*ibl + block_size*ib; + for (int i = 0; i < block_size; ++i) weight[i] = qw[i] * sqrtf(sigma2 + xb[i]*xb[i]); + } else { + for (int i = 0; i < block_size; ++i) weight[i] = xb[i]*xb[i]; + } + for (int i = 0; i < block_size; ++i) waux[i] = sqrtf(weight[i]); + for (int k = 0; k < bs8; ++k) { + uint8_t s = 0; + for (int i = 0; i < 8; ++i) { + if (xb[8*k + i] >= 0) xval[8*k + i] = xb[8*k + i]; + else { + xval[8*k + i] = -xb[8*k + i]; s |= (1 << i); + } + } + block_signs[k] = s; + } + float max = xval[0]; + for (int i = 1; i < block_size; ++i) max = MAX(max, xval[i]); + if (!max) { + scales[ib] = 0; + continue; + } + float best = 0; + float scale = max/(2*kMaxQ-1); + for (int is = -15; is <= 15; ++is) { + float id = (2*kMaxQ-1+is*0.2f)/max; + float this_scale = 1/id; + for (int k = 0; k < bs4; ++k) { + for (int i = 0; i < 4; ++i) { + int l = nearest_int(0.5f*(id*xval[4*k+i]-1)); + Laux[4*k+i] = MAX(0, MIN(kMaxQ-1, l)); + } + uint16_t u = 0; + for (int i = 0; i < 4; ++i) u |= (Laux[4*k+i] << 3*i); + int grid_index = kmap_q3xs[u]; + is_on_grid_aux[k] = true; + if (grid_index < 0) { + is_on_grid_aux[k] = false; + const uint16_t * neighbours = kneighbors_q3xs - kmap_q3xs[u] - 1; + grid_index = iq3_find_best_neighbour(neighbours, kgrid_q3xs, xval + 4*k, waux + 4*k, this_scale, Laux + 4*k); + } + } + float sumqx = 0, sumq2 = 0; + for (int i = 0; i < block_size; ++i) { + float w = weight[i]; + float q = 2*Laux[i] + 1; + sumqx += w*xval[i]*q; + sumq2 += w*q*q; + } + if (sumq2 > 0 && sumqx*sumqx > best*sumq2) { + scale = sumqx/sumq2; best = scale*sumqx; + for (int i = 0; i < block_size; ++i) L[i] = Laux[i]; + for (int k = 0; k < bs4; ++k) is_on_grid[k] = is_on_grid_aux[k]; + } + } + int n_not_ongrid = 0; + for (int k = 0; k < bs4; ++k) if (!is_on_grid[k]) ++n_not_ongrid; + if (n_not_ongrid > 0 && scale > 0) { + float id = 1/scale; + for (int k = 0; k < bs4; ++k) { + if (is_on_grid[k]) continue; + uint16_t u = 0; + for (int i = 0; i < 4; ++i) { + int l = nearest_int(0.5f*(id*xval[4*k+i]-1)); + l = MAX(0, MIN(kMaxQ-1, l)); + u |= (l << 3*i); + } + int grid_index = kmap_q3xs[u]; + if (grid_index < 0) { + const uint16_t * neighbours = kneighbors_q3xs - kmap_q3xs[u] - 1; + grid_index = iq3_find_best_neighbour(neighbours, kgrid_q3xs, xval + 4*k, waux + 4*k, scale, L + 4*k); + } + const int8_t * pg = (const int8_t *)(kgrid_q3xs + grid_index); + for (int i = 0; i < 4; ++i) L[4*k+i] = (pg[i] - 1)/2; + } + float sumqx = 0, sumq2 = 0; + for (int i = 0; i < block_size; ++i) { + float w = weight[i]; + float q = 2*L[i] + 1; + sumqx += w*xval[i]*q; + sumq2 += w*q*q; + } + if (sumq2 > 0) scale = sumqx/sumq2; + } + if (scale < 0) { + // This should never happen, but just in case, flip scale so that it is positive (we use uint's to encode the scale) + // and correspondingly flip quant signs. + scale = -scale; + for (int k = 0; k < bs8; ++k) block_signs[k] = ~block_signs[k]; + } + for (int k = 0; k < bs4; ++k) { + uint16_t u = 0; + for (int i = 0; i < 4; ++i) u |= (L[4*k+i] << 3*i); + int grid_index = kmap_q3xs[u]; + if (grid_index < 0) { + printf("Oops: found point %u not on grid:", u); + for (int i = 0; i < 4; ++i) printf(" %d", L[4*k+i]); + printf("\n"); + GGML_ASSERT(false); + } + qs[k] = grid_index & 255; + qh[(ib*bs4+k)/8] |= ((grid_index >> 8) << ((ib*bs4+k)%8)); + } + qs += bs4; + for (int k = 0; k < bs8; ++k) signs[k] = block_signs[k]; + signs += bs8; + GGML_ASSERT(scale >= 0); + scales[ib] = scale; + max_scale = MAX(max_scale, scale); + } + + if (!max_scale) { + continue; + } + + float d = max_scale/31; + y[ibl].d = GGML_FP32_TO_FP16(d); + float id = 1/d; + for (int ib = 0; ib < QK_K/block_size; ib += 2) { + int l1 = nearest_int(0.5f*(id*scales[ib+0]-1)); + l1 = MAX(0, MIN(15, l1)); + int l2 = nearest_int(0.5f*(id*scales[ib+1]-1)); + l2 = MAX(0, MIN(15, l2)); + y[ibl].scales[ib/2] = l1 | (l2 << 4); + } + + } +} + +#define IQ3S_BLOCK_SIZE 32 +size_t quantize_iq3_s(const float * src, void * dst, int nrow, int n_per_row, int64_t * hist, const float * quant_weights) { + (void)hist; + GGML_ASSERT(n_per_row%QK_K == 0); + int nblock = n_per_row/QK_K; + float scales[QK_K/IQ3S_BLOCK_SIZE]; + float weight[IQ3S_BLOCK_SIZE]; + float xval[IQ3S_BLOCK_SIZE]; + int8_t L[IQ3S_BLOCK_SIZE]; + int8_t Laux[IQ3S_BLOCK_SIZE]; + float waux[IQ3S_BLOCK_SIZE]; + bool is_on_grid[IQ3S_BLOCK_SIZE/4]; + bool is_on_grid_aux[IQ3S_BLOCK_SIZE/4]; + uint8_t block_signs[IQ3S_BLOCK_SIZE/8]; + char * qrow = (char *)dst; + for (int row = 0; row < nrow; ++row) { + quantize_row_iq3_s_impl(IQ3S_BLOCK_SIZE, src, qrow, n_per_row, quant_weights, + scales, weight, xval, L, Laux, waux, is_on_grid, is_on_grid_aux, block_signs); + src += n_per_row; + qrow += nblock*sizeof(block_iq3_s); + } + return nrow * nblock * sizeof(block_iq3_s); +} + +void quantize_row_iq3_s(const float * restrict x, void * restrict vy, int k) { + assert(k % QK_K == 0); + block_iq3_s * restrict y = vy; + quantize_row_iq3_s_reference(x, y, k); +} + +void quantize_row_iq3_s_reference(const float * restrict x, block_iq3_s * restrict y, int k) { + assert(k % QK_K == 0); + quantize_iq3_s(x, y, 1, k, NULL, NULL); +} + + // =================================== 1.5 bpw =================================================== static int iq1_find_best_neighbour(const uint16_t * restrict neighbours, const uint64_t * restrict grid, diff --git a/ggml-quants.h b/ggml-quants.h index 113623b62..303b0b6f9 100644 --- a/ggml-quants.h +++ b/ggml-quants.h @@ -191,6 +191,21 @@ typedef struct { } block_iq3_xxs; static_assert(sizeof(block_iq3_xxs) == sizeof(ggml_fp16_t) + 3*(QK_K/8), "wrong iq3_xxs block size/padding"); +// 3.4375 bpw +#if QK_K == 64 +#define IQ3S_N_SCALE 2 +#else +#define IQ3S_N_SCALE QK_K/64 +#endif +typedef struct { + ggml_fp16_t d; + uint8_t qs[QK_K/4]; + uint8_t qh[QK_K/32]; + uint8_t signs[QK_K/8]; + uint8_t scales[IQ3S_N_SCALE]; +} block_iq3_s; +static_assert(sizeof(block_iq3_s) == sizeof(ggml_fp16_t) + 13*(QK_K/32) + IQ3S_N_SCALE, "wrong iq3_s block size/padding"); + typedef struct { ggml_fp16_t d; uint8_t qs[QK_K/8]; @@ -226,6 +241,7 @@ void quantize_row_q6_K_reference(const float * GGML_RESTRICT x, block_q6_K * GGM void quantize_row_q8_K_reference(const float * GGML_RESTRICT x, block_q8_K * GGML_RESTRICT y, int k); void quantize_row_iq3_xxs_reference(const float * GGML_RESTRICT x, block_iq3_xxs * GGML_RESTRICT y, int k); void quantize_row_iq4_nl_reference (const float * GGML_RESTRICT x, block_iq4_nl * GGML_RESTRICT y, int k); +void quantize_row_iq3_s_reference (const float * GGML_RESTRICT x, block_iq3_s * GGML_RESTRICT y, int k); void quantize_row_q4_0(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int k); void quantize_row_q4_1(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int k); @@ -242,6 +258,7 @@ void quantize_row_q6_K(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, in void quantize_row_q8_K(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int k); void quantize_row_iq3_xxs(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int k); void quantize_row_iq4_nl (const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int k); +void quantize_row_iq3_s (const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int k); // Dequantization void dequantize_row_q4_0(const block_q4_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int k); @@ -262,6 +279,7 @@ void dequantize_row_iq2_xs (const block_iq2_xs * GGML_RESTRICT x, float * GGML_ void dequantize_row_iq3_xxs(const block_iq3_xxs * GGML_RESTRICT x, float * GGML_RESTRICT y, int k); void dequantize_row_iq1_s (const block_iq1_s * GGML_RESTRICT x, float * GGML_RESTRICT y, int k); void dequantize_row_iq4_nl (const block_iq4_nl * GGML_RESTRICT x, float * GGML_RESTRICT y, int k); +void dequantize_row_iq3_s (const block_iq3_s * GGML_RESTRICT x, float * GGML_RESTRICT y, int k); // Dot product void ggml_vec_dot_q4_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); @@ -280,6 +298,7 @@ void ggml_vec_dot_iq2_xs_q8_K (int n, float * GGML_RESTRICT s, size_t bs, const void ggml_vec_dot_iq3_xxs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); void ggml_vec_dot_iq1_s_q8_K (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); void ggml_vec_dot_iq4_nl_q8_0 (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); +void ggml_vec_dot_iq3_s_q8_K (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); // // Quantization utilizing an importance matrix (a.k.a. "Activation aWare Quantization") @@ -289,6 +308,7 @@ size_t quantize_iq2_xs (const float * src, void * dst, int nrows, int n_per_row, size_t quantize_iq3_xxs(const float * src, void * dst, int nrows, int n_per_row, int64_t * hist, const float * imatrix); size_t quantize_iq1_s (const float * src, void * dst, int nrows, int n_per_row, int64_t * hist, const float * imatrix); size_t quantize_iq4_nl (const float * src, void * dst, int nrows, int n_per_row, int64_t * hist, const float * imatrix); +size_t quantize_iq3_s (const float * src, void * dst, int nrows, int n_per_row, int64_t * hist, const float * imatrix); size_t quantize_q2_K (const float * src, void * dst, int nrows, int n_per_row, int64_t * hist, const float * imatrix); size_t quantize_q3_K (const float * src, void * dst, int nrows, int n_per_row, int64_t * hist, const float * imatrix); size_t quantize_q4_K (const float * src, void * dst, int nrows, int n_per_row, int64_t * hist, const float * imatrix); diff --git a/ggml.c b/ggml.c index d710fe702..c09a3cad6 100644 --- a/ggml.c +++ b/ggml.c @@ -678,6 +678,18 @@ static const ggml_type_traits_t type_traits[GGML_TYPE_COUNT] = { .vec_dot_type = GGML_TYPE_Q8_K, .nrows = 1, }, + [GGML_TYPE_IQ3_S] = { + .type_name = "iq3_s", + .blck_size = QK_K, + .type_size = sizeof(block_iq3_s), + .is_quantized = true, + .to_float = (ggml_to_float_t) dequantize_row_iq3_s, + .from_float = quantize_row_iq3_s, + .from_float_reference = (ggml_from_float_t)quantize_row_iq3_s_reference, + .vec_dot = ggml_vec_dot_iq3_s_q8_K, + .vec_dot_type = GGML_TYPE_Q8_K, + .nrows = 1, + }, [GGML_TYPE_IQ1_S] = { .type_name = "iq1_s", .blck_size = QK_K, @@ -2304,6 +2316,7 @@ enum ggml_type ggml_ftype_to_ggml_type(enum ggml_ftype ftype) { case GGML_FTYPE_MOSTLY_IQ3_XXS: wtype = GGML_TYPE_IQ3_XXS; break; case GGML_FTYPE_MOSTLY_IQ1_S: wtype = GGML_TYPE_IQ1_S; break; case GGML_FTYPE_MOSTLY_IQ4_NL: wtype = GGML_TYPE_IQ4_NL; break; + case GGML_FTYPE_MOSTLY_IQ3_S: wtype = GGML_TYPE_IQ3_S; break; case GGML_FTYPE_UNKNOWN: wtype = GGML_TYPE_COUNT; break; case GGML_FTYPE_MOSTLY_Q4_1_SOME_F16: wtype = GGML_TYPE_COUNT; break; } @@ -7738,6 +7751,7 @@ static void ggml_compute_forward_add( case GGML_TYPE_IQ3_XXS: case GGML_TYPE_IQ1_S: case GGML_TYPE_IQ4_NL: + case GGML_TYPE_IQ3_S: { ggml_compute_forward_add_q_f32(params, dst); } break; @@ -8017,6 +8031,7 @@ static void ggml_compute_forward_add1( case GGML_TYPE_IQ3_XXS: case GGML_TYPE_IQ1_S: case GGML_TYPE_IQ4_NL: + case GGML_TYPE_IQ3_S: { ggml_compute_forward_add1_q_f32(params, dst); } break; @@ -8141,6 +8156,7 @@ static void ggml_compute_forward_acc( case GGML_TYPE_IQ3_XXS: case GGML_TYPE_IQ1_S: case GGML_TYPE_IQ4_NL: + case GGML_TYPE_IQ3_S: default: { GGML_ASSERT(false); @@ -11039,6 +11055,7 @@ static void ggml_compute_forward_out_prod( case GGML_TYPE_IQ3_XXS: case GGML_TYPE_IQ1_S: case GGML_TYPE_IQ4_NL: + case GGML_TYPE_IQ3_S: { ggml_compute_forward_out_prod_q_f32(params, dst); } break; @@ -11227,6 +11244,7 @@ static void ggml_compute_forward_set( case GGML_TYPE_IQ3_XXS: case GGML_TYPE_IQ1_S: case GGML_TYPE_IQ4_NL: + case GGML_TYPE_IQ3_S: default: { GGML_ASSERT(false); @@ -11429,6 +11447,7 @@ static void ggml_compute_forward_get_rows( case GGML_TYPE_IQ3_XXS: case GGML_TYPE_IQ1_S: case GGML_TYPE_IQ4_NL: + case GGML_TYPE_IQ3_S: { ggml_compute_forward_get_rows_q(params, dst); } break; @@ -12129,6 +12148,7 @@ static void ggml_compute_forward_alibi( case GGML_TYPE_IQ3_XXS: case GGML_TYPE_IQ1_S: case GGML_TYPE_IQ4_NL: + case GGML_TYPE_IQ3_S: case GGML_TYPE_Q8_K: case GGML_TYPE_I8: case GGML_TYPE_I16: @@ -12212,6 +12232,7 @@ static void ggml_compute_forward_clamp( case GGML_TYPE_IQ3_XXS: case GGML_TYPE_IQ1_S: case GGML_TYPE_IQ4_NL: + case GGML_TYPE_IQ3_S: case GGML_TYPE_Q8_K: case GGML_TYPE_I8: case GGML_TYPE_I16: @@ -19463,6 +19484,7 @@ void ggml_quantize_init(enum ggml_type type) { case GGML_TYPE_IQ2_XS: case GGML_TYPE_IQ1_S: iq2xs_init_impl(type); break; case GGML_TYPE_IQ3_XXS: iq3xs_init_impl(256); break; + case GGML_TYPE_IQ3_S: iq3xs_init_impl(512); break; default: // nothing break; } @@ -19737,6 +19759,15 @@ size_t ggml_quantize_chunk(enum ggml_type type, const float * src, void * dst, i result = quantize_iq3_xxs(src + start, (char *)dst + start_row * row_size, nrows, n_per_row, hist, imatrix); GGML_ASSERT(result == row_size * nrows); } break; + case GGML_TYPE_IQ3_S: + { + GGML_ASSERT(start % QK_K == 0); + GGML_ASSERT(start % n_per_row == 0); + size_t start_row = start / n_per_row; + size_t row_size = ggml_row_size(type, n_per_row); + result = quantize_iq3_s(src + start, (char *)dst + start_row * row_size, nrows, n_per_row, hist, imatrix); + GGML_ASSERT(result == row_size * nrows); + } break; case GGML_TYPE_IQ1_S: { GGML_ASSERT(start % QK_K == 0); diff --git a/ggml.h b/ggml.h index 37eff6279..a4166e1f7 100644 --- a/ggml.h +++ b/ggml.h @@ -350,6 +350,7 @@ extern "C" { GGML_TYPE_IQ3_XXS = 18, GGML_TYPE_IQ1_S = 19, GGML_TYPE_IQ4_NL = 20, + GGML_TYPE_IQ3_S = 21, GGML_TYPE_I8, GGML_TYPE_I16, GGML_TYPE_I32, @@ -389,6 +390,7 @@ extern "C" { GGML_FTYPE_MOSTLY_IQ3_XXS = 17, // except 1d tensors GGML_FTYPE_MOSTLY_IQ1_S = 18, // except 1d tensors GGML_FTYPE_MOSTLY_IQ4_NL = 19, // except 1d tensors + GGML_FTYPE_MOSTLY_IQ3_S = 20, // except 1d tensors }; // available tensor operations: diff --git a/llama.cpp b/llama.cpp index 37477e6ef..1f6b6cff4 100644 --- a/llama.cpp +++ b/llama.cpp @@ -2545,6 +2545,7 @@ struct llama_model_loader { case GGML_TYPE_IQ3_XXS: ftype = LLAMA_FTYPE_MOSTLY_IQ3_XXS; break; case GGML_TYPE_IQ1_S: ftype = LLAMA_FTYPE_MOSTLY_IQ1_S; break; case GGML_TYPE_IQ4_NL: ftype = LLAMA_FTYPE_MOSTLY_IQ4_NL; break; + case GGML_TYPE_IQ3_S: ftype = LLAMA_FTYPE_MOSTLY_IQ3_S; break; default: { LLAMA_LOG_WARN("%s: unknown type %s\n", __func__, ggml_type_name(type_max)); @@ -2890,6 +2891,8 @@ static std::string llama_model_ftype_name(llama_ftype ftype) { case LLAMA_FTYPE_MOSTLY_IQ3_XXS:return "IQ3_XXS - 3.0625 bpw"; case LLAMA_FTYPE_MOSTLY_IQ1_S :return "IQ1_S - 1.5625 bpw"; case LLAMA_FTYPE_MOSTLY_IQ4_NL: return "IQ4_NL - 4.5 bpw"; + case LLAMA_FTYPE_MOSTLY_IQ3_S: return "IQ3_S - 3.4375 bpw"; + case LLAMA_FTYPE_MOSTLY_IQ3_M: return "IQ3_S mix - 3.66 bpw"; default: return "unknown, may not work"; } @@ -10544,6 +10547,12 @@ static ggml_type get_k_quant_type(quantize_state_internal & qs, ggml_type new_ty else if (ftype == LLAMA_FTYPE_MOSTLY_IQ3_XXS) { new_type = qs.model.hparams.n_gqa() >= 4 ? GGML_TYPE_Q4_K : !qs.has_imatrix ? GGML_TYPE_Q3_K : GGML_TYPE_IQ3_XXS; } + else if (ftype == LLAMA_FTYPE_MOSTLY_IQ3_S && qs.model.hparams.n_gqa() >= 4) { + new_type = GGML_TYPE_Q4_K; + } + else if (ftype == LLAMA_FTYPE_MOSTLY_IQ3_M) { + new_type = GGML_TYPE_Q4_K; + } else if (ftype == LLAMA_FTYPE_MOSTLY_Q3_K_M) { new_type = qs.i_attention_wv < 2 ? GGML_TYPE_Q5_K : GGML_TYPE_Q4_K; } @@ -10575,13 +10584,17 @@ static ggml_type get_k_quant_type(quantize_state_internal & qs, ggml_type new_ty new_type = GGML_TYPE_Q8_0; } else if (ftype == LLAMA_FTYPE_MOSTLY_Q3_K_XS) { - new_type = GGML_TYPE_Q2_K; + new_type = GGML_TYPE_IQ3_XXS; + } + } else if (name.find("attn_q.weight") != std::string::npos) { + if (ftype == LLAMA_FTYPE_MOSTLY_Q3_K_XS) { + new_type = GGML_TYPE_IQ3_XXS; } } else if (name.find("ffn_down") != std::string::npos) { auto info = layer_info(qs.i_ffn_down, qs.n_ffn_down, name.c_str()); int i_layer = info.first, n_layer = info.second; if (ftype == LLAMA_FTYPE_MOSTLY_Q2_K) new_type = GGML_TYPE_Q3_K; - else if (ftype == LLAMA_FTYPE_MOSTLY_Q2_K_S || ftype == LLAMA_FTYPE_MOSTLY_Q3_K_XS) { + else if (ftype == LLAMA_FTYPE_MOSTLY_Q2_K_S) { if (i_layer < n_layer/8) new_type = GGML_TYPE_Q4_K; } else if (ftype == LLAMA_FTYPE_MOSTLY_IQ3_XXS && !qs.has_imatrix) { @@ -10592,6 +10605,10 @@ static ggml_type get_k_quant_type(quantize_state_internal & qs, ggml_type new_ty : arch != LLM_ARCH_FALCON || use_more_bits(i_layer, n_layer) ? GGML_TYPE_Q4_K : GGML_TYPE_Q3_K; } + else if (ftype == LLAMA_FTYPE_MOSTLY_IQ3_M && (i_layer < n_layer/8 || + (qs.model.hparams.n_expert == 8 && use_more_bits(i_layer, n_layer)))) { + new_type = GGML_TYPE_Q4_K; + } else if (ftype == LLAMA_FTYPE_MOSTLY_Q3_K_L) { new_type = arch == LLM_ARCH_FALCON ? GGML_TYPE_Q4_K : GGML_TYPE_Q5_K; } @@ -10623,37 +10640,41 @@ static ggml_type get_k_quant_type(quantize_state_internal & qs, ggml_type new_ty if (qs.model.hparams.n_expert == 8) { if (ftype == LLAMA_FTYPE_MOSTLY_Q2_K || ftype == LLAMA_FTYPE_MOSTLY_Q3_K_XS || ftype == LLAMA_FTYPE_MOSTLY_IQ3_XXS || ftype == LLAMA_FTYPE_MOSTLY_Q3_K_S || ftype == LLAMA_FTYPE_MOSTLY_Q3_K_M || ftype == LLAMA_FTYPE_MOSTLY_IQ4_NL || - ftype == LLAMA_FTYPE_MOSTLY_Q4_K_S || ftype == LLAMA_FTYPE_MOSTLY_Q4_K_M) { + ftype == LLAMA_FTYPE_MOSTLY_Q4_K_S || ftype == LLAMA_FTYPE_MOSTLY_Q4_K_M || ftype == LLAMA_FTYPE_MOSTLY_IQ3_S || + ftype == LLAMA_FTYPE_MOSTLY_IQ3_M) { new_type = GGML_TYPE_Q5_K; } } else { - if (ftype == LLAMA_FTYPE_MOSTLY_Q2_K ) new_type = GGML_TYPE_Q3_K; + if (ftype == LLAMA_FTYPE_MOSTLY_Q2_K ) new_type = GGML_TYPE_Q3_K; else if (ftype == LLAMA_FTYPE_MOSTLY_IQ3_XXS) new_type = GGML_TYPE_Q3_K; - else if (ftype == LLAMA_FTYPE_MOSTLY_Q3_K_M) new_type = GGML_TYPE_Q4_K; - else if (ftype == LLAMA_FTYPE_MOSTLY_Q3_K_L) new_type = GGML_TYPE_Q5_K; + else if (ftype == LLAMA_FTYPE_MOSTLY_Q3_K_M ) new_type = GGML_TYPE_Q4_K; + else if (ftype == LLAMA_FTYPE_MOSTLY_Q3_K_L ) new_type = GGML_TYPE_Q5_K; + else if (ftype == LLAMA_FTYPE_MOSTLY_IQ3_M ) new_type = GGML_TYPE_Q4_K; } } else { if (ftype == LLAMA_FTYPE_MOSTLY_Q3_K_L) new_type = GGML_TYPE_Q4_K; } } else if (name.find("attn_qkv.weight") != std::string::npos) { - if (ftype == LLAMA_FTYPE_MOSTLY_Q3_K_M || ftype == LLAMA_FTYPE_MOSTLY_Q3_K_L) new_type = GGML_TYPE_Q4_K; + if (ftype == LLAMA_FTYPE_MOSTLY_Q3_K_M || ftype == LLAMA_FTYPE_MOSTLY_Q3_K_L || ftype == LLAMA_FTYPE_MOSTLY_IQ3_M) { + new_type = GGML_TYPE_Q4_K; + } else if (ftype == LLAMA_FTYPE_MOSTLY_Q4_K_M) new_type = GGML_TYPE_Q5_K; else if (ftype == LLAMA_FTYPE_MOSTLY_Q5_K_M) new_type = GGML_TYPE_Q6_K; } else if (name.find("ffn_gate") != std::string::npos) { auto info = layer_info(qs.i_ffn_gate, qs.n_ffn_gate, name.c_str()); int i_layer = info.first, n_layer = info.second; - if (ftype == LLAMA_FTYPE_MOSTLY_Q3_K_XS && !use_more_bits(i_layer, n_layer)) { - new_type = GGML_TYPE_Q2_K; + if (ftype == LLAMA_FTYPE_MOSTLY_Q3_K_XS && (i_layer >= n_layer/8 && i_layer < 7*n_layer/8)) { + new_type = GGML_TYPE_IQ3_XXS; } ++qs.i_ffn_gate; } else if (name.find("ffn_up") != std::string::npos) { auto info = layer_info(qs.i_ffn_up, qs.n_ffn_up, name.c_str()); int i_layer = info.first, n_layer = info.second; - if (ftype == LLAMA_FTYPE_MOSTLY_Q3_K_XS && !use_more_bits(i_layer, n_layer)) { - new_type = GGML_TYPE_Q2_K; + if (ftype == LLAMA_FTYPE_MOSTLY_Q3_K_XS && (i_layer >= n_layer/8 && i_layer < 7*n_layer/8)) { + new_type = GGML_TYPE_IQ3_XXS; } ++qs.i_ffn_up; } @@ -10673,7 +10694,7 @@ static ggml_type get_k_quant_type(quantize_state_internal & qs, ggml_type new_ty if (new_type == GGML_TYPE_Q2_K || new_type == GGML_TYPE_Q3_K || new_type == GGML_TYPE_Q4_K || new_type == GGML_TYPE_Q5_K || new_type == GGML_TYPE_Q6_K || new_type == GGML_TYPE_IQ2_XS || new_type == GGML_TYPE_IQ2_XXS || - new_type == GGML_TYPE_IQ3_XXS || ftype == LLAMA_FTYPE_MOSTLY_IQ1_S) { + new_type == GGML_TYPE_IQ3_XXS || ftype == LLAMA_FTYPE_MOSTLY_IQ1_S || new_type == GGML_TYPE_IQ3_S) { int nx = tensor->ne[0]; int ny = tensor->ne[1]; if (nx % QK_K != 0) { @@ -10688,6 +10709,7 @@ static ggml_type get_k_quant_type(quantize_state_internal & qs, ggml_type new_ty case GGML_TYPE_IQ2_XXS: case GGML_TYPE_IQ2_XS: case GGML_TYPE_IQ3_XXS: + case GGML_TYPE_IQ3_S: case GGML_TYPE_IQ1_S: case GGML_TYPE_Q2_K: case GGML_TYPE_Q3_K: new_type = GGML_TYPE_IQ4_NL; break; @@ -10719,7 +10741,7 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s // K-quants case LLAMA_FTYPE_MOSTLY_Q2_K_S: case LLAMA_FTYPE_MOSTLY_Q2_K: quantized_type = GGML_TYPE_Q2_K; break; - case LLAMA_FTYPE_MOSTLY_Q3_K_XS: + case LLAMA_FTYPE_MOSTLY_Q3_K_XS: quantized_type = GGML_TYPE_IQ3_S; break; case LLAMA_FTYPE_MOSTLY_Q3_K_S: case LLAMA_FTYPE_MOSTLY_Q3_K_M: case LLAMA_FTYPE_MOSTLY_Q3_K_L: quantized_type = GGML_TYPE_Q3_K; break; @@ -10733,6 +10755,8 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s case LLAMA_FTYPE_MOSTLY_IQ3_XXS: quantized_type = GGML_TYPE_IQ3_XXS; break; case LLAMA_FTYPE_MOSTLY_IQ1_S: quantized_type = GGML_TYPE_IQ1_S; break; case LLAMA_FTYPE_MOSTLY_IQ4_NL: quantized_type = GGML_TYPE_IQ4_NL; break; + case LLAMA_FTYPE_MOSTLY_IQ3_S: quantized_type = GGML_TYPE_IQ3_S; break; + case LLAMA_FTYPE_MOSTLY_IQ3_M: quantized_type = GGML_TYPE_IQ3_S; break; default: throw std::runtime_error(format("invalid output file type %d\n", ftype)); } diff --git a/llama.h b/llama.h index 84f196b3b..889edf4d9 100644 --- a/llama.h +++ b/llama.h @@ -102,6 +102,8 @@ extern "C" { LLAMA_FTYPE_MOSTLY_IQ3_XXS = 23, // except 1d tensors LLAMA_FTYPE_MOSTLY_IQ1_S = 24, // except 1d tensors LLAMA_FTYPE_MOSTLY_IQ4_NL = 25, // except 1d tensors + LLAMA_FTYPE_MOSTLY_IQ3_S = 26, // except 1d tensors + LLAMA_FTYPE_MOSTLY_IQ3_M = 27, // except 1d tensors LLAMA_FTYPE_GUESSED = 1024, // not specified in the model file }; diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index 55db42bf6..f8574588b 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -1918,7 +1918,7 @@ static bool test_backend(ggml_backend_t backend, test_mode mode, const char * op GGML_TYPE_Q6_K, GGML_TYPE_IQ2_XXS, GGML_TYPE_IQ2_XS, GGML_TYPE_IQ3_XXS, GGML_TYPE_IQ1_S, - GGML_TYPE_IQ4_NL, + GGML_TYPE_IQ4_NL, GGML_TYPE_IQ3_S, }; // unary ops diff --git a/tests/test-quantize-fns.cpp b/tests/test-quantize-fns.cpp index 5e92d5742..04656bb9e 100644 --- a/tests/test-quantize-fns.cpp +++ b/tests/test-quantize-fns.cpp @@ -151,6 +151,7 @@ int main(int argc, char * argv[]) { const float max_quantization_error = type == GGML_TYPE_Q2_K ? MAX_QUANTIZATION_TOTAL_ERROR_2BITS : type == GGML_TYPE_Q3_K ? MAX_QUANTIZATION_TOTAL_ERROR_3BITS : + type == GGML_TYPE_IQ3_S ? MAX_QUANTIZATION_TOTAL_ERROR_3BITS : type == GGML_TYPE_IQ3_XXS ? MAX_QUANTIZATION_TOTAL_ERROR_3BITS_XXS : MAX_QUANTIZATION_TOTAL_ERROR; failed = !(total_error < max_quantization_error); num_failed += failed; @@ -167,7 +168,8 @@ int main(int argc, char * argv[]) { const float vec_dot_error = dot_product_error(qfns, test_size, test_data.data(), test_data2.data()); const float max_allowed_error = type == GGML_TYPE_Q2_K || type == GGML_TYPE_IQ2_XS || type == GGML_TYPE_IQ2_XXS || - type == GGML_TYPE_IQ3_XXS ? MAX_DOT_PRODUCT_ERROR_LOWBIT : MAX_DOT_PRODUCT_ERROR; + type == GGML_TYPE_IQ3_XXS || type == GGML_TYPE_IQ3_S ? MAX_DOT_PRODUCT_ERROR_LOWBIT + : MAX_DOT_PRODUCT_ERROR; failed = !(vec_dot_error < max_allowed_error); num_failed += failed; if (failed || verbose) { From 9e359a4f47c1b2dceb99e29706c9f7403d32ab5e Mon Sep 17 00:00:00 2001 From: Pierrick Hymbert Date: Sat, 24 Feb 2024 19:16:04 +0100 Subject: [PATCH 450/812] server: continue to update other slots on embedding concurrent request (#5699) * server: #5655 - continue to update other slots on embedding concurrent request. * server: tests: add multi users embeddings as fixed * server: tests: adding OAI compatible embedding concurrent endpoint * server: tests: adding OAI compatible embedding with multiple inputs --- examples/server/server.cpp | 2 +- examples/server/tests/features/issues.feature | 34 +--- .../server/tests/features/parallel.feature | 46 ++++++ examples/server/tests/features/server.feature | 13 ++ examples/server/tests/features/steps/steps.py | 151 +++++++++++++----- 5 files changed, 168 insertions(+), 78 deletions(-) diff --git a/examples/server/server.cpp b/examples/server/server.cpp index 9fb436c2a..19a8c1067 100644 --- a/examples/server/server.cpp +++ b/examples/server/server.cpp @@ -1836,7 +1836,7 @@ struct llama_server_context send_embedding(slot); slot.release(); slot.i_batch = -1; - return true; + continue; } completion_token_output result; diff --git a/examples/server/tests/features/issues.feature b/examples/server/tests/features/issues.feature index 542006d9a..bf5a175a3 100644 --- a/examples/server/tests/features/issues.feature +++ b/examples/server/tests/features/issues.feature @@ -1,36 +1,4 @@ # List of ongoing issues @bug Feature: Issues - # Issue #5655 - Scenario: Multi users embeddings - Given a server listening on localhost:8080 - And a model file stories260K.gguf - And a model alias tinyllama-2 - And 42 as server seed - And 64 KV cache size - And 2 slots - And continuous batching - And embeddings extraction - Then the server is starting - Then the server is healthy - - Given a prompt: - """ - Write a very long story about AI. - """ - And a prompt: - """ - Write another very long music lyrics. - """ - And a prompt: - """ - Write a very long poem. - """ - And a prompt: - """ - Write a very long joke. - """ - Given concurrent embedding requests - Then the server is busy - Then the server is idle - Then all embeddings are generated + # No confirmed issue at the moment diff --git a/examples/server/tests/features/parallel.feature b/examples/server/tests/features/parallel.feature index 802d624ff..c85f9de1d 100644 --- a/examples/server/tests/features/parallel.feature +++ b/examples/server/tests/features/parallel.feature @@ -8,6 +8,7 @@ Feature: Parallel And 42 as server seed And 64 KV cache size And 2 slots + And embeddings extraction And continuous batching Then the server is starting Then the server is healthy @@ -75,3 +76,48 @@ Feature: Parallel Then the server is busy Then the server is idle Then all prompts are predicted + + Scenario: Multi users embeddings + Given a prompt: + """ + Write a very long story about AI. + """ + And a prompt: + """ + Write another very long music lyrics. + """ + And a prompt: + """ + Write a very long poem. + """ + And a prompt: + """ + Write a very long joke. + """ + Given concurrent embedding requests + Then the server is busy + Then the server is idle + Then all embeddings are generated + + Scenario: Multi users OAI compatibility embeddings + Given a prompt: + """ + In which country Paris is located ? + """ + And a prompt: + """ + Is Madrid the capital of Spain ? + """ + And a prompt: + """ + What is the biggest US city ? + """ + And a prompt: + """ + What is the capital of Bulgaria ? + """ + And a model tinyllama-2 + Given concurrent OAI embedding requests + Then the server is busy + Then the server is idle + Then all embeddings are generated diff --git a/examples/server/tests/features/server.feature b/examples/server/tests/features/server.feature index fedcfe5ae..5f81d256a 100644 --- a/examples/server/tests/features/server.feature +++ b/examples/server/tests/features/server.feature @@ -60,6 +60,19 @@ Feature: llama.cpp server """ Then embeddings are generated + Scenario: OAI Embeddings compatibility with multiple inputs + Given a model tinyllama-2 + Given a prompt: + """ + In which country Paris is located ? + """ + And a prompt: + """ + Is Madrid the capital of Spain ? + """ + When an OAI compatible embeddings computation request for multiple inputs + Then embeddings are generated + Scenario: Tokenize / Detokenize When tokenizing: diff --git a/examples/server/tests/features/steps/steps.py b/examples/server/tests/features/steps/steps.py index 50f2b641e..9c825fdbc 100644 --- a/examples/server/tests/features/steps/steps.py +++ b/examples/server/tests/features/steps/steps.py @@ -1,4 +1,5 @@ import asyncio +import collections import json import os import re @@ -261,35 +262,35 @@ def step_a_prompt_prompt(context, prompt): @step(u'concurrent completion requests') @async_run_until_complete() async def step_concurrent_completion_requests(context): - await concurrent_completion_requests(context, - request_completion, - # prompt is inserted automatically - context.base_url, - debug=context.debug, - n_predict=context.n_predict if hasattr(context, 'n_predict') else None, - server_seed=context.server_seed if hasattr(context, 'server_seed') else None, - user_api_key=context.user_api_key if hasattr(context, - 'user_api_key') else None) + await concurrent_requests(context, + request_completion, + # prompt is inserted automatically + context.base_url, + debug=context.debug, + n_predict=context.n_predict if hasattr(context, 'n_predict') else None, + server_seed=context.server_seed if hasattr(context, 'server_seed') else None, + user_api_key=context.user_api_key if hasattr(context, + 'user_api_key') else None) @step(u'concurrent OAI completions requests') @async_run_until_complete async def step_oai_chat_completions(context): - await concurrent_completion_requests(context, oai_chat_completions, - # user_prompt is inserted automatically - context.system_prompt, - context.base_url, - True, # async_client - model=context.model - if hasattr(context, 'model') else None, - n_predict=context.n_predict - if hasattr(context, 'n_predict') else None, - enable_streaming=context.enable_streaming - if hasattr(context, 'enable_streaming') else None, - server_seed=context.server_seed - if hasattr(context, 'server_seed') else None, - user_api_key=context.user_api_key - if hasattr(context, 'user_api_key') else None) + await concurrent_requests(context, oai_chat_completions, + # user_prompt is inserted automatically + context.system_prompt, + context.base_url, + True, # async_client + model=context.model + if hasattr(context, 'model') else None, + n_predict=context.n_predict + if hasattr(context, 'n_predict') else None, + enable_streaming=context.enable_streaming + if hasattr(context, 'enable_streaming') else None, + server_seed=context.server_seed + if hasattr(context, 'server_seed') else None, + user_api_key=context.user_api_key + if hasattr(context, 'user_api_key') else None) @step(u'all prompts are predicted') @@ -316,36 +317,58 @@ async def all_prompts_are_predicted(context, expected_predicted_n=None): @step(u'embeddings are computed for') @async_run_until_complete async def step_compute_embedding(context): - content = context.text - base_url = context.base_url - context.embeddings = await request_embedding(content, base_url) + context.embeddings = await request_embedding(context.text, base_url=context.base_url) @step(u'embeddings are generated') def step_assert_embeddings(context): - assert_embeddings(context.embeddings) + if len(context.prompts) == 0: + assert_embeddings(context.embeddings) + else: + assert len(context.embeddings) == len(context.prompts), (f"unexpected response:\n" + f"context.prompts={context.prompts}\n" + f"context.embeddings={context.embeddings}") + for embedding in context.embeddings: + context.prompts.pop() + assert_embeddings(embedding) @step(u'an OAI compatible embeddings computation request for') -def step_oai_compute_embedding(context): - openai.api_key = 'nope' # openai client always expects an api_keu - if context.user_api_key is not None: - openai.api_key = context.user_api_key - openai.api_base = f'{context.base_url}/v1' - embeddings = openai.Embedding.create( - model=context.model, - input=context.text, - ) - context.embeddings = embeddings +@async_run_until_complete +async def step_oai_compute_embeddings(context): + context.embeddings = await request_oai_embeddings(context.text, + base_url=context.base_url, + user_api_key=context.user_api_key, + model=context.model) + + +@step(u'an OAI compatible embeddings computation request for multiple inputs') +@async_run_until_complete +async def step_oai_compute_embeddings_multiple_inputs(context): + context.embeddings = await request_oai_embeddings(context.prompts, + base_url=context.base_url, + user_api_key=context.user_api_key, + model=context.model) @step(u'concurrent embedding requests') @async_run_until_complete() async def step_concurrent_embedding_requests(context): - await concurrent_completion_requests(context, - request_embedding, - # prompt is inserted automatically - context.base_url) + await concurrent_requests(context, + request_embedding, + # prompt is inserted automatically + base_url=context.base_url) + + +@step(u'concurrent OAI embedding requests') +@async_run_until_complete() +async def step_concurrent_oai_embedding_requests(context): + await concurrent_requests(context, + request_oai_embeddings, + # prompt is inserted automatically + base_url=context.base_url, + async_client=True, + model=context.model) @step(u'all embeddings are generated') @@ -401,7 +424,7 @@ def step_check_options_header_value(context, cors_header, cors_header_value): assert context.options_response.headers[cors_header] == cors_header_value -async def concurrent_completion_requests(context, f_completion, *args, **kwargs): +async def concurrent_requests(context, f_completion, *args, **kwargs): n_prompts = len(context.prompts) if context.debug: print(f"starting {n_prompts} concurrent completion requests...") @@ -565,7 +588,7 @@ async def oai_chat_completions(user_prompt, return completion_response -async def request_embedding(content, base_url): +async def request_embedding(content, base_url=None): async with aiohttp.ClientSession() as session: async with session.post(f'{base_url}/embedding', json={ @@ -576,6 +599,46 @@ async def request_embedding(content, base_url): return response_json['embedding'] +async def request_oai_embeddings(input, + base_url=None, user_api_key=None, + model=None, async_client=False): + # openai client always expects an api_key + user_api_key = user_api_key if user_api_key is not None else 'nope' + if async_client: + origin = 'llama.cpp' + if user_api_key is not None: + headers = {'Authorization': f'Bearer {user_api_key}', 'Origin': origin} + async with aiohttp.ClientSession() as session: + async with session.post(f'{base_url}/v1/embeddings', + json={ + "input": input, + "model": model, + }, + headers=headers) as response: + assert response.status == 200, f"received status code not expected: {response.status}" + assert response.headers['Access-Control-Allow-Origin'] == origin + assert response.headers['Content-Type'] == "application/json; charset=utf-8" + response_json = await response.json() + assert response_json['model'] == model, f"invalid model received: {response_json['model']}" + assert response_json['object'] == 'list' + return response_json['data'] + else: + openai.api_key = user_api_key + openai.api_base = f'{base_url}/v1' + oai_embeddings = openai.Embedding.create( + model=model, + input=input, + ) + + if isinstance(input, collections.abc.Sequence): + embeddings = [] + for an_oai_embeddings in oai_embeddings.data: + embeddings.append(an_oai_embeddings.embedding) + else: + embeddings = oai_embeddings.data.embedding + return embeddings + + def assert_n_tokens_predicted(completion_response, expected_predicted_n=None, re_content=None): content = completion_response['content'] n_predicted = completion_response['timings']['predicted_n'] From 69917dfa55674c608360638bb4d6a12a315e2810 Mon Sep 17 00:00:00 2001 From: Anas Ahouzi <112881240+aahouzi@users.noreply.github.com> Date: Sun, 25 Feb 2024 10:54:04 +0100 Subject: [PATCH 451/812] py : fix StableLM conversion after config.json changes (#5703) * Fix issues during StableLM models conversion * Fix hard coded layer_norm_eps * Support layer_norm_eps for LlavaStableLM Co-authored-by: Jared Van Bortel * Add missing parenthesis Co-authored-by: Jared Van Bortel * Support rotary_factor for LlavaStableLM Co-authored-by: Jared Van Bortel * fix typo * Add StableLMEpochForCausalLM for safety Co-authored-by: compilade <113953597+compilade@users.noreply.github.com> * Add StableLMEpochForCausalLM for safety 2 Co-authored-by: compilade <113953597+compilade@users.noreply.github.com> --------- Co-authored-by: Jared Van Bortel Co-authored-by: Jared Van Bortel Co-authored-by: compilade <113953597+compilade@users.noreply.github.com> --- convert-hf-to-gguf.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/convert-hf-to-gguf.py b/convert-hf-to-gguf.py index 32d54b45f..ae30b2a76 100755 --- a/convert-hf-to-gguf.py +++ b/convert-hf-to-gguf.py @@ -192,7 +192,7 @@ class Model: return RefactModel if model_architecture == "PersimmonForCausalLM": return PersimmonModel - if model_architecture in ("StableLMEpochForCausalLM", "LlavaStableLMEpochForCausalLM"): + if model_architecture in ("StableLmForCausalLM", "StableLMEpochForCausalLM", "LlavaStableLMEpochForCausalLM"): return StableLMModel if model_architecture == "QWenLMHeadModel": return QwenModel @@ -253,7 +253,7 @@ class Model: return gguf.MODEL_ARCH.REFACT if arch == "PersimmonForCausalLM": return gguf.MODEL_ARCH.PERSIMMON - if arch in ("StableLMEpochForCausalLM", "LlavaStableLMEpochForCausalLM"): + if arch in ("StableLmForCausalLM", "StableLMEpochForCausalLM", "LlavaStableLMEpochForCausalLM"): return gguf.MODEL_ARCH.STABLELM if arch == "QWenLMHeadModel": return gguf.MODEL_ARCH.QWEN @@ -1074,10 +1074,11 @@ class StableLMModel(Model): self.gguf_writer.add_embedding_length(hparams["hidden_size"]) self.gguf_writer.add_block_count(block_count) self.gguf_writer.add_feed_forward_length(hparams["intermediate_size"]) - self.gguf_writer.add_rope_dimension_count(int(hparams["rope_pct"] * (hparams["hidden_size"] // hparams["num_attention_heads"]))) + rotary_factor = self.find_hparam(["partial_rotary_factor", "rope_pct"]) + self.gguf_writer.add_rope_dimension_count(int(rotary_factor * (hparams["hidden_size"] // hparams["num_attention_heads"]))) self.gguf_writer.add_head_count(hparams["num_attention_heads"]) self.gguf_writer.add_parallel_residual(hparams["use_parallel_residual"] if "use_parallel_residual" in hparams else True) - self.gguf_writer.add_layer_norm_eps(1e-5) + self.gguf_writer.add_layer_norm_eps(self.find_hparam(["layer_norm_eps", "norm_eps"])) class MixtralModel(Model): From ab336a9d5e5352ecdcdf4c12d2d54cf4ef82ce31 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Sun, 25 Feb 2024 12:09:09 +0200 Subject: [PATCH 452/812] code : normalize enum names (#5697) * coda : normalize enum names ggml-ci * code : cont * code : cont --- common/common.cpp | 18 +- common/common.h | 4 +- common/train.cpp | 10 +- examples/baby-llama/baby-llama.cpp | 2 +- examples/finetune/finetune.cpp | 2 +- examples/llama-bench/llama-bench.cpp | 14 +- examples/llava/llava.cpp | 2 +- examples/server/server.cpp | 18 +- .../train-text-from-scratch.cpp | 2 +- ggml-cuda.cu | 138 +++---- ggml-metal.m | 4 +- ggml-opencl.cpp | 50 +-- ggml-sycl.cpp | 152 ++++---- ggml-vulkan.cpp | 102 ++--- ggml.c | 350 +++++++++--------- ggml.h | 38 +- llama.cpp | 64 ++-- llama.h | 28 +- tests/test-backend-ops.cpp | 4 +- tests/test-opt.cpp | 2 +- 20 files changed, 502 insertions(+), 502 deletions(-) diff --git a/common/common.cpp b/common/common.cpp index 10ef11829..ec596f5a0 100644 --- a/common/common.cpp +++ b/common/common.cpp @@ -295,9 +295,9 @@ bool gpt_params_parse_ex(int argc, char ** argv, gpt_params & params) { break; } std::string value(argv[i]); - /**/ if (value == "none") { params.rope_scaling_type = LLAMA_ROPE_SCALING_NONE; } - else if (value == "linear") { params.rope_scaling_type = LLAMA_ROPE_SCALING_LINEAR; } - else if (value == "yarn") { params.rope_scaling_type = LLAMA_ROPE_SCALING_YARN; } + /**/ if (value == "none") { params.rope_scaling_type = LLAMA_ROPE_SCALING_TYPE_NONE; } + else if (value == "linear") { params.rope_scaling_type = LLAMA_ROPE_SCALING_TYPE_LINEAR; } + else if (value == "yarn") { params.rope_scaling_type = LLAMA_ROPE_SCALING_TYPE_YARN; } else { invalid_param = true; break; } } else if (arg == "--rope-scale") { if (++i >= argc) { @@ -630,11 +630,11 @@ bool gpt_params_parse_ex(int argc, char ** argv, gpt_params & params) { } std::string arg_next = argv[i]; if (arg_next == "none") { - params.split_mode = LLAMA_SPLIT_NONE; + params.split_mode = LLAMA_SPLIT_MODE_NONE; } else if (arg_next == "layer") { - params.split_mode = LLAMA_SPLIT_LAYER; + params.split_mode = LLAMA_SPLIT_MODE_LAYER; } else if (arg_next == "row") { - params.split_mode = LLAMA_SPLIT_ROW; + params.split_mode = LLAMA_SPLIT_MODE_ROW; } else { invalid_param = true; break; @@ -837,15 +837,15 @@ bool gpt_params_parse_ex(int argc, char ** argv, gpt_params & params) { sep++; if (strncmp(sep, "int:", 4) == 0) { sep += 4; - kvo.tag = LLAMA_KV_OVERRIDE_INT; + kvo.tag = LLAMA_KV_OVERRIDE_TYPE_INT; kvo.int_value = std::atol(sep); } else if (strncmp(sep, "float:", 6) == 0) { sep += 6; - kvo.tag = LLAMA_KV_OVERRIDE_FLOAT; + kvo.tag = LLAMA_KV_OVERRIDE_TYPE_FLOAT; kvo.float_value = std::atof(sep); } else if (strncmp(sep, "bool:", 5) == 0) { sep += 5; - kvo.tag = LLAMA_KV_OVERRIDE_BOOL; + kvo.tag = LLAMA_KV_OVERRIDE_TYPE_BOOL; if (std::strcmp(sep, "true") == 0) { kvo.bool_value = true; } else if (std::strcmp(sep, "false") == 0) { diff --git a/common/common.h b/common/common.h index 935771d44..3e21579b0 100644 --- a/common/common.h +++ b/common/common.h @@ -61,7 +61,7 @@ struct gpt_params { float p_split = 0.1f; // speculative decoding split probability int32_t n_gpu_layers = -1; // number of layers to store in VRAM (-1 - use default) int32_t n_gpu_layers_draft = -1; // number of layers to store in VRAM for the draft model (-1 - use default) - llama_split_mode split_mode = LLAMA_SPLIT_LAYER; // how to split the model across GPUs + llama_split_mode split_mode = LLAMA_SPLIT_MODE_LAYER; // how to split the model across GPUs int32_t main_gpu = 0; // the GPU that is used for scratch and small tensors float tensor_split[128] = {0}; // how split tensors should be distributed across GPUs int32_t n_beams = 0; // if non-zero then use beam search of given width. @@ -75,7 +75,7 @@ struct gpt_params { float yarn_beta_fast = 32.0f; // YaRN low correction dim float yarn_beta_slow = 1.0f; // YaRN high correction dim int32_t yarn_orig_ctx = 0; // YaRN original context length - int32_t rope_scaling_type = LLAMA_ROPE_SCALING_UNSPECIFIED; + int32_t rope_scaling_type = LLAMA_ROPE_SCALING_TYPE_UNSPECIFIED; ggml_numa_strategy numa = GGML_NUMA_STRATEGY_DISABLED; // // sampling parameters diff --git a/common/train.cpp b/common/train.cpp index e4c3d5df6..0dbfd24df 100644 --- a/common/train.cpp +++ b/common/train.cpp @@ -31,7 +31,7 @@ struct train_state * init_train_state() { state->opt = new struct ggml_opt_context; state->opt->ctx = NULL; - state->opt->params = ggml_opt_default_params(GGML_OPT_ADAM); + state->opt->params = ggml_opt_default_params(GGML_OPT_TYPE_ADAM); state->opt->params.graph_size = LLAMA_TRAIN_MAX_NODES; state->opt->loss_after = 0.0f; @@ -556,7 +556,7 @@ void load_opt_context_gguf(struct gguf_context * fctx, struct ggml_context * f_g std::string opt_type; GGUF_GET_KEY(fctx, opt_type, gguf_get_val_str, GGUF_TYPE_STRING, true, LLM_KV_OPTIMIZER_TYPE); if (opt_type == LLM_KV_OPTIMIZER_TYPE_ADAM) { - opt->params.type = GGML_OPT_ADAM; + opt->params.type = GGML_OPT_TYPE_ADAM; GGUF_GET_KEY(fctx, opt->adam.fx_best, gguf_get_val_f32, GGUF_TYPE_FLOAT32, true, LLM_KV_OPTIMIZER_ADAM_BEST_LOSS); GGUF_GET_KEY(fctx, opt->adam.fx_prev, gguf_get_val_f32, GGUF_TYPE_FLOAT32, true, LLM_KV_OPTIMIZER_ADAM_PREVIOUS_LOSS); @@ -568,7 +568,7 @@ void load_opt_context_gguf(struct gguf_context * fctx, struct ggml_context * f_g copy_tensor_by_name(opt->adam.v, f_ggml_ctx, LLM_TENSOR_OPTIMIZER_ADAM_SECOND_MOMENTS); copy_tensor_by_name(opt->adam.pf, f_ggml_ctx, LLM_TENSOR_OPTIMIZER_ADAM_PAST_LOSS_VALUES); } else if (opt_type == LLM_KV_OPTIMIZER_TYPE_LBFGS) { - opt->params.type = GGML_OPT_LBFGS; + opt->params.type = GGML_OPT_TYPE_LBFGS; GGUF_GET_KEY(fctx, opt->params.lbfgs.m, gguf_get_val_u32, GGUF_TYPE_UINT32, true, LLM_KV_OPTIMIZER_LBFGS_APPROX_HESSIAN_COUNT); GGUF_GET_KEY(fctx, opt->lbfgs.fx_best, gguf_get_val_f32, GGUF_TYPE_FLOAT32, true, LLM_KV_OPTIMIZER_LBFGS_BEST_LOSS); @@ -603,7 +603,7 @@ void save_opt_context_gguf(struct gguf_context * fctx, struct ggml_opt_context * gguf_set_val_bool(fctx, LLM_KV_OPTIMIZER_JUST_INITIALIZED, opt->just_initialized); switch (opt->params.type) { - case GGML_OPT_ADAM: + case GGML_OPT_TYPE_ADAM: { gguf_set_val_str(fctx, LLM_KV_OPTIMIZER_TYPE, LLM_KV_OPTIMIZER_TYPE_ADAM); gguf_set_val_f32(fctx, LLM_KV_OPTIMIZER_ADAM_BEST_LOSS, opt->adam.fx_best); @@ -622,7 +622,7 @@ void save_opt_context_gguf(struct gguf_context * fctx, struct ggml_opt_context * gguf_add_tensor(fctx, opt->adam.pf); } } break; - case GGML_OPT_LBFGS: + case GGML_OPT_TYPE_LBFGS: { gguf_set_val_str(fctx, LLM_KV_OPTIMIZER_TYPE, LLM_KV_OPTIMIZER_TYPE_LBFGS); gguf_set_val_u32(fctx, LLM_KV_OPTIMIZER_LBFGS_APPROX_HESSIAN_COUNT, opt->params.lbfgs.m); diff --git a/examples/baby-llama/baby-llama.cpp b/examples/baby-llama/baby-llama.cpp index 65bb238a0..bf0125e75 100644 --- a/examples/baby-llama/baby-llama.cpp +++ b/examples/baby-llama/baby-llama.cpp @@ -1547,7 +1547,7 @@ int main(int argc, char ** argv) { float error_before_opt = ggml_get_f32_1d(e, 0); - struct ggml_opt_params opt_params_lbfgs = ggml_opt_default_params(GGML_OPT_LBFGS); + struct ggml_opt_params opt_params_lbfgs = ggml_opt_default_params(GGML_OPT_TYPE_LBFGS); opt_params_lbfgs.print_forward_graph = false; opt_params_lbfgs.print_backward_graph = false; opt_params_lbfgs.lbfgs.n_iter = 16; diff --git a/examples/finetune/finetune.cpp b/examples/finetune/finetune.cpp index 98bf5a07a..3da5317b3 100644 --- a/examples/finetune/finetune.cpp +++ b/examples/finetune/finetune.cpp @@ -1531,7 +1531,7 @@ int main(int argc, char ** argv) { lora.hparams.n_rank_output = n_rank_output; // set opt params from command line - opt->params = ggml_opt_default_params(GGML_OPT_ADAM); + opt->params = ggml_opt_default_params(GGML_OPT_TYPE_ADAM); opt->params.print_forward_graph = false; opt->params.print_backward_graph = false; opt->params.graph_size = LLAMA_TRAIN_MAX_NODES; diff --git a/examples/llama-bench/llama-bench.cpp b/examples/llama-bench/llama-bench.cpp index 11410f8ae..8fec3d43d 100644 --- a/examples/llama-bench/llama-bench.cpp +++ b/examples/llama-bench/llama-bench.cpp @@ -157,9 +157,9 @@ static const char * output_format_str(output_formats format) { static const char * split_mode_str(llama_split_mode mode) { switch (mode) { - case LLAMA_SPLIT_NONE: return "none"; - case LLAMA_SPLIT_LAYER: return "layer"; - case LLAMA_SPLIT_ROW: return "row"; + case LLAMA_SPLIT_MODE_NONE: return "none"; + case LLAMA_SPLIT_MODE_LAYER: return "layer"; + case LLAMA_SPLIT_MODE_ROW: return "row"; default: GGML_ASSERT(!"invalid split mode"); } } @@ -193,7 +193,7 @@ static const cmd_params cmd_params_defaults = { /* type_v */ {GGML_TYPE_F16}, /* n_threads */ {get_num_physical_cores()}, /* n_gpu_layers */ {99}, - /* split_mode */ {LLAMA_SPLIT_LAYER}, + /* split_mode */ {LLAMA_SPLIT_MODE_LAYER}, /* main_gpu */ {0}, /* no_kv_offload */ {false}, /* mul_mat_q */ {true}, @@ -358,11 +358,11 @@ static cmd_params parse_cmd_params(int argc, char ** argv) { for (const auto & m : p) { llama_split_mode mode; if (m == "none") { - mode = LLAMA_SPLIT_NONE; + mode = LLAMA_SPLIT_MODE_NONE; } else if (m == "layer") { - mode = LLAMA_SPLIT_LAYER; + mode = LLAMA_SPLIT_MODE_LAYER; } else if (m == "row") { - mode = LLAMA_SPLIT_ROW; + mode = LLAMA_SPLIT_MODE_ROW; } else { invalid_param = true; break; diff --git a/examples/llava/llava.cpp b/examples/llava/llava.cpp index 1a1cf7c78..980128166 100644 --- a/examples/llava/llava.cpp +++ b/examples/llava/llava.cpp @@ -152,7 +152,7 @@ static bool clip_llava_handle_patches(clip_ctx * ctx_clip, std::vector ggml_tensor * newline_tmp = clip_get_newline_tensor(ctx_clip); model.newline = ggml_new_tensor_1d(model.ctx, GGML_TYPE_F32, newline_tmp->ne[0]); - if (newline_tmp->backend != GGML_BACKEND_CPU) { + if (newline_tmp->backend != GGML_BACKEND_TYPE_CPU) { if (newline_tmp->buffer == NULL) { printf("newline_tmp tensor buffer is NULL\n"); } diff --git a/examples/server/server.cpp b/examples/server/server.cpp index 19a8c1067..780862ef6 100644 --- a/examples/server/server.cpp +++ b/examples/server/server.cpp @@ -2086,9 +2086,9 @@ static void server_params_parse(int argc, char **argv, server_params &sparams, break; } std::string value(argv[i]); - /**/ if (value == "none") { params.rope_scaling_type = LLAMA_ROPE_SCALING_NONE; } - else if (value == "linear") { params.rope_scaling_type = LLAMA_ROPE_SCALING_LINEAR; } - else if (value == "yarn") { params.rope_scaling_type = LLAMA_ROPE_SCALING_YARN; } + /**/ if (value == "none") { params.rope_scaling_type = LLAMA_ROPE_SCALING_TYPE_NONE; } + else if (value == "linear") { params.rope_scaling_type = LLAMA_ROPE_SCALING_TYPE_LINEAR; } + else if (value == "yarn") { params.rope_scaling_type = LLAMA_ROPE_SCALING_TYPE_YARN; } else { invalid_param = true; break; } } else if (arg == "--rope-freq-base") @@ -2212,15 +2212,15 @@ static void server_params_parse(int argc, char **argv, server_params &sparams, std::string arg_next = argv[i]; if (arg_next == "none") { - params.split_mode = LLAMA_SPLIT_NONE; + params.split_mode = LLAMA_SPLIT_MODE_NONE; } else if (arg_next == "layer") { - params.split_mode = LLAMA_SPLIT_LAYER; + params.split_mode = LLAMA_SPLIT_MODE_LAYER; } else if (arg_next == "row") { - params.split_mode = LLAMA_SPLIT_ROW; + params.split_mode = LLAMA_SPLIT_MODE_ROW; } else { invalid_param = true; @@ -2447,15 +2447,15 @@ static void server_params_parse(int argc, char **argv, server_params &sparams, sep++; if (strncmp(sep, "int:", 4) == 0) { sep += 4; - kvo.tag = LLAMA_KV_OVERRIDE_INT; + kvo.tag = LLAMA_KV_OVERRIDE_TYPE_INT; kvo.int_value = std::atol(sep); } else if (strncmp(sep, "float:", 6) == 0) { sep += 6; - kvo.tag = LLAMA_KV_OVERRIDE_FLOAT; + kvo.tag = LLAMA_KV_OVERRIDE_TYPE_FLOAT; kvo.float_value = std::atof(sep); } else if (strncmp(sep, "bool:", 5) == 0) { sep += 5; - kvo.tag = LLAMA_KV_OVERRIDE_BOOL; + kvo.tag = LLAMA_KV_OVERRIDE_TYPE_BOOL; if (std::strcmp(sep, "true") == 0) { kvo.bool_value = true; } else if (std::strcmp(sep, "false") == 0) { diff --git a/examples/train-text-from-scratch/train-text-from-scratch.cpp b/examples/train-text-from-scratch/train-text-from-scratch.cpp index e78ab185d..7eafe8515 100644 --- a/examples/train-text-from-scratch/train-text-from-scratch.cpp +++ b/examples/train-text-from-scratch/train-text-from-scratch.cpp @@ -960,7 +960,7 @@ int main(int argc, char ** argv) { struct ggml_opt_context * opt = train->opt; // set opt params from command line - opt->params = ggml_opt_default_params(GGML_OPT_ADAM); + opt->params = ggml_opt_default_params(GGML_OPT_TYPE_ADAM); opt->params.print_forward_graph = false; opt->params.print_backward_graph = false; opt->params.graph_size = LLAMA_TRAIN_MAX_NODES; diff --git a/ggml-cuda.cu b/ggml-cuda.cu index 21c612cb7..fb6d4f7d2 100644 --- a/ggml-cuda.cu +++ b/ggml-cuda.cu @@ -6369,11 +6369,11 @@ static __global__ void k_argsort_f32_i32(const float * x, int * dst, const int n int ixj = col ^ j; if (ixj > col) { if ((col & k) == 0) { - if (order == GGML_SORT_ASC ? x_row[dst_row[col]] > x_row[dst_row[ixj]] : x_row[dst_row[col]] < x_row[dst_row[ixj]]) { + if (order == GGML_SORT_ORDER_ASC ? x_row[dst_row[col]] > x_row[dst_row[ixj]] : x_row[dst_row[col]] < x_row[dst_row[ixj]]) { swap(dst_row[col], dst_row[ixj]); } } else { - if (order == GGML_SORT_ASC ? x_row[dst_row[col]] < x_row[dst_row[ixj]] : x_row[dst_row[col]] > x_row[dst_row[ixj]]) { + if (order == GGML_SORT_ORDER_ASC ? x_row[dst_row[col]] < x_row[dst_row[ixj]] : x_row[dst_row[col]] > x_row[dst_row[ixj]]) { swap(dst_row[col], dst_row[ixj]); } } @@ -7927,10 +7927,10 @@ static void argsort_f32_i32_cuda(const float * x, int * dst, const int ncols, co const dim3 block_dims(ncols, 1, 1); const dim3 block_nums(1, nrows, 1); - if (order == GGML_SORT_ASC) { - k_argsort_f32_i32<<>>(x, dst, ncols); - } else if (order == GGML_SORT_DESC) { - k_argsort_f32_i32<<>>(x, dst, ncols); + if (order == GGML_SORT_ORDER_ASC) { + k_argsort_f32_i32<<>>(x, dst, ncols); + } else if (order == GGML_SORT_ORDER_DESC) { + k_argsort_f32_i32<<>>(x, dst, ncols); } else { GGML_ASSERT(false); } @@ -8362,11 +8362,11 @@ static cudaError_t ggml_cuda_cpy_tensor_2d( cudaMemcpyKind kind; char * src_ptr; - if (src->backend == GGML_BACKEND_CPU) { + if (src->backend == GGML_BACKEND_TYPE_CPU) { kind = cudaMemcpyHostToDevice; src_ptr = (char *) src->data; - } else if (src->backend == GGML_BACKEND_GPU || src->backend == GGML_BACKEND_GPU_SPLIT) { - GGML_ASSERT(src->backend != GGML_BACKEND_GPU_SPLIT || (i1_low == 0 && i1_high == src->ne[1])); + } else if (src->backend == GGML_BACKEND_TYPE_GPU || src->backend == GGML_BACKEND_TYPE_GPU_SPLIT) { + GGML_ASSERT(src->backend != GGML_BACKEND_TYPE_GPU_SPLIT || (i1_low == 0 && i1_high == src->ne[1])); kind = cudaMemcpyDeviceToDevice; ggml_tensor_extra_gpu * extra = (ggml_tensor_extra_gpu *) src->extra; int id; @@ -8771,7 +8771,7 @@ static void ggml_cuda_op_mul_mat_q( // the main device has a larger memory buffer to hold the results from all GPUs // nrows_dst == nrows of the matrix that the kernel writes into - const int64_t nrows_dst = dst->backend == GGML_BACKEND_GPU && id == g_main_device ? ne0 : row_diff; + const int64_t nrows_dst = dst->backend == GGML_BACKEND_TYPE_GPU && id == g_main_device ? ne0 : row_diff; switch (src0->type) { case GGML_TYPE_Q4_0: @@ -8920,7 +8920,7 @@ static void ggml_cuda_op_mul_mat_vec_q( // the main device has a larger memory buffer to hold the results from all GPUs // nrows_dst == nrows of the matrix that the kernel writes into - const int64_t nrows_dst = dst->backend == GGML_BACKEND_GPU && id == g_main_device ? ne0 : row_diff; + const int64_t nrows_dst = dst->backend == GGML_BACKEND_TYPE_GPU && id == g_main_device ? ne0 : row_diff; switch (src0->type) { case GGML_TYPE_Q4_0: @@ -9096,7 +9096,7 @@ static void ggml_cuda_op_mul_mat_cublas( // the main device has a larger memory buffer to hold the results from all GPUs // ldc == nrows of the matrix that cuBLAS writes into - int ldc = dst->backend == GGML_BACKEND_GPU && id == g_main_device ? ne0 : row_diff; + int ldc = dst->backend == GGML_BACKEND_TYPE_GPU && id == g_main_device ? ne0 : row_diff; const int compute_capability = g_device_caps[id].cc; @@ -9444,7 +9444,7 @@ static void ggml_cuda_op_soft_max( const bool use_src2 = src2 != nullptr; if (use_src2) { - const bool src2_on_device = src2->backend == GGML_BACKEND_GPU; + const bool src2_on_device = src2->backend == GGML_BACKEND_TYPE_GPU; if (src2_on_device) { ggml_tensor_extra_gpu * src2_extra = (ggml_tensor_extra_gpu *) src2->extra; @@ -9502,16 +9502,16 @@ static void ggml_cuda_op_flatten(const ggml_tensor * src0, const ggml_tensor * s const bool use_src1 = src1 != nullptr; const int64_t nrows1 = use_src1 ? ggml_nrows(src1) : 1; - GGML_ASSERT(!use_src1 || src1->backend != GGML_BACKEND_GPU_SPLIT); - GGML_ASSERT( dst->backend != GGML_BACKEND_GPU_SPLIT); + GGML_ASSERT(!use_src1 || src1->backend != GGML_BACKEND_TYPE_GPU_SPLIT); + GGML_ASSERT( dst->backend != GGML_BACKEND_TYPE_GPU_SPLIT); ggml_tensor_extra_gpu * src0_extra = (ggml_tensor_extra_gpu *) src0->extra; ggml_tensor_extra_gpu * src1_extra = use_src1 ? (ggml_tensor_extra_gpu *) src1->extra : nullptr; ggml_tensor_extra_gpu * dst_extra = (ggml_tensor_extra_gpu *) dst->extra; - const bool src0_on_device = src0->backend == GGML_BACKEND_GPU || src0->backend == GGML_BACKEND_GPU_SPLIT; - const bool src1_on_device = use_src1 && src1->backend == GGML_BACKEND_GPU; - const bool dst_on_device = dst->backend == GGML_BACKEND_GPU; + const bool src0_on_device = src0->backend == GGML_BACKEND_TYPE_GPU || src0->backend == GGML_BACKEND_TYPE_GPU_SPLIT; + const bool src1_on_device = use_src1 && src1->backend == GGML_BACKEND_TYPE_GPU; + const bool dst_on_device = dst->backend == GGML_BACKEND_TYPE_GPU; // dd = data device float * src0_ddf = nullptr; @@ -9555,7 +9555,7 @@ static void ggml_cuda_op_flatten(const ggml_tensor * src0, const ggml_tensor * s CUDA_CHECK(cudaMemcpyAsync(dst->data, dst_ddf, ggml_nbytes(dst), cudaMemcpyDeviceToHost, main_stream)); } - if (dst->backend == GGML_BACKEND_CPU) { + if (dst->backend == GGML_BACKEND_TYPE_CPU) { CUDA_CHECK(cudaDeviceSynchronize()); } } @@ -9636,8 +9636,8 @@ static void ggml_cuda_op_mul_mat( const int nb2 = dst->nb[2]; const int nb3 = dst->nb[3]; - GGML_ASSERT(dst->backend != GGML_BACKEND_GPU_SPLIT); - GGML_ASSERT(src1->backend != GGML_BACKEND_GPU_SPLIT); + GGML_ASSERT(dst->backend != GGML_BACKEND_TYPE_GPU_SPLIT); + GGML_ASSERT(src1->backend != GGML_BACKEND_TYPE_GPU_SPLIT); GGML_ASSERT(src1->type == GGML_TYPE_F32 || (src1->ne[2] == 1 && src1->ne[3] == 1)); GGML_ASSERT(ne12 >= ne02 && ne12 % ne02 == 0); @@ -9653,20 +9653,20 @@ static void ggml_cuda_op_mul_mat( ggml_tensor_extra_gpu * src1_extra = (ggml_tensor_extra_gpu *) src1->extra; ggml_tensor_extra_gpu * dst_extra = (ggml_tensor_extra_gpu *) dst->extra; - const bool src0_on_device = src0->backend == GGML_BACKEND_GPU || src0->backend == GGML_BACKEND_GPU_SPLIT; + const bool src0_on_device = src0->backend == GGML_BACKEND_TYPE_GPU || src0->backend == GGML_BACKEND_TYPE_GPU_SPLIT; const bool src0_is_contiguous = ggml_is_contiguous(src0); const bool src1_is_contiguous = ggml_is_contiguous(src1); const int64_t src1_padded_col_size = GGML_PAD(ne10, MATRIX_ROW_PADDING); - const bool split = src0->backend == GGML_BACKEND_GPU_SPLIT; + const bool split = src0->backend == GGML_BACKEND_TYPE_GPU_SPLIT; GGML_ASSERT(!(split && ne02 > 1)); GGML_ASSERT(!(split && ne03 > 1)); GGML_ASSERT(!(split && ne02 < ne12)); std::array tensor_split; if (split) { - // TODO: check that src0->buffer->buft is a split buffer type, replace GGML_BACKEND_GPU_SPLIT check + // TODO: check that src0->buffer->buft is a split buffer type, replace GGML_BACKEND_TYPE_GPU_SPLIT check // GGML_ASSERT(src0->buffer != nullptr && src0->buffer->buft == ...); ggml_backend_cuda_split_buffer_type_context * buft_ctx = (ggml_backend_cuda_split_buffer_type_context *) src0->buffer->buft->context; tensor_split = buft_ctx->tensor_split; @@ -9724,8 +9724,8 @@ static void ggml_cuda_op_mul_mat( used_devices++; - const bool src1_on_device = src1->backend == GGML_BACKEND_GPU && id == g_main_device; - const bool dst_on_device = dst->backend == GGML_BACKEND_GPU && id == g_main_device; + const bool src1_on_device = src1->backend == GGML_BACKEND_TYPE_GPU && id == g_main_device; + const bool dst_on_device = dst->backend == GGML_BACKEND_TYPE_GPU && id == g_main_device; ggml_cuda_set_device(id); cudaStream_t stream = g_cudaStreams[id][0]; @@ -9776,8 +9776,8 @@ static void ggml_cuda_op_mul_mat( continue; } - const bool src1_on_device = src1->backend == GGML_BACKEND_GPU && id == g_main_device; - const bool dst_on_device = dst->backend == GGML_BACKEND_GPU && id == g_main_device; + const bool src1_on_device = src1->backend == GGML_BACKEND_TYPE_GPU && id == g_main_device; + const bool dst_on_device = dst->backend == GGML_BACKEND_TYPE_GPU && id == g_main_device; const int64_t row_diff = dev[id].row_high - dev[id].row_low; ggml_cuda_set_device(id); @@ -9802,12 +9802,12 @@ static void ggml_cuda_op_mul_mat( // the main device memory buffer can be on VRAM scratch, with space for all partial results // in that case an offset on dst_ddf_i is needed - if (dst->backend == GGML_BACKEND_GPU && id == g_main_device) { + if (dst->backend == GGML_BACKEND_TYPE_GPU && id == g_main_device) { dst_dd_i += dev[id].row_low; // offset is 0 if no tensor split } // copy src0, src1 to device if necessary - if (src1->backend == GGML_BACKEND_GPU && src1_is_contiguous) { + if (src1->backend == GGML_BACKEND_TYPE_GPU && src1_is_contiguous) { if (id != g_main_device) { if (convert_src1_to_q8_1) { char * src1_ddq_i_source = dev[g_main_device].src1_ddq + src1_ddq_i_offset; @@ -9820,14 +9820,14 @@ static void ggml_cuda_op_mul_mat( src1_ncols*ne10*sizeof(float), stream)); } } - } else if (src1->backend == GGML_BACKEND_CPU || (src1_on_device && !src1_is_contiguous)) { + } else if (src1->backend == GGML_BACKEND_TYPE_CPU || (src1_on_device && !src1_is_contiguous)) { CUDA_CHECK(ggml_cuda_cpy_tensor_2d( src1_ddf_i, src1, i03, i02, src1_col_0, src1_col_0+src1_ncols, stream)); } else { GGML_ASSERT(false); } - if (convert_src1_to_q8_1 && (src1->backend == GGML_BACKEND_CPU || !src1_is_contiguous)) { + if (convert_src1_to_q8_1 && (src1->backend == GGML_BACKEND_TYPE_CPU || !src1_is_contiguous)) { quantize_row_q8_1_cuda(src1_ddf_i, src1_ddq_i, ne10, src1_ncols, src1_padded_col_size, stream); CUDA_CHECK(cudaGetLastError()); } @@ -9845,10 +9845,10 @@ static void ggml_cuda_op_mul_mat( if (!dst_on_device) { void * dst_off_device; cudaMemcpyKind kind; - if (dst->backend == GGML_BACKEND_CPU) { + if (dst->backend == GGML_BACKEND_TYPE_CPU) { dst_off_device = dst->data; kind = cudaMemcpyDeviceToHost; - } else if (dst->backend == GGML_BACKEND_GPU) { + } else if (dst->backend == GGML_BACKEND_TYPE_GPU) { dst_off_device = dst_extra->data_device[g_main_device]; kind = cudaMemcpyDeviceToDevice; } else { @@ -9913,7 +9913,7 @@ static void ggml_cuda_op_mul_mat( } } - if (dst->backend == GGML_BACKEND_CPU) { + if (dst->backend == GGML_BACKEND_TYPE_CPU) { ggml_cuda_set_device(g_main_device); CUDA_CHECK(cudaDeviceSynchronize()); } @@ -10019,7 +10019,7 @@ GGML_CALL bool ggml_cuda_can_mul_mat(const struct ggml_tensor * src0, const stru static void ggml_cuda_mul_mat_vec_p021(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst){ GGML_ASSERT(ggml_is_permuted(src0) && ggml_is_permuted(src1)); - GGML_ASSERT(src0->backend != GGML_BACKEND_GPU_SPLIT); + GGML_ASSERT(src0->backend != GGML_BACKEND_TYPE_GPU_SPLIT); GGML_ASSERT(src0->nb[0] <= src0->nb[1] && src0->nb[2] <= src0->nb[3]); // 0213 permutation GGML_ASSERT(src1->nb[0] <= src1->nb[1] && src1->nb[2] <= src1->nb[3]); // 0213 permutation GGML_ASSERT(src0->type == GGML_TYPE_F16); @@ -10050,7 +10050,7 @@ static void ggml_cuda_mul_mat_vec_nc(const ggml_tensor * src0, const ggml_tensor GGML_ASSERT(!ggml_is_transposed(src0)); GGML_ASSERT(!ggml_is_transposed(src1)); GGML_ASSERT(!ggml_is_permuted(src0)); - GGML_ASSERT(src0->backend != GGML_BACKEND_GPU_SPLIT); + GGML_ASSERT(src0->backend != GGML_BACKEND_TYPE_GPU_SPLIT); GGML_ASSERT(src0->type == GGML_TYPE_F16); GGML_ASSERT(src1->type == GGML_TYPE_F32); @@ -10109,7 +10109,7 @@ static void ggml_cuda_mul_mat_batched_cublas(const ggml_tensor * src0, const ggm GGML_ASSERT(!ggml_is_transposed(src0)); GGML_ASSERT(!ggml_is_transposed(src1)); - GGML_ASSERT(src0->backend != GGML_BACKEND_GPU_SPLIT); + GGML_ASSERT(src0->backend != GGML_BACKEND_TYPE_GPU_SPLIT); GGML_ASSERT(src0->type == GGML_TYPE_F16); GGML_TENSOR_BINARY_OP_LOCALS @@ -10255,11 +10255,11 @@ static void ggml_cuda_mul_mat_batched_cublas(const ggml_tensor * src0, const ggm static void ggml_cuda_mul_mat(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { const bool all_on_device = - (src0->backend == GGML_BACKEND_GPU || src0->backend == GGML_BACKEND_GPU_SPLIT) && - (src1->backend == GGML_BACKEND_GPU) && - ( dst->backend == GGML_BACKEND_GPU); + (src0->backend == GGML_BACKEND_TYPE_GPU || src0->backend == GGML_BACKEND_TYPE_GPU_SPLIT) && + (src1->backend == GGML_BACKEND_TYPE_GPU) && + ( dst->backend == GGML_BACKEND_TYPE_GPU); - const bool split = src0->backend == GGML_BACKEND_GPU_SPLIT; + const bool split = src0->backend == GGML_BACKEND_TYPE_GPU_SPLIT; int64_t min_compute_capability = INT_MAX; @@ -10409,7 +10409,7 @@ static void ggml_cuda_mul_mat_id_cublas(ggml_tensor * dst) { GGML_ASSERT(!ggml_is_transposed(src00)); GGML_ASSERT(!ggml_is_transposed(src1)); - GGML_ASSERT(src00->backend != GGML_BACKEND_GPU_SPLIT); + GGML_ASSERT(src00->backend != GGML_BACKEND_TYPE_GPU_SPLIT); GGML_ASSERT(src1->type == GGML_TYPE_F32); const int64_t ne00 = src00->ne[0]; GGML_UNUSED(ne00); @@ -10553,7 +10553,7 @@ static void ggml_cuda_mul_mat_id(const ggml_tensor * src0, const ggml_tensor * s cudaStream_t stream = g_cudaStreams[g_main_device][0]; - if (ids->backend == GGML_BACKEND_GPU) { + if (ids->backend == GGML_BACKEND_TYPE_GPU) { const char * ids_dev = (const char *)((const ggml_tensor_extra_gpu *)ids->extra)->data_device[g_main_device]; CUDA_CHECK(cudaMemcpyAsync(ids_host.data(), ids_dev, ggml_nbytes(ids), cudaMemcpyDeviceToHost, stream)); CUDA_CHECK(cudaStreamSynchronize(stream)); @@ -10570,20 +10570,20 @@ static void ggml_cuda_mul_mat_id(const ggml_tensor * src0, const ggml_tensor * s ggml_tensor src1_row = *src1; ggml_tensor dst_row = *dst; - src1_row.backend = GGML_BACKEND_GPU; - dst_row.backend = GGML_BACKEND_GPU; + src1_row.backend = GGML_BACKEND_TYPE_GPU; + dst_row.backend = GGML_BACKEND_TYPE_GPU; src1_row.extra = &src1_row_extra; dst_row.extra = &dst_row_extra; - char * src1_original = src1->backend == GGML_BACKEND_CPU ? + char * src1_original = src1->backend == GGML_BACKEND_TYPE_CPU ? (char *) src1->data : (char *) src1_extra->data_device[g_main_device]; - char * dst_original = dst->backend == GGML_BACKEND_CPU ? + char * dst_original = dst->backend == GGML_BACKEND_TYPE_CPU ? (char *) dst->data : (char *) dst_extra->data_device[g_main_device]; if (src1->ne[1] == 1) { - GGML_ASSERT(src1->backend == GGML_BACKEND_GPU); - GGML_ASSERT(dst->backend == GGML_BACKEND_GPU); + GGML_ASSERT(src1->backend == GGML_BACKEND_TYPE_GPU); + GGML_ASSERT(dst->backend == GGML_BACKEND_TYPE_GPU); for (int64_t i01 = 0; i01 < ids->ne[1]; i01++) { //int32_t row_id; @@ -10611,9 +10611,9 @@ static void ggml_cuda_mul_mat_id(const ggml_tensor * src0, const ggml_tensor * s src1_row_extra.data_device[g_main_device] = src1_contiguous.get(); dst_row_extra.data_device[g_main_device] = dst_contiguous.get(); - const cudaMemcpyKind src1_kind = src1->backend == GGML_BACKEND_CPU ? + const cudaMemcpyKind src1_kind = src1->backend == GGML_BACKEND_TYPE_CPU ? cudaMemcpyHostToDevice : cudaMemcpyDeviceToDevice; - const cudaMemcpyKind dst_kind = dst->backend == GGML_BACKEND_CPU ? + const cudaMemcpyKind dst_kind = dst->backend == GGML_BACKEND_TYPE_CPU ? cudaMemcpyDeviceToHost : cudaMemcpyDeviceToDevice; for (int32_t row_id = 0; row_id < n_as; ++row_id) { @@ -10668,7 +10668,7 @@ static void ggml_cuda_mul_mat_id(const ggml_tensor * src0, const ggml_tensor * s } } - if (dst->backend == GGML_BACKEND_CPU) { + if (dst->backend == GGML_BACKEND_TYPE_CPU) { CUDA_CHECK(cudaStreamSynchronize(stream)); } } @@ -10685,8 +10685,8 @@ static void ggml_cuda_cpy(const ggml_tensor * src0, const ggml_tensor * src1, gg const int64_t ne = ggml_nelements(src0); GGML_ASSERT(ne == ggml_nelements(src1)); - GGML_ASSERT(src0->backend == GGML_BACKEND_GPU); - GGML_ASSERT(src1->backend == GGML_BACKEND_GPU); + GGML_ASSERT(src0->backend == GGML_BACKEND_TYPE_GPU); + GGML_ASSERT(src1->backend == GGML_BACKEND_TYPE_GPU); GGML_ASSERT(ggml_nbytes(src0) <= INT_MAX); GGML_ASSERT(ggml_nbytes(src1) <= INT_MAX); @@ -10817,9 +10817,9 @@ GGML_CALL bool ggml_cuda_compute_forward(struct ggml_compute_params * params, st if (!g_cublas_loaded) return false; ggml_cuda_func_t func; - const bool any_on_device = tensor->backend == GGML_BACKEND_GPU - || (tensor->src[0] != nullptr && (tensor->src[0]->backend == GGML_BACKEND_GPU || tensor->src[0]->backend == GGML_BACKEND_GPU_SPLIT)) - || (tensor->src[1] != nullptr && tensor->src[1]->backend == GGML_BACKEND_GPU); + const bool any_on_device = tensor->backend == GGML_BACKEND_TYPE_GPU + || (tensor->src[0] != nullptr && (tensor->src[0]->backend == GGML_BACKEND_TYPE_GPU || tensor->src[0]->backend == GGML_BACKEND_TYPE_GPU_SPLIT)) + || (tensor->src[1] != nullptr && tensor->src[1]->backend == GGML_BACKEND_TYPE_GPU); if (!any_on_device && tensor->op != GGML_OP_MUL_MAT && tensor->op != GGML_OP_MUL_MAT_ID) { return false; @@ -10966,14 +10966,14 @@ GGML_CALL bool ggml_cuda_compute_forward(struct ggml_compute_params * params, st return false; } - if (tensor->src[0] != nullptr && tensor->src[0]->backend == GGML_BACKEND_GPU_SPLIT) { + if (tensor->src[0] != nullptr && tensor->src[0]->backend == GGML_BACKEND_TYPE_GPU_SPLIT) { ggml_cuda_set_peer_access(tensor->src[1]->ne[1]); } if (params->ith != 0) { return true; } - if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) { + if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) { return true; } func(tensor->src[0], tensor->src[1], tensor); @@ -11072,7 +11072,7 @@ GGML_CALL static void ggml_backend_cuda_buffer_init_tensor(ggml_backend_buffer_t extra->data_device[ctx->device] = tensor->data; - tensor->backend = GGML_BACKEND_GPU; + tensor->backend = GGML_BACKEND_TYPE_GPU; tensor->extra = extra; if (ggml_is_quantized(tensor->type)) { @@ -11087,7 +11087,7 @@ GGML_CALL static void ggml_backend_cuda_buffer_init_tensor(ggml_backend_buffer_t } GGML_CALL static void ggml_backend_cuda_buffer_set_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor, const void * data, size_t offset, size_t size) { - GGML_ASSERT(tensor->backend == GGML_BACKEND_GPU); + GGML_ASSERT(tensor->backend == GGML_BACKEND_TYPE_GPU); ggml_backend_cuda_buffer_context * ctx = (ggml_backend_cuda_buffer_context *)buffer->context; @@ -11098,7 +11098,7 @@ GGML_CALL static void ggml_backend_cuda_buffer_set_tensor(ggml_backend_buffer_t } GGML_CALL static void ggml_backend_cuda_buffer_get_tensor(ggml_backend_buffer_t buffer, const ggml_tensor * tensor, void * data, size_t offset, size_t size) { - GGML_ASSERT(tensor->backend == GGML_BACKEND_GPU); + GGML_ASSERT(tensor->backend == GGML_BACKEND_TYPE_GPU); ggml_backend_cuda_buffer_context * ctx = (ggml_backend_cuda_buffer_context *)buffer->context; @@ -11333,7 +11333,7 @@ GGML_CALL static void ggml_backend_cuda_split_buffer_init_tensor(ggml_backend_bu CUDA_CHECK(cudaEventCreateWithFlags(&extra->events[id][is], cudaEventDisableTiming)); } } - tensor->backend = GGML_BACKEND_GPU_SPLIT; + tensor->backend = GGML_BACKEND_TYPE_GPU_SPLIT; tensor->extra = extra; } @@ -11605,7 +11605,7 @@ GGML_CALL static void ggml_backend_cuda_set_tensor_async(ggml_backend_t backend, ggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *)backend->context; GGML_ASSERT(tensor->buffer->buft == ggml_backend_cuda_buffer_type(cuda_ctx->device) && "unsupported buffer type"); - GGML_ASSERT(tensor->backend == GGML_BACKEND_GPU); + GGML_ASSERT(tensor->backend == GGML_BACKEND_TYPE_GPU); CUDA_CHECK(cudaMemcpyAsync((char *)tensor->data + offset, data, size, cudaMemcpyHostToDevice, g_cudaStreams[cuda_ctx->device][0])); } @@ -11614,7 +11614,7 @@ GGML_CALL static void ggml_backend_cuda_get_tensor_async(ggml_backend_t backend, ggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *)backend->context; GGML_ASSERT(tensor->buffer->buft == ggml_backend_cuda_buffer_type(cuda_ctx->device) && "unsupported buffer type"); - GGML_ASSERT(tensor->backend == GGML_BACKEND_GPU); + GGML_ASSERT(tensor->backend == GGML_BACKEND_TYPE_GPU); CUDA_CHECK(cudaMemcpyAsync(data, (const char *)tensor->data + offset, size, cudaMemcpyDeviceToHost, g_cudaStreams[cuda_ctx->device][0])); } @@ -11644,7 +11644,7 @@ GGML_CALL static bool ggml_backend_cuda_graph_compute(ggml_backend_t backend, gg ggml_cuda_set_main_device(cuda_ctx->device); ggml_compute_params params = {}; - params.type = GGML_TASK_COMPUTE; + params.type = GGML_TASK_TYPE_COMPUTE; params.ith = 0; for (int i = 0; i < cgraph->n_nodes; i++) { ggml_tensor * node = cgraph->nodes[i]; @@ -11654,13 +11654,13 @@ GGML_CALL static bool ggml_backend_cuda_graph_compute(ggml_backend_t backend, gg } #ifndef NDEBUG - assert(node->backend == GGML_BACKEND_GPU || node->backend == GGML_BACKEND_GPU_SPLIT); + assert(node->backend == GGML_BACKEND_TYPE_GPU || node->backend == GGML_BACKEND_TYPE_GPU_SPLIT); assert(node->buffer->buft == ggml_backend_cuda_buffer_type(cuda_ctx->device)); assert(node->extra != nullptr); for (int j = 0; j < GGML_MAX_SRC; j++) { if (node->src[j] != nullptr) { - assert(node->src[j]->backend == GGML_BACKEND_GPU || node->src[j]->backend == GGML_BACKEND_GPU_SPLIT); + assert(node->src[j]->backend == GGML_BACKEND_TYPE_GPU || node->src[j]->backend == GGML_BACKEND_TYPE_GPU_SPLIT); assert(node->src[j]->buffer->buft == ggml_backend_cuda_buffer_type(cuda_ctx->device) || ggml_backend_buffer_is_cuda_split(node->src[j]->buffer)); assert(node->src[j]->extra != nullptr); } diff --git a/ggml-metal.m b/ggml-metal.m index ee584cfa7..3d6b01263 100644 --- a/ggml-metal.m +++ b/ggml-metal.m @@ -2262,8 +2262,8 @@ static bool ggml_metal_graph_compute( id pipeline = nil; switch (order) { - case GGML_SORT_ASC: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_ARGSORT_F32_I32_ASC].pipeline; break; - case GGML_SORT_DESC: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_ARGSORT_F32_I32_DESC].pipeline; break; + case GGML_SORT_ORDER_ASC: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_ARGSORT_F32_I32_ASC].pipeline; break; + case GGML_SORT_ORDER_DESC: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_ARGSORT_F32_I32_DESC].pipeline; break; default: GGML_ASSERT(false); }; diff --git a/ggml-opencl.cpp b/ggml-opencl.cpp index 797bee667..df619a884 100644 --- a/ggml-opencl.cpp +++ b/ggml-opencl.cpp @@ -1354,7 +1354,7 @@ static void ggml_cl_pool_free(cl_mem mem, size_t size) { } void ggml_cl_free_data(const struct ggml_tensor* tensor) { - if (tensor->backend != GGML_BACKEND_GPU) { + if (tensor->backend != GGML_BACKEND_TYPE_GPU) { return; } @@ -1412,7 +1412,7 @@ static cl_int ggml_cl_h2d_tensor_2d(cl_command_queue queue, cl_mem dst, size_t o } static void ggml_cl_mul_f32(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { - GGML_ASSERT(src1->backend == GGML_BACKEND_GPU); + GGML_ASSERT(src1->backend == GGML_BACKEND_TYPE_GPU); const int64_t ne00 = src0->ne[0]; const int64_t ne01 = src0->ne[1]; const int64_t ne02 = src0->ne[2]; @@ -1476,7 +1476,7 @@ void ggml_cl_mul(const struct ggml_tensor * src0, const struct ggml_tensor * src } static void ggml_cl_add_f32(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { - GGML_ASSERT(src1->backend == GGML_BACKEND_GPU); + GGML_ASSERT(src1->backend == GGML_BACKEND_TYPE_GPU); const int64_t ne00 = src0->ne[0]; const int64_t ne01 = src0->ne[1]; const int64_t ne02 = src0->ne[2]; @@ -1566,13 +1566,13 @@ static void ggml_cl_mul_mat_f32(const ggml_tensor * src0, const ggml_tensor * sr size_t y_size; size_t d_size; cl_mem d_X; - if (src0->backend == GGML_BACKEND_GPU) { // NOLINT + if (src0->backend == GGML_BACKEND_TYPE_GPU) { // NOLINT d_X = (cl_mem) src0->extra; } else { d_X = ggml_cl_pool_malloc(sizeof(float) * x_ne, &x_size); } - cl_mem d_Y = src1->backend == GGML_BACKEND_GPU ? (cl_mem) src1->extra : ggml_cl_pool_malloc(sizeof(float) * y_ne, &y_size); - cl_mem d_D = dst->backend == GGML_BACKEND_GPU ? (cl_mem) dst->extra : ggml_cl_pool_malloc(sizeof(float) * d_ne, &d_size); + cl_mem d_Y = src1->backend == GGML_BACKEND_TYPE_GPU ? (cl_mem) src1->extra : ggml_cl_pool_malloc(sizeof(float) * y_ne, &y_size); + cl_mem d_D = dst->backend == GGML_BACKEND_TYPE_GPU ? (cl_mem) dst->extra : ggml_cl_pool_malloc(sizeof(float) * d_ne, &d_size); size_t x_offset = 0; @@ -1580,7 +1580,7 @@ static void ggml_cl_mul_mat_f32(const ggml_tensor * src0, const ggml_tensor * sr // TODO: copy src0 here when r3>1 for (int64_t i13 = i03 * r3, e13 = i13 + r3; i13 < e13; i13++) { for (int64_t i02 = 0; i02 < ne02; i02++) { - if (src0->backend == GGML_BACKEND_GPU) { + if (src0->backend == GGML_BACKEND_TYPE_GPU) { x_offset = (i03 * ne02 + i02) * x_ne; } else { // copy src0 to device @@ -1589,7 +1589,7 @@ static void ggml_cl_mul_mat_f32(const ggml_tensor * src0, const ggml_tensor * sr for (int64_t i12 = i02 * r2, e12 = i12 + r2; i12 < e12; i12++) { // copy src1 to device - if (src1->backend == GGML_BACKEND_CPU) { + if (src1->backend == GGML_BACKEND_TYPE_CPU) { CL_CHECK(ggml_cl_h2d_tensor_2d(queue, d_Y, 0, src1, i13, i12, NULL)); } @@ -1612,7 +1612,7 @@ static void ggml_cl_mul_mat_f32(const ggml_tensor * src0, const ggml_tensor * sr } // copy dst to host - if (dst->backend == GGML_BACKEND_CPU) { + if (dst->backend == GGML_BACKEND_TYPE_CPU) { float * d = (float *) ((char *) dst->data + i12*nb2 + i13*nb3); CL_CHECK(clEnqueueReadBuffer(queue, d_D, true, 0, sizeof(float) * d_ne, d, 1, &ev_sgemm, NULL)); } @@ -1621,13 +1621,13 @@ static void ggml_cl_mul_mat_f32(const ggml_tensor * src0, const ggml_tensor * sr } } - if (src0->backend != GGML_BACKEND_GPU) { + if (src0->backend != GGML_BACKEND_TYPE_GPU) { ggml_cl_pool_free(d_X, x_size); } - if (src1->backend != GGML_BACKEND_GPU) { + if (src1->backend != GGML_BACKEND_TYPE_GPU) { ggml_cl_pool_free(d_Y, y_size); } - if (dst->backend != GGML_BACKEND_GPU) { + if (dst->backend != GGML_BACKEND_TYPE_GPU) { ggml_cl_pool_free(d_D, d_size); } } @@ -1670,7 +1670,7 @@ static void ggml_cl_mul_mat_f16(const ggml_tensor * src0, const ggml_tensor * sr size_t y_size; size_t d_size; cl_mem d_X; - if (src0->backend == GGML_BACKEND_GPU) { // NOLINT + if (src0->backend == GGML_BACKEND_TYPE_GPU) { // NOLINT d_X = (cl_mem) src0->extra; } else { d_X = ggml_cl_pool_malloc(sizeof(ggml_fp16_t) * x_ne, &x_size); @@ -1687,7 +1687,7 @@ static void ggml_cl_mul_mat_f16(const ggml_tensor * src0, const ggml_tensor * sr // TODO: copy src0 here when r3>1 for (int64_t i13 = i03 * r3, e13 = i13 + r3; i13 < e13; i13++) { for (int64_t i02 = 0; i02 < ne02; i02++) { - if (src0->backend == GGML_BACKEND_GPU) { + if (src0->backend == GGML_BACKEND_TYPE_GPU) { x_offset = (i03 * ne02 + i02) * x_ne; } else { // copy src0 to device @@ -1741,7 +1741,7 @@ static void ggml_cl_mul_mat_f16(const ggml_tensor * src0, const ggml_tensor * sr } // copy dst to host, then convert to float - if (dst->backend == GGML_BACKEND_CPU) { + if (dst->backend == GGML_BACKEND_TYPE_CPU) { CL_CHECK(clEnqueueReadBuffer(queue, d_D, true, 0, sizeof(ggml_fp16_t) * d_ne, tmp, 1, &ev_sgemm, NULL)); float * d = (float *) ((char *) dst->data + i12*nb2 + i13*nb3); ggml_fp16_to_fp32_row(tmp, d, d_ne); @@ -1753,7 +1753,7 @@ static void ggml_cl_mul_mat_f16(const ggml_tensor * src0, const ggml_tensor * sr } } - if (src0->backend != GGML_BACKEND_GPU) { + if (src0->backend != GGML_BACKEND_TYPE_GPU) { ggml_cl_pool_free(d_X, x_size); } ggml_cl_pool_free(d_Y, y_size); @@ -1798,7 +1798,7 @@ static void ggml_cl_mul_mat_q_f32(const ggml_tensor * src0, const ggml_tensor * cl_mem d_Y = ggml_cl_pool_malloc(sizeof(float) * y_ne, &y_size); cl_mem d_D = ggml_cl_pool_malloc(sizeof(float) * d_ne, &d_size); cl_mem d_Q; - if (src0->backend == GGML_BACKEND_CPU) { + if (src0->backend == GGML_BACKEND_TYPE_CPU) { d_Q = ggml_cl_pool_malloc(q_sz, &q_size); } @@ -1817,10 +1817,10 @@ static void ggml_cl_mul_mat_q_f32(const ggml_tensor * src0, const ggml_tensor * for (int64_t i13 = i03 * r3, e13 = i13 + r3; i13 < e13; i13++) { for (int64_t i02 = 0; i02 < ne02; i02++) { // copy src0 to device if necessary - if (src0->backend == GGML_BACKEND_CPU) { + if (src0->backend == GGML_BACKEND_TYPE_CPU) { events.emplace_back(); CL_CHECK(ggml_cl_h2d_tensor_2d(queue, d_Q, 0, src0, i03, i02, events.data() + ev_idx++)); - } else if (src0->backend == GGML_BACKEND_GPU) { + } else if (src0->backend == GGML_BACKEND_TYPE_GPU) { d_Q = (cl_mem) src0->extra; } else { GGML_ASSERT(false); @@ -1829,7 +1829,7 @@ static void ggml_cl_mul_mat_q_f32(const ggml_tensor * src0, const ggml_tensor * if (!mul_mat_vec) { // convert src0 to fp32 on device const size_t global = x_ne / global_denom; - const size_t offset = src0->backend == GGML_BACKEND_GPU ? (i03 * ne02 + i02) * x_bps : 0; + const size_t offset = src0->backend == GGML_BACKEND_TYPE_GPU ? (i03 * ne02 + i02) * x_bps : 0; CL_CHECK(clSetKernelArg(*to_fp32_cl, 0, sizeof(cl_mem), &d_Q)); CL_CHECK(clSetKernelArg(*to_fp32_cl, 1, sizeof(cl_mem), &d_X)); CL_CHECK(clEnqueueNDRangeKernel(queue, *to_fp32_cl, 1, &offset, &global, local > 0 ? &local : NULL, events.size(), !events.empty() ? events.data() : NULL, NULL)); @@ -1843,7 +1843,7 @@ static void ggml_cl_mul_mat_q_f32(const ggml_tensor * src0, const ggml_tensor * // compute const size_t global = ne01 * local; - const size_t offset = src0->backend == GGML_BACKEND_GPU ? (i03 * ne02 + i02) * x_bps : 0; + const size_t offset = src0->backend == GGML_BACKEND_TYPE_GPU ? (i03 * ne02 + i02) * x_bps : 0; const cl_int ncols = ne00; events.emplace_back(); CL_CHECK(clSetKernelArg(*dmmv, 0, sizeof(cl_mem), &d_Q)); @@ -1895,7 +1895,7 @@ static void ggml_cl_mul_mat_q_f32(const ggml_tensor * src0, const ggml_tensor * } ggml_cl_pool_free(d_Y, y_size); ggml_cl_pool_free(d_D, d_size); - if (src0->backend == GGML_BACKEND_CPU) { + if (src0->backend == GGML_BACKEND_TYPE_CPU) { ggml_cl_pool_free(d_Q, q_size); } } @@ -1911,7 +1911,7 @@ bool ggml_cl_can_mul_mat(const struct ggml_tensor * src0, const struct ggml_tens if ((src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16 || ggml_is_quantized(src0->type)) && src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32 && - ((ne0 >= 32 && ne1 >= 32 && ne10 >= 32) || src0->backend == GGML_BACKEND_GPU)) { + ((ne0 >= 32 && ne1 >= 32 && ne10 >= 32) || src0->backend == GGML_BACKEND_TYPE_GPU)) { return true; } @@ -1993,7 +1993,7 @@ void ggml_cl_transform_tensor(void * data, ggml_tensor * tensor) { CL_CHECK(clFinish(queue)); tensor->extra = dst; - GGML_ASSERT(tensor->backend == GGML_BACKEND_GPU); + GGML_ASSERT(tensor->backend == GGML_BACKEND_TYPE_GPU); } // ggml-backend @@ -2045,7 +2045,7 @@ static void ggml_backend_opencl_buffer_init_tensor(ggml_backend_buffer_t buffer, ctx->sub_buffers.push_back(sub_buffer); tensor->extra = sub_buffer; } - tensor->backend = GGML_BACKEND_GPU; + tensor->backend = GGML_BACKEND_TYPE_GPU; } static void ggml_backend_opencl_buffer_set_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor, const void * data, size_t offset, size_t size) { diff --git a/ggml-sycl.cpp b/ggml-sycl.cpp index b897828f9..c6c3c6e6f 100644 --- a/ggml-sycl.cpp +++ b/ggml-sycl.cpp @@ -3338,7 +3338,7 @@ void print_ggml_tensor(const char*name, struct ggml_tensor *src){ size_t total_elements = ggml_nelements(src); - const bool src_on_device = src->backend == GGML_BACKEND_GPU || src->backend == GGML_BACKEND_GPU_SPLIT; + const bool src_on_device = src->backend == GGML_BACKEND_TYPE_GPU || src->backend == GGML_BACKEND_TYPE_GPU_SPLIT; float *src_data =NULL; if(src_on_device) { ggml_tensor_extra_gpu * src_extra = (ggml_tensor_extra_gpu *) src->extra; @@ -8086,11 +8086,11 @@ static void k_argsort_f32_i32(const float * x, int * dst, const int ncols, int ixj = col ^ j; if (ixj > col) { if ((col & k) == 0) { - if (order == GGML_SORT_ASC ? x_row[dst_row[col]] > x_row[dst_row[ixj]] : x_row[dst_row[col]] < x_row[dst_row[ixj]]) { + if (order == GGML_SORT_ORDER_ASC ? x_row[dst_row[col]] > x_row[dst_row[ixj]] : x_row[dst_row[col]] < x_row[dst_row[ixj]]) { swap(dst_row[col], dst_row[ixj]); } } else { - if (order == GGML_SORT_ASC ? x_row[dst_row[col]] < x_row[dst_row[ixj]] : x_row[dst_row[col]] > x_row[dst_row[ixj]]) { + if (order == GGML_SORT_ORDER_ASC ? x_row[dst_row[col]] < x_row[dst_row[ixj]] : x_row[dst_row[col]] > x_row[dst_row[ixj]]) { swap(dst_row[col], dst_row[ixj]); } } @@ -10825,7 +10825,7 @@ static void argsort_f32_i32_sycl(const float *x, int *dst, const int ncols, const sycl::range<3> block_dims(1, 1, ncols); const sycl::range<3> block_nums(1, nrows, 1); - if (order == GGML_SORT_ASC) { + if (order == GGML_SORT_ORDER_ASC) { /* DPCT1049:44: The work-group size passed to the SYCL kernel may exceed the limit. To get the device limit, query @@ -10834,9 +10834,9 @@ static void argsort_f32_i32_sycl(const float *x, int *dst, const int ncols, stream->parallel_for( sycl::nd_range<3>(block_nums * block_dims, block_dims), [=](sycl::nd_item<3> item_ct1) { - k_argsort_f32_i32(x, dst, ncols, item_ct1); + k_argsort_f32_i32(x, dst, ncols, item_ct1); }); - } else if (order == GGML_SORT_DESC) { + } else if (order == GGML_SORT_ORDER_DESC) { /* DPCT1049:45: The work-group size passed to the SYCL kernel may exceed the limit. To get the device limit, query @@ -10845,7 +10845,7 @@ static void argsort_f32_i32_sycl(const float *x, int *dst, const int ncols, stream->parallel_for( sycl::nd_range<3>(block_nums * block_dims, block_dims), [=](sycl::nd_item<3> item_ct1) { - k_argsort_f32_i32(x, dst, ncols, item_ct1); + k_argsort_f32_i32(x, dst, ncols, item_ct1); }); } else { GGML_ASSERT(false); @@ -11407,12 +11407,12 @@ static dpct::err0 ggml_sycl_cpy_tensor_2d(void *dst, dpct::memcpy_direction kind; char * src_ptr; - if (src->backend == GGML_BACKEND_CPU) { + if (src->backend == GGML_BACKEND_TYPE_CPU) { kind = dpct::host_to_device; src_ptr = (char *) src->data; - // GGML_SYCL_DEBUG("ggml_sycl_cpy_tensor_2d GGML_BACKEND_CPU src_ptr %p\n", src_ptr); - } else if (src->backend == GGML_BACKEND_GPU || src->backend == GGML_BACKEND_GPU_SPLIT) { - GGML_ASSERT(src->backend != GGML_BACKEND_GPU_SPLIT || (i1_low == 0 && i1_high == src->ne[1])); + // GGML_SYCL_DEBUG("ggml_sycl_cpy_tensor_2d GGML_BACKEND_TYPE_CPU src_ptr %p\n", src_ptr); + } else if (src->backend == GGML_BACKEND_TYPE_GPU || src->backend == GGML_BACKEND_TYPE_GPU_SPLIT) { + GGML_ASSERT(src->backend != GGML_BACKEND_TYPE_GPU_SPLIT || (i1_low == 0 && i1_high == src->ne[1])); kind = dpct::device_to_device; ggml_tensor_extra_gpu * extra = (ggml_tensor_extra_gpu *) src->extra; int id; @@ -11846,7 +11846,7 @@ inline void ggml_sycl_op_mul_mat_q( // the main device has a larger memory buffer to hold the results from all GPUs // nrows_dst == nrows of the matrix that the dequantize_mul_mat kernel writes into - const int64_t nrows_dst = dst->backend == GGML_BACKEND_GPU && device_id == g_main_device ? ne0 : row_diff; + const int64_t nrows_dst = dst->backend == GGML_BACKEND_TYPE_GPU && device_id == g_main_device ? ne0 : row_diff; switch (src0->type) { case GGML_TYPE_Q4_0: @@ -12119,7 +12119,7 @@ inline void ggml_sycl_op_mul_mat_sycl( // the main device has a larger memory buffer to hold the results from all GPUs // ldc == nrows of the matrix that cuBLAS writes into - int ldc = dst->backend == GGML_BACKEND_GPU && device_id == g_main_device ? ne0 : row_diff; + int ldc = dst->backend == GGML_BACKEND_TYPE_GPU && device_id == g_main_device ? ne0 : row_diff; #ifdef GGML_SYCL_F16 bool use_fp16 = true; // TODO(Yu) SYCL capability check @@ -12501,16 +12501,16 @@ static void ggml_sycl_op_flatten(const ggml_tensor *src0, const bool use_src1 = src1 != nullptr; const int64_t nrows1 = use_src1 ? ggml_nrows(src1) : 1; - GGML_ASSERT(!use_src1 || src1->backend != GGML_BACKEND_GPU_SPLIT); - GGML_ASSERT( dst->backend != GGML_BACKEND_GPU_SPLIT); + GGML_ASSERT(!use_src1 || src1->backend != GGML_BACKEND_TYPE_GPU_SPLIT); + GGML_ASSERT( dst->backend != GGML_BACKEND_TYPE_GPU_SPLIT); ggml_tensor_extra_gpu * src0_extra = (ggml_tensor_extra_gpu *) src0->extra; ggml_tensor_extra_gpu * src1_extra = use_src1 ? (ggml_tensor_extra_gpu *) src1->extra : nullptr; ggml_tensor_extra_gpu * dst_extra = (ggml_tensor_extra_gpu *) dst->extra; - const bool src0_on_device = src0->backend == GGML_BACKEND_GPU || src0->backend == GGML_BACKEND_GPU_SPLIT; - const bool src1_on_device = use_src1 && src1->backend == GGML_BACKEND_GPU; - const bool dst_on_device = dst->backend == GGML_BACKEND_GPU; + const bool src0_on_device = src0->backend == GGML_BACKEND_TYPE_GPU || src0->backend == GGML_BACKEND_TYPE_GPU_SPLIT; + const bool src1_on_device = use_src1 && src1->backend == GGML_BACKEND_TYPE_GPU; + const bool dst_on_device = dst->backend == GGML_BACKEND_TYPE_GPU; // dd = data device float * src0_ddf = nullptr; @@ -12565,7 +12565,7 @@ static void ggml_sycl_op_flatten(const ggml_tensor *src0, main_stream->memcpy(dst->data, dst_ddf, ggml_nbytes(dst)))); } - if (dst->backend == GGML_BACKEND_CPU) { + if (dst->backend == GGML_BACKEND_TYPE_CPU) { SYCL_CHECK(CHECK_TRY_ERROR( dpct::get_current_device().queues_wait_and_throw())); } @@ -12640,8 +12640,8 @@ static void ggml_sycl_op_mul_mat(const ggml_tensor *src0, const int nb2 = dst->nb[2]; const int nb3 = dst->nb[3]; - GGML_ASSERT(dst->backend != GGML_BACKEND_GPU_SPLIT); - GGML_ASSERT(src1->backend != GGML_BACKEND_GPU_SPLIT); + GGML_ASSERT(dst->backend != GGML_BACKEND_TYPE_GPU_SPLIT); + GGML_ASSERT(src1->backend != GGML_BACKEND_TYPE_GPU_SPLIT); GGML_ASSERT(ne12 >= ne02 && ne12 % ne02 == 0); @@ -12656,13 +12656,13 @@ static void ggml_sycl_op_mul_mat(const ggml_tensor *src0, ggml_tensor_extra_gpu * src1_extra = (ggml_tensor_extra_gpu *) src1->extra; ggml_tensor_extra_gpu * dst_extra = (ggml_tensor_extra_gpu *) dst->extra; - const bool src0_on_device = src0->backend == GGML_BACKEND_GPU || src0->backend == GGML_BACKEND_GPU_SPLIT; + const bool src0_on_device = src0->backend == GGML_BACKEND_TYPE_GPU || src0->backend == GGML_BACKEND_TYPE_GPU_SPLIT; const bool src0_is_contiguous = ggml_is_contiguous(src0); const bool src1_is_contiguous = ggml_is_contiguous(src1); int64_t src1_padded_col_size = GGML_PAD(ne10, MATRIX_ROW_PADDING); - const bool split = src0->backend == GGML_BACKEND_GPU_SPLIT; + const bool split = src0->backend == GGML_BACKEND_TYPE_GPU_SPLIT; GGML_ASSERT(!(split && ne02 > 1)); GGML_ASSERT(!(split && ne03 > 1)); GGML_ASSERT(!(split && ne02 < ne12)); @@ -12717,8 +12717,8 @@ static void ggml_sycl_op_mul_mat(const ggml_tensor *src0, used_devices++; - const bool src1_on_device = src1->backend == GGML_BACKEND_GPU && id == g_main_device_index; - const bool dst_on_device = dst->backend == GGML_BACKEND_GPU && id == g_main_device_index; + const bool src1_on_device = src1->backend == GGML_BACKEND_TYPE_GPU && id == g_main_device_index; + const bool dst_on_device = dst->backend == GGML_BACKEND_TYPE_GPU && id == g_main_device_index; ggml_sycl_set_device(get_device_id_by_index(id)); const dpct::queue_ptr stream = g_syclStreams[id][0]; @@ -12782,8 +12782,8 @@ static void ggml_sycl_op_mul_mat(const ggml_tensor *src0, continue; } - const bool src1_on_device = src1->backend == GGML_BACKEND_GPU && id == g_main_device_index; - const bool dst_on_device = dst->backend == GGML_BACKEND_GPU && id == g_main_device_index; + const bool src1_on_device = src1->backend == GGML_BACKEND_TYPE_GPU && id == g_main_device_index; + const bool dst_on_device = dst->backend == GGML_BACKEND_TYPE_GPU && id == g_main_device_index; const int64_t row_diff = row_high[id] - row_low[id]; ggml_sycl_set_device(get_device_id_by_index(id)); @@ -12809,12 +12809,12 @@ static void ggml_sycl_op_mul_mat(const ggml_tensor *src0, // the main device memory buffer can be on VRAM scratch, with space for all partial results // in that case an offset on dst_ddf_i is needed - if (dst->backend == GGML_BACKEND_GPU && id == g_main_device_index) { + if (dst->backend == GGML_BACKEND_TYPE_GPU && id == g_main_device_index) { dst_dd_i += row_low[id]; // offset is 0 if no tensor split } // copy src0, src1 to device if necessary - if (src1->backend == GGML_BACKEND_GPU && src1_is_contiguous) { + if (src1->backend == GGML_BACKEND_TYPE_GPU && src1_is_contiguous) { if (id != g_main_device_index) { if (convert_src1_to_q8_1) { char * src1_ddq_i_source = src1_ddq[g_main_device_index] + src1_ddq_i_offset; @@ -12830,14 +12830,14 @@ static void ggml_sycl_op_mul_mat(const ggml_tensor *src0, src1_ncols * ne10 * sizeof(float)))); } } - } else if (src1->backend == GGML_BACKEND_CPU || (src1_on_device && !src1_is_contiguous)) { + } else if (src1->backend == GGML_BACKEND_TYPE_CPU || (src1_on_device && !src1_is_contiguous)) { SYCL_CHECK(ggml_sycl_cpy_tensor_2d( src1_ddf_i, src1, i03, i02, src1_col_0, src1_col_0+src1_ncols, stream)); } else { GGML_ASSERT(false); } - if (convert_src1_to_q8_1 && (src1->backend == GGML_BACKEND_CPU || !src1_is_contiguous)) { + if (convert_src1_to_q8_1 && (src1->backend == GGML_BACKEND_TYPE_CPU || !src1_is_contiguous)) { quantize_row_q8_1_sycl(src1_ddf_i, src1_ddq_i, ne10, src1_ncols, src1_padded_col_size, stream); /* DPCT1010:92: SYCL uses exceptions to report errors and does @@ -12867,10 +12867,10 @@ static void ggml_sycl_op_mul_mat(const ggml_tensor *src0, if (!dst_on_device) { void * dst_off_device; dpct::memcpy_direction kind; - if (dst->backend == GGML_BACKEND_CPU) { + if (dst->backend == GGML_BACKEND_TYPE_CPU) { dst_off_device = dst->data; kind = dpct::device_to_host; - } else if (dst->backend == GGML_BACKEND_GPU) { + } else if (dst->backend == GGML_BACKEND_TYPE_GPU) { dst_off_device = dst_extra->data_device[g_main_device_index]; kind = dpct::device_to_device; } else { @@ -12954,7 +12954,7 @@ static void ggml_sycl_op_mul_mat(const ggml_tensor *src0, } } - if (dst->backend == GGML_BACKEND_CPU) { + if (dst->backend == GGML_BACKEND_TYPE_CPU) { SYCL_CHECK(ggml_sycl_set_device(g_main_device)); SYCL_CHECK(CHECK_TRY_ERROR( dpct::get_current_device().queues_wait_and_throw())); @@ -13091,7 +13091,7 @@ static void ggml_sycl_mul_mat_vec_p021(const ggml_tensor *src0, const ggml_tensor *src1, ggml_tensor *dst) try { GGML_ASSERT(ggml_is_permuted(src0) && ggml_is_permuted(src1)); - GGML_ASSERT(src0->backend != GGML_BACKEND_GPU_SPLIT); + GGML_ASSERT(src0->backend != GGML_BACKEND_TYPE_GPU_SPLIT); GGML_ASSERT(src0->nb[0] <= src0->nb[1] && src0->nb[2] <= src0->nb[3]); // 0213 permutation GGML_ASSERT(src1->nb[0] <= src1->nb[1] && src1->nb[2] <= src1->nb[3]); // 0213 permutation GGML_ASSERT(src0->type == GGML_TYPE_F16); @@ -13129,7 +13129,7 @@ static void ggml_sycl_mul_mat_vec_nc(const ggml_tensor *src0, GGML_ASSERT(!ggml_is_transposed(src0)); GGML_ASSERT(!ggml_is_transposed(src1)); GGML_ASSERT(!ggml_is_permuted(src0)); - GGML_ASSERT(src0->backend != GGML_BACKEND_GPU_SPLIT); + GGML_ASSERT(src0->backend != GGML_BACKEND_TYPE_GPU_SPLIT); GGML_ASSERT(src0->type == GGML_TYPE_F16); GGML_ASSERT(src1->type == GGML_TYPE_F32); @@ -13196,7 +13196,7 @@ static void ggml_sycl_mul_mat_mat_batched_sycl(const ggml_tensor *src0, GGML_ASSERT(!ggml_is_transposed(src0)); GGML_ASSERT(!ggml_is_transposed(src1)); - GGML_ASSERT(src0->backend != GGML_BACKEND_GPU_SPLIT); + GGML_ASSERT(src0->backend != GGML_BACKEND_TYPE_GPU_SPLIT); GGML_ASSERT(src0->type == GGML_TYPE_F16); GGML_ASSERT(src1->type == GGML_TYPE_F32); @@ -13372,11 +13372,11 @@ catch (sycl::exception const &exc) { static void ggml_sycl_mul_mat(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { const bool all_on_device = - (src0->backend == GGML_BACKEND_GPU || src0->backend == GGML_BACKEND_GPU_SPLIT) && - (src1->backend == GGML_BACKEND_GPU) && - ( dst->backend == GGML_BACKEND_GPU); + (src0->backend == GGML_BACKEND_TYPE_GPU || src0->backend == GGML_BACKEND_TYPE_GPU_SPLIT) && + (src1->backend == GGML_BACKEND_TYPE_GPU) && + ( dst->backend == GGML_BACKEND_TYPE_GPU); - const bool split = src0->backend == GGML_BACKEND_GPU_SPLIT; + const bool split = src0->backend == GGML_BACKEND_TYPE_GPU_SPLIT; int64_t min_compute_capability = INT_MAX; for (int64_t id = 0; id < g_device_count; ++id) { @@ -13505,7 +13505,7 @@ static void ggml_sycl_mul_mat_id_sycl(ggml_tensor * dst) { GGML_ASSERT(!ggml_is_transposed(src00)); GGML_ASSERT(!ggml_is_transposed(src1)); - GGML_ASSERT(src00->backend != GGML_BACKEND_GPU_SPLIT); + GGML_ASSERT(src00->backend != GGML_BACKEND_TYPE_GPU_SPLIT); GGML_ASSERT(src1->type == GGML_TYPE_F32); GGML_TENSOR_LOCALS(int64_t, ne0, src00, ne); @@ -13643,7 +13643,7 @@ static void ggml_sycl_mul_mat_id(const ggml_tensor *src0, const dpct::queue_ptr stream = g_syclStreams[g_main_device_index][0]; - if (ids->backend == GGML_BACKEND_GPU) { + if (ids->backend == GGML_BACKEND_TYPE_GPU) { const char * ids_dev = (const char *)((const ggml_tensor_extra_gpu *)ids->extra)->data_device[g_main_device_index]; SYCL_CHECK(CHECK_TRY_ERROR( stream->memcpy(ids_host.data(), ids_dev, ggml_nbytes(ids)))); @@ -13661,20 +13661,20 @@ static void ggml_sycl_mul_mat_id(const ggml_tensor *src0, ggml_tensor src1_row = *src1; ggml_tensor dst_row = *dst; - src1_row.backend = GGML_BACKEND_GPU; - dst_row.backend = GGML_BACKEND_GPU; + src1_row.backend = GGML_BACKEND_TYPE_GPU; + dst_row.backend = GGML_BACKEND_TYPE_GPU; src1_row.extra = &src1_row_extra; dst_row.extra = &dst_row_extra; - char * src1_original = src1->backend == GGML_BACKEND_CPU ? + char * src1_original = src1->backend == GGML_BACKEND_TYPE_CPU ? (char *) src1->data : (char *) src1_extra->data_device[g_main_device_index]; - char * dst_original = dst->backend == GGML_BACKEND_CPU ? + char * dst_original = dst->backend == GGML_BACKEND_TYPE_CPU ? (char *) dst->data : (char *) dst_extra->data_device[g_main_device_index]; if (src1->ne[1] == 1) { - GGML_ASSERT(src1->backend == GGML_BACKEND_GPU); - GGML_ASSERT(dst->backend == GGML_BACKEND_GPU); + GGML_ASSERT(src1->backend == GGML_BACKEND_TYPE_GPU); + GGML_ASSERT(dst->backend == GGML_BACKEND_TYPE_GPU); for (int64_t i01 = 0; i01 < ids->ne[1]; i01++) { //int32_t row_id; @@ -13756,7 +13756,7 @@ static void ggml_sycl_mul_mat_id(const ggml_tensor *src0, } } - if (dst->backend == GGML_BACKEND_CPU) { + if (dst->backend == GGML_BACKEND_TYPE_CPU) { SYCL_CHECK(CHECK_TRY_ERROR(stream->wait())); } } @@ -13779,8 +13779,8 @@ static void ggml_sycl_cpy(const ggml_tensor *src0, const ggml_tensor *src1, const int64_t ne = ggml_nelements(src0); GGML_ASSERT(ne == ggml_nelements(src1)); - GGML_ASSERT(src0->backend == GGML_BACKEND_GPU); - GGML_ASSERT(src1->backend == GGML_BACKEND_GPU); + GGML_ASSERT(src0->backend == GGML_BACKEND_TYPE_GPU); + GGML_ASSERT(src1->backend == GGML_BACKEND_TYPE_GPU); GGML_ASSERT(ggml_nbytes(src0) <= INT_MAX); GGML_ASSERT(ggml_nbytes(src1) <= INT_MAX); @@ -13887,17 +13887,17 @@ void ggml_sycl_transform_tensor(void *data, struct ggml_tensor *tensor) try { memset(extra, 0, sizeof(*extra)); for (int64_t id = 0; id < g_device_count; ++id) { - if (backend == GGML_BACKEND_GPU && id != g_main_device_index) { + if (backend == GGML_BACKEND_TYPE_GPU && id != g_main_device_index) { continue; } ggml_sycl_set_device(get_device_id_by_index(id)); const dpct::queue_ptr stream = g_syclStreams[id][0]; int64_t row_low, row_high; - if (backend == GGML_BACKEND_GPU) { + if (backend == GGML_BACKEND_TYPE_GPU) { row_low = 0; row_high = nrows; - } else if (backend == GGML_BACKEND_GPU_SPLIT) { + } else if (backend == GGML_BACKEND_TYPE_GPU_SPLIT) { const int64_t rounding = get_row_rounding(tensor->type); row_low = id == 0 ? 0 : nrows*g_tensor_split[id]; @@ -13946,7 +13946,7 @@ void ggml_sycl_transform_tensor(void *data, struct ggml_tensor *tensor) try { extra->data_device[id] = buf; - if (backend == GGML_BACKEND_GPU_SPLIT) { + if (backend == GGML_BACKEND_TYPE_GPU_SPLIT) { for (int64_t is = 0; is < MAX_STREAMS; ++is) { SYCL_CHECK(CHECK_TRY_ERROR(extra->events[id][is] = new sycl::event())); @@ -13963,7 +13963,7 @@ catch (sycl::exception const &exc) { } void ggml_sycl_free_data(struct ggml_tensor *tensor) try { - if (!tensor || !tensor->extra || (tensor->backend != GGML_BACKEND_GPU && tensor->backend != GGML_BACKEND_GPU_SPLIT) ) { + if (!tensor || !tensor->extra || (tensor->backend != GGML_BACKEND_TYPE_GPU && tensor->backend != GGML_BACKEND_TYPE_GPU_SPLIT) ) { return; } @@ -14016,15 +14016,15 @@ static void ggml_sycl_assign_buffers_impl(struct ggml_tensor *tensor, return; } - tensor->backend = GGML_BACKEND_GPU; + tensor->backend = GGML_BACKEND_TYPE_GPU; - if (tensor->src[0] != nullptr && tensor->src[0]->backend == GGML_BACKEND_CPU) { + if (tensor->src[0] != nullptr && tensor->src[0]->backend == GGML_BACKEND_TYPE_CPU) { const ggml_op src0_op = tensor->src[0]->op; if (src0_op == GGML_OP_RESHAPE || src0_op == GGML_OP_TRANSPOSE || src0_op == GGML_OP_VIEW || src0_op == GGML_OP_PERMUTE) { ggml_sycl_assign_buffers_impl(tensor->src[0], scratch, force_inplace, no_alloc); } } - if (tensor->op == GGML_OP_CPY && tensor->src[1]->backend == GGML_BACKEND_CPU) { + if (tensor->op == GGML_OP_CPY && tensor->src[1]->backend == GGML_BACKEND_TYPE_CPU) { ggml_sycl_assign_buffers_impl(tensor->src[1], scratch, force_inplace, no_alloc); } @@ -14042,7 +14042,7 @@ static void ggml_sycl_assign_buffers_impl(struct ggml_tensor *tensor, SYCL_CHECK(ggml_sycl_set_device(g_main_device)); const dpct::queue_ptr stream = g_syclStreams[g_main_device_index][0]; - if (inplace && (tensor->src[0]->backend == GGML_BACKEND_GPU || tensor->src[0]->backend == GGML_BACKEND_GPU_SPLIT)) { + if (inplace && (tensor->src[0]->backend == GGML_BACKEND_TYPE_GPU || tensor->src[0]->backend == GGML_BACKEND_TYPE_GPU_SPLIT)) { ggml_tensor_extra_gpu * src0_extra = (ggml_tensor_extra_gpu * ) tensor->src[0]->extra; char * src0_ddc = (char *) src0_extra->data_device[g_main_device_index]; size_t offset = 0; @@ -14111,7 +14111,7 @@ void ggml_sycl_assign_scratch_offset(struct ggml_tensor *tensor, const bool inplace = tensor->view_src != nullptr; - if (inplace && (tensor->view_src->backend == GGML_BACKEND_GPU || tensor->view_src->backend == GGML_BACKEND_GPU_SPLIT)) { + if (inplace && (tensor->view_src->backend == GGML_BACKEND_TYPE_GPU || tensor->view_src->backend == GGML_BACKEND_TYPE_GPU_SPLIT)) { ggml_tensor_extra_gpu * src0_extra = (ggml_tensor_extra_gpu * ) tensor->view_src->extra; char * src0_ddc = (char *) src0_extra->data_device[g_main_device_index]; size_t view_offset = 0; @@ -14132,7 +14132,7 @@ catch (sycl::exception const &exc) { } void ggml_sycl_copy_to_device(struct ggml_tensor *tensor) try { - GGML_ASSERT(tensor->backend == GGML_BACKEND_GPU); + GGML_ASSERT(tensor->backend == GGML_BACKEND_TYPE_GPU); GGML_ASSERT(ggml_is_contiguous(tensor)); ggml_tensor_extra_gpu * extra = (ggml_tensor_extra_gpu *) tensor->extra; @@ -14219,9 +14219,9 @@ bool ggml_sycl_compute_forward(struct ggml_compute_params * params, struct ggml_ if (!g_sycl_loaded) return false; ggml_sycl_func_t func; - const bool any_on_device = tensor->backend == GGML_BACKEND_GPU - || (tensor->src[0] != nullptr && (tensor->src[0]->backend == GGML_BACKEND_GPU || tensor->src[0]->backend == GGML_BACKEND_GPU_SPLIT)) - || (tensor->src[1] != nullptr && tensor->src[1]->backend == GGML_BACKEND_GPU); + const bool any_on_device = tensor->backend == GGML_BACKEND_TYPE_GPU + || (tensor->src[0] != nullptr && (tensor->src[0]->backend == GGML_BACKEND_TYPE_GPU || tensor->src[0]->backend == GGML_BACKEND_TYPE_GPU_SPLIT)) + || (tensor->src[1] != nullptr && tensor->src[1]->backend == GGML_BACKEND_TYPE_GPU); if (!any_on_device && tensor->op != GGML_OP_MUL_MAT && tensor->op != GGML_OP_MUL_MAT_ID) { return false; @@ -14359,14 +14359,14 @@ bool ggml_sycl_compute_forward(struct ggml_compute_params * params, struct ggml_ return false; } - if (tensor->src[0] != nullptr && tensor->src[0]->backend == GGML_BACKEND_GPU_SPLIT) { + if (tensor->src[0] != nullptr && tensor->src[0]->backend == GGML_BACKEND_TYPE_GPU_SPLIT) { ggml_sycl_set_peer_access(tensor->src[1]->ne[1]); } if (params->ith != 0) { return true; } - if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) { + if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) { return true; } func(tensor->src[0], tensor->src[1], tensor); @@ -14517,7 +14517,7 @@ static void ggml_backend_sycl_buffer_init_tensor(ggml_backend_buffer_t buffer, extra->data_device[ctx->device] = tensor->data; - tensor->backend = GGML_BACKEND_GPU; + tensor->backend = GGML_BACKEND_TYPE_GPU; tensor->extra = extra; if (ggml_is_quantized(tensor->type)) { @@ -14548,7 +14548,7 @@ static void ggml_backend_sycl_buffer_set_tensor(ggml_backend_buffer_t buffer, ggml_tensor *tensor, const void *data, size_t offset, size_t size) try { - GGML_ASSERT(tensor->backend == GGML_BACKEND_GPU); + GGML_ASSERT(tensor->backend == GGML_BACKEND_TYPE_GPU); ggml_backend_sycl_buffer_context * ctx = ( ggml_backend_sycl_buffer_context *)buffer->context; @@ -14573,7 +14573,7 @@ static void ggml_backend_sycl_buffer_get_tensor(ggml_backend_buffer_t buffer, const ggml_tensor *tensor, void *data, size_t offset, size_t size) try { - GGML_ASSERT(tensor->backend == GGML_BACKEND_GPU); + GGML_ASSERT(tensor->backend == GGML_BACKEND_TYPE_GPU); ggml_backend_sycl_buffer_context * ctx = ( ggml_backend_sycl_buffer_context *)buffer->context; @@ -14809,7 +14809,7 @@ static void ggml_backend_sycl_set_tensor_async(ggml_backend_t backend, ggml_backend_sycl_context * sycl_ctx = (ggml_backend_sycl_context *)backend->context; GGML_ASSERT(tensor->buffer->buft == ggml_backend_sycl_buffer_type(sycl_ctx->device) && "unsupported buffer type"); - GGML_ASSERT(tensor->backend == GGML_BACKEND_GPU); + GGML_ASSERT(tensor->backend == GGML_BACKEND_TYPE_GPU); SYCL_CHECK(CHECK_TRY_ERROR(g_syclStreams[sycl_ctx->device][0]->memcpy( (char *)tensor->data + offset, data, size))); @@ -14827,7 +14827,7 @@ static void ggml_backend_sycl_get_tensor_async(ggml_backend_t backend, ggml_backend_sycl_context * sycl_ctx = (ggml_backend_sycl_context *)backend->context; GGML_ASSERT(tensor->buffer->buft == ggml_backend_sycl_buffer_type(sycl_ctx->device) && "unsupported buffer type"); - GGML_ASSERT(tensor->backend == GGML_BACKEND_GPU); + GGML_ASSERT(tensor->backend == GGML_BACKEND_TYPE_GPU); SYCL_CHECK(CHECK_TRY_ERROR(g_syclStreams[sycl_ctx->device][0]->memcpy( data, (const char *)tensor->data + offset, size))); @@ -14880,7 +14880,7 @@ static bool ggml_backend_sycl_graph_compute(ggml_backend_t backend, ggml_cgraph ggml_sycl_set_main_device(sycl_ctx->device); ggml_compute_params params = {}; - params.type = GGML_TASK_COMPUTE; + params.type = GGML_TASK_TYPE_COMPUTE; params.ith = 0; for (int i = 0; i < cgraph->n_nodes; i++) { ggml_tensor * node = cgraph->nodes[i]; @@ -14888,13 +14888,13 @@ static bool ggml_backend_sycl_graph_compute(ggml_backend_t backend, ggml_cgraph if (node->op == GGML_OP_RESHAPE || node->op == GGML_OP_TRANSPOSE || node->op == GGML_OP_VIEW || node->op == GGML_OP_PERMUTE) continue; - assert(node->backend == GGML_BACKEND_GPU); + assert(node->backend == GGML_BACKEND_TYPE_GPU); assert(node->buffer->buft == ggml_backend_sycl_buffer_type(sycl_ctx->device)); assert(node->extra != nullptr); for (int j = 0; j < GGML_MAX_SRC; j++) { if (node->src[j] != nullptr) { - assert(node->src[j]->backend == GGML_BACKEND_GPU); + assert(node->src[j]->backend == GGML_BACKEND_TYPE_GPU); assert(node->src[j]->buffer->buft == ggml_backend_sycl_buffer_type(sycl_ctx->device)); assert(node->src[j]->extra != nullptr); } diff --git a/ggml-vulkan.cpp b/ggml-vulkan.cpp index 4e5eaff15..6caafb822 100644 --- a/ggml-vulkan.cpp +++ b/ggml-vulkan.cpp @@ -2320,8 +2320,8 @@ static void ggml_vk_mul_mat_q_f16(ggml_backend_vk_context * ctx, vk_context * su src1_uma = d_Qy != nullptr; } - const bool load_x = src0->backend != GGML_BACKEND_GPU && !src0_uma; - const bool load_y = src1->backend != GGML_BACKEND_GPU && !src1_uma; + const bool load_x = src0->backend != GGML_BACKEND_TYPE_GPU && !src0_uma; + const bool load_y = src1->backend != GGML_BACKEND_TYPE_GPU && !src1_uma; const bool x_non_contig = !load_x && !ggml_vk_dim01_contiguous(src0); const bool y_non_contig = !load_y && !ggml_vk_dim01_contiguous(src1); @@ -2453,7 +2453,7 @@ static void ggml_vk_mul_mat_q_f16(ggml_backend_vk_context * ctx, vk_context * su // compute ggml_vk_matmul(ctx, subctx, *pipeline, { d_X, x_buf_offset, x_sz * ne02 * ne03 }, { d_Y, y_buf_offset, y_sz * ne12 * ne13 }, { d_D, d_buf_offset, d_sz * ne12 * ne13 }, { ctx->prealloc_split_k, 0, d_sz * ne12 * ne13 * split_k }, ne01, ne11, ne10, ne10, ne10, ne01, split_k, ne12*ne13, ne02, ne12, r2, r3, stride_batch_x, stride_batch_y, ne20*ne21); // NOLINT - if (dst->backend == GGML_BACKEND_CPU) { + if (dst->backend == GGML_BACKEND_TYPE_CPU) { // copy dst to host float * d = (float *) ((char *) dst->data); ggml_vk_buffer_read_async(ctx, subctx, d_D, 0, d, sizeof(float) * d_ne * ne12 * ne13); @@ -2506,8 +2506,8 @@ static void ggml_vk_mul_mat_vec_q_f16(ggml_backend_vk_context * ctx, vk_context src1_uma = d_Qy != nullptr; } - const bool load_x = src0->backend != GGML_BACKEND_GPU && !src0_uma; - const bool load_y = src1->backend != GGML_BACKEND_GPU && !src1_uma; + const bool load_x = src0->backend != GGML_BACKEND_TYPE_GPU && !src0_uma; + const bool load_y = src1->backend != GGML_BACKEND_TYPE_GPU && !src1_uma; const bool x_non_contig = !load_x && !ggml_vk_dim01_contiguous(src0); const bool y_non_contig = !load_y && !ggml_vk_dim01_contiguous(src1); @@ -2630,7 +2630,7 @@ static void ggml_vk_mul_mat_vec_q_f16(ggml_backend_vk_context * ctx, vk_context ggml_vk_sync_buffers(subctx); ggml_vk_dispatch_pipeline(ctx, subctx, *dmmv, { { d_X, x_offset, x_sz }, { d_Y, y_buffer_offset, y_sz + y_shader_offset }, { d_D, d_buffer_offset, d_sz + d_shader_offset } }, 3 * sizeof(int), &pc, { (uint32_t)ne01, 1, 1}); - if (dst->backend == GGML_BACKEND_CPU) { + if (dst->backend == GGML_BACKEND_TYPE_CPU) { // copy dst to host float * d = (float *) ((char *) dst->data + i12*nb2 + i13*nb3); ggml_vk_sync_buffers(subctx); @@ -2647,7 +2647,7 @@ static void ggml_vk_mul_mat_vec_p021_f16_f32(ggml_backend_vk_context * ctx, vk_c std::cerr << "), (" << dst << ", name=" << dst->name << ", type=" << dst->type << ", backend=" << dst->backend << ", ne0=" << dst->ne[0] << ", ne1=" << dst->ne[1] << ", ne2=" << dst->ne[2] << ", ne3=" << dst->ne[3] << ", nb0=" << dst->nb[0] << ", nb1=" << dst->nb[1] << ", nb2=" << dst->nb[2] << ", nb3=" << dst->nb[3] << "),)" << std::endl; #endif GGML_ASSERT(ggml_is_permuted(src0) && ggml_is_permuted(src1)); - GGML_ASSERT(src0->backend == GGML_BACKEND_GPU); + GGML_ASSERT(src0->backend == GGML_BACKEND_TYPE_GPU); GGML_ASSERT(src0->nb[0] <= src0->nb[1] && src0->nb[2] <= src0->nb[3]); // NOLINT GGML_ASSERT(src1->nb[0] <= src1->nb[1] && src1->nb[2] <= src1->nb[3]); // NOLINT GGML_ASSERT(src0->type == GGML_TYPE_F16); @@ -2679,7 +2679,7 @@ static void ggml_vk_mul_mat_vec_p021_f16_f32(ggml_backend_vk_context * ctx, vk_c src1_uma = d_Qy != nullptr; } - const bool load_y = src1->backend != GGML_BACKEND_GPU && !src1_uma; + const bool load_y = src1->backend != GGML_BACKEND_TYPE_GPU && !src1_uma; const uint64_t x_ne = ne00 * ne01 * ne02; const uint64_t y_ne = ne10 * ne11 * ne12; @@ -2721,7 +2721,7 @@ static void ggml_vk_mul_mat_vec_p021_f16_f32(ggml_backend_vk_context * ctx, vk_c ggml_vk_sync_buffers(subctx); ggml_vk_dispatch_pipeline(ctx, subctx, ctx->pipeline_mul_mat_vec_p021_f16_f32, { { d_Qx, qx_buf_offset, qx_sz }, { d_Qy, qy_buffer_offset, qy_sz + qy_shader_offset }, { d_D, d_buffer_offset, d_sz + d_shader_offset } }, 6 * sizeof(uint32_t), &pc, { 1, (uint32_t)ne01, (uint32_t)ne12 }); - if (dst->backend == GGML_BACKEND_CPU) { + if (dst->backend == GGML_BACKEND_TYPE_CPU) { // copy dst to host float * d = (float *) dst->data; ggml_vk_sync_buffers(subctx); @@ -2738,7 +2738,7 @@ static void ggml_vk_mul_mat_vec_nc_f16_f32(ggml_backend_vk_context * ctx, vk_con GGML_ASSERT(!ggml_is_transposed(src0)); GGML_ASSERT(!ggml_is_transposed(src1)); GGML_ASSERT(!ggml_is_permuted(src0)); - GGML_ASSERT(src0->backend == GGML_BACKEND_GPU); + GGML_ASSERT(src0->backend == GGML_BACKEND_TYPE_GPU); GGML_ASSERT(src0->type == GGML_TYPE_F16); GGML_ASSERT(src1->type == GGML_TYPE_F32); @@ -2771,7 +2771,7 @@ static void ggml_vk_mul_mat_vec_nc_f16_f32(ggml_backend_vk_context * ctx, vk_con src1_uma = d_Qy != nullptr; } - const bool load_y = src1->backend != GGML_BACKEND_GPU && !src1_uma; + const bool load_y = src1->backend != GGML_BACKEND_TYPE_GPU && !src1_uma; const uint64_t d_ne = ne01 * ne11 * ne12; @@ -2814,7 +2814,7 @@ static void ggml_vk_mul_mat_vec_nc_f16_f32(ggml_backend_vk_context * ctx, vk_con ggml_vk_sync_buffers(subctx); ggml_vk_dispatch_pipeline(ctx, subctx, ctx->pipeline_mul_mat_vec_nc_f16_f32, { { d_Qx, qx_buf_offset, qx_sz }, { d_Qy, qy_buffer_offset, qy_sz + qy_shader_offset }, { d_D, d_buffer_offset, d_sz + d_shader_offset } }, 7 * sizeof(uint32_t), &pc, { 1, (uint32_t)ne01, (uint32_t)ne12 }); - if (dst->backend == GGML_BACKEND_CPU) { + if (dst->backend == GGML_BACKEND_TYPE_CPU) { // copy dst to host float * d = (float *) dst->data; ggml_vk_sync_buffers(subctx); @@ -2832,7 +2832,7 @@ static bool ggml_vk_can_mul_mat(const ggml_tensor * src0, const ggml_tensor * sr return (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16 || ggml_is_quantized(src0->type)) && (src1->type == GGML_TYPE_F32 || src1->type == GGML_TYPE_F16 || ggml_is_quantized(src1->type)) && dst->type == GGML_TYPE_F32 && - ((ne0 >= 32 && ne1 >= 32 && ne10 >= 32) || src0->backend == GGML_BACKEND_GPU); + ((ne0 >= 32 && ne1 >= 32 && ne10 >= 32) || src0->backend == GGML_BACKEND_TYPE_GPU); } static void ggml_vk_mul_mat(ggml_backend_vk_context * ctx, vk_context * subctx, const struct ggml_tensor * src0, const struct ggml_tensor * src1, struct ggml_tensor * dst) { @@ -2880,8 +2880,8 @@ static void ggml_vk_op_repeat(ggml_backend_vk_context * ctx, vk_context * subctx // TODO: support for transposed / permuted tensors GGML_ASSERT(nb0 == sizeof(float)); GGML_ASSERT(nb00 == sizeof(float)); - GGML_ASSERT(src0->backend == GGML_BACKEND_GPU); - GGML_ASSERT(dst->backend == GGML_BACKEND_GPU); + GGML_ASSERT(src0->backend == GGML_BACKEND_TYPE_GPU); + GGML_ASSERT(dst->backend == GGML_BACKEND_TYPE_GPU); ggml_tensor_extra_gpu * extra = (ggml_tensor_extra_gpu *) dst->extra; ggml_tensor_extra_gpu * extra_src0 = (ggml_tensor_extra_gpu *) src0->extra; @@ -3110,8 +3110,8 @@ static void ggml_vk_op_f32(ggml_backend_vk_context * ctx, vk_context * subctx, c } } - const bool transfer_src0 = src0->backend != GGML_BACKEND_GPU && !src0_uma; - const bool transfer_src1 = use_src1 && src1->backend != GGML_BACKEND_GPU && !src1_uma; + const bool transfer_src0 = src0->backend != GGML_BACKEND_TYPE_GPU && !src0_uma; + const bool transfer_src1 = use_src1 && src1->backend != GGML_BACKEND_TYPE_GPU && !src1_uma; uint64_t x_sz = ggml_vk_align_size(ggml_type_size(src0->type) * ne0, ctx->device.lock()->properties.limits.minStorageBufferOffsetAlignment); uint64_t y_sz = use_src1 ? ggml_vk_align_size(ggml_type_size(src1->type) * ne1, ctx->device.lock()->properties.limits.minStorageBufferOffsetAlignment) : 0; @@ -3120,7 +3120,7 @@ static void ggml_vk_op_f32(ggml_backend_vk_context * ctx, vk_context * subctx, c vk_buffer d_D = extra->buffer_gpu.lock(); // Workaround for tiny tensor inputs on ROPE - if (use_src1 && src1->backend == GGML_BACKEND_GPU && y_sz > d_D->size) { + if (use_src1 && src1->backend == GGML_BACKEND_TYPE_GPU && y_sz > d_D->size) { y_sz = VK_WHOLE_SIZE; } @@ -3209,9 +3209,9 @@ static void ggml_vk_op_f32(ggml_backend_vk_context * ctx, vk_context * subctx, c ggml_vk_sync_buffers(subctx); ggml_vk_dispatch_pipeline(ctx, subctx, *pipeline, { { d_X, x_buf_offset, x_sz }, { d_D, d_buf_offset, d_sz } }, sizeof(PC), &pc, elements); } - if (dst->backend == GGML_BACKEND_CPU && op == GGML_OP_CPY) { + if (dst->backend == GGML_BACKEND_TYPE_CPU && op == GGML_OP_CPY) { ggml_vk_d2h_tensor_2d(ctx, subctx, d_D, 0, dst); - } else if(dst->backend == GGML_BACKEND_CPU) { + } else if(dst->backend == GGML_BACKEND_TYPE_CPU) { // copy dst to host float * d = (float *) dst->data; ggml_vk_buffer_read_async(ctx, subctx, d_D, 0, d, d_sz); @@ -3253,7 +3253,7 @@ static void ggml_vk_op_f32(ggml_backend_vk_context * ctx, vk_context * subctx, c ggml_vk_sync_buffers(subctx); ggml_vk_dispatch_pipeline(ctx, subctx, *pipeline, { { d_X, x_buf_offset + x_offset, x_sz }, { d_D, d_buf_offset + d_offset, d_sz } }, sizeof(PC), &pc, elements); } - if (dst->backend == GGML_BACKEND_CPU) { + if (dst->backend == GGML_BACKEND_TYPE_CPU) { // copy dst to host ggml_vk_buffer_read_async(ctx, subctx, d_D, d_buf_offset + d_offset, (char *) dst->data + i02*nb2 + i03*nb3, d_sz); } @@ -3359,7 +3359,7 @@ static void ggml_vk_rope(ggml_backend_vk_context * ctx, vk_context * subctx, con static void ggml_vk_nop(ggml_backend_vk_context * ctx, vk_context * subctx, const ggml_tensor * src0, ggml_tensor * dst) { // If backend is CPU, data from src0 has to be copied off the device - if (dst->backend == GGML_BACKEND_CPU) { + if (dst->backend == GGML_BACKEND_TYPE_CPU) { ggml_tensor_extra_gpu * extra_src0 = (ggml_tensor_extra_gpu *) src0->extra; vk_buffer d_D = extra_src0->buffer_gpu.lock(); ggml_vk_sync_buffers(subctx); @@ -3994,9 +3994,9 @@ static void ggml_vk_preallocate_buffers_graph(ggml_backend_vk_context * ctx, ggm #ifdef GGML_VULKAN_DEBUG std::cerr << "ggml_vk_preallocate_buffers_graph(" << node << ")" << std::endl; #endif - const bool any_on_device = node->backend == GGML_BACKEND_GPU - || (node->src[0] != nullptr && (node->src[0]->backend == GGML_BACKEND_GPU || node->src[0]->backend == GGML_BACKEND_GPU_SPLIT)) - || (node->src[1] != nullptr && (node->src[1]->backend == GGML_BACKEND_GPU)); + const bool any_on_device = node->backend == GGML_BACKEND_TYPE_GPU + || (node->src[0] != nullptr && (node->src[0]->backend == GGML_BACKEND_TYPE_GPU || node->src[0]->backend == GGML_BACKEND_TYPE_GPU_SPLIT)) + || (node->src[1] != nullptr && (node->src[1]->backend == GGML_BACKEND_TYPE_GPU)); if (ctx->disable || (!any_on_device && node->op != GGML_OP_MUL_MAT)) { return; @@ -4215,9 +4215,9 @@ static void ggml_vk_preallocate_buffers(ggml_backend_vk_context * ctx) { } static void ggml_vk_build_graph(ggml_backend_vk_context * ctx, ggml_tensor * node, bool last_node){ - const bool any_on_device = node->backend == GGML_BACKEND_GPU - || (node->src[0] != nullptr && (node->src[0]->backend == GGML_BACKEND_GPU || node->src[0]->backend == GGML_BACKEND_GPU_SPLIT)) - || (node->src[1] != nullptr && node->src[1]->backend == GGML_BACKEND_GPU); + const bool any_on_device = node->backend == GGML_BACKEND_TYPE_GPU + || (node->src[0] != nullptr && (node->src[0]->backend == GGML_BACKEND_TYPE_GPU || node->src[0]->backend == GGML_BACKEND_TYPE_GPU_SPLIT)) + || (node->src[1] != nullptr && node->src[1]->backend == GGML_BACKEND_TYPE_GPU); if (ctx->disable || (!any_on_device && node->op != GGML_OP_MUL_MAT) || (node->op == GGML_OP_MUL_MAT && !any_on_device && !ggml_vk_can_mul_mat(node->src[0], node->src[1], node))) { return; @@ -4371,7 +4371,7 @@ static void ggml_vk_build_graph(ggml_backend_vk_context * ctx, ggml_tensor * nod last_node = true; #endif - if (node->backend == GGML_BACKEND_CPU || last_node) { + if (node->backend == GGML_BACKEND_TYPE_CPU || last_node) { ggml_vk_ctx_end(ctx->compute_ctx); ctx->compute_ctx->exit_tensor = node; ctx->compute_ctx = nullptr; @@ -4379,9 +4379,9 @@ static void ggml_vk_build_graph(ggml_backend_vk_context * ctx, ggml_tensor * nod } static bool ggml_vk_compute_forward(ggml_backend_vk_context * ctx, ggml_compute_params * params, ggml_tensor * tensor){ - const bool any_on_device = tensor->backend == GGML_BACKEND_GPU - || (tensor->src[0] != nullptr && (tensor->src[0]->backend == GGML_BACKEND_GPU || tensor->src[0]->backend == GGML_BACKEND_GPU_SPLIT)) - || (tensor->src[1] != nullptr && tensor->src[1]->backend == GGML_BACKEND_GPU); + const bool any_on_device = tensor->backend == GGML_BACKEND_TYPE_GPU + || (tensor->src[0] != nullptr && (tensor->src[0]->backend == GGML_BACKEND_TYPE_GPU || tensor->src[0]->backend == GGML_BACKEND_TYPE_GPU_SPLIT)) + || (tensor->src[1] != nullptr && tensor->src[1]->backend == GGML_BACKEND_TYPE_GPU); if (ctx->disable || (!any_on_device && tensor->op != GGML_OP_MUL_MAT)) { return false; @@ -4442,7 +4442,7 @@ static bool ggml_vk_compute_forward(ggml_backend_vk_context * ctx, ggml_compute_ if (params->ith != 0) { return true; } - if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) { + if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) { return true; } @@ -4745,7 +4745,7 @@ GGML_CALL static void ggml_backend_vk_buffer_init_tensor(ggml_backend_buffer_t b extra->offset = (uint8_t *) tensor->data - (uint8_t *) vk_ptr_base; } - tensor->backend = GGML_BACKEND_GPU; + tensor->backend = GGML_BACKEND_TYPE_GPU; tensor->extra = extra; } @@ -4753,7 +4753,7 @@ GGML_CALL static void ggml_backend_vk_buffer_set_tensor(ggml_backend_buffer_t bu #ifdef GGML_VULKAN_DEBUG std::cerr << "ggml_backend_vk_buffer_set_tensor(" << buffer << ", " << tensor << ", " << data << ", " << offset << ", " << size << ")" << std::endl; #endif - GGML_ASSERT(tensor->backend == GGML_BACKEND_GPU); + GGML_ASSERT(tensor->backend == GGML_BACKEND_TYPE_GPU); ggml_backend_vk_buffer_context * ctx = (ggml_backend_vk_buffer_context *)buffer->context; @@ -4768,7 +4768,7 @@ GGML_CALL static void ggml_backend_vk_buffer_get_tensor(ggml_backend_buffer_t bu #ifdef GGML_VULKAN_DEBUG std::cerr << "ggml_backend_vk_buffer_get_tensor(" << buffer << ", " << tensor << ", " << data << ", " << offset << ", " << size << ")" << std::endl; #endif - GGML_ASSERT(tensor->backend == GGML_BACKEND_GPU); + GGML_ASSERT(tensor->backend == GGML_BACKEND_TYPE_GPU); ggml_backend_vk_buffer_context * ctx = (ggml_backend_vk_buffer_context *)buffer->context; @@ -4999,7 +4999,7 @@ GGML_CALL static void ggml_backend_vk_set_tensor_async(ggml_backend_t backend, g #endif ggml_backend_vk_context * ctx = (ggml_backend_vk_context *)backend->context; GGML_ASSERT((tensor->buffer->buft == ggml_backend_vk_buffer_type(ctx->idx) || tensor->buffer->buft == ggml_backend_vk_host_buffer_type()) && "unsupported buffer type"); - GGML_ASSERT(tensor->backend == GGML_BACKEND_GPU); + GGML_ASSERT(tensor->backend == GGML_BACKEND_TYPE_GPU); ggml_tensor_extra_gpu * extra = (ggml_tensor_extra_gpu *) tensor->extra; @@ -5020,7 +5020,7 @@ GGML_CALL static void ggml_backend_vk_get_tensor_async(ggml_backend_t backend, c #endif ggml_backend_vk_context * ctx = (ggml_backend_vk_context *)backend->context; GGML_ASSERT((tensor->buffer->buft == ggml_backend_vk_buffer_type(ctx->idx) || tensor->buffer->buft == ggml_backend_vk_host_buffer_type()) && "unsupported buffer type"); - GGML_ASSERT(tensor->backend == GGML_BACKEND_GPU); + GGML_ASSERT(tensor->backend == GGML_BACKEND_TYPE_GPU); ggml_tensor_extra_gpu * extra = (ggml_tensor_extra_gpu *) tensor->extra; @@ -5097,7 +5097,7 @@ GGML_CALL static bool ggml_backend_vk_graph_compute(ggml_backend_t backend, ggml int last_node = cgraph->n_nodes - 1; // If the last op in the cgraph isn't backend GPU, the command buffer doesn't get closed properly - while (last_node > 0 && cgraph->nodes[last_node]->backend != GGML_BACKEND_GPU) { + while (last_node > 0 && cgraph->nodes[last_node]->backend != GGML_BACKEND_TYPE_GPU) { last_node -= 1; } @@ -5106,7 +5106,7 @@ GGML_CALL static bool ggml_backend_vk_graph_compute(ggml_backend_t backend, ggml } ggml_compute_params params = {}; - params.type = GGML_TASK_COMPUTE; + params.type = GGML_TASK_TYPE_COMPUTE; params.ith = 0; for (int i = 0; i < cgraph->n_nodes; i++) { ggml_tensor * node = cgraph->nodes[i]; @@ -5410,7 +5410,7 @@ static void ggml_vk_print_tensor_area(const ggml_tensor * tensor, const void * d static void ggml_vk_print_tensor(ggml_backend_vk_context * ctx, const ggml_tensor * tensor, const char * name) { void * tensor_data = tensor->data; - if (tensor->backend == GGML_BACKEND_GPU) { + if (tensor->backend == GGML_BACKEND_TYPE_GPU) { const size_t tensor_size = ggml_nbytes(tensor); tensor_data = malloc(tensor_size); @@ -5436,14 +5436,14 @@ static void ggml_vk_print_tensor(ggml_backend_vk_context * ctx, const ggml_tenso std::vector done; ggml_vk_print_graph_origin(tensor, done); - if (tensor->backend == GGML_BACKEND_GPU) { + if (tensor->backend == GGML_BACKEND_TYPE_GPU) { free(tensor_data); } } static void ggml_vk_check_tensor(const std::string& name, const ggml_tensor * tensor) { return; - GGML_ASSERT(tensor->backend == GGML_BACKEND_CPU); + GGML_ASSERT(tensor->backend == GGML_BACKEND_TYPE_CPU); if (tensor->type != GGML_TYPE_F32 && tensor->type != GGML_TYPE_F16) { return; } @@ -5481,7 +5481,7 @@ static void ggml_vk_check_results_0(ggml_backend_vk_context * ctx, ggml_compute_ if (params->ith != 0) { return; } - if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE || tensor->op == GGML_OP_TRANSPOSE) { + if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE || tensor->op == GGML_OP_TRANSPOSE) { return; } @@ -5518,10 +5518,10 @@ static void ggml_vk_check_results_0(ggml_backend_vk_context * ctx, ggml_compute_ src0_buffer = malloc(src0_size); src0_clone->data = src0_buffer; - if (src0->backend == GGML_BACKEND_CPU) { + if (src0->backend == GGML_BACKEND_TYPE_CPU) { memcpy(src0_clone->data, src0->data, src0_size); memcpy(src0_clone->nb, src0->nb, sizeof(size_t) * GGML_MAX_DIMS); - } else if (src0->backend == GGML_BACKEND_GPU) { + } else if (src0->backend == GGML_BACKEND_TYPE_GPU) { ggml_tensor_extra_gpu * extra = (ggml_tensor_extra_gpu *) src0->extra; uint64_t offset = extra->offset; if (!ggml_is_contiguous(src0) && ggml_vk_dim01_contiguous(src0)) { @@ -5561,10 +5561,10 @@ static void ggml_vk_check_results_0(ggml_backend_vk_context * ctx, ggml_compute_ src1_buffer = malloc(src1_size); src1_clone->data = src1_buffer; - if (src1->backend == GGML_BACKEND_CPU) { + if (src1->backend == GGML_BACKEND_TYPE_CPU) { memcpy(src1_clone->data, src1->data, src1_size); memcpy(src1_clone->nb, src1->nb, sizeof(size_t) * GGML_MAX_DIMS); - } else if (src1->backend == GGML_BACKEND_GPU) { + } else if (src1->backend == GGML_BACKEND_TYPE_GPU) { ggml_tensor_extra_gpu * extra = (ggml_tensor_extra_gpu *) src1->extra; uint64_t offset = extra->offset; if (!ggml_is_contiguous(src1) && ggml_vk_dim01_contiguous(src1)) { @@ -5723,7 +5723,7 @@ static void ggml_vk_check_results_1(ggml_backend_vk_context * ctx, ggml_compute_ if (params->ith != 0) { return; } - if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE || tensor->op == GGML_OP_TRANSPOSE) { + if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE || tensor->op == GGML_OP_TRANSPOSE) { return; } if (!(vk_output_tensor > 0 && vk_output_tensor == check_counter) && check_counter <= vk_skip_checks) { @@ -5735,7 +5735,7 @@ static void ggml_vk_check_results_1(ggml_backend_vk_context * ctx, ggml_compute_ void * tensor_data = tensor->data; - if (tensor->backend == GGML_BACKEND_GPU) { + if (tensor->backend == GGML_BACKEND_TYPE_GPU) { size_t tensor_size = ggml_nbytes(tensor); tensor_data = malloc(tensor_size); @@ -5868,7 +5868,7 @@ static void ggml_vk_check_results_1(ggml_backend_vk_context * ctx, ggml_compute_ comp_result = nullptr; comp_size = 0; - if (tensor->backend == GGML_BACKEND_GPU) { + if (tensor->backend == GGML_BACKEND_TYPE_GPU) { free(tensor_data); } } diff --git a/ggml.c b/ggml.c index c09a3cad6..1d81553f4 100644 --- a/ggml.c +++ b/ggml.c @@ -2721,7 +2721,7 @@ static struct ggml_tensor * ggml_new_tensor_impl( } } - struct ggml_object * const obj_new = ggml_new_object(ctx, GGML_OBJECT_TENSOR, GGML_TENSOR_SIZE + obj_alloc_size); + struct ggml_object * const obj_new = ggml_new_object(ctx, GGML_OBJECT_TYPE_TENSOR, GGML_TENSOR_SIZE + obj_alloc_size); // TODO: for recoverable errors, we would need to free the data allocated from the scratch buffer here @@ -2729,7 +2729,7 @@ static struct ggml_tensor * ggml_new_tensor_impl( *result = (struct ggml_tensor) { /*.type =*/ type, - /*.backend =*/ GGML_BACKEND_CPU, + /*.backend =*/ GGML_BACKEND_TYPE_CPU, /*.buffer =*/ NULL, /*.ne =*/ { 1, 1, 1, 1 }, /*.nb =*/ { 0, 0, 0, 0 }, @@ -3302,7 +3302,7 @@ struct ggml_tensor * ggml_get_first_tensor(const struct ggml_context * ctx) { char * const mem_buffer = ctx->mem_buffer; while (obj != NULL) { - if (obj->type == GGML_OBJECT_TENSOR) { + if (obj->type == GGML_OBJECT_TYPE_TENSOR) { return (struct ggml_tensor *)(mem_buffer + obj->offs); } @@ -3319,7 +3319,7 @@ struct ggml_tensor * ggml_get_next_tensor(const struct ggml_context * ctx, struc char * const mem_buffer = ctx->mem_buffer; while (obj != NULL) { - if (obj->type == GGML_OBJECT_TENSOR) { + if (obj->type == GGML_OBJECT_TYPE_TENSOR) { return (struct ggml_tensor *)(mem_buffer + obj->offs); } @@ -3335,7 +3335,7 @@ struct ggml_tensor * ggml_get_tensor(struct ggml_context * ctx, const char * nam char * const mem_buffer = ctx->mem_buffer; while (obj != NULL) { - if (obj->type == GGML_OBJECT_TENSOR) { + if (obj->type == GGML_OBJECT_TYPE_TENSOR) { struct ggml_tensor * cur = (struct ggml_tensor *)(mem_buffer + obj->offs); if (strcmp(cur->name, name) == 0) { return cur; @@ -5879,7 +5879,7 @@ struct ggml_tensor * ggml_top_k( int k) { GGML_ASSERT(a->ne[0] >= k); - struct ggml_tensor * result = ggml_argsort(ctx, a, GGML_SORT_DESC); + struct ggml_tensor * result = ggml_argsort(ctx, a, GGML_SORT_ORDER_DESC); result = ggml_view_4d(ctx, result, k, result->ne[1], result->ne[2], result->ne[3], @@ -6673,7 +6673,7 @@ static void ggml_compute_forward_dup_same_cont( GGML_ASSERT(ggml_is_contiguous(dst) && ggml_is_contiguous(src0)); GGML_ASSERT(src0->type == dst->type); - if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) { + if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) { return; } @@ -6705,7 +6705,7 @@ static void ggml_compute_forward_dup_f16( GGML_ASSERT(ggml_nelements(dst) == ggml_nelements(src0)); - if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) { + if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) { return; } @@ -6978,7 +6978,7 @@ static void ggml_compute_forward_dup_f32( GGML_ASSERT(ggml_nelements(dst) == ggml_nelements(src0)); - if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) { + if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) { return; } @@ -7231,7 +7231,7 @@ static void ggml_compute_forward_dup_bytes( GGML_ASSERT(ggml_nelements(dst) == ggml_nelements(src0)); GGML_ASSERT(src0->type == dst->type); - if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) { + if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) { return; } @@ -7411,7 +7411,7 @@ static void ggml_compute_forward_add_f32( GGML_ASSERT(ggml_can_repeat(src1, src0) && ggml_are_same_shape(src0, dst)); - if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) { + if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) { return; } @@ -7419,7 +7419,7 @@ static void ggml_compute_forward_add_f32( const int nth = params->nth; #ifdef GGML_USE_CLBLAST - if (src1->backend == GGML_BACKEND_GPU) { + if (src1->backend == GGML_BACKEND_TYPE_GPU) { // TODO: OpenCL kernel support full broadcast GGML_ASSERT(ggml_can_repeat_rows(src1, src0)); if (ith == 0) { @@ -7501,7 +7501,7 @@ static void ggml_compute_forward_add_f16_f32( GGML_ASSERT(ggml_are_same_shape(src0, src1) && ggml_are_same_shape(src0, dst)); - if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) { + if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) { return; } @@ -7580,7 +7580,7 @@ static void ggml_compute_forward_add_f16_f16( GGML_ASSERT(ggml_are_same_shape(src0, src1) && ggml_are_same_shape(src0, dst)); - if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) { + if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) { return; } @@ -7636,7 +7636,7 @@ static void ggml_compute_forward_add_q_f32( GGML_ASSERT(ggml_are_same_shape(src0, src1) && ggml_are_same_shape(src0, dst)); - if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) { + if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) { return; } @@ -7774,7 +7774,7 @@ static void ggml_compute_forward_add1_f32( GGML_ASSERT(ggml_are_same_shape(src0, dst)); GGML_ASSERT(ggml_is_scalar(src1)); - if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) { + if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) { return; } @@ -7828,7 +7828,7 @@ static void ggml_compute_forward_add1_f16_f32( GGML_ASSERT(ggml_are_same_shape(src0, dst)); GGML_ASSERT(ggml_is_scalar(src1)); - if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) { + if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) { return; } @@ -7880,7 +7880,7 @@ static void ggml_compute_forward_add1_f16_f16( GGML_ASSERT(ggml_are_same_shape(src0, dst)); GGML_ASSERT(ggml_is_scalar(src1)); - if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) { + if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) { return; } @@ -7932,7 +7932,7 @@ static void ggml_compute_forward_add1_q_f32( GGML_ASSERT(ggml_are_same_shape(src0, dst)); GGML_ASSERT(ggml_is_scalar(src1)); - if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) { + if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) { return; } @@ -8062,7 +8062,7 @@ static void ggml_compute_forward_acc_f32( size_t offset = ((int32_t *) dst->op_params)[3]; bool inplace = (bool) ((int32_t *) dst->op_params)[4]; - if (!inplace && (params->type == GGML_TASK_INIT)) { + if (!inplace && (params->type == GGML_TASK_TYPE_INIT)) { if (params->ith != 0) { return; } @@ -8074,7 +8074,7 @@ static void ggml_compute_forward_acc_f32( ggml_nbytes(dst)); } - if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) { + if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) { return; } @@ -8176,7 +8176,7 @@ static void ggml_compute_forward_sub_f32( assert(params->ith == 0); assert(ggml_are_same_shape(src0, src1) && ggml_are_same_shape(src0, dst)); - if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) { + if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) { return; } @@ -8257,14 +8257,14 @@ static void ggml_compute_forward_mul_f32( GGML_ASSERT(ggml_can_repeat(src1, src0) && ggml_are_same_shape(src0, dst)); - if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) { + if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) { return; } const int ith = params->ith; const int nth = params->nth; #if defined(GGML_USE_CLBLAST) - if (src1->backend == GGML_BACKEND_GPU) { + if (src1->backend == GGML_BACKEND_TYPE_GPU) { // TODO: OpenCL kernel support full broadcast GGML_ASSERT(ggml_can_repeat_rows(src1, src0)); if (ith == 0) { @@ -8365,7 +8365,7 @@ static void ggml_compute_forward_div_f32( GGML_ASSERT(ggml_can_repeat(src1, src0) && ggml_are_same_shape(src0, dst)); - if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) { + if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) { return; } @@ -8460,7 +8460,7 @@ static void ggml_compute_forward_sqr_f32( assert(params->ith == 0); assert(ggml_are_same_shape(src0, dst)); - if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) { + if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) { return; } @@ -8506,7 +8506,7 @@ static void ggml_compute_forward_sqrt_f32( assert(params->ith == 0); assert(ggml_are_same_shape(src0, dst)); - if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) { + if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) { return; } @@ -8552,7 +8552,7 @@ static void ggml_compute_forward_log_f32( GGML_ASSERT(params->ith == 0); GGML_ASSERT(ggml_are_same_shape(src0, dst)); - if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) { + if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) { return; } @@ -8598,7 +8598,7 @@ static void ggml_compute_forward_sum_f32( assert(params->ith == 0); assert(ggml_is_scalar(dst)); - if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) { + if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) { return; } @@ -8633,7 +8633,7 @@ static void ggml_compute_forward_sum_f16( assert(params->ith == 0); assert(ggml_is_scalar(dst)); - if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) { + if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) { return; } @@ -8690,7 +8690,7 @@ static void ggml_compute_forward_sum_rows_f32( GGML_ASSERT(params->ith == 0); - if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) { + if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) { return; } @@ -8745,7 +8745,7 @@ static void ggml_compute_forward_mean_f32( assert(params->ith == 0); - if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) { + if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) { return; } @@ -8804,7 +8804,7 @@ static void ggml_compute_forward_argmax_f32( assert(params->ith == 0); - if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) { + if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) { return; } @@ -8855,7 +8855,7 @@ static void ggml_compute_forward_repeat_f32( GGML_ASSERT(params->ith == 0); GGML_ASSERT(ggml_can_repeat(src0, dst)); - if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) { + if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) { return; } @@ -8900,7 +8900,7 @@ static void ggml_compute_forward_repeat_f16( GGML_ASSERT(params->ith == 0); GGML_ASSERT(ggml_can_repeat(src0, dst)); - if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) { + if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) { return; } @@ -8974,7 +8974,7 @@ static void ggml_compute_forward_repeat_back_f32( GGML_ASSERT(params->ith == 0); GGML_ASSERT(ggml_can_repeat(dst, src0)); - if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) { + if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) { return; } @@ -9051,7 +9051,7 @@ static void ggml_compute_forward_concat_f32( const struct ggml_tensor * src0 = dst->src[0]; const struct ggml_tensor * src1 = dst->src[1]; - if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) { + if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) { return; } @@ -9123,7 +9123,7 @@ static void ggml_compute_forward_abs_f32( assert(params->ith == 0); assert(ggml_are_same_shape(src0, dst)); - if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) { + if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) { return; } @@ -9169,7 +9169,7 @@ static void ggml_compute_forward_sgn_f32( assert(params->ith == 0); assert(ggml_are_same_shape(src0, dst)); - if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) { + if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) { return; } @@ -9215,7 +9215,7 @@ static void ggml_compute_forward_neg_f32( assert(params->ith == 0); assert(ggml_are_same_shape(src0, dst)); - if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) { + if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) { return; } @@ -9261,7 +9261,7 @@ static void ggml_compute_forward_step_f32( assert(params->ith == 0); assert(ggml_are_same_shape(src0, dst)); - if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) { + if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) { return; } @@ -9307,7 +9307,7 @@ static void ggml_compute_forward_tanh_f32( assert(params->ith == 0); assert(ggml_are_same_shape(src0, dst)); - if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) { + if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) { return; } @@ -9353,7 +9353,7 @@ static void ggml_compute_forward_elu_f32( assert(params->ith == 0); assert(ggml_are_same_shape(src0, dst)); - if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) { + if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) { return; } @@ -9399,7 +9399,7 @@ static void ggml_compute_forward_relu_f32( assert(params->ith == 0); assert(ggml_are_same_shape(src0, dst)); - if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) { + if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) { return; } @@ -9446,7 +9446,7 @@ static void ggml_compute_forward_gelu_f32( GGML_ASSERT(ggml_is_contiguous_except_dim_1(dst)); GGML_ASSERT(ggml_are_same_shape(src0, dst)); - if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) { + if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) { return; } @@ -9509,7 +9509,7 @@ static void ggml_compute_forward_gelu_quick_f32( GGML_ASSERT(ggml_is_contiguous_except_dim_1(dst)); GGML_ASSERT(ggml_are_same_shape(src0, dst)); - if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) { + if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) { return; } @@ -9572,7 +9572,7 @@ static void ggml_compute_forward_silu_f32( GGML_ASSERT(ggml_is_contiguous_except_dim_1(dst)); GGML_ASSERT(ggml_are_same_shape(src0, dst)); - if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) { + if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) { return; } @@ -9633,7 +9633,7 @@ static void ggml_compute_forward_leaky_relu_f32( assert(params->ith == 0); assert(ggml_are_same_shape(src0, dst)); - if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) { + if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) { return; } @@ -9686,7 +9686,7 @@ static void ggml_compute_forward_silu_back_f32( GGML_ASSERT(ggml_are_same_shape(src0, dst)); GGML_ASSERT(ggml_are_same_shape(src0, grad)); - if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) { + if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) { return; } @@ -9748,7 +9748,7 @@ static void ggml_compute_forward_hardswish_f32( assert(params->ith == 0); assert(ggml_are_same_shape(src0, dst)); - if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) { + if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) { return; } @@ -9791,7 +9791,7 @@ static void ggml_compute_forward_hardsigmoid_f32( assert(params->ith == 0); assert(ggml_are_same_shape(src0, dst)); - if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) { + if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) { return; } @@ -9837,7 +9837,7 @@ static void ggml_compute_forward_norm_f32( GGML_ASSERT(ggml_are_same_shape(src0, dst)); - if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) { + if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) { return; } @@ -9912,7 +9912,7 @@ static void ggml_compute_forward_rms_norm_f32( GGML_ASSERT(ggml_are_same_shape(src0, dst)); - if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) { + if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) { return; } @@ -9983,7 +9983,7 @@ static void ggml_compute_forward_rms_norm_back_f32( GGML_ASSERT(ggml_are_same_shape(src0, dst) && ggml_are_same_shape(src0, src1)); - if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) { + if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) { return; } @@ -10161,7 +10161,7 @@ static void ggml_compute_forward_group_norm_f32( GGML_ASSERT(ggml_are_same_shape(src0, dst)); - if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) { + if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) { return; } @@ -10328,7 +10328,7 @@ static void ggml_compute_forward_mul_mat( #if defined(GGML_USE_CLBLAST) if (ggml_cl_can_mul_mat(src0, src1, dst)) { - if (params->ith == 0 && params->type == GGML_TASK_COMPUTE) { + if (params->ith == 0 && params->type == GGML_TASK_TYPE_COMPUTE) { ggml_cl_mul_mat(src0, src1, dst, params->wdata, params->wsize); } return; @@ -10341,7 +10341,7 @@ static void ggml_compute_forward_mul_mat( const size_t desired_wsize = ne13*ne12*ne_plane*sizeof(float); UNUSED(desired_wsize); - if (params->type == GGML_TASK_INIT) { + if (params->type == GGML_TASK_TYPE_INIT) { if (type != GGML_TYPE_F32) { assert(params->wsize >= desired_wsize); // parallelize by src0 rows @@ -10364,7 +10364,7 @@ static void ggml_compute_forward_mul_mat( return; } - if (params->type == GGML_TASK_FINALIZE) { + if (params->type == GGML_TASK_TYPE_FINALIZE) { return; } @@ -10402,7 +10402,7 @@ static void ggml_compute_forward_mul_mat( } #endif - if (params->type == GGML_TASK_INIT) { + if (params->type == GGML_TASK_TYPE_INIT) { if (ith != 0) { return; } @@ -10426,7 +10426,7 @@ static void ggml_compute_forward_mul_mat( return; } - if (params->type == GGML_TASK_FINALIZE) { + if (params->type == GGML_TASK_TYPE_FINALIZE) { return; } @@ -10583,7 +10583,7 @@ static void ggml_compute_forward_mul_mat_id( #define MMID_MATRIX_ROW(row_id, i1) matrix_rows[(row_id)*ne11 + (i1)] - if (params->type == GGML_TASK_INIT) { + if (params->type == GGML_TASK_TYPE_INIT) { if (ith != 0) { return; } @@ -10620,7 +10620,7 @@ static void ggml_compute_forward_mul_mat_id( return; } - if (params->type == GGML_TASK_FINALIZE) { + if (params->type == GGML_TASK_TYPE_FINALIZE) { return; } @@ -10768,7 +10768,7 @@ static void ggml_compute_forward_out_prod_f32( (ggml_is_contiguous(src1) || ggml_is_transposed(src1)); #endif - if (params->type == GGML_TASK_INIT) { + if (params->type == GGML_TASK_TYPE_INIT) { #if defined(GGML_USE_ACCELERATE) || defined(GGML_USE_OPENBLAS) // gemm beta will zero dst if (use_blas) { return; @@ -10781,7 +10781,7 @@ static void ggml_compute_forward_out_prod_f32( return; } - if (params->type == GGML_TASK_FINALIZE) { + if (params->type == GGML_TASK_TYPE_FINALIZE) { return; } @@ -10961,7 +10961,7 @@ static void ggml_compute_forward_out_prod_q_f32( // TODO: #if defined(GGML_USE_CUBLAS) ggml_cuda_out_prod // TODO: #if defined(GGML_USE_ACCELERATE) || defined(GGML_USE_OPENBLAS) || defined(GGML_USE_CLBLAST) - if (params->type == GGML_TASK_INIT) { + if (params->type == GGML_TASK_TYPE_INIT) { if (ith != 0) { return; } @@ -10969,7 +10969,7 @@ static void ggml_compute_forward_out_prod_q_f32( return; } - if (params->type == GGML_TASK_FINALIZE) { + if (params->type == GGML_TASK_TYPE_FINALIZE) { return; } @@ -11087,7 +11087,7 @@ static void ggml_compute_forward_scale_f32( GGML_ASSERT(ggml_is_contiguous(dst)); GGML_ASSERT(ggml_are_same_shape(src0, dst)); - if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) { + if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) { return; } @@ -11159,7 +11159,7 @@ static void ggml_compute_forward_set_f32( size_t offset = ((int32_t *) dst->op_params)[3]; bool inplace = (bool) ((int32_t *) dst->op_params)[4]; - if (!inplace && (params->type == GGML_TASK_INIT)) { + if (!inplace && (params->type == GGML_TASK_TYPE_INIT)) { if (params->ith != 0) { return; } @@ -11171,7 +11171,7 @@ static void ggml_compute_forward_set_f32( ggml_nbytes(dst)); } - if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) { + if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) { return; } @@ -11319,7 +11319,7 @@ static void ggml_compute_forward_get_rows_q( assert(params->ith == 0); - if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) { + if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) { return; } @@ -11359,7 +11359,7 @@ static void ggml_compute_forward_get_rows_f16( assert(params->ith == 0); - if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) { + if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) { return; } @@ -11396,7 +11396,7 @@ static void ggml_compute_forward_get_rows_f32( assert(params->ith == 0); - if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) { + if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) { return; } @@ -11499,14 +11499,14 @@ static void ggml_compute_forward_get_rows_back_f32_f16( // ggml_compute_forward_dup_same_cont(params, opt0, dst); - if (params->type == GGML_TASK_INIT) { + if (params->type == GGML_TASK_TYPE_INIT) { if (params->ith != 0) { return; } memset(dst->data, 0, ggml_nbytes(dst)); } - if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) { + if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) { return; } @@ -11538,14 +11538,14 @@ static void ggml_compute_forward_get_rows_back_f32( // ggml_compute_forward_dup_same_cont(params, opt0, dst); - if (params->type == GGML_TASK_INIT) { + if (params->type == GGML_TASK_TYPE_INIT) { if (params->ith != 0) { return; } memset(dst->data, 0, ggml_nbytes(dst)); } - if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) { + if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) { return; } @@ -11615,7 +11615,7 @@ static void ggml_compute_forward_diag_f32( GGML_ASSERT(params->ith == 0); - if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) { + if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) { return; } @@ -11684,7 +11684,7 @@ static void ggml_compute_forward_diag_mask_f32( GGML_ASSERT(n_past >= 0); - if (!inplace && (params->type == GGML_TASK_INIT)) { + if (!inplace && (params->type == GGML_TASK_TYPE_INIT)) { if (ith != 0) { return; } @@ -11698,7 +11698,7 @@ static void ggml_compute_forward_diag_mask_f32( ggml_nbytes(dst)); } - if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) { + if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) { return; } @@ -11772,7 +11772,7 @@ static void ggml_compute_forward_soft_max_f32( assert(ggml_is_contiguous(dst)); assert(ggml_are_same_shape(src0, dst)); - if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) { + if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) { return; } @@ -11910,7 +11910,7 @@ static void ggml_compute_forward_soft_max_back_f32( GGML_ASSERT(ggml_are_same_shape(src0, dst)); GGML_ASSERT(ggml_are_same_shape(src1, dst)); - if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) { + if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) { return; } @@ -12004,7 +12004,7 @@ static void ggml_compute_forward_alibi_f32( assert(params->ith == 0); - if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) { + if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) { return; } @@ -12063,7 +12063,7 @@ static void ggml_compute_forward_alibi_f16( assert(params->ith == 0); - if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) { + if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) { return; } @@ -12170,7 +12170,7 @@ static void ggml_compute_forward_clamp_f32( assert(params->ith == 0); - if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) { + if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) { return; } @@ -12310,7 +12310,7 @@ static void ggml_compute_forward_rope_f32( const struct ggml_tensor * src0 = dst->src[0]; const struct ggml_tensor * src1 = dst->src[1]; - if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) { + if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) { return; } @@ -12488,7 +12488,7 @@ static void ggml_compute_forward_rope_f16( const struct ggml_tensor * src0 = dst->src[0]; const struct ggml_tensor * src1 = dst->src[1]; - if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) { + if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) { return; } @@ -12719,7 +12719,7 @@ static void ggml_compute_forward_conv_transpose_1d_f16_f32( GGML_ASSERT(nb00 == sizeof(ggml_fp16_t)); GGML_ASSERT(nb10 == sizeof(float)); - if (params->type == GGML_TASK_INIT) { + if (params->type == GGML_TASK_TYPE_INIT) { if (ith != 0) { return; } @@ -12759,7 +12759,7 @@ static void ggml_compute_forward_conv_transpose_1d_f16_f32( return; } - if (params->type == GGML_TASK_FINALIZE) { + if (params->type == GGML_TASK_TYPE_FINALIZE) { return; } @@ -12818,7 +12818,7 @@ static void ggml_compute_forward_conv_transpose_1d_f32( GGML_ASSERT(nb00 == sizeof(float)); GGML_ASSERT(nb10 == sizeof(float)); - if (params->type == GGML_TASK_INIT) { + if (params->type == GGML_TASK_TYPE_INIT) { if (ith != 0) { return; } @@ -12858,7 +12858,7 @@ static void ggml_compute_forward_conv_transpose_1d_f32( return; } - if (params->type == GGML_TASK_FINALIZE) { + if (params->type == GGML_TASK_TYPE_FINALIZE) { return; } @@ -12962,11 +12962,11 @@ static void ggml_compute_forward_im2col_f32( GGML_ASSERT(nb00 == sizeof(ggml_fp16_t)); GGML_ASSERT(nb10 == sizeof(float)); - if (params->type == GGML_TASK_INIT) { + if (params->type == GGML_TASK_TYPE_INIT) { return; } - if (params->type == GGML_TASK_FINALIZE) { + if (params->type == GGML_TASK_TYPE_FINALIZE) { return; } @@ -13050,11 +13050,11 @@ static void ggml_compute_forward_im2col_f16( GGML_ASSERT(nb00 == sizeof(ggml_fp16_t)); GGML_ASSERT(nb10 == sizeof(float)); - if (params->type == GGML_TASK_INIT) { + if (params->type == GGML_TASK_TYPE_INIT) { return; } - if (params->type == GGML_TASK_FINALIZE) { + if (params->type == GGML_TASK_TYPE_FINALIZE) { return; } @@ -13136,7 +13136,7 @@ static void ggml_compute_forward_conv_transpose_2d( GGML_ASSERT(nb00 == sizeof(ggml_fp16_t)); GGML_ASSERT(nb10 == sizeof(float)); - if (params->type == GGML_TASK_INIT) { + if (params->type == GGML_TASK_TYPE_INIT) { if (ith != 0) { return; } @@ -13178,7 +13178,7 @@ static void ggml_compute_forward_conv_transpose_2d( return; } - if (params->type == GGML_TASK_FINALIZE) { + if (params->type == GGML_TASK_TYPE_FINALIZE) { return; } @@ -13230,7 +13230,7 @@ static void ggml_compute_forward_pool_1d_sk_p0( assert(src->type == GGML_TYPE_F32); assert(params->ith == 0); - if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) { + if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) { return; } @@ -13299,7 +13299,7 @@ static void ggml_compute_forward_pool_2d( GGML_ASSERT(src->type == GGML_TYPE_F32); GGML_ASSERT(params->ith == 0); - if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) { + if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) { return; } @@ -13372,7 +13372,7 @@ static void ggml_compute_forward_upscale_f32( const struct ggml_tensor * src0 = dst->src[0]; - if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) { + if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) { return; } @@ -13432,7 +13432,7 @@ static void ggml_compute_forward_pad_f32( const struct ggml_tensor * src0 = dst->src[0]; - if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) { + if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) { return; } @@ -13493,7 +13493,7 @@ static void ggml_compute_forward_argsort_f32( const struct ggml_tensor * src0 = dst->src[0]; - if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) { + if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) { return; } @@ -13519,8 +13519,8 @@ static void ggml_compute_forward_argsort_f32( // C doesn't have a functional sort, so we do a bubble sort instead for (int64_t j = 0; j < ne0; j++) { for (int64_t k = j + 1; k < ne0; k++) { - if ((order == GGML_SORT_ASC && src_data[dst_data[j]] > src_data[dst_data[k]]) || - (order == GGML_SORT_DESC && src_data[dst_data[j]] < src_data[dst_data[k]])) { + if ((order == GGML_SORT_ORDER_ASC && src_data[dst_data[j]] > src_data[dst_data[k]]) || + (order == GGML_SORT_ORDER_DESC && src_data[dst_data[j]] < src_data[dst_data[k]])) { int32_t tmp = dst_data[j]; dst_data[j] = dst_data[k]; dst_data[k] = tmp; @@ -13603,11 +13603,11 @@ static void ggml_compute_forward_flash_attn_f32( GGML_ASSERT(nb1 <= nb2); GGML_ASSERT(nb2 <= nb3); - if (params->type == GGML_TASK_INIT) { + if (params->type == GGML_TASK_TYPE_INIT) { return; } - if (params->type == GGML_TASK_FINALIZE) { + if (params->type == GGML_TASK_TYPE_FINALIZE) { return; } @@ -13795,11 +13795,11 @@ static void ggml_compute_forward_flash_attn_f16( GGML_ASSERT(nb1 <= nb2); GGML_ASSERT(nb2 <= nb3); - if (params->type == GGML_TASK_INIT) { + if (params->type == GGML_TASK_TYPE_INIT) { return; } - if (params->type == GGML_TASK_FINALIZE) { + if (params->type == GGML_TASK_TYPE_FINALIZE) { return; } @@ -14054,11 +14054,11 @@ static void ggml_compute_forward_flash_ff_f16( GGML_ASSERT(nb1 <= nb2); GGML_ASSERT(nb2 <= nb3); - if (params->type == GGML_TASK_INIT) { + if (params->type == GGML_TASK_TYPE_INIT) { return; } - if (params->type == GGML_TASK_FINALIZE) { + if (params->type == GGML_TASK_TYPE_FINALIZE) { return; } @@ -14213,14 +14213,14 @@ static void ggml_compute_forward_flash_attn_back_f32( GGML_ASSERT(nb1 <= nb2); GGML_ASSERT(nb2 <= nb3); - if (params->type == GGML_TASK_INIT) { + if (params->type == GGML_TASK_TYPE_INIT) { if (ith == 0) { memset(dst->data, 0, nb0*ne0*ne1*ne2*ne3); } return; } - if (params->type == GGML_TASK_FINALIZE) { + if (params->type == GGML_TASK_TYPE_FINALIZE) { return; } @@ -14536,7 +14536,7 @@ static void ggml_compute_forward_win_part_f32( const struct ggml_tensor * src0 = dst->src[0]; - if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) { + if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) { return; } @@ -14602,7 +14602,7 @@ static void ggml_compute_forward_win_unpart_f32( const struct ggml_tensor * src0 = dst->src[0]; - if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) { + if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) { return; } @@ -14730,7 +14730,7 @@ static void ggml_compute_forward_get_rel_pos_f16( const struct ggml_tensor * src0 = dst->src[0]; - if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) { + if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) { return; } @@ -14782,14 +14782,14 @@ static void ggml_compute_forward_add_rel_pos_f32( const struct ggml_tensor * src2 = dst->src[2]; const bool inplace = (bool) ((int32_t *) dst->op_params)[0]; - if (!inplace && params->type == GGML_TASK_INIT) { + if (!inplace && params->type == GGML_TASK_TYPE_INIT) { if (params->ith != 0) { return; } memcpy((char *) dst->data, (char *) src0->data, ggml_nbytes(dst)); return; } - if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) { + if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) { return; } @@ -14871,7 +14871,7 @@ static void ggml_compute_forward_map_unary_f32( GGML_ASSERT(ggml_are_same_shape(src0, dst)); - if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) { + if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) { return; } @@ -14920,7 +14920,7 @@ static void ggml_compute_forward_map_binary_f32( assert(params->ith == 0); assert(ggml_are_same_shape(src0, src1) && ggml_are_same_shape(src0, dst)); - if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) { + if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) { return; } @@ -14969,7 +14969,7 @@ static void ggml_compute_forward_map_custom1_f32( assert(params->ith == 0); - if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) { + if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) { return; } @@ -14988,7 +14988,7 @@ static void ggml_compute_forward_map_custom2_f32( assert(params->ith == 0); - if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) { + if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) { return; } @@ -15008,7 +15008,7 @@ static void ggml_compute_forward_map_custom3_f32( assert(params->ith == 0); - if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) { + if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) { return; } @@ -15023,7 +15023,7 @@ static void ggml_compute_forward_map_custom1( const struct ggml_tensor * a = dst->src[0]; - if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) { + if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) { return; } @@ -15041,7 +15041,7 @@ static void ggml_compute_forward_map_custom2( const struct ggml_tensor * a = dst->src[0]; const struct ggml_tensor * b = dst->src[1]; - if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) { + if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) { return; } @@ -15060,7 +15060,7 @@ static void ggml_compute_forward_map_custom3( const struct ggml_tensor * b = dst->src[1]; const struct ggml_tensor * c = dst->src[2]; - if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) { + if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) { return; } @@ -15094,14 +15094,14 @@ static void ggml_compute_forward_cross_entropy_loss_f32( GGML_ASSERT(params->wsize >= sizeof(float) * (nth + nth * nc)); - if (params->type == GGML_TASK_INIT) { + if (params->type == GGML_TASK_TYPE_INIT) { if (ith == 0) { memset(sums, 0, sizeof(float) * (nth + nth * nc)); } return; } - if (params->type == GGML_TASK_FINALIZE) { + if (params->type == GGML_TASK_TYPE_FINALIZE) { if (ith == 0) { float * dp = (float *) dst->data; ggml_vec_sum_f32(nth, dp, sums); @@ -15216,7 +15216,7 @@ static void ggml_compute_forward_cross_entropy_loss_back_f32( const int64_t ith = params->ith; const int64_t nth = params->nth; - if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) { + if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) { return; } @@ -15323,8 +15323,8 @@ static void ggml_compute_forward(struct ggml_compute_params * params, struct ggm if (skip_cpu) { return; } - GGML_ASSERT(tensor->src[0] == NULL || tensor->src[0]->backend == GGML_BACKEND_CPU); - GGML_ASSERT(tensor->src[1] == NULL || tensor->src[1]->backend == GGML_BACKEND_CPU); + GGML_ASSERT(tensor->src[0] == NULL || tensor->src[0]->backend == GGML_BACKEND_TYPE_CPU); + GGML_ASSERT(tensor->src[1] == NULL || tensor->src[1]->backend == GGML_BACKEND_TYPE_CPU); #elif defined(GGML_USE_VULKAN) const bool skip_cpu = ggml_vk_compute_forward_cpu_assist(params, tensor); #ifdef GGML_VULKAN_CHECK_RESULTS @@ -15335,8 +15335,8 @@ static void ggml_compute_forward(struct ggml_compute_params * params, struct ggm if (skip_cpu) { return; } - GGML_ASSERT(tensor->src[0] == NULL || tensor->src[0]->backend == GGML_BACKEND_CPU); - GGML_ASSERT(tensor->src[1] == NULL || tensor->src[1]->backend == GGML_BACKEND_CPU); + GGML_ASSERT(tensor->src[0] == NULL || tensor->src[0]->backend == GGML_BACKEND_TYPE_CPU); + GGML_ASSERT(tensor->src[1] == NULL || tensor->src[1]->backend == GGML_BACKEND_TYPE_CPU); #endif // GGML_USE_CUBLAS #ifdef GGML_USE_SYCL @@ -16882,7 +16882,7 @@ size_t ggml_graph_overhead(void) { struct ggml_cgraph * ggml_new_graph_custom(struct ggml_context * ctx, size_t size, bool grads) { const size_t obj_size = ggml_graph_nbytes(size, grads); - struct ggml_object * obj = ggml_new_object(ctx, GGML_OBJECT_GRAPH, obj_size); + struct ggml_object * obj = ggml_new_object(ctx, GGML_OBJECT_TYPE_GRAPH, obj_size); struct ggml_cgraph * cgraph = (struct ggml_cgraph *) ((char *) ctx->mem_buffer + obj->offs); struct ggml_tensor ** data_start = (struct ggml_tensor **) (cgraph + 1); @@ -17429,7 +17429,7 @@ static thread_ret_t ggml_graph_compute_thread(void * data) { set_numa_thread_affinity(state->ith); int node_n = -1; - int task_phase = GGML_TASK_FINALIZE; + int task_phase = GGML_TASK_TYPE_FINALIZE; while (true) { if (cplan->abort_callback && cplan->abort_callback(cplan->abort_callback_data)) { @@ -17441,7 +17441,7 @@ static thread_ret_t ggml_graph_compute_thread(void * data) { // all other threads are finished and spinning // do finalize and init here so we don't have synchronize again struct ggml_compute_params params = { - /*.type =*/ GGML_TASK_FINALIZE, + /*.type =*/ GGML_TASK_TYPE_FINALIZE, /*.ith =*/ 0, /*.nth =*/ 0, /*.wsize =*/ cplan->work_size, @@ -17472,17 +17472,17 @@ static thread_ret_t ggml_graph_compute_thread(void * data) { if (n_tasks == 1) { /* INIT */ if (GGML_OP_HAS_INIT[node->op]) { - params.type = GGML_TASK_INIT; + params.type = GGML_TASK_TYPE_INIT; ggml_compute_forward(¶ms, node); } // TODO: maybe push node_n to the atomic but if other threads see n_tasks is 1, // they do something more efficient than spinning (?) - params.type = GGML_TASK_COMPUTE; + params.type = GGML_TASK_TYPE_COMPUTE; ggml_compute_forward(¶ms, node); if (GGML_OP_HAS_FINALIZE[node->op]) { - params.type = GGML_TASK_FINALIZE; + params.type = GGML_TASK_TYPE_FINALIZE; ggml_compute_forward(¶ms, node); } @@ -17496,7 +17496,7 @@ static thread_ret_t ggml_graph_compute_thread(void * data) { } } - task_phase = GGML_TASK_INIT; + task_phase = GGML_TASK_TYPE_INIT; atomic_store(&state->shared->n_active, n_threads); atomic_store(&state->shared->node_n, node_n); atomic_store(&state->shared->node_task, task_phase); @@ -17513,7 +17513,7 @@ static thread_ret_t ggml_graph_compute_thread(void * data) { const int n_tasks = ggml_get_n_tasks(node, n_threads); struct ggml_compute_params params = { - /*.type =*/ GGML_TASK_INIT, + /*.type =*/ GGML_TASK_TYPE_INIT, /*.ith =*/ state->ith, /*.nth =*/ n_tasks, /*.wsize =*/ cplan->work_size, @@ -17527,7 +17527,7 @@ static thread_ret_t ggml_graph_compute_thread(void * data) { } if (atomic_fetch_sub(&state->shared->n_active, 1) == 1) { - task_phase = GGML_TASK_COMPUTE; + task_phase = GGML_TASK_TYPE_COMPUTE; atomic_store(&state->shared->n_active, n_threads); atomic_store(&state->shared->node_task, task_phase); } @@ -17542,12 +17542,12 @@ static thread_ret_t ggml_graph_compute_thread(void * data) { } if (state->ith < n_tasks) { - params.type = GGML_TASK_COMPUTE; + params.type = GGML_TASK_TYPE_COMPUTE; ggml_compute_forward(¶ms, node); } if (atomic_fetch_sub(&state->shared->n_active, 1) == 1) { - task_phase = GGML_TASK_FINALIZE; + task_phase = GGML_TASK_TYPE_FINALIZE; atomic_store(&state->shared->n_active, n_threads); atomic_store(&state->shared->node_task, task_phase); } @@ -17783,7 +17783,7 @@ int ggml_graph_compute(struct ggml_cgraph * cgraph, struct ggml_cplan * cplan) { /*.n_threads =*/ n_threads, /*.n_active =*/ n_threads, /*.node_n =*/ -1, - /*.node_task =*/ GGML_TASK_FINALIZE, + /*.node_task =*/ GGML_TASK_TYPE_FINALIZE, /*.abort_callback =*/ NULL, /*.abort_callback_data =*/ NULL, }; @@ -17851,7 +17851,7 @@ int ggml_graph_compute(struct ggml_cgraph * cgraph, struct ggml_cplan * cplan) { void ggml_graph_compute_with_ctx(struct ggml_context * ctx, struct ggml_cgraph * cgraph, int n_threads) { struct ggml_cplan cplan = ggml_graph_plan(cgraph, n_threads); - struct ggml_object * obj = ggml_new_object(ctx, GGML_OBJECT_WORK_BUFFER, cplan.work_size); + struct ggml_object * obj = ggml_new_object(ctx, GGML_OBJECT_TYPE_WORK_BUFFER, cplan.work_size); cplan.work_data = (uint8_t *)ctx->mem_buffer + obj->offs; @@ -18659,7 +18659,7 @@ static enum ggml_opt_result ggml_opt_adam( float * pf = params.past > 0 ? opt->adam.pf->data : NULL; // past function values struct ggml_cplan cplan = ggml_graph_plan(gb, params.n_threads); - struct ggml_object * obj = ggml_new_object(ctx, GGML_OBJECT_WORK_BUFFER, cplan.work_size); + struct ggml_object * obj = ggml_new_object(ctx, GGML_OBJECT_TYPE_WORK_BUFFER, cplan.work_size); cplan.work_data = (uint8_t *)ctx->mem_buffer + obj->offs; bool cancel = false; @@ -18671,7 +18671,7 @@ static enum ggml_opt_result ggml_opt_adam( if (callback) { callback(callback_data, accum_step, &sched, &cancel); if (cancel) { - return GGML_OPT_CANCEL; + return GGML_OPT_RESULT_CANCEL; } } // ggml_graph_reset (gf); @@ -18762,7 +18762,7 @@ static enum ggml_opt_result ggml_opt_adam( if (callback) { callback(callback_data, accum_step, &sched, &cancel); if (cancel) { - return GGML_OPT_CANCEL;; + return GGML_OPT_RESULT_CANCEL;; } } // ggml_graph_reset (gf); @@ -18779,7 +18779,7 @@ static enum ggml_opt_result ggml_opt_adam( if (fabsf(fx - fx_prev[0])/fx < params.adam.eps_f) { GGML_PRINT_DEBUG("converged\n"); - return GGML_OPT_OK; + return GGML_OPT_RESULT_OK; } // delta-based convergence test @@ -18789,7 +18789,7 @@ static enum ggml_opt_result ggml_opt_adam( const float rate = (pf[(iter0 + t)%params.past] - fx)/fx; if (fabsf(rate) < params.delta) { - return GGML_OPT_OK; + return GGML_OPT_RESULT_OK; } } @@ -18805,7 +18805,7 @@ static enum ggml_opt_result ggml_opt_adam( ++n_no_improvement[0]; if (n_no_improvement[0] >= params.max_no_improvement) { - return GGML_OPT_OK; + return GGML_OPT_RESULT_OK; } } } @@ -18823,7 +18823,7 @@ static enum ggml_opt_result ggml_opt_adam( } } - return GGML_OPT_DID_NOT_CONVERGE; + return GGML_OPT_RESULT_DID_NOT_CONVERGE; } // @@ -18904,7 +18904,7 @@ static enum ggml_opt_result linesearch_backtracking( float sched = 0; callback(callback_data, accum_step, &sched, cancel); if (*cancel) { - return GGML_OPT_CANCEL; + return GGML_OPT_RESULT_CANCEL; } } // ggml_graph_reset (gf); @@ -18977,7 +18977,7 @@ static enum ggml_opt_result ggml_opt_lbfgs( if (params.lbfgs.linesearch == GGML_LINESEARCH_BACKTRACKING_WOLFE || params.lbfgs.linesearch == GGML_LINESEARCH_BACKTRACKING_STRONG_WOLFE) { if (params.lbfgs.wolfe <= params.lbfgs.ftol || 1.f <= params.lbfgs.wolfe) { - return GGML_OPT_INVALID_WOLFE; + return GGML_OPT_RESULT_INVALID_WOLFE; } } @@ -19006,7 +19006,7 @@ static enum ggml_opt_result ggml_opt_lbfgs( } struct ggml_cplan cplan = ggml_graph_plan(gb, params.n_threads); - struct ggml_object * obj = ggml_new_object(ctx, GGML_OBJECT_WORK_BUFFER, cplan.work_size); + struct ggml_object * obj = ggml_new_object(ctx, GGML_OBJECT_TYPE_WORK_BUFFER, cplan.work_size); cplan.work_data = (uint8_t *)ctx->mem_buffer + obj->offs; float * x = opt->lbfgs.x->data; // current parameters @@ -19047,7 +19047,7 @@ static enum ggml_opt_result ggml_opt_lbfgs( float sched = 0; callback(callback_data, accum_step, &sched, &cancel); if (cancel) { - return GGML_OPT_CANCEL; + return GGML_OPT_RESULT_CANCEL; } } // ggml_graph_reset (gf); @@ -19075,7 +19075,7 @@ static enum ggml_opt_result ggml_opt_lbfgs( // already optimized if (gnorm/xnorm <= params.lbfgs.eps) { - return GGML_OPT_OK; + return GGML_OPT_RESULT_OK; } if (opt->just_initialized) { @@ -19120,7 +19120,7 @@ static enum ggml_opt_result ggml_opt_lbfgs( // way to test and don't want to break something with so many changes lined up ls = linesearch_backtracking(¶ms, nx, x, &fx, g, d, step, xp, f, gb, &cplan, np, ps, &cancel, callback, callback_data); if (cancel) { - return GGML_OPT_CANCEL; + return GGML_OPT_RESULT_CANCEL; } if (ls < 0) { @@ -19143,7 +19143,7 @@ static enum ggml_opt_result ggml_opt_lbfgs( } if (gnorm/xnorm <= params.lbfgs.eps) { // converged - return GGML_OPT_OK; + return GGML_OPT_RESULT_OK; } // delta-based convergence test @@ -19153,7 +19153,7 @@ static enum ggml_opt_result ggml_opt_lbfgs( const float rate = (pf[k[0]%params.past] - fx)/fx; if (fabsf(rate) < params.delta) { - return GGML_OPT_OK; + return GGML_OPT_RESULT_OK; } } @@ -19169,14 +19169,14 @@ static enum ggml_opt_result ggml_opt_lbfgs( n_no_improvement[0]++; if (n_no_improvement[0] >= params.max_no_improvement) { - return GGML_OPT_OK; + return GGML_OPT_RESULT_OK; } } } if (params.lbfgs.n_iter != 0 && params.lbfgs.n_iter < it + 1) { // reached the maximum number of iterations - return GGML_OPT_DID_NOT_CONVERGE; + return GGML_OPT_RESULT_DID_NOT_CONVERGE; } // update vectors s and y: @@ -19232,17 +19232,17 @@ static enum ggml_opt_result ggml_opt_lbfgs( GGML_ASSERT(false && "lbfgs failed"); - return GGML_OPT_DID_NOT_CONVERGE; + return GGML_OPT_RESULT_DID_NOT_CONVERGE; } struct ggml_opt_params ggml_opt_default_params(enum ggml_opt_type type) { struct ggml_opt_params result; switch (type) { - case GGML_OPT_ADAM: + case GGML_OPT_TYPE_ADAM: { result = (struct ggml_opt_params) { - .type = GGML_OPT_ADAM, + .type = GGML_OPT_TYPE_ADAM, .graph_size = GGML_DEFAULT_GRAPH_SIZE, .n_threads = 1, // FIXME: GGML_DEFAULT_N_THREADS ? .past = 0, @@ -19270,10 +19270,10 @@ struct ggml_opt_params ggml_opt_default_params(enum ggml_opt_type type) { }, }; } break; - case GGML_OPT_LBFGS: + case GGML_OPT_TYPE_LBFGS: { result = (struct ggml_opt_params) { - .type = GGML_OPT_LBFGS, + .type = GGML_OPT_TYPE_LBFGS, .graph_size = GGML_DEFAULT_GRAPH_SIZE, .n_threads = 1, .past = 0, @@ -19318,12 +19318,12 @@ GGML_API void ggml_opt_init( opt->just_initialized = true; if (opt->ctx == NULL) { struct ggml_init_params ctx_opt_params; - if (opt->params.type == GGML_OPT_ADAM) { + if (opt->params.type == GGML_OPT_TYPE_ADAM) { ctx_opt_params.mem_size = GGML_MEM_ALIGN*3 + ggml_tensor_overhead()*3 + ggml_type_size(GGML_TYPE_F32)*nx*3; if (opt->params.past > 0) { ctx_opt_params.mem_size += GGML_MEM_ALIGN + ggml_tensor_overhead() + ggml_type_size(GGML_TYPE_F32)*opt->params.past; } - } else if (opt->params.type == GGML_OPT_LBFGS) { + } else if (opt->params.type == GGML_OPT_TYPE_LBFGS) { ctx_opt_params.mem_size = GGML_MEM_ALIGN*9 + ggml_tensor_overhead()*9 + ggml_type_size(GGML_TYPE_F32)*(nx*5 + opt->params.lbfgs.m*2 + nx*opt->params.lbfgs.m*2); if (opt->params.past > 0) { ctx_opt_params.mem_size += GGML_MEM_ALIGN + ggml_tensor_overhead() + ggml_type_size(GGML_TYPE_F32)*opt->params.past; @@ -19335,7 +19335,7 @@ GGML_API void ggml_opt_init( opt->ctx = ggml_init(ctx_opt_params); } switch (opt->params.type) { - case GGML_OPT_ADAM: + case GGML_OPT_TYPE_ADAM: { opt->adam.g = ggml_new_tensor_1d(opt->ctx, GGML_TYPE_F32, nx); opt->adam.m = ggml_new_tensor_1d(opt->ctx, GGML_TYPE_F32, nx); @@ -19349,7 +19349,7 @@ GGML_API void ggml_opt_init( ggml_set_zero(opt->adam.pf); } } break; - case GGML_OPT_LBFGS: + case GGML_OPT_TYPE_LBFGS: { opt->lbfgs.x = ggml_new_tensor_1d(opt->ctx, GGML_TYPE_F32, nx); opt->lbfgs.xp = ggml_new_tensor_1d(opt->ctx, GGML_TYPE_F32, nx); @@ -19393,13 +19393,13 @@ enum ggml_opt_result ggml_opt( ctx = ggml_init(params_ctx); if (ctx == NULL) { - return GGML_OPT_NO_CONTEXT; + return GGML_OPT_RESULT_NO_CONTEXT; } free_ctx = true; } - enum ggml_opt_result result = GGML_OPT_OK; + enum ggml_opt_result result = GGML_OPT_RESULT_OK; struct ggml_opt_context * opt = (struct ggml_opt_context *) alloca(sizeof(struct ggml_opt_context)); @@ -19438,14 +19438,14 @@ enum ggml_opt_result ggml_opt_resume_g( void * callback_data) { // build forward + backward compute graphs - enum ggml_opt_result result = GGML_OPT_OK; + enum ggml_opt_result result = GGML_OPT_RESULT_OK; switch (opt->params.type) { - case GGML_OPT_ADAM: + case GGML_OPT_TYPE_ADAM: { result = ggml_opt_adam(ctx, opt, opt->params, f, gf, gb, callback, callback_data); } break; - case GGML_OPT_LBFGS: + case GGML_OPT_TYPE_LBFGS: { result = ggml_opt_lbfgs(ctx, opt, opt->params, f, gf, gb, callback, callback_data); } break; diff --git a/ggml.h b/ggml.h index a4166e1f7..75fd035a4 100644 --- a/ggml.h +++ b/ggml.h @@ -364,9 +364,9 @@ extern "C" { }; enum ggml_backend_type { - GGML_BACKEND_CPU = 0, - GGML_BACKEND_GPU = 10, - GGML_BACKEND_GPU_SPLIT = 20, + GGML_BACKEND_TYPE_CPU = 0, + GGML_BACKEND_TYPE_GPU = 10, + GGML_BACKEND_TYPE_GPU_SPLIT = 20, }; // model file types @@ -498,9 +498,9 @@ extern "C" { }; enum ggml_object_type { - GGML_OBJECT_TENSOR, - GGML_OBJECT_GRAPH, - GGML_OBJECT_WORK_BUFFER + GGML_OBJECT_TYPE_TENSOR, + GGML_OBJECT_TYPE_GRAPH, + GGML_OBJECT_TYPE_WORK_BUFFER }; enum ggml_log_level { @@ -642,9 +642,9 @@ extern "C" { // NOTE: the INIT or FINALIZE pass is not scheduled unless explicitly enabled. // This behavior was changed since https://github.com/ggerganov/llama.cpp/pull/1995. enum ggml_task_type { - GGML_TASK_INIT = 0, - GGML_TASK_COMPUTE, - GGML_TASK_FINALIZE, + GGML_TASK_TYPE_INIT = 0, + GGML_TASK_TYPE_COMPUTE, + GGML_TASK_TYPE_FINALIZE, }; struct ggml_compute_params { @@ -1649,8 +1649,8 @@ extern "C" { // sort rows enum ggml_sort_order { - GGML_SORT_ASC, - GGML_SORT_DESC, + GGML_SORT_ORDER_ASC, + GGML_SORT_ORDER_DESC, }; GGML_API struct ggml_tensor * ggml_argsort( @@ -1943,8 +1943,8 @@ extern "C" { // optimization methods enum ggml_opt_type { - GGML_OPT_ADAM, - GGML_OPT_LBFGS, + GGML_OPT_TYPE_ADAM, + GGML_OPT_TYPE_LBFGS, }; // linesearch methods @@ -1958,12 +1958,12 @@ extern "C" { // optimization return values enum ggml_opt_result { - GGML_OPT_OK = 0, - GGML_OPT_DID_NOT_CONVERGE, - GGML_OPT_NO_CONTEXT, - GGML_OPT_INVALID_WOLFE, - GGML_OPT_FAIL, - GGML_OPT_CANCEL, + GGML_OPT_RESULT_OK = 0, + GGML_OPT_RESULT_DID_NOT_CONVERGE, + GGML_OPT_RESULT_NO_CONTEXT, + GGML_OPT_RESULT_INVALID_WOLFE, + GGML_OPT_RESULT_FAIL, + GGML_OPT_RESULT_CANCEL, GGML_LINESEARCH_FAIL = -128, GGML_LINESEARCH_MINIMUM_STEP, diff --git a/llama.cpp b/llama.cpp index 1f6b6cff4..acd9be08a 100644 --- a/llama.cpp +++ b/llama.cpp @@ -850,9 +850,9 @@ struct LLM_TN { // static std::map LLAMA_ROPE_SCALING_TYPES = { - { LLAMA_ROPE_SCALING_NONE, "none" }, - { LLAMA_ROPE_SCALING_LINEAR, "linear" }, - { LLAMA_ROPE_SCALING_YARN, "yarn" }, + { LLAMA_ROPE_SCALING_TYPE_NONE, "none" }, + { LLAMA_ROPE_SCALING_TYPE_LINEAR, "linear" }, + { LLAMA_ROPE_SCALING_TYPE_YARN, "yarn" }, }; static int32_t llama_rope_scaling_type_from_string(const std::string & name) { @@ -862,7 +862,7 @@ static int32_t llama_rope_scaling_type_from_string(const std::string & name) { } } - return LLAMA_ROPE_SCALING_UNSPECIFIED; + return LLAMA_ROPE_SCALING_TYPE_UNSPECIFIED; } static std::string gguf_data_to_str(enum gguf_type type, const void * data, int i) { @@ -1580,7 +1580,7 @@ struct llama_hparams { bool causal_attn = true; bool need_kq_pos = false; - uint32_t pooling_type = LLAMA_POOLING_NONE; + uint32_t pooling_type = LLAMA_POOLING_TYPE_NONE; bool operator!=(const llama_hparams & other) const { if (this->vocab_only != other.vocab_only) return true; @@ -2345,9 +2345,9 @@ namespace GGUFMeta { static const char * override_type_to_str(const llama_model_kv_override_type ty) { switch (ty) { - case LLAMA_KV_OVERRIDE_BOOL: return "bool"; - case LLAMA_KV_OVERRIDE_INT: return "int"; - case LLAMA_KV_OVERRIDE_FLOAT: return "float"; + case LLAMA_KV_OVERRIDE_TYPE_BOOL: return "bool"; + case LLAMA_KV_OVERRIDE_TYPE_INT: return "int"; + case LLAMA_KV_OVERRIDE_TYPE_FLOAT: return "float"; } return "unknown"; } @@ -2358,13 +2358,13 @@ namespace GGUFMeta { LLAMA_LOG_INFO("%s: Using metadata override (%5s) '%s' = ", __func__, override_type_to_str(override->tag), override->key); switch (override->tag) { - case LLAMA_KV_OVERRIDE_BOOL: { + case LLAMA_KV_OVERRIDE_TYPE_BOOL: { LLAMA_LOG_INFO("%s\n", override->bool_value ? "true" : "false"); } break; - case LLAMA_KV_OVERRIDE_INT: { + case LLAMA_KV_OVERRIDE_TYPE_INT: { LLAMA_LOG_INFO("%" PRId64 "\n", override->int_value); } break; - case LLAMA_KV_OVERRIDE_FLOAT: { + case LLAMA_KV_OVERRIDE_TYPE_FLOAT: { LLAMA_LOG_INFO("%.6f\n", override->float_value); } break; default: @@ -2383,7 +2383,7 @@ namespace GGUFMeta { template static typename std::enable_if::value, bool>::type try_override(OT & target, const struct llama_model_kv_override *override) { - if (validate_override(LLAMA_KV_OVERRIDE_BOOL, override)) { + if (validate_override(LLAMA_KV_OVERRIDE_TYPE_BOOL, override)) { target = override->bool_value; return true; } @@ -2393,7 +2393,7 @@ namespace GGUFMeta { template static typename std::enable_if::value && std::is_integral::value, bool>::type try_override(OT & target, const struct llama_model_kv_override *override) { - if (validate_override(LLAMA_KV_OVERRIDE_INT, override)) { + if (validate_override(LLAMA_KV_OVERRIDE_TYPE_INT, override)) { target = override->int_value; return true; } @@ -2403,7 +2403,7 @@ namespace GGUFMeta { template static typename std::enable_if::value, bool>::type try_override(T & target, const struct llama_model_kv_override *override) { - if (validate_override(LLAMA_KV_OVERRIDE_FLOAT, override)) { + if (validate_override(LLAMA_KV_OVERRIDE_TYPE_FLOAT, override)) { target = override->float_value; return true; } @@ -2999,7 +2999,7 @@ static void llm_load_hparams( std::string rope_scaling("linear"); ml.get_key(LLM_KV_ROPE_SCALING_TYPE, rope_scaling, false); hparams.rope_scaling_type_train = llama_rope_scaling_type_from_string(rope_scaling); - GGML_ASSERT(hparams.rope_scaling_type_train != LLAMA_ROPE_SCALING_UNSPECIFIED); + GGML_ASSERT(hparams.rope_scaling_type_train != LLAMA_ROPE_SCALING_TYPE_UNSPECIFIED); // rope_freq_scale (inverse of the kv) is optional float ropescale = 0.0f; @@ -3643,7 +3643,7 @@ static bool llm_load_tensors( model.buft_layer[i] = llama_default_buffer_type_cpu(true); } - if (split_mode == LLAMA_SPLIT_LAYER) { + if (split_mode == LLAMA_SPLIT_MODE_LAYER) { // calculate the split points int device_count = llama_get_device_count(); bool all_zero = tensor_split == nullptr || std::all_of(tensor_split, tensor_split + device_count, [](float x) { return x == 0.0f; }); @@ -3682,10 +3682,10 @@ static bool llm_load_tensors( } } else { ggml_backend_buffer_type_t split_buft; - if (split_mode == LLAMA_SPLIT_ROW) { + if (split_mode == LLAMA_SPLIT_MODE_ROW) { split_buft = llama_default_buffer_type_split(main_gpu, tensor_split); } else { - // LLAMA_SPLIT_NONE or LLAMA_SPLIT_LAYER in backends where it is not supported + // LLAMA_SPLIT_MODE_NONE or LLAMA_SPLIT_MODE_LAYER in backends where it is not supported split_buft = llama_default_buffer_type_offload(main_gpu); } // assign the repeating layers @@ -5070,7 +5070,7 @@ struct llm_build_context { kv_head (worst_case ? n_ctx - n_tokens : kv_self.head), n_orig_ctx (cparams.n_yarn_orig_ctx), do_rope_shift (worst_case || kv_self.has_shift), - pooling_type (cparams.do_pooling ? hparams.pooling_type : (uint32_t)LLAMA_POOLING_NONE), + pooling_type (cparams.do_pooling ? hparams.pooling_type : (uint32_t)LLAMA_POOLING_TYPE_NONE), cb (cb), buf_compute_meta (lctx.buf_compute_meta) { // all initializations should be done in init() @@ -6050,12 +6050,12 @@ struct llm_build_context { cur = inpL; // pooling layer - if (pooling_type == LLAMA_POOLING_MEAN) { + if (pooling_type == LLAMA_POOLING_TYPE_MEAN) { cur = ggml_mul_mat(ctx0, ggml_cont(ctx0, ggml_transpose(ctx0, cur)), inp_mean); - } else if (pooling_type == LLAMA_POOLING_CLS) { + } else if (pooling_type == LLAMA_POOLING_TYPE_CLS) { cur = ggml_get_rows(ctx0, cur, inp_cls); } else { - GGML_ASSERT(pooling_type == LLAMA_POOLING_NONE && "Invalid pooling type"); + GGML_ASSERT(pooling_type == LLAMA_POOLING_TYPE_NONE && "Invalid pooling type"); } cb(cur, "result_embd", -1); @@ -7754,7 +7754,7 @@ static void llama_set_inputs(llama_context & lctx, const llama_batch & batch) { } } - if (cparams.do_pooling && hparams.pooling_type == LLAMA_POOLING_MEAN) { + if (cparams.do_pooling && hparams.pooling_type == LLAMA_POOLING_TYPE_MEAN) { const int64_t n_tokens = batch.n_tokens; GGML_ASSERT(ggml_backend_buffer_is_host(lctx.inp_mean->buffer)); @@ -7782,7 +7782,7 @@ static void llama_set_inputs(llama_context & lctx, const llama_batch & batch) { } } - if (cparams.do_pooling && hparams.pooling_type == LLAMA_POOLING_CLS) { + if (cparams.do_pooling && hparams.pooling_type == LLAMA_POOLING_TYPE_CLS) { const int64_t n_tokens = batch.n_tokens; GGML_ASSERT(ggml_backend_buffer_is_host(lctx.inp_cls->buffer)); @@ -11351,7 +11351,7 @@ static int llama_apply_lora_from_file_internal( struct llama_model_params llama_model_default_params() { struct llama_model_params result = { /*.n_gpu_layers =*/ 0, - /*.split_mode =*/ LLAMA_SPLIT_LAYER, + /*.split_mode =*/ LLAMA_SPLIT_MODE_LAYER, /*.main_gpu =*/ 0, /*.tensor_split =*/ nullptr, /*.progress_callback =*/ nullptr, @@ -11377,7 +11377,7 @@ struct llama_context_params llama_context_default_params() { /*.n_batch =*/ 512, /*.n_threads =*/ GGML_DEFAULT_N_THREADS, // TODO: better default /*.n_threads_batch =*/ GGML_DEFAULT_N_THREADS, - /*.rope_scaling_type =*/ LLAMA_ROPE_SCALING_UNSPECIFIED, + /*.rope_scaling_type =*/ LLAMA_ROPE_SCALING_TYPE_UNSPECIFIED, /*.rope_freq_base =*/ 0.0f, /*.rope_freq_scale =*/ 0.0f, /*.yarn_ext_factor =*/ -1.0f, @@ -11565,16 +11565,16 @@ struct llama_context * llama_new_context_with_model( cparams.cb_eval_user_data = params.cb_eval_user_data; auto rope_scaling_type = params.rope_scaling_type; - if (rope_scaling_type == LLAMA_ROPE_SCALING_UNSPECIFIED) { + if (rope_scaling_type == LLAMA_ROPE_SCALING_TYPE_UNSPECIFIED) { rope_scaling_type = hparams.rope_scaling_type_train; } - if (rope_scaling_type == LLAMA_ROPE_SCALING_NONE) { + if (rope_scaling_type == LLAMA_ROPE_SCALING_TYPE_NONE) { cparams.rope_freq_scale = 1.0f; // never scale if scaling type is none } if (cparams.yarn_ext_factor < 0.0f) { // negative indicates 'not set' - cparams.yarn_ext_factor = rope_scaling_type == LLAMA_ROPE_SCALING_YARN ? 1.0f : 0.0f; + cparams.yarn_ext_factor = rope_scaling_type == LLAMA_ROPE_SCALING_TYPE_YARN ? 1.0f : 0.0f; } if (params.seed == LLAMA_DEFAULT_SEED) { @@ -11608,8 +11608,8 @@ struct llama_context * llama_new_context_with_model( } #elif defined(GGML_USE_CUBLAS) if (model->n_gpu_layers > 0) { - // with split_mode LLAMA_SPLIT_NONE or LLAMA_SPLIT_ROW, only the main GPU backend is used - if (model->split_mode == LLAMA_SPLIT_NONE || model->split_mode == LLAMA_SPLIT_ROW) { + // with split_mode LLAMA_SPLIT_MODE_NONE or LLAMA_SPLIT_MODE_ROW, only the main GPU backend is used + if (model->split_mode == LLAMA_SPLIT_MODE_NONE || model->split_mode == LLAMA_SPLIT_MODE_ROW) { ggml_backend_t backend = ggml_backend_cuda_init(model->main_gpu); if (backend == nullptr) { LLAMA_LOG_ERROR("%s: failed to initialize CUDA%d backend\n", __func__, model->main_gpu); @@ -11618,7 +11618,7 @@ struct llama_context * llama_new_context_with_model( } ctx->backends.push_back(backend); } else { - // LLAMA_SPLIT_LAYER requires a backend for each GPU + // LLAMA_SPLIT_MODE_LAYER requires a backend for each GPU for (int device = 0; device < ggml_backend_cuda_get_device_count(); ++device) { ggml_backend_t backend = ggml_backend_cuda_init(device); if (backend == nullptr) { diff --git a/llama.h b/llama.h index 889edf4d9..947284ea2 100644 --- a/llama.h +++ b/llama.h @@ -109,23 +109,23 @@ extern "C" { }; enum llama_rope_scaling_type { - LLAMA_ROPE_SCALING_UNSPECIFIED = -1, - LLAMA_ROPE_SCALING_NONE = 0, - LLAMA_ROPE_SCALING_LINEAR = 1, - LLAMA_ROPE_SCALING_YARN = 2, - LLAMA_ROPE_SCALING_MAX_VALUE = LLAMA_ROPE_SCALING_YARN, + LLAMA_ROPE_SCALING_TYPE_UNSPECIFIED = -1, + LLAMA_ROPE_SCALING_TYPE_NONE = 0, + LLAMA_ROPE_SCALING_TYPE_LINEAR = 1, + LLAMA_ROPE_SCALING_TYPE_YARN = 2, + LLAMA_ROPE_SCALING_TYPE_MAX_VALUE = LLAMA_ROPE_SCALING_TYPE_YARN, }; enum llama_pooling_type { - LLAMA_POOLING_NONE = 0, - LLAMA_POOLING_MEAN = 1, - LLAMA_POOLING_CLS = 2, + LLAMA_POOLING_TYPE_NONE = 0, + LLAMA_POOLING_TYPE_MEAN = 1, + LLAMA_POOLING_TYPE_CLS = 2, }; enum llama_split_mode { - LLAMA_SPLIT_NONE = 0, // single GPU - LLAMA_SPLIT_LAYER = 1, // split layers and KV across GPUs - LLAMA_SPLIT_ROW = 2, // split rows across GPUs + LLAMA_SPLIT_MODE_NONE = 0, // single GPU + LLAMA_SPLIT_MODE_LAYER = 1, // split layers and KV across GPUs + LLAMA_SPLIT_MODE_ROW = 2, // split rows across GPUs }; typedef struct llama_token_data { @@ -173,9 +173,9 @@ extern "C" { } llama_batch; enum llama_model_kv_override_type { - LLAMA_KV_OVERRIDE_INT, - LLAMA_KV_OVERRIDE_FLOAT, - LLAMA_KV_OVERRIDE_BOOL, + LLAMA_KV_OVERRIDE_TYPE_INT, + LLAMA_KV_OVERRIDE_TYPE_FLOAT, + LLAMA_KV_OVERRIDE_TYPE_BOOL, }; struct llama_model_kv_override { diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index f8574588b..24d12ef14 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -1264,7 +1264,7 @@ struct test_argsort : public test_case { test_argsort(ggml_type type = GGML_TYPE_F32, std::array ne = {16, 10, 10, 10}, - ggml_sort_order order = GGML_SORT_ASC) + ggml_sort_order order = GGML_SORT_ORDER_ASC) : type(type), ne(ne), order(order) {} ggml_tensor * build_graph(ggml_context * ctx) override { @@ -2116,7 +2116,7 @@ static bool test_backend(ggml_backend_t backend, test_mode mode, const char * op test_cases.emplace_back(new test_concat(GGML_TYPE_F32)); test_cases.emplace_back(new test_concat(GGML_TYPE_I32)); - for (ggml_sort_order order : {GGML_SORT_ASC, GGML_SORT_DESC}) { + for (ggml_sort_order order : {GGML_SORT_ORDER_ASC, GGML_SORT_ORDER_DESC}) { test_cases.emplace_back(new test_argsort(GGML_TYPE_F32, {8, 1, 1, 1}, order)); test_cases.emplace_back(new test_argsort(GGML_TYPE_F32, {16, 10, 10, 10}, order)); } diff --git a/tests/test-opt.cpp b/tests/test-opt.cpp index 2c9997fca..546ca230b 100644 --- a/tests/test-opt.cpp +++ b/tests/test-opt.cpp @@ -118,7 +118,7 @@ int main(void) { const float fe = ggml_get_f32_1d(e, 0); printf("%s: e = %.4f\n", __func__, fe); - struct ggml_opt_params opt_params = ggml_opt_default_params(GGML_OPT_ADAM); + struct ggml_opt_params opt_params = ggml_opt_default_params(GGML_OPT_TYPE_ADAM); ggml_opt(ctx, opt_params, e); From 12894088170f62e4cad4f8d6a3043c185b414bab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Gryta?= Date: Sun, 25 Feb 2024 11:53:11 +0100 Subject: [PATCH 453/812] cmake : fix compilation for Android armeabi-v7a (#5702) --- CMakeLists.txt | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 3c4629001..48880f720 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -936,10 +936,16 @@ if (CMAKE_OSX_ARCHITECTURES STREQUAL "arm64" OR CMAKE_GENERATOR_PLATFORM_LWR STR list(APPEND ARCH_FLAGS -mfpu=neon-fp-armv8 -mno-unaligned-access) endif() if (${CMAKE_SYSTEM_PROCESSOR} MATCHES "armv7") - # Raspberry Pi 2 - list(APPEND ARCH_FLAGS -mfpu=neon-fp-armv8 -mno-unaligned-access -funsafe-math-optimizations) + if ("${CMAKE_SYSTEM_NAME}" STREQUAL "Android") + # Android armeabi-v7a + list(APPEND ARCH_FLAGS -mfpu=neon-vfpv4 -mno-unaligned-access -funsafe-math-optimizations) + else() + # Raspberry Pi 2 + list(APPEND ARCH_FLAGS -mfpu=neon-fp-armv8 -mno-unaligned-access -funsafe-math-optimizations) + endif() endif() if (${CMAKE_SYSTEM_PROCESSOR} MATCHES "armv8") + # Android arm64-v8a # Raspberry Pi 3, 4, Zero 2 (32-bit) list(APPEND ARCH_FLAGS -mno-unaligned-access) endif() From d52d7819b8ced70c642a88a59da8c78208dc58ec Mon Sep 17 00:00:00 2001 From: Pierrick Hymbert Date: Sun, 25 Feb 2024 13:49:43 +0100 Subject: [PATCH 454/812] server: concurrency fix + monitoring - add /metrics prometheus compatible endpoint (#5708) * server: monitoring - add /metrics prometheus compatible endpoint * server: concurrency issue, when 2 task are waiting for results, only one call thread is notified * server: metrics - move to a dedicated struct --- examples/server/README.md | 13 ++ examples/server/server.cpp | 150 +++++++++++++++++- examples/server/tests/features/environment.py | 2 + examples/server/tests/features/server.feature | 2 + examples/server/tests/features/steps/steps.py | 27 ++++ examples/server/tests/requirements.txt | 1 + examples/server/utils.hpp | 4 +- 7 files changed, 191 insertions(+), 8 deletions(-) diff --git a/examples/server/README.md b/examples/server/README.md index 0c43ac4c9..2129f7fb2 100644 --- a/examples/server/README.md +++ b/examples/server/README.md @@ -41,6 +41,7 @@ see https://github.com/ggerganov/llama.cpp/issues/1437 - `--grp-attn-w`: Set the group attention width to extend context size through self-extend(default: 512), used together with group attention factor `--grp-attn-n` - `-n, --n-predict`: Set the maximum tokens to predict (default: -1) - `--slots-endpoint-disable`: To disable slots state monitoring endpoint. Slots state may contain user data, prompts included. +- `--metrics`: enable prometheus `/metrics` compatible endpoint (default: disabled) - `--chat-template JINJA_TEMPLATE`: Set custom jinja chat template. This parameter accepts a string, not a file name (default: template taken from model's metadata). We only support [some pre-defined templates](https://github.com/ggerganov/llama.cpp/wiki/Templates-supported-by-llama_chat_apply_template) ## Build @@ -457,6 +458,18 @@ Notice that each `probs` is an array of length `n_probs`. ] ``` +- **GET** `/metrics`: [Prometheus](https://prometheus.io/) compatible metrics exporter endpoint if `--metrics` is enabled: + +Available metrics: +- `llamacpp:prompt_tokens_total`: Number of prompt tokens processed. +- `llamacpp:tokens_predicted_total`: Number of generation tokens processed. +- `llamacpp:prompt_tokens_seconds`: Average prompt throughput in tokens/s. +- `llamacpp:predicted_tokens_seconds`: Average generation throughput in tokens/s. +- `llamacpp:kv_cache_usage_ratio`: KV-cache usage. 1 means 100 percent usage. +- `llamacpp:kv_cache_tokens`: KV-cache tokens. +- `llamacpp:requests_processing`: Number of request processing. +- `llamacpp:requests_deferred`: Number of request deferred. + ## More examples ### Change system prompt on runtime diff --git a/examples/server/server.cpp b/examples/server/server.cpp index 780862ef6..811495915 100644 --- a/examples/server/server.cpp +++ b/examples/server/server.cpp @@ -43,6 +43,7 @@ struct server_params int32_t read_timeout = 600; int32_t write_timeout = 600; bool slots_endpoint = true; + bool metrics_endpoint = false; }; bool server_verbose = false; @@ -310,6 +311,39 @@ struct llama_client_slot } }; +struct llama_metrics { + uint64_t n_prompt_tokens_processed_total = 0; + uint64_t n_tokens_predicted_total = 0; + + uint64_t n_prompt_tokens_processed = 0; + uint64_t t_prompt_processing = 0; + + uint64_t n_tokens_predicted = 0; + uint64_t t_tokens_generation = 0; + + + void on_prompt_eval(const llama_client_slot &slot) { + n_prompt_tokens_processed_total += slot.num_prompt_tokens_processed; + + n_prompt_tokens_processed += slot.num_prompt_tokens_processed; + t_prompt_processing += slot.t_prompt_processing; + } + + void on_prediction(const llama_client_slot &slot) { + n_tokens_predicted_total += slot.n_decoded; + + n_tokens_predicted += slot.n_decoded; + t_tokens_generation += slot.t_token_generation; + } + + void reset_bucket() { + n_prompt_tokens_processed = 0; + t_prompt_processing = 0; + n_tokens_predicted = 0; + t_tokens_generation = 0; + } +}; + struct llama_server_context { llama_model *model = nullptr; @@ -344,6 +378,8 @@ struct llama_server_context llama_server_queue queue_tasks; llama_server_response queue_results; + llama_metrics metrics; + ~llama_server_context() { if (ctx) @@ -1404,7 +1440,7 @@ struct llama_server_context case TASK_TYPE_NEXT_RESPONSE: { // do nothing } break; - case TASK_TYPE_SLOTS_DATA: { + case TASK_TYPE_METRICS: { json slots_data = json::array(); int n_idle_slots = 0; int n_processing_slots = 0; @@ -1438,10 +1474,24 @@ struct llama_server_context res.stop = true; res.error = false; res.result_json = { - { "idle", n_idle_slots }, - { "processing", n_processing_slots }, - { "slots", slots_data } + { "idle", n_idle_slots }, + { "processing", n_processing_slots }, + { "deferred", queue_tasks.queue_tasks_deferred.size() }, + + { "n_prompt_tokens_processed_total", metrics.n_prompt_tokens_processed_total}, + { "n_tokens_predicted_total", metrics.n_tokens_predicted_total}, + + { "n_prompt_tokens_processed", metrics.n_prompt_tokens_processed}, + { "t_prompt_processing", metrics.t_prompt_processing}, + { "n_tokens_predicted", metrics.n_tokens_predicted}, + { "t_tokens_generation", metrics.t_tokens_generation}, + + { "kv_cache_tokens_count", llama_get_kv_cache_token_count(ctx)}, + { "kv_cache_used_cells", llama_get_kv_cache_used_cells(ctx)}, + + { "slots", slots_data }, }; + metrics.reset_bucket(); queue_results.send(res); } break; } @@ -1849,6 +1899,7 @@ struct llama_server_context { slot.t_start_genereration = ggml_time_us(); slot.t_prompt_processing = (slot.t_start_genereration - slot.t_start_process_prompt) / 1e3; + metrics.on_prompt_eval(slot); } llama_token_data_array cur_p = { slot.ctx_sampling->cur.data(), slot.ctx_sampling->cur.size(), false }; @@ -1871,6 +1922,7 @@ struct llama_server_context slot.release(); slot.print_timings(); send_final_response(slot); + metrics.on_prediction(slot); } slot.i_batch = -1; @@ -1955,6 +2007,7 @@ static void server_print_usage(const char *argv0, const gpt_params ¶ms, printf(" --mmproj MMPROJ_FILE path to a multimodal projector file for LLaVA.\n"); printf(" --log-disable disables logging to a file.\n"); printf(" --slots-endpoint-disable disables slots monitoring endpoint.\n"); + printf(" --metrics enable prometheus compatible metrics endpoint (default: %s).\n", sparams.metrics_endpoint ? "enabled" : "disabled"); printf("\n"); printf(" -n, --n-predict maximum tokens to predict (default: %d)\n", params.n_predict); printf(" --override-kv KEY=TYPE:VALUE\n"); @@ -2414,6 +2467,10 @@ static void server_params_parse(int argc, char **argv, server_params &sparams, { sparams.slots_endpoint = false; } + else if (arg == "--metrics") + { + sparams.metrics_endpoint = true; + } else if (arg == "--chat-template") { if (++i >= argc) @@ -2621,7 +2678,7 @@ int main(int argc, char **argv) // request slots data using task queue task_server task; task.id = llama.queue_tasks.get_new_id(); - task.type = TASK_TYPE_SLOTS_DATA; + task.type = TASK_TYPE_METRICS; task.target_id = -1; llama.queue_results.add_waiting_task_id(task.id); @@ -2668,7 +2725,7 @@ int main(int argc, char **argv) // request slots data using task queue task_server task; task.id = llama.queue_tasks.get_new_id(); - task.type = TASK_TYPE_SLOTS_DATA; + task.type = TASK_TYPE_METRICS; task.target_id = -1; llama.queue_results.add_waiting_task_id(task.id); @@ -2683,6 +2740,87 @@ int main(int argc, char **argv) }); } + if (sparams.metrics_endpoint) { + svr.Get("/metrics", [&](const httplib::Request&, httplib::Response& res) { + // request slots data using task queue + task_server task; + task.id = llama.queue_tasks.get_new_id(); + task.type = TASK_TYPE_METRICS; + task.target_id = -1; + + llama.queue_results.add_waiting_task_id(task.id); + llama.queue_tasks.post(task); + + // get the result + task_result result = llama.queue_results.recv(task.id); + llama.queue_results.remove_waiting_task_id(task.id); + + json data = result.result_json; + + uint64_t n_prompt_tokens_processed = data["n_prompt_tokens_processed"]; + uint64_t t_prompt_processing = data["t_prompt_processing"]; + + uint64_t n_tokens_predicted = data["n_tokens_predicted"]; + uint64_t t_tokens_generation = data["t_tokens_generation"]; + + int32_t kv_cache_used_cells = data["kv_cache_used_cells"]; + + // metrics definition: https://prometheus.io/docs/practices/naming/#metric-names + json all_metrics_def = json { + {"counter", {{ + {"name", "prompt_tokens_total"}, + {"help", "Number of prompt tokens processed."}, + {"value", data["n_prompt_tokens_processed_total"]} + }, { + {"name", "tokens_predicted_total"}, + {"help", "Number of generation tokens processed."}, + {"value", data["n_tokens_predicted_total"]} + }}}, + {"gauge", {{ + {"name", "prompt_tokens_seconds"}, + {"help", "Average prompt throughput in tokens/s."}, + {"value", n_prompt_tokens_processed ? 1e3 / t_prompt_processing * n_prompt_tokens_processed : 0} + },{ + {"name", "predicted_tokens_seconds"}, + {"help", "Average generation throughput in tokens/s."}, + {"value", n_tokens_predicted ? 1e3 / t_tokens_generation * n_tokens_predicted : 0} + },{ + {"name", "kv_cache_usage_ratio"}, + {"help", "KV-cache usage. 1 means 100 percent usage."}, + {"value", 1. * kv_cache_used_cells / params.n_ctx} + },{ + {"name", "kv_cache_tokens"}, + {"help", "KV-cache tokens."}, + {"value", data["kv_cache_tokens_count"]} + },{ + {"name", "requests_processing"}, + {"help", "Number of request processing."}, + {"value", data["processing"]} + },{ + {"name", "requests_deferred"}, + {"help", "Number of request deferred."}, + {"value", data["deferred"]} + }}} + }; + + std::stringstream prometheus; + for (const auto& el : all_metrics_def.items()) { + const auto& type = el.key(); + const auto& metrics_def = el.value(); + for (const auto& metric_def : metrics_def) { + std::string name = metric_def["name"]; + std::string help = metric_def["help"]; + prometheus << "# HELP llamacpp:" << name << " " << help << "\n" + << "# TYPE llamacpp:" << name << " " << type << "\n" + << "llamacpp:" << name << " " << metric_def["value"] << "\n"; + } + } + + res.set_content(prometheus.str(), "text/plain; version=0.0.4"); + res.status = 200; // HTTP OK + }); + } + svr.set_logger(log_server_request); svr.set_exception_handler([](const httplib::Request &, httplib::Response &res, std::exception_ptr ep) diff --git a/examples/server/tests/features/environment.py b/examples/server/tests/features/environment.py index 13cc84101..09e826747 100644 --- a/examples/server/tests/features/environment.py +++ b/examples/server/tests/features/environment.py @@ -16,6 +16,8 @@ def before_scenario(context, scenario): def after_scenario(context, scenario): + if context.server_process is None: + return if scenario.status == "failed": if 'GITHUB_ACTIONS' in os.environ: print(f"\x1b[33;101mSCENARIO FAILED: {scenario.name} server logs:\x1b[0m\n\n") diff --git a/examples/server/tests/features/server.feature b/examples/server/tests/features/server.feature index 5f81d256a..0139f89d8 100644 --- a/examples/server/tests/features/server.feature +++ b/examples/server/tests/features/server.feature @@ -13,6 +13,7 @@ Feature: llama.cpp server And 1 slots And embeddings extraction And 32 server max tokens to predict + And prometheus compatible metrics exposed Then the server is starting Then the server is healthy @@ -25,6 +26,7 @@ Feature: llama.cpp server And max tokens to predict And a completion request with no api error Then tokens are predicted matching + And prometheus metrics are exposed Examples: Prompts | prompt | n_predict | re_content | n_predicted | diff --git a/examples/server/tests/features/steps/steps.py b/examples/server/tests/features/steps/steps.py index 9c825fdbc..051fd440c 100644 --- a/examples/server/tests/features/steps/steps.py +++ b/examples/server/tests/features/steps/steps.py @@ -13,6 +13,7 @@ import aiohttp import openai from behave import step from behave.api.async_step import async_run_until_complete +from prometheus_client import parser @step(u"a server listening on {server_fqdn}:{server_port}") @@ -34,6 +35,8 @@ def step_server_config(context, server_fqdn, server_port): context.server_api_key = None context.server_continuous_batching = False context.server_embeddings = False + context.server_metrics = False + context.server_process = None context.server_seed = None context.user_api_key = None @@ -82,6 +85,11 @@ def step_server_embeddings(context): context.server_embeddings = True +@step(u'prometheus compatible metrics exposed') +def step_server_metrics(context): + context.server_metrics = True + + @step(u"the server is starting") def step_start_server(context): start_server_background(context) @@ -424,6 +432,23 @@ def step_check_options_header_value(context, cors_header, cors_header_value): assert context.options_response.headers[cors_header] == cors_header_value +@step(u'prometheus metrics are exposed') +@async_run_until_complete +async def step_prometheus_metrics_exported(context): + async with aiohttp.ClientSession() as session: + async with await session.get(f'{context.base_url}/metrics') as metrics_response: + assert metrics_response.status == 200 + assert metrics_response.headers['Content-Type'] == "text/plain; version=0.0.4" + metrics_raw = await metrics_response.text() + metric_exported = False + for metric in parser.text_string_to_metric_families(metrics_raw): + match metric.name: + case "llamacpp:kv_cache_usage_ratio": + assert len(metric.samples) > 0 + metric_exported = True + assert metric_exported, "No metrics exported" + + async def concurrent_requests(context, f_completion, *args, **kwargs): n_prompts = len(context.prompts) if context.debug: @@ -753,6 +778,8 @@ def start_server_background(context): server_args.append('--cont-batching') if context.server_embeddings: server_args.append('--embedding') + if context.server_metrics: + server_args.append('--metrics') if context.model_alias is not None: server_args.extend(['--alias', context.model_alias]) if context.n_ctx is not None: diff --git a/examples/server/tests/requirements.txt b/examples/server/tests/requirements.txt index 3e51b12dc..334fa4a70 100644 --- a/examples/server/tests/requirements.txt +++ b/examples/server/tests/requirements.txt @@ -1,3 +1,4 @@ aiohttp~=3.9.3 behave~=1.2.6 openai~=0.25.0 +prometheus-client~=0.20.0 diff --git a/examples/server/utils.hpp b/examples/server/utils.hpp index 88545eb69..71cc5b0b8 100644 --- a/examples/server/utils.hpp +++ b/examples/server/utils.hpp @@ -50,7 +50,7 @@ enum task_type { TASK_TYPE_COMPLETION, TASK_TYPE_CANCEL, TASK_TYPE_NEXT_RESPONSE, - TASK_TYPE_SLOTS_DATA + TASK_TYPE_METRICS }; struct task_server { @@ -441,7 +441,7 @@ struct llama_server_response { { LOG_VERBOSE("queue_results.push_back", {}); queue_results.push_back(result); - condition_results.notify_one(); + condition_results.notify_all(); return; } } From 930b1780269a69948d106e2d1b838ab7661f679a Mon Sep 17 00:00:00 2001 From: Pierrick Hymbert Date: Sun, 25 Feb 2024 13:50:32 +0100 Subject: [PATCH 455/812] server: logs - unified format and --log-format option (#5700) * server: logs - always use JSON logger, add add thread_id in message, log task_id and slot_id * server : skip GH copilot requests from logging * server : change message format of server_log() * server : no need to repeat log in comment * server : log style consistency * server : fix compile warning * server : fix tests regex patterns on M2 Ultra * server: logs: PR feedback on log level * server: logs: allow to choose log format in json or plain text * server: tests: output server logs in text * server: logs switch init logs to server logs macro * server: logs ensure value json value does not raised error * server: logs reduce level VERBOSE to VERB to max 4 chars * server: logs lower case as other log messages * server: logs avoid static in general Co-authored-by: Georgi Gerganov * server: logs PR feedback: change text log format to: LEVEL [function_name] message | additional=data --------- Co-authored-by: Georgi Gerganov --- examples/server/README.md | 4 +- examples/server/server.cpp | 218 ++++++++++++++---- examples/server/tests/README.md | 1 + examples/server/tests/features/server.feature | 6 +- examples/server/tests/features/steps/steps.py | 2 + examples/server/utils.hpp | 80 ++++--- 6 files changed, 231 insertions(+), 80 deletions(-) diff --git a/examples/server/README.md b/examples/server/README.md index 2129f7fb2..cb3fd6054 100644 --- a/examples/server/README.md +++ b/examples/server/README.md @@ -39,10 +39,12 @@ see https://github.com/ggerganov/llama.cpp/issues/1437 - `--mmproj MMPROJ_FILE`: Path to a multimodal projector file for LLaVA. - `--grp-attn-n`: Set the group attention factor to extend context size through self-extend(default: 1=disabled), used together with group attention width `--grp-attn-w` - `--grp-attn-w`: Set the group attention width to extend context size through self-extend(default: 512), used together with group attention factor `--grp-attn-n` -- `-n, --n-predict`: Set the maximum tokens to predict (default: -1) +- `-n N, --n-predict N`: Set the maximum tokens to predict (default: -1) - `--slots-endpoint-disable`: To disable slots state monitoring endpoint. Slots state may contain user data, prompts included. - `--metrics`: enable prometheus `/metrics` compatible endpoint (default: disabled) - `--chat-template JINJA_TEMPLATE`: Set custom jinja chat template. This parameter accepts a string, not a file name (default: template taken from model's metadata). We only support [some pre-defined templates](https://github.com/ggerganov/llama.cpp/wiki/Templates-supported-by-llama_chat_apply_template) +- `--log-disable`: Output logs to stdout only, default: enabled. +- `--log-format FORMAT`: Define the log output to FORMAT: json or text (default: json) ## Build diff --git a/examples/server/server.cpp b/examples/server/server.cpp index 811495915..d970202d2 100644 --- a/examples/server/server.cpp +++ b/examples/server/server.cpp @@ -47,6 +47,7 @@ struct server_params }; bool server_verbose = false; +bool server_log_json = true; static size_t common_part(const std::vector &a, const std::vector &b) { @@ -302,12 +303,43 @@ struct llama_client_slot } void print_timings() const { - LOG_TEE("\n"); - LOG_TEE("%s: prompt eval time = %10.2f ms / %5d tokens (%8.2f ms per token, %8.2f tokens per second)\n", - __func__, t_prompt_processing, num_prompt_tokens_processed, t_prompt_processing / num_prompt_tokens_processed, 1e3 / t_prompt_processing * num_prompt_tokens_processed); - LOG_TEE("%s: eval time = %10.2f ms / %5d runs (%8.2f ms per token, %8.2f tokens per second)\n", - __func__, t_token_generation, n_decoded,t_token_generation / n_decoded, 1e3 / t_token_generation * n_decoded); - LOG_TEE("%s: total time = %10.2f ms\n", __func__, t_prompt_processing + t_token_generation); + char buffer[512]; + double t_token = t_prompt_processing / num_prompt_tokens_processed; + double n_tokens_second = 1e3 / t_prompt_processing * num_prompt_tokens_processed; + sprintf(buffer, "prompt eval time = %10.2f ms / %5d tokens (%8.2f ms per token, %8.2f tokens per second)", + t_prompt_processing, num_prompt_tokens_processed, + t_token, n_tokens_second); + LOG_INFO(buffer, { + {"slot_id", id}, + {"task_id", task_id}, + {"t_prompt_processing", t_prompt_processing}, + {"num_prompt_tokens_processed", num_prompt_tokens_processed}, + {"t_token", t_token}, + {"n_tokens_second", n_tokens_second}, + }); + + t_token = t_token_generation / n_decoded; + n_tokens_second = 1e3 / t_token_generation * n_decoded; + sprintf(buffer, "generation eval time = %10.2f ms / %5d runs (%8.2f ms per token, %8.2f tokens per second)", + t_token_generation, n_decoded, + t_token, n_tokens_second); + LOG_INFO(buffer, { + {"slot_id", id}, + {"task_id", task_id}, + {"t_token_generation", t_token_generation}, + {"n_decoded", n_decoded}, + {"t_token", t_token}, + {"n_tokens_second", n_tokens_second}, + }); + + sprintf(buffer, " total time = %10.2f ms", t_prompt_processing + t_token_generation); + LOG_INFO(buffer, { + {"slot_id", id}, + {"task_id", task_id}, + {"t_prompt_processing", t_prompt_processing}, + {"t_token_generation", t_token_generation}, + {"t_total", t_prompt_processing + t_token_generation}, + }); } }; @@ -399,7 +431,7 @@ struct llama_server_context params = params_; if (!params.mmproj.empty()) { multimodal = true; - LOG_TEE("Multi Modal Mode Enabled"); + LOG_INFO("Multi Modal Mode Enabled", {}); clp_ctx = clip_model_load(params.mmproj.c_str(), /*verbosity=*/ 1); if(clp_ctx == nullptr) { LOG_ERROR("unable to load clip model", {{"model", params.mmproj}}); @@ -452,7 +484,7 @@ struct llama_server_context const int32_t n_ctx_slot = n_ctx / params.n_parallel; - LOG_TEE("Available slots:\n"); + LOG_INFO("initializing slots", {{"n_slots", params.n_parallel}}); for (int i = 0; i < params.n_parallel; i++) { llama_client_slot slot; @@ -461,7 +493,10 @@ struct llama_server_context slot.n_ctx = n_ctx_slot; slot.n_predict = params.n_predict; - LOG_TEE(" -> Slot %i - max context: %i\n", slot.id, n_ctx_slot); + LOG_INFO("new slot", { + {"slot_id", slot.id}, + {"n_ctx_slot", slot.n_ctx} + }); const int ga_n = params.grp_attn_n; const int ga_w = params.grp_attn_w; @@ -471,7 +506,12 @@ struct llama_server_context GGML_ASSERT(ga_w % ga_n == 0 && "ga_w must be a multiple of ga_n"); // NOLINT //GGML_ASSERT(n_ctx_train % ga_w == 0 && "n_ctx_train must be a multiple of ga_w"); // NOLINT //GGML_ASSERT(n_ctx >= n_ctx_train * ga_n && "n_ctx must be at least n_ctx_train * ga_n"); // NOLINT - LOG_TEE(" -> Slot %i - self-extend: ga_n = %d, ga_w = %d\n", slot.id, ga_n, ga_w); + + LOG_INFO("slot self-extend", { + {"slot_id", slot.id}, + {"ga_n", ga_n}, + {"ga_w", ga_w} + }); } slot.ga_i = 0; @@ -765,10 +805,16 @@ struct llama_server_context img_sl.img_data = clip_image_u8_init(); if (!clip_image_load_from_bytes(image_buffer.data(), image_buffer.size(), img_sl.img_data)) { - LOG_TEE("slot %i - failed to load image [id: %i]\n", slot->id, img_sl.id); + LOG_ERROR("failed to load image", { + {"slot_id", slot->id}, + {"img_sl_id", img_sl.id} + }); return false; } - LOG_TEE("slot %i - loaded image\n", slot->id); + LOG_VERBOSE("image loaded", { + {"slot_id", slot->id}, + {"img_sl_id", img_sl.id} + }); img_sl.request_encode_image = true; slot->images.push_back(img_sl); } @@ -828,7 +874,10 @@ struct llama_server_context all_slots_are_idle = false; - LOG_TEE("slot %i is processing [task id: %i]\n", slot->id, slot->task_id); + LOG_INFO("slot is processing task", { + {"slot_id", slot->id}, + {"task_id", slot->task_id}, + }); return true; } @@ -1391,7 +1440,7 @@ struct llama_server_context if (slot == nullptr) { // if no slot is available, we defer this task for processing later - LOG_VERBOSE("no slot is available", {}); + LOG_VERBOSE("no slot is available", {{"task_id", task.id}}); queue_tasks.defer(task); break; } @@ -1467,7 +1516,17 @@ struct llama_server_context } slots_data.push_back(slot_data); } - LOG_TEE("task %i - slots data: idle=%i processing=%i\n", task.id, n_idle_slots, n_processing_slots); + LOG_INFO("slot data", { + {"task_id", task.id}, + {"n_idle_slots", n_idle_slots}, + {"n_processing_slots", n_processing_slots} + }); + LOG_VERBOSE("slot data", { + {"task_id", task.id}, + {"n_idle_slots", n_idle_slots}, + {"n_processing_slots", n_processing_slots}, + {"slots", slots_data} + }); task_result res; res.id = task.id; res.multitask_id = task.multitask_id; @@ -1519,7 +1578,7 @@ struct llama_server_context bool update_slots() { if (system_need_update) { - LOG_TEE("updating system prompt\n"); + LOG_INFO("updating system prompt", {}); update_system_prompt(); } @@ -1529,12 +1588,13 @@ struct llama_server_context { if (system_prompt.empty() && clean_kv_cache) { - LOG_TEE("all slots are idle and system prompt is empty, clear the KV cache\n"); + LOG_INFO("all slots are idle and system prompt is empty, clear the KV cache", {}); kv_cache_clear(); } return true; } + LOG_VERBOSE("posting NEXT_RESPONSE", {}); task_server task; task.type = TASK_TYPE_NEXT_RESPONSE; task.target_id = -1; @@ -1548,10 +1608,20 @@ struct llama_server_context { // Shift context const int n_keep = slot.params.n_keep + add_bos_token; - const int n_left = system_tokens.size() + slot.n_past - n_keep; + const int n_left = (int) system_tokens.size() + slot.n_past - n_keep; const int n_discard = n_left / 2; - LOG_TEE("slot %d: context shift - n_keep = %d, n_left = %d, n_discard = %d\n", slot.id, n_keep, n_left, n_discard); + LOG_INFO("slot context shift", { + {"slot_id", slot.id}, + {"task_id", slot.task_id}, + {"n_keep", n_keep}, + {"n_left", n_left}, + {"n_discard", n_discard}, + {"n_ctx", n_ctx}, + {"n_past", slot.n_past}, + {"n_system_tokens", system_tokens.size()}, + {"n_cache_tokens", slot.cache_tokens.size()} + }); llama_kv_cache_seq_rm (ctx, slot.id, n_keep , n_keep + n_discard); llama_kv_cache_seq_shift(ctx, slot.id, n_keep + n_discard, system_tokens.size() + slot.n_past, -n_discard); @@ -1565,17 +1635,12 @@ struct llama_server_context slot.n_past -= n_discard; slot.truncated = true; - - LOG_VERBOSE("context shift", { - { "n_ctx", n_ctx }, - { "n_keep", n_keep }, - { "n_left", n_left }, - }); } } } // decode any currently ongoing sequences + LOG_VERBOSE("decoding ongoing sequences", {}); for (auto & slot : slots) { // release the slot @@ -1585,7 +1650,15 @@ struct llama_server_context slot.command = NONE; slot.t_last_used = ggml_time_us(); - LOG_TEE("slot %d released (%d tokens in cache)\n", slot.id, (int) slot.cache_tokens.size()); + LOG_INFO("slot released", { + {"slot_id", slot.id}, + {"task_id", slot.task_id}, + {"n_ctx", n_ctx}, + {"n_past", slot.n_past}, + {"n_system_tokens", system_tokens.size()}, + {"n_cache_tokens", slot.cache_tokens.size()}, + {"truncated", slot.truncated} + }); queue_tasks.notify_slot_changed(); continue; @@ -1733,7 +1806,12 @@ struct llama_server_context slot.ga_i = ga_i; } - LOG_TEE("slot %d : in cache: %i tokens | to process: %i tokens\n", slot.id, slot.n_past, slot.num_prompt_tokens_processed); + LOG_INFO("slot progression", { + { "slot_id", slot.id }, + { "task_id", slot.task_id }, + { "n_past", slot.n_past }, + { "num_prompt_tokens_processed", slot.num_prompt_tokens_processed } + }); } slot.cache_tokens = prompt_tokens; @@ -1741,7 +1819,10 @@ struct llama_server_context if (slot.n_past == slot.num_prompt_tokens && slot.n_past > 0) { // we have to evaluate at least 1 token to generate logits. - LOG_TEE("slot %d : we have to evaluate at least 1 token to generate logits\n", slot.id); + LOG_INFO("we have to evaluate at least 1 token to generate logits", { + { "slot_id", slot.id }, + { "task_id", slot.task_id } + }); slot.n_past--; if (slot.ga_i > 0) { @@ -1749,9 +1830,13 @@ struct llama_server_context } } - LOG_TEE("slot %d : kv cache rm - [%d, end)\n", slot.id, (int) system_tokens.size() + slot.n_past); - - llama_kv_cache_seq_rm(ctx, slot.id, system_tokens.size() + slot.n_past, -1); + int p0 = (int) system_tokens.size() + slot.n_past; + LOG_INFO("kv cache rm [p0, end)", { + { "slot_id", slot.id }, + { "task_id", slot.task_id }, + { "p0", p0 } + }); + llama_kv_cache_seq_rm(ctx, slot.id, p0, -1); LOG_VERBOSE("prompt ingested", { {"n_past", slot.n_past}, @@ -1786,7 +1871,13 @@ struct llama_server_context if (has_images && !ingest_images(slot, n_batch)) { - LOG_TEE("failed processing images\n"); + LOG_ERROR("failed processing images", { + "slot_id", slot.id, + "task_id", slot.task_id, + }); + // FIXME @phymbert: to be properly tested + // early returning without changing the slot state will block the slot for ever + // no one at the moment is checking the return value return false; } @@ -1928,6 +2019,8 @@ struct llama_server_context slot.i_batch = -1; } } + + LOG_VERBOSE("slots updated", {}); return true; } @@ -2005,6 +2098,7 @@ static void server_print_usage(const char *argv0, const gpt_params ¶ms, printf(" -ctv TYPE, --cache-type-v TYPE\n"); printf(" KV cache data type for V (default: f16)\n"); printf(" --mmproj MMPROJ_FILE path to a multimodal projector file for LLaVA.\n"); + printf(" --log-format log output format: json or text (default: json)\n"); printf(" --log-disable disables logging to a file.\n"); printf(" --slots-endpoint-disable disables slots monitoring endpoint.\n"); printf(" --metrics enable prometheus compatible metrics endpoint (default: %s).\n", sparams.metrics_endpoint ? "enabled" : "disabled"); @@ -2458,6 +2552,27 @@ static void server_params_parse(int argc, char **argv, server_params &sparams, } params.mmproj = argv[i]; } + else if (arg == "--log-format") + { + if (++i >= argc) + { + invalid_param = true; + break; + } + if (std::strcmp(argv[i], "json") == 0) + { + server_log_json = true; + } + else if (std::strcmp(argv[i], "text") == 0) + { + server_log_json = false; + } + else + { + invalid_param = true; + break; + } + } else if (arg == "--log-disable") { log_set_target(stdout); @@ -2571,32 +2686,40 @@ static json format_partial_response( static json format_tokenizer_response(const std::vector &tokens) { - return json{ - {"tokens", tokens}}; + return json { + {"tokens", tokens} + }; } static json format_detokenized_response(std::string content) { - return json{ - {"content", content}}; + return json { + {"content", content} + }; } static void log_server_request(const httplib::Request &req, const httplib::Response &res) { + // skip GH copilot requests when using default port + if (req.path == "/v1/health" || req.path == "/v1/completions") + { + return; + } + LOG_INFO("request", { - {"remote_addr", req.remote_addr}, - {"remote_port", req.remote_port}, - {"status", res.status}, - {"method", req.method}, - {"path", req.path}, - {"params", req.params}, - }); + {"remote_addr", req.remote_addr}, + {"remote_port", req.remote_port}, + {"status", res.status}, + {"method", req.method}, + {"path", req.path}, + {"params", req.params}, + }); LOG_VERBOSE("request", { - {"request", req.body}, - {"response", res.body}, - }); + {"request", req.body}, + {"response", res.body}, + }); } struct token_translator @@ -2873,9 +2996,6 @@ int main(int argc, char **argv) // Set the base directory for serving static files svr.set_base_dir(sparams.public_path); - // to make it ctrl+clickable: - LOG_TEE("\nllama server listening at http://%s:%d\n\n", sparams.hostname.c_str(), sparams.port); - std::unordered_map log_data; log_data["hostname"] = sparams.hostname; log_data["port"] = std::to_string(sparams.port); diff --git a/examples/server/tests/README.md b/examples/server/tests/README.md index e44c5c286..0b9fdc4e7 100644 --- a/examples/server/tests/README.md +++ b/examples/server/tests/README.md @@ -32,6 +32,7 @@ It's possible to override some scenario steps values with environment variables: - `PORT` -> `context.server_port` to set the listening port of the server during scenario, default: `8080` - `LLAMA_SERVER_BIN_PATH` -> to change the server binary path, default: `../../../build/bin/server` - `DEBUG` -> "ON" to enable steps and server verbose mode `--verbose` + - `SERVER_LOG_FORMAT_JSON` -> if set switch server logs to json format ### Run @bug, @wip or @wrong_usage annotated scenario diff --git a/examples/server/tests/features/server.feature b/examples/server/tests/features/server.feature index 0139f89d8..b571582a7 100644 --- a/examples/server/tests/features/server.feature +++ b/examples/server/tests/features/server.feature @@ -29,9 +29,9 @@ Feature: llama.cpp server And prometheus metrics are exposed Examples: Prompts - | prompt | n_predict | re_content | n_predicted | - | I believe the meaning of life is | 8 | read | 8 | - | Write a joke about AI | 64 | (parkfriendsscared)+ | 32 | + | prompt | n_predict | re_content | n_predicted | + | I believe the meaning of life is | 8 | (readgoing)+ | 8 | + | Write a joke about AI | 64 | (parkfriendsscaredalways)+ | 32 | Scenario Outline: OAI Compatibility Given a model diff --git a/examples/server/tests/features/steps/steps.py b/examples/server/tests/features/steps/steps.py index 051fd440c..8e4babf20 100644 --- a/examples/server/tests/features/steps/steps.py +++ b/examples/server/tests/features/steps/steps.py @@ -792,6 +792,8 @@ def start_server_background(context): server_args.extend(['--api-key', context.server_api_key]) if context.debug: server_args.append('--verbose') + if 'SERVER_LOG_FORMAT_JSON' not in os.environ: + server_args.extend(['--log-format', "text"]) print(f"starting server with: {context.server_path}", *server_args) context.server_process = subprocess.Popen( [str(arg) for arg in [context.server_path, *server_args]], diff --git a/examples/server/utils.hpp b/examples/server/utils.hpp index 71cc5b0b8..d7abd7cbb 100644 --- a/examples/server/utils.hpp +++ b/examples/server/utils.hpp @@ -14,6 +14,7 @@ using json = nlohmann::json; extern bool server_verbose; +extern bool server_log_json; #ifndef SERVER_VERBOSE #define SERVER_VERBOSE 1 @@ -27,14 +28,14 @@ extern bool server_verbose; { \ if (server_verbose) \ { \ - server_log("VERBOSE", __func__, __LINE__, MSG, __VA_ARGS__); \ + server_log("VERB", __func__, __LINE__, MSG, __VA_ARGS__); \ } \ } while (0) #endif -#define LOG_ERROR( MSG, ...) server_log("ERROR", __func__, __LINE__, MSG, __VA_ARGS__) -#define LOG_WARNING(MSG, ...) server_log("WARNING", __func__, __LINE__, MSG, __VA_ARGS__) -#define LOG_INFO( MSG, ...) server_log("INFO", __func__, __LINE__, MSG, __VA_ARGS__) +#define LOG_ERROR( MSG, ...) server_log("ERR", __func__, __LINE__, MSG, __VA_ARGS__) +#define LOG_WARNING(MSG, ...) server_log("WARN", __func__, __LINE__, MSG, __VA_ARGS__) +#define LOG_INFO( MSG, ...) server_log("INFO", __func__, __LINE__, MSG, __VA_ARGS__) // // parallel @@ -133,26 +134,48 @@ struct completion_token_output std::string text_to_send; }; -static inline void server_log(const char *level, const char *function, int line, - const char *message, const nlohmann::ordered_json &extra) +static inline void server_log(const char *level, const char *function, int line, const char *message, const nlohmann::ordered_json &extra) { - nlohmann::ordered_json log - { + std::stringstream ss_tid; + ss_tid << std::this_thread::get_id(); + json log = nlohmann::ordered_json{ + {"tid", ss_tid.str()}, {"timestamp", time(nullptr)}, - {"level", level}, - {"function", function}, - {"line", line}, - {"message", message}, }; - if (!extra.empty()) - { - log.merge_patch(extra); - } + if (server_log_json) { + log.merge_patch( + { + {"level", level}, + {"function", function}, + {"line", line}, + {"msg", message}, + }); + if (!extra.empty()) { + log.merge_patch(extra); + } - const std::string str = log.dump(-1, ' ', false, json::error_handler_t::replace); - printf("%.*s\n", (int)str.size(), str.data()); - fflush(stdout); + std::cout << log.dump(-1, ' ', false, json::error_handler_t::replace) << "\n" << std::flush; + } else { + char buf[1024]; + snprintf(buf, 1024, "%4s [%24s] %s", level, function, message); + + if (!extra.empty()) { + log.merge_patch(extra); + } + std::stringstream ss; + ss << buf << " |"; + for (const auto& el : log.items()) + { + const std::string value = el.value().dump(-1, ' ', false, json::error_handler_t::replace); + snprintf(buf, 1024, " %s=%s", el.key().c_str(), value.c_str()); + ss << buf; + } + + const std::string str = ss.str(); + printf("%.*s\n", (int)str.size(), str.data()); + fflush(stdout); + } } // @@ -234,6 +257,7 @@ struct llama_server_queue { std::unique_lock lock(mutex_tasks); if (task.id == -1) { task.id = id++; + LOG_VERBOSE("new task id", {{"new_id", task.id}}); } queue_tasks.push_back(std::move(task)); condition_tasks.notify_one(); @@ -249,7 +273,9 @@ struct llama_server_queue { // Get the next id for creating anew task int get_new_id() { std::unique_lock lock(mutex_tasks); - return id++; + int new_id = id++; + LOG_VERBOSE("new task id", {{"new_id", new_id}}); + return new_id; } // Register function to process a new task @@ -290,8 +316,7 @@ struct llama_server_queue { void start_loop() { running = true; while (true) { - // new task arrived - LOG_VERBOSE("have new task", {}); + LOG_VERBOSE("new task may arrive", {}); { while (true) { @@ -303,7 +328,7 @@ struct llama_server_queue { task_server task = queue_tasks.front(); queue_tasks.erase(queue_tasks.begin()); lock.unlock(); - LOG_VERBOSE("callback_new_task", {}); + LOG_VERBOSE("callback_new_task", {{"task_id", task.id}}); callback_new_task(task); } LOG_VERBOSE("callback_all_task_finished", {}); @@ -384,11 +409,13 @@ struct llama_server_response { std::condition_variable condition_results; void add_waiting_task_id(int task_id) { + LOG_VERBOSE("waiting for task id", {{"task_id", task_id}}); std::unique_lock lock(mutex_results); waiting_task_ids.insert(task_id); } void remove_waiting_task_id(int task_id) { + LOG_VERBOSE("remove waiting for task id", {{"task_id", task_id}}); std::unique_lock lock(mutex_results); waiting_task_ids.erase(task_id); } @@ -401,7 +428,6 @@ struct llama_server_response { condition_results.wait(lock, [&]{ return !queue_results.empty(); }); - LOG_VERBOSE("condition_results unblock", {}); for (int i = 0; i < (int) queue_results.size(); i++) { @@ -426,20 +452,20 @@ struct llama_server_response { // Send a new result to a waiting task_id void send(task_result result) { std::unique_lock lock(mutex_results); - LOG_VERBOSE("send new result", {}); + LOG_VERBOSE("send new result", {{"task_id", result.id}}); for (auto& task_id : waiting_task_ids) { // LOG_TEE("waiting task id %i \n", task_id); // for now, tasks that have associated parent multitasks just get erased once multitask picks up the result if (result.multitask_id == task_id) { - LOG_VERBOSE("callback_update_multitask", {}); + LOG_VERBOSE("callback_update_multitask", {{"task_id", task_id}}); callback_update_multitask(task_id, result.id, result); continue; } if (result.id == task_id) { - LOG_VERBOSE("queue_results.push_back", {}); + LOG_VERBOSE("queue_results.push_back", {{"task_id", task_id}}); queue_results.push_back(result); condition_results.notify_all(); return; From 7d548a1827f6fc6aece6db74c9d112da42c40d68 Mon Sep 17 00:00:00 2001 From: Ashok Gelal <401055+ashokgelal@users.noreply.github.com> Date: Sun, 25 Feb 2024 10:57:34 -0500 Subject: [PATCH 456/812] readme : add Msty to UI list (#5618) --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 3bc512af0..d61f9171b 100644 --- a/README.md +++ b/README.md @@ -155,6 +155,7 @@ Unless otherwise noted these projects are open-source with permissive licensing: - [semperai/amica](https://github.com/semperai/amica) - [withcatai/catai](https://github.com/withcatai/catai) - [Mobile-Artificial-Intelligence/maid](https://github.com/Mobile-Artificial-Intelligence/maid) (MIT) +- [Msty](https://msty.app) (proprietary) --- From f1a98c52546d009f742bdec2154c2a314ea950a6 Mon Sep 17 00:00:00 2001 From: kwin1412 <42286931+kwin1412@users.noreply.github.com> Date: Mon, 26 Feb 2024 00:46:49 +0800 Subject: [PATCH 457/812] make : fix nvcc version is empty (#5713) fix nvcc version is empty --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index f03faf6ed..068f6ed02 100644 --- a/Makefile +++ b/Makefile @@ -597,7 +597,7 @@ $(info I CC: $(shell $(CC) --version | head -n 1)) $(info I CXX: $(shell $(CXX) --version | head -n 1)) ifdef LLAMA_CUBLAS $(info I NVCC: $(shell $(NVCC) --version | tail -n 1)) -CUDA_VERSION := $(shell nvcc --version | grep -oP 'release (\K[0-9]+\.[0-9])') +CUDA_VERSION := $(shell $(NVCC) --version | grep -oP 'release (\K[0-9]+\.[0-9])') ifeq ($(shell awk -v "v=$(CUDA_VERSION)" 'BEGIN { print (v < 11.7) }'),1) ifndef CUDA_DOCKER_ARCH ifndef CUDA_POWER_ARCH From abbabc5e51d0d4656b438aec10b7fae9479ef37d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Gryta?= Date: Sun, 25 Feb 2024 19:43:00 +0100 Subject: [PATCH 458/812] ggml-quants : provide ggml_vqtbl1q_u8 for 64bit compatibility (#5711) * [ggml-quants] Provide ggml_vqtbl1q_u8 for 64bit compatibility vqtbl1q_u8 is not part of arm v7 neon library * [android-example] Remove abi filter after arm v7a fix * [github-workflows] Do not skip Android armeabi-v7a build --- .github/workflows/build.yml | 3 +- examples/llama.android/app/build.gradle.kts | 8 ++--- ggml-quants.c | 33 ++++++++++++++++++--- 3 files changed, 32 insertions(+), 12 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 03d76d455..66ad85938 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -669,8 +669,7 @@ jobs: run: | cd examples/llama.android - # Skip armeabi-v7a for now (https://github.com/llvm/llvm-project/issues/65820). - ./gradlew build --no-daemon -Pskip-armeabi-v7a + ./gradlew build --no-daemon # freeBSD-latest: # runs-on: macos-12 diff --git a/examples/llama.android/app/build.gradle.kts b/examples/llama.android/app/build.gradle.kts index aadbe22c9..d42140efe 100644 --- a/examples/llama.android/app/build.gradle.kts +++ b/examples/llama.android/app/build.gradle.kts @@ -21,12 +21,8 @@ android { useSupportLibrary = true } ndk { - // Workaround for https://github.com/llvm/llvm-project/issues/65820 - // affecting armeabi-v7a. Skip armeabi-v7a when invoked with - // -Pskip-armeabi-v7a (e.g., ./gradlew build -Pskip-armeabi-v7a). - if (project.hasProperty("skip-armeabi-v7a")) { - abiFilters += listOf("arm64-v8a", "x86_64", "x86") - } + // Add NDK properties if wanted, e.g. + // abiFilters += listOf("arm64-v8a") } externalNativeBuild { cmake { diff --git a/ggml-quants.c b/ggml-quants.c index 5c5f2ce1b..3d94d166d 100644 --- a/ggml-quants.c +++ b/ggml-quants.c @@ -462,6 +462,30 @@ inline static int8x16_t ggml_vqtbl1q_s8(int8x16_t a, uint8x16_t b) { return res; } +// NOTE: not tested +inline static int8x16_t ggml_vqtbl1q_u8(uint8x16_t a, uint8x16_t b) { + int8x16_t res; + + res[ 0] = a[b[ 0]]; + res[ 1] = a[b[ 1]]; + res[ 2] = a[b[ 2]]; + res[ 3] = a[b[ 3]]; + res[ 4] = a[b[ 4]]; + res[ 5] = a[b[ 5]]; + res[ 6] = a[b[ 6]]; + res[ 7] = a[b[ 7]]; + res[ 8] = a[b[ 8]]; + res[ 9] = a[b[ 9]]; + res[10] = a[b[10]]; + res[11] = a[b[11]]; + res[12] = a[b[12]]; + res[13] = a[b[13]]; + res[14] = a[b[14]]; + res[15] = a[b[15]]; + + return res; +} + #else #define ggml_int16x8x2_t int16x8x2_t @@ -476,6 +500,7 @@ inline static int8x16_t ggml_vqtbl1q_s8(int8x16_t a, uint8x16_t b) { #define ggml_vld1q_s8_x2 vld1q_s8_x2 #define ggml_vld1q_s8_x4 vld1q_s8_x4 #define ggml_vqtbl1q_s8 vqtbl1q_s8 +#define ggml_vqtbl1q_u8 vqtbl1q_u8 #endif @@ -9488,8 +9513,8 @@ void ggml_vec_dot_iq3_s_q8_K (int n, float * GGML_RESTRICT s, size_t bs, const v qs += 16; vs.val[0] = vreinterpretq_u8_u32(vdupq_n_u32(signs[0] | (signs[1] << 16))); - vs.val[1] = vandq_u8(vqtbl1q_u8(vs.val[0], mask1.val[1]), mask2); - vs.val[0] = vandq_u8(vqtbl1q_u8(vs.val[0], mask1.val[0]), mask2); + vs.val[1] = vandq_u8(ggml_vqtbl1q_u8(vs.val[0], mask1.val[1]), mask2); + vs.val[0] = vandq_u8(ggml_vqtbl1q_u8(vs.val[0], mask1.val[0]), mask2); vs.val[0] = vceqq_u8(vs.val[0], mask2); vs.val[1] = vceqq_u8(vs.val[1], mask2); @@ -9497,8 +9522,8 @@ void ggml_vec_dot_iq3_s_q8_K (int n, float * GGML_RESTRICT s, size_t bs, const v q3s.val[1] = vsubq_s8(vreinterpretq_s8_u8(veorq_u8(vs.val[1], vreinterpretq_u8_u32(aux32x4_1))), vreinterpretq_s8_u8(vs.val[1])); vs.val[0] = vreinterpretq_u8_u32(vdupq_n_u32(signs[2] | (signs[3] << 16))); - vs.val[1] = vandq_u8(vqtbl1q_u8(vs.val[0], mask1.val[1]), mask2); - vs.val[0] = vandq_u8(vqtbl1q_u8(vs.val[0], mask1.val[0]), mask2); + vs.val[1] = vandq_u8(ggml_vqtbl1q_u8(vs.val[0], mask1.val[1]), mask2); + vs.val[0] = vandq_u8(ggml_vqtbl1q_u8(vs.val[0], mask1.val[0]), mask2); vs.val[0] = vceqq_u8(vs.val[0], mask2); vs.val[1] = vceqq_u8(vs.val[1], mask2); From f7625019c51ca437a5840576d92362cfa710e4a2 Mon Sep 17 00:00:00 2001 From: compilade <113953597+compilade@users.noreply.github.com> Date: Sun, 25 Feb 2024 13:43:50 -0500 Subject: [PATCH 459/812] server : fix crash when system prompt is bigger than batch size (#5714) The system prompt is now decoded in batches. * server : fix off-by-one n_past when start of prompt matches whole cache The tokens right after the matching part would otherwise skip a pos value. --- examples/server/server.cpp | 28 +++++++++++++++++++++++++--- 1 file changed, 25 insertions(+), 3 deletions(-) diff --git a/examples/server/server.cpp b/examples/server/server.cpp index d970202d2..c1eb61678 100644 --- a/examples/server/server.cpp +++ b/examples/server/server.cpp @@ -902,10 +902,24 @@ struct llama_server_context llama_batch_add(batch, system_tokens[i], i, { 0 }, false); } - if (llama_decode(ctx, batch) != 0) + for (int32_t i = 0; i < (int32_t) batch.n_tokens; i += params.n_batch) { - LOG_TEE("%s: llama_decode() failed\n", __func__); - return; + const int32_t n_tokens = std::min(params.n_batch, (int32_t) (batch.n_tokens - i)); + llama_batch batch_view = { + n_tokens, + batch.token + i, + nullptr, + batch.pos + i, + batch.n_seq_id + i, + batch.seq_id + i, + batch.logits + i, + 0, 0, 0, // unused + }; + if (llama_decode(ctx, batch_view) != 0) + { + LOG_TEE("%s: llama_decode() failed\n", __func__); + return; + } } // assign the system KV cache to all parallel sequences @@ -1785,6 +1799,14 @@ struct llama_server_context } slot.n_past = common_part(slot.cache_tokens, prompt_tokens); + + // the last token of the cache is not in the KV cache until the next call to llama_decode + // (it was sampled, pushed into the "cache_tokens", but not yet put in the context) + if (slot.n_past > 0 && slot.n_past == (int32_t) slot.cache_tokens.size()) + { + slot.n_past -= 1; + } + slot.num_prompt_tokens_processed = slot.num_prompt_tokens - slot.n_past; if (slot.ga_n != 1) From bf08e00643fd529f748f0a858fd79f3061e3fa18 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Sun, 25 Feb 2024 22:12:24 +0200 Subject: [PATCH 460/812] llama : refactor k-shift implementation + KV defragmentation (#5691) * llama : refactor k-shift implementation ggml-ci * llama : rename llama_kv_cache_seq_shift to llama_kv_cache_seq_add * llama : cont k-shift refactoring + normalize type names ggml-ci * minor : fix MPI builds * llama : reuse n_rot from the build context ggml-ci * llama : revert enum name changes from this PR ggml-ci * llama : update llama_rope_type * llama : add comment about rope values * llama : fix build * passkey : apply kv cache updates explicitly ggml-ci * llama : change name to llama_kv_cache_update() * llama : add llama_kv_cache_seq_pos_max() * passkey : fix llama_kv_cache_seq_pos_max() usage * llama : some llama_kv_cell simplifications * llama : add llama_kv_cache_compress (EXPERIMENTAL) * llama : add alternative KV cache merging (EXPERIMENTAL) * llama : add llama_kv_cache_defrag * llama : comments * llama : remove llama_kv_cache_compress will add in a separate PR ggml-ci * llama : defragment via non-overlapping moves * llama : ggml_graph based defrag implementation ggml-ci * llama : switch the loop order in build_defrag * llama : add comments --- examples/infill/infill.cpp | 4 +- examples/main/main.cpp | 10 +- examples/passkey/passkey.cpp | 25 +- examples/server/server.cpp | 8 +- llama.cpp | 869 ++++++++++++++++++++++++----------- llama.h | 34 +- 6 files changed, 646 insertions(+), 304 deletions(-) diff --git a/examples/infill/infill.cpp b/examples/infill/infill.cpp index 92c67b7cf..d4b8729dd 100644 --- a/examples/infill/infill.cpp +++ b/examples/infill/infill.cpp @@ -447,8 +447,8 @@ int main(int argc, char ** argv) { LOG("context full, swapping: n_past = %d, n_left = %d, n_ctx = %d, n_keep = %d, n_discard = %d\n", n_past, n_left, n_ctx, params.n_keep, n_discard); - llama_kv_cache_seq_rm (ctx, 0, params.n_keep + 1 , params.n_keep + n_discard + 1); - llama_kv_cache_seq_shift(ctx, 0, params.n_keep + 1 + n_discard, n_past, -n_discard); + llama_kv_cache_seq_rm (ctx, 0, params.n_keep + 1 , params.n_keep + n_discard + 1); + llama_kv_cache_seq_add(ctx, 0, params.n_keep + 1 + n_discard, n_past, -n_discard); n_past -= n_discard; diff --git a/examples/main/main.cpp b/examples/main/main.cpp index 7555dffe4..34e84d0d4 100644 --- a/examples/main/main.cpp +++ b/examples/main/main.cpp @@ -548,8 +548,8 @@ int main(int argc, char ** argv) { LOG("context full, swapping: n_past = %d, n_left = %d, n_ctx = %d, n_keep = %d, n_discard = %d\n", n_past, n_left, n_ctx, params.n_keep, n_discard); - llama_kv_cache_seq_rm (ctx, 0, params.n_keep , params.n_keep + n_discard); - llama_kv_cache_seq_shift(ctx, 0, params.n_keep + n_discard, n_past, -n_discard); + llama_kv_cache_seq_rm (ctx, 0, params.n_keep , params.n_keep + n_discard); + llama_kv_cache_seq_add(ctx, 0, params.n_keep + n_discard, n_past, -n_discard); n_past -= n_discard; @@ -576,9 +576,9 @@ int main(int argc, char ** argv) { LOG("div: [%6d, %6d] / %6d -> [%6d, %6d]\n", ga_i + ib*bd, ga_i + ib*bd + ga_w, ga_n, (ga_i + ib*bd)/ga_n, (ga_i + ib*bd + ga_w)/ga_n); LOG("shift: [%6d, %6d] + %6d -> [%6d, %6d]\n", ga_i + ib*bd + ga_w, n_past + ib*bd, dd, ga_i + ib*bd + ga_w + dd, n_past + ib*bd + dd); - llama_kv_cache_seq_shift(ctx, 0, ga_i, n_past, ib*bd); - llama_kv_cache_seq_div (ctx, 0, ga_i + ib*bd, ga_i + ib*bd + ga_w, ga_n); - llama_kv_cache_seq_shift(ctx, 0, ga_i + ib*bd + ga_w, n_past + ib*bd, dd); + llama_kv_cache_seq_add(ctx, 0, ga_i, n_past, ib*bd); + llama_kv_cache_seq_div(ctx, 0, ga_i + ib*bd, ga_i + ib*bd + ga_w, ga_n); + llama_kv_cache_seq_add(ctx, 0, ga_i + ib*bd + ga_w, n_past + ib*bd, dd); n_past -= bd; diff --git a/examples/passkey/passkey.cpp b/examples/passkey/passkey.cpp index e12a1cdf1..47de67a93 100644 --- a/examples/passkey/passkey.cpp +++ b/examples/passkey/passkey.cpp @@ -126,7 +126,7 @@ int main(int argc, char ** argv) { const int n_batch = ctx_params.n_batch; const int n_batch_grp = ctx_params.n_batch/n_grp; - LOG_TEE("\n%s: n_len = %d, n_ctx = %d, n_kv_req = %d, n_grp = %d, n_batch = %d\n", __func__, n_len, n_ctx, n_kv_req, n_grp, n_batch); + LOG_TEE("\n%s: n_len = %d, n_ctx = %d, n_kv_req = %d, n_grp = %d, n_batch = %d, n_junk = %d, i_pos = %d\n", __func__, n_len, n_ctx, n_kv_req, n_grp, n_batch, n_junk, i_pos); // print the prompt token-by-token @@ -146,10 +146,11 @@ int main(int argc, char ** argv) { const int ib = i/n_batch - 1; const int bd = n_batch_grp*(n_grp - 1); - llama_kv_cache_seq_shift(ctx, 0, n_past - n_batch, n_past, ib*bd); - llama_kv_cache_seq_div (ctx, 0, n_past - n_batch + ib*bd, n_past + ib*bd, n_grp); + llama_kv_cache_seq_add (ctx, 0, n_past - n_batch, n_past, ib*bd); + llama_kv_cache_seq_div (ctx, 0, n_past - n_batch + ib*bd, n_past + ib*bd, n_grp); + llama_kv_cache_update (ctx); - n_past -= bd; + n_past = llama_kv_cache_seq_pos_max(ctx, 0) + 1; } llama_batch_clear(batch); @@ -179,10 +180,12 @@ int main(int argc, char ** argv) { LOG_TEE("%s: shifting KV cache with %d\n", __func__, n_discard); - llama_kv_cache_seq_rm (ctx, 0, n_keep , n_keep + n_discard); - llama_kv_cache_seq_shift(ctx, 0, n_keep + n_discard, n_ctx, -n_discard); + llama_kv_cache_seq_rm (ctx, 0, n_keep , n_keep + n_discard); + llama_kv_cache_seq_add(ctx, 0, n_keep + n_discard, n_ctx, -n_discard); + llama_kv_cache_defrag (ctx); + llama_kv_cache_update (ctx); - n_past -= n_discard; + n_past = llama_kv_cache_seq_pos_max(ctx, 0) + 1; llama_batch_clear(batch); @@ -208,10 +211,12 @@ int main(int argc, char ** argv) { if (n_discard > 0) { LOG_TEE("%s: shifting KV cache with %d to free space for the answer\n", __func__, n_discard); - llama_kv_cache_seq_rm (ctx, 0, n_keep , n_keep + n_discard); - llama_kv_cache_seq_shift(ctx, 0, n_keep + n_discard, n_ctx, -n_discard); + llama_kv_cache_seq_rm (ctx, 0, n_keep , n_keep + n_discard); + llama_kv_cache_seq_add(ctx, 0, n_keep + n_discard, n_ctx, -n_discard); + llama_kv_cache_defrag (ctx); + llama_kv_cache_update (ctx); - n_past -= n_discard; + n_past = llama_kv_cache_seq_pos_max(ctx, 0) + 1; } } diff --git a/examples/server/server.cpp b/examples/server/server.cpp index c1eb61678..8aadc95a9 100644 --- a/examples/server/server.cpp +++ b/examples/server/server.cpp @@ -1636,8 +1636,8 @@ struct llama_server_context {"n_system_tokens", system_tokens.size()}, {"n_cache_tokens", slot.cache_tokens.size()} }); - llama_kv_cache_seq_rm (ctx, slot.id, n_keep , n_keep + n_discard); - llama_kv_cache_seq_shift(ctx, slot.id, n_keep + n_discard, system_tokens.size() + slot.n_past, -n_discard); + llama_kv_cache_seq_rm (ctx, slot.id, n_keep , n_keep + n_discard); + llama_kv_cache_seq_add(ctx, slot.id, n_keep + n_discard, system_tokens.size() + slot.n_past, -n_discard); for (size_t i = n_keep + n_discard; i < slot.cache_tokens.size(); i++) { @@ -1941,9 +1941,9 @@ struct llama_server_context LOG_TEE("div: [%6d, %6d] / %6d -> [%6d, %6d]\n", slot.ga_i + ib * bd, slot.ga_i + ib * bd + slot.ga_w, slot.ga_n, (slot.ga_i + ib * bd) / slot.ga_n, (slot.ga_i + ib * bd + slot.ga_w) / slot.ga_n); LOG_TEE("shift: [%6d, %6d] + %6d -> [%6d, %6d]\n", slot.ga_i + ib * bd + slot.ga_w, slot.n_past_se + ib * bd, dd, slot.ga_i + ib * bd + slot.ga_w + dd, slot.n_past_se + ib * bd + dd); - llama_kv_cache_seq_shift(ctx, slot.id, slot.ga_i, slot.n_past_se, ib * bd); + llama_kv_cache_seq_add(ctx, slot.id, slot.ga_i, slot.n_past_se, ib * bd); llama_kv_cache_seq_div(ctx, slot.id, slot.ga_i + ib * bd, slot.ga_i + ib * bd + slot.ga_w,slot.ga_n); - llama_kv_cache_seq_shift(ctx, slot.id, slot.ga_i + ib * bd + slot.ga_w,slot.n_past_se + ib * bd, dd); + llama_kv_cache_seq_add(ctx, slot.id, slot.ga_i + ib * bd + slot.ga_w,slot.n_past_se + ib * bd, dd); slot.n_past_se -= bd; diff --git a/llama.cpp b/llama.cpp index acd9be08a..3424b1999 100644 --- a/llama.cpp +++ b/llama.cpp @@ -1550,8 +1550,9 @@ static const size_t MiB = 1024*kiB; static const size_t GiB = 1024*MiB; struct llama_hparams { - bool vocab_only; - bool rope_finetuned; + bool vocab_only; + bool rope_finetuned; + uint32_t n_vocab; uint32_t n_ctx_train; // context size the model was trained on uint32_t n_embd; @@ -1580,7 +1581,8 @@ struct llama_hparams { bool causal_attn = true; bool need_kq_pos = false; - uint32_t pooling_type = LLAMA_POOLING_TYPE_NONE; + enum llama_pooling_type pooling_type = LLAMA_POOLING_TYPE_NONE; + enum llama_rope_type rope_type = LLAMA_ROPE_TYPE_NONE; bool operator!=(const llama_hparams & other) const { if (this->vocab_only != other.vocab_only) return true; @@ -1707,11 +1709,20 @@ struct llama_kv_cell { bool has_seq_id(const llama_seq_id & id) const { return seq_id.find(id) != seq_id.end(); } + + bool is_empty() const { + return seq_id.empty(); + } + + bool is_same_seq(const llama_kv_cell & other) const { + return seq_id == other.seq_id; + } }; // ring-buffer of cached KV data struct llama_kv_cache { bool has_shift = false; + bool do_defrag = false; // Note: The value of head isn't only used to optimize searching // for a free KV slot. llama_decode_internal also uses it, so it @@ -1723,6 +1734,9 @@ struct llama_kv_cache { // computed before each graph build uint32_t n = 0; + ggml_type type_k = GGML_TYPE_F16; + ggml_type type_v = GGML_TYPE_F16; + std::vector cells; std::vector k_l; // per layer @@ -1958,8 +1972,8 @@ struct llama_context { static bool llama_kv_cache_init( struct llama_kv_cache & cache, const llama_model & model, - ggml_type ktype, - ggml_type vtype, + ggml_type type_k, + ggml_type type_v, uint32_t n_ctx, bool offload) { const struct llama_hparams & hparams = model.hparams; @@ -1974,6 +1988,9 @@ static bool llama_kv_cache_init( cache.size = n_ctx; cache.used = 0; + cache.type_k = type_k; + cache.type_v = type_v; + cache.cells.clear(); cache.cells.resize(n_ctx); @@ -2014,8 +2031,8 @@ static bool llama_kv_cache_init( for (int i = 0; i < (int) n_layer; i++) { struct ggml_context * ctx = offload ? ctx_map.at(model.buft_layer[i].buft) : cache.ctxs.front(); - ggml_tensor * k = ggml_new_tensor_1d(ctx, ktype, n_embd_k_gqa*n_ctx); - ggml_tensor * v = ggml_new_tensor_1d(ctx, vtype, n_embd_v_gqa*n_ctx); + ggml_tensor * k = ggml_new_tensor_1d(ctx, type_k, n_embd_k_gqa*n_ctx); + ggml_tensor * v = ggml_new_tensor_1d(ctx, type_v, n_embd_v_gqa*n_ctx); ggml_format_name(k, "cache_k_l%d", i); ggml_format_name(v, "cache_v_l%d", i); cache.k_l.push_back(k); @@ -2099,7 +2116,7 @@ static bool llama_kv_cache_find_slot( // find how many cells are currently in use static int32_t llama_kv_cache_cell_max(const struct llama_kv_cache & cache) { for (uint32_t i = cache.size - 1; i > 0; --i) { - if (cache.cells[i].pos >= 0 && !cache.cells[i].seq_id.empty()) { + if (cache.cells[i].pos >= 0 && !cache.cells[i].is_empty()) { return i + 1; } } @@ -2135,7 +2152,7 @@ static void llama_kv_cache_seq_rm( } else { continue; } - if (cache.cells[i].seq_id.empty()) { + if (cache.cells[i].is_empty()) { // keep count of the number of used cells if (cache.cells[i].pos >= 0) cache.used--; @@ -2186,7 +2203,7 @@ static void llama_kv_cache_seq_keep(struct llama_kv_cache & cache, llama_seq_id if (new_head != cache.size && new_head < cache.head) cache.head = new_head; } -static void llama_kv_cache_seq_shift( +static void llama_kv_cache_seq_add( struct llama_kv_cache & cache, llama_seq_id seq_id, llama_pos p0, @@ -2204,10 +2221,14 @@ static void llama_kv_cache_seq_shift( cache.cells[i].delta += delta; if (cache.cells[i].pos < 0) { - if (!cache.cells[i].seq_id.empty()) cache.used--; + if (!cache.cells[i].is_empty()) { + cache.used--; + } cache.cells[i].pos = -1; cache.cells[i].seq_id.clear(); - if (new_head == cache.size) new_head = i; + if (new_head == cache.size) { + new_head = i; + } } } } @@ -2239,6 +2260,22 @@ static void llama_kv_cache_seq_div( } } +static llama_pos llama_kv_cache_seq_pos_max(struct llama_kv_cache & cache, llama_seq_id seq_id) { + llama_pos result = 0; + + for (uint32_t i = 0; i < cache.size; ++i) { + if (cache.cells[i].has_seq_id(seq_id)) { + result = std::max(result, cache.cells[i].pos); + } + } + + return result; +} + +static void llama_kv_cache_defrag(struct llama_kv_cache & cache) { + cache.do_defrag = true; +} + // // model loading and saving // @@ -2310,7 +2347,7 @@ namespace GGUFMeta { } }; - struct ArrayInfo{ + struct ArrayInfo { const gguf_type gt; const size_t length; const void * data; @@ -2329,7 +2366,7 @@ namespace GGUFMeta { }; template - class GKV: public GKV_Base { + class GKV : public GKV_Base { GKV() = delete; public: @@ -2352,39 +2389,39 @@ namespace GGUFMeta { return "unknown"; } - static bool validate_override(const llama_model_kv_override_type expected_type, const struct llama_model_kv_override *override) { - if (!override) { return false; } - if (override->tag == expected_type) { + static bool validate_override(const llama_model_kv_override_type expected_type, const struct llama_model_kv_override * ovrd) { + if (!ovrd) { return false; } + if (ovrd->tag == expected_type) { LLAMA_LOG_INFO("%s: Using metadata override (%5s) '%s' = ", - __func__, override_type_to_str(override->tag), override->key); - switch (override->tag) { + __func__, override_type_to_str(ovrd->tag), ovrd->key); + switch (ovrd->tag) { case LLAMA_KV_OVERRIDE_TYPE_BOOL: { - LLAMA_LOG_INFO("%s\n", override->bool_value ? "true" : "false"); + LLAMA_LOG_INFO("%s\n", ovrd->bool_value ? "true" : "false"); } break; case LLAMA_KV_OVERRIDE_TYPE_INT: { - LLAMA_LOG_INFO("%" PRId64 "\n", override->int_value); + LLAMA_LOG_INFO("%" PRId64 "\n", ovrd->int_value); } break; case LLAMA_KV_OVERRIDE_TYPE_FLOAT: { - LLAMA_LOG_INFO("%.6f\n", override->float_value); + LLAMA_LOG_INFO("%.6f\n", ovrd->float_value); } break; default: // Shouldn't be possible to end up here, but just in case... throw std::runtime_error( format("Unsupported attempt to override %s type for metadata key %s\n", - override_type_to_str(override->tag), override->key)); + override_type_to_str(ovrd->tag), ovrd->key)); } return true; } LLAMA_LOG_WARN("%s: Warning: Bad metadata override type for key '%s', expected %s but got %s\n", - __func__, override->key, override_type_to_str(expected_type), override_type_to_str(override->tag)); + __func__, ovrd->key, override_type_to_str(expected_type), override_type_to_str(ovrd->tag)); return false; } template static typename std::enable_if::value, bool>::type - try_override(OT & target, const struct llama_model_kv_override *override) { - if (validate_override(LLAMA_KV_OVERRIDE_TYPE_BOOL, override)) { - target = override->bool_value; + try_override(OT & target, const struct llama_model_kv_override * ovrd) { + if (validate_override(LLAMA_KV_OVERRIDE_TYPE_BOOL, ovrd)) { + target = ovrd->bool_value; return true; } return false; @@ -2392,9 +2429,9 @@ namespace GGUFMeta { template static typename std::enable_if::value && std::is_integral::value, bool>::type - try_override(OT & target, const struct llama_model_kv_override *override) { - if (validate_override(LLAMA_KV_OVERRIDE_TYPE_INT, override)) { - target = override->int_value; + try_override(OT & target, const struct llama_model_kv_override * ovrd) { + if (validate_override(LLAMA_KV_OVERRIDE_TYPE_INT, ovrd)) { + target = ovrd->int_value; return true; } return false; @@ -2402,9 +2439,9 @@ namespace GGUFMeta { template static typename std::enable_if::value, bool>::type - try_override(T & target, const struct llama_model_kv_override *override) { - if (validate_override(LLAMA_KV_OVERRIDE_TYPE_FLOAT, override)) { - target = override->float_value; + try_override(T & target, const struct llama_model_kv_override * ovrd) { + if (validate_override(LLAMA_KV_OVERRIDE_TYPE_FLOAT, ovrd)) { + target = ovrd->float_value; return true; } return false; @@ -2412,17 +2449,17 @@ namespace GGUFMeta { template static typename std::enable_if::value, bool>::type - try_override(T & target, const struct llama_model_kv_override *override) { + try_override(T & target, const struct llama_model_kv_override * ovrd) { (void)target; - (void)override; - if (!override) { return false; } + (void)ovrd; + if (!ovrd) { return false; } // Currently, we should never end up here so it would be a bug if we do. throw std::runtime_error(format("Unsupported attempt to override string type for metadata key %s\n", - override ? override->key : "NULL")); + ovrd ? ovrd->key : "NULL")); } - static bool set(const gguf_context * ctx, const int k, T & target, const struct llama_model_kv_override *override = nullptr) { - if (try_override(target, override)) { + static bool set(const gguf_context * ctx, const int k, T & target, const struct llama_model_kv_override * ovrd = nullptr) { + if (try_override(target, ovrd)) { return true; } if (k < 0) { return false; } @@ -2430,12 +2467,12 @@ namespace GGUFMeta { return true; } - static bool set(const gguf_context * ctx, const char * key, T & target, const struct llama_model_kv_override *override = nullptr) { - return set(ctx, gguf_find_key(ctx, key), target, override); + static bool set(const gguf_context * ctx, const char * key, T & target, const struct llama_model_kv_override * ovrd = nullptr) { + return set(ctx, gguf_find_key(ctx, key), target, ovrd); } - static bool set(const gguf_context * ctx, const std::string & key, T & target, const struct llama_model_kv_override *override = nullptr) { - return set(ctx, key.c_str(), target, override); + static bool set(const gguf_context * ctx, const std::string & key, T & target, const struct llama_model_kv_override * ovrd = nullptr) { + return set(ctx, key.c_str(), target, ovrd); } }; } @@ -2846,6 +2883,15 @@ struct llama_model_loader { } }; +template<> +bool llama_model_loader::get_key(const enum llm_kv kid, enum llama_pooling_type & result, const bool required) { + uint32_t tmp; + const bool found = get_key(kid, tmp, required); + result = (enum llama_pooling_type) tmp; + return found; +} + + // // load LLaMA models // @@ -2926,16 +2972,16 @@ static const char * llama_model_type_name(e_model type) { default: return "?B"; } } + static const char * llama_model_vocab_type_name(enum llama_vocab_type type){ switch (type) { - case LLAMA_VOCAB_TYPE_SPM: return "SPM"; - case LLAMA_VOCAB_TYPE_BPE: return "BPE"; - case LLAMA_VOCAB_TYPE_WPM: return "WPM"; - default: return "unknown"; + case LLAMA_VOCAB_TYPE_SPM: return "SPM"; + case LLAMA_VOCAB_TYPE_BPE: return "BPE"; + case LLAMA_VOCAB_TYPE_WPM: return "WPM"; + default: return "unknown"; } } - static void llm_load_arch(llama_model_loader & ml, llama_model & model) { model.arch = ml.get_arch(); if (model.arch == LLM_ARCH_UNKNOWN) { @@ -3112,10 +3158,10 @@ static void llm_load_hparams( } break; case LLM_ARCH_BERT: { - ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps); - ml.get_key(LLM_KV_ATTENTION_CAUSAL, hparams.causal_attn); + ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps); + ml.get_key(LLM_KV_ATTENTION_CAUSAL, hparams.causal_attn); ml.get_key(LLM_KV_TOKENIZER_TOKEN_TYPE_COUNT, hparams.n_vocab_type); - ml.get_key(LLM_KV_POOLING_TYPE, hparams.pooling_type); + ml.get_key(LLM_KV_POOLING_TYPE, hparams.pooling_type); switch (hparams.n_layer) { case 3: @@ -3133,10 +3179,10 @@ static void llm_load_hparams( } break; case LLM_ARCH_NOMIC_BERT: { - ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps); - ml.get_key(LLM_KV_ATTENTION_CAUSAL, hparams.causal_attn); + ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps); + ml.get_key(LLM_KV_ATTENTION_CAUSAL, hparams.causal_attn); ml.get_key(LLM_KV_TOKENIZER_TOKEN_TYPE_COUNT, hparams.n_vocab_type); - ml.get_key(LLM_KV_POOLING_TYPE, hparams.pooling_type); + ml.get_key(LLM_KV_POOLING_TYPE, hparams.pooling_type); if (hparams.n_layer == 12 && hparams.n_embd == 768) { model.type = e_model::MODEL_137M; @@ -3275,6 +3321,8 @@ static void llm_load_hparams( if (hparams.f_max_alibi_bias > 0.0f) { hparams.need_kq_pos = true; } + + hparams.rope_type = llama_rope_type(&model); } // TODO: This should probably be in llama.h @@ -3577,6 +3625,8 @@ static void llm_load_print_meta(llama_model_loader & ml, llama_model & model) { LLAMA_LOG_INFO("%s: n_ff = %u\n", __func__, hparams.n_ff); LLAMA_LOG_INFO("%s: n_expert = %u\n", __func__, hparams.n_expert); LLAMA_LOG_INFO("%s: n_expert_used = %u\n", __func__, hparams.n_expert_used); + LLAMA_LOG_INFO("%s: pooling type = %d\n", __func__, hparams.pooling_type); + LLAMA_LOG_INFO("%s: rope type = %d\n", __func__, hparams.rope_type); LLAMA_LOG_INFO("%s: rope scaling = %s\n", __func__, rope_scaling_type); LLAMA_LOG_INFO("%s: freq_base_train = %.1f\n", __func__, hparams.rope_freq_base_train); LLAMA_LOG_INFO("%s: freq_scale_train = %g\n", __func__, hparams.rope_freq_scale_train); @@ -4598,12 +4648,6 @@ static int llama_model_load(const std::string & fname, llama_model & model, llam using llm_build_cb = std::function; -enum llm_rope_type { - LLM_ROPE, - LLM_ROPE_NEOX, - LLM_ROPE_GLM, -}; - enum llm_ffn_op_type { LLM_FFN_SILU, LLM_FFN_GELU, @@ -4649,55 +4693,6 @@ static struct ggml_tensor * llm_build_inp_embd( return inpL; } -// Persimmon: n_rot = n_embd_head_k/2 -// Other: n_rot = n_embd_head_k -static void llm_build_k_shift( - struct ggml_context * ctx, - const llama_hparams & hparams, - const llama_cparams & cparams, - const llama_kv_cache & kv, - struct ggml_cgraph * graph, - struct ggml_tensor * K_shift, - llm_rope_type type, - int64_t n_ctx, - float freq_base, - float freq_scale, - const llm_build_cb & cb) { - const int64_t n_layer = hparams.n_layer; - const int64_t n_head_kv = hparams.n_head_kv; - const int64_t n_embd_head_k = hparams.n_embd_head_k; - const int64_t n_embd_k_gqa = hparams.n_embd_k_gqa(); - const int32_t n_rot = hparams.n_rot; - const int32_t n_orig_ctx = cparams.n_yarn_orig_ctx; - const float ext_factor = cparams.yarn_ext_factor; - const float attn_factor = cparams.yarn_attn_factor; - const float beta_fast = cparams.yarn_beta_fast; - const float beta_slow = cparams.yarn_beta_slow; - - int rope_type = 0; - - switch (type) { - case LLM_ROPE: rope_type = 0; break; - case LLM_ROPE_NEOX: rope_type = 2; break; - case LLM_ROPE_GLM: rope_type = 4; break; - } - - for (int il = 0; il < n_layer; ++il) { - struct ggml_tensor * tmp = - // we rotate only the first n_rot dimensions - ggml_rope_custom_inplace(ctx, - ggml_view_3d(ctx, kv.k_l[il], - n_embd_head_k, n_head_kv, n_ctx, - ggml_row_size(kv.k_l[il]->type, n_embd_head_k), - ggml_row_size(kv.k_l[il]->type, n_embd_k_gqa), - 0), - K_shift, n_rot, rope_type, 0, n_orig_ctx, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); - cb(tmp, "K_shifted", il); - ggml_build_forward_expand(graph, tmp); - } -} - static void llm_build_kv_store( struct ggml_context * ctx, const llama_hparams & hparams, @@ -5001,6 +4996,7 @@ struct llm_build_context { const int64_t n_embd; const int64_t n_layer; + const int64_t n_rot; const int64_t n_ctx; // user-specified context size (can be different from n_ctx_train) const int64_t n_head; const int64_t n_head_kv; @@ -5025,8 +5021,8 @@ struct llm_build_context { const int32_t kv_head; // index of where we store new KV data in the cache const int32_t n_orig_ctx; - const bool do_rope_shift; - const uint32_t pooling_type; + const enum llama_pooling_type pooling_type; + const enum llama_rope_type rope_type; const llm_build_cb & cb; @@ -5048,6 +5044,7 @@ struct llm_build_context { kv_self (lctx.kv_self), n_embd (hparams.n_embd), n_layer (hparams.n_layer), + n_rot (hparams.n_rot), n_ctx (cparams.n_ctx), n_head (hparams.n_head), n_head_kv (hparams.n_head_kv), @@ -5069,8 +5066,8 @@ struct llm_build_context { n_kv (worst_case ? n_ctx : kv_self.n), kv_head (worst_case ? n_ctx - n_tokens : kv_self.head), n_orig_ctx (cparams.n_yarn_orig_ctx), - do_rope_shift (worst_case || kv_self.has_shift), - pooling_type (cparams.do_pooling ? hparams.pooling_type : (uint32_t)LLAMA_POOLING_TYPE_NONE), + pooling_type (cparams.do_pooling ? hparams.pooling_type : LLAMA_POOLING_TYPE_NONE), + rope_type (hparams.rope_type), cb (cb), buf_compute_meta (lctx.buf_compute_meta) { // all initializations should be done in init() @@ -5093,6 +5090,74 @@ struct llm_build_context { } } + struct ggml_cgraph * build_k_shift() { + struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, LLAMA_MAX_NODES, false); + + for (int il = 0; il < n_layer; ++il) { + struct ggml_tensor * tmp = + // we rotate only the first n_rot dimensions + ggml_rope_custom_inplace(ctx0, + ggml_view_3d(ctx0, kv_self.k_l[il], + n_embd_head_k, n_head_kv, n_ctx, + ggml_row_size(kv_self.k_l[il]->type, n_embd_head_k), + ggml_row_size(kv_self.k_l[il]->type, n_embd_k_gqa), + 0), + lctx.inp_K_shift, n_rot, rope_type, 0, n_orig_ctx, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow); + cb(tmp, "K_shifted", il); + ggml_build_forward_expand(gf, tmp); + } + + return gf; + } + + struct ggml_cgraph * build_defrag(const std::vector & ids) { + struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, LLAMA_MAX_NODES, false); + + for (int i = 0; i < n_kv; ++i) { + const int id = ids[i]; + + if (i == id || id == n_kv) { + continue; + } + + int nm = 1; + + while (i + nm < n_kv && (int) ids[i + nm] == id + nm) { + nm++; + } + + for (int il = 0; il < n_layer; ++il) { + ggml_tensor * view_k_src = ggml_view_2d(ctx0, kv_self.k_l[il], + n_embd_k_gqa, nm, + ggml_row_size(kv_self.k_l[il]->type, n_embd_k_gqa), + ggml_row_size(kv_self.k_l[il]->type, n_embd_k_gqa*i)); + + ggml_tensor * view_k_dst = ggml_view_2d(ctx0, kv_self.k_l[il], + n_embd_k_gqa, nm, + ggml_row_size(kv_self.k_l[il]->type, n_embd_k_gqa), + ggml_row_size(kv_self.k_l[il]->type, n_embd_k_gqa*id)); + + ggml_tensor * view_v_src = ggml_view_2d(ctx0, kv_self.v_l[il], + nm, n_embd_v_gqa, + ggml_row_size(kv_self.v_l[il]->type, kv_self.size), + ggml_row_size(kv_self.v_l[il]->type, i)); + + ggml_tensor * view_v_dst = ggml_view_2d(ctx0, kv_self.v_l[il], + nm, n_embd_v_gqa, + ggml_row_size(kv_self.v_l[il]->type, kv_self.size), + ggml_row_size(kv_self.v_l[il]->type, id)); + + ggml_build_forward_expand(gf, ggml_cpy(ctx0, view_k_src, view_k_dst)); + ggml_build_forward_expand(gf, ggml_cpy(ctx0, view_v_src, view_v_dst)); + } + + i += nm - 1; + } + + return gf; + } + struct ggml_cgraph * build_llama() { struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, LLAMA_MAX_NODES, false); @@ -5114,11 +5179,6 @@ struct llm_build_context { struct ggml_tensor * KQ_mask = ggml_view_2d(ctx0, lctx.inp_KQ_mask, n_kv, n_tokens, n_kv*ggml_type_size(lctx.inp_KQ_mask->type), 0); cb(KQ_mask, "KQ_mask", -1); - // shift the entire K-cache if needed - if (do_rope_shift) { - llm_build_k_shift(ctx0, hparams, cparams, kv_self, gf, lctx.inp_K_shift, LLM_ROPE, n_ctx, freq_base, freq_scale, cb); - } - for (int il = 0; il < n_layer; ++il) { struct ggml_tensor * inpSA = inpL; @@ -5154,14 +5214,14 @@ struct llm_build_context { Qcur = ggml_rope_custom( ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, - hparams.n_rot, 0, 0, n_orig_ctx, freq_base, freq_scale, + n_rot, rope_type, 0, n_orig_ctx, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); cb(Qcur, "Qcur", il); Kcur = ggml_rope_custom( ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, - hparams.n_rot, 0, 0, n_orig_ctx, freq_base, freq_scale, + n_rot, rope_type, 0, n_orig_ctx, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); cb(Kcur, "Kcur", il); @@ -5302,11 +5362,6 @@ struct llm_build_context { struct ggml_tensor * KQ_pos = ggml_view_1d(ctx0, lctx.inp_KQ_pos, n_kv, 0); cb(KQ_pos, "KQ_pos", -1); - // shift the entire K-cache if needed - if (do_rope_shift) { - llm_build_k_shift(ctx0, hparams, cparams, kv_self, gf, lctx.inp_K_shift, LLM_ROPE, n_ctx, freq_base, freq_scale, cb); - } - for (int il = 0; il < n_layer; ++il) { struct ggml_tensor * inpSA = inpL; @@ -5330,12 +5385,12 @@ struct llm_build_context { case MODEL_7B: Qcur = ggml_rope_custom( ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, - hparams.n_rot, 0, 0, n_orig_ctx, freq_base, freq_scale, + n_rot, rope_type, 0, n_orig_ctx, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); Kcur = ggml_rope_custom( ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, - hparams.n_rot, 0, 0, n_orig_ctx, freq_base, freq_scale, + n_rot, rope_type, 0, n_orig_ctx, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); break; @@ -5420,11 +5475,6 @@ struct llm_build_context { struct ggml_tensor * KQ_mask = ggml_view_2d(ctx0, lctx.inp_KQ_mask, n_kv, n_tokens, n_kv*ggml_type_size(lctx.inp_KQ_mask->type), 0); cb(KQ_mask, "KQ_mask", -1); - // shift the entire K-cache if needed - if (do_rope_shift) { - llm_build_k_shift(ctx0, hparams, cparams, kv_self, gf, lctx.inp_K_shift, LLM_ROPE_NEOX, n_ctx, freq_base, freq_scale, cb); - } - for (int il = 0; il < n_layer; ++il) { struct ggml_tensor * attn_norm; @@ -5463,13 +5513,13 @@ struct llm_build_context { // using mode = 2 for neox mode Qcur = ggml_rope_custom( - ctx0, Qcur, inp_pos, hparams.n_rot, 2, 0, n_orig_ctx, + ctx0, Qcur, inp_pos, n_rot, rope_type, 0, n_orig_ctx, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); cb(Qcur, "Qcur", il); Kcur = ggml_rope_custom( - ctx0, Kcur, inp_pos, hparams.n_rot, 2, 0, n_orig_ctx, + ctx0, Kcur, inp_pos, n_rot, rope_type, 0, n_orig_ctx, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); cb(Kcur, "Kcur", il); @@ -5639,10 +5689,6 @@ struct llm_build_context { struct ggml_tensor * KQ_mask = ggml_view_2d(ctx0, lctx.inp_KQ_mask, n_kv, n_tokens, n_kv*ggml_type_size(lctx.inp_KQ_mask->type), 0); cb(KQ_mask, "KQ_mask", -1); - if (do_rope_shift) { - llm_build_k_shift(ctx0, hparams, cparams, kv_self, gf, lctx.inp_K_shift, LLM_ROPE_NEOX, n_ctx, freq_base, freq_scale, cb); - } - for (int il = 0; il < n_layer; ++il) { struct ggml_tensor * residual = inpL; @@ -5700,7 +5746,7 @@ struct llm_build_context { // RoPE the first n_rot of q/k, pass the other half, and concat. struct ggml_tensor * qrot = ggml_view_3d( - ctx0, tmpq, hparams.n_rot, n_head, n_tokens, + ctx0, tmpq, n_rot, n_head, n_tokens, ggml_element_size(tmpq) * n_embd_head, ggml_element_size(tmpq) * n_embd_head * n_head, 0 @@ -5708,7 +5754,7 @@ struct llm_build_context { cb(qrot, "qrot", il); struct ggml_tensor * krot = ggml_view_3d( - ctx0, tmpk, hparams.n_rot, n_head, n_tokens, + ctx0, tmpk, n_rot, n_head, n_tokens, ggml_element_size(tmpk) * n_embd_head, ggml_element_size(tmpk) * n_embd_head * n_head, 0 @@ -5717,29 +5763,29 @@ struct llm_build_context { // get the second half of tmpq, e.g tmpq[n_rot:, :, :] struct ggml_tensor * qpass = ggml_view_3d( - ctx0, tmpq, hparams.n_rot, n_head, n_tokens, + ctx0, tmpq, n_rot, n_head, n_tokens, ggml_element_size(tmpq) * n_embd_head, ggml_element_size(tmpq) * n_embd_head * n_head, - ggml_element_size(tmpq) * hparams.n_rot + ggml_element_size(tmpq) * n_rot ); cb(qpass, "qpass", il); struct ggml_tensor * kpass = ggml_view_3d( - ctx0, tmpk, hparams.n_rot, n_head, n_tokens, + ctx0, tmpk, n_rot, n_head, n_tokens, ggml_element_size(tmpk) * n_embd_head, ggml_element_size(tmpk) * n_embd_head * n_head, - ggml_element_size(tmpk) * hparams.n_rot + ggml_element_size(tmpk) * n_rot ); cb(kpass, "kpass", il); struct ggml_tensor * qrotated = ggml_rope_custom( - ctx0, qrot, inp_pos, hparams.n_rot, 2, 0, n_orig_ctx, + ctx0, qrot, inp_pos, n_rot, rope_type, 0, n_orig_ctx, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); cb(qrotated, "qrotated", il); struct ggml_tensor * krotated = ggml_rope_custom( - ctx0, krot, inp_pos, hparams.n_rot, 2, 0, n_orig_ctx, + ctx0, krot, inp_pos, n_rot, rope_type, 0, n_orig_ctx, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); cb(krotated, "krotated", il); @@ -5991,14 +6037,14 @@ struct llm_build_context { Qcur = ggml_rope_custom( ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, - hparams.n_rot, 2, 0, n_orig_ctx, freq_base, freq_scale, + n_rot, rope_type, 0, n_orig_ctx, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); cb(Qcur, "Qcur", il); Kcur = ggml_rope_custom( ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, - hparams.n_rot, 2, 0, n_orig_ctx, freq_base, freq_scale, + n_rot, rope_type, 0, n_orig_ctx, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); cb(Kcur, "Kcur", il); @@ -6287,11 +6333,6 @@ struct llm_build_context { struct ggml_tensor * KQ_mask = ggml_view_2d(ctx0, lctx.inp_KQ_mask, n_kv, n_tokens, n_kv*ggml_type_size(lctx.inp_KQ_mask->type), 0); cb(KQ_mask, "KQ_mask", -1); - // shift the entire K-cache if needed - if (do_rope_shift) { - llm_build_k_shift(ctx0, hparams, cparams, kv_self, gf, lctx.inp_K_shift, LLM_ROPE_NEOX, n_ctx, freq_base, freq_scale, cb); - } - for (int il = 0; il < n_layer; ++il) { struct ggml_tensor * inpSA = inpL; @@ -6328,14 +6369,14 @@ struct llm_build_context { Qcur = ggml_rope_custom( ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, - hparams.n_rot, 2, 0, n_orig_ctx, freq_base, freq_scale, + n_rot, rope_type, 0, n_orig_ctx, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); cb(Qcur, "Qcur", il); Kcur = ggml_rope_custom( ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, - hparams.n_rot, 2, 0, n_orig_ctx, freq_base, freq_scale, + n_rot, rope_type, 0, n_orig_ctx, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); cb(Kcur, "Kcur", il); @@ -6410,11 +6451,6 @@ struct llm_build_context { struct ggml_tensor * KQ_mask = ggml_view_2d(ctx0, lctx.inp_KQ_mask, n_kv, n_tokens, n_kv*ggml_type_size(lctx.inp_KQ_mask->type), 0); cb(KQ_mask, "KQ_mask", -1); - // shift the entire K-cache if needed - if (do_rope_shift) { - llm_build_k_shift(ctx0, hparams, cparams, kv_self, gf, lctx.inp_K_shift, LLM_ROPE_NEOX, n_ctx, freq_base, freq_scale, cb); - } - for (int il = 0; il < n_layer; ++il) { struct ggml_tensor * inpSA = inpL; @@ -6444,13 +6480,13 @@ struct llm_build_context { // using mode = 2 for neox mode Qcur = ggml_rope_custom( - ctx0, Qcur, inp_pos, hparams.n_rot, 2, 0, n_orig_ctx, + ctx0, Qcur, inp_pos, n_rot, rope_type, 0, n_orig_ctx, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); cb(Qcur, "Qcur", il); Kcur = ggml_rope_custom( - ctx0, Kcur, inp_pos, hparams.n_rot, 2, 0, n_orig_ctx, + ctx0, Kcur, inp_pos, n_rot, rope_type, 0, n_orig_ctx, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); cb(Kcur, "Kcur", il); @@ -6524,11 +6560,6 @@ struct llm_build_context { struct ggml_tensor * KQ_mask = ggml_view_2d(ctx0, lctx.inp_KQ_mask, n_kv, n_tokens, n_kv*ggml_type_size(lctx.inp_KQ_mask->type), 0); cb(KQ_mask, "KQ_mask", -1); - // shift the entire K-cache if needed - if (do_rope_shift) { - llm_build_k_shift(ctx0, hparams, cparams, kv_self, gf, lctx.inp_K_shift, LLM_ROPE_NEOX, n_ctx, freq_base, freq_scale, cb); - } - for (int il = 0; il < n_layer; ++il) { struct ggml_tensor * inpSA = inpL; @@ -6564,14 +6595,14 @@ struct llm_build_context { Qcur = ggml_rope_custom( ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, - hparams.n_rot, 2, 0, n_orig_ctx, freq_base, freq_scale, + n_rot, rope_type, 0, n_orig_ctx, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); cb(Qcur, "Qcur", il); Kcur = ggml_rope_custom( ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, - hparams.n_rot, 2, 0, n_orig_ctx, freq_base, freq_scale, + n_rot, rope_type, 0, n_orig_ctx, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); cb(Kcur, "Kcur", il); @@ -6645,11 +6676,6 @@ struct llm_build_context { struct ggml_tensor * KQ_mask = ggml_view_2d(ctx0, lctx.inp_KQ_mask, n_kv, n_tokens, n_kv*ggml_type_size(lctx.inp_KQ_mask->type), 0); cb(KQ_mask, "KQ_mask", -1); - // shift the entire K-cache if needed - if (do_rope_shift) { - llm_build_k_shift(ctx0, hparams, cparams, kv_self, gf, lctx.inp_K_shift, LLM_ROPE_NEOX, n_ctx, freq_base, freq_scale, cb); - } - for (int il = 0; il < n_layer; ++il) { attn_norm_output = llm_build_norm(ctx0, inpL, hparams, model.layers[il].attn_norm, @@ -6687,7 +6713,7 @@ struct llm_build_context { Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens); Qcur = ggml_rope_custom( - ctx0, Qcur, inp_pos, hparams.n_rot, 2, 0, n_orig_ctx, + ctx0, Qcur, inp_pos, n_rot, rope_type, 0, n_orig_ctx, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); cb(Qcur, "Qcur", il); @@ -6698,7 +6724,7 @@ struct llm_build_context { cb(Qcur, "Qcur", il); Kcur = ggml_rope_custom( - ctx0, Kcur, inp_pos, hparams.n_rot, 2, 0, n_orig_ctx, + ctx0, Kcur, inp_pos, n_rot, rope_type, 0, n_orig_ctx, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); cb(Kcur, "Kcur", il); @@ -6767,11 +6793,6 @@ struct llm_build_context { struct ggml_tensor * KQ_mask = ggml_view_2d(ctx0, lctx.inp_KQ_mask, n_kv, n_tokens, n_kv*ggml_type_size(lctx.inp_KQ_mask->type), 0); cb(KQ_mask, "KQ_mask", -1); - // shift the entire K-cache if needed - if (do_rope_shift) { - llm_build_k_shift(ctx0, hparams, cparams, kv_self, gf, lctx.inp_K_shift, LLM_ROPE, n_ctx, freq_base, freq_scale, cb); - } - for (int il = 0; il < n_layer; ++il) { // norm @@ -6795,14 +6816,14 @@ struct llm_build_context { cb(Vcur, "Vcur", il); Qcur = ggml_rope_custom( - ctx0, ggml_reshape_3d(ctx0, Qcur, hparams.n_rot, n_head, n_tokens), inp_pos, - n_embd_head, 2, 0, n_orig_ctx, freq_base, freq_scale, + ctx0, ggml_reshape_3d(ctx0, Qcur, n_rot, n_head, n_tokens), inp_pos, + n_embd_head, rope_type, 0, n_orig_ctx, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow); cb(Qcur, "Qcur", il); Kcur = ggml_rope_custom( - ctx0, ggml_reshape_3d(ctx0, Kcur, hparams.n_rot, n_head_kv, n_tokens), inp_pos, - n_embd_head, 2, 0, n_orig_ctx, freq_base, freq_scale, + ctx0, ggml_reshape_3d(ctx0, Kcur, n_rot, n_head_kv, n_tokens), inp_pos, + n_embd_head, rope_type, 0, n_orig_ctx, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow); cb(Kcur, "Kcur", il); @@ -6972,11 +6993,6 @@ struct llm_build_context { struct ggml_tensor * KQ_mask = ggml_view_2d(ctx0, lctx.inp_KQ_mask, n_kv, n_tokens, n_kv*ggml_type_size(lctx.inp_KQ_mask->type), 0); cb(KQ_mask, "KQ_mask", -1); - // shift the entire K-cache if needed - if (do_rope_shift) { - llm_build_k_shift(ctx0, hparams, cparams, kv_self, gf, lctx.inp_K_shift, LLM_ROPE, n_ctx, freq_base, freq_scale, cb); - } - for (int il = 0; il < n_layer; ++il) { cur = llm_build_norm(ctx0, inpL, hparams, model.layers[il].attn_norm, @@ -7002,14 +7018,14 @@ struct llm_build_context { struct ggml_tensor * Qcur = ggml_rope_custom( ctx0, ggml_reshape_3d(ctx0, tmpq, n_embd_head, n_head, n_tokens), inp_pos, - hparams.n_rot, 2, 0, n_orig_ctx, freq_base, freq_scale, + n_rot, rope_type, 0, n_orig_ctx, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); cb(Qcur, "Qcur", il); struct ggml_tensor * Kcur = ggml_rope_custom( ctx0, ggml_reshape_3d(ctx0, tmpk, n_embd_head, n_head_kv, n_tokens), inp_pos, - hparams.n_rot, 2, 0, n_orig_ctx, freq_base, freq_scale, + n_rot, rope_type, 0, n_orig_ctx, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); cb(Kcur, "Kcur", il); @@ -7080,11 +7096,6 @@ struct llm_build_context { struct ggml_tensor * KQ_mask = ggml_view_2d(ctx0, lctx.inp_KQ_mask, n_kv, n_tokens, n_kv*ggml_type_size(lctx.inp_KQ_mask->type), 0); cb(KQ_mask, "KQ_mask", -1); - // shift the entire K-cache if needed - if (do_rope_shift) { - llm_build_k_shift(ctx0, hparams, cparams, kv_self, gf, lctx.inp_K_shift, LLM_ROPE, n_ctx, freq_base, freq_scale, cb); - } - for (int il = 0; il < n_layer; ++il) { struct ggml_tensor * inpSA = inpL; @@ -7120,14 +7131,14 @@ struct llm_build_context { Qcur = ggml_rope_custom( ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, - hparams.n_rot, 2, 0, n_orig_ctx, freq_base, freq_scale, + n_rot, rope_type, 0, n_orig_ctx, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); cb(Qcur, "Qcur", il); Kcur = ggml_rope_custom( ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, - hparams.n_rot, 2, 0, n_orig_ctx, freq_base, freq_scale, + n_rot, rope_type, 0, n_orig_ctx, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); cb(Kcur, "Kcur", il); @@ -7199,11 +7210,6 @@ struct llm_build_context { struct ggml_tensor * KQ_mask = ggml_view_2d(ctx0, lctx.inp_KQ_mask, n_kv, n_tokens, n_kv*ggml_type_size(lctx.inp_KQ_mask->type), 0); cb(KQ_mask, "KQ_mask", -1); - // shift the entire K-cache if needed - if (do_rope_shift) { - llm_build_k_shift(ctx0, hparams, cparams, kv_self, gf, lctx.inp_K_shift, LLM_ROPE, n_ctx, freq_base, freq_scale, cb); - } - for (int il = 0; il < n_layer; ++il) { struct ggml_tensor * inpSA = inpL; @@ -7239,14 +7245,14 @@ struct llm_build_context { Qcur = ggml_rope_custom( ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, - hparams.n_rot, 0, 0, n_orig_ctx, freq_base, freq_scale, + n_rot, rope_type, 0, n_orig_ctx, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); cb(Qcur, "Qcur", il); Kcur = ggml_rope_custom( ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, - hparams.n_rot, 0, 0, n_orig_ctx, freq_base, freq_scale, + n_rot, rope_type, 0, n_orig_ctx, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); cb(Kcur, "Kcur", il); @@ -7331,11 +7337,6 @@ struct llm_build_context { struct ggml_tensor * KQ_mask = ggml_view_2d(ctx0, lctx.inp_KQ_mask, n_kv, n_tokens, n_kv*ggml_type_size(lctx.inp_KQ_mask->type), 0); cb(KQ_mask, "KQ_mask", -1); - // shift the entire K-cache if needed - if (do_rope_shift) { - llm_build_k_shift(ctx0, hparams, cparams, kv_self, gf, lctx.inp_K_shift, LLM_ROPE, n_ctx, freq_base, freq_scale, cb); - } - for (int il = 0; il < n_layer; ++il) { struct ggml_tensor * inpSA = inpL; @@ -7371,14 +7372,14 @@ struct llm_build_context { Qcur = ggml_rope_custom( ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, - hparams.n_rot, 0, 0, n_orig_ctx, freq_base, freq_scale, + n_rot, rope_type, 0, n_orig_ctx, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); cb(Qcur, "Qcur", il); Kcur = ggml_rope_custom( ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, - hparams.n_rot, 0, 0, n_orig_ctx, freq_base, freq_scale, + n_rot, rope_type, 0, n_orig_ctx, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); cb(Kcur, "Kcur", il); @@ -7467,11 +7468,6 @@ struct llm_build_context { struct ggml_tensor * KQ_mask = ggml_view_2d(ctx0, lctx.inp_KQ_mask, n_kv, n_tokens, n_kv*ggml_type_size(lctx.inp_KQ_mask->type), 0); cb(KQ_mask, "KQ_mask", -1); - // shift the entire K-cache if needed - if (do_rope_shift) { - llm_build_k_shift(ctx0, hparams, cparams, kv_self, gf, lctx.inp_K_shift, LLM_ROPE, n_ctx, freq_base, freq_scale, cb); - } - for (int il = 0; il < n_layer; ++il) { // norm @@ -7494,7 +7490,7 @@ struct llm_build_context { Qcur = ggml_rope_custom( ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head_k, n_head, n_tokens), inp_pos, - n_embd_head_k, 2, 0, n_orig_ctx, freq_base, freq_scale, + n_embd_head_k, rope_type, 0, n_orig_ctx, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow); cb(Qcur, "Qcur", il); @@ -7503,7 +7499,7 @@ struct llm_build_context { Kcur = ggml_rope_custom( ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head_k, n_head_kv, n_tokens), inp_pos, - n_embd_head_k, 2, 0, n_orig_ctx, freq_base, freq_scale, + n_embd_head_k, rope_type, 0, n_orig_ctx, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow); cb(Kcur, "Kcur", il); @@ -7556,6 +7552,40 @@ struct llm_build_context { } }; +static struct ggml_cgraph * llama_build_graph_defrag(llama_context & lctx, const std::vector & ids) { + llama_batch dummy; + dummy.n_tokens = 0; + + llm_build_cb cb = [&](struct ggml_tensor * , const char * , int ) { }; + + struct llm_build_context llm(lctx, dummy, cb, false); + + llm.init(); + + struct ggml_cgraph * result = llm.build_defrag(ids); + + llm.free(); + + return result; +} + +static struct ggml_cgraph * llama_build_graph_k_shift(llama_context & lctx) { + llama_batch dummy; + dummy.n_tokens = 0; + + llm_build_cb cb = [&](struct ggml_tensor * , const char * , int ) { }; + + struct llm_build_context llm(lctx, dummy, cb, false); + + llm.init(); + + struct ggml_cgraph * result = llm.build_k_shift(); + + llm.free(); + + return result; +} + static struct ggml_cgraph * llama_build_graph( llama_context & lctx, const llama_batch & batch, @@ -7675,6 +7705,20 @@ static struct ggml_cgraph * llama_build_graph( return result; } +static void llama_set_k_shift(llama_context & lctx) { + const auto & cparams = lctx.cparams; + + const int64_t n_ctx = cparams.n_ctx; + + assert(ggml_backend_buffer_is_host(lctx.inp_K_shift->buffer)); + + int32_t * data = (int32_t *) lctx.inp_K_shift->data; + + for (int i = 0; i < n_ctx; ++i) { + data[i] = lctx.kv_self.cells[i].delta; + } +} + static void llama_set_inputs(llama_context & lctx, const llama_batch & batch) { // // set input data @@ -7742,18 +7786,6 @@ static void llama_set_inputs(llama_context & lctx, const llama_batch & batch) { } } - if (kv_self.has_shift) { - const int64_t n_ctx = cparams.n_ctx; - - assert(ggml_backend_buffer_is_host(lctx.inp_K_shift->buffer)); - - int32_t * data = (int32_t *) lctx.inp_K_shift->data; - - for (int i = 0; i < n_ctx; ++i) { - data[i] = lctx.kv_self.cells[i].delta; - } - } - if (cparams.do_pooling && hparams.pooling_type == LLAMA_POOLING_TYPE_MEAN) { const int64_t n_tokens = batch.n_tokens; @@ -7798,6 +7830,34 @@ static void llama_set_inputs(llama_context & lctx, const llama_batch & batch) { } } +static void llama_graph_compute( + llama_context & lctx, + ggml_cgraph * gf, + int n_threads) { +#ifdef GGML_USE_MPI + const int64_t n_layer = lctx.model.hparams.n_layer; + ggml_mpi_graph_compute_pre(lctx.ctx_mpi, gf, n_layer); +#endif + +#ifdef GGML_USE_METAL + if (ggml_backend_is_metal(lctx.backend_metal)) { + ggml_backend_metal_set_n_cb(lctx.backend_metal, n_threads); + } +#endif + + if (lctx.backend_cpu != nullptr) { + ggml_backend_cpu_set_n_threads(lctx.backend_cpu, n_threads); + } + + ggml_backend_sched_graph_compute(lctx.sched, gf); + + // fprintf(stderr, "splits: %d\n", ggml_backend_sched_get_n_splits(lctx.sched)); + +#ifdef GGML_USE_MPI + ggml_mpi_graph_compute_post(lctx.ctx_mpi, gf, n_layer); +#endif +} + // decode a batch of tokens by evaluating the transformer // // - lctx: llama context @@ -7893,14 +7953,17 @@ static int llama_decode_internal( //printf("kv_self.n = %5d, kv_self.used = %5d, kv_self.head = %5d\n", kv_self.n, kv_self.used, kv_self.head); + llama_kv_cache_update(&lctx); + ggml_backend_sched_reset(lctx.sched); ggml_backend_sched_set_eval_callback(lctx.sched, lctx.cparams.cb_eval, lctx.cparams.cb_eval_user_data); ggml_cgraph * gf = llama_build_graph(lctx, batch, false); // the output is always the last tensor in the graph - struct ggml_tensor * res = gf->nodes[gf->n_nodes - 1]; + struct ggml_tensor * res = gf->nodes[gf->n_nodes - 1]; struct ggml_tensor * embeddings = gf->nodes[gf->n_nodes - 2]; + if (strcmp(res->name, "result_output") == 0) { // the embeddings could be the second to last tensor, or the third to last tensor if (strcmp(embeddings->name, "result_norm") != 0) { @@ -7927,40 +7990,12 @@ static int llama_decode_internal( n_threads = std::min(4, n_threads); } -#ifdef GGML_USE_MPI - const int64_t n_layer = hparams.n_layer; - ggml_mpi_graph_compute_pre(lctx.ctx_mpi, gf, n_layer); -#endif - -#ifdef GGML_USE_METAL - if (ggml_backend_is_metal(lctx.backend_metal)) { - ggml_backend_metal_set_n_cb(lctx.backend_metal, n_threads); - } -#endif - - if (lctx.backend_cpu != nullptr) { - ggml_backend_cpu_set_n_threads(lctx.backend_cpu, n_threads); - } - llama_set_inputs(lctx, batch); - ggml_backend_sched_graph_compute(lctx.sched, gf); - - // fprintf(stderr, "splits: %d\n", ggml_backend_sched_get_n_splits(lctx.sched)); - -#ifdef GGML_USE_MPI - ggml_mpi_graph_compute_post(lctx.ctx_mpi, gf, n_layer); -#endif + llama_graph_compute(lctx, gf, n_threads); // update the kv ring buffer { - if (kv_self.has_shift) { - kv_self.has_shift = false; - for (uint32_t i = 0; i < kv_self.size; ++i) { - kv_self.cells[i].delta = 0; - } - } - kv_self.head += n_tokens; // Ensure kv cache head points to a valid index. @@ -8056,6 +8091,221 @@ static int llama_decode_internal( return 0; } +// find holes from the beginning of the KV cache and fill them by moving data from the end of the cache +static void llama_kv_cache_defrag_internal(struct llama_context & lctx) { + auto & kv_self = lctx.kv_self; + + const uint32_t n_kv = llama_kv_cache_cell_max(kv_self); + const uint32_t n_used = kv_self.used; + + assert(n_used <= n_kv); + + const int64_t t_start = ggml_time_us(); + + // number of cells moved + uint32_t n_moves = 0; + + // determine which KV cells to move where + // + // cell i moves to ids[i] + // + // if ids[i] == i || ids[i] == n_kv, then cell i is not moved + // + std::vector ids(n_kv, n_kv); + + for (uint32_t i0 = 0; i0 < n_used; ++i0) { + const auto & cell0 = kv_self.cells[i0]; + + if (!cell0.is_empty()) { + ids[i0] = i0; + + continue; + } + + // found a hole - fill it with data from the end of the cache + + // determine the size of the hole + uint32_t nh = 1; + while (i0 + nh < n_used && kv_self.cells[i0 + nh].is_empty()) { + nh++; + } + + // starting from the end, find nh non-empty cells + uint32_t nf = 0; + uint32_t is = n_kv - 1; + for (; is > i0; --is) { + const auto & cell1 = kv_self.cells[is]; + + if (cell1.is_empty() || ids[is] != n_kv) { + continue; + } + + // non-empty cell which is not yet moved + nf++; + + if (nf == nh) { + break; + } + } + + // this can only happen if `n_used` is not accurate, which would be a bug + GGML_ASSERT(nf == nh && "KV defrag bug: nf != nh"); + + nf = 0; + + // go back and move the nf cells to the hole + for (uint32_t i1 = is; i1 < n_kv; ++i1) { + const auto & cell1 = kv_self.cells[i1]; + + if (cell1.is_empty() || ids[i1] != n_kv) { + continue; + } + + // this cell goes to (i0 + nf) + ids[i1] = i0 + nf; + + // move the cell meta data + kv_self.cells[i0 + nf] = cell1; + + n_moves++; + nf++; + } + + LLAMA_LOG_INFO("(tmp log) KV defrag: move [%u, %u) to [%u, %u)\n", is, n_kv, i0, i0 + nh); + + i0 += nh - 1; + } + + if (n_moves == 0) { + return; + } + + LLAMA_LOG_INFO("(tmp log) KV defrag cell moves: %u\n", n_moves); + + kv_self.head = n_used; + kv_self.used = n_used; + + // zero the rest of the cells + for (uint32_t i = n_used; i < n_kv; ++i) { + kv_self.cells[i] = llama_kv_cell(); + } + +#if 0 + // CPU defrag + // + // TODO: optimizations are possible: + // - multiple threads + // - avoid copying to the host memory when already there + // + // likely not worth the effort, as we have ggml_graph based defrag + // + + const auto & hparams = lctx.model.hparams; + + const uint32_t n_layer = hparams.n_layer; + const uint32_t n_embd_k_gqa = hparams.n_embd_k_gqa(); + const uint32_t n_embd_v_gqa = hparams.n_embd_v_gqa(); + + const uint32_t kv_size = kv_self.size; + + std::vector buf_k; + std::vector buf_v; + + for (uint32_t il = 0; il < n_layer; ++il) { + const size_t k_size_row = ggml_row_size(kv_self.k_l[il]->type, n_embd_k_gqa); + const size_t k_size = ggml_row_size(kv_self.k_l[il]->type, n_embd_k_gqa*kv_size); + + const size_t v_size_el = ggml_type_size(kv_self.v_l[il]->type); + const size_t v_size = ggml_row_size (kv_self.v_l[il]->type, n_embd_v_gqa*kv_size); + + buf_k.resize(k_size); + buf_v.resize(v_size); + + ggml_backend_tensor_get(kv_self.k_l[il], buf_k.data(), 0, buf_k.size()); + ggml_backend_tensor_get(kv_self.v_l[il], buf_v.data(), 0, buf_v.size()); + + // batch move [i, i+nm) to [id, id+nm) + // note: cells can move only to a lower index + for (uint32_t i = 0; i < n_kv; ++i) { + const uint32_t id = ids[i]; + + if (i == id || id == n_kv) { + continue; + } + + uint32_t nm = 1; + + while (i + nm < n_kv && ids[i + nm] == id + nm) { + nm++; + } + + // move keys + { + const int64_t os = i*k_size_row; + const int64_t od = id*k_size_row; + + memcpy(buf_k.data() + od, buf_k.data() + os, nm*k_size_row); + } + + // move values (note: they are transposed) + { + const int64_t os = i; + const int64_t od = id; + + for (uint32_t j = 0; j < n_embd_v_gqa; ++j) { + memcpy(buf_v.data() + (od + j*kv_size)*v_size_el, buf_v.data() + (os + j*kv_size)*v_size_el, nm*v_size_el); + } + } + + i += nm - 1; + } + + ggml_backend_tensor_set(kv_self.k_l[il], buf_k.data(), 0, buf_k.size()); + ggml_backend_tensor_set(kv_self.v_l[il], buf_v.data(), 0, buf_v.size()); + } +#else + // ggml_graph defrag + + ggml_cgraph * gf = llama_build_graph_defrag(lctx, ids); + + llama_graph_compute(lctx, gf, lctx.cparams.n_threads); +#endif + + const int64_t t_end = ggml_time_us(); + + LLAMA_LOG_INFO("(tmp log) KV defrag time: %.3f ms\n", (t_end - t_start)/1000.0); +} + +static void llama_kv_cache_update_internal(struct llama_context & lctx) { + // apply K-shift if needed + if (lctx.model.hparams.rope_type != LLAMA_ROPE_TYPE_NONE && lctx.kv_self.has_shift) { + llama_set_k_shift(lctx); + + { + ggml_cgraph * gf = llama_build_graph_k_shift(lctx); + + llama_graph_compute(lctx, gf, lctx.cparams.n_threads); + } + + { + auto & kv_self = lctx.kv_self; + + kv_self.has_shift = false; + + for (uint32_t i = 0; i < kv_self.size; ++i) { + kv_self.cells[i].delta = 0; + } + } + } + + // defragment the KV cache if needed + if (lctx.kv_self.do_defrag) { + llama_kv_cache_defrag_internal(lctx); + + lctx.kv_self.do_defrag = false; + } +} + // // tokenizer // @@ -11671,8 +11921,7 @@ struct llama_context * llama_new_context_with_model( } ctx->backends.push_back(ctx->backend_cpu); - if (!llama_kv_cache_init(ctx->kv_self, ctx->model, type_k, type_v, - cparams.n_ctx, cparams.offload_kqv)) { + if (!llama_kv_cache_init(ctx->kv_self, ctx->model, type_k, type_v, cparams.n_ctx, cparams.offload_kqv)) { LLAMA_LOG_ERROR("%s: llama_kv_cache_init() failed for self-attention cache\n", __func__); llama_free(ctx); return nullptr; @@ -11820,6 +12069,49 @@ enum llama_vocab_type llama_vocab_type(const struct llama_model * model) { return model->vocab.type; } +enum llama_rope_type llama_rope_type(const struct llama_model * model) { + switch (model->arch) { + // these models do not use RoPE + case LLM_ARCH_GPT2: + case LLM_ARCH_GPTJ: + case LLM_ARCH_GPTNEOX: + case LLM_ARCH_MPT: + case LLM_ARCH_REFACT: + case LLM_ARCH_BLOOM: + return LLAMA_ROPE_TYPE_NONE; + + // use what we call a normal RoPE, operating on pairs of consecutive head values + case LLM_ARCH_LLAMA: + case LLM_ARCH_BAICHUAN: + case LLM_ARCH_STARCODER: + case LLM_ARCH_PLAMO: + case LLM_ARCH_CODESHELL: + case LLM_ARCH_ORION: + case LLM_ARCH_INTERNLM2: + case LLM_ARCH_MINICPM: + case LLM_ARCH_GEMMA: + return LLAMA_ROPE_TYPE_NORM; + + // the pairs of head values are offset by n_rot/2 + case LLM_ARCH_FALCON: + case LLM_ARCH_PERSIMMON: + case LLM_ARCH_BERT: + case LLM_ARCH_NOMIC_BERT: + case LLM_ARCH_STABLELM: + case LLM_ARCH_QWEN: + case LLM_ARCH_QWEN2: + case LLM_ARCH_PHI2: + return LLAMA_ROPE_TYPE_NEOX; + + // all model arches should be listed explicitly here + case LLM_ARCH_UNKNOWN: + GGML_ASSERT(false && "unknown architecture"); + break; + } + + return LLAMA_ROPE_TYPE_NONE; +} + int32_t llama_n_vocab(const struct llama_model * model) { return model->vocab.id_to_token.size(); } @@ -12062,12 +12354,12 @@ void llama_kv_cache_seq_keep(struct llama_context * ctx, llama_seq_id seq_id) { llama_kv_cache_seq_keep(ctx->kv_self, seq_id); } -void llama_kv_cache_seq_shift(struct llama_context * ctx, llama_seq_id seq_id, llama_pos p0, llama_pos p1, llama_pos delta) { +void llama_kv_cache_seq_add(struct llama_context * ctx, llama_seq_id seq_id, llama_pos p0, llama_pos p1, llama_pos delta) { if (delta == 0) { return; } - llama_kv_cache_seq_shift(ctx->kv_self, seq_id, p0, p1, delta); + llama_kv_cache_seq_add(ctx->kv_self, seq_id, p0, p1, delta); } void llama_kv_cache_seq_div(struct llama_context * ctx, llama_seq_id seq_id, llama_pos p0, llama_pos p1, int d) { @@ -12078,6 +12370,19 @@ void llama_kv_cache_seq_div(struct llama_context * ctx, llama_seq_id seq_id, lla llama_kv_cache_seq_div(ctx->kv_self, seq_id, p0, p1, d); } +llama_pos llama_kv_cache_seq_pos_max(struct llama_context * ctx, llama_seq_id seq_id) { + return llama_kv_cache_seq_pos_max(ctx->kv_self, seq_id); +} + +void llama_kv_cache_defrag(struct llama_context * ctx) { + llama_kv_cache_defrag(ctx->kv_self); +} + +void llama_kv_cache_update(struct llama_context * ctx) { + llama_kv_cache_update_internal(*ctx); +} + + // Returns the *maximum* size of the state size_t llama_get_state_size(const struct llama_context * ctx) { // we don't know size of rng until we actually serialize it. so reserve more than enough memory for its serialized state. @@ -12204,10 +12509,10 @@ static void llama_copy_state_data_internal(struct llama_context * ctx, llama_dat const auto & hparams = ctx->model.hparams; const auto & cparams = ctx->cparams; - const auto n_layer = hparams.n_layer; - const auto n_embd_k_gqa = hparams.n_embd_k_gqa(); - const auto n_embd_v_gqa = hparams.n_embd_v_gqa(); - const auto n_ctx = cparams.n_ctx; + const uint32_t n_layer = hparams.n_layer; + const uint32_t n_embd_k_gqa = hparams.n_embd_k_gqa(); + const uint32_t n_embd_v_gqa = hparams.n_embd_v_gqa(); + const uint32_t n_ctx = cparams.n_ctx; const size_t kv_buf_size = kv_self.total_size(); const uint32_t kv_head = kv_self.head; @@ -12222,14 +12527,16 @@ static void llama_copy_state_data_internal(struct llama_context * ctx, llama_dat if (kv_buf_size) { std::vector tmp_buf; for (int il = 0; il < (int) n_layer; ++il) { - size_t k_size = ggml_row_size(kv_self.k_l[il]->type, n_embd_k_gqa*kv_head); + const size_t k_size = ggml_row_size(kv_self.k_l[il]->type, n_embd_k_gqa*kv_head); + tmp_buf.resize(k_size); ggml_backend_tensor_get(kv_self.k_l[il], tmp_buf.data(), 0, tmp_buf.size()); data_ctx->write(tmp_buf.data(), tmp_buf.size()); // v is not contiguous, copy row by row - size_t v_row_size = ggml_row_size(kv_self.v_l[il]->type, kv_head); - size_t v_row_stride = ggml_row_size(kv_self.v_l[il]->type, n_ctx); + const size_t v_row_size = ggml_row_size(kv_self.v_l[il]->type, kv_head); + const size_t v_row_stride = ggml_row_size(kv_self.v_l[il]->type, n_ctx); + tmp_buf.resize(v_row_size); for (int ir = 0; ir < (int) n_embd_v_gqa; ++ir) { ggml_backend_tensor_get(kv_self.v_l[il], tmp_buf.data(), ir*v_row_stride, tmp_buf.size()); @@ -12316,10 +12623,10 @@ size_t llama_set_state_data(struct llama_context * ctx, uint8_t * src) { const auto & hparams = ctx->model.hparams; const auto & cparams = ctx->cparams; - const int n_layer = hparams.n_layer; - const int n_embd_k_gqa = hparams.n_embd_k_gqa(); - const int n_embd_v_gqa = hparams.n_embd_v_gqa(); - const int n_ctx = cparams.n_ctx; + const uint32_t n_layer = hparams.n_layer; + const uint32_t n_embd_k_gqa = hparams.n_embd_k_gqa(); + const uint32_t n_embd_v_gqa = hparams.n_embd_v_gqa(); + const uint32_t n_ctx = cparams.n_ctx; size_t kv_buf_size; uint32_t kv_head; @@ -12335,13 +12642,15 @@ size_t llama_set_state_data(struct llama_context * ctx, uint8_t * src) { GGML_ASSERT(kv_self.total_size() == kv_buf_size); for (int il = 0; il < (int) n_layer; ++il) { - size_t k_size = ggml_row_size(kv_self.k_l[il]->type, n_embd_k_gqa*kv_head); + const size_t k_size = ggml_row_size(kv_self.k_l[il]->type, n_embd_k_gqa*kv_head); + ggml_backend_tensor_set(kv_self.k_l[il], inp, 0, k_size); inp += k_size; // v is not contiguous, copy row by row - size_t v_row_size = ggml_row_size(kv_self.v_l[il]->type, kv_head); - size_t v_row_stride = ggml_row_size(kv_self.v_l[il]->type, n_ctx); + const size_t v_row_size = ggml_row_size(kv_self.v_l[il]->type, kv_head); + const size_t v_row_stride = ggml_row_size(kv_self.v_l[il]->type, n_ctx); + for (int ir = 0; ir < (int) n_embd_v_gqa; ++ir) { ggml_backend_tensor_set(kv_self.v_l[il], inp, ir*v_row_stride, v_row_size); inp += v_row_size; diff --git a/llama.h b/llama.h index 947284ea2..ff131996d 100644 --- a/llama.h +++ b/llama.h @@ -64,6 +64,15 @@ extern "C" { LLAMA_VOCAB_TYPE_WPM = 2, // WordPiece }; + // note: these values should be synchronized with ggml_rope + // TODO: maybe move this enum to ggml.h (ggml_rope_type) + enum llama_rope_type { + LLAMA_ROPE_TYPE_NONE = -1, + LLAMA_ROPE_TYPE_NORM = 0, + LLAMA_ROPE_TYPE_NEOX = 2, + LLAMA_ROPE_TYPE_GLM = 4, + }; + enum llama_token_type { LLAMA_TOKEN_TYPE_UNDEFINED = 0, LLAMA_TOKEN_TYPE_NORMAL = 1, @@ -360,6 +369,7 @@ extern "C" { LLAMA_API uint32_t llama_n_batch (const struct llama_context * ctx); LLAMA_API enum llama_vocab_type llama_vocab_type(const struct llama_model * model); + LLAMA_API enum llama_rope_type llama_rope_type (const struct llama_model * model); LLAMA_API int32_t llama_n_vocab (const struct llama_model * model); LLAMA_API int32_t llama_n_ctx_train(const struct llama_model * model); @@ -514,10 +524,12 @@ extern "C" { llama_seq_id seq_id); // Adds relative position "delta" to all tokens that belong to the specified sequence and have positions in [p0, p1) - // If the KV cache is RoPEd, the KV data is updated accordingly + // If the KV cache is RoPEd, the KV data is updated accordingly: + // - lazily on next llama_decode() + // - explicitly with llama_kv_cache_update() // p0 < 0 : [0, p1] // p1 < 0 : [p0, inf) - LLAMA_API void llama_kv_cache_seq_shift( + LLAMA_API void llama_kv_cache_seq_add( struct llama_context * ctx, llama_seq_id seq_id, llama_pos p0, @@ -525,7 +537,9 @@ extern "C" { llama_pos delta); // Integer division of the positions by factor of `d > 1` - // If the KV cache is RoPEd, the KV data is updated accordingly + // If the KV cache is RoPEd, the KV data is updated accordingly: + // - lazily on next llama_decode() + // - explicitly with llama_kv_cache_update() // p0 < 0 : [0, p1] // p1 < 0 : [p0, inf) LLAMA_API void llama_kv_cache_seq_div( @@ -535,6 +549,20 @@ extern "C" { llama_pos p1, int d); + // Returns the largest position present in the KV cache for the specified sequence + LLAMA_API llama_pos llama_kv_cache_seq_pos_max( + struct llama_context * ctx, + llama_seq_id seq_id); + + // Defragment the KV cache + // This will be applied: + // - lazily on next llama_decode() + // - explicitly with llama_kv_cache_update() + LLAMA_API void llama_kv_cache_defrag(struct llama_context * ctx); + + // Apply the KV cache updates (such as K-shifts, defragmentation, etc.) + LLAMA_API void llama_kv_cache_update(struct llama_context * ctx); + // // State / sessions // From 8b350356b28f782deab63d8b0e9ae103ceb25fcd Mon Sep 17 00:00:00 2001 From: Pierrick Hymbert Date: Sun, 25 Feb 2024 21:46:29 +0100 Subject: [PATCH 461/812] server: docs - refresh and tease a little bit more the http server (#5718) * server: docs - refresh and tease a little bit more the http server * Rephrase README.md server doc Co-authored-by: Georgi Gerganov * Update examples/server/README.md Co-authored-by: Georgi Gerganov * Update examples/server/README.md Co-authored-by: Georgi Gerganov * Update README.md --------- Co-authored-by: Georgi Gerganov --- README.md | 3 +++ examples/server/README.md | 18 +++++++++++++++--- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index d61f9171b..d0af5d0b9 100644 --- a/README.md +++ b/README.md @@ -114,6 +114,9 @@ Typically finetunes of the base models below are supported as well. - [x] [MobileVLM 1.7B/3B models](https://huggingface.co/models?search=mobileVLM) - [x] [Yi-VL](https://huggingface.co/models?search=Yi-VL) +**HTTP server** + +[llama.cpp web server](./examples/server) is a lightweight [OpenAI API](https://github.com/openai/openai-openapi) compatible HTTP server that can be used to serve local models and easily connect them to existing clients. **Bindings:** diff --git a/examples/server/README.md b/examples/server/README.md index cb3fd6054..0e9bd7fd4 100644 --- a/examples/server/README.md +++ b/examples/server/README.md @@ -1,8 +1,20 @@ -# llama.cpp/example/server +# LLaMA.cpp HTTP Server -This example demonstrates a simple HTTP API server and a simple web front end to interact with llama.cpp. +Fast, lightweight, pure C/C++ HTTP server based on [httplib](https://github.com/yhirose/cpp-httplib), [nlohmann::json](https://github.com/nlohmann/json) and **llama.cpp**. -Command line options: +Set of LLM REST APIs and a simple web front end to interact with llama.cpp. + +**Features:** + * LLM inference of F16 and quantum models on GPU and CPU + * [OpenAI API](https://github.com/openai/openai-openapi) compatible chat completions and embeddings routes + * Parallel decoding with multi-user support + * Continuous batching + * Multimodal (wip) + * Monitoring endpoints + +The project is under active development, and we are [looking for feedback and contributors](https://github.com/ggerganov/llama.cpp/issues/4216). + +**Command line options:** - `--threads N`, `-t N`: Set the number of threads to use during generation. - `-tb N, --threads-batch N`: Set the number of threads to use during batch and prompt processing. If not specified, the number of threads will be set to the number of threads used for generation. From e3965cf35aac00d4e24998c8a3d0093ae1d98bd3 Mon Sep 17 00:00:00 2001 From: Pierrick Hymbert Date: Sun, 25 Feb 2024 22:48:33 +0100 Subject: [PATCH 462/812] server: tests - slow inference causes timeout on the CI (#5715) * server: tests - longer inference timeout for CI --- common/sampling.cpp | 2 +- examples/server/tests/features/steps/steps.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/common/sampling.cpp b/common/sampling.cpp index de4331a11..e67096bea 100644 --- a/common/sampling.cpp +++ b/common/sampling.cpp @@ -266,7 +266,7 @@ static llama_token llama_sampling_sample_impl( // } //} - LOG("sampled token: %5d: '%s'\n", id, llama_token_to_piece(ctx_main, id).c_str()); + //LOG("sampled token: %5d: '%s'\n", id, llama_token_to_piece(ctx_main, id).c_str()); } } diff --git a/examples/server/tests/features/steps/steps.py b/examples/server/tests/features/steps/steps.py index 8e4babf20..ad87fcb82 100644 --- a/examples/server/tests/features/steps/steps.py +++ b/examples/server/tests/features/steps/steps.py @@ -699,6 +699,8 @@ async def wait_for_health_status(context, if context.debug: print(f"Starting checking for health for expected_health_status={expected_health_status}") timeout = 3 # seconds + if expected_health_status == 'ok': + timeout = 10 # CI slow inference interval = 0.5 counter = 0 async with aiohttp.ClientSession() as session: @@ -736,7 +738,7 @@ async def wait_for_health_status(context, if n_completions > 0: return - assert False, 'timeout exceeded' + assert False, f'{expected_health_status} timeout exceeded {counter}s>={timeout}' def assert_embeddings(embeddings): From c39373398803c669056304090050fe3f44b41bf9 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Sun, 25 Feb 2024 00:17:11 +0000 Subject: [PATCH 463/812] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/5863c27340ba4de8f83e7e3c023b9599c3cb3c80' (2024-02-16) → 'github:NixOS/nixpkgs/cbc4211f0afffe6dfd2478a62615dd5175a13f9a' (2024-02-23) --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index 47d6448b5..9f659ba8f 100644 --- a/flake.lock +++ b/flake.lock @@ -20,11 +20,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1708118438, - "narHash": "sha256-kk9/0nuVgA220FcqH/D2xaN6uGyHp/zoxPNUmPCMmEE=", + "lastModified": 1708655239, + "narHash": "sha256-ZrP/yACUvDB+zbqYJsln4iwotbH6CTZiTkANJ0AgDv4=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "5863c27340ba4de8f83e7e3c023b9599c3cb3c80", + "rev": "cbc4211f0afffe6dfd2478a62615dd5175a13f9a", "type": "github" }, "original": { From 269de86ba073b5dc9ce687c11a3bc4d7d873b962 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Mon, 26 Feb 2024 08:30:17 +0200 Subject: [PATCH 464/812] llama : fix Gemma rope type (#5691) --- llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llama.cpp b/llama.cpp index 3424b1999..28430254f 100644 --- a/llama.cpp +++ b/llama.cpp @@ -12089,7 +12089,6 @@ enum llama_rope_type llama_rope_type(const struct llama_model * model) { case LLM_ARCH_ORION: case LLM_ARCH_INTERNLM2: case LLM_ARCH_MINICPM: - case LLM_ARCH_GEMMA: return LLAMA_ROPE_TYPE_NORM; // the pairs of head values are offset by n_rot/2 @@ -12101,6 +12100,7 @@ enum llama_rope_type llama_rope_type(const struct llama_model * model) { case LLM_ARCH_QWEN: case LLM_ARCH_QWEN2: case LLM_ARCH_PHI2: + case LLM_ARCH_GEMMA: return LLAMA_ROPE_TYPE_NEOX; // all model arches should be listed explicitly here From 8a533f0d9078396ebaee9ba213038a1322976dee Mon Sep 17 00:00:00 2001 From: Pierrick Hymbert Date: Mon, 26 Feb 2024 09:56:10 +0100 Subject: [PATCH 465/812] server: CI tests reduce build matrix (#5725) --- .github/workflows/server.yml | 78 ++++++++---------------------------- 1 file changed, 17 insertions(+), 61 deletions(-) diff --git a/.github/workflows/server.yml b/.github/workflows/server.yml index ed27dc528..1211ba128 100644 --- a/.github/workflows/server.yml +++ b/.github/workflows/server.yml @@ -6,11 +6,10 @@ on: push: branches: - master - - test/server-add-ci-test # FIXME remove - paths: ['.github/workflows/**', '**/CMakeLists.txt', '**/Makefile', '**/*.h', '**/*.hpp', '**/*.c', '**/*.cpp', '**/*.cu', '**/*.swift', '**/*.m', 'examples/server/**.*'] + paths: ['.github/workflows/server.yml', '**/CMakeLists.txt', '**/Makefile', '**/*.h', '**/*.hpp', '**/*.c', '**/*.cpp', '**/*.cu', '**/*.swift', '**/*.m', 'examples/server/tests/**.*'] pull_request: types: [opened, synchronize, reopened] - paths: ['**/CMakeLists.txt', '**/Makefile', '**/*.h', '**/*.hpp', '**/*.c', '**/*.cpp', '**/*.cu', '**/*.swift', '**/*.m', 'examples/server/**.*'] + paths: ['.github/workflows/server.yml', '**/CMakeLists.txt', '**/Makefile', '**/*.h', '**/*.hpp', '**/*.c', '**/*.cpp', '**/*.cu', '**/*.swift', '**/*.m', 'examples/server/tests/**.*'] jobs: server: @@ -18,45 +17,21 @@ jobs: strategy: matrix: - build: [noavx, avx2, avx, avx512, cublas, clblast, openblas, kompute, vulkan] sanitizer: [ADDRESS, THREAD, UNDEFINED] build_type: [Debug, Release] include: - - build: 'noavx' - defines: '-DLLAMA_NATIVE=OFF -DLLAMA_BUILD_SERVER=ON -DLLAMA_AVX=OFF -DLLAMA_AVX2=OFF -DLLAMA_FMA=OFF' - image: ubuntu:latest - - build: 'avx2' - defines: '-DLLAMA_NATIVE=OFF -DLLAMA_BUILD_SERVER=ON' - image: ubuntu:latest - - build: 'avx' - defines: '-DLLAMA_NATIVE=OFF -DLLAMA_BUILD_SERVER=ON -DLLAMA_AVX2=OFF' - image: ubuntu:latest - - build: 'avx512' - defines: '-DLLAMA_NATIVE=OFF -DLLAMA_BUILD_SERVER=ON -DLLAMA_AVX512=ON' - image: ubuntu:latest - experimental: true - - build: 'cublas' - defines: '-DLLAMA_NATIVE=OFF -DLLAMA_BUILD_SERVER=ON -DLLAMA_CUBLAS=ON' - image: nvidia/cuda:12.3.1-devel-ubuntu22.04 - arch_not_available: true # require nvidia docker engine - - build: 'clblast' - defines: '-DLLAMA_NATIVE=OFF -DLLAMA_BUILD_SERVER=ON -DLLAMA_CLBLAST=ON' - image: ubuntu:latest - arch_not_available: true - - build: 'openblas' - defines: '-DLLAMA_NATIVE=OFF -DLLAMA_BUILD_SERVER=ON -DLLAMA_BLAS=ON -DLLAMA_BLAS_VENDOR=OpenBLAS' - image: ubuntu:latest - - build: 'kompute' - defines: '-DLLAMA_NATIVE=OFF -DLLAMA_BUILD_SERVER=ON -DLLAMA_KOMPUTE=ON -DKOMPUTE_OPT_DISABLE_VULKAN_VERSION_CHECK=ON' - image: ubuntu:latest - arch_not_available: true - - build: 'vulkan' - defines: '-DLLAMA_NATIVE=OFF -DLLAMA_BUILD_SERVER=ON -DLLAMA_VULKAN=ON' - image: ubuntu:latest - arch_not_available: true + - build_type: Release + sanitizer: "" + exclude: + - build_type: Release + sanitizer: ADDRESS + - build_type: Release + sanitizer: THREAD + - build_type: Release + sanitizer: UNDEFINED container: - image: ${{ matrix.image }} + image: ubuntu:latest ports: - 8888 options: --cpus 4 @@ -72,40 +47,22 @@ jobs: apt-get update apt-get -y install \ build-essential \ - pkg-config \ git \ cmake \ python3-pip \ wget \ psmisc - - name: Download CLBlast - id: get_clblast - if: ${{ matrix.build == 'clblast' }} - run: | - apt install -y libclblast-dev - - - name: Download OpenBLAS - id: get_openblas - if: ${{ matrix.build == 'openblas' }} - run: | - apt-get -y install libopenblas-dev - - - name: Install Vulkan SDK - id: get_vulkan - if: ${{ matrix.build == 'kompute' || matrix.build == 'vulkan' }} - run: | - wget -qO- https://packages.lunarg.com/lunarg-signing-key-pub.asc | tee /etc/apt/trusted.gpg.d/lunarg.asc - wget -qO /etc/apt/sources.list.d/lunarg-vulkan-jammy.list http://packages.lunarg.com/vulkan/lunarg-vulkan-jammy.list - apt-get update - apt-get -y install vulkan-sdk - - name: Build id: cmake_build run: | mkdir build cd build - cmake .. -DLLAMA_SANITIZE_${{ matrix.sanitizer }}=ON -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} ${{ matrix.defines }} + cmake .. \ + -DLLAMA_NATIVE=OFF \ + -DLLAMA_BUILD_SERVER=ON \ + -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} \ + -DLLAMA_SANITIZE_${{ matrix.sanitizer }}=ON ; cmake --build . --config ${{ matrix.build_type }} -j $(nproc) --target server - name: Tests dependencies @@ -121,7 +78,6 @@ jobs: - name: Tests id: server_integration_test - continue-on-error: ${{ matrix.experimental || matrix.arch_not_available }} run: | cd examples/server/tests PORT=8888 ./tests.sh From 4804215cb833841ffb15a710a16b77ca0a29eb4b Mon Sep 17 00:00:00 2001 From: Pierrick Hymbert Date: Mon, 26 Feb 2024 11:41:34 +0100 Subject: [PATCH 466/812] server: CI fix trailing space (#5728) --- .github/workflows/server.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/server.yml b/.github/workflows/server.yml index 1211ba128..0b6f6669b 100644 --- a/.github/workflows/server.yml +++ b/.github/workflows/server.yml @@ -62,7 +62,7 @@ jobs: -DLLAMA_NATIVE=OFF \ -DLLAMA_BUILD_SERVER=ON \ -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} \ - -DLLAMA_SANITIZE_${{ matrix.sanitizer }}=ON ; + -DLLAMA_SANITIZE_${{ matrix.sanitizer }}=ON ; cmake --build . --config ${{ matrix.build_type }} -j $(nproc) --target server - name: Tests dependencies From 67fd33132fab93e6c2087bd6fa656a8a57419efa Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Mon, 26 Feb 2024 14:02:12 +0200 Subject: [PATCH 467/812] unicode : reuse iterator (#5726) --- unicode.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/unicode.h b/unicode.h index 263260702..10a5dab01 100644 --- a/unicode.h +++ b/unicode.h @@ -404,7 +404,8 @@ static std::unordered_map codepoint_type_map() { static int codepoint_type(uint32_t cp) { static std::unordered_map codepoint_types = codepoint_type_map(); - return codepoint_types.find(cp) == codepoint_types.end() ? CODEPOINT_TYPE_UNIDENTIFIED : codepoint_types.at(cp); + const auto it = codepoint_types.find(cp); + return it == codepoint_types.end() ? CODEPOINT_TYPE_UNIDENTIFIED : it->second; } static int codepoint_type(const std::string & utf8) { From e849078c6e09e72fdd2c95ba61f5fba9a7b2d9ef Mon Sep 17 00:00:00 2001 From: AidanBeltonS <87009434+AidanBeltonS@users.noreply.github.com> Date: Mon, 26 Feb 2024 14:02:11 +0000 Subject: [PATCH 468/812] [SYCL] Add support for soft_max ALiBi (#5639) * Add support for bias * Update pre-processor * rm commented code * fix format * fix CI --------- Co-authored-by: Abhilash Majumder <30946547+abhilash1910@users.noreply.github.com> --- ggml-sycl.cpp | 248 +++++++++++++++++++++++++++++++++----------------- llama.cpp | 4 +- 2 files changed, 168 insertions(+), 84 deletions(-) diff --git a/ggml-sycl.cpp b/ggml-sycl.cpp index c6c3c6e6f..835967fb6 100644 --- a/ggml-sycl.cpp +++ b/ggml-sycl.cpp @@ -8126,23 +8126,51 @@ static void diag_mask_inf_f32(const float * x, float * dst, const int ncols, con dst[i] = x[i] - (col > n_past + row % rows_per_channel) * FLT_MAX; } -static void soft_max_f32(const float * x, const float * y, float * dst, const int ncols, const int nrows_y, const float scale, - const sycl::nd_item<3> &item_ct1, float *buf) { + +template +static void soft_max_f32(const float * x, const float * mask, const float *pos, float * dst, const int ncols_par, + const int nrows_y, const float scale, const float max_bias, const float m0, + const float m1, uint32_t n_head_log2, const sycl::nd_item<3> &item_ct1, float *buf) { + const int ncols = ncols_template == 0 ? ncols_par : ncols_template; + const int tid = item_ct1.get_local_id(2); const int rowx = item_ct1.get_group(2); const int rowy = rowx % nrows_y; // broadcast the mask (y) in the row dimension - const int block_size = item_ct1.get_local_range(2); + const int block_size = block_size_template == 0 ? item_ct1.get_local_range(2) : block_size_template; const int warp_id = item_ct1.get_local_id(2) / WARP_SIZE; const int lane_id = item_ct1.get_local_id(2) % WARP_SIZE; + float slope = 0.0f; + + // ALiBi + if (max_bias > 0.0f) { + const uint32_t h = rowx/nrows_y; // head index + + const float base = h < n_head_log2 ? m0 : m1; + const int exp = h < n_head_log2 ? h + 1 : 2*(h - n_head_log2) + 1; + + slope = sycl::pow(base, float(exp)); + } + + float * vals = vals_smem ? buf + WARP_SIZE : dst + rowx*ncols; float max_val = -INFINITY; - for (int col = tid; col < ncols; col += block_size) { + for (int col0 = 0; col0 < ncols; col0 += block_size) { + const int col = col0 + tid; + + if (ncols_template == 0 && col >= ncols) { + break; + } + const int ix = rowx*ncols + col; const int iy = rowy*ncols + col; - max_val = sycl::max(max_val, x[ix] * scale + (y ? y[iy] : 0.0f)); + + const float val = x[ix]*scale + (mask ? mask[iy] : 0.0f) + (pos ? slope*pos[col] : 0.0f); + + vals[col] = val; + max_val = sycl::max(max_val, val); } // find the max value in the block @@ -8151,30 +8179,12 @@ static void soft_max_f32(const float * x, const float * y, float * dst, const in if (warp_id == 0) { buf[lane_id] = -INFINITY; } - /* - DPCT1118:12: SYCL group functions and algorithms must be encountered in - converged control flow. You may need to adjust the code. - */ - /* - DPCT1065:60: Consider replacing sycl::nd_item::barrier() with - sycl::nd_item::barrier(sycl::access::fence_space::local_space) for - better performance if there is no access to global memory. - */ - item_ct1.barrier(); + item_ct1.barrier(sycl::access::fence_space::local_space); if (lane_id == 0) { buf[warp_id] = max_val; } - /* - DPCT1118:13: SYCL group functions and algorithms must be encountered in - converged control flow. You may need to adjust the code. - */ - /* - DPCT1065:61: Consider replacing sycl::nd_item::barrier() with - sycl::nd_item::barrier(sycl::access::fence_space::local_space) for - better performance if there is no access to global memory. - */ - item_ct1.barrier(); + item_ct1.barrier(sycl::access::fence_space::local_space); max_val = buf[lane_id]; max_val = warp_reduce_max(max_val, item_ct1); @@ -8182,13 +8192,16 @@ static void soft_max_f32(const float * x, const float * y, float * dst, const in float tmp = 0.f; - for (int col = tid; col < ncols; col += block_size) { - const int ix = rowx*ncols + col; - const int iy = rowy*ncols + col; - const float val = - sycl::native::exp((x[ix] * scale + (y ? y[iy] : 0.0f)) - max_val); +#pragma unroll + for (int col0 = 0; col0 < ncols; col0 += block_size) { + const int col = col0 + tid; + if (ncols_template == 0 && col >= ncols) { + break; + } + + const float val = sycl::native::exp(vals[col] - max_val); tmp += val; - dst[ix] = val; + vals[col] = val; } // find the sum of exps in the block @@ -8197,40 +8210,29 @@ static void soft_max_f32(const float * x, const float * y, float * dst, const in if (warp_id == 0) { buf[lane_id] = 0.f; } - /* - DPCT1118:14: SYCL group functions and algorithms must be encountered in - converged control flow. You may need to adjust the code. - */ - /* - DPCT1065:62: Consider replacing sycl::nd_item::barrier() with - sycl::nd_item::barrier(sycl::access::fence_space::local_space) for - better performance if there is no access to global memory. - */ - item_ct1.barrier(); + item_ct1.barrier(sycl::access::fence_space::local_space); if (lane_id == 0) { buf[warp_id] = tmp; } - /* - DPCT1118:15: SYCL group functions and algorithms must be encountered in - converged control flow. You may need to adjust the code. - */ - /* - DPCT1065:63: Consider replacing sycl::nd_item::barrier() with - sycl::nd_item::barrier(sycl::access::fence_space::local_space) for - better performance if there is no access to global memory. - */ - item_ct1.barrier(); + item_ct1.barrier(sycl::access::fence_space::local_space); tmp = buf[lane_id]; tmp = warp_reduce_sum(tmp, item_ct1); } - const float inv_tmp = 1.f / tmp; + const float inv_sum = 1.f / tmp; - for (int col = tid; col < ncols; col += block_size) { - const int i = rowx*ncols + col; - dst[i] *= inv_tmp; +#pragma unroll + for (int col0 = 0; col0 < ncols; col0 += block_size) { + const int col = col0 + tid; + + if (ncols_template == 0 && col >= ncols) { + return; + } + + const int idst = rowx*ncols + col; + dst[idst] = vals[col] * inv_sum; } } @@ -10867,35 +10869,96 @@ static void diag_mask_inf_f32_sycl(const float *x, float *dst, }); } -static void soft_max_f32_sycl(const float *x, const float *y, float *dst, - const int ncols_x, const int nrows_x, - const int nrows_y, const float scale, +template +static void soft_max_f32_submitter(const float * x, const float * mask, const float *pos, float * dst, const int ncols_par, + const int nrows_y, const float scale, const float max_bias, const float m0, + const float m1, uint32_t n_head_log2, sycl::range<3> block_nums, sycl::range<3> block_dims, + const size_t n_local_scratch, dpct::queue_ptr stream) { + stream->submit([&](sycl::handler &cgh) { + sycl::local_accessor local_buf_acc(n_local_scratch, cgh); + + cgh.parallel_for( + sycl::nd_range<3>(block_nums * block_dims, block_dims), + [=](sycl::nd_item<3> item_ct1) [[intel::reqd_sub_group_size(32)]] { + soft_max_f32(x, mask, pos, dst, ncols_par, + nrows_y, scale, max_bias, m0, + m1, n_head_log2, item_ct1, + local_buf_acc.get_pointer()); + }); + }); +} + +static void soft_max_f32_sycl(const float * x, const float * mask, const float * pos, + float * dst, const int ncols_x, const int nrows_x, + const int nrows_y, const float scale, const float max_bias, dpct::queue_ptr stream) { int nth = WARP_SIZE; while (nth < ncols_x && nth < SYCL_SOFT_MAX_BLOCK_SIZE) nth *= 2; const sycl::range<3> block_dims(1, 1, nth); const sycl::range<3> block_nums(1, 1, nrows_x); - /* - DPCT1049:46: The work-group size passed to the SYCL kernel may exceed the - limit. To get the device limit, query info::device::max_work_group_size. - Adjust the work-group size if needed. - */ - stream->submit([&](sycl::handler &cgh) { - /* - DPCT1101:96: 'SYCL_SOFT_MAX_BLOCK_SIZE/WARP_SIZE' expression was - replaced with a value. Modify the code to use the original expression, - provided in comments, if it is correct. - */ - sycl::local_accessor buf_acc_ct1( - sycl::range<1>(32 /*SYCL_SOFT_MAX_BLOCK_SIZE/WARP_SIZE*/), cgh); + const size_t n_local_scratch = (GGML_PAD(ncols_x, WARP_SIZE) + WARP_SIZE); + static_assert(SYCL_SOFT_MAX_BLOCK_SIZE == 1024, "These values need to be adjusted."); - cgh.parallel_for( - sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) [[intel::reqd_sub_group_size(32)]] { - soft_max_f32(x, y, dst, ncols_x, nrows_y, scale, item_ct1, - buf_acc_ct1.get_pointer()); - }); - }); + const uint32_t n_head_kv = nrows_x/nrows_y; + const uint32_t n_head_log2 = 1u << (uint32_t) floorf(log2f((float) n_head_kv)); + + const float m0 = powf(2.0f, -(max_bias ) / n_head_log2); + const float m1 = powf(2.0f, -(max_bias / 2.0f) / n_head_log2); + + const size_t local_mem_size = stream->get_device().get_info(); + if (n_local_scratch*sizeof(float) < local_mem_size) { + switch (ncols_x) { + case 32: + soft_max_f32_submitter(x, mask, pos, dst, ncols_x, nrows_y, scale, + max_bias, m0, m1, n_head_log2, block_nums, + block_dims, n_local_scratch, stream); + break; + case 64: + soft_max_f32_submitter(x, mask, pos, dst, ncols_x, nrows_y, scale, + max_bias, m0, m1, n_head_log2, block_nums, + block_dims, n_local_scratch, stream); + break; + case 128: + soft_max_f32_submitter(x, mask, pos, dst, ncols_x, nrows_y, scale, + max_bias, m0, m1, n_head_log2, block_nums, + block_dims, n_local_scratch, stream); + break; + case 256: + soft_max_f32_submitter(x, mask, pos, dst, ncols_x, nrows_y, scale, + max_bias, m0, m1, n_head_log2, block_nums, + block_dims, n_local_scratch, stream); + break; + case 512: + soft_max_f32_submitter(x, mask, pos, dst, ncols_x, nrows_y, scale, + max_bias, m0, m1, n_head_log2, block_nums, + block_dims, n_local_scratch, stream); + break; + case 1024: + soft_max_f32_submitter(x, mask, pos, dst, ncols_x, nrows_y, scale, + max_bias, m0, m1, n_head_log2, block_nums, + block_dims, n_local_scratch, stream); + break; + case 2048: + soft_max_f32_submitter(x, mask, pos, dst, ncols_x, nrows_y, scale, + max_bias, m0, m1, n_head_log2, block_nums, + block_dims, n_local_scratch, stream); + break; + case 4096: + soft_max_f32_submitter(x, mask, pos, dst, ncols_x, nrows_y, scale, + max_bias, m0, m1, n_head_log2, block_nums, + block_dims, n_local_scratch, stream); + break; + default: + soft_max_f32_submitter(x, mask, pos, dst, ncols_x, nrows_y, scale, + max_bias, m0, m1, n_head_log2, block_nums, + block_dims, n_local_scratch, stream); + break; + } + } else { + soft_max_f32_submitter(x, mask, pos, dst, ncols_x, nrows_y, scale, + max_bias, m0, m1, n_head_log2, block_nums, + block_dims, WARP_SIZE, stream); + } } template @@ -12435,14 +12498,35 @@ inline void ggml_sycl_op_soft_max(const ggml_tensor *src0, const int64_t ne00 = src0->ne[0]; const int64_t nrows_x = ggml_nrows(src0); - const int64_t nrows_y = src1 ? ggml_nrows(src1) : 1; + const int64_t nrows_y = src0->ne[1]; float scale = 1.0f; - memcpy(&scale, dst->op_params, sizeof(float)); + float max_bias = 0.0f; - soft_max_f32_sycl(src0_dd, src1 ? src1_dd : nullptr, dst_dd, ne00, nrows_x, nrows_y, scale, main_stream); + memcpy(&scale, dst->op_params + 0, sizeof(float)); + memcpy(&max_bias, dst->op_params + 1, sizeof(float)); - (void) dst; + // positions tensor + float * src2_dd = nullptr; + sycl_pool_alloc src2_f; + + ggml_tensor * src2 = dst->src[2]; + const bool use_src2 = src2 != nullptr; + + if (use_src2) { + const bool src2_on_device = src2->backend == GGML_BACKEND_TYPE_GPU; + + if (src2_on_device) { + ggml_tensor_extra_gpu * src2_extra = (ggml_tensor_extra_gpu *) src2->extra; + src2_dd = (float *) src2_extra->data_device[g_main_device]; + } else { + src2_dd = src2_f.alloc(ggml_nelements(src2)); + SYCL_CHECK(ggml_sycl_cpy_tensor_2d(src2_dd, src2, 0, 0, 0, 1, main_stream)); + } + } + + soft_max_f32_sycl(src0_dd, src1 ? src1_dd : nullptr, src2_dd, dst_dd, ne00, + nrows_x, nrows_y, scale, max_bias, main_stream); } inline void ggml_sycl_op_scale(const ggml_tensor *src0, const ggml_tensor *src1, diff --git a/llama.cpp b/llama.cpp index 28430254f..f549e7d04 100644 --- a/llama.cpp +++ b/llama.cpp @@ -4894,8 +4894,8 @@ static struct ggml_tensor * llm_build_kqv( ggml_mul_mat_set_prec(kq, GGML_PREC_F32); } -#if defined(GGML_USE_VULKAN) || defined(GGML_USE_KOMPUTE) || defined(GGML_USE_SYCL) -#pragma message("TODO: ALiBi support in ggml_soft_max_ext is not implemented for Vulkan, Kompute, and SYCL") +#if defined(GGML_USE_VULKAN) || defined(GGML_USE_KOMPUTE) +#pragma message("TODO: ALiBi support in ggml_soft_max_ext is not implemented for Vulkan, and Kompute") #pragma message(" Falling back to ggml_alibi(). Will become an error in Mar 2024") #pragma message("ref: https://github.com/ggerganov/llama.cpp/pull/5488") if (hparams.f_max_alibi_bias > 0.0f) { From c4d7f8178608440506e5489bae0109e4ca12e44a Mon Sep 17 00:00:00 2001 From: Artem Date: Mon, 26 Feb 2024 17:15:28 +0300 Subject: [PATCH 469/812] readme : update ui list (#5731) * Add LLMFarm (ui for iOS) to list --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index d0af5d0b9..507a2888b 100644 --- a/README.md +++ b/README.md @@ -159,6 +159,7 @@ Unless otherwise noted these projects are open-source with permissive licensing: - [withcatai/catai](https://github.com/withcatai/catai) - [Mobile-Artificial-Intelligence/maid](https://github.com/Mobile-Artificial-Intelligence/maid) (MIT) - [Msty](https://msty.app) (proprietary) +- [LLMFarm](https://github.com/guinmoon/LLMFarm?tab=readme-ov-file) (MIT) --- From 47bb7b48c7cec9d8f57d56812ce811ec130b89a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20G=C3=A4=C3=9Fler?= Date: Mon, 26 Feb 2024 15:36:38 +0100 Subject: [PATCH 470/812] CUDA: fix DEBUG_CUDA_MALLOC (#5729) --- ggml-cuda.cu | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ggml-cuda.cu b/ggml-cuda.cu index fb6d4f7d2..15322fb59 100644 --- a/ggml-cuda.cu +++ b/ggml-cuda.cu @@ -8079,8 +8079,8 @@ static void * ggml_cuda_pool_malloc_leg(int device, size_t size, size_t * actual *actual_size = look_ahead_size; g_cuda_pool_size[device] += look_ahead_size; #ifdef DEBUG_CUDA_MALLOC - fprintf(stderr, "%s[%d]: %d buffers, max_size = %u MB, pool_size = %u MB, requested %u MB\n", __func__, id, nnz, - (uint32_t)(max_size/1024/1024), (uint32_t)(g_cuda_pool_size[id]/1024/1024), (uint32_t)(size/1024/1024)); + fprintf(stderr, "%s[%d]: %d buffers, max_size = %u MB, pool_size = %u MB, requested %u MB\n", __func__, device, nnz, + (uint32_t)(max_size/1024/1024), (uint32_t)(g_cuda_pool_size[device]/1024/1024), (uint32_t)(size/1024/1024)); #endif return ptr; } @@ -8166,7 +8166,7 @@ static void * ggml_cuda_pool_malloc_vmm(int device, size_t size, size_t * actual g_cuda_pool_used[device] += size; #ifdef DEBUG_CUDA_MALLOC - printf("cuda pool[%d]: allocated %llu bytes at %llx [%s]\n", id, (unsigned long long) size, ptr); + printf("cuda pool[%d]: allocated %llu bytes at %llx\n", device, (unsigned long long) size, ptr); #endif return ptr; @@ -8176,7 +8176,7 @@ static void ggml_cuda_pool_free_vmm(int device, void * ptr, size_t size) { scoped_spin_lock lock(g_cuda_pool_lock); #ifdef DEBUG_CUDA_MALLOC - printf("cuda pool[%d]: freed %llu bytes at %llx\n", id, (unsigned long long) size, ptr); + printf("cuda pool[%d]: freed %llu bytes at %llx\n", device, (unsigned long long) size, ptr); #endif g_cuda_pool_used[device] -= size; From a33e6a0d2a66104ea9a906bdbf8a94d050189d91 Mon Sep 17 00:00:00 2001 From: Kawrakow <48489457+ikawrakow@users.noreply.github.com> Date: Mon, 26 Feb 2024 18:28:38 +0200 Subject: [PATCH 471/812] Adding IQ2_S and IQ2_M to complete coverage of the 2-3 bit quantization range (#5721) * Adding IQ2_S and IQ2_M as a single cumulative commit * Update examples/quantize/quantize.cpp Co-authored-by: Georgi Gerganov --------- Co-authored-by: Iwan Kawrakow Co-authored-by: Georgi Gerganov --- examples/quantize/quantize.cpp | 7 +- ggml-cuda.cu | 358 ++++++++++++++- ggml-metal.m | 37 +- ggml-metal.metal | 487 +++++++++++++++++++++ ggml-quants.c | 775 ++++++++++++++++++++++++++++++++- ggml-quants.h | 14 + ggml.c | 31 ++ ggml.h | 2 + llama.cpp | 71 ++- llama.h | 4 +- tests/test-backend-ops.cpp | 2 +- tests/test-quantize-fns.cpp | 4 +- 12 files changed, 1754 insertions(+), 38 deletions(-) diff --git a/examples/quantize/quantize.cpp b/examples/quantize/quantize.cpp index ab7e72aaf..2d187823f 100644 --- a/examples/quantize/quantize.cpp +++ b/examples/quantize/quantize.cpp @@ -23,14 +23,16 @@ static const std::vector QUANT_OPTIONS = { { "Q5_1", LLAMA_FTYPE_MOSTLY_Q5_1, " 4.70G, +0.0349 ppl @ LLaMA-v1-7B", }, { "IQ2_XXS",LLAMA_FTYPE_MOSTLY_IQ2_XXS," 2.06 bpw quantization", }, { "IQ2_XS", LLAMA_FTYPE_MOSTLY_IQ2_XS, " 2.31 bpw quantization", }, + { "IQ2_S", LLAMA_FTYPE_MOSTLY_IQ2_S, " 2.5 bpw quantization", }, + { "IQ2_M", LLAMA_FTYPE_MOSTLY_IQ2_M, " 2.7 bpw quantization", }, { "IQ1_S", LLAMA_FTYPE_MOSTLY_IQ1_S, " 1.56 bpw quantization", }, { "Q2_K", LLAMA_FTYPE_MOSTLY_Q2_K, " 2.63G, +0.6717 ppl @ LLaMA-v1-7B", }, { "Q2_K_S", LLAMA_FTYPE_MOSTLY_Q2_K_S, " 2.16G, +9.0634 ppl @ LLaMA-v1-7B", }, { "IQ3_XXS",LLAMA_FTYPE_MOSTLY_IQ3_XXS," 3.06 bpw quantization", }, { "IQ3_S", LLAMA_FTYPE_MOSTLY_IQ3_S, " 3.44 bpw quantization", }, - { "IQ3_M", LLAMA_FTYPE_MOSTLY_IQ3_M, " 3.66 bpw quantization mix", }, + { "IQ3_M", LLAMA_FTYPE_MOSTLY_IQ3_M, " 3.66 bpw quantization mix", }, { "Q3_K", LLAMA_FTYPE_MOSTLY_Q3_K_M, "alias for Q3_K_M" }, - { "Q3_K_XS",LLAMA_FTYPE_MOSTLY_Q3_K_XS,"3-bit extra small quantization" , }, + { "IQ3_XS", LLAMA_FTYPE_MOSTLY_IQ3_XS, " 3.3 bpw quantization" , }, { "Q3_K_S", LLAMA_FTYPE_MOSTLY_Q3_K_S, " 2.75G, +0.5551 ppl @ LLaMA-v1-7B", }, { "Q3_K_M", LLAMA_FTYPE_MOSTLY_Q3_K_M, " 3.07G, +0.2496 ppl @ LLaMA-v1-7B", }, { "Q3_K_L", LLAMA_FTYPE_MOSTLY_Q3_K_L, " 3.35G, +0.1764 ppl @ LLaMA-v1-7B", }, @@ -292,6 +294,7 @@ int main(int argc, char ** argv) { } if ((params.ftype == LLAMA_FTYPE_MOSTLY_IQ2_XS || params.ftype == LLAMA_FTYPE_MOSTLY_IQ2_XXS || + params.ftype == LLAMA_FTYPE_MOSTLY_IQ2_S || params.ftype == LLAMA_FTYPE_MOSTLY_Q2_K_S || params.ftype == LLAMA_FTYPE_MOSTLY_IQ1_S) && imatrix_data.empty()) { fprintf(stderr, "\n===============================================================================================\n"); fprintf(stderr, "Please do not use IQ1_S, IQ2_XXS, IQ2_XS or Q2_K_S quantization without an importance matrix\n"); diff --git a/ggml-cuda.cu b/ggml-cuda.cu index 15322fb59..964fb7351 100644 --- a/ggml-cuda.cu +++ b/ggml-cuda.cu @@ -523,6 +523,17 @@ typedef struct { } block_iq2_xs; static_assert(sizeof(block_iq2_xs) == sizeof(ggml_fp16_t) + QK_K/8*sizeof(uint16_t) + QK_K/32, "wrong iq2_xs block size/padding"); +// 2.5625 bpw quants +#define QR2_S 8 +#define QI2_S (QK_K / (4*QR2_S)) +typedef struct { + half d; + uint8_t qs[QK_K/4]; + uint8_t qh[QK_K/32]; + uint8_t scales[QK_K/32]; +} block_iq2_s; +static_assert(sizeof(block_iq2_s) == sizeof(ggml_fp16_t) + QK_K/4 + QK_K/16, "wrong iq2_s block size/padding"); + #define QR3_XXS 8 #define QI3_XXS (QK_K / (4*QR3_XXS)) typedef struct { @@ -1689,6 +1700,265 @@ static const __device__ uint64_t iq2xs_grid[512] = { 0x2b2b2b2b082b2b08, 0x2b2b2b2b082b2b2b, 0x2b2b2b2b2b190819, 0x2b2b2b2b2b2b2b2b, }; +static const __device__ uint64_t iq2s_grid[1024] = { + 0x0808080808080808, 0x080808080808082b, 0x0808080808081919, 0x0808080808082b08, + 0x0808080808082b2b, 0x0808080808190819, 0x0808080808191908, 0x080808080819192b, + 0x0808080808192b19, 0x08080808082b0808, 0x08080808082b082b, 0x08080808082b1919, + 0x08080808082b2b08, 0x0808080819080819, 0x0808080819081908, 0x080808081908192b, + 0x0808080819082b19, 0x0808080819190808, 0x080808081919082b, 0x0808080819191919, + 0x0808080819192b08, 0x08080808192b0819, 0x08080808192b1908, 0x08080808192b192b, + 0x08080808192b2b19, 0x080808082b080808, 0x080808082b08082b, 0x080808082b081919, + 0x080808082b082b08, 0x080808082b190819, 0x080808082b191908, 0x080808082b2b0808, + 0x080808082b2b1919, 0x080808082b2b2b2b, 0x0808081908080819, 0x0808081908081908, + 0x080808190808192b, 0x0808081908082b19, 0x0808081908190808, 0x080808190819082b, + 0x0808081908191919, 0x0808081908192b08, 0x08080819082b0819, 0x08080819082b1908, + 0x0808081919080808, 0x080808191908082b, 0x0808081919081919, 0x0808081919082b08, + 0x0808081919190819, 0x0808081919191908, 0x080808191919192b, 0x0808081919192b19, + 0x08080819192b0808, 0x08080819192b1919, 0x08080819192b2b08, 0x080808192b080819, + 0x080808192b081908, 0x080808192b190808, 0x080808192b19082b, 0x080808192b191919, + 0x080808192b2b0819, 0x080808192b2b1908, 0x0808082b08080808, 0x0808082b0808082b, + 0x0808082b08081919, 0x0808082b08082b08, 0x0808082b08190819, 0x0808082b08191908, + 0x0808082b082b0808, 0x0808082b082b2b2b, 0x0808082b19080819, 0x0808082b19081908, + 0x0808082b1908192b, 0x0808082b19082b19, 0x0808082b19190808, 0x0808082b19191919, + 0x0808082b2b080808, 0x0808082b2b081919, 0x0808082b2b082b2b, 0x0808082b2b191908, + 0x0808082b2b2b082b, 0x0808190808080819, 0x0808190808081908, 0x080819080808192b, + 0x0808190808082b19, 0x0808190808190808, 0x080819080819082b, 0x0808190808191919, + 0x0808190808192b08, 0x08081908082b0819, 0x08081908082b1908, 0x08081908082b192b, + 0x08081908082b2b19, 0x0808190819080808, 0x080819081908082b, 0x0808190819081919, + 0x0808190819082b08, 0x0808190819082b2b, 0x0808190819190819, 0x0808190819191908, + 0x080819081919192b, 0x0808190819192b19, 0x08081908192b0808, 0x08081908192b082b, + 0x08081908192b1919, 0x080819082b080819, 0x080819082b081908, 0x080819082b08192b, + 0x080819082b082b19, 0x080819082b190808, 0x080819082b191919, 0x080819082b192b08, + 0x080819082b2b0819, 0x080819082b2b1908, 0x0808191908080808, 0x080819190808082b, + 0x0808191908081919, 0x0808191908082b08, 0x0808191908082b2b, 0x0808191908190819, + 0x0808191908191908, 0x080819190819192b, 0x0808191908192b19, 0x08081919082b0808, + 0x08081919082b1919, 0x08081919082b2b08, 0x0808191919080819, 0x0808191919081908, + 0x080819191908192b, 0x0808191919082b19, 0x0808191919190808, 0x080819191919082b, + 0x0808191919191919, 0x0808191919192b08, 0x08081919192b0819, 0x08081919192b1908, + 0x080819192b080808, 0x080819192b08082b, 0x080819192b081919, 0x080819192b082b08, + 0x080819192b190819, 0x080819192b191908, 0x080819192b2b0808, 0x0808192b08080819, + 0x0808192b08081908, 0x0808192b0808192b, 0x0808192b08082b19, 0x0808192b08190808, + 0x0808192b08191919, 0x0808192b19080808, 0x0808192b19081919, 0x0808192b19082b08, + 0x0808192b19190819, 0x0808192b19191908, 0x0808192b192b0808, 0x0808192b2b080819, + 0x0808192b2b081908, 0x0808192b2b190808, 0x08082b0808080808, 0x08082b080808082b, + 0x08082b0808081919, 0x08082b0808082b08, 0x08082b0808190819, 0x08082b0808191908, + 0x08082b080819192b, 0x08082b0808192b19, 0x08082b08082b0808, 0x08082b08082b1919, + 0x08082b08082b2b2b, 0x08082b0819080819, 0x08082b0819081908, 0x08082b081908192b, + 0x08082b0819082b19, 0x08082b0819190808, 0x08082b081919082b, 0x08082b0819191919, + 0x08082b0819192b08, 0x08082b08192b0819, 0x08082b08192b1908, 0x08082b082b080808, + 0x08082b082b081919, 0x08082b082b191908, 0x08082b082b2b2b2b, 0x08082b1908080819, + 0x08082b1908081908, 0x08082b1908190808, 0x08082b190819082b, 0x08082b1908191919, + 0x08082b1908192b08, 0x08082b19082b0819, 0x08082b1919080808, 0x08082b1919081919, + 0x08082b1919082b08, 0x08082b1919190819, 0x08082b1919191908, 0x08082b19192b0808, + 0x08082b192b080819, 0x08082b192b190808, 0x08082b2b08080808, 0x08082b2b08190819, + 0x08082b2b08191908, 0x08082b2b082b082b, 0x08082b2b082b2b08, 0x08082b2b082b2b2b, + 0x08082b2b19190808, 0x08082b2b2b192b19, 0x0819080808080819, 0x0819080808081908, + 0x081908080808192b, 0x0819080808082b19, 0x0819080808190808, 0x081908080819082b, + 0x0819080808191919, 0x0819080808192b08, 0x08190808082b0819, 0x08190808082b1908, + 0x08190808082b192b, 0x0819080819080808, 0x081908081908082b, 0x0819080819081919, + 0x0819080819082b08, 0x0819080819190819, 0x0819080819191908, 0x081908081919192b, + 0x0819080819192b19, 0x08190808192b0808, 0x08190808192b082b, 0x08190808192b1919, + 0x08190808192b2b08, 0x081908082b080819, 0x081908082b081908, 0x081908082b08192b, + 0x081908082b190808, 0x081908082b191919, 0x081908082b192b08, 0x081908082b2b0819, + 0x081908082b2b1908, 0x0819081908080808, 0x081908190808082b, 0x0819081908081919, + 0x0819081908082b08, 0x0819081908082b2b, 0x0819081908190819, 0x0819081908191908, + 0x081908190819192b, 0x0819081908192b19, 0x08190819082b0808, 0x08190819082b082b, + 0x08190819082b1919, 0x08190819082b2b08, 0x0819081919080819, 0x0819081919081908, + 0x081908191908192b, 0x0819081919082b19, 0x0819081919190808, 0x081908191919082b, + 0x0819081919191919, 0x0819081919192b08, 0x08190819192b0819, 0x08190819192b1908, + 0x081908192b080808, 0x081908192b08082b, 0x081908192b081919, 0x081908192b082b08, + 0x081908192b190819, 0x081908192b191908, 0x0819082b08080819, 0x0819082b08081908, + 0x0819082b08082b19, 0x0819082b08190808, 0x0819082b08191919, 0x0819082b082b0819, + 0x0819082b082b1908, 0x0819082b19080808, 0x0819082b19081919, 0x0819082b19190819, + 0x0819082b19191908, 0x0819082b2b080819, 0x0819082b2b081908, 0x0819082b2b190808, + 0x0819190808080808, 0x081919080808082b, 0x0819190808081919, 0x0819190808082b08, + 0x0819190808190819, 0x0819190808191908, 0x081919080819192b, 0x0819190808192b19, + 0x08191908082b0808, 0x08191908082b1919, 0x08191908082b2b08, 0x0819190819080819, + 0x0819190819081908, 0x081919081908192b, 0x0819190819082b19, 0x0819190819190808, + 0x081919081919082b, 0x0819190819191919, 0x0819190819192b08, 0x08191908192b0819, + 0x08191908192b1908, 0x081919082b080808, 0x081919082b08082b, 0x081919082b081919, + 0x081919082b082b08, 0x081919082b190819, 0x081919082b191908, 0x081919082b2b0808, + 0x0819191908080819, 0x0819191908081908, 0x081919190808192b, 0x0819191908082b19, + 0x0819191908190808, 0x081919190819082b, 0x0819191908191919, 0x0819191908192b08, + 0x08191919082b0819, 0x08191919082b1908, 0x0819191919080808, 0x081919191908082b, + 0x0819191919081919, 0x0819191919082b08, 0x0819191919190819, 0x0819191919191908, + 0x08191919192b0808, 0x081919192b080819, 0x081919192b081908, 0x081919192b190808, + 0x0819192b08080808, 0x0819192b08081919, 0x0819192b08082b08, 0x0819192b08190819, + 0x0819192b08191908, 0x0819192b082b0808, 0x0819192b19080819, 0x0819192b19081908, + 0x0819192b19190808, 0x0819192b2b080808, 0x0819192b2b2b2b2b, 0x08192b0808080819, + 0x08192b0808081908, 0x08192b080808192b, 0x08192b0808082b19, 0x08192b0808190808, + 0x08192b0808191919, 0x08192b0808192b08, 0x08192b08082b0819, 0x08192b0819080808, + 0x08192b081908082b, 0x08192b0819081919, 0x08192b0819082b08, 0x08192b0819190819, + 0x08192b0819191908, 0x08192b08192b0808, 0x08192b082b080819, 0x08192b082b081908, + 0x08192b1908080808, 0x08192b190808082b, 0x08192b1908081919, 0x08192b1908082b08, + 0x08192b1908190819, 0x08192b1908191908, 0x08192b19082b0808, 0x08192b1919080819, + 0x08192b1919081908, 0x08192b1919190808, 0x08192b19192b2b19, 0x08192b192b2b082b, + 0x08192b2b08081908, 0x08192b2b08190808, 0x08192b2b19080808, 0x08192b2b1919192b, + 0x082b080808080808, 0x082b08080808082b, 0x082b080808081919, 0x082b080808082b08, + 0x082b080808190819, 0x082b080808191908, 0x082b08080819192b, 0x082b080808192b19, + 0x082b0808082b0808, 0x082b0808082b1919, 0x082b0808082b2b2b, 0x082b080819080819, + 0x082b080819081908, 0x082b080819190808, 0x082b08081919082b, 0x082b080819191919, + 0x082b0808192b1908, 0x082b08082b080808, 0x082b08082b082b2b, 0x082b08082b191908, + 0x082b08082b2b2b2b, 0x082b081908080819, 0x082b081908081908, 0x082b081908190808, + 0x082b08190819082b, 0x082b081908191919, 0x082b0819082b0819, 0x082b081919080808, + 0x082b08191908082b, 0x082b081919081919, 0x082b081919190819, 0x082b081919191908, + 0x082b0819192b0808, 0x082b08192b080819, 0x082b08192b081908, 0x082b08192b190808, + 0x082b082b08080808, 0x082b082b08082b2b, 0x082b082b082b082b, 0x082b082b082b2b08, + 0x082b082b082b2b2b, 0x082b082b19081908, 0x082b082b19190808, 0x082b082b2b082b08, + 0x082b082b2b082b2b, 0x082b082b2b2b2b08, 0x082b190808080819, 0x082b190808081908, + 0x082b19080808192b, 0x082b190808082b19, 0x082b190808190808, 0x082b190808191919, + 0x082b190808192b08, 0x082b1908082b0819, 0x082b1908082b1908, 0x082b190819080808, + 0x082b19081908082b, 0x082b190819081919, 0x082b190819082b08, 0x082b190819190819, + 0x082b190819191908, 0x082b1908192b0808, 0x082b19082b080819, 0x082b19082b081908, + 0x082b19082b190808, 0x082b191908080808, 0x082b191908081919, 0x082b191908082b08, + 0x082b191908190819, 0x082b191908191908, 0x082b1919082b0808, 0x082b191919080819, + 0x082b191919081908, 0x082b191919190808, 0x082b1919192b192b, 0x082b19192b080808, + 0x082b192b08080819, 0x082b192b08081908, 0x082b192b08190808, 0x082b192b19080808, + 0x082b192b19192b19, 0x082b2b0808080808, 0x082b2b0808081919, 0x082b2b0808190819, + 0x082b2b0808191908, 0x082b2b0819080819, 0x082b2b0819081908, 0x082b2b0819190808, + 0x082b2b082b082b2b, 0x082b2b082b2b2b2b, 0x082b2b1908080819, 0x082b2b1908081908, + 0x082b2b1908190808, 0x082b2b192b191919, 0x082b2b2b08082b2b, 0x082b2b2b082b082b, + 0x082b2b2b192b1908, 0x082b2b2b2b082b08, 0x082b2b2b2b082b2b, 0x1908080808080819, + 0x1908080808081908, 0x190808080808192b, 0x1908080808082b19, 0x1908080808190808, + 0x190808080819082b, 0x1908080808191919, 0x1908080808192b08, 0x1908080808192b2b, + 0x19080808082b0819, 0x19080808082b1908, 0x19080808082b192b, 0x1908080819080808, + 0x190808081908082b, 0x1908080819081919, 0x1908080819082b08, 0x1908080819082b2b, + 0x1908080819190819, 0x1908080819191908, 0x190808081919192b, 0x1908080819192b19, + 0x19080808192b0808, 0x19080808192b082b, 0x19080808192b1919, 0x190808082b080819, + 0x190808082b081908, 0x190808082b190808, 0x190808082b191919, 0x190808082b192b08, + 0x190808082b2b0819, 0x190808082b2b1908, 0x1908081908080808, 0x190808190808082b, + 0x1908081908081919, 0x1908081908082b08, 0x1908081908190819, 0x1908081908191908, + 0x190808190819192b, 0x1908081908192b19, 0x19080819082b0808, 0x19080819082b082b, + 0x19080819082b1919, 0x1908081919080819, 0x1908081919081908, 0x190808191908192b, + 0x1908081919082b19, 0x1908081919190808, 0x190808191919082b, 0x1908081919191919, + 0x1908081919192b08, 0x19080819192b0819, 0x19080819192b1908, 0x190808192b080808, + 0x190808192b08082b, 0x190808192b081919, 0x190808192b082b08, 0x190808192b190819, + 0x190808192b191908, 0x190808192b2b0808, 0x1908082b08080819, 0x1908082b08081908, + 0x1908082b08190808, 0x1908082b0819082b, 0x1908082b08191919, 0x1908082b08192b08, + 0x1908082b082b1908, 0x1908082b19080808, 0x1908082b19081919, 0x1908082b19082b08, + 0x1908082b19190819, 0x1908082b19191908, 0x1908082b192b0808, 0x1908082b2b080819, + 0x1908082b2b081908, 0x1908190808080808, 0x190819080808082b, 0x1908190808081919, + 0x1908190808082b08, 0x1908190808082b2b, 0x1908190808190819, 0x1908190808191908, + 0x190819080819192b, 0x1908190808192b19, 0x19081908082b0808, 0x19081908082b082b, + 0x19081908082b1919, 0x19081908082b2b08, 0x1908190819080819, 0x1908190819081908, + 0x190819081908192b, 0x1908190819082b19, 0x1908190819190808, 0x190819081919082b, + 0x1908190819191919, 0x1908190819192b08, 0x19081908192b0819, 0x19081908192b1908, + 0x190819082b080808, 0x190819082b08082b, 0x190819082b081919, 0x190819082b082b08, + 0x190819082b190819, 0x190819082b191908, 0x190819082b2b0808, 0x1908191908080819, + 0x1908191908081908, 0x190819190808192b, 0x1908191908082b19, 0x1908191908190808, + 0x190819190819082b, 0x1908191908191919, 0x1908191908192b08, 0x19081919082b0819, + 0x19081919082b1908, 0x1908191919080808, 0x190819191908082b, 0x1908191919081919, + 0x1908191919082b08, 0x1908191919190819, 0x1908191919191908, 0x19081919192b0808, + 0x19081919192b2b2b, 0x190819192b080819, 0x190819192b081908, 0x190819192b190808, + 0x1908192b08080808, 0x1908192b0808082b, 0x1908192b08081919, 0x1908192b08082b08, + 0x1908192b08190819, 0x1908192b08191908, 0x1908192b082b0808, 0x1908192b19080819, + 0x1908192b19081908, 0x1908192b19190808, 0x1908192b2b080808, 0x1908192b2b2b1919, + 0x19082b0808080819, 0x19082b0808081908, 0x19082b0808082b19, 0x19082b0808190808, + 0x19082b080819082b, 0x19082b0808191919, 0x19082b0808192b08, 0x19082b08082b0819, + 0x19082b08082b1908, 0x19082b0819080808, 0x19082b081908082b, 0x19082b0819081919, + 0x19082b0819082b08, 0x19082b0819190819, 0x19082b0819191908, 0x19082b08192b0808, + 0x19082b082b081908, 0x19082b082b190808, 0x19082b1908080808, 0x19082b190808082b, + 0x19082b1908081919, 0x19082b1908082b08, 0x19082b1908190819, 0x19082b1908191908, + 0x19082b19082b0808, 0x19082b1919080819, 0x19082b1919081908, 0x19082b1919190808, + 0x19082b192b080808, 0x19082b192b19192b, 0x19082b2b08080819, 0x19082b2b08081908, + 0x19082b2b08190808, 0x19082b2b19080808, 0x1919080808080808, 0x191908080808082b, + 0x1919080808081919, 0x1919080808082b08, 0x1919080808190819, 0x1919080808191908, + 0x191908080819192b, 0x1919080808192b19, 0x19190808082b0808, 0x19190808082b082b, + 0x19190808082b1919, 0x19190808082b2b08, 0x1919080819080819, 0x1919080819081908, + 0x191908081908192b, 0x1919080819082b19, 0x1919080819190808, 0x191908081919082b, + 0x1919080819191919, 0x1919080819192b08, 0x19190808192b0819, 0x19190808192b1908, + 0x191908082b080808, 0x191908082b08082b, 0x191908082b081919, 0x191908082b082b08, + 0x191908082b190819, 0x191908082b191908, 0x1919081908080819, 0x1919081908081908, + 0x191908190808192b, 0x1919081908082b19, 0x1919081908190808, 0x191908190819082b, + 0x1919081908191919, 0x1919081908192b08, 0x19190819082b0819, 0x19190819082b1908, + 0x1919081919080808, 0x191908191908082b, 0x1919081919081919, 0x1919081919082b08, + 0x1919081919190819, 0x1919081919191908, 0x19190819192b0808, 0x191908192b080819, + 0x191908192b081908, 0x191908192b190808, 0x1919082b08080808, 0x1919082b08081919, + 0x1919082b08082b08, 0x1919082b08190819, 0x1919082b08191908, 0x1919082b082b0808, + 0x1919082b19080819, 0x1919082b19081908, 0x1919082b19190808, 0x1919082b192b2b19, + 0x1919082b2b080808, 0x1919190808080819, 0x1919190808081908, 0x191919080808192b, + 0x1919190808082b19, 0x1919190808190808, 0x191919080819082b, 0x1919190808191919, + 0x1919190808192b08, 0x19191908082b0819, 0x19191908082b1908, 0x1919190819080808, + 0x191919081908082b, 0x1919190819081919, 0x1919190819082b08, 0x1919190819190819, + 0x1919190819191908, 0x19191908192b0808, 0x191919082b080819, 0x191919082b081908, + 0x191919082b190808, 0x1919191908080808, 0x191919190808082b, 0x1919191908081919, + 0x1919191908082b08, 0x1919191908190819, 0x1919191908191908, 0x19191919082b0808, + 0x1919191919080819, 0x1919191919081908, 0x1919191919190808, 0x191919192b080808, + 0x1919192b08080819, 0x1919192b08081908, 0x1919192b08190808, 0x1919192b082b192b, + 0x1919192b19080808, 0x19192b0808080808, 0x19192b080808082b, 0x19192b0808081919, + 0x19192b0808082b08, 0x19192b0808190819, 0x19192b0808191908, 0x19192b08082b0808, + 0x19192b0819080819, 0x19192b0819081908, 0x19192b0819190808, 0x19192b0819192b2b, + 0x19192b082b080808, 0x19192b1908080819, 0x19192b1908081908, 0x19192b1908190808, + 0x19192b1919080808, 0x19192b2b08080808, 0x19192b2b08192b19, 0x19192b2b2b081919, + 0x19192b2b2b2b2b08, 0x192b080808080819, 0x192b080808081908, 0x192b08080808192b, + 0x192b080808190808, 0x192b08080819082b, 0x192b080808191919, 0x192b080808192b08, + 0x192b0808082b0819, 0x192b0808082b1908, 0x192b080819080808, 0x192b080819081919, + 0x192b080819082b08, 0x192b080819190819, 0x192b080819191908, 0x192b0808192b0808, + 0x192b08082b081908, 0x192b08082b190808, 0x192b081908080808, 0x192b08190808082b, + 0x192b081908081919, 0x192b081908082b08, 0x192b081908190819, 0x192b081908191908, + 0x192b0819082b0808, 0x192b081919080819, 0x192b081919081908, 0x192b081919190808, + 0x192b08192b080808, 0x192b08192b192b19, 0x192b082b08081908, 0x192b082b08190808, + 0x192b082b19080808, 0x192b082b1919192b, 0x192b082b2b2b0819, 0x192b190808080808, + 0x192b190808081919, 0x192b190808082b08, 0x192b190808190819, 0x192b190808191908, + 0x192b1908082b0808, 0x192b190819080819, 0x192b190819081908, 0x192b190819190808, + 0x192b19082b080808, 0x192b191908080819, 0x192b191908081908, 0x192b191908190808, + 0x192b191919080808, 0x192b191919082b2b, 0x192b1919192b2b08, 0x192b19192b19082b, + 0x192b192b08080808, 0x192b192b2b191908, 0x192b2b0808080819, 0x192b2b0808081908, + 0x192b2b0808190808, 0x192b2b08192b1919, 0x192b2b082b192b08, 0x192b2b1908080808, + 0x192b2b19082b2b2b, 0x192b2b2b1908082b, 0x192b2b2b2b2b0819, 0x2b08080808080808, + 0x2b0808080808082b, 0x2b08080808081919, 0x2b08080808082b08, 0x2b08080808190819, + 0x2b08080808191908, 0x2b08080808192b19, 0x2b080808082b0808, 0x2b080808082b1919, + 0x2b08080819080819, 0x2b08080819081908, 0x2b08080819190808, 0x2b0808081919082b, + 0x2b08080819191919, 0x2b08080819192b08, 0x2b080808192b0819, 0x2b0808082b080808, + 0x2b0808082b081919, 0x2b0808082b190819, 0x2b0808082b191908, 0x2b08081908080819, + 0x2b08081908081908, 0x2b08081908082b19, 0x2b08081908190808, 0x2b0808190819082b, + 0x2b08081908191919, 0x2b08081908192b08, 0x2b080819082b0819, 0x2b080819082b1908, + 0x2b08081919080808, 0x2b0808191908082b, 0x2b08081919081919, 0x2b08081919082b08, + 0x2b08081919190819, 0x2b08081919191908, 0x2b0808192b080819, 0x2b0808192b081908, + 0x2b0808192b190808, 0x2b0808192b2b2b19, 0x2b08082b08080808, 0x2b08082b08081919, + 0x2b08082b08082b2b, 0x2b08082b08190819, 0x2b08082b08191908, 0x2b08082b19080819, + 0x2b08082b19081908, 0x2b08082b19190808, 0x2b08190808080819, 0x2b08190808081908, + 0x2b0819080808192b, 0x2b08190808082b19, 0x2b08190808190808, 0x2b0819080819082b, + 0x2b08190808191919, 0x2b08190808192b08, 0x2b081908082b0819, 0x2b08190819080808, + 0x2b0819081908082b, 0x2b08190819081919, 0x2b08190819082b08, 0x2b08190819190819, + 0x2b08190819191908, 0x2b081908192b0808, 0x2b0819082b080819, 0x2b0819082b081908, + 0x2b0819082b190808, 0x2b08191908080808, 0x2b0819190808082b, 0x2b08191908081919, + 0x2b08191908082b08, 0x2b08191908190819, 0x2b08191908191908, 0x2b081919082b0808, + 0x2b08191919080819, 0x2b08191919081908, 0x2b08191919190808, 0x2b0819192b080808, + 0x2b0819192b082b2b, 0x2b08192b08080819, 0x2b08192b08081908, 0x2b08192b08190808, + 0x2b08192b082b2b19, 0x2b08192b19080808, 0x2b082b0808080808, 0x2b082b0808081919, + 0x2b082b0808190819, 0x2b082b0808191908, 0x2b082b0819080819, 0x2b082b0819081908, + 0x2b082b0819190808, 0x2b082b082b2b082b, 0x2b082b1908080819, 0x2b082b1908081908, + 0x2b082b1919080808, 0x2b082b19192b1919, 0x2b082b2b082b082b, 0x2b082b2b19192b08, + 0x2b082b2b19192b2b, 0x2b082b2b2b08082b, 0x2b082b2b2b2b082b, 0x2b19080808080819, + 0x2b19080808081908, 0x2b19080808082b19, 0x2b19080808190808, 0x2b1908080819082b, + 0x2b19080808191919, 0x2b19080808192b08, 0x2b190808082b1908, 0x2b19080819080808, + 0x2b1908081908082b, 0x2b19080819081919, 0x2b19080819082b08, 0x2b19080819190819, + 0x2b19080819191908, 0x2b190808192b0808, 0x2b1908082b080819, 0x2b1908082b081908, + 0x2b1908082b190808, 0x2b19081908080808, 0x2b19081908081919, 0x2b19081908190819, + 0x2b19081908191908, 0x2b19081919080819, 0x2b19081919081908, 0x2b19081919190808, + 0x2b19081919192b2b, 0x2b19082b08080819, 0x2b19082b08081908, 0x2b19082b08190808, + 0x2b19082b19080808, 0x2b19082b2b2b192b, 0x2b19190808080808, 0x2b1919080808082b, + 0x2b19190808081919, 0x2b19190808082b08, 0x2b19190808190819, 0x2b19190808191908, + 0x2b191908082b0808, 0x2b19190819080819, 0x2b19190819081908, 0x2b19190819190808, + 0x2b1919082b080808, 0x2b1919082b19192b, 0x2b19191908080819, 0x2b19191908081908, + 0x2b19191908190808, 0x2b19191919080808, 0x2b1919192b192b08, 0x2b1919192b2b0819, + 0x2b19192b08080808, 0x2b19192b1908192b, 0x2b19192b192b1908, 0x2b192b0808080819, + 0x2b192b0808081908, 0x2b192b0808190808, 0x2b192b08082b192b, 0x2b192b0819080808, + 0x2b192b082b2b2b19, 0x2b192b1908080808, 0x2b192b1919082b19, 0x2b192b191919082b, + 0x2b192b2b2b190808, 0x2b2b080808080808, 0x2b2b080808081919, 0x2b2b080808082b2b, + 0x2b2b080808191908, 0x2b2b0808082b082b, 0x2b2b0808082b2b2b, 0x2b2b080819080819, + 0x2b2b080819081908, 0x2b2b080819190808, 0x2b2b08082b2b082b, 0x2b2b08082b2b2b2b, + 0x2b2b081919080808, 0x2b2b0819192b1919, 0x2b2b082b0808082b, 0x2b2b082b08082b2b, + 0x2b2b082b082b082b, 0x2b2b082b082b2b08, 0x2b2b082b082b2b2b, 0x2b2b082b2b08082b, + 0x2b2b082b2b082b08, 0x2b2b082b2b082b2b, 0x2b2b082b2b2b2b08, 0x2b2b190808080819, + 0x2b2b190808081908, 0x2b2b190808190808, 0x2b2b190819080808, 0x2b2b19082b082b19, + 0x2b2b19082b2b1908, 0x2b2b191908080808, 0x2b2b191908192b19, 0x2b2b192b19190819, + 0x2b2b2b0808082b2b, 0x2b2b2b08082b2b08, 0x2b2b2b082b2b082b, 0x2b2b2b1919191908, + 0x2b2b2b192b08192b, 0x2b2b2b2b08082b08, 0x2b2b2b2b08082b2b, 0x2b2b2b2b082b0808, + 0x2b2b2b2b082b082b, 0x2b2b2b2b082b2b08, 0x2b2b2b2b2b082b08, 0x2b2b2b2b2b2b2b2b, +}; + static const __device__ uint32_t iq3xxs_grid[256] = { 0x04040404, 0x04040414, 0x04040424, 0x04040c0c, 0x04040c1c, 0x04040c3e, 0x04041404, 0x04041414, 0x04041c0c, 0x04042414, 0x04043e1c, 0x04043e2c, 0x040c040c, 0x040c041c, 0x040c0c04, 0x040c0c14, @@ -2037,6 +2307,27 @@ static __global__ void dequantize_block_iq2_xs(const void * __restrict__ vx, dst } +template +static __global__ void dequantize_block_iq2_s(const void * __restrict__ vx, dst_t * __restrict__ yy) { + + const int i = blockIdx.x; + const block_iq2_s * x = (const block_iq2_s *) vx; + + const int tid = threadIdx.x; +#if QK_K == 256 + const int il = tid/8; // 0...3 + const int ib = tid%8; // 0...7 + dst_t * y = yy + i*QK_K + 32*ib + 8*il; + const uint8_t * grid = (const uint8_t *)(iq2s_grid + (x[i].qs[4*ib+il] | ((x[i].qh[ib] << (8-2*il)) & 0x300))); + const float d = (float)x[i].d * (0.5f + ((x[i].scales[ib] >> 4*(il/2)) & 0xf)) * 0.25f; + const uint8_t signs = x[i].qs[QK_K/8+4*ib+il]; + for (int j = 0; j < 8; ++j) y[j] = d * grid[j] * (signs & kmask_iq2xs[j] ? -1.f : 1.f); +#else + assert(false); +#endif + +} + template static __global__ void dequantize_block_iq3_xxs(const void * __restrict__ vx, dst_t * __restrict__ yy) { @@ -4800,6 +5091,54 @@ static __device__ __forceinline__ float vec_dot_iq2_xs_q8_1( #endif } +// TODO +static __device__ __forceinline__ float vec_dot_iq2_s_q8_1( + const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) { +#if __CUDA_ARCH__ >= MIN_CC_DP4A // lowest compute capability for integer intrinsics +#if QK_K == 256 + const block_iq2_s * bq2 = (const block_iq2_s *) vbq; + + const int ib32 = iqs; + const int8_t * q8 = bq8_1[ib32].qs; + const uint8_t * signs = bq2->qs + QK_K/8 + 4*ib32; + const uint8_t ls1 = bq2->scales[ib32] & 0xf; + const uint8_t ls2 = bq2->scales[ib32] >> 4; + int sumi1 = 0; + for (int l = 0; l < 2; ++l) { + const uint32_t * grid = (const uint32_t *)(iq2s_grid + (bq2->qs[4*ib32+l] | ((bq2->qh[ib32] << (8-2*l)) & 0x300))); + const uint32_t signs0 = __vcmpeq4(((signs[l] & 0xf) * 0x01010101) & 0x08040201, 0x08040201); + const uint32_t signs1 = __vcmpeq4(((signs[l] >> 4) * 0x01010101) & 0x08040201, 0x08040201); + const int grid_l = __vsub4(grid[0] ^ signs0, signs0); + const int grid_h = __vsub4(grid[1] ^ signs1, signs1); + sumi1 = __dp4a(grid_l, *((const int *)q8 + 0), sumi1); + sumi1 = __dp4a(grid_h, *((const int *)q8 + 1), sumi1); + q8 += 8; + } + int sumi2 = 0; + for (int l = 2; l < 4; ++l) { + const uint32_t * grid = (const uint32_t *)(iq2s_grid + (bq2->qs[4*ib32+l] | ((bq2->qh[ib32] << (8-2*l)) & 0x300))); + const uint32_t signs0 = __vcmpeq4(((signs[l] & 0xf) * 0x01010101) & 0x08040201, 0x08040201); + const uint32_t signs1 = __vcmpeq4(((signs[l] >> 4) * 0x01010101) & 0x08040201, 0x08040201); + const int grid_l = __vsub4(grid[0] ^ signs0, signs0); + const int grid_h = __vsub4(grid[1] ^ signs1, signs1); + sumi2 = __dp4a(grid_l, *((const int *)q8 + 0), sumi2); + sumi2 = __dp4a(grid_h, *((const int *)q8 + 1), sumi2); + q8 += 8; + } + const float d = (float)bq2->d * __low2float(bq8_1[ib32].ds) * 0.25f; + return d * ((0.5f + ls1) * sumi1 + (0.5f + ls2) * sumi2); +#else + (void) ksigns64; + assert(false); + return 0.f; +#endif +#else + (void) ksigns64; + assert(false); + return 0.f; +#endif +} + static __device__ __forceinline__ float vec_dot_iq3_xxs_q8_1( const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) { #if __CUDA_ARCH__ >= MIN_CC_DP4A // lowest compute capability for integer intrinsics @@ -6996,6 +7335,12 @@ static void dequantize_row_iq2_xs_cuda(const void * vx, dst_t * y, const int k, dequantize_block_iq2_xs<<>>(vx, y); } +template +static void dequantize_row_iq2_s_cuda(const void * vx, dst_t * y, const int k, cudaStream_t stream) { + const int nb = k / QK_K; + dequantize_block_iq2_s<<>>(vx, y); +} + template static void dequantize_row_iq3_xxs_cuda(const void * vx, dst_t * y, const int k, cudaStream_t stream) { const int nb = k / QK_K; @@ -7057,6 +7402,8 @@ static to_fp16_cuda_t ggml_get_to_fp16_cuda(ggml_type type) { return dequantize_row_iq2_xxs_cuda; case GGML_TYPE_IQ2_XS: return dequantize_row_iq2_xs_cuda; + case GGML_TYPE_IQ2_S: + return dequantize_row_iq2_s_cuda; case GGML_TYPE_IQ3_XXS: return dequantize_row_iq3_xxs_cuda; case GGML_TYPE_IQ1_S: @@ -7098,6 +7445,8 @@ static to_fp32_cuda_t ggml_get_to_fp32_cuda(ggml_type type) { return dequantize_row_iq2_xxs_cuda; case GGML_TYPE_IQ2_XS: return dequantize_row_iq2_xs_cuda; + case GGML_TYPE_IQ2_S: + return dequantize_row_iq2_s_cuda; case GGML_TYPE_IQ3_XXS: return dequantize_row_iq3_xxs_cuda; case GGML_TYPE_IQ1_S: @@ -8848,6 +9197,7 @@ static int64_t get_row_rounding(ggml_type type, const std::array (src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_padded_row_size, src1_ncols, nrows_dst, stream); break; + case GGML_TYPE_IQ2_S: + mul_mat_vec_q_cuda + (src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_padded_row_size, src1_ncols, nrows_dst, stream); + break; case GGML_TYPE_IQ3_XXS: mul_mat_vec_q_cuda (src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_padded_row_size, src1_ncols, nrows_dst, stream); @@ -11710,7 +12065,8 @@ GGML_CALL static bool ggml_backend_cuda_supports_op(ggml_backend_t backend, cons } ggml_type a_type = a->type; if (a_type == GGML_TYPE_IQ2_XXS || a_type == GGML_TYPE_IQ2_XS || a_type == GGML_TYPE_IQ3_XXS || - a_type == GGML_TYPE_IQ1_S || a_type == GGML_TYPE_IQ4_NL || a_type == GGML_TYPE_IQ3_S) { + a_type == GGML_TYPE_IQ1_S || a_type == GGML_TYPE_IQ4_NL || a_type == GGML_TYPE_IQ3_S || + a_type == GGML_TYPE_IQ2_S) { if (b->ne[1] == 1 && ggml_nrows(b) > 1) { return false; } diff --git a/ggml-metal.m b/ggml-metal.m index 3d6b01263..251d04fb0 100644 --- a/ggml-metal.m +++ b/ggml-metal.m @@ -62,6 +62,7 @@ enum ggml_metal_kernel_type { GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ2_XS, GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ3_XXS, GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ3_S, + GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ2_S, GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ1_S, GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ4_NL, GGML_METAL_KERNEL_TYPE_GET_ROWS_I32, @@ -87,6 +88,7 @@ enum ggml_metal_kernel_type { GGML_METAL_KERNEL_TYPE_MUL_MV_IQ2_XS_F32, GGML_METAL_KERNEL_TYPE_MUL_MV_IQ3_XXS_F32, GGML_METAL_KERNEL_TYPE_MUL_MV_IQ3_S_F32, + GGML_METAL_KERNEL_TYPE_MUL_MV_IQ2_S_F32, GGML_METAL_KERNEL_TYPE_MUL_MV_IQ1_S_F32, GGML_METAL_KERNEL_TYPE_MUL_MV_IQ4_NL_F32, GGML_METAL_KERNEL_TYPE_MUL_MV_ID_F32_F32, @@ -108,6 +110,7 @@ enum ggml_metal_kernel_type { GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ2_XS_F32, GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ3_XXS_F32, GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ3_S_F32, + GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ2_S_F32, GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ1_S_F32, GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ4_NL_F32, GGML_METAL_KERNEL_TYPE_MUL_MM_F32_F32, @@ -126,6 +129,7 @@ enum ggml_metal_kernel_type { GGML_METAL_KERNEL_TYPE_MUL_MM_IQ2_XS_F32, GGML_METAL_KERNEL_TYPE_MUL_MM_IQ3_XXS_F32, GGML_METAL_KERNEL_TYPE_MUL_MM_IQ3_S_F32, + GGML_METAL_KERNEL_TYPE_MUL_MM_IQ2_S_F32, GGML_METAL_KERNEL_TYPE_MUL_MM_IQ1_S_F32, GGML_METAL_KERNEL_TYPE_MUL_MM_IQ4_NL_F32, GGML_METAL_KERNEL_TYPE_MUL_MM_ID_F32_F32, @@ -144,6 +148,7 @@ enum ggml_metal_kernel_type { GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ2_XS_F32, GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ3_XXS_F32, GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ3_S_F32, + GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ2_S_F32, GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ1_S_F32, GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ4_NL_F32, GGML_METAL_KERNEL_TYPE_ROPE_F32, @@ -458,6 +463,7 @@ static struct ggml_metal_context * ggml_metal_init(int n_cb) { GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ2_XS, get_rows_iq2_xs, true); GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ3_XXS, get_rows_iq3_xxs, true); GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ3_S, get_rows_iq3_s, true); + GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ2_S, get_rows_iq2_s, true); GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ1_S, get_rows_iq1_s, true); GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ4_NL, get_rows_iq4_nl, true); GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_I32, get_rows_i32, true); @@ -483,6 +489,7 @@ static struct ggml_metal_context * ggml_metal_init(int n_cb) { GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_IQ2_XS_F32, mul_mv_iq2_xs_f32, ctx->support_simdgroup_reduction); GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_IQ3_XXS_F32, mul_mv_iq3_xxs_f32, ctx->support_simdgroup_reduction); GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_IQ3_S_F32, mul_mv_iq3_s_f32, ctx->support_simdgroup_reduction); + GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_IQ2_S_F32, mul_mv_iq2_s_f32, ctx->support_simdgroup_reduction); GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_IQ1_S_F32, mul_mv_iq1_s_f32, ctx->support_simdgroup_reduction); GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_IQ4_NL_F32, mul_mv_iq4_nl_f32, ctx->support_simdgroup_reduction); GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_F32_F32, mul_mv_id_f32_f32, ctx->support_simdgroup_reduction); @@ -504,6 +511,7 @@ static struct ggml_metal_context * ggml_metal_init(int n_cb) { GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ2_XS_F32, mul_mv_id_iq2_xs_f32, ctx->support_simdgroup_reduction); GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ3_XXS_F32, mul_mv_id_iq3_xxs_f32, ctx->support_simdgroup_reduction); GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ3_S_F32, mul_mv_id_iq3_s_f32, ctx->support_simdgroup_reduction); + GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ2_S_F32, mul_mv_id_iq2_s_f32, ctx->support_simdgroup_reduction); GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ1_S_F32, mul_mv_id_iq1_s_f32, ctx->support_simdgroup_reduction); GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ4_NL_F32, mul_mv_id_iq4_nl_f32, ctx->support_simdgroup_reduction); GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_F32_F32, mul_mm_f32_f32, ctx->support_simdgroup_mm); @@ -522,6 +530,7 @@ static struct ggml_metal_context * ggml_metal_init(int n_cb) { GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_IQ2_XS_F32, mul_mm_iq2_xs_f32, ctx->support_simdgroup_mm); GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_IQ3_XXS_F32, mul_mm_iq3_xxs_f32, ctx->support_simdgroup_mm); GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_IQ3_S_F32, mul_mm_iq3_s_f32, ctx->support_simdgroup_mm); + GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_IQ2_S_F32, mul_mm_iq2_s_f32, ctx->support_simdgroup_mm); GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_IQ1_S_F32, mul_mm_iq1_s_f32, ctx->support_simdgroup_mm); GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_IQ4_NL_F32, mul_mm_iq4_nl_f32, ctx->support_simdgroup_mm); GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_F32_F32, mul_mm_id_f32_f32, ctx->support_simdgroup_mm); @@ -540,6 +549,7 @@ static struct ggml_metal_context * ggml_metal_init(int n_cb) { GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ2_XS_F32, mul_mm_id_iq2_xs_f32, ctx->support_simdgroup_mm); GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ3_XXS_F32, mul_mm_id_iq3_xxs_f32, ctx->support_simdgroup_mm); GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ3_S_F32, mul_mm_id_iq3_s_f32, ctx->support_simdgroup_mm); + GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ2_S_F32, mul_mm_id_iq2_s_f32, ctx->support_simdgroup_mm); GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ1_S_F32, mul_mm_id_iq1_s_f32, ctx->support_simdgroup_mm); GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ4_NL_F32, mul_mm_id_iq4_nl_f32, ctx->support_simdgroup_mm); GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_ROPE_F32, rope_f32, true); @@ -1358,6 +1368,7 @@ static bool ggml_metal_graph_compute( case GGML_TYPE_IQ2_XS: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_IQ2_XS_F32 ].pipeline; break; case GGML_TYPE_IQ3_XXS: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_IQ3_XXS_F32].pipeline; break; case GGML_TYPE_IQ3_S: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_IQ3_S_F32 ].pipeline; break; + case GGML_TYPE_IQ2_S: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_IQ2_S_F32 ].pipeline; break; case GGML_TYPE_IQ1_S: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_IQ1_S_F32 ].pipeline; break; case GGML_TYPE_IQ4_NL: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_IQ4_NL_F32 ].pipeline; break; default: GGML_ASSERT(false && "MUL MAT-MAT not implemented"); @@ -1500,6 +1511,12 @@ static bool ggml_metal_graph_compute( nth1 = 16; pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_IQ3_S_F32].pipeline; } break; + case GGML_TYPE_IQ2_S: + { + nth0 = 4; + nth1 = 16; + pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_IQ2_S_F32].pipeline; + } break; case GGML_TYPE_IQ1_S: { nth0 = 4; @@ -1544,9 +1561,9 @@ static bool ggml_metal_graph_compute( [encoder setBytes:&r2 length:sizeof(r2) atIndex:17]; [encoder setBytes:&r3 length:sizeof(r3) atIndex:18]; - if (src0t == GGML_TYPE_Q4_0 || src0t == GGML_TYPE_Q4_1 || - src0t == GGML_TYPE_Q5_0 || src0t == GGML_TYPE_Q5_1 || src0t == GGML_TYPE_Q8_0 || - src0t == GGML_TYPE_Q2_K || src0t == GGML_TYPE_IQ1_S) { // || src0t == GGML_TYPE_Q4_K) { + if (src0t == GGML_TYPE_Q4_0 || src0t == GGML_TYPE_Q4_1 || + src0t == GGML_TYPE_Q5_0 || src0t == GGML_TYPE_Q5_1 || src0t == GGML_TYPE_Q8_0 || + src0t == GGML_TYPE_Q2_K || src0t == GGML_TYPE_IQ1_S || src0t == GGML_TYPE_IQ2_S) { [encoder dispatchThreadgroups:MTLSizeMake((ne01 + 7)/8, ne11, ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)]; } else if (src0t == GGML_TYPE_IQ2_XXS || src0t == GGML_TYPE_IQ2_XS) { @@ -1658,6 +1675,7 @@ static bool ggml_metal_graph_compute( case GGML_TYPE_IQ2_XS: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ2_XS_F32 ].pipeline; break; case GGML_TYPE_IQ3_XXS: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ3_XXS_F32].pipeline; break; case GGML_TYPE_IQ3_S: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ3_S_F32 ].pipeline; break; + case GGML_TYPE_IQ2_S: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ2_S_F32 ].pipeline; break; case GGML_TYPE_IQ1_S: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ1_S_F32 ].pipeline; break; case GGML_TYPE_IQ4_NL: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ4_NL_F32 ].pipeline; break; default: GGML_ASSERT(false && "MUL_MAT_ID not implemented"); @@ -1803,6 +1821,12 @@ static bool ggml_metal_graph_compute( nth1 = 16; pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ3_S_F32].pipeline; } break; + case GGML_TYPE_IQ2_S: + { + nth0 = 4; + nth1 = 16; + pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ2_S_F32].pipeline; + } break; case GGML_TYPE_IQ1_S: { nth0 = 4; @@ -1863,9 +1887,9 @@ static bool ggml_metal_graph_compute( [encoder setBuffer:id_src_cur offset:offs_src_cur atIndex:23 + j]; } - if (src2t == GGML_TYPE_Q4_0 || src2t == GGML_TYPE_Q4_1 || - src2t == GGML_TYPE_Q5_0 || src2t == GGML_TYPE_Q5_1 || src2t == GGML_TYPE_Q8_0 || - src2t == GGML_TYPE_Q2_K || src2t == GGML_TYPE_IQ1_S) { // || src2t == GGML_TYPE_Q4_K) { + if (src2t == GGML_TYPE_Q4_0 || src2t == GGML_TYPE_Q4_1 || + src2t == GGML_TYPE_Q5_0 || src2t == GGML_TYPE_Q5_1 || src2t == GGML_TYPE_Q8_0 || + src2t == GGML_TYPE_Q2_K || src2t == GGML_TYPE_IQ1_S || src2t == GGML_TYPE_IQ2_S) { [encoder dispatchThreadgroups:MTLSizeMake((ne21 + 7)/8, _ne1, ne01*ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)]; } else if (src2t == GGML_TYPE_IQ2_XXS || src2t == GGML_TYPE_IQ2_XS) { @@ -1925,6 +1949,7 @@ static bool ggml_metal_graph_compute( case GGML_TYPE_IQ2_XS: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ2_XS ].pipeline; break; case GGML_TYPE_IQ3_XXS: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ3_XXS].pipeline; break; case GGML_TYPE_IQ3_S: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ3_S ].pipeline; break; + case GGML_TYPE_IQ2_S: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ2_S ].pipeline; break; case GGML_TYPE_IQ1_S: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ1_S ].pipeline; break; case GGML_TYPE_IQ4_NL: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ4_NL ].pipeline; break; case GGML_TYPE_I32: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GET_ROWS_I32 ].pipeline; break; diff --git a/ggml-metal.metal b/ggml-metal.metal index b3bf40539..47354e952 100644 --- a/ggml-metal.metal +++ b/ggml-metal.metal @@ -2519,6 +2519,14 @@ typedef struct { } block_iq2_xs; // 74 bytes / block for QK_K = 256, so 2.3125 bpw +// 2.5625 bpw quants +typedef struct { + half d; + uint8_t qs[QK_K/4]; + uint8_t qh[QK_K/32]; + uint8_t scales[QK_K/32]; +} block_iq2_s; + typedef struct { half d; uint8_t qs[3*QK_K/8]; @@ -3774,6 +3782,265 @@ constexpr constant static uint64_t iq2xs_grid[512] = { 0x2b2b2b2b082b2b08, 0x2b2b2b2b082b2b2b, 0x2b2b2b2b2b190819, 0x2b2b2b2b2b2b2b2b, }; +constexpr constant static uint64_t iq2s_grid[1024] = { + 0x0808080808080808, 0x080808080808082b, 0x0808080808081919, 0x0808080808082b08, + 0x0808080808082b2b, 0x0808080808190819, 0x0808080808191908, 0x080808080819192b, + 0x0808080808192b19, 0x08080808082b0808, 0x08080808082b082b, 0x08080808082b1919, + 0x08080808082b2b08, 0x0808080819080819, 0x0808080819081908, 0x080808081908192b, + 0x0808080819082b19, 0x0808080819190808, 0x080808081919082b, 0x0808080819191919, + 0x0808080819192b08, 0x08080808192b0819, 0x08080808192b1908, 0x08080808192b192b, + 0x08080808192b2b19, 0x080808082b080808, 0x080808082b08082b, 0x080808082b081919, + 0x080808082b082b08, 0x080808082b190819, 0x080808082b191908, 0x080808082b2b0808, + 0x080808082b2b1919, 0x080808082b2b2b2b, 0x0808081908080819, 0x0808081908081908, + 0x080808190808192b, 0x0808081908082b19, 0x0808081908190808, 0x080808190819082b, + 0x0808081908191919, 0x0808081908192b08, 0x08080819082b0819, 0x08080819082b1908, + 0x0808081919080808, 0x080808191908082b, 0x0808081919081919, 0x0808081919082b08, + 0x0808081919190819, 0x0808081919191908, 0x080808191919192b, 0x0808081919192b19, + 0x08080819192b0808, 0x08080819192b1919, 0x08080819192b2b08, 0x080808192b080819, + 0x080808192b081908, 0x080808192b190808, 0x080808192b19082b, 0x080808192b191919, + 0x080808192b2b0819, 0x080808192b2b1908, 0x0808082b08080808, 0x0808082b0808082b, + 0x0808082b08081919, 0x0808082b08082b08, 0x0808082b08190819, 0x0808082b08191908, + 0x0808082b082b0808, 0x0808082b082b2b2b, 0x0808082b19080819, 0x0808082b19081908, + 0x0808082b1908192b, 0x0808082b19082b19, 0x0808082b19190808, 0x0808082b19191919, + 0x0808082b2b080808, 0x0808082b2b081919, 0x0808082b2b082b2b, 0x0808082b2b191908, + 0x0808082b2b2b082b, 0x0808190808080819, 0x0808190808081908, 0x080819080808192b, + 0x0808190808082b19, 0x0808190808190808, 0x080819080819082b, 0x0808190808191919, + 0x0808190808192b08, 0x08081908082b0819, 0x08081908082b1908, 0x08081908082b192b, + 0x08081908082b2b19, 0x0808190819080808, 0x080819081908082b, 0x0808190819081919, + 0x0808190819082b08, 0x0808190819082b2b, 0x0808190819190819, 0x0808190819191908, + 0x080819081919192b, 0x0808190819192b19, 0x08081908192b0808, 0x08081908192b082b, + 0x08081908192b1919, 0x080819082b080819, 0x080819082b081908, 0x080819082b08192b, + 0x080819082b082b19, 0x080819082b190808, 0x080819082b191919, 0x080819082b192b08, + 0x080819082b2b0819, 0x080819082b2b1908, 0x0808191908080808, 0x080819190808082b, + 0x0808191908081919, 0x0808191908082b08, 0x0808191908082b2b, 0x0808191908190819, + 0x0808191908191908, 0x080819190819192b, 0x0808191908192b19, 0x08081919082b0808, + 0x08081919082b1919, 0x08081919082b2b08, 0x0808191919080819, 0x0808191919081908, + 0x080819191908192b, 0x0808191919082b19, 0x0808191919190808, 0x080819191919082b, + 0x0808191919191919, 0x0808191919192b08, 0x08081919192b0819, 0x08081919192b1908, + 0x080819192b080808, 0x080819192b08082b, 0x080819192b081919, 0x080819192b082b08, + 0x080819192b190819, 0x080819192b191908, 0x080819192b2b0808, 0x0808192b08080819, + 0x0808192b08081908, 0x0808192b0808192b, 0x0808192b08082b19, 0x0808192b08190808, + 0x0808192b08191919, 0x0808192b19080808, 0x0808192b19081919, 0x0808192b19082b08, + 0x0808192b19190819, 0x0808192b19191908, 0x0808192b192b0808, 0x0808192b2b080819, + 0x0808192b2b081908, 0x0808192b2b190808, 0x08082b0808080808, 0x08082b080808082b, + 0x08082b0808081919, 0x08082b0808082b08, 0x08082b0808190819, 0x08082b0808191908, + 0x08082b080819192b, 0x08082b0808192b19, 0x08082b08082b0808, 0x08082b08082b1919, + 0x08082b08082b2b2b, 0x08082b0819080819, 0x08082b0819081908, 0x08082b081908192b, + 0x08082b0819082b19, 0x08082b0819190808, 0x08082b081919082b, 0x08082b0819191919, + 0x08082b0819192b08, 0x08082b08192b0819, 0x08082b08192b1908, 0x08082b082b080808, + 0x08082b082b081919, 0x08082b082b191908, 0x08082b082b2b2b2b, 0x08082b1908080819, + 0x08082b1908081908, 0x08082b1908190808, 0x08082b190819082b, 0x08082b1908191919, + 0x08082b1908192b08, 0x08082b19082b0819, 0x08082b1919080808, 0x08082b1919081919, + 0x08082b1919082b08, 0x08082b1919190819, 0x08082b1919191908, 0x08082b19192b0808, + 0x08082b192b080819, 0x08082b192b190808, 0x08082b2b08080808, 0x08082b2b08190819, + 0x08082b2b08191908, 0x08082b2b082b082b, 0x08082b2b082b2b08, 0x08082b2b082b2b2b, + 0x08082b2b19190808, 0x08082b2b2b192b19, 0x0819080808080819, 0x0819080808081908, + 0x081908080808192b, 0x0819080808082b19, 0x0819080808190808, 0x081908080819082b, + 0x0819080808191919, 0x0819080808192b08, 0x08190808082b0819, 0x08190808082b1908, + 0x08190808082b192b, 0x0819080819080808, 0x081908081908082b, 0x0819080819081919, + 0x0819080819082b08, 0x0819080819190819, 0x0819080819191908, 0x081908081919192b, + 0x0819080819192b19, 0x08190808192b0808, 0x08190808192b082b, 0x08190808192b1919, + 0x08190808192b2b08, 0x081908082b080819, 0x081908082b081908, 0x081908082b08192b, + 0x081908082b190808, 0x081908082b191919, 0x081908082b192b08, 0x081908082b2b0819, + 0x081908082b2b1908, 0x0819081908080808, 0x081908190808082b, 0x0819081908081919, + 0x0819081908082b08, 0x0819081908082b2b, 0x0819081908190819, 0x0819081908191908, + 0x081908190819192b, 0x0819081908192b19, 0x08190819082b0808, 0x08190819082b082b, + 0x08190819082b1919, 0x08190819082b2b08, 0x0819081919080819, 0x0819081919081908, + 0x081908191908192b, 0x0819081919082b19, 0x0819081919190808, 0x081908191919082b, + 0x0819081919191919, 0x0819081919192b08, 0x08190819192b0819, 0x08190819192b1908, + 0x081908192b080808, 0x081908192b08082b, 0x081908192b081919, 0x081908192b082b08, + 0x081908192b190819, 0x081908192b191908, 0x0819082b08080819, 0x0819082b08081908, + 0x0819082b08082b19, 0x0819082b08190808, 0x0819082b08191919, 0x0819082b082b0819, + 0x0819082b082b1908, 0x0819082b19080808, 0x0819082b19081919, 0x0819082b19190819, + 0x0819082b19191908, 0x0819082b2b080819, 0x0819082b2b081908, 0x0819082b2b190808, + 0x0819190808080808, 0x081919080808082b, 0x0819190808081919, 0x0819190808082b08, + 0x0819190808190819, 0x0819190808191908, 0x081919080819192b, 0x0819190808192b19, + 0x08191908082b0808, 0x08191908082b1919, 0x08191908082b2b08, 0x0819190819080819, + 0x0819190819081908, 0x081919081908192b, 0x0819190819082b19, 0x0819190819190808, + 0x081919081919082b, 0x0819190819191919, 0x0819190819192b08, 0x08191908192b0819, + 0x08191908192b1908, 0x081919082b080808, 0x081919082b08082b, 0x081919082b081919, + 0x081919082b082b08, 0x081919082b190819, 0x081919082b191908, 0x081919082b2b0808, + 0x0819191908080819, 0x0819191908081908, 0x081919190808192b, 0x0819191908082b19, + 0x0819191908190808, 0x081919190819082b, 0x0819191908191919, 0x0819191908192b08, + 0x08191919082b0819, 0x08191919082b1908, 0x0819191919080808, 0x081919191908082b, + 0x0819191919081919, 0x0819191919082b08, 0x0819191919190819, 0x0819191919191908, + 0x08191919192b0808, 0x081919192b080819, 0x081919192b081908, 0x081919192b190808, + 0x0819192b08080808, 0x0819192b08081919, 0x0819192b08082b08, 0x0819192b08190819, + 0x0819192b08191908, 0x0819192b082b0808, 0x0819192b19080819, 0x0819192b19081908, + 0x0819192b19190808, 0x0819192b2b080808, 0x0819192b2b2b2b2b, 0x08192b0808080819, + 0x08192b0808081908, 0x08192b080808192b, 0x08192b0808082b19, 0x08192b0808190808, + 0x08192b0808191919, 0x08192b0808192b08, 0x08192b08082b0819, 0x08192b0819080808, + 0x08192b081908082b, 0x08192b0819081919, 0x08192b0819082b08, 0x08192b0819190819, + 0x08192b0819191908, 0x08192b08192b0808, 0x08192b082b080819, 0x08192b082b081908, + 0x08192b1908080808, 0x08192b190808082b, 0x08192b1908081919, 0x08192b1908082b08, + 0x08192b1908190819, 0x08192b1908191908, 0x08192b19082b0808, 0x08192b1919080819, + 0x08192b1919081908, 0x08192b1919190808, 0x08192b19192b2b19, 0x08192b192b2b082b, + 0x08192b2b08081908, 0x08192b2b08190808, 0x08192b2b19080808, 0x08192b2b1919192b, + 0x082b080808080808, 0x082b08080808082b, 0x082b080808081919, 0x082b080808082b08, + 0x082b080808190819, 0x082b080808191908, 0x082b08080819192b, 0x082b080808192b19, + 0x082b0808082b0808, 0x082b0808082b1919, 0x082b0808082b2b2b, 0x082b080819080819, + 0x082b080819081908, 0x082b080819190808, 0x082b08081919082b, 0x082b080819191919, + 0x082b0808192b1908, 0x082b08082b080808, 0x082b08082b082b2b, 0x082b08082b191908, + 0x082b08082b2b2b2b, 0x082b081908080819, 0x082b081908081908, 0x082b081908190808, + 0x082b08190819082b, 0x082b081908191919, 0x082b0819082b0819, 0x082b081919080808, + 0x082b08191908082b, 0x082b081919081919, 0x082b081919190819, 0x082b081919191908, + 0x082b0819192b0808, 0x082b08192b080819, 0x082b08192b081908, 0x082b08192b190808, + 0x082b082b08080808, 0x082b082b08082b2b, 0x082b082b082b082b, 0x082b082b082b2b08, + 0x082b082b082b2b2b, 0x082b082b19081908, 0x082b082b19190808, 0x082b082b2b082b08, + 0x082b082b2b082b2b, 0x082b082b2b2b2b08, 0x082b190808080819, 0x082b190808081908, + 0x082b19080808192b, 0x082b190808082b19, 0x082b190808190808, 0x082b190808191919, + 0x082b190808192b08, 0x082b1908082b0819, 0x082b1908082b1908, 0x082b190819080808, + 0x082b19081908082b, 0x082b190819081919, 0x082b190819082b08, 0x082b190819190819, + 0x082b190819191908, 0x082b1908192b0808, 0x082b19082b080819, 0x082b19082b081908, + 0x082b19082b190808, 0x082b191908080808, 0x082b191908081919, 0x082b191908082b08, + 0x082b191908190819, 0x082b191908191908, 0x082b1919082b0808, 0x082b191919080819, + 0x082b191919081908, 0x082b191919190808, 0x082b1919192b192b, 0x082b19192b080808, + 0x082b192b08080819, 0x082b192b08081908, 0x082b192b08190808, 0x082b192b19080808, + 0x082b192b19192b19, 0x082b2b0808080808, 0x082b2b0808081919, 0x082b2b0808190819, + 0x082b2b0808191908, 0x082b2b0819080819, 0x082b2b0819081908, 0x082b2b0819190808, + 0x082b2b082b082b2b, 0x082b2b082b2b2b2b, 0x082b2b1908080819, 0x082b2b1908081908, + 0x082b2b1908190808, 0x082b2b192b191919, 0x082b2b2b08082b2b, 0x082b2b2b082b082b, + 0x082b2b2b192b1908, 0x082b2b2b2b082b08, 0x082b2b2b2b082b2b, 0x1908080808080819, + 0x1908080808081908, 0x190808080808192b, 0x1908080808082b19, 0x1908080808190808, + 0x190808080819082b, 0x1908080808191919, 0x1908080808192b08, 0x1908080808192b2b, + 0x19080808082b0819, 0x19080808082b1908, 0x19080808082b192b, 0x1908080819080808, + 0x190808081908082b, 0x1908080819081919, 0x1908080819082b08, 0x1908080819082b2b, + 0x1908080819190819, 0x1908080819191908, 0x190808081919192b, 0x1908080819192b19, + 0x19080808192b0808, 0x19080808192b082b, 0x19080808192b1919, 0x190808082b080819, + 0x190808082b081908, 0x190808082b190808, 0x190808082b191919, 0x190808082b192b08, + 0x190808082b2b0819, 0x190808082b2b1908, 0x1908081908080808, 0x190808190808082b, + 0x1908081908081919, 0x1908081908082b08, 0x1908081908190819, 0x1908081908191908, + 0x190808190819192b, 0x1908081908192b19, 0x19080819082b0808, 0x19080819082b082b, + 0x19080819082b1919, 0x1908081919080819, 0x1908081919081908, 0x190808191908192b, + 0x1908081919082b19, 0x1908081919190808, 0x190808191919082b, 0x1908081919191919, + 0x1908081919192b08, 0x19080819192b0819, 0x19080819192b1908, 0x190808192b080808, + 0x190808192b08082b, 0x190808192b081919, 0x190808192b082b08, 0x190808192b190819, + 0x190808192b191908, 0x190808192b2b0808, 0x1908082b08080819, 0x1908082b08081908, + 0x1908082b08190808, 0x1908082b0819082b, 0x1908082b08191919, 0x1908082b08192b08, + 0x1908082b082b1908, 0x1908082b19080808, 0x1908082b19081919, 0x1908082b19082b08, + 0x1908082b19190819, 0x1908082b19191908, 0x1908082b192b0808, 0x1908082b2b080819, + 0x1908082b2b081908, 0x1908190808080808, 0x190819080808082b, 0x1908190808081919, + 0x1908190808082b08, 0x1908190808082b2b, 0x1908190808190819, 0x1908190808191908, + 0x190819080819192b, 0x1908190808192b19, 0x19081908082b0808, 0x19081908082b082b, + 0x19081908082b1919, 0x19081908082b2b08, 0x1908190819080819, 0x1908190819081908, + 0x190819081908192b, 0x1908190819082b19, 0x1908190819190808, 0x190819081919082b, + 0x1908190819191919, 0x1908190819192b08, 0x19081908192b0819, 0x19081908192b1908, + 0x190819082b080808, 0x190819082b08082b, 0x190819082b081919, 0x190819082b082b08, + 0x190819082b190819, 0x190819082b191908, 0x190819082b2b0808, 0x1908191908080819, + 0x1908191908081908, 0x190819190808192b, 0x1908191908082b19, 0x1908191908190808, + 0x190819190819082b, 0x1908191908191919, 0x1908191908192b08, 0x19081919082b0819, + 0x19081919082b1908, 0x1908191919080808, 0x190819191908082b, 0x1908191919081919, + 0x1908191919082b08, 0x1908191919190819, 0x1908191919191908, 0x19081919192b0808, + 0x19081919192b2b2b, 0x190819192b080819, 0x190819192b081908, 0x190819192b190808, + 0x1908192b08080808, 0x1908192b0808082b, 0x1908192b08081919, 0x1908192b08082b08, + 0x1908192b08190819, 0x1908192b08191908, 0x1908192b082b0808, 0x1908192b19080819, + 0x1908192b19081908, 0x1908192b19190808, 0x1908192b2b080808, 0x1908192b2b2b1919, + 0x19082b0808080819, 0x19082b0808081908, 0x19082b0808082b19, 0x19082b0808190808, + 0x19082b080819082b, 0x19082b0808191919, 0x19082b0808192b08, 0x19082b08082b0819, + 0x19082b08082b1908, 0x19082b0819080808, 0x19082b081908082b, 0x19082b0819081919, + 0x19082b0819082b08, 0x19082b0819190819, 0x19082b0819191908, 0x19082b08192b0808, + 0x19082b082b081908, 0x19082b082b190808, 0x19082b1908080808, 0x19082b190808082b, + 0x19082b1908081919, 0x19082b1908082b08, 0x19082b1908190819, 0x19082b1908191908, + 0x19082b19082b0808, 0x19082b1919080819, 0x19082b1919081908, 0x19082b1919190808, + 0x19082b192b080808, 0x19082b192b19192b, 0x19082b2b08080819, 0x19082b2b08081908, + 0x19082b2b08190808, 0x19082b2b19080808, 0x1919080808080808, 0x191908080808082b, + 0x1919080808081919, 0x1919080808082b08, 0x1919080808190819, 0x1919080808191908, + 0x191908080819192b, 0x1919080808192b19, 0x19190808082b0808, 0x19190808082b082b, + 0x19190808082b1919, 0x19190808082b2b08, 0x1919080819080819, 0x1919080819081908, + 0x191908081908192b, 0x1919080819082b19, 0x1919080819190808, 0x191908081919082b, + 0x1919080819191919, 0x1919080819192b08, 0x19190808192b0819, 0x19190808192b1908, + 0x191908082b080808, 0x191908082b08082b, 0x191908082b081919, 0x191908082b082b08, + 0x191908082b190819, 0x191908082b191908, 0x1919081908080819, 0x1919081908081908, + 0x191908190808192b, 0x1919081908082b19, 0x1919081908190808, 0x191908190819082b, + 0x1919081908191919, 0x1919081908192b08, 0x19190819082b0819, 0x19190819082b1908, + 0x1919081919080808, 0x191908191908082b, 0x1919081919081919, 0x1919081919082b08, + 0x1919081919190819, 0x1919081919191908, 0x19190819192b0808, 0x191908192b080819, + 0x191908192b081908, 0x191908192b190808, 0x1919082b08080808, 0x1919082b08081919, + 0x1919082b08082b08, 0x1919082b08190819, 0x1919082b08191908, 0x1919082b082b0808, + 0x1919082b19080819, 0x1919082b19081908, 0x1919082b19190808, 0x1919082b192b2b19, + 0x1919082b2b080808, 0x1919190808080819, 0x1919190808081908, 0x191919080808192b, + 0x1919190808082b19, 0x1919190808190808, 0x191919080819082b, 0x1919190808191919, + 0x1919190808192b08, 0x19191908082b0819, 0x19191908082b1908, 0x1919190819080808, + 0x191919081908082b, 0x1919190819081919, 0x1919190819082b08, 0x1919190819190819, + 0x1919190819191908, 0x19191908192b0808, 0x191919082b080819, 0x191919082b081908, + 0x191919082b190808, 0x1919191908080808, 0x191919190808082b, 0x1919191908081919, + 0x1919191908082b08, 0x1919191908190819, 0x1919191908191908, 0x19191919082b0808, + 0x1919191919080819, 0x1919191919081908, 0x1919191919190808, 0x191919192b080808, + 0x1919192b08080819, 0x1919192b08081908, 0x1919192b08190808, 0x1919192b082b192b, + 0x1919192b19080808, 0x19192b0808080808, 0x19192b080808082b, 0x19192b0808081919, + 0x19192b0808082b08, 0x19192b0808190819, 0x19192b0808191908, 0x19192b08082b0808, + 0x19192b0819080819, 0x19192b0819081908, 0x19192b0819190808, 0x19192b0819192b2b, + 0x19192b082b080808, 0x19192b1908080819, 0x19192b1908081908, 0x19192b1908190808, + 0x19192b1919080808, 0x19192b2b08080808, 0x19192b2b08192b19, 0x19192b2b2b081919, + 0x19192b2b2b2b2b08, 0x192b080808080819, 0x192b080808081908, 0x192b08080808192b, + 0x192b080808190808, 0x192b08080819082b, 0x192b080808191919, 0x192b080808192b08, + 0x192b0808082b0819, 0x192b0808082b1908, 0x192b080819080808, 0x192b080819081919, + 0x192b080819082b08, 0x192b080819190819, 0x192b080819191908, 0x192b0808192b0808, + 0x192b08082b081908, 0x192b08082b190808, 0x192b081908080808, 0x192b08190808082b, + 0x192b081908081919, 0x192b081908082b08, 0x192b081908190819, 0x192b081908191908, + 0x192b0819082b0808, 0x192b081919080819, 0x192b081919081908, 0x192b081919190808, + 0x192b08192b080808, 0x192b08192b192b19, 0x192b082b08081908, 0x192b082b08190808, + 0x192b082b19080808, 0x192b082b1919192b, 0x192b082b2b2b0819, 0x192b190808080808, + 0x192b190808081919, 0x192b190808082b08, 0x192b190808190819, 0x192b190808191908, + 0x192b1908082b0808, 0x192b190819080819, 0x192b190819081908, 0x192b190819190808, + 0x192b19082b080808, 0x192b191908080819, 0x192b191908081908, 0x192b191908190808, + 0x192b191919080808, 0x192b191919082b2b, 0x192b1919192b2b08, 0x192b19192b19082b, + 0x192b192b08080808, 0x192b192b2b191908, 0x192b2b0808080819, 0x192b2b0808081908, + 0x192b2b0808190808, 0x192b2b08192b1919, 0x192b2b082b192b08, 0x192b2b1908080808, + 0x192b2b19082b2b2b, 0x192b2b2b1908082b, 0x192b2b2b2b2b0819, 0x2b08080808080808, + 0x2b0808080808082b, 0x2b08080808081919, 0x2b08080808082b08, 0x2b08080808190819, + 0x2b08080808191908, 0x2b08080808192b19, 0x2b080808082b0808, 0x2b080808082b1919, + 0x2b08080819080819, 0x2b08080819081908, 0x2b08080819190808, 0x2b0808081919082b, + 0x2b08080819191919, 0x2b08080819192b08, 0x2b080808192b0819, 0x2b0808082b080808, + 0x2b0808082b081919, 0x2b0808082b190819, 0x2b0808082b191908, 0x2b08081908080819, + 0x2b08081908081908, 0x2b08081908082b19, 0x2b08081908190808, 0x2b0808190819082b, + 0x2b08081908191919, 0x2b08081908192b08, 0x2b080819082b0819, 0x2b080819082b1908, + 0x2b08081919080808, 0x2b0808191908082b, 0x2b08081919081919, 0x2b08081919082b08, + 0x2b08081919190819, 0x2b08081919191908, 0x2b0808192b080819, 0x2b0808192b081908, + 0x2b0808192b190808, 0x2b0808192b2b2b19, 0x2b08082b08080808, 0x2b08082b08081919, + 0x2b08082b08082b2b, 0x2b08082b08190819, 0x2b08082b08191908, 0x2b08082b19080819, + 0x2b08082b19081908, 0x2b08082b19190808, 0x2b08190808080819, 0x2b08190808081908, + 0x2b0819080808192b, 0x2b08190808082b19, 0x2b08190808190808, 0x2b0819080819082b, + 0x2b08190808191919, 0x2b08190808192b08, 0x2b081908082b0819, 0x2b08190819080808, + 0x2b0819081908082b, 0x2b08190819081919, 0x2b08190819082b08, 0x2b08190819190819, + 0x2b08190819191908, 0x2b081908192b0808, 0x2b0819082b080819, 0x2b0819082b081908, + 0x2b0819082b190808, 0x2b08191908080808, 0x2b0819190808082b, 0x2b08191908081919, + 0x2b08191908082b08, 0x2b08191908190819, 0x2b08191908191908, 0x2b081919082b0808, + 0x2b08191919080819, 0x2b08191919081908, 0x2b08191919190808, 0x2b0819192b080808, + 0x2b0819192b082b2b, 0x2b08192b08080819, 0x2b08192b08081908, 0x2b08192b08190808, + 0x2b08192b082b2b19, 0x2b08192b19080808, 0x2b082b0808080808, 0x2b082b0808081919, + 0x2b082b0808190819, 0x2b082b0808191908, 0x2b082b0819080819, 0x2b082b0819081908, + 0x2b082b0819190808, 0x2b082b082b2b082b, 0x2b082b1908080819, 0x2b082b1908081908, + 0x2b082b1919080808, 0x2b082b19192b1919, 0x2b082b2b082b082b, 0x2b082b2b19192b08, + 0x2b082b2b19192b2b, 0x2b082b2b2b08082b, 0x2b082b2b2b2b082b, 0x2b19080808080819, + 0x2b19080808081908, 0x2b19080808082b19, 0x2b19080808190808, 0x2b1908080819082b, + 0x2b19080808191919, 0x2b19080808192b08, 0x2b190808082b1908, 0x2b19080819080808, + 0x2b1908081908082b, 0x2b19080819081919, 0x2b19080819082b08, 0x2b19080819190819, + 0x2b19080819191908, 0x2b190808192b0808, 0x2b1908082b080819, 0x2b1908082b081908, + 0x2b1908082b190808, 0x2b19081908080808, 0x2b19081908081919, 0x2b19081908190819, + 0x2b19081908191908, 0x2b19081919080819, 0x2b19081919081908, 0x2b19081919190808, + 0x2b19081919192b2b, 0x2b19082b08080819, 0x2b19082b08081908, 0x2b19082b08190808, + 0x2b19082b19080808, 0x2b19082b2b2b192b, 0x2b19190808080808, 0x2b1919080808082b, + 0x2b19190808081919, 0x2b19190808082b08, 0x2b19190808190819, 0x2b19190808191908, + 0x2b191908082b0808, 0x2b19190819080819, 0x2b19190819081908, 0x2b19190819190808, + 0x2b1919082b080808, 0x2b1919082b19192b, 0x2b19191908080819, 0x2b19191908081908, + 0x2b19191908190808, 0x2b19191919080808, 0x2b1919192b192b08, 0x2b1919192b2b0819, + 0x2b19192b08080808, 0x2b19192b1908192b, 0x2b19192b192b1908, 0x2b192b0808080819, + 0x2b192b0808081908, 0x2b192b0808190808, 0x2b192b08082b192b, 0x2b192b0819080808, + 0x2b192b082b2b2b19, 0x2b192b1908080808, 0x2b192b1919082b19, 0x2b192b191919082b, + 0x2b192b2b2b190808, 0x2b2b080808080808, 0x2b2b080808081919, 0x2b2b080808082b2b, + 0x2b2b080808191908, 0x2b2b0808082b082b, 0x2b2b0808082b2b2b, 0x2b2b080819080819, + 0x2b2b080819081908, 0x2b2b080819190808, 0x2b2b08082b2b082b, 0x2b2b08082b2b2b2b, + 0x2b2b081919080808, 0x2b2b0819192b1919, 0x2b2b082b0808082b, 0x2b2b082b08082b2b, + 0x2b2b082b082b082b, 0x2b2b082b082b2b08, 0x2b2b082b082b2b2b, 0x2b2b082b2b08082b, + 0x2b2b082b2b082b08, 0x2b2b082b2b082b2b, 0x2b2b082b2b2b2b08, 0x2b2b190808080819, + 0x2b2b190808081908, 0x2b2b190808190808, 0x2b2b190819080808, 0x2b2b19082b082b19, + 0x2b2b19082b2b1908, 0x2b2b191908080808, 0x2b2b191908192b19, 0x2b2b192b19190819, + 0x2b2b2b0808082b2b, 0x2b2b2b08082b2b08, 0x2b2b2b082b2b082b, 0x2b2b2b1919191908, + 0x2b2b2b192b08192b, 0x2b2b2b2b08082b08, 0x2b2b2b2b08082b2b, 0x2b2b2b2b082b0808, + 0x2b2b2b2b082b082b, 0x2b2b2b2b082b2b08, 0x2b2b2b2b2b082b08, 0x2b2b2b2b2b2b2b2b, +}; + constexpr constant static uint32_t iq3xxs_grid[256] = { 0x04040404, 0x04040414, 0x04040424, 0x04040c0c, 0x04040c1c, 0x04040c3e, 0x04041404, 0x04041414, 0x04041c0c, 0x04042414, 0x04043e1c, 0x04043e2c, 0x040c040c, 0x040c041c, 0x040c0c04, 0x040c0c14, @@ -4572,6 +4839,139 @@ kernel void kernel_mul_mv_iq3_s_f32( kernel_mul_mv_iq3_s_f32_impl(src0, src1, dst, ne00, ne01, ne02, ne10, ne12, ne0, ne1, r2, r3, shared_values, tgpig, tiisg, sgitg); } +void kernel_mul_mv_iq2_s_f32_impl( + device const void * src0, + device const float * src1, + device float * dst, + constant int64_t & ne00, + constant int64_t & ne01, + constant int64_t & ne02, + constant int64_t & ne10, + constant int64_t & ne12, + constant int64_t & ne0, + constant int64_t & ne1, + constant uint & r2, + constant uint & r3, + threadgroup int8_t * shared_values [[threadgroup(0)]], + uint3 tgpig[[threadgroup_position_in_grid]], + uint tiisg[[thread_index_in_simdgroup]], + uint sgitg[[simdgroup_index_in_threadgroup]]) { + + const int nb = ne00/QK_K; + const int r0 = tgpig.x; + const int r1 = tgpig.y; + const int im = tgpig.z; + + const int first_row = (r0 * N_SIMDGROUP + sgitg) * N_DST; + const int ib_row = first_row * nb; + + const uint i12 = im%ne12; + const uint i13 = im/ne12; + + const uint offset0 = (i12/r2)*(nb*ne01) + (i13/r3)*(nb*ne01*ne02); + + device const block_iq2_s * x = (device const block_iq2_s *) src0 + ib_row + offset0; + device const float * y = (device const float *) src1 + r1*ne10 + im*ne00*ne1; + + float yl[32]; + float sumf[N_DST]={0.f}, all_sum; + + const int nb32 = nb * (QK_K / 32); + + //threadgroup uint64_t * values = (threadgroup uint64_t *)shared_values; + //{ + // int nval = 32; + // int pos = (32*sgitg + tiisg)*nval; + // for (int i = 0; i < nval; ++i) values[pos + i] = iq2s_grid[pos + i]; + // threadgroup_barrier(mem_flags::mem_threadgroup); + //} + + const int ix = tiisg; + + device const float * y4 = y + 32 * ix; + + for (int ib32 = ix; ib32 < nb32; ib32 += 32) { + + for (int i = 0; i < 32; ++i) { + yl[i] = y4[i]; + } + + const int ibl = ib32 / (QK_K / 32); + const int ib = ib32 % (QK_K / 32); + + device const block_iq2_s * xr = x + ibl; + device const uint8_t * qs = xr->qs + 4 * ib; + device const uint8_t * qh = xr->qh + ib; + device const uint8_t * sc = xr->scales + ib; + device const uint8_t * signs = qs + QK_K/8; + device const half * dh = &xr->d; + + for (int row = 0; row < N_DST; row++) { + + const float db = dh[0]; + const float d1 = db * (0.5f + (sc[0] & 0xf)); + const float d2 = db * (0.5f + (sc[0] >> 4)); + + float2 sum = {0}; + for (int l = 0; l < 2; ++l) { + //const threadgroup uint8_t * grid1 = (const threadgroup uint8_t *)(values + (qs[l+0] | ((qh[0] << (8-2*l)) & 0x300))); + //const threadgroup uint8_t * grid2 = (const threadgroup uint8_t *)(values + (qs[l+2] | ((qh[0] << (4-2*l)) & 0x300))); + constant uint8_t * grid1 = (constant uint8_t *)(iq2s_grid + (qs[l+0] | ((qh[0] << (8-2*l)) & 0x300))); + constant uint8_t * grid2 = (constant uint8_t *)(iq2s_grid + (qs[l+2] | ((qh[0] << (4-2*l)) & 0x300))); + for (int j = 0; j < 8; ++j) { + sum[0] += yl[8*l + j + 0] * grid1[j] * select(1, -1, signs[l+0] & kmask_iq2xs[j]); + sum[1] += yl[8*l + j + 16] * grid2[j] * select(1, -1, signs[l+2] & kmask_iq2xs[j]); + } + } + sumf[row] += d1 * sum[0] + d2 * sum[1]; + + dh += nb*sizeof(block_iq2_s)/2; + qs += nb*sizeof(block_iq2_s); + qh += nb*sizeof(block_iq2_s); + sc += nb*sizeof(block_iq2_s); + signs += nb*sizeof(block_iq2_s); + } + + y4 += 32 * 32; + } + + for (int row = 0; row < N_DST; ++row) { + all_sum = simd_sum(sumf[row]); + if (tiisg == 0) { + dst[r1*ne0 + im*ne0*ne1 + first_row + row] = all_sum * 0.25f; + } + } +} + +[[host_name("kernel_mul_mv_iq2_s_f32")]] +kernel void kernel_mul_mv_iq2_s_f32( + device const void * src0, + device const float * src1, + device float * dst, + constant int64_t & ne00, + constant int64_t & ne01, + constant int64_t & ne02, + constant uint64_t & nb00, + constant uint64_t & nb01, + constant uint64_t & nb02, + constant int64_t & ne10, + constant int64_t & ne11, + constant int64_t & ne12, + constant uint64_t & nb10, + constant uint64_t & nb11, + constant uint64_t & nb12, + constant int64_t & ne0, + constant int64_t & ne1, + constant uint & r2, + constant uint & r3, + threadgroup int8_t * shared_values [[threadgroup(0)]], + uint3 tgpig[[threadgroup_position_in_grid]], + uint tiisg[[thread_index_in_simdgroup]], + uint sgitg[[simdgroup_index_in_threadgroup]]) { + + kernel_mul_mv_iq2_s_f32_impl(src0, src1, dst, ne00, ne01, ne02, ne10, ne12, ne0, ne1, r2, r3, shared_values, tgpig, tiisg, sgitg); +} + void kernel_mul_mv_iq1_s_f32_impl( device const void * src0, device const float * src1, @@ -5188,6 +5588,25 @@ void dequantize_iq3_s(device const block_iq3_s * xb, short il, thread type4x4 & } } +template +void dequantize_iq2_s(device const block_iq2_s * xb, short il, thread type4x4 & reg) { + // il is 0...15 for QK_K = 256 => index of block of 32 is il/2 + const float d = xb->d; + const int ib32 = il/2; + il = il%2; + // il = 0 or 1. il = 0 processes the first 16 quants in a block of 32, il = 1 the second 16 + device const uint8_t * qs = xb->qs + 4*ib32 + 2*il; + device const uint8_t * signs = qs + QK_K/8; + const uint8_t qh = xb->qh[ib32] >> 4*il; + const float dl = d * (0.5f + ((xb->scales[ib32] >> 4*il) & 0xf)) * 0.25f; + constant uint8_t * grid1 = (constant uint8_t *)(iq2s_grid + (qs[0] | ((qh << 8) & 0x300))); + constant uint8_t * grid2 = (constant uint8_t *)(iq2s_grid + (qs[1] | ((qh << 6) & 0x300))); + for (int i = 0; i < 8; ++i) { + reg[i/4+0][i%4] = dl * grid1[i] * select(1, -1, signs[0] & kmask_iq2xs[i]); + reg[i/4+2][i%4] = dl * grid2[i] * select(1, -1, signs[1] & kmask_iq2xs[i]); + } +} + template void dequantize_iq1_s(device const block_iq1_s * xb, short il, thread type4x4 & reg) { // il is 0...15 for QK_K = 256 => index of block of 32 is il/2 @@ -5762,6 +6181,7 @@ template [[host_name("kernel_get_rows_iq2_xxs")]] kernel get_rows_t kernel_get_r template [[host_name("kernel_get_rows_iq2_xs")]] kernel get_rows_t kernel_get_rows; template [[host_name("kernel_get_rows_iq3_xxs")]] kernel get_rows_t kernel_get_rows; template [[host_name("kernel_get_rows_iq3_s")]] kernel get_rows_t kernel_get_rows; +template [[host_name("kernel_get_rows_iq2_s")]] kernel get_rows_t kernel_get_rows; template [[host_name("kernel_get_rows_iq1_s")]] kernel get_rows_t kernel_get_rows; template [[host_name("kernel_get_rows_iq4_nl")]] kernel get_rows_t kernel_get_rows; @@ -5804,6 +6224,7 @@ template [[host_name("kernel_mul_mm_iq2_xxs_f32")]] kernel mat_mm_t kernel_mul_m template [[host_name("kernel_mul_mm_iq2_xs_f32")]] kernel mat_mm_t kernel_mul_mm; template [[host_name("kernel_mul_mm_iq3_xxs_f32")]] kernel mat_mm_t kernel_mul_mm; template [[host_name("kernel_mul_mm_iq3_s_f32")]] kernel mat_mm_t kernel_mul_mm; +template [[host_name("kernel_mul_mm_iq2_s_f32")]] kernel mat_mm_t kernel_mul_mm; template [[host_name("kernel_mul_mm_iq1_s_f32")]] kernel mat_mm_t kernel_mul_mm; template [[host_name("kernel_mul_mm_iq4_nl_f32")]] kernel mat_mm_t kernel_mul_mm; @@ -5858,6 +6279,7 @@ template [[host_name("kernel_mul_mm_id_iq2_xxs_f32")]] kernel mat_mm_id_t kernel template [[host_name("kernel_mul_mm_id_iq2_xs_f32")]] kernel mat_mm_id_t kernel_mul_mm_id; template [[host_name("kernel_mul_mm_id_iq3_xxs_f32")]] kernel mat_mm_id_t kernel_mul_mm_id; template [[host_name("kernel_mul_mm_id_iq3_s_f32")]] kernel mat_mm_id_t kernel_mul_mm_id; +template [[host_name("kernel_mul_mm_id_iq2_s_f32")]] kernel mat_mm_id_t kernel_mul_mm_id; template [[host_name("kernel_mul_mm_id_iq1_s_f32")]] kernel mat_mm_id_t kernel_mul_mm_id; template [[host_name("kernel_mul_mm_id_iq4_nl_f32")]] kernel mat_mm_id_t kernel_mul_mm_id; @@ -6893,6 +7315,71 @@ kernel void kernel_mul_mv_id_iq3_s_f32( sgitg); } +[[host_name("kernel_mul_mv_id_iq2_s_f32")]] +kernel void kernel_mul_mv_id_iq2_s_f32( + device const char * ids, + device const char * src1, + device float * dst, + constant uint64_t & nbi1, + constant int64_t & ne00, + constant int64_t & ne01, + constant int64_t & ne02, + constant uint64_t & nb00, + constant uint64_t & nb01, + constant uint64_t & nb02, + constant int64_t & ne10, + constant int64_t & ne11, + constant int64_t & ne12, + constant int64_t & ne13, + constant uint64_t & nb10, + constant uint64_t & nb11, + constant uint64_t & nb12, + constant int64_t & ne0, + constant int64_t & ne1, + constant uint64_t & nb1, + constant uint & r2, + constant uint & r3, + constant int & idx, + device const char * src00, + device const char * src01, + device const char * src02, + device const char * src03, + device const char * src04, + device const char * src05, + device const char * src06, + device const char * src07, + threadgroup int8_t * shared_values [[threadgroup(0)]], + uint3 tgpig[[threadgroup_position_in_grid]], + uint tiitg[[thread_index_in_threadgroup]], + uint tiisg[[thread_index_in_simdgroup]], + uint sgitg[[simdgroup_index_in_threadgroup]]) { + device const char * src0[8] = {src00, src01, src02, src03, src04, src05, src06, src07}; + + const int64_t bid = tgpig.z/(ne12*ne13); + + tgpig.z = tgpig.z%(ne12*ne13); + + const int32_t id = ((device int32_t *) (ids + bid*nbi1))[idx]; + + kernel_mul_mv_iq2_s_f32_impl( + src0[id], + (device const float *) (src1 + bid*nb11), + dst + bid*ne0, + ne00, + ne01, + ne02, + ne10, + ne12, + ne0, + ne1, + r2, + r3, + shared_values, + tgpig, + tiisg, + sgitg); +} + [[host_name("kernel_mul_mv_id_iq1_s_f32")]] kernel void kernel_mul_mv_id_iq1_s_f32( device const char * ids, diff --git a/ggml-quants.c b/ggml-quants.c index 3d94d166d..ce654f094 100644 --- a/ggml-quants.c +++ b/ggml-quants.c @@ -3495,6 +3495,265 @@ static const uint64_t iq2xs_grid[512] = { 0x2b2b2b2b082b2b08, 0x2b2b2b2b082b2b2b, 0x2b2b2b2b2b190819, 0x2b2b2b2b2b2b2b2b, }; +static const uint64_t iq2s_grid[1024] = { + 0x0808080808080808, 0x080808080808082b, 0x0808080808081919, 0x0808080808082b08, + 0x0808080808082b2b, 0x0808080808190819, 0x0808080808191908, 0x080808080819192b, + 0x0808080808192b19, 0x08080808082b0808, 0x08080808082b082b, 0x08080808082b1919, + 0x08080808082b2b08, 0x0808080819080819, 0x0808080819081908, 0x080808081908192b, + 0x0808080819082b19, 0x0808080819190808, 0x080808081919082b, 0x0808080819191919, + 0x0808080819192b08, 0x08080808192b0819, 0x08080808192b1908, 0x08080808192b192b, + 0x08080808192b2b19, 0x080808082b080808, 0x080808082b08082b, 0x080808082b081919, + 0x080808082b082b08, 0x080808082b190819, 0x080808082b191908, 0x080808082b2b0808, + 0x080808082b2b1919, 0x080808082b2b2b2b, 0x0808081908080819, 0x0808081908081908, + 0x080808190808192b, 0x0808081908082b19, 0x0808081908190808, 0x080808190819082b, + 0x0808081908191919, 0x0808081908192b08, 0x08080819082b0819, 0x08080819082b1908, + 0x0808081919080808, 0x080808191908082b, 0x0808081919081919, 0x0808081919082b08, + 0x0808081919190819, 0x0808081919191908, 0x080808191919192b, 0x0808081919192b19, + 0x08080819192b0808, 0x08080819192b1919, 0x08080819192b2b08, 0x080808192b080819, + 0x080808192b081908, 0x080808192b190808, 0x080808192b19082b, 0x080808192b191919, + 0x080808192b2b0819, 0x080808192b2b1908, 0x0808082b08080808, 0x0808082b0808082b, + 0x0808082b08081919, 0x0808082b08082b08, 0x0808082b08190819, 0x0808082b08191908, + 0x0808082b082b0808, 0x0808082b082b2b2b, 0x0808082b19080819, 0x0808082b19081908, + 0x0808082b1908192b, 0x0808082b19082b19, 0x0808082b19190808, 0x0808082b19191919, + 0x0808082b2b080808, 0x0808082b2b081919, 0x0808082b2b082b2b, 0x0808082b2b191908, + 0x0808082b2b2b082b, 0x0808190808080819, 0x0808190808081908, 0x080819080808192b, + 0x0808190808082b19, 0x0808190808190808, 0x080819080819082b, 0x0808190808191919, + 0x0808190808192b08, 0x08081908082b0819, 0x08081908082b1908, 0x08081908082b192b, + 0x08081908082b2b19, 0x0808190819080808, 0x080819081908082b, 0x0808190819081919, + 0x0808190819082b08, 0x0808190819082b2b, 0x0808190819190819, 0x0808190819191908, + 0x080819081919192b, 0x0808190819192b19, 0x08081908192b0808, 0x08081908192b082b, + 0x08081908192b1919, 0x080819082b080819, 0x080819082b081908, 0x080819082b08192b, + 0x080819082b082b19, 0x080819082b190808, 0x080819082b191919, 0x080819082b192b08, + 0x080819082b2b0819, 0x080819082b2b1908, 0x0808191908080808, 0x080819190808082b, + 0x0808191908081919, 0x0808191908082b08, 0x0808191908082b2b, 0x0808191908190819, + 0x0808191908191908, 0x080819190819192b, 0x0808191908192b19, 0x08081919082b0808, + 0x08081919082b1919, 0x08081919082b2b08, 0x0808191919080819, 0x0808191919081908, + 0x080819191908192b, 0x0808191919082b19, 0x0808191919190808, 0x080819191919082b, + 0x0808191919191919, 0x0808191919192b08, 0x08081919192b0819, 0x08081919192b1908, + 0x080819192b080808, 0x080819192b08082b, 0x080819192b081919, 0x080819192b082b08, + 0x080819192b190819, 0x080819192b191908, 0x080819192b2b0808, 0x0808192b08080819, + 0x0808192b08081908, 0x0808192b0808192b, 0x0808192b08082b19, 0x0808192b08190808, + 0x0808192b08191919, 0x0808192b19080808, 0x0808192b19081919, 0x0808192b19082b08, + 0x0808192b19190819, 0x0808192b19191908, 0x0808192b192b0808, 0x0808192b2b080819, + 0x0808192b2b081908, 0x0808192b2b190808, 0x08082b0808080808, 0x08082b080808082b, + 0x08082b0808081919, 0x08082b0808082b08, 0x08082b0808190819, 0x08082b0808191908, + 0x08082b080819192b, 0x08082b0808192b19, 0x08082b08082b0808, 0x08082b08082b1919, + 0x08082b08082b2b2b, 0x08082b0819080819, 0x08082b0819081908, 0x08082b081908192b, + 0x08082b0819082b19, 0x08082b0819190808, 0x08082b081919082b, 0x08082b0819191919, + 0x08082b0819192b08, 0x08082b08192b0819, 0x08082b08192b1908, 0x08082b082b080808, + 0x08082b082b081919, 0x08082b082b191908, 0x08082b082b2b2b2b, 0x08082b1908080819, + 0x08082b1908081908, 0x08082b1908190808, 0x08082b190819082b, 0x08082b1908191919, + 0x08082b1908192b08, 0x08082b19082b0819, 0x08082b1919080808, 0x08082b1919081919, + 0x08082b1919082b08, 0x08082b1919190819, 0x08082b1919191908, 0x08082b19192b0808, + 0x08082b192b080819, 0x08082b192b190808, 0x08082b2b08080808, 0x08082b2b08190819, + 0x08082b2b08191908, 0x08082b2b082b082b, 0x08082b2b082b2b08, 0x08082b2b082b2b2b, + 0x08082b2b19190808, 0x08082b2b2b192b19, 0x0819080808080819, 0x0819080808081908, + 0x081908080808192b, 0x0819080808082b19, 0x0819080808190808, 0x081908080819082b, + 0x0819080808191919, 0x0819080808192b08, 0x08190808082b0819, 0x08190808082b1908, + 0x08190808082b192b, 0x0819080819080808, 0x081908081908082b, 0x0819080819081919, + 0x0819080819082b08, 0x0819080819190819, 0x0819080819191908, 0x081908081919192b, + 0x0819080819192b19, 0x08190808192b0808, 0x08190808192b082b, 0x08190808192b1919, + 0x08190808192b2b08, 0x081908082b080819, 0x081908082b081908, 0x081908082b08192b, + 0x081908082b190808, 0x081908082b191919, 0x081908082b192b08, 0x081908082b2b0819, + 0x081908082b2b1908, 0x0819081908080808, 0x081908190808082b, 0x0819081908081919, + 0x0819081908082b08, 0x0819081908082b2b, 0x0819081908190819, 0x0819081908191908, + 0x081908190819192b, 0x0819081908192b19, 0x08190819082b0808, 0x08190819082b082b, + 0x08190819082b1919, 0x08190819082b2b08, 0x0819081919080819, 0x0819081919081908, + 0x081908191908192b, 0x0819081919082b19, 0x0819081919190808, 0x081908191919082b, + 0x0819081919191919, 0x0819081919192b08, 0x08190819192b0819, 0x08190819192b1908, + 0x081908192b080808, 0x081908192b08082b, 0x081908192b081919, 0x081908192b082b08, + 0x081908192b190819, 0x081908192b191908, 0x0819082b08080819, 0x0819082b08081908, + 0x0819082b08082b19, 0x0819082b08190808, 0x0819082b08191919, 0x0819082b082b0819, + 0x0819082b082b1908, 0x0819082b19080808, 0x0819082b19081919, 0x0819082b19190819, + 0x0819082b19191908, 0x0819082b2b080819, 0x0819082b2b081908, 0x0819082b2b190808, + 0x0819190808080808, 0x081919080808082b, 0x0819190808081919, 0x0819190808082b08, + 0x0819190808190819, 0x0819190808191908, 0x081919080819192b, 0x0819190808192b19, + 0x08191908082b0808, 0x08191908082b1919, 0x08191908082b2b08, 0x0819190819080819, + 0x0819190819081908, 0x081919081908192b, 0x0819190819082b19, 0x0819190819190808, + 0x081919081919082b, 0x0819190819191919, 0x0819190819192b08, 0x08191908192b0819, + 0x08191908192b1908, 0x081919082b080808, 0x081919082b08082b, 0x081919082b081919, + 0x081919082b082b08, 0x081919082b190819, 0x081919082b191908, 0x081919082b2b0808, + 0x0819191908080819, 0x0819191908081908, 0x081919190808192b, 0x0819191908082b19, + 0x0819191908190808, 0x081919190819082b, 0x0819191908191919, 0x0819191908192b08, + 0x08191919082b0819, 0x08191919082b1908, 0x0819191919080808, 0x081919191908082b, + 0x0819191919081919, 0x0819191919082b08, 0x0819191919190819, 0x0819191919191908, + 0x08191919192b0808, 0x081919192b080819, 0x081919192b081908, 0x081919192b190808, + 0x0819192b08080808, 0x0819192b08081919, 0x0819192b08082b08, 0x0819192b08190819, + 0x0819192b08191908, 0x0819192b082b0808, 0x0819192b19080819, 0x0819192b19081908, + 0x0819192b19190808, 0x0819192b2b080808, 0x0819192b2b2b2b2b, 0x08192b0808080819, + 0x08192b0808081908, 0x08192b080808192b, 0x08192b0808082b19, 0x08192b0808190808, + 0x08192b0808191919, 0x08192b0808192b08, 0x08192b08082b0819, 0x08192b0819080808, + 0x08192b081908082b, 0x08192b0819081919, 0x08192b0819082b08, 0x08192b0819190819, + 0x08192b0819191908, 0x08192b08192b0808, 0x08192b082b080819, 0x08192b082b081908, + 0x08192b1908080808, 0x08192b190808082b, 0x08192b1908081919, 0x08192b1908082b08, + 0x08192b1908190819, 0x08192b1908191908, 0x08192b19082b0808, 0x08192b1919080819, + 0x08192b1919081908, 0x08192b1919190808, 0x08192b19192b2b19, 0x08192b192b2b082b, + 0x08192b2b08081908, 0x08192b2b08190808, 0x08192b2b19080808, 0x08192b2b1919192b, + 0x082b080808080808, 0x082b08080808082b, 0x082b080808081919, 0x082b080808082b08, + 0x082b080808190819, 0x082b080808191908, 0x082b08080819192b, 0x082b080808192b19, + 0x082b0808082b0808, 0x082b0808082b1919, 0x082b0808082b2b2b, 0x082b080819080819, + 0x082b080819081908, 0x082b080819190808, 0x082b08081919082b, 0x082b080819191919, + 0x082b0808192b1908, 0x082b08082b080808, 0x082b08082b082b2b, 0x082b08082b191908, + 0x082b08082b2b2b2b, 0x082b081908080819, 0x082b081908081908, 0x082b081908190808, + 0x082b08190819082b, 0x082b081908191919, 0x082b0819082b0819, 0x082b081919080808, + 0x082b08191908082b, 0x082b081919081919, 0x082b081919190819, 0x082b081919191908, + 0x082b0819192b0808, 0x082b08192b080819, 0x082b08192b081908, 0x082b08192b190808, + 0x082b082b08080808, 0x082b082b08082b2b, 0x082b082b082b082b, 0x082b082b082b2b08, + 0x082b082b082b2b2b, 0x082b082b19081908, 0x082b082b19190808, 0x082b082b2b082b08, + 0x082b082b2b082b2b, 0x082b082b2b2b2b08, 0x082b190808080819, 0x082b190808081908, + 0x082b19080808192b, 0x082b190808082b19, 0x082b190808190808, 0x082b190808191919, + 0x082b190808192b08, 0x082b1908082b0819, 0x082b1908082b1908, 0x082b190819080808, + 0x082b19081908082b, 0x082b190819081919, 0x082b190819082b08, 0x082b190819190819, + 0x082b190819191908, 0x082b1908192b0808, 0x082b19082b080819, 0x082b19082b081908, + 0x082b19082b190808, 0x082b191908080808, 0x082b191908081919, 0x082b191908082b08, + 0x082b191908190819, 0x082b191908191908, 0x082b1919082b0808, 0x082b191919080819, + 0x082b191919081908, 0x082b191919190808, 0x082b1919192b192b, 0x082b19192b080808, + 0x082b192b08080819, 0x082b192b08081908, 0x082b192b08190808, 0x082b192b19080808, + 0x082b192b19192b19, 0x082b2b0808080808, 0x082b2b0808081919, 0x082b2b0808190819, + 0x082b2b0808191908, 0x082b2b0819080819, 0x082b2b0819081908, 0x082b2b0819190808, + 0x082b2b082b082b2b, 0x082b2b082b2b2b2b, 0x082b2b1908080819, 0x082b2b1908081908, + 0x082b2b1908190808, 0x082b2b192b191919, 0x082b2b2b08082b2b, 0x082b2b2b082b082b, + 0x082b2b2b192b1908, 0x082b2b2b2b082b08, 0x082b2b2b2b082b2b, 0x1908080808080819, + 0x1908080808081908, 0x190808080808192b, 0x1908080808082b19, 0x1908080808190808, + 0x190808080819082b, 0x1908080808191919, 0x1908080808192b08, 0x1908080808192b2b, + 0x19080808082b0819, 0x19080808082b1908, 0x19080808082b192b, 0x1908080819080808, + 0x190808081908082b, 0x1908080819081919, 0x1908080819082b08, 0x1908080819082b2b, + 0x1908080819190819, 0x1908080819191908, 0x190808081919192b, 0x1908080819192b19, + 0x19080808192b0808, 0x19080808192b082b, 0x19080808192b1919, 0x190808082b080819, + 0x190808082b081908, 0x190808082b190808, 0x190808082b191919, 0x190808082b192b08, + 0x190808082b2b0819, 0x190808082b2b1908, 0x1908081908080808, 0x190808190808082b, + 0x1908081908081919, 0x1908081908082b08, 0x1908081908190819, 0x1908081908191908, + 0x190808190819192b, 0x1908081908192b19, 0x19080819082b0808, 0x19080819082b082b, + 0x19080819082b1919, 0x1908081919080819, 0x1908081919081908, 0x190808191908192b, + 0x1908081919082b19, 0x1908081919190808, 0x190808191919082b, 0x1908081919191919, + 0x1908081919192b08, 0x19080819192b0819, 0x19080819192b1908, 0x190808192b080808, + 0x190808192b08082b, 0x190808192b081919, 0x190808192b082b08, 0x190808192b190819, + 0x190808192b191908, 0x190808192b2b0808, 0x1908082b08080819, 0x1908082b08081908, + 0x1908082b08190808, 0x1908082b0819082b, 0x1908082b08191919, 0x1908082b08192b08, + 0x1908082b082b1908, 0x1908082b19080808, 0x1908082b19081919, 0x1908082b19082b08, + 0x1908082b19190819, 0x1908082b19191908, 0x1908082b192b0808, 0x1908082b2b080819, + 0x1908082b2b081908, 0x1908190808080808, 0x190819080808082b, 0x1908190808081919, + 0x1908190808082b08, 0x1908190808082b2b, 0x1908190808190819, 0x1908190808191908, + 0x190819080819192b, 0x1908190808192b19, 0x19081908082b0808, 0x19081908082b082b, + 0x19081908082b1919, 0x19081908082b2b08, 0x1908190819080819, 0x1908190819081908, + 0x190819081908192b, 0x1908190819082b19, 0x1908190819190808, 0x190819081919082b, + 0x1908190819191919, 0x1908190819192b08, 0x19081908192b0819, 0x19081908192b1908, + 0x190819082b080808, 0x190819082b08082b, 0x190819082b081919, 0x190819082b082b08, + 0x190819082b190819, 0x190819082b191908, 0x190819082b2b0808, 0x1908191908080819, + 0x1908191908081908, 0x190819190808192b, 0x1908191908082b19, 0x1908191908190808, + 0x190819190819082b, 0x1908191908191919, 0x1908191908192b08, 0x19081919082b0819, + 0x19081919082b1908, 0x1908191919080808, 0x190819191908082b, 0x1908191919081919, + 0x1908191919082b08, 0x1908191919190819, 0x1908191919191908, 0x19081919192b0808, + 0x19081919192b2b2b, 0x190819192b080819, 0x190819192b081908, 0x190819192b190808, + 0x1908192b08080808, 0x1908192b0808082b, 0x1908192b08081919, 0x1908192b08082b08, + 0x1908192b08190819, 0x1908192b08191908, 0x1908192b082b0808, 0x1908192b19080819, + 0x1908192b19081908, 0x1908192b19190808, 0x1908192b2b080808, 0x1908192b2b2b1919, + 0x19082b0808080819, 0x19082b0808081908, 0x19082b0808082b19, 0x19082b0808190808, + 0x19082b080819082b, 0x19082b0808191919, 0x19082b0808192b08, 0x19082b08082b0819, + 0x19082b08082b1908, 0x19082b0819080808, 0x19082b081908082b, 0x19082b0819081919, + 0x19082b0819082b08, 0x19082b0819190819, 0x19082b0819191908, 0x19082b08192b0808, + 0x19082b082b081908, 0x19082b082b190808, 0x19082b1908080808, 0x19082b190808082b, + 0x19082b1908081919, 0x19082b1908082b08, 0x19082b1908190819, 0x19082b1908191908, + 0x19082b19082b0808, 0x19082b1919080819, 0x19082b1919081908, 0x19082b1919190808, + 0x19082b192b080808, 0x19082b192b19192b, 0x19082b2b08080819, 0x19082b2b08081908, + 0x19082b2b08190808, 0x19082b2b19080808, 0x1919080808080808, 0x191908080808082b, + 0x1919080808081919, 0x1919080808082b08, 0x1919080808190819, 0x1919080808191908, + 0x191908080819192b, 0x1919080808192b19, 0x19190808082b0808, 0x19190808082b082b, + 0x19190808082b1919, 0x19190808082b2b08, 0x1919080819080819, 0x1919080819081908, + 0x191908081908192b, 0x1919080819082b19, 0x1919080819190808, 0x191908081919082b, + 0x1919080819191919, 0x1919080819192b08, 0x19190808192b0819, 0x19190808192b1908, + 0x191908082b080808, 0x191908082b08082b, 0x191908082b081919, 0x191908082b082b08, + 0x191908082b190819, 0x191908082b191908, 0x1919081908080819, 0x1919081908081908, + 0x191908190808192b, 0x1919081908082b19, 0x1919081908190808, 0x191908190819082b, + 0x1919081908191919, 0x1919081908192b08, 0x19190819082b0819, 0x19190819082b1908, + 0x1919081919080808, 0x191908191908082b, 0x1919081919081919, 0x1919081919082b08, + 0x1919081919190819, 0x1919081919191908, 0x19190819192b0808, 0x191908192b080819, + 0x191908192b081908, 0x191908192b190808, 0x1919082b08080808, 0x1919082b08081919, + 0x1919082b08082b08, 0x1919082b08190819, 0x1919082b08191908, 0x1919082b082b0808, + 0x1919082b19080819, 0x1919082b19081908, 0x1919082b19190808, 0x1919082b192b2b19, + 0x1919082b2b080808, 0x1919190808080819, 0x1919190808081908, 0x191919080808192b, + 0x1919190808082b19, 0x1919190808190808, 0x191919080819082b, 0x1919190808191919, + 0x1919190808192b08, 0x19191908082b0819, 0x19191908082b1908, 0x1919190819080808, + 0x191919081908082b, 0x1919190819081919, 0x1919190819082b08, 0x1919190819190819, + 0x1919190819191908, 0x19191908192b0808, 0x191919082b080819, 0x191919082b081908, + 0x191919082b190808, 0x1919191908080808, 0x191919190808082b, 0x1919191908081919, + 0x1919191908082b08, 0x1919191908190819, 0x1919191908191908, 0x19191919082b0808, + 0x1919191919080819, 0x1919191919081908, 0x1919191919190808, 0x191919192b080808, + 0x1919192b08080819, 0x1919192b08081908, 0x1919192b08190808, 0x1919192b082b192b, + 0x1919192b19080808, 0x19192b0808080808, 0x19192b080808082b, 0x19192b0808081919, + 0x19192b0808082b08, 0x19192b0808190819, 0x19192b0808191908, 0x19192b08082b0808, + 0x19192b0819080819, 0x19192b0819081908, 0x19192b0819190808, 0x19192b0819192b2b, + 0x19192b082b080808, 0x19192b1908080819, 0x19192b1908081908, 0x19192b1908190808, + 0x19192b1919080808, 0x19192b2b08080808, 0x19192b2b08192b19, 0x19192b2b2b081919, + 0x19192b2b2b2b2b08, 0x192b080808080819, 0x192b080808081908, 0x192b08080808192b, + 0x192b080808190808, 0x192b08080819082b, 0x192b080808191919, 0x192b080808192b08, + 0x192b0808082b0819, 0x192b0808082b1908, 0x192b080819080808, 0x192b080819081919, + 0x192b080819082b08, 0x192b080819190819, 0x192b080819191908, 0x192b0808192b0808, + 0x192b08082b081908, 0x192b08082b190808, 0x192b081908080808, 0x192b08190808082b, + 0x192b081908081919, 0x192b081908082b08, 0x192b081908190819, 0x192b081908191908, + 0x192b0819082b0808, 0x192b081919080819, 0x192b081919081908, 0x192b081919190808, + 0x192b08192b080808, 0x192b08192b192b19, 0x192b082b08081908, 0x192b082b08190808, + 0x192b082b19080808, 0x192b082b1919192b, 0x192b082b2b2b0819, 0x192b190808080808, + 0x192b190808081919, 0x192b190808082b08, 0x192b190808190819, 0x192b190808191908, + 0x192b1908082b0808, 0x192b190819080819, 0x192b190819081908, 0x192b190819190808, + 0x192b19082b080808, 0x192b191908080819, 0x192b191908081908, 0x192b191908190808, + 0x192b191919080808, 0x192b191919082b2b, 0x192b1919192b2b08, 0x192b19192b19082b, + 0x192b192b08080808, 0x192b192b2b191908, 0x192b2b0808080819, 0x192b2b0808081908, + 0x192b2b0808190808, 0x192b2b08192b1919, 0x192b2b082b192b08, 0x192b2b1908080808, + 0x192b2b19082b2b2b, 0x192b2b2b1908082b, 0x192b2b2b2b2b0819, 0x2b08080808080808, + 0x2b0808080808082b, 0x2b08080808081919, 0x2b08080808082b08, 0x2b08080808190819, + 0x2b08080808191908, 0x2b08080808192b19, 0x2b080808082b0808, 0x2b080808082b1919, + 0x2b08080819080819, 0x2b08080819081908, 0x2b08080819190808, 0x2b0808081919082b, + 0x2b08080819191919, 0x2b08080819192b08, 0x2b080808192b0819, 0x2b0808082b080808, + 0x2b0808082b081919, 0x2b0808082b190819, 0x2b0808082b191908, 0x2b08081908080819, + 0x2b08081908081908, 0x2b08081908082b19, 0x2b08081908190808, 0x2b0808190819082b, + 0x2b08081908191919, 0x2b08081908192b08, 0x2b080819082b0819, 0x2b080819082b1908, + 0x2b08081919080808, 0x2b0808191908082b, 0x2b08081919081919, 0x2b08081919082b08, + 0x2b08081919190819, 0x2b08081919191908, 0x2b0808192b080819, 0x2b0808192b081908, + 0x2b0808192b190808, 0x2b0808192b2b2b19, 0x2b08082b08080808, 0x2b08082b08081919, + 0x2b08082b08082b2b, 0x2b08082b08190819, 0x2b08082b08191908, 0x2b08082b19080819, + 0x2b08082b19081908, 0x2b08082b19190808, 0x2b08190808080819, 0x2b08190808081908, + 0x2b0819080808192b, 0x2b08190808082b19, 0x2b08190808190808, 0x2b0819080819082b, + 0x2b08190808191919, 0x2b08190808192b08, 0x2b081908082b0819, 0x2b08190819080808, + 0x2b0819081908082b, 0x2b08190819081919, 0x2b08190819082b08, 0x2b08190819190819, + 0x2b08190819191908, 0x2b081908192b0808, 0x2b0819082b080819, 0x2b0819082b081908, + 0x2b0819082b190808, 0x2b08191908080808, 0x2b0819190808082b, 0x2b08191908081919, + 0x2b08191908082b08, 0x2b08191908190819, 0x2b08191908191908, 0x2b081919082b0808, + 0x2b08191919080819, 0x2b08191919081908, 0x2b08191919190808, 0x2b0819192b080808, + 0x2b0819192b082b2b, 0x2b08192b08080819, 0x2b08192b08081908, 0x2b08192b08190808, + 0x2b08192b082b2b19, 0x2b08192b19080808, 0x2b082b0808080808, 0x2b082b0808081919, + 0x2b082b0808190819, 0x2b082b0808191908, 0x2b082b0819080819, 0x2b082b0819081908, + 0x2b082b0819190808, 0x2b082b082b2b082b, 0x2b082b1908080819, 0x2b082b1908081908, + 0x2b082b1919080808, 0x2b082b19192b1919, 0x2b082b2b082b082b, 0x2b082b2b19192b08, + 0x2b082b2b19192b2b, 0x2b082b2b2b08082b, 0x2b082b2b2b2b082b, 0x2b19080808080819, + 0x2b19080808081908, 0x2b19080808082b19, 0x2b19080808190808, 0x2b1908080819082b, + 0x2b19080808191919, 0x2b19080808192b08, 0x2b190808082b1908, 0x2b19080819080808, + 0x2b1908081908082b, 0x2b19080819081919, 0x2b19080819082b08, 0x2b19080819190819, + 0x2b19080819191908, 0x2b190808192b0808, 0x2b1908082b080819, 0x2b1908082b081908, + 0x2b1908082b190808, 0x2b19081908080808, 0x2b19081908081919, 0x2b19081908190819, + 0x2b19081908191908, 0x2b19081919080819, 0x2b19081919081908, 0x2b19081919190808, + 0x2b19081919192b2b, 0x2b19082b08080819, 0x2b19082b08081908, 0x2b19082b08190808, + 0x2b19082b19080808, 0x2b19082b2b2b192b, 0x2b19190808080808, 0x2b1919080808082b, + 0x2b19190808081919, 0x2b19190808082b08, 0x2b19190808190819, 0x2b19190808191908, + 0x2b191908082b0808, 0x2b19190819080819, 0x2b19190819081908, 0x2b19190819190808, + 0x2b1919082b080808, 0x2b1919082b19192b, 0x2b19191908080819, 0x2b19191908081908, + 0x2b19191908190808, 0x2b19191919080808, 0x2b1919192b192b08, 0x2b1919192b2b0819, + 0x2b19192b08080808, 0x2b19192b1908192b, 0x2b19192b192b1908, 0x2b192b0808080819, + 0x2b192b0808081908, 0x2b192b0808190808, 0x2b192b08082b192b, 0x2b192b0819080808, + 0x2b192b082b2b2b19, 0x2b192b1908080808, 0x2b192b1919082b19, 0x2b192b191919082b, + 0x2b192b2b2b190808, 0x2b2b080808080808, 0x2b2b080808081919, 0x2b2b080808082b2b, + 0x2b2b080808191908, 0x2b2b0808082b082b, 0x2b2b0808082b2b2b, 0x2b2b080819080819, + 0x2b2b080819081908, 0x2b2b080819190808, 0x2b2b08082b2b082b, 0x2b2b08082b2b2b2b, + 0x2b2b081919080808, 0x2b2b0819192b1919, 0x2b2b082b0808082b, 0x2b2b082b08082b2b, + 0x2b2b082b082b082b, 0x2b2b082b082b2b08, 0x2b2b082b082b2b2b, 0x2b2b082b2b08082b, + 0x2b2b082b2b082b08, 0x2b2b082b2b082b2b, 0x2b2b082b2b2b2b08, 0x2b2b190808080819, + 0x2b2b190808081908, 0x2b2b190808190808, 0x2b2b190819080808, 0x2b2b19082b082b19, + 0x2b2b19082b2b1908, 0x2b2b191908080808, 0x2b2b191908192b19, 0x2b2b192b19190819, + 0x2b2b2b0808082b2b, 0x2b2b2b08082b2b08, 0x2b2b2b082b2b082b, 0x2b2b2b1919191908, + 0x2b2b2b192b08192b, 0x2b2b2b2b08082b08, 0x2b2b2b2b08082b2b, 0x2b2b2b2b082b0808, + 0x2b2b2b2b082b082b, 0x2b2b2b2b082b2b08, 0x2b2b2b2b2b082b08, 0x2b2b2b2b2b2b2b2b, +}; + static const uint32_t iq3xxs_grid[256] = { 0x04040404, 0x04040414, 0x04040424, 0x04040c0c, 0x04040c1c, 0x04040c3e, 0x04041404, 0x04041414, 0x04041c0c, 0x04042414, 0x04043e1c, 0x04043e2c, 0x040c040c, 0x040c041c, 0x040c0c04, 0x040c0c14, @@ -3796,6 +4055,38 @@ void dequantize_row_iq2_xs(const block_iq2_xs * restrict x, float * restrict y, } } +// ====================== 2.5625 bpw (de)-quantization + +void dequantize_row_iq2_s(const block_iq2_s * restrict x, float * restrict y, int k) { + assert(k % QK_K == 0); + const int nb = k / QK_K; + + float db[2]; + + for (int i = 0; i < nb; i++) { + + const float d = GGML_FP16_TO_FP32(x[i].d); + const uint8_t * qs = x[i].qs; + const uint8_t * qh = x[i].qh; + const uint8_t * signs = qs + QK_K/8; + + for (int ib32 = 0; ib32 < QK_K/32; ++ib32) { + db[0] = d * (0.5f + (x[i].scales[ib32] & 0xf)) * 0.25f; + db[1] = d * (0.5f + (x[i].scales[ib32] >> 4)) * 0.25f; + for (int l = 0; l < 4; ++l) { + const float dl = db[l/2]; + const uint8_t * grid = (const uint8_t *)(iq2s_grid + (qs[l] | (qh[ib32] << (8-2*l) & 0x300))); + for (int j = 0; j < 8; ++j) { + y[j] = dl * grid[j] * (signs[l] & kmask_iq2xs[j] ? -1.f : 1.f); + } + y += 8; + } + qs += 4; + signs += 4; + } + } +} + // ====================== 3.0625 bpw (de)-quantization void dequantize_row_iq3_xxs(const block_iq3_xxs * restrict x, float * restrict y, int k) { @@ -9330,6 +9621,210 @@ void ggml_vec_dot_iq2_xs_q8_K(int n, float * restrict s, size_t bs, const void * #endif } +void ggml_vec_dot_iq2_s_q8_K(int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) { + assert(n % QK_K == 0); + assert(nrc == 1); + UNUSED(nrc); + UNUSED(bx); + UNUSED(by); + UNUSED(bs); + + const block_iq2_s * restrict x = vx; + const block_q8_K * restrict y = vy; + + const int nb = n / QK_K; + +#if defined(__ARM_NEON) + + static const uint8_t k_mask1[32] = {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03 + }; + + static const uint8_t k_mask2[16] = {0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80,}; + + const uint8x16x2_t mask1 = vld1q_u8_x2(k_mask1); + const uint8x16_t mask2 = vld1q_u8(k_mask2); + const uint8x16_t m1 = vdupq_n_u8(1); + const int32x4_t vzero = vdupq_n_s32(0); + + uint8x16x2_t vs; + ggml_int8x16x4_t q2s; + ggml_int8x16x4_t q8b; + + float sumf = 0; + for (int i = 0; i < nb; ++i) { + + const float d = GGML_FP16_TO_FP32(x[i].d) * y[i].d; + + const uint8_t * restrict qs = x[i].qs; + const uint8_t * restrict qh = x[i].qh; + const uint16_t * restrict signs = (const uint16_t *)(x[i].qs + QK_K/8); + const int8_t * restrict q8 = y[i].qs; + + int sumi1 = 0, sumi2 = 0; + for (int ib32 = 0; ib32 < QK_K/32; ib32 += 2) { + q8b = ggml_vld1q_s8_x4(q8); q8 += 64; + q2s.val[0] = vcombine_s8(vld1_s8((const int8_t *)(iq2s_grid + (qs[0] | ((qh[ib32+0] << 8) & 0x300)))), + vld1_s8((const int8_t *)(iq2s_grid + (qs[1] | ((qh[ib32+0] << 6) & 0x300))))); + q2s.val[1] = vcombine_s8(vld1_s8((const int8_t *)(iq2s_grid + (qs[2] | ((qh[ib32+0] << 4) & 0x300)))), + vld1_s8((const int8_t *)(iq2s_grid + (qs[3] | ((qh[ib32+0] << 2) & 0x300))))); + q2s.val[2] = vcombine_s8(vld1_s8((const int8_t *)(iq2s_grid + (qs[4] | ((qh[ib32+1] << 8) & 0x300)))), + vld1_s8((const int8_t *)(iq2s_grid + (qs[5] | ((qh[ib32+1] << 6) & 0x300))))); + q2s.val[3] = vcombine_s8(vld1_s8((const int8_t *)(iq2s_grid + (qs[6] | ((qh[ib32+1] << 4) & 0x300)))), + vld1_s8((const int8_t *)(iq2s_grid + (qs[7] | ((qh[ib32+1] << 2) & 0x300))))); + qs += 8; + + vs.val[0] = vreinterpretq_u8_u32(vdupq_n_u32(signs[0] | (signs[1] << 16))); + vs.val[1] = vandq_u8(vqtbl1q_u8(vs.val[0], mask1.val[1]), mask2); + vs.val[0] = vandq_u8(vqtbl1q_u8(vs.val[0], mask1.val[0]), mask2); + vs.val[0] = vceqq_u8(vs.val[0], mask2); + vs.val[1] = vceqq_u8(vs.val[1], mask2); + + q2s.val[0] = vmulq_s8(vreinterpretq_s8_u8(vorrq_u8(vs.val[0], m1)), q2s.val[0]); + q2s.val[1] = vmulq_s8(vreinterpretq_s8_u8(vorrq_u8(vs.val[1], m1)), q2s.val[1]); + + vs.val[0] = vreinterpretq_u8_u32(vdupq_n_u32(signs[2] | (signs[3] << 16))); + vs.val[1] = vandq_u8(vqtbl1q_u8(vs.val[0], mask1.val[1]), mask2); + vs.val[0] = vandq_u8(vqtbl1q_u8(vs.val[0], mask1.val[0]), mask2); + vs.val[0] = vceqq_u8(vs.val[0], mask2); + vs.val[1] = vceqq_u8(vs.val[1], mask2); + + signs += 4; + + q2s.val[2] = vmulq_s8(vreinterpretq_s8_u8(vorrq_u8(vs.val[0], m1)), q2s.val[2]); + q2s.val[3] = vmulq_s8(vreinterpretq_s8_u8(vorrq_u8(vs.val[1], m1)), q2s.val[3]); + + const int32x4_t p1 = ggml_vdotq_s32(vzero, q2s.val[0], q8b.val[0]); + const int32x4_t p2 = ggml_vdotq_s32(vzero, q2s.val[1], q8b.val[1]); + const int32x4_t p3 = ggml_vdotq_s32(vzero, q2s.val[2], q8b.val[2]); + const int32x4_t p4 = ggml_vdotq_s32(vzero, q2s.val[3], q8b.val[3]); + + sumi1 += vaddvq_s32(p1) * (1 + 2*(x[i].scales[ib32+0] & 0xf)); + sumi2 += vaddvq_s32(p2) * (1 + 2*(x[i].scales[ib32+0] >> 4)); + sumi1 += vaddvq_s32(p3) * (1 + 2*(x[i].scales[ib32+1] & 0xf)); + sumi2 += vaddvq_s32(p4) * (1 + 2*(x[i].scales[ib32+1] >> 4)); + } + sumf += d*(sumi1 + sumi2); + } + + *s = 0.125f * sumf; + +#elif defined(__AVX2__) + + static const uint8_t k_mask1[32] = {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03 + }; + + static const uint8_t k_mask2[32] = {0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, + 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, + }; + + const __m128i m4 = _mm_set1_epi8(0xf); + const __m128i m1 = _mm_set1_epi8(1); + + const __m256i mask1 = _mm256_loadu_si256((const __m256i*)k_mask1); + const __m256i mask2 = _mm256_loadu_si256((const __m256i*)k_mask2); + + uint64_t aux64; + + __m256 accumf = _mm256_setzero_ps(); + for (int i = 0; i < nb; ++i) { + const float d = GGML_FP16_TO_FP32(x[i].d) * y[i].d; + const uint8_t * restrict qs = x[i].qs; + const uint8_t * restrict qh = x[i].qh; + const uint16_t * restrict signs = (const uint16_t *)(x[i].qs + QK_K/8); + const int8_t * restrict q8 = y[i].qs; + + memcpy(&aux64, x[i].scales, 8); + const __m128i scales8 = _mm_add_epi8(_mm_slli_epi16(_mm_and_si128(_mm_set_epi64x(aux64 >> 4, aux64), m4), 1), m1); + const __m256i scales16 = _mm256_cvtepi8_epi16(scales8); // 0 2 4 6 8 10 12 14 1 3 5 7 9 11 13 15 + + __m256i sumi1 = _mm256_setzero_si256(); + __m256i sumi2 = _mm256_setzero_si256(); + for (int ib32 = 0; ib32 < QK_K/32; ib32 += 2) { + const __m256i q8_1 = _mm256_loadu_si256((const __m256i *)q8); q8 += 32; + const __m256i q8_2 = _mm256_loadu_si256((const __m256i *)q8); q8 += 32; + const __m256i q2_1 = _mm256_set_epi64x(iq2s_grid[qs[3] | ((qh[ib32+0] << 2) & 0x300)], + iq2s_grid[qs[2] | ((qh[ib32+0] << 4) & 0x300)], + iq2s_grid[qs[1] | ((qh[ib32+0] << 6) & 0x300)], + iq2s_grid[qs[0] | ((qh[ib32+0] << 8) & 0x300)]); + const __m256i q2_2 = _mm256_set_epi64x(iq2s_grid[qs[7] | ((qh[ib32+1] << 2) & 0x300)], + iq2s_grid[qs[6] | ((qh[ib32+1] << 4) & 0x300)], + iq2s_grid[qs[5] | ((qh[ib32+1] << 6) & 0x300)], + iq2s_grid[qs[4] | ((qh[ib32+1] << 8) & 0x300)]); + qs += 8; + + __m256i aux256 = _mm256_set1_epi32(signs[0] | (signs[1] << 16)); + aux256 = _mm256_and_si256(_mm256_shuffle_epi8(aux256,mask1), mask2); + const __m256i s2_1 = _mm256_cmpeq_epi8(aux256, mask2); + const __m256i q8s_1 = _mm256_sub_epi8(_mm256_xor_si256(s2_1, q8_1), s2_1); + + aux256 = _mm256_set1_epi32(signs[2] | (signs[3] << 16)); + aux256 = _mm256_and_si256(_mm256_shuffle_epi8(aux256,mask1), mask2); + const __m256i s2_2 = _mm256_cmpeq_epi8(aux256, mask2); + const __m256i q8s_2 = _mm256_sub_epi8(_mm256_xor_si256(s2_2, q8_2), s2_2); + + signs += 4; + + const __m256i dot1 = _mm256_maddubs_epi16(q2_1, q8s_1); // blocks 2*ib32+0, 2*ib32+1 + const __m256i dot2 = _mm256_maddubs_epi16(q2_2, q8s_2); // blocks 2*ib32+2, 2*ib32+3 + + const __m256i p1 = _mm256_madd_epi16(dot1, _mm256_shuffle_epi8(scales16, get_scale_shuffle_k4(ib32+0))); + const __m256i p2 = _mm256_madd_epi16(dot2, _mm256_shuffle_epi8(scales16, get_scale_shuffle_k4(ib32+1))); + sumi1 = _mm256_add_epi32(sumi1, p1); + sumi2 = _mm256_add_epi32(sumi2, p2); + } + + accumf = _mm256_fmadd_ps(_mm256_set1_ps(d), _mm256_cvtepi32_ps(_mm256_add_epi32(sumi1, sumi2)), accumf); + + } + + *s = 0.125f * hsum_float_8(accumf); + +#else + + float sumf = 0; + for (int i = 0; i < nb; i++) { + + const float d = GGML_FP16_TO_FP32(x[i].d) * y[i].d; + const int8_t * q8 = y[i].qs; + const uint8_t * qs = x[i].qs; + const uint8_t * qh = x[i].qh; + const uint8_t * signs = qs + QK_K/8; + + int bsum = 0; + for (int ib32 = 0; ib32 < QK_K/32; ++ib32) { + int ls1 = 1 + 2*(x[i].scales[ib32] & 0xf); + int ls2 = 1 + 2*(x[i].scales[ib32] >> 4); + int sumi1 = 0, sumi2 = 0; + for (int l = 0; l < 2; ++l) { + const uint8_t * grid = (const uint8_t *)(iq2s_grid + (qs[l] | (qh[ib32] << (8-2*l) & 0x300))); + for (int j = 0; j < 8; ++j) { + sumi1 += q8[j] * grid[j] * (signs[l] & kmask_iq2xs[j] ? -1 : 1); + } + q8 += 8; + } + for (int l = 2; l < 4; ++l) { + const uint8_t * grid = (const uint8_t *)(iq2s_grid + (qs[l] | (qh[ib32] << (8-2*l) & 0x300))); + for (int j = 0; j < 8; ++j) { + sumi2 += q8[j] * grid[j] * (signs[l] & kmask_iq2xs[j] ? -1 : 1); + } + q8 += 8; + } + bsum += ls1 * sumi1 + ls2 * sumi2; + qs += 4; + signs += 4; + } + + sumf += d * bsum; + } + + *s = 0.125f * sumf; + +#endif + +} + void ggml_vec_dot_iq3_xxs_q8_K(int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) { assert(n % QK_K == 0); assert(nrc == 1); @@ -9934,22 +10429,25 @@ typedef struct { uint16_t * neighbours; } iq2_entry_t; -static iq2_entry_t iq2_data[3] = { +static iq2_entry_t iq2_data[4] = { + {NULL, NULL, NULL}, {NULL, NULL, NULL}, {NULL, NULL, NULL}, {NULL, NULL, NULL}, }; static inline int iq2_data_index(enum ggml_type type) { - GGML_ASSERT(type == GGML_TYPE_IQ2_XXS || type == GGML_TYPE_IQ2_XS || type == GGML_TYPE_IQ1_S); + GGML_ASSERT(type == GGML_TYPE_IQ2_XXS || type == GGML_TYPE_IQ2_XS || type == GGML_TYPE_IQ1_S || type == GGML_TYPE_IQ2_S); return type == GGML_TYPE_IQ2_XXS ? 0 : - type == GGML_TYPE_IQ2_XS ? 1 : 2; + type == GGML_TYPE_IQ2_XS ? 1 : + type == GGML_TYPE_IQ1_S ? 2 : 3; } static inline int iq2_grid_size(enum ggml_type type) { - GGML_ASSERT(type == GGML_TYPE_IQ2_XXS || type == GGML_TYPE_IQ2_XS || type == GGML_TYPE_IQ1_S); + GGML_ASSERT(type == GGML_TYPE_IQ2_XXS || type == GGML_TYPE_IQ2_XS || type == GGML_TYPE_IQ1_S || type == GGML_TYPE_IQ2_S); return type == GGML_TYPE_IQ2_XXS ? 256 : - type == GGML_TYPE_IQ2_XS ? 512 : 512; + type == GGML_TYPE_IQ2_XS ? 512 : + type == GGML_TYPE_IQ1_S ? 512 : 1024; } static int iq2_compare_func(const void * left, const void * right) { @@ -10050,11 +10548,79 @@ void iq2xs_init_impl(enum ggml_type type) { 41557, 41633, 41989, 42021, 42056, 42068, 42074, 42113, 42242, 42265, 42274, 42325, 42340, 42402, 42501, 42512, 42533, 42624, 42632, 42666, 43040, 43093, 43106, 43168, 43176, 43264, 43286, 43345, 43429, 43590, 43618, 43680, }; + static const uint16_t kgrid_2bit_1024[1024] = { + 0, 2, 5, 8, 10, 17, 20, 22, 25, 32, 34, 37, 40, 65, 68, 70, + 73, 80, 82, 85, 88, 97, 100, 102, 105, 128, 130, 133, 136, 145, 148, 160, + 165, 170, 257, 260, 262, 265, 272, 274, 277, 280, 289, 292, 320, 322, 325, 328, + 337, 340, 342, 345, 352, 357, 360, 385, 388, 400, 402, 405, 417, 420, 512, 514, + 517, 520, 529, 532, 544, 554, 577, 580, 582, 585, 592, 597, 640, 645, 650, 660, + 674, 1025, 1028, 1030, 1033, 1040, 1042, 1045, 1048, 1057, 1060, 1062, 1065, 1088, 1090, 1093, + 1096, 1098, 1105, 1108, 1110, 1113, 1120, 1122, 1125, 1153, 1156, 1158, 1161, 1168, 1173, 1176, + 1185, 1188, 1280, 1282, 1285, 1288, 1290, 1297, 1300, 1302, 1305, 1312, 1317, 1320, 1345, 1348, + 1350, 1353, 1360, 1362, 1365, 1368, 1377, 1380, 1408, 1410, 1413, 1416, 1425, 1428, 1440, 1537, + 1540, 1542, 1545, 1552, 1557, 1600, 1605, 1608, 1617, 1620, 1632, 1665, 1668, 1680, 2048, 2050, + 2053, 2056, 2065, 2068, 2070, 2073, 2080, 2085, 2090, 2113, 2116, 2118, 2121, 2128, 2130, 2133, + 2136, 2145, 2148, 2176, 2181, 2196, 2218, 2305, 2308, 2320, 2322, 2325, 2328, 2337, 2368, 2373, + 2376, 2385, 2388, 2400, 2433, 2448, 2560, 2577, 2580, 2594, 2600, 2602, 2640, 2713, 4097, 4100, + 4102, 4105, 4112, 4114, 4117, 4120, 4129, 4132, 4134, 4160, 4162, 4165, 4168, 4177, 4180, 4182, + 4185, 4192, 4194, 4197, 4200, 4225, 4228, 4230, 4240, 4245, 4248, 4257, 4260, 4352, 4354, 4357, + 4360, 4362, 4369, 4372, 4374, 4377, 4384, 4386, 4389, 4392, 4417, 4420, 4422, 4425, 4432, 4434, + 4437, 4440, 4449, 4452, 4480, 4482, 4485, 4488, 4497, 4500, 4609, 4612, 4617, 4624, 4629, 4641, + 4644, 4672, 4677, 4689, 4692, 4737, 4740, 4752, 5120, 5122, 5125, 5128, 5137, 5140, 5142, 5145, + 5152, 5157, 5160, 5185, 5188, 5190, 5193, 5200, 5202, 5205, 5208, 5217, 5220, 5248, 5250, 5253, + 5256, 5265, 5268, 5280, 5377, 5380, 5382, 5385, 5392, 5394, 5397, 5400, 5409, 5412, 5440, 5442, + 5445, 5448, 5457, 5460, 5472, 5505, 5508, 5520, 5632, 5637, 5640, 5649, 5652, 5664, 5697, 5700, + 5712, 5760, 5802, 6145, 6148, 6150, 6153, 6160, 6165, 6168, 6177, 6208, 6210, 6213, 6216, 6225, + 6228, 6240, 6273, 6276, 6400, 6402, 6405, 6408, 6417, 6420, 6432, 6465, 6468, 6480, 6505, 6562, + 6660, 6672, 6720, 6742, 8192, 8194, 8197, 8200, 8209, 8212, 8214, 8217, 8224, 8229, 8234, 8257, + 8260, 8272, 8274, 8277, 8292, 8320, 8330, 8340, 8362, 8449, 8452, 8464, 8466, 8469, 8481, 8512, + 8514, 8517, 8529, 8532, 8544, 8577, 8580, 8592, 8704, 8714, 8738, 8744, 8746, 8772, 8784, 8840, + 8842, 8872, 9217, 9220, 9222, 9225, 9232, 9237, 9240, 9249, 9252, 9280, 9282, 9285, 9288, 9297, + 9300, 9312, 9345, 9348, 9360, 9472, 9477, 9480, 9489, 9492, 9504, 9537, 9540, 9552, 9574, 9600, + 9729, 9732, 9744, 9792, 9817, 10240, 10245, 10257, 10260, 10305, 10308, 10320, 10378, 10410, 10497, 10500, + 10512, 10645, 10762, 10786, 10852, 10888, 10890, 16385, 16388, 16390, 16393, 16400, 16402, 16405, 16408, 16410, + 16417, 16420, 16422, 16448, 16450, 16453, 16456, 16458, 16465, 16468, 16470, 16473, 16480, 16482, 16485, 16513, + 16516, 16528, 16533, 16536, 16545, 16548, 16640, 16642, 16645, 16648, 16657, 16660, 16662, 16665, 16672, 16674, + 16677, 16705, 16708, 16710, 16713, 16720, 16722, 16725, 16728, 16737, 16740, 16768, 16770, 16773, 16776, 16785, + 16788, 16800, 16897, 16900, 16912, 16914, 16917, 16920, 16932, 16960, 16965, 16968, 16977, 16980, 16992, 17025, + 17028, 17408, 17410, 17413, 17416, 17418, 17425, 17428, 17430, 17433, 17440, 17442, 17445, 17448, 17473, 17476, + 17478, 17481, 17488, 17490, 17493, 17496, 17505, 17508, 17536, 17538, 17541, 17544, 17553, 17556, 17568, 17665, + 17668, 17670, 17673, 17680, 17682, 17685, 17688, 17697, 17700, 17728, 17730, 17733, 17736, 17745, 17748, 17760, + 17770, 17793, 17796, 17808, 17920, 17922, 17925, 17928, 17937, 17940, 17952, 17985, 17988, 18000, 18048, 18085, + 18433, 18436, 18441, 18448, 18450, 18453, 18456, 18465, 18468, 18496, 18498, 18501, 18504, 18513, 18516, 18528, + 18564, 18576, 18688, 18690, 18693, 18696, 18705, 18708, 18720, 18753, 18756, 18768, 18816, 18838, 18945, 18948, + 18960, 19008, 20480, 20482, 20485, 20488, 20497, 20500, 20502, 20505, 20512, 20514, 20517, 20520, 20545, 20548, + 20550, 20553, 20560, 20562, 20565, 20568, 20577, 20580, 20608, 20610, 20613, 20616, 20625, 20628, 20737, 20740, + 20742, 20745, 20752, 20754, 20757, 20760, 20769, 20772, 20800, 20802, 20805, 20808, 20817, 20820, 20832, 20865, + 20868, 20880, 20992, 20997, 21000, 21009, 21012, 21024, 21057, 21060, 21072, 21097, 21120, 21505, 21508, 21510, + 21513, 21520, 21522, 21525, 21528, 21537, 21540, 21568, 21570, 21573, 21576, 21585, 21588, 21600, 21633, 21636, + 21648, 21760, 21762, 21765, 21768, 21777, 21780, 21792, 21825, 21828, 21840, 21888, 22017, 22020, 22032, 22054, + 22080, 22528, 22530, 22533, 22536, 22545, 22548, 22560, 22593, 22596, 22608, 22618, 22656, 22785, 22788, 22800, + 22848, 23040, 23065, 23173, 23208, 24577, 24580, 24582, 24592, 24594, 24597, 24600, 24609, 24612, 24640, 24645, + 24648, 24657, 24660, 24672, 24708, 24720, 24832, 24834, 24837, 24840, 24849, 24852, 24864, 24897, 24900, 24912, + 24960, 24985, 25092, 25104, 25152, 25174, 25249, 25600, 25605, 25608, 25617, 25620, 25632, 25665, 25668, 25680, + 25728, 25857, 25860, 25872, 25920, 25930, 25960, 26002, 26112, 26260, 26625, 26628, 26640, 26725, 26776, 26880, + 26922, 27202, 27297, 32768, 32770, 32773, 32776, 32785, 32788, 32793, 32800, 32805, 32833, 32836, 32848, 32850, + 32853, 32856, 32865, 32896, 32901, 32913, 32916, 33025, 33028, 33033, 33040, 33042, 33045, 33048, 33057, 33060, + 33088, 33090, 33093, 33096, 33105, 33108, 33153, 33156, 33168, 33193, 33280, 33285, 33290, 33297, 33300, 33345, + 33348, 33360, 33793, 33796, 33798, 33801, 33808, 33810, 33813, 33816, 33825, 33856, 33858, 33861, 33864, 33873, + 33876, 33888, 33921, 33924, 33936, 34048, 34050, 34053, 34056, 34065, 34068, 34080, 34113, 34116, 34128, 34176, + 34186, 34305, 34308, 34320, 34345, 34368, 34816, 34821, 34833, 34836, 34881, 34884, 34896, 34978, 35073, 35076, + 35136, 35173, 35362, 35416, 35418, 35458, 35490, 36865, 36868, 36873, 36880, 36882, 36885, 36888, 36900, 36928, + 36930, 36933, 36936, 36945, 36948, 36960, 36993, 36996, 37008, 37120, 37125, 37137, 37140, 37185, 37188, 37200, + 37210, 37377, 37380, 37392, 37440, 37542, 37888, 37890, 37893, 37896, 37905, 37908, 37920, 37953, 37956, 37968, + 38016, 38038, 38145, 38148, 38160, 38208, 38296, 38305, 38400, 38470, 38500, 38913, 38916, 38928, 38950, 38976, + 39081, 39168, 39241, 39250, 39568, 40960, 40965, 40970, 40980, 40994, 41002, 41025, 41028, 41040, 41122, 41130, + 41280, 41317, 41474, 41482, 41506, 41512, 41514, 41602, 41608, 41610, 41640, 41985, 41988, 42000, 42048, 42121, + 42148, 42240, 42265, 42577, 43018, 43048, 43170, 43348, 43398, 43528, 43530, 43552, 43554, 43560, 43656, 43690, + }; const int kmap_size = 43692; - const int nwant = type == GGML_TYPE_IQ1_S ? 3 : 2; + //const int nwant = type == GGML_TYPE_IQ1_S ? 3 : 2; + const int nwant = type == GGML_TYPE_IQ1_S ? 3 : type == GGML_TYPE_IQ2_S ? 1 : 2; const uint16_t * kgrid = type == GGML_TYPE_IQ2_XXS ? kgrid_2bit_256 : - type == GGML_TYPE_IQ2_XS ? kgrid_2bit_512 : kgrid_1bit_512; + type == GGML_TYPE_IQ2_XS ? kgrid_2bit_512 : + type == GGML_TYPE_IQ1_S ? kgrid_1bit_512 : kgrid_2bit_1024; uint64_t * kgrid_q2xs; int * kmap_q2xs; uint16_t * kneighbors_q2xs; @@ -10151,7 +10717,7 @@ void iq2xs_init_impl(enum ggml_type type) { } void iq2xs_free_impl(enum ggml_type type) { - GGML_ASSERT(type == GGML_TYPE_IQ2_XXS || type == GGML_TYPE_IQ2_XS || type == GGML_TYPE_IQ1_S); + GGML_ASSERT(type == GGML_TYPE_IQ2_XXS || type == GGML_TYPE_IQ2_XS || type == GGML_TYPE_IQ1_S || type == GGML_TYPE_IQ2_S); const int gindex = iq2_data_index(type); if (iq2_data[gindex].grid) { free(iq2_data[gindex].grid); iq2_data[gindex].grid = NULL; @@ -11557,3 +12123,196 @@ void quantize_row_iq4_nl_reference(const float * restrict x, block_iq4_nl * rest quantize_iq4_nl(x, y, 1, k, NULL, NULL); } +// =============================== 2.5625 bpw + +static void quantize_row_iq2_s_impl(const float * restrict x, void * restrict vy, int n, const float * restrict quant_weights) { + + const int gindex = iq2_data_index(GGML_TYPE_IQ2_S); + + const uint64_t * kgrid_q2xs = iq2_data[gindex].grid; + const int * kmap_q2xs = iq2_data[gindex].map; + const uint16_t * kneighbors_q2xs = iq2_data[gindex].neighbours; + + GGML_ASSERT(kmap_q2xs && "forgot to call ggml_quantize_init()?"); + GGML_ASSERT(kgrid_q2xs && "forgot to call ggml_quantize_init()?"); + GGML_ASSERT(kneighbors_q2xs && "forgot to call ggml_quantize_init()?"); + GGML_ASSERT(n%QK_K == 0); + + const int kMaxQ = 3; + + const int nbl = n/256; + + block_iq2_s * y = vy; + + float scales[QK_K/16]; + float weight[16]; + float xval[16]; + int8_t L[16]; + int8_t Laux[16]; + float waux[16]; + bool is_on_grid[2]; + bool is_on_grid_aux[2]; + uint8_t block_signs[2]; + + for (int ibl = 0; ibl < nbl; ++ibl) { + + memset(&y[ibl], 0, sizeof(block_iq2_s)); + y[ibl].d = GGML_FP32_TO_FP16(0.f); + + float max_scale = 0; + + const float * xbl = x + QK_K*ibl; + float sumx2 = 0; + for (int i = 0; i < QK_K; ++i) sumx2 += xbl[i]*xbl[i]; + float sigma2 = 2*sumx2/QK_K; + + for (int ib = 0; ib < QK_K/16; ++ib) { + const float * xb = xbl + 16*ib; + if (quant_weights) { + const float * qw = quant_weights + QK_K*ibl + 16*ib; + for (int i = 0; i < 16; ++i) weight[i] = qw[i] * sqrtf(sigma2 + xb[i]*xb[i]); + } else { + for (int i = 0; i < 16; ++i) weight[i] = 0.25f*sigma2 + xb[i]*xb[i]; + } + for (int i = 0; i < 16; ++i) waux[i] = sqrtf(weight[i]); + for (int k = 0; k < 2; ++k) { + uint8_t s = 0; + for (int i = 0; i < 8; ++i) { + if (xb[8*k + i] >= 0) xval[8*k + i] = xb[8*k + i]; + else { + xval[8*k + i] = -xb[8*k + i]; s |= (1 << i); + } + } + block_signs[k] = s; + } + float max = xval[0]; + for (int i = 1; i < 16; ++i) max = MAX(max, xval[i]); + if (!max) { + scales[ib] = 0; + continue; + } + float best = 0; + float scale = max/(2*kMaxQ-1); + is_on_grid[0] = is_on_grid[1] = true; + for (int is = -9; is <= 9; ++is) { + float id = (2*kMaxQ-1+is*0.1f)/max; + float this_scale = 1/id; + for (int k = 0; k < 2; ++k) { + for (int i = 0; i < 8; ++i) { + int l = nearest_int(0.5f*(id*xval[8*k+i]-1)); + Laux[8*k+i] = MAX(0, MIN(kMaxQ-1, l)); + } + uint16_t u = 0; + for (int i = 0; i < 8; ++i) u |= (Laux[8*k+i] << 2*i); + int grid_index = kmap_q2xs[u]; + is_on_grid_aux[k] = true; + if (grid_index < 0) { + is_on_grid_aux[k] = false; + const uint16_t * neighbours = kneighbors_q2xs - kmap_q2xs[u] - 1; + grid_index = iq2_find_best_neighbour(neighbours, kgrid_q2xs, xval + 8*k, waux + 8*k, this_scale, Laux + 8*k); + } + } + float sumqx = 0, sumq2 = 0; + for (int i = 0; i < 16; ++i) { + float w = weight[i]; + float q = 2*Laux[i] + 1; + sumqx += w*xval[i]*q; + sumq2 += w*q*q; + } + if (sumq2 > 0 && sumqx*sumqx > best*sumq2) { + scale = sumqx/sumq2; best = scale*sumqx; + for (int i = 0; i < 16; ++i) L[i] = Laux[i]; + for (int k = 0; k < 2; ++k) is_on_grid[k] = is_on_grid_aux[k]; + } + } + int n_not_ongrid = 0; + for (int k = 0; k < 2; ++k) if (!is_on_grid[k]) ++n_not_ongrid; + if (n_not_ongrid > 0 && scale > 0) { + float id = 1/scale; + for (int k = 0; k < 2; ++k) { + if (is_on_grid[k]) continue; + uint16_t u = 0; + for (int i = 0; i < 8; ++i) { + int l = nearest_int(0.5f*(id*xval[8*k+i]-1)); + l = MAX(0, MIN(kMaxQ-1, l)); + u |= (l << 2*i); + L[8*k + i] = l; + } + int grid_index = kmap_q2xs[u]; + if (grid_index < 0) { + const uint16_t * neighbours = kneighbors_q2xs - kmap_q2xs[u] - 1; + grid_index = iq2_find_best_neighbour(neighbours, kgrid_q2xs, xval + 8*k, waux + 8*k, scale, L + 8*k); + } + } + float sumqx = 0, sumq2 = 0; + for (int i = 0; i < 16; ++i) { + float w = weight[i]; + float q = 2*L[i] + 1; + sumqx += w*xval[i]*q; + sumq2 += w*q*q; + } + if (sumq2 > 0) scale = sumqx/sumq2; + } + if (scale < 0) { + scale = -scale; + for (int k = 0; k < 2; ++k) block_signs[k] = ~block_signs[k]; + } + for (int k = 0; k < 2; ++k) { + uint16_t u = 0; + for (int i = 0; i < 8; ++i) u |= (L[8*k+i] << 2*i); + int grid_index = kmap_q2xs[u]; + if (grid_index < 0) { + printf("Oops: found point %u not on grid:", u); + for (int i = 0; i < 8; ++i) printf(" %d", L[8*k+i]); + printf("\n"); + GGML_ASSERT(false); + } + const int i8 = 2*ib + k; + y[ibl].qs[i8] = grid_index & 255; + y[ibl].qh[i8/4] |= ((grid_index >> 8) << 2*(i8%4)); + y[ibl].qs[QK_K/8 + i8] = block_signs[k]; + } + GGML_ASSERT(scale >= 0); + scales[ib] = scale; + max_scale = MAX(max_scale, scale); + } + + if (!max_scale) { + continue; + } + + float d = max_scale/31; + y[ibl].d = GGML_FP32_TO_FP16(d * 0.9875f); + float id = 1/d; + for (int ib = 0; ib < QK_K/16; ++ib) { + int l = nearest_int(0.5f*(id*scales[ib]-1)); + l = MAX(0, MIN(15, l)); + if (ib%2 == 0) y[ibl].scales[ib/2] = l; + else y[ibl].scales[ib/2] |= (l << 4); + } + } +} + +size_t quantize_iq2_s(const float * src, void * dst, int nrow, int n_per_row, int64_t * hist, const float * quant_weights) { + (void)hist; + GGML_ASSERT(n_per_row%QK_K == 0); + int nblock = n_per_row/QK_K; + char * qrow = (char *)dst; + for (int row = 0; row < nrow; ++row) { + quantize_row_iq2_s_impl(src, qrow, n_per_row, quant_weights); + src += n_per_row; + qrow += nblock*sizeof(block_iq2_s); + } + return nrow * nblock * sizeof(block_iq2_s); +} + +void quantize_row_iq2_s_reference(const float * restrict x, block_iq2_s * restrict y, int k) { + assert(k % QK_K == 0); + quantize_iq2_s(x, y, 1, k, NULL, NULL); +} + +void quantize_row_iq2_s(const float * restrict x, void * restrict vy, int k) { + assert(k % QK_K == 0); + block_iq2_s * restrict y = vy; + quantize_row_iq2_s_reference(x, y, k); +} diff --git a/ggml-quants.h b/ggml-quants.h index 303b0b6f9..4731dde0c 100644 --- a/ggml-quants.h +++ b/ggml-quants.h @@ -182,6 +182,15 @@ typedef struct { } block_iq2_xs; static_assert(sizeof(block_iq2_xs) == sizeof(ggml_fp16_t) + QK_K/8*sizeof(uint16_t) + QK_K/32, "wrong iq2_xs block size/padding"); +// 2.5625 bpw quants +typedef struct { + ggml_fp16_t d; + uint8_t qs[QK_K/4]; + uint8_t qh[QK_K/32]; + uint8_t scales[QK_K/32]; +} block_iq2_s; +static_assert(sizeof(block_iq2_s) == sizeof(ggml_fp16_t) + QK_K/4 + QK_K/16, "wrong iq2_s block size/padding"); + // (Almost) "true" 3-bit quantization. // Due to the need to use blocks as per ggml design, it ends up using // 3.0625 bpw because of the 16-bit scale for each block of 256. @@ -242,6 +251,7 @@ void quantize_row_q8_K_reference(const float * GGML_RESTRICT x, block_q8_K * GGM void quantize_row_iq3_xxs_reference(const float * GGML_RESTRICT x, block_iq3_xxs * GGML_RESTRICT y, int k); void quantize_row_iq4_nl_reference (const float * GGML_RESTRICT x, block_iq4_nl * GGML_RESTRICT y, int k); void quantize_row_iq3_s_reference (const float * GGML_RESTRICT x, block_iq3_s * GGML_RESTRICT y, int k); +void quantize_row_iq2_s_reference (const float * GGML_RESTRICT x, block_iq2_s * GGML_RESTRICT y, int k); void quantize_row_q4_0(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int k); void quantize_row_q4_1(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int k); @@ -259,6 +269,7 @@ void quantize_row_q8_K(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, in void quantize_row_iq3_xxs(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int k); void quantize_row_iq4_nl (const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int k); void quantize_row_iq3_s (const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int k); +void quantize_row_iq2_s (const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int k); // Dequantization void dequantize_row_q4_0(const block_q4_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int k); @@ -276,6 +287,7 @@ void dequantize_row_q6_K(const block_q6_K * GGML_RESTRICT x, float * GGML_RESTRI void dequantize_row_q8_K(const block_q8_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int k); void dequantize_row_iq2_xxs(const block_iq2_xxs * GGML_RESTRICT x, float * GGML_RESTRICT y, int k); void dequantize_row_iq2_xs (const block_iq2_xs * GGML_RESTRICT x, float * GGML_RESTRICT y, int k); +void dequantize_row_iq2_s (const block_iq2_s * GGML_RESTRICT x, float * GGML_RESTRICT y, int k); void dequantize_row_iq3_xxs(const block_iq3_xxs * GGML_RESTRICT x, float * GGML_RESTRICT y, int k); void dequantize_row_iq1_s (const block_iq1_s * GGML_RESTRICT x, float * GGML_RESTRICT y, int k); void dequantize_row_iq4_nl (const block_iq4_nl * GGML_RESTRICT x, float * GGML_RESTRICT y, int k); @@ -295,6 +307,7 @@ void ggml_vec_dot_q5_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voi void ggml_vec_dot_q6_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); void ggml_vec_dot_iq2_xxs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); void ggml_vec_dot_iq2_xs_q8_K (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); +void ggml_vec_dot_iq2_s_q8_K (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); void ggml_vec_dot_iq3_xxs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); void ggml_vec_dot_iq1_s_q8_K (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); void ggml_vec_dot_iq4_nl_q8_0 (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); @@ -305,6 +318,7 @@ void ggml_vec_dot_iq3_s_q8_K (int n, float * GGML_RESTRICT s, size_t bs, const // size_t quantize_iq2_xxs(const float * src, void * dst, int nrows, int n_per_row, int64_t * hist, const float * imatrix); size_t quantize_iq2_xs (const float * src, void * dst, int nrows, int n_per_row, int64_t * hist, const float * imatrix); +size_t quantize_iq2_s (const float * src, void * dst, int nrows, int n_per_row, int64_t * hist, const float * imatrix); size_t quantize_iq3_xxs(const float * src, void * dst, int nrows, int n_per_row, int64_t * hist, const float * imatrix); size_t quantize_iq1_s (const float * src, void * dst, int nrows, int n_per_row, int64_t * hist, const float * imatrix); size_t quantize_iq4_nl (const float * src, void * dst, int nrows, int n_per_row, int64_t * hist, const float * imatrix); diff --git a/ggml.c b/ggml.c index 1d81553f4..6be07bb6f 100644 --- a/ggml.c +++ b/ggml.c @@ -690,6 +690,18 @@ static const ggml_type_traits_t type_traits[GGML_TYPE_COUNT] = { .vec_dot_type = GGML_TYPE_Q8_K, .nrows = 1, }, + [GGML_TYPE_IQ2_S] = { + .type_name = "iq2_s", + .blck_size = QK_K, + .type_size = sizeof(block_iq2_s), + .is_quantized = true, + .to_float = (ggml_to_float_t) dequantize_row_iq2_s, + .from_float = quantize_row_iq2_s, + .from_float_reference = (ggml_from_float_t)quantize_row_iq2_s_reference, + .vec_dot = ggml_vec_dot_iq2_s_q8_K, + .vec_dot_type = GGML_TYPE_Q8_K, + .nrows = 1, + }, [GGML_TYPE_IQ1_S] = { .type_name = "iq1_s", .blck_size = QK_K, @@ -2317,6 +2329,7 @@ enum ggml_type ggml_ftype_to_ggml_type(enum ggml_ftype ftype) { case GGML_FTYPE_MOSTLY_IQ1_S: wtype = GGML_TYPE_IQ1_S; break; case GGML_FTYPE_MOSTLY_IQ4_NL: wtype = GGML_TYPE_IQ4_NL; break; case GGML_FTYPE_MOSTLY_IQ3_S: wtype = GGML_TYPE_IQ3_S; break; + case GGML_FTYPE_MOSTLY_IQ2_S: wtype = GGML_TYPE_IQ2_S; break; case GGML_FTYPE_UNKNOWN: wtype = GGML_TYPE_COUNT; break; case GGML_FTYPE_MOSTLY_Q4_1_SOME_F16: wtype = GGML_TYPE_COUNT; break; } @@ -7752,6 +7765,7 @@ static void ggml_compute_forward_add( case GGML_TYPE_IQ1_S: case GGML_TYPE_IQ4_NL: case GGML_TYPE_IQ3_S: + case GGML_TYPE_IQ2_S: { ggml_compute_forward_add_q_f32(params, dst); } break; @@ -8032,6 +8046,7 @@ static void ggml_compute_forward_add1( case GGML_TYPE_IQ1_S: case GGML_TYPE_IQ4_NL: case GGML_TYPE_IQ3_S: + case GGML_TYPE_IQ2_S: { ggml_compute_forward_add1_q_f32(params, dst); } break; @@ -8157,6 +8172,7 @@ static void ggml_compute_forward_acc( case GGML_TYPE_IQ1_S: case GGML_TYPE_IQ4_NL: case GGML_TYPE_IQ3_S: + case GGML_TYPE_IQ2_S: default: { GGML_ASSERT(false); @@ -11056,6 +11072,7 @@ static void ggml_compute_forward_out_prod( case GGML_TYPE_IQ1_S: case GGML_TYPE_IQ4_NL: case GGML_TYPE_IQ3_S: + case GGML_TYPE_IQ2_S: { ggml_compute_forward_out_prod_q_f32(params, dst); } break; @@ -11245,6 +11262,7 @@ static void ggml_compute_forward_set( case GGML_TYPE_IQ1_S: case GGML_TYPE_IQ4_NL: case GGML_TYPE_IQ3_S: + case GGML_TYPE_IQ2_S: default: { GGML_ASSERT(false); @@ -11448,6 +11466,7 @@ static void ggml_compute_forward_get_rows( case GGML_TYPE_IQ1_S: case GGML_TYPE_IQ4_NL: case GGML_TYPE_IQ3_S: + case GGML_TYPE_IQ2_S: { ggml_compute_forward_get_rows_q(params, dst); } break; @@ -12149,6 +12168,7 @@ static void ggml_compute_forward_alibi( case GGML_TYPE_IQ1_S: case GGML_TYPE_IQ4_NL: case GGML_TYPE_IQ3_S: + case GGML_TYPE_IQ2_S: case GGML_TYPE_Q8_K: case GGML_TYPE_I8: case GGML_TYPE_I16: @@ -12233,6 +12253,7 @@ static void ggml_compute_forward_clamp( case GGML_TYPE_IQ1_S: case GGML_TYPE_IQ4_NL: case GGML_TYPE_IQ3_S: + case GGML_TYPE_IQ2_S: case GGML_TYPE_Q8_K: case GGML_TYPE_I8: case GGML_TYPE_I16: @@ -19482,6 +19503,7 @@ void ggml_quantize_init(enum ggml_type type) { switch (type) { case GGML_TYPE_IQ2_XXS: case GGML_TYPE_IQ2_XS: + case GGML_TYPE_IQ2_S: case GGML_TYPE_IQ1_S: iq2xs_init_impl(type); break; case GGML_TYPE_IQ3_XXS: iq3xs_init_impl(256); break; case GGML_TYPE_IQ3_S: iq3xs_init_impl(512); break; @@ -19768,6 +19790,15 @@ size_t ggml_quantize_chunk(enum ggml_type type, const float * src, void * dst, i result = quantize_iq3_s(src + start, (char *)dst + start_row * row_size, nrows, n_per_row, hist, imatrix); GGML_ASSERT(result == row_size * nrows); } break; + case GGML_TYPE_IQ2_S: + { + GGML_ASSERT(start % QK_K == 0); + GGML_ASSERT(start % n_per_row == 0); + size_t start_row = start / n_per_row; + size_t row_size = ggml_row_size(type, n_per_row); + result = quantize_iq2_s(src + start, (char *)dst + start_row * row_size, nrows, n_per_row, hist, imatrix); + GGML_ASSERT(result == row_size * nrows); + } break; case GGML_TYPE_IQ1_S: { GGML_ASSERT(start % QK_K == 0); diff --git a/ggml.h b/ggml.h index 75fd035a4..8c7ca4588 100644 --- a/ggml.h +++ b/ggml.h @@ -351,6 +351,7 @@ extern "C" { GGML_TYPE_IQ1_S = 19, GGML_TYPE_IQ4_NL = 20, GGML_TYPE_IQ3_S = 21, + GGML_TYPE_IQ2_S = 22, GGML_TYPE_I8, GGML_TYPE_I16, GGML_TYPE_I32, @@ -391,6 +392,7 @@ extern "C" { GGML_FTYPE_MOSTLY_IQ1_S = 18, // except 1d tensors GGML_FTYPE_MOSTLY_IQ4_NL = 19, // except 1d tensors GGML_FTYPE_MOSTLY_IQ3_S = 20, // except 1d tensors + GGML_FTYPE_MOSTLY_IQ2_S = 21, // except 1d tensors }; // available tensor operations: diff --git a/llama.cpp b/llama.cpp index f549e7d04..80dc4d166 100644 --- a/llama.cpp +++ b/llama.cpp @@ -2579,6 +2579,7 @@ struct llama_model_loader { case GGML_TYPE_Q6_K: ftype = LLAMA_FTYPE_MOSTLY_Q6_K; break; case GGML_TYPE_IQ2_XXS: ftype = LLAMA_FTYPE_MOSTLY_IQ2_XXS; break; case GGML_TYPE_IQ2_XS: ftype = LLAMA_FTYPE_MOSTLY_IQ2_XS; break; + case GGML_TYPE_IQ2_S: ftype = LLAMA_FTYPE_MOSTLY_IQ2_S; break; case GGML_TYPE_IQ3_XXS: ftype = LLAMA_FTYPE_MOSTLY_IQ3_XXS; break; case GGML_TYPE_IQ1_S: ftype = LLAMA_FTYPE_MOSTLY_IQ1_S; break; case GGML_TYPE_IQ4_NL: ftype = LLAMA_FTYPE_MOSTLY_IQ4_NL; break; @@ -2933,7 +2934,9 @@ static std::string llama_model_ftype_name(llama_ftype ftype) { case LLAMA_FTYPE_MOSTLY_Q6_K: return "Q6_K"; case LLAMA_FTYPE_MOSTLY_IQ2_XXS:return "IQ2_XXS - 2.0625 bpw"; case LLAMA_FTYPE_MOSTLY_IQ2_XS: return "IQ2_XS - 2.3125 bpw"; - case LLAMA_FTYPE_MOSTLY_Q3_K_XS:return "Q3_K - Extra small"; + case LLAMA_FTYPE_MOSTLY_IQ2_S: return "IQ2_S - 2.5 bpw"; + case LLAMA_FTYPE_MOSTLY_IQ2_M: return "IQ2_M - 2.7 bpw"; + case LLAMA_FTYPE_MOSTLY_IQ3_XS: return "IQ3_XS - 3.3 bpw"; case LLAMA_FTYPE_MOSTLY_IQ3_XXS:return "IQ3_XXS - 3.0625 bpw"; case LLAMA_FTYPE_MOSTLY_IQ1_S :return "IQ1_S - 1.5625 bpw"; case LLAMA_FTYPE_MOSTLY_IQ4_NL: return "IQ4_NL - 4.5 bpw"; @@ -10761,31 +10764,47 @@ static ggml_type get_k_quant_type(quantize_state_internal & qs, ggml_type new_ty if (arch == LLM_ARCH_FALCON || nx % QK_K != 0) { new_type = GGML_TYPE_Q8_0; } - else if (ftype == LLAMA_FTYPE_MOSTLY_IQ2_XXS || ftype == LLAMA_FTYPE_MOSTLY_IQ2_XS || ftype == LLAMA_FTYPE_MOSTLY_IQ1_S) { + else if (ftype == LLAMA_FTYPE_MOSTLY_IQ2_XXS || ftype == LLAMA_FTYPE_MOSTLY_IQ2_XS || ftype == LLAMA_FTYPE_MOSTLY_IQ3_XXS || + ftype == LLAMA_FTYPE_MOSTLY_IQ1_S || ftype == LLAMA_FTYPE_MOSTLY_IQ2_S || ftype == LLAMA_FTYPE_MOSTLY_IQ2_M) { new_type = GGML_TYPE_Q5_K; } else if (new_type != GGML_TYPE_Q8_0) { new_type = GGML_TYPE_Q6_K; } } else if (name == "token_embd.weight") { - if (ftype == LLAMA_FTYPE_MOSTLY_IQ2_XXS || ftype == LLAMA_FTYPE_MOSTLY_IQ2_XS || ftype == LLAMA_FTYPE_MOSTLY_IQ1_S) { + if (ftype == LLAMA_FTYPE_MOSTLY_IQ2_XXS || ftype == LLAMA_FTYPE_MOSTLY_IQ2_XS || + ftype == LLAMA_FTYPE_MOSTLY_IQ1_S) { new_type = GGML_TYPE_Q2_K; } - else if (ftype == LLAMA_FTYPE_MOSTLY_IQ3_XXS) { - new_type = GGML_TYPE_Q4_K; + else if (ftype == LLAMA_FTYPE_MOSTLY_IQ2_S || ftype == LLAMA_FTYPE_MOSTLY_IQ2_M) { + new_type = GGML_TYPE_IQ3_S; } - } else if (ftype == LLAMA_FTYPE_MOSTLY_IQ2_XXS || ftype == LLAMA_FTYPE_MOSTLY_IQ2_XS || ftype == LLAMA_FTYPE_MOSTLY_IQ1_S) { + else if (ftype == LLAMA_FTYPE_MOSTLY_IQ3_XXS) { + new_type = GGML_TYPE_IQ3_S; + } + } else if (ftype == LLAMA_FTYPE_MOSTLY_IQ2_XXS || ftype == LLAMA_FTYPE_MOSTLY_IQ2_XS || ftype == LLAMA_FTYPE_MOSTLY_IQ1_S || + ftype == LLAMA_FTYPE_MOSTLY_IQ2_S || ftype == LLAMA_FTYPE_MOSTLY_IQ2_M) { if (name.find("attn_v.weight") != std::string::npos) { if (qs.model.hparams.n_gqa() >= 4 || qs.model.hparams.n_expert >= 4) new_type = GGML_TYPE_Q4_K; - else new_type = GGML_TYPE_Q2_K; + else new_type = ftype == LLAMA_FTYPE_MOSTLY_IQ2_S || ftype == LLAMA_FTYPE_MOSTLY_IQ2_M ? GGML_TYPE_IQ3_S : GGML_TYPE_Q2_K; ++qs.i_attention_wv; } + else if (qs.model.hparams.n_expert == 8 && name.find("attn_k.weight") != std::string::npos) { + new_type = GGML_TYPE_Q4_K; + } else if (name.find("ffn_down") != std::string::npos) { - if (qs.i_ffn_down < qs.n_ffn_down/8) new_type = GGML_TYPE_Q2_K; + if (qs.i_ffn_down < qs.n_ffn_down/8) { + new_type = ftype == LLAMA_FTYPE_MOSTLY_IQ2_S || ftype == LLAMA_FTYPE_MOSTLY_IQ2_M ? GGML_TYPE_IQ3_S : GGML_TYPE_Q2_K; + } ++qs.i_ffn_down; } else if (name.find("attn_output.weight") != std::string::npos) { - if (ftype == LLAMA_FTYPE_MOSTLY_IQ1_S) new_type = GGML_TYPE_IQ2_XXS; + if (qs.model.hparams.n_expert == 8) { + new_type = GGML_TYPE_Q5_K; + } else { + if (ftype == LLAMA_FTYPE_MOSTLY_IQ1_S) new_type = GGML_TYPE_IQ2_XXS; + else if (ftype == LLAMA_FTYPE_MOSTLY_IQ2_S || ftype == LLAMA_FTYPE_MOSTLY_IQ2_M) new_type = GGML_TYPE_IQ3_S; + } } } else if (name.find("attn_v.weight") != std::string::npos) { if (ftype == LLAMA_FTYPE_MOSTLY_Q2_K) { @@ -10795,7 +10814,13 @@ static ggml_type get_k_quant_type(quantize_state_internal & qs, ggml_type new_ty new_type = GGML_TYPE_Q4_K; } else if (ftype == LLAMA_FTYPE_MOSTLY_IQ3_XXS) { - new_type = qs.model.hparams.n_gqa() >= 4 ? GGML_TYPE_Q4_K : !qs.has_imatrix ? GGML_TYPE_Q3_K : GGML_TYPE_IQ3_XXS; + new_type = qs.model.hparams.n_gqa() >= 4 ? GGML_TYPE_Q4_K : !qs.has_imatrix ? GGML_TYPE_IQ3_S : GGML_TYPE_IQ3_XXS; + } + else if (ftype == LLAMA_FTYPE_MOSTLY_IQ3_S && qs.model.hparams.n_gqa() >= 4) { + new_type = GGML_TYPE_Q4_K; + } + else if (ftype == LLAMA_FTYPE_MOSTLY_IQ3_M) { + new_type = GGML_TYPE_Q4_K; } else if (ftype == LLAMA_FTYPE_MOSTLY_IQ3_S && qs.model.hparams.n_gqa() >= 4) { new_type = GGML_TYPE_Q4_K; @@ -10833,13 +10858,19 @@ static ggml_type get_k_quant_type(quantize_state_internal & qs, ggml_type new_ty // TODO: explore better strategies new_type = GGML_TYPE_Q8_0; } - else if (ftype == LLAMA_FTYPE_MOSTLY_Q3_K_XS) { + else if (ftype == LLAMA_FTYPE_MOSTLY_IQ3_XS) { new_type = GGML_TYPE_IQ3_XXS; } + else if (ftype == LLAMA_FTYPE_MOSTLY_IQ3_XXS) { + new_type = GGML_TYPE_IQ2_S; + } } else if (name.find("attn_q.weight") != std::string::npos) { - if (ftype == LLAMA_FTYPE_MOSTLY_Q3_K_XS) { + if (ftype == LLAMA_FTYPE_MOSTLY_IQ3_XS) { new_type = GGML_TYPE_IQ3_XXS; } + else if (ftype == LLAMA_FTYPE_MOSTLY_IQ3_XXS) { + new_type = GGML_TYPE_IQ2_S; + } } else if (name.find("ffn_down") != std::string::npos) { auto info = layer_info(qs.i_ffn_down, qs.n_ffn_down, name.c_str()); int i_layer = info.first, n_layer = info.second; @@ -10888,7 +10919,7 @@ static ggml_type get_k_quant_type(quantize_state_internal & qs, ggml_type new_ty } else if (name.find("attn_output.weight") != std::string::npos) { if (arch != LLM_ARCH_FALCON) { if (qs.model.hparams.n_expert == 8) { - if (ftype == LLAMA_FTYPE_MOSTLY_Q2_K || ftype == LLAMA_FTYPE_MOSTLY_Q3_K_XS || ftype == LLAMA_FTYPE_MOSTLY_IQ3_XXS || + if (ftype == LLAMA_FTYPE_MOSTLY_Q2_K || ftype == LLAMA_FTYPE_MOSTLY_IQ3_XS || ftype == LLAMA_FTYPE_MOSTLY_IQ3_XXS || ftype == LLAMA_FTYPE_MOSTLY_Q3_K_S || ftype == LLAMA_FTYPE_MOSTLY_Q3_K_M || ftype == LLAMA_FTYPE_MOSTLY_IQ4_NL || ftype == LLAMA_FTYPE_MOSTLY_Q4_K_S || ftype == LLAMA_FTYPE_MOSTLY_Q4_K_M || ftype == LLAMA_FTYPE_MOSTLY_IQ3_S || ftype == LLAMA_FTYPE_MOSTLY_IQ3_M) { @@ -10896,7 +10927,7 @@ static ggml_type get_k_quant_type(quantize_state_internal & qs, ggml_type new_ty } } else { if (ftype == LLAMA_FTYPE_MOSTLY_Q2_K ) new_type = GGML_TYPE_Q3_K; - else if (ftype == LLAMA_FTYPE_MOSTLY_IQ3_XXS) new_type = GGML_TYPE_Q3_K; + else if (ftype == LLAMA_FTYPE_MOSTLY_IQ3_XXS) new_type = GGML_TYPE_IQ3_S; else if (ftype == LLAMA_FTYPE_MOSTLY_Q3_K_M ) new_type = GGML_TYPE_Q4_K; else if (ftype == LLAMA_FTYPE_MOSTLY_Q3_K_L ) new_type = GGML_TYPE_Q5_K; else if (ftype == LLAMA_FTYPE_MOSTLY_IQ3_M ) new_type = GGML_TYPE_Q4_K; @@ -10915,7 +10946,7 @@ static ggml_type get_k_quant_type(quantize_state_internal & qs, ggml_type new_ty else if (name.find("ffn_gate") != std::string::npos) { auto info = layer_info(qs.i_ffn_gate, qs.n_ffn_gate, name.c_str()); int i_layer = info.first, n_layer = info.second; - if (ftype == LLAMA_FTYPE_MOSTLY_Q3_K_XS && (i_layer >= n_layer/8 && i_layer < 7*n_layer/8)) { + if (ftype == LLAMA_FTYPE_MOSTLY_IQ3_XS && (i_layer >= n_layer/8 && i_layer < 7*n_layer/8)) { new_type = GGML_TYPE_IQ3_XXS; } ++qs.i_ffn_gate; @@ -10923,7 +10954,7 @@ static ggml_type get_k_quant_type(quantize_state_internal & qs, ggml_type new_ty else if (name.find("ffn_up") != std::string::npos) { auto info = layer_info(qs.i_ffn_up, qs.n_ffn_up, name.c_str()); int i_layer = info.first, n_layer = info.second; - if (ftype == LLAMA_FTYPE_MOSTLY_Q3_K_XS && (i_layer >= n_layer/8 && i_layer < 7*n_layer/8)) { + if (ftype == LLAMA_FTYPE_MOSTLY_IQ3_XS && (i_layer >= n_layer/8 && i_layer < 7*n_layer/8)) { new_type = GGML_TYPE_IQ3_XXS; } ++qs.i_ffn_up; @@ -10943,7 +10974,7 @@ static ggml_type get_k_quant_type(quantize_state_internal & qs, ggml_type new_ty bool convert_incompatible_tensor = false; if (new_type == GGML_TYPE_Q2_K || new_type == GGML_TYPE_Q3_K || new_type == GGML_TYPE_Q4_K || new_type == GGML_TYPE_Q5_K || new_type == GGML_TYPE_Q6_K || - new_type == GGML_TYPE_IQ2_XS || new_type == GGML_TYPE_IQ2_XXS || + new_type == GGML_TYPE_IQ2_XS || new_type == GGML_TYPE_IQ2_XXS || new_type == GGML_TYPE_IQ2_S || new_type == GGML_TYPE_IQ3_XXS || ftype == LLAMA_FTYPE_MOSTLY_IQ1_S || new_type == GGML_TYPE_IQ3_S) { int nx = tensor->ne[0]; int ny = tensor->ne[1]; @@ -10958,6 +10989,7 @@ static ggml_type get_k_quant_type(quantize_state_internal & qs, ggml_type new_ty switch (new_type) { case GGML_TYPE_IQ2_XXS: case GGML_TYPE_IQ2_XS: + case GGML_TYPE_IQ2_S: case GGML_TYPE_IQ3_XXS: case GGML_TYPE_IQ3_S: case GGML_TYPE_IQ1_S: @@ -10991,7 +11023,7 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s // K-quants case LLAMA_FTYPE_MOSTLY_Q2_K_S: case LLAMA_FTYPE_MOSTLY_Q2_K: quantized_type = GGML_TYPE_Q2_K; break; - case LLAMA_FTYPE_MOSTLY_Q3_K_XS: quantized_type = GGML_TYPE_IQ3_S; break; + case LLAMA_FTYPE_MOSTLY_IQ3_XS: quantized_type = GGML_TYPE_IQ3_S; break; case LLAMA_FTYPE_MOSTLY_Q3_K_S: case LLAMA_FTYPE_MOSTLY_Q3_K_M: case LLAMA_FTYPE_MOSTLY_Q3_K_L: quantized_type = GGML_TYPE_Q3_K; break; @@ -11002,6 +11034,8 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s case LLAMA_FTYPE_MOSTLY_Q6_K: quantized_type = GGML_TYPE_Q6_K; break; case LLAMA_FTYPE_MOSTLY_IQ2_XXS: quantized_type = GGML_TYPE_IQ2_XXS; break; case LLAMA_FTYPE_MOSTLY_IQ2_XS: quantized_type = GGML_TYPE_IQ2_XS; break; + case LLAMA_FTYPE_MOSTLY_IQ2_S: quantized_type = GGML_TYPE_IQ2_XS; break; + case LLAMA_FTYPE_MOSTLY_IQ2_M: quantized_type = GGML_TYPE_IQ2_S; break; case LLAMA_FTYPE_MOSTLY_IQ3_XXS: quantized_type = GGML_TYPE_IQ3_XXS; break; case LLAMA_FTYPE_MOSTLY_IQ1_S: quantized_type = GGML_TYPE_IQ1_S; break; case LLAMA_FTYPE_MOSTLY_IQ4_NL: quantized_type = GGML_TYPE_IQ4_NL; break; @@ -11180,6 +11214,7 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s } if ((new_type == GGML_TYPE_IQ2_XXS || new_type == GGML_TYPE_IQ2_XS || + new_type == GGML_TYPE_IQ2_S || new_type == GGML_TYPE_IQ1_S || (new_type == GGML_TYPE_Q2_K && params->ftype == LLAMA_FTYPE_MOSTLY_Q2_K_S && strcmp(tensor->name, "token_embd.weight") != 0)) && !imatrix) { LLAMA_LOG_ERROR("\n\n============================================================\n"); diff --git a/llama.h b/llama.h index ff131996d..3ff77d5a8 100644 --- a/llama.h +++ b/llama.h @@ -107,12 +107,14 @@ extern "C" { LLAMA_FTYPE_MOSTLY_IQ2_XXS = 19, // except 1d tensors LLAMA_FTYPE_MOSTLY_IQ2_XS = 20, // except 1d tensors LLAMA_FTYPE_MOSTLY_Q2_K_S = 21, // except 1d tensors - LLAMA_FTYPE_MOSTLY_Q3_K_XS = 22, // except 1d tensors + LLAMA_FTYPE_MOSTLY_IQ3_XS = 22, // except 1d tensors LLAMA_FTYPE_MOSTLY_IQ3_XXS = 23, // except 1d tensors LLAMA_FTYPE_MOSTLY_IQ1_S = 24, // except 1d tensors LLAMA_FTYPE_MOSTLY_IQ4_NL = 25, // except 1d tensors LLAMA_FTYPE_MOSTLY_IQ3_S = 26, // except 1d tensors LLAMA_FTYPE_MOSTLY_IQ3_M = 27, // except 1d tensors + LLAMA_FTYPE_MOSTLY_IQ2_S = 28, // except 1d tensors + LLAMA_FTYPE_MOSTLY_IQ2_M = 29, // except 1d tensors LLAMA_FTYPE_GUESSED = 1024, // not specified in the model file }; diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index 24d12ef14..60a852779 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -1916,7 +1916,7 @@ static bool test_backend(ggml_backend_t backend, test_mode mode, const char * op GGML_TYPE_Q2_K, GGML_TYPE_Q3_K, GGML_TYPE_Q4_K, GGML_TYPE_Q5_K, GGML_TYPE_Q6_K, - GGML_TYPE_IQ2_XXS, GGML_TYPE_IQ2_XS, + GGML_TYPE_IQ2_XXS, GGML_TYPE_IQ2_XS, GGML_TYPE_IQ2_S, GGML_TYPE_IQ3_XXS, GGML_TYPE_IQ1_S, GGML_TYPE_IQ4_NL, GGML_TYPE_IQ3_S, }; diff --git a/tests/test-quantize-fns.cpp b/tests/test-quantize-fns.cpp index 04656bb9e..f615b612d 100644 --- a/tests/test-quantize-fns.cpp +++ b/tests/test-quantize-fns.cpp @@ -150,6 +150,7 @@ int main(int argc, char * argv[]) { const float total_error = total_quantization_error(qfns, test_size, test_data.data()); const float max_quantization_error = type == GGML_TYPE_Q2_K ? MAX_QUANTIZATION_TOTAL_ERROR_2BITS : + type == GGML_TYPE_IQ2_S ? MAX_QUANTIZATION_TOTAL_ERROR_2BITS : type == GGML_TYPE_Q3_K ? MAX_QUANTIZATION_TOTAL_ERROR_3BITS : type == GGML_TYPE_IQ3_S ? MAX_QUANTIZATION_TOTAL_ERROR_3BITS : type == GGML_TYPE_IQ3_XXS ? MAX_QUANTIZATION_TOTAL_ERROR_3BITS_XXS : MAX_QUANTIZATION_TOTAL_ERROR; @@ -168,7 +169,8 @@ int main(int argc, char * argv[]) { const float vec_dot_error = dot_product_error(qfns, test_size, test_data.data(), test_data2.data()); const float max_allowed_error = type == GGML_TYPE_Q2_K || type == GGML_TYPE_IQ2_XS || type == GGML_TYPE_IQ2_XXS || - type == GGML_TYPE_IQ3_XXS || type == GGML_TYPE_IQ3_S ? MAX_DOT_PRODUCT_ERROR_LOWBIT + type == GGML_TYPE_IQ3_XXS || type == GGML_TYPE_IQ3_S || type == GGML_TYPE_IQ2_S + ? MAX_DOT_PRODUCT_ERROR_LOWBIT : MAX_DOT_PRODUCT_ERROR; failed = !(vec_dot_error < max_allowed_error); num_failed += failed; From b11a93df41921846a10628a7c306d5c82a549939 Mon Sep 17 00:00:00 2001 From: Xuan Son Nguyen Date: Mon, 26 Feb 2024 23:15:48 +0100 Subject: [PATCH 472/812] fix server hangs on empty prompt (#5733) --- examples/server/server.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/examples/server/server.cpp b/examples/server/server.cpp index 8aadc95a9..846ef7e5f 100644 --- a/examples/server/server.cpp +++ b/examples/server/server.cpp @@ -1336,6 +1336,10 @@ struct llama_server_context split_multiprompt_task(task_id, task); } } else { + // an empty prompt can make slot become buggy + if (task.data.contains("prompt") && task.data["prompt"].is_string() && task.data["prompt"].get().empty()) { + task.data["prompt"] = " "; // add a space so that we have one token + } queue_tasks.post(task); } } From cbbd1efa06f8c09f9dff58ff9d9af509cc4c152b Mon Sep 17 00:00:00 2001 From: "le.chang" Date: Tue, 27 Feb 2024 10:03:06 +0800 Subject: [PATCH 473/812] Makefile: use variables for cublas (#5689) * make: use arch variable for cublas * fix UNAME_M * check opt first --------- Co-authored-by: lindeer --- Makefile | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 068f6ed02..4f26c0463 100644 --- a/Makefile +++ b/Makefile @@ -381,8 +381,13 @@ ifdef LLAMA_BLIS endif # LLAMA_BLIS ifdef LLAMA_CUBLAS - MK_CPPFLAGS += -DGGML_USE_CUBLAS -I/usr/local/cuda/include -I/opt/cuda/include -I$(CUDA_PATH)/targets/x86_64-linux/include -I/usr/local/cuda/targets/aarch64-linux/include - MK_LDFLAGS += -lcuda -lcublas -lculibos -lcudart -lcublasLt -lpthread -ldl -lrt -L/usr/local/cuda/lib64 -L/opt/cuda/lib64 -L$(CUDA_PATH)/targets/x86_64-linux/lib -L/usr/local/cuda/targets/aarch64-linux/lib -L/usr/lib/wsl/lib + ifneq ('', '$(wildcard /opt/cuda)') + CUDA_PATH ?= /opt/cuda + else + CUDA_PATH ?= /usr/local/cuda + endif + MK_CPPFLAGS += -DGGML_USE_CUBLAS -I$(CUDA_PATH)/include -I$(CUDA_PATH)/targets/$(UNAME_M)-linux/include + MK_LDFLAGS += -lcuda -lcublas -lculibos -lcudart -lcublasLt -lpthread -ldl -lrt -L$(CUDA_PATH)/lib64 -L/usr/lib64 -L$(CUDA_PATH)/targets/$(UNAME_M)-linux/lib -L/usr/lib/wsl/lib OBJS += ggml-cuda.o MK_NVCCFLAGS += -use_fast_math ifdef LLAMA_FATAL_WARNINGS From 9d533a77d0c3850ce09d736bc1baa67fd6ad27b3 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Tue, 27 Feb 2024 14:35:51 +0200 Subject: [PATCH 474/812] llama : fix defrag bugs + add parameter (#5735) * llama : fix defrag bugs + enable by default ggml-ci * llama : add defrag_thold parameter ggml-ci * llama : cont * llama : disable log message ggml-ci * llama : fix graph size check during defrag --- common/common.cpp | 9 ++++ common/common.h | 1 + examples/passkey/passkey.cpp | 4 +- llama.cpp | 97 +++++++++++++++++++++++++----------- llama.h | 1 + 5 files changed, 82 insertions(+), 30 deletions(-) diff --git a/common/common.cpp b/common/common.cpp index ec596f5a0..18289755c 100644 --- a/common/common.cpp +++ b/common/common.cpp @@ -335,6 +335,12 @@ bool gpt_params_parse_ex(int argc, char ** argv, gpt_params & params) { break; } params.yarn_beta_slow = std::stof(argv[i]); + } else if (arg == "--defrag-thold" || arg == "-dt") { + if (++i >= argc) { + invalid_param = true; + break; + } + params.defrag_thold = std::stof(argv[i]); } else if (arg == "--samplers") { if (++i >= argc) { invalid_param = true; @@ -1004,6 +1010,8 @@ void gpt_print_usage(int /*argc*/, char ** argv, const gpt_params & params) { printf(" --yarn-attn-factor N YaRN: scale sqrt(t) or attention magnitude (default: 1.0)\n"); printf(" --yarn-beta-slow N YaRN: high correction dim or alpha (default: %.1f)\n", params.yarn_beta_slow); printf(" --yarn-beta-fast N YaRN: low correction dim or beta (default: %.1f)\n", params.yarn_beta_fast); + printf(" -dt N, --defrag-thold N\n"); + printf(" KV cache defragmentation threshold (default: %.1f, < 0 - disabled)\n", params.defrag_thold); printf(" --ignore-eos ignore end of stream token and continue generating (implies --logit-bias 2-inf)\n"); printf(" --no-penalize-nl do not penalize newline token\n"); printf(" --temp N temperature (default: %.1f)\n", (double)sparams.temp); @@ -1285,6 +1293,7 @@ struct llama_context_params llama_context_params_from_gpt_params(const gpt_param cparams.yarn_beta_fast = params.yarn_beta_fast; cparams.yarn_beta_slow = params.yarn_beta_slow; cparams.yarn_orig_ctx = params.yarn_orig_ctx; + cparams.defrag_thold = params.defrag_thold; cparams.offload_kqv = !params.no_kv_offload; cparams.type_k = kv_cache_type_from_str(params.cache_type_k); diff --git a/common/common.h b/common/common.h index 3e21579b0..25003df26 100644 --- a/common/common.h +++ b/common/common.h @@ -75,6 +75,7 @@ struct gpt_params { float yarn_beta_fast = 32.0f; // YaRN low correction dim float yarn_beta_slow = 1.0f; // YaRN high correction dim int32_t yarn_orig_ctx = 0; // YaRN original context length + float defrag_thold = -1.0f; // KV cache defragmentation threshold int32_t rope_scaling_type = LLAMA_ROPE_SCALING_TYPE_UNSPECIFIED; ggml_numa_strategy numa = GGML_NUMA_STRATEGY_DISABLED; diff --git a/examples/passkey/passkey.cpp b/examples/passkey/passkey.cpp index 47de67a93..2cbc9e1fa 100644 --- a/examples/passkey/passkey.cpp +++ b/examples/passkey/passkey.cpp @@ -182,7 +182,7 @@ int main(int argc, char ** argv) { llama_kv_cache_seq_rm (ctx, 0, n_keep , n_keep + n_discard); llama_kv_cache_seq_add(ctx, 0, n_keep + n_discard, n_ctx, -n_discard); - llama_kv_cache_defrag (ctx); + //llama_kv_cache_defrag (ctx); llama_kv_cache_update (ctx); n_past = llama_kv_cache_seq_pos_max(ctx, 0) + 1; @@ -213,7 +213,7 @@ int main(int argc, char ** argv) { llama_kv_cache_seq_rm (ctx, 0, n_keep , n_keep + n_discard); llama_kv_cache_seq_add(ctx, 0, n_keep + n_discard, n_ctx, -n_discard); - llama_kv_cache_defrag (ctx); + //llama_kv_cache_defrag (ctx); llama_kv_cache_update (ctx); n_past = llama_kv_cache_seq_pos_max(ctx, 0) + 1; diff --git a/llama.cpp b/llama.cpp index 80dc4d166..6729bb99c 100644 --- a/llama.cpp +++ b/llama.cpp @@ -1641,6 +1641,7 @@ struct llama_cparams { float yarn_attn_factor; float yarn_beta_fast; float yarn_beta_slow; + float defrag_thold; bool mul_mat_q; bool offload_kqv; @@ -5117,16 +5118,16 @@ struct llm_build_context { struct ggml_cgraph * build_defrag(const std::vector & ids) { struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, LLAMA_MAX_NODES, false); - for (int i = 0; i < n_kv; ++i) { - const int id = ids[i]; + for (uint32_t i = 0; i < ids.size(); ++i) { + const uint32_t id = ids[i]; - if (i == id || id == n_kv) { + if (i == id || id == ids.size()) { continue; } - int nm = 1; + uint32_t nm = 1; - while (i + nm < n_kv && (int) ids[i + nm] == id + nm) { + while (i + nm < ids.size() && ids[i + nm] == id + nm) { nm++; } @@ -5158,6 +5159,8 @@ struct llm_build_context { i += nm - 1; } + //LLAMA_LOG_INFO("gf->n_nodes = %d\n", gf->n_nodes); + return gf; } @@ -7938,6 +7941,8 @@ static int llama_decode_internal( batch.seq_id = seq_id_arr.data(); } + llama_kv_cache_update(&lctx); + // if we have enough unused cells before the current head -> // better to start searching from the beginning of the cache, hoping to fill it if (kv_self.head > kv_self.used + 2*n_tokens) { @@ -7956,8 +7961,6 @@ static int llama_decode_internal( //printf("kv_self.n = %5d, kv_self.used = %5d, kv_self.head = %5d\n", kv_self.n, kv_self.used, kv_self.head); - llama_kv_cache_update(&lctx); - ggml_backend_sched_reset(lctx.sched); ggml_backend_sched_set_eval_callback(lctx.sched, lctx.cparams.cb_eval, lctx.cparams.cb_eval_user_data); @@ -8007,6 +8010,18 @@ static int llama_decode_internal( } } + // decide if we need to defrag the kv cache + if (cparams.defrag_thold >= 0.0f) { + const float fragmentation = kv_self.n >= 128 ? 1.0f - float(kv_self.used + n_tokens)/float(kv_self.n) : 0.0f; + + // queue defragmentation for next llama_kv_cache_update + if (fragmentation > cparams.defrag_thold) { + //LLAMA_LOG_INFO("fragmentation: %.2f\n", fragmentation); + + llama_kv_cache_defrag(kv_self); + } + } + #ifdef GGML_PERF // print timing information per ggml operation (for debugging purposes) // requires GGML_PERF to be defined @@ -8098,12 +8113,16 @@ static int llama_decode_internal( static void llama_kv_cache_defrag_internal(struct llama_context & lctx) { auto & kv_self = lctx.kv_self; + const auto & hparams = lctx.model.hparams; + + const uint32_t n_layer = hparams.n_layer; + const uint32_t n_kv = llama_kv_cache_cell_max(kv_self); const uint32_t n_used = kv_self.used; assert(n_used <= n_kv); - const int64_t t_start = ggml_time_us(); + //const int64_t t_start = ggml_time_us(); // number of cells moved uint32_t n_moves = 0; @@ -8127,15 +8146,26 @@ static void llama_kv_cache_defrag_internal(struct llama_context & lctx) { // found a hole - fill it with data from the end of the cache - // determine the size of the hole uint32_t nh = 1; + + // determine the size of the hole while (i0 + nh < n_used && kv_self.cells[i0 + nh].is_empty()) { nh++; } - // starting from the end, find nh non-empty cells + // each move requires 6*n_layer tensors (see build_defrag) + // - source view, destination view, copy operation + // - x2 for keys and values + // + if (6*(n_moves + nh)*n_layer >= LLAMA_MAX_NODES) { + // the graph is too big, we cannot move more cells + break; + } + uint32_t nf = 0; uint32_t is = n_kv - 1; + + // starting from the end, find nh non-empty cells for (; is > i0; --is) { const auto & cell1 = kv_self.cells[is]; @@ -8156,11 +8186,17 @@ static void llama_kv_cache_defrag_internal(struct llama_context & lctx) { nf = 0; + uint32_t i1 = is; + + // are we moving a continuous block of memory? + bool cont = false; + // go back and move the nf cells to the hole - for (uint32_t i1 = is; i1 < n_kv; ++i1) { - const auto & cell1 = kv_self.cells[i1]; + for (; i1 < n_kv; ++i1) { + auto & cell1 = kv_self.cells[i1]; if (cell1.is_empty() || ids[i1] != n_kv) { + cont = false; continue; } @@ -8170,11 +8206,23 @@ static void llama_kv_cache_defrag_internal(struct llama_context & lctx) { // move the cell meta data kv_self.cells[i0 + nf] = cell1; - n_moves++; + // clear the old cell and move the head there + cell1 = llama_kv_cell(); + kv_self.head = n_used; + + if (!cont) { + n_moves++; + cont = true; + } + nf++; + + if (nf == nh) { + break; + } } - LLAMA_LOG_INFO("(tmp log) KV defrag: move [%u, %u) to [%u, %u)\n", is, n_kv, i0, i0 + nh); + //LLAMA_LOG_INFO("(tmp log) KV defrag: move [%u, %u) to [%u, %u)\n", is, i1 + 1, i0, i0 + nh); i0 += nh - 1; } @@ -8183,15 +8231,9 @@ static void llama_kv_cache_defrag_internal(struct llama_context & lctx) { return; } - LLAMA_LOG_INFO("(tmp log) KV defrag cell moves: %u\n", n_moves); + //LLAMA_LOG_INFO("(tmp log) KV defrag cell moves: %u\n", n_moves); - kv_self.head = n_used; - kv_self.used = n_used; - - // zero the rest of the cells - for (uint32_t i = n_used; i < n_kv; ++i) { - kv_self.cells[i] = llama_kv_cell(); - } + //LLAMA_LOG_INFO("expected gf nodes: %u\n", 6*n_moves*n_layer); #if 0 // CPU defrag @@ -8203,9 +8245,6 @@ static void llama_kv_cache_defrag_internal(struct llama_context & lctx) { // likely not worth the effort, as we have ggml_graph based defrag // - const auto & hparams = lctx.model.hparams; - - const uint32_t n_layer = hparams.n_layer; const uint32_t n_embd_k_gqa = hparams.n_embd_k_gqa(); const uint32_t n_embd_v_gqa = hparams.n_embd_v_gqa(); @@ -8274,9 +8313,9 @@ static void llama_kv_cache_defrag_internal(struct llama_context & lctx) { llama_graph_compute(lctx, gf, lctx.cparams.n_threads); #endif - const int64_t t_end = ggml_time_us(); + //const int64_t t_end = ggml_time_us(); - LLAMA_LOG_INFO("(tmp log) KV defrag time: %.3f ms\n", (t_end - t_start)/1000.0); + //LLAMA_LOG_INFO("(tmp log) KV defrag time: %.3f ms\n", (t_end - t_start)/1000.0); } static void llama_kv_cache_update_internal(struct llama_context & lctx) { @@ -11670,6 +11709,7 @@ struct llama_context_params llama_context_default_params() { /*.yarn_beta_fast =*/ 32.0f, /*.yarn_beta_slow =*/ 1.0f, /*.yarn_orig_ctx =*/ 0, + /*.defrag_thold =*/ -1.0f, /*.cb_eval =*/ nullptr, /*.cb_eval_user_data =*/ nullptr, /*.type_k =*/ GGML_TYPE_F16, @@ -11834,6 +11874,7 @@ struct llama_context * llama_new_context_with_model( cparams.yarn_attn_factor = params.yarn_attn_factor; cparams.yarn_beta_fast = params.yarn_beta_fast; cparams.yarn_beta_slow = params.yarn_beta_slow; + cparams.defrag_thold = params.defrag_thold; cparams.mul_mat_q = params.mul_mat_q; cparams.offload_kqv = params.offload_kqv; cparams.do_pooling = params.do_pooling; @@ -12035,7 +12076,7 @@ struct llama_context * llama_new_context_with_model( } // buffer used to store the computation graph and the tensor meta data - ctx->buf_compute_meta.resize(ggml_tensor_overhead()*LLAMA_MAX_NODES + ggml_graph_overhead()); + ctx->buf_compute_meta.resize(ggml_tensor_overhead()*LLAMA_MAX_NODES + ggml_graph_overhead_custom(LLAMA_MAX_NODES, false)); ctx->sched = ggml_backend_sched_new(ctx->backends.data(), backend_buft.data(), ctx->backends.size(), LLAMA_MAX_NODES); diff --git a/llama.h b/llama.h index 3ff77d5a8..604161808 100644 --- a/llama.h +++ b/llama.h @@ -245,6 +245,7 @@ extern "C" { float yarn_beta_fast; // YaRN low correction dim float yarn_beta_slow; // YaRN high correction dim uint32_t yarn_orig_ctx; // YaRN original context size + float defrag_thold; // defragment the KV cache if holes/size > thold, < 0 disabled (default) ggml_backend_sched_eval_callback cb_eval; void * cb_eval_user_data; From 1f30b7a9f1b86baa455072d3182b9ebeee0cd845 Mon Sep 17 00:00:00 2001 From: Engininja2 <139037756+Engininja2@users.noreply.github.com> Date: Tue, 27 Feb 2024 06:50:18 -0600 Subject: [PATCH 475/812] ggml-quants : fix avx2 iq1_s vec_dot when compiled with gcc (#5742) --- ggml-quants.c | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/ggml-quants.c b/ggml-quants.c index ce654f094..73c3bb412 100644 --- a/ggml-quants.c +++ b/ggml-quants.c @@ -10248,8 +10248,12 @@ void ggml_vec_dot_iq1_s_q8_K (int n, float * GGML_RESTRICT s, size_t bs, const uint64_t aux64; - __m256i v_gindex; - const uint16_t * gindex = (const uint16_t *)&v_gindex; + typedef union m256i_uint16 { + __m256i reg; + uint16_t s[16]; + } m256i_uint16_t; + + m256i_uint16_t v_gindex; __m256 accum = _mm256_setzero_ps(); for (int i = 0; i < nb; ++i) { @@ -10264,13 +10268,13 @@ void ggml_vec_dot_iq1_s_q8_K (int n, float * GGML_RESTRICT s, size_t bs, const memcpy(&aux64, sc, 8); sc += 8; const __m128i qh = _mm_shuffle_epi8(_mm_set_epi64x(aux64 >> 4, aux64), shuffle_h); const __m256i hbit = _mm256_cvtepu8_epi16(_mm_and_si128(qh, m8)); - v_gindex = _mm256_or_si256(_mm256_cvtepu8_epi16(ql), _mm256_slli_epi16(hbit, 5)); + v_gindex.reg = _mm256_or_si256(_mm256_cvtepu8_epi16(ql), _mm256_slli_epi16(hbit, 5)); const __m128i scales = _mm_or_si128(_mm_slli_epi16(_mm_and_si128(qh, m7), 1), m1); for (int i32 = 0; i32 < 4; ++i32) { const __m256i q8b = _mm256_loadu_si256((const __m256i*)q8); q8 += 32; - const __m256i q1b = _mm256_set_epi64x(iq1s_grid[gindex[4*i32+3]], iq1s_grid[gindex[4*i32+2]], - iq1s_grid[gindex[4*i32+1]], iq1s_grid[gindex[4*i32+0]]); + const __m256i q1b = _mm256_set_epi64x(iq1s_grid[v_gindex.s[4*i32+3]], iq1s_grid[v_gindex.s[4*i32+2]], + iq1s_grid[v_gindex.s[4*i32+1]], iq1s_grid[v_gindex.s[4*i32+0]]); const __m256i dot = mul_add_epi8(q1b, q8b); const __m256i s16 = _mm256_cvtepi8_epi16(_mm_shuffle_epi8(scales, shuffle_s[i32])); const __m256i p = _mm256_madd_epi16(s16, dot); From c24a2a6e6005e5d424301525a42ba45a4a362d30 Mon Sep 17 00:00:00 2001 From: Engininja2 <139037756+Engininja2@users.noreply.github.com> Date: Tue, 27 Feb 2024 07:22:45 -0600 Subject: [PATCH 476/812] cuda : replace remaining shfl_xor with calls to warp_reduce functions (#5744) --- ggml-cuda.cu | 73 +++++++++++++++++----------------------------------- 1 file changed, 24 insertions(+), 49 deletions(-) diff --git a/ggml-cuda.cu b/ggml-cuda.cu index 964fb7351..caef65de5 100644 --- a/ggml-cuda.cu +++ b/ggml-cuda.cu @@ -696,18 +696,20 @@ static __device__ __forceinline__ float2 warp_reduce_sum(float2 a) { return a; } -//static __device__ __forceinline__ half2 warp_reduce_sum(half2 a) { -//#if !(defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)) && __CUDA_ARCH__ >= CC_PASCAL -//#pragma unroll -// for (int mask = 16; mask > 0; mask >>= 1) { -// a = __hadd2(a, __shfl_xor_sync(0xffffffff, a, mask, 32)); -// } -// return a; -//#else -// (void) a; -// NO_DEVICE_CODE; -//#endif // !(defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)) && __CUDA_ARCH__ >= CC_PASCAL -//} +#ifdef GGML_CUDA_F16 +static __device__ __forceinline__ half2 warp_reduce_sum(half2 a) { +#if !(defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)) && __CUDA_ARCH__ >= CC_PASCAL +#pragma unroll + for (int mask = 16; mask > 0; mask >>= 1) { + a = __hadd2(a, __shfl_xor_sync(0xffffffff, a, mask, 32)); + } + return a; +#else + (void) a; + NO_DEVICE_CODE; +#endif // !(defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)) && __CUDA_ARCH__ >= CC_PASCAL +} +#endif // GGML_CUDA_F16 static __device__ __forceinline__ float warp_reduce_max(float x) { #pragma unroll @@ -2521,10 +2523,7 @@ static __global__ void dequantize_mul_mat_vec_q2_k(const void * __restrict__ vx, #endif // sum up partial sums and write back result -#pragma unroll - for (int mask = 16; mask > 0; mask >>= 1) { - tmp += __shfl_xor_sync(0xffffffff, tmp, mask, 32); - } + tmp = warp_reduce_sum(tmp); if (threadIdx.x == 0) { dst[row] = tmp; @@ -2625,10 +2624,7 @@ static __global__ void dequantize_mul_mat_vec_q3_k(const void * __restrict__ vx, #endif // sum up partial sums and write back result -#pragma unroll - for (int mask = 16; mask > 0; mask >>= 1) { - tmp += __shfl_xor_sync(0xffffffff, tmp, mask, 32); - } + tmp = warp_reduce_sum(tmp); if (threadIdx.x == 0) { dst[row] = tmp; @@ -2761,10 +2757,7 @@ static __global__ void dequantize_mul_mat_vec_q4_k(const void * __restrict__ vx, #endif // sum up partial sums and write back result -#pragma unroll - for (int mask = 16; mask > 0; mask >>= 1) { - tmp += __shfl_xor_sync(0xffffffff, tmp, mask, 32); - } + tmp = warp_reduce_sum(tmp); if (tid == 0) { dst[row] = tmp; @@ -2877,10 +2870,7 @@ static __global__ void dequantize_mul_mat_vec_q5_k(const void * __restrict__ vx, #endif // sum up partial sums and write back result -#pragma unroll - for (int mask = 16; mask > 0; mask >>= 1) { - tmp += __shfl_xor_sync(0xffffffff, tmp, mask, 32); - } + tmp = warp_reduce_sum(tmp); if (threadIdx.x == 0) { dst[row] = tmp; @@ -2987,10 +2977,7 @@ static __global__ void dequantize_mul_mat_vec_q6_k(const void * __restrict__ vx, #endif // sum up partial sums and write back result -#pragma unroll - for (int mask = 16; mask > 0; mask >>= 1) { - tmp += __shfl_xor_sync(0xffffffff, tmp, mask, 32); - } + tmp = warp_reduce_sum(tmp); if (tid == 0) { dst[row] = tmp; @@ -3025,11 +3012,8 @@ static __global__ void quantize_q8_1(const float * __restrict__ x, void * __rest float amax = fabsf(xi); float sum = xi; -#pragma unroll - for (int mask = 16; mask > 0; mask >>= 1) { - amax = fmaxf(amax, __shfl_xor_sync(0xffffffff, amax, mask, 32)); - sum += __shfl_xor_sync(0xffffffff, sum, mask, 32); - } + amax = warp_reduce_max(amax); + sum = warp_reduce_sum(sum); const float d = amax / 127; const int8_t q = amax == 0.0f ? 0 : roundf(xi / d); @@ -6222,10 +6206,7 @@ static __global__ void dequantize_mul_mat_vec(const void * __restrict__ vx, cons } // sum up partial sums and write back result -#pragma unroll - for (int mask = 16; mask > 0; mask >>= 1) { - tmp += __shfl_xor_sync(0xffffffff, tmp, mask, 32); - } + tmp = warp_reduce_sum(tmp); if (tid == 0) { #ifdef GGML_CUDA_F16 @@ -6275,10 +6256,7 @@ static __global__ void mul_mat_p021_f16_f32( const int idst = channel*nrows_dst + row_dst; // sum up partial sums and write back result -#pragma unroll - for (int mask = 16; mask > 0; mask >>= 1) { - tmp += __shfl_xor_sync(0xffffffff, tmp, mask, 32); - } + tmp = warp_reduce_sum(tmp); if (threadIdx.x == 0) { dst[idst] = tmp; @@ -6321,10 +6299,7 @@ static __global__ void mul_mat_vec_nc_f16_f32( // nc == non-contiguous } // sum up partial sums and write back result -#pragma unroll - for (int mask = 16; mask > 0; mask >>= 1) { - tmp += __shfl_xor_sync(0xffffffff, tmp, mask, 32); - } + tmp = warp_reduce_sum(tmp); if (threadIdx.x == 0) { dst[idst] = tmp; From 0becb22ac05b6542bd9d5f2235691aa1d3d4d307 Mon Sep 17 00:00:00 2001 From: Kawrakow <48489457+ikawrakow@users.noreply.github.com> Date: Tue, 27 Feb 2024 16:34:24 +0200 Subject: [PATCH 477/812] IQ4_XS: a 4.25 bpw quantization (#5747) * Try IQ4_NL with blocks of 64 - does not look good * iq4_xs: go to super-blocks of 256 and 6-bit scales for blocks of 32 * iq4_xs: CUDA works - 133.2 t/s * iq4_xs: AVX2 dot product * iq4_xs: ARM_NEON dot product * iq4_nl: Metal implementation As usual, Metal / Apple Silicon don't like my quants. * iq3_xs: minor fix * iq4_xs: shrink by using IQ3_S for attn_k and attn_q * iq4_xs: revert using IQ3_S for attn_k and attn_v PPL vs size is good, but CPU performance suffers: on M2 Max TG-128 drops to 21.7 t/s from 28.8, and on a Ryzen-7950X to 14.5 t/s from 15.8 t/s. On CUDA we have 135 t/s when using IQ3_S vs 133 t/s with pure IQ4_XS. * Fix CI * iq4_xs: Added forgotten check for 256 divisibility --------- Co-authored-by: Iwan Kawrakow --- examples/quantize/quantize.cpp | 3 +- ggml-cuda.cu | 119 ++++++++++++++- ggml-metal.m | 29 +++- ggml-metal.metal | 224 +++++++++++++++++++++++++++- ggml-quants.c | 261 ++++++++++++++++++++++++++++++--- ggml-quants.h | 13 ++ ggml.c | 30 ++++ ggml.h | 2 + llama.cpp | 22 +-- llama.h | 1 + tests/test-backend-ops.cpp | 2 +- 11 files changed, 668 insertions(+), 38 deletions(-) diff --git a/examples/quantize/quantize.cpp b/examples/quantize/quantize.cpp index 2d187823f..7662ec80c 100644 --- a/examples/quantize/quantize.cpp +++ b/examples/quantize/quantize.cpp @@ -36,7 +36,8 @@ static const std::vector QUANT_OPTIONS = { { "Q3_K_S", LLAMA_FTYPE_MOSTLY_Q3_K_S, " 2.75G, +0.5551 ppl @ LLaMA-v1-7B", }, { "Q3_K_M", LLAMA_FTYPE_MOSTLY_Q3_K_M, " 3.07G, +0.2496 ppl @ LLaMA-v1-7B", }, { "Q3_K_L", LLAMA_FTYPE_MOSTLY_Q3_K_L, " 3.35G, +0.1764 ppl @ LLaMA-v1-7B", }, - { "IQ4_NL", LLAMA_FTYPE_MOSTLY_IQ4_NL, " 4.25 bpw non-linear quantization", }, + { "IQ4_NL", LLAMA_FTYPE_MOSTLY_IQ4_NL, " 4.50 bpw non-linear quantization", }, + { "IQ4_XS", LLAMA_FTYPE_MOSTLY_IQ4_XS, " 4.25 bpw non-linear quantization", }, { "Q4_K", LLAMA_FTYPE_MOSTLY_Q4_K_M, "alias for Q4_K_M", }, { "Q4_K_S", LLAMA_FTYPE_MOSTLY_Q4_K_S, " 3.59G, +0.0992 ppl @ LLaMA-v1-7B", }, { "Q4_K_M", LLAMA_FTYPE_MOSTLY_Q4_K_M, " 3.80G, +0.0532 ppl @ LLaMA-v1-7B", }, diff --git a/ggml-cuda.cu b/ggml-cuda.cu index caef65de5..dfd28df62 100644 --- a/ggml-cuda.cu +++ b/ggml-cuda.cu @@ -571,6 +571,18 @@ typedef struct { } block_iq4_nl; static_assert(sizeof(block_iq4_nl) == sizeof(ggml_fp16_t) + QK4_NL/2, "wrong iq4_nl block size/padding"); +// QR4_XS = 8 is very slightly faster than QR4_XS = 4 +#define QR4_XS 8 +#define QI4_XS (QK_K / (4*QR4_XS)) +typedef struct { + half d; + uint16_t scales_h; + uint8_t scales_l[QK_K/64]; + uint8_t qs[QK_K/2]; +} block_iq4_xs; +static_assert(sizeof(block_iq4_xs) == sizeof(ggml_fp16_t) + sizeof(uint16_t) + QK_K/64 + QK_K/2, "wrong iq4_xs block size/padding"); + + #define WARP_SIZE 32 #define MATRIX_ROW_PADDING 512 // last row of quant. matrices is a multiple of this to avoid out-of-bounds memory accesses @@ -2427,6 +2439,25 @@ static __global__ void dequantize_block_iq4_nl(const void * __restrict__ vx, dst } +template +static __global__ void dequantize_block_iq4_xs(const void * __restrict__ vx, dst_t * __restrict__ yy) { + + const int i = blockIdx.x; + const block_iq4_xs * x = (const block_iq4_xs *)vx; + + const int tid = threadIdx.x; + const int il = tid/8; // 0...3 + const int ib = tid%8; // 0...7 + dst_t * y = yy + i*QK_K + 32*ib + 4*il; + const uint8_t * q4 = x[i].qs + 16*ib + 4*il; + const float d = (float)x[i].d * ((((x[i].scales_l[ib/2] >> 4*(ib%2)) & 0xf) | (((x[i].scales_h >> 2*ib) & 3) << 4)) - 32); + for (int j = 0; j < 4; ++j) { + y[j+ 0] = d * kvalues_iq4nl[q4[j] & 0xf]; + y[j+16] = d * kvalues_iq4nl[q4[j] >> 4]; + } + +} + static __global__ void dequantize_mul_mat_vec_q2_k(const void * __restrict__ vx, const float * __restrict__ yy, float * __restrict__ dst, const int ncols, int nrows) { static_assert(16%K_QUANTS_PER_ITERATION == 0, "16 must be divisible by K_QUANTS_PER_ITERATION"); @@ -5286,6 +5317,76 @@ static __device__ __forceinline__ float vec_dot_iq4_nl_q8_1( return d * (sumi1 + sumi2); } +static __device__ __forceinline__ float vec_dot_iq4_xs_q8_1( + const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) { + +#if QK_K == 256 +#if __CUDA_ARCH__ >= MIN_CC_DP4A // lowest compute capability for integer intrinsics + + const block_iq4_xs * bq4 = (const block_iq4_xs *) vbq; + const uint8_t * values = (const uint8_t *)kvalues_iq4nl; + + //// iqs is 0...7 + //const int ib64 = iqs/2; + //const int il = iqs%2; + //const int32_t * q8_1 = (const int *)bq8_1[2*ib64+0].qs + 2*il; + //const int32_t * q8_2 = (const int *)bq8_1[2*ib64+1].qs + 2*il; + //const uint32_t * q4_1 = (const uint32_t *)bq4->qs + 8*ib64 + 2*il; + //const uint32_t * q4_2 = q4_1 + 4; + //const int8_t ls1 = (bq4->scales_l[ib64] & 0xf) | (((bq4->scales_h >> (4*ib64+0)) & 3) << 4); + //const int8_t ls2 = (bq4->scales_l[ib64] >> 4) | (((bq4->scales_h >> (4*ib64+2)) & 3) << 4); + //const float d1 = (float)bq4->d * (ls1 - 32) * __low2float(bq8_1[2*ib64+0].ds); + //const float d2 = (float)bq4->d * (ls2 - 32) * __low2float(bq8_1[2*ib64+1].ds); + //int v1, v2; + //int sumi1 = 0, sumi2 = 0; + //for (int j = 0; j < 2; ++j) { + // get_int_from_table_16(q4_1[j], values, v1, v2); + // sumi1 = __dp4a(v2, q8_1[j+4], __dp4a(v1, q8_1[j+0], sumi1)); + // get_int_from_table_16(q4_2[j], values, v1, v2); + // sumi2 = __dp4a(v2, q8_2[j+4], __dp4a(v1, q8_2[j+0], sumi2)); + //} + //return d1 * sumi1 + d2 * sumi2; + + // iqs is 0...7 + const int ib32 = iqs; + const int32_t * q8 = (const int *)bq8_1[ib32].qs; + const uint32_t * q4 = (const uint32_t *)bq4->qs + 4*ib32; + const int8_t ls = ((bq4->scales_l[ib32/2] >> 4*(ib32%2)) & 0xf) | (((bq4->scales_h >> 2*ib32) & 3) << 4); + const float d = (float)bq4->d * (ls - 32) * __low2float(bq8_1[ib32].ds); + int v1, v2; + int sumi1 = 0, sumi2 = 0; + for (int j = 0; j < 4; ++j) { + get_int_from_table_16(q4[j], values, v1, v2); + sumi1 = __dp4a(v1, q8[j+0], sumi1); + sumi2 = __dp4a(v2, q8[j+4], sumi2); + } + return d * (sumi1 + sumi2); + + //// iqs is 0...15 + //const int ib32 = iqs/2; + //const int il = iqs%2; + //const int32_t * q8 = (const int *)bq8_1[ib32].qs + 2*il; + //const uint32_t * q4 = (const uint32_t *)bq4->qs + 4*ib32 + 2*il; + //const int8_t ls = ((bq4->scales_l[ib32/2] >> 4*(ib32%2)) & 0xf) | (((bq4->scales_h >> 2*ib32) & 3) << 4); + //const float d = (float)bq4->d * (ls - 32) * __low2float(bq8_1[ib32].ds); + //int v1, v2; + //int sumi1 = 0, sumi2 = 0; + //for (int j = 0; j < 2; ++j) { + // get_int_from_table_16(q4[j], values, v1, v2); + // sumi1 = __dp4a(v1, q8[j+0], sumi1); + // sumi2 = __dp4a(v2, q8[j+4], sumi2); + //} + //return d * (sumi1 + sumi2); +#else + assert(false); + return 0.f; +#endif +#else + assert(false); + return 0.f; +#endif +} + template static __device__ __forceinline__ void mul_mat_q( @@ -7340,6 +7441,12 @@ static void dequantize_row_iq4_nl_cuda(const void * vx, dst_t * y, const int k, dequantize_block_iq4_nl<<>>(vx, y); } +template +static void dequantize_row_iq4_xs_cuda(const void * vx, dst_t * y, const int k, cudaStream_t stream) { + const int nb = (k + QK_K - 1) / QK_K; + dequantize_block_iq4_xs<<>>(vx, y); +} + template static void convert_unary_cuda(const void * __restrict__ vx, dst_t * __restrict__ y, const int k, cudaStream_t stream) { const int num_blocks = (k + CUDA_DEQUANTIZE_BLOCK_SIZE - 1) / CUDA_DEQUANTIZE_BLOCK_SIZE; @@ -7385,6 +7492,8 @@ static to_fp16_cuda_t ggml_get_to_fp16_cuda(ggml_type type) { return dequantize_row_iq1_s_cuda; case GGML_TYPE_IQ4_NL: return dequantize_row_iq4_nl_cuda; + case GGML_TYPE_IQ4_XS: + return dequantize_row_iq4_xs_cuda; case GGML_TYPE_IQ3_S: return dequantize_row_iq3_s_cuda; case GGML_TYPE_F32: @@ -7428,6 +7537,8 @@ static to_fp32_cuda_t ggml_get_to_fp32_cuda(ggml_type type) { return dequantize_row_iq1_s_cuda; case GGML_TYPE_IQ4_NL: return dequantize_row_iq4_nl_cuda; + case GGML_TYPE_IQ4_XS: + return dequantize_row_iq4_xs_cuda; case GGML_TYPE_IQ3_S: return dequantize_row_iq3_s_cuda; case GGML_TYPE_F16: @@ -9176,6 +9287,7 @@ static int64_t get_row_rounding(ggml_type type, const std::array= CC_RDNA2 ? 128 : 64; default: @@ -9203,6 +9315,7 @@ static int64_t get_row_rounding(ggml_type type, const std::array= CC_VOLTA ? 128 : 64; case GGML_TYPE_Q6_K: @@ -9313,6 +9426,10 @@ static void ggml_cuda_op_mul_mat_vec_q( mul_mat_vec_q_cuda (src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_padded_row_size, src1_ncols, nrows_dst, stream); break; + case GGML_TYPE_IQ4_XS: + mul_mat_vec_q_cuda + (src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_padded_row_size, src1_ncols, nrows_dst, stream); + break; case GGML_TYPE_IQ3_S: mul_mat_vec_q_cuda (src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_padded_row_size, src1_ncols, nrows_dst, stream); @@ -12041,7 +12158,7 @@ GGML_CALL static bool ggml_backend_cuda_supports_op(ggml_backend_t backend, cons ggml_type a_type = a->type; if (a_type == GGML_TYPE_IQ2_XXS || a_type == GGML_TYPE_IQ2_XS || a_type == GGML_TYPE_IQ3_XXS || a_type == GGML_TYPE_IQ1_S || a_type == GGML_TYPE_IQ4_NL || a_type == GGML_TYPE_IQ3_S || - a_type == GGML_TYPE_IQ2_S) { + a_type == GGML_TYPE_IQ2_S || a_type == GGML_TYPE_IQ4_XS) { if (b->ne[1] == 1 && ggml_nrows(b) > 1) { return false; } diff --git a/ggml-metal.m b/ggml-metal.m index 251d04fb0..9eba2f5d2 100644 --- a/ggml-metal.m +++ b/ggml-metal.m @@ -65,6 +65,7 @@ enum ggml_metal_kernel_type { GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ2_S, GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ1_S, GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ4_NL, + GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ4_XS, GGML_METAL_KERNEL_TYPE_GET_ROWS_I32, GGML_METAL_KERNEL_TYPE_RMS_NORM, GGML_METAL_KERNEL_TYPE_GROUP_NORM, @@ -91,6 +92,7 @@ enum ggml_metal_kernel_type { GGML_METAL_KERNEL_TYPE_MUL_MV_IQ2_S_F32, GGML_METAL_KERNEL_TYPE_MUL_MV_IQ1_S_F32, GGML_METAL_KERNEL_TYPE_MUL_MV_IQ4_NL_F32, + GGML_METAL_KERNEL_TYPE_MUL_MV_IQ4_XS_F32, GGML_METAL_KERNEL_TYPE_MUL_MV_ID_F32_F32, //GGML_METAL_KERNEL_TYPE_MUL_MV_ID_F16_F16, GGML_METAL_KERNEL_TYPE_MUL_MV_ID_F16_F32, @@ -113,6 +115,7 @@ enum ggml_metal_kernel_type { GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ2_S_F32, GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ1_S_F32, GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ4_NL_F32, + GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ4_XS_F32, GGML_METAL_KERNEL_TYPE_MUL_MM_F32_F32, GGML_METAL_KERNEL_TYPE_MUL_MM_F16_F32, GGML_METAL_KERNEL_TYPE_MUL_MM_Q4_0_F32, @@ -132,6 +135,7 @@ enum ggml_metal_kernel_type { GGML_METAL_KERNEL_TYPE_MUL_MM_IQ2_S_F32, GGML_METAL_KERNEL_TYPE_MUL_MM_IQ1_S_F32, GGML_METAL_KERNEL_TYPE_MUL_MM_IQ4_NL_F32, + GGML_METAL_KERNEL_TYPE_MUL_MM_IQ4_XS_F32, GGML_METAL_KERNEL_TYPE_MUL_MM_ID_F32_F32, GGML_METAL_KERNEL_TYPE_MUL_MM_ID_F16_F32, GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q4_0_F32, @@ -151,6 +155,7 @@ enum ggml_metal_kernel_type { GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ2_S_F32, GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ1_S_F32, GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ4_NL_F32, + GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ4_XS_F32, GGML_METAL_KERNEL_TYPE_ROPE_F32, GGML_METAL_KERNEL_TYPE_ROPE_F16, GGML_METAL_KERNEL_TYPE_ALIBI_F32, @@ -466,6 +471,7 @@ static struct ggml_metal_context * ggml_metal_init(int n_cb) { GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ2_S, get_rows_iq2_s, true); GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ1_S, get_rows_iq1_s, true); GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ4_NL, get_rows_iq4_nl, true); + GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ4_XS, get_rows_iq4_xs, true); GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_I32, get_rows_i32, true); GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_RMS_NORM, rms_norm, ctx->support_simdgroup_reduction); GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GROUP_NORM, group_norm, ctx->support_simdgroup_reduction); @@ -492,6 +498,7 @@ static struct ggml_metal_context * ggml_metal_init(int n_cb) { GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_IQ2_S_F32, mul_mv_iq2_s_f32, ctx->support_simdgroup_reduction); GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_IQ1_S_F32, mul_mv_iq1_s_f32, ctx->support_simdgroup_reduction); GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_IQ4_NL_F32, mul_mv_iq4_nl_f32, ctx->support_simdgroup_reduction); + GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_IQ4_XS_F32, mul_mv_iq4_xs_f32, ctx->support_simdgroup_reduction); GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_F32_F32, mul_mv_id_f32_f32, ctx->support_simdgroup_reduction); //GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_F16_F16, mul_mv_id_f16_f16, ctx->support_simdgroup_reduction); GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_F16_F32, mul_mv_id_f16_f32, ctx->support_simdgroup_reduction); @@ -514,6 +521,7 @@ static struct ggml_metal_context * ggml_metal_init(int n_cb) { GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ2_S_F32, mul_mv_id_iq2_s_f32, ctx->support_simdgroup_reduction); GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ1_S_F32, mul_mv_id_iq1_s_f32, ctx->support_simdgroup_reduction); GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ4_NL_F32, mul_mv_id_iq4_nl_f32, ctx->support_simdgroup_reduction); + GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ4_XS_F32, mul_mv_id_iq4_xs_f32, ctx->support_simdgroup_reduction); GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_F32_F32, mul_mm_f32_f32, ctx->support_simdgroup_mm); GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_F16_F32, mul_mm_f16_f32, ctx->support_simdgroup_mm); GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_Q4_0_F32, mul_mm_q4_0_f32, ctx->support_simdgroup_mm); @@ -533,6 +541,7 @@ static struct ggml_metal_context * ggml_metal_init(int n_cb) { GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_IQ2_S_F32, mul_mm_iq2_s_f32, ctx->support_simdgroup_mm); GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_IQ1_S_F32, mul_mm_iq1_s_f32, ctx->support_simdgroup_mm); GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_IQ4_NL_F32, mul_mm_iq4_nl_f32, ctx->support_simdgroup_mm); + GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_IQ4_XS_F32, mul_mm_iq4_xs_f32, ctx->support_simdgroup_mm); GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_F32_F32, mul_mm_id_f32_f32, ctx->support_simdgroup_mm); GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_F16_F32, mul_mm_id_f16_f32, ctx->support_simdgroup_mm); GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q4_0_F32, mul_mm_id_q4_0_f32, ctx->support_simdgroup_mm); @@ -552,6 +561,7 @@ static struct ggml_metal_context * ggml_metal_init(int n_cb) { GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ2_S_F32, mul_mm_id_iq2_s_f32, ctx->support_simdgroup_mm); GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ1_S_F32, mul_mm_id_iq1_s_f32, ctx->support_simdgroup_mm); GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ4_NL_F32, mul_mm_id_iq4_nl_f32, ctx->support_simdgroup_mm); + GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ4_XS_F32, mul_mm_id_iq4_xs_f32, ctx->support_simdgroup_mm); GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_ROPE_F32, rope_f32, true); GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_ROPE_F16, rope_f16, true); GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_ALIBI_F32, alibi_f32, true); @@ -1371,6 +1381,7 @@ static bool ggml_metal_graph_compute( case GGML_TYPE_IQ2_S: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_IQ2_S_F32 ].pipeline; break; case GGML_TYPE_IQ1_S: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_IQ1_S_F32 ].pipeline; break; case GGML_TYPE_IQ4_NL: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_IQ4_NL_F32 ].pipeline; break; + case GGML_TYPE_IQ4_XS: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_IQ4_XS_F32 ].pipeline; break; default: GGML_ASSERT(false && "MUL MAT-MAT not implemented"); } @@ -1529,6 +1540,12 @@ static bool ggml_metal_graph_compute( nth1 = 16; pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_IQ4_NL_F32].pipeline; } break; + case GGML_TYPE_IQ4_XS: + { + nth0 = 4; + nth1 = 16; + pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_IQ4_XS_F32].pipeline; + } break; default: { GGML_METAL_LOG_ERROR("Asserting on type %d\n", (int)src0t); @@ -1576,7 +1593,7 @@ static bool ggml_metal_graph_compute( [encoder setThreadgroupMemoryLength:mem_size atIndex:0]; [encoder dispatchThreadgroups:MTLSizeMake((ne01 + 7)/8, ne11, ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)]; } - else if (src0t == GGML_TYPE_IQ4_NL) { + else if (src0t == GGML_TYPE_IQ4_NL || src0t == GGML_TYPE_IQ4_XS) { const int mem_size = 32*sizeof(float); [encoder setThreadgroupMemoryLength:mem_size atIndex:0]; [encoder dispatchThreadgroups:MTLSizeMake((ne01 + 3)/4, ne11, ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)]; @@ -1678,6 +1695,7 @@ static bool ggml_metal_graph_compute( case GGML_TYPE_IQ2_S: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ2_S_F32 ].pipeline; break; case GGML_TYPE_IQ1_S: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ1_S_F32 ].pipeline; break; case GGML_TYPE_IQ4_NL: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ4_NL_F32 ].pipeline; break; + case GGML_TYPE_IQ4_XS: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ4_XS_F32 ].pipeline; break; default: GGML_ASSERT(false && "MUL_MAT_ID not implemented"); } @@ -1839,6 +1857,12 @@ static bool ggml_metal_graph_compute( nth1 = 16; pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ4_NL_F32].pipeline; } break; + case GGML_TYPE_IQ4_XS: + { + nth0 = 4; + nth1 = 16; + pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ4_XS_F32].pipeline; + } break; default: { GGML_METAL_LOG_ERROR("Asserting on type %d\n", (int)src2t); @@ -1902,7 +1926,7 @@ static bool ggml_metal_graph_compute( [encoder setThreadgroupMemoryLength:mem_size atIndex:0]; [encoder dispatchThreadgroups:MTLSizeMake((ne21 + 7)/8, _ne1, ne01*ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)]; } - else if (src2t == GGML_TYPE_IQ4_NL) { + else if (src2t == GGML_TYPE_IQ4_NL || src2t == GGML_TYPE_IQ4_XS) { const int mem_size = 32*sizeof(float); [encoder setThreadgroupMemoryLength:mem_size atIndex:0]; [encoder dispatchThreadgroups:MTLSizeMake((ne21 + 3)/4, _ne1, ne01*ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)]; @@ -1952,6 +1976,7 @@ static bool ggml_metal_graph_compute( case GGML_TYPE_IQ2_S: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ2_S ].pipeline; break; case GGML_TYPE_IQ1_S: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ1_S ].pipeline; break; case GGML_TYPE_IQ4_NL: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ4_NL ].pipeline; break; + case GGML_TYPE_IQ4_XS: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ4_XS ].pipeline; break; case GGML_TYPE_I32: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GET_ROWS_I32 ].pipeline; break; default: GGML_ASSERT(false && "not implemented"); } diff --git a/ggml-metal.metal b/ggml-metal.metal index 47354e952..689411903 100644 --- a/ggml-metal.metal +++ b/ggml-metal.metal @@ -2560,6 +2560,13 @@ typedef struct { uint8_t qs[QK4_NL/2]; } block_iq4_nl; +typedef struct { + half d; + uint16_t scales_h; + uint8_t scales_l[QK_K/64]; + uint8_t qs[QK_K/2]; +} block_iq4_xs; + //====================================== dot products ========================= void kernel_mul_mv_q2_K_f32_impl( @@ -5160,6 +5167,100 @@ void kernel_mul_mv_iq4_nl_f32_impl( } } +void kernel_mul_mv_iq4_xs_f32_impl( + device const void * src0, + device const float * src1, + device float * dst, + constant int64_t & ne00, + constant int64_t & ne01, + constant int64_t & ne02, + constant int64_t & ne10, + constant int64_t & ne12, + constant int64_t & ne0, + constant int64_t & ne1, + constant uint & r2, + constant uint & r3, + threadgroup float * shared_values [[threadgroup(0)]], + uint3 tgpig[[threadgroup_position_in_grid]], + uint tiisg[[thread_index_in_simdgroup]], + uint sgitg[[simdgroup_index_in_threadgroup]]) { + + const int nb = ne00/QK_K; + const int r0 = tgpig.x; + const int r1 = tgpig.y; + const int im = tgpig.z; + const int first_row = (r0 * 2 + sgitg) * 2; + const int ib_row = first_row * nb; + + const uint i12 = im%ne12; + const uint i13 = im/ne12; + + const uint offset0 = (i12/r2)*(nb*ne01) + (i13/r3)*(nb*ne01*ne02); + device const block_iq4_xs * x = (device const block_iq4_xs *) src0 + ib_row + offset0; + device const float * y = (device const float *) src1 + r1*ne10 + im*ne00*ne1; + + const int ix = tiisg/16; // 0 or 1 + const int it = tiisg%16; // 0...15 + const int ib = it/2; + const int il = it%2; + + shared_values[tiisg] = kvalues_iq4nl_f[tiisg%16]; + threadgroup_barrier(mem_flags::mem_threadgroup); + + float4 yl[4]; + float sumf[2]={0.f}, all_sum; + + device const float * yb = y + ix * QK_K + ib * 32 + il * 8; + + uint32_t aux32[2]; + thread const uint8_t * q8 = (thread const uint8_t *)aux32; + + float4 qf1, qf2; + + for (int ibl = ix; ibl < nb; ibl += 2) { + + device const float4 * y4 = (device const float4 *)yb; + yl[0] = y4[0]; yl[1] = y4[4]; yl[2] = y4[1]; yl[3] = y4[5]; + + for (int row = 0; row < 2; ++row) { + + device const block_iq4_xs & xb = x[row*nb + ibl]; + device const uint32_t * q4 = (device const uint32_t *)(xb.qs + 16*ib + 8*il); + + float4 acc1 = {0.f}, acc2 = {0.f}; + + aux32[0] = q4[0] & 0x0f0f0f0f; + aux32[1] = (q4[0] >> 4) & 0x0f0f0f0f; + qf1 = {shared_values[q8[0]], shared_values[q8[1]], shared_values[q8[2]], shared_values[q8[3]]}; + qf2 = {shared_values[q8[4]], shared_values[q8[5]], shared_values[q8[6]], shared_values[q8[7]]}; + acc1 += yl[0] * qf1; + acc2 += yl[1] * qf2; + + aux32[0] = q4[1] & 0x0f0f0f0f; + aux32[1] = (q4[1] >> 4) & 0x0f0f0f0f; + qf1 = {shared_values[q8[0]], shared_values[q8[1]], shared_values[q8[2]], shared_values[q8[3]]}; + qf2 = {shared_values[q8[4]], shared_values[q8[5]], shared_values[q8[6]], shared_values[q8[7]]}; + acc1 += yl[2] * qf1; + acc2 += yl[3] * qf2; + + acc1 += acc2; + + const int ls = (((xb.scales_l[ib/2] >> 4*(ib%2)) & 0xf) | (((xb.scales_h >> 2*ib) & 3) << 4)) - 32; + sumf[row] += (float)xb.d * ls * (acc1[0] + acc1[1] + acc1[2] + acc1[3]); + + } + + yb += 2 * QK_K; + } + + for (int row = 0; row < 2; ++row) { + all_sum = simd_sum(sumf[row]); + if (tiisg == 0) { + dst[r1*ne0 + im*ne0*ne1 + first_row + row] = all_sum; + } + } +} + [[host_name("kernel_mul_mv_iq1_s_f32")]] kernel void kernel_mul_mv_iq1_s_f32( device const void * src0, @@ -5217,6 +5318,35 @@ kernel void kernel_mul_mv_iq4_nl_f32( kernel_mul_mv_iq4_nl_f32_impl(src0, src1, dst, ne00, ne01, ne02, ne10, ne12, ne0, ne1, r2, r3, shared_values, tgpig, tiisg, sgitg); } +[[host_name("kernel_mul_mv_iq4_xs_f32")]] +kernel void kernel_mul_mv_iq4_xs_f32( + device const void * src0, + device const float * src1, + device float * dst, + constant int64_t & ne00, + constant int64_t & ne01, + constant int64_t & ne02, + constant uint64_t & nb00, + constant uint64_t & nb01, + constant uint64_t & nb02, + constant int64_t & ne10, + constant int64_t & ne11, + constant int64_t & ne12, + constant uint64_t & nb10, + constant uint64_t & nb11, + constant uint64_t & nb12, + constant int64_t & ne0, + constant int64_t & ne1, + constant uint & r2, + constant uint & r3, + threadgroup float * shared_values [[threadgroup(0)]], + uint3 tgpig[[threadgroup_position_in_grid]], + uint tiisg[[thread_index_in_simdgroup]], + uint sgitg[[simdgroup_index_in_threadgroup]]) { + + kernel_mul_mv_iq4_xs_f32_impl(src0, src1, dst, ne00, ne01, ne02, ne10, ne12, ne0, ne1, r2, r3, shared_values, tgpig, tiisg, sgitg); +} + //============================= templates and their specializations ============================= // NOTE: this is not dequantizing - we are simply fitting the template @@ -5638,6 +5768,26 @@ void dequantize_iq4_nl(device const block_iq4_nl * xb, short il, thread type4x4 } } +template +void dequantize_iq4_xs(device const block_iq4_xs * xb, short il, thread type4x4 & reg) { + // il is 0...15 for QK_K = 256 => index of block of 32 is il/2 + const int ib32 = il/2; + il = il%2; + // il = 0 or 1. il = 0 processes the first 16 quants in a block of 32, il = 1 the second 16 + device const uint32_t * q4 = (device const uint32_t *)xb->qs + 4*ib32; + const int ls = ((xb->scales_l[ib32/2] >> 4*(ib32%2)) & 0xf) | (((xb->scales_h >> 2*ib32) & 3) << 4); + const float d = (float)xb->d * (ls - 32); + uint32_t aux32; + thread const uint8_t * q8 = (thread const uint8_t *)&aux32; + for (int i = 0; i < 4; ++i) { + aux32 = (q4[i] >> 4*il) & 0x0f0f0f0f; + reg[i][0] = d * kvalues_iq4nl_f[q8[0]]; + reg[i][1] = d * kvalues_iq4nl_f[q8[1]]; + reg[i][2] = d * kvalues_iq4nl_f[q8[2]]; + reg[i][3] = d * kvalues_iq4nl_f[q8[3]]; + } +} + template kernel void kernel_get_rows( device const void * src0, @@ -6183,7 +6333,8 @@ template [[host_name("kernel_get_rows_iq3_xxs")]] kernel get_rows_t kernel_get_r template [[host_name("kernel_get_rows_iq3_s")]] kernel get_rows_t kernel_get_rows; template [[host_name("kernel_get_rows_iq2_s")]] kernel get_rows_t kernel_get_rows; template [[host_name("kernel_get_rows_iq1_s")]] kernel get_rows_t kernel_get_rows; -template [[host_name("kernel_get_rows_iq4_nl")]] kernel get_rows_t kernel_get_rows; +template [[host_name("kernel_get_rows_iq4_nl")]] kernel get_rows_t kernel_get_rows; +template [[host_name("kernel_get_rows_iq4_xs")]] kernel get_rows_t kernel_get_rows; // // matrix-matrix multiplication @@ -6226,7 +6377,8 @@ template [[host_name("kernel_mul_mm_iq3_xxs_f32")]] kernel mat_mm_t kernel_mul_m template [[host_name("kernel_mul_mm_iq3_s_f32")]] kernel mat_mm_t kernel_mul_mm; template [[host_name("kernel_mul_mm_iq2_s_f32")]] kernel mat_mm_t kernel_mul_mm; template [[host_name("kernel_mul_mm_iq1_s_f32")]] kernel mat_mm_t kernel_mul_mm; -template [[host_name("kernel_mul_mm_iq4_nl_f32")]] kernel mat_mm_t kernel_mul_mm; +template [[host_name("kernel_mul_mm_iq4_nl_f32")]] kernel mat_mm_t kernel_mul_mm; +template [[host_name("kernel_mul_mm_iq4_xs_f32")]] kernel mat_mm_t kernel_mul_mm; // // indirect matrix-matrix multiplication @@ -6281,7 +6433,8 @@ template [[host_name("kernel_mul_mm_id_iq3_xxs_f32")]] kernel mat_mm_id_t kernel template [[host_name("kernel_mul_mm_id_iq3_s_f32")]] kernel mat_mm_id_t kernel_mul_mm_id; template [[host_name("kernel_mul_mm_id_iq2_s_f32")]] kernel mat_mm_id_t kernel_mul_mm_id; template [[host_name("kernel_mul_mm_id_iq1_s_f32")]] kernel mat_mm_id_t kernel_mul_mm_id; -template [[host_name("kernel_mul_mm_id_iq4_nl_f32")]] kernel mat_mm_id_t kernel_mul_mm_id; +template [[host_name("kernel_mul_mm_id_iq4_nl_f32")]] kernel mat_mm_id_t kernel_mul_mm_id; +template [[host_name("kernel_mul_mm_id_iq4_xs_f32")]] kernel mat_mm_id_t kernel_mul_mm_id; // // matrix-vector multiplication @@ -7507,3 +7660,68 @@ kernel void kernel_mul_mv_id_iq4_nl_f32( tiisg, sgitg); } + +[[host_name("kernel_mul_mv_id_iq4_xs_f32")]] +kernel void kernel_mul_mv_id_iq4_xs_f32( + device const char * ids, + device const char * src1, + device float * dst, + constant uint64_t & nbi1, + constant int64_t & ne00, + constant int64_t & ne01, + constant int64_t & ne02, + constant uint64_t & nb00, + constant uint64_t & nb01, + constant uint64_t & nb02, + constant int64_t & ne10, + constant int64_t & ne11, + constant int64_t & ne12, + constant int64_t & ne13, + constant uint64_t & nb10, + constant uint64_t & nb11, + constant uint64_t & nb12, + constant int64_t & ne0, + constant int64_t & ne1, + constant uint64_t & nb1, + constant uint & r2, + constant uint & r3, + constant int & idx, + device const char * src00, + device const char * src01, + device const char * src02, + device const char * src03, + device const char * src04, + device const char * src05, + device const char * src06, + device const char * src07, + threadgroup float * shared_values [[threadgroup(0)]], + uint3 tgpig[[threadgroup_position_in_grid]], + uint tiitg[[thread_index_in_threadgroup]], + uint tiisg[[thread_index_in_simdgroup]], + uint sgitg[[simdgroup_index_in_threadgroup]]) { + device const char * src0[8] = {src00, src01, src02, src03, src04, src05, src06, src07}; + + const int64_t bid = tgpig.z/(ne12*ne13); + + tgpig.z = tgpig.z%(ne12*ne13); + + const int32_t id = ((device int32_t *) (ids + bid*nbi1))[idx]; + + kernel_mul_mv_iq4_xs_f32_impl( + src0[id], + (device const float *) (src1 + bid*nb11), + dst + bid*ne0, + ne00, + ne01, + ne02, + ne10, + ne12, + ne0, + ne1, + r2, + r3, + shared_values, + tgpig, + tiisg, + sgitg); +} diff --git a/ggml-quants.c b/ggml-quants.c index 73c3bb412..607d50925 100644 --- a/ggml-quants.c +++ b/ggml-quants.c @@ -4225,6 +4225,29 @@ void dequantize_row_iq4_nl(const block_iq4_nl * restrict x, float * restrict y, } } +void dequantize_row_iq4_xs(const block_iq4_xs * restrict x, float * restrict y, int k) { + assert(k % QK_K == 0); + const int nb = k / QK_K; + + for (int i = 0; i < nb; i++) { + + const uint8_t * qs = x[i].qs; + + const float d = GGML_FP16_TO_FP32(x[i].d); + + for (int ib = 0; ib < QK_K/32; ++ib) { + const int ls = ((x[i].scales_l[ib/2] >> 4*(ib%2)) & 0xf) | (((x[i].scales_h >> 2*ib) & 3) << 4); + const float dl = d * (ls - 32); + for (int j = 0; j < 16; ++j) { + y[j+ 0] = dl * kvalues_iq4nl[qs[j] & 0xf]; + y[j+16] = dl * kvalues_iq4nl[qs[j] >> 4]; + } + y += 32; + qs += 16; + } + } +} + //===================================== Q8_K ============================================== void quantize_row_q8_K_reference(const float * restrict x, block_q8_K * restrict y, int k) { @@ -9675,8 +9698,8 @@ void ggml_vec_dot_iq2_s_q8_K(int n, float * restrict s, size_t bs, const void * qs += 8; vs.val[0] = vreinterpretq_u8_u32(vdupq_n_u32(signs[0] | (signs[1] << 16))); - vs.val[1] = vandq_u8(vqtbl1q_u8(vs.val[0], mask1.val[1]), mask2); - vs.val[0] = vandq_u8(vqtbl1q_u8(vs.val[0], mask1.val[0]), mask2); + vs.val[1] = vandq_u8(ggml_vqtbl1q_u8(vs.val[0], mask1.val[1]), mask2); + vs.val[0] = vandq_u8(ggml_vqtbl1q_u8(vs.val[0], mask1.val[0]), mask2); vs.val[0] = vceqq_u8(vs.val[0], mask2); vs.val[1] = vceqq_u8(vs.val[1], mask2); @@ -9684,8 +9707,8 @@ void ggml_vec_dot_iq2_s_q8_K(int n, float * restrict s, size_t bs, const void * q2s.val[1] = vmulq_s8(vreinterpretq_s8_u8(vorrq_u8(vs.val[1], m1)), q2s.val[1]); vs.val[0] = vreinterpretq_u8_u32(vdupq_n_u32(signs[2] | (signs[3] << 16))); - vs.val[1] = vandq_u8(vqtbl1q_u8(vs.val[0], mask1.val[1]), mask2); - vs.val[0] = vandq_u8(vqtbl1q_u8(vs.val[0], mask1.val[0]), mask2); + vs.val[1] = vandq_u8(ggml_vqtbl1q_u8(vs.val[0], mask1.val[1]), mask2); + vs.val[0] = vandq_u8(ggml_vqtbl1q_u8(vs.val[0], mask1.val[0]), mask2); vs.val[0] = vceqq_u8(vs.val[0], mask2); vs.val[1] = vceqq_u8(vs.val[1], mask2); @@ -10425,6 +10448,134 @@ void ggml_vec_dot_iq4_nl_q8_0(int n, float * restrict s, size_t bs, const void * #endif } +void ggml_vec_dot_iq4_xs_q8_K(int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) { + assert(nrc == 1); + UNUSED(nrc); + UNUSED(bx); + UNUSED(by); + UNUSED(bs); + assert(n % QK_K == 0); + + const block_iq4_xs * restrict x = vx; + const block_q8_K * restrict y = vy; + + const int nb = n / QK_K; + +#if defined __ARM_NEON + const int8x16_t values = vld1q_s8(kvalues_iq4nl); + const uint8x16_t m4b = vdupq_n_u8(0x0f); + uint8x16x2_t q4bits; + int8x16x4_t q4b; + int8x16x4_t q8b; + int32x4_t prod_1, prod_2; + + float sumf = 0; + + for (int ibl = 0; ibl < nb; ++ibl) { + + const int8_t * q8 = y[ibl].qs; + const uint8_t * q4 = x[ibl].qs; + uint16_t h = x[ibl].scales_h; + + int sumi1 = 0, sumi2 = 0; + for (int ib = 0; ib < QK_K/64; ++ib) { + + q4bits = ggml_vld1q_u8_x2(q4); q4 += 32; + q8b = ggml_vld1q_s8_x4(q8); q8 += 64; + + q4b.val[0] = ggml_vqtbl1q_s8(values, vandq_u8 (q4bits.val[0], m4b)); + q4b.val[1] = ggml_vqtbl1q_s8(values, vshrq_n_u8(q4bits.val[0], 4)); + q4b.val[2] = ggml_vqtbl1q_s8(values, vandq_u8 (q4bits.val[1], m4b)); + q4b.val[3] = ggml_vqtbl1q_s8(values, vshrq_n_u8(q4bits.val[1], 4)); + + prod_1 = ggml_vdotq_s32(ggml_vdotq_s32(vdupq_n_s32(0), q4b.val[0], q8b.val[0]), q4b.val[1], q8b.val[1]); + prod_2 = ggml_vdotq_s32(ggml_vdotq_s32(vdupq_n_s32(0), q4b.val[2], q8b.val[2]), q4b.val[3], q8b.val[3]); + + int ls1 = ((x[ibl].scales_l[ib] & 0xf) | ((h << 4) & 0x30)) - 32; + int ls2 = ((x[ibl].scales_l[ib] >> 4) | ((h << 2) & 0x30)) - 32; + h >>= 4; + sumi1 += vaddvq_s32(prod_1) * ls1; + sumi2 += vaddvq_s32(prod_2) * ls2; + + } + + sumf += GGML_FP16_TO_FP32(x[ibl].d) * y[ibl].d * (sumi1 + sumi2); + } + + *s = sumf; + +#elif defined __AVX2__ + + const __m128i values128 = _mm_loadu_si128((const __m128i*)kvalues_iq4nl); + const __m128i m4b = _mm_set1_epi8(0x0f); + + __m256 accum = _mm256_setzero_ps(); + for (int ibl = 0; ibl < nb; ++ibl) { + const uint8_t * qs = x[ibl].qs; + const int8_t * q8 = y[ibl].qs; + uint16_t sh = x[ibl].scales_h; + __m256i sumi1 = _mm256_setzero_si256(); + __m256i sumi2 = _mm256_setzero_si256(); + for (int ib = 0; ib < QK_K/32; ib += 2) { + const __m128i q4bits_1 = _mm_loadu_si128((const __m128i*)qs); qs += 16; + const __m128i q4bits_2 = _mm_loadu_si128((const __m128i*)qs); qs += 16; + const __m256i q8b_1 = _mm256_loadu_si256((const __m256i *)q8); q8 += 32; + const __m256i q8b_2 = _mm256_loadu_si256((const __m256i *)q8); q8 += 32; + const __m256i q4b_1 = _mm256_set_m128i(_mm_shuffle_epi8(values128, _mm_and_si128(_mm_srli_epi16(q4bits_1, 4), m4b)), + _mm_shuffle_epi8(values128, _mm_and_si128(q4bits_1, m4b))); + const __m256i q4b_2 = _mm256_set_m128i(_mm_shuffle_epi8(values128, _mm_and_si128(_mm_srli_epi16(q4bits_2, 4), m4b)), + _mm_shuffle_epi8(values128, _mm_and_si128(q4bits_2, m4b))); + const __m256i p16_1 = mul_add_epi8(q4b_1, q8b_1); + const __m256i p16_2 = mul_add_epi8(q4b_2, q8b_2); + const int16_t ls1 = ((x[ibl].scales_l[ib/2] & 0xf) | ((sh << 4) & 0x30)) - 32; + const int16_t ls2 = ((x[ibl].scales_l[ib/2] >> 4) | ((sh << 2) & 0x30)) - 32; + sh >>= 4; + const __m256i p_1 = _mm256_madd_epi16(p16_1, _mm256_set1_epi16(ls1)); + const __m256i p_2 = _mm256_madd_epi16(p16_2, _mm256_set1_epi16(ls2)); + sumi1 = _mm256_add_epi32(p_1, sumi1); + sumi2 = _mm256_add_epi32(p_2, sumi2); + } + accum = _mm256_fmadd_ps(_mm256_set1_ps(GGML_FP16_TO_FP32(x[ibl].d)*y[ibl].d), + _mm256_cvtepi32_ps(_mm256_add_epi32(sumi1, sumi2)), accum); + } + + *s = hsum_float_8(accum); + +#else + float sumf = 0; + for (int ibl = 0; ibl < nb; ++ibl) { + const float d4d8 = GGML_FP16_TO_FP32(x[ibl].d) * y[ibl].d; + uint16_t h = x[ibl].scales_h; + const uint8_t * qs = x[ibl].qs; + const int8_t * q8 = y[ibl].qs; + for (int ib = 0; ib < QK_K/32; ib += 2) { + const uint8_t ls1 = (x[ibl].scales_l[ib/2] & 0xf) | ((h << 4) & 0x30); + const uint8_t ls2 = (x[ibl].scales_l[ib/2] >> 4) | ((h << 2) & 0x30); + h >>= 4; + const float d1 = d4d8*(ls1 - 32); + const float d2 = d4d8*(ls2 - 32); + int sumi1 = 0, sumi2 = 0; + for (int j = 0; j < 16; ++j) { + sumi1 += q8[j+ 0] * kvalues_iq4nl[qs[j] & 0xf]; + sumi2 += q8[j+16] * kvalues_iq4nl[qs[j] >> 4]; + } + sumf += d1 * (sumi1 + sumi2); + qs += 16; + q8 += 32; + sumi1 = sumi2 = 0; + for (int j = 0; j < 16; ++j) { + sumi1 += q8[j+ 0] * kvalues_iq4nl[qs[j] & 0xf]; + sumi2 += q8[j+16] * kvalues_iq4nl[qs[j] >> 4]; + } + sumf += d2 * (sumi1 + sumi2); + qs += 16; + q8 += 32; + } + } + *s = sumf; +#endif +} + // ================================ IQ2 quantization ============================================= typedef struct { @@ -12021,23 +12172,23 @@ static inline int best_index_int8(int n, const int8_t * val, float x) { return x - val[mu-1] < val[mu] - x ? mu-1 : mu; } -static void quantize_row_iq4_nl_impl(const int block_size, const float * GGML_RESTRICT x, - ggml_fp16_t * dh, uint8_t * q4, - float * weight, uint8_t * L, +static void quantize_row_iq4_nl_impl(const int super_block_size, const int block_size, const float * GGML_RESTRICT x, + ggml_fp16_t * dh, uint8_t * q4, uint16_t * scales_h, uint8_t * scales_l, + float * scales, float * weight, uint8_t * L, const int8_t * values, const float * quant_weights) { const int ntry = 7; float sigma2 = 0; - for (int j = 0; j < QK4_NL; ++j) sigma2 += x[j]*x[j]; - sigma2 *= 2.f/QK4_NL; + for (int j = 0; j < super_block_size; ++j) sigma2 += x[j]*x[j]; + sigma2 *= 2.f/super_block_size; - const int nb = QK4_NL/block_size; + memset(q4, 0, super_block_size/2); + dh[0] = GGML_FP32_TO_FP16(0.f); - memset(q4, 0, QK4_NL/2); - for (int ib = 0; ib < nb; ++ib) { - dh[ib] = GGML_FP32_TO_FP16(0.f); + float max_scale = 0, amax_scale = 0; + for (int ib = 0; ib < super_block_size/block_size; ++ib) { const float * xb = x + ib*block_size; if (quant_weights) { const float * qw = quant_weights + ib*block_size; @@ -12053,6 +12204,7 @@ static void quantize_row_iq4_nl_impl(const int block_size, const float * GGML_RE } } if (!amax) { + scales[ib] = 0; continue; } float d = -max/values[0]; @@ -12066,7 +12218,6 @@ static void quantize_row_iq4_nl_impl(const int block_size, const float * GGML_RE sumqx += w*q*xb[j]; sumq2 += w*q*q; } - float best_id = id; d = sumqx/sumq2; float best = d*sumqx; for (int itry = -ntry; itry <= ntry; ++itry) { @@ -12082,15 +12233,47 @@ static void quantize_row_iq4_nl_impl(const int block_size, const float * GGML_RE } if (sumq2 > 0 && sumqx*sumqx > best*sumq2) { d = sumqx/sumq2; best = d * sumqx; - best_id = id; } } - dh[ib] = GGML_FP32_TO_FP16(d); - for (int j = 0; j < block_size; ++j) { - L[ib*block_size + j] = best_index_int8(16, values, best_id*xb[j]); + scales[ib] = d; + float abs_d = fabsf(d); + if (abs_d > amax_scale) { + amax_scale = abs_d; max_scale = d; } } - for (int i = 0; i < QK4_NL/32; ++i) { + + if (super_block_size/block_size > 1) { + int nb = super_block_size/block_size; + memset(scales_h, 0, ((nb+7)/8)*sizeof(uint16_t)); + float d = -max_scale/32; + dh[0] = GGML_FP32_TO_FP16(d); + float id = d ? 1/d : 0.f; + for (int ib = 0; ib < super_block_size/block_size; ++ib) { + int l = nearest_int(id*scales[ib]); + l = MAX(-32, MIN(31, l)); + float dl = d * l; + float idl = dl ? 1/dl : 0.f; + uint8_t * Lb = L + ib*block_size; + const float * xb = x + ib*block_size; + for (int j = 0; j < block_size; ++j) { + Lb[j] = best_index_int8(16, values, idl*xb[j]); + } + l += 32; + uint8_t l_l = l & 0xf; + uint8_t l_h = l >> 4; + if (ib%2 == 0) scales_l[ib/2] = l_l; + else scales_l[ib/2] |= (l_l << 4); + scales_h[ib/8] |= (l_h << 2*(ib%8)); + } + } else { + dh[0] = GGML_FP32_TO_FP16(scales[0]); + float id = scales[0] ? 1/scales[0] : 0; + for (int j = 0; j < super_block_size; ++j) { + L[j] = best_index_int8(16, values, id*x[j]); + } + } + + for (int i = 0; i < super_block_size/32; ++i) { for (int j = 0; j < 16; ++j) { q4[16*i + j] = L[32*i + j] | (L[32*i + 16 + j] << 4); } @@ -12103,12 +12286,16 @@ size_t quantize_iq4_nl(const float * src, void * dst, int nrow, int n_per_row, i int nblock = n_per_row/QK4_NL; char * qrow = (char *)dst; uint8_t L[QK4_NL]; - float weight[32]; + float weight[QK4_NL]; + uint16_t unused_h; + uint8_t * unused_l = NULL; + float scale; for (int row = 0; row < nrow; ++row) { block_iq4_nl * iq4 = (block_iq4_nl *)qrow; for (int ibl = 0; ibl < nblock; ++ibl) { const float * qw = quant_weights ? quant_weights + QK4_NL*ibl : NULL; - quantize_row_iq4_nl_impl(32, src + QK4_NL*ibl, &iq4[ibl].d, iq4[ibl].qs, weight, L, kvalues_iq4nl, qw); + quantize_row_iq4_nl_impl(QK4_NL, 32, src + QK4_NL*ibl, &iq4[ibl].d, iq4[ibl].qs, &unused_h, unused_l, + &scale, weight, L, kvalues_iq4nl, qw); } src += n_per_row; qrow += nblock*sizeof(block_iq4_nl); @@ -12127,6 +12314,38 @@ void quantize_row_iq4_nl_reference(const float * restrict x, block_iq4_nl * rest quantize_iq4_nl(x, y, 1, k, NULL, NULL); } +size_t quantize_iq4_xs(const float * src, void * dst, int nrow, int n_per_row, int64_t * hist, const float * quant_weights) { + (void)hist; + GGML_ASSERT(n_per_row%QK_K == 0); + int nblock = n_per_row/QK_K; + char * qrow = (char *)dst; + uint8_t L[QK_K]; + float weight[32]; + float scales[QK_K/32]; + for (int row = 0; row < nrow; ++row) { + block_iq4_xs * iq4 = (block_iq4_xs *)qrow; + for (int ibl = 0; ibl < nblock; ++ibl) { + const float * qw = quant_weights ? quant_weights + QK_K*ibl : NULL; + quantize_row_iq4_nl_impl(QK_K, 32, src + QK_K*ibl, &iq4[ibl].d, iq4[ibl].qs, &iq4[ibl].scales_h, iq4[ibl].scales_l, + scales, weight, L, kvalues_iq4nl, qw); + } + src += n_per_row; + qrow += nblock*sizeof(block_iq4_xs); + } + return nrow * nblock * sizeof(block_iq4_xs); +} + +void quantize_row_iq4_xs(const float * restrict x, void * restrict vy, int k) { + assert(k % QK_K == 0); + block_iq4_xs * restrict y = vy; + quantize_row_iq4_xs_reference(x, y, k); +} + +void quantize_row_iq4_xs_reference(const float * restrict x, block_iq4_xs * restrict y, int k) { + assert(k % QK_K == 0); + quantize_iq4_xs(x, y, 1, k, NULL, NULL); +} + // =============================== 2.5625 bpw static void quantize_row_iq2_s_impl(const float * restrict x, void * restrict vy, int n, const float * restrict quant_weights) { diff --git a/ggml-quants.h b/ggml-quants.h index 4731dde0c..2c61134c4 100644 --- a/ggml-quants.h +++ b/ggml-quants.h @@ -230,6 +230,14 @@ typedef struct { } block_iq4_nl; static_assert(sizeof(block_iq4_nl) == sizeof(ggml_fp16_t) + QK4_NL/2, "wrong iq4_nl block size/padding"); +typedef struct { + ggml_fp16_t d; + uint16_t scales_h; + uint8_t scales_l[QK_K/64]; + uint8_t qs[QK_K/2]; +} block_iq4_xs; +static_assert(sizeof(block_iq4_xs) == sizeof(ggml_fp16_t) + sizeof(uint16_t) + QK_K/64 + QK_K/2, "wrong iq4_xs block size/padding"); + #ifdef __cplusplus extern "C" { #endif @@ -250,6 +258,7 @@ void quantize_row_q6_K_reference(const float * GGML_RESTRICT x, block_q6_K * GGM void quantize_row_q8_K_reference(const float * GGML_RESTRICT x, block_q8_K * GGML_RESTRICT y, int k); void quantize_row_iq3_xxs_reference(const float * GGML_RESTRICT x, block_iq3_xxs * GGML_RESTRICT y, int k); void quantize_row_iq4_nl_reference (const float * GGML_RESTRICT x, block_iq4_nl * GGML_RESTRICT y, int k); +void quantize_row_iq4_xs_reference (const float * GGML_RESTRICT x, block_iq4_xs * GGML_RESTRICT y, int k); void quantize_row_iq3_s_reference (const float * GGML_RESTRICT x, block_iq3_s * GGML_RESTRICT y, int k); void quantize_row_iq2_s_reference (const float * GGML_RESTRICT x, block_iq2_s * GGML_RESTRICT y, int k); @@ -268,6 +277,7 @@ void quantize_row_q6_K(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, in void quantize_row_q8_K(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int k); void quantize_row_iq3_xxs(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int k); void quantize_row_iq4_nl (const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int k); +void quantize_row_iq4_xs (const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int k); void quantize_row_iq3_s (const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int k); void quantize_row_iq2_s (const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int k); @@ -291,6 +301,7 @@ void dequantize_row_iq2_s (const block_iq2_s * GGML_RESTRICT x, float * GGML_ void dequantize_row_iq3_xxs(const block_iq3_xxs * GGML_RESTRICT x, float * GGML_RESTRICT y, int k); void dequantize_row_iq1_s (const block_iq1_s * GGML_RESTRICT x, float * GGML_RESTRICT y, int k); void dequantize_row_iq4_nl (const block_iq4_nl * GGML_RESTRICT x, float * GGML_RESTRICT y, int k); +void dequantize_row_iq4_xs (const block_iq4_xs * GGML_RESTRICT x, float * GGML_RESTRICT y, int k); void dequantize_row_iq3_s (const block_iq3_s * GGML_RESTRICT x, float * GGML_RESTRICT y, int k); // Dot product @@ -311,6 +322,7 @@ void ggml_vec_dot_iq2_s_q8_K (int n, float * GGML_RESTRICT s, size_t bs, const void ggml_vec_dot_iq3_xxs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); void ggml_vec_dot_iq1_s_q8_K (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); void ggml_vec_dot_iq4_nl_q8_0 (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); +void ggml_vec_dot_iq4_xs_q8_K (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); void ggml_vec_dot_iq3_s_q8_K (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); // @@ -322,6 +334,7 @@ size_t quantize_iq2_s (const float * src, void * dst, int nrows, int n_per_row, size_t quantize_iq3_xxs(const float * src, void * dst, int nrows, int n_per_row, int64_t * hist, const float * imatrix); size_t quantize_iq1_s (const float * src, void * dst, int nrows, int n_per_row, int64_t * hist, const float * imatrix); size_t quantize_iq4_nl (const float * src, void * dst, int nrows, int n_per_row, int64_t * hist, const float * imatrix); +size_t quantize_iq4_xs (const float * src, void * dst, int nrows, int n_per_row, int64_t * hist, const float * imatrix); size_t quantize_iq3_s (const float * src, void * dst, int nrows, int n_per_row, int64_t * hist, const float * imatrix); size_t quantize_q2_K (const float * src, void * dst, int nrows, int n_per_row, int64_t * hist, const float * imatrix); size_t quantize_q3_K (const float * src, void * dst, int nrows, int n_per_row, int64_t * hist, const float * imatrix); diff --git a/ggml.c b/ggml.c index 6be07bb6f..d66db3352 100644 --- a/ggml.c +++ b/ggml.c @@ -726,6 +726,18 @@ static const ggml_type_traits_t type_traits[GGML_TYPE_COUNT] = { .vec_dot_type = GGML_TYPE_Q8_0, .nrows = 1, }, + [GGML_TYPE_IQ4_XS] = { + .type_name = "iq4_xs", + .blck_size = QK_K, + .type_size = sizeof(block_iq4_xs), + .is_quantized = true, + .to_float = (ggml_to_float_t) dequantize_row_iq4_xs, + .from_float = quantize_row_iq4_xs, + .from_float_reference = (ggml_from_float_t)quantize_row_iq4_xs_reference, + .vec_dot = ggml_vec_dot_iq4_xs_q8_K, + .vec_dot_type = GGML_TYPE_Q8_K, + .nrows = 1, + }, [GGML_TYPE_Q8_K] = { .type_name = "q8_K", .blck_size = QK_K, @@ -2328,6 +2340,7 @@ enum ggml_type ggml_ftype_to_ggml_type(enum ggml_ftype ftype) { case GGML_FTYPE_MOSTLY_IQ3_XXS: wtype = GGML_TYPE_IQ3_XXS; break; case GGML_FTYPE_MOSTLY_IQ1_S: wtype = GGML_TYPE_IQ1_S; break; case GGML_FTYPE_MOSTLY_IQ4_NL: wtype = GGML_TYPE_IQ4_NL; break; + case GGML_FTYPE_MOSTLY_IQ4_XS: wtype = GGML_TYPE_IQ4_XS; break; case GGML_FTYPE_MOSTLY_IQ3_S: wtype = GGML_TYPE_IQ3_S; break; case GGML_FTYPE_MOSTLY_IQ2_S: wtype = GGML_TYPE_IQ2_S; break; case GGML_FTYPE_UNKNOWN: wtype = GGML_TYPE_COUNT; break; @@ -7764,6 +7777,7 @@ static void ggml_compute_forward_add( case GGML_TYPE_IQ3_XXS: case GGML_TYPE_IQ1_S: case GGML_TYPE_IQ4_NL: + case GGML_TYPE_IQ4_XS: case GGML_TYPE_IQ3_S: case GGML_TYPE_IQ2_S: { @@ -8045,6 +8059,7 @@ static void ggml_compute_forward_add1( case GGML_TYPE_IQ3_XXS: case GGML_TYPE_IQ1_S: case GGML_TYPE_IQ4_NL: + case GGML_TYPE_IQ4_XS: case GGML_TYPE_IQ3_S: case GGML_TYPE_IQ2_S: { @@ -8171,6 +8186,7 @@ static void ggml_compute_forward_acc( case GGML_TYPE_IQ3_XXS: case GGML_TYPE_IQ1_S: case GGML_TYPE_IQ4_NL: + case GGML_TYPE_IQ4_XS: case GGML_TYPE_IQ3_S: case GGML_TYPE_IQ2_S: default: @@ -11071,6 +11087,7 @@ static void ggml_compute_forward_out_prod( case GGML_TYPE_IQ3_XXS: case GGML_TYPE_IQ1_S: case GGML_TYPE_IQ4_NL: + case GGML_TYPE_IQ4_XS: case GGML_TYPE_IQ3_S: case GGML_TYPE_IQ2_S: { @@ -11261,6 +11278,7 @@ static void ggml_compute_forward_set( case GGML_TYPE_IQ3_XXS: case GGML_TYPE_IQ1_S: case GGML_TYPE_IQ4_NL: + case GGML_TYPE_IQ4_XS: case GGML_TYPE_IQ3_S: case GGML_TYPE_IQ2_S: default: @@ -11465,6 +11483,7 @@ static void ggml_compute_forward_get_rows( case GGML_TYPE_IQ3_XXS: case GGML_TYPE_IQ1_S: case GGML_TYPE_IQ4_NL: + case GGML_TYPE_IQ4_XS: case GGML_TYPE_IQ3_S: case GGML_TYPE_IQ2_S: { @@ -12167,6 +12186,7 @@ static void ggml_compute_forward_alibi( case GGML_TYPE_IQ3_XXS: case GGML_TYPE_IQ1_S: case GGML_TYPE_IQ4_NL: + case GGML_TYPE_IQ4_XS: case GGML_TYPE_IQ3_S: case GGML_TYPE_IQ2_S: case GGML_TYPE_Q8_K: @@ -12252,6 +12272,7 @@ static void ggml_compute_forward_clamp( case GGML_TYPE_IQ3_XXS: case GGML_TYPE_IQ1_S: case GGML_TYPE_IQ4_NL: + case GGML_TYPE_IQ4_XS: case GGML_TYPE_IQ3_S: case GGML_TYPE_IQ2_S: case GGML_TYPE_Q8_K: @@ -19817,6 +19838,15 @@ size_t ggml_quantize_chunk(enum ggml_type type, const float * src, void * dst, i result = quantize_iq4_nl(src + start, (char *)dst + start_row * row_size, nrows, n_per_row, hist, imatrix); GGML_ASSERT(result == row_size * nrows); } break; + case GGML_TYPE_IQ4_XS: + { + GGML_ASSERT(start % QK4_NL == 0); + GGML_ASSERT(start % n_per_row == 0); + size_t start_row = start / n_per_row; + size_t row_size = ggml_row_size(type, n_per_row); + result = quantize_iq4_xs(src + start, (char *)dst + start_row * row_size, nrows, n_per_row, hist, imatrix); + GGML_ASSERT(result == row_size * nrows); + } break; case GGML_TYPE_F16: { size_t elemsize = sizeof(ggml_fp16_t); diff --git a/ggml.h b/ggml.h index 8c7ca4588..23b768640 100644 --- a/ggml.h +++ b/ggml.h @@ -352,6 +352,7 @@ extern "C" { GGML_TYPE_IQ4_NL = 20, GGML_TYPE_IQ3_S = 21, GGML_TYPE_IQ2_S = 22, + GGML_TYPE_IQ4_XS = 23, GGML_TYPE_I8, GGML_TYPE_I16, GGML_TYPE_I32, @@ -393,6 +394,7 @@ extern "C" { GGML_FTYPE_MOSTLY_IQ4_NL = 19, // except 1d tensors GGML_FTYPE_MOSTLY_IQ3_S = 20, // except 1d tensors GGML_FTYPE_MOSTLY_IQ2_S = 21, // except 1d tensors + GGML_FTYPE_MOSTLY_IQ4_XS = 22, // except 1d tensors }; // available tensor operations: diff --git a/llama.cpp b/llama.cpp index 6729bb99c..464e1b89b 100644 --- a/llama.cpp +++ b/llama.cpp @@ -2584,6 +2584,7 @@ struct llama_model_loader { case GGML_TYPE_IQ3_XXS: ftype = LLAMA_FTYPE_MOSTLY_IQ3_XXS; break; case GGML_TYPE_IQ1_S: ftype = LLAMA_FTYPE_MOSTLY_IQ1_S; break; case GGML_TYPE_IQ4_NL: ftype = LLAMA_FTYPE_MOSTLY_IQ4_NL; break; + case GGML_TYPE_IQ4_XS: ftype = LLAMA_FTYPE_MOSTLY_IQ4_XS; break; case GGML_TYPE_IQ3_S: ftype = LLAMA_FTYPE_MOSTLY_IQ3_S; break; default: { @@ -2941,6 +2942,7 @@ static std::string llama_model_ftype_name(llama_ftype ftype) { case LLAMA_FTYPE_MOSTLY_IQ3_XXS:return "IQ3_XXS - 3.0625 bpw"; case LLAMA_FTYPE_MOSTLY_IQ1_S :return "IQ1_S - 1.5625 bpw"; case LLAMA_FTYPE_MOSTLY_IQ4_NL: return "IQ4_NL - 4.5 bpw"; + case LLAMA_FTYPE_MOSTLY_IQ4_XS: return "IQ4_XS - 4.25 bpw"; case LLAMA_FTYPE_MOSTLY_IQ3_S: return "IQ3_S - 3.4375 bpw"; case LLAMA_FTYPE_MOSTLY_IQ3_M: return "IQ3_S mix - 3.66 bpw"; @@ -10871,7 +10873,7 @@ static ggml_type get_k_quant_type(quantize_state_internal & qs, ggml_type new_ty new_type = qs.i_attention_wv < 2 ? GGML_TYPE_Q5_K : GGML_TYPE_Q4_K; } else if (ftype == LLAMA_FTYPE_MOSTLY_Q3_K_L) new_type = GGML_TYPE_Q5_K; - else if (ftype == LLAMA_FTYPE_MOSTLY_IQ4_NL && qs.model.hparams.n_gqa() >= 4) { + else if ((ftype == LLAMA_FTYPE_MOSTLY_IQ4_NL || ftype == LLAMA_FTYPE_MOSTLY_IQ4_XS) && qs.model.hparams.n_gqa() >= 4) { new_type = GGML_TYPE_Q5_K; } else if ((ftype == LLAMA_FTYPE_MOSTLY_Q4_K_M || ftype == LLAMA_FTYPE_MOSTLY_Q5_K_M) && @@ -10940,8 +10942,8 @@ static ggml_type get_k_quant_type(quantize_state_internal & qs, ggml_type new_ty if (use_more_bits(i_layer, n_layer)) new_type = GGML_TYPE_Q6_K; } } - else if (ftype == LLAMA_FTYPE_MOSTLY_IQ4_NL && !qs.has_imatrix) { - if (i_layer < n_layer/8) new_type = GGML_TYPE_Q5_K; + else if (i_layer < n_layer/8 && (ftype == LLAMA_FTYPE_MOSTLY_IQ4_NL || ftype == LLAMA_FTYPE_MOSTLY_IQ4_XS) && !qs.has_imatrix) { + new_type = GGML_TYPE_Q5_K; } else if (ftype == LLAMA_FTYPE_MOSTLY_Q5_K_M && use_more_bits(i_layer, n_layer)) new_type = GGML_TYPE_Q6_K; else if (ftype == LLAMA_FTYPE_MOSTLY_Q4_K_S && arch != LLM_ARCH_FALCON && i_layer < n_layer/8) { @@ -10961,7 +10963,7 @@ static ggml_type get_k_quant_type(quantize_state_internal & qs, ggml_type new_ty if (ftype == LLAMA_FTYPE_MOSTLY_Q2_K || ftype == LLAMA_FTYPE_MOSTLY_IQ3_XS || ftype == LLAMA_FTYPE_MOSTLY_IQ3_XXS || ftype == LLAMA_FTYPE_MOSTLY_Q3_K_S || ftype == LLAMA_FTYPE_MOSTLY_Q3_K_M || ftype == LLAMA_FTYPE_MOSTLY_IQ4_NL || ftype == LLAMA_FTYPE_MOSTLY_Q4_K_S || ftype == LLAMA_FTYPE_MOSTLY_Q4_K_M || ftype == LLAMA_FTYPE_MOSTLY_IQ3_S || - ftype == LLAMA_FTYPE_MOSTLY_IQ3_M) { + ftype == LLAMA_FTYPE_MOSTLY_IQ3_M || ftype == LLAMA_FTYPE_MOSTLY_IQ4_XS) { new_type = GGML_TYPE_Q5_K; } } else { @@ -11012,7 +11014,7 @@ static ggml_type get_k_quant_type(quantize_state_internal & qs, ggml_type new_ty //} bool convert_incompatible_tensor = false; if (new_type == GGML_TYPE_Q2_K || new_type == GGML_TYPE_Q3_K || new_type == GGML_TYPE_Q4_K || - new_type == GGML_TYPE_Q5_K || new_type == GGML_TYPE_Q6_K || + new_type == GGML_TYPE_Q5_K || new_type == GGML_TYPE_Q6_K || new_type == GGML_TYPE_IQ4_XS || new_type == GGML_TYPE_IQ2_XS || new_type == GGML_TYPE_IQ2_XXS || new_type == GGML_TYPE_IQ2_S || new_type == GGML_TYPE_IQ3_XXS || ftype == LLAMA_FTYPE_MOSTLY_IQ1_S || new_type == GGML_TYPE_IQ3_S) { int nx = tensor->ne[0]; @@ -11033,10 +11035,11 @@ static ggml_type get_k_quant_type(quantize_state_internal & qs, ggml_type new_ty case GGML_TYPE_IQ3_S: case GGML_TYPE_IQ1_S: case GGML_TYPE_Q2_K: - case GGML_TYPE_Q3_K: new_type = GGML_TYPE_IQ4_NL; break; - case GGML_TYPE_Q4_K: new_type = GGML_TYPE_Q5_0; break; - case GGML_TYPE_Q5_K: new_type = GGML_TYPE_Q5_1; break; - case GGML_TYPE_Q6_K: new_type = GGML_TYPE_Q8_0; break; + case GGML_TYPE_Q3_K: + case GGML_TYPE_IQ4_XS: new_type = GGML_TYPE_IQ4_NL; break; + case GGML_TYPE_Q4_K: new_type = GGML_TYPE_Q5_0; break; + case GGML_TYPE_Q5_K: new_type = GGML_TYPE_Q5_1; break; + case GGML_TYPE_Q6_K: new_type = GGML_TYPE_Q8_0; break; default: throw std::runtime_error("\nUnsupported tensor size encountered\n"); } LLAMA_LOG_WARN(" - using fallback quantization %s\n", ggml_type_name(new_type)); @@ -11078,6 +11081,7 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s case LLAMA_FTYPE_MOSTLY_IQ3_XXS: quantized_type = GGML_TYPE_IQ3_XXS; break; case LLAMA_FTYPE_MOSTLY_IQ1_S: quantized_type = GGML_TYPE_IQ1_S; break; case LLAMA_FTYPE_MOSTLY_IQ4_NL: quantized_type = GGML_TYPE_IQ4_NL; break; + case LLAMA_FTYPE_MOSTLY_IQ4_XS: quantized_type = GGML_TYPE_IQ4_XS; break; case LLAMA_FTYPE_MOSTLY_IQ3_S: quantized_type = GGML_TYPE_IQ3_S; break; case LLAMA_FTYPE_MOSTLY_IQ3_M: quantized_type = GGML_TYPE_IQ3_S; break; diff --git a/llama.h b/llama.h index 604161808..16e28e91d 100644 --- a/llama.h +++ b/llama.h @@ -115,6 +115,7 @@ extern "C" { LLAMA_FTYPE_MOSTLY_IQ3_M = 27, // except 1d tensors LLAMA_FTYPE_MOSTLY_IQ2_S = 28, // except 1d tensors LLAMA_FTYPE_MOSTLY_IQ2_M = 29, // except 1d tensors + LLAMA_FTYPE_MOSTLY_IQ4_XS = 30, // except 1d tensors LLAMA_FTYPE_GUESSED = 1024, // not specified in the model file }; diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index 60a852779..d4cea805f 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -1918,7 +1918,7 @@ static bool test_backend(ggml_backend_t backend, test_mode mode, const char * op GGML_TYPE_Q6_K, GGML_TYPE_IQ2_XXS, GGML_TYPE_IQ2_XS, GGML_TYPE_IQ2_S, GGML_TYPE_IQ3_XXS, GGML_TYPE_IQ1_S, - GGML_TYPE_IQ4_NL, GGML_TYPE_IQ3_S, + GGML_TYPE_IQ4_NL, GGML_TYPE_IQ3_S, GGML_TYPE_IQ4_XS, }; // unary ops From cb49e0f8c906e5da49e9f6d64a57742a9a241c6a Mon Sep 17 00:00:00 2001 From: Kawrakow <48489457+ikawrakow@users.noreply.github.com> Date: Tue, 27 Feb 2024 19:16:49 +0200 Subject: [PATCH 478/812] Attempt to fix android build (#5752) Co-authored-by: Iwan Kawrakow --- ggml-quants.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ggml-quants.c b/ggml-quants.c index 607d50925..f73d17ce2 100644 --- a/ggml-quants.c +++ b/ggml-quants.c @@ -10464,9 +10464,9 @@ void ggml_vec_dot_iq4_xs_q8_K(int n, float * restrict s, size_t bs, const void * #if defined __ARM_NEON const int8x16_t values = vld1q_s8(kvalues_iq4nl); const uint8x16_t m4b = vdupq_n_u8(0x0f); - uint8x16x2_t q4bits; - int8x16x4_t q4b; - int8x16x4_t q8b; + ggml_uint8x16x2_t q4bits; + ggml_int8x16x4_t q4b; + ggml_int8x16x4_t q8b; int32x4_t prod_1, prod_2; float sumf = 0; From 7c4263d4261d6ee6f0539d53eb9e1b4d120ba8af Mon Sep 17 00:00:00 2001 From: Kawrakow <48489457+ikawrakow@users.noreply.github.com> Date: Wed, 28 Feb 2024 10:37:02 +0200 Subject: [PATCH 479/812] ggml : make i-quants work with super-blocks of 64 (CPU,Metal) (#5760) * WIP: make i-quants work for QK_K = 64 * iq2_xs: attempt to fix AVX dot product for QK_K = 64 Tests pass, but I get gibberish. * QK_K = 64 tests pass on ARM_NEON and Metal Sadly, that does not mean it actually works. * Make CUDA compile with QK_K = 64 Tests don't pass, plus we get misaligned access * Q2_K: fixed bug in imatrix quantization for QK_K = 64 * iq1_s: turn off SIMD implementation for QK_K = 64 (it does not work) --------- Co-authored-by: Iwan Kawrakow --- ggml-cuda.cu | 27 ++++++--- ggml-metal.metal | 58 ++++++++++--------- ggml-quants.c | 148 +++++++++++++++++++++++++++++++++++++++-------- ggml-quants.h | 5 ++ ggml.c | 15 ++++- 5 files changed, 194 insertions(+), 59 deletions(-) diff --git a/ggml-cuda.cu b/ggml-cuda.cu index dfd28df62..831c84efb 100644 --- a/ggml-cuda.cu +++ b/ggml-cuda.cu @@ -544,14 +544,19 @@ static_assert(sizeof(block_iq3_xxs) == sizeof(ggml_fp16_t) + 3*(QK_K/8), "wrong #define QR3_XS 8 #define QI3_XS (QK_K / (4*QR3_XS)) +#if QK_K == 64 +#define IQ3S_N_SCALE 2 +#else +#define IQ3S_N_SCALE QK_K/64 +#endif typedef struct { half d; uint8_t qs[QK_K/4]; uint8_t qh[QK_K/32]; uint8_t signs[QK_K/8]; - uint8_t scales[QK_K/64]; + uint8_t scales[IQ3S_N_SCALE]; } block_iq3_s; -static_assert(sizeof(block_iq3_s) == sizeof(ggml_fp16_t) + 27*(QK_K/64), "wrong iq3_s block size/padding"); +static_assert(sizeof(block_iq3_s) == sizeof(ggml_fp16_t) + 13*(QK_K/32) + IQ3S_N_SCALE, "wrong iq3_s block size/padding"); #define QR1_S 8 #define QI1_S (QK_K / (4*QR1_S)) @@ -571,6 +576,11 @@ typedef struct { } block_iq4_nl; static_assert(sizeof(block_iq4_nl) == sizeof(ggml_fp16_t) + QK4_NL/2, "wrong iq4_nl block size/padding"); +#if QK_K == 64 +#define block_iq4_xs block_iq4_nl +#define QR4_XS QR4_NL +#define QI4_XS QI4_NL +#else // QR4_XS = 8 is very slightly faster than QR4_XS = 4 #define QR4_XS 8 #define QI4_XS (QK_K / (4*QR4_XS)) @@ -581,7 +591,7 @@ typedef struct { uint8_t qs[QK_K/2]; } block_iq4_xs; static_assert(sizeof(block_iq4_xs) == sizeof(ggml_fp16_t) + sizeof(uint16_t) + QK_K/64 + QK_K/2, "wrong iq4_xs block size/padding"); - +#endif #define WARP_SIZE 32 #define MATRIX_ROW_PADDING 512 // last row of quant. matrices is a multiple of this to avoid out-of-bounds memory accesses @@ -2439,9 +2449,9 @@ static __global__ void dequantize_block_iq4_nl(const void * __restrict__ vx, dst } +#if QK_K != 64 template static __global__ void dequantize_block_iq4_xs(const void * __restrict__ vx, dst_t * __restrict__ yy) { - const int i = blockIdx.x; const block_iq4_xs * x = (const block_iq4_xs *)vx; @@ -2455,8 +2465,8 @@ static __global__ void dequantize_block_iq4_xs(const void * __restrict__ vx, dst y[j+ 0] = d * kvalues_iq4nl[q4[j] & 0xf]; y[j+16] = d * kvalues_iq4nl[q4[j] >> 4]; } - } +#endif static __global__ void dequantize_mul_mat_vec_q2_k(const void * __restrict__ vx, const float * __restrict__ yy, float * __restrict__ dst, const int ncols, int nrows) { @@ -5382,8 +5392,7 @@ static __device__ __forceinline__ float vec_dot_iq4_xs_q8_1( return 0.f; #endif #else - assert(false); - return 0.f; + return vec_dot_iq4_xs_q8_1(vbq, bq8_1, iqs); #endif } @@ -7444,7 +7453,11 @@ static void dequantize_row_iq4_nl_cuda(const void * vx, dst_t * y, const int k, template static void dequantize_row_iq4_xs_cuda(const void * vx, dst_t * y, const int k, cudaStream_t stream) { const int nb = (k + QK_K - 1) / QK_K; +#if QK_K == 64 + dequantize_block_iq4_nl<<>>(vx, y); +#else dequantize_block_iq4_xs<<>>(vx, y); +#endif } template diff --git a/ggml-metal.metal b/ggml-metal.metal index 689411903..74a5e0b03 100644 --- a/ggml-metal.metal +++ b/ggml-metal.metal @@ -2560,12 +2560,16 @@ typedef struct { uint8_t qs[QK4_NL/2]; } block_iq4_nl; +#if QK_K == 64 +#define block_iq4_xs block_iq4_nl +#else typedef struct { half d; uint16_t scales_h; uint8_t scales_l[QK_K/64]; uint8_t qs[QK_K/2]; } block_iq4_xs; +#endif //====================================== dot products ========================= @@ -4346,7 +4350,6 @@ void kernel_mul_mv_iq2_xxs_f32_impl( threadgroup_barrier(mem_flags::mem_threadgroup); } -#if QK_K == 256 const int ix = tiisg; device const float * y4 = y + 32 * ix; @@ -4387,12 +4390,6 @@ void kernel_mul_mv_iq2_xxs_f32_impl( y4 += 32 * 32; } -#else - (void) x; - (void) y; - (void) yl; - (void) nb32; -#endif for (int row = 0; row < N_DST; ++row) { all_sum = simd_sum(sumf[row]); @@ -4482,7 +4479,6 @@ void kernel_mul_mv_iq2_xs_f32_impl( threadgroup_barrier(mem_flags::mem_threadgroup); } -#if QK_K == 256 const int ix = tiisg; device const float * y4 = y + 32 * ix; @@ -4533,12 +4529,6 @@ void kernel_mul_mv_iq2_xs_f32_impl( y4 += 32 * 32; } -#else - (void) x; - (void) y; - (void) yl; - (void) nb32; -#endif for (int row = 0; row < N_DST; ++row) { all_sum = simd_sum(sumf[row]); @@ -4628,7 +4618,6 @@ void kernel_mul_mv_iq3_xxs_f32_impl( threadgroup_barrier(mem_flags::mem_threadgroup); } -#if QK_K == 256 const int ix = tiisg; device const float * y4 = y + 32 * ix; @@ -4672,12 +4661,6 @@ void kernel_mul_mv_iq3_xxs_f32_impl( y4 += 32 * 32; } -#else - (void) x; - (void) y; - (void) yl; - (void) nb32; -#endif for (int row = 0; row < N_DST; ++row) { all_sum = simd_sum(sumf[row]); @@ -5016,7 +4999,6 @@ void kernel_mul_mv_iq1_s_f32_impl( const int nb32 = nb * (QK_K / 32); -#if QK_K == 256 const int ix = tiisg/2; const int il = tiisg%2; @@ -5055,12 +5037,6 @@ void kernel_mul_mv_iq1_s_f32_impl( y4 += 16 * 32; } -#else - (void) x; - (void) y; - (void) yl; - (void) nb32; -#endif for (int row = 0; row < N_DST; ++row) { all_sum = simd_sum(sumf[row]); @@ -5167,6 +5143,7 @@ void kernel_mul_mv_iq4_nl_f32_impl( } } +#if QK_K != 64 void kernel_mul_mv_iq4_xs_f32_impl( device const void * src0, device const float * src1, @@ -5260,6 +5237,7 @@ void kernel_mul_mv_iq4_xs_f32_impl( } } } +#endif [[host_name("kernel_mul_mv_iq1_s_f32")]] kernel void kernel_mul_mv_iq1_s_f32( @@ -5344,7 +5322,11 @@ kernel void kernel_mul_mv_iq4_xs_f32( uint tiisg[[thread_index_in_simdgroup]], uint sgitg[[simdgroup_index_in_threadgroup]]) { +#if QK_K == 64 + kernel_mul_mv_iq4_nl_f32_impl(src0, src1, dst, ne00, ne01, ne02, ne10, ne12, ne0, ne1, r2, r3, shared_values, tgpig, tiisg, sgitg); +#else kernel_mul_mv_iq4_xs_f32_impl(src0, src1, dst, ne00, ne01, ne02, ne10, ne12, ne0, ne1, r2, r3, shared_values, tgpig, tiisg, sgitg); +#endif } //============================= templates and their specializations ============================= @@ -5770,6 +5752,9 @@ void dequantize_iq4_nl(device const block_iq4_nl * xb, short il, thread type4x4 template void dequantize_iq4_xs(device const block_iq4_xs * xb, short il, thread type4x4 & reg) { +#if QK_K == 64 + dequantize_iq4_nl(xb, il, reg); +#else // il is 0...15 for QK_K = 256 => index of block of 32 is il/2 const int ib32 = il/2; il = il%2; @@ -5786,6 +5771,7 @@ void dequantize_iq4_xs(device const block_iq4_xs * xb, short il, thread type4x4 reg[i][2] = d * kvalues_iq4nl_f[q8[2]]; reg[i][3] = d * kvalues_iq4nl_f[q8[3]]; } +#endif } template @@ -6334,7 +6320,11 @@ template [[host_name("kernel_get_rows_iq3_s")]] kernel get_rows_t kernel_get_r template [[host_name("kernel_get_rows_iq2_s")]] kernel get_rows_t kernel_get_rows; template [[host_name("kernel_get_rows_iq1_s")]] kernel get_rows_t kernel_get_rows; template [[host_name("kernel_get_rows_iq4_nl")]] kernel get_rows_t kernel_get_rows; +#if QK_K == 64 +template [[host_name("kernel_get_rows_iq4_xs")]] kernel get_rows_t kernel_get_rows; +#else template [[host_name("kernel_get_rows_iq4_xs")]] kernel get_rows_t kernel_get_rows; +#endif // // matrix-matrix multiplication @@ -6378,7 +6368,11 @@ template [[host_name("kernel_mul_mm_iq3_s_f32")]] kernel mat_mm_t kernel_mul_m template [[host_name("kernel_mul_mm_iq2_s_f32")]] kernel mat_mm_t kernel_mul_mm; template [[host_name("kernel_mul_mm_iq1_s_f32")]] kernel mat_mm_t kernel_mul_mm; template [[host_name("kernel_mul_mm_iq4_nl_f32")]] kernel mat_mm_t kernel_mul_mm; +#if QK_K == 64 +template [[host_name("kernel_mul_mm_iq4_xs_f32")]] kernel mat_mm_t kernel_mul_mm; +#else template [[host_name("kernel_mul_mm_iq4_xs_f32")]] kernel mat_mm_t kernel_mul_mm; +#endif // // indirect matrix-matrix multiplication @@ -6434,7 +6428,11 @@ template [[host_name("kernel_mul_mm_id_iq3_s_f32")]] kernel mat_mm_id_t kernel template [[host_name("kernel_mul_mm_id_iq2_s_f32")]] kernel mat_mm_id_t kernel_mul_mm_id; template [[host_name("kernel_mul_mm_id_iq1_s_f32")]] kernel mat_mm_id_t kernel_mul_mm_id; template [[host_name("kernel_mul_mm_id_iq4_nl_f32")]] kernel mat_mm_id_t kernel_mul_mm_id; +#if QK_K == 64 +template [[host_name("kernel_mul_mm_id_iq4_xs_f32")]] kernel mat_mm_id_t kernel_mul_mm_id; +#else template [[host_name("kernel_mul_mm_id_iq4_xs_f32")]] kernel mat_mm_id_t kernel_mul_mm_id; +#endif // // matrix-vector multiplication @@ -7707,7 +7705,11 @@ kernel void kernel_mul_mv_id_iq4_xs_f32( const int32_t id = ((device int32_t *) (ids + bid*nbi1))[idx]; +#if QK_K == 64 + kernel_mul_mv_iq4_nl_f32_impl( +#else kernel_mul_mv_iq4_xs_f32_impl( +#endif src0[id], (device const float *) (src1 + bid*nb11), dst + bid*ne0, diff --git a/ggml-quants.c b/ggml-quants.c index f73d17ce2..371826f14 100644 --- a/ggml-quants.c +++ b/ggml-quants.c @@ -1877,7 +1877,7 @@ static void quantize_row_q2_K_impl(const float * restrict x, block_q2_K * restri float mins[QK_K/16]; float scales[QK_K/16]; float sw[QK_K/16]; - float weight[QK_K/16]; + float weight[16]; uint8_t Ls[QK_K/16], Lm[QK_K/16]; for (int i = 0; i < nb; i++) { @@ -1887,13 +1887,42 @@ static void quantize_row_q2_K_impl(const float * restrict x, block_q2_K * restri float sigma2 = sumx2/QK_K; for (int j = 0; j < QK_K/16; ++j) { const float * restrict qw = quant_weights + QK_K * i + 16*j; - for (int l = 0; l < QK_K/16; ++l) weight[l] = qw[l] * sqrtf(sigma2 + x[16*j + l]*x[16*j + l]); + for (int l = 0; l < 16; ++l) weight[l] = qw[l] * sqrtf(sigma2 + x[16*j + l]*x[16*j + l]); for (int l = 0; l < QK_K/16; ++l) sw[j] += weight[l]; - scales[j] = make_qkx3_quants(QK_K/16, 3, x + 16*j, weight, L + 16*j, &mins[j], Laux, -0.9f, 0.05f, 36, false); + scales[j] = make_qkx3_quants(16, 3, x + 16*j, weight, L + 16*j, &mins[j], Laux, -0.9f, 0.05f, 36, false); } - float dm = make_qp_quants(QK_K/16, 15, scales, Ls, sw); - float mm = make_qp_quants(QK_K/16, 15, mins, Lm, sw); + float dm, mm; +#if QK_K == 64 + float max_scale = 0, max_min = 0; + for (int j = 0; j < QK_K/16; ++j) { + max_scale = MAX(max_scale, scales[j]); + max_min = MAX(max_min, mins[j]); + } + dm = max_scale/15; + mm = max_min/15; + if (max_scale) { + float id = 1/dm; + for (int j = 0; j < QK_K/16; ++j) { + int l = nearest_int(id*scales[j]); + Ls[j] = MAX(0, MIN(15, l)); + } + } else { + memset(Ls, 0, QK_K/16); + } + if (max_min) { + float id = 1/mm; + for (int j = 0; j < QK_K/16; ++j) { + int l = nearest_int(id*mins[j]); + Lm[j] = MAX(0, MIN(15, l)); + } + } else { + memset(Lm, 0, QK_K/16); + } +#else + dm = make_qp_quants(QK_K/16, 15, scales, Ls, sw); + mm = make_qp_quants(QK_K/16, 15, mins, Lm, sw); +#endif y[i].d = GGML_FP32_TO_FP16(dm); y[i].dmin = GGML_FP32_TO_FP16(mm); dm = GGML_FP16_TO_FP32(y[i].d); @@ -4227,6 +4256,9 @@ void dequantize_row_iq4_nl(const block_iq4_nl * restrict x, float * restrict y, void dequantize_row_iq4_xs(const block_iq4_xs * restrict x, float * restrict y, int k) { assert(k % QK_K == 0); +#if QK_K == 64 + dequantize_row_iq4_nl((const block_iq4_nl *)x, y, k); +#else const int nb = k / QK_K; for (int i = 0; i < nb; i++) { @@ -4246,6 +4278,7 @@ void dequantize_row_iq4_xs(const block_iq4_xs * restrict x, float * restrict y, qs += 16; } } +#endif } //===================================== Q8_K ============================================== @@ -6306,7 +6339,7 @@ void ggml_vec_dot_q2_K_q8_K(int n, float * restrict s, size_t bs, const void * r float sumf = 0; - int isum[4]; + int isum[QK_K/16]; for (int i = 0; i < nb; ++i) { @@ -6322,14 +6355,14 @@ void ggml_vec_dot_q2_K_q8_K(int n, float * restrict s, size_t bs, const void * r const float dall = y[i].d * GGML_FP16_TO_FP32(x[i].d); const float dmin = y[i].d * GGML_FP16_TO_FP32(x[i].dmin); - isum[0] = isum[1] = isum[2] = isum[3] = 0; + memset(isum, 0, (QK_K/16)*sizeof(int)); for (int l = 0; l < 16; ++l) { isum[0] += q8[l+ 0] * ((q2[l] >> 0) & 3); isum[1] += q8[l+16] * ((q2[l] >> 2) & 3); isum[2] += q8[l+32] * ((q2[l] >> 4) & 3); isum[3] += q8[l+48] * ((q2[l] >> 6) & 3); } - for (int l = 0; l < 4; ++l) { + for (int l = 0; l < QK_K/16; ++l) { isum[l] *= (sc[l] & 0xF); } sumf += dall * (isum[0] + isum[1] + isum[2] + isum[3]) - dmin * summs; @@ -9488,15 +9521,7 @@ void ggml_vec_dot_iq2_xs_q8_K(int n, float * restrict s, size_t bs, const void * #elif defined(__AVX2__) - const __m128i m4 = _mm_set1_epi8(0xf); - const __m128i m1 = _mm_set1_epi8(1); - const __m256i m511 = _mm256_set1_epi16(511); const __m256i mone = _mm256_set1_epi8(1); - - static const uint8_t k_bit_helper[32] = { - 0x00, 0x80, 0x80, 0x00, 0x80, 0x00, 0x00, 0x80, 0x80, 0x00, 0x00, 0x80, 0x00, 0x80, 0x80, 0x00, - 0x00, 0x80, 0x80, 0x00, 0x80, 0x00, 0x00, 0x80, 0x80, 0x00, 0x00, 0x80, 0x00, 0x80, 0x80, 0x00, - }; static const char block_sign_shuffle_mask_1[32] = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, @@ -9510,11 +9535,77 @@ void ggml_vec_dot_iq2_xs_q8_K(int n, float * restrict s, size_t bs, const void * 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, }; - const __m256i bit_helper = _mm256_loadu_si256((const __m256i*)k_bit_helper); const __m256i bit_selector_mask = _mm256_loadu_si256((const __m256i*)bit_selector_mask_bytes); const __m256i block_sign_shuffle_1 = _mm256_loadu_si256((const __m256i*)block_sign_shuffle_mask_1); const __m256i block_sign_shuffle_2 = _mm256_loadu_si256((const __m256i*)block_sign_shuffle_mask_2); +#if QK_K == 64 + static const uint8_t k_bit_helper[16] = { + 0x00, 0x80, 0x80, 0x00, 0x80, 0x00, 0x00, 0x80, 0x80, 0x00, 0x00, 0x80, 0x00, 0x80, 0x80, 0x00, + }; + const __m128i bit_helper = _mm_loadu_si128((const __m128i*)k_bit_helper); + const __m128i m511 = _mm_set1_epi16(511); + typedef union { + __m128i vec_index; + uint16_t index[8]; + } index_t; + + index_t idx; + __m256 accumf = _mm256_setzero_ps(); + for (int i = 0; i < nb; ++i) { + const float d = GGML_FP16_TO_FP32(x[i].d) * y[i].d; + const __m128i q2_data = _mm_loadu_si128((const __m128i*)x[i].qs); + idx.vec_index = _mm_and_si128(q2_data, m511); + + const __m128i partial_sign_bits = _mm_srli_epi16(q2_data, 9); + const __m128i partial_sign_bits_upper = _mm_srli_epi16(q2_data, 13); + const __m128i partial_sign_bits_for_counting = _mm_xor_si128(partial_sign_bits, partial_sign_bits_upper); + + const __m128i odd_bits = _mm_shuffle_epi8(bit_helper, partial_sign_bits_for_counting); + const __m128i full_sign_bits = _mm_or_si128(partial_sign_bits, odd_bits); + const __m256i full_signs = _mm256_set_m128i(full_sign_bits, full_sign_bits); + + const __m256i q8_1 = _mm256_loadu_si256((const __m256i *)y[i].qs); + const __m256i q8_2 = _mm256_loadu_si256((const __m256i *)(y[i].qs+32)); + + const __m256i q2_1 = _mm256_set_epi64x(iq2xs_grid[idx.index[3]], iq2xs_grid[idx.index[2]], + iq2xs_grid[idx.index[1]], iq2xs_grid[idx.index[0]]); + const __m256i q2_2 = _mm256_set_epi64x(iq2xs_grid[idx.index[7]], iq2xs_grid[idx.index[6]], + iq2xs_grid[idx.index[5]], iq2xs_grid[idx.index[4]]); + + __m256i signs; + signs = _mm256_shuffle_epi8(full_signs, block_sign_shuffle_1); + signs = _mm256_cmpeq_epi8(_mm256_and_si256(signs, bit_selector_mask), bit_selector_mask); + const __m256i q8s_1 = _mm256_sign_epi8(q8_1, _mm256_or_si256(signs, mone)); + + signs = _mm256_shuffle_epi8(full_signs, block_sign_shuffle_2); + signs = _mm256_cmpeq_epi8(_mm256_and_si256(signs, bit_selector_mask), bit_selector_mask); + const __m256i q8s_2 = _mm256_sign_epi8(q8_2, _mm256_or_si256(signs, mone)); + + const __m256i dot1 = _mm256_maddubs_epi16(q2_1, q8s_1); + const __m256i dot2 = _mm256_maddubs_epi16(q2_2, q8s_2); + + const __m256i sc1 = _mm256_set_m128i(_mm_set1_epi16(2*(x[i].scales[0] >> 4)+1), _mm_set1_epi16(2*(x[i].scales[0] & 0xf)+1)); + const __m256i sc2 = _mm256_set_m128i(_mm_set1_epi16(2*(x[i].scales[1] >> 4)+1), _mm_set1_epi16(2*(x[i].scales[1] & 0xf)+1)); + + const __m256i sum = _mm256_add_epi32(_mm256_madd_epi16(sc1, dot1), _mm256_madd_epi16(sc2, dot2)); + + accumf = _mm256_fmadd_ps(_mm256_set1_ps(d), _mm256_cvtepi32_ps(sum), accumf); + + } + + *s = 0.125f * hsum_float_8(accumf); +#else + + static const uint8_t k_bit_helper[32] = { + 0x00, 0x80, 0x80, 0x00, 0x80, 0x00, 0x00, 0x80, 0x80, 0x00, 0x00, 0x80, 0x00, 0x80, 0x80, 0x00, + 0x00, 0x80, 0x80, 0x00, 0x80, 0x00, 0x00, 0x80, 0x80, 0x00, 0x00, 0x80, 0x00, 0x80, 0x80, 0x00, + }; + const __m256i bit_helper = _mm256_loadu_si256((const __m256i*)k_bit_helper); + const __m256i m511 = _mm256_set1_epi16(511); + const __m128i m4 = _mm_set1_epi8(0xf); + const __m128i m1 = _mm_set1_epi8(1); + uint64_t aux64; // somewhat hacky, but gives a significant boost in performance @@ -9603,6 +9694,7 @@ void ggml_vec_dot_iq2_xs_q8_K(int n, float * restrict s, size_t bs, const void * } *s = 0.125f * hsum_float_8(accumf); +#endif #else @@ -10199,7 +10291,8 @@ void ggml_vec_dot_iq1_s_q8_K (int n, float * GGML_RESTRICT s, size_t bs, const const int nb = n / QK_K; -#if defined __ARM_NEON + // TODO: implement for QK_K = 64 +#if defined __ARM_NEON && QK_K == 256 const uint8x16_t m8 = vdupq_n_u8(0x08); const uint8x16_t m7 = vdupq_n_u8(0x07); @@ -10256,7 +10349,8 @@ void ggml_vec_dot_iq1_s_q8_K (int n, float * GGML_RESTRICT s, size_t bs, const *s = sumf; -#elif defined __AVX2__ + // TODO: implement for QK_K = 64 +#elif defined __AVX2__ && QK_K == 256 const __m128i m8 = _mm_set1_epi8(0x08); const __m128i m7 = _mm_set1_epi8(0x07); @@ -10455,6 +10549,9 @@ void ggml_vec_dot_iq4_xs_q8_K(int n, float * restrict s, size_t bs, const void * UNUSED(by); UNUSED(bs); assert(n % QK_K == 0); +#if QK_K == 64 + ggml_vec_dot_iq4_nl_q8_0(n, s, bs, vx, bx, vy, by, nrc); +#else const block_iq4_xs * restrict x = vx; const block_q8_K * restrict y = vy; @@ -10574,6 +10671,7 @@ void ggml_vec_dot_iq4_xs_q8_K(int n, float * restrict s, size_t bs, const void * } *s = sumf; #endif +#endif } // ================================ IQ2 quantization ============================================= @@ -10921,7 +11019,7 @@ static void quantize_row_iq2_xxs_impl(const float * restrict x, void * restrict const int kMaxQ = 3; - const int nbl = n/256; + const int nbl = n/QK_K; block_iq2_xxs * y = vy; @@ -11094,7 +11192,7 @@ static void quantize_row_iq2_xs_impl(const float * restrict x, void * restrict v const int kMaxQ = 3; - const int nbl = n/256; + const int nbl = n/QK_K; block_iq2_xs * y = vy; @@ -12037,7 +12135,7 @@ static void quantize_row_iq1_s_impl(const float * restrict x, void * restrict vy GGML_ASSERT(kneighbors_q2xs && "forgot to call ggml_quantize_init()?"); GGML_ASSERT(n%QK_K == 0); - const int nbl = n/256; + const int nbl = n/QK_K; block_iq1_s * y = vy; @@ -12315,6 +12413,9 @@ void quantize_row_iq4_nl_reference(const float * restrict x, block_iq4_nl * rest } size_t quantize_iq4_xs(const float * src, void * dst, int nrow, int n_per_row, int64_t * hist, const float * quant_weights) { +#if QK_K == 64 + return quantize_iq4_nl(src, dst, nrow, n_per_row, hist, quant_weights); +#else (void)hist; GGML_ASSERT(n_per_row%QK_K == 0); int nblock = n_per_row/QK_K; @@ -12333,6 +12434,7 @@ size_t quantize_iq4_xs(const float * src, void * dst, int nrow, int n_per_row, i qrow += nblock*sizeof(block_iq4_xs); } return nrow * nblock * sizeof(block_iq4_xs); +#endif } void quantize_row_iq4_xs(const float * restrict x, void * restrict vy, int k) { @@ -12363,7 +12465,7 @@ static void quantize_row_iq2_s_impl(const float * restrict x, void * restrict vy const int kMaxQ = 3; - const int nbl = n/256; + const int nbl = n/QK_K; block_iq2_s * y = vy; diff --git a/ggml-quants.h b/ggml-quants.h index 2c61134c4..316e35687 100644 --- a/ggml-quants.h +++ b/ggml-quants.h @@ -230,6 +230,10 @@ typedef struct { } block_iq4_nl; static_assert(sizeof(block_iq4_nl) == sizeof(ggml_fp16_t) + QK4_NL/2, "wrong iq4_nl block size/padding"); +#if QK_K == 64 +#define block_iq4_xs block_iq4_nl +//typedef struct block_iq4_nl block_iq4_xs; +#else typedef struct { ggml_fp16_t d; uint16_t scales_h; @@ -237,6 +241,7 @@ typedef struct { uint8_t qs[QK_K/2]; } block_iq4_xs; static_assert(sizeof(block_iq4_xs) == sizeof(ggml_fp16_t) + sizeof(uint16_t) + QK_K/64 + QK_K/2, "wrong iq4_xs block size/padding"); +#endif #ifdef __cplusplus extern "C" { diff --git a/ggml.c b/ggml.c index d66db3352..4591644ad 100644 --- a/ggml.c +++ b/ggml.c @@ -728,14 +728,22 @@ static const ggml_type_traits_t type_traits[GGML_TYPE_COUNT] = { }, [GGML_TYPE_IQ4_XS] = { .type_name = "iq4_xs", +#if QK_K == 64 + .blck_size = QK4_NL, +#else .blck_size = QK_K, +#endif .type_size = sizeof(block_iq4_xs), .is_quantized = true, .to_float = (ggml_to_float_t) dequantize_row_iq4_xs, .from_float = quantize_row_iq4_xs, .from_float_reference = (ggml_from_float_t)quantize_row_iq4_xs_reference, .vec_dot = ggml_vec_dot_iq4_xs_q8_K, +#if QK_K == 64 + .vec_dot_type = GGML_TYPE_Q8_0, +#else .vec_dot_type = GGML_TYPE_Q8_K, +#endif .nrows = 1, }, [GGML_TYPE_Q8_K] = { @@ -19830,6 +19838,9 @@ size_t ggml_quantize_chunk(enum ggml_type type, const float * src, void * dst, i GGML_ASSERT(result == row_size * nrows); } break; case GGML_TYPE_IQ4_NL: +#if QK_K == 64 + case GGML_TYPE_IQ4_XS: +#endif { GGML_ASSERT(start % QK4_NL == 0); GGML_ASSERT(start % n_per_row == 0); @@ -19838,15 +19849,17 @@ size_t ggml_quantize_chunk(enum ggml_type type, const float * src, void * dst, i result = quantize_iq4_nl(src + start, (char *)dst + start_row * row_size, nrows, n_per_row, hist, imatrix); GGML_ASSERT(result == row_size * nrows); } break; +#if QK_K != 64 case GGML_TYPE_IQ4_XS: { - GGML_ASSERT(start % QK4_NL == 0); + GGML_ASSERT(start % QK_K == 0); GGML_ASSERT(start % n_per_row == 0); size_t start_row = start / n_per_row; size_t row_size = ggml_row_size(type, n_per_row); result = quantize_iq4_xs(src + start, (char *)dst + start_row * row_size, nrows, n_per_row, hist, imatrix); GGML_ASSERT(result == row_size * nrows); } break; +#endif case GGML_TYPE_F16: { size_t elemsize = sizeof(ggml_fp16_t); From efc72253f7987ed7bdc8bde9d9fa5c7cac2f6292 Mon Sep 17 00:00:00 2001 From: Jorge A <161275481+jorgealias@users.noreply.github.com> Date: Wed, 28 Feb 2024 01:39:15 -0700 Subject: [PATCH 480/812] server : add "/chat/completions" alias for "/v1/...` (#5722) * Add "/chat/completions" as alias for "/v1/chat/completions" * merge to upstream master * minor : fix trailing whitespace --------- Co-authored-by: Georgi Gerganov --- examples/server/server.cpp | 133 +++++++++--------- .../server/tests/features/parallel.feature | 22 +++ examples/server/tests/features/steps/steps.py | 28 +++- 3 files changed, 115 insertions(+), 68 deletions(-) diff --git a/examples/server/server.cpp b/examples/server/server.cpp index 846ef7e5f..6b3ee531c 100644 --- a/examples/server/server.cpp +++ b/examples/server/server.cpp @@ -3211,87 +3211,88 @@ int main(int argc, char **argv) res.set_content(models.dump(), "application/json; charset=utf-8"); }); + const auto chat_completions = [&llama, &validate_api_key, &sparams](const httplib::Request &req, httplib::Response &res) + { + res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin")); + if (!validate_api_key(req, res)) { + return; + } + json data = oaicompat_completion_params_parse(llama.model, json::parse(req.body), sparams.chat_template); - // TODO: add mount point without "/v1" prefix -- how? - svr.Post("/v1/chat/completions", [&llama, &validate_api_key, &sparams](const httplib::Request &req, httplib::Response &res) - { - res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin")); - if (!validate_api_key(req, res)) { - return; - } - json data = oaicompat_completion_params_parse(llama.model, json::parse(req.body), sparams.chat_template); + const int task_id = llama.queue_tasks.get_new_id(); + llama.queue_results.add_waiting_task_id(task_id); + llama.request_completion(task_id, data, false, false, -1); - const int task_id = llama.queue_tasks.get_new_id(); - llama.queue_results.add_waiting_task_id(task_id); - llama.request_completion(task_id, data, false, false, -1); + if (!json_value(data, "stream", false)) { + std::string completion_text; + task_result result = llama.queue_results.recv(task_id); - if (!json_value(data, "stream", false)) { - std::string completion_text; - task_result result = llama.queue_results.recv(task_id); + if (!result.error && result.stop) { + json oaicompat_result = format_final_response_oaicompat(data, result); - if (!result.error && result.stop) { - json oaicompat_result = format_final_response_oaicompat(data, result); + res.set_content(oaicompat_result.dump(-1, ' ', false, + json::error_handler_t::replace), + "application/json; charset=utf-8"); + } else { + res.status = 500; + res.set_content(result.result_json["content"], "text/plain; charset=utf-8"); + } + llama.queue_results.remove_waiting_task_id(task_id); + } else { + const auto chunked_content_provider = [task_id, &llama](size_t, httplib::DataSink &sink) { + while (true) { + task_result llama_result = llama.queue_results.recv(task_id); + if (!llama_result.error) { + std::vector result_array = format_partial_response_oaicompat( llama_result); - res.set_content(oaicompat_result.dump(-1, ' ', false, - json::error_handler_t::replace), - "application/json; charset=utf-8"); - } else { - res.status = 500; - res.set_content(result.result_json["content"], "text/plain; charset=utf-8"); - } - llama.queue_results.remove_waiting_task_id(task_id); - } else { - const auto chunked_content_provider = [task_id, &llama](size_t, httplib::DataSink &sink) { - while (true) { - task_result llama_result = llama.queue_results.recv(task_id); - if (!llama_result.error) { - std::vector result_array = format_partial_response_oaicompat( llama_result); - - for (auto it = result_array.begin(); it != result_array.end(); ++it) - { - if (!it->empty()) { - const std::string str = - "data: " + - it->dump(-1, ' ', false, json::error_handler_t::replace) + - "\n\n"; - LOG_VERBOSE("data stream", {{"to_send", str}}); - if (!sink.write(str.c_str(), str.size())) { - llama.queue_results.remove_waiting_task_id(task_id); - return false; - } - } - } - if (llama_result.stop) { - break; - } - } else { + for (auto it = result_array.begin(); it != result_array.end(); ++it) + { + if (!it->empty()) { const std::string str = - "error: " + - llama_result.result_json.dump(-1, ' ', false, - json::error_handler_t::replace) + + "data: " + + it->dump(-1, ' ', false, json::error_handler_t::replace) + "\n\n"; LOG_VERBOSE("data stream", {{"to_send", str}}); if (!sink.write(str.c_str(), str.size())) { llama.queue_results.remove_waiting_task_id(task_id); return false; } - break; } } - sink.done(); - llama.queue_results.remove_waiting_task_id(task_id); - return true; - }; - - auto on_complete = [task_id, &llama](bool) { - // cancel request - llama.request_cancel(task_id); - llama.queue_results.remove_waiting_task_id(task_id); - }; - - res.set_chunked_content_provider("text/event-stream", chunked_content_provider, on_complete); + if (llama_result.stop) { + break; + } + } else { + const std::string str = + "error: " + + llama_result.result_json.dump(-1, ' ', false, + json::error_handler_t::replace) + + "\n\n"; + LOG_VERBOSE("data stream", {{"to_send", str}}); + if (!sink.write(str.c_str(), str.size())) { + llama.queue_results.remove_waiting_task_id(task_id); + return false; + } + break; + } } - }); + sink.done(); + llama.queue_results.remove_waiting_task_id(task_id); + return true; + }; + + auto on_complete = [task_id, &llama](bool) { + // cancel request + llama.request_cancel(task_id); + llama.queue_results.remove_waiting_task_id(task_id); + }; + + res.set_chunked_content_provider("text/event-stream", chunked_content_provider, on_complete); + } + }; + + svr.Post("/chat/completions", chat_completions); + svr.Post("/v1/chat/completions", chat_completions); svr.Post("/infill", [&llama, &validate_api_key](const httplib::Request &req, httplib::Response &res) { diff --git a/examples/server/tests/features/parallel.feature b/examples/server/tests/features/parallel.feature index c85f9de1d..5f895cf90 100644 --- a/examples/server/tests/features/parallel.feature +++ b/examples/server/tests/features/parallel.feature @@ -54,6 +54,28 @@ Feature: Parallel | disabled | 128 | | enabled | 64 | + Scenario Outline: Multi users OAI completions compatibility no v1 + Given a system prompt You are a writer. + And a model tinyllama-2 + Given a prompt: + """ + Write a very long book. + """ + And a prompt: + """ + Write another a poem. + """ + And max tokens to predict + And streaming is + Given concurrent OAI completions requests no v1 + Then the server is busy + Then the server is idle + Then all prompts are predicted with tokens + Examples: + | streaming | n_predict | + | disabled | 128 | + | enabled | 64 | + Scenario: Multi users with total number of tokens to predict exceeds the KV Cache size #3969 Given a prompt: """ diff --git a/examples/server/tests/features/steps/steps.py b/examples/server/tests/features/steps/steps.py index ad87fcb82..381da105e 100644 --- a/examples/server/tests/features/steps/steps.py +++ b/examples/server/tests/features/steps/steps.py @@ -231,6 +231,7 @@ async def step_oai_chat_completions(context, api_error): completion = await oai_chat_completions(context.prompts.pop(), context.system_prompt, context.base_url, + '/v1/chat', False, model=context.model if hasattr(context, 'model') else None, @@ -288,6 +289,28 @@ async def step_oai_chat_completions(context): # user_prompt is inserted automatically context.system_prompt, context.base_url, + '/v1/chat/completions', + True, # async_client + model=context.model + if hasattr(context, 'model') else None, + n_predict=context.n_predict + if hasattr(context, 'n_predict') else None, + enable_streaming=context.enable_streaming + if hasattr(context, 'enable_streaming') else None, + server_seed=context.server_seed + if hasattr(context, 'server_seed') else None, + user_api_key=context.user_api_key + if hasattr(context, 'user_api_key') else None) + + +@step(u'concurrent OAI completions requests no v1') +@async_run_until_complete +async def step_oai_chat_completions(context): + await concurrent_requests(context, oai_chat_completions, + # user_prompt is inserted automatically + context.system_prompt, + context.base_url, + '/chat/completions', True, # async_client model=context.model if hasattr(context, 'model') else None, @@ -497,6 +520,7 @@ async def request_completion(prompt, async def oai_chat_completions(user_prompt, system_prompt, base_url, + base_path, async_client, debug=False, model=None, @@ -537,7 +561,7 @@ async def oai_chat_completions(user_prompt, origin = 'llama.cpp' headers = {'Authorization': f'Bearer {user_api_key}', 'Origin': origin} async with aiohttp.ClientSession() as session: - async with session.post(f'{base_url}/v1/chat/completions', + async with session.post(f'{base_url}{base_path}', json=payload, headers=headers) as response: if enable_streaming: @@ -579,7 +603,7 @@ async def oai_chat_completions(user_prompt, else: try: openai.api_key = user_api_key - openai.api_base = f'{base_url}/v1/chat' + openai.api_base = f'{base_url}{base_path}' chat_completion = openai.Completion.create( messages=payload['messages'], model=model, From 6c4416868df2e5455da7d20547f62bcf9735ba8e Mon Sep 17 00:00:00 2001 From: Daniel Bevenius Date: Wed, 28 Feb 2024 09:39:39 +0100 Subject: [PATCH 481/812] readme : add link to LLaVA 1.6 models (#5758) Signed-off-by: Daniel Bevenius --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 507a2888b..5401e197f 100644 --- a/README.md +++ b/README.md @@ -107,7 +107,7 @@ Typically finetunes of the base models below are supported as well. **Multimodal models:** -- [x] [LLaVA 1.5 models](https://huggingface.co/collections/liuhaotian/llava-15-653aac15d994e992e2677a7e) +- [x] [LLaVA 1.5 models](https://huggingface.co/collections/liuhaotian/llava-15-653aac15d994e992e2677a7e), [LLaVA 1.6 models](https://huggingface.co/collections/liuhaotian/llava-16-65b9e40155f60fd046a5ccf2) - [x] [BakLLaVA](https://huggingface.co/models?search=SkunkworksAI/Bakllava) - [x] [Obsidian](https://huggingface.co/NousResearch/Obsidian-3B-V0.5) - [x] [ShareGPT4V](https://huggingface.co/models?search=Lin-Chen/ShareGPT4V) From 177628bfd85565070916ad66a5ac4071ee0527d8 Mon Sep 17 00:00:00 2001 From: Douglas Hanley Date: Wed, 28 Feb 2024 02:51:11 -0600 Subject: [PATCH 482/812] llama : improve BERT tokenization (#5740) * implement nfd for stripping accents in wpm tokenizer * sort nfd map; reuse iterator * use builtin tolower * add locale include * Simplify to_lower cases Co-authored-by: Jared Van Bortel --------- Co-authored-by: Jared Van Bortel --- llama.cpp | 137 +++++++++------------------- unicode.h | 262 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 305 insertions(+), 94 deletions(-) diff --git a/llama.cpp b/llama.cpp index 464e1b89b..356ca1076 100644 --- a/llama.cpp +++ b/llama.cpp @@ -68,10 +68,12 @@ #include #include #include +#include #include #include #include #include +#include #include #include #include @@ -8941,37 +8943,46 @@ struct llm_tokenizer_wpm { } std::vector preprocess(const std::string & text) { - std::string ori_str = normalize(text); - uint64_t ori_size = ori_str.size(); + // normalalization form D + std::vector codepoints = codepoints_from_utf8(text); + std::vector nfd_codepoints; + for (uint32_t code : codepoints) { + auto it = nfd_map.find(code); + if (it != nfd_map.end()) { + for (uint32_t c : it->second) { + nfd_codepoints.push_back(c); + } + } else { + nfd_codepoints.push_back(code); + } + } - // single punct / single symbol / single digit - // baseline: add whitespace on the left and right of punct and chinese characters - std::vector words; + // strip accents, strip control, uniformize whitespace, + // to lowercase, pad chinese characters, pad punctuation std::string new_str = ""; - uint64_t i = 0; - while (i < ori_size) { - int utf_char_len = utf8_len(ori_str[i]); - if ((utf_char_len == 1) && ispunct(ori_str[i])) { - new_str += " "; - new_str += ori_str[i]; - new_str += " "; - i += 1; + for (uint32_t code : nfd_codepoints) { + int type = codepoint_type(code); + if (type == CODEPOINT_TYPE_ACCENT_MARK || type == CODEPOINT_TYPE_CONTROL) { + continue; } - else if ((utf_char_len == 3) && is_chinese_char(ori_str.substr(i, 3))) { - new_str += " "; - new_str += ori_str.substr(i, 3); - new_str += " "; - i += 3; + code = to_lower(code); + if (type == CODEPOINT_TYPE_WHITESPACE) { + code = ' '; } - else { - new_str += ori_str[i]; - i += 1; + std::string s = codepoint_to_utf8(code); + if (type == CODEPOINT_TYPE_PUNCTUATION || is_ascii_punct(code) || is_chinese_char(code)) { + new_str += " "; + new_str += s; + new_str += " "; + } else { + new_str += s; } } // split by whitespace uint64_t l = 0; uint64_t r = 0; + std::vector words; while (r < new_str.size()) { // if is whitespace if (isspace(new_str[r])) { @@ -8989,47 +9000,20 @@ struct llm_tokenizer_wpm { return words; } - std::string normalize(const std::string & text) { - // TODO: handle chinese characters? https://github.com/huggingface/tokenizers/blob/ef5f50605ddf9f8caef1598c0e4853862b9707a7/tokenizers/src/normalizers/bert.rs#L98 - std::string text2 = strip_accents(text); - for (size_t i = 0; i < text2.size(); i += utf8_len(text2[i])) { - char c = text2[i]; - if (c >= 'A' && c <= 'Z') { - text2[i] = c - 'A' + 'a'; - } + uint32_t to_lower(uint32_t code) { +#if defined(_WIN32) + if (code > 0xFFFF) { + return code; } - return text2; +#endif + return std::tolower(wchar_t(code), std::locale("en_US.UTF-8")); } - bool is_chinese_char(const std::string & str) { - int len = str.length(); - unsigned int codepoint = 0; - int num_bytes = 0; - int i = 0; - unsigned char ch = static_cast(str[i]); - if (ch <= 0x7f) { - codepoint = ch; - num_bytes = 1; - } else if ((ch >> 5) == 0x06) { - codepoint = ch & 0x1f; - num_bytes = 2; - } else if ((ch >> 4) == 0x0e) { - codepoint = ch & 0x0f; - num_bytes = 3; - } else if ((ch >> 3) == 0x1e) { - codepoint = ch & 0x07; - num_bytes = 4; - } - for (int j = 1; j < num_bytes; ++j) { - if (i + j >= len) { - return false; // incomplete UTF-8 character - } - unsigned char next_ch = static_cast(str[i + j]); - if ((next_ch >> 6) != 0x02) { - return false; // invalid trailing byte - } - codepoint = (codepoint << 6) | (next_ch & 0x3f); - } + bool is_ascii_punct(uint32_t code) { + return code < 256 && ispunct(code); + } + + bool is_chinese_char(uint32_t codepoint) { if ((codepoint >= 0x4E00 && codepoint <= 0x9FFF) || (codepoint >= 0x3400 && codepoint <= 0x4DBF) || (codepoint >= 0x20000 && codepoint <= 0x2A6DF) || @@ -9045,41 +9029,6 @@ struct llm_tokenizer_wpm { return false; } - std::string strip_accents(const std::string & input_string) { - std::string resultString; - std::map accent_map = { - {"À", 'A'}, {"Á", 'A'}, {"Â", 'A'}, {"Ã", 'A'}, {"Ä", 'A'}, {"Å", 'A'}, - {"à", 'a'}, {"á", 'a'}, {"â", 'a'}, {"ã", 'a'}, {"ä", 'a'}, {"å", 'a'}, - {"È", 'E'}, {"É", 'E'}, {"Ê", 'E'}, {"Ë", 'E'}, {"è", 'e'}, {"é", 'e'}, - {"ê", 'e'}, {"ë", 'e'}, {"Ì", 'I'}, {"Í", 'I'}, {"Î", 'I'}, {"Ï", 'I'}, - {"ì", 'i'}, {"í", 'i'}, {"î", 'i'}, {"ï", 'i'}, {"Ò", 'O'}, {"Ó", 'O'}, - {"Ô", 'O'}, {"Õ", 'O'}, {"Ö", 'O'}, {"ò", 'o'}, {"ó", 'o'}, {"ô", 'o'}, - {"õ", 'o'}, {"ö", 'o'}, {"Ù", 'U'}, {"Ú", 'U'}, {"Û", 'U'}, {"Ü", 'U'}, - {"ù", 'u'}, {"ú", 'u'}, {"û", 'u'}, {"ü", 'u'}, {"Ý", 'Y'}, {"ý", 'y'}, - {"Ç", 'C'}, {"ç", 'c'}, {"Ñ", 'N'}, {"ñ", 'n'}, - }; - - for (size_t i = 0; i < input_string.length();) { - int len = utf8_len(input_string[i]); - std::string curChar = input_string.substr(i, len); - auto iter = accent_map.find(curChar); - if (iter != accent_map.end()) { - resultString += iter->second; - } else { - resultString += curChar; - } - i += len; - } - - return resultString; - } - - static size_t utf8_len(char src) { - const size_t lookup[] = {1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 3, 4}; - uint8_t highbits = static_cast(src) >> 4; - return lookup[highbits]; - } - const llama_vocab & vocab; }; diff --git a/unicode.h b/unicode.h index 10a5dab01..620e2b580 100644 --- a/unicode.h +++ b/unicode.h @@ -223,6 +223,268 @@ static const std::vector> control_ranges = { {0x2B81E, 0x2B81F}, {0x2CEA2, 0x2CEAF}, {0x2EBE1, 0x2F7FF}, {0x2FA1E, 0x2FFFF}, {0x3134B, 0xE00FF}, {0xE01F0, 0x10FFFF}, }; +static const std::unordered_map> nfd_map = { +{0xC0, {0x41, 0x300}}, {0xC1, {0x41, 0x301}}, {0xC2, {0x41, 0x302}}, {0xC3, {0x41, 0x303}}, {0xC4, {0x41, 0x308}}, {0xC5, {0x41, 0x30A}}, {0xC7, {0x43, 0x327}}, {0xC8, {0x45, 0x300}}, +{0xC9, {0x45, 0x301}}, {0xCA, {0x45, 0x302}}, {0xCB, {0x45, 0x308}}, {0xCC, {0x49, 0x300}}, {0xCD, {0x49, 0x301}}, {0xCE, {0x49, 0x302}}, {0xCF, {0x49, 0x308}}, {0xD1, {0x4E, 0x303}}, +{0xD2, {0x4F, 0x300}}, {0xD3, {0x4F, 0x301}}, {0xD4, {0x4F, 0x302}}, {0xD5, {0x4F, 0x303}}, {0xD6, {0x4F, 0x308}}, {0xD9, {0x55, 0x300}}, {0xDA, {0x55, 0x301}}, {0xDB, {0x55, 0x302}}, +{0xDC, {0x55, 0x308}}, {0xDD, {0x59, 0x301}}, {0xE0, {0x61, 0x300}}, {0xE1, {0x61, 0x301}}, {0xE2, {0x61, 0x302}}, {0xE3, {0x61, 0x303}}, {0xE4, {0x61, 0x308}}, {0xE5, {0x61, 0x30A}}, +{0xE7, {0x63, 0x327}}, {0xE8, {0x65, 0x300}}, {0xE9, {0x65, 0x301}}, {0xEA, {0x65, 0x302}}, {0xEB, {0x65, 0x308}}, {0xEC, {0x69, 0x300}}, {0xED, {0x69, 0x301}}, {0xEE, {0x69, 0x302}}, +{0xEF, {0x69, 0x308}}, {0xF1, {0x6E, 0x303}}, {0xF2, {0x6F, 0x300}}, {0xF3, {0x6F, 0x301}}, {0xF4, {0x6F, 0x302}}, {0xF5, {0x6F, 0x303}}, {0xF6, {0x6F, 0x308}}, {0xF9, {0x75, 0x300}}, +{0xFA, {0x75, 0x301}}, {0xFB, {0x75, 0x302}}, {0xFC, {0x75, 0x308}}, {0xFD, {0x79, 0x301}}, {0xFF, {0x79, 0x308}}, {0x100, {0x41, 0x304}}, {0x101, {0x61, 0x304}}, {0x102, {0x41, 0x306}}, +{0x103, {0x61, 0x306}}, {0x104, {0x41, 0x328}}, {0x105, {0x61, 0x328}}, {0x106, {0x43, 0x301}}, {0x107, {0x63, 0x301}}, {0x108, {0x43, 0x302}}, {0x109, {0x63, 0x302}}, {0x10A, {0x43, 0x307}}, +{0x10B, {0x63, 0x307}}, {0x10C, {0x43, 0x30C}}, {0x10D, {0x63, 0x30C}}, {0x10E, {0x44, 0x30C}}, {0x10F, {0x64, 0x30C}}, {0x112, {0x45, 0x304}}, {0x113, {0x65, 0x304}}, {0x114, {0x45, 0x306}}, +{0x115, {0x65, 0x306}}, {0x116, {0x45, 0x307}}, {0x117, {0x65, 0x307}}, {0x118, {0x45, 0x328}}, {0x119, {0x65, 0x328}}, {0x11A, {0x45, 0x30C}}, {0x11B, {0x65, 0x30C}}, {0x11C, {0x47, 0x302}}, +{0x11D, {0x67, 0x302}}, {0x11E, {0x47, 0x306}}, {0x11F, {0x67, 0x306}}, {0x120, {0x47, 0x307}}, {0x121, {0x67, 0x307}}, {0x122, {0x47, 0x327}}, {0x123, {0x67, 0x327}}, {0x124, {0x48, 0x302}}, +{0x125, {0x68, 0x302}}, {0x128, {0x49, 0x303}}, {0x129, {0x69, 0x303}}, {0x12A, {0x49, 0x304}}, {0x12B, {0x69, 0x304}}, {0x12C, {0x49, 0x306}}, {0x12D, {0x69, 0x306}}, {0x12E, {0x49, 0x328}}, +{0x12F, {0x69, 0x328}}, {0x130, {0x49, 0x307}}, {0x134, {0x4A, 0x302}}, {0x135, {0x6A, 0x302}}, {0x136, {0x4B, 0x327}}, {0x137, {0x6B, 0x327}}, {0x139, {0x4C, 0x301}}, {0x13A, {0x6C, 0x301}}, +{0x13B, {0x4C, 0x327}}, {0x13C, {0x6C, 0x327}}, {0x13D, {0x4C, 0x30C}}, {0x13E, {0x6C, 0x30C}}, {0x143, {0x4E, 0x301}}, {0x144, {0x6E, 0x301}}, {0x145, {0x4E, 0x327}}, {0x146, {0x6E, 0x327}}, +{0x147, {0x4E, 0x30C}}, {0x148, {0x6E, 0x30C}}, {0x14C, {0x4F, 0x304}}, {0x14D, {0x6F, 0x304}}, {0x14E, {0x4F, 0x306}}, {0x14F, {0x6F, 0x306}}, {0x150, {0x4F, 0x30B}}, {0x151, {0x6F, 0x30B}}, +{0x154, {0x52, 0x301}}, {0x155, {0x72, 0x301}}, {0x156, {0x52, 0x327}}, {0x157, {0x72, 0x327}}, {0x158, {0x52, 0x30C}}, {0x159, {0x72, 0x30C}}, {0x15A, {0x53, 0x301}}, {0x15B, {0x73, 0x301}}, +{0x15C, {0x53, 0x302}}, {0x15D, {0x73, 0x302}}, {0x15E, {0x53, 0x327}}, {0x15F, {0x73, 0x327}}, {0x160, {0x53, 0x30C}}, {0x161, {0x73, 0x30C}}, {0x162, {0x54, 0x327}}, {0x163, {0x74, 0x327}}, +{0x164, {0x54, 0x30C}}, {0x165, {0x74, 0x30C}}, {0x168, {0x55, 0x303}}, {0x169, {0x75, 0x303}}, {0x16A, {0x55, 0x304}}, {0x16B, {0x75, 0x304}}, {0x16C, {0x55, 0x306}}, {0x16D, {0x75, 0x306}}, +{0x16E, {0x55, 0x30A}}, {0x16F, {0x75, 0x30A}}, {0x170, {0x55, 0x30B}}, {0x171, {0x75, 0x30B}}, {0x172, {0x55, 0x328}}, {0x173, {0x75, 0x328}}, {0x174, {0x57, 0x302}}, {0x175, {0x77, 0x302}}, +{0x176, {0x59, 0x302}}, {0x177, {0x79, 0x302}}, {0x178, {0x59, 0x308}}, {0x179, {0x5A, 0x301}}, {0x17A, {0x7A, 0x301}}, {0x17B, {0x5A, 0x307}}, {0x17C, {0x7A, 0x307}}, {0x17D, {0x5A, 0x30C}}, +{0x17E, {0x7A, 0x30C}}, {0x1A0, {0x4F, 0x31B}}, {0x1A1, {0x6F, 0x31B}}, {0x1AF, {0x55, 0x31B}}, {0x1B0, {0x75, 0x31B}}, {0x1CD, {0x41, 0x30C}}, {0x1CE, {0x61, 0x30C}}, {0x1CF, {0x49, 0x30C}}, +{0x1D0, {0x69, 0x30C}}, {0x1D1, {0x4F, 0x30C}}, {0x1D2, {0x6F, 0x30C}}, {0x1D3, {0x55, 0x30C}}, {0x1D4, {0x75, 0x30C}}, {0x1D5, {0x55, 0x308, 0x304}}, {0x1D6, {0x75, 0x308, 0x304}}, +{0x1D7, {0x55, 0x308, 0x301}}, {0x1D8, {0x75, 0x308, 0x301}}, {0x1D9, {0x55, 0x308, 0x30C}}, {0x1DA, {0x75, 0x308, 0x30C}}, {0x1DB, {0x55, 0x308, 0x300}}, {0x1DC, {0x75, 0x308, 0x300}}, +{0x1DE, {0x41, 0x308, 0x304}}, {0x1DF, {0x61, 0x308, 0x304}}, {0x1E0, {0x41, 0x307, 0x304}}, {0x1E1, {0x61, 0x307, 0x304}}, {0x1E2, {0xC6, 0x304}}, {0x1E3, {0xE6, 0x304}}, {0x1E6, {0x47, 0x30C}}, +{0x1E7, {0x67, 0x30C}}, {0x1E8, {0x4B, 0x30C}}, {0x1E9, {0x6B, 0x30C}}, {0x1EA, {0x4F, 0x328}}, {0x1EB, {0x6F, 0x328}}, {0x1EC, {0x4F, 0x328, 0x304}}, {0x1ED, {0x6F, 0x328, 0x304}}, +{0x1EE, {0x1B7, 0x30C}}, {0x1EF, {0x292, 0x30C}}, {0x1F0, {0x6A, 0x30C}}, {0x1F4, {0x47, 0x301}}, {0x1F5, {0x67, 0x301}}, {0x1F8, {0x4E, 0x300}}, {0x1F9, {0x6E, 0x300}}, {0x1FA, {0x41, 0x30A, 0x301}}, +{0x1FB, {0x61, 0x30A, 0x301}}, {0x1FC, {0xC6, 0x301}}, {0x1FD, {0xE6, 0x301}}, {0x1FE, {0xD8, 0x301}}, {0x1FF, {0xF8, 0x301}}, {0x200, {0x41, 0x30F}}, {0x201, {0x61, 0x30F}}, {0x202, {0x41, 0x311}}, +{0x203, {0x61, 0x311}}, {0x204, {0x45, 0x30F}}, {0x205, {0x65, 0x30F}}, {0x206, {0x45, 0x311}}, {0x207, {0x65, 0x311}}, {0x208, {0x49, 0x30F}}, {0x209, {0x69, 0x30F}}, {0x20A, {0x49, 0x311}}, +{0x20B, {0x69, 0x311}}, {0x20C, {0x4F, 0x30F}}, {0x20D, {0x6F, 0x30F}}, {0x20E, {0x4F, 0x311}}, {0x20F, {0x6F, 0x311}}, {0x210, {0x52, 0x30F}}, {0x211, {0x72, 0x30F}}, {0x212, {0x52, 0x311}}, +{0x213, {0x72, 0x311}}, {0x214, {0x55, 0x30F}}, {0x215, {0x75, 0x30F}}, {0x216, {0x55, 0x311}}, {0x217, {0x75, 0x311}}, {0x218, {0x53, 0x326}}, {0x219, {0x73, 0x326}}, {0x21A, {0x54, 0x326}}, +{0x21B, {0x74, 0x326}}, {0x21E, {0x48, 0x30C}}, {0x21F, {0x68, 0x30C}}, {0x226, {0x41, 0x307}}, {0x227, {0x61, 0x307}}, {0x228, {0x45, 0x327}}, {0x229, {0x65, 0x327}}, {0x22A, {0x4F, 0x308, 0x304}}, +{0x22B, {0x6F, 0x308, 0x304}}, {0x22C, {0x4F, 0x303, 0x304}}, {0x22D, {0x6F, 0x303, 0x304}}, {0x22E, {0x4F, 0x307}}, {0x22F, {0x6F, 0x307}}, {0x230, {0x4F, 0x307, 0x304}}, +{0x231, {0x6F, 0x307, 0x304}}, {0x232, {0x59, 0x304}}, {0x233, {0x79, 0x304}}, {0x340, {0x300}}, {0x341, {0x301}}, {0x343, {0x313}}, {0x344, {0x308, 0x301}}, {0x374, {0x2B9}}, {0x37E, {0x3B}}, +{0x385, {0xA8, 0x301}}, {0x386, {0x391, 0x301}}, {0x387, {0xB7}}, {0x388, {0x395, 0x301}}, {0x389, {0x397, 0x301}}, {0x38A, {0x399, 0x301}}, {0x38C, {0x39F, 0x301}}, {0x38E, {0x3A5, 0x301}}, +{0x38F, {0x3A9, 0x301}}, {0x390, {0x3B9, 0x308, 0x301}}, {0x3AA, {0x399, 0x308}}, {0x3AB, {0x3A5, 0x308}}, {0x3AC, {0x3B1, 0x301}}, {0x3AD, {0x3B5, 0x301}}, {0x3AE, {0x3B7, 0x301}}, +{0x3AF, {0x3B9, 0x301}}, {0x3B0, {0x3C5, 0x308, 0x301}}, {0x3CA, {0x3B9, 0x308}}, {0x3CB, {0x3C5, 0x308}}, {0x3CC, {0x3BF, 0x301}}, {0x3CD, {0x3C5, 0x301}}, {0x3CE, {0x3C9, 0x301}}, +{0x3D3, {0x3D2, 0x301}}, {0x3D4, {0x3D2, 0x308}}, {0x400, {0x415, 0x300}}, {0x401, {0x415, 0x308}}, {0x403, {0x413, 0x301}}, {0x407, {0x406, 0x308}}, {0x40C, {0x41A, 0x301}}, {0x40D, {0x418, 0x300}}, +{0x40E, {0x423, 0x306}}, {0x419, {0x418, 0x306}}, {0x439, {0x438, 0x306}}, {0x450, {0x435, 0x300}}, {0x451, {0x435, 0x308}}, {0x453, {0x433, 0x301}}, {0x457, {0x456, 0x308}}, {0x45C, {0x43A, 0x301}}, +{0x45D, {0x438, 0x300}}, {0x45E, {0x443, 0x306}}, {0x476, {0x474, 0x30F}}, {0x477, {0x475, 0x30F}}, {0x4C1, {0x416, 0x306}}, {0x4C2, {0x436, 0x306}}, {0x4D0, {0x410, 0x306}}, {0x4D1, {0x430, 0x306}}, +{0x4D2, {0x410, 0x308}}, {0x4D3, {0x430, 0x308}}, {0x4D6, {0x415, 0x306}}, {0x4D7, {0x435, 0x306}}, {0x4DA, {0x4D8, 0x308}}, {0x4DB, {0x4D9, 0x308}}, {0x4DC, {0x416, 0x308}}, {0x4DD, {0x436, 0x308}}, +{0x4DE, {0x417, 0x308}}, {0x4DF, {0x437, 0x308}}, {0x4E2, {0x418, 0x304}}, {0x4E3, {0x438, 0x304}}, {0x4E4, {0x418, 0x308}}, {0x4E5, {0x438, 0x308}}, {0x4E6, {0x41E, 0x308}}, {0x4E7, {0x43E, 0x308}}, +{0x4EA, {0x4E8, 0x308}}, {0x4EB, {0x4E9, 0x308}}, {0x4EC, {0x42D, 0x308}}, {0x4ED, {0x44D, 0x308}}, {0x4EE, {0x423, 0x304}}, {0x4EF, {0x443, 0x304}}, {0x4F0, {0x423, 0x308}}, {0x4F1, {0x443, 0x308}}, +{0x4F2, {0x423, 0x30B}}, {0x4F3, {0x443, 0x30B}}, {0x4F4, {0x427, 0x308}}, {0x4F5, {0x447, 0x308}}, {0x4F8, {0x42B, 0x308}}, {0x4F9, {0x44B, 0x308}}, {0x622, {0x627, 0x653}}, {0x623, {0x627, 0x654}}, +{0x624, {0x648, 0x654}}, {0x625, {0x627, 0x655}}, {0x626, {0x64A, 0x654}}, {0x6C0, {0x6D5, 0x654}}, {0x6C2, {0x6C1, 0x654}}, {0x6D3, {0x6D2, 0x654}}, {0x929, {0x928, 0x93C}}, {0x931, {0x930, 0x93C}}, +{0x934, {0x933, 0x93C}}, {0x958, {0x915, 0x93C}}, {0x959, {0x916, 0x93C}}, {0x95A, {0x917, 0x93C}}, {0x95B, {0x91C, 0x93C}}, {0x95C, {0x921, 0x93C}}, {0x95D, {0x922, 0x93C}}, {0x95E, {0x92B, 0x93C}}, +{0x95F, {0x92F, 0x93C}}, {0x9CB, {0x9C7, 0x9BE}}, {0x9CC, {0x9C7, 0x9D7}}, {0x9DC, {0x9A1, 0x9BC}}, {0x9DD, {0x9A2, 0x9BC}}, {0x9DF, {0x9AF, 0x9BC}}, {0xA33, {0xA32, 0xA3C}}, {0xA36, {0xA38, 0xA3C}}, +{0xA59, {0xA16, 0xA3C}}, {0xA5A, {0xA17, 0xA3C}}, {0xA5B, {0xA1C, 0xA3C}}, {0xA5E, {0xA2B, 0xA3C}}, {0xB48, {0xB47, 0xB56}}, {0xB4B, {0xB47, 0xB3E}}, {0xB4C, {0xB47, 0xB57}}, {0xB5C, {0xB21, 0xB3C}}, +{0xB5D, {0xB22, 0xB3C}}, {0xB94, {0xB92, 0xBD7}}, {0xBCA, {0xBC6, 0xBBE}}, {0xBCB, {0xBC7, 0xBBE}}, {0xBCC, {0xBC6, 0xBD7}}, {0xC48, {0xC46, 0xC56}}, {0xCC0, {0xCBF, 0xCD5}}, {0xCC7, {0xCC6, 0xCD5}}, +{0xCC8, {0xCC6, 0xCD6}}, {0xCCA, {0xCC6, 0xCC2}}, {0xCCB, {0xCC6, 0xCC2, 0xCD5}}, {0xD4A, {0xD46, 0xD3E}}, {0xD4B, {0xD47, 0xD3E}}, {0xD4C, {0xD46, 0xD57}}, {0xDDA, {0xDD9, 0xDCA}}, +{0xDDC, {0xDD9, 0xDCF}}, {0xDDD, {0xDD9, 0xDCF, 0xDCA}}, {0xDDE, {0xDD9, 0xDDF}}, {0xF43, {0xF42, 0xFB7}}, {0xF4D, {0xF4C, 0xFB7}}, {0xF52, {0xF51, 0xFB7}}, {0xF57, {0xF56, 0xFB7}}, +{0xF5C, {0xF5B, 0xFB7}}, {0xF69, {0xF40, 0xFB5}}, {0xF73, {0xF71, 0xF72}}, {0xF75, {0xF71, 0xF74}}, {0xF76, {0xFB2, 0xF80}}, {0xF78, {0xFB3, 0xF80}}, {0xF81, {0xF71, 0xF80}}, {0xF93, {0xF92, 0xFB7}}, +{0xF9D, {0xF9C, 0xFB7}}, {0xFA2, {0xFA1, 0xFB7}}, {0xFA7, {0xFA6, 0xFB7}}, {0xFAC, {0xFAB, 0xFB7}}, {0xFB9, {0xF90, 0xFB5}}, {0x1026, {0x1025, 0x102E}}, {0x1B06, {0x1B05, 0x1B35}}, +{0x1B08, {0x1B07, 0x1B35}}, {0x1B0A, {0x1B09, 0x1B35}}, {0x1B0C, {0x1B0B, 0x1B35}}, {0x1B0E, {0x1B0D, 0x1B35}}, {0x1B12, {0x1B11, 0x1B35}}, {0x1B3B, {0x1B3A, 0x1B35}}, {0x1B3D, {0x1B3C, 0x1B35}}, +{0x1B40, {0x1B3E, 0x1B35}}, {0x1B41, {0x1B3F, 0x1B35}}, {0x1B43, {0x1B42, 0x1B35}}, {0x1E00, {0x41, 0x325}}, {0x1E01, {0x61, 0x325}}, {0x1E02, {0x42, 0x307}}, {0x1E03, {0x62, 0x307}}, +{0x1E04, {0x42, 0x323}}, {0x1E05, {0x62, 0x323}}, {0x1E06, {0x42, 0x331}}, {0x1E07, {0x62, 0x331}}, {0x1E08, {0x43, 0x327, 0x301}}, {0x1E09, {0x63, 0x327, 0x301}}, {0x1E0A, {0x44, 0x307}}, +{0x1E0B, {0x64, 0x307}}, {0x1E0C, {0x44, 0x323}}, {0x1E0D, {0x64, 0x323}}, {0x1E0E, {0x44, 0x331}}, {0x1E0F, {0x64, 0x331}}, {0x1E10, {0x44, 0x327}}, {0x1E11, {0x64, 0x327}}, {0x1E12, {0x44, 0x32D}}, +{0x1E13, {0x64, 0x32D}}, {0x1E14, {0x45, 0x304, 0x300}}, {0x1E15, {0x65, 0x304, 0x300}}, {0x1E16, {0x45, 0x304, 0x301}}, {0x1E17, {0x65, 0x304, 0x301}}, {0x1E18, {0x45, 0x32D}}, +{0x1E19, {0x65, 0x32D}}, {0x1E1A, {0x45, 0x330}}, {0x1E1B, {0x65, 0x330}}, {0x1E1C, {0x45, 0x327, 0x306}}, {0x1E1D, {0x65, 0x327, 0x306}}, {0x1E1E, {0x46, 0x307}}, {0x1E1F, {0x66, 0x307}}, +{0x1E20, {0x47, 0x304}}, {0x1E21, {0x67, 0x304}}, {0x1E22, {0x48, 0x307}}, {0x1E23, {0x68, 0x307}}, {0x1E24, {0x48, 0x323}}, {0x1E25, {0x68, 0x323}}, {0x1E26, {0x48, 0x308}}, {0x1E27, {0x68, 0x308}}, +{0x1E28, {0x48, 0x327}}, {0x1E29, {0x68, 0x327}}, {0x1E2A, {0x48, 0x32E}}, {0x1E2B, {0x68, 0x32E}}, {0x1E2C, {0x49, 0x330}}, {0x1E2D, {0x69, 0x330}}, {0x1E2E, {0x49, 0x308, 0x301}}, +{0x1E2F, {0x69, 0x308, 0x301}}, {0x1E30, {0x4B, 0x301}}, {0x1E31, {0x6B, 0x301}}, {0x1E32, {0x4B, 0x323}}, {0x1E33, {0x6B, 0x323}}, {0x1E34, {0x4B, 0x331}}, {0x1E35, {0x6B, 0x331}}, +{0x1E36, {0x4C, 0x323}}, {0x1E37, {0x6C, 0x323}}, {0x1E38, {0x4C, 0x323, 0x304}}, {0x1E39, {0x6C, 0x323, 0x304}}, {0x1E3A, {0x4C, 0x331}}, {0x1E3B, {0x6C, 0x331}}, {0x1E3C, {0x4C, 0x32D}}, +{0x1E3D, {0x6C, 0x32D}}, {0x1E3E, {0x4D, 0x301}}, {0x1E3F, {0x6D, 0x301}}, {0x1E40, {0x4D, 0x307}}, {0x1E41, {0x6D, 0x307}}, {0x1E42, {0x4D, 0x323}}, {0x1E43, {0x6D, 0x323}}, {0x1E44, {0x4E, 0x307}}, +{0x1E45, {0x6E, 0x307}}, {0x1E46, {0x4E, 0x323}}, {0x1E47, {0x6E, 0x323}}, {0x1E48, {0x4E, 0x331}}, {0x1E49, {0x6E, 0x331}}, {0x1E4A, {0x4E, 0x32D}}, {0x1E4B, {0x6E, 0x32D}}, +{0x1E4C, {0x4F, 0x303, 0x301}}, {0x1E4D, {0x6F, 0x303, 0x301}}, {0x1E4E, {0x4F, 0x303, 0x308}}, {0x1E4F, {0x6F, 0x303, 0x308}}, {0x1E50, {0x4F, 0x304, 0x300}}, {0x1E51, {0x6F, 0x304, 0x300}}, +{0x1E52, {0x4F, 0x304, 0x301}}, {0x1E53, {0x6F, 0x304, 0x301}}, {0x1E54, {0x50, 0x301}}, {0x1E55, {0x70, 0x301}}, {0x1E56, {0x50, 0x307}}, {0x1E57, {0x70, 0x307}}, {0x1E58, {0x52, 0x307}}, +{0x1E59, {0x72, 0x307}}, {0x1E5A, {0x52, 0x323}}, {0x1E5B, {0x72, 0x323}}, {0x1E5C, {0x52, 0x323, 0x304}}, {0x1E5D, {0x72, 0x323, 0x304}}, {0x1E5E, {0x52, 0x331}}, {0x1E5F, {0x72, 0x331}}, +{0x1E60, {0x53, 0x307}}, {0x1E61, {0x73, 0x307}}, {0x1E62, {0x53, 0x323}}, {0x1E63, {0x73, 0x323}}, {0x1E64, {0x53, 0x301, 0x307}}, {0x1E65, {0x73, 0x301, 0x307}}, {0x1E66, {0x53, 0x30C, 0x307}}, +{0x1E67, {0x73, 0x30C, 0x307}}, {0x1E68, {0x53, 0x323, 0x307}}, {0x1E69, {0x73, 0x323, 0x307}}, {0x1E6A, {0x54, 0x307}}, {0x1E6B, {0x74, 0x307}}, {0x1E6C, {0x54, 0x323}}, {0x1E6D, {0x74, 0x323}}, +{0x1E6E, {0x54, 0x331}}, {0x1E6F, {0x74, 0x331}}, {0x1E70, {0x54, 0x32D}}, {0x1E71, {0x74, 0x32D}}, {0x1E72, {0x55, 0x324}}, {0x1E73, {0x75, 0x324}}, {0x1E74, {0x55, 0x330}}, {0x1E75, {0x75, 0x330}}, +{0x1E76, {0x55, 0x32D}}, {0x1E77, {0x75, 0x32D}}, {0x1E78, {0x55, 0x303, 0x301}}, {0x1E79, {0x75, 0x303, 0x301}}, {0x1E7A, {0x55, 0x304, 0x308}}, {0x1E7B, {0x75, 0x304, 0x308}}, +{0x1E7C, {0x56, 0x303}}, {0x1E7D, {0x76, 0x303}}, {0x1E7E, {0x56, 0x323}}, {0x1E7F, {0x76, 0x323}}, {0x1E80, {0x57, 0x300}}, {0x1E81, {0x77, 0x300}}, {0x1E82, {0x57, 0x301}}, {0x1E83, {0x77, 0x301}}, +{0x1E84, {0x57, 0x308}}, {0x1E85, {0x77, 0x308}}, {0x1E86, {0x57, 0x307}}, {0x1E87, {0x77, 0x307}}, {0x1E88, {0x57, 0x323}}, {0x1E89, {0x77, 0x323}}, {0x1E8A, {0x58, 0x307}}, {0x1E8B, {0x78, 0x307}}, +{0x1E8C, {0x58, 0x308}}, {0x1E8D, {0x78, 0x308}}, {0x1E8E, {0x59, 0x307}}, {0x1E8F, {0x79, 0x307}}, {0x1E90, {0x5A, 0x302}}, {0x1E91, {0x7A, 0x302}}, {0x1E92, {0x5A, 0x323}}, {0x1E93, {0x7A, 0x323}}, +{0x1E94, {0x5A, 0x331}}, {0x1E95, {0x7A, 0x331}}, {0x1E96, {0x68, 0x331}}, {0x1E97, {0x74, 0x308}}, {0x1E98, {0x77, 0x30A}}, {0x1E99, {0x79, 0x30A}}, {0x1E9B, {0x17F, 0x307}}, {0x1EA0, {0x41, 0x323}}, +{0x1EA1, {0x61, 0x323}}, {0x1EA2, {0x41, 0x309}}, {0x1EA3, {0x61, 0x309}}, {0x1EA4, {0x41, 0x302, 0x301}}, {0x1EA5, {0x61, 0x302, 0x301}}, {0x1EA6, {0x41, 0x302, 0x300}}, +{0x1EA7, {0x61, 0x302, 0x300}}, {0x1EA8, {0x41, 0x302, 0x309}}, {0x1EA9, {0x61, 0x302, 0x309}}, {0x1EAA, {0x41, 0x302, 0x303}}, {0x1EAB, {0x61, 0x302, 0x303}}, {0x1EAC, {0x41, 0x323, 0x302}}, +{0x1EAD, {0x61, 0x323, 0x302}}, {0x1EAE, {0x41, 0x306, 0x301}}, {0x1EAF, {0x61, 0x306, 0x301}}, {0x1EB0, {0x41, 0x306, 0x300}}, {0x1EB1, {0x61, 0x306, 0x300}}, {0x1EB2, {0x41, 0x306, 0x309}}, +{0x1EB3, {0x61, 0x306, 0x309}}, {0x1EB4, {0x41, 0x306, 0x303}}, {0x1EB5, {0x61, 0x306, 0x303}}, {0x1EB6, {0x41, 0x323, 0x306}}, {0x1EB7, {0x61, 0x323, 0x306}}, {0x1EB8, {0x45, 0x323}}, +{0x1EB9, {0x65, 0x323}}, {0x1EBA, {0x45, 0x309}}, {0x1EBB, {0x65, 0x309}}, {0x1EBC, {0x45, 0x303}}, {0x1EBD, {0x65, 0x303}}, {0x1EBE, {0x45, 0x302, 0x301}}, {0x1EBF, {0x65, 0x302, 0x301}}, +{0x1EC0, {0x45, 0x302, 0x300}}, {0x1EC1, {0x65, 0x302, 0x300}}, {0x1EC2, {0x45, 0x302, 0x309}}, {0x1EC3, {0x65, 0x302, 0x309}}, {0x1EC4, {0x45, 0x302, 0x303}}, {0x1EC5, {0x65, 0x302, 0x303}}, +{0x1EC6, {0x45, 0x323, 0x302}}, {0x1EC7, {0x65, 0x323, 0x302}}, {0x1EC8, {0x49, 0x309}}, {0x1EC9, {0x69, 0x309}}, {0x1ECA, {0x49, 0x323}}, {0x1ECB, {0x69, 0x323}}, {0x1ECC, {0x4F, 0x323}}, +{0x1ECD, {0x6F, 0x323}}, {0x1ECE, {0x4F, 0x309}}, {0x1ECF, {0x6F, 0x309}}, {0x1ED0, {0x4F, 0x302, 0x301}}, {0x1ED1, {0x6F, 0x302, 0x301}}, {0x1ED2, {0x4F, 0x302, 0x300}}, +{0x1ED3, {0x6F, 0x302, 0x300}}, {0x1ED4, {0x4F, 0x302, 0x309}}, {0x1ED5, {0x6F, 0x302, 0x309}}, {0x1ED6, {0x4F, 0x302, 0x303}}, {0x1ED7, {0x6F, 0x302, 0x303}}, {0x1ED8, {0x4F, 0x323, 0x302}}, +{0x1ED9, {0x6F, 0x323, 0x302}}, {0x1EDA, {0x4F, 0x31B, 0x301}}, {0x1EDB, {0x6F, 0x31B, 0x301}}, {0x1EDC, {0x4F, 0x31B, 0x300}}, {0x1EDD, {0x6F, 0x31B, 0x300}}, {0x1EDE, {0x4F, 0x31B, 0x309}}, +{0x1EDF, {0x6F, 0x31B, 0x309}}, {0x1EE0, {0x4F, 0x31B, 0x303}}, {0x1EE1, {0x6F, 0x31B, 0x303}}, {0x1EE2, {0x4F, 0x31B, 0x323}}, {0x1EE3, {0x6F, 0x31B, 0x323}}, {0x1EE4, {0x55, 0x323}}, +{0x1EE5, {0x75, 0x323}}, {0x1EE6, {0x55, 0x309}}, {0x1EE7, {0x75, 0x309}}, {0x1EE8, {0x55, 0x31B, 0x301}}, {0x1EE9, {0x75, 0x31B, 0x301}}, {0x1EEA, {0x55, 0x31B, 0x300}}, +{0x1EEB, {0x75, 0x31B, 0x300}}, {0x1EEC, {0x55, 0x31B, 0x309}}, {0x1EED, {0x75, 0x31B, 0x309}}, {0x1EEE, {0x55, 0x31B, 0x303}}, {0x1EEF, {0x75, 0x31B, 0x303}}, {0x1EF0, {0x55, 0x31B, 0x323}}, +{0x1EF1, {0x75, 0x31B, 0x323}}, {0x1EF2, {0x59, 0x300}}, {0x1EF3, {0x79, 0x300}}, {0x1EF4, {0x59, 0x323}}, {0x1EF5, {0x79, 0x323}}, {0x1EF6, {0x59, 0x309}}, {0x1EF7, {0x79, 0x309}}, +{0x1EF8, {0x59, 0x303}}, {0x1EF9, {0x79, 0x303}}, {0x1F00, {0x3B1, 0x313}}, {0x1F01, {0x3B1, 0x314}}, {0x1F02, {0x3B1, 0x313, 0x300}}, {0x1F03, {0x3B1, 0x314, 0x300}}, {0x1F04, {0x3B1, 0x313, 0x301}}, +{0x1F05, {0x3B1, 0x314, 0x301}}, {0x1F06, {0x3B1, 0x313, 0x342}}, {0x1F07, {0x3B1, 0x314, 0x342}}, {0x1F08, {0x391, 0x313}}, {0x1F09, {0x391, 0x314}}, {0x1F0A, {0x391, 0x313, 0x300}}, +{0x1F0B, {0x391, 0x314, 0x300}}, {0x1F0C, {0x391, 0x313, 0x301}}, {0x1F0D, {0x391, 0x314, 0x301}}, {0x1F0E, {0x391, 0x313, 0x342}}, {0x1F0F, {0x391, 0x314, 0x342}}, {0x1F10, {0x3B5, 0x313}}, +{0x1F11, {0x3B5, 0x314}}, {0x1F12, {0x3B5, 0x313, 0x300}}, {0x1F13, {0x3B5, 0x314, 0x300}}, {0x1F14, {0x3B5, 0x313, 0x301}}, {0x1F15, {0x3B5, 0x314, 0x301}}, {0x1F18, {0x395, 0x313}}, +{0x1F19, {0x395, 0x314}}, {0x1F1A, {0x395, 0x313, 0x300}}, {0x1F1B, {0x395, 0x314, 0x300}}, {0x1F1C, {0x395, 0x313, 0x301}}, {0x1F1D, {0x395, 0x314, 0x301}}, {0x1F20, {0x3B7, 0x313}}, +{0x1F21, {0x3B7, 0x314}}, {0x1F22, {0x3B7, 0x313, 0x300}}, {0x1F23, {0x3B7, 0x314, 0x300}}, {0x1F24, {0x3B7, 0x313, 0x301}}, {0x1F25, {0x3B7, 0x314, 0x301}}, {0x1F26, {0x3B7, 0x313, 0x342}}, +{0x1F27, {0x3B7, 0x314, 0x342}}, {0x1F28, {0x397, 0x313}}, {0x1F29, {0x397, 0x314}}, {0x1F2A, {0x397, 0x313, 0x300}}, {0x1F2B, {0x397, 0x314, 0x300}}, {0x1F2C, {0x397, 0x313, 0x301}}, +{0x1F2D, {0x397, 0x314, 0x301}}, {0x1F2E, {0x397, 0x313, 0x342}}, {0x1F2F, {0x397, 0x314, 0x342}}, {0x1F30, {0x3B9, 0x313}}, {0x1F31, {0x3B9, 0x314}}, {0x1F32, {0x3B9, 0x313, 0x300}}, +{0x1F33, {0x3B9, 0x314, 0x300}}, {0x1F34, {0x3B9, 0x313, 0x301}}, {0x1F35, {0x3B9, 0x314, 0x301}}, {0x1F36, {0x3B9, 0x313, 0x342}}, {0x1F37, {0x3B9, 0x314, 0x342}}, {0x1F38, {0x399, 0x313}}, +{0x1F39, {0x399, 0x314}}, {0x1F3A, {0x399, 0x313, 0x300}}, {0x1F3B, {0x399, 0x314, 0x300}}, {0x1F3C, {0x399, 0x313, 0x301}}, {0x1F3D, {0x399, 0x314, 0x301}}, {0x1F3E, {0x399, 0x313, 0x342}}, +{0x1F3F, {0x399, 0x314, 0x342}}, {0x1F40, {0x3BF, 0x313}}, {0x1F41, {0x3BF, 0x314}}, {0x1F42, {0x3BF, 0x313, 0x300}}, {0x1F43, {0x3BF, 0x314, 0x300}}, {0x1F44, {0x3BF, 0x313, 0x301}}, +{0x1F45, {0x3BF, 0x314, 0x301}}, {0x1F48, {0x39F, 0x313}}, {0x1F49, {0x39F, 0x314}}, {0x1F4A, {0x39F, 0x313, 0x300}}, {0x1F4B, {0x39F, 0x314, 0x300}}, {0x1F4C, {0x39F, 0x313, 0x301}}, +{0x1F4D, {0x39F, 0x314, 0x301}}, {0x1F50, {0x3C5, 0x313}}, {0x1F51, {0x3C5, 0x314}}, {0x1F52, {0x3C5, 0x313, 0x300}}, {0x1F53, {0x3C5, 0x314, 0x300}}, {0x1F54, {0x3C5, 0x313, 0x301}}, +{0x1F55, {0x3C5, 0x314, 0x301}}, {0x1F56, {0x3C5, 0x313, 0x342}}, {0x1F57, {0x3C5, 0x314, 0x342}}, {0x1F59, {0x3A5, 0x314}}, {0x1F5B, {0x3A5, 0x314, 0x300}}, {0x1F5D, {0x3A5, 0x314, 0x301}}, +{0x1F5F, {0x3A5, 0x314, 0x342}}, {0x1F60, {0x3C9, 0x313}}, {0x1F61, {0x3C9, 0x314}}, {0x1F62, {0x3C9, 0x313, 0x300}}, {0x1F63, {0x3C9, 0x314, 0x300}}, {0x1F64, {0x3C9, 0x313, 0x301}}, +{0x1F65, {0x3C9, 0x314, 0x301}}, {0x1F66, {0x3C9, 0x313, 0x342}}, {0x1F67, {0x3C9, 0x314, 0x342}}, {0x1F68, {0x3A9, 0x313}}, {0x1F69, {0x3A9, 0x314}}, {0x1F6A, {0x3A9, 0x313, 0x300}}, +{0x1F6B, {0x3A9, 0x314, 0x300}}, {0x1F6C, {0x3A9, 0x313, 0x301}}, {0x1F6D, {0x3A9, 0x314, 0x301}}, {0x1F6E, {0x3A9, 0x313, 0x342}}, {0x1F6F, {0x3A9, 0x314, 0x342}}, {0x1F70, {0x3B1, 0x300}}, +{0x1F71, {0x3B1, 0x301}}, {0x1F72, {0x3B5, 0x300}}, {0x1F73, {0x3B5, 0x301}}, {0x1F74, {0x3B7, 0x300}}, {0x1F75, {0x3B7, 0x301}}, {0x1F76, {0x3B9, 0x300}}, {0x1F77, {0x3B9, 0x301}}, +{0x1F78, {0x3BF, 0x300}}, {0x1F79, {0x3BF, 0x301}}, {0x1F7A, {0x3C5, 0x300}}, {0x1F7B, {0x3C5, 0x301}}, {0x1F7C, {0x3C9, 0x300}}, {0x1F7D, {0x3C9, 0x301}}, {0x1F80, {0x3B1, 0x313, 0x345}}, +{0x1F81, {0x3B1, 0x314, 0x345}}, {0x1F82, {0x3B1, 0x313, 0x300, 0x345}}, {0x1F83, {0x3B1, 0x314, 0x300, 0x345}}, {0x1F84, {0x3B1, 0x313, 0x301, 0x345}}, {0x1F85, {0x3B1, 0x314, 0x301, 0x345}}, +{0x1F86, {0x3B1, 0x313, 0x342, 0x345}}, {0x1F87, {0x3B1, 0x314, 0x342, 0x345}}, {0x1F88, {0x391, 0x313, 0x345}}, {0x1F89, {0x391, 0x314, 0x345}}, {0x1F8A, {0x391, 0x313, 0x300, 0x345}}, +{0x1F8B, {0x391, 0x314, 0x300, 0x345}}, {0x1F8C, {0x391, 0x313, 0x301, 0x345}}, {0x1F8D, {0x391, 0x314, 0x301, 0x345}}, {0x1F8E, {0x391, 0x313, 0x342, 0x345}}, {0x1F8F, {0x391, 0x314, 0x342, 0x345}}, +{0x1F90, {0x3B7, 0x313, 0x345}}, {0x1F91, {0x3B7, 0x314, 0x345}}, {0x1F92, {0x3B7, 0x313, 0x300, 0x345}}, {0x1F93, {0x3B7, 0x314, 0x300, 0x345}}, {0x1F94, {0x3B7, 0x313, 0x301, 0x345}}, +{0x1F95, {0x3B7, 0x314, 0x301, 0x345}}, {0x1F96, {0x3B7, 0x313, 0x342, 0x345}}, {0x1F97, {0x3B7, 0x314, 0x342, 0x345}}, {0x1F98, {0x397, 0x313, 0x345}}, {0x1F99, {0x397, 0x314, 0x345}}, +{0x1F9A, {0x397, 0x313, 0x300, 0x345}}, {0x1F9B, {0x397, 0x314, 0x300, 0x345}}, {0x1F9C, {0x397, 0x313, 0x301, 0x345}}, {0x1F9D, {0x397, 0x314, 0x301, 0x345}}, {0x1F9E, {0x397, 0x313, 0x342, 0x345}}, +{0x1F9F, {0x397, 0x314, 0x342, 0x345}}, {0x1FA0, {0x3C9, 0x313, 0x345}}, {0x1FA1, {0x3C9, 0x314, 0x345}}, {0x1FA2, {0x3C9, 0x313, 0x300, 0x345}}, {0x1FA3, {0x3C9, 0x314, 0x300, 0x345}}, +{0x1FA4, {0x3C9, 0x313, 0x301, 0x345}}, {0x1FA5, {0x3C9, 0x314, 0x301, 0x345}}, {0x1FA6, {0x3C9, 0x313, 0x342, 0x345}}, {0x1FA7, {0x3C9, 0x314, 0x342, 0x345}}, {0x1FA8, {0x3A9, 0x313, 0x345}}, +{0x1FA9, {0x3A9, 0x314, 0x345}}, {0x1FAA, {0x3A9, 0x313, 0x300, 0x345}}, {0x1FAB, {0x3A9, 0x314, 0x300, 0x345}}, {0x1FAC, {0x3A9, 0x313, 0x301, 0x345}}, {0x1FAD, {0x3A9, 0x314, 0x301, 0x345}}, +{0x1FAE, {0x3A9, 0x313, 0x342, 0x345}}, {0x1FAF, {0x3A9, 0x314, 0x342, 0x345}}, {0x1FB0, {0x3B1, 0x306}}, {0x1FB1, {0x3B1, 0x304}}, {0x1FB2, {0x3B1, 0x300, 0x345}}, {0x1FB3, {0x3B1, 0x345}}, +{0x1FB4, {0x3B1, 0x301, 0x345}}, {0x1FB6, {0x3B1, 0x342}}, {0x1FB7, {0x3B1, 0x342, 0x345}}, {0x1FB8, {0x391, 0x306}}, {0x1FB9, {0x391, 0x304}}, {0x1FBA, {0x391, 0x300}}, {0x1FBB, {0x391, 0x301}}, +{0x1FBC, {0x391, 0x345}}, {0x1FBE, {0x3B9}}, {0x1FC1, {0xA8, 0x342}}, {0x1FC2, {0x3B7, 0x300, 0x345}}, {0x1FC3, {0x3B7, 0x345}}, {0x1FC4, {0x3B7, 0x301, 0x345}}, {0x1FC6, {0x3B7, 0x342}}, +{0x1FC7, {0x3B7, 0x342, 0x345}}, {0x1FC8, {0x395, 0x300}}, {0x1FC9, {0x395, 0x301}}, {0x1FCA, {0x397, 0x300}}, {0x1FCB, {0x397, 0x301}}, {0x1FCC, {0x397, 0x345}}, {0x1FCD, {0x1FBF, 0x300}}, +{0x1FCE, {0x1FBF, 0x301}}, {0x1FCF, {0x1FBF, 0x342}}, {0x1FD0, {0x3B9, 0x306}}, {0x1FD1, {0x3B9, 0x304}}, {0x1FD2, {0x3B9, 0x308, 0x300}}, {0x1FD3, {0x3B9, 0x308, 0x301}}, {0x1FD6, {0x3B9, 0x342}}, +{0x1FD7, {0x3B9, 0x308, 0x342}}, {0x1FD8, {0x399, 0x306}}, {0x1FD9, {0x399, 0x304}}, {0x1FDA, {0x399, 0x300}}, {0x1FDB, {0x399, 0x301}}, {0x1FDD, {0x1FFE, 0x300}}, {0x1FDE, {0x1FFE, 0x301}}, +{0x1FDF, {0x1FFE, 0x342}}, {0x1FE0, {0x3C5, 0x306}}, {0x1FE1, {0x3C5, 0x304}}, {0x1FE2, {0x3C5, 0x308, 0x300}}, {0x1FE3, {0x3C5, 0x308, 0x301}}, {0x1FE4, {0x3C1, 0x313}}, {0x1FE5, {0x3C1, 0x314}}, +{0x1FE6, {0x3C5, 0x342}}, {0x1FE7, {0x3C5, 0x308, 0x342}}, {0x1FE8, {0x3A5, 0x306}}, {0x1FE9, {0x3A5, 0x304}}, {0x1FEA, {0x3A5, 0x300}}, {0x1FEB, {0x3A5, 0x301}}, {0x1FEC, {0x3A1, 0x314}}, +{0x1FED, {0xA8, 0x300}}, {0x1FEE, {0xA8, 0x301}}, {0x1FEF, {0x60}}, {0x1FF2, {0x3C9, 0x300, 0x345}}, {0x1FF3, {0x3C9, 0x345}}, {0x1FF4, {0x3C9, 0x301, 0x345}}, {0x1FF6, {0x3C9, 0x342}}, +{0x1FF7, {0x3C9, 0x342, 0x345}}, {0x1FF8, {0x39F, 0x300}}, {0x1FF9, {0x39F, 0x301}}, {0x1FFA, {0x3A9, 0x300}}, {0x1FFB, {0x3A9, 0x301}}, {0x1FFC, {0x3A9, 0x345}}, {0x1FFD, {0xB4}}, {0x2000, {0x2002}}, +{0x2001, {0x2003}}, {0x2126, {0x3A9}}, {0x212A, {0x4B}}, {0x212B, {0x41, 0x30A}}, {0x219A, {0x2190, 0x338}}, {0x219B, {0x2192, 0x338}}, {0x21AE, {0x2194, 0x338}}, {0x21CD, {0x21D0, 0x338}}, +{0x21CE, {0x21D4, 0x338}}, {0x21CF, {0x21D2, 0x338}}, {0x2204, {0x2203, 0x338}}, {0x2209, {0x2208, 0x338}}, {0x220C, {0x220B, 0x338}}, {0x2224, {0x2223, 0x338}}, {0x2226, {0x2225, 0x338}}, +{0x2241, {0x223C, 0x338}}, {0x2244, {0x2243, 0x338}}, {0x2247, {0x2245, 0x338}}, {0x2249, {0x2248, 0x338}}, {0x2260, {0x3D, 0x338}}, {0x2262, {0x2261, 0x338}}, {0x226D, {0x224D, 0x338}}, +{0x226E, {0x3C, 0x338}}, {0x226F, {0x3E, 0x338}}, {0x2270, {0x2264, 0x338}}, {0x2271, {0x2265, 0x338}}, {0x2274, {0x2272, 0x338}}, {0x2275, {0x2273, 0x338}}, {0x2278, {0x2276, 0x338}}, +{0x2279, {0x2277, 0x338}}, {0x2280, {0x227A, 0x338}}, {0x2281, {0x227B, 0x338}}, {0x2284, {0x2282, 0x338}}, {0x2285, {0x2283, 0x338}}, {0x2288, {0x2286, 0x338}}, {0x2289, {0x2287, 0x338}}, +{0x22AC, {0x22A2, 0x338}}, {0x22AD, {0x22A8, 0x338}}, {0x22AE, {0x22A9, 0x338}}, {0x22AF, {0x22AB, 0x338}}, {0x22E0, {0x227C, 0x338}}, {0x22E1, {0x227D, 0x338}}, {0x22E2, {0x2291, 0x338}}, +{0x22E3, {0x2292, 0x338}}, {0x22EA, {0x22B2, 0x338}}, {0x22EB, {0x22B3, 0x338}}, {0x22EC, {0x22B4, 0x338}}, {0x22ED, {0x22B5, 0x338}}, {0x2329, {0x3008}}, {0x232A, {0x3009}}, +{0x2ADC, {0x2ADD, 0x338}}, {0x304C, {0x304B, 0x3099}}, {0x304E, {0x304D, 0x3099}}, {0x3050, {0x304F, 0x3099}}, {0x3052, {0x3051, 0x3099}}, {0x3054, {0x3053, 0x3099}}, {0x3056, {0x3055, 0x3099}}, +{0x3058, {0x3057, 0x3099}}, {0x305A, {0x3059, 0x3099}}, {0x305C, {0x305B, 0x3099}}, {0x305E, {0x305D, 0x3099}}, {0x3060, {0x305F, 0x3099}}, {0x3062, {0x3061, 0x3099}}, {0x3065, {0x3064, 0x3099}}, +{0x3067, {0x3066, 0x3099}}, {0x3069, {0x3068, 0x3099}}, {0x3070, {0x306F, 0x3099}}, {0x3071, {0x306F, 0x309A}}, {0x3073, {0x3072, 0x3099}}, {0x3074, {0x3072, 0x309A}}, {0x3076, {0x3075, 0x3099}}, +{0x3077, {0x3075, 0x309A}}, {0x3079, {0x3078, 0x3099}}, {0x307A, {0x3078, 0x309A}}, {0x307C, {0x307B, 0x3099}}, {0x307D, {0x307B, 0x309A}}, {0x3094, {0x3046, 0x3099}}, {0x309E, {0x309D, 0x3099}}, +{0x30AC, {0x30AB, 0x3099}}, {0x30AE, {0x30AD, 0x3099}}, {0x30B0, {0x30AF, 0x3099}}, {0x30B2, {0x30B1, 0x3099}}, {0x30B4, {0x30B3, 0x3099}}, {0x30B6, {0x30B5, 0x3099}}, {0x30B8, {0x30B7, 0x3099}}, +{0x30BA, {0x30B9, 0x3099}}, {0x30BC, {0x30BB, 0x3099}}, {0x30BE, {0x30BD, 0x3099}}, {0x30C0, {0x30BF, 0x3099}}, {0x30C2, {0x30C1, 0x3099}}, {0x30C5, {0x30C4, 0x3099}}, {0x30C7, {0x30C6, 0x3099}}, +{0x30C9, {0x30C8, 0x3099}}, {0x30D0, {0x30CF, 0x3099}}, {0x30D1, {0x30CF, 0x309A}}, {0x30D3, {0x30D2, 0x3099}}, {0x30D4, {0x30D2, 0x309A}}, {0x30D6, {0x30D5, 0x3099}}, {0x30D7, {0x30D5, 0x309A}}, +{0x30D9, {0x30D8, 0x3099}}, {0x30DA, {0x30D8, 0x309A}}, {0x30DC, {0x30DB, 0x3099}}, {0x30DD, {0x30DB, 0x309A}}, {0x30F4, {0x30A6, 0x3099}}, {0x30F7, {0x30EF, 0x3099}}, {0x30F8, {0x30F0, 0x3099}}, +{0x30F9, {0x30F1, 0x3099}}, {0x30FA, {0x30F2, 0x3099}}, {0x30FE, {0x30FD, 0x3099}}, {0xF900, {0x8C48}}, {0xF901, {0x66F4}}, {0xF902, {0x8ECA}}, {0xF903, {0x8CC8}}, {0xF904, {0x6ED1}}, +{0xF905, {0x4E32}}, {0xF906, {0x53E5}}, {0xF907, {0x9F9C}}, {0xF908, {0x9F9C}}, {0xF909, {0x5951}}, {0xF90A, {0x91D1}}, {0xF90B, {0x5587}}, {0xF90C, {0x5948}}, {0xF90D, {0x61F6}}, {0xF90E, {0x7669}}, +{0xF90F, {0x7F85}}, {0xF910, {0x863F}}, {0xF911, {0x87BA}}, {0xF912, {0x88F8}}, {0xF913, {0x908F}}, {0xF914, {0x6A02}}, {0xF915, {0x6D1B}}, {0xF916, {0x70D9}}, {0xF917, {0x73DE}}, {0xF918, {0x843D}}, +{0xF919, {0x916A}}, {0xF91A, {0x99F1}}, {0xF91B, {0x4E82}}, {0xF91C, {0x5375}}, {0xF91D, {0x6B04}}, {0xF91E, {0x721B}}, {0xF91F, {0x862D}}, {0xF920, {0x9E1E}}, {0xF921, {0x5D50}}, {0xF922, {0x6FEB}}, +{0xF923, {0x85CD}}, {0xF924, {0x8964}}, {0xF925, {0x62C9}}, {0xF926, {0x81D8}}, {0xF927, {0x881F}}, {0xF928, {0x5ECA}}, {0xF929, {0x6717}}, {0xF92A, {0x6D6A}}, {0xF92B, {0x72FC}}, {0xF92C, {0x90CE}}, +{0xF92D, {0x4F86}}, {0xF92E, {0x51B7}}, {0xF92F, {0x52DE}}, {0xF930, {0x64C4}}, {0xF931, {0x6AD3}}, {0xF932, {0x7210}}, {0xF933, {0x76E7}}, {0xF934, {0x8001}}, {0xF935, {0x8606}}, {0xF936, {0x865C}}, +{0xF937, {0x8DEF}}, {0xF938, {0x9732}}, {0xF939, {0x9B6F}}, {0xF93A, {0x9DFA}}, {0xF93B, {0x788C}}, {0xF93C, {0x797F}}, {0xF93D, {0x7DA0}}, {0xF93E, {0x83C9}}, {0xF93F, {0x9304}}, {0xF940, {0x9E7F}}, +{0xF941, {0x8AD6}}, {0xF942, {0x58DF}}, {0xF943, {0x5F04}}, {0xF944, {0x7C60}}, {0xF945, {0x807E}}, {0xF946, {0x7262}}, {0xF947, {0x78CA}}, {0xF948, {0x8CC2}}, {0xF949, {0x96F7}}, {0xF94A, {0x58D8}}, +{0xF94B, {0x5C62}}, {0xF94C, {0x6A13}}, {0xF94D, {0x6DDA}}, {0xF94E, {0x6F0F}}, {0xF94F, {0x7D2F}}, {0xF950, {0x7E37}}, {0xF951, {0x964B}}, {0xF952, {0x52D2}}, {0xF953, {0x808B}}, {0xF954, {0x51DC}}, +{0xF955, {0x51CC}}, {0xF956, {0x7A1C}}, {0xF957, {0x7DBE}}, {0xF958, {0x83F1}}, {0xF959, {0x9675}}, {0xF95A, {0x8B80}}, {0xF95B, {0x62CF}}, {0xF95C, {0x6A02}}, {0xF95D, {0x8AFE}}, {0xF95E, {0x4E39}}, +{0xF95F, {0x5BE7}}, {0xF960, {0x6012}}, {0xF961, {0x7387}}, {0xF962, {0x7570}}, {0xF963, {0x5317}}, {0xF964, {0x78FB}}, {0xF965, {0x4FBF}}, {0xF966, {0x5FA9}}, {0xF967, {0x4E0D}}, {0xF968, {0x6CCC}}, +{0xF969, {0x6578}}, {0xF96A, {0x7D22}}, {0xF96B, {0x53C3}}, {0xF96C, {0x585E}}, {0xF96D, {0x7701}}, {0xF96E, {0x8449}}, {0xF96F, {0x8AAA}}, {0xF970, {0x6BBA}}, {0xF971, {0x8FB0}}, {0xF972, {0x6C88}}, +{0xF973, {0x62FE}}, {0xF974, {0x82E5}}, {0xF975, {0x63A0}}, {0xF976, {0x7565}}, {0xF977, {0x4EAE}}, {0xF978, {0x5169}}, {0xF979, {0x51C9}}, {0xF97A, {0x6881}}, {0xF97B, {0x7CE7}}, {0xF97C, {0x826F}}, +{0xF97D, {0x8AD2}}, {0xF97E, {0x91CF}}, {0xF97F, {0x52F5}}, {0xF980, {0x5442}}, {0xF981, {0x5973}}, {0xF982, {0x5EEC}}, {0xF983, {0x65C5}}, {0xF984, {0x6FFE}}, {0xF985, {0x792A}}, {0xF986, {0x95AD}}, +{0xF987, {0x9A6A}}, {0xF988, {0x9E97}}, {0xF989, {0x9ECE}}, {0xF98A, {0x529B}}, {0xF98B, {0x66C6}}, {0xF98C, {0x6B77}}, {0xF98D, {0x8F62}}, {0xF98E, {0x5E74}}, {0xF98F, {0x6190}}, {0xF990, {0x6200}}, +{0xF991, {0x649A}}, {0xF992, {0x6F23}}, {0xF993, {0x7149}}, {0xF994, {0x7489}}, {0xF995, {0x79CA}}, {0xF996, {0x7DF4}}, {0xF997, {0x806F}}, {0xF998, {0x8F26}}, {0xF999, {0x84EE}}, {0xF99A, {0x9023}}, +{0xF99B, {0x934A}}, {0xF99C, {0x5217}}, {0xF99D, {0x52A3}}, {0xF99E, {0x54BD}}, {0xF99F, {0x70C8}}, {0xF9A0, {0x88C2}}, {0xF9A1, {0x8AAA}}, {0xF9A2, {0x5EC9}}, {0xF9A3, {0x5FF5}}, {0xF9A4, {0x637B}}, +{0xF9A5, {0x6BAE}}, {0xF9A6, {0x7C3E}}, {0xF9A7, {0x7375}}, {0xF9A8, {0x4EE4}}, {0xF9A9, {0x56F9}}, {0xF9AA, {0x5BE7}}, {0xF9AB, {0x5DBA}}, {0xF9AC, {0x601C}}, {0xF9AD, {0x73B2}}, {0xF9AE, {0x7469}}, +{0xF9AF, {0x7F9A}}, {0xF9B0, {0x8046}}, {0xF9B1, {0x9234}}, {0xF9B2, {0x96F6}}, {0xF9B3, {0x9748}}, {0xF9B4, {0x9818}}, {0xF9B5, {0x4F8B}}, {0xF9B6, {0x79AE}}, {0xF9B7, {0x91B4}}, {0xF9B8, {0x96B8}}, +{0xF9B9, {0x60E1}}, {0xF9BA, {0x4E86}}, {0xF9BB, {0x50DA}}, {0xF9BC, {0x5BEE}}, {0xF9BD, {0x5C3F}}, {0xF9BE, {0x6599}}, {0xF9BF, {0x6A02}}, {0xF9C0, {0x71CE}}, {0xF9C1, {0x7642}}, {0xF9C2, {0x84FC}}, +{0xF9C3, {0x907C}}, {0xF9C4, {0x9F8D}}, {0xF9C5, {0x6688}}, {0xF9C6, {0x962E}}, {0xF9C7, {0x5289}}, {0xF9C8, {0x677B}}, {0xF9C9, {0x67F3}}, {0xF9CA, {0x6D41}}, {0xF9CB, {0x6E9C}}, {0xF9CC, {0x7409}}, +{0xF9CD, {0x7559}}, {0xF9CE, {0x786B}}, {0xF9CF, {0x7D10}}, {0xF9D0, {0x985E}}, {0xF9D1, {0x516D}}, {0xF9D2, {0x622E}}, {0xF9D3, {0x9678}}, {0xF9D4, {0x502B}}, {0xF9D5, {0x5D19}}, {0xF9D6, {0x6DEA}}, +{0xF9D7, {0x8F2A}}, {0xF9D8, {0x5F8B}}, {0xF9D9, {0x6144}}, {0xF9DA, {0x6817}}, {0xF9DB, {0x7387}}, {0xF9DC, {0x9686}}, {0xF9DD, {0x5229}}, {0xF9DE, {0x540F}}, {0xF9DF, {0x5C65}}, {0xF9E0, {0x6613}}, +{0xF9E1, {0x674E}}, {0xF9E2, {0x68A8}}, {0xF9E3, {0x6CE5}}, {0xF9E4, {0x7406}}, {0xF9E5, {0x75E2}}, {0xF9E6, {0x7F79}}, {0xF9E7, {0x88CF}}, {0xF9E8, {0x88E1}}, {0xF9E9, {0x91CC}}, {0xF9EA, {0x96E2}}, +{0xF9EB, {0x533F}}, {0xF9EC, {0x6EBA}}, {0xF9ED, {0x541D}}, {0xF9EE, {0x71D0}}, {0xF9EF, {0x7498}}, {0xF9F0, {0x85FA}}, {0xF9F1, {0x96A3}}, {0xF9F2, {0x9C57}}, {0xF9F3, {0x9E9F}}, {0xF9F4, {0x6797}}, +{0xF9F5, {0x6DCB}}, {0xF9F6, {0x81E8}}, {0xF9F7, {0x7ACB}}, {0xF9F8, {0x7B20}}, {0xF9F9, {0x7C92}}, {0xF9FA, {0x72C0}}, {0xF9FB, {0x7099}}, {0xF9FC, {0x8B58}}, {0xF9FD, {0x4EC0}}, {0xF9FE, {0x8336}}, +{0xF9FF, {0x523A}}, {0xFA00, {0x5207}}, {0xFA01, {0x5EA6}}, {0xFA02, {0x62D3}}, {0xFA03, {0x7CD6}}, {0xFA04, {0x5B85}}, {0xFA05, {0x6D1E}}, {0xFA06, {0x66B4}}, {0xFA07, {0x8F3B}}, {0xFA08, {0x884C}}, +{0xFA09, {0x964D}}, {0xFA0A, {0x898B}}, {0xFA0B, {0x5ED3}}, {0xFA0C, {0x5140}}, {0xFA0D, {0x55C0}}, {0xFA10, {0x585A}}, {0xFA12, {0x6674}}, {0xFA15, {0x51DE}}, {0xFA16, {0x732A}}, {0xFA17, {0x76CA}}, +{0xFA18, {0x793C}}, {0xFA19, {0x795E}}, {0xFA1A, {0x7965}}, {0xFA1B, {0x798F}}, {0xFA1C, {0x9756}}, {0xFA1D, {0x7CBE}}, {0xFA1E, {0x7FBD}}, {0xFA20, {0x8612}}, {0xFA22, {0x8AF8}}, {0xFA25, {0x9038}}, +{0xFA26, {0x90FD}}, {0xFA2A, {0x98EF}}, {0xFA2B, {0x98FC}}, {0xFA2C, {0x9928}}, {0xFA2D, {0x9DB4}}, {0xFA2E, {0x90DE}}, {0xFA2F, {0x96B7}}, {0xFA30, {0x4FAE}}, {0xFA31, {0x50E7}}, {0xFA32, {0x514D}}, +{0xFA33, {0x52C9}}, {0xFA34, {0x52E4}}, {0xFA35, {0x5351}}, {0xFA36, {0x559D}}, {0xFA37, {0x5606}}, {0xFA38, {0x5668}}, {0xFA39, {0x5840}}, {0xFA3A, {0x58A8}}, {0xFA3B, {0x5C64}}, {0xFA3C, {0x5C6E}}, +{0xFA3D, {0x6094}}, {0xFA3E, {0x6168}}, {0xFA3F, {0x618E}}, {0xFA40, {0x61F2}}, {0xFA41, {0x654F}}, {0xFA42, {0x65E2}}, {0xFA43, {0x6691}}, {0xFA44, {0x6885}}, {0xFA45, {0x6D77}}, {0xFA46, {0x6E1A}}, +{0xFA47, {0x6F22}}, {0xFA48, {0x716E}}, {0xFA49, {0x722B}}, {0xFA4A, {0x7422}}, {0xFA4B, {0x7891}}, {0xFA4C, {0x793E}}, {0xFA4D, {0x7949}}, {0xFA4E, {0x7948}}, {0xFA4F, {0x7950}}, {0xFA50, {0x7956}}, +{0xFA51, {0x795D}}, {0xFA52, {0x798D}}, {0xFA53, {0x798E}}, {0xFA54, {0x7A40}}, {0xFA55, {0x7A81}}, {0xFA56, {0x7BC0}}, {0xFA57, {0x7DF4}}, {0xFA58, {0x7E09}}, {0xFA59, {0x7E41}}, {0xFA5A, {0x7F72}}, +{0xFA5B, {0x8005}}, {0xFA5C, {0x81ED}}, {0xFA5D, {0x8279}}, {0xFA5E, {0x8279}}, {0xFA5F, {0x8457}}, {0xFA60, {0x8910}}, {0xFA61, {0x8996}}, {0xFA62, {0x8B01}}, {0xFA63, {0x8B39}}, {0xFA64, {0x8CD3}}, +{0xFA65, {0x8D08}}, {0xFA66, {0x8FB6}}, {0xFA67, {0x9038}}, {0xFA68, {0x96E3}}, {0xFA69, {0x97FF}}, {0xFA6A, {0x983B}}, {0xFA6B, {0x6075}}, {0xFA6C, {0x242EE}}, {0xFA6D, {0x8218}}, {0xFA70, {0x4E26}}, +{0xFA71, {0x51B5}}, {0xFA72, {0x5168}}, {0xFA73, {0x4F80}}, {0xFA74, {0x5145}}, {0xFA75, {0x5180}}, {0xFA76, {0x52C7}}, {0xFA77, {0x52FA}}, {0xFA78, {0x559D}}, {0xFA79, {0x5555}}, {0xFA7A, {0x5599}}, +{0xFA7B, {0x55E2}}, {0xFA7C, {0x585A}}, {0xFA7D, {0x58B3}}, {0xFA7E, {0x5944}}, {0xFA7F, {0x5954}}, {0xFA80, {0x5A62}}, {0xFA81, {0x5B28}}, {0xFA82, {0x5ED2}}, {0xFA83, {0x5ED9}}, {0xFA84, {0x5F69}}, +{0xFA85, {0x5FAD}}, {0xFA86, {0x60D8}}, {0xFA87, {0x614E}}, {0xFA88, {0x6108}}, {0xFA89, {0x618E}}, {0xFA8A, {0x6160}}, {0xFA8B, {0x61F2}}, {0xFA8C, {0x6234}}, {0xFA8D, {0x63C4}}, {0xFA8E, {0x641C}}, +{0xFA8F, {0x6452}}, {0xFA90, {0x6556}}, {0xFA91, {0x6674}}, {0xFA92, {0x6717}}, {0xFA93, {0x671B}}, {0xFA94, {0x6756}}, {0xFA95, {0x6B79}}, {0xFA96, {0x6BBA}}, {0xFA97, {0x6D41}}, {0xFA98, {0x6EDB}}, +{0xFA99, {0x6ECB}}, {0xFA9A, {0x6F22}}, {0xFA9B, {0x701E}}, {0xFA9C, {0x716E}}, {0xFA9D, {0x77A7}}, {0xFA9E, {0x7235}}, {0xFA9F, {0x72AF}}, {0xFAA0, {0x732A}}, {0xFAA1, {0x7471}}, {0xFAA2, {0x7506}}, +{0xFAA3, {0x753B}}, {0xFAA4, {0x761D}}, {0xFAA5, {0x761F}}, {0xFAA6, {0x76CA}}, {0xFAA7, {0x76DB}}, {0xFAA8, {0x76F4}}, {0xFAA9, {0x774A}}, {0xFAAA, {0x7740}}, {0xFAAB, {0x78CC}}, {0xFAAC, {0x7AB1}}, +{0xFAAD, {0x7BC0}}, {0xFAAE, {0x7C7B}}, {0xFAAF, {0x7D5B}}, {0xFAB0, {0x7DF4}}, {0xFAB1, {0x7F3E}}, {0xFAB2, {0x8005}}, {0xFAB3, {0x8352}}, {0xFAB4, {0x83EF}}, {0xFAB5, {0x8779}}, {0xFAB6, {0x8941}}, +{0xFAB7, {0x8986}}, {0xFAB8, {0x8996}}, {0xFAB9, {0x8ABF}}, {0xFABA, {0x8AF8}}, {0xFABB, {0x8ACB}}, {0xFABC, {0x8B01}}, {0xFABD, {0x8AFE}}, {0xFABE, {0x8AED}}, {0xFABF, {0x8B39}}, {0xFAC0, {0x8B8A}}, +{0xFAC1, {0x8D08}}, {0xFAC2, {0x8F38}}, {0xFAC3, {0x9072}}, {0xFAC4, {0x9199}}, {0xFAC5, {0x9276}}, {0xFAC6, {0x967C}}, {0xFAC7, {0x96E3}}, {0xFAC8, {0x9756}}, {0xFAC9, {0x97DB}}, {0xFACA, {0x97FF}}, +{0xFACB, {0x980B}}, {0xFACC, {0x983B}}, {0xFACD, {0x9B12}}, {0xFACE, {0x9F9C}}, {0xFACF, {0x2284A}}, {0xFAD0, {0x22844}}, {0xFAD1, {0x233D5}}, {0xFAD2, {0x3B9D}}, {0xFAD3, {0x4018}}, +{0xFAD4, {0x4039}}, {0xFAD5, {0x25249}}, {0xFAD6, {0x25CD0}}, {0xFAD7, {0x27ED3}}, {0xFAD8, {0x9F43}}, {0xFAD9, {0x9F8E}}, {0xFB1D, {0x5D9, 0x5B4}}, {0xFB1F, {0x5F2, 0x5B7}}, {0xFB2A, {0x5E9, 0x5C1}}, +{0xFB2B, {0x5E9, 0x5C2}}, {0xFB2C, {0x5E9, 0x5BC, 0x5C1}}, {0xFB2D, {0x5E9, 0x5BC, 0x5C2}}, {0xFB2E, {0x5D0, 0x5B7}}, {0xFB2F, {0x5D0, 0x5B8}}, {0xFB30, {0x5D0, 0x5BC}}, {0xFB31, {0x5D1, 0x5BC}}, +{0xFB32, {0x5D2, 0x5BC}}, {0xFB33, {0x5D3, 0x5BC}}, {0xFB34, {0x5D4, 0x5BC}}, {0xFB35, {0x5D5, 0x5BC}}, {0xFB36, {0x5D6, 0x5BC}}, {0xFB38, {0x5D8, 0x5BC}}, {0xFB39, {0x5D9, 0x5BC}}, +{0xFB3A, {0x5DA, 0x5BC}}, {0xFB3B, {0x5DB, 0x5BC}}, {0xFB3C, {0x5DC, 0x5BC}}, {0xFB3E, {0x5DE, 0x5BC}}, {0xFB40, {0x5E0, 0x5BC}}, {0xFB41, {0x5E1, 0x5BC}}, {0xFB43, {0x5E3, 0x5BC}}, +{0xFB44, {0x5E4, 0x5BC}}, {0xFB46, {0x5E6, 0x5BC}}, {0xFB47, {0x5E7, 0x5BC}}, {0xFB48, {0x5E8, 0x5BC}}, {0xFB49, {0x5E9, 0x5BC}}, {0xFB4A, {0x5EA, 0x5BC}}, {0xFB4B, {0x5D5, 0x5B9}}, +{0xFB4C, {0x5D1, 0x5BF}}, {0xFB4D, {0x5DB, 0x5BF}}, {0xFB4E, {0x5E4, 0x5BF}}, {0x1109A, {0x11099, 0x110BA}}, {0x1109C, {0x1109B, 0x110BA}}, {0x110AB, {0x110A5, 0x110BA}}, +{0x1112E, {0x11131, 0x11127}}, {0x1112F, {0x11132, 0x11127}}, {0x1134B, {0x11347, 0x1133E}}, {0x1134C, {0x11347, 0x11357}}, {0x114BB, {0x114B9, 0x114BA}}, {0x114BC, {0x114B9, 0x114B0}}, +{0x114BE, {0x114B9, 0x114BD}}, {0x115BA, {0x115B8, 0x115AF}}, {0x115BB, {0x115B9, 0x115AF}}, {0x1D15E, {0x1D157, 0x1D165}}, {0x1D15F, {0x1D158, 0x1D165}}, {0x1D160, {0x1D158, 0x1D165, 0x1D16E}}, +{0x1D161, {0x1D158, 0x1D165, 0x1D16F}}, {0x1D162, {0x1D158, 0x1D165, 0x1D170}}, {0x1D163, {0x1D158, 0x1D165, 0x1D171}}, {0x1D164, {0x1D158, 0x1D165, 0x1D172}}, {0x1D1BB, {0x1D1B9, 0x1D165}}, +{0x1D1BC, {0x1D1BA, 0x1D165}}, {0x1D1BD, {0x1D1B9, 0x1D165, 0x1D16E}}, {0x1D1BE, {0x1D1BA, 0x1D165, 0x1D16E}}, {0x1D1BF, {0x1D1B9, 0x1D165, 0x1D16F}}, {0x1D1C0, {0x1D1BA, 0x1D165, 0x1D16F}}, +{0x2F800, {0x4E3D}}, {0x2F801, {0x4E38}}, {0x2F802, {0x4E41}}, {0x2F803, {0x20122}}, {0x2F804, {0x4F60}}, {0x2F805, {0x4FAE}}, {0x2F806, {0x4FBB}}, {0x2F807, {0x5002}}, {0x2F808, {0x507A}}, +{0x2F809, {0x5099}}, {0x2F80A, {0x50E7}}, {0x2F80B, {0x50CF}}, {0x2F80C, {0x349E}}, {0x2F80D, {0x2063A}}, {0x2F80E, {0x514D}}, {0x2F80F, {0x5154}}, {0x2F810, {0x5164}}, {0x2F811, {0x5177}}, +{0x2F812, {0x2051C}}, {0x2F813, {0x34B9}}, {0x2F814, {0x5167}}, {0x2F815, {0x518D}}, {0x2F816, {0x2054B}}, {0x2F817, {0x5197}}, {0x2F818, {0x51A4}}, {0x2F819, {0x4ECC}}, {0x2F81A, {0x51AC}}, +{0x2F81B, {0x51B5}}, {0x2F81C, {0x291DF}}, {0x2F81D, {0x51F5}}, {0x2F81E, {0x5203}}, {0x2F81F, {0x34DF}}, {0x2F820, {0x523B}}, {0x2F821, {0x5246}}, {0x2F822, {0x5272}}, {0x2F823, {0x5277}}, +{0x2F824, {0x3515}}, {0x2F825, {0x52C7}}, {0x2F826, {0x52C9}}, {0x2F827, {0x52E4}}, {0x2F828, {0x52FA}}, {0x2F829, {0x5305}}, {0x2F82A, {0x5306}}, {0x2F82B, {0x5317}}, {0x2F82C, {0x5349}}, +{0x2F82D, {0x5351}}, {0x2F82E, {0x535A}}, {0x2F82F, {0x5373}}, {0x2F830, {0x537D}}, {0x2F831, {0x537F}}, {0x2F832, {0x537F}}, {0x2F833, {0x537F}}, {0x2F834, {0x20A2C}}, {0x2F835, {0x7070}}, +{0x2F836, {0x53CA}}, {0x2F837, {0x53DF}}, {0x2F838, {0x20B63}}, {0x2F839, {0x53EB}}, {0x2F83A, {0x53F1}}, {0x2F83B, {0x5406}}, {0x2F83C, {0x549E}}, {0x2F83D, {0x5438}}, {0x2F83E, {0x5448}}, +{0x2F83F, {0x5468}}, {0x2F840, {0x54A2}}, {0x2F841, {0x54F6}}, {0x2F842, {0x5510}}, {0x2F843, {0x5553}}, {0x2F844, {0x5563}}, {0x2F845, {0x5584}}, {0x2F846, {0x5584}}, {0x2F847, {0x5599}}, +{0x2F848, {0x55AB}}, {0x2F849, {0x55B3}}, {0x2F84A, {0x55C2}}, {0x2F84B, {0x5716}}, {0x2F84C, {0x5606}}, {0x2F84D, {0x5717}}, {0x2F84E, {0x5651}}, {0x2F84F, {0x5674}}, {0x2F850, {0x5207}}, +{0x2F851, {0x58EE}}, {0x2F852, {0x57CE}}, {0x2F853, {0x57F4}}, {0x2F854, {0x580D}}, {0x2F855, {0x578B}}, {0x2F856, {0x5832}}, {0x2F857, {0x5831}}, {0x2F858, {0x58AC}}, {0x2F859, {0x214E4}}, +{0x2F85A, {0x58F2}}, {0x2F85B, {0x58F7}}, {0x2F85C, {0x5906}}, {0x2F85D, {0x591A}}, {0x2F85E, {0x5922}}, {0x2F85F, {0x5962}}, {0x2F860, {0x216A8}}, {0x2F861, {0x216EA}}, {0x2F862, {0x59EC}}, +{0x2F863, {0x5A1B}}, {0x2F864, {0x5A27}}, {0x2F865, {0x59D8}}, {0x2F866, {0x5A66}}, {0x2F867, {0x36EE}}, {0x2F868, {0x36FC}}, {0x2F869, {0x5B08}}, {0x2F86A, {0x5B3E}}, {0x2F86B, {0x5B3E}}, +{0x2F86C, {0x219C8}}, {0x2F86D, {0x5BC3}}, {0x2F86E, {0x5BD8}}, {0x2F86F, {0x5BE7}}, {0x2F870, {0x5BF3}}, {0x2F871, {0x21B18}}, {0x2F872, {0x5BFF}}, {0x2F873, {0x5C06}}, {0x2F874, {0x5F53}}, +{0x2F875, {0x5C22}}, {0x2F876, {0x3781}}, {0x2F877, {0x5C60}}, {0x2F878, {0x5C6E}}, {0x2F879, {0x5CC0}}, {0x2F87A, {0x5C8D}}, {0x2F87B, {0x21DE4}}, {0x2F87C, {0x5D43}}, {0x2F87D, {0x21DE6}}, +{0x2F87E, {0x5D6E}}, {0x2F87F, {0x5D6B}}, {0x2F880, {0x5D7C}}, {0x2F881, {0x5DE1}}, {0x2F882, {0x5DE2}}, {0x2F883, {0x382F}}, {0x2F884, {0x5DFD}}, {0x2F885, {0x5E28}}, {0x2F886, {0x5E3D}}, +{0x2F887, {0x5E69}}, {0x2F888, {0x3862}}, {0x2F889, {0x22183}}, {0x2F88A, {0x387C}}, {0x2F88B, {0x5EB0}}, {0x2F88C, {0x5EB3}}, {0x2F88D, {0x5EB6}}, {0x2F88E, {0x5ECA}}, {0x2F88F, {0x2A392}}, +{0x2F890, {0x5EFE}}, {0x2F891, {0x22331}}, {0x2F892, {0x22331}}, {0x2F893, {0x8201}}, {0x2F894, {0x5F22}}, {0x2F895, {0x5F22}}, {0x2F896, {0x38C7}}, {0x2F897, {0x232B8}}, {0x2F898, {0x261DA}}, +{0x2F899, {0x5F62}}, {0x2F89A, {0x5F6B}}, {0x2F89B, {0x38E3}}, {0x2F89C, {0x5F9A}}, {0x2F89D, {0x5FCD}}, {0x2F89E, {0x5FD7}}, {0x2F89F, {0x5FF9}}, {0x2F8A0, {0x6081}}, {0x2F8A1, {0x393A}}, +{0x2F8A2, {0x391C}}, {0x2F8A3, {0x6094}}, {0x2F8A4, {0x226D4}}, {0x2F8A5, {0x60C7}}, {0x2F8A6, {0x6148}}, {0x2F8A7, {0x614C}}, {0x2F8A8, {0x614E}}, {0x2F8A9, {0x614C}}, {0x2F8AA, {0x617A}}, +{0x2F8AB, {0x618E}}, {0x2F8AC, {0x61B2}}, {0x2F8AD, {0x61A4}}, {0x2F8AE, {0x61AF}}, {0x2F8AF, {0x61DE}}, {0x2F8B0, {0x61F2}}, {0x2F8B1, {0x61F6}}, {0x2F8B2, {0x6210}}, {0x2F8B3, {0x621B}}, +{0x2F8B4, {0x625D}}, {0x2F8B5, {0x62B1}}, {0x2F8B6, {0x62D4}}, {0x2F8B7, {0x6350}}, {0x2F8B8, {0x22B0C}}, {0x2F8B9, {0x633D}}, {0x2F8BA, {0x62FC}}, {0x2F8BB, {0x6368}}, {0x2F8BC, {0x6383}}, +{0x2F8BD, {0x63E4}}, {0x2F8BE, {0x22BF1}}, {0x2F8BF, {0x6422}}, {0x2F8C0, {0x63C5}}, {0x2F8C1, {0x63A9}}, {0x2F8C2, {0x3A2E}}, {0x2F8C3, {0x6469}}, {0x2F8C4, {0x647E}}, {0x2F8C5, {0x649D}}, +{0x2F8C6, {0x6477}}, {0x2F8C7, {0x3A6C}}, {0x2F8C8, {0x654F}}, {0x2F8C9, {0x656C}}, {0x2F8CA, {0x2300A}}, {0x2F8CB, {0x65E3}}, {0x2F8CC, {0x66F8}}, {0x2F8CD, {0x6649}}, {0x2F8CE, {0x3B19}}, +{0x2F8CF, {0x6691}}, {0x2F8D0, {0x3B08}}, {0x2F8D1, {0x3AE4}}, {0x2F8D2, {0x5192}}, {0x2F8D3, {0x5195}}, {0x2F8D4, {0x6700}}, {0x2F8D5, {0x669C}}, {0x2F8D6, {0x80AD}}, {0x2F8D7, {0x43D9}}, +{0x2F8D8, {0x6717}}, {0x2F8D9, {0x671B}}, {0x2F8DA, {0x6721}}, {0x2F8DB, {0x675E}}, {0x2F8DC, {0x6753}}, {0x2F8DD, {0x233C3}}, {0x2F8DE, {0x3B49}}, {0x2F8DF, {0x67FA}}, {0x2F8E0, {0x6785}}, +{0x2F8E1, {0x6852}}, {0x2F8E2, {0x6885}}, {0x2F8E3, {0x2346D}}, {0x2F8E4, {0x688E}}, {0x2F8E5, {0x681F}}, {0x2F8E6, {0x6914}}, {0x2F8E7, {0x3B9D}}, {0x2F8E8, {0x6942}}, {0x2F8E9, {0x69A3}}, +{0x2F8EA, {0x69EA}}, {0x2F8EB, {0x6AA8}}, {0x2F8EC, {0x236A3}}, {0x2F8ED, {0x6ADB}}, {0x2F8EE, {0x3C18}}, {0x2F8EF, {0x6B21}}, {0x2F8F0, {0x238A7}}, {0x2F8F1, {0x6B54}}, {0x2F8F2, {0x3C4E}}, +{0x2F8F3, {0x6B72}}, {0x2F8F4, {0x6B9F}}, {0x2F8F5, {0x6BBA}}, {0x2F8F6, {0x6BBB}}, {0x2F8F7, {0x23A8D}}, {0x2F8F8, {0x21D0B}}, {0x2F8F9, {0x23AFA}}, {0x2F8FA, {0x6C4E}}, {0x2F8FB, {0x23CBC}}, +{0x2F8FC, {0x6CBF}}, {0x2F8FD, {0x6CCD}}, {0x2F8FE, {0x6C67}}, {0x2F8FF, {0x6D16}}, {0x2F900, {0x6D3E}}, {0x2F901, {0x6D77}}, {0x2F902, {0x6D41}}, {0x2F903, {0x6D69}}, {0x2F904, {0x6D78}}, +{0x2F905, {0x6D85}}, {0x2F906, {0x23D1E}}, {0x2F907, {0x6D34}}, {0x2F908, {0x6E2F}}, {0x2F909, {0x6E6E}}, {0x2F90A, {0x3D33}}, {0x2F90B, {0x6ECB}}, {0x2F90C, {0x6EC7}}, {0x2F90D, {0x23ED1}}, +{0x2F90E, {0x6DF9}}, {0x2F90F, {0x6F6E}}, {0x2F910, {0x23F5E}}, {0x2F911, {0x23F8E}}, {0x2F912, {0x6FC6}}, {0x2F913, {0x7039}}, {0x2F914, {0x701E}}, {0x2F915, {0x701B}}, {0x2F916, {0x3D96}}, +{0x2F917, {0x704A}}, {0x2F918, {0x707D}}, {0x2F919, {0x7077}}, {0x2F91A, {0x70AD}}, {0x2F91B, {0x20525}}, {0x2F91C, {0x7145}}, {0x2F91D, {0x24263}}, {0x2F91E, {0x719C}}, {0x2F91F, {0x243AB}}, +{0x2F920, {0x7228}}, {0x2F921, {0x7235}}, {0x2F922, {0x7250}}, {0x2F923, {0x24608}}, {0x2F924, {0x7280}}, {0x2F925, {0x7295}}, {0x2F926, {0x24735}}, {0x2F927, {0x24814}}, {0x2F928, {0x737A}}, +{0x2F929, {0x738B}}, {0x2F92A, {0x3EAC}}, {0x2F92B, {0x73A5}}, {0x2F92C, {0x3EB8}}, {0x2F92D, {0x3EB8}}, {0x2F92E, {0x7447}}, {0x2F92F, {0x745C}}, {0x2F930, {0x7471}}, {0x2F931, {0x7485}}, +{0x2F932, {0x74CA}}, {0x2F933, {0x3F1B}}, {0x2F934, {0x7524}}, {0x2F935, {0x24C36}}, {0x2F936, {0x753E}}, {0x2F937, {0x24C92}}, {0x2F938, {0x7570}}, {0x2F939, {0x2219F}}, {0x2F93A, {0x7610}}, +{0x2F93B, {0x24FA1}}, {0x2F93C, {0x24FB8}}, {0x2F93D, {0x25044}}, {0x2F93E, {0x3FFC}}, {0x2F93F, {0x4008}}, {0x2F940, {0x76F4}}, {0x2F941, {0x250F3}}, {0x2F942, {0x250F2}}, {0x2F943, {0x25119}}, +{0x2F944, {0x25133}}, {0x2F945, {0x771E}}, {0x2F946, {0x771F}}, {0x2F947, {0x771F}}, {0x2F948, {0x774A}}, {0x2F949, {0x4039}}, {0x2F94A, {0x778B}}, {0x2F94B, {0x4046}}, {0x2F94C, {0x4096}}, +{0x2F94D, {0x2541D}}, {0x2F94E, {0x784E}}, {0x2F94F, {0x788C}}, {0x2F950, {0x78CC}}, {0x2F951, {0x40E3}}, {0x2F952, {0x25626}}, {0x2F953, {0x7956}}, {0x2F954, {0x2569A}}, {0x2F955, {0x256C5}}, +{0x2F956, {0x798F}}, {0x2F957, {0x79EB}}, {0x2F958, {0x412F}}, {0x2F959, {0x7A40}}, {0x2F95A, {0x7A4A}}, {0x2F95B, {0x7A4F}}, {0x2F95C, {0x2597C}}, {0x2F95D, {0x25AA7}}, {0x2F95E, {0x25AA7}}, +{0x2F95F, {0x7AEE}}, {0x2F960, {0x4202}}, {0x2F961, {0x25BAB}}, {0x2F962, {0x7BC6}}, {0x2F963, {0x7BC9}}, {0x2F964, {0x4227}}, {0x2F965, {0x25C80}}, {0x2F966, {0x7CD2}}, {0x2F967, {0x42A0}}, +{0x2F968, {0x7CE8}}, {0x2F969, {0x7CE3}}, {0x2F96A, {0x7D00}}, {0x2F96B, {0x25F86}}, {0x2F96C, {0x7D63}}, {0x2F96D, {0x4301}}, {0x2F96E, {0x7DC7}}, {0x2F96F, {0x7E02}}, {0x2F970, {0x7E45}}, +{0x2F971, {0x4334}}, {0x2F972, {0x26228}}, {0x2F973, {0x26247}}, {0x2F974, {0x4359}}, {0x2F975, {0x262D9}}, {0x2F976, {0x7F7A}}, {0x2F977, {0x2633E}}, {0x2F978, {0x7F95}}, {0x2F979, {0x7FFA}}, +{0x2F97A, {0x8005}}, {0x2F97B, {0x264DA}}, {0x2F97C, {0x26523}}, {0x2F97D, {0x8060}}, {0x2F97E, {0x265A8}}, {0x2F97F, {0x8070}}, {0x2F980, {0x2335F}}, {0x2F981, {0x43D5}}, {0x2F982, {0x80B2}}, +{0x2F983, {0x8103}}, {0x2F984, {0x440B}}, {0x2F985, {0x813E}}, {0x2F986, {0x5AB5}}, {0x2F987, {0x267A7}}, {0x2F988, {0x267B5}}, {0x2F989, {0x23393}}, {0x2F98A, {0x2339C}}, {0x2F98B, {0x8201}}, +{0x2F98C, {0x8204}}, {0x2F98D, {0x8F9E}}, {0x2F98E, {0x446B}}, {0x2F98F, {0x8291}}, {0x2F990, {0x828B}}, {0x2F991, {0x829D}}, {0x2F992, {0x52B3}}, {0x2F993, {0x82B1}}, {0x2F994, {0x82B3}}, +{0x2F995, {0x82BD}}, {0x2F996, {0x82E6}}, {0x2F997, {0x26B3C}}, {0x2F998, {0x82E5}}, {0x2F999, {0x831D}}, {0x2F99A, {0x8363}}, {0x2F99B, {0x83AD}}, {0x2F99C, {0x8323}}, {0x2F99D, {0x83BD}}, +{0x2F99E, {0x83E7}}, {0x2F99F, {0x8457}}, {0x2F9A0, {0x8353}}, {0x2F9A1, {0x83CA}}, {0x2F9A2, {0x83CC}}, {0x2F9A3, {0x83DC}}, {0x2F9A4, {0x26C36}}, {0x2F9A5, {0x26D6B}}, {0x2F9A6, {0x26CD5}}, +{0x2F9A7, {0x452B}}, {0x2F9A8, {0x84F1}}, {0x2F9A9, {0x84F3}}, {0x2F9AA, {0x8516}}, {0x2F9AB, {0x273CA}}, {0x2F9AC, {0x8564}}, {0x2F9AD, {0x26F2C}}, {0x2F9AE, {0x455D}}, {0x2F9AF, {0x4561}}, +{0x2F9B0, {0x26FB1}}, {0x2F9B1, {0x270D2}}, {0x2F9B2, {0x456B}}, {0x2F9B3, {0x8650}}, {0x2F9B4, {0x865C}}, {0x2F9B5, {0x8667}}, {0x2F9B6, {0x8669}}, {0x2F9B7, {0x86A9}}, {0x2F9B8, {0x8688}}, +{0x2F9B9, {0x870E}}, {0x2F9BA, {0x86E2}}, {0x2F9BB, {0x8779}}, {0x2F9BC, {0x8728}}, {0x2F9BD, {0x876B}}, {0x2F9BE, {0x8786}}, {0x2F9BF, {0x45D7}}, {0x2F9C0, {0x87E1}}, {0x2F9C1, {0x8801}}, +{0x2F9C2, {0x45F9}}, {0x2F9C3, {0x8860}}, {0x2F9C4, {0x8863}}, {0x2F9C5, {0x27667}}, {0x2F9C6, {0x88D7}}, {0x2F9C7, {0x88DE}}, {0x2F9C8, {0x4635}}, {0x2F9C9, {0x88FA}}, {0x2F9CA, {0x34BB}}, +{0x2F9CB, {0x278AE}}, {0x2F9CC, {0x27966}}, {0x2F9CD, {0x46BE}}, {0x2F9CE, {0x46C7}}, {0x2F9CF, {0x8AA0}}, {0x2F9D0, {0x8AED}}, {0x2F9D1, {0x8B8A}}, {0x2F9D2, {0x8C55}}, {0x2F9D3, {0x27CA8}}, +{0x2F9D4, {0x8CAB}}, {0x2F9D5, {0x8CC1}}, {0x2F9D6, {0x8D1B}}, {0x2F9D7, {0x8D77}}, {0x2F9D8, {0x27F2F}}, {0x2F9D9, {0x20804}}, {0x2F9DA, {0x8DCB}}, {0x2F9DB, {0x8DBC}}, {0x2F9DC, {0x8DF0}}, +{0x2F9DD, {0x208DE}}, {0x2F9DE, {0x8ED4}}, {0x2F9DF, {0x8F38}}, {0x2F9E0, {0x285D2}}, {0x2F9E1, {0x285ED}}, {0x2F9E2, {0x9094}}, {0x2F9E3, {0x90F1}}, {0x2F9E4, {0x9111}}, {0x2F9E5, {0x2872E}}, +{0x2F9E6, {0x911B}}, {0x2F9E7, {0x9238}}, {0x2F9E8, {0x92D7}}, {0x2F9E9, {0x92D8}}, {0x2F9EA, {0x927C}}, {0x2F9EB, {0x93F9}}, {0x2F9EC, {0x9415}}, {0x2F9ED, {0x28BFA}}, {0x2F9EE, {0x958B}}, +{0x2F9EF, {0x4995}}, {0x2F9F0, {0x95B7}}, {0x2F9F1, {0x28D77}}, {0x2F9F2, {0x49E6}}, {0x2F9F3, {0x96C3}}, {0x2F9F4, {0x5DB2}}, {0x2F9F5, {0x9723}}, {0x2F9F6, {0x29145}}, {0x2F9F7, {0x2921A}}, +{0x2F9F8, {0x4A6E}}, {0x2F9F9, {0x4A76}}, {0x2F9FA, {0x97E0}}, {0x2F9FB, {0x2940A}}, {0x2F9FC, {0x4AB2}}, {0x2F9FD, {0x29496}}, {0x2F9FE, {0x980B}}, {0x2F9FF, {0x980B}}, {0x2FA00, {0x9829}}, +{0x2FA01, {0x295B6}}, {0x2FA02, {0x98E2}}, {0x2FA03, {0x4B33}}, {0x2FA04, {0x9929}}, {0x2FA05, {0x99A7}}, {0x2FA06, {0x99C2}}, {0x2FA07, {0x99FE}}, {0x2FA08, {0x4BCE}}, {0x2FA09, {0x29B30}}, +{0x2FA0A, {0x9B12}}, {0x2FA0B, {0x9C40}}, {0x2FA0C, {0x9CFD}}, {0x2FA0D, {0x4CCE}}, {0x2FA0E, {0x4CED}}, {0x2FA0F, {0x9D67}}, {0x2FA10, {0x2A0CE}}, {0x2FA11, {0x4CF8}}, {0x2FA12, {0x2A105}}, +{0x2FA13, {0x2A20E}}, {0x2FA14, {0x2A291}}, {0x2FA15, {0x9EBB}}, {0x2FA16, {0x4D56}}, {0x2FA17, {0x9EF9}}, {0x2FA18, {0x9EFE}}, {0x2FA19, {0x9F05}}, {0x2FA1A, {0x9F0F}}, {0x2FA1B, {0x9F16}}, +{0x2FA1D, {0x2A600}}, +}; + static std::string codepoint_to_utf8(uint32_t cp) { std::string result; if (/* 0x00 <= cp && */ cp <= 0x7f) { From adcb12a9bad87bc96f2f158c95892b3d04aa7ffb Mon Sep 17 00:00:00 2001 From: compilade <113953597+compilade@users.noreply.github.com> Date: Wed, 28 Feb 2024 03:52:56 -0500 Subject: [PATCH 483/812] llama : fix non-quantization of expert gating tensors (#5754) This reverts a single line from #5475 --- llama.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/llama.cpp b/llama.cpp index 356ca1076..893bcdbc0 100644 --- a/llama.cpp +++ b/llama.cpp @@ -11162,7 +11162,8 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s quantize &= !params->only_copy; // do not quantize expert gating tensors - quantize &= name != LLM_TN(model.arch)(LLM_TENSOR_FFN_GATE_INP, "weight"); + // NOTE: can't use LLM_TN here because the layer number is not known + quantize &= name.find("ffn_gate_inp.weight") == std::string::npos; // do not quantize positional embeddings and token types (BERT) quantize &= name != LLM_TN(model.arch)(LLM_TENSOR_POS_EMBD, "weight"); From a693bea1e6762a17b78b6ddf4611e54136941ea2 Mon Sep 17 00:00:00 2001 From: Xuan Son Nguyen Date: Wed, 28 Feb 2024 09:55:37 +0100 Subject: [PATCH 484/812] server : hit Ctrl+C twice to exit (#5734) * server: twice ctrl+C to exit * std::atomic_flag * sigint: message * sigint: stderr * Update examples/server/server.cpp Co-authored-by: Jared Van Bortel --------- Co-authored-by: Jared Van Bortel --- examples/server/server.cpp | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/examples/server/server.cpp b/examples/server/server.cpp index 6b3ee531c..080fa9bd5 100644 --- a/examples/server/server.cpp +++ b/examples/server/server.cpp @@ -2772,7 +2772,16 @@ static void append_to_generated_text_from_generated_token_probs(llama_server_con } std::function shutdown_handler; -inline void signal_handler(int signal) { shutdown_handler(signal); } +std::atomic_flag is_terminating = ATOMIC_FLAG_INIT; +inline void signal_handler(int signal) { + if (is_terminating.test_and_set()) { + // in case it hangs, we can force terminate the server by hitting Ctrl+C twice + // this is for better developer experience, we can remove when the server is stable enough + fprintf(stderr, "Received second interrupt, terminating immediately.\n"); + exit(1); + } + shutdown_handler(signal); +} int main(int argc, char **argv) { From 5f706718566e3a5147916dc381f3b99de0ffad47 Mon Sep 17 00:00:00 2001 From: "UEXTM.com" <84163508+uextm@users.noreply.github.com> Date: Sat, 24 Feb 2024 11:27:36 -0500 Subject: [PATCH 485/812] Introduce backend GUIDs (ggml/743) * Introduce backend GUIDs Initial proposed implementation of backend GUIDs (Discussed in https://github.com/ggerganov/ggml/pull/741) Hardcoded CPU backend GUID (for now) Change ggml_backend_is_cpu logic to use GUID * Remove redundant functions Remove redundant functions `ggml_backend_i::get_name` and `ggml_backend_guid` which are not desired for future expansion * Add spaces to match style Co-authored-by: slaren * Fix brace style to match Co-authored-by: slaren * Add void to () in function signature Co-authored-by: slaren * Add back ggml_backend_guid and make CPU_GUID a local static in ggml_backend_cpu_guid * add guids to all backends ggml-ci --------- Co-authored-by: slaren --- ggml-backend-impl.h | 2 ++ ggml-backend.c | 16 ++++++++++++++-- ggml-backend.h | 2 +- ggml-cuda.cu | 8 +++++++- ggml-kompute.cpp | 8 +++++++- ggml-metal.m | 8 +++++++- ggml-sycl.cpp | 8 +++++++- ggml-vulkan.cpp | 8 +++++++- ggml.c | 4 ++++ ggml.h | 10 ++++++++++ 10 files changed, 66 insertions(+), 8 deletions(-) diff --git a/ggml-backend-impl.h b/ggml-backend-impl.h index f95df47f7..0e5bf0ae1 100644 --- a/ggml-backend-impl.h +++ b/ggml-backend-impl.h @@ -104,6 +104,8 @@ extern "C" { }; struct ggml_backend { + ggml_guid_t guid; + struct ggml_backend_i iface; ggml_backend_context_t context; diff --git a/ggml-backend.c b/ggml-backend.c index 5076d9e5e..c86673b04 100644 --- a/ggml-backend.c +++ b/ggml-backend.c @@ -12,7 +12,6 @@ #define MAX(a, b) ((a) > (b) ? (a) : (b)) - // backend buffer type const char * ggml_backend_buft_name(ggml_backend_buffer_type_t buft) { @@ -159,6 +158,13 @@ bool ggml_backend_buffer_copy_tensor(const struct ggml_tensor * src, struct ggml // backend +ggml_guid_t ggml_backend_guid(ggml_backend_t backend) { + if (backend == NULL) { + return NULL; + } + return backend->guid; +} + const char * ggml_backend_name(ggml_backend_t backend) { if (backend == NULL) { return "NULL"; @@ -781,6 +787,11 @@ static struct ggml_backend_i cpu_backend_i = { /* .supports_op = */ ggml_backend_cpu_supports_op, }; +static ggml_guid_t ggml_backend_cpu_guid(void) { + static ggml_guid guid = { 0xaa, 0x67, 0xc7, 0x43, 0x96, 0xe6, 0xa3, 0x8a, 0xe3, 0xaf, 0xea, 0x92, 0x36, 0xbc, 0xfc, 0x89 }; + return &guid; +} + ggml_backend_t ggml_backend_cpu_init(void) { struct ggml_backend_cpu_context * ctx = malloc(sizeof(struct ggml_backend_cpu_context)); if (ctx == NULL) { @@ -800,6 +811,7 @@ ggml_backend_t ggml_backend_cpu_init(void) { } *cpu_backend = (struct ggml_backend) { + /* .guid = */ ggml_backend_cpu_guid(), /* .interface = */ cpu_backend_i, /* .context = */ ctx }; @@ -807,7 +819,7 @@ ggml_backend_t ggml_backend_cpu_init(void) { } GGML_CALL bool ggml_backend_is_cpu(ggml_backend_t backend) { - return backend && backend->iface.get_name == ggml_backend_cpu_name; + return backend != NULL && ggml_guid_matches(backend->guid, ggml_backend_cpu_guid()); } void ggml_backend_cpu_set_n_threads(ggml_backend_t backend_cpu, int n_threads) { diff --git a/ggml-backend.h b/ggml-backend.h index f13c69bff..8fb54bd92 100644 --- a/ggml-backend.h +++ b/ggml-backend.h @@ -49,7 +49,7 @@ extern "C" { // Backend // - + GGML_API ggml_guid_t ggml_backend_guid(ggml_backend_t backend); GGML_API const char * ggml_backend_name(ggml_backend_t backend); GGML_API void ggml_backend_free(ggml_backend_t backend); diff --git a/ggml-cuda.cu b/ggml-cuda.cu index 831c84efb..0c6501e98 100644 --- a/ggml-cuda.cu +++ b/ggml-cuda.cu @@ -12277,6 +12277,11 @@ static ggml_backend_i ggml_backend_cuda_interface = { /* .supports_op = */ ggml_backend_cuda_supports_op, }; +static ggml_guid_t ggml_backend_cuda_guid() { + static ggml_guid guid = { 0x2c, 0xdd, 0xe8, 0x1c, 0x65, 0xb3, 0x65, 0x73, 0x6a, 0x12, 0x88, 0x61, 0x1c, 0xc9, 0xdc, 0x25 }; + return &guid; +} + GGML_CALL ggml_backend_t ggml_backend_cuda_init(int device) { ggml_init_cublas(); // TODO: remove from ggml.c @@ -12294,6 +12299,7 @@ GGML_CALL ggml_backend_t ggml_backend_cuda_init(int device) { }; ggml_backend_t cuda_backend = new ggml_backend { + /* .guid = */ ggml_backend_cuda_guid(), /* .interface = */ ggml_backend_cuda_interface, /* .context = */ ctx }; @@ -12302,7 +12308,7 @@ GGML_CALL ggml_backend_t ggml_backend_cuda_init(int device) { } GGML_CALL bool ggml_backend_is_cuda(ggml_backend_t backend) { - return backend && backend->iface.get_name == ggml_backend_cuda_name; + return backend != NULL && ggml_guid_matches(backend->guid, ggml_backend_cuda_guid()); } GGML_CALL int ggml_backend_cuda_get_device_count() { diff --git a/ggml-kompute.cpp b/ggml-kompute.cpp index 51c5af8ec..e740a76d1 100644 --- a/ggml-kompute.cpp +++ b/ggml-kompute.cpp @@ -1953,11 +1953,17 @@ static struct ggml_backend_i kompute_backend_i = { /* .supports_op = */ ggml_backend_kompute_supports_op, }; +static ggml_guid_t ggml_backend_kompute_guid() { + static ggml_guid guid = { 0x7b, 0x57, 0xdc, 0xaf, 0xde, 0x12, 0x1d, 0x49, 0xfb, 0x35, 0xfa, 0x9b, 0x18, 0x31, 0x1d, 0xca }; + return &guid; +} + ggml_backend_t ggml_backend_kompute_init(int device) { GGML_ASSERT(s_kompute_context == nullptr); s_kompute_context = new ggml_kompute_context(device); ggml_backend_t kompute_backend = new ggml_backend { + /* .guid = */ ggml_backend_kompute_guid(), /* .interface = */ kompute_backend_i, /* .context = */ s_kompute_context, }; @@ -1966,7 +1972,7 @@ ggml_backend_t ggml_backend_kompute_init(int device) { } bool ggml_backend_is_kompute(ggml_backend_t backend) { - return backend && backend->iface.get_name == ggml_backend_kompute_name; + return backend != NULL && ggml_guid_matches(backend->guid, ggml_backend_kompute_guid()); } static ggml_backend_t ggml_backend_reg_kompute_init(const char * params, void * user_data) { diff --git a/ggml-metal.m b/ggml-metal.m index 9eba2f5d2..71fcca560 100644 --- a/ggml-metal.m +++ b/ggml-metal.m @@ -2771,6 +2771,11 @@ void ggml_backend_metal_log_set_callback(ggml_log_callback log_callback, void * ggml_metal_log_user_data = user_data; } +static ggml_guid_t ggml_backend_metal_guid(void) { + static ggml_guid guid = { 0x81, 0xa1, 0x8b, 0x1e, 0x71, 0xec, 0x79, 0xed, 0x2b, 0x85, 0xdc, 0x8a, 0x61, 0x98, 0x30, 0xe6 }; + return &guid; +} + ggml_backend_t ggml_backend_metal_init(void) { struct ggml_metal_context * ctx = ggml_metal_init(GGML_DEFAULT_N_THREADS); @@ -2781,6 +2786,7 @@ ggml_backend_t ggml_backend_metal_init(void) { ggml_backend_t metal_backend = malloc(sizeof(struct ggml_backend)); *metal_backend = (struct ggml_backend) { + /* .guid = */ ggml_backend_metal_guid(), /* .interface = */ ggml_backend_metal_i, /* .context = */ ctx, }; @@ -2789,7 +2795,7 @@ ggml_backend_t ggml_backend_metal_init(void) { } bool ggml_backend_is_metal(ggml_backend_t backend) { - return backend && backend->iface.get_name == ggml_backend_metal_name; + return backend != NULL && ggml_guid_matches(backend->guid, ggml_backend_metal_guid()); } void ggml_backend_metal_set_n_cb(ggml_backend_t backend, int n_cb) { diff --git a/ggml-sycl.cpp b/ggml-sycl.cpp index 835967fb6..a054ec8b9 100644 --- a/ggml-sycl.cpp +++ b/ggml-sycl.cpp @@ -15162,6 +15162,11 @@ static ggml_backend_i ggml_backend_sycl_interface = { /* .supports_op = */ ggml_backend_sycl_supports_op, }; +static ggml_guid_t ggml_backend_sycl_guid() { + static ggml_guid guid = { 0x58, 0x05, 0x13, 0x8f, 0xcd, 0x3a, 0x61, 0x9d, 0xe7, 0xcd, 0x98, 0xa9, 0x03, 0xfd, 0x7c, 0x53 }; + return &guid; +} + ggml_backend_t ggml_backend_sycl_init(int device) { ggml_init_sycl(); // TODO: remove from ggml.c @@ -15179,6 +15184,7 @@ ggml_backend_t ggml_backend_sycl_init(int device) { }; ggml_backend_t sycl_backend = new ggml_backend { + /* .guid = */ ggml_backend_sycl_guid(), /* .interface = */ ggml_backend_sycl_interface, /* .context = */ ctx }; @@ -15187,7 +15193,7 @@ ggml_backend_t ggml_backend_sycl_init(int device) { } bool ggml_backend_is_sycl(ggml_backend_t backend) { - return backend->iface.get_name == ggml_backend_sycl_name; + return backend != NULL && ggml_guid_matches(backend->guid, ggml_backend_sycl_guid()); } static ggml_backend_t ggml_backend_reg_sycl_init(const char * params, void * user_data) { diff --git a/ggml-vulkan.cpp b/ggml-vulkan.cpp index 6caafb822..314e3d7a9 100644 --- a/ggml-vulkan.cpp +++ b/ggml-vulkan.cpp @@ -5244,6 +5244,11 @@ static ggml_backend_i ggml_backend_vk_interface = { /* .supports_op = */ ggml_backend_vk_supports_op, }; +static ggml_guid_t ggml_backend_vk_guid() { + static ggml_guid guid = { 0xb8, 0xf7, 0x4f, 0x86, 0x40, 0x3c, 0xe1, 0x02, 0x91, 0xc8, 0xdd, 0xe9, 0x02, 0x3f, 0xc0, 0x2b }; + return &guid; +} + GGML_CALL ggml_backend_t ggml_backend_vk_init(size_t idx) { if (vk_instance.initialized[idx]) { return vk_instance.backends[idx]; @@ -5262,6 +5267,7 @@ GGML_CALL ggml_backend_t ggml_backend_vk_init(size_t idx) { vk_instance.initialized[idx] = true; ggml_backend_t vk_backend = new ggml_backend { + /* .guid = */ ggml_backend_vk_guid(), /* .interface = */ ggml_backend_vk_interface, /* .context = */ &vk_instance.contexts[ctx->idx], }; @@ -5272,7 +5278,7 @@ GGML_CALL ggml_backend_t ggml_backend_vk_init(size_t idx) { } GGML_CALL bool ggml_backend_is_vk(ggml_backend_t backend) { - return backend && backend->iface.get_name == ggml_backend_vk_name; + return backend != NULL && ggml_guid_matches(backend->guid, ggml_backend_vk_guid()); } GGML_CALL int ggml_backend_vk_get_device_count() { diff --git a/ggml.c b/ggml.c index 4591644ad..68ac6201e 100644 --- a/ggml.c +++ b/ggml.c @@ -355,6 +355,10 @@ void ggml_fp32_to_fp16_row(const float * x, ggml_fp16_t * y, int n) { } } +bool ggml_guid_matches(ggml_guid_t guid_a, ggml_guid_t guid_b) { + return memcmp(guid_a, guid_b, sizeof(ggml_guid)) == 0; +} + // // timing // diff --git a/ggml.h b/ggml.h index 23b768640..0a6d3c051 100644 --- a/ggml.h +++ b/ggml.h @@ -672,6 +672,16 @@ extern "C" { GGML_NUMA_STRATEGY_COUNT }; + // + // GUID + // + + // GUID types + typedef uint8_t ggml_guid[16]; + typedef ggml_guid * ggml_guid_t; + + GGML_API bool ggml_guid_matches(ggml_guid_t guid_a, ggml_guid_t guid_b); + // misc GGML_API void ggml_time_init(void); // call this once at the beginning of the program From 2774b0c97427ee3ad3e2ee121354d078794e89d9 Mon Sep 17 00:00:00 2001 From: slaren Date: Sun, 25 Feb 2024 20:41:35 +0100 Subject: [PATCH 486/812] add google magika inference example (ggml/748) * add magika inference example * ggml : fix unaligned accesses in custom ops * ggml : fix FP32 GELU for values that exceed the FP16 range * use ggml_pool_1d * add README * Update README.md * pad inputs if the files are too small * cleanup ggml-ci --- ggml.c | 54 ++++++++++++++++++++++++++++++++++-------------------- 1 file changed, 34 insertions(+), 20 deletions(-) diff --git a/ggml.c b/ggml.c index 68ac6201e..f29b9f13f 100644 --- a/ggml.c +++ b/ggml.c @@ -1608,9 +1608,15 @@ inline static void ggml_vec_gelu_f16(const int n, ggml_fp16_t * y, const ggml_fp inline static void ggml_vec_gelu_f32(const int n, float * y, const float * x) { uint16_t t; for (int i = 0; i < n; ++i) { - ggml_fp16_t fp16 = GGML_FP32_TO_FP16(x[i]); - memcpy(&t, &fp16, sizeof(uint16_t)); - y[i] = GGML_FP16_TO_FP32(ggml_table_gelu_f16[t]); + if (x[i] <= -10.0f) { + y[i] = 0.0f; + } else if (x[i] >= 10.0f) { + y[i] = x[i]; + } else { + ggml_fp16_t fp16 = GGML_FP32_TO_FP16(x[i]); + memcpy(&t, &fp16, sizeof(uint16_t)); + y[i] = GGML_FP16_TO_FP32(ggml_table_gelu_f16[t]); + } } } #else @@ -5780,11 +5786,13 @@ struct ggml_tensor * ggml_pool_1d( is_node = true; } - const int64_t ne[2] = { + const int64_t ne[4] = { ggml_calc_pool_output_size(a->ne[0], k0, s0, p0), a->ne[1], + a->ne[2], + a->ne[3], }; - struct ggml_tensor * result = ggml_new_tensor(ctx, GGML_TYPE_F32, 2, ne); + struct ggml_tensor * result = ggml_new_tensor(ctx, GGML_TYPE_F32, 4, ne); int32_t params[] = { op, k0, s0, p0 }; ggml_set_op_params(result, params, sizeof(params)); @@ -15081,9 +15089,10 @@ static void ggml_compute_forward_map_custom1( return; } - struct ggml_map_custom1_op_params * p = (struct ggml_map_custom1_op_params *) dst->op_params; + struct ggml_map_custom1_op_params p; + memcpy(&p, dst->op_params, sizeof(p)); - p->fun(dst, a, params->ith, params->nth, p->userdata); + p.fun(dst, a, params->ith, params->nth, p.userdata); } // ggml_compute_forward_map_custom2 @@ -15099,9 +15108,10 @@ static void ggml_compute_forward_map_custom2( return; } - struct ggml_map_custom2_op_params * p = (struct ggml_map_custom2_op_params *) dst->op_params; + struct ggml_map_custom2_op_params p; + memcpy(&p, dst->op_params, sizeof(p)); - p->fun(dst, a, b, params->ith, params->nth, p->userdata); + p.fun(dst, a, b, params->ith, params->nth, p.userdata); } // ggml_compute_forward_map_custom3 @@ -15118,9 +15128,10 @@ static void ggml_compute_forward_map_custom3( return; } - struct ggml_map_custom3_op_params * p = (struct ggml_map_custom3_op_params *) dst->op_params; + struct ggml_map_custom3_op_params p; + memcpy(&p, dst->op_params, sizeof(p)); - p->fun(dst, a, b, c, params->ith, params->nth, p->userdata); + p.fun(dst, a, b, c, params->ith, params->nth, p.userdata); } // ggml_compute_forward_cross_entropy_loss @@ -17386,29 +17397,32 @@ static int ggml_get_n_tasks(struct ggml_tensor * node, int n_threads) { } break; case GGML_OP_MAP_CUSTOM1: { - struct ggml_map_custom1_op_params * p = (struct ggml_map_custom1_op_params *) node->op_params; - if (p->n_tasks == GGML_N_TASKS_MAX) { + struct ggml_map_custom1_op_params p; + memcpy(&p, node->op_params, sizeof(p)); + if (p.n_tasks == GGML_N_TASKS_MAX) { n_tasks = n_threads; } else { - n_tasks = MIN(p->n_tasks, n_threads); + n_tasks = MIN(p.n_tasks, n_threads); } } break; case GGML_OP_MAP_CUSTOM2: { - struct ggml_map_custom2_op_params * p = (struct ggml_map_custom2_op_params *) node->op_params; - if (p->n_tasks == GGML_N_TASKS_MAX) { + struct ggml_map_custom2_op_params p; + memcpy(&p, node->op_params, sizeof(p)); + if (p.n_tasks == GGML_N_TASKS_MAX) { n_tasks = n_threads; } else { - n_tasks = MIN(p->n_tasks, n_threads); + n_tasks = MIN(p.n_tasks, n_threads); } } break; case GGML_OP_MAP_CUSTOM3: { - struct ggml_map_custom3_op_params * p = (struct ggml_map_custom3_op_params *) node->op_params; - if (p->n_tasks == GGML_N_TASKS_MAX) { + struct ggml_map_custom3_op_params p; + memcpy(&p, node->op_params, sizeof(p)); + if (p.n_tasks == GGML_N_TASKS_MAX) { n_tasks = n_threads; } else { - n_tasks = MIN(p->n_tasks, n_threads); + n_tasks = MIN(p.n_tasks, n_threads); } } break; case GGML_OP_CROSS_ENTROPY_LOSS: From 8c0e8f4e73e275756ad69f9c99b26ead085ca9f0 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Wed, 28 Feb 2024 11:17:32 +0200 Subject: [PATCH 487/812] sync : ggml --- scripts/sync-ggml.last | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/sync-ggml.last b/scripts/sync-ggml.last index 59de34370..389c0bdfe 100644 --- a/scripts/sync-ggml.last +++ b/scripts/sync-ggml.last @@ -1 +1 @@ -8cdf783f288a98eddf521b0ab1b4d405be9e18ba +b458250b736a7473f7ff3560d47c93f1644f3290 From 78aacf36344df724cdca9f1e1af849b2d2519cb8 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Wed, 28 Feb 2024 17:36:53 +0200 Subject: [PATCH 488/812] awq-py : remove (#5768) --- awq-py/README.md | 116 ------------------ awq-py/awq/apply_awq.py | 254 ---------------------------------------- awq-py/requirements.txt | 2 - 3 files changed, 372 deletions(-) delete mode 100644 awq-py/README.md delete mode 100644 awq-py/awq/apply_awq.py delete mode 100644 awq-py/requirements.txt diff --git a/awq-py/README.md b/awq-py/README.md deleted file mode 100644 index 16e68d027..000000000 --- a/awq-py/README.md +++ /dev/null @@ -1,116 +0,0 @@ -# AWQ: Activation-aware Weight Quantization for LLM - version apply to llamacpp -[[Paper](https://arxiv.org/abs/2306.00978)][[Original Repo](https://github.com/mit-han-lab/llm-awq)][[Easy-to-use Repo](https://github.com/casper-hansen/AutoAWQ)] - -**Supported models:** - -- [X] LLaMA -- [x] LLaMA 2 -- [X] MPT -- [X] Mistral AI v0.1 -- [ ] Bloom -- [ ] Mixtral MoE - -**TODO:** -- [x] Update version work with both MPT and MPT-AWQ model -- [ ] Add OPT model -- [ ] Add Bloom model -- [ ] Add Mixtral MoE -- [ ] Support w3, w2 - - -## Contents - -- [Install](##Install) -- [Convert](##Convert) -- [Quantize](##Quantize) -- [Test](##Test) -- [Benchmark](##Benchmark) -- [Results](##Results) - -## Install -Install requirements -```bash -pip install -r requirements.txt -``` -Get the pre-computed AWQ search results for multiple model families, including LLaMA, LLaMA2, MPT, OPT -```bash -git clone https://huggingface.co/datasets/mit-han-lab/awq-model-zoo awq_cache -``` - -## Convert -Example for llama model -```bash -# For llama7b and llama2 models -python convert.py models/llama-7b/ --awq-path awq_cache/llama-7b-w4-g128.pt --outfile models/llama_7b_fp16.gguf -# For mistral and mpt models -python convert-hf-to-gguf.py models/mpt-7b/ --awq-path awq_cache/mpt-7b-w4-g128.pt --outfile models/mpt_7b_fp16.gguf -``` - -## Quantize -```bash -# We only benchmark and confirm the results on q4_0, q4_1, and q2_k types. -./quantize models/llama_7b_fp16.gguf models/llama_7b_q4_0.gguf q4_0 -``` - -## Test -```bash -# For all models. -./build/bin/main -m models/llama_7b_q4_0.gguf -n 128 --prompt "Once upon a time" -``` - -## Benchmark -The perplexity measurements in table above are done against the `wikitext2` test dataset (https://paperswithcode.com/dataset/wikitext-2), with context length of 512. -```bash -# For llama and llama2, and mistral models. -./perplexity -m models/llama_7b_q4_0.gguf -f datasets/wikitext-2-raw/wiki.test.raw -``` - -## Results -Results are run on OpenBLAS (CPU) and CuBLAS (GPU) for fair comparison -We use three types of llamacpp quantization methods to work with our version, including q4_0, q4_1, and q2_k - -### Llama 7B (Build with OpenBLAS) - -| Model | Measure | F16 | Q4_0 | Q4_1 | Q2_K | -|-----------:|--------------|-------:|-------:|-------:|-------:| -|Llama 7B | perplexity | 5.9066 | 6.1214 | 6.0643 | 6.5808 | -|Llama 7B | file size | 12.9G | 3.5G | 3.9G | 2.7G | -|Llama 7B | bits/weight | 16.0 | 4.5 | 5.0 | 2.6 | -|AWQ-LLama 7B| perplexity | 5.9175 | 6.0252 | 5.9987 | 6.3692 | -|AWQ-LLama 7B| file size | 12.9G | 3.5G | 3.9G | 2.7G | -|AWQ-LLama 7B| bits/weight | 16.0 | 4.5 | 5.0 | 2.6 | - - -### Llama2 7B (Build with CuBLAS) - -| Model | Measure | F16 | Q4_0 | Q4_1 | Q2_K | -|------------:|--------------|-------:|-------:|-------:|-------:| -|Llama2 7B | perplexity | 5.8664 | 6.0260 | 6.0656 | 6.4496 | -|Llama2 7B | file size | 12.9G | 3.5G | 3.9G | 2.7G | -|Llama2 7B | bits/weight | 16.0 | 4.5 | 5.0 | 2.6 | -|AWQ-LLama2 7B| perplexity | 5.8801 | 6.0054 | 5.9849 | 6.3650 | -|AWQ-LLama2 7B| file size | 12.9G | 3.5G | 3.9G | 2.7G | -|AWQ-LLama2 7B| bits/weight | 16.0 | 4.5 | 5.0 | 2.6 | - - -### Mistral 7B v0.1 (Build with CuBLAS) - -| Model | Measure | F16 | Q4_0 | Q4_1 | Q2_K | -|-------------:|--------------|-------:|-------:|-------:|-------:| -|Mistral 7B | perplexity | 5.6931 | 5.8202 | 5.8268 | 6.1645 | -|Mistral 7B | file size | 14.5G | 4.1G | 4.5G | 3.1G | -|Mistral 7B | bits/weight | 16.0 | 4.5 | 5.0 | 2.6 | -|AWQ-Mistral 7B| perplexity | 5.6934 | 5.8020 | 5.7691 | 6.0426 | -|AWQ-Mistral 7B| file size | 14.5G | 4.1G | 4.5G | 3.1G | -|AWQ-Mistral 7B| bits/weight | 16.0 | 4.5 | 5.0 | 2.6 | - -### MPT 7B (Build with OpenBLAS) - -| Model | Measure | F16 | Q4_0 | Q4_1 | Q2_K | -|---------:|--------------|-------:|-------:|-------:|--------:| -|MPT 7B | perplexity | 8.4369 | 8.7956 | 8.6265 | 11.4913 | -|MPT 7B | file size | 13.7G | 3.9G | 4.3G | 2.8G | -|MPT 7B | bits/weight | 16.0 | 4.5 | 5.0 | 2.6 | -|AWQ-MPT 7B| perplexity | 8.4944 | 8.7053 | 8.6750 | 10.2873| -|AWQ-MPT 7B| file size | 13.7G | 3.9G | 4.3G | 2.8G | -|AWQ-MPT 7B| bits/weight | 16.0 | 4.5 | 5.0 | 2.6 | diff --git a/awq-py/awq/apply_awq.py b/awq-py/awq/apply_awq.py deleted file mode 100644 index 11132c5d2..000000000 --- a/awq-py/awq/apply_awq.py +++ /dev/null @@ -1,254 +0,0 @@ -""" -Implements the AWQ for llama.cpp use cases. -Original paper: https://arxiv.org/abs/2306.00978 - -This code is based on versions of the AWQ implementation found in the following repositories: -* https://github.com/mit-han-lab/llm-awq -* https://github.com/casper-hansen/AutoAWQ -""" - -import os -import torch -import torch.nn as nn - -from transformers import AutoModelForCausalLM, AutoConfig -from transformers.models.bloom.modeling_bloom import BloomGelu -from transformers.models.llama.modeling_llama import LlamaRMSNorm -from transformers.activations import GELUActivation - - -class ScaledActivation(nn.Module): - """ - ScaledActivation module wraps an existing activation function and applies a - scale factor to its output. - - Args: - module (nn.Module): The activation function to be scaled. - scales (torch.Tensor): A tensor of size (num_features,) containing the initial - scale factors for each feature. - - Returns: - torch.Tensor: The scaled output of the activation function. - """ - - def __init__(self, module, scales): - super().__init__() - self.act = module - self.scales = nn.Parameter(scales.data) - - def forward(self, x): - return self.act(x) / self.scales.view(1, 1, -1).to(x.device) - - -def set_op_by_name(layer, name, new_module): - """ - Set the new module for given module's name. - - Args: - layer (nn.Module): The layer in which to replace the submodule. - name (str): The path to the submodule to be replaced, using dot notation - to access nested modules. - new_module (nn.Module): The new module to replace the existing one. - """ - levels = name.split(".") - if len(levels) > 1: - mod_ = layer - for l_idx in range(len(levels) - 1): - if levels[l_idx].isdigit(): - mod_ = mod_[int(levels[l_idx])] - else: - mod_ = getattr(mod_, levels[l_idx]) - setattr(mod_, levels[-1], new_module) - else: - setattr(layer, name, new_module) - - -def get_op_by_name(module, op_name): - """ - Retrieves a submodule within a given layer based on its name. - - Args: - module (nn.Module): The layer containing the submodule to find. - op_name (str): The name of the submodule. - - Returns: - nn.Module: The requested submodule found within the given layer. - - Raises: - ValueError: If the specified submodule cannot be found within the layer. - """ - for name, m in module.named_modules(): - if name == op_name: - return m - raise ValueError(f"Cannot find op {op_name} in module {module}") - - -@torch.no_grad() -def scale_ln_fcs(ln, fcs, scales): - """ - Scales the weights of a LayerNorm and a list of fully-connected layers proportionally. - - Args: - ln (nn.LayerNorm): The LayerNorm module to be scaled. - fcs (List[nn.Linear]): A list of fully-connected layers to be scaled. - scales (torch.Tensor): A 1D tensor of size (num_features,). - """ - - if not isinstance(fcs, list): - fcs = [fcs] - - scales = scales.to(ln.weight.device) - - ln.weight.div_(scales) - if hasattr(ln, "bias") and ln.bias is not None: - ln.bias.div_(scales) - - for fc in fcs: - fc.weight.mul_(scales.view(1, -1)) - - for p in ln.parameters(): - assert torch.isnan(p).sum() == 0 - for fc in fcs: - for p in fc.parameters(): - assert torch.isnan(p).sum() == 0 - - -@torch.no_grad() -def scale_fc_fc(fc1, fc2, scales): - """ - Scales the weights of two fully-connected layers in a specific pattern. - - Args: - fc1 (nn.Linear): The first fully-connected layer to be scaled. - fc2 (nn.Linear): The second fully-connected layer to be scaled. - scales (torch.Tensor): A 1D tensor of size (num_features,). - """ - assert isinstance(fc1, nn.Linear) - assert isinstance(fc2, nn.Linear) - - scales = scales.to(fc1.weight.device) - - fc1.weight[-scales.size(0):].div_(scales.view(-1, 1)) - if fc1.bias is not None: - fc1.bias.div_(scales.view(-1)) - - fc2.weight.mul_(scales.view(1, -1)) - - for p in fc1.parameters(): - assert torch.isnan(p).sum() == 0 - for p in fc2.parameters(): - assert torch.isnan(p).sum() == 0 - - -@torch.no_grad() -def scale_gelu_fc(gelu, fc, scales): - """ - Scales the weight of a GELU activation and a fully-connected layer proportionally. - - Args: - gelu (Union[nn.GELU, BloomGelu, GELUActivation]): The GELU activation module to be scaled. - fc (nn.Linear): The fully-connected layer to be scaled. - scales (torch.Tensor): A 1D tensor of size (num_features,). - - Raises: - TypeError: If the `gelu` module is not of type `nn.GELU`, `BloomGelu`, or `GELUActivation`. - TypeError: If the `fc` module is not of type `nn.Linear`. - """ - assert isinstance(gelu, (nn.GELU, BloomGelu, GELUActivation)) - assert isinstance(fc, nn.Linear) - - fc.weight.mul_(scales.view(1, -1).to(fc.weight.device)) - - for p in fc.parameters(): - assert torch.isnan(p).sum() == 0 - - -def apply_scale(module, scales_list, input_feat_dict=None): - """ - Applies different scaling strategies to layers based on their type and hierarchy within a given module. - - Args: - module (nn.Module): The module containing the layers to be scaled. - scales_list (List[Tuple[str, List[str], torch.Tensor]]): A list of tuples containing: - * prev_op_name (str): The name of the preceding operation or module, - relative to which the layers to be scaled are located. - * layer_names (List[str]): A list of names of the layers to be scaled, relative to the preceding operation. - * scales (torch.Tensor): A 1D tensor of size (num_features,) containing the scaling factors for each feature. - input_feat_dict (Optional[Dict[str, torch.Tensor]]): A dictionary mapping layer names to their corresponding - input features (optional). - """ - for prev_op_name, layer_names, scales in scales_list: - prev_op = get_op_by_name(module, prev_op_name) - layers = [get_op_by_name(module, name) for name in layer_names] - - prev_op.cuda() - for layer in layers: - layer.cuda() - scales.cuda() - - if isinstance(prev_op, nn.Linear): - assert len(layers) == 1 - scale_fc_fc(prev_op, layers[0], scales) - elif isinstance(prev_op, (nn.LayerNorm, LlamaRMSNorm)) or "rmsnorm" in str(prev_op.__class__).lower(): - scale_ln_fcs(prev_op, layers, scales) - elif isinstance(prev_op, (nn.GELU, BloomGelu, GELUActivation)): - new_module = ScaledActivation(prev_op, scales) - set_op_by_name(module, prev_op_name, new_module) - scale_gelu_fc(prev_op, layers[0], scales) - else: - raise NotImplementedError(f"prev_op {type(prev_op)} not supported yet!") - - # apply the scaling to input feat if given; prepare it for clipping - if input_feat_dict is not None: - for layer_name in layer_names: - inp = input_feat_dict[layer_name] - inp.div_(scales.view(1, -1).to(inp.device)) - - prev_op.cpu() - for layer in layers: - layer.cpu() - scales.cpu() - - -@torch.no_grad() -def apply_clip(module, clip_list): - """ - Applies element-wise clipping to the weight of a specific layer within a given module. - - Args: - module (nn.Module): The module containing the layer to be clipped. - clip_list (List[Tuple[str, torch.Tensor]]): A list of tuples containing: - * name (str): The name of the layer to be clipped, relative to the root of the module. - * max_val (torch.Tensor): A 1D or 2D tensor defining the upper bound for each element of the layer's weight. - """ - for name, max_val in clip_list: - layer = get_op_by_name(module, name) - layer.cuda() - max_val = max_val.to(layer.weight.device) - org_shape = layer.weight.shape - layer.weight.data = layer.weight.data.reshape(*max_val.shape[:2], -1) - layer.weight.data = torch.clamp(layer.weight.data, -max_val, max_val) - layer.weight.data = layer.weight.data.reshape(org_shape) - layer.cpu() - - -def add_scale_weights(model_path, scale_path, tmp_path): - """ - Adds pre-computed Activation Weight Quantization (AWQ) results to a model, - including scaling factors and clipping bounds. - - Args: - model_path (str): Path to the pre-trained model to be equipped with AWQ. - scale_path (str): Path to the AWQ scale factors (.pt file). - tmp_path (str): Path to the temporary directory where the equipped model will be saved. - """ - config = AutoConfig.from_pretrained(model_path, trust_remote_code=True) - model = AutoModelForCausalLM.from_pretrained( - model_path, config=config, trust_remote_code=True - ) - model.eval() - awq_results = torch.load(str(scale_path), map_location="cpu") - apply_scale(model, awq_results["scale"]) - apply_clip(model, awq_results["clip"]) - model.save_pretrained(str(tmp_path)) - os.system(f"cp {str(model_path)}/tokenizer* {str(tmp_path)}") diff --git a/awq-py/requirements.txt b/awq-py/requirements.txt deleted file mode 100644 index 991896116..000000000 --- a/awq-py/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -torch>=2.1.1 -transformers>=4.32.0 From 08c5ee87e4cceb603ecceac90734fcdade57311b Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Wed, 28 Feb 2024 18:43:38 +0200 Subject: [PATCH 489/812] llama : remove deprecated API (#5770) ggml-ci --- llama.cpp | 88 +------------------------------------------------------ llama.h | 45 ---------------------------- 2 files changed, 1 insertion(+), 132 deletions(-) diff --git a/llama.cpp b/llama.cpp index 893bcdbc0..30d5eb32d 100644 --- a/llama.cpp +++ b/llama.cpp @@ -7894,9 +7894,9 @@ static int llama_decode_internal( const auto n_batch = cparams.n_batch; GGML_ASSERT(n_tokens <= n_batch); + GGML_ASSERT((!batch.token && batch.embd) || (batch.token && !batch.embd)); // NOLINT int n_threads = n_tokens == 1 ? cparams.n_threads : cparams.n_threads_batch; - GGML_ASSERT((!batch.token && batch.embd) || (batch.token && !batch.embd)); // NOLINT const int64_t t_start_us = ggml_time_us(); @@ -10062,10 +10062,6 @@ void llama_sample_temp(struct llama_context * ctx, llama_token_data_array * cand } } -void llama_sample_temperature(struct llama_context * ctx, llama_token_data_array * candidates_p, float temp) { - llama_sample_temp(ctx, candidates_p, temp); -} - void llama_sample_repetition_penalties( struct llama_context * ctx, llama_token_data_array * candidates, @@ -10192,38 +10188,6 @@ void llama_sample_apply_guidance( ctx->t_sample_us += ggml_time_us() - t_start_sample_us; } -void llama_sample_classifier_free_guidance( - struct llama_context * ctx, - llama_token_data_array * candidates, - struct llama_context * guidance_ctx, - float scale) { - GGML_ASSERT(ctx); - int64_t t_start_sample_us; - - t_start_sample_us = ggml_time_us(); - const size_t n_vocab = llama_n_vocab(llama_get_model(ctx)); - - GGML_ASSERT(n_vocab == candidates->size); - GGML_ASSERT(!candidates->sorted); - - std::vector logits_base(n_vocab); - for (size_t i = 0; i < n_vocab; ++i) { - logits_base[i] = candidates->data[i].logit; - } - - float * logits_guidance = llama_get_logits(guidance_ctx); - - ctx->t_sample_us += ggml_time_us() - t_start_sample_us; - llama_sample_apply_guidance(ctx, logits_base.data(), logits_guidance, scale); - t_start_sample_us = ggml_time_us(); - - for (size_t i = 0; i < n_vocab; ++i) { - candidates->data[i].logit = logits_base[i]; - } - - ctx->t_sample_us += ggml_time_us() - t_start_sample_us; -} - llama_token llama_sample_token_mirostat(struct llama_context * ctx, llama_token_data_array * candidates, float tau, float eta, int32_t m, float * mu) { GGML_ASSERT(ctx); @@ -11724,15 +11688,6 @@ bool llama_supports_gpu_offload(void) { #endif } -// deprecated: -bool llama_mmap_supported(void) { - return llama_supports_mmap(); -} - -bool llama_mlock_supported(void) { - return llama_supports_mlock(); -} - void llama_backend_init(void) { ggml_time_init(); @@ -12244,15 +12199,6 @@ uint32_t llama_model_quantize( } } -int32_t llama_apply_lora_from_file(struct llama_context * ctx, const char * path_lora, float scale, const char * path_base_model, int32_t n_threads) { - try { - return llama_apply_lora_from_file_internal(ctx->model, path_lora, scale, path_base_model, n_threads); - } catch (const std::exception & err) { - LLAMA_LOG_ERROR("%s: failed to apply lora adapter: %s\n", __func__, err.what()); - return 1; - } -} - int32_t llama_model_apply_lora_from_file(const struct llama_model * model, const char * path_lora, float scale, const char * path_base_model, int32_t n_threads) { try { return llama_apply_lora_from_file_internal(*model, path_lora, scale, path_base_model, n_threads); @@ -12802,38 +12748,6 @@ bool llama_save_session_file(struct llama_context * ctx, const char * path_sessi return true; } -int llama_eval( - struct llama_context * ctx, - llama_token * tokens, - int32_t n_tokens, - int32_t n_past) { - llama_kv_cache_seq_rm(ctx->kv_self, -1, n_past, -1); - - const int ret = llama_decode_internal(*ctx, llama_batch_get_one(tokens, n_tokens, n_past, 0)); - if (ret < 0) { - LLAMA_LOG_ERROR("%s: failed to decode, ret = %d\n", __func__, ret); - } - - return ret; -} - -int llama_eval_embd( - struct llama_context * ctx, - float * embd, - int32_t n_tokens, - int32_t n_past) { - llama_kv_cache_seq_rm(ctx->kv_self, -1, n_past, -1); - - llama_batch batch = { n_tokens, nullptr, embd, nullptr, nullptr, nullptr, nullptr, n_past, 1, 0, }; - - const int ret = llama_decode_internal(*ctx, batch); - if (ret < 0) { - LLAMA_LOG_ERROR("%s: failed to decode, ret = %d\n", __func__, ret); - } - - return ret; -} - void llama_set_n_threads(struct llama_context * ctx, uint32_t n_threads, uint32_t n_threads_batch) { ctx->cparams.n_threads = n_threads; ctx->cparams.n_threads_batch = n_threads_batch; diff --git a/llama.h b/llama.h index 16e28e91d..a6823bb2b 100644 --- a/llama.h +++ b/llama.h @@ -364,9 +364,6 @@ extern "C" { LLAMA_API bool llama_supports_mlock (void); LLAMA_API bool llama_supports_gpu_offload(void); - LLAMA_API DEPRECATED(bool llama_mmap_supported (void), "use llama_supports_mmap() instead"); - LLAMA_API DEPRECATED(bool llama_mlock_supported(void), "use llama_supports_mlock() instead"); - LLAMA_API const struct llama_model * llama_get_model(const struct llama_context * ctx); LLAMA_API uint32_t llama_n_ctx (const struct llama_context * ctx); @@ -423,14 +420,6 @@ extern "C" { // The model needs to be reloaded before applying a new adapter, otherwise the adapter // will be applied on top of the previous one // Returns 0 on success - LLAMA_API DEPRECATED(int32_t llama_apply_lora_from_file( - struct llama_context * ctx, - const char * path_lora, - float scale, - const char * path_base_model, - int32_t n_threads), - "use llama_model_apply_lora_from_file instead"); - LLAMA_API int32_t llama_model_apply_lora_from_file( const struct llama_model * model, const char * path_lora, @@ -606,27 +595,6 @@ extern "C" { // Decoding // - // Run the llama inference to obtain the logits and probabilities for the next token(s). - // tokens + n_tokens is the provided batch of new tokens to process - // n_past is the number of tokens to use from previous eval calls - // Returns 0 on success - // DEPRECATED: use llama_decode() instead - LLAMA_API DEPRECATED(int llama_eval( - struct llama_context * ctx, - llama_token * tokens, - int32_t n_tokens, - int32_t n_past), - "use llama_decode() instead"); - - // Same as llama_eval, but use float matrix input directly. - // DEPRECATED: use llama_decode() instead - LLAMA_API DEPRECATED(int llama_eval_embd( - struct llama_context * ctx, - float * embd, - int32_t n_tokens, - int32_t n_past), - "use llama_decode() instead"); - // Return batch for single sequence of tokens starting at pos_0 // // NOTE: this is a helper function to facilitate transition to the new batch API - avoid using it @@ -800,13 +768,6 @@ extern "C" { float * logits_guidance, float scale); - LLAMA_API DEPRECATED(void llama_sample_classifier_free_guidance( - struct llama_context * ctx, - llama_token_data_array * candidates, - struct llama_context * guidance_ctx, - float scale), - "use llama_sample_apply_guidance() instead"); - /// @details Sorts candidate tokens by their logits in descending order and calculate probabilities based on logits. LLAMA_API void llama_sample_softmax( struct llama_context * ctx, @@ -860,12 +821,6 @@ extern "C" { llama_token_data_array * candidates, float temp); - LLAMA_API DEPRECATED(void llama_sample_temperature( - struct llama_context * ctx, - llama_token_data_array * candidates, - float temp), - "use llama_sample_temp instead"); - /// @details Apply constraints from grammar LLAMA_API void llama_sample_grammar( struct llama_context * ctx, From 317709b2a81dbaf87850202686ec5bb2602a504e Mon Sep 17 00:00:00 2001 From: Eve <139727413+netrunnereve@users.noreply.github.com> Date: Wed, 28 Feb 2024 19:33:37 +0000 Subject: [PATCH 490/812] make portability_enumeration_ext apple only (#5757) --- ggml-vulkan.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/ggml-vulkan.cpp b/ggml-vulkan.cpp index 314e3d7a9..896c290b2 100644 --- a/ggml-vulkan.cpp +++ b/ggml-vulkan.cpp @@ -1106,7 +1106,9 @@ void ggml_vk_instance_init() { const std::vector instance_extensions = vk::enumerateInstanceExtensionProperties(); const bool validation_ext = ggml_vk_instance_validation_ext_available(instance_extensions); +#ifdef __APPLE__ const bool portability_enumeration_ext = ggml_vk_instance_portability_enumeration_ext_available(instance_extensions); +#endif std::vector layers; @@ -1117,13 +1119,17 @@ void ggml_vk_instance_init() { if (validation_ext) { extensions.push_back("VK_EXT_validation_features"); } +#ifdef __APPLE__ if (portability_enumeration_ext) { extensions.push_back("VK_KHR_portability_enumeration"); } +#endif vk::InstanceCreateInfo instance_create_info(vk::InstanceCreateFlags{}, &app_info, layers, extensions); +#ifdef __APPLE__ if (portability_enumeration_ext) { instance_create_info.flags |= vk::InstanceCreateFlagBits::eEnumeratePortabilityKHR; } +#endif std::vector features_enable; vk::ValidationFeaturesEXT validation_features; From 87c91c07663b707e831c59ec373b5e665ff9d64a Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Wed, 28 Feb 2024 21:44:21 +0200 Subject: [PATCH 491/812] ci : reduce 3b ppl chunks to 1 to avoid timeout (#5771) ggml-ci --- ci/run.sh | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/ci/run.sh b/ci/run.sh index f3a29c2e9..35eb3c7aa 100755 --- a/ci/run.sh +++ b/ci/run.sh @@ -272,19 +272,19 @@ function gg_run_open_llama_3b_v2 { (time ./bin/main --model ${model_q5_k} -s 1234 -n 64 --ignore-eos -p "I believe the meaning of life is" ) 2>&1 | tee -a $OUT/${ci}-tg-q5_k.log (time ./bin/main --model ${model_q6_k} -s 1234 -n 64 --ignore-eos -p "I believe the meaning of life is" ) 2>&1 | tee -a $OUT/${ci}-tg-q6_k.log - (time ./bin/perplexity --model ${model_f16} -f ${wiki_test_60} -c 128 -b 128 --chunks 2 ) 2>&1 | tee -a $OUT/${ci}-tg-f16.log - (time ./bin/perplexity --model ${model_q8_0} -f ${wiki_test_60} -c 128 -b 128 --chunks 2 ) 2>&1 | tee -a $OUT/${ci}-tg-q8_0.log - (time ./bin/perplexity --model ${model_q4_0} -f ${wiki_test_60} -c 128 -b 128 --chunks 2 ) 2>&1 | tee -a $OUT/${ci}-tg-q4_0.log - (time ./bin/perplexity --model ${model_q4_1} -f ${wiki_test_60} -c 128 -b 128 --chunks 2 ) 2>&1 | tee -a $OUT/${ci}-tg-q4_1.log - (time ./bin/perplexity --model ${model_q5_0} -f ${wiki_test_60} -c 128 -b 128 --chunks 2 ) 2>&1 | tee -a $OUT/${ci}-tg-q5_0.log - (time ./bin/perplexity --model ${model_q5_1} -f ${wiki_test_60} -c 128 -b 128 --chunks 2 ) 2>&1 | tee -a $OUT/${ci}-tg-q5_1.log - (time ./bin/perplexity --model ${model_q2_k} -f ${wiki_test_60} -c 128 -b 128 --chunks 2 ) 2>&1 | tee -a $OUT/${ci}-tg-q2_k.log - (time ./bin/perplexity --model ${model_q3_k} -f ${wiki_test_60} -c 128 -b 128 --chunks 2 ) 2>&1 | tee -a $OUT/${ci}-tg-q3_k.log - (time ./bin/perplexity --model ${model_q4_k} -f ${wiki_test_60} -c 128 -b 128 --chunks 2 ) 2>&1 | tee -a $OUT/${ci}-tg-q4_k.log - (time ./bin/perplexity --model ${model_q5_k} -f ${wiki_test_60} -c 128 -b 128 --chunks 2 ) 2>&1 | tee -a $OUT/${ci}-tg-q5_k.log - (time ./bin/perplexity --model ${model_q6_k} -f ${wiki_test_60} -c 128 -b 128 --chunks 2 ) 2>&1 | tee -a $OUT/${ci}-tg-q6_k.log + (time ./bin/perplexity --model ${model_f16} -f ${wiki_test_60} -c 128 -b 128 --chunks 1 ) 2>&1 | tee -a $OUT/${ci}-tg-f16.log + (time ./bin/perplexity --model ${model_q8_0} -f ${wiki_test_60} -c 128 -b 128 --chunks 1 ) 2>&1 | tee -a $OUT/${ci}-tg-q8_0.log + (time ./bin/perplexity --model ${model_q4_0} -f ${wiki_test_60} -c 128 -b 128 --chunks 1 ) 2>&1 | tee -a $OUT/${ci}-tg-q4_0.log + (time ./bin/perplexity --model ${model_q4_1} -f ${wiki_test_60} -c 128 -b 128 --chunks 1 ) 2>&1 | tee -a $OUT/${ci}-tg-q4_1.log + (time ./bin/perplexity --model ${model_q5_0} -f ${wiki_test_60} -c 128 -b 128 --chunks 1 ) 2>&1 | tee -a $OUT/${ci}-tg-q5_0.log + (time ./bin/perplexity --model ${model_q5_1} -f ${wiki_test_60} -c 128 -b 128 --chunks 1 ) 2>&1 | tee -a $OUT/${ci}-tg-q5_1.log + (time ./bin/perplexity --model ${model_q2_k} -f ${wiki_test_60} -c 128 -b 128 --chunks 1 ) 2>&1 | tee -a $OUT/${ci}-tg-q2_k.log + (time ./bin/perplexity --model ${model_q3_k} -f ${wiki_test_60} -c 128 -b 128 --chunks 1 ) 2>&1 | tee -a $OUT/${ci}-tg-q3_k.log + (time ./bin/perplexity --model ${model_q4_k} -f ${wiki_test_60} -c 128 -b 128 --chunks 1 ) 2>&1 | tee -a $OUT/${ci}-tg-q4_k.log + (time ./bin/perplexity --model ${model_q5_k} -f ${wiki_test_60} -c 128 -b 128 --chunks 1 ) 2>&1 | tee -a $OUT/${ci}-tg-q5_k.log + (time ./bin/perplexity --model ${model_q6_k} -f ${wiki_test_60} -c 128 -b 128 --chunks 1 ) 2>&1 | tee -a $OUT/${ci}-tg-q6_k.log - (time ./bin/imatrix --model ${model_f16} -f ${wiki_test_60} -c 128 -b 128 --chunks 2 ) 2>&1 | tee -a $OUT/${ci}-imatrix.log + (time ./bin/imatrix --model ${model_f16} -f ${wiki_test_60} -c 128 -b 128 --chunks 1 ) 2>&1 | tee -a $OUT/${ci}-imatrix.log (time ./bin/save-load-state --model ${model_q4_0} ) 2>&1 | tee -a $OUT/${ci}-save-load-state.log @@ -343,17 +343,17 @@ function gg_run_open_llama_3b_v2 { python3 ../convert-lora-to-ggml.py ${path_lora} # f16 - (time ./bin/perplexity --model ${model_f16} -f ${shakespeare} -c 128 -b 128 --chunks 2 ) 2>&1 | tee -a $OUT/${ci}-ppl-shakespeare-f16.log - (time ./bin/perplexity --model ${model_f16} -f ${shakespeare} --lora ${lora_shakespeare} -c 128 -b 128 --chunks 2 ) 2>&1 | tee -a $OUT/${ci}-ppl-shakespeare-lora-f16.log + (time ./bin/perplexity --model ${model_f16} -f ${shakespeare} -c 128 -b 128 --chunks 1 ) 2>&1 | tee -a $OUT/${ci}-ppl-shakespeare-f16.log + (time ./bin/perplexity --model ${model_f16} -f ${shakespeare} --lora ${lora_shakespeare} -c 128 -b 128 --chunks 1 ) 2>&1 | tee -a $OUT/${ci}-ppl-shakespeare-lora-f16.log compare_ppl "f16 shakespeare" "$(cat $OUT/${ci}-ppl-shakespeare-f16.log | grep "^\[1\]")" "$(cat $OUT/${ci}-ppl-shakespeare-lora-f16.log | grep "^\[1\]")" | tee -a $OUT/${ci}-lora-ppl.log # q8_0 - (time ./bin/perplexity --model ${model_q8_0} -f ${shakespeare} -c 128 -b 128 --chunks 2 ) 2>&1 | tee -a $OUT/${ci}-ppl-shakespeare-q8_0.log - (time ./bin/perplexity --model ${model_q8_0} -f ${shakespeare} --lora ${lora_shakespeare} -c 128 -b 128 --chunks 2 ) 2>&1 | tee -a $OUT/${ci}-ppl-shakespeare-lora-q8_0.log + (time ./bin/perplexity --model ${model_q8_0} -f ${shakespeare} -c 128 -b 128 --chunks 1 ) 2>&1 | tee -a $OUT/${ci}-ppl-shakespeare-q8_0.log + (time ./bin/perplexity --model ${model_q8_0} -f ${shakespeare} --lora ${lora_shakespeare} -c 128 -b 128 --chunks 1 ) 2>&1 | tee -a $OUT/${ci}-ppl-shakespeare-lora-q8_0.log compare_ppl "q8_0 shakespeare" "$(cat $OUT/${ci}-ppl-shakespeare-q8_0.log | grep "^\[1\]")" "$(cat $OUT/${ci}-ppl-shakespeare-lora-q8_0.log | grep "^\[1\]")" | tee -a $OUT/${ci}-lora-ppl.log # q8_0 + f16 lora-base - (time ./bin/perplexity --model ${model_q8_0} -f ${shakespeare} --lora ${lora_shakespeare} --lora-base ${model_f16} -c 128 -b 128 --chunks 2 ) 2>&1 | tee -a $OUT/${ci}-ppl-shakespeare-lora-q8_0-f16.log + (time ./bin/perplexity --model ${model_q8_0} -f ${shakespeare} --lora ${lora_shakespeare} --lora-base ${model_f16} -c 128 -b 128 --chunks 1 ) 2>&1 | tee -a $OUT/${ci}-ppl-shakespeare-lora-q8_0-f16.log compare_ppl "q8_0 / f16 base shakespeare" "$(cat $OUT/${ci}-ppl-shakespeare-q8_0.log | grep "^\[1\]")" "$(cat $OUT/${ci}-ppl-shakespeare-lora-q8_0-f16.log | grep "^\[1\]")" | tee -a $OUT/${ci}-lora-ppl.log set +e From d5ab29757ebc59a30f03e408294ec20628a6374e Mon Sep 17 00:00:00 2001 From: Marcus Dunn <51931484+MarcusDunn@users.noreply.github.com> Date: Thu, 29 Feb 2024 00:17:23 -0800 Subject: [PATCH 492/812] llama : constified `llama_set_state_data`'s `src` (#5774) --- llama.cpp | 6 +++--- llama.h | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/llama.cpp b/llama.cpp index 30d5eb32d..62699ce52 100644 --- a/llama.cpp +++ b/llama.cpp @@ -12545,8 +12545,8 @@ size_t llama_copy_state_data(struct llama_context * ctx, uint8_t * dst) { } // Sets the state reading from the specified source address -size_t llama_set_state_data(struct llama_context * ctx, uint8_t * src) { - uint8_t * inp = src; +size_t llama_set_state_data(struct llama_context * ctx, const uint8_t * src) { + const uint8_t * inp = src; // set rng { @@ -12555,7 +12555,7 @@ size_t llama_set_state_data(struct llama_context * ctx, uint8_t * src) { GGML_ASSERT(rng_size <= LLAMA_MAX_RNG_STATE); - std::string rng_str((char *)inp, rng_size); inp += rng_size; + std::string rng_str((const char *)inp, rng_size); inp += rng_size; std::istringstream rng_ss(rng_str); rng_ss >> ctx->rng; diff --git a/llama.h b/llama.h index a6823bb2b..4d0ebe37d 100644 --- a/llama.h +++ b/llama.h @@ -575,7 +575,7 @@ extern "C" { // Returns the number of bytes read LLAMA_API size_t llama_set_state_data( struct llama_context * ctx, - uint8_t * src); + const uint8_t * src); // Save/load session file LLAMA_API bool llama_load_session_file( From 052051d8ae4639a1c3c61e7da3237bcc572469d4 Mon Sep 17 00:00:00 2001 From: Xuan Son Nguyen Date: Thu, 29 Feb 2024 21:42:11 +0100 Subject: [PATCH 493/812] Server: normalize naming (#5779) * server: normalize naming * fix spacing --- examples/server/server.cpp | 370 ++++++++++++++++--------------------- examples/server/utils.hpp | 186 ++++++++++++------- 2 files changed, 277 insertions(+), 279 deletions(-) diff --git a/examples/server/server.cpp b/examples/server/server.cpp index 080fa9bd5..bf20e0cf1 100644 --- a/examples/server/server.cpp +++ b/examples/server/server.cpp @@ -33,8 +33,7 @@ using json = nlohmann::json; -struct server_params -{ +struct server_params { std::string hostname = "127.0.0.1"; std::vector api_keys; std::string public_path = "examples/server/public"; @@ -49,103 +48,50 @@ struct server_params bool server_verbose = false; bool server_log_json = true; -static size_t common_part(const std::vector &a, const std::vector &b) -{ - size_t i; - for (i = 0; i < a.size() && i < b.size() && a[i] == b[i]; i++) - { - } - return i; -} - -enum stop_type -{ +enum stop_type { STOP_FULL, STOP_PARTIAL, }; -static bool ends_with(const std::string &str, const std::string &suffix) -{ - return str.size() >= suffix.size() && - 0 == str.compare(str.size() - suffix.size(), suffix.size(), suffix); -} +// TODO: can become bool if we can't find use of more states +enum slot_state { + IDLE, + PROCESSING, +}; -static size_t find_partial_stop_string(const std::string &stop, - const std::string &text) -{ - if (!text.empty() && !stop.empty()) - { - const char text_last_char = text.back(); - for (int64_t char_index = stop.size() - 1; char_index >= 0; char_index--) - { - if (stop[char_index] == text_last_char) - { - const std::string current_partial = stop.substr(0, char_index + 1); - if (ends_with(text, current_partial)) - { - return text.size() - char_index - 1; - } - } - } - } - return std::string::npos; -} +enum slot_command { + NONE, + LOAD_PROMPT, + RELEASE, +}; -// TODO: reuse llama_detokenize -template -static std::string tokens_to_str(llama_context *ctx, Iter begin, Iter end) -{ - std::string ret; - for (; begin != end; ++begin) - { - ret += llama_token_to_piece(ctx, *begin); - } - return ret; -} +struct slot_params { + bool stream = true; + bool cache_prompt = false; // remember the prompt to avoid reprocessing all prompt -// format incomplete utf-8 multibyte character for output -static std::string tokens_to_output_formatted_string(const llama_context *ctx, const llama_token token) -{ - std::string out = token == -1 ? "" : llama_token_to_piece(ctx, token); - // if the size is 1 and first bit is 1, meaning it's a partial character - // (size > 1 meaning it's already a known token) - if (out.size() == 1 && (out[0] & 0x80) == 0x80) - { - std::stringstream ss; - ss << std::hex << (out[0] & 0xff); - std::string res(ss.str()); - out = "byte: \\x" + res; - } - return out; -} + uint32_t seed = -1; // RNG seed + int32_t n_keep = 0; // number of tokens to keep from initial prompt + int32_t n_predict = -1; // new tokens to predict -// convert a vector of completion_token_output to json -static json probs_vector_to_json(const llama_context *ctx, const std::vector &probs) -{ - json out = json::array(); - for (const auto &prob : probs) - { - json probs_for_token = json::array(); - for (const auto &p : prob.probs) - { - std::string tok_str = tokens_to_output_formatted_string(ctx, p.tok); - probs_for_token.push_back(json - { - {"tok_str", tok_str}, - {"prob", p.prob}, - }); - } - std::string tok_str = tokens_to_output_formatted_string(ctx, prob.tok); - out.push_back(json{ - {"content", tok_str}, - {"probs", probs_for_token}, - }); - } - return out; -} + std::vector antiprompt; -struct llama_client_slot -{ + json input_prefix; + json input_suffix; +}; + +struct slot_image { + int32_t id; + + bool request_encode_image = false; + float * image_embedding = nullptr; + int32_t image_tokens = 0; + + clip_image_u8 * img_data; + + std::string prefix_prompt; // before of this image +}; + +struct server_slot { int id; int task_id = -1; @@ -165,8 +111,8 @@ struct llama_client_slot int32_t i_batch = -1; int32_t n_predict = -1; - int32_t num_prompt_tokens = 0; - int32_t num_prompt_tokens_processed = 0; + int32_t n_prompt_tokens = 0; + int32_t n_prompt_tokens_processed = 0; json prompt; std::string generated_text; @@ -201,8 +147,8 @@ struct llama_client_slot std::vector images; // stats - size_t sent_count = 0; - size_t sent_token_probs_index = 0; + size_t n_sent_text = 0; // number of sent text character + size_t n_sent_token_probs = 0; int64_t t_start_process_prompt; int64_t t_start_genereration; @@ -214,7 +160,7 @@ struct llama_client_slot int multitask_id = -1; void reset() { - num_prompt_tokens = 0; + n_prompt_tokens = 0; generated_text = ""; truncated = false; stopped_eos = false; @@ -222,16 +168,15 @@ struct llama_client_slot stopped_limit = false; stopping_word = ""; n_past = 0; - sent_count = 0; - sent_token_probs_index = 0; + n_sent_text = 0; + n_sent_token_probs = 0; infill = false; ga_i = 0; n_past_se = 0; generated_token_probs.clear(); - for (slot_image & img : images) - { + for (slot_image & img : images) { free(img.image_embedding); if (img.img_data) { clip_image_u8_free(img.img_data); @@ -243,19 +188,15 @@ struct llama_client_slot } bool has_budget(gpt_params &global_params) { - if (params.n_predict == -1 && global_params.n_predict == -1) - { + if (params.n_predict == -1 && global_params.n_predict == -1) { return true; // limitless } n_remaining = -1; - if (params.n_predict != -1) - { + if (params.n_predict != -1) { n_remaining = params.n_predict - n_decoded; - } - else if (global_params.n_predict != -1) - { + } else if (global_params.n_predict != -1) { n_remaining = global_params.n_predict - n_decoded; } @@ -271,8 +212,7 @@ struct llama_client_slot } void add_token_string(const completion_token_output &token) { - if (command == RELEASE) - { + if (command == RELEASE) { return; } cache_tokens.push_back(token.tok); @@ -290,10 +230,10 @@ struct llama_client_slot json get_formated_timings() { return json { - {"prompt_n", num_prompt_tokens_processed}, + {"prompt_n", n_prompt_tokens_processed}, {"prompt_ms", t_prompt_processing}, - {"prompt_per_token_ms", t_prompt_processing / num_prompt_tokens_processed}, - {"prompt_per_second", 1e3 / t_prompt_processing * num_prompt_tokens_processed}, + {"prompt_per_token_ms", t_prompt_processing / n_prompt_tokens_processed}, + {"prompt_per_second", 1e3 / t_prompt_processing * n_prompt_tokens_processed}, {"predicted_n", n_decoded}, {"predicted_ms", t_token_generation}, @@ -304,18 +244,18 @@ struct llama_client_slot void print_timings() const { char buffer[512]; - double t_token = t_prompt_processing / num_prompt_tokens_processed; - double n_tokens_second = 1e3 / t_prompt_processing * num_prompt_tokens_processed; + double t_token = t_prompt_processing / n_prompt_tokens_processed; + double n_tokens_second = 1e3 / t_prompt_processing * n_prompt_tokens_processed; sprintf(buffer, "prompt eval time = %10.2f ms / %5d tokens (%8.2f ms per token, %8.2f tokens per second)", - t_prompt_processing, num_prompt_tokens_processed, + t_prompt_processing, n_prompt_tokens_processed, t_token, n_tokens_second); LOG_INFO(buffer, { - {"slot_id", id}, - {"task_id", task_id}, - {"t_prompt_processing", t_prompt_processing}, - {"num_prompt_tokens_processed", num_prompt_tokens_processed}, - {"t_token", t_token}, - {"n_tokens_second", n_tokens_second}, + {"slot_id", id}, + {"task_id", task_id}, + {"t_prompt_processing", t_prompt_processing}, + {"n_prompt_tokens_processed", n_prompt_tokens_processed}, + {"t_token", t_token}, + {"n_tokens_second", n_tokens_second}, }); t_token = t_token_generation / n_decoded; @@ -343,7 +283,7 @@ struct llama_client_slot } }; -struct llama_metrics { +struct server_metrics { uint64_t n_prompt_tokens_processed_total = 0; uint64_t n_tokens_predicted_total = 0; @@ -354,18 +294,16 @@ struct llama_metrics { uint64_t t_tokens_generation = 0; - void on_prompt_eval(const llama_client_slot &slot) { - n_prompt_tokens_processed_total += slot.num_prompt_tokens_processed; - - n_prompt_tokens_processed += slot.num_prompt_tokens_processed; - t_prompt_processing += slot.t_prompt_processing; + void on_prompt_eval(const server_slot &slot) { + n_prompt_tokens_processed_total += slot.n_prompt_tokens_processed; + n_prompt_tokens_processed += slot.n_prompt_tokens_processed; + t_prompt_processing += slot.t_prompt_processing; } - void on_prediction(const llama_client_slot &slot) { + void on_prediction(const server_slot &slot) { n_tokens_predicted_total += slot.n_decoded; - - n_tokens_predicted += slot.n_decoded; - t_tokens_generation += slot.t_token_generation; + n_tokens_predicted += slot.n_decoded; + t_tokens_generation += slot.t_token_generation; } void reset_bucket() { @@ -404,13 +342,13 @@ struct llama_server_context std::string name_assistant; // slots / clients - std::vector slots; + std::vector slots; json default_generation_settings_for_props; - llama_server_queue queue_tasks; + llama_server_queue queue_tasks; llama_server_response queue_results; - llama_metrics metrics; + server_metrics metrics; ~llama_server_context() { @@ -487,7 +425,7 @@ struct llama_server_context LOG_INFO("initializing slots", {{"n_slots", params.n_parallel}}); for (int i = 0; i < params.n_parallel; i++) { - llama_client_slot slot; + server_slot slot; slot.id = i; slot.n_ctx = n_ctx_slot; @@ -579,11 +517,11 @@ struct llama_server_context return prompt_tokens; } - llama_client_slot* get_slot(int id) { + server_slot* get_slot(int id) { int64_t t_last = ggml_time_us(); - llama_client_slot *last_used = nullptr; + server_slot *last_used = nullptr; - for (llama_client_slot & slot : slots) + for (server_slot & slot : slots) { if (slot.id == id && slot.available()) { @@ -600,7 +538,7 @@ struct llama_server_context return last_used; } - bool launch_slot_with_data(llama_client_slot* &slot, json data) { + bool launch_slot_with_data(server_slot* &slot, json data) { slot_params default_params; llama_sampling_params default_sparams; @@ -888,7 +826,7 @@ struct llama_server_context clean_kv_cache = false; } - void update_system_prompt() { + void system_prompt_update() { kv_cache_clear(); system_tokens.clear(); @@ -933,9 +871,9 @@ struct llama_server_context system_need_update = false; } - void notify_system_prompt_changed() { + void system_prompt_notify() { // release all slots - for (llama_client_slot &slot : slots) + for (server_slot &slot : slots) { slot.release(); } @@ -943,17 +881,17 @@ struct llama_server_context system_need_update = true; } - void process_system_prompt_data(const json &sys_props) { + void system_prompt_process(const json &sys_props) { system_prompt = sys_props.value("prompt", ""); name_user = sys_props.value("anti_prompt", ""); name_assistant = sys_props.value("assistant_name", ""); - notify_system_prompt_changed(); + system_prompt_notify(); } static size_t find_stopping_strings(const std::string &text, const size_t last_token_size, - const stop_type type, llama_client_slot &slot) + const stop_type type, server_slot &slot) { size_t stop_pos = std::string::npos; @@ -975,8 +913,8 @@ struct llama_server_context { if (type == STOP_FULL) { - slot.stopped_word = true; - slot.stopping_word = word; + slot.stopped_word = true; + slot.stopping_word = word; slot.has_next_token = false; } stop_pos = pos; @@ -986,7 +924,7 @@ struct llama_server_context return stop_pos; } - bool process_token(completion_token_output &result, llama_client_slot &slot) { + bool process_token(completion_token_output &result, server_slot &slot) { // remember which tokens were sampled - used for repetition penalties during sampling const std::string token_str = llama_token_to_piece(ctx, result.tok); slot.sampled = result.tok; @@ -1032,7 +970,7 @@ struct llama_server_context if (!incomplete) { - size_t pos = std::min(slot.sent_count, slot.generated_text.size()); + size_t pos = std::min(slot.n_sent_text, slot.generated_text.size()); const std::string str_test = slot.generated_text.substr(pos); bool is_stop_full = false; size_t stop_pos = find_stopping_strings(str_test, token_str.size(), STOP_FULL, slot); @@ -1042,7 +980,7 @@ struct llama_server_context slot.generated_text.erase( slot.generated_text.begin() + pos + stop_pos, slot.generated_text.end()); - pos = std::min(slot.sent_count, slot.generated_text.size()); + pos = std::min(slot.n_sent_text, slot.generated_text.size()); } else { @@ -1055,7 +993,7 @@ struct llama_server_context { // no send the stop word in the response result.text_to_send = slot.generated_text.substr(pos, std::string::npos); - slot.sent_count += result.text_to_send.size(); + slot.n_sent_text += result.text_to_send.size(); // add the token to slot queue and cache } slot.add_token_string(result); @@ -1099,7 +1037,7 @@ struct llama_server_context return slot.has_next_token; // continue } - bool process_images(llama_client_slot &slot) const + bool process_images(server_slot &slot) const { for (slot_image &img : slot.images) { @@ -1132,7 +1070,7 @@ struct llama_server_context queue_results.send(res); } - json get_formated_generation(llama_client_slot &slot) + json get_formated_generation(server_slot &slot) { const auto eos_bias = slot.sparams.logit_bias.find(llama_token_eos(model)); const bool ignore_eos = eos_bias != slot.sparams.logit_bias.end() && @@ -1179,7 +1117,7 @@ struct llama_server_context }; } - void send_partial_response(llama_client_slot &slot, completion_token_output tkn) + void send_partial_response(server_slot &slot, completion_token_output tkn) { task_result res; res.id = slot.task_id; @@ -1199,13 +1137,13 @@ struct llama_server_context { std::vector probs_output = {}; const std::vector to_send_toks = llama_tokenize(ctx, tkn.text_to_send, false); - size_t probs_pos = std::min(slot.sent_token_probs_index, slot.generated_token_probs.size()); - size_t probs_stop_pos = std::min(slot.sent_token_probs_index + to_send_toks.size(), slot.generated_token_probs.size()); + size_t probs_pos = std::min(slot.n_sent_token_probs, slot.generated_token_probs.size()); + size_t probs_stop_pos = std::min(slot.n_sent_token_probs + to_send_toks.size(), slot.generated_token_probs.size()); if (probs_pos < probs_stop_pos) { probs_output = std::vector(slot.generated_token_probs.begin() + probs_pos, slot.generated_token_probs.begin() + probs_stop_pos); } - slot.sent_token_probs_index = probs_stop_pos; + slot.n_sent_token_probs = probs_stop_pos; res.result_json["completion_probabilities"] = probs_vector_to_json(ctx, probs_output); } @@ -1218,7 +1156,7 @@ struct llama_server_context queue_results.send(res); } - void send_final_response(llama_client_slot &slot) + void send_final_response(server_slot &slot) { task_result res; res.id = slot.task_id; @@ -1233,7 +1171,7 @@ struct llama_server_context {"stop", true}, {"model", params.model_alias}, {"tokens_predicted", slot.n_decoded}, - {"tokens_evaluated", slot.num_prompt_tokens}, + {"tokens_evaluated", slot.n_prompt_tokens}, {"generation_settings", get_formated_generation(slot)}, {"prompt", slot.prompt}, {"truncated", slot.truncated}, @@ -1271,7 +1209,7 @@ struct llama_server_context queue_results.send(res); } - void send_embedding(llama_client_slot &slot) + void send_embedding(server_slot &slot) { task_result res; res.id = slot.task_id; @@ -1282,9 +1220,7 @@ struct llama_server_context const int n_embd = llama_n_embd(model); if (!params.embedding) { - LOG_WARNING("embedding disabled", { - {"params.embedding", params.embedding}, - }); + LOG_WARNING("embedding disabled", {{"params.embedding", params.embedding}}); res.result_json = json { {"embedding", std::vector(n_embd, 0.0f)}, @@ -1296,7 +1232,7 @@ struct llama_server_context std::vector embedding(data, data + n_embd); res.result_json = json { - {"embedding", embedding }, + {"embedding", embedding}, }; } queue_results.send(res); @@ -1345,7 +1281,7 @@ struct llama_server_context } // for multiple images processing - bool ingest_images(llama_client_slot &slot, int n_batch) + bool ingest_images(server_slot &slot, int n_batch) { int image_idx = 0; @@ -1384,7 +1320,17 @@ struct llama_server_context } const int n_embd = llama_n_embd(model); - llama_batch batch_img = { n_eval, nullptr, (img.image_embedding + i * n_embd), nullptr, nullptr, nullptr, nullptr, slot.n_past, 1, 0, }; + llama_batch batch_img = { + n_eval, + nullptr, + (img.image_embedding + i * n_embd), + nullptr, + nullptr, + nullptr, + nullptr, + slot.n_past, + 1, 0 + }; if (llama_decode(ctx, batch_img)) { LOG_TEE("%s : failed to eval image\n", __func__); @@ -1454,7 +1400,7 @@ struct llama_server_context switch (task.type) { case TASK_TYPE_COMPLETION: { - llama_client_slot *slot = get_slot(json_value(task.data, "slot_id", -1)); + server_slot *slot = get_slot(json_value(task.data, "slot_id", -1)); if (slot == nullptr) { // if no slot is available, we defer this task for processing later @@ -1469,10 +1415,10 @@ struct llama_server_context send_error(task, "system prompt can only be updated when all slots are idle"); break; } - process_system_prompt_data(task.data["system_prompt"]); + system_prompt_process(task.data["system_prompt"]); // reset cache_tokens for all slots - for (llama_client_slot &slot : slots) + for (server_slot &slot : slots) { slot.cache_tokens.clear(); slot.n_past = 0; @@ -1512,20 +1458,20 @@ struct llama_server_context int n_idle_slots = 0; int n_processing_slots = 0; - for (llama_client_slot &slot: slots) { + for (server_slot &slot: slots) { json slot_data = get_formated_generation(slot); slot_data["id"] = slot.id; slot_data["task_id"] = slot.task_id; slot_data["state"] = slot.state; slot_data["prompt"] = slot.prompt; slot_data["next_token"] = { - {"has_next_token", slot.has_next_token}, - {"n_remain", slot.n_remaining}, + {"has_next_token", slot.has_next_token}, + {"n_remain", slot.n_remaining}, {"num_tokens_predicted", slot.n_decoded}, - {"stopped_eos", slot.stopped_eos}, - {"stopped_word", slot.stopped_word}, - {"stopped_limit", slot.stopped_limit}, - {"stopping_word", slot.stopping_word}, + {"stopped_eos", slot.stopped_eos}, + {"stopped_word", slot.stopped_word}, + {"stopped_limit", slot.stopped_limit}, + {"stopping_word", slot.stopping_word}, }; if (slot_data["state"] == IDLE) { n_idle_slots++; @@ -1563,10 +1509,10 @@ struct llama_server_context { "n_tokens_predicted", metrics.n_tokens_predicted}, { "t_tokens_generation", metrics.t_tokens_generation}, - { "kv_cache_tokens_count", llama_get_kv_cache_token_count(ctx)}, - { "kv_cache_used_cells", llama_get_kv_cache_used_cells(ctx)}, + { "kv_cache_tokens_count", llama_get_kv_cache_token_count(ctx)}, + { "kv_cache_used_cells", llama_get_kv_cache_used_cells(ctx)}, - { "slots", slots_data }, + { "slots", slots_data }, }; metrics.reset_bucket(); queue_results.send(res); @@ -1597,7 +1543,7 @@ struct llama_server_context if (system_need_update) { LOG_INFO("updating system prompt", {}); - update_system_prompt(); + system_prompt_update(); } llama_batch_clear(batch); @@ -1618,7 +1564,7 @@ struct llama_server_context task.target_id = -1; queue_tasks.post(task); - for (llama_client_slot &slot : slots) + for (server_slot &slot : slots) { if (slot.ga_n == 1) { @@ -1754,45 +1700,50 @@ struct llama_server_context prompt_tokens = tokenize(slot.prompt, system_prompt.empty() && add_bos_token); // add BOS if there isn't system prompt } - slot.num_prompt_tokens = prompt_tokens.size(); + slot.n_prompt_tokens = prompt_tokens.size(); if (slot.params.n_keep < 0) { - slot.params.n_keep = slot.num_prompt_tokens; + slot.params.n_keep = slot.n_prompt_tokens; } slot.params.n_keep = std::min(slot.n_ctx - 4, slot.params.n_keep); // if input prompt is too big, truncate it - if (slot.num_prompt_tokens >= slot.n_ctx) + if (slot.n_prompt_tokens >= slot.n_ctx) { const int n_left = slot.n_ctx - slot.params.n_keep; const int n_block_size = n_left / 2; - const int erased_blocks = (slot.num_prompt_tokens - slot.params.n_keep - n_block_size) / n_block_size; + const int erased_blocks = (slot.n_prompt_tokens - slot.params.n_keep - n_block_size) / n_block_size; - std::vector new_tokens(prompt_tokens.begin(), prompt_tokens.begin() + slot.params.n_keep); - new_tokens.insert(new_tokens.end(), prompt_tokens.begin() + slot.params.n_keep + erased_blocks * n_block_size, prompt_tokens.end()); + std::vector new_tokens( + prompt_tokens.begin(), + prompt_tokens.begin() + slot.params.n_keep); + new_tokens.insert( + new_tokens.end(), + prompt_tokens.begin() + slot.params.n_keep + erased_blocks * n_block_size, + prompt_tokens.end()); LOG_VERBOSE("input truncated", { - {"n_ctx", slot.n_ctx}, - {"n_keep", slot.params.n_keep}, - {"n_left", n_left}, + {"n_ctx", slot.n_ctx}, + {"n_keep", slot.params.n_keep}, + {"n_left", n_left}, {"new_tokens", tokens_to_str(ctx, new_tokens.cbegin(), new_tokens.cend())}, }); slot.truncated = true; prompt_tokens = new_tokens; - slot.num_prompt_tokens = prompt_tokens.size(); - GGML_ASSERT(slot.num_prompt_tokens < slot.n_ctx); + slot.n_prompt_tokens = prompt_tokens.size(); + GGML_ASSERT(slot.n_prompt_tokens < slot.n_ctx); } if (!slot.params.cache_prompt) { llama_sampling_reset(slot.ctx_sampling); - slot.n_past = 0; + slot.n_past = 0; slot.n_past_se = 0; - slot.ga_i = 0; - slot.num_prompt_tokens_processed = slot.num_prompt_tokens; + slot.ga_i = 0; + slot.n_prompt_tokens_processed = slot.n_prompt_tokens; } else { @@ -1811,7 +1762,7 @@ struct llama_server_context slot.n_past -= 1; } - slot.num_prompt_tokens_processed = slot.num_prompt_tokens - slot.n_past; + slot.n_prompt_tokens_processed = slot.n_prompt_tokens - slot.n_past; if (slot.ga_n != 1) { @@ -1836,13 +1787,13 @@ struct llama_server_context { "slot_id", slot.id }, { "task_id", slot.task_id }, { "n_past", slot.n_past }, - { "num_prompt_tokens_processed", slot.num_prompt_tokens_processed } + { "n_prompt_tokens_processed", slot.n_prompt_tokens_processed } }); } slot.cache_tokens = prompt_tokens; - if (slot.n_past == slot.num_prompt_tokens && slot.n_past > 0) + if (slot.n_past == slot.n_prompt_tokens && slot.n_past > 0) { // we have to evaluate at least 1 token to generate logits. LOG_INFO("we have to evaluate at least 1 token to generate logits", { @@ -1898,8 +1849,8 @@ struct llama_server_context if (has_images && !ingest_images(slot, n_batch)) { LOG_ERROR("failed processing images", { - "slot_id", slot.id, - "task_id", slot.task_id, + {"slot_id", slot.id}, + {"task_id", slot.task_id}, }); // FIXME @phymbert: to be properly tested // early returning without changing the slot state will block the slot for ever @@ -2049,10 +2000,6 @@ struct llama_server_context LOG_VERBOSE("slots updated", {}); return true; } - - void run_on_all_tasks_finished() { - update_slots(); - } }; static void server_print_usage(const char *argv0, const gpt_params ¶ms, @@ -2561,7 +2508,7 @@ static void server_params_parse(int argc, char **argv, server_params &sparams, std::istreambuf_iterator(), std::back_inserter(systm_content) ); - llama.process_system_prompt_data(json::parse(systm_content)); + llama.system_prompt_process(json::parse(systm_content)); } else if (arg == "-ctk" || arg == "--cache-type-k") { params.cache_type_k = argv[++i]; @@ -2692,7 +2639,7 @@ static void server_params_parse(int argc, char **argv, server_params &sparams, /* llama.cpp completion api semantics */ static json format_partial_response( - llama_server_context &llama, llama_client_slot *slot, const std::string &content, const std::vector &probs + llama_server_context &llama, server_slot *slot, const std::string &content, const std::vector &probs ) { json res = json { @@ -2748,14 +2695,7 @@ static void log_server_request(const httplib::Request &req, const httplib::Respo }); } -struct token_translator -{ - llama_context * ctx; - std::string operator()(llama_token tok) const { return llama_token_to_piece(ctx, tok); } - std::string operator()(const completion_token_output &cto) const { return (*this)(cto.tok); } -}; - -static void append_to_generated_text_from_generated_token_probs(llama_server_context &llama, llama_client_slot *slot) +static void append_to_generated_text_from_generated_token_probs(llama_server_context &llama, server_slot *slot) { auto & gtps = slot->generated_token_probs; auto translator = token_translator{llama.ctx}; @@ -3526,8 +3466,8 @@ int main(int argc, char **argv) &llama_server_context::process_single_task, &llama, std::placeholders::_1)); llama.queue_tasks.on_finish_multitask(std::bind( &llama_server_context::on_finish_multitask, &llama, std::placeholders::_1)); - llama.queue_tasks.on_all_tasks_finished(std::bind( - &llama_server_context::run_on_all_tasks_finished, &llama)); + llama.queue_tasks.on_run_slots(std::bind( + &llama_server_context::update_slots, &llama)); llama.queue_results.on_multitask_update(std::bind( &llama_server_queue::update_multitask, &llama.queue_tasks, diff --git a/examples/server/utils.hpp b/examples/server/utils.hpp index d7abd7cbb..d98541f26 100644 --- a/examples/server/utils.hpp +++ b/examples/server/utils.hpp @@ -37,10 +37,6 @@ extern bool server_log_json; #define LOG_WARNING(MSG, ...) server_log("WARN", __func__, __LINE__, MSG, __VA_ARGS__) #define LOG_INFO( MSG, ...) server_log("INFO", __func__, __LINE__, MSG, __VA_ARGS__) -// -// parallel -// - enum server_state { SERVER_STATE_LOADING_MODEL, // Server is starting up, model not fully loaded yet SERVER_STATE_READY, // Server is ready and model is loaded @@ -78,51 +74,8 @@ struct task_multi { std::vector results{}; }; -// TODO: can become bool if we can't find use of more states -enum slot_state -{ - IDLE, - PROCESSING, -}; - -enum slot_command -{ - NONE, - LOAD_PROMPT, - RELEASE, -}; - -struct slot_params -{ - bool stream = true; - bool cache_prompt = false; // remember the prompt to avoid reprocessing all prompt - - uint32_t seed = -1; // RNG seed - int32_t n_keep = 0; // number of tokens to keep from initial prompt - int32_t n_predict = -1; // new tokens to predict - - std::vector antiprompt; - - json input_prefix; - json input_suffix; -}; - -struct slot_image -{ - int32_t id; - - bool request_encode_image = false; - float * image_embedding = nullptr; - int32_t image_tokens = 0; - - clip_image_u8 * img_data; - - std::string prefix_prompt; // before of this image -}; - // completion token output with probabilities -struct completion_token_output -{ +struct completion_token_output { struct token_prob { llama_token tok; @@ -134,8 +87,13 @@ struct completion_token_output std::string text_to_send; }; -static inline void server_log(const char *level, const char *function, int line, const char *message, const nlohmann::ordered_json &extra) -{ +struct token_translator { + llama_context * ctx; + std::string operator()(llama_token tok) const { return llama_token_to_piece(ctx, tok); } + std::string operator()(const completion_token_output &cto) const { return (*this)(cto.tok); } +}; + +static inline void server_log(const char *level, const char *function, int line, const char *message, const nlohmann::ordered_json &extra) { std::stringstream ss_tid; ss_tid << std::this_thread::get_id(); json log = nlohmann::ordered_json{ @@ -183,8 +141,7 @@ static inline void server_log(const char *level, const char *function, int line, // template -static T json_value(const json &body, const std::string &key, const T &default_value) -{ +static T json_value(const json &body, const std::string &key, const T &default_value) { // Fallback null to default value return body.contains(key) && !body.at(key).is_null() ? body.value(key, default_value) @@ -200,8 +157,7 @@ inline bool verify_custom_template(const std::string & tmpl) { } // Format given chat. If tmpl is empty, we take the template from model metadata -inline std::string format_chat(const struct llama_model * model, const std::string & tmpl, const std::vector & messages) -{ +inline std::string format_chat(const struct llama_model * model, const std::string & tmpl, const std::vector & messages) { size_t alloc_size = 0; // vector holding all allocated string to be passed to llama_chat_apply_template std::vector str(messages.size() * 2); @@ -250,7 +206,7 @@ struct llama_server_queue { // callback functions std::function callback_new_task; std::function callback_finish_multitask; - std::function callback_all_task_finished; + std::function callback_run_slots; // Add a new task to the end of the queue int post(task_server task) { @@ -283,14 +239,14 @@ struct llama_server_queue { callback_new_task = callback; } - // Register function to process a multitask + // Register function to process a multitask when it is finished void on_finish_multitask(std::function callback) { callback_finish_multitask = callback; } - // Register the function to be called when the batch of tasks is finished - void on_all_tasks_finished(std::function callback) { - callback_all_task_finished = callback; + // Register the function to be called when all slots data is ready to be processed + void on_run_slots(std::function callback) { + callback_run_slots = callback; } // Call when the state of one slot is changed @@ -312,7 +268,13 @@ struct llama_server_queue { condition_tasks.notify_all(); } - // Start the main loop. + /** + * Main loop consists of these steps: + * - Wait until a new task arrives + * - Process the task (i.e. maybe copy data into slot) + * - Check if multitask is finished + * - Run all slots + */ void start_loop() { running = true; while (true) { @@ -331,8 +293,8 @@ struct llama_server_queue { LOG_VERBOSE("callback_new_task", {{"task_id", task.id}}); callback_new_task(task); } - LOG_VERBOSE("callback_all_task_finished", {}); - // process and update all the multitasks + LOG_VERBOSE("update_multitasks", {}); + // check if we have any finished multitasks auto queue_iterator = queue_multitasks.begin(); while (queue_iterator != queue_multitasks.end()) { @@ -349,8 +311,9 @@ struct llama_server_queue { ++queue_iterator; } } - // all tasks in the current loop is finished - callback_all_task_finished(); + // all tasks in the current loop is processed, slots data is now ready + LOG_VERBOSE("callback_run_slots", {}); + callback_run_slots(); } LOG_VERBOSE("wait for new task", {}); // wait for new task @@ -408,12 +371,14 @@ struct llama_server_response { std::mutex mutex_results; std::condition_variable condition_results; + // add the task_id to the list of tasks waiting for response void add_waiting_task_id(int task_id) { LOG_VERBOSE("waiting for task id", {{"task_id", task_id}}); std::unique_lock lock(mutex_results); waiting_task_ids.insert(task_id); } + // when the request is finished, we can remove task associated with it void remove_waiting_task_id(int task_id) { LOG_VERBOSE("remove waiting for task id", {{"task_id", task_id}}); std::unique_lock lock(mutex_results); @@ -574,3 +539,96 @@ static std::string gen_chatcmplid() chatcmplid << "chatcmpl-" << random_string(); return chatcmplid.str(); } + +// +// other common utils +// + +static size_t common_part(const std::vector &a, const std::vector &b) +{ + size_t i; + for (i = 0; i < a.size() && i < b.size() && a[i] == b[i]; i++) + { + } + return i; +} + +static bool ends_with(const std::string &str, const std::string &suffix) +{ + return str.size() >= suffix.size() && + 0 == str.compare(str.size() - suffix.size(), suffix.size(), suffix); +} + +static size_t find_partial_stop_string(const std::string &stop, + const std::string &text) +{ + if (!text.empty() && !stop.empty()) + { + const char text_last_char = text.back(); + for (int64_t char_index = stop.size() - 1; char_index >= 0; char_index--) + { + if (stop[char_index] == text_last_char) + { + const std::string current_partial = stop.substr(0, char_index + 1); + if (ends_with(text, current_partial)) + { + return text.size() - char_index - 1; + } + } + } + } + return std::string::npos; +} + +// TODO: reuse llama_detokenize +template +static std::string tokens_to_str(llama_context *ctx, Iter begin, Iter end) +{ + std::string ret; + for (; begin != end; ++begin) + { + ret += llama_token_to_piece(ctx, *begin); + } + return ret; +} + +// format incomplete utf-8 multibyte character for output +static std::string tokens_to_output_formatted_string(const llama_context *ctx, const llama_token token) +{ + std::string out = token == -1 ? "" : llama_token_to_piece(ctx, token); + // if the size is 1 and first bit is 1, meaning it's a partial character + // (size > 1 meaning it's already a known token) + if (out.size() == 1 && (out[0] & 0x80) == 0x80) + { + std::stringstream ss; + ss << std::hex << (out[0] & 0xff); + std::string res(ss.str()); + out = "byte: \\x" + res; + } + return out; +} + +// convert a vector of completion_token_output to json +static json probs_vector_to_json(const llama_context *ctx, const std::vector &probs) +{ + json out = json::array(); + for (const auto &prob : probs) + { + json probs_for_token = json::array(); + for (const auto &p : prob.probs) + { + std::string tok_str = tokens_to_output_formatted_string(ctx, p.tok); + probs_for_token.push_back(json + { + {"tok_str", tok_str}, + {"prob", p.prob}, + }); + } + std::string tok_str = tokens_to_output_formatted_string(ctx, prob.tok); + out.push_back(json{ + {"content", tok_str}, + {"probs", probs_for_token}, + }); + } + return out; +} From 38d152160898b0173ffe4dc7df5daadcbd2eceb0 Mon Sep 17 00:00:00 2001 From: AidanBeltonS <87009434+AidanBeltonS@users.noreply.github.com> Date: Fri, 1 Mar 2024 07:36:47 +0000 Subject: [PATCH 494/812] [SYCL] Use batched mul_mat pathway (#5591) * Use batched mul_mat pathway * rm extra line * Explicitly state scaled data type --------- Co-authored-by: Abhilash Majumder <30946547+abhilash1910@users.noreply.github.com> --- ggml-sycl.cpp | 107 +++++++++++++++++++++----------------------------- 1 file changed, 44 insertions(+), 63 deletions(-) diff --git a/ggml-sycl.cpp b/ggml-sycl.cpp index a054ec8b9..6f391b0c6 100644 --- a/ggml-sycl.cpp +++ b/ggml-sycl.cpp @@ -12726,6 +12726,7 @@ static void ggml_sycl_op_mul_mat(const ggml_tensor *src0, GGML_ASSERT(dst->backend != GGML_BACKEND_TYPE_GPU_SPLIT); GGML_ASSERT(src1->backend != GGML_BACKEND_TYPE_GPU_SPLIT); + GGML_ASSERT(src1->type == GGML_TYPE_F32 || (src1->ne[2] == 1 && src1->ne[3] == 1)); GGML_ASSERT(ne12 >= ne02 && ne12 % ne02 == 0); @@ -13269,31 +13270,23 @@ static void k_compute_batched_ptrs(const sycl::half *src0_as_f16, int64_t i03 = i13 / r3; int64_t i02 = i12 / r2; - ptrs_src[0*ne23 + i12 + i13*ne12] = (const char *) src0_as_f16 + i02*nb02 + i03*nb03; - ptrs_src[1*ne23 + i12 + i13*ne12] = (const char *) src1_as_f16 + i12*nb12/2 + i13*nb13/2; - ptrs_dst[0*ne23 + i12 + i13*ne12] = ( char *) dst + i12*nbd2 + i13*nbd3; + ptrs_src[0*ne23 + i12 + i13*ne12] = (const char *) src0_as_f16 + i02*nb02 + i03*nb03; + ptrs_src[1*ne23 + i12 + i13*ne12] = (const char *) src1_as_f16 + i12*nb12 + i13*nb13; + ptrs_dst[0*ne23 + i12 + i13*ne12] = ( char *) dst + i12*nbd2 + i13*nbd3; } -static void ggml_sycl_mul_mat_mat_batched_sycl(const ggml_tensor *src0, - const ggml_tensor *src1, - ggml_tensor *dst) try { +static void ggml_sycl_mul_mat_batched_sycl(const ggml_tensor *src0, + const ggml_tensor *src1, + ggml_tensor *dst) try { GGML_ASSERT(!ggml_is_transposed(src0)); GGML_ASSERT(!ggml_is_transposed(src1)); GGML_ASSERT(src0->backend != GGML_BACKEND_TYPE_GPU_SPLIT); GGML_ASSERT(src0->type == GGML_TYPE_F16); - GGML_ASSERT(src1->type == GGML_TYPE_F32); - GGML_TENSOR_LOCALS(int64_t, ne0, src0, ne); + GGML_TENSOR_BINARY_OP_LOCALS - GGML_TENSOR_LOCALS(int64_t, nb0, src0, nb); - - GGML_TENSOR_LOCALS(int64_t, ne1, src1, ne); - - GGML_TENSOR_LOCALS(int64_t, nb1, src1, nb); - - const int64_t ne1 = ggml_nelements(src1); - const int64_t ne = ggml_nelements(dst); + const int64_t ne_dst = ggml_nelements(dst); SYCL_CHECK(ggml_sycl_set_device(g_main_device)); dpct::queue_ptr main_stream = g_syclStreams[g_main_device_index][0]; @@ -13312,11 +13305,16 @@ static void ggml_sycl_mul_mat_mat_batched_sycl(const ggml_tensor *src0, float * dst_ddf = (float *) dst_extra->data_device[g_main_device_index]; // convert src1 to fp16 - const to_fp16_sycl_t to_fp16_sycl = ggml_get_to_fp16_sycl(src1->type); - GGML_ASSERT(to_fp16_sycl != nullptr); - - sycl_pool_alloc src1_as_f16(ne1); - to_fp16_sycl(src1_ddf, src1_as_f16.get(), ne1, main_stream); + sycl_pool_alloc src1_f16_alloc; + if (src1->type != GGML_TYPE_F16) { + const to_fp16_sycl_t to_fp16_sycl = ggml_get_to_fp16_sycl(src1->type); + const int64_t ne_src1 = ggml_nelements(src1); + src1_f16_alloc.alloc(ne_src1); + GGML_ASSERT(to_fp16_sycl != nullptr); + to_fp16_sycl(src1_ddf, src1_f16_alloc.get(), ne_src1, main_stream); + } + sycl::half *src1_f16 = src1->type == GGML_TYPE_F16 ? (sycl::half *)src1_ddf + : src1_f16_alloc.get(); sycl_pool_alloc dst_f16; char * dst_t; @@ -13337,20 +13335,12 @@ static void ggml_sycl_mul_mat_mat_batched_sycl(const ggml_tensor *src0, const void * alpha = &alpha_f16; const void * beta = &beta_f16; - if (dst->op_params[0] == GGML_PREC_DEFAULT) { - dst_t = (char *) dst_f16.alloc(ne); + // TODO: Renable (dst->op_params[0] =! GGML_PREC_DEFAULT) pathway + // once oneMKL open source supports half, half, float, float: datatypes + dst_t = (char *) dst_f16.alloc(ne_dst); - nbd2 /= sizeof(float) / sizeof(sycl::half); - nbd3 /= sizeof(float) / sizeof(sycl::half); - } else { - dst_t = (char *) dst_ddf; - - cu_compute_type = dpct::library_data_t::real_float; - cu_data_type = dpct::library_data_t::real_float; - - alpha = &alpha_f32; - beta = &beta_f32; - } + nbd2 /= sizeof(float) / sizeof(sycl::half); + nbd3 /= sizeof(float) / sizeof(sycl::half); GGML_ASSERT(ne12 % ne02 == 0); GGML_ASSERT(ne13 % ne03 == 0); @@ -13386,10 +13376,10 @@ static void ggml_sycl_mul_mat_mat_batched_sycl(const ggml_tensor *src0, *g_sycl_handles[g_main_device_index], oneapi::mkl::transpose::trans, oneapi::mkl::transpose::nontrans, ne01, ne11, ne10, alpha, (const char *)src0_as_f16, dpct::library_data_t::real_half, - nb01 / sizeof(sycl::half), src0->nb[2] / sizeof(sycl::half), - (const char *)src1_as_f16.get(), dpct::library_data_t::real_half, - nb11 / sizeof(float), src1->nb[2] / sizeof(float), beta, - (char *)dst_t, cu_data_type, ne01, dst->nb[2] / sizeof(float), + nb01 / nb00, nb02 / nb00, + (const char *)src1_f16, dpct::library_data_t::real_half, + nb11 / nb10, nb12 / nb10, beta, + (char *)dst_t, cu_data_type, ne01, nb2 / nb0, ne12 * ne13, cu_compute_type))); } else { // use syclGemmBatchedEx @@ -13409,44 +13399,35 @@ static void ggml_sycl_mul_mat_mat_batched_sycl(const ggml_tensor *src0, {sycl::aspect::fp16}); main_stream->submit([&](sycl::handler &cgh) { - const sycl::half *src1_as_f16_get_ct1 = src1_as_f16.get(); - const void **ptrs_src_get_ct3 = ptrs_src.get(); - void **ptrs_dst_get_ct4 = ptrs_dst.get(); - + const void **ptrs_src_get = ptrs_src.get(); + void **ptrs_dst_get = ptrs_dst.get(); + size_t nb12_scaled = src1->type == GGML_TYPE_F16 ? nb12 : nb12 / 2; + size_t nb13_scaled = src1->type == GGML_TYPE_F16 ? nb13 : nb13 / 2; cgh.parallel_for(sycl::nd_range<3>(block_dims, block_dims), [=](sycl::nd_item<3> item_ct1) { k_compute_batched_ptrs( - src0_as_f16, src1_as_f16_get_ct1, - dst_t, ptrs_src_get_ct3, - ptrs_dst_get_ct4, ne12, ne13, ne23, - nb02, nb03, nb12, nb13, nbd2, nbd3, r2, - r3, item_ct1); + src0_as_f16, src1_f16, + dst_t, ptrs_src_get, + ptrs_dst_get, ne12, ne13, ne23, + nb02, nb03, nb12_scaled, nb13_scaled, + nbd2, nbd3, r2, r3, item_ct1); }); }); } - /* - DPCT1010:95: SYCL uses exceptions to report errors and does not use the - error codes. The call was replaced with 0. You need to rewrite this - code. - */ - SYCL_CHECK(0); - SYCL_CHECK(CHECK_TRY_ERROR(dpct::gemm_batch( *g_sycl_handles[g_main_device_index], oneapi::mkl::transpose::trans, oneapi::mkl::transpose::nontrans, ne01, ne11, ne10, alpha, (const void **)(ptrs_src.get() + 0 * ne23), - dpct::library_data_t::real_half, nb01 / sizeof(sycl::half), + dpct::library_data_t::real_half, nb01 / nb00, (const void **)(ptrs_src.get() + 1 * ne23), - dpct::library_data_t::real_half, nb11 / sizeof(float), beta, + dpct::library_data_t::real_half, nb11 / nb10, beta, (void **)(ptrs_dst.get() + 0 * ne23), cu_data_type, ne01, ne23, cu_compute_type))); } #endif - if (dst->op_params[0] == GGML_PREC_DEFAULT) { - const to_fp32_sycl_t to_fp32_sycl = ggml_get_to_fp32_sycl(GGML_TYPE_F16); - to_fp32_sycl(dst_f16.get(), dst_ddf, ne, main_stream); - } + const to_fp32_sycl_t to_fp32_sycl = ggml_get_to_fp32_sycl(GGML_TYPE_F16); + to_fp32_sycl(dst_f16.get(), dst_ddf, ne_dst, main_stream); } catch (sycl::exception const &exc) { std::cerr << exc.what() << "Exception caught at file:" << __FILE__ @@ -13491,10 +13472,10 @@ static void ggml_sycl_mul_mat(const ggml_tensor * src0, const ggml_tensor * src1 // KQV single-batch // GGML_SYCL_DEBUG("ggml_sycl_mul_mat_vec_nc\n"); ggml_sycl_mul_mat_vec_nc(src0, src1, dst); - } else if (!split && all_on_device && use_xmx && src0->type == GGML_TYPE_F16 && src1->type == GGML_TYPE_F32 && !ggml_is_transposed(src0) && !ggml_is_transposed(src1)) { + } else if (!split && all_on_device && use_xmx && src0->type == GGML_TYPE_F16 && !ggml_is_transposed(src0) && !ggml_is_transposed(src1)) { // KQ + KQV multi-batch - // GGML_SYCL_DEBUG("ggml_sycl_mul_mat_mat_batched_sycl\n"); - ggml_sycl_mul_mat_mat_batched_sycl(src0, src1, dst); + // GGML_SYCL_DEBUG("ggml_sycl_mul_mat_batched_sycl\n"); + ggml_sycl_mul_mat_batched_sycl(src0, src1, dst); } else if (src0->type == GGML_TYPE_F32) { // GGML_SYCL_DEBUG("ggml_sycl_op_mul_mat\n"); ggml_sycl_op_mul_mat(src0, src1, dst, ggml_sycl_op_mul_mat_sycl, false); From f105471ef6aa4727afac8240da398590d7277f45 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Fri, 1 Mar 2024 09:59:43 +0200 Subject: [PATCH 495/812] server : fix newlines in help (#5785) --- examples/server/server.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/server/server.cpp b/examples/server/server.cpp index bf20e0cf1..45c4aec4d 100644 --- a/examples/server/server.cpp +++ b/examples/server/server.cpp @@ -2080,8 +2080,8 @@ static void server_print_usage(const char *argv0, const gpt_params ¶ms, printf(" --override-kv KEY=TYPE:VALUE\n"); printf(" advanced option to override model metadata by key. may be specified multiple times.\n"); printf(" types: int, float, bool. example: --override-kv tokenizer.ggml.add_bos_token=bool:false\n"); - printf(" -gan N, --grp-attn-n N set the group attention factor to extend context size through self-extend(default: 1=disabled), used together with group attention width `--grp-attn-w`"); - printf(" -gaw N, --grp-attn-w N set the group attention width to extend context size through self-extend(default: 512), used together with group attention factor `--grp-attn-n`"); + printf(" -gan N, --grp-attn-n N set the group attention factor to extend context size through self-extend(default: 1=disabled), used together with group attention width `--grp-attn-w`\n"); + printf(" -gaw N, --grp-attn-w N set the group attention width to extend context size through self-extend(default: 512), used together with group attention factor `--grp-attn-n`\n"); printf(" --chat-template JINJA_TEMPLATE\n"); printf(" set custom jinja chat template (default: template taken from model's metadata)\n"); printf(" Note: only commonly used templates are accepted, since we don't have jinja parser\n"); From 6ea0f010ff6967034528d9e0b8330b9b0f0b7c13 Mon Sep 17 00:00:00 2001 From: Eve <139727413+netrunnereve@users.noreply.github.com> Date: Fri, 1 Mar 2024 08:54:53 +0000 Subject: [PATCH 496/812] ci : add Ubuntu 22 Vulkan CI run (#5789) --- .github/workflows/build.yml | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 66ad85938..9144f9266 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -145,6 +145,28 @@ jobs: cd build ctest -L main --verbose + ubuntu-22-cmake-vulkan: + runs-on: ubuntu-22.04 + + steps: + - name: Clone + id: checkout + uses: actions/checkout@v3 + + - name: Dependencies + id: depends + run: | + sudo apt-get update + sudo apt-get install build-essential libvulkan-dev + + - name: Build + id: cmake_build + run: | + mkdir build + cd build + cmake -DLLAMA_VULKAN=ON .. + cmake --build . --config Release -j $(nproc) + ubuntu-22-cmake-sycl: runs-on: ubuntu-22.04 From 5cb02b4a012bb16c6c699c0c62c05ffa653eee0f Mon Sep 17 00:00:00 2001 From: Pierrick Hymbert Date: Fri, 1 Mar 2024 10:08:08 +0100 Subject: [PATCH 497/812] server: allow to override threads server pool with --threads-http (#5794) --- examples/server/README.md | 1 + examples/server/server.cpp | 16 ++++++++++++++++ 2 files changed, 17 insertions(+) diff --git a/examples/server/README.md b/examples/server/README.md index 0e9bd7fd4..ad35306c6 100644 --- a/examples/server/README.md +++ b/examples/server/README.md @@ -18,6 +18,7 @@ The project is under active development, and we are [looking for feedback and co - `--threads N`, `-t N`: Set the number of threads to use during generation. - `-tb N, --threads-batch N`: Set the number of threads to use during batch and prompt processing. If not specified, the number of threads will be set to the number of threads used for generation. +- `--threads-http N`: number of threads in the http server pool to process requests (default: `std::thread::hardware_concurrency()`) - `-m FNAME`, `--model FNAME`: Specify the path to the LLaMA model file (e.g., `models/7B/ggml-model.gguf`). - `-a ALIAS`, `--alias ALIAS`: Set an alias for the model. The alias will be returned in API responses. - `-c N`, `--ctx-size N`: Set the size of the prompt context. The default is 512, but LLaMA models were built with a context of 2048, which will provide better results for longer input/inference. The size may differ in other models, for example, baichuan models were build with a context of 4096. diff --git a/examples/server/server.cpp b/examples/server/server.cpp index 45c4aec4d..eea987966 100644 --- a/examples/server/server.cpp +++ b/examples/server/server.cpp @@ -43,6 +43,7 @@ struct server_params { int32_t write_timeout = 600; bool slots_endpoint = true; bool metrics_endpoint = false; + int n_threads_http = -1; }; bool server_verbose = false; @@ -2012,6 +2013,7 @@ static void server_print_usage(const char *argv0, const gpt_params ¶ms, printf(" -v, --verbose verbose output (default: %s)\n", server_verbose ? "enabled" : "disabled"); printf(" -t N, --threads N number of threads to use during computation (default: %d)\n", params.n_threads); printf(" -tb N, --threads-batch N number of threads to use during batch and prompt processing (default: same as --threads)\n"); + printf(" --threads-http N number of threads in the http server pool to process requests (default: hardware concurrency)\n"); printf(" -c N, --ctx-size N size of the prompt context (default: %d)\n", params.n_ctx); printf(" --rope-scaling {none,linear,yarn}\n"); printf(" RoPE frequency scaling method, defaults to linear unless specified by the model\n"); @@ -2298,6 +2300,15 @@ static void server_params_parse(int argc, char **argv, server_params &sparams, } params.n_threads_batch = std::stoi(argv[i]); } + else if (arg == "--threads-http") + { + if (++i >= argc) + { + invalid_param = true; + break; + } + sparams.n_threads_http = std::stoi(argv[i]); + } else if (arg == "-b" || arg == "--batch-size") { if (++i >= argc) @@ -3449,6 +3460,11 @@ int main(int argc, char **argv) }*/ //); + if (sparams.n_threads_http > 0) { + log_data["n_threads_http"] = std::to_string(sparams.n_threads_http); + svr.new_task_queue = [&sparams] { return new httplib::ThreadPool(sparams.n_threads_http); }; + } + LOG_INFO("HTTP server listening", log_data); // run the HTTP server in a thread - see comment below std::thread t([&]() From 9600d59e010c18f5872580a21734ea1bf1968d04 Mon Sep 17 00:00:00 2001 From: Douglas Hanley Date: Fri, 1 Mar 2024 03:15:36 -0600 Subject: [PATCH 498/812] unicode : switch to multimap based nfd_map (#5799) * switch to multimap based nfd_map due to compile time issues * simplify multimap keys * dont construct new locale every time --- llama.cpp | 11 +- unicode.h | 566 +++++++++++++++++++++++++++++------------------------- 2 files changed, 312 insertions(+), 265 deletions(-) diff --git a/llama.cpp b/llama.cpp index 62699ce52..a35f07aa4 100644 --- a/llama.cpp +++ b/llama.cpp @@ -8947,10 +8947,10 @@ struct llm_tokenizer_wpm { std::vector codepoints = codepoints_from_utf8(text); std::vector nfd_codepoints; for (uint32_t code : codepoints) { - auto it = nfd_map.find(code); - if (it != nfd_map.end()) { - for (uint32_t c : it->second) { - nfd_codepoints.push_back(c); + auto it = nfd_map.equal_range(code); + if (it.first != it.second) { + for (auto jt = it.first; jt != it.second; jt++) { + nfd_codepoints.push_back(jt->second); } } else { nfd_codepoints.push_back(code); @@ -9001,12 +9001,13 @@ struct llm_tokenizer_wpm { } uint32_t to_lower(uint32_t code) { + static const std::locale locale("en_US.UTF-8"); #if defined(_WIN32) if (code > 0xFFFF) { return code; } #endif - return std::tolower(wchar_t(code), std::locale("en_US.UTF-8")); + return std::tolower(wchar_t(code), locale); } bool is_ascii_punct(uint32_t code) { diff --git a/unicode.h b/unicode.h index 620e2b580..f6be4549b 100644 --- a/unicode.h +++ b/unicode.h @@ -1,6 +1,7 @@ #pragma once #include +#include #include #include #include @@ -223,266 +224,311 @@ static const std::vector> control_ranges = { {0x2B81E, 0x2B81F}, {0x2CEA2, 0x2CEAF}, {0x2EBE1, 0x2F7FF}, {0x2FA1E, 0x2FFFF}, {0x3134B, 0xE00FF}, {0xE01F0, 0x10FFFF}, }; -static const std::unordered_map> nfd_map = { -{0xC0, {0x41, 0x300}}, {0xC1, {0x41, 0x301}}, {0xC2, {0x41, 0x302}}, {0xC3, {0x41, 0x303}}, {0xC4, {0x41, 0x308}}, {0xC5, {0x41, 0x30A}}, {0xC7, {0x43, 0x327}}, {0xC8, {0x45, 0x300}}, -{0xC9, {0x45, 0x301}}, {0xCA, {0x45, 0x302}}, {0xCB, {0x45, 0x308}}, {0xCC, {0x49, 0x300}}, {0xCD, {0x49, 0x301}}, {0xCE, {0x49, 0x302}}, {0xCF, {0x49, 0x308}}, {0xD1, {0x4E, 0x303}}, -{0xD2, {0x4F, 0x300}}, {0xD3, {0x4F, 0x301}}, {0xD4, {0x4F, 0x302}}, {0xD5, {0x4F, 0x303}}, {0xD6, {0x4F, 0x308}}, {0xD9, {0x55, 0x300}}, {0xDA, {0x55, 0x301}}, {0xDB, {0x55, 0x302}}, -{0xDC, {0x55, 0x308}}, {0xDD, {0x59, 0x301}}, {0xE0, {0x61, 0x300}}, {0xE1, {0x61, 0x301}}, {0xE2, {0x61, 0x302}}, {0xE3, {0x61, 0x303}}, {0xE4, {0x61, 0x308}}, {0xE5, {0x61, 0x30A}}, -{0xE7, {0x63, 0x327}}, {0xE8, {0x65, 0x300}}, {0xE9, {0x65, 0x301}}, {0xEA, {0x65, 0x302}}, {0xEB, {0x65, 0x308}}, {0xEC, {0x69, 0x300}}, {0xED, {0x69, 0x301}}, {0xEE, {0x69, 0x302}}, -{0xEF, {0x69, 0x308}}, {0xF1, {0x6E, 0x303}}, {0xF2, {0x6F, 0x300}}, {0xF3, {0x6F, 0x301}}, {0xF4, {0x6F, 0x302}}, {0xF5, {0x6F, 0x303}}, {0xF6, {0x6F, 0x308}}, {0xF9, {0x75, 0x300}}, -{0xFA, {0x75, 0x301}}, {0xFB, {0x75, 0x302}}, {0xFC, {0x75, 0x308}}, {0xFD, {0x79, 0x301}}, {0xFF, {0x79, 0x308}}, {0x100, {0x41, 0x304}}, {0x101, {0x61, 0x304}}, {0x102, {0x41, 0x306}}, -{0x103, {0x61, 0x306}}, {0x104, {0x41, 0x328}}, {0x105, {0x61, 0x328}}, {0x106, {0x43, 0x301}}, {0x107, {0x63, 0x301}}, {0x108, {0x43, 0x302}}, {0x109, {0x63, 0x302}}, {0x10A, {0x43, 0x307}}, -{0x10B, {0x63, 0x307}}, {0x10C, {0x43, 0x30C}}, {0x10D, {0x63, 0x30C}}, {0x10E, {0x44, 0x30C}}, {0x10F, {0x64, 0x30C}}, {0x112, {0x45, 0x304}}, {0x113, {0x65, 0x304}}, {0x114, {0x45, 0x306}}, -{0x115, {0x65, 0x306}}, {0x116, {0x45, 0x307}}, {0x117, {0x65, 0x307}}, {0x118, {0x45, 0x328}}, {0x119, {0x65, 0x328}}, {0x11A, {0x45, 0x30C}}, {0x11B, {0x65, 0x30C}}, {0x11C, {0x47, 0x302}}, -{0x11D, {0x67, 0x302}}, {0x11E, {0x47, 0x306}}, {0x11F, {0x67, 0x306}}, {0x120, {0x47, 0x307}}, {0x121, {0x67, 0x307}}, {0x122, {0x47, 0x327}}, {0x123, {0x67, 0x327}}, {0x124, {0x48, 0x302}}, -{0x125, {0x68, 0x302}}, {0x128, {0x49, 0x303}}, {0x129, {0x69, 0x303}}, {0x12A, {0x49, 0x304}}, {0x12B, {0x69, 0x304}}, {0x12C, {0x49, 0x306}}, {0x12D, {0x69, 0x306}}, {0x12E, {0x49, 0x328}}, -{0x12F, {0x69, 0x328}}, {0x130, {0x49, 0x307}}, {0x134, {0x4A, 0x302}}, {0x135, {0x6A, 0x302}}, {0x136, {0x4B, 0x327}}, {0x137, {0x6B, 0x327}}, {0x139, {0x4C, 0x301}}, {0x13A, {0x6C, 0x301}}, -{0x13B, {0x4C, 0x327}}, {0x13C, {0x6C, 0x327}}, {0x13D, {0x4C, 0x30C}}, {0x13E, {0x6C, 0x30C}}, {0x143, {0x4E, 0x301}}, {0x144, {0x6E, 0x301}}, {0x145, {0x4E, 0x327}}, {0x146, {0x6E, 0x327}}, -{0x147, {0x4E, 0x30C}}, {0x148, {0x6E, 0x30C}}, {0x14C, {0x4F, 0x304}}, {0x14D, {0x6F, 0x304}}, {0x14E, {0x4F, 0x306}}, {0x14F, {0x6F, 0x306}}, {0x150, {0x4F, 0x30B}}, {0x151, {0x6F, 0x30B}}, -{0x154, {0x52, 0x301}}, {0x155, {0x72, 0x301}}, {0x156, {0x52, 0x327}}, {0x157, {0x72, 0x327}}, {0x158, {0x52, 0x30C}}, {0x159, {0x72, 0x30C}}, {0x15A, {0x53, 0x301}}, {0x15B, {0x73, 0x301}}, -{0x15C, {0x53, 0x302}}, {0x15D, {0x73, 0x302}}, {0x15E, {0x53, 0x327}}, {0x15F, {0x73, 0x327}}, {0x160, {0x53, 0x30C}}, {0x161, {0x73, 0x30C}}, {0x162, {0x54, 0x327}}, {0x163, {0x74, 0x327}}, -{0x164, {0x54, 0x30C}}, {0x165, {0x74, 0x30C}}, {0x168, {0x55, 0x303}}, {0x169, {0x75, 0x303}}, {0x16A, {0x55, 0x304}}, {0x16B, {0x75, 0x304}}, {0x16C, {0x55, 0x306}}, {0x16D, {0x75, 0x306}}, -{0x16E, {0x55, 0x30A}}, {0x16F, {0x75, 0x30A}}, {0x170, {0x55, 0x30B}}, {0x171, {0x75, 0x30B}}, {0x172, {0x55, 0x328}}, {0x173, {0x75, 0x328}}, {0x174, {0x57, 0x302}}, {0x175, {0x77, 0x302}}, -{0x176, {0x59, 0x302}}, {0x177, {0x79, 0x302}}, {0x178, {0x59, 0x308}}, {0x179, {0x5A, 0x301}}, {0x17A, {0x7A, 0x301}}, {0x17B, {0x5A, 0x307}}, {0x17C, {0x7A, 0x307}}, {0x17D, {0x5A, 0x30C}}, -{0x17E, {0x7A, 0x30C}}, {0x1A0, {0x4F, 0x31B}}, {0x1A1, {0x6F, 0x31B}}, {0x1AF, {0x55, 0x31B}}, {0x1B0, {0x75, 0x31B}}, {0x1CD, {0x41, 0x30C}}, {0x1CE, {0x61, 0x30C}}, {0x1CF, {0x49, 0x30C}}, -{0x1D0, {0x69, 0x30C}}, {0x1D1, {0x4F, 0x30C}}, {0x1D2, {0x6F, 0x30C}}, {0x1D3, {0x55, 0x30C}}, {0x1D4, {0x75, 0x30C}}, {0x1D5, {0x55, 0x308, 0x304}}, {0x1D6, {0x75, 0x308, 0x304}}, -{0x1D7, {0x55, 0x308, 0x301}}, {0x1D8, {0x75, 0x308, 0x301}}, {0x1D9, {0x55, 0x308, 0x30C}}, {0x1DA, {0x75, 0x308, 0x30C}}, {0x1DB, {0x55, 0x308, 0x300}}, {0x1DC, {0x75, 0x308, 0x300}}, -{0x1DE, {0x41, 0x308, 0x304}}, {0x1DF, {0x61, 0x308, 0x304}}, {0x1E0, {0x41, 0x307, 0x304}}, {0x1E1, {0x61, 0x307, 0x304}}, {0x1E2, {0xC6, 0x304}}, {0x1E3, {0xE6, 0x304}}, {0x1E6, {0x47, 0x30C}}, -{0x1E7, {0x67, 0x30C}}, {0x1E8, {0x4B, 0x30C}}, {0x1E9, {0x6B, 0x30C}}, {0x1EA, {0x4F, 0x328}}, {0x1EB, {0x6F, 0x328}}, {0x1EC, {0x4F, 0x328, 0x304}}, {0x1ED, {0x6F, 0x328, 0x304}}, -{0x1EE, {0x1B7, 0x30C}}, {0x1EF, {0x292, 0x30C}}, {0x1F0, {0x6A, 0x30C}}, {0x1F4, {0x47, 0x301}}, {0x1F5, {0x67, 0x301}}, {0x1F8, {0x4E, 0x300}}, {0x1F9, {0x6E, 0x300}}, {0x1FA, {0x41, 0x30A, 0x301}}, -{0x1FB, {0x61, 0x30A, 0x301}}, {0x1FC, {0xC6, 0x301}}, {0x1FD, {0xE6, 0x301}}, {0x1FE, {0xD8, 0x301}}, {0x1FF, {0xF8, 0x301}}, {0x200, {0x41, 0x30F}}, {0x201, {0x61, 0x30F}}, {0x202, {0x41, 0x311}}, -{0x203, {0x61, 0x311}}, {0x204, {0x45, 0x30F}}, {0x205, {0x65, 0x30F}}, {0x206, {0x45, 0x311}}, {0x207, {0x65, 0x311}}, {0x208, {0x49, 0x30F}}, {0x209, {0x69, 0x30F}}, {0x20A, {0x49, 0x311}}, -{0x20B, {0x69, 0x311}}, {0x20C, {0x4F, 0x30F}}, {0x20D, {0x6F, 0x30F}}, {0x20E, {0x4F, 0x311}}, {0x20F, {0x6F, 0x311}}, {0x210, {0x52, 0x30F}}, {0x211, {0x72, 0x30F}}, {0x212, {0x52, 0x311}}, -{0x213, {0x72, 0x311}}, {0x214, {0x55, 0x30F}}, {0x215, {0x75, 0x30F}}, {0x216, {0x55, 0x311}}, {0x217, {0x75, 0x311}}, {0x218, {0x53, 0x326}}, {0x219, {0x73, 0x326}}, {0x21A, {0x54, 0x326}}, -{0x21B, {0x74, 0x326}}, {0x21E, {0x48, 0x30C}}, {0x21F, {0x68, 0x30C}}, {0x226, {0x41, 0x307}}, {0x227, {0x61, 0x307}}, {0x228, {0x45, 0x327}}, {0x229, {0x65, 0x327}}, {0x22A, {0x4F, 0x308, 0x304}}, -{0x22B, {0x6F, 0x308, 0x304}}, {0x22C, {0x4F, 0x303, 0x304}}, {0x22D, {0x6F, 0x303, 0x304}}, {0x22E, {0x4F, 0x307}}, {0x22F, {0x6F, 0x307}}, {0x230, {0x4F, 0x307, 0x304}}, -{0x231, {0x6F, 0x307, 0x304}}, {0x232, {0x59, 0x304}}, {0x233, {0x79, 0x304}}, {0x340, {0x300}}, {0x341, {0x301}}, {0x343, {0x313}}, {0x344, {0x308, 0x301}}, {0x374, {0x2B9}}, {0x37E, {0x3B}}, -{0x385, {0xA8, 0x301}}, {0x386, {0x391, 0x301}}, {0x387, {0xB7}}, {0x388, {0x395, 0x301}}, {0x389, {0x397, 0x301}}, {0x38A, {0x399, 0x301}}, {0x38C, {0x39F, 0x301}}, {0x38E, {0x3A5, 0x301}}, -{0x38F, {0x3A9, 0x301}}, {0x390, {0x3B9, 0x308, 0x301}}, {0x3AA, {0x399, 0x308}}, {0x3AB, {0x3A5, 0x308}}, {0x3AC, {0x3B1, 0x301}}, {0x3AD, {0x3B5, 0x301}}, {0x3AE, {0x3B7, 0x301}}, -{0x3AF, {0x3B9, 0x301}}, {0x3B0, {0x3C5, 0x308, 0x301}}, {0x3CA, {0x3B9, 0x308}}, {0x3CB, {0x3C5, 0x308}}, {0x3CC, {0x3BF, 0x301}}, {0x3CD, {0x3C5, 0x301}}, {0x3CE, {0x3C9, 0x301}}, -{0x3D3, {0x3D2, 0x301}}, {0x3D4, {0x3D2, 0x308}}, {0x400, {0x415, 0x300}}, {0x401, {0x415, 0x308}}, {0x403, {0x413, 0x301}}, {0x407, {0x406, 0x308}}, {0x40C, {0x41A, 0x301}}, {0x40D, {0x418, 0x300}}, -{0x40E, {0x423, 0x306}}, {0x419, {0x418, 0x306}}, {0x439, {0x438, 0x306}}, {0x450, {0x435, 0x300}}, {0x451, {0x435, 0x308}}, {0x453, {0x433, 0x301}}, {0x457, {0x456, 0x308}}, {0x45C, {0x43A, 0x301}}, -{0x45D, {0x438, 0x300}}, {0x45E, {0x443, 0x306}}, {0x476, {0x474, 0x30F}}, {0x477, {0x475, 0x30F}}, {0x4C1, {0x416, 0x306}}, {0x4C2, {0x436, 0x306}}, {0x4D0, {0x410, 0x306}}, {0x4D1, {0x430, 0x306}}, -{0x4D2, {0x410, 0x308}}, {0x4D3, {0x430, 0x308}}, {0x4D6, {0x415, 0x306}}, {0x4D7, {0x435, 0x306}}, {0x4DA, {0x4D8, 0x308}}, {0x4DB, {0x4D9, 0x308}}, {0x4DC, {0x416, 0x308}}, {0x4DD, {0x436, 0x308}}, -{0x4DE, {0x417, 0x308}}, {0x4DF, {0x437, 0x308}}, {0x4E2, {0x418, 0x304}}, {0x4E3, {0x438, 0x304}}, {0x4E4, {0x418, 0x308}}, {0x4E5, {0x438, 0x308}}, {0x4E6, {0x41E, 0x308}}, {0x4E7, {0x43E, 0x308}}, -{0x4EA, {0x4E8, 0x308}}, {0x4EB, {0x4E9, 0x308}}, {0x4EC, {0x42D, 0x308}}, {0x4ED, {0x44D, 0x308}}, {0x4EE, {0x423, 0x304}}, {0x4EF, {0x443, 0x304}}, {0x4F0, {0x423, 0x308}}, {0x4F1, {0x443, 0x308}}, -{0x4F2, {0x423, 0x30B}}, {0x4F3, {0x443, 0x30B}}, {0x4F4, {0x427, 0x308}}, {0x4F5, {0x447, 0x308}}, {0x4F8, {0x42B, 0x308}}, {0x4F9, {0x44B, 0x308}}, {0x622, {0x627, 0x653}}, {0x623, {0x627, 0x654}}, -{0x624, {0x648, 0x654}}, {0x625, {0x627, 0x655}}, {0x626, {0x64A, 0x654}}, {0x6C0, {0x6D5, 0x654}}, {0x6C2, {0x6C1, 0x654}}, {0x6D3, {0x6D2, 0x654}}, {0x929, {0x928, 0x93C}}, {0x931, {0x930, 0x93C}}, -{0x934, {0x933, 0x93C}}, {0x958, {0x915, 0x93C}}, {0x959, {0x916, 0x93C}}, {0x95A, {0x917, 0x93C}}, {0x95B, {0x91C, 0x93C}}, {0x95C, {0x921, 0x93C}}, {0x95D, {0x922, 0x93C}}, {0x95E, {0x92B, 0x93C}}, -{0x95F, {0x92F, 0x93C}}, {0x9CB, {0x9C7, 0x9BE}}, {0x9CC, {0x9C7, 0x9D7}}, {0x9DC, {0x9A1, 0x9BC}}, {0x9DD, {0x9A2, 0x9BC}}, {0x9DF, {0x9AF, 0x9BC}}, {0xA33, {0xA32, 0xA3C}}, {0xA36, {0xA38, 0xA3C}}, -{0xA59, {0xA16, 0xA3C}}, {0xA5A, {0xA17, 0xA3C}}, {0xA5B, {0xA1C, 0xA3C}}, {0xA5E, {0xA2B, 0xA3C}}, {0xB48, {0xB47, 0xB56}}, {0xB4B, {0xB47, 0xB3E}}, {0xB4C, {0xB47, 0xB57}}, {0xB5C, {0xB21, 0xB3C}}, -{0xB5D, {0xB22, 0xB3C}}, {0xB94, {0xB92, 0xBD7}}, {0xBCA, {0xBC6, 0xBBE}}, {0xBCB, {0xBC7, 0xBBE}}, {0xBCC, {0xBC6, 0xBD7}}, {0xC48, {0xC46, 0xC56}}, {0xCC0, {0xCBF, 0xCD5}}, {0xCC7, {0xCC6, 0xCD5}}, -{0xCC8, {0xCC6, 0xCD6}}, {0xCCA, {0xCC6, 0xCC2}}, {0xCCB, {0xCC6, 0xCC2, 0xCD5}}, {0xD4A, {0xD46, 0xD3E}}, {0xD4B, {0xD47, 0xD3E}}, {0xD4C, {0xD46, 0xD57}}, {0xDDA, {0xDD9, 0xDCA}}, -{0xDDC, {0xDD9, 0xDCF}}, {0xDDD, {0xDD9, 0xDCF, 0xDCA}}, {0xDDE, {0xDD9, 0xDDF}}, {0xF43, {0xF42, 0xFB7}}, {0xF4D, {0xF4C, 0xFB7}}, {0xF52, {0xF51, 0xFB7}}, {0xF57, {0xF56, 0xFB7}}, -{0xF5C, {0xF5B, 0xFB7}}, {0xF69, {0xF40, 0xFB5}}, {0xF73, {0xF71, 0xF72}}, {0xF75, {0xF71, 0xF74}}, {0xF76, {0xFB2, 0xF80}}, {0xF78, {0xFB3, 0xF80}}, {0xF81, {0xF71, 0xF80}}, {0xF93, {0xF92, 0xFB7}}, -{0xF9D, {0xF9C, 0xFB7}}, {0xFA2, {0xFA1, 0xFB7}}, {0xFA7, {0xFA6, 0xFB7}}, {0xFAC, {0xFAB, 0xFB7}}, {0xFB9, {0xF90, 0xFB5}}, {0x1026, {0x1025, 0x102E}}, {0x1B06, {0x1B05, 0x1B35}}, -{0x1B08, {0x1B07, 0x1B35}}, {0x1B0A, {0x1B09, 0x1B35}}, {0x1B0C, {0x1B0B, 0x1B35}}, {0x1B0E, {0x1B0D, 0x1B35}}, {0x1B12, {0x1B11, 0x1B35}}, {0x1B3B, {0x1B3A, 0x1B35}}, {0x1B3D, {0x1B3C, 0x1B35}}, -{0x1B40, {0x1B3E, 0x1B35}}, {0x1B41, {0x1B3F, 0x1B35}}, {0x1B43, {0x1B42, 0x1B35}}, {0x1E00, {0x41, 0x325}}, {0x1E01, {0x61, 0x325}}, {0x1E02, {0x42, 0x307}}, {0x1E03, {0x62, 0x307}}, -{0x1E04, {0x42, 0x323}}, {0x1E05, {0x62, 0x323}}, {0x1E06, {0x42, 0x331}}, {0x1E07, {0x62, 0x331}}, {0x1E08, {0x43, 0x327, 0x301}}, {0x1E09, {0x63, 0x327, 0x301}}, {0x1E0A, {0x44, 0x307}}, -{0x1E0B, {0x64, 0x307}}, {0x1E0C, {0x44, 0x323}}, {0x1E0D, {0x64, 0x323}}, {0x1E0E, {0x44, 0x331}}, {0x1E0F, {0x64, 0x331}}, {0x1E10, {0x44, 0x327}}, {0x1E11, {0x64, 0x327}}, {0x1E12, {0x44, 0x32D}}, -{0x1E13, {0x64, 0x32D}}, {0x1E14, {0x45, 0x304, 0x300}}, {0x1E15, {0x65, 0x304, 0x300}}, {0x1E16, {0x45, 0x304, 0x301}}, {0x1E17, {0x65, 0x304, 0x301}}, {0x1E18, {0x45, 0x32D}}, -{0x1E19, {0x65, 0x32D}}, {0x1E1A, {0x45, 0x330}}, {0x1E1B, {0x65, 0x330}}, {0x1E1C, {0x45, 0x327, 0x306}}, {0x1E1D, {0x65, 0x327, 0x306}}, {0x1E1E, {0x46, 0x307}}, {0x1E1F, {0x66, 0x307}}, -{0x1E20, {0x47, 0x304}}, {0x1E21, {0x67, 0x304}}, {0x1E22, {0x48, 0x307}}, {0x1E23, {0x68, 0x307}}, {0x1E24, {0x48, 0x323}}, {0x1E25, {0x68, 0x323}}, {0x1E26, {0x48, 0x308}}, {0x1E27, {0x68, 0x308}}, -{0x1E28, {0x48, 0x327}}, {0x1E29, {0x68, 0x327}}, {0x1E2A, {0x48, 0x32E}}, {0x1E2B, {0x68, 0x32E}}, {0x1E2C, {0x49, 0x330}}, {0x1E2D, {0x69, 0x330}}, {0x1E2E, {0x49, 0x308, 0x301}}, -{0x1E2F, {0x69, 0x308, 0x301}}, {0x1E30, {0x4B, 0x301}}, {0x1E31, {0x6B, 0x301}}, {0x1E32, {0x4B, 0x323}}, {0x1E33, {0x6B, 0x323}}, {0x1E34, {0x4B, 0x331}}, {0x1E35, {0x6B, 0x331}}, -{0x1E36, {0x4C, 0x323}}, {0x1E37, {0x6C, 0x323}}, {0x1E38, {0x4C, 0x323, 0x304}}, {0x1E39, {0x6C, 0x323, 0x304}}, {0x1E3A, {0x4C, 0x331}}, {0x1E3B, {0x6C, 0x331}}, {0x1E3C, {0x4C, 0x32D}}, -{0x1E3D, {0x6C, 0x32D}}, {0x1E3E, {0x4D, 0x301}}, {0x1E3F, {0x6D, 0x301}}, {0x1E40, {0x4D, 0x307}}, {0x1E41, {0x6D, 0x307}}, {0x1E42, {0x4D, 0x323}}, {0x1E43, {0x6D, 0x323}}, {0x1E44, {0x4E, 0x307}}, -{0x1E45, {0x6E, 0x307}}, {0x1E46, {0x4E, 0x323}}, {0x1E47, {0x6E, 0x323}}, {0x1E48, {0x4E, 0x331}}, {0x1E49, {0x6E, 0x331}}, {0x1E4A, {0x4E, 0x32D}}, {0x1E4B, {0x6E, 0x32D}}, -{0x1E4C, {0x4F, 0x303, 0x301}}, {0x1E4D, {0x6F, 0x303, 0x301}}, {0x1E4E, {0x4F, 0x303, 0x308}}, {0x1E4F, {0x6F, 0x303, 0x308}}, {0x1E50, {0x4F, 0x304, 0x300}}, {0x1E51, {0x6F, 0x304, 0x300}}, -{0x1E52, {0x4F, 0x304, 0x301}}, {0x1E53, {0x6F, 0x304, 0x301}}, {0x1E54, {0x50, 0x301}}, {0x1E55, {0x70, 0x301}}, {0x1E56, {0x50, 0x307}}, {0x1E57, {0x70, 0x307}}, {0x1E58, {0x52, 0x307}}, -{0x1E59, {0x72, 0x307}}, {0x1E5A, {0x52, 0x323}}, {0x1E5B, {0x72, 0x323}}, {0x1E5C, {0x52, 0x323, 0x304}}, {0x1E5D, {0x72, 0x323, 0x304}}, {0x1E5E, {0x52, 0x331}}, {0x1E5F, {0x72, 0x331}}, -{0x1E60, {0x53, 0x307}}, {0x1E61, {0x73, 0x307}}, {0x1E62, {0x53, 0x323}}, {0x1E63, {0x73, 0x323}}, {0x1E64, {0x53, 0x301, 0x307}}, {0x1E65, {0x73, 0x301, 0x307}}, {0x1E66, {0x53, 0x30C, 0x307}}, -{0x1E67, {0x73, 0x30C, 0x307}}, {0x1E68, {0x53, 0x323, 0x307}}, {0x1E69, {0x73, 0x323, 0x307}}, {0x1E6A, {0x54, 0x307}}, {0x1E6B, {0x74, 0x307}}, {0x1E6C, {0x54, 0x323}}, {0x1E6D, {0x74, 0x323}}, -{0x1E6E, {0x54, 0x331}}, {0x1E6F, {0x74, 0x331}}, {0x1E70, {0x54, 0x32D}}, {0x1E71, {0x74, 0x32D}}, {0x1E72, {0x55, 0x324}}, {0x1E73, {0x75, 0x324}}, {0x1E74, {0x55, 0x330}}, {0x1E75, {0x75, 0x330}}, -{0x1E76, {0x55, 0x32D}}, {0x1E77, {0x75, 0x32D}}, {0x1E78, {0x55, 0x303, 0x301}}, {0x1E79, {0x75, 0x303, 0x301}}, {0x1E7A, {0x55, 0x304, 0x308}}, {0x1E7B, {0x75, 0x304, 0x308}}, -{0x1E7C, {0x56, 0x303}}, {0x1E7D, {0x76, 0x303}}, {0x1E7E, {0x56, 0x323}}, {0x1E7F, {0x76, 0x323}}, {0x1E80, {0x57, 0x300}}, {0x1E81, {0x77, 0x300}}, {0x1E82, {0x57, 0x301}}, {0x1E83, {0x77, 0x301}}, -{0x1E84, {0x57, 0x308}}, {0x1E85, {0x77, 0x308}}, {0x1E86, {0x57, 0x307}}, {0x1E87, {0x77, 0x307}}, {0x1E88, {0x57, 0x323}}, {0x1E89, {0x77, 0x323}}, {0x1E8A, {0x58, 0x307}}, {0x1E8B, {0x78, 0x307}}, -{0x1E8C, {0x58, 0x308}}, {0x1E8D, {0x78, 0x308}}, {0x1E8E, {0x59, 0x307}}, {0x1E8F, {0x79, 0x307}}, {0x1E90, {0x5A, 0x302}}, {0x1E91, {0x7A, 0x302}}, {0x1E92, {0x5A, 0x323}}, {0x1E93, {0x7A, 0x323}}, -{0x1E94, {0x5A, 0x331}}, {0x1E95, {0x7A, 0x331}}, {0x1E96, {0x68, 0x331}}, {0x1E97, {0x74, 0x308}}, {0x1E98, {0x77, 0x30A}}, {0x1E99, {0x79, 0x30A}}, {0x1E9B, {0x17F, 0x307}}, {0x1EA0, {0x41, 0x323}}, -{0x1EA1, {0x61, 0x323}}, {0x1EA2, {0x41, 0x309}}, {0x1EA3, {0x61, 0x309}}, {0x1EA4, {0x41, 0x302, 0x301}}, {0x1EA5, {0x61, 0x302, 0x301}}, {0x1EA6, {0x41, 0x302, 0x300}}, -{0x1EA7, {0x61, 0x302, 0x300}}, {0x1EA8, {0x41, 0x302, 0x309}}, {0x1EA9, {0x61, 0x302, 0x309}}, {0x1EAA, {0x41, 0x302, 0x303}}, {0x1EAB, {0x61, 0x302, 0x303}}, {0x1EAC, {0x41, 0x323, 0x302}}, -{0x1EAD, {0x61, 0x323, 0x302}}, {0x1EAE, {0x41, 0x306, 0x301}}, {0x1EAF, {0x61, 0x306, 0x301}}, {0x1EB0, {0x41, 0x306, 0x300}}, {0x1EB1, {0x61, 0x306, 0x300}}, {0x1EB2, {0x41, 0x306, 0x309}}, -{0x1EB3, {0x61, 0x306, 0x309}}, {0x1EB4, {0x41, 0x306, 0x303}}, {0x1EB5, {0x61, 0x306, 0x303}}, {0x1EB6, {0x41, 0x323, 0x306}}, {0x1EB7, {0x61, 0x323, 0x306}}, {0x1EB8, {0x45, 0x323}}, -{0x1EB9, {0x65, 0x323}}, {0x1EBA, {0x45, 0x309}}, {0x1EBB, {0x65, 0x309}}, {0x1EBC, {0x45, 0x303}}, {0x1EBD, {0x65, 0x303}}, {0x1EBE, {0x45, 0x302, 0x301}}, {0x1EBF, {0x65, 0x302, 0x301}}, -{0x1EC0, {0x45, 0x302, 0x300}}, {0x1EC1, {0x65, 0x302, 0x300}}, {0x1EC2, {0x45, 0x302, 0x309}}, {0x1EC3, {0x65, 0x302, 0x309}}, {0x1EC4, {0x45, 0x302, 0x303}}, {0x1EC5, {0x65, 0x302, 0x303}}, -{0x1EC6, {0x45, 0x323, 0x302}}, {0x1EC7, {0x65, 0x323, 0x302}}, {0x1EC8, {0x49, 0x309}}, {0x1EC9, {0x69, 0x309}}, {0x1ECA, {0x49, 0x323}}, {0x1ECB, {0x69, 0x323}}, {0x1ECC, {0x4F, 0x323}}, -{0x1ECD, {0x6F, 0x323}}, {0x1ECE, {0x4F, 0x309}}, {0x1ECF, {0x6F, 0x309}}, {0x1ED0, {0x4F, 0x302, 0x301}}, {0x1ED1, {0x6F, 0x302, 0x301}}, {0x1ED2, {0x4F, 0x302, 0x300}}, -{0x1ED3, {0x6F, 0x302, 0x300}}, {0x1ED4, {0x4F, 0x302, 0x309}}, {0x1ED5, {0x6F, 0x302, 0x309}}, {0x1ED6, {0x4F, 0x302, 0x303}}, {0x1ED7, {0x6F, 0x302, 0x303}}, {0x1ED8, {0x4F, 0x323, 0x302}}, -{0x1ED9, {0x6F, 0x323, 0x302}}, {0x1EDA, {0x4F, 0x31B, 0x301}}, {0x1EDB, {0x6F, 0x31B, 0x301}}, {0x1EDC, {0x4F, 0x31B, 0x300}}, {0x1EDD, {0x6F, 0x31B, 0x300}}, {0x1EDE, {0x4F, 0x31B, 0x309}}, -{0x1EDF, {0x6F, 0x31B, 0x309}}, {0x1EE0, {0x4F, 0x31B, 0x303}}, {0x1EE1, {0x6F, 0x31B, 0x303}}, {0x1EE2, {0x4F, 0x31B, 0x323}}, {0x1EE3, {0x6F, 0x31B, 0x323}}, {0x1EE4, {0x55, 0x323}}, -{0x1EE5, {0x75, 0x323}}, {0x1EE6, {0x55, 0x309}}, {0x1EE7, {0x75, 0x309}}, {0x1EE8, {0x55, 0x31B, 0x301}}, {0x1EE9, {0x75, 0x31B, 0x301}}, {0x1EEA, {0x55, 0x31B, 0x300}}, -{0x1EEB, {0x75, 0x31B, 0x300}}, {0x1EEC, {0x55, 0x31B, 0x309}}, {0x1EED, {0x75, 0x31B, 0x309}}, {0x1EEE, {0x55, 0x31B, 0x303}}, {0x1EEF, {0x75, 0x31B, 0x303}}, {0x1EF0, {0x55, 0x31B, 0x323}}, -{0x1EF1, {0x75, 0x31B, 0x323}}, {0x1EF2, {0x59, 0x300}}, {0x1EF3, {0x79, 0x300}}, {0x1EF4, {0x59, 0x323}}, {0x1EF5, {0x79, 0x323}}, {0x1EF6, {0x59, 0x309}}, {0x1EF7, {0x79, 0x309}}, -{0x1EF8, {0x59, 0x303}}, {0x1EF9, {0x79, 0x303}}, {0x1F00, {0x3B1, 0x313}}, {0x1F01, {0x3B1, 0x314}}, {0x1F02, {0x3B1, 0x313, 0x300}}, {0x1F03, {0x3B1, 0x314, 0x300}}, {0x1F04, {0x3B1, 0x313, 0x301}}, -{0x1F05, {0x3B1, 0x314, 0x301}}, {0x1F06, {0x3B1, 0x313, 0x342}}, {0x1F07, {0x3B1, 0x314, 0x342}}, {0x1F08, {0x391, 0x313}}, {0x1F09, {0x391, 0x314}}, {0x1F0A, {0x391, 0x313, 0x300}}, -{0x1F0B, {0x391, 0x314, 0x300}}, {0x1F0C, {0x391, 0x313, 0x301}}, {0x1F0D, {0x391, 0x314, 0x301}}, {0x1F0E, {0x391, 0x313, 0x342}}, {0x1F0F, {0x391, 0x314, 0x342}}, {0x1F10, {0x3B5, 0x313}}, -{0x1F11, {0x3B5, 0x314}}, {0x1F12, {0x3B5, 0x313, 0x300}}, {0x1F13, {0x3B5, 0x314, 0x300}}, {0x1F14, {0x3B5, 0x313, 0x301}}, {0x1F15, {0x3B5, 0x314, 0x301}}, {0x1F18, {0x395, 0x313}}, -{0x1F19, {0x395, 0x314}}, {0x1F1A, {0x395, 0x313, 0x300}}, {0x1F1B, {0x395, 0x314, 0x300}}, {0x1F1C, {0x395, 0x313, 0x301}}, {0x1F1D, {0x395, 0x314, 0x301}}, {0x1F20, {0x3B7, 0x313}}, -{0x1F21, {0x3B7, 0x314}}, {0x1F22, {0x3B7, 0x313, 0x300}}, {0x1F23, {0x3B7, 0x314, 0x300}}, {0x1F24, {0x3B7, 0x313, 0x301}}, {0x1F25, {0x3B7, 0x314, 0x301}}, {0x1F26, {0x3B7, 0x313, 0x342}}, -{0x1F27, {0x3B7, 0x314, 0x342}}, {0x1F28, {0x397, 0x313}}, {0x1F29, {0x397, 0x314}}, {0x1F2A, {0x397, 0x313, 0x300}}, {0x1F2B, {0x397, 0x314, 0x300}}, {0x1F2C, {0x397, 0x313, 0x301}}, -{0x1F2D, {0x397, 0x314, 0x301}}, {0x1F2E, {0x397, 0x313, 0x342}}, {0x1F2F, {0x397, 0x314, 0x342}}, {0x1F30, {0x3B9, 0x313}}, {0x1F31, {0x3B9, 0x314}}, {0x1F32, {0x3B9, 0x313, 0x300}}, -{0x1F33, {0x3B9, 0x314, 0x300}}, {0x1F34, {0x3B9, 0x313, 0x301}}, {0x1F35, {0x3B9, 0x314, 0x301}}, {0x1F36, {0x3B9, 0x313, 0x342}}, {0x1F37, {0x3B9, 0x314, 0x342}}, {0x1F38, {0x399, 0x313}}, -{0x1F39, {0x399, 0x314}}, {0x1F3A, {0x399, 0x313, 0x300}}, {0x1F3B, {0x399, 0x314, 0x300}}, {0x1F3C, {0x399, 0x313, 0x301}}, {0x1F3D, {0x399, 0x314, 0x301}}, {0x1F3E, {0x399, 0x313, 0x342}}, -{0x1F3F, {0x399, 0x314, 0x342}}, {0x1F40, {0x3BF, 0x313}}, {0x1F41, {0x3BF, 0x314}}, {0x1F42, {0x3BF, 0x313, 0x300}}, {0x1F43, {0x3BF, 0x314, 0x300}}, {0x1F44, {0x3BF, 0x313, 0x301}}, -{0x1F45, {0x3BF, 0x314, 0x301}}, {0x1F48, {0x39F, 0x313}}, {0x1F49, {0x39F, 0x314}}, {0x1F4A, {0x39F, 0x313, 0x300}}, {0x1F4B, {0x39F, 0x314, 0x300}}, {0x1F4C, {0x39F, 0x313, 0x301}}, -{0x1F4D, {0x39F, 0x314, 0x301}}, {0x1F50, {0x3C5, 0x313}}, {0x1F51, {0x3C5, 0x314}}, {0x1F52, {0x3C5, 0x313, 0x300}}, {0x1F53, {0x3C5, 0x314, 0x300}}, {0x1F54, {0x3C5, 0x313, 0x301}}, -{0x1F55, {0x3C5, 0x314, 0x301}}, {0x1F56, {0x3C5, 0x313, 0x342}}, {0x1F57, {0x3C5, 0x314, 0x342}}, {0x1F59, {0x3A5, 0x314}}, {0x1F5B, {0x3A5, 0x314, 0x300}}, {0x1F5D, {0x3A5, 0x314, 0x301}}, -{0x1F5F, {0x3A5, 0x314, 0x342}}, {0x1F60, {0x3C9, 0x313}}, {0x1F61, {0x3C9, 0x314}}, {0x1F62, {0x3C9, 0x313, 0x300}}, {0x1F63, {0x3C9, 0x314, 0x300}}, {0x1F64, {0x3C9, 0x313, 0x301}}, -{0x1F65, {0x3C9, 0x314, 0x301}}, {0x1F66, {0x3C9, 0x313, 0x342}}, {0x1F67, {0x3C9, 0x314, 0x342}}, {0x1F68, {0x3A9, 0x313}}, {0x1F69, {0x3A9, 0x314}}, {0x1F6A, {0x3A9, 0x313, 0x300}}, -{0x1F6B, {0x3A9, 0x314, 0x300}}, {0x1F6C, {0x3A9, 0x313, 0x301}}, {0x1F6D, {0x3A9, 0x314, 0x301}}, {0x1F6E, {0x3A9, 0x313, 0x342}}, {0x1F6F, {0x3A9, 0x314, 0x342}}, {0x1F70, {0x3B1, 0x300}}, -{0x1F71, {0x3B1, 0x301}}, {0x1F72, {0x3B5, 0x300}}, {0x1F73, {0x3B5, 0x301}}, {0x1F74, {0x3B7, 0x300}}, {0x1F75, {0x3B7, 0x301}}, {0x1F76, {0x3B9, 0x300}}, {0x1F77, {0x3B9, 0x301}}, -{0x1F78, {0x3BF, 0x300}}, {0x1F79, {0x3BF, 0x301}}, {0x1F7A, {0x3C5, 0x300}}, {0x1F7B, {0x3C5, 0x301}}, {0x1F7C, {0x3C9, 0x300}}, {0x1F7D, {0x3C9, 0x301}}, {0x1F80, {0x3B1, 0x313, 0x345}}, -{0x1F81, {0x3B1, 0x314, 0x345}}, {0x1F82, {0x3B1, 0x313, 0x300, 0x345}}, {0x1F83, {0x3B1, 0x314, 0x300, 0x345}}, {0x1F84, {0x3B1, 0x313, 0x301, 0x345}}, {0x1F85, {0x3B1, 0x314, 0x301, 0x345}}, -{0x1F86, {0x3B1, 0x313, 0x342, 0x345}}, {0x1F87, {0x3B1, 0x314, 0x342, 0x345}}, {0x1F88, {0x391, 0x313, 0x345}}, {0x1F89, {0x391, 0x314, 0x345}}, {0x1F8A, {0x391, 0x313, 0x300, 0x345}}, -{0x1F8B, {0x391, 0x314, 0x300, 0x345}}, {0x1F8C, {0x391, 0x313, 0x301, 0x345}}, {0x1F8D, {0x391, 0x314, 0x301, 0x345}}, {0x1F8E, {0x391, 0x313, 0x342, 0x345}}, {0x1F8F, {0x391, 0x314, 0x342, 0x345}}, -{0x1F90, {0x3B7, 0x313, 0x345}}, {0x1F91, {0x3B7, 0x314, 0x345}}, {0x1F92, {0x3B7, 0x313, 0x300, 0x345}}, {0x1F93, {0x3B7, 0x314, 0x300, 0x345}}, {0x1F94, {0x3B7, 0x313, 0x301, 0x345}}, -{0x1F95, {0x3B7, 0x314, 0x301, 0x345}}, {0x1F96, {0x3B7, 0x313, 0x342, 0x345}}, {0x1F97, {0x3B7, 0x314, 0x342, 0x345}}, {0x1F98, {0x397, 0x313, 0x345}}, {0x1F99, {0x397, 0x314, 0x345}}, -{0x1F9A, {0x397, 0x313, 0x300, 0x345}}, {0x1F9B, {0x397, 0x314, 0x300, 0x345}}, {0x1F9C, {0x397, 0x313, 0x301, 0x345}}, {0x1F9D, {0x397, 0x314, 0x301, 0x345}}, {0x1F9E, {0x397, 0x313, 0x342, 0x345}}, -{0x1F9F, {0x397, 0x314, 0x342, 0x345}}, {0x1FA0, {0x3C9, 0x313, 0x345}}, {0x1FA1, {0x3C9, 0x314, 0x345}}, {0x1FA2, {0x3C9, 0x313, 0x300, 0x345}}, {0x1FA3, {0x3C9, 0x314, 0x300, 0x345}}, -{0x1FA4, {0x3C9, 0x313, 0x301, 0x345}}, {0x1FA5, {0x3C9, 0x314, 0x301, 0x345}}, {0x1FA6, {0x3C9, 0x313, 0x342, 0x345}}, {0x1FA7, {0x3C9, 0x314, 0x342, 0x345}}, {0x1FA8, {0x3A9, 0x313, 0x345}}, -{0x1FA9, {0x3A9, 0x314, 0x345}}, {0x1FAA, {0x3A9, 0x313, 0x300, 0x345}}, {0x1FAB, {0x3A9, 0x314, 0x300, 0x345}}, {0x1FAC, {0x3A9, 0x313, 0x301, 0x345}}, {0x1FAD, {0x3A9, 0x314, 0x301, 0x345}}, -{0x1FAE, {0x3A9, 0x313, 0x342, 0x345}}, {0x1FAF, {0x3A9, 0x314, 0x342, 0x345}}, {0x1FB0, {0x3B1, 0x306}}, {0x1FB1, {0x3B1, 0x304}}, {0x1FB2, {0x3B1, 0x300, 0x345}}, {0x1FB3, {0x3B1, 0x345}}, -{0x1FB4, {0x3B1, 0x301, 0x345}}, {0x1FB6, {0x3B1, 0x342}}, {0x1FB7, {0x3B1, 0x342, 0x345}}, {0x1FB8, {0x391, 0x306}}, {0x1FB9, {0x391, 0x304}}, {0x1FBA, {0x391, 0x300}}, {0x1FBB, {0x391, 0x301}}, -{0x1FBC, {0x391, 0x345}}, {0x1FBE, {0x3B9}}, {0x1FC1, {0xA8, 0x342}}, {0x1FC2, {0x3B7, 0x300, 0x345}}, {0x1FC3, {0x3B7, 0x345}}, {0x1FC4, {0x3B7, 0x301, 0x345}}, {0x1FC6, {0x3B7, 0x342}}, -{0x1FC7, {0x3B7, 0x342, 0x345}}, {0x1FC8, {0x395, 0x300}}, {0x1FC9, {0x395, 0x301}}, {0x1FCA, {0x397, 0x300}}, {0x1FCB, {0x397, 0x301}}, {0x1FCC, {0x397, 0x345}}, {0x1FCD, {0x1FBF, 0x300}}, -{0x1FCE, {0x1FBF, 0x301}}, {0x1FCF, {0x1FBF, 0x342}}, {0x1FD0, {0x3B9, 0x306}}, {0x1FD1, {0x3B9, 0x304}}, {0x1FD2, {0x3B9, 0x308, 0x300}}, {0x1FD3, {0x3B9, 0x308, 0x301}}, {0x1FD6, {0x3B9, 0x342}}, -{0x1FD7, {0x3B9, 0x308, 0x342}}, {0x1FD8, {0x399, 0x306}}, {0x1FD9, {0x399, 0x304}}, {0x1FDA, {0x399, 0x300}}, {0x1FDB, {0x399, 0x301}}, {0x1FDD, {0x1FFE, 0x300}}, {0x1FDE, {0x1FFE, 0x301}}, -{0x1FDF, {0x1FFE, 0x342}}, {0x1FE0, {0x3C5, 0x306}}, {0x1FE1, {0x3C5, 0x304}}, {0x1FE2, {0x3C5, 0x308, 0x300}}, {0x1FE3, {0x3C5, 0x308, 0x301}}, {0x1FE4, {0x3C1, 0x313}}, {0x1FE5, {0x3C1, 0x314}}, -{0x1FE6, {0x3C5, 0x342}}, {0x1FE7, {0x3C5, 0x308, 0x342}}, {0x1FE8, {0x3A5, 0x306}}, {0x1FE9, {0x3A5, 0x304}}, {0x1FEA, {0x3A5, 0x300}}, {0x1FEB, {0x3A5, 0x301}}, {0x1FEC, {0x3A1, 0x314}}, -{0x1FED, {0xA8, 0x300}}, {0x1FEE, {0xA8, 0x301}}, {0x1FEF, {0x60}}, {0x1FF2, {0x3C9, 0x300, 0x345}}, {0x1FF3, {0x3C9, 0x345}}, {0x1FF4, {0x3C9, 0x301, 0x345}}, {0x1FF6, {0x3C9, 0x342}}, -{0x1FF7, {0x3C9, 0x342, 0x345}}, {0x1FF8, {0x39F, 0x300}}, {0x1FF9, {0x39F, 0x301}}, {0x1FFA, {0x3A9, 0x300}}, {0x1FFB, {0x3A9, 0x301}}, {0x1FFC, {0x3A9, 0x345}}, {0x1FFD, {0xB4}}, {0x2000, {0x2002}}, -{0x2001, {0x2003}}, {0x2126, {0x3A9}}, {0x212A, {0x4B}}, {0x212B, {0x41, 0x30A}}, {0x219A, {0x2190, 0x338}}, {0x219B, {0x2192, 0x338}}, {0x21AE, {0x2194, 0x338}}, {0x21CD, {0x21D0, 0x338}}, -{0x21CE, {0x21D4, 0x338}}, {0x21CF, {0x21D2, 0x338}}, {0x2204, {0x2203, 0x338}}, {0x2209, {0x2208, 0x338}}, {0x220C, {0x220B, 0x338}}, {0x2224, {0x2223, 0x338}}, {0x2226, {0x2225, 0x338}}, -{0x2241, {0x223C, 0x338}}, {0x2244, {0x2243, 0x338}}, {0x2247, {0x2245, 0x338}}, {0x2249, {0x2248, 0x338}}, {0x2260, {0x3D, 0x338}}, {0x2262, {0x2261, 0x338}}, {0x226D, {0x224D, 0x338}}, -{0x226E, {0x3C, 0x338}}, {0x226F, {0x3E, 0x338}}, {0x2270, {0x2264, 0x338}}, {0x2271, {0x2265, 0x338}}, {0x2274, {0x2272, 0x338}}, {0x2275, {0x2273, 0x338}}, {0x2278, {0x2276, 0x338}}, -{0x2279, {0x2277, 0x338}}, {0x2280, {0x227A, 0x338}}, {0x2281, {0x227B, 0x338}}, {0x2284, {0x2282, 0x338}}, {0x2285, {0x2283, 0x338}}, {0x2288, {0x2286, 0x338}}, {0x2289, {0x2287, 0x338}}, -{0x22AC, {0x22A2, 0x338}}, {0x22AD, {0x22A8, 0x338}}, {0x22AE, {0x22A9, 0x338}}, {0x22AF, {0x22AB, 0x338}}, {0x22E0, {0x227C, 0x338}}, {0x22E1, {0x227D, 0x338}}, {0x22E2, {0x2291, 0x338}}, -{0x22E3, {0x2292, 0x338}}, {0x22EA, {0x22B2, 0x338}}, {0x22EB, {0x22B3, 0x338}}, {0x22EC, {0x22B4, 0x338}}, {0x22ED, {0x22B5, 0x338}}, {0x2329, {0x3008}}, {0x232A, {0x3009}}, -{0x2ADC, {0x2ADD, 0x338}}, {0x304C, {0x304B, 0x3099}}, {0x304E, {0x304D, 0x3099}}, {0x3050, {0x304F, 0x3099}}, {0x3052, {0x3051, 0x3099}}, {0x3054, {0x3053, 0x3099}}, {0x3056, {0x3055, 0x3099}}, -{0x3058, {0x3057, 0x3099}}, {0x305A, {0x3059, 0x3099}}, {0x305C, {0x305B, 0x3099}}, {0x305E, {0x305D, 0x3099}}, {0x3060, {0x305F, 0x3099}}, {0x3062, {0x3061, 0x3099}}, {0x3065, {0x3064, 0x3099}}, -{0x3067, {0x3066, 0x3099}}, {0x3069, {0x3068, 0x3099}}, {0x3070, {0x306F, 0x3099}}, {0x3071, {0x306F, 0x309A}}, {0x3073, {0x3072, 0x3099}}, {0x3074, {0x3072, 0x309A}}, {0x3076, {0x3075, 0x3099}}, -{0x3077, {0x3075, 0x309A}}, {0x3079, {0x3078, 0x3099}}, {0x307A, {0x3078, 0x309A}}, {0x307C, {0x307B, 0x3099}}, {0x307D, {0x307B, 0x309A}}, {0x3094, {0x3046, 0x3099}}, {0x309E, {0x309D, 0x3099}}, -{0x30AC, {0x30AB, 0x3099}}, {0x30AE, {0x30AD, 0x3099}}, {0x30B0, {0x30AF, 0x3099}}, {0x30B2, {0x30B1, 0x3099}}, {0x30B4, {0x30B3, 0x3099}}, {0x30B6, {0x30B5, 0x3099}}, {0x30B8, {0x30B7, 0x3099}}, -{0x30BA, {0x30B9, 0x3099}}, {0x30BC, {0x30BB, 0x3099}}, {0x30BE, {0x30BD, 0x3099}}, {0x30C0, {0x30BF, 0x3099}}, {0x30C2, {0x30C1, 0x3099}}, {0x30C5, {0x30C4, 0x3099}}, {0x30C7, {0x30C6, 0x3099}}, -{0x30C9, {0x30C8, 0x3099}}, {0x30D0, {0x30CF, 0x3099}}, {0x30D1, {0x30CF, 0x309A}}, {0x30D3, {0x30D2, 0x3099}}, {0x30D4, {0x30D2, 0x309A}}, {0x30D6, {0x30D5, 0x3099}}, {0x30D7, {0x30D5, 0x309A}}, -{0x30D9, {0x30D8, 0x3099}}, {0x30DA, {0x30D8, 0x309A}}, {0x30DC, {0x30DB, 0x3099}}, {0x30DD, {0x30DB, 0x309A}}, {0x30F4, {0x30A6, 0x3099}}, {0x30F7, {0x30EF, 0x3099}}, {0x30F8, {0x30F0, 0x3099}}, -{0x30F9, {0x30F1, 0x3099}}, {0x30FA, {0x30F2, 0x3099}}, {0x30FE, {0x30FD, 0x3099}}, {0xF900, {0x8C48}}, {0xF901, {0x66F4}}, {0xF902, {0x8ECA}}, {0xF903, {0x8CC8}}, {0xF904, {0x6ED1}}, -{0xF905, {0x4E32}}, {0xF906, {0x53E5}}, {0xF907, {0x9F9C}}, {0xF908, {0x9F9C}}, {0xF909, {0x5951}}, {0xF90A, {0x91D1}}, {0xF90B, {0x5587}}, {0xF90C, {0x5948}}, {0xF90D, {0x61F6}}, {0xF90E, {0x7669}}, -{0xF90F, {0x7F85}}, {0xF910, {0x863F}}, {0xF911, {0x87BA}}, {0xF912, {0x88F8}}, {0xF913, {0x908F}}, {0xF914, {0x6A02}}, {0xF915, {0x6D1B}}, {0xF916, {0x70D9}}, {0xF917, {0x73DE}}, {0xF918, {0x843D}}, -{0xF919, {0x916A}}, {0xF91A, {0x99F1}}, {0xF91B, {0x4E82}}, {0xF91C, {0x5375}}, {0xF91D, {0x6B04}}, {0xF91E, {0x721B}}, {0xF91F, {0x862D}}, {0xF920, {0x9E1E}}, {0xF921, {0x5D50}}, {0xF922, {0x6FEB}}, -{0xF923, {0x85CD}}, {0xF924, {0x8964}}, {0xF925, {0x62C9}}, {0xF926, {0x81D8}}, {0xF927, {0x881F}}, {0xF928, {0x5ECA}}, {0xF929, {0x6717}}, {0xF92A, {0x6D6A}}, {0xF92B, {0x72FC}}, {0xF92C, {0x90CE}}, -{0xF92D, {0x4F86}}, {0xF92E, {0x51B7}}, {0xF92F, {0x52DE}}, {0xF930, {0x64C4}}, {0xF931, {0x6AD3}}, {0xF932, {0x7210}}, {0xF933, {0x76E7}}, {0xF934, {0x8001}}, {0xF935, {0x8606}}, {0xF936, {0x865C}}, -{0xF937, {0x8DEF}}, {0xF938, {0x9732}}, {0xF939, {0x9B6F}}, {0xF93A, {0x9DFA}}, {0xF93B, {0x788C}}, {0xF93C, {0x797F}}, {0xF93D, {0x7DA0}}, {0xF93E, {0x83C9}}, {0xF93F, {0x9304}}, {0xF940, {0x9E7F}}, -{0xF941, {0x8AD6}}, {0xF942, {0x58DF}}, {0xF943, {0x5F04}}, {0xF944, {0x7C60}}, {0xF945, {0x807E}}, {0xF946, {0x7262}}, {0xF947, {0x78CA}}, {0xF948, {0x8CC2}}, {0xF949, {0x96F7}}, {0xF94A, {0x58D8}}, -{0xF94B, {0x5C62}}, {0xF94C, {0x6A13}}, {0xF94D, {0x6DDA}}, {0xF94E, {0x6F0F}}, {0xF94F, {0x7D2F}}, {0xF950, {0x7E37}}, {0xF951, {0x964B}}, {0xF952, {0x52D2}}, {0xF953, {0x808B}}, {0xF954, {0x51DC}}, -{0xF955, {0x51CC}}, {0xF956, {0x7A1C}}, {0xF957, {0x7DBE}}, {0xF958, {0x83F1}}, {0xF959, {0x9675}}, {0xF95A, {0x8B80}}, {0xF95B, {0x62CF}}, {0xF95C, {0x6A02}}, {0xF95D, {0x8AFE}}, {0xF95E, {0x4E39}}, -{0xF95F, {0x5BE7}}, {0xF960, {0x6012}}, {0xF961, {0x7387}}, {0xF962, {0x7570}}, {0xF963, {0x5317}}, {0xF964, {0x78FB}}, {0xF965, {0x4FBF}}, {0xF966, {0x5FA9}}, {0xF967, {0x4E0D}}, {0xF968, {0x6CCC}}, -{0xF969, {0x6578}}, {0xF96A, {0x7D22}}, {0xF96B, {0x53C3}}, {0xF96C, {0x585E}}, {0xF96D, {0x7701}}, {0xF96E, {0x8449}}, {0xF96F, {0x8AAA}}, {0xF970, {0x6BBA}}, {0xF971, {0x8FB0}}, {0xF972, {0x6C88}}, -{0xF973, {0x62FE}}, {0xF974, {0x82E5}}, {0xF975, {0x63A0}}, {0xF976, {0x7565}}, {0xF977, {0x4EAE}}, {0xF978, {0x5169}}, {0xF979, {0x51C9}}, {0xF97A, {0x6881}}, {0xF97B, {0x7CE7}}, {0xF97C, {0x826F}}, -{0xF97D, {0x8AD2}}, {0xF97E, {0x91CF}}, {0xF97F, {0x52F5}}, {0xF980, {0x5442}}, {0xF981, {0x5973}}, {0xF982, {0x5EEC}}, {0xF983, {0x65C5}}, {0xF984, {0x6FFE}}, {0xF985, {0x792A}}, {0xF986, {0x95AD}}, -{0xF987, {0x9A6A}}, {0xF988, {0x9E97}}, {0xF989, {0x9ECE}}, {0xF98A, {0x529B}}, {0xF98B, {0x66C6}}, {0xF98C, {0x6B77}}, {0xF98D, {0x8F62}}, {0xF98E, {0x5E74}}, {0xF98F, {0x6190}}, {0xF990, {0x6200}}, -{0xF991, {0x649A}}, {0xF992, {0x6F23}}, {0xF993, {0x7149}}, {0xF994, {0x7489}}, {0xF995, {0x79CA}}, {0xF996, {0x7DF4}}, {0xF997, {0x806F}}, {0xF998, {0x8F26}}, {0xF999, {0x84EE}}, {0xF99A, {0x9023}}, -{0xF99B, {0x934A}}, {0xF99C, {0x5217}}, {0xF99D, {0x52A3}}, {0xF99E, {0x54BD}}, {0xF99F, {0x70C8}}, {0xF9A0, {0x88C2}}, {0xF9A1, {0x8AAA}}, {0xF9A2, {0x5EC9}}, {0xF9A3, {0x5FF5}}, {0xF9A4, {0x637B}}, -{0xF9A5, {0x6BAE}}, {0xF9A6, {0x7C3E}}, {0xF9A7, {0x7375}}, {0xF9A8, {0x4EE4}}, {0xF9A9, {0x56F9}}, {0xF9AA, {0x5BE7}}, {0xF9AB, {0x5DBA}}, {0xF9AC, {0x601C}}, {0xF9AD, {0x73B2}}, {0xF9AE, {0x7469}}, -{0xF9AF, {0x7F9A}}, {0xF9B0, {0x8046}}, {0xF9B1, {0x9234}}, {0xF9B2, {0x96F6}}, {0xF9B3, {0x9748}}, {0xF9B4, {0x9818}}, {0xF9B5, {0x4F8B}}, {0xF9B6, {0x79AE}}, {0xF9B7, {0x91B4}}, {0xF9B8, {0x96B8}}, -{0xF9B9, {0x60E1}}, {0xF9BA, {0x4E86}}, {0xF9BB, {0x50DA}}, {0xF9BC, {0x5BEE}}, {0xF9BD, {0x5C3F}}, {0xF9BE, {0x6599}}, {0xF9BF, {0x6A02}}, {0xF9C0, {0x71CE}}, {0xF9C1, {0x7642}}, {0xF9C2, {0x84FC}}, -{0xF9C3, {0x907C}}, {0xF9C4, {0x9F8D}}, {0xF9C5, {0x6688}}, {0xF9C6, {0x962E}}, {0xF9C7, {0x5289}}, {0xF9C8, {0x677B}}, {0xF9C9, {0x67F3}}, {0xF9CA, {0x6D41}}, {0xF9CB, {0x6E9C}}, {0xF9CC, {0x7409}}, -{0xF9CD, {0x7559}}, {0xF9CE, {0x786B}}, {0xF9CF, {0x7D10}}, {0xF9D0, {0x985E}}, {0xF9D1, {0x516D}}, {0xF9D2, {0x622E}}, {0xF9D3, {0x9678}}, {0xF9D4, {0x502B}}, {0xF9D5, {0x5D19}}, {0xF9D6, {0x6DEA}}, -{0xF9D7, {0x8F2A}}, {0xF9D8, {0x5F8B}}, {0xF9D9, {0x6144}}, {0xF9DA, {0x6817}}, {0xF9DB, {0x7387}}, {0xF9DC, {0x9686}}, {0xF9DD, {0x5229}}, {0xF9DE, {0x540F}}, {0xF9DF, {0x5C65}}, {0xF9E0, {0x6613}}, -{0xF9E1, {0x674E}}, {0xF9E2, {0x68A8}}, {0xF9E3, {0x6CE5}}, {0xF9E4, {0x7406}}, {0xF9E5, {0x75E2}}, {0xF9E6, {0x7F79}}, {0xF9E7, {0x88CF}}, {0xF9E8, {0x88E1}}, {0xF9E9, {0x91CC}}, {0xF9EA, {0x96E2}}, -{0xF9EB, {0x533F}}, {0xF9EC, {0x6EBA}}, {0xF9ED, {0x541D}}, {0xF9EE, {0x71D0}}, {0xF9EF, {0x7498}}, {0xF9F0, {0x85FA}}, {0xF9F1, {0x96A3}}, {0xF9F2, {0x9C57}}, {0xF9F3, {0x9E9F}}, {0xF9F4, {0x6797}}, -{0xF9F5, {0x6DCB}}, {0xF9F6, {0x81E8}}, {0xF9F7, {0x7ACB}}, {0xF9F8, {0x7B20}}, {0xF9F9, {0x7C92}}, {0xF9FA, {0x72C0}}, {0xF9FB, {0x7099}}, {0xF9FC, {0x8B58}}, {0xF9FD, {0x4EC0}}, {0xF9FE, {0x8336}}, -{0xF9FF, {0x523A}}, {0xFA00, {0x5207}}, {0xFA01, {0x5EA6}}, {0xFA02, {0x62D3}}, {0xFA03, {0x7CD6}}, {0xFA04, {0x5B85}}, {0xFA05, {0x6D1E}}, {0xFA06, {0x66B4}}, {0xFA07, {0x8F3B}}, {0xFA08, {0x884C}}, -{0xFA09, {0x964D}}, {0xFA0A, {0x898B}}, {0xFA0B, {0x5ED3}}, {0xFA0C, {0x5140}}, {0xFA0D, {0x55C0}}, {0xFA10, {0x585A}}, {0xFA12, {0x6674}}, {0xFA15, {0x51DE}}, {0xFA16, {0x732A}}, {0xFA17, {0x76CA}}, -{0xFA18, {0x793C}}, {0xFA19, {0x795E}}, {0xFA1A, {0x7965}}, {0xFA1B, {0x798F}}, {0xFA1C, {0x9756}}, {0xFA1D, {0x7CBE}}, {0xFA1E, {0x7FBD}}, {0xFA20, {0x8612}}, {0xFA22, {0x8AF8}}, {0xFA25, {0x9038}}, -{0xFA26, {0x90FD}}, {0xFA2A, {0x98EF}}, {0xFA2B, {0x98FC}}, {0xFA2C, {0x9928}}, {0xFA2D, {0x9DB4}}, {0xFA2E, {0x90DE}}, {0xFA2F, {0x96B7}}, {0xFA30, {0x4FAE}}, {0xFA31, {0x50E7}}, {0xFA32, {0x514D}}, -{0xFA33, {0x52C9}}, {0xFA34, {0x52E4}}, {0xFA35, {0x5351}}, {0xFA36, {0x559D}}, {0xFA37, {0x5606}}, {0xFA38, {0x5668}}, {0xFA39, {0x5840}}, {0xFA3A, {0x58A8}}, {0xFA3B, {0x5C64}}, {0xFA3C, {0x5C6E}}, -{0xFA3D, {0x6094}}, {0xFA3E, {0x6168}}, {0xFA3F, {0x618E}}, {0xFA40, {0x61F2}}, {0xFA41, {0x654F}}, {0xFA42, {0x65E2}}, {0xFA43, {0x6691}}, {0xFA44, {0x6885}}, {0xFA45, {0x6D77}}, {0xFA46, {0x6E1A}}, -{0xFA47, {0x6F22}}, {0xFA48, {0x716E}}, {0xFA49, {0x722B}}, {0xFA4A, {0x7422}}, {0xFA4B, {0x7891}}, {0xFA4C, {0x793E}}, {0xFA4D, {0x7949}}, {0xFA4E, {0x7948}}, {0xFA4F, {0x7950}}, {0xFA50, {0x7956}}, -{0xFA51, {0x795D}}, {0xFA52, {0x798D}}, {0xFA53, {0x798E}}, {0xFA54, {0x7A40}}, {0xFA55, {0x7A81}}, {0xFA56, {0x7BC0}}, {0xFA57, {0x7DF4}}, {0xFA58, {0x7E09}}, {0xFA59, {0x7E41}}, {0xFA5A, {0x7F72}}, -{0xFA5B, {0x8005}}, {0xFA5C, {0x81ED}}, {0xFA5D, {0x8279}}, {0xFA5E, {0x8279}}, {0xFA5F, {0x8457}}, {0xFA60, {0x8910}}, {0xFA61, {0x8996}}, {0xFA62, {0x8B01}}, {0xFA63, {0x8B39}}, {0xFA64, {0x8CD3}}, -{0xFA65, {0x8D08}}, {0xFA66, {0x8FB6}}, {0xFA67, {0x9038}}, {0xFA68, {0x96E3}}, {0xFA69, {0x97FF}}, {0xFA6A, {0x983B}}, {0xFA6B, {0x6075}}, {0xFA6C, {0x242EE}}, {0xFA6D, {0x8218}}, {0xFA70, {0x4E26}}, -{0xFA71, {0x51B5}}, {0xFA72, {0x5168}}, {0xFA73, {0x4F80}}, {0xFA74, {0x5145}}, {0xFA75, {0x5180}}, {0xFA76, {0x52C7}}, {0xFA77, {0x52FA}}, {0xFA78, {0x559D}}, {0xFA79, {0x5555}}, {0xFA7A, {0x5599}}, -{0xFA7B, {0x55E2}}, {0xFA7C, {0x585A}}, {0xFA7D, {0x58B3}}, {0xFA7E, {0x5944}}, {0xFA7F, {0x5954}}, {0xFA80, {0x5A62}}, {0xFA81, {0x5B28}}, {0xFA82, {0x5ED2}}, {0xFA83, {0x5ED9}}, {0xFA84, {0x5F69}}, -{0xFA85, {0x5FAD}}, {0xFA86, {0x60D8}}, {0xFA87, {0x614E}}, {0xFA88, {0x6108}}, {0xFA89, {0x618E}}, {0xFA8A, {0x6160}}, {0xFA8B, {0x61F2}}, {0xFA8C, {0x6234}}, {0xFA8D, {0x63C4}}, {0xFA8E, {0x641C}}, -{0xFA8F, {0x6452}}, {0xFA90, {0x6556}}, {0xFA91, {0x6674}}, {0xFA92, {0x6717}}, {0xFA93, {0x671B}}, {0xFA94, {0x6756}}, {0xFA95, {0x6B79}}, {0xFA96, {0x6BBA}}, {0xFA97, {0x6D41}}, {0xFA98, {0x6EDB}}, -{0xFA99, {0x6ECB}}, {0xFA9A, {0x6F22}}, {0xFA9B, {0x701E}}, {0xFA9C, {0x716E}}, {0xFA9D, {0x77A7}}, {0xFA9E, {0x7235}}, {0xFA9F, {0x72AF}}, {0xFAA0, {0x732A}}, {0xFAA1, {0x7471}}, {0xFAA2, {0x7506}}, -{0xFAA3, {0x753B}}, {0xFAA4, {0x761D}}, {0xFAA5, {0x761F}}, {0xFAA6, {0x76CA}}, {0xFAA7, {0x76DB}}, {0xFAA8, {0x76F4}}, {0xFAA9, {0x774A}}, {0xFAAA, {0x7740}}, {0xFAAB, {0x78CC}}, {0xFAAC, {0x7AB1}}, -{0xFAAD, {0x7BC0}}, {0xFAAE, {0x7C7B}}, {0xFAAF, {0x7D5B}}, {0xFAB0, {0x7DF4}}, {0xFAB1, {0x7F3E}}, {0xFAB2, {0x8005}}, {0xFAB3, {0x8352}}, {0xFAB4, {0x83EF}}, {0xFAB5, {0x8779}}, {0xFAB6, {0x8941}}, -{0xFAB7, {0x8986}}, {0xFAB8, {0x8996}}, {0xFAB9, {0x8ABF}}, {0xFABA, {0x8AF8}}, {0xFABB, {0x8ACB}}, {0xFABC, {0x8B01}}, {0xFABD, {0x8AFE}}, {0xFABE, {0x8AED}}, {0xFABF, {0x8B39}}, {0xFAC0, {0x8B8A}}, -{0xFAC1, {0x8D08}}, {0xFAC2, {0x8F38}}, {0xFAC3, {0x9072}}, {0xFAC4, {0x9199}}, {0xFAC5, {0x9276}}, {0xFAC6, {0x967C}}, {0xFAC7, {0x96E3}}, {0xFAC8, {0x9756}}, {0xFAC9, {0x97DB}}, {0xFACA, {0x97FF}}, -{0xFACB, {0x980B}}, {0xFACC, {0x983B}}, {0xFACD, {0x9B12}}, {0xFACE, {0x9F9C}}, {0xFACF, {0x2284A}}, {0xFAD0, {0x22844}}, {0xFAD1, {0x233D5}}, {0xFAD2, {0x3B9D}}, {0xFAD3, {0x4018}}, -{0xFAD4, {0x4039}}, {0xFAD5, {0x25249}}, {0xFAD6, {0x25CD0}}, {0xFAD7, {0x27ED3}}, {0xFAD8, {0x9F43}}, {0xFAD9, {0x9F8E}}, {0xFB1D, {0x5D9, 0x5B4}}, {0xFB1F, {0x5F2, 0x5B7}}, {0xFB2A, {0x5E9, 0x5C1}}, -{0xFB2B, {0x5E9, 0x5C2}}, {0xFB2C, {0x5E9, 0x5BC, 0x5C1}}, {0xFB2D, {0x5E9, 0x5BC, 0x5C2}}, {0xFB2E, {0x5D0, 0x5B7}}, {0xFB2F, {0x5D0, 0x5B8}}, {0xFB30, {0x5D0, 0x5BC}}, {0xFB31, {0x5D1, 0x5BC}}, -{0xFB32, {0x5D2, 0x5BC}}, {0xFB33, {0x5D3, 0x5BC}}, {0xFB34, {0x5D4, 0x5BC}}, {0xFB35, {0x5D5, 0x5BC}}, {0xFB36, {0x5D6, 0x5BC}}, {0xFB38, {0x5D8, 0x5BC}}, {0xFB39, {0x5D9, 0x5BC}}, -{0xFB3A, {0x5DA, 0x5BC}}, {0xFB3B, {0x5DB, 0x5BC}}, {0xFB3C, {0x5DC, 0x5BC}}, {0xFB3E, {0x5DE, 0x5BC}}, {0xFB40, {0x5E0, 0x5BC}}, {0xFB41, {0x5E1, 0x5BC}}, {0xFB43, {0x5E3, 0x5BC}}, -{0xFB44, {0x5E4, 0x5BC}}, {0xFB46, {0x5E6, 0x5BC}}, {0xFB47, {0x5E7, 0x5BC}}, {0xFB48, {0x5E8, 0x5BC}}, {0xFB49, {0x5E9, 0x5BC}}, {0xFB4A, {0x5EA, 0x5BC}}, {0xFB4B, {0x5D5, 0x5B9}}, -{0xFB4C, {0x5D1, 0x5BF}}, {0xFB4D, {0x5DB, 0x5BF}}, {0xFB4E, {0x5E4, 0x5BF}}, {0x1109A, {0x11099, 0x110BA}}, {0x1109C, {0x1109B, 0x110BA}}, {0x110AB, {0x110A5, 0x110BA}}, -{0x1112E, {0x11131, 0x11127}}, {0x1112F, {0x11132, 0x11127}}, {0x1134B, {0x11347, 0x1133E}}, {0x1134C, {0x11347, 0x11357}}, {0x114BB, {0x114B9, 0x114BA}}, {0x114BC, {0x114B9, 0x114B0}}, -{0x114BE, {0x114B9, 0x114BD}}, {0x115BA, {0x115B8, 0x115AF}}, {0x115BB, {0x115B9, 0x115AF}}, {0x1D15E, {0x1D157, 0x1D165}}, {0x1D15F, {0x1D158, 0x1D165}}, {0x1D160, {0x1D158, 0x1D165, 0x1D16E}}, -{0x1D161, {0x1D158, 0x1D165, 0x1D16F}}, {0x1D162, {0x1D158, 0x1D165, 0x1D170}}, {0x1D163, {0x1D158, 0x1D165, 0x1D171}}, {0x1D164, {0x1D158, 0x1D165, 0x1D172}}, {0x1D1BB, {0x1D1B9, 0x1D165}}, -{0x1D1BC, {0x1D1BA, 0x1D165}}, {0x1D1BD, {0x1D1B9, 0x1D165, 0x1D16E}}, {0x1D1BE, {0x1D1BA, 0x1D165, 0x1D16E}}, {0x1D1BF, {0x1D1B9, 0x1D165, 0x1D16F}}, {0x1D1C0, {0x1D1BA, 0x1D165, 0x1D16F}}, -{0x2F800, {0x4E3D}}, {0x2F801, {0x4E38}}, {0x2F802, {0x4E41}}, {0x2F803, {0x20122}}, {0x2F804, {0x4F60}}, {0x2F805, {0x4FAE}}, {0x2F806, {0x4FBB}}, {0x2F807, {0x5002}}, {0x2F808, {0x507A}}, -{0x2F809, {0x5099}}, {0x2F80A, {0x50E7}}, {0x2F80B, {0x50CF}}, {0x2F80C, {0x349E}}, {0x2F80D, {0x2063A}}, {0x2F80E, {0x514D}}, {0x2F80F, {0x5154}}, {0x2F810, {0x5164}}, {0x2F811, {0x5177}}, -{0x2F812, {0x2051C}}, {0x2F813, {0x34B9}}, {0x2F814, {0x5167}}, {0x2F815, {0x518D}}, {0x2F816, {0x2054B}}, {0x2F817, {0x5197}}, {0x2F818, {0x51A4}}, {0x2F819, {0x4ECC}}, {0x2F81A, {0x51AC}}, -{0x2F81B, {0x51B5}}, {0x2F81C, {0x291DF}}, {0x2F81D, {0x51F5}}, {0x2F81E, {0x5203}}, {0x2F81F, {0x34DF}}, {0x2F820, {0x523B}}, {0x2F821, {0x5246}}, {0x2F822, {0x5272}}, {0x2F823, {0x5277}}, -{0x2F824, {0x3515}}, {0x2F825, {0x52C7}}, {0x2F826, {0x52C9}}, {0x2F827, {0x52E4}}, {0x2F828, {0x52FA}}, {0x2F829, {0x5305}}, {0x2F82A, {0x5306}}, {0x2F82B, {0x5317}}, {0x2F82C, {0x5349}}, -{0x2F82D, {0x5351}}, {0x2F82E, {0x535A}}, {0x2F82F, {0x5373}}, {0x2F830, {0x537D}}, {0x2F831, {0x537F}}, {0x2F832, {0x537F}}, {0x2F833, {0x537F}}, {0x2F834, {0x20A2C}}, {0x2F835, {0x7070}}, -{0x2F836, {0x53CA}}, {0x2F837, {0x53DF}}, {0x2F838, {0x20B63}}, {0x2F839, {0x53EB}}, {0x2F83A, {0x53F1}}, {0x2F83B, {0x5406}}, {0x2F83C, {0x549E}}, {0x2F83D, {0x5438}}, {0x2F83E, {0x5448}}, -{0x2F83F, {0x5468}}, {0x2F840, {0x54A2}}, {0x2F841, {0x54F6}}, {0x2F842, {0x5510}}, {0x2F843, {0x5553}}, {0x2F844, {0x5563}}, {0x2F845, {0x5584}}, {0x2F846, {0x5584}}, {0x2F847, {0x5599}}, -{0x2F848, {0x55AB}}, {0x2F849, {0x55B3}}, {0x2F84A, {0x55C2}}, {0x2F84B, {0x5716}}, {0x2F84C, {0x5606}}, {0x2F84D, {0x5717}}, {0x2F84E, {0x5651}}, {0x2F84F, {0x5674}}, {0x2F850, {0x5207}}, -{0x2F851, {0x58EE}}, {0x2F852, {0x57CE}}, {0x2F853, {0x57F4}}, {0x2F854, {0x580D}}, {0x2F855, {0x578B}}, {0x2F856, {0x5832}}, {0x2F857, {0x5831}}, {0x2F858, {0x58AC}}, {0x2F859, {0x214E4}}, -{0x2F85A, {0x58F2}}, {0x2F85B, {0x58F7}}, {0x2F85C, {0x5906}}, {0x2F85D, {0x591A}}, {0x2F85E, {0x5922}}, {0x2F85F, {0x5962}}, {0x2F860, {0x216A8}}, {0x2F861, {0x216EA}}, {0x2F862, {0x59EC}}, -{0x2F863, {0x5A1B}}, {0x2F864, {0x5A27}}, {0x2F865, {0x59D8}}, {0x2F866, {0x5A66}}, {0x2F867, {0x36EE}}, {0x2F868, {0x36FC}}, {0x2F869, {0x5B08}}, {0x2F86A, {0x5B3E}}, {0x2F86B, {0x5B3E}}, -{0x2F86C, {0x219C8}}, {0x2F86D, {0x5BC3}}, {0x2F86E, {0x5BD8}}, {0x2F86F, {0x5BE7}}, {0x2F870, {0x5BF3}}, {0x2F871, {0x21B18}}, {0x2F872, {0x5BFF}}, {0x2F873, {0x5C06}}, {0x2F874, {0x5F53}}, -{0x2F875, {0x5C22}}, {0x2F876, {0x3781}}, {0x2F877, {0x5C60}}, {0x2F878, {0x5C6E}}, {0x2F879, {0x5CC0}}, {0x2F87A, {0x5C8D}}, {0x2F87B, {0x21DE4}}, {0x2F87C, {0x5D43}}, {0x2F87D, {0x21DE6}}, -{0x2F87E, {0x5D6E}}, {0x2F87F, {0x5D6B}}, {0x2F880, {0x5D7C}}, {0x2F881, {0x5DE1}}, {0x2F882, {0x5DE2}}, {0x2F883, {0x382F}}, {0x2F884, {0x5DFD}}, {0x2F885, {0x5E28}}, {0x2F886, {0x5E3D}}, -{0x2F887, {0x5E69}}, {0x2F888, {0x3862}}, {0x2F889, {0x22183}}, {0x2F88A, {0x387C}}, {0x2F88B, {0x5EB0}}, {0x2F88C, {0x5EB3}}, {0x2F88D, {0x5EB6}}, {0x2F88E, {0x5ECA}}, {0x2F88F, {0x2A392}}, -{0x2F890, {0x5EFE}}, {0x2F891, {0x22331}}, {0x2F892, {0x22331}}, {0x2F893, {0x8201}}, {0x2F894, {0x5F22}}, {0x2F895, {0x5F22}}, {0x2F896, {0x38C7}}, {0x2F897, {0x232B8}}, {0x2F898, {0x261DA}}, -{0x2F899, {0x5F62}}, {0x2F89A, {0x5F6B}}, {0x2F89B, {0x38E3}}, {0x2F89C, {0x5F9A}}, {0x2F89D, {0x5FCD}}, {0x2F89E, {0x5FD7}}, {0x2F89F, {0x5FF9}}, {0x2F8A0, {0x6081}}, {0x2F8A1, {0x393A}}, -{0x2F8A2, {0x391C}}, {0x2F8A3, {0x6094}}, {0x2F8A4, {0x226D4}}, {0x2F8A5, {0x60C7}}, {0x2F8A6, {0x6148}}, {0x2F8A7, {0x614C}}, {0x2F8A8, {0x614E}}, {0x2F8A9, {0x614C}}, {0x2F8AA, {0x617A}}, -{0x2F8AB, {0x618E}}, {0x2F8AC, {0x61B2}}, {0x2F8AD, {0x61A4}}, {0x2F8AE, {0x61AF}}, {0x2F8AF, {0x61DE}}, {0x2F8B0, {0x61F2}}, {0x2F8B1, {0x61F6}}, {0x2F8B2, {0x6210}}, {0x2F8B3, {0x621B}}, -{0x2F8B4, {0x625D}}, {0x2F8B5, {0x62B1}}, {0x2F8B6, {0x62D4}}, {0x2F8B7, {0x6350}}, {0x2F8B8, {0x22B0C}}, {0x2F8B9, {0x633D}}, {0x2F8BA, {0x62FC}}, {0x2F8BB, {0x6368}}, {0x2F8BC, {0x6383}}, -{0x2F8BD, {0x63E4}}, {0x2F8BE, {0x22BF1}}, {0x2F8BF, {0x6422}}, {0x2F8C0, {0x63C5}}, {0x2F8C1, {0x63A9}}, {0x2F8C2, {0x3A2E}}, {0x2F8C3, {0x6469}}, {0x2F8C4, {0x647E}}, {0x2F8C5, {0x649D}}, -{0x2F8C6, {0x6477}}, {0x2F8C7, {0x3A6C}}, {0x2F8C8, {0x654F}}, {0x2F8C9, {0x656C}}, {0x2F8CA, {0x2300A}}, {0x2F8CB, {0x65E3}}, {0x2F8CC, {0x66F8}}, {0x2F8CD, {0x6649}}, {0x2F8CE, {0x3B19}}, -{0x2F8CF, {0x6691}}, {0x2F8D0, {0x3B08}}, {0x2F8D1, {0x3AE4}}, {0x2F8D2, {0x5192}}, {0x2F8D3, {0x5195}}, {0x2F8D4, {0x6700}}, {0x2F8D5, {0x669C}}, {0x2F8D6, {0x80AD}}, {0x2F8D7, {0x43D9}}, -{0x2F8D8, {0x6717}}, {0x2F8D9, {0x671B}}, {0x2F8DA, {0x6721}}, {0x2F8DB, {0x675E}}, {0x2F8DC, {0x6753}}, {0x2F8DD, {0x233C3}}, {0x2F8DE, {0x3B49}}, {0x2F8DF, {0x67FA}}, {0x2F8E0, {0x6785}}, -{0x2F8E1, {0x6852}}, {0x2F8E2, {0x6885}}, {0x2F8E3, {0x2346D}}, {0x2F8E4, {0x688E}}, {0x2F8E5, {0x681F}}, {0x2F8E6, {0x6914}}, {0x2F8E7, {0x3B9D}}, {0x2F8E8, {0x6942}}, {0x2F8E9, {0x69A3}}, -{0x2F8EA, {0x69EA}}, {0x2F8EB, {0x6AA8}}, {0x2F8EC, {0x236A3}}, {0x2F8ED, {0x6ADB}}, {0x2F8EE, {0x3C18}}, {0x2F8EF, {0x6B21}}, {0x2F8F0, {0x238A7}}, {0x2F8F1, {0x6B54}}, {0x2F8F2, {0x3C4E}}, -{0x2F8F3, {0x6B72}}, {0x2F8F4, {0x6B9F}}, {0x2F8F5, {0x6BBA}}, {0x2F8F6, {0x6BBB}}, {0x2F8F7, {0x23A8D}}, {0x2F8F8, {0x21D0B}}, {0x2F8F9, {0x23AFA}}, {0x2F8FA, {0x6C4E}}, {0x2F8FB, {0x23CBC}}, -{0x2F8FC, {0x6CBF}}, {0x2F8FD, {0x6CCD}}, {0x2F8FE, {0x6C67}}, {0x2F8FF, {0x6D16}}, {0x2F900, {0x6D3E}}, {0x2F901, {0x6D77}}, {0x2F902, {0x6D41}}, {0x2F903, {0x6D69}}, {0x2F904, {0x6D78}}, -{0x2F905, {0x6D85}}, {0x2F906, {0x23D1E}}, {0x2F907, {0x6D34}}, {0x2F908, {0x6E2F}}, {0x2F909, {0x6E6E}}, {0x2F90A, {0x3D33}}, {0x2F90B, {0x6ECB}}, {0x2F90C, {0x6EC7}}, {0x2F90D, {0x23ED1}}, -{0x2F90E, {0x6DF9}}, {0x2F90F, {0x6F6E}}, {0x2F910, {0x23F5E}}, {0x2F911, {0x23F8E}}, {0x2F912, {0x6FC6}}, {0x2F913, {0x7039}}, {0x2F914, {0x701E}}, {0x2F915, {0x701B}}, {0x2F916, {0x3D96}}, -{0x2F917, {0x704A}}, {0x2F918, {0x707D}}, {0x2F919, {0x7077}}, {0x2F91A, {0x70AD}}, {0x2F91B, {0x20525}}, {0x2F91C, {0x7145}}, {0x2F91D, {0x24263}}, {0x2F91E, {0x719C}}, {0x2F91F, {0x243AB}}, -{0x2F920, {0x7228}}, {0x2F921, {0x7235}}, {0x2F922, {0x7250}}, {0x2F923, {0x24608}}, {0x2F924, {0x7280}}, {0x2F925, {0x7295}}, {0x2F926, {0x24735}}, {0x2F927, {0x24814}}, {0x2F928, {0x737A}}, -{0x2F929, {0x738B}}, {0x2F92A, {0x3EAC}}, {0x2F92B, {0x73A5}}, {0x2F92C, {0x3EB8}}, {0x2F92D, {0x3EB8}}, {0x2F92E, {0x7447}}, {0x2F92F, {0x745C}}, {0x2F930, {0x7471}}, {0x2F931, {0x7485}}, -{0x2F932, {0x74CA}}, {0x2F933, {0x3F1B}}, {0x2F934, {0x7524}}, {0x2F935, {0x24C36}}, {0x2F936, {0x753E}}, {0x2F937, {0x24C92}}, {0x2F938, {0x7570}}, {0x2F939, {0x2219F}}, {0x2F93A, {0x7610}}, -{0x2F93B, {0x24FA1}}, {0x2F93C, {0x24FB8}}, {0x2F93D, {0x25044}}, {0x2F93E, {0x3FFC}}, {0x2F93F, {0x4008}}, {0x2F940, {0x76F4}}, {0x2F941, {0x250F3}}, {0x2F942, {0x250F2}}, {0x2F943, {0x25119}}, -{0x2F944, {0x25133}}, {0x2F945, {0x771E}}, {0x2F946, {0x771F}}, {0x2F947, {0x771F}}, {0x2F948, {0x774A}}, {0x2F949, {0x4039}}, {0x2F94A, {0x778B}}, {0x2F94B, {0x4046}}, {0x2F94C, {0x4096}}, -{0x2F94D, {0x2541D}}, {0x2F94E, {0x784E}}, {0x2F94F, {0x788C}}, {0x2F950, {0x78CC}}, {0x2F951, {0x40E3}}, {0x2F952, {0x25626}}, {0x2F953, {0x7956}}, {0x2F954, {0x2569A}}, {0x2F955, {0x256C5}}, -{0x2F956, {0x798F}}, {0x2F957, {0x79EB}}, {0x2F958, {0x412F}}, {0x2F959, {0x7A40}}, {0x2F95A, {0x7A4A}}, {0x2F95B, {0x7A4F}}, {0x2F95C, {0x2597C}}, {0x2F95D, {0x25AA7}}, {0x2F95E, {0x25AA7}}, -{0x2F95F, {0x7AEE}}, {0x2F960, {0x4202}}, {0x2F961, {0x25BAB}}, {0x2F962, {0x7BC6}}, {0x2F963, {0x7BC9}}, {0x2F964, {0x4227}}, {0x2F965, {0x25C80}}, {0x2F966, {0x7CD2}}, {0x2F967, {0x42A0}}, -{0x2F968, {0x7CE8}}, {0x2F969, {0x7CE3}}, {0x2F96A, {0x7D00}}, {0x2F96B, {0x25F86}}, {0x2F96C, {0x7D63}}, {0x2F96D, {0x4301}}, {0x2F96E, {0x7DC7}}, {0x2F96F, {0x7E02}}, {0x2F970, {0x7E45}}, -{0x2F971, {0x4334}}, {0x2F972, {0x26228}}, {0x2F973, {0x26247}}, {0x2F974, {0x4359}}, {0x2F975, {0x262D9}}, {0x2F976, {0x7F7A}}, {0x2F977, {0x2633E}}, {0x2F978, {0x7F95}}, {0x2F979, {0x7FFA}}, -{0x2F97A, {0x8005}}, {0x2F97B, {0x264DA}}, {0x2F97C, {0x26523}}, {0x2F97D, {0x8060}}, {0x2F97E, {0x265A8}}, {0x2F97F, {0x8070}}, {0x2F980, {0x2335F}}, {0x2F981, {0x43D5}}, {0x2F982, {0x80B2}}, -{0x2F983, {0x8103}}, {0x2F984, {0x440B}}, {0x2F985, {0x813E}}, {0x2F986, {0x5AB5}}, {0x2F987, {0x267A7}}, {0x2F988, {0x267B5}}, {0x2F989, {0x23393}}, {0x2F98A, {0x2339C}}, {0x2F98B, {0x8201}}, -{0x2F98C, {0x8204}}, {0x2F98D, {0x8F9E}}, {0x2F98E, {0x446B}}, {0x2F98F, {0x8291}}, {0x2F990, {0x828B}}, {0x2F991, {0x829D}}, {0x2F992, {0x52B3}}, {0x2F993, {0x82B1}}, {0x2F994, {0x82B3}}, -{0x2F995, {0x82BD}}, {0x2F996, {0x82E6}}, {0x2F997, {0x26B3C}}, {0x2F998, {0x82E5}}, {0x2F999, {0x831D}}, {0x2F99A, {0x8363}}, {0x2F99B, {0x83AD}}, {0x2F99C, {0x8323}}, {0x2F99D, {0x83BD}}, -{0x2F99E, {0x83E7}}, {0x2F99F, {0x8457}}, {0x2F9A0, {0x8353}}, {0x2F9A1, {0x83CA}}, {0x2F9A2, {0x83CC}}, {0x2F9A3, {0x83DC}}, {0x2F9A4, {0x26C36}}, {0x2F9A5, {0x26D6B}}, {0x2F9A6, {0x26CD5}}, -{0x2F9A7, {0x452B}}, {0x2F9A8, {0x84F1}}, {0x2F9A9, {0x84F3}}, {0x2F9AA, {0x8516}}, {0x2F9AB, {0x273CA}}, {0x2F9AC, {0x8564}}, {0x2F9AD, {0x26F2C}}, {0x2F9AE, {0x455D}}, {0x2F9AF, {0x4561}}, -{0x2F9B0, {0x26FB1}}, {0x2F9B1, {0x270D2}}, {0x2F9B2, {0x456B}}, {0x2F9B3, {0x8650}}, {0x2F9B4, {0x865C}}, {0x2F9B5, {0x8667}}, {0x2F9B6, {0x8669}}, {0x2F9B7, {0x86A9}}, {0x2F9B8, {0x8688}}, -{0x2F9B9, {0x870E}}, {0x2F9BA, {0x86E2}}, {0x2F9BB, {0x8779}}, {0x2F9BC, {0x8728}}, {0x2F9BD, {0x876B}}, {0x2F9BE, {0x8786}}, {0x2F9BF, {0x45D7}}, {0x2F9C0, {0x87E1}}, {0x2F9C1, {0x8801}}, -{0x2F9C2, {0x45F9}}, {0x2F9C3, {0x8860}}, {0x2F9C4, {0x8863}}, {0x2F9C5, {0x27667}}, {0x2F9C6, {0x88D7}}, {0x2F9C7, {0x88DE}}, {0x2F9C8, {0x4635}}, {0x2F9C9, {0x88FA}}, {0x2F9CA, {0x34BB}}, -{0x2F9CB, {0x278AE}}, {0x2F9CC, {0x27966}}, {0x2F9CD, {0x46BE}}, {0x2F9CE, {0x46C7}}, {0x2F9CF, {0x8AA0}}, {0x2F9D0, {0x8AED}}, {0x2F9D1, {0x8B8A}}, {0x2F9D2, {0x8C55}}, {0x2F9D3, {0x27CA8}}, -{0x2F9D4, {0x8CAB}}, {0x2F9D5, {0x8CC1}}, {0x2F9D6, {0x8D1B}}, {0x2F9D7, {0x8D77}}, {0x2F9D8, {0x27F2F}}, {0x2F9D9, {0x20804}}, {0x2F9DA, {0x8DCB}}, {0x2F9DB, {0x8DBC}}, {0x2F9DC, {0x8DF0}}, -{0x2F9DD, {0x208DE}}, {0x2F9DE, {0x8ED4}}, {0x2F9DF, {0x8F38}}, {0x2F9E0, {0x285D2}}, {0x2F9E1, {0x285ED}}, {0x2F9E2, {0x9094}}, {0x2F9E3, {0x90F1}}, {0x2F9E4, {0x9111}}, {0x2F9E5, {0x2872E}}, -{0x2F9E6, {0x911B}}, {0x2F9E7, {0x9238}}, {0x2F9E8, {0x92D7}}, {0x2F9E9, {0x92D8}}, {0x2F9EA, {0x927C}}, {0x2F9EB, {0x93F9}}, {0x2F9EC, {0x9415}}, {0x2F9ED, {0x28BFA}}, {0x2F9EE, {0x958B}}, -{0x2F9EF, {0x4995}}, {0x2F9F0, {0x95B7}}, {0x2F9F1, {0x28D77}}, {0x2F9F2, {0x49E6}}, {0x2F9F3, {0x96C3}}, {0x2F9F4, {0x5DB2}}, {0x2F9F5, {0x9723}}, {0x2F9F6, {0x29145}}, {0x2F9F7, {0x2921A}}, -{0x2F9F8, {0x4A6E}}, {0x2F9F9, {0x4A76}}, {0x2F9FA, {0x97E0}}, {0x2F9FB, {0x2940A}}, {0x2F9FC, {0x4AB2}}, {0x2F9FD, {0x29496}}, {0x2F9FE, {0x980B}}, {0x2F9FF, {0x980B}}, {0x2FA00, {0x9829}}, -{0x2FA01, {0x295B6}}, {0x2FA02, {0x98E2}}, {0x2FA03, {0x4B33}}, {0x2FA04, {0x9929}}, {0x2FA05, {0x99A7}}, {0x2FA06, {0x99C2}}, {0x2FA07, {0x99FE}}, {0x2FA08, {0x4BCE}}, {0x2FA09, {0x29B30}}, -{0x2FA0A, {0x9B12}}, {0x2FA0B, {0x9C40}}, {0x2FA0C, {0x9CFD}}, {0x2FA0D, {0x4CCE}}, {0x2FA0E, {0x4CED}}, {0x2FA0F, {0x9D67}}, {0x2FA10, {0x2A0CE}}, {0x2FA11, {0x4CF8}}, {0x2FA12, {0x2A105}}, -{0x2FA13, {0x2A20E}}, {0x2FA14, {0x2A291}}, {0x2FA15, {0x9EBB}}, {0x2FA16, {0x4D56}}, {0x2FA17, {0x9EF9}}, {0x2FA18, {0x9EFE}}, {0x2FA19, {0x9F05}}, {0x2FA1A, {0x9F0F}}, {0x2FA1B, {0x9F16}}, -{0x2FA1D, {0x2A600}}, +static const std::multimap nfd_map = { +{0xC0, 0x41}, {0xC0, 0x300}, {0xC1, 0x41}, {0xC1, 0x301}, {0xC2, 0x41}, {0xC2, 0x302}, {0xC3, 0x41}, {0xC3, 0x303}, {0xC4, 0x41}, {0xC4, 0x308}, {0xC5, 0x41}, {0xC5, 0x30A}, {0xC7, 0x43}, +{0xC7, 0x327}, {0xC8, 0x45}, {0xC8, 0x300}, {0xC9, 0x45}, {0xC9, 0x301}, {0xCA, 0x45}, {0xCA, 0x302}, {0xCB, 0x45}, {0xCB, 0x308}, {0xCC, 0x49}, {0xCC, 0x300}, {0xCD, 0x49}, {0xCD, 0x301}, +{0xCE, 0x49}, {0xCE, 0x302}, {0xCF, 0x49}, {0xCF, 0x308}, {0xD1, 0x4E}, {0xD1, 0x303}, {0xD2, 0x4F}, {0xD2, 0x300}, {0xD3, 0x4F}, {0xD3, 0x301}, {0xD4, 0x4F}, {0xD4, 0x302}, {0xD5, 0x4F}, +{0xD5, 0x303}, {0xD6, 0x4F}, {0xD6, 0x308}, {0xD9, 0x55}, {0xD9, 0x300}, {0xDA, 0x55}, {0xDA, 0x301}, {0xDB, 0x55}, {0xDB, 0x302}, {0xDC, 0x55}, {0xDC, 0x308}, {0xDD, 0x59}, {0xDD, 0x301}, +{0xE0, 0x61}, {0xE0, 0x300}, {0xE1, 0x61}, {0xE1, 0x301}, {0xE2, 0x61}, {0xE2, 0x302}, {0xE3, 0x61}, {0xE3, 0x303}, {0xE4, 0x61}, {0xE4, 0x308}, {0xE5, 0x61}, {0xE5, 0x30A}, {0xE7, 0x63}, +{0xE7, 0x327}, {0xE8, 0x65}, {0xE8, 0x300}, {0xE9, 0x65}, {0xE9, 0x301}, {0xEA, 0x65}, {0xEA, 0x302}, {0xEB, 0x65}, {0xEB, 0x308}, {0xEC, 0x69}, {0xEC, 0x300}, {0xED, 0x69}, {0xED, 0x301}, +{0xEE, 0x69}, {0xEE, 0x302}, {0xEF, 0x69}, {0xEF, 0x308}, {0xF1, 0x6E}, {0xF1, 0x303}, {0xF2, 0x6F}, {0xF2, 0x300}, {0xF3, 0x6F}, {0xF3, 0x301}, {0xF4, 0x6F}, {0xF4, 0x302}, {0xF5, 0x6F}, +{0xF5, 0x303}, {0xF6, 0x6F}, {0xF6, 0x308}, {0xF9, 0x75}, {0xF9, 0x300}, {0xFA, 0x75}, {0xFA, 0x301}, {0xFB, 0x75}, {0xFB, 0x302}, {0xFC, 0x75}, {0xFC, 0x308}, {0xFD, 0x79}, {0xFD, 0x301}, +{0xFF, 0x79}, {0xFF, 0x308}, {0x100, 0x41}, {0x100, 0x304}, {0x101, 0x61}, {0x101, 0x304}, {0x102, 0x41}, {0x102, 0x306}, {0x103, 0x61}, {0x103, 0x306}, {0x104, 0x41}, {0x104, 0x328}, {0x105, 0x61}, +{0x105, 0x328}, {0x106, 0x43}, {0x106, 0x301}, {0x107, 0x63}, {0x107, 0x301}, {0x108, 0x43}, {0x108, 0x302}, {0x109, 0x63}, {0x109, 0x302}, {0x10A, 0x43}, {0x10A, 0x307}, {0x10B, 0x63}, +{0x10B, 0x307}, {0x10C, 0x43}, {0x10C, 0x30C}, {0x10D, 0x63}, {0x10D, 0x30C}, {0x10E, 0x44}, {0x10E, 0x30C}, {0x10F, 0x64}, {0x10F, 0x30C}, {0x112, 0x45}, {0x112, 0x304}, {0x113, 0x65}, +{0x113, 0x304}, {0x114, 0x45}, {0x114, 0x306}, {0x115, 0x65}, {0x115, 0x306}, {0x116, 0x45}, {0x116, 0x307}, {0x117, 0x65}, {0x117, 0x307}, {0x118, 0x45}, {0x118, 0x328}, {0x119, 0x65}, +{0x119, 0x328}, {0x11A, 0x45}, {0x11A, 0x30C}, {0x11B, 0x65}, {0x11B, 0x30C}, {0x11C, 0x47}, {0x11C, 0x302}, {0x11D, 0x67}, {0x11D, 0x302}, {0x11E, 0x47}, {0x11E, 0x306}, {0x11F, 0x67}, +{0x11F, 0x306}, {0x120, 0x47}, {0x120, 0x307}, {0x121, 0x67}, {0x121, 0x307}, {0x122, 0x47}, {0x122, 0x327}, {0x123, 0x67}, {0x123, 0x327}, {0x124, 0x48}, {0x124, 0x302}, {0x125, 0x68}, +{0x125, 0x302}, {0x128, 0x49}, {0x128, 0x303}, {0x129, 0x69}, {0x129, 0x303}, {0x12A, 0x49}, {0x12A, 0x304}, {0x12B, 0x69}, {0x12B, 0x304}, {0x12C, 0x49}, {0x12C, 0x306}, {0x12D, 0x69}, +{0x12D, 0x306}, {0x12E, 0x49}, {0x12E, 0x328}, {0x12F, 0x69}, {0x12F, 0x328}, {0x130, 0x49}, {0x130, 0x307}, {0x134, 0x4A}, {0x134, 0x302}, {0x135, 0x6A}, {0x135, 0x302}, {0x136, 0x4B}, +{0x136, 0x327}, {0x137, 0x6B}, {0x137, 0x327}, {0x139, 0x4C}, {0x139, 0x301}, {0x13A, 0x6C}, {0x13A, 0x301}, {0x13B, 0x4C}, {0x13B, 0x327}, {0x13C, 0x6C}, {0x13C, 0x327}, {0x13D, 0x4C}, +{0x13D, 0x30C}, {0x13E, 0x6C}, {0x13E, 0x30C}, {0x143, 0x4E}, {0x143, 0x301}, {0x144, 0x6E}, {0x144, 0x301}, {0x145, 0x4E}, {0x145, 0x327}, {0x146, 0x6E}, {0x146, 0x327}, {0x147, 0x4E}, +{0x147, 0x30C}, {0x148, 0x6E}, {0x148, 0x30C}, {0x14C, 0x4F}, {0x14C, 0x304}, {0x14D, 0x6F}, {0x14D, 0x304}, {0x14E, 0x4F}, {0x14E, 0x306}, {0x14F, 0x6F}, {0x14F, 0x306}, {0x150, 0x4F}, +{0x150, 0x30B}, {0x151, 0x6F}, {0x151, 0x30B}, {0x154, 0x52}, {0x154, 0x301}, {0x155, 0x72}, {0x155, 0x301}, {0x156, 0x52}, {0x156, 0x327}, {0x157, 0x72}, {0x157, 0x327}, {0x158, 0x52}, +{0x158, 0x30C}, {0x159, 0x72}, {0x159, 0x30C}, {0x15A, 0x53}, {0x15A, 0x301}, {0x15B, 0x73}, {0x15B, 0x301}, {0x15C, 0x53}, {0x15C, 0x302}, {0x15D, 0x73}, {0x15D, 0x302}, {0x15E, 0x53}, +{0x15E, 0x327}, {0x15F, 0x73}, {0x15F, 0x327}, {0x160, 0x53}, {0x160, 0x30C}, {0x161, 0x73}, {0x161, 0x30C}, {0x162, 0x54}, {0x162, 0x327}, {0x163, 0x74}, {0x163, 0x327}, {0x164, 0x54}, +{0x164, 0x30C}, {0x165, 0x74}, {0x165, 0x30C}, {0x168, 0x55}, {0x168, 0x303}, {0x169, 0x75}, {0x169, 0x303}, {0x16A, 0x55}, {0x16A, 0x304}, {0x16B, 0x75}, {0x16B, 0x304}, {0x16C, 0x55}, +{0x16C, 0x306}, {0x16D, 0x75}, {0x16D, 0x306}, {0x16E, 0x55}, {0x16E, 0x30A}, {0x16F, 0x75}, {0x16F, 0x30A}, {0x170, 0x55}, {0x170, 0x30B}, {0x171, 0x75}, {0x171, 0x30B}, {0x172, 0x55}, +{0x172, 0x328}, {0x173, 0x75}, {0x173, 0x328}, {0x174, 0x57}, {0x174, 0x302}, {0x175, 0x77}, {0x175, 0x302}, {0x176, 0x59}, {0x176, 0x302}, {0x177, 0x79}, {0x177, 0x302}, {0x178, 0x59}, +{0x178, 0x308}, {0x179, 0x5A}, {0x179, 0x301}, {0x17A, 0x7A}, {0x17A, 0x301}, {0x17B, 0x5A}, {0x17B, 0x307}, {0x17C, 0x7A}, {0x17C, 0x307}, {0x17D, 0x5A}, {0x17D, 0x30C}, {0x17E, 0x7A}, +{0x17E, 0x30C}, {0x1A0, 0x4F}, {0x1A0, 0x31B}, {0x1A1, 0x6F}, {0x1A1, 0x31B}, {0x1AF, 0x55}, {0x1AF, 0x31B}, {0x1B0, 0x75}, {0x1B0, 0x31B}, {0x1CD, 0x41}, {0x1CD, 0x30C}, {0x1CE, 0x61}, +{0x1CE, 0x30C}, {0x1CF, 0x49}, {0x1CF, 0x30C}, {0x1D0, 0x69}, {0x1D0, 0x30C}, {0x1D1, 0x4F}, {0x1D1, 0x30C}, {0x1D2, 0x6F}, {0x1D2, 0x30C}, {0x1D3, 0x55}, {0x1D3, 0x30C}, {0x1D4, 0x75}, +{0x1D4, 0x30C}, {0x1D5, 0x55}, {0x1D5, 0x308}, {0x1D5, 0x304}, {0x1D6, 0x75}, {0x1D6, 0x308}, {0x1D6, 0x304}, {0x1D7, 0x55}, {0x1D7, 0x308}, {0x1D7, 0x301}, {0x1D8, 0x75}, {0x1D8, 0x308}, +{0x1D8, 0x301}, {0x1D9, 0x55}, {0x1D9, 0x308}, {0x1D9, 0x30C}, {0x1DA, 0x75}, {0x1DA, 0x308}, {0x1DA, 0x30C}, {0x1DB, 0x55}, {0x1DB, 0x308}, {0x1DB, 0x300}, {0x1DC, 0x75}, {0x1DC, 0x308}, +{0x1DC, 0x300}, {0x1DE, 0x41}, {0x1DE, 0x308}, {0x1DE, 0x304}, {0x1DF, 0x61}, {0x1DF, 0x308}, {0x1DF, 0x304}, {0x1E0, 0x41}, {0x1E0, 0x307}, {0x1E0, 0x304}, {0x1E1, 0x61}, {0x1E1, 0x307}, +{0x1E1, 0x304}, {0x1E2, 0xC6}, {0x1E2, 0x304}, {0x1E3, 0xE6}, {0x1E3, 0x304}, {0x1E6, 0x47}, {0x1E6, 0x30C}, {0x1E7, 0x67}, {0x1E7, 0x30C}, {0x1E8, 0x4B}, {0x1E8, 0x30C}, {0x1E9, 0x6B}, +{0x1E9, 0x30C}, {0x1EA, 0x4F}, {0x1EA, 0x328}, {0x1EB, 0x6F}, {0x1EB, 0x328}, {0x1EC, 0x4F}, {0x1EC, 0x328}, {0x1EC, 0x304}, {0x1ED, 0x6F}, {0x1ED, 0x328}, {0x1ED, 0x304}, {0x1EE, 0x1B7}, +{0x1EE, 0x30C}, {0x1EF, 0x292}, {0x1EF, 0x30C}, {0x1F0, 0x6A}, {0x1F0, 0x30C}, {0x1F4, 0x47}, {0x1F4, 0x301}, {0x1F5, 0x67}, {0x1F5, 0x301}, {0x1F8, 0x4E}, {0x1F8, 0x300}, {0x1F9, 0x6E}, +{0x1F9, 0x300}, {0x1FA, 0x41}, {0x1FA, 0x30A}, {0x1FA, 0x301}, {0x1FB, 0x61}, {0x1FB, 0x30A}, {0x1FB, 0x301}, {0x1FC, 0xC6}, {0x1FC, 0x301}, {0x1FD, 0xE6}, {0x1FD, 0x301}, {0x1FE, 0xD8}, +{0x1FE, 0x301}, {0x1FF, 0xF8}, {0x1FF, 0x301}, {0x200, 0x41}, {0x200, 0x30F}, {0x201, 0x61}, {0x201, 0x30F}, {0x202, 0x41}, {0x202, 0x311}, {0x203, 0x61}, {0x203, 0x311}, {0x204, 0x45}, +{0x204, 0x30F}, {0x205, 0x65}, {0x205, 0x30F}, {0x206, 0x45}, {0x206, 0x311}, {0x207, 0x65}, {0x207, 0x311}, {0x208, 0x49}, {0x208, 0x30F}, {0x209, 0x69}, {0x209, 0x30F}, {0x20A, 0x49}, +{0x20A, 0x311}, {0x20B, 0x69}, {0x20B, 0x311}, {0x20C, 0x4F}, {0x20C, 0x30F}, {0x20D, 0x6F}, {0x20D, 0x30F}, {0x20E, 0x4F}, {0x20E, 0x311}, {0x20F, 0x6F}, {0x20F, 0x311}, {0x210, 0x52}, +{0x210, 0x30F}, {0x211, 0x72}, {0x211, 0x30F}, {0x212, 0x52}, {0x212, 0x311}, {0x213, 0x72}, {0x213, 0x311}, {0x214, 0x55}, {0x214, 0x30F}, {0x215, 0x75}, {0x215, 0x30F}, {0x216, 0x55}, +{0x216, 0x311}, {0x217, 0x75}, {0x217, 0x311}, {0x218, 0x53}, {0x218, 0x326}, {0x219, 0x73}, {0x219, 0x326}, {0x21A, 0x54}, {0x21A, 0x326}, {0x21B, 0x74}, {0x21B, 0x326}, {0x21E, 0x48}, +{0x21E, 0x30C}, {0x21F, 0x68}, {0x21F, 0x30C}, {0x226, 0x41}, {0x226, 0x307}, {0x227, 0x61}, {0x227, 0x307}, {0x228, 0x45}, {0x228, 0x327}, {0x229, 0x65}, {0x229, 0x327}, {0x22A, 0x4F}, +{0x22A, 0x308}, {0x22A, 0x304}, {0x22B, 0x6F}, {0x22B, 0x308}, {0x22B, 0x304}, {0x22C, 0x4F}, {0x22C, 0x303}, {0x22C, 0x304}, {0x22D, 0x6F}, {0x22D, 0x303}, {0x22D, 0x304}, {0x22E, 0x4F}, +{0x22E, 0x307}, {0x22F, 0x6F}, {0x22F, 0x307}, {0x230, 0x4F}, {0x230, 0x307}, {0x230, 0x304}, {0x231, 0x6F}, {0x231, 0x307}, {0x231, 0x304}, {0x232, 0x59}, {0x232, 0x304}, {0x233, 0x79}, +{0x233, 0x304}, {0x340, 0x300}, {0x341, 0x301}, {0x343, 0x313}, {0x344, 0x308}, {0x344, 0x301}, {0x374, 0x2B9}, {0x37E, 0x3B}, {0x385, 0xA8}, {0x385, 0x301}, {0x386, 0x391}, {0x386, 0x301}, +{0x387, 0xB7}, {0x388, 0x395}, {0x388, 0x301}, {0x389, 0x397}, {0x389, 0x301}, {0x38A, 0x399}, {0x38A, 0x301}, {0x38C, 0x39F}, {0x38C, 0x301}, {0x38E, 0x3A5}, {0x38E, 0x301}, {0x38F, 0x3A9}, +{0x38F, 0x301}, {0x390, 0x3B9}, {0x390, 0x308}, {0x390, 0x301}, {0x3AA, 0x399}, {0x3AA, 0x308}, {0x3AB, 0x3A5}, {0x3AB, 0x308}, {0x3AC, 0x3B1}, {0x3AC, 0x301}, {0x3AD, 0x3B5}, {0x3AD, 0x301}, +{0x3AE, 0x3B7}, {0x3AE, 0x301}, {0x3AF, 0x3B9}, {0x3AF, 0x301}, {0x3B0, 0x3C5}, {0x3B0, 0x308}, {0x3B0, 0x301}, {0x3CA, 0x3B9}, {0x3CA, 0x308}, {0x3CB, 0x3C5}, {0x3CB, 0x308}, {0x3CC, 0x3BF}, +{0x3CC, 0x301}, {0x3CD, 0x3C5}, {0x3CD, 0x301}, {0x3CE, 0x3C9}, {0x3CE, 0x301}, {0x3D3, 0x3D2}, {0x3D3, 0x301}, {0x3D4, 0x3D2}, {0x3D4, 0x308}, {0x400, 0x415}, {0x400, 0x300}, {0x401, 0x415}, +{0x401, 0x308}, {0x403, 0x413}, {0x403, 0x301}, {0x407, 0x406}, {0x407, 0x308}, {0x40C, 0x41A}, {0x40C, 0x301}, {0x40D, 0x418}, {0x40D, 0x300}, {0x40E, 0x423}, {0x40E, 0x306}, {0x419, 0x418}, +{0x419, 0x306}, {0x439, 0x438}, {0x439, 0x306}, {0x450, 0x435}, {0x450, 0x300}, {0x451, 0x435}, {0x451, 0x308}, {0x453, 0x433}, {0x453, 0x301}, {0x457, 0x456}, {0x457, 0x308}, {0x45C, 0x43A}, +{0x45C, 0x301}, {0x45D, 0x438}, {0x45D, 0x300}, {0x45E, 0x443}, {0x45E, 0x306}, {0x476, 0x474}, {0x476, 0x30F}, {0x477, 0x475}, {0x477, 0x30F}, {0x4C1, 0x416}, {0x4C1, 0x306}, {0x4C2, 0x436}, +{0x4C2, 0x306}, {0x4D0, 0x410}, {0x4D0, 0x306}, {0x4D1, 0x430}, {0x4D1, 0x306}, {0x4D2, 0x410}, {0x4D2, 0x308}, {0x4D3, 0x430}, {0x4D3, 0x308}, {0x4D6, 0x415}, {0x4D6, 0x306}, {0x4D7, 0x435}, +{0x4D7, 0x306}, {0x4DA, 0x4D8}, {0x4DA, 0x308}, {0x4DB, 0x4D9}, {0x4DB, 0x308}, {0x4DC, 0x416}, {0x4DC, 0x308}, {0x4DD, 0x436}, {0x4DD, 0x308}, {0x4DE, 0x417}, {0x4DE, 0x308}, {0x4DF, 0x437}, +{0x4DF, 0x308}, {0x4E2, 0x418}, {0x4E2, 0x304}, {0x4E3, 0x438}, {0x4E3, 0x304}, {0x4E4, 0x418}, {0x4E4, 0x308}, {0x4E5, 0x438}, {0x4E5, 0x308}, {0x4E6, 0x41E}, {0x4E6, 0x308}, {0x4E7, 0x43E}, +{0x4E7, 0x308}, {0x4EA, 0x4E8}, {0x4EA, 0x308}, {0x4EB, 0x4E9}, {0x4EB, 0x308}, {0x4EC, 0x42D}, {0x4EC, 0x308}, {0x4ED, 0x44D}, {0x4ED, 0x308}, {0x4EE, 0x423}, {0x4EE, 0x304}, {0x4EF, 0x443}, +{0x4EF, 0x304}, {0x4F0, 0x423}, {0x4F0, 0x308}, {0x4F1, 0x443}, {0x4F1, 0x308}, {0x4F2, 0x423}, {0x4F2, 0x30B}, {0x4F3, 0x443}, {0x4F3, 0x30B}, {0x4F4, 0x427}, {0x4F4, 0x308}, {0x4F5, 0x447}, +{0x4F5, 0x308}, {0x4F8, 0x42B}, {0x4F8, 0x308}, {0x4F9, 0x44B}, {0x4F9, 0x308}, {0x622, 0x627}, {0x622, 0x653}, {0x623, 0x627}, {0x623, 0x654}, {0x624, 0x648}, {0x624, 0x654}, {0x625, 0x627}, +{0x625, 0x655}, {0x626, 0x64A}, {0x626, 0x654}, {0x6C0, 0x6D5}, {0x6C0, 0x654}, {0x6C2, 0x6C1}, {0x6C2, 0x654}, {0x6D3, 0x6D2}, {0x6D3, 0x654}, {0x929, 0x928}, {0x929, 0x93C}, {0x931, 0x930}, +{0x931, 0x93C}, {0x934, 0x933}, {0x934, 0x93C}, {0x958, 0x915}, {0x958, 0x93C}, {0x959, 0x916}, {0x959, 0x93C}, {0x95A, 0x917}, {0x95A, 0x93C}, {0x95B, 0x91C}, {0x95B, 0x93C}, {0x95C, 0x921}, +{0x95C, 0x93C}, {0x95D, 0x922}, {0x95D, 0x93C}, {0x95E, 0x92B}, {0x95E, 0x93C}, {0x95F, 0x92F}, {0x95F, 0x93C}, {0x9CB, 0x9C7}, {0x9CB, 0x9BE}, {0x9CC, 0x9C7}, {0x9CC, 0x9D7}, {0x9DC, 0x9A1}, +{0x9DC, 0x9BC}, {0x9DD, 0x9A2}, {0x9DD, 0x9BC}, {0x9DF, 0x9AF}, {0x9DF, 0x9BC}, {0xA33, 0xA32}, {0xA33, 0xA3C}, {0xA36, 0xA38}, {0xA36, 0xA3C}, {0xA59, 0xA16}, {0xA59, 0xA3C}, {0xA5A, 0xA17}, +{0xA5A, 0xA3C}, {0xA5B, 0xA1C}, {0xA5B, 0xA3C}, {0xA5E, 0xA2B}, {0xA5E, 0xA3C}, {0xB48, 0xB47}, {0xB48, 0xB56}, {0xB4B, 0xB47}, {0xB4B, 0xB3E}, {0xB4C, 0xB47}, {0xB4C, 0xB57}, {0xB5C, 0xB21}, +{0xB5C, 0xB3C}, {0xB5D, 0xB22}, {0xB5D, 0xB3C}, {0xB94, 0xB92}, {0xB94, 0xBD7}, {0xBCA, 0xBC6}, {0xBCA, 0xBBE}, {0xBCB, 0xBC7}, {0xBCB, 0xBBE}, {0xBCC, 0xBC6}, {0xBCC, 0xBD7}, {0xC48, 0xC46}, +{0xC48, 0xC56}, {0xCC0, 0xCBF}, {0xCC0, 0xCD5}, {0xCC7, 0xCC6}, {0xCC7, 0xCD5}, {0xCC8, 0xCC6}, {0xCC8, 0xCD6}, {0xCCA, 0xCC6}, {0xCCA, 0xCC2}, {0xCCB, 0xCC6}, {0xCCB, 0xCC2}, {0xCCB, 0xCD5}, +{0xD4A, 0xD46}, {0xD4A, 0xD3E}, {0xD4B, 0xD47}, {0xD4B, 0xD3E}, {0xD4C, 0xD46}, {0xD4C, 0xD57}, {0xDDA, 0xDD9}, {0xDDA, 0xDCA}, {0xDDC, 0xDD9}, {0xDDC, 0xDCF}, {0xDDD, 0xDD9}, {0xDDD, 0xDCF}, +{0xDDD, 0xDCA}, {0xDDE, 0xDD9}, {0xDDE, 0xDDF}, {0xF43, 0xF42}, {0xF43, 0xFB7}, {0xF4D, 0xF4C}, {0xF4D, 0xFB7}, {0xF52, 0xF51}, {0xF52, 0xFB7}, {0xF57, 0xF56}, {0xF57, 0xFB7}, {0xF5C, 0xF5B}, +{0xF5C, 0xFB7}, {0xF69, 0xF40}, {0xF69, 0xFB5}, {0xF73, 0xF71}, {0xF73, 0xF72}, {0xF75, 0xF71}, {0xF75, 0xF74}, {0xF76, 0xFB2}, {0xF76, 0xF80}, {0xF78, 0xFB3}, {0xF78, 0xF80}, {0xF81, 0xF71}, +{0xF81, 0xF80}, {0xF93, 0xF92}, {0xF93, 0xFB7}, {0xF9D, 0xF9C}, {0xF9D, 0xFB7}, {0xFA2, 0xFA1}, {0xFA2, 0xFB7}, {0xFA7, 0xFA6}, {0xFA7, 0xFB7}, {0xFAC, 0xFAB}, {0xFAC, 0xFB7}, {0xFB9, 0xF90}, +{0xFB9, 0xFB5}, {0x1026, 0x1025}, {0x1026, 0x102E}, {0x1B06, 0x1B05}, {0x1B06, 0x1B35}, {0x1B08, 0x1B07}, {0x1B08, 0x1B35}, {0x1B0A, 0x1B09}, {0x1B0A, 0x1B35}, {0x1B0C, 0x1B0B}, {0x1B0C, 0x1B35}, +{0x1B0E, 0x1B0D}, {0x1B0E, 0x1B35}, {0x1B12, 0x1B11}, {0x1B12, 0x1B35}, {0x1B3B, 0x1B3A}, {0x1B3B, 0x1B35}, {0x1B3D, 0x1B3C}, {0x1B3D, 0x1B35}, {0x1B40, 0x1B3E}, {0x1B40, 0x1B35}, {0x1B41, 0x1B3F}, +{0x1B41, 0x1B35}, {0x1B43, 0x1B42}, {0x1B43, 0x1B35}, {0x1E00, 0x41}, {0x1E00, 0x325}, {0x1E01, 0x61}, {0x1E01, 0x325}, {0x1E02, 0x42}, {0x1E02, 0x307}, {0x1E03, 0x62}, {0x1E03, 0x307}, +{0x1E04, 0x42}, {0x1E04, 0x323}, {0x1E05, 0x62}, {0x1E05, 0x323}, {0x1E06, 0x42}, {0x1E06, 0x331}, {0x1E07, 0x62}, {0x1E07, 0x331}, {0x1E08, 0x43}, {0x1E08, 0x327}, {0x1E08, 0x301}, {0x1E09, 0x63}, +{0x1E09, 0x327}, {0x1E09, 0x301}, {0x1E0A, 0x44}, {0x1E0A, 0x307}, {0x1E0B, 0x64}, {0x1E0B, 0x307}, {0x1E0C, 0x44}, {0x1E0C, 0x323}, {0x1E0D, 0x64}, {0x1E0D, 0x323}, {0x1E0E, 0x44}, {0x1E0E, 0x331}, +{0x1E0F, 0x64}, {0x1E0F, 0x331}, {0x1E10, 0x44}, {0x1E10, 0x327}, {0x1E11, 0x64}, {0x1E11, 0x327}, {0x1E12, 0x44}, {0x1E12, 0x32D}, {0x1E13, 0x64}, {0x1E13, 0x32D}, {0x1E14, 0x45}, {0x1E14, 0x304}, +{0x1E14, 0x300}, {0x1E15, 0x65}, {0x1E15, 0x304}, {0x1E15, 0x300}, {0x1E16, 0x45}, {0x1E16, 0x304}, {0x1E16, 0x301}, {0x1E17, 0x65}, {0x1E17, 0x304}, {0x1E17, 0x301}, {0x1E18, 0x45}, {0x1E18, 0x32D}, +{0x1E19, 0x65}, {0x1E19, 0x32D}, {0x1E1A, 0x45}, {0x1E1A, 0x330}, {0x1E1B, 0x65}, {0x1E1B, 0x330}, {0x1E1C, 0x45}, {0x1E1C, 0x327}, {0x1E1C, 0x306}, {0x1E1D, 0x65}, {0x1E1D, 0x327}, {0x1E1D, 0x306}, +{0x1E1E, 0x46}, {0x1E1E, 0x307}, {0x1E1F, 0x66}, {0x1E1F, 0x307}, {0x1E20, 0x47}, {0x1E20, 0x304}, {0x1E21, 0x67}, {0x1E21, 0x304}, {0x1E22, 0x48}, {0x1E22, 0x307}, {0x1E23, 0x68}, {0x1E23, 0x307}, +{0x1E24, 0x48}, {0x1E24, 0x323}, {0x1E25, 0x68}, {0x1E25, 0x323}, {0x1E26, 0x48}, {0x1E26, 0x308}, {0x1E27, 0x68}, {0x1E27, 0x308}, {0x1E28, 0x48}, {0x1E28, 0x327}, {0x1E29, 0x68}, {0x1E29, 0x327}, +{0x1E2A, 0x48}, {0x1E2A, 0x32E}, {0x1E2B, 0x68}, {0x1E2B, 0x32E}, {0x1E2C, 0x49}, {0x1E2C, 0x330}, {0x1E2D, 0x69}, {0x1E2D, 0x330}, {0x1E2E, 0x49}, {0x1E2E, 0x308}, {0x1E2E, 0x301}, {0x1E2F, 0x69}, +{0x1E2F, 0x308}, {0x1E2F, 0x301}, {0x1E30, 0x4B}, {0x1E30, 0x301}, {0x1E31, 0x6B}, {0x1E31, 0x301}, {0x1E32, 0x4B}, {0x1E32, 0x323}, {0x1E33, 0x6B}, {0x1E33, 0x323}, {0x1E34, 0x4B}, {0x1E34, 0x331}, +{0x1E35, 0x6B}, {0x1E35, 0x331}, {0x1E36, 0x4C}, {0x1E36, 0x323}, {0x1E37, 0x6C}, {0x1E37, 0x323}, {0x1E38, 0x4C}, {0x1E38, 0x323}, {0x1E38, 0x304}, {0x1E39, 0x6C}, {0x1E39, 0x323}, {0x1E39, 0x304}, +{0x1E3A, 0x4C}, {0x1E3A, 0x331}, {0x1E3B, 0x6C}, {0x1E3B, 0x331}, {0x1E3C, 0x4C}, {0x1E3C, 0x32D}, {0x1E3D, 0x6C}, {0x1E3D, 0x32D}, {0x1E3E, 0x4D}, {0x1E3E, 0x301}, {0x1E3F, 0x6D}, {0x1E3F, 0x301}, +{0x1E40, 0x4D}, {0x1E40, 0x307}, {0x1E41, 0x6D}, {0x1E41, 0x307}, {0x1E42, 0x4D}, {0x1E42, 0x323}, {0x1E43, 0x6D}, {0x1E43, 0x323}, {0x1E44, 0x4E}, {0x1E44, 0x307}, {0x1E45, 0x6E}, {0x1E45, 0x307}, +{0x1E46, 0x4E}, {0x1E46, 0x323}, {0x1E47, 0x6E}, {0x1E47, 0x323}, {0x1E48, 0x4E}, {0x1E48, 0x331}, {0x1E49, 0x6E}, {0x1E49, 0x331}, {0x1E4A, 0x4E}, {0x1E4A, 0x32D}, {0x1E4B, 0x6E}, {0x1E4B, 0x32D}, +{0x1E4C, 0x4F}, {0x1E4C, 0x303}, {0x1E4C, 0x301}, {0x1E4D, 0x6F}, {0x1E4D, 0x303}, {0x1E4D, 0x301}, {0x1E4E, 0x4F}, {0x1E4E, 0x303}, {0x1E4E, 0x308}, {0x1E4F, 0x6F}, {0x1E4F, 0x303}, {0x1E4F, 0x308}, +{0x1E50, 0x4F}, {0x1E50, 0x304}, {0x1E50, 0x300}, {0x1E51, 0x6F}, {0x1E51, 0x304}, {0x1E51, 0x300}, {0x1E52, 0x4F}, {0x1E52, 0x304}, {0x1E52, 0x301}, {0x1E53, 0x6F}, {0x1E53, 0x304}, {0x1E53, 0x301}, +{0x1E54, 0x50}, {0x1E54, 0x301}, {0x1E55, 0x70}, {0x1E55, 0x301}, {0x1E56, 0x50}, {0x1E56, 0x307}, {0x1E57, 0x70}, {0x1E57, 0x307}, {0x1E58, 0x52}, {0x1E58, 0x307}, {0x1E59, 0x72}, {0x1E59, 0x307}, +{0x1E5A, 0x52}, {0x1E5A, 0x323}, {0x1E5B, 0x72}, {0x1E5B, 0x323}, {0x1E5C, 0x52}, {0x1E5C, 0x323}, {0x1E5C, 0x304}, {0x1E5D, 0x72}, {0x1E5D, 0x323}, {0x1E5D, 0x304}, {0x1E5E, 0x52}, {0x1E5E, 0x331}, +{0x1E5F, 0x72}, {0x1E5F, 0x331}, {0x1E60, 0x53}, {0x1E60, 0x307}, {0x1E61, 0x73}, {0x1E61, 0x307}, {0x1E62, 0x53}, {0x1E62, 0x323}, {0x1E63, 0x73}, {0x1E63, 0x323}, {0x1E64, 0x53}, {0x1E64, 0x301}, +{0x1E64, 0x307}, {0x1E65, 0x73}, {0x1E65, 0x301}, {0x1E65, 0x307}, {0x1E66, 0x53}, {0x1E66, 0x30C}, {0x1E66, 0x307}, {0x1E67, 0x73}, {0x1E67, 0x30C}, {0x1E67, 0x307}, {0x1E68, 0x53}, {0x1E68, 0x323}, +{0x1E68, 0x307}, {0x1E69, 0x73}, {0x1E69, 0x323}, {0x1E69, 0x307}, {0x1E6A, 0x54}, {0x1E6A, 0x307}, {0x1E6B, 0x74}, {0x1E6B, 0x307}, {0x1E6C, 0x54}, {0x1E6C, 0x323}, {0x1E6D, 0x74}, {0x1E6D, 0x323}, +{0x1E6E, 0x54}, {0x1E6E, 0x331}, {0x1E6F, 0x74}, {0x1E6F, 0x331}, {0x1E70, 0x54}, {0x1E70, 0x32D}, {0x1E71, 0x74}, {0x1E71, 0x32D}, {0x1E72, 0x55}, {0x1E72, 0x324}, {0x1E73, 0x75}, {0x1E73, 0x324}, +{0x1E74, 0x55}, {0x1E74, 0x330}, {0x1E75, 0x75}, {0x1E75, 0x330}, {0x1E76, 0x55}, {0x1E76, 0x32D}, {0x1E77, 0x75}, {0x1E77, 0x32D}, {0x1E78, 0x55}, {0x1E78, 0x303}, {0x1E78, 0x301}, {0x1E79, 0x75}, +{0x1E79, 0x303}, {0x1E79, 0x301}, {0x1E7A, 0x55}, {0x1E7A, 0x304}, {0x1E7A, 0x308}, {0x1E7B, 0x75}, {0x1E7B, 0x304}, {0x1E7B, 0x308}, {0x1E7C, 0x56}, {0x1E7C, 0x303}, {0x1E7D, 0x76}, {0x1E7D, 0x303}, +{0x1E7E, 0x56}, {0x1E7E, 0x323}, {0x1E7F, 0x76}, {0x1E7F, 0x323}, {0x1E80, 0x57}, {0x1E80, 0x300}, {0x1E81, 0x77}, {0x1E81, 0x300}, {0x1E82, 0x57}, {0x1E82, 0x301}, {0x1E83, 0x77}, {0x1E83, 0x301}, +{0x1E84, 0x57}, {0x1E84, 0x308}, {0x1E85, 0x77}, {0x1E85, 0x308}, {0x1E86, 0x57}, {0x1E86, 0x307}, {0x1E87, 0x77}, {0x1E87, 0x307}, {0x1E88, 0x57}, {0x1E88, 0x323}, {0x1E89, 0x77}, {0x1E89, 0x323}, +{0x1E8A, 0x58}, {0x1E8A, 0x307}, {0x1E8B, 0x78}, {0x1E8B, 0x307}, {0x1E8C, 0x58}, {0x1E8C, 0x308}, {0x1E8D, 0x78}, {0x1E8D, 0x308}, {0x1E8E, 0x59}, {0x1E8E, 0x307}, {0x1E8F, 0x79}, {0x1E8F, 0x307}, +{0x1E90, 0x5A}, {0x1E90, 0x302}, {0x1E91, 0x7A}, {0x1E91, 0x302}, {0x1E92, 0x5A}, {0x1E92, 0x323}, {0x1E93, 0x7A}, {0x1E93, 0x323}, {0x1E94, 0x5A}, {0x1E94, 0x331}, {0x1E95, 0x7A}, {0x1E95, 0x331}, +{0x1E96, 0x68}, {0x1E96, 0x331}, {0x1E97, 0x74}, {0x1E97, 0x308}, {0x1E98, 0x77}, {0x1E98, 0x30A}, {0x1E99, 0x79}, {0x1E99, 0x30A}, {0x1E9B, 0x17F}, {0x1E9B, 0x307}, {0x1EA0, 0x41}, {0x1EA0, 0x323}, +{0x1EA1, 0x61}, {0x1EA1, 0x323}, {0x1EA2, 0x41}, {0x1EA2, 0x309}, {0x1EA3, 0x61}, {0x1EA3, 0x309}, {0x1EA4, 0x41}, {0x1EA4, 0x302}, {0x1EA4, 0x301}, {0x1EA5, 0x61}, {0x1EA5, 0x302}, {0x1EA5, 0x301}, +{0x1EA6, 0x41}, {0x1EA6, 0x302}, {0x1EA6, 0x300}, {0x1EA7, 0x61}, {0x1EA7, 0x302}, {0x1EA7, 0x300}, {0x1EA8, 0x41}, {0x1EA8, 0x302}, {0x1EA8, 0x309}, {0x1EA9, 0x61}, {0x1EA9, 0x302}, {0x1EA9, 0x309}, +{0x1EAA, 0x41}, {0x1EAA, 0x302}, {0x1EAA, 0x303}, {0x1EAB, 0x61}, {0x1EAB, 0x302}, {0x1EAB, 0x303}, {0x1EAC, 0x41}, {0x1EAC, 0x323}, {0x1EAC, 0x302}, {0x1EAD, 0x61}, {0x1EAD, 0x323}, {0x1EAD, 0x302}, +{0x1EAE, 0x41}, {0x1EAE, 0x306}, {0x1EAE, 0x301}, {0x1EAF, 0x61}, {0x1EAF, 0x306}, {0x1EAF, 0x301}, {0x1EB0, 0x41}, {0x1EB0, 0x306}, {0x1EB0, 0x300}, {0x1EB1, 0x61}, {0x1EB1, 0x306}, {0x1EB1, 0x300}, +{0x1EB2, 0x41}, {0x1EB2, 0x306}, {0x1EB2, 0x309}, {0x1EB3, 0x61}, {0x1EB3, 0x306}, {0x1EB3, 0x309}, {0x1EB4, 0x41}, {0x1EB4, 0x306}, {0x1EB4, 0x303}, {0x1EB5, 0x61}, {0x1EB5, 0x306}, {0x1EB5, 0x303}, +{0x1EB6, 0x41}, {0x1EB6, 0x323}, {0x1EB6, 0x306}, {0x1EB7, 0x61}, {0x1EB7, 0x323}, {0x1EB7, 0x306}, {0x1EB8, 0x45}, {0x1EB8, 0x323}, {0x1EB9, 0x65}, {0x1EB9, 0x323}, {0x1EBA, 0x45}, {0x1EBA, 0x309}, +{0x1EBB, 0x65}, {0x1EBB, 0x309}, {0x1EBC, 0x45}, {0x1EBC, 0x303}, {0x1EBD, 0x65}, {0x1EBD, 0x303}, {0x1EBE, 0x45}, {0x1EBE, 0x302}, {0x1EBE, 0x301}, {0x1EBF, 0x65}, {0x1EBF, 0x302}, {0x1EBF, 0x301}, +{0x1EC0, 0x45}, {0x1EC0, 0x302}, {0x1EC0, 0x300}, {0x1EC1, 0x65}, {0x1EC1, 0x302}, {0x1EC1, 0x300}, {0x1EC2, 0x45}, {0x1EC2, 0x302}, {0x1EC2, 0x309}, {0x1EC3, 0x65}, {0x1EC3, 0x302}, {0x1EC3, 0x309}, +{0x1EC4, 0x45}, {0x1EC4, 0x302}, {0x1EC4, 0x303}, {0x1EC5, 0x65}, {0x1EC5, 0x302}, {0x1EC5, 0x303}, {0x1EC6, 0x45}, {0x1EC6, 0x323}, {0x1EC6, 0x302}, {0x1EC7, 0x65}, {0x1EC7, 0x323}, {0x1EC7, 0x302}, +{0x1EC8, 0x49}, {0x1EC8, 0x309}, {0x1EC9, 0x69}, {0x1EC9, 0x309}, {0x1ECA, 0x49}, {0x1ECA, 0x323}, {0x1ECB, 0x69}, {0x1ECB, 0x323}, {0x1ECC, 0x4F}, {0x1ECC, 0x323}, {0x1ECD, 0x6F}, {0x1ECD, 0x323}, +{0x1ECE, 0x4F}, {0x1ECE, 0x309}, {0x1ECF, 0x6F}, {0x1ECF, 0x309}, {0x1ED0, 0x4F}, {0x1ED0, 0x302}, {0x1ED0, 0x301}, {0x1ED1, 0x6F}, {0x1ED1, 0x302}, {0x1ED1, 0x301}, {0x1ED2, 0x4F}, {0x1ED2, 0x302}, +{0x1ED2, 0x300}, {0x1ED3, 0x6F}, {0x1ED3, 0x302}, {0x1ED3, 0x300}, {0x1ED4, 0x4F}, {0x1ED4, 0x302}, {0x1ED4, 0x309}, {0x1ED5, 0x6F}, {0x1ED5, 0x302}, {0x1ED5, 0x309}, {0x1ED6, 0x4F}, {0x1ED6, 0x302}, +{0x1ED6, 0x303}, {0x1ED7, 0x6F}, {0x1ED7, 0x302}, {0x1ED7, 0x303}, {0x1ED8, 0x4F}, {0x1ED8, 0x323}, {0x1ED8, 0x302}, {0x1ED9, 0x6F}, {0x1ED9, 0x323}, {0x1ED9, 0x302}, {0x1EDA, 0x4F}, {0x1EDA, 0x31B}, +{0x1EDA, 0x301}, {0x1EDB, 0x6F}, {0x1EDB, 0x31B}, {0x1EDB, 0x301}, {0x1EDC, 0x4F}, {0x1EDC, 0x31B}, {0x1EDC, 0x300}, {0x1EDD, 0x6F}, {0x1EDD, 0x31B}, {0x1EDD, 0x300}, {0x1EDE, 0x4F}, {0x1EDE, 0x31B}, +{0x1EDE, 0x309}, {0x1EDF, 0x6F}, {0x1EDF, 0x31B}, {0x1EDF, 0x309}, {0x1EE0, 0x4F}, {0x1EE0, 0x31B}, {0x1EE0, 0x303}, {0x1EE1, 0x6F}, {0x1EE1, 0x31B}, {0x1EE1, 0x303}, {0x1EE2, 0x4F}, {0x1EE2, 0x31B}, +{0x1EE2, 0x323}, {0x1EE3, 0x6F}, {0x1EE3, 0x31B}, {0x1EE3, 0x323}, {0x1EE4, 0x55}, {0x1EE4, 0x323}, {0x1EE5, 0x75}, {0x1EE5, 0x323}, {0x1EE6, 0x55}, {0x1EE6, 0x309}, {0x1EE7, 0x75}, {0x1EE7, 0x309}, +{0x1EE8, 0x55}, {0x1EE8, 0x31B}, {0x1EE8, 0x301}, {0x1EE9, 0x75}, {0x1EE9, 0x31B}, {0x1EE9, 0x301}, {0x1EEA, 0x55}, {0x1EEA, 0x31B}, {0x1EEA, 0x300}, {0x1EEB, 0x75}, {0x1EEB, 0x31B}, {0x1EEB, 0x300}, +{0x1EEC, 0x55}, {0x1EEC, 0x31B}, {0x1EEC, 0x309}, {0x1EED, 0x75}, {0x1EED, 0x31B}, {0x1EED, 0x309}, {0x1EEE, 0x55}, {0x1EEE, 0x31B}, {0x1EEE, 0x303}, {0x1EEF, 0x75}, {0x1EEF, 0x31B}, {0x1EEF, 0x303}, +{0x1EF0, 0x55}, {0x1EF0, 0x31B}, {0x1EF0, 0x323}, {0x1EF1, 0x75}, {0x1EF1, 0x31B}, {0x1EF1, 0x323}, {0x1EF2, 0x59}, {0x1EF2, 0x300}, {0x1EF3, 0x79}, {0x1EF3, 0x300}, {0x1EF4, 0x59}, {0x1EF4, 0x323}, +{0x1EF5, 0x79}, {0x1EF5, 0x323}, {0x1EF6, 0x59}, {0x1EF6, 0x309}, {0x1EF7, 0x79}, {0x1EF7, 0x309}, {0x1EF8, 0x59}, {0x1EF8, 0x303}, {0x1EF9, 0x79}, {0x1EF9, 0x303}, {0x1F00, 0x3B1}, {0x1F00, 0x313}, +{0x1F01, 0x3B1}, {0x1F01, 0x314}, {0x1F02, 0x3B1}, {0x1F02, 0x313}, {0x1F02, 0x300}, {0x1F03, 0x3B1}, {0x1F03, 0x314}, {0x1F03, 0x300}, {0x1F04, 0x3B1}, {0x1F04, 0x313}, {0x1F04, 0x301}, +{0x1F05, 0x3B1}, {0x1F05, 0x314}, {0x1F05, 0x301}, {0x1F06, 0x3B1}, {0x1F06, 0x313}, {0x1F06, 0x342}, {0x1F07, 0x3B1}, {0x1F07, 0x314}, {0x1F07, 0x342}, {0x1F08, 0x391}, {0x1F08, 0x313}, +{0x1F09, 0x391}, {0x1F09, 0x314}, {0x1F0A, 0x391}, {0x1F0A, 0x313}, {0x1F0A, 0x300}, {0x1F0B, 0x391}, {0x1F0B, 0x314}, {0x1F0B, 0x300}, {0x1F0C, 0x391}, {0x1F0C, 0x313}, {0x1F0C, 0x301}, +{0x1F0D, 0x391}, {0x1F0D, 0x314}, {0x1F0D, 0x301}, {0x1F0E, 0x391}, {0x1F0E, 0x313}, {0x1F0E, 0x342}, {0x1F0F, 0x391}, {0x1F0F, 0x314}, {0x1F0F, 0x342}, {0x1F10, 0x3B5}, {0x1F10, 0x313}, +{0x1F11, 0x3B5}, {0x1F11, 0x314}, {0x1F12, 0x3B5}, {0x1F12, 0x313}, {0x1F12, 0x300}, {0x1F13, 0x3B5}, {0x1F13, 0x314}, {0x1F13, 0x300}, {0x1F14, 0x3B5}, {0x1F14, 0x313}, {0x1F14, 0x301}, +{0x1F15, 0x3B5}, {0x1F15, 0x314}, {0x1F15, 0x301}, {0x1F18, 0x395}, {0x1F18, 0x313}, {0x1F19, 0x395}, {0x1F19, 0x314}, {0x1F1A, 0x395}, {0x1F1A, 0x313}, {0x1F1A, 0x300}, {0x1F1B, 0x395}, +{0x1F1B, 0x314}, {0x1F1B, 0x300}, {0x1F1C, 0x395}, {0x1F1C, 0x313}, {0x1F1C, 0x301}, {0x1F1D, 0x395}, {0x1F1D, 0x314}, {0x1F1D, 0x301}, {0x1F20, 0x3B7}, {0x1F20, 0x313}, {0x1F21, 0x3B7}, +{0x1F21, 0x314}, {0x1F22, 0x3B7}, {0x1F22, 0x313}, {0x1F22, 0x300}, {0x1F23, 0x3B7}, {0x1F23, 0x314}, {0x1F23, 0x300}, {0x1F24, 0x3B7}, {0x1F24, 0x313}, {0x1F24, 0x301}, {0x1F25, 0x3B7}, +{0x1F25, 0x314}, {0x1F25, 0x301}, {0x1F26, 0x3B7}, {0x1F26, 0x313}, {0x1F26, 0x342}, {0x1F27, 0x3B7}, {0x1F27, 0x314}, {0x1F27, 0x342}, {0x1F28, 0x397}, {0x1F28, 0x313}, {0x1F29, 0x397}, +{0x1F29, 0x314}, {0x1F2A, 0x397}, {0x1F2A, 0x313}, {0x1F2A, 0x300}, {0x1F2B, 0x397}, {0x1F2B, 0x314}, {0x1F2B, 0x300}, {0x1F2C, 0x397}, {0x1F2C, 0x313}, {0x1F2C, 0x301}, {0x1F2D, 0x397}, +{0x1F2D, 0x314}, {0x1F2D, 0x301}, {0x1F2E, 0x397}, {0x1F2E, 0x313}, {0x1F2E, 0x342}, {0x1F2F, 0x397}, {0x1F2F, 0x314}, {0x1F2F, 0x342}, {0x1F30, 0x3B9}, {0x1F30, 0x313}, {0x1F31, 0x3B9}, +{0x1F31, 0x314}, {0x1F32, 0x3B9}, {0x1F32, 0x313}, {0x1F32, 0x300}, {0x1F33, 0x3B9}, {0x1F33, 0x314}, {0x1F33, 0x300}, {0x1F34, 0x3B9}, {0x1F34, 0x313}, {0x1F34, 0x301}, {0x1F35, 0x3B9}, +{0x1F35, 0x314}, {0x1F35, 0x301}, {0x1F36, 0x3B9}, {0x1F36, 0x313}, {0x1F36, 0x342}, {0x1F37, 0x3B9}, {0x1F37, 0x314}, {0x1F37, 0x342}, {0x1F38, 0x399}, {0x1F38, 0x313}, {0x1F39, 0x399}, +{0x1F39, 0x314}, {0x1F3A, 0x399}, {0x1F3A, 0x313}, {0x1F3A, 0x300}, {0x1F3B, 0x399}, {0x1F3B, 0x314}, {0x1F3B, 0x300}, {0x1F3C, 0x399}, {0x1F3C, 0x313}, {0x1F3C, 0x301}, {0x1F3D, 0x399}, +{0x1F3D, 0x314}, {0x1F3D, 0x301}, {0x1F3E, 0x399}, {0x1F3E, 0x313}, {0x1F3E, 0x342}, {0x1F3F, 0x399}, {0x1F3F, 0x314}, {0x1F3F, 0x342}, {0x1F40, 0x3BF}, {0x1F40, 0x313}, {0x1F41, 0x3BF}, +{0x1F41, 0x314}, {0x1F42, 0x3BF}, {0x1F42, 0x313}, {0x1F42, 0x300}, {0x1F43, 0x3BF}, {0x1F43, 0x314}, {0x1F43, 0x300}, {0x1F44, 0x3BF}, {0x1F44, 0x313}, {0x1F44, 0x301}, {0x1F45, 0x3BF}, +{0x1F45, 0x314}, {0x1F45, 0x301}, {0x1F48, 0x39F}, {0x1F48, 0x313}, {0x1F49, 0x39F}, {0x1F49, 0x314}, {0x1F4A, 0x39F}, {0x1F4A, 0x313}, {0x1F4A, 0x300}, {0x1F4B, 0x39F}, {0x1F4B, 0x314}, +{0x1F4B, 0x300}, {0x1F4C, 0x39F}, {0x1F4C, 0x313}, {0x1F4C, 0x301}, {0x1F4D, 0x39F}, {0x1F4D, 0x314}, {0x1F4D, 0x301}, {0x1F50, 0x3C5}, {0x1F50, 0x313}, {0x1F51, 0x3C5}, {0x1F51, 0x314}, +{0x1F52, 0x3C5}, {0x1F52, 0x313}, {0x1F52, 0x300}, {0x1F53, 0x3C5}, {0x1F53, 0x314}, {0x1F53, 0x300}, {0x1F54, 0x3C5}, {0x1F54, 0x313}, {0x1F54, 0x301}, {0x1F55, 0x3C5}, {0x1F55, 0x314}, +{0x1F55, 0x301}, {0x1F56, 0x3C5}, {0x1F56, 0x313}, {0x1F56, 0x342}, {0x1F57, 0x3C5}, {0x1F57, 0x314}, {0x1F57, 0x342}, {0x1F59, 0x3A5}, {0x1F59, 0x314}, {0x1F5B, 0x3A5}, {0x1F5B, 0x314}, +{0x1F5B, 0x300}, {0x1F5D, 0x3A5}, {0x1F5D, 0x314}, {0x1F5D, 0x301}, {0x1F5F, 0x3A5}, {0x1F5F, 0x314}, {0x1F5F, 0x342}, {0x1F60, 0x3C9}, {0x1F60, 0x313}, {0x1F61, 0x3C9}, {0x1F61, 0x314}, +{0x1F62, 0x3C9}, {0x1F62, 0x313}, {0x1F62, 0x300}, {0x1F63, 0x3C9}, {0x1F63, 0x314}, {0x1F63, 0x300}, {0x1F64, 0x3C9}, {0x1F64, 0x313}, {0x1F64, 0x301}, {0x1F65, 0x3C9}, {0x1F65, 0x314}, +{0x1F65, 0x301}, {0x1F66, 0x3C9}, {0x1F66, 0x313}, {0x1F66, 0x342}, {0x1F67, 0x3C9}, {0x1F67, 0x314}, {0x1F67, 0x342}, {0x1F68, 0x3A9}, {0x1F68, 0x313}, {0x1F69, 0x3A9}, {0x1F69, 0x314}, +{0x1F6A, 0x3A9}, {0x1F6A, 0x313}, {0x1F6A, 0x300}, {0x1F6B, 0x3A9}, {0x1F6B, 0x314}, {0x1F6B, 0x300}, {0x1F6C, 0x3A9}, {0x1F6C, 0x313}, {0x1F6C, 0x301}, {0x1F6D, 0x3A9}, {0x1F6D, 0x314}, +{0x1F6D, 0x301}, {0x1F6E, 0x3A9}, {0x1F6E, 0x313}, {0x1F6E, 0x342}, {0x1F6F, 0x3A9}, {0x1F6F, 0x314}, {0x1F6F, 0x342}, {0x1F70, 0x3B1}, {0x1F70, 0x300}, {0x1F71, 0x3B1}, {0x1F71, 0x301}, +{0x1F72, 0x3B5}, {0x1F72, 0x300}, {0x1F73, 0x3B5}, {0x1F73, 0x301}, {0x1F74, 0x3B7}, {0x1F74, 0x300}, {0x1F75, 0x3B7}, {0x1F75, 0x301}, {0x1F76, 0x3B9}, {0x1F76, 0x300}, {0x1F77, 0x3B9}, +{0x1F77, 0x301}, {0x1F78, 0x3BF}, {0x1F78, 0x300}, {0x1F79, 0x3BF}, {0x1F79, 0x301}, {0x1F7A, 0x3C5}, {0x1F7A, 0x300}, {0x1F7B, 0x3C5}, {0x1F7B, 0x301}, {0x1F7C, 0x3C9}, {0x1F7C, 0x300}, +{0x1F7D, 0x3C9}, {0x1F7D, 0x301}, {0x1F80, 0x3B1}, {0x1F80, 0x313}, {0x1F80, 0x345}, {0x1F81, 0x3B1}, {0x1F81, 0x314}, {0x1F81, 0x345}, {0x1F82, 0x3B1}, {0x1F82, 0x313}, {0x1F82, 0x300}, +{0x1F82, 0x345}, {0x1F83, 0x3B1}, {0x1F83, 0x314}, {0x1F83, 0x300}, {0x1F83, 0x345}, {0x1F84, 0x3B1}, {0x1F84, 0x313}, {0x1F84, 0x301}, {0x1F84, 0x345}, {0x1F85, 0x3B1}, {0x1F85, 0x314}, +{0x1F85, 0x301}, {0x1F85, 0x345}, {0x1F86, 0x3B1}, {0x1F86, 0x313}, {0x1F86, 0x342}, {0x1F86, 0x345}, {0x1F87, 0x3B1}, {0x1F87, 0x314}, {0x1F87, 0x342}, {0x1F87, 0x345}, {0x1F88, 0x391}, +{0x1F88, 0x313}, {0x1F88, 0x345}, {0x1F89, 0x391}, {0x1F89, 0x314}, {0x1F89, 0x345}, {0x1F8A, 0x391}, {0x1F8A, 0x313}, {0x1F8A, 0x300}, {0x1F8A, 0x345}, {0x1F8B, 0x391}, {0x1F8B, 0x314}, +{0x1F8B, 0x300}, {0x1F8B, 0x345}, {0x1F8C, 0x391}, {0x1F8C, 0x313}, {0x1F8C, 0x301}, {0x1F8C, 0x345}, {0x1F8D, 0x391}, {0x1F8D, 0x314}, {0x1F8D, 0x301}, {0x1F8D, 0x345}, {0x1F8E, 0x391}, +{0x1F8E, 0x313}, {0x1F8E, 0x342}, {0x1F8E, 0x345}, {0x1F8F, 0x391}, {0x1F8F, 0x314}, {0x1F8F, 0x342}, {0x1F8F, 0x345}, {0x1F90, 0x3B7}, {0x1F90, 0x313}, {0x1F90, 0x345}, {0x1F91, 0x3B7}, +{0x1F91, 0x314}, {0x1F91, 0x345}, {0x1F92, 0x3B7}, {0x1F92, 0x313}, {0x1F92, 0x300}, {0x1F92, 0x345}, {0x1F93, 0x3B7}, {0x1F93, 0x314}, {0x1F93, 0x300}, {0x1F93, 0x345}, {0x1F94, 0x3B7}, +{0x1F94, 0x313}, {0x1F94, 0x301}, {0x1F94, 0x345}, {0x1F95, 0x3B7}, {0x1F95, 0x314}, {0x1F95, 0x301}, {0x1F95, 0x345}, {0x1F96, 0x3B7}, {0x1F96, 0x313}, {0x1F96, 0x342}, {0x1F96, 0x345}, +{0x1F97, 0x3B7}, {0x1F97, 0x314}, {0x1F97, 0x342}, {0x1F97, 0x345}, {0x1F98, 0x397}, {0x1F98, 0x313}, {0x1F98, 0x345}, {0x1F99, 0x397}, {0x1F99, 0x314}, {0x1F99, 0x345}, {0x1F9A, 0x397}, +{0x1F9A, 0x313}, {0x1F9A, 0x300}, {0x1F9A, 0x345}, {0x1F9B, 0x397}, {0x1F9B, 0x314}, {0x1F9B, 0x300}, {0x1F9B, 0x345}, {0x1F9C, 0x397}, {0x1F9C, 0x313}, {0x1F9C, 0x301}, {0x1F9C, 0x345}, +{0x1F9D, 0x397}, {0x1F9D, 0x314}, {0x1F9D, 0x301}, {0x1F9D, 0x345}, {0x1F9E, 0x397}, {0x1F9E, 0x313}, {0x1F9E, 0x342}, {0x1F9E, 0x345}, {0x1F9F, 0x397}, {0x1F9F, 0x314}, {0x1F9F, 0x342}, +{0x1F9F, 0x345}, {0x1FA0, 0x3C9}, {0x1FA0, 0x313}, {0x1FA0, 0x345}, {0x1FA1, 0x3C9}, {0x1FA1, 0x314}, {0x1FA1, 0x345}, {0x1FA2, 0x3C9}, {0x1FA2, 0x313}, {0x1FA2, 0x300}, {0x1FA2, 0x345}, +{0x1FA3, 0x3C9}, {0x1FA3, 0x314}, {0x1FA3, 0x300}, {0x1FA3, 0x345}, {0x1FA4, 0x3C9}, {0x1FA4, 0x313}, {0x1FA4, 0x301}, {0x1FA4, 0x345}, {0x1FA5, 0x3C9}, {0x1FA5, 0x314}, {0x1FA5, 0x301}, +{0x1FA5, 0x345}, {0x1FA6, 0x3C9}, {0x1FA6, 0x313}, {0x1FA6, 0x342}, {0x1FA6, 0x345}, {0x1FA7, 0x3C9}, {0x1FA7, 0x314}, {0x1FA7, 0x342}, {0x1FA7, 0x345}, {0x1FA8, 0x3A9}, {0x1FA8, 0x313}, +{0x1FA8, 0x345}, {0x1FA9, 0x3A9}, {0x1FA9, 0x314}, {0x1FA9, 0x345}, {0x1FAA, 0x3A9}, {0x1FAA, 0x313}, {0x1FAA, 0x300}, {0x1FAA, 0x345}, {0x1FAB, 0x3A9}, {0x1FAB, 0x314}, {0x1FAB, 0x300}, +{0x1FAB, 0x345}, {0x1FAC, 0x3A9}, {0x1FAC, 0x313}, {0x1FAC, 0x301}, {0x1FAC, 0x345}, {0x1FAD, 0x3A9}, {0x1FAD, 0x314}, {0x1FAD, 0x301}, {0x1FAD, 0x345}, {0x1FAE, 0x3A9}, {0x1FAE, 0x313}, +{0x1FAE, 0x342}, {0x1FAE, 0x345}, {0x1FAF, 0x3A9}, {0x1FAF, 0x314}, {0x1FAF, 0x342}, {0x1FAF, 0x345}, {0x1FB0, 0x3B1}, {0x1FB0, 0x306}, {0x1FB1, 0x3B1}, {0x1FB1, 0x304}, {0x1FB2, 0x3B1}, +{0x1FB2, 0x300}, {0x1FB2, 0x345}, {0x1FB3, 0x3B1}, {0x1FB3, 0x345}, {0x1FB4, 0x3B1}, {0x1FB4, 0x301}, {0x1FB4, 0x345}, {0x1FB6, 0x3B1}, {0x1FB6, 0x342}, {0x1FB7, 0x3B1}, {0x1FB7, 0x342}, +{0x1FB7, 0x345}, {0x1FB8, 0x391}, {0x1FB8, 0x306}, {0x1FB9, 0x391}, {0x1FB9, 0x304}, {0x1FBA, 0x391}, {0x1FBA, 0x300}, {0x1FBB, 0x391}, {0x1FBB, 0x301}, {0x1FBC, 0x391}, {0x1FBC, 0x345}, +{0x1FBE, 0x3B9}, {0x1FC1, 0xA8}, {0x1FC1, 0x342}, {0x1FC2, 0x3B7}, {0x1FC2, 0x300}, {0x1FC2, 0x345}, {0x1FC3, 0x3B7}, {0x1FC3, 0x345}, {0x1FC4, 0x3B7}, {0x1FC4, 0x301}, {0x1FC4, 0x345}, +{0x1FC6, 0x3B7}, {0x1FC6, 0x342}, {0x1FC7, 0x3B7}, {0x1FC7, 0x342}, {0x1FC7, 0x345}, {0x1FC8, 0x395}, {0x1FC8, 0x300}, {0x1FC9, 0x395}, {0x1FC9, 0x301}, {0x1FCA, 0x397}, {0x1FCA, 0x300}, +{0x1FCB, 0x397}, {0x1FCB, 0x301}, {0x1FCC, 0x397}, {0x1FCC, 0x345}, {0x1FCD, 0x1FBF}, {0x1FCD, 0x300}, {0x1FCE, 0x1FBF}, {0x1FCE, 0x301}, {0x1FCF, 0x1FBF}, {0x1FCF, 0x342}, {0x1FD0, 0x3B9}, +{0x1FD0, 0x306}, {0x1FD1, 0x3B9}, {0x1FD1, 0x304}, {0x1FD2, 0x3B9}, {0x1FD2, 0x308}, {0x1FD2, 0x300}, {0x1FD3, 0x3B9}, {0x1FD3, 0x308}, {0x1FD3, 0x301}, {0x1FD6, 0x3B9}, {0x1FD6, 0x342}, +{0x1FD7, 0x3B9}, {0x1FD7, 0x308}, {0x1FD7, 0x342}, {0x1FD8, 0x399}, {0x1FD8, 0x306}, {0x1FD9, 0x399}, {0x1FD9, 0x304}, {0x1FDA, 0x399}, {0x1FDA, 0x300}, {0x1FDB, 0x399}, {0x1FDB, 0x301}, +{0x1FDD, 0x1FFE}, {0x1FDD, 0x300}, {0x1FDE, 0x1FFE}, {0x1FDE, 0x301}, {0x1FDF, 0x1FFE}, {0x1FDF, 0x342}, {0x1FE0, 0x3C5}, {0x1FE0, 0x306}, {0x1FE1, 0x3C5}, {0x1FE1, 0x304}, {0x1FE2, 0x3C5}, +{0x1FE2, 0x308}, {0x1FE2, 0x300}, {0x1FE3, 0x3C5}, {0x1FE3, 0x308}, {0x1FE3, 0x301}, {0x1FE4, 0x3C1}, {0x1FE4, 0x313}, {0x1FE5, 0x3C1}, {0x1FE5, 0x314}, {0x1FE6, 0x3C5}, {0x1FE6, 0x342}, +{0x1FE7, 0x3C5}, {0x1FE7, 0x308}, {0x1FE7, 0x342}, {0x1FE8, 0x3A5}, {0x1FE8, 0x306}, {0x1FE9, 0x3A5}, {0x1FE9, 0x304}, {0x1FEA, 0x3A5}, {0x1FEA, 0x300}, {0x1FEB, 0x3A5}, {0x1FEB, 0x301}, +{0x1FEC, 0x3A1}, {0x1FEC, 0x314}, {0x1FED, 0xA8}, {0x1FED, 0x300}, {0x1FEE, 0xA8}, {0x1FEE, 0x301}, {0x1FEF, 0x60}, {0x1FF2, 0x3C9}, {0x1FF2, 0x300}, {0x1FF2, 0x345}, {0x1FF3, 0x3C9}, {0x1FF3, 0x345}, +{0x1FF4, 0x3C9}, {0x1FF4, 0x301}, {0x1FF4, 0x345}, {0x1FF6, 0x3C9}, {0x1FF6, 0x342}, {0x1FF7, 0x3C9}, {0x1FF7, 0x342}, {0x1FF7, 0x345}, {0x1FF8, 0x39F}, {0x1FF8, 0x300}, {0x1FF9, 0x39F}, +{0x1FF9, 0x301}, {0x1FFA, 0x3A9}, {0x1FFA, 0x300}, {0x1FFB, 0x3A9}, {0x1FFB, 0x301}, {0x1FFC, 0x3A9}, {0x1FFC, 0x345}, {0x1FFD, 0xB4}, {0x2000, 0x2002}, {0x2001, 0x2003}, {0x2126, 0x3A9}, +{0x212A, 0x4B}, {0x212B, 0x41}, {0x212B, 0x30A}, {0x219A, 0x2190}, {0x219A, 0x338}, {0x219B, 0x2192}, {0x219B, 0x338}, {0x21AE, 0x2194}, {0x21AE, 0x338}, {0x21CD, 0x21D0}, {0x21CD, 0x338}, +{0x21CE, 0x21D4}, {0x21CE, 0x338}, {0x21CF, 0x21D2}, {0x21CF, 0x338}, {0x2204, 0x2203}, {0x2204, 0x338}, {0x2209, 0x2208}, {0x2209, 0x338}, {0x220C, 0x220B}, {0x220C, 0x338}, {0x2224, 0x2223}, +{0x2224, 0x338}, {0x2226, 0x2225}, {0x2226, 0x338}, {0x2241, 0x223C}, {0x2241, 0x338}, {0x2244, 0x2243}, {0x2244, 0x338}, {0x2247, 0x2245}, {0x2247, 0x338}, {0x2249, 0x2248}, {0x2249, 0x338}, +{0x2260, 0x3D}, {0x2260, 0x338}, {0x2262, 0x2261}, {0x2262, 0x338}, {0x226D, 0x224D}, {0x226D, 0x338}, {0x226E, 0x3C}, {0x226E, 0x338}, {0x226F, 0x3E}, {0x226F, 0x338}, {0x2270, 0x2264}, +{0x2270, 0x338}, {0x2271, 0x2265}, {0x2271, 0x338}, {0x2274, 0x2272}, {0x2274, 0x338}, {0x2275, 0x2273}, {0x2275, 0x338}, {0x2278, 0x2276}, {0x2278, 0x338}, {0x2279, 0x2277}, {0x2279, 0x338}, +{0x2280, 0x227A}, {0x2280, 0x338}, {0x2281, 0x227B}, {0x2281, 0x338}, {0x2284, 0x2282}, {0x2284, 0x338}, {0x2285, 0x2283}, {0x2285, 0x338}, {0x2288, 0x2286}, {0x2288, 0x338}, {0x2289, 0x2287}, +{0x2289, 0x338}, {0x22AC, 0x22A2}, {0x22AC, 0x338}, {0x22AD, 0x22A8}, {0x22AD, 0x338}, {0x22AE, 0x22A9}, {0x22AE, 0x338}, {0x22AF, 0x22AB}, {0x22AF, 0x338}, {0x22E0, 0x227C}, {0x22E0, 0x338}, +{0x22E1, 0x227D}, {0x22E1, 0x338}, {0x22E2, 0x2291}, {0x22E2, 0x338}, {0x22E3, 0x2292}, {0x22E3, 0x338}, {0x22EA, 0x22B2}, {0x22EA, 0x338}, {0x22EB, 0x22B3}, {0x22EB, 0x338}, {0x22EC, 0x22B4}, +{0x22EC, 0x338}, {0x22ED, 0x22B5}, {0x22ED, 0x338}, {0x2329, 0x3008}, {0x232A, 0x3009}, {0x2ADC, 0x2ADD}, {0x2ADC, 0x338}, {0x304C, 0x304B}, {0x304C, 0x3099}, {0x304E, 0x304D}, {0x304E, 0x3099}, +{0x3050, 0x304F}, {0x3050, 0x3099}, {0x3052, 0x3051}, {0x3052, 0x3099}, {0x3054, 0x3053}, {0x3054, 0x3099}, {0x3056, 0x3055}, {0x3056, 0x3099}, {0x3058, 0x3057}, {0x3058, 0x3099}, {0x305A, 0x3059}, +{0x305A, 0x3099}, {0x305C, 0x305B}, {0x305C, 0x3099}, {0x305E, 0x305D}, {0x305E, 0x3099}, {0x3060, 0x305F}, {0x3060, 0x3099}, {0x3062, 0x3061}, {0x3062, 0x3099}, {0x3065, 0x3064}, {0x3065, 0x3099}, +{0x3067, 0x3066}, {0x3067, 0x3099}, {0x3069, 0x3068}, {0x3069, 0x3099}, {0x3070, 0x306F}, {0x3070, 0x3099}, {0x3071, 0x306F}, {0x3071, 0x309A}, {0x3073, 0x3072}, {0x3073, 0x3099}, {0x3074, 0x3072}, +{0x3074, 0x309A}, {0x3076, 0x3075}, {0x3076, 0x3099}, {0x3077, 0x3075}, {0x3077, 0x309A}, {0x3079, 0x3078}, {0x3079, 0x3099}, {0x307A, 0x3078}, {0x307A, 0x309A}, {0x307C, 0x307B}, {0x307C, 0x3099}, +{0x307D, 0x307B}, {0x307D, 0x309A}, {0x3094, 0x3046}, {0x3094, 0x3099}, {0x309E, 0x309D}, {0x309E, 0x3099}, {0x30AC, 0x30AB}, {0x30AC, 0x3099}, {0x30AE, 0x30AD}, {0x30AE, 0x3099}, {0x30B0, 0x30AF}, +{0x30B0, 0x3099}, {0x30B2, 0x30B1}, {0x30B2, 0x3099}, {0x30B4, 0x30B3}, {0x30B4, 0x3099}, {0x30B6, 0x30B5}, {0x30B6, 0x3099}, {0x30B8, 0x30B7}, {0x30B8, 0x3099}, {0x30BA, 0x30B9}, {0x30BA, 0x3099}, +{0x30BC, 0x30BB}, {0x30BC, 0x3099}, {0x30BE, 0x30BD}, {0x30BE, 0x3099}, {0x30C0, 0x30BF}, {0x30C0, 0x3099}, {0x30C2, 0x30C1}, {0x30C2, 0x3099}, {0x30C5, 0x30C4}, {0x30C5, 0x3099}, {0x30C7, 0x30C6}, +{0x30C7, 0x3099}, {0x30C9, 0x30C8}, {0x30C9, 0x3099}, {0x30D0, 0x30CF}, {0x30D0, 0x3099}, {0x30D1, 0x30CF}, {0x30D1, 0x309A}, {0x30D3, 0x30D2}, {0x30D3, 0x3099}, {0x30D4, 0x30D2}, {0x30D4, 0x309A}, +{0x30D6, 0x30D5}, {0x30D6, 0x3099}, {0x30D7, 0x30D5}, {0x30D7, 0x309A}, {0x30D9, 0x30D8}, {0x30D9, 0x3099}, {0x30DA, 0x30D8}, {0x30DA, 0x309A}, {0x30DC, 0x30DB}, {0x30DC, 0x3099}, {0x30DD, 0x30DB}, +{0x30DD, 0x309A}, {0x30F4, 0x30A6}, {0x30F4, 0x3099}, {0x30F7, 0x30EF}, {0x30F7, 0x3099}, {0x30F8, 0x30F0}, {0x30F8, 0x3099}, {0x30F9, 0x30F1}, {0x30F9, 0x3099}, {0x30FA, 0x30F2}, {0x30FA, 0x3099}, +{0x30FE, 0x30FD}, {0x30FE, 0x3099}, {0xF900, 0x8C48}, {0xF901, 0x66F4}, {0xF902, 0x8ECA}, {0xF903, 0x8CC8}, {0xF904, 0x6ED1}, {0xF905, 0x4E32}, {0xF906, 0x53E5}, {0xF907, 0x9F9C}, {0xF908, 0x9F9C}, +{0xF909, 0x5951}, {0xF90A, 0x91D1}, {0xF90B, 0x5587}, {0xF90C, 0x5948}, {0xF90D, 0x61F6}, {0xF90E, 0x7669}, {0xF90F, 0x7F85}, {0xF910, 0x863F}, {0xF911, 0x87BA}, {0xF912, 0x88F8}, {0xF913, 0x908F}, +{0xF914, 0x6A02}, {0xF915, 0x6D1B}, {0xF916, 0x70D9}, {0xF917, 0x73DE}, {0xF918, 0x843D}, {0xF919, 0x916A}, {0xF91A, 0x99F1}, {0xF91B, 0x4E82}, {0xF91C, 0x5375}, {0xF91D, 0x6B04}, {0xF91E, 0x721B}, +{0xF91F, 0x862D}, {0xF920, 0x9E1E}, {0xF921, 0x5D50}, {0xF922, 0x6FEB}, {0xF923, 0x85CD}, {0xF924, 0x8964}, {0xF925, 0x62C9}, {0xF926, 0x81D8}, {0xF927, 0x881F}, {0xF928, 0x5ECA}, {0xF929, 0x6717}, +{0xF92A, 0x6D6A}, {0xF92B, 0x72FC}, {0xF92C, 0x90CE}, {0xF92D, 0x4F86}, {0xF92E, 0x51B7}, {0xF92F, 0x52DE}, {0xF930, 0x64C4}, {0xF931, 0x6AD3}, {0xF932, 0x7210}, {0xF933, 0x76E7}, {0xF934, 0x8001}, +{0xF935, 0x8606}, {0xF936, 0x865C}, {0xF937, 0x8DEF}, {0xF938, 0x9732}, {0xF939, 0x9B6F}, {0xF93A, 0x9DFA}, {0xF93B, 0x788C}, {0xF93C, 0x797F}, {0xF93D, 0x7DA0}, {0xF93E, 0x83C9}, {0xF93F, 0x9304}, +{0xF940, 0x9E7F}, {0xF941, 0x8AD6}, {0xF942, 0x58DF}, {0xF943, 0x5F04}, {0xF944, 0x7C60}, {0xF945, 0x807E}, {0xF946, 0x7262}, {0xF947, 0x78CA}, {0xF948, 0x8CC2}, {0xF949, 0x96F7}, {0xF94A, 0x58D8}, +{0xF94B, 0x5C62}, {0xF94C, 0x6A13}, {0xF94D, 0x6DDA}, {0xF94E, 0x6F0F}, {0xF94F, 0x7D2F}, {0xF950, 0x7E37}, {0xF951, 0x964B}, {0xF952, 0x52D2}, {0xF953, 0x808B}, {0xF954, 0x51DC}, {0xF955, 0x51CC}, +{0xF956, 0x7A1C}, {0xF957, 0x7DBE}, {0xF958, 0x83F1}, {0xF959, 0x9675}, {0xF95A, 0x8B80}, {0xF95B, 0x62CF}, {0xF95C, 0x6A02}, {0xF95D, 0x8AFE}, {0xF95E, 0x4E39}, {0xF95F, 0x5BE7}, {0xF960, 0x6012}, +{0xF961, 0x7387}, {0xF962, 0x7570}, {0xF963, 0x5317}, {0xF964, 0x78FB}, {0xF965, 0x4FBF}, {0xF966, 0x5FA9}, {0xF967, 0x4E0D}, {0xF968, 0x6CCC}, {0xF969, 0x6578}, {0xF96A, 0x7D22}, {0xF96B, 0x53C3}, +{0xF96C, 0x585E}, {0xF96D, 0x7701}, {0xF96E, 0x8449}, {0xF96F, 0x8AAA}, {0xF970, 0x6BBA}, {0xF971, 0x8FB0}, {0xF972, 0x6C88}, {0xF973, 0x62FE}, {0xF974, 0x82E5}, {0xF975, 0x63A0}, {0xF976, 0x7565}, +{0xF977, 0x4EAE}, {0xF978, 0x5169}, {0xF979, 0x51C9}, {0xF97A, 0x6881}, {0xF97B, 0x7CE7}, {0xF97C, 0x826F}, {0xF97D, 0x8AD2}, {0xF97E, 0x91CF}, {0xF97F, 0x52F5}, {0xF980, 0x5442}, {0xF981, 0x5973}, +{0xF982, 0x5EEC}, {0xF983, 0x65C5}, {0xF984, 0x6FFE}, {0xF985, 0x792A}, {0xF986, 0x95AD}, {0xF987, 0x9A6A}, {0xF988, 0x9E97}, {0xF989, 0x9ECE}, {0xF98A, 0x529B}, {0xF98B, 0x66C6}, {0xF98C, 0x6B77}, +{0xF98D, 0x8F62}, {0xF98E, 0x5E74}, {0xF98F, 0x6190}, {0xF990, 0x6200}, {0xF991, 0x649A}, {0xF992, 0x6F23}, {0xF993, 0x7149}, {0xF994, 0x7489}, {0xF995, 0x79CA}, {0xF996, 0x7DF4}, {0xF997, 0x806F}, +{0xF998, 0x8F26}, {0xF999, 0x84EE}, {0xF99A, 0x9023}, {0xF99B, 0x934A}, {0xF99C, 0x5217}, {0xF99D, 0x52A3}, {0xF99E, 0x54BD}, {0xF99F, 0x70C8}, {0xF9A0, 0x88C2}, {0xF9A1, 0x8AAA}, {0xF9A2, 0x5EC9}, +{0xF9A3, 0x5FF5}, {0xF9A4, 0x637B}, {0xF9A5, 0x6BAE}, {0xF9A6, 0x7C3E}, {0xF9A7, 0x7375}, {0xF9A8, 0x4EE4}, {0xF9A9, 0x56F9}, {0xF9AA, 0x5BE7}, {0xF9AB, 0x5DBA}, {0xF9AC, 0x601C}, {0xF9AD, 0x73B2}, +{0xF9AE, 0x7469}, {0xF9AF, 0x7F9A}, {0xF9B0, 0x8046}, {0xF9B1, 0x9234}, {0xF9B2, 0x96F6}, {0xF9B3, 0x9748}, {0xF9B4, 0x9818}, {0xF9B5, 0x4F8B}, {0xF9B6, 0x79AE}, {0xF9B7, 0x91B4}, {0xF9B8, 0x96B8}, +{0xF9B9, 0x60E1}, {0xF9BA, 0x4E86}, {0xF9BB, 0x50DA}, {0xF9BC, 0x5BEE}, {0xF9BD, 0x5C3F}, {0xF9BE, 0x6599}, {0xF9BF, 0x6A02}, {0xF9C0, 0x71CE}, {0xF9C1, 0x7642}, {0xF9C2, 0x84FC}, {0xF9C3, 0x907C}, +{0xF9C4, 0x9F8D}, {0xF9C5, 0x6688}, {0xF9C6, 0x962E}, {0xF9C7, 0x5289}, {0xF9C8, 0x677B}, {0xF9C9, 0x67F3}, {0xF9CA, 0x6D41}, {0xF9CB, 0x6E9C}, {0xF9CC, 0x7409}, {0xF9CD, 0x7559}, {0xF9CE, 0x786B}, +{0xF9CF, 0x7D10}, {0xF9D0, 0x985E}, {0xF9D1, 0x516D}, {0xF9D2, 0x622E}, {0xF9D3, 0x9678}, {0xF9D4, 0x502B}, {0xF9D5, 0x5D19}, {0xF9D6, 0x6DEA}, {0xF9D7, 0x8F2A}, {0xF9D8, 0x5F8B}, {0xF9D9, 0x6144}, +{0xF9DA, 0x6817}, {0xF9DB, 0x7387}, {0xF9DC, 0x9686}, {0xF9DD, 0x5229}, {0xF9DE, 0x540F}, {0xF9DF, 0x5C65}, {0xF9E0, 0x6613}, {0xF9E1, 0x674E}, {0xF9E2, 0x68A8}, {0xF9E3, 0x6CE5}, {0xF9E4, 0x7406}, +{0xF9E5, 0x75E2}, {0xF9E6, 0x7F79}, {0xF9E7, 0x88CF}, {0xF9E8, 0x88E1}, {0xF9E9, 0x91CC}, {0xF9EA, 0x96E2}, {0xF9EB, 0x533F}, {0xF9EC, 0x6EBA}, {0xF9ED, 0x541D}, {0xF9EE, 0x71D0}, {0xF9EF, 0x7498}, +{0xF9F0, 0x85FA}, {0xF9F1, 0x96A3}, {0xF9F2, 0x9C57}, {0xF9F3, 0x9E9F}, {0xF9F4, 0x6797}, {0xF9F5, 0x6DCB}, {0xF9F6, 0x81E8}, {0xF9F7, 0x7ACB}, {0xF9F8, 0x7B20}, {0xF9F9, 0x7C92}, {0xF9FA, 0x72C0}, +{0xF9FB, 0x7099}, {0xF9FC, 0x8B58}, {0xF9FD, 0x4EC0}, {0xF9FE, 0x8336}, {0xF9FF, 0x523A}, {0xFA00, 0x5207}, {0xFA01, 0x5EA6}, {0xFA02, 0x62D3}, {0xFA03, 0x7CD6}, {0xFA04, 0x5B85}, {0xFA05, 0x6D1E}, +{0xFA06, 0x66B4}, {0xFA07, 0x8F3B}, {0xFA08, 0x884C}, {0xFA09, 0x964D}, {0xFA0A, 0x898B}, {0xFA0B, 0x5ED3}, {0xFA0C, 0x5140}, {0xFA0D, 0x55C0}, {0xFA10, 0x585A}, {0xFA12, 0x6674}, {0xFA15, 0x51DE}, +{0xFA16, 0x732A}, {0xFA17, 0x76CA}, {0xFA18, 0x793C}, {0xFA19, 0x795E}, {0xFA1A, 0x7965}, {0xFA1B, 0x798F}, {0xFA1C, 0x9756}, {0xFA1D, 0x7CBE}, {0xFA1E, 0x7FBD}, {0xFA20, 0x8612}, {0xFA22, 0x8AF8}, +{0xFA25, 0x9038}, {0xFA26, 0x90FD}, {0xFA2A, 0x98EF}, {0xFA2B, 0x98FC}, {0xFA2C, 0x9928}, {0xFA2D, 0x9DB4}, {0xFA2E, 0x90DE}, {0xFA2F, 0x96B7}, {0xFA30, 0x4FAE}, {0xFA31, 0x50E7}, {0xFA32, 0x514D}, +{0xFA33, 0x52C9}, {0xFA34, 0x52E4}, {0xFA35, 0x5351}, {0xFA36, 0x559D}, {0xFA37, 0x5606}, {0xFA38, 0x5668}, {0xFA39, 0x5840}, {0xFA3A, 0x58A8}, {0xFA3B, 0x5C64}, {0xFA3C, 0x5C6E}, {0xFA3D, 0x6094}, +{0xFA3E, 0x6168}, {0xFA3F, 0x618E}, {0xFA40, 0x61F2}, {0xFA41, 0x654F}, {0xFA42, 0x65E2}, {0xFA43, 0x6691}, {0xFA44, 0x6885}, {0xFA45, 0x6D77}, {0xFA46, 0x6E1A}, {0xFA47, 0x6F22}, {0xFA48, 0x716E}, +{0xFA49, 0x722B}, {0xFA4A, 0x7422}, {0xFA4B, 0x7891}, {0xFA4C, 0x793E}, {0xFA4D, 0x7949}, {0xFA4E, 0x7948}, {0xFA4F, 0x7950}, {0xFA50, 0x7956}, {0xFA51, 0x795D}, {0xFA52, 0x798D}, {0xFA53, 0x798E}, +{0xFA54, 0x7A40}, {0xFA55, 0x7A81}, {0xFA56, 0x7BC0}, {0xFA57, 0x7DF4}, {0xFA58, 0x7E09}, {0xFA59, 0x7E41}, {0xFA5A, 0x7F72}, {0xFA5B, 0x8005}, {0xFA5C, 0x81ED}, {0xFA5D, 0x8279}, {0xFA5E, 0x8279}, +{0xFA5F, 0x8457}, {0xFA60, 0x8910}, {0xFA61, 0x8996}, {0xFA62, 0x8B01}, {0xFA63, 0x8B39}, {0xFA64, 0x8CD3}, {0xFA65, 0x8D08}, {0xFA66, 0x8FB6}, {0xFA67, 0x9038}, {0xFA68, 0x96E3}, {0xFA69, 0x97FF}, +{0xFA6A, 0x983B}, {0xFA6B, 0x6075}, {0xFA6C, 0x242EE}, {0xFA6D, 0x8218}, {0xFA70, 0x4E26}, {0xFA71, 0x51B5}, {0xFA72, 0x5168}, {0xFA73, 0x4F80}, {0xFA74, 0x5145}, {0xFA75, 0x5180}, {0xFA76, 0x52C7}, +{0xFA77, 0x52FA}, {0xFA78, 0x559D}, {0xFA79, 0x5555}, {0xFA7A, 0x5599}, {0xFA7B, 0x55E2}, {0xFA7C, 0x585A}, {0xFA7D, 0x58B3}, {0xFA7E, 0x5944}, {0xFA7F, 0x5954}, {0xFA80, 0x5A62}, {0xFA81, 0x5B28}, +{0xFA82, 0x5ED2}, {0xFA83, 0x5ED9}, {0xFA84, 0x5F69}, {0xFA85, 0x5FAD}, {0xFA86, 0x60D8}, {0xFA87, 0x614E}, {0xFA88, 0x6108}, {0xFA89, 0x618E}, {0xFA8A, 0x6160}, {0xFA8B, 0x61F2}, {0xFA8C, 0x6234}, +{0xFA8D, 0x63C4}, {0xFA8E, 0x641C}, {0xFA8F, 0x6452}, {0xFA90, 0x6556}, {0xFA91, 0x6674}, {0xFA92, 0x6717}, {0xFA93, 0x671B}, {0xFA94, 0x6756}, {0xFA95, 0x6B79}, {0xFA96, 0x6BBA}, {0xFA97, 0x6D41}, +{0xFA98, 0x6EDB}, {0xFA99, 0x6ECB}, {0xFA9A, 0x6F22}, {0xFA9B, 0x701E}, {0xFA9C, 0x716E}, {0xFA9D, 0x77A7}, {0xFA9E, 0x7235}, {0xFA9F, 0x72AF}, {0xFAA0, 0x732A}, {0xFAA1, 0x7471}, {0xFAA2, 0x7506}, +{0xFAA3, 0x753B}, {0xFAA4, 0x761D}, {0xFAA5, 0x761F}, {0xFAA6, 0x76CA}, {0xFAA7, 0x76DB}, {0xFAA8, 0x76F4}, {0xFAA9, 0x774A}, {0xFAAA, 0x7740}, {0xFAAB, 0x78CC}, {0xFAAC, 0x7AB1}, {0xFAAD, 0x7BC0}, +{0xFAAE, 0x7C7B}, {0xFAAF, 0x7D5B}, {0xFAB0, 0x7DF4}, {0xFAB1, 0x7F3E}, {0xFAB2, 0x8005}, {0xFAB3, 0x8352}, {0xFAB4, 0x83EF}, {0xFAB5, 0x8779}, {0xFAB6, 0x8941}, {0xFAB7, 0x8986}, {0xFAB8, 0x8996}, +{0xFAB9, 0x8ABF}, {0xFABA, 0x8AF8}, {0xFABB, 0x8ACB}, {0xFABC, 0x8B01}, {0xFABD, 0x8AFE}, {0xFABE, 0x8AED}, {0xFABF, 0x8B39}, {0xFAC0, 0x8B8A}, {0xFAC1, 0x8D08}, {0xFAC2, 0x8F38}, {0xFAC3, 0x9072}, +{0xFAC4, 0x9199}, {0xFAC5, 0x9276}, {0xFAC6, 0x967C}, {0xFAC7, 0x96E3}, {0xFAC8, 0x9756}, {0xFAC9, 0x97DB}, {0xFACA, 0x97FF}, {0xFACB, 0x980B}, {0xFACC, 0x983B}, {0xFACD, 0x9B12}, {0xFACE, 0x9F9C}, +{0xFACF, 0x2284A}, {0xFAD0, 0x22844}, {0xFAD1, 0x233D5}, {0xFAD2, 0x3B9D}, {0xFAD3, 0x4018}, {0xFAD4, 0x4039}, {0xFAD5, 0x25249}, {0xFAD6, 0x25CD0}, {0xFAD7, 0x27ED3}, {0xFAD8, 0x9F43}, +{0xFAD9, 0x9F8E}, {0xFB1D, 0x5D9}, {0xFB1D, 0x5B4}, {0xFB1F, 0x5F2}, {0xFB1F, 0x5B7}, {0xFB2A, 0x5E9}, {0xFB2A, 0x5C1}, {0xFB2B, 0x5E9}, {0xFB2B, 0x5C2}, {0xFB2C, 0x5E9}, {0xFB2C, 0x5BC}, +{0xFB2C, 0x5C1}, {0xFB2D, 0x5E9}, {0xFB2D, 0x5BC}, {0xFB2D, 0x5C2}, {0xFB2E, 0x5D0}, {0xFB2E, 0x5B7}, {0xFB2F, 0x5D0}, {0xFB2F, 0x5B8}, {0xFB30, 0x5D0}, {0xFB30, 0x5BC}, {0xFB31, 0x5D1}, +{0xFB31, 0x5BC}, {0xFB32, 0x5D2}, {0xFB32, 0x5BC}, {0xFB33, 0x5D3}, {0xFB33, 0x5BC}, {0xFB34, 0x5D4}, {0xFB34, 0x5BC}, {0xFB35, 0x5D5}, {0xFB35, 0x5BC}, {0xFB36, 0x5D6}, {0xFB36, 0x5BC}, +{0xFB38, 0x5D8}, {0xFB38, 0x5BC}, {0xFB39, 0x5D9}, {0xFB39, 0x5BC}, {0xFB3A, 0x5DA}, {0xFB3A, 0x5BC}, {0xFB3B, 0x5DB}, {0xFB3B, 0x5BC}, {0xFB3C, 0x5DC}, {0xFB3C, 0x5BC}, {0xFB3E, 0x5DE}, +{0xFB3E, 0x5BC}, {0xFB40, 0x5E0}, {0xFB40, 0x5BC}, {0xFB41, 0x5E1}, {0xFB41, 0x5BC}, {0xFB43, 0x5E3}, {0xFB43, 0x5BC}, {0xFB44, 0x5E4}, {0xFB44, 0x5BC}, {0xFB46, 0x5E6}, {0xFB46, 0x5BC}, +{0xFB47, 0x5E7}, {0xFB47, 0x5BC}, {0xFB48, 0x5E8}, {0xFB48, 0x5BC}, {0xFB49, 0x5E9}, {0xFB49, 0x5BC}, {0xFB4A, 0x5EA}, {0xFB4A, 0x5BC}, {0xFB4B, 0x5D5}, {0xFB4B, 0x5B9}, {0xFB4C, 0x5D1}, +{0xFB4C, 0x5BF}, {0xFB4D, 0x5DB}, {0xFB4D, 0x5BF}, {0xFB4E, 0x5E4}, {0xFB4E, 0x5BF}, {0x1109A, 0x11099}, {0x1109A, 0x110BA}, {0x1109C, 0x1109B}, {0x1109C, 0x110BA}, {0x110AB, 0x110A5}, +{0x110AB, 0x110BA}, {0x1112E, 0x11131}, {0x1112E, 0x11127}, {0x1112F, 0x11132}, {0x1112F, 0x11127}, {0x1134B, 0x11347}, {0x1134B, 0x1133E}, {0x1134C, 0x11347}, {0x1134C, 0x11357}, {0x114BB, 0x114B9}, +{0x114BB, 0x114BA}, {0x114BC, 0x114B9}, {0x114BC, 0x114B0}, {0x114BE, 0x114B9}, {0x114BE, 0x114BD}, {0x115BA, 0x115B8}, {0x115BA, 0x115AF}, {0x115BB, 0x115B9}, {0x115BB, 0x115AF}, {0x1D15E, 0x1D157}, +{0x1D15E, 0x1D165}, {0x1D15F, 0x1D158}, {0x1D15F, 0x1D165}, {0x1D160, 0x1D158}, {0x1D160, 0x1D165}, {0x1D160, 0x1D16E}, {0x1D161, 0x1D158}, {0x1D161, 0x1D165}, {0x1D161, 0x1D16F}, {0x1D162, 0x1D158}, +{0x1D162, 0x1D165}, {0x1D162, 0x1D170}, {0x1D163, 0x1D158}, {0x1D163, 0x1D165}, {0x1D163, 0x1D171}, {0x1D164, 0x1D158}, {0x1D164, 0x1D165}, {0x1D164, 0x1D172}, {0x1D1BB, 0x1D1B9}, {0x1D1BB, 0x1D165}, +{0x1D1BC, 0x1D1BA}, {0x1D1BC, 0x1D165}, {0x1D1BD, 0x1D1B9}, {0x1D1BD, 0x1D165}, {0x1D1BD, 0x1D16E}, {0x1D1BE, 0x1D1BA}, {0x1D1BE, 0x1D165}, {0x1D1BE, 0x1D16E}, {0x1D1BF, 0x1D1B9}, {0x1D1BF, 0x1D165}, +{0x1D1BF, 0x1D16F}, {0x1D1C0, 0x1D1BA}, {0x1D1C0, 0x1D165}, {0x1D1C0, 0x1D16F}, {0x2F800, 0x4E3D}, {0x2F801, 0x4E38}, {0x2F802, 0x4E41}, {0x2F803, 0x20122}, {0x2F804, 0x4F60}, {0x2F805, 0x4FAE}, +{0x2F806, 0x4FBB}, {0x2F807, 0x5002}, {0x2F808, 0x507A}, {0x2F809, 0x5099}, {0x2F80A, 0x50E7}, {0x2F80B, 0x50CF}, {0x2F80C, 0x349E}, {0x2F80D, 0x2063A}, {0x2F80E, 0x514D}, {0x2F80F, 0x5154}, +{0x2F810, 0x5164}, {0x2F811, 0x5177}, {0x2F812, 0x2051C}, {0x2F813, 0x34B9}, {0x2F814, 0x5167}, {0x2F815, 0x518D}, {0x2F816, 0x2054B}, {0x2F817, 0x5197}, {0x2F818, 0x51A4}, {0x2F819, 0x4ECC}, +{0x2F81A, 0x51AC}, {0x2F81B, 0x51B5}, {0x2F81C, 0x291DF}, {0x2F81D, 0x51F5}, {0x2F81E, 0x5203}, {0x2F81F, 0x34DF}, {0x2F820, 0x523B}, {0x2F821, 0x5246}, {0x2F822, 0x5272}, {0x2F823, 0x5277}, +{0x2F824, 0x3515}, {0x2F825, 0x52C7}, {0x2F826, 0x52C9}, {0x2F827, 0x52E4}, {0x2F828, 0x52FA}, {0x2F829, 0x5305}, {0x2F82A, 0x5306}, {0x2F82B, 0x5317}, {0x2F82C, 0x5349}, {0x2F82D, 0x5351}, +{0x2F82E, 0x535A}, {0x2F82F, 0x5373}, {0x2F830, 0x537D}, {0x2F831, 0x537F}, {0x2F832, 0x537F}, {0x2F833, 0x537F}, {0x2F834, 0x20A2C}, {0x2F835, 0x7070}, {0x2F836, 0x53CA}, {0x2F837, 0x53DF}, +{0x2F838, 0x20B63}, {0x2F839, 0x53EB}, {0x2F83A, 0x53F1}, {0x2F83B, 0x5406}, {0x2F83C, 0x549E}, {0x2F83D, 0x5438}, {0x2F83E, 0x5448}, {0x2F83F, 0x5468}, {0x2F840, 0x54A2}, {0x2F841, 0x54F6}, +{0x2F842, 0x5510}, {0x2F843, 0x5553}, {0x2F844, 0x5563}, {0x2F845, 0x5584}, {0x2F846, 0x5584}, {0x2F847, 0x5599}, {0x2F848, 0x55AB}, {0x2F849, 0x55B3}, {0x2F84A, 0x55C2}, {0x2F84B, 0x5716}, +{0x2F84C, 0x5606}, {0x2F84D, 0x5717}, {0x2F84E, 0x5651}, {0x2F84F, 0x5674}, {0x2F850, 0x5207}, {0x2F851, 0x58EE}, {0x2F852, 0x57CE}, {0x2F853, 0x57F4}, {0x2F854, 0x580D}, {0x2F855, 0x578B}, +{0x2F856, 0x5832}, {0x2F857, 0x5831}, {0x2F858, 0x58AC}, {0x2F859, 0x214E4}, {0x2F85A, 0x58F2}, {0x2F85B, 0x58F7}, {0x2F85C, 0x5906}, {0x2F85D, 0x591A}, {0x2F85E, 0x5922}, {0x2F85F, 0x5962}, +{0x2F860, 0x216A8}, {0x2F861, 0x216EA}, {0x2F862, 0x59EC}, {0x2F863, 0x5A1B}, {0x2F864, 0x5A27}, {0x2F865, 0x59D8}, {0x2F866, 0x5A66}, {0x2F867, 0x36EE}, {0x2F868, 0x36FC}, {0x2F869, 0x5B08}, +{0x2F86A, 0x5B3E}, {0x2F86B, 0x5B3E}, {0x2F86C, 0x219C8}, {0x2F86D, 0x5BC3}, {0x2F86E, 0x5BD8}, {0x2F86F, 0x5BE7}, {0x2F870, 0x5BF3}, {0x2F871, 0x21B18}, {0x2F872, 0x5BFF}, {0x2F873, 0x5C06}, +{0x2F874, 0x5F53}, {0x2F875, 0x5C22}, {0x2F876, 0x3781}, {0x2F877, 0x5C60}, {0x2F878, 0x5C6E}, {0x2F879, 0x5CC0}, {0x2F87A, 0x5C8D}, {0x2F87B, 0x21DE4}, {0x2F87C, 0x5D43}, {0x2F87D, 0x21DE6}, +{0x2F87E, 0x5D6E}, {0x2F87F, 0x5D6B}, {0x2F880, 0x5D7C}, {0x2F881, 0x5DE1}, {0x2F882, 0x5DE2}, {0x2F883, 0x382F}, {0x2F884, 0x5DFD}, {0x2F885, 0x5E28}, {0x2F886, 0x5E3D}, {0x2F887, 0x5E69}, +{0x2F888, 0x3862}, {0x2F889, 0x22183}, {0x2F88A, 0x387C}, {0x2F88B, 0x5EB0}, {0x2F88C, 0x5EB3}, {0x2F88D, 0x5EB6}, {0x2F88E, 0x5ECA}, {0x2F88F, 0x2A392}, {0x2F890, 0x5EFE}, {0x2F891, 0x22331}, +{0x2F892, 0x22331}, {0x2F893, 0x8201}, {0x2F894, 0x5F22}, {0x2F895, 0x5F22}, {0x2F896, 0x38C7}, {0x2F897, 0x232B8}, {0x2F898, 0x261DA}, {0x2F899, 0x5F62}, {0x2F89A, 0x5F6B}, {0x2F89B, 0x38E3}, +{0x2F89C, 0x5F9A}, {0x2F89D, 0x5FCD}, {0x2F89E, 0x5FD7}, {0x2F89F, 0x5FF9}, {0x2F8A0, 0x6081}, {0x2F8A1, 0x393A}, {0x2F8A2, 0x391C}, {0x2F8A3, 0x6094}, {0x2F8A4, 0x226D4}, {0x2F8A5, 0x60C7}, +{0x2F8A6, 0x6148}, {0x2F8A7, 0x614C}, {0x2F8A8, 0x614E}, {0x2F8A9, 0x614C}, {0x2F8AA, 0x617A}, {0x2F8AB, 0x618E}, {0x2F8AC, 0x61B2}, {0x2F8AD, 0x61A4}, {0x2F8AE, 0x61AF}, {0x2F8AF, 0x61DE}, +{0x2F8B0, 0x61F2}, {0x2F8B1, 0x61F6}, {0x2F8B2, 0x6210}, {0x2F8B3, 0x621B}, {0x2F8B4, 0x625D}, {0x2F8B5, 0x62B1}, {0x2F8B6, 0x62D4}, {0x2F8B7, 0x6350}, {0x2F8B8, 0x22B0C}, {0x2F8B9, 0x633D}, +{0x2F8BA, 0x62FC}, {0x2F8BB, 0x6368}, {0x2F8BC, 0x6383}, {0x2F8BD, 0x63E4}, {0x2F8BE, 0x22BF1}, {0x2F8BF, 0x6422}, {0x2F8C0, 0x63C5}, {0x2F8C1, 0x63A9}, {0x2F8C2, 0x3A2E}, {0x2F8C3, 0x6469}, +{0x2F8C4, 0x647E}, {0x2F8C5, 0x649D}, {0x2F8C6, 0x6477}, {0x2F8C7, 0x3A6C}, {0x2F8C8, 0x654F}, {0x2F8C9, 0x656C}, {0x2F8CA, 0x2300A}, {0x2F8CB, 0x65E3}, {0x2F8CC, 0x66F8}, {0x2F8CD, 0x6649}, +{0x2F8CE, 0x3B19}, {0x2F8CF, 0x6691}, {0x2F8D0, 0x3B08}, {0x2F8D1, 0x3AE4}, {0x2F8D2, 0x5192}, {0x2F8D3, 0x5195}, {0x2F8D4, 0x6700}, {0x2F8D5, 0x669C}, {0x2F8D6, 0x80AD}, {0x2F8D7, 0x43D9}, +{0x2F8D8, 0x6717}, {0x2F8D9, 0x671B}, {0x2F8DA, 0x6721}, {0x2F8DB, 0x675E}, {0x2F8DC, 0x6753}, {0x2F8DD, 0x233C3}, {0x2F8DE, 0x3B49}, {0x2F8DF, 0x67FA}, {0x2F8E0, 0x6785}, {0x2F8E1, 0x6852}, +{0x2F8E2, 0x6885}, {0x2F8E3, 0x2346D}, {0x2F8E4, 0x688E}, {0x2F8E5, 0x681F}, {0x2F8E6, 0x6914}, {0x2F8E7, 0x3B9D}, {0x2F8E8, 0x6942}, {0x2F8E9, 0x69A3}, {0x2F8EA, 0x69EA}, {0x2F8EB, 0x6AA8}, +{0x2F8EC, 0x236A3}, {0x2F8ED, 0x6ADB}, {0x2F8EE, 0x3C18}, {0x2F8EF, 0x6B21}, {0x2F8F0, 0x238A7}, {0x2F8F1, 0x6B54}, {0x2F8F2, 0x3C4E}, {0x2F8F3, 0x6B72}, {0x2F8F4, 0x6B9F}, {0x2F8F5, 0x6BBA}, +{0x2F8F6, 0x6BBB}, {0x2F8F7, 0x23A8D}, {0x2F8F8, 0x21D0B}, {0x2F8F9, 0x23AFA}, {0x2F8FA, 0x6C4E}, {0x2F8FB, 0x23CBC}, {0x2F8FC, 0x6CBF}, {0x2F8FD, 0x6CCD}, {0x2F8FE, 0x6C67}, {0x2F8FF, 0x6D16}, +{0x2F900, 0x6D3E}, {0x2F901, 0x6D77}, {0x2F902, 0x6D41}, {0x2F903, 0x6D69}, {0x2F904, 0x6D78}, {0x2F905, 0x6D85}, {0x2F906, 0x23D1E}, {0x2F907, 0x6D34}, {0x2F908, 0x6E2F}, {0x2F909, 0x6E6E}, +{0x2F90A, 0x3D33}, {0x2F90B, 0x6ECB}, {0x2F90C, 0x6EC7}, {0x2F90D, 0x23ED1}, {0x2F90E, 0x6DF9}, {0x2F90F, 0x6F6E}, {0x2F910, 0x23F5E}, {0x2F911, 0x23F8E}, {0x2F912, 0x6FC6}, {0x2F913, 0x7039}, +{0x2F914, 0x701E}, {0x2F915, 0x701B}, {0x2F916, 0x3D96}, {0x2F917, 0x704A}, {0x2F918, 0x707D}, {0x2F919, 0x7077}, {0x2F91A, 0x70AD}, {0x2F91B, 0x20525}, {0x2F91C, 0x7145}, {0x2F91D, 0x24263}, +{0x2F91E, 0x719C}, {0x2F91F, 0x243AB}, {0x2F920, 0x7228}, {0x2F921, 0x7235}, {0x2F922, 0x7250}, {0x2F923, 0x24608}, {0x2F924, 0x7280}, {0x2F925, 0x7295}, {0x2F926, 0x24735}, {0x2F927, 0x24814}, +{0x2F928, 0x737A}, {0x2F929, 0x738B}, {0x2F92A, 0x3EAC}, {0x2F92B, 0x73A5}, {0x2F92C, 0x3EB8}, {0x2F92D, 0x3EB8}, {0x2F92E, 0x7447}, {0x2F92F, 0x745C}, {0x2F930, 0x7471}, {0x2F931, 0x7485}, +{0x2F932, 0x74CA}, {0x2F933, 0x3F1B}, {0x2F934, 0x7524}, {0x2F935, 0x24C36}, {0x2F936, 0x753E}, {0x2F937, 0x24C92}, {0x2F938, 0x7570}, {0x2F939, 0x2219F}, {0x2F93A, 0x7610}, {0x2F93B, 0x24FA1}, +{0x2F93C, 0x24FB8}, {0x2F93D, 0x25044}, {0x2F93E, 0x3FFC}, {0x2F93F, 0x4008}, {0x2F940, 0x76F4}, {0x2F941, 0x250F3}, {0x2F942, 0x250F2}, {0x2F943, 0x25119}, {0x2F944, 0x25133}, {0x2F945, 0x771E}, +{0x2F946, 0x771F}, {0x2F947, 0x771F}, {0x2F948, 0x774A}, {0x2F949, 0x4039}, {0x2F94A, 0x778B}, {0x2F94B, 0x4046}, {0x2F94C, 0x4096}, {0x2F94D, 0x2541D}, {0x2F94E, 0x784E}, {0x2F94F, 0x788C}, +{0x2F950, 0x78CC}, {0x2F951, 0x40E3}, {0x2F952, 0x25626}, {0x2F953, 0x7956}, {0x2F954, 0x2569A}, {0x2F955, 0x256C5}, {0x2F956, 0x798F}, {0x2F957, 0x79EB}, {0x2F958, 0x412F}, {0x2F959, 0x7A40}, +{0x2F95A, 0x7A4A}, {0x2F95B, 0x7A4F}, {0x2F95C, 0x2597C}, {0x2F95D, 0x25AA7}, {0x2F95E, 0x25AA7}, {0x2F95F, 0x7AEE}, {0x2F960, 0x4202}, {0x2F961, 0x25BAB}, {0x2F962, 0x7BC6}, {0x2F963, 0x7BC9}, +{0x2F964, 0x4227}, {0x2F965, 0x25C80}, {0x2F966, 0x7CD2}, {0x2F967, 0x42A0}, {0x2F968, 0x7CE8}, {0x2F969, 0x7CE3}, {0x2F96A, 0x7D00}, {0x2F96B, 0x25F86}, {0x2F96C, 0x7D63}, {0x2F96D, 0x4301}, +{0x2F96E, 0x7DC7}, {0x2F96F, 0x7E02}, {0x2F970, 0x7E45}, {0x2F971, 0x4334}, {0x2F972, 0x26228}, {0x2F973, 0x26247}, {0x2F974, 0x4359}, {0x2F975, 0x262D9}, {0x2F976, 0x7F7A}, {0x2F977, 0x2633E}, +{0x2F978, 0x7F95}, {0x2F979, 0x7FFA}, {0x2F97A, 0x8005}, {0x2F97B, 0x264DA}, {0x2F97C, 0x26523}, {0x2F97D, 0x8060}, {0x2F97E, 0x265A8}, {0x2F97F, 0x8070}, {0x2F980, 0x2335F}, {0x2F981, 0x43D5}, +{0x2F982, 0x80B2}, {0x2F983, 0x8103}, {0x2F984, 0x440B}, {0x2F985, 0x813E}, {0x2F986, 0x5AB5}, {0x2F987, 0x267A7}, {0x2F988, 0x267B5}, {0x2F989, 0x23393}, {0x2F98A, 0x2339C}, {0x2F98B, 0x8201}, +{0x2F98C, 0x8204}, {0x2F98D, 0x8F9E}, {0x2F98E, 0x446B}, {0x2F98F, 0x8291}, {0x2F990, 0x828B}, {0x2F991, 0x829D}, {0x2F992, 0x52B3}, {0x2F993, 0x82B1}, {0x2F994, 0x82B3}, {0x2F995, 0x82BD}, +{0x2F996, 0x82E6}, {0x2F997, 0x26B3C}, {0x2F998, 0x82E5}, {0x2F999, 0x831D}, {0x2F99A, 0x8363}, {0x2F99B, 0x83AD}, {0x2F99C, 0x8323}, {0x2F99D, 0x83BD}, {0x2F99E, 0x83E7}, {0x2F99F, 0x8457}, +{0x2F9A0, 0x8353}, {0x2F9A1, 0x83CA}, {0x2F9A2, 0x83CC}, {0x2F9A3, 0x83DC}, {0x2F9A4, 0x26C36}, {0x2F9A5, 0x26D6B}, {0x2F9A6, 0x26CD5}, {0x2F9A7, 0x452B}, {0x2F9A8, 0x84F1}, {0x2F9A9, 0x84F3}, +{0x2F9AA, 0x8516}, {0x2F9AB, 0x273CA}, {0x2F9AC, 0x8564}, {0x2F9AD, 0x26F2C}, {0x2F9AE, 0x455D}, {0x2F9AF, 0x4561}, {0x2F9B0, 0x26FB1}, {0x2F9B1, 0x270D2}, {0x2F9B2, 0x456B}, {0x2F9B3, 0x8650}, +{0x2F9B4, 0x865C}, {0x2F9B5, 0x8667}, {0x2F9B6, 0x8669}, {0x2F9B7, 0x86A9}, {0x2F9B8, 0x8688}, {0x2F9B9, 0x870E}, {0x2F9BA, 0x86E2}, {0x2F9BB, 0x8779}, {0x2F9BC, 0x8728}, {0x2F9BD, 0x876B}, +{0x2F9BE, 0x8786}, {0x2F9BF, 0x45D7}, {0x2F9C0, 0x87E1}, {0x2F9C1, 0x8801}, {0x2F9C2, 0x45F9}, {0x2F9C3, 0x8860}, {0x2F9C4, 0x8863}, {0x2F9C5, 0x27667}, {0x2F9C6, 0x88D7}, {0x2F9C7, 0x88DE}, +{0x2F9C8, 0x4635}, {0x2F9C9, 0x88FA}, {0x2F9CA, 0x34BB}, {0x2F9CB, 0x278AE}, {0x2F9CC, 0x27966}, {0x2F9CD, 0x46BE}, {0x2F9CE, 0x46C7}, {0x2F9CF, 0x8AA0}, {0x2F9D0, 0x8AED}, {0x2F9D1, 0x8B8A}, +{0x2F9D2, 0x8C55}, {0x2F9D3, 0x27CA8}, {0x2F9D4, 0x8CAB}, {0x2F9D5, 0x8CC1}, {0x2F9D6, 0x8D1B}, {0x2F9D7, 0x8D77}, {0x2F9D8, 0x27F2F}, {0x2F9D9, 0x20804}, {0x2F9DA, 0x8DCB}, {0x2F9DB, 0x8DBC}, +{0x2F9DC, 0x8DF0}, {0x2F9DD, 0x208DE}, {0x2F9DE, 0x8ED4}, {0x2F9DF, 0x8F38}, {0x2F9E0, 0x285D2}, {0x2F9E1, 0x285ED}, {0x2F9E2, 0x9094}, {0x2F9E3, 0x90F1}, {0x2F9E4, 0x9111}, {0x2F9E5, 0x2872E}, +{0x2F9E6, 0x911B}, {0x2F9E7, 0x9238}, {0x2F9E8, 0x92D7}, {0x2F9E9, 0x92D8}, {0x2F9EA, 0x927C}, {0x2F9EB, 0x93F9}, {0x2F9EC, 0x9415}, {0x2F9ED, 0x28BFA}, {0x2F9EE, 0x958B}, {0x2F9EF, 0x4995}, +{0x2F9F0, 0x95B7}, {0x2F9F1, 0x28D77}, {0x2F9F2, 0x49E6}, {0x2F9F3, 0x96C3}, {0x2F9F4, 0x5DB2}, {0x2F9F5, 0x9723}, {0x2F9F6, 0x29145}, {0x2F9F7, 0x2921A}, {0x2F9F8, 0x4A6E}, {0x2F9F9, 0x4A76}, +{0x2F9FA, 0x97E0}, {0x2F9FB, 0x2940A}, {0x2F9FC, 0x4AB2}, {0x2F9FD, 0x29496}, {0x2F9FE, 0x980B}, {0x2F9FF, 0x980B}, {0x2FA00, 0x9829}, {0x2FA01, 0x295B6}, {0x2FA02, 0x98E2}, {0x2FA03, 0x4B33}, +{0x2FA04, 0x9929}, {0x2FA05, 0x99A7}, {0x2FA06, 0x99C2}, {0x2FA07, 0x99FE}, {0x2FA08, 0x4BCE}, {0x2FA09, 0x29B30}, {0x2FA0A, 0x9B12}, {0x2FA0B, 0x9C40}, {0x2FA0C, 0x9CFD}, {0x2FA0D, 0x4CCE}, +{0x2FA0E, 0x4CED}, {0x2FA0F, 0x9D67}, {0x2FA10, 0x2A0CE}, {0x2FA11, 0x4CF8}, {0x2FA12, 0x2A105}, {0x2FA13, 0x2A20E}, {0x2FA14, 0x2A291}, {0x2FA15, 0x9EBB}, {0x2FA16, 0x4D56}, {0x2FA17, 0x9EF9}, +{0x2FA18, 0x9EFE}, {0x2FA19, 0x9F05}, {0x2FA1A, 0x9F0F}, {0x2FA1B, 0x9F16}, {0x2FA1D, 0x2A600}, }; static std::string codepoint_to_utf8(uint32_t cp) { From 3ab8b3a92ede46df88bc5a2dfca3777de4a2b2b6 Mon Sep 17 00:00:00 2001 From: Pierrick Hymbert Date: Fri, 1 Mar 2024 12:39:06 +0100 Subject: [PATCH 499/812] llama : cleanup unused mmq flags (#5772) * cleanup unused --no-mul-mat-q,-nommq, -mmq, --mul-mat-q, mul_mat_q * remove: mul_mat_q in compare llama bench and usage * update llama-bench --------- Co-authored-by: slaren --- common/common.cpp | 2 -- common/common.h | 1 - examples/batched-bench/batched-bench.cpp | 18 +++++--------- examples/llama-bench/README.md | 1 - examples/llama-bench/llama-bench.cpp | 30 +++--------------------- examples/server/server.cpp | 8 ------- llama.cpp | 3 --- llama.h | 1 - scripts/compare-llama-bench.py | 2 +- 9 files changed, 10 insertions(+), 56 deletions(-) diff --git a/common/common.cpp b/common/common.cpp index 18289755c..bf1ed8a66 100644 --- a/common/common.cpp +++ b/common/common.cpp @@ -1281,7 +1281,6 @@ struct llama_context_params llama_context_params_from_gpt_params(const gpt_param cparams.n_batch = params.n_batch; cparams.n_threads = params.n_threads; cparams.n_threads_batch = params.n_threads_batch == -1 ? params.n_threads : params.n_threads_batch; - cparams.mul_mat_q = params.mul_mat_q; cparams.seed = params.seed; cparams.logits_all = params.logits_all; cparams.embedding = params.embedding; @@ -1725,7 +1724,6 @@ void dump_non_result_info_yaml(FILE * stream, const gpt_params & params, const l fprintf(stream, "n_predict: %d # default: -1 (unlimited)\n", params.n_predict); fprintf(stream, "n_probs: %d # only used by server binary, default: 0\n", sparams.n_probs); fprintf(stream, "no_mmap: %s # default: false\n", !params.use_mmap ? "true" : "false"); - fprintf(stream, "no_mul_mat_q: %s # default: false\n", !params.mul_mat_q ? "true" : "false"); fprintf(stream, "no_penalize_nl: %s # default: false\n", !sparams.penalize_nl ? "true" : "false"); fprintf(stream, "ppl_output_type: %d # default: 0\n", params.ppl_output_type); fprintf(stream, "ppl_stride: %d # default: 0\n", params.ppl_stride); diff --git a/common/common.h b/common/common.h index 25003df26..ab62bdb82 100644 --- a/common/common.h +++ b/common/common.h @@ -115,7 +115,6 @@ struct gpt_params { bool kl_divergence = false; // compute KL-divergence - bool mul_mat_q = true; // if true, use mul_mat_q kernels instead of cuBLAS bool random_prompt = false; // do not randomize prompt if none provided bool use_color = false; // use color to distinguish generations and inputs bool interactive = false; // interactive mode diff --git a/examples/batched-bench/batched-bench.cpp b/examples/batched-bench/batched-bench.cpp index b4b8a38e1..19aff18ae 100644 --- a/examples/batched-bench/batched-bench.cpp +++ b/examples/batched-bench/batched-bench.cpp @@ -32,16 +32,15 @@ int main(int argc, char ** argv) { gpt_params params; if (argc == 1 || argv[1][0] == '-') { - printf("usage: %s MODEL_PATH [N_KV_MAX] [IS_PP_SHARED] [NGL] [MMQ] \n" , argv[0]); + printf("usage: %s MODEL_PATH [N_KV_MAX] [IS_PP_SHARED] [NGL] \n" , argv[0]); printf(" , and PL are comma-separated lists of numbers without spaces\n\n"); - printf(" example: %s ggml-model-f16.gguf 2048 0 999 0 128,256,512 128,256 1,2,4,8,16,32\n\n", argv[0]); + printf(" example: %s ggml-model-f16.gguf 2048 0 999 128,256,512 128,256 1,2,4,8,16,32\n\n", argv[0]); return 1 ; } int n_kv_max = 2048; int is_pp_shared = 0; int n_gpu_layers = 0; - int mmq = 0; std::vector n_pp = { 128, 256, 512, 1024, 2048, 3584, 7680, }; std::vector n_tg = { 128, 256, }; @@ -65,19 +64,15 @@ int main(int argc, char ** argv) { } if (argc >= 6) { - mmq = std::atoi(argv[5]); + n_pp = parse_list(argv[5]); } if (argc >= 7) { - n_pp = parse_list(argv[6]); + n_tg = parse_list(argv[6]); } if (argc >= 8) { - n_tg = parse_list(argv[7]); - } - - if (argc >= 9) { - n_pl = parse_list(argv[8]); + n_pl = parse_list(argv[7]); } // init LLM @@ -106,7 +101,6 @@ int main(int argc, char ** argv) { ctx_params.seed = 1234; ctx_params.n_ctx = n_kv_max; ctx_params.n_batch = 512; - ctx_params.mul_mat_q = mmq; ctx_params.n_threads = params.n_threads; ctx_params.n_threads_batch = params.n_threads_batch == -1 ? params.n_threads : params.n_threads_batch; @@ -159,7 +153,7 @@ int main(int argc, char ** argv) { } LOG_TEE("\n"); - LOG_TEE("%s: n_kv_max = %d, is_pp_shared = %d, n_gpu_layers = %d, mmq = %d, n_threads = %u, n_threads_batch = %u\n", __func__, n_kv_max, is_pp_shared, n_gpu_layers, mmq, ctx_params.n_threads, ctx_params.n_threads_batch); + LOG_TEE("%s: n_kv_max = %d, is_pp_shared = %d, n_gpu_layers = %d, n_threads = %u, n_threads_batch = %u\n", __func__, n_kv_max, is_pp_shared, n_gpu_layers, ctx_params.n_threads, ctx_params.n_threads_batch); LOG_TEE("\n"); LOG_TEE("|%6s | %6s | %4s | %6s | %8s | %8s | %8s | %8s | %8s | %8s |\n", "PP", "TG", "B", "N_KV", "T_PP s", "S_PP t/s", "T_TG s", "S_TG t/s", "T s", "S t/s"); diff --git a/examples/llama-bench/README.md b/examples/llama-bench/README.md index 374e40a7d..10f37b441 100644 --- a/examples/llama-bench/README.md +++ b/examples/llama-bench/README.md @@ -35,7 +35,6 @@ options: -mg, --main-gpu (default: 0) -nkvo, --no-kv-offload <0|1> (default: 0) -mmp, --mmap <0|1> (default: 1) - -mmq, --mul-mat-q <0|1> (default: 1) -ts, --tensor_split (default: 0) -r, --repetitions (default: 5) -o, --output (default: md) diff --git a/examples/llama-bench/llama-bench.cpp b/examples/llama-bench/llama-bench.cpp index 8fec3d43d..c2155b2ac 100644 --- a/examples/llama-bench/llama-bench.cpp +++ b/examples/llama-bench/llama-bench.cpp @@ -176,7 +176,6 @@ struct cmd_params { std::vector split_mode; std::vector main_gpu; std::vector no_kv_offload; - std::vector mul_mat_q; std::vector> tensor_split; std::vector use_mmap; int reps; @@ -196,7 +195,6 @@ static const cmd_params cmd_params_defaults = { /* split_mode */ {LLAMA_SPLIT_MODE_LAYER}, /* main_gpu */ {0}, /* no_kv_offload */ {false}, - /* mul_mat_q */ {true}, /* tensor_split */ {std::vector(llama_max_devices(), 0.0f)}, /* use_mmap */ {true}, /* reps */ 5, @@ -221,7 +219,6 @@ static void print_usage(int /* argc */, char ** argv) { printf(" -mg, --main-gpu (default: %s)\n", join(cmd_params_defaults.main_gpu, ",").c_str()); printf(" -nkvo, --no-kv-offload <0|1> (default: %s)\n", join(cmd_params_defaults.no_kv_offload, ",").c_str()); printf(" -mmp, --mmap <0|1> (default: %s)\n", join(cmd_params_defaults.use_mmap, ",").c_str()); - printf(" -mmq, --mul-mat-q <0|1> (default: %s)\n", join(cmd_params_defaults.mul_mat_q, ",").c_str()); printf(" -ts, --tensor_split (default: 0)\n"); printf(" -r, --repetitions (default: %d)\n", cmd_params_defaults.reps); printf(" -o, --output (default: %s)\n", output_format_str(cmd_params_defaults.output_format)); @@ -383,13 +380,6 @@ static cmd_params parse_cmd_params(int argc, char ** argv) { } auto p = split(argv[i], split_delim); params.no_kv_offload.insert(params.no_kv_offload.end(), p.begin(), p.end()); - } else if (arg == "-mmq" || arg == "--mul-mat-q") { - if (++i >= argc) { - invalid_param = true; - break; - } - auto p = split(argv[i], split_delim); - params.mul_mat_q.insert(params.mul_mat_q.end(), p.begin(), p.end()); } else if (arg == "-mmp" || arg == "--mmap") { if (++i >= argc) { invalid_param = true; @@ -466,7 +456,6 @@ static cmd_params parse_cmd_params(int argc, char ** argv) { if (params.split_mode.empty()) { params.split_mode = cmd_params_defaults.split_mode; } if (params.main_gpu.empty()) { params.main_gpu = cmd_params_defaults.main_gpu; } if (params.no_kv_offload.empty()){ params.no_kv_offload = cmd_params_defaults.no_kv_offload; } - if (params.mul_mat_q.empty()) { params.mul_mat_q = cmd_params_defaults.mul_mat_q; } if (params.tensor_split.empty()) { params.tensor_split = cmd_params_defaults.tensor_split; } if (params.use_mmap.empty()) { params.use_mmap = cmd_params_defaults.use_mmap; } if (params.n_threads.empty()) { params.n_threads = cmd_params_defaults.n_threads; } @@ -486,7 +475,6 @@ struct cmd_params_instance { llama_split_mode split_mode; int main_gpu; bool no_kv_offload; - bool mul_mat_q; std::vector tensor_split; bool use_mmap; @@ -518,7 +506,6 @@ struct cmd_params_instance { cparams.n_batch = n_batch; cparams.type_k = type_k; cparams.type_v = type_v; - cparams.mul_mat_q = mul_mat_q; cparams.offload_kqv = !no_kv_offload; return cparams; @@ -538,7 +525,6 @@ static std::vector get_cmd_params_instances(const cmd_param for (const auto & nb : params.n_batch) for (const auto & tk : params.type_k) for (const auto & tv : params.type_v) - for (const auto & mmq : params.mul_mat_q) for (const auto & nkvo : params.no_kv_offload) for (const auto & nt : params.n_threads) { for (const auto & n_prompt : params.n_prompt) { @@ -557,7 +543,6 @@ static std::vector get_cmd_params_instances(const cmd_param /* .split_mode = */ sm, /* .main_gpu = */ mg, /* .no_kv_offload= */ nkvo, - /* .mul_mat_q = */ mmq, /* .tensor_split = */ ts, /* .use_mmap = */ mmp, }; @@ -580,7 +565,6 @@ static std::vector get_cmd_params_instances(const cmd_param /* .split_mode = */ sm, /* .main_gpu = */ mg, /* .no_kv_offload= */ nkvo, - /* .mul_mat_q = */ mmq, /* .tensor_split = */ ts, /* .use_mmap = */ mmp, }; @@ -616,7 +600,6 @@ struct test { llama_split_mode split_mode; int main_gpu; bool no_kv_offload; - bool mul_mat_q; std::vector tensor_split; bool use_mmap; int n_prompt; @@ -639,7 +622,6 @@ struct test { split_mode = inst.split_mode; main_gpu = inst.main_gpu; no_kv_offload = inst.no_kv_offload; - mul_mat_q = inst.mul_mat_q; tensor_split = inst.tensor_split; use_mmap = inst.use_mmap; n_prompt = inst.n_prompt; @@ -713,7 +695,7 @@ struct test { "n_batch", "n_threads", "type_k", "type_v", "n_gpu_layers", "split_mode", "main_gpu", "no_kv_offload", - "mul_mat_q", "tensor_split", "use_mmap", + "tensor_split", "use_mmap", "n_prompt", "n_gen", "test_time", "avg_ns", "stddev_ns", "avg_ts", "stddev_ts" @@ -733,7 +715,7 @@ struct test { } if (field == "cuda" || field == "opencl" || field == "vulkan" || field == "kompute" || field == "metal" || field == "gpu_blas" || field == "blas" || field == "sycl" ||field == "f16_kv" || field == "no_kv_offload" || - field == "mul_mat_q" || field == "use_mmap") { + field == "use_mmap") { return BOOL; } if (field == "avg_ts" || field == "stddev_ts") { @@ -767,7 +749,7 @@ struct test { std::to_string(n_batch), std::to_string(n_threads), ggml_type_name(type_k), ggml_type_name(type_v), std::to_string(n_gpu_layers), split_mode_str(split_mode), std::to_string(main_gpu), std::to_string(no_kv_offload), - std::to_string(mul_mat_q), tensor_split_str, std::to_string(use_mmap), + tensor_split_str, std::to_string(use_mmap), std::to_string(n_prompt), std::to_string(n_gen), test_time, std::to_string(avg_ns()), std::to_string(stdev_ns()), std::to_string(avg_ts()), std::to_string(stdev_ts()) @@ -931,9 +913,6 @@ struct markdown_printer : public printer { if (field == "n_threads") { return "threads"; } - if (field == "mul_mat_q") { - return "mmq"; - } if (field == "no_kv_offload") { return "nkvo"; } @@ -974,9 +953,6 @@ struct markdown_printer : public printer { if (params.split_mode.size() > 1 || params.split_mode != cmd_params_defaults.split_mode) { fields.emplace_back("split_mode"); } - if (params.mul_mat_q.size() > 1 || params.mul_mat_q != cmd_params_defaults.mul_mat_q) { - fields.emplace_back("mul_mat_q"); - } if (params.no_kv_offload.size() > 1 || params.no_kv_offload != cmd_params_defaults.no_kv_offload) { fields.emplace_back("no_kv_offload"); } diff --git a/examples/server/server.cpp b/examples/server/server.cpp index eea987966..2b2f4a0f4 100644 --- a/examples/server/server.cpp +++ b/examples/server/server.cpp @@ -2390,14 +2390,6 @@ static void server_params_parse(int argc, char **argv, server_params &sparams, } #else LOG_WARNING("llama.cpp was compiled without cuBLAS. It is not possible to set a tensor split.\n", {}); -#endif // GGML_USE_CUBLAS - } - else if (arg == "--no-mul-mat-q" || arg == "-nommq") - { -#if defined(GGML_USE_CUBLAS) || defined(GGML_USE_SYCL) - params.mul_mat_q = false; -#else - LOG_WARNING("warning: llama.cpp was compiled without cuBLAS. Disabling mul_mat_q kernels has no effect.\n", {}); #endif // GGML_USE_CUBLAS } else if (arg == "--main-gpu" || arg == "-mg") diff --git a/llama.cpp b/llama.cpp index a35f07aa4..073fd3b70 100644 --- a/llama.cpp +++ b/llama.cpp @@ -1645,7 +1645,6 @@ struct llama_cparams { float yarn_beta_slow; float defrag_thold; - bool mul_mat_q; bool offload_kqv; bool do_pooling; @@ -11633,7 +11632,6 @@ struct llama_context_params llama_context_default_params() { /*.cb_eval_user_data =*/ nullptr, /*.type_k =*/ GGML_TYPE_F16, /*.type_v =*/ GGML_TYPE_F16, - /*.mul_mat_q =*/ true, /*.logits_all =*/ false, /*.embedding =*/ false, /*.offload_kqv =*/ true, @@ -11785,7 +11783,6 @@ struct llama_context * llama_new_context_with_model( cparams.yarn_beta_fast = params.yarn_beta_fast; cparams.yarn_beta_slow = params.yarn_beta_slow; cparams.defrag_thold = params.defrag_thold; - cparams.mul_mat_q = params.mul_mat_q; cparams.offload_kqv = params.offload_kqv; cparams.do_pooling = params.do_pooling; diff --git a/llama.h b/llama.h index 4d0ebe37d..ed51f478a 100644 --- a/llama.h +++ b/llama.h @@ -255,7 +255,6 @@ extern "C" { enum ggml_type type_v; // data type for V cache // Keep the booleans together to avoid misalignment during copy-by-value. - bool mul_mat_q; // if true, use experimental mul_mat_q kernels (DEPRECATED - always true) bool logits_all; // the llama_eval() call computes all logits, not just the last one (DEPRECATED - set llama_batch.logits instead) bool embedding; // embedding mode only bool offload_kqv; // whether to offload the KQV ops (including the KV cache) to GPU diff --git a/scripts/compare-llama-bench.py b/scripts/compare-llama-bench.py index 70737f976..39c3e52e5 100755 --- a/scripts/compare-llama-bench.py +++ b/scripts/compare-llama-bench.py @@ -31,7 +31,7 @@ PRETTY_NAMES = { "model_size": "Model Size [GiB]", "model_n_params": "Num. of Parameters", "n_batch": "Batch size", "n_threads": "Threads", "type_k": "K type", "type_v": "V type", "n_gpu_layers": "GPU layers", "main_gpu": "Main GPU", "no_kv_offload": "NKVO", - "mul_mat_q": "MMQ", "tensor_split": "Tensor split" + "tensor_split": "Tensor split" } DEFAULT_SHOW = ["model_type"] # Always show these properties by default. From f49a5356865ced0eca1df9f9d84631dfef71b9dc Mon Sep 17 00:00:00 2001 From: Miwa / Ensan <63481257+ensan-hcl@users.noreply.github.com> Date: Fri, 1 Mar 2024 22:48:56 +0900 Subject: [PATCH 500/812] common : fix flag `--logits-all` to `--all-logits` (#5805) --- common/common.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/common.cpp b/common/common.cpp index bf1ed8a66..938c428cf 100644 --- a/common/common.cpp +++ b/common/common.cpp @@ -1015,7 +1015,7 @@ void gpt_print_usage(int /*argc*/, char ** argv, const gpt_params & params) { printf(" --ignore-eos ignore end of stream token and continue generating (implies --logit-bias 2-inf)\n"); printf(" --no-penalize-nl do not penalize newline token\n"); printf(" --temp N temperature (default: %.1f)\n", (double)sparams.temp); - printf(" --logits-all return logits for all tokens in the batch (default: disabled)\n"); + printf(" --all-logits return logits for all tokens in the batch (default: disabled)\n"); printf(" --hellaswag compute HellaSwag score over random tasks from datafile supplied with -f\n"); printf(" --hellaswag-tasks N number of tasks to use when computing the HellaSwag score (default: %zu)\n", params.hellaswag_tasks); printf(" --winogrande compute Winogrande score over random tasks from datafile supplied with -f\n"); From e7433867288d2f142cffe596f3751bda5d7ee2c7 Mon Sep 17 00:00:00 2001 From: kunal-vaishnavi <115581922+kunal-vaishnavi@users.noreply.github.com> Date: Fri, 1 Mar 2024 06:08:08 -0800 Subject: [PATCH 501/812] gemma : fix bfloat16 -> float16 conversion issue (#5810) --- convert-hf-to-gguf.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/convert-hf-to-gguf.py b/convert-hf-to-gguf.py index ae30b2a76..d3e8ec1f6 100755 --- a/convert-hf-to-gguf.py +++ b/convert-hf-to-gguf.py @@ -1811,16 +1811,15 @@ class GemmaModel(Model): tensor_map = gguf.get_tensor_name_map(self.model_arch, block_count) for name, data_torch in self.get_tensors(): - # ref: https://github.com/huggingface/transformers/blob/fc37f38915372c15992b540dfcbbe00a916d4fc6/src/transformers/models/gemma/modeling_gemma.py#L89 - if name.endswith("norm.weight"): - data_torch = data_torch + 1 - old_dtype = data_torch.dtype # convert any unsupported data types to float32 if data_torch.dtype not in (torch.float16, torch.float32): data_torch = data_torch.to(torch.float32) + # ref: https://github.com/huggingface/transformers/blob/fc37f38915372c15992b540dfcbbe00a916d4fc6/src/transformers/models/gemma/modeling_gemma.py#L89 + if name.endswith("norm.weight"): + data_torch = data_torch + 1 data = data_torch.squeeze().numpy() # map tensor names From c2224f003bf9cf558b1a3c57033563e11a4de9a5 Mon Sep 17 00:00:00 2001 From: ddpasa <112642920+ddpasa@users.noreply.github.com> Date: Fri, 1 Mar 2024 18:00:00 +0100 Subject: [PATCH 502/812] ggml-vulkan: fix VULKAN_CHECK_RESULTS flag, which was previously broken (#5813) --- ggml-vulkan.cpp | 34 ++++++++++++++++++---------------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/ggml-vulkan.cpp b/ggml-vulkan.cpp index 896c290b2..ae9cb3c1c 100644 --- a/ggml-vulkan.cpp +++ b/ggml-vulkan.cpp @@ -5428,7 +5428,8 @@ static void ggml_vk_print_tensor(ggml_backend_vk_context * ctx, const ggml_tenso ggml_tensor_extra_gpu * extra = (ggml_tensor_extra_gpu *) tensor->extra; - ggml_vk_buffer_read(ctx, extra->buffer_gpu, extra->offset, tensor_data, tensor_size); + vk_buffer buffer_gpu = extra->buffer_gpu.lock(); + ggml_vk_buffer_read(ctx, buffer_gpu, extra->offset, tensor_data, tensor_size); } std::cerr << "TENSOR CHECK " << name << " (" << tensor->name << "): " << ggml_op_name(tensor->op) << std::endl; @@ -5540,7 +5541,8 @@ static void ggml_vk_check_results_0(ggml_backend_vk_context * ctx, ggml_compute_ for (int i3 = 0; i3 < src0->ne[3]; i3++) { for (int i2 = 0; i2 < src0->ne[2]; i2++) { const int idx = i3*src0->ne[2] + i2; - ggml_vk_buffer_read(ctx, extra->buffer_gpu, offset + idx * src0->nb[2], ((char *)src0_clone->data + idx * src0_clone->nb[2]), src0->ne[1] * src0->nb[1]); + vk_buffer buffer_gpu = extra->buffer_gpu.lock(); + ggml_vk_buffer_read(ctx, buffer_gpu, offset + idx * src0->nb[2], ((char *)src0_clone->data + idx * src0_clone->nb[2]), src0->ne[1] * src0->nb[1]); } } @@ -5550,10 +5552,11 @@ static void ggml_vk_check_results_0(ggml_backend_vk_context * ctx, ggml_compute_ src0_clone->nb[i] = src0_clone->nb[i - 1]*src0_clone->ne[i - 1]; } } else { - if (offset + src0_size >= extra->buffer_gpu->size) { - src0_size = extra->buffer_gpu->size - offset; + vk_buffer buffer_gpu = extra->buffer_gpu.lock(); + if (offset + src0_size >= buffer_gpu->size) { + src0_size = buffer_gpu->size - offset; } - ggml_vk_buffer_read(ctx, extra->buffer_gpu, offset, src0_clone->data, src0_size); + ggml_vk_buffer_read(ctx, buffer_gpu, offset, src0_clone->data, src0_size); memcpy(src0_clone->nb, src0->nb, sizeof(size_t) * GGML_MAX_DIMS); } } else { @@ -5583,7 +5586,8 @@ static void ggml_vk_check_results_0(ggml_backend_vk_context * ctx, ggml_compute_ for (int i3 = 0; i3 < src1->ne[3]; i3++) { for (int i2 = 0; i2 < src1->ne[2]; i2++) { const int idx = i3*src1->ne[2] + i2; - ggml_vk_buffer_read(ctx, extra->buffer_gpu, offset + idx * src1->nb[2], ((char *)src1_clone->data + idx * src1_clone->nb[2]), src1->ne[1] * src1->nb[1]); + vk_buffer buffer_gpu = extra->buffer_gpu.lock(); + ggml_vk_buffer_read(ctx, buffer_gpu, offset + idx * src1->nb[2], ((char *)src1_clone->data + idx * src1_clone->nb[2]), src1->ne[1] * src1->nb[1]); } } @@ -5593,10 +5597,11 @@ static void ggml_vk_check_results_0(ggml_backend_vk_context * ctx, ggml_compute_ src1_clone->nb[i] = src1_clone->nb[i - 1]*src1_clone->ne[i - 1]; } } else { - if (offset + src1_size >= extra->buffer_gpu->size) { - src1_size = extra->buffer_gpu->size - offset; + vk_buffer buffer_gpu = extra->buffer_gpu.lock(); + if (offset + src1_size >= buffer_gpu->size) { + src1_size = buffer_gpu->size - offset; } - ggml_vk_buffer_read(ctx, extra->buffer_gpu, offset, src1_clone->data, src1_size); + ggml_vk_buffer_read(ctx, buffer_gpu, offset, src1_clone->data, src1_size); memcpy(src1_clone->nb, src1->nb, sizeof(size_t) * GGML_MAX_DIMS); } } else { @@ -5643,11 +5648,7 @@ static void ggml_vk_check_results_0(ggml_backend_vk_context * ctx, ggml_compute_ } else if (tensor->op == GGML_OP_RMS_NORM) { tensor_clone = ggml_rms_norm(ggml_ctx, src0_clone, *(float *)tensor->op_params); } else if (tensor->op == GGML_OP_SOFT_MAX) { - if (src1 != nullptr) { - tensor_clone = ggml_soft_max_ext(ggml_ctx, src0_clone, src1_clone, *(float *)tensor->op_params); - } else { tensor_clone = ggml_soft_max(ggml_ctx, src0_clone); - } } else if (tensor->op == GGML_OP_DIAG_MASK_INF) { tensor_clone = ggml_diag_mask_inf(ggml_ctx, src0_clone, *(float *)tensor->op_params); } else if (tensor->op == GGML_OP_ROPE) { @@ -5753,11 +5754,12 @@ static void ggml_vk_check_results_1(ggml_backend_vk_context * ctx, ggml_compute_ ggml_tensor_extra_gpu * extra = (ggml_tensor_extra_gpu *) tensor->extra; - if (extra->offset + tensor_size >= extra->buffer_gpu->size) { - tensor_size = extra->buffer_gpu->size - (extra->offset); + vk_buffer buffer_gpu = extra->buffer_gpu.lock(); + if (extra->offset + tensor_size >= buffer_gpu->size) { + tensor_size = buffer_gpu->size - (extra->offset); } - ggml_vk_buffer_read(ctx, extra->buffer_gpu, extra->offset, tensor_data, tensor_size); + ggml_vk_buffer_read(ctx, buffer_gpu, extra->offset, tensor_data, tensor_size); } float first_error_result = -1.0f; From 38d16b142624bdd7c41d9955752b7f7b59c5e048 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Fri, 1 Mar 2024 20:00:58 +0200 Subject: [PATCH 503/812] server : remove api_like_OAI.py proxy script (#5808) --- README.md | 1 + examples/server/README.md | 17 +-- examples/server/api_like_OAI.py | 228 -------------------------------- 3 files changed, 3 insertions(+), 243 deletions(-) delete mode 100755 examples/server/api_like_OAI.py diff --git a/README.md b/README.md index 5401e197f..67717c1e3 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,7 @@ Inference of Meta's [LLaMA](https://arxiv.org/abs/2302.13971) model (and others) ### Hot topics +- The `api_like_OAI.py` script has been removed - use `server` instead ([#5766](https://github.com/ggerganov/llama.cpp/issues/5766#issuecomment-1969037761)) - Support for chat templates: [Wiki (contributions welcome)](https://github.com/ggerganov/llama.cpp/wiki/Templates-supported-by-llama_chat_apply_template) - Support for Gemma models: https://github.com/ggerganov/llama.cpp/pull/5631 - Non-linear quantization IQ4_NL: https://github.com/ggerganov/llama.cpp/pull/5590 diff --git a/examples/server/README.md b/examples/server/README.md index ad35306c6..397ee8252 100644 --- a/examples/server/README.md +++ b/examples/server/README.md @@ -326,7 +326,7 @@ Notice that each `probs` is an array of length `n_probs`. - `default_generation_settings` - the default generation settings for the `/completion` endpoint, has the same fields as the `generation_settings` response object from the `/completion` endpoint. - `total_slots` - the total number of slots for process requests (defined by `--parallel` option) -- **POST** `/v1/chat/completions`: OpenAI-compatible Chat Completions API. Given a ChatML-formatted json description in `messages`, it returns the predicted completion. Both synchronous and streaming mode are supported, so scripted and interactive applications work fine. While no strong claims of compatibility with OpenAI API spec is being made, in our experience it suffices to support many apps. Only ChatML-tuned models, such as Dolphin, OpenOrca, OpenHermes, OpenChat-3.5, etc can be used with this endpoint. Compared to `api_like_OAI.py` this API implementation does not require a wrapper to be served. +- **POST** `/v1/chat/completions`: OpenAI-compatible Chat Completions API. Given a ChatML-formatted json description in `messages`, it returns the predicted completion. Both synchronous and streaming mode are supported, so scripted and interactive applications work fine. While no strong claims of compatibility with OpenAI API spec is being made, in our experience it suffices to support many apps. Only ChatML-tuned models, such as Dolphin, OpenOrca, OpenHermes, OpenChat-3.5, etc can be used with this endpoint. *Options:* @@ -528,20 +528,7 @@ bash chat.sh ### API like OAI -API example using Python Flask: [api_like_OAI.py](api_like_OAI.py) -This example must be used with server.cpp - -```sh -python api_like_OAI.py -``` - -After running the API server, you can use it in Python by setting the API base URL. - -```python -openai.api_base = "http://:port" -``` - -Then you can utilize llama.cpp as an OpenAI's **chat.completion** or **text_completion** API +The HTTP server supports OAI-like API ### Extending or building alternative Web Front End diff --git a/examples/server/api_like_OAI.py b/examples/server/api_like_OAI.py deleted file mode 100755 index 607fe49d3..000000000 --- a/examples/server/api_like_OAI.py +++ /dev/null @@ -1,228 +0,0 @@ -#!/usr/bin/env python3 -import argparse -from flask import Flask, jsonify, request, Response -import urllib.parse -import requests -import time -import json - - -app = Flask(__name__) -slot_id = -1 - -parser = argparse.ArgumentParser(description="An example of using server.cpp with a similar API to OAI. It must be used together with server.cpp.") -parser.add_argument("--chat-prompt", type=str, help="the top prompt in chat completions(default: 'A chat between a curious user and an artificial intelligence assistant. The assistant follows the given rules no matter what.')", default='A chat between a curious user and an artificial intelligence assistant. The assistant follows the given rules no matter what.') -parser.add_argument("--user-name", type=str, help="USER name in chat completions(default: 'USER: ')", default="USER: ") -parser.add_argument("--ai-name", type=str, help="ASSISTANT name in chat completions(default: 'ASSISTANT: ')", default="ASSISTANT: ") -parser.add_argument("--system-name", type=str, help="SYSTEM name in chat completions(default: 'ASSISTANT's RULE: ')", default="ASSISTANT's RULE: ") -parser.add_argument("--stop", type=str, help="the end of response in chat completions(default: '')", default="") -parser.add_argument("--llama-api", type=str, help="Set the address of server.cpp in llama.cpp(default: http://127.0.0.1:8080)", default='http://127.0.0.1:8080') -parser.add_argument("--api-key", type=str, help="Set the api key to allow only few user(default: NULL)", default="") -parser.add_argument("--host", type=str, help="Set the ip address to listen.(default: 127.0.0.1)", default='127.0.0.1') -parser.add_argument("--port", type=int, help="Set the port to listen.(default: 8081)", default=8081) - -args = parser.parse_args() - -def is_present(json, key): - try: - buf = json[key] - except KeyError: - return False - if json[key] == None: - return False - return True - -#convert chat to prompt -def convert_chat(messages): - - system_n = args.system_name - user_n = args.user_name - ai_n = args.ai_name - stop = args.stop - - prompt = "" + args.chat_prompt + stop - - for line in messages: - if (line["role"] == "system"): - prompt += f"{system_n}{line['content']}{stop}" - if (line["role"] == "user"): - prompt += f"{user_n}{line['content']}{stop}" - if (line["role"] == "assistant"): - prompt += f"{ai_n}{line['content']}{stop}" - prompt += ai_n.rstrip() - - return prompt - -def make_postData(body, chat=False, stream=False): - postData = {} - if (chat): - postData["prompt"] = convert_chat(body["messages"]) - else: - postData["prompt"] = body["prompt"] - if(is_present(body, "temperature")): postData["temperature"] = body["temperature"] - if(is_present(body, "top_k")): postData["top_k"] = body["top_k"] - if(is_present(body, "top_p")): postData["top_p"] = body["top_p"] - if(is_present(body, "max_tokens")): postData["n_predict"] = body["max_tokens"] - if(is_present(body, "presence_penalty")): postData["presence_penalty"] = body["presence_penalty"] - if(is_present(body, "frequency_penalty")): postData["frequency_penalty"] = body["frequency_penalty"] - if(is_present(body, "repeat_penalty")): postData["repeat_penalty"] = body["repeat_penalty"] - if(is_present(body, "mirostat")): postData["mirostat"] = body["mirostat"] - if(is_present(body, "mirostat_tau")): postData["mirostat_tau"] = body["mirostat_tau"] - if(is_present(body, "mirostat_eta")): postData["mirostat_eta"] = body["mirostat_eta"] - if(is_present(body, "seed")): postData["seed"] = body["seed"] - if(is_present(body, "grammar")): postData["grammar"] = body["grammar"] - if(is_present(body, "logit_bias")): postData["logit_bias"] = [[int(token), body["logit_bias"][token]] for token in body["logit_bias"].keys()] - if (args.stop != ""): - postData["stop"] = [args.stop] - else: - postData["stop"] = [] - if(is_present(body, "stop")): postData["stop"] += body["stop"] - postData["n_keep"] = -1 - postData["stream"] = stream - postData["cache_prompt"] = True - postData["slot_id"] = slot_id - return postData - -def make_resData(data, chat=False, promptToken=[]): - resData = { - "id": "chatcmpl" if (chat) else "cmpl", - "object": "chat.completion" if (chat) else "text_completion", - "created": int(time.time()), - "truncated": data["truncated"], - "model": "LLaMA_CPP", - "usage": { - "prompt_tokens": data["tokens_evaluated"], - "completion_tokens": data["tokens_predicted"], - "total_tokens": data["tokens_evaluated"] + data["tokens_predicted"] - } - } - if (len(promptToken) != 0): - resData["promptToken"] = promptToken - if (chat): - #only one choice is supported - resData["choices"] = [{ - "index": 0, - "message": { - "role": "assistant", - "content": data["content"], - }, - "finish_reason": "stop" if (data["stopped_eos"] or data["stopped_word"]) else "length" - }] - else: - #only one choice is supported - resData["choices"] = [{ - "text": data["content"], - "index": 0, - "logprobs": None, - "finish_reason": "stop" if (data["stopped_eos"] or data["stopped_word"]) else "length" - }] - return resData - -def make_resData_stream(data, chat=False, time_now = 0, start=False): - resData = { - "id": "chatcmpl" if (chat) else "cmpl", - "object": "chat.completion.chunk" if (chat) else "text_completion.chunk", - "created": time_now, - "model": "LLaMA_CPP", - "choices": [ - { - "finish_reason": None, - "index": 0 - } - ] - } - slot_id = data.get("slot_id") - if (chat): - if (start): - resData["choices"][0]["delta"] = { - "role": "assistant" - } - else: - resData["choices"][0]["delta"] = { - "content": data["content"] - } - if (data["stop"]): - resData["choices"][0]["finish_reason"] = "stop" if (data["stopped_eos"] or data["stopped_word"]) else "length" - else: - resData["choices"][0]["text"] = data["content"] - if (data["stop"]): - resData["choices"][0]["finish_reason"] = "stop" if (data["stopped_eos"] or data["stopped_word"]) else "length" - - return resData - - -@app.route('/chat/completions', methods=['POST', 'OPTIONS']) -@app.route('/v1/chat/completions', methods=['POST', 'OPTIONS']) -def chat_completions(): - if (args.api_key != "" and request.headers["Authorization"].split()[1] != args.api_key): - return Response(status=403) - if request.method == 'OPTIONS': - return Response(headers={"Access-Control-Allow-Origin": "*", "Access-Control-Allow-Headers": "*"}) - body = request.get_json() - stream = False - tokenize = False - if(is_present(body, "stream")): stream = body["stream"] - if(is_present(body, "tokenize")): tokenize = body["tokenize"] - postData = make_postData(body, chat=True, stream=stream) - - promptToken = [] - if (tokenize): - tokenData = requests.request("POST", urllib.parse.urljoin(args.llama_api, "/tokenize"), data=json.dumps({"content": postData["prompt"]})).json() - promptToken = tokenData["tokens"] - - if (not stream): - data = requests.request("POST", urllib.parse.urljoin(args.llama_api, "/completion"), data=json.dumps(postData)) - print(data.json()) - resData = make_resData(data.json(), chat=True, promptToken=promptToken) - return jsonify(resData) - else: - def generate(): - data = requests.request("POST", urllib.parse.urljoin(args.llama_api, "/completion"), data=json.dumps(postData), stream=True) - time_now = int(time.time()) - resData = make_resData_stream({}, chat=True, time_now=time_now, start=True) - yield 'data: {}\n\n'.format(json.dumps(resData)) - for line in data.iter_lines(): - if line: - decoded_line = line.decode('utf-8') - resData = make_resData_stream(json.loads(decoded_line[6:]), chat=True, time_now=time_now) - yield 'data: {}\n\n'.format(json.dumps(resData)) - return Response(generate(), mimetype='text/event-stream', headers={"Access-Control-Allow-Origin": "*", "Access-Control-Allow-Headers": "*"}) - - -@app.route('/completions', methods=['POST', 'OPTIONS']) -@app.route('/v1/completions', methods=['POST', 'OPTIONS']) -def completion(): - if (args.api_key != "" and request.headers["Authorization"].split()[1] != args.api_key): - return Response(status=403) - if request.method == 'OPTIONS': - return Response(headers={"Access-Control-Allow-Origin": "*", "Access-Control-Allow-Headers": "*"}) - body = request.get_json() - stream = False - tokenize = False - if(is_present(body, "stream")): stream = body["stream"] - if(is_present(body, "tokenize")): tokenize = body["tokenize"] - postData = make_postData(body, chat=False, stream=stream) - - promptToken = [] - if (tokenize): - tokenData = requests.request("POST", urllib.parse.urljoin(args.llama_api, "/tokenize"), data=json.dumps({"content": postData["prompt"]})).json() - promptToken = tokenData["tokens"] - - if (not stream): - data = requests.request("POST", urllib.parse.urljoin(args.llama_api, "/completion"), data=json.dumps(postData)) - print(data.json()) - resData = make_resData(data.json(), chat=False, promptToken=promptToken) - return jsonify(resData) - else: - def generate(): - data = requests.request("POST", urllib.parse.urljoin(args.llama_api, "/completion"), data=json.dumps(postData), stream=True) - time_now = int(time.time()) - for line in data.iter_lines(): - if line: - decoded_line = line.decode('utf-8') - resData = make_resData_stream(json.loads(decoded_line[6:]), chat=False, time_now=time_now) - yield 'data: {}\n\n'.format(json.dumps(resData)) - return Response(generate(), mimetype='text/event-stream', headers={"Access-Control-Allow-Origin": "*", "Access-Control-Allow-Headers": "*"}) - -if __name__ == '__main__': - app.run(args.host, port=args.port) From c29af7e2252d288f2ea58a7d437c1cb7c0abf160 Mon Sep 17 00:00:00 2001 From: Sourab Mangrulkar <13534540+pacman100@users.noreply.github.com> Date: Sat, 2 Mar 2024 01:00:46 +0530 Subject: [PATCH 504/812] llama : add StarCoder2 support (#5795) * Add support for starcoder2 * handle rope type * skip rope freq and rotary embeddings from being serialized * resolve comments * Update llama.cpp * remove redundant changes * handle `rope-theta` * llama : change starcoder2 rope type * address comment --------- Co-authored-by: Georgi Gerganov --- convert-hf-to-gguf.py | 8 +- gguf-py/gguf/constants.py | 21 ++++ gguf-py/gguf/tensor_mapping.py | 2 + llama.cpp | 199 +++++++++++++++++++++++++++++++++ 4 files changed, 229 insertions(+), 1 deletion(-) diff --git a/convert-hf-to-gguf.py b/convert-hf-to-gguf.py index d3e8ec1f6..28b92ac38 100755 --- a/convert-hf-to-gguf.py +++ b/convert-hf-to-gguf.py @@ -96,9 +96,11 @@ class Model: if (n_head_kv := self.hparams.get("num_key_value_heads")) is not None: self.gguf_writer.add_head_count_kv(n_head_kv) + if (rope_theta := self.hparams.get("rope_theta")) is not None: + self.gguf_writer.add_rope_freq_base(rope_theta) if (f_rms_eps := self.hparams.get("rms_norm_eps")) is not None: self.gguf_writer.add_layer_norm_rms_eps(f_rms_eps) - if (f_norm_eps := self.find_hparam(["layer_norm_eps", "layer_norm_epsilon"], optional=True)) is not None: + if (f_norm_eps := self.find_hparam(["layer_norm_eps", "layer_norm_epsilon", "norm_epsilon"], optional=True)) is not None: self.gguf_writer.add_layer_norm_eps(f_norm_eps) if (n_experts := self.hparams.get("num_local_experts")) is not None: self.gguf_writer.add_expert_count(n_experts) @@ -220,6 +222,8 @@ class Model: return NomicBertModel if model_architecture == "GemmaForCausalLM": return GemmaModel + if model_architecture == "Starcoder2ForCausalLM": + return Model return Model def _is_model_safetensors(self) -> bool: @@ -281,6 +285,8 @@ class Model: return gguf.MODEL_ARCH.NOMIC_BERT if arch == "GemmaForCausalLM": return gguf.MODEL_ARCH.GEMMA + if arch == "Starcoder2ForCausalLM": + return gguf.MODEL_ARCH.STARCODER2 raise NotImplementedError(f'Architecture "{arch}" not supported!') diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py index 8f9139d1b..5db760cb1 100644 --- a/gguf-py/gguf/constants.py +++ b/gguf-py/gguf/constants.py @@ -112,6 +112,7 @@ class MODEL_ARCH(IntEnum): INTERNLM2 = auto() MINICPM = auto() GEMMA = auto() + STARCODER2 = auto() class MODEL_TENSOR(IntEnum): @@ -169,6 +170,7 @@ MODEL_ARCH_NAMES: dict[MODEL_ARCH, str] = { MODEL_ARCH.INTERNLM2: "internlm2", MODEL_ARCH.MINICPM: "minicpm", MODEL_ARCH.GEMMA: "gemma", + MODEL_ARCH.STARCODER2: "starcoder2", } TENSOR_NAMES: dict[MODEL_TENSOR, str] = { @@ -526,6 +528,21 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = { MODEL_TENSOR.FFN_UP, MODEL_TENSOR.FFN_NORM, ], + MODEL_ARCH.STARCODER2: [ + MODEL_TENSOR.TOKEN_EMBD, + MODEL_TENSOR.OUTPUT_NORM, + MODEL_TENSOR.OUTPUT, + MODEL_TENSOR.ROPE_FREQS, + MODEL_TENSOR.ATTN_NORM, + MODEL_TENSOR.ATTN_Q, + MODEL_TENSOR.ATTN_K, + MODEL_TENSOR.ATTN_V, + MODEL_TENSOR.ATTN_OUT, + MODEL_TENSOR.ATTN_ROT_EMBD, + MODEL_TENSOR.FFN_NORM, + MODEL_TENSOR.FFN_DOWN, + MODEL_TENSOR.FFN_UP, + ], # TODO } @@ -554,6 +571,10 @@ MODEL_TENSOR_SKIP: dict[MODEL_ARCH, list[MODEL_TENSOR]] = { MODEL_TENSOR.ROPE_FREQS, MODEL_TENSOR.ATTN_ROT_EMBD, ], + MODEL_ARCH.STARCODER2: [ + MODEL_TENSOR.ROPE_FREQS, + MODEL_TENSOR.ATTN_ROT_EMBD, + ], } # diff --git a/gguf-py/gguf/tensor_mapping.py b/gguf-py/gguf/tensor_mapping.py index 861003776..db2ec9704 100644 --- a/gguf-py/gguf/tensor_mapping.py +++ b/gguf-py/gguf/tensor_mapping.py @@ -210,6 +210,7 @@ class TensorNameMap: "model.layers.layers.{bid}.mlp.up_proj", # plamo "model.layers.{bid}.feed_forward.w3", # internlm2 "encoder.layers.{bid}.mlp.fc11", # nomic-bert + "model.layers.{bid}.mlp.c_fc", # starcoder2 ), MODEL_TENSOR.FFN_UP_EXP: ( @@ -256,6 +257,7 @@ class TensorNameMap: "model.layers.layers.{bid}.mlp.down_proj", # plamo "model.layers.{bid}.feed_forward.w2", # internlm2 "encoder.layers.{bid}.mlp.fc2", # nomic-bert + "model.layers.{bid}.mlp.c_proj", # starcoder2 ), MODEL_TENSOR.FFN_DOWN_EXP: ( diff --git a/llama.cpp b/llama.cpp index 073fd3b70..b1db5b179 100644 --- a/llama.cpp +++ b/llama.cpp @@ -211,6 +211,7 @@ enum llm_arch { LLM_ARCH_INTERNLM2, LLM_ARCH_MINICPM, LLM_ARCH_GEMMA, + LLM_ARCH_STARCODER2, LLM_ARCH_UNKNOWN, }; @@ -238,6 +239,7 @@ static std::map LLM_ARCH_NAMES = { { LLM_ARCH_INTERNLM2, "internlm2" }, { LLM_ARCH_MINICPM, "minicpm" }, { LLM_ARCH_GEMMA, "gemma" }, + { LLM_ARCH_STARCODER2, "starcoder2" }, }; enum llm_kv { @@ -779,6 +781,24 @@ static std::map> LLM_TENSOR_NAMES = { LLM_TENSOR_FFN_UP, "blk.%d.ffn_up" }, }, }, + { + LLM_ARCH_STARCODER2, + { + { LLM_TENSOR_TOKEN_EMBD, "token_embd" }, + { LLM_TENSOR_OUTPUT_NORM, "output_norm" }, + { LLM_TENSOR_OUTPUT, "output" }, + { LLM_TENSOR_ROPE_FREQS, "rope_freqs" }, + { LLM_TENSOR_ATTN_NORM, "blk.%d.attn_norm" }, + { LLM_TENSOR_ATTN_Q, "blk.%d.attn_q" }, + { LLM_TENSOR_ATTN_K, "blk.%d.attn_k" }, + { LLM_TENSOR_ATTN_V, "blk.%d.attn_v" }, + { LLM_TENSOR_ATTN_OUT, "blk.%d.attn_output" }, + { LLM_TENSOR_ATTN_ROT_EMBD, "blk.%d.attn_rot_embd" }, + { LLM_TENSOR_FFN_NORM, "blk.%d.ffn_norm" }, + { LLM_TENSOR_FFN_DOWN, "blk.%d.ffn_down" }, + { LLM_TENSOR_FFN_UP, "blk.%d.ffn_up" }, + }, + }, { LLM_ARCH_UNKNOWN, { @@ -3320,6 +3340,16 @@ static void llm_load_hparams( default: model.type = e_model::MODEL_UNKNOWN; } } break; + case LLM_ARCH_STARCODER2: + { + ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps); + switch (hparams.n_layer) { + case 30: model.type = e_model::MODEL_3B; break; + case 32: model.type = e_model::MODEL_7B; break; + case 40: model.type = e_model::MODEL_15B; break; + default: model.type = e_model::MODEL_UNKNOWN; + } + } break; default: (void)0; } @@ -4490,6 +4520,56 @@ static bool llm_load_tensors( layer.ffn_down = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_DOWN, "weight", i), { n_ff, n_embd}); } } break; + case LLM_ARCH_STARCODER2: + { + model.tok_embd = ml.create_tensor(ctx_input, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}); + + // output + { + model.output_norm = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}); + model.output_norm_b = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "bias"), {n_embd}); + + model.output = ml.create_tensor(ctx_output_split, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, false); + // if output is NULL, init from the input tok embed + if (model.output == NULL) { + model.output = ml.create_tensor(ctx_output, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}); + ml.n_created--; // artificial tensor + ml.size_data += ggml_nbytes(model.output); + } + + } + + for (int i = 0; i < n_layer; ++i) { + ggml_context * ctx_layer = ctx_for_layer(i); + ggml_context * ctx_split = ctx_for_layer_split(i); + + auto & layer = model.layers[i]; + + layer.attn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}); + layer.attn_norm_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM, "bias", i), {n_embd}); + + layer.wq = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_embd}); + layer.wk = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_K, "weight", i), {n_embd, n_embd_gqa}); + layer.wv = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_V, "weight", i), {n_embd, n_embd_gqa}); + layer.wo = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd}); + + // optional bias tensors + layer.bq = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_Q, "bias", i), {n_embd}); + layer.bk = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_K, "bias", i), {n_embd_gqa}); + layer.bv = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_V, "bias", i), {n_embd_gqa}); + layer.bo = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_OUT, "bias", i), {n_embd}); + + layer.ffn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}); + layer.ffn_norm_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_NORM, "bias", i), {n_embd}); + + layer.ffn_down = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_DOWN, "weight", i), { n_ff, n_embd}); + layer.ffn_up = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff}); + + // optional bias tensors + layer.ffn_down_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_DOWN, "bias", i), {n_embd}); + layer.ffn_up_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_UP , "bias", i), { n_ff}); + } + } break; default: throw std::runtime_error("unknown architecture"); } @@ -7559,6 +7639,120 @@ struct llm_build_context { return gf; } + + struct ggml_cgraph * build_starcoder2() { + struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, LLAMA_MAX_NODES, false); + + const int64_t n_embd_head = hparams.n_embd_head_v; + GGML_ASSERT(n_embd_head == hparams.n_embd_head_k); + GGML_ASSERT(n_embd_head == hparams.n_rot); + + struct ggml_tensor * cur; + struct ggml_tensor * inpL; + + inpL = llm_build_inp_embd(ctx0, hparams, batch, model.tok_embd, lctx.inp_tokens, lctx.inp_embd, cb); + cb(inpL, "inp_embd", -1); + + // inp_pos - contains the positions + struct ggml_tensor * inp_pos = ggml_view_1d(ctx0, lctx.inp_pos, n_tokens, 0); + cb(inp_pos, "inp_pos", -1); + + // KQ_mask (mask for 1 head, it will be broadcasted to all heads) + struct ggml_tensor * KQ_mask = ggml_view_2d(ctx0, lctx.inp_KQ_mask, n_kv, n_tokens, n_kv*ggml_type_size(lctx.inp_KQ_mask->type), 0); + cb(KQ_mask, "KQ_mask", -1); + + for (int il = 0; il < n_layer; ++il) { + struct ggml_tensor * inpSA = inpL; + + // norm + cur = llm_build_norm(ctx0, inpL, hparams, + model.layers[il].attn_norm, model.layers[il].attn_norm_b, + LLM_NORM, cb, il); + cb(cur, "attn_norm", il); + + // self-attention + { + // compute Q and K and RoPE them + struct ggml_tensor * Qcur = ggml_mul_mat(ctx0, model.layers[il].wq, cur); + cb(Qcur, "Qcur", il); + if (model.layers[il].bq) { + Qcur = ggml_add(ctx0, Qcur, model.layers[il].bq); + cb(Qcur, "Qcur", il); + } + + struct ggml_tensor * Kcur = ggml_mul_mat(ctx0, model.layers[il].wk, cur); + cb(Kcur, "Kcur", il); + if (model.layers[il].bk) { + Kcur = ggml_add(ctx0, Kcur, model.layers[il].bk); + cb(Kcur, "Kcur", il); + } + + struct ggml_tensor * Vcur = ggml_mul_mat(ctx0, model.layers[il].wv, cur); + cb(Vcur, "Vcur", il); + if (model.layers[il].bv) { + Vcur = ggml_add(ctx0, Vcur, model.layers[il].bv); + cb(Vcur, "Vcur", il); + } + + Qcur = ggml_rope_custom( + ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, + n_rot, rope_type, 0, n_orig_ctx, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow + ); + cb(Qcur, "Qcur", il); + + Kcur = ggml_rope_custom( + ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, + n_rot, rope_type, 0, n_orig_ctx, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow + ); + cb(Kcur, "Kcur", il); + + cur = llm_build_kv(ctx0, model, hparams, kv_self, gf, + model.layers[il].wo, model.layers[il].bo, + Kcur, Vcur, Qcur, KQ_mask, nullptr, n_ctx, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il); + cb(cur, "kqv_out", il); + } + + struct ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA); + cb(ffn_inp, "ffn_inp", il); + + // feed-forward network + + cur = llm_build_norm(ctx0, ffn_inp, hparams, + model.layers[il].ffn_norm, model.layers[il].ffn_norm_b, + LLM_NORM, cb, il); + cb(cur, "ffn_norm", il); + + cur = llm_build_ffn(ctx0, cur, + model.layers[il].ffn_up, model.layers[il].ffn_up_b, + NULL, NULL, + model.layers[il].ffn_down, model.layers[il].ffn_down_b, + NULL, + LLM_FFN_GELU, LLM_FFN_SEQ, cb, il); + cb(cur, "ffn_out", il); + cur = ggml_add(ctx0, cur, ffn_inp); + cb(cur, "l_out", il); + + // input for next layer + inpL = cur; + } + + cur = inpL; + + cur = llm_build_norm(ctx0, cur, hparams, + model.output_norm, model.output_norm_b, + LLM_NORM, cb, -1); + cb(cur, "result_norm", -1); + + // lm_head + cur = ggml_mul_mat(ctx0, model.output, cur); + cb(cur, "result_output", -1); + + ggml_build_forward_expand(gf, cur); + + return gf; + } }; static struct ggml_cgraph * llama_build_graph_defrag(llama_context & lctx, const std::vector & ids) { @@ -7705,6 +7899,10 @@ static struct ggml_cgraph * llama_build_graph( { result = llm.build_gemma(); } break; + case LLM_ARCH_STARCODER2: + { + result = llm.build_starcoder2(); + } break; default: GGML_ASSERT(false); } @@ -12084,6 +12282,7 @@ enum llama_rope_type llama_rope_type(const struct llama_model * model) { case LLM_ARCH_QWEN2: case LLM_ARCH_PHI2: case LLM_ARCH_GEMMA: + case LLM_ARCH_STARCODER2: return LLAMA_ROPE_TYPE_NEOX; // all model arches should be listed explicitly here From da3b9ba2b710c0f8b44398a0eb9e5a7ae2ad967a Mon Sep 17 00:00:00 2001 From: nold Date: Fri, 1 Mar 2024 22:51:12 +0100 Subject: [PATCH 505/812] convert-hf-to-gguf : require einops for InternLM2ForCausalLM (#5792) --- requirements/requirements-convert-hf-to-gguf.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements/requirements-convert-hf-to-gguf.txt b/requirements/requirements-convert-hf-to-gguf.txt index 6ac402610..6ce840d73 100644 --- a/requirements/requirements-convert-hf-to-gguf.txt +++ b/requirements/requirements-convert-hf-to-gguf.txt @@ -1,2 +1,3 @@ -r ./requirements-convert.txt torch~=2.1.1 +einops~=0.7.0 From cb5e8f7fc4ee57d4bcccafbe04a82cededd35486 Mon Sep 17 00:00:00 2001 From: Tushar Date: Sat, 2 Mar 2024 04:48:26 +0530 Subject: [PATCH 506/812] build(nix): Introduce flake.formatter for `nix fmt` (#5687) * build(nix): Introduce flake.formatter for `nix fmt` * chore: Switch to pkgs.nixfmt-rfc-style --- .devops/nix/sif.nix | 2 +- flake.nix | 10 +++++++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/.devops/nix/sif.nix b/.devops/nix/sif.nix index 7535ca0f3..7a5e1dd0f 100644 --- a/.devops/nix/sif.nix +++ b/.devops/nix/sif.nix @@ -7,7 +7,7 @@ }: let - optionalInt = cond: x: if cond then x else 0; + optionalInt = cond: x: if cond then x else 0; in singularity-tools.buildImage rec { inherit (llama-cpp) name; diff --git a/flake.nix b/flake.nix index dc4e503c3..45f9deda0 100644 --- a/flake.nix +++ b/flake.nix @@ -107,11 +107,12 @@ # ``` # # Cf. https://nixos.org/manual/nix/unstable/command-ref/new-cli/nix3-flake.html?highlight=flake#flake-format - flake.overlays.default = - (final: prev: { + flake.overlays.default = ( + final: prev: { llamaPackages = final.callPackage .devops/nix/scope.nix { inherit llamaVersion; }; inherit (final.llamaPackages) llama-cpp; - }); + } + ); systems = [ "aarch64-darwin" @@ -131,6 +132,9 @@ ... }: { + # For standardised reproducible formatting with `nix fmt` + formatter = pkgs.nixfmt-rfc-style; + # Unlike `.#packages`, legacyPackages may contain values of # arbitrary types (including nested attrsets) and may even throw # exceptions. This attribute isn't recursed into by `nix flake From 9bf297a02bfbd474e51912409a470dd797e2fe13 Mon Sep 17 00:00:00 2001 From: crasm Date: Sat, 2 Mar 2024 00:11:06 -0500 Subject: [PATCH 507/812] workflows : remove nocleanup arg for check-requirements.sh (#5826) Reduces peak tmpfs usage and should prevent the check from failing from running out of space. Fixes the 'No space left on device' issue mentioned in #5703. --- .github/workflows/python-check-requirements.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/python-check-requirements.yml b/.github/workflows/python-check-requirements.yml index 92e1108b3..b82205992 100644 --- a/.github/workflows/python-check-requirements.yml +++ b/.github/workflows/python-check-requirements.yml @@ -3,12 +3,14 @@ name: Python check requirements.txt on: push: paths: + - '.github/workflows/python-check-requirements.yml' - 'scripts/check-requirements.sh' - 'convert*.py' - 'requirements.txt' - 'requirements/*.txt' pull_request: paths: + - '.github/workflows/python-check-requirements.yml' - 'scripts/check-requirements.sh' - 'convert*.py' - 'requirements.txt' @@ -26,4 +28,4 @@ jobs: with: python-version: "3.11" - name: Run check-requirements.sh script - run: bash scripts/check-requirements.sh nocleanup + run: bash scripts/check-requirements.sh From 715641391dda1ff9762dc5d99d9a30acce99f2c6 Mon Sep 17 00:00:00 2001 From: Neo Zhang Jianyu Date: Sat, 2 Mar 2024 19:49:30 +0800 Subject: [PATCH 508/812] Support multiple GPUs (split mode) on SYCL backend (#5806) * suport multiple cards: split-mode - layer|row * rm warning * rebase with master, support tow new OPs, close feature for -sm=row, fix for unit test * update news * fix merge error * update according to review comments --- README-sycl.md | 21 + common/common.cpp | 4 + examples/llama-bench/llama-bench.cpp | 17 +- examples/sycl/ls-sycl-device.cpp | 2 +- examples/sycl/run-llama2.sh | 17 +- ggml-sycl.cpp | 2205 +++++++++++++++++--------- ggml-sycl.h | 5 + llama.cpp | 49 +- 8 files changed, 1506 insertions(+), 814 deletions(-) diff --git a/README-sycl.md b/README-sycl.md index dd5bf9dea..85eb16f2b 100644 --- a/README-sycl.md +++ b/README-sycl.md @@ -1,6 +1,7 @@ # llama.cpp for SYCL - [Background](#background) +- [News](#news) - [OS](#os) - [Intel GPU](#intel-gpu) - [Docker](#docker) @@ -25,6 +26,21 @@ The llama.cpp for SYCL is used to support Intel GPUs. For Intel CPU, recommend to use llama.cpp for X86 (Intel MKL building). +## News + +- 2024.3 + - Support multiple cards: **--split-mode**: [none|layer]; not support [row], it's on developing. + - Support to assign main GPU by **--main-gpu**, replace $GGML_SYCL_DEVICE. + - Support detecting all GPUs with level-zero and same top **Max compute units**. + - Support OPs + - hardsigmoid + - hardswish + - pool2d + +- 2024.1 + - Create SYCL backend for Intel GPU. + - Support Windows build + ## OS |OS|Status|Verified| @@ -449,6 +465,7 @@ Using device **0** (Intel(R) Arc(TM) A770 Graphics) as main device |-|-|-| |GGML_SYCL_DEVICE|0 (default) or 1|Set the device id used. Check the device ids by default running output| |GGML_SYCL_DEBUG|0 (default) or 1|Enable log function by macro: GGML_SYCL_DEBUG| +|ZES_ENABLE_SYSMAN| 0 (default) or 1|Support to get free memory of GPU by sycl::aspect::ext_intel_free_memory.
    Recommended to use when --split-mode = layer| ## Known Issue @@ -458,6 +475,10 @@ Using device **0** (Intel(R) Arc(TM) A770 Graphics) as main device Solution: add **--no-mmap** or **--mmap 0**. +- Split-mode: [row] is not supported + + It's on developing. + ## Q&A - Error: `error while loading shared libraries: libsycl.so.7: cannot open shared object file: No such file or directory`. diff --git a/common/common.cpp b/common/common.cpp index 938c428cf..1c0b7c403 100644 --- a/common/common.cpp +++ b/common/common.cpp @@ -640,6 +640,10 @@ bool gpt_params_parse_ex(int argc, char ** argv, gpt_params & params) { } else if (arg_next == "layer") { params.split_mode = LLAMA_SPLIT_MODE_LAYER; } else if (arg_next == "row") { +#ifdef GGML_USE_SYCL + fprintf(stderr, "warning: The split mode value:[row] is not supported by llama.cpp with SYCL. It's developing.\nExit!\n"); + exit(1); +#endif // GGML_USE_SYCL params.split_mode = LLAMA_SPLIT_MODE_ROW; } else { invalid_param = true; diff --git a/examples/llama-bench/llama-bench.cpp b/examples/llama-bench/llama-bench.cpp index c2155b2ac..aa79d002a 100644 --- a/examples/llama-bench/llama-bench.cpp +++ b/examples/llama-bench/llama-bench.cpp @@ -123,20 +123,15 @@ static std::string get_gpu_info() { } #endif #ifdef GGML_USE_SYCL - int device_list[GGML_SYCL_MAX_DEVICES]; - ggml_sycl_get_gpu_list(device_list, GGML_SYCL_MAX_DEVICES); - - for (int i = 0; i < GGML_SYCL_MAX_DEVICES; i++) { - if (device_list[i] >0 ){ - char buf[128]; - ggml_sycl_get_device_description(i, buf, sizeof(buf)); - id += buf; + int count = ggml_backend_sycl_get_device_count(); + for (int i = 0; i < count; i++) { + char buf[128]; + ggml_sycl_get_device_description(i, buf, sizeof(buf)); + id += buf; + if (i < count - 1) { id += "/"; } } - if (id.length() >2 ) { - id.pop_back(); - } #endif // TODO: other backends return id; diff --git a/examples/sycl/ls-sycl-device.cpp b/examples/sycl/ls-sycl-device.cpp index 52442e4ca..74a8b7fd8 100644 --- a/examples/sycl/ls-sycl-device.cpp +++ b/examples/sycl/ls-sycl-device.cpp @@ -7,7 +7,7 @@ #include "ggml-sycl.h" -int main(int argc, char ** argv) { +int main() { ggml_backend_sycl_print_sycl_devices(); return 0; } diff --git a/examples/sycl/run-llama2.sh b/examples/sycl/run-llama2.sh index f5f4c1e98..52f7c01a4 100755 --- a/examples/sycl/run-llama2.sh +++ b/examples/sycl/run-llama2.sh @@ -8,12 +8,19 @@ INPUT2="Building a website can be done in 10 simple steps:\nStep 1:" source /opt/intel/oneapi/setvars.sh if [ $# -gt 0 ]; then - export GGML_SYCL_DEVICE=$1 + GGML_SYCL_DEVICE=$1 else - export GGML_SYCL_DEVICE=0 + GGML_SYCL_DEVICE=0 fi -echo GGML_SYCL_DEVICE=$GGML_SYCL_DEVICE +echo "use $GGML_SYCL_DEVICE as main GPU" #export GGML_SYCL_DEBUG=1 -./build/bin/main -m models/llama-2-7b.Q4_0.gguf -p "${INPUT2}" -n 400 -e -ngl 33 -s 0 -#./build/bin/main -m models/llama-2-7b.Q4_0.gguf -p "${INPUT2}" -n 5 -e -ngl 33 -t 1 -s 0 + + +#ZES_ENABLE_SYSMAN=1, Support to get free memory of GPU by sycl::aspect::ext_intel_free_memory. Recommended to use when --split-mode = layer. + +#use all GPUs with same max compute units +ZES_ENABLE_SYSMAN=1 ./build/bin/main -m models/llama-2-7b.Q4_0.gguf -p "${INPUT2}" -n 400 -e -ngl 33 -s 0 + +#use main GPU only +#ZES_ENABLE_SYSMAN=1 ./build/bin/main -m models/llama-2-7b.Q4_0.gguf -p "${INPUT2}" -n 400 -e -ngl 33 -s 0 -mg $GGML_SYCL_DEVICE -sm none diff --git a/ggml-sycl.cpp b/ggml-sycl.cpp index 6f391b0c6..cad08d610 100644 --- a/ggml-sycl.cpp +++ b/ggml-sycl.cpp @@ -661,26 +661,29 @@ namespace dpct /// \param [out] total_memory The number of bytes of total memory on the SYCL device. void get_memory_info(size_t &free_memory, size_t &total_memory) { + total_memory = get_device_info().get_global_mem_size(); + const char *warning_info = "get_memory_info: [warning] ext_intel_free_memory is not " + "supported (export/set ZES_ENABLE_SYSMAN=1 to support), " + "use total memory as free memory"; #if (defined(__SYCL_COMPILER_VERSION) && __SYCL_COMPILER_VERSION >= 20221105) if (!has(sycl::aspect::ext_intel_free_memory)) { - std::cerr << "get_memory_info: ext_intel_free_memory is not supported." << std::endl; - free_memory = 0; + std::cerr << warning_info << std::endl; + free_memory = total_memory; } else { free_memory = get_info(); } #else - std::cerr << "get_memory_info: ext_intel_free_memory is not supported." << std::endl; - free_memory = 0; + std::cerr << warning_info << std::endl; + free_memory = total_memory; #if defined(_MSC_VER) && !defined(__clang__) #pragma message("Querying the number of bytes of free memory is not supported") #else #warning "Querying the number of bytes of free memory is not supported" #endif #endif - total_memory = get_device_info().get_global_mem_size(); } void get_device_info(device_info &out) const @@ -738,15 +741,25 @@ namespace dpct #endif // DPCT_USM_LEVEL_NONE } - sycl::queue *create_in_order_queue(bool enable_exception_handler = false) - { - std::lock_guard lock(m_mutex); - return create_queue_impl(enable_exception_handler, - sycl::property::queue::in_order()); + sycl::queue *create_queue(sycl::context context, sycl::device device, + bool enable_exception_handler = false) { + return create_in_order_queue(context, device, enable_exception_handler); } - sycl::queue *create_out_of_order_queue(bool enable_exception_handler = false) - { + sycl::queue *create_in_order_queue(bool enable_exception_handler = false) { + std::lock_guard lock(m_mutex); + return create_queue_impl(enable_exception_handler, + sycl::property::queue::in_order()); + } + + sycl::queue *create_in_order_queue(sycl::context context, sycl::device device, + bool enable_exception_handler = false) { + std::lock_guard lock(m_mutex); + return create_queue_impl(context, device, enable_exception_handler, + sycl::property::queue::in_order()); + } + + sycl::queue *create_out_of_order_queue(bool enable_exception_handler = false) { std::lock_guard lock(m_mutex); return create_queue_impl(enable_exception_handler); } @@ -809,6 +822,25 @@ namespace dpct return _queues.back().get(); } + template + sycl::queue *create_queue_impl(sycl::context context, sycl::device device, + bool enable_exception_handler, + Properties... properties) { + sycl::async_handler eh = {}; + if (enable_exception_handler) { + eh = exception_handler; + } + _queues.push_back(std::make_shared( + context, device, eh, + sycl::property_list( + #ifdef DPCT_PROFILING_ENABLED + sycl::property::queue::enable_profiling(), + #endif + properties...))); + + return _queues.back().get(); + } + void get_version(int &major, int &minor) const { detail::get_version(*this, major, minor); @@ -2943,14 +2975,11 @@ bool ggml_sycl_loaded(void); void * ggml_sycl_host_malloc(size_t size); void ggml_sycl_host_free(void * ptr); bool ggml_sycl_can_mul_mat(const struct ggml_tensor * src0, const struct ggml_tensor * src1, struct ggml_tensor * dst); -void ggml_sycl_set_tensor_split(const float * tensor_split); -void ggml_sycl_transform_tensor(void * data, struct ggml_tensor * tensor); void ggml_sycl_free_data(struct ggml_tensor * tensor); void ggml_sycl_assign_buffers(struct ggml_tensor * tensor); void ggml_sycl_assign_buffers_no_scratch(struct ggml_tensor * tensor); void ggml_sycl_assign_buffers_force_inplace(struct ggml_tensor * tensor); void ggml_sycl_assign_buffers_no_alloc(struct ggml_tensor * tensor); -void ggml_sycl_assign_scratch_offset(struct ggml_tensor * tensor, size_t offset); void ggml_sycl_copy_to_device(struct ggml_tensor * tensor); void ggml_sycl_set_main_device(int main_device); void ggml_sycl_set_mul_mat_q(bool mul_mat_q); @@ -2963,6 +2992,14 @@ int get_main_device(); void print_ggml_tensor(const char*name, struct ggml_tensor *src); void log_tensor_with_cnt(const char* name, struct ggml_tensor * src, int stop_cnt); +void dev2dev_memcpy(sycl::queue &q_dst, sycl::queue &q_src, void *ptr_dst, + const void *ptr_src, size_t size) { + char *host_buf = (char *)malloc(size); + q_src.memcpy(host_buf, (const char *)ptr_src, size).wait(); + q_dst.memcpy((char *)ptr_dst, host_buf, size).wait(); + free(host_buf); +} + static __dpct_inline__ int get_int_from_int8(const int8_t *x8, const int &i32) { const uint16_t * x16 = (const uint16_t *) (x8 + sizeof(int) * i32); // assume at least 2 byte alignment @@ -3180,6 +3217,8 @@ static_assert(sizeof(block_q6_K) == sizeof(ggml_fp16_t) + 13*QK_K/16, "wrong q6_ #define SYCL_SILU_BLOCK_SIZE 256 #define SYCL_TANH_BLOCK_SIZE 256 #define SYCL_RELU_BLOCK_SIZE 256 +#define SYCL_HARDSIGMOID_BLOCK_SIZE 256 +#define SYCL_HARDSWISH_BLOCK_SIZE 256 #define SYCL_SQR_BLOCK_SIZE 256 #define SYCL_CPY_BLOCK_SIZE 32 #define SYCL_SCALE_BLOCK_SIZE 256 @@ -3196,6 +3235,7 @@ static_assert(sizeof(block_q6_K) == sizeof(ggml_fp16_t) + 13*QK_K/16, "wrong q6_ #define SYCL_PAD_BLOCK_SIZE 256 #define SYCL_ACC_BLOCK_SIZE 256 #define SYCL_IM2COL_BLOCK_SIZE 256 +#define SYCL_POOL2D_BLOCK_SIZE 256 // dmmv = dequantize_mul_mat_vec #ifndef GGML_SYCL_DMMV_X @@ -3218,8 +3258,7 @@ static_assert(K_QUANTS_PER_ITERATION == 1 || K_QUANTS_PER_ITERATION == 2, "K_QUA #define MUL_MAT_SRC1_COL_STRIDE 128 #define MAX_STREAMS 8 -static dpct::queue_ptr g_syclStreams[GGML_SYCL_MAX_DEVICES][MAX_STREAMS] = { - {0}}; +static dpct::queue_ptr g_syclStreams[GGML_SYCL_MAX_DEVICES][MAX_STREAMS] = {{0}}; struct ggml_tensor_extra_gpu { void * data_device[GGML_SYCL_MAX_DEVICES]; // 1 pointer for each device for split tensors @@ -3228,30 +3267,108 @@ struct ggml_tensor_extra_gpu { [MAX_STREAMS]; // events for synchronizing multiple GPUs }; -inline dpct::err0 ggml_sycl_set_device(const int device) try { - int current_device; +class sycl_gpu_mgr { + public: + std::vector gpus; + std::vector devices; + sycl::queue *first_queue; + sycl::context co_ctx; + int max_compute_units = 0; + int work_group_size = 0; + std::string gpus_list = ""; - SYCL_CHECK(CHECK_TRY_ERROR( - current_device = dpct::dev_mgr::instance().current_device_id())); + sycl_gpu_mgr() { + detect_sycl_gpu_list_with_max_cu(); + get_allow_gpus(); + create_context_with_gpus(); + } - // GGML_SYCL_DEBUG("ggml_sycl_set_device device=%d, current_device=%d\n", device, current_device); - if (device == current_device) { - return 0; - } + void create_context_with_gpus() { + sycl::context ctx = sycl::context(devices); + assert(gpus.size() > 0); + first_queue = dpct::get_current_device().create_queue(ctx, devices[0]); + co_ctx = first_queue->get_context(); + } - return CHECK_TRY_ERROR(dpct::select_device(device)); -} -catch (sycl::exception const &exc) { - std::cerr << exc.what() << "Exception caught at file:" << __FILE__ - << ", line:" << __LINE__ << std::endl; - crash(); - std::exit(1); -} + sycl::context &get_co_ctx() { return co_ctx; } + void get_allow_gpus() { + gpus_list = ""; + for (size_t i = 0; i < gpus.size(); ++i) { + gpus_list += std::to_string(gpus[i]); + gpus_list += ","; + } + if (gpus_list.length() > 2) { + gpus_list.pop_back(); + } + } + + bool is_allowed_gpu(int device_id) { + return std::find(gpus.begin(), gpus.end(), device_id) != gpus.end(); + } + + void detect_sycl_gpu_list_with_max_cu() try { + int device_count = dpct::dev_mgr::instance().device_count(); + + for (int id = 0; id < device_count; id++) { + sycl::device device = dpct::dev_mgr::instance().get_device(id); + if (!device.is_gpu()) + continue; + dpct::device_info prop; + dpct::get_device_info(prop, device); + if (max_compute_units < prop.get_max_compute_units()) + max_compute_units = prop.get_max_compute_units(); + } + + for (int id = 0; id < device_count; id++) { + sycl::device device = dpct::dev_mgr::instance().get_device(id); + if (!device.is_gpu()) + continue; + dpct::device_info prop; + dpct::get_device_info(prop, device); + if (max_compute_units == prop.get_max_compute_units() && + prop.get_major_version() == 1) { + gpus.push_back(id); + devices.push_back(device); + work_group_size = prop.get_max_work_group_size(); + } + } + return; + } catch (sycl::exception const &exc) { + std::cerr << exc.what() << "Exception caught at file:" << __FILE__ + << ", line:" << __LINE__ << std::endl; + std::exit(1); + } + + int get_gpu_count() { return (int)gpus.size(); } + + int get_index(int id) { + for (int i = 0; i < (int)gpus.size(); i++) { + if (gpus[i] == id) + return i; + } + assert(false); + return -1; + } + + int get_next_index(int id) { + int cur_index = get_index(id); + for (int i = cur_index + 1; i < (int)gpus.size(); i++) { + if (gpus[i] == id) + return i; + } + assert(false); + return -1; + } +}; + +static sycl_gpu_mgr *g_sycl_gpu_mgr = NULL; static int g_device_count = -1; static int g_all_sycl_device_count = -1; static int g_main_device = -1; -static int g_main_device_index = -1; +static int g_main_device_id = -1; + +static std::array g_default_tensor_split = {}; static float g_tensor_split[GGML_SYCL_MAX_DEVICES] = {0}; @@ -3268,8 +3385,6 @@ struct sycl_device_id2index { int index; }; -static sycl_device_id2index g_sycl_device_id2index[GGML_SYCL_MAX_DEVICES] = { {-1} }; - static void * g_scratch_buffer = nullptr; static size_t g_scratch_size = 0; // disabled by default static size_t g_scratch_offset = 0; @@ -3290,6 +3405,63 @@ static void bad_arch(const sycl::stream &stream_ct1) { (void) bad_arch; // suppress unused function warning } +/* +device_index: device index from 0 to n (continue numbers). + It is used for device select/set in SYCL backend internal data structure. +*/ +void check_allow_gpu_index(const int device_index) { + if (device_index >= g_device_count) { + char error_buf[256]; + snprintf(error_buf, sizeof(error_buf), + "%s error: device_index:%d is out of range: [0-%d]", __func__, + device_index, g_device_count - 1); + fprintf(stderr, "%s\n", error_buf); + assert(false); + } +} + +/* +device_id: device ID is shown by ggml_backend_sycl_print_sycl_devices(). + It is only used to set current working device. +*/ +void check_allow_gpu_id(const int device_id) { + if (!g_sycl_gpu_mgr->is_allowed_gpu(device_id)) { + char error_buf[256]; + snprintf(error_buf, sizeof(error_buf), + "error: cannot set device=%d, which is not allowed. Please " + "set GPU ID in: [%s]", + device_id, g_sycl_gpu_mgr->gpus_list.c_str()); + fprintf(stderr, "%s\n", error_buf); + throw std::invalid_argument(error_buf); + } +} + +int get_current_device_id() { + return dpct::dev_mgr::instance().current_device_id(); +} + +inline dpct::err0 ggml_sycl_set_device(const int device) try { + + int device_id = g_sycl_gpu_mgr->gpus[device]; + check_allow_gpu_id(device_id); + + int current_device_id; + SYCL_CHECK(CHECK_TRY_ERROR(current_device_id = get_current_device_id())); + + // GGML_SYCL_DEBUG("ggml_sycl_set_device device_id=%d, + // current_device_id=%d\n", device, current_device); + if (device_id == current_device_id) { + return 0; + } + + return CHECK_TRY_ERROR(dpct::select_device(device_id)); +} catch (sycl::exception const &exc) { + std::cerr << exc.what() << "Exception caught at file:" << __FILE__ + << ", line:" << __LINE__ << std::endl; + crash(); + std::exit(1); +} + void log_ggml_var_device(const char*name, float *src, size_t total_elements, bool src_on_device){ if(!g_ggml_sycl_debug) return; if(!src){ @@ -3302,22 +3474,18 @@ void log_ggml_var_device(const char*name, float *src, size_t total_elements, boo size_t total_size = total_elements*sizeof(float); float *local_buf = NULL; - // printf("total_size %d2, src_on_device %d\n", total_size, src_on_device); if(src_on_device) { local_buf = (float *) ggml_sycl_host_malloc(total_size); - // printf("local buf %p size %d bytes\n", local_buf, total_size); ggml_sycl_set_device(g_main_device); - dpct::queue_ptr main_stream = g_syclStreams[g_main_device_index][0]; + dpct::queue_ptr main_stream = g_syclStreams[g_main_device][0]; main_stream->memcpy(local_buf, src, total_size); } else { local_buf = (float *)src; - // printf("local buf from src-> data %p\n", local_buf); } std::ofstream logfile; logfile.open(filename); - // printf("local buf element %d\n", total_elements); for(size_t i=0; iextra; - src_data = (float*)src_extra->data_device[g_main_device_index]; + src_data = (float*)src_extra->data_device[g_main_device]; } else { src_data = (float *)src->data; @@ -3359,10 +3527,6 @@ void log_tensor_with_cnt(const char* name, struct ggml_tensor * src, int stop_cn sprintf(filename, "%s_%07d", name, log_file_name_idx); log_file_name_idx++; print_ggml_tensor(filename, src); - // print_ggml_tensor("ggml_sycl_rms_norm_src0", (ggml_tensor *)src0); - // print_ggml_tensor("ggml_sycl_rms_norm_src1", (ggml_tensor *)src1); - // int *ptr = NULL; - // *ptr = 0; } static __dpct_inline__ float warp_reduce_sum(float x, @@ -3583,6 +3747,28 @@ static void relu_f32(const float * x, float * dst, const int k, dst[i] = sycl::fmax((float)(x[i]), (float)0); } +static void hardsigmoid_f32(const float * x, float * dst, const int k, + const sycl::nd_item<3> &item_ct1) { + const int i = item_ct1.get_local_range(2) * item_ct1.get_group(2) + + item_ct1.get_local_id(2); + + if (i >= k) { + return; + } + dst[i] = sycl::fmin(1.0f, sycl::fmax(0.0f, (x[i] + 3.0f) / 6.0f)); +} + +static void hardswish_f32(const float * x, float * dst, const int k, + const sycl::nd_item<3> &item_ct1) { + const int i = item_ct1.get_local_range(2) * item_ct1.get_group(2) + + item_ct1.get_local_id(2); + + if (i >= k) { + return; + } + dst[i] = x[i] * sycl::fmin(1.0f, sycl::fmax(0.0f, (x[i] + 3.0f) / 6.0f)); +} + static void leaky_relu_f32(const float *x, float *dst, const int k, const float negative_slope, const sycl::nd_item<3> &item_ct1) { const int i = item_ct1.get_local_range(2) * item_ct1.get_group(2) + @@ -4964,8 +5150,8 @@ static void k_get_rows_float( template static void dequantize_block(const void * __restrict__ vx, dst_t * __restrict__ y, const int k, const sycl::nd_item<3> &item_ct1) { - const int i = item_ct1.get_local_range(2) * item_ct1.get_group(2) + - 2 * item_ct1.get_local_id(2); + const int i = 2 * (item_ct1.get_local_range(2) * item_ct1.get_group(2) + + item_ct1.get_local_id(2)); if (i >= k) { return; @@ -7695,7 +7881,7 @@ static void cpy_1_f16_f16(const char * cxi, char * cdsti) { static void cpy_1_f16_f32(const char * cxi, char * cdsti) { const sycl::half *xi = (const sycl::half *)cxi; - float *dsti = (float *)cdsti; + float * dsti = (float *) cdsti; *dsti = *xi; } @@ -8297,6 +8483,62 @@ static void im2col_kernel(const float *x, T *dst, int offset_delta, } } +template +static void pool2d_nchw_kernel( + const int ih, const int iw, const int oh, const int ow, + const int kh, const int kw, const int sh, const int sw, + const int ph, const int pw, const int parallel_elements, + const Ti* src, To* dst, const enum ggml_op_pool op, + const sycl::nd_item<3> &item_ct1) { + int idx = item_ct1.get_local_id(2) + + item_ct1.get_group(2) * item_ct1.get_local_range(2); + if (idx >= parallel_elements) { + return; + } + + const int I_HW = ih * iw; + const int O_HW = oh * ow; + const int nc = idx / O_HW; + const int cur_oh = idx % O_HW / ow; + const int cur_ow = idx % O_HW % ow; + const Ti* i_ptr = src + nc * I_HW; + To* o_ptr = dst + nc * O_HW; + const int start_h = cur_oh * sh - ph; + const int bh = sycl::max(0, start_h); + const int eh = sycl::min(ih, start_h + kh); + const int start_w = cur_ow * sw - pw; + const int bw = sycl::max(0, start_w); + const int ew = sycl::min(iw, start_w + kw); + + To res = 0; + + switch (op) { + case GGML_OP_POOL_AVG: res = 0; break; + case GGML_OP_POOL_MAX: res = -FLT_MAX; break; + } + + for (int i = bh; i < eh; i += 1) { + for (int j = bw; j < ew; j += 1) { +#if DPCT_COMPATIBILITY_TEMP >= 350 + /* + DPCT1098:106: The '*' expression is used instead of the __ldg + call. These two expressions do not provide the exact same + functionality. Check the generated code for potential precision + and/or performance issues. + */ + Ti cur = *(i_ptr + i * iw + j); +#else + Ti cur = i_ptr[i * iw + j]; +#endif + switch (op) { + case GGML_OP_POOL_AVG: res += (cur / (kh * kw)); break; + case GGML_OP_POOL_MAX: res = sycl::max(res, (To)cur); break; + } + } + } + o_ptr[cur_oh * ow + cur_ow] = res; +} + template static void get_rows_sycl(const ggml_tensor *src0, const ggml_tensor *src1, ggml_tensor *dst, const void *src0_dd, @@ -8585,6 +8827,30 @@ static void relu_f32_sycl(const float *x, float *dst, const int k, }); } +static void hardsigmoid_f32_sycl(const float *x, float *dst, const int k, + dpct::queue_ptr stream) { + const int num_blocks = (k + SYCL_HARDSIGMOID_BLOCK_SIZE - 1) / SYCL_HARDSIGMOID_BLOCK_SIZE; + stream->parallel_for( + sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * + sycl::range<3>(1, 1, SYCL_HARDSIGMOID_BLOCK_SIZE), + sycl::range<3>(1, 1, SYCL_HARDSIGMOID_BLOCK_SIZE)), + [=](sycl::nd_item<3> item_ct1) { + hardsigmoid_f32(x, dst, k, item_ct1); + }); +} + +static void hardswish_f32_sycl(const float *x, float *dst, const int k, + dpct::queue_ptr stream) { + const int num_blocks = (k + SYCL_HARDSWISH_BLOCK_SIZE - 1) / SYCL_HARDSWISH_BLOCK_SIZE; + stream->parallel_for( + sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * + sycl::range<3>(1, 1, SYCL_HARDSWISH_BLOCK_SIZE), + sycl::range<3>(1, 1, SYCL_HARDSWISH_BLOCK_SIZE)), + [=](sycl::nd_item<3> item_ct1) { + hardswish_f32(x, dst, k, item_ct1); + }); +} + static void leaky_relu_f32_sycl(const float *x, float *dst, const int k, const float negative_slope, dpct::queue_ptr stream) { @@ -8811,11 +9077,10 @@ template static void dequantize_block_sycl(const void *__restrict__ vx, dst_t *__restrict__ y, const int k, dpct::queue_ptr stream) { - const int num_blocks = (k + SYCL_DEQUANTIZE_BLOCK_SIZE - 1) / SYCL_DEQUANTIZE_BLOCK_SIZE; + const int num_blocks = (k + 2*SYCL_DEQUANTIZE_BLOCK_SIZE - 1) / (2*SYCL_DEQUANTIZE_BLOCK_SIZE); { dpct::has_capability_or_fail(stream->get_device(), {sycl::aspect::fp16}); - stream->parallel_for( sycl::nd_range<3>( sycl::range<3>(1, 1, num_blocks) * @@ -9208,24 +9473,6 @@ static void mul_mat_vec_q_sycl_submitter(const void *vx, const void *vy, }); } -int get_device_index_by_id(int id){ - int res = g_sycl_device_id2index[id].index; - // GGML_SYCL_DEBUG("get_device_index_by_id id=%d device_index=%d\n", id, res); - GGML_ASSERT(res>=0); - return res; -} - -int get_device_id_by_index(int index){ - int res = g_device_caps[index].device_id; - GGML_ASSERT(res>=0); - return res; -} - - -int get_current_device_index(){ - return get_device_index_by_id(dpct::dev_mgr::instance().current_device_id()); -} - static void ggml_mul_mat_q4_0_q8_1_sycl(const void *vx, const void *vy, float *dst, const int ncols_x, const int nrows_x, const int ncols_y, @@ -9234,7 +9481,7 @@ static void ggml_mul_mat_q4_0_q8_1_sycl(const void *vx, const void *vy, int id; SYCL_CHECK( - CHECK_TRY_ERROR(id = get_current_device_index())); + CHECK_TRY_ERROR(id = get_current_device_id())); const int compute_capability = g_device_caps[id].cc; int mmq_x, mmq_y, nwarps; @@ -9349,7 +9596,7 @@ static void ggml_mul_mat_q4_1_q8_1_sycl(const void *vx, const void *vy, int id; SYCL_CHECK( - CHECK_TRY_ERROR(id = get_current_device_index())); + CHECK_TRY_ERROR(id = get_current_device_id())); const int compute_capability = g_device_caps[id].cc; int mmq_x, mmq_y, nwarps; @@ -9464,7 +9711,7 @@ static void ggml_mul_mat_q5_0_q8_1_sycl(const void *vx, const void *vy, int id; SYCL_CHECK( - CHECK_TRY_ERROR(id = get_current_device_index())); + CHECK_TRY_ERROR(id = get_current_device_id())); const int compute_capability = g_device_caps[id].cc; int mmq_x, mmq_y, nwarps; @@ -9579,7 +9826,7 @@ static void ggml_mul_mat_q5_1_q8_1_sycl(const void *vx, const void *vy, int id; SYCL_CHECK( - CHECK_TRY_ERROR(id = get_current_device_index())); + CHECK_TRY_ERROR(id = get_current_device_id())); const int compute_capability = g_device_caps[id].cc; int mmq_x, mmq_y, nwarps; @@ -9694,7 +9941,7 @@ static void ggml_mul_mat_q8_0_q8_1_sycl(const void *vx, const void *vy, int id; SYCL_CHECK( - CHECK_TRY_ERROR(id = get_current_device_index())); + CHECK_TRY_ERROR(id = get_current_device_id())); const int compute_capability = g_device_caps[id].cc; int mmq_x, mmq_y, nwarps; @@ -9809,7 +10056,7 @@ static void ggml_mul_mat_q2_K_q8_1_sycl(const void *vx, const void *vy, int id; SYCL_CHECK( - CHECK_TRY_ERROR(id = get_current_device_index())); + CHECK_TRY_ERROR(id = get_current_device_id())); const int compute_capability = g_device_caps[id].cc; int mmq_x, mmq_y, nwarps; @@ -9932,7 +10179,7 @@ static void ggml_mul_mat_q3_K_q8_1_sycl(const void *vx, const void *vy, int id; SYCL_CHECK( - CHECK_TRY_ERROR(id = get_current_device_index())); + CHECK_TRY_ERROR(id = get_current_device_id())); const int compute_capability = g_device_caps[id].cc; int mmq_x, mmq_y, nwarps; @@ -10060,7 +10307,7 @@ static void ggml_mul_mat_q4_K_q8_1_sycl(const void *vx, const void *vy, int id; SYCL_CHECK( - CHECK_TRY_ERROR(id = get_current_device_index())); + CHECK_TRY_ERROR(id = get_current_device_id())); const int compute_capability = g_device_caps[id].cc; int mmq_x, mmq_y, nwarps; @@ -10181,7 +10428,7 @@ static void ggml_mul_mat_q5_K_q8_1_sycl(const void *vx, const void *vy, int id; SYCL_CHECK( - CHECK_TRY_ERROR(id = get_current_device_index())); + CHECK_TRY_ERROR(id = get_current_device_id())); const int compute_capability = g_device_caps[id].cc; int mmq_x, mmq_y, nwarps; @@ -10302,7 +10549,7 @@ static void ggml_mul_mat_q6_K_q8_1_sycl(const void *vx, const void *vy, int id; SYCL_CHECK( - CHECK_TRY_ERROR(id = get_current_device_index())); + CHECK_TRY_ERROR(id = get_current_device_id())); const int compute_capability = g_device_caps[id].cc; int mmq_x, mmq_y, nwarps; @@ -10458,6 +10705,31 @@ static void ggml_mul_mat_vec_nc_f16_f32_sycl( } } +static void +ggml_cpy_f16_f32_sycl(const char *cx, char *cdst, const int ne, const int ne00, + const int ne01, const int ne02, const int nb00, + const int nb01, const int nb02, const int nb03, + const int ne10, const int ne11, const int ne12, + const int nb10, const int nb11, const int nb12, + const int nb13, dpct::queue_ptr stream) { + + const int num_blocks = (ne + SYCL_CPY_BLOCK_SIZE - 1) / SYCL_CPY_BLOCK_SIZE; + { + dpct::has_capability_or_fail(stream->get_device(), + {sycl::aspect::fp16}); + + stream->parallel_for( + sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * + sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), + sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), + [=](sycl::nd_item<3> item_ct1) { + cpy_f32_f16(cx, cdst, ne, ne00, ne01, ne02, nb00, + nb01, nb02, nb03, ne10, ne11, ne12, + nb10, nb11, nb12, nb13, item_ct1); + }); + } +} + static void ggml_cpy_f32_f32_sycl(const char *cx, char *cdst, const int ne, const int ne00, const int ne01, const int ne02, const int nb00, @@ -11014,12 +11286,9 @@ struct sycl_buffer { static sycl_buffer g_sycl_buffer_pool[GGML_SYCL_MAX_DEVICES][MAX_SYCL_BUFFERS]; static size_t g_sycl_pool_size[GGML_SYCL_MAX_DEVICES] = {0}; -static void *ggml_sycl_pool_malloc_leg(size_t size, size_t *actual_size) try { +static void *ggml_sycl_pool_malloc_leg(int device_index, size_t size, size_t *actual_size) try { scoped_spin_lock lock(g_sycl_pool_lock); - int id; - SYCL_CHECK( - CHECK_TRY_ERROR(id = get_current_device_index())); - // GGML_SYCL_DEBUG("ggml_sycl_pool_malloc_leg index %d\n", id); + // GGML_SYCL_DEBUG("ggml_sycl_pool_malloc_leg device_index %d size=%lu\n", device_index, size); #ifdef DEBUG_SYCL_MALLOC int nnz = 0; size_t max_size = 0; @@ -11027,7 +11296,7 @@ static void *ggml_sycl_pool_malloc_leg(size_t size, size_t *actual_size) try { size_t best_diff = 1ull << 36; int ibest = -1; for (int i = 0; i < MAX_SYCL_BUFFERS; ++i) { - sycl_buffer& b = g_sycl_buffer_pool[id][i]; + sycl_buffer& b = g_sycl_buffer_pool[device_index][i]; if (b.ptr != nullptr) { #ifdef DEBUG_SYCL_MALLOC ++nnz; @@ -11043,7 +11312,7 @@ static void *ggml_sycl_pool_malloc_leg(size_t size, size_t *actual_size) try { *actual_size = b.size; b.ptr = nullptr; b.size = 0; - // GGML_SYCL_DEBUG("ggml_sycl_pool_malloc_leg return 1 %p\n", ptr); + // GGML_SYCL_DEBUG("ggml_sycl_pool_malloc_leg return 1 %p and rm in pool\n", ptr); return ptr; } } @@ -11051,30 +11320,30 @@ static void *ggml_sycl_pool_malloc_leg(size_t size, size_t *actual_size) try { } } if (ibest >= 0) { - sycl_buffer& b = g_sycl_buffer_pool[id][ibest]; + sycl_buffer& b = g_sycl_buffer_pool[device_index][ibest]; void * ptr = b.ptr; *actual_size = b.size; b.ptr = nullptr; b.size = 0; - // GGML_SYCL_DEBUG("ggml_sycl_pool_malloc_leg return 2 %p\n", ptr); + // GGML_SYCL_DEBUG("ggml_sycl_pool_malloc_leg return 2 %p and rm in pool\n", ptr); return ptr; } void * ptr; size_t look_ahead_size = (size_t) (1.05 * size); look_ahead_size = 256 * ((look_ahead_size + 255)/256); - const dpct::queue_ptr stream = g_syclStreams[id][0]; + const dpct::queue_ptr stream = g_syclStreams[device_index][0]; SYCL_CHECK( CHECK_TRY_ERROR(ptr = (void *)sycl::malloc_device( look_ahead_size, *stream))); *actual_size = look_ahead_size; - g_sycl_pool_size[id] += look_ahead_size; + g_sycl_pool_size[device_index] += look_ahead_size; #ifdef DEBUG_SYCL_MALLOC fprintf(stderr, "%s[%d]: %d buffers, max_size = %u MB, pool_size = %u MB, requested %u MB\n", __func__, id, nnz, (uint32_t)(max_size/1024/1024), (uint32_t)(g_sycl_pool_size[id]/1024/1024), (uint32_t)(size/1024/1024)); #endif - // GGML_SYCL_DEBUG("ggml_sycl_pool_malloc_leg return %p\n", ptr); + // GGML_SYCL_DEBUG("ggml_sycl_pool_malloc_leg look_ahead_size=%lu, return %p\n", look_ahead_size, ptr); return ptr; } catch (sycl::exception const &exc) { @@ -11083,15 +11352,11 @@ catch (sycl::exception const &exc) { std::exit(1); } -static void ggml_sycl_pool_free_leg(void *ptr, size_t size) try { +static void ggml_sycl_pool_free_leg(int device_index, void *ptr, size_t size) try { scoped_spin_lock lock(g_sycl_pool_lock); - int id; - SYCL_CHECK( - CHECK_TRY_ERROR(id = get_current_device_index())); - - const dpct::queue_ptr stream = g_syclStreams[id][0]; + const dpct::queue_ptr stream = g_syclStreams[device_index][0]; for (int i = 0; i < MAX_SYCL_BUFFERS; ++i) { - sycl_buffer& b = g_sycl_buffer_pool[id][i]; + sycl_buffer& b = g_sycl_buffer_pool[device_index][i]; if (b.ptr == nullptr) { b.ptr = ptr; b.size = size; @@ -11100,7 +11365,7 @@ static void ggml_sycl_pool_free_leg(void *ptr, size_t size) try { } fprintf(stderr, "WARNING: sycl buffer pool full, increase MAX_SYCL_BUFFERS\n"); SYCL_CHECK(CHECK_TRY_ERROR(sycl::free(ptr, *stream))); - g_sycl_pool_size[id] -= size; + g_sycl_pool_size[device_index] -= size; } catch (sycl::exception const &exc) { std::cerr << exc.what() << "Exception caught at file:" << __FILE__ @@ -11117,7 +11382,8 @@ DPCT1082:64: Migration of CUmemGenericAllocationHandle type is not supported. static dpct::device_ptr g_sycl_pool_addr[GGML_SYCL_MAX_DEVICES] = {0}; static size_t g_sycl_pool_used[GGML_SYCL_MAX_DEVICES] = {0}; -static void *ggml_sycl_pool_malloc_vmm(size_t size, size_t *actual_size) try { +static void *ggml_sycl_pool_malloc_vmm(int device_index, size_t size, size_t *actual_size) try { + GGML_UNUSED(device_index); GGML_UNUSED(size); GGML_UNUSED(actual_size); return NULL; @@ -11128,20 +11394,16 @@ catch (sycl::exception const &exc) { std::exit(1); } -static void ggml_sycl_pool_free_vmm(void *ptr, size_t size) try { +static void ggml_sycl_pool_free_vmm(int device_index, void *ptr, size_t size) try { scoped_spin_lock lock(g_sycl_pool_lock); - int id; - SYCL_CHECK( - CHECK_TRY_ERROR(id = dpct::dev_mgr::instance().current_device_id())); - #ifdef DEBUG_SYCL_MALLOC - printf("sycl pool[%d]: freed %llu bytes at %llx\n", id, (unsigned long long) size, ptr); + printf("sycl pool[%d]: freed %llu bytes at %llx\n", device_index, (unsigned long long) size, ptr); #endif - g_sycl_pool_used[id] -= size; + g_sycl_pool_used[device_index] -= size; // all deallocations must be in reverse order of the allocations - GGML_ASSERT(ptr == (void *) (g_sycl_pool_addr[id] + g_sycl_pool_used[id])); + GGML_ASSERT(ptr == (void *) (g_sycl_pool_addr[device_index] + g_sycl_pool_used[device_index])); } catch (sycl::exception const &exc) { std::cerr << exc.what() << "Exception caught at file:" << __FILE__ @@ -11149,14 +11411,11 @@ catch (sycl::exception const &exc) { std::exit(1); } -static void *ggml_sycl_pool_malloc(size_t size, size_t *actual_size) try { - int id; - SYCL_CHECK( - CHECK_TRY_ERROR(id = get_current_device_index())); - if (g_device_caps[id].vmm) { - return ggml_sycl_pool_malloc_vmm(size, actual_size); +static void *ggml_sycl_pool_malloc(int device_index, size_t size, size_t *actual_size) try { + if (g_device_caps[device_index].vmm) { + return ggml_sycl_pool_malloc_vmm(device_index, size, actual_size); } else { - return ggml_sycl_pool_malloc_leg(size, actual_size); + return ggml_sycl_pool_malloc_leg(device_index, size, actual_size); } } catch (sycl::exception const &exc) { @@ -11165,14 +11424,11 @@ catch (sycl::exception const &exc) { std::exit(1); } -static void ggml_sycl_pool_free(void *ptr, size_t size) try { - int id; - SYCL_CHECK( - CHECK_TRY_ERROR(id = get_current_device_index())); - if (g_device_caps[id].vmm) { - ggml_sycl_pool_free_vmm(ptr, size); +static void ggml_sycl_pool_free(int device_index, void *ptr, size_t size) try { + if (g_device_caps[device_index].vmm) { + ggml_sycl_pool_free_vmm(device_index, ptr, size); } else { - ggml_sycl_pool_free_leg(ptr, size); + ggml_sycl_pool_free_leg(device_index, ptr, size); } } catch (sycl::exception const &exc) { @@ -11184,13 +11440,17 @@ catch (sycl::exception const &exc) { template struct sycl_pool_alloc { + int device_index = -1; + int device_id = -1; T * ptr = nullptr; size_t actual_size = 0; // size is in number of elements T * alloc(size_t size) { GGML_ASSERT(ptr == nullptr); - ptr = (T *) ggml_sycl_pool_malloc(size * sizeof(T), &this->actual_size); + device_id = get_current_device_id(); + device_index = g_sycl_gpu_mgr->get_index(device_id); + ptr = (T *) ggml_sycl_pool_malloc(device_index, size * sizeof(T), &this->actual_size); // GGML_SYCL_DEBUG("alloc %lu return %p actual size=%lu\n", size * sizeof(T), ptr, this->actual_size); return ptr; } @@ -11201,7 +11461,7 @@ struct sycl_pool_alloc { ~sycl_pool_alloc() { if (ptr != nullptr) { - ggml_sycl_pool_free(ptr, actual_size); + ggml_sycl_pool_free(device_index, ptr, actual_size); } } @@ -11222,44 +11482,57 @@ bool ggml_sycl_loaded(void) { return g_sycl_loaded; } -void ggml_backend_sycl_print_sycl_devices(){ - int device_count = dpct::dev_mgr::instance().device_count(); - fprintf(stderr, "found %d SYCL devices:\n", device_count); - for (int id = 0; id < device_count; ++id) { - dpct::device_info prop; - SYCL_CHECK(CHECK_TRY_ERROR(dpct::get_device_info( - prop, dpct::dev_mgr::instance().get_device(id)))); - sycl::device cur_device = dpct::dev_mgr::instance().get_device(id); - fprintf(stderr, " Device %d: %s,\tcompute capability %d.%d,\n\tmax compute_units %d,\tmax work group size %d,\tmax sub group size %d,\tglobal mem size %lu\n", id, - prop.get_name(), prop.get_major_version(), - prop.get_minor_version(), - prop.get_max_compute_units(), - prop.get_max_work_group_size(), - prop.get_max_sub_group_size(), - prop.get_global_mem_size() - ); - } - // fprintf(stderr, "\n"); +void print_device_detail(int id) { + dpct::device_info prop; + SYCL_CHECK(CHECK_TRY_ERROR( + dpct::get_device_info(prop, dpct::dev_mgr::instance().get_device(id)))); + sycl::device cur_device = dpct::dev_mgr::instance().get_device(id); + std::string version; + version += std::to_string(prop.get_major_version()); + version += "."; + version += std::to_string(prop.get_minor_version()); + + fprintf(stderr, "|%2d|%45s|%18s|%17d|%14d|%13d|%15lu|\n", id, + prop.get_name(), version.c_str(), prop.get_max_compute_units(), + prop.get_max_work_group_size(), prop.get_max_sub_group_size(), + prop.get_global_mem_size()); } -int get_sycl_env(const char* env_name, int default_val){ - char * user_device_string = getenv(env_name); +void ggml_backend_sycl_print_sycl_devices() { + int device_count = dpct::dev_mgr::instance().device_count(); + fprintf(stderr, "found %d SYCL devices:\n", device_count); + fprintf(stderr, "|ID| Name |compute capability|Max compute units|Max work group|Max sub group|Global mem size|\n"); + fprintf(stderr, "|--|---------------------------------------------|------------------|-----------------|--------------|-------------|---------------|\n"); + for (int id = 0; id < device_count; ++id) { + print_device_detail(id); + } +} + +void print_gpu_device_list() { + fprintf(stderr, "detect %d SYCL GPUs: [%s] with Max compute units:%d\n", + g_sycl_gpu_mgr->get_gpu_count(), + g_sycl_gpu_mgr->gpus_list.c_str(), + g_sycl_gpu_mgr->max_compute_units); +} + +int get_sycl_env(const char *env_name, int default_val) { + char *user_device_string = getenv(env_name); int user_number = default_val; unsigned n; - if (user_device_string != NULL && sscanf(user_device_string, " %u", &n) == 1) { - user_number = (int)n; - } else { - user_number=default_val; - } + if (user_device_string != NULL && + sscanf(user_device_string, " %u", &n) == 1) { + user_number = (int)n; + } else { + user_number = default_val; + } return user_number; } -int get_work_group_size(int user_device_id){ +int get_work_group_size(int user_device_id) { dpct::device_info prop; - dpct::get_device_info( - prop, - dpct::dev_mgr::instance().get_device(user_device_id)); + dpct::get_device_info(prop, + dpct::dev_mgr::instance().get_device(user_device_id)); return prop.get_max_work_group_size(); } @@ -11268,113 +11541,81 @@ void ggml_init_sycl() try { if (!initialized) { g_ggml_sycl_debug = get_sycl_env("GGML_SYCL_DEBUG", 0); + fprintf(stderr, "%s: GGML_SYCL_DEBUG: %d\n", __func__, g_ggml_sycl_debug); - printf("GGML_SYCL_DEBUG=%d\n", g_ggml_sycl_debug); - - int user_device_id = get_sycl_env("GGML_SYCL_DEVICE", 0); - +#if defined(GGML_SYCL_F16) + fprintf(stderr, "%s: GGML_SYCL_F16: yes\n", __func__); +#else + fprintf(stderr, "%s: GGML_SYCL_F16: no\n", __func__); +#endif if (CHECK_TRY_ERROR(g_all_sycl_device_count = - dpct::dev_mgr::instance().device_count()) != - 0) { + dpct::dev_mgr::instance().device_count()) != 0) { initialized = true; g_sycl_loaded = false; return; } GGML_ASSERT(g_all_sycl_device_count <= GGML_SYCL_MAX_DEVICES); + ggml_backend_sycl_print_sycl_devices(); + + if (!g_sycl_gpu_mgr) g_sycl_gpu_mgr = new sycl_gpu_mgr(); + + g_device_count = g_sycl_gpu_mgr->get_gpu_count(); + g_work_group_size = g_sycl_gpu_mgr->work_group_size; + + print_gpu_device_list(); + int64_t total_vram = 0; -#if defined(GGML_SYCL_F16) - fprintf(stderr, "%s: GGML_SYCL_F16: yes\n", __func__); -#else - fprintf(stderr, "%s: GGML_SYCL_F16: no\n", __func__); -#endif - - +/* NOT REMOVE, keep it for next optimize for XMX. #if defined(SYCL_USE_XMX) fprintf(stderr, "%s: SYCL_USE_XMX: yes\n", __func__); #else fprintf(stderr, "%s: SYCL_USE_XMX: no\n", __func__); #endif - ggml_backend_sycl_print_sycl_devices(); +*/ for (int id = 0; id < GGML_SYCL_MAX_DEVICES; ++id) { - g_sycl_device_id2index[id].index = -1; g_device_caps[id].vmm = 0; g_device_caps[id].device_id = -1; g_device_caps[id].cc = 0; g_tensor_split[id] = 0; + g_default_tensor_split[id] = 0; } - int device_inx = -1; - for (int id = 0; id < g_all_sycl_device_count; ++id) { - if(id!=user_device_id) continue; - - device_inx++; - - g_device_caps[device_inx].vmm = 0; - g_device_caps[device_inx].device_id = id; - g_sycl_device_id2index[id].index = device_inx; + for (int i = 0; i < g_device_count; ++i) { + int device_id = g_sycl_gpu_mgr->gpus[i]; + g_device_caps[i].vmm = 0; dpct::device_info prop; SYCL_CHECK(CHECK_TRY_ERROR(dpct::get_device_info( - prop, dpct::dev_mgr::instance().get_device(id)))); + prop, dpct::dev_mgr::instance().get_device(device_id)))); - g_tensor_split[device_inx] = total_vram; + g_default_tensor_split[i] = total_vram; total_vram += prop.get_global_mem_size(); - g_device_caps[device_inx].cc = + g_device_caps[i].cc = 100 * prop.get_major_version() + 10 * prop.get_minor_version(); - - } - device_inx = -1; - for (int id = 0; id < g_all_sycl_device_count; ++id) { - if(id!=user_device_id) continue; - device_inx++; - g_tensor_split[device_inx] /= total_vram; } - device_inx = -1; - for (int id = 0; id < g_all_sycl_device_count; ++id) { - if(id!=user_device_id) continue; - device_inx++; - SYCL_CHECK(ggml_sycl_set_device(id)); + for (int i = 0; i < g_device_count; ++i) { + g_default_tensor_split[i] /= total_vram; + } + + for (int i = 0; i < g_device_count; ++i) { + SYCL_CHECK(ggml_sycl_set_device(i)); // create sycl streams for (int is = 0; is < MAX_STREAMS; ++is) { - /* - DPCT1025:88: The SYCL queue is created ignoring the flag and - priority options. - */ SYCL_CHECK(CHECK_TRY_ERROR( - g_syclStreams[device_inx][is] = - dpct::get_current_device().create_queue())); + g_syclStreams[i][is] = + dpct::get_current_device().create_queue( + g_sycl_gpu_mgr->get_co_ctx(), dpct::get_current_device()))); } - const dpct::queue_ptr stream = g_syclStreams[device_inx][0]; + const dpct::queue_ptr stream = g_syclStreams[i][0]; // create sycl handle - SYCL_CHECK(CHECK_TRY_ERROR(g_sycl_handles[device_inx] = - stream)); - /* - DPCT1027:89: The call to syclSetMathMode was replaced with 0 - because this functionality is redundant in SYCL. - */ - SYCL_CHECK(0); + SYCL_CHECK(CHECK_TRY_ERROR(g_sycl_handles[i] = stream)); } - // configure logging to stdout - // SYCL_CHECK(syclLoggerConfigure(1, 1, 0, nullptr)); - - //hardcode, force set to 1 device - g_device_count = 1; - ggml_sycl_set_main_device(user_device_id); - ggml_sycl_set_device(user_device_id); - g_work_group_size = get_work_group_size(user_device_id); - // fprintf(stderr, "Using Device %d\n", user_device_id); - - // for (int id = 0; id < g_all_sycl_device_count; ++id) { - // GGML_SYCL_DEBUG("id=%d g_device_caps[%d].device_id=%d g_sycl_device_id2index[%d].index=%d ", id, id, - // g_device_caps[id].device_id, id, g_sycl_device_id2index[id].index); - // } - initialized = true; g_sycl_loaded = true; } @@ -11385,31 +11626,6 @@ catch (sycl::exception const &exc) { std::exit(1); } - -void ggml_sycl_set_tensor_split(const float * tensor_split) { - if (tensor_split == nullptr) { - return; - } - bool all_zero = true; - for (int i = 0; i < g_device_count; ++i) { - if (tensor_split[i] != 0.0f) { - all_zero = false; - break; - } - } - if (all_zero) { - return; - } - float split_sum = 0.0f; - for (int i = 0; i < g_device_count; ++i) { - g_tensor_split[i] = split_sum; - split_sum += tensor_split[i]; - } - for (int i = 0; i < g_device_count; ++i) { - g_tensor_split[i] /= split_sum; - } -} - void *ggml_sycl_host_malloc(size_t size) try { if (getenv("GGML_SYCL_NO_PINNED") != nullptr) { return nullptr; @@ -11419,28 +11635,14 @@ void *ggml_sycl_host_malloc(size_t size) try { //allow to use dpct::get_in_order_queue() for host malloc dpct::err0 err = CHECK_TRY_ERROR( ptr = (void *)sycl::malloc_host(size, dpct::get_in_order_queue())); - /* - DPCT1000:82: Error handling if-stmt was detected but could not be rewritten. - */ + if (err != 0) { // clear the error - /* - DPCT1026:83: The call to syclGetLastError was removed because this - functionality is redundant in SYCL. - */ - /* - DPCT1001:81: The statement could not be removed. - */ fprintf( stderr, "WARNING: failed to allocate %.2f MB of pinned memory: %s\n", - /* - DPCT1009:84: SYCL uses exceptions to report errors and does not use - the error codes. The original code was commented out and a warning - string was inserted. You need to rewrite this code. - */ size / 1024.0 / 1024.0, - "syclGetErrorString is not supported" /*syclGetErrorString(err)*/); + "syclGetErrorString is not supported"); return nullptr; } @@ -11480,7 +11682,7 @@ static dpct::err0 ggml_sycl_cpy_tensor_2d(void *dst, ggml_tensor_extra_gpu * extra = (ggml_tensor_extra_gpu *) src->extra; int id; SYCL_CHECK(CHECK_TRY_ERROR( - id = get_current_device_index())); + id = get_current_device_id())); // GGML_SYCL_DEBUG("current device index %d\n", id); src_ptr = (char *) extra->data_device[id]; } else { @@ -11714,7 +11916,6 @@ inline void ggml_sycl_op_tanh(const ggml_tensor *src0, const ggml_tensor *src1, GGML_ASSERT(src0->type == GGML_TYPE_F32); GGML_ASSERT( dst->type == GGML_TYPE_F32); - tanh_f32_sycl(src0_dd, dst_dd, ggml_nelements(src0), main_stream); (void) src1; @@ -11737,6 +11938,37 @@ inline void ggml_sycl_op_relu(const ggml_tensor *src0, const ggml_tensor *src1, (void) src1_dd; } +static void ggml_sycl_op_hardsigmoid(const ggml_tensor *src0, + const ggml_tensor *src1, ggml_tensor *dst, + const float *src0_dd, const float *src1_dd, + float *dst_dd, + const dpct::queue_ptr &main_stream) { + + GGML_ASSERT(src0->type == GGML_TYPE_F32); + GGML_ASSERT( dst->type == GGML_TYPE_F32); + + hardsigmoid_f32_sycl(src0_dd, dst_dd, ggml_nelements(src0), main_stream); + + (void) src1; + (void) dst; + (void) src1_dd; +} + +static void ggml_sycl_op_hardswish(const ggml_tensor *src0, + const ggml_tensor *src1, ggml_tensor *dst, + const float *src0_dd, const float *src1_dd, + float *dst_dd, const dpct::queue_ptr &main_stream) { + + GGML_ASSERT(src0->type == GGML_TYPE_F32); + GGML_ASSERT( dst->type == GGML_TYPE_F32); + + hardswish_f32_sycl(src0_dd, dst_dd, ggml_nelements(src0), main_stream); + + (void) src1; + (void) dst; + (void) src1_dd; +} + inline void ggml_sycl_op_leaky_relu(const ggml_tensor *src0, const ggml_tensor *src1, ggml_tensor *dst, const float *src0_dd, const float *src1_dd, @@ -11905,7 +12137,7 @@ inline void ggml_sycl_op_mul_mat_q( int device_id; SYCL_CHECK( - CHECK_TRY_ERROR(device_id = dpct::dev_mgr::instance().current_device_id())); + CHECK_TRY_ERROR(device_id = get_current_device_id())); // the main device has a larger memory buffer to hold the results from all GPUs // nrows_dst == nrows of the matrix that the dequantize_mul_mat kernel writes into @@ -11957,16 +12189,16 @@ catch (sycl::exception const &exc) { std::exit(1); } -static int64_t get_row_rounding(ggml_type type) { +static int64_t get_row_rounding(ggml_type type, const std::array & tensor_split) { int64_t min_compute_capability = INT_MAX; int64_t max_compute_capability = INT_MIN; - for (int64_t id = 0; id < g_device_count; ++id) { - if (g_tensor_split[id] < (id + 1 < g_device_count ? g_tensor_split[id + 1] : 1.0f)) { - if (min_compute_capability > g_device_caps[id].cc) { - min_compute_capability = g_device_caps[id].cc; + for (int i = 0; i < g_device_count; ++i) { + if (tensor_split[i] < (i + 1 < g_device_count ? tensor_split[i + 1] : 1.0f)) { + if (min_compute_capability > g_device_caps[i].cc) { + min_compute_capability = g_device_caps[i].cc; } - if (max_compute_capability < g_device_caps[id].cc) { - max_compute_capability = g_device_caps[id].cc; + if (max_compute_capability < g_device_caps[i].cc) { + max_compute_capability = g_device_caps[i].cc; } } } @@ -11986,12 +12218,16 @@ static int64_t get_row_rounding(ggml_type type) { case GGML_TYPE_Q3_K: case GGML_TYPE_Q4_K: case GGML_TYPE_Q5_K: + case GGML_TYPE_IQ2_XXS: + case GGML_TYPE_IQ2_XS: + case GGML_TYPE_IQ3_XXS: return max_compute_capability >= VER_GEN9 ? 128 : 64; case GGML_TYPE_Q6_K: return 64; default: GGML_ASSERT(false); } + } inline void ggml_sycl_op_mul_mat_vec_q( @@ -12176,27 +12412,22 @@ inline void ggml_sycl_op_mul_mat_sycl( const int64_t row_diff = row_high - row_low; int id; - int device_id = dpct::dev_mgr::instance().current_device_id(); SYCL_CHECK( - CHECK_TRY_ERROR(id = get_current_device_index())); + CHECK_TRY_ERROR(id = get_current_device_id())); // the main device has a larger memory buffer to hold the results from all GPUs // ldc == nrows of the matrix that cuBLAS writes into - int ldc = dst->backend == GGML_BACKEND_TYPE_GPU && device_id == g_main_device ? ne0 : row_diff; + int ldc = dst->backend == GGML_BACKEND_TYPE_GPU && id == g_main_device ? ne0 : row_diff; #ifdef GGML_SYCL_F16 bool use_fp16 = true; // TODO(Yu) SYCL capability check #else bool use_fp16 = false; #endif - // if (compute_capability >= VER_GEN9 && (src0->type == GGML_TYPE_F16 || - // ggml_is_quantized(src0->type)) && ggml_is_contiguous(src0) && row_diff == - // src0->ne[1] && dst->op_params[0] == GGML_PREC_DEFAULT) { if ((src0->type == GGML_TYPE_F16 || ggml_is_quantized(src0->type)) && use_fp16 && ggml_is_contiguous(src0) && row_diff == src0->ne[1] && dst->op_params[0] == GGML_PREC_DEFAULT) { - // convert src0 and src1 to fp16, multiply as fp16, convert dst to fp32 // GGML_SYCL_DEBUG("ggml_sycl_op_mul_mat_sycl - fp16 path\n"); sycl_pool_alloc src0_as_f16; if (src0->type != GGML_TYPE_F16) { @@ -12225,7 +12456,6 @@ inline void ggml_sycl_op_mul_mat_sycl( const sycl::half alpha_f16 = 1.0f; const sycl::half beta_f16 = 0.0f; - SYCL_CHECK(CHECK_TRY_ERROR(g_sycl_handles[id] = stream)); SYCL_CHECK(CHECK_TRY_ERROR(dpct::gemm( *g_sycl_handles[id], oneapi::mkl::transpose::trans, @@ -12241,14 +12471,21 @@ inline void ggml_sycl_op_mul_mat_sycl( else { // GGML_SYCL_DEBUG("ggml_sycl_op_mul_mat_sycl - fp32 path\n"); sycl_pool_alloc src0_ddq_as_f32; - + sycl_pool_alloc src1_ddq_as_f32; if (src0->type != GGML_TYPE_F32) { const to_fp32_sycl_t to_fp32_sycl = ggml_get_to_fp32_sycl(src0->type); GGML_ASSERT(to_fp32_sycl != nullptr); src0_ddq_as_f32.alloc(row_diff*ne00); to_fp32_sycl(src0_dd_i, src0_ddq_as_f32.get(), row_diff*ne00, stream); } + if (src1->type != GGML_TYPE_F32) { + const to_fp32_sycl_t to_fp32_sycl = ggml_get_to_fp32_sycl(src1->type); + GGML_ASSERT(to_fp32_sycl != nullptr); + src1_ddq_as_f32.alloc(src1_ncols*ne10); + to_fp32_sycl(src1_ddf_i, src1_ddq_as_f32.get(), src1_ncols*ne10, stream); + } const float * src0_ddf_i = src0->type == GGML_TYPE_F32 ? (const float *) src0_dd_i : src0_ddq_as_f32.get(); + const float * src1_ddf1_i = src1->type == GGML_TYPE_F32 ? (const float *) src1_ddf_i : src1_ddq_as_f32.get(); const float alpha = 1.0f; const float beta = 0.0f; @@ -12261,7 +12498,6 @@ inline void ggml_sycl_op_mul_mat_sycl( src1_ddf_i, ne10, dpct::get_value(&beta, *g_sycl_handles[id]), dst_dd_i, ldc))); } - (void) dst; (void) src1_ddq_i; (void) src1_padded_row_size; @@ -12382,6 +12618,48 @@ inline void ggml_sycl_op_alibi(const ggml_tensor *src0, const ggml_tensor *src1, (void) src1_dd; } +static void ggml_sycl_op_pool2d(const ggml_tensor *src0, + const ggml_tensor *src1, ggml_tensor *dst, + const float *src0_dd, const float *src1_dd, + float *dst_dd, const dpct::queue_ptr &main_stream) { + + GGML_ASSERT(src0->type == GGML_TYPE_F32); + GGML_ASSERT( dst->type == GGML_TYPE_F32); + + const int32_t * opts = (const int32_t *)dst->op_params; + enum ggml_op_pool op = static_cast(opts[0]); + const int k0 = opts[1]; + const int k1 = opts[2]; + const int s0 = opts[3]; + const int s1 = opts[4]; + const int p0 = opts[5]; + const int p1 = opts[6]; + + const int64_t IH = src0->ne[1]; + const int64_t IW = src0->ne[0]; + + const int64_t N = dst->ne[3]; + const int64_t OC = dst->ne[2]; + const int64_t OH = dst->ne[1]; + const int64_t OW = dst->ne[0]; + + const int parallel_elements = N * OC * OH * OW; + const int num_blocks = (parallel_elements + SYCL_POOL2D_BLOCK_SIZE - 1) / SYCL_POOL2D_BLOCK_SIZE; + sycl::range<3> block_nums(1, 1, num_blocks); + main_stream->parallel_for( + sycl::nd_range<3>(block_nums * + sycl::range<3>(1, 1, SYCL_IM2COL_BLOCK_SIZE), + sycl::range<3>(1, 1, SYCL_IM2COL_BLOCK_SIZE)), + [=](sycl::nd_item<3> item_ct1) { + pool2d_nchw_kernel(IH, IW, OH, OW, k1, k0, s1, s0, p1, p0, + parallel_elements, src0_dd, dst_dd, op, + item_ct1); + }); + + (void) src1; + (void) src1_dd; +} + inline void ggml_sycl_op_im2col(const ggml_tensor *src0, const ggml_tensor *src1, ggml_tensor *dst, const float *src0_dd, const float *src1_dd, @@ -12606,12 +12884,12 @@ static void ggml_sycl_op_flatten(const ggml_tensor *src0, sycl_pool_alloc dst_f; ggml_sycl_set_device(g_main_device); - dpct::queue_ptr main_stream = g_syclStreams[g_main_device_index][0]; - // GGML_SYCL_DEBUG("g_main_device_index=%d, main_stream=%p src0_on_device=%d, src1_on_device=%d, dst_on_device=%d\n", - // g_main_device_index, main_stream, src0_on_device, src1_on_device, dst_on_device); + dpct::queue_ptr main_stream = g_syclStreams[g_main_device][0]; + // GGML_SYCL_DEBUG("g_main_device=%d, main_stream=%p src0_on_device=%d, src1_on_device=%d, dst_on_device=%d\n", + // g_main_device, main_stream, src0_on_device, src1_on_device, dst_on_device); if (src0_on_device) { - src0_ddf = (float *) src0_extra->data_device[g_main_device_index]; + src0_ddf = (float *) src0_extra->data_device[g_main_device]; } else { src0_ddf = src0_f.alloc(ggml_nelements(src0)); // GGML_SYCL_DEBUG("before ggml_sycl_cpy_tensor_2d src0_ddf=%p, src0=%p\n", src0_ddf, src0); @@ -12620,15 +12898,14 @@ static void ggml_sycl_op_flatten(const ggml_tensor *src0, if (use_src1) { if (src1_on_device) { - src1_ddf = (float *) src1_extra->data_device[g_main_device_index]; + src1_ddf = (float *) src1_extra->data_device[g_main_device]; } else { src1_ddf = src1_f.alloc(ggml_nelements(src1)); SYCL_CHECK(ggml_sycl_cpy_tensor_2d(src1_ddf, src1, 0, 0, 0, nrows1, main_stream)); } } if (dst_on_device) { - dst_ddf = (float *) dst_extra->data_device[g_main_device_index]; - // printf("zjy dst_ddf=%p main_stream=%p g_main_device_index=%d\n", dst_ddf, main_stream, g_main_device_index); + dst_ddf = (float *) dst_extra->data_device[g_main_device]; } else { dst_ddf = dst_f.alloc(ggml_nelements(dst)); } @@ -12672,21 +12949,19 @@ static void ggml_sycl_set_peer_access(const int n_tokens) { } #ifdef NDEBUG - for (int id = 0; id < g_device_count; ++id) { - SYCL_CHECK(ggml_sycl_set_device(get_device_id_by_index(id))); + for (int i = 0; i < g_device_count; ++i) { + SYCL_CHECK(ggml_sycl_set_device(i)); // SYCL_CHECK(syclDeviceSynchronize()); } - for (int id = 0; id < g_device_count; ++id) { - SYCL_CHECK(ggml_sycl_set_device(get_device_id_by_index(id))); - int device_id = g_device_caps[id].device_id; + for (int i = 0; i < g_device_count; ++i) { + SYCL_CHECK(ggml_sycl_set_device(i)); for (int id_other = 0; id_other < g_device_count; ++id_other) { - int device_id_other = g_device_caps[id_other].device_id; - if (device_id == id_other) { + if (i == id_other) { continue; } - if (device_id != g_main_device && device_id_other != g_main_device) { + if (i != g_main_device && id_other != g_main_device) { continue; } @@ -12706,6 +12981,10 @@ static void ggml_sycl_set_peer_access(const int n_tokens) { peer_access_enabled = enable_peer_access; } +struct ggml_backend_sycl_split_buffer_type_context { + std::array tensor_split; +}; + static void ggml_sycl_op_mul_mat(const ggml_tensor *src0, const ggml_tensor *src1, ggml_tensor *dst, ggml_sycl_op_mul_mat_t op, @@ -12752,80 +13031,90 @@ static void ggml_sycl_op_mul_mat(const ggml_tensor *src0, GGML_ASSERT(!(split && ne03 > 1)); GGML_ASSERT(!(split && ne02 < ne12)); - // dd = data device - char * src0_dd[GGML_SYCL_MAX_DEVICES] = {nullptr}; - float * src1_ddf[GGML_SYCL_MAX_DEVICES] = {nullptr}; // float - char * src1_ddq[GGML_SYCL_MAX_DEVICES] = {nullptr}; // q8_1 - float * dst_dd[GGML_SYCL_MAX_DEVICES] = {nullptr}; + std::array tensor_split; + if (split) { + // TODO: check that src0->buffer->buft is a split buffer type, replace GGML_BACKEND_TYPE_GPU_SPLIT check + // GGML_ASSERT(src0->buffer != nullptr && src0->buffer->buft == ...); + ggml_backend_sycl_split_buffer_type_context * buft_ctx = (ggml_backend_sycl_split_buffer_type_context *) src0->buffer->buft->context; + tensor_split = buft_ctx->tensor_split; + } - // as = actual size - size_t src0_as[GGML_SYCL_MAX_DEVICES] = {0}; - size_t src1_asf[GGML_SYCL_MAX_DEVICES] = {0}; - size_t src1_asq[GGML_SYCL_MAX_DEVICES] = {0}; - size_t dst_as[GGML_SYCL_MAX_DEVICES] = {0}; + struct dev_data { + sycl_pool_alloc src0_dd_alloc; + sycl_pool_alloc src1_ddf_alloc; + sycl_pool_alloc src1_ddq_alloc; + sycl_pool_alloc dst_dd_alloc; - int64_t row_low[GGML_SYCL_MAX_DEVICES]; - int64_t row_high[GGML_SYCL_MAX_DEVICES]; + char *src0_dd = nullptr; + float *src1_ddf = nullptr; // float + char *src1_ddq = nullptr; // q8_1 + float *dst_dd = nullptr; + + int64_t row_low; + int64_t row_high; + }; + + dev_data dev[GGML_SYCL_MAX_DEVICES]; int used_devices = 0; + dpct::queue_ptr main_stream = g_syclStreams[g_main_device][0]; - for (int64_t id = 0; id < g_device_count; ++id) { + for (int i = 0; i < g_device_count; ++i) { // by default, use all rows - row_low[id] = 0; - row_high[id] = ne01; + dev[i].row_low = 0; + dev[i].row_high = ne01; // for multi GPU, get the row boundaries from tensor split // and round to mul_mat_q tile sizes if (split) { - const int64_t rounding = get_row_rounding(src0->type); + const int64_t rounding = get_row_rounding(src0->type, tensor_split); - if (id != 0) { - row_low[id] = ne01*g_tensor_split[id]; - if (row_low[id] < ne01) { - row_low[id] -= row_low[id] % rounding; + if (i != 0) { + dev[i].row_low = ne01*tensor_split[i]; + if (dev[i].row_low < ne01) { + dev[i].row_low -= dev[i].row_low % rounding; } } - if (id != g_device_count - 1) { - row_high[id] = ne01*g_tensor_split[id + 1]; - if (row_high[id] < ne01) { - row_high[id] -= row_high[id] % rounding; + if (i != g_device_count - 1) { + dev[i].row_high = ne01*tensor_split[i + 1]; + if (dev[i].row_high < ne01) { + dev[i].row_high -= dev[i].row_high % rounding; } } } } - for (int64_t id = 0; id < g_device_count; ++id) { - if ((!split && id != g_main_device_index) || row_low[id] == row_high[id]) { + for (int i = 0; i < g_device_count; ++i) { + if ((!split && i != g_main_device) || dev[i].row_low == dev[i].row_high) { continue; } used_devices++; - const bool src1_on_device = src1->backend == GGML_BACKEND_TYPE_GPU && id == g_main_device_index; - const bool dst_on_device = dst->backend == GGML_BACKEND_TYPE_GPU && id == g_main_device_index; + const bool src1_on_device = src1->backend == GGML_BACKEND_TYPE_GPU && i == g_main_device; + const bool dst_on_device = dst->backend == GGML_BACKEND_TYPE_GPU && i == g_main_device; - ggml_sycl_set_device(get_device_id_by_index(id)); - const dpct::queue_ptr stream = g_syclStreams[id][0]; + ggml_sycl_set_device(i); + dpct::queue_ptr stream = g_syclStreams[i][0]; if (src0_on_device && src0_is_contiguous) { - src0_dd[id] = (char *) src0_extra->data_device[id]; + dev[i].src0_dd = (char *) src0_extra->data_device[i]; } else { - // const size_t size_src0_ddq = split ? (row_high[id]-row_low[id])*ne00 * src0_ts/src0_bs : ggml_nbytes(src0); - src0_dd[id] = (char *) ggml_sycl_pool_malloc(ggml_nbytes(src0), &src0_as[id]); + dev[i].src0_dd = dev[i].src0_dd_alloc.alloc(ggml_nbytes(src0)); } if (src1_on_device && src1_is_contiguous) { - src1_ddf[id] = (float *) src1_extra->data_device[id]; + dev[i].src1_ddf = (float *) src1_extra->data_device[i]; } else { - src1_ddf[id] = (float *) ggml_sycl_pool_malloc(ggml_nbytes(src1), &src1_asf[id]); + dev[i].src1_ddf = dev[i].src1_ddf_alloc.alloc(ggml_nelements(src1)); } if (convert_src1_to_q8_1) { - src1_ddq[id] = (char *) ggml_sycl_pool_malloc(nrows1*src1_padded_col_size*q8_1_ts/q8_1_bs, &src1_asq[id]); + dev[i].src1_ddq = dev[i].src1_ddq_alloc.alloc(nrows1*src1_padded_col_size*q8_1_ts/q8_1_bs); if (src1_on_device && src1_is_contiguous) { - quantize_row_q8_1_sycl(src1_ddf[id], src1_ddq[id], ne10, nrows1, src1_padded_col_size, stream); + quantize_row_q8_1_sycl(dev[i].src1_ddf, dev[i].src1_ddq, ne10, nrows1, src1_padded_col_size, stream); /* DPCT1010:90: SYCL uses exceptions to report errors and does not use the error codes. The call was replaced with 0. You need to @@ -12836,25 +13125,25 @@ static void ggml_sycl_op_mul_mat(const ggml_tensor *src0, } if (dst_on_device) { - dst_dd[id] = (float *) dst_extra->data_device[id]; + dev[i].dst_dd = (float *) dst_extra->data_device[i]; } else { - const size_t size_dst_ddf = split ? (row_high[id]-row_low[id])*ne1*sizeof(float) : ggml_nbytes(dst); - dst_dd[id] = (float *) ggml_sycl_pool_malloc(size_dst_ddf, &dst_as[id]); + const size_t size_dst_ddf = split ? (dev[i].row_high - dev[i].row_low)*ne1 : ggml_nelements(dst); + dev[i].dst_dd = dev[i].dst_dd_alloc.alloc(size_dst_ddf); } } // if multiple devices are used they need to wait for the main device // here an event is recorded that signals that the main device has finished calculating the input data if (split && used_devices > 1) { - SYCL_CHECK(ggml_sycl_set_device(g_main_device)); + ggml_sycl_set_device(g_main_device); /* DPCT1024:91: The original code returned the error code that was further consumed by the program logic. This original code was replaced with 0. You may need to rewrite the program logic consuming the error code. */ SYCL_CHECK(CHECK_TRY_ERROR( - *src0_extra->events[g_main_device_index][0] = - g_syclStreams[g_main_device_index][0]->ext_oneapi_submit_barrier())); + *src0_extra->events[g_main_device][0] = + g_syclStreams[g_main_device][0]->ext_oneapi_submit_barrier())); } const int64_t src1_col_stride = split && used_devices > 1 ? MUL_MAT_SRC1_COL_STRIDE : ne11; @@ -12862,22 +13151,27 @@ static void ggml_sycl_op_mul_mat(const ggml_tensor *src0, const int64_t is = split ? (src1_col_0/src1_col_stride) % MAX_STREAMS : 0; const int64_t src1_ncols = src1_col_0 + src1_col_stride > ne11 ? ne11 - src1_col_0 : src1_col_stride; - for (int64_t id = 0; id < g_device_count; ++id) { - if ((!split && id != g_main_device_index) || row_low[id] == row_high[id]) { + for (int i = 0; i < g_device_count; ++i) { + if ((!split && i != g_main_device) || dev[i].row_low == dev[i].row_high) { continue; } - const bool src1_on_device = src1->backend == GGML_BACKEND_TYPE_GPU && id == g_main_device_index; - const bool dst_on_device = dst->backend == GGML_BACKEND_TYPE_GPU && id == g_main_device_index; - const int64_t row_diff = row_high[id] - row_low[id]; + const bool src1_on_device = src1->backend == GGML_BACKEND_TYPE_GPU && i == g_main_device; + const bool dst_on_device = dst->backend == GGML_BACKEND_TYPE_GPU && i == g_main_device; + const int64_t row_diff = dev[i].row_high - dev[i].row_low; - ggml_sycl_set_device(get_device_id_by_index(id)); - const dpct::queue_ptr stream = g_syclStreams[id][is]; + ggml_sycl_set_device(i); + dpct::queue_ptr stream = g_syclStreams[i][is]; // wait for main GPU data if necessary - if (split && (id != g_main_device_index || is != 0)) { + if (split && (i != g_main_device || is != 0)) { + /* + DPCT1009:163: SYCL uses exceptions to report errors and does not + use the error codes. The original code was commented out and a + warning string was inserted. You need to rewrite this code. + */ SYCL_CHECK(CHECK_TRY_ERROR(stream->ext_oneapi_submit_barrier( - {*src0_extra->events[g_main_device_index][0]}))); + {*src0_extra->events[g_main_device][0]}))); } for (int64_t i0 = 0; i0 < ne13*ne12; ++i0) { @@ -12887,30 +13181,32 @@ static void ggml_sycl_op_mul_mat(const ggml_tensor *src0, const size_t src1_ddq_i_offset = (i0*ne11 + src1_col_0) * src1_padded_col_size*q8_1_ts/q8_1_bs; // for split tensors the data begins at i0 == i0_offset_low - char * src0_dd_i = src0_dd[id] + (i0/i02_divisor) * (ne01*ne00*src0_ts)/src0_bs; - float * src1_ddf_i = src1_ddf[id] + (i0*ne11 + src1_col_0) * ne10; - char * src1_ddq_i = src1_ddq[id] + src1_ddq_i_offset; - float * dst_dd_i = dst_dd[id] + (i0*ne1 + src1_col_0) * (dst_on_device ? ne0 : row_diff); + char * src0_dd_i = dev[i].src0_dd + (i0/i02_divisor) * (ne01*ne00*src0_ts)/src0_bs; + float * src1_ddf_i = dev[i].src1_ddf + (i0*ne11 + src1_col_0) * ne10; + char * src1_ddq_i = dev[i].src1_ddq + src1_ddq_i_offset; + float * dst_dd_i = dev[i].dst_dd + (i0*ne1 + src1_col_0) * (dst_on_device ? ne0 : row_diff); // the main device memory buffer can be on VRAM scratch, with space for all partial results // in that case an offset on dst_ddf_i is needed - if (dst->backend == GGML_BACKEND_TYPE_GPU && id == g_main_device_index) { - dst_dd_i += row_low[id]; // offset is 0 if no tensor split + if (dst->backend == GGML_BACKEND_TYPE_GPU && i == g_main_device) { + dst_dd_i += dev[i].row_low; // offset is 0 if no tensor split } // copy src0, src1 to device if necessary if (src1->backend == GGML_BACKEND_TYPE_GPU && src1_is_contiguous) { - if (id != g_main_device_index) { + if (i != g_main_device) { if (convert_src1_to_q8_1) { - char * src1_ddq_i_source = src1_ddq[g_main_device_index] + src1_ddq_i_offset; - SYCL_CHECK(CHECK_TRY_ERROR(stream->memcpy( + char * src1_ddq_i_source = dev[g_main_device].src1_ddq + src1_ddq_i_offset; + SYCL_CHECK(CHECK_TRY_ERROR(stream->memcpy( src1_ddq_i, src1_ddq_i_source, src1_ncols * src1_padded_col_size * q8_1_ts / q8_1_bs))); } else { - float * src1_ddf_i_source = (float *) src1_extra->data_device[g_main_device_index]; + + float * src1_ddf_i_source = (float *) src1_extra->data_device[g_main_device]; src1_ddf_i_source += (i0*ne11 + src1_col_0) * ne10; - SYCL_CHECK(CHECK_TRY_ERROR(stream->memcpy( + + SYCL_CHECK(CHECK_TRY_ERROR(dev2dev_memcpy(*stream, *main_stream, src1_ddf_i, src1_ddf_i_source, src1_ncols * ne10 * sizeof(float)))); } @@ -12933,14 +13229,14 @@ static void ggml_sycl_op_mul_mat(const ggml_tensor *src0, } if (src1_col_0 == 0 && (!src0_on_device || !src0_is_contiguous) && i02 % i02_divisor == 0) { - SYCL_CHECK(ggml_sycl_cpy_tensor_2d(src0_dd_i, src0, i03, i02/i02_divisor, row_low[id], row_high[id], stream)); + SYCL_CHECK(ggml_sycl_cpy_tensor_2d(src0_dd_i, src0, i03, i02/i02_divisor, dev[i].row_low, dev[i].row_high, stream)); } if (src1->type == GGML_TYPE_F16) { src1_padded_col_size = (i0 * ne11 + src1_col_0) * ne10; } // do the computation op(src0, src1, dst, src0_dd_i, src1_ddf_i, src1_ddq_i, dst_dd_i, - row_low[id], row_high[id], src1_ncols, src1_padded_col_size, stream); + dev[i].row_low, dev[i].row_high, src1_ncols, src1_padded_col_size, stream); /* DPCT1010:93: SYCL uses exceptions to report errors and does not use the error codes. The call was replaced with 0. You need to @@ -12956,7 +13252,7 @@ static void ggml_sycl_op_mul_mat(const ggml_tensor *src0, dst_off_device = dst->data; kind = dpct::device_to_host; } else if (dst->backend == GGML_BACKEND_TYPE_GPU) { - dst_off_device = dst_extra->data_device[g_main_device_index]; + dst_off_device = dst_extra->data_device[g_main_device]; kind = dpct::device_to_device; } else { GGML_ASSERT(false); @@ -12969,11 +13265,29 @@ static void ggml_sycl_op_mul_mat(const ggml_tensor *src0, // If dst is a vector with ne0 == 1 then you don't have to do this but it still produces correct results. float * dhf_dst_i = (float *) ((char *) dst_off_device + i02*nb2 + i03*nb3); GGML_ASSERT(dst->nb[1] == ne0*sizeof(float)); - dhf_dst_i += src1_col_0*ne0 + row_low[id]; - SYCL_CHECK(CHECK_TRY_ERROR(dpct::async_dpct_memcpy( - dhf_dst_i, ne0 * sizeof(float), dst_dd_i, - row_diff * sizeof(float), row_diff * sizeof(float), - src1_ncols, kind, *stream))); + dhf_dst_i += src1_col_0*ne0 + dev[i].row_low; + + //todo, dirty solution. Need be updated when device2device memcpy() is supported. + if (kind == dpct::device_to_device) { + size_t dst_size = ggml_nbytes_pad(dst); + float *host_buf = (float *)malloc(dst_size); + SYCL_CHECK(CHECK_TRY_ERROR(dpct::async_dpct_memcpy( + host_buf, ne0 * sizeof(float), dst_dd_i, + row_diff * sizeof(float), row_diff * sizeof(float), + src1_ncols, dpct::device_to_host, *stream))); + dpct::dev_mgr::instance().get_device(g_sycl_gpu_mgr->gpus[i]).queues_wait_and_throw(); + SYCL_CHECK(CHECK_TRY_ERROR(dpct::async_dpct_memcpy( + dhf_dst_i, ne0 * sizeof(float), host_buf, + row_diff * sizeof(float), row_diff * sizeof(float), + src1_ncols, dpct::host_to_device, *main_stream))); + dpct::dev_mgr::instance().get_device(g_sycl_gpu_mgr->gpus[g_main_device]).queues_wait_and_throw(); + free(host_buf); + } else { + SYCL_CHECK(CHECK_TRY_ERROR(dpct::async_dpct_memcpy( + dhf_dst_i, ne0 * sizeof(float), dst_dd_i, + row_diff * sizeof(float), row_diff * sizeof(float), + src1_ncols, kind, *stream))); + } } else { float * dhf_dst_i = (float *) ((char *) dst_off_device + i02*nb2 + i03*nb3); GGML_ASSERT(dst->nb[1] == ne0*sizeof(float)); @@ -12985,7 +13299,7 @@ static void ggml_sycl_op_mul_mat(const ggml_tensor *src0, } // add event for the main device to wait on until other device is done - if (split && (id != g_main_device_index || is != 0)) { + if (split && (i != g_main_device || is != 0)) { /* DPCT1024:94: The original code returned the error code that was further consumed by the program logic. This original @@ -12993,48 +13307,27 @@ static void ggml_sycl_op_mul_mat(const ggml_tensor *src0, program logic consuming the error code. */ SYCL_CHECK(CHECK_TRY_ERROR( - *src0_extra->events[id][is] = + *src0_extra->events[i][is] = stream->ext_oneapi_submit_barrier())); } } } } - for (int64_t id = 0; id < g_device_count; ++id) { - if ((!split && id != g_main_device_index) || row_low[id] == row_high[id]) { - continue; - } - SYCL_CHECK(ggml_sycl_set_device(get_device_id_by_index(id))); - - // free buffers again when done - if (dst_as[id] > 0) { - ggml_sycl_pool_free(dst_dd[id], dst_as[id]); - } - if (src1_asq[id] > 0) { - ggml_sycl_pool_free(src1_ddq[id], src1_asq[id]); - } - if (src1_asf[id] > 0) { - ggml_sycl_pool_free(src1_ddf[id], src1_asf[id]); - } - if (src0_as[id] > 0) { - ggml_sycl_pool_free(src0_dd[id], src0_as[id]); - } - } - // main device waits for all other devices to be finished if (split && g_device_count > 1) { int64_t is_max = (ne11 + MUL_MAT_SRC1_COL_STRIDE - 1) / MUL_MAT_SRC1_COL_STRIDE; is_max = is_max <= MAX_STREAMS ? is_max : MAX_STREAMS; - SYCL_CHECK(ggml_sycl_set_device(g_main_device)); - for (int64_t id = 0; id < g_device_count; ++id) { - if (row_low[id] == row_high[id]) { + ggml_sycl_set_device(g_main_device); + for (int i = 0; i < g_device_count; ++i) { + if (dev[i].row_low == dev[i].row_high) { continue; } for (int64_t is = 0; is < is_max; ++is) { SYCL_CHECK(CHECK_TRY_ERROR( - g_syclStreams[g_main_device_index][0]->ext_oneapi_submit_barrier( - {*src0_extra->events[id][is]}))); + g_syclStreams[g_main_device][0]->ext_oneapi_submit_barrier( + {*src0_extra->events[i][is]}))); } } } @@ -13051,110 +13344,132 @@ catch (sycl::exception const &exc) { std::exit(1); } + static void ggml_sycl_repeat(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { GGML_SYCL_DEBUG("call %s\n", __func__); ggml_sycl_op_flatten(src0, src1, dst, ggml_sycl_op_repeat); + GGML_SYCL_DEBUG("call %s done\n", __func__); } static void ggml_sycl_get_rows(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { GGML_SYCL_DEBUG("call %s\n", __func__); ggml_sycl_op_flatten(src0, src1, dst, ggml_sycl_op_get_rows); + GGML_SYCL_DEBUG("call %s done\n", __func__); } static void ggml_sycl_add(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { GGML_SYCL_DEBUG("call %s\n", __func__); ggml_sycl_op_flatten(src0, src1, dst, ggml_sycl_op_add); - // log_tensor_with_cnt("log_ggml_sycl_add_src0", (struct ggml_tensor *) src0, 6); - // log_tensor_with_cnt("log_ggml_sycl_add_src1", (struct ggml_tensor *)src1, 6); - // log_tensor_with_cnt("log_ggml_sycl_add_dst", dst, 6); + GGML_SYCL_DEBUG("call %s done\n", __func__); } static void ggml_sycl_acc(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { GGML_SYCL_DEBUG("call %s\n", __func__); ggml_sycl_op_flatten(src0, src1, dst, ggml_sycl_op_acc); + GGML_SYCL_DEBUG("call %s done\n", __func__); } static void ggml_sycl_mul(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { GGML_SYCL_DEBUG("call %s\n", __func__); ggml_sycl_op_flatten(src0, src1, dst, ggml_sycl_op_mul); - // log_tensor_with_cnt("log_ggml_sycl_mul_src0", (struct ggml_tensor *)src0, 6); - // log_tensor_with_cnt("log_ggml_sycl_mul_src1", (struct ggml_tensor *)src1, 6); - // log_tensor_with_cnt("log_ggml_sycl_mul_dst", dst, 6); - + GGML_SYCL_DEBUG("call %s done\n", __func__); } static void ggml_sycl_div(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { GGML_SYCL_DEBUG("call %s\n", __func__); ggml_sycl_op_flatten(src0, src1, dst, ggml_sycl_op_div); + GGML_SYCL_DEBUG("call %s done\n", __func__); } static void ggml_sycl_gelu(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { GGML_SYCL_DEBUG("call %s\n", __func__); ggml_sycl_op_flatten(src0, src1, dst, ggml_sycl_op_gelu); + GGML_SYCL_DEBUG("call %s done\n", __func__); } static void ggml_sycl_silu(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { GGML_SYCL_DEBUG("call %s\n", __func__); ggml_sycl_op_flatten(src0, src1, dst, ggml_sycl_op_silu); + GGML_SYCL_DEBUG("call %s done\n", __func__); } static void ggml_sycl_gelu_quick(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { GGML_SYCL_DEBUG("call %s\n", __func__); ggml_sycl_op_flatten(src0, src1, dst, ggml_sycl_op_gelu_quick); + GGML_SYCL_DEBUG("call %s done\n", __func__); } static void ggml_sycl_tanh(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { GGML_SYCL_DEBUG("call %s\n", __func__); ggml_sycl_op_flatten(src0, src1, dst, ggml_sycl_op_tanh); + GGML_SYCL_DEBUG("call %s done\n", __func__); } static void ggml_sycl_relu(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { GGML_SYCL_DEBUG("call %s\n", __func__); ggml_sycl_op_flatten(src0, src1, dst, ggml_sycl_op_relu); + GGML_SYCL_DEBUG("call %s done\n", __func__); +} + +static void ggml_sycl_hardsigmoid(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + GGML_SYCL_DEBUG("call %s\n", __func__); + ggml_sycl_op_flatten(src0, src1, dst, ggml_sycl_op_hardsigmoid); + GGML_SYCL_DEBUG("call %s done\n", __func__); +} + +static void ggml_sycl_hardswish(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + GGML_SYCL_DEBUG("call %s\n", __func__); + ggml_sycl_op_flatten(src0, src1, dst, ggml_sycl_op_hardswish); + GGML_SYCL_DEBUG("call %s done\n", __func__); } static void ggml_sycl_leaky_relu(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { GGML_SYCL_DEBUG("call %s\n", __func__); ggml_sycl_op_flatten(src0, src1, dst, ggml_sycl_op_leaky_relu); + GGML_SYCL_DEBUG("call %s done\n", __func__); } static void ggml_sycl_sqr(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { GGML_SYCL_DEBUG("call %s\n", __func__); ggml_sycl_op_flatten(src0, src1, dst, ggml_sycl_op_sqr); + GGML_SYCL_DEBUG("call %s done\n", __func__); } static void ggml_sycl_norm(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { GGML_SYCL_DEBUG("call %s\n", __func__); ggml_sycl_op_flatten(src0, src1, dst, ggml_sycl_op_norm); + GGML_SYCL_DEBUG("call %s done\n", __func__); } static void ggml_sycl_group_norm(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { GGML_SYCL_DEBUG("call %s\n", __func__); ggml_sycl_op_flatten(src0, src1, dst, ggml_sycl_op_group_norm); + GGML_SYCL_DEBUG("call %s done\n", __func__); } static void ggml_sycl_concat(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { GGML_SYCL_DEBUG("call %s\n", __func__); ggml_sycl_op_flatten(src0, src1, dst, ggml_sycl_op_concat); + GGML_SYCL_DEBUG("call %s done\n", __func__); } static void ggml_sycl_upscale(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { GGML_SYCL_DEBUG("call %s\n", __func__); ggml_sycl_op_flatten(src0, src1, dst, ggml_sycl_op_upscale); + GGML_SYCL_DEBUG("call %s done\n", __func__); } static void ggml_sycl_pad(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { GGML_SYCL_DEBUG("call %s\n", __func__); ggml_sycl_op_flatten(src0, src1, dst, ggml_sycl_op_pad); + GGML_SYCL_DEBUG("call %s done\n", __func__); } static void ggml_sycl_rms_norm(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { GGML_SYCL_DEBUG("call %s\n", __func__); ggml_sycl_op_flatten(src0, src1, dst, ggml_sycl_op_rms_norm); - // log_tensor_with_cnt("log_ggml_sycl_rms_norm_src0", (struct ggml_tensor *)src0, 6); - // log_tensor_with_cnt("log_ggml_sycl_rms_norm_src1", (struct ggml_tensor *)src1, 6); - // log_tensor_with_cnt("log_ggml_sycl_rms_norm_dst", dst, 6); + GGML_SYCL_DEBUG("call %s done\n", __func__); } bool ggml_sycl_can_mul_mat(const struct ggml_tensor * src0, const struct ggml_tensor * src1, struct ggml_tensor * dst) { @@ -13189,16 +13504,16 @@ static void ggml_sycl_mul_mat_vec_p021(const ggml_tensor *src0, const int64_t ne12 = src1->ne[2]; SYCL_CHECK(ggml_sycl_set_device(g_main_device)); - dpct::queue_ptr main_stream = g_syclStreams[g_main_device_index][0]; + dpct::queue_ptr main_stream = g_syclStreams[g_main_device][0]; ggml_tensor_extra_gpu * src0_extra = (ggml_tensor_extra_gpu *) src0->extra; - void * src0_ddq = src0_extra->data_device[g_main_device_index]; + void * src0_ddq = src0_extra->data_device[g_main_device]; ggml_tensor_extra_gpu * src1_extra = (ggml_tensor_extra_gpu *) src1->extra; - float * src1_ddf = (float *) src1_extra->data_device[g_main_device_index]; + float * src1_ddf = (float *) src1_extra->data_device[g_main_device]; ggml_tensor_extra_gpu * dst_extra = (ggml_tensor_extra_gpu *) dst->extra; - float * dst_ddf = (float *) dst_extra->data_device[g_main_device_index]; + float * dst_ddf = (float *) dst_extra->data_device[g_main_device]; ggml_mul_mat_p021_f16_f32_sycl(src0_ddq, src1_ddf, dst_ddf, ne00, ne01, ne02, ne12, main_stream); } @@ -13228,16 +13543,16 @@ static void ggml_sycl_mul_mat_vec_nc(const ggml_tensor *src0, const int64_t ne12 = src1->ne[2]; SYCL_CHECK(ggml_sycl_set_device(g_main_device)); - dpct::queue_ptr main_stream = g_syclStreams[g_main_device_index][0]; + dpct::queue_ptr main_stream = g_syclStreams[g_main_device][0]; ggml_tensor_extra_gpu * src0_extra = (ggml_tensor_extra_gpu *) src0->extra; - void * src0_ddq = src0_extra->data_device[g_main_device_index]; + void * src0_ddq = src0_extra->data_device[g_main_device]; ggml_tensor_extra_gpu * src1_extra = (ggml_tensor_extra_gpu *) src1->extra; - float * src1_ddf = (float *) src1_extra->data_device[g_main_device_index]; + float * src1_ddf = (float *) src1_extra->data_device[g_main_device]; ggml_tensor_extra_gpu * dst_extra = (ggml_tensor_extra_gpu *) dst->extra; - float * dst_ddf = (float *) dst_extra->data_device[g_main_device_index]; + float * dst_ddf = (float *) dst_extra->data_device[g_main_device]; const int64_t row_stride_x = nb01 / sizeof(sycl::half); const int64_t channel_stride_x = nb02 / sizeof(sycl::half); @@ -13280,38 +13595,37 @@ static void ggml_sycl_mul_mat_batched_sycl(const ggml_tensor *src0, ggml_tensor *dst) try { GGML_ASSERT(!ggml_is_transposed(src0)); GGML_ASSERT(!ggml_is_transposed(src1)); - GGML_ASSERT(src0->backend != GGML_BACKEND_TYPE_GPU_SPLIT); GGML_ASSERT(src0->type == GGML_TYPE_F16); GGML_TENSOR_BINARY_OP_LOCALS - const int64_t ne_dst = ggml_nelements(dst); + const int64_t ne_dst = ggml_nelements(dst); SYCL_CHECK(ggml_sycl_set_device(g_main_device)); - dpct::queue_ptr main_stream = g_syclStreams[g_main_device_index][0]; + dpct::queue_ptr main_stream = g_syclStreams[g_main_device][0]; SYCL_CHECK( - CHECK_TRY_ERROR(g_sycl_handles[g_main_device_index] = main_stream)); + CHECK_TRY_ERROR(g_sycl_handles[g_main_device] = main_stream)); ggml_tensor_extra_gpu * src0_extra = (ggml_tensor_extra_gpu *) src0->extra; - void * src0_ddq = src0_extra->data_device[g_main_device_index]; + void * src0_ddq = src0_extra->data_device[g_main_device]; sycl::half *src0_as_f16 = (sycl::half *)src0_ddq; ggml_tensor_extra_gpu * src1_extra = (ggml_tensor_extra_gpu *) src1->extra; - float * src1_ddf = (float *) src1_extra->data_device[g_main_device_index]; + float * src1_ddf = (float *) src1_extra->data_device[g_main_device]; ggml_tensor_extra_gpu * dst_extra = (ggml_tensor_extra_gpu *) dst->extra; - float * dst_ddf = (float *) dst_extra->data_device[g_main_device_index]; + float * dst_ddf = (float *) dst_extra->data_device[g_main_device]; // convert src1 to fp16 sycl_pool_alloc src1_f16_alloc; if (src1->type != GGML_TYPE_F16) { - const to_fp16_sycl_t to_fp16_sycl = ggml_get_to_fp16_sycl(src1->type); - const int64_t ne_src1 = ggml_nelements(src1); - src1_f16_alloc.alloc(ne_src1); - GGML_ASSERT(to_fp16_sycl != nullptr); - to_fp16_sycl(src1_ddf, src1_f16_alloc.get(), ne_src1, main_stream); + const to_fp16_sycl_t to_fp16_sycl = ggml_get_to_fp16_sycl(src1->type); + const int64_t ne_src1 = ggml_nelements(src1); + src1_f16_alloc.alloc(ne_src1); + GGML_ASSERT(to_fp16_sycl != nullptr); + to_fp16_sycl(src1_ddf, src1_f16_alloc.get(), ne_src1, main_stream); } sycl::half *src1_f16 = src1->type == GGML_TYPE_F16 ? (sycl::half *)src1_ddf : src1_f16_alloc.get(); @@ -13358,7 +13672,7 @@ static void ggml_sycl_mul_mat_batched_sycl(const ggml_tensor *src0, int i02 = i12 / r2; SYCL_CHECK( - syclGemmEx(g_sycl_handles[g_main_device_index], CUBLAS_OP_T, CUBLAS_OP_N, + syclGemmEx(g_sycl_handles[g_main_device], CUBLAS_OP_T, CUBLAS_OP_N, ne01, ne11, ne10, alpha, (const char *) src0_as_f16 + i02*src0->nb[2] + i03*src0->nb[3] , SYCL_R_16F, nb01/sizeof(half), (const char *) src1_as_f16 + i12*src1->nb[2]/2 + i13*src1->nb[3]/2, SYCL_R_16F, nb11/sizeof(float), @@ -13371,9 +13685,8 @@ static void ggml_sycl_mul_mat_batched_sycl(const ggml_tensor *src0, #else if (r2 == 1 && r3 == 1 && src0->nb[2]*src0->ne[2] == src0->nb[3] && src1->nb[2]*src1->ne[2] == src1->nb[3]) { // there is no broadcast and src0, src1 are contiguous across dims 2, 3 - // use syclGemmStridedBatchedEx SYCL_CHECK(CHECK_TRY_ERROR(dpct::gemm_batch( - *g_sycl_handles[g_main_device_index], oneapi::mkl::transpose::trans, + *g_sycl_handles[g_main_device], oneapi::mkl::transpose::trans, oneapi::mkl::transpose::nontrans, ne01, ne11, ne10, alpha, (const char *)src0_as_f16, dpct::library_data_t::real_half, nb01 / nb00, nb02 / nb00, @@ -13382,7 +13695,6 @@ static void ggml_sycl_mul_mat_batched_sycl(const ggml_tensor *src0, (char *)dst_t, cu_data_type, ne01, nb2 / nb0, ne12 * ne13, cu_compute_type))); } else { - // use syclGemmBatchedEx const int ne23 = ne12*ne13; sycl_pool_alloc ptrs_src(2*ne23); @@ -13415,7 +13727,7 @@ static void ggml_sycl_mul_mat_batched_sycl(const ggml_tensor *src0, }); } SYCL_CHECK(CHECK_TRY_ERROR(dpct::gemm_batch( - *g_sycl_handles[g_main_device_index], oneapi::mkl::transpose::trans, + *g_sycl_handles[g_main_device], oneapi::mkl::transpose::trans, oneapi::mkl::transpose::nontrans, ne01, ne11, ne10, alpha, (const void **)(ptrs_src.get() + 0 * ne23), dpct::library_data_t::real_half, nb01 / nb00, @@ -13435,6 +13747,7 @@ catch (sycl::exception const &exc) { std::exit(1); } + static void ggml_sycl_mul_mat(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { const bool all_on_device = (src0->backend == GGML_BACKEND_TYPE_GPU || src0->backend == GGML_BACKEND_TYPE_GPU_SPLIT) && @@ -13444,9 +13757,9 @@ static void ggml_sycl_mul_mat(const ggml_tensor * src0, const ggml_tensor * src1 const bool split = src0->backend == GGML_BACKEND_TYPE_GPU_SPLIT; int64_t min_compute_capability = INT_MAX; - for (int64_t id = 0; id < g_device_count; ++id) { - if (min_compute_capability > g_device_caps[id].cc && g_tensor_split[id] < (id + 1 < g_device_count ? g_tensor_split[id + 1] : 1.0f)) { - min_compute_capability = g_device_caps[id].cc; + for (int i = 0; i < g_device_count; ++i) { + if (min_compute_capability > g_device_caps[i].cc && g_tensor_split[i] < (i + 1 < g_device_count ? g_tensor_split[i + 1] : 1.0f)) { + min_compute_capability = g_device_caps[i].cc; } } @@ -13587,30 +13900,30 @@ static void ggml_sycl_mul_mat_id_sycl(ggml_tensor * dst) { const int64_t ne = ggml_nelements(dst); SYCL_CHECK(ggml_sycl_set_device(g_main_device)); - syclStream_t main_stream = g_syclStreams[g_main_device_index][0]; + syclStream_t main_stream = g_syclStreams[g_main_device][0]; - SYCL_CHECK(syclSetStream(g_sycl_handles[g_main_device_index], main_stream)); + SYCL_CHECK(syclSetStream(g_sycl_handles[g_main_device], main_stream)); //ggml_tensor_extra_gpu * src0_extra = (ggml_tensor_extra_gpu *) src0->extra; - //void * src0_ddq = src0_extra->data_device[g_main_device_index]; + //void * src0_ddq = src0_extra->data_device[g_main_device]; //half * src0_as_f16 = (half *) src0_ddq; ggml_tensor_extra_gpu * src1_extra = (ggml_tensor_extra_gpu *) src1->extra; - float * src1_ddf = (float *) src1_extra->data_device[g_main_device_index]; + float * src1_ddf = (float *) src1_extra->data_device[g_main_device]; ggml_tensor_extra_gpu * dst_extra = (ggml_tensor_extra_gpu *) dst->extra; - float * dst_ddf = (float *) dst_extra->data_device[g_main_device_index]; + float * dst_ddf = (float *) dst_extra->data_device[g_main_device]; // convert src1 to fp16 const to_fp16_sycl_t to_fp16_sycl = ggml_get_to_fp16_sycl(src1->type); GGML_ASSERT(to_fp16_sycl != nullptr); size_t src1_as = 0; - half * src1_as_f16 = (half *) ggml_sycl_pool_malloc(ne1 * sizeof(half), &src1_as); + half * src1_as_f16 = (half *) ggml_sycl_pool_malloc(g_main_device, ne1 * sizeof(half), &src1_as); to_fp16_sycl(src1_ddf, src1_as_f16, ne1, main_stream); size_t dst_as = 0; - half * dst_f16 = (half *) ggml_sycl_pool_malloc(ne * sizeof(half), &dst_as); + half * dst_f16 = (half *) ggml_sycl_pool_malloc(g_main_device, ne * sizeof(half), &dst_as); GGML_ASSERT(ne12 % ne02 == 0); GGML_ASSERT(ne13 % ne03 == 0); @@ -13631,14 +13944,14 @@ static void ggml_sycl_mul_mat_id_sycl(ggml_tensor * dst) { size_t ptrs_src_s = 0; size_t ptrs_dst_s = 0; - ptrs_src = (const void **) ggml_sycl_pool_malloc(2*ne23*sizeof(void *), &ptrs_src_s); - ptrs_dst = ( void **) ggml_sycl_pool_malloc(1*ne23*sizeof(void *), &ptrs_dst_s); + ptrs_src = (const void **) ggml_sycl_pool_malloc(g_main_device, 2*ne23*sizeof(void *), &ptrs_src_s); + ptrs_dst = ( void **) ggml_sycl_pool_malloc(g_main_device, 1*ne23*sizeof(void *), &ptrs_dst_s); int64_t src0_ne = ggml_nelements(src00); half * src0_as_f16 = nullptr; size_t src0_as = 0; if (src00->type != GGML_TYPE_F16) { - src0_as_f16 = (half *) ggml_sycl_pool_malloc(src0_ne * sizeof(half), &src0_as); + src0_as_f16 = (half *) ggml_sycl_pool_malloc(g_main_device, src0_ne * sizeof(half), &src0_as); } static_assert(GGML_MAX_SRC == 6, "GGML_MAX_SRC == 6"); @@ -13653,16 +13966,16 @@ static void ggml_sycl_mul_mat_id_sycl(ggml_tensor * dst) { r2, r3, src00->type, src0_as_f16, src0_ne, src1_as_f16, dst_f16, - (const int *)((ggml_tensor_extra_gpu *)ids->extra)->data_device[g_main_device_index], id, - dst->src[2] ? (const half *)((ggml_tensor_extra_gpu *)dst->src[2]->extra)->data_device[g_main_device_index] : nullptr, - dst->src[3] ? (const half *)((ggml_tensor_extra_gpu *)dst->src[3]->extra)->data_device[g_main_device_index] : nullptr, - dst->src[4] ? (const half *)((ggml_tensor_extra_gpu *)dst->src[4]->extra)->data_device[g_main_device_index] : nullptr, - dst->src[5] ? (const half *)((ggml_tensor_extra_gpu *)dst->src[5]->extra)->data_device[g_main_device_index] : nullptr + (const int *)((ggml_tensor_extra_gpu *)ids->extra)->data_device[g_main_device], id, + dst->src[2] ? (const half *)((ggml_tensor_extra_gpu *)dst->src[2]->extra)->data_device[g_main_device] : nullptr, + dst->src[3] ? (const half *)((ggml_tensor_extra_gpu *)dst->src[3]->extra)->data_device[g_main_device] : nullptr, + dst->src[4] ? (const half *)((ggml_tensor_extra_gpu *)dst->src[4]->extra)->data_device[g_main_device] : nullptr, + dst->src[5] ? (const half *)((ggml_tensor_extra_gpu *)dst->src[5]->extra)->data_device[g_main_device] : nullptr ); SYCL_CHECK(syclGetLastError()); SYCL_CHECK( - syclGemmBatchedEx(g_sycl_handles[g_main_device_index], CUBLAS_OP_T, CUBLAS_OP_N, + syclGemmBatchedEx(g_sycl_handles[g_main_device], CUBLAS_OP_T, CUBLAS_OP_N, ne01, ne11, ne10, &alpha_f16, (const void **) (ptrs_src + 0*ne23), SYCL_R_16F, ne00, (const void **) (ptrs_src + 1*ne23), SYCL_R_16F, ne10, @@ -13672,20 +13985,20 @@ static void ggml_sycl_mul_mat_id_sycl(ggml_tensor * dst) { CUBLAS_GEMM_DEFAULT_TENSOR_OP)); if (src0_as != 0) { - ggml_sycl_pool_free(src0_as_f16, src0_as); + ggml_sycl_pool_free(g_main_device, src0_as_f16, src0_as); } if (ptrs_src_s != 0) { - ggml_sycl_pool_free(ptrs_src, ptrs_src_s); + ggml_sycl_pool_free(g_main_device, ptrs_src, ptrs_src_s); } if (ptrs_dst_s != 0) { - ggml_sycl_pool_free(ptrs_dst, ptrs_dst_s); + ggml_sycl_pool_free(g_main_device, ptrs_dst, ptrs_dst_s); } const to_fp32_sycl_t to_fp32_sycl = ggml_get_to_fp32_sycl(GGML_TYPE_F16); to_fp32_sycl(dst_f16, dst_ddf, ne, main_stream); - ggml_sycl_pool_free(src1_as_f16, src1_as); - ggml_sycl_pool_free(dst_f16, dst_as); + ggml_sycl_pool_free(g_main_device, src1_as_f16, src1_as); + ggml_sycl_pool_free(g_main_device, dst_f16, dst_as); } #endif @@ -13706,10 +14019,10 @@ static void ggml_sycl_mul_mat_id(const ggml_tensor *src0, std::vector ids_host(ggml_nbytes(ids)); - const dpct::queue_ptr stream = g_syclStreams[g_main_device_index][0]; + const dpct::queue_ptr stream = g_syclStreams[g_main_device][0]; if (ids->backend == GGML_BACKEND_TYPE_GPU) { - const char * ids_dev = (const char *)((const ggml_tensor_extra_gpu *)ids->extra)->data_device[g_main_device_index]; + const char * ids_dev = (const char *)((const ggml_tensor_extra_gpu *)ids->extra)->data_device[g_main_device]; SYCL_CHECK(CHECK_TRY_ERROR( stream->memcpy(ids_host.data(), ids_dev, ggml_nbytes(ids)))); SYCL_CHECK(CHECK_TRY_ERROR(stream->wait())); @@ -13733,9 +14046,9 @@ static void ggml_sycl_mul_mat_id(const ggml_tensor *src0, dst_row.extra = &dst_row_extra; char * src1_original = src1->backend == GGML_BACKEND_TYPE_CPU ? - (char *) src1->data : (char *) src1_extra->data_device[g_main_device_index]; + (char *) src1->data : (char *) src1_extra->data_device[g_main_device]; char * dst_original = dst->backend == GGML_BACKEND_TYPE_CPU ? - (char *) dst->data : (char *) dst_extra->data_device[g_main_device_index]; + (char *) dst->data : (char *) dst_extra->data_device[g_main_device]; if (src1->ne[1] == 1) { GGML_ASSERT(src1->backend == GGML_BACKEND_TYPE_GPU); @@ -13752,10 +14065,10 @@ static void ggml_sycl_mul_mat_id(const ggml_tensor *src0, const struct ggml_tensor * src0_row = dst->src[row_id + 2]; - src1_row_extra.data_device[g_main_device_index] = src1_original + i01*src1->nb[1]; + src1_row_extra.data_device[g_main_device] = src1_original + i01*src1->nb[1]; src1_row.data = (char *) src1->data + i01*src1->nb[1]; // TODO why is this set? - dst_row_extra.data_device[g_main_device_index] = dst_original + i01*dst->nb[1]; + dst_row_extra.data_device[g_main_device] = dst_original + i01*dst->nb[1]; dst_row.data = (char *) dst->data + i01*dst->nb[1]; // TODO why is this set? ggml_sycl_mul_mat(src0_row, &src1_row, &dst_row); @@ -13764,8 +14077,8 @@ static void ggml_sycl_mul_mat_id(const ggml_tensor *src0, sycl_pool_alloc src1_contiguous(sizeof(float)*ggml_nelements(src1)); sycl_pool_alloc dst_contiguous(sizeof(float)*ggml_nelements(dst)); - src1_row_extra.data_device[g_main_device_index] = src1_contiguous.get(); - dst_row_extra.data_device[g_main_device_index] = dst_contiguous.get(); + src1_row_extra.data_device[g_main_device] = src1_contiguous.get(); + dst_row_extra.data_device[g_main_device] = dst_contiguous.get(); for (int32_t row_id = 0; row_id < n_as; ++row_id) { const struct ggml_tensor * src0_row = dst->src[row_id + 2]; @@ -13853,13 +14166,13 @@ static void ggml_sycl_cpy(const ggml_tensor *src0, const ggml_tensor *src1, GGML_TENSOR_BINARY_OP_LOCALS; SYCL_CHECK(ggml_sycl_set_device(g_main_device)); - dpct::queue_ptr main_stream = g_syclStreams[g_main_device_index][0]; + dpct::queue_ptr main_stream = g_syclStreams[g_main_device][0]; const ggml_tensor_extra_gpu * src0_extra = (ggml_tensor_extra_gpu *) src0->extra; const ggml_tensor_extra_gpu * src1_extra = (ggml_tensor_extra_gpu *) src1->extra; - char * src0_ddc = (char *) src0_extra->data_device[g_main_device_index]; - char * src1_ddc = (char *) src1_extra->data_device[g_main_device_index]; + char * src0_ddc = (char *) src0_extra->data_device[g_main_device]; + char * src1_ddc = (char *) src1_extra->data_device[g_main_device]; if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_F32) { ggml_cpy_f32_f32_sycl (src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream); @@ -13871,6 +14184,8 @@ static void ggml_sycl_cpy(const ggml_tensor *src0, const ggml_tensor *src1, ggml_cpy_f32_q4_0_sycl(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream); } else if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_Q4_1) { ggml_cpy_f32_q4_1_sycl(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream); + } else if (src0->type == GGML_TYPE_F16 && src1->type == GGML_TYPE_F32) { + ggml_cpy_f16_f32_sycl (src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream); } else if (src0->type == GGML_TYPE_F16 && src1->type == GGML_TYPE_F16) { ggml_cpy_f16_f16_sycl (src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream); } else if (src0->type == GGML_TYPE_I16 && src1->type == GGML_TYPE_I16) { @@ -13914,6 +14229,10 @@ static void ggml_sycl_alibi(const ggml_tensor * src0, const ggml_tensor * src1, ggml_sycl_op_flatten(src0, src1, dst, ggml_sycl_op_alibi); } +static void ggml_sycl_pool2d(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + ggml_sycl_op_flatten(src0, src1, dst, ggml_sycl_op_pool2d); +} + static void ggml_sycl_im2col(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { ggml_sycl_op_flatten(src0, src1, dst, ggml_sycl_op_im2col); } @@ -13940,93 +14259,6 @@ static size_t ggml_nbytes_split(const struct ggml_tensor * tensor, int nrows_spl return nrows_split*ggml_row_size(tensor->type, tensor->ne[0]); } -void ggml_sycl_transform_tensor(void *data, struct ggml_tensor *tensor) try { - const int64_t nrows = ggml_nrows(tensor); - - const int64_t ne0 = tensor->ne[0]; - - const size_t nb1 = tensor->nb[1]; - - ggml_backend_type backend = tensor->backend; - ggml_tensor_extra_gpu * extra = new struct ggml_tensor_extra_gpu; - memset(extra, 0, sizeof(*extra)); - - for (int64_t id = 0; id < g_device_count; ++id) { - if (backend == GGML_BACKEND_TYPE_GPU && id != g_main_device_index) { - continue; - } - ggml_sycl_set_device(get_device_id_by_index(id)); - const dpct::queue_ptr stream = g_syclStreams[id][0]; - - int64_t row_low, row_high; - if (backend == GGML_BACKEND_TYPE_GPU) { - row_low = 0; - row_high = nrows; - } else if (backend == GGML_BACKEND_TYPE_GPU_SPLIT) { - const int64_t rounding = get_row_rounding(tensor->type); - - row_low = id == 0 ? 0 : nrows*g_tensor_split[id]; - row_low -= row_low % rounding; - - if (id == g_device_count - 1) { - row_high = nrows; - } else { - row_high = nrows*g_tensor_split[id + 1]; - row_high -= row_high % rounding; - } - } else { - GGML_ASSERT(false); - } - if (row_low == row_high) { - continue; - } - - int64_t nrows_split = row_high - row_low; - - const size_t offset_split = row_low*nb1; - size_t size = ggml_nbytes_split(tensor, nrows_split); - const size_t original_size = size; - - // pad last row to a multiple of 512 elements to avoid out-of-bounds memory accesses - if (ne0 % MATRIX_ROW_PADDING != 0) { - size += ggml_row_size(tensor->type, MATRIX_ROW_PADDING - ne0 % MATRIX_ROW_PADDING); - } - - char * buf; - SYCL_CHECK(CHECK_TRY_ERROR(buf = (char *)sycl::malloc_device( - size, *stream))); - char * buf_host = (char *)data + offset_split; - - // set padding to 0 to avoid possible NaN values - if (size > original_size) { - SYCL_CHECK(CHECK_TRY_ERROR( - (*stream) - .memset(buf + original_size, 0, size - original_size) - .wait())); - } - - SYCL_CHECK(CHECK_TRY_ERROR((*stream) - .memcpy(buf, buf_host, original_size) - .wait())); - - extra->data_device[id] = buf; - - if (backend == GGML_BACKEND_TYPE_GPU_SPLIT) { - for (int64_t is = 0; is < MAX_STREAMS; ++is) { - SYCL_CHECK(CHECK_TRY_ERROR(extra->events[id][is] = - new sycl::event())); - } - } - } - - tensor->extra = extra; -} -catch (sycl::exception const &exc) { - std::cerr << exc.what() << "Exception caught at file:" << __FILE__ - << ", line:" << __LINE__ << std::endl; - std::exit(1); -} - void ggml_sycl_free_data(struct ggml_tensor *tensor) try { if (!tensor || !tensor->extra || (tensor->backend != GGML_BACKEND_TYPE_GPU && tensor->backend != GGML_BACKEND_TYPE_GPU_SPLIT) ) { return; @@ -14034,18 +14266,18 @@ void ggml_sycl_free_data(struct ggml_tensor *tensor) try { ggml_tensor_extra_gpu * extra = (ggml_tensor_extra_gpu *) tensor->extra; - for (int64_t id = 0; id < g_device_count; ++id) { - const dpct::queue_ptr stream = g_syclStreams[id][0]; - if (extra->data_device[id] != nullptr) { - SYCL_CHECK(ggml_sycl_set_device(get_device_id_by_index(id))); - SYCL_CHECK(CHECK_TRY_ERROR(sycl::free(extra->data_device[id], *stream))); + for (int i = 0; i < g_device_count; ++i) { + const dpct::queue_ptr stream = g_syclStreams[i][0]; + if (extra->data_device[i] != nullptr) { + SYCL_CHECK(ggml_sycl_set_device(i)); + SYCL_CHECK(CHECK_TRY_ERROR(sycl::free(extra->data_device[i], *stream))); } for (int64_t is = 0; is < MAX_STREAMS; ++is) { - if (extra->events[id][is] != nullptr) { - SYCL_CHECK(ggml_sycl_set_device(get_device_id_by_index(id))); + if (extra->events[i][is] != nullptr) { + SYCL_CHECK(ggml_sycl_set_device(i)); SYCL_CHECK(CHECK_TRY_ERROR( - dpct::destroy_event(extra->events[id][is]))); + dpct::destroy_event(extra->events[i][is]))); } } } @@ -14105,22 +14337,22 @@ static void ggml_sycl_assign_buffers_impl(struct ggml_tensor *tensor, const size_t size = ggml_nbytes(tensor); SYCL_CHECK(ggml_sycl_set_device(g_main_device)); - const dpct::queue_ptr stream = g_syclStreams[g_main_device_index][0]; + const dpct::queue_ptr stream = g_syclStreams[g_main_device][0]; if (inplace && (tensor->src[0]->backend == GGML_BACKEND_TYPE_GPU || tensor->src[0]->backend == GGML_BACKEND_TYPE_GPU_SPLIT)) { ggml_tensor_extra_gpu * src0_extra = (ggml_tensor_extra_gpu * ) tensor->src[0]->extra; - char * src0_ddc = (char *) src0_extra->data_device[g_main_device_index]; + char * src0_ddc = (char *) src0_extra->data_device[g_main_device]; size_t offset = 0; if (tensor->op == GGML_OP_VIEW) { memcpy(&offset, tensor->op_params, sizeof(size_t)); } extra = ggml_sycl_alloc_temp_tensor_extra(); - extra->data_device[g_main_device_index] = src0_ddc + offset; + extra->data_device[g_main_device] = src0_ddc + offset; } else if (tensor->op == GGML_OP_CPY) { ggml_tensor_extra_gpu * src1_extra = (ggml_tensor_extra_gpu * ) tensor->src[1]->extra; - void * src1_ddv = src1_extra->data_device[g_main_device_index]; + void * src1_ddv = src1_extra->data_device[g_main_device]; extra = ggml_sycl_alloc_temp_tensor_extra(); - extra->data_device[g_main_device_index] = src1_ddv; + extra->data_device[g_main_device] = src1_ddv; } else if (scratch) { GGML_ASSERT(size <= g_scratch_size); if (g_scratch_offset + size > g_scratch_size) { @@ -14135,7 +14367,7 @@ static void ggml_sycl_assign_buffers_impl(struct ggml_tensor *tensor, g_scratch_buffer = data; } extra = ggml_sycl_alloc_temp_tensor_extra(); - extra->data_device[g_main_device_index] = data + g_scratch_offset; + extra->data_device[g_main_device] = data + g_scratch_offset; g_scratch_offset += size; @@ -14148,44 +14380,7 @@ static void ggml_sycl_assign_buffers_impl(struct ggml_tensor *tensor, (*stream).memset(data, 0, size).wait())); extra = new ggml_tensor_extra_gpu; memset(extra, 0, sizeof(*extra)); - extra->data_device[g_main_device_index] = data; - } - - tensor->extra = extra; -} -catch (sycl::exception const &exc) { - std::cerr << exc.what() << "Exception caught at file:" << __FILE__ - << ", line:" << __LINE__ << std::endl; - std::exit(1); -} - -void ggml_sycl_assign_scratch_offset(struct ggml_tensor *tensor, - size_t offset) try { - if (g_scratch_size == 0) { - return; - } - if (g_scratch_buffer == nullptr) { - ggml_sycl_set_device(g_main_device); - const dpct::queue_ptr stream = g_syclStreams[g_main_device_index][0]; - SYCL_CHECK( - CHECK_TRY_ERROR(g_scratch_buffer = (void *)sycl::malloc_device( - g_scratch_size, *stream))); - } - - ggml_tensor_extra_gpu * extra = ggml_sycl_alloc_temp_tensor_extra(); - - const bool inplace = tensor->view_src != nullptr; - - if (inplace && (tensor->view_src->backend == GGML_BACKEND_TYPE_GPU || tensor->view_src->backend == GGML_BACKEND_TYPE_GPU_SPLIT)) { - ggml_tensor_extra_gpu * src0_extra = (ggml_tensor_extra_gpu * ) tensor->view_src->extra; - char * src0_ddc = (char *) src0_extra->data_device[g_main_device_index]; - size_t view_offset = 0; - if (tensor->op == GGML_OP_VIEW) { - memcpy(&view_offset, tensor->op_params, sizeof(size_t)); - } - extra->data_device[g_main_device_index] = src0_ddc + view_offset; - } else { - extra->data_device[g_main_device_index] = (char *) g_scratch_buffer + offset; + extra->data_device[g_main_device] = data; } tensor->extra = extra; @@ -14202,9 +14397,9 @@ void ggml_sycl_copy_to_device(struct ggml_tensor *tensor) try { ggml_tensor_extra_gpu * extra = (ggml_tensor_extra_gpu *) tensor->extra; SYCL_CHECK(ggml_sycl_set_device(g_main_device)); - const dpct::queue_ptr stream = g_syclStreams[g_main_device_index][0]; + const dpct::queue_ptr stream = g_syclStreams[g_main_device][0]; SYCL_CHECK(CHECK_TRY_ERROR((*stream) - .memcpy(extra->data_device[g_main_device_index], + .memcpy(extra->data_device[g_main_device], tensor->data, ggml_nbytes(tensor)) .wait())); } @@ -14231,21 +14426,17 @@ void ggml_sycl_assign_buffers_force_inplace(struct ggml_tensor * tensor) { } void ggml_sycl_set_main_device(const int main_device) try { + if (g_main_device == main_device) return; + check_allow_gpu_index(main_device); + g_main_device = main_device; + g_main_device_id = g_sycl_gpu_mgr->gpus[main_device]; - if (main_device >= g_all_sycl_device_count) { - fprintf(stderr, "warning: cannot set main_device=%d because there are only %d devices. Using device %d instead.\n", - main_device, g_all_sycl_device_count, g_main_device); - return; - } - - if (g_main_device != main_device && g_device_count >= 1) { - g_main_device = main_device; - g_main_device_index = get_device_index_by_id(g_main_device); + if (g_ggml_sycl_debug) { dpct::device_info prop; SYCL_CHECK(CHECK_TRY_ERROR(dpct::get_device_info( - prop, dpct::dev_mgr::instance().get_device(g_main_device)))); + prop, dpct::dev_mgr::instance().get_device(g_main_device_id)))); fprintf(stderr, "Using device %d (%s) as main device\n", - g_main_device, prop.get_name()); + g_main_device_id, prop.get_name()); } } catch (sycl::exception const &exc) { @@ -14268,7 +14459,7 @@ void ggml_sycl_free_scratch() try { return; } ggml_sycl_set_device(g_main_device); - const dpct::queue_ptr stream = g_syclStreams[g_main_device_index][0]; + const dpct::queue_ptr stream = g_syclStreams[g_main_device][0]; SYCL_CHECK(CHECK_TRY_ERROR( sycl::free(g_scratch_buffer, *stream))); @@ -14340,6 +14531,12 @@ bool ggml_sycl_compute_forward(struct ggml_compute_params * params, struct ggml_ case GGML_UNARY_OP_RELU: func = ggml_sycl_relu; break; + case GGML_UNARY_OP_HARDSIGMOID: + func = ggml_sycl_hardsigmoid; + break; + case GGML_UNARY_OP_HARDSWISH: + func = ggml_sycl_hardswish; + break; default: return false; } @@ -14414,6 +14611,9 @@ bool ggml_sycl_compute_forward(struct ggml_compute_params * params, struct ggml_ case GGML_OP_IM2COL: func = ggml_sycl_im2col; break; + case GGML_OP_POOL_2D: + func = ggml_sycl_pool2d; + break; case GGML_OP_SUM_ROWS: func = ggml_sycl_sum_rows; break; @@ -14439,27 +14639,15 @@ bool ggml_sycl_compute_forward(struct ggml_compute_params * params, struct ggml_ } GGML_API GGML_CALL void ggml_sycl_get_gpu_list(int *id_list, int max_len) try { - int max_compute_units = -1; - for(int i=0;igpus.size();i++){ + if (i>=max_len) break; + id_list[i] = g_sycl_gpu_mgr->gpus[i]; } return; } @@ -14486,8 +14674,9 @@ catch (sycl::exception const &exc) { GGML_API GGML_CALL void ggml_sycl_get_device_description(int device, char *description, size_t description_size) try { dpct::device_info prop; + int device_id = g_sycl_gpu_mgr->gpus[device]; SYCL_CHECK(CHECK_TRY_ERROR(dpct::get_device_info( - prop, dpct::dev_mgr::instance().get_device(device)))); + prop, dpct::dev_mgr::instance().get_device(device_id)))); snprintf(description, description_size, "%s", prop.get_name()); } catch (sycl::exception const &exc) { @@ -14496,17 +14685,36 @@ catch (sycl::exception const &exc) { std::exit(1); } +GGML_CALL void ggml_backend_sycl_get_device_memory(int device, size_t *free, + size_t *total) try { + ggml_sycl_set_device(device); + + /* + DPCT1009:218: SYCL uses exceptions to report errors and does not use the + error codes. The original code was commented out and a warning string was + inserted. You need to rewrite this code. + */ + /* + DPCT1106:217: 'cudaMemGetInfo' was migrated with the Intel extensions for + device information which may not be supported by all compilers or runtimes. + You may need to adjust the code. + */ + int device_id = g_sycl_gpu_mgr->gpus[device]; + SYCL_CHECK(CHECK_TRY_ERROR( + dpct::dev_mgr::instance().get_device(device_id).get_memory_info(*free, *total))); +} +catch (sycl::exception const &exc) { + std::cerr << exc.what() << "Exception caught at file:" << __FILE__ + << ", line:" << __LINE__ << std::endl; + std::exit(1); +} + //////////////////////////////////////////////////////////////////////////////// // backend interface #define UNUSED GGML_UNUSED -struct ggml_backend_sycl_context { - int device; - std::string name; -}; - // sycl buffer struct ggml_backend_sycl_buffer_context { @@ -14516,7 +14724,12 @@ struct ggml_backend_sycl_buffer_context { size_t temp_tensor_extra_index = 0; std::string name; - ggml_backend_sycl_buffer_context(int device, void * dev_ptr) : device(device), dev_ptr(dev_ptr) {} + ggml_backend_sycl_buffer_context(int device, void * dev_ptr) : + device(device), dev_ptr(dev_ptr) { + check_allow_gpu_index(device); + int id = g_sycl_gpu_mgr->gpus[device]; + name = (GGML_SYCL_NAME + std::to_string(id)); + } ~ ggml_backend_sycl_buffer_context() { delete[] temp_tensor_extras; @@ -14547,10 +14760,9 @@ GGML_CALL static bool ggml_backend_buffer_is_sycl(ggml_backend_buffer_t buffer) static void ggml_backend_sycl_buffer_free_buffer(ggml_backend_buffer_t buffer) try { - ggml_backend_sycl_buffer_context * ctx = ( ggml_backend_sycl_buffer_context *)buffer->context; + ggml_backend_sycl_buffer_context * ctx = ( ggml_backend_sycl_buffer_context *)buffer->context; ggml_sycl_set_device(ctx->device); - int device_index = get_device_index_by_id(ctx->device); - const dpct::queue_ptr stream = g_syclStreams[device_index][0]; + const dpct::queue_ptr stream = g_syclStreams[ctx->device][0]; SYCL_CHECK( CHECK_TRY_ERROR(sycl::free(ctx->dev_ptr, *stream))); @@ -14563,13 +14775,14 @@ catch (sycl::exception const &exc) { } static void * ggml_backend_sycl_buffer_get_base(ggml_backend_buffer_t buffer) { - ggml_backend_sycl_buffer_context * ctx = ( ggml_backend_sycl_buffer_context *)buffer->context; + ggml_backend_sycl_buffer_context * ctx = ( ggml_backend_sycl_buffer_context *)buffer->context; return ctx->dev_ptr; } -static void ggml_backend_sycl_buffer_init_tensor(ggml_backend_buffer_t buffer, - ggml_tensor *tensor) try { - ggml_backend_sycl_buffer_context * ctx = ( ggml_backend_sycl_buffer_context *)buffer->context; +GGML_CALL static void +ggml_backend_sycl_buffer_init_tensor(ggml_backend_buffer_t buffer, + ggml_tensor *tensor) try { + ggml_backend_sycl_buffer_context * ctx = (ggml_backend_sycl_buffer_context *)buffer->context; if (tensor->view_src != NULL && tensor->view_offs == 0) { assert(tensor->view_src->buffer->buft == buffer->buft); @@ -14581,27 +14794,20 @@ static void ggml_backend_sycl_buffer_init_tensor(ggml_backend_buffer_t buffer, ggml_tensor_extra_gpu * extra = ctx->ggml_sycl_alloc_temp_tensor_extra(); extra->data_device[ctx->device] = tensor->data; - tensor->backend = GGML_BACKEND_TYPE_GPU; tensor->extra = extra; if (ggml_is_quantized(tensor->type)) { // initialize padding to 0 to avoid possible NaN values - int64_t row_low = 0; - int64_t row_high = ggml_nrows(tensor); - int64_t nrows_split = row_high - row_low; - - size_t original_size = ggml_nbytes_split(tensor, nrows_split); + size_t original_size = ggml_nbytes(tensor); size_t padded_size = ggml_backend_buft_get_alloc_size(buffer->buft, tensor); if (padded_size > original_size && tensor->view_src == nullptr) { SYCL_CHECK(CHECK_TRY_ERROR(g_syclStreams[ctx->device][0]->memset( (char *)tensor->data + original_size, 0, - padded_size - original_size))); + padded_size - original_size).wait())); } } - - UNUSED(buffer); } catch (sycl::exception const &exc) { std::cerr << exc.what() << "Exception caught at file:" << __FILE__ @@ -14615,13 +14821,12 @@ static void ggml_backend_sycl_buffer_set_tensor(ggml_backend_buffer_t buffer, size_t size) try { GGML_ASSERT(tensor->backend == GGML_BACKEND_TYPE_GPU); - ggml_backend_sycl_buffer_context * ctx = ( ggml_backend_sycl_buffer_context *)buffer->context; + ggml_backend_sycl_buffer_context * ctx = ( ggml_backend_sycl_buffer_context *)buffer->context; ggml_sycl_set_device(ctx->device); - int device_index = get_device_index_by_id(ctx->device); - const dpct::queue_ptr stream = g_syclStreams[device_index][0]; + const dpct::queue_ptr stream = g_syclStreams[ctx->device][0]; SYCL_CHECK( - CHECK_TRY_ERROR(dpct::get_current_device().queues_wait_and_throw())); + CHECK_TRY_ERROR(dpct::dev_mgr::instance().get_device(ctx->device).queues_wait_and_throw())); SYCL_CHECK( CHECK_TRY_ERROR((*stream) @@ -14640,14 +14845,13 @@ static void ggml_backend_sycl_buffer_get_tensor(ggml_backend_buffer_t buffer, size_t size) try { GGML_ASSERT(tensor->backend == GGML_BACKEND_TYPE_GPU); - ggml_backend_sycl_buffer_context * ctx = ( ggml_backend_sycl_buffer_context *)buffer->context; + ggml_backend_sycl_buffer_context * ctx = ( ggml_backend_sycl_buffer_context *)buffer->context; ggml_sycl_set_device(ctx->device); - int device_index = get_device_index_by_id(ctx->device); - const dpct::queue_ptr stream = g_syclStreams[device_index][0]; + const dpct::queue_ptr stream = g_syclStreams[ctx->device][0]; SYCL_CHECK( - CHECK_TRY_ERROR(dpct::get_current_device().queues_wait_and_throw())); + CHECK_TRY_ERROR(dpct::dev_mgr::instance().get_device(ctx->device).queues_wait_and_throw())); SYCL_CHECK(CHECK_TRY_ERROR( (*stream) @@ -14660,13 +14864,73 @@ catch (sycl::exception const &exc) { std::exit(1); } +GGML_CALL static bool +ggml_backend_sycl_buffer_cpy_tensor(ggml_backend_buffer_t buffer, + const ggml_tensor *src, + ggml_tensor *dst) try { + if (ggml_backend_buffer_is_sycl(src->buffer)) { + ggml_backend_sycl_buffer_context * src_ctx = (ggml_backend_sycl_buffer_context *)src->buffer->context; + ggml_backend_sycl_buffer_context * dst_ctx = (ggml_backend_sycl_buffer_context *)buffer->context; + + ggml_sycl_set_device(src_ctx->device); + /* + DPCT1009:198: SYCL uses exceptions to report errors and does not use the + error codes. The original code was commented out and a warning string + was inserted. You need to rewrite this code. + */ + SYCL_CHECK(CHECK_TRY_ERROR( + dpct::dev_mgr::instance().get_device(src_ctx->device).queues_wait_and_throw())); + ggml_sycl_set_device(dst_ctx->device); + /* + DPCT1009:199: SYCL uses exceptions to report errors and does not use the + error codes. The original code was commented out and a warning string + was inserted. You need to rewrite this code. + */ + SYCL_CHECK(CHECK_TRY_ERROR( + dpct::dev_mgr::instance().get_device(dst_ctx->device).queues_wait_and_throw())); + /* + DPCT1009:200: SYCL uses exceptions to report errors and does not use the + error codes. The original code was commented out and a warning string + was inserted. You need to rewrite this code. + */ + + dpct::queue_ptr stream_dst = g_syclStreams[dst_ctx->device][0]; + dpct::queue_ptr stream_src = g_syclStreams[src_ctx->device][0]; + size_t size = ggml_nbytes(src); + + //todo. it's dirty solutino to walkaroud known issue:device2device cross GPUs. + dev2dev_memcpy(*stream_dst, *stream_src, dst->data, src->data, size); + +//todo, it's known issue:error in device2device cross GPUs. reused when the issue is fixed. DON"T remove +#if 0 + SYCL_CHECK(CHECK_TRY_ERROR((*stream).memcpy( + (char *)dst->data, (const char *)src->data, size).wait())); + + /* + DPCT1009:201: SYCL uses exceptions to report errors and does not use the + error codes. The original code was commented out and a warning string + was inserted. You need to rewrite this code. + */ + SYCL_CHECK(CHECK_TRY_ERROR( + dpct::dev_mgr::instance().get_device(dst_ctx->device).queues_wait_and_throw())); +#endif + return true; + } + return false; +} +catch (sycl::exception const &exc) { + std::cerr << exc.what() << "Exception caught at file:" << __FILE__ + << ", line:" << __LINE__ << std::endl; + std::exit(1); +} + + static void ggml_backend_sycl_buffer_clear(ggml_backend_buffer_t buffer, uint8_t value) try { ggml_backend_sycl_buffer_context * ctx = ( ggml_backend_sycl_buffer_context *)buffer->context; ggml_sycl_set_device(ctx->device); - int device_index = get_device_index_by_id(ctx->device); - const dpct::queue_ptr stream = g_syclStreams[device_index][0]; + const dpct::queue_ptr stream = g_syclStreams[ctx->device][0]; SYCL_CHECK( CHECK_TRY_ERROR(dpct::get_current_device().queues_wait_and_throw())); @@ -14687,7 +14951,7 @@ static struct ggml_backend_buffer_i ggml_backend_sycl_buffer_interface = { /* .init_tensor = */ ggml_backend_sycl_buffer_init_tensor, /* .set_tensor = */ ggml_backend_sycl_buffer_set_tensor, /* .get_tensor = */ ggml_backend_sycl_buffer_get_tensor, - /* .cpy_tensor = */ NULL, + /* .cpy_tensor = */ ggml_backend_sycl_buffer_cpy_tensor, /* .clear = */ ggml_backend_sycl_buffer_clear, /* .reset = */ NULL, }; @@ -14698,29 +14962,28 @@ struct ggml_backend_sycl_buffer_type_context { std::string name; }; +struct ggml_backend_sycl_context { + int device; + std::string name; +}; + GGML_CALL static const char * ggml_backend_sycl_buffer_type_name(ggml_backend_buffer_type_t buft) { ggml_backend_sycl_buffer_type_context * ctx = (ggml_backend_sycl_buffer_type_context *)buft->context; return ctx->name.c_str(); } - -static ggml_backend_buffer_t +GGML_CALL static ggml_backend_buffer_t ggml_backend_sycl_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size) try { ggml_backend_sycl_buffer_type_context * buft_ctx = (ggml_backend_sycl_buffer_type_context *)buft->context; - int device = (int) buft_ctx->device; - - ggml_sycl_set_device(device); - int device_index = get_device_index_by_id(device); - const dpct::queue_ptr stream = g_syclStreams[device_index][0]; + ggml_sycl_set_device(buft_ctx->device); + const dpct::queue_ptr stream = g_syclStreams[buft_ctx->device][0]; size = std::max(size, (size_t)1); // syclMalloc returns null for size 0 void * dev_ptr; SYCL_CHECK(CHECK_TRY_ERROR(dev_ptr = (void *)sycl::malloc_device( size, *stream))); - - ggml_backend_sycl_buffer_context * ctx = new ggml_backend_sycl_buffer_context(device, dev_ptr); - + ggml_backend_sycl_buffer_context * ctx = new ggml_backend_sycl_buffer_context(buft_ctx->device, dev_ptr); return ggml_backend_buffer_init(buft, ggml_backend_sycl_buffer_interface, ctx, size); } catch (sycl::exception const &exc) { @@ -14729,9 +14992,8 @@ catch (sycl::exception const &exc) { std::exit(1); } -static size_t ggml_backend_sycl_buffer_type_get_alignment(ggml_backend_buffer_type_t buft) { +GGML_CALL static size_t ggml_backend_sycl_buffer_type_get_alignment(ggml_backend_buffer_type_t buft) { return 128; - UNUSED(buft); } @@ -14741,13 +15003,8 @@ static size_t ggml_backend_sycl_buffer_type_get_max_size(ggml_backend_buffer_typ UNUSED(buft); } -static size_t ggml_backend_sycl_buffer_type_get_alloc_size(ggml_backend_buffer_type_t buft, const ggml_tensor * tensor) { - int64_t row_low = 0; - int64_t row_high = ggml_nrows(tensor); - int64_t nrows_split = row_high - row_low; - - size_t size = ggml_nbytes_split(tensor, nrows_split); - +GGML_CALL static size_t ggml_backend_sycl_buffer_type_get_alloc_size(ggml_backend_buffer_type_t buft, const ggml_tensor * tensor) { + size_t size = ggml_nbytes(tensor); int64_t ne0 = tensor->ne[0]; if (ggml_is_quantized(tensor->type)) { @@ -14761,10 +15018,13 @@ static size_t ggml_backend_sycl_buffer_type_get_alloc_size(ggml_backend_buffer_t UNUSED(buft); } -static bool ggml_backend_sycl_buffer_type_supports_backend(ggml_backend_buffer_type_t buft, ggml_backend_t backend) { - return ggml_backend_is_sycl(backend); - - UNUSED(buft); +GGML_CALL static bool ggml_backend_sycl_buffer_type_supports_backend(ggml_backend_buffer_type_t buft, ggml_backend_t backend) { + if (!ggml_backend_is_sycl(backend)) { + return false; + } + ggml_backend_sycl_buffer_type_context * buft_ctx = (ggml_backend_sycl_buffer_type_context *)buft->context; + ggml_backend_sycl_context * sycl_ctx = (ggml_backend_sycl_context *)backend->context; + return buft_ctx->device == sycl_ctx->device; } static ggml_backend_buffer_type_i ggml_backend_sycl_buffer_type_interface = { @@ -14783,10 +15043,10 @@ ggml_backend_buffer_type_t ggml_backend_sycl_buffer_type(int device) { static bool ggml_backend_sycl_buffer_type_initialized = false; if (!ggml_backend_sycl_buffer_type_initialized) { - for (int i = 0; i < GGML_SYCL_MAX_DEVICES; i++) { + for (int i = 0; i < g_device_count; i++) { ggml_backend_sycl_buffer_types[i] = { /* .iface = */ ggml_backend_sycl_buffer_type_interface, - /* .context = */ new ggml_backend_sycl_buffer_type_context{i, GGML_SYCL_NAME + std::to_string(i)}, + /* .context = */ new ggml_backend_sycl_buffer_type_context{i, GGML_SYCL_NAME + std::to_string(g_sycl_gpu_mgr->gpus[i])}, }; } ggml_backend_sycl_buffer_type_initialized = true; @@ -14795,6 +15055,391 @@ ggml_backend_buffer_type_t ggml_backend_sycl_buffer_type(int device) { return &ggml_backend_sycl_buffer_types[device]; } +// sycl split buffer type +static void get_row_split(int64_t * row_low, int64_t * row_high, const ggml_tensor * tensor, const std::array & tensor_split, int id) { + const int64_t nrows = ggml_nrows(tensor); + const int64_t rounding = get_row_rounding(tensor->type, tensor_split); + + *row_low = id == 0 ? 0 : nrows*tensor_split[id]; + *row_low -= *row_low % rounding; + if (id == g_device_count - 1) { + *row_high = nrows; + } else { + *row_high = nrows*tensor_split[id + 1]; + *row_high -= *row_high % rounding; + } +} + +struct ggml_backend_sycl_split_buffer_context { + ~ggml_backend_sycl_split_buffer_context() try { + for (ggml_tensor_extra_gpu * extra : tensor_extras) { + for (int i = 0; i < g_device_count; ++i) { + // int id = g_sycl_gpu_mgr->gpus[i]; + for (int64_t is = 0; is < MAX_STREAMS; ++is) { + if (extra->events[i][is] != nullptr) { + /* + DPCT1009:206: SYCL uses exceptions to report errors and + does not use the error codes. The original code was + commented out and a warning string was inserted. You + need to rewrite this code. + */ + SYCL_CHECK(CHECK_TRY_ERROR( + dpct::destroy_event(extra->events[i][is]))); + } + } + if (extra->data_device[i] != nullptr) { + /* + DPCT1009:207: SYCL uses exceptions to report errors and does + not use the error codes. The original code was commented out + and a warning string was inserted. You need to rewrite this + code. + */ + ggml_sycl_set_device(i); + SYCL_CHECK(CHECK_TRY_ERROR(sycl::free( + extra->data_device[i], *g_syclStreams[i][0]))); + } + } + delete extra; + } + } + catch (sycl::exception const &exc) { + std::cerr << exc.what() << "Exception caught at file:" << __FILE__ + << ", line:" << __LINE__ << std::endl; + std::exit(1); + } + + std::vector tensor_extras; +}; + +GGML_CALL static const char * ggml_backend_sycl_split_buffer_get_name(ggml_backend_buffer_t buffer) { + return GGML_SYCL_NAME "_Split"; + + UNUSED(buffer); +} + +// unused at the moment +//static bool ggml_backend_buffer_is_sycl_split(ggml_backend_buffer_t buffer) { +// return buffer->iface.get_name == ggml_backend_sycl_split_buffer_get_name; +//} + +GGML_CALL static void ggml_backend_sycl_split_buffer_free_buffer(ggml_backend_buffer_t buffer) { + ggml_backend_sycl_split_buffer_context * ctx = (ggml_backend_sycl_split_buffer_context *)buffer->context; + delete ctx; +} + +GGML_CALL static void * ggml_backend_sycl_split_buffer_get_base(ggml_backend_buffer_t buffer) { + // the pointers are stored in the tensor extras, this is just a dummy address and never dereferenced + return (void *)0x1000; + + UNUSED(buffer); +} + +GGML_CALL static void +ggml_backend_sycl_split_buffer_init_tensor(ggml_backend_buffer_t buffer, + ggml_tensor *tensor) try { + GGML_ASSERT(tensor->view_src == nullptr); // views of split tensors are not supported + + ggml_backend_sycl_split_buffer_context * ctx = (ggml_backend_sycl_split_buffer_context *)buffer->context; + ggml_backend_sycl_split_buffer_type_context * buft_ctx = (ggml_backend_sycl_split_buffer_type_context *)buffer->buft->context; + + const int64_t ne0 = tensor->ne[0]; + + ggml_tensor_extra_gpu * extra = new ggml_tensor_extra_gpu{}; + + ctx->tensor_extras.push_back(extra); + + for (int i = 0; i < g_device_count; ++i) { + // int id = g_sycl_gpu_mgr->gpus[i]; + int64_t row_low, row_high; + get_row_split(&row_low, &row_high, tensor, buft_ctx->tensor_split, i); + + int64_t nrows_split = row_high - row_low; + if (nrows_split == 0) { + continue; + } + + size_t size = ggml_nbytes_split(tensor, nrows_split); + const size_t original_size = size; + + // pad last row to a multiple of 512 elements to avoid out-of-bounds memory accesses + if (ne0 % MATRIX_ROW_PADDING != 0) { + size += ggml_row_size(tensor->type, MATRIX_ROW_PADDING - ne0 % MATRIX_ROW_PADDING); + } + + // FIXME: do not crash if cudaMalloc fails + // currently, init_tensor cannot fail, it needs to be fixed in ggml-backend first + ggml_sycl_set_device(i); + char * buf; + /* + DPCT1009:208: SYCL uses exceptions to report errors and does not use the + error codes. The original code was commented out and a warning string + was inserted. You need to rewrite this code. + */ + SYCL_CHECK(CHECK_TRY_ERROR(buf = (char *)sycl::malloc_device( + size, *g_syclStreams[i][0]))); + + // set padding to 0 to avoid possible NaN values + if (size > original_size) { + /* + DPCT1009:209: SYCL uses exceptions to report errors and does not use + the error codes. The original code was commented out and a warning + string was inserted. You need to rewrite this code. + */ + SYCL_CHECK(CHECK_TRY_ERROR( + (*g_syclStreams[i][0]) + .memset(buf + original_size, 0, size - original_size) + .wait())); + } + + extra->data_device[i] = buf; + + for (int64_t is = 0; is < MAX_STREAMS; ++is) { + /* + DPCT1009:210: SYCL uses exceptions to report errors and does not use + the error codes. The original code was commented out and a warning + string was inserted. You need to rewrite this code. + */ + SYCL_CHECK( + CHECK_TRY_ERROR(extra->events[i][is] = new sycl::event())); + } + } + tensor->backend = GGML_BACKEND_TYPE_GPU_SPLIT; + tensor->extra = extra; +} +catch (sycl::exception const &exc) { + std::cerr << exc.what() << "Exception caught at file:" << __FILE__ + << ", line:" << __LINE__ << std::endl; + std::exit(1); +} + +GGML_CALL static void +ggml_backend_sycl_split_buffer_set_tensor(ggml_backend_buffer_t buffer, + ggml_tensor *tensor, const void *data, + size_t offset, size_t size) try { + // split tensors must always be set in their entirety at once + GGML_ASSERT(offset == 0); + GGML_ASSERT(size == ggml_nbytes(tensor)); + + ggml_backend_sycl_split_buffer_type_context * buft_ctx = (ggml_backend_sycl_split_buffer_type_context *)buffer->buft->context; + + const int64_t ne0 = tensor->ne[0]; + const size_t nb1 = tensor->nb[1]; + ggml_tensor_extra_gpu * extra = (ggml_tensor_extra_gpu *)tensor->extra; + + for (int i = 0; i < g_device_count; ++i) { + // int id = g_sycl_gpu_mgr->gpus[i]; + int64_t row_low, row_high; + get_row_split(&row_low, &row_high, tensor, buft_ctx->tensor_split, i); + + int64_t nrows_split = row_high - row_low; + if (nrows_split == 0) { + continue; + } + + const size_t offset_split = row_low*nb1; + size_t size = ggml_nbytes_split(tensor, nrows_split); + const size_t original_size = size; + + // pad last row to a multiple of 512 elements to avoid out-of-bounds memory accesses + if (ne0 % MATRIX_ROW_PADDING != 0) { + size += ggml_row_size(tensor->type, MATRIX_ROW_PADDING - ne0 % MATRIX_ROW_PADDING); + } + + const char * buf_host = (const char *)data + offset_split; + /* + DPCT1009:211: SYCL uses exceptions to report errors and does not use the + error codes. The original code was commented out and a warning string + was inserted. You need to rewrite this code. + */ + ggml_sycl_set_device(i); + SYCL_CHECK(CHECK_TRY_ERROR( + (*g_syclStreams[i][0]) + .memcpy(extra->data_device[i], buf_host, original_size) + .wait())); + } +} +catch (sycl::exception const &exc) { + std::cerr << exc.what() << "Exception caught at file:" << __FILE__ + << ", line:" << __LINE__ << std::endl; + std::exit(1); +} + +GGML_CALL static void +ggml_backend_sycl_split_buffer_get_tensor(ggml_backend_buffer_t buffer, + const ggml_tensor *tensor, void *data, + size_t offset, size_t size) try { + // split tensors must always be set in their entirety at once + GGML_ASSERT(offset == 0); + GGML_ASSERT(size == ggml_nbytes(tensor)); + + ggml_backend_sycl_split_buffer_type_context * buft_ctx = (ggml_backend_sycl_split_buffer_type_context *)buffer->buft->context; + + const int64_t ne0 = tensor->ne[0]; + const size_t nb1 = tensor->nb[1]; + ggml_tensor_extra_gpu * extra = (ggml_tensor_extra_gpu *)tensor->extra; + + for (int i = 0; i < g_device_count; ++i) { + // int id = g_sycl_gpu_mgr->gpus[i]; + int64_t row_low, row_high; + get_row_split(&row_low, &row_high, tensor, buft_ctx->tensor_split, i); + + int64_t nrows_split = row_high - row_low; + if (nrows_split == 0) { + continue; + } + + const size_t offset_split = row_low*nb1; + size_t size = ggml_nbytes_split(tensor, nrows_split); + const size_t original_size = size; + + // pad last row to a multiple of 512 elements to avoid out-of-bounds memory accesses + if (ne0 % MATRIX_ROW_PADDING != 0) { + size += ggml_row_size(tensor->type, MATRIX_ROW_PADDING - ne0 % MATRIX_ROW_PADDING); + } + + char * buf_host = (char *)data + offset_split; + /* + DPCT1009:212: SYCL uses exceptions to report errors and does not use the + error codes. The original code was commented out and a warning string + was inserted. You need to rewrite this code. + */ + ggml_sycl_set_device(i); + SYCL_CHECK(CHECK_TRY_ERROR( + (*g_syclStreams[i][0]) + .memcpy(buf_host, extra->data_device[i], original_size) + .wait())); + } +} +catch (sycl::exception const &exc) { + std::cerr << exc.what() << "Exception caught at file:" << __FILE__ + << ", line:" << __LINE__ << std::endl; + std::exit(1); +} + +GGML_CALL static void ggml_backend_sycl_split_buffer_clear(ggml_backend_buffer_t buffer, uint8_t value) { + UNUSED(buffer); + UNUSED(value); +} + +static struct ggml_backend_buffer_i ggml_backend_sycl_split_buffer_interface = { + /* .get_name = */ ggml_backend_sycl_split_buffer_get_name, + /* .free_buffer = */ ggml_backend_sycl_split_buffer_free_buffer, + /* .get_base = */ ggml_backend_sycl_split_buffer_get_base, + /* .init_tensor = */ ggml_backend_sycl_split_buffer_init_tensor, + /* .set_tensor = */ ggml_backend_sycl_split_buffer_set_tensor, + /* .get_tensor = */ ggml_backend_sycl_split_buffer_get_tensor, + /* .cpy_tensor = */ NULL, + /* .clear = */ ggml_backend_sycl_split_buffer_clear, + /* .reset = */ NULL, +}; + +GGML_CALL static const char * ggml_backend_sycl_split_buffer_type_name(ggml_backend_buffer_type_t buft) { + return GGML_SYCL_NAME "_Split"; + + UNUSED(buft); +} + +GGML_CALL static ggml_backend_buffer_t ggml_backend_sycl_split_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size) { + // since we don't know the exact split after rounding, we cannot allocate the device buffers at this point + // instead, we allocate them for each tensor separately in init_tensor + // however, the size still represents the maximum cumulative size of all the device buffers after the tensors are allocated, + // as returned by get_alloc_size. this limit is enforced during tensor allocation by ggml-alloc, so it must be correct. + ggml_backend_sycl_split_buffer_context * ctx = new ggml_backend_sycl_split_buffer_context(); + + return ggml_backend_buffer_init(buft, ggml_backend_sycl_split_buffer_interface, ctx, size); +} + +GGML_CALL static size_t ggml_backend_sycl_split_buffer_type_get_alignment(ggml_backend_buffer_type_t buft) { + return 128; + UNUSED(buft); +} + +GGML_CALL static size_t ggml_backend_sycl_split_buffer_type_get_alloc_size(ggml_backend_buffer_type_t buft, const ggml_tensor * tensor) { + ggml_backend_sycl_split_buffer_type_context * ctx = (ggml_backend_sycl_split_buffer_type_context *)buft->context; + + size_t total_size = 0; + + const int64_t ne0 = tensor->ne[0]; + + for (int i = 0; i < g_device_count; ++i) { + // int id = g_sycl_gpu_mgr->gpus[i]; + int64_t row_low, row_high; + get_row_split(&row_low, &row_high, tensor, ctx->tensor_split, i); + + int64_t nrows_split = row_high - row_low; + if (nrows_split == 0) { + continue; + } + + total_size += ggml_nbytes_split(tensor, nrows_split); + + // pad last row to a multiple of 512 elements to avoid out-of-bounds memory accesses + if (ne0 % MATRIX_ROW_PADDING != 0) { + total_size += ggml_row_size(tensor->type, MATRIX_ROW_PADDING - ne0 % MATRIX_ROW_PADDING); + } + } + + return total_size; +} + +GGML_CALL static bool ggml_backend_sycl_split_buffer_type_supports_backend(ggml_backend_buffer_type_t buft, ggml_backend_t backend) { + return ggml_backend_is_sycl(backend); + + UNUSED(buft); +} + +GGML_CALL static bool ggml_backend_sycl_split_buffer_type_is_host(ggml_backend_buffer_type_t buft) { + return false; + + UNUSED(buft); +} + +static ggml_backend_buffer_type_i ggml_backend_sycl_split_buffer_type_interface = { + /* .get_name = */ ggml_backend_sycl_split_buffer_type_name, + /* .alloc_buffer = */ ggml_backend_sycl_split_buffer_type_alloc_buffer, + /* .get_alignment = */ ggml_backend_sycl_split_buffer_type_get_alignment, + /* .get_max_size = */ NULL, // defaults to SIZE_MAX + /* .get_alloc_size = */ ggml_backend_sycl_split_buffer_type_get_alloc_size, + /* .supports_backend = */ ggml_backend_sycl_split_buffer_type_supports_backend, + /* .is_host = */ ggml_backend_sycl_split_buffer_type_is_host, +}; + +GGML_CALL ggml_backend_buffer_type_t ggml_backend_sycl_split_buffer_type(const float * tensor_split) { + // FIXME: this is not thread safe + static std::map, struct ggml_backend_buffer_type> buft_map; + + std::array tensor_split_arr = {}; + + bool all_zero = tensor_split == nullptr || std::all_of(tensor_split, tensor_split + GGML_SYCL_MAX_DEVICES, [](float x) { return x == 0.0f; }); + if (all_zero) { + tensor_split_arr = g_default_tensor_split; + } else { + float split_sum = 0.0f; + for (int i = 0; i < g_device_count; ++i) { + // int id = g_sycl_gpu_mgr->gpus[i]; + tensor_split_arr[i] = split_sum; + split_sum += tensor_split[i]; + } + for (int i = 0; i < g_device_count; ++i) { + // int id = g_sycl_gpu_mgr->gpus[i]; + tensor_split_arr[i] /= split_sum; + } + } + + auto it = buft_map.find(tensor_split_arr); + if (it != buft_map.end()) { + return &it->second; + } + + struct ggml_backend_buffer_type buft { + /* .iface = */ ggml_backend_sycl_split_buffer_type_interface, + /* .context = */ new ggml_backend_sycl_split_buffer_type_context{tensor_split_arr}, + }; + + auto result = buft_map.emplace(tensor_split_arr, buft); + return &result.first->second; +} + // host buffer type GGML_CALL static const char * ggml_backend_sycl_host_buffer_type_name(ggml_backend_buffer_type_t buft) { @@ -14824,6 +15469,7 @@ static ggml_backend_buffer_t ggml_backend_sycl_host_buffer_type_alloc_buffer(ggm // FIXME: this is a hack to avoid having to implement a new buffer type ggml_backend_buffer_t buffer = ggml_backend_cpu_buffer_from_ptr(ptr, size); buffer->buft = buft; + buffer->iface.get_name = ggml_backend_sycl_host_buffer_name; buffer->iface.free_buffer = ggml_backend_sycl_host_buffer_free_buffer; return buffer; @@ -14848,34 +15494,33 @@ ggml_backend_buffer_type_t ggml_backend_sycl_host_buffer_type() { // backend -static const char * ggml_backend_sycl_name(ggml_backend_t backend) { - return GGML_SYCL_NAME; +GGML_CALL static const char * ggml_backend_sycl_name(ggml_backend_t backend) { - UNUSED(backend); + ggml_backend_sycl_context * sycl_ctx = (ggml_backend_sycl_context *)backend->context; + + return sycl_ctx->name.c_str(); } -static void ggml_backend_sycl_free(ggml_backend_t backend) { +GGML_CALL static void ggml_backend_sycl_free(ggml_backend_t backend) { ggml_backend_sycl_context * sycl_ctx = (ggml_backend_sycl_context *)backend->context; delete sycl_ctx; delete backend; } -static ggml_backend_buffer_type_t ggml_backend_sycl_get_default_buffer_type(ggml_backend_t backend) { - ggml_backend_sycl_context * sycl_ctx = (ggml_backend_sycl_context *)backend->context; +GGML_CALL static ggml_backend_buffer_type_t ggml_backend_sycl_get_default_buffer_type(ggml_backend_t backend) { + ggml_backend_sycl_context * sycl_ctx = (ggml_backend_sycl_context *)backend->context; return ggml_backend_sycl_buffer_type(sycl_ctx->device); } -static void ggml_backend_sycl_set_tensor_async(ggml_backend_t backend, +GGML_CALL static void ggml_backend_sycl_set_tensor_async(ggml_backend_t backend, ggml_tensor *tensor, const void *data, size_t offset, size_t size) try { ggml_backend_sycl_context * sycl_ctx = (ggml_backend_sycl_context *)backend->context; - GGML_ASSERT(tensor->buffer->buft == ggml_backend_sycl_buffer_type(sycl_ctx->device) && "unsupported buffer type"); GGML_ASSERT(tensor->backend == GGML_BACKEND_TYPE_GPU); - SYCL_CHECK(CHECK_TRY_ERROR(g_syclStreams[sycl_ctx->device][0]->memcpy( (char *)tensor->data + offset, data, size))); } @@ -14885,15 +15530,13 @@ catch (sycl::exception const &exc) { std::exit(1); } -static void ggml_backend_sycl_get_tensor_async(ggml_backend_t backend, +GGML_CALL static void ggml_backend_sycl_get_tensor_async(ggml_backend_t backend, const ggml_tensor *tensor, void *data, size_t offset, size_t size) try { ggml_backend_sycl_context * sycl_ctx = (ggml_backend_sycl_context *)backend->context; - GGML_ASSERT(tensor->buffer->buft == ggml_backend_sycl_buffer_type(sycl_ctx->device) && "unsupported buffer type"); GGML_ASSERT(tensor->backend == GGML_BACKEND_TYPE_GPU); - SYCL_CHECK(CHECK_TRY_ERROR(g_syclStreams[sycl_ctx->device][0]->memcpy( data, (const char *)tensor->data + offset, size))); } @@ -14903,9 +15546,31 @@ catch (sycl::exception const &exc) { std::exit(1); } +GGML_CALL static bool ggml_backend_sycl_cpy_tensor_async(ggml_backend_t backend, + const ggml_tensor *src, + ggml_tensor *dst) try { + ggml_backend_sycl_context * sycl_ctx = (ggml_backend_sycl_context *)backend->context; + if (dst->buffer->buft == ggml_backend_sycl_buffer_type(sycl_ctx->device) && ggml_backend_buffer_is_sycl(src->buffer)) { + /* + DPCT1009:215: SYCL uses exceptions to report errors and does not use the + error codes. The original code was commented out and a warning string + was inserted. You need to rewrite this code. + */ + SYCL_CHECK(CHECK_TRY_ERROR(g_syclStreams[sycl_ctx->device][0]->memcpy( + dst->data, src->data, ggml_nbytes(dst)))); + return true; + } + + return false; +} +catch (sycl::exception const &exc) { + std::cerr << exc.what() << "Exception caught at file:" << __FILE__ + << ", line:" << __LINE__ << std::endl; + std::exit(1); +} + static void ggml_backend_sycl_synchronize(ggml_backend_t backend) try { ggml_backend_sycl_context * sycl_ctx = (ggml_backend_sycl_context *)backend->context; - SYCL_CHECK(CHECK_TRY_ERROR(g_syclStreams[sycl_ctx->device][0]->wait())); UNUSED(backend); @@ -14916,32 +15581,8 @@ catch (sycl::exception const &exc) { std::exit(1); } -static ggml_backend_graph_plan_t ggml_backend_sycl_graph_plan_create(ggml_backend_t backend, const ggml_cgraph * cgraph) { - GGML_ASSERT(!"not implemented"); - - return nullptr; - - UNUSED(backend); - UNUSED(cgraph); -} - -static void ggml_backend_sycl_graph_plan_free(ggml_backend_t backend, ggml_backend_graph_plan_t plan) { - GGML_ASSERT(!"not implemented"); - - UNUSED(backend); - UNUSED(plan); -} - -static void ggml_backend_sycl_graph_plan_compute(ggml_backend_t backend, ggml_backend_graph_plan_t plan) { - GGML_ASSERT(!"not implemented"); - - UNUSED(backend); - UNUSED(plan); -} - -static bool ggml_backend_sycl_graph_compute(ggml_backend_t backend, ggml_cgraph * cgraph) { +GGML_CALL static bool ggml_backend_sycl_graph_compute(ggml_backend_t backend, ggml_cgraph * cgraph) { ggml_backend_sycl_context * sycl_ctx = (ggml_backend_sycl_context *)backend->context; - ggml_sycl_set_main_device(sycl_ctx->device); ggml_compute_params params = {}; @@ -14949,63 +15590,41 @@ static bool ggml_backend_sycl_graph_compute(ggml_backend_t backend, ggml_cgraph params.ith = 0; for (int i = 0; i < cgraph->n_nodes; i++) { ggml_tensor * node = cgraph->nodes[i]; - - if (node->op == GGML_OP_RESHAPE || node->op == GGML_OP_TRANSPOSE || node->op == GGML_OP_VIEW || node->op == GGML_OP_PERMUTE) + if (node->op == GGML_OP_RESHAPE || node->op == GGML_OP_TRANSPOSE || node->op == GGML_OP_VIEW || node->op == GGML_OP_PERMUTE || node->op == GGML_OP_NONE) { continue; - - assert(node->backend == GGML_BACKEND_TYPE_GPU); + } +#ifndef NDEBUG + assert(node->backend == GGML_BACKEND_TYPE_GPU || node->backend == GGML_BACKEND_TYPE_GPU_SPLIT); assert(node->buffer->buft == ggml_backend_sycl_buffer_type(sycl_ctx->device)); assert(node->extra != nullptr); for (int j = 0; j < GGML_MAX_SRC; j++) { if (node->src[j] != nullptr) { - assert(node->src[j]->backend == GGML_BACKEND_TYPE_GPU); + assert(node->src[j]->backend == GGML_BACKEND_TYPE_GPU || node->src[j]->backend == GGML_BACKEND_TYPE_GPU_SPLIT); assert(node->src[j]->buffer->buft == ggml_backend_sycl_buffer_type(sycl_ctx->device)); assert(node->src[j]->extra != nullptr); } } - +#endif bool ok = ggml_sycl_compute_forward(¶ms, node); if (!ok) { fprintf(stderr, "%s: error: op not supported %s (%s)\n", __func__, node->name, ggml_op_name(node->op)); } GGML_ASSERT(ok); - -#if 0 - if (node->type == GGML_TYPE_F32) { - syclDeviceSynchronize(); - std::vector tmp(ggml_nelements(node), 0.0f); - syclMemcpy(tmp.data(), node->data, ggml_nelements(node)*sizeof(float), syclMemcpyDeviceToHost); - printf("\n%s (%s) (%s %s) (%s %s): ", node->name, ggml_op_name(node->op), - ggml_type_name(node->src[0]->type), - node->src[1] ? ggml_type_name(node->src[1]->type) : "none", - node->src[0]->name, - node->src[1] ? node->src[1]->name : "none"); - double sum = 0.0; - double sq_sum = 0.0; - for (int i = 0; i < ggml_nelements(node); i++) { - printf("%f ", tmp[i]); - sum += tmp[i]; - sq_sum += tmp[i]*tmp[i]; - } - printf("\n"); - printf("sum: %f, ", sum); - printf("sq_sum: %f\n", sq_sum); - } -#endif } - UNUSED(backend); return true; } -static bool ggml_backend_sycl_supports_op(ggml_backend_t backend, const ggml_tensor * op) { +GGML_CALL static bool ggml_backend_sycl_supports_op(ggml_backend_t backend, const ggml_tensor * op) { switch (op->op) { case GGML_OP_UNARY: switch (ggml_get_unary_op(op)) { case GGML_UNARY_OP_GELU: case GGML_UNARY_OP_SILU: case GGML_UNARY_OP_RELU: + case GGML_UNARY_OP_HARDSIGMOID: + case GGML_UNARY_OP_HARDSWISH: case GGML_UNARY_OP_GELU_QUICK: case GGML_UNARY_OP_TANH: return true; @@ -15081,16 +15700,17 @@ static bool ggml_backend_sycl_supports_op(ggml_backend_t backend, const ggml_ten if (src0_type == GGML_TYPE_F16 && src1_type == GGML_TYPE_F16) { return true; } + if (src0_type == GGML_TYPE_F16 && src1_type == GGML_TYPE_F32) { + return true; + } return false; } break; + case GGML_OP_DUP: + case GGML_OP_REPEAT: case GGML_OP_CONCAT: { ggml_type src0_type = op->src[0]->type; - if (src0_type == GGML_TYPE_F32) { - return true; - } else { - return false; - } + return src0_type != GGML_TYPE_I32 && src0_type != GGML_TYPE_I16; } break; case GGML_OP_NONE: case GGML_OP_RESHAPE: @@ -15098,8 +15718,6 @@ static bool ggml_backend_sycl_supports_op(ggml_backend_t backend, const ggml_ten case GGML_OP_PERMUTE: case GGML_OP_TRANSPOSE: case GGML_OP_NORM: - case GGML_OP_REPEAT: - case GGML_OP_DUP: case GGML_OP_ADD: case GGML_OP_MUL: case GGML_OP_DIV: @@ -15113,6 +15731,7 @@ static bool ggml_backend_sycl_supports_op(ggml_backend_t backend, const ggml_ten case GGML_OP_ROPE: case GGML_OP_ALIBI: case GGML_OP_IM2COL: + case GGML_OP_POOL_2D: case GGML_OP_SUM_ROWS: case GGML_OP_ARGSORT: case GGML_OP_ACC: @@ -15134,11 +15753,11 @@ static ggml_backend_i ggml_backend_sycl_interface = { /* .get_default_buffer_type = */ ggml_backend_sycl_get_default_buffer_type, /* .set_tensor_async = */ ggml_backend_sycl_set_tensor_async, /* .get_tensor_async = */ ggml_backend_sycl_get_tensor_async, - /* .cpy_tensor_async = */ NULL, + /* .cpy_tensor_async = */ ggml_backend_sycl_cpy_tensor_async, /* .synchronize = */ ggml_backend_sycl_synchronize, - /* .graph_plan_create = */ ggml_backend_sycl_graph_plan_create, - /* .graph_plan_free = */ ggml_backend_sycl_graph_plan_free, - /* .graph_plan_compute = */ ggml_backend_sycl_graph_plan_compute, + /* .graph_plan_create = */ NULL, + /* .graph_plan_free = */ NULL, + /* .graph_plan_compute = */ NULL, /* .graph_compute = */ ggml_backend_sycl_graph_compute, /* .supports_op = */ ggml_backend_sycl_supports_op, }; @@ -15148,20 +15767,17 @@ static ggml_guid_t ggml_backend_sycl_guid() { return &guid; } -ggml_backend_t ggml_backend_sycl_init(int device) { +GGML_CALL ggml_backend_t ggml_backend_sycl_init(int device) { ggml_init_sycl(); // TODO: remove from ggml.c - if (device < 0 || device >= ggml_sycl_get_device_count()) { - fprintf(stderr, "%s: error: invalid device %d\n", __func__, device); - return nullptr; - } + check_allow_gpu_index(device); // not strictly necessary, but it may reduce the overhead of the first graph_compute ggml_sycl_set_main_device(device); - + int id = g_sycl_gpu_mgr->gpus[device]; ggml_backend_sycl_context * ctx = new ggml_backend_sycl_context { /* .device = */ device, - /* .name = */ GGML_SYCL_NAME + std::to_string(device), + /* .name = */ GGML_SYCL_NAME + std::to_string(id), }; ggml_backend_t sycl_backend = new ggml_backend { @@ -15177,22 +15793,33 @@ bool ggml_backend_is_sycl(ggml_backend_t backend) { return backend != NULL && ggml_guid_matches(backend->guid, ggml_backend_sycl_guid()); } -static ggml_backend_t ggml_backend_reg_sycl_init(const char * params, void * user_data) { +GGML_CALL int ggml_backend_sycl_get_device_count() { + if (!g_sycl_gpu_mgr) g_sycl_gpu_mgr = new sycl_gpu_mgr(); + return g_sycl_gpu_mgr->get_gpu_count(); +} + +GGML_CALL static ggml_backend_t ggml_backend_reg_sycl_init(const char * params, void * user_data) { ggml_backend_t sycl_backend = ggml_backend_sycl_init((int) (intptr_t) user_data); return sycl_backend; UNUSED(params); } +GGML_API GGML_CALL int ggml_backend_sycl_get_device_index(int device_id) { + return g_sycl_gpu_mgr->get_index(device_id); +} + extern "C" int ggml_backend_sycl_reg_devices(); int ggml_backend_sycl_reg_devices() { - int device_count = ggml_sycl_get_device_count(); - - for (int i = 0; i < device_count; i++) { + if (!g_sycl_gpu_mgr) g_sycl_gpu_mgr = new sycl_gpu_mgr(); + g_device_count = g_sycl_gpu_mgr->get_gpu_count(); + assert(g_device_count>0); + for (int i = 0; i < g_device_count; i++) { + int id = g_sycl_gpu_mgr->gpus[i]; char name[128]; - snprintf(name, sizeof(name), "%s%d", GGML_SYCL_NAME, i); + snprintf(name, sizeof(name), "%s%d", GGML_SYCL_NAME, id); ggml_backend_register(name, ggml_backend_reg_sycl_init, ggml_backend_sycl_buffer_type(i), (void *) (intptr_t) i); } - return device_count; + return g_device_count; } diff --git a/ggml-sycl.h b/ggml-sycl.h index 891f2d00a..bf5b11b36 100644 --- a/ggml-sycl.h +++ b/ggml-sycl.h @@ -24,6 +24,11 @@ GGML_API ggml_backend_buffer_type_t ggml_backend_sycl_host_buffer_type(void); GGML_API void ggml_backend_sycl_print_sycl_devices(void); GGML_API GGML_CALL void ggml_sycl_get_gpu_list(int *id_list, int max_len); GGML_API GGML_CALL void ggml_sycl_get_device_description(int device, char *description, size_t description_size); +GGML_API GGML_CALL int ggml_backend_sycl_get_device_count(); +GGML_API GGML_CALL ggml_backend_buffer_type_t ggml_backend_sycl_split_buffer_type(const float * tensor_split); +GGML_API GGML_CALL void ggml_backend_sycl_get_device_memory(int device, size_t *free, size_t *total); +GGML_API GGML_CALL int ggml_backend_sycl_get_device_index(int device_id); + #ifdef __cplusplus } #endif diff --git a/llama.cpp b/llama.cpp index b1db5b179..cb6266a43 100644 --- a/llama.cpp +++ b/llama.cpp @@ -104,6 +104,7 @@ #define LLAMA_MAX_NODES 8192 #define LLAMA_MAX_EXPERTS 8 + // // logging // @@ -1429,7 +1430,9 @@ static ggml_backend_buffer_type_t llama_default_buffer_type_cpu(bool host_buffer buft = ggml_backend_cuda_host_buffer_type(); } #elif defined(GGML_USE_SYCL) - buft = ggml_backend_sycl_host_buffer_type(); + if (host_buffer) { + buft = ggml_backend_sycl_host_buffer_type(); + } #elif defined(GGML_USE_CPU_HBM) buft = ggml_backend_cpu_hbm_buffer_type(); #elif defined(GGML_USE_VULKAN) @@ -1483,6 +1486,12 @@ static ggml_backend_buffer_type_t llama_default_buffer_type_split(int fallback_g } #endif +#ifdef GGML_USE_SYCL + if (ggml_backend_sycl_get_device_count() > 1) { + buft = ggml_backend_sycl_split_buffer_type(tensor_split); + } +#endif + if (buft == nullptr) { buft = llama_default_buffer_type_offload(fallback_gpu); } @@ -1494,6 +1503,8 @@ static ggml_backend_buffer_type_t llama_default_buffer_type_split(int fallback_g static size_t llama_get_device_count() { #if defined(GGML_USE_CUBLAS) return ggml_backend_cuda_get_device_count(); +#elif defined(GGML_USE_SYCL) + return ggml_backend_sycl_get_device_count(); #elif defined(GGML_USE_VULKAN) return ggml_backend_vk_get_device_count(); #else @@ -1507,6 +1518,11 @@ static size_t llama_get_device_memory(int device) { size_t free; ggml_backend_cuda_get_device_memory(device, &total, &free); return free; +#elif defined(GGML_USE_SYCL) + size_t total; + size_t free; + ggml_backend_sycl_get_device_memory(device, &total, &free); + return free; #elif defined(GGML_USE_VULKAN) size_t total; size_t free; @@ -12075,13 +12091,31 @@ struct llama_context * llama_new_context_with_model( } #elif defined(GGML_USE_SYCL) if (model->n_gpu_layers > 0) { - ggml_backend_t backend = ggml_backend_sycl_init(model->main_gpu); - if (backend == nullptr) { - LLAMA_LOG_ERROR("%s: failed to initialize SYCL%d backend\n", __func__, model->main_gpu); - llama_free(ctx); - return nullptr; + // with split_mode LLAMA_SPLIT_MODE_NONE or LLAMA_SPLIT_MODE_ROW, only the main GPU backend is used + if (model->split_mode == LLAMA_SPLIT_MODE_NONE || model->split_mode == LLAMA_SPLIT_MODE_ROW) { + int main_gpu_index = ggml_backend_sycl_get_device_index(model->main_gpu); + ggml_backend_t backend = ggml_backend_sycl_init(main_gpu_index); + if (backend == nullptr) { + LLAMA_LOG_ERROR("%s: failed to initialize SYCL%d (index %d)backend\n", __func__, model->main_gpu, main_gpu_index); + llama_free(ctx); + return nullptr; + } + ctx->backends.push_back(backend); + } else { + // LLAMA_SPLIT_LAYER requires a backend for each GPU + int id_list[GGML_SYCL_MAX_DEVICES]; + ggml_sycl_get_gpu_list(id_list, GGML_SYCL_MAX_DEVICES); + for (int i = 0; i < ggml_backend_sycl_get_device_count(); ++i) { + int device_id = id_list[i]; + ggml_backend_t backend = ggml_backend_sycl_init(i); + if (backend == nullptr) { + LLAMA_LOG_ERROR("%s: failed to initialize SYCL%d (index %d)backend\n", __func__, device_id, i); + llama_free(ctx); + return nullptr; + } + ctx->backends.push_back(backend); + } } - ctx->backends.push_back(backend); } #elif defined(GGML_USE_KOMPUTE) if (model->n_gpu_layers > 0) { @@ -12161,7 +12195,6 @@ struct llama_context * llama_new_context_with_model( ggml_set_name(ctx->inp_cls, "inp_cls"); ctx->buf_input = ggml_backend_alloc_ctx_tensors_from_buft(ctx->ctx_input, llama_default_buffer_type_cpu(true)); - LLAMA_LOG_INFO("%s: %10s input buffer size = %8.2f MiB\n", __func__, ggml_backend_buffer_name(ctx->buf_input), ggml_backend_buffer_get_size(ctx->buf_input) / 1024.0 / 1024.0); From 802da0091ba646ecf02e1a8fae2da0b8e76409bd Mon Sep 17 00:00:00 2001 From: compilade <113953597+compilade@users.noreply.github.com> Date: Sat, 2 Mar 2024 08:42:56 -0500 Subject: [PATCH 509/812] llama : fix segfault from unknown model arch name (#5820) * llama : fix segfault from unknown model arch name * llama : make all LLM maps const This also requires using `std::map::at` instead of its `operator[]` which does not exist for const maps. * llama : name LLM_ARCH_UNKNOWN to "(unknown)" This avoids errors from `std::map::at` when getting the general name of the model architecture. Using "(unknown)" instead of an empty string as per suggestion https://github.com/ggerganov/llama.cpp/pull/5820#issuecomment-1973735284 * llama : remove redundant inner const for LLM_TENSOR_NAMES The extra const won't do anything here as const maps return const references to values. Co-authored-by: Jared Van Bortel * llama : remove redundant nullptr check in llm_arch_from_string Since LLM_ARCH_NAMES is a const map, no spurious elements with a NULL name are inserted anymore, so this check is dead code. --------- Co-authored-by: Jared Van Bortel --- llama.cpp | 31 ++++++++++++++++--------------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/llama.cpp b/llama.cpp index cb6266a43..790c2740f 100644 --- a/llama.cpp +++ b/llama.cpp @@ -216,7 +216,7 @@ enum llm_arch { LLM_ARCH_UNKNOWN, }; -static std::map LLM_ARCH_NAMES = { +static const std::map LLM_ARCH_NAMES = { { LLM_ARCH_LLAMA, "llama" }, { LLM_ARCH_FALCON, "falcon" }, { LLM_ARCH_GPT2, "gpt2" }, @@ -241,6 +241,7 @@ static std::map LLM_ARCH_NAMES = { { LLM_ARCH_MINICPM, "minicpm" }, { LLM_ARCH_GEMMA, "gemma" }, { LLM_ARCH_STARCODER2, "starcoder2" }, + { LLM_ARCH_UNKNOWN, "(unknown)" }, }; enum llm_kv { @@ -301,7 +302,7 @@ enum llm_kv { LLM_KV_TOKENIZER_RWKV, }; -static std::map LLM_KV_NAMES = { +static const std::map LLM_KV_NAMES = { { LLM_KV_GENERAL_ARCHITECTURE, "general.architecture" }, { LLM_KV_GENERAL_QUANTIZATION_VERSION, "general.quantization_version" }, { LLM_KV_GENERAL_ALIGNMENT, "general.alignment" }, @@ -365,7 +366,7 @@ struct LLM_KV { llm_arch arch; std::string operator()(llm_kv kv) const { - return ::format(LLM_KV_NAMES[kv], LLM_ARCH_NAMES[arch]); + return ::format(LLM_KV_NAMES.at(kv), LLM_ARCH_NAMES.at(arch)); } }; @@ -400,7 +401,7 @@ enum llm_tensor { LLM_TENSOR_LAYER_OUT_NORM, }; -static std::map> LLM_TENSOR_NAMES = { +static const std::map> LLM_TENSOR_NAMES = { { LLM_ARCH_LLAMA, { @@ -833,38 +834,38 @@ struct LLM_TN { llm_arch arch; std::string operator()(llm_tensor tensor) const { - if (LLM_TENSOR_NAMES[arch].find(tensor) == LLM_TENSOR_NAMES[arch].end()) { + if (LLM_TENSOR_NAMES.at(arch).find(tensor) == LLM_TENSOR_NAMES.at(arch).end()) { return "__missing__"; } - return LLM_TENSOR_NAMES[arch].at(tensor); + return LLM_TENSOR_NAMES.at(arch).at(tensor); } std::string operator()(llm_tensor tensor, const std::string & suffix) const { - if (LLM_TENSOR_NAMES[arch].find(tensor) == LLM_TENSOR_NAMES[arch].end()) { + if (LLM_TENSOR_NAMES.at(arch).find(tensor) == LLM_TENSOR_NAMES.at(arch).end()) { return "__missing__"; } - return LLM_TENSOR_NAMES[arch].at(tensor) + "." + suffix; + return LLM_TENSOR_NAMES.at(arch).at(tensor) + "." + suffix; } std::string operator()(llm_tensor tensor, int bid) const { - if (LLM_TENSOR_NAMES[arch].find(tensor) == LLM_TENSOR_NAMES[arch].end()) { + if (LLM_TENSOR_NAMES.at(arch).find(tensor) == LLM_TENSOR_NAMES.at(arch).end()) { return "__missing__"; } - return ::format(LLM_TENSOR_NAMES[arch].at(tensor).c_str(), bid); + return ::format(LLM_TENSOR_NAMES.at(arch).at(tensor).c_str(), bid); } std::string operator()(llm_tensor tensor, const std::string & suffix, int bid) const { - if (LLM_TENSOR_NAMES[arch].find(tensor) == LLM_TENSOR_NAMES[arch].end()) { + if (LLM_TENSOR_NAMES.at(arch).find(tensor) == LLM_TENSOR_NAMES.at(arch).end()) { return "__missing__"; } - return ::format(LLM_TENSOR_NAMES[arch].at(tensor).c_str(), bid) + "." + suffix; + return ::format(LLM_TENSOR_NAMES.at(arch).at(tensor).c_str(), bid) + "." + suffix; } std::string operator()(llm_tensor tensor, const std::string & suffix, int bid, int xid) const { - if (LLM_TENSOR_NAMES[arch].find(tensor) == LLM_TENSOR_NAMES[arch].end()) { + if (LLM_TENSOR_NAMES.at(arch).find(tensor) == LLM_TENSOR_NAMES.at(arch).end()) { return "__missing__"; } - return ::format(LLM_TENSOR_NAMES[arch].at(tensor).c_str(), bid, xid) + "." + suffix; + return ::format(LLM_TENSOR_NAMES.at(arch).at(tensor).c_str(), bid, xid) + "." + suffix; } }; @@ -872,7 +873,7 @@ struct LLM_TN { // gguf helpers // -static std::map LLAMA_ROPE_SCALING_TYPES = { +static const std::map LLAMA_ROPE_SCALING_TYPES = { { LLAMA_ROPE_SCALING_TYPE_NONE, "none" }, { LLAMA_ROPE_SCALING_TYPE_LINEAR, "linear" }, { LLAMA_ROPE_SCALING_TYPE_YARN, "yarn" }, From 6c32d8c7ad8ba7b6ad2a162e929a21dd04fcdca0 Mon Sep 17 00:00:00 2001 From: Xuan Son Nguyen Date: Sat, 2 Mar 2024 15:19:09 +0100 Subject: [PATCH 510/812] llama : refactor internal quantization functions (#5830) --- llama.cpp | 81 +++++++++++++++++++++++++++++-------------------------- 1 file changed, 43 insertions(+), 38 deletions(-) diff --git a/llama.cpp b/llama.cpp index 790c2740f..697e85e89 100644 --- a/llama.cpp +++ b/llama.cpp @@ -10836,7 +10836,7 @@ struct quantize_state_internal { {} }; -static void llama_convert_tensor_internal( +static void llama_tensor_dequantize_internal( struct ggml_tensor * tensor, std::vector> & output, std::vector & workers, const size_t nelements, const int nthread ) { @@ -11177,6 +11177,46 @@ static ggml_type get_k_quant_type(quantize_state_internal & qs, ggml_type new_ty return new_type; } +static int32_t llama_tensor_quantize_internal(enum ggml_type new_type, const float * f32_data, void * new_data, const int chunk_size, int nrows, int n_per_row, int64_t * hist_cur, const float * imatrix, std::vector & workers, const int nthread) { + std::mutex mutex; + int counter = 0; + size_t new_size = 0; + if (nthread < 2) { + // single-thread + return ggml_quantize_chunk(new_type, f32_data, new_data, 0, nrows, n_per_row, hist_cur, imatrix); + } + auto compute = [&mutex, &counter, &hist_cur, &new_size, new_type, f32_data, new_data, chunk_size, + nrows, n_per_row, imatrix]() { + std::array local_hist = {}; + const int nrows_per_chunk = chunk_size / n_per_row; + size_t local_size = 0; + while (true) { + std::unique_lock lock(mutex); + int first_row = counter; counter += nrows_per_chunk; + if (first_row >= nrows) { + if (local_size > 0) { + for (int j=0; jftype; @@ -11289,7 +11329,6 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s std::vector workers; workers.reserve(nthread); - std::mutex mutex; int idx = 0; @@ -11403,7 +11442,7 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s } else if (ggml_is_quantized(tensor->type) && !params->allow_requantize) { throw std::runtime_error(format("requantizing from type %s is disabled", ggml_type_name(tensor->type))); } else { - llama_convert_tensor_internal(tensor, f32_conv_buf, workers, nelements, nthread); + llama_tensor_dequantize_internal(tensor, f32_conv_buf, workers, nelements, nthread); f32_data = (float *) f32_conv_buf.data(); } @@ -11424,41 +11463,7 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s const int nchunk = (nelements + chunk_size - 1)/chunk_size; const int nthread_use = nthread > 1 ? std::max(1, std::min(nthread, nchunk)) : 1; - if (nthread_use < 2) { - new_size = ggml_quantize_chunk(new_type, f32_data, new_data, 0, nrows, n_per_row, hist_cur.data(), imatrix); - } else { - int counter = 0; - new_size = 0; - auto compute = [&mutex, &counter, &hist_cur, &new_size, new_type, f32_data, new_data, chunk_size, - nrows, n_per_row, imatrix]() { - std::array local_hist = {}; - const int nrows_per_chunk = chunk_size / n_per_row; - size_t local_size = 0; - while (true) { - std::unique_lock lock(mutex); - int first_row = counter; counter += nrows_per_chunk; - if (first_row >= nrows) { - if (local_size > 0) { - for (int j=0; j %8.2f MiB", ggml_nbytes(tensor)/1024.0/1024.0, new_size/1024.0/1024.0); int64_t tot_count = 0; From ef2cd694c4155fbf25bae61c5178c47eb3676dba Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Sat, 2 Mar 2024 16:54:08 +0200 Subject: [PATCH 511/812] scripts : add pod-llama.sh --- scripts/pod-llama.sh | 213 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 213 insertions(+) create mode 100644 scripts/pod-llama.sh diff --git a/scripts/pod-llama.sh b/scripts/pod-llama.sh new file mode 100644 index 000000000..6cf1ab4f3 --- /dev/null +++ b/scripts/pod-llama.sh @@ -0,0 +1,213 @@ +#!/bin/bash +# +# Use this script only on fresh pods (runpod.io)! +# Otherwise, it can break your environment! +# + +if [ -z "$1" ]; then + echo "Usage: $0 " + echo " 0: no models" + echo " 1: tinyllama-1b" + echo " 2: codellama-7b" + echo " 3: codellama-13b" + echo " 4: codellama-34b" + echo " 5: codellama-7b-instruct" + echo " 6: codellama-13b-instruct" + echo " 7: codellama-34b-instruct" + + exit 1 +fi + +set -x + +# setup deps +apt-get update +apt-get install -y git-lfs cmake cmake-curses-gui vim ruby +git-lfs install + +if [ ! -d "/workspace" ]; then + ln -sfn $(pwd) /workspace +fi + +# download data +cd /workspace + +# this is useful to git clone repos without doubling the disk size due to .git +git clone https://github.com/iboB/git-lfs-download +ln -sfn /workspace/git-lfs-download/git-lfs-download /usr/local/bin/git-lfs-download + +# llama.cpp +cd /workspace +git clone https://github.com/ggerganov/llama.cpp + +cd llama.cpp + +LLAMA_CUBLAS=1 make -j + +ln -sfn /workspace/TinyLlama-1.1B-Chat-v0.3 ./models/tinyllama-1b +ln -sfn /workspace/CodeLlama-7b-hf ./models/codellama-7b +ln -sfn /workspace/CodeLlama-13b-hf ./models/codellama-13b +ln -sfn /workspace/CodeLlama-34b-hf ./models/codellama-34b +ln -sfn /workspace/CodeLlama-7b-Instruct-hf ./models/codellama-7b-instruct +ln -sfn /workspace/CodeLlama-13b-Instruct-hf ./models/codellama-13b-instruct +ln -sfn /workspace/CodeLlama-34b-Instruct-hf ./models/codellama-34b-instruct + +pip install -r requirements.txt + +# cmake +cd /workspace/llama.cpp + +mkdir build-cublas +cd build-cublas + +cmake -DLLAMA_CUBLAS=1 ../ +make -j + +if [ "$1" -eq "0" ]; then + exit 0 +fi + +# more models +if [ "$1" -eq "1" ]; then + cd /workspace + + git-lfs-download https://huggingface.co/PY007/TinyLlama-1.1B-Chat-v0.3 + + cd /workspace/llama.cpp + + python3 convert.py ./models/tinyllama-1b --outfile ./models/tinyllama-1b/ggml-model-f16.gguf --outtype f16 + + ./quantize ./models/tinyllama-1b/ggml-model-f16.gguf ./models/tinyllama-1b/ggml-model-q4_0.gguf q4_0 + ./quantize ./models/tinyllama-1b/ggml-model-f16.gguf ./models/tinyllama-1b/ggml-model-q4_k.gguf q4_k + ./quantize ./models/tinyllama-1b/ggml-model-f16.gguf ./models/tinyllama-1b/ggml-model-q8_0.gguf q8_0 +fi + +if [ "$1" -eq "2" ]; then + cd /workspace + + git-lfs-download https://huggingface.co/codellama/CodeLlama-7b-hf --without *safetensors* + rm -v ./CodeLlama-7b-hf/*safetensors* + + cd /workspace/llama.cpp + + python3 convert.py ./models/codellama-7b --outfile ./models/codellama-7b/ggml-model-f16.gguf --outtype f16 + + ./quantize ./models/codellama-7b/ggml-model-f16.gguf ./models/codellama-7b/ggml-model-q4_0.gguf q4_0 + ./quantize ./models/codellama-7b/ggml-model-f16.gguf ./models/codellama-7b/ggml-model-q4_k.gguf q4_k + ./quantize ./models/codellama-7b/ggml-model-f16.gguf ./models/codellama-7b/ggml-model-q8_0.gguf q8_0 +fi + +if [ "$1" -eq "3" ]; then + cd /workspace + + git-lfs-download https://huggingface.co/codellama/CodeLlama-13b-hf --without *safetensors* + rm -v ./CodeLlama-13b-hf/*safetensors* + + cd /workspace/llama.cpp + + python3 convert.py ./models/codellama-13b --outfile ./models/codellama-13b/ggml-model-f16.gguf --outtype f16 + + ./quantize ./models/codellama-13b/ggml-model-f16.gguf ./models/codellama-13b/ggml-model-q4_0.gguf q4_0 + ./quantize ./models/codellama-13b/ggml-model-f16.gguf ./models/codellama-13b/ggml-model-q4_k.gguf q4_k + ./quantize ./models/codellama-13b/ggml-model-f16.gguf ./models/codellama-13b/ggml-model-q8_0.gguf q8_0 +fi + +if [ "$1" -eq "4" ]; then + cd /workspace + + git-lfs-download https://huggingface.co/codellama/CodeLlama-34b-hf --without *safetensors* + rm -v ./CodeLlama-34b-hf/*safetensors* + + cd /workspace/llama.cpp + + python3 convert.py ./models/codellama-34b --outfile ./models/codellama-34b/ggml-model-f16.gguf --outtype f16 + + ./quantize ./models/codellama-34b/ggml-model-f16.gguf ./models/codellama-34b/ggml-model-q4_0.gguf q4_0 + ./quantize ./models/codellama-34b/ggml-model-f16.gguf ./models/codellama-34b/ggml-model-q4_k.gguf q4_k + ./quantize ./models/codellama-34b/ggml-model-f16.gguf ./models/codellama-34b/ggml-model-q8_0.gguf q8_0 +fi + +if [ "$1" -eq "5" ]; then + cd /workspace + + git-lfs-download https://huggingface.co/codellama/CodeLlama-7b-Instruct-hf --without *safetensors* + rm -v ./CodeLlama-7b-Instruct-hf/*safetensors* + + cd /workspace/llama.cpp + + python3 convert.py ./models/codellama-7b-instruct --outfile ./models/codellama-7b-instruct/ggml-model-f16.gguf --outtype f16 + + ./quantize ./models/codellama-7b-instruct/ggml-model-f16.gguf ./models/codellama-7b-instruct/ggml-model-q4_0.gguf q4_0 + ./quantize ./models/codellama-7b-instruct/ggml-model-f16.gguf ./models/codellama-7b-instruct/ggml-model-q4_k.gguf q4_k + ./quantize ./models/codellama-7b-instruct/ggml-model-f16.gguf ./models/codellama-7b-instruct/ggml-model-q8_0.gguf q8_0 +fi + +if [ "$1" -eq "6" ]; then + cd /workspace + + git-lfs-download https://huggingface.co/codellama/CodeLlama-13b-Instruct-hf --without *safetensors* + rm -v ./CodeLlama-13b-Instruct-hf/*safetensors* + + cd /workspace/llama.cpp + + python3 convert.py ./models/codellama-13b-instruct --outfile ./models/codellama-13b-instruct/ggml-model-f16.gguf --outtype f16 + + ./quantize ./models/codellama-13b-instruct/ggml-model-f16.gguf ./models/codellama-13b-instruct/ggml-model-q4_0.gguf q4_0 + ./quantize ./models/codellama-13b-instruct/ggml-model-f16.gguf ./models/codellama-13b-instruct/ggml-model-q4_k.gguf q4_k + ./quantize ./models/codellama-13b-instruct/ggml-model-f16.gguf ./models/codellama-13b-instruct/ggml-model-q8_0.gguf q8_0 +fi + +if [ "$1" -eq "7" ]; then + cd /workspace + + git-lfs-download https://huggingface.co/codellama/CodeLlama-34b-Instruct-hf --without *safetensors* + rm -v ./CodeLlama-34b-Instruct-hf/*safetensors* + + cd /workspace/llama.cpp + + python3 convert.py ./models/codellama-34b-instruct --outfile ./models/codellama-34b-instruct/ggml-model-f16.gguf --outtype f16 + + ./quantize ./models/codellama-34b-instruct/ggml-model-f16.gguf ./models/codellama-34b-instruct/ggml-model-q4_0.gguf q4_0 + ./quantize ./models/codellama-34b-instruct/ggml-model-f16.gguf ./models/codellama-34b-instruct/ggml-model-q4_k.gguf q4_k + ./quantize ./models/codellama-34b-instruct/ggml-model-f16.gguf ./models/codellama-34b-instruct/ggml-model-q8_0.gguf q8_0 +fi + +if [ "$1" -eq "1" ]; then + # perf + perplexity + cd /workspace/llama.cpp/build-cublas + + make -j && ../scripts/run-all-perf.sh tinyllama-1b "f16" "-ngl 99 -t 1 -p 1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,32,64,128,256,512,1024,2048 -n 128" + + ../scripts/get-wikitext-2.sh + unzip wikitext-2-raw-v1.zip + + make -j && ./bin/perplexity -m ../models/tinyllama-1b/ggml-model-f16.gguf -f ./wikitext-2-raw/wiki.test.raw -ngl 100 --chunks 32 + + # batched + cd /workspace/llama.cpp + + LLAMA_CUBLAS=1 make -j && ./batched ./models/tinyllama-1b/ggml-model-f16.gguf "Hello, my name is" 8 128 999 + + # batched-bench + cd /workspace/llama.cpp + + LLAMA_CUBLAS=1 make -j && ./batched-bench ./models/tinyllama-1b/ggml-model-f16.gguf 4608 1 99 0 512 128 1,2,3,4,5,6,7,8,16,32 + + # parallel + cd /workspace/llama.cpp + + LLAMA_CUBLAS=1 make -j && ./parallel -m ./models/tinyllama-1b/ggml-model-f16.gguf -t 1 -ngl 100 -c 4096 -b 512 -s 1 -np 8 -ns 128 -n 100 -cb + +fi + +# speculative +#if [ "$1" -eq "7" ]; then +# cd /workspace/llama.cpp +# +# LLAMA_CUBLAS=1 make -j && ./speculative -m ./models/codellama-34b-instruct/ggml-model-f16.gguf -md ./models/codellama-7b-instruct/ggml-model-q4_0.gguf -p "# Dijkstra's shortest path algorithm in Python (4 spaces indentation) + complexity analysis:\n\n" -e -ngl 999 -ngld 999 -t 4 -n 512 -c 4096 -s 21 --draft 16 -np 1 --temp 0.0 +#fi + +# more benches +#LLAMA_CUBLAS=1 make -j && ./batched-bench ./models/codellama-7b/ggml-model-q4_k.gguf 4096 1 99 1 512,3200 128,128,800 1 +#LLAMA_CUBLAS=1 make -j && ./batched-bench ./models/codellama-13b/ggml-model-q4_k.gguf 4096 1 99 1 512,3200 128,128,800 1 + From bbde6eb2561153aabbdfac5001c690fe00cad639 Mon Sep 17 00:00:00 2001 From: Kawrakow <48489457+ikawrakow@users.noreply.github.com> Date: Sat, 2 Mar 2024 17:00:51 +0200 Subject: [PATCH 512/812] ggml : IQ3_S improvements (#5829) * iq3_s: somewhat faster AVX2 dot product On Ryzen a 7950X TG-128 increases to 16 t/s from 15.5 t/s using 16 threads. For 8 threads it is 13.85 t/s vs 11.75 t/s. PP-512 increases to 28.5 t/s from 23.8 t/s. * iq3_s: somewhat faster ARM_NEON dot product Still dog slow - 10.7 t/s up from 9.9 t/s. * iq3_s: another small ARM_NEON improvement 10.7 -> 11.0 t/s. Using vmulq_s8 is faster than the xor - sub trick that works best on AVX2. * iq3_s: minor improvement on Metal 49.4 t/s -> 50.3 t/s * iq3_s: PPL improvement E.g., for a context of 4096 LLaMA-v2-7B goes to 5.1340 from 5.1653. * iq3_s: use new grid everywhere * Fix ARM_NEON --------- Co-authored-by: Iwan Kawrakow --- ggml-cuda.cu | 143 ++++++++++++------------ ggml-metal.metal | 152 ++++++++++++------------- ggml-quants.c | 280 +++++++++++++++++++++++++++-------------------- 3 files changed, 310 insertions(+), 265 deletions(-) diff --git a/ggml-cuda.cu b/ggml-cuda.cu index 0c6501e98..7ed97430f 100644 --- a/ggml-cuda.cu +++ b/ggml-cuda.cu @@ -2018,74 +2018,73 @@ static const __device__ uint32_t iq3xxs_grid[256] = { 0x3e1c1c1c, 0x3e1c3404, 0x3e24140c, 0x3e24240c, 0x3e2c0404, 0x3e2c0414, 0x3e2c1424, 0x3e341c04, }; -static const __device__ uint32_t iq3xs_grid[512] = { - 0x04040404, 0x0404040c, 0x04040414, 0x0404042c, 0x0404043e, 0x04040c04, 0x04040c0c, 0x04040c14, - 0x04040c24, 0x04040c34, 0x04041404, 0x0404140c, 0x0404142c, 0x04041c1c, 0x04042404, 0x04042414, - 0x0404242c, 0x0404243e, 0x04042c0c, 0x04042c1c, 0x04043404, 0x04043414, 0x04043e0c, 0x04043e24, - 0x04043e3e, 0x040c0404, 0x040c040c, 0x040c0414, 0x040c0424, 0x040c0c04, 0x040c0c0c, 0x040c0c2c, - 0x040c1404, 0x040c141c, 0x040c143e, 0x040c1c0c, 0x040c1c2c, 0x040c2424, 0x040c340c, 0x040c342c, - 0x040c3e14, 0x04140404, 0x0414040c, 0x0414042c, 0x0414043e, 0x04140c04, 0x04140c1c, 0x04140c34, - 0x0414140c, 0x0414142c, 0x04141c04, 0x04141c24, 0x04142414, 0x0414242c, 0x0414243e, 0x04142c0c, - 0x04142c1c, 0x04143e04, 0x04143e1c, 0x041c041c, 0x041c0c0c, 0x041c0c2c, 0x041c1404, 0x041c1414, - 0x041c1c0c, 0x041c1c1c, 0x041c1c34, 0x041c2424, 0x041c2c04, 0x041c2c14, 0x041c343e, 0x041c3e0c, - 0x041c3e2c, 0x04240404, 0x04240c1c, 0x04240c3e, 0x0424140c, 0x04241424, 0x04241c14, 0x04242404, - 0x0424241c, 0x04242c0c, 0x04243e04, 0x042c0414, 0x042c0424, 0x042c1404, 0x042c1414, 0x042c1434, - 0x042c1c1c, 0x042c240c, 0x042c242c, 0x042c243e, 0x042c3434, 0x042c3e1c, 0x04340434, 0x04340c0c, - 0x04340c1c, 0x04341c0c, 0x04342c14, 0x04343e0c, 0x043e0404, 0x043e0414, 0x043e0424, 0x043e1404, - 0x043e1414, 0x043e1434, 0x043e1c1c, 0x043e2c04, 0x043e2c24, 0x0c040404, 0x0c04040c, 0x0c040414, - 0x0c040424, 0x0c040c04, 0x0c040c0c, 0x0c040c1c, 0x0c040c2c, 0x0c040c3e, 0x0c041404, 0x0c041414, - 0x0c041c0c, 0x0c041c24, 0x0c041c34, 0x0c042c24, 0x0c042c34, 0x0c04340c, 0x0c043e14, 0x0c0c0404, - 0x0c0c040c, 0x0c0c041c, 0x0c0c0434, 0x0c0c0c04, 0x0c0c0c24, 0x0c0c140c, 0x0c0c1c04, 0x0c0c1c1c, - 0x0c0c240c, 0x0c0c2c04, 0x0c0c2c14, 0x0c0c3e04, 0x0c0c3e34, 0x0c140404, 0x0c140c14, 0x0c140c2c, - 0x0c140c3e, 0x0c141404, 0x0c141424, 0x0c141c14, 0x0c142404, 0x0c14241c, 0x0c142c2c, 0x0c143404, - 0x0c143e14, 0x0c1c040c, 0x0c1c0424, 0x0c1c043e, 0x0c1c0c04, 0x0c1c0c1c, 0x0c1c140c, 0x0c1c143e, - 0x0c1c1c04, 0x0c1c1c24, 0x0c1c240c, 0x0c1c3414, 0x0c1c3e04, 0x0c24041c, 0x0c24042c, 0x0c240c14, - 0x0c240c24, 0x0c241c0c, 0x0c241c1c, 0x0c242414, 0x0c242434, 0x0c242c04, 0x0c242c24, 0x0c2c040c, - 0x0c2c0c04, 0x0c2c0c1c, 0x0c2c140c, 0x0c2c1c04, 0x0c2c1c14, 0x0c2c2c0c, 0x0c341404, 0x0c341424, - 0x0c34143e, 0x0c342424, 0x0c342434, 0x0c3e040c, 0x0c3e041c, 0x0c3e0c04, 0x0c3e0c14, 0x0c3e140c, - 0x0c3e1c2c, 0x0c3e240c, 0x0c3e3414, 0x0c3e3e04, 0x14040404, 0x1404040c, 0x1404041c, 0x1404042c, - 0x1404043e, 0x14040c04, 0x14040c14, 0x14040c24, 0x14040c34, 0x1404140c, 0x1404141c, 0x1404143e, - 0x14041c04, 0x14041c14, 0x1404240c, 0x1404241c, 0x1404242c, 0x14042c04, 0x14042c14, 0x1404343e, - 0x14043e04, 0x14043e1c, 0x14043e2c, 0x140c0404, 0x140c0414, 0x140c0c04, 0x140c0c1c, 0x140c0c3e, - 0x140c1414, 0x140c142c, 0x140c1c0c, 0x140c1c24, 0x140c2414, 0x140c2c0c, 0x1414040c, 0x14140424, - 0x1414043e, 0x1414140c, 0x1414141c, 0x14141c04, 0x14141c3e, 0x1414240c, 0x14142c1c, 0x14142c3e, - 0x14143e0c, 0x14143e24, 0x141c0404, 0x141c0414, 0x141c042c, 0x141c0c0c, 0x141c1414, 0x141c1424, - 0x141c1c0c, 0x141c1c1c, 0x141c2414, 0x141c2c04, 0x141c3434, 0x1424040c, 0x1424043e, 0x14241404, - 0x1424141c, 0x14241c14, 0x14241c2c, 0x1424240c, 0x14243e14, 0x14243e2c, 0x142c0424, 0x142c0c0c, - 0x142c1414, 0x142c1c3e, 0x142c2404, 0x142c2c1c, 0x142c3e04, 0x14340404, 0x14340414, 0x1434043e, - 0x1434140c, 0x14342c2c, 0x1434340c, 0x143e042c, 0x143e0c0c, 0x143e1434, 0x143e1c04, 0x143e241c, - 0x143e2c04, 0x1c040414, 0x1c040c0c, 0x1c040c1c, 0x1c040c2c, 0x1c040c3e, 0x1c041414, 0x1c041c0c, - 0x1c041c1c, 0x1c041c2c, 0x1c042414, 0x1c042424, 0x1c04243e, 0x1c042c0c, 0x1c04341c, 0x1c043e0c, - 0x1c0c040c, 0x1c0c041c, 0x1c0c042c, 0x1c0c0c24, 0x1c0c140c, 0x1c0c141c, 0x1c0c2404, 0x1c0c3404, - 0x1c0c3e14, 0x1c0c3e34, 0x1c140404, 0x1c140c14, 0x1c141404, 0x1c141c14, 0x1c141c24, 0x1c142c04, - 0x1c1c040c, 0x1c1c0c04, 0x1c1c0c24, 0x1c1c140c, 0x1c1c141c, 0x1c1c143e, 0x1c1c1c04, 0x1c1c240c, - 0x1c1c241c, 0x1c1c243e, 0x1c1c2c2c, 0x1c1c3e1c, 0x1c24041c, 0x1c240c0c, 0x1c240c34, 0x1c241414, - 0x1c241c0c, 0x1c242c14, 0x1c243404, 0x1c243424, 0x1c2c040c, 0x1c2c0c04, 0x1c2c0c14, 0x1c2c142c, - 0x1c2c1c14, 0x1c2c2424, 0x1c2c2c34, 0x1c2c3e1c, 0x1c340c34, 0x1c34240c, 0x1c3e040c, 0x1c3e041c, - 0x1c3e1404, 0x1c3e1414, 0x1c3e1c2c, 0x24040404, 0x24040424, 0x24040c14, 0x24041404, 0x24041424, - 0x2404143e, 0x24041c14, 0x2404240c, 0x24042c04, 0x24043e04, 0x240c0414, 0x240c043e, 0x240c0c0c, - 0x240c0c1c, 0x240c1414, 0x240c1c04, 0x240c1c2c, 0x240c241c, 0x240c2c0c, 0x240c2c2c, 0x2414040c, - 0x2414041c, 0x24140c04, 0x24140c2c, 0x2414140c, 0x24141c1c, 0x24142404, 0x24142c3e, 0x24143414, - 0x24143e04, 0x241c0424, 0x241c0c0c, 0x241c0c1c, 0x241c1404, 0x241c1414, 0x241c1c0c, 0x241c1c2c, - 0x24240404, 0x24240414, 0x24241424, 0x24241c3e, 0x24242404, 0x24243e0c, 0x242c042c, 0x242c043e, - 0x242c140c, 0x242c3414, 0x24340c1c, 0x24341c24, 0x24343404, 0x243e0c04, 0x243e0c2c, 0x243e1c04, - 0x243e241c, 0x243e2c0c, 0x2c040414, 0x2c040c04, 0x2c040c24, 0x2c041414, 0x2c042404, 0x2c042424, - 0x2c04243e, 0x2c042c14, 0x2c043434, 0x2c043e24, 0x2c0c040c, 0x2c0c041c, 0x2c0c042c, 0x2c0c0c14, - 0x2c0c140c, 0x2c0c1c14, 0x2c0c3e14, 0x2c140404, 0x2c140c0c, 0x2c14141c, 0x2c141c04, 0x2c141c34, - 0x2c142c1c, 0x2c1c0414, 0x2c1c043e, 0x2c1c0c04, 0x2c1c143e, 0x2c1c2424, 0x2c1c2c0c, 0x2c1c342c, - 0x2c1c3e1c, 0x2c24040c, 0x2c240424, 0x2c241404, 0x2c241c14, 0x2c242434, 0x2c2c0c14, 0x2c2c1434, - 0x2c2c2c0c, 0x2c2c2c1c, 0x2c342414, 0x2c3e0414, 0x2c3e0424, 0x2c3e1414, 0x34040c0c, 0x34040c1c, - 0x34040c2c, 0x34041c0c, 0x34041c1c, 0x34043404, 0x340c0404, 0x340c1404, 0x340c143e, 0x340c3424, - 0x34140c14, 0x34141c24, 0x34142414, 0x34142c2c, 0x34143414, 0x34143e04, 0x341c0404, 0x341c0c24, - 0x341c140c, 0x341c2404, 0x3424142c, 0x3424241c, 0x34243414, 0x342c0404, 0x342c041c, 0x342c1c24, - 0x342c3404, 0x3434042c, 0x34342404, 0x343e0c0c, 0x343e0c1c, 0x3e040404, 0x3e040424, 0x3e04043e, - 0x3e041404, 0x3e041414, 0x3e041c34, 0x3e042404, 0x3e042c24, 0x3e043414, 0x3e0c0414, 0x3e0c0c0c, - 0x3e0c1424, 0x3e0c241c, 0x3e0c242c, 0x3e14040c, 0x3e140424, 0x3e140c04, 0x3e140c34, 0x3e14140c, - 0x3e141c04, 0x3e142c0c, 0x3e1c0414, 0x3e1c1c14, 0x3e1c1c2c, 0x3e1c2c1c, 0x3e24040c, 0x3e24042c, - 0x3e240c1c, 0x3e241404, 0x3e242c04, 0x3e2c1414, 0x3e2c2414, 0x3e340414, 0x3e341c0c, 0x3e3e0404, +static const __device__ uint32_t iq3s_grid[512] = { + 0x01010101, 0x01010103, 0x01010105, 0x0101010b, 0x0101010f, 0x01010301, 0x01010303, 0x01010305, + 0x01010309, 0x0101030d, 0x01010501, 0x01010503, 0x0101050b, 0x01010707, 0x01010901, 0x01010905, + 0x0101090b, 0x0101090f, 0x01010b03, 0x01010b07, 0x01010d01, 0x01010d05, 0x01010f03, 0x01010f09, + 0x01010f0f, 0x01030101, 0x01030103, 0x01030105, 0x01030109, 0x01030301, 0x01030303, 0x0103030b, + 0x01030501, 0x01030507, 0x0103050f, 0x01030703, 0x0103070b, 0x01030909, 0x01030d03, 0x01030d0b, + 0x01030f05, 0x01050101, 0x01050103, 0x0105010b, 0x0105010f, 0x01050301, 0x01050307, 0x0105030d, + 0x01050503, 0x0105050b, 0x01050701, 0x01050709, 0x01050905, 0x0105090b, 0x0105090f, 0x01050b03, + 0x01050b07, 0x01050f01, 0x01050f07, 0x01070107, 0x01070303, 0x0107030b, 0x01070501, 0x01070505, + 0x01070703, 0x01070707, 0x0107070d, 0x01070909, 0x01070b01, 0x01070b05, 0x01070d0f, 0x01070f03, + 0x01070f0b, 0x01090101, 0x01090307, 0x0109030f, 0x01090503, 0x01090509, 0x01090705, 0x01090901, + 0x01090907, 0x01090b03, 0x01090f01, 0x010b0105, 0x010b0109, 0x010b0501, 0x010b0505, 0x010b050d, + 0x010b0707, 0x010b0903, 0x010b090b, 0x010b090f, 0x010b0d0d, 0x010b0f07, 0x010d010d, 0x010d0303, + 0x010d0307, 0x010d0703, 0x010d0b05, 0x010d0f03, 0x010f0101, 0x010f0105, 0x010f0109, 0x010f0501, + 0x010f0505, 0x010f050d, 0x010f0707, 0x010f0b01, 0x010f0b09, 0x03010101, 0x03010103, 0x03010105, + 0x03010109, 0x03010301, 0x03010303, 0x03010307, 0x0301030b, 0x0301030f, 0x03010501, 0x03010505, + 0x03010703, 0x03010709, 0x0301070d, 0x03010b09, 0x03010b0d, 0x03010d03, 0x03010f05, 0x03030101, + 0x03030103, 0x03030107, 0x0303010d, 0x03030301, 0x03030309, 0x03030503, 0x03030701, 0x03030707, + 0x03030903, 0x03030b01, 0x03030b05, 0x03030f01, 0x03030f0d, 0x03050101, 0x03050305, 0x0305030b, + 0x0305030f, 0x03050501, 0x03050509, 0x03050705, 0x03050901, 0x03050907, 0x03050b0b, 0x03050d01, + 0x03050f05, 0x03070103, 0x03070109, 0x0307010f, 0x03070301, 0x03070307, 0x03070503, 0x0307050f, + 0x03070701, 0x03070709, 0x03070903, 0x03070d05, 0x03070f01, 0x03090107, 0x0309010b, 0x03090305, + 0x03090309, 0x03090703, 0x03090707, 0x03090905, 0x0309090d, 0x03090b01, 0x03090b09, 0x030b0103, + 0x030b0301, 0x030b0307, 0x030b0503, 0x030b0701, 0x030b0705, 0x030b0b03, 0x030d0501, 0x030d0509, + 0x030d050f, 0x030d0909, 0x030d090d, 0x030f0103, 0x030f0107, 0x030f0301, 0x030f0305, 0x030f0503, + 0x030f070b, 0x030f0903, 0x030f0d05, 0x030f0f01, 0x05010101, 0x05010103, 0x05010107, 0x0501010b, + 0x0501010f, 0x05010301, 0x05010305, 0x05010309, 0x0501030d, 0x05010503, 0x05010507, 0x0501050f, + 0x05010701, 0x05010705, 0x05010903, 0x05010907, 0x0501090b, 0x05010b01, 0x05010b05, 0x05010d0f, + 0x05010f01, 0x05010f07, 0x05010f0b, 0x05030101, 0x05030105, 0x05030301, 0x05030307, 0x0503030f, + 0x05030505, 0x0503050b, 0x05030703, 0x05030709, 0x05030905, 0x05030b03, 0x05050103, 0x05050109, + 0x0505010f, 0x05050503, 0x05050507, 0x05050701, 0x0505070f, 0x05050903, 0x05050b07, 0x05050b0f, + 0x05050f03, 0x05050f09, 0x05070101, 0x05070105, 0x0507010b, 0x05070303, 0x05070505, 0x05070509, + 0x05070703, 0x05070707, 0x05070905, 0x05070b01, 0x05070d0d, 0x05090103, 0x0509010f, 0x05090501, + 0x05090507, 0x05090705, 0x0509070b, 0x05090903, 0x05090f05, 0x05090f0b, 0x050b0109, 0x050b0303, + 0x050b0505, 0x050b070f, 0x050b0901, 0x050b0b07, 0x050b0f01, 0x050d0101, 0x050d0105, 0x050d010f, + 0x050d0503, 0x050d0b0b, 0x050d0d03, 0x050f010b, 0x050f0303, 0x050f050d, 0x050f0701, 0x050f0907, + 0x050f0b01, 0x07010105, 0x07010303, 0x07010307, 0x0701030b, 0x0701030f, 0x07010505, 0x07010703, + 0x07010707, 0x0701070b, 0x07010905, 0x07010909, 0x0701090f, 0x07010b03, 0x07010d07, 0x07010f03, + 0x07030103, 0x07030107, 0x0703010b, 0x07030309, 0x07030503, 0x07030507, 0x07030901, 0x07030d01, + 0x07030f05, 0x07030f0d, 0x07050101, 0x07050305, 0x07050501, 0x07050705, 0x07050709, 0x07050b01, + 0x07070103, 0x07070301, 0x07070309, 0x07070503, 0x07070507, 0x0707050f, 0x07070701, 0x07070903, + 0x07070907, 0x0707090f, 0x07070b0b, 0x07070f07, 0x07090107, 0x07090303, 0x0709030d, 0x07090505, + 0x07090703, 0x07090b05, 0x07090d01, 0x07090d09, 0x070b0103, 0x070b0301, 0x070b0305, 0x070b050b, + 0x070b0705, 0x070b0909, 0x070b0b0d, 0x070b0f07, 0x070d030d, 0x070d0903, 0x070f0103, 0x070f0107, + 0x070f0501, 0x070f0505, 0x070f070b, 0x09010101, 0x09010109, 0x09010305, 0x09010501, 0x09010509, + 0x0901050f, 0x09010705, 0x09010903, 0x09010b01, 0x09010f01, 0x09030105, 0x0903010f, 0x09030303, + 0x09030307, 0x09030505, 0x09030701, 0x0903070b, 0x09030907, 0x09030b03, 0x09030b0b, 0x09050103, + 0x09050107, 0x09050301, 0x0905030b, 0x09050503, 0x09050707, 0x09050901, 0x09050b0f, 0x09050d05, + 0x09050f01, 0x09070109, 0x09070303, 0x09070307, 0x09070501, 0x09070505, 0x09070703, 0x0907070b, + 0x09090101, 0x09090105, 0x09090509, 0x0909070f, 0x09090901, 0x09090f03, 0x090b010b, 0x090b010f, + 0x090b0503, 0x090b0d05, 0x090d0307, 0x090d0709, 0x090d0d01, 0x090f0301, 0x090f030b, 0x090f0701, + 0x090f0907, 0x090f0b03, 0x0b010105, 0x0b010301, 0x0b010309, 0x0b010505, 0x0b010901, 0x0b010909, + 0x0b01090f, 0x0b010b05, 0x0b010d0d, 0x0b010f09, 0x0b030103, 0x0b030107, 0x0b03010b, 0x0b030305, + 0x0b030503, 0x0b030705, 0x0b030f05, 0x0b050101, 0x0b050303, 0x0b050507, 0x0b050701, 0x0b05070d, + 0x0b050b07, 0x0b070105, 0x0b07010f, 0x0b070301, 0x0b07050f, 0x0b070909, 0x0b070b03, 0x0b070d0b, + 0x0b070f07, 0x0b090103, 0x0b090109, 0x0b090501, 0x0b090705, 0x0b09090d, 0x0b0b0305, 0x0b0b050d, + 0x0b0b0b03, 0x0b0b0b07, 0x0b0d0905, 0x0b0f0105, 0x0b0f0109, 0x0b0f0505, 0x0d010303, 0x0d010307, + 0x0d01030b, 0x0d010703, 0x0d010707, 0x0d010d01, 0x0d030101, 0x0d030501, 0x0d03050f, 0x0d030d09, + 0x0d050305, 0x0d050709, 0x0d050905, 0x0d050b0b, 0x0d050d05, 0x0d050f01, 0x0d070101, 0x0d070309, + 0x0d070503, 0x0d070901, 0x0d09050b, 0x0d090907, 0x0d090d05, 0x0d0b0101, 0x0d0b0107, 0x0d0b0709, + 0x0d0b0d01, 0x0d0d010b, 0x0d0d0901, 0x0d0f0303, 0x0d0f0307, 0x0f010101, 0x0f010109, 0x0f01010f, + 0x0f010501, 0x0f010505, 0x0f01070d, 0x0f010901, 0x0f010b09, 0x0f010d05, 0x0f030105, 0x0f030303, + 0x0f030509, 0x0f030907, 0x0f03090b, 0x0f050103, 0x0f050109, 0x0f050301, 0x0f05030d, 0x0f050503, + 0x0f050701, 0x0f050b03, 0x0f070105, 0x0f070705, 0x0f07070b, 0x0f070b07, 0x0f090103, 0x0f09010b, + 0x0f090307, 0x0f090501, 0x0f090b01, 0x0f0b0505, 0x0f0b0905, 0x0f0d0105, 0x0f0d0703, 0x0f0f0101, }; - static const __device__ uint64_t iq1s_grid[512] = { 0xffffffffffff0101, 0xffffffffff01ff00, 0xffffffffff010100, 0xffffffff00000000, 0xffffffff01ff00ff, 0xffffffff01ff0001, 0xffffffff0101ffff, 0xffffffff0101ff01, @@ -2392,9 +2391,9 @@ static __global__ void dequantize_block_iq3_s(const void * __restrict__ vx, dst_ const int ib = tid%8; // 0...7 dst_t * y = yy + i*QK_K + 32*ib + 8*il; const uint8_t * qs = x[i].qs + 8*ib; - const uint8_t * grid1 = (const uint8_t *)(iq3xs_grid + (qs[2*il+0] | ((x[i].qh[ib] << (8-2*il)) & 256))); - const uint8_t * grid2 = (const uint8_t *)(iq3xs_grid + (qs[2*il+1] | ((x[i].qh[ib] << (7-2*il)) & 256))); - const float d = (float)x[i].d * (0.5f + ((x[i].scales[ib/2] >> 4*(ib%2)) & 0xf)) * 0.5f; + const uint8_t * grid1 = (const uint8_t *)(iq3s_grid + (qs[2*il+0] | ((x[i].qh[ib] << (8-2*il)) & 256))); + const uint8_t * grid2 = (const uint8_t *)(iq3s_grid + (qs[2*il+1] | ((x[i].qh[ib] << (7-2*il)) & 256))); + const float d = (float)x[i].d * (1 + 2*((x[i].scales[ib/2] >> 4*(ib%2)) & 0xf)); const uint8_t signs = x[i].signs[4*ib + il]; for (int j = 0; j < 4; ++j) { y[j+0] = d * grid1[j] * (signs & kmask_iq2xs[j+0] ? -1.f : 1.f); @@ -5211,8 +5210,8 @@ static __device__ __forceinline__ float vec_dot_iq3_s_q8_1( const int8_t * q8 = bq8_1[ib32].qs; int sumi = 0; for (int l = 0; l < 4; ++l) { - const uint32_t * grid1 = iq3xs_grid + (qs[2*l+0] | ((bq2->qh[ib32] << (8 - 2*l)) & 256)); - const uint32_t * grid2 = iq3xs_grid + (qs[2*l+1] | ((bq2->qh[ib32] << (7 - 2*l)) & 256)); + const uint32_t * grid1 = iq3s_grid + (qs[2*l+0] | ((bq2->qh[ib32] << (8 - 2*l)) & 256)); + const uint32_t * grid2 = iq3s_grid + (qs[2*l+1] | ((bq2->qh[ib32] << (7 - 2*l)) & 256)); uint32_t signs0 = __vcmpeq4(((bq2->signs[4*ib32+l] & 0xf) * 0x01010101) & 0x08040201, 0x08040201); uint32_t signs1 = __vcmpeq4(((bq2->signs[4*ib32+l] >> 4) * 0x01010101) & 0x08040201, 0x08040201); const int grid_l = __vsub4(grid1[0] ^ signs0, signs0); @@ -5221,7 +5220,7 @@ static __device__ __forceinline__ float vec_dot_iq3_s_q8_1( sumi = __dp4a(grid_h, *((int *)q8+1), sumi); q8 += 8; } - const float d = (float)bq2->d * (0.5f + ((bq2->scales[ib32/2] >> 4*(ib32%2)) & 0xf)) * __low2float(bq8_1[ib32].ds) * 0.5f; + const float d = (float)bq2->d * (1 + 2*((bq2->scales[ib32/2] >> 4*(ib32%2)) & 0xf)) * __low2float(bq8_1[ib32].ds); return d * sumi; #else assert(false); diff --git a/ggml-metal.metal b/ggml-metal.metal index 74a5e0b03..8b9488437 100644 --- a/ggml-metal.metal +++ b/ggml-metal.metal @@ -4087,71 +4087,71 @@ constexpr constant static uint32_t iq3xxs_grid[256] = { 0x3e1c1c1c, 0x3e1c3404, 0x3e24140c, 0x3e24240c, 0x3e2c0404, 0x3e2c0414, 0x3e2c1424, 0x3e341c04, }; -constexpr constant static uint32_t iq3xs_grid[512] = { - 0x04040404, 0x0404040c, 0x04040414, 0x0404042c, 0x0404043e, 0x04040c04, 0x04040c0c, 0x04040c14, - 0x04040c24, 0x04040c34, 0x04041404, 0x0404140c, 0x0404142c, 0x04041c1c, 0x04042404, 0x04042414, - 0x0404242c, 0x0404243e, 0x04042c0c, 0x04042c1c, 0x04043404, 0x04043414, 0x04043e0c, 0x04043e24, - 0x04043e3e, 0x040c0404, 0x040c040c, 0x040c0414, 0x040c0424, 0x040c0c04, 0x040c0c0c, 0x040c0c2c, - 0x040c1404, 0x040c141c, 0x040c143e, 0x040c1c0c, 0x040c1c2c, 0x040c2424, 0x040c340c, 0x040c342c, - 0x040c3e14, 0x04140404, 0x0414040c, 0x0414042c, 0x0414043e, 0x04140c04, 0x04140c1c, 0x04140c34, - 0x0414140c, 0x0414142c, 0x04141c04, 0x04141c24, 0x04142414, 0x0414242c, 0x0414243e, 0x04142c0c, - 0x04142c1c, 0x04143e04, 0x04143e1c, 0x041c041c, 0x041c0c0c, 0x041c0c2c, 0x041c1404, 0x041c1414, - 0x041c1c0c, 0x041c1c1c, 0x041c1c34, 0x041c2424, 0x041c2c04, 0x041c2c14, 0x041c343e, 0x041c3e0c, - 0x041c3e2c, 0x04240404, 0x04240c1c, 0x04240c3e, 0x0424140c, 0x04241424, 0x04241c14, 0x04242404, - 0x0424241c, 0x04242c0c, 0x04243e04, 0x042c0414, 0x042c0424, 0x042c1404, 0x042c1414, 0x042c1434, - 0x042c1c1c, 0x042c240c, 0x042c242c, 0x042c243e, 0x042c3434, 0x042c3e1c, 0x04340434, 0x04340c0c, - 0x04340c1c, 0x04341c0c, 0x04342c14, 0x04343e0c, 0x043e0404, 0x043e0414, 0x043e0424, 0x043e1404, - 0x043e1414, 0x043e1434, 0x043e1c1c, 0x043e2c04, 0x043e2c24, 0x0c040404, 0x0c04040c, 0x0c040414, - 0x0c040424, 0x0c040c04, 0x0c040c0c, 0x0c040c1c, 0x0c040c2c, 0x0c040c3e, 0x0c041404, 0x0c041414, - 0x0c041c0c, 0x0c041c24, 0x0c041c34, 0x0c042c24, 0x0c042c34, 0x0c04340c, 0x0c043e14, 0x0c0c0404, - 0x0c0c040c, 0x0c0c041c, 0x0c0c0434, 0x0c0c0c04, 0x0c0c0c24, 0x0c0c140c, 0x0c0c1c04, 0x0c0c1c1c, - 0x0c0c240c, 0x0c0c2c04, 0x0c0c2c14, 0x0c0c3e04, 0x0c0c3e34, 0x0c140404, 0x0c140c14, 0x0c140c2c, - 0x0c140c3e, 0x0c141404, 0x0c141424, 0x0c141c14, 0x0c142404, 0x0c14241c, 0x0c142c2c, 0x0c143404, - 0x0c143e14, 0x0c1c040c, 0x0c1c0424, 0x0c1c043e, 0x0c1c0c04, 0x0c1c0c1c, 0x0c1c140c, 0x0c1c143e, - 0x0c1c1c04, 0x0c1c1c24, 0x0c1c240c, 0x0c1c3414, 0x0c1c3e04, 0x0c24041c, 0x0c24042c, 0x0c240c14, - 0x0c240c24, 0x0c241c0c, 0x0c241c1c, 0x0c242414, 0x0c242434, 0x0c242c04, 0x0c242c24, 0x0c2c040c, - 0x0c2c0c04, 0x0c2c0c1c, 0x0c2c140c, 0x0c2c1c04, 0x0c2c1c14, 0x0c2c2c0c, 0x0c341404, 0x0c341424, - 0x0c34143e, 0x0c342424, 0x0c342434, 0x0c3e040c, 0x0c3e041c, 0x0c3e0c04, 0x0c3e0c14, 0x0c3e140c, - 0x0c3e1c2c, 0x0c3e240c, 0x0c3e3414, 0x0c3e3e04, 0x14040404, 0x1404040c, 0x1404041c, 0x1404042c, - 0x1404043e, 0x14040c04, 0x14040c14, 0x14040c24, 0x14040c34, 0x1404140c, 0x1404141c, 0x1404143e, - 0x14041c04, 0x14041c14, 0x1404240c, 0x1404241c, 0x1404242c, 0x14042c04, 0x14042c14, 0x1404343e, - 0x14043e04, 0x14043e1c, 0x14043e2c, 0x140c0404, 0x140c0414, 0x140c0c04, 0x140c0c1c, 0x140c0c3e, - 0x140c1414, 0x140c142c, 0x140c1c0c, 0x140c1c24, 0x140c2414, 0x140c2c0c, 0x1414040c, 0x14140424, - 0x1414043e, 0x1414140c, 0x1414141c, 0x14141c04, 0x14141c3e, 0x1414240c, 0x14142c1c, 0x14142c3e, - 0x14143e0c, 0x14143e24, 0x141c0404, 0x141c0414, 0x141c042c, 0x141c0c0c, 0x141c1414, 0x141c1424, - 0x141c1c0c, 0x141c1c1c, 0x141c2414, 0x141c2c04, 0x141c3434, 0x1424040c, 0x1424043e, 0x14241404, - 0x1424141c, 0x14241c14, 0x14241c2c, 0x1424240c, 0x14243e14, 0x14243e2c, 0x142c0424, 0x142c0c0c, - 0x142c1414, 0x142c1c3e, 0x142c2404, 0x142c2c1c, 0x142c3e04, 0x14340404, 0x14340414, 0x1434043e, - 0x1434140c, 0x14342c2c, 0x1434340c, 0x143e042c, 0x143e0c0c, 0x143e1434, 0x143e1c04, 0x143e241c, - 0x143e2c04, 0x1c040414, 0x1c040c0c, 0x1c040c1c, 0x1c040c2c, 0x1c040c3e, 0x1c041414, 0x1c041c0c, - 0x1c041c1c, 0x1c041c2c, 0x1c042414, 0x1c042424, 0x1c04243e, 0x1c042c0c, 0x1c04341c, 0x1c043e0c, - 0x1c0c040c, 0x1c0c041c, 0x1c0c042c, 0x1c0c0c24, 0x1c0c140c, 0x1c0c141c, 0x1c0c2404, 0x1c0c3404, - 0x1c0c3e14, 0x1c0c3e34, 0x1c140404, 0x1c140c14, 0x1c141404, 0x1c141c14, 0x1c141c24, 0x1c142c04, - 0x1c1c040c, 0x1c1c0c04, 0x1c1c0c24, 0x1c1c140c, 0x1c1c141c, 0x1c1c143e, 0x1c1c1c04, 0x1c1c240c, - 0x1c1c241c, 0x1c1c243e, 0x1c1c2c2c, 0x1c1c3e1c, 0x1c24041c, 0x1c240c0c, 0x1c240c34, 0x1c241414, - 0x1c241c0c, 0x1c242c14, 0x1c243404, 0x1c243424, 0x1c2c040c, 0x1c2c0c04, 0x1c2c0c14, 0x1c2c142c, - 0x1c2c1c14, 0x1c2c2424, 0x1c2c2c34, 0x1c2c3e1c, 0x1c340c34, 0x1c34240c, 0x1c3e040c, 0x1c3e041c, - 0x1c3e1404, 0x1c3e1414, 0x1c3e1c2c, 0x24040404, 0x24040424, 0x24040c14, 0x24041404, 0x24041424, - 0x2404143e, 0x24041c14, 0x2404240c, 0x24042c04, 0x24043e04, 0x240c0414, 0x240c043e, 0x240c0c0c, - 0x240c0c1c, 0x240c1414, 0x240c1c04, 0x240c1c2c, 0x240c241c, 0x240c2c0c, 0x240c2c2c, 0x2414040c, - 0x2414041c, 0x24140c04, 0x24140c2c, 0x2414140c, 0x24141c1c, 0x24142404, 0x24142c3e, 0x24143414, - 0x24143e04, 0x241c0424, 0x241c0c0c, 0x241c0c1c, 0x241c1404, 0x241c1414, 0x241c1c0c, 0x241c1c2c, - 0x24240404, 0x24240414, 0x24241424, 0x24241c3e, 0x24242404, 0x24243e0c, 0x242c042c, 0x242c043e, - 0x242c140c, 0x242c3414, 0x24340c1c, 0x24341c24, 0x24343404, 0x243e0c04, 0x243e0c2c, 0x243e1c04, - 0x243e241c, 0x243e2c0c, 0x2c040414, 0x2c040c04, 0x2c040c24, 0x2c041414, 0x2c042404, 0x2c042424, - 0x2c04243e, 0x2c042c14, 0x2c043434, 0x2c043e24, 0x2c0c040c, 0x2c0c041c, 0x2c0c042c, 0x2c0c0c14, - 0x2c0c140c, 0x2c0c1c14, 0x2c0c3e14, 0x2c140404, 0x2c140c0c, 0x2c14141c, 0x2c141c04, 0x2c141c34, - 0x2c142c1c, 0x2c1c0414, 0x2c1c043e, 0x2c1c0c04, 0x2c1c143e, 0x2c1c2424, 0x2c1c2c0c, 0x2c1c342c, - 0x2c1c3e1c, 0x2c24040c, 0x2c240424, 0x2c241404, 0x2c241c14, 0x2c242434, 0x2c2c0c14, 0x2c2c1434, - 0x2c2c2c0c, 0x2c2c2c1c, 0x2c342414, 0x2c3e0414, 0x2c3e0424, 0x2c3e1414, 0x34040c0c, 0x34040c1c, - 0x34040c2c, 0x34041c0c, 0x34041c1c, 0x34043404, 0x340c0404, 0x340c1404, 0x340c143e, 0x340c3424, - 0x34140c14, 0x34141c24, 0x34142414, 0x34142c2c, 0x34143414, 0x34143e04, 0x341c0404, 0x341c0c24, - 0x341c140c, 0x341c2404, 0x3424142c, 0x3424241c, 0x34243414, 0x342c0404, 0x342c041c, 0x342c1c24, - 0x342c3404, 0x3434042c, 0x34342404, 0x343e0c0c, 0x343e0c1c, 0x3e040404, 0x3e040424, 0x3e04043e, - 0x3e041404, 0x3e041414, 0x3e041c34, 0x3e042404, 0x3e042c24, 0x3e043414, 0x3e0c0414, 0x3e0c0c0c, - 0x3e0c1424, 0x3e0c241c, 0x3e0c242c, 0x3e14040c, 0x3e140424, 0x3e140c04, 0x3e140c34, 0x3e14140c, - 0x3e141c04, 0x3e142c0c, 0x3e1c0414, 0x3e1c1c14, 0x3e1c1c2c, 0x3e1c2c1c, 0x3e24040c, 0x3e24042c, - 0x3e240c1c, 0x3e241404, 0x3e242c04, 0x3e2c1414, 0x3e2c2414, 0x3e340414, 0x3e341c0c, 0x3e3e0404, +constexpr constant static uint32_t iq3s_grid[512] = { + 0x01010101, 0x01010103, 0x01010105, 0x0101010b, 0x0101010f, 0x01010301, 0x01010303, 0x01010305, + 0x01010309, 0x0101030d, 0x01010501, 0x01010503, 0x0101050b, 0x01010707, 0x01010901, 0x01010905, + 0x0101090b, 0x0101090f, 0x01010b03, 0x01010b07, 0x01010d01, 0x01010d05, 0x01010f03, 0x01010f09, + 0x01010f0f, 0x01030101, 0x01030103, 0x01030105, 0x01030109, 0x01030301, 0x01030303, 0x0103030b, + 0x01030501, 0x01030507, 0x0103050f, 0x01030703, 0x0103070b, 0x01030909, 0x01030d03, 0x01030d0b, + 0x01030f05, 0x01050101, 0x01050103, 0x0105010b, 0x0105010f, 0x01050301, 0x01050307, 0x0105030d, + 0x01050503, 0x0105050b, 0x01050701, 0x01050709, 0x01050905, 0x0105090b, 0x0105090f, 0x01050b03, + 0x01050b07, 0x01050f01, 0x01050f07, 0x01070107, 0x01070303, 0x0107030b, 0x01070501, 0x01070505, + 0x01070703, 0x01070707, 0x0107070d, 0x01070909, 0x01070b01, 0x01070b05, 0x01070d0f, 0x01070f03, + 0x01070f0b, 0x01090101, 0x01090307, 0x0109030f, 0x01090503, 0x01090509, 0x01090705, 0x01090901, + 0x01090907, 0x01090b03, 0x01090f01, 0x010b0105, 0x010b0109, 0x010b0501, 0x010b0505, 0x010b050d, + 0x010b0707, 0x010b0903, 0x010b090b, 0x010b090f, 0x010b0d0d, 0x010b0f07, 0x010d010d, 0x010d0303, + 0x010d0307, 0x010d0703, 0x010d0b05, 0x010d0f03, 0x010f0101, 0x010f0105, 0x010f0109, 0x010f0501, + 0x010f0505, 0x010f050d, 0x010f0707, 0x010f0b01, 0x010f0b09, 0x03010101, 0x03010103, 0x03010105, + 0x03010109, 0x03010301, 0x03010303, 0x03010307, 0x0301030b, 0x0301030f, 0x03010501, 0x03010505, + 0x03010703, 0x03010709, 0x0301070d, 0x03010b09, 0x03010b0d, 0x03010d03, 0x03010f05, 0x03030101, + 0x03030103, 0x03030107, 0x0303010d, 0x03030301, 0x03030309, 0x03030503, 0x03030701, 0x03030707, + 0x03030903, 0x03030b01, 0x03030b05, 0x03030f01, 0x03030f0d, 0x03050101, 0x03050305, 0x0305030b, + 0x0305030f, 0x03050501, 0x03050509, 0x03050705, 0x03050901, 0x03050907, 0x03050b0b, 0x03050d01, + 0x03050f05, 0x03070103, 0x03070109, 0x0307010f, 0x03070301, 0x03070307, 0x03070503, 0x0307050f, + 0x03070701, 0x03070709, 0x03070903, 0x03070d05, 0x03070f01, 0x03090107, 0x0309010b, 0x03090305, + 0x03090309, 0x03090703, 0x03090707, 0x03090905, 0x0309090d, 0x03090b01, 0x03090b09, 0x030b0103, + 0x030b0301, 0x030b0307, 0x030b0503, 0x030b0701, 0x030b0705, 0x030b0b03, 0x030d0501, 0x030d0509, + 0x030d050f, 0x030d0909, 0x030d090d, 0x030f0103, 0x030f0107, 0x030f0301, 0x030f0305, 0x030f0503, + 0x030f070b, 0x030f0903, 0x030f0d05, 0x030f0f01, 0x05010101, 0x05010103, 0x05010107, 0x0501010b, + 0x0501010f, 0x05010301, 0x05010305, 0x05010309, 0x0501030d, 0x05010503, 0x05010507, 0x0501050f, + 0x05010701, 0x05010705, 0x05010903, 0x05010907, 0x0501090b, 0x05010b01, 0x05010b05, 0x05010d0f, + 0x05010f01, 0x05010f07, 0x05010f0b, 0x05030101, 0x05030105, 0x05030301, 0x05030307, 0x0503030f, + 0x05030505, 0x0503050b, 0x05030703, 0x05030709, 0x05030905, 0x05030b03, 0x05050103, 0x05050109, + 0x0505010f, 0x05050503, 0x05050507, 0x05050701, 0x0505070f, 0x05050903, 0x05050b07, 0x05050b0f, + 0x05050f03, 0x05050f09, 0x05070101, 0x05070105, 0x0507010b, 0x05070303, 0x05070505, 0x05070509, + 0x05070703, 0x05070707, 0x05070905, 0x05070b01, 0x05070d0d, 0x05090103, 0x0509010f, 0x05090501, + 0x05090507, 0x05090705, 0x0509070b, 0x05090903, 0x05090f05, 0x05090f0b, 0x050b0109, 0x050b0303, + 0x050b0505, 0x050b070f, 0x050b0901, 0x050b0b07, 0x050b0f01, 0x050d0101, 0x050d0105, 0x050d010f, + 0x050d0503, 0x050d0b0b, 0x050d0d03, 0x050f010b, 0x050f0303, 0x050f050d, 0x050f0701, 0x050f0907, + 0x050f0b01, 0x07010105, 0x07010303, 0x07010307, 0x0701030b, 0x0701030f, 0x07010505, 0x07010703, + 0x07010707, 0x0701070b, 0x07010905, 0x07010909, 0x0701090f, 0x07010b03, 0x07010d07, 0x07010f03, + 0x07030103, 0x07030107, 0x0703010b, 0x07030309, 0x07030503, 0x07030507, 0x07030901, 0x07030d01, + 0x07030f05, 0x07030f0d, 0x07050101, 0x07050305, 0x07050501, 0x07050705, 0x07050709, 0x07050b01, + 0x07070103, 0x07070301, 0x07070309, 0x07070503, 0x07070507, 0x0707050f, 0x07070701, 0x07070903, + 0x07070907, 0x0707090f, 0x07070b0b, 0x07070f07, 0x07090107, 0x07090303, 0x0709030d, 0x07090505, + 0x07090703, 0x07090b05, 0x07090d01, 0x07090d09, 0x070b0103, 0x070b0301, 0x070b0305, 0x070b050b, + 0x070b0705, 0x070b0909, 0x070b0b0d, 0x070b0f07, 0x070d030d, 0x070d0903, 0x070f0103, 0x070f0107, + 0x070f0501, 0x070f0505, 0x070f070b, 0x09010101, 0x09010109, 0x09010305, 0x09010501, 0x09010509, + 0x0901050f, 0x09010705, 0x09010903, 0x09010b01, 0x09010f01, 0x09030105, 0x0903010f, 0x09030303, + 0x09030307, 0x09030505, 0x09030701, 0x0903070b, 0x09030907, 0x09030b03, 0x09030b0b, 0x09050103, + 0x09050107, 0x09050301, 0x0905030b, 0x09050503, 0x09050707, 0x09050901, 0x09050b0f, 0x09050d05, + 0x09050f01, 0x09070109, 0x09070303, 0x09070307, 0x09070501, 0x09070505, 0x09070703, 0x0907070b, + 0x09090101, 0x09090105, 0x09090509, 0x0909070f, 0x09090901, 0x09090f03, 0x090b010b, 0x090b010f, + 0x090b0503, 0x090b0d05, 0x090d0307, 0x090d0709, 0x090d0d01, 0x090f0301, 0x090f030b, 0x090f0701, + 0x090f0907, 0x090f0b03, 0x0b010105, 0x0b010301, 0x0b010309, 0x0b010505, 0x0b010901, 0x0b010909, + 0x0b01090f, 0x0b010b05, 0x0b010d0d, 0x0b010f09, 0x0b030103, 0x0b030107, 0x0b03010b, 0x0b030305, + 0x0b030503, 0x0b030705, 0x0b030f05, 0x0b050101, 0x0b050303, 0x0b050507, 0x0b050701, 0x0b05070d, + 0x0b050b07, 0x0b070105, 0x0b07010f, 0x0b070301, 0x0b07050f, 0x0b070909, 0x0b070b03, 0x0b070d0b, + 0x0b070f07, 0x0b090103, 0x0b090109, 0x0b090501, 0x0b090705, 0x0b09090d, 0x0b0b0305, 0x0b0b050d, + 0x0b0b0b03, 0x0b0b0b07, 0x0b0d0905, 0x0b0f0105, 0x0b0f0109, 0x0b0f0505, 0x0d010303, 0x0d010307, + 0x0d01030b, 0x0d010703, 0x0d010707, 0x0d010d01, 0x0d030101, 0x0d030501, 0x0d03050f, 0x0d030d09, + 0x0d050305, 0x0d050709, 0x0d050905, 0x0d050b0b, 0x0d050d05, 0x0d050f01, 0x0d070101, 0x0d070309, + 0x0d070503, 0x0d070901, 0x0d09050b, 0x0d090907, 0x0d090d05, 0x0d0b0101, 0x0d0b0107, 0x0d0b0709, + 0x0d0b0d01, 0x0d0d010b, 0x0d0d0901, 0x0d0f0303, 0x0d0f0307, 0x0f010101, 0x0f010109, 0x0f01010f, + 0x0f010501, 0x0f010505, 0x0f01070d, 0x0f010901, 0x0f010b09, 0x0f010d05, 0x0f030105, 0x0f030303, + 0x0f030509, 0x0f030907, 0x0f03090b, 0x0f050103, 0x0f050109, 0x0f050301, 0x0f05030d, 0x0f050503, + 0x0f050701, 0x0f050b03, 0x0f070105, 0x0f070705, 0x0f07070b, 0x0f070b07, 0x0f090103, 0x0f09010b, + 0x0f090307, 0x0f090501, 0x0f090b01, 0x0f0b0505, 0x0f0b0905, 0x0f0d0105, 0x0f0d0703, 0x0f0f0101, }; #define NGRID_IQ1S 512 @@ -4742,7 +4742,7 @@ void kernel_mul_mv_iq3_s_f32_impl( { int nval = 8; int pos = (32*sgitg + tiisg)*nval; - for (int i = 0; i < nval; ++i) values[pos + i] = iq3xs_grid[pos + i]; + for (int i = 0; i < nval; ++i) values[pos + i] = iq3s_grid[pos + i]; threadgroup_barrier(mem_flags::mem_threadgroup); } @@ -4769,12 +4769,14 @@ void kernel_mul_mv_iq3_s_f32_impl( for (int row = 0; row < N_DST; row++) { const float db = dh[0]; - const float d = db * (0.5f + ((sc[0] >> 4*(ib%2)) & 0xf)); + const float d = db * (1 + 2*((sc[0] >> 4*(ib%2)) & 0xf)); float2 sum = {0}; for (int l = 0; l < 4; ++l) { - const threadgroup uint8_t * grid1 = (const threadgroup uint8_t *)(values + (qs[2*l+0] | ((qh[0] << (8-2*l)) & 256))); - const threadgroup uint8_t * grid2 = (const threadgroup uint8_t *)(values + (qs[2*l+1] | ((qh[0] << (7-2*l)) & 256))); + const threadgroup uint32_t * table1 = qh[0] & kmask_iq2xs[2*l+0] ? values + 256 : values; + const threadgroup uint32_t * table2 = qh[0] & kmask_iq2xs[2*l+1] ? values + 256 : values; + const threadgroup uint8_t * grid1 = (const threadgroup uint8_t *)(table1 + qs[2*l+0]); + const threadgroup uint8_t * grid2 = (const threadgroup uint8_t *)(table2 + qs[2*l+1]); for (int j = 0; j < 4; ++j) { sum[0] += yl[8*l + j + 0] * grid1[j] * select(1, -1, signs[l] & kmask_iq2xs[j+0]); sum[1] += yl[8*l + j + 4] * grid2[j] * select(1, -1, signs[l] & kmask_iq2xs[j+4]); @@ -4795,7 +4797,7 @@ void kernel_mul_mv_iq3_s_f32_impl( for (int row = 0; row < N_DST; ++row) { all_sum = simd_sum(sumf[row]); if (tiisg == 0) { - dst[r1*ne0 + im*ne0*ne1 + first_row + row] = all_sum * 0.5f; + dst[r1*ne0 + im*ne0*ne1 + first_row + row] = all_sum; } } } @@ -5685,15 +5687,15 @@ void dequantize_iq3_s(device const block_iq3_s * xb, short il, thread type4x4 & device const uint8_t * qs = xb->qs + 8*ib32; device const uint8_t * signs = xb->signs + 4*ib32 + 2*il; const uint8_t qh = xb->qh[ib32] >> 4*il; - const float dl = d * (0.5f + ((xb->scales[ib32/2] >> 4*(ib32%2)) & 0xf)) * 0.5f; - constant uint8_t * grid1 = (constant uint8_t *)(iq3xs_grid + (qs[4*il+0] | ((qh << 8) & 256))); - constant uint8_t * grid2 = (constant uint8_t *)(iq3xs_grid + (qs[4*il+1] | ((qh << 7) & 256))); + const float dl = d * (1 + 2*((xb->scales[ib32/2] >> 4*(ib32%2)) & 0xf)); + constant uint8_t * grid1 = (constant uint8_t *)(iq3s_grid + (qs[4*il+0] | ((qh << 8) & 256))); + constant uint8_t * grid2 = (constant uint8_t *)(iq3s_grid + (qs[4*il+1] | ((qh << 7) & 256))); for (int i = 0; i < 4; ++i) { reg[0][i] = dl * grid1[i] * select(1, -1, signs[0] & kmask_iq2xs[i+0]); reg[1][i] = dl * grid2[i] * select(1, -1, signs[0] & kmask_iq2xs[i+4]); } - grid1 = (constant uint8_t *)(iq3xs_grid + (qs[4*il+2] | ((qh << 6) & 256))); - grid2 = (constant uint8_t *)(iq3xs_grid + (qs[4*il+3] | ((qh << 5) & 256))); + grid1 = (constant uint8_t *)(iq3s_grid + (qs[4*il+2] | ((qh << 6) & 256))); + grid2 = (constant uint8_t *)(iq3s_grid + (qs[4*il+3] | ((qh << 5) & 256))); for (int i = 0; i < 4; ++i) { reg[2][i] = dl * grid1[i] * select(1, -1, signs[1] & kmask_iq2xs[i+0]); reg[3][i] = dl * grid2[i] * select(1, -1, signs[1] & kmask_iq2xs[i+4]); diff --git a/ggml-quants.c b/ggml-quants.c index 371826f14..492a1b9a6 100644 --- a/ggml-quants.c +++ b/ggml-quants.c @@ -3818,71 +3818,71 @@ static const uint32_t iq3xxs_grid[256] = { 0x3e1c1c1c, 0x3e1c3404, 0x3e24140c, 0x3e24240c, 0x3e2c0404, 0x3e2c0414, 0x3e2c1424, 0x3e341c04, }; -static const uint32_t iq3xs_grid[512] = { - 0x04040404, 0x0404040c, 0x04040414, 0x0404042c, 0x0404043e, 0x04040c04, 0x04040c0c, 0x04040c14, - 0x04040c24, 0x04040c34, 0x04041404, 0x0404140c, 0x0404142c, 0x04041c1c, 0x04042404, 0x04042414, - 0x0404242c, 0x0404243e, 0x04042c0c, 0x04042c1c, 0x04043404, 0x04043414, 0x04043e0c, 0x04043e24, - 0x04043e3e, 0x040c0404, 0x040c040c, 0x040c0414, 0x040c0424, 0x040c0c04, 0x040c0c0c, 0x040c0c2c, - 0x040c1404, 0x040c141c, 0x040c143e, 0x040c1c0c, 0x040c1c2c, 0x040c2424, 0x040c340c, 0x040c342c, - 0x040c3e14, 0x04140404, 0x0414040c, 0x0414042c, 0x0414043e, 0x04140c04, 0x04140c1c, 0x04140c34, - 0x0414140c, 0x0414142c, 0x04141c04, 0x04141c24, 0x04142414, 0x0414242c, 0x0414243e, 0x04142c0c, - 0x04142c1c, 0x04143e04, 0x04143e1c, 0x041c041c, 0x041c0c0c, 0x041c0c2c, 0x041c1404, 0x041c1414, - 0x041c1c0c, 0x041c1c1c, 0x041c1c34, 0x041c2424, 0x041c2c04, 0x041c2c14, 0x041c343e, 0x041c3e0c, - 0x041c3e2c, 0x04240404, 0x04240c1c, 0x04240c3e, 0x0424140c, 0x04241424, 0x04241c14, 0x04242404, - 0x0424241c, 0x04242c0c, 0x04243e04, 0x042c0414, 0x042c0424, 0x042c1404, 0x042c1414, 0x042c1434, - 0x042c1c1c, 0x042c240c, 0x042c242c, 0x042c243e, 0x042c3434, 0x042c3e1c, 0x04340434, 0x04340c0c, - 0x04340c1c, 0x04341c0c, 0x04342c14, 0x04343e0c, 0x043e0404, 0x043e0414, 0x043e0424, 0x043e1404, - 0x043e1414, 0x043e1434, 0x043e1c1c, 0x043e2c04, 0x043e2c24, 0x0c040404, 0x0c04040c, 0x0c040414, - 0x0c040424, 0x0c040c04, 0x0c040c0c, 0x0c040c1c, 0x0c040c2c, 0x0c040c3e, 0x0c041404, 0x0c041414, - 0x0c041c0c, 0x0c041c24, 0x0c041c34, 0x0c042c24, 0x0c042c34, 0x0c04340c, 0x0c043e14, 0x0c0c0404, - 0x0c0c040c, 0x0c0c041c, 0x0c0c0434, 0x0c0c0c04, 0x0c0c0c24, 0x0c0c140c, 0x0c0c1c04, 0x0c0c1c1c, - 0x0c0c240c, 0x0c0c2c04, 0x0c0c2c14, 0x0c0c3e04, 0x0c0c3e34, 0x0c140404, 0x0c140c14, 0x0c140c2c, - 0x0c140c3e, 0x0c141404, 0x0c141424, 0x0c141c14, 0x0c142404, 0x0c14241c, 0x0c142c2c, 0x0c143404, - 0x0c143e14, 0x0c1c040c, 0x0c1c0424, 0x0c1c043e, 0x0c1c0c04, 0x0c1c0c1c, 0x0c1c140c, 0x0c1c143e, - 0x0c1c1c04, 0x0c1c1c24, 0x0c1c240c, 0x0c1c3414, 0x0c1c3e04, 0x0c24041c, 0x0c24042c, 0x0c240c14, - 0x0c240c24, 0x0c241c0c, 0x0c241c1c, 0x0c242414, 0x0c242434, 0x0c242c04, 0x0c242c24, 0x0c2c040c, - 0x0c2c0c04, 0x0c2c0c1c, 0x0c2c140c, 0x0c2c1c04, 0x0c2c1c14, 0x0c2c2c0c, 0x0c341404, 0x0c341424, - 0x0c34143e, 0x0c342424, 0x0c342434, 0x0c3e040c, 0x0c3e041c, 0x0c3e0c04, 0x0c3e0c14, 0x0c3e140c, - 0x0c3e1c2c, 0x0c3e240c, 0x0c3e3414, 0x0c3e3e04, 0x14040404, 0x1404040c, 0x1404041c, 0x1404042c, - 0x1404043e, 0x14040c04, 0x14040c14, 0x14040c24, 0x14040c34, 0x1404140c, 0x1404141c, 0x1404143e, - 0x14041c04, 0x14041c14, 0x1404240c, 0x1404241c, 0x1404242c, 0x14042c04, 0x14042c14, 0x1404343e, - 0x14043e04, 0x14043e1c, 0x14043e2c, 0x140c0404, 0x140c0414, 0x140c0c04, 0x140c0c1c, 0x140c0c3e, - 0x140c1414, 0x140c142c, 0x140c1c0c, 0x140c1c24, 0x140c2414, 0x140c2c0c, 0x1414040c, 0x14140424, - 0x1414043e, 0x1414140c, 0x1414141c, 0x14141c04, 0x14141c3e, 0x1414240c, 0x14142c1c, 0x14142c3e, - 0x14143e0c, 0x14143e24, 0x141c0404, 0x141c0414, 0x141c042c, 0x141c0c0c, 0x141c1414, 0x141c1424, - 0x141c1c0c, 0x141c1c1c, 0x141c2414, 0x141c2c04, 0x141c3434, 0x1424040c, 0x1424043e, 0x14241404, - 0x1424141c, 0x14241c14, 0x14241c2c, 0x1424240c, 0x14243e14, 0x14243e2c, 0x142c0424, 0x142c0c0c, - 0x142c1414, 0x142c1c3e, 0x142c2404, 0x142c2c1c, 0x142c3e04, 0x14340404, 0x14340414, 0x1434043e, - 0x1434140c, 0x14342c2c, 0x1434340c, 0x143e042c, 0x143e0c0c, 0x143e1434, 0x143e1c04, 0x143e241c, - 0x143e2c04, 0x1c040414, 0x1c040c0c, 0x1c040c1c, 0x1c040c2c, 0x1c040c3e, 0x1c041414, 0x1c041c0c, - 0x1c041c1c, 0x1c041c2c, 0x1c042414, 0x1c042424, 0x1c04243e, 0x1c042c0c, 0x1c04341c, 0x1c043e0c, - 0x1c0c040c, 0x1c0c041c, 0x1c0c042c, 0x1c0c0c24, 0x1c0c140c, 0x1c0c141c, 0x1c0c2404, 0x1c0c3404, - 0x1c0c3e14, 0x1c0c3e34, 0x1c140404, 0x1c140c14, 0x1c141404, 0x1c141c14, 0x1c141c24, 0x1c142c04, - 0x1c1c040c, 0x1c1c0c04, 0x1c1c0c24, 0x1c1c140c, 0x1c1c141c, 0x1c1c143e, 0x1c1c1c04, 0x1c1c240c, - 0x1c1c241c, 0x1c1c243e, 0x1c1c2c2c, 0x1c1c3e1c, 0x1c24041c, 0x1c240c0c, 0x1c240c34, 0x1c241414, - 0x1c241c0c, 0x1c242c14, 0x1c243404, 0x1c243424, 0x1c2c040c, 0x1c2c0c04, 0x1c2c0c14, 0x1c2c142c, - 0x1c2c1c14, 0x1c2c2424, 0x1c2c2c34, 0x1c2c3e1c, 0x1c340c34, 0x1c34240c, 0x1c3e040c, 0x1c3e041c, - 0x1c3e1404, 0x1c3e1414, 0x1c3e1c2c, 0x24040404, 0x24040424, 0x24040c14, 0x24041404, 0x24041424, - 0x2404143e, 0x24041c14, 0x2404240c, 0x24042c04, 0x24043e04, 0x240c0414, 0x240c043e, 0x240c0c0c, - 0x240c0c1c, 0x240c1414, 0x240c1c04, 0x240c1c2c, 0x240c241c, 0x240c2c0c, 0x240c2c2c, 0x2414040c, - 0x2414041c, 0x24140c04, 0x24140c2c, 0x2414140c, 0x24141c1c, 0x24142404, 0x24142c3e, 0x24143414, - 0x24143e04, 0x241c0424, 0x241c0c0c, 0x241c0c1c, 0x241c1404, 0x241c1414, 0x241c1c0c, 0x241c1c2c, - 0x24240404, 0x24240414, 0x24241424, 0x24241c3e, 0x24242404, 0x24243e0c, 0x242c042c, 0x242c043e, - 0x242c140c, 0x242c3414, 0x24340c1c, 0x24341c24, 0x24343404, 0x243e0c04, 0x243e0c2c, 0x243e1c04, - 0x243e241c, 0x243e2c0c, 0x2c040414, 0x2c040c04, 0x2c040c24, 0x2c041414, 0x2c042404, 0x2c042424, - 0x2c04243e, 0x2c042c14, 0x2c043434, 0x2c043e24, 0x2c0c040c, 0x2c0c041c, 0x2c0c042c, 0x2c0c0c14, - 0x2c0c140c, 0x2c0c1c14, 0x2c0c3e14, 0x2c140404, 0x2c140c0c, 0x2c14141c, 0x2c141c04, 0x2c141c34, - 0x2c142c1c, 0x2c1c0414, 0x2c1c043e, 0x2c1c0c04, 0x2c1c143e, 0x2c1c2424, 0x2c1c2c0c, 0x2c1c342c, - 0x2c1c3e1c, 0x2c24040c, 0x2c240424, 0x2c241404, 0x2c241c14, 0x2c242434, 0x2c2c0c14, 0x2c2c1434, - 0x2c2c2c0c, 0x2c2c2c1c, 0x2c342414, 0x2c3e0414, 0x2c3e0424, 0x2c3e1414, 0x34040c0c, 0x34040c1c, - 0x34040c2c, 0x34041c0c, 0x34041c1c, 0x34043404, 0x340c0404, 0x340c1404, 0x340c143e, 0x340c3424, - 0x34140c14, 0x34141c24, 0x34142414, 0x34142c2c, 0x34143414, 0x34143e04, 0x341c0404, 0x341c0c24, - 0x341c140c, 0x341c2404, 0x3424142c, 0x3424241c, 0x34243414, 0x342c0404, 0x342c041c, 0x342c1c24, - 0x342c3404, 0x3434042c, 0x34342404, 0x343e0c0c, 0x343e0c1c, 0x3e040404, 0x3e040424, 0x3e04043e, - 0x3e041404, 0x3e041414, 0x3e041c34, 0x3e042404, 0x3e042c24, 0x3e043414, 0x3e0c0414, 0x3e0c0c0c, - 0x3e0c1424, 0x3e0c241c, 0x3e0c242c, 0x3e14040c, 0x3e140424, 0x3e140c04, 0x3e140c34, 0x3e14140c, - 0x3e141c04, 0x3e142c0c, 0x3e1c0414, 0x3e1c1c14, 0x3e1c1c2c, 0x3e1c2c1c, 0x3e24040c, 0x3e24042c, - 0x3e240c1c, 0x3e241404, 0x3e242c04, 0x3e2c1414, 0x3e2c2414, 0x3e340414, 0x3e341c0c, 0x3e3e0404, +static const uint32_t iq3s_grid[512] = { + 0x01010101, 0x01010103, 0x01010105, 0x0101010b, 0x0101010f, 0x01010301, 0x01010303, 0x01010305, + 0x01010309, 0x0101030d, 0x01010501, 0x01010503, 0x0101050b, 0x01010707, 0x01010901, 0x01010905, + 0x0101090b, 0x0101090f, 0x01010b03, 0x01010b07, 0x01010d01, 0x01010d05, 0x01010f03, 0x01010f09, + 0x01010f0f, 0x01030101, 0x01030103, 0x01030105, 0x01030109, 0x01030301, 0x01030303, 0x0103030b, + 0x01030501, 0x01030507, 0x0103050f, 0x01030703, 0x0103070b, 0x01030909, 0x01030d03, 0x01030d0b, + 0x01030f05, 0x01050101, 0x01050103, 0x0105010b, 0x0105010f, 0x01050301, 0x01050307, 0x0105030d, + 0x01050503, 0x0105050b, 0x01050701, 0x01050709, 0x01050905, 0x0105090b, 0x0105090f, 0x01050b03, + 0x01050b07, 0x01050f01, 0x01050f07, 0x01070107, 0x01070303, 0x0107030b, 0x01070501, 0x01070505, + 0x01070703, 0x01070707, 0x0107070d, 0x01070909, 0x01070b01, 0x01070b05, 0x01070d0f, 0x01070f03, + 0x01070f0b, 0x01090101, 0x01090307, 0x0109030f, 0x01090503, 0x01090509, 0x01090705, 0x01090901, + 0x01090907, 0x01090b03, 0x01090f01, 0x010b0105, 0x010b0109, 0x010b0501, 0x010b0505, 0x010b050d, + 0x010b0707, 0x010b0903, 0x010b090b, 0x010b090f, 0x010b0d0d, 0x010b0f07, 0x010d010d, 0x010d0303, + 0x010d0307, 0x010d0703, 0x010d0b05, 0x010d0f03, 0x010f0101, 0x010f0105, 0x010f0109, 0x010f0501, + 0x010f0505, 0x010f050d, 0x010f0707, 0x010f0b01, 0x010f0b09, 0x03010101, 0x03010103, 0x03010105, + 0x03010109, 0x03010301, 0x03010303, 0x03010307, 0x0301030b, 0x0301030f, 0x03010501, 0x03010505, + 0x03010703, 0x03010709, 0x0301070d, 0x03010b09, 0x03010b0d, 0x03010d03, 0x03010f05, 0x03030101, + 0x03030103, 0x03030107, 0x0303010d, 0x03030301, 0x03030309, 0x03030503, 0x03030701, 0x03030707, + 0x03030903, 0x03030b01, 0x03030b05, 0x03030f01, 0x03030f0d, 0x03050101, 0x03050305, 0x0305030b, + 0x0305030f, 0x03050501, 0x03050509, 0x03050705, 0x03050901, 0x03050907, 0x03050b0b, 0x03050d01, + 0x03050f05, 0x03070103, 0x03070109, 0x0307010f, 0x03070301, 0x03070307, 0x03070503, 0x0307050f, + 0x03070701, 0x03070709, 0x03070903, 0x03070d05, 0x03070f01, 0x03090107, 0x0309010b, 0x03090305, + 0x03090309, 0x03090703, 0x03090707, 0x03090905, 0x0309090d, 0x03090b01, 0x03090b09, 0x030b0103, + 0x030b0301, 0x030b0307, 0x030b0503, 0x030b0701, 0x030b0705, 0x030b0b03, 0x030d0501, 0x030d0509, + 0x030d050f, 0x030d0909, 0x030d090d, 0x030f0103, 0x030f0107, 0x030f0301, 0x030f0305, 0x030f0503, + 0x030f070b, 0x030f0903, 0x030f0d05, 0x030f0f01, 0x05010101, 0x05010103, 0x05010107, 0x0501010b, + 0x0501010f, 0x05010301, 0x05010305, 0x05010309, 0x0501030d, 0x05010503, 0x05010507, 0x0501050f, + 0x05010701, 0x05010705, 0x05010903, 0x05010907, 0x0501090b, 0x05010b01, 0x05010b05, 0x05010d0f, + 0x05010f01, 0x05010f07, 0x05010f0b, 0x05030101, 0x05030105, 0x05030301, 0x05030307, 0x0503030f, + 0x05030505, 0x0503050b, 0x05030703, 0x05030709, 0x05030905, 0x05030b03, 0x05050103, 0x05050109, + 0x0505010f, 0x05050503, 0x05050507, 0x05050701, 0x0505070f, 0x05050903, 0x05050b07, 0x05050b0f, + 0x05050f03, 0x05050f09, 0x05070101, 0x05070105, 0x0507010b, 0x05070303, 0x05070505, 0x05070509, + 0x05070703, 0x05070707, 0x05070905, 0x05070b01, 0x05070d0d, 0x05090103, 0x0509010f, 0x05090501, + 0x05090507, 0x05090705, 0x0509070b, 0x05090903, 0x05090f05, 0x05090f0b, 0x050b0109, 0x050b0303, + 0x050b0505, 0x050b070f, 0x050b0901, 0x050b0b07, 0x050b0f01, 0x050d0101, 0x050d0105, 0x050d010f, + 0x050d0503, 0x050d0b0b, 0x050d0d03, 0x050f010b, 0x050f0303, 0x050f050d, 0x050f0701, 0x050f0907, + 0x050f0b01, 0x07010105, 0x07010303, 0x07010307, 0x0701030b, 0x0701030f, 0x07010505, 0x07010703, + 0x07010707, 0x0701070b, 0x07010905, 0x07010909, 0x0701090f, 0x07010b03, 0x07010d07, 0x07010f03, + 0x07030103, 0x07030107, 0x0703010b, 0x07030309, 0x07030503, 0x07030507, 0x07030901, 0x07030d01, + 0x07030f05, 0x07030f0d, 0x07050101, 0x07050305, 0x07050501, 0x07050705, 0x07050709, 0x07050b01, + 0x07070103, 0x07070301, 0x07070309, 0x07070503, 0x07070507, 0x0707050f, 0x07070701, 0x07070903, + 0x07070907, 0x0707090f, 0x07070b0b, 0x07070f07, 0x07090107, 0x07090303, 0x0709030d, 0x07090505, + 0x07090703, 0x07090b05, 0x07090d01, 0x07090d09, 0x070b0103, 0x070b0301, 0x070b0305, 0x070b050b, + 0x070b0705, 0x070b0909, 0x070b0b0d, 0x070b0f07, 0x070d030d, 0x070d0903, 0x070f0103, 0x070f0107, + 0x070f0501, 0x070f0505, 0x070f070b, 0x09010101, 0x09010109, 0x09010305, 0x09010501, 0x09010509, + 0x0901050f, 0x09010705, 0x09010903, 0x09010b01, 0x09010f01, 0x09030105, 0x0903010f, 0x09030303, + 0x09030307, 0x09030505, 0x09030701, 0x0903070b, 0x09030907, 0x09030b03, 0x09030b0b, 0x09050103, + 0x09050107, 0x09050301, 0x0905030b, 0x09050503, 0x09050707, 0x09050901, 0x09050b0f, 0x09050d05, + 0x09050f01, 0x09070109, 0x09070303, 0x09070307, 0x09070501, 0x09070505, 0x09070703, 0x0907070b, + 0x09090101, 0x09090105, 0x09090509, 0x0909070f, 0x09090901, 0x09090f03, 0x090b010b, 0x090b010f, + 0x090b0503, 0x090b0d05, 0x090d0307, 0x090d0709, 0x090d0d01, 0x090f0301, 0x090f030b, 0x090f0701, + 0x090f0907, 0x090f0b03, 0x0b010105, 0x0b010301, 0x0b010309, 0x0b010505, 0x0b010901, 0x0b010909, + 0x0b01090f, 0x0b010b05, 0x0b010d0d, 0x0b010f09, 0x0b030103, 0x0b030107, 0x0b03010b, 0x0b030305, + 0x0b030503, 0x0b030705, 0x0b030f05, 0x0b050101, 0x0b050303, 0x0b050507, 0x0b050701, 0x0b05070d, + 0x0b050b07, 0x0b070105, 0x0b07010f, 0x0b070301, 0x0b07050f, 0x0b070909, 0x0b070b03, 0x0b070d0b, + 0x0b070f07, 0x0b090103, 0x0b090109, 0x0b090501, 0x0b090705, 0x0b09090d, 0x0b0b0305, 0x0b0b050d, + 0x0b0b0b03, 0x0b0b0b07, 0x0b0d0905, 0x0b0f0105, 0x0b0f0109, 0x0b0f0505, 0x0d010303, 0x0d010307, + 0x0d01030b, 0x0d010703, 0x0d010707, 0x0d010d01, 0x0d030101, 0x0d030501, 0x0d03050f, 0x0d030d09, + 0x0d050305, 0x0d050709, 0x0d050905, 0x0d050b0b, 0x0d050d05, 0x0d050f01, 0x0d070101, 0x0d070309, + 0x0d070503, 0x0d070901, 0x0d09050b, 0x0d090907, 0x0d090d05, 0x0d0b0101, 0x0d0b0107, 0x0d0b0709, + 0x0d0b0d01, 0x0d0d010b, 0x0d0d0901, 0x0d0f0303, 0x0d0f0307, 0x0f010101, 0x0f010109, 0x0f01010f, + 0x0f010501, 0x0f010505, 0x0f01070d, 0x0f010901, 0x0f010b09, 0x0f010d05, 0x0f030105, 0x0f030303, + 0x0f030509, 0x0f030907, 0x0f03090b, 0x0f050103, 0x0f050109, 0x0f050301, 0x0f05030d, 0x0f050503, + 0x0f050701, 0x0f050b03, 0x0f070105, 0x0f070705, 0x0f07070b, 0x0f070b07, 0x0f090103, 0x0f09010b, + 0x0f090307, 0x0f090501, 0x0f090b01, 0x0f0b0505, 0x0f0b0905, 0x0f0d0105, 0x0f0d0703, 0x0f0f0101, }; #define NGRID_IQ2XXS 512 @@ -4162,11 +4162,11 @@ void dequantize_row_iq3_s(const block_iq3_s * restrict x, float * restrict y, in const uint8_t * signs = x[i].signs; for (int ib32 = 0; ib32 < QK_K/32; ib32 += 2) { - const float db1 = d * (0.5f + (x[i].scales[ib32/2] & 0xf)) * 0.5f; - const float db2 = d * (0.5f + (x[i].scales[ib32/2] >> 4)) * 0.5f; + const float db1 = d * (1 + 2*(x[i].scales[ib32/2] & 0xf)); + const float db2 = d * (1 + 2*(x[i].scales[ib32/2] >> 4)); for (int l = 0; l < 4; ++l) { - const uint8_t * grid1 = (const uint8_t *)(iq3xs_grid + (qs[2*l+0] | ((qh[0] << (8-2*l)) & 256))); - const uint8_t * grid2 = (const uint8_t *)(iq3xs_grid + (qs[2*l+1] | ((qh[0] << (7-2*l)) & 256))); + const uint8_t * grid1 = (const uint8_t *)(iq3s_grid + (qs[2*l+0] | ((qh[0] << (8-2*l)) & 256))); + const uint8_t * grid2 = (const uint8_t *)(iq3s_grid + (qs[2*l+1] | ((qh[0] << (7-2*l)) & 256))); for (int j = 0; j < 4; ++j) { y[j+0] = db1 * grid1[j] * (signs[l] & kmask_iq2xs[j+0] ? -1.f : 1.f); y[j+4] = db1 * grid2[j] * (signs[l] & kmask_iq2xs[j+4] ? -1.f : 1.f); @@ -4176,8 +4176,8 @@ void dequantize_row_iq3_s(const block_iq3_s * restrict x, float * restrict y, in qs += 8; signs += 4; for (int l = 0; l < 4; ++l) { - const uint8_t * grid1 = (const uint8_t *)(iq3xs_grid + (qs[2*l+0] | ((qh[1] << (8-2*l)) & 256))); - const uint8_t * grid2 = (const uint8_t *)(iq3xs_grid + (qs[2*l+1] | ((qh[1] << (7-2*l)) & 256))); + const uint8_t * grid1 = (const uint8_t *)(iq3s_grid + (qs[2*l+0] | ((qh[1] << (8-2*l)) & 256))); + const uint8_t * grid2 = (const uint8_t *)(iq3s_grid + (qs[2*l+1] | ((qh[1] << (7-2*l)) & 256))); for (int j = 0; j < 4; ++j) { y[j+0] = db2 * grid1[j] * (signs[l] & kmask_iq2xs[j+0] ? -1.f : 1.f); y[j+4] = db2 * grid2[j] * (signs[l] & kmask_iq2xs[j+4] ? -1.f : 1.f); @@ -10089,18 +10089,34 @@ void ggml_vec_dot_iq3_s_q8_K (int n, float * GGML_RESTRICT s, size_t bs, const v #if defined(__ARM_NEON) + typedef union { + uint16x8_t vec_index; + uint16_t index[8]; + } vec_index_t; + static const uint8_t k_mask1[32] = {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03 }; static const uint8_t k_mask2[16] = {0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80,}; - const uint8x16x2_t mask1 = vld1q_u8_x2(k_mask1); - const uint8x16_t mask2 = vld1q_u8(k_mask2); + static const int16_t k_shift[8] = {8, 7, 6, 5, 4, 3, 2, 1}; + + const uint8x16x2_t mask1 = vld1q_u8_x2(k_mask1); + const uint8x16_t mask2 = vld1q_u8(k_mask2); + const int16x8_t hshift = vld1q_s16(k_shift); + const uint16x8_t m256 = vdupq_n_u16(256); + const uint8x16_t m1 = vdupq_n_u8(1); uint8x16x2_t vs; ggml_int8x16x4_t q3s; ggml_int8x16x4_t q8b; + vec_index_t idx; + +#if QK_K == 256 + uint32_t scales32[2]; + const uint8_t * scales8 = (const uint8_t *)scales32; +#endif float sumf = 0; for (int i = 0; i < nb; ++i) { @@ -10109,47 +10125,63 @@ void ggml_vec_dot_iq3_s_q8_K (int n, float * GGML_RESTRICT s, size_t bs, const v const uint8_t * restrict qh = x[i].qh; const uint16_t * restrict signs = (const uint16_t *)x[i].signs; const int8_t * restrict q8 = y[i].qs; + +#if QK_K == 256 + memcpy(scales32, x[i].scales, 4); + scales32[1] = (((scales32[0] >> 4) & 0x0f0f0f0f) << 1) | 0x01010101; + scales32[0] = ((scales32[0] & 0x0f0f0f0f) << 1) | 0x01010101; +#endif + int sumi1 = 0, sumi2 = 0; for (int ib32 = 0; ib32 < QK_K/32; ib32 += 2) { q8b = ggml_vld1q_s8_x4(q8); q8 += 64; - const uint32x4_t aux32x4_0 = {iq3xs_grid[qs[ 0] | ((qh[ib32+0] << 8) & 256)], iq3xs_grid[qs[ 1] | ((qh[ib32+0] << 7) & 256)], - iq3xs_grid[qs[ 2] | ((qh[ib32+0] << 6) & 256)], iq3xs_grid[qs[ 3] | ((qh[ib32+0] << 5) & 256)]}; - const uint32x4_t aux32x4_1 = {iq3xs_grid[qs[ 4] | ((qh[ib32+0] << 4) & 256)], iq3xs_grid[qs[ 5] | ((qh[ib32+0] << 3) & 256)], - iq3xs_grid[qs[ 6] | ((qh[ib32+0] << 2) & 256)], iq3xs_grid[qs[ 7] | ((qh[ib32+0] << 1) & 256)]}; - const uint32x4_t aux32x4_2 = {iq3xs_grid[qs[ 8] | ((qh[ib32+1] << 8) & 256)], iq3xs_grid[qs[ 9] | ((qh[ib32+1] << 7) & 256)], - iq3xs_grid[qs[10] | ((qh[ib32+1] << 6) & 256)], iq3xs_grid[qs[11] | ((qh[ib32+1] << 5) & 256)]}; - const uint32x4_t aux32x4_3 = {iq3xs_grid[qs[12] | ((qh[ib32+1] << 4) & 256)], iq3xs_grid[qs[13] | ((qh[ib32+1] << 3) & 256)], - iq3xs_grid[qs[14] | ((qh[ib32+1] << 2) & 256)], iq3xs_grid[qs[15] | ((qh[ib32+1] << 1) & 256)]}; - qs += 16; + + const uint8x16_t idx_l = vld1q_u8(qs); qs += 16; + idx.vec_index = vorrq_u16(vmovl_u8(vget_low_u8 (idx_l)), vandq_u16(vshlq_u16(vdupq_n_u16(qh[ib32+0]), hshift), m256)); + const uint32x4_t aux32x4_0 = {iq3s_grid[idx.index[0]], iq3s_grid[idx.index[1]], + iq3s_grid[idx.index[2]], iq3s_grid[idx.index[3]]}; + const uint32x4_t aux32x4_1 = {iq3s_grid[idx.index[4]], iq3s_grid[idx.index[5]], + iq3s_grid[idx.index[6]], iq3s_grid[idx.index[7]]}; + idx.vec_index = vorrq_u16(vmovl_u8(vget_high_u8(idx_l)), vandq_u16(vshlq_u16(vdupq_n_u16(qh[ib32+1]), hshift), m256)); + const uint32x4_t aux32x4_2 = {iq3s_grid[idx.index[0]], iq3s_grid[idx.index[1]], + iq3s_grid[idx.index[2]], iq3s_grid[idx.index[3]]}; + const uint32x4_t aux32x4_3 = {iq3s_grid[idx.index[4]], iq3s_grid[idx.index[5]], + iq3s_grid[idx.index[6]], iq3s_grid[idx.index[7]]}; + vs.val[0] = vreinterpretq_u8_u32(vdupq_n_u32(signs[0] | (signs[1] << 16))); vs.val[1] = vandq_u8(ggml_vqtbl1q_u8(vs.val[0], mask1.val[1]), mask2); vs.val[0] = vandq_u8(ggml_vqtbl1q_u8(vs.val[0], mask1.val[0]), mask2); - vs.val[0] = vceqq_u8(vs.val[0], mask2); - vs.val[1] = vceqq_u8(vs.val[1], mask2); + vs.val[0] = vorrq_u8(vceqq_u8(vs.val[0], mask2), m1); + vs.val[1] = vorrq_u8(vceqq_u8(vs.val[1], mask2), m1); - q3s.val[0] = vsubq_s8(vreinterpretq_s8_u8(veorq_u8(vs.val[0], vreinterpretq_u8_u32(aux32x4_0))), vreinterpretq_s8_u8(vs.val[0])); - q3s.val[1] = vsubq_s8(vreinterpretq_s8_u8(veorq_u8(vs.val[1], vreinterpretq_u8_u32(aux32x4_1))), vreinterpretq_s8_u8(vs.val[1])); + q3s.val[0] = vmulq_s8(vreinterpretq_s8_u8(vs.val[0]), vreinterpretq_s8_u32(aux32x4_0)); + q3s.val[1] = vmulq_s8(vreinterpretq_s8_u8(vs.val[1]), vreinterpretq_s8_u32(aux32x4_1)); vs.val[0] = vreinterpretq_u8_u32(vdupq_n_u32(signs[2] | (signs[3] << 16))); vs.val[1] = vandq_u8(ggml_vqtbl1q_u8(vs.val[0], mask1.val[1]), mask2); vs.val[0] = vandq_u8(ggml_vqtbl1q_u8(vs.val[0], mask1.val[0]), mask2); - vs.val[0] = vceqq_u8(vs.val[0], mask2); - vs.val[1] = vceqq_u8(vs.val[1], mask2); + vs.val[0] = vorrq_u8(vceqq_u8(vs.val[0], mask2), m1); + vs.val[1] = vorrq_u8(vceqq_u8(vs.val[1], mask2), m1); signs += 4; - q3s.val[2] = vsubq_s8(vreinterpretq_s8_u8(veorq_u8(vs.val[0], vreinterpretq_u8_u32(aux32x4_2))), vreinterpretq_s8_u8(vs.val[0])); - q3s.val[3] = vsubq_s8(vreinterpretq_s8_u8(veorq_u8(vs.val[1], vreinterpretq_u8_u32(aux32x4_3))), vreinterpretq_s8_u8(vs.val[1])); + q3s.val[2] = vmulq_s8(vreinterpretq_s8_u8(vs.val[0]), vreinterpretq_s8_u32(aux32x4_2)); + q3s.val[3] = vmulq_s8(vreinterpretq_s8_u8(vs.val[1]), vreinterpretq_s8_u32(aux32x4_3)); const int32x4_t p1 = ggml_vdotq_s32(ggml_vdotq_s32(vdupq_n_s32(0), q3s.val[0], q8b.val[0]), q3s.val[1], q8b.val[1]); const int32x4_t p2 = ggml_vdotq_s32(ggml_vdotq_s32(vdupq_n_s32(0), q3s.val[2], q8b.val[2]), q3s.val[3], q8b.val[3]); +#if QK_K == 256 + sumi1 += vaddvq_s32(p1) * scales8[ib32/2+0]; + sumi2 += vaddvq_s32(p2) * scales8[ib32/2+4]; +#else sumi1 += vaddvq_s32(p1) * (1 + 2*(x[i].scales[ib32/2] & 0xf)); sumi2 += vaddvq_s32(p2) * (1 + 2*(x[i].scales[ib32/2] >> 4)); +#endif } sumf += d*(sumi1 + sumi2); } - *s = 0.25f * sumf; + *s = sumf; #elif defined(__AVX2__) @@ -10164,6 +10196,16 @@ void ggml_vec_dot_iq3_s_q8_K (int n, float * GGML_RESTRICT s, size_t bs, const v const __m256i mask1 = _mm256_loadu_si256((const __m256i*)k_mask1); const __m256i mask2 = _mm256_loadu_si256((const __m256i*)k_mask2); + const __m256i idx_shift = _mm256_set_epi32(1, 2, 3, 4, 5, 6, 7, 8); + const __m256i idx_mask = _mm256_set1_epi32(256); + + typedef union { + __m256i vec[2]; + uint32_t index[16]; + } index_t; + + index_t idx; + __m256 accumf = _mm256_setzero_ps(); for (int i = 0; i < nb; ++i) { const float d = GGML_FP16_TO_FP32(x[i].d) * y[i].d; @@ -10176,24 +10218,25 @@ void ggml_vec_dot_iq3_s_q8_K (int n, float * GGML_RESTRICT s, size_t bs, const v for (int ib32 = 0; ib32 < QK_K/32; ib32 += 2) { const __m256i q8_1 = _mm256_loadu_si256((const __m256i *)q8); q8 += 32; const __m256i q8_2 = _mm256_loadu_si256((const __m256i *)q8); q8 += 32; - const __m256i q2_1 = _mm256_set_epi32(iq3xs_grid[qs[7] | ((qh[ib32+0] << 1) & 256)], - iq3xs_grid[qs[6] | ((qh[ib32+0] << 2) & 256)], - iq3xs_grid[qs[5] | ((qh[ib32+0] << 3) & 256)], - iq3xs_grid[qs[4] | ((qh[ib32+0] << 4) & 256)], - iq3xs_grid[qs[3] | ((qh[ib32+0] << 5) & 256)], - iq3xs_grid[qs[2] | ((qh[ib32+0] << 6) & 256)], - iq3xs_grid[qs[1] | ((qh[ib32+0] << 7) & 256)], - iq3xs_grid[qs[0] | ((qh[ib32+0] << 8) & 256)]); - qs += 8; - const __m256i q2_2 = _mm256_set_epi32(iq3xs_grid[qs[7] | ((qh[ib32+1] << 1) & 256)], - iq3xs_grid[qs[6] | ((qh[ib32+1] << 2) & 256)], - iq3xs_grid[qs[5] | ((qh[ib32+1] << 3) & 256)], - iq3xs_grid[qs[4] | ((qh[ib32+1] << 4) & 256)], - iq3xs_grid[qs[3] | ((qh[ib32+1] << 5) & 256)], - iq3xs_grid[qs[2] | ((qh[ib32+1] << 6) & 256)], - iq3xs_grid[qs[1] | ((qh[ib32+1] << 7) & 256)], - iq3xs_grid[qs[0] | ((qh[ib32+1] << 8) & 256)]); - qs += 8; + const __m256i idx_l = _mm256_cvtepu8_epi16(_mm_loadu_si128((const __m128i *)qs)); qs += 16; + idx.vec[0] = _mm256_set1_epi32(qh[ib32+0]); + idx.vec[1] = _mm256_set1_epi32(qh[ib32+1]); + idx.vec[0] = _mm256_and_si256(_mm256_sllv_epi32(idx.vec[0], idx_shift), idx_mask); + idx.vec[1] = _mm256_and_si256(_mm256_sllv_epi32(idx.vec[1], idx_shift), idx_mask); + idx.vec[0] = _mm256_or_si256(idx.vec[0], _mm256_cvtepi16_epi32(_mm256_castsi256_si128(idx_l))); + idx.vec[1] = _mm256_or_si256(idx.vec[1], _mm256_cvtepi16_epi32(_mm256_extractf128_si256(idx_l, 1))); + + // At leat on my CPU (Ryzen 7950X), using _mm256_i32gather_epi32 is slower than _mm256_set_epi32. Strange. + //const __m256i q2_1 = _mm256_i32gather_epi32((const int *)iq3s_grid, idx.vec[0], 4); + //const __m256i q2_2 = _mm256_i32gather_epi32((const int *)iq3s_grid, idx.vec[1], 4); + const __m256i q2_1 = _mm256_set_epi32( + iq3s_grid[idx.index[7]], iq3s_grid[idx.index[6]], iq3s_grid[idx.index[5]], iq3s_grid[idx.index[4]], + iq3s_grid[idx.index[3]], iq3s_grid[idx.index[1]], iq3s_grid[idx.index[1]], iq3s_grid[idx.index[0]] + ); + const __m256i q2_2 = _mm256_set_epi32( + iq3s_grid[idx.index[15]], iq3s_grid[idx.index[14]], iq3s_grid[idx.index[13]], iq3s_grid[idx.index[12]], + iq3s_grid[idx.index[11]], iq3s_grid[idx.index[10]], iq3s_grid[idx.index[ 9]], iq3s_grid[idx.index[ 8]] + ); __m256i aux256 = _mm256_set1_epi32(signs[0] | (signs[1] << 16)); aux256 = _mm256_and_si256(_mm256_shuffle_epi8(aux256,mask1), mask2); @@ -10221,7 +10264,7 @@ void ggml_vec_dot_iq3_s_q8_K (int n, float * GGML_RESTRICT s, size_t bs, const v } - *s = 0.25f * hsum_float_8(accumf); + *s = hsum_float_8(accumf); #else @@ -10238,8 +10281,8 @@ void ggml_vec_dot_iq3_s_q8_K (int n, float * GGML_RESTRICT s, size_t bs, const v const uint32_t ls2 = 2*(x[i].scales[ib32/2] >> 4) + 1; int32_t sumi = 0; for (int l = 0; l < 4; ++l) { - const uint8_t * grid1 = (const uint8_t *)(iq3xs_grid + (qs[2*l+0] | ((qh[ib32+0] << (8-2*l)) & 256))); - const uint8_t * grid2 = (const uint8_t *)(iq3xs_grid + (qs[2*l+1] | ((qh[ib32+0] << (7-2*l)) & 256))); + const uint8_t * grid1 = (const uint8_t *)(iq3s_grid + (qs[2*l+0] | ((qh[ib32+0] << (8-2*l)) & 256))); + const uint8_t * grid2 = (const uint8_t *)(iq3s_grid + (qs[2*l+1] | ((qh[ib32+0] << (7-2*l)) & 256))); for (int j = 0; j < 4; ++j) { sumi += grid1[j] * q8[j+0] * (signs[l] & kmask_iq2xs[j+0] ? -1 : 1); sumi += grid2[j] * q8[j+4] * (signs[l] & kmask_iq2xs[j+4] ? -1 : 1); @@ -10251,8 +10294,8 @@ void ggml_vec_dot_iq3_s_q8_K (int n, float * GGML_RESTRICT s, size_t bs, const v bsum += sumi * ls1; sumi = 0; for (int l = 0; l < 4; ++l) { - const uint8_t * grid1 = (const uint8_t *)(iq3xs_grid + (qs[2*l+0] | ((qh[ib32+1] << (8-2*l)) & 256))); - const uint8_t * grid2 = (const uint8_t *)(iq3xs_grid + (qs[2*l+1] | ((qh[ib32+1] << (7-2*l)) & 256))); + const uint8_t * grid1 = (const uint8_t *)(iq3s_grid + (qs[2*l+0] | ((qh[ib32+1] << (8-2*l)) & 256))); + const uint8_t * grid2 = (const uint8_t *)(iq3s_grid + (qs[2*l+1] | ((qh[ib32+1] << (7-2*l)) & 256))); for (int j = 0; j < 4; ++j) { sumi += grid1[j] * q8[j+0] * (signs[l] & kmask_iq2xs[j+0] ? -1 : 1); sumi += grid2[j] * q8[j+4] * (signs[l] & kmask_iq2xs[j+4] ? -1 : 1); @@ -10265,7 +10308,7 @@ void ggml_vec_dot_iq3_s_q8_K (int n, float * GGML_RESTRICT s, size_t bs, const v } sumf += d * bsum; } - *s = 0.25f * sumf; + *s = sumf; #endif } @@ -11912,7 +11955,8 @@ static void quantize_row_iq3_s_impl(int block_size, const float * restrict x, vo } float best = 0; float scale = max/(2*kMaxQ-1); - for (int is = -15; is <= 15; ++is) { + for (int k = 0; k < bs4; ++k) is_on_grid[k] = false; + for (int is = -9; is <= 9; ++is) { float id = (2*kMaxQ-1+is*0.2f)/max; float this_scale = 1/id; for (int k = 0; k < bs4; ++k) { @@ -11948,7 +11992,7 @@ static void quantize_row_iq3_s_impl(int block_size, const float * restrict x, vo if (n_not_ongrid > 0 && scale > 0) { float id = 1/scale; for (int k = 0; k < bs4; ++k) { - if (is_on_grid[k]) continue; + //if (is_on_grid[k]) continue; uint16_t u = 0; for (int i = 0; i < 4; ++i) { int l = nearest_int(0.5f*(id*xval[4*k+i]-1)); @@ -12004,7 +12048,7 @@ static void quantize_row_iq3_s_impl(int block_size, const float * restrict x, vo } float d = max_scale/31; - y[ibl].d = GGML_FP32_TO_FP16(d); + y[ibl].d = GGML_FP32_TO_FP16(d * 1.033f); float id = 1/d; for (int ib = 0; ib < QK_K/block_size; ib += 2) { int l1 = nearest_int(0.5f*(id*scales[ib+0]-1)); From c7a0ad8ec9ebb5ddb1c1c80c82f2ee041c525d47 Mon Sep 17 00:00:00 2001 From: Jared Van Bortel Date: Sat, 2 Mar 2024 12:21:47 -0500 Subject: [PATCH 513/812] convert-hf : make model class definitions self-contained (#5825) --- convert-hf-to-gguf.py | 204 ++++++++++++++++++------------------ gguf-py/gguf/gguf_writer.py | 2 +- 2 files changed, 101 insertions(+), 105 deletions(-) diff --git a/convert-hf-to-gguf.py b/convert-hf-to-gguf.py index 28b92ac38..fa9d4f22f 100755 --- a/convert-hf-to-gguf.py +++ b/convert-hf-to-gguf.py @@ -8,9 +8,10 @@ import json import os import re import sys +from abc import ABC, abstractmethod from enum import IntEnum from pathlib import Path -from typing import TYPE_CHECKING, Any, ContextManager, Iterator, Sequence, cast +from typing import TYPE_CHECKING, Any, Callable, ContextManager, Iterator, Sequence, TypeVar, cast import numpy as np import torch @@ -35,8 +36,11 @@ class SentencePieceTokenTypes(IntEnum): UNUSED = 5 BYTE = 6 +AnyModel = TypeVar("AnyModel", bound="type[Model]") + +class Model(ABC): + _model_classes: dict[str, type[Model]] = {} -class Model: def __init__(self, dir_model: Path, ftype: int, fname_out: Path, is_big_endian: bool): self.dir_model = dir_model self.ftype = ftype @@ -47,10 +51,14 @@ class Model: self.num_parts = Model.count_model_parts(self.dir_model, ".safetensors" if self.is_safetensors else ".bin") self.part_names = self._get_part_names() self.hparams = Model.load_hparams(self.dir_model) - self.model_arch = self._get_model_architecture() self.gguf_writer = gguf.GGUFWriter(fname_out, gguf.MODEL_ARCH_NAMES[self.model_arch], endianess=self.endianess, use_temp_file=False) self.block_count = self.find_hparam(["n_layers", "num_hidden_layers", "n_layer"]) + @property + @abstractmethod + def model_arch(self) -> gguf.MODEL_ARCH: + pass + def find_hparam(self, keys: Sequence[str], optional: bool = False) -> Any: key = next((k for k in keys if k in self.hparams), None) if key is not None: @@ -176,55 +184,21 @@ class Model: with open(dir_model / "config.json", "r", encoding="utf-8") as f: return json.load(f) - @staticmethod - def from_model_architecture(model_architecture): - if model_architecture == "GPTNeoXForCausalLM": - return GPTNeoXModel - if model_architecture == "BloomForCausalLM": - return BloomModel - if model_architecture == "MPTForCausalLM": - return MPTModel - if model_architecture in ("BaichuanForCausalLM", "BaiChuanForCausalLM"): - return BaichuanModel - if model_architecture in ("FalconForCausalLM", "RWForCausalLM"): - return FalconModel - if model_architecture == "GPTBigCodeForCausalLM": - return StarCoderModel - if model_architecture == "GPTRefactForCausalLM": - return RefactModel - if model_architecture == "PersimmonForCausalLM": - return PersimmonModel - if model_architecture in ("StableLmForCausalLM", "StableLMEpochForCausalLM", "LlavaStableLMEpochForCausalLM"): - return StableLMModel - if model_architecture == "QWenLMHeadModel": - return QwenModel - if model_architecture == "Qwen2ForCausalLM": - return Model - if model_architecture == "MixtralForCausalLM": - return MixtralModel - if model_architecture == "GPT2LMHeadModel": - return GPT2Model - if model_architecture == "PhiForCausalLM": - return Phi2Model - if model_architecture == "PlamoForCausalLM": - return PlamoModel - if model_architecture == "CodeShellForCausalLM": - return CodeShellModel - if model_architecture == "OrionForCausalLM": - return OrionModel - if model_architecture == "InternLM2ForCausalLM": - return InternLM2Model - if model_architecture == "MiniCPMForCausalLM": - return MiniCPMModel - if model_architecture == "BertModel": - return BertModel - if model_architecture == "NomicBertModel": - return NomicBertModel - if model_architecture == "GemmaForCausalLM": - return GemmaModel - if model_architecture == "Starcoder2ForCausalLM": - return Model - return Model + @classmethod + def register(cls, *names: str) -> Callable[[AnyModel], AnyModel]: + assert names + def func(modelcls: type[Model]): + for name in names: + cls._model_classes[name] = modelcls + return modelcls + return func + + @classmethod + def from_model_architecture(cls, arch): + try: + return cls._model_classes[arch] + except KeyError: + raise NotImplementedError(f'Architecture {arch!r} not supported!') from None def _is_model_safetensors(self) -> bool: return Model.count_model_parts(self.dir_model, ".safetensors") > 0 @@ -239,57 +213,6 @@ class Model: return ("pytorch_model.bin",) return (f"pytorch_model-{n:05}-of-{self.num_parts:05}.bin" for n in range(1, self.num_parts + 1)) - def _get_model_architecture(self) -> gguf.MODEL_ARCH: - arch = self.hparams["architectures"][0] - if arch == "GPTNeoXForCausalLM": - return gguf.MODEL_ARCH.GPTNEOX - if arch == "BloomForCausalLM": - return gguf.MODEL_ARCH.BLOOM - if arch == "MPTForCausalLM": - return gguf.MODEL_ARCH.MPT - if arch in ("BaichuanForCausalLM", "BaiChuanForCausalLM"): - return gguf.MODEL_ARCH.BAICHUAN - if arch in ("FalconForCausalLM", "RWForCausalLM"): - return gguf.MODEL_ARCH.FALCON - if arch == "GPTBigCodeForCausalLM": - return gguf.MODEL_ARCH.STARCODER - if arch == "GPTRefactForCausalLM": - return gguf.MODEL_ARCH.REFACT - if arch == "PersimmonForCausalLM": - return gguf.MODEL_ARCH.PERSIMMON - if arch in ("StableLmForCausalLM", "StableLMEpochForCausalLM", "LlavaStableLMEpochForCausalLM"): - return gguf.MODEL_ARCH.STABLELM - if arch == "QWenLMHeadModel": - return gguf.MODEL_ARCH.QWEN - if arch == "Qwen2ForCausalLM": - return gguf.MODEL_ARCH.QWEN2 - if arch == "MixtralForCausalLM": - return gguf.MODEL_ARCH.LLAMA - if arch == "GPT2LMHeadModel": - return gguf.MODEL_ARCH.GPT2 - if arch == "PhiForCausalLM": - return gguf.MODEL_ARCH.PHI2 - if arch == "PlamoForCausalLM": - return gguf.MODEL_ARCH.PLAMO - if arch == "CodeShellForCausalLM": - return gguf.MODEL_ARCH.CODESHELL - if arch == "OrionForCausalLM": - return gguf.MODEL_ARCH.ORION - if arch == "InternLM2ForCausalLM": - return gguf.MODEL_ARCH.INTERNLM2 - if arch == "MiniCPMForCausalLM": - return gguf.MODEL_ARCH.MINICPM - if arch == "BertModel": - return gguf.MODEL_ARCH.BERT - if arch == "NomicBertModel": - return gguf.MODEL_ARCH.NOMIC_BERT - if arch == "GemmaForCausalLM": - return gguf.MODEL_ARCH.GEMMA - if arch == "Starcoder2ForCausalLM": - return gguf.MODEL_ARCH.STARCODER2 - - raise NotImplementedError(f'Architecture "{arch}" not supported!') - def _set_vocab_gpt2(self): dir_model = self.dir_model hparams = self.hparams @@ -457,7 +380,10 @@ class Model: special_vocab.add_to_gguf(self.gguf_writer) +@Model.register("GPTNeoXForCausalLM") class GPTNeoXModel(Model): + model_arch = gguf.MODEL_ARCH.GPTNEOX + def set_gguf_parameters(self): block_count = self.hparams["num_hidden_layers"] @@ -474,7 +400,10 @@ class GPTNeoXModel(Model): self.gguf_writer.add_layer_norm_eps(self.hparams["layer_norm_eps"]) +@Model.register("BloomForCausalLM") class BloomModel(Model): + model_arch = gguf.MODEL_ARCH.BLOOM + def set_gguf_parameters(self): self.gguf_writer.add_name("Bloom") n_embed = self.hparams.get("hidden_size", self.hparams.get("n_embed")) @@ -566,7 +495,10 @@ class BloomModel(Model): print(name, f"=> output.weight, shape = {data.shape}, {old_dtype} --> {data.dtype}") +@Model.register("MPTForCausalLM") class MPTModel(Model): + model_arch = gguf.MODEL_ARCH.MPT + def set_gguf_parameters(self): block_count = self.hparams["n_layers"] self.gguf_writer.add_name(self.dir_model.name) @@ -629,7 +561,10 @@ class MPTModel(Model): self.gguf_writer.add_tensor(new_name, data) +@Model.register("OrionForCausalLM") class OrionModel(Model): + model_arch = gguf.MODEL_ARCH.ORION + def set_vocab(self): self._set_vocab_sentencepiece() @@ -708,7 +643,10 @@ class OrionModel(Model): self.gguf_writer.add_tensor(new_name, data) +@Model.register("BaichuanForCausalLM", "BaiChuanForCausalLM") class BaichuanModel(Model): + model_arch = gguf.MODEL_ARCH.BAICHUAN + def set_vocab(self): self._set_vocab_sentencepiece() @@ -823,7 +761,10 @@ class BaichuanModel(Model): return weights[r * n_part:r * n_part + r, ...] +@Model.register("FalconForCausalLM", "RWForCausalLM") class FalconModel(Model): + model_arch = gguf.MODEL_ARCH.FALCON + def set_gguf_parameters(self): block_count = self.hparams.get("num_hidden_layers") if block_count is None: @@ -916,7 +857,10 @@ class FalconModel(Model): self.gguf_writer.add_tensor(new_name, data) +@Model.register("GPTBigCodeForCausalLM") class StarCoderModel(Model): + model_arch = gguf.MODEL_ARCH.STARCODER + def set_gguf_parameters(self): block_count = self.hparams["n_layer"] @@ -931,7 +875,10 @@ class StarCoderModel(Model): self.gguf_writer.add_file_type(self.ftype) +@Model.register("GPTRefactForCausalLM") class RefactModel(Model): + model_arch = gguf.MODEL_ARCH.REFACT + def set_gguf_parameters(self): hidden_dim = self.hparams["n_embd"] inner_dim = 4 * hidden_dim @@ -1015,7 +962,10 @@ class RefactModel(Model): self.gguf_writer.add_tensor(new_name, data) +@Model.register("PersimmonForCausalLM") class PersimmonModel(Model): + model_arch = gguf.MODEL_ARCH.PERSIMMON + def set_gguf_parameters(self): block_count = self.hparams.get("num_layers", self.hparams.get("num_hidden_layers")) head_count = self.hparams["num_attention_heads"] @@ -1063,7 +1013,10 @@ class PersimmonModel(Model): self.gguf_writer.add_tensor(new_name, data) +@Model.register("StableLmForCausalLM", "StableLMEpochForCausalLM", "LlavaStableLMEpochForCausalLM") class StableLMModel(Model): + model_arch = gguf.MODEL_ARCH.STABLELM + def set_vocab(self): if (self.dir_model / "tokenizer.json").is_file(): self._set_vocab_gpt2() @@ -1087,12 +1040,18 @@ class StableLMModel(Model): self.gguf_writer.add_layer_norm_eps(self.find_hparam(["layer_norm_eps", "norm_eps"])) +@Model.register("MixtralForCausalLM") class MixtralModel(Model): + model_arch = gguf.MODEL_ARCH.LLAMA + def set_vocab(self): self._set_vocab_sentencepiece() +@Model.register("MiniCPMForCausalLM") class MiniCPMModel(Model): + model_arch = gguf.MODEL_ARCH.MINICPM + def set_gguf_parameters(self): block_count = self.hparams["num_hidden_layers"] self.gguf_writer.add_name("MiniCPM") @@ -1169,7 +1128,10 @@ class MiniCPMModel(Model): self.gguf_writer.add_tensor(new_name, data) +@Model.register("QWenLMHeadModel") class QwenModel(Model): + model_arch = gguf.MODEL_ARCH.QWEN + @staticmethod def token_bytes_to_string(b): from transformers.models.gpt2.tokenization_gpt2 import bytes_to_unicode @@ -1249,7 +1211,15 @@ class QwenModel(Model): self.gguf_writer.add_tensor(new_name, data) +@Model.register("Qwen2ForCausalLM") +class Qwen2Model(Model): + model_arch = gguf.MODEL_ARCH.QWEN2 + + +@Model.register("GPT2LMHeadModel") class GPT2Model(Model): + model_arch = gguf.MODEL_ARCH.GPT2 + def set_gguf_parameters(self): self.gguf_writer.add_name(self.dir_model.name) self.gguf_writer.add_block_count(self.hparams["n_layer"]) @@ -1311,7 +1281,10 @@ class GPT2Model(Model): self.gguf_writer.add_tensor("output.weight", data) +@Model.register("PhiForCausalLM") class Phi2Model(Model): + model_arch = gguf.MODEL_ARCH.PHI2 + def set_gguf_parameters(self): block_count = self.find_hparam(["num_hidden_layers", "n_layer"]) @@ -1333,7 +1306,10 @@ class Phi2Model(Model): self.gguf_writer.add_add_bos_token(False) +@Model.register("PlamoForCausalLM") class PlamoModel(Model): + model_arch = gguf.MODEL_ARCH.PLAMO + def set_vocab(self): self._set_vocab_sentencepiece() @@ -1412,7 +1388,10 @@ class PlamoModel(Model): self.gguf_writer.add_tensor(new_name, data) +@Model.register("CodeShellForCausalLM") class CodeShellModel(Model): + model_arch = gguf.MODEL_ARCH.CODESHELL + def set_gguf_parameters(self): block_count = self.hparams["n_layer"] @@ -1477,7 +1456,10 @@ class CodeShellModel(Model): print(name, f"=> output.weight, shape = {data.shape}, {old_dtype} --> {data.dtype}") +@Model.register("InternLM2ForCausalLM") class InternLM2Model(Model): + model_arch = gguf.MODEL_ARCH.INTERNLM2 + def set_vocab(self): # (TODO): Is there a better way? # Copy from _set_vocab_sentencepiece, The only difference is that we will treat the character @@ -1649,7 +1631,10 @@ in chat mode so that the conversation can end normally.") self.post_write_tensors(tensor_map, name, data_torch) +@Model.register("BertModel") class BertModel(Model): + model_arch = gguf.MODEL_ARCH.BERT + def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.vocab_size = None @@ -1679,7 +1664,7 @@ class BertModel(Model): else: raise NotImplementedError("Only MEAN and CLS pooling types supported") - self.gguf_writer.add_pooling_type(pooling_type.value) + self.gguf_writer.add_pooling_type(pooling_type) def set_vocab(self): path = self.dir_model @@ -1755,7 +1740,10 @@ class BertModel(Model): self.gguf_writer.add_tensor(new_name, data) +@Model.register("NomicBertModel") class NomicBertModel(BertModel): + model_arch = gguf.MODEL_ARCH.NOMIC_BERT + def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) @@ -1792,7 +1780,10 @@ class NomicBertModel(BertModel): yield name, data +@Model.register("GemmaForCausalLM") class GemmaModel(Model): + model_arch = gguf.MODEL_ARCH.GEMMA + def set_vocab(self): self._set_vocab_sentencepiece() @@ -1848,6 +1839,11 @@ class GemmaModel(Model): self.gguf_writer.add_tensor(new_name, data) +@Model.register("Starcoder2ForCausalLM") +class StarCoder2Model(Model): + model_arch = gguf.MODEL_ARCH.STARCODER2 + + ###### CONVERSION LOGIC ###### diff --git a/gguf-py/gguf/gguf_writer.py b/gguf-py/gguf/gguf_writer.py index e4681475c..801160832 100644 --- a/gguf-py/gguf/gguf_writer.py +++ b/gguf-py/gguf/gguf_writer.py @@ -362,7 +362,7 @@ class GGUFWriter: self.add_bool(Keys.Attention.CAUSAL.format(arch=self.arch), value) def add_pooling_type(self, value: PoolingType) -> None: - self.add_uint32(Keys.LLM.POOLING_TYPE.format(arch=self.arch), value) + self.add_uint32(Keys.LLM.POOLING_TYPE.format(arch=self.arch), value.value) def add_rope_dimension_count(self, count: int) -> None: self.add_uint32(Keys.Rope.DIMENSION_COUNT.format(arch=self.arch), count) From 4d4d2366fc9c54d4a275065cfe9299c6cf7c5b78 Mon Sep 17 00:00:00 2001 From: Jared Van Bortel Date: Sat, 2 Mar 2024 12:27:26 -0500 Subject: [PATCH 514/812] convert : automatically fall back to HfVocab if tokenizer.model doesn't exist (#5821) --- README.md | 4 +- convert-llama-ggml-to-gguf.py | 6 +-- convert.py | 72 +++++++++++++++++------------------ examples/infill/infill.cpp | 4 +- 4 files changed, 41 insertions(+), 45 deletions(-) diff --git a/README.md b/README.md index 67717c1e3..939646753 100644 --- a/README.md +++ b/README.md @@ -786,7 +786,7 @@ And after 4.45 hours, you will have the final perplexity. ### Interactive mode If you want a more ChatGPT-like experience, you can run in interactive mode by passing `-i` as a parameter. -In this mode, you can always interrupt generation by pressing Ctrl+C and entering one or more lines of text, which will be converted into tokens and appended to the current context. You can also specify a *reverse prompt* with the parameter `-r "reverse prompt string"`. This will result in user input being prompted whenever the exact tokens of the reverse prompt string are encountered in the generation. A typical use is to use a prompt that makes LLaMa emulate a chat between multiple users, say Alice and Bob, and pass `-r "Alice:"`. +In this mode, you can always interrupt generation by pressing Ctrl+C and entering one or more lines of text, which will be converted into tokens and appended to the current context. You can also specify a *reverse prompt* with the parameter `-r "reverse prompt string"`. This will result in user input being prompted whenever the exact tokens of the reverse prompt string are encountered in the generation. A typical use is to use a prompt that makes LLaMA emulate a chat between multiple users, say Alice and Bob, and pass `-r "Alice:"`. Here is an example of a few-shot interaction, invoked with the command @@ -850,7 +850,7 @@ Sample run: ``` == Running in interactive mode. == - Press Ctrl+C to interject at any time. - - Press Return to return control to LLaMa. + - Press Return to return control to LLaMA. - If you want to submit another line, end your input in '\'. Below is an instruction that describes a task. Write a response that appropriately completes the request. diff --git a/convert-llama-ggml-to-gguf.py b/convert-llama-ggml-to-gguf.py index b33108062..cd9644fcb 100755 --- a/convert-llama-ggml-to-gguf.py +++ b/convert-llama-ggml-to-gguf.py @@ -373,7 +373,7 @@ def handle_metadata(cfg, hp): raise ValueError('Unable to load metadata') vocab_path = Path(cfg.vocab_dir if cfg.vocab_dir is not None else cfg.model_metadata_dir) vocab_factory = convert.VocabFactory(vocab_path) - vocab, special_vocab = vocab_factory.load_vocab(cfg.vocabtype, cfg.model_metadata_dir) + vocab, special_vocab = vocab_factory.load_vocab(cfg.vocabtype.split(","), cfg.model_metadata_dir) convert.check_vocab_size(params, vocab) return params, vocab, special_vocab @@ -398,8 +398,8 @@ def handle_args(): help ='Load HuggingFace/.pth vocab and metadata from the specified directory') parser.add_argument("--vocab-dir", type=Path, help="directory containing tokenizer.model, if separate from model file - only meaningful with --model-metadata-dir") - parser.add_argument("--vocabtype", choices=["spm", "bpe"], default="spm", - help="vocab format - only meaningful with --model-metadata-dir and/or --vocab-dir (default: spm)") + parser.add_argument("--vocabtype", default="spm,hfft", + help="vocab format - only meaningful with --model-metadata-dir and/or --vocab-dir (default: spm,hfft)") return parser.parse_args() diff --git a/convert.py b/convert.py index 63a0a5d78..6e3a0319b 100755 --- a/convert.py +++ b/convert.py @@ -1282,35 +1282,32 @@ def load_some_model(path: Path) -> ModelPlus: class VocabFactory: + _FILES = {"spm": "tokenizer.model", "bpe": "vocab.json", "hfft": "tokenizer.json"} + def __init__(self, path: Path): self.path = path - self.files: dict[str, Path | None] = { - "tokenizer.model": None, - "vocab.json": None, - "tokenizer.json": None, - } - self._detect_files() + self.file_paths = self._detect_files() + print(f"Found vocab files: {self.file_paths}") - def _detect_files(self): - for file in self.files.keys(): - file_path = self.path / file - parent_file_path = self.path.parent / file - if file_path.exists(): - self.files[file] = file_path - elif parent_file_path.exists(): - self.files[file] = parent_file_path - print(f"Found vocab files: {self.files}") + def _detect_files(self) -> dict[str, Path | None]: + def locate(file: str) -> Path | None: + if (path := self.path / file).exists(): + return path + if (path := self.path.parent / file).exists(): + return path + return None - def _select_file(self, vocabtype: str | None) -> Path: - if vocabtype in ["spm", "bpe"]: - for file_key in self.files.keys(): - if (file := self.files[file_key]) is not None: - return file - raise FileNotFoundError(f"{vocabtype} vocab not found.") - if vocabtype == "hfft": - # For Hugging Face Fast Tokenizer, return the directory path instead of a specific file - return self.path - raise ValueError(f"Unsupported vocabulary type {vocabtype}") + return {vt: locate(f) for vt, f in self._FILES.items()} + + def _select_file(self, vocab_types: list[str]) -> tuple[str, Path]: + for vtype in vocab_types: + try: + path = self.file_paths[vtype] + except KeyError: + raise ValueError(f"Unsupported vocabulary type {vtype}") from None + if path is not None: + return vtype, path + raise FileNotFoundError(f"Could not find any of {[self._FILES[vt] for vt in vocab_types]}") def _create_special_vocab(self, vocab: Vocab, vocabtype: str, model_parent_path: Path) -> gguf.SpecialVocab: load_merges = vocabtype == "bpe" @@ -1322,30 +1319,30 @@ class VocabFactory: n_vocab=n_vocab, ) - def load_vocab(self, vocabtype: str, model_parent_path: Path) -> tuple[Vocab, gguf.SpecialVocab]: - path = self._select_file(vocabtype) - print(f"Loading vocab file '{path}', type '{vocabtype}'") + def load_vocab(self, vocab_types: list[str], model_parent_path: Path) -> tuple[Vocab, gguf.SpecialVocab]: + vocab_type, path = self._select_file(vocab_types) + print(f"Loading vocab file {path!r}, type {vocab_type!r}") added_tokens_path = path.parent / "added_tokens.json" vocab: Vocab - if vocabtype == "bpe": + if vocab_type == "bpe": vocab = BpeVocab( path, added_tokens_path if added_tokens_path.exists() else None ) - elif vocabtype == "spm": + elif vocab_type == "spm": vocab = SentencePieceVocab( path, added_tokens_path if added_tokens_path.exists() else None ) - elif vocabtype == "hfft": + elif vocab_type == "hfft": vocab = HfVocab( - path, added_tokens_path if added_tokens_path.exists() else None + path.parent, added_tokens_path if added_tokens_path.exists() else None ) else: - raise ValueError(f"Unsupported vocabulary type {vocabtype}") + raise ValueError(vocab_type) # FIXME: Respect --vocab-dir? special_vocab = self._create_special_vocab( vocab, - vocabtype, + vocab_type, model_parent_path, ) return vocab, special_vocab @@ -1379,15 +1376,14 @@ def main(args_in: list[str] | None = None) -> None: if np.uint32(1) == np.uint32(1).newbyteorder("<"): # We currently only support Q8_0 output on little endian systems. output_choices.append("q8_0") - vocab_types = ["spm", "bpe", "hfft"] - parser = argparse.ArgumentParser(description="Convert a LLaMa model to a GGML compatible file") + parser = argparse.ArgumentParser(description="Convert a LLaMA model to a GGML compatible file") parser.add_argument("--awq-path", type=Path, help="Path to scale awq cache file", default=None) parser.add_argument("--dump", action="store_true", help="don't convert, just show what's in the model") parser.add_argument("--dump-single", action="store_true", help="don't convert, just show what's in a single model file") parser.add_argument("--vocab-only", action="store_true", help="extract only the vocab") parser.add_argument("--outtype", choices=output_choices, help="output format - note: q8_0 may be very slow (default: f16 or f32 based on input)") parser.add_argument("--vocab-dir", type=Path, help="directory containing tokenizer.model, if separate from model file") - parser.add_argument("--vocab-type", choices=vocab_types, help="The vocabulary format used to define the tokenizer model (default: spm)", default="spm") + parser.add_argument("--vocab-type", help="vocab types to try in order, choose from 'spm', 'bpe', 'hfft' (default: spm,hfft)", default="spm,hfft") parser.add_argument("--outfile", type=Path, help="path to write to; default: based on input") parser.add_argument("model", type=Path, help="directory containing model file, or model file itself (*.pth, *.pt, *.bin)") parser.add_argument("--ctx", type=int, help="model training context (default: based on input)") @@ -1448,7 +1444,7 @@ def main(args_in: list[str] | None = None) -> None: model_parent_path = model_plus.paths[0].parent vocab_path = Path(args.vocab_dir or args.model or model_parent_path) vocab_factory = VocabFactory(vocab_path) - vocab, special_vocab = vocab_factory.load_vocab(args.vocab_type, model_parent_path) + vocab, special_vocab = vocab_factory.load_vocab(args.vocab_type.split(","), model_parent_path) if args.vocab_only: if not args.outfile: diff --git a/examples/infill/infill.cpp b/examples/infill/infill.cpp index d4b8729dd..91c39c5ae 100644 --- a/examples/infill/infill.cpp +++ b/examples/infill/infill.cpp @@ -378,10 +378,10 @@ int main(int argc, char ** argv) { if (params.interactive) { const char *control_message; if (params.multiline_input) { - control_message = " - To return control to LLaMa, end your input with '\\'.\n" + control_message = " - To return control to LLaMA, end your input with '\\'.\n" " - To return control without starting a new line, end your input with '/'.\n"; } else { - control_message = " - Press Return to return control to LLaMa.\n" + control_message = " - Press Return to return control to LLaMA.\n" " - To return control without starting a new line, end your input with '/'.\n" " - If you want to submit another line, end your input with '\\'.\n"; } From 494c87032613e31c0be99b2735e732871f2c4e4d Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Sat, 2 Mar 2024 20:00:49 +0200 Subject: [PATCH 515/812] ggml : fix IQ3_S AVX implementation (#5834) ggml-ci --- ggml-quants.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ggml-quants.c b/ggml-quants.c index 492a1b9a6..2a8881d73 100644 --- a/ggml-quants.c +++ b/ggml-quants.c @@ -10231,7 +10231,7 @@ void ggml_vec_dot_iq3_s_q8_K (int n, float * GGML_RESTRICT s, size_t bs, const v //const __m256i q2_2 = _mm256_i32gather_epi32((const int *)iq3s_grid, idx.vec[1], 4); const __m256i q2_1 = _mm256_set_epi32( iq3s_grid[idx.index[7]], iq3s_grid[idx.index[6]], iq3s_grid[idx.index[5]], iq3s_grid[idx.index[4]], - iq3s_grid[idx.index[3]], iq3s_grid[idx.index[1]], iq3s_grid[idx.index[1]], iq3s_grid[idx.index[0]] + iq3s_grid[idx.index[3]], iq3s_grid[idx.index[2]], iq3s_grid[idx.index[1]], iq3s_grid[idx.index[0]] ); const __m256i q2_2 = _mm256_set_epi32( iq3s_grid[idx.index[15]], iq3s_grid[idx.index[14]], iq3s_grid[idx.index[13]], iq3s_grid[idx.index[12]], From 4a6e2d6142ab815c964924896891e9ab3e050632 Mon Sep 17 00:00:00 2001 From: Michael Podvitskiy Date: Sat, 2 Mar 2024 20:52:25 +0100 Subject: [PATCH 516/812] llama : add abort_callback to interrupt computation (#5409) * using abort_callback from ggml to stop llama computation * format fix * a brief explaining comment --------- Co-authored-by: Georgi Gerganov --- llama.cpp | 18 ++++++++++++++++-- llama.h | 13 +++++++++++-- 2 files changed, 27 insertions(+), 4 deletions(-) diff --git a/llama.cpp b/llama.cpp index 697e85e89..d4c7a965b 100644 --- a/llama.cpp +++ b/llama.cpp @@ -1987,6 +1987,9 @@ struct llama_context { std::vector buf_compute_meta; ggml_backend_sched_t sched = nullptr; + ggml_abort_callback abort_callback = nullptr; + void * abort_callback_data = nullptr; + // input tensors ggml_backend_buffer_t buf_input = nullptr; ggml_context * ctx_input = nullptr; @@ -8071,6 +8074,7 @@ static void llama_graph_compute( if (lctx.backend_cpu != nullptr) { ggml_backend_cpu_set_n_threads(lctx.backend_cpu, n_threads); + ggml_backend_cpu_set_abort_callback(lctx.backend_cpu, lctx.abort_callback, lctx.abort_callback_data); } ggml_backend_sched_graph_compute(lctx.sched, gf); @@ -11856,6 +11860,8 @@ struct llama_context_params llama_context_default_params() { /*.embedding =*/ false, /*.offload_kqv =*/ true, /*.do_pooling =*/ true, + /*.abort_callback =*/ nullptr, + /*.abort_callback_data =*/ nullptr, }; return result; @@ -12038,8 +12044,11 @@ struct llama_context * llama_new_context_with_model( LLAMA_LOG_INFO("%s: freq_base = %.1f\n", __func__, cparams.rope_freq_base); LLAMA_LOG_INFO("%s: freq_scale = %g\n", __func__, cparams.rope_freq_scale); - ctx->rng = std::mt19937(params.seed); - ctx->logits_all = params.logits_all; + ctx->abort_callback = params.abort_callback; + ctx->abort_callback_data = params.abort_callback_data; + + ctx->rng = std::mt19937(params.seed); + ctx->logits_all = params.logits_all; const ggml_type type_k = params.type_k; const ggml_type type_v = params.type_v; @@ -12989,6 +12998,11 @@ void llama_set_n_threads(struct llama_context * ctx, uint32_t n_threads, uint32_ ctx->cparams.n_threads_batch = n_threads_batch; } +void llama_set_abort_callback(struct llama_context * ctx, bool (*abort_callback)(void * data), void * abort_callback_data) { + ctx->abort_callback = abort_callback; + ctx->abort_callback_data = abort_callback_data; +} + struct llama_batch llama_batch_get_one( llama_token * tokens, int32_t n_tokens, diff --git a/llama.h b/llama.h index ed51f478a..6406b5270 100644 --- a/llama.h +++ b/llama.h @@ -255,10 +255,16 @@ extern "C" { enum ggml_type type_v; // data type for V cache // Keep the booleans together to avoid misalignment during copy-by-value. - bool logits_all; // the llama_eval() call computes all logits, not just the last one (DEPRECATED - set llama_batch.logits instead) + bool logits_all; // the llama_decode() call computes all logits, not just the last one (DEPRECATED - set llama_batch.logits instead) bool embedding; // embedding mode only bool offload_kqv; // whether to offload the KQV ops (including the KV cache) to GPU bool do_pooling; // whether to pool (sum) embedding results by sequence id (ignored if no pooling layer) + + // Abort callback + // if it returns true, execution of llama_decode() will be aborted + // currently works only with CPU execution + ggml_abort_callback abort_callback; + void * abort_callback_data; }; // model quantization parameters @@ -632,7 +638,10 @@ extern "C" { // n_threads_batch is the number of threads used for prompt and batch processing (multiple tokens) LLAMA_API void llama_set_n_threads(struct llama_context * ctx, uint32_t n_threads, uint32_t n_threads_batch); - // Token logits obtained from the last call to llama_eval() + // Set abort callback + LLAMA_API void llama_set_abort_callback(struct llama_context * ctx, ggml_abort_callback abort_callback, void * abort_callback_data); + + // Token logits obtained from the last call to llama_decode() // The logits for the last token are stored in the last row // Logits for which llama_batch.logits[i] == 0 are undefined // Rows: n_tokens provided with llama_batch From 9731134296af3a6839cd682e51d9c2109a871de5 Mon Sep 17 00:00:00 2001 From: Pierrick Hymbert Date: Sat, 2 Mar 2024 22:00:14 +0100 Subject: [PATCH 517/812] server: tests: passkey challenge / self-extend with context shift demo (#5832) * server: tests: add models endpoint scenario * server: /v1/models add some metadata * server: tests: add debug field in context before scenario * server: tests: download model from HF, add batch size * server: tests: add passkey test * server: tests: add group attention params * server: do not truncate prompt tokens if self-extend through group attention is enabled * server: logs: do not truncate log values * server: tests - passkey - first good working value of nga * server: tests: fix server timeout * server: tests: fix passkey, add doc, fix regex content matching, fix timeout * server: tests: fix regex content matching * server: tests: schedule slow tests on master * server: metrics: fix when no prompt processed * server: tests: self-extend add llama-2-7B and Mixtral-8x7B-v0.1 * server: tests: increase timeout for completion * server: tests: keep only the PHI-2 test * server: tests: passkey add a negative test --- .github/workflows/server.yml | 17 +- examples/server/server.cpp | 46 +++- examples/server/tests/README.md | 50 +++- examples/server/tests/features/environment.py | 5 +- examples/server/tests/features/issues.feature | 1 + .../server/tests/features/parallel.feature | 5 +- .../server/tests/features/passkey.feature | 55 ++++ .../server/tests/features/security.feature | 3 +- examples/server/tests/features/server.feature | 23 +- examples/server/tests/features/steps/steps.py | 259 ++++++++++++++---- .../tests/features/wrong_usages.feature | 5 +- examples/server/tests/requirements.txt | 1 + examples/server/tests/tests.sh | 2 +- examples/server/utils.hpp | 3 +- 14 files changed, 363 insertions(+), 112 deletions(-) create mode 100644 examples/server/tests/features/passkey.feature diff --git a/.github/workflows/server.yml b/.github/workflows/server.yml index 0b6f6669b..8c6312508 100644 --- a/.github/workflows/server.yml +++ b/.github/workflows/server.yml @@ -10,6 +10,8 @@ on: pull_request: types: [opened, synchronize, reopened] paths: ['.github/workflows/server.yml', '**/CMakeLists.txt', '**/Makefile', '**/*.h', '**/*.hpp', '**/*.c', '**/*.cpp', '**/*.cu', '**/*.swift', '**/*.m', 'examples/server/tests/**.*'] + schedule: + - cron: '00 0 * * *' jobs: server: @@ -70,14 +72,15 @@ jobs: run: | pip install -r examples/server/tests/requirements.txt - - name: Download models - id: download_models - run: | - cd examples/server/tests - ../../../scripts/hf.sh --repo ggml-org/models --file tinyllamas/stories260K.gguf - - name: Tests - id: server_integration_test + id: server_integration_tests run: | cd examples/server/tests PORT=8888 ./tests.sh + + - name: Slow tests + id: server_integration_tests_slow + if: github.event.schedule != '' + run: | + cd examples/server/tests + PORT=8888 ./tests.sh --stop --no-skipped --no-capture --tags slow diff --git a/examples/server/server.cpp b/examples/server/server.cpp index 2b2f4a0f4..52daf9e7a 100644 --- a/examples/server/server.cpp +++ b/examples/server/server.cpp @@ -441,8 +441,8 @@ struct llama_server_context const int ga_w = params.grp_attn_w; if (ga_n != 1) { - GGML_ASSERT(ga_n > 0 && "ga_n must be positive"); // NOLINT - GGML_ASSERT(ga_w % ga_n == 0 && "ga_w must be a multiple of ga_n"); // NOLINT + GGML_ASSERT(ga_n > 0 && "ga_n must be positive"); // NOLINT + GGML_ASSERT(ga_w % ga_n == 0 && "ga_w must be a multiple of ga_n"); // NOLINT //GGML_ASSERT(n_ctx_train % ga_w == 0 && "n_ctx_train must be a multiple of ga_w"); // NOLINT //GGML_ASSERT(n_ctx >= n_ctx_train * ga_n && "n_ctx must be at least n_ctx_train * ga_n"); // NOLINT @@ -1709,8 +1709,8 @@ struct llama_server_context } slot.params.n_keep = std::min(slot.n_ctx - 4, slot.params.n_keep); - // if input prompt is too big, truncate it - if (slot.n_prompt_tokens >= slot.n_ctx) + // if input prompt is too big, truncate it, if group attention self-extend is disabled + if (slot.ga_n == 1 && slot.n_prompt_tokens >= slot.n_ctx) { const int n_left = slot.n_ctx - slot.params.n_keep; const int n_block_size = n_left / 2; @@ -1785,9 +1785,11 @@ struct llama_server_context } LOG_INFO("slot progression", { - { "slot_id", slot.id }, - { "task_id", slot.task_id }, - { "n_past", slot.n_past }, + { "slot_id", slot.id }, + { "task_id", slot.task_id }, + { "n_past", slot.n_past }, + { "n_past_se", slot.n_past_se }, + { "ga_i", slot.ga_i }, { "n_prompt_tokens_processed", slot.n_prompt_tokens_processed } }); } @@ -2001,6 +2003,17 @@ struct llama_server_context LOG_VERBOSE("slots updated", {}); return true; } + + json model_meta() { + return json{ + {"vocab_type", llama_vocab_type(model)}, + {"n_vocab", llama_n_vocab(model)}, + {"n_ctx_train", llama_n_ctx_train(model)}, + {"n_embd", llama_n_embd(model)}, + {"n_params", llama_model_n_params(model)}, + {"size", llama_model_size(model)}, + }; + } }; static void server_print_usage(const char *argv0, const gpt_params ¶ms, @@ -2911,9 +2924,10 @@ int main(int argc, char **argv) for (const auto& metric_def : metrics_def) { std::string name = metric_def["name"]; std::string help = metric_def["help"]; - prometheus << "# HELP llamacpp:" << name << " " << help << "\n" - << "# TYPE llamacpp:" << name << " " << type << "\n" - << "llamacpp:" << name << " " << metric_def["value"] << "\n"; + auto value = json_value(metric_def, "value", 0); + prometheus << "# HELP llamacpp:" << name << " " << help << "\n" + << "# TYPE llamacpp:" << name << " " << type << "\n" + << "llamacpp:" << name << " " << value << "\n"; } } @@ -2994,6 +3008,7 @@ int main(int argc, char **argv) state.store(SERVER_STATE_READY); LOG_INFO("model loaded", {}); } + const auto model_meta = llama.model_meta(); if (sparams.chat_template.empty()) { // custom chat template is not supplied // check if the template comes with the model is supported by us @@ -3143,7 +3158,7 @@ int main(int argc, char **argv) } }); - svr.Get("/v1/models", [¶ms](const httplib::Request& req, httplib::Response& res) + svr.Get("/v1/models", [¶ms, &model_meta](const httplib::Request& req, httplib::Response& res) { res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin")); std::time_t t = std::time(0); @@ -3152,10 +3167,11 @@ int main(int argc, char **argv) {"object", "list"}, {"data", { { - {"id", params.model_alias}, - {"object", "model"}, - {"created", t}, - {"owned_by", "llamacpp"} + {"id", params.model_alias}, + {"object", "model"}, + {"created", t}, + {"owned_by", "llamacpp"}, + {"meta", model_meta} }, }} }; diff --git a/examples/server/tests/README.md b/examples/server/tests/README.md index 0b9fdc4e7..95a0353b6 100644 --- a/examples/server/tests/README.md +++ b/examples/server/tests/README.md @@ -1,22 +1,30 @@ # Server tests -Python based server tests scenario using [BDD](https://en.wikipedia.org/wiki/Behavior-driven_development) and [behave](https://behave.readthedocs.io/en/latest/): - * [issues.feature](./features/issues.feature) Pending issues scenario - * [parallel.feature](./features/parallel.feature) Scenario involving multi slots and concurrent requests - * [security.feature](./features/security.feature) Security, CORS and API Key - * [server.feature](./features/server.feature) Server base scenario: completion, embedding, tokenization, etc... +Python based server tests scenario using [BDD](https://en.wikipedia.org/wiki/Behavior-driven_development) +and [behave](https://behave.readthedocs.io/en/latest/): + +* [issues.feature](./features/issues.feature) Pending issues scenario +* [parallel.feature](./features/parallel.feature) Scenario involving multi slots and concurrent requests +* [security.feature](./features/security.feature) Security, CORS and API Key +* [server.feature](./features/server.feature) Server base scenario: completion, embedding, tokenization, etc... Tests target GitHub workflows job runners with 4 vCPU. -Requests are using [aiohttp](https://docs.aiohttp.org/en/stable/client_reference.html), [asyncio](https://docs.python.org/fr/3/library/asyncio.html) based http client. +Requests are +using [aiohttp](https://docs.aiohttp.org/en/stable/client_reference.html), [asyncio](https://docs.python.org/fr/3/library/asyncio.html) +based http client. -Note: If the host architecture inference speed is faster than GitHub runners one, parallel scenario may randomly fail. To mitigate it, you can increase values in `n_predict`, `kv_size`. +Note: If the host architecture inference speed is faster than GitHub runners one, parallel scenario may randomly fail. +To mitigate it, you can increase values in `n_predict`, `kv_size`. ### Install dependencies + `pip install -r requirements.txt` ### Run tests + 1. Build the server + ```shell cd ../../.. mkdir build @@ -24,24 +32,36 @@ cd build cmake ../ cmake --build . --target server ``` -2. download required models: - 1. `../../../scripts/hf.sh --repo ggml-org/models --file tinyllamas/stories260K.gguf` -3. Start the test: `./tests.sh` + +2. Start the test: `./tests.sh` It's possible to override some scenario steps values with environment variables: - - `PORT` -> `context.server_port` to set the listening port of the server during scenario, default: `8080` - - `LLAMA_SERVER_BIN_PATH` -> to change the server binary path, default: `../../../build/bin/server` - - `DEBUG` -> "ON" to enable steps and server verbose mode `--verbose` - - `SERVER_LOG_FORMAT_JSON` -> if set switch server logs to json format + +| variable | description | +|--------------------------|------------------------------------------------------------------------------------------------| +| `PORT` | `context.server_port` to set the listening port of the server during scenario, default: `8080` | +| `LLAMA_SERVER_BIN_PATH` | to change the server binary path, default: `../../../build/bin/server` | +| `DEBUG` | "ON" to enable steps and server verbose mode `--verbose` | +| `SERVER_LOG_FORMAT_JSON` | if set switch server logs to json format | +| `N_GPU_LAYERS` | number of model layers to offload to VRAM `-ngl --n-gpu-layers` | ### Run @bug, @wip or @wrong_usage annotated scenario Feature or Scenario must be annotated with `@llama.cpp` to be included in the default scope. + - `@bug` annotation aims to link a scenario with a GitHub issue. - `@wrong_usage` are meant to show user issue that are actually an expected behavior - `@wip` to focus on a scenario working in progress +- `@slow` heavy test, disabled by default To run a scenario annotated with `@bug`, start: -`DEBUG=ON ./tests.sh --no-skipped --tags bug` + +```shell +DEBUG=ON ./tests.sh --no-skipped --tags bug +``` After changing logic in `steps.py`, ensure that `@bug` and `@wrong_usage` scenario are updated. + +```shell +./tests.sh --no-skipped --tags bug,wrong_usage || echo "should failed but compile" +``` diff --git a/examples/server/tests/features/environment.py b/examples/server/tests/features/environment.py index 09e826747..9fd330db6 100644 --- a/examples/server/tests/features/environment.py +++ b/examples/server/tests/features/environment.py @@ -7,7 +7,10 @@ from signal import SIGKILL def before_scenario(context, scenario): - print(f"\x1b[33;42mStarting new scenario: {scenario.name}!\x1b[0m") + context.debug = 'DEBUG' in os.environ and os.environ['DEBUG'] == 'ON' + if context.debug: + print("DEBUG=ON\n") + print(f"\x1b[33;42mStarting new scenario: {scenario.name}!\x1b[0m\n") port = 8080 if 'PORT' in os.environ: port = int(os.environ['PORT']) diff --git a/examples/server/tests/features/issues.feature b/examples/server/tests/features/issues.feature index bf5a175a3..7b13e44ca 100644 --- a/examples/server/tests/features/issues.feature +++ b/examples/server/tests/features/issues.feature @@ -1,4 +1,5 @@ # List of ongoing issues +# run with: DEBUG=ON ./tests.sh --no-skipped --tags bug @bug Feature: Issues # No confirmed issue at the moment diff --git a/examples/server/tests/features/parallel.feature b/examples/server/tests/features/parallel.feature index 5f895cf90..86cdf7282 100644 --- a/examples/server/tests/features/parallel.feature +++ b/examples/server/tests/features/parallel.feature @@ -1,11 +1,12 @@ @llama.cpp +@parallel Feature: Parallel Background: Server startup Given a server listening on localhost:8080 - And a model file stories260K.gguf - And a model alias tinyllama-2 + And a model file tinyllamas/stories260K.gguf from HF repo ggml-org/models And 42 as server seed + And 512 as batch size And 64 KV cache size And 2 slots And embeddings extraction diff --git a/examples/server/tests/features/passkey.feature b/examples/server/tests/features/passkey.feature new file mode 100644 index 000000000..1bde7aab8 --- /dev/null +++ b/examples/server/tests/features/passkey.feature @@ -0,0 +1,55 @@ +# run with: ./tests.sh --no-skipped --tags passkey +@passkey +@slow +Feature: Passkey / Self-extend with context shift + + Background: Server startup + Given a server listening on localhost:8080 + + # Generates a long text of junk and inserts a secret passkey number inside it. + # Then we query the LLM for the secret passkey. + # see #3856 and #4810 + Scenario Outline: Passkey + Given a model file from HF repo + And as batch size + And as number of junk + And server max tokens to predict + And 42 as seed + And KV cache size + And 1 slots + And group attention factor to extend context size through self-extend + And group attention width to extend context size through self-extend + # Can be override with N_GPU_LAYERS + And GPU offloaded layers + Then the server is starting + Then the server is healthy + Given available models + Then model 0 is trained on tokens context + Given a prefix prompt: + """ + here is an important info hidden inside a lot of irrelevant text. Find it and memorize them. I will quiz you about the important information there. + """ + And a passkey prompt template: + """ + The pass key is Remember it. is the pass key. + """ + And a junk suffix prompt: + """ + The grass is green. The sky is blue. The sun is yellow. Here we go. There and back again. + """ + And a suffix prompt: + """ + What is the pass key? The pass key is + """ + Given a "" passkey challenge prompt with the passkey inserted every junk + And a completion request with no api error + Then tokens are predicted matching + + Examples: + | hf_repo | hf_file | n_ctx_train | ngl | n_ctx | n_batch | n_ga | n_ga_w | n_junk | i_pos | passkey | n_predicted | re_content | + | TheBloke/phi-2-GGUF | phi-2.Q4_K_M.gguf | 2048 | 5 | 8192 | 512 | 4 | 512 | 250 | 50 | 42 | 1 | 42 | + | TheBloke/phi-2-GGUF | phi-2.Q4_K_M.gguf | 2048 | 5 | 8192 | 512 | 2 | 512 | 250 | 50 | 42 | 1 | \b((?!42)\w)+\b | + #| TheBloke/Llama-2-7B-GGUF | llama-2-7b.Q2_K.gguf | 4096 | 3 | 16384 | 512 | 4 | 512 | 500 | 300 | 1234 | 5 | 1234 | + #| TheBloke/Mixtral-8x7B-v0.1-GGUF | mixtral-8x7b-v0.1.Q2_K.gguf | 32768 | 2 | 16384 | 512 | 4 | 512 | 500 | 100 | 0987 | 5 | 0 + # 987 | + diff --git a/examples/server/tests/features/security.feature b/examples/server/tests/features/security.feature index db06d3977..42a6709a5 100644 --- a/examples/server/tests/features/security.feature +++ b/examples/server/tests/features/security.feature @@ -1,9 +1,10 @@ @llama.cpp +@security Feature: Security Background: Server startup with an api key defined Given a server listening on localhost:8080 - And a model file stories260K.gguf + And a model file tinyllamas/stories260K.gguf from HF repo ggml-org/models And a server api key llama.cpp Then the server is starting Then the server is healthy diff --git a/examples/server/tests/features/server.feature b/examples/server/tests/features/server.feature index b571582a7..7c977bcce 100644 --- a/examples/server/tests/features/server.feature +++ b/examples/server/tests/features/server.feature @@ -1,15 +1,17 @@ @llama.cpp +@server Feature: llama.cpp server Background: Server startup Given a server listening on localhost:8080 - And a model file stories260K.gguf + And a model file tinyllamas/stories260K.gguf from HF repo ggml-org/models And a model alias tinyllama-2 And 42 as server seed # KV Cache corresponds to the total amount of tokens # that can be stored across all independent sequences: #4130 # see --ctx-size and #5568 And 32 KV cache size + And 512 as batch size And 1 slots And embeddings extraction And 32 server max tokens to predict @@ -29,9 +31,9 @@ Feature: llama.cpp server And prometheus metrics are exposed Examples: Prompts - | prompt | n_predict | re_content | n_predicted | - | I believe the meaning of life is | 8 | (readgoing)+ | 8 | - | Write a joke about AI | 64 | (parkfriendsscaredalways)+ | 32 | + | prompt | n_predict | re_content | n_predicted | + | I believe the meaning of life is | 8 | (read\|going)+ | 8 | + | Write a joke about AI | 64 | (park\|friends\|scared\|always)+ | 32 | Scenario Outline: OAI Compatibility Given a model @@ -43,9 +45,9 @@ Feature: llama.cpp server Then tokens are predicted matching Examples: Prompts - | model | system_prompt | user_prompt | max_tokens | re_content | n_predicted | enable_streaming | - | llama-2 | Book | What is the best book | 8 | (Momwhat)+ | 8 | disabled | - | codellama70b | You are a coding assistant. | Write the fibonacci function in c++. | 64 | (thankshappybird)+ | 32 | enabled | + | model | system_prompt | user_prompt | max_tokens | re_content | n_predicted | enable_streaming | + | llama-2 | Book | What is the best book | 8 | (Mom\|what)+ | 8 | disabled | + | codellama70b | You are a coding assistant. | Write the fibonacci function in c++. | 64 | (thanks\|happy\|bird)+ | 32 | enabled | Scenario: Embedding When embeddings are computed for: @@ -75,10 +77,15 @@ Feature: llama.cpp server When an OAI compatible embeddings computation request for multiple inputs Then embeddings are generated - Scenario: Tokenize / Detokenize When tokenizing: """ What is the capital of France ? """ Then tokens can be detokenize + + Scenario: Models available + Given available models + Then 1 models are supported + Then model 0 is identified by tinyllama-2 + Then model 0 is trained on 128 tokens context diff --git a/examples/server/tests/features/steps/steps.py b/examples/server/tests/features/steps/steps.py index 381da105e..319527802 100644 --- a/examples/server/tests/features/steps/steps.py +++ b/examples/server/tests/features/steps/steps.py @@ -13,6 +13,7 @@ import aiohttp import openai from behave import step from behave.api.async_step import async_run_until_complete +from huggingface_hub import hf_hub_download from prometheus_client import parser @@ -26,17 +27,23 @@ def step_server_config(context, server_fqdn, server_port): context.base_url = f'http://{context.server_fqdn}:{context.server_port}' - context.debug = 'DEBUG' in os.environ and os.environ['DEBUG'] == 'ON' context.model_alias = None + context.n_batch = None context.n_ctx = None + context.n_ga = None + context.n_ga_w = None + context.n_gpu_layer = None context.n_predict = None context.n_server_predict = None context.n_slots = None + context.prompt_prefix = None + context.prompt_suffix = None context.server_api_key = None context.server_continuous_batching = False context.server_embeddings = False context.server_metrics = False context.server_process = None + context.seed = None context.server_seed = None context.user_api_key = None @@ -45,9 +52,11 @@ def step_server_config(context, server_fqdn, server_port): context.prompts = [] -@step(u'a model file {model_file}') -def step_model_file(context, model_file): - context.model_file = model_file +@step(u'a model file {hf_file} from HF repo {hf_repo}') +def step_download_hf_model(context, hf_file, hf_repo): + context.model_file = hf_hub_download(repo_id=hf_repo, filename=hf_file) + if context.debug: + print(f"model file: {context.model_file}\n") @step(u'a model alias {model_alias}') @@ -55,24 +64,34 @@ def step_model_alias(context, model_alias): context.model_alias = model_alias -@step(u'{seed} as server seed') +@step(u'{seed:d} as server seed') def step_seed(context, seed): - context.server_seed = int(seed) + context.server_seed = seed -@step(u'{n_ctx} KV cache size') +@step(u'{ngl:d} GPU offloaded layers') +def step_n_gpu_layer(context, ngl): + if 'N_GPU_LAYERS' in os.environ: + new_ngl = int(os.environ['N_GPU_LAYERS']) + if context.debug: + print(f"-ngl upgraded from {ngl} to {new_ngl}") + ngl = new_ngl + context.n_gpu_layer = ngl + + +@step(u'{n_ctx:d} KV cache size') def step_n_ctx(context, n_ctx): - context.n_ctx = int(n_ctx) + context.n_ctx = n_ctx -@step(u'{n_slots} slots') +@step(u'{n_slots:d} slots') def step_n_slots(context, n_slots): - context.n_slots = int(n_slots) + context.n_slots = n_slots -@step(u'{n_predict} server max tokens to predict') +@step(u'{n_predict:d} server max tokens to predict') def step_server_n_predict(context, n_predict): - context.n_server_predict = int(n_predict) + context.n_server_predict = n_predict @step(u'continuous batching') @@ -116,11 +135,13 @@ async def step_wait_for_the_server_to_be_started(context, expecting_status): case 'ready' | 'idle': await wait_for_health_status(context, context.base_url, 200, 'ok', + timeout=10, params={'fail_on_no_slot': 0, 'include_slots': 0}, slots_idle=context.n_slots, slots_processing=0, expected_slots=[{'id': slot_id, 'state': 0} - for slot_id in range(context.n_slots)]) + for slot_id in + range(context.n_slots if context.n_slots else 1)]) case 'busy': await wait_for_health_status(context, context.base_url, 503, 'no slot available', @@ -128,7 +149,8 @@ async def step_wait_for_the_server_to_be_started(context, expecting_status): slots_idle=0, slots_processing=context.n_slots, expected_slots=[{'id': slot_id, 'state': 1} - for slot_id in range(context.n_slots)]) + for slot_id in + range(context.n_slots if context.n_slots else 1)]) case _: assert False, "unknown status" @@ -157,24 +179,24 @@ async def step_request_completion(context, api_error): context.base_url, debug=context.debug, n_predict=context.n_predict, - server_seed=context.server_seed, + seed=await completions_seed(context), expect_api_error=expect_api_error, user_api_key=context.user_api_key) context.tasks_result.append(completion) if context.debug: - print(f"Completion response: {completion}") + print(f"Completion response: {completion}\n") if expect_api_error: assert completion == 401, f"completion must be an 401 status code: {completion}" -@step(u'{predicted_n} tokens are predicted matching {re_content}') +@step(u'{predicted_n:d} tokens are predicted matching {re_content}') def step_n_tokens_predicted_with_content(context, predicted_n, re_content): - assert_n_tokens_predicted(context.tasks_result.pop(), int(predicted_n), re_content) + assert_n_tokens_predicted(context.tasks_result.pop(), predicted_n, re_content) -@step(u'{predicted_n} tokens are predicted') +@step(u'{predicted_n:d} tokens are predicted') def step_n_tokens_predicted(context, predicted_n): - assert_n_tokens_predicted(context.tasks_result.pop(), int(predicted_n)) + assert_n_tokens_predicted(context.tasks_result.pop(), predicted_n) @step(u'a user prompt {user_prompt}') @@ -192,9 +214,9 @@ def step_model(context, model): context.model = model -@step(u'{max_tokens} max tokens to predict') +@step(u'{max_tokens:d} max tokens to predict') def step_max_tokens(context, max_tokens): - context.n_predict = int(max_tokens) + context.n_predict = max_tokens @step(u'streaming is {enable_streaming}') @@ -222,11 +244,70 @@ def step_server_api_key(context, server_api_key): context.server_api_key = server_api_key +@step(u'{n_junk:d} as number of junk') +def step_n_junk(context, n_junk): + context.n_junk = n_junk + + +@step(u'{n_batch:d} as batch size') +def step_n_batch(context, n_batch): + context.n_batch = n_batch + + +@step(u'{seed:d} as seed') +def step_seed(context, seed): + context.seed = seed + + +@step(u'a prefix prompt') +def step_prompt_prefix(context): + context.prompt_prefix = context.text + + +@step(u'a junk suffix prompt') +def step_prompt_junk_suffix(context): + context.prompt_junk_suffix = context.text + + +@step(u'a suffix prompt') +def step_prompt_suffix(context): + context.prompt_suffix = context.text + + +@step(u'{n_ga:d} group attention factor' + u' to extend context size through self-extend') +def step_impl(context, n_ga): + context.n_ga = n_ga + + +@step(u'{n_ga_w:d} group attention width to extend context size through self-extend') +def step_impl(context, n_ga_w): + context.n_ga_w = n_ga_w + + +@step(u'a passkey prompt template') +def step_prompt_passkey(context): + context.prompt_passkey = context.text + + +@step(u'a "{passkey}" passkey challenge prompt with the passkey inserted every {i_pos:d} junk') +def step_prompt_passkey(context, passkey, i_pos): + prompt = "" + for i in range(context.n_junk): + if i % context.n_junk == i_pos: + prompt += context.prompt_passkey # the passkey is already substituted + prompt += context.prompt_junk_suffix + if context.debug: + passkey_highlight = "\x1b[33m" + passkey + "\x1b[0m" + print(f"Passkey challenge:\n```{prompt.replace(passkey, passkey_highlight)}```\n") + context.prompts.append(context.prompt_prefix + prompt + context.prompt_suffix) + + @step(u'an OAI compatible chat completions request with {api_error} api error') @async_run_until_complete async def step_oai_chat_completions(context, api_error): if context.debug: - print(f"Submitting OAI compatible completions request...") + print(f"Submitting OAI compatible completions request...\n") expect_api_error = api_error == 'raised' completion = await oai_chat_completions(context.prompts.pop(), context.system_prompt, @@ -241,8 +322,7 @@ async def step_oai_chat_completions(context, api_error): enable_streaming=context.enable_streaming if hasattr(context, 'enable_streaming') else None, - server_seed=context.server_seed - if hasattr(context, 'server_seed') else None, + seed=await completions_seed(context), user_api_key=context.user_api_key if hasattr(context, 'user_api_key') else None, @@ -276,8 +356,10 @@ async def step_concurrent_completion_requests(context): # prompt is inserted automatically context.base_url, debug=context.debug, + prompt_prefix=context.prompt_prefix, + prompt_suffix=context.prompt_suffix, n_predict=context.n_predict if hasattr(context, 'n_predict') else None, - server_seed=context.server_seed if hasattr(context, 'server_seed') else None, + seed=await completions_seed(context), user_api_key=context.user_api_key if hasattr(context, 'user_api_key') else None) @@ -297,8 +379,7 @@ async def step_oai_chat_completions(context): if hasattr(context, 'n_predict') else None, enable_streaming=context.enable_streaming if hasattr(context, 'enable_streaming') else None, - server_seed=context.server_seed - if hasattr(context, 'server_seed') else None, + seed=await completions_seed(context), user_api_key=context.user_api_key if hasattr(context, 'user_api_key') else None) @@ -318,7 +399,9 @@ async def step_oai_chat_completions(context): if hasattr(context, 'n_predict') else None, enable_streaming=context.enable_streaming if hasattr(context, 'enable_streaming') else None, - server_seed=context.server_seed + seed=context.seed + if hasattr(context, 'seed') else + context.server_seed if hasattr(context, 'server_seed') else None, user_api_key=context.user_api_key if hasattr(context, 'user_api_key') else None) @@ -330,11 +413,10 @@ async def step_all_prompts_are_predicted(context): await all_prompts_are_predicted(context) -@step(u'all prompts are predicted with {n_predict} tokens') +@step(u'all prompts are predicted with {n_expected_predicted:d} tokens') @async_run_until_complete -async def step_all_prompts_are_predicted_with_n_tokens(context, n_predict): - expected_predicted_n = int(n_predict) - await all_prompts_are_predicted(context, expected_predicted_n) +async def step_all_prompts_are_predicted_with_n_tokens(context, n_expected_predicted): + await all_prompts_are_predicted(context, n_expected_predicted) async def all_prompts_are_predicted(context, expected_predicted_n=None): @@ -464,6 +546,8 @@ async def step_prometheus_metrics_exported(context): assert metrics_response.headers['Content-Type'] == "text/plain; version=0.0.4" metrics_raw = await metrics_response.text() metric_exported = False + if context.debug: + print(f"/metrics answer:\n{metrics_raw}\n") for metric in parser.text_string_to_metric_families(metrics_raw): match metric.name: case "llamacpp:kv_cache_usage_ratio": @@ -472,6 +556,37 @@ async def step_prometheus_metrics_exported(context): assert metric_exported, "No metrics exported" +@step(u'available models') +def step_available_models(context): + # openai client always expects an api_key + openai.api_key = context.user_api_key if context.user_api_key is not None else 'nope' + openai.api_base = f'{context.base_url}/v1' + context.models = openai.Model.list().data + + +@step(u'{n_model:d} models are supported') +def step_supported_models(context, n_model): + if context.debug: + print("server models available:", context.models) + assert len(context.models) == n_model + + +@step(u'model {i_model:d} is {param} {preposition} {param_value}') +def step_supported_models(context, i_model, param, preposition, param_value): + assert i_model < len(context.models) + model = context.models[i_model] + + param_value = param_value.split(' ', 1)[0] + match param: + case 'identified': + value = model.id + case 'trained': + value = str(model.meta.n_ctx_train) + case _: + assert False, "param {param} not supported" + assert param_value == value, f"model param {param} {value} != {param_value}" + + async def concurrent_requests(context, f_completion, *args, **kwargs): n_prompts = len(context.prompts) if context.debug: @@ -486,8 +601,10 @@ async def concurrent_requests(context, f_completion, *args, **kwargs): async def request_completion(prompt, base_url, debug=False, + prompt_prefix=None, + prompt_suffix=None, n_predict=None, - server_seed=None, + seed=None, expect_api_error=None, user_api_key=None): if debug: @@ -504,11 +621,14 @@ async def request_completion(prompt, async with aiohttp.ClientSession() as session: async with session.post(f'{base_url}/completion', json={ + "input_prefix": prompt_prefix, "prompt": prompt, - "n_predict": int(n_predict) if n_predict is not None else -1, - "seed": server_seed if server_seed is not None else 42 + "input_suffix": prompt_suffix, + "n_predict": n_predict if n_predict is not None else -1, + "seed": seed if seed is not None else 42 }, - headers=headers) as response: + headers=headers, + timeout=3600) as response: if expect_api_error is None or not expect_api_error: assert response.status == 200 assert response.headers['Access-Control-Allow-Origin'] == origin @@ -526,14 +646,14 @@ async def oai_chat_completions(user_prompt, model=None, n_predict=None, enable_streaming=None, - server_seed=None, + seed=None, user_api_key=None, expect_api_error=None): if debug: print(f"Sending OAI Chat completions request: {user_prompt}") # openai client always expects an api key user_api_key = user_api_key if user_api_key is not None else 'nope' - seed = server_seed if server_seed is not None else 42 + seed = seed if seed is not None else 42 enable_streaming = enable_streaming if enable_streaming is not None else False payload = { "messages": [ @@ -692,20 +812,32 @@ def assert_n_tokens_predicted(completion_response, expected_predicted_n=None, re content = completion_response['content'] n_predicted = completion_response['timings']['predicted_n'] assert len(content) > 0, "no token predicted" - if expected_predicted_n is not None: + if re_content is not None: + p = re.compile(re_content, flags=RegexFlag.IGNORECASE | RegexFlag.MULTILINE | RegexFlag.DOTALL) + matches = p.finditer(content) + last_match = 0 + highlighted = '' + for match in matches: + start, end = match.span() + highlighted += content[last_match: start] + highlighted += '\x1b[33m' + highlighted += content[start: end] + highlighted += '\x1b[0m' + last_match = end + highlighted += content[last_match:] + if 'DEBUG' in os.environ and os.environ['DEBUG'] == 'ON': + print(f"Checking completion response: {highlighted}\n") + assert last_match > 0, f'/{re_content}/ must match ```{highlighted}```' + if expected_predicted_n and expected_predicted_n > 0: assert n_predicted == expected_predicted_n, (f'invalid number of tokens predicted:' f' {n_predicted} <> {expected_predicted_n}') - if re_content is not None: - re_content = '^.*' + re_content.replace('', '|') + '.*$' - assert re.match(re_content, content, flags=RegexFlag.IGNORECASE | RegexFlag.MULTILINE | RegexFlag.DOTALL), ( - f'invalid tokens predicted:' - f' ```\n{content}\n``` do not match /{re_content}/') + async def gather_tasks_results(context): n_tasks = len(context.concurrent_tasks) if context.debug: - print(f"Waiting for all {n_tasks} tasks results...") + print(f"Waiting for all {n_tasks} tasks results...\n") for task_no in range(n_tasks): context.tasks_result.append(await context.concurrent_tasks.pop()) n_completions = len(context.tasks_result) @@ -716,15 +848,13 @@ async def wait_for_health_status(context, base_url, expected_http_status_code, expected_health_status, + timeout=3, params=None, slots_idle=None, slots_processing=None, expected_slots=None): if context.debug: - print(f"Starting checking for health for expected_health_status={expected_health_status}") - timeout = 3 # seconds - if expected_health_status == 'ok': - timeout = 10 # CI slow inference + print(f"Starting checking for health for expected_health_status={expected_health_status}\n") interval = 0.5 counter = 0 async with aiohttp.ClientSession() as session: @@ -734,7 +864,7 @@ async def wait_for_health_status(context, health = await health_response.json() if context.debug: print(f"HEALTH - response for expected health status='{expected_health_status}' on " - f"'{base_url}/health'?{params} is {health}") + f"'{base_url}/health'?{params} is {health}\n") if (status_code == expected_http_status_code and health['status'] == expected_health_status and (slots_idle is None or health['slots_idle'] == slots_idle) @@ -757,7 +887,7 @@ async def wait_for_health_status(context, if expected_http_status_code == 503: if len(context.tasks_result) == 0: print("\x1b[5;37;43mWARNING: forcing concurrent tasks," - " busy health check missed, probably too fast inference\x1b[0m") + " busy health check missed, probably too fast inference\x1b[0m\n") n_completions = await gather_tasks_results(context) if n_completions > 0: return @@ -791,6 +921,11 @@ def assert_slots_status(slots, expected_slots): f" = {expected[key]} != {slot[key]}") +async def completions_seed(context): + return context.seed if hasattr(context, 'seed') and context.seed is not None \ + else context.server_seed if hasattr(context, 'server_seed') else None + + def start_server_background(context): context.server_path = '../../../build/bin/server' if 'LLAMA_SERVER_BIN_PATH' in os.environ: @@ -800,27 +935,35 @@ def start_server_background(context): '--port', context.server_port, '--model', context.model_file ] + if context.n_batch: + server_args.extend(['--batch-size', context.n_batch]) + if context.n_gpu_layer: + server_args.extend(['--n-gpu-layers', context.n_gpu_layer]) if context.server_continuous_batching: server_args.append('--cont-batching') if context.server_embeddings: server_args.append('--embedding') if context.server_metrics: server_args.append('--metrics') - if context.model_alias is not None: + if context.model_alias: server_args.extend(['--alias', context.model_alias]) - if context.n_ctx is not None: + if context.n_ctx: server_args.extend(['--ctx-size', context.n_ctx]) - if context.n_slots is not None: + if context.n_slots: server_args.extend(['--parallel', context.n_slots]) - if context.n_server_predict is not None: + if context.n_server_predict: server_args.extend(['--n-predict', context.n_server_predict]) - if context.server_api_key is not None: + if context.server_api_key: server_args.extend(['--api-key', context.server_api_key]) + if context.n_ga: + server_args.extend(['--grp-attn-n', context.n_ga]) + if context.n_ga_w: + server_args.extend(['--grp-attn-w', context.n_ga_w]) if context.debug: server_args.append('--verbose') if 'SERVER_LOG_FORMAT_JSON' not in os.environ: server_args.extend(['--log-format', "text"]) - print(f"starting server with: {context.server_path}", *server_args) + print(f"starting server with: {context.server_path} {server_args}\n") context.server_process = subprocess.Popen( [str(arg) for arg in [context.server_path, *server_args]], close_fds=True) diff --git a/examples/server/tests/features/wrong_usages.feature b/examples/server/tests/features/wrong_usages.feature index e228b2371..cf14b3b44 100644 --- a/examples/server/tests/features/wrong_usages.feature +++ b/examples/server/tests/features/wrong_usages.feature @@ -1,4 +1,4 @@ -# run with ./test.sh --tags wrong_usage +# run with: ./tests.sh --no-skipped --tags wrong_usage @wrong_usage Feature: Wrong usage of llama.cpp server @@ -7,7 +7,7 @@ Feature: Wrong usage of llama.cpp server # or pass n_predict/max_tokens in the request. Scenario: Infinite loop Given a server listening on localhost:8080 - And a model file stories260K.gguf + And a model file tinyllamas/stories260K.gguf from HF repo ggml-org/models # Uncomment below to fix the issue #And 64 server max tokens to predict Then the server is starting @@ -18,4 +18,5 @@ Feature: Wrong usage of llama.cpp server # Uncomment below to fix the issue #And 128 max tokens to predict Given concurrent completion requests + Then the server is idle Then all prompts are predicted diff --git a/examples/server/tests/requirements.txt b/examples/server/tests/requirements.txt index 334fa4a70..5d4210164 100644 --- a/examples/server/tests/requirements.txt +++ b/examples/server/tests/requirements.txt @@ -1,4 +1,5 @@ aiohttp~=3.9.3 behave~=1.2.6 +huggingface_hub~=0.20.3 openai~=0.25.0 prometheus-client~=0.20.0 diff --git a/examples/server/tests/tests.sh b/examples/server/tests/tests.sh index 17a4e6fc6..1c6c5695f 100755 --- a/examples/server/tests/tests.sh +++ b/examples/server/tests/tests.sh @@ -5,7 +5,7 @@ set -eu if [ $# -lt 1 ] then # Start @llama.cpp scenario - behave --summary --stop --no-capture --exclude 'issues|wrong_usages' --tags llama.cpp + behave --summary --stop --no-capture --exclude 'issues|wrong_usages|passkey' --tags llama.cpp else behave "$@" fi diff --git a/examples/server/utils.hpp b/examples/server/utils.hpp index d98541f26..b6e49d8b9 100644 --- a/examples/server/utils.hpp +++ b/examples/server/utils.hpp @@ -126,8 +126,7 @@ static inline void server_log(const char *level, const char *function, int line, for (const auto& el : log.items()) { const std::string value = el.value().dump(-1, ' ', false, json::error_handler_t::replace); - snprintf(buf, 1024, " %s=%s", el.key().c_str(), value.c_str()); - ss << buf; + ss << " " << el.key() << "=" << value; } const std::string str = ss.str(); From fa974646e1a2024fc7dc9e6f27cf1f2f5d4a3763 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Sun, 3 Mar 2024 06:11:31 +0200 Subject: [PATCH 518/812] flake.lock: Update (#5842) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'flake-parts': 'github:hercules-ci/flake-parts/b253292d9c0a5ead9bc98c4e9a26c6312e27d69f' (2024-02-01) → 'github:hercules-ci/flake-parts/f7b3c975cf067e56e7cda6cb098ebe3fb4d74ca2' (2024-03-01) • Updated input 'flake-parts/nixpkgs-lib': 'github:NixOS/nixpkgs/97b17f32362e475016f942bbdfda4a4a72a8a652?dir=lib' (2024-01-29) → 'github:NixOS/nixpkgs/1536926ef5621b09bba54035ae2bb6d806d72ac8?dir=lib' (2024-02-29) • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/cbc4211f0afffe6dfd2478a62615dd5175a13f9a' (2024-02-23) → 'github:NixOS/nixpkgs/1536926ef5621b09bba54035ae2bb6d806d72ac8' (2024-02-29) Co-authored-by: github-actions[bot] --- flake.lock | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/flake.lock b/flake.lock index 9f659ba8f..b1b091656 100644 --- a/flake.lock +++ b/flake.lock @@ -5,11 +5,11 @@ "nixpkgs-lib": "nixpkgs-lib" }, "locked": { - "lastModified": 1706830856, - "narHash": "sha256-a0NYyp+h9hlb7ddVz4LUn1vT/PLwqfrWYcHMvFB1xYg=", + "lastModified": 1709336216, + "narHash": "sha256-Dt/wOWeW6Sqm11Yh+2+t0dfEWxoMxGBvv3JpIocFl9E=", "owner": "hercules-ci", "repo": "flake-parts", - "rev": "b253292d9c0a5ead9bc98c4e9a26c6312e27d69f", + "rev": "f7b3c975cf067e56e7cda6cb098ebe3fb4d74ca2", "type": "github" }, "original": { @@ -20,11 +20,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1708655239, - "narHash": "sha256-ZrP/yACUvDB+zbqYJsln4iwotbH6CTZiTkANJ0AgDv4=", + "lastModified": 1709237383, + "narHash": "sha256-cy6ArO4k5qTx+l5o+0mL9f5fa86tYUX3ozE1S+Txlds=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "cbc4211f0afffe6dfd2478a62615dd5175a13f9a", + "rev": "1536926ef5621b09bba54035ae2bb6d806d72ac8", "type": "github" }, "original": { @@ -37,11 +37,11 @@ "nixpkgs-lib": { "locked": { "dir": "lib", - "lastModified": 1706550542, - "narHash": "sha256-UcsnCG6wx++23yeER4Hg18CXWbgNpqNXcHIo5/1Y+hc=", + "lastModified": 1709237383, + "narHash": "sha256-cy6ArO4k5qTx+l5o+0mL9f5fa86tYUX3ozE1S+Txlds=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "97b17f32362e475016f942bbdfda4a4a72a8a652", + "rev": "1536926ef5621b09bba54035ae2bb6d806d72ac8", "type": "github" }, "original": { From 8ef969afcec1645d2d9c3ab1fc82263bba968989 Mon Sep 17 00:00:00 2001 From: Pierrick Hymbert Date: Sun, 3 Mar 2024 08:48:36 +0100 Subject: [PATCH 519/812] server : init http requests thread pool with --parallel if set (#5836) --- examples/server/README.md | 2 +- examples/server/server.cpp | 10 ++++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/examples/server/README.md b/examples/server/README.md index 397ee8252..21da7a0a0 100644 --- a/examples/server/README.md +++ b/examples/server/README.md @@ -18,7 +18,7 @@ The project is under active development, and we are [looking for feedback and co - `--threads N`, `-t N`: Set the number of threads to use during generation. - `-tb N, --threads-batch N`: Set the number of threads to use during batch and prompt processing. If not specified, the number of threads will be set to the number of threads used for generation. -- `--threads-http N`: number of threads in the http server pool to process requests (default: `std::thread::hardware_concurrency()`) +- `--threads-http N`: number of threads in the http server pool to process requests (default: `max(std::thread::hardware_concurrency() - 1, --parallel N + 2)`) - `-m FNAME`, `--model FNAME`: Specify the path to the LLaMA model file (e.g., `models/7B/ggml-model.gguf`). - `-a ALIAS`, `--alias ALIAS`: Set an alias for the model. The alias will be returned in API responses. - `-c N`, `--ctx-size N`: Set the size of the prompt context. The default is 512, but LLaMA models were built with a context of 2048, which will provide better results for longer input/inference. The size may differ in other models, for example, baichuan models were build with a context of 4096. diff --git a/examples/server/server.cpp b/examples/server/server.cpp index 52daf9e7a..0ca388f47 100644 --- a/examples/server/server.cpp +++ b/examples/server/server.cpp @@ -2026,7 +2026,7 @@ static void server_print_usage(const char *argv0, const gpt_params ¶ms, printf(" -v, --verbose verbose output (default: %s)\n", server_verbose ? "enabled" : "disabled"); printf(" -t N, --threads N number of threads to use during computation (default: %d)\n", params.n_threads); printf(" -tb N, --threads-batch N number of threads to use during batch and prompt processing (default: same as --threads)\n"); - printf(" --threads-http N number of threads in the http server pool to process requests (default: hardware concurrency)\n"); + printf(" --threads-http N number of threads in the http server pool to process requests (default: max(hardware concurrency - 1, --parallel N + 2))\n"); printf(" -c N, --ctx-size N size of the prompt context (default: %d)\n", params.n_ctx); printf(" --rope-scaling {none,linear,yarn}\n"); printf(" RoPE frequency scaling method, defaults to linear unless specified by the model\n"); @@ -3468,10 +3468,12 @@ int main(int argc, char **argv) }*/ //); - if (sparams.n_threads_http > 0) { - log_data["n_threads_http"] = std::to_string(sparams.n_threads_http); - svr.new_task_queue = [&sparams] { return new httplib::ThreadPool(sparams.n_threads_http); }; + if (sparams.n_threads_http < 1) { + // +2 threads for monitoring endpoints + sparams.n_threads_http = std::max(params.n_parallel + 2, (int32_t) std::thread::hardware_concurrency() - 1); } + log_data["n_threads_http"] = std::to_string(sparams.n_threads_http); + svr.new_task_queue = [&sparams] { return new httplib::ThreadPool(sparams.n_threads_http); }; LOG_INFO("HTTP server listening", log_data); // run the HTTP server in a thread - see comment below From e6029348e86c3810d4435faee54ba822cb43e2ef Mon Sep 17 00:00:00 2001 From: Pierrick Hymbert Date: Sun, 3 Mar 2024 09:35:23 +0100 Subject: [PATCH 520/812] ci : schedule slow server tests only on Release or on demand (#5839) --- .github/workflows/server.yml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/server.yml b/.github/workflows/server.yml index 8c6312508..04e3fc0c1 100644 --- a/.github/workflows/server.yml +++ b/.github/workflows/server.yml @@ -3,6 +3,11 @@ name: Server on: workflow_dispatch: # allows manual triggering + inputs: + slow_tests: + description: 'Run slow tests' + required: true + type: boolean push: branches: - master @@ -11,7 +16,7 @@ on: types: [opened, synchronize, reopened] paths: ['.github/workflows/server.yml', '**/CMakeLists.txt', '**/Makefile', '**/*.h', '**/*.hpp', '**/*.c', '**/*.cpp', '**/*.cu', '**/*.swift', '**/*.m', 'examples/server/tests/**.*'] schedule: - - cron: '00 0 * * *' + - cron: '0 0 * * *' jobs: server: @@ -80,7 +85,7 @@ jobs: - name: Slow tests id: server_integration_tests_slow - if: github.event.schedule != '' + if: ${{ github.event.schedule != '' && matrix.build_type == 'Release' || github.event.inputs.slow_tests == 'true' }} run: | cd examples/server/tests PORT=8888 ./tests.sh --stop --no-skipped --no-capture --tags slow From de9692a7d2db66e29e5cb373c6551acc49145ccd Mon Sep 17 00:00:00 2001 From: compilade <113953597+compilade@users.noreply.github.com> Date: Sun, 3 Mar 2024 03:41:55 -0500 Subject: [PATCH 521/812] llama : fix llama_copy_state_data with fragmented KV cache (#5840) The row size of the saved states was based on kv_self.head while it should be based on llama_kv_cache_cell_max. Existing session files should still work. * llama : fix llama_kv_cache_cell_max inability to return 1 I've also changed its return type to uint32_t, because this function is always used to set the value of uint32_t variables, and because the index already has this type. * llama : fix state size calculation Some bytes in the state were unaccounted for in llama_get_state_size. Since the logits reserve so much space, it did not cause problems. --- llama.cpp | 47 ++++++++++++++++++++++++++++++----------------- 1 file changed, 30 insertions(+), 17 deletions(-) diff --git a/llama.cpp b/llama.cpp index d4c7a965b..41d0000da 100644 --- a/llama.cpp +++ b/llama.cpp @@ -2156,10 +2156,12 @@ static bool llama_kv_cache_find_slot( } // find how many cells are currently in use -static int32_t llama_kv_cache_cell_max(const struct llama_kv_cache & cache) { - for (uint32_t i = cache.size - 1; i > 0; --i) { - if (cache.cells[i].pos >= 0 && !cache.cells[i].is_empty()) { - return i + 1; +static uint32_t llama_kv_cache_cell_max(const struct llama_kv_cache & cache) { + for (uint32_t i = cache.size; i > 0; --i) { + const llama_kv_cell & cell = cache.cells[i - 1]; + + if (cell.pos >= 0 && !cell.is_empty()) { + return i; } } @@ -8178,7 +8180,7 @@ static int llama_decode_internal( // a heuristic, to avoid attending the full cache if it is not yet utilized // after enough generations, the benefit from this heuristic disappears // if we start defragmenting the cache, the benefit from this will be more important - kv_self.n = std::min((int32_t) cparams.n_ctx, std::max(32, GGML_PAD(llama_kv_cache_cell_max(kv_self), 32))); + kv_self.n = std::min(cparams.n_ctx, std::max(32u, GGML_PAD(llama_kv_cache_cell_max(kv_self), 32))); //kv_self.n = llama_kv_cache_cell_max(kv_self); //printf("kv_self.n = %5d, kv_self.used = %5d, kv_self.head = %5d\n", kv_self.n, kv_self.used, kv_self.head); @@ -12615,9 +12617,14 @@ size_t llama_get_state_size(const struct llama_context * ctx) { const size_t s_logits = ctx->logits.capacity() * sizeof(float); const size_t s_embedding_size = sizeof(size_t); const size_t s_embedding = ctx->embedding.size() * sizeof(float); - const size_t s_kv_size = sizeof(size_t); - const size_t s_kv_ntok = sizeof(int); + const size_t s_kv_buf_size = sizeof(size_t); + const size_t s_kv_head = sizeof(uint32_t); + const size_t s_kv_size = sizeof(uint32_t); + const size_t s_kv_used = sizeof(uint32_t); const size_t s_kv = ctx->kv_self.total_size(); + // TODO: assume the max is more than 1 seq_id per KV cell + const size_t s_kv_cell = sizeof(llama_pos) + sizeof(size_t) + sizeof(llama_seq_id); + const size_t s_kv_cells = ctx->kv_self.size * s_kv_cell; const size_t s_total = ( + s_rng_size @@ -12626,9 +12633,12 @@ size_t llama_get_state_size(const struct llama_context * ctx) { + s_logits + s_embedding_size + s_embedding + + s_kv_buf_size + + s_kv_head + s_kv_size - + s_kv_ntok + + s_kv_used + s_kv + + s_kv_cells ); return s_total; @@ -12728,15 +12738,13 @@ static void llama_copy_state_data_internal(struct llama_context * ctx, llama_dat { const auto & kv_self = ctx->kv_self; const auto & hparams = ctx->model.hparams; - const auto & cparams = ctx->cparams; const uint32_t n_layer = hparams.n_layer; const uint32_t n_embd_k_gqa = hparams.n_embd_k_gqa(); const uint32_t n_embd_v_gqa = hparams.n_embd_v_gqa(); - const uint32_t n_ctx = cparams.n_ctx; const size_t kv_buf_size = kv_self.total_size(); - const uint32_t kv_head = kv_self.head; + const uint32_t kv_head = llama_kv_cache_cell_max(kv_self); const uint32_t kv_size = kv_self.size; const uint32_t kv_used = kv_self.used; @@ -12756,7 +12764,7 @@ static void llama_copy_state_data_internal(struct llama_context * ctx, llama_dat // v is not contiguous, copy row by row const size_t v_row_size = ggml_row_size(kv_self.v_l[il]->type, kv_head); - const size_t v_row_stride = ggml_row_size(kv_self.v_l[il]->type, n_ctx); + const size_t v_row_stride = ggml_row_size(kv_self.v_l[il]->type, kv_size); tmp_buf.resize(v_row_size); for (int ir = 0; ir < (int) n_embd_v_gqa; ++ir) { @@ -12766,7 +12774,7 @@ static void llama_copy_state_data_internal(struct llama_context * ctx, llama_dat } } - for (uint32_t i = 0; i < kv_size; ++i) { + for (uint32_t i = 0; i < kv_head; ++i) { const auto & cell = kv_self.cells[i]; const llama_pos pos = cell.pos; @@ -12842,12 +12850,10 @@ size_t llama_set_state_data(struct llama_context * ctx, const uint8_t * src) { { const auto & kv_self = ctx->kv_self; const auto & hparams = ctx->model.hparams; - const auto & cparams = ctx->cparams; const uint32_t n_layer = hparams.n_layer; const uint32_t n_embd_k_gqa = hparams.n_embd_k_gqa(); const uint32_t n_embd_v_gqa = hparams.n_embd_v_gqa(); - const uint32_t n_ctx = cparams.n_ctx; size_t kv_buf_size; uint32_t kv_head; @@ -12870,7 +12876,7 @@ size_t llama_set_state_data(struct llama_context * ctx, const uint8_t * src) { // v is not contiguous, copy row by row const size_t v_row_size = ggml_row_size(kv_self.v_l[il]->type, kv_head); - const size_t v_row_stride = ggml_row_size(kv_self.v_l[il]->type, n_ctx); + const size_t v_row_stride = ggml_row_size(kv_self.v_l[il]->type, kv_size); for (int ir = 0; ir < (int) n_embd_v_gqa; ++ir) { ggml_backend_tensor_set(kv_self.v_l[il], inp, ir*v_row_stride, v_row_size); @@ -12879,13 +12885,15 @@ size_t llama_set_state_data(struct llama_context * ctx, const uint8_t * src) { } } + GGML_ASSERT(kv_self.size == kv_size); + ctx->kv_self.head = kv_head; ctx->kv_self.size = kv_size; ctx->kv_self.used = kv_used; ctx->kv_self.cells.resize(kv_size); - for (uint32_t i = 0; i < kv_size; ++i) { + for (uint32_t i = 0; i < kv_head; ++i) { llama_pos pos; size_t seq_id_size; @@ -12901,6 +12909,11 @@ size_t llama_set_state_data(struct llama_context * ctx, const uint8_t * src) { ctx->kv_self.cells[i].seq_id.insert(seq_id); } } + + for (uint32_t i = kv_head; i < kv_size; ++i) { + ctx->kv_self.cells[i].pos = -1; + ctx->kv_self.cells[i].seq_id.clear(); + } } const size_t nread = inp - src; From 87c2e8b2797860a06af3d6c06b8488a8ff1a09ab Mon Sep 17 00:00:00 2001 From: Nindaleth Date: Sun, 3 Mar 2024 09:43:42 +0100 Subject: [PATCH 522/812] gguf-dump : support i-quants (#5841) Co-authored-by: Black_Fox --- gguf-py/gguf/constants.py | 72 ++++++++++++++++++++++++--------------- 1 file changed, 44 insertions(+), 28 deletions(-) diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py index 5db760cb1..a62139811 100644 --- a/gguf-py/gguf/constants.py +++ b/gguf-py/gguf/constants.py @@ -604,20 +604,28 @@ class PoolingType(IntEnum): class GGMLQuantizationType(IntEnum): - F32 = 0 - F16 = 1 - Q4_0 = 2 - Q4_1 = 3 - Q5_0 = 6 - Q5_1 = 7 - Q8_0 = 8 - Q8_1 = 9 - Q2_K = 10 - Q3_K = 11 - Q4_K = 12 - Q5_K = 13 - Q6_K = 14 - Q8_K = 15 + F32 = 0 + F16 = 1 + Q4_0 = 2 + Q4_1 = 3 + Q5_0 = 6 + Q5_1 = 7 + Q8_0 = 8 + Q8_1 = 9 + Q2_K = 10 + Q3_K = 11 + Q4_K = 12 + Q5_K = 13 + Q6_K = 14 + Q8_K = 15 + IQ2_XXS = 16 + IQ2_XS = 17 + IQ3_XXS = 18 + IQ1_S = 19 + IQ4_NL = 20 + IQ3_S = 21 + IQ2_S = 22 + IQ4_XS = 23 class GGUFEndian(IntEnum): @@ -662,20 +670,28 @@ class GGUFValueType(IntEnum): QK_K = 256 # Items here are (block size, type size) GGML_QUANT_SIZES = { - GGMLQuantizationType.F32: (1, 4), - GGMLQuantizationType.F16: (1, 2), - GGMLQuantizationType.Q4_0: (32, 2 + 16), - GGMLQuantizationType.Q4_1: (32, 2 + 2 + 16), - GGMLQuantizationType.Q5_0: (32, 2 + 4 + 16), - GGMLQuantizationType.Q5_1: (32, 2 + 2 + 4 + 16), - GGMLQuantizationType.Q8_0: (32, 2 + 32), - GGMLQuantizationType.Q8_1: (32, 4 + 4 + 32), - GGMLQuantizationType.Q2_K: (256, 2 + 2 + QK_K // 16 + QK_K // 4), - GGMLQuantizationType.Q3_K: (256, 2 + QK_K // 4 + QK_K // 8 + 12), - GGMLQuantizationType.Q4_K: (256, 2 + 2 + QK_K // 2 + 12), - GGMLQuantizationType.Q5_K: (256, 2 + 2 + QK_K // 2 + QK_K // 8 + 12), - GGMLQuantizationType.Q6_K: (256, 2 + QK_K // 2 + QK_K // 4 + QK_K // 16), - GGMLQuantizationType.Q8_K: (256, 4 + QK_K + QK_K // 8), + GGMLQuantizationType.F32: (1, 4), + GGMLQuantizationType.F16: (1, 2), + GGMLQuantizationType.Q4_0: (32, 2 + 16), + GGMLQuantizationType.Q4_1: (32, 2 + 2 + 16), + GGMLQuantizationType.Q5_0: (32, 2 + 4 + 16), + GGMLQuantizationType.Q5_1: (32, 2 + 2 + 4 + 16), + GGMLQuantizationType.Q8_0: (32, 2 + 32), + GGMLQuantizationType.Q8_1: (32, 4 + 4 + 32), + GGMLQuantizationType.Q2_K: (256, 2 + 2 + QK_K // 16 + QK_K // 4), + GGMLQuantizationType.Q3_K: (256, 2 + QK_K // 4 + QK_K // 8 + 12), + GGMLQuantizationType.Q4_K: (256, 2 + 2 + QK_K // 2 + 12), + GGMLQuantizationType.Q5_K: (256, 2 + 2 + QK_K // 2 + QK_K // 8 + 12), + GGMLQuantizationType.Q6_K: (256, 2 + QK_K // 2 + QK_K // 4 + QK_K // 16), + GGMLQuantizationType.Q8_K: (256, 4 + QK_K + QK_K // 8), + GGMLQuantizationType.IQ2_XXS: (256, 2 + QK_K // 4), + GGMLQuantizationType.IQ2_XS: (256, 2 + QK_K // 4 + QK_K // 32), + GGMLQuantizationType.IQ3_XXS: (256, 2 + QK_K // 4 + QK_K // 8), + GGMLQuantizationType.IQ1_S: (256, 2 + QK_K // 8 + QK_K // 16), + GGMLQuantizationType.IQ4_NL: (32, 2 + 16), + GGMLQuantizationType.IQ3_S: (256, 2 + QK_K // 4 + QK_K // 8 + QK_K // 32 + 4), + GGMLQuantizationType.IQ2_S: (256, 2 + QK_K // 4 + QK_K // 16), + GGMLQuantizationType.IQ4_XS: (256, 2 + 2 + QK_K // 2 + QK_K // 64), } From 475df1d6cf817060028d3ff763cb8097d4ec40d6 Mon Sep 17 00:00:00 2001 From: Douglas Hanley Date: Sun, 3 Mar 2024 04:40:27 -0600 Subject: [PATCH 523/812] llama : allow for user specified embedding pooling type (#5849) * allow for user specified pooling type * llama : use enum types over int --------- Co-authored-by: Georgi Gerganov --- common/common.cpp | 13 +++++++++++++ common/common.h | 7 +++++-- convert-hf-to-gguf.py | 18 +++++++++--------- llama.cpp | 44 +++++++++++++++++++++++++++---------------- llama.h | 7 +++++-- 5 files changed, 60 insertions(+), 29 deletions(-) diff --git a/common/common.cpp b/common/common.cpp index 1c0b7c403..dbe7e9229 100644 --- a/common/common.cpp +++ b/common/common.cpp @@ -335,6 +335,16 @@ bool gpt_params_parse_ex(int argc, char ** argv, gpt_params & params) { break; } params.yarn_beta_slow = std::stof(argv[i]); + } else if (arg == "--pooling") { + if (++i >= argc) { + invalid_param = true; + break; + } + std::string value(argv[i]); + /**/ if (value == "none") { params.pooling_type = LLAMA_POOLING_TYPE_NONE; } + else if (value == "mean") { params.pooling_type = LLAMA_POOLING_TYPE_MEAN; } + else if (value == "cls") { params.pooling_type = LLAMA_POOLING_TYPE_CLS; } + else { invalid_param = true; break; } } else if (arg == "--defrag-thold" || arg == "-dt") { if (++i >= argc) { invalid_param = true; @@ -1014,6 +1024,8 @@ void gpt_print_usage(int /*argc*/, char ** argv, const gpt_params & params) { printf(" --yarn-attn-factor N YaRN: scale sqrt(t) or attention magnitude (default: 1.0)\n"); printf(" --yarn-beta-slow N YaRN: high correction dim or alpha (default: %.1f)\n", params.yarn_beta_slow); printf(" --yarn-beta-fast N YaRN: low correction dim or beta (default: %.1f)\n", params.yarn_beta_fast); + printf(" --pooling {none,mean,cls}\n"); + printf(" pooling type for embeddings, use model default if unspecified\n"); printf(" -dt N, --defrag-thold N\n"); printf(" KV cache defragmentation threshold (default: %.1f, < 0 - disabled)\n", params.defrag_thold); printf(" --ignore-eos ignore end of stream token and continue generating (implies --logit-bias 2-inf)\n"); @@ -1296,6 +1308,7 @@ struct llama_context_params llama_context_params_from_gpt_params(const gpt_param cparams.yarn_beta_fast = params.yarn_beta_fast; cparams.yarn_beta_slow = params.yarn_beta_slow; cparams.yarn_orig_ctx = params.yarn_orig_ctx; + cparams.pooling_type = params.pooling_type; cparams.defrag_thold = params.defrag_thold; cparams.offload_kqv = !params.no_kv_offload; diff --git a/common/common.h b/common/common.h index ab62bdb82..d3682b7ad 100644 --- a/common/common.h +++ b/common/common.h @@ -76,8 +76,11 @@ struct gpt_params { float yarn_beta_slow = 1.0f; // YaRN high correction dim int32_t yarn_orig_ctx = 0; // YaRN original context length float defrag_thold = -1.0f; // KV cache defragmentation threshold - int32_t rope_scaling_type = LLAMA_ROPE_SCALING_TYPE_UNSPECIFIED; - ggml_numa_strategy numa = GGML_NUMA_STRATEGY_DISABLED; + + ggml_numa_strategy numa = GGML_NUMA_STRATEGY_DISABLED; + + llama_rope_scaling_type rope_scaling_type = LLAMA_ROPE_SCALING_TYPE_UNSPECIFIED; + llama_pooling_type pooling_type = LLAMA_POOLING_TYPE_UNSPECIFIED; // pooling type for embeddings // // sampling parameters struct llama_sampling_params sparams; diff --git a/convert-hf-to-gguf.py b/convert-hf-to-gguf.py index fa9d4f22f..ffdba7444 100755 --- a/convert-hf-to-gguf.py +++ b/convert-hf-to-gguf.py @@ -1644,16 +1644,17 @@ class BertModel(Model): self.gguf_writer.add_causal_attention(False) # get pooling path - with open(self.dir_model / "modules.json", encoding="utf-8") as f: - modules = json.load(f) pooling_path = None - for mod in modules: - if mod["type"] == "sentence_transformers.models.Pooling": - pooling_path = mod["path"] - break + module_path = self.dir_model / "modules.json" + if module_path.is_file(): + with open(module_path, encoding="utf-8") as f: + modules = json.load(f) + for mod in modules: + if mod["type"] == "sentence_transformers.models.Pooling": + pooling_path = mod["path"] + break # get pooling type - pooling_type = gguf.PoolingType.NONE if pooling_path is not None: with open(self.dir_model / pooling_path / "config.json", encoding="utf-8") as f: pooling = json.load(f) @@ -1663,8 +1664,7 @@ class BertModel(Model): pooling_type = gguf.PoolingType.CLS else: raise NotImplementedError("Only MEAN and CLS pooling types supported") - - self.gguf_writer.add_pooling_type(pooling_type) + self.gguf_writer.add_pooling_type(pooling_type) def set_vocab(self): path = self.dir_model diff --git a/llama.cpp b/llama.cpp index 41d0000da..c1f015791 100644 --- a/llama.cpp +++ b/llama.cpp @@ -873,16 +873,16 @@ struct LLM_TN { // gguf helpers // -static const std::map LLAMA_ROPE_SCALING_TYPES = { +static const std::map LLAMA_ROPE_SCALING_TYPES = { { LLAMA_ROPE_SCALING_TYPE_NONE, "none" }, { LLAMA_ROPE_SCALING_TYPE_LINEAR, "linear" }, { LLAMA_ROPE_SCALING_TYPE_YARN, "yarn" }, }; -static int32_t llama_rope_scaling_type_from_string(const std::string & name) { +static llama_rope_scaling_type llama_rope_scaling_type_from_string(const std::string & name) { for (const auto & kv : LLAMA_ROPE_SCALING_TYPES) { if (kv.second == name) { - return kv.first; + return (llama_rope_scaling_type) kv.first; } } @@ -1612,7 +1612,6 @@ struct llama_hparams { float rope_freq_base_train; float rope_freq_scale_train; uint32_t n_yarn_orig_ctx; - int32_t rope_scaling_type_train; float f_clamp_kqv = 0.0f; float f_max_alibi_bias = 0.0f; @@ -1620,8 +1619,9 @@ struct llama_hparams { bool causal_attn = true; bool need_kq_pos = false; - enum llama_pooling_type pooling_type = LLAMA_POOLING_TYPE_NONE; - enum llama_rope_type rope_type = LLAMA_ROPE_TYPE_NONE; + enum llama_pooling_type pooling_type = LLAMA_POOLING_TYPE_NONE; + enum llama_rope_type rope_type = LLAMA_ROPE_TYPE_NONE; + enum llama_rope_scaling_type rope_scaling_type_train = LLAMA_ROPE_SCALING_TYPE_NONE; bool operator!=(const llama_hparams & other) const { if (this->vocab_only != other.vocab_only) return true; @@ -1670,8 +1670,8 @@ struct llama_cparams { uint32_t n_threads; // number of threads to use for generation uint32_t n_threads_batch; // number of threads to use for batch processing - float rope_freq_base; - float rope_freq_scale; + float rope_freq_base; + float rope_freq_scale; uint32_t n_yarn_orig_ctx; // These hyperparameters are not exposed in GGUF, because all @@ -1683,7 +1683,7 @@ struct llama_cparams { float defrag_thold; bool offload_kqv; - bool do_pooling; + enum llama_pooling_type pooling_type; ggml_backend_sched_eval_callback cb_eval; void * cb_eval_user_data; @@ -2933,7 +2933,11 @@ template<> bool llama_model_loader::get_key(const enum llm_kv kid, enum llama_pooling_type & result, const bool required) { uint32_t tmp; const bool found = get_key(kid, tmp, required); - result = (enum llama_pooling_type) tmp; + if (found) { + result = (enum llama_pooling_type) tmp; + } else { + result = LLAMA_POOLING_TYPE_UNSPECIFIED; + } return found; } @@ -3210,7 +3214,7 @@ static void llm_load_hparams( ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps); ml.get_key(LLM_KV_ATTENTION_CAUSAL, hparams.causal_attn); ml.get_key(LLM_KV_TOKENIZER_TOKEN_TYPE_COUNT, hparams.n_vocab_type); - ml.get_key(LLM_KV_POOLING_TYPE, hparams.pooling_type); + ml.get_key(LLM_KV_POOLING_TYPE, hparams.pooling_type, false); switch (hparams.n_layer) { case 3: @@ -5175,7 +5179,7 @@ struct llm_build_context { n_kv (worst_case ? n_ctx : kv_self.n), kv_head (worst_case ? n_ctx - n_tokens : kv_self.head), n_orig_ctx (cparams.n_yarn_orig_ctx), - pooling_type (cparams.do_pooling ? hparams.pooling_type : LLAMA_POOLING_TYPE_NONE), + pooling_type (cparams.pooling_type), rope_type (hparams.rope_type), cb (cb), buf_compute_meta (lctx.buf_compute_meta) { @@ -8015,7 +8019,7 @@ static void llama_set_inputs(llama_context & lctx, const llama_batch & batch) { } } - if (cparams.do_pooling && hparams.pooling_type == LLAMA_POOLING_TYPE_MEAN) { + if (cparams.pooling_type == LLAMA_POOLING_TYPE_MEAN) { const int64_t n_tokens = batch.n_tokens; GGML_ASSERT(ggml_backend_buffer_is_host(lctx.inp_mean->buffer)); @@ -8043,7 +8047,7 @@ static void llama_set_inputs(llama_context & lctx, const llama_batch & batch) { } } - if (cparams.do_pooling && hparams.pooling_type == LLAMA_POOLING_TYPE_CLS) { + if (cparams.pooling_type == LLAMA_POOLING_TYPE_CLS) { const int64_t n_tokens = batch.n_tokens; GGML_ASSERT(ggml_backend_buffer_is_host(lctx.inp_cls->buffer)); @@ -11846,6 +11850,7 @@ struct llama_context_params llama_context_default_params() { /*.n_threads =*/ GGML_DEFAULT_N_THREADS, // TODO: better default /*.n_threads_batch =*/ GGML_DEFAULT_N_THREADS, /*.rope_scaling_type =*/ LLAMA_ROPE_SCALING_TYPE_UNSPECIFIED, + /*.pooling_type =*/ LLAMA_POOLING_TYPE_UNSPECIFIED, /*.rope_freq_base =*/ 0.0f, /*.rope_freq_scale =*/ 0.0f, /*.yarn_ext_factor =*/ -1.0f, @@ -11861,7 +11866,6 @@ struct llama_context_params llama_context_default_params() { /*.logits_all =*/ false, /*.embedding =*/ false, /*.offload_kqv =*/ true, - /*.do_pooling =*/ true, /*.abort_callback =*/ nullptr, /*.abort_callback_data =*/ nullptr, }; @@ -12012,7 +12016,7 @@ struct llama_context * llama_new_context_with_model( cparams.yarn_beta_slow = params.yarn_beta_slow; cparams.defrag_thold = params.defrag_thold; cparams.offload_kqv = params.offload_kqv; - cparams.do_pooling = params.do_pooling; + cparams.pooling_type = params.pooling_type; cparams.n_ctx = params.n_ctx == 0 ? hparams.n_ctx_train : params.n_ctx; cparams.rope_freq_base = params.rope_freq_base == 0.0f ? hparams.rope_freq_base_train : params.rope_freq_base; @@ -12038,6 +12042,14 @@ struct llama_context * llama_new_context_with_model( cparams.yarn_ext_factor = rope_scaling_type == LLAMA_ROPE_SCALING_TYPE_YARN ? 1.0f : 0.0f; } + if (cparams.pooling_type == LLAMA_POOLING_TYPE_UNSPECIFIED) { + if (hparams.pooling_type == LLAMA_POOLING_TYPE_UNSPECIFIED) { + cparams.pooling_type = LLAMA_POOLING_TYPE_NONE; + } else { + cparams.pooling_type = hparams.pooling_type; + } + } + if (params.seed == LLAMA_DEFAULT_SEED) { params.seed = time(NULL); } diff --git a/llama.h b/llama.h index 6406b5270..70da4cb3f 100644 --- a/llama.h +++ b/llama.h @@ -129,6 +129,7 @@ extern "C" { }; enum llama_pooling_type { + LLAMA_POOLING_TYPE_UNSPECIFIED = -1, LLAMA_POOLING_TYPE_NONE = 0, LLAMA_POOLING_TYPE_MEAN = 1, LLAMA_POOLING_TYPE_CLS = 2, @@ -236,7 +237,10 @@ extern "C" { uint32_t n_batch; // prompt processing maximum batch size uint32_t n_threads; // number of threads to use for generation uint32_t n_threads_batch; // number of threads to use for batch processing - int32_t rope_scaling_type; // RoPE scaling type, from `enum llama_rope_scaling_type` + + enum llama_rope_scaling_type rope_scaling_type; // RoPE scaling type, from `enum llama_rope_scaling_type` + enum llama_pooling_type pooling_type; // whether to pool (sum) embedding results by sequence id + // (ignored if no pooling layer) // ref: https://github.com/ggerganov/llama.cpp/pull/2054 float rope_freq_base; // RoPE base frequency, 0 = from model @@ -258,7 +262,6 @@ extern "C" { bool logits_all; // the llama_decode() call computes all logits, not just the last one (DEPRECATED - set llama_batch.logits instead) bool embedding; // embedding mode only bool offload_kqv; // whether to offload the KQV ops (including the KV cache) to GPU - bool do_pooling; // whether to pool (sum) embedding results by sequence id (ignored if no pooling layer) // Abort callback // if it returns true, execution of llama_decode() will be aborted From 231ae28f078c3148d097b301f2145f1e3e816cc1 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Sun, 3 Mar 2024 12:44:03 +0200 Subject: [PATCH 524/812] readme : add API changes section --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 939646753..45c5d06f3 100644 --- a/README.md +++ b/README.md @@ -8,6 +8,10 @@ Inference of Meta's [LLaMA](https://arxiv.org/abs/2302.13971) model (and others) in pure C/C++ +### Recent API changes + +- [2024 Mar 3] `struct llama_context_params` https://github.com/ggerganov/llama.cpp/pull/5849 + ### Hot topics - The `api_like_OAI.py` script has been removed - use `server` instead ([#5766](https://github.com/ggerganov/llama.cpp/issues/5766#issuecomment-1969037761)) From 67be2ce1015d070b3b2cd488bcb041eefb61de72 Mon Sep 17 00:00:00 2001 From: slaren Date: Sun, 3 Mar 2024 14:26:18 +0100 Subject: [PATCH 525/812] cuda : fix data race in soft max (#5853) --- ggml-cuda.cu | 1 + 1 file changed, 1 insertion(+) diff --git a/ggml-cuda.cu b/ggml-cuda.cu index 7ed97430f..04c6cb1b8 100644 --- a/ggml-cuda.cu +++ b/ggml-cuda.cu @@ -6904,6 +6904,7 @@ static __global__ void soft_max_f32(const float * x, const float * mask, const f // find the sum of exps in the block tmp = warp_reduce_sum(tmp); if (block_size > WARP_SIZE) { + __syncthreads(); if (warp_id == 0) { buf_iw[lane_id] = 0.0f; } From 5a51cc1bb4592f0d71f9af89cd08b11a066ba447 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?DAN=E2=84=A2?= Date: Mon, 4 Mar 2024 02:57:20 -0500 Subject: [PATCH 526/812] main : support special tokens as reverse/anti prompt (#5847) * Support special tokens as reverse/anti prompt. * Tokenize antiprompts only once. * main : minor --------- Co-authored-by: Georgi Gerganov --- examples/main/main.cpp | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/examples/main/main.cpp b/examples/main/main.cpp index 34e84d0d4..47059e582 100644 --- a/examples/main/main.cpp +++ b/examples/main/main.cpp @@ -511,6 +511,14 @@ int main(int argc, char ** argv) { std::vector embd; std::vector embd_guidance; + // tokenized antiprompts + std::vector> antiprompt_ids; + + antiprompt_ids.reserve(params.antiprompt.size()); + for (const std::string & antiprompt : params.antiprompt) { + antiprompt_ids.emplace_back(::llama_tokenize(ctx, antiprompt, false, true)); + } + struct llama_sampling_context * ctx_sampling = llama_sampling_init(sparams); while ((n_remain != 0 && !is_antiprompt) || params.interactive) { @@ -769,6 +777,18 @@ int main(int argc, char ** argv) { } } + // check for reverse prompt using special tokens + llama_token last_token = llama_sampling_last(ctx_sampling); + for (std::vector ids : antiprompt_ids) { + if (ids.size() == 1 && last_token == ids[0]) { + if (params.interactive) { + is_interacting = true; + } + is_antiprompt = true; + break; + } + } + if (is_antiprompt) { LOG("found antiprompt: %s\n", last_output.c_str()); } From 82f3e668adafba647de703f835991e91a96b5ac4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?DAN=E2=84=A2?= Date: Mon, 4 Mar 2024 03:08:19 -0500 Subject: [PATCH 527/812] common : use LLAMA_DEFAULT_SEED (#5855) --- common/common.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/common.h b/common/common.h index d3682b7ad..b2868833b 100644 --- a/common/common.h +++ b/common/common.h @@ -43,7 +43,7 @@ extern char const *LLAMA_BUILD_TARGET; int32_t get_num_physical_cores(); struct gpt_params { - uint32_t seed = -1; // RNG seed + uint32_t seed = LLAMA_DEFAULT_SEED; // RNG seed int32_t n_threads = get_num_physical_cores(); int32_t n_threads_draft = -1; From 7d43c585dc174bb586775c22c15e5db9242b5b4b Mon Sep 17 00:00:00 2001 From: leejet Date: Sun, 3 Mar 2024 20:23:52 +0800 Subject: [PATCH 528/812] add some new ops, fix some operators and add batch operations to certain operators. (ggml/747) * cuda: fix group_norm * cuda: add batch inference support for ggml_pad/ggml_upscale * add ggml_arrange * add ggml_timestep_embedding * update ggml_arange/ggml_timestep_embedding tests * cuda: fix im2col * add ggml_arange/ggml_timestep_embbeding support for metal backend * fix some bugs * fix some bugs * Update ggml.h Co-authored-by: Georgi Gerganov * Update ggml-cuda.cu Co-authored-by: Georgi Gerganov * Update ggml-metal.m Co-authored-by: Georgi Gerganov * Update ggml-metal.m Co-authored-by: Georgi Gerganov * Update ggml-metal.metal Co-authored-by: Georgi Gerganov * modify according to the review comments * ggml : fix compile warnings + code style * ggml : normalize compute_forward calls + fix seg fault in debug * minor --------- Co-authored-by: Georgi Gerganov Co-authored-by: slaren --- ggml-cuda.cu | 227 ++++++++++++++++++++++++++++++------- ggml-metal.m | 62 +++++++++- ggml-metal.metal | 43 +++++++ ggml.c | 207 +++++++++++++++++++++++++++++++-- ggml.h | 17 +++ tests/test-backend-ops.cpp | 46 ++++++++ 6 files changed, 550 insertions(+), 52 deletions(-) diff --git a/ggml-cuda.cu b/ggml-cuda.cu index 04c6cb1b8..7d027a30a 100644 --- a/ggml-cuda.cu +++ b/ggml-cuda.cu @@ -616,6 +616,8 @@ static_assert(sizeof(block_iq4_xs) == sizeof(ggml_fp16_t) + sizeof(uint16_t) + Q #define CUDA_UPSCALE_BLOCK_SIZE 256 #define CUDA_CONCAT_BLOCK_SIZE 256 #define CUDA_PAD_BLOCK_SIZE 256 +#define CUDA_ARANGE_BLOCK_SIZE 256 +#define CUDA_TIMESTEP_EMBEDDING_BLOCK_SIZE 256 #define CUDA_ACC_BLOCK_SIZE 256 #define CUDA_IM2COL_BLOCK_SIZE 256 #define CUDA_POOL2D_BLOCK_SIZE 256 @@ -990,17 +992,21 @@ static __global__ void concat_f32(const float * x,const float * y, float * dst, nidx + blockIdx.y * ne0 + blockIdx.z * ne0 * gridDim.y; - dst[offset_dst] = x[offset_src]; + dst[offset_dst] = x[offset_src]; } else { int offset_src = nidx + blockIdx.y * ne0 + (blockIdx.z - ne02) * ne0 * gridDim.y; - dst[offset_dst] = y[offset_src]; + dst[offset_dst] = y[offset_src]; } } -static __global__ void upscale_f32(const float * x, float * dst, const int ne00, const int nb02, const int scale_factor) { +static __global__ void upscale_f32(const float * x, float * dst, const int ne00, const int ne00xne01, const int scale_factor) { + // blockIdx.z: idx of ne02*ne03 + // blockIdx.y: idx of ne01*scale_factor, aka ne1 + // blockIDx.x: idx of ne00*scale_factor / BLOCK_SIZE + // ne00xne01: ne00 * ne01 int ne0 = ne00 * scale_factor; int nidx = threadIdx.x + blockIdx.x * blockDim.x; if (nidx >= ne0) { @@ -1012,7 +1018,7 @@ static __global__ void upscale_f32(const float * x, float * dst, const int ne00, int offset_src = i00 + i01 * ne00 + - blockIdx.z * nb02; + blockIdx.z * ne00xne01; int offset_dst = nidx + blockIdx.y * ne0 + @@ -1020,7 +1026,10 @@ static __global__ void upscale_f32(const float * x, float * dst, const int ne00, dst[offset_dst] = x[offset_src]; } -static __global__ void pad_f32(const float * x, float * dst, const int ne0, const int ne00, const int ne01, const int ne02) { +static __global__ void pad_f32(const float * x, float * dst, const int ne0, const int ne00, const int ne01, const int ne02, const int ne03) { + // blockIdx.z: idx of ne2*ne3, aka ne02*ne03 + // blockIdx.y: idx of ne1 + // blockIDx.x: idx of ne0 / BLOCK_SIZE int nidx = threadIdx.x + blockIdx.x * blockDim.x; if (nidx >= ne0) { return; @@ -1031,19 +1040,53 @@ static __global__ void pad_f32(const float * x, float * dst, const int ne0, cons nidx + blockIdx.y * ne0 + blockIdx.z * ne0 * gridDim.y; - if (nidx < ne00 && blockIdx.y < ne01 && blockIdx.z < ne02) { + if (nidx < ne00 && blockIdx.y < ne01 && blockIdx.z < ne02*ne03) { int offset_src = nidx + blockIdx.y * ne00 + blockIdx.z * ne00 * ne01; - dst[offset_dst] = x[offset_src]; + dst[offset_dst] = x[offset_src]; } else { dst[offset_dst] = 0.0f; } } +static __global__ void arange_f32(float * dst, const int ne0, const float start, const float step) { + // blockIDx.x: idx of ne0 / BLOCK_SIZE + int nidx = threadIdx.x + blockIdx.x * blockDim.x; + if (nidx >= ne0) { + return; + } + dst[nidx] = start + step * nidx; +} + +static __global__ void timestep_embedding_f32(const float * timesteps, float * dst, const int nb1, const int dim, const int max_period) { + // blockIDx.y: idx of timesteps->ne[0] + // blockIDx.x: idx of ((dim + 1) / 2) / BLOCK_SIZE + int i = blockIdx.y; + int j = threadIdx.x + blockIdx.x * blockDim.x; + float * embed_data = (float *)((char *)dst + i*nb1); + + if (dim % 2 != 0 && j == ((dim + 1) / 2)) { + embed_data[dim] = 0.f; + } + + int half = dim / 2; + if (j >= half) { + return; + } + + float timestep = timesteps[i]; + float freq = (float)expf(-logf(max_period) * j / half); + float arg = timestep * freq; + embed_data[j] = cosf(arg); + embed_data[j + half] = sinf(arg); +} + template static __global__ void group_norm_f32(const float * x, float * dst, const int group_size, const int ne_elements, const float eps) { + // blockIdx.x: num_groups idx + // threadIdx.x: block_size idx int start = blockIdx.x * group_size; int end = start + group_size; @@ -6448,7 +6491,7 @@ static __global__ void cpy_f32_f16(const char * cx, char * cdst, const int ne, const int ne00, const int ne01, const int ne02, const int nb00, const int nb01, const int nb02, const int nb03, const int ne10, const int ne11, const int ne12, const int nb10, const int nb11, const int nb12, const int nb13) { - const int i = blockDim.x*blockIdx.x + threadIdx.x; + const int64_t i = blockDim.x*blockIdx.x + threadIdx.x; if (i >= ne) { return; @@ -6456,17 +6499,17 @@ static __global__ void cpy_f32_f16(const char * cx, char * cdst, const int ne, // determine indices i03/i13, i02/i12, i01/i11, i00/i10 as a function of index i of flattened tensor // then combine those indices with the corresponding byte offsets to get the total offsets - const int i03 = i/(ne00 * ne01 * ne02); - const int i02 = (i - i03*ne00*ne01*ne02 )/ (ne00*ne01); - const int i01 = (i - i03*ne00*ne01*ne02 - i02*ne01*ne00) / ne00; - const int i00 = i - i03*ne00*ne01*ne02 - i02*ne01*ne00 - i01*ne00; - const int x_offset = i00*nb00 + i01*nb01 + i02*nb02 + i03 * nb03; + const int64_t i03 = i/(ne00 * ne01 * ne02); + const int64_t i02 = (i - i03*ne00*ne01*ne02 )/ (ne00*ne01); + const int64_t i01 = (i - i03*ne00*ne01*ne02 - i02*ne01*ne00) / ne00; + const int64_t i00 = i - i03*ne00*ne01*ne02 - i02*ne01*ne00 - i01*ne00; + const int64_t x_offset = i00*nb00 + i01*nb01 + i02*nb02 + i03 * nb03; - const int i13 = i/(ne10 * ne11 * ne12); - const int i12 = (i - i13*ne10*ne11*ne12) / (ne10*ne11); - const int i11 = (i - i13*ne10*ne11*ne12 - i12*ne10*ne11) / ne10; - const int i10 = i - i13*ne10*ne11*ne12 - i12*ne10*ne11 - i11*ne10; - const int dst_offset = i10*nb10 + i11*nb11 + i12*nb12 + i13 * nb13; + const int64_t i13 = i/(ne10 * ne11 * ne12); + const int64_t i12 = (i - i13*ne10*ne11*ne12) / (ne10*ne11); + const int64_t i11 = (i - i13*ne10*ne11*ne12 - i12*ne10*ne11) / ne10; + const int64_t i10 = i - i13*ne10*ne11*ne12 - i12*ne10*ne11 - i11*ne10; + const int64_t dst_offset = i10*nb10 + i11*nb11 + i12*nb12 + i13 * nb13; cpy_1(cx + x_offset, cdst + dst_offset); } @@ -6956,23 +6999,23 @@ static __global__ void clamp_f32(const float * x, float * dst, const float min, template static __global__ void im2col_kernel( - const float * x, T * dst, int batch_offset, - int offset_delta, int IC, int IW, int IH, int OH, int OW, int KW, int KH, int pelements, int CHW, + const float * x, T * dst, int64_t batch_offset, + int64_t offset_delta, int64_t IC, int64_t IW, int64_t IH, int64_t OH, int64_t OW, int64_t KW, int64_t KH, int64_t pelements, int64_t CHW, int s0, int s1, int p0, int p1, int d0, int d1) { - const int i = threadIdx.x + blockIdx.x * blockDim.x; + const int64_t i = threadIdx.x + blockIdx.x * blockDim.x; if (i >= pelements) { return; } - const int ksize = OW * (KH > 1 ? KW : 1); - const int kx = i / ksize; - const int kd = kx * ksize; - const int ky = (i - kd) / OW; - const int ix = i % OW; + const int64_t ksize = OW * (KH > 1 ? KW : 1); + const int64_t kx = i / ksize; + const int64_t kd = kx * ksize; + const int64_t ky = (i - kd) / OW; + const int64_t ix = i % OW; - const int oh = blockIdx.y; - const int batch = blockIdx.z / IC; - const int ic = blockIdx.z % IC; + const int64_t oh = blockIdx.y; + const int64_t batch = blockIdx.z / IC; + const int64_t ic = blockIdx.z % IC; const int64_t iiw = ix * s0 + kx * d0 - p0; const int64_t iih = oh * s1 + ky * d1 - p1; @@ -7298,19 +7341,33 @@ static void concat_f32_cuda(const float * x, const float * y, float * dst, const concat_f32<<>>(x, y, dst, ne0, ne02); } -static void upscale_f32_cuda(const float * x, float * dst, const int ne00, const int ne01, const int ne02, const int scale_factor, cudaStream_t stream) { +static void upscale_f32_cuda(const float * x, float * dst, const int ne00, const int ne01, const int ne02, const int ne03, + const int scale_factor, cudaStream_t stream) { int ne0 = (ne00 * scale_factor); int num_blocks = (ne0 + CUDA_UPSCALE_BLOCK_SIZE - 1) / CUDA_UPSCALE_BLOCK_SIZE; - dim3 gridDim(num_blocks, (ne01 * scale_factor), ne02); + dim3 gridDim(num_blocks, (ne01 * scale_factor), ne02*ne03); upscale_f32<<>>(x, dst, ne00, ne00 * ne01, scale_factor); } static void pad_f32_cuda(const float * x, float * dst, - const int ne00, const int ne01, const int ne02, - const int ne0, const int ne1, const int ne2, cudaStream_t stream) { + const int ne00, const int ne01, const int ne02, const int ne03, + const int ne0, const int ne1, const int ne2, const int ne3, cudaStream_t stream) { int num_blocks = (ne0 + CUDA_PAD_BLOCK_SIZE - 1) / CUDA_PAD_BLOCK_SIZE; - dim3 gridDim(num_blocks, ne1, ne2); - pad_f32<<>>(x, dst, ne0, ne00, ne01, ne02); + dim3 gridDim(num_blocks, ne1, ne2*ne3); + pad_f32<<>>(x, dst, ne0, ne00, ne01, ne02, ne03); +} + +static void arange_f32_cuda(float * dst, const int ne0, const float start, const float step, cudaStream_t stream) { + int num_blocks = (ne0 + CUDA_ARANGE_BLOCK_SIZE - 1) / CUDA_ARANGE_BLOCK_SIZE; + arange_f32<<>>(dst, ne0, start, step); +} + +static void timestep_embedding_f32_cuda(const float * x, float * dst, const int ne00, const int nb1, + const int dim, const int max_period, cudaStream_t stream) { + int half_ceil = (dim + 1) / 2; + int num_blocks = (half_ceil + CUDA_TIMESTEP_EMBEDDING_BLOCK_SIZE - 1) / CUDA_TIMESTEP_EMBEDDING_BLOCK_SIZE; + dim3 gridDim(num_blocks, ne00, 1); + timestep_embedding_f32<<>>(x, dst, nb1, dim, max_period); } static void rms_norm_f32_cuda(const float * x, float * dst, const int ncols, const int nrows, const float eps, cudaStream_t stream) { @@ -8443,8 +8500,8 @@ static void soft_max_f32_cuda(const float * x, const float * mask, const float * template static void im2col_cuda(const float* x, T* dst, - int IW, int IH, int OW, int OH, int KW, int KH, int IC, - int batch, int batch_offset, int offset_delta, + int64_t IW, int64_t IH, int64_t OW, int64_t OH, int64_t KW, int64_t KH, int64_t IC, + int64_t batch, int64_t batch_offset, int64_t offset_delta, int s0,int s1,int p0,int p1,int d0,int d1, cudaStream_t stream) { const int parallel_elements = OW * KW * KH; const int num_blocks = (parallel_elements + CUDA_IM2COL_BLOCK_SIZE - 1) / CUDA_IM2COL_BLOCK_SIZE; @@ -9123,7 +9180,7 @@ static void ggml_cuda_op_group_norm( int num_groups = dst->op_params[0]; int group_size = src0->ne[0] * src0->ne[1] * ((src0->ne[2] + num_groups - 1) / num_groups); - group_norm_f32_cuda(src0_dd, dst_dd, num_groups, group_size, src0->ne[0] * src0->ne[1] * src0->ne[2], main_stream); + group_norm_f32_cuda(src0_dd, dst_dd, num_groups * src0->ne[3], group_size, ggml_nelements(src0), main_stream); (void) src1; (void) dst; @@ -9156,7 +9213,7 @@ static void ggml_cuda_op_upscale( const int scale_factor = dst->op_params[0]; - upscale_f32_cuda(src0_dd, dst_dd, src0->ne[0], src0->ne[1], src0->ne[2], scale_factor, main_stream); + upscale_f32_cuda(src0_dd, dst_dd, src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3], scale_factor, main_stream); (void) src1; (void) dst; @@ -9172,8 +9229,49 @@ static void ggml_cuda_op_pad( GGML_ASSERT(src0->ne[3] == 1 && dst->ne[3] == 1); // just 3D tensors pad_f32_cuda(src0_dd, dst_dd, - src0->ne[0], src0->ne[1], src0->ne[2], - dst->ne[0], dst->ne[1], dst->ne[2], main_stream); + src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3], + dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], main_stream); + + (void) src1; + (void) dst; + (void) src1_dd; +} + +static void ggml_cuda_op_arange( + const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, + const float * src0_dd, const float * src1_dd, float * dst_dd, cudaStream_t main_stream) { + + GGML_ASSERT(dst->type == GGML_TYPE_F32); + + float start; + float stop; + float step; + memcpy(&start, (float *)dst->op_params + 0, sizeof(float)); + memcpy(&stop, (float *)dst->op_params + 1, sizeof(float)); + memcpy(&step, (float *)dst->op_params + 2, sizeof(float)); + + int64_t steps = (int64_t)ceil((stop - start) / step); + GGML_ASSERT(ggml_nelements(dst) == steps); + + arange_f32_cuda(dst_dd, dst->ne[0], start, step, main_stream); + + (void) src0; + (void) src1; + (void) src0_dd; + (void) src1_dd; +} + +static void ggml_cuda_op_timestep_embedding( + const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, + const float * src0_dd, const float * src1_dd, float * dst_dd, cudaStream_t main_stream) { + + GGML_ASSERT(src0->type == GGML_TYPE_F32); + GGML_ASSERT(dst->type == GGML_TYPE_F32); + + const int dim = dst->op_params[0]; + const int max_period = dst->op_params[1]; + + timestep_embedding_f32_cuda(src0_dd, dst_dd, src0->ne[0], dst->nb[1], dim, max_period, main_stream); (void) src1; (void) dst; @@ -10458,6 +10556,45 @@ static void ggml_cuda_pad(const ggml_tensor * src0, const ggml_tensor * src1, gg ggml_cuda_op_flatten(src0, src1, dst, ggml_cuda_op_pad); } +static void ggml_cuda_arange(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + ggml_tensor_extra_gpu * dst_extra = (ggml_tensor_extra_gpu *) dst->extra; + + const bool dst_on_device = dst->backend == GGML_BACKEND_TYPE_GPU; + + // dd = data device + float * src0_ddf = nullptr; + float * src1_ddf = nullptr; + float * dst_ddf = nullptr; + + cuda_pool_alloc dst_f; + + ggml_cuda_set_device(g_main_device); + cudaStream_t main_stream = g_cudaStreams[g_main_device][0]; + + if (dst_on_device) { + dst_ddf = (float *) dst_extra->data_device[g_main_device]; + } else { + dst_ddf = dst_f.alloc(ggml_nelements(dst)); + } + + // do the computation + ggml_cuda_op_arange(src0, src1, dst, src0_ddf, src1_ddf, dst_ddf, main_stream); + CUDA_CHECK(cudaGetLastError()); + + // copy dst to host if necessary + if (!dst_on_device) { + CUDA_CHECK(cudaMemcpyAsync(dst->data, dst_ddf, ggml_nbytes(dst), cudaMemcpyDeviceToHost, main_stream)); + } + + if (dst->backend == GGML_BACKEND_TYPE_CPU) { + CUDA_CHECK(cudaDeviceSynchronize()); + } +} + +static void ggml_cuda_timestep_embedding(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + ggml_cuda_op_flatten(src0, src1, dst, ggml_cuda_op_timestep_embedding); +} + static void ggml_cuda_rms_norm(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { ggml_cuda_op_flatten(src0, src1, dst, ggml_cuda_op_rms_norm); } @@ -11358,6 +11495,12 @@ GGML_CALL bool ggml_cuda_compute_forward(struct ggml_compute_params * params, st case GGML_OP_PAD: func = ggml_cuda_pad; break; + case GGML_OP_ARANGE: + func = ggml_cuda_arange; + break; + case GGML_OP_TIMESTEP_EMBEDDING: + func = ggml_cuda_timestep_embedding; + break; case GGML_OP_LEAKY_RELU: func = ggml_cuda_leaky_relu; break; @@ -12253,6 +12396,8 @@ GGML_CALL static bool ggml_backend_cuda_supports_op(ggml_backend_t backend, cons case GGML_OP_GROUP_NORM: case GGML_OP_UPSCALE: case GGML_OP_PAD: + case GGML_OP_ARANGE: + case GGML_OP_TIMESTEP_EMBEDDING: case GGML_OP_LEAKY_RELU: return true; default: diff --git a/ggml-metal.m b/ggml-metal.m index 71fcca560..6b5a8fdf5 100644 --- a/ggml-metal.m +++ b/ggml-metal.m @@ -163,6 +163,8 @@ enum ggml_metal_kernel_type { GGML_METAL_KERNEL_TYPE_IM2COL_F32, GGML_METAL_KERNEL_TYPE_UPSCALE_F32, GGML_METAL_KERNEL_TYPE_PAD_F32, + GGML_METAL_KERNEL_TYPE_ARANGE_F32, + GGML_METAL_KERNEL_TYPE_TIMESTEP_EMBEDDING_F32, GGML_METAL_KERNEL_TYPE_ARGSORT_F32_I32_ASC, GGML_METAL_KERNEL_TYPE_ARGSORT_F32_I32_DESC, GGML_METAL_KERNEL_TYPE_LEAKY_RELU_F32, @@ -569,6 +571,8 @@ static struct ggml_metal_context * ggml_metal_init(int n_cb) { GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_IM2COL_F32, im2col_f32, true); GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_UPSCALE_F32, upscale_f32, true); GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_PAD_F32, pad_f32, true); + GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_TIMESTEP_EMBEDDING_F32, timestep_embedding_f32, true); + GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_ARANGE_F32, arange_f32, true); GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_ARGSORT_F32_I32_ASC, argsort_f32_i32_asc, true); GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_ARGSORT_F32_I32_DESC, argsort_f32_i32_desc, true); GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_LEAKY_RELU_F32, leaky_relu_f32, true); @@ -697,6 +701,8 @@ static bool ggml_metal_supports_op(const struct ggml_metal_context * ctx, const return false; case GGML_OP_UPSCALE: case GGML_OP_PAD: + case GGML_OP_ARANGE: + case GGML_OP_TIMESTEP_EMBEDDING: case GGML_OP_ARGSORT: case GGML_OP_LEAKY_RELU: return true; @@ -1091,7 +1097,8 @@ static bool ggml_metal_graph_compute( { GGML_ASSERT(ggml_is_contiguous(src0)); - const float scale = *(const float *) dst->op_params; + float scale; + memcpy(&scale, dst->op_params, sizeof(scale)); int64_t n = ggml_nelements(dst); @@ -1250,11 +1257,15 @@ static bool ggml_metal_graph_compute( pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_SOFT_MAX].pipeline; } - const float scale = ((float *) dst->op_params)[0]; - const float max_bias = ((float *) dst->op_params)[1]; + float scale; + float max_bias; + + memcpy(&scale, ((int32_t *) dst->op_params) + 0, sizeof(scale)); + memcpy(&max_bias, ((int32_t *) dst->op_params) + 1, sizeof(max_bias)); const int64_t nrows_x = ggml_nrows(src0); const int64_t nrows_y = src0->ne[1]; + const uint32_t n_head_kv = nrows_x/nrows_y; const uint32_t n_head_log2 = 1u << (uint32_t) floorf(log2f((float) n_head_kv)); @@ -2086,6 +2097,7 @@ static bool ggml_metal_graph_compute( //const int n_past = ((int32_t *) dst->op_params)[0]; const int n_head = ((int32_t *) dst->op_params)[1]; + float max_bias; memcpy(&max_bias, (int32_t *) dst->op_params + 2, sizeof(float)); @@ -2300,6 +2312,50 @@ static bool ggml_metal_graph_compute( [encoder dispatchThreadgroups:MTLSizeMake(ne1, ne2, ne3) threadsPerThreadgroup:MTLSizeMake(nth, 1, 1)]; } break; + case GGML_OP_ARANGE: + { + GGML_ASSERT(dst->type == GGML_TYPE_F32); + + float start; + float step; + + memcpy(&start, ((int32_t *) dst->op_params) + 0, sizeof(float)); + memcpy(&step, ((int32_t *) dst->op_params) + 2, sizeof(float)); + + id pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_ARANGE_F32].pipeline; + + [encoder setComputePipelineState:pipeline]; + [encoder setBuffer:id_dst offset:offs_dst atIndex:0]; + [encoder setBytes:&ne0 length:sizeof(ne0) atIndex:1]; + [encoder setBytes:&start length:sizeof(start) atIndex:2]; + [encoder setBytes:&step length:sizeof(step) atIndex:3]; + + const int nth = MIN(1024, ne0); + + [encoder dispatchThreadgroups:MTLSizeMake(1, 1, 1) threadsPerThreadgroup:MTLSizeMake(nth, 1, 1)]; + } break; + case GGML_OP_TIMESTEP_EMBEDDING: + { + GGML_ASSERT(src0->type == GGML_TYPE_F32); + + const int dim = dst->op_params[0]; + const int max_period = dst->op_params[1]; + + const int half = dim / 2; + + id pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_TIMESTEP_EMBEDDING_F32].pipeline; + + [encoder setComputePipelineState:pipeline]; + [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0]; + [encoder setBuffer:id_dst offset:offs_dst atIndex:1]; + [encoder setBytes:&nb1 length:sizeof(nb1) atIndex:2]; + [encoder setBytes:&dim length:sizeof(dim) atIndex:3]; + [encoder setBytes:&max_period length:sizeof(max_period) atIndex:4]; + + const int nth = MIN(1024, half); + + [encoder dispatchThreadgroups:MTLSizeMake(ne00, 1, 1) threadsPerThreadgroup:MTLSizeMake(nth, 1, 1)]; + } break; case GGML_OP_ARGSORT: { GGML_ASSERT(src0->type == GGML_TYPE_F32); diff --git a/ggml-metal.metal b/ggml-metal.metal index 8b9488437..a65d12641 100644 --- a/ggml-metal.metal +++ b/ggml-metal.metal @@ -1959,6 +1959,49 @@ kernel void kernel_pad_f32( } } +kernel void kernel_arange_f32( + device char * dst, + constant int64_t & ne0, + constant float & start, + constant float & step, + uint3 tgpig[[threadgroup_position_in_grid]], + uint3 tpitg[[thread_position_in_threadgroup]], + uint3 ntg[[threads_per_threadgroup]]) { + + device float * dst_ptr = (device float *) dst; + + for (int i0 = tpitg.x; i0 < ne0; i0 += ntg.x) { + dst_ptr[i0] = start + step * i0; + } +} + +kernel void kernel_timestep_embedding_f32( + device const char * src0, + device char * dst, + constant uint64_t & nb1, + constant int & dim, + constant int & max_period, + uint3 tgpig[[threadgroup_position_in_grid]], + uint3 tpitg[[thread_position_in_threadgroup]], + uint3 ntg[[threads_per_threadgroup]]) { + + int i = tgpig.x; + device float * embed_data = (device float *)(dst + i*nb1); + + int half_ = dim / 2; + for (int j = tpitg.x; j < half_; j += ntg.x) { + float timestep = ((device float *)src0)[i]; + float freq = (float)exp(-log((float)max_period) * j / half_); + float arg = timestep * freq; + embed_data[j ] = cos(arg); + embed_data[j + half_] = sin(arg); + } + + if (dim % 2 != 0 && tpitg.x == 0) { + embed_data[dim] = 0.f; + } +} + // bitonic sort implementation following the CUDA kernels as reference typedef void (argsort_t)( device const float * x, diff --git a/ggml.c b/ggml.c index f29b9f13f..870e41614 100644 --- a/ggml.c +++ b/ggml.c @@ -1822,6 +1822,8 @@ static const char * GGML_OP_NAME[GGML_OP_COUNT] = { "POOL_2D", "UPSCALE", "PAD", + "ARANGE", + "TIMESTEP_EMBEDDING", "ARGSORT", "LEAKY_RELU", @@ -1850,7 +1852,7 @@ static const char * GGML_OP_NAME[GGML_OP_COUNT] = { "CROSS_ENTROPY_LOSS_BACK", }; -static_assert(GGML_OP_COUNT == 72, "GGML_OP_COUNT != 72"); +static_assert(GGML_OP_COUNT == 74, "GGML_OP_COUNT != 74"); static const char * GGML_OP_SYMBOL[GGML_OP_COUNT] = { "none", @@ -1908,6 +1910,8 @@ static const char * GGML_OP_SYMBOL[GGML_OP_COUNT] = { "pool_2d(x)", "upscale(x)", "pad(x)", + "arange(start, stop, step)", + "timestep_embedding(timesteps, dim, max_period)", "argsort(x)", "leaky_relu(x)", @@ -1936,7 +1940,7 @@ static const char * GGML_OP_SYMBOL[GGML_OP_COUNT] = { "cross_entropy_loss_back(x,y)", }; -static_assert(GGML_OP_COUNT == 72, "GGML_OP_COUNT != 72"); +static_assert(GGML_OP_COUNT == 74, "GGML_OP_COUNT != 74"); static_assert(GGML_OP_POOL_COUNT == 2, "GGML_OP_POOL_COUNT != 2"); @@ -2895,11 +2899,21 @@ static int32_t ggml_get_op_params_i32(const struct ggml_tensor * tensor, uint32_ return ((const int32_t *)(tensor->op_params))[i]; } +static float ggml_get_op_params_f32(const struct ggml_tensor * tensor, uint32_t i) { + assert(i < GGML_MAX_OP_PARAMS / sizeof(float)); + return ((const float *)(tensor->op_params))[i]; +} + static void ggml_set_op_params_i32(struct ggml_tensor * tensor, uint32_t i, int32_t value) { assert(i < GGML_MAX_OP_PARAMS / sizeof(int32_t)); ((int32_t *)(tensor->op_params))[i] = value; } +static void ggml_set_op_params_f32(struct ggml_tensor * tensor, uint32_t i, float value) { + assert(i < GGML_MAX_OP_PARAMS / sizeof(float)); + ((float *)(tensor->op_params))[i] = value; +} + struct ggml_tensor * ggml_set_zero(struct ggml_tensor * tensor) { memset(tensor->data, 0, ggml_nbytes(tensor)); return tensor; @@ -5898,6 +5912,55 @@ struct ggml_tensor * ggml_upscale( return ggml_upscale_impl(ctx, a, scale_factor); } +struct ggml_tensor * ggml_arange( + struct ggml_context * ctx, + float start, + float stop, + float step) { + + GGML_ASSERT(stop > start); + + const int64_t steps = (int64_t) ceilf((stop - start) / step); + + struct ggml_tensor * result = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, steps); + + result->op = GGML_OP_ARANGE; + ggml_set_op_params_f32(result, 0, start); + ggml_set_op_params_f32(result, 1, stop); + ggml_set_op_params_f32(result, 2, step); + + return result; +} + +struct ggml_tensor * ggml_timestep_embedding( + struct ggml_context * ctx, + struct ggml_tensor * timesteps, + int dim, + int max_period) { + bool is_node = false; + + if (timesteps->grad) { + GGML_ASSERT(false); // TODO: implement backward + is_node = true; + } + + int actual_dim = dim; + if (dim % 2 != 0) { + actual_dim = dim + 1; + } + + struct ggml_tensor * result = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, actual_dim, timesteps->ne[0]); + + result->op = GGML_OP_TIMESTEP_EMBEDDING; + ggml_set_op_params_i32(result, 0, dim); + ggml_set_op_params_i32(result, 1, max_period); + + result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL; + result->src[0] = timesteps; + + return result; +} + // ggml_argsort struct ggml_tensor * ggml_argsort( @@ -10231,7 +10294,7 @@ static void ggml_compute_forward_group_norm_f32( int n_channels = src0->ne[2]; int n_groups = dst->op_params[0]; int n_channels_per_group = (n_channels + n_groups - 1) / n_groups; - for (int i = ith; i < n_groups; i+=nth) { + for (int i = ith; i < n_groups; i += nth) { int start = i * n_channels_per_group; int end = start + n_channels_per_group; if (end > n_channels) { @@ -10245,28 +10308,32 @@ static void ggml_compute_forward_group_norm_f32( for (int64_t i01 = 0; i01 < ne01; i01++) { const float * x = (float *)((char *) src0->data + i01 * nb01 + i02 * nb02 + i03 * nb03); + ggml_float sumr = 0.0; for (int64_t i00 = 0; i00 < ne00; i00++) { - sum += (ggml_float)x[i00]; + sumr += (ggml_float)x[i00]; } + sum += sumr; } } - float mean = sum / (ne00 * ne01 * step); - ggml_float sum2 = 0.0; + const float mean = sum / (ne00 * ne01 * step); + ggml_float sum2 = 0.0; for (int64_t i02 = start; i02 < end; i02++) { for (int64_t i01 = 0; i01 < ne01; i01++) { const float * x = (float *)((char *) src0->data + i01 * nb01 + i02 * nb02 + i03 * nb03); float * y = (float *)((char *) dst->data + i01 * nb1 + i02 * nb2 + i03 * nb3); + ggml_float sumr = 0.0; for (int64_t i00 = 0; i00 < ne00; i00++) { float v = x[i00] - mean; y[i00] = v; - sum2 += (ggml_float)(v * v); + sumr += (ggml_float)(v * v); } + sum2 += sumr; } } - float variance = sum2 / (ne00 * ne01 * step); + const float variance = sum2 / (ne00 * ne01 * step); const float scale = 1.0f / sqrtf(variance + eps); for (int64_t i02 = start; i02 < end; i02++) { @@ -13547,6 +13614,106 @@ static void ggml_compute_forward_pad( } } + +// ggml_compute_forward_arange + +static void ggml_compute_forward_arange_f32( + const struct ggml_compute_params * params, + struct ggml_tensor * dst) { + + if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) { + return; + } + + GGML_ASSERT(dst->nb[0] == sizeof(float)); + + const int ith = params->ith; + const int nth = params->nth; + + const float start = ggml_get_op_params_f32(dst, 0); + const float stop = ggml_get_op_params_f32(dst, 1); + const float step = ggml_get_op_params_f32(dst, 2); + + const int64_t steps = (int64_t) ceilf((stop - start) / step); + + GGML_ASSERT(ggml_nelements(dst) == steps); + + for (int64_t i = ith; i < steps; i+= nth) { + float value = start + step * i; + ((float *)dst->data)[i] = value; + } +} + +static void ggml_compute_forward_arange( + const struct ggml_compute_params * params, + struct ggml_tensor * dst) { + switch (dst->type) { + case GGML_TYPE_F32: + { + ggml_compute_forward_arange_f32(params, dst); + } break; + default: + { + GGML_ASSERT(false); + } break; + } +} + +static void ggml_compute_forward_timestep_embedding_f32( + const struct ggml_compute_params * params, + struct ggml_tensor * dst) { + + if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) { + return; + } + + const struct ggml_tensor * src0 = dst->src[0]; + + GGML_ASSERT(src0->nb[0] == sizeof(float)); + + const int ith = params->ith; + const int nth = params->nth; + + GGML_TENSOR_UNARY_OP_LOCALS + + const int dim = ggml_get_op_params_i32(dst, 0); + const int max_period = ggml_get_op_params_i32(dst, 1); + + int half = dim / 2; + + for (int64_t i = 0; i < ne00; i++) { + float * embed_data = (float *)((char *) dst->data + i*nb1); + for (int64_t j = ith; j < half; j += nth) { + float timestep = ((float *)src0->data)[i]; + float freq = (float)expf(-logf(max_period) * j / half); + float arg = timestep * freq; + embed_data[j] = cosf(arg); + embed_data[j + half] = sinf(arg); + } + if (dim % 2 != 0 && ith == 0) { + embed_data[dim] = 0.f; + } + } +} + +static void ggml_compute_forward_timestep_embedding( + const struct ggml_compute_params * params, + struct ggml_tensor * dst) { + + const struct ggml_tensor * src0 = dst->src[0]; + + switch (src0->type) { + case GGML_TYPE_F32: + { + ggml_compute_forward_timestep_embedding_f32(params, dst); + } break; + default: + { + GGML_ASSERT(false); + } break; + } +} + // ggml_compute_forward_argsort static void ggml_compute_forward_argsort_f32( @@ -15615,6 +15782,14 @@ static void ggml_compute_forward(struct ggml_compute_params * params, struct ggm { ggml_compute_forward_pad(params, tensor); } break; + case GGML_OP_ARANGE: + { + ggml_compute_forward_arange(params, tensor); + } break; + case GGML_OP_TIMESTEP_EMBEDDING: + { + ggml_compute_forward_timestep_embedding(params, tensor); + } break; case GGML_OP_ARGSORT: { ggml_compute_forward_argsort(params, tensor); @@ -16617,6 +16792,14 @@ static void ggml_compute_backward(struct ggml_context * ctx, struct ggml_tensor { GGML_ASSERT(false); // TODO: not implemented } break; + case GGML_OP_ARANGE: + { + GGML_ASSERT(false); // TODO: not implemented + } break; + case GGML_OP_TIMESTEP_EMBEDDING: + { + GGML_ASSERT(false); // TODO: not implemented + } break; case GGML_OP_ARGSORT: { GGML_ASSERT(false); // TODO: not implemented @@ -17368,6 +17551,14 @@ static int ggml_get_n_tasks(struct ggml_tensor * node, int n_threads) { { n_tasks = n_threads; } break; + case GGML_OP_ARANGE: + { + n_tasks = n_threads; + } break; + case GGML_OP_TIMESTEP_EMBEDDING: + { + n_tasks = n_threads; + } break; case GGML_OP_ARGSORT: { n_tasks = n_threads; diff --git a/ggml.h b/ggml.h index 0a6d3c051..98cfc7bf8 100644 --- a/ggml.h +++ b/ggml.h @@ -454,6 +454,8 @@ extern "C" { GGML_OP_POOL_2D, GGML_OP_UPSCALE, // nearest interpolate GGML_OP_PAD, + GGML_OP_ARANGE, + GGML_OP_TIMESTEP_EMBEDDING, GGML_OP_ARGSORT, GGML_OP_LEAKY_RELU, @@ -1661,6 +1663,15 @@ extern "C" { int p2, int p3); + // Ref: https://github.com/CompVis/stable-diffusion/blob/main/ldm/modules/diffusionmodules/util.py#L151 + // timesteps: [N,] + // return: [N, dim] + GGML_API struct ggml_tensor * ggml_timestep_embedding( + struct ggml_context * ctx, + struct ggml_tensor * timesteps, + int dim, + int max_period); + // sort rows enum ggml_sort_order { GGML_SORT_ORDER_ASC, @@ -1672,6 +1683,12 @@ extern "C" { struct ggml_tensor * a, enum ggml_sort_order order); + GGML_API struct ggml_tensor * ggml_arange( + struct ggml_context * ctx, + float start, + float stop, + float step); + // top k elements per row GGML_API struct ggml_tensor * ggml_top_k( struct ggml_context * ctx, diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index d4cea805f..8a6999f21 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -1412,6 +1412,50 @@ struct test_pad : public test_case { } }; +// GGML_OP_ARANGE +struct test_arange : public test_case { + const ggml_type type; + const float start; + const float stop; + const float step; + + std::string vars() override { + return VARS_TO_STR4(type, start, stop, step); + } + + test_arange(ggml_type type = GGML_TYPE_F32, + float start = 0.f, float stop = 10.f, float step = 1.f) + : type(type), start(start), stop(stop), step(step) {} + + ggml_tensor * build_graph(ggml_context * ctx) override { + ggml_tensor * out = ggml_arange(ctx, start, stop, step); + return out; + } +}; + +// GGML_OP_TIMESTEP_EMBEDDING +struct test_timestep_embedding : public test_case { + const ggml_type type; + const std::array ne_a; + const int dim; + const int max_period; + + std::string vars() override { + return VARS_TO_STR4(type, ne_a, dim, max_period); + } + + test_timestep_embedding(ggml_type type = GGML_TYPE_F32, + std::array ne_a = {2, 1, 1, 1}, + int dim = 320, int max_period=10000) + : type(type), ne_a(ne_a), dim(dim), max_period(max_period) {} + + ggml_tensor * build_graph(ggml_context * ctx) override { + ggml_tensor * a = ggml_new_tensor(ctx, type, 4, ne_a.data()); + ggml_tensor * out = ggml_timestep_embedding(ctx, a, dim, max_period); + return out; + } +}; + // GGML_OP_LEAKY_RELU struct test_leaky_relu : public test_case { const ggml_type type; @@ -2126,6 +2170,8 @@ static bool test_backend(ggml_backend_t backend, test_mode mode, const char * op test_cases.emplace_back(new test_group_norm()); test_cases.emplace_back(new test_acc()); test_cases.emplace_back(new test_pad()); + test_cases.emplace_back(new test_arange()); + test_cases.emplace_back(new test_timestep_embedding()); test_cases.emplace_back(new test_leaky_relu()); // these tests are disabled to save execution time, but they can be handy for debugging From a0fc62661f0fd2a9edd10ae5617345bbbf972f42 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Mon, 4 Mar 2024 10:40:04 +0200 Subject: [PATCH 529/812] sync : ggml --- scripts/sync-ggml.last | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/sync-ggml.last b/scripts/sync-ggml.last index 389c0bdfe..4a1b0bab4 100644 --- a/scripts/sync-ggml.last +++ b/scripts/sync-ggml.last @@ -1 +1 @@ -b458250b736a7473f7ff3560d47c93f1644f3290 +274680868e12427373bab4bec87554431b954704 From 4ffcdce2ff877ebb683cd217ea38faf20faa5ffe Mon Sep 17 00:00:00 2001 From: Xuan Son Nguyen Date: Mon, 4 Mar 2024 12:22:08 +0100 Subject: [PATCH 530/812] add alias for chat template (#5858) --- examples/server/server.cpp | 2 +- llama.cpp | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/examples/server/server.cpp b/examples/server/server.cpp index 0ca388f47..208edd571 100644 --- a/examples/server/server.cpp +++ b/examples/server/server.cpp @@ -413,7 +413,7 @@ struct llama_server_context int res = llama_chat_apply_template(model, nullptr, chat, 1, true, buf.data(), buf.size()); if (res < 0) { LOG_ERROR("The chat template comes with this model is not yet supported, falling back to chatml. This may cause the model to output suboptimal responses", {}); - sparams.chat_template = "<|im_start|>"; // llama_chat_apply_template only checks if <|im_start|> exist in the template + sparams.chat_template = "chatml"; } } diff --git a/llama.cpp b/llama.cpp index c1f015791..de579d9e3 100644 --- a/llama.cpp +++ b/llama.cpp @@ -13282,7 +13282,7 @@ static int32_t llama_chat_apply_template_internal( std::string & dest, bool add_ass) { // Taken from the research: https://github.com/ggerganov/llama.cpp/issues/5527 std::stringstream ss; - if (tmpl.find("<|im_start|>") != std::string::npos) { + if (tmpl == "chatml" || tmpl.find("<|im_start|>") != std::string::npos) { // chatml template for (auto message : chat) { ss << "<|im_start|>" << message->role << "\n" << message->content << "<|im_end|>\n"; @@ -13290,7 +13290,7 @@ static int32_t llama_chat_apply_template_internal( if (add_ass) { ss << "<|im_start|>assistant\n"; } - } else if (tmpl.find("[INST]") != std::string::npos) { + } else if (tmpl == "llama2" || tmpl.find("[INST]") != std::string::npos) { // llama2 template and its variants // [variant] support system message bool support_system_message = tmpl.find("<>") != std::string::npos; @@ -13325,7 +13325,7 @@ static int32_t llama_chat_apply_template_internal( } } // llama2 templates seem to not care about "add_generation_prompt" - } else if (tmpl.find("<|user|>") != std::string::npos) { + } else if (tmpl == "zephyr" || tmpl.find("<|user|>") != std::string::npos) { // zephyr template for (auto message : chat) { ss << "<|" << message->role << "|>" << "\n" << message->content << "<|endoftext|>\n"; @@ -13333,7 +13333,7 @@ static int32_t llama_chat_apply_template_internal( if (add_ass) { ss << "<|assistant|>\n"; } - } else if (tmpl.find("bos_token + message['role']") != std::string::npos) { + } else if (tmpl == "monarch" || tmpl.find("bos_token + message['role']") != std::string::npos) { // mlabonne/AlphaMonarch-7B template (the is included inside history) for (auto message : chat) { std::string bos = (message == chat.front()) ? "" : ""; // skip BOS for first message @@ -13342,7 +13342,7 @@ static int32_t llama_chat_apply_template_internal( if (add_ass) { ss << "assistant\n"; } - } else if (tmpl.find("") != std::string::npos) { + } else if (tmpl == "gemma" || tmpl.find("") != std::string::npos) { // google/gemma-7b-it std::string system_prompt = ""; for (auto message : chat) { @@ -13389,7 +13389,7 @@ LLAMA_API int32_t llama_chat_apply_template( int32_t res = llama_model_meta_val_str(model, template_key.c_str(), model_template.data(), model_template.size()); if (res < 0) { // worst case: there is no information about template, we will use chatml by default - curr_tmpl = "<|im_start|>"; // see llama_chat_apply_template_internal + curr_tmpl = "chatml"; // see llama_chat_apply_template_internal } else { curr_tmpl = std::string(model_template.data(), model_template.size()); } From 6d341ab6c53cd51f2921d986d0090cc8b049b39a Mon Sep 17 00:00:00 2001 From: Minsoo Cheong <54794500+mscheong01@users.noreply.github.com> Date: Tue, 5 Mar 2024 03:24:00 +0900 Subject: [PATCH 531/812] speculative : implement stochastic speculative sampling (#5625) * (WIP) Implement stochastic speculative decoding * sample from residual distribution on draft accept failure * fix #5657: force greedy sampling with probs when temp is 0 * remove p_accept parameter * fix style * remove unused variables * add srand() in speculative.cpp * replace use of rand() with mt19937 sampling * fixes based on review (@JohannesGaessler) * fix r random generation * randomly select next sequence to verify + fix bug in memory freeing * fix bug in active_seqs sync * fix uniform int distribution initialization * remove warnings from comparison between int and size_t * check grammar in `llama_sample_probability_distribution_impl` * remove malloc code by utilizing vectors * add PR link to README --- common/common.cpp | 7 - common/common.h | 3 +- common/sampling.cpp | 79 ++++++++++ common/sampling.h | 7 + examples/speculative/README.md | 1 + examples/speculative/speculative.cpp | 224 ++++++++++++++++++++------- 6 files changed, 260 insertions(+), 61 deletions(-) diff --git a/common/common.cpp b/common/common.cpp index dbe7e9229..036a98134 100644 --- a/common/common.cpp +++ b/common/common.cpp @@ -513,12 +513,6 @@ bool gpt_params_parse_ex(int argc, char ** argv, gpt_params & params) { break; } params.n_sequences = std::stoi(argv[i]); - } else if (arg == "--p-accept" || arg == "-pa") { - if (++i >= argc) { - invalid_param = true; - break; - } - params.p_accept = std::stof(argv[i]); } else if (arg == "--p-split" || arg == "-ps") { if (++i >= argc) { invalid_param = true; @@ -1044,7 +1038,6 @@ void gpt_print_usage(int /*argc*/, char ** argv, const gpt_params & params) { printf(" --chunks N max number of chunks to process (default: %d, -1 = all)\n", params.n_chunks); printf(" -np N, --parallel N number of parallel sequences to decode (default: %d)\n", params.n_parallel); printf(" -ns N, --sequences N number of sequences to decode (default: %d)\n", params.n_sequences); - printf(" -pa N, --p-accept N speculative decoding accept probability (default: %.1f)\n", (double)params.p_accept); printf(" -ps N, --p-split N speculative decoding split probability (default: %.1f)\n", (double)params.p_split); printf(" -cb, --cont-batching enable continuous batching (a.k.a dynamic batching) (default: disabled)\n"); printf(" --mmproj MMPROJ_FILE path to a multimodal projector file for LLaVA. see examples/llava/README.md\n"); diff --git a/common/common.h b/common/common.h index b2868833b..977ce419f 100644 --- a/common/common.h +++ b/common/common.h @@ -53,11 +53,10 @@ struct gpt_params { int32_t n_ctx = 512; // context size int32_t n_batch = 512; // batch size for prompt processing (must be >=32 to use BLAS) int32_t n_keep = 0; // number of tokens to keep from initial prompt - int32_t n_draft = 8; // number of tokens to draft during speculative decoding + int32_t n_draft = 5; // number of tokens to draft during speculative decoding int32_t n_chunks = -1; // max number of chunks to process (-1 = unlimited) int32_t n_parallel = 1; // number of parallel sequences to decode int32_t n_sequences = 1; // number of sequences to decode - float p_accept = 0.5f; // speculative decoding accept probability float p_split = 0.1f; // speculative decoding split probability int32_t n_gpu_layers = -1; // number of layers to store in VRAM (-1 - use default) int32_t n_gpu_layers_draft = -1; // number of layers to store in VRAM for the draft model (-1 - use default) diff --git a/common/sampling.cpp b/common/sampling.cpp index e67096bea..823031feb 100644 --- a/common/sampling.cpp +++ b/common/sampling.cpp @@ -295,6 +295,77 @@ static llama_token llama_sampling_sample_impl( return id; } +static llama_token_data_array llama_sample_probability_distribution_impl( + struct llama_sampling_context * ctx_sampling, + struct llama_context * ctx_main, + struct llama_context * ctx_cfg, + const int idx) { + const llama_sampling_params & params = ctx_sampling->params; + + const int n_vocab = llama_n_vocab(llama_get_model(ctx_main)); + + const int32_t penalty_last_n = params.penalty_last_n < 0 ? params.n_prev : params.penalty_last_n; + const float penalty_repeat = params.penalty_repeat; + const float penalty_freq = params.penalty_freq; + const float penalty_present = params.penalty_present; + const bool penalize_nl = params.penalize_nl; + + auto & prev = ctx_sampling->prev; + auto & cur = ctx_sampling->cur; + + // Get a pointer to the logits + float * logits = llama_get_logits_ith(ctx_main, idx); + + // Declare original_logits at the beginning of the function scope + std::vector original_logits; + + // apply params.logit_bias map + for (auto it = params.logit_bias.begin(); it != params.logit_bias.end(); it++) { + logits[it->first] += it->second; + } + + if (ctx_cfg) { + float * logits_guidance = llama_get_logits_ith(ctx_cfg, idx); + llama_sample_apply_guidance(ctx_main, logits, logits_guidance, params.cfg_scale); + } + + cur.clear(); + + for (llama_token token_id = 0; token_id < n_vocab; token_id++) { + cur.emplace_back(llama_token_data{token_id, logits[token_id], 0.0f}); + } + + llama_token_data_array cur_p = { cur.data(), cur.size(), false }; + + // apply penalties + const auto& penalty_tokens = params.use_penalty_prompt_tokens ? params.penalty_prompt_tokens : prev; + const int penalty_tokens_used_size = std::min((int)penalty_tokens.size(), penalty_last_n); + if (penalty_tokens_used_size) { + const float nl_logit = logits[llama_token_nl(llama_get_model(ctx_main))]; + + llama_sample_repetition_penalties(ctx_main, &cur_p, + penalty_tokens.data() + penalty_tokens.size() - penalty_tokens_used_size, + penalty_tokens_used_size, penalty_repeat, penalty_freq, penalty_present); + + if (!penalize_nl) { + for (size_t idx = 0; idx < cur_p.size; idx++) { + if (cur_p.data[idx].id == llama_token_nl(llama_get_model(ctx_main))) { + cur_p.data[idx].logit = nl_logit; + break; + } + } + } + } + + // apply grammar checks + if (ctx_sampling->grammar != NULL) { + llama_sample_grammar(ctx_main, &cur_p, ctx_sampling->grammar); + } + + llama_sample_softmax(ctx_main, &cur_p); + return cur_p; +} + llama_token llama_sampling_sample( struct llama_sampling_context * ctx_sampling, struct llama_context * ctx_main, @@ -304,6 +375,14 @@ llama_token llama_sampling_sample( return llama_sampling_sample_impl(ctx_sampling, ctx_main, ctx_cfg, idx, false); } +llama_token_data_array llama_sampling_probability_distribution( + struct llama_sampling_context * ctx_sampling, + struct llama_context * ctx_main, + struct llama_context * ctx_cfg, + const int idx) { + return llama_sample_probability_distribution_impl(ctx_sampling,ctx_main, ctx_cfg, idx); +} + void llama_sampling_accept( struct llama_sampling_context * ctx_sampling, struct llama_context * ctx_main, diff --git a/common/sampling.h b/common/sampling.h index 95d875394..48b2459d1 100644 --- a/common/sampling.h +++ b/common/sampling.h @@ -131,6 +131,13 @@ llama_token llama_sampling_sample( struct llama_context * ctx_cfg, int idx = 0); +// returns the probability that token of given id will be sampled +llama_token_data_array llama_sampling_probability_distribution( + struct llama_sampling_context * ctx_sampling, + struct llama_context * ctx_main, + struct llama_context * ctx_cfg, + int idx = 0); + void llama_sampling_accept( struct llama_sampling_context * ctx_sampling, struct llama_context * ctx_main, diff --git a/examples/speculative/README.md b/examples/speculative/README.md index 814efa592..a6608c5fe 100644 --- a/examples/speculative/README.md +++ b/examples/speculative/README.md @@ -6,3 +6,4 @@ More info: - https://github.com/ggerganov/llama.cpp/pull/2926 - https://github.com/ggerganov/llama.cpp/pull/3624 +- https://github.com/ggerganov/llama.cpp/pull/5625 diff --git a/examples/speculative/speculative.cpp b/examples/speculative/speculative.cpp index 3848791d4..85bc0a762 100644 --- a/examples/speculative/speculative.cpp +++ b/examples/speculative/speculative.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #define SPEC_VOCAB_MAX_SIZE_DIFFERENCE 100 #define SPEC_VOCAB_CHECK_START_TOKEN_ID 5 @@ -18,6 +19,7 @@ struct seq_draft { std::vector i_batch_tgt; std::vector tokens; + std::vector> dists; struct llama_sampling_context * ctx_sampling; }; @@ -37,12 +39,15 @@ int main(int argc, char ** argv) { // max number of parallel drafting sequences (i.e. tree branches) const int n_seq_dft = params.n_parallel; - // probability threshold for accepting a token from the draft model - const float p_accept = params.p_accept; - // probability threshold for splitting a draft branch (only for n_seq_dft > 1) const float p_split = params.p_split; + if (params.seed == LLAMA_DEFAULT_SEED) { + params.seed = time(NULL); + } + std::default_random_engine rng(params.seed); + std::uniform_real_distribution<> u_dist; + #ifndef LOG_DISABLE_LOGS log_set_target(log_filename_generator("speculative", "log")); LOG_TEE("Log start\n"); @@ -166,7 +171,9 @@ int main(int argc, char ** argv) { std::vector drafts(n_seq_dft); params.sparams.grammar.clear(); // the draft samplers will copy the target sampler's grammar - params.sparams.temp = -1.0f; // force greedy sampling with probs for the draft model + if (params.sparams.temp == 0) { + params.sparams.temp = -1.0f; // force greedy sampling with probs for the draft model + } for (int s = 0; s < n_seq_dft; ++s) { drafts[s].ctx_sampling = llama_sampling_init(params.sparams); @@ -182,12 +189,15 @@ int main(int argc, char ** argv) { drafts[0].i_batch_tgt[0] = 0; while (true) { + std::set active_seqs = {}; + // print current draft sequences for (int s = 0; s < n_seq_dft; ++s) { if (!drafts[s].active) { continue; } + active_seqs.insert(s); const auto & tokens = drafts[s].tokens; LOG("draft %d: %s\n", s, LOG_TOKENS_TOSTR_PRETTY(ctx_dft, tokens).c_str()); @@ -196,48 +206,156 @@ int main(int argc, char ** argv) { int i_dft = 0; int s_keep = 0; + llama_token token_id; + std::string token_str; + + // loop until we fail to accept a drafted token or we run out of drafted tokens while (true) { - LOG("sampling target: s_keep = %3d, i_dft = %3d, i_batch_tgt = %3d\n", s_keep, i_dft, drafts[s_keep].i_batch_tgt[i_dft]); - - // sample from the target model - llama_token id = llama_sampling_sample(ctx_sampling, ctx_tgt, NULL, drafts[s_keep].i_batch_tgt[i_dft]); - - llama_sampling_accept(ctx_sampling, ctx_tgt, id, true); - - //LOG("last: %s\n", LOG_TOKENS_TOSTR_PRETTY(ctx_tgt, ctx_sampling->prev).c_str()); - - const std::string token_str = llama_token_to_piece(ctx_tgt, id); - - if (!params.use_color) { - printf("%s", token_str.c_str()); - } - - if (id == llama_token_eos(model_tgt)) { - has_eos = true; - } - - ++n_predict; // check if the target token matches any of the drafts + // for stochastic sampling, attempt to match the token with the drafted tokens { - bool matches = false; + bool accept = false; + if (params.sparams.temp > 0) { + // stochastic verification - for (int s = 0; s < n_seq_dft; ++s) { - if (!drafts[s].active) { - continue; + llama_token_data_array dist_tgt = llama_sampling_probability_distribution(ctx_sampling, ctx_tgt, NULL, drafts[s_keep].i_batch_tgt[i_dft]); + float p_tgt = 0, p_dft = 0; + + // GGML_ASSERT(dist_tgt.size() == dist_dft.size()); + + while (active_seqs.size() > 0) { + // randomly select a sequence to verify from active sequences + std::uniform_int_distribution u_int_dist(0, active_seqs.size() - 1); + int s = *std::next(active_seqs.begin(), u_int_dist(rng)); + if (i_dft >= (int) drafts[s].tokens.size()) { + drafts[s].active = false; + active_seqs.erase(s); + continue; + } + if (accept) { + // if we already accepted a token, we can skip the rest + if (drafts[s].tokens[i_dft] != drafts[s_keep].tokens[i_dft]) { + drafts[s].active = false; + active_seqs.erase(s); + } + continue; + } + LOG("verifying sequence #%d at pos #%d from %d active sequence(s)\n", s, i_dft, (int) active_seqs.size()); + float r = u_dist(rng); + llama_token_data_array dist_dft = { drafts[s].dists[i_dft].data() , drafts[s].dists[i_dft].size(), true }; + // acquire the token probabilities assigned by the draft and target models + for (size_t i = 0; i < dist_tgt.size; i++) { + if (dist_tgt.data[i].id == drafts[s].tokens[i_dft]) { + p_tgt = dist_tgt.data[i].p; + } + if (dist_dft.data[i].id == drafts[s].tokens[i_dft]) { + p_dft = dist_dft.data[i].p; + } + if (p_tgt && p_dft) { + break; + } + } + LOG("r = %f, p_dft = %f, p_tgt = %f\n", r, p_dft, p_tgt); + if (r <= p_tgt / p_dft) { + s_keep = s; + accept = true; + token_id = drafts[s].tokens[i_dft]; + token_str = llama_token_to_piece(ctx_tgt, token_id); + llama_sampling_accept(ctx_sampling, ctx_tgt, token_id, true); + + LOG("draft token %d of sequence %d (%d, '%s') accepted\n", i_dft, s, token_id, token_str.c_str()); + break; + } else { + LOG("draft token %d of sequence %d (%d, '%s') rejected\n", i_dft, s, drafts[s].tokens[i_dft], llama_token_to_piece(ctx_tgt, drafts[s].tokens[i_dft]).c_str()); + drafts[s].active = false; + + // calculate residual probability + GGML_ASSERT(dist_tgt.sorted); + GGML_ASSERT(dist_dft.sorted); + float sum_probs = 0.0f; + + // sort dist by id + std::sort(dist_tgt.data, dist_tgt.data + dist_tgt.size, [](const llama_token_data &a, const llama_token_data &b) { + return a.id < b.id; + }); + std::sort(dist_dft.data, dist_dft.data + dist_dft.size, [](const llama_token_data &a, const llama_token_data &b) { + return a.id < b.id; + }); + + for (size_t i = 0; i < dist_tgt.size; i++) { + dist_tgt.data[i].p = std::max(0.0f, dist_tgt.data[i].p - dist_dft.data[i].p); + sum_probs += dist_tgt.data[i].p; + } + for (size_t i = 0; i < dist_tgt.size; i++) { + dist_tgt.data[i].p /= sum_probs; + } + + // sort dist_tgt by p desc + std::sort(dist_tgt.data, dist_tgt.data + dist_tgt.size, [](const llama_token_data &a, const llama_token_data &b) { + return a.p > b.p; + }); + } + + active_seqs.erase(s); + for(int i = 0; i < n_seq_dft; i++) { + if (i == s) { + continue; + } + if (drafts[i].tokens[i_dft] == drafts[s].tokens[i_dft]) { + // synchronize active status for sequences with the same drafted token + drafts[i].active = drafts[i].active && accept; + if (!drafts[i].active) { + active_seqs.erase(s); + } + } + } } - if (i_dft < (int) drafts[s].tokens.size() && id == drafts[s].tokens[i_dft]) { - LOG("the sampled target token matches the %dth drafted token of sequence %d (%d, '%s') - accepted\n", i_dft, s, id, token_str.c_str()); + if (!accept) { + // all drafted tokens were rejected + // sample from the target model + LOG("all drafted tokens were rejected, sampling from residual distribution\n"); + token_id = llama_sample_token(ctx_tgt, &dist_tgt); + llama_sampling_accept(ctx_sampling, ctx_tgt, token_id, true); + token_str = llama_token_to_piece(ctx_tgt, token_id); + } - s_keep = s; - matches = true; - } else { - drafts[s].active = false; + } else { + // greedy verification + + // sample from the target model + LOG("sampling target: s_keep = %3d, i_dft = %3d, i_batch_tgt = %3d\n", s_keep, i_dft, drafts[s_keep].i_batch_tgt[i_dft]); + token_id = llama_sampling_sample(ctx_sampling, ctx_tgt, NULL, drafts[s_keep].i_batch_tgt[i_dft]); + + llama_sampling_accept(ctx_sampling, ctx_tgt, token_id, true); + + //LOG("last: %s\n", LOG_TOKENS_TOSTR_PRETTY(ctx_tgt, ctx_sampling->prev).c_str()); + + token_str = llama_token_to_piece(ctx_tgt, token_id); + + for (int s = 0; s < n_seq_dft; ++s) { + if (!drafts[s].active) { + continue; + } + + if (i_dft < (int) drafts[s].tokens.size() && token_id == drafts[s].tokens[i_dft]) { + LOG("the sampled target token matches the %dth drafted token of sequence %d (%d, '%s') - accepted\n", i_dft, s, token_id, token_str.c_str()); + + s_keep = s; + accept = true; + } else { + drafts[s].active = false; + } } } - if (matches) { + if (token_id == llama_token_eos(model_tgt)) { + has_eos = true; + } + ++n_predict; + + if (accept) { ++n_accept; ++n_past_tgt; ++n_past_dft; @@ -245,17 +363,21 @@ int main(int argc, char ** argv) { if (params.use_color) { // Color token according to its origin sequence printf("\u001b[%dm%s\u001b[37m", (36 - s_keep % 6), token_str.c_str()); - fflush(stdout); + } else { + printf("%s", token_str.c_str()); } + fflush(stdout); continue; + } else { + printf("%s", token_str.c_str()); + fflush(stdout); + break; } } - if (params.use_color) { - printf("%s", token_str.c_str()); - } - fflush(stdout); + } - LOG("the sampled target token (%d, '%s') did not match, or we ran out of drafted tokens\n", id, token_str.c_str()); + { + LOG("the sampled target token (%d, '%s') did not match, or we ran out of drafted tokens\n", token_id, token_str.c_str()); // TODO: simplify { @@ -275,21 +397,21 @@ int main(int argc, char ** argv) { drafts[s].active = false; drafts[s].tokens.clear(); drafts[s].i_batch_tgt.clear(); + drafts[s].dists.clear(); } // note: will be erased after the speculation phase - drafts[0].tokens.push_back(id); + drafts[0].tokens.push_back(token_id); + drafts[0].dists.push_back(std::vector()); drafts[0].i_batch_tgt.push_back(0); llama_batch_clear(batch_dft); - llama_batch_add (batch_dft, id, n_past_dft, { 0 }, true); + llama_batch_add (batch_dft, token_id, n_past_dft, { 0 }, true); llama_kv_cache_seq_rm(ctx_dft, 0, n_past_dft, -1); // LOG("dft batch: %s\n", LOG_BATCH_TOSTR_PRETTY(ctx_dft, batch_dft).c_str()); - llama_decode (ctx_dft, batch_dft); + llama_decode(ctx_dft, batch_dft); ++n_past_dft; - - break; } if (n_predict > params.n_predict || has_eos) { @@ -334,12 +456,6 @@ int main(int argc, char ** argv) { k, s, i, cur_p[k].id, cur_p[k].p, llama_token_to_piece(ctx_dft, cur_p[k].id).c_str()); } - if (cur_p[0].p < p_accept) { - LOG("stopping drafting for seq %3d, probability too low: %.3f < %.3f\n", s, cur_p[0].p, p_accept); - drafts[s].drafting = false; - continue; - } - std::vector sa(1, s); // attempt to split the branch if the probability is high enough @@ -367,6 +483,7 @@ int main(int argc, char ** argv) { drafts[n_seq_cur].skip = true; drafts[n_seq_cur].tokens = drafts[s].tokens; + drafts[n_seq_cur].dists = drafts[s].dists; drafts[n_seq_cur].i_batch_dft = drafts[s].i_batch_dft; drafts[n_seq_cur].i_batch_tgt = drafts[s].i_batch_tgt; @@ -389,6 +506,8 @@ int main(int argc, char ** argv) { llama_sampling_accept(drafts[s].ctx_sampling, ctx_dft, id, true); drafts[s].tokens.push_back(id); + // save cur_p.data into drafts[s].dists + drafts[s].dists.push_back(cur_p); // add unique drafted tokens to the target batch drafts[s].i_batch_tgt.push_back(batch_tgt.n_tokens); @@ -440,6 +559,7 @@ int main(int argc, char ** argv) { } drafts[s].tokens.erase(drafts[s].tokens.begin()); + drafts[s].dists.erase(drafts[s].dists.begin()); } } From fe52be11e35358d2fd249f19d7ef5b6f9c08b16b Mon Sep 17 00:00:00 2001 From: Dane Madsen Date: Tue, 5 Mar 2024 05:26:55 +1100 Subject: [PATCH 532/812] cmake : handle cases where git index is not found in .git (#5844) * Update CMakeLists.txt * Update CMakeLists.txt --- common/CMakeLists.txt | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/common/CMakeLists.txt b/common/CMakeLists.txt index f79acfef1..350bbdf7f 100644 --- a/common/CMakeLists.txt +++ b/common/CMakeLists.txt @@ -19,7 +19,12 @@ if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/../.git") endif() endif() - set(GIT_INDEX "${GIT_DIR}/index") + if(EXISTS "${GIT_DIR}/index") + set(GIT_INDEX "${GIT_DIR}/index") + else() + message(WARNING "Git index not found in git repository.") + set(GIT_INDEX "") + endif() else() message(WARNING "Git repository not found; to enable automatic generation of build info, make sure Git is installed and the project is a Git repository.") set(GIT_INDEX "") From 9fa262734733573fa629ffc97dfcb971fe3f4832 Mon Sep 17 00:00:00 2001 From: Michael Podvitskiy Date: Mon, 4 Mar 2024 10:05:42 +0100 Subject: [PATCH 533/812] ggml : introduce ggml_status (ggml/750) * using enum as an exit code instead of macros * update return type from enum to unsigned int * indentation fix * compound update ggml_compute_exit_code -> ggml_status changed ggml_status from a bit-field type to simple codes ggml_status to string cast * ggml_status to string cast * GGML_CALL was removed Co-authored-by: slaren --------- Co-authored-by: slaren Co-authored-by: Georgi Gerganov --- ggml-backend-impl.h | 7 ++++--- ggml-backend.c | 39 ++++++++++++++++++--------------------- ggml-backend.h | 31 ++++++++++++++++--------------- ggml-cuda.cu | 4 ++-- ggml-kompute.cpp | 4 ++-- ggml-metal.m | 8 ++++---- ggml-opencl.cpp | 4 ++-- ggml-sycl.cpp | 4 ++-- ggml-vulkan.cpp | 4 ++-- ggml.c | 29 +++++++++++++++++++++++------ ggml.h | 17 +++++++++++++---- 11 files changed, 88 insertions(+), 63 deletions(-) diff --git a/ggml-backend-impl.h b/ggml-backend-impl.h index 0e5bf0ae1..2e9ba58a9 100644 --- a/ggml-backend-impl.h +++ b/ggml-backend-impl.h @@ -91,13 +91,14 @@ extern "C" { // (optional) complete all pending operations void (*GGML_CALL synchronize)(ggml_backend_t backend); - // compute graph with a plan + // create a plan for ggml_cgraph and free it ggml_backend_graph_plan_t (*GGML_CALL graph_plan_create) (ggml_backend_t backend, const struct ggml_cgraph * cgraph); void (*GGML_CALL graph_plan_free) (ggml_backend_t backend, ggml_backend_graph_plan_t plan); - void (*GGML_CALL graph_plan_compute)(ggml_backend_t backend, ggml_backend_graph_plan_t plan); + // compute graph with a plan + enum ggml_status (*GGML_CALL graph_plan_compute)(ggml_backend_t backend, ggml_backend_graph_plan_t plan); // compute graph without a plan (async) - bool (*GGML_CALL graph_compute)(ggml_backend_t backend, struct ggml_cgraph * cgraph); + enum ggml_status (*GGML_CALL graph_compute) (ggml_backend_t backend, struct ggml_cgraph * cgraph); // check if the backend supports an operation bool (*GGML_CALL supports_op)(ggml_backend_t backend, const struct ggml_tensor * op); diff --git a/ggml-backend.c b/ggml-backend.c index c86673b04..d60d98414 100644 --- a/ggml-backend.c +++ b/ggml-backend.c @@ -262,11 +262,11 @@ void ggml_backend_graph_plan_free(ggml_backend_t backend, ggml_backend_graph_pla backend->iface.graph_plan_free(backend, plan); } -void ggml_backend_graph_plan_compute(ggml_backend_t backend, ggml_backend_graph_plan_t plan) { - backend->iface.graph_plan_compute(backend, plan); +enum ggml_status ggml_backend_graph_plan_compute(ggml_backend_t backend, ggml_backend_graph_plan_t plan) { + return backend->iface.graph_plan_compute(backend, plan); } -bool ggml_backend_graph_compute(ggml_backend_t backend, struct ggml_cgraph * cgraph) { +enum ggml_status ggml_backend_graph_compute(ggml_backend_t backend, struct ggml_cgraph * cgraph) { return backend->iface.graph_compute(backend, cgraph); } @@ -732,15 +732,15 @@ GGML_CALL static void ggml_backend_cpu_graph_plan_free(ggml_backend_t backend, g GGML_UNUSED(backend); } -GGML_CALL static void ggml_backend_cpu_graph_plan_compute(ggml_backend_t backend, ggml_backend_graph_plan_t plan) { +GGML_CALL static enum ggml_status ggml_backend_cpu_graph_plan_compute(ggml_backend_t backend, ggml_backend_graph_plan_t plan) { struct ggml_backend_plan_cpu * cpu_plan = (struct ggml_backend_plan_cpu *)plan; - ggml_graph_compute(&cpu_plan->cgraph, &cpu_plan->cplan); + return ggml_graph_compute(&cpu_plan->cgraph, &cpu_plan->cplan); GGML_UNUSED(backend); } -GGML_CALL static bool ggml_backend_cpu_graph_compute(ggml_backend_t backend, struct ggml_cgraph * cgraph) { +GGML_CALL static enum ggml_status ggml_backend_cpu_graph_compute(ggml_backend_t backend, struct ggml_cgraph * cgraph) { struct ggml_backend_cpu_context * cpu_ctx = (struct ggml_backend_cpu_context *)backend->context; struct ggml_cplan cplan = ggml_graph_plan(cgraph, cpu_ctx->n_threads); @@ -755,8 +755,7 @@ GGML_CALL static bool ggml_backend_cpu_graph_compute(ggml_backend_t backend, str cplan.abort_callback = cpu_ctx->abort_callback; cplan.abort_callback_data = cpu_ctx->abort_callback_data; - ggml_graph_compute(cgraph, &cplan); - return true; + return ggml_graph_compute(cgraph, &cplan); } GGML_CALL static bool ggml_backend_cpu_supports_op(ggml_backend_t backend, const struct ggml_tensor * op) { @@ -1437,7 +1436,7 @@ static bool ggml_backend_sched_alloc_splits(ggml_backend_sched_t sched) { return true; } -static bool ggml_backend_sched_compute_splits(ggml_backend_sched_t sched) { +static enum ggml_status ggml_backend_sched_compute_splits(ggml_backend_sched_t sched) { uint64_t copy_us[GGML_MAX_BACKENDS] = {0}; uint64_t compute_us[GGML_MAX_BACKENDS] = {0}; @@ -1472,8 +1471,9 @@ static bool ggml_backend_sched_compute_splits(ggml_backend_sched_t sched) { uint64_t compute_start_us = ggml_time_us(); if (!sched->callback_eval) { - if (!ggml_backend_graph_compute(split_backend, &split->graph)) { - return false; + enum ggml_status ec = ggml_backend_graph_compute(split_backend, &split->graph); + if (ec != GGML_STATUS_SUCCESS) { + return ec; } //ggml_backend_synchronize(split_backend); // necessary to measure compute time } else { @@ -1494,8 +1494,9 @@ static bool ggml_backend_sched_compute_splits(ggml_backend_sched_t sched) { struct ggml_cgraph gv = ggml_graph_view(&split->graph, j0, j1 + 1); - if (!ggml_backend_graph_compute(split_backend, &gv)) { - return false; + enum ggml_status ec = ggml_backend_graph_compute(split_backend, &gv); + if (ec != GGML_STATUS_SUCCESS) { + return ec; } if (need && !sched->callback_eval(t, false, sched->callback_eval_user_data)) { @@ -1519,7 +1520,7 @@ static bool ggml_backend_sched_compute_splits(ggml_backend_sched_t sched) { } #endif - return true; + return GGML_STATUS_SUCCESS; } ggml_backend_sched_t ggml_backend_sched_new(ggml_backend_t * backends, ggml_backend_buffer_type_t * bufts, int n_backends, size_t graph_size) { @@ -1581,7 +1582,7 @@ bool ggml_backend_sched_reserve(ggml_backend_sched_t sched, struct ggml_cgraph * return true; } -bool ggml_backend_sched_graph_compute(ggml_backend_sched_t sched, struct ggml_cgraph * graph) { +enum ggml_status ggml_backend_sched_graph_compute(ggml_backend_sched_t sched, struct ggml_cgraph * graph) { GGML_ASSERT((int)sched->hash_set.size >= graph->n_nodes + GGML_MAX_SPLITS*GGML_MAX_SPLIT_INPUTS); if (!sched->is_reset) { @@ -1590,14 +1591,10 @@ bool ggml_backend_sched_graph_compute(ggml_backend_sched_t sched, struct ggml_cg ggml_backend_sched_split_graph(sched, graph); if (!ggml_backend_sched_alloc_splits(sched)) { - return false; + return GGML_STATUS_ALLOC_FAILED; } - if (!ggml_backend_sched_compute_splits(sched)) { - return false; - } - - return true; + return ggml_backend_sched_compute_splits(sched); } void ggml_backend_sched_set_eval_callback(ggml_backend_sched_t sched, ggml_backend_sched_eval_callback callback, void * user_data) { diff --git a/ggml-backend.h b/ggml-backend.h index 8fb54bd92..8bed22578 100644 --- a/ggml-backend.h +++ b/ggml-backend.h @@ -66,12 +66,13 @@ extern "C" { GGML_API void ggml_backend_synchronize(ggml_backend_t backend); - GGML_API ggml_backend_graph_plan_t ggml_backend_graph_plan_create (ggml_backend_t backend, struct ggml_cgraph * cgraph); + GGML_API ggml_backend_graph_plan_t ggml_backend_graph_plan_create(ggml_backend_t backend, struct ggml_cgraph * cgraph); + GGML_API void ggml_backend_graph_plan_free (ggml_backend_t backend, ggml_backend_graph_plan_t plan); - GGML_API void ggml_backend_graph_plan_free (ggml_backend_t backend, ggml_backend_graph_plan_t plan); - GGML_API void ggml_backend_graph_plan_compute(ggml_backend_t backend, ggml_backend_graph_plan_t plan); - GGML_API bool ggml_backend_graph_compute (ggml_backend_t backend, struct ggml_cgraph * cgraph); - GGML_API bool ggml_backend_supports_op (ggml_backend_t backend, const struct ggml_tensor * op); + GGML_API enum ggml_status ggml_backend_graph_plan_compute(ggml_backend_t backend, ggml_backend_graph_plan_t plan); + GGML_API enum ggml_status ggml_backend_graph_compute (ggml_backend_t backend, struct ggml_cgraph * cgraph); + + GGML_API bool ggml_backend_supports_op(ggml_backend_t backend, const struct ggml_tensor * op); // tensor copy between different backends GGML_API void ggml_backend_tensor_copy(struct ggml_tensor * src, struct ggml_tensor * dst); @@ -157,26 +158,26 @@ extern "C" { typedef bool (*ggml_backend_sched_eval_callback)(struct ggml_tensor * t, bool ask, void * user_data); // Initialize a backend scheduler - GGML_API ggml_backend_sched_t ggml_backend_sched_new(ggml_backend_t * backends, ggml_backend_buffer_type_t * bufts, int n_backends, size_t graph_size); - GGML_API void ggml_backend_sched_free(ggml_backend_sched_t sched); + GGML_API ggml_backend_sched_t ggml_backend_sched_new(ggml_backend_t * backends, ggml_backend_buffer_type_t * bufts, int n_backends, size_t graph_size); + GGML_API void ggml_backend_sched_free(ggml_backend_sched_t sched); // Initialize backend buffers from a measure graph - GGML_API bool ggml_backend_sched_reserve(ggml_backend_sched_t sched, struct ggml_cgraph * measure_graph); + GGML_API bool ggml_backend_sched_reserve(ggml_backend_sched_t sched, struct ggml_cgraph * measure_graph); // Get the number of splits of the last graph - GGML_API int ggml_backend_sched_get_n_splits(ggml_backend_sched_t sched); + GGML_API int ggml_backend_sched_get_n_splits(ggml_backend_sched_t sched); - GGML_API size_t ggml_backend_sched_get_buffer_size(ggml_backend_sched_t sched, ggml_backend_t backend); + GGML_API size_t ggml_backend_sched_get_buffer_size(ggml_backend_sched_t sched, ggml_backend_t backend); - GGML_API void ggml_backend_sched_set_node_backend(ggml_backend_sched_t sched, struct ggml_tensor * node, ggml_backend_t backend); - GGML_API ggml_backend_t ggml_backend_sched_get_node_backend(ggml_backend_sched_t sched, struct ggml_tensor * node); + GGML_API void ggml_backend_sched_set_node_backend(ggml_backend_sched_t sched, struct ggml_tensor * node, ggml_backend_t backend); + GGML_API ggml_backend_t ggml_backend_sched_get_node_backend(ggml_backend_sched_t sched, struct ggml_tensor * node); // Allocate and compute graph on the backend scheduler - GGML_API bool ggml_backend_sched_graph_compute(ggml_backend_sched_t sched, struct ggml_cgraph * graph); + GGML_API enum ggml_status ggml_backend_sched_graph_compute(ggml_backend_sched_t sched, struct ggml_cgraph * graph); // Reset all assignments and allocators - must be called before changing the node backends - GGML_API void ggml_backend_sched_reset(ggml_backend_sched_t sched); + GGML_API void ggml_backend_sched_reset(ggml_backend_sched_t sched); // Set a callback to be called for each resulting node during graph compute - GGML_API void ggml_backend_sched_set_eval_callback(ggml_backend_sched_t sched, ggml_backend_sched_eval_callback callback, void * user_data); + GGML_API void ggml_backend_sched_set_eval_callback(ggml_backend_sched_t sched, ggml_backend_sched_eval_callback callback, void * user_data); // // Utils diff --git a/ggml-cuda.cu b/ggml-cuda.cu index 7d027a30a..72bcec8cd 100644 --- a/ggml-cuda.cu +++ b/ggml-cuda.cu @@ -12241,7 +12241,7 @@ GGML_CALL static void ggml_backend_cuda_synchronize(ggml_backend_t backend) { UNUSED(backend); } -GGML_CALL static bool ggml_backend_cuda_graph_compute(ggml_backend_t backend, ggml_cgraph * cgraph) { +GGML_CALL static enum ggml_status ggml_backend_cuda_graph_compute(ggml_backend_t backend, ggml_cgraph * cgraph) { ggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *)backend->context; ggml_cuda_set_main_device(cuda_ctx->device); @@ -12277,7 +12277,7 @@ GGML_CALL static bool ggml_backend_cuda_graph_compute(ggml_backend_t backend, gg GGML_ASSERT(ok); } - return true; + return GGML_STATUS_SUCCESS; } GGML_CALL static bool ggml_backend_cuda_supports_op(ggml_backend_t backend, const ggml_tensor * op) { diff --git a/ggml-kompute.cpp b/ggml-kompute.cpp index e740a76d1..83a7822fd 100644 --- a/ggml-kompute.cpp +++ b/ggml-kompute.cpp @@ -1927,10 +1927,10 @@ static ggml_backend_buffer_type_t ggml_backend_kompute_get_default_buffer_type(g return ggml_backend_kompute_buffer_type(ctx->device); } -static bool ggml_backend_kompute_graph_compute(ggml_backend_t backend, struct ggml_cgraph * cgraph) { +static ggml_status ggml_backend_kompute_graph_compute(ggml_backend_t backend, struct ggml_cgraph * cgraph) { auto * ctx = static_cast(backend->context); ggml_vk_graph_compute(ctx, cgraph); - return true; + return GGML_STATUS_SUCCESS; } static bool ggml_backend_kompute_supports_op(ggml_backend_t backend, const struct ggml_tensor * op) { diff --git a/ggml-metal.m b/ggml-metal.m index 6b5a8fdf5..00df22838 100644 --- a/ggml-metal.m +++ b/ggml-metal.m @@ -748,7 +748,7 @@ static bool ggml_metal_supports_op(const struct ggml_metal_context * ctx, const } } -static bool ggml_metal_graph_compute( +static enum ggml_status ggml_metal_graph_compute( struct ggml_metal_context * ctx, struct ggml_cgraph * gf) { @@ -2484,7 +2484,7 @@ static bool ggml_metal_graph_compute( MTLCommandBufferStatus status = [command_buffer status]; if (status != MTLCommandBufferStatusCompleted) { GGML_METAL_LOG_INFO("%s: command buffer %d failed with status %lu\n", __func__, i, status); - return false; + return GGML_STATUS_FAILED; } } @@ -2493,7 +2493,7 @@ static bool ggml_metal_graph_compute( } } - return true; + return GGML_STATUS_SUCCESS; } //////////////////////////////////////////////////////////////////////////////// @@ -2795,7 +2795,7 @@ GGML_CALL static ggml_backend_buffer_type_t ggml_backend_metal_get_default_buffe UNUSED(backend); } -GGML_CALL static bool ggml_backend_metal_graph_compute(ggml_backend_t backend, struct ggml_cgraph * cgraph) { +GGML_CALL static enum ggml_status ggml_backend_metal_graph_compute(ggml_backend_t backend, struct ggml_cgraph * cgraph) { struct ggml_metal_context * metal_ctx = (struct ggml_metal_context *)backend->context; return ggml_metal_graph_compute(metal_ctx, cgraph); diff --git a/ggml-opencl.cpp b/ggml-opencl.cpp index df619a884..aa73d67df 100644 --- a/ggml-opencl.cpp +++ b/ggml-opencl.cpp @@ -2231,7 +2231,7 @@ static ggml_backend_buffer_type_t ggml_backend_opencl_get_default_buffer_type(gg GGML_UNUSED(backend); } -static bool ggml_backend_opencl_graph_compute(ggml_backend_t backend, ggml_cgraph * graph) { +static ggml_status ggml_backend_opencl_graph_compute(ggml_backend_t backend, ggml_cgraph * graph) { for (int i = 0; i < graph->n_nodes; ++i) { ggml_tensor * node = graph->nodes[i]; switch (node->op) { @@ -2246,7 +2246,7 @@ static bool ggml_backend_opencl_graph_compute(ggml_backend_t backend, ggml_cgrap } } - return true; + return GGML_STATUS_SUCCESS; GGML_UNUSED(backend); } diff --git a/ggml-sycl.cpp b/ggml-sycl.cpp index cad08d610..47a605b01 100644 --- a/ggml-sycl.cpp +++ b/ggml-sycl.cpp @@ -15581,7 +15581,7 @@ catch (sycl::exception const &exc) { std::exit(1); } -GGML_CALL static bool ggml_backend_sycl_graph_compute(ggml_backend_t backend, ggml_cgraph * cgraph) { +GGML_CALL static ggml_status ggml_backend_sycl_graph_compute(ggml_backend_t backend, ggml_cgraph * cgraph) { ggml_backend_sycl_context * sycl_ctx = (ggml_backend_sycl_context *)backend->context; ggml_sycl_set_main_device(sycl_ctx->device); @@ -15613,7 +15613,7 @@ GGML_CALL static bool ggml_backend_sycl_graph_compute(ggml_backend_t backend, gg GGML_ASSERT(ok); } - return true; + return GGML_STATUS_SUCCESS; } GGML_CALL static bool ggml_backend_sycl_supports_op(ggml_backend_t backend, const ggml_tensor * op) { diff --git a/ggml-vulkan.cpp b/ggml-vulkan.cpp index ae9cb3c1c..bc316c3f3 100644 --- a/ggml-vulkan.cpp +++ b/ggml-vulkan.cpp @@ -5092,7 +5092,7 @@ GGML_CALL static void ggml_backend_vk_synchronize(ggml_backend_t backend) { ctx->transfer_ctx = nullptr; } -GGML_CALL static bool ggml_backend_vk_graph_compute(ggml_backend_t backend, ggml_cgraph * cgraph) { +GGML_CALL static ggml_status ggml_backend_vk_graph_compute(ggml_backend_t backend, ggml_cgraph * cgraph) { ggml_backend_vk_context * ctx = (ggml_backend_vk_context *)backend->context; for (int i = 0; i < cgraph->n_nodes; i++) { @@ -5135,7 +5135,7 @@ GGML_CALL static bool ggml_backend_vk_graph_compute(ggml_backend_t backend, ggml ggml_vk_graph_cleanup(ctx); - return true; + return GGML_STATUS_SUCCESS; UNUSED(backend); } diff --git a/ggml.c b/ggml.c index 870e41614..3c2e94c1b 100644 --- a/ggml.c +++ b/ggml.c @@ -320,6 +320,16 @@ static ggml_fp16_t ggml_table_exp_f16[1 << 16]; // precomputed f32 table for f16 (256 KB) (ggml-impl.h) float ggml_table_f32_f16[1 << 16]; +const char * ggml_status_to_string(enum ggml_status status) { + switch (status) { + case GGML_STATUS_ALLOC_FAILED: return "GGML status: error (failed to allocate memory)"; + case GGML_STATUS_FAILED: return "GGML status: error (operation failed)"; + case GGML_STATUS_SUCCESS: return "GGML status: success"; + case GGML_STATUS_ABORTED: return "GGML status: warning (operation aborted)"; + default: GGML_ASSERT(false); + } +} + // note: do not use these inside ggml.c // these are meant to be used via the ggml.h API float ggml_fp16_to_fp32(ggml_fp16_t x) { @@ -17400,6 +17410,7 @@ struct ggml_compute_state { ggml_thread_t thrd; int ith; struct ggml_compute_state_shared * shared; + enum ggml_status ec; }; static void ggml_graph_compute_perf_stats_node(struct ggml_tensor * node, const struct ggml_compute_state_shared * st) { @@ -17693,7 +17704,8 @@ static thread_ret_t ggml_graph_compute_thread(void * data) { while (true) { if (cplan->abort_callback && cplan->abort_callback(cplan->abort_callback_data)) { state->shared->node_n += 1; - return (thread_ret_t) GGML_EXIT_ABORTED; + state->ec = GGML_STATUS_ABORTED; + return 0; } if (atomic_fetch_sub(&state->shared->n_active, 1) == 1) { @@ -17815,7 +17827,7 @@ static thread_ret_t ggml_graph_compute_thread(void * data) { } } - return GGML_EXIT_SUCCESS; + return 0; } struct ggml_cplan ggml_graph_plan(const struct ggml_cgraph * cgraph, int n_threads) { @@ -18011,7 +18023,7 @@ struct ggml_cplan ggml_graph_plan(const struct ggml_cgraph * cgraph, int n_threa return cplan; } -int ggml_graph_compute(struct ggml_cgraph * cgraph, struct ggml_cplan * cplan) { +enum ggml_status ggml_graph_compute(struct ggml_cgraph * cgraph, struct ggml_cplan * cplan) { { GGML_ASSERT(cplan); GGML_ASSERT(cplan->n_threads > 0); @@ -18055,6 +18067,7 @@ int ggml_graph_compute(struct ggml_cgraph * cgraph, struct ggml_cplan * cplan) { .thrd = 0, .ith = j, .shared = &state_shared, + .ec = GGML_STATUS_SUCCESS, }; const int rc = ggml_thread_create(&workers[j].thrd, NULL, ggml_graph_compute_thread, &workers[j]); @@ -18065,12 +18078,14 @@ int ggml_graph_compute(struct ggml_cgraph * cgraph, struct ggml_cplan * cplan) { workers[0].ith = 0; workers[0].shared = &state_shared; + workers[0].ec = GGML_STATUS_SUCCESS; const int64_t perf_start_cycles = ggml_perf_cycles(); const int64_t perf_start_time_us = ggml_perf_time_us(); // this is a work thread too - int compute_status = (size_t) ggml_graph_compute_thread(&workers[0]); + ggml_graph_compute_thread(&workers[0]); + enum ggml_status compute_status = workers[0].ec; // don't leave affinity set on the main thread clear_numa_thread_affinity(); @@ -18080,6 +18095,8 @@ int ggml_graph_compute(struct ggml_cgraph * cgraph, struct ggml_cplan * cplan) { for (int j = 1; j < n_threads; j++) { const int rc = ggml_thread_join(workers[j].thrd, NULL); GGML_ASSERT(rc == 0); + if (workers[j].ec != GGML_STATUS_SUCCESS) + compute_status = workers[j].ec; } } @@ -18107,14 +18124,14 @@ int ggml_graph_compute(struct ggml_cgraph * cgraph, struct ggml_cplan * cplan) { return compute_status; } -void ggml_graph_compute_with_ctx(struct ggml_context * ctx, struct ggml_cgraph * cgraph, int n_threads) { +enum ggml_status ggml_graph_compute_with_ctx(struct ggml_context * ctx, struct ggml_cgraph * cgraph, int n_threads) { struct ggml_cplan cplan = ggml_graph_plan(cgraph, n_threads); struct ggml_object * obj = ggml_new_object(ctx, GGML_OBJECT_TYPE_WORK_BUFFER, cplan.work_size); cplan.work_data = (uint8_t *)ctx->mem_buffer + obj->offs; - ggml_graph_compute(cgraph, &cplan); + return ggml_graph_compute(cgraph, &cplan); } struct ggml_tensor * ggml_graph_get_tensor(struct ggml_cgraph * cgraph, const char * name) { diff --git a/ggml.h b/ggml.h index 98cfc7bf8..0ea4f8847 100644 --- a/ggml.h +++ b/ggml.h @@ -315,6 +315,16 @@ extern "C" { #endif + enum ggml_status { + GGML_STATUS_ALLOC_FAILED = -2, + GGML_STATUS_FAILED = -1, + GGML_STATUS_SUCCESS = 0, + GGML_STATUS_ABORTED = 1, + }; + + // get ggml_status name string + GGML_API GGML_CALL const char * ggml_status_to_string(enum ggml_status status); + typedef uint16_t ggml_fp16_t; // convert FP16 <-> FP32 @@ -1940,12 +1950,11 @@ extern "C" { // ggml_graph_plan() has to be called before ggml_graph_compute() // when plan.work_size > 0, caller must allocate memory for plan.work_data - GGML_API struct ggml_cplan ggml_graph_plan (const struct ggml_cgraph * cgraph, int n_threads /*= GGML_DEFAULT_N_THREADS*/); - GGML_API int ggml_graph_compute( struct ggml_cgraph * cgraph, struct ggml_cplan * cplan); - + GGML_API struct ggml_cplan ggml_graph_plan (const struct ggml_cgraph * cgraph, int n_threads /*= GGML_DEFAULT_N_THREADS*/); + GGML_API enum ggml_status ggml_graph_compute ( struct ggml_cgraph * cgraph, struct ggml_cplan * cplan); // same as ggml_graph_compute() but the work data is allocated as a part of the context // note: the drawback of this API is that you must have ensured that the context has enough memory for the work data - GGML_API void ggml_graph_compute_with_ctx(struct ggml_context * ctx, struct ggml_cgraph * cgraph, int n_threads); + GGML_API enum ggml_status ggml_graph_compute_with_ctx(struct ggml_context * ctx, struct ggml_cgraph * cgraph, int n_threads); GGML_API struct ggml_tensor * ggml_graph_get_tensor(struct ggml_cgraph * cgraph, const char * name); From efd8533ef8d0752cef7119eb5dbee412c4dba270 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Mon, 4 Mar 2024 11:06:39 +0200 Subject: [PATCH 534/812] sync : ggml ggml-ci --- scripts/sync-ggml.last | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/sync-ggml.last b/scripts/sync-ggml.last index 4a1b0bab4..afe68077d 100644 --- a/scripts/sync-ggml.last +++ b/scripts/sync-ggml.last @@ -1 +1 @@ -274680868e12427373bab4bec87554431b954704 +8695910a39102609073d0e099aa7c97d6bcb3bf9 From a1c6d96ed8f906aa1cda439f7386b1171a22bf9f Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Mon, 4 Mar 2024 20:53:27 +0200 Subject: [PATCH 535/812] ggml : fix unknown status (#0) --- ggml.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/ggml.c b/ggml.c index 3c2e94c1b..6a10bbcb4 100644 --- a/ggml.c +++ b/ggml.c @@ -323,11 +323,12 @@ float ggml_table_f32_f16[1 << 16]; const char * ggml_status_to_string(enum ggml_status status) { switch (status) { case GGML_STATUS_ALLOC_FAILED: return "GGML status: error (failed to allocate memory)"; - case GGML_STATUS_FAILED: return "GGML status: error (operation failed)"; - case GGML_STATUS_SUCCESS: return "GGML status: success"; - case GGML_STATUS_ABORTED: return "GGML status: warning (operation aborted)"; - default: GGML_ASSERT(false); + case GGML_STATUS_FAILED: return "GGML status: error (operation failed)"; + case GGML_STATUS_SUCCESS: return "GGML status: success"; + case GGML_STATUS_ABORTED: return "GGML status: warning (operation aborted)"; } + + return "GGML status: unknown"; } // note: do not use these inside ggml.c From e0843afe1b37890b631bc7d3d2da2ed36c862b91 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Mon, 4 Mar 2024 21:50:50 +0200 Subject: [PATCH 536/812] flake : fix --- convert-hf-to-gguf.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/convert-hf-to-gguf.py b/convert-hf-to-gguf.py index ffdba7444..f6369af38 100755 --- a/convert-hf-to-gguf.py +++ b/convert-hf-to-gguf.py @@ -36,8 +36,10 @@ class SentencePieceTokenTypes(IntEnum): UNUSED = 5 BYTE = 6 + AnyModel = TypeVar("AnyModel", bound="type[Model]") + class Model(ABC): _model_classes: dict[str, type[Model]] = {} @@ -187,6 +189,7 @@ class Model(ABC): @classmethod def register(cls, *names: str) -> Callable[[AnyModel], AnyModel]: assert names + def func(modelcls: type[Model]): for name in names: cls._model_classes[name] = modelcls From 29ae62d2ae163e2b68aa0ad3bf2ab4636de0c957 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Mon, 4 Mar 2024 22:31:20 +0200 Subject: [PATCH 537/812] llama : fix embeddings (#5796) * llama : fix embeddings ggml-ci * llama : do not use KV cache for non-causal models ggml-ci * embeddings : fix llama_batch_init arg * llama : add pooling switch * llama : distinguish token vs sequence embeddings ggml-ci * llama : assert pooling tensor * llama : simplify causal mask condition ggml-ci * llama : assert input batch with pooling enabled * readme : update API changes list --- README.md | 1 + common/common.cpp | 2 +- examples/embedding/embedding.cpp | 28 ++- examples/server-embd.py | 34 +++ examples/server/server.cpp | 53 ++++- llama.cpp | 357 +++++++++++++++++++++---------- llama.h | 18 +- 7 files changed, 359 insertions(+), 134 deletions(-) create mode 100644 examples/server-embd.py diff --git a/README.md b/README.md index 45c5d06f3..f754022de 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,7 @@ Inference of Meta's [LLaMA](https://arxiv.org/abs/2302.13971) model (and others) ### Recent API changes +- [2024 Mar 4] Embeddings API updated https://github.com/ggerganov/llama.cpp/pull/5796 - [2024 Mar 3] `struct llama_context_params` https://github.com/ggerganov/llama.cpp/pull/5849 ### Hot topics diff --git a/common/common.cpp b/common/common.cpp index 036a98134..c244db644 100644 --- a/common/common.cpp +++ b/common/common.cpp @@ -1292,7 +1292,7 @@ struct llama_context_params llama_context_params_from_gpt_params(const gpt_param cparams.n_threads_batch = params.n_threads_batch == -1 ? params.n_threads : params.n_threads_batch; cparams.seed = params.seed; cparams.logits_all = params.logits_all; - cparams.embedding = params.embedding; + cparams.embeddings = params.embedding; cparams.rope_scaling_type = params.rope_scaling_type; cparams.rope_freq_base = params.rope_freq_base; cparams.rope_freq_scale = params.rope_freq_scale; diff --git a/examples/embedding/embedding.cpp b/examples/embedding/embedding.cpp index acff715e9..ff5883da6 100644 --- a/examples/embedding/embedding.cpp +++ b/examples/embedding/embedding.cpp @@ -19,11 +19,11 @@ static std::vector split_lines(const std::string & s) { static void batch_add_seq(llama_batch & batch, const std::vector & tokens, int seq_id) { for (size_t i = 0; i < tokens.size(); i++) { - llama_batch_add(batch, tokens[i], i, { seq_id }, false); + llama_batch_add(batch, tokens[i], i, { seq_id }, i == tokens.size() - 1); } } -static void normalize(float * vec, float * out, int n) { +static void normalize(const float * vec, float * out, int n) { float norm = 0; for (int i = 0; i < n; i++) { norm += vec[i] * vec[i]; @@ -45,10 +45,23 @@ static void batch_decode(llama_context * ctx, llama_batch & batch, float * outpu } // normalize on copy - for (int k = 0; k < n_seq; k++) { - float * emb = llama_get_embeddings_ith(ctx, k); - float * out = output + k * n_embd; - normalize(emb, out, n_embd); + for (int i = 0; i < batch.n_tokens; i++) { + if (!batch.logits[i]) { + continue; + } + + // try to get sequence embeddings - supported only when pooling_type is not NONE + const float * embd = llama_get_embeddings_seq(ctx, batch.seq_id[i][0]); + if (embd == NULL) { + embd = llama_get_embeddings_ith(ctx, i); + if (embd == NULL) { + fprintf(stderr, "%s: failed to get embeddings for token %d\n", __func__, i); + continue; + } + } + + float * out = output + batch.seq_id[i][0] * n_embd; + normalize(embd, out, n_embd); } } @@ -132,7 +145,7 @@ int main(int argc, char ** argv) { // initialize batch const int n_prompts = prompts.size(); - struct llama_batch batch = llama_batch_init(n_batch, 0, n_prompts); + struct llama_batch batch = llama_batch_init(n_batch, 0, 1); // allocate output const int n_embd = llama_n_embd(model); @@ -145,6 +158,7 @@ int main(int argc, char ** argv) { for (int k = 0; k < n_prompts; k++) { // clamp to n_batch tokens auto & inp = inputs[k]; + const uint64_t n_toks = inp.size(); // encode if at capacity diff --git a/examples/server-embd.py b/examples/server-embd.py new file mode 100644 index 000000000..c5c4ea87b --- /dev/null +++ b/examples/server-embd.py @@ -0,0 +1,34 @@ +import asyncio +import requests +import numpy as np + +n = 8 + +result = [] + +async def requests_post_async(*args, **kwargs): + return await asyncio.to_thread(requests.post, *args, **kwargs) + +async def main(): + model_url = "http://127.0.0.1:6900" + responses: list[requests.Response] = await asyncio.gather(*[requests_post_async( + url= f"{model_url}/embedding", + json= {"content": str(i)*1024} + ) for i in range(n)]) + + for response in responses: + embedding = response.json()["embedding"] + print(embedding[-8:]) + result.append(embedding) + +asyncio.run(main()) + +# compute cosine similarity + +for i in range(n-1): + for j in range(i+1, n): + embedding1 = np.array(result[i]) + embedding2 = np.array(result[j]) + similarity = np.dot(embedding1, embedding2) / (np.linalg.norm(embedding1) * np.linalg.norm(embedding2)) + print(f"Similarity between {i} and {j}: {similarity:.2f}") + diff --git a/examples/server/server.cpp b/examples/server/server.cpp index 208edd571..8fe5e0b19 100644 --- a/examples/server/server.cpp +++ b/examples/server/server.cpp @@ -1210,7 +1210,7 @@ struct llama_server_context queue_results.send(res); } - void send_embedding(server_slot &slot) + void send_embedding(server_slot & slot, const llama_batch & batch) { task_result res; res.id = slot.task_id; @@ -1219,6 +1219,7 @@ struct llama_server_context res.stop = true; const int n_embd = llama_n_embd(model); + if (!params.embedding) { LOG_WARNING("embedding disabled", {{"params.embedding", params.embedding}}); @@ -1229,12 +1230,29 @@ struct llama_server_context } else { - const float *data = llama_get_embeddings(ctx); - std::vector embedding(data, data + n_embd); - res.result_json = json - { - {"embedding", embedding}, - }; + for (int i = 0; i < batch.n_tokens; ++i) { + if (!batch.logits[i] || batch.seq_id[i][0] != slot.id) { + continue; + } + + const float * embd = llama_get_embeddings_seq(ctx, batch.seq_id[i][0]); + if (embd == NULL) { + embd = llama_get_embeddings_ith(ctx, i); + if (embd == NULL) { + LOG_ERROR("failed to get embeddings for token", {{"token", batch.token[i]}, {"seq_id", batch.seq_id[i][0]}}); + res.result_json = json + { + {"embedding", std::vector(n_embd, 0.0f)}, + }; + continue; + } + } + + res.result_json = json + { + {"embedding", std::vector(embd, embd + n_embd)}, + }; + } } queue_results.send(res); } @@ -1845,7 +1863,7 @@ struct llama_server_context ga_i += ga_w/ga_n; } } - llama_batch_add(batch, prefix_tokens[slot.n_past], system_tokens.size() + slot_npast, {slot.id }, false); + llama_batch_add(batch, prefix_tokens[slot.n_past], system_tokens.size() + slot_npast, { slot.id }, false); slot_npast++; } @@ -1881,7 +1899,7 @@ struct llama_server_context for (int32_t i = 0; i < (int32_t) batch.n_tokens; i += n_batch) { - const int32_t n_tokens = std::min(n_batch, (int32_t) (batch.n_tokens - i)); + const int32_t n_tokens = std::min(n_batch, batch.n_tokens - i); for (auto & slot : slots) { @@ -1954,7 +1972,7 @@ struct llama_server_context // prompt evaluated for embedding if (slot.embedding) { - send_embedding(slot); + send_embedding(slot, batch_view); slot.release(); slot.i_batch = -1; continue; @@ -2036,6 +2054,8 @@ static void server_print_usage(const char *argv0, const gpt_params ¶ms, printf(" --yarn-attn-factor N YaRN: scale sqrt(t) or attention magnitude (default: 1.0)\n"); printf(" --yarn-beta-slow N YaRN: high correction dim or alpha (default: %.1f)\n", params.yarn_beta_slow); printf(" --yarn-beta-fast N YaRN: low correction dim or beta (default: %.1f)\n", params.yarn_beta_fast); + printf(" --pooling {none,mean,cls}\n"); + printf(" pooling type for embeddings, use model default if unspecified\n"); printf(" -b N, --batch-size N batch size for prompt processing (default: %d)\n", params.n_batch); printf(" --memory-f32 use f32 instead of f16 for memory key+value (default: disabled)\n"); printf(" not recommended: doubles context memory required and no measurable increase in quality\n"); @@ -2276,6 +2296,18 @@ static void server_params_parse(int argc, char **argv, server_params &sparams, } params.yarn_beta_slow = std::stof(argv[i]); } + else if (arg == "--pooling") + { + if (++i >= argc) { + invalid_param = true; + break; + } + std::string value(argv[i]); + /**/ if (value == "none") { params.pooling_type = LLAMA_POOLING_TYPE_NONE; } + else if (value == "mean") { params.pooling_type = LLAMA_POOLING_TYPE_MEAN; } + else if (value == "cls") { params.pooling_type = LLAMA_POOLING_TYPE_CLS; } + else { invalid_param = true; break; } + } else if (arg == "--threads" || arg == "-t") { if (++i >= argc) @@ -2330,7 +2362,6 @@ static void server_params_parse(int argc, char **argv, server_params &sparams, break; } params.n_batch = std::stoi(argv[i]); - params.n_batch = std::min(512, params.n_batch); } else if (arg == "--gpu-layers" || arg == "-ngl" || arg == "--n-gpu-layers") { diff --git a/llama.cpp b/llama.cpp index de579d9e3..76afcbc13 100644 --- a/llama.cpp +++ b/llama.cpp @@ -1665,7 +1665,7 @@ struct llama_hparams { }; struct llama_cparams { - uint32_t n_ctx; // context size used during inference + uint32_t n_ctx; // context size used during inference uint32_t n_batch; uint32_t n_threads; // number of threads to use for generation uint32_t n_threads_batch; // number of threads to use for batch processing @@ -1682,7 +1682,9 @@ struct llama_cparams { float yarn_beta_slow; float defrag_thold; + bool embeddings; bool offload_kqv; + enum llama_pooling_type pooling_type; ggml_backend_sched_eval_callback cb_eval; @@ -1972,7 +1974,7 @@ struct llama_context { int32_t n_p_eval = 0; // number of tokens in eval calls for the prompt (with batch size > 1) int32_t n_eval = 0; // number of eval calls - // decode output (2-dimensional array: [n_tokens][n_vocab]) + // logits output (2-dimensional array: [n_tokens][n_vocab]) std::vector logits; #ifndef NDEBUG // guard against access to unset logits @@ -1980,8 +1982,13 @@ struct llama_context { #endif bool logits_all = false; - // input embedding (1-dimensional array: [n_embd]) - std::vector embedding; + // embeddings output (2-dimensional array: [n_tokens][n_embd]) + // populated only when pooling_type == LLAMA_POOLING_TYPE_NONE + std::vector embd; + + // sequence embeddings output (map of [n_embd] vectors) + // populated only when pooling_type != LLAMA_POOLING_TYPE_NONE + std::map> embd_seq; // memory buffers used to evaluate the model std::vector buf_compute_meta; @@ -5092,6 +5099,7 @@ static struct ggml_tensor * llm_build_kv( llm_build_kv_store(ctx, hparams, kv, graph, k_cur, v_cur, n_ctx, n_tokens, kv_head, cb, il); struct ggml_tensor * cur; + cur = llm_build_kqv(ctx, model, hparams, kv, graph, wo, wo_b, q_cur, kq_mask, kq_pos, n_ctx, n_tokens, n_kv, kq_scale, cb, il); cb(cur, "kqv_out", il); @@ -6085,6 +6093,7 @@ struct llm_build_context { const int64_t n_embd_head = hparams.n_embd_head_v; const int64_t n_embd_gqa = hparams.n_embd_v_gqa(); + GGML_ASSERT(n_embd_head == hparams.n_embd_head_k); struct ggml_tensor * cur; @@ -6092,9 +6101,10 @@ struct llm_build_context { // get input vectors with right size const size_t stride1 = n_tokens * ggml_type_size(lctx.inp_tokens->type); - struct ggml_tensor * inp_pos = ggml_view_1d(ctx0, lctx.inp_pos, n_tokens, 0); + + struct ggml_tensor * inp_pos = ggml_view_1d(ctx0, lctx.inp_pos, n_tokens, 0); struct ggml_tensor * inp_mean = ggml_view_2d(ctx0, lctx.inp_mean, n_tokens, n_tokens, stride1, 0); - struct ggml_tensor * inp_cls = ggml_view_1d(ctx0, lctx.inp_cls, n_tokens, 0); + struct ggml_tensor * inp_cls = ggml_view_1d(ctx0, lctx.inp_cls, n_tokens, 0); // construct input embeddings (token, type, position) inpL = llm_build_inp_embd(ctx0, hparams, batch, model.tok_embd, lctx.inp_tokens, lctx.inp_embd, cb); @@ -6112,39 +6122,38 @@ struct llm_build_context { cb(inpL, "inp_norm", -1); // KQ_mask (mask for 1 head, it will be broadcasted to all heads) - struct ggml_tensor * KQ_mask = ggml_view_2d(ctx0, lctx.inp_KQ_mask, n_kv, n_tokens, n_kv*ggml_type_size(lctx.inp_KQ_mask->type), 0); - cb(KQ_mask, "KQ_mask", -1); // [n_kv, n_tokens] + struct ggml_tensor * KQ_mask = ggml_cont(ctx0, ggml_view_2d(ctx0, lctx.inp_KQ_mask, n_tokens, n_tokens, n_tokens*ggml_type_size(lctx.inp_KQ_mask->type), 0)); + cb(KQ_mask, "KQ_mask", -1); // [n_tokens, n_tokens] // iterate layers for (int il = 0; il < n_layer; ++il) { struct ggml_tensor * cur = inpL; + struct ggml_tensor * Qcur; + struct ggml_tensor * Kcur; + struct ggml_tensor * Vcur; + // self-attention if (model.arch == LLM_ARCH_BERT) { - struct ggml_tensor * Qcur = ggml_add(ctx0, ggml_mul_mat(ctx0, model.layers[il].wq, cur), model.layers[il].bq); + Qcur = ggml_add(ctx0, ggml_mul_mat(ctx0, model.layers[il].wq, cur), model.layers[il].bq); cb(Qcur, "Qcur", il); - struct ggml_tensor * Kcur = ggml_add(ctx0, ggml_mul_mat(ctx0, model.layers[il].wk, cur), model.layers[il].bk); + Kcur = ggml_add(ctx0, ggml_mul_mat(ctx0, model.layers[il].wk, cur), model.layers[il].bk); cb(Kcur, "Kcur", il); - struct ggml_tensor * Vcur = ggml_add(ctx0, ggml_mul_mat(ctx0, model.layers[il].wv, cur), model.layers[il].bv); + Vcur = ggml_add(ctx0, ggml_mul_mat(ctx0, model.layers[il].wv, cur), model.layers[il].bv); cb(Vcur, "Vcur", il); - // seems like we just need to do this for Q? - Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); - - cur = llm_build_kv(ctx0, model, hparams, kv_self, gf, - model.layers[il].wo, model.layers[il].bo, - Kcur, Vcur, Qcur, KQ_mask, nullptr, n_ctx, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il); - cb(cur, "kqv_out", il); + Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); + Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens); } else { // compute Q and K and RoPE them cur = ggml_mul_mat(ctx0, model.layers[il].wqkv, cur); cb(cur, "wqkv", il); - struct ggml_tensor * Qcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd, n_tokens, cur->nb[1], 0*sizeof(float)*(n_embd))); - struct ggml_tensor * Kcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd_gqa, n_tokens, cur->nb[1], 1*sizeof(float)*(n_embd))); - struct ggml_tensor * Vcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd_gqa, n_tokens, cur->nb[1], 1*sizeof(float)*(n_embd + n_embd_gqa))); + Qcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd, n_tokens, cur->nb[1], 0*sizeof(float)*(n_embd))); + Kcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd_gqa, n_tokens, cur->nb[1], 1*sizeof(float)*(n_embd))); + Vcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd_gqa, n_tokens, cur->nb[1], 1*sizeof(float)*(n_embd + n_embd_gqa))); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); @@ -6163,13 +6172,41 @@ struct llm_build_context { ext_factor, attn_factor, beta_fast, beta_slow ); cb(Kcur, "Kcur", il); - - cur = llm_build_kv(ctx0, model, hparams, kv_self, gf, - model.layers[il].wo, model.layers[il].bo, - Kcur, Vcur, Qcur, KQ_mask, nullptr, n_ctx, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il); - cb(cur, "kqv_out", il); } + struct ggml_tensor * q = ggml_permute(ctx0, Qcur, 0, 2, 1, 3); + struct ggml_tensor * k = ggml_cont(ctx0, ggml_permute(ctx0, Kcur, 0, 2, 1, 3)); + + struct ggml_tensor * kq = ggml_mul_mat(ctx0, k, q); + cb(kq, "kq", il); + + kq = ggml_soft_max_ext(ctx0, kq, KQ_mask, nullptr, 1.0f/sqrtf(float(n_embd_head)), hparams.f_max_alibi_bias); + cb(kq, "kq_soft_max_ext", il); + + struct ggml_tensor * v = ggml_cont(ctx0, ggml_transpose(ctx0, ggml_reshape_2d(ctx0, Vcur, n_embd_gqa, n_tokens))); + cb(v, "v", il); + + struct ggml_tensor * kqv = ggml_mul_mat(ctx0, ggml_reshape_3d(ctx0, v, n_tokens, n_embd_head, n_head_kv), kq); + cb(kqv, "kqv", il); + + struct ggml_tensor * kqv_merged = ggml_permute(ctx0, kqv, 0, 2, 1, 3); + cb(kqv_merged, "kqv_merged", il); + + cur = ggml_cont_2d(ctx0, kqv_merged, n_embd_gqa, n_tokens); + cb(cur, "kqv_merged_cont", il); + + ggml_build_forward_expand(gf, cur); + + cur = ggml_mul_mat(ctx0, model.layers[il].wo, cur); + if (model.layers[il].bo) { + cb(cur, "kqv_wo", il); + } + + if (model.layers[il].bo) { + cur = ggml_add(ctx0, cur, model.layers[il].bo); + } + cb(cur, "kqv_out", il); + // re-add the layer input cur = ggml_add(ctx0, cur, inpL); @@ -6209,16 +6246,29 @@ struct llm_build_context { // final output cur = inpL; + cb(cur, "result_embd", -1); // pooling layer - if (pooling_type == LLAMA_POOLING_TYPE_MEAN) { - cur = ggml_mul_mat(ctx0, ggml_cont(ctx0, ggml_transpose(ctx0, cur)), inp_mean); - } else if (pooling_type == LLAMA_POOLING_TYPE_CLS) { - cur = ggml_get_rows(ctx0, cur, inp_cls); - } else { - GGML_ASSERT(pooling_type == LLAMA_POOLING_TYPE_NONE && "Invalid pooling type"); + switch (pooling_type) { + case LLAMA_POOLING_TYPE_NONE: + { + // nop + } break; + case LLAMA_POOLING_TYPE_MEAN: + { + cur = ggml_mul_mat(ctx0, ggml_cont(ctx0, ggml_transpose(ctx0, cur)), inp_mean); + cb(cur, "result_embd_pooled", -1); + } break; + case LLAMA_POOLING_TYPE_CLS: + { + cur = ggml_get_rows(ctx0, cur, inp_cls); + cb(cur, "result_embd_pooled", -1); + } break; + case LLAMA_POOLING_TYPE_UNSPECIFIED: + { + GGML_ASSERT(false && "Invalid pooling type"); + } break; } - cb(cur, "result_embd", -1); ggml_build_forward_expand(gf, cur); @@ -7980,7 +8030,7 @@ static void llama_set_inputs(llama_context & lctx, const llama_batch & batch) { ggml_backend_tensor_set(lctx.inp_pos, batch.pos, 0, n_tokens*ggml_element_size(lctx.inp_pos)); } - { + if (hparams.causal_attn) { const int64_t n_kv = kv_self.n; const int64_t n_tokens = batch.n_tokens; @@ -7995,16 +8045,40 @@ static void llama_set_inputs(llama_context & lctx, const llama_batch & batch) { for (int i = 0; i < n_kv; ++i) { float f; - if (!lctx.kv_self.cells[i].has_seq_id(seq_id) || - (hparams.causal_attn && lctx.kv_self.cells[i].pos > pos)) { + if (!lctx.kv_self.cells[i].has_seq_id(seq_id) || lctx.kv_self.cells[i].pos > pos) { f = -INFINITY; } else { - f = 0; + f = 0.0f; } data[h*(n_kv*n_tokens) + j*n_kv + i] = f; } } } + } else { + // non-causal attention attends only the tokens within the batch (i.e. the KV cache is not used) + const int64_t n_tokens = batch.n_tokens; + + assert(ggml_backend_buffer_is_host(lctx.inp_KQ_mask->buffer)); + + float * data = (float *) lctx.inp_KQ_mask->data; + + for (int h = 0; h < 1; ++h) { + for (int j = 0; j < n_tokens; ++j) { + const llama_seq_id seq_id = batch.seq_id[j][0]; + + for (int i = 0; i < n_tokens; ++i) { + float f = -INFINITY; + for (int s = 0; s < batch.n_seq_id[i]; ++s) { + if (batch.seq_id[i][s] == seq_id) { + f = 0.0f; + break; + } + } + + data[h*(n_tokens*n_tokens) + j*n_tokens + i] = f; + } + } + } } if (hparams.need_kq_pos) { @@ -8023,13 +8097,16 @@ static void llama_set_inputs(llama_context & lctx, const llama_batch & batch) { const int64_t n_tokens = batch.n_tokens; GGML_ASSERT(ggml_backend_buffer_is_host(lctx.inp_mean->buffer)); - float * data = (float *) lctx.inp_mean->data; + float * data = (float *) lctx.inp_mean->data; memset(lctx.inp_mean->data, 0, n_tokens * n_tokens * ggml_element_size(lctx.inp_mean)); std::vector sum(n_tokens, 0); for (int i = 0; i < n_tokens; ++i) { const llama_seq_id seq_id = batch.seq_id[i][0]; + + GGML_ASSERT(seq_id < n_tokens && "seq_id cannot be larger than n_tokens with pooling_type == MEAN"); + sum[seq_id] += 1; } @@ -8051,11 +8128,16 @@ static void llama_set_inputs(llama_context & lctx, const llama_batch & batch) { const int64_t n_tokens = batch.n_tokens; GGML_ASSERT(ggml_backend_buffer_is_host(lctx.inp_cls->buffer)); + uint32_t * data = (uint32_t *) lctx.inp_cls->data; + memset(lctx.inp_cls->data, 0, n_tokens * ggml_element_size(lctx.inp_cls)); for (int i = 0; i < n_tokens; ++i) { const llama_seq_id seq_id = batch.seq_id[i][0]; - const llama_pos pos = batch.pos[i]; + const llama_pos pos = batch.pos[i]; + + GGML_ASSERT(seq_id < n_tokens && "seq_id cannot be larger than n_tokens with pooling_type == CLS"); + if (pos == 0) { data[seq_id] = i; } @@ -8169,24 +8251,27 @@ static int llama_decode_internal( batch.seq_id = seq_id_arr.data(); } - llama_kv_cache_update(&lctx); + // non-causal masks do not use the KV cache + if (hparams.causal_attn) { + llama_kv_cache_update(&lctx); - // if we have enough unused cells before the current head -> - // better to start searching from the beginning of the cache, hoping to fill it - if (kv_self.head > kv_self.used + 2*n_tokens) { - kv_self.head = 0; + // if we have enough unused cells before the current head -> + // better to start searching from the beginning of the cache, hoping to fill it + if (kv_self.head > kv_self.used + 2*n_tokens) { + kv_self.head = 0; + } + + if (!llama_kv_cache_find_slot(kv_self, batch)) { + return 1; + } + + // a heuristic, to avoid attending the full cache if it is not yet utilized + // after enough generations, the benefit from this heuristic disappears + // if we start defragmenting the cache, the benefit from this will be more important + kv_self.n = std::min(cparams.n_ctx, std::max(32u, GGML_PAD(llama_kv_cache_cell_max(kv_self), 32))); + //kv_self.n = llama_kv_cache_cell_max(kv_self); } - if (!llama_kv_cache_find_slot(kv_self, batch)) { - return 1; - } - - // a heuristic, to avoid attending the full cache if it is not yet utilized - // after enough generations, the benefit from this heuristic disappears - // if we start defragmenting the cache, the benefit from this will be more important - kv_self.n = std::min(cparams.n_ctx, std::max(32u, GGML_PAD(llama_kv_cache_cell_max(kv_self), 32))); - //kv_self.n = llama_kv_cache_cell_max(kv_self); - //printf("kv_self.n = %5d, kv_self.used = %5d, kv_self.head = %5d\n", kv_self.n, kv_self.used, kv_self.head); ggml_backend_sched_reset(lctx.sched); @@ -8195,20 +8280,26 @@ static int llama_decode_internal( ggml_cgraph * gf = llama_build_graph(lctx, batch, false); // the output is always the last tensor in the graph - struct ggml_tensor * res = gf->nodes[gf->n_nodes - 1]; - struct ggml_tensor * embeddings = gf->nodes[gf->n_nodes - 2]; + struct ggml_tensor * res = gf->nodes[gf->n_nodes - 1]; + struct ggml_tensor * embd = gf->nodes[gf->n_nodes - 2]; - if (strcmp(res->name, "result_output") == 0) { - // the embeddings could be the second to last tensor, or the third to last tensor - if (strcmp(embeddings->name, "result_norm") != 0) { - embeddings = gf->nodes[gf->n_nodes - 3]; - GGML_ASSERT(strcmp(embeddings->name, "result_norm") == 0); - } - } else if (strcmp(res->name, "result_embd") == 0) { - embeddings = res; - res = nullptr; + if (!hparams.causal_attn) { + res = nullptr; // do not extract logits for embedding models such as BERT + + // token or sequence embeddings + embd = gf->nodes[gf->n_nodes - 1]; + + GGML_ASSERT(strcmp(embd->name, "result_embd") == 0 || strcmp(embd->name, "result_embd_pooled") == 0); } else { - GGML_ASSERT(false); + if (strcmp(res->name, "result_output") == 0) { + // the token embeddings could be the second to last tensor, or the third to last tensor + if (strcmp(embd->name, "result_norm") != 0) { + embd = gf->nodes[gf->n_nodes - 3]; + GGML_ASSERT(strcmp(embd->name, "result_norm") == 0); + } + } else { + GGML_ASSERT(false && "missing result_output tensor"); + } } // LLAMA_LOG_INFO("graph build time: %.3f ms (%d nodes, %d leafs)\n", (ggml_time_us() - t_start_us)/1000.0, gf->n_nodes, gf->n_leafs); @@ -8275,46 +8366,82 @@ static int llama_decode_internal( logits_out.clear(); #endif - ggml_backend_t res_backend = ggml_backend_sched_get_node_backend(lctx.sched, res); - GGML_ASSERT(res_backend != nullptr); + ggml_backend_t backend_res = ggml_backend_sched_get_node_backend(lctx.sched, res); + GGML_ASSERT(backend_res != nullptr); + if (batch.logits) { logits_out.resize(n_vocab * n_tokens); for (uint32_t i = 0; i < n_tokens; i++) { if (batch.logits[i] == 0) { continue; } - ggml_backend_tensor_get_async(res_backend, res, logits_out.data() + (n_vocab*i), (n_vocab*i)*sizeof(float), n_vocab*sizeof(float)); + ggml_backend_tensor_get_async(backend_res, res, logits_out.data() + (n_vocab*i), (n_vocab*i)*sizeof(float), n_vocab*sizeof(float)); #ifndef NDEBUG logits_valid[i] = true; #endif } } else if (lctx.logits_all) { logits_out.resize(n_vocab * n_tokens); - ggml_backend_tensor_get_async(res_backend, res, logits_out.data(), 0, n_vocab*n_tokens*sizeof(float)); + ggml_backend_tensor_get_async(backend_res, res, logits_out.data(), 0, n_vocab*n_tokens*sizeof(float)); #ifndef NDEBUG std::fill(logits_valid.begin(), logits_valid.end(), true); #endif } else { logits_out.resize(n_vocab); - ggml_backend_tensor_get_async(res_backend, res, logits_out.data(), (n_vocab*(n_tokens - 1))*sizeof(float), n_vocab*sizeof(float)); + ggml_backend_tensor_get_async(backend_res, res, logits_out.data(), (n_vocab*(n_tokens - 1))*sizeof(float), n_vocab*sizeof(float)); #ifndef NDEBUG logits_valid[0] = true; #endif } - ggml_backend_synchronize(res_backend); + ggml_backend_synchronize(backend_res); } // extract embeddings - if (!lctx.embedding.empty()) { - auto & embedding_out = lctx.embedding; + if (cparams.embeddings && embd) { + ggml_backend_t backend_embd = ggml_backend_sched_get_node_backend(lctx.sched, embd); + GGML_ASSERT(backend_embd != nullptr); - const int64_t embd_pos = res ? n_embd * (n_tokens-1) : 0; - const int64_t embd_size = res ? n_embd : n_embd * n_tokens; + switch (cparams.pooling_type) { + case LLAMA_POOLING_TYPE_NONE: + { + // extract token embeddings + auto & embd_out = lctx.embd; - embedding_out.resize(embd_size); - ggml_backend_t embeddings_backend = ggml_backend_sched_get_node_backend(lctx.sched, embeddings); - ggml_backend_tensor_get_async(embeddings_backend, embeddings, embedding_out.data(), embd_pos*sizeof(float), embd_size*sizeof(float)); - ggml_backend_synchronize(embeddings_backend); + if (batch.logits) { + embd_out.resize(n_embd * n_tokens); + for (uint32_t i = 0; i < n_tokens; i++) { + if (batch.logits[i] == 0) { + continue; + } + + ggml_backend_tensor_get_async(backend_embd, embd, embd_out.data() + (n_embd*i), (n_embd*i)*sizeof(float), n_embd*sizeof(float)); + } + } + } break; + case LLAMA_POOLING_TYPE_CLS: + case LLAMA_POOLING_TYPE_MEAN: + { + GGML_ASSERT(strcmp(embd->name, "result_embd_pooled") == 0); + + // extract sequence embeddings + auto & embd_seq_out = lctx.embd_seq; + embd_seq_out.clear(); + + for (uint32_t i = 0; i < n_tokens; i++) { + const llama_seq_id seq_id = batch.seq_id[i][0]; + if (embd_seq_out.find(seq_id) != embd_seq_out.end()) { + continue; + } + embd_seq_out[seq_id].resize(n_embd); + ggml_backend_tensor_get_async(backend_embd, embd, embd_seq_out[seq_id].data(), (n_embd*seq_id)*sizeof(float), n_embd*sizeof(float)); + } + } break; + case LLAMA_POOLING_TYPE_UNSPECIFIED: + { + GGML_ASSERT(false && "unknown pooling type"); + } break; + } + ggml_backend_synchronize(backend_embd); } // measure the performance only for the single-token evals @@ -8608,19 +8735,19 @@ static uint8_t llama_token_to_byte(const llama_vocab& vocab, llama_token id) { GGML_ASSERT(llama_is_byte_token(vocab, id)); const auto& token_data = vocab.id_to_token.at(id); switch (llama_vocab_get_type(vocab)) { - case LLAMA_VOCAB_TYPE_SPM: { - auto buf = token_data.text.substr(3, 2); - return strtol(buf.c_str(), NULL, 16); - } - case LLAMA_VOCAB_TYPE_BPE: { - GGML_ASSERT(false); - return unicode_to_bytes_bpe(token_data.text); - } - case LLAMA_VOCAB_TYPE_WPM: { - GGML_ASSERT(false); - } - default: - GGML_ASSERT(false); + case LLAMA_VOCAB_TYPE_SPM: { + auto buf = token_data.text.substr(3, 2); + return strtol(buf.c_str(), NULL, 16); + } + case LLAMA_VOCAB_TYPE_BPE: { + GGML_ASSERT(false); + return unicode_to_bytes_bpe(token_data.text); + } + case LLAMA_VOCAB_TYPE_WPM: { + GGML_ASSERT(false); + } + default: + GGML_ASSERT(false); } } @@ -11864,7 +11991,7 @@ struct llama_context_params llama_context_default_params() { /*.type_k =*/ GGML_TYPE_F16, /*.type_v =*/ GGML_TYPE_F16, /*.logits_all =*/ false, - /*.embedding =*/ false, + /*.embeddings =*/ false, /*.offload_kqv =*/ true, /*.abort_callback =*/ nullptr, /*.abort_callback_data =*/ nullptr, @@ -12015,6 +12142,7 @@ struct llama_context * llama_new_context_with_model( cparams.yarn_beta_fast = params.yarn_beta_fast; cparams.yarn_beta_slow = params.yarn_beta_slow; cparams.defrag_thold = params.defrag_thold; + cparams.embeddings = params.embeddings; cparams.offload_kqv = params.offload_kqv; cparams.pooling_type = params.pooling_type; @@ -12192,8 +12320,8 @@ struct llama_context * llama_new_context_with_model( // resized during inference, reserve maximum ctx->logits.reserve(hparams.n_vocab*cparams.n_batch); - if (params.embedding) { - ctx->embedding.resize(hparams.n_embd); + if (params.embeddings) { + ctx->embd.reserve(hparams.n_embd*cparams.n_batch); } // graph inputs @@ -12628,7 +12756,7 @@ size_t llama_get_state_size(const struct llama_context * ctx) { // assume worst case for logits although only currently set ones are serialized const size_t s_logits = ctx->logits.capacity() * sizeof(float); const size_t s_embedding_size = sizeof(size_t); - const size_t s_embedding = ctx->embedding.size() * sizeof(float); + const size_t s_embedding = ctx->embd.capacity() * sizeof(float); const size_t s_kv_buf_size = sizeof(size_t); const size_t s_kv_head = sizeof(uint32_t); const size_t s_kv_size = sizeof(uint32_t); @@ -12737,12 +12865,12 @@ static void llama_copy_state_data_internal(struct llama_context * ctx, llama_dat // copy embeddings { - const size_t embedding_size = ctx->embedding.size(); + const size_t embeddings_size = ctx->embd.size(); - data_ctx->write(&embedding_size, sizeof(embedding_size)); + data_ctx->write(&embeddings_size, sizeof(embeddings_size)); - if (embedding_size) { - data_ctx->write(ctx->embedding.data(), embedding_size * sizeof(float)); + if (embeddings_size) { + data_ctx->write(ctx->embd.data(), embeddings_size * sizeof(float)); } } @@ -12846,15 +12974,17 @@ size_t llama_set_state_data(struct llama_context * ctx, const uint8_t * src) { // set embeddings { - size_t embedding_size; + size_t embeddings_size; - memcpy(&embedding_size, inp, sizeof(embedding_size)); inp += sizeof(embedding_size); + memcpy(&embeddings_size, inp, sizeof(embeddings_size)); inp += sizeof(embeddings_size); - GGML_ASSERT(ctx->embedding.capacity() == embedding_size); + GGML_ASSERT(ctx->embd.capacity() == embeddings_size); - if (embedding_size) { - memcpy(ctx->embedding.data(), inp, embedding_size * sizeof(float)); - inp += embedding_size * sizeof(float); + if (embeddings_size) { + ctx->embd.resize(embeddings_size); + + memcpy(ctx->embd.data(), inp, embeddings_size * sizeof(float)); + inp += embeddings_size * sizeof(float); } } @@ -13104,11 +13234,20 @@ float * llama_get_logits_ith(struct llama_context * ctx, int32_t i) { } float * llama_get_embeddings(struct llama_context * ctx) { - return ctx->embedding.data(); + return ctx->embd.data(); } float * llama_get_embeddings_ith(struct llama_context * ctx, int32_t i) { - return ctx->embedding.data() + i*ctx->model.hparams.n_embd; + return ctx->embd.data() + i*ctx->model.hparams.n_embd; +} + +float * llama_get_embeddings_seq(struct llama_context * ctx, llama_seq_id seq_id) { + auto it = ctx->embd_seq.find(seq_id); + if (it == ctx->embd_seq.end()) { + return nullptr; + } + + return it->second.data(); } const char * llama_token_get_text(const struct llama_model * model, llama_token token) { diff --git a/llama.h b/llama.h index 70da4cb3f..3dc162b07 100644 --- a/llama.h +++ b/llama.h @@ -163,7 +163,7 @@ extern "C" { // - embd : token embeddings (i.e. float vector of size n_embd) (used when token is NULL) // - pos : the positions of the respective token in the sequence // - seq_id : the sequence to which the respective token belongs - // - logits : if zero, the logits for the respective token will not be output + // - logits : if zero, the logits (and/or the embeddings) for the respective token will not be output // typedef struct llama_batch { int32_t n_tokens; @@ -173,7 +173,7 @@ extern "C" { llama_pos * pos; int32_t * n_seq_id; llama_seq_id ** seq_id; - int8_t * logits; + int8_t * logits; // TODO: rename this to "output" // NOTE: helpers for smooth API transition - can be deprecated in the future // for future-proof code, use the above fields instead and ignore everything below @@ -260,7 +260,7 @@ extern "C" { // Keep the booleans together to avoid misalignment during copy-by-value. bool logits_all; // the llama_decode() call computes all logits, not just the last one (DEPRECATED - set llama_batch.logits instead) - bool embedding; // embedding mode only + bool embeddings; // if true, extract embeddings (together with logits) bool offload_kqv; // whether to offload the KQV ops (including the KV cache) to GPU // Abort callback @@ -655,14 +655,20 @@ extern "C" { // llama_get_logits(ctx) + i*n_vocab LLAMA_API float * llama_get_logits_ith(struct llama_context * ctx, int32_t i); - // Get the embeddings for the input - // shape: [n_embd] (1-dimensional) + // Get all output token embeddings + // shape: [n_tokens*n_embd] (1-dimensional) LLAMA_API float * llama_get_embeddings(struct llama_context * ctx); - // Get the embeddings for the ith sequence + // Get the embeddings for the ith token // llama_get_embeddings(ctx) + i*n_embd + // shape: [n_embd] (1-dimensional) LLAMA_API float * llama_get_embeddings_ith(struct llama_context * ctx, int32_t i); + // Get the embeddings for a sequence id + // Returns NULL if pooling_type is LLAMA_POOLING_TYPE_NONE + // shape: [n_embd] (1-dimensional) + LLAMA_API float * llama_get_embeddings_seq(struct llama_context * ctx, llama_seq_id seq_id); + // // Vocab // From 1d41d6f7c2a666eb9c18a686a4684c4b03289bf3 Mon Sep 17 00:00:00 2001 From: hutli <6594598+hutli@users.noreply.github.com> Date: Tue, 5 Mar 2024 02:33:08 +0100 Subject: [PATCH 538/812] nix: static build (#5814) --- .devops/nix/package.nix | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/.devops/nix/package.nix b/.devops/nix/package.nix index 815db6a2d..01c99185b 100644 --- a/.devops/nix/package.nix +++ b/.devops/nix/package.nix @@ -1,5 +1,6 @@ { lib, + glibc, config, stdenv, mkShell, @@ -30,6 +31,11 @@ useRocm ? config.rocmSupport, useVulkan ? false, llamaVersion ? "0.0.0", # Arbitrary version, substituted by the flake + + # It's necessary to consistently use backendStdenv when building with CUDA support, + # otherwise we get libstdc++ errors downstream. + effectiveStdenv ? if useCuda then cudaPackages.backendStdenv else stdenv, + enableStatic ? effectiveStdenv.hostPlatform.isStatic }@inputs: let @@ -40,11 +46,8 @@ let strings versionOlder ; - - # It's necessary to consistently use backendStdenv when building with CUDA support, - # otherwise we get libstdc++ errors downstream. + stdenv = throw "Use effectiveStdenv instead"; - effectiveStdenv = if useCuda then cudaPackages.backendStdenv else inputs.stdenv; suffices = lib.optionals useBlas [ "BLAS" ] @@ -167,6 +170,9 @@ effectiveStdenv.mkDerivation ( # TODO: Replace with autoAddDriverRunpath # once https://github.com/NixOS/nixpkgs/pull/275241 has been merged cudaPackages.autoAddOpenGLRunpathHook + ] + ++ optionals (effectiveStdenv.hostPlatform.isGnu && enableStatic) [ + glibc.static ]; buildInputs = @@ -181,7 +187,7 @@ effectiveStdenv.mkDerivation ( [ (cmakeBool "LLAMA_NATIVE" false) (cmakeBool "LLAMA_BUILD_SERVER" true) - (cmakeBool "BUILD_SHARED_LIBS" true) + (cmakeBool "BUILD_SHARED_LIBS" (!enableStatic)) (cmakeBool "CMAKE_SKIP_BUILD_RPATH" true) (cmakeBool "LLAMA_BLAS" useBlas) (cmakeBool "LLAMA_CLBLAST" useOpenCL) @@ -190,6 +196,7 @@ effectiveStdenv.mkDerivation ( (cmakeBool "LLAMA_METAL" useMetalKit) (cmakeBool "LLAMA_MPI" useMpi) (cmakeBool "LLAMA_VULKAN" useVulkan) + (cmakeBool "LLAMA_STATIC" enableStatic) ] ++ optionals useCuda [ ( From 29eee404746e4696143a4f3a642660a4793a15d8 Mon Sep 17 00:00:00 2001 From: Jeffrey Quesnelle Date: Mon, 4 Mar 2024 19:23:06 -0800 Subject: [PATCH 539/812] fix speculative decoding build on windows (#5874) --- examples/speculative/speculative.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/speculative/speculative.cpp b/examples/speculative/speculative.cpp index 85bc0a762..e991b8846 100644 --- a/examples/speculative/speculative.cpp +++ b/examples/speculative/speculative.cpp @@ -226,7 +226,7 @@ int main(int argc, char ** argv) { while (active_seqs.size() > 0) { // randomly select a sequence to verify from active sequences - std::uniform_int_distribution u_int_dist(0, active_seqs.size() - 1); + std::uniform_int_distribution u_int_dist(0, active_seqs.size() - 1); int s = *std::next(active_seqs.begin(), u_int_dist(rng)); if (i_dft >= (int) drafts[s].tokens.size()) { drafts[s].active = false; From 6a87ac3a52668e117d97bcea07b529c93188b303 Mon Sep 17 00:00:00 2001 From: Minsoo Cheong <54794500+mscheong01@users.noreply.github.com> Date: Tue, 5 Mar 2024 15:12:23 +0900 Subject: [PATCH 540/812] fix editorconfig check break (#5879) --- .devops/nix/package.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.devops/nix/package.nix b/.devops/nix/package.nix index 01c99185b..c7fa2203a 100644 --- a/.devops/nix/package.nix +++ b/.devops/nix/package.nix @@ -46,7 +46,7 @@ let strings versionOlder ; - + stdenv = throw "Use effectiveStdenv instead"; suffices = From 21b08674331e1ea1b599f17c5ca91f0ed173be31 Mon Sep 17 00:00:00 2001 From: Neo Zhang Jianyu Date: Tue, 5 Mar 2024 16:08:35 +0800 Subject: [PATCH 541/812] [SYCL] fix mul_mat fault in CI/unit-test (#5862) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix mul_mat fault in cpy_f32_f16 * rm unused function * add wait() for memcpy * restore ci/run.sh, rename struct defination, fix bug in ggml_sycl_op_mul_mat_sycl * fix format issue * llama : fix segfault from unknown model arch name (#5820) * llama : fix segfault from unknown model arch name * llama : make all LLM maps const This also requires using `std::map::at` instead of its `operator[]` which does not exist for const maps. * llama : name LLM_ARCH_UNKNOWN to "(unknown)" This avoids errors from `std::map::at` when getting the general name of the model architecture. Using "(unknown)" instead of an empty string as per suggestion https://github.com/ggerganov/llama.cpp/pull/5820#issuecomment-1973735284 * llama : remove redundant inner const for LLM_TENSOR_NAMES The extra const won't do anything here as const maps return const references to values. Co-authored-by: Jared Van Bortel * llama : remove redundant nullptr check in llm_arch_from_string Since LLM_ARCH_NAMES is a const map, no spurious elements with a NULL name are inserted anymore, so this check is dead code. --------- Co-authored-by: Jared Van Bortel * llama : refactor internal quantization functions (#5830) * scripts : add pod-llama.sh * ggml : IQ3_S improvements (#5829) * iq3_s: somewhat faster AVX2 dot product On Ryzen a 7950X TG-128 increases to 16 t/s from 15.5 t/s using 16 threads. For 8 threads it is 13.85 t/s vs 11.75 t/s. PP-512 increases to 28.5 t/s from 23.8 t/s. * iq3_s: somewhat faster ARM_NEON dot product Still dog slow - 10.7 t/s up from 9.9 t/s. * iq3_s: another small ARM_NEON improvement 10.7 -> 11.0 t/s. Using vmulq_s8 is faster than the xor - sub trick that works best on AVX2. * iq3_s: minor improvement on Metal 49.4 t/s -> 50.3 t/s * iq3_s: PPL improvement E.g., for a context of 4096 LLaMA-v2-7B goes to 5.1340 from 5.1653. * iq3_s: use new grid everywhere * Fix ARM_NEON --------- Co-authored-by: Iwan Kawrakow * convert-hf : make model class definitions self-contained (#5825) * convert : automatically fall back to HfVocab if tokenizer.model doesn't exist (#5821) * ggml : fix IQ3_S AVX implementation (#5834) ggml-ci * llama : add abort_callback to interrupt computation (#5409) * using abort_callback from ggml to stop llama computation * format fix * a brief explaining comment --------- Co-authored-by: Georgi Gerganov * server: tests: passkey challenge / self-extend with context shift demo (#5832) * server: tests: add models endpoint scenario * server: /v1/models add some metadata * server: tests: add debug field in context before scenario * server: tests: download model from HF, add batch size * server: tests: add passkey test * server: tests: add group attention params * server: do not truncate prompt tokens if self-extend through group attention is enabled * server: logs: do not truncate log values * server: tests - passkey - first good working value of nga * server: tests: fix server timeout * server: tests: fix passkey, add doc, fix regex content matching, fix timeout * server: tests: fix regex content matching * server: tests: schedule slow tests on master * server: metrics: fix when no prompt processed * server: tests: self-extend add llama-2-7B and Mixtral-8x7B-v0.1 * server: tests: increase timeout for completion * server: tests: keep only the PHI-2 test * server: tests: passkey add a negative test * flake.lock: Update (#5842) Flake lock file updates: • Updated input 'flake-parts': 'github:hercules-ci/flake-parts/b253292d9c0a5ead9bc98c4e9a26c6312e27d69f' (2024-02-01) → 'github:hercules-ci/flake-parts/f7b3c975cf067e56e7cda6cb098ebe3fb4d74ca2' (2024-03-01) • Updated input 'flake-parts/nixpkgs-lib': 'github:NixOS/nixpkgs/97b17f32362e475016f942bbdfda4a4a72a8a652?dir=lib' (2024-01-29) → 'github:NixOS/nixpkgs/1536926ef5621b09bba54035ae2bb6d806d72ac8?dir=lib' (2024-02-29) • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/cbc4211f0afffe6dfd2478a62615dd5175a13f9a' (2024-02-23) → 'github:NixOS/nixpkgs/1536926ef5621b09bba54035ae2bb6d806d72ac8' (2024-02-29) Co-authored-by: github-actions[bot] * server : init http requests thread pool with --parallel if set (#5836) * ci : schedule slow server tests only on Release or on demand (#5839) * llama : fix llama_copy_state_data with fragmented KV cache (#5840) The row size of the saved states was based on kv_self.head while it should be based on llama_kv_cache_cell_max. Existing session files should still work. * llama : fix llama_kv_cache_cell_max inability to return 1 I've also changed its return type to uint32_t, because this function is always used to set the value of uint32_t variables, and because the index already has this type. * llama : fix state size calculation Some bytes in the state were unaccounted for in llama_get_state_size. Since the logits reserve so much space, it did not cause problems. * gguf-dump : support i-quants (#5841) Co-authored-by: Black_Fox * llama : allow for user specified embedding pooling type (#5849) * allow for user specified pooling type * llama : use enum types over int --------- Co-authored-by: Georgi Gerganov * readme : add API changes section * cuda : fix data race in soft max (#5853) * main : support special tokens as reverse/anti prompt (#5847) * Support special tokens as reverse/anti prompt. * Tokenize antiprompts only once. * main : minor --------- Co-authored-by: Georgi Gerganov * common : use LLAMA_DEFAULT_SEED (#5855) * add some new ops, fix some operators and add batch operations to certain operators. (ggml/747) * cuda: fix group_norm * cuda: add batch inference support for ggml_pad/ggml_upscale * add ggml_arrange * add ggml_timestep_embedding * update ggml_arange/ggml_timestep_embedding tests * cuda: fix im2col * add ggml_arange/ggml_timestep_embbeding support for metal backend * fix some bugs * fix some bugs * Update ggml.h Co-authored-by: Georgi Gerganov * Update ggml-cuda.cu Co-authored-by: Georgi Gerganov * Update ggml-metal.m Co-authored-by: Georgi Gerganov * Update ggml-metal.m Co-authored-by: Georgi Gerganov * Update ggml-metal.metal Co-authored-by: Georgi Gerganov * modify according to the review comments * ggml : fix compile warnings + code style * ggml : normalize compute_forward calls + fix seg fault in debug * minor --------- Co-authored-by: Georgi Gerganov Co-authored-by: slaren * sync : ggml * add alias for chat template (#5858) * speculative : implement stochastic speculative sampling (#5625) * (WIP) Implement stochastic speculative decoding * sample from residual distribution on draft accept failure * fix #5657: force greedy sampling with probs when temp is 0 * remove p_accept parameter * fix style * remove unused variables * add srand() in speculative.cpp * replace use of rand() with mt19937 sampling * fixes based on review (@JohannesGaessler) * fix r random generation * randomly select next sequence to verify + fix bug in memory freeing * fix bug in active_seqs sync * fix uniform int distribution initialization * remove warnings from comparison between int and size_t * check grammar in `llama_sample_probability_distribution_impl` * remove malloc code by utilizing vectors * add PR link to README * cmake : handle cases where git index is not found in .git (#5844) * Update CMakeLists.txt * Update CMakeLists.txt * ggml : introduce ggml_status (ggml/750) * using enum as an exit code instead of macros * update return type from enum to unsigned int * indentation fix * compound update ggml_compute_exit_code -> ggml_status changed ggml_status from a bit-field type to simple codes ggml_status to string cast * ggml_status to string cast * GGML_CALL was removed Co-authored-by: slaren --------- Co-authored-by: slaren Co-authored-by: Georgi Gerganov * sync : ggml ggml-ci * ggml : fix unknown status (#0) * flake : fix * llama : fix embeddings (#5796) * llama : fix embeddings ggml-ci * llama : do not use KV cache for non-causal models ggml-ci * embeddings : fix llama_batch_init arg * llama : add pooling switch * llama : distinguish token vs sequence embeddings ggml-ci * llama : assert pooling tensor * llama : simplify causal mask condition ggml-ci * llama : assert input batch with pooling enabled * readme : update API changes list * nix: static build (#5814) * fix speculative decoding build on windows (#5874) * rebase and rm tailing space --------- Co-authored-by: LiangtaoJin Co-authored-by: compilade <113953597+compilade@users.noreply.github.com> Co-authored-by: Jared Van Bortel Co-authored-by: Xuan Son Nguyen Co-authored-by: Georgi Gerganov Co-authored-by: Kawrakow <48489457+ikawrakow@users.noreply.github.com> Co-authored-by: Iwan Kawrakow Co-authored-by: Jared Van Bortel Co-authored-by: Michael Podvitskiy Co-authored-by: Pierrick Hymbert Co-authored-by: github-actions[bot] Co-authored-by: Nindaleth Co-authored-by: Black_Fox Co-authored-by: Douglas Hanley Co-authored-by: slaren Co-authored-by: DAN™ Co-authored-by: leejet Co-authored-by: Minsoo Cheong <54794500+mscheong01@users.noreply.github.com> Co-authored-by: Dane Madsen Co-authored-by: hutli <6594598+hutli@users.noreply.github.com> Co-authored-by: Jeffrey Quesnelle --- ggml-sycl.cpp | 1966 +++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 1810 insertions(+), 156 deletions(-) diff --git a/ggml-sycl.cpp b/ggml-sycl.cpp index 47a605b01..477f5cb02 100644 --- a/ggml-sycl.cpp +++ b/ggml-sycl.cpp @@ -2894,6 +2894,254 @@ namespace dpct using err0 = detail::generic_error_type; using err1 = detail::generic_error_type; + static inline void dpct_free(void *ptr, sycl::queue &q = get_default_queue()) { + detail::dpct_free(ptr, q); + } + + /// dpct accessor used as device function parameter. + template class accessor; + template class accessor { + public: + using memory_t = detail::memory_traits; + using element_t = typename memory_t::element_t; + using pointer_t = typename memory_t::pointer_t; + using accessor_t = typename memory_t::template accessor_t<3>; + accessor(pointer_t data, const sycl::range<3> &in_range) + : _data(data), _range(in_range) {} + template + accessor(typename std::enable_if::type &acc) + : accessor(acc, acc.get_range()) {} + accessor(const accessor_t &acc, const sycl::range<3> &in_range) + : accessor(acc.get_pointer(), in_range) {} + accessor operator[](size_t index) const { + sycl::range<2> sub(_range.get(1), _range.get(2)); + return accessor(_data + index * sub.size(), sub); + } + + pointer_t get_ptr() const { return _data; } + + private: + pointer_t _data; + sycl::range<3> _range; + }; + template class accessor { + public: + using memory_t = detail::memory_traits; + using element_t = typename memory_t::element_t; + using pointer_t = typename memory_t::pointer_t; + using accessor_t = typename memory_t::template accessor_t<2>; + accessor(pointer_t data, const sycl::range<2> &in_range) + : _data(data), _range(in_range) {} + template + accessor(typename std::enable_if::type &acc) + : accessor(acc, acc.get_range()) {} + accessor(const accessor_t &acc, const sycl::range<2> &in_range) + : accessor(acc.get_pointer(), in_range) {} + + pointer_t operator[](size_t index) const { + return _data + _range.get(1) * index; + } + + pointer_t get_ptr() const { return _data; } + + private: + pointer_t _data; + sycl::range<2> _range; + }; + + namespace detail { + /// Device variable with address space of shared, global or constant. + template class device_memory { + public: + using accessor_t = + typename detail::memory_traits::template accessor_t; + using value_t = typename detail::memory_traits::value_t; + using dpct_accessor_t = dpct::accessor; + + device_memory() : device_memory(sycl::range(1)) {} + + /// Constructor of 1-D array with initializer list + device_memory(const sycl::range &in_range, + std::initializer_list &&init_list) + : device_memory(in_range) { + assert(init_list.size() <= in_range.size()); + _host_ptr = (value_t *)std::malloc(_size); + std::memset(_host_ptr, 0, _size); + std::memcpy(_host_ptr, init_list.begin(), init_list.size() * sizeof(T)); + } + + /// Constructor of 2-D array with initializer list + template + device_memory( + const typename std::enable_if>::type &in_range, + std::initializer_list> &&init_list) + : device_memory(in_range) { + assert(init_list.size() <= in_range[0]); + _host_ptr = (value_t *)std::malloc(_size); + std::memset(_host_ptr, 0, _size); + auto tmp_data = _host_ptr; + for (auto sub_list : init_list) { + assert(sub_list.size() <= in_range[1]); + std::memcpy(tmp_data, sub_list.begin(), + sub_list.size() * sizeof(T)); + tmp_data += in_range[1]; + } + } + + /// Constructor with range + device_memory(const sycl::range &range_in) + : _size(range_in.size() * sizeof(T)), _range(range_in), + _reference(false), _host_ptr(nullptr), _device_ptr(nullptr) { + static_assert( + (Memory == global) || (Memory == constant) || (Memory == shared), + "device memory region should be global, constant or shared"); + // Make sure that singleton class mem_mgr and dev_mgr will destruct + // later than this. + detail::mem_mgr::instance(); + dev_mgr::instance(); + } + + /// Constructor with range + template + device_memory(Args... Arguments) + : device_memory(sycl::range(Arguments...)) {} + + ~device_memory() { + if (_device_ptr && !_reference) + dpct::dpct_free(_device_ptr); + if (_host_ptr) + std::free(_host_ptr); + } + + /// Allocate memory with default queue, and init memory if has initial + /// value. + void init() { init(dpct::get_default_queue()); } + /// Allocate memory with specified queue, and init memory if has initial + /// value. + void init(sycl::queue &q) { + if (_device_ptr) + return; + if (!_size) + return; + allocate_device(q); + if (_host_ptr) + detail::dpct_memcpy(q, _device_ptr, _host_ptr, _size, + host_to_device); + } + + /// The variable is assigned to a device pointer. + void assign(value_t *src, size_t size) { + this->~device_memory(); + new (this) device_memory(src, size); + } + + /// Get memory pointer of the memory object, which is virtual pointer when + /// usm is not used, and device pointer when usm is used. + value_t *get_ptr() { return get_ptr(get_default_queue()); } + /// Get memory pointer of the memory object, which is virtual pointer when + /// usm is not used, and device pointer when usm is used. + value_t *get_ptr(sycl::queue &q) { + init(q); + return _device_ptr; + } + + /// Get the device memory object size in bytes. + size_t get_size() { return _size; } + + template + typename std::enable_if::type &operator[](size_t index) { + init(); + #ifdef DPCT_USM_LEVEL_NONE + return dpct::get_buffer::type>( + _device_ptr) + .template get_access()[index]; + #else + return _device_ptr[index]; + #endif // DPCT_USM_LEVEL_NONE + } + + #ifdef DPCT_USM_LEVEL_NONE + /// Get sycl::accessor for the device memory object when usm is not used. + accessor_t get_access(sycl::handler &cgh) { + return get_buffer(_device_ptr) + .template reinterpret(_range) + .template get_access::mode, + detail::memory_traits::target>(cgh); + } + #else + /// Get dpct::accessor with dimension info for the device memory object + /// when usm is used and dimension is greater than 1. + template + typename std::enable_if::type + get_access(sycl::handler &cgh) { + return dpct_accessor_t((T *)_device_ptr, _range); + } + #endif // DPCT_USM_LEVEL_NONE + + private: + device_memory(value_t *memory_ptr, size_t size) + : _size(size), _range(size / sizeof(T)), _reference(true), + _device_ptr(memory_ptr) {} + + void allocate_device(sycl::queue &q) { + #ifndef DPCT_USM_LEVEL_NONE + if (Memory == shared) { + _device_ptr = (value_t *)sycl::malloc_shared(_size, q.get_device(), + q.get_context()); + return; + } + #ifdef SYCL_EXT_ONEAPI_USM_DEVICE_READ_ONLY + if (Memory == constant) { + _device_ptr = (value_t *)sycl::malloc_device( + _size, q.get_device(), q.get_context(), + sycl::ext::oneapi::property::usm::device_read_only()); + return; + } + #endif + #endif + _device_ptr = (value_t *)detail::dpct_malloc(_size, q); + } + + size_t _size; + sycl::range _range; + bool _reference; + value_t *_host_ptr; + value_t *_device_ptr; + }; + template + class device_memory : public device_memory { + public: + using base = device_memory; + using value_t = typename base::value_t; + using accessor_t = + typename detail::memory_traits::template accessor_t<0>; + + /// Constructor with initial value. + device_memory(const value_t &val) : base(sycl::range<1>(1), {val}) {} + + /// Default constructor + device_memory() : base(1) {} + + #ifdef DPCT_USM_LEVEL_NONE + /// Get sycl::accessor for the device memory object when usm is not used. + accessor_t get_access(sycl::handler &cgh) { + auto buf = get_buffer(base::get_ptr()) + .template reinterpret(sycl::range<1>(1)); + return accessor_t(buf, cgh); + } + #endif // DPCT_USM_LEVEL_NONE + }; + } // namespace detail + + template + using global_memory = detail::device_memory; + template + using constant_memory = detail::device_memory; + template + using shared_memory = detail::device_memory; + + } // COPY from DPCT head files @@ -2938,6 +3186,15 @@ static int g_work_group_size = 0; #pragma warning(disable: 4244 4267) // possible loss of data #endif +// dmmv = dequantize_mul_mat_vec +#ifndef GGML_SYCL_DMMV_X +#define GGML_SYCL_DMMV_X 32 +#endif +#ifndef GGML_SYCL_MMV_Y +#define GGML_SYCL_MMV_Y 1 +#endif + + static_assert(sizeof(sycl::half) == sizeof(ggml_fp16_t), "wrong fp16 size"); static void crash(){ @@ -3060,7 +3317,7 @@ typedef void (*ggml_sycl_op_flatten_t)(const ggml_tensor *src0, #define QK4_0 32 #define QR4_0 2 #define QI4_0 (QK4_0 / (4 * QR4_0)) -typedef struct dpct_type_471834 { +typedef struct dpct_type_block_q4_0 { sycl::half d; // delta uint8_t qs[QK4_0 / 2]; // nibbles / quants } block_q4_0; @@ -3069,7 +3326,7 @@ static_assert(sizeof(block_q4_0) == sizeof(ggml_fp16_t) + QK4_0 / 2, "wrong q4_0 #define QK4_1 32 #define QR4_1 2 #define QI4_1 (QK4_1 / (4 * QR4_1)) -typedef struct dpct_type_143705 { +typedef struct dpct_type_block_q4_1 { sycl::half2 dm; // dm.x = delta, dm.y = min uint8_t qs[QK4_1 / 2]; // nibbles / quants } block_q4_1; @@ -3078,7 +3335,7 @@ static_assert(sizeof(block_q4_1) == sizeof(ggml_fp16_t) * 2 + QK4_1 / 2, "wrong #define QK5_0 32 #define QR5_0 2 #define QI5_0 (QK5_0 / (4 * QR5_0)) -typedef struct dpct_type_673649 { +typedef struct dpct_type_block_q5_0 { sycl::half d; // delta uint8_t qh[4]; // 5-th bit of quants uint8_t qs[QK5_0 / 2]; // nibbles / quants @@ -3088,7 +3345,7 @@ static_assert(sizeof(block_q5_0) == sizeof(ggml_fp16_t) + sizeof(uint32_t) + QK5 #define QK5_1 32 #define QR5_1 2 #define QI5_1 (QK5_1 / (4 * QR5_1)) -typedef struct dpct_type_135589 { +typedef struct dpct_type_block_q5_1 { sycl::half2 dm; // dm.x = delta, dm.y = min uint8_t qh[4]; // 5-th bit of quants uint8_t qs[QK5_1 / 2]; // nibbles / quants @@ -3098,7 +3355,7 @@ static_assert(sizeof(block_q5_1) == 2 * sizeof(ggml_fp16_t) + sizeof(uint32_t) + #define QK8_0 32 #define QR8_0 1 #define QI8_0 (QK8_0 / (4 * QR8_0)) -typedef struct dpct_type_122878 { +typedef struct dpct_type_block_q8_0 { sycl::half d; // delta int8_t qs[QK8_0]; // quants } block_q8_0; @@ -3107,7 +3364,7 @@ static_assert(sizeof(block_q8_0) == sizeof(ggml_fp16_t) + QK8_0, "wrong q8_0 blo #define QK8_1 32 #define QR8_1 1 #define QI8_1 (QK8_1 / (4 * QR8_1)) -typedef struct dpct_type_143721 { +typedef struct dpct_type_block_q8_1 { sycl::half2 ds; // ds.x = delta, ds.y = sum int8_t qs[QK8_0]; // quants } block_q8_1; @@ -3141,7 +3398,7 @@ typedef float (*vec_dot_q_mul_mat_sycl_t)( #define QR2_K 4 #define QI2_K (QK_K / (4*QR2_K)) -typedef struct dpct_type_619598 { +typedef struct dpct_type_block_q2_K { uint8_t scales[QK_K/16]; // scales and mins, quantized with 4 bits uint8_t qs[QK_K/4]; // quants sycl::half2 dm; // super-block scale for quantized scales/mins @@ -3150,7 +3407,7 @@ static_assert(sizeof(block_q2_K) == 2*sizeof(ggml_fp16_t) + QK_K/16 + QK_K/4, "w #define QR3_K 4 #define QI3_K (QK_K / (4*QR3_K)) -typedef struct dpct_type_138576 { +typedef struct dpct_type_block_q3_K { uint8_t hmask[QK_K/8]; // quants - high bit uint8_t qs[QK_K/4]; // quants - low 2 bits #ifdef GGML_QKK_64 @@ -3166,13 +3423,13 @@ typedef struct dpct_type_138576 { #define QI4_K (QK_K / (4*QR4_K)) #ifdef GGML_QKK_64 typedef struct { - half dm[2]; // super-block scales/mins + sycl::half dm[2]; // super-block scales/mins uint8_t scales[2]; // 4-bit block scales/mins uint8_t qs[QK_K/2]; // 4--bit quants } block_q4_K; -static_assert(sizeof(block_q4_K) == sizeof(half2) + QK_K/2 + 2, "wrong q4_K block size/padding"); +static_assert(sizeof(block_q4_K) == sizeof(sycl::half2) + QK_K/2 + 2, "wrong q4_K block size/padding"); #else -typedef struct dpct_type_154943 { +typedef struct dpct_type_block_q4_K { sycl::half2 dm; // super-block scale for quantized scales/mins uint8_t scales[3*QK_K/64]; // scales, quantized with 6 bits uint8_t qs[QK_K/2]; // 4--bit quants @@ -3184,14 +3441,14 @@ static_assert(sizeof(block_q4_K) == 2*sizeof(ggml_fp16_t) + 3*QK_K/64 + QK_K/2, #define QI5_K (QK_K / (4*QR5_K)) #ifdef GGML_QKK_64 typedef struct { - half d; // super-block scale + sycl::half d; // super-block scale int8_t scales[QK_K/16]; // block scales uint8_t qh[QK_K/8]; // quants, high bit uint8_t qs[QK_K/2]; // quants, low 4 bits } block_q5_K; static_assert(sizeof(block_q5_K) == sizeof(ggml_fp16_t) + QK_K/2 + QK_K/8 + QK_K/16, "wrong q5_K block size/padding"); #else -typedef struct dpct_type_866817 { +typedef struct dpct_type_block_q5_K { sycl::half2 dm; // super-block scale for quantized scales/mins uint8_t scales[K_SCALE_SIZE]; // scales and mins, quantized with 6 bits uint8_t qh[QK_K/8]; // quants, high bit @@ -3202,7 +3459,7 @@ static_assert(sizeof(block_q5_K) == 2*sizeof(ggml_fp16_t) + K_SCALE_SIZE + QK_K/ #define QR6_K 2 #define QI6_K (QK_K / (4*QR6_K)) -typedef struct dpct_type_107281 { +typedef struct dpct_type_block_q6_K { uint8_t ql[QK_K/2]; // quants, lower 4 bits uint8_t qh[QK_K/4]; // quants, upper 2 bits int8_t scales[QK_K/16]; // scales @@ -3210,6 +3467,31 @@ typedef struct dpct_type_107281 { } block_q6_K; static_assert(sizeof(block_q6_K) == sizeof(ggml_fp16_t) + 13*QK_K/16, "wrong q6_K block size/padding"); +#define QR2_XXS 8 +#define QI2_XXS (QK_K / (4*QR2_XXS)) +typedef struct dpct_type_block_iq2_xxs { + sycl::half d; + uint16_t qs[QK_K/8]; +} block_iq2_xxs; +static_assert(sizeof(block_iq2_xxs) == sizeof(ggml_fp16_t) + QK_K/8*sizeof(uint16_t), "wrong iq2_xxs block size/padding"); + +#define QR2_XS 8 +#define QI2_XS (QK_K / (4*QR2_XS)) +typedef struct dpct_type_block_iq2_xs { + sycl::half d; + uint16_t qs[QK_K/8]; + uint8_t scales[QK_K/32]; +} block_iq2_xs; +static_assert(sizeof(block_iq2_xs) == sizeof(ggml_fp16_t) + QK_K/8*sizeof(uint16_t) + QK_K/32, "wrong iq2_xs block size/padding"); + +#define QR3_XXS 8 +#define QI3_XXS (QK_K / (4*QR3_XXS)) +typedef struct dpct_type_block_iq3_xxs { + sycl::half d; + uint8_t qs[3*(QK_K/8)]; +} block_iq3_xxs; +static_assert(sizeof(block_iq3_xxs) == sizeof(ggml_fp16_t) + 3*(QK_K/8), "wrong iq3_xxs block size/padding"); + #define WARP_SIZE 32 #define MATRIX_ROW_PADDING 512 // last row of quant. matrices is a multiple of this to avoid out-of-bounds memory accesses @@ -3478,7 +3760,7 @@ void log_ggml_var_device(const char*name, float *src, size_t total_elements, boo local_buf = (float *) ggml_sycl_host_malloc(total_size); ggml_sycl_set_device(g_main_device); dpct::queue_ptr main_stream = g_syclStreams[g_main_device][0]; - main_stream->memcpy(local_buf, src, total_size); + main_stream->memcpy(local_buf, src, total_size).wait(); } else { local_buf = (float *)src; @@ -4129,6 +4411,66 @@ static __dpct_inline__ void dequantize_q8_0(const void *vx, const int ib, #endif // GGML_SYCL_F16 } +template +static void dequantize_block_q4_0(const void * __restrict__ vx, dst_t * __restrict__ yy, int nb32, + const sycl::nd_item<3> &item_ct1) { + + const int i = item_ct1.get_group(2); + + // assume 32 threads + const int tid = item_ct1.get_local_id(2); + const int il = tid/8; + const int ir = tid%8; + const int ib = 8*i + ir; + if (ib >= nb32) { + return; + } + + dst_t * y = yy + 256*i + 32*ir + 4*il; + + const block_q4_0 * x = (const block_q4_0 *)vx + ib; + const float d = sycl::vec(x->d) + .convert()[0]; + const float dm = -8*d; + + const uint8_t * q = x->qs + 4*il; + + for (int l = 0; l < 4; ++l) { + y[l+ 0] = d * (q[l] & 0xF) + dm; + y[l+16] = d * (q[l] >> 4) + dm; + } +} + +template +static void dequantize_block_q4_1(const void * __restrict__ vx, dst_t * __restrict__ yy, int nb32, + const sycl::nd_item<3> &item_ct1) { + + const int i = item_ct1.get_group(2); + + // assume 32 threads + const int tid = item_ct1.get_local_id(2); + const int il = tid/8; + const int ir = tid%8; + const int ib = 8*i + ir; + if (ib >= nb32) { + return; + } + + dst_t * y = yy + 256*i + 32*ir + 4*il; + + const block_q4_1 * x = (const block_q4_1 *)vx + ib; + const sycl::float2 d = + x->dm.convert(); + + const uint8_t * q = x->qs + 4*il; + + for (int l = 0; l < 4; ++l) { + y[l + 0] = d.x() * (q[l] & 0xF) + d.y(); + y[l + 16] = d.x() * (q[l] >> 4) + d.y(); + } +} + + //================================== k-quants template @@ -4158,8 +4500,9 @@ static void dequantize_block_q2_K(const void * __restrict__ vx, dst_t * __restri const int il = tid%16; // 0...15 const uint8_t q = x[i].qs[il] >> (2*is); dst_t * y = yy + i*QK_K + 16*is + il; - float dall = __low2half(x[i].dm); - float dmin = __high2half(x[i].dm); + + float dall = x[i].dm[0]; + float dmin = x[i].dm[1]; y[ 0] = dall * (x[i].scales[is+0] & 0xF) * ((q >> 0) & 3) - dmin * (x[i].scales[is+0] >> 4); y[32] = dall * (x[i].scales[is+2] & 0xF) * ((q >> 4) & 3) - dmin * (x[i].scales[is+2] >> 4); #endif @@ -4198,7 +4541,7 @@ static void dequantize_block_q3_K(const void * __restrict__ vx, dst_t * __restri for (int l = l0; l < l0+4; ++l) y[l] = dl * ((int8_t)((q[l] >> shift) & 3) - ((hm[l] & m) ? 0 : 4)); #else - const int tid = threadIdx.x; + const int tid = item_ct1.get_local_id(2); const int is = tid/16; // 0 or 1 const int il = tid%16; // 0...15 const int im = il/8; // 0...1 @@ -4264,7 +4607,7 @@ static void dequantize_block_q4_K(const void * __restrict__ vx, dst_t * __restri y[l +32] = d2 * (q[l] >> 4) - m2; } #else - const int tid = threadIdx.x; + const int tid = item_ct1.get_local_id(2); const uint8_t * q = x[i].qs; dst_t * y = yy + i*QK_K; const float d = (float)x[i].dm[0]; @@ -4309,7 +4652,7 @@ static void dequantize_block_q5_K(const void * __restrict__ vx, dst_t * __restri y[32] = d2 * ((ql[ 0] >> 4) + (qh[ 0] & hm ? 16 : 0)) - m2; y[33] = d2 * ((ql[ 1] >> 4) + (qh[ 1] & hm ? 16 : 0)) - m2; #else - const int tid = threadIdx.x; + const int tid = item_ct1.get_local_id(2); const uint8_t q = x[i].qs[tid]; const int im = tid/8; // 0...3 const int in = tid%8; // 0...7 @@ -4351,7 +4694,7 @@ static void dequantize_block_q6_K(const void * __restrict__ vx, dst_t * __restri #else // assume 32 threads - const int tid = threadIdx.x; + const int tid = item_ct1.get_local_id(2); const int ip = tid/16; // 0 or 1 const int il = tid - 16*ip; // 0...15 @@ -4368,6 +4711,474 @@ static void dequantize_block_q6_K(const void * __restrict__ vx, dst_t * __restri #endif } +static dpct::global_memory + iq2xxs_grid(sycl::range<1>(256), + { + 0x0808080808080808, 0x080808080808082b, 0x0808080808081919, + 0x0808080808082b08, 0x0808080808082b2b, 0x0808080808190819, + 0x0808080808191908, 0x08080808082b0808, 0x08080808082b082b, + 0x08080808082b2b08, 0x08080808082b2b2b, 0x0808080819080819, + 0x0808080819081908, 0x0808080819190808, 0x0808080819192b08, + 0x08080808192b0819, 0x08080808192b1908, 0x080808082b080808, + 0x080808082b08082b, 0x080808082b082b2b, 0x080808082b2b082b, + 0x0808081908080819, 0x0808081908081908, 0x0808081908190808, + 0x0808081908191919, 0x0808081919080808, 0x080808192b081908, + 0x080808192b192b08, 0x0808082b08080808, 0x0808082b0808082b, + 0x0808082b082b082b, 0x0808082b2b08082b, 0x0808190808080819, + 0x0808190808081908, 0x0808190808190808, 0x08081908082b0819, + 0x08081908082b1908, 0x0808190819080808, 0x080819081908082b, + 0x0808190819082b08, 0x08081908192b0808, 0x080819082b080819, + 0x080819082b081908, 0x080819082b190808, 0x080819082b2b1908, + 0x0808191908080808, 0x080819190808082b, 0x0808191908082b08, + 0x08081919082b0808, 0x080819191908192b, 0x08081919192b2b19, + 0x080819192b080808, 0x080819192b190819, 0x0808192b08082b19, + 0x0808192b08190808, 0x0808192b19080808, 0x0808192b2b081908, + 0x0808192b2b2b1908, 0x08082b0808080808, 0x08082b0808081919, + 0x08082b0808082b08, 0x08082b0808191908, 0x08082b08082b2b08, + 0x08082b0819080819, 0x08082b0819081908, 0x08082b0819190808, + 0x08082b081919082b, 0x08082b082b082b08, 0x08082b1908081908, + 0x08082b1919080808, 0x08082b2b0808082b, 0x08082b2b08191908, + 0x0819080808080819, 0x0819080808081908, 0x0819080808190808, + 0x08190808082b0819, 0x0819080819080808, 0x08190808192b0808, + 0x081908082b081908, 0x081908082b190808, 0x081908082b191919, + 0x0819081908080808, 0x0819081908082b08, 0x08190819082b0808, + 0x0819081919190808, 0x0819081919192b2b, 0x081908192b080808, + 0x0819082b082b1908, 0x0819082b19081919, 0x0819190808080808, + 0x0819190808082b08, 0x08191908082b0808, 0x08191908082b1919, + 0x0819190819082b19, 0x081919082b080808, 0x0819191908192b08, + 0x08191919192b082b, 0x0819192b08080808, 0x0819192b0819192b, + 0x08192b0808080819, 0x08192b0808081908, 0x08192b0808190808, + 0x08192b0819080808, 0x08192b082b080819, 0x08192b1908080808, + 0x08192b1908081919, 0x08192b192b2b0808, 0x08192b2b19190819, + 0x082b080808080808, 0x082b08080808082b, 0x082b080808082b2b, + 0x082b080819081908, 0x082b0808192b0819, 0x082b08082b080808, + 0x082b08082b08082b, 0x082b0819082b2b19, 0x082b081919082b08, + 0x082b082b08080808, 0x082b082b0808082b, 0x082b190808080819, + 0x082b190808081908, 0x082b190808190808, 0x082b190819080808, + 0x082b19081919192b, 0x082b191908080808, 0x082b191919080819, + 0x082b1919192b1908, 0x082b192b2b190808, 0x082b2b0808082b08, + 0x082b2b08082b0808, 0x082b2b082b191908, 0x082b2b2b19081908, + 0x1908080808080819, 0x1908080808081908, 0x1908080808190808, + 0x1908080808192b08, 0x19080808082b0819, 0x19080808082b1908, + 0x1908080819080808, 0x1908080819082b08, 0x190808081919192b, + 0x19080808192b0808, 0x190808082b080819, 0x190808082b081908, + 0x190808082b190808, 0x1908081908080808, 0x19080819082b0808, + 0x19080819192b0819, 0x190808192b080808, 0x190808192b081919, + 0x1908082b08080819, 0x1908082b08190808, 0x1908082b19082b08, + 0x1908082b1919192b, 0x1908082b192b2b08, 0x1908190808080808, + 0x1908190808082b08, 0x19081908082b0808, 0x190819082b080808, + 0x190819082b192b19, 0x190819190819082b, 0x19081919082b1908, + 0x1908192b08080808, 0x19082b0808080819, 0x19082b0808081908, + 0x19082b0808190808, 0x19082b0819080808, 0x19082b0819081919, + 0x19082b1908080808, 0x19082b1919192b08, 0x19082b19192b0819, + 0x19082b192b08082b, 0x19082b2b19081919, 0x19082b2b2b190808, + 0x1919080808080808, 0x1919080808082b08, 0x1919080808190819, + 0x1919080808192b19, 0x19190808082b0808, 0x191908082b080808, + 0x191908082b082b08, 0x1919081908081908, 0x191908191908082b, + 0x191908192b2b1908, 0x1919082b2b190819, 0x191919082b190808, + 0x191919082b19082b, 0x1919191908082b2b, 0x1919192b08080819, + 0x1919192b19191908, 0x19192b0808080808, 0x19192b0808190819, + 0x19192b0808192b19, 0x19192b08192b1908, 0x19192b1919080808, + 0x19192b2b08082b08, 0x192b080808081908, 0x192b080808190808, + 0x192b080819080808, 0x192b0808192b2b08, 0x192b081908080808, + 0x192b081919191919, 0x192b082b08192b08, 0x192b082b192b0808, + 0x192b190808080808, 0x192b190808081919, 0x192b191908190808, + 0x192b19190819082b, 0x192b19192b081908, 0x192b2b081908082b, + 0x2b08080808080808, 0x2b0808080808082b, 0x2b08080808082b2b, + 0x2b08080819080819, 0x2b0808082b08082b, 0x2b08081908081908, + 0x2b08081908192b08, 0x2b08081919080808, 0x2b08082b08190819, + 0x2b08190808080819, 0x2b08190808081908, 0x2b08190808190808, + 0x2b08190808191919, 0x2b08190819080808, 0x2b081908192b0808, + 0x2b08191908080808, 0x2b0819191908192b, 0x2b0819192b191908, + 0x2b08192b08082b19, 0x2b08192b19080808, 0x2b08192b192b0808, + 0x2b082b080808082b, 0x2b082b1908081908, 0x2b082b2b08190819, + 0x2b19080808081908, 0x2b19080808190808, 0x2b190808082b1908, + 0x2b19080819080808, 0x2b1908082b2b0819, 0x2b1908190819192b, + 0x2b1908192b080808, 0x2b19082b19081919, 0x2b19190808080808, + 0x2b191908082b082b, 0x2b19190819081908, 0x2b19191919190819, + 0x2b192b082b080819, 0x2b192b19082b0808, 0x2b2b08080808082b, + 0x2b2b080819190808, 0x2b2b08082b081919, 0x2b2b081908082b19, + 0x2b2b082b08080808, 0x2b2b190808192b08, 0x2b2b2b0819190808, + 0x2b2b2b1908081908, + }); + +static dpct::global_memory + iq2xs_grid(sycl::range<1>(512), + { + 0x0808080808080808, 0x080808080808082b, 0x0808080808081919, + 0x0808080808082b08, 0x0808080808082b2b, 0x0808080808190819, + 0x0808080808191908, 0x080808080819192b, 0x0808080808192b19, + 0x08080808082b0808, 0x08080808082b082b, 0x08080808082b1919, + 0x08080808082b2b08, 0x0808080819080819, 0x0808080819081908, + 0x080808081908192b, 0x0808080819082b19, 0x0808080819190808, + 0x080808081919082b, 0x0808080819191919, 0x0808080819192b08, + 0x08080808192b0819, 0x08080808192b1908, 0x080808082b080808, + 0x080808082b08082b, 0x080808082b081919, 0x080808082b082b08, + 0x080808082b190819, 0x080808082b191908, 0x080808082b192b19, + 0x080808082b2b0808, 0x0808081908080819, 0x0808081908081908, + 0x080808190808192b, 0x0808081908082b19, 0x0808081908190808, + 0x080808190819082b, 0x0808081908191919, 0x0808081908192b08, + 0x0808081908192b2b, 0x08080819082b0819, 0x08080819082b1908, + 0x0808081919080808, 0x080808191908082b, 0x0808081919081919, + 0x0808081919082b08, 0x0808081919190819, 0x0808081919191908, + 0x08080819192b0808, 0x08080819192b2b08, 0x080808192b080819, + 0x080808192b081908, 0x080808192b190808, 0x0808082b08080808, + 0x0808082b0808082b, 0x0808082b08081919, 0x0808082b08082b08, + 0x0808082b08190819, 0x0808082b08191908, 0x0808082b082b0808, + 0x0808082b19080819, 0x0808082b19081908, 0x0808082b19190808, + 0x0808082b19191919, 0x0808082b2b080808, 0x0808082b2b082b2b, + 0x0808190808080819, 0x0808190808081908, 0x080819080808192b, + 0x0808190808082b19, 0x0808190808190808, 0x080819080819082b, + 0x0808190808191919, 0x0808190808192b08, 0x08081908082b0819, + 0x08081908082b1908, 0x0808190819080808, 0x080819081908082b, + 0x0808190819081919, 0x0808190819082b08, 0x0808190819190819, + 0x0808190819191908, 0x080819081919192b, 0x08081908192b0808, + 0x080819082b080819, 0x080819082b081908, 0x080819082b190808, + 0x0808191908080808, 0x080819190808082b, 0x0808191908081919, + 0x0808191908082b08, 0x0808191908190819, 0x0808191908191908, + 0x08081919082b0808, 0x0808191919080819, 0x0808191919081908, + 0x0808191919190808, 0x08081919192b0819, 0x080819192b080808, + 0x0808192b08080819, 0x0808192b08081908, 0x0808192b08190808, + 0x0808192b082b192b, 0x0808192b19080808, 0x0808192b1908082b, + 0x0808192b2b081908, 0x08082b0808080808, 0x08082b080808082b, + 0x08082b0808081919, 0x08082b0808082b08, 0x08082b0808082b2b, + 0x08082b0808190819, 0x08082b0808191908, 0x08082b08082b0808, + 0x08082b08082b1919, 0x08082b0819080819, 0x08082b0819081908, + 0x08082b0819190808, 0x08082b0819192b08, 0x08082b082b080808, + 0x08082b082b2b0808, 0x08082b082b2b2b2b, 0x08082b1908080819, + 0x08082b1908081908, 0x08082b1908190808, 0x08082b1919080808, + 0x08082b192b080819, 0x08082b192b082b19, 0x08082b2b08080808, + 0x08082b2b082b0808, 0x08082b2b082b2b08, 0x08082b2b2b19192b, + 0x08082b2b2b2b0808, 0x0819080808080819, 0x0819080808081908, + 0x081908080808192b, 0x0819080808082b19, 0x0819080808190808, + 0x081908080819082b, 0x0819080808191919, 0x0819080808192b08, + 0x08190808082b0819, 0x08190808082b1908, 0x0819080819080808, + 0x081908081908082b, 0x0819080819081919, 0x0819080819082b08, + 0x0819080819190819, 0x0819080819191908, 0x08190808192b0808, + 0x08190808192b2b2b, 0x081908082b080819, 0x081908082b081908, + 0x081908082b190808, 0x0819081908080808, 0x081908190808082b, + 0x0819081908081919, 0x0819081908082b08, 0x0819081908190819, + 0x0819081908191908, 0x08190819082b0808, 0x0819081919080819, + 0x0819081919081908, 0x0819081919190808, 0x081908192b080808, + 0x081908192b191908, 0x081908192b19192b, 0x0819082b08080819, + 0x0819082b08081908, 0x0819082b0808192b, 0x0819082b08190808, + 0x0819082b19080808, 0x0819082b192b0808, 0x0819190808080808, + 0x081919080808082b, 0x0819190808081919, 0x0819190808082b08, + 0x0819190808190819, 0x0819190808191908, 0x08191908082b0808, + 0x0819190819080819, 0x0819190819081908, 0x0819190819082b19, + 0x0819190819190808, 0x08191908192b1908, 0x081919082b080808, + 0x0819191908080819, 0x0819191908081908, 0x0819191908190808, + 0x0819191919080808, 0x0819192b08080808, 0x0819192b08191908, + 0x0819192b19082b19, 0x08192b0808080819, 0x08192b0808081908, + 0x08192b0808190808, 0x08192b080819082b, 0x08192b0819080808, + 0x08192b0819191908, 0x08192b082b08192b, 0x08192b1908080808, + 0x08192b1908081919, 0x08192b19192b192b, 0x08192b2b19190819, + 0x08192b2b2b2b2b19, 0x082b080808080808, 0x082b08080808082b, + 0x082b080808081919, 0x082b080808082b08, 0x082b080808082b2b, + 0x082b080808190819, 0x082b080808191908, 0x082b0808082b0808, + 0x082b080819080819, 0x082b080819081908, 0x082b080819190808, + 0x082b08082b080808, 0x082b08082b2b0808, 0x082b081908080819, + 0x082b081908081908, 0x082b081908190808, 0x082b081919080808, + 0x082b081919082b08, 0x082b0819192b1919, 0x082b082b08080808, + 0x082b082b082b082b, 0x082b082b2b080808, 0x082b082b2b2b2b08, + 0x082b190808080819, 0x082b190808081908, 0x082b190808190808, + 0x082b1908082b2b19, 0x082b190819080808, 0x082b191908080808, + 0x082b191919080819, 0x082b19191919082b, 0x082b19192b192b19, + 0x082b192b08080819, 0x082b192b08192b2b, 0x082b192b2b2b192b, + 0x082b2b0808080808, 0x082b2b0808082b08, 0x082b2b0808082b2b, + 0x082b2b08082b0808, 0x082b2b0819191919, 0x082b2b082b082b08, + 0x082b2b082b2b082b, 0x082b2b19192b2b08, 0x082b2b192b190808, + 0x082b2b2b08082b08, 0x082b2b2b082b0808, 0x082b2b2b2b08082b, + 0x082b2b2b2b082b08, 0x082b2b2b2b082b2b, 0x1908080808080819, + 0x1908080808081908, 0x190808080808192b, 0x1908080808082b19, + 0x1908080808190808, 0x190808080819082b, 0x1908080808191919, + 0x1908080808192b08, 0x19080808082b0819, 0x19080808082b1908, + 0x1908080819080808, 0x190808081908082b, 0x1908080819081919, + 0x1908080819082b08, 0x1908080819082b2b, 0x1908080819190819, + 0x1908080819191908, 0x19080808192b0808, 0x19080808192b1919, + 0x190808082b080819, 0x190808082b081908, 0x190808082b190808, + 0x1908081908080808, 0x190808190808082b, 0x1908081908081919, + 0x1908081908082b08, 0x1908081908190819, 0x1908081908191908, + 0x19080819082b0808, 0x1908081919080819, 0x1908081919081908, + 0x1908081919190808, 0x190808192b080808, 0x190808192b081919, + 0x190808192b2b082b, 0x1908082b08080819, 0x1908082b08081908, + 0x1908082b08190808, 0x1908082b0819082b, 0x1908082b082b2b19, + 0x1908082b19080808, 0x1908190808080808, 0x190819080808082b, + 0x1908190808081919, 0x1908190808082b08, 0x1908190808190819, + 0x1908190808191908, 0x1908190808192b19, 0x19081908082b0808, + 0x1908190819080819, 0x1908190819081908, 0x1908190819190808, + 0x190819082b080808, 0x190819082b191908, 0x1908191908080819, + 0x1908191908081908, 0x1908191908190808, 0x19081919082b1908, + 0x1908191919080808, 0x190819192b192b2b, 0x1908192b08080808, + 0x1908192b08082b2b, 0x1908192b19081908, 0x1908192b19190808, + 0x19082b0808080819, 0x19082b0808081908, 0x19082b0808190808, + 0x19082b0819080808, 0x19082b0819081919, 0x19082b0819191908, + 0x19082b08192b082b, 0x19082b1908080808, 0x19082b1908190819, + 0x19082b1919081908, 0x19082b1919190808, 0x19082b19192b2b19, + 0x19082b2b08081908, 0x1919080808080808, 0x191908080808082b, + 0x1919080808081919, 0x1919080808082b08, 0x1919080808190819, + 0x1919080808191908, 0x19190808082b0808, 0x19190808082b2b08, + 0x1919080819080819, 0x1919080819081908, 0x1919080819190808, + 0x191908082b080808, 0x1919081908080819, 0x1919081908081908, + 0x1919081908190808, 0x1919081908191919, 0x1919081919080808, + 0x191908191908082b, 0x1919082b08080808, 0x1919082b19081908, + 0x1919082b2b2b2b2b, 0x1919190808080819, 0x1919190808081908, + 0x1919190808190808, 0x19191908082b0819, 0x1919190819080808, + 0x19191908192b0808, 0x191919082b080819, 0x191919082b2b0819, + 0x1919191908080808, 0x1919191908082b08, 0x191919192b080808, + 0x191919192b082b08, 0x1919192b082b0819, 0x1919192b192b2b08, + 0x1919192b2b2b0819, 0x19192b0808080808, 0x19192b0808191908, + 0x19192b0819080819, 0x19192b0819190808, 0x19192b082b192b19, + 0x19192b1908192b2b, 0x19192b1919080808, 0x19192b191908082b, + 0x19192b2b2b081919, 0x192b080808080819, 0x192b080808081908, + 0x192b080808190808, 0x192b080819080808, 0x192b080819191908, + 0x192b0808192b082b, 0x192b08082b08192b, 0x192b08082b2b2b19, + 0x192b081908080808, 0x192b082b082b1908, 0x192b082b19082b2b, + 0x192b082b2b19082b, 0x192b190808080808, 0x192b19080819192b, + 0x192b191908190808, 0x192b191919080808, 0x192b191919081919, + 0x192b19192b2b1908, 0x192b2b0808080819, 0x192b2b08192b2b2b, + 0x192b2b19082b1919, 0x192b2b2b0808192b, 0x192b2b2b19191908, + 0x192b2b2b192b082b, 0x2b08080808080808, 0x2b0808080808082b, + 0x2b08080808081919, 0x2b08080808082b08, 0x2b08080808190819, + 0x2b08080808191908, 0x2b080808082b0808, 0x2b080808082b2b2b, + 0x2b08080819080819, 0x2b08080819081908, 0x2b08080819190808, + 0x2b0808082b080808, 0x2b0808082b08082b, 0x2b0808082b2b2b08, + 0x2b0808082b2b2b2b, 0x2b08081908080819, 0x2b08081908081908, + 0x2b0808190808192b, 0x2b08081908190808, 0x2b08081919080808, + 0x2b08081919190819, 0x2b08081919192b19, 0x2b08082b08080808, + 0x2b08082b082b0808, 0x2b08082b2b080808, 0x2b08082b2b08082b, + 0x2b08082b2b2b0808, 0x2b08082b2b2b2b08, 0x2b08190808080819, + 0x2b08190808081908, 0x2b08190808190808, 0x2b0819080819082b, + 0x2b08190808191919, 0x2b08190819080808, 0x2b081908192b0808, + 0x2b0819082b082b19, 0x2b08191908080808, 0x2b08191919081908, + 0x2b0819192b2b1919, 0x2b08192b08192b08, 0x2b08192b192b2b2b, + 0x2b082b0808080808, 0x2b082b0808082b08, 0x2b082b08082b1919, + 0x2b082b0819192b2b, 0x2b082b082b080808, 0x2b082b082b08082b, + 0x2b082b082b2b2b08, 0x2b082b190808192b, 0x2b082b2b082b082b, + 0x2b082b2b2b080808, 0x2b082b2b2b082b08, 0x2b082b2b2b19192b, + 0x2b082b2b2b2b2b08, 0x2b19080808080819, 0x2b19080808081908, + 0x2b19080808190808, 0x2b19080819080808, 0x2b1908081919192b, + 0x2b1908082b081908, 0x2b19081908080808, 0x2b190819082b082b, + 0x2b190819192b1908, 0x2b19082b1919192b, 0x2b19082b2b082b19, + 0x2b19190808080808, 0x2b19190808081919, 0x2b19190819081908, + 0x2b19190819190808, 0x2b19190819192b08, 0x2b191919082b2b19, + 0x2b1919192b190808, 0x2b1919192b19082b, 0x2b19192b19080819, + 0x2b192b0819190819, 0x2b192b082b2b192b, 0x2b192b1919082b19, + 0x2b192b2b08191919, 0x2b192b2b192b0808, 0x2b2b080808080808, + 0x2b2b08080808082b, 0x2b2b080808082b08, 0x2b2b080808082b2b, + 0x2b2b0808082b0808, 0x2b2b0808082b2b2b, 0x2b2b08082b2b0808, + 0x2b2b081919190819, 0x2b2b081919192b19, 0x2b2b08192b2b192b, + 0x2b2b082b08080808, 0x2b2b082b0808082b, 0x2b2b082b08082b08, + 0x2b2b082b082b2b2b, 0x2b2b082b2b080808, 0x2b2b082b2b2b0808, + 0x2b2b190819080808, 0x2b2b19082b191919, 0x2b2b192b192b1919, + 0x2b2b192b2b192b08, 0x2b2b2b0808082b2b, 0x2b2b2b08082b0808, + 0x2b2b2b08082b082b, 0x2b2b2b08082b2b08, 0x2b2b2b082b2b0808, + 0x2b2b2b082b2b2b08, 0x2b2b2b1908081908, 0x2b2b2b192b081908, + 0x2b2b2b192b08192b, 0x2b2b2b2b082b2b08, 0x2b2b2b2b082b2b2b, + 0x2b2b2b2b2b190819, 0x2b2b2b2b2b2b2b2b, + }); + +static dpct::global_memory iq3xxs_grid( + sycl::range<1>(256), + { + 0x04040404, 0x04040414, 0x04040424, 0x04040c0c, 0x04040c1c, 0x04040c3e, + 0x04041404, 0x04041414, 0x04041c0c, 0x04042414, 0x04043e1c, 0x04043e2c, + 0x040c040c, 0x040c041c, 0x040c0c04, 0x040c0c14, 0x040c140c, 0x040c142c, + 0x040c1c04, 0x040c1c14, 0x040c240c, 0x040c2c24, 0x040c3e04, 0x04140404, + 0x04140414, 0x04140424, 0x04140c0c, 0x04141404, 0x04141414, 0x04141c0c, + 0x04141c1c, 0x04141c3e, 0x04142c0c, 0x04142c3e, 0x04143e2c, 0x041c040c, + 0x041c043e, 0x041c0c04, 0x041c0c14, 0x041c142c, 0x041c3e04, 0x04240c1c, + 0x04241c3e, 0x04242424, 0x04242c3e, 0x04243e1c, 0x04243e2c, 0x042c040c, + 0x042c043e, 0x042c1c14, 0x042c2c14, 0x04341c2c, 0x04343424, 0x043e0c04, + 0x043e0c24, 0x043e0c34, 0x043e241c, 0x043e340c, 0x0c04040c, 0x0c04041c, + 0x0c040c04, 0x0c040c14, 0x0c04140c, 0x0c04141c, 0x0c041c04, 0x0c041c14, + 0x0c041c24, 0x0c04243e, 0x0c042c04, 0x0c0c0404, 0x0c0c0414, 0x0c0c0c0c, + 0x0c0c1404, 0x0c0c1414, 0x0c14040c, 0x0c14041c, 0x0c140c04, 0x0c140c14, + 0x0c14140c, 0x0c141c04, 0x0c143e14, 0x0c1c0404, 0x0c1c0414, 0x0c1c1404, + 0x0c1c1c0c, 0x0c1c2434, 0x0c1c3434, 0x0c24040c, 0x0c24042c, 0x0c242c04, + 0x0c2c1404, 0x0c2c1424, 0x0c2c2434, 0x0c2c3e0c, 0x0c34042c, 0x0c3e1414, + 0x0c3e2404, 0x14040404, 0x14040414, 0x14040c0c, 0x14040c1c, 0x14041404, + 0x14041414, 0x14041434, 0x14041c0c, 0x14042414, 0x140c040c, 0x140c041c, + 0x140c042c, 0x140c0c04, 0x140c0c14, 0x140c140c, 0x140c1c04, 0x140c341c, + 0x140c343e, 0x140c3e04, 0x14140404, 0x14140414, 0x14140c0c, 0x14140c3e, + 0x14141404, 0x14141414, 0x14141c3e, 0x14142404, 0x14142c2c, 0x141c040c, + 0x141c0c04, 0x141c0c24, 0x141c3e04, 0x141c3e24, 0x14241c2c, 0x14242c1c, + 0x142c041c, 0x142c143e, 0x142c240c, 0x142c3e24, 0x143e040c, 0x143e041c, + 0x143e0c34, 0x143e242c, 0x1c04040c, 0x1c040c04, 0x1c040c14, 0x1c04140c, + 0x1c04141c, 0x1c042c04, 0x1c04342c, 0x1c043e14, 0x1c0c0404, 0x1c0c0414, + 0x1c0c1404, 0x1c0c1c0c, 0x1c0c2424, 0x1c0c2434, 0x1c14040c, 0x1c14041c, + 0x1c140c04, 0x1c14142c, 0x1c142c14, 0x1c143e14, 0x1c1c0c0c, 0x1c1c1c1c, + 0x1c241c04, 0x1c24243e, 0x1c243e14, 0x1c2c0404, 0x1c2c0434, 0x1c2c1414, + 0x1c2c2c2c, 0x1c340c24, 0x1c341c34, 0x1c34341c, 0x1c3e1c1c, 0x1c3e3404, + 0x24040424, 0x24040c3e, 0x24041c2c, 0x24041c3e, 0x24042c1c, 0x24042c3e, + 0x240c3e24, 0x24141404, 0x24141c3e, 0x24142404, 0x24143404, 0x24143434, + 0x241c043e, 0x241c242c, 0x24240424, 0x24242c0c, 0x24243424, 0x242c142c, + 0x242c241c, 0x242c3e04, 0x243e042c, 0x243e0c04, 0x243e0c14, 0x243e1c04, + 0x2c040c14, 0x2c04240c, 0x2c043e04, 0x2c0c0404, 0x2c0c0434, 0x2c0c1434, + 0x2c0c2c2c, 0x2c140c24, 0x2c141c14, 0x2c143e14, 0x2c1c0414, 0x2c1c2c1c, + 0x2c240c04, 0x2c24141c, 0x2c24143e, 0x2c243e14, 0x2c2c0414, 0x2c2c1c0c, + 0x2c342c04, 0x2c3e1424, 0x2c3e2414, 0x34041424, 0x34042424, 0x34042434, + 0x34043424, 0x340c140c, 0x340c340c, 0x34140c3e, 0x34143424, 0x341c1c04, + 0x341c1c34, 0x34242424, 0x342c042c, 0x342c2c14, 0x34341c1c, 0x343e041c, + 0x343e140c, 0x3e04041c, 0x3e04042c, 0x3e04043e, 0x3e040c04, 0x3e041c14, + 0x3e042c14, 0x3e0c1434, 0x3e0c2404, 0x3e140c14, 0x3e14242c, 0x3e142c14, + 0x3e1c0404, 0x3e1c0c2c, 0x3e1c1c1c, 0x3e1c3404, 0x3e24140c, 0x3e24240c, + 0x3e2c0404, 0x3e2c0414, 0x3e2c1424, 0x3e341c04, + }); + +static dpct::global_memory ksigns_iq2xs( + sycl::range<1>(128), + { + 0, 129, 130, 3, 132, 5, 6, 135, 136, 9, 10, 139, 12, + 141, 142, 15, 144, 17, 18, 147, 20, 149, 150, 23, 24, 153, + 154, 27, 156, 29, 30, 159, 160, 33, 34, 163, 36, 165, 166, + 39, 40, 169, 170, 43, 172, 45, 46, 175, 48, 177, 178, 51, + 180, 53, 54, 183, 184, 57, 58, 187, 60, 189, 190, 63, 192, + 65, 66, 195, 68, 197, 198, 71, 72, 201, 202, 75, 204, 77, + 78, 207, 80, 209, 210, 83, 212, 85, 86, 215, 216, 89, 90, + 219, 92, 221, 222, 95, 96, 225, 226, 99, 228, 101, 102, 231, + 232, 105, 106, 235, 108, 237, 238, 111, 240, 113, 114, 243, 116, + 245, 246, 119, 120, 249, 250, 123, 252, 125, 126, 255, + }); + +static dpct::global_memory + ksigns64(sycl::range<1>(128), + { + 0x0000000000000000, 0xff000000000000ff, 0xff0000000000ff00, + 0x000000000000ffff, 0xff00000000ff0000, 0x0000000000ff00ff, + 0x0000000000ffff00, 0xff00000000ffffff, 0xff000000ff000000, + 0x00000000ff0000ff, 0x00000000ff00ff00, 0xff000000ff00ffff, + 0x00000000ffff0000, 0xff000000ffff00ff, 0xff000000ffffff00, + 0x00000000ffffffff, 0xff0000ff00000000, 0x000000ff000000ff, + 0x000000ff0000ff00, 0xff0000ff0000ffff, 0x000000ff00ff0000, + 0xff0000ff00ff00ff, 0xff0000ff00ffff00, 0x000000ff00ffffff, + 0x000000ffff000000, 0xff0000ffff0000ff, 0xff0000ffff00ff00, + 0x000000ffff00ffff, 0xff0000ffffff0000, 0x000000ffffff00ff, + 0x000000ffffffff00, 0xff0000ffffffffff, 0xff00ff0000000000, + 0x0000ff00000000ff, 0x0000ff000000ff00, 0xff00ff000000ffff, + 0x0000ff0000ff0000, 0xff00ff0000ff00ff, 0xff00ff0000ffff00, + 0x0000ff0000ffffff, 0x0000ff00ff000000, 0xff00ff00ff0000ff, + 0xff00ff00ff00ff00, 0x0000ff00ff00ffff, 0xff00ff00ffff0000, + 0x0000ff00ffff00ff, 0x0000ff00ffffff00, 0xff00ff00ffffffff, + 0x0000ffff00000000, 0xff00ffff000000ff, 0xff00ffff0000ff00, + 0x0000ffff0000ffff, 0xff00ffff00ff0000, 0x0000ffff00ff00ff, + 0x0000ffff00ffff00, 0xff00ffff00ffffff, 0xff00ffffff000000, + 0x0000ffffff0000ff, 0x0000ffffff00ff00, 0xff00ffffff00ffff, + 0x0000ffffffff0000, 0xff00ffffffff00ff, 0xff00ffffffffff00, + 0x0000ffffffffffff, 0xffff000000000000, 0x00ff0000000000ff, + 0x00ff00000000ff00, 0xffff00000000ffff, 0x00ff000000ff0000, + 0xffff000000ff00ff, 0xffff000000ffff00, 0x00ff000000ffffff, + 0x00ff0000ff000000, 0xffff0000ff0000ff, 0xffff0000ff00ff00, + 0x00ff0000ff00ffff, 0xffff0000ffff0000, 0x00ff0000ffff00ff, + 0x00ff0000ffffff00, 0xffff0000ffffffff, 0x00ff00ff00000000, + 0xffff00ff000000ff, 0xffff00ff0000ff00, 0x00ff00ff0000ffff, + 0xffff00ff00ff0000, 0x00ff00ff00ff00ff, 0x00ff00ff00ffff00, + 0xffff00ff00ffffff, 0xffff00ffff000000, 0x00ff00ffff0000ff, + 0x00ff00ffff00ff00, 0xffff00ffff00ffff, 0x00ff00ffffff0000, + 0xffff00ffffff00ff, 0xffff00ffffffff00, 0x00ff00ffffffffff, + 0x00ffff0000000000, 0xffffff00000000ff, 0xffffff000000ff00, + 0x00ffff000000ffff, 0xffffff0000ff0000, 0x00ffff0000ff00ff, + 0x00ffff0000ffff00, 0xffffff0000ffffff, 0xffffff00ff000000, + 0x00ffff00ff0000ff, 0x00ffff00ff00ff00, 0xffffff00ff00ffff, + 0x00ffff00ffff0000, 0xffffff00ffff00ff, 0xffffff00ffffff00, + 0x00ffff00ffffffff, 0xffffffff00000000, 0x00ffffff000000ff, + 0x00ffffff0000ff00, 0xffffffff0000ffff, 0x00ffffff00ff0000, + 0xffffffff00ff00ff, 0xffffffff00ffff00, 0x00ffffff00ffffff, + 0x00ffffffff000000, 0xffffffffff0000ff, 0xffffffffff00ff00, + 0x00ffffffff00ffff, 0xffffffffffff0000, 0x00ffffffffff00ff, + 0x00ffffffffffff00, 0xffffffffffffffff, + }); +//#endif + +static dpct::global_memory + kmask_iq2xs(sycl::range<1>(8), {1, 2, 4, 8, 16, 32, 64, 128}); + +template +static void dequantize_block_iq2_xxs(const void * __restrict__ vx, dst_t * __restrict__ yy, + const sycl::nd_item<3> &item_ct1, + const uint64_t *iq2xxs_grid_ptr, + const uint8_t *ksigns_iq2xs_ptr, + const uint8_t *kmask_iq2xs_ptr) { + + const int i = item_ct1.get_group(2); + const block_iq2_xxs * x = (const block_iq2_xxs *) vx; + + const int tid = item_ct1.get_local_id(2); +#if QK_K == 256 + const int il = tid/8; // 0...3 + const int ib = tid%8; // 0...7 + dst_t * y = yy + i*QK_K + 32*ib + 8*il; + const uint16_t * q2 = x[i].qs + 4*ib; + const uint8_t * aux8 = (const uint8_t *)q2; + const uint8_t * grid = (const uint8_t *)(iq2xxs_grid_ptr + aux8[il]); + const uint32_t aux32 = q2[2] | (q2[3] << 16); + const float d = (float)x[i].d * (0.5f + (aux32 >> 28)) * 0.25f; + const uint8_t signs = ksigns_iq2xs_ptr[(aux32 >> 7*il) & 127]; + for (int j = 0; j < 8; ++j) y[j] = d * grid[j] * (signs & kmask_iq2xs_ptr[j] ? -1.f : 1.f); +#else + assert(false); +#endif + +} + +template +static void dequantize_block_iq2_xs(const void * __restrict__ vx, dst_t * __restrict__ yy, + const sycl::nd_item<3> &item_ct1, + const uint64_t *iq2xs_grid, + const uint8_t *ksigns_iq2xs, + const uint8_t *kmask_iq2xs) { + + const int i = item_ct1.get_group(2); + const block_iq2_xs * x = (const block_iq2_xs *) vx; + + const int tid = item_ct1.get_local_id(2); +#if QK_K == 256 + const int il = tid/8; // 0...3 + const int ib = tid%8; // 0...7 + dst_t * y = yy + i*QK_K + 32*ib + 8*il; + const uint16_t * q2 = x[i].qs + 4*ib; + const uint8_t * grid = (const uint8_t *)(iq2xs_grid + (q2[il] & 511)); + const float d = (float)x[i].d * (0.5f + ((x[i].scales[ib] >> 4*(il/2)) & 0xf)) * 0.25f; + const uint8_t signs = ksigns_iq2xs[q2[il] >> 9]; + for (int j = 0; j < 8; ++j) y[j] = d * grid[j] * (signs & kmask_iq2xs[j] ? -1.f : 1.f); +#else + assert(false); +#endif + +} + +template +static void dequantize_block_iq3_xxs(const void * __restrict__ vx, dst_t * __restrict__ yy, + const sycl::nd_item<3> &item_ct1, + const uint32_t *iq3xxs_grid, + const uint8_t *ksigns_iq2xs, + const uint8_t *kmask_iq2xs) { + + const int i = item_ct1.get_group(2); + const block_iq3_xxs * x = (const block_iq3_xxs *) vx; + + const int tid = item_ct1.get_local_id(2); +#if QK_K == 256 + const int il = tid/8; // 0...3 + const int ib = tid%8; // 0...7 + dst_t * y = yy + i*QK_K + 32*ib + 8*il; + const uint8_t * q3 = x[i].qs + 8*ib; + const uint16_t * gas = (const uint16_t *)(x[i].qs + QK_K/4) + 2*ib; + const uint8_t * grid1 = (const uint8_t *)(iq3xxs_grid + q3[2*il+0]); + const uint8_t * grid2 = (const uint8_t *)(iq3xxs_grid + q3[2*il+1]); + const uint32_t aux32 = gas[0] | (gas[1] << 16); + const float d = (float)x[i].d * (0.5f + (aux32 >> 28)) * 0.5f; + const uint8_t signs = ksigns_iq2xs[(aux32 >> 7*il) & 127]; + for (int j = 0; j < 4; ++j) { + y[j+0] = d * grid1[j] * (signs & kmask_iq2xs[j+0] ? -1.f : 1.f); + y[j+4] = d * grid2[j] * (signs & kmask_iq2xs[j+4] ? -1.f : 1.f); + } +#else + assert(false); +#endif + +} + /* DPCT1110:4: The total declared local variable size in device function dequantize_mul_mat_vec_q2_k exceeds 128 bytes and may cause high register @@ -4446,13 +5257,16 @@ static void dequantize_mul_mat_vec_q2_k(const void *__restrict__ vx, } #else - const int tid = threadIdx.x/(2*K_QUANTS_PER_ITERATION); // 0...15 or 0...7 - const int ix = threadIdx.x%(2*K_QUANTS_PER_ITERATION); // 0....1 or 0...3 + const int tid = item_ct1.get_local_id(2) / + (2 * K_QUANTS_PER_ITERATION); // 0...15 or 0...7 + const int ix = item_ct1.get_local_id(2) % + (2 * K_QUANTS_PER_ITERATION); // 0....1 or 0...3 const int offset = tid * K_QUANTS_PER_ITERATION; uint32_t uaux[2]; const uint8_t * d = (const uint8_t *)uaux; + for (int i = ix; i < num_blocks_per_row; i += 2*K_QUANTS_PER_ITERATION) { const float * y = yy + i * QK_K + offset; @@ -4462,7 +5276,8 @@ static void dequantize_mul_mat_vec_q2_k(const void *__restrict__ vx, uaux[0] = s[0] & 0x0f0f0f0f; uaux[1] = (s[0] >> 4) & 0x0f0f0f0f; - const float2 dall = __half22float2(x[i].dm); + const sycl::float2 dall = + x[i].dm.convert(); float sum1 = 0, sum2 = 0; for (int l = 0; l < K_QUANTS_PER_ITERATION; ++l) { @@ -4473,8 +5288,9 @@ static void dequantize_mul_mat_vec_q2_k(const void *__restrict__ vx, + y[l+48] * d[3] * ((ql >> 6) & 3); sum2 += y[l+0] * d[4] + y[l+16] * d[5] + y[l+32] * d[6] + y[l+48] * d[7]; } - tmp += dall.x * sum1 - dall.y * sum2; + tmp += dall.x() * sum1 - dall.y() * sum2; } + #endif // sum up partial sums and write back result @@ -4569,8 +5385,8 @@ static void dequantize_mul_mat_vec_q3_k(const void *__restrict__ vx, } #else - const int tid = threadIdx.x/(2*K_QUANTS_PER_ITERATION); // 0...15 or 0...7 - const int ix = threadIdx.x%(2*K_QUANTS_PER_ITERATION); // 0....1 or 0...3 + const int tid = item_ct1.get_local_id(2)/(2*K_QUANTS_PER_ITERATION); // 0...15 or 0...7 + const int ix = item_ct1.get_local_id(2)%(2*K_QUANTS_PER_ITERATION); // 0....1 or 0...3 const int offset = tid * K_QUANTS_PER_ITERATION; // 0...15 or 0...14 const int in = offset/8; // 0 or 1 const int im = offset%8; // 0...7 @@ -4719,8 +5535,8 @@ static void dequantize_mul_mat_vec_q4_k(const void *__restrict__ vx, } #else - const int tid = threadIdx.x/(2*K_QUANTS_PER_ITERATION); // 0...15 - const int ix = threadIdx.x%(2*K_QUANTS_PER_ITERATION); + const int tid = item_ct1.get_local_id(2)/(2*K_QUANTS_PER_ITERATION); // 0...15 + const int ix = item_ct1.get_local_id(2)%(2*K_QUANTS_PER_ITERATION); const int step = tid * K_QUANTS_PER_ITERATION; @@ -4860,8 +5676,8 @@ static void dequantize_mul_mat_vec_q5_k(const void *__restrict__ vx, } #else - const int tid = threadIdx.x/(2*K_QUANTS_PER_ITERATION); // 0...15 - const int ix = threadIdx.x%(2*K_QUANTS_PER_ITERATION); + const int tid = item_ct1.get_local_id(2)/(2*K_QUANTS_PER_ITERATION); // 0...15 + const int ix = item_ct1.get_local_id(2)%(2*K_QUANTS_PER_ITERATION); const int step = tid * K_QUANTS_PER_ITERATION; const int im = step/8; const int in = step%8; @@ -4969,8 +5785,8 @@ static void dequantize_mul_mat_vec_q6_k(const void * __restrict__ vx, const floa #else - const int tid = threadIdx.x/(2*K_QUANTS_PER_ITERATION); // 0...7 - const int ix = threadIdx.x%(2*K_QUANTS_PER_ITERATION); // 0...3 + const int tid = item_ct1.get_local_id(2)/(2*K_QUANTS_PER_ITERATION); // 0...7 + const int ix = item_ct1.get_local_id(2)%(2*K_QUANTS_PER_ITERATION); // 0...3 const int step = tid * K_QUANTS_PER_ITERATION; @@ -5170,6 +5986,21 @@ static void dequantize_block(const void * __restrict__ vx, dst_t * __restrict__ y[iybs + iqs + y_offset] = v.y(); } +template +static void convert_unary(const void * __restrict__ vx, dst_t * __restrict__ y, const int k, + const sycl::nd_item<3> &item_ct1) { + const int i = item_ct1.get_local_range(2) * item_ct1.get_group(2) + + item_ct1.get_local_id(2); + + if (i >= k) { + return; + } + + const src_t * x = (src_t *) vx; + + y[i] = x[i]; +} + // VDR = vec dot ratio, how many contiguous integers each thread processes when the vec dot kernel is called // MMVQ = mul_mat_vec_q, MMQ = mul_mat_q @@ -6588,8 +7419,8 @@ vec_dot_q4_K_q8_1(const void *__restrict__ vbq, const float dall = bq4_K->dm[0]; const float dmin = bq4_K->dm[1]; - const float d8_1 = __low2float(bq8_1[0].ds); - const float d8_2 = __low2float(bq8_1[1].ds); + const float d8_1 = bq8_1[0].ds[0]; + const float d8_2 = bq8_1[1].ds[1]; const int ui1 = *((const int *)bq8_1[0].qs + (iqs/2)); const int ui2 = *((const int *)bq8_1[0].qs + (iqs/2) + 4); @@ -6600,10 +7431,10 @@ vec_dot_q4_K_q8_1(const void *__restrict__ vbq, const int v1 = q4[0]; const int v2 = q4[4]; - const int dot1 = __dp4a(ui2, v2 & 0x0f0f0f0f, __dp4a(ui1, v1 & 0x0f0f0f0f, 0)); - const int dot2 = __dp4a(ui4, (v2 >> 4) & 0x0f0f0f0f, __dp4a(ui3, (v1 >> 4) & 0x0f0f0f0f, 0)); - const int dot3 = __dp4a(0x01010101, ui2, __dp4a(0x01010101, ui1, 0)); - const int dot4 = __dp4a(0x01010101, ui4, __dp4a(0x01010101, ui3, 0)); + const int dot1 = dpct::dp4a(ui2, v2 & 0x0f0f0f0f, dpct::dp4a(ui1, v1 & 0x0f0f0f0f, 0)); + const int dot2 = dpct::dp4a(ui4, (v2 >> 4) & 0x0f0f0f0f, dpct::dp4a(ui3, (v1 >> 4) & 0x0f0f0f0f, 0)); + const int dot3 = dpct::dp4a(0x01010101, ui2, dpct::dp4a(0x01010101, ui1, 0)); + const int dot4 = dpct::dp4a(0x01010101, ui4, dpct::dp4a(0x01010101, ui3, 0)); sumf_d += d8_1 * (dot1 * s[0]) + d8_2 * (dot2 * s[1]); sumf_m += d8_1 * (dot3 * s[2]) + d8_2 * (dot4 * s[3]); @@ -6772,8 +7603,8 @@ vec_dot_q5_K_q8_1(const void *__restrict__ vbq, const float d = bq5_K->d; - const float d8_1 = __low2half(bq8_1[0].ds); - const float d8_2 = __low2half(bq8_1[1].ds); + const float d8_1 = bq8_1[0].ds[0]; + const float d8_2 = bq8_1[1].ds[1]; const int ui1 = *((const int *)bq8_1[0].qs + (iqs/2)); const int ui2 = *((const int *)bq8_1[0].qs + (iqs/2) + 4); @@ -6794,8 +7625,8 @@ vec_dot_q5_K_q8_1(const void *__restrict__ vbq, const int v3 = (((vh >> 0) & 0x10101010) ^ 0x10101010) | ((vl1 >> 4) & 0x0f0f0f0f); const int v4 = (((vh >> 2) & 0x10101010) ^ 0x10101010) | ((vl2 >> 4) & 0x0f0f0f0f); - const float sumf_d = d8_1 * (__dp4a(ui1, v1, 0) * s[0] + __dp4a(ui2, v2, 0) * s[1]) - + d8_2 * (__dp4a(ui3, v3, 0) * s[2] + __dp4a(ui4, v4, 0) * s[3]); + const float sumf_d = d8_1 * (dpct::dp4a(ui1, v1, 0) * s[0] + dpct::dp4a(ui2, v2, 0) * s[1]) + + d8_2 * (dpct::dp4a(ui3, v3, 0) * s[2] + dpct::dp4a(ui4, v4, 0) * s[3]); return d * sumf_d; @@ -7051,6 +7882,150 @@ static __dpct_inline__ float vec_dot_q6_K_q8_1_mul_mat( return vec_dot_q6_K_q8_1_impl_mmq(&x_ql[index_x], &y_qs[index_y], sc, x_dmf[i * (WARP_SIZE/QI6_K) + i/QI6_K], &y_df[index_y/QI8_1]); } + +static __dpct_inline__ float +vec_dot_iq2_xxs_q8_1(const void *__restrict__ vbq, + const block_q8_1 *__restrict__ bq8_1, const int &iqs, + const uint64_t *iq2xxs_grid, const uint8_t *ksigns_iq2xs, + const uint8_t *kmask_iq2xs) { +#if QK_K == 256 + const block_iq2_xxs * bq2 = (const block_iq2_xxs *) vbq; + +#if QR2_XXS == 8 + const int ib32 = iqs; + const uint16_t * q2 = bq2->qs + 4*ib32; + const uint8_t * aux8 = (const uint8_t *)q2; + const int8_t * q8 = bq8_1[ib32].qs; + uint32_t aux32 = q2[2] | (q2[3] << 16); + int sumi = 0; + for (int l = 0; l < 4; ++l) { + const uint8_t * grid = (const uint8_t *)(iq2xxs_grid + aux8[l]); + const uint8_t signs = ksigns_iq2xs[aux32 & 127]; + for (int j = 0; j < 8; ++j) { + sumi += q8[j] * grid[j] * (signs & kmask_iq2xs[j] ? -1 : 1); + } + q8 += 8; + aux32 >>= 7; + } + const float d = (float)bq2->d * (0.5f + aux32) * bq8_1[ib32].ds[0] * 0.25f; + return d * sumi; +#else + // iqs is 0...15 + const int ib32 = iqs/2; + const int il = iqs%2; + const uint16_t * q2 = bq2->qs + 4*ib32; + const uint8_t * aux8 = (const uint8_t *)q2; + const uint8_t * grid1 = (const uint8_t *)(iq2xxs_grid + aux8[2*il+0]); + const uint8_t * grid2 = (const uint8_t *)(iq2xxs_grid + aux8[2*il+1]); + const uint32_t aux32 = q2[2] | (q2[3] << 16); + const float d = (float)bq2->d * (0.5f + (aux32 >> 28)) * bq8_1[ib32].ds[0] * 0.25f; + const uint8_t signs1 = ksigns_iq2xs[(aux32 >> 14*il) & 127]; + const uint8_t signs2 = ksigns_iq2xs[(aux32 >> (14*il + 7)) & 127]; + const int8_t * q8 = bq8_1[ib32].qs + 16*il; + int sumi1 = 0, sumi2 = 0; + for (int j = 0; j < 8; ++j) { + sumi1 += q8[j+0] * grid1[j] * (signs1 & kmask_iq2xs[j] ? -1 : 1); + sumi2 += q8[j+8] * grid2[j] * (signs2 & kmask_iq2xs[j] ? -1 : 1); + } + return d * (sumi1 + sumi2); +#endif +#else + assert(false); + return 0.f; +#endif +} + +static __dpct_inline__ float +vec_dot_iq2_xs_q8_1(const void *__restrict__ vbq, + const block_q8_1 *__restrict__ bq8_1, const int &iqs, + const uint64_t *iq2xs_grid, const uint64_t *ksigns64) { +#if DPCT_COMPATIBILITY_TEMP >= \ + MIN_CC_DP4A // lowest compute capability for integer intrinsics +#if QK_K == 256 + const block_iq2_xs * bq2 = (const block_iq2_xs *) vbq; + + const int ib32 = iqs; + const uint16_t * q2 = bq2->qs + 4*ib32; + const int8_t * q8 = bq8_1[ib32].qs; + const uint8_t ls1 = bq2->scales[ib32] & 0xf; + const uint8_t ls2 = bq2->scales[ib32] >> 4; + int sumi1 = 0; + for (int l = 0; l < 2; ++l) { + const uint32_t * grid = (const uint32_t *)(iq2xs_grid + (q2[l] & 511)); + const uint32_t * signs = (const uint32_t *)(ksigns64 + (q2[l] >> 9)); + const int grid_l = dpct::vectorized_binary( + grid[0] ^ signs[0], signs[0], std::minus<>()); + const int grid_h = dpct::vectorized_binary( + grid[1] ^ signs[1], signs[1], std::minus<>()); + sumi1 = dpct::dp4a(grid_l, *((const int *)q8 + 0), sumi1); + sumi1 = dpct::dp4a(grid_h, *((const int *)q8 + 1), sumi1); + q8 += 8; + } + int sumi2 = 0; + for (int l = 2; l < 4; ++l) { + const uint32_t * grid = (const uint32_t *)(iq2xs_grid + (q2[l] & 511)); + const uint32_t * signs = (const uint32_t *)(ksigns64 + (q2[l] >> 9)); + const int grid_l = dpct::vectorized_binary( + grid[0] ^ signs[0], signs[0], std::minus<>()); + const int grid_h = dpct::vectorized_binary( + grid[1] ^ signs[1], signs[1], std::minus<>()); + sumi2 = dpct::dp4a(grid_l, *((const int *)q8 + 0), sumi2); + sumi2 = dpct::dp4a(grid_h, *((const int *)q8 + 1), sumi2); + q8 += 8; + } + const float d = (float)bq2->d * bq8_1[ib32].ds[0] * 0.25f; + return d * ((0.5f + ls1) * sumi1 + (0.5f + ls2) * sumi2); +#else + assert(false); + return 0.f; +#endif +#else + assert(false); + return 0.f; +#endif +} + +static __dpct_inline__ float +vec_dot_iq3_xxs_q8_1(const void *__restrict__ vbq, + const block_q8_1 *__restrict__ bq8_1, const int &iqs, + const uint32_t *iq3xxs_grid, const uint64_t *ksigns64) { +#if DPCT_COMPATIBILITY_TEMP >= \ + MIN_CC_DP4A // lowest compute capability for integer intrinsics +#if QK_K == 256 + const block_iq3_xxs * bq2 = (const block_iq3_xxs *) vbq; + + const int ib32 = iqs; + const uint8_t * q3 = bq2->qs + 8*ib32; + const uint16_t * gas = (const uint16_t *)(bq2->qs + QK_K/4) + 2*ib32; + const int8_t * q8 = bq8_1[ib32].qs; + uint32_t aux32 = gas[0] | (gas[1] << 16); + int sumi = 0; + for (int l = 0; l < 4; ++l) { + const uint32_t * grid1 = iq3xxs_grid + q3[2*l+0]; + const uint32_t * grid2 = iq3xxs_grid + q3[2*l+1]; + const uint32_t * signs = (const uint32_t *)(ksigns64 + (aux32 & 127)); + const int grid_l = dpct::vectorized_binary( + grid1[0] ^ signs[0], signs[0], std::minus<>()); + const int grid_h = dpct::vectorized_binary( + grid2[0] ^ signs[1], signs[1], std::minus<>()); + sumi = dpct::dp4a(grid_l, *((int *)q8 + 0), sumi); + sumi = dpct::dp4a(grid_h, *((int *)q8 + 1), sumi); + q8 += 8; + aux32 >>= 7; + } + const float d = (float)bq2->d * (0.5f + aux32) * bq8_1[ib32].ds[0] * 0.5f; + return d * sumi; +#else + assert(false); + return 0.f; +#endif +#else + assert(false); + return 0.f; +#endif +} + + template @@ -7632,7 +8607,8 @@ template static void template static void mul_mat_vec_q(const void * __restrict__ vx, const void * __restrict__ vy, float * __restrict__ dst, const int ncols, const int nrows, - const sycl::nd_item<3> &item_ct1) { + const sycl::nd_item<3> &item_ct1, + const uint32_t *iq3xxs_grid_ptr, const uint64_t *ksigns64_ptr) { const int row = item_ct1.get_group(2) * item_ct1.get_local_range(1) + item_ct1.get_local_id(1); @@ -7649,12 +8625,11 @@ static void mul_mat_vec_q(const void * __restrict__ vx, const void * __restrict_ const block_q_t * x = (const block_q_t *) vx; const block_q8_1 * y = (const block_q8_1 *) vy; - for (int i = 0; i < blocks_per_row; i += blocks_per_warp) { - const int ibx = row * blocks_per_row + i + - item_ct1.get_local_id(2) / (qi / vdr); // x block index + for (int i = item_ct1.get_local_id(2) / (qi / vdr); i < blocks_per_row; + i += blocks_per_warp) { + const int ibx = row*blocks_per_row + i; // x block index - const int iby = (i + item_ct1.get_local_id(2) / (qi / vdr)) * - (qk / QK8_1); // y block index that aligns with ibx + const int iby = i * (qk/QK8_1); // y block index that aligns with ibx const int iqs = vdr * @@ -7676,6 +8651,145 @@ static void mul_mat_vec_q(const void * __restrict__ vx, const void * __restrict_ } } +template +static void mul_mat_vec_q_iq2_xxs_q8_1(const void * __restrict__ vx, const void * __restrict__ vy, float * __restrict__ dst, const int ncols, const int nrows, + const sycl::nd_item<3> &item_ct1, + const uint64_t *iq2xxs_grid_ptr, const uint8_t *ksigns_iq2xs_ptr, + const uint8_t *kmask_iq2xs_ptr ) { + const int row = item_ct1.get_group(2) * item_ct1.get_local_range(1) + + item_ct1.get_local_id(1); + + if (row >= nrows) { + return; + } + + const int blocks_per_row = ncols / qk; + const int blocks_per_warp = vdr * WARP_SIZE / qi; + +// partial sum for each thread + float tmp = 0.0f; + + const block_q_t * x = (const block_q_t *) vx; + const block_q8_1 * y = (const block_q8_1 *) vy; + + for (int i = item_ct1.get_local_id(2) / (qi / vdr); i < blocks_per_row; + i += blocks_per_warp) { + const int ibx = row*blocks_per_row + i; // x block index + + const int iby = i * (qk/QK8_1); // y block index that aligns with ibx + + const int iqs = + vdr * + (item_ct1.get_local_id(2) % + (qi / vdr)); // x block quant index when casting the quants to int + + tmp += vec_dot_iq2_xxs_q8_1(&x[ibx], &y[iby], iqs, iq2xxs_grid_ptr, ksigns_iq2xs_ptr, kmask_iq2xs_ptr); + } + + // sum up partial sums and write back result +#pragma unroll + for (int mask = 16; mask > 0; mask >>= 1) { + tmp += + dpct::permute_sub_group_by_xor(item_ct1.get_sub_group(), tmp, mask); + } + + if (item_ct1.get_local_id(2) == 0) { + dst[row] = tmp; + } +} + +template +static void mul_mat_vec_q_iq2_xs_q8_1(const void * __restrict__ vx, const void * __restrict__ vy, float * __restrict__ dst, const int ncols, const int nrows, + const sycl::nd_item<3> &item_ct1, + const uint64_t *iq2xs_grid_ptr, const uint64_t *ksigns64_ptr ) { + const int row = item_ct1.get_group(2) * item_ct1.get_local_range(1) + + item_ct1.get_local_id(1); + + if (row >= nrows) { + return; + } + + const int blocks_per_row = ncols / qk; + const int blocks_per_warp = vdr * WARP_SIZE / qi; + +// partial sum for each thread + float tmp = 0.0f; + + const block_q_t * x = (const block_q_t *) vx; + const block_q8_1 * y = (const block_q8_1 *) vy; + + for (int i = item_ct1.get_local_id(2) / (qi / vdr); i < blocks_per_row; + i += blocks_per_warp) { + const int ibx = row*blocks_per_row + i; // x block index + + const int iby = i * (qk/QK8_1); // y block index that aligns with ibx + + const int iqs = + vdr * + (item_ct1.get_local_id(2) % + (qi / vdr)); // x block quant index when casting the quants to int + + tmp += vec_dot_iq2_xs_q8_1(&x[ibx], &y[iby], iqs, iq2xs_grid_ptr, ksigns64_ptr); + } + + // sum up partial sums and write back result +#pragma unroll + for (int mask = 16; mask > 0; mask >>= 1) { + tmp += + dpct::permute_sub_group_by_xor(item_ct1.get_sub_group(), tmp, mask); + } + + if (item_ct1.get_local_id(2) == 0) { + dst[row] = tmp; + } +} + +template +static void mul_mat_vec_q_iq3_xxs_q8_1(const void * __restrict__ vx, const void * __restrict__ vy, float * __restrict__ dst, const int ncols, const int nrows, + const sycl::nd_item<3> &item_ct1, + const uint32_t *iq3xxs_grid_ptr, const uint64_t *ksigns64_ptr ) { + const int row = item_ct1.get_group(2) * item_ct1.get_local_range(1) + + item_ct1.get_local_id(1); + + if (row >= nrows) { + return; + } + + const int blocks_per_row = ncols / qk; + const int blocks_per_warp = vdr * WARP_SIZE / qi; + +// partial sum for each thread + float tmp = 0.0f; + + const block_q_t * x = (const block_q_t *) vx; + const block_q8_1 * y = (const block_q8_1 *) vy; + + for (int i = item_ct1.get_local_id(2) / (qi / vdr); i < blocks_per_row; + i += blocks_per_warp) { + const int ibx = row*blocks_per_row + i; // x block index + + const int iby = i * (qk/QK8_1); // y block index that aligns with ibx + + const int iqs = + vdr * + (item_ct1.get_local_id(2) % + (qi / vdr)); // x block quant index when casting the quants to int + + tmp += vec_dot_iq3_xxs_q8_1(&x[ibx], &y[iby], iqs, iq3xxs_grid_ptr, ksigns64_ptr); + } + + // sum up partial sums and write back result +#pragma unroll + for (int mask = 16; mask > 0; mask >>= 1) { + tmp += + dpct::permute_sub_group_by_xor(item_ct1.get_sub_group(), tmp, mask); + } + + if (item_ct1.get_local_id(2) == 0) { + dst[row] = tmp; + } +} + template static void dequantize_mul_mat_vec(const void * __restrict__ vx, const dfloat * __restrict__ y, float * __restrict__ dst, const int ncols, const int nrows, const sycl::nd_item<3> &item_ct1) { @@ -9109,7 +10223,18 @@ static void dequantize_row_q2_K_sycl(const void *vx, dst_t *y, const int k, }); } #else - dequantize_block_q2_K<<>>(vx, y); + { + dpct::has_capability_or_fail(stream->get_device(), + {sycl::aspect::fp16}); + + stream->parallel_for(sycl::nd_range<3>(sycl::range<3>(1, 1, nb) * + sycl::range<3>(1, 1, 32), + sycl::range<3>(1, 1, 32)), + [=](sycl::nd_item<3> item_ct1) { + dequantize_block_q2_K(vx, y, item_ct1); + }); + } + #endif } @@ -9130,10 +10255,57 @@ static void dequantize_row_q3_K_sycl(const void *vx, dst_t *y, const int k, }); } #else - dequantize_block_q3_K<<>>(vx, y); + { + dpct::has_capability_or_fail(stream->get_device(), + {sycl::aspect::fp16}); + + stream->parallel_for(sycl::nd_range<3>(sycl::range<3>(1, 1, nb) * + sycl::range<3>(1, 1, 32), + sycl::range<3>(1, 1, 32)), + [=](sycl::nd_item<3> item_ct1) { + dequantize_block_q3_K(vx, y, item_ct1); + }); + } #endif } +template +static void dequantize_row_q4_0_sycl(const void *vx, dst_t *y, const int k, + dpct::queue_ptr stream) { + const int nb32 = k / 32; + const int nb = (k + 255) / 256; + { + dpct::has_capability_or_fail(stream->get_device(), + {sycl::aspect::fp16}); + + stream->parallel_for(sycl::nd_range<3>(sycl::range<3>(1, 1, nb) * + sycl::range<3>(1, 1, 32), + sycl::range<3>(1, 1, 32)), + [=](sycl::nd_item<3> item_ct1) { + dequantize_block_q4_0(vx, y, nb32, item_ct1); + }); + } +} + +template +static void dequantize_row_q4_1_sycl(const void *vx, dst_t *y, const int k, + dpct::queue_ptr stream) { + const int nb32 = k / 32; + const int nb = (k + 255) / 256; + { + dpct::has_capability_or_fail(stream->get_device(), + {sycl::aspect::fp16}); + + stream->parallel_for(sycl::nd_range<3>(sycl::range<3>(1, 1, nb) * + sycl::range<3>(1, 1, 32), + sycl::range<3>(1, 1, 32)), + [=](sycl::nd_item<3> item_ct1) { + dequantize_block_q4_1(vx, y, nb32, item_ct1); + }); + } +} + + template static void dequantize_row_q4_K_sycl(const void *vx, dst_t *y, const int k, dpct::queue_ptr stream) { @@ -9168,7 +10340,18 @@ static void dequantize_row_q5_K_sycl(const void *vx, dst_t *y, const int k, }); } #else - dequantize_block_q5_K<<>>(vx, y); + { + dpct::has_capability_or_fail(stream->get_device(), + {sycl::aspect::fp16}); + + stream->parallel_for(sycl::nd_range<3>(sycl::range<3>(1, 1, nb) * + sycl::range<3>(1, 1, 32), + sycl::range<3>(1, 1, 32)), + [=](sycl::nd_item<3> item_ct1) { + dequantize_block_q5_K(vx, y, item_ct1); + }); + } + #endif } @@ -9189,11 +10372,132 @@ static void dequantize_row_q6_K_sycl(const void *vx, dst_t *y, const int k, }); } #else - dequantize_block_q6_K<<>>(vx, y); + { + dpct::has_capability_or_fail(stream->get_device(), + {sycl::aspect::fp16}); + + stream->parallel_for(sycl::nd_range<3>(sycl::range<3>(1, 1, nb) * + sycl::range<3>(1, 1, 32), + sycl::range<3>(1, 1, 32)), + [=](sycl::nd_item<3> item_ct1) { + dequantize_block_q6_K(vx, y, item_ct1); + }); + } + #endif } -static to_fp16_sycl_t ggml_get_to_fp16_sycl(ggml_type type) { + +template +static void dequantize_row_iq2_xxs_sycl(const void *vx, dst_t *y, const int k, + dpct::queue_ptr stream) { + const int nb = k / QK_K; + { + iq2xxs_grid.init(*stream); + ksigns_iq2xs.init(*stream); + kmask_iq2xs.init(*stream); + + dpct::has_capability_or_fail(stream->get_device(), + {sycl::aspect::fp16}); + + stream->submit([&](sycl::handler &cgh) { + auto iq2xxs_grid_ptr_ct1 = iq2xxs_grid.get_ptr(); + auto ksigns_iq2xs_ptr_ct1 = ksigns_iq2xs.get_ptr(); + auto kmask_iq2xs_ptr_ct1 = kmask_iq2xs.get_ptr(); + + cgh.parallel_for(sycl::nd_range<3>(sycl::range<3>(1, 1, nb) * + sycl::range<3>(1, 1, 32), + sycl::range<3>(1, 1, 32)), + [=](sycl::nd_item<3> item_ct1) { + dequantize_block_iq2_xxs( + vx, y, item_ct1, iq2xxs_grid_ptr_ct1, + ksigns_iq2xs_ptr_ct1, kmask_iq2xs_ptr_ct1); + }); + }); + } +} + +template +static void dequantize_row_iq2_xs_sycl(const void *vx, dst_t *y, const int k, + dpct::queue_ptr stream) { + const int nb = k / QK_K; + { + iq2xs_grid.init(*stream); + ksigns_iq2xs.init(*stream); + kmask_iq2xs.init(*stream); + + dpct::has_capability_or_fail(stream->get_device(), + {sycl::aspect::fp16}); + + stream->submit([&](sycl::handler &cgh) { + auto iq2xs_grid_ptr_ct1 = iq2xs_grid.get_ptr(); + auto ksigns_iq2xs_ptr_ct1 = ksigns_iq2xs.get_ptr(); + auto kmask_iq2xs_ptr_ct1 = kmask_iq2xs.get_ptr(); + + cgh.parallel_for(sycl::nd_range<3>(sycl::range<3>(1, 1, nb) * + sycl::range<3>(1, 1, 32), + sycl::range<3>(1, 1, 32)), + [=](sycl::nd_item<3> item_ct1) { + dequantize_block_iq2_xs( + vx, y, item_ct1, iq2xs_grid_ptr_ct1, + ksigns_iq2xs_ptr_ct1, kmask_iq2xs_ptr_ct1); + }); + }); + } +} + +template +static void dequantize_row_iq3_xxs_sycl(const void *vx, dst_t *y, const int k, + dpct::queue_ptr stream) { + const int nb = k / QK_K; + { + iq3xxs_grid.init(*stream); + ksigns_iq2xs.init(*stream); + kmask_iq2xs.init(*stream); + + dpct::has_capability_or_fail(stream->get_device(), + {sycl::aspect::fp16}); + + stream->submit([&](sycl::handler &cgh) { + auto iq3xxs_grid_ptr_ct1 = iq3xxs_grid.get_ptr(); + auto ksigns_iq2xs_ptr_ct1 = ksigns_iq2xs.get_ptr(); + auto kmask_iq2xs_ptr_ct1 = kmask_iq2xs.get_ptr(); + + cgh.parallel_for(sycl::nd_range<3>(sycl::range<3>(1, 1, nb) * + sycl::range<3>(1, 1, 32), + sycl::range<3>(1, 1, 32)), + [=](sycl::nd_item<3> item_ct1) { + dequantize_block_iq3_xxs( + vx, y, item_ct1, iq3xxs_grid_ptr_ct1, + ksigns_iq2xs_ptr_ct1, kmask_iq2xs_ptr_ct1); + }); + }); + } +} + +template +static void convert_unary_sycl(const void *__restrict__ vx, + dst_t *__restrict__ y, const int k, + dpct::queue_ptr stream) { + const int num_blocks = (k + SYCL_DEQUANTIZE_BLOCK_SIZE - 1) / SYCL_DEQUANTIZE_BLOCK_SIZE; + { + dpct::has_capability_or_fail(stream->get_device(), + {sycl::aspect::fp16}); + + stream->parallel_for( + sycl::nd_range<3>( + sycl::range<3>(1, 1, num_blocks) * + sycl::range<3>(1, 1, SYCL_DEQUANTIZE_BLOCK_SIZE), + sycl::range<3>(1, 1, SYCL_DEQUANTIZE_BLOCK_SIZE)), + [=](sycl::nd_item<3> item_ct1) { + convert_unary(vx, y, k, item_ct1); + }); + } +} + + +static to_fp16_sycl_t ggml_get_to_fp16_sycl(ggml_type type) try { + int id; switch (type) { case GGML_TYPE_Q4_0: return dequantize_block_sycl; @@ -9215,19 +10519,30 @@ static to_fp16_sycl_t ggml_get_to_fp16_sycl(ggml_type type) { return dequantize_row_q5_K_sycl; case GGML_TYPE_Q6_K: return dequantize_row_q6_K_sycl; + case GGML_TYPE_IQ2_XXS: + return dequantize_row_iq2_xxs_sycl; + case GGML_TYPE_IQ2_XS: + return dequantize_row_iq2_xs_sycl; + case GGML_TYPE_IQ3_XXS: + return dequantize_row_iq3_xxs_sycl; case GGML_TYPE_F32: - return dequantize_block_sycl<1, 1, convert_f32>; + return convert_unary_sycl; default: return nullptr; } } +catch (sycl::exception const &exc) { + std::cerr << exc.what() << "Exception caught at file:" << __FILE__ + << ", line:" << __LINE__ << std::endl; + std::exit(1); +} static to_fp32_sycl_t ggml_get_to_fp32_sycl(ggml_type type) { switch (type) { case GGML_TYPE_Q4_0: - return dequantize_block_sycl; + return dequantize_row_q4_0_sycl; case GGML_TYPE_Q4_1: - return dequantize_block_sycl; + return dequantize_row_q4_1_sycl; case GGML_TYPE_Q5_0: return dequantize_block_sycl; case GGML_TYPE_Q5_1: @@ -9244,8 +10559,14 @@ static to_fp32_sycl_t ggml_get_to_fp32_sycl(ggml_type type) { return dequantize_row_q5_K_sycl; case GGML_TYPE_Q6_K: return dequantize_row_q6_K_sycl; + case GGML_TYPE_IQ2_XXS: + return dequantize_row_iq2_xxs_sycl; + case GGML_TYPE_IQ2_XS: + return dequantize_row_iq2_xs_sycl; + case GGML_TYPE_IQ3_XXS: + return dequantize_row_iq3_xxs_sycl; case GGML_TYPE_F16: - return dequantize_block_sycl<1, 1, convert_f16>; + return convert_unary_sycl; default: return nullptr; } @@ -9455,24 +10776,385 @@ static void convert_mul_mat_vec_f16_sycl(const void *vx, const dfloat *y, } } -template -static void mul_mat_vec_q_sycl_submitter(const void *vx, const void *vy, + +static void mul_mat_vec_q4_0_q8_1_sycl(const void *vx, const void *vy, + float *dst, const int ncols, + const int nrows, + dpct::queue_ptr stream) { + GGML_ASSERT(ncols % QK4_0 == 0); + const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y; + const sycl::range<3> block_nums(1, 1, block_num_y); + const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE); + { + iq3xxs_grid.init(*stream); + ksigns64.init(*stream); + + stream->submit([&](sycl::handler &cgh) { + auto iq3xxs_grid_ptr_ct1 = iq3xxs_grid.get_ptr(); + auto ksigns64_ptr_ct1 = ksigns64.get_ptr(); + + cgh.parallel_for( + sycl::nd_range<3>(block_nums * block_dims, block_dims), + [=](sycl::nd_item<3> item_ct1) + [[intel::reqd_sub_group_size(32)]] { + mul_mat_vec_q( + vx, vy, dst, ncols, nrows, item_ct1, + iq3xxs_grid_ptr_ct1, ksigns64_ptr_ct1); + }); + }); + } +} + +static void mul_mat_vec_q4_1_q8_1_sycl(const void *vx, const void *vy, + float *dst, const int ncols, + const int nrows, + dpct::queue_ptr stream) { + GGML_ASSERT(ncols % QK4_1 == 0); + const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y; + const sycl::range<3> block_nums(1, 1, block_num_y); + const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE); + { + iq3xxs_grid.init(*stream); + ksigns64.init(*stream); + + stream->submit([&](sycl::handler &cgh) { + auto iq3xxs_grid_ptr_ct1 = iq3xxs_grid.get_ptr(); + auto ksigns64_ptr_ct1 = ksigns64.get_ptr(); + + cgh.parallel_for( + sycl::nd_range<3>(block_nums * block_dims, block_dims), + [=](sycl::nd_item<3> item_ct1) + [[intel::reqd_sub_group_size(32)]] { + mul_mat_vec_q( + vx, vy, dst, ncols, nrows, item_ct1, + iq3xxs_grid_ptr_ct1, ksigns64_ptr_ct1); + }); + }); + } +} + +static void mul_mat_vec_q5_0_q8_1_sycl(const void *vx, const void *vy, + float *dst, const int ncols, + const int nrows, + dpct::queue_ptr stream) { + GGML_ASSERT(ncols % QK5_0 == 0); + const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y; + const sycl::range<3> block_nums(1, 1, block_num_y); + const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE); + { + iq3xxs_grid.init(*stream); + ksigns64.init(*stream); + + stream->submit([&](sycl::handler &cgh) { + auto iq3xxs_grid_ptr_ct1 = iq3xxs_grid.get_ptr(); + auto ksigns64_ptr_ct1 = ksigns64.get_ptr(); + + cgh.parallel_for( + sycl::nd_range<3>(block_nums * block_dims, block_dims), + [=](sycl::nd_item<3> item_ct1) + [[intel::reqd_sub_group_size(32)]] { + mul_mat_vec_q( + vx, vy, dst, ncols, nrows, item_ct1, + iq3xxs_grid_ptr_ct1, ksigns64_ptr_ct1); + }); + }); + } +} + +static void mul_mat_vec_q5_1_q8_1_sycl(const void *vx, const void *vy, + float *dst, const int ncols, + const int nrows, + dpct::queue_ptr stream) { + GGML_ASSERT(ncols % QK5_1 == 0); + const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y; + const sycl::range<3> block_nums(1, 1, block_num_y); + const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE); + { + iq3xxs_grid.init(*stream); + ksigns64.init(*stream); + + stream->submit([&](sycl::handler &cgh) { + auto iq3xxs_grid_ptr_ct1 = iq3xxs_grid.get_ptr(); + auto ksigns64_ptr_ct1 = ksigns64.get_ptr(); + + cgh.parallel_for( + sycl::nd_range<3>(block_nums * block_dims, block_dims), + [=](sycl::nd_item<3> item_ct1) + [[intel::reqd_sub_group_size(32)]] { + mul_mat_vec_q( + vx, vy, dst, ncols, nrows, item_ct1, + iq3xxs_grid_ptr_ct1, ksigns64_ptr_ct1); + }); + }); + } +} + +static void mul_mat_vec_q8_0_q8_1_sycl(const void *vx, const void *vy, + float *dst, const int ncols, + const int nrows, + dpct::queue_ptr stream) { + GGML_ASSERT(ncols % QK8_0 == 0); + const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y; + const sycl::range<3> block_nums(1, 1, block_num_y); + const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE); + { + iq3xxs_grid.init(*stream); + ksigns64.init(*stream); + + stream->submit([&](sycl::handler &cgh) { + auto iq3xxs_grid_ptr_ct1 = iq3xxs_grid.get_ptr(); + auto ksigns64_ptr_ct1 = ksigns64.get_ptr(); + + cgh.parallel_for( + sycl::nd_range<3>(block_nums * block_dims, block_dims), + [=](sycl::nd_item<3> item_ct1) + [[intel::reqd_sub_group_size(32)]] { + mul_mat_vec_q( + vx, vy, dst, ncols, nrows, item_ct1, + iq3xxs_grid_ptr_ct1, ksigns64_ptr_ct1); + }); + }); + } +} + +static void mul_mat_vec_q2_K_q8_1_sycl(const void *vx, const void *vy, + float *dst, const int ncols, + const int nrows, + dpct::queue_ptr stream) { + GGML_ASSERT(ncols % QK_K == 0); + const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y; + const sycl::range<3> block_nums(1, 1, block_num_y); + const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE); + { + iq3xxs_grid.init(*stream); + ksigns64.init(*stream); + + stream->submit([&](sycl::handler &cgh) { + auto iq3xxs_grid_ptr_ct1 = iq3xxs_grid.get_ptr(); + auto ksigns64_ptr_ct1 = ksigns64.get_ptr(); + + cgh.parallel_for( + sycl::nd_range<3>(block_nums * block_dims, block_dims), + [=](sycl::nd_item<3> item_ct1) + [[intel::reqd_sub_group_size(32)]] { + mul_mat_vec_q( + vx, vy, dst, ncols, nrows, item_ct1, + iq3xxs_grid_ptr_ct1, ksigns64_ptr_ct1); + }); + }); + } +} + +static void mul_mat_vec_q3_K_q8_1_sycl(const void *vx, const void *vy, + float *dst, const int ncols, + const int nrows, + dpct::queue_ptr stream) { + GGML_ASSERT(ncols % QK_K == 0); + const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y; + const sycl::range<3> block_nums(1, 1, block_num_y); + const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE); + { + iq3xxs_grid.init(*stream); + ksigns64.init(*stream); + + stream->submit([&](sycl::handler &cgh) { + auto iq3xxs_grid_ptr_ct1 = iq3xxs_grid.get_ptr(); + auto ksigns64_ptr_ct1 = ksigns64.get_ptr(); + + cgh.parallel_for( + sycl::nd_range<3>(block_nums * block_dims, block_dims), + [=](sycl::nd_item<3> item_ct1) + [[intel::reqd_sub_group_size(32)]] { + mul_mat_vec_q( + vx, vy, dst, ncols, nrows, item_ct1, + iq3xxs_grid_ptr_ct1, ksigns64_ptr_ct1); + }); + }); + } +} + +static void mul_mat_vec_q4_K_q8_1_sycl(const void *vx, const void *vy, + float *dst, const int ncols, + const int nrows, + dpct::queue_ptr stream) { + GGML_ASSERT(ncols % QK_K == 0); + const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y; + const sycl::range<3> block_nums(1, 1, block_num_y); + const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE); + { + iq3xxs_grid.init(*stream); + ksigns64.init(*stream); + + stream->submit([&](sycl::handler &cgh) { + auto iq3xxs_grid_ptr_ct1 = iq3xxs_grid.get_ptr(); + auto ksigns64_ptr_ct1 = ksigns64.get_ptr(); + + cgh.parallel_for( + sycl::nd_range<3>(block_nums * block_dims, block_dims), + [=](sycl::nd_item<3> item_ct1) + [[intel::reqd_sub_group_size(32)]] { + mul_mat_vec_q( + vx, vy, dst, ncols, nrows, item_ct1, + iq3xxs_grid_ptr_ct1, ksigns64_ptr_ct1); + }); + }); + } +} + +static void mul_mat_vec_q5_K_q8_1_sycl(const void *vx, const void *vy, + float *dst, const int ncols, + const int nrows, + dpct::queue_ptr stream) { + GGML_ASSERT(ncols % QK_K == 0); + const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y; + const sycl::range<3> block_nums(1, 1, block_num_y); + const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE); + { + iq3xxs_grid.init(*stream); + ksigns64.init(*stream); + + stream->submit([&](sycl::handler &cgh) { + auto iq3xxs_grid_ptr_ct1 = iq3xxs_grid.get_ptr(); + auto ksigns64_ptr_ct1 = ksigns64.get_ptr(); + + cgh.parallel_for( + sycl::nd_range<3>(block_nums * block_dims, block_dims), + [=](sycl::nd_item<3> item_ct1) + [[intel::reqd_sub_group_size(32)]] { + mul_mat_vec_q( + vx, vy, dst, ncols, nrows, item_ct1, + iq3xxs_grid_ptr_ct1, ksigns64_ptr_ct1); + }); + }); + } +} + +static void mul_mat_vec_q6_K_q8_1_sycl(const void *vx, const void *vy, + float *dst, const int ncols, + const int nrows, + dpct::queue_ptr stream) { + GGML_ASSERT(ncols % QK_K == 0); + const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y; + const sycl::range<3> block_nums(1, 1, block_num_y); + const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE); + { + iq3xxs_grid.init(*stream); + ksigns64.init(*stream); + + stream->submit([&](sycl::handler &cgh) { + auto iq3xxs_grid_ptr_ct1 = iq3xxs_grid.get_ptr(); + auto ksigns64_ptr_ct1 = ksigns64.get_ptr(); + + cgh.parallel_for( + sycl::nd_range<3>(block_nums * block_dims, block_dims), + [=](sycl::nd_item<3> item_ct1) + [[intel::reqd_sub_group_size(32)]] { + mul_mat_vec_q( + vx, vy, dst, ncols, nrows, item_ct1, + iq3xxs_grid_ptr_ct1, ksigns64_ptr_ct1); + }); + }); + } +} + +static void mul_mat_vec_iq2_xxs_q8_1_sycl(const void *vx, const void *vy, + float *dst, const int ncols, + const int nrows, + dpct::queue_ptr stream) { + GGML_ASSERT(ncols % QK_K == 0); + const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y; + const sycl::range<3> block_nums(1, 1, block_num_y); + const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE); + { + iq2xxs_grid.init(*stream); + ksigns_iq2xs.init(*stream); + kmask_iq2xs.init(*stream); + + + stream->submit([&](sycl::handler &cgh) { + auto iq2xxs_grid_ptr_ct1 = iq2xxs_grid.get_ptr(); + auto ksigns_iq2xs_ptr_ct1 = ksigns_iq2xs.get_ptr(); + auto kmask_iq2xs_ptr_ct1 = kmask_iq2xs.get_ptr(); + + cgh.parallel_for( + sycl::nd_range<3>(block_nums * block_dims, block_dims), + [=](sycl::nd_item<3> item_ct1) + [[intel::reqd_sub_group_size(32)]] { + mul_mat_vec_q_iq2_xxs_q8_1( + vx, vy, dst, ncols, nrows, item_ct1, + iq2xxs_grid_ptr_ct1, ksigns_iq2xs_ptr_ct1, kmask_iq2xs_ptr_ct1); + }); + }); + } +} + +static void mul_mat_vec_iq2_xs_q8_1_sycl(const void *vx, const void *vy, float *dst, const int ncols, const int nrows, dpct::queue_ptr stream) { - GGML_ASSERT(ncols % QK4_0 == 0); - const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y; - const sycl::range<3> block_nums(1, 1, block_num_y); - const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE); - stream->parallel_for( - sycl::nd_range<3>(block_nums * block_dims, block_dims), [= - ](sycl::nd_item<3> item_ct1) [[intel::reqd_sub_group_size(32)]] { - mul_mat_vec_q( - vx, vy, dst, ncols, nrows, item_ct1); - }); + GGML_ASSERT(ncols % QK_K == 0); + const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y; + const sycl::range<3> block_nums(1, 1, block_num_y); + const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE); + { + iq2xs_grid.init(*stream); + ksigns64.init(*stream); + + stream->submit([&](sycl::handler &cgh) { + auto iq2xs_grid_ptr_ct1 = iq2xs_grid.get_ptr(); + auto ksigns64_ptr_ct1 = ksigns64.get_ptr(); + + cgh.parallel_for( + sycl::nd_range<3>(block_nums * block_dims, block_dims), + [=](sycl::nd_item<3> item_ct1) + [[intel::reqd_sub_group_size(32)]] { + mul_mat_vec_q_iq2_xs_q8_1( + vx, vy, dst, ncols, nrows, item_ct1, + iq2xs_grid_ptr_ct1, ksigns64_ptr_ct1); + }); + }); + } } +static void mul_mat_vec_iq3_xxs_q8_1_sycl(const void *vx, const void *vy, + float *dst, const int ncols, + const int nrows, + dpct::queue_ptr stream) { + GGML_ASSERT(ncols % QK_K == 0); + const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y; + const sycl::range<3> block_nums(1, 1, block_num_y); + const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE); + { + iq3xxs_grid.init(*stream); + ksigns64.init(*stream); + + stream->submit([&](sycl::handler &cgh) { + auto iq3xxs_grid_ptr_ct1 = iq3xxs_grid.get_ptr(); + auto ksigns64_ptr_ct1 = ksigns64.get_ptr(); + + cgh.parallel_for( + sycl::nd_range<3>(block_nums * block_dims, block_dims), + [=](sycl::nd_item<3> item_ct1) + [[intel::reqd_sub_group_size(32)]] { + mul_mat_vec_q_iq3_xxs_q8_1( + vx, vy, dst, ncols, nrows, item_ct1, + iq3xxs_grid_ptr_ct1, ksigns64_ptr_ct1); + }); + }); + } +} + + static void ggml_mul_mat_q4_0_q8_1_sycl(const void *vx, const void *vy, float *dst, const int ncols_x, const int nrows_x, const int ncols_y, @@ -11451,7 +13133,7 @@ struct sycl_pool_alloc { device_id = get_current_device_id(); device_index = g_sycl_gpu_mgr->get_index(device_id); ptr = (T *) ggml_sycl_pool_malloc(device_index, size * sizeof(T), &this->actual_size); - // GGML_SYCL_DEBUG("alloc %lu return %p actual size=%lu\n", size * sizeof(T), ptr, this->actual_size); + // GGML_SYCL_DEBUG("sycl_pool_alloc %lu return %p actual size=%lu\n", size * sizeof(T), ptr, this->actual_size); return ptr; } @@ -12242,63 +13924,46 @@ inline void ggml_sycl_op_mul_mat_vec_q( const int64_t ne00 = src0->ne[0]; const int64_t row_diff = row_high - row_low; - // TODO: support these quantization types - GGML_ASSERT(!(src0->type == GGML_TYPE_IQ2_XXS || - src0->type == GGML_TYPE_IQ2_XS || - src0->type == GGML_TYPE_IQ3_XXS || - src0->type == GGML_TYPE_IQ1_S)); - switch (src0->type) { case GGML_TYPE_Q4_0: - mul_mat_vec_q_sycl_submitter( - src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, stream); - break; + mul_mat_vec_q4_0_q8_1_sycl(src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, stream); + break; case GGML_TYPE_Q4_1: - mul_mat_vec_q_sycl_submitter( - src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, stream); - break; + mul_mat_vec_q4_1_q8_1_sycl(src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, stream); + break; case GGML_TYPE_Q5_0: - mul_mat_vec_q_sycl_submitter( - src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, stream); - break; + mul_mat_vec_q5_0_q8_1_sycl(src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, stream); + break; case GGML_TYPE_Q5_1: - mul_mat_vec_q_sycl_submitter( - src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, stream); - break; + mul_mat_vec_q5_1_q8_1_sycl(src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, stream); + break; case GGML_TYPE_Q8_0: - mul_mat_vec_q_sycl_submitter( - src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, stream); - break; + mul_mat_vec_q8_0_q8_1_sycl(src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, stream); + break; case GGML_TYPE_Q2_K: - mul_mat_vec_q_sycl_submitter( - src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, stream); - break; + mul_mat_vec_q2_K_q8_1_sycl(src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, stream); + break; case GGML_TYPE_Q3_K: - mul_mat_vec_q_sycl_submitter( - src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, stream); - break; + mul_mat_vec_q3_K_q8_1_sycl(src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, stream); + break; case GGML_TYPE_Q4_K: - mul_mat_vec_q_sycl_submitter( - src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, stream); - break; + mul_mat_vec_q4_K_q8_1_sycl(src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, stream); + break; case GGML_TYPE_Q5_K: - mul_mat_vec_q_sycl_submitter( - src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, stream); - break; + mul_mat_vec_q5_K_q8_1_sycl(src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, stream); + break; case GGML_TYPE_Q6_K: - mul_mat_vec_q_sycl_submitter( - src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, stream); - break; + mul_mat_vec_q6_K_q8_1_sycl(src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, stream); + break; + case GGML_TYPE_IQ2_XXS: + mul_mat_vec_iq2_xxs_q8_1_sycl(src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, stream); + break; + case GGML_TYPE_IQ2_XS: + mul_mat_vec_iq2_xs_q8_1_sycl(src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, stream); + break; + case GGML_TYPE_IQ3_XXS: + mul_mat_vec_iq3_xxs_q8_1_sycl(src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, stream); + break; default: GGML_ASSERT(false); break; @@ -12311,6 +13976,7 @@ inline void ggml_sycl_op_mul_mat_vec_q( (void) src1_padded_row_size; } + inline void ggml_sycl_op_dequantize_mul_mat_vec( const ggml_tensor *src0, const ggml_tensor *src1, ggml_tensor *dst, const char *src0_dd_i, const float *src1_ddf_i, const char *src1_ddq_i, @@ -12318,10 +13984,11 @@ inline void ggml_sycl_op_dequantize_mul_mat_vec( const int64_t src1_ncols, const int64_t src1_padded_row_size, const dpct::queue_ptr &stream) { - GGML_TENSOR_BINARY_OP_LOCALS; - + const int64_t ne00 = src0->ne[0]; const int64_t row_diff = row_high - row_low; + GGML_ASSERT(src1->type == GGML_TYPE_F32); + // on some GPUs it is faster to convert src1 to half and to use half precision intrinsics #ifdef GGML_SYCL_F16 sycl_pool_alloc src1_dfloat_a; @@ -12333,15 +14000,10 @@ inline void ggml_sycl_op_dequantize_mul_mat_vec( src0->type == GGML_TYPE_Q8_0 || src0->type == GGML_TYPE_F16; if (src1_convert_f16) { - if (src1->type == GGML_TYPE_F16) { - src1_dfloat = (sycl::half *)src1->data + src1_padded_row_size; - } else { - src1_dfloat = src1_dfloat_a.alloc(ne00); - ggml_cpy_f32_f16_sycl((const char *)src1_ddf_i, (char *)src1_dfloat, - ne00, ne00, ne01, ne02, nb00, nb01, nb02, - nb03, ne10, ne11, ne12, nb10, nb11, nb12, - nb13, stream); - } + src1_dfloat = src1_dfloat_a.alloc(ne00); + const to_fp16_sycl_t to_fp16_sycl = ggml_get_to_fp16_sycl(src1->type); + GGML_ASSERT(to_fp16_sycl != nullptr); + to_fp16_sycl(src1_ddf_i, src1_dfloat, ne00, stream); } #else const dfloat * src1_dfloat = (const dfloat *) src1_ddf_i; // dfloat == float, no conversion @@ -12495,7 +14157,7 @@ inline void ggml_sycl_op_mul_mat_sycl( *g_sycl_handles[id], oneapi::mkl::transpose::trans, oneapi::mkl::transpose::nontrans, row_diff, src1_ncols, ne10, dpct::get_value(&alpha, *g_sycl_handles[id]), src0_ddf_i, ne00, - src1_ddf_i, ne10, dpct::get_value(&beta, *g_sycl_handles[id]), + src1_ddf1_i, ne10, dpct::get_value(&beta, *g_sycl_handles[id]), dst_dd_i, ldc))); } (void) dst; @@ -12923,7 +14585,7 @@ static void ggml_sycl_op_flatten(const ggml_tensor *src0, // copy dst to host if necessary if (!dst_on_device) { SYCL_CHECK(CHECK_TRY_ERROR( - main_stream->memcpy(dst->data, dst_ddf, ggml_nbytes(dst)))); + main_stream->memcpy(dst->data, dst_ddf, ggml_nbytes(dst)).wait())); } if (dst->backend == GGML_BACKEND_TYPE_CPU) { @@ -13200,7 +14862,7 @@ static void ggml_sycl_op_mul_mat(const ggml_tensor *src0, SYCL_CHECK(CHECK_TRY_ERROR(stream->memcpy( src1_ddq_i, src1_ddq_i_source, src1_ncols * src1_padded_col_size * q8_1_ts / - q8_1_bs))); + q8_1_bs).wait())); } else { float * src1_ddf_i_source = (float *) src1_extra->data_device[g_main_device]; @@ -13294,7 +14956,7 @@ static void ggml_sycl_op_mul_mat(const ggml_tensor *src0, dhf_dst_i += src1_col_0*ne0; SYCL_CHECK(CHECK_TRY_ERROR( stream->memcpy(dhf_dst_i, dst_dd_i, - src1_ncols * ne0 * sizeof(float)))); + src1_ncols * ne0 * sizeof(float)).wait())); } } @@ -13852,13 +15514,13 @@ static __global__ void k_compute_batched_ptrs_id( src0_f16 = (half *) srcs_ar[i]; } else { src0_f16 = src0_as_f16; - if (threadIdx.x == 0 && threadIdx.y == 0) { + if (item_ct1.get_local_id(2) == 0 && threadIdx.y == 0) { const to_fp16_sycl_t to_fp16 = ggml_get_to_fp16_sycl(src0_type); to_fp16(srcs_ar[i], src0_f16, src0_ne, syclStreamFireAndForget); } } - int i13 = blockIdx.x * blockDim.x + threadIdx.x; + int i13 = blockIdx.x * blockDim.x + item_ct1.get_local_id(2); int i12 = blockIdx.y * blockDim.y + threadIdx.y; if (i13 >= ne13 || i12 >= ne12) { @@ -14024,8 +15686,8 @@ static void ggml_sycl_mul_mat_id(const ggml_tensor *src0, if (ids->backend == GGML_BACKEND_TYPE_GPU) { const char * ids_dev = (const char *)((const ggml_tensor_extra_gpu *)ids->extra)->data_device[g_main_device]; SYCL_CHECK(CHECK_TRY_ERROR( - stream->memcpy(ids_host.data(), ids_dev, ggml_nbytes(ids)))); - SYCL_CHECK(CHECK_TRY_ERROR(stream->wait())); + stream->memcpy(ids_host.data(), ids_dev, ggml_nbytes(ids)).wait())); + // SYCL_CHECK(CHECK_TRY_ERROR(stream->wait())); } else { memcpy(ids_host.data(), ids->data, ggml_nbytes(ids)); } @@ -14095,7 +15757,7 @@ static void ggml_sycl_mul_mat_id(const ggml_tensor *src0, SYCL_CHECK(CHECK_TRY_ERROR( stream->memcpy(src1_contiguous.get() + num_src1_rows * nb11, - src1_original + i01 * nb11, nb11))); + src1_original + i01 * nb11, nb11).wait())); num_src1_rows++; } @@ -14128,7 +15790,7 @@ static void ggml_sycl_mul_mat_id(const ggml_tensor *src0, SYCL_CHECK(CHECK_TRY_ERROR(stream->memcpy( dst_original + i01 * nb1, - dst_contiguous.get() + num_src1_rows * nb1, nb1))); + dst_contiguous.get() + num_src1_rows * nb1, nb1).wait())); num_src1_rows++; } } @@ -15522,7 +17184,7 @@ GGML_CALL static void ggml_backend_sycl_set_tensor_async(ggml_backend_t backend, GGML_ASSERT(tensor->buffer->buft == ggml_backend_sycl_buffer_type(sycl_ctx->device) && "unsupported buffer type"); GGML_ASSERT(tensor->backend == GGML_BACKEND_TYPE_GPU); SYCL_CHECK(CHECK_TRY_ERROR(g_syclStreams[sycl_ctx->device][0]->memcpy( - (char *)tensor->data + offset, data, size))); + (char *)tensor->data + offset, data, size).wait())); } catch (sycl::exception const &exc) { std::cerr << exc.what() << "Exception caught at file:" << __FILE__ @@ -15538,7 +17200,7 @@ GGML_CALL static void ggml_backend_sycl_get_tensor_async(ggml_backend_t backend, GGML_ASSERT(tensor->buffer->buft == ggml_backend_sycl_buffer_type(sycl_ctx->device) && "unsupported buffer type"); GGML_ASSERT(tensor->backend == GGML_BACKEND_TYPE_GPU); SYCL_CHECK(CHECK_TRY_ERROR(g_syclStreams[sycl_ctx->device][0]->memcpy( - data, (const char *)tensor->data + offset, size))); + data, (const char *)tensor->data + offset, size).wait())); } catch (sycl::exception const &exc) { std::cerr << exc.what() << "Exception caught at file:" << __FILE__ @@ -15557,7 +17219,7 @@ GGML_CALL static bool ggml_backend_sycl_cpy_tensor_async(ggml_backend_t backend, was inserted. You need to rewrite this code. */ SYCL_CHECK(CHECK_TRY_ERROR(g_syclStreams[sycl_ctx->device][0]->memcpy( - dst->data, src->data, ggml_nbytes(dst)))); + dst->data, src->data, ggml_nbytes(dst)).wait())); return true; } @@ -15647,20 +17309,12 @@ GGML_CALL static bool ggml_backend_sycl_supports_op(ggml_backend_t backend, cons if (a->ne[3] != b->ne[3]) { return false; } - - if (a->type == GGML_TYPE_IQ1_S) { + ggml_type a_type = a->type; + if (a_type == GGML_TYPE_IQ2_XXS || a_type == GGML_TYPE_IQ2_XS || a_type == GGML_TYPE_IQ3_XXS || + a_type == GGML_TYPE_IQ1_S || a_type == GGML_TYPE_IQ4_NL || a_type == GGML_TYPE_IQ3_S || + a_type == GGML_TYPE_IQ2_S || a_type == GGML_TYPE_IQ4_XS) { return false; } - if (a->type == GGML_TYPE_IQ3_XXS) { - return false; - } - if (a->type == GGML_TYPE_IQ2_XXS) { - return false; - } - if (a->type == GGML_TYPE_IQ2_XS) { - return false; - } - return true; } break; case GGML_OP_GET_ROWS: @@ -15705,15 +17359,15 @@ GGML_CALL static bool ggml_backend_sycl_supports_op(ggml_backend_t backend, cons } return false; } break; - case GGML_OP_DUP: - case GGML_OP_REPEAT: case GGML_OP_CONCAT: { ggml_type src0_type = op->src[0]->type; return src0_type != GGML_TYPE_I32 && src0_type != GGML_TYPE_I16; } break; + case GGML_OP_DUP: case GGML_OP_NONE: case GGML_OP_RESHAPE: + case GGML_OP_REPEAT: case GGML_OP_VIEW: case GGML_OP_PERMUTE: case GGML_OP_TRANSPOSE: From 61d1c88e155515dd03940913a5707ea84a8b119b Mon Sep 17 00:00:00 2001 From: 0cc4m Date: Tue, 5 Mar 2024 13:33:42 +0100 Subject: [PATCH 542/812] Vulkan Improvements (#5835) * Improve dequant shaders, add fast q4_0 dequant * Optimize dmmv non-kquants for GCN Remove unnecessary SPIR-V shader duplication * Fix q4_0 dequant dispatch sizes Fix backend free bug * Optimize dequant shaders for q4_1, q5_0, q5_1 and q8_0 * Add unary and binary op shader templates * Fix Vulkan check results * Enable non-contiguous support for simple ops * Add argsort Basic q4_0 mmq shader and unit test * Speed up q4_0 dequant code, enable mmq for q4_0 * Rework matmul pipeline selection * Add soft_max alibi support * Add q4_1, q5_0, q5_1 and q8_0 dequant mat mat mul shaders * Add environment variable GGML_VK_FORCE_MAX_ALLOCATION_SIZE to limit max buffer size Rename GGML_VULKAN_DISABLE_F16 to GGML_VK_DISABLE_F16 for consistency --- ggml-vulkan-shaders.hpp | 86931 ++++++++++++++++------------------ ggml-vulkan.cpp | 2128 +- ggml-vulkan.h | 1 + ggml_vk_generate_shaders.py | 1207 +- llama.cpp | 4 +- 5 files changed, 43476 insertions(+), 46795 deletions(-) diff --git a/ggml-vulkan-shaders.hpp b/ggml-vulkan-shaders.hpp index e5e7a8414..1eca4d6e0 100644 --- a/ggml-vulkan-shaders.hpp +++ b/ggml-vulkan-shaders.hpp @@ -2,50 +2,1847 @@ unsigned char add_f32_data[] = { 0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, -0x3e,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, +0x67,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, 0x01,0x00,0x00,0x00,0x0b,0x00,0x06,0x00,0x01,0x00,0x00,0x00, 0x47,0x4c,0x53,0x4c,0x2e,0x73,0x74,0x64,0x2e,0x34,0x35,0x30, 0x00,0x00,0x00,0x00,0x0e,0x00,0x03,0x00,0x00,0x00,0x00,0x00, 0x01,0x00,0x00,0x00,0x0f,0x00,0x0a,0x00,0x05,0x00,0x00,0x00, 0x04,0x00,0x00,0x00,0x6d,0x61,0x69,0x6e,0x00,0x00,0x00,0x00, -0x0b,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x22,0x00,0x00,0x00, -0x27,0x00,0x00,0x00,0x2f,0x00,0x00,0x00,0x10,0x00,0x06,0x00, +0x16,0x00,0x00,0x00,0x2e,0x01,0x00,0x00,0x3f,0x01,0x00,0x00, +0x4b,0x01,0x00,0x00,0x56,0x01,0x00,0x00,0x10,0x00,0x06,0x00, 0x04,0x00,0x00,0x00,0x11,0x00,0x00,0x00,0x00,0x02,0x00,0x00, +0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x14,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x14,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x14,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x14,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x14,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x10,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x14,0x00,0x00,0x00,0x05,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x14,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x14,0x00,0x00,0x00, +0x07,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x1c,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x14,0x00,0x00,0x00,0x08,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x14,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x24,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x14,0x00,0x00,0x00, +0x0a,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x28,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x14,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x2c,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x14,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x30,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x14,0x00,0x00,0x00, +0x0d,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x34,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x14,0x00,0x00,0x00,0x0e,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x38,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x14,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x3c,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x14,0x00,0x00,0x00, +0x10,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x40,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x14,0x00,0x00,0x00,0x11,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x44,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x14,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x48,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x14,0x00,0x00,0x00, +0x13,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x4c,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x14,0x00,0x00,0x00,0x14,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x50,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x14,0x00,0x00,0x00,0x15,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x54,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x14,0x00,0x00,0x00, +0x16,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x58,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x14,0x00,0x00,0x00,0x17,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x5c,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x14,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x60,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x14,0x00,0x00,0x00, +0x19,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x64,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x14,0x00,0x00,0x00,0x1a,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x68,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x14,0x00,0x00,0x00,0x1b,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x14,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x2e,0x01,0x00,0x00, +0x0b,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x3c,0x01,0x00,0x00,0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x48,0x00,0x04,0x00,0x3d,0x01,0x00,0x00,0x00,0x00,0x00,0x00, +0x19,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x3d,0x01,0x00,0x00, +0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x03,0x00,0x3d,0x01,0x00,0x00,0x02,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x3f,0x01,0x00,0x00,0x22,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x3f,0x01,0x00,0x00, +0x21,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x48,0x01,0x00,0x00,0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x48,0x00,0x04,0x00,0x49,0x01,0x00,0x00,0x00,0x00,0x00,0x00, +0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x49,0x01,0x00,0x00, +0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x03,0x00,0x49,0x01,0x00,0x00,0x02,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x4b,0x01,0x00,0x00,0x22,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x4b,0x01,0x00,0x00, +0x21,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x53,0x01,0x00,0x00,0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x48,0x00,0x04,0x00,0x54,0x01,0x00,0x00,0x00,0x00,0x00,0x00, +0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x54,0x01,0x00,0x00, +0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x03,0x00,0x54,0x01,0x00,0x00,0x02,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x56,0x01,0x00,0x00,0x22,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x56,0x01,0x00,0x00, +0x21,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x61,0x01,0x00,0x00,0x0b,0x00,0x00,0x00,0x19,0x00,0x00,0x00, +0x13,0x00,0x02,0x00,0x02,0x00,0x00,0x00,0x21,0x00,0x03,0x00, +0x03,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x15,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x1e,0x00,0x1e,0x00,0x14,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x15,0x00,0x00,0x00,0x09,0x00,0x00,0x00, +0x14,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x15,0x00,0x00,0x00, +0x16,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x15,0x00,0x04,0x00, +0x17,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x17,0x00,0x00,0x00,0x18,0x00,0x00,0x00, +0x03,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x19,0x00,0x00,0x00, +0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x17,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x17,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x17,0x00,0x00,0x00, +0x57,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x17,0x00,0x00,0x00,0x5c,0x00,0x00,0x00,0x07,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x17,0x00,0x00,0x00,0x62,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x17,0x00,0x00,0x00, +0x68,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x17,0x00,0x00,0x00,0xac,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x17,0x00,0x00,0x00,0xb0,0x00,0x00,0x00, +0x10,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x17,0x00,0x00,0x00, +0xb5,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x17,0x00,0x00,0x00,0xb9,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x17,0x00,0x00,0x00,0xbf,0x00,0x00,0x00, +0x0a,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x17,0x00,0x00,0x00, +0xc3,0x00,0x00,0x00,0x0e,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x17,0x00,0x00,0x00,0xc9,0x00,0x00,0x00,0x09,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x17,0x00,0x00,0x00,0xcd,0x00,0x00,0x00, +0x0d,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x17,0x00,0x00,0x00, +0xd6,0x00,0x00,0x00,0x13,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x17,0x00,0x00,0x00,0xd9,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x17,0x00,0x00,0x00,0xdd,0x00,0x00,0x00, +0x11,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x17,0x00,0x00,0x00, +0x14,0x01,0x00,0x00,0x18,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x17,0x00,0x00,0x00,0x19,0x01,0x00,0x00,0x17,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x17,0x00,0x00,0x00,0x1f,0x01,0x00,0x00, +0x16,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x17,0x00,0x00,0x00, +0x25,0x01,0x00,0x00,0x15,0x00,0x00,0x00,0x17,0x00,0x04,0x00, +0x2c,0x01,0x00,0x00,0x06,0x00,0x00,0x00,0x03,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x2d,0x01,0x00,0x00,0x01,0x00,0x00,0x00, +0x2c,0x01,0x00,0x00,0x3b,0x00,0x04,0x00,0x2d,0x01,0x00,0x00, +0x2e,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x2f,0x01,0x00,0x00,0x00,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x30,0x01,0x00,0x00,0x01,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x17,0x00,0x00,0x00, +0x33,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x14,0x00,0x02,0x00, +0x36,0x01,0x00,0x00,0x16,0x00,0x03,0x00,0x3b,0x01,0x00,0x00, +0x20,0x00,0x00,0x00,0x1d,0x00,0x03,0x00,0x3c,0x01,0x00,0x00, +0x3b,0x01,0x00,0x00,0x1e,0x00,0x03,0x00,0x3d,0x01,0x00,0x00, +0x3c,0x01,0x00,0x00,0x20,0x00,0x04,0x00,0x3e,0x01,0x00,0x00, +0x0c,0x00,0x00,0x00,0x3d,0x01,0x00,0x00,0x3b,0x00,0x04,0x00, +0x3e,0x01,0x00,0x00,0x3f,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x17,0x00,0x00,0x00,0x40,0x01,0x00,0x00, +0x19,0x00,0x00,0x00,0x1d,0x00,0x03,0x00,0x48,0x01,0x00,0x00, +0x3b,0x01,0x00,0x00,0x1e,0x00,0x03,0x00,0x49,0x01,0x00,0x00, +0x48,0x01,0x00,0x00,0x20,0x00,0x04,0x00,0x4a,0x01,0x00,0x00, +0x0c,0x00,0x00,0x00,0x49,0x01,0x00,0x00,0x3b,0x00,0x04,0x00, +0x4a,0x01,0x00,0x00,0x4b,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x50,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, +0x3b,0x01,0x00,0x00,0x1d,0x00,0x03,0x00,0x53,0x01,0x00,0x00, +0x3b,0x01,0x00,0x00,0x1e,0x00,0x03,0x00,0x54,0x01,0x00,0x00, +0x53,0x01,0x00,0x00,0x20,0x00,0x04,0x00,0x55,0x01,0x00,0x00, +0x0c,0x00,0x00,0x00,0x54,0x01,0x00,0x00,0x3b,0x00,0x04,0x00, +0x55,0x01,0x00,0x00,0x56,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x5f,0x01,0x00,0x00, +0x00,0x02,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x60,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x2c,0x00,0x06,0x00, +0x2c,0x01,0x00,0x00,0x61,0x01,0x00,0x00,0x5f,0x01,0x00,0x00, +0x60,0x01,0x00,0x00,0x60,0x01,0x00,0x00,0x36,0x00,0x05,0x00, +0x02,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x03,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x05,0x00,0x00,0x00, +0xf7,0x00,0x03,0x00,0x62,0x01,0x00,0x00,0x00,0x00,0x00,0x00, +0xfb,0x00,0x03,0x00,0x2f,0x01,0x00,0x00,0x63,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x63,0x01,0x00,0x00,0x41,0x00,0x05,0x00, +0x30,0x01,0x00,0x00,0x31,0x01,0x00,0x00,0x2e,0x01,0x00,0x00, +0x2f,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x32,0x01,0x00,0x00,0x31,0x01,0x00,0x00,0x41,0x00,0x05,0x00, +0x19,0x00,0x00,0x00,0x34,0x01,0x00,0x00,0x16,0x00,0x00,0x00, +0x33,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x35,0x01,0x00,0x00,0x34,0x01,0x00,0x00,0xae,0x00,0x05,0x00, +0x36,0x01,0x00,0x00,0x37,0x01,0x00,0x00,0x32,0x01,0x00,0x00, +0x35,0x01,0x00,0x00,0xf7,0x00,0x03,0x00,0x39,0x01,0x00,0x00, +0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x37,0x01,0x00,0x00, +0x38,0x01,0x00,0x00,0x39,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x38,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x62,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x39,0x01,0x00,0x00,0x41,0x00,0x05,0x00, +0x19,0x00,0x00,0x00,0x41,0x01,0x00,0x00,0x16,0x00,0x00,0x00, +0x40,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x42,0x01,0x00,0x00,0x41,0x01,0x00,0x00,0x41,0x00,0x05,0x00, +0x19,0x00,0x00,0x00,0x71,0x01,0x00,0x00,0x16,0x00,0x00,0x00, +0xd6,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x72,0x01,0x00,0x00,0x71,0x01,0x00,0x00,0x41,0x00,0x05,0x00, +0x19,0x00,0x00,0x00,0x73,0x01,0x00,0x00,0x16,0x00,0x00,0x00, +0xd9,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x74,0x01,0x00,0x00,0x73,0x01,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x75,0x01,0x00,0x00,0x72,0x01,0x00,0x00, +0x74,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0x19,0x00,0x00,0x00, +0x76,0x01,0x00,0x00,0x16,0x00,0x00,0x00,0xdd,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x77,0x01,0x00,0x00, +0x76,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x78,0x01,0x00,0x00,0x75,0x01,0x00,0x00,0x77,0x01,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x79,0x01,0x00,0x00, +0x32,0x01,0x00,0x00,0x78,0x01,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x7d,0x01,0x00,0x00,0x79,0x01,0x00,0x00, +0x72,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x80,0x01,0x00,0x00,0x7d,0x01,0x00,0x00,0x74,0x01,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x83,0x01,0x00,0x00, +0x80,0x01,0x00,0x00,0x77,0x01,0x00,0x00,0x82,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x86,0x01,0x00,0x00,0x32,0x01,0x00,0x00, +0x83,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x8b,0x01,0x00,0x00,0x74,0x01,0x00,0x00,0x77,0x01,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8c,0x01,0x00,0x00, +0x86,0x01,0x00,0x00,0x8b,0x01,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x90,0x01,0x00,0x00,0x8c,0x01,0x00,0x00, +0x74,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x93,0x01,0x00,0x00,0x90,0x01,0x00,0x00,0x77,0x01,0x00,0x00, +0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x98,0x01,0x00,0x00, +0x86,0x01,0x00,0x00,0x93,0x01,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x9b,0x01,0x00,0x00,0x98,0x01,0x00,0x00, +0x77,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xa4,0x01,0x00,0x00,0x9b,0x01,0x00,0x00,0x77,0x01,0x00,0x00, +0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa5,0x01,0x00,0x00, +0x98,0x01,0x00,0x00,0xa4,0x01,0x00,0x00,0x41,0x00,0x05,0x00, +0x19,0x00,0x00,0x00,0xa7,0x01,0x00,0x00,0x16,0x00,0x00,0x00, +0x14,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xa8,0x01,0x00,0x00,0xa7,0x01,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xa9,0x01,0x00,0x00,0x79,0x01,0x00,0x00, +0xa8,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0x19,0x00,0x00,0x00, +0xab,0x01,0x00,0x00,0x16,0x00,0x00,0x00,0x19,0x01,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xac,0x01,0x00,0x00, +0xab,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xad,0x01,0x00,0x00,0x8c,0x01,0x00,0x00,0xac,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xae,0x01,0x00,0x00, +0xa9,0x01,0x00,0x00,0xad,0x01,0x00,0x00,0x41,0x00,0x05,0x00, +0x19,0x00,0x00,0x00,0xb0,0x01,0x00,0x00,0x16,0x00,0x00,0x00, +0x1f,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xb1,0x01,0x00,0x00,0xb0,0x01,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xb2,0x01,0x00,0x00,0x9b,0x01,0x00,0x00, +0xb1,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xb3,0x01,0x00,0x00,0xae,0x01,0x00,0x00,0xb2,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0x19,0x00,0x00,0x00,0xb5,0x01,0x00,0x00, +0x16,0x00,0x00,0x00,0x25,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xb6,0x01,0x00,0x00,0xb5,0x01,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xb7,0x01,0x00,0x00, +0xa5,0x01,0x00,0x00,0xb6,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xb8,0x01,0x00,0x00,0xb3,0x01,0x00,0x00, +0xb7,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x47,0x01,0x00,0x00,0x42,0x01,0x00,0x00,0xb8,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0x19,0x00,0x00,0x00,0xc2,0x01,0x00,0x00, +0x16,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xc3,0x01,0x00,0x00,0xc2,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0x19,0x00,0x00,0x00,0xc4,0x01,0x00,0x00, +0x16,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xc5,0x01,0x00,0x00,0xc4,0x01,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xc6,0x01,0x00,0x00, +0xc3,0x01,0x00,0x00,0xc5,0x01,0x00,0x00,0x41,0x00,0x05,0x00, +0x19,0x00,0x00,0x00,0xc7,0x01,0x00,0x00,0x16,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xc8,0x01,0x00,0x00,0xc7,0x01,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xc9,0x01,0x00,0x00,0xc6,0x01,0x00,0x00, +0xc8,0x01,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xca,0x01,0x00,0x00,0x32,0x01,0x00,0x00,0xc9,0x01,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xce,0x01,0x00,0x00, +0xca,0x01,0x00,0x00,0xc3,0x01,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xd1,0x01,0x00,0x00,0xce,0x01,0x00,0x00, +0xc5,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xd4,0x01,0x00,0x00,0xd1,0x01,0x00,0x00,0xc8,0x01,0x00,0x00, +0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xd7,0x01,0x00,0x00, +0x32,0x01,0x00,0x00,0xd4,0x01,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xdc,0x01,0x00,0x00,0xc5,0x01,0x00,0x00, +0xc8,0x01,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xdd,0x01,0x00,0x00,0xd7,0x01,0x00,0x00,0xdc,0x01,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xe1,0x01,0x00,0x00, +0xdd,0x01,0x00,0x00,0xc5,0x01,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xe4,0x01,0x00,0x00,0xe1,0x01,0x00,0x00, +0xc8,0x01,0x00,0x00,0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xe9,0x01,0x00,0x00,0xd7,0x01,0x00,0x00,0xe4,0x01,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xec,0x01,0x00,0x00, +0xe9,0x01,0x00,0x00,0xc8,0x01,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xf5,0x01,0x00,0x00,0xec,0x01,0x00,0x00, +0xc8,0x01,0x00,0x00,0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xf6,0x01,0x00,0x00,0xe9,0x01,0x00,0x00,0xf5,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0x19,0x00,0x00,0x00,0xf8,0x01,0x00,0x00, +0x16,0x00,0x00,0x00,0x57,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xf9,0x01,0x00,0x00,0xf8,0x01,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xfa,0x01,0x00,0x00, +0xca,0x01,0x00,0x00,0xf9,0x01,0x00,0x00,0x41,0x00,0x05,0x00, +0x19,0x00,0x00,0x00,0xfc,0x01,0x00,0x00,0x16,0x00,0x00,0x00, +0x5c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xfd,0x01,0x00,0x00,0xfc,0x01,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xfe,0x01,0x00,0x00,0xdd,0x01,0x00,0x00, +0xfd,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xff,0x01,0x00,0x00,0xfa,0x01,0x00,0x00,0xfe,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0x19,0x00,0x00,0x00,0x01,0x02,0x00,0x00, +0x16,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x02,0x02,0x00,0x00,0x01,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x03,0x02,0x00,0x00, +0xec,0x01,0x00,0x00,0x02,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x04,0x02,0x00,0x00,0xff,0x01,0x00,0x00, +0x03,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x19,0x00,0x00,0x00, +0x06,0x02,0x00,0x00,0x16,0x00,0x00,0x00,0x68,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x07,0x02,0x00,0x00, +0x06,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x08,0x02,0x00,0x00,0xf6,0x01,0x00,0x00,0x07,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x09,0x02,0x00,0x00, +0x04,0x02,0x00,0x00,0x08,0x02,0x00,0x00,0x41,0x00,0x06,0x00, +0x50,0x01,0x00,0x00,0x51,0x01,0x00,0x00,0x4b,0x01,0x00,0x00, +0x33,0x01,0x00,0x00,0x09,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, +0x3b,0x01,0x00,0x00,0x52,0x01,0x00,0x00,0x51,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0x19,0x00,0x00,0x00,0x49,0x02,0x00,0x00, +0x16,0x00,0x00,0x00,0xac,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x4a,0x02,0x00,0x00,0x49,0x02,0x00,0x00, +0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x4b,0x02,0x00,0x00, +0xca,0x01,0x00,0x00,0x4a,0x02,0x00,0x00,0x41,0x00,0x05,0x00, +0x19,0x00,0x00,0x00,0x4c,0x02,0x00,0x00,0x16,0x00,0x00,0x00, +0xb0,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x4d,0x02,0x00,0x00,0x4c,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x4e,0x02,0x00,0x00,0x4b,0x02,0x00,0x00, +0x4d,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x19,0x00,0x00,0x00, +0x50,0x02,0x00,0x00,0x16,0x00,0x00,0x00,0xb5,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x51,0x02,0x00,0x00, +0x50,0x02,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x52,0x02,0x00,0x00,0xdd,0x01,0x00,0x00,0x51,0x02,0x00,0x00, +0x41,0x00,0x05,0x00,0x19,0x00,0x00,0x00,0x53,0x02,0x00,0x00, +0x16,0x00,0x00,0x00,0xb9,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x54,0x02,0x00,0x00,0x53,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x55,0x02,0x00,0x00, +0x52,0x02,0x00,0x00,0x54,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x56,0x02,0x00,0x00,0x4e,0x02,0x00,0x00, +0x55,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x19,0x00,0x00,0x00, +0x58,0x02,0x00,0x00,0x16,0x00,0x00,0x00,0xbf,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x59,0x02,0x00,0x00, +0x58,0x02,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x5a,0x02,0x00,0x00,0xec,0x01,0x00,0x00,0x59,0x02,0x00,0x00, +0x41,0x00,0x05,0x00,0x19,0x00,0x00,0x00,0x5b,0x02,0x00,0x00, +0x16,0x00,0x00,0x00,0xc3,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x5c,0x02,0x00,0x00,0x5b,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x5d,0x02,0x00,0x00, +0x5a,0x02,0x00,0x00,0x5c,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x5e,0x02,0x00,0x00,0x56,0x02,0x00,0x00, +0x5d,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x19,0x00,0x00,0x00, +0x60,0x02,0x00,0x00,0x16,0x00,0x00,0x00,0xc9,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x61,0x02,0x00,0x00, +0x60,0x02,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x62,0x02,0x00,0x00,0xf6,0x01,0x00,0x00,0x61,0x02,0x00,0x00, +0x41,0x00,0x05,0x00,0x19,0x00,0x00,0x00,0x63,0x02,0x00,0x00, +0x16,0x00,0x00,0x00,0xcd,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x64,0x02,0x00,0x00,0x63,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x65,0x02,0x00,0x00, +0x62,0x02,0x00,0x00,0x64,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x66,0x02,0x00,0x00,0x5e,0x02,0x00,0x00, +0x65,0x02,0x00,0x00,0x41,0x00,0x06,0x00,0x50,0x01,0x00,0x00, +0x5b,0x01,0x00,0x00,0x56,0x01,0x00,0x00,0x33,0x01,0x00,0x00, +0x66,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0x3b,0x01,0x00,0x00, +0x5c,0x01,0x00,0x00,0x5b,0x01,0x00,0x00,0x81,0x00,0x05,0x00, +0x3b,0x01,0x00,0x00,0x5d,0x01,0x00,0x00,0x52,0x01,0x00,0x00, +0x5c,0x01,0x00,0x00,0x41,0x00,0x06,0x00,0x50,0x01,0x00,0x00, +0x5e,0x01,0x00,0x00,0x3f,0x01,0x00,0x00,0x33,0x01,0x00,0x00, +0x47,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x5e,0x01,0x00,0x00, +0x5d,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x62,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x62,0x01,0x00,0x00,0xfd,0x00,0x01,0x00, +0x38,0x00,0x01,0x00, +}; +const uint64_t add_f32_len = 4276; + +unsigned char argsort_f32_data[] = { +0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, +0x2b,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, +0x01,0x00,0x00,0x00,0x0b,0x00,0x06,0x00,0x01,0x00,0x00,0x00, +0x47,0x4c,0x53,0x4c,0x2e,0x73,0x74,0x64,0x2e,0x34,0x35,0x30, +0x00,0x00,0x00,0x00,0x0e,0x00,0x03,0x00,0x00,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x0f,0x00,0x0a,0x00,0x05,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x6d,0x61,0x69,0x6e,0x00,0x00,0x00,0x00, +0x13,0x00,0x00,0x00,0x24,0x00,0x00,0x00,0x2b,0x00,0x00,0x00, +0x33,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x10,0x00,0x06,0x00, +0x04,0x00,0x00,0x00,0x11,0x00,0x00,0x00,0x00,0x04,0x00,0x00, 0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x0b,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x1c,0x00,0x00,0x00, +0x10,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x11,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, +0x11,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x21,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x24,0x00,0x00,0x00, +0x0b,0x00,0x00,0x00,0x1b,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x2b,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x1a,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x31,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x31,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x31,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x81,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00, +0x82,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x82,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, +0x82,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x84,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x84,0x00,0x00,0x00,0x21,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x06,0x01,0x00,0x00, +0x0b,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x13,0x00,0x02,0x00, +0x02,0x00,0x00,0x00,0x21,0x00,0x03,0x00,0x03,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x15,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x15,0x00,0x04,0x00, +0x0d,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x1d,0x00,0x03,0x00,0x10,0x00,0x00,0x00,0x0d,0x00,0x00,0x00, +0x1e,0x00,0x03,0x00,0x11,0x00,0x00,0x00,0x10,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x12,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x11,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x12,0x00,0x00,0x00, +0x13,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x0d,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x16,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x0d,0x00,0x00,0x00,0x17,0x00,0x04,0x00,0x22,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x23,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x22,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0x23,0x00,0x00,0x00,0x24,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x25,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x26,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0x23,0x00,0x00,0x00,0x2b,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x2c,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x1e,0x00,0x04,0x00, +0x31,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x32,0x00,0x00,0x00,0x09,0x00,0x00,0x00, +0x31,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x32,0x00,0x00,0x00, +0x33,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x34,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x14,0x00,0x02,0x00,0x37,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x50,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x51,0x00,0x00,0x00, +0x08,0x01,0x00,0x00,0x2b,0x00,0x04,0x00,0x0d,0x00,0x00,0x00, +0x78,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x16,0x00,0x03,0x00, +0x80,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, +0x81,0x00,0x00,0x00,0x80,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, +0x82,0x00,0x00,0x00,0x81,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x83,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x82,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0x83,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x8e,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x80,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x05,0x01,0x00,0x00,0x00,0x04,0x00,0x00, +0x2c,0x00,0x06,0x00,0x22,0x00,0x00,0x00,0x06,0x01,0x00,0x00, +0x05,0x01,0x00,0x00,0x2c,0x00,0x00,0x00,0x2c,0x00,0x00,0x00, +0x36,0x00,0x05,0x00,0x02,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0x05,0x00,0x00,0x00,0xf7,0x00,0x03,0x00,0x07,0x01,0x00,0x00, +0x00,0x00,0x00,0x00,0xfb,0x00,0x03,0x00,0x25,0x00,0x00,0x00, +0x08,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x08,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0x26,0x00,0x00,0x00,0x27,0x00,0x00,0x00, +0x24,0x00,0x00,0x00,0x25,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0x27,0x00,0x00,0x00, +0x7c,0x00,0x04,0x00,0x0d,0x00,0x00,0x00,0x29,0x00,0x00,0x00, +0x28,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x26,0x00,0x00,0x00, +0x2d,0x00,0x00,0x00,0x2b,0x00,0x00,0x00,0x2c,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, +0x2d,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x30,0x00,0x00,0x00,0x29,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x34,0x00,0x00,0x00,0x35,0x00,0x00,0x00,0x33,0x00,0x00,0x00, +0x14,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x36,0x00,0x00,0x00,0x35,0x00,0x00,0x00,0xae,0x00,0x05,0x00, +0x37,0x00,0x00,0x00,0x38,0x00,0x00,0x00,0x30,0x00,0x00,0x00, +0x36,0x00,0x00,0x00,0xf7,0x00,0x03,0x00,0x3a,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x38,0x00,0x00,0x00, +0x39,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0x39,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x07,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x3a,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, +0x36,0x00,0x00,0x00,0xb0,0x00,0x05,0x00,0x37,0x00,0x00,0x00, +0x4a,0x00,0x00,0x00,0x30,0x00,0x00,0x00,0x36,0x00,0x00,0x00, +0xf7,0x00,0x03,0x00,0x4c,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0x4b,0x00,0x00,0x00, +0x4c,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x4b,0x00,0x00,0x00, +0x41,0x00,0x06,0x00,0x16,0x00,0x00,0x00,0x4f,0x00,0x00,0x00, +0x13,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x29,0x00,0x00,0x00, +0x3e,0x00,0x03,0x00,0x4f,0x00,0x00,0x00,0x29,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x4c,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0x4c,0x00,0x00,0x00,0xe0,0x00,0x04,0x00,0x50,0x00,0x00,0x00, +0x50,0x00,0x00,0x00,0x51,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x53,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x53,0x00,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x26,0x01,0x00,0x00, +0x50,0x00,0x00,0x00,0x4c,0x00,0x00,0x00,0x04,0x01,0x00,0x00, +0x56,0x00,0x00,0x00,0xb2,0x00,0x05,0x00,0x37,0x00,0x00,0x00, +0x5b,0x00,0x00,0x00,0x26,0x01,0x00,0x00,0x36,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0x55,0x00,0x00,0x00,0x56,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x5b,0x00,0x00,0x00, +0x54,0x00,0x00,0x00,0x55,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0x54,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x5e,0x00,0x00,0x00,0x26,0x01,0x00,0x00,0x50,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x5f,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0x5f,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0x27,0x01,0x00,0x00,0x5e,0x00,0x00,0x00,0x54,0x00,0x00,0x00, +0x02,0x01,0x00,0x00,0x62,0x00,0x00,0x00,0xac,0x00,0x05,0x00, +0x37,0x00,0x00,0x00,0x65,0x00,0x00,0x00,0x27,0x01,0x00,0x00, +0x25,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x61,0x00,0x00,0x00, +0x62,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x65,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x61,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0x60,0x00,0x00,0x00,0xc6,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x6a,0x00,0x00,0x00,0x30,0x00,0x00,0x00, +0x27,0x01,0x00,0x00,0xac,0x00,0x05,0x00,0x37,0x00,0x00,0x00, +0x6e,0x00,0x00,0x00,0x6a,0x00,0x00,0x00,0x30,0x00,0x00,0x00, +0xf7,0x00,0x03,0x00,0x70,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x6e,0x00,0x00,0x00,0x6f,0x00,0x00,0x00, +0x70,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x6f,0x00,0x00,0x00, +0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x74,0x00,0x00,0x00, +0x30,0x00,0x00,0x00,0x26,0x01,0x00,0x00,0xaa,0x00,0x05,0x00, +0x37,0x00,0x00,0x00,0x75,0x00,0x00,0x00,0x74,0x00,0x00,0x00, +0x25,0x00,0x00,0x00,0xf7,0x00,0x03,0x00,0x77,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x75,0x00,0x00,0x00, +0x76,0x00,0x00,0x00,0xc0,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0x76,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x34,0x00,0x00,0x00, +0x79,0x00,0x00,0x00,0x33,0x00,0x00,0x00,0x78,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x7a,0x00,0x00,0x00, +0x79,0x00,0x00,0x00,0xab,0x00,0x05,0x00,0x37,0x00,0x00,0x00, +0x7b,0x00,0x00,0x00,0x7a,0x00,0x00,0x00,0x25,0x00,0x00,0x00, +0xf7,0x00,0x03,0x00,0x7f,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x7b,0x00,0x00,0x00,0x7e,0x00,0x00,0x00, +0x9c,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x7e,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x89,0x00,0x00,0x00, +0x40,0x00,0x00,0x00,0x30,0x00,0x00,0x00,0x41,0x00,0x06,0x00, +0x16,0x00,0x00,0x00,0x8a,0x00,0x00,0x00,0x13,0x00,0x00,0x00, +0x14,0x00,0x00,0x00,0x89,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x0d,0x00,0x00,0x00,0x8b,0x00,0x00,0x00,0x8a,0x00,0x00,0x00, +0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x8c,0x00,0x00,0x00, +0x8b,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x8d,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x8c,0x00,0x00,0x00, +0x41,0x00,0x06,0x00,0x8e,0x00,0x00,0x00,0x8f,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x8d,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x80,0x00,0x00,0x00,0x90,0x00,0x00,0x00, +0x8f,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x94,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x6a,0x00,0x00,0x00, +0x41,0x00,0x06,0x00,0x16,0x00,0x00,0x00,0x95,0x00,0x00,0x00, +0x13,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x94,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x0d,0x00,0x00,0x00,0x96,0x00,0x00,0x00, +0x95,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x97,0x00,0x00,0x00,0x96,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x98,0x00,0x00,0x00,0x40,0x00,0x00,0x00, +0x97,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x8e,0x00,0x00,0x00, +0x99,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x14,0x00,0x00,0x00, +0x98,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x80,0x00,0x00,0x00, +0x9a,0x00,0x00,0x00,0x99,0x00,0x00,0x00,0xba,0x00,0x05,0x00, +0x37,0x00,0x00,0x00,0x9b,0x00,0x00,0x00,0x90,0x00,0x00,0x00, +0x9a,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x7f,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0x9c,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xa1,0x00,0x00,0x00,0x40,0x00,0x00,0x00, +0x30,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x16,0x00,0x00,0x00, +0xa2,0x00,0x00,0x00,0x13,0x00,0x00,0x00,0x14,0x00,0x00,0x00, +0xa1,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x0d,0x00,0x00,0x00, +0xa3,0x00,0x00,0x00,0xa2,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xa4,0x00,0x00,0x00,0xa3,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa5,0x00,0x00,0x00, +0x40,0x00,0x00,0x00,0xa4,0x00,0x00,0x00,0x41,0x00,0x06,0x00, +0x8e,0x00,0x00,0x00,0xa6,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0x14,0x00,0x00,0x00,0xa5,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x80,0x00,0x00,0x00,0xa7,0x00,0x00,0x00,0xa6,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xab,0x00,0x00,0x00, +0x40,0x00,0x00,0x00,0x6a,0x00,0x00,0x00,0x41,0x00,0x06,0x00, +0x16,0x00,0x00,0x00,0xac,0x00,0x00,0x00,0x13,0x00,0x00,0x00, +0x14,0x00,0x00,0x00,0xab,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x0d,0x00,0x00,0x00,0xad,0x00,0x00,0x00,0xac,0x00,0x00,0x00, +0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xae,0x00,0x00,0x00, +0xad,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xaf,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0xae,0x00,0x00,0x00, +0x41,0x00,0x06,0x00,0x8e,0x00,0x00,0x00,0xb0,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0xaf,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x80,0x00,0x00,0x00,0xb1,0x00,0x00,0x00, +0xb0,0x00,0x00,0x00,0xb8,0x00,0x05,0x00,0x37,0x00,0x00,0x00, +0xb2,0x00,0x00,0x00,0xa7,0x00,0x00,0x00,0xb1,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x7f,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0x7f,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x37,0x00,0x00,0x00, +0x2a,0x01,0x00,0x00,0x9b,0x00,0x00,0x00,0x7e,0x00,0x00,0x00, +0xb2,0x00,0x00,0x00,0x9c,0x00,0x00,0x00,0xf7,0x00,0x03,0x00, +0xb5,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x2a,0x01,0x00,0x00,0xb4,0x00,0x00,0x00,0xb5,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xb4,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xb9,0x00,0x00,0x00,0x40,0x00,0x00,0x00, +0x30,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xbc,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x6a,0x00,0x00,0x00, +0x41,0x00,0x06,0x00,0x16,0x00,0x00,0x00,0x0f,0x01,0x00,0x00, +0x13,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0xb9,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x0d,0x00,0x00,0x00,0x10,0x01,0x00,0x00, +0x0f,0x01,0x00,0x00,0x41,0x00,0x06,0x00,0x16,0x00,0x00,0x00, +0x13,0x01,0x00,0x00,0x13,0x00,0x00,0x00,0x14,0x00,0x00,0x00, +0xbc,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x0d,0x00,0x00,0x00, +0x14,0x01,0x00,0x00,0x13,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, +0x0f,0x01,0x00,0x00,0x14,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, +0x13,0x01,0x00,0x00,0x10,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0xb5,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xb5,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x77,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xc0,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x34,0x00,0x00,0x00, +0xc1,0x00,0x00,0x00,0x33,0x00,0x00,0x00,0x78,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xc2,0x00,0x00,0x00, +0xc1,0x00,0x00,0x00,0xab,0x00,0x05,0x00,0x37,0x00,0x00,0x00, +0xc3,0x00,0x00,0x00,0xc2,0x00,0x00,0x00,0x25,0x00,0x00,0x00, +0xf7,0x00,0x03,0x00,0xc6,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0xc3,0x00,0x00,0x00,0xc5,0x00,0x00,0x00, +0xdd,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xc5,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xcb,0x00,0x00,0x00, +0x40,0x00,0x00,0x00,0x30,0x00,0x00,0x00,0x41,0x00,0x06,0x00, +0x16,0x00,0x00,0x00,0xcc,0x00,0x00,0x00,0x13,0x00,0x00,0x00, +0x14,0x00,0x00,0x00,0xcb,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x0d,0x00,0x00,0x00,0xcd,0x00,0x00,0x00,0xcc,0x00,0x00,0x00, +0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xce,0x00,0x00,0x00, +0xcd,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xcf,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0xce,0x00,0x00,0x00, +0x41,0x00,0x06,0x00,0x8e,0x00,0x00,0x00,0xd0,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0xcf,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x80,0x00,0x00,0x00,0xd1,0x00,0x00,0x00, +0xd0,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xd5,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x6a,0x00,0x00,0x00, +0x41,0x00,0x06,0x00,0x16,0x00,0x00,0x00,0xd6,0x00,0x00,0x00, +0x13,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0xd5,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x0d,0x00,0x00,0x00,0xd7,0x00,0x00,0x00, +0xd6,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xd8,0x00,0x00,0x00,0xd7,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xd9,0x00,0x00,0x00,0x40,0x00,0x00,0x00, +0xd8,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x8e,0x00,0x00,0x00, +0xda,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x14,0x00,0x00,0x00, +0xd9,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x80,0x00,0x00,0x00, +0xdb,0x00,0x00,0x00,0xda,0x00,0x00,0x00,0xb8,0x00,0x05,0x00, +0x37,0x00,0x00,0x00,0xdc,0x00,0x00,0x00,0xd1,0x00,0x00,0x00, +0xdb,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xc6,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xdd,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xe2,0x00,0x00,0x00,0x40,0x00,0x00,0x00, +0x30,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x16,0x00,0x00,0x00, +0xe3,0x00,0x00,0x00,0x13,0x00,0x00,0x00,0x14,0x00,0x00,0x00, +0xe2,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x0d,0x00,0x00,0x00, +0xe4,0x00,0x00,0x00,0xe3,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xe5,0x00,0x00,0x00,0xe4,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xe6,0x00,0x00,0x00, +0x40,0x00,0x00,0x00,0xe5,0x00,0x00,0x00,0x41,0x00,0x06,0x00, +0x8e,0x00,0x00,0x00,0xe7,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0x14,0x00,0x00,0x00,0xe6,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x80,0x00,0x00,0x00,0xe8,0x00,0x00,0x00,0xe7,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xec,0x00,0x00,0x00, +0x40,0x00,0x00,0x00,0x6a,0x00,0x00,0x00,0x41,0x00,0x06,0x00, +0x16,0x00,0x00,0x00,0xed,0x00,0x00,0x00,0x13,0x00,0x00,0x00, +0x14,0x00,0x00,0x00,0xec,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x0d,0x00,0x00,0x00,0xee,0x00,0x00,0x00,0xed,0x00,0x00,0x00, +0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xef,0x00,0x00,0x00, +0xee,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xf0,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0xef,0x00,0x00,0x00, +0x41,0x00,0x06,0x00,0x8e,0x00,0x00,0x00,0xf1,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0xf0,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x80,0x00,0x00,0x00,0xf2,0x00,0x00,0x00, +0xf1,0x00,0x00,0x00,0xba,0x00,0x05,0x00,0x37,0x00,0x00,0x00, +0xf3,0x00,0x00,0x00,0xe8,0x00,0x00,0x00,0xf2,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xc6,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xc6,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x37,0x00,0x00,0x00, +0x29,0x01,0x00,0x00,0xdc,0x00,0x00,0x00,0xc5,0x00,0x00,0x00, +0xf3,0x00,0x00,0x00,0xdd,0x00,0x00,0x00,0xf7,0x00,0x03,0x00, +0xf6,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x29,0x01,0x00,0x00,0xf5,0x00,0x00,0x00,0xf6,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xf5,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xfa,0x00,0x00,0x00,0x40,0x00,0x00,0x00, +0x30,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xfd,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x6a,0x00,0x00,0x00, +0x41,0x00,0x06,0x00,0x16,0x00,0x00,0x00,0x1c,0x01,0x00,0x00, +0x13,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0xfa,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x0d,0x00,0x00,0x00,0x1d,0x01,0x00,0x00, +0x1c,0x01,0x00,0x00,0x41,0x00,0x06,0x00,0x16,0x00,0x00,0x00, +0x20,0x01,0x00,0x00,0x13,0x00,0x00,0x00,0x14,0x00,0x00,0x00, +0xfd,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x0d,0x00,0x00,0x00, +0x21,0x01,0x00,0x00,0x20,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, +0x1c,0x01,0x00,0x00,0x21,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, +0x20,0x01,0x00,0x00,0x1d,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0xf6,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xf6,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x77,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0x77,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x70,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0x70,0x00,0x00,0x00,0xe0,0x00,0x04,0x00, +0x50,0x00,0x00,0x00,0x50,0x00,0x00,0x00,0x51,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x62,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0x62,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x02,0x01,0x00,0x00,0x27,0x01,0x00,0x00,0x50,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x5f,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0x61,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x56,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0x56,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x04,0x01,0x00,0x00,0x26,0x01,0x00,0x00, +0x50,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x53,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0x55,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x07,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x07,0x01,0x00,0x00, +0xfd,0x00,0x01,0x00,0x38,0x00,0x01,0x00, +}; +const uint64_t argsort_f32_len = 3752; + +unsigned char clamp_f32_data[] = { +0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, +0xb6,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, +0x01,0x00,0x00,0x00,0x0b,0x00,0x06,0x00,0x01,0x00,0x00,0x00, +0x47,0x4c,0x53,0x4c,0x2e,0x73,0x74,0x64,0x2e,0x34,0x35,0x30, +0x00,0x00,0x00,0x00,0x0e,0x00,0x03,0x00,0x00,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x0f,0x00,0x09,0x00,0x05,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x6d,0x61,0x69,0x6e,0x00,0x00,0x00,0x00, +0x14,0x00,0x00,0x00,0xc7,0x00,0x00,0x00,0xd9,0x00,0x00,0x00, +0xe4,0x00,0x00,0x00,0x10,0x00,0x06,0x00,0x04,0x00,0x00,0x00, +0x11,0x00,0x00,0x00,0x00,0x02,0x00,0x00,0x01,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x12,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x12,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x12,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x08,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x12,0x00,0x00,0x00, +0x03,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x12,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x12,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x14,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x12,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x18,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x12,0x00,0x00,0x00,0x07,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x12,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x12,0x00,0x00,0x00, +0x09,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x24,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x12,0x00,0x00,0x00,0x0a,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x12,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x2c,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x12,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x30,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x12,0x00,0x00,0x00,0x0d,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x12,0x00,0x00,0x00,0x0e,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x38,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x12,0x00,0x00,0x00, +0x0f,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x3c,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x12,0x00,0x00,0x00,0x10,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x12,0x00,0x00,0x00,0x11,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x44,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x12,0x00,0x00,0x00, +0x12,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x48,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x12,0x00,0x00,0x00,0x13,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x4c,0x00,0x00,0x00,0x47,0x00,0x03,0x00, +0x12,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0xc7,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x1c,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0xd6,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0xd7,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0xd7,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0xd7,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xd9,0x00,0x00,0x00, +0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0xd9,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0xe1,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0xe2,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0xe2,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0xe2,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xe4,0x00,0x00,0x00, +0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0xe4,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x0a,0x01,0x00,0x00,0x0b,0x00,0x00,0x00, +0x19,0x00,0x00,0x00,0x13,0x00,0x02,0x00,0x02,0x00,0x00,0x00, +0x21,0x00,0x03,0x00,0x03,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x15,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x16,0x00,0x03,0x00,0x11,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0x1e,0x00,0x16,0x00,0x12,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x11,0x00,0x00,0x00,0x11,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x14,0x00,0x00,0x00, +0x09,0x00,0x00,0x00,0x15,0x00,0x04,0x00,0x15,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x15,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x03,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x17,0x00,0x00,0x00,0x09,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x15,0x00,0x00,0x00, +0x1a,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x15,0x00,0x00,0x00,0x1e,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x15,0x00,0x00,0x00,0x55,0x00,0x00,0x00, +0x08,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x15,0x00,0x00,0x00, +0x5a,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x15,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x15,0x00,0x00,0x00,0x66,0x00,0x00,0x00, +0x05,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x15,0x00,0x00,0x00, +0x6f,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x15,0x00,0x00,0x00,0x72,0x00,0x00,0x00,0x0a,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x15,0x00,0x00,0x00,0x76,0x00,0x00,0x00, +0x09,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x15,0x00,0x00,0x00, +0xad,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x15,0x00,0x00,0x00,0xb2,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x15,0x00,0x00,0x00,0xb8,0x00,0x00,0x00, +0x0e,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x15,0x00,0x00,0x00, +0xbe,0x00,0x00,0x00,0x0d,0x00,0x00,0x00,0x17,0x00,0x04,0x00, +0xc5,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x03,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0xc6,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0xc5,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0xc6,0x00,0x00,0x00, +0xc7,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xc8,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0xc9,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x15,0x00,0x00,0x00, +0xcc,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x14,0x00,0x02,0x00, +0xcf,0x00,0x00,0x00,0x1d,0x00,0x03,0x00,0xd6,0x00,0x00,0x00, +0x11,0x00,0x00,0x00,0x1e,0x00,0x03,0x00,0xd7,0x00,0x00,0x00, +0xd6,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0xd8,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0xd7,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0xd8,0x00,0x00,0x00,0xd9,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0xde,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x11,0x00,0x00,0x00,0x1d,0x00,0x03,0x00,0xe1,0x00,0x00,0x00, +0x11,0x00,0x00,0x00,0x1e,0x00,0x03,0x00,0xe2,0x00,0x00,0x00, +0xe1,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0xe3,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0xe2,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0xe3,0x00,0x00,0x00,0xe4,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x15,0x00,0x00,0x00,0xe5,0x00,0x00,0x00, +0x11,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x15,0x00,0x00,0x00, +0xee,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0xef,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x11,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x15,0x00,0x00,0x00,0xfa,0x00,0x00,0x00, +0x13,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x08,0x01,0x00,0x00,0x00,0x02,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x09,0x01,0x00,0x00,0x01,0x00,0x00,0x00, +0x2c,0x00,0x06,0x00,0xc5,0x00,0x00,0x00,0x0a,0x01,0x00,0x00, +0x08,0x01,0x00,0x00,0x09,0x01,0x00,0x00,0x09,0x01,0x00,0x00, +0x36,0x00,0x05,0x00,0x02,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0x05,0x00,0x00,0x00,0xf7,0x00,0x03,0x00,0x0b,0x01,0x00,0x00, +0x00,0x00,0x00,0x00,0xfb,0x00,0x03,0x00,0xc8,0x00,0x00,0x00, +0x0c,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x0c,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0xc9,0x00,0x00,0x00,0xca,0x00,0x00,0x00, +0xc7,0x00,0x00,0x00,0xc8,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xcb,0x00,0x00,0x00,0xca,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00,0xcd,0x00,0x00,0x00, +0x14,0x00,0x00,0x00,0xcc,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xce,0x00,0x00,0x00,0xcd,0x00,0x00,0x00, +0xae,0x00,0x05,0x00,0xcf,0x00,0x00,0x00,0xd0,0x00,0x00,0x00, +0xcb,0x00,0x00,0x00,0xce,0x00,0x00,0x00,0xf7,0x00,0x03,0x00, +0xd2,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0xd0,0x00,0x00,0x00,0xd1,0x00,0x00,0x00,0xd2,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xd1,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x0b,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xd2,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00,0x1a,0x01,0x00,0x00, +0x14,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x1b,0x01,0x00,0x00,0x1a,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00,0x1c,0x01,0x00,0x00, +0x14,0x00,0x00,0x00,0x1a,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x1d,0x01,0x00,0x00,0x1c,0x01,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x1e,0x01,0x00,0x00, +0x1b,0x01,0x00,0x00,0x1d,0x01,0x00,0x00,0x41,0x00,0x05,0x00, +0x17,0x00,0x00,0x00,0x1f,0x01,0x00,0x00,0x14,0x00,0x00,0x00, +0x1e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x20,0x01,0x00,0x00,0x1f,0x01,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x21,0x01,0x00,0x00,0x1e,0x01,0x00,0x00, +0x20,0x01,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x22,0x01,0x00,0x00,0xcb,0x00,0x00,0x00,0x21,0x01,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x26,0x01,0x00,0x00, +0x22,0x01,0x00,0x00,0x1b,0x01,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x29,0x01,0x00,0x00,0x26,0x01,0x00,0x00, +0x1d,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x2c,0x01,0x00,0x00,0x29,0x01,0x00,0x00,0x20,0x01,0x00,0x00, +0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x2f,0x01,0x00,0x00, +0xcb,0x00,0x00,0x00,0x2c,0x01,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x34,0x01,0x00,0x00,0x1d,0x01,0x00,0x00, +0x20,0x01,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x35,0x01,0x00,0x00,0x2f,0x01,0x00,0x00,0x34,0x01,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x39,0x01,0x00,0x00, +0x35,0x01,0x00,0x00,0x1d,0x01,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x3c,0x01,0x00,0x00,0x39,0x01,0x00,0x00, +0x20,0x01,0x00,0x00,0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x41,0x01,0x00,0x00,0x2f,0x01,0x00,0x00,0x3c,0x01,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x44,0x01,0x00,0x00, +0x41,0x01,0x00,0x00,0x20,0x01,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x4d,0x01,0x00,0x00,0x44,0x01,0x00,0x00, +0x20,0x01,0x00,0x00,0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x4e,0x01,0x00,0x00,0x41,0x01,0x00,0x00,0x4d,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00,0x50,0x01,0x00,0x00, +0x14,0x00,0x00,0x00,0x55,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x51,0x01,0x00,0x00,0x50,0x01,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x52,0x01,0x00,0x00, +0x22,0x01,0x00,0x00,0x51,0x01,0x00,0x00,0x41,0x00,0x05,0x00, +0x17,0x00,0x00,0x00,0x54,0x01,0x00,0x00,0x14,0x00,0x00,0x00, +0x5a,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x55,0x01,0x00,0x00,0x54,0x01,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x56,0x01,0x00,0x00,0x35,0x01,0x00,0x00, +0x55,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x57,0x01,0x00,0x00,0x52,0x01,0x00,0x00,0x56,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00,0x59,0x01,0x00,0x00, +0x14,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x5a,0x01,0x00,0x00,0x59,0x01,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x5b,0x01,0x00,0x00, +0x44,0x01,0x00,0x00,0x5a,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x5c,0x01,0x00,0x00,0x57,0x01,0x00,0x00, +0x5b,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00, +0x5e,0x01,0x00,0x00,0x14,0x00,0x00,0x00,0x66,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x5f,0x01,0x00,0x00, +0x5e,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x60,0x01,0x00,0x00,0x4e,0x01,0x00,0x00,0x5f,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x61,0x01,0x00,0x00, +0x5c,0x01,0x00,0x00,0x60,0x01,0x00,0x00,0x41,0x00,0x06,0x00, +0xde,0x00,0x00,0x00,0xdf,0x00,0x00,0x00,0xd9,0x00,0x00,0x00, +0xcc,0x00,0x00,0x00,0x61,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, +0x11,0x00,0x00,0x00,0xe0,0x00,0x00,0x00,0xdf,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00,0xe6,0x00,0x00,0x00, +0x14,0x00,0x00,0x00,0xe5,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xe7,0x00,0x00,0x00,0xe6,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00,0x6b,0x01,0x00,0x00, +0x14,0x00,0x00,0x00,0x6f,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x6c,0x01,0x00,0x00,0x6b,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00,0x6d,0x01,0x00,0x00, +0x14,0x00,0x00,0x00,0x72,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x6e,0x01,0x00,0x00,0x6d,0x01,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x6f,0x01,0x00,0x00, +0x6c,0x01,0x00,0x00,0x6e,0x01,0x00,0x00,0x41,0x00,0x05,0x00, +0x17,0x00,0x00,0x00,0x70,0x01,0x00,0x00,0x14,0x00,0x00,0x00, +0x76,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x71,0x01,0x00,0x00,0x70,0x01,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x72,0x01,0x00,0x00,0x6f,0x01,0x00,0x00, +0x71,0x01,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x73,0x01,0x00,0x00,0xcb,0x00,0x00,0x00,0x72,0x01,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x77,0x01,0x00,0x00, +0x73,0x01,0x00,0x00,0x6c,0x01,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x7a,0x01,0x00,0x00,0x77,0x01,0x00,0x00, +0x6e,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x7d,0x01,0x00,0x00,0x7a,0x01,0x00,0x00,0x71,0x01,0x00,0x00, +0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x80,0x01,0x00,0x00, +0xcb,0x00,0x00,0x00,0x7d,0x01,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x85,0x01,0x00,0x00,0x6e,0x01,0x00,0x00, +0x71,0x01,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x86,0x01,0x00,0x00,0x80,0x01,0x00,0x00,0x85,0x01,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8a,0x01,0x00,0x00, +0x86,0x01,0x00,0x00,0x6e,0x01,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x8d,0x01,0x00,0x00,0x8a,0x01,0x00,0x00, +0x71,0x01,0x00,0x00,0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x92,0x01,0x00,0x00,0x80,0x01,0x00,0x00,0x8d,0x01,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x95,0x01,0x00,0x00, +0x92,0x01,0x00,0x00,0x71,0x01,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x9e,0x01,0x00,0x00,0x95,0x01,0x00,0x00, +0x71,0x01,0x00,0x00,0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x9f,0x01,0x00,0x00,0x92,0x01,0x00,0x00,0x9e,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00,0xa1,0x01,0x00,0x00, +0x14,0x00,0x00,0x00,0xad,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xa2,0x01,0x00,0x00,0xa1,0x01,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa3,0x01,0x00,0x00, +0x73,0x01,0x00,0x00,0xa2,0x01,0x00,0x00,0x41,0x00,0x05,0x00, +0x17,0x00,0x00,0x00,0xa5,0x01,0x00,0x00,0x14,0x00,0x00,0x00, +0xb2,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xa6,0x01,0x00,0x00,0xa5,0x01,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xa7,0x01,0x00,0x00,0x86,0x01,0x00,0x00, +0xa6,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xa8,0x01,0x00,0x00,0xa3,0x01,0x00,0x00,0xa7,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00,0xaa,0x01,0x00,0x00, +0x14,0x00,0x00,0x00,0xb8,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xab,0x01,0x00,0x00,0xaa,0x01,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xac,0x01,0x00,0x00, +0x95,0x01,0x00,0x00,0xab,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xad,0x01,0x00,0x00,0xa8,0x01,0x00,0x00, +0xac,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00, +0xaf,0x01,0x00,0x00,0x14,0x00,0x00,0x00,0xbe,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xb0,0x01,0x00,0x00, +0xaf,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xb1,0x01,0x00,0x00,0x9f,0x01,0x00,0x00,0xb0,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xb2,0x01,0x00,0x00, +0xad,0x01,0x00,0x00,0xb1,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xec,0x00,0x00,0x00,0xe7,0x00,0x00,0x00, +0xb2,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0xef,0x00,0x00,0x00, +0xf0,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0xee,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x11,0x00,0x00,0x00,0xf1,0x00,0x00,0x00, +0xf0,0x00,0x00,0x00,0xb8,0x00,0x05,0x00,0xcf,0x00,0x00,0x00, +0xf2,0x00,0x00,0x00,0xe0,0x00,0x00,0x00,0xf1,0x00,0x00,0x00, +0xf7,0x00,0x03,0x00,0xf5,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0xf2,0x00,0x00,0x00,0xf4,0x00,0x00,0x00, +0xf8,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xf4,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xf5,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xf8,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0xef,0x00,0x00,0x00, +0xfb,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0xfa,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x11,0x00,0x00,0x00,0xfc,0x00,0x00,0x00, +0xfb,0x00,0x00,0x00,0xba,0x00,0x05,0x00,0xcf,0x00,0x00,0x00, +0xfd,0x00,0x00,0x00,0xe0,0x00,0x00,0x00,0xfc,0x00,0x00,0x00, +0xa9,0x00,0x06,0x00,0x11,0x00,0x00,0x00,0xb5,0x01,0x00,0x00, +0xfd,0x00,0x00,0x00,0xfc,0x00,0x00,0x00,0xe0,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xf5,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xf5,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x11,0x00,0x00,0x00, +0xb4,0x01,0x00,0x00,0xf1,0x00,0x00,0x00,0xf4,0x00,0x00,0x00, +0xb5,0x01,0x00,0x00,0xf8,0x00,0x00,0x00,0x41,0x00,0x06,0x00, +0xde,0x00,0x00,0x00,0x07,0x01,0x00,0x00,0xe4,0x00,0x00,0x00, +0xcc,0x00,0x00,0x00,0xec,0x00,0x00,0x00,0x3e,0x00,0x03,0x00, +0x07,0x01,0x00,0x00,0xb4,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0x0b,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x0b,0x01,0x00,0x00, +0xfd,0x00,0x01,0x00,0x38,0x00,0x01,0x00, +}; +const uint64_t clamp_f32_len = 3512; + +unsigned char cpy_f16_f16_data[] = { +0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, +0x98,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, +0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x51,0x11,0x00,0x00, +0x0b,0x00,0x06,0x00,0x01,0x00,0x00,0x00,0x47,0x4c,0x53,0x4c, +0x2e,0x73,0x74,0x64,0x2e,0x34,0x35,0x30,0x00,0x00,0x00,0x00, +0x0e,0x00,0x03,0x00,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x0f,0x00,0x09,0x00,0x05,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x6d,0x61,0x69,0x6e,0x00,0x00,0x00,0x00,0x14,0x00,0x00,0x00, +0xc7,0x00,0x00,0x00,0xd8,0x00,0x00,0x00,0xe4,0x00,0x00,0x00, +0x10,0x00,0x06,0x00,0x04,0x00,0x00,0x00,0x11,0x00,0x00,0x00, +0x00,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00, 0x48,0x00,0x05,0x00,0x12,0x00,0x00,0x00,0x00,0x00,0x00,0x00, 0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00, 0x12,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00, 0x04,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x12,0x00,0x00,0x00, 0x02,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x08,0x00,0x00,0x00, 0x48,0x00,0x05,0x00,0x12,0x00,0x00,0x00,0x03,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x47,0x00,0x03,0x00, -0x12,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x1f,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x48,0x00,0x04,0x00,0x20,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x19,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x20,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x12,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x10,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x12,0x00,0x00,0x00, +0x05,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x14,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x12,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x12,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x1c,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x12,0x00,0x00,0x00, +0x08,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x12,0x00,0x00,0x00,0x09,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x24,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x12,0x00,0x00,0x00,0x0a,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x28,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x12,0x00,0x00,0x00, +0x0b,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x2c,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x12,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x30,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x12,0x00,0x00,0x00,0x0d,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x34,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x12,0x00,0x00,0x00, +0x0e,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x38,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x12,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x3c,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x12,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x40,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x12,0x00,0x00,0x00, +0x11,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x44,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x12,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x48,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x12,0x00,0x00,0x00,0x13,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x4c,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x12,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xc7,0x00,0x00,0x00, +0x0b,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0xd5,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x48,0x00,0x04,0x00,0xd6,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x19,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0xd6,0x00,0x00,0x00, 0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x03,0x00,0x20,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x22,0x00,0x00,0x00,0x22,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x22,0x00,0x00,0x00, -0x21,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x24,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x48,0x00,0x04,0x00,0x25,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x25,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x03,0x00,0x25,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x27,0x00,0x00,0x00,0x22,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x27,0x00,0x00,0x00, -0x21,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x2c,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x48,0x00,0x04,0x00,0x2d,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x2d,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x03,0x00,0x2d,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x2f,0x00,0x00,0x00,0x22,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x2f,0x00,0x00,0x00, +0x47,0x00,0x03,0x00,0xd6,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0xd8,0x00,0x00,0x00,0x22,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xd8,0x00,0x00,0x00, 0x21,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x3b,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x19,0x00,0x00,0x00, +0xe1,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x48,0x00,0x04,0x00,0xe2,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0xe2,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x03,0x00,0xe2,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0xe4,0x00,0x00,0x00,0x22,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xe4,0x00,0x00,0x00, +0x21,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0xef,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x19,0x00,0x00,0x00, +0x13,0x00,0x02,0x00,0x02,0x00,0x00,0x00,0x21,0x00,0x03,0x00, +0x03,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x15,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x16,0x00,0x03,0x00,0x11,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0x1e,0x00,0x16,0x00,0x12,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x11,0x00,0x00,0x00, +0x11,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x13,0x00,0x00,0x00, +0x09,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x09,0x00,0x00,0x00, +0x15,0x00,0x04,0x00,0x15,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x15,0x00,0x00,0x00, +0x16,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x17,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x15,0x00,0x00,0x00,0x1a,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x15,0x00,0x00,0x00, +0x1e,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x15,0x00,0x00,0x00,0x55,0x00,0x00,0x00,0x08,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x15,0x00,0x00,0x00,0x5a,0x00,0x00,0x00, +0x07,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x15,0x00,0x00,0x00, +0x60,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x15,0x00,0x00,0x00,0x66,0x00,0x00,0x00,0x05,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x15,0x00,0x00,0x00,0x6f,0x00,0x00,0x00, +0x0b,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x15,0x00,0x00,0x00, +0x72,0x00,0x00,0x00,0x0a,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x15,0x00,0x00,0x00,0x76,0x00,0x00,0x00,0x09,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x15,0x00,0x00,0x00,0xad,0x00,0x00,0x00, +0x10,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x15,0x00,0x00,0x00, +0xb2,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x15,0x00,0x00,0x00,0xb8,0x00,0x00,0x00,0x0e,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x15,0x00,0x00,0x00,0xbe,0x00,0x00,0x00, +0x0d,0x00,0x00,0x00,0x17,0x00,0x04,0x00,0xc5,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0xc6,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0xc5,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0xc6,0x00,0x00,0x00,0xc7,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xc8,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0xc9,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x15,0x00,0x00,0x00,0xcc,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x14,0x00,0x02,0x00,0xcf,0x00,0x00,0x00, +0x16,0x00,0x03,0x00,0xd4,0x00,0x00,0x00,0x10,0x00,0x00,0x00, +0x1d,0x00,0x03,0x00,0xd5,0x00,0x00,0x00,0xd4,0x00,0x00,0x00, +0x1e,0x00,0x03,0x00,0xd6,0x00,0x00,0x00,0xd5,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0xd7,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0xd6,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0xd7,0x00,0x00,0x00, +0xd8,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x15,0x00,0x00,0x00,0xd9,0x00,0x00,0x00,0x11,0x00,0x00,0x00, +0x1d,0x00,0x03,0x00,0xe1,0x00,0x00,0x00,0xd4,0x00,0x00,0x00, +0x1e,0x00,0x03,0x00,0xe2,0x00,0x00,0x00,0xe1,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0xe3,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0xe2,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0xe3,0x00,0x00,0x00, +0xe4,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0xe9,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0xd4,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xed,0x00,0x00,0x00, +0x00,0x02,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xee,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x2c,0x00,0x06,0x00, +0xc5,0x00,0x00,0x00,0xef,0x00,0x00,0x00,0xed,0x00,0x00,0x00, +0xee,0x00,0x00,0x00,0xee,0x00,0x00,0x00,0x36,0x00,0x05,0x00, +0x02,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x03,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x05,0x00,0x00,0x00, +0xf7,0x00,0x03,0x00,0xf0,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0xfb,0x00,0x03,0x00,0xc8,0x00,0x00,0x00,0xf1,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xf1,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0xc9,0x00,0x00,0x00,0xca,0x00,0x00,0x00,0xc7,0x00,0x00,0x00, +0xc8,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xcb,0x00,0x00,0x00,0xca,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x17,0x00,0x00,0x00,0xcd,0x00,0x00,0x00,0x14,0x00,0x00,0x00, +0xcc,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xce,0x00,0x00,0x00,0xcd,0x00,0x00,0x00,0xae,0x00,0x05,0x00, +0xcf,0x00,0x00,0x00,0xd0,0x00,0x00,0x00,0xcb,0x00,0x00,0x00, +0xce,0x00,0x00,0x00,0xf7,0x00,0x03,0x00,0xd2,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xd0,0x00,0x00,0x00, +0xd1,0x00,0x00,0x00,0xd2,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xd1,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xf0,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xd2,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x17,0x00,0x00,0x00,0xda,0x00,0x00,0x00,0x14,0x00,0x00,0x00, +0xd9,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xdb,0x00,0x00,0x00,0xda,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x17,0x00,0x00,0x00,0xff,0x00,0x00,0x00,0x14,0x00,0x00,0x00, +0x6f,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x00,0x01,0x00,0x00,0xff,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x17,0x00,0x00,0x00,0x01,0x01,0x00,0x00,0x14,0x00,0x00,0x00, +0x72,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x02,0x01,0x00,0x00,0x01,0x01,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x03,0x01,0x00,0x00,0x00,0x01,0x00,0x00, +0x02,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00, +0x04,0x01,0x00,0x00,0x14,0x00,0x00,0x00,0x76,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x05,0x01,0x00,0x00, +0x04,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x06,0x01,0x00,0x00,0x03,0x01,0x00,0x00,0x05,0x01,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x07,0x01,0x00,0x00, +0xcb,0x00,0x00,0x00,0x06,0x01,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x0b,0x01,0x00,0x00,0x07,0x01,0x00,0x00, +0x00,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x0e,0x01,0x00,0x00,0x0b,0x01,0x00,0x00,0x02,0x01,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x11,0x01,0x00,0x00, +0x0e,0x01,0x00,0x00,0x05,0x01,0x00,0x00,0x82,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x14,0x01,0x00,0x00,0xcb,0x00,0x00,0x00, +0x11,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x19,0x01,0x00,0x00,0x02,0x01,0x00,0x00,0x05,0x01,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x1a,0x01,0x00,0x00, +0x14,0x01,0x00,0x00,0x19,0x01,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x1e,0x01,0x00,0x00,0x1a,0x01,0x00,0x00, +0x02,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x21,0x01,0x00,0x00,0x1e,0x01,0x00,0x00,0x05,0x01,0x00,0x00, +0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x26,0x01,0x00,0x00, +0x14,0x01,0x00,0x00,0x21,0x01,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x29,0x01,0x00,0x00,0x26,0x01,0x00,0x00, +0x05,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x32,0x01,0x00,0x00,0x29,0x01,0x00,0x00,0x05,0x01,0x00,0x00, +0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x33,0x01,0x00,0x00, +0x26,0x01,0x00,0x00,0x32,0x01,0x00,0x00,0x41,0x00,0x05,0x00, +0x17,0x00,0x00,0x00,0x35,0x01,0x00,0x00,0x14,0x00,0x00,0x00, +0xad,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x36,0x01,0x00,0x00,0x35,0x01,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x37,0x01,0x00,0x00,0x07,0x01,0x00,0x00, +0x36,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00, +0x39,0x01,0x00,0x00,0x14,0x00,0x00,0x00,0xb2,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x3a,0x01,0x00,0x00, +0x39,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x3b,0x01,0x00,0x00,0x1a,0x01,0x00,0x00,0x3a,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3c,0x01,0x00,0x00, +0x37,0x01,0x00,0x00,0x3b,0x01,0x00,0x00,0x41,0x00,0x05,0x00, +0x17,0x00,0x00,0x00,0x3e,0x01,0x00,0x00,0x14,0x00,0x00,0x00, +0xb8,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x3f,0x01,0x00,0x00,0x3e,0x01,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x40,0x01,0x00,0x00,0x29,0x01,0x00,0x00, +0x3f,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x41,0x01,0x00,0x00,0x3c,0x01,0x00,0x00,0x40,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00,0x43,0x01,0x00,0x00, +0x14,0x00,0x00,0x00,0xbe,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x44,0x01,0x00,0x00,0x43,0x01,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x45,0x01,0x00,0x00, +0x33,0x01,0x00,0x00,0x44,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x46,0x01,0x00,0x00,0x41,0x01,0x00,0x00, +0x45,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xe0,0x00,0x00,0x00,0xdb,0x00,0x00,0x00,0x46,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00,0x50,0x01,0x00,0x00, +0x14,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x51,0x01,0x00,0x00,0x50,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00,0x52,0x01,0x00,0x00, +0x14,0x00,0x00,0x00,0x1a,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x53,0x01,0x00,0x00,0x52,0x01,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x54,0x01,0x00,0x00, +0x51,0x01,0x00,0x00,0x53,0x01,0x00,0x00,0x41,0x00,0x05,0x00, +0x17,0x00,0x00,0x00,0x55,0x01,0x00,0x00,0x14,0x00,0x00,0x00, +0x1e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x56,0x01,0x00,0x00,0x55,0x01,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x57,0x01,0x00,0x00,0x54,0x01,0x00,0x00, +0x56,0x01,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x58,0x01,0x00,0x00,0xcb,0x00,0x00,0x00,0x57,0x01,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x5c,0x01,0x00,0x00, +0x58,0x01,0x00,0x00,0x51,0x01,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x5f,0x01,0x00,0x00,0x5c,0x01,0x00,0x00, +0x53,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x62,0x01,0x00,0x00,0x5f,0x01,0x00,0x00,0x56,0x01,0x00,0x00, +0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x65,0x01,0x00,0x00, +0xcb,0x00,0x00,0x00,0x62,0x01,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x6a,0x01,0x00,0x00,0x53,0x01,0x00,0x00, +0x56,0x01,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x6b,0x01,0x00,0x00,0x65,0x01,0x00,0x00,0x6a,0x01,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x6f,0x01,0x00,0x00, +0x6b,0x01,0x00,0x00,0x53,0x01,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x72,0x01,0x00,0x00,0x6f,0x01,0x00,0x00, +0x56,0x01,0x00,0x00,0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x77,0x01,0x00,0x00,0x65,0x01,0x00,0x00,0x72,0x01,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x7a,0x01,0x00,0x00, +0x77,0x01,0x00,0x00,0x56,0x01,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x83,0x01,0x00,0x00,0x7a,0x01,0x00,0x00, +0x56,0x01,0x00,0x00,0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x84,0x01,0x00,0x00,0x77,0x01,0x00,0x00,0x83,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00,0x86,0x01,0x00,0x00, +0x14,0x00,0x00,0x00,0x55,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x87,0x01,0x00,0x00,0x86,0x01,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x88,0x01,0x00,0x00, +0x58,0x01,0x00,0x00,0x87,0x01,0x00,0x00,0x41,0x00,0x05,0x00, +0x17,0x00,0x00,0x00,0x8a,0x01,0x00,0x00,0x14,0x00,0x00,0x00, +0x5a,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x8b,0x01,0x00,0x00,0x8a,0x01,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x8c,0x01,0x00,0x00,0x6b,0x01,0x00,0x00, +0x8b,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x8d,0x01,0x00,0x00,0x88,0x01,0x00,0x00,0x8c,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00,0x8f,0x01,0x00,0x00, +0x14,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x90,0x01,0x00,0x00,0x8f,0x01,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x91,0x01,0x00,0x00, +0x7a,0x01,0x00,0x00,0x90,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x92,0x01,0x00,0x00,0x8d,0x01,0x00,0x00, +0x91,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00, +0x94,0x01,0x00,0x00,0x14,0x00,0x00,0x00,0x66,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x95,0x01,0x00,0x00, +0x94,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x96,0x01,0x00,0x00,0x84,0x01,0x00,0x00,0x95,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x97,0x01,0x00,0x00, +0x92,0x01,0x00,0x00,0x96,0x01,0x00,0x00,0x41,0x00,0x06,0x00, +0xe9,0x00,0x00,0x00,0xea,0x00,0x00,0x00,0xe4,0x00,0x00,0x00, +0xcc,0x00,0x00,0x00,0x97,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, +0xd4,0x00,0x00,0x00,0xeb,0x00,0x00,0x00,0xea,0x00,0x00,0x00, +0x41,0x00,0x06,0x00,0xe9,0x00,0x00,0x00,0xec,0x00,0x00,0x00, +0xd8,0x00,0x00,0x00,0xcc,0x00,0x00,0x00,0xe0,0x00,0x00,0x00, +0x3e,0x00,0x03,0x00,0xec,0x00,0x00,0x00,0xeb,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xf0,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xf0,0x00,0x00,0x00,0xfd,0x00,0x01,0x00,0x38,0x00,0x01,0x00, + +}; +const uint64_t cpy_f16_f16_len = 3252; + +unsigned char cpy_f32_f16_data[] = { +0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, +0x9a,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, +0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x51,0x11,0x00,0x00, +0x0b,0x00,0x06,0x00,0x01,0x00,0x00,0x00,0x47,0x4c,0x53,0x4c, +0x2e,0x73,0x74,0x64,0x2e,0x34,0x35,0x30,0x00,0x00,0x00,0x00, +0x0e,0x00,0x03,0x00,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x0f,0x00,0x09,0x00,0x05,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x6d,0x61,0x69,0x6e,0x00,0x00,0x00,0x00,0x14,0x00,0x00,0x00, +0xc7,0x00,0x00,0x00,0xd8,0x00,0x00,0x00,0xe4,0x00,0x00,0x00, +0x10,0x00,0x06,0x00,0x04,0x00,0x00,0x00,0x11,0x00,0x00,0x00, +0x00,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x12,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x12,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x12,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x08,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x12,0x00,0x00,0x00,0x03,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x12,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x10,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x12,0x00,0x00,0x00, +0x05,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x14,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x12,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x12,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x1c,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x12,0x00,0x00,0x00, +0x08,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x12,0x00,0x00,0x00,0x09,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x24,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x12,0x00,0x00,0x00,0x0a,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x28,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x12,0x00,0x00,0x00, +0x0b,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x2c,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x12,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x30,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x12,0x00,0x00,0x00,0x0d,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x34,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x12,0x00,0x00,0x00, +0x0e,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x38,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x12,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x3c,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x12,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x40,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x12,0x00,0x00,0x00, +0x11,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x44,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x12,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x48,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x12,0x00,0x00,0x00,0x13,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x4c,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x12,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xc7,0x00,0x00,0x00, +0x0b,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0xd5,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x48,0x00,0x04,0x00,0xd6,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x19,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0xd6,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x03,0x00,0xd6,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0xd8,0x00,0x00,0x00,0x22,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xd8,0x00,0x00,0x00, +0x21,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0xe1,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x48,0x00,0x04,0x00,0xe2,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0xe2,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x03,0x00,0xe2,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0xe4,0x00,0x00,0x00,0x22,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xe4,0x00,0x00,0x00, +0x21,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0xf1,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x19,0x00,0x00,0x00, +0x13,0x00,0x02,0x00,0x02,0x00,0x00,0x00,0x21,0x00,0x03,0x00, +0x03,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x15,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x16,0x00,0x03,0x00,0x11,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0x1e,0x00,0x16,0x00,0x12,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x11,0x00,0x00,0x00, +0x11,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x13,0x00,0x00,0x00, +0x09,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x09,0x00,0x00,0x00, +0x15,0x00,0x04,0x00,0x15,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x15,0x00,0x00,0x00, +0x16,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x17,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x15,0x00,0x00,0x00,0x1a,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x15,0x00,0x00,0x00, +0x1e,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x15,0x00,0x00,0x00,0x55,0x00,0x00,0x00,0x08,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x15,0x00,0x00,0x00,0x5a,0x00,0x00,0x00, +0x07,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x15,0x00,0x00,0x00, +0x60,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x15,0x00,0x00,0x00,0x66,0x00,0x00,0x00,0x05,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x15,0x00,0x00,0x00,0x6f,0x00,0x00,0x00, +0x0b,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x15,0x00,0x00,0x00, +0x72,0x00,0x00,0x00,0x0a,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x15,0x00,0x00,0x00,0x76,0x00,0x00,0x00,0x09,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x15,0x00,0x00,0x00,0xad,0x00,0x00,0x00, +0x10,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x15,0x00,0x00,0x00, +0xb2,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x15,0x00,0x00,0x00,0xb8,0x00,0x00,0x00,0x0e,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x15,0x00,0x00,0x00,0xbe,0x00,0x00,0x00, +0x0d,0x00,0x00,0x00,0x17,0x00,0x04,0x00,0xc5,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0xc6,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0xc5,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0xc6,0x00,0x00,0x00,0xc7,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xc8,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0xc9,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x15,0x00,0x00,0x00,0xcc,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x14,0x00,0x02,0x00,0xcf,0x00,0x00,0x00, +0x16,0x00,0x03,0x00,0xd4,0x00,0x00,0x00,0x10,0x00,0x00,0x00, +0x1d,0x00,0x03,0x00,0xd5,0x00,0x00,0x00,0xd4,0x00,0x00,0x00, +0x1e,0x00,0x03,0x00,0xd6,0x00,0x00,0x00,0xd5,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0xd7,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0xd6,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0xd7,0x00,0x00,0x00, +0xd8,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x15,0x00,0x00,0x00,0xd9,0x00,0x00,0x00,0x11,0x00,0x00,0x00, +0x1d,0x00,0x03,0x00,0xe1,0x00,0x00,0x00,0x11,0x00,0x00,0x00, +0x1e,0x00,0x03,0x00,0xe2,0x00,0x00,0x00,0xe1,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0xe3,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0xe2,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0xe3,0x00,0x00,0x00, +0xe4,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0xe9,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x11,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0xed,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0xd4,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xef,0x00,0x00,0x00,0x00,0x02,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xf0,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x2c,0x00,0x06,0x00,0xc5,0x00,0x00,0x00,0xf1,0x00,0x00,0x00, +0xef,0x00,0x00,0x00,0xf0,0x00,0x00,0x00,0xf0,0x00,0x00,0x00, +0x36,0x00,0x05,0x00,0x02,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0x05,0x00,0x00,0x00,0xf7,0x00,0x03,0x00,0xf2,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0xfb,0x00,0x03,0x00,0xc8,0x00,0x00,0x00, +0xf3,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xf3,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0xc9,0x00,0x00,0x00,0xca,0x00,0x00,0x00, +0xc7,0x00,0x00,0x00,0xc8,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xcb,0x00,0x00,0x00,0xca,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00,0xcd,0x00,0x00,0x00, +0x14,0x00,0x00,0x00,0xcc,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xce,0x00,0x00,0x00,0xcd,0x00,0x00,0x00, +0xae,0x00,0x05,0x00,0xcf,0x00,0x00,0x00,0xd0,0x00,0x00,0x00, +0xcb,0x00,0x00,0x00,0xce,0x00,0x00,0x00,0xf7,0x00,0x03,0x00, +0xd2,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0xd0,0x00,0x00,0x00,0xd1,0x00,0x00,0x00,0xd2,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xd1,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xf2,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xd2,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00,0xda,0x00,0x00,0x00, +0x14,0x00,0x00,0x00,0xd9,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xdb,0x00,0x00,0x00,0xda,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00,0x01,0x01,0x00,0x00, +0x14,0x00,0x00,0x00,0x6f,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x02,0x01,0x00,0x00,0x01,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00,0x03,0x01,0x00,0x00, +0x14,0x00,0x00,0x00,0x72,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x04,0x01,0x00,0x00,0x03,0x01,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x05,0x01,0x00,0x00, +0x02,0x01,0x00,0x00,0x04,0x01,0x00,0x00,0x41,0x00,0x05,0x00, +0x17,0x00,0x00,0x00,0x06,0x01,0x00,0x00,0x14,0x00,0x00,0x00, +0x76,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x07,0x01,0x00,0x00,0x06,0x01,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x08,0x01,0x00,0x00,0x05,0x01,0x00,0x00, +0x07,0x01,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x09,0x01,0x00,0x00,0xcb,0x00,0x00,0x00,0x08,0x01,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x0d,0x01,0x00,0x00, +0x09,0x01,0x00,0x00,0x02,0x01,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x10,0x01,0x00,0x00,0x0d,0x01,0x00,0x00, +0x04,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x13,0x01,0x00,0x00,0x10,0x01,0x00,0x00,0x07,0x01,0x00,0x00, +0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x16,0x01,0x00,0x00, +0xcb,0x00,0x00,0x00,0x13,0x01,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x1b,0x01,0x00,0x00,0x04,0x01,0x00,0x00, +0x07,0x01,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x1c,0x01,0x00,0x00,0x16,0x01,0x00,0x00,0x1b,0x01,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x20,0x01,0x00,0x00, +0x1c,0x01,0x00,0x00,0x04,0x01,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x23,0x01,0x00,0x00,0x20,0x01,0x00,0x00, +0x07,0x01,0x00,0x00,0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x28,0x01,0x00,0x00,0x16,0x01,0x00,0x00,0x23,0x01,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x2b,0x01,0x00,0x00, +0x28,0x01,0x00,0x00,0x07,0x01,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x34,0x01,0x00,0x00,0x2b,0x01,0x00,0x00, +0x07,0x01,0x00,0x00,0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x35,0x01,0x00,0x00,0x28,0x01,0x00,0x00,0x34,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00,0x37,0x01,0x00,0x00, +0x14,0x00,0x00,0x00,0xad,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x38,0x01,0x00,0x00,0x37,0x01,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x39,0x01,0x00,0x00, +0x09,0x01,0x00,0x00,0x38,0x01,0x00,0x00,0x41,0x00,0x05,0x00, +0x17,0x00,0x00,0x00,0x3b,0x01,0x00,0x00,0x14,0x00,0x00,0x00, +0xb2,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x3c,0x01,0x00,0x00,0x3b,0x01,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x3d,0x01,0x00,0x00,0x1c,0x01,0x00,0x00, +0x3c,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x3e,0x01,0x00,0x00,0x39,0x01,0x00,0x00,0x3d,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00,0x40,0x01,0x00,0x00, +0x14,0x00,0x00,0x00,0xb8,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x41,0x01,0x00,0x00,0x40,0x01,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x42,0x01,0x00,0x00, +0x2b,0x01,0x00,0x00,0x41,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x43,0x01,0x00,0x00,0x3e,0x01,0x00,0x00, +0x42,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00, +0x45,0x01,0x00,0x00,0x14,0x00,0x00,0x00,0xbe,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x46,0x01,0x00,0x00, +0x45,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x47,0x01,0x00,0x00,0x35,0x01,0x00,0x00,0x46,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x48,0x01,0x00,0x00, +0x43,0x01,0x00,0x00,0x47,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xe0,0x00,0x00,0x00,0xdb,0x00,0x00,0x00, +0x48,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00, +0x52,0x01,0x00,0x00,0x14,0x00,0x00,0x00,0x16,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x53,0x01,0x00,0x00, +0x52,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00, +0x54,0x01,0x00,0x00,0x14,0x00,0x00,0x00,0x1a,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x55,0x01,0x00,0x00, +0x54,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x56,0x01,0x00,0x00,0x53,0x01,0x00,0x00,0x55,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00,0x57,0x01,0x00,0x00, +0x14,0x00,0x00,0x00,0x1e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x58,0x01,0x00,0x00,0x57,0x01,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x59,0x01,0x00,0x00, +0x56,0x01,0x00,0x00,0x58,0x01,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x5a,0x01,0x00,0x00,0xcb,0x00,0x00,0x00, +0x59,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x5e,0x01,0x00,0x00,0x5a,0x01,0x00,0x00,0x53,0x01,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x61,0x01,0x00,0x00, +0x5e,0x01,0x00,0x00,0x55,0x01,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x64,0x01,0x00,0x00,0x61,0x01,0x00,0x00, +0x58,0x01,0x00,0x00,0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x67,0x01,0x00,0x00,0xcb,0x00,0x00,0x00,0x64,0x01,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x6c,0x01,0x00,0x00, +0x55,0x01,0x00,0x00,0x58,0x01,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x6d,0x01,0x00,0x00,0x67,0x01,0x00,0x00, +0x6c,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x71,0x01,0x00,0x00,0x6d,0x01,0x00,0x00,0x55,0x01,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x74,0x01,0x00,0x00, +0x71,0x01,0x00,0x00,0x58,0x01,0x00,0x00,0x82,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x79,0x01,0x00,0x00,0x67,0x01,0x00,0x00, +0x74,0x01,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x7c,0x01,0x00,0x00,0x79,0x01,0x00,0x00,0x58,0x01,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x85,0x01,0x00,0x00, +0x7c,0x01,0x00,0x00,0x58,0x01,0x00,0x00,0x82,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x86,0x01,0x00,0x00,0x79,0x01,0x00,0x00, +0x85,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00, +0x88,0x01,0x00,0x00,0x14,0x00,0x00,0x00,0x55,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x89,0x01,0x00,0x00, +0x88,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x8a,0x01,0x00,0x00,0x5a,0x01,0x00,0x00,0x89,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00,0x8c,0x01,0x00,0x00, +0x14,0x00,0x00,0x00,0x5a,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x8d,0x01,0x00,0x00,0x8c,0x01,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8e,0x01,0x00,0x00, +0x6d,0x01,0x00,0x00,0x8d,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x8f,0x01,0x00,0x00,0x8a,0x01,0x00,0x00, +0x8e,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00, +0x91,0x01,0x00,0x00,0x14,0x00,0x00,0x00,0x60,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x92,0x01,0x00,0x00, +0x91,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x93,0x01,0x00,0x00,0x7c,0x01,0x00,0x00,0x92,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x94,0x01,0x00,0x00, +0x8f,0x01,0x00,0x00,0x93,0x01,0x00,0x00,0x41,0x00,0x05,0x00, +0x17,0x00,0x00,0x00,0x96,0x01,0x00,0x00,0x14,0x00,0x00,0x00, +0x66,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x97,0x01,0x00,0x00,0x96,0x01,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x98,0x01,0x00,0x00,0x86,0x01,0x00,0x00, +0x97,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x99,0x01,0x00,0x00,0x94,0x01,0x00,0x00,0x98,0x01,0x00,0x00, +0x41,0x00,0x06,0x00,0xe9,0x00,0x00,0x00,0xea,0x00,0x00,0x00, +0xe4,0x00,0x00,0x00,0xcc,0x00,0x00,0x00,0x99,0x01,0x00,0x00, +0x3d,0x00,0x04,0x00,0x11,0x00,0x00,0x00,0xeb,0x00,0x00,0x00, +0xea,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0xd4,0x00,0x00,0x00, +0xec,0x00,0x00,0x00,0xeb,0x00,0x00,0x00,0x41,0x00,0x06,0x00, +0xed,0x00,0x00,0x00,0xee,0x00,0x00,0x00,0xd8,0x00,0x00,0x00, +0xcc,0x00,0x00,0x00,0xe0,0x00,0x00,0x00,0x3e,0x00,0x03,0x00, +0xee,0x00,0x00,0x00,0xec,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xf2,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xf2,0x00,0x00,0x00, +0xfd,0x00,0x01,0x00,0x38,0x00,0x01,0x00, +}; +const uint64_t cpy_f32_f16_len = 3284; + +unsigned char cpy_f32_f32_data[] = { +0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, +0x97,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, +0x01,0x00,0x00,0x00,0x0b,0x00,0x06,0x00,0x01,0x00,0x00,0x00, +0x47,0x4c,0x53,0x4c,0x2e,0x73,0x74,0x64,0x2e,0x34,0x35,0x30, +0x00,0x00,0x00,0x00,0x0e,0x00,0x03,0x00,0x00,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x0f,0x00,0x09,0x00,0x05,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x6d,0x61,0x69,0x6e,0x00,0x00,0x00,0x00, +0x14,0x00,0x00,0x00,0xc7,0x00,0x00,0x00,0xd7,0x00,0x00,0x00, +0xe3,0x00,0x00,0x00,0x10,0x00,0x06,0x00,0x04,0x00,0x00,0x00, +0x11,0x00,0x00,0x00,0x00,0x02,0x00,0x00,0x01,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x12,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x12,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x12,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x08,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x12,0x00,0x00,0x00, +0x03,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x12,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x12,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x14,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x12,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x18,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x12,0x00,0x00,0x00,0x07,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x12,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x12,0x00,0x00,0x00, +0x09,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x24,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x12,0x00,0x00,0x00,0x0a,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x12,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x2c,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x12,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x30,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x12,0x00,0x00,0x00,0x0d,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x12,0x00,0x00,0x00,0x0e,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x38,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x12,0x00,0x00,0x00, +0x0f,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x3c,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x12,0x00,0x00,0x00,0x10,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x12,0x00,0x00,0x00,0x11,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x44,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x12,0x00,0x00,0x00, +0x12,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x48,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x12,0x00,0x00,0x00,0x13,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x4c,0x00,0x00,0x00,0x47,0x00,0x03,0x00, +0x12,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0xc7,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x1c,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0xd4,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0xd5,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0xd5,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0xd5,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xd7,0x00,0x00,0x00, +0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0xd7,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0xe0,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0xe1,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0xe1,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0xe1,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xe3,0x00,0x00,0x00, +0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0xe3,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0xee,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, +0x19,0x00,0x00,0x00,0x13,0x00,0x02,0x00,0x02,0x00,0x00,0x00, +0x21,0x00,0x03,0x00,0x03,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x15,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x16,0x00,0x03,0x00,0x11,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0x1e,0x00,0x16,0x00,0x12,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x11,0x00,0x00,0x00,0x11,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x14,0x00,0x00,0x00, +0x09,0x00,0x00,0x00,0x15,0x00,0x04,0x00,0x15,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x15,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x03,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x17,0x00,0x00,0x00,0x09,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x15,0x00,0x00,0x00, +0x1a,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x15,0x00,0x00,0x00,0x1e,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x15,0x00,0x00,0x00,0x55,0x00,0x00,0x00, +0x08,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x15,0x00,0x00,0x00, +0x5a,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x15,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x15,0x00,0x00,0x00,0x66,0x00,0x00,0x00, +0x05,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x15,0x00,0x00,0x00, +0x6f,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x15,0x00,0x00,0x00,0x72,0x00,0x00,0x00,0x0a,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x15,0x00,0x00,0x00,0x76,0x00,0x00,0x00, +0x09,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x15,0x00,0x00,0x00, +0xad,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x15,0x00,0x00,0x00,0xb2,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x15,0x00,0x00,0x00,0xb8,0x00,0x00,0x00, +0x0e,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x15,0x00,0x00,0x00, +0xbe,0x00,0x00,0x00,0x0d,0x00,0x00,0x00,0x17,0x00,0x04,0x00, +0xc5,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x03,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0xc6,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0xc5,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0xc6,0x00,0x00,0x00, +0xc7,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xc8,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0xc9,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x15,0x00,0x00,0x00, +0xcc,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x14,0x00,0x02,0x00, +0xcf,0x00,0x00,0x00,0x1d,0x00,0x03,0x00,0xd4,0x00,0x00,0x00, +0x11,0x00,0x00,0x00,0x1e,0x00,0x03,0x00,0xd5,0x00,0x00,0x00, +0xd4,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0xd6,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0xd5,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0xd6,0x00,0x00,0x00,0xd7,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x15,0x00,0x00,0x00,0xd8,0x00,0x00,0x00, +0x11,0x00,0x00,0x00,0x1d,0x00,0x03,0x00,0xe0,0x00,0x00,0x00, +0x11,0x00,0x00,0x00,0x1e,0x00,0x03,0x00,0xe1,0x00,0x00,0x00, +0xe0,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0xe2,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0xe1,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0xe2,0x00,0x00,0x00,0xe3,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0xe8,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x11,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xec,0x00,0x00,0x00,0x00,0x02,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xed,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x2c,0x00,0x06,0x00,0xc5,0x00,0x00,0x00,0xee,0x00,0x00,0x00, +0xec,0x00,0x00,0x00,0xed,0x00,0x00,0x00,0xed,0x00,0x00,0x00, +0x36,0x00,0x05,0x00,0x02,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0x05,0x00,0x00,0x00,0xf7,0x00,0x03,0x00,0xef,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0xfb,0x00,0x03,0x00,0xc8,0x00,0x00,0x00, +0xf0,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xf0,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0xc9,0x00,0x00,0x00,0xca,0x00,0x00,0x00, +0xc7,0x00,0x00,0x00,0xc8,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xcb,0x00,0x00,0x00,0xca,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00,0xcd,0x00,0x00,0x00, +0x14,0x00,0x00,0x00,0xcc,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xce,0x00,0x00,0x00,0xcd,0x00,0x00,0x00, +0xae,0x00,0x05,0x00,0xcf,0x00,0x00,0x00,0xd0,0x00,0x00,0x00, +0xcb,0x00,0x00,0x00,0xce,0x00,0x00,0x00,0xf7,0x00,0x03,0x00, +0xd2,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0xd0,0x00,0x00,0x00,0xd1,0x00,0x00,0x00,0xd2,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xd1,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xef,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xd2,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00,0xd9,0x00,0x00,0x00, +0x14,0x00,0x00,0x00,0xd8,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xda,0x00,0x00,0x00,0xd9,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00,0xfe,0x00,0x00,0x00, +0x14,0x00,0x00,0x00,0x6f,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xff,0x00,0x00,0x00,0xfe,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00,0x00,0x01,0x00,0x00, +0x14,0x00,0x00,0x00,0x72,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x01,0x01,0x00,0x00,0x00,0x01,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x02,0x01,0x00,0x00, +0xff,0x00,0x00,0x00,0x01,0x01,0x00,0x00,0x41,0x00,0x05,0x00, +0x17,0x00,0x00,0x00,0x03,0x01,0x00,0x00,0x14,0x00,0x00,0x00, +0x76,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x04,0x01,0x00,0x00,0x03,0x01,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x05,0x01,0x00,0x00,0x02,0x01,0x00,0x00, +0x04,0x01,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x06,0x01,0x00,0x00,0xcb,0x00,0x00,0x00,0x05,0x01,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x0a,0x01,0x00,0x00, +0x06,0x01,0x00,0x00,0xff,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x0d,0x01,0x00,0x00,0x0a,0x01,0x00,0x00, +0x01,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x10,0x01,0x00,0x00,0x0d,0x01,0x00,0x00,0x04,0x01,0x00,0x00, +0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x13,0x01,0x00,0x00, +0xcb,0x00,0x00,0x00,0x10,0x01,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x18,0x01,0x00,0x00,0x01,0x01,0x00,0x00, +0x04,0x01,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x19,0x01,0x00,0x00,0x13,0x01,0x00,0x00,0x18,0x01,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x1d,0x01,0x00,0x00, +0x19,0x01,0x00,0x00,0x01,0x01,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x20,0x01,0x00,0x00,0x1d,0x01,0x00,0x00, +0x04,0x01,0x00,0x00,0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x25,0x01,0x00,0x00,0x13,0x01,0x00,0x00,0x20,0x01,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x28,0x01,0x00,0x00, +0x25,0x01,0x00,0x00,0x04,0x01,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x31,0x01,0x00,0x00,0x28,0x01,0x00,0x00, +0x04,0x01,0x00,0x00,0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x32,0x01,0x00,0x00,0x25,0x01,0x00,0x00,0x31,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00,0x34,0x01,0x00,0x00, +0x14,0x00,0x00,0x00,0xad,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x35,0x01,0x00,0x00,0x34,0x01,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x36,0x01,0x00,0x00, +0x06,0x01,0x00,0x00,0x35,0x01,0x00,0x00,0x41,0x00,0x05,0x00, +0x17,0x00,0x00,0x00,0x38,0x01,0x00,0x00,0x14,0x00,0x00,0x00, +0xb2,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x39,0x01,0x00,0x00,0x38,0x01,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x3a,0x01,0x00,0x00,0x19,0x01,0x00,0x00, +0x39,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x3b,0x01,0x00,0x00,0x36,0x01,0x00,0x00,0x3a,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00,0x3d,0x01,0x00,0x00, +0x14,0x00,0x00,0x00,0xb8,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x3e,0x01,0x00,0x00,0x3d,0x01,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3f,0x01,0x00,0x00, +0x28,0x01,0x00,0x00,0x3e,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x40,0x01,0x00,0x00,0x3b,0x01,0x00,0x00, +0x3f,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00, +0x42,0x01,0x00,0x00,0x14,0x00,0x00,0x00,0xbe,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x43,0x01,0x00,0x00, +0x42,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x44,0x01,0x00,0x00,0x32,0x01,0x00,0x00,0x43,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x45,0x01,0x00,0x00, +0x40,0x01,0x00,0x00,0x44,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xdf,0x00,0x00,0x00,0xda,0x00,0x00,0x00, +0x45,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00, +0x4f,0x01,0x00,0x00,0x14,0x00,0x00,0x00,0x16,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x50,0x01,0x00,0x00, +0x4f,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00, +0x51,0x01,0x00,0x00,0x14,0x00,0x00,0x00,0x1a,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x52,0x01,0x00,0x00, +0x51,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x53,0x01,0x00,0x00,0x50,0x01,0x00,0x00,0x52,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00,0x54,0x01,0x00,0x00, +0x14,0x00,0x00,0x00,0x1e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x55,0x01,0x00,0x00,0x54,0x01,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x56,0x01,0x00,0x00, +0x53,0x01,0x00,0x00,0x55,0x01,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x57,0x01,0x00,0x00,0xcb,0x00,0x00,0x00, +0x56,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x5b,0x01,0x00,0x00,0x57,0x01,0x00,0x00,0x50,0x01,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x5e,0x01,0x00,0x00, +0x5b,0x01,0x00,0x00,0x52,0x01,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x61,0x01,0x00,0x00,0x5e,0x01,0x00,0x00, +0x55,0x01,0x00,0x00,0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x64,0x01,0x00,0x00,0xcb,0x00,0x00,0x00,0x61,0x01,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x69,0x01,0x00,0x00, +0x52,0x01,0x00,0x00,0x55,0x01,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x6a,0x01,0x00,0x00,0x64,0x01,0x00,0x00, +0x69,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x6e,0x01,0x00,0x00,0x6a,0x01,0x00,0x00,0x52,0x01,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x71,0x01,0x00,0x00, +0x6e,0x01,0x00,0x00,0x55,0x01,0x00,0x00,0x82,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x76,0x01,0x00,0x00,0x64,0x01,0x00,0x00, +0x71,0x01,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x79,0x01,0x00,0x00,0x76,0x01,0x00,0x00,0x55,0x01,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x82,0x01,0x00,0x00, +0x79,0x01,0x00,0x00,0x55,0x01,0x00,0x00,0x82,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x83,0x01,0x00,0x00,0x76,0x01,0x00,0x00, +0x82,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00, +0x85,0x01,0x00,0x00,0x14,0x00,0x00,0x00,0x55,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x86,0x01,0x00,0x00, +0x85,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x87,0x01,0x00,0x00,0x57,0x01,0x00,0x00,0x86,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00,0x89,0x01,0x00,0x00, +0x14,0x00,0x00,0x00,0x5a,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x8a,0x01,0x00,0x00,0x89,0x01,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8b,0x01,0x00,0x00, +0x6a,0x01,0x00,0x00,0x8a,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x8c,0x01,0x00,0x00,0x87,0x01,0x00,0x00, +0x8b,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00, +0x8e,0x01,0x00,0x00,0x14,0x00,0x00,0x00,0x60,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x8f,0x01,0x00,0x00, +0x8e,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x90,0x01,0x00,0x00,0x79,0x01,0x00,0x00,0x8f,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x91,0x01,0x00,0x00, +0x8c,0x01,0x00,0x00,0x90,0x01,0x00,0x00,0x41,0x00,0x05,0x00, +0x17,0x00,0x00,0x00,0x93,0x01,0x00,0x00,0x14,0x00,0x00,0x00, +0x66,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x94,0x01,0x00,0x00,0x93,0x01,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x95,0x01,0x00,0x00,0x83,0x01,0x00,0x00, +0x94,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x96,0x01,0x00,0x00,0x91,0x01,0x00,0x00,0x95,0x01,0x00,0x00, +0x41,0x00,0x06,0x00,0xe8,0x00,0x00,0x00,0xe9,0x00,0x00,0x00, +0xe3,0x00,0x00,0x00,0xcc,0x00,0x00,0x00,0x96,0x01,0x00,0x00, +0x3d,0x00,0x04,0x00,0x11,0x00,0x00,0x00,0xea,0x00,0x00,0x00, +0xe9,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0xe8,0x00,0x00,0x00, +0xeb,0x00,0x00,0x00,0xd7,0x00,0x00,0x00,0xcc,0x00,0x00,0x00, +0xdf,0x00,0x00,0x00,0x3e,0x00,0x03,0x00,0xeb,0x00,0x00,0x00, +0xea,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xef,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xef,0x00,0x00,0x00,0xfd,0x00,0x01,0x00, +0x38,0x00,0x01,0x00, +}; +const uint64_t cpy_f32_f32_len = 3232; + +unsigned char dequant_f32_data[] = { +0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, +0x1f,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, +0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x51,0x11,0x00,0x00, +0x0b,0x00,0x06,0x00,0x01,0x00,0x00,0x00,0x47,0x4c,0x53,0x4c, +0x2e,0x73,0x74,0x64,0x2e,0x34,0x35,0x30,0x00,0x00,0x00,0x00, +0x0e,0x00,0x03,0x00,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x0f,0x00,0x09,0x00,0x05,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x6d,0x61,0x69,0x6e,0x00,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, +0x15,0x00,0x00,0x00,0x2c,0x00,0x00,0x00,0x35,0x00,0x00,0x00, +0x10,0x00,0x06,0x00,0x04,0x00,0x00,0x00,0x11,0x00,0x00,0x00, +0x00,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x0b,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, +0x1c,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x13,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x13,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x13,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x08,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x13,0x00,0x00,0x00, +0x03,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x13,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x47,0x00,0x03,0x00, +0x13,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x29,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x48,0x00,0x04,0x00,0x2a,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x19,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x2a,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x03,0x00,0x2a,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x2c,0x00,0x00,0x00,0x22,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x2c,0x00,0x00,0x00, +0x21,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x32,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x48,0x00,0x04,0x00,0x33,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x33,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x03,0x00,0x33,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x35,0x00,0x00,0x00,0x22,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x35,0x00,0x00,0x00, +0x21,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x44,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x19,0x00,0x00,0x00, 0x13,0x00,0x02,0x00,0x02,0x00,0x00,0x00,0x21,0x00,0x03,0x00, 0x03,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x15,0x00,0x04,0x00, 0x06,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x00,0x00,0x00,0x00, @@ -55,114 +1852,1021 @@ unsigned char add_f32_data[] = { 0x0a,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x01,0x00,0x00,0x00, 0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, 0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x0d,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x16,0x00,0x03,0x00, -0x11,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x1e,0x00,0x06,0x00, -0x12,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x11,0x00,0x00,0x00,0x11,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x14,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x15,0x00,0x04,0x00,0x15,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x15,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x17,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x14,0x00,0x02,0x00,0x1a,0x00,0x00,0x00, -0x1d,0x00,0x03,0x00,0x1f,0x00,0x00,0x00,0x11,0x00,0x00,0x00, -0x1e,0x00,0x03,0x00,0x20,0x00,0x00,0x00,0x1f,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x21,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x21,0x00,0x00,0x00, -0x22,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, -0x24,0x00,0x00,0x00,0x11,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, -0x25,0x00,0x00,0x00,0x24,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x26,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x25,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x26,0x00,0x00,0x00,0x27,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x29,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x11,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, -0x2c,0x00,0x00,0x00,0x11,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, -0x2d,0x00,0x00,0x00,0x2c,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x2e,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x2d,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x2e,0x00,0x00,0x00,0x2f,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x15,0x00,0x00,0x00, -0x31,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x00,0x02,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x2c,0x00,0x06,0x00,0x09,0x00,0x00,0x00, -0x3b,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, -0x3a,0x00,0x00,0x00,0x36,0x00,0x05,0x00,0x02,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x10,0x00,0x00,0x00, +0x1e,0x00,0x07,0x00,0x13,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x14,0x00,0x00,0x00, +0x09,0x00,0x00,0x00,0x13,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0x14,0x00,0x00,0x00,0x15,0x00,0x00,0x00,0x09,0x00,0x00,0x00, +0x15,0x00,0x04,0x00,0x16,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x16,0x00,0x00,0x00, +0x17,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x18,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x14,0x00,0x02,0x00,0x1b,0x00,0x00,0x00,0x16,0x00,0x03,0x00, +0x28,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, +0x29,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, +0x2a,0x00,0x00,0x00,0x29,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x2b,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x2a,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0x2b,0x00,0x00,0x00,0x2c,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x16,0x00,0x00,0x00, +0x2d,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x16,0x00,0x03,0x00, +0x31,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, +0x32,0x00,0x00,0x00,0x31,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, +0x33,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x34,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x33,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0x34,0x00,0x00,0x00,0x35,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x39,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x31,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x3d,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x28,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x42,0x00,0x00,0x00, +0x00,0x01,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x43,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x2c,0x00,0x06,0x00, +0x09,0x00,0x00,0x00,0x44,0x00,0x00,0x00,0x42,0x00,0x00,0x00, +0x43,0x00,0x00,0x00,0x43,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x11,0x01,0x00,0x00,0x02,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x12,0x01,0x00,0x00, +0x03,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x13,0x01,0x00,0x00,0x04,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x14,0x01,0x00,0x00,0x05,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x15,0x01,0x00,0x00, +0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x16,0x01,0x00,0x00,0x07,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x17,0x01,0x00,0x00,0x08,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x18,0x01,0x00,0x00, +0x09,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x19,0x01,0x00,0x00,0x0a,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x1a,0x01,0x00,0x00,0x0b,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x1b,0x01,0x00,0x00, +0x0c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x1c,0x01,0x00,0x00,0x0d,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x1d,0x01,0x00,0x00,0x0e,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x1e,0x01,0x00,0x00, +0x0f,0x00,0x00,0x00,0x36,0x00,0x05,0x00,0x02,0x00,0x00,0x00, 0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x03,0x00,0x00,0x00, 0xf8,0x00,0x02,0x00,0x05,0x00,0x00,0x00,0xf7,0x00,0x03,0x00, -0x3c,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xfb,0x00,0x03,0x00, -0x0c,0x00,0x00,0x00,0x3d,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x3d,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00, +0x45,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xfb,0x00,0x03,0x00, +0x0c,0x00,0x00,0x00,0x46,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0x46,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00, 0x0e,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, 0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, -0x0e,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x19,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0xae,0x00,0x05,0x00,0x1a,0x00,0x00,0x00, -0x1b,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x19,0x00,0x00,0x00, -0xf7,0x00,0x03,0x00,0x1d,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x1b,0x00,0x00,0x00,0x1c,0x00,0x00,0x00, -0x1d,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x1c,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x3c,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x1d,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x29,0x00,0x00,0x00, -0x2a,0x00,0x00,0x00,0x27,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0x0f,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x11,0x00,0x00,0x00, -0x2b,0x00,0x00,0x00,0x2a,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x17,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x14,0x00,0x00,0x00, -0x31,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x33,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x89,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, -0x33,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x29,0x00,0x00,0x00, -0x35,0x00,0x00,0x00,0x2f,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0x34,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x11,0x00,0x00,0x00, -0x36,0x00,0x00,0x00,0x35,0x00,0x00,0x00,0x81,0x00,0x05,0x00, -0x11,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x2b,0x00,0x00,0x00, -0x36,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x29,0x00,0x00,0x00, -0x38,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0x0f,0x00,0x00,0x00,0x3e,0x00,0x03,0x00,0x38,0x00,0x00,0x00, -0x37,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x3c,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0x3c,0x00,0x00,0x00,0xfd,0x00,0x01,0x00, -0x38,0x00,0x01,0x00, +0x0e,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x11,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x10,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x18,0x00,0x00,0x00,0x19,0x00,0x00,0x00, +0x15,0x00,0x00,0x00,0x17,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x1a,0x00,0x00,0x00,0x19,0x00,0x00,0x00, +0xae,0x00,0x05,0x00,0x1b,0x00,0x00,0x00,0x1c,0x00,0x00,0x00, +0x11,0x00,0x00,0x00,0x1a,0x00,0x00,0x00,0xf7,0x00,0x03,0x00, +0x1e,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x1c,0x00,0x00,0x00,0x1d,0x00,0x00,0x00,0x1e,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0x1d,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x45,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x1e,0x00,0x00,0x00, +0x41,0x00,0x06,0x00,0x39,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, +0x35,0x00,0x00,0x00,0x2d,0x00,0x00,0x00,0x11,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x31,0x00,0x00,0x00,0x3b,0x00,0x00,0x00, +0x3a,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x28,0x00,0x00,0x00, +0x3c,0x00,0x00,0x00,0x3b,0x00,0x00,0x00,0x41,0x00,0x06,0x00, +0x3d,0x00,0x00,0x00,0x3e,0x00,0x00,0x00,0x2c,0x00,0x00,0x00, +0x2d,0x00,0x00,0x00,0x11,0x00,0x00,0x00,0x3e,0x00,0x03,0x00, +0x3e,0x00,0x00,0x00,0x3c,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x52,0x00,0x00,0x00,0x11,0x00,0x00,0x00, +0x43,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x39,0x00,0x00,0x00, +0x54,0x00,0x00,0x00,0x35,0x00,0x00,0x00,0x2d,0x00,0x00,0x00, +0x52,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x31,0x00,0x00,0x00, +0x55,0x00,0x00,0x00,0x54,0x00,0x00,0x00,0x73,0x00,0x04,0x00, +0x28,0x00,0x00,0x00,0x56,0x00,0x00,0x00,0x55,0x00,0x00,0x00, +0x41,0x00,0x06,0x00,0x3d,0x00,0x00,0x00,0x57,0x00,0x00,0x00, +0x2c,0x00,0x00,0x00,0x2d,0x00,0x00,0x00,0x52,0x00,0x00,0x00, +0x3e,0x00,0x03,0x00,0x57,0x00,0x00,0x00,0x56,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x5f,0x00,0x00,0x00, +0x11,0x00,0x00,0x00,0x11,0x01,0x00,0x00,0x41,0x00,0x06,0x00, +0x39,0x00,0x00,0x00,0x61,0x00,0x00,0x00,0x35,0x00,0x00,0x00, +0x2d,0x00,0x00,0x00,0x5f,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x31,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x61,0x00,0x00,0x00, +0x73,0x00,0x04,0x00,0x28,0x00,0x00,0x00,0x63,0x00,0x00,0x00, +0x62,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x3d,0x00,0x00,0x00, +0x64,0x00,0x00,0x00,0x2c,0x00,0x00,0x00,0x2d,0x00,0x00,0x00, +0x5f,0x00,0x00,0x00,0x3e,0x00,0x03,0x00,0x64,0x00,0x00,0x00, +0x63,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x11,0x00,0x00,0x00,0x12,0x01,0x00,0x00, +0x41,0x00,0x06,0x00,0x39,0x00,0x00,0x00,0x6e,0x00,0x00,0x00, +0x35,0x00,0x00,0x00,0x2d,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x31,0x00,0x00,0x00,0x6f,0x00,0x00,0x00, +0x6e,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x28,0x00,0x00,0x00, +0x70,0x00,0x00,0x00,0x6f,0x00,0x00,0x00,0x41,0x00,0x06,0x00, +0x3d,0x00,0x00,0x00,0x71,0x00,0x00,0x00,0x2c,0x00,0x00,0x00, +0x2d,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x3e,0x00,0x03,0x00, +0x71,0x00,0x00,0x00,0x70,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x79,0x00,0x00,0x00,0x11,0x00,0x00,0x00, +0x13,0x01,0x00,0x00,0x41,0x00,0x06,0x00,0x39,0x00,0x00,0x00, +0x7b,0x00,0x00,0x00,0x35,0x00,0x00,0x00,0x2d,0x00,0x00,0x00, +0x79,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x31,0x00,0x00,0x00, +0x7c,0x00,0x00,0x00,0x7b,0x00,0x00,0x00,0x73,0x00,0x04,0x00, +0x28,0x00,0x00,0x00,0x7d,0x00,0x00,0x00,0x7c,0x00,0x00,0x00, +0x41,0x00,0x06,0x00,0x3d,0x00,0x00,0x00,0x7e,0x00,0x00,0x00, +0x2c,0x00,0x00,0x00,0x2d,0x00,0x00,0x00,0x79,0x00,0x00,0x00, +0x3e,0x00,0x03,0x00,0x7e,0x00,0x00,0x00,0x7d,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0x11,0x00,0x00,0x00,0x14,0x01,0x00,0x00,0x41,0x00,0x06,0x00, +0x39,0x00,0x00,0x00,0x88,0x00,0x00,0x00,0x35,0x00,0x00,0x00, +0x2d,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x31,0x00,0x00,0x00,0x89,0x00,0x00,0x00,0x88,0x00,0x00,0x00, +0x73,0x00,0x04,0x00,0x28,0x00,0x00,0x00,0x8a,0x00,0x00,0x00, +0x89,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x3d,0x00,0x00,0x00, +0x8b,0x00,0x00,0x00,0x2c,0x00,0x00,0x00,0x2d,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0x3e,0x00,0x03,0x00,0x8b,0x00,0x00,0x00, +0x8a,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x93,0x00,0x00,0x00,0x11,0x00,0x00,0x00,0x15,0x01,0x00,0x00, +0x41,0x00,0x06,0x00,0x39,0x00,0x00,0x00,0x95,0x00,0x00,0x00, +0x35,0x00,0x00,0x00,0x2d,0x00,0x00,0x00,0x93,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x31,0x00,0x00,0x00,0x96,0x00,0x00,0x00, +0x95,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x28,0x00,0x00,0x00, +0x97,0x00,0x00,0x00,0x96,0x00,0x00,0x00,0x41,0x00,0x06,0x00, +0x3d,0x00,0x00,0x00,0x98,0x00,0x00,0x00,0x2c,0x00,0x00,0x00, +0x2d,0x00,0x00,0x00,0x93,0x00,0x00,0x00,0x3e,0x00,0x03,0x00, +0x98,0x00,0x00,0x00,0x97,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xa0,0x00,0x00,0x00,0x11,0x00,0x00,0x00, +0x16,0x01,0x00,0x00,0x41,0x00,0x06,0x00,0x39,0x00,0x00,0x00, +0xa2,0x00,0x00,0x00,0x35,0x00,0x00,0x00,0x2d,0x00,0x00,0x00, +0xa0,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x31,0x00,0x00,0x00, +0xa3,0x00,0x00,0x00,0xa2,0x00,0x00,0x00,0x73,0x00,0x04,0x00, +0x28,0x00,0x00,0x00,0xa4,0x00,0x00,0x00,0xa3,0x00,0x00,0x00, +0x41,0x00,0x06,0x00,0x3d,0x00,0x00,0x00,0xa5,0x00,0x00,0x00, +0x2c,0x00,0x00,0x00,0x2d,0x00,0x00,0x00,0xa0,0x00,0x00,0x00, +0x3e,0x00,0x03,0x00,0xa5,0x00,0x00,0x00,0xa4,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xad,0x00,0x00,0x00, +0x11,0x00,0x00,0x00,0x17,0x01,0x00,0x00,0x41,0x00,0x06,0x00, +0x39,0x00,0x00,0x00,0xaf,0x00,0x00,0x00,0x35,0x00,0x00,0x00, +0x2d,0x00,0x00,0x00,0xad,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x31,0x00,0x00,0x00,0xb0,0x00,0x00,0x00,0xaf,0x00,0x00,0x00, +0x73,0x00,0x04,0x00,0x28,0x00,0x00,0x00,0xb1,0x00,0x00,0x00, +0xb0,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x3d,0x00,0x00,0x00, +0xb2,0x00,0x00,0x00,0x2c,0x00,0x00,0x00,0x2d,0x00,0x00,0x00, +0xad,0x00,0x00,0x00,0x3e,0x00,0x03,0x00,0xb2,0x00,0x00,0x00, +0xb1,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xba,0x00,0x00,0x00,0x11,0x00,0x00,0x00,0x18,0x01,0x00,0x00, +0x41,0x00,0x06,0x00,0x39,0x00,0x00,0x00,0xbc,0x00,0x00,0x00, +0x35,0x00,0x00,0x00,0x2d,0x00,0x00,0x00,0xba,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x31,0x00,0x00,0x00,0xbd,0x00,0x00,0x00, +0xbc,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x28,0x00,0x00,0x00, +0xbe,0x00,0x00,0x00,0xbd,0x00,0x00,0x00,0x41,0x00,0x06,0x00, +0x3d,0x00,0x00,0x00,0xbf,0x00,0x00,0x00,0x2c,0x00,0x00,0x00, +0x2d,0x00,0x00,0x00,0xba,0x00,0x00,0x00,0x3e,0x00,0x03,0x00, +0xbf,0x00,0x00,0x00,0xbe,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xc7,0x00,0x00,0x00,0x11,0x00,0x00,0x00, +0x19,0x01,0x00,0x00,0x41,0x00,0x06,0x00,0x39,0x00,0x00,0x00, +0xc9,0x00,0x00,0x00,0x35,0x00,0x00,0x00,0x2d,0x00,0x00,0x00, +0xc7,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x31,0x00,0x00,0x00, +0xca,0x00,0x00,0x00,0xc9,0x00,0x00,0x00,0x73,0x00,0x04,0x00, +0x28,0x00,0x00,0x00,0xcb,0x00,0x00,0x00,0xca,0x00,0x00,0x00, +0x41,0x00,0x06,0x00,0x3d,0x00,0x00,0x00,0xcc,0x00,0x00,0x00, +0x2c,0x00,0x00,0x00,0x2d,0x00,0x00,0x00,0xc7,0x00,0x00,0x00, +0x3e,0x00,0x03,0x00,0xcc,0x00,0x00,0x00,0xcb,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xd4,0x00,0x00,0x00, +0x11,0x00,0x00,0x00,0x1a,0x01,0x00,0x00,0x41,0x00,0x06,0x00, +0x39,0x00,0x00,0x00,0xd6,0x00,0x00,0x00,0x35,0x00,0x00,0x00, +0x2d,0x00,0x00,0x00,0xd4,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x31,0x00,0x00,0x00,0xd7,0x00,0x00,0x00,0xd6,0x00,0x00,0x00, +0x73,0x00,0x04,0x00,0x28,0x00,0x00,0x00,0xd8,0x00,0x00,0x00, +0xd7,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x3d,0x00,0x00,0x00, +0xd9,0x00,0x00,0x00,0x2c,0x00,0x00,0x00,0x2d,0x00,0x00,0x00, +0xd4,0x00,0x00,0x00,0x3e,0x00,0x03,0x00,0xd9,0x00,0x00,0x00, +0xd8,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xe1,0x00,0x00,0x00,0x11,0x00,0x00,0x00,0x1b,0x01,0x00,0x00, +0x41,0x00,0x06,0x00,0x39,0x00,0x00,0x00,0xe3,0x00,0x00,0x00, +0x35,0x00,0x00,0x00,0x2d,0x00,0x00,0x00,0xe1,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x31,0x00,0x00,0x00,0xe4,0x00,0x00,0x00, +0xe3,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x28,0x00,0x00,0x00, +0xe5,0x00,0x00,0x00,0xe4,0x00,0x00,0x00,0x41,0x00,0x06,0x00, +0x3d,0x00,0x00,0x00,0xe6,0x00,0x00,0x00,0x2c,0x00,0x00,0x00, +0x2d,0x00,0x00,0x00,0xe1,0x00,0x00,0x00,0x3e,0x00,0x03,0x00, +0xe6,0x00,0x00,0x00,0xe5,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xee,0x00,0x00,0x00,0x11,0x00,0x00,0x00, +0x1c,0x01,0x00,0x00,0x41,0x00,0x06,0x00,0x39,0x00,0x00,0x00, +0xf0,0x00,0x00,0x00,0x35,0x00,0x00,0x00,0x2d,0x00,0x00,0x00, +0xee,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x31,0x00,0x00,0x00, +0xf1,0x00,0x00,0x00,0xf0,0x00,0x00,0x00,0x73,0x00,0x04,0x00, +0x28,0x00,0x00,0x00,0xf2,0x00,0x00,0x00,0xf1,0x00,0x00,0x00, +0x41,0x00,0x06,0x00,0x3d,0x00,0x00,0x00,0xf3,0x00,0x00,0x00, +0x2c,0x00,0x00,0x00,0x2d,0x00,0x00,0x00,0xee,0x00,0x00,0x00, +0x3e,0x00,0x03,0x00,0xf3,0x00,0x00,0x00,0xf2,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xfb,0x00,0x00,0x00, +0x11,0x00,0x00,0x00,0x1d,0x01,0x00,0x00,0x41,0x00,0x06,0x00, +0x39,0x00,0x00,0x00,0xfd,0x00,0x00,0x00,0x35,0x00,0x00,0x00, +0x2d,0x00,0x00,0x00,0xfb,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x31,0x00,0x00,0x00,0xfe,0x00,0x00,0x00,0xfd,0x00,0x00,0x00, +0x73,0x00,0x04,0x00,0x28,0x00,0x00,0x00,0xff,0x00,0x00,0x00, +0xfe,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x3d,0x00,0x00,0x00, +0x00,0x01,0x00,0x00,0x2c,0x00,0x00,0x00,0x2d,0x00,0x00,0x00, +0xfb,0x00,0x00,0x00,0x3e,0x00,0x03,0x00,0x00,0x01,0x00,0x00, +0xff,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x08,0x01,0x00,0x00,0x11,0x00,0x00,0x00,0x1e,0x01,0x00,0x00, +0x41,0x00,0x06,0x00,0x39,0x00,0x00,0x00,0x0a,0x01,0x00,0x00, +0x35,0x00,0x00,0x00,0x2d,0x00,0x00,0x00,0x08,0x01,0x00,0x00, +0x3d,0x00,0x04,0x00,0x31,0x00,0x00,0x00,0x0b,0x01,0x00,0x00, +0x0a,0x01,0x00,0x00,0x73,0x00,0x04,0x00,0x28,0x00,0x00,0x00, +0x0c,0x01,0x00,0x00,0x0b,0x01,0x00,0x00,0x41,0x00,0x06,0x00, +0x3d,0x00,0x00,0x00,0x0d,0x01,0x00,0x00,0x2c,0x00,0x00,0x00, +0x2d,0x00,0x00,0x00,0x08,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, +0x0d,0x01,0x00,0x00,0x0c,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0x45,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x45,0x00,0x00,0x00, +0xfd,0x00,0x01,0x00,0x38,0x00,0x01,0x00, }; -const uint64_t add_f32_len = 1456; +const uint64_t dequant_f32_len = 3200; -unsigned char clamp_f32_data[] = { +unsigned char dequant_q2_K_data[] = { 0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, -0x55,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, -0x01,0x00,0x00,0x00,0x0b,0x00,0x06,0x00,0x01,0x00,0x00,0x00, +0x10,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, +0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x27,0x00,0x00,0x00, +0x11,0x00,0x02,0x00,0x51,0x11,0x00,0x00,0x11,0x00,0x02,0x00, +0x60,0x11,0x00,0x00,0x0b,0x00,0x06,0x00,0x01,0x00,0x00,0x00, 0x47,0x4c,0x53,0x4c,0x2e,0x73,0x74,0x64,0x2e,0x34,0x35,0x30, 0x00,0x00,0x00,0x00,0x0e,0x00,0x03,0x00,0x00,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x0f,0x00,0x09,0x00,0x05,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x0f,0x00,0x0a,0x00,0x05,0x00,0x00,0x00, 0x04,0x00,0x00,0x00,0x6d,0x61,0x69,0x6e,0x00,0x00,0x00,0x00, -0x0b,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x24,0x00,0x00,0x00, -0x2c,0x00,0x00,0x00,0x10,0x00,0x06,0x00,0x04,0x00,0x00,0x00, -0x11,0x00,0x00,0x00,0x00,0x02,0x00,0x00,0x01,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x0b,0x00,0x00,0x00, -0x0b,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x12,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x12,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x12,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x12,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x12,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x21,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00, -0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x16,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x30,0x00,0x00,0x00, +0x5e,0x00,0x00,0x00,0x79,0x00,0x00,0x00,0x10,0x00,0x06,0x00, +0x04,0x00,0x00,0x00,0x11,0x00,0x00,0x00,0x40,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x16,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x1a,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x1e,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x1e,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x1e,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x08,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x1e,0x00,0x00,0x00,0x03,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x1e,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x10,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x1e,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x30,0x00,0x00,0x00, +0x0b,0x00,0x00,0x00,0x1b,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x55,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x57,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x5a,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x5a,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x5a,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x50,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x5b,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x54,0x00,0x00,0x00,0x48,0x00,0x04,0x00, +0x5c,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x5c,0x00,0x00,0x00,0x00,0x00,0x00,0x00, 0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, -0x22,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x24,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x24,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x29,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00, -0x2a,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x19,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x2a,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x5c,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x5e,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x5e,0x00,0x00,0x00,0x21,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x76,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x48,0x00,0x04,0x00, +0x77,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x19,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x77,0x00,0x00,0x00,0x00,0x00,0x00,0x00, 0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, -0x2a,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x2c,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x2c,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x4b,0x00,0x00,0x00, +0x77,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x79,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x79,0x00,0x00,0x00,0x21,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x00,0x01,0x00,0x00, +0x0b,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x13,0x00,0x02,0x00, +0x02,0x00,0x00,0x00,0x21,0x00,0x03,0x00,0x03,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x15,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x10,0x00,0x00,0x00, +0x00,0x01,0x00,0x00,0x14,0x00,0x02,0x00,0x11,0x00,0x00,0x00, +0x17,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x03,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x15,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0x15,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x17,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x1e,0x00,0x07,0x00,0x1e,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x1f,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x1e,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0x1f,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0x09,0x00,0x00,0x00,0x15,0x00,0x04,0x00,0x21,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x21,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x23,0x00,0x00,0x00,0x09,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x21,0x00,0x00,0x00, +0x26,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0x15,0x00,0x00,0x00,0x30,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x35,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x3d,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x41,0x00,0x00,0x00,0x10,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x47,0x00,0x00,0x00, +0x80,0x00,0x00,0x00,0x15,0x00,0x04,0x00,0x52,0x00,0x00,0x00, +0x08,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x1c,0x00,0x04,0x00, +0x55,0x00,0x00,0x00,0x52,0x00,0x00,0x00,0x41,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x56,0x00,0x00,0x00, +0x40,0x00,0x00,0x00,0x1c,0x00,0x04,0x00,0x57,0x00,0x00,0x00, +0x52,0x00,0x00,0x00,0x56,0x00,0x00,0x00,0x16,0x00,0x03,0x00, +0x58,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x17,0x00,0x04,0x00, +0x59,0x00,0x00,0x00,0x58,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x1e,0x00,0x05,0x00,0x5a,0x00,0x00,0x00,0x55,0x00,0x00,0x00, +0x57,0x00,0x00,0x00,0x59,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, +0x5b,0x00,0x00,0x00,0x5a,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, +0x5c,0x00,0x00,0x00,0x5b,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x5d,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x5c,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0x5d,0x00,0x00,0x00,0x5e,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x64,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x52,0x00,0x00,0x00,0x16,0x00,0x03,0x00, +0x67,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x21,0x00,0x00,0x00,0x6b,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x6c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x58,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x72,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, +0x76,0x00,0x00,0x00,0x58,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, +0x77,0x00,0x00,0x00,0x76,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x78,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x77,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0x78,0x00,0x00,0x00,0x79,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x21,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x21,0x00,0x00,0x00,0x8a,0x00,0x00,0x00,0x03,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x21,0x00,0x00,0x00,0x95,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xa1,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xc1,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xdd,0x00,0x00,0x00, +0x60,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xe2,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x21,0x00,0x00,0x00,0xea,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x2c,0x00,0x06,0x00,0x14,0x00,0x00,0x00,0x00,0x01,0x00,0x00, +0x56,0x00,0x00,0x00,0x72,0x00,0x00,0x00,0x72,0x00,0x00,0x00, +0x2a,0x00,0x03,0x00,0x11,0x00,0x00,0x00,0x03,0x01,0x00,0x00, +0x29,0x00,0x03,0x00,0x11,0x00,0x00,0x00,0x06,0x01,0x00,0x00, +0x36,0x00,0x05,0x00,0x02,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0x05,0x00,0x00,0x00,0xf7,0x00,0x03,0x00,0x01,0x01,0x00,0x00, +0x00,0x00,0x00,0x00,0xfb,0x00,0x03,0x00,0x09,0x00,0x00,0x00, +0x02,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x02,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0x0a,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0x0a,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0x09,0x01,0x00,0x00,0x09,0x00,0x00,0x00,0x02,0x01,0x00,0x00, +0xff,0x00,0x00,0x00,0x0d,0x00,0x00,0x00,0xb0,0x00,0x05,0x00, +0x11,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x09,0x01,0x00,0x00, +0x10,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x0c,0x00,0x00,0x00, +0x0d,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x12,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0x0b,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x17,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x16,0x00,0x00,0x00, +0x09,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x19,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x1a,0x00,0x00,0x00,0x19,0x00,0x00,0x00, +0x10,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x1c,0x00,0x00,0x00,0x1a,0x00,0x00,0x00,0x09,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0x23,0x00,0x00,0x00,0x24,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x25,0x00,0x00,0x00,0x24,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x23,0x00,0x00,0x00,0x27,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0x26,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0x27,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x29,0x00,0x00,0x00, +0x25,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x2a,0x00,0x00,0x00,0x29,0x00,0x00,0x00, +0x10,0x00,0x00,0x00,0xae,0x00,0x05,0x00,0x11,0x00,0x00,0x00, +0x2b,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x2a,0x00,0x00,0x00, +0xf7,0x00,0x03,0x00,0x2d,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x2b,0x00,0x00,0x00,0x2c,0x00,0x00,0x00, +0x2d,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x2c,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x0c,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0x2d,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00, +0x31,0x00,0x00,0x00,0x30,0x00,0x00,0x00,0x09,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x32,0x00,0x00,0x00, +0x31,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x36,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x35,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, +0x35,0x00,0x00,0x00,0x36,0x00,0x00,0x00,0x82,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x3b,0x00,0x00,0x00,0x32,0x00,0x00,0x00, +0x3a,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x3f,0x00,0x00,0x00,0x3d,0x00,0x00,0x00,0x36,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x42,0x00,0x00,0x00, +0x3b,0x00,0x00,0x00,0x41,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x43,0x00,0x00,0x00,0x3f,0x00,0x00,0x00, +0x42,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x46,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x10,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x49,0x00,0x00,0x00, +0x47,0x00,0x00,0x00,0x36,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x4a,0x00,0x00,0x00,0x46,0x00,0x00,0x00, +0x49,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x4c,0x00,0x00,0x00,0x4a,0x00,0x00,0x00,0x3b,0x00,0x00,0x00, +0x41,0x00,0x08,0x00,0x64,0x00,0x00,0x00,0x65,0x00,0x00,0x00, +0x5e,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x1c,0x00,0x00,0x00, +0x26,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x52,0x00,0x00,0x00,0x66,0x00,0x00,0x00,0x65,0x00,0x00,0x00, +0x41,0x00,0x08,0x00,0x6c,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, +0x5e,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x1c,0x00,0x00,0x00, +0x6b,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x58,0x00,0x00,0x00,0x6e,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, +0x73,0x00,0x04,0x00,0x67,0x00,0x00,0x00,0x6f,0x00,0x00,0x00, +0x6e,0x00,0x00,0x00,0x41,0x00,0x08,0x00,0x6c,0x00,0x00,0x00, +0x73,0x00,0x00,0x00,0x5e,0x00,0x00,0x00,0x22,0x00,0x00,0x00, +0x1c,0x00,0x00,0x00,0x6b,0x00,0x00,0x00,0x72,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x58,0x00,0x00,0x00,0x74,0x00,0x00,0x00, +0x73,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x67,0x00,0x00,0x00, +0x75,0x00,0x00,0x00,0x74,0x00,0x00,0x00,0x41,0x00,0x08,0x00, +0x64,0x00,0x00,0x00,0x80,0x00,0x00,0x00,0x5e,0x00,0x00,0x00, +0x22,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x22,0x00,0x00,0x00, +0x43,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x52,0x00,0x00,0x00, +0x81,0x00,0x00,0x00,0x80,0x00,0x00,0x00,0x71,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x82,0x00,0x00,0x00,0x81,0x00,0x00,0x00, +0x7c,0x00,0x04,0x00,0x21,0x00,0x00,0x00,0x83,0x00,0x00,0x00, +0x82,0x00,0x00,0x00,0xc7,0x00,0x05,0x00,0x21,0x00,0x00,0x00, +0x85,0x00,0x00,0x00,0x83,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0xc2,0x00,0x05,0x00,0x52,0x00,0x00,0x00,0x87,0x00,0x00,0x00, +0x66,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x71,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x88,0x00,0x00,0x00,0x87,0x00,0x00,0x00, +0x7c,0x00,0x04,0x00,0x21,0x00,0x00,0x00,0x89,0x00,0x00,0x00, +0x88,0x00,0x00,0x00,0xc7,0x00,0x05,0x00,0x21,0x00,0x00,0x00, +0x8b,0x00,0x00,0x00,0x89,0x00,0x00,0x00,0x8a,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x21,0x00,0x00,0x00,0x8c,0x00,0x00,0x00, +0x85,0x00,0x00,0x00,0x8b,0x00,0x00,0x00,0x6f,0x00,0x04,0x00, +0x67,0x00,0x00,0x00,0x8d,0x00,0x00,0x00,0x8c,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x52,0x00,0x00,0x00,0x94,0x00,0x00,0x00, +0x80,0x00,0x00,0x00,0xc2,0x00,0x05,0x00,0x52,0x00,0x00,0x00, +0x96,0x00,0x00,0x00,0x94,0x00,0x00,0x00,0x95,0x00,0x00,0x00, +0x70,0x00,0x04,0x00,0x67,0x00,0x00,0x00,0x97,0x00,0x00,0x00, +0x96,0x00,0x00,0x00,0x85,0x00,0x05,0x00,0x67,0x00,0x00,0x00, +0x98,0x00,0x00,0x00,0x75,0x00,0x00,0x00,0x97,0x00,0x00,0x00, +0x7f,0x00,0x04,0x00,0x67,0x00,0x00,0x00,0x0c,0x01,0x00,0x00, +0x98,0x00,0x00,0x00,0x0c,0x00,0x08,0x00,0x67,0x00,0x00,0x00, +0x99,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00, +0x6f,0x00,0x00,0x00,0x8d,0x00,0x00,0x00,0x0c,0x01,0x00,0x00, +0x73,0x00,0x04,0x00,0x58,0x00,0x00,0x00,0x9a,0x00,0x00,0x00, +0x99,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x6c,0x00,0x00,0x00, +0x9b,0x00,0x00,0x00,0x79,0x00,0x00,0x00,0x22,0x00,0x00,0x00, +0x4c,0x00,0x00,0x00,0x3e,0x00,0x03,0x00,0x9b,0x00,0x00,0x00, +0x9a,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x9d,0x00,0x00,0x00,0x4c,0x00,0x00,0x00,0x35,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa2,0x00,0x00,0x00, +0x43,0x00,0x00,0x00,0xa1,0x00,0x00,0x00,0x41,0x00,0x08,0x00, +0x64,0x00,0x00,0x00,0xa3,0x00,0x00,0x00,0x5e,0x00,0x00,0x00, +0x22,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x22,0x00,0x00,0x00, +0xa2,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x52,0x00,0x00,0x00, +0xa4,0x00,0x00,0x00,0xa3,0x00,0x00,0x00,0x71,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xa5,0x00,0x00,0x00,0xa4,0x00,0x00,0x00, +0x7c,0x00,0x04,0x00,0x21,0x00,0x00,0x00,0xa6,0x00,0x00,0x00, +0xa5,0x00,0x00,0x00,0xc7,0x00,0x05,0x00,0x21,0x00,0x00,0x00, +0xa7,0x00,0x00,0x00,0xa6,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0xc2,0x00,0x05,0x00,0x52,0x00,0x00,0x00,0xa9,0x00,0x00,0x00, +0x66,0x00,0x00,0x00,0x6b,0x00,0x00,0x00,0x71,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xaa,0x00,0x00,0x00,0xa9,0x00,0x00,0x00, +0x7c,0x00,0x04,0x00,0x21,0x00,0x00,0x00,0xab,0x00,0x00,0x00, +0xaa,0x00,0x00,0x00,0xc7,0x00,0x05,0x00,0x21,0x00,0x00,0x00, +0xac,0x00,0x00,0x00,0xab,0x00,0x00,0x00,0x8a,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x21,0x00,0x00,0x00,0xad,0x00,0x00,0x00, +0xa7,0x00,0x00,0x00,0xac,0x00,0x00,0x00,0x6f,0x00,0x04,0x00, +0x67,0x00,0x00,0x00,0xae,0x00,0x00,0x00,0xad,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x52,0x00,0x00,0x00,0xb5,0x00,0x00,0x00, +0xa3,0x00,0x00,0x00,0xc2,0x00,0x05,0x00,0x52,0x00,0x00,0x00, +0xb6,0x00,0x00,0x00,0xb5,0x00,0x00,0x00,0x95,0x00,0x00,0x00, +0x70,0x00,0x04,0x00,0x67,0x00,0x00,0x00,0xb7,0x00,0x00,0x00, +0xb6,0x00,0x00,0x00,0x85,0x00,0x05,0x00,0x67,0x00,0x00,0x00, +0xb8,0x00,0x00,0x00,0x75,0x00,0x00,0x00,0xb7,0x00,0x00,0x00, +0x7f,0x00,0x04,0x00,0x67,0x00,0x00,0x00,0x0d,0x01,0x00,0x00, +0xb8,0x00,0x00,0x00,0x0c,0x00,0x08,0x00,0x67,0x00,0x00,0x00, +0xb9,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00, +0x6f,0x00,0x00,0x00,0xae,0x00,0x00,0x00,0x0d,0x01,0x00,0x00, +0x73,0x00,0x04,0x00,0x58,0x00,0x00,0x00,0xba,0x00,0x00,0x00, +0xb9,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x6c,0x00,0x00,0x00, +0xbb,0x00,0x00,0x00,0x79,0x00,0x00,0x00,0x22,0x00,0x00,0x00, +0x9d,0x00,0x00,0x00,0x3e,0x00,0x03,0x00,0xbb,0x00,0x00,0x00, +0xba,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xbd,0x00,0x00,0x00,0x4c,0x00,0x00,0x00,0x56,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xc2,0x00,0x00,0x00, +0x43,0x00,0x00,0x00,0xc1,0x00,0x00,0x00,0x41,0x00,0x08,0x00, +0x64,0x00,0x00,0x00,0xc3,0x00,0x00,0x00,0x5e,0x00,0x00,0x00, +0x22,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x22,0x00,0x00,0x00, +0xc2,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x52,0x00,0x00,0x00, +0xc4,0x00,0x00,0x00,0xc3,0x00,0x00,0x00,0x71,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xc5,0x00,0x00,0x00,0xc4,0x00,0x00,0x00, +0x7c,0x00,0x04,0x00,0x21,0x00,0x00,0x00,0xc6,0x00,0x00,0x00, +0xc5,0x00,0x00,0x00,0xc7,0x00,0x05,0x00,0x21,0x00,0x00,0x00, +0xc7,0x00,0x00,0x00,0xc6,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0xc2,0x00,0x05,0x00,0x52,0x00,0x00,0x00,0xc9,0x00,0x00,0x00, +0x66,0x00,0x00,0x00,0x95,0x00,0x00,0x00,0x71,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xca,0x00,0x00,0x00,0xc9,0x00,0x00,0x00, +0x7c,0x00,0x04,0x00,0x21,0x00,0x00,0x00,0xcb,0x00,0x00,0x00, +0xca,0x00,0x00,0x00,0xc7,0x00,0x05,0x00,0x21,0x00,0x00,0x00, +0xcc,0x00,0x00,0x00,0xcb,0x00,0x00,0x00,0x8a,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x21,0x00,0x00,0x00,0xcd,0x00,0x00,0x00, +0xc7,0x00,0x00,0x00,0xcc,0x00,0x00,0x00,0x6f,0x00,0x04,0x00, +0x67,0x00,0x00,0x00,0xce,0x00,0x00,0x00,0xcd,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x52,0x00,0x00,0x00,0xd5,0x00,0x00,0x00, +0xc3,0x00,0x00,0x00,0xc2,0x00,0x05,0x00,0x52,0x00,0x00,0x00, +0xd6,0x00,0x00,0x00,0xd5,0x00,0x00,0x00,0x95,0x00,0x00,0x00, +0x70,0x00,0x04,0x00,0x67,0x00,0x00,0x00,0xd7,0x00,0x00,0x00, +0xd6,0x00,0x00,0x00,0x85,0x00,0x05,0x00,0x67,0x00,0x00,0x00, +0xd8,0x00,0x00,0x00,0x75,0x00,0x00,0x00,0xd7,0x00,0x00,0x00, +0x7f,0x00,0x04,0x00,0x67,0x00,0x00,0x00,0x0e,0x01,0x00,0x00, +0xd8,0x00,0x00,0x00,0x0c,0x00,0x08,0x00,0x67,0x00,0x00,0x00, +0xd9,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00, +0x6f,0x00,0x00,0x00,0xce,0x00,0x00,0x00,0x0e,0x01,0x00,0x00, +0x73,0x00,0x04,0x00,0x58,0x00,0x00,0x00,0xda,0x00,0x00,0x00, +0xd9,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x6c,0x00,0x00,0x00, +0xdb,0x00,0x00,0x00,0x79,0x00,0x00,0x00,0x22,0x00,0x00,0x00, +0xbd,0x00,0x00,0x00,0x3e,0x00,0x03,0x00,0xdb,0x00,0x00,0x00, +0xda,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xde,0x00,0x00,0x00,0x4c,0x00,0x00,0x00,0xdd,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xe3,0x00,0x00,0x00, +0x43,0x00,0x00,0x00,0xe2,0x00,0x00,0x00,0x41,0x00,0x08,0x00, +0x64,0x00,0x00,0x00,0xe4,0x00,0x00,0x00,0x5e,0x00,0x00,0x00, +0x22,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x22,0x00,0x00,0x00, +0xe3,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x52,0x00,0x00,0x00, +0xe5,0x00,0x00,0x00,0xe4,0x00,0x00,0x00,0x71,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xe6,0x00,0x00,0x00,0xe5,0x00,0x00,0x00, +0x7c,0x00,0x04,0x00,0x21,0x00,0x00,0x00,0xe7,0x00,0x00,0x00, +0xe6,0x00,0x00,0x00,0xc7,0x00,0x05,0x00,0x21,0x00,0x00,0x00, +0xe8,0x00,0x00,0x00,0xe7,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0xc2,0x00,0x05,0x00,0x52,0x00,0x00,0x00,0xeb,0x00,0x00,0x00, +0x66,0x00,0x00,0x00,0xea,0x00,0x00,0x00,0x71,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xec,0x00,0x00,0x00,0xeb,0x00,0x00,0x00, +0x7c,0x00,0x04,0x00,0x21,0x00,0x00,0x00,0xed,0x00,0x00,0x00, +0xec,0x00,0x00,0x00,0xc7,0x00,0x05,0x00,0x21,0x00,0x00,0x00, +0xee,0x00,0x00,0x00,0xed,0x00,0x00,0x00,0x8a,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x21,0x00,0x00,0x00,0xef,0x00,0x00,0x00, +0xe8,0x00,0x00,0x00,0xee,0x00,0x00,0x00,0x6f,0x00,0x04,0x00, +0x67,0x00,0x00,0x00,0xf0,0x00,0x00,0x00,0xef,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x52,0x00,0x00,0x00,0xf7,0x00,0x00,0x00, +0xe4,0x00,0x00,0x00,0xc2,0x00,0x05,0x00,0x52,0x00,0x00,0x00, +0xf8,0x00,0x00,0x00,0xf7,0x00,0x00,0x00,0x95,0x00,0x00,0x00, +0x70,0x00,0x04,0x00,0x67,0x00,0x00,0x00,0xf9,0x00,0x00,0x00, +0xf8,0x00,0x00,0x00,0x85,0x00,0x05,0x00,0x67,0x00,0x00,0x00, +0xfa,0x00,0x00,0x00,0x75,0x00,0x00,0x00,0xf9,0x00,0x00,0x00, +0x7f,0x00,0x04,0x00,0x67,0x00,0x00,0x00,0x0f,0x01,0x00,0x00, +0xfa,0x00,0x00,0x00,0x0c,0x00,0x08,0x00,0x67,0x00,0x00,0x00, +0xfb,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00, +0x6f,0x00,0x00,0x00,0xf0,0x00,0x00,0x00,0x0f,0x01,0x00,0x00, +0x73,0x00,0x04,0x00,0x58,0x00,0x00,0x00,0xfc,0x00,0x00,0x00, +0xfb,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x6c,0x00,0x00,0x00, +0xfd,0x00,0x00,0x00,0x79,0x00,0x00,0x00,0x22,0x00,0x00,0x00, +0xde,0x00,0x00,0x00,0x3e,0x00,0x03,0x00,0xfd,0x00,0x00,0x00, +0xfc,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x0d,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0x0d,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xff,0x00,0x00,0x00,0x09,0x01,0x00,0x00, +0x26,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x0a,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0x0c,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, +0x11,0x00,0x00,0x00,0x0a,0x01,0x00,0x00,0x03,0x01,0x00,0x00, +0x0a,0x00,0x00,0x00,0x06,0x01,0x00,0x00,0x2c,0x00,0x00,0x00, +0xf7,0x00,0x03,0x00,0x07,0x01,0x00,0x00,0x00,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x0a,0x01,0x00,0x00,0x01,0x01,0x00,0x00, +0x07,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x07,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0x01,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x01,0x01,0x00,0x00,0xfd,0x00,0x01,0x00,0x38,0x00,0x01,0x00, + +}; +const uint64_t dequant_q2_K_len = 4032; + +unsigned char dequant_q3_K_data[] = { +0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, +0x3f,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, +0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x27,0x00,0x00,0x00, +0x11,0x00,0x02,0x00,0x51,0x11,0x00,0x00,0x11,0x00,0x02,0x00, +0x60,0x11,0x00,0x00,0x0b,0x00,0x06,0x00,0x01,0x00,0x00,0x00, +0x47,0x4c,0x53,0x4c,0x2e,0x73,0x74,0x64,0x2e,0x34,0x35,0x30, +0x00,0x00,0x00,0x00,0x0e,0x00,0x03,0x00,0x00,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x0f,0x00,0x0a,0x00,0x05,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x6d,0x61,0x69,0x6e,0x00,0x00,0x00,0x00, +0x16,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x30,0x00,0x00,0x00, +0x77,0x00,0x00,0x00,0x06,0x01,0x00,0x00,0x10,0x00,0x06,0x00, +0x04,0x00,0x00,0x00,0x11,0x00,0x00,0x00,0x40,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x16,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x1a,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x1e,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x1e,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x1e,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x08,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x1e,0x00,0x00,0x00,0x03,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x1e,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x10,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x1e,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x30,0x00,0x00,0x00, +0x0b,0x00,0x00,0x00,0x1b,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x6d,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x6f,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x71,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x73,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x73,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x73,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x73,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x74,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x6e,0x00,0x00,0x00,0x48,0x00,0x04,0x00, +0x75,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x75,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, +0x75,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x77,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x77,0x00,0x00,0x00,0x21,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x03,0x01,0x00,0x00, +0x06,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x48,0x00,0x04,0x00, +0x04,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x19,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x04,0x01,0x00,0x00,0x00,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, +0x04,0x01,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x06,0x01,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x06,0x01,0x00,0x00,0x21,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x2c,0x01,0x00,0x00, +0x0b,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x13,0x00,0x02,0x00, +0x02,0x00,0x00,0x00,0x21,0x00,0x03,0x00,0x03,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x15,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x10,0x00,0x00,0x00, +0x00,0x01,0x00,0x00,0x14,0x00,0x02,0x00,0x11,0x00,0x00,0x00, +0x17,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x03,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x15,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0x15,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x17,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x1e,0x00,0x07,0x00,0x1e,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x1f,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x1e,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0x1f,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0x09,0x00,0x00,0x00,0x15,0x00,0x04,0x00,0x21,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x21,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x23,0x00,0x00,0x00,0x09,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x21,0x00,0x00,0x00, +0x26,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0x15,0x00,0x00,0x00,0x30,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x33,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x37,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x3d,0x00,0x00,0x00,0x10,0x00,0x00,0x00, +0x15,0x00,0x04,0x00,0x4d,0x00,0x00,0x00,0x08,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x15,0x00,0x04,0x00,0x55,0x00,0x00,0x00, +0x08,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x59,0x00,0x00,0x00,0x08,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0x1c,0x00,0x04,0x00,0x6d,0x00,0x00,0x00, +0x4d,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x6e,0x00,0x00,0x00,0x40,0x00,0x00,0x00, +0x1c,0x00,0x04,0x00,0x6f,0x00,0x00,0x00,0x4d,0x00,0x00,0x00, +0x6e,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x70,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x1c,0x00,0x04,0x00, +0x71,0x00,0x00,0x00,0x4d,0x00,0x00,0x00,0x70,0x00,0x00,0x00, +0x16,0x00,0x03,0x00,0x72,0x00,0x00,0x00,0x10,0x00,0x00,0x00, +0x1e,0x00,0x06,0x00,0x73,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, +0x6f,0x00,0x00,0x00,0x71,0x00,0x00,0x00,0x72,0x00,0x00,0x00, +0x1d,0x00,0x03,0x00,0x74,0x00,0x00,0x00,0x73,0x00,0x00,0x00, +0x1e,0x00,0x03,0x00,0x75,0x00,0x00,0x00,0x74,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x76,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x75,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x76,0x00,0x00,0x00, +0x77,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x21,0x00,0x00,0x00,0x79,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x7c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x4d,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x21,0x00,0x00,0x00, +0x81,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x21,0x00,0x00,0x00,0x8b,0x00,0x00,0x00,0x03,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x21,0x00,0x00,0x00,0x8d,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x21,0x00,0x00,0x00, +0xd0,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x16,0x00,0x03,0x00, +0xdb,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0xdf,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x72,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x21,0x00,0x00,0x00,0xe7,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xee,0x00,0x00,0x00,0x80,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, +0x03,0x01,0x00,0x00,0x72,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, +0x04,0x01,0x00,0x00,0x03,0x01,0x00,0x00,0x20,0x00,0x04,0x00, +0x05,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x04,0x01,0x00,0x00, +0x3b,0x00,0x04,0x00,0x05,0x01,0x00,0x00,0x06,0x01,0x00,0x00, +0x0c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x2b,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x2c,0x00,0x06,0x00, +0x14,0x00,0x00,0x00,0x2c,0x01,0x00,0x00,0x6e,0x00,0x00,0x00, +0x2b,0x01,0x00,0x00,0x2b,0x01,0x00,0x00,0x2a,0x00,0x03,0x00, +0x11,0x00,0x00,0x00,0x2f,0x01,0x00,0x00,0x29,0x00,0x03,0x00, +0x11,0x00,0x00,0x00,0x32,0x01,0x00,0x00,0x36,0x00,0x05,0x00, +0x02,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x03,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x05,0x00,0x00,0x00, +0xf7,0x00,0x03,0x00,0x2d,0x01,0x00,0x00,0x00,0x00,0x00,0x00, +0xfb,0x00,0x03,0x00,0x09,0x00,0x00,0x00,0x2e,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x2e,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0x0a,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x0a,0x00,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x35,0x01,0x00,0x00, +0x09,0x00,0x00,0x00,0x2e,0x01,0x00,0x00,0x2a,0x01,0x00,0x00, +0x0d,0x00,0x00,0x00,0xb0,0x00,0x05,0x00,0x11,0x00,0x00,0x00, +0x12,0x00,0x00,0x00,0x35,0x01,0x00,0x00,0x10,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0x0c,0x00,0x00,0x00,0x0d,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x12,0x00,0x00,0x00, +0x0b,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0x0b,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00, +0x18,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x09,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x19,0x00,0x00,0x00, +0x18,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x1a,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x10,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x1c,0x00,0x00,0x00, +0x1a,0x00,0x00,0x00,0x35,0x01,0x00,0x00,0x41,0x00,0x05,0x00, +0x23,0x00,0x00,0x00,0x24,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0x22,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x25,0x00,0x00,0x00,0x24,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x23,0x00,0x00,0x00,0x27,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0x26,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x28,0x00,0x00,0x00,0x27,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x29,0x00,0x00,0x00,0x25,0x00,0x00,0x00, +0x28,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x2a,0x00,0x00,0x00,0x29,0x00,0x00,0x00,0x10,0x00,0x00,0x00, +0xae,0x00,0x05,0x00,0x11,0x00,0x00,0x00,0x2b,0x00,0x00,0x00, +0x1c,0x00,0x00,0x00,0x2a,0x00,0x00,0x00,0xf7,0x00,0x03,0x00, +0x2d,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x2b,0x00,0x00,0x00,0x2c,0x00,0x00,0x00,0x2d,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0x2c,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x0c,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x2d,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00,0x31,0x00,0x00,0x00, +0x30,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x31,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x34,0x00,0x00,0x00, +0x32,0x00,0x00,0x00,0x33,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x38,0x00,0x00,0x00,0x34,0x00,0x00,0x00, +0x37,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x3b,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0x37,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3f,0x00,0x00,0x00, +0x3d,0x00,0x00,0x00,0x3b,0x00,0x00,0x00,0x89,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x42,0x00,0x00,0x00,0x32,0x00,0x00,0x00, +0x33,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x43,0x00,0x00,0x00,0x33,0x00,0x00,0x00,0x42,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x44,0x00,0x00,0x00, +0x3f,0x00,0x00,0x00,0x43,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x47,0x00,0x00,0x00,0x38,0x00,0x00,0x00, +0x33,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x4b,0x00,0x00,0x00,0x33,0x00,0x00,0x00,0x47,0x00,0x00,0x00, +0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x4c,0x00,0x00,0x00, +0x38,0x00,0x00,0x00,0x4b,0x00,0x00,0x00,0xc4,0x00,0x05,0x00, +0x21,0x00,0x00,0x00,0x54,0x00,0x00,0x00,0x26,0x00,0x00,0x00, +0x38,0x00,0x00,0x00,0x72,0x00,0x04,0x00,0x55,0x00,0x00,0x00, +0x56,0x00,0x00,0x00,0x54,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, +0x4d,0x00,0x00,0x00,0x57,0x00,0x00,0x00,0x56,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x5b,0x00,0x00,0x00, +0x59,0x00,0x00,0x00,0x47,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x5d,0x00,0x00,0x00,0x37,0x00,0x00,0x00, +0x4c,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x5e,0x00,0x00,0x00,0x5b,0x00,0x00,0x00,0x5d,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x60,0x00,0x00,0x00, +0x5e,0x00,0x00,0x00,0x3b,0x00,0x00,0x00,0xb0,0x00,0x05,0x00, +0x11,0x00,0x00,0x00,0x67,0x00,0x00,0x00,0x60,0x00,0x00,0x00, +0x33,0x00,0x00,0x00,0xf7,0x00,0x03,0x00,0x6b,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x67,0x00,0x00,0x00, +0x6a,0x00,0x00,0x00,0x90,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0x6a,0x00,0x00,0x00,0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x7b,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x09,0x00,0x00,0x00, +0x41,0x00,0x08,0x00,0x7c,0x00,0x00,0x00,0x7d,0x00,0x00,0x00, +0x77,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x1c,0x00,0x00,0x00, +0x79,0x00,0x00,0x00,0x7b,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x4d,0x00,0x00,0x00,0x7e,0x00,0x00,0x00,0x7d,0x00,0x00,0x00, +0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x7f,0x00,0x00,0x00, +0x7e,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x21,0x00,0x00,0x00, +0x80,0x00,0x00,0x00,0x7f,0x00,0x00,0x00,0xc7,0x00,0x05,0x00, +0x21,0x00,0x00,0x00,0x82,0x00,0x00,0x00,0x80,0x00,0x00,0x00, +0x81,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x85,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x59,0x00,0x00,0x00, +0x41,0x00,0x08,0x00,0x7c,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0x77,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x1c,0x00,0x00,0x00, +0x79,0x00,0x00,0x00,0x85,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x4d,0x00,0x00,0x00,0x87,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0xc2,0x00,0x05,0x00,0x4d,0x00,0x00,0x00,0x88,0x00,0x00,0x00, +0x87,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x71,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x89,0x00,0x00,0x00,0x88,0x00,0x00,0x00, +0x7c,0x00,0x04,0x00,0x21,0x00,0x00,0x00,0x8a,0x00,0x00,0x00, +0x89,0x00,0x00,0x00,0xc7,0x00,0x05,0x00,0x21,0x00,0x00,0x00, +0x8c,0x00,0x00,0x00,0x8a,0x00,0x00,0x00,0x8b,0x00,0x00,0x00, +0xc4,0x00,0x05,0x00,0x21,0x00,0x00,0x00,0x8e,0x00,0x00,0x00, +0x8c,0x00,0x00,0x00,0x8d,0x00,0x00,0x00,0xc5,0x00,0x05,0x00, +0x21,0x00,0x00,0x00,0x8f,0x00,0x00,0x00,0x82,0x00,0x00,0x00, +0x8e,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x6b,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0x90,0x00,0x00,0x00,0xb0,0x00,0x05,0x00, +0x11,0x00,0x00,0x00,0x92,0x00,0x00,0x00,0x60,0x00,0x00,0x00, +0x59,0x00,0x00,0x00,0xf7,0x00,0x03,0x00,0x95,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x92,0x00,0x00,0x00, +0x94,0x00,0x00,0x00,0xa9,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0x94,0x00,0x00,0x00,0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x98,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x09,0x00,0x00,0x00, +0x41,0x00,0x08,0x00,0x7c,0x00,0x00,0x00,0x99,0x00,0x00,0x00, +0x77,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x1c,0x00,0x00,0x00, +0x79,0x00,0x00,0x00,0x98,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x4d,0x00,0x00,0x00,0x9a,0x00,0x00,0x00,0x99,0x00,0x00,0x00, +0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x9b,0x00,0x00,0x00, +0x9a,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x21,0x00,0x00,0x00, +0x9c,0x00,0x00,0x00,0x9b,0x00,0x00,0x00,0xc7,0x00,0x05,0x00, +0x21,0x00,0x00,0x00,0x9d,0x00,0x00,0x00,0x9c,0x00,0x00,0x00, +0x81,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xa0,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x33,0x00,0x00,0x00, +0x41,0x00,0x08,0x00,0x7c,0x00,0x00,0x00,0xa1,0x00,0x00,0x00, +0x77,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x1c,0x00,0x00,0x00, +0x79,0x00,0x00,0x00,0xa0,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x4d,0x00,0x00,0x00,0xa2,0x00,0x00,0x00,0xa1,0x00,0x00,0x00, +0xc2,0x00,0x05,0x00,0x4d,0x00,0x00,0x00,0xa3,0x00,0x00,0x00, +0xa2,0x00,0x00,0x00,0x79,0x00,0x00,0x00,0x71,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xa4,0x00,0x00,0x00,0xa3,0x00,0x00,0x00, +0x7c,0x00,0x04,0x00,0x21,0x00,0x00,0x00,0xa5,0x00,0x00,0x00, +0xa4,0x00,0x00,0x00,0xc7,0x00,0x05,0x00,0x21,0x00,0x00,0x00, +0xa6,0x00,0x00,0x00,0xa5,0x00,0x00,0x00,0x8b,0x00,0x00,0x00, +0xc4,0x00,0x05,0x00,0x21,0x00,0x00,0x00,0xa7,0x00,0x00,0x00, +0xa6,0x00,0x00,0x00,0x8d,0x00,0x00,0x00,0xc5,0x00,0x05,0x00, +0x21,0x00,0x00,0x00,0xa8,0x00,0x00,0x00,0x9d,0x00,0x00,0x00, +0xa7,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x95,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xa9,0x00,0x00,0x00,0xb0,0x00,0x05,0x00, +0x11,0x00,0x00,0x00,0xab,0x00,0x00,0x00,0x60,0x00,0x00,0x00, +0x70,0x00,0x00,0x00,0xf7,0x00,0x03,0x00,0xae,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xab,0x00,0x00,0x00, +0xad,0x00,0x00,0x00,0xc2,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xad,0x00,0x00,0x00,0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xb1,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x59,0x00,0x00,0x00, +0x41,0x00,0x08,0x00,0x7c,0x00,0x00,0x00,0xb2,0x00,0x00,0x00, +0x77,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x1c,0x00,0x00,0x00, +0x79,0x00,0x00,0x00,0xb1,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x4d,0x00,0x00,0x00,0xb3,0x00,0x00,0x00,0xb2,0x00,0x00,0x00, +0xc2,0x00,0x05,0x00,0x4d,0x00,0x00,0x00,0xb4,0x00,0x00,0x00, +0xb3,0x00,0x00,0x00,0x8d,0x00,0x00,0x00,0x71,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xb5,0x00,0x00,0x00,0xb4,0x00,0x00,0x00, +0x7c,0x00,0x04,0x00,0x21,0x00,0x00,0x00,0xb6,0x00,0x00,0x00, +0xb5,0x00,0x00,0x00,0x41,0x00,0x08,0x00,0x7c,0x00,0x00,0x00, +0xba,0x00,0x00,0x00,0x77,0x00,0x00,0x00,0x22,0x00,0x00,0x00, +0x1c,0x00,0x00,0x00,0x79,0x00,0x00,0x00,0x60,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00,0xbb,0x00,0x00,0x00, +0xba,0x00,0x00,0x00,0xc2,0x00,0x05,0x00,0x4d,0x00,0x00,0x00, +0xbc,0x00,0x00,0x00,0xbb,0x00,0x00,0x00,0x8d,0x00,0x00,0x00, +0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xbd,0x00,0x00,0x00, +0xbc,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x21,0x00,0x00,0x00, +0xbe,0x00,0x00,0x00,0xbd,0x00,0x00,0x00,0xc7,0x00,0x05,0x00, +0x21,0x00,0x00,0x00,0xbf,0x00,0x00,0x00,0xbe,0x00,0x00,0x00, +0x8b,0x00,0x00,0x00,0xc4,0x00,0x05,0x00,0x21,0x00,0x00,0x00, +0xc0,0x00,0x00,0x00,0xbf,0x00,0x00,0x00,0x8d,0x00,0x00,0x00, +0xc5,0x00,0x05,0x00,0x21,0x00,0x00,0x00,0xc1,0x00,0x00,0x00, +0xb6,0x00,0x00,0x00,0xc0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xae,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xc2,0x00,0x00,0x00, +0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xc5,0x00,0x00,0x00, +0x60,0x00,0x00,0x00,0x59,0x00,0x00,0x00,0x41,0x00,0x08,0x00, +0x7c,0x00,0x00,0x00,0xc6,0x00,0x00,0x00,0x77,0x00,0x00,0x00, +0x22,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x79,0x00,0x00,0x00, +0xc5,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, +0xc7,0x00,0x00,0x00,0xc6,0x00,0x00,0x00,0xc2,0x00,0x05,0x00, +0x4d,0x00,0x00,0x00,0xc8,0x00,0x00,0x00,0xc7,0x00,0x00,0x00, +0x8d,0x00,0x00,0x00,0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xc9,0x00,0x00,0x00,0xc8,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, +0x21,0x00,0x00,0x00,0xca,0x00,0x00,0x00,0xc9,0x00,0x00,0x00, +0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xcd,0x00,0x00,0x00, +0x60,0x00,0x00,0x00,0x33,0x00,0x00,0x00,0x41,0x00,0x08,0x00, +0x7c,0x00,0x00,0x00,0xce,0x00,0x00,0x00,0x77,0x00,0x00,0x00, +0x22,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x79,0x00,0x00,0x00, +0xcd,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, +0xcf,0x00,0x00,0x00,0xce,0x00,0x00,0x00,0xc2,0x00,0x05,0x00, +0x4d,0x00,0x00,0x00,0xd1,0x00,0x00,0x00,0xcf,0x00,0x00,0x00, +0xd0,0x00,0x00,0x00,0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xd2,0x00,0x00,0x00,0xd1,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, +0x21,0x00,0x00,0x00,0xd3,0x00,0x00,0x00,0xd2,0x00,0x00,0x00, +0xc7,0x00,0x05,0x00,0x21,0x00,0x00,0x00,0xd4,0x00,0x00,0x00, +0xd3,0x00,0x00,0x00,0x8b,0x00,0x00,0x00,0xc4,0x00,0x05,0x00, +0x21,0x00,0x00,0x00,0xd5,0x00,0x00,0x00,0xd4,0x00,0x00,0x00, +0x8d,0x00,0x00,0x00,0xc5,0x00,0x05,0x00,0x21,0x00,0x00,0x00, +0xd6,0x00,0x00,0x00,0xca,0x00,0x00,0x00,0xd5,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xae,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xae,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x21,0x00,0x00,0x00, +0x36,0x01,0x00,0x00,0xc1,0x00,0x00,0x00,0xad,0x00,0x00,0x00, +0xd6,0x00,0x00,0x00,0xc2,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x95,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x95,0x00,0x00,0x00, +0xf5,0x00,0x07,0x00,0x21,0x00,0x00,0x00,0x37,0x01,0x00,0x00, +0xa8,0x00,0x00,0x00,0x94,0x00,0x00,0x00,0x36,0x01,0x00,0x00, +0xae,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x6b,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0x6b,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, +0x21,0x00,0x00,0x00,0x38,0x01,0x00,0x00,0x8f,0x00,0x00,0x00, +0x6a,0x00,0x00,0x00,0x37,0x01,0x00,0x00,0x95,0x00,0x00,0x00, +0x72,0x00,0x04,0x00,0x55,0x00,0x00,0x00,0xda,0x00,0x00,0x00, +0x38,0x01,0x00,0x00,0x41,0x00,0x07,0x00,0xdf,0x00,0x00,0x00, +0xe0,0x00,0x00,0x00,0x77,0x00,0x00,0x00,0x22,0x00,0x00,0x00, +0x1c,0x00,0x00,0x00,0x8b,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x72,0x00,0x00,0x00,0xe1,0x00,0x00,0x00,0xe0,0x00,0x00,0x00, +0x73,0x00,0x04,0x00,0xdb,0x00,0x00,0x00,0xe2,0x00,0x00,0x00, +0xe1,0x00,0x00,0x00,0x72,0x00,0x04,0x00,0x21,0x00,0x00,0x00, +0xe6,0x00,0x00,0x00,0xda,0x00,0x00,0x00,0x82,0x00,0x05,0x00, +0x21,0x00,0x00,0x00,0xe8,0x00,0x00,0x00,0xe6,0x00,0x00,0x00, +0xe7,0x00,0x00,0x00,0x6f,0x00,0x04,0x00,0xdb,0x00,0x00,0x00, +0xe9,0x00,0x00,0x00,0xe8,0x00,0x00,0x00,0x85,0x00,0x05,0x00, +0xdb,0x00,0x00,0x00,0xea,0x00,0x00,0x00,0xe2,0x00,0x00,0x00, +0xe9,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xed,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x10,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xf0,0x00,0x00,0x00, +0xee,0x00,0x00,0x00,0x47,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xf1,0x00,0x00,0x00,0xed,0x00,0x00,0x00, +0xf0,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xf3,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x4c,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xf4,0x00,0x00,0x00, +0xf1,0x00,0x00,0x00,0xf3,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xf7,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, +0x47,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xfa,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xfa,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0x39,0x01,0x00,0x00,0x44,0x00,0x00,0x00, +0x6b,0x00,0x00,0x00,0x28,0x01,0x00,0x00,0xfb,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x01,0x01,0x00,0x00, +0x44,0x00,0x00,0x00,0x33,0x00,0x00,0x00,0xb0,0x00,0x05,0x00, +0x11,0x00,0x00,0x00,0x02,0x01,0x00,0x00,0x39,0x01,0x00,0x00, +0x01,0x01,0x00,0x00,0xf6,0x00,0x04,0x00,0xfc,0x00,0x00,0x00, +0xfb,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x02,0x01,0x00,0x00,0xfb,0x00,0x00,0x00,0xfc,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xfb,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x09,0x01,0x00,0x00,0xf4,0x00,0x00,0x00, +0x39,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x0e,0x01,0x00,0x00,0xf7,0x00,0x00,0x00,0x39,0x01,0x00,0x00, +0x41,0x00,0x08,0x00,0x7c,0x00,0x00,0x00,0x0f,0x01,0x00,0x00, +0x77,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x1c,0x00,0x00,0x00, +0x26,0x00,0x00,0x00,0x0e,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, +0x4d,0x00,0x00,0x00,0x10,0x01,0x00,0x00,0x0f,0x01,0x00,0x00, +0xc2,0x00,0x05,0x00,0x4d,0x00,0x00,0x00,0x12,0x01,0x00,0x00, +0x10,0x01,0x00,0x00,0x5d,0x00,0x00,0x00,0x71,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x13,0x01,0x00,0x00,0x12,0x01,0x00,0x00, +0x7c,0x00,0x04,0x00,0x21,0x00,0x00,0x00,0x14,0x01,0x00,0x00, +0x13,0x01,0x00,0x00,0xc7,0x00,0x05,0x00,0x21,0x00,0x00,0x00, +0x15,0x01,0x00,0x00,0x14,0x01,0x00,0x00,0x8b,0x00,0x00,0x00, +0x72,0x00,0x04,0x00,0x55,0x00,0x00,0x00,0x16,0x01,0x00,0x00, +0x15,0x01,0x00,0x00,0x72,0x00,0x04,0x00,0x21,0x00,0x00,0x00, +0x17,0x01,0x00,0x00,0x16,0x01,0x00,0x00,0x41,0x00,0x08,0x00, +0x7c,0x00,0x00,0x00,0x1a,0x01,0x00,0x00,0x77,0x00,0x00,0x00, +0x22,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x22,0x00,0x00,0x00, +0x39,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, +0x1b,0x01,0x00,0x00,0x1a,0x01,0x00,0x00,0xc7,0x00,0x05,0x00, +0x4d,0x00,0x00,0x00,0x1d,0x01,0x00,0x00,0x1b,0x01,0x00,0x00, +0x57,0x00,0x00,0x00,0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x1e,0x01,0x00,0x00,0x1d,0x01,0x00,0x00,0x7c,0x00,0x04,0x00, +0x21,0x00,0x00,0x00,0x1f,0x01,0x00,0x00,0x1e,0x01,0x00,0x00, +0xab,0x00,0x05,0x00,0x11,0x00,0x00,0x00,0x20,0x01,0x00,0x00, +0x1f,0x01,0x00,0x00,0x22,0x00,0x00,0x00,0xa9,0x00,0x06,0x00, +0x21,0x00,0x00,0x00,0x21,0x01,0x00,0x00,0x20,0x01,0x00,0x00, +0x22,0x00,0x00,0x00,0x8d,0x00,0x00,0x00,0x82,0x00,0x05,0x00, +0x21,0x00,0x00,0x00,0x22,0x01,0x00,0x00,0x17,0x01,0x00,0x00, +0x21,0x01,0x00,0x00,0x6f,0x00,0x04,0x00,0xdb,0x00,0x00,0x00, +0x23,0x01,0x00,0x00,0x22,0x01,0x00,0x00,0x85,0x00,0x05,0x00, +0xdb,0x00,0x00,0x00,0x24,0x01,0x00,0x00,0xea,0x00,0x00,0x00, +0x23,0x01,0x00,0x00,0x73,0x00,0x04,0x00,0x72,0x00,0x00,0x00, +0x25,0x01,0x00,0x00,0x24,0x01,0x00,0x00,0x41,0x00,0x06,0x00, +0xdf,0x00,0x00,0x00,0x26,0x01,0x00,0x00,0x06,0x01,0x00,0x00, +0x22,0x00,0x00,0x00,0x09,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, +0x26,0x01,0x00,0x00,0x25,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x28,0x01,0x00,0x00,0x39,0x01,0x00,0x00, +0x26,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xfa,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xfc,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x0d,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x0d,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x2a,0x01,0x00,0x00, +0x35,0x01,0x00,0x00,0x26,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x0a,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x0c,0x00,0x00,0x00, +0xf5,0x00,0x07,0x00,0x11,0x00,0x00,0x00,0x3e,0x01,0x00,0x00, +0x2f,0x01,0x00,0x00,0x0a,0x00,0x00,0x00,0x32,0x01,0x00,0x00, +0x2c,0x00,0x00,0x00,0xf7,0x00,0x03,0x00,0x33,0x01,0x00,0x00, +0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x3e,0x01,0x00,0x00, +0x2d,0x01,0x00,0x00,0x33,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x33,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x2d,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x2d,0x01,0x00,0x00,0xfd,0x00,0x01,0x00, +0x38,0x00,0x01,0x00, +}; +const uint64_t dequant_q3_K_len = 4804; + +unsigned char dequant_q4_0_data[] = { +0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, +0x83,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, +0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x27,0x00,0x00,0x00, +0x11,0x00,0x02,0x00,0x51,0x11,0x00,0x00,0x11,0x00,0x02,0x00, +0x60,0x11,0x00,0x00,0x0b,0x00,0x06,0x00,0x01,0x00,0x00,0x00, +0x47,0x4c,0x53,0x4c,0x2e,0x73,0x74,0x64,0x2e,0x34,0x35,0x30, +0x00,0x00,0x00,0x00,0x0e,0x00,0x03,0x00,0x00,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x0f,0x00,0x0a,0x00,0x05,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x6d,0x61,0x69,0x6e,0x00,0x00,0x00,0x00, +0x0b,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x2b,0x00,0x00,0x00, +0x4d,0x00,0x00,0x00,0x66,0x00,0x00,0x00,0x10,0x00,0x06,0x00, +0x04,0x00,0x00,0x00,0x11,0x00,0x00,0x00,0x00,0x01,0x00,0x00, +0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x0b,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x1a,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x12,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, +0x1b,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x29,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x29,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x29,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x08,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x29,0x00,0x00,0x00, +0x03,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x29,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x47,0x00,0x03,0x00, +0x29,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x48,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x49,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x49,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x48,0x00,0x04,0x00, +0x4b,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x4b,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, +0x4b,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x4d,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x4d,0x00,0x00,0x00,0x21,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x63,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x48,0x00,0x04,0x00, +0x64,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x19,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x64,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, +0x64,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x66,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x66,0x00,0x00,0x00,0x21,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x94,0x00,0x00,0x00, 0x0b,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x13,0x00,0x02,0x00, 0x02,0x00,0x00,0x00,0x21,0x00,0x03,0x00,0x03,0x00,0x00,0x00, 0x02,0x00,0x00,0x00,0x15,0x00,0x04,0x00,0x06,0x00,0x00,0x00, @@ -173,3186 +2877,389 @@ unsigned char clamp_f32_data[] = { 0x0b,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, 0x06,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,0x00,0x00,0x00, 0x20,0x00,0x04,0x00,0x0d,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x16,0x00,0x03,0x00,0x11,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x1e,0x00,0x06,0x00,0x12,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x11,0x00,0x00,0x00, -0x11,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x13,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x15,0x00,0x04,0x00,0x15,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x15,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x17,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x14,0x00,0x02,0x00,0x1a,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, -0x21,0x00,0x00,0x00,0x11,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, -0x22,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x23,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x22,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x23,0x00,0x00,0x00,0x24,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x26,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x11,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, -0x29,0x00,0x00,0x00,0x11,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, -0x2a,0x00,0x00,0x00,0x29,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x2b,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x2a,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x2b,0x00,0x00,0x00,0x2c,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x15,0x00,0x00,0x00, -0x2f,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x30,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x11,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x15,0x00,0x00,0x00,0x3b,0x00,0x00,0x00, -0x03,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x49,0x00,0x00,0x00,0x00,0x02,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x4a,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x2c,0x00,0x06,0x00,0x09,0x00,0x00,0x00,0x4b,0x00,0x00,0x00, -0x49,0x00,0x00,0x00,0x4a,0x00,0x00,0x00,0x4a,0x00,0x00,0x00, -0x36,0x00,0x05,0x00,0x02,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x05,0x00,0x00,0x00,0xf7,0x00,0x03,0x00,0x4c,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0xfb,0x00,0x03,0x00,0x0c,0x00,0x00,0x00, -0x4d,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x4d,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00,0x0e,0x00,0x00,0x00, -0x0b,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x0e,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00,0x18,0x00,0x00,0x00, -0x14,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x18,0x00,0x00,0x00, -0xae,0x00,0x05,0x00,0x1a,0x00,0x00,0x00,0x1b,0x00,0x00,0x00, -0x0f,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0xf7,0x00,0x03,0x00, -0x1d,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x1b,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x1d,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0x1c,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x4c,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x1d,0x00,0x00,0x00, -0x41,0x00,0x06,0x00,0x26,0x00,0x00,0x00,0x27,0x00,0x00,0x00, -0x24,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x11,0x00,0x00,0x00,0x28,0x00,0x00,0x00, -0x27,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x30,0x00,0x00,0x00, -0x31,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x2f,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x11,0x00,0x00,0x00,0x32,0x00,0x00,0x00, -0x31,0x00,0x00,0x00,0xb8,0x00,0x05,0x00,0x1a,0x00,0x00,0x00, -0x33,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0x32,0x00,0x00,0x00, -0xf7,0x00,0x03,0x00,0x36,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x33,0x00,0x00,0x00,0x35,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x35,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x36,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x39,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x30,0x00,0x00,0x00, -0x3c,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x3b,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x11,0x00,0x00,0x00,0x3d,0x00,0x00,0x00, -0x3c,0x00,0x00,0x00,0xba,0x00,0x05,0x00,0x1a,0x00,0x00,0x00, -0x3e,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0x3d,0x00,0x00,0x00, -0xa9,0x00,0x06,0x00,0x11,0x00,0x00,0x00,0x54,0x00,0x00,0x00, -0x3e,0x00,0x00,0x00,0x3d,0x00,0x00,0x00,0x28,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x36,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x36,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x11,0x00,0x00,0x00, -0x53,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x35,0x00,0x00,0x00, -0x54,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x41,0x00,0x06,0x00, -0x26,0x00,0x00,0x00,0x48,0x00,0x00,0x00,0x2c,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x3e,0x00,0x03,0x00, -0x48,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x4c,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x4c,0x00,0x00,0x00, -0xfd,0x00,0x01,0x00,0x38,0x00,0x01,0x00, -}; -const uint64_t clamp_f32_len = 1448; - -unsigned char cpy_f16_f16_data[] = { -0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, -0xaa,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, -0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x51,0x11,0x00,0x00, -0x0b,0x00,0x06,0x00,0x01,0x00,0x00,0x00,0x47,0x4c,0x53,0x4c, -0x2e,0x73,0x74,0x64,0x2e,0x34,0x35,0x30,0x00,0x00,0x00,0x00, -0x0e,0x00,0x03,0x00,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x0f,0x00,0x09,0x00,0x05,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x6d,0x61,0x69,0x6e,0x00,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x10,0x00,0x00,0x00,0x96,0x00,0x00,0x00,0x9f,0x00,0x00,0x00, -0x10,0x00,0x06,0x00,0x04,0x00,0x00,0x00,0x11,0x00,0x00,0x00, -0x00,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x1c,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x0e,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x0e,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x0e,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x08,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x0e,0x00,0x00,0x00, -0x03,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x0e,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x0e,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x14,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x0e,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x18,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x0e,0x00,0x00,0x00,0x07,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x0e,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x0e,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x24,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x0e,0x00,0x00,0x00,0x0a,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x0e,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x2c,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x0e,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x93,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x48,0x00,0x04,0x00, -0x94,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x19,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x94,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, -0x94,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x96,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x96,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x9c,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x48,0x00,0x04,0x00, -0x9d,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x9d,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, -0x9d,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x9f,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x9f,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xa7,0x00,0x00,0x00, -0x0b,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x13,0x00,0x02,0x00, -0x02,0x00,0x00,0x00,0x21,0x00,0x03,0x00,0x03,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x15,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x17,0x00,0x04,0x00, -0x07,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x03,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x08,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x07,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x08,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x0a,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x0b,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x1e,0x00,0x0e,0x00,0x0e,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x10,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0x0a,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x15,0x00,0x00,0x00, +0x40,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x1e,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x1e,0x00,0x07,0x00, +0x29,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, 0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x0f,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x0e,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x0f,0x00,0x00,0x00, -0x10,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x15,0x00,0x04,0x00, -0x11,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x11,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x13,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x2a,0x00,0x00,0x00,0x09,0x00,0x00,0x00, +0x29,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x2a,0x00,0x00,0x00, +0x2b,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x15,0x00,0x04,0x00, +0x2c,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x2c,0x00,0x00,0x00,0x2d,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x2e,0x00,0x00,0x00, 0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x14,0x00,0x02,0x00, -0x16,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x11,0x00,0x00,0x00, -0x1f,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x11,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x11,0x00,0x00,0x00,0x47,0x00,0x00,0x00, -0x03,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x11,0x00,0x00,0x00, -0x4c,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x11,0x00,0x00,0x00,0x52,0x00,0x00,0x00,0x05,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x11,0x00,0x00,0x00,0x5a,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x11,0x00,0x00,0x00, -0x5d,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x11,0x00,0x00,0x00,0x82,0x00,0x00,0x00,0x08,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x11,0x00,0x00,0x00,0x87,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x11,0x00,0x00,0x00, -0x8d,0x00,0x00,0x00,0x0a,0x00,0x00,0x00,0x16,0x00,0x03,0x00, -0x92,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, -0x93,0x00,0x00,0x00,0x92,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, -0x94,0x00,0x00,0x00,0x93,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x95,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x94,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x95,0x00,0x00,0x00,0x96,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x11,0x00,0x00,0x00, -0x97,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, -0x9c,0x00,0x00,0x00,0x92,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, -0x9d,0x00,0x00,0x00,0x9c,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x9e,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x9d,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x9e,0x00,0x00,0x00,0x9f,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0xa1,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x92,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xa5,0x00,0x00,0x00,0x00,0x02,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xa6,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x2c,0x00,0x06,0x00,0x07,0x00,0x00,0x00, -0xa7,0x00,0x00,0x00,0xa5,0x00,0x00,0x00,0xa6,0x00,0x00,0x00, -0xa6,0x00,0x00,0x00,0x36,0x00,0x05,0x00,0x02,0x00,0x00,0x00, +0x32,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x38,0x00,0x00,0x00,0x00,0x04,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x3e,0x00,0x00,0x00,0x08,0x00,0x00,0x00, +0x16,0x00,0x03,0x00,0x42,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0x16,0x00,0x03,0x00,0x45,0x00,0x00,0x00,0x10,0x00,0x00,0x00, +0x15,0x00,0x04,0x00,0x46,0x00,0x00,0x00,0x08,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x47,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x1c,0x00,0x04,0x00, +0x48,0x00,0x00,0x00,0x46,0x00,0x00,0x00,0x47,0x00,0x00,0x00, +0x1e,0x00,0x04,0x00,0x49,0x00,0x00,0x00,0x45,0x00,0x00,0x00, +0x48,0x00,0x00,0x00,0x1d,0x00,0x03,0x00,0x4a,0x00,0x00,0x00, +0x49,0x00,0x00,0x00,0x1e,0x00,0x03,0x00,0x4b,0x00,0x00,0x00, +0x4a,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x4c,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x4b,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0x4c,0x00,0x00,0x00,0x4d,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x2c,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x50,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x45,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x42,0x00,0x00,0x00,0x55,0x00,0x00,0x00,0x00,0x00,0x00,0xc1, +0x1d,0x00,0x03,0x00,0x63,0x00,0x00,0x00,0x45,0x00,0x00,0x00, +0x1e,0x00,0x03,0x00,0x64,0x00,0x00,0x00,0x63,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x65,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x64,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x65,0x00,0x00,0x00, +0x66,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x2c,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x71,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x46,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x2c,0x00,0x00,0x00, +0x76,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x92,0x00,0x00,0x00,0x00,0x01,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x93,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x2c,0x00,0x06,0x00,0x09,0x00,0x00,0x00, +0x94,0x00,0x00,0x00,0x92,0x00,0x00,0x00,0x93,0x00,0x00,0x00, +0x93,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x77,0x01,0x00,0x00,0x11,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x78,0x01,0x00,0x00,0x02,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x79,0x01,0x00,0x00, +0x12,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x7a,0x01,0x00,0x00,0x03,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x7b,0x01,0x00,0x00,0x13,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x7c,0x01,0x00,0x00, +0x14,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x7d,0x01,0x00,0x00,0x05,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x7e,0x01,0x00,0x00,0x15,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x7f,0x01,0x00,0x00, +0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x80,0x01,0x00,0x00,0x16,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x81,0x01,0x00,0x00,0x07,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x82,0x01,0x00,0x00, +0x17,0x00,0x00,0x00,0x36,0x00,0x05,0x00,0x02,0x00,0x00,0x00, 0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x03,0x00,0x00,0x00, 0xf8,0x00,0x02,0x00,0x05,0x00,0x00,0x00,0xf7,0x00,0x03,0x00, -0xa8,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xfb,0x00,0x03,0x00, -0x0a,0x00,0x00,0x00,0xa9,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xa9,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x0b,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x0a,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x0d,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x13,0x00,0x00,0x00, -0x14,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x15,0x00,0x00,0x00, -0x14,0x00,0x00,0x00,0xae,0x00,0x05,0x00,0x16,0x00,0x00,0x00, -0x17,0x00,0x00,0x00,0x0d,0x00,0x00,0x00,0x15,0x00,0x00,0x00, -0xf7,0x00,0x03,0x00,0x19,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x17,0x00,0x00,0x00,0x18,0x00,0x00,0x00, -0x19,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x18,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xa8,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x19,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x13,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x1f,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x13,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x22,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x24,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x25,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x24,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x26,0x00,0x00,0x00, -0x0d,0x00,0x00,0x00,0x25,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x2d,0x00,0x00,0x00,0x26,0x00,0x00,0x00, -0x24,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x30,0x00,0x00,0x00,0x2d,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x31,0x00,0x00,0x00, -0x0d,0x00,0x00,0x00,0x30,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0x31,0x00,0x00,0x00, -0x21,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x43,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x44,0x00,0x00,0x00, -0x31,0x00,0x00,0x00,0x43,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x13,0x00,0x00,0x00,0x48,0x00,0x00,0x00,0x10,0x00,0x00,0x00, -0x47,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x49,0x00,0x00,0x00,0x48,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x4a,0x00,0x00,0x00,0x44,0x00,0x00,0x00, -0x49,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x13,0x00,0x00,0x00, -0x4d,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x4c,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, -0x4d,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x4f,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x50,0x00,0x00,0x00, -0x4a,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x13,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x10,0x00,0x00,0x00, -0x52,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x54,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x55,0x00,0x00,0x00,0x26,0x00,0x00,0x00, -0x54,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x56,0x00,0x00,0x00,0x50,0x00,0x00,0x00,0x55,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x13,0x00,0x00,0x00,0x5b,0x00,0x00,0x00, -0x10,0x00,0x00,0x00,0x5a,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x5c,0x00,0x00,0x00,0x5b,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x13,0x00,0x00,0x00,0x5e,0x00,0x00,0x00, -0x10,0x00,0x00,0x00,0x5d,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x5f,0x00,0x00,0x00,0x5e,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x60,0x00,0x00,0x00, -0x5c,0x00,0x00,0x00,0x5f,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x61,0x00,0x00,0x00,0x0d,0x00,0x00,0x00, -0x60,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x68,0x00,0x00,0x00,0x61,0x00,0x00,0x00,0x5f,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x6b,0x00,0x00,0x00, -0x68,0x00,0x00,0x00,0x5c,0x00,0x00,0x00,0x82,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x0d,0x00,0x00,0x00, -0x6b,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x6f,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x5c,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x7e,0x00,0x00,0x00, -0x6f,0x00,0x00,0x00,0x5c,0x00,0x00,0x00,0x82,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x7f,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x7e,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x13,0x00,0x00,0x00, -0x83,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x82,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0x83,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x85,0x00,0x00,0x00,0x7f,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x13,0x00,0x00,0x00,0x88,0x00,0x00,0x00, -0x10,0x00,0x00,0x00,0x87,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x89,0x00,0x00,0x00,0x88,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8a,0x00,0x00,0x00, -0x6f,0x00,0x00,0x00,0x89,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x8b,0x00,0x00,0x00,0x85,0x00,0x00,0x00, -0x8a,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x13,0x00,0x00,0x00, -0x8e,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x8d,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x8f,0x00,0x00,0x00, -0x8e,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x90,0x00,0x00,0x00,0x61,0x00,0x00,0x00,0x8f,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x91,0x00,0x00,0x00, -0x8b,0x00,0x00,0x00,0x90,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x13,0x00,0x00,0x00,0x98,0x00,0x00,0x00,0x10,0x00,0x00,0x00, -0x97,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x99,0x00,0x00,0x00,0x98,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x9b,0x00,0x00,0x00,0x99,0x00,0x00,0x00, -0x91,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0xa1,0x00,0x00,0x00, -0xa2,0x00,0x00,0x00,0x9f,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x56,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x92,0x00,0x00,0x00, -0xa3,0x00,0x00,0x00,0xa2,0x00,0x00,0x00,0x41,0x00,0x06,0x00, -0xa1,0x00,0x00,0x00,0xa4,0x00,0x00,0x00,0x96,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x9b,0x00,0x00,0x00,0x3e,0x00,0x03,0x00, -0xa4,0x00,0x00,0x00,0xa3,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xa8,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xa8,0x00,0x00,0x00, -0xfd,0x00,0x01,0x00,0x38,0x00,0x01,0x00, -}; -const uint64_t cpy_f16_f16_len = 2480; - -unsigned char cpy_f32_f16_data[] = { -0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, -0xad,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, -0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x51,0x11,0x00,0x00, -0x0b,0x00,0x06,0x00,0x01,0x00,0x00,0x00,0x47,0x4c,0x53,0x4c, -0x2e,0x73,0x74,0x64,0x2e,0x34,0x35,0x30,0x00,0x00,0x00,0x00, -0x0e,0x00,0x03,0x00,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x0f,0x00,0x09,0x00,0x05,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x6d,0x61,0x69,0x6e,0x00,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x10,0x00,0x00,0x00,0x96,0x00,0x00,0x00,0xa0,0x00,0x00,0x00, -0x10,0x00,0x06,0x00,0x04,0x00,0x00,0x00,0x11,0x00,0x00,0x00, -0x00,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x1c,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x0e,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x0e,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x0e,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x08,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x0e,0x00,0x00,0x00, -0x03,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x0e,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x0e,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x14,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x0e,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x18,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x0e,0x00,0x00,0x00,0x07,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x0e,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x0e,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x24,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x0e,0x00,0x00,0x00,0x0a,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x0e,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x2c,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x0e,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x93,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x48,0x00,0x04,0x00, -0x94,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x19,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x94,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, -0x94,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x96,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x96,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x9d,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00, -0x9e,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x9e,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, -0x9e,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0xa0,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0xa0,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xaa,0x00,0x00,0x00, -0x0b,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x13,0x00,0x02,0x00, -0x02,0x00,0x00,0x00,0x21,0x00,0x03,0x00,0x03,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x15,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x17,0x00,0x04,0x00, -0x07,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x03,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x08,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x07,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x08,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x0a,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x0b,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x1e,0x00,0x0e,0x00,0x0e,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x0f,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x0e,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x0f,0x00,0x00,0x00, -0x10,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x15,0x00,0x04,0x00, -0x11,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x11,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x13,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x14,0x00,0x02,0x00, -0x16,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x11,0x00,0x00,0x00, -0x1f,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x11,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x11,0x00,0x00,0x00,0x47,0x00,0x00,0x00, -0x03,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x11,0x00,0x00,0x00, -0x4c,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x11,0x00,0x00,0x00,0x52,0x00,0x00,0x00,0x05,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x11,0x00,0x00,0x00,0x5a,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x11,0x00,0x00,0x00, -0x5d,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x11,0x00,0x00,0x00,0x82,0x00,0x00,0x00,0x08,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x11,0x00,0x00,0x00,0x87,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x11,0x00,0x00,0x00, -0x8d,0x00,0x00,0x00,0x0a,0x00,0x00,0x00,0x16,0x00,0x03,0x00, -0x92,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, -0x93,0x00,0x00,0x00,0x92,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, -0x94,0x00,0x00,0x00,0x93,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x95,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x94,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x95,0x00,0x00,0x00,0x96,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x11,0x00,0x00,0x00, -0x97,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x16,0x00,0x03,0x00, -0x9c,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, -0x9d,0x00,0x00,0x00,0x9c,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, -0x9e,0x00,0x00,0x00,0x9d,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x9f,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x9e,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x9f,0x00,0x00,0x00,0xa0,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0xa2,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x9c,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0xa6,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x92,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xa8,0x00,0x00,0x00, -0x00,0x02,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xa9,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x2c,0x00,0x06,0x00, -0x07,0x00,0x00,0x00,0xaa,0x00,0x00,0x00,0xa8,0x00,0x00,0x00, -0xa9,0x00,0x00,0x00,0xa9,0x00,0x00,0x00,0x36,0x00,0x05,0x00, -0x02,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x03,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x05,0x00,0x00,0x00, -0xf7,0x00,0x03,0x00,0xab,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0xfb,0x00,0x03,0x00,0x0a,0x00,0x00,0x00,0xac,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xac,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x0b,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x0a,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x0d,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x13,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x10,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x15,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0xae,0x00,0x05,0x00, -0x16,0x00,0x00,0x00,0x17,0x00,0x00,0x00,0x0d,0x00,0x00,0x00, -0x15,0x00,0x00,0x00,0xf7,0x00,0x03,0x00,0x19,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x17,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x18,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xab,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0x19,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x13,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x10,0x00,0x00,0x00, -0x1f,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x21,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x13,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x10,0x00,0x00,0x00, -0x22,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x24,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x25,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x24,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x26,0x00,0x00,0x00,0x0d,0x00,0x00,0x00,0x25,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x2d,0x00,0x00,0x00, -0x26,0x00,0x00,0x00,0x24,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x30,0x00,0x00,0x00,0x2d,0x00,0x00,0x00, -0x21,0x00,0x00,0x00,0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x31,0x00,0x00,0x00,0x0d,0x00,0x00,0x00,0x30,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x34,0x00,0x00,0x00, -0x31,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x43,0x00,0x00,0x00,0x34,0x00,0x00,0x00, -0x21,0x00,0x00,0x00,0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x44,0x00,0x00,0x00,0x31,0x00,0x00,0x00,0x43,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x13,0x00,0x00,0x00,0x48,0x00,0x00,0x00, -0x10,0x00,0x00,0x00,0x47,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x49,0x00,0x00,0x00,0x48,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x4a,0x00,0x00,0x00, -0x44,0x00,0x00,0x00,0x49,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x13,0x00,0x00,0x00,0x4d,0x00,0x00,0x00,0x10,0x00,0x00,0x00, -0x4c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x4e,0x00,0x00,0x00,0x4d,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x34,0x00,0x00,0x00, -0x4e,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x50,0x00,0x00,0x00,0x4a,0x00,0x00,0x00,0x4f,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x13,0x00,0x00,0x00,0x53,0x00,0x00,0x00, -0x10,0x00,0x00,0x00,0x52,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x54,0x00,0x00,0x00,0x53,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x55,0x00,0x00,0x00, -0x26,0x00,0x00,0x00,0x54,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x56,0x00,0x00,0x00,0x50,0x00,0x00,0x00, -0x55,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x13,0x00,0x00,0x00, -0x5b,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x5a,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x5c,0x00,0x00,0x00, -0x5b,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x13,0x00,0x00,0x00, -0x5e,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x5d,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x5f,0x00,0x00,0x00, -0x5e,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x60,0x00,0x00,0x00,0x5c,0x00,0x00,0x00,0x5f,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x61,0x00,0x00,0x00, -0x0d,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x68,0x00,0x00,0x00,0x61,0x00,0x00,0x00, -0x5f,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x6b,0x00,0x00,0x00,0x68,0x00,0x00,0x00,0x5c,0x00,0x00,0x00, -0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x0d,0x00,0x00,0x00,0x6b,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x6f,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x5c,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x7e,0x00,0x00,0x00,0x6f,0x00,0x00,0x00,0x5c,0x00,0x00,0x00, -0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x7f,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x7e,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x13,0x00,0x00,0x00,0x83,0x00,0x00,0x00,0x10,0x00,0x00,0x00, -0x82,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0x83,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x85,0x00,0x00,0x00,0x7f,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x13,0x00,0x00,0x00, -0x88,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x87,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x89,0x00,0x00,0x00, -0x88,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x8a,0x00,0x00,0x00,0x6f,0x00,0x00,0x00,0x89,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8b,0x00,0x00,0x00, -0x85,0x00,0x00,0x00,0x8a,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x13,0x00,0x00,0x00,0x8e,0x00,0x00,0x00,0x10,0x00,0x00,0x00, -0x8d,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x8f,0x00,0x00,0x00,0x8e,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x90,0x00,0x00,0x00,0x61,0x00,0x00,0x00, -0x8f,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x91,0x00,0x00,0x00,0x8b,0x00,0x00,0x00,0x90,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x13,0x00,0x00,0x00,0x98,0x00,0x00,0x00, -0x10,0x00,0x00,0x00,0x97,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x99,0x00,0x00,0x00,0x98,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9b,0x00,0x00,0x00, -0x99,0x00,0x00,0x00,0x91,0x00,0x00,0x00,0x41,0x00,0x06,0x00, -0xa2,0x00,0x00,0x00,0xa3,0x00,0x00,0x00,0xa0,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x56,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x9c,0x00,0x00,0x00,0xa4,0x00,0x00,0x00,0xa3,0x00,0x00,0x00, -0x73,0x00,0x04,0x00,0x92,0x00,0x00,0x00,0xa5,0x00,0x00,0x00, -0xa4,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0xa6,0x00,0x00,0x00, -0xa7,0x00,0x00,0x00,0x96,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x9b,0x00,0x00,0x00,0x3e,0x00,0x03,0x00,0xa7,0x00,0x00,0x00, -0xa5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xab,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xab,0x00,0x00,0x00,0xfd,0x00,0x01,0x00, -0x38,0x00,0x01,0x00, -}; -const uint64_t cpy_f32_f16_len = 2524; - -unsigned char cpy_f32_f32_data[] = { -0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, -0xaa,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, -0x01,0x00,0x00,0x00,0x0b,0x00,0x06,0x00,0x01,0x00,0x00,0x00, -0x47,0x4c,0x53,0x4c,0x2e,0x73,0x74,0x64,0x2e,0x34,0x35,0x30, -0x00,0x00,0x00,0x00,0x0e,0x00,0x03,0x00,0x00,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x0f,0x00,0x09,0x00,0x05,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x6d,0x61,0x69,0x6e,0x00,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x96,0x00,0x00,0x00, -0x9f,0x00,0x00,0x00,0x10,0x00,0x06,0x00,0x04,0x00,0x00,0x00, -0x11,0x00,0x00,0x00,0x00,0x02,0x00,0x00,0x01,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x0b,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x0e,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x0e,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x0e,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x0e,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x0e,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x10,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x0e,0x00,0x00,0x00,0x05,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x0e,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x0e,0x00,0x00,0x00, -0x07,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x1c,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x0e,0x00,0x00,0x00,0x08,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x0e,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x24,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x0e,0x00,0x00,0x00, -0x0a,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x28,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x0e,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x2c,0x00,0x00,0x00,0x47,0x00,0x03,0x00, -0x0e,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x93,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x48,0x00,0x04,0x00,0x94,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x19,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x94,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x03,0x00,0x94,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x96,0x00,0x00,0x00,0x22,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x96,0x00,0x00,0x00, -0x21,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x9c,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x48,0x00,0x04,0x00,0x9d,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x9d,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x03,0x00,0x9d,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x9f,0x00,0x00,0x00,0x22,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x9f,0x00,0x00,0x00, -0x21,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0xa7,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x19,0x00,0x00,0x00, -0x13,0x00,0x02,0x00,0x02,0x00,0x00,0x00,0x21,0x00,0x03,0x00, -0x03,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x15,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x17,0x00,0x04,0x00,0x07,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x03,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x08,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x08,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x0a,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x0b,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x1e,0x00,0x0e,0x00, -0x0e,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x0f,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x0e,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x0f,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x15,0x00,0x04,0x00,0x11,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x11,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x14,0x00,0x02,0x00,0x16,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x11,0x00,0x00,0x00,0x1f,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x11,0x00,0x00,0x00,0x22,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x11,0x00,0x00,0x00, -0x47,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x11,0x00,0x00,0x00,0x4c,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x11,0x00,0x00,0x00,0x52,0x00,0x00,0x00, -0x05,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x11,0x00,0x00,0x00, -0x5a,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x11,0x00,0x00,0x00,0x5d,0x00,0x00,0x00,0x07,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x11,0x00,0x00,0x00,0x82,0x00,0x00,0x00, -0x08,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x11,0x00,0x00,0x00, -0x87,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x11,0x00,0x00,0x00,0x8d,0x00,0x00,0x00,0x0a,0x00,0x00,0x00, -0x16,0x00,0x03,0x00,0x92,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x1d,0x00,0x03,0x00,0x93,0x00,0x00,0x00,0x92,0x00,0x00,0x00, -0x1e,0x00,0x03,0x00,0x94,0x00,0x00,0x00,0x93,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x95,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x94,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x95,0x00,0x00,0x00, -0x96,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x11,0x00,0x00,0x00,0x97,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x1d,0x00,0x03,0x00,0x9c,0x00,0x00,0x00,0x92,0x00,0x00,0x00, -0x1e,0x00,0x03,0x00,0x9d,0x00,0x00,0x00,0x9c,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x9e,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x9d,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x9e,0x00,0x00,0x00, -0x9f,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0xa1,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x92,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xa5,0x00,0x00,0x00, -0x00,0x02,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xa6,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x2c,0x00,0x06,0x00, -0x07,0x00,0x00,0x00,0xa7,0x00,0x00,0x00,0xa5,0x00,0x00,0x00, -0xa6,0x00,0x00,0x00,0xa6,0x00,0x00,0x00,0x36,0x00,0x05,0x00, -0x02,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x03,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x05,0x00,0x00,0x00, -0xf7,0x00,0x03,0x00,0xa8,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0xfb,0x00,0x03,0x00,0x0a,0x00,0x00,0x00,0xa9,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xa9,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x0b,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x0a,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x0d,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x13,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x10,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x15,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0xae,0x00,0x05,0x00, -0x16,0x00,0x00,0x00,0x17,0x00,0x00,0x00,0x0d,0x00,0x00,0x00, -0x15,0x00,0x00,0x00,0xf7,0x00,0x03,0x00,0x19,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x17,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x18,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xa8,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0x19,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x13,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x10,0x00,0x00,0x00, -0x1f,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x21,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x13,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x10,0x00,0x00,0x00, -0x22,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x24,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x25,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x24,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x26,0x00,0x00,0x00,0x0d,0x00,0x00,0x00,0x25,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x2d,0x00,0x00,0x00, -0x26,0x00,0x00,0x00,0x24,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x30,0x00,0x00,0x00,0x2d,0x00,0x00,0x00, -0x21,0x00,0x00,0x00,0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x31,0x00,0x00,0x00,0x0d,0x00,0x00,0x00,0x30,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x34,0x00,0x00,0x00, -0x31,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x43,0x00,0x00,0x00,0x34,0x00,0x00,0x00, -0x21,0x00,0x00,0x00,0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x44,0x00,0x00,0x00,0x31,0x00,0x00,0x00,0x43,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x13,0x00,0x00,0x00,0x48,0x00,0x00,0x00, -0x10,0x00,0x00,0x00,0x47,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x49,0x00,0x00,0x00,0x48,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x4a,0x00,0x00,0x00, -0x44,0x00,0x00,0x00,0x49,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x13,0x00,0x00,0x00,0x4d,0x00,0x00,0x00,0x10,0x00,0x00,0x00, -0x4c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x4e,0x00,0x00,0x00,0x4d,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x34,0x00,0x00,0x00, -0x4e,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x50,0x00,0x00,0x00,0x4a,0x00,0x00,0x00,0x4f,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x13,0x00,0x00,0x00,0x53,0x00,0x00,0x00, -0x10,0x00,0x00,0x00,0x52,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x54,0x00,0x00,0x00,0x53,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x55,0x00,0x00,0x00, -0x26,0x00,0x00,0x00,0x54,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x56,0x00,0x00,0x00,0x50,0x00,0x00,0x00, -0x55,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x13,0x00,0x00,0x00, -0x5b,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x5a,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x5c,0x00,0x00,0x00, -0x5b,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x13,0x00,0x00,0x00, -0x5e,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x5d,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x5f,0x00,0x00,0x00, -0x5e,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x60,0x00,0x00,0x00,0x5c,0x00,0x00,0x00,0x5f,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x61,0x00,0x00,0x00, -0x0d,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x68,0x00,0x00,0x00,0x61,0x00,0x00,0x00, -0x5f,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x6b,0x00,0x00,0x00,0x68,0x00,0x00,0x00,0x5c,0x00,0x00,0x00, -0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x0d,0x00,0x00,0x00,0x6b,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x6f,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x5c,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x7e,0x00,0x00,0x00,0x6f,0x00,0x00,0x00,0x5c,0x00,0x00,0x00, -0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x7f,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x7e,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x13,0x00,0x00,0x00,0x83,0x00,0x00,0x00,0x10,0x00,0x00,0x00, -0x82,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0x83,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x85,0x00,0x00,0x00,0x7f,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x13,0x00,0x00,0x00, -0x88,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x87,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x89,0x00,0x00,0x00, -0x88,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x8a,0x00,0x00,0x00,0x6f,0x00,0x00,0x00,0x89,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8b,0x00,0x00,0x00, -0x85,0x00,0x00,0x00,0x8a,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x13,0x00,0x00,0x00,0x8e,0x00,0x00,0x00,0x10,0x00,0x00,0x00, -0x8d,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x8f,0x00,0x00,0x00,0x8e,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x90,0x00,0x00,0x00,0x61,0x00,0x00,0x00, -0x8f,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x91,0x00,0x00,0x00,0x8b,0x00,0x00,0x00,0x90,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x13,0x00,0x00,0x00,0x98,0x00,0x00,0x00, -0x10,0x00,0x00,0x00,0x97,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x99,0x00,0x00,0x00,0x98,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9b,0x00,0x00,0x00, -0x99,0x00,0x00,0x00,0x91,0x00,0x00,0x00,0x41,0x00,0x06,0x00, -0xa1,0x00,0x00,0x00,0xa2,0x00,0x00,0x00,0x9f,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x56,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x92,0x00,0x00,0x00,0xa3,0x00,0x00,0x00,0xa2,0x00,0x00,0x00, -0x41,0x00,0x06,0x00,0xa1,0x00,0x00,0x00,0xa4,0x00,0x00,0x00, -0x96,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x9b,0x00,0x00,0x00, -0x3e,0x00,0x03,0x00,0xa4,0x00,0x00,0x00,0xa3,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xa8,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xa8,0x00,0x00,0x00,0xfd,0x00,0x01,0x00,0x38,0x00,0x01,0x00, +0x95,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xfb,0x00,0x03,0x00, +0x0c,0x00,0x00,0x00,0x96,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0x96,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00, +0x0e,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, +0x0e,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x11,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x10,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00,0x13,0x00,0x00,0x00, +0x12,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x13,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x16,0x00,0x00,0x00, +0x14,0x00,0x00,0x00,0x15,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x17,0x00,0x00,0x00,0x11,0x00,0x00,0x00, +0x16,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x1b,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x15,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x1f,0x00,0x00,0x00, +0x1b,0x00,0x00,0x00,0x1e,0x00,0x00,0x00,0x89,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x1b,0x00,0x00,0x00, +0x1e,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x25,0x00,0x00,0x00,0x1e,0x00,0x00,0x00,0x17,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x27,0x00,0x00,0x00, +0x25,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x2e,0x00,0x00,0x00,0x2f,0x00,0x00,0x00,0x2b,0x00,0x00,0x00, +0x2d,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x30,0x00,0x00,0x00,0x2f,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x31,0x00,0x00,0x00,0x30,0x00,0x00,0x00, +0x1e,0x00,0x00,0x00,0xae,0x00,0x05,0x00,0x32,0x00,0x00,0x00, +0x33,0x00,0x00,0x00,0x27,0x00,0x00,0x00,0x31,0x00,0x00,0x00, +0xf7,0x00,0x03,0x00,0x35,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x33,0x00,0x00,0x00,0x34,0x00,0x00,0x00, +0x35,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x34,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x95,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0x35,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x3a,0x00,0x00,0x00,0x38,0x00,0x00,0x00,0x17,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3c,0x00,0x00,0x00, +0x1e,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x3d,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, +0x3c,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x40,0x00,0x00,0x00,0x3e,0x00,0x00,0x00,0x1f,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x41,0x00,0x00,0x00, +0x3d,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x41,0x00,0x07,0x00, +0x50,0x00,0x00,0x00,0x51,0x00,0x00,0x00,0x4d,0x00,0x00,0x00, +0x4e,0x00,0x00,0x00,0x27,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x45,0x00,0x00,0x00,0x52,0x00,0x00,0x00, +0x51,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x42,0x00,0x00,0x00, +0x53,0x00,0x00,0x00,0x52,0x00,0x00,0x00,0x85,0x00,0x05,0x00, +0x42,0x00,0x00,0x00,0x57,0x00,0x00,0x00,0x55,0x00,0x00,0x00, +0x53,0x00,0x00,0x00,0x41,0x00,0x08,0x00,0x71,0x00,0x00,0x00, +0x72,0x00,0x00,0x00,0x4d,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, +0x27,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x40,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x46,0x00,0x00,0x00,0x73,0x00,0x00,0x00, +0x72,0x00,0x00,0x00,0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x74,0x00,0x00,0x00,0x73,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, +0x2c,0x00,0x00,0x00,0x75,0x00,0x00,0x00,0x74,0x00,0x00,0x00, +0xc7,0x00,0x05,0x00,0x2c,0x00,0x00,0x00,0x77,0x00,0x00,0x00, +0x75,0x00,0x00,0x00,0x76,0x00,0x00,0x00,0x6f,0x00,0x04,0x00, +0x42,0x00,0x00,0x00,0x78,0x00,0x00,0x00,0x77,0x00,0x00,0x00, +0x0c,0x00,0x08,0x00,0x42,0x00,0x00,0x00,0x7b,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x53,0x00,0x00,0x00, +0x78,0x00,0x00,0x00,0x57,0x00,0x00,0x00,0x73,0x00,0x04,0x00, +0x45,0x00,0x00,0x00,0x7c,0x00,0x00,0x00,0x7b,0x00,0x00,0x00, +0x41,0x00,0x06,0x00,0x50,0x00,0x00,0x00,0x7d,0x00,0x00,0x00, +0x66,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x41,0x00,0x00,0x00, +0x3e,0x00,0x03,0x00,0x7d,0x00,0x00,0x00,0x7c,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x81,0x00,0x00,0x00, +0x41,0x00,0x00,0x00,0x47,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x46,0x00,0x00,0x00,0x88,0x00,0x00,0x00,0x72,0x00,0x00,0x00, +0xc2,0x00,0x05,0x00,0x46,0x00,0x00,0x00,0x89,0x00,0x00,0x00, +0x88,0x00,0x00,0x00,0x2d,0x00,0x00,0x00,0x70,0x00,0x04,0x00, +0x42,0x00,0x00,0x00,0x8a,0x00,0x00,0x00,0x89,0x00,0x00,0x00, +0x0c,0x00,0x08,0x00,0x42,0x00,0x00,0x00,0x8d,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x53,0x00,0x00,0x00, +0x8a,0x00,0x00,0x00,0x57,0x00,0x00,0x00,0x73,0x00,0x04,0x00, +0x45,0x00,0x00,0x00,0x8e,0x00,0x00,0x00,0x8d,0x00,0x00,0x00, +0x41,0x00,0x06,0x00,0x50,0x00,0x00,0x00,0x8f,0x00,0x00,0x00, +0x66,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x81,0x00,0x00,0x00, +0x3e,0x00,0x03,0x00,0x8f,0x00,0x00,0x00,0x8e,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa2,0x00,0x00,0x00, +0x41,0x00,0x00,0x00,0x93,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xa4,0x00,0x00,0x00,0x40,0x00,0x00,0x00, +0x93,0x00,0x00,0x00,0x41,0x00,0x08,0x00,0x71,0x00,0x00,0x00, +0xa5,0x00,0x00,0x00,0x4d,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, +0x27,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0xa4,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x46,0x00,0x00,0x00,0xa6,0x00,0x00,0x00, +0xa5,0x00,0x00,0x00,0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xa7,0x00,0x00,0x00,0xa6,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, +0x2c,0x00,0x00,0x00,0xa8,0x00,0x00,0x00,0xa7,0x00,0x00,0x00, +0xc7,0x00,0x05,0x00,0x2c,0x00,0x00,0x00,0xa9,0x00,0x00,0x00, +0xa8,0x00,0x00,0x00,0x76,0x00,0x00,0x00,0x6f,0x00,0x04,0x00, +0x42,0x00,0x00,0x00,0xaa,0x00,0x00,0x00,0xa9,0x00,0x00,0x00, +0x0c,0x00,0x08,0x00,0x42,0x00,0x00,0x00,0xac,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x53,0x00,0x00,0x00, +0xaa,0x00,0x00,0x00,0x57,0x00,0x00,0x00,0x73,0x00,0x04,0x00, +0x45,0x00,0x00,0x00,0xad,0x00,0x00,0x00,0xac,0x00,0x00,0x00, +0x41,0x00,0x06,0x00,0x50,0x00,0x00,0x00,0xae,0x00,0x00,0x00, +0x66,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0xa2,0x00,0x00,0x00, +0x3e,0x00,0x03,0x00,0xae,0x00,0x00,0x00,0xad,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xb0,0x00,0x00,0x00, +0x41,0x00,0x00,0x00,0x77,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, +0x46,0x00,0x00,0x00,0xb3,0x00,0x00,0x00,0xa5,0x00,0x00,0x00, +0xc2,0x00,0x05,0x00,0x46,0x00,0x00,0x00,0xb4,0x00,0x00,0x00, +0xb3,0x00,0x00,0x00,0x2d,0x00,0x00,0x00,0x70,0x00,0x04,0x00, +0x42,0x00,0x00,0x00,0xb5,0x00,0x00,0x00,0xb4,0x00,0x00,0x00, +0x0c,0x00,0x08,0x00,0x42,0x00,0x00,0x00,0xb7,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x53,0x00,0x00,0x00, +0xb5,0x00,0x00,0x00,0x57,0x00,0x00,0x00,0x73,0x00,0x04,0x00, +0x45,0x00,0x00,0x00,0xb8,0x00,0x00,0x00,0xb7,0x00,0x00,0x00, +0x41,0x00,0x06,0x00,0x50,0x00,0x00,0x00,0xb9,0x00,0x00,0x00, +0x66,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0xb0,0x00,0x00,0x00, +0x3e,0x00,0x03,0x00,0xb9,0x00,0x00,0x00,0xb8,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xc1,0x00,0x00,0x00, +0x41,0x00,0x00,0x00,0x78,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xc3,0x00,0x00,0x00,0x40,0x00,0x00,0x00, +0x78,0x01,0x00,0x00,0x41,0x00,0x08,0x00,0x71,0x00,0x00,0x00, +0xc4,0x00,0x00,0x00,0x4d,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, +0x27,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0xc3,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x46,0x00,0x00,0x00,0xc5,0x00,0x00,0x00, +0xc4,0x00,0x00,0x00,0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xc6,0x00,0x00,0x00,0xc5,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, +0x2c,0x00,0x00,0x00,0xc7,0x00,0x00,0x00,0xc6,0x00,0x00,0x00, +0xc7,0x00,0x05,0x00,0x2c,0x00,0x00,0x00,0xc8,0x00,0x00,0x00, +0xc7,0x00,0x00,0x00,0x76,0x00,0x00,0x00,0x6f,0x00,0x04,0x00, +0x42,0x00,0x00,0x00,0xc9,0x00,0x00,0x00,0xc8,0x00,0x00,0x00, +0x0c,0x00,0x08,0x00,0x42,0x00,0x00,0x00,0xcb,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x53,0x00,0x00,0x00, +0xc9,0x00,0x00,0x00,0x57,0x00,0x00,0x00,0x73,0x00,0x04,0x00, +0x45,0x00,0x00,0x00,0xcc,0x00,0x00,0x00,0xcb,0x00,0x00,0x00, +0x41,0x00,0x06,0x00,0x50,0x00,0x00,0x00,0xcd,0x00,0x00,0x00, +0x66,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0xc1,0x00,0x00,0x00, +0x3e,0x00,0x03,0x00,0xcd,0x00,0x00,0x00,0xcc,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xcf,0x00,0x00,0x00, +0x41,0x00,0x00,0x00,0x79,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, +0x46,0x00,0x00,0x00,0xd2,0x00,0x00,0x00,0xc4,0x00,0x00,0x00, +0xc2,0x00,0x05,0x00,0x46,0x00,0x00,0x00,0xd3,0x00,0x00,0x00, +0xd2,0x00,0x00,0x00,0x2d,0x00,0x00,0x00,0x70,0x00,0x04,0x00, +0x42,0x00,0x00,0x00,0xd4,0x00,0x00,0x00,0xd3,0x00,0x00,0x00, +0x0c,0x00,0x08,0x00,0x42,0x00,0x00,0x00,0xd6,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x53,0x00,0x00,0x00, +0xd4,0x00,0x00,0x00,0x57,0x00,0x00,0x00,0x73,0x00,0x04,0x00, +0x45,0x00,0x00,0x00,0xd7,0x00,0x00,0x00,0xd6,0x00,0x00,0x00, +0x41,0x00,0x06,0x00,0x50,0x00,0x00,0x00,0xd8,0x00,0x00,0x00, +0x66,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0xcf,0x00,0x00,0x00, +0x3e,0x00,0x03,0x00,0xd8,0x00,0x00,0x00,0xd7,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xe0,0x00,0x00,0x00, +0x41,0x00,0x00,0x00,0x7a,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xe2,0x00,0x00,0x00,0x40,0x00,0x00,0x00, +0x7a,0x01,0x00,0x00,0x41,0x00,0x08,0x00,0x71,0x00,0x00,0x00, +0xe3,0x00,0x00,0x00,0x4d,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, +0x27,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0xe2,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x46,0x00,0x00,0x00,0xe4,0x00,0x00,0x00, +0xe3,0x00,0x00,0x00,0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xe5,0x00,0x00,0x00,0xe4,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, +0x2c,0x00,0x00,0x00,0xe6,0x00,0x00,0x00,0xe5,0x00,0x00,0x00, +0xc7,0x00,0x05,0x00,0x2c,0x00,0x00,0x00,0xe7,0x00,0x00,0x00, +0xe6,0x00,0x00,0x00,0x76,0x00,0x00,0x00,0x6f,0x00,0x04,0x00, +0x42,0x00,0x00,0x00,0xe8,0x00,0x00,0x00,0xe7,0x00,0x00,0x00, +0x0c,0x00,0x08,0x00,0x42,0x00,0x00,0x00,0xea,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x53,0x00,0x00,0x00, +0xe8,0x00,0x00,0x00,0x57,0x00,0x00,0x00,0x73,0x00,0x04,0x00, +0x45,0x00,0x00,0x00,0xeb,0x00,0x00,0x00,0xea,0x00,0x00,0x00, +0x41,0x00,0x06,0x00,0x50,0x00,0x00,0x00,0xec,0x00,0x00,0x00, +0x66,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0xe0,0x00,0x00,0x00, +0x3e,0x00,0x03,0x00,0xec,0x00,0x00,0x00,0xeb,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xee,0x00,0x00,0x00, +0x41,0x00,0x00,0x00,0x7b,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, +0x46,0x00,0x00,0x00,0xf1,0x00,0x00,0x00,0xe3,0x00,0x00,0x00, +0xc2,0x00,0x05,0x00,0x46,0x00,0x00,0x00,0xf2,0x00,0x00,0x00, +0xf1,0x00,0x00,0x00,0x2d,0x00,0x00,0x00,0x70,0x00,0x04,0x00, +0x42,0x00,0x00,0x00,0xf3,0x00,0x00,0x00,0xf2,0x00,0x00,0x00, +0x0c,0x00,0x08,0x00,0x42,0x00,0x00,0x00,0xf5,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x53,0x00,0x00,0x00, +0xf3,0x00,0x00,0x00,0x57,0x00,0x00,0x00,0x73,0x00,0x04,0x00, +0x45,0x00,0x00,0x00,0xf6,0x00,0x00,0x00,0xf5,0x00,0x00,0x00, +0x41,0x00,0x06,0x00,0x50,0x00,0x00,0x00,0xf7,0x00,0x00,0x00, +0x66,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0xee,0x00,0x00,0x00, +0x3e,0x00,0x03,0x00,0xf7,0x00,0x00,0x00,0xf6,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xff,0x00,0x00,0x00, +0x41,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x01,0x01,0x00,0x00,0x40,0x00,0x00,0x00, +0x10,0x00,0x00,0x00,0x41,0x00,0x08,0x00,0x71,0x00,0x00,0x00, +0x02,0x01,0x00,0x00,0x4d,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, +0x27,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x01,0x01,0x00,0x00, +0x3d,0x00,0x04,0x00,0x46,0x00,0x00,0x00,0x03,0x01,0x00,0x00, +0x02,0x01,0x00,0x00,0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x04,0x01,0x00,0x00,0x03,0x01,0x00,0x00,0x7c,0x00,0x04,0x00, +0x2c,0x00,0x00,0x00,0x05,0x01,0x00,0x00,0x04,0x01,0x00,0x00, +0xc7,0x00,0x05,0x00,0x2c,0x00,0x00,0x00,0x06,0x01,0x00,0x00, +0x05,0x01,0x00,0x00,0x76,0x00,0x00,0x00,0x6f,0x00,0x04,0x00, +0x42,0x00,0x00,0x00,0x07,0x01,0x00,0x00,0x06,0x01,0x00,0x00, +0x0c,0x00,0x08,0x00,0x42,0x00,0x00,0x00,0x09,0x01,0x00,0x00, +0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x53,0x00,0x00,0x00, +0x07,0x01,0x00,0x00,0x57,0x00,0x00,0x00,0x73,0x00,0x04,0x00, +0x45,0x00,0x00,0x00,0x0a,0x01,0x00,0x00,0x09,0x01,0x00,0x00, +0x41,0x00,0x06,0x00,0x50,0x00,0x00,0x00,0x0b,0x01,0x00,0x00, +0x66,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0xff,0x00,0x00,0x00, +0x3e,0x00,0x03,0x00,0x0b,0x01,0x00,0x00,0x0a,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x0d,0x01,0x00,0x00, +0x41,0x00,0x00,0x00,0x7c,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, +0x46,0x00,0x00,0x00,0x10,0x01,0x00,0x00,0x02,0x01,0x00,0x00, +0xc2,0x00,0x05,0x00,0x46,0x00,0x00,0x00,0x11,0x01,0x00,0x00, +0x10,0x01,0x00,0x00,0x2d,0x00,0x00,0x00,0x70,0x00,0x04,0x00, +0x42,0x00,0x00,0x00,0x12,0x01,0x00,0x00,0x11,0x01,0x00,0x00, +0x0c,0x00,0x08,0x00,0x42,0x00,0x00,0x00,0x14,0x01,0x00,0x00, +0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x53,0x00,0x00,0x00, +0x12,0x01,0x00,0x00,0x57,0x00,0x00,0x00,0x73,0x00,0x04,0x00, +0x45,0x00,0x00,0x00,0x15,0x01,0x00,0x00,0x14,0x01,0x00,0x00, +0x41,0x00,0x06,0x00,0x50,0x00,0x00,0x00,0x16,0x01,0x00,0x00, +0x66,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x0d,0x01,0x00,0x00, +0x3e,0x00,0x03,0x00,0x16,0x01,0x00,0x00,0x15,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x1e,0x01,0x00,0x00, +0x41,0x00,0x00,0x00,0x7d,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x20,0x01,0x00,0x00,0x40,0x00,0x00,0x00, +0x7d,0x01,0x00,0x00,0x41,0x00,0x08,0x00,0x71,0x00,0x00,0x00, +0x21,0x01,0x00,0x00,0x4d,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, +0x27,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x20,0x01,0x00,0x00, +0x3d,0x00,0x04,0x00,0x46,0x00,0x00,0x00,0x22,0x01,0x00,0x00, +0x21,0x01,0x00,0x00,0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x23,0x01,0x00,0x00,0x22,0x01,0x00,0x00,0x7c,0x00,0x04,0x00, +0x2c,0x00,0x00,0x00,0x24,0x01,0x00,0x00,0x23,0x01,0x00,0x00, +0xc7,0x00,0x05,0x00,0x2c,0x00,0x00,0x00,0x25,0x01,0x00,0x00, +0x24,0x01,0x00,0x00,0x76,0x00,0x00,0x00,0x6f,0x00,0x04,0x00, +0x42,0x00,0x00,0x00,0x26,0x01,0x00,0x00,0x25,0x01,0x00,0x00, +0x0c,0x00,0x08,0x00,0x42,0x00,0x00,0x00,0x28,0x01,0x00,0x00, +0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x53,0x00,0x00,0x00, +0x26,0x01,0x00,0x00,0x57,0x00,0x00,0x00,0x73,0x00,0x04,0x00, +0x45,0x00,0x00,0x00,0x29,0x01,0x00,0x00,0x28,0x01,0x00,0x00, +0x41,0x00,0x06,0x00,0x50,0x00,0x00,0x00,0x2a,0x01,0x00,0x00, +0x66,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x1e,0x01,0x00,0x00, +0x3e,0x00,0x03,0x00,0x2a,0x01,0x00,0x00,0x29,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x2c,0x01,0x00,0x00, +0x41,0x00,0x00,0x00,0x7e,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, +0x46,0x00,0x00,0x00,0x2f,0x01,0x00,0x00,0x21,0x01,0x00,0x00, +0xc2,0x00,0x05,0x00,0x46,0x00,0x00,0x00,0x30,0x01,0x00,0x00, +0x2f,0x01,0x00,0x00,0x2d,0x00,0x00,0x00,0x70,0x00,0x04,0x00, +0x42,0x00,0x00,0x00,0x31,0x01,0x00,0x00,0x30,0x01,0x00,0x00, +0x0c,0x00,0x08,0x00,0x42,0x00,0x00,0x00,0x33,0x01,0x00,0x00, +0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x53,0x00,0x00,0x00, +0x31,0x01,0x00,0x00,0x57,0x00,0x00,0x00,0x73,0x00,0x04,0x00, +0x45,0x00,0x00,0x00,0x34,0x01,0x00,0x00,0x33,0x01,0x00,0x00, +0x41,0x00,0x06,0x00,0x50,0x00,0x00,0x00,0x35,0x01,0x00,0x00, +0x66,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x2c,0x01,0x00,0x00, +0x3e,0x00,0x03,0x00,0x35,0x01,0x00,0x00,0x34,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3d,0x01,0x00,0x00, +0x41,0x00,0x00,0x00,0x7f,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x3f,0x01,0x00,0x00,0x40,0x00,0x00,0x00, +0x7f,0x01,0x00,0x00,0x41,0x00,0x08,0x00,0x71,0x00,0x00,0x00, +0x40,0x01,0x00,0x00,0x4d,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, +0x27,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x3f,0x01,0x00,0x00, +0x3d,0x00,0x04,0x00,0x46,0x00,0x00,0x00,0x41,0x01,0x00,0x00, +0x40,0x01,0x00,0x00,0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x42,0x01,0x00,0x00,0x41,0x01,0x00,0x00,0x7c,0x00,0x04,0x00, +0x2c,0x00,0x00,0x00,0x43,0x01,0x00,0x00,0x42,0x01,0x00,0x00, +0xc7,0x00,0x05,0x00,0x2c,0x00,0x00,0x00,0x44,0x01,0x00,0x00, +0x43,0x01,0x00,0x00,0x76,0x00,0x00,0x00,0x6f,0x00,0x04,0x00, +0x42,0x00,0x00,0x00,0x45,0x01,0x00,0x00,0x44,0x01,0x00,0x00, +0x0c,0x00,0x08,0x00,0x42,0x00,0x00,0x00,0x47,0x01,0x00,0x00, +0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x53,0x00,0x00,0x00, +0x45,0x01,0x00,0x00,0x57,0x00,0x00,0x00,0x73,0x00,0x04,0x00, +0x45,0x00,0x00,0x00,0x48,0x01,0x00,0x00,0x47,0x01,0x00,0x00, +0x41,0x00,0x06,0x00,0x50,0x00,0x00,0x00,0x49,0x01,0x00,0x00, +0x66,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x3d,0x01,0x00,0x00, +0x3e,0x00,0x03,0x00,0x49,0x01,0x00,0x00,0x48,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x4b,0x01,0x00,0x00, +0x41,0x00,0x00,0x00,0x80,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, +0x46,0x00,0x00,0x00,0x4e,0x01,0x00,0x00,0x40,0x01,0x00,0x00, +0xc2,0x00,0x05,0x00,0x46,0x00,0x00,0x00,0x4f,0x01,0x00,0x00, +0x4e,0x01,0x00,0x00,0x2d,0x00,0x00,0x00,0x70,0x00,0x04,0x00, +0x42,0x00,0x00,0x00,0x50,0x01,0x00,0x00,0x4f,0x01,0x00,0x00, +0x0c,0x00,0x08,0x00,0x42,0x00,0x00,0x00,0x52,0x01,0x00,0x00, +0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x53,0x00,0x00,0x00, +0x50,0x01,0x00,0x00,0x57,0x00,0x00,0x00,0x73,0x00,0x04,0x00, +0x45,0x00,0x00,0x00,0x53,0x01,0x00,0x00,0x52,0x01,0x00,0x00, +0x41,0x00,0x06,0x00,0x50,0x00,0x00,0x00,0x54,0x01,0x00,0x00, +0x66,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x4b,0x01,0x00,0x00, +0x3e,0x00,0x03,0x00,0x54,0x01,0x00,0x00,0x53,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x5c,0x01,0x00,0x00, +0x41,0x00,0x00,0x00,0x81,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x5e,0x01,0x00,0x00,0x40,0x00,0x00,0x00, +0x81,0x01,0x00,0x00,0x41,0x00,0x08,0x00,0x71,0x00,0x00,0x00, +0x5f,0x01,0x00,0x00,0x4d,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, +0x27,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x5e,0x01,0x00,0x00, +0x3d,0x00,0x04,0x00,0x46,0x00,0x00,0x00,0x60,0x01,0x00,0x00, +0x5f,0x01,0x00,0x00,0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x61,0x01,0x00,0x00,0x60,0x01,0x00,0x00,0x7c,0x00,0x04,0x00, +0x2c,0x00,0x00,0x00,0x62,0x01,0x00,0x00,0x61,0x01,0x00,0x00, +0xc7,0x00,0x05,0x00,0x2c,0x00,0x00,0x00,0x63,0x01,0x00,0x00, +0x62,0x01,0x00,0x00,0x76,0x00,0x00,0x00,0x6f,0x00,0x04,0x00, +0x42,0x00,0x00,0x00,0x64,0x01,0x00,0x00,0x63,0x01,0x00,0x00, +0x0c,0x00,0x08,0x00,0x42,0x00,0x00,0x00,0x66,0x01,0x00,0x00, +0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x53,0x00,0x00,0x00, +0x64,0x01,0x00,0x00,0x57,0x00,0x00,0x00,0x73,0x00,0x04,0x00, +0x45,0x00,0x00,0x00,0x67,0x01,0x00,0x00,0x66,0x01,0x00,0x00, +0x41,0x00,0x06,0x00,0x50,0x00,0x00,0x00,0x68,0x01,0x00,0x00, +0x66,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x5c,0x01,0x00,0x00, +0x3e,0x00,0x03,0x00,0x68,0x01,0x00,0x00,0x67,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x6a,0x01,0x00,0x00, +0x41,0x00,0x00,0x00,0x82,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, +0x46,0x00,0x00,0x00,0x6d,0x01,0x00,0x00,0x5f,0x01,0x00,0x00, +0xc2,0x00,0x05,0x00,0x46,0x00,0x00,0x00,0x6e,0x01,0x00,0x00, +0x6d,0x01,0x00,0x00,0x2d,0x00,0x00,0x00,0x70,0x00,0x04,0x00, +0x42,0x00,0x00,0x00,0x6f,0x01,0x00,0x00,0x6e,0x01,0x00,0x00, +0x0c,0x00,0x08,0x00,0x42,0x00,0x00,0x00,0x71,0x01,0x00,0x00, +0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x53,0x00,0x00,0x00, +0x6f,0x01,0x00,0x00,0x57,0x00,0x00,0x00,0x73,0x00,0x04,0x00, +0x45,0x00,0x00,0x00,0x72,0x01,0x00,0x00,0x71,0x01,0x00,0x00, +0x41,0x00,0x06,0x00,0x50,0x00,0x00,0x00,0x73,0x01,0x00,0x00, +0x66,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x6a,0x01,0x00,0x00, +0x3e,0x00,0x03,0x00,0x73,0x01,0x00,0x00,0x72,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0x95,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0x95,0x00,0x00,0x00,0xfd,0x00,0x01,0x00,0x38,0x00,0x01,0x00, }; -const uint64_t cpy_f32_f32_len = 2472; - -unsigned char dequant_f16_data[] = { -0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, -0x86,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, -0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x51,0x11,0x00,0x00, -0x0b,0x00,0x06,0x00,0x01,0x00,0x00,0x00,0x47,0x4c,0x53,0x4c, -0x2e,0x73,0x74,0x64,0x2e,0x34,0x35,0x30,0x00,0x00,0x00,0x00, -0x0e,0x00,0x03,0x00,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x0f,0x00,0x09,0x00,0x05,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x6d,0x61,0x69,0x6e,0x00,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x50,0x00,0x00,0x00,0x60,0x00,0x00,0x00, -0x10,0x00,0x06,0x00,0x04,0x00,0x00,0x00,0x11,0x00,0x00,0x00, -0x00,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x0c,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x1c,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x14,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x14,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x14,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x08,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x14,0x00,0x00,0x00, -0x03,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x47,0x00,0x03,0x00,0x14,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x4d,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0x4e,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x4e,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x4e,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x50,0x00,0x00,0x00, -0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x50,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x5d,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0x5e,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x5e,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x5e,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x60,0x00,0x00,0x00, -0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x60,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x83,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x19,0x00,0x00,0x00,0x13,0x00,0x02,0x00,0x02,0x00,0x00,0x00, -0x21,0x00,0x03,0x00,0x03,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x15,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x15,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x17,0x00,0x04,0x00, -0x0a,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x03,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x0b,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x0a,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x0b,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x0e,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x1e,0x00,0x06,0x00,0x14,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x15,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x15,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x17,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x18,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x14,0x00,0x02,0x00, -0x23,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x2d,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x36,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x16,0x00,0x03,0x00,0x48,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x16,0x00,0x03,0x00,0x4c,0x00,0x00,0x00,0x10,0x00,0x00,0x00, -0x1d,0x00,0x03,0x00,0x4d,0x00,0x00,0x00,0x4c,0x00,0x00,0x00, -0x1e,0x00,0x03,0x00,0x4e,0x00,0x00,0x00,0x4d,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x4f,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x4e,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x4f,0x00,0x00,0x00, -0x50,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x53,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x4c,0x00,0x00,0x00, -0x1d,0x00,0x03,0x00,0x5d,0x00,0x00,0x00,0x4c,0x00,0x00,0x00, -0x1e,0x00,0x03,0x00,0x5e,0x00,0x00,0x00,0x5d,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x5f,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x5e,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x5f,0x00,0x00,0x00, -0x60,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x03,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x7b,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x82,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x2c,0x00,0x06,0x00, -0x0a,0x00,0x00,0x00,0x83,0x00,0x00,0x00,0x82,0x00,0x00,0x00, -0x7b,0x00,0x00,0x00,0x7b,0x00,0x00,0x00,0x36,0x00,0x05,0x00, -0x02,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x03,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x05,0x00,0x00,0x00, -0xf7,0x00,0x03,0x00,0x84,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0xfb,0x00,0x03,0x00,0x0d,0x00,0x00,0x00,0x85,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0x85,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x0e,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x0d,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x10,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x11,0x00,0x00,0x00,0x10,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x18,0x00,0x00,0x00,0x19,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x17,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x1a,0x00,0x00,0x00,0x19,0x00,0x00,0x00, -0x87,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x1b,0x00,0x00,0x00, -0x1a,0x00,0x00,0x00,0x17,0x00,0x00,0x00,0x8b,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x11,0x00,0x00,0x00, -0x1b,0x00,0x00,0x00,0x87,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x22,0x00,0x00,0x00,0x11,0x00,0x00,0x00,0x1b,0x00,0x00,0x00, -0xaf,0x00,0x05,0x00,0x23,0x00,0x00,0x00,0x28,0x00,0x00,0x00, -0x1c,0x00,0x00,0x00,0x1a,0x00,0x00,0x00,0xa8,0x00,0x04,0x00, -0x23,0x00,0x00,0x00,0x29,0x00,0x00,0x00,0x28,0x00,0x00,0x00, -0xf7,0x00,0x03,0x00,0x2b,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x29,0x00,0x00,0x00,0x2a,0x00,0x00,0x00, -0x2b,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x2a,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x18,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x2d,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x2f,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0xaf,0x00,0x05,0x00,0x23,0x00,0x00,0x00,0x30,0x00,0x00,0x00, -0x22,0x00,0x00,0x00,0x2f,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x2b,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x2b,0x00,0x00,0x00, -0xf5,0x00,0x07,0x00,0x23,0x00,0x00,0x00,0x31,0x00,0x00,0x00, -0x28,0x00,0x00,0x00,0x85,0x00,0x00,0x00,0x30,0x00,0x00,0x00, -0x2a,0x00,0x00,0x00,0xf7,0x00,0x03,0x00,0x33,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x31,0x00,0x00,0x00, -0x32,0x00,0x00,0x00,0x33,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x32,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x84,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0x33,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x18,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0x36,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x38,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x87,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x38,0x00,0x00,0x00, -0x17,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x3d,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3f,0x00,0x00,0x00, -0x3d,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x41,0x00,0x06,0x00, -0x53,0x00,0x00,0x00,0x54,0x00,0x00,0x00,0x50,0x00,0x00,0x00, -0x2d,0x00,0x00,0x00,0x3f,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4c,0x00,0x00,0x00,0x55,0x00,0x00,0x00,0x54,0x00,0x00,0x00, -0x73,0x00,0x04,0x00,0x48,0x00,0x00,0x00,0x56,0x00,0x00,0x00, -0x55,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x58,0x00,0x00,0x00,0x3f,0x00,0x00,0x00,0x17,0x00,0x00,0x00, -0x41,0x00,0x06,0x00,0x53,0x00,0x00,0x00,0x59,0x00,0x00,0x00, -0x50,0x00,0x00,0x00,0x2d,0x00,0x00,0x00,0x58,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4c,0x00,0x00,0x00,0x5a,0x00,0x00,0x00, -0x59,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x48,0x00,0x00,0x00, -0x5b,0x00,0x00,0x00,0x5a,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x18,0x00,0x00,0x00,0x63,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x64,0x00,0x00,0x00,0x63,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x65,0x00,0x00,0x00,0x22,0x00,0x00,0x00, -0x64,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x68,0x00,0x00,0x00,0x65,0x00,0x00,0x00,0x1c,0x00,0x00,0x00, -0x73,0x00,0x04,0x00,0x4c,0x00,0x00,0x00,0x6f,0x00,0x00,0x00, -0x56,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x53,0x00,0x00,0x00, -0x70,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x2d,0x00,0x00,0x00, -0x68,0x00,0x00,0x00,0x3e,0x00,0x03,0x00,0x70,0x00,0x00,0x00, -0x6f,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x7a,0x00,0x00,0x00,0x68,0x00,0x00,0x00,0x17,0x00,0x00,0x00, -0x73,0x00,0x04,0x00,0x4c,0x00,0x00,0x00,0x7e,0x00,0x00,0x00, -0x5b,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x53,0x00,0x00,0x00, -0x7f,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x2d,0x00,0x00,0x00, -0x7a,0x00,0x00,0x00,0x3e,0x00,0x03,0x00,0x7f,0x00,0x00,0x00, -0x7e,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x84,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0x84,0x00,0x00,0x00,0xfd,0x00,0x01,0x00, -0x38,0x00,0x01,0x00, -}; -const uint64_t dequant_f16_len = 1816; - -unsigned char dequant_q2_K_data[] = { -0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, -0x13,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, -0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x27,0x00,0x00,0x00, -0x11,0x00,0x02,0x00,0x51,0x11,0x00,0x00,0x11,0x00,0x02,0x00, -0x60,0x11,0x00,0x00,0x0b,0x00,0x06,0x00,0x01,0x00,0x00,0x00, -0x47,0x4c,0x53,0x4c,0x2e,0x73,0x74,0x64,0x2e,0x34,0x35,0x30, -0x00,0x00,0x00,0x00,0x0e,0x00,0x03,0x00,0x00,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x0f,0x00,0x0a,0x00,0x05,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x6d,0x61,0x69,0x6e,0x00,0x00,0x00,0x00, -0x17,0x00,0x00,0x00,0x25,0x00,0x00,0x00,0x33,0x00,0x00,0x00, -0x63,0x00,0x00,0x00,0x7e,0x00,0x00,0x00,0x10,0x00,0x06,0x00, -0x04,0x00,0x00,0x00,0x11,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x17,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x1a,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x23,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x23,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x08,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x23,0x00,0x00,0x00,0x03,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x47,0x00,0x03,0x00, -0x23,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x33,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x1b,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x5a,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x5c,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x5f,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x5f,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x10,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x5f,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x50,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x60,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x54,0x00,0x00,0x00, -0x48,0x00,0x04,0x00,0x61,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x61,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x03,0x00,0x61,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x63,0x00,0x00,0x00,0x22,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x63,0x00,0x00,0x00, -0x21,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x7b,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x48,0x00,0x04,0x00,0x7c,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x19,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x7c,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x03,0x00,0x7c,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x7e,0x00,0x00,0x00,0x22,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x7e,0x00,0x00,0x00, -0x21,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x03,0x01,0x00,0x00,0x0b,0x00,0x00,0x00,0x19,0x00,0x00,0x00, -0x13,0x00,0x02,0x00,0x02,0x00,0x00,0x00,0x21,0x00,0x03,0x00, -0x03,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x15,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x10,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x14,0x00,0x02,0x00, -0x11,0x00,0x00,0x00,0x15,0x00,0x04,0x00,0x14,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x17,0x00,0x04,0x00, -0x15,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x03,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x16,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x15,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x16,0x00,0x00,0x00, -0x17,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x14,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x19,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x14,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x14,0x00,0x00,0x00, -0x1c,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x1e,0x00,0x06,0x00, -0x23,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x24,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x24,0x00,0x00,0x00,0x25,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x26,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x29,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x16,0x00,0x00,0x00,0x33,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x41,0x00,0x00,0x00,0x08,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x45,0x00,0x00,0x00, -0x10,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x4b,0x00,0x00,0x00,0x80,0x00,0x00,0x00,0x15,0x00,0x04,0x00, -0x56,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0x59,0x00,0x00,0x00, -0x10,0x00,0x00,0x00,0x1c,0x00,0x04,0x00,0x5a,0x00,0x00,0x00, -0x56,0x00,0x00,0x00,0x59,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x14,0x00,0x00,0x00,0x5b,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x1c,0x00,0x04,0x00,0x5c,0x00,0x00,0x00,0x56,0x00,0x00,0x00, -0x5b,0x00,0x00,0x00,0x16,0x00,0x03,0x00,0x5d,0x00,0x00,0x00, -0x10,0x00,0x00,0x00,0x17,0x00,0x04,0x00,0x5e,0x00,0x00,0x00, -0x5d,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x1e,0x00,0x05,0x00, -0x5f,0x00,0x00,0x00,0x5a,0x00,0x00,0x00,0x5c,0x00,0x00,0x00, -0x5e,0x00,0x00,0x00,0x1d,0x00,0x03,0x00,0x60,0x00,0x00,0x00, -0x5f,0x00,0x00,0x00,0x1e,0x00,0x03,0x00,0x61,0x00,0x00,0x00, -0x60,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x62,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x61,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x62,0x00,0x00,0x00,0x63,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x69,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x56,0x00,0x00,0x00,0x16,0x00,0x03,0x00,0x6c,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x70,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x71,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x5d,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0x77,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x1d,0x00,0x03,0x00,0x7b,0x00,0x00,0x00, -0x5d,0x00,0x00,0x00,0x1e,0x00,0x03,0x00,0x7c,0x00,0x00,0x00, -0x7b,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x7d,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x7c,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x7d,0x00,0x00,0x00,0x7e,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x89,0x00,0x00,0x00, -0x0f,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x8f,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x9a,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xc1,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xe1,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xe6,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x2c,0x00,0x06,0x00,0x15,0x00,0x00,0x00,0x03,0x01,0x00,0x00, -0x5b,0x00,0x00,0x00,0x77,0x00,0x00,0x00,0x77,0x00,0x00,0x00, -0x2a,0x00,0x03,0x00,0x11,0x00,0x00,0x00,0x06,0x01,0x00,0x00, -0x29,0x00,0x03,0x00,0x11,0x00,0x00,0x00,0x09,0x01,0x00,0x00, -0x36,0x00,0x05,0x00,0x02,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x05,0x00,0x00,0x00,0xf7,0x00,0x03,0x00,0x04,0x01,0x00,0x00, -0x00,0x00,0x00,0x00,0xfb,0x00,0x03,0x00,0x18,0x00,0x00,0x00, -0x05,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x05,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0x0a,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x0a,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0x0c,0x01,0x00,0x00,0x09,0x00,0x00,0x00,0x05,0x01,0x00,0x00, -0x02,0x01,0x00,0x00,0x0d,0x00,0x00,0x00,0xb1,0x00,0x05,0x00, -0x11,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x0c,0x01,0x00,0x00, -0x10,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x0c,0x00,0x00,0x00, -0x0d,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x12,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0x0b,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x19,0x00,0x00,0x00,0x1a,0x00,0x00,0x00,0x17,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x14,0x00,0x00,0x00, -0x1b,0x00,0x00,0x00,0x1a,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x14,0x00,0x00,0x00,0x1d,0x00,0x00,0x00,0x1b,0x00,0x00,0x00, -0x1c,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x14,0x00,0x00,0x00, -0x1f,0x00,0x00,0x00,0x0c,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x14,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x1d,0x00,0x00,0x00, -0x1f,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x21,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x26,0x00,0x00,0x00,0x27,0x00,0x00,0x00,0x25,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x28,0x00,0x00,0x00,0x27,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x26,0x00,0x00,0x00,0x2a,0x00,0x00,0x00,0x25,0x00,0x00,0x00, -0x29,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x2b,0x00,0x00,0x00,0x2a,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x2c,0x00,0x00,0x00,0x28,0x00,0x00,0x00, -0x2b,0x00,0x00,0x00,0x87,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x2d,0x00,0x00,0x00,0x2c,0x00,0x00,0x00,0x10,0x00,0x00,0x00, -0xaf,0x00,0x05,0x00,0x11,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x21,0x00,0x00,0x00,0x2d,0x00,0x00,0x00,0xf7,0x00,0x03,0x00, -0x30,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x2e,0x00,0x00,0x00,0x2f,0x00,0x00,0x00,0x30,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0x2f,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x0c,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x30,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x19,0x00,0x00,0x00,0x34,0x00,0x00,0x00, -0x33,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x14,0x00,0x00,0x00,0x35,0x00,0x00,0x00,0x34,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x36,0x00,0x00,0x00, -0x35,0x00,0x00,0x00,0x87,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x3a,0x00,0x00,0x00,0x36,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3e,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x82,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x3f,0x00,0x00,0x00,0x36,0x00,0x00,0x00, -0x3e,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x43,0x00,0x00,0x00,0x41,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, -0x87,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x46,0x00,0x00,0x00, -0x3f,0x00,0x00,0x00,0x45,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x47,0x00,0x00,0x00,0x43,0x00,0x00,0x00, -0x46,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x4a,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x10,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x4d,0x00,0x00,0x00, -0x4b,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x4a,0x00,0x00,0x00, -0x4d,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x50,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x3f,0x00,0x00,0x00, -0x41,0x00,0x08,0x00,0x69,0x00,0x00,0x00,0x6a,0x00,0x00,0x00, -0x63,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x29,0x00,0x00,0x00,0x36,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x56,0x00,0x00,0x00,0x6b,0x00,0x00,0x00,0x6a,0x00,0x00,0x00, -0x41,0x00,0x08,0x00,0x71,0x00,0x00,0x00,0x72,0x00,0x00,0x00, -0x63,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x70,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x5d,0x00,0x00,0x00,0x73,0x00,0x00,0x00,0x72,0x00,0x00,0x00, -0x73,0x00,0x04,0x00,0x6c,0x00,0x00,0x00,0x74,0x00,0x00,0x00, -0x73,0x00,0x00,0x00,0x41,0x00,0x08,0x00,0x71,0x00,0x00,0x00, -0x78,0x00,0x00,0x00,0x63,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x21,0x00,0x00,0x00,0x70,0x00,0x00,0x00,0x77,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x5d,0x00,0x00,0x00,0x79,0x00,0x00,0x00, -0x78,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x6c,0x00,0x00,0x00, -0x7a,0x00,0x00,0x00,0x79,0x00,0x00,0x00,0x41,0x00,0x08,0x00, -0x69,0x00,0x00,0x00,0x85,0x00,0x00,0x00,0x63,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x47,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x56,0x00,0x00,0x00, -0x86,0x00,0x00,0x00,0x85,0x00,0x00,0x00,0x71,0x00,0x04,0x00, -0x14,0x00,0x00,0x00,0x87,0x00,0x00,0x00,0x86,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x88,0x00,0x00,0x00, -0x87,0x00,0x00,0x00,0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x8a,0x00,0x00,0x00,0x88,0x00,0x00,0x00,0x89,0x00,0x00,0x00, -0xc2,0x00,0x05,0x00,0x56,0x00,0x00,0x00,0x8c,0x00,0x00,0x00, -0x6b,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x71,0x00,0x04,0x00, -0x14,0x00,0x00,0x00,0x8d,0x00,0x00,0x00,0x8c,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x8e,0x00,0x00,0x00, -0x8d,0x00,0x00,0x00,0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x90,0x00,0x00,0x00,0x8e,0x00,0x00,0x00,0x8f,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x91,0x00,0x00,0x00, -0x8a,0x00,0x00,0x00,0x90,0x00,0x00,0x00,0x6f,0x00,0x04,0x00, -0x6c,0x00,0x00,0x00,0x92,0x00,0x00,0x00,0x91,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x56,0x00,0x00,0x00,0x99,0x00,0x00,0x00, -0x85,0x00,0x00,0x00,0xc2,0x00,0x05,0x00,0x56,0x00,0x00,0x00, -0x9b,0x00,0x00,0x00,0x99,0x00,0x00,0x00,0x9a,0x00,0x00,0x00, -0x70,0x00,0x04,0x00,0x6c,0x00,0x00,0x00,0x9c,0x00,0x00,0x00, -0x9b,0x00,0x00,0x00,0x85,0x00,0x05,0x00,0x6c,0x00,0x00,0x00, -0x9d,0x00,0x00,0x00,0x7a,0x00,0x00,0x00,0x9c,0x00,0x00,0x00, -0x7f,0x00,0x04,0x00,0x6c,0x00,0x00,0x00,0x0f,0x01,0x00,0x00, -0x9d,0x00,0x00,0x00,0x0c,0x00,0x08,0x00,0x6c,0x00,0x00,0x00, -0x9e,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00, -0x74,0x00,0x00,0x00,0x92,0x00,0x00,0x00,0x0f,0x01,0x00,0x00, -0x73,0x00,0x04,0x00,0x5d,0x00,0x00,0x00,0x9f,0x00,0x00,0x00, -0x9e,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x71,0x00,0x00,0x00, -0xa0,0x00,0x00,0x00,0x7e,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x50,0x00,0x00,0x00,0x3e,0x00,0x03,0x00,0xa0,0x00,0x00,0x00, -0x9f,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xa2,0x00,0x00,0x00,0x50,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa6,0x00,0x00,0x00, -0x47,0x00,0x00,0x00,0x70,0x00,0x00,0x00,0x41,0x00,0x08,0x00, -0x69,0x00,0x00,0x00,0xa7,0x00,0x00,0x00,0x63,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0xa6,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x56,0x00,0x00,0x00, -0xa8,0x00,0x00,0x00,0xa7,0x00,0x00,0x00,0x71,0x00,0x04,0x00, -0x14,0x00,0x00,0x00,0xa9,0x00,0x00,0x00,0xa8,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xaa,0x00,0x00,0x00, -0xa9,0x00,0x00,0x00,0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xab,0x00,0x00,0x00,0xaa,0x00,0x00,0x00,0x89,0x00,0x00,0x00, -0xc2,0x00,0x05,0x00,0x56,0x00,0x00,0x00,0xad,0x00,0x00,0x00, -0x6b,0x00,0x00,0x00,0x70,0x00,0x00,0x00,0x71,0x00,0x04,0x00, -0x14,0x00,0x00,0x00,0xae,0x00,0x00,0x00,0xad,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xaf,0x00,0x00,0x00, -0xae,0x00,0x00,0x00,0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xb0,0x00,0x00,0x00,0xaf,0x00,0x00,0x00,0x8f,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xb1,0x00,0x00,0x00, -0xab,0x00,0x00,0x00,0xb0,0x00,0x00,0x00,0x6f,0x00,0x04,0x00, -0x6c,0x00,0x00,0x00,0xb2,0x00,0x00,0x00,0xb1,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x56,0x00,0x00,0x00,0xb9,0x00,0x00,0x00, -0xa7,0x00,0x00,0x00,0xc2,0x00,0x05,0x00,0x56,0x00,0x00,0x00, -0xba,0x00,0x00,0x00,0xb9,0x00,0x00,0x00,0x9a,0x00,0x00,0x00, -0x70,0x00,0x04,0x00,0x6c,0x00,0x00,0x00,0xbb,0x00,0x00,0x00, -0xba,0x00,0x00,0x00,0x85,0x00,0x05,0x00,0x6c,0x00,0x00,0x00, -0xbc,0x00,0x00,0x00,0x7a,0x00,0x00,0x00,0xbb,0x00,0x00,0x00, -0x7f,0x00,0x04,0x00,0x6c,0x00,0x00,0x00,0x10,0x01,0x00,0x00, -0xbc,0x00,0x00,0x00,0x0c,0x00,0x08,0x00,0x6c,0x00,0x00,0x00, -0xbd,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00, -0x74,0x00,0x00,0x00,0xb2,0x00,0x00,0x00,0x10,0x01,0x00,0x00, -0x73,0x00,0x04,0x00,0x5d,0x00,0x00,0x00,0xbe,0x00,0x00,0x00, -0xbd,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x71,0x00,0x00,0x00, -0xbf,0x00,0x00,0x00,0x7e,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0xa2,0x00,0x00,0x00,0x3e,0x00,0x03,0x00,0xbf,0x00,0x00,0x00, -0xbe,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xc2,0x00,0x00,0x00,0x50,0x00,0x00,0x00,0xc1,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xc6,0x00,0x00,0x00, -0x47,0x00,0x00,0x00,0x9a,0x00,0x00,0x00,0x41,0x00,0x08,0x00, -0x69,0x00,0x00,0x00,0xc7,0x00,0x00,0x00,0x63,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0xc6,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x56,0x00,0x00,0x00, -0xc8,0x00,0x00,0x00,0xc7,0x00,0x00,0x00,0x71,0x00,0x04,0x00, -0x14,0x00,0x00,0x00,0xc9,0x00,0x00,0x00,0xc8,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xca,0x00,0x00,0x00, -0xc9,0x00,0x00,0x00,0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xcb,0x00,0x00,0x00,0xca,0x00,0x00,0x00,0x89,0x00,0x00,0x00, -0xc2,0x00,0x05,0x00,0x56,0x00,0x00,0x00,0xcd,0x00,0x00,0x00, -0x6b,0x00,0x00,0x00,0x9a,0x00,0x00,0x00,0x71,0x00,0x04,0x00, -0x14,0x00,0x00,0x00,0xce,0x00,0x00,0x00,0xcd,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xcf,0x00,0x00,0x00, -0xce,0x00,0x00,0x00,0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xd0,0x00,0x00,0x00,0xcf,0x00,0x00,0x00,0x8f,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xd1,0x00,0x00,0x00, -0xcb,0x00,0x00,0x00,0xd0,0x00,0x00,0x00,0x6f,0x00,0x04,0x00, -0x6c,0x00,0x00,0x00,0xd2,0x00,0x00,0x00,0xd1,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x56,0x00,0x00,0x00,0xd9,0x00,0x00,0x00, -0xc7,0x00,0x00,0x00,0xc2,0x00,0x05,0x00,0x56,0x00,0x00,0x00, -0xda,0x00,0x00,0x00,0xd9,0x00,0x00,0x00,0x9a,0x00,0x00,0x00, -0x70,0x00,0x04,0x00,0x6c,0x00,0x00,0x00,0xdb,0x00,0x00,0x00, -0xda,0x00,0x00,0x00,0x85,0x00,0x05,0x00,0x6c,0x00,0x00,0x00, -0xdc,0x00,0x00,0x00,0x7a,0x00,0x00,0x00,0xdb,0x00,0x00,0x00, -0x7f,0x00,0x04,0x00,0x6c,0x00,0x00,0x00,0x11,0x01,0x00,0x00, -0xdc,0x00,0x00,0x00,0x0c,0x00,0x08,0x00,0x6c,0x00,0x00,0x00, -0xdd,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00, -0x74,0x00,0x00,0x00,0xd2,0x00,0x00,0x00,0x11,0x01,0x00,0x00, -0x73,0x00,0x04,0x00,0x5d,0x00,0x00,0x00,0xde,0x00,0x00,0x00, -0xdd,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x71,0x00,0x00,0x00, -0xdf,0x00,0x00,0x00,0x7e,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0xc2,0x00,0x00,0x00,0x3e,0x00,0x03,0x00,0xdf,0x00,0x00,0x00, -0xde,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xe2,0x00,0x00,0x00,0x50,0x00,0x00,0x00,0xe1,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xe7,0x00,0x00,0x00, -0x47,0x00,0x00,0x00,0xe6,0x00,0x00,0x00,0x41,0x00,0x08,0x00, -0x69,0x00,0x00,0x00,0xe8,0x00,0x00,0x00,0x63,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0xe7,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x56,0x00,0x00,0x00, -0xe9,0x00,0x00,0x00,0xe8,0x00,0x00,0x00,0x71,0x00,0x04,0x00, -0x14,0x00,0x00,0x00,0xea,0x00,0x00,0x00,0xe9,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xeb,0x00,0x00,0x00, -0xea,0x00,0x00,0x00,0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xec,0x00,0x00,0x00,0xeb,0x00,0x00,0x00,0x89,0x00,0x00,0x00, -0xc2,0x00,0x05,0x00,0x56,0x00,0x00,0x00,0xee,0x00,0x00,0x00, -0x6b,0x00,0x00,0x00,0xe6,0x00,0x00,0x00,0x71,0x00,0x04,0x00, -0x14,0x00,0x00,0x00,0xef,0x00,0x00,0x00,0xee,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xf0,0x00,0x00,0x00, -0xef,0x00,0x00,0x00,0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xf1,0x00,0x00,0x00,0xf0,0x00,0x00,0x00,0x8f,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xf2,0x00,0x00,0x00, -0xec,0x00,0x00,0x00,0xf1,0x00,0x00,0x00,0x6f,0x00,0x04,0x00, -0x6c,0x00,0x00,0x00,0xf3,0x00,0x00,0x00,0xf2,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x56,0x00,0x00,0x00,0xfa,0x00,0x00,0x00, -0xe8,0x00,0x00,0x00,0xc2,0x00,0x05,0x00,0x56,0x00,0x00,0x00, -0xfb,0x00,0x00,0x00,0xfa,0x00,0x00,0x00,0x9a,0x00,0x00,0x00, -0x70,0x00,0x04,0x00,0x6c,0x00,0x00,0x00,0xfc,0x00,0x00,0x00, -0xfb,0x00,0x00,0x00,0x85,0x00,0x05,0x00,0x6c,0x00,0x00,0x00, -0xfd,0x00,0x00,0x00,0x7a,0x00,0x00,0x00,0xfc,0x00,0x00,0x00, -0x7f,0x00,0x04,0x00,0x6c,0x00,0x00,0x00,0x12,0x01,0x00,0x00, -0xfd,0x00,0x00,0x00,0x0c,0x00,0x08,0x00,0x6c,0x00,0x00,0x00, -0xfe,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00, -0x74,0x00,0x00,0x00,0xf3,0x00,0x00,0x00,0x12,0x01,0x00,0x00, -0x73,0x00,0x04,0x00,0x5d,0x00,0x00,0x00,0xff,0x00,0x00,0x00, -0xfe,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x71,0x00,0x00,0x00, -0x00,0x01,0x00,0x00,0x7e,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0xe2,0x00,0x00,0x00,0x3e,0x00,0x03,0x00,0x00,0x01,0x00,0x00, -0xff,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x0d,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0x0d,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x02,0x01,0x00,0x00,0x0c,0x01,0x00,0x00, -0x29,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x0a,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0x0c,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, -0x11,0x00,0x00,0x00,0x0d,0x01,0x00,0x00,0x06,0x01,0x00,0x00, -0x0a,0x00,0x00,0x00,0x09,0x01,0x00,0x00,0x2f,0x00,0x00,0x00, -0xf7,0x00,0x03,0x00,0x0a,0x01,0x00,0x00,0x00,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x0d,0x01,0x00,0x00,0x04,0x01,0x00,0x00, -0x0a,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x0a,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0x04,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x04,0x01,0x00,0x00,0xfd,0x00,0x01,0x00,0x38,0x00,0x01,0x00, - -}; -const uint64_t dequant_q2_K_len = 4056; - -unsigned char dequant_q3_K_data[] = { -0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, -0x42,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, -0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x27,0x00,0x00,0x00, -0x11,0x00,0x02,0x00,0x51,0x11,0x00,0x00,0x11,0x00,0x02,0x00, -0x60,0x11,0x00,0x00,0x0b,0x00,0x06,0x00,0x01,0x00,0x00,0x00, -0x47,0x4c,0x53,0x4c,0x2e,0x73,0x74,0x64,0x2e,0x34,0x35,0x30, -0x00,0x00,0x00,0x00,0x0e,0x00,0x03,0x00,0x00,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x0f,0x00,0x0a,0x00,0x05,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x6d,0x61,0x69,0x6e,0x00,0x00,0x00,0x00, -0x17,0x00,0x00,0x00,0x25,0x00,0x00,0x00,0x33,0x00,0x00,0x00, -0x7b,0x00,0x00,0x00,0x09,0x01,0x00,0x00,0x10,0x00,0x06,0x00, -0x04,0x00,0x00,0x00,0x11,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x17,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x1a,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x23,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x23,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x08,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x23,0x00,0x00,0x00,0x03,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x47,0x00,0x03,0x00, -0x23,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x33,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x1b,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x71,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x73,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x75,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x77,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x77,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x77,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x60,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x77,0x00,0x00,0x00,0x03,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x78,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x6e,0x00,0x00,0x00, -0x48,0x00,0x04,0x00,0x79,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x79,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x03,0x00,0x79,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x7b,0x00,0x00,0x00,0x22,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x7b,0x00,0x00,0x00, -0x21,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x06,0x01,0x00,0x00,0x06,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x48,0x00,0x04,0x00,0x07,0x01,0x00,0x00,0x00,0x00,0x00,0x00, -0x19,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x07,0x01,0x00,0x00, -0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x03,0x00,0x07,0x01,0x00,0x00,0x02,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x09,0x01,0x00,0x00,0x22,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x09,0x01,0x00,0x00, -0x21,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x2f,0x01,0x00,0x00,0x0b,0x00,0x00,0x00,0x19,0x00,0x00,0x00, -0x13,0x00,0x02,0x00,0x02,0x00,0x00,0x00,0x21,0x00,0x03,0x00, -0x03,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x15,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x10,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x14,0x00,0x02,0x00, -0x11,0x00,0x00,0x00,0x15,0x00,0x04,0x00,0x14,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x17,0x00,0x04,0x00, -0x15,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x03,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x16,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x15,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x16,0x00,0x00,0x00, -0x17,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x14,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x19,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x14,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x14,0x00,0x00,0x00, -0x1c,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x1e,0x00,0x06,0x00, -0x23,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x24,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x24,0x00,0x00,0x00,0x25,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x26,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x29,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x16,0x00,0x00,0x00,0x33,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x37,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x3b,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x41,0x00,0x00,0x00, -0x10,0x00,0x00,0x00,0x15,0x00,0x04,0x00,0x52,0x00,0x00,0x00, -0x08,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x15,0x00,0x04,0x00, -0x5a,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x5e,0x00,0x00,0x00, -0x08,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x14,0x00,0x00,0x00, -0x70,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x1c,0x00,0x04,0x00, -0x71,0x00,0x00,0x00,0x52,0x00,0x00,0x00,0x70,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0x72,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0x1c,0x00,0x04,0x00,0x73,0x00,0x00,0x00, -0x52,0x00,0x00,0x00,0x72,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x14,0x00,0x00,0x00,0x74,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x1c,0x00,0x04,0x00,0x75,0x00,0x00,0x00,0x52,0x00,0x00,0x00, -0x74,0x00,0x00,0x00,0x16,0x00,0x03,0x00,0x76,0x00,0x00,0x00, -0x10,0x00,0x00,0x00,0x1e,0x00,0x06,0x00,0x77,0x00,0x00,0x00, -0x71,0x00,0x00,0x00,0x73,0x00,0x00,0x00,0x75,0x00,0x00,0x00, -0x76,0x00,0x00,0x00,0x1d,0x00,0x03,0x00,0x78,0x00,0x00,0x00, -0x77,0x00,0x00,0x00,0x1e,0x00,0x03,0x00,0x79,0x00,0x00,0x00, -0x78,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x7a,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x79,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x7a,0x00,0x00,0x00,0x7b,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x7f,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x52,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x8e,0x00,0x00,0x00,0x03,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xad,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xd3,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x16,0x00,0x03,0x00, -0xde,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0xe2,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x76,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xea,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xf1,0x00,0x00,0x00,0x80,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, -0x06,0x01,0x00,0x00,0x76,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, -0x07,0x01,0x00,0x00,0x06,0x01,0x00,0x00,0x20,0x00,0x04,0x00, -0x08,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x07,0x01,0x00,0x00, -0x3b,0x00,0x04,0x00,0x08,0x01,0x00,0x00,0x09,0x01,0x00,0x00, -0x0c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x14,0x00,0x00,0x00, -0x2e,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x2c,0x00,0x06,0x00, -0x15,0x00,0x00,0x00,0x2f,0x01,0x00,0x00,0x72,0x00,0x00,0x00, -0x2e,0x01,0x00,0x00,0x2e,0x01,0x00,0x00,0x2a,0x00,0x03,0x00, -0x11,0x00,0x00,0x00,0x32,0x01,0x00,0x00,0x29,0x00,0x03,0x00, -0x11,0x00,0x00,0x00,0x35,0x01,0x00,0x00,0x36,0x00,0x05,0x00, -0x02,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x03,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x05,0x00,0x00,0x00, -0xf7,0x00,0x03,0x00,0x30,0x01,0x00,0x00,0x00,0x00,0x00,0x00, -0xfb,0x00,0x03,0x00,0x18,0x00,0x00,0x00,0x31,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x31,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0x0a,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x0a,0x00,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x38,0x01,0x00,0x00, -0x09,0x00,0x00,0x00,0x31,0x01,0x00,0x00,0x2d,0x01,0x00,0x00, -0x0d,0x00,0x00,0x00,0xb1,0x00,0x05,0x00,0x11,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x38,0x01,0x00,0x00,0x10,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0x0c,0x00,0x00,0x00,0x0d,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x12,0x00,0x00,0x00, -0x0b,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x0b,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x19,0x00,0x00,0x00, -0x1a,0x00,0x00,0x00,0x17,0x00,0x00,0x00,0x18,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0x1b,0x00,0x00,0x00, -0x1a,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x14,0x00,0x00,0x00, -0x1d,0x00,0x00,0x00,0x1b,0x00,0x00,0x00,0x1c,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0x1f,0x00,0x00,0x00, -0x38,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x14,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x1d,0x00,0x00,0x00,0x1f,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x26,0x00,0x00,0x00, -0x27,0x00,0x00,0x00,0x25,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x28,0x00,0x00,0x00, -0x27,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x26,0x00,0x00,0x00, -0x2a,0x00,0x00,0x00,0x25,0x00,0x00,0x00,0x29,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x2b,0x00,0x00,0x00, -0x2a,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x2c,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0x2b,0x00,0x00,0x00, -0x87,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x2d,0x00,0x00,0x00, -0x2c,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0xaf,0x00,0x05,0x00, -0x11,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x2d,0x00,0x00,0x00,0xf7,0x00,0x03,0x00,0x30,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x2e,0x00,0x00,0x00, -0x2f,0x00,0x00,0x00,0x30,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x2f,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x0c,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0x30,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x19,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0x33,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x14,0x00,0x00,0x00, -0x35,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x36,0x00,0x00,0x00,0x35,0x00,0x00,0x00, -0x87,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x38,0x00,0x00,0x00, -0x36,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x87,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x3c,0x00,0x00,0x00,0x38,0x00,0x00,0x00, -0x3b,0x00,0x00,0x00,0x8b,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x3f,0x00,0x00,0x00,0x38,0x00,0x00,0x00,0x3b,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x43,0x00,0x00,0x00, -0x41,0x00,0x00,0x00,0x3f,0x00,0x00,0x00,0x8b,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x47,0x00,0x00,0x00,0x36,0x00,0x00,0x00, -0x37,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x48,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x47,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x49,0x00,0x00,0x00, -0x43,0x00,0x00,0x00,0x48,0x00,0x00,0x00,0x87,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x4c,0x00,0x00,0x00,0x3c,0x00,0x00,0x00, -0x37,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x50,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x4c,0x00,0x00,0x00, -0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x51,0x00,0x00,0x00, -0x3c,0x00,0x00,0x00,0x50,0x00,0x00,0x00,0xc4,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x59,0x00,0x00,0x00,0x29,0x00,0x00,0x00, -0x3c,0x00,0x00,0x00,0x72,0x00,0x04,0x00,0x5a,0x00,0x00,0x00, -0x5b,0x00,0x00,0x00,0x59,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x52,0x00,0x00,0x00,0x5c,0x00,0x00,0x00,0x5b,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x60,0x00,0x00,0x00, -0x5e,0x00,0x00,0x00,0x4c,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x3b,0x00,0x00,0x00, -0x51,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x63,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x65,0x00,0x00,0x00, -0x63,0x00,0x00,0x00,0x3f,0x00,0x00,0x00,0xb1,0x00,0x05,0x00, -0x11,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x65,0x00,0x00,0x00, -0x37,0x00,0x00,0x00,0xf7,0x00,0x03,0x00,0x6f,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x6c,0x00,0x00,0x00, -0x6e,0x00,0x00,0x00,0x92,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x6e,0x00,0x00,0x00,0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x7e,0x00,0x00,0x00,0x65,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x41,0x00,0x08,0x00,0x7f,0x00,0x00,0x00,0x80,0x00,0x00,0x00, -0x7b,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x3b,0x00,0x00,0x00,0x7e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x52,0x00,0x00,0x00,0x81,0x00,0x00,0x00,0x80,0x00,0x00,0x00, -0x71,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0x82,0x00,0x00,0x00, -0x81,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x83,0x00,0x00,0x00,0x82,0x00,0x00,0x00,0xc7,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x85,0x00,0x00,0x00,0x83,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x88,0x00,0x00,0x00,0x65,0x00,0x00,0x00,0x5e,0x00,0x00,0x00, -0x41,0x00,0x08,0x00,0x7f,0x00,0x00,0x00,0x89,0x00,0x00,0x00, -0x7b,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x3b,0x00,0x00,0x00,0x88,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x52,0x00,0x00,0x00,0x8a,0x00,0x00,0x00,0x89,0x00,0x00,0x00, -0xc2,0x00,0x05,0x00,0x52,0x00,0x00,0x00,0x8b,0x00,0x00,0x00, -0x8a,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x71,0x00,0x04,0x00, -0x14,0x00,0x00,0x00,0x8c,0x00,0x00,0x00,0x8b,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x8d,0x00,0x00,0x00, -0x8c,0x00,0x00,0x00,0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x8f,0x00,0x00,0x00,0x8d,0x00,0x00,0x00,0x8e,0x00,0x00,0x00, -0xc4,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x90,0x00,0x00,0x00, -0x8f,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0xc5,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x91,0x00,0x00,0x00,0x85,0x00,0x00,0x00, -0x90,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x6f,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0x92,0x00,0x00,0x00,0xb1,0x00,0x05,0x00, -0x11,0x00,0x00,0x00,0x94,0x00,0x00,0x00,0x65,0x00,0x00,0x00, -0x5e,0x00,0x00,0x00,0xf7,0x00,0x03,0x00,0x97,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x94,0x00,0x00,0x00, -0x96,0x00,0x00,0x00,0xab,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x96,0x00,0x00,0x00,0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x9a,0x00,0x00,0x00,0x65,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x41,0x00,0x08,0x00,0x7f,0x00,0x00,0x00,0x9b,0x00,0x00,0x00, -0x7b,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x3b,0x00,0x00,0x00,0x9a,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x52,0x00,0x00,0x00,0x9c,0x00,0x00,0x00,0x9b,0x00,0x00,0x00, -0x71,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0x9d,0x00,0x00,0x00, -0x9c,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x9e,0x00,0x00,0x00,0x9d,0x00,0x00,0x00,0xc7,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x9f,0x00,0x00,0x00,0x9e,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xa2,0x00,0x00,0x00,0x65,0x00,0x00,0x00,0x37,0x00,0x00,0x00, -0x41,0x00,0x08,0x00,0x7f,0x00,0x00,0x00,0xa3,0x00,0x00,0x00, -0x7b,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x3b,0x00,0x00,0x00,0xa2,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x52,0x00,0x00,0x00,0xa4,0x00,0x00,0x00,0xa3,0x00,0x00,0x00, -0xc2,0x00,0x05,0x00,0x52,0x00,0x00,0x00,0xa5,0x00,0x00,0x00, -0xa4,0x00,0x00,0x00,0x3b,0x00,0x00,0x00,0x71,0x00,0x04,0x00, -0x14,0x00,0x00,0x00,0xa6,0x00,0x00,0x00,0xa5,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xa7,0x00,0x00,0x00, -0xa6,0x00,0x00,0x00,0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xa8,0x00,0x00,0x00,0xa7,0x00,0x00,0x00,0x8e,0x00,0x00,0x00, -0xc4,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa9,0x00,0x00,0x00, -0xa8,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0xc5,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xaa,0x00,0x00,0x00,0x9f,0x00,0x00,0x00, -0xa9,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x97,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xab,0x00,0x00,0x00,0xb1,0x00,0x05,0x00, -0x11,0x00,0x00,0x00,0xae,0x00,0x00,0x00,0x65,0x00,0x00,0x00, -0xad,0x00,0x00,0x00,0xf7,0x00,0x03,0x00,0xb1,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xae,0x00,0x00,0x00, -0xb0,0x00,0x00,0x00,0xc5,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xb0,0x00,0x00,0x00,0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xb4,0x00,0x00,0x00,0x65,0x00,0x00,0x00,0x5e,0x00,0x00,0x00, -0x41,0x00,0x08,0x00,0x7f,0x00,0x00,0x00,0xb5,0x00,0x00,0x00, -0x7b,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x3b,0x00,0x00,0x00,0xb4,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x52,0x00,0x00,0x00,0xb6,0x00,0x00,0x00,0xb5,0x00,0x00,0x00, -0xc2,0x00,0x05,0x00,0x52,0x00,0x00,0x00,0xb7,0x00,0x00,0x00, -0xb6,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x71,0x00,0x04,0x00, -0x14,0x00,0x00,0x00,0xb8,0x00,0x00,0x00,0xb7,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xb9,0x00,0x00,0x00, -0xb8,0x00,0x00,0x00,0x41,0x00,0x08,0x00,0x7f,0x00,0x00,0x00, -0xbd,0x00,0x00,0x00,0x7b,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x21,0x00,0x00,0x00,0x3b,0x00,0x00,0x00,0x65,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x52,0x00,0x00,0x00,0xbe,0x00,0x00,0x00, -0xbd,0x00,0x00,0x00,0xc2,0x00,0x05,0x00,0x52,0x00,0x00,0x00, -0xbf,0x00,0x00,0x00,0xbe,0x00,0x00,0x00,0x37,0x00,0x00,0x00, -0x71,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0xc0,0x00,0x00,0x00, -0xbf,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xc1,0x00,0x00,0x00,0xc0,0x00,0x00,0x00,0xc7,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xc2,0x00,0x00,0x00,0xc1,0x00,0x00,0x00, -0x8e,0x00,0x00,0x00,0xc4,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xc3,0x00,0x00,0x00,0xc2,0x00,0x00,0x00,0x37,0x00,0x00,0x00, -0xc5,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xc4,0x00,0x00,0x00, -0xb9,0x00,0x00,0x00,0xc3,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xb1,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xc5,0x00,0x00,0x00, -0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xc8,0x00,0x00,0x00, -0x65,0x00,0x00,0x00,0x5e,0x00,0x00,0x00,0x41,0x00,0x08,0x00, -0x7f,0x00,0x00,0x00,0xc9,0x00,0x00,0x00,0x7b,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x3b,0x00,0x00,0x00, -0xc8,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x52,0x00,0x00,0x00, -0xca,0x00,0x00,0x00,0xc9,0x00,0x00,0x00,0xc2,0x00,0x05,0x00, -0x52,0x00,0x00,0x00,0xcb,0x00,0x00,0x00,0xca,0x00,0x00,0x00, -0x37,0x00,0x00,0x00,0x71,0x00,0x04,0x00,0x14,0x00,0x00,0x00, -0xcc,0x00,0x00,0x00,0xcb,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xcd,0x00,0x00,0x00,0xcc,0x00,0x00,0x00, -0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xd0,0x00,0x00,0x00, -0x65,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x41,0x00,0x08,0x00, -0x7f,0x00,0x00,0x00,0xd1,0x00,0x00,0x00,0x7b,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x3b,0x00,0x00,0x00, -0xd0,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x52,0x00,0x00,0x00, -0xd2,0x00,0x00,0x00,0xd1,0x00,0x00,0x00,0xc2,0x00,0x05,0x00, -0x52,0x00,0x00,0x00,0xd4,0x00,0x00,0x00,0xd2,0x00,0x00,0x00, -0xd3,0x00,0x00,0x00,0x71,0x00,0x04,0x00,0x14,0x00,0x00,0x00, -0xd5,0x00,0x00,0x00,0xd4,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xd6,0x00,0x00,0x00,0xd5,0x00,0x00,0x00, -0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xd7,0x00,0x00,0x00, -0xd6,0x00,0x00,0x00,0x8e,0x00,0x00,0x00,0xc4,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xd8,0x00,0x00,0x00,0xd7,0x00,0x00,0x00, -0x37,0x00,0x00,0x00,0xc5,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xd9,0x00,0x00,0x00,0xcd,0x00,0x00,0x00,0xd8,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xb1,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xb1,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0x39,0x01,0x00,0x00,0xc4,0x00,0x00,0x00,0xb0,0x00,0x00,0x00, -0xd9,0x00,0x00,0x00,0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x97,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x97,0x00,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x3a,0x01,0x00,0x00, -0xaa,0x00,0x00,0x00,0x96,0x00,0x00,0x00,0x39,0x01,0x00,0x00, -0xb1,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x6f,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0x6f,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0x3b,0x01,0x00,0x00,0x91,0x00,0x00,0x00, -0x6e,0x00,0x00,0x00,0x3a,0x01,0x00,0x00,0x97,0x00,0x00,0x00, -0x72,0x00,0x04,0x00,0x5a,0x00,0x00,0x00,0xdd,0x00,0x00,0x00, -0x3b,0x01,0x00,0x00,0x41,0x00,0x07,0x00,0xe2,0x00,0x00,0x00, -0xe3,0x00,0x00,0x00,0x7b,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x21,0x00,0x00,0x00,0x8e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x76,0x00,0x00,0x00,0xe4,0x00,0x00,0x00,0xe3,0x00,0x00,0x00, -0x73,0x00,0x04,0x00,0xde,0x00,0x00,0x00,0xe5,0x00,0x00,0x00, -0xe4,0x00,0x00,0x00,0x72,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xe9,0x00,0x00,0x00,0xdd,0x00,0x00,0x00,0x82,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xeb,0x00,0x00,0x00,0xe9,0x00,0x00,0x00, -0xea,0x00,0x00,0x00,0x6f,0x00,0x04,0x00,0xde,0x00,0x00,0x00, -0xec,0x00,0x00,0x00,0xeb,0x00,0x00,0x00,0x85,0x00,0x05,0x00, -0xde,0x00,0x00,0x00,0xed,0x00,0x00,0x00,0xe5,0x00,0x00,0x00, -0xec,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xf0,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x10,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xf3,0x00,0x00,0x00, -0xf1,0x00,0x00,0x00,0x4c,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xf4,0x00,0x00,0x00,0xf0,0x00,0x00,0x00, -0xf3,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xf6,0x00,0x00,0x00,0xea,0x00,0x00,0x00,0x51,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xf7,0x00,0x00,0x00, -0xf4,0x00,0x00,0x00,0xf6,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xfa,0x00,0x00,0x00,0xea,0x00,0x00,0x00, -0x4c,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xfd,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xfd,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0x3c,0x01,0x00,0x00,0x49,0x00,0x00,0x00, -0x6f,0x00,0x00,0x00,0x2b,0x01,0x00,0x00,0xfe,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x04,0x01,0x00,0x00, -0x49,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0xb1,0x00,0x05,0x00, -0x11,0x00,0x00,0x00,0x05,0x01,0x00,0x00,0x3c,0x01,0x00,0x00, -0x04,0x01,0x00,0x00,0xf6,0x00,0x04,0x00,0xff,0x00,0x00,0x00, -0xfe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x05,0x01,0x00,0x00,0xfe,0x00,0x00,0x00,0xff,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xfe,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x0c,0x01,0x00,0x00,0xf7,0x00,0x00,0x00, -0x3c,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x11,0x01,0x00,0x00,0xfa,0x00,0x00,0x00,0x3c,0x01,0x00,0x00, -0x41,0x00,0x08,0x00,0x7f,0x00,0x00,0x00,0x12,0x01,0x00,0x00, -0x7b,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x29,0x00,0x00,0x00,0x11,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, -0x52,0x00,0x00,0x00,0x13,0x01,0x00,0x00,0x12,0x01,0x00,0x00, -0xc2,0x00,0x05,0x00,0x52,0x00,0x00,0x00,0x15,0x01,0x00,0x00, -0x13,0x01,0x00,0x00,0x62,0x00,0x00,0x00,0x71,0x00,0x04,0x00, -0x14,0x00,0x00,0x00,0x16,0x01,0x00,0x00,0x15,0x01,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x17,0x01,0x00,0x00, -0x16,0x01,0x00,0x00,0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x18,0x01,0x00,0x00,0x17,0x01,0x00,0x00,0x8e,0x00,0x00,0x00, -0x72,0x00,0x04,0x00,0x5a,0x00,0x00,0x00,0x19,0x01,0x00,0x00, -0x18,0x01,0x00,0x00,0x72,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x1a,0x01,0x00,0x00,0x19,0x01,0x00,0x00,0x41,0x00,0x08,0x00, -0x7f,0x00,0x00,0x00,0x1d,0x01,0x00,0x00,0x7b,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x3c,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0x52,0x00,0x00,0x00, -0x1e,0x01,0x00,0x00,0x1d,0x01,0x00,0x00,0xc7,0x00,0x05,0x00, -0x52,0x00,0x00,0x00,0x20,0x01,0x00,0x00,0x1e,0x01,0x00,0x00, -0x5c,0x00,0x00,0x00,0x71,0x00,0x04,0x00,0x14,0x00,0x00,0x00, -0x21,0x01,0x00,0x00,0x20,0x01,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x22,0x01,0x00,0x00,0x21,0x01,0x00,0x00, -0xab,0x00,0x05,0x00,0x11,0x00,0x00,0x00,0x23,0x01,0x00,0x00, -0x22,0x01,0x00,0x00,0x09,0x00,0x00,0x00,0xa9,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x24,0x01,0x00,0x00,0x23,0x01,0x00,0x00, -0x09,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x82,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x25,0x01,0x00,0x00,0x1a,0x01,0x00,0x00, -0x24,0x01,0x00,0x00,0x6f,0x00,0x04,0x00,0xde,0x00,0x00,0x00, -0x26,0x01,0x00,0x00,0x25,0x01,0x00,0x00,0x85,0x00,0x05,0x00, -0xde,0x00,0x00,0x00,0x27,0x01,0x00,0x00,0xed,0x00,0x00,0x00, -0x26,0x01,0x00,0x00,0x73,0x00,0x04,0x00,0x76,0x00,0x00,0x00, -0x28,0x01,0x00,0x00,0x27,0x01,0x00,0x00,0x41,0x00,0x06,0x00, -0xe2,0x00,0x00,0x00,0x29,0x01,0x00,0x00,0x09,0x01,0x00,0x00, -0x09,0x00,0x00,0x00,0x0c,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0x29,0x01,0x00,0x00,0x28,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x2b,0x01,0x00,0x00,0x3c,0x01,0x00,0x00, -0x29,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xfd,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xff,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x0d,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x0d,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x2d,0x01,0x00,0x00, -0x38,0x01,0x00,0x00,0x29,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x0a,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x0c,0x00,0x00,0x00, -0xf5,0x00,0x07,0x00,0x11,0x00,0x00,0x00,0x41,0x01,0x00,0x00, -0x32,0x01,0x00,0x00,0x0a,0x00,0x00,0x00,0x35,0x01,0x00,0x00, -0x2f,0x00,0x00,0x00,0xf7,0x00,0x03,0x00,0x36,0x01,0x00,0x00, -0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x41,0x01,0x00,0x00, -0x30,0x01,0x00,0x00,0x36,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x36,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x30,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x30,0x01,0x00,0x00,0xfd,0x00,0x01,0x00, -0x38,0x00,0x01,0x00, -}; -const uint64_t dequant_q3_K_len = 4828; - -unsigned char dequant_q4_0_data[] = { -0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, -0x19,0x03,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, -0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x51,0x11,0x00,0x00, -0x11,0x00,0x02,0x00,0x60,0x11,0x00,0x00,0x0b,0x00,0x06,0x00, -0x01,0x00,0x00,0x00,0x47,0x4c,0x53,0x4c,0x2e,0x73,0x74,0x64, -0x2e,0x34,0x35,0x30,0x00,0x00,0x00,0x00,0x0e,0x00,0x03,0x00, -0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x0f,0x00,0x09,0x00, -0x05,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x6d,0x61,0x69,0x6e, -0x00,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0x55,0x00,0x00,0x00,0x78,0x00,0x00,0x00,0x10,0x00,0x06,0x00, -0x04,0x00,0x00,0x00,0x11,0x00,0x00,0x00,0x00,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x0c,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x1c,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x14,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x14,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x14,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x08,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x14,0x00,0x00,0x00,0x03,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x47,0x00,0x03,0x00, -0x14,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x50,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x51,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x51,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x52,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x48,0x00,0x04,0x00, -0x53,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x53,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, -0x53,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x55,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x55,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x75,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x48,0x00,0x04,0x00, -0x76,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x19,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x76,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, -0x76,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x78,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x78,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x9a,0x00,0x00,0x00, -0x0b,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x13,0x00,0x02,0x00, -0x02,0x00,0x00,0x00,0x21,0x00,0x03,0x00,0x03,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x15,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x15,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x17,0x00,0x04,0x00,0x0a,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x03,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x0b,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x0a,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x0b,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x0d,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x0e,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x1e,0x00,0x06,0x00, -0x14,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x15,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x14,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x15,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x17,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x18,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x1b,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x14,0x00,0x02,0x00,0x24,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x37,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x48,0x00,0x00,0x00,0x10,0x00,0x00,0x00, -0x16,0x00,0x03,0x00,0x4a,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x16,0x00,0x03,0x00,0x4d,0x00,0x00,0x00,0x10,0x00,0x00,0x00, -0x15,0x00,0x04,0x00,0x4e,0x00,0x00,0x00,0x08,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x4f,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x1c,0x00,0x04,0x00, -0x50,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x4f,0x00,0x00,0x00, -0x1e,0x00,0x04,0x00,0x51,0x00,0x00,0x00,0x4d,0x00,0x00,0x00, -0x50,0x00,0x00,0x00,0x1d,0x00,0x03,0x00,0x52,0x00,0x00,0x00, -0x51,0x00,0x00,0x00,0x1e,0x00,0x03,0x00,0x53,0x00,0x00,0x00, -0x52,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x54,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x54,0x00,0x00,0x00,0x55,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x57,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x4d,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x5f,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x17,0x00,0x04,0x00, -0x63,0x00,0x00,0x00,0x4a,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x67,0x00,0x00,0x00, -0x0f,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x6b,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x4a,0x00,0x00,0x00,0x70,0x00,0x00,0x00,0x00,0x00,0x00,0x41, -0x1d,0x00,0x03,0x00,0x75,0x00,0x00,0x00,0x4d,0x00,0x00,0x00, -0x1e,0x00,0x03,0x00,0x76,0x00,0x00,0x00,0x75,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x77,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x76,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x77,0x00,0x00,0x00, -0x78,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x7a,0x00,0x00,0x00,0x03,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x92,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x99,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x2c,0x00,0x06,0x00, -0x0a,0x00,0x00,0x00,0x9a,0x00,0x00,0x00,0x99,0x00,0x00,0x00, -0x92,0x00,0x00,0x00,0x92,0x00,0x00,0x00,0x2c,0x00,0x05,0x00, -0x63,0x00,0x00,0x00,0xa5,0x00,0x00,0x00,0x70,0x00,0x00,0x00, -0x70,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xff,0x02,0x00,0x00,0x11,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x00,0x03,0x00,0x00,0x12,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x01,0x03,0x00,0x00, -0x13,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x02,0x03,0x00,0x00,0x14,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x03,0x03,0x00,0x00,0x05,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x04,0x03,0x00,0x00, -0x15,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x05,0x03,0x00,0x00,0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x06,0x03,0x00,0x00,0x16,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x07,0x03,0x00,0x00, -0x07,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x08,0x03,0x00,0x00,0x17,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x09,0x03,0x00,0x00,0x08,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x0a,0x03,0x00,0x00, -0x18,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x0b,0x03,0x00,0x00,0x09,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x0c,0x03,0x00,0x00,0x19,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x0d,0x03,0x00,0x00, -0x0a,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x0e,0x03,0x00,0x00,0x1a,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x0f,0x03,0x00,0x00,0x0b,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x10,0x03,0x00,0x00, -0x1b,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x11,0x03,0x00,0x00,0x0c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x12,0x03,0x00,0x00,0x1c,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x13,0x03,0x00,0x00, -0x0d,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x14,0x03,0x00,0x00,0x1d,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x15,0x03,0x00,0x00,0x0e,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x16,0x03,0x00,0x00, -0x1e,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x17,0x03,0x00,0x00,0x0f,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x18,0x03,0x00,0x00,0x1f,0x00,0x00,0x00, -0x36,0x00,0x05,0x00,0x02,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x05,0x00,0x00,0x00,0xf7,0x00,0x03,0x00,0x9b,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0xfb,0x00,0x03,0x00,0x0d,0x00,0x00,0x00, -0x9c,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x9c,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x0e,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x0d,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x11,0x00,0x00,0x00, -0x10,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x18,0x00,0x00,0x00, -0x19,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x17,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x1a,0x00,0x00,0x00, -0x19,0x00,0x00,0x00,0x87,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x1c,0x00,0x00,0x00,0x1a,0x00,0x00,0x00,0x1b,0x00,0x00,0x00, -0x8b,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x1d,0x00,0x00,0x00, -0x11,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x87,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x11,0x00,0x00,0x00, -0x1c,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x26,0x00,0x00,0x00,0x1d,0x00,0x00,0x00,0x1b,0x00,0x00,0x00, -0xaf,0x00,0x05,0x00,0x24,0x00,0x00,0x00,0x29,0x00,0x00,0x00, -0x26,0x00,0x00,0x00,0x1a,0x00,0x00,0x00,0xa8,0x00,0x04,0x00, -0x24,0x00,0x00,0x00,0x2a,0x00,0x00,0x00,0x29,0x00,0x00,0x00, -0xf7,0x00,0x03,0x00,0x2c,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x2a,0x00,0x00,0x00,0x2b,0x00,0x00,0x00, -0x2c,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x2b,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x18,0x00,0x00,0x00,0x2f,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x30,0x00,0x00,0x00,0x2f,0x00,0x00,0x00, -0xaf,0x00,0x05,0x00,0x24,0x00,0x00,0x00,0x31,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x30,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x2c,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x2c,0x00,0x00,0x00, -0xf5,0x00,0x07,0x00,0x24,0x00,0x00,0x00,0x32,0x00,0x00,0x00, -0x29,0x00,0x00,0x00,0x9c,0x00,0x00,0x00,0x31,0x00,0x00,0x00, -0x2b,0x00,0x00,0x00,0xf7,0x00,0x03,0x00,0x34,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x32,0x00,0x00,0x00, -0x33,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x33,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x9b,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0x34,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x18,0x00,0x00,0x00,0x38,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0x37,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x38,0x00,0x00,0x00,0x87,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x1b,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x3e,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x3e,0x00,0x00,0x00,0x1d,0x00,0x00,0x00,0x41,0x00,0x07,0x00, -0x57,0x00,0x00,0x00,0x58,0x00,0x00,0x00,0x55,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00,0x59,0x00,0x00,0x00, -0x58,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0x5a,0x00,0x00,0x00,0x59,0x00,0x00,0x00,0x41,0x00,0x08,0x00, -0x5f,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x55,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x17,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x4e,0x00,0x00,0x00, -0x61,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x71,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x61,0x00,0x00,0x00, -0xc7,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0x68,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x67,0x00,0x00,0x00,0x70,0x00,0x04,0x00, -0x4a,0x00,0x00,0x00,0x69,0x00,0x00,0x00,0x68,0x00,0x00,0x00, -0xc2,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x6b,0x00,0x00,0x00,0x70,0x00,0x04,0x00, -0x4a,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x50,0x00,0x05,0x00,0x63,0x00,0x00,0x00,0x6e,0x00,0x00,0x00, -0x69,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x83,0x00,0x05,0x00, -0x63,0x00,0x00,0x00,0x72,0x00,0x00,0x00,0x6e,0x00,0x00,0x00, -0xa5,0x00,0x00,0x00,0x8e,0x00,0x05,0x00,0x63,0x00,0x00,0x00, -0x74,0x00,0x00,0x00,0x72,0x00,0x00,0x00,0x5a,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x18,0x00,0x00,0x00,0x7b,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x7a,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x7c,0x00,0x00,0x00,0x7b,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x7d,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x7c,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x80,0x00,0x00,0x00,0x7d,0x00,0x00,0x00, -0x26,0x00,0x00,0x00,0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00, -0x85,0x00,0x00,0x00,0x74,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x73,0x00,0x04,0x00,0x4d,0x00,0x00,0x00,0x86,0x00,0x00,0x00, -0x85,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x57,0x00,0x00,0x00, -0x87,0x00,0x00,0x00,0x78,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x80,0x00,0x00,0x00,0x3e,0x00,0x03,0x00,0x87,0x00,0x00,0x00, -0x86,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x91,0x00,0x00,0x00,0x80,0x00,0x00,0x00,0x48,0x00,0x00,0x00, -0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00,0x94,0x00,0x00,0x00, -0x74,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x73,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0x95,0x00,0x00,0x00,0x94,0x00,0x00,0x00, -0x41,0x00,0x06,0x00,0x57,0x00,0x00,0x00,0x96,0x00,0x00,0x00, -0x78,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x91,0x00,0x00,0x00, -0x3e,0x00,0x03,0x00,0x96,0x00,0x00,0x00,0x95,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00,0xac,0x00,0x00,0x00, -0x58,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0xad,0x00,0x00,0x00,0xac,0x00,0x00,0x00,0x41,0x00,0x08,0x00, -0x5f,0x00,0x00,0x00,0xae,0x00,0x00,0x00,0x55,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x17,0x00,0x00,0x00, -0x17,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x4e,0x00,0x00,0x00, -0xaf,0x00,0x00,0x00,0xae,0x00,0x00,0x00,0x71,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0xb0,0x00,0x00,0x00,0xaf,0x00,0x00,0x00, -0xc7,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0xb1,0x00,0x00,0x00, -0xb0,0x00,0x00,0x00,0x67,0x00,0x00,0x00,0x70,0x00,0x04,0x00, -0x4a,0x00,0x00,0x00,0xb2,0x00,0x00,0x00,0xb1,0x00,0x00,0x00, -0xc2,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0xb3,0x00,0x00,0x00, -0xb0,0x00,0x00,0x00,0x6b,0x00,0x00,0x00,0x70,0x00,0x04,0x00, -0x4a,0x00,0x00,0x00,0xb4,0x00,0x00,0x00,0xb3,0x00,0x00,0x00, -0x50,0x00,0x05,0x00,0x63,0x00,0x00,0x00,0xb5,0x00,0x00,0x00, -0xb2,0x00,0x00,0x00,0xb4,0x00,0x00,0x00,0x83,0x00,0x05,0x00, -0x63,0x00,0x00,0x00,0xb6,0x00,0x00,0x00,0xb5,0x00,0x00,0x00, -0xa5,0x00,0x00,0x00,0x8e,0x00,0x05,0x00,0x63,0x00,0x00,0x00, -0xb7,0x00,0x00,0x00,0xb6,0x00,0x00,0x00,0xad,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xbd,0x00,0x00,0x00, -0x80,0x00,0x00,0x00,0x17,0x00,0x00,0x00,0x51,0x00,0x05,0x00, -0x4a,0x00,0x00,0x00,0xbf,0x00,0x00,0x00,0xb7,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0xc0,0x00,0x00,0x00,0xbf,0x00,0x00,0x00,0x41,0x00,0x06,0x00, -0x57,0x00,0x00,0x00,0xc1,0x00,0x00,0x00,0x78,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0xbd,0x00,0x00,0x00,0x3e,0x00,0x03,0x00, -0xc1,0x00,0x00,0x00,0xc0,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xc8,0x00,0x00,0x00,0x80,0x00,0x00,0x00, -0xff,0x02,0x00,0x00,0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00, -0xc9,0x00,0x00,0x00,0xb7,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x73,0x00,0x04,0x00,0x4d,0x00,0x00,0x00,0xca,0x00,0x00,0x00, -0xc9,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x57,0x00,0x00,0x00, -0xcb,0x00,0x00,0x00,0x78,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0xc8,0x00,0x00,0x00,0x3e,0x00,0x03,0x00,0xcb,0x00,0x00,0x00, -0xca,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0xd4,0x00,0x00,0x00,0x58,0x00,0x00,0x00,0x73,0x00,0x04,0x00, -0x4a,0x00,0x00,0x00,0xd5,0x00,0x00,0x00,0xd4,0x00,0x00,0x00, -0x41,0x00,0x08,0x00,0x5f,0x00,0x00,0x00,0xd6,0x00,0x00,0x00, -0x55,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x17,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4e,0x00,0x00,0x00,0xd7,0x00,0x00,0x00,0xd6,0x00,0x00,0x00, -0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0xd8,0x00,0x00,0x00, -0xd7,0x00,0x00,0x00,0xc7,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0xd9,0x00,0x00,0x00,0xd8,0x00,0x00,0x00,0x67,0x00,0x00,0x00, -0x70,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0xda,0x00,0x00,0x00, -0xd9,0x00,0x00,0x00,0xc2,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0xdb,0x00,0x00,0x00,0xd8,0x00,0x00,0x00,0x6b,0x00,0x00,0x00, -0x70,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0xdc,0x00,0x00,0x00, -0xdb,0x00,0x00,0x00,0x50,0x00,0x05,0x00,0x63,0x00,0x00,0x00, -0xdd,0x00,0x00,0x00,0xda,0x00,0x00,0x00,0xdc,0x00,0x00,0x00, -0x83,0x00,0x05,0x00,0x63,0x00,0x00,0x00,0xde,0x00,0x00,0x00, -0xdd,0x00,0x00,0x00,0xa5,0x00,0x00,0x00,0x8e,0x00,0x05,0x00, -0x63,0x00,0x00,0x00,0xdf,0x00,0x00,0x00,0xde,0x00,0x00,0x00, -0xd5,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xe5,0x00,0x00,0x00,0x80,0x00,0x00,0x00,0x37,0x00,0x00,0x00, -0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00,0xe7,0x00,0x00,0x00, -0xdf,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x73,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0xe8,0x00,0x00,0x00,0xe7,0x00,0x00,0x00, -0x41,0x00,0x06,0x00,0x57,0x00,0x00,0x00,0xe9,0x00,0x00,0x00, -0x78,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0xe5,0x00,0x00,0x00, -0x3e,0x00,0x03,0x00,0xe9,0x00,0x00,0x00,0xe8,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xf0,0x00,0x00,0x00, -0x80,0x00,0x00,0x00,0x00,0x03,0x00,0x00,0x51,0x00,0x05,0x00, -0x4a,0x00,0x00,0x00,0xf1,0x00,0x00,0x00,0xdf,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0xf2,0x00,0x00,0x00,0xf1,0x00,0x00,0x00,0x41,0x00,0x06,0x00, -0x57,0x00,0x00,0x00,0xf3,0x00,0x00,0x00,0x78,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0xf0,0x00,0x00,0x00,0x3e,0x00,0x03,0x00, -0xf3,0x00,0x00,0x00,0xf2,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0xfc,0x00,0x00,0x00,0x58,0x00,0x00,0x00, -0x73,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0xfd,0x00,0x00,0x00, -0xfc,0x00,0x00,0x00,0x41,0x00,0x08,0x00,0x5f,0x00,0x00,0x00, -0xfe,0x00,0x00,0x00,0x55,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0x17,0x00,0x00,0x00,0x7a,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4e,0x00,0x00,0x00,0xff,0x00,0x00,0x00, -0xfe,0x00,0x00,0x00,0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x00,0x01,0x00,0x00,0xff,0x00,0x00,0x00,0xc7,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0x01,0x01,0x00,0x00,0x00,0x01,0x00,0x00, -0x67,0x00,0x00,0x00,0x70,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0x02,0x01,0x00,0x00,0x01,0x01,0x00,0x00,0xc2,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0x03,0x01,0x00,0x00,0x00,0x01,0x00,0x00, -0x6b,0x00,0x00,0x00,0x70,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0x04,0x01,0x00,0x00,0x03,0x01,0x00,0x00,0x50,0x00,0x05,0x00, -0x63,0x00,0x00,0x00,0x05,0x01,0x00,0x00,0x02,0x01,0x00,0x00, -0x04,0x01,0x00,0x00,0x83,0x00,0x05,0x00,0x63,0x00,0x00,0x00, -0x06,0x01,0x00,0x00,0x05,0x01,0x00,0x00,0xa5,0x00,0x00,0x00, -0x8e,0x00,0x05,0x00,0x63,0x00,0x00,0x00,0x07,0x01,0x00,0x00, -0x06,0x01,0x00,0x00,0xfd,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x0d,0x01,0x00,0x00,0x80,0x00,0x00,0x00, -0x7a,0x00,0x00,0x00,0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00, -0x0f,0x01,0x00,0x00,0x07,0x01,0x00,0x00,0x00,0x00,0x00,0x00, -0x73,0x00,0x04,0x00,0x4d,0x00,0x00,0x00,0x10,0x01,0x00,0x00, -0x0f,0x01,0x00,0x00,0x41,0x00,0x06,0x00,0x57,0x00,0x00,0x00, -0x11,0x01,0x00,0x00,0x78,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x0d,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x11,0x01,0x00,0x00, -0x10,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x18,0x01,0x00,0x00,0x80,0x00,0x00,0x00,0x01,0x03,0x00,0x00, -0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00,0x19,0x01,0x00,0x00, -0x07,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x73,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0x1a,0x01,0x00,0x00,0x19,0x01,0x00,0x00, -0x41,0x00,0x06,0x00,0x57,0x00,0x00,0x00,0x1b,0x01,0x00,0x00, -0x78,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x18,0x01,0x00,0x00, -0x3e,0x00,0x03,0x00,0x1b,0x01,0x00,0x00,0x1a,0x01,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00,0x24,0x01,0x00,0x00, -0x58,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0x25,0x01,0x00,0x00,0x24,0x01,0x00,0x00,0x41,0x00,0x08,0x00, -0x5f,0x00,0x00,0x00,0x26,0x01,0x00,0x00,0x55,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x17,0x00,0x00,0x00, -0x6b,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x4e,0x00,0x00,0x00, -0x27,0x01,0x00,0x00,0x26,0x01,0x00,0x00,0x71,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x28,0x01,0x00,0x00,0x27,0x01,0x00,0x00, -0xc7,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0x29,0x01,0x00,0x00, -0x28,0x01,0x00,0x00,0x67,0x00,0x00,0x00,0x70,0x00,0x04,0x00, -0x4a,0x00,0x00,0x00,0x2a,0x01,0x00,0x00,0x29,0x01,0x00,0x00, -0xc2,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0x2b,0x01,0x00,0x00, -0x28,0x01,0x00,0x00,0x6b,0x00,0x00,0x00,0x70,0x00,0x04,0x00, -0x4a,0x00,0x00,0x00,0x2c,0x01,0x00,0x00,0x2b,0x01,0x00,0x00, -0x50,0x00,0x05,0x00,0x63,0x00,0x00,0x00,0x2d,0x01,0x00,0x00, -0x2a,0x01,0x00,0x00,0x2c,0x01,0x00,0x00,0x83,0x00,0x05,0x00, -0x63,0x00,0x00,0x00,0x2e,0x01,0x00,0x00,0x2d,0x01,0x00,0x00, -0xa5,0x00,0x00,0x00,0x8e,0x00,0x05,0x00,0x63,0x00,0x00,0x00, -0x2f,0x01,0x00,0x00,0x2e,0x01,0x00,0x00,0x25,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x35,0x01,0x00,0x00, -0x80,0x00,0x00,0x00,0x6b,0x00,0x00,0x00,0x51,0x00,0x05,0x00, -0x4a,0x00,0x00,0x00,0x37,0x01,0x00,0x00,0x2f,0x01,0x00,0x00, -0x00,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0x38,0x01,0x00,0x00,0x37,0x01,0x00,0x00,0x41,0x00,0x06,0x00, -0x57,0x00,0x00,0x00,0x39,0x01,0x00,0x00,0x78,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x35,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0x39,0x01,0x00,0x00,0x38,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x40,0x01,0x00,0x00,0x80,0x00,0x00,0x00, -0x02,0x03,0x00,0x00,0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00, -0x41,0x01,0x00,0x00,0x2f,0x01,0x00,0x00,0x01,0x00,0x00,0x00, -0x73,0x00,0x04,0x00,0x4d,0x00,0x00,0x00,0x42,0x01,0x00,0x00, -0x41,0x01,0x00,0x00,0x41,0x00,0x06,0x00,0x57,0x00,0x00,0x00, -0x43,0x01,0x00,0x00,0x78,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x40,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x43,0x01,0x00,0x00, -0x42,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0x4c,0x01,0x00,0x00,0x58,0x00,0x00,0x00,0x73,0x00,0x04,0x00, -0x4a,0x00,0x00,0x00,0x4d,0x01,0x00,0x00,0x4c,0x01,0x00,0x00, -0x41,0x00,0x08,0x00,0x5f,0x00,0x00,0x00,0x4e,0x01,0x00,0x00, -0x55,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x17,0x00,0x00,0x00,0x03,0x03,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4e,0x00,0x00,0x00,0x4f,0x01,0x00,0x00,0x4e,0x01,0x00,0x00, -0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x50,0x01,0x00,0x00, -0x4f,0x01,0x00,0x00,0xc7,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0x51,0x01,0x00,0x00,0x50,0x01,0x00,0x00,0x67,0x00,0x00,0x00, -0x70,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0x52,0x01,0x00,0x00, -0x51,0x01,0x00,0x00,0xc2,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0x53,0x01,0x00,0x00,0x50,0x01,0x00,0x00,0x6b,0x00,0x00,0x00, -0x70,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0x54,0x01,0x00,0x00, -0x53,0x01,0x00,0x00,0x50,0x00,0x05,0x00,0x63,0x00,0x00,0x00, -0x55,0x01,0x00,0x00,0x52,0x01,0x00,0x00,0x54,0x01,0x00,0x00, -0x83,0x00,0x05,0x00,0x63,0x00,0x00,0x00,0x56,0x01,0x00,0x00, -0x55,0x01,0x00,0x00,0xa5,0x00,0x00,0x00,0x8e,0x00,0x05,0x00, -0x63,0x00,0x00,0x00,0x57,0x01,0x00,0x00,0x56,0x01,0x00,0x00, -0x4d,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x5d,0x01,0x00,0x00,0x80,0x00,0x00,0x00,0x03,0x03,0x00,0x00, -0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00,0x5f,0x01,0x00,0x00, -0x57,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x73,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0x60,0x01,0x00,0x00,0x5f,0x01,0x00,0x00, -0x41,0x00,0x06,0x00,0x57,0x00,0x00,0x00,0x61,0x01,0x00,0x00, -0x78,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x5d,0x01,0x00,0x00, -0x3e,0x00,0x03,0x00,0x61,0x01,0x00,0x00,0x60,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x68,0x01,0x00,0x00, -0x80,0x00,0x00,0x00,0x04,0x03,0x00,0x00,0x51,0x00,0x05,0x00, -0x4a,0x00,0x00,0x00,0x69,0x01,0x00,0x00,0x57,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0x6a,0x01,0x00,0x00,0x69,0x01,0x00,0x00,0x41,0x00,0x06,0x00, -0x57,0x00,0x00,0x00,0x6b,0x01,0x00,0x00,0x78,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x68,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0x6b,0x01,0x00,0x00,0x6a,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0x74,0x01,0x00,0x00,0x58,0x00,0x00,0x00, -0x73,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0x75,0x01,0x00,0x00, -0x74,0x01,0x00,0x00,0x41,0x00,0x08,0x00,0x5f,0x00,0x00,0x00, -0x76,0x01,0x00,0x00,0x55,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0x17,0x00,0x00,0x00,0x05,0x03,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4e,0x00,0x00,0x00,0x77,0x01,0x00,0x00, -0x76,0x01,0x00,0x00,0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x78,0x01,0x00,0x00,0x77,0x01,0x00,0x00,0xc7,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0x79,0x01,0x00,0x00,0x78,0x01,0x00,0x00, -0x67,0x00,0x00,0x00,0x70,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0x7a,0x01,0x00,0x00,0x79,0x01,0x00,0x00,0xc2,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0x7b,0x01,0x00,0x00,0x78,0x01,0x00,0x00, -0x6b,0x00,0x00,0x00,0x70,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0x7c,0x01,0x00,0x00,0x7b,0x01,0x00,0x00,0x50,0x00,0x05,0x00, -0x63,0x00,0x00,0x00,0x7d,0x01,0x00,0x00,0x7a,0x01,0x00,0x00, -0x7c,0x01,0x00,0x00,0x83,0x00,0x05,0x00,0x63,0x00,0x00,0x00, -0x7e,0x01,0x00,0x00,0x7d,0x01,0x00,0x00,0xa5,0x00,0x00,0x00, -0x8e,0x00,0x05,0x00,0x63,0x00,0x00,0x00,0x7f,0x01,0x00,0x00, -0x7e,0x01,0x00,0x00,0x75,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x85,0x01,0x00,0x00,0x80,0x00,0x00,0x00, -0x05,0x03,0x00,0x00,0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00, -0x87,0x01,0x00,0x00,0x7f,0x01,0x00,0x00,0x00,0x00,0x00,0x00, -0x73,0x00,0x04,0x00,0x4d,0x00,0x00,0x00,0x88,0x01,0x00,0x00, -0x87,0x01,0x00,0x00,0x41,0x00,0x06,0x00,0x57,0x00,0x00,0x00, -0x89,0x01,0x00,0x00,0x78,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x85,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x89,0x01,0x00,0x00, -0x88,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x90,0x01,0x00,0x00,0x80,0x00,0x00,0x00,0x06,0x03,0x00,0x00, -0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00,0x91,0x01,0x00,0x00, -0x7f,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x73,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0x92,0x01,0x00,0x00,0x91,0x01,0x00,0x00, -0x41,0x00,0x06,0x00,0x57,0x00,0x00,0x00,0x93,0x01,0x00,0x00, -0x78,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x90,0x01,0x00,0x00, -0x3e,0x00,0x03,0x00,0x93,0x01,0x00,0x00,0x92,0x01,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00,0x9c,0x01,0x00,0x00, -0x58,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0x9d,0x01,0x00,0x00,0x9c,0x01,0x00,0x00,0x41,0x00,0x08,0x00, -0x5f,0x00,0x00,0x00,0x9e,0x01,0x00,0x00,0x55,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x17,0x00,0x00,0x00, -0x07,0x03,0x00,0x00,0x3d,0x00,0x04,0x00,0x4e,0x00,0x00,0x00, -0x9f,0x01,0x00,0x00,0x9e,0x01,0x00,0x00,0x71,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0xa0,0x01,0x00,0x00,0x9f,0x01,0x00,0x00, -0xc7,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0xa1,0x01,0x00,0x00, -0xa0,0x01,0x00,0x00,0x67,0x00,0x00,0x00,0x70,0x00,0x04,0x00, -0x4a,0x00,0x00,0x00,0xa2,0x01,0x00,0x00,0xa1,0x01,0x00,0x00, -0xc2,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0xa3,0x01,0x00,0x00, -0xa0,0x01,0x00,0x00,0x6b,0x00,0x00,0x00,0x70,0x00,0x04,0x00, -0x4a,0x00,0x00,0x00,0xa4,0x01,0x00,0x00,0xa3,0x01,0x00,0x00, -0x50,0x00,0x05,0x00,0x63,0x00,0x00,0x00,0xa5,0x01,0x00,0x00, -0xa2,0x01,0x00,0x00,0xa4,0x01,0x00,0x00,0x83,0x00,0x05,0x00, -0x63,0x00,0x00,0x00,0xa6,0x01,0x00,0x00,0xa5,0x01,0x00,0x00, -0xa5,0x00,0x00,0x00,0x8e,0x00,0x05,0x00,0x63,0x00,0x00,0x00, -0xa7,0x01,0x00,0x00,0xa6,0x01,0x00,0x00,0x9d,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xad,0x01,0x00,0x00, -0x80,0x00,0x00,0x00,0x07,0x03,0x00,0x00,0x51,0x00,0x05,0x00, -0x4a,0x00,0x00,0x00,0xaf,0x01,0x00,0x00,0xa7,0x01,0x00,0x00, -0x00,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0xb0,0x01,0x00,0x00,0xaf,0x01,0x00,0x00,0x41,0x00,0x06,0x00, -0x57,0x00,0x00,0x00,0xb1,0x01,0x00,0x00,0x78,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0xad,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0xb1,0x01,0x00,0x00,0xb0,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xb8,0x01,0x00,0x00,0x80,0x00,0x00,0x00, -0x08,0x03,0x00,0x00,0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00, -0xb9,0x01,0x00,0x00,0xa7,0x01,0x00,0x00,0x01,0x00,0x00,0x00, -0x73,0x00,0x04,0x00,0x4d,0x00,0x00,0x00,0xba,0x01,0x00,0x00, -0xb9,0x01,0x00,0x00,0x41,0x00,0x06,0x00,0x57,0x00,0x00,0x00, -0xbb,0x01,0x00,0x00,0x78,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0xb8,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0xbb,0x01,0x00,0x00, -0xba,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0xc4,0x01,0x00,0x00,0x58,0x00,0x00,0x00,0x73,0x00,0x04,0x00, -0x4a,0x00,0x00,0x00,0xc5,0x01,0x00,0x00,0xc4,0x01,0x00,0x00, -0x41,0x00,0x08,0x00,0x5f,0x00,0x00,0x00,0xc6,0x01,0x00,0x00, -0x55,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x17,0x00,0x00,0x00,0x09,0x03,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4e,0x00,0x00,0x00,0xc7,0x01,0x00,0x00,0xc6,0x01,0x00,0x00, -0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0xc8,0x01,0x00,0x00, -0xc7,0x01,0x00,0x00,0xc7,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0xc9,0x01,0x00,0x00,0xc8,0x01,0x00,0x00,0x67,0x00,0x00,0x00, -0x70,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0xca,0x01,0x00,0x00, -0xc9,0x01,0x00,0x00,0xc2,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0xcb,0x01,0x00,0x00,0xc8,0x01,0x00,0x00,0x6b,0x00,0x00,0x00, -0x70,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0xcc,0x01,0x00,0x00, -0xcb,0x01,0x00,0x00,0x50,0x00,0x05,0x00,0x63,0x00,0x00,0x00, -0xcd,0x01,0x00,0x00,0xca,0x01,0x00,0x00,0xcc,0x01,0x00,0x00, -0x83,0x00,0x05,0x00,0x63,0x00,0x00,0x00,0xce,0x01,0x00,0x00, -0xcd,0x01,0x00,0x00,0xa5,0x00,0x00,0x00,0x8e,0x00,0x05,0x00, -0x63,0x00,0x00,0x00,0xcf,0x01,0x00,0x00,0xce,0x01,0x00,0x00, -0xc5,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xd5,0x01,0x00,0x00,0x80,0x00,0x00,0x00,0x09,0x03,0x00,0x00, -0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00,0xd7,0x01,0x00,0x00, -0xcf,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x73,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0xd8,0x01,0x00,0x00,0xd7,0x01,0x00,0x00, -0x41,0x00,0x06,0x00,0x57,0x00,0x00,0x00,0xd9,0x01,0x00,0x00, -0x78,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0xd5,0x01,0x00,0x00, -0x3e,0x00,0x03,0x00,0xd9,0x01,0x00,0x00,0xd8,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xe0,0x01,0x00,0x00, -0x80,0x00,0x00,0x00,0x0a,0x03,0x00,0x00,0x51,0x00,0x05,0x00, -0x4a,0x00,0x00,0x00,0xe1,0x01,0x00,0x00,0xcf,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0xe2,0x01,0x00,0x00,0xe1,0x01,0x00,0x00,0x41,0x00,0x06,0x00, -0x57,0x00,0x00,0x00,0xe3,0x01,0x00,0x00,0x78,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0xe0,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0xe3,0x01,0x00,0x00,0xe2,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0xec,0x01,0x00,0x00,0x58,0x00,0x00,0x00, -0x73,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0xed,0x01,0x00,0x00, -0xec,0x01,0x00,0x00,0x41,0x00,0x08,0x00,0x5f,0x00,0x00,0x00, -0xee,0x01,0x00,0x00,0x55,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0x17,0x00,0x00,0x00,0x0b,0x03,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4e,0x00,0x00,0x00,0xef,0x01,0x00,0x00, -0xee,0x01,0x00,0x00,0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0xf0,0x01,0x00,0x00,0xef,0x01,0x00,0x00,0xc7,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0xf1,0x01,0x00,0x00,0xf0,0x01,0x00,0x00, -0x67,0x00,0x00,0x00,0x70,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0xf2,0x01,0x00,0x00,0xf1,0x01,0x00,0x00,0xc2,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0xf3,0x01,0x00,0x00,0xf0,0x01,0x00,0x00, -0x6b,0x00,0x00,0x00,0x70,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0xf4,0x01,0x00,0x00,0xf3,0x01,0x00,0x00,0x50,0x00,0x05,0x00, -0x63,0x00,0x00,0x00,0xf5,0x01,0x00,0x00,0xf2,0x01,0x00,0x00, -0xf4,0x01,0x00,0x00,0x83,0x00,0x05,0x00,0x63,0x00,0x00,0x00, -0xf6,0x01,0x00,0x00,0xf5,0x01,0x00,0x00,0xa5,0x00,0x00,0x00, -0x8e,0x00,0x05,0x00,0x63,0x00,0x00,0x00,0xf7,0x01,0x00,0x00, -0xf6,0x01,0x00,0x00,0xed,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xfd,0x01,0x00,0x00,0x80,0x00,0x00,0x00, -0x0b,0x03,0x00,0x00,0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00, -0xff,0x01,0x00,0x00,0xf7,0x01,0x00,0x00,0x00,0x00,0x00,0x00, -0x73,0x00,0x04,0x00,0x4d,0x00,0x00,0x00,0x00,0x02,0x00,0x00, -0xff,0x01,0x00,0x00,0x41,0x00,0x06,0x00,0x57,0x00,0x00,0x00, -0x01,0x02,0x00,0x00,0x78,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0xfd,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x01,0x02,0x00,0x00, -0x00,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x08,0x02,0x00,0x00,0x80,0x00,0x00,0x00,0x0c,0x03,0x00,0x00, -0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00,0x09,0x02,0x00,0x00, -0xf7,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x73,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0x0a,0x02,0x00,0x00,0x09,0x02,0x00,0x00, -0x41,0x00,0x06,0x00,0x57,0x00,0x00,0x00,0x0b,0x02,0x00,0x00, -0x78,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x08,0x02,0x00,0x00, -0x3e,0x00,0x03,0x00,0x0b,0x02,0x00,0x00,0x0a,0x02,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00,0x14,0x02,0x00,0x00, -0x58,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0x15,0x02,0x00,0x00,0x14,0x02,0x00,0x00,0x41,0x00,0x08,0x00, -0x5f,0x00,0x00,0x00,0x16,0x02,0x00,0x00,0x55,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x17,0x00,0x00,0x00, -0x0d,0x03,0x00,0x00,0x3d,0x00,0x04,0x00,0x4e,0x00,0x00,0x00, -0x17,0x02,0x00,0x00,0x16,0x02,0x00,0x00,0x71,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x18,0x02,0x00,0x00,0x17,0x02,0x00,0x00, -0xc7,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0x19,0x02,0x00,0x00, -0x18,0x02,0x00,0x00,0x67,0x00,0x00,0x00,0x70,0x00,0x04,0x00, -0x4a,0x00,0x00,0x00,0x1a,0x02,0x00,0x00,0x19,0x02,0x00,0x00, -0xc2,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0x1b,0x02,0x00,0x00, -0x18,0x02,0x00,0x00,0x6b,0x00,0x00,0x00,0x70,0x00,0x04,0x00, -0x4a,0x00,0x00,0x00,0x1c,0x02,0x00,0x00,0x1b,0x02,0x00,0x00, -0x50,0x00,0x05,0x00,0x63,0x00,0x00,0x00,0x1d,0x02,0x00,0x00, -0x1a,0x02,0x00,0x00,0x1c,0x02,0x00,0x00,0x83,0x00,0x05,0x00, -0x63,0x00,0x00,0x00,0x1e,0x02,0x00,0x00,0x1d,0x02,0x00,0x00, -0xa5,0x00,0x00,0x00,0x8e,0x00,0x05,0x00,0x63,0x00,0x00,0x00, -0x1f,0x02,0x00,0x00,0x1e,0x02,0x00,0x00,0x15,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x25,0x02,0x00,0x00, -0x80,0x00,0x00,0x00,0x0d,0x03,0x00,0x00,0x51,0x00,0x05,0x00, -0x4a,0x00,0x00,0x00,0x27,0x02,0x00,0x00,0x1f,0x02,0x00,0x00, -0x00,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0x28,0x02,0x00,0x00,0x27,0x02,0x00,0x00,0x41,0x00,0x06,0x00, -0x57,0x00,0x00,0x00,0x29,0x02,0x00,0x00,0x78,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x25,0x02,0x00,0x00,0x3e,0x00,0x03,0x00, -0x29,0x02,0x00,0x00,0x28,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x30,0x02,0x00,0x00,0x80,0x00,0x00,0x00, -0x0e,0x03,0x00,0x00,0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00, -0x31,0x02,0x00,0x00,0x1f,0x02,0x00,0x00,0x01,0x00,0x00,0x00, -0x73,0x00,0x04,0x00,0x4d,0x00,0x00,0x00,0x32,0x02,0x00,0x00, -0x31,0x02,0x00,0x00,0x41,0x00,0x06,0x00,0x57,0x00,0x00,0x00, -0x33,0x02,0x00,0x00,0x78,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x30,0x02,0x00,0x00,0x3e,0x00,0x03,0x00,0x33,0x02,0x00,0x00, -0x32,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0x3c,0x02,0x00,0x00,0x58,0x00,0x00,0x00,0x73,0x00,0x04,0x00, -0x4a,0x00,0x00,0x00,0x3d,0x02,0x00,0x00,0x3c,0x02,0x00,0x00, -0x41,0x00,0x08,0x00,0x5f,0x00,0x00,0x00,0x3e,0x02,0x00,0x00, -0x55,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x17,0x00,0x00,0x00,0x0f,0x03,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4e,0x00,0x00,0x00,0x3f,0x02,0x00,0x00,0x3e,0x02,0x00,0x00, -0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x40,0x02,0x00,0x00, -0x3f,0x02,0x00,0x00,0xc7,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0x41,0x02,0x00,0x00,0x40,0x02,0x00,0x00,0x67,0x00,0x00,0x00, -0x70,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0x42,0x02,0x00,0x00, -0x41,0x02,0x00,0x00,0xc2,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0x43,0x02,0x00,0x00,0x40,0x02,0x00,0x00,0x6b,0x00,0x00,0x00, -0x70,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0x44,0x02,0x00,0x00, -0x43,0x02,0x00,0x00,0x50,0x00,0x05,0x00,0x63,0x00,0x00,0x00, -0x45,0x02,0x00,0x00,0x42,0x02,0x00,0x00,0x44,0x02,0x00,0x00, -0x83,0x00,0x05,0x00,0x63,0x00,0x00,0x00,0x46,0x02,0x00,0x00, -0x45,0x02,0x00,0x00,0xa5,0x00,0x00,0x00,0x8e,0x00,0x05,0x00, -0x63,0x00,0x00,0x00,0x47,0x02,0x00,0x00,0x46,0x02,0x00,0x00, -0x3d,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x4d,0x02,0x00,0x00,0x80,0x00,0x00,0x00,0x0f,0x03,0x00,0x00, -0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00,0x4f,0x02,0x00,0x00, -0x47,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x73,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0x50,0x02,0x00,0x00,0x4f,0x02,0x00,0x00, -0x41,0x00,0x06,0x00,0x57,0x00,0x00,0x00,0x51,0x02,0x00,0x00, -0x78,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x4d,0x02,0x00,0x00, -0x3e,0x00,0x03,0x00,0x51,0x02,0x00,0x00,0x50,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x58,0x02,0x00,0x00, -0x80,0x00,0x00,0x00,0x10,0x03,0x00,0x00,0x51,0x00,0x05,0x00, -0x4a,0x00,0x00,0x00,0x59,0x02,0x00,0x00,0x47,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0x5a,0x02,0x00,0x00,0x59,0x02,0x00,0x00,0x41,0x00,0x06,0x00, -0x57,0x00,0x00,0x00,0x5b,0x02,0x00,0x00,0x78,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x58,0x02,0x00,0x00,0x3e,0x00,0x03,0x00, -0x5b,0x02,0x00,0x00,0x5a,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0x64,0x02,0x00,0x00,0x58,0x00,0x00,0x00, -0x73,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0x65,0x02,0x00,0x00, -0x64,0x02,0x00,0x00,0x41,0x00,0x08,0x00,0x5f,0x00,0x00,0x00, -0x66,0x02,0x00,0x00,0x55,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0x17,0x00,0x00,0x00,0x11,0x03,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4e,0x00,0x00,0x00,0x67,0x02,0x00,0x00, -0x66,0x02,0x00,0x00,0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x68,0x02,0x00,0x00,0x67,0x02,0x00,0x00,0xc7,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0x69,0x02,0x00,0x00,0x68,0x02,0x00,0x00, -0x67,0x00,0x00,0x00,0x70,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0x6a,0x02,0x00,0x00,0x69,0x02,0x00,0x00,0xc2,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0x6b,0x02,0x00,0x00,0x68,0x02,0x00,0x00, -0x6b,0x00,0x00,0x00,0x70,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0x6c,0x02,0x00,0x00,0x6b,0x02,0x00,0x00,0x50,0x00,0x05,0x00, -0x63,0x00,0x00,0x00,0x6d,0x02,0x00,0x00,0x6a,0x02,0x00,0x00, -0x6c,0x02,0x00,0x00,0x83,0x00,0x05,0x00,0x63,0x00,0x00,0x00, -0x6e,0x02,0x00,0x00,0x6d,0x02,0x00,0x00,0xa5,0x00,0x00,0x00, -0x8e,0x00,0x05,0x00,0x63,0x00,0x00,0x00,0x6f,0x02,0x00,0x00, -0x6e,0x02,0x00,0x00,0x65,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x75,0x02,0x00,0x00,0x80,0x00,0x00,0x00, -0x11,0x03,0x00,0x00,0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00, -0x77,0x02,0x00,0x00,0x6f,0x02,0x00,0x00,0x00,0x00,0x00,0x00, -0x73,0x00,0x04,0x00,0x4d,0x00,0x00,0x00,0x78,0x02,0x00,0x00, -0x77,0x02,0x00,0x00,0x41,0x00,0x06,0x00,0x57,0x00,0x00,0x00, -0x79,0x02,0x00,0x00,0x78,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x75,0x02,0x00,0x00,0x3e,0x00,0x03,0x00,0x79,0x02,0x00,0x00, -0x78,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x80,0x02,0x00,0x00,0x80,0x00,0x00,0x00,0x12,0x03,0x00,0x00, -0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00,0x81,0x02,0x00,0x00, -0x6f,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0x73,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0x82,0x02,0x00,0x00,0x81,0x02,0x00,0x00, -0x41,0x00,0x06,0x00,0x57,0x00,0x00,0x00,0x83,0x02,0x00,0x00, -0x78,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x80,0x02,0x00,0x00, -0x3e,0x00,0x03,0x00,0x83,0x02,0x00,0x00,0x82,0x02,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00,0x8c,0x02,0x00,0x00, -0x58,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0x8d,0x02,0x00,0x00,0x8c,0x02,0x00,0x00,0x41,0x00,0x08,0x00, -0x5f,0x00,0x00,0x00,0x8e,0x02,0x00,0x00,0x55,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x17,0x00,0x00,0x00, -0x13,0x03,0x00,0x00,0x3d,0x00,0x04,0x00,0x4e,0x00,0x00,0x00, -0x8f,0x02,0x00,0x00,0x8e,0x02,0x00,0x00,0x71,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x90,0x02,0x00,0x00,0x8f,0x02,0x00,0x00, -0xc7,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0x91,0x02,0x00,0x00, -0x90,0x02,0x00,0x00,0x67,0x00,0x00,0x00,0x70,0x00,0x04,0x00, -0x4a,0x00,0x00,0x00,0x92,0x02,0x00,0x00,0x91,0x02,0x00,0x00, -0xc2,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0x93,0x02,0x00,0x00, -0x90,0x02,0x00,0x00,0x6b,0x00,0x00,0x00,0x70,0x00,0x04,0x00, -0x4a,0x00,0x00,0x00,0x94,0x02,0x00,0x00,0x93,0x02,0x00,0x00, -0x50,0x00,0x05,0x00,0x63,0x00,0x00,0x00,0x95,0x02,0x00,0x00, -0x92,0x02,0x00,0x00,0x94,0x02,0x00,0x00,0x83,0x00,0x05,0x00, -0x63,0x00,0x00,0x00,0x96,0x02,0x00,0x00,0x95,0x02,0x00,0x00, -0xa5,0x00,0x00,0x00,0x8e,0x00,0x05,0x00,0x63,0x00,0x00,0x00, -0x97,0x02,0x00,0x00,0x96,0x02,0x00,0x00,0x8d,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9d,0x02,0x00,0x00, -0x80,0x00,0x00,0x00,0x13,0x03,0x00,0x00,0x51,0x00,0x05,0x00, -0x4a,0x00,0x00,0x00,0x9f,0x02,0x00,0x00,0x97,0x02,0x00,0x00, -0x00,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0xa0,0x02,0x00,0x00,0x9f,0x02,0x00,0x00,0x41,0x00,0x06,0x00, -0x57,0x00,0x00,0x00,0xa1,0x02,0x00,0x00,0x78,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x9d,0x02,0x00,0x00,0x3e,0x00,0x03,0x00, -0xa1,0x02,0x00,0x00,0xa0,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xa8,0x02,0x00,0x00,0x80,0x00,0x00,0x00, -0x14,0x03,0x00,0x00,0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00, -0xa9,0x02,0x00,0x00,0x97,0x02,0x00,0x00,0x01,0x00,0x00,0x00, -0x73,0x00,0x04,0x00,0x4d,0x00,0x00,0x00,0xaa,0x02,0x00,0x00, -0xa9,0x02,0x00,0x00,0x41,0x00,0x06,0x00,0x57,0x00,0x00,0x00, -0xab,0x02,0x00,0x00,0x78,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0xa8,0x02,0x00,0x00,0x3e,0x00,0x03,0x00,0xab,0x02,0x00,0x00, -0xaa,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0xb4,0x02,0x00,0x00,0x58,0x00,0x00,0x00,0x73,0x00,0x04,0x00, -0x4a,0x00,0x00,0x00,0xb5,0x02,0x00,0x00,0xb4,0x02,0x00,0x00, -0x41,0x00,0x08,0x00,0x5f,0x00,0x00,0x00,0xb6,0x02,0x00,0x00, -0x55,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x17,0x00,0x00,0x00,0x15,0x03,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4e,0x00,0x00,0x00,0xb7,0x02,0x00,0x00,0xb6,0x02,0x00,0x00, -0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0xb8,0x02,0x00,0x00, -0xb7,0x02,0x00,0x00,0xc7,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0xb9,0x02,0x00,0x00,0xb8,0x02,0x00,0x00,0x67,0x00,0x00,0x00, -0x70,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0xba,0x02,0x00,0x00, -0xb9,0x02,0x00,0x00,0xc2,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0xbb,0x02,0x00,0x00,0xb8,0x02,0x00,0x00,0x6b,0x00,0x00,0x00, -0x70,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0xbc,0x02,0x00,0x00, -0xbb,0x02,0x00,0x00,0x50,0x00,0x05,0x00,0x63,0x00,0x00,0x00, -0xbd,0x02,0x00,0x00,0xba,0x02,0x00,0x00,0xbc,0x02,0x00,0x00, -0x83,0x00,0x05,0x00,0x63,0x00,0x00,0x00,0xbe,0x02,0x00,0x00, -0xbd,0x02,0x00,0x00,0xa5,0x00,0x00,0x00,0x8e,0x00,0x05,0x00, -0x63,0x00,0x00,0x00,0xbf,0x02,0x00,0x00,0xbe,0x02,0x00,0x00, -0xb5,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xc5,0x02,0x00,0x00,0x80,0x00,0x00,0x00,0x15,0x03,0x00,0x00, -0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00,0xc7,0x02,0x00,0x00, -0xbf,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x73,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0xc8,0x02,0x00,0x00,0xc7,0x02,0x00,0x00, -0x41,0x00,0x06,0x00,0x57,0x00,0x00,0x00,0xc9,0x02,0x00,0x00, -0x78,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0xc5,0x02,0x00,0x00, -0x3e,0x00,0x03,0x00,0xc9,0x02,0x00,0x00,0xc8,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xd0,0x02,0x00,0x00, -0x80,0x00,0x00,0x00,0x16,0x03,0x00,0x00,0x51,0x00,0x05,0x00, -0x4a,0x00,0x00,0x00,0xd1,0x02,0x00,0x00,0xbf,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0xd2,0x02,0x00,0x00,0xd1,0x02,0x00,0x00,0x41,0x00,0x06,0x00, -0x57,0x00,0x00,0x00,0xd3,0x02,0x00,0x00,0x78,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0xd0,0x02,0x00,0x00,0x3e,0x00,0x03,0x00, -0xd3,0x02,0x00,0x00,0xd2,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0xdc,0x02,0x00,0x00,0x58,0x00,0x00,0x00, -0x73,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0xdd,0x02,0x00,0x00, -0xdc,0x02,0x00,0x00,0x41,0x00,0x08,0x00,0x5f,0x00,0x00,0x00, -0xde,0x02,0x00,0x00,0x55,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0x17,0x00,0x00,0x00,0x17,0x03,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4e,0x00,0x00,0x00,0xdf,0x02,0x00,0x00, -0xde,0x02,0x00,0x00,0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0xe0,0x02,0x00,0x00,0xdf,0x02,0x00,0x00,0xc7,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0xe1,0x02,0x00,0x00,0xe0,0x02,0x00,0x00, -0x67,0x00,0x00,0x00,0x70,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0xe2,0x02,0x00,0x00,0xe1,0x02,0x00,0x00,0xc2,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0xe3,0x02,0x00,0x00,0xe0,0x02,0x00,0x00, -0x6b,0x00,0x00,0x00,0x70,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0xe4,0x02,0x00,0x00,0xe3,0x02,0x00,0x00,0x50,0x00,0x05,0x00, -0x63,0x00,0x00,0x00,0xe5,0x02,0x00,0x00,0xe2,0x02,0x00,0x00, -0xe4,0x02,0x00,0x00,0x83,0x00,0x05,0x00,0x63,0x00,0x00,0x00, -0xe6,0x02,0x00,0x00,0xe5,0x02,0x00,0x00,0xa5,0x00,0x00,0x00, -0x8e,0x00,0x05,0x00,0x63,0x00,0x00,0x00,0xe7,0x02,0x00,0x00, -0xe6,0x02,0x00,0x00,0xdd,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xed,0x02,0x00,0x00,0x80,0x00,0x00,0x00, -0x17,0x03,0x00,0x00,0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00, -0xef,0x02,0x00,0x00,0xe7,0x02,0x00,0x00,0x00,0x00,0x00,0x00, -0x73,0x00,0x04,0x00,0x4d,0x00,0x00,0x00,0xf0,0x02,0x00,0x00, -0xef,0x02,0x00,0x00,0x41,0x00,0x06,0x00,0x57,0x00,0x00,0x00, -0xf1,0x02,0x00,0x00,0x78,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0xed,0x02,0x00,0x00,0x3e,0x00,0x03,0x00,0xf1,0x02,0x00,0x00, -0xf0,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xf8,0x02,0x00,0x00,0x80,0x00,0x00,0x00,0x18,0x03,0x00,0x00, -0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00,0xf9,0x02,0x00,0x00, -0xe7,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0x73,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0xfa,0x02,0x00,0x00,0xf9,0x02,0x00,0x00, -0x41,0x00,0x06,0x00,0x57,0x00,0x00,0x00,0xfb,0x02,0x00,0x00, -0x78,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0xf8,0x02,0x00,0x00, -0x3e,0x00,0x03,0x00,0xfb,0x02,0x00,0x00,0xfa,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0x9b,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x9b,0x00,0x00,0x00,0xfd,0x00,0x01,0x00,0x38,0x00,0x01,0x00, - -}; -const uint64_t dequant_q4_0_len = 8856; +const uint64_t dequant_q4_0_len = 5184; unsigned char dequant_q4_1_data[] = { 0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, -0x59,0x03,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, -0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x51,0x11,0x00,0x00, -0x11,0x00,0x02,0x00,0x60,0x11,0x00,0x00,0x0b,0x00,0x06,0x00, -0x01,0x00,0x00,0x00,0x47,0x4c,0x53,0x4c,0x2e,0x73,0x74,0x64, -0x2e,0x34,0x35,0x30,0x00,0x00,0x00,0x00,0x0e,0x00,0x03,0x00, -0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x0f,0x00,0x09,0x00, -0x05,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x6d,0x61,0x69,0x6e, -0x00,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0x55,0x00,0x00,0x00,0x7d,0x00,0x00,0x00,0x10,0x00,0x06,0x00, -0x04,0x00,0x00,0x00,0x11,0x00,0x00,0x00,0x00,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x0c,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x1c,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x14,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x14,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x14,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x08,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x14,0x00,0x00,0x00,0x03,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x47,0x00,0x03,0x00, -0x14,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x50,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x51,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x51,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x51,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x52,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x14,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0x53,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x53,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x53,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x55,0x00,0x00,0x00, -0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x55,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x7a,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0x7b,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x7b,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x7b,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x7d,0x00,0x00,0x00, -0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x7d,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x9f,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x19,0x00,0x00,0x00,0x13,0x00,0x02,0x00,0x02,0x00,0x00,0x00, -0x21,0x00,0x03,0x00,0x03,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x15,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x15,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x17,0x00,0x04,0x00, -0x0a,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x03,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x0b,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x0a,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x0b,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x0e,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x1e,0x00,0x06,0x00,0x14,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x15,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x15,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x17,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x18,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x1b,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x14,0x00,0x02,0x00,0x24,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x37,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x48,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x16,0x00,0x03,0x00, -0x4a,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x16,0x00,0x03,0x00, -0x4d,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x15,0x00,0x04,0x00, -0x4e,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x4f,0x00,0x00,0x00, -0x10,0x00,0x00,0x00,0x1c,0x00,0x04,0x00,0x50,0x00,0x00,0x00, -0x4e,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x1e,0x00,0x05,0x00, -0x51,0x00,0x00,0x00,0x4d,0x00,0x00,0x00,0x4d,0x00,0x00,0x00, -0x50,0x00,0x00,0x00,0x1d,0x00,0x03,0x00,0x52,0x00,0x00,0x00, -0x51,0x00,0x00,0x00,0x1e,0x00,0x03,0x00,0x53,0x00,0x00,0x00, -0x52,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x54,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x54,0x00,0x00,0x00,0x55,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x57,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x4d,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x64,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x17,0x00,0x04,0x00, -0x68,0x00,0x00,0x00,0x4a,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x0f,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x70,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, -0x7a,0x00,0x00,0x00,0x4d,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, -0x7b,0x00,0x00,0x00,0x7a,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x7c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x7b,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x7c,0x00,0x00,0x00,0x7d,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x7f,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x97,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x9e,0x00,0x00,0x00, -0x00,0x01,0x00,0x00,0x2c,0x00,0x06,0x00,0x0a,0x00,0x00,0x00, -0x9f,0x00,0x00,0x00,0x9e,0x00,0x00,0x00,0x97,0x00,0x00,0x00, -0x97,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x3f,0x03,0x00,0x00,0x11,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x40,0x03,0x00,0x00,0x12,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x41,0x03,0x00,0x00, -0x13,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x42,0x03,0x00,0x00,0x14,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x43,0x03,0x00,0x00,0x05,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x44,0x03,0x00,0x00, -0x15,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x45,0x03,0x00,0x00,0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x46,0x03,0x00,0x00,0x16,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x47,0x03,0x00,0x00, -0x07,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x48,0x03,0x00,0x00,0x17,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x49,0x03,0x00,0x00,0x08,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x4a,0x03,0x00,0x00, -0x18,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x4b,0x03,0x00,0x00,0x09,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x4c,0x03,0x00,0x00,0x19,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x4d,0x03,0x00,0x00, -0x0a,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x4e,0x03,0x00,0x00,0x1a,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x4f,0x03,0x00,0x00,0x0b,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x50,0x03,0x00,0x00, -0x1b,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x51,0x03,0x00,0x00,0x0c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x52,0x03,0x00,0x00,0x1c,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x53,0x03,0x00,0x00, -0x0d,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x54,0x03,0x00,0x00,0x1d,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x55,0x03,0x00,0x00,0x0e,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x56,0x03,0x00,0x00, -0x1e,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x57,0x03,0x00,0x00,0x0f,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x58,0x03,0x00,0x00,0x1f,0x00,0x00,0x00, -0x36,0x00,0x05,0x00,0x02,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x05,0x00,0x00,0x00,0xf7,0x00,0x03,0x00,0xa0,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0xfb,0x00,0x03,0x00,0x0d,0x00,0x00,0x00, -0xa1,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xa1,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x0e,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x0d,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x11,0x00,0x00,0x00, -0x10,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x18,0x00,0x00,0x00, -0x19,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x17,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x1a,0x00,0x00,0x00, -0x19,0x00,0x00,0x00,0x87,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x1c,0x00,0x00,0x00,0x1a,0x00,0x00,0x00,0x1b,0x00,0x00,0x00, -0x8b,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x1d,0x00,0x00,0x00, -0x11,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x87,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x11,0x00,0x00,0x00, -0x1c,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x26,0x00,0x00,0x00,0x1d,0x00,0x00,0x00,0x1b,0x00,0x00,0x00, -0xaf,0x00,0x05,0x00,0x24,0x00,0x00,0x00,0x29,0x00,0x00,0x00, -0x26,0x00,0x00,0x00,0x1a,0x00,0x00,0x00,0xa8,0x00,0x04,0x00, -0x24,0x00,0x00,0x00,0x2a,0x00,0x00,0x00,0x29,0x00,0x00,0x00, -0xf7,0x00,0x03,0x00,0x2c,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x2a,0x00,0x00,0x00,0x2b,0x00,0x00,0x00, -0x2c,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x2b,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x18,0x00,0x00,0x00,0x2f,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x30,0x00,0x00,0x00,0x2f,0x00,0x00,0x00, -0xaf,0x00,0x05,0x00,0x24,0x00,0x00,0x00,0x31,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x30,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x2c,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x2c,0x00,0x00,0x00, -0xf5,0x00,0x07,0x00,0x24,0x00,0x00,0x00,0x32,0x00,0x00,0x00, -0x29,0x00,0x00,0x00,0xa1,0x00,0x00,0x00,0x31,0x00,0x00,0x00, -0x2b,0x00,0x00,0x00,0xf7,0x00,0x03,0x00,0x34,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x32,0x00,0x00,0x00, -0x33,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x33,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xa0,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0x34,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x18,0x00,0x00,0x00,0x38,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0x37,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x38,0x00,0x00,0x00,0x87,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x1b,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x3e,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x3e,0x00,0x00,0x00,0x1d,0x00,0x00,0x00,0x41,0x00,0x07,0x00, -0x57,0x00,0x00,0x00,0x58,0x00,0x00,0x00,0x55,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00,0x59,0x00,0x00,0x00, -0x58,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0x5a,0x00,0x00,0x00,0x59,0x00,0x00,0x00,0x41,0x00,0x07,0x00, -0x57,0x00,0x00,0x00,0x5d,0x00,0x00,0x00,0x55,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x17,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00,0x5e,0x00,0x00,0x00, -0x5d,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0x5f,0x00,0x00,0x00,0x5e,0x00,0x00,0x00,0x41,0x00,0x08,0x00, -0x64,0x00,0x00,0x00,0x65,0x00,0x00,0x00,0x55,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x37,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x4e,0x00,0x00,0x00, -0x66,0x00,0x00,0x00,0x65,0x00,0x00,0x00,0x71,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x67,0x00,0x00,0x00,0x66,0x00,0x00,0x00, -0xc7,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, -0x67,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x70,0x00,0x04,0x00, -0x4a,0x00,0x00,0x00,0x6e,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, -0xc2,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0x71,0x00,0x00,0x00, -0x67,0x00,0x00,0x00,0x70,0x00,0x00,0x00,0x70,0x00,0x04,0x00, -0x4a,0x00,0x00,0x00,0x72,0x00,0x00,0x00,0x71,0x00,0x00,0x00, -0x50,0x00,0x05,0x00,0x68,0x00,0x00,0x00,0x73,0x00,0x00,0x00, -0x6e,0x00,0x00,0x00,0x72,0x00,0x00,0x00,0x8e,0x00,0x05,0x00, -0x68,0x00,0x00,0x00,0x76,0x00,0x00,0x00,0x73,0x00,0x00,0x00, -0x5a,0x00,0x00,0x00,0x50,0x00,0x05,0x00,0x68,0x00,0x00,0x00, -0x78,0x00,0x00,0x00,0x5f,0x00,0x00,0x00,0x5f,0x00,0x00,0x00, -0x81,0x00,0x05,0x00,0x68,0x00,0x00,0x00,0x79,0x00,0x00,0x00, -0x76,0x00,0x00,0x00,0x78,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x18,0x00,0x00,0x00,0x80,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0x7f,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x81,0x00,0x00,0x00,0x80,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x82,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x81,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x85,0x00,0x00,0x00,0x82,0x00,0x00,0x00,0x26,0x00,0x00,0x00, -0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00,0x8a,0x00,0x00,0x00, -0x79,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x73,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0x8b,0x00,0x00,0x00,0x8a,0x00,0x00,0x00, -0x41,0x00,0x06,0x00,0x57,0x00,0x00,0x00,0x8c,0x00,0x00,0x00, -0x7d,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x85,0x00,0x00,0x00, -0x3e,0x00,0x03,0x00,0x8c,0x00,0x00,0x00,0x8b,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x96,0x00,0x00,0x00, -0x85,0x00,0x00,0x00,0x48,0x00,0x00,0x00,0x51,0x00,0x05,0x00, -0x4a,0x00,0x00,0x00,0x99,0x00,0x00,0x00,0x79,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0x9a,0x00,0x00,0x00,0x99,0x00,0x00,0x00,0x41,0x00,0x06,0x00, -0x57,0x00,0x00,0x00,0x9b,0x00,0x00,0x00,0x7d,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x96,0x00,0x00,0x00,0x3e,0x00,0x03,0x00, -0x9b,0x00,0x00,0x00,0x9a,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0xb0,0x00,0x00,0x00,0x58,0x00,0x00,0x00, -0x73,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0xb1,0x00,0x00,0x00, -0xb0,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0xb3,0x00,0x00,0x00,0x5d,0x00,0x00,0x00,0x73,0x00,0x04,0x00, -0x4a,0x00,0x00,0x00,0xb4,0x00,0x00,0x00,0xb3,0x00,0x00,0x00, -0x41,0x00,0x08,0x00,0x64,0x00,0x00,0x00,0xb5,0x00,0x00,0x00, -0x55,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x37,0x00,0x00,0x00,0x17,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4e,0x00,0x00,0x00,0xb6,0x00,0x00,0x00,0xb5,0x00,0x00,0x00, -0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0xb7,0x00,0x00,0x00, -0xb6,0x00,0x00,0x00,0xc7,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0xb8,0x00,0x00,0x00,0xb7,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x70,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0xb9,0x00,0x00,0x00, -0xb8,0x00,0x00,0x00,0xc2,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0xba,0x00,0x00,0x00,0xb7,0x00,0x00,0x00,0x70,0x00,0x00,0x00, -0x70,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0xbb,0x00,0x00,0x00, -0xba,0x00,0x00,0x00,0x50,0x00,0x05,0x00,0x68,0x00,0x00,0x00, -0xbc,0x00,0x00,0x00,0xb9,0x00,0x00,0x00,0xbb,0x00,0x00,0x00, -0x8e,0x00,0x05,0x00,0x68,0x00,0x00,0x00,0xbd,0x00,0x00,0x00, -0xbc,0x00,0x00,0x00,0xb1,0x00,0x00,0x00,0x50,0x00,0x05,0x00, -0x68,0x00,0x00,0x00,0xbe,0x00,0x00,0x00,0xb4,0x00,0x00,0x00, -0xb4,0x00,0x00,0x00,0x81,0x00,0x05,0x00,0x68,0x00,0x00,0x00, -0xbf,0x00,0x00,0x00,0xbd,0x00,0x00,0x00,0xbe,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xc5,0x00,0x00,0x00, -0x85,0x00,0x00,0x00,0x17,0x00,0x00,0x00,0x51,0x00,0x05,0x00, -0x4a,0x00,0x00,0x00,0xc7,0x00,0x00,0x00,0xbf,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0xc8,0x00,0x00,0x00,0xc7,0x00,0x00,0x00,0x41,0x00,0x06,0x00, -0x57,0x00,0x00,0x00,0xc9,0x00,0x00,0x00,0x7d,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0xc5,0x00,0x00,0x00,0x3e,0x00,0x03,0x00, -0xc9,0x00,0x00,0x00,0xc8,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xd0,0x00,0x00,0x00,0x85,0x00,0x00,0x00, -0x3f,0x03,0x00,0x00,0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00, -0xd1,0x00,0x00,0x00,0xbf,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x73,0x00,0x04,0x00,0x4d,0x00,0x00,0x00,0xd2,0x00,0x00,0x00, -0xd1,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x57,0x00,0x00,0x00, -0xd3,0x00,0x00,0x00,0x7d,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0xd0,0x00,0x00,0x00,0x3e,0x00,0x03,0x00,0xd3,0x00,0x00,0x00, -0xd2,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0xdc,0x00,0x00,0x00,0x58,0x00,0x00,0x00,0x73,0x00,0x04,0x00, -0x4a,0x00,0x00,0x00,0xdd,0x00,0x00,0x00,0xdc,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00,0xdf,0x00,0x00,0x00, -0x5d,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0xe0,0x00,0x00,0x00,0xdf,0x00,0x00,0x00,0x41,0x00,0x08,0x00, -0x64,0x00,0x00,0x00,0xe1,0x00,0x00,0x00,0x55,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x37,0x00,0x00,0x00, -0x37,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x4e,0x00,0x00,0x00, -0xe2,0x00,0x00,0x00,0xe1,0x00,0x00,0x00,0x71,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0xe3,0x00,0x00,0x00,0xe2,0x00,0x00,0x00, -0xc7,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0xe4,0x00,0x00,0x00, -0xe3,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x70,0x00,0x04,0x00, -0x4a,0x00,0x00,0x00,0xe5,0x00,0x00,0x00,0xe4,0x00,0x00,0x00, -0xc2,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0xe6,0x00,0x00,0x00, -0xe3,0x00,0x00,0x00,0x70,0x00,0x00,0x00,0x70,0x00,0x04,0x00, -0x4a,0x00,0x00,0x00,0xe7,0x00,0x00,0x00,0xe6,0x00,0x00,0x00, -0x50,0x00,0x05,0x00,0x68,0x00,0x00,0x00,0xe8,0x00,0x00,0x00, -0xe5,0x00,0x00,0x00,0xe7,0x00,0x00,0x00,0x8e,0x00,0x05,0x00, -0x68,0x00,0x00,0x00,0xe9,0x00,0x00,0x00,0xe8,0x00,0x00,0x00, -0xdd,0x00,0x00,0x00,0x50,0x00,0x05,0x00,0x68,0x00,0x00,0x00, -0xea,0x00,0x00,0x00,0xe0,0x00,0x00,0x00,0xe0,0x00,0x00,0x00, -0x81,0x00,0x05,0x00,0x68,0x00,0x00,0x00,0xeb,0x00,0x00,0x00, -0xe9,0x00,0x00,0x00,0xea,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xf1,0x00,0x00,0x00,0x85,0x00,0x00,0x00, -0x37,0x00,0x00,0x00,0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00, -0xf3,0x00,0x00,0x00,0xeb,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x73,0x00,0x04,0x00,0x4d,0x00,0x00,0x00,0xf4,0x00,0x00,0x00, -0xf3,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x57,0x00,0x00,0x00, -0xf5,0x00,0x00,0x00,0x7d,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0xf1,0x00,0x00,0x00,0x3e,0x00,0x03,0x00,0xf5,0x00,0x00,0x00, -0xf4,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xfc,0x00,0x00,0x00,0x85,0x00,0x00,0x00,0x40,0x03,0x00,0x00, -0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00,0xfd,0x00,0x00,0x00, -0xeb,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x73,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0xfe,0x00,0x00,0x00,0xfd,0x00,0x00,0x00, -0x41,0x00,0x06,0x00,0x57,0x00,0x00,0x00,0xff,0x00,0x00,0x00, -0x7d,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0xfc,0x00,0x00,0x00, -0x3e,0x00,0x03,0x00,0xff,0x00,0x00,0x00,0xfe,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00,0x08,0x01,0x00,0x00, -0x58,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0x09,0x01,0x00,0x00,0x08,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0x0b,0x01,0x00,0x00,0x5d,0x00,0x00,0x00, -0x73,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0x0c,0x01,0x00,0x00, -0x0b,0x01,0x00,0x00,0x41,0x00,0x08,0x00,0x64,0x00,0x00,0x00, -0x0d,0x01,0x00,0x00,0x55,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x7f,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4e,0x00,0x00,0x00,0x0e,0x01,0x00,0x00, -0x0d,0x01,0x00,0x00,0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x0f,0x01,0x00,0x00,0x0e,0x01,0x00,0x00,0xc7,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0x10,0x01,0x00,0x00,0x0f,0x01,0x00,0x00, -0x6c,0x00,0x00,0x00,0x70,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0x11,0x01,0x00,0x00,0x10,0x01,0x00,0x00,0xc2,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0x12,0x01,0x00,0x00,0x0f,0x01,0x00,0x00, -0x70,0x00,0x00,0x00,0x70,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0x13,0x01,0x00,0x00,0x12,0x01,0x00,0x00,0x50,0x00,0x05,0x00, -0x68,0x00,0x00,0x00,0x14,0x01,0x00,0x00,0x11,0x01,0x00,0x00, -0x13,0x01,0x00,0x00,0x8e,0x00,0x05,0x00,0x68,0x00,0x00,0x00, -0x15,0x01,0x00,0x00,0x14,0x01,0x00,0x00,0x09,0x01,0x00,0x00, -0x50,0x00,0x05,0x00,0x68,0x00,0x00,0x00,0x16,0x01,0x00,0x00, -0x0c,0x01,0x00,0x00,0x0c,0x01,0x00,0x00,0x81,0x00,0x05,0x00, -0x68,0x00,0x00,0x00,0x17,0x01,0x00,0x00,0x15,0x01,0x00,0x00, -0x16,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x1d,0x01,0x00,0x00,0x85,0x00,0x00,0x00,0x7f,0x00,0x00,0x00, -0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00,0x1f,0x01,0x00,0x00, -0x17,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x73,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0x20,0x01,0x00,0x00,0x1f,0x01,0x00,0x00, -0x41,0x00,0x06,0x00,0x57,0x00,0x00,0x00,0x21,0x01,0x00,0x00, -0x7d,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x1d,0x01,0x00,0x00, -0x3e,0x00,0x03,0x00,0x21,0x01,0x00,0x00,0x20,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x28,0x01,0x00,0x00, -0x85,0x00,0x00,0x00,0x41,0x03,0x00,0x00,0x51,0x00,0x05,0x00, -0x4a,0x00,0x00,0x00,0x29,0x01,0x00,0x00,0x17,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0x2a,0x01,0x00,0x00,0x29,0x01,0x00,0x00,0x41,0x00,0x06,0x00, -0x57,0x00,0x00,0x00,0x2b,0x01,0x00,0x00,0x7d,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x28,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0x2b,0x01,0x00,0x00,0x2a,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0x34,0x01,0x00,0x00,0x58,0x00,0x00,0x00, -0x73,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0x35,0x01,0x00,0x00, -0x34,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0x37,0x01,0x00,0x00,0x5d,0x00,0x00,0x00,0x73,0x00,0x04,0x00, -0x4a,0x00,0x00,0x00,0x38,0x01,0x00,0x00,0x37,0x01,0x00,0x00, -0x41,0x00,0x08,0x00,0x64,0x00,0x00,0x00,0x39,0x01,0x00,0x00, -0x55,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x37,0x00,0x00,0x00,0x70,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4e,0x00,0x00,0x00,0x3a,0x01,0x00,0x00,0x39,0x01,0x00,0x00, -0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x3b,0x01,0x00,0x00, -0x3a,0x01,0x00,0x00,0xc7,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0x3c,0x01,0x00,0x00,0x3b,0x01,0x00,0x00,0x6c,0x00,0x00,0x00, -0x70,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0x3d,0x01,0x00,0x00, -0x3c,0x01,0x00,0x00,0xc2,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0x3e,0x01,0x00,0x00,0x3b,0x01,0x00,0x00,0x70,0x00,0x00,0x00, -0x70,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0x3f,0x01,0x00,0x00, -0x3e,0x01,0x00,0x00,0x50,0x00,0x05,0x00,0x68,0x00,0x00,0x00, -0x40,0x01,0x00,0x00,0x3d,0x01,0x00,0x00,0x3f,0x01,0x00,0x00, -0x8e,0x00,0x05,0x00,0x68,0x00,0x00,0x00,0x41,0x01,0x00,0x00, -0x40,0x01,0x00,0x00,0x35,0x01,0x00,0x00,0x50,0x00,0x05,0x00, -0x68,0x00,0x00,0x00,0x42,0x01,0x00,0x00,0x38,0x01,0x00,0x00, -0x38,0x01,0x00,0x00,0x81,0x00,0x05,0x00,0x68,0x00,0x00,0x00, -0x43,0x01,0x00,0x00,0x41,0x01,0x00,0x00,0x42,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x49,0x01,0x00,0x00, -0x85,0x00,0x00,0x00,0x70,0x00,0x00,0x00,0x51,0x00,0x05,0x00, -0x4a,0x00,0x00,0x00,0x4b,0x01,0x00,0x00,0x43,0x01,0x00,0x00, -0x00,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0x4c,0x01,0x00,0x00,0x4b,0x01,0x00,0x00,0x41,0x00,0x06,0x00, -0x57,0x00,0x00,0x00,0x4d,0x01,0x00,0x00,0x7d,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x49,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0x4d,0x01,0x00,0x00,0x4c,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x54,0x01,0x00,0x00,0x85,0x00,0x00,0x00, -0x42,0x03,0x00,0x00,0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00, -0x55,0x01,0x00,0x00,0x43,0x01,0x00,0x00,0x01,0x00,0x00,0x00, -0x73,0x00,0x04,0x00,0x4d,0x00,0x00,0x00,0x56,0x01,0x00,0x00, -0x55,0x01,0x00,0x00,0x41,0x00,0x06,0x00,0x57,0x00,0x00,0x00, -0x57,0x01,0x00,0x00,0x7d,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x54,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x57,0x01,0x00,0x00, -0x56,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0x60,0x01,0x00,0x00,0x58,0x00,0x00,0x00,0x73,0x00,0x04,0x00, -0x4a,0x00,0x00,0x00,0x61,0x01,0x00,0x00,0x60,0x01,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00,0x63,0x01,0x00,0x00, -0x5d,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0x64,0x01,0x00,0x00,0x63,0x01,0x00,0x00,0x41,0x00,0x08,0x00, -0x64,0x00,0x00,0x00,0x65,0x01,0x00,0x00,0x55,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x37,0x00,0x00,0x00, -0x43,0x03,0x00,0x00,0x3d,0x00,0x04,0x00,0x4e,0x00,0x00,0x00, -0x66,0x01,0x00,0x00,0x65,0x01,0x00,0x00,0x71,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x67,0x01,0x00,0x00,0x66,0x01,0x00,0x00, -0xc7,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0x68,0x01,0x00,0x00, -0x67,0x01,0x00,0x00,0x6c,0x00,0x00,0x00,0x70,0x00,0x04,0x00, -0x4a,0x00,0x00,0x00,0x69,0x01,0x00,0x00,0x68,0x01,0x00,0x00, -0xc2,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0x6a,0x01,0x00,0x00, -0x67,0x01,0x00,0x00,0x70,0x00,0x00,0x00,0x70,0x00,0x04,0x00, -0x4a,0x00,0x00,0x00,0x6b,0x01,0x00,0x00,0x6a,0x01,0x00,0x00, -0x50,0x00,0x05,0x00,0x68,0x00,0x00,0x00,0x6c,0x01,0x00,0x00, -0x69,0x01,0x00,0x00,0x6b,0x01,0x00,0x00,0x8e,0x00,0x05,0x00, -0x68,0x00,0x00,0x00,0x6d,0x01,0x00,0x00,0x6c,0x01,0x00,0x00, -0x61,0x01,0x00,0x00,0x50,0x00,0x05,0x00,0x68,0x00,0x00,0x00, -0x6e,0x01,0x00,0x00,0x64,0x01,0x00,0x00,0x64,0x01,0x00,0x00, -0x81,0x00,0x05,0x00,0x68,0x00,0x00,0x00,0x6f,0x01,0x00,0x00, -0x6d,0x01,0x00,0x00,0x6e,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x75,0x01,0x00,0x00,0x85,0x00,0x00,0x00, -0x43,0x03,0x00,0x00,0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00, -0x77,0x01,0x00,0x00,0x6f,0x01,0x00,0x00,0x00,0x00,0x00,0x00, -0x73,0x00,0x04,0x00,0x4d,0x00,0x00,0x00,0x78,0x01,0x00,0x00, -0x77,0x01,0x00,0x00,0x41,0x00,0x06,0x00,0x57,0x00,0x00,0x00, -0x79,0x01,0x00,0x00,0x7d,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x75,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x79,0x01,0x00,0x00, -0x78,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x80,0x01,0x00,0x00,0x85,0x00,0x00,0x00,0x44,0x03,0x00,0x00, -0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00,0x81,0x01,0x00,0x00, -0x6f,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x73,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0x82,0x01,0x00,0x00,0x81,0x01,0x00,0x00, -0x41,0x00,0x06,0x00,0x57,0x00,0x00,0x00,0x83,0x01,0x00,0x00, -0x7d,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x80,0x01,0x00,0x00, -0x3e,0x00,0x03,0x00,0x83,0x01,0x00,0x00,0x82,0x01,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00,0x8c,0x01,0x00,0x00, -0x58,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0x8d,0x01,0x00,0x00,0x8c,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0x8f,0x01,0x00,0x00,0x5d,0x00,0x00,0x00, -0x73,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0x90,0x01,0x00,0x00, -0x8f,0x01,0x00,0x00,0x41,0x00,0x08,0x00,0x64,0x00,0x00,0x00, -0x91,0x01,0x00,0x00,0x55,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x45,0x03,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4e,0x00,0x00,0x00,0x92,0x01,0x00,0x00, -0x91,0x01,0x00,0x00,0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x93,0x01,0x00,0x00,0x92,0x01,0x00,0x00,0xc7,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0x94,0x01,0x00,0x00,0x93,0x01,0x00,0x00, -0x6c,0x00,0x00,0x00,0x70,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0x95,0x01,0x00,0x00,0x94,0x01,0x00,0x00,0xc2,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0x96,0x01,0x00,0x00,0x93,0x01,0x00,0x00, -0x70,0x00,0x00,0x00,0x70,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0x97,0x01,0x00,0x00,0x96,0x01,0x00,0x00,0x50,0x00,0x05,0x00, -0x68,0x00,0x00,0x00,0x98,0x01,0x00,0x00,0x95,0x01,0x00,0x00, -0x97,0x01,0x00,0x00,0x8e,0x00,0x05,0x00,0x68,0x00,0x00,0x00, -0x99,0x01,0x00,0x00,0x98,0x01,0x00,0x00,0x8d,0x01,0x00,0x00, -0x50,0x00,0x05,0x00,0x68,0x00,0x00,0x00,0x9a,0x01,0x00,0x00, -0x90,0x01,0x00,0x00,0x90,0x01,0x00,0x00,0x81,0x00,0x05,0x00, -0x68,0x00,0x00,0x00,0x9b,0x01,0x00,0x00,0x99,0x01,0x00,0x00, -0x9a,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xa1,0x01,0x00,0x00,0x85,0x00,0x00,0x00,0x45,0x03,0x00,0x00, -0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00,0xa3,0x01,0x00,0x00, -0x9b,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x73,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0xa4,0x01,0x00,0x00,0xa3,0x01,0x00,0x00, -0x41,0x00,0x06,0x00,0x57,0x00,0x00,0x00,0xa5,0x01,0x00,0x00, -0x7d,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0xa1,0x01,0x00,0x00, -0x3e,0x00,0x03,0x00,0xa5,0x01,0x00,0x00,0xa4,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xac,0x01,0x00,0x00, -0x85,0x00,0x00,0x00,0x46,0x03,0x00,0x00,0x51,0x00,0x05,0x00, -0x4a,0x00,0x00,0x00,0xad,0x01,0x00,0x00,0x9b,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0xae,0x01,0x00,0x00,0xad,0x01,0x00,0x00,0x41,0x00,0x06,0x00, -0x57,0x00,0x00,0x00,0xaf,0x01,0x00,0x00,0x7d,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0xac,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0xaf,0x01,0x00,0x00,0xae,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0xb8,0x01,0x00,0x00,0x58,0x00,0x00,0x00, -0x73,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0xb9,0x01,0x00,0x00, -0xb8,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0xbb,0x01,0x00,0x00,0x5d,0x00,0x00,0x00,0x73,0x00,0x04,0x00, -0x4a,0x00,0x00,0x00,0xbc,0x01,0x00,0x00,0xbb,0x01,0x00,0x00, -0x41,0x00,0x08,0x00,0x64,0x00,0x00,0x00,0xbd,0x01,0x00,0x00, -0x55,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x37,0x00,0x00,0x00,0x47,0x03,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4e,0x00,0x00,0x00,0xbe,0x01,0x00,0x00,0xbd,0x01,0x00,0x00, -0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0xbf,0x01,0x00,0x00, -0xbe,0x01,0x00,0x00,0xc7,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0xc0,0x01,0x00,0x00,0xbf,0x01,0x00,0x00,0x6c,0x00,0x00,0x00, -0x70,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0xc1,0x01,0x00,0x00, -0xc0,0x01,0x00,0x00,0xc2,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0xc2,0x01,0x00,0x00,0xbf,0x01,0x00,0x00,0x70,0x00,0x00,0x00, -0x70,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0xc3,0x01,0x00,0x00, -0xc2,0x01,0x00,0x00,0x50,0x00,0x05,0x00,0x68,0x00,0x00,0x00, -0xc4,0x01,0x00,0x00,0xc1,0x01,0x00,0x00,0xc3,0x01,0x00,0x00, -0x8e,0x00,0x05,0x00,0x68,0x00,0x00,0x00,0xc5,0x01,0x00,0x00, -0xc4,0x01,0x00,0x00,0xb9,0x01,0x00,0x00,0x50,0x00,0x05,0x00, -0x68,0x00,0x00,0x00,0xc6,0x01,0x00,0x00,0xbc,0x01,0x00,0x00, -0xbc,0x01,0x00,0x00,0x81,0x00,0x05,0x00,0x68,0x00,0x00,0x00, -0xc7,0x01,0x00,0x00,0xc5,0x01,0x00,0x00,0xc6,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xcd,0x01,0x00,0x00, -0x85,0x00,0x00,0x00,0x47,0x03,0x00,0x00,0x51,0x00,0x05,0x00, -0x4a,0x00,0x00,0x00,0xcf,0x01,0x00,0x00,0xc7,0x01,0x00,0x00, -0x00,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0xd0,0x01,0x00,0x00,0xcf,0x01,0x00,0x00,0x41,0x00,0x06,0x00, -0x57,0x00,0x00,0x00,0xd1,0x01,0x00,0x00,0x7d,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0xcd,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0xd1,0x01,0x00,0x00,0xd0,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xd8,0x01,0x00,0x00,0x85,0x00,0x00,0x00, -0x48,0x03,0x00,0x00,0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00, -0xd9,0x01,0x00,0x00,0xc7,0x01,0x00,0x00,0x01,0x00,0x00,0x00, -0x73,0x00,0x04,0x00,0x4d,0x00,0x00,0x00,0xda,0x01,0x00,0x00, -0xd9,0x01,0x00,0x00,0x41,0x00,0x06,0x00,0x57,0x00,0x00,0x00, -0xdb,0x01,0x00,0x00,0x7d,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0xd8,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0xdb,0x01,0x00,0x00, -0xda,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0xe4,0x01,0x00,0x00,0x58,0x00,0x00,0x00,0x73,0x00,0x04,0x00, -0x4a,0x00,0x00,0x00,0xe5,0x01,0x00,0x00,0xe4,0x01,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00,0xe7,0x01,0x00,0x00, -0x5d,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0xe8,0x01,0x00,0x00,0xe7,0x01,0x00,0x00,0x41,0x00,0x08,0x00, -0x64,0x00,0x00,0x00,0xe9,0x01,0x00,0x00,0x55,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x37,0x00,0x00,0x00, -0x49,0x03,0x00,0x00,0x3d,0x00,0x04,0x00,0x4e,0x00,0x00,0x00, -0xea,0x01,0x00,0x00,0xe9,0x01,0x00,0x00,0x71,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0xeb,0x01,0x00,0x00,0xea,0x01,0x00,0x00, -0xc7,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0xec,0x01,0x00,0x00, -0xeb,0x01,0x00,0x00,0x6c,0x00,0x00,0x00,0x70,0x00,0x04,0x00, -0x4a,0x00,0x00,0x00,0xed,0x01,0x00,0x00,0xec,0x01,0x00,0x00, -0xc2,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0xee,0x01,0x00,0x00, -0xeb,0x01,0x00,0x00,0x70,0x00,0x00,0x00,0x70,0x00,0x04,0x00, -0x4a,0x00,0x00,0x00,0xef,0x01,0x00,0x00,0xee,0x01,0x00,0x00, -0x50,0x00,0x05,0x00,0x68,0x00,0x00,0x00,0xf0,0x01,0x00,0x00, -0xed,0x01,0x00,0x00,0xef,0x01,0x00,0x00,0x8e,0x00,0x05,0x00, -0x68,0x00,0x00,0x00,0xf1,0x01,0x00,0x00,0xf0,0x01,0x00,0x00, -0xe5,0x01,0x00,0x00,0x50,0x00,0x05,0x00,0x68,0x00,0x00,0x00, -0xf2,0x01,0x00,0x00,0xe8,0x01,0x00,0x00,0xe8,0x01,0x00,0x00, -0x81,0x00,0x05,0x00,0x68,0x00,0x00,0x00,0xf3,0x01,0x00,0x00, -0xf1,0x01,0x00,0x00,0xf2,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xf9,0x01,0x00,0x00,0x85,0x00,0x00,0x00, -0x49,0x03,0x00,0x00,0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00, -0xfb,0x01,0x00,0x00,0xf3,0x01,0x00,0x00,0x00,0x00,0x00,0x00, -0x73,0x00,0x04,0x00,0x4d,0x00,0x00,0x00,0xfc,0x01,0x00,0x00, -0xfb,0x01,0x00,0x00,0x41,0x00,0x06,0x00,0x57,0x00,0x00,0x00, -0xfd,0x01,0x00,0x00,0x7d,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0xf9,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0xfd,0x01,0x00,0x00, -0xfc,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x04,0x02,0x00,0x00,0x85,0x00,0x00,0x00,0x4a,0x03,0x00,0x00, -0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00,0x05,0x02,0x00,0x00, -0xf3,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x73,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0x06,0x02,0x00,0x00,0x05,0x02,0x00,0x00, -0x41,0x00,0x06,0x00,0x57,0x00,0x00,0x00,0x07,0x02,0x00,0x00, -0x7d,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x04,0x02,0x00,0x00, -0x3e,0x00,0x03,0x00,0x07,0x02,0x00,0x00,0x06,0x02,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00,0x10,0x02,0x00,0x00, -0x58,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0x11,0x02,0x00,0x00,0x10,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0x13,0x02,0x00,0x00,0x5d,0x00,0x00,0x00, -0x73,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0x14,0x02,0x00,0x00, -0x13,0x02,0x00,0x00,0x41,0x00,0x08,0x00,0x64,0x00,0x00,0x00, -0x15,0x02,0x00,0x00,0x55,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x4b,0x03,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4e,0x00,0x00,0x00,0x16,0x02,0x00,0x00, -0x15,0x02,0x00,0x00,0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x17,0x02,0x00,0x00,0x16,0x02,0x00,0x00,0xc7,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0x18,0x02,0x00,0x00,0x17,0x02,0x00,0x00, -0x6c,0x00,0x00,0x00,0x70,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0x19,0x02,0x00,0x00,0x18,0x02,0x00,0x00,0xc2,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0x1a,0x02,0x00,0x00,0x17,0x02,0x00,0x00, -0x70,0x00,0x00,0x00,0x70,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0x1b,0x02,0x00,0x00,0x1a,0x02,0x00,0x00,0x50,0x00,0x05,0x00, -0x68,0x00,0x00,0x00,0x1c,0x02,0x00,0x00,0x19,0x02,0x00,0x00, -0x1b,0x02,0x00,0x00,0x8e,0x00,0x05,0x00,0x68,0x00,0x00,0x00, -0x1d,0x02,0x00,0x00,0x1c,0x02,0x00,0x00,0x11,0x02,0x00,0x00, -0x50,0x00,0x05,0x00,0x68,0x00,0x00,0x00,0x1e,0x02,0x00,0x00, -0x14,0x02,0x00,0x00,0x14,0x02,0x00,0x00,0x81,0x00,0x05,0x00, -0x68,0x00,0x00,0x00,0x1f,0x02,0x00,0x00,0x1d,0x02,0x00,0x00, -0x1e,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x25,0x02,0x00,0x00,0x85,0x00,0x00,0x00,0x4b,0x03,0x00,0x00, -0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00,0x27,0x02,0x00,0x00, -0x1f,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x73,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0x28,0x02,0x00,0x00,0x27,0x02,0x00,0x00, -0x41,0x00,0x06,0x00,0x57,0x00,0x00,0x00,0x29,0x02,0x00,0x00, -0x7d,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x25,0x02,0x00,0x00, -0x3e,0x00,0x03,0x00,0x29,0x02,0x00,0x00,0x28,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x30,0x02,0x00,0x00, -0x85,0x00,0x00,0x00,0x4c,0x03,0x00,0x00,0x51,0x00,0x05,0x00, -0x4a,0x00,0x00,0x00,0x31,0x02,0x00,0x00,0x1f,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0x32,0x02,0x00,0x00,0x31,0x02,0x00,0x00,0x41,0x00,0x06,0x00, -0x57,0x00,0x00,0x00,0x33,0x02,0x00,0x00,0x7d,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x30,0x02,0x00,0x00,0x3e,0x00,0x03,0x00, -0x33,0x02,0x00,0x00,0x32,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0x3c,0x02,0x00,0x00,0x58,0x00,0x00,0x00, -0x73,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0x3d,0x02,0x00,0x00, -0x3c,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0x3f,0x02,0x00,0x00,0x5d,0x00,0x00,0x00,0x73,0x00,0x04,0x00, -0x4a,0x00,0x00,0x00,0x40,0x02,0x00,0x00,0x3f,0x02,0x00,0x00, -0x41,0x00,0x08,0x00,0x64,0x00,0x00,0x00,0x41,0x02,0x00,0x00, -0x55,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x37,0x00,0x00,0x00,0x4d,0x03,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4e,0x00,0x00,0x00,0x42,0x02,0x00,0x00,0x41,0x02,0x00,0x00, -0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x43,0x02,0x00,0x00, -0x42,0x02,0x00,0x00,0xc7,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0x44,0x02,0x00,0x00,0x43,0x02,0x00,0x00,0x6c,0x00,0x00,0x00, -0x70,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0x45,0x02,0x00,0x00, -0x44,0x02,0x00,0x00,0xc2,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0x46,0x02,0x00,0x00,0x43,0x02,0x00,0x00,0x70,0x00,0x00,0x00, -0x70,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0x47,0x02,0x00,0x00, -0x46,0x02,0x00,0x00,0x50,0x00,0x05,0x00,0x68,0x00,0x00,0x00, -0x48,0x02,0x00,0x00,0x45,0x02,0x00,0x00,0x47,0x02,0x00,0x00, -0x8e,0x00,0x05,0x00,0x68,0x00,0x00,0x00,0x49,0x02,0x00,0x00, -0x48,0x02,0x00,0x00,0x3d,0x02,0x00,0x00,0x50,0x00,0x05,0x00, -0x68,0x00,0x00,0x00,0x4a,0x02,0x00,0x00,0x40,0x02,0x00,0x00, -0x40,0x02,0x00,0x00,0x81,0x00,0x05,0x00,0x68,0x00,0x00,0x00, -0x4b,0x02,0x00,0x00,0x49,0x02,0x00,0x00,0x4a,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x51,0x02,0x00,0x00, -0x85,0x00,0x00,0x00,0x4d,0x03,0x00,0x00,0x51,0x00,0x05,0x00, -0x4a,0x00,0x00,0x00,0x53,0x02,0x00,0x00,0x4b,0x02,0x00,0x00, -0x00,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0x54,0x02,0x00,0x00,0x53,0x02,0x00,0x00,0x41,0x00,0x06,0x00, -0x57,0x00,0x00,0x00,0x55,0x02,0x00,0x00,0x7d,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x51,0x02,0x00,0x00,0x3e,0x00,0x03,0x00, -0x55,0x02,0x00,0x00,0x54,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x5c,0x02,0x00,0x00,0x85,0x00,0x00,0x00, -0x4e,0x03,0x00,0x00,0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00, -0x5d,0x02,0x00,0x00,0x4b,0x02,0x00,0x00,0x01,0x00,0x00,0x00, -0x73,0x00,0x04,0x00,0x4d,0x00,0x00,0x00,0x5e,0x02,0x00,0x00, -0x5d,0x02,0x00,0x00,0x41,0x00,0x06,0x00,0x57,0x00,0x00,0x00, -0x5f,0x02,0x00,0x00,0x7d,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x5c,0x02,0x00,0x00,0x3e,0x00,0x03,0x00,0x5f,0x02,0x00,0x00, -0x5e,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0x68,0x02,0x00,0x00,0x58,0x00,0x00,0x00,0x73,0x00,0x04,0x00, -0x4a,0x00,0x00,0x00,0x69,0x02,0x00,0x00,0x68,0x02,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00,0x6b,0x02,0x00,0x00, -0x5d,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0x6c,0x02,0x00,0x00,0x6b,0x02,0x00,0x00,0x41,0x00,0x08,0x00, -0x64,0x00,0x00,0x00,0x6d,0x02,0x00,0x00,0x55,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x37,0x00,0x00,0x00, -0x4f,0x03,0x00,0x00,0x3d,0x00,0x04,0x00,0x4e,0x00,0x00,0x00, -0x6e,0x02,0x00,0x00,0x6d,0x02,0x00,0x00,0x71,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x6f,0x02,0x00,0x00,0x6e,0x02,0x00,0x00, -0xc7,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0x70,0x02,0x00,0x00, -0x6f,0x02,0x00,0x00,0x6c,0x00,0x00,0x00,0x70,0x00,0x04,0x00, -0x4a,0x00,0x00,0x00,0x71,0x02,0x00,0x00,0x70,0x02,0x00,0x00, -0xc2,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0x72,0x02,0x00,0x00, -0x6f,0x02,0x00,0x00,0x70,0x00,0x00,0x00,0x70,0x00,0x04,0x00, -0x4a,0x00,0x00,0x00,0x73,0x02,0x00,0x00,0x72,0x02,0x00,0x00, -0x50,0x00,0x05,0x00,0x68,0x00,0x00,0x00,0x74,0x02,0x00,0x00, -0x71,0x02,0x00,0x00,0x73,0x02,0x00,0x00,0x8e,0x00,0x05,0x00, -0x68,0x00,0x00,0x00,0x75,0x02,0x00,0x00,0x74,0x02,0x00,0x00, -0x69,0x02,0x00,0x00,0x50,0x00,0x05,0x00,0x68,0x00,0x00,0x00, -0x76,0x02,0x00,0x00,0x6c,0x02,0x00,0x00,0x6c,0x02,0x00,0x00, -0x81,0x00,0x05,0x00,0x68,0x00,0x00,0x00,0x77,0x02,0x00,0x00, -0x75,0x02,0x00,0x00,0x76,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x7d,0x02,0x00,0x00,0x85,0x00,0x00,0x00, -0x4f,0x03,0x00,0x00,0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00, -0x7f,0x02,0x00,0x00,0x77,0x02,0x00,0x00,0x00,0x00,0x00,0x00, -0x73,0x00,0x04,0x00,0x4d,0x00,0x00,0x00,0x80,0x02,0x00,0x00, -0x7f,0x02,0x00,0x00,0x41,0x00,0x06,0x00,0x57,0x00,0x00,0x00, -0x81,0x02,0x00,0x00,0x7d,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x7d,0x02,0x00,0x00,0x3e,0x00,0x03,0x00,0x81,0x02,0x00,0x00, -0x80,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x88,0x02,0x00,0x00,0x85,0x00,0x00,0x00,0x50,0x03,0x00,0x00, -0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00,0x89,0x02,0x00,0x00, -0x77,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0x73,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0x8a,0x02,0x00,0x00,0x89,0x02,0x00,0x00, -0x41,0x00,0x06,0x00,0x57,0x00,0x00,0x00,0x8b,0x02,0x00,0x00, -0x7d,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x88,0x02,0x00,0x00, -0x3e,0x00,0x03,0x00,0x8b,0x02,0x00,0x00,0x8a,0x02,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00,0x94,0x02,0x00,0x00, -0x58,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0x95,0x02,0x00,0x00,0x94,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0x97,0x02,0x00,0x00,0x5d,0x00,0x00,0x00, -0x73,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0x98,0x02,0x00,0x00, -0x97,0x02,0x00,0x00,0x41,0x00,0x08,0x00,0x64,0x00,0x00,0x00, -0x99,0x02,0x00,0x00,0x55,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x51,0x03,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4e,0x00,0x00,0x00,0x9a,0x02,0x00,0x00, -0x99,0x02,0x00,0x00,0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x9b,0x02,0x00,0x00,0x9a,0x02,0x00,0x00,0xc7,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0x9c,0x02,0x00,0x00,0x9b,0x02,0x00,0x00, -0x6c,0x00,0x00,0x00,0x70,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0x9d,0x02,0x00,0x00,0x9c,0x02,0x00,0x00,0xc2,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0x9e,0x02,0x00,0x00,0x9b,0x02,0x00,0x00, -0x70,0x00,0x00,0x00,0x70,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0x9f,0x02,0x00,0x00,0x9e,0x02,0x00,0x00,0x50,0x00,0x05,0x00, -0x68,0x00,0x00,0x00,0xa0,0x02,0x00,0x00,0x9d,0x02,0x00,0x00, -0x9f,0x02,0x00,0x00,0x8e,0x00,0x05,0x00,0x68,0x00,0x00,0x00, -0xa1,0x02,0x00,0x00,0xa0,0x02,0x00,0x00,0x95,0x02,0x00,0x00, -0x50,0x00,0x05,0x00,0x68,0x00,0x00,0x00,0xa2,0x02,0x00,0x00, -0x98,0x02,0x00,0x00,0x98,0x02,0x00,0x00,0x81,0x00,0x05,0x00, -0x68,0x00,0x00,0x00,0xa3,0x02,0x00,0x00,0xa1,0x02,0x00,0x00, -0xa2,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xa9,0x02,0x00,0x00,0x85,0x00,0x00,0x00,0x51,0x03,0x00,0x00, -0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00,0xab,0x02,0x00,0x00, -0xa3,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x73,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0xac,0x02,0x00,0x00,0xab,0x02,0x00,0x00, -0x41,0x00,0x06,0x00,0x57,0x00,0x00,0x00,0xad,0x02,0x00,0x00, -0x7d,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0xa9,0x02,0x00,0x00, -0x3e,0x00,0x03,0x00,0xad,0x02,0x00,0x00,0xac,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xb4,0x02,0x00,0x00, -0x85,0x00,0x00,0x00,0x52,0x03,0x00,0x00,0x51,0x00,0x05,0x00, -0x4a,0x00,0x00,0x00,0xb5,0x02,0x00,0x00,0xa3,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0xb6,0x02,0x00,0x00,0xb5,0x02,0x00,0x00,0x41,0x00,0x06,0x00, -0x57,0x00,0x00,0x00,0xb7,0x02,0x00,0x00,0x7d,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0xb4,0x02,0x00,0x00,0x3e,0x00,0x03,0x00, -0xb7,0x02,0x00,0x00,0xb6,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0xc0,0x02,0x00,0x00,0x58,0x00,0x00,0x00, -0x73,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0xc1,0x02,0x00,0x00, -0xc0,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0xc3,0x02,0x00,0x00,0x5d,0x00,0x00,0x00,0x73,0x00,0x04,0x00, -0x4a,0x00,0x00,0x00,0xc4,0x02,0x00,0x00,0xc3,0x02,0x00,0x00, -0x41,0x00,0x08,0x00,0x64,0x00,0x00,0x00,0xc5,0x02,0x00,0x00, -0x55,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x37,0x00,0x00,0x00,0x53,0x03,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4e,0x00,0x00,0x00,0xc6,0x02,0x00,0x00,0xc5,0x02,0x00,0x00, -0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0xc7,0x02,0x00,0x00, -0xc6,0x02,0x00,0x00,0xc7,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0xc8,0x02,0x00,0x00,0xc7,0x02,0x00,0x00,0x6c,0x00,0x00,0x00, -0x70,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0xc9,0x02,0x00,0x00, -0xc8,0x02,0x00,0x00,0xc2,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0xca,0x02,0x00,0x00,0xc7,0x02,0x00,0x00,0x70,0x00,0x00,0x00, -0x70,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0xcb,0x02,0x00,0x00, -0xca,0x02,0x00,0x00,0x50,0x00,0x05,0x00,0x68,0x00,0x00,0x00, -0xcc,0x02,0x00,0x00,0xc9,0x02,0x00,0x00,0xcb,0x02,0x00,0x00, -0x8e,0x00,0x05,0x00,0x68,0x00,0x00,0x00,0xcd,0x02,0x00,0x00, -0xcc,0x02,0x00,0x00,0xc1,0x02,0x00,0x00,0x50,0x00,0x05,0x00, -0x68,0x00,0x00,0x00,0xce,0x02,0x00,0x00,0xc4,0x02,0x00,0x00, -0xc4,0x02,0x00,0x00,0x81,0x00,0x05,0x00,0x68,0x00,0x00,0x00, -0xcf,0x02,0x00,0x00,0xcd,0x02,0x00,0x00,0xce,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xd5,0x02,0x00,0x00, -0x85,0x00,0x00,0x00,0x53,0x03,0x00,0x00,0x51,0x00,0x05,0x00, -0x4a,0x00,0x00,0x00,0xd7,0x02,0x00,0x00,0xcf,0x02,0x00,0x00, -0x00,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0xd8,0x02,0x00,0x00,0xd7,0x02,0x00,0x00,0x41,0x00,0x06,0x00, -0x57,0x00,0x00,0x00,0xd9,0x02,0x00,0x00,0x7d,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0xd5,0x02,0x00,0x00,0x3e,0x00,0x03,0x00, -0xd9,0x02,0x00,0x00,0xd8,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xe0,0x02,0x00,0x00,0x85,0x00,0x00,0x00, -0x54,0x03,0x00,0x00,0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00, -0xe1,0x02,0x00,0x00,0xcf,0x02,0x00,0x00,0x01,0x00,0x00,0x00, -0x73,0x00,0x04,0x00,0x4d,0x00,0x00,0x00,0xe2,0x02,0x00,0x00, -0xe1,0x02,0x00,0x00,0x41,0x00,0x06,0x00,0x57,0x00,0x00,0x00, -0xe3,0x02,0x00,0x00,0x7d,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0xe0,0x02,0x00,0x00,0x3e,0x00,0x03,0x00,0xe3,0x02,0x00,0x00, -0xe2,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0xec,0x02,0x00,0x00,0x58,0x00,0x00,0x00,0x73,0x00,0x04,0x00, -0x4a,0x00,0x00,0x00,0xed,0x02,0x00,0x00,0xec,0x02,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00,0xef,0x02,0x00,0x00, -0x5d,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0xf0,0x02,0x00,0x00,0xef,0x02,0x00,0x00,0x41,0x00,0x08,0x00, -0x64,0x00,0x00,0x00,0xf1,0x02,0x00,0x00,0x55,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x37,0x00,0x00,0x00, -0x55,0x03,0x00,0x00,0x3d,0x00,0x04,0x00,0x4e,0x00,0x00,0x00, -0xf2,0x02,0x00,0x00,0xf1,0x02,0x00,0x00,0x71,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0xf3,0x02,0x00,0x00,0xf2,0x02,0x00,0x00, -0xc7,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0xf4,0x02,0x00,0x00, -0xf3,0x02,0x00,0x00,0x6c,0x00,0x00,0x00,0x70,0x00,0x04,0x00, -0x4a,0x00,0x00,0x00,0xf5,0x02,0x00,0x00,0xf4,0x02,0x00,0x00, -0xc2,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0xf6,0x02,0x00,0x00, -0xf3,0x02,0x00,0x00,0x70,0x00,0x00,0x00,0x70,0x00,0x04,0x00, -0x4a,0x00,0x00,0x00,0xf7,0x02,0x00,0x00,0xf6,0x02,0x00,0x00, -0x50,0x00,0x05,0x00,0x68,0x00,0x00,0x00,0xf8,0x02,0x00,0x00, -0xf5,0x02,0x00,0x00,0xf7,0x02,0x00,0x00,0x8e,0x00,0x05,0x00, -0x68,0x00,0x00,0x00,0xf9,0x02,0x00,0x00,0xf8,0x02,0x00,0x00, -0xed,0x02,0x00,0x00,0x50,0x00,0x05,0x00,0x68,0x00,0x00,0x00, -0xfa,0x02,0x00,0x00,0xf0,0x02,0x00,0x00,0xf0,0x02,0x00,0x00, -0x81,0x00,0x05,0x00,0x68,0x00,0x00,0x00,0xfb,0x02,0x00,0x00, -0xf9,0x02,0x00,0x00,0xfa,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x01,0x03,0x00,0x00,0x85,0x00,0x00,0x00, -0x55,0x03,0x00,0x00,0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00, -0x03,0x03,0x00,0x00,0xfb,0x02,0x00,0x00,0x00,0x00,0x00,0x00, -0x73,0x00,0x04,0x00,0x4d,0x00,0x00,0x00,0x04,0x03,0x00,0x00, -0x03,0x03,0x00,0x00,0x41,0x00,0x06,0x00,0x57,0x00,0x00,0x00, -0x05,0x03,0x00,0x00,0x7d,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x01,0x03,0x00,0x00,0x3e,0x00,0x03,0x00,0x05,0x03,0x00,0x00, -0x04,0x03,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x0c,0x03,0x00,0x00,0x85,0x00,0x00,0x00,0x56,0x03,0x00,0x00, -0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00,0x0d,0x03,0x00,0x00, -0xfb,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0x73,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0x0e,0x03,0x00,0x00,0x0d,0x03,0x00,0x00, -0x41,0x00,0x06,0x00,0x57,0x00,0x00,0x00,0x0f,0x03,0x00,0x00, -0x7d,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x0c,0x03,0x00,0x00, -0x3e,0x00,0x03,0x00,0x0f,0x03,0x00,0x00,0x0e,0x03,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00,0x18,0x03,0x00,0x00, -0x58,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0x19,0x03,0x00,0x00,0x18,0x03,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0x1b,0x03,0x00,0x00,0x5d,0x00,0x00,0x00, -0x73,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0x1c,0x03,0x00,0x00, -0x1b,0x03,0x00,0x00,0x41,0x00,0x08,0x00,0x64,0x00,0x00,0x00, -0x1d,0x03,0x00,0x00,0x55,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x57,0x03,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4e,0x00,0x00,0x00,0x1e,0x03,0x00,0x00, -0x1d,0x03,0x00,0x00,0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x1f,0x03,0x00,0x00,0x1e,0x03,0x00,0x00,0xc7,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0x20,0x03,0x00,0x00,0x1f,0x03,0x00,0x00, -0x6c,0x00,0x00,0x00,0x70,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0x21,0x03,0x00,0x00,0x20,0x03,0x00,0x00,0xc2,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0x22,0x03,0x00,0x00,0x1f,0x03,0x00,0x00, -0x70,0x00,0x00,0x00,0x70,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0x23,0x03,0x00,0x00,0x22,0x03,0x00,0x00,0x50,0x00,0x05,0x00, -0x68,0x00,0x00,0x00,0x24,0x03,0x00,0x00,0x21,0x03,0x00,0x00, -0x23,0x03,0x00,0x00,0x8e,0x00,0x05,0x00,0x68,0x00,0x00,0x00, -0x25,0x03,0x00,0x00,0x24,0x03,0x00,0x00,0x19,0x03,0x00,0x00, -0x50,0x00,0x05,0x00,0x68,0x00,0x00,0x00,0x26,0x03,0x00,0x00, -0x1c,0x03,0x00,0x00,0x1c,0x03,0x00,0x00,0x81,0x00,0x05,0x00, -0x68,0x00,0x00,0x00,0x27,0x03,0x00,0x00,0x25,0x03,0x00,0x00, -0x26,0x03,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x2d,0x03,0x00,0x00,0x85,0x00,0x00,0x00,0x57,0x03,0x00,0x00, -0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00,0x2f,0x03,0x00,0x00, -0x27,0x03,0x00,0x00,0x00,0x00,0x00,0x00,0x73,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0x30,0x03,0x00,0x00,0x2f,0x03,0x00,0x00, -0x41,0x00,0x06,0x00,0x57,0x00,0x00,0x00,0x31,0x03,0x00,0x00, -0x7d,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x2d,0x03,0x00,0x00, -0x3e,0x00,0x03,0x00,0x31,0x03,0x00,0x00,0x30,0x03,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x38,0x03,0x00,0x00, -0x85,0x00,0x00,0x00,0x58,0x03,0x00,0x00,0x51,0x00,0x05,0x00, -0x4a,0x00,0x00,0x00,0x39,0x03,0x00,0x00,0x27,0x03,0x00,0x00, -0x01,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0x3a,0x03,0x00,0x00,0x39,0x03,0x00,0x00,0x41,0x00,0x06,0x00, -0x57,0x00,0x00,0x00,0x3b,0x03,0x00,0x00,0x7d,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x38,0x03,0x00,0x00,0x3e,0x00,0x03,0x00, -0x3b,0x03,0x00,0x00,0x3a,0x03,0x00,0x00,0xf9,0x00,0x02,0x00, -0xa0,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xa0,0x00,0x00,0x00, -0xfd,0x00,0x01,0x00,0x38,0x00,0x01,0x00, -}; -const uint64_t dequant_q4_1_len = 9704; - -unsigned char dequant_q4_K_data[] = { -0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, -0xb1,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, +0x85,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, 0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x27,0x00,0x00,0x00, 0x11,0x00,0x02,0x00,0x51,0x11,0x00,0x00,0x11,0x00,0x02,0x00, 0x60,0x11,0x00,0x00,0x0b,0x00,0x06,0x00,0x01,0x00,0x00,0x00, @@ -3360,29 +3267,970 @@ unsigned char dequant_q4_K_data[] = { 0x00,0x00,0x00,0x00,0x0e,0x00,0x03,0x00,0x00,0x00,0x00,0x00, 0x01,0x00,0x00,0x00,0x0f,0x00,0x0a,0x00,0x05,0x00,0x00,0x00, 0x04,0x00,0x00,0x00,0x6d,0x61,0x69,0x6e,0x00,0x00,0x00,0x00, -0x17,0x00,0x00,0x00,0x25,0x00,0x00,0x00,0x33,0x00,0x00,0x00, -0x50,0x00,0x00,0x00,0x0b,0x01,0x00,0x00,0x10,0x00,0x06,0x00, +0x0b,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x2b,0x00,0x00,0x00, +0x4d,0x00,0x00,0x00,0x68,0x00,0x00,0x00,0x10,0x00,0x06,0x00, +0x04,0x00,0x00,0x00,0x11,0x00,0x00,0x00,0x00,0x01,0x00,0x00, +0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x0b,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x1a,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x12,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, +0x1b,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x29,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x29,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x29,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x08,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x29,0x00,0x00,0x00, +0x03,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x29,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x47,0x00,0x03,0x00, +0x29,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x48,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x49,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x49,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x49,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x14,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0x4b,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x4b,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x4b,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, +0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x4d,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x65,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0x66,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x66,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x66,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x68,0x00,0x00,0x00, +0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x68,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x96,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, +0x19,0x00,0x00,0x00,0x13,0x00,0x02,0x00,0x02,0x00,0x00,0x00, +0x21,0x00,0x03,0x00,0x03,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x15,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x17,0x00,0x04,0x00,0x09,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x0a,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x09,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x0d,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x10,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00, +0x12,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x15,0x00,0x00,0x00,0x40,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x1e,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0x1e,0x00,0x07,0x00,0x29,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x2a,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x29,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0x2a,0x00,0x00,0x00,0x2b,0x00,0x00,0x00, +0x09,0x00,0x00,0x00,0x15,0x00,0x04,0x00,0x2c,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x2c,0x00,0x00,0x00,0x2d,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x2e,0x00,0x00,0x00,0x09,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x14,0x00,0x02,0x00,0x32,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x38,0x00,0x00,0x00, +0x00,0x04,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x3e,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x16,0x00,0x03,0x00, +0x42,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x16,0x00,0x03,0x00, +0x45,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x15,0x00,0x04,0x00, +0x46,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x47,0x00,0x00,0x00, +0x10,0x00,0x00,0x00,0x1c,0x00,0x04,0x00,0x48,0x00,0x00,0x00, +0x46,0x00,0x00,0x00,0x47,0x00,0x00,0x00,0x1e,0x00,0x05,0x00, +0x49,0x00,0x00,0x00,0x45,0x00,0x00,0x00,0x45,0x00,0x00,0x00, +0x48,0x00,0x00,0x00,0x1d,0x00,0x03,0x00,0x4a,0x00,0x00,0x00, +0x49,0x00,0x00,0x00,0x1e,0x00,0x03,0x00,0x4b,0x00,0x00,0x00, +0x4a,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x4c,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x4b,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0x4c,0x00,0x00,0x00,0x4d,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x2c,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x50,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x45,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x2c,0x00,0x00,0x00,0x56,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x1d,0x00,0x03,0x00,0x65,0x00,0x00,0x00,0x45,0x00,0x00,0x00, +0x1e,0x00,0x03,0x00,0x66,0x00,0x00,0x00,0x65,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x67,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x66,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x67,0x00,0x00,0x00, +0x68,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x2c,0x00,0x00,0x00,0x6f,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x73,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x46,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x2c,0x00,0x00,0x00, +0x78,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x94,0x00,0x00,0x00,0x00,0x01,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x95,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x2c,0x00,0x06,0x00,0x09,0x00,0x00,0x00, +0x96,0x00,0x00,0x00,0x94,0x00,0x00,0x00,0x95,0x00,0x00,0x00, +0x95,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x79,0x01,0x00,0x00,0x11,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x7a,0x01,0x00,0x00,0x02,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x7b,0x01,0x00,0x00, +0x12,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x7c,0x01,0x00,0x00,0x03,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x7d,0x01,0x00,0x00,0x13,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x7e,0x01,0x00,0x00, +0x14,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x7f,0x01,0x00,0x00,0x05,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x80,0x01,0x00,0x00,0x15,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x81,0x01,0x00,0x00, +0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x82,0x01,0x00,0x00,0x16,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x83,0x01,0x00,0x00,0x07,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x84,0x01,0x00,0x00, +0x17,0x00,0x00,0x00,0x36,0x00,0x05,0x00,0x02,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x03,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0x05,0x00,0x00,0x00,0xf7,0x00,0x03,0x00, +0x97,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xfb,0x00,0x03,0x00, +0x0c,0x00,0x00,0x00,0x98,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0x98,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00, +0x0e,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, +0x0e,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x11,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x10,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00,0x13,0x00,0x00,0x00, +0x12,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x13,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x16,0x00,0x00,0x00, +0x14,0x00,0x00,0x00,0x15,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x17,0x00,0x00,0x00,0x11,0x00,0x00,0x00, +0x16,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x1b,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x15,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x1f,0x00,0x00,0x00, +0x1b,0x00,0x00,0x00,0x1e,0x00,0x00,0x00,0x89,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x1b,0x00,0x00,0x00, +0x1e,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x25,0x00,0x00,0x00,0x1e,0x00,0x00,0x00,0x17,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x27,0x00,0x00,0x00, +0x25,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x2e,0x00,0x00,0x00,0x2f,0x00,0x00,0x00,0x2b,0x00,0x00,0x00, +0x2d,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x30,0x00,0x00,0x00,0x2f,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x31,0x00,0x00,0x00,0x30,0x00,0x00,0x00, +0x1e,0x00,0x00,0x00,0xae,0x00,0x05,0x00,0x32,0x00,0x00,0x00, +0x33,0x00,0x00,0x00,0x27,0x00,0x00,0x00,0x31,0x00,0x00,0x00, +0xf7,0x00,0x03,0x00,0x35,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x33,0x00,0x00,0x00,0x34,0x00,0x00,0x00, +0x35,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x34,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x97,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0x35,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x3a,0x00,0x00,0x00,0x38,0x00,0x00,0x00,0x17,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3c,0x00,0x00,0x00, +0x1e,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x3d,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, +0x3c,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x40,0x00,0x00,0x00,0x3e,0x00,0x00,0x00,0x1f,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x41,0x00,0x00,0x00, +0x3d,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x41,0x00,0x07,0x00, +0x50,0x00,0x00,0x00,0x51,0x00,0x00,0x00,0x4d,0x00,0x00,0x00, +0x4e,0x00,0x00,0x00,0x27,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x45,0x00,0x00,0x00,0x52,0x00,0x00,0x00, +0x51,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x42,0x00,0x00,0x00, +0x53,0x00,0x00,0x00,0x52,0x00,0x00,0x00,0x41,0x00,0x07,0x00, +0x50,0x00,0x00,0x00,0x57,0x00,0x00,0x00,0x4d,0x00,0x00,0x00, +0x4e,0x00,0x00,0x00,0x27,0x00,0x00,0x00,0x56,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x45,0x00,0x00,0x00,0x58,0x00,0x00,0x00, +0x57,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x42,0x00,0x00,0x00, +0x59,0x00,0x00,0x00,0x58,0x00,0x00,0x00,0x41,0x00,0x08,0x00, +0x73,0x00,0x00,0x00,0x74,0x00,0x00,0x00,0x4d,0x00,0x00,0x00, +0x4e,0x00,0x00,0x00,0x27,0x00,0x00,0x00,0x6f,0x00,0x00,0x00, +0x40,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x46,0x00,0x00,0x00, +0x75,0x00,0x00,0x00,0x74,0x00,0x00,0x00,0x71,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x76,0x00,0x00,0x00,0x75,0x00,0x00,0x00, +0x7c,0x00,0x04,0x00,0x2c,0x00,0x00,0x00,0x77,0x00,0x00,0x00, +0x76,0x00,0x00,0x00,0xc7,0x00,0x05,0x00,0x2c,0x00,0x00,0x00, +0x79,0x00,0x00,0x00,0x77,0x00,0x00,0x00,0x78,0x00,0x00,0x00, +0x6f,0x00,0x04,0x00,0x42,0x00,0x00,0x00,0x7a,0x00,0x00,0x00, +0x79,0x00,0x00,0x00,0x0c,0x00,0x08,0x00,0x42,0x00,0x00,0x00, +0x7d,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00, +0x53,0x00,0x00,0x00,0x7a,0x00,0x00,0x00,0x59,0x00,0x00,0x00, +0x73,0x00,0x04,0x00,0x45,0x00,0x00,0x00,0x7e,0x00,0x00,0x00, +0x7d,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x50,0x00,0x00,0x00, +0x7f,0x00,0x00,0x00,0x68,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, +0x41,0x00,0x00,0x00,0x3e,0x00,0x03,0x00,0x7f,0x00,0x00,0x00, +0x7e,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x83,0x00,0x00,0x00,0x41,0x00,0x00,0x00,0x47,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x46,0x00,0x00,0x00,0x8a,0x00,0x00,0x00, +0x74,0x00,0x00,0x00,0xc2,0x00,0x05,0x00,0x46,0x00,0x00,0x00, +0x8b,0x00,0x00,0x00,0x8a,0x00,0x00,0x00,0x2d,0x00,0x00,0x00, +0x70,0x00,0x04,0x00,0x42,0x00,0x00,0x00,0x8c,0x00,0x00,0x00, +0x8b,0x00,0x00,0x00,0x0c,0x00,0x08,0x00,0x42,0x00,0x00,0x00, +0x8f,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00, +0x53,0x00,0x00,0x00,0x8c,0x00,0x00,0x00,0x59,0x00,0x00,0x00, +0x73,0x00,0x04,0x00,0x45,0x00,0x00,0x00,0x90,0x00,0x00,0x00, +0x8f,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x50,0x00,0x00,0x00, +0x91,0x00,0x00,0x00,0x68,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, +0x83,0x00,0x00,0x00,0x3e,0x00,0x03,0x00,0x91,0x00,0x00,0x00, +0x90,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xa4,0x00,0x00,0x00,0x41,0x00,0x00,0x00,0x95,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa6,0x00,0x00,0x00, +0x40,0x00,0x00,0x00,0x95,0x00,0x00,0x00,0x41,0x00,0x08,0x00, +0x73,0x00,0x00,0x00,0xa7,0x00,0x00,0x00,0x4d,0x00,0x00,0x00, +0x4e,0x00,0x00,0x00,0x27,0x00,0x00,0x00,0x6f,0x00,0x00,0x00, +0xa6,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x46,0x00,0x00,0x00, +0xa8,0x00,0x00,0x00,0xa7,0x00,0x00,0x00,0x71,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xa9,0x00,0x00,0x00,0xa8,0x00,0x00,0x00, +0x7c,0x00,0x04,0x00,0x2c,0x00,0x00,0x00,0xaa,0x00,0x00,0x00, +0xa9,0x00,0x00,0x00,0xc7,0x00,0x05,0x00,0x2c,0x00,0x00,0x00, +0xab,0x00,0x00,0x00,0xaa,0x00,0x00,0x00,0x78,0x00,0x00,0x00, +0x6f,0x00,0x04,0x00,0x42,0x00,0x00,0x00,0xac,0x00,0x00,0x00, +0xab,0x00,0x00,0x00,0x0c,0x00,0x08,0x00,0x42,0x00,0x00,0x00, +0xae,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00, +0x53,0x00,0x00,0x00,0xac,0x00,0x00,0x00,0x59,0x00,0x00,0x00, +0x73,0x00,0x04,0x00,0x45,0x00,0x00,0x00,0xaf,0x00,0x00,0x00, +0xae,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x50,0x00,0x00,0x00, +0xb0,0x00,0x00,0x00,0x68,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, +0xa4,0x00,0x00,0x00,0x3e,0x00,0x03,0x00,0xb0,0x00,0x00,0x00, +0xaf,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xb2,0x00,0x00,0x00,0x41,0x00,0x00,0x00,0x79,0x01,0x00,0x00, +0x3d,0x00,0x04,0x00,0x46,0x00,0x00,0x00,0xb5,0x00,0x00,0x00, +0xa7,0x00,0x00,0x00,0xc2,0x00,0x05,0x00,0x46,0x00,0x00,0x00, +0xb6,0x00,0x00,0x00,0xb5,0x00,0x00,0x00,0x2d,0x00,0x00,0x00, +0x70,0x00,0x04,0x00,0x42,0x00,0x00,0x00,0xb7,0x00,0x00,0x00, +0xb6,0x00,0x00,0x00,0x0c,0x00,0x08,0x00,0x42,0x00,0x00,0x00, +0xb9,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00, +0x53,0x00,0x00,0x00,0xb7,0x00,0x00,0x00,0x59,0x00,0x00,0x00, +0x73,0x00,0x04,0x00,0x45,0x00,0x00,0x00,0xba,0x00,0x00,0x00, +0xb9,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x50,0x00,0x00,0x00, +0xbb,0x00,0x00,0x00,0x68,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, +0xb2,0x00,0x00,0x00,0x3e,0x00,0x03,0x00,0xbb,0x00,0x00,0x00, +0xba,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xc3,0x00,0x00,0x00,0x41,0x00,0x00,0x00,0x7a,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xc5,0x00,0x00,0x00, +0x40,0x00,0x00,0x00,0x7a,0x01,0x00,0x00,0x41,0x00,0x08,0x00, +0x73,0x00,0x00,0x00,0xc6,0x00,0x00,0x00,0x4d,0x00,0x00,0x00, +0x4e,0x00,0x00,0x00,0x27,0x00,0x00,0x00,0x6f,0x00,0x00,0x00, +0xc5,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x46,0x00,0x00,0x00, +0xc7,0x00,0x00,0x00,0xc6,0x00,0x00,0x00,0x71,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xc8,0x00,0x00,0x00,0xc7,0x00,0x00,0x00, +0x7c,0x00,0x04,0x00,0x2c,0x00,0x00,0x00,0xc9,0x00,0x00,0x00, +0xc8,0x00,0x00,0x00,0xc7,0x00,0x05,0x00,0x2c,0x00,0x00,0x00, +0xca,0x00,0x00,0x00,0xc9,0x00,0x00,0x00,0x78,0x00,0x00,0x00, +0x6f,0x00,0x04,0x00,0x42,0x00,0x00,0x00,0xcb,0x00,0x00,0x00, +0xca,0x00,0x00,0x00,0x0c,0x00,0x08,0x00,0x42,0x00,0x00,0x00, +0xcd,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00, +0x53,0x00,0x00,0x00,0xcb,0x00,0x00,0x00,0x59,0x00,0x00,0x00, +0x73,0x00,0x04,0x00,0x45,0x00,0x00,0x00,0xce,0x00,0x00,0x00, +0xcd,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x50,0x00,0x00,0x00, +0xcf,0x00,0x00,0x00,0x68,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, +0xc3,0x00,0x00,0x00,0x3e,0x00,0x03,0x00,0xcf,0x00,0x00,0x00, +0xce,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xd1,0x00,0x00,0x00,0x41,0x00,0x00,0x00,0x7b,0x01,0x00,0x00, +0x3d,0x00,0x04,0x00,0x46,0x00,0x00,0x00,0xd4,0x00,0x00,0x00, +0xc6,0x00,0x00,0x00,0xc2,0x00,0x05,0x00,0x46,0x00,0x00,0x00, +0xd5,0x00,0x00,0x00,0xd4,0x00,0x00,0x00,0x2d,0x00,0x00,0x00, +0x70,0x00,0x04,0x00,0x42,0x00,0x00,0x00,0xd6,0x00,0x00,0x00, +0xd5,0x00,0x00,0x00,0x0c,0x00,0x08,0x00,0x42,0x00,0x00,0x00, +0xd8,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00, +0x53,0x00,0x00,0x00,0xd6,0x00,0x00,0x00,0x59,0x00,0x00,0x00, +0x73,0x00,0x04,0x00,0x45,0x00,0x00,0x00,0xd9,0x00,0x00,0x00, +0xd8,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x50,0x00,0x00,0x00, +0xda,0x00,0x00,0x00,0x68,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, +0xd1,0x00,0x00,0x00,0x3e,0x00,0x03,0x00,0xda,0x00,0x00,0x00, +0xd9,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xe2,0x00,0x00,0x00,0x41,0x00,0x00,0x00,0x7c,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xe4,0x00,0x00,0x00, +0x40,0x00,0x00,0x00,0x7c,0x01,0x00,0x00,0x41,0x00,0x08,0x00, +0x73,0x00,0x00,0x00,0xe5,0x00,0x00,0x00,0x4d,0x00,0x00,0x00, +0x4e,0x00,0x00,0x00,0x27,0x00,0x00,0x00,0x6f,0x00,0x00,0x00, +0xe4,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x46,0x00,0x00,0x00, +0xe6,0x00,0x00,0x00,0xe5,0x00,0x00,0x00,0x71,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xe7,0x00,0x00,0x00,0xe6,0x00,0x00,0x00, +0x7c,0x00,0x04,0x00,0x2c,0x00,0x00,0x00,0xe8,0x00,0x00,0x00, +0xe7,0x00,0x00,0x00,0xc7,0x00,0x05,0x00,0x2c,0x00,0x00,0x00, +0xe9,0x00,0x00,0x00,0xe8,0x00,0x00,0x00,0x78,0x00,0x00,0x00, +0x6f,0x00,0x04,0x00,0x42,0x00,0x00,0x00,0xea,0x00,0x00,0x00, +0xe9,0x00,0x00,0x00,0x0c,0x00,0x08,0x00,0x42,0x00,0x00,0x00, +0xec,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00, +0x53,0x00,0x00,0x00,0xea,0x00,0x00,0x00,0x59,0x00,0x00,0x00, +0x73,0x00,0x04,0x00,0x45,0x00,0x00,0x00,0xed,0x00,0x00,0x00, +0xec,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x50,0x00,0x00,0x00, +0xee,0x00,0x00,0x00,0x68,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, +0xe2,0x00,0x00,0x00,0x3e,0x00,0x03,0x00,0xee,0x00,0x00,0x00, +0xed,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xf0,0x00,0x00,0x00,0x41,0x00,0x00,0x00,0x7d,0x01,0x00,0x00, +0x3d,0x00,0x04,0x00,0x46,0x00,0x00,0x00,0xf3,0x00,0x00,0x00, +0xe5,0x00,0x00,0x00,0xc2,0x00,0x05,0x00,0x46,0x00,0x00,0x00, +0xf4,0x00,0x00,0x00,0xf3,0x00,0x00,0x00,0x2d,0x00,0x00,0x00, +0x70,0x00,0x04,0x00,0x42,0x00,0x00,0x00,0xf5,0x00,0x00,0x00, +0xf4,0x00,0x00,0x00,0x0c,0x00,0x08,0x00,0x42,0x00,0x00,0x00, +0xf7,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00, +0x53,0x00,0x00,0x00,0xf5,0x00,0x00,0x00,0x59,0x00,0x00,0x00, +0x73,0x00,0x04,0x00,0x45,0x00,0x00,0x00,0xf8,0x00,0x00,0x00, +0xf7,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x50,0x00,0x00,0x00, +0xf9,0x00,0x00,0x00,0x68,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, +0xf0,0x00,0x00,0x00,0x3e,0x00,0x03,0x00,0xf9,0x00,0x00,0x00, +0xf8,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x01,0x01,0x00,0x00,0x41,0x00,0x00,0x00,0x10,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x03,0x01,0x00,0x00, +0x40,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x41,0x00,0x08,0x00, +0x73,0x00,0x00,0x00,0x04,0x01,0x00,0x00,0x4d,0x00,0x00,0x00, +0x4e,0x00,0x00,0x00,0x27,0x00,0x00,0x00,0x6f,0x00,0x00,0x00, +0x03,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0x46,0x00,0x00,0x00, +0x05,0x01,0x00,0x00,0x04,0x01,0x00,0x00,0x71,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x06,0x01,0x00,0x00,0x05,0x01,0x00,0x00, +0x7c,0x00,0x04,0x00,0x2c,0x00,0x00,0x00,0x07,0x01,0x00,0x00, +0x06,0x01,0x00,0x00,0xc7,0x00,0x05,0x00,0x2c,0x00,0x00,0x00, +0x08,0x01,0x00,0x00,0x07,0x01,0x00,0x00,0x78,0x00,0x00,0x00, +0x6f,0x00,0x04,0x00,0x42,0x00,0x00,0x00,0x09,0x01,0x00,0x00, +0x08,0x01,0x00,0x00,0x0c,0x00,0x08,0x00,0x42,0x00,0x00,0x00, +0x0b,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00, +0x53,0x00,0x00,0x00,0x09,0x01,0x00,0x00,0x59,0x00,0x00,0x00, +0x73,0x00,0x04,0x00,0x45,0x00,0x00,0x00,0x0c,0x01,0x00,0x00, +0x0b,0x01,0x00,0x00,0x41,0x00,0x06,0x00,0x50,0x00,0x00,0x00, +0x0d,0x01,0x00,0x00,0x68,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, +0x01,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x0d,0x01,0x00,0x00, +0x0c,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x0f,0x01,0x00,0x00,0x41,0x00,0x00,0x00,0x7e,0x01,0x00,0x00, +0x3d,0x00,0x04,0x00,0x46,0x00,0x00,0x00,0x12,0x01,0x00,0x00, +0x04,0x01,0x00,0x00,0xc2,0x00,0x05,0x00,0x46,0x00,0x00,0x00, +0x13,0x01,0x00,0x00,0x12,0x01,0x00,0x00,0x2d,0x00,0x00,0x00, +0x70,0x00,0x04,0x00,0x42,0x00,0x00,0x00,0x14,0x01,0x00,0x00, +0x13,0x01,0x00,0x00,0x0c,0x00,0x08,0x00,0x42,0x00,0x00,0x00, +0x16,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00, +0x53,0x00,0x00,0x00,0x14,0x01,0x00,0x00,0x59,0x00,0x00,0x00, +0x73,0x00,0x04,0x00,0x45,0x00,0x00,0x00,0x17,0x01,0x00,0x00, +0x16,0x01,0x00,0x00,0x41,0x00,0x06,0x00,0x50,0x00,0x00,0x00, +0x18,0x01,0x00,0x00,0x68,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, +0x0f,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x18,0x01,0x00,0x00, +0x17,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x20,0x01,0x00,0x00,0x41,0x00,0x00,0x00,0x7f,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x22,0x01,0x00,0x00, +0x40,0x00,0x00,0x00,0x7f,0x01,0x00,0x00,0x41,0x00,0x08,0x00, +0x73,0x00,0x00,0x00,0x23,0x01,0x00,0x00,0x4d,0x00,0x00,0x00, +0x4e,0x00,0x00,0x00,0x27,0x00,0x00,0x00,0x6f,0x00,0x00,0x00, +0x22,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0x46,0x00,0x00,0x00, +0x24,0x01,0x00,0x00,0x23,0x01,0x00,0x00,0x71,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x25,0x01,0x00,0x00,0x24,0x01,0x00,0x00, +0x7c,0x00,0x04,0x00,0x2c,0x00,0x00,0x00,0x26,0x01,0x00,0x00, +0x25,0x01,0x00,0x00,0xc7,0x00,0x05,0x00,0x2c,0x00,0x00,0x00, +0x27,0x01,0x00,0x00,0x26,0x01,0x00,0x00,0x78,0x00,0x00,0x00, +0x6f,0x00,0x04,0x00,0x42,0x00,0x00,0x00,0x28,0x01,0x00,0x00, +0x27,0x01,0x00,0x00,0x0c,0x00,0x08,0x00,0x42,0x00,0x00,0x00, +0x2a,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00, +0x53,0x00,0x00,0x00,0x28,0x01,0x00,0x00,0x59,0x00,0x00,0x00, +0x73,0x00,0x04,0x00,0x45,0x00,0x00,0x00,0x2b,0x01,0x00,0x00, +0x2a,0x01,0x00,0x00,0x41,0x00,0x06,0x00,0x50,0x00,0x00,0x00, +0x2c,0x01,0x00,0x00,0x68,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, +0x20,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x2c,0x01,0x00,0x00, +0x2b,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x2e,0x01,0x00,0x00,0x41,0x00,0x00,0x00,0x80,0x01,0x00,0x00, +0x3d,0x00,0x04,0x00,0x46,0x00,0x00,0x00,0x31,0x01,0x00,0x00, +0x23,0x01,0x00,0x00,0xc2,0x00,0x05,0x00,0x46,0x00,0x00,0x00, +0x32,0x01,0x00,0x00,0x31,0x01,0x00,0x00,0x2d,0x00,0x00,0x00, +0x70,0x00,0x04,0x00,0x42,0x00,0x00,0x00,0x33,0x01,0x00,0x00, +0x32,0x01,0x00,0x00,0x0c,0x00,0x08,0x00,0x42,0x00,0x00,0x00, +0x35,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00, +0x53,0x00,0x00,0x00,0x33,0x01,0x00,0x00,0x59,0x00,0x00,0x00, +0x73,0x00,0x04,0x00,0x45,0x00,0x00,0x00,0x36,0x01,0x00,0x00, +0x35,0x01,0x00,0x00,0x41,0x00,0x06,0x00,0x50,0x00,0x00,0x00, +0x37,0x01,0x00,0x00,0x68,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, +0x2e,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x37,0x01,0x00,0x00, +0x36,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x3f,0x01,0x00,0x00,0x41,0x00,0x00,0x00,0x81,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x41,0x01,0x00,0x00, +0x40,0x00,0x00,0x00,0x81,0x01,0x00,0x00,0x41,0x00,0x08,0x00, +0x73,0x00,0x00,0x00,0x42,0x01,0x00,0x00,0x4d,0x00,0x00,0x00, +0x4e,0x00,0x00,0x00,0x27,0x00,0x00,0x00,0x6f,0x00,0x00,0x00, +0x41,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0x46,0x00,0x00,0x00, +0x43,0x01,0x00,0x00,0x42,0x01,0x00,0x00,0x71,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x44,0x01,0x00,0x00,0x43,0x01,0x00,0x00, +0x7c,0x00,0x04,0x00,0x2c,0x00,0x00,0x00,0x45,0x01,0x00,0x00, +0x44,0x01,0x00,0x00,0xc7,0x00,0x05,0x00,0x2c,0x00,0x00,0x00, +0x46,0x01,0x00,0x00,0x45,0x01,0x00,0x00,0x78,0x00,0x00,0x00, +0x6f,0x00,0x04,0x00,0x42,0x00,0x00,0x00,0x47,0x01,0x00,0x00, +0x46,0x01,0x00,0x00,0x0c,0x00,0x08,0x00,0x42,0x00,0x00,0x00, +0x49,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00, +0x53,0x00,0x00,0x00,0x47,0x01,0x00,0x00,0x59,0x00,0x00,0x00, +0x73,0x00,0x04,0x00,0x45,0x00,0x00,0x00,0x4a,0x01,0x00,0x00, +0x49,0x01,0x00,0x00,0x41,0x00,0x06,0x00,0x50,0x00,0x00,0x00, +0x4b,0x01,0x00,0x00,0x68,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, +0x3f,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x4b,0x01,0x00,0x00, +0x4a,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x4d,0x01,0x00,0x00,0x41,0x00,0x00,0x00,0x82,0x01,0x00,0x00, +0x3d,0x00,0x04,0x00,0x46,0x00,0x00,0x00,0x50,0x01,0x00,0x00, +0x42,0x01,0x00,0x00,0xc2,0x00,0x05,0x00,0x46,0x00,0x00,0x00, +0x51,0x01,0x00,0x00,0x50,0x01,0x00,0x00,0x2d,0x00,0x00,0x00, +0x70,0x00,0x04,0x00,0x42,0x00,0x00,0x00,0x52,0x01,0x00,0x00, +0x51,0x01,0x00,0x00,0x0c,0x00,0x08,0x00,0x42,0x00,0x00,0x00, +0x54,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00, +0x53,0x00,0x00,0x00,0x52,0x01,0x00,0x00,0x59,0x00,0x00,0x00, +0x73,0x00,0x04,0x00,0x45,0x00,0x00,0x00,0x55,0x01,0x00,0x00, +0x54,0x01,0x00,0x00,0x41,0x00,0x06,0x00,0x50,0x00,0x00,0x00, +0x56,0x01,0x00,0x00,0x68,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, +0x4d,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x56,0x01,0x00,0x00, +0x55,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x5e,0x01,0x00,0x00,0x41,0x00,0x00,0x00,0x83,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x60,0x01,0x00,0x00, +0x40,0x00,0x00,0x00,0x83,0x01,0x00,0x00,0x41,0x00,0x08,0x00, +0x73,0x00,0x00,0x00,0x61,0x01,0x00,0x00,0x4d,0x00,0x00,0x00, +0x4e,0x00,0x00,0x00,0x27,0x00,0x00,0x00,0x6f,0x00,0x00,0x00, +0x60,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0x46,0x00,0x00,0x00, +0x62,0x01,0x00,0x00,0x61,0x01,0x00,0x00,0x71,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x63,0x01,0x00,0x00,0x62,0x01,0x00,0x00, +0x7c,0x00,0x04,0x00,0x2c,0x00,0x00,0x00,0x64,0x01,0x00,0x00, +0x63,0x01,0x00,0x00,0xc7,0x00,0x05,0x00,0x2c,0x00,0x00,0x00, +0x65,0x01,0x00,0x00,0x64,0x01,0x00,0x00,0x78,0x00,0x00,0x00, +0x6f,0x00,0x04,0x00,0x42,0x00,0x00,0x00,0x66,0x01,0x00,0x00, +0x65,0x01,0x00,0x00,0x0c,0x00,0x08,0x00,0x42,0x00,0x00,0x00, +0x68,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00, +0x53,0x00,0x00,0x00,0x66,0x01,0x00,0x00,0x59,0x00,0x00,0x00, +0x73,0x00,0x04,0x00,0x45,0x00,0x00,0x00,0x69,0x01,0x00,0x00, +0x68,0x01,0x00,0x00,0x41,0x00,0x06,0x00,0x50,0x00,0x00,0x00, +0x6a,0x01,0x00,0x00,0x68,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, +0x5e,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x6a,0x01,0x00,0x00, +0x69,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x6c,0x01,0x00,0x00,0x41,0x00,0x00,0x00,0x84,0x01,0x00,0x00, +0x3d,0x00,0x04,0x00,0x46,0x00,0x00,0x00,0x6f,0x01,0x00,0x00, +0x61,0x01,0x00,0x00,0xc2,0x00,0x05,0x00,0x46,0x00,0x00,0x00, +0x70,0x01,0x00,0x00,0x6f,0x01,0x00,0x00,0x2d,0x00,0x00,0x00, +0x70,0x00,0x04,0x00,0x42,0x00,0x00,0x00,0x71,0x01,0x00,0x00, +0x70,0x01,0x00,0x00,0x0c,0x00,0x08,0x00,0x42,0x00,0x00,0x00, +0x73,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00, +0x53,0x00,0x00,0x00,0x71,0x01,0x00,0x00,0x59,0x00,0x00,0x00, +0x73,0x00,0x04,0x00,0x45,0x00,0x00,0x00,0x74,0x01,0x00,0x00, +0x73,0x01,0x00,0x00,0x41,0x00,0x06,0x00,0x50,0x00,0x00,0x00, +0x75,0x01,0x00,0x00,0x68,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, +0x6c,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x75,0x01,0x00,0x00, +0x74,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x97,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0x97,0x00,0x00,0x00,0xfd,0x00,0x01,0x00, +0x38,0x00,0x01,0x00, +}; +const uint64_t dequant_q4_1_len = 5248; + +unsigned char dequant_q4_K_data[] = { +0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, +0xae,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, +0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x27,0x00,0x00,0x00, +0x11,0x00,0x02,0x00,0x51,0x11,0x00,0x00,0x11,0x00,0x02,0x00, +0x60,0x11,0x00,0x00,0x0b,0x00,0x06,0x00,0x01,0x00,0x00,0x00, +0x47,0x4c,0x53,0x4c,0x2e,0x73,0x74,0x64,0x2e,0x34,0x35,0x30, +0x00,0x00,0x00,0x00,0x0e,0x00,0x03,0x00,0x00,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x0f,0x00,0x0a,0x00,0x05,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x6d,0x61,0x69,0x6e,0x00,0x00,0x00,0x00, +0x16,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x30,0x00,0x00,0x00, +0x4c,0x00,0x00,0x00,0x08,0x01,0x00,0x00,0x10,0x00,0x06,0x00, 0x04,0x00,0x00,0x00,0x11,0x00,0x00,0x00,0x20,0x00,0x00,0x00, 0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x17,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x1a,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x16,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x1a,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x1e,0x00,0x00,0x00,0x00,0x00,0x00,0x00, 0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x23,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x23,0x00,0x00,0x00, +0x1e,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x1e,0x00,0x00,0x00, 0x02,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x08,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x23,0x00,0x00,0x00,0x03,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x47,0x00,0x03,0x00, -0x23,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x33,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x1b,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x49,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x4b,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x1e,0x00,0x00,0x00,0x03,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x1e,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x10,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x1e,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x30,0x00,0x00,0x00, +0x0b,0x00,0x00,0x00,0x1b,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x45,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x47,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x48,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x48,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x48,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x10,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x49,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x90,0x00,0x00,0x00,0x48,0x00,0x04,0x00, +0x4a,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x4a,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, +0x4a,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x4c,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x4c,0x00,0x00,0x00,0x21,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x05,0x01,0x00,0x00, +0x06,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x48,0x00,0x04,0x00, +0x06,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x19,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x06,0x01,0x00,0x00,0x00,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, +0x06,0x01,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x08,0x01,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x08,0x01,0x00,0x00,0x21,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x33,0x01,0x00,0x00, +0x0b,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x13,0x00,0x02,0x00, +0x02,0x00,0x00,0x00,0x21,0x00,0x03,0x00,0x03,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x15,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x10,0x00,0x00,0x00, +0x00,0x01,0x00,0x00,0x14,0x00,0x02,0x00,0x11,0x00,0x00,0x00, +0x17,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x03,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x15,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0x15,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x17,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x1e,0x00,0x07,0x00,0x1e,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x1f,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x1e,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0x1f,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0x09,0x00,0x00,0x00,0x15,0x00,0x04,0x00,0x21,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x21,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x23,0x00,0x00,0x00,0x09,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x21,0x00,0x00,0x00, +0x26,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0x15,0x00,0x00,0x00,0x30,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x35,0x00,0x00,0x00, +0x08,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x3b,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x16,0x00,0x03,0x00, +0x3e,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x16,0x00,0x03,0x00, +0x41,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x17,0x00,0x04,0x00, +0x42,0x00,0x00,0x00,0x41,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x15,0x00,0x04,0x00,0x43,0x00,0x00,0x00,0x08,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x44,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x1c,0x00,0x04,0x00, +0x45,0x00,0x00,0x00,0x43,0x00,0x00,0x00,0x44,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x46,0x00,0x00,0x00, +0x80,0x00,0x00,0x00,0x1c,0x00,0x04,0x00,0x47,0x00,0x00,0x00, +0x43,0x00,0x00,0x00,0x46,0x00,0x00,0x00,0x1e,0x00,0x05,0x00, +0x48,0x00,0x00,0x00,0x42,0x00,0x00,0x00,0x45,0x00,0x00,0x00, +0x47,0x00,0x00,0x00,0x1d,0x00,0x03,0x00,0x49,0x00,0x00,0x00, +0x48,0x00,0x00,0x00,0x1e,0x00,0x03,0x00,0x4a,0x00,0x00,0x00, +0x49,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x4b,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x4a,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0x4b,0x00,0x00,0x00,0x4c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x4e,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x41,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x54,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x5b,0x00,0x00,0x00,0x40,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x5f,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x64,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x72,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x43,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x21,0x00,0x00,0x00,0x77,0x00,0x00,0x00, +0x3f,0x00,0x00,0x00,0x15,0x00,0x04,0x00,0x79,0x00,0x00,0x00, +0x08,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x21,0x00,0x00,0x00,0x8f,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x21,0x00,0x00,0x00,0x96,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x21,0x00,0x00,0x00, +0x98,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xc6,0x00,0x00,0x00,0x05,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xda,0x00,0x00,0x00, +0x03,0x00,0x00,0x00,0x1d,0x00,0x03,0x00,0x05,0x01,0x00,0x00, +0x41,0x00,0x00,0x00,0x1e,0x00,0x03,0x00,0x06,0x01,0x00,0x00, +0x05,0x01,0x00,0x00,0x20,0x00,0x04,0x00,0x07,0x01,0x00,0x00, +0x0c,0x00,0x00,0x00,0x06,0x01,0x00,0x00,0x3b,0x00,0x04,0x00, +0x07,0x01,0x00,0x00,0x08,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x21,0x00,0x00,0x00,0x0e,0x01,0x00,0x00, +0x02,0x00,0x00,0x00,0x2c,0x00,0x06,0x00,0x14,0x00,0x00,0x00, +0x33,0x01,0x00,0x00,0x64,0x00,0x00,0x00,0x54,0x00,0x00,0x00, +0x54,0x00,0x00,0x00,0x2a,0x00,0x03,0x00,0x11,0x00,0x00,0x00, +0x36,0x01,0x00,0x00,0x29,0x00,0x03,0x00,0x11,0x00,0x00,0x00, +0x39,0x01,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xa7,0x01,0x00,0x00,0x21,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xaa,0x01,0x00,0x00,0x22,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xad,0x01,0x00,0x00, +0x23,0x00,0x00,0x00,0x36,0x00,0x05,0x00,0x02,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x03,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0x05,0x00,0x00,0x00,0xf7,0x00,0x03,0x00, +0x34,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0xfb,0x00,0x03,0x00, +0x09,0x00,0x00,0x00,0x35,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x35,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x0a,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0x0a,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0x3c,0x01,0x00,0x00,0x09,0x00,0x00,0x00, +0x35,0x01,0x00,0x00,0x32,0x01,0x00,0x00,0x0d,0x00,0x00,0x00, +0xb0,0x00,0x05,0x00,0x11,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0x3c,0x01,0x00,0x00,0x10,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0x0c,0x00,0x00,0x00,0x0d,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x12,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x0b,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00,0x18,0x00,0x00,0x00, +0x16,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x18,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x1a,0x00,0x00,0x00, +0x19,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x1a,0x00,0x00,0x00, +0x3c,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0x23,0x00,0x00,0x00, +0x24,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x22,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x25,0x00,0x00,0x00, +0x24,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x23,0x00,0x00,0x00, +0x27,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x26,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x28,0x00,0x00,0x00, +0x27,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x29,0x00,0x00,0x00,0x25,0x00,0x00,0x00,0x28,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x2a,0x00,0x00,0x00, +0x29,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0xae,0x00,0x05,0x00, +0x11,0x00,0x00,0x00,0x2b,0x00,0x00,0x00,0x1c,0x00,0x00,0x00, +0x2a,0x00,0x00,0x00,0xf7,0x00,0x03,0x00,0x2d,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x2b,0x00,0x00,0x00, +0x2c,0x00,0x00,0x00,0x2d,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0x2c,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x0c,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0x2d,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x17,0x00,0x00,0x00,0x31,0x00,0x00,0x00,0x30,0x00,0x00,0x00, +0x09,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x32,0x00,0x00,0x00,0x31,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x36,0x00,0x00,0x00,0x32,0x00,0x00,0x00, +0x35,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x39,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x35,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3d,0x00,0x00,0x00, +0x3b,0x00,0x00,0x00,0x36,0x00,0x00,0x00,0x41,0x00,0x08,0x00, +0x4e,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x4c,0x00,0x00,0x00, +0x22,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x22,0x00,0x00,0x00, +0x09,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x41,0x00,0x00,0x00, +0x50,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x73,0x00,0x04,0x00, +0x3e,0x00,0x00,0x00,0x51,0x00,0x00,0x00,0x50,0x00,0x00,0x00, +0x41,0x00,0x08,0x00,0x4e,0x00,0x00,0x00,0x55,0x00,0x00,0x00, +0x4c,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x1c,0x00,0x00,0x00, +0x22,0x00,0x00,0x00,0x54,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x41,0x00,0x00,0x00,0x56,0x00,0x00,0x00,0x55,0x00,0x00,0x00, +0x73,0x00,0x04,0x00,0x3e,0x00,0x00,0x00,0x57,0x00,0x00,0x00, +0x56,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x5a,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x10,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x5d,0x00,0x00,0x00, +0x5b,0x00,0x00,0x00,0x36,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x5e,0x00,0x00,0x00,0x5a,0x00,0x00,0x00, +0x5d,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x61,0x00,0x00,0x00,0x5f,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x62,0x00,0x00,0x00, +0x5e,0x00,0x00,0x00,0x61,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x66,0x00,0x00,0x00,0x64,0x00,0x00,0x00, +0x36,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x69,0x00,0x00,0x00,0x66,0x00,0x00,0x00,0x61,0x00,0x00,0x00, +0xb0,0x00,0x05,0x00,0x11,0x00,0x00,0x00,0x6b,0x00,0x00,0x00, +0x3d,0x00,0x00,0x00,0x5f,0x00,0x00,0x00,0xf7,0x00,0x03,0x00, +0x6d,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x6b,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x87,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0x6c,0x00,0x00,0x00,0x41,0x00,0x08,0x00, +0x72,0x00,0x00,0x00,0x73,0x00,0x00,0x00,0x4c,0x00,0x00,0x00, +0x22,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x26,0x00,0x00,0x00, +0x3d,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x43,0x00,0x00,0x00, +0x74,0x00,0x00,0x00,0x73,0x00,0x00,0x00,0x71,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x75,0x00,0x00,0x00,0x74,0x00,0x00,0x00, +0x7c,0x00,0x04,0x00,0x21,0x00,0x00,0x00,0x76,0x00,0x00,0x00, +0x75,0x00,0x00,0x00,0xc7,0x00,0x05,0x00,0x21,0x00,0x00,0x00, +0x78,0x00,0x00,0x00,0x76,0x00,0x00,0x00,0x77,0x00,0x00,0x00, +0x72,0x00,0x04,0x00,0x79,0x00,0x00,0x00,0x7a,0x00,0x00,0x00, +0x78,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x43,0x00,0x00,0x00, +0x7b,0x00,0x00,0x00,0x7a,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x7f,0x00,0x00,0x00,0x3d,0x00,0x00,0x00, +0x5f,0x00,0x00,0x00,0x41,0x00,0x08,0x00,0x72,0x00,0x00,0x00, +0x80,0x00,0x00,0x00,0x4c,0x00,0x00,0x00,0x22,0x00,0x00,0x00, +0x1c,0x00,0x00,0x00,0x26,0x00,0x00,0x00,0x7f,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x43,0x00,0x00,0x00,0x81,0x00,0x00,0x00, +0x80,0x00,0x00,0x00,0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x82,0x00,0x00,0x00,0x81,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, +0x21,0x00,0x00,0x00,0x83,0x00,0x00,0x00,0x82,0x00,0x00,0x00, +0xc7,0x00,0x05,0x00,0x21,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0x83,0x00,0x00,0x00,0x77,0x00,0x00,0x00,0x72,0x00,0x04,0x00, +0x79,0x00,0x00,0x00,0x85,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0x7c,0x00,0x04,0x00,0x43,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0x85,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x6d,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0x87,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x8a,0x00,0x00,0x00,0x3d,0x00,0x00,0x00, +0x5f,0x00,0x00,0x00,0x41,0x00,0x08,0x00,0x72,0x00,0x00,0x00, +0x8b,0x00,0x00,0x00,0x4c,0x00,0x00,0x00,0x22,0x00,0x00,0x00, +0x1c,0x00,0x00,0x00,0x26,0x00,0x00,0x00,0x8a,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x43,0x00,0x00,0x00,0x8c,0x00,0x00,0x00, +0x8b,0x00,0x00,0x00,0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x8d,0x00,0x00,0x00,0x8c,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, +0x21,0x00,0x00,0x00,0x8e,0x00,0x00,0x00,0x8d,0x00,0x00,0x00, +0xc7,0x00,0x05,0x00,0x21,0x00,0x00,0x00,0x90,0x00,0x00,0x00, +0x8e,0x00,0x00,0x00,0x8f,0x00,0x00,0x00,0x82,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x93,0x00,0x00,0x00,0x3d,0x00,0x00,0x00, +0x5f,0x00,0x00,0x00,0x41,0x00,0x08,0x00,0x72,0x00,0x00,0x00, +0x94,0x00,0x00,0x00,0x4c,0x00,0x00,0x00,0x22,0x00,0x00,0x00, +0x1c,0x00,0x00,0x00,0x26,0x00,0x00,0x00,0x93,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x43,0x00,0x00,0x00,0x95,0x00,0x00,0x00, +0x94,0x00,0x00,0x00,0xc2,0x00,0x05,0x00,0x43,0x00,0x00,0x00, +0x97,0x00,0x00,0x00,0x95,0x00,0x00,0x00,0x96,0x00,0x00,0x00, +0xc4,0x00,0x05,0x00,0x43,0x00,0x00,0x00,0x99,0x00,0x00,0x00, +0x97,0x00,0x00,0x00,0x98,0x00,0x00,0x00,0x71,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x9a,0x00,0x00,0x00,0x99,0x00,0x00,0x00, +0x7c,0x00,0x04,0x00,0x21,0x00,0x00,0x00,0x9b,0x00,0x00,0x00, +0x9a,0x00,0x00,0x00,0xc5,0x00,0x05,0x00,0x21,0x00,0x00,0x00, +0x9c,0x00,0x00,0x00,0x90,0x00,0x00,0x00,0x9b,0x00,0x00,0x00, +0x72,0x00,0x04,0x00,0x79,0x00,0x00,0x00,0x9d,0x00,0x00,0x00, +0x9c,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x43,0x00,0x00,0x00, +0x9e,0x00,0x00,0x00,0x9d,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x43,0x00,0x00,0x00,0xa3,0x00,0x00,0x00,0x8b,0x00,0x00,0x00, +0xc2,0x00,0x05,0x00,0x43,0x00,0x00,0x00,0xa4,0x00,0x00,0x00, +0xa3,0x00,0x00,0x00,0x98,0x00,0x00,0x00,0x41,0x00,0x08,0x00, +0x72,0x00,0x00,0x00,0xa7,0x00,0x00,0x00,0x4c,0x00,0x00,0x00, +0x22,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x26,0x00,0x00,0x00, +0x3d,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x43,0x00,0x00,0x00, +0xa8,0x00,0x00,0x00,0xa7,0x00,0x00,0x00,0xc2,0x00,0x05,0x00, +0x43,0x00,0x00,0x00,0xa9,0x00,0x00,0x00,0xa8,0x00,0x00,0x00, +0x96,0x00,0x00,0x00,0xc4,0x00,0x05,0x00,0x43,0x00,0x00,0x00, +0xaa,0x00,0x00,0x00,0xa9,0x00,0x00,0x00,0x98,0x00,0x00,0x00, +0xc5,0x00,0x05,0x00,0x43,0x00,0x00,0x00,0xab,0x00,0x00,0x00, +0xa4,0x00,0x00,0x00,0xaa,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x6d,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x6d,0x00,0x00,0x00, +0xf5,0x00,0x07,0x00,0x43,0x00,0x00,0x00,0x3e,0x01,0x00,0x00, +0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0xab,0x00,0x00,0x00, +0x87,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x43,0x00,0x00,0x00, +0x3d,0x01,0x00,0x00,0x7b,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, +0x9e,0x00,0x00,0x00,0x87,0x00,0x00,0x00,0x70,0x00,0x04,0x00, +0x3e,0x00,0x00,0x00,0xaf,0x00,0x00,0x00,0x3d,0x01,0x00,0x00, +0x85,0x00,0x05,0x00,0x3e,0x00,0x00,0x00,0xb0,0x00,0x00,0x00, +0x51,0x00,0x00,0x00,0xaf,0x00,0x00,0x00,0x70,0x00,0x04,0x00, +0x3e,0x00,0x00,0x00,0xb4,0x00,0x00,0x00,0x3e,0x01,0x00,0x00, +0x85,0x00,0x05,0x00,0x3e,0x00,0x00,0x00,0xb5,0x00,0x00,0x00, +0x57,0x00,0x00,0x00,0xb4,0x00,0x00,0x00,0xf7,0x00,0x03,0x00, +0xb9,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x6b,0x00,0x00,0x00,0xb8,0x00,0x00,0x00,0xcf,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xb8,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xbc,0x00,0x00,0x00,0x3d,0x00,0x00,0x00, +0x54,0x00,0x00,0x00,0x41,0x00,0x08,0x00,0x72,0x00,0x00,0x00, +0xbd,0x00,0x00,0x00,0x4c,0x00,0x00,0x00,0x22,0x00,0x00,0x00, +0x1c,0x00,0x00,0x00,0x26,0x00,0x00,0x00,0xbc,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x43,0x00,0x00,0x00,0xbe,0x00,0x00,0x00, +0xbd,0x00,0x00,0x00,0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xbf,0x00,0x00,0x00,0xbe,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, +0x21,0x00,0x00,0x00,0xc0,0x00,0x00,0x00,0xbf,0x00,0x00,0x00, +0xc7,0x00,0x05,0x00,0x21,0x00,0x00,0x00,0xc1,0x00,0x00,0x00, +0xc0,0x00,0x00,0x00,0x77,0x00,0x00,0x00,0x72,0x00,0x04,0x00, +0x79,0x00,0x00,0x00,0xc2,0x00,0x00,0x00,0xc1,0x00,0x00,0x00, +0x7c,0x00,0x04,0x00,0x43,0x00,0x00,0x00,0xc3,0x00,0x00,0x00, +0xc2,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xc7,0x00,0x00,0x00,0x3d,0x00,0x00,0x00,0xc6,0x00,0x00,0x00, +0x41,0x00,0x08,0x00,0x72,0x00,0x00,0x00,0xc8,0x00,0x00,0x00, +0x4c,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x1c,0x00,0x00,0x00, +0x26,0x00,0x00,0x00,0xc7,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x43,0x00,0x00,0x00,0xc9,0x00,0x00,0x00,0xc8,0x00,0x00,0x00, +0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xca,0x00,0x00,0x00, +0xc9,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x21,0x00,0x00,0x00, +0xcb,0x00,0x00,0x00,0xca,0x00,0x00,0x00,0xc7,0x00,0x05,0x00, +0x21,0x00,0x00,0x00,0xcc,0x00,0x00,0x00,0xcb,0x00,0x00,0x00, +0x77,0x00,0x00,0x00,0x72,0x00,0x04,0x00,0x79,0x00,0x00,0x00, +0xcd,0x00,0x00,0x00,0xcc,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, +0x43,0x00,0x00,0x00,0xce,0x00,0x00,0x00,0xcd,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xb9,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xcf,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xd2,0x00,0x00,0x00,0x3d,0x00,0x00,0x00,0xc6,0x00,0x00,0x00, +0x41,0x00,0x08,0x00,0x72,0x00,0x00,0x00,0xd3,0x00,0x00,0x00, +0x4c,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x1c,0x00,0x00,0x00, +0x26,0x00,0x00,0x00,0xd2,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x43,0x00,0x00,0x00,0xd4,0x00,0x00,0x00,0xd3,0x00,0x00,0x00, +0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xd5,0x00,0x00,0x00, +0xd4,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x21,0x00,0x00,0x00, +0xd6,0x00,0x00,0x00,0xd5,0x00,0x00,0x00,0xc7,0x00,0x05,0x00, +0x21,0x00,0x00,0x00,0xd7,0x00,0x00,0x00,0xd6,0x00,0x00,0x00, +0x8f,0x00,0x00,0x00,0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xdb,0x00,0x00,0x00,0x3d,0x00,0x00,0x00,0xda,0x00,0x00,0x00, +0x41,0x00,0x08,0x00,0x72,0x00,0x00,0x00,0xdc,0x00,0x00,0x00, +0x4c,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x1c,0x00,0x00,0x00, +0x26,0x00,0x00,0x00,0xdb,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x43,0x00,0x00,0x00,0xdd,0x00,0x00,0x00,0xdc,0x00,0x00,0x00, +0xc2,0x00,0x05,0x00,0x43,0x00,0x00,0x00,0xde,0x00,0x00,0x00, +0xdd,0x00,0x00,0x00,0x96,0x00,0x00,0x00,0xc4,0x00,0x05,0x00, +0x43,0x00,0x00,0x00,0xdf,0x00,0x00,0x00,0xde,0x00,0x00,0x00, +0x98,0x00,0x00,0x00,0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xe0,0x00,0x00,0x00,0xdf,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, +0x21,0x00,0x00,0x00,0xe1,0x00,0x00,0x00,0xe0,0x00,0x00,0x00, +0xc5,0x00,0x05,0x00,0x21,0x00,0x00,0x00,0xe2,0x00,0x00,0x00, +0xd7,0x00,0x00,0x00,0xe1,0x00,0x00,0x00,0x72,0x00,0x04,0x00, +0x79,0x00,0x00,0x00,0xe3,0x00,0x00,0x00,0xe2,0x00,0x00,0x00, +0x7c,0x00,0x04,0x00,0x43,0x00,0x00,0x00,0xe4,0x00,0x00,0x00, +0xe3,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x43,0x00,0x00,0x00, +0xe9,0x00,0x00,0x00,0xd3,0x00,0x00,0x00,0xc2,0x00,0x05,0x00, +0x43,0x00,0x00,0x00,0xea,0x00,0x00,0x00,0xe9,0x00,0x00,0x00, +0x98,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xed,0x00,0x00,0x00,0x3d,0x00,0x00,0x00,0x54,0x00,0x00,0x00, +0x41,0x00,0x08,0x00,0x72,0x00,0x00,0x00,0xee,0x00,0x00,0x00, +0x4c,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x1c,0x00,0x00,0x00, +0x26,0x00,0x00,0x00,0xed,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x43,0x00,0x00,0x00,0xef,0x00,0x00,0x00,0xee,0x00,0x00,0x00, +0xc2,0x00,0x05,0x00,0x43,0x00,0x00,0x00,0xf0,0x00,0x00,0x00, +0xef,0x00,0x00,0x00,0x96,0x00,0x00,0x00,0xc4,0x00,0x05,0x00, +0x43,0x00,0x00,0x00,0xf1,0x00,0x00,0x00,0xf0,0x00,0x00,0x00, +0x98,0x00,0x00,0x00,0xc5,0x00,0x05,0x00,0x43,0x00,0x00,0x00, +0xf2,0x00,0x00,0x00,0xea,0x00,0x00,0x00,0xf1,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xb9,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xb9,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x43,0x00,0x00,0x00, +0x40,0x01,0x00,0x00,0xce,0x00,0x00,0x00,0xb8,0x00,0x00,0x00, +0xf2,0x00,0x00,0x00,0xcf,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, +0x43,0x00,0x00,0x00,0x3f,0x01,0x00,0x00,0xc3,0x00,0x00,0x00, +0xb8,0x00,0x00,0x00,0xe4,0x00,0x00,0x00,0xcf,0x00,0x00,0x00, +0x70,0x00,0x04,0x00,0x3e,0x00,0x00,0x00,0xf6,0x00,0x00,0x00, +0x3f,0x01,0x00,0x00,0x85,0x00,0x05,0x00,0x3e,0x00,0x00,0x00, +0xf7,0x00,0x00,0x00,0x51,0x00,0x00,0x00,0xf6,0x00,0x00,0x00, +0x70,0x00,0x04,0x00,0x3e,0x00,0x00,0x00,0xfb,0x00,0x00,0x00, +0x40,0x01,0x00,0x00,0x85,0x00,0x05,0x00,0x3e,0x00,0x00,0x00, +0xfc,0x00,0x00,0x00,0x57,0x00,0x00,0x00,0xfb,0x00,0x00,0x00, +0x41,0x00,0x08,0x00,0x72,0x00,0x00,0x00,0x12,0x01,0x00,0x00, +0x4c,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x1c,0x00,0x00,0x00, +0x0e,0x01,0x00,0x00,0x69,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x43,0x00,0x00,0x00,0x13,0x01,0x00,0x00,0x12,0x01,0x00,0x00, +0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x14,0x01,0x00,0x00, +0x13,0x01,0x00,0x00,0x7c,0x00,0x04,0x00,0x21,0x00,0x00,0x00, +0x15,0x01,0x00,0x00,0x14,0x01,0x00,0x00,0xc7,0x00,0x05,0x00, +0x21,0x00,0x00,0x00,0x16,0x01,0x00,0x00,0x15,0x01,0x00,0x00, +0x8f,0x00,0x00,0x00,0x6f,0x00,0x04,0x00,0x3e,0x00,0x00,0x00, +0x17,0x01,0x00,0x00,0x16,0x01,0x00,0x00,0x7f,0x00,0x04,0x00, +0x3e,0x00,0x00,0x00,0xa4,0x01,0x00,0x00,0xb5,0x00,0x00,0x00, +0x0c,0x00,0x08,0x00,0x3e,0x00,0x00,0x00,0x1a,0x01,0x00,0x00, +0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0xb0,0x00,0x00,0x00, +0x17,0x01,0x00,0x00,0xa4,0x01,0x00,0x00,0x73,0x00,0x04,0x00, +0x41,0x00,0x00,0x00,0x1b,0x01,0x00,0x00,0x1a,0x01,0x00,0x00, +0x41,0x00,0x06,0x00,0x4e,0x00,0x00,0x00,0x1c,0x01,0x00,0x00, +0x08,0x01,0x00,0x00,0x22,0x00,0x00,0x00,0x62,0x00,0x00,0x00, +0x3e,0x00,0x03,0x00,0x1c,0x01,0x00,0x00,0x1b,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x20,0x01,0x00,0x00, +0x62,0x00,0x00,0x00,0x64,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x43,0x00,0x00,0x00,0x27,0x01,0x00,0x00,0x12,0x01,0x00,0x00, +0xc2,0x00,0x05,0x00,0x43,0x00,0x00,0x00,0x28,0x01,0x00,0x00, +0x27,0x01,0x00,0x00,0x98,0x00,0x00,0x00,0x70,0x00,0x04,0x00, +0x3e,0x00,0x00,0x00,0x29,0x01,0x00,0x00,0x28,0x01,0x00,0x00, +0x7f,0x00,0x04,0x00,0x3e,0x00,0x00,0x00,0xa5,0x01,0x00,0x00, +0xfc,0x00,0x00,0x00,0x0c,0x00,0x08,0x00,0x3e,0x00,0x00,0x00, +0x2c,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00, +0xf7,0x00,0x00,0x00,0x29,0x01,0x00,0x00,0xa5,0x01,0x00,0x00, +0x73,0x00,0x04,0x00,0x41,0x00,0x00,0x00,0x2d,0x01,0x00,0x00, +0x2c,0x01,0x00,0x00,0x41,0x00,0x06,0x00,0x4e,0x00,0x00,0x00, +0x2e,0x01,0x00,0x00,0x08,0x01,0x00,0x00,0x22,0x00,0x00,0x00, +0x20,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x2e,0x01,0x00,0x00, +0x2d,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x4f,0x01,0x00,0x00,0x62,0x00,0x00,0x00,0x54,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x50,0x01,0x00,0x00, +0x69,0x00,0x00,0x00,0x54,0x00,0x00,0x00,0x41,0x00,0x08,0x00, +0x72,0x00,0x00,0x00,0x51,0x01,0x00,0x00,0x4c,0x00,0x00,0x00, +0x22,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x0e,0x01,0x00,0x00, +0x50,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0x43,0x00,0x00,0x00, +0x52,0x01,0x00,0x00,0x51,0x01,0x00,0x00,0x71,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x53,0x01,0x00,0x00,0x52,0x01,0x00,0x00, +0x7c,0x00,0x04,0x00,0x21,0x00,0x00,0x00,0x54,0x01,0x00,0x00, +0x53,0x01,0x00,0x00,0xc7,0x00,0x05,0x00,0x21,0x00,0x00,0x00, +0x55,0x01,0x00,0x00,0x54,0x01,0x00,0x00,0x8f,0x00,0x00,0x00, +0x6f,0x00,0x04,0x00,0x3e,0x00,0x00,0x00,0x56,0x01,0x00,0x00, +0x55,0x01,0x00,0x00,0x0c,0x00,0x08,0x00,0x3e,0x00,0x00,0x00, +0x58,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00, +0xb0,0x00,0x00,0x00,0x56,0x01,0x00,0x00,0xa4,0x01,0x00,0x00, +0x73,0x00,0x04,0x00,0x41,0x00,0x00,0x00,0x59,0x01,0x00,0x00, +0x58,0x01,0x00,0x00,0x41,0x00,0x06,0x00,0x4e,0x00,0x00,0x00, +0x5a,0x01,0x00,0x00,0x08,0x01,0x00,0x00,0x22,0x00,0x00,0x00, +0x4f,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x5a,0x01,0x00,0x00, +0x59,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x5c,0x01,0x00,0x00,0x62,0x00,0x00,0x00,0xa7,0x01,0x00,0x00, +0x3d,0x00,0x04,0x00,0x43,0x00,0x00,0x00,0x5f,0x01,0x00,0x00, +0x51,0x01,0x00,0x00,0xc2,0x00,0x05,0x00,0x43,0x00,0x00,0x00, +0x60,0x01,0x00,0x00,0x5f,0x01,0x00,0x00,0x98,0x00,0x00,0x00, +0x70,0x00,0x04,0x00,0x3e,0x00,0x00,0x00,0x61,0x01,0x00,0x00, +0x60,0x01,0x00,0x00,0x0c,0x00,0x08,0x00,0x3e,0x00,0x00,0x00, +0x63,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00, +0xf7,0x00,0x00,0x00,0x61,0x01,0x00,0x00,0xa5,0x01,0x00,0x00, +0x73,0x00,0x04,0x00,0x41,0x00,0x00,0x00,0x64,0x01,0x00,0x00, +0x63,0x01,0x00,0x00,0x41,0x00,0x06,0x00,0x4e,0x00,0x00,0x00, +0x65,0x01,0x00,0x00,0x08,0x01,0x00,0x00,0x22,0x00,0x00,0x00, +0x5c,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x65,0x01,0x00,0x00, +0x64,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x6d,0x01,0x00,0x00,0x62,0x00,0x00,0x00,0x3b,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x6e,0x01,0x00,0x00, +0x69,0x00,0x00,0x00,0x3b,0x00,0x00,0x00,0x41,0x00,0x08,0x00, +0x72,0x00,0x00,0x00,0x6f,0x01,0x00,0x00,0x4c,0x00,0x00,0x00, +0x22,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x0e,0x01,0x00,0x00, +0x6e,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0x43,0x00,0x00,0x00, +0x70,0x01,0x00,0x00,0x6f,0x01,0x00,0x00,0x71,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x71,0x01,0x00,0x00,0x70,0x01,0x00,0x00, +0x7c,0x00,0x04,0x00,0x21,0x00,0x00,0x00,0x72,0x01,0x00,0x00, +0x71,0x01,0x00,0x00,0xc7,0x00,0x05,0x00,0x21,0x00,0x00,0x00, +0x73,0x01,0x00,0x00,0x72,0x01,0x00,0x00,0x8f,0x00,0x00,0x00, +0x6f,0x00,0x04,0x00,0x3e,0x00,0x00,0x00,0x74,0x01,0x00,0x00, +0x73,0x01,0x00,0x00,0x0c,0x00,0x08,0x00,0x3e,0x00,0x00,0x00, +0x76,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00, +0xb0,0x00,0x00,0x00,0x74,0x01,0x00,0x00,0xa4,0x01,0x00,0x00, +0x73,0x00,0x04,0x00,0x41,0x00,0x00,0x00,0x77,0x01,0x00,0x00, +0x76,0x01,0x00,0x00,0x41,0x00,0x06,0x00,0x4e,0x00,0x00,0x00, +0x78,0x01,0x00,0x00,0x08,0x01,0x00,0x00,0x22,0x00,0x00,0x00, +0x6d,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x78,0x01,0x00,0x00, +0x77,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x7a,0x01,0x00,0x00,0x62,0x00,0x00,0x00,0xaa,0x01,0x00,0x00, +0x3d,0x00,0x04,0x00,0x43,0x00,0x00,0x00,0x7d,0x01,0x00,0x00, +0x6f,0x01,0x00,0x00,0xc2,0x00,0x05,0x00,0x43,0x00,0x00,0x00, +0x7e,0x01,0x00,0x00,0x7d,0x01,0x00,0x00,0x98,0x00,0x00,0x00, +0x70,0x00,0x04,0x00,0x3e,0x00,0x00,0x00,0x7f,0x01,0x00,0x00, +0x7e,0x01,0x00,0x00,0x0c,0x00,0x08,0x00,0x3e,0x00,0x00,0x00, +0x81,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00, +0xf7,0x00,0x00,0x00,0x7f,0x01,0x00,0x00,0xa5,0x01,0x00,0x00, +0x73,0x00,0x04,0x00,0x41,0x00,0x00,0x00,0x82,0x01,0x00,0x00, +0x81,0x01,0x00,0x00,0x41,0x00,0x06,0x00,0x4e,0x00,0x00,0x00, +0x83,0x01,0x00,0x00,0x08,0x01,0x00,0x00,0x22,0x00,0x00,0x00, +0x7a,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x83,0x01,0x00,0x00, +0x82,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x8b,0x01,0x00,0x00,0x62,0x00,0x00,0x00,0xda,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8c,0x01,0x00,0x00, +0x69,0x00,0x00,0x00,0xda,0x00,0x00,0x00,0x41,0x00,0x08,0x00, +0x72,0x00,0x00,0x00,0x8d,0x01,0x00,0x00,0x4c,0x00,0x00,0x00, +0x22,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x0e,0x01,0x00,0x00, +0x8c,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0x43,0x00,0x00,0x00, +0x8e,0x01,0x00,0x00,0x8d,0x01,0x00,0x00,0x71,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x8f,0x01,0x00,0x00,0x8e,0x01,0x00,0x00, +0x7c,0x00,0x04,0x00,0x21,0x00,0x00,0x00,0x90,0x01,0x00,0x00, +0x8f,0x01,0x00,0x00,0xc7,0x00,0x05,0x00,0x21,0x00,0x00,0x00, +0x91,0x01,0x00,0x00,0x90,0x01,0x00,0x00,0x8f,0x00,0x00,0x00, +0x6f,0x00,0x04,0x00,0x3e,0x00,0x00,0x00,0x92,0x01,0x00,0x00, +0x91,0x01,0x00,0x00,0x0c,0x00,0x08,0x00,0x3e,0x00,0x00,0x00, +0x94,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00, +0xb0,0x00,0x00,0x00,0x92,0x01,0x00,0x00,0xa4,0x01,0x00,0x00, +0x73,0x00,0x04,0x00,0x41,0x00,0x00,0x00,0x95,0x01,0x00,0x00, +0x94,0x01,0x00,0x00,0x41,0x00,0x06,0x00,0x4e,0x00,0x00,0x00, +0x96,0x01,0x00,0x00,0x08,0x01,0x00,0x00,0x22,0x00,0x00,0x00, +0x8b,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x96,0x01,0x00,0x00, +0x95,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x98,0x01,0x00,0x00,0x62,0x00,0x00,0x00,0xad,0x01,0x00,0x00, +0x3d,0x00,0x04,0x00,0x43,0x00,0x00,0x00,0x9b,0x01,0x00,0x00, +0x8d,0x01,0x00,0x00,0xc2,0x00,0x05,0x00,0x43,0x00,0x00,0x00, +0x9c,0x01,0x00,0x00,0x9b,0x01,0x00,0x00,0x98,0x00,0x00,0x00, +0x70,0x00,0x04,0x00,0x3e,0x00,0x00,0x00,0x9d,0x01,0x00,0x00, +0x9c,0x01,0x00,0x00,0x0c,0x00,0x08,0x00,0x3e,0x00,0x00,0x00, +0x9f,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00, +0xf7,0x00,0x00,0x00,0x9d,0x01,0x00,0x00,0xa5,0x01,0x00,0x00, +0x73,0x00,0x04,0x00,0x41,0x00,0x00,0x00,0xa0,0x01,0x00,0x00, +0x9f,0x01,0x00,0x00,0x41,0x00,0x06,0x00,0x4e,0x00,0x00,0x00, +0xa1,0x01,0x00,0x00,0x08,0x01,0x00,0x00,0x22,0x00,0x00,0x00, +0x98,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0xa1,0x01,0x00,0x00, +0xa0,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x0d,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0x0d,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x32,0x01,0x00,0x00,0x3c,0x01,0x00,0x00, +0x26,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x0a,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0x0c,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, +0x11,0x00,0x00,0x00,0x45,0x01,0x00,0x00,0x36,0x01,0x00,0x00, +0x0a,0x00,0x00,0x00,0x39,0x01,0x00,0x00,0x2c,0x00,0x00,0x00, +0xf7,0x00,0x03,0x00,0x3a,0x01,0x00,0x00,0x00,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x45,0x01,0x00,0x00,0x34,0x01,0x00,0x00, +0x3a,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x3a,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0x34,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x34,0x01,0x00,0x00,0xfd,0x00,0x01,0x00,0x38,0x00,0x01,0x00, + +}; +const uint64_t dequant_q4_K_len = 5916; + +unsigned char dequant_q5_0_data[] = { +0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, +0xb7,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, +0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x51,0x11,0x00,0x00, +0x11,0x00,0x02,0x00,0x60,0x11,0x00,0x00,0x0b,0x00,0x06,0x00, +0x01,0x00,0x00,0x00,0x47,0x4c,0x53,0x4c,0x2e,0x73,0x74,0x64, +0x2e,0x34,0x35,0x30,0x00,0x00,0x00,0x00,0x0e,0x00,0x03,0x00, +0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x0f,0x00,0x0a,0x00, +0x05,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x6d,0x61,0x69,0x6e, +0x00,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0x2b,0x00,0x00,0x00,0x50,0x00,0x00,0x00,0x7f,0x00,0x00,0x00, +0x10,0x00,0x06,0x00,0x04,0x00,0x00,0x00,0x11,0x00,0x00,0x00, +0x00,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x0b,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, +0x1a,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x12,0x00,0x00,0x00, +0x0b,0x00,0x00,0x00,0x1b,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x29,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x29,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x29,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x29,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x29,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x10,0x00,0x00,0x00, +0x47,0x00,0x03,0x00,0x29,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x48,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x4b,0x00,0x00,0x00, 0x06,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x48,0x00,0x05,0x00, 0x4c,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, 0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x4c,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x02,0x00,0x00,0x00, 0x48,0x00,0x05,0x00,0x4c,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x90,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x4d,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x16,0x00,0x00,0x00, 0x48,0x00,0x04,0x00,0x4e,0x00,0x00,0x00,0x00,0x00,0x00,0x00, 0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x4e,0x00,0x00,0x00, 0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, @@ -3390,2770 +4238,1065 @@ unsigned char dequant_q4_K_data[] = { 0x47,0x00,0x04,0x00,0x50,0x00,0x00,0x00,0x22,0x00,0x00,0x00, 0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x50,0x00,0x00,0x00, 0x21,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x08,0x01,0x00,0x00,0x06,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x48,0x00,0x04,0x00,0x09,0x01,0x00,0x00,0x00,0x00,0x00,0x00, -0x19,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x09,0x01,0x00,0x00, +0x7c,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x48,0x00,0x04,0x00,0x7d,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x19,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x7d,0x00,0x00,0x00, 0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x03,0x00,0x09,0x01,0x00,0x00,0x02,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x0b,0x01,0x00,0x00,0x22,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x0b,0x01,0x00,0x00, +0x47,0x00,0x03,0x00,0x7d,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x7f,0x00,0x00,0x00,0x22,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x7f,0x00,0x00,0x00, 0x21,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x36,0x01,0x00,0x00,0x0b,0x00,0x00,0x00,0x19,0x00,0x00,0x00, +0xab,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x19,0x00,0x00,0x00, 0x13,0x00,0x02,0x00,0x02,0x00,0x00,0x00,0x21,0x00,0x03,0x00, 0x03,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x15,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x10,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x14,0x00,0x02,0x00, -0x11,0x00,0x00,0x00,0x15,0x00,0x04,0x00,0x14,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x17,0x00,0x04,0x00, -0x15,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x03,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x16,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x15,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x16,0x00,0x00,0x00, -0x17,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x14,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x19,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x14,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x14,0x00,0x00,0x00, -0x1c,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x1e,0x00,0x06,0x00, -0x23,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x24,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x24,0x00,0x00,0x00,0x25,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x26,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x29,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x16,0x00,0x00,0x00,0x33,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x17,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x03,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x0a,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0x0a,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x0d,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00,0x12,0x00,0x00,0x00, 0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x3f,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x16,0x00,0x03,0x00,0x42,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x16,0x00,0x03,0x00,0x45,0x00,0x00,0x00,0x10,0x00,0x00,0x00, -0x17,0x00,0x04,0x00,0x46,0x00,0x00,0x00,0x45,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x15,0x00,0x04,0x00,0x47,0x00,0x00,0x00, +0x15,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x1e,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0x1e,0x00,0x07,0x00,0x29,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x2a,0x00,0x00,0x00, +0x09,0x00,0x00,0x00,0x29,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0x2a,0x00,0x00,0x00,0x2b,0x00,0x00,0x00,0x09,0x00,0x00,0x00, +0x15,0x00,0x04,0x00,0x2c,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x2c,0x00,0x00,0x00, +0x2d,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x2e,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x14,0x00,0x02,0x00,0x32,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x38,0x00,0x00,0x00,0x00,0x04,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x3e,0x00,0x00,0x00, +0x08,0x00,0x00,0x00,0x16,0x00,0x03,0x00,0x42,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0x16,0x00,0x03,0x00,0x45,0x00,0x00,0x00, +0x10,0x00,0x00,0x00,0x15,0x00,0x04,0x00,0x46,0x00,0x00,0x00, +0x10,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x47,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x1c,0x00,0x04,0x00,0x48,0x00,0x00,0x00,0x46,0x00,0x00,0x00, +0x47,0x00,0x00,0x00,0x15,0x00,0x04,0x00,0x49,0x00,0x00,0x00, 0x08,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x14,0x00,0x00,0x00,0x48,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x1c,0x00,0x04,0x00,0x49,0x00,0x00,0x00,0x47,0x00,0x00,0x00, -0x48,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x14,0x00,0x00,0x00, -0x4a,0x00,0x00,0x00,0x80,0x00,0x00,0x00,0x1c,0x00,0x04,0x00, -0x4b,0x00,0x00,0x00,0x47,0x00,0x00,0x00,0x4a,0x00,0x00,0x00, -0x1e,0x00,0x05,0x00,0x4c,0x00,0x00,0x00,0x46,0x00,0x00,0x00, -0x49,0x00,0x00,0x00,0x4b,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, -0x4d,0x00,0x00,0x00,0x4c,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, -0x4e,0x00,0x00,0x00,0x4d,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x4f,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x4f,0x00,0x00,0x00,0x50,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x52,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x45,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x14,0x00,0x00,0x00,0x58,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x5f,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x63,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x68,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x76,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x47,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x7b,0x00,0x00,0x00,0x3f,0x00,0x00,0x00,0x15,0x00,0x04,0x00, -0x7d,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x93,0x00,0x00,0x00, -0x0f,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x9a,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xc9,0x00,0x00,0x00,0x05,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xdd,0x00,0x00,0x00, -0x03,0x00,0x00,0x00,0x1d,0x00,0x03,0x00,0x08,0x01,0x00,0x00, -0x45,0x00,0x00,0x00,0x1e,0x00,0x03,0x00,0x09,0x01,0x00,0x00, -0x08,0x01,0x00,0x00,0x20,0x00,0x04,0x00,0x0a,0x01,0x00,0x00, -0x0c,0x00,0x00,0x00,0x09,0x01,0x00,0x00,0x3b,0x00,0x04,0x00, -0x0a,0x01,0x00,0x00,0x0b,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0x35,0x01,0x00,0x00, -0x20,0x00,0x00,0x00,0x2c,0x00,0x06,0x00,0x15,0x00,0x00,0x00, -0x36,0x01,0x00,0x00,0x35,0x01,0x00,0x00,0x58,0x00,0x00,0x00, -0x58,0x00,0x00,0x00,0x2a,0x00,0x03,0x00,0x11,0x00,0x00,0x00, -0x39,0x01,0x00,0x00,0x29,0x00,0x03,0x00,0x11,0x00,0x00,0x00, -0x3c,0x01,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xaa,0x01,0x00,0x00,0x21,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xad,0x01,0x00,0x00,0x22,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xb0,0x01,0x00,0x00, -0x23,0x00,0x00,0x00,0x36,0x00,0x05,0x00,0x02,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x03,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0x05,0x00,0x00,0x00,0xf7,0x00,0x03,0x00, -0x37,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0xfb,0x00,0x03,0x00, -0x18,0x00,0x00,0x00,0x38,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x38,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x0a,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0x0a,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0x3f,0x01,0x00,0x00,0x09,0x00,0x00,0x00, -0x38,0x01,0x00,0x00,0x34,0x01,0x00,0x00,0x0d,0x00,0x00,0x00, -0xb1,0x00,0x05,0x00,0x11,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x3f,0x01,0x00,0x00,0x10,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0x0c,0x00,0x00,0x00,0x0d,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x12,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x0b,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x19,0x00,0x00,0x00,0x1a,0x00,0x00,0x00, -0x17,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x14,0x00,0x00,0x00,0x1b,0x00,0x00,0x00,0x1a,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x14,0x00,0x00,0x00,0x1d,0x00,0x00,0x00, -0x1b,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x14,0x00,0x00,0x00,0x1f,0x00,0x00,0x00,0x3f,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x14,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x1d,0x00,0x00,0x00,0x1f,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x26,0x00,0x00,0x00,0x27,0x00,0x00,0x00, -0x25,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0x27,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x26,0x00,0x00,0x00,0x2a,0x00,0x00,0x00, -0x25,0x00,0x00,0x00,0x29,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x2b,0x00,0x00,0x00,0x2a,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x2c,0x00,0x00,0x00, -0x28,0x00,0x00,0x00,0x2b,0x00,0x00,0x00,0x87,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x2d,0x00,0x00,0x00,0x2c,0x00,0x00,0x00, -0x10,0x00,0x00,0x00,0xaf,0x00,0x05,0x00,0x11,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x2d,0x00,0x00,0x00, -0xf7,0x00,0x03,0x00,0x30,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x2e,0x00,0x00,0x00,0x2f,0x00,0x00,0x00, -0x30,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x2f,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x0c,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x30,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x19,0x00,0x00,0x00, -0x34,0x00,0x00,0x00,0x33,0x00,0x00,0x00,0x18,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0x35,0x00,0x00,0x00, -0x34,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x36,0x00,0x00,0x00,0x35,0x00,0x00,0x00,0x87,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x36,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x8b,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x3d,0x00,0x00,0x00,0x36,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x41,0x00,0x00,0x00, -0x3f,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x41,0x00,0x08,0x00, -0x52,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x50,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x45,0x00,0x00,0x00, -0x54,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x73,0x00,0x04,0x00, -0x42,0x00,0x00,0x00,0x55,0x00,0x00,0x00,0x54,0x00,0x00,0x00, -0x41,0x00,0x08,0x00,0x52,0x00,0x00,0x00,0x59,0x00,0x00,0x00, -0x50,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x58,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x45,0x00,0x00,0x00,0x5a,0x00,0x00,0x00,0x59,0x00,0x00,0x00, -0x73,0x00,0x04,0x00,0x42,0x00,0x00,0x00,0x5b,0x00,0x00,0x00, -0x5a,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x5e,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x10,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x61,0x00,0x00,0x00, -0x5f,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x5e,0x00,0x00,0x00, -0x61,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x65,0x00,0x00,0x00,0x63,0x00,0x00,0x00,0x3d,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x66,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x65,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x6a,0x00,0x00,0x00,0x68,0x00,0x00,0x00, -0x3a,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x6d,0x00,0x00,0x00,0x6a,0x00,0x00,0x00,0x65,0x00,0x00,0x00, -0xb1,0x00,0x05,0x00,0x11,0x00,0x00,0x00,0x6f,0x00,0x00,0x00, -0x41,0x00,0x00,0x00,0x63,0x00,0x00,0x00,0xf7,0x00,0x03,0x00, -0x71,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x6f,0x00,0x00,0x00,0x70,0x00,0x00,0x00,0x8b,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0x70,0x00,0x00,0x00,0x41,0x00,0x08,0x00, -0x76,0x00,0x00,0x00,0x77,0x00,0x00,0x00,0x50,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x29,0x00,0x00,0x00, -0x41,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x47,0x00,0x00,0x00, -0x78,0x00,0x00,0x00,0x77,0x00,0x00,0x00,0x71,0x00,0x04,0x00, -0x14,0x00,0x00,0x00,0x79,0x00,0x00,0x00,0x78,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x7a,0x00,0x00,0x00, -0x79,0x00,0x00,0x00,0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x7c,0x00,0x00,0x00,0x7a,0x00,0x00,0x00,0x7b,0x00,0x00,0x00, -0x72,0x00,0x04,0x00,0x7d,0x00,0x00,0x00,0x7e,0x00,0x00,0x00, -0x7c,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x47,0x00,0x00,0x00, -0x7f,0x00,0x00,0x00,0x7e,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x83,0x00,0x00,0x00,0x41,0x00,0x00,0x00, -0x63,0x00,0x00,0x00,0x41,0x00,0x08,0x00,0x76,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0x50,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x21,0x00,0x00,0x00,0x29,0x00,0x00,0x00,0x83,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x47,0x00,0x00,0x00,0x85,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0x71,0x00,0x04,0x00,0x14,0x00,0x00,0x00, -0x86,0x00,0x00,0x00,0x85,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x87,0x00,0x00,0x00,0x86,0x00,0x00,0x00, -0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x88,0x00,0x00,0x00, -0x87,0x00,0x00,0x00,0x7b,0x00,0x00,0x00,0x72,0x00,0x04,0x00, -0x7d,0x00,0x00,0x00,0x89,0x00,0x00,0x00,0x88,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x47,0x00,0x00,0x00,0x8a,0x00,0x00,0x00, -0x89,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x71,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0x8b,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x8e,0x00,0x00,0x00,0x41,0x00,0x00,0x00, -0x63,0x00,0x00,0x00,0x41,0x00,0x08,0x00,0x76,0x00,0x00,0x00, -0x8f,0x00,0x00,0x00,0x50,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x21,0x00,0x00,0x00,0x29,0x00,0x00,0x00,0x8e,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x47,0x00,0x00,0x00,0x90,0x00,0x00,0x00, -0x8f,0x00,0x00,0x00,0x71,0x00,0x04,0x00,0x14,0x00,0x00,0x00, -0x91,0x00,0x00,0x00,0x90,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x92,0x00,0x00,0x00,0x91,0x00,0x00,0x00, -0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x94,0x00,0x00,0x00, -0x92,0x00,0x00,0x00,0x93,0x00,0x00,0x00,0x82,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x97,0x00,0x00,0x00,0x41,0x00,0x00,0x00, -0x63,0x00,0x00,0x00,0x41,0x00,0x08,0x00,0x76,0x00,0x00,0x00, -0x98,0x00,0x00,0x00,0x50,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x21,0x00,0x00,0x00,0x29,0x00,0x00,0x00,0x97,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x47,0x00,0x00,0x00,0x99,0x00,0x00,0x00, -0x98,0x00,0x00,0x00,0xc2,0x00,0x05,0x00,0x47,0x00,0x00,0x00, -0x9b,0x00,0x00,0x00,0x99,0x00,0x00,0x00,0x9a,0x00,0x00,0x00, -0xc4,0x00,0x05,0x00,0x47,0x00,0x00,0x00,0x9c,0x00,0x00,0x00, -0x9b,0x00,0x00,0x00,0x63,0x00,0x00,0x00,0x71,0x00,0x04,0x00, -0x14,0x00,0x00,0x00,0x9d,0x00,0x00,0x00,0x9c,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x9e,0x00,0x00,0x00, -0x9d,0x00,0x00,0x00,0xc5,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x9f,0x00,0x00,0x00,0x94,0x00,0x00,0x00,0x9e,0x00,0x00,0x00, -0x72,0x00,0x04,0x00,0x7d,0x00,0x00,0x00,0xa0,0x00,0x00,0x00, -0x9f,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x47,0x00,0x00,0x00, -0xa1,0x00,0x00,0x00,0xa0,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x47,0x00,0x00,0x00,0xa6,0x00,0x00,0x00,0x8f,0x00,0x00,0x00, -0xc2,0x00,0x05,0x00,0x47,0x00,0x00,0x00,0xa7,0x00,0x00,0x00, -0xa6,0x00,0x00,0x00,0x63,0x00,0x00,0x00,0x41,0x00,0x08,0x00, -0x76,0x00,0x00,0x00,0xaa,0x00,0x00,0x00,0x50,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x29,0x00,0x00,0x00, -0x41,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x47,0x00,0x00,0x00, -0xab,0x00,0x00,0x00,0xaa,0x00,0x00,0x00,0xc2,0x00,0x05,0x00, -0x47,0x00,0x00,0x00,0xac,0x00,0x00,0x00,0xab,0x00,0x00,0x00, -0x9a,0x00,0x00,0x00,0xc4,0x00,0x05,0x00,0x47,0x00,0x00,0x00, -0xad,0x00,0x00,0x00,0xac,0x00,0x00,0x00,0x63,0x00,0x00,0x00, -0xc5,0x00,0x05,0x00,0x47,0x00,0x00,0x00,0xae,0x00,0x00,0x00, -0xa7,0x00,0x00,0x00,0xad,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x71,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x71,0x00,0x00,0x00, -0xf5,0x00,0x07,0x00,0x47,0x00,0x00,0x00,0x41,0x01,0x00,0x00, -0x8a,0x00,0x00,0x00,0x70,0x00,0x00,0x00,0xae,0x00,0x00,0x00, -0x8b,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x47,0x00,0x00,0x00, -0x40,0x01,0x00,0x00,0x7f,0x00,0x00,0x00,0x70,0x00,0x00,0x00, -0xa1,0x00,0x00,0x00,0x8b,0x00,0x00,0x00,0x70,0x00,0x04,0x00, -0x42,0x00,0x00,0x00,0xb2,0x00,0x00,0x00,0x40,0x01,0x00,0x00, -0x85,0x00,0x05,0x00,0x42,0x00,0x00,0x00,0xb3,0x00,0x00,0x00, -0x55,0x00,0x00,0x00,0xb2,0x00,0x00,0x00,0x70,0x00,0x04,0x00, -0x42,0x00,0x00,0x00,0xb7,0x00,0x00,0x00,0x41,0x01,0x00,0x00, -0x85,0x00,0x05,0x00,0x42,0x00,0x00,0x00,0xb8,0x00,0x00,0x00, -0x5b,0x00,0x00,0x00,0xb7,0x00,0x00,0x00,0xf7,0x00,0x03,0x00, -0xbc,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x6f,0x00,0x00,0x00,0xbb,0x00,0x00,0x00,0xd2,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xbb,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xbf,0x00,0x00,0x00,0x41,0x00,0x00,0x00, -0x29,0x00,0x00,0x00,0x41,0x00,0x08,0x00,0x76,0x00,0x00,0x00, -0xc0,0x00,0x00,0x00,0x50,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x21,0x00,0x00,0x00,0x29,0x00,0x00,0x00,0xbf,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x47,0x00,0x00,0x00,0xc1,0x00,0x00,0x00, -0xc0,0x00,0x00,0x00,0x71,0x00,0x04,0x00,0x14,0x00,0x00,0x00, -0xc2,0x00,0x00,0x00,0xc1,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xc3,0x00,0x00,0x00,0xc2,0x00,0x00,0x00, -0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xc4,0x00,0x00,0x00, -0xc3,0x00,0x00,0x00,0x7b,0x00,0x00,0x00,0x72,0x00,0x04,0x00, -0x7d,0x00,0x00,0x00,0xc5,0x00,0x00,0x00,0xc4,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x47,0x00,0x00,0x00,0xc6,0x00,0x00,0x00, -0xc5,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xca,0x00,0x00,0x00,0x41,0x00,0x00,0x00,0xc9,0x00,0x00,0x00, -0x41,0x00,0x08,0x00,0x76,0x00,0x00,0x00,0xcb,0x00,0x00,0x00, -0x50,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x29,0x00,0x00,0x00,0xca,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x47,0x00,0x00,0x00,0xcc,0x00,0x00,0x00,0xcb,0x00,0x00,0x00, -0x71,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0xcd,0x00,0x00,0x00, -0xcc,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xce,0x00,0x00,0x00,0xcd,0x00,0x00,0x00,0xc7,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xcf,0x00,0x00,0x00,0xce,0x00,0x00,0x00, -0x7b,0x00,0x00,0x00,0x72,0x00,0x04,0x00,0x7d,0x00,0x00,0x00, -0xd0,0x00,0x00,0x00,0xcf,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x47,0x00,0x00,0x00,0xd1,0x00,0x00,0x00,0xd0,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xbc,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xd2,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xd5,0x00,0x00,0x00,0x41,0x00,0x00,0x00,0xc9,0x00,0x00,0x00, -0x41,0x00,0x08,0x00,0x76,0x00,0x00,0x00,0xd6,0x00,0x00,0x00, -0x50,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x29,0x00,0x00,0x00,0xd5,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x47,0x00,0x00,0x00,0xd7,0x00,0x00,0x00,0xd6,0x00,0x00,0x00, -0x71,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0xd8,0x00,0x00,0x00, -0xd7,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xd9,0x00,0x00,0x00,0xd8,0x00,0x00,0x00,0xc7,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xda,0x00,0x00,0x00,0xd9,0x00,0x00,0x00, -0x93,0x00,0x00,0x00,0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xde,0x00,0x00,0x00,0x41,0x00,0x00,0x00,0xdd,0x00,0x00,0x00, -0x41,0x00,0x08,0x00,0x76,0x00,0x00,0x00,0xdf,0x00,0x00,0x00, -0x50,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x29,0x00,0x00,0x00,0xde,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x47,0x00,0x00,0x00,0xe0,0x00,0x00,0x00,0xdf,0x00,0x00,0x00, -0xc2,0x00,0x05,0x00,0x47,0x00,0x00,0x00,0xe1,0x00,0x00,0x00, -0xe0,0x00,0x00,0x00,0x9a,0x00,0x00,0x00,0xc4,0x00,0x05,0x00, -0x47,0x00,0x00,0x00,0xe2,0x00,0x00,0x00,0xe1,0x00,0x00,0x00, -0x63,0x00,0x00,0x00,0x71,0x00,0x04,0x00,0x14,0x00,0x00,0x00, -0xe3,0x00,0x00,0x00,0xe2,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xe4,0x00,0x00,0x00,0xe3,0x00,0x00,0x00, -0xc5,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xe5,0x00,0x00,0x00, -0xda,0x00,0x00,0x00,0xe4,0x00,0x00,0x00,0x72,0x00,0x04,0x00, -0x7d,0x00,0x00,0x00,0xe6,0x00,0x00,0x00,0xe5,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x47,0x00,0x00,0x00,0xe7,0x00,0x00,0x00, -0xe6,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x47,0x00,0x00,0x00, -0xec,0x00,0x00,0x00,0xd6,0x00,0x00,0x00,0xc2,0x00,0x05,0x00, -0x47,0x00,0x00,0x00,0xed,0x00,0x00,0x00,0xec,0x00,0x00,0x00, -0x63,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xf0,0x00,0x00,0x00,0x41,0x00,0x00,0x00,0x29,0x00,0x00,0x00, -0x41,0x00,0x08,0x00,0x76,0x00,0x00,0x00,0xf1,0x00,0x00,0x00, -0x50,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x29,0x00,0x00,0x00,0xf0,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x47,0x00,0x00,0x00,0xf2,0x00,0x00,0x00,0xf1,0x00,0x00,0x00, -0xc2,0x00,0x05,0x00,0x47,0x00,0x00,0x00,0xf3,0x00,0x00,0x00, -0xf2,0x00,0x00,0x00,0x9a,0x00,0x00,0x00,0xc4,0x00,0x05,0x00, -0x47,0x00,0x00,0x00,0xf4,0x00,0x00,0x00,0xf3,0x00,0x00,0x00, -0x63,0x00,0x00,0x00,0xc5,0x00,0x05,0x00,0x47,0x00,0x00,0x00, -0xf5,0x00,0x00,0x00,0xed,0x00,0x00,0x00,0xf4,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xbc,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xbc,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x47,0x00,0x00,0x00, -0x43,0x01,0x00,0x00,0xd1,0x00,0x00,0x00,0xbb,0x00,0x00,0x00, -0xf5,0x00,0x00,0x00,0xd2,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, -0x47,0x00,0x00,0x00,0x42,0x01,0x00,0x00,0xc6,0x00,0x00,0x00, -0xbb,0x00,0x00,0x00,0xe7,0x00,0x00,0x00,0xd2,0x00,0x00,0x00, -0x70,0x00,0x04,0x00,0x42,0x00,0x00,0x00,0xf9,0x00,0x00,0x00, -0x42,0x01,0x00,0x00,0x85,0x00,0x05,0x00,0x42,0x00,0x00,0x00, -0xfa,0x00,0x00,0x00,0x55,0x00,0x00,0x00,0xf9,0x00,0x00,0x00, -0x70,0x00,0x04,0x00,0x42,0x00,0x00,0x00,0xfe,0x00,0x00,0x00, -0x43,0x01,0x00,0x00,0x85,0x00,0x05,0x00,0x42,0x00,0x00,0x00, -0xff,0x00,0x00,0x00,0x5b,0x00,0x00,0x00,0xfe,0x00,0x00,0x00, -0x41,0x00,0x08,0x00,0x76,0x00,0x00,0x00,0x14,0x01,0x00,0x00, -0x50,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x3f,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x47,0x00,0x00,0x00,0x15,0x01,0x00,0x00,0x14,0x01,0x00,0x00, -0x71,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0x16,0x01,0x00,0x00, -0x15,0x01,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x17,0x01,0x00,0x00,0x16,0x01,0x00,0x00,0xc7,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x18,0x01,0x00,0x00,0x17,0x01,0x00,0x00, -0x93,0x00,0x00,0x00,0x6f,0x00,0x04,0x00,0x42,0x00,0x00,0x00, -0x19,0x01,0x00,0x00,0x18,0x01,0x00,0x00,0x7f,0x00,0x04,0x00, -0x42,0x00,0x00,0x00,0xa7,0x01,0x00,0x00,0xb8,0x00,0x00,0x00, -0x0c,0x00,0x08,0x00,0x42,0x00,0x00,0x00,0x1c,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0xb3,0x00,0x00,0x00, -0x19,0x01,0x00,0x00,0xa7,0x01,0x00,0x00,0x73,0x00,0x04,0x00, -0x45,0x00,0x00,0x00,0x1d,0x01,0x00,0x00,0x1c,0x01,0x00,0x00, -0x41,0x00,0x06,0x00,0x52,0x00,0x00,0x00,0x1e,0x01,0x00,0x00, -0x0b,0x01,0x00,0x00,0x09,0x00,0x00,0x00,0x66,0x00,0x00,0x00, -0x3e,0x00,0x03,0x00,0x1e,0x01,0x00,0x00,0x1d,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x22,0x01,0x00,0x00, -0x66,0x00,0x00,0x00,0x68,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x47,0x00,0x00,0x00,0x29,0x01,0x00,0x00,0x14,0x01,0x00,0x00, -0xc2,0x00,0x05,0x00,0x47,0x00,0x00,0x00,0x2a,0x01,0x00,0x00, -0x29,0x01,0x00,0x00,0x63,0x00,0x00,0x00,0x70,0x00,0x04,0x00, -0x42,0x00,0x00,0x00,0x2b,0x01,0x00,0x00,0x2a,0x01,0x00,0x00, -0x7f,0x00,0x04,0x00,0x42,0x00,0x00,0x00,0xa8,0x01,0x00,0x00, -0xff,0x00,0x00,0x00,0x0c,0x00,0x08,0x00,0x42,0x00,0x00,0x00, -0x2e,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00, -0xfa,0x00,0x00,0x00,0x2b,0x01,0x00,0x00,0xa8,0x01,0x00,0x00, -0x73,0x00,0x04,0x00,0x45,0x00,0x00,0x00,0x2f,0x01,0x00,0x00, -0x2e,0x01,0x00,0x00,0x41,0x00,0x06,0x00,0x52,0x00,0x00,0x00, -0x30,0x01,0x00,0x00,0x0b,0x01,0x00,0x00,0x09,0x00,0x00,0x00, -0x22,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x30,0x01,0x00,0x00, -0x2f,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x52,0x01,0x00,0x00,0x66,0x00,0x00,0x00,0x29,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x53,0x01,0x00,0x00, -0x6d,0x00,0x00,0x00,0x29,0x00,0x00,0x00,0x41,0x00,0x08,0x00, -0x76,0x00,0x00,0x00,0x54,0x01,0x00,0x00,0x50,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x3f,0x00,0x00,0x00, -0x53,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0x47,0x00,0x00,0x00, -0x55,0x01,0x00,0x00,0x54,0x01,0x00,0x00,0x71,0x00,0x04,0x00, -0x14,0x00,0x00,0x00,0x56,0x01,0x00,0x00,0x55,0x01,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x57,0x01,0x00,0x00, -0x56,0x01,0x00,0x00,0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x58,0x01,0x00,0x00,0x57,0x01,0x00,0x00,0x93,0x00,0x00,0x00, -0x6f,0x00,0x04,0x00,0x42,0x00,0x00,0x00,0x59,0x01,0x00,0x00, -0x58,0x01,0x00,0x00,0x0c,0x00,0x08,0x00,0x42,0x00,0x00,0x00, -0x5b,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00, -0xb3,0x00,0x00,0x00,0x59,0x01,0x00,0x00,0xa7,0x01,0x00,0x00, -0x73,0x00,0x04,0x00,0x45,0x00,0x00,0x00,0x5c,0x01,0x00,0x00, -0x5b,0x01,0x00,0x00,0x41,0x00,0x06,0x00,0x52,0x00,0x00,0x00, -0x5d,0x01,0x00,0x00,0x0b,0x01,0x00,0x00,0x09,0x00,0x00,0x00, -0x52,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x5d,0x01,0x00,0x00, -0x5c,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x5f,0x01,0x00,0x00,0x66,0x00,0x00,0x00,0xaa,0x01,0x00,0x00, -0x3d,0x00,0x04,0x00,0x47,0x00,0x00,0x00,0x62,0x01,0x00,0x00, -0x54,0x01,0x00,0x00,0xc2,0x00,0x05,0x00,0x47,0x00,0x00,0x00, -0x63,0x01,0x00,0x00,0x62,0x01,0x00,0x00,0x63,0x00,0x00,0x00, -0x70,0x00,0x04,0x00,0x42,0x00,0x00,0x00,0x64,0x01,0x00,0x00, -0x63,0x01,0x00,0x00,0x0c,0x00,0x08,0x00,0x42,0x00,0x00,0x00, -0x66,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00, -0xfa,0x00,0x00,0x00,0x64,0x01,0x00,0x00,0xa8,0x01,0x00,0x00, -0x73,0x00,0x04,0x00,0x45,0x00,0x00,0x00,0x67,0x01,0x00,0x00, -0x66,0x01,0x00,0x00,0x41,0x00,0x06,0x00,0x52,0x00,0x00,0x00, -0x68,0x01,0x00,0x00,0x0b,0x01,0x00,0x00,0x09,0x00,0x00,0x00, -0x5f,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x68,0x01,0x00,0x00, -0x67,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x70,0x01,0x00,0x00,0x66,0x00,0x00,0x00,0x3f,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x71,0x01,0x00,0x00, -0x6d,0x00,0x00,0x00,0x3f,0x00,0x00,0x00,0x41,0x00,0x08,0x00, -0x76,0x00,0x00,0x00,0x72,0x01,0x00,0x00,0x50,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x3f,0x00,0x00,0x00, -0x71,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0x47,0x00,0x00,0x00, -0x73,0x01,0x00,0x00,0x72,0x01,0x00,0x00,0x71,0x00,0x04,0x00, -0x14,0x00,0x00,0x00,0x74,0x01,0x00,0x00,0x73,0x01,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x75,0x01,0x00,0x00, -0x74,0x01,0x00,0x00,0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x76,0x01,0x00,0x00,0x75,0x01,0x00,0x00,0x93,0x00,0x00,0x00, -0x6f,0x00,0x04,0x00,0x42,0x00,0x00,0x00,0x77,0x01,0x00,0x00, -0x76,0x01,0x00,0x00,0x0c,0x00,0x08,0x00,0x42,0x00,0x00,0x00, -0x79,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00, -0xb3,0x00,0x00,0x00,0x77,0x01,0x00,0x00,0xa7,0x01,0x00,0x00, -0x73,0x00,0x04,0x00,0x45,0x00,0x00,0x00,0x7a,0x01,0x00,0x00, -0x79,0x01,0x00,0x00,0x41,0x00,0x06,0x00,0x52,0x00,0x00,0x00, -0x7b,0x01,0x00,0x00,0x0b,0x01,0x00,0x00,0x09,0x00,0x00,0x00, -0x70,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x7b,0x01,0x00,0x00, -0x7a,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x7d,0x01,0x00,0x00,0x66,0x00,0x00,0x00,0xad,0x01,0x00,0x00, -0x3d,0x00,0x04,0x00,0x47,0x00,0x00,0x00,0x80,0x01,0x00,0x00, -0x72,0x01,0x00,0x00,0xc2,0x00,0x05,0x00,0x47,0x00,0x00,0x00, -0x81,0x01,0x00,0x00,0x80,0x01,0x00,0x00,0x63,0x00,0x00,0x00, -0x70,0x00,0x04,0x00,0x42,0x00,0x00,0x00,0x82,0x01,0x00,0x00, -0x81,0x01,0x00,0x00,0x0c,0x00,0x08,0x00,0x42,0x00,0x00,0x00, -0x84,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00, -0xfa,0x00,0x00,0x00,0x82,0x01,0x00,0x00,0xa8,0x01,0x00,0x00, -0x73,0x00,0x04,0x00,0x45,0x00,0x00,0x00,0x85,0x01,0x00,0x00, -0x84,0x01,0x00,0x00,0x41,0x00,0x06,0x00,0x52,0x00,0x00,0x00, -0x86,0x01,0x00,0x00,0x0b,0x01,0x00,0x00,0x09,0x00,0x00,0x00, -0x7d,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x86,0x01,0x00,0x00, -0x85,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x8e,0x01,0x00,0x00,0x66,0x00,0x00,0x00,0xdd,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8f,0x01,0x00,0x00, -0x6d,0x00,0x00,0x00,0xdd,0x00,0x00,0x00,0x41,0x00,0x08,0x00, -0x76,0x00,0x00,0x00,0x90,0x01,0x00,0x00,0x50,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x3f,0x00,0x00,0x00, -0x8f,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0x47,0x00,0x00,0x00, -0x91,0x01,0x00,0x00,0x90,0x01,0x00,0x00,0x71,0x00,0x04,0x00, -0x14,0x00,0x00,0x00,0x92,0x01,0x00,0x00,0x91,0x01,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x93,0x01,0x00,0x00, -0x92,0x01,0x00,0x00,0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x94,0x01,0x00,0x00,0x93,0x01,0x00,0x00,0x93,0x00,0x00,0x00, -0x6f,0x00,0x04,0x00,0x42,0x00,0x00,0x00,0x95,0x01,0x00,0x00, -0x94,0x01,0x00,0x00,0x0c,0x00,0x08,0x00,0x42,0x00,0x00,0x00, -0x97,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00, -0xb3,0x00,0x00,0x00,0x95,0x01,0x00,0x00,0xa7,0x01,0x00,0x00, -0x73,0x00,0x04,0x00,0x45,0x00,0x00,0x00,0x98,0x01,0x00,0x00, -0x97,0x01,0x00,0x00,0x41,0x00,0x06,0x00,0x52,0x00,0x00,0x00, -0x99,0x01,0x00,0x00,0x0b,0x01,0x00,0x00,0x09,0x00,0x00,0x00, -0x8e,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x99,0x01,0x00,0x00, -0x98,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x9b,0x01,0x00,0x00,0x66,0x00,0x00,0x00,0xb0,0x01,0x00,0x00, -0x3d,0x00,0x04,0x00,0x47,0x00,0x00,0x00,0x9e,0x01,0x00,0x00, -0x90,0x01,0x00,0x00,0xc2,0x00,0x05,0x00,0x47,0x00,0x00,0x00, -0x9f,0x01,0x00,0x00,0x9e,0x01,0x00,0x00,0x63,0x00,0x00,0x00, -0x70,0x00,0x04,0x00,0x42,0x00,0x00,0x00,0xa0,0x01,0x00,0x00, -0x9f,0x01,0x00,0x00,0x0c,0x00,0x08,0x00,0x42,0x00,0x00,0x00, -0xa2,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00, -0xfa,0x00,0x00,0x00,0xa0,0x01,0x00,0x00,0xa8,0x01,0x00,0x00, -0x73,0x00,0x04,0x00,0x45,0x00,0x00,0x00,0xa3,0x01,0x00,0x00, -0xa2,0x01,0x00,0x00,0x41,0x00,0x06,0x00,0x52,0x00,0x00,0x00, -0xa4,0x01,0x00,0x00,0x0b,0x01,0x00,0x00,0x09,0x00,0x00,0x00, -0x9b,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0xa4,0x01,0x00,0x00, -0xa3,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x0d,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0x0d,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x34,0x01,0x00,0x00,0x3f,0x01,0x00,0x00, -0x29,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x0a,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0x0c,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, -0x11,0x00,0x00,0x00,0x48,0x01,0x00,0x00,0x39,0x01,0x00,0x00, -0x0a,0x00,0x00,0x00,0x3c,0x01,0x00,0x00,0x2f,0x00,0x00,0x00, -0xf7,0x00,0x03,0x00,0x3d,0x01,0x00,0x00,0x00,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x48,0x01,0x00,0x00,0x37,0x01,0x00,0x00, -0x3d,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x3d,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0x37,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x37,0x01,0x00,0x00,0xfd,0x00,0x01,0x00,0x38,0x00,0x01,0x00, - -}; -const uint64_t dequant_q4_K_len = 5940; - -unsigned char dequant_q5_0_data[] = { -0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, -0x9b,0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, -0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x51,0x11,0x00,0x00, -0x11,0x00,0x02,0x00,0x60,0x11,0x00,0x00,0x0b,0x00,0x06,0x00, -0x01,0x00,0x00,0x00,0x47,0x4c,0x53,0x4c,0x2e,0x73,0x74,0x64, -0x2e,0x34,0x35,0x30,0x00,0x00,0x00,0x00,0x0e,0x00,0x03,0x00, -0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x0f,0x00,0x09,0x00, -0x05,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x6d,0x61,0x69,0x6e, -0x00,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0x58,0x00,0x00,0x00,0xa1,0x00,0x00,0x00,0x10,0x00,0x06,0x00, -0x04,0x00,0x00,0x00,0x11,0x00,0x00,0x00,0x00,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x0c,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x1c,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x14,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x14,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x14,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x08,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x14,0x00,0x00,0x00,0x03,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x47,0x00,0x03,0x00, -0x14,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x50,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x53,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x54,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x54,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x54,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x55,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x48,0x00,0x04,0x00, -0x56,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x56,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, -0x56,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x58,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x58,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x9e,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x48,0x00,0x04,0x00, -0x9f,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x19,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x9f,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, -0x9f,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0xa1,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0xa1,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xc2,0x00,0x00,0x00, -0x0b,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x13,0x00,0x02,0x00, -0x02,0x00,0x00,0x00,0x21,0x00,0x03,0x00,0x03,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x15,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x15,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x17,0x00,0x04,0x00,0x0a,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x03,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x0b,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x0a,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x0b,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x0d,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x0e,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x1e,0x00,0x06,0x00, -0x14,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x15,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x14,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x15,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x17,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x18,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x1b,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x14,0x00,0x02,0x00,0x24,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x37,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x48,0x00,0x00,0x00,0x10,0x00,0x00,0x00, -0x16,0x00,0x03,0x00,0x4a,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x16,0x00,0x03,0x00,0x4d,0x00,0x00,0x00,0x10,0x00,0x00,0x00, -0x15,0x00,0x04,0x00,0x4e,0x00,0x00,0x00,0x10,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x4f,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x1c,0x00,0x04,0x00, -0x50,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x4f,0x00,0x00,0x00, -0x15,0x00,0x04,0x00,0x51,0x00,0x00,0x00,0x08,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x52,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x1c,0x00,0x04,0x00, -0x53,0x00,0x00,0x00,0x51,0x00,0x00,0x00,0x52,0x00,0x00,0x00, -0x1e,0x00,0x05,0x00,0x54,0x00,0x00,0x00,0x4d,0x00,0x00,0x00, -0x50,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, -0x55,0x00,0x00,0x00,0x54,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, -0x56,0x00,0x00,0x00,0x55,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x57,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x56,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x57,0x00,0x00,0x00,0x58,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x5a,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x4d,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x61,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x71,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x77,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x80,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x51,0x00,0x00,0x00, -0x17,0x00,0x04,0x00,0x84,0x00,0x00,0x00,0x4a,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x88,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x91,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0x99,0x00,0x00,0x00, -0x00,0x00,0x80,0x41,0x1d,0x00,0x03,0x00,0x9e,0x00,0x00,0x00, -0x4d,0x00,0x00,0x00,0x1e,0x00,0x03,0x00,0x9f,0x00,0x00,0x00, -0x9e,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0xa0,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x9f,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0xa0,0x00,0x00,0x00,0xa1,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xa3,0x00,0x00,0x00, -0x03,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0xc1,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x2c,0x00,0x06,0x00, -0x0a,0x00,0x00,0x00,0xc2,0x00,0x00,0x00,0xc1,0x00,0x00,0x00, -0x91,0x00,0x00,0x00,0x91,0x00,0x00,0x00,0x2c,0x00,0x05,0x00, -0x84,0x00,0x00,0x00,0xcf,0x00,0x00,0x00,0x99,0x00,0x00,0x00, -0x99,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x82,0x04,0x00,0x00,0x0d,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x83,0x04,0x00,0x00,0x11,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x84,0x04,0x00,0x00, -0x0e,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x85,0x04,0x00,0x00,0x12,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x86,0x04,0x00,0x00,0x0f,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x87,0x04,0x00,0x00, +0x06,0x00,0x00,0x00,0x4a,0x00,0x00,0x00,0x10,0x00,0x00,0x00, +0x1c,0x00,0x04,0x00,0x4b,0x00,0x00,0x00,0x49,0x00,0x00,0x00, +0x4a,0x00,0x00,0x00,0x1e,0x00,0x05,0x00,0x4c,0x00,0x00,0x00, +0x45,0x00,0x00,0x00,0x48,0x00,0x00,0x00,0x4b,0x00,0x00,0x00, +0x1d,0x00,0x03,0x00,0x4d,0x00,0x00,0x00,0x4c,0x00,0x00,0x00, +0x1e,0x00,0x03,0x00,0x4e,0x00,0x00,0x00,0x4d,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x4f,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x4e,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x4f,0x00,0x00,0x00, +0x50,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x2c,0x00,0x00,0x00,0x51,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x53,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x45,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x2c,0x00,0x00,0x00, +0x59,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x5a,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x46,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x2c,0x00,0x00,0x00,0x5e,0x00,0x00,0x00, +0x10,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x2c,0x00,0x00,0x00, +0x76,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x78,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x49,0x00,0x00,0x00, +0x1d,0x00,0x03,0x00,0x7c,0x00,0x00,0x00,0x45,0x00,0x00,0x00, +0x1e,0x00,0x03,0x00,0x7d,0x00,0x00,0x00,0x7c,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x7e,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x7d,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x7e,0x00,0x00,0x00, +0x7f,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x42,0x00,0x00,0x00,0x8f,0x00,0x00,0x00, +0x00,0x00,0x80,0x41,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x9d,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xa9,0x00,0x00,0x00,0x00,0x01,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xaa,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x2c,0x00,0x06,0x00,0x09,0x00,0x00,0x00, +0xab,0x00,0x00,0x00,0xa9,0x00,0x00,0x00,0xaa,0x00,0x00,0x00, +0xaa,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xaa,0x01,0x00,0x00,0x11,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xab,0x01,0x00,0x00,0x0d,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xac,0x01,0x00,0x00, +0x12,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xad,0x01,0x00,0x00,0x0e,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xae,0x01,0x00,0x00,0x03,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xaf,0x01,0x00,0x00, 0x13,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x88,0x04,0x00,0x00,0x14,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x89,0x04,0x00,0x00,0x05,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x8a,0x04,0x00,0x00, +0xb0,0x01,0x00,0x00,0x14,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xb1,0x01,0x00,0x00,0x05,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xb2,0x01,0x00,0x00, 0x15,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x8b,0x04,0x00,0x00,0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x8c,0x04,0x00,0x00,0x16,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x8d,0x04,0x00,0x00, +0xb3,0x01,0x00,0x00,0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xb4,0x01,0x00,0x00,0x16,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xb5,0x01,0x00,0x00, 0x07,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x8e,0x04,0x00,0x00,0x17,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x8f,0x04,0x00,0x00,0x08,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x90,0x04,0x00,0x00, -0x18,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x91,0x04,0x00,0x00,0x09,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x92,0x04,0x00,0x00,0x19,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x93,0x04,0x00,0x00, -0x0a,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x94,0x04,0x00,0x00,0x1a,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x95,0x04,0x00,0x00,0x0b,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x96,0x04,0x00,0x00, -0x1b,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x97,0x04,0x00,0x00,0x1c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x98,0x04,0x00,0x00,0x1d,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x99,0x04,0x00,0x00, -0x1e,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x9a,0x04,0x00,0x00,0x1f,0x00,0x00,0x00,0x36,0x00,0x05,0x00, +0xb6,0x01,0x00,0x00,0x17,0x00,0x00,0x00,0x36,0x00,0x05,0x00, 0x02,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x00, 0x03,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x05,0x00,0x00,0x00, -0xf7,0x00,0x03,0x00,0xc3,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0xfb,0x00,0x03,0x00,0x0d,0x00,0x00,0x00,0xc4,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xc4,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x0e,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x0d,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x10,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x11,0x00,0x00,0x00,0x10,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x18,0x00,0x00,0x00,0x19,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x17,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x1a,0x00,0x00,0x00,0x19,0x00,0x00,0x00, -0x87,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x1c,0x00,0x00,0x00, -0x1a,0x00,0x00,0x00,0x1b,0x00,0x00,0x00,0x8b,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x1d,0x00,0x00,0x00,0x11,0x00,0x00,0x00, -0x1c,0x00,0x00,0x00,0x87,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x11,0x00,0x00,0x00,0x1c,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x26,0x00,0x00,0x00, -0x1d,0x00,0x00,0x00,0x1b,0x00,0x00,0x00,0xaf,0x00,0x05,0x00, -0x24,0x00,0x00,0x00,0x29,0x00,0x00,0x00,0x26,0x00,0x00,0x00, -0x1a,0x00,0x00,0x00,0xa8,0x00,0x04,0x00,0x24,0x00,0x00,0x00, -0x2a,0x00,0x00,0x00,0x29,0x00,0x00,0x00,0xf7,0x00,0x03,0x00, -0x2c,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x2a,0x00,0x00,0x00,0x2b,0x00,0x00,0x00,0x2c,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0x2b,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x18,0x00,0x00,0x00,0x2f,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x30,0x00,0x00,0x00,0x2f,0x00,0x00,0x00,0xaf,0x00,0x05,0x00, -0x24,0x00,0x00,0x00,0x31,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x30,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x2c,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0x2c,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, -0x24,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x29,0x00,0x00,0x00, -0xc4,0x00,0x00,0x00,0x31,0x00,0x00,0x00,0x2b,0x00,0x00,0x00, -0xf7,0x00,0x03,0x00,0x34,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x32,0x00,0x00,0x00,0x33,0x00,0x00,0x00, -0x34,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x33,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xc3,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x34,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x18,0x00,0x00,0x00, -0x38,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x37,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x38,0x00,0x00,0x00,0x87,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x3a,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x1b,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3e,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0xf7,0x00,0x03,0x00,0xac,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0xfb,0x00,0x03,0x00,0x0c,0x00,0x00,0x00,0xad,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xad,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x0d,0x00,0x00,0x00,0x0e,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x0f,0x00,0x00,0x00,0x0e,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x11,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, +0x10,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00, +0x13,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x14,0x00,0x00,0x00, +0x13,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x16,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x15,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x17,0x00,0x00,0x00, +0x11,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x89,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x1b,0x00,0x00,0x00,0x14,0x00,0x00,0x00, +0x15,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x1f,0x00,0x00,0x00,0x1b,0x00,0x00,0x00,0x1e,0x00,0x00,0x00, +0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x22,0x00,0x00,0x00, +0x1b,0x00,0x00,0x00,0x1e,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x25,0x00,0x00,0x00,0x1e,0x00,0x00,0x00, +0x17,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x27,0x00,0x00,0x00,0x25,0x00,0x00,0x00,0x22,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x2e,0x00,0x00,0x00,0x2f,0x00,0x00,0x00, +0x2b,0x00,0x00,0x00,0x2d,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x30,0x00,0x00,0x00,0x2f,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x31,0x00,0x00,0x00, +0x30,0x00,0x00,0x00,0x1e,0x00,0x00,0x00,0xae,0x00,0x05,0x00, +0x32,0x00,0x00,0x00,0x33,0x00,0x00,0x00,0x27,0x00,0x00,0x00, +0x31,0x00,0x00,0x00,0xf7,0x00,0x03,0x00,0x35,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x33,0x00,0x00,0x00, +0x34,0x00,0x00,0x00,0x35,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0x34,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xac,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0x35,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x38,0x00,0x00,0x00, +0x17,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x3c,0x00,0x00,0x00,0x1e,0x00,0x00,0x00,0x22,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3d,0x00,0x00,0x00, +0x3a,0x00,0x00,0x00,0x3c,0x00,0x00,0x00,0x84,0x00,0x05,0x00, 0x06,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x3e,0x00,0x00,0x00, -0x1d,0x00,0x00,0x00,0x41,0x00,0x07,0x00,0x5a,0x00,0x00,0x00, -0x5b,0x00,0x00,0x00,0x58,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0x5c,0x00,0x00,0x00,0x5b,0x00,0x00,0x00, -0x73,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0x5d,0x00,0x00,0x00, -0x5c,0x00,0x00,0x00,0x41,0x00,0x08,0x00,0x61,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x58,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0x17,0x00,0x00,0x00,0x17,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4e,0x00,0x00,0x00,0x63,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x64,0x00,0x00,0x00,0x63,0x00,0x00,0x00,0xc4,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0x65,0x00,0x00,0x00,0x64,0x00,0x00,0x00, -0x48,0x00,0x00,0x00,0x41,0x00,0x08,0x00,0x61,0x00,0x00,0x00, -0x67,0x00,0x00,0x00,0x58,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0x17,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4e,0x00,0x00,0x00,0x68,0x00,0x00,0x00, -0x67,0x00,0x00,0x00,0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x69,0x00,0x00,0x00,0x68,0x00,0x00,0x00,0xc5,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0x6a,0x00,0x00,0x00,0x65,0x00,0x00,0x00, -0x69,0x00,0x00,0x00,0xc2,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0x70,0x00,0x00,0x00,0x6a,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0xc4,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0x72,0x00,0x00,0x00, -0x70,0x00,0x00,0x00,0x71,0x00,0x00,0x00,0xc7,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0x73,0x00,0x00,0x00,0x72,0x00,0x00,0x00, -0x52,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x74,0x00,0x00,0x00,0x73,0x00,0x00,0x00,0xc2,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0x79,0x00,0x00,0x00,0x6a,0x00,0x00,0x00, -0x77,0x00,0x00,0x00,0xc7,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0x7a,0x00,0x00,0x00,0x79,0x00,0x00,0x00,0x52,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x7b,0x00,0x00,0x00, -0x7a,0x00,0x00,0x00,0x41,0x00,0x08,0x00,0x80,0x00,0x00,0x00, -0x81,0x00,0x00,0x00,0x58,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x51,0x00,0x00,0x00,0x82,0x00,0x00,0x00, -0x81,0x00,0x00,0x00,0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x83,0x00,0x00,0x00,0x82,0x00,0x00,0x00,0xc7,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0x89,0x00,0x00,0x00,0x83,0x00,0x00,0x00, -0x88,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x8c,0x00,0x00,0x00,0x74,0x00,0x00,0x00,0xc5,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0x8d,0x00,0x00,0x00,0x89,0x00,0x00,0x00, -0x8c,0x00,0x00,0x00,0x70,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0x8e,0x00,0x00,0x00,0x8d,0x00,0x00,0x00,0xc2,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0x90,0x00,0x00,0x00,0x83,0x00,0x00,0x00, -0x71,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x94,0x00,0x00,0x00,0x7b,0x00,0x00,0x00,0xc5,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0x95,0x00,0x00,0x00,0x90,0x00,0x00,0x00, -0x94,0x00,0x00,0x00,0x70,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0x96,0x00,0x00,0x00,0x95,0x00,0x00,0x00,0x50,0x00,0x05,0x00, -0x84,0x00,0x00,0x00,0x97,0x00,0x00,0x00,0x8e,0x00,0x00,0x00, -0x96,0x00,0x00,0x00,0x83,0x00,0x05,0x00,0x84,0x00,0x00,0x00, -0x9b,0x00,0x00,0x00,0x97,0x00,0x00,0x00,0xcf,0x00,0x00,0x00, -0x8e,0x00,0x05,0x00,0x84,0x00,0x00,0x00,0x9d,0x00,0x00,0x00, -0x9b,0x00,0x00,0x00,0x5d,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x18,0x00,0x00,0x00,0xa4,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0xa3,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xa5,0x00,0x00,0x00,0xa4,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xa6,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0xa5,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xa9,0x00,0x00,0x00,0xa6,0x00,0x00,0x00,0x26,0x00,0x00,0x00, -0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00,0xae,0x00,0x00,0x00, -0x9d,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x73,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0xaf,0x00,0x00,0x00,0xae,0x00,0x00,0x00, -0x41,0x00,0x06,0x00,0x5a,0x00,0x00,0x00,0xb0,0x00,0x00,0x00, -0xa1,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0xa9,0x00,0x00,0x00, -0x3e,0x00,0x03,0x00,0xb0,0x00,0x00,0x00,0xaf,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xba,0x00,0x00,0x00, -0xa9,0x00,0x00,0x00,0x48,0x00,0x00,0x00,0x51,0x00,0x05,0x00, -0x4a,0x00,0x00,0x00,0xbc,0x00,0x00,0x00,0x9d,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0xbd,0x00,0x00,0x00,0xbc,0x00,0x00,0x00,0x41,0x00,0x06,0x00, -0x5a,0x00,0x00,0x00,0xbe,0x00,0x00,0x00,0xa1,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0xba,0x00,0x00,0x00,0x3e,0x00,0x03,0x00, -0xbe,0x00,0x00,0x00,0xbd,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0xd6,0x00,0x00,0x00,0x5b,0x00,0x00,0x00, -0x73,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0xd7,0x00,0x00,0x00, -0xd6,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x4e,0x00,0x00,0x00, -0xd9,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x71,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0xda,0x00,0x00,0x00,0xd9,0x00,0x00,0x00, -0xc4,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0xdb,0x00,0x00,0x00, -0xda,0x00,0x00,0x00,0x48,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4e,0x00,0x00,0x00,0xdd,0x00,0x00,0x00,0x67,0x00,0x00,0x00, -0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0xde,0x00,0x00,0x00, -0xdd,0x00,0x00,0x00,0xc5,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0xdf,0x00,0x00,0x00,0xdb,0x00,0x00,0x00,0xde,0x00,0x00,0x00, -0xc2,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0xe0,0x00,0x00,0x00, -0xdf,0x00,0x00,0x00,0x17,0x00,0x00,0x00,0xc4,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0xe1,0x00,0x00,0x00,0xe0,0x00,0x00,0x00, -0x71,0x00,0x00,0x00,0xc7,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0xe2,0x00,0x00,0x00,0xe1,0x00,0x00,0x00,0x52,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xe3,0x00,0x00,0x00, -0xe2,0x00,0x00,0x00,0xc2,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0xe5,0x00,0x00,0x00,0xdf,0x00,0x00,0x00,0x82,0x04,0x00,0x00, -0xc7,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0xe6,0x00,0x00,0x00, -0xe5,0x00,0x00,0x00,0x52,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xe7,0x00,0x00,0x00,0xe6,0x00,0x00,0x00, -0x41,0x00,0x08,0x00,0x80,0x00,0x00,0x00,0xe9,0x00,0x00,0x00, -0x58,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x37,0x00,0x00,0x00,0x17,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x51,0x00,0x00,0x00,0xea,0x00,0x00,0x00,0xe9,0x00,0x00,0x00, -0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0xeb,0x00,0x00,0x00, -0xea,0x00,0x00,0x00,0xc7,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0xec,0x00,0x00,0x00,0xeb,0x00,0x00,0x00,0x88,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0xee,0x00,0x00,0x00, -0xe3,0x00,0x00,0x00,0xc5,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0xef,0x00,0x00,0x00,0xec,0x00,0x00,0x00,0xee,0x00,0x00,0x00, -0x70,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0xf0,0x00,0x00,0x00, -0xef,0x00,0x00,0x00,0xc2,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0xf1,0x00,0x00,0x00,0xeb,0x00,0x00,0x00,0x71,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0xf3,0x00,0x00,0x00, -0xe7,0x00,0x00,0x00,0xc5,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0xf4,0x00,0x00,0x00,0xf1,0x00,0x00,0x00,0xf3,0x00,0x00,0x00, -0x70,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0xf5,0x00,0x00,0x00, -0xf4,0x00,0x00,0x00,0x50,0x00,0x05,0x00,0x84,0x00,0x00,0x00, -0xf6,0x00,0x00,0x00,0xf0,0x00,0x00,0x00,0xf5,0x00,0x00,0x00, -0x83,0x00,0x05,0x00,0x84,0x00,0x00,0x00,0xf7,0x00,0x00,0x00, -0xf6,0x00,0x00,0x00,0xcf,0x00,0x00,0x00,0x8e,0x00,0x05,0x00, -0x84,0x00,0x00,0x00,0xf8,0x00,0x00,0x00,0xf7,0x00,0x00,0x00, -0xd7,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xfe,0x00,0x00,0x00,0xa9,0x00,0x00,0x00,0x17,0x00,0x00,0x00, -0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00,0x00,0x01,0x00,0x00, -0xf8,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x73,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0x01,0x01,0x00,0x00,0x00,0x01,0x00,0x00, -0x41,0x00,0x06,0x00,0x5a,0x00,0x00,0x00,0x02,0x01,0x00,0x00, -0xa1,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0xfe,0x00,0x00,0x00, -0x3e,0x00,0x03,0x00,0x02,0x01,0x00,0x00,0x01,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x09,0x01,0x00,0x00, -0xa9,0x00,0x00,0x00,0x83,0x04,0x00,0x00,0x51,0x00,0x05,0x00, -0x4a,0x00,0x00,0x00,0x0a,0x01,0x00,0x00,0xf8,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0x0b,0x01,0x00,0x00,0x0a,0x01,0x00,0x00,0x41,0x00,0x06,0x00, -0x5a,0x00,0x00,0x00,0x0c,0x01,0x00,0x00,0xa1,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x09,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0x0c,0x01,0x00,0x00,0x0b,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0x15,0x01,0x00,0x00,0x5b,0x00,0x00,0x00, -0x73,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0x16,0x01,0x00,0x00, -0x15,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0x4e,0x00,0x00,0x00, -0x18,0x01,0x00,0x00,0x62,0x00,0x00,0x00,0x71,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x19,0x01,0x00,0x00,0x18,0x01,0x00,0x00, -0xc4,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0x1a,0x01,0x00,0x00, -0x19,0x01,0x00,0x00,0x48,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4e,0x00,0x00,0x00,0x1c,0x01,0x00,0x00,0x67,0x00,0x00,0x00, -0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x1d,0x01,0x00,0x00, -0x1c,0x01,0x00,0x00,0xc5,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0x1e,0x01,0x00,0x00,0x1a,0x01,0x00,0x00,0x1d,0x01,0x00,0x00, -0xc2,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0x1f,0x01,0x00,0x00, -0x1e,0x01,0x00,0x00,0x37,0x00,0x00,0x00,0xc4,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0x20,0x01,0x00,0x00,0x1f,0x01,0x00,0x00, -0x71,0x00,0x00,0x00,0xc7,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0x21,0x01,0x00,0x00,0x20,0x01,0x00,0x00,0x52,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x22,0x01,0x00,0x00, -0x21,0x01,0x00,0x00,0xc2,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0x24,0x01,0x00,0x00,0x1e,0x01,0x00,0x00,0x84,0x04,0x00,0x00, -0xc7,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0x25,0x01,0x00,0x00, -0x24,0x01,0x00,0x00,0x52,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x26,0x01,0x00,0x00,0x25,0x01,0x00,0x00, -0x41,0x00,0x08,0x00,0x80,0x00,0x00,0x00,0x28,0x01,0x00,0x00, -0x58,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x37,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x51,0x00,0x00,0x00,0x29,0x01,0x00,0x00,0x28,0x01,0x00,0x00, -0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x2a,0x01,0x00,0x00, -0x29,0x01,0x00,0x00,0xc7,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0x2b,0x01,0x00,0x00,0x2a,0x01,0x00,0x00,0x88,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x2d,0x01,0x00,0x00, -0x22,0x01,0x00,0x00,0xc5,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0x2e,0x01,0x00,0x00,0x2b,0x01,0x00,0x00,0x2d,0x01,0x00,0x00, -0x70,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0x2f,0x01,0x00,0x00, -0x2e,0x01,0x00,0x00,0xc2,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0x30,0x01,0x00,0x00,0x2a,0x01,0x00,0x00,0x71,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x32,0x01,0x00,0x00, -0x26,0x01,0x00,0x00,0xc5,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0x33,0x01,0x00,0x00,0x30,0x01,0x00,0x00,0x32,0x01,0x00,0x00, -0x70,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0x34,0x01,0x00,0x00, -0x33,0x01,0x00,0x00,0x50,0x00,0x05,0x00,0x84,0x00,0x00,0x00, -0x35,0x01,0x00,0x00,0x2f,0x01,0x00,0x00,0x34,0x01,0x00,0x00, -0x83,0x00,0x05,0x00,0x84,0x00,0x00,0x00,0x36,0x01,0x00,0x00, -0x35,0x01,0x00,0x00,0xcf,0x00,0x00,0x00,0x8e,0x00,0x05,0x00, -0x84,0x00,0x00,0x00,0x37,0x01,0x00,0x00,0x36,0x01,0x00,0x00, -0x16,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x3d,0x01,0x00,0x00,0xa9,0x00,0x00,0x00,0x37,0x00,0x00,0x00, -0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00,0x3f,0x01,0x00,0x00, -0x37,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x73,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0x40,0x01,0x00,0x00,0x3f,0x01,0x00,0x00, -0x41,0x00,0x06,0x00,0x5a,0x00,0x00,0x00,0x41,0x01,0x00,0x00, -0xa1,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x3d,0x01,0x00,0x00, -0x3e,0x00,0x03,0x00,0x41,0x01,0x00,0x00,0x40,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x48,0x01,0x00,0x00, -0xa9,0x00,0x00,0x00,0x85,0x04,0x00,0x00,0x51,0x00,0x05,0x00, -0x4a,0x00,0x00,0x00,0x49,0x01,0x00,0x00,0x37,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0x4a,0x01,0x00,0x00,0x49,0x01,0x00,0x00,0x41,0x00,0x06,0x00, -0x5a,0x00,0x00,0x00,0x4b,0x01,0x00,0x00,0xa1,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x48,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0x4b,0x01,0x00,0x00,0x4a,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0x54,0x01,0x00,0x00,0x5b,0x00,0x00,0x00, -0x73,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0x55,0x01,0x00,0x00, -0x54,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0x4e,0x00,0x00,0x00, -0x57,0x01,0x00,0x00,0x62,0x00,0x00,0x00,0x71,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x58,0x01,0x00,0x00,0x57,0x01,0x00,0x00, -0xc4,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0x59,0x01,0x00,0x00, -0x58,0x01,0x00,0x00,0x48,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4e,0x00,0x00,0x00,0x5b,0x01,0x00,0x00,0x67,0x00,0x00,0x00, -0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x5c,0x01,0x00,0x00, -0x5b,0x01,0x00,0x00,0xc5,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0x5d,0x01,0x00,0x00,0x59,0x01,0x00,0x00,0x5c,0x01,0x00,0x00, -0xc2,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0x5e,0x01,0x00,0x00, -0x5d,0x01,0x00,0x00,0xa3,0x00,0x00,0x00,0xc4,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0x5f,0x01,0x00,0x00,0x5e,0x01,0x00,0x00, -0x71,0x00,0x00,0x00,0xc7,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0x60,0x01,0x00,0x00,0x5f,0x01,0x00,0x00,0x52,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x61,0x01,0x00,0x00, -0x60,0x01,0x00,0x00,0xc2,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0x63,0x01,0x00,0x00,0x5d,0x01,0x00,0x00,0x86,0x04,0x00,0x00, -0xc7,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0x64,0x01,0x00,0x00, -0x63,0x01,0x00,0x00,0x52,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x65,0x01,0x00,0x00,0x64,0x01,0x00,0x00, -0x41,0x00,0x08,0x00,0x80,0x00,0x00,0x00,0x67,0x01,0x00,0x00, -0x58,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x37,0x00,0x00,0x00,0xa3,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x51,0x00,0x00,0x00,0x68,0x01,0x00,0x00,0x67,0x01,0x00,0x00, -0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x69,0x01,0x00,0x00, -0x68,0x01,0x00,0x00,0xc7,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0x6a,0x01,0x00,0x00,0x69,0x01,0x00,0x00,0x88,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x6c,0x01,0x00,0x00, -0x61,0x01,0x00,0x00,0xc5,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0x6d,0x01,0x00,0x00,0x6a,0x01,0x00,0x00,0x6c,0x01,0x00,0x00, -0x70,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0x6e,0x01,0x00,0x00, -0x6d,0x01,0x00,0x00,0xc2,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0x6f,0x01,0x00,0x00,0x69,0x01,0x00,0x00,0x71,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x71,0x01,0x00,0x00, -0x65,0x01,0x00,0x00,0xc5,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0x72,0x01,0x00,0x00,0x6f,0x01,0x00,0x00,0x71,0x01,0x00,0x00, -0x70,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0x73,0x01,0x00,0x00, -0x72,0x01,0x00,0x00,0x50,0x00,0x05,0x00,0x84,0x00,0x00,0x00, -0x74,0x01,0x00,0x00,0x6e,0x01,0x00,0x00,0x73,0x01,0x00,0x00, -0x83,0x00,0x05,0x00,0x84,0x00,0x00,0x00,0x75,0x01,0x00,0x00, -0x74,0x01,0x00,0x00,0xcf,0x00,0x00,0x00,0x8e,0x00,0x05,0x00, -0x84,0x00,0x00,0x00,0x76,0x01,0x00,0x00,0x75,0x01,0x00,0x00, -0x55,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x7c,0x01,0x00,0x00,0xa9,0x00,0x00,0x00,0xa3,0x00,0x00,0x00, -0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00,0x7e,0x01,0x00,0x00, -0x76,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x73,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0x7f,0x01,0x00,0x00,0x7e,0x01,0x00,0x00, -0x41,0x00,0x06,0x00,0x5a,0x00,0x00,0x00,0x80,0x01,0x00,0x00, -0xa1,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x7c,0x01,0x00,0x00, -0x3e,0x00,0x03,0x00,0x80,0x01,0x00,0x00,0x7f,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x87,0x01,0x00,0x00, -0xa9,0x00,0x00,0x00,0x87,0x04,0x00,0x00,0x51,0x00,0x05,0x00, -0x4a,0x00,0x00,0x00,0x88,0x01,0x00,0x00,0x76,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0x89,0x01,0x00,0x00,0x88,0x01,0x00,0x00,0x41,0x00,0x06,0x00, -0x5a,0x00,0x00,0x00,0x8a,0x01,0x00,0x00,0xa1,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x87,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0x8a,0x01,0x00,0x00,0x89,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0x93,0x01,0x00,0x00,0x5b,0x00,0x00,0x00, -0x73,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0x94,0x01,0x00,0x00, -0x93,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0x4e,0x00,0x00,0x00, -0x96,0x01,0x00,0x00,0x62,0x00,0x00,0x00,0x71,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x97,0x01,0x00,0x00,0x96,0x01,0x00,0x00, -0xc4,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0x98,0x01,0x00,0x00, -0x97,0x01,0x00,0x00,0x48,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4e,0x00,0x00,0x00,0x9a,0x01,0x00,0x00,0x67,0x00,0x00,0x00, -0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x9b,0x01,0x00,0x00, -0x9a,0x01,0x00,0x00,0xc5,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0x9c,0x01,0x00,0x00,0x98,0x01,0x00,0x00,0x9b,0x01,0x00,0x00, -0xc2,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0x9d,0x01,0x00,0x00, -0x9c,0x01,0x00,0x00,0x71,0x00,0x00,0x00,0xc4,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0x9e,0x01,0x00,0x00,0x9d,0x01,0x00,0x00, -0x71,0x00,0x00,0x00,0xc7,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0x9f,0x01,0x00,0x00,0x9e,0x01,0x00,0x00,0x52,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xa0,0x01,0x00,0x00, -0x9f,0x01,0x00,0x00,0xc2,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0xa2,0x01,0x00,0x00,0x9c,0x01,0x00,0x00,0x48,0x00,0x00,0x00, -0xc7,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0xa3,0x01,0x00,0x00, -0xa2,0x01,0x00,0x00,0x52,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xa4,0x01,0x00,0x00,0xa3,0x01,0x00,0x00, -0x41,0x00,0x08,0x00,0x80,0x00,0x00,0x00,0xa6,0x01,0x00,0x00, -0x58,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x37,0x00,0x00,0x00,0x71,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x51,0x00,0x00,0x00,0xa7,0x01,0x00,0x00,0xa6,0x01,0x00,0x00, -0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0xa8,0x01,0x00,0x00, -0xa7,0x01,0x00,0x00,0xc7,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0xa9,0x01,0x00,0x00,0xa8,0x01,0x00,0x00,0x88,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0xab,0x01,0x00,0x00, -0xa0,0x01,0x00,0x00,0xc5,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0xac,0x01,0x00,0x00,0xa9,0x01,0x00,0x00,0xab,0x01,0x00,0x00, -0x70,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0xad,0x01,0x00,0x00, -0xac,0x01,0x00,0x00,0xc2,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0xae,0x01,0x00,0x00,0xa8,0x01,0x00,0x00,0x71,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0xb0,0x01,0x00,0x00, -0xa4,0x01,0x00,0x00,0xc5,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0xb1,0x01,0x00,0x00,0xae,0x01,0x00,0x00,0xb0,0x01,0x00,0x00, -0x70,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0xb2,0x01,0x00,0x00, -0xb1,0x01,0x00,0x00,0x50,0x00,0x05,0x00,0x84,0x00,0x00,0x00, -0xb3,0x01,0x00,0x00,0xad,0x01,0x00,0x00,0xb2,0x01,0x00,0x00, -0x83,0x00,0x05,0x00,0x84,0x00,0x00,0x00,0xb4,0x01,0x00,0x00, -0xb3,0x01,0x00,0x00,0xcf,0x00,0x00,0x00,0x8e,0x00,0x05,0x00, -0x84,0x00,0x00,0x00,0xb5,0x01,0x00,0x00,0xb4,0x01,0x00,0x00, -0x94,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xbb,0x01,0x00,0x00,0xa9,0x00,0x00,0x00,0x71,0x00,0x00,0x00, -0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00,0xbd,0x01,0x00,0x00, -0xb5,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x73,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0xbe,0x01,0x00,0x00,0xbd,0x01,0x00,0x00, -0x41,0x00,0x06,0x00,0x5a,0x00,0x00,0x00,0xbf,0x01,0x00,0x00, -0xa1,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0xbb,0x01,0x00,0x00, -0x3e,0x00,0x03,0x00,0xbf,0x01,0x00,0x00,0xbe,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xc6,0x01,0x00,0x00, -0xa9,0x00,0x00,0x00,0x88,0x04,0x00,0x00,0x51,0x00,0x05,0x00, -0x4a,0x00,0x00,0x00,0xc7,0x01,0x00,0x00,0xb5,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0xc8,0x01,0x00,0x00,0xc7,0x01,0x00,0x00,0x41,0x00,0x06,0x00, -0x5a,0x00,0x00,0x00,0xc9,0x01,0x00,0x00,0xa1,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0xc6,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0xc9,0x01,0x00,0x00,0xc8,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0xd2,0x01,0x00,0x00,0x5b,0x00,0x00,0x00, -0x73,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0xd3,0x01,0x00,0x00, -0xd2,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0x4e,0x00,0x00,0x00, -0xd5,0x01,0x00,0x00,0x62,0x00,0x00,0x00,0x71,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0xd6,0x01,0x00,0x00,0xd5,0x01,0x00,0x00, -0xc4,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0xd7,0x01,0x00,0x00, -0xd6,0x01,0x00,0x00,0x48,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4e,0x00,0x00,0x00,0xd9,0x01,0x00,0x00,0x67,0x00,0x00,0x00, -0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0xda,0x01,0x00,0x00, -0xd9,0x01,0x00,0x00,0xc5,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0xdb,0x01,0x00,0x00,0xd7,0x01,0x00,0x00,0xda,0x01,0x00,0x00, -0xc2,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0xdc,0x01,0x00,0x00, -0xdb,0x01,0x00,0x00,0x89,0x04,0x00,0x00,0xc4,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0xdd,0x01,0x00,0x00,0xdc,0x01,0x00,0x00, -0x71,0x00,0x00,0x00,0xc7,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0xde,0x01,0x00,0x00,0xdd,0x01,0x00,0x00,0x52,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xdf,0x01,0x00,0x00, -0xde,0x01,0x00,0x00,0xc2,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0xe1,0x01,0x00,0x00,0xdb,0x01,0x00,0x00,0x83,0x04,0x00,0x00, -0xc7,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0xe2,0x01,0x00,0x00, -0xe1,0x01,0x00,0x00,0x52,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xe3,0x01,0x00,0x00,0xe2,0x01,0x00,0x00, -0x41,0x00,0x08,0x00,0x80,0x00,0x00,0x00,0xe5,0x01,0x00,0x00, -0x58,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x37,0x00,0x00,0x00,0x89,0x04,0x00,0x00,0x3d,0x00,0x04,0x00, -0x51,0x00,0x00,0x00,0xe6,0x01,0x00,0x00,0xe5,0x01,0x00,0x00, -0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0xe7,0x01,0x00,0x00, -0xe6,0x01,0x00,0x00,0xc7,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0xe8,0x01,0x00,0x00,0xe7,0x01,0x00,0x00,0x88,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0xea,0x01,0x00,0x00, -0xdf,0x01,0x00,0x00,0xc5,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0xeb,0x01,0x00,0x00,0xe8,0x01,0x00,0x00,0xea,0x01,0x00,0x00, -0x70,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0xec,0x01,0x00,0x00, -0xeb,0x01,0x00,0x00,0xc2,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0xed,0x01,0x00,0x00,0xe7,0x01,0x00,0x00,0x71,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0xef,0x01,0x00,0x00, -0xe3,0x01,0x00,0x00,0xc5,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0xf0,0x01,0x00,0x00,0xed,0x01,0x00,0x00,0xef,0x01,0x00,0x00, -0x70,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0xf1,0x01,0x00,0x00, -0xf0,0x01,0x00,0x00,0x50,0x00,0x05,0x00,0x84,0x00,0x00,0x00, -0xf2,0x01,0x00,0x00,0xec,0x01,0x00,0x00,0xf1,0x01,0x00,0x00, -0x83,0x00,0x05,0x00,0x84,0x00,0x00,0x00,0xf3,0x01,0x00,0x00, -0xf2,0x01,0x00,0x00,0xcf,0x00,0x00,0x00,0x8e,0x00,0x05,0x00, -0x84,0x00,0x00,0x00,0xf4,0x01,0x00,0x00,0xf3,0x01,0x00,0x00, -0xd3,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xfa,0x01,0x00,0x00,0xa9,0x00,0x00,0x00,0x89,0x04,0x00,0x00, -0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00,0xfc,0x01,0x00,0x00, -0xf4,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x73,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0xfd,0x01,0x00,0x00,0xfc,0x01,0x00,0x00, -0x41,0x00,0x06,0x00,0x5a,0x00,0x00,0x00,0xfe,0x01,0x00,0x00, -0xa1,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0xfa,0x01,0x00,0x00, -0x3e,0x00,0x03,0x00,0xfe,0x01,0x00,0x00,0xfd,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x05,0x02,0x00,0x00, -0xa9,0x00,0x00,0x00,0x8a,0x04,0x00,0x00,0x51,0x00,0x05,0x00, -0x4a,0x00,0x00,0x00,0x06,0x02,0x00,0x00,0xf4,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0x07,0x02,0x00,0x00,0x06,0x02,0x00,0x00,0x41,0x00,0x06,0x00, -0x5a,0x00,0x00,0x00,0x08,0x02,0x00,0x00,0xa1,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x05,0x02,0x00,0x00,0x3e,0x00,0x03,0x00, -0x08,0x02,0x00,0x00,0x07,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0x11,0x02,0x00,0x00,0x5b,0x00,0x00,0x00, -0x73,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0x12,0x02,0x00,0x00, -0x11,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0x4e,0x00,0x00,0x00, -0x14,0x02,0x00,0x00,0x62,0x00,0x00,0x00,0x71,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x15,0x02,0x00,0x00,0x14,0x02,0x00,0x00, -0xc4,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0x16,0x02,0x00,0x00, -0x15,0x02,0x00,0x00,0x48,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4e,0x00,0x00,0x00,0x18,0x02,0x00,0x00,0x67,0x00,0x00,0x00, -0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x19,0x02,0x00,0x00, -0x18,0x02,0x00,0x00,0xc5,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0x1a,0x02,0x00,0x00,0x16,0x02,0x00,0x00,0x19,0x02,0x00,0x00, -0xc2,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0x1b,0x02,0x00,0x00, -0x1a,0x02,0x00,0x00,0x8b,0x04,0x00,0x00,0xc4,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0x1c,0x02,0x00,0x00,0x1b,0x02,0x00,0x00, -0x71,0x00,0x00,0x00,0xc7,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0x1d,0x02,0x00,0x00,0x1c,0x02,0x00,0x00,0x52,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x1e,0x02,0x00,0x00, -0x1d,0x02,0x00,0x00,0xc2,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0x20,0x02,0x00,0x00,0x1a,0x02,0x00,0x00,0x85,0x04,0x00,0x00, -0xc7,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0x21,0x02,0x00,0x00, -0x20,0x02,0x00,0x00,0x52,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x22,0x02,0x00,0x00,0x21,0x02,0x00,0x00, -0x41,0x00,0x08,0x00,0x80,0x00,0x00,0x00,0x24,0x02,0x00,0x00, -0x58,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x37,0x00,0x00,0x00,0x8b,0x04,0x00,0x00,0x3d,0x00,0x04,0x00, -0x51,0x00,0x00,0x00,0x25,0x02,0x00,0x00,0x24,0x02,0x00,0x00, -0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x26,0x02,0x00,0x00, -0x25,0x02,0x00,0x00,0xc7,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0x27,0x02,0x00,0x00,0x26,0x02,0x00,0x00,0x88,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x29,0x02,0x00,0x00, -0x1e,0x02,0x00,0x00,0xc5,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0x2a,0x02,0x00,0x00,0x27,0x02,0x00,0x00,0x29,0x02,0x00,0x00, -0x70,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0x2b,0x02,0x00,0x00, -0x2a,0x02,0x00,0x00,0xc2,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0x2c,0x02,0x00,0x00,0x26,0x02,0x00,0x00,0x71,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x2e,0x02,0x00,0x00, -0x22,0x02,0x00,0x00,0xc5,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0x2f,0x02,0x00,0x00,0x2c,0x02,0x00,0x00,0x2e,0x02,0x00,0x00, -0x70,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0x30,0x02,0x00,0x00, -0x2f,0x02,0x00,0x00,0x50,0x00,0x05,0x00,0x84,0x00,0x00,0x00, -0x31,0x02,0x00,0x00,0x2b,0x02,0x00,0x00,0x30,0x02,0x00,0x00, -0x83,0x00,0x05,0x00,0x84,0x00,0x00,0x00,0x32,0x02,0x00,0x00, -0x31,0x02,0x00,0x00,0xcf,0x00,0x00,0x00,0x8e,0x00,0x05,0x00, -0x84,0x00,0x00,0x00,0x33,0x02,0x00,0x00,0x32,0x02,0x00,0x00, -0x12,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x39,0x02,0x00,0x00,0xa9,0x00,0x00,0x00,0x8b,0x04,0x00,0x00, -0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00,0x3b,0x02,0x00,0x00, -0x33,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x73,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0x3c,0x02,0x00,0x00,0x3b,0x02,0x00,0x00, -0x41,0x00,0x06,0x00,0x5a,0x00,0x00,0x00,0x3d,0x02,0x00,0x00, -0xa1,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x39,0x02,0x00,0x00, -0x3e,0x00,0x03,0x00,0x3d,0x02,0x00,0x00,0x3c,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x44,0x02,0x00,0x00, -0xa9,0x00,0x00,0x00,0x8c,0x04,0x00,0x00,0x51,0x00,0x05,0x00, -0x4a,0x00,0x00,0x00,0x45,0x02,0x00,0x00,0x33,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0x46,0x02,0x00,0x00,0x45,0x02,0x00,0x00,0x41,0x00,0x06,0x00, -0x5a,0x00,0x00,0x00,0x47,0x02,0x00,0x00,0xa1,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x44,0x02,0x00,0x00,0x3e,0x00,0x03,0x00, -0x47,0x02,0x00,0x00,0x46,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0x50,0x02,0x00,0x00,0x5b,0x00,0x00,0x00, -0x73,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0x51,0x02,0x00,0x00, -0x50,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0x4e,0x00,0x00,0x00, -0x53,0x02,0x00,0x00,0x62,0x00,0x00,0x00,0x71,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x54,0x02,0x00,0x00,0x53,0x02,0x00,0x00, -0xc4,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0x55,0x02,0x00,0x00, -0x54,0x02,0x00,0x00,0x48,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4e,0x00,0x00,0x00,0x57,0x02,0x00,0x00,0x67,0x00,0x00,0x00, -0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x58,0x02,0x00,0x00, -0x57,0x02,0x00,0x00,0xc5,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0x59,0x02,0x00,0x00,0x55,0x02,0x00,0x00,0x58,0x02,0x00,0x00, -0xc2,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0x5a,0x02,0x00,0x00, -0x59,0x02,0x00,0x00,0x8d,0x04,0x00,0x00,0xc4,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0x5b,0x02,0x00,0x00,0x5a,0x02,0x00,0x00, -0x71,0x00,0x00,0x00,0xc7,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0x5c,0x02,0x00,0x00,0x5b,0x02,0x00,0x00,0x52,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x5d,0x02,0x00,0x00, -0x5c,0x02,0x00,0x00,0xc2,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0x5f,0x02,0x00,0x00,0x59,0x02,0x00,0x00,0x87,0x04,0x00,0x00, -0xc7,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0x60,0x02,0x00,0x00, -0x5f,0x02,0x00,0x00,0x52,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x61,0x02,0x00,0x00,0x60,0x02,0x00,0x00, -0x41,0x00,0x08,0x00,0x80,0x00,0x00,0x00,0x63,0x02,0x00,0x00, -0x58,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x37,0x00,0x00,0x00,0x8d,0x04,0x00,0x00,0x3d,0x00,0x04,0x00, -0x51,0x00,0x00,0x00,0x64,0x02,0x00,0x00,0x63,0x02,0x00,0x00, -0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x65,0x02,0x00,0x00, -0x64,0x02,0x00,0x00,0xc7,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0x66,0x02,0x00,0x00,0x65,0x02,0x00,0x00,0x88,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x68,0x02,0x00,0x00, -0x5d,0x02,0x00,0x00,0xc5,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0x69,0x02,0x00,0x00,0x66,0x02,0x00,0x00,0x68,0x02,0x00,0x00, -0x70,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0x6a,0x02,0x00,0x00, -0x69,0x02,0x00,0x00,0xc2,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0x6b,0x02,0x00,0x00,0x65,0x02,0x00,0x00,0x71,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x6d,0x02,0x00,0x00, -0x61,0x02,0x00,0x00,0xc5,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0x6e,0x02,0x00,0x00,0x6b,0x02,0x00,0x00,0x6d,0x02,0x00,0x00, -0x70,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0x6f,0x02,0x00,0x00, -0x6e,0x02,0x00,0x00,0x50,0x00,0x05,0x00,0x84,0x00,0x00,0x00, -0x70,0x02,0x00,0x00,0x6a,0x02,0x00,0x00,0x6f,0x02,0x00,0x00, -0x83,0x00,0x05,0x00,0x84,0x00,0x00,0x00,0x71,0x02,0x00,0x00, -0x70,0x02,0x00,0x00,0xcf,0x00,0x00,0x00,0x8e,0x00,0x05,0x00, -0x84,0x00,0x00,0x00,0x72,0x02,0x00,0x00,0x71,0x02,0x00,0x00, -0x51,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x78,0x02,0x00,0x00,0xa9,0x00,0x00,0x00,0x8d,0x04,0x00,0x00, -0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00,0x7a,0x02,0x00,0x00, -0x72,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x73,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0x7b,0x02,0x00,0x00,0x7a,0x02,0x00,0x00, -0x41,0x00,0x06,0x00,0x5a,0x00,0x00,0x00,0x7c,0x02,0x00,0x00, -0xa1,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x78,0x02,0x00,0x00, -0x3e,0x00,0x03,0x00,0x7c,0x02,0x00,0x00,0x7b,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x83,0x02,0x00,0x00, -0xa9,0x00,0x00,0x00,0x8e,0x04,0x00,0x00,0x51,0x00,0x05,0x00, -0x4a,0x00,0x00,0x00,0x84,0x02,0x00,0x00,0x72,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0x85,0x02,0x00,0x00,0x84,0x02,0x00,0x00,0x41,0x00,0x06,0x00, -0x5a,0x00,0x00,0x00,0x86,0x02,0x00,0x00,0xa1,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x83,0x02,0x00,0x00,0x3e,0x00,0x03,0x00, -0x86,0x02,0x00,0x00,0x85,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0x8f,0x02,0x00,0x00,0x5b,0x00,0x00,0x00, -0x73,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0x90,0x02,0x00,0x00, -0x8f,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0x4e,0x00,0x00,0x00, -0x92,0x02,0x00,0x00,0x62,0x00,0x00,0x00,0x71,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x93,0x02,0x00,0x00,0x92,0x02,0x00,0x00, -0xc4,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0x94,0x02,0x00,0x00, -0x93,0x02,0x00,0x00,0x48,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4e,0x00,0x00,0x00,0x96,0x02,0x00,0x00,0x67,0x00,0x00,0x00, -0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x97,0x02,0x00,0x00, -0x96,0x02,0x00,0x00,0xc5,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0x98,0x02,0x00,0x00,0x94,0x02,0x00,0x00,0x97,0x02,0x00,0x00, -0xc2,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0x99,0x02,0x00,0x00, -0x98,0x02,0x00,0x00,0x8f,0x04,0x00,0x00,0xc4,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0x9a,0x02,0x00,0x00,0x99,0x02,0x00,0x00, -0x71,0x00,0x00,0x00,0xc7,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0x9b,0x02,0x00,0x00,0x9a,0x02,0x00,0x00,0x52,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x9c,0x02,0x00,0x00, -0x9b,0x02,0x00,0x00,0xc2,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0x9e,0x02,0x00,0x00,0x98,0x02,0x00,0x00,0x88,0x04,0x00,0x00, -0xc7,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0x9f,0x02,0x00,0x00, -0x9e,0x02,0x00,0x00,0x52,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xa0,0x02,0x00,0x00,0x9f,0x02,0x00,0x00, -0x41,0x00,0x08,0x00,0x80,0x00,0x00,0x00,0xa2,0x02,0x00,0x00, -0x58,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x37,0x00,0x00,0x00,0x8f,0x04,0x00,0x00,0x3d,0x00,0x04,0x00, -0x51,0x00,0x00,0x00,0xa3,0x02,0x00,0x00,0xa2,0x02,0x00,0x00, -0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0xa4,0x02,0x00,0x00, -0xa3,0x02,0x00,0x00,0xc7,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0xa5,0x02,0x00,0x00,0xa4,0x02,0x00,0x00,0x88,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0xa7,0x02,0x00,0x00, -0x9c,0x02,0x00,0x00,0xc5,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0xa8,0x02,0x00,0x00,0xa5,0x02,0x00,0x00,0xa7,0x02,0x00,0x00, -0x70,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0xa9,0x02,0x00,0x00, -0xa8,0x02,0x00,0x00,0xc2,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0xaa,0x02,0x00,0x00,0xa4,0x02,0x00,0x00,0x71,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0xac,0x02,0x00,0x00, -0xa0,0x02,0x00,0x00,0xc5,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0xad,0x02,0x00,0x00,0xaa,0x02,0x00,0x00,0xac,0x02,0x00,0x00, -0x70,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0xae,0x02,0x00,0x00, -0xad,0x02,0x00,0x00,0x50,0x00,0x05,0x00,0x84,0x00,0x00,0x00, -0xaf,0x02,0x00,0x00,0xa9,0x02,0x00,0x00,0xae,0x02,0x00,0x00, -0x83,0x00,0x05,0x00,0x84,0x00,0x00,0x00,0xb0,0x02,0x00,0x00, -0xaf,0x02,0x00,0x00,0xcf,0x00,0x00,0x00,0x8e,0x00,0x05,0x00, -0x84,0x00,0x00,0x00,0xb1,0x02,0x00,0x00,0xb0,0x02,0x00,0x00, -0x90,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xb7,0x02,0x00,0x00,0xa9,0x00,0x00,0x00,0x8f,0x04,0x00,0x00, -0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00,0xb9,0x02,0x00,0x00, -0xb1,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x73,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0xba,0x02,0x00,0x00,0xb9,0x02,0x00,0x00, -0x41,0x00,0x06,0x00,0x5a,0x00,0x00,0x00,0xbb,0x02,0x00,0x00, -0xa1,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0xb7,0x02,0x00,0x00, -0x3e,0x00,0x03,0x00,0xbb,0x02,0x00,0x00,0xba,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xc2,0x02,0x00,0x00, -0xa9,0x00,0x00,0x00,0x90,0x04,0x00,0x00,0x51,0x00,0x05,0x00, -0x4a,0x00,0x00,0x00,0xc3,0x02,0x00,0x00,0xb1,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0xc4,0x02,0x00,0x00,0xc3,0x02,0x00,0x00,0x41,0x00,0x06,0x00, -0x5a,0x00,0x00,0x00,0xc5,0x02,0x00,0x00,0xa1,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0xc2,0x02,0x00,0x00,0x3e,0x00,0x03,0x00, -0xc5,0x02,0x00,0x00,0xc4,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0xce,0x02,0x00,0x00,0x5b,0x00,0x00,0x00, -0x73,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0xcf,0x02,0x00,0x00, -0xce,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0x4e,0x00,0x00,0x00, -0xd1,0x02,0x00,0x00,0x62,0x00,0x00,0x00,0x71,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0xd2,0x02,0x00,0x00,0xd1,0x02,0x00,0x00, -0xc4,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0xd3,0x02,0x00,0x00, -0xd2,0x02,0x00,0x00,0x48,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4e,0x00,0x00,0x00,0xd5,0x02,0x00,0x00,0x67,0x00,0x00,0x00, -0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0xd6,0x02,0x00,0x00, -0xd5,0x02,0x00,0x00,0xc5,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0xd7,0x02,0x00,0x00,0xd3,0x02,0x00,0x00,0xd6,0x02,0x00,0x00, -0xc2,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0xd8,0x02,0x00,0x00, -0xd7,0x02,0x00,0x00,0x91,0x04,0x00,0x00,0xc4,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0xd9,0x02,0x00,0x00,0xd8,0x02,0x00,0x00, -0x71,0x00,0x00,0x00,0xc7,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0xda,0x02,0x00,0x00,0xd9,0x02,0x00,0x00,0x52,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xdb,0x02,0x00,0x00, -0xda,0x02,0x00,0x00,0xc2,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0xdd,0x02,0x00,0x00,0xd7,0x02,0x00,0x00,0x8a,0x04,0x00,0x00, -0xc7,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0xde,0x02,0x00,0x00, -0xdd,0x02,0x00,0x00,0x52,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xdf,0x02,0x00,0x00,0xde,0x02,0x00,0x00, -0x41,0x00,0x08,0x00,0x80,0x00,0x00,0x00,0xe1,0x02,0x00,0x00, -0x58,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x37,0x00,0x00,0x00,0x91,0x04,0x00,0x00,0x3d,0x00,0x04,0x00, -0x51,0x00,0x00,0x00,0xe2,0x02,0x00,0x00,0xe1,0x02,0x00,0x00, -0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0xe3,0x02,0x00,0x00, -0xe2,0x02,0x00,0x00,0xc7,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0xe4,0x02,0x00,0x00,0xe3,0x02,0x00,0x00,0x88,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0xe6,0x02,0x00,0x00, -0xdb,0x02,0x00,0x00,0xc5,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0xe7,0x02,0x00,0x00,0xe4,0x02,0x00,0x00,0xe6,0x02,0x00,0x00, -0x70,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0xe8,0x02,0x00,0x00, -0xe7,0x02,0x00,0x00,0xc2,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0xe9,0x02,0x00,0x00,0xe3,0x02,0x00,0x00,0x71,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0xeb,0x02,0x00,0x00, -0xdf,0x02,0x00,0x00,0xc5,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0xec,0x02,0x00,0x00,0xe9,0x02,0x00,0x00,0xeb,0x02,0x00,0x00, -0x70,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0xed,0x02,0x00,0x00, -0xec,0x02,0x00,0x00,0x50,0x00,0x05,0x00,0x84,0x00,0x00,0x00, -0xee,0x02,0x00,0x00,0xe8,0x02,0x00,0x00,0xed,0x02,0x00,0x00, -0x83,0x00,0x05,0x00,0x84,0x00,0x00,0x00,0xef,0x02,0x00,0x00, -0xee,0x02,0x00,0x00,0xcf,0x00,0x00,0x00,0x8e,0x00,0x05,0x00, -0x84,0x00,0x00,0x00,0xf0,0x02,0x00,0x00,0xef,0x02,0x00,0x00, -0xcf,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xf6,0x02,0x00,0x00,0xa9,0x00,0x00,0x00,0x91,0x04,0x00,0x00, -0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00,0xf8,0x02,0x00,0x00, -0xf0,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x73,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0xf9,0x02,0x00,0x00,0xf8,0x02,0x00,0x00, -0x41,0x00,0x06,0x00,0x5a,0x00,0x00,0x00,0xfa,0x02,0x00,0x00, -0xa1,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0xf6,0x02,0x00,0x00, -0x3e,0x00,0x03,0x00,0xfa,0x02,0x00,0x00,0xf9,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x01,0x03,0x00,0x00, -0xa9,0x00,0x00,0x00,0x92,0x04,0x00,0x00,0x51,0x00,0x05,0x00, -0x4a,0x00,0x00,0x00,0x02,0x03,0x00,0x00,0xf0,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0x03,0x03,0x00,0x00,0x02,0x03,0x00,0x00,0x41,0x00,0x06,0x00, -0x5a,0x00,0x00,0x00,0x04,0x03,0x00,0x00,0xa1,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x01,0x03,0x00,0x00,0x3e,0x00,0x03,0x00, -0x04,0x03,0x00,0x00,0x03,0x03,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0x0d,0x03,0x00,0x00,0x5b,0x00,0x00,0x00, -0x73,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0x0e,0x03,0x00,0x00, -0x0d,0x03,0x00,0x00,0x3d,0x00,0x04,0x00,0x4e,0x00,0x00,0x00, -0x10,0x03,0x00,0x00,0x62,0x00,0x00,0x00,0x71,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x11,0x03,0x00,0x00,0x10,0x03,0x00,0x00, -0xc4,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0x12,0x03,0x00,0x00, -0x11,0x03,0x00,0x00,0x48,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4e,0x00,0x00,0x00,0x14,0x03,0x00,0x00,0x67,0x00,0x00,0x00, -0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x15,0x03,0x00,0x00, -0x14,0x03,0x00,0x00,0xc5,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0x16,0x03,0x00,0x00,0x12,0x03,0x00,0x00,0x15,0x03,0x00,0x00, -0xc2,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0x17,0x03,0x00,0x00, -0x16,0x03,0x00,0x00,0x93,0x04,0x00,0x00,0xc4,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0x18,0x03,0x00,0x00,0x17,0x03,0x00,0x00, -0x71,0x00,0x00,0x00,0xc7,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0x19,0x03,0x00,0x00,0x18,0x03,0x00,0x00,0x52,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x1a,0x03,0x00,0x00, -0x19,0x03,0x00,0x00,0xc2,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0x1c,0x03,0x00,0x00,0x16,0x03,0x00,0x00,0x8c,0x04,0x00,0x00, -0xc7,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0x1d,0x03,0x00,0x00, -0x1c,0x03,0x00,0x00,0x52,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x1e,0x03,0x00,0x00,0x1d,0x03,0x00,0x00, -0x41,0x00,0x08,0x00,0x80,0x00,0x00,0x00,0x20,0x03,0x00,0x00, -0x58,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x37,0x00,0x00,0x00,0x93,0x04,0x00,0x00,0x3d,0x00,0x04,0x00, -0x51,0x00,0x00,0x00,0x21,0x03,0x00,0x00,0x20,0x03,0x00,0x00, -0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x22,0x03,0x00,0x00, -0x21,0x03,0x00,0x00,0xc7,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0x23,0x03,0x00,0x00,0x22,0x03,0x00,0x00,0x88,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x25,0x03,0x00,0x00, -0x1a,0x03,0x00,0x00,0xc5,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0x26,0x03,0x00,0x00,0x23,0x03,0x00,0x00,0x25,0x03,0x00,0x00, -0x70,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0x27,0x03,0x00,0x00, -0x26,0x03,0x00,0x00,0xc2,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0x28,0x03,0x00,0x00,0x22,0x03,0x00,0x00,0x71,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x2a,0x03,0x00,0x00, -0x1e,0x03,0x00,0x00,0xc5,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0x2b,0x03,0x00,0x00,0x28,0x03,0x00,0x00,0x2a,0x03,0x00,0x00, -0x70,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0x2c,0x03,0x00,0x00, -0x2b,0x03,0x00,0x00,0x50,0x00,0x05,0x00,0x84,0x00,0x00,0x00, -0x2d,0x03,0x00,0x00,0x27,0x03,0x00,0x00,0x2c,0x03,0x00,0x00, -0x83,0x00,0x05,0x00,0x84,0x00,0x00,0x00,0x2e,0x03,0x00,0x00, -0x2d,0x03,0x00,0x00,0xcf,0x00,0x00,0x00,0x8e,0x00,0x05,0x00, -0x84,0x00,0x00,0x00,0x2f,0x03,0x00,0x00,0x2e,0x03,0x00,0x00, -0x0e,0x03,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x35,0x03,0x00,0x00,0xa9,0x00,0x00,0x00,0x93,0x04,0x00,0x00, -0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00,0x37,0x03,0x00,0x00, -0x2f,0x03,0x00,0x00,0x00,0x00,0x00,0x00,0x73,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0x38,0x03,0x00,0x00,0x37,0x03,0x00,0x00, -0x41,0x00,0x06,0x00,0x5a,0x00,0x00,0x00,0x39,0x03,0x00,0x00, -0xa1,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x35,0x03,0x00,0x00, -0x3e,0x00,0x03,0x00,0x39,0x03,0x00,0x00,0x38,0x03,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x40,0x03,0x00,0x00, -0xa9,0x00,0x00,0x00,0x94,0x04,0x00,0x00,0x51,0x00,0x05,0x00, -0x4a,0x00,0x00,0x00,0x41,0x03,0x00,0x00,0x2f,0x03,0x00,0x00, -0x01,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0x42,0x03,0x00,0x00,0x41,0x03,0x00,0x00,0x41,0x00,0x06,0x00, -0x5a,0x00,0x00,0x00,0x43,0x03,0x00,0x00,0xa1,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x40,0x03,0x00,0x00,0x3e,0x00,0x03,0x00, -0x43,0x03,0x00,0x00,0x42,0x03,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0x4c,0x03,0x00,0x00,0x5b,0x00,0x00,0x00, -0x73,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0x4d,0x03,0x00,0x00, -0x4c,0x03,0x00,0x00,0x3d,0x00,0x04,0x00,0x4e,0x00,0x00,0x00, -0x4f,0x03,0x00,0x00,0x62,0x00,0x00,0x00,0x71,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x50,0x03,0x00,0x00,0x4f,0x03,0x00,0x00, -0xc4,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0x51,0x03,0x00,0x00, -0x50,0x03,0x00,0x00,0x48,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4e,0x00,0x00,0x00,0x53,0x03,0x00,0x00,0x67,0x00,0x00,0x00, -0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x54,0x03,0x00,0x00, -0x53,0x03,0x00,0x00,0xc5,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0x55,0x03,0x00,0x00,0x51,0x03,0x00,0x00,0x54,0x03,0x00,0x00, -0xc2,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0x56,0x03,0x00,0x00, -0x55,0x03,0x00,0x00,0x95,0x04,0x00,0x00,0xc4,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0x57,0x03,0x00,0x00,0x56,0x03,0x00,0x00, -0x71,0x00,0x00,0x00,0xc7,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0x58,0x03,0x00,0x00,0x57,0x03,0x00,0x00,0x52,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x59,0x03,0x00,0x00, -0x58,0x03,0x00,0x00,0xc2,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0x5b,0x03,0x00,0x00,0x55,0x03,0x00,0x00,0x8e,0x04,0x00,0x00, -0xc7,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0x5c,0x03,0x00,0x00, -0x5b,0x03,0x00,0x00,0x52,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x5d,0x03,0x00,0x00,0x5c,0x03,0x00,0x00, -0x41,0x00,0x08,0x00,0x80,0x00,0x00,0x00,0x5f,0x03,0x00,0x00, -0x58,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x37,0x00,0x00,0x00,0x95,0x04,0x00,0x00,0x3d,0x00,0x04,0x00, -0x51,0x00,0x00,0x00,0x60,0x03,0x00,0x00,0x5f,0x03,0x00,0x00, -0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x61,0x03,0x00,0x00, -0x60,0x03,0x00,0x00,0xc7,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0x62,0x03,0x00,0x00,0x61,0x03,0x00,0x00,0x88,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x64,0x03,0x00,0x00, -0x59,0x03,0x00,0x00,0xc5,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0x65,0x03,0x00,0x00,0x62,0x03,0x00,0x00,0x64,0x03,0x00,0x00, -0x70,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0x66,0x03,0x00,0x00, -0x65,0x03,0x00,0x00,0xc2,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0x67,0x03,0x00,0x00,0x61,0x03,0x00,0x00,0x71,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x69,0x03,0x00,0x00, -0x5d,0x03,0x00,0x00,0xc5,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0x6a,0x03,0x00,0x00,0x67,0x03,0x00,0x00,0x69,0x03,0x00,0x00, -0x70,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0x6b,0x03,0x00,0x00, -0x6a,0x03,0x00,0x00,0x50,0x00,0x05,0x00,0x84,0x00,0x00,0x00, -0x6c,0x03,0x00,0x00,0x66,0x03,0x00,0x00,0x6b,0x03,0x00,0x00, -0x83,0x00,0x05,0x00,0x84,0x00,0x00,0x00,0x6d,0x03,0x00,0x00, -0x6c,0x03,0x00,0x00,0xcf,0x00,0x00,0x00,0x8e,0x00,0x05,0x00, -0x84,0x00,0x00,0x00,0x6e,0x03,0x00,0x00,0x6d,0x03,0x00,0x00, -0x4d,0x03,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x74,0x03,0x00,0x00,0xa9,0x00,0x00,0x00,0x95,0x04,0x00,0x00, -0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00,0x76,0x03,0x00,0x00, -0x6e,0x03,0x00,0x00,0x00,0x00,0x00,0x00,0x73,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0x77,0x03,0x00,0x00,0x76,0x03,0x00,0x00, -0x41,0x00,0x06,0x00,0x5a,0x00,0x00,0x00,0x78,0x03,0x00,0x00, -0xa1,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x74,0x03,0x00,0x00, -0x3e,0x00,0x03,0x00,0x78,0x03,0x00,0x00,0x77,0x03,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x7f,0x03,0x00,0x00, -0xa9,0x00,0x00,0x00,0x96,0x04,0x00,0x00,0x51,0x00,0x05,0x00, -0x4a,0x00,0x00,0x00,0x80,0x03,0x00,0x00,0x6e,0x03,0x00,0x00, -0x01,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0x81,0x03,0x00,0x00,0x80,0x03,0x00,0x00,0x41,0x00,0x06,0x00, -0x5a,0x00,0x00,0x00,0x82,0x03,0x00,0x00,0xa1,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x7f,0x03,0x00,0x00,0x3e,0x00,0x03,0x00, -0x82,0x03,0x00,0x00,0x81,0x03,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0x8b,0x03,0x00,0x00,0x5b,0x00,0x00,0x00, -0x73,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0x8c,0x03,0x00,0x00, -0x8b,0x03,0x00,0x00,0x3d,0x00,0x04,0x00,0x4e,0x00,0x00,0x00, -0x8e,0x03,0x00,0x00,0x62,0x00,0x00,0x00,0x71,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x8f,0x03,0x00,0x00,0x8e,0x03,0x00,0x00, -0xc4,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0x90,0x03,0x00,0x00, -0x8f,0x03,0x00,0x00,0x48,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4e,0x00,0x00,0x00,0x92,0x03,0x00,0x00,0x67,0x00,0x00,0x00, -0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x93,0x03,0x00,0x00, -0x92,0x03,0x00,0x00,0xc5,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0x94,0x03,0x00,0x00,0x90,0x03,0x00,0x00,0x93,0x03,0x00,0x00, -0xc2,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0x95,0x03,0x00,0x00, -0x94,0x03,0x00,0x00,0x77,0x00,0x00,0x00,0xc4,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0x96,0x03,0x00,0x00,0x95,0x03,0x00,0x00, -0x71,0x00,0x00,0x00,0xc7,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0x97,0x03,0x00,0x00,0x96,0x03,0x00,0x00,0x52,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x98,0x03,0x00,0x00, -0x97,0x03,0x00,0x00,0xc2,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0x9a,0x03,0x00,0x00,0x94,0x03,0x00,0x00,0x90,0x04,0x00,0x00, -0xc7,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0x9b,0x03,0x00,0x00, -0x9a,0x03,0x00,0x00,0x52,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x9c,0x03,0x00,0x00,0x9b,0x03,0x00,0x00, -0x41,0x00,0x08,0x00,0x80,0x00,0x00,0x00,0x9e,0x03,0x00,0x00, -0x58,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x37,0x00,0x00,0x00,0x77,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x51,0x00,0x00,0x00,0x9f,0x03,0x00,0x00,0x9e,0x03,0x00,0x00, -0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0xa0,0x03,0x00,0x00, -0x9f,0x03,0x00,0x00,0xc7,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0xa1,0x03,0x00,0x00,0xa0,0x03,0x00,0x00,0x88,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0xa3,0x03,0x00,0x00, -0x98,0x03,0x00,0x00,0xc5,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0xa4,0x03,0x00,0x00,0xa1,0x03,0x00,0x00,0xa3,0x03,0x00,0x00, -0x70,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0xa5,0x03,0x00,0x00, -0xa4,0x03,0x00,0x00,0xc2,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0xa6,0x03,0x00,0x00,0xa0,0x03,0x00,0x00,0x71,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0xa8,0x03,0x00,0x00, -0x9c,0x03,0x00,0x00,0xc5,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0xa9,0x03,0x00,0x00,0xa6,0x03,0x00,0x00,0xa8,0x03,0x00,0x00, -0x70,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0xaa,0x03,0x00,0x00, -0xa9,0x03,0x00,0x00,0x50,0x00,0x05,0x00,0x84,0x00,0x00,0x00, -0xab,0x03,0x00,0x00,0xa5,0x03,0x00,0x00,0xaa,0x03,0x00,0x00, -0x83,0x00,0x05,0x00,0x84,0x00,0x00,0x00,0xac,0x03,0x00,0x00, -0xab,0x03,0x00,0x00,0xcf,0x00,0x00,0x00,0x8e,0x00,0x05,0x00, -0x84,0x00,0x00,0x00,0xad,0x03,0x00,0x00,0xac,0x03,0x00,0x00, -0x8c,0x03,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xb3,0x03,0x00,0x00,0xa9,0x00,0x00,0x00,0x77,0x00,0x00,0x00, -0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00,0xb5,0x03,0x00,0x00, -0xad,0x03,0x00,0x00,0x00,0x00,0x00,0x00,0x73,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0xb6,0x03,0x00,0x00,0xb5,0x03,0x00,0x00, -0x41,0x00,0x06,0x00,0x5a,0x00,0x00,0x00,0xb7,0x03,0x00,0x00, -0xa1,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0xb3,0x03,0x00,0x00, -0x3e,0x00,0x03,0x00,0xb7,0x03,0x00,0x00,0xb6,0x03,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xbe,0x03,0x00,0x00, -0xa9,0x00,0x00,0x00,0x97,0x04,0x00,0x00,0x51,0x00,0x05,0x00, -0x4a,0x00,0x00,0x00,0xbf,0x03,0x00,0x00,0xad,0x03,0x00,0x00, -0x01,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0xc0,0x03,0x00,0x00,0xbf,0x03,0x00,0x00,0x41,0x00,0x06,0x00, -0x5a,0x00,0x00,0x00,0xc1,0x03,0x00,0x00,0xa1,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0xbe,0x03,0x00,0x00,0x3e,0x00,0x03,0x00, -0xc1,0x03,0x00,0x00,0xc0,0x03,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0xca,0x03,0x00,0x00,0x5b,0x00,0x00,0x00, -0x73,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0xcb,0x03,0x00,0x00, -0xca,0x03,0x00,0x00,0x3d,0x00,0x04,0x00,0x4e,0x00,0x00,0x00, -0xcd,0x03,0x00,0x00,0x62,0x00,0x00,0x00,0x71,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0xce,0x03,0x00,0x00,0xcd,0x03,0x00,0x00, -0xc4,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0xcf,0x03,0x00,0x00, -0xce,0x03,0x00,0x00,0x48,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4e,0x00,0x00,0x00,0xd1,0x03,0x00,0x00,0x67,0x00,0x00,0x00, -0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0xd2,0x03,0x00,0x00, -0xd1,0x03,0x00,0x00,0xc5,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0xd3,0x03,0x00,0x00,0xcf,0x03,0x00,0x00,0xd2,0x03,0x00,0x00, -0xc2,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0xd4,0x03,0x00,0x00, -0xd3,0x03,0x00,0x00,0x82,0x04,0x00,0x00,0xc4,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0xd5,0x03,0x00,0x00,0xd4,0x03,0x00,0x00, -0x71,0x00,0x00,0x00,0xc7,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0xd6,0x03,0x00,0x00,0xd5,0x03,0x00,0x00,0x52,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xd7,0x03,0x00,0x00, -0xd6,0x03,0x00,0x00,0xc2,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0xd9,0x03,0x00,0x00,0xd3,0x03,0x00,0x00,0x92,0x04,0x00,0x00, -0xc7,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0xda,0x03,0x00,0x00, -0xd9,0x03,0x00,0x00,0x52,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xdb,0x03,0x00,0x00,0xda,0x03,0x00,0x00, -0x41,0x00,0x08,0x00,0x80,0x00,0x00,0x00,0xdd,0x03,0x00,0x00, -0x58,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x37,0x00,0x00,0x00,0x82,0x04,0x00,0x00,0x3d,0x00,0x04,0x00, -0x51,0x00,0x00,0x00,0xde,0x03,0x00,0x00,0xdd,0x03,0x00,0x00, -0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0xdf,0x03,0x00,0x00, -0xde,0x03,0x00,0x00,0xc7,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0xe0,0x03,0x00,0x00,0xdf,0x03,0x00,0x00,0x88,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0xe2,0x03,0x00,0x00, -0xd7,0x03,0x00,0x00,0xc5,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0xe3,0x03,0x00,0x00,0xe0,0x03,0x00,0x00,0xe2,0x03,0x00,0x00, -0x70,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0xe4,0x03,0x00,0x00, -0xe3,0x03,0x00,0x00,0xc2,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0xe5,0x03,0x00,0x00,0xdf,0x03,0x00,0x00,0x71,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0xe7,0x03,0x00,0x00, -0xdb,0x03,0x00,0x00,0xc5,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0xe8,0x03,0x00,0x00,0xe5,0x03,0x00,0x00,0xe7,0x03,0x00,0x00, -0x70,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0xe9,0x03,0x00,0x00, -0xe8,0x03,0x00,0x00,0x50,0x00,0x05,0x00,0x84,0x00,0x00,0x00, -0xea,0x03,0x00,0x00,0xe4,0x03,0x00,0x00,0xe9,0x03,0x00,0x00, -0x83,0x00,0x05,0x00,0x84,0x00,0x00,0x00,0xeb,0x03,0x00,0x00, -0xea,0x03,0x00,0x00,0xcf,0x00,0x00,0x00,0x8e,0x00,0x05,0x00, -0x84,0x00,0x00,0x00,0xec,0x03,0x00,0x00,0xeb,0x03,0x00,0x00, -0xcb,0x03,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xf2,0x03,0x00,0x00,0xa9,0x00,0x00,0x00,0x82,0x04,0x00,0x00, -0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00,0xf4,0x03,0x00,0x00, -0xec,0x03,0x00,0x00,0x00,0x00,0x00,0x00,0x73,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0xf5,0x03,0x00,0x00,0xf4,0x03,0x00,0x00, -0x41,0x00,0x06,0x00,0x5a,0x00,0x00,0x00,0xf6,0x03,0x00,0x00, -0xa1,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0xf2,0x03,0x00,0x00, -0x3e,0x00,0x03,0x00,0xf6,0x03,0x00,0x00,0xf5,0x03,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xfd,0x03,0x00,0x00, -0xa9,0x00,0x00,0x00,0x98,0x04,0x00,0x00,0x51,0x00,0x05,0x00, -0x4a,0x00,0x00,0x00,0xfe,0x03,0x00,0x00,0xec,0x03,0x00,0x00, -0x01,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0xff,0x03,0x00,0x00,0xfe,0x03,0x00,0x00,0x41,0x00,0x06,0x00, -0x5a,0x00,0x00,0x00,0x00,0x04,0x00,0x00,0xa1,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0xfd,0x03,0x00,0x00,0x3e,0x00,0x03,0x00, -0x00,0x04,0x00,0x00,0xff,0x03,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0x09,0x04,0x00,0x00,0x5b,0x00,0x00,0x00, -0x73,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0x0a,0x04,0x00,0x00, -0x09,0x04,0x00,0x00,0x3d,0x00,0x04,0x00,0x4e,0x00,0x00,0x00, -0x0c,0x04,0x00,0x00,0x62,0x00,0x00,0x00,0x71,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x0d,0x04,0x00,0x00,0x0c,0x04,0x00,0x00, -0xc4,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0x0e,0x04,0x00,0x00, -0x0d,0x04,0x00,0x00,0x48,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4e,0x00,0x00,0x00,0x10,0x04,0x00,0x00,0x67,0x00,0x00,0x00, -0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x11,0x04,0x00,0x00, -0x10,0x04,0x00,0x00,0xc5,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0x12,0x04,0x00,0x00,0x0e,0x04,0x00,0x00,0x11,0x04,0x00,0x00, -0xc2,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0x13,0x04,0x00,0x00, -0x12,0x04,0x00,0x00,0x84,0x04,0x00,0x00,0xc4,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0x14,0x04,0x00,0x00,0x13,0x04,0x00,0x00, -0x71,0x00,0x00,0x00,0xc7,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0x15,0x04,0x00,0x00,0x14,0x04,0x00,0x00,0x52,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x16,0x04,0x00,0x00, -0x15,0x04,0x00,0x00,0xc2,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0x18,0x04,0x00,0x00,0x12,0x04,0x00,0x00,0x94,0x04,0x00,0x00, -0xc7,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0x19,0x04,0x00,0x00, -0x18,0x04,0x00,0x00,0x52,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x1a,0x04,0x00,0x00,0x19,0x04,0x00,0x00, -0x41,0x00,0x08,0x00,0x80,0x00,0x00,0x00,0x1c,0x04,0x00,0x00, -0x58,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x37,0x00,0x00,0x00,0x84,0x04,0x00,0x00,0x3d,0x00,0x04,0x00, -0x51,0x00,0x00,0x00,0x1d,0x04,0x00,0x00,0x1c,0x04,0x00,0x00, -0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x1e,0x04,0x00,0x00, -0x1d,0x04,0x00,0x00,0xc7,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0x1f,0x04,0x00,0x00,0x1e,0x04,0x00,0x00,0x88,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x21,0x04,0x00,0x00, -0x16,0x04,0x00,0x00,0xc5,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0x22,0x04,0x00,0x00,0x1f,0x04,0x00,0x00,0x21,0x04,0x00,0x00, -0x70,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0x23,0x04,0x00,0x00, -0x22,0x04,0x00,0x00,0xc2,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0x24,0x04,0x00,0x00,0x1e,0x04,0x00,0x00,0x71,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x26,0x04,0x00,0x00, -0x1a,0x04,0x00,0x00,0xc5,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0x27,0x04,0x00,0x00,0x24,0x04,0x00,0x00,0x26,0x04,0x00,0x00, -0x70,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0x28,0x04,0x00,0x00, -0x27,0x04,0x00,0x00,0x50,0x00,0x05,0x00,0x84,0x00,0x00,0x00, -0x29,0x04,0x00,0x00,0x23,0x04,0x00,0x00,0x28,0x04,0x00,0x00, -0x83,0x00,0x05,0x00,0x84,0x00,0x00,0x00,0x2a,0x04,0x00,0x00, -0x29,0x04,0x00,0x00,0xcf,0x00,0x00,0x00,0x8e,0x00,0x05,0x00, -0x84,0x00,0x00,0x00,0x2b,0x04,0x00,0x00,0x2a,0x04,0x00,0x00, -0x0a,0x04,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x31,0x04,0x00,0x00,0xa9,0x00,0x00,0x00,0x84,0x04,0x00,0x00, -0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00,0x33,0x04,0x00,0x00, -0x2b,0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x73,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0x34,0x04,0x00,0x00,0x33,0x04,0x00,0x00, -0x41,0x00,0x06,0x00,0x5a,0x00,0x00,0x00,0x35,0x04,0x00,0x00, -0xa1,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x31,0x04,0x00,0x00, -0x3e,0x00,0x03,0x00,0x35,0x04,0x00,0x00,0x34,0x04,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3c,0x04,0x00,0x00, -0xa9,0x00,0x00,0x00,0x99,0x04,0x00,0x00,0x51,0x00,0x05,0x00, -0x4a,0x00,0x00,0x00,0x3d,0x04,0x00,0x00,0x2b,0x04,0x00,0x00, -0x01,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0x3e,0x04,0x00,0x00,0x3d,0x04,0x00,0x00,0x41,0x00,0x06,0x00, -0x5a,0x00,0x00,0x00,0x3f,0x04,0x00,0x00,0xa1,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x3c,0x04,0x00,0x00,0x3e,0x00,0x03,0x00, -0x3f,0x04,0x00,0x00,0x3e,0x04,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0x48,0x04,0x00,0x00,0x5b,0x00,0x00,0x00, -0x73,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0x49,0x04,0x00,0x00, -0x48,0x04,0x00,0x00,0x3d,0x00,0x04,0x00,0x4e,0x00,0x00,0x00, -0x4b,0x04,0x00,0x00,0x62,0x00,0x00,0x00,0x71,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x4c,0x04,0x00,0x00,0x4b,0x04,0x00,0x00, -0xc4,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0x4d,0x04,0x00,0x00, -0x4c,0x04,0x00,0x00,0x48,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4e,0x00,0x00,0x00,0x4f,0x04,0x00,0x00,0x67,0x00,0x00,0x00, -0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x50,0x04,0x00,0x00, -0x4f,0x04,0x00,0x00,0xc5,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0x51,0x04,0x00,0x00,0x4d,0x04,0x00,0x00,0x50,0x04,0x00,0x00, -0xc2,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0x52,0x04,0x00,0x00, -0x51,0x04,0x00,0x00,0x86,0x04,0x00,0x00,0xc4,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0x53,0x04,0x00,0x00,0x52,0x04,0x00,0x00, -0x71,0x00,0x00,0x00,0xc7,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0x54,0x04,0x00,0x00,0x53,0x04,0x00,0x00,0x52,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x55,0x04,0x00,0x00, -0x54,0x04,0x00,0x00,0xc2,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0x57,0x04,0x00,0x00,0x51,0x04,0x00,0x00,0x96,0x04,0x00,0x00, -0xc7,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0x58,0x04,0x00,0x00, -0x57,0x04,0x00,0x00,0x52,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x59,0x04,0x00,0x00,0x58,0x04,0x00,0x00, -0x41,0x00,0x08,0x00,0x80,0x00,0x00,0x00,0x5b,0x04,0x00,0x00, -0x58,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x37,0x00,0x00,0x00,0x86,0x04,0x00,0x00,0x3d,0x00,0x04,0x00, -0x51,0x00,0x00,0x00,0x5c,0x04,0x00,0x00,0x5b,0x04,0x00,0x00, -0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x5d,0x04,0x00,0x00, -0x5c,0x04,0x00,0x00,0xc7,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0x5e,0x04,0x00,0x00,0x5d,0x04,0x00,0x00,0x88,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x60,0x04,0x00,0x00, -0x55,0x04,0x00,0x00,0xc5,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0x61,0x04,0x00,0x00,0x5e,0x04,0x00,0x00,0x60,0x04,0x00,0x00, -0x70,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0x62,0x04,0x00,0x00, -0x61,0x04,0x00,0x00,0xc2,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0x63,0x04,0x00,0x00,0x5d,0x04,0x00,0x00,0x71,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x65,0x04,0x00,0x00, -0x59,0x04,0x00,0x00,0xc5,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0x66,0x04,0x00,0x00,0x63,0x04,0x00,0x00,0x65,0x04,0x00,0x00, -0x70,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0x67,0x04,0x00,0x00, -0x66,0x04,0x00,0x00,0x50,0x00,0x05,0x00,0x84,0x00,0x00,0x00, -0x68,0x04,0x00,0x00,0x62,0x04,0x00,0x00,0x67,0x04,0x00,0x00, -0x83,0x00,0x05,0x00,0x84,0x00,0x00,0x00,0x69,0x04,0x00,0x00, -0x68,0x04,0x00,0x00,0xcf,0x00,0x00,0x00,0x8e,0x00,0x05,0x00, -0x84,0x00,0x00,0x00,0x6a,0x04,0x00,0x00,0x69,0x04,0x00,0x00, -0x49,0x04,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x70,0x04,0x00,0x00,0xa9,0x00,0x00,0x00,0x86,0x04,0x00,0x00, -0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00,0x72,0x04,0x00,0x00, -0x6a,0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x73,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0x73,0x04,0x00,0x00,0x72,0x04,0x00,0x00, -0x41,0x00,0x06,0x00,0x5a,0x00,0x00,0x00,0x74,0x04,0x00,0x00, -0xa1,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x70,0x04,0x00,0x00, -0x3e,0x00,0x03,0x00,0x74,0x04,0x00,0x00,0x73,0x04,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x7b,0x04,0x00,0x00, -0xa9,0x00,0x00,0x00,0x9a,0x04,0x00,0x00,0x51,0x00,0x05,0x00, -0x4a,0x00,0x00,0x00,0x7c,0x04,0x00,0x00,0x6a,0x04,0x00,0x00, -0x01,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0x7d,0x04,0x00,0x00,0x7c,0x04,0x00,0x00,0x41,0x00,0x06,0x00, -0x5a,0x00,0x00,0x00,0x7e,0x04,0x00,0x00,0xa1,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x7b,0x04,0x00,0x00,0x3e,0x00,0x03,0x00, -0x7e,0x04,0x00,0x00,0x7d,0x04,0x00,0x00,0xf9,0x00,0x02,0x00, -0xc3,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xc3,0x00,0x00,0x00, +0x1f,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x41,0x00,0x00,0x00,0x3d,0x00,0x00,0x00,0x40,0x00,0x00,0x00, +0x41,0x00,0x07,0x00,0x53,0x00,0x00,0x00,0x54,0x00,0x00,0x00, +0x50,0x00,0x00,0x00,0x51,0x00,0x00,0x00,0x27,0x00,0x00,0x00, +0x51,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x45,0x00,0x00,0x00, +0x55,0x00,0x00,0x00,0x54,0x00,0x00,0x00,0x73,0x00,0x04,0x00, +0x42,0x00,0x00,0x00,0x56,0x00,0x00,0x00,0x55,0x00,0x00,0x00, +0x41,0x00,0x08,0x00,0x5a,0x00,0x00,0x00,0x5b,0x00,0x00,0x00, +0x50,0x00,0x00,0x00,0x51,0x00,0x00,0x00,0x27,0x00,0x00,0x00, +0x59,0x00,0x00,0x00,0x59,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x46,0x00,0x00,0x00,0x5c,0x00,0x00,0x00,0x5b,0x00,0x00,0x00, +0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x5d,0x00,0x00,0x00, +0x5c,0x00,0x00,0x00,0xc4,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x5f,0x00,0x00,0x00,0x5d,0x00,0x00,0x00,0x5e,0x00,0x00,0x00, +0x41,0x00,0x08,0x00,0x5a,0x00,0x00,0x00,0x61,0x00,0x00,0x00, +0x50,0x00,0x00,0x00,0x51,0x00,0x00,0x00,0x27,0x00,0x00,0x00, +0x59,0x00,0x00,0x00,0x51,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x46,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x61,0x00,0x00,0x00, +0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x63,0x00,0x00,0x00, +0x62,0x00,0x00,0x00,0xc5,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x64,0x00,0x00,0x00,0x5f,0x00,0x00,0x00,0x63,0x00,0x00,0x00, +0x41,0x00,0x08,0x00,0x78,0x00,0x00,0x00,0x79,0x00,0x00,0x00, +0x50,0x00,0x00,0x00,0x51,0x00,0x00,0x00,0x27,0x00,0x00,0x00, +0x76,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x49,0x00,0x00,0x00,0x7a,0x00,0x00,0x00,0x79,0x00,0x00,0x00, +0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x7b,0x00,0x00,0x00, +0x7a,0x00,0x00,0x00,0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x87,0x00,0x00,0x00,0x7b,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0xc2,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8a,0x00,0x00,0x00, +0x64,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0xc4,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x8b,0x00,0x00,0x00,0x8a,0x00,0x00,0x00, +0x2d,0x00,0x00,0x00,0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x8c,0x00,0x00,0x00,0x8b,0x00,0x00,0x00,0x4a,0x00,0x00,0x00, +0xc5,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8d,0x00,0x00,0x00, +0x87,0x00,0x00,0x00,0x8c,0x00,0x00,0x00,0x70,0x00,0x04,0x00, +0x42,0x00,0x00,0x00,0x8e,0x00,0x00,0x00,0x8d,0x00,0x00,0x00, +0x83,0x00,0x05,0x00,0x42,0x00,0x00,0x00,0x90,0x00,0x00,0x00, +0x8e,0x00,0x00,0x00,0x8f,0x00,0x00,0x00,0x85,0x00,0x05,0x00, +0x42,0x00,0x00,0x00,0x91,0x00,0x00,0x00,0x56,0x00,0x00,0x00, +0x90,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x45,0x00,0x00,0x00, +0x92,0x00,0x00,0x00,0x91,0x00,0x00,0x00,0x41,0x00,0x06,0x00, +0x53,0x00,0x00,0x00,0x93,0x00,0x00,0x00,0x7f,0x00,0x00,0x00, +0x51,0x00,0x00,0x00,0x41,0x00,0x00,0x00,0x3e,0x00,0x03,0x00, +0x93,0x00,0x00,0x00,0x92,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x97,0x00,0x00,0x00,0x41,0x00,0x00,0x00, +0x4a,0x00,0x00,0x00,0xc2,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x9a,0x00,0x00,0x00,0x7b,0x00,0x00,0x00,0x2d,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9e,0x00,0x00,0x00, +0x40,0x00,0x00,0x00,0x9d,0x00,0x00,0x00,0xc2,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x9f,0x00,0x00,0x00,0x64,0x00,0x00,0x00, +0x9e,0x00,0x00,0x00,0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xa0,0x00,0x00,0x00,0x9f,0x00,0x00,0x00,0x4a,0x00,0x00,0x00, +0xc5,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa1,0x00,0x00,0x00, +0x9a,0x00,0x00,0x00,0xa0,0x00,0x00,0x00,0x70,0x00,0x04,0x00, +0x42,0x00,0x00,0x00,0xa2,0x00,0x00,0x00,0xa1,0x00,0x00,0x00, +0x83,0x00,0x05,0x00,0x42,0x00,0x00,0x00,0xa3,0x00,0x00,0x00, +0xa2,0x00,0x00,0x00,0x8f,0x00,0x00,0x00,0x85,0x00,0x05,0x00, +0x42,0x00,0x00,0x00,0xa4,0x00,0x00,0x00,0x56,0x00,0x00,0x00, +0xa3,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x45,0x00,0x00,0x00, +0xa5,0x00,0x00,0x00,0xa4,0x00,0x00,0x00,0x41,0x00,0x06,0x00, +0x53,0x00,0x00,0x00,0xa6,0x00,0x00,0x00,0x7f,0x00,0x00,0x00, +0x51,0x00,0x00,0x00,0x97,0x00,0x00,0x00,0x3e,0x00,0x03,0x00, +0xa6,0x00,0x00,0x00,0xa5,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xb9,0x00,0x00,0x00,0x40,0x00,0x00,0x00, +0xaa,0x00,0x00,0x00,0x41,0x00,0x08,0x00,0x78,0x00,0x00,0x00, +0xba,0x00,0x00,0x00,0x50,0x00,0x00,0x00,0x51,0x00,0x00,0x00, +0x27,0x00,0x00,0x00,0x76,0x00,0x00,0x00,0xb9,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x49,0x00,0x00,0x00,0xbb,0x00,0x00,0x00, +0xba,0x00,0x00,0x00,0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xbc,0x00,0x00,0x00,0xbb,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xbd,0x00,0x00,0x00,0x41,0x00,0x00,0x00, +0xaa,0x00,0x00,0x00,0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xbf,0x00,0x00,0x00,0xbc,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0xc2,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xc0,0x00,0x00,0x00, +0x64,0x00,0x00,0x00,0xb9,0x00,0x00,0x00,0xc4,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xc1,0x00,0x00,0x00,0xc0,0x00,0x00,0x00, +0x2d,0x00,0x00,0x00,0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xc2,0x00,0x00,0x00,0xc1,0x00,0x00,0x00,0x4a,0x00,0x00,0x00, +0xc5,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xc3,0x00,0x00,0x00, +0xbf,0x00,0x00,0x00,0xc2,0x00,0x00,0x00,0x70,0x00,0x04,0x00, +0x42,0x00,0x00,0x00,0xc4,0x00,0x00,0x00,0xc3,0x00,0x00,0x00, +0x83,0x00,0x05,0x00,0x42,0x00,0x00,0x00,0xc5,0x00,0x00,0x00, +0xc4,0x00,0x00,0x00,0x8f,0x00,0x00,0x00,0x85,0x00,0x05,0x00, +0x42,0x00,0x00,0x00,0xc6,0x00,0x00,0x00,0x56,0x00,0x00,0x00, +0xc5,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x45,0x00,0x00,0x00, +0xc7,0x00,0x00,0x00,0xc6,0x00,0x00,0x00,0x41,0x00,0x06,0x00, +0x53,0x00,0x00,0x00,0xc8,0x00,0x00,0x00,0x7f,0x00,0x00,0x00, +0x51,0x00,0x00,0x00,0xbd,0x00,0x00,0x00,0x3e,0x00,0x03,0x00, +0xc8,0x00,0x00,0x00,0xc7,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xca,0x00,0x00,0x00,0x41,0x00,0x00,0x00, +0xaa,0x01,0x00,0x00,0xc2,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xcb,0x00,0x00,0x00,0xbc,0x00,0x00,0x00,0x2d,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xcc,0x00,0x00,0x00, +0x40,0x00,0x00,0x00,0xab,0x01,0x00,0x00,0xc2,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xcd,0x00,0x00,0x00,0x64,0x00,0x00,0x00, +0xcc,0x00,0x00,0x00,0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xce,0x00,0x00,0x00,0xcd,0x00,0x00,0x00,0x4a,0x00,0x00,0x00, +0xc5,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xcf,0x00,0x00,0x00, +0xcb,0x00,0x00,0x00,0xce,0x00,0x00,0x00,0x70,0x00,0x04,0x00, +0x42,0x00,0x00,0x00,0xd0,0x00,0x00,0x00,0xcf,0x00,0x00,0x00, +0x83,0x00,0x05,0x00,0x42,0x00,0x00,0x00,0xd1,0x00,0x00,0x00, +0xd0,0x00,0x00,0x00,0x8f,0x00,0x00,0x00,0x85,0x00,0x05,0x00, +0x42,0x00,0x00,0x00,0xd2,0x00,0x00,0x00,0x56,0x00,0x00,0x00, +0xd1,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x45,0x00,0x00,0x00, +0xd3,0x00,0x00,0x00,0xd2,0x00,0x00,0x00,0x41,0x00,0x06,0x00, +0x53,0x00,0x00,0x00,0xd4,0x00,0x00,0x00,0x7f,0x00,0x00,0x00, +0x51,0x00,0x00,0x00,0xca,0x00,0x00,0x00,0x3e,0x00,0x03,0x00, +0xd4,0x00,0x00,0x00,0xd3,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xdc,0x00,0x00,0x00,0x40,0x00,0x00,0x00, +0x47,0x00,0x00,0x00,0x41,0x00,0x08,0x00,0x78,0x00,0x00,0x00, +0xdd,0x00,0x00,0x00,0x50,0x00,0x00,0x00,0x51,0x00,0x00,0x00, +0x27,0x00,0x00,0x00,0x76,0x00,0x00,0x00,0xdc,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x49,0x00,0x00,0x00,0xde,0x00,0x00,0x00, +0xdd,0x00,0x00,0x00,0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xdf,0x00,0x00,0x00,0xde,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xe0,0x00,0x00,0x00,0x41,0x00,0x00,0x00, +0x47,0x00,0x00,0x00,0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xe2,0x00,0x00,0x00,0xdf,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0xc2,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xe3,0x00,0x00,0x00, +0x64,0x00,0x00,0x00,0xdc,0x00,0x00,0x00,0xc4,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xe4,0x00,0x00,0x00,0xe3,0x00,0x00,0x00, +0x2d,0x00,0x00,0x00,0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xe5,0x00,0x00,0x00,0xe4,0x00,0x00,0x00,0x4a,0x00,0x00,0x00, +0xc5,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xe6,0x00,0x00,0x00, +0xe2,0x00,0x00,0x00,0xe5,0x00,0x00,0x00,0x70,0x00,0x04,0x00, +0x42,0x00,0x00,0x00,0xe7,0x00,0x00,0x00,0xe6,0x00,0x00,0x00, +0x83,0x00,0x05,0x00,0x42,0x00,0x00,0x00,0xe8,0x00,0x00,0x00, +0xe7,0x00,0x00,0x00,0x8f,0x00,0x00,0x00,0x85,0x00,0x05,0x00, +0x42,0x00,0x00,0x00,0xe9,0x00,0x00,0x00,0x56,0x00,0x00,0x00, +0xe8,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x45,0x00,0x00,0x00, +0xea,0x00,0x00,0x00,0xe9,0x00,0x00,0x00,0x41,0x00,0x06,0x00, +0x53,0x00,0x00,0x00,0xeb,0x00,0x00,0x00,0x7f,0x00,0x00,0x00, +0x51,0x00,0x00,0x00,0xe0,0x00,0x00,0x00,0x3e,0x00,0x03,0x00, +0xeb,0x00,0x00,0x00,0xea,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xed,0x00,0x00,0x00,0x41,0x00,0x00,0x00, +0xac,0x01,0x00,0x00,0xc2,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xee,0x00,0x00,0x00,0xdf,0x00,0x00,0x00,0x2d,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xef,0x00,0x00,0x00, +0x40,0x00,0x00,0x00,0xad,0x01,0x00,0x00,0xc2,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xf0,0x00,0x00,0x00,0x64,0x00,0x00,0x00, +0xef,0x00,0x00,0x00,0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xf1,0x00,0x00,0x00,0xf0,0x00,0x00,0x00,0x4a,0x00,0x00,0x00, +0xc5,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xf2,0x00,0x00,0x00, +0xee,0x00,0x00,0x00,0xf1,0x00,0x00,0x00,0x70,0x00,0x04,0x00, +0x42,0x00,0x00,0x00,0xf3,0x00,0x00,0x00,0xf2,0x00,0x00,0x00, +0x83,0x00,0x05,0x00,0x42,0x00,0x00,0x00,0xf4,0x00,0x00,0x00, +0xf3,0x00,0x00,0x00,0x8f,0x00,0x00,0x00,0x85,0x00,0x05,0x00, +0x42,0x00,0x00,0x00,0xf5,0x00,0x00,0x00,0x56,0x00,0x00,0x00, +0xf4,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x45,0x00,0x00,0x00, +0xf6,0x00,0x00,0x00,0xf5,0x00,0x00,0x00,0x41,0x00,0x06,0x00, +0x53,0x00,0x00,0x00,0xf7,0x00,0x00,0x00,0x7f,0x00,0x00,0x00, +0x51,0x00,0x00,0x00,0xed,0x00,0x00,0x00,0x3e,0x00,0x03,0x00, +0xf7,0x00,0x00,0x00,0xf6,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xff,0x00,0x00,0x00,0x40,0x00,0x00,0x00, +0xae,0x01,0x00,0x00,0x41,0x00,0x08,0x00,0x78,0x00,0x00,0x00, +0x00,0x01,0x00,0x00,0x50,0x00,0x00,0x00,0x51,0x00,0x00,0x00, +0x27,0x00,0x00,0x00,0x76,0x00,0x00,0x00,0xff,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x49,0x00,0x00,0x00,0x01,0x01,0x00,0x00, +0x00,0x01,0x00,0x00,0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x02,0x01,0x00,0x00,0x01,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x03,0x01,0x00,0x00,0x41,0x00,0x00,0x00, +0xae,0x01,0x00,0x00,0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x05,0x01,0x00,0x00,0x02,0x01,0x00,0x00,0x86,0x00,0x00,0x00, +0xc2,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x06,0x01,0x00,0x00, +0x64,0x00,0x00,0x00,0xff,0x00,0x00,0x00,0xc4,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x07,0x01,0x00,0x00,0x06,0x01,0x00,0x00, +0x2d,0x00,0x00,0x00,0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x08,0x01,0x00,0x00,0x07,0x01,0x00,0x00,0x4a,0x00,0x00,0x00, +0xc5,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x09,0x01,0x00,0x00, +0x05,0x01,0x00,0x00,0x08,0x01,0x00,0x00,0x70,0x00,0x04,0x00, +0x42,0x00,0x00,0x00,0x0a,0x01,0x00,0x00,0x09,0x01,0x00,0x00, +0x83,0x00,0x05,0x00,0x42,0x00,0x00,0x00,0x0b,0x01,0x00,0x00, +0x0a,0x01,0x00,0x00,0x8f,0x00,0x00,0x00,0x85,0x00,0x05,0x00, +0x42,0x00,0x00,0x00,0x0c,0x01,0x00,0x00,0x56,0x00,0x00,0x00, +0x0b,0x01,0x00,0x00,0x73,0x00,0x04,0x00,0x45,0x00,0x00,0x00, +0x0d,0x01,0x00,0x00,0x0c,0x01,0x00,0x00,0x41,0x00,0x06,0x00, +0x53,0x00,0x00,0x00,0x0e,0x01,0x00,0x00,0x7f,0x00,0x00,0x00, +0x51,0x00,0x00,0x00,0x03,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, +0x0e,0x01,0x00,0x00,0x0d,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x10,0x01,0x00,0x00,0x41,0x00,0x00,0x00, +0xaf,0x01,0x00,0x00,0xc2,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x11,0x01,0x00,0x00,0x02,0x01,0x00,0x00,0x2d,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x12,0x01,0x00,0x00, +0x40,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0xc2,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x13,0x01,0x00,0x00,0x64,0x00,0x00,0x00, +0x12,0x01,0x00,0x00,0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x14,0x01,0x00,0x00,0x13,0x01,0x00,0x00,0x4a,0x00,0x00,0x00, +0xc5,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x15,0x01,0x00,0x00, +0x11,0x01,0x00,0x00,0x14,0x01,0x00,0x00,0x70,0x00,0x04,0x00, +0x42,0x00,0x00,0x00,0x16,0x01,0x00,0x00,0x15,0x01,0x00,0x00, +0x83,0x00,0x05,0x00,0x42,0x00,0x00,0x00,0x17,0x01,0x00,0x00, +0x16,0x01,0x00,0x00,0x8f,0x00,0x00,0x00,0x85,0x00,0x05,0x00, +0x42,0x00,0x00,0x00,0x18,0x01,0x00,0x00,0x56,0x00,0x00,0x00, +0x17,0x01,0x00,0x00,0x73,0x00,0x04,0x00,0x45,0x00,0x00,0x00, +0x19,0x01,0x00,0x00,0x18,0x01,0x00,0x00,0x41,0x00,0x06,0x00, +0x53,0x00,0x00,0x00,0x1a,0x01,0x00,0x00,0x7f,0x00,0x00,0x00, +0x51,0x00,0x00,0x00,0x10,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, +0x1a,0x01,0x00,0x00,0x19,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x22,0x01,0x00,0x00,0x40,0x00,0x00,0x00, +0x10,0x00,0x00,0x00,0x41,0x00,0x08,0x00,0x78,0x00,0x00,0x00, +0x23,0x01,0x00,0x00,0x50,0x00,0x00,0x00,0x51,0x00,0x00,0x00, +0x27,0x00,0x00,0x00,0x76,0x00,0x00,0x00,0x22,0x01,0x00,0x00, +0x3d,0x00,0x04,0x00,0x49,0x00,0x00,0x00,0x24,0x01,0x00,0x00, +0x23,0x01,0x00,0x00,0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x25,0x01,0x00,0x00,0x24,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x26,0x01,0x00,0x00,0x41,0x00,0x00,0x00, +0x10,0x00,0x00,0x00,0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x28,0x01,0x00,0x00,0x25,0x01,0x00,0x00,0x86,0x00,0x00,0x00, +0xc2,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x29,0x01,0x00,0x00, +0x64,0x00,0x00,0x00,0x22,0x01,0x00,0x00,0xc4,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x2a,0x01,0x00,0x00,0x29,0x01,0x00,0x00, +0x2d,0x00,0x00,0x00,0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x2b,0x01,0x00,0x00,0x2a,0x01,0x00,0x00,0x4a,0x00,0x00,0x00, +0xc5,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x2c,0x01,0x00,0x00, +0x28,0x01,0x00,0x00,0x2b,0x01,0x00,0x00,0x70,0x00,0x04,0x00, +0x42,0x00,0x00,0x00,0x2d,0x01,0x00,0x00,0x2c,0x01,0x00,0x00, +0x83,0x00,0x05,0x00,0x42,0x00,0x00,0x00,0x2e,0x01,0x00,0x00, +0x2d,0x01,0x00,0x00,0x8f,0x00,0x00,0x00,0x85,0x00,0x05,0x00, +0x42,0x00,0x00,0x00,0x2f,0x01,0x00,0x00,0x56,0x00,0x00,0x00, +0x2e,0x01,0x00,0x00,0x73,0x00,0x04,0x00,0x45,0x00,0x00,0x00, +0x30,0x01,0x00,0x00,0x2f,0x01,0x00,0x00,0x41,0x00,0x06,0x00, +0x53,0x00,0x00,0x00,0x31,0x01,0x00,0x00,0x7f,0x00,0x00,0x00, +0x51,0x00,0x00,0x00,0x26,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, +0x31,0x01,0x00,0x00,0x30,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x33,0x01,0x00,0x00,0x41,0x00,0x00,0x00, +0xb0,0x01,0x00,0x00,0xc2,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x34,0x01,0x00,0x00,0x25,0x01,0x00,0x00,0x2d,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x35,0x01,0x00,0x00, +0x40,0x00,0x00,0x00,0x4a,0x00,0x00,0x00,0xc2,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x36,0x01,0x00,0x00,0x64,0x00,0x00,0x00, +0x35,0x01,0x00,0x00,0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x37,0x01,0x00,0x00,0x36,0x01,0x00,0x00,0x4a,0x00,0x00,0x00, +0xc5,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x38,0x01,0x00,0x00, +0x34,0x01,0x00,0x00,0x37,0x01,0x00,0x00,0x70,0x00,0x04,0x00, +0x42,0x00,0x00,0x00,0x39,0x01,0x00,0x00,0x38,0x01,0x00,0x00, +0x83,0x00,0x05,0x00,0x42,0x00,0x00,0x00,0x3a,0x01,0x00,0x00, +0x39,0x01,0x00,0x00,0x8f,0x00,0x00,0x00,0x85,0x00,0x05,0x00, +0x42,0x00,0x00,0x00,0x3b,0x01,0x00,0x00,0x56,0x00,0x00,0x00, +0x3a,0x01,0x00,0x00,0x73,0x00,0x04,0x00,0x45,0x00,0x00,0x00, +0x3c,0x01,0x00,0x00,0x3b,0x01,0x00,0x00,0x41,0x00,0x06,0x00, +0x53,0x00,0x00,0x00,0x3d,0x01,0x00,0x00,0x7f,0x00,0x00,0x00, +0x51,0x00,0x00,0x00,0x33,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, +0x3d,0x01,0x00,0x00,0x3c,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x45,0x01,0x00,0x00,0x40,0x00,0x00,0x00, +0xb1,0x01,0x00,0x00,0x41,0x00,0x08,0x00,0x78,0x00,0x00,0x00, +0x46,0x01,0x00,0x00,0x50,0x00,0x00,0x00,0x51,0x00,0x00,0x00, +0x27,0x00,0x00,0x00,0x76,0x00,0x00,0x00,0x45,0x01,0x00,0x00, +0x3d,0x00,0x04,0x00,0x49,0x00,0x00,0x00,0x47,0x01,0x00,0x00, +0x46,0x01,0x00,0x00,0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x48,0x01,0x00,0x00,0x47,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x49,0x01,0x00,0x00,0x41,0x00,0x00,0x00, +0xb1,0x01,0x00,0x00,0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x4b,0x01,0x00,0x00,0x48,0x01,0x00,0x00,0x86,0x00,0x00,0x00, +0xc2,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x4c,0x01,0x00,0x00, +0x64,0x00,0x00,0x00,0x45,0x01,0x00,0x00,0xc4,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x4d,0x01,0x00,0x00,0x4c,0x01,0x00,0x00, +0x2d,0x00,0x00,0x00,0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x4e,0x01,0x00,0x00,0x4d,0x01,0x00,0x00,0x4a,0x00,0x00,0x00, +0xc5,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x4f,0x01,0x00,0x00, +0x4b,0x01,0x00,0x00,0x4e,0x01,0x00,0x00,0x70,0x00,0x04,0x00, +0x42,0x00,0x00,0x00,0x50,0x01,0x00,0x00,0x4f,0x01,0x00,0x00, +0x83,0x00,0x05,0x00,0x42,0x00,0x00,0x00,0x51,0x01,0x00,0x00, +0x50,0x01,0x00,0x00,0x8f,0x00,0x00,0x00,0x85,0x00,0x05,0x00, +0x42,0x00,0x00,0x00,0x52,0x01,0x00,0x00,0x56,0x00,0x00,0x00, +0x51,0x01,0x00,0x00,0x73,0x00,0x04,0x00,0x45,0x00,0x00,0x00, +0x53,0x01,0x00,0x00,0x52,0x01,0x00,0x00,0x41,0x00,0x06,0x00, +0x53,0x00,0x00,0x00,0x54,0x01,0x00,0x00,0x7f,0x00,0x00,0x00, +0x51,0x00,0x00,0x00,0x49,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, +0x54,0x01,0x00,0x00,0x53,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x56,0x01,0x00,0x00,0x41,0x00,0x00,0x00, +0xb2,0x01,0x00,0x00,0xc2,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x57,0x01,0x00,0x00,0x48,0x01,0x00,0x00,0x2d,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x58,0x01,0x00,0x00, +0x40,0x00,0x00,0x00,0xaa,0x01,0x00,0x00,0xc2,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x59,0x01,0x00,0x00,0x64,0x00,0x00,0x00, +0x58,0x01,0x00,0x00,0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x5a,0x01,0x00,0x00,0x59,0x01,0x00,0x00,0x4a,0x00,0x00,0x00, +0xc5,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x5b,0x01,0x00,0x00, +0x57,0x01,0x00,0x00,0x5a,0x01,0x00,0x00,0x70,0x00,0x04,0x00, +0x42,0x00,0x00,0x00,0x5c,0x01,0x00,0x00,0x5b,0x01,0x00,0x00, +0x83,0x00,0x05,0x00,0x42,0x00,0x00,0x00,0x5d,0x01,0x00,0x00, +0x5c,0x01,0x00,0x00,0x8f,0x00,0x00,0x00,0x85,0x00,0x05,0x00, +0x42,0x00,0x00,0x00,0x5e,0x01,0x00,0x00,0x56,0x00,0x00,0x00, +0x5d,0x01,0x00,0x00,0x73,0x00,0x04,0x00,0x45,0x00,0x00,0x00, +0x5f,0x01,0x00,0x00,0x5e,0x01,0x00,0x00,0x41,0x00,0x06,0x00, +0x53,0x00,0x00,0x00,0x60,0x01,0x00,0x00,0x7f,0x00,0x00,0x00, +0x51,0x00,0x00,0x00,0x56,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, +0x60,0x01,0x00,0x00,0x5f,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x68,0x01,0x00,0x00,0x40,0x00,0x00,0x00, +0xb3,0x01,0x00,0x00,0x41,0x00,0x08,0x00,0x78,0x00,0x00,0x00, +0x69,0x01,0x00,0x00,0x50,0x00,0x00,0x00,0x51,0x00,0x00,0x00, +0x27,0x00,0x00,0x00,0x76,0x00,0x00,0x00,0x68,0x01,0x00,0x00, +0x3d,0x00,0x04,0x00,0x49,0x00,0x00,0x00,0x6a,0x01,0x00,0x00, +0x69,0x01,0x00,0x00,0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x6b,0x01,0x00,0x00,0x6a,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x6c,0x01,0x00,0x00,0x41,0x00,0x00,0x00, +0xb3,0x01,0x00,0x00,0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x6e,0x01,0x00,0x00,0x6b,0x01,0x00,0x00,0x86,0x00,0x00,0x00, +0xc2,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x6f,0x01,0x00,0x00, +0x64,0x00,0x00,0x00,0x68,0x01,0x00,0x00,0xc4,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x70,0x01,0x00,0x00,0x6f,0x01,0x00,0x00, +0x2d,0x00,0x00,0x00,0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x71,0x01,0x00,0x00,0x70,0x01,0x00,0x00,0x4a,0x00,0x00,0x00, +0xc5,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x72,0x01,0x00,0x00, +0x6e,0x01,0x00,0x00,0x71,0x01,0x00,0x00,0x70,0x00,0x04,0x00, +0x42,0x00,0x00,0x00,0x73,0x01,0x00,0x00,0x72,0x01,0x00,0x00, +0x83,0x00,0x05,0x00,0x42,0x00,0x00,0x00,0x74,0x01,0x00,0x00, +0x73,0x01,0x00,0x00,0x8f,0x00,0x00,0x00,0x85,0x00,0x05,0x00, +0x42,0x00,0x00,0x00,0x75,0x01,0x00,0x00,0x56,0x00,0x00,0x00, +0x74,0x01,0x00,0x00,0x73,0x00,0x04,0x00,0x45,0x00,0x00,0x00, +0x76,0x01,0x00,0x00,0x75,0x01,0x00,0x00,0x41,0x00,0x06,0x00, +0x53,0x00,0x00,0x00,0x77,0x01,0x00,0x00,0x7f,0x00,0x00,0x00, +0x51,0x00,0x00,0x00,0x6c,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, +0x77,0x01,0x00,0x00,0x76,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x79,0x01,0x00,0x00,0x41,0x00,0x00,0x00, +0xb4,0x01,0x00,0x00,0xc2,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x7a,0x01,0x00,0x00,0x6b,0x01,0x00,0x00,0x2d,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x7b,0x01,0x00,0x00, +0x40,0x00,0x00,0x00,0xac,0x01,0x00,0x00,0xc2,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x7c,0x01,0x00,0x00,0x64,0x00,0x00,0x00, +0x7b,0x01,0x00,0x00,0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x7d,0x01,0x00,0x00,0x7c,0x01,0x00,0x00,0x4a,0x00,0x00,0x00, +0xc5,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x7e,0x01,0x00,0x00, +0x7a,0x01,0x00,0x00,0x7d,0x01,0x00,0x00,0x70,0x00,0x04,0x00, +0x42,0x00,0x00,0x00,0x7f,0x01,0x00,0x00,0x7e,0x01,0x00,0x00, +0x83,0x00,0x05,0x00,0x42,0x00,0x00,0x00,0x80,0x01,0x00,0x00, +0x7f,0x01,0x00,0x00,0x8f,0x00,0x00,0x00,0x85,0x00,0x05,0x00, +0x42,0x00,0x00,0x00,0x81,0x01,0x00,0x00,0x56,0x00,0x00,0x00, +0x80,0x01,0x00,0x00,0x73,0x00,0x04,0x00,0x45,0x00,0x00,0x00, +0x82,0x01,0x00,0x00,0x81,0x01,0x00,0x00,0x41,0x00,0x06,0x00, +0x53,0x00,0x00,0x00,0x83,0x01,0x00,0x00,0x7f,0x00,0x00,0x00, +0x51,0x00,0x00,0x00,0x79,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, +0x83,0x01,0x00,0x00,0x82,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x8b,0x01,0x00,0x00,0x40,0x00,0x00,0x00, +0xb5,0x01,0x00,0x00,0x41,0x00,0x08,0x00,0x78,0x00,0x00,0x00, +0x8c,0x01,0x00,0x00,0x50,0x00,0x00,0x00,0x51,0x00,0x00,0x00, +0x27,0x00,0x00,0x00,0x76,0x00,0x00,0x00,0x8b,0x01,0x00,0x00, +0x3d,0x00,0x04,0x00,0x49,0x00,0x00,0x00,0x8d,0x01,0x00,0x00, +0x8c,0x01,0x00,0x00,0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x8e,0x01,0x00,0x00,0x8d,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x8f,0x01,0x00,0x00,0x41,0x00,0x00,0x00, +0xb5,0x01,0x00,0x00,0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x91,0x01,0x00,0x00,0x8e,0x01,0x00,0x00,0x86,0x00,0x00,0x00, +0xc2,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x92,0x01,0x00,0x00, +0x64,0x00,0x00,0x00,0x8b,0x01,0x00,0x00,0xc4,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x93,0x01,0x00,0x00,0x92,0x01,0x00,0x00, +0x2d,0x00,0x00,0x00,0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x94,0x01,0x00,0x00,0x93,0x01,0x00,0x00,0x4a,0x00,0x00,0x00, +0xc5,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x95,0x01,0x00,0x00, +0x91,0x01,0x00,0x00,0x94,0x01,0x00,0x00,0x70,0x00,0x04,0x00, +0x42,0x00,0x00,0x00,0x96,0x01,0x00,0x00,0x95,0x01,0x00,0x00, +0x83,0x00,0x05,0x00,0x42,0x00,0x00,0x00,0x97,0x01,0x00,0x00, +0x96,0x01,0x00,0x00,0x8f,0x00,0x00,0x00,0x85,0x00,0x05,0x00, +0x42,0x00,0x00,0x00,0x98,0x01,0x00,0x00,0x56,0x00,0x00,0x00, +0x97,0x01,0x00,0x00,0x73,0x00,0x04,0x00,0x45,0x00,0x00,0x00, +0x99,0x01,0x00,0x00,0x98,0x01,0x00,0x00,0x41,0x00,0x06,0x00, +0x53,0x00,0x00,0x00,0x9a,0x01,0x00,0x00,0x7f,0x00,0x00,0x00, +0x51,0x00,0x00,0x00,0x8f,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, +0x9a,0x01,0x00,0x00,0x99,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x9c,0x01,0x00,0x00,0x41,0x00,0x00,0x00, +0xb6,0x01,0x00,0x00,0xc2,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x9d,0x01,0x00,0x00,0x8e,0x01,0x00,0x00,0x2d,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9e,0x01,0x00,0x00, +0x40,0x00,0x00,0x00,0xaf,0x01,0x00,0x00,0xc2,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x9f,0x01,0x00,0x00,0x64,0x00,0x00,0x00, +0x9e,0x01,0x00,0x00,0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xa0,0x01,0x00,0x00,0x9f,0x01,0x00,0x00,0x4a,0x00,0x00,0x00, +0xc5,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa1,0x01,0x00,0x00, +0x9d,0x01,0x00,0x00,0xa0,0x01,0x00,0x00,0x70,0x00,0x04,0x00, +0x42,0x00,0x00,0x00,0xa2,0x01,0x00,0x00,0xa1,0x01,0x00,0x00, +0x83,0x00,0x05,0x00,0x42,0x00,0x00,0x00,0xa3,0x01,0x00,0x00, +0xa2,0x01,0x00,0x00,0x8f,0x00,0x00,0x00,0x85,0x00,0x05,0x00, +0x42,0x00,0x00,0x00,0xa4,0x01,0x00,0x00,0x56,0x00,0x00,0x00, +0xa3,0x01,0x00,0x00,0x73,0x00,0x04,0x00,0x45,0x00,0x00,0x00, +0xa5,0x01,0x00,0x00,0xa4,0x01,0x00,0x00,0x41,0x00,0x06,0x00, +0x53,0x00,0x00,0x00,0xa6,0x01,0x00,0x00,0x7f,0x00,0x00,0x00, +0x51,0x00,0x00,0x00,0x9c,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, +0xa6,0x01,0x00,0x00,0xa5,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0xac,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xac,0x00,0x00,0x00, 0xfd,0x00,0x01,0x00,0x38,0x00,0x01,0x00, }; -const uint64_t dequant_q5_0_len = 13952; +const uint64_t dequant_q5_0_len = 6644; unsigned char dequant_q5_1_data[] = { 0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, -0x95,0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, +0xb4,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, 0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x51,0x11,0x00,0x00, 0x11,0x00,0x02,0x00,0x60,0x11,0x00,0x00,0x0b,0x00,0x06,0x00, 0x01,0x00,0x00,0x00,0x47,0x4c,0x53,0x4c,0x2e,0x73,0x74,0x64, 0x2e,0x34,0x35,0x30,0x00,0x00,0x00,0x00,0x0e,0x00,0x03,0x00, -0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x0f,0x00,0x09,0x00, +0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x0f,0x00,0x0a,0x00, 0x05,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x6d,0x61,0x69,0x6e, -0x00,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0x55,0x00,0x00,0x00,0x9d,0x00,0x00,0x00,0x10,0x00,0x06,0x00, -0x04,0x00,0x00,0x00,0x11,0x00,0x00,0x00,0x00,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x0c,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x1c,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x14,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x14,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x14,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x08,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x14,0x00,0x00,0x00,0x03,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x47,0x00,0x03,0x00, -0x14,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x50,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x51,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x51,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x51,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x51,0x00,0x00,0x00,0x03,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x52,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x18,0x00,0x00,0x00, -0x48,0x00,0x04,0x00,0x53,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x53,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0x2b,0x00,0x00,0x00,0x4d,0x00,0x00,0x00,0x7a,0x00,0x00,0x00, +0x10,0x00,0x06,0x00,0x04,0x00,0x00,0x00,0x11,0x00,0x00,0x00, +0x00,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x0b,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, +0x1a,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x12,0x00,0x00,0x00, +0x0b,0x00,0x00,0x00,0x1b,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x29,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x29,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x29,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x29,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x29,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x10,0x00,0x00,0x00, +0x47,0x00,0x03,0x00,0x29,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x48,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x49,0x00,0x00,0x00, 0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x03,0x00,0x53,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x55,0x00,0x00,0x00,0x22,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x55,0x00,0x00,0x00, -0x21,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x9a,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x48,0x00,0x04,0x00,0x9b,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x19,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x9b,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x03,0x00,0x9b,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x9d,0x00,0x00,0x00,0x22,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x9d,0x00,0x00,0x00, -0x21,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0xbd,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x19,0x00,0x00,0x00, -0x13,0x00,0x02,0x00,0x02,0x00,0x00,0x00,0x21,0x00,0x03,0x00, -0x03,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x15,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x15,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x17,0x00,0x04,0x00,0x0a,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x0b,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x0a,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x0b,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x0d,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x0e,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x1e,0x00,0x06,0x00,0x14,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x49,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x49,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x49,0x00,0x00,0x00, +0x03,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x08,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x18,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0x4b,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x4b,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x4b,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, +0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x4d,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x77,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0x78,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x78,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x78,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x7a,0x00,0x00,0x00, +0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x7a,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0xa7,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, +0x19,0x00,0x00,0x00,0x13,0x00,0x02,0x00,0x02,0x00,0x00,0x00, +0x21,0x00,0x03,0x00,0x03,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x15,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x17,0x00,0x04,0x00,0x09,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x0a,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x09,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x0d,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x10,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00, +0x12,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x15,0x00,0x00,0x00,0x40,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x1e,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0x1e,0x00,0x07,0x00,0x29,0x00,0x00,0x00, 0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x15,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x14,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x15,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x17,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x18,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x1b,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x14,0x00,0x02,0x00, -0x24,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x48,0x00,0x00,0x00, -0x10,0x00,0x00,0x00,0x16,0x00,0x03,0x00,0x4a,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x16,0x00,0x03,0x00,0x4d,0x00,0x00,0x00, -0x10,0x00,0x00,0x00,0x15,0x00,0x04,0x00,0x4e,0x00,0x00,0x00, -0x08,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x10,0x00,0x00,0x00, -0x1c,0x00,0x04,0x00,0x50,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, -0x4f,0x00,0x00,0x00,0x1e,0x00,0x06,0x00,0x51,0x00,0x00,0x00, -0x4d,0x00,0x00,0x00,0x4d,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x50,0x00,0x00,0x00,0x1d,0x00,0x03,0x00,0x52,0x00,0x00,0x00, -0x51,0x00,0x00,0x00,0x1e,0x00,0x03,0x00,0x53,0x00,0x00,0x00, -0x52,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x54,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x54,0x00,0x00,0x00,0x55,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x57,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x4d,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x64,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x69,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x71,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x2a,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x29,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0x2a,0x00,0x00,0x00,0x2b,0x00,0x00,0x00, +0x09,0x00,0x00,0x00,0x15,0x00,0x04,0x00,0x2c,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x2c,0x00,0x00,0x00,0x2d,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x2e,0x00,0x00,0x00,0x09,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x14,0x00,0x02,0x00,0x32,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x38,0x00,0x00,0x00, +0x00,0x04,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x3e,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x16,0x00,0x03,0x00, +0x42,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x16,0x00,0x03,0x00, +0x45,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x15,0x00,0x04,0x00, +0x46,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x47,0x00,0x00,0x00, +0x10,0x00,0x00,0x00,0x1c,0x00,0x04,0x00,0x48,0x00,0x00,0x00, +0x46,0x00,0x00,0x00,0x47,0x00,0x00,0x00,0x1e,0x00,0x06,0x00, +0x49,0x00,0x00,0x00,0x45,0x00,0x00,0x00,0x45,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x48,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, +0x4a,0x00,0x00,0x00,0x49,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, +0x4b,0x00,0x00,0x00,0x4a,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x4c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x4b,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0x4c,0x00,0x00,0x00,0x4d,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x2c,0x00,0x00,0x00, +0x4e,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x50,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x45,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x2c,0x00,0x00,0x00,0x56,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x2c,0x00,0x00,0x00, +0x5c,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x5d,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x2c,0x00,0x00,0x00,0x71,0x00,0x00,0x00, +0x03,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x73,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x46,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, +0x77,0x00,0x00,0x00,0x45,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, +0x78,0x00,0x00,0x00,0x77,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x79,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x78,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0x79,0x00,0x00,0x00,0x7a,0x00,0x00,0x00, 0x0c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x7a,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x7c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, -0x17,0x00,0x04,0x00,0x80,0x00,0x00,0x00,0x4a,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x8d,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x1d,0x00,0x03,0x00,0x9a,0x00,0x00,0x00,0x4d,0x00,0x00,0x00, -0x1e,0x00,0x03,0x00,0x9b,0x00,0x00,0x00,0x9a,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x9c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x9b,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x9c,0x00,0x00,0x00, -0x9d,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0xbc,0x00,0x00,0x00,0x00,0x01,0x00,0x00, -0x2c,0x00,0x06,0x00,0x0a,0x00,0x00,0x00,0xbd,0x00,0x00,0x00, -0xbc,0x00,0x00,0x00,0x8d,0x00,0x00,0x00,0x8d,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x7c,0x04,0x00,0x00, +0x81,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x98,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xa5,0x00,0x00,0x00, +0x00,0x01,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xa6,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x2c,0x00,0x06,0x00, +0x09,0x00,0x00,0x00,0xa7,0x00,0x00,0x00,0xa5,0x00,0x00,0x00, +0xa6,0x00,0x00,0x00,0xa6,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xa6,0x01,0x00,0x00,0x11,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xa7,0x01,0x00,0x00, 0x0d,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x7d,0x04,0x00,0x00,0x11,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x7e,0x04,0x00,0x00,0x0e,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x7f,0x04,0x00,0x00, -0x12,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x80,0x04,0x00,0x00,0x0f,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x81,0x04,0x00,0x00,0x13,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x82,0x04,0x00,0x00, +0xa8,0x01,0x00,0x00,0x02,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xa9,0x01,0x00,0x00,0x12,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xaa,0x01,0x00,0x00, +0x0e,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xab,0x01,0x00,0x00,0x03,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xac,0x01,0x00,0x00,0x13,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xad,0x01,0x00,0x00, 0x14,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x83,0x04,0x00,0x00,0x05,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x84,0x04,0x00,0x00,0x15,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x85,0x04,0x00,0x00, +0xae,0x01,0x00,0x00,0x05,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xaf,0x01,0x00,0x00,0x15,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xb0,0x01,0x00,0x00, 0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x86,0x04,0x00,0x00,0x16,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x87,0x04,0x00,0x00,0x07,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x88,0x04,0x00,0x00, -0x17,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x89,0x04,0x00,0x00,0x08,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x8a,0x04,0x00,0x00,0x18,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x8b,0x04,0x00,0x00, -0x09,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x8c,0x04,0x00,0x00,0x19,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x8d,0x04,0x00,0x00,0x0a,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x8e,0x04,0x00,0x00, -0x1a,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x8f,0x04,0x00,0x00,0x0b,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x90,0x04,0x00,0x00,0x1b,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x91,0x04,0x00,0x00, -0x1c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x92,0x04,0x00,0x00,0x1d,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x93,0x04,0x00,0x00,0x1e,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x94,0x04,0x00,0x00, -0x1f,0x00,0x00,0x00,0x36,0x00,0x05,0x00,0x02,0x00,0x00,0x00, +0xb1,0x01,0x00,0x00,0x16,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xb2,0x01,0x00,0x00,0x07,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xb3,0x01,0x00,0x00, +0x17,0x00,0x00,0x00,0x36,0x00,0x05,0x00,0x02,0x00,0x00,0x00, 0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x03,0x00,0x00,0x00, 0xf8,0x00,0x02,0x00,0x05,0x00,0x00,0x00,0xf7,0x00,0x03,0x00, -0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xfb,0x00,0x03,0x00, -0x0d,0x00,0x00,0x00,0xbf,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xbf,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x0e,0x00,0x00,0x00, -0x0f,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x0d,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x10,0x00,0x00,0x00, -0x0f,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x11,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x18,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0x17,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x1a,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x87,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x1a,0x00,0x00,0x00, -0x1b,0x00,0x00,0x00,0x8b,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x1d,0x00,0x00,0x00,0x11,0x00,0x00,0x00,0x1c,0x00,0x00,0x00, -0x87,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x11,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x26,0x00,0x00,0x00,0x1d,0x00,0x00,0x00, -0x1b,0x00,0x00,0x00,0xaf,0x00,0x05,0x00,0x24,0x00,0x00,0x00, -0x29,0x00,0x00,0x00,0x26,0x00,0x00,0x00,0x1a,0x00,0x00,0x00, -0xa8,0x00,0x04,0x00,0x24,0x00,0x00,0x00,0x2a,0x00,0x00,0x00, -0x29,0x00,0x00,0x00,0xf7,0x00,0x03,0x00,0x2c,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x2a,0x00,0x00,0x00, -0x2b,0x00,0x00,0x00,0x2c,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x2b,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x18,0x00,0x00,0x00, -0x2f,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x30,0x00,0x00,0x00, -0x2f,0x00,0x00,0x00,0xaf,0x00,0x05,0x00,0x24,0x00,0x00,0x00, -0x31,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x30,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x2c,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x2c,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x24,0x00,0x00,0x00, -0x32,0x00,0x00,0x00,0x29,0x00,0x00,0x00,0xbf,0x00,0x00,0x00, -0x31,0x00,0x00,0x00,0x2b,0x00,0x00,0x00,0xf7,0x00,0x03,0x00, -0x34,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x32,0x00,0x00,0x00,0x33,0x00,0x00,0x00,0x34,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0x33,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xbe,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x34,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x18,0x00,0x00,0x00,0x38,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x38,0x00,0x00,0x00, -0x87,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x1b,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x3e,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x3a,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0x3e,0x00,0x00,0x00,0x1d,0x00,0x00,0x00, -0x41,0x00,0x07,0x00,0x57,0x00,0x00,0x00,0x58,0x00,0x00,0x00, -0x55,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0x59,0x00,0x00,0x00,0x58,0x00,0x00,0x00,0x73,0x00,0x04,0x00, -0x4a,0x00,0x00,0x00,0x5a,0x00,0x00,0x00,0x59,0x00,0x00,0x00, -0x41,0x00,0x07,0x00,0x57,0x00,0x00,0x00,0x5d,0x00,0x00,0x00, -0x55,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x17,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0x5e,0x00,0x00,0x00,0x5d,0x00,0x00,0x00,0x73,0x00,0x04,0x00, -0x4a,0x00,0x00,0x00,0x5f,0x00,0x00,0x00,0x5e,0x00,0x00,0x00, -0x41,0x00,0x07,0x00,0x64,0x00,0x00,0x00,0x65,0x00,0x00,0x00, -0x55,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x37,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x66,0x00,0x00,0x00,0x65,0x00,0x00,0x00,0xc2,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0x68,0x00,0x00,0x00,0x66,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0xc4,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0x6a,0x00,0x00,0x00,0x68,0x00,0x00,0x00,0x69,0x00,0x00,0x00, -0xc7,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0x6b,0x00,0x00,0x00, -0x6a,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x6b,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x6f,0x00,0x00,0x00, -0x65,0x00,0x00,0x00,0xc2,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0x73,0x00,0x00,0x00,0x6f,0x00,0x00,0x00,0x71,0x00,0x00,0x00, -0xc7,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0x74,0x00,0x00,0x00, -0x73,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x75,0x00,0x00,0x00,0x74,0x00,0x00,0x00, -0x41,0x00,0x08,0x00,0x7c,0x00,0x00,0x00,0x7d,0x00,0x00,0x00, -0x55,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x7a,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4e,0x00,0x00,0x00,0x7e,0x00,0x00,0x00,0x7d,0x00,0x00,0x00, -0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x7f,0x00,0x00,0x00, -0x7e,0x00,0x00,0x00,0xc7,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0x85,0x00,0x00,0x00,0x7f,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x88,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0xc5,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0x89,0x00,0x00,0x00,0x85,0x00,0x00,0x00,0x88,0x00,0x00,0x00, -0x70,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0x8a,0x00,0x00,0x00, -0x89,0x00,0x00,0x00,0xc2,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0x8c,0x00,0x00,0x00,0x7f,0x00,0x00,0x00,0x69,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x90,0x00,0x00,0x00, -0x75,0x00,0x00,0x00,0xc5,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0x91,0x00,0x00,0x00,0x8c,0x00,0x00,0x00,0x90,0x00,0x00,0x00, -0x70,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0x92,0x00,0x00,0x00, -0x91,0x00,0x00,0x00,0x50,0x00,0x05,0x00,0x80,0x00,0x00,0x00, -0x93,0x00,0x00,0x00,0x8a,0x00,0x00,0x00,0x92,0x00,0x00,0x00, -0x8e,0x00,0x05,0x00,0x80,0x00,0x00,0x00,0x96,0x00,0x00,0x00, -0x93,0x00,0x00,0x00,0x5a,0x00,0x00,0x00,0x50,0x00,0x05,0x00, -0x80,0x00,0x00,0x00,0x98,0x00,0x00,0x00,0x5f,0x00,0x00,0x00, -0x5f,0x00,0x00,0x00,0x81,0x00,0x05,0x00,0x80,0x00,0x00,0x00, -0x99,0x00,0x00,0x00,0x96,0x00,0x00,0x00,0x98,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x18,0x00,0x00,0x00,0x9f,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x7a,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xa0,0x00,0x00,0x00,0x9f,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa1,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0xa0,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xa4,0x00,0x00,0x00,0xa1,0x00,0x00,0x00, -0x26,0x00,0x00,0x00,0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00, -0xa9,0x00,0x00,0x00,0x99,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x73,0x00,0x04,0x00,0x4d,0x00,0x00,0x00,0xaa,0x00,0x00,0x00, -0xa9,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x57,0x00,0x00,0x00, -0xab,0x00,0x00,0x00,0x9d,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0xa4,0x00,0x00,0x00,0x3e,0x00,0x03,0x00,0xab,0x00,0x00,0x00, -0xaa,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xb5,0x00,0x00,0x00,0xa4,0x00,0x00,0x00,0x48,0x00,0x00,0x00, -0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00,0xb7,0x00,0x00,0x00, -0x99,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x73,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0xb8,0x00,0x00,0x00,0xb7,0x00,0x00,0x00, -0x41,0x00,0x06,0x00,0x57,0x00,0x00,0x00,0xb9,0x00,0x00,0x00, -0x9d,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0xb5,0x00,0x00,0x00, -0x3e,0x00,0x03,0x00,0xb9,0x00,0x00,0x00,0xb8,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00,0xd0,0x00,0x00,0x00, -0x58,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0xd1,0x00,0x00,0x00,0xd0,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0xd3,0x00,0x00,0x00,0x5d,0x00,0x00,0x00, -0x73,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0xd4,0x00,0x00,0x00, -0xd3,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0xd6,0x00,0x00,0x00,0x65,0x00,0x00,0x00,0xc2,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0xd7,0x00,0x00,0x00,0xd6,0x00,0x00,0x00, -0x17,0x00,0x00,0x00,0xc4,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0xd8,0x00,0x00,0x00,0xd7,0x00,0x00,0x00,0x69,0x00,0x00,0x00, -0xc7,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0xd9,0x00,0x00,0x00, -0xd8,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xda,0x00,0x00,0x00,0xd9,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0xdc,0x00,0x00,0x00, -0x65,0x00,0x00,0x00,0xc2,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0xde,0x00,0x00,0x00,0xdc,0x00,0x00,0x00,0x7c,0x04,0x00,0x00, -0xc7,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0xdf,0x00,0x00,0x00, -0xde,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xe0,0x00,0x00,0x00,0xdf,0x00,0x00,0x00, -0x41,0x00,0x08,0x00,0x7c,0x00,0x00,0x00,0xe2,0x00,0x00,0x00, -0x55,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x7a,0x00,0x00,0x00,0x17,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4e,0x00,0x00,0x00,0xe3,0x00,0x00,0x00,0xe2,0x00,0x00,0x00, -0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0xe4,0x00,0x00,0x00, -0xe3,0x00,0x00,0x00,0xc7,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0xe5,0x00,0x00,0x00,0xe4,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0xe7,0x00,0x00,0x00, -0xda,0x00,0x00,0x00,0xc5,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0xe8,0x00,0x00,0x00,0xe5,0x00,0x00,0x00,0xe7,0x00,0x00,0x00, -0x70,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0xe9,0x00,0x00,0x00, -0xe8,0x00,0x00,0x00,0xc2,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0xea,0x00,0x00,0x00,0xe4,0x00,0x00,0x00,0x69,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0xec,0x00,0x00,0x00, -0xe0,0x00,0x00,0x00,0xc5,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0xed,0x00,0x00,0x00,0xea,0x00,0x00,0x00,0xec,0x00,0x00,0x00, -0x70,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0xee,0x00,0x00,0x00, -0xed,0x00,0x00,0x00,0x50,0x00,0x05,0x00,0x80,0x00,0x00,0x00, -0xef,0x00,0x00,0x00,0xe9,0x00,0x00,0x00,0xee,0x00,0x00,0x00, -0x8e,0x00,0x05,0x00,0x80,0x00,0x00,0x00,0xf0,0x00,0x00,0x00, -0xef,0x00,0x00,0x00,0xd1,0x00,0x00,0x00,0x50,0x00,0x05,0x00, -0x80,0x00,0x00,0x00,0xf1,0x00,0x00,0x00,0xd4,0x00,0x00,0x00, -0xd4,0x00,0x00,0x00,0x81,0x00,0x05,0x00,0x80,0x00,0x00,0x00, -0xf2,0x00,0x00,0x00,0xf0,0x00,0x00,0x00,0xf1,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xf8,0x00,0x00,0x00, -0xa4,0x00,0x00,0x00,0x17,0x00,0x00,0x00,0x51,0x00,0x05,0x00, -0x4a,0x00,0x00,0x00,0xfa,0x00,0x00,0x00,0xf2,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0xfb,0x00,0x00,0x00,0xfa,0x00,0x00,0x00,0x41,0x00,0x06,0x00, -0x57,0x00,0x00,0x00,0xfc,0x00,0x00,0x00,0x9d,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0xf8,0x00,0x00,0x00,0x3e,0x00,0x03,0x00, -0xfc,0x00,0x00,0x00,0xfb,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x03,0x01,0x00,0x00,0xa4,0x00,0x00,0x00, -0x7d,0x04,0x00,0x00,0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00, -0x04,0x01,0x00,0x00,0xf2,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x73,0x00,0x04,0x00,0x4d,0x00,0x00,0x00,0x05,0x01,0x00,0x00, -0x04,0x01,0x00,0x00,0x41,0x00,0x06,0x00,0x57,0x00,0x00,0x00, -0x06,0x01,0x00,0x00,0x9d,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x03,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x06,0x01,0x00,0x00, -0x05,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0x0f,0x01,0x00,0x00,0x58,0x00,0x00,0x00,0x73,0x00,0x04,0x00, -0x4a,0x00,0x00,0x00,0x10,0x01,0x00,0x00,0x0f,0x01,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00,0x12,0x01,0x00,0x00, -0x5d,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0x13,0x01,0x00,0x00,0x12,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x15,0x01,0x00,0x00,0x65,0x00,0x00,0x00, -0xc2,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0x16,0x01,0x00,0x00, -0x15,0x01,0x00,0x00,0x37,0x00,0x00,0x00,0xc4,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0x17,0x01,0x00,0x00,0x16,0x01,0x00,0x00, -0x69,0x00,0x00,0x00,0xc7,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0x18,0x01,0x00,0x00,0x17,0x01,0x00,0x00,0x4f,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x19,0x01,0x00,0x00, -0x18,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x1b,0x01,0x00,0x00,0x65,0x00,0x00,0x00,0xc2,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0x1d,0x01,0x00,0x00,0x1b,0x01,0x00,0x00, -0x7e,0x04,0x00,0x00,0xc7,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0x1e,0x01,0x00,0x00,0x1d,0x01,0x00,0x00,0x4f,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x1f,0x01,0x00,0x00, -0x1e,0x01,0x00,0x00,0x41,0x00,0x08,0x00,0x7c,0x00,0x00,0x00, -0x21,0x01,0x00,0x00,0x55,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0x7a,0x00,0x00,0x00,0x37,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4e,0x00,0x00,0x00,0x22,0x01,0x00,0x00, -0x21,0x01,0x00,0x00,0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x23,0x01,0x00,0x00,0x22,0x01,0x00,0x00,0xc7,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0x24,0x01,0x00,0x00,0x23,0x01,0x00,0x00, -0x84,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x26,0x01,0x00,0x00,0x19,0x01,0x00,0x00,0xc5,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0x27,0x01,0x00,0x00,0x24,0x01,0x00,0x00, -0x26,0x01,0x00,0x00,0x70,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0x28,0x01,0x00,0x00,0x27,0x01,0x00,0x00,0xc2,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0x29,0x01,0x00,0x00,0x23,0x01,0x00,0x00, -0x69,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x2b,0x01,0x00,0x00,0x1f,0x01,0x00,0x00,0xc5,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0x2c,0x01,0x00,0x00,0x29,0x01,0x00,0x00, -0x2b,0x01,0x00,0x00,0x70,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0x2d,0x01,0x00,0x00,0x2c,0x01,0x00,0x00,0x50,0x00,0x05,0x00, -0x80,0x00,0x00,0x00,0x2e,0x01,0x00,0x00,0x28,0x01,0x00,0x00, -0x2d,0x01,0x00,0x00,0x8e,0x00,0x05,0x00,0x80,0x00,0x00,0x00, -0x2f,0x01,0x00,0x00,0x2e,0x01,0x00,0x00,0x10,0x01,0x00,0x00, -0x50,0x00,0x05,0x00,0x80,0x00,0x00,0x00,0x30,0x01,0x00,0x00, -0x13,0x01,0x00,0x00,0x13,0x01,0x00,0x00,0x81,0x00,0x05,0x00, -0x80,0x00,0x00,0x00,0x31,0x01,0x00,0x00,0x2f,0x01,0x00,0x00, -0x30,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x37,0x01,0x00,0x00,0xa4,0x00,0x00,0x00,0x37,0x00,0x00,0x00, -0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00,0x39,0x01,0x00,0x00, -0x31,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x73,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0x3a,0x01,0x00,0x00,0x39,0x01,0x00,0x00, -0x41,0x00,0x06,0x00,0x57,0x00,0x00,0x00,0x3b,0x01,0x00,0x00, -0x9d,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x37,0x01,0x00,0x00, -0x3e,0x00,0x03,0x00,0x3b,0x01,0x00,0x00,0x3a,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x42,0x01,0x00,0x00, -0xa4,0x00,0x00,0x00,0x7f,0x04,0x00,0x00,0x51,0x00,0x05,0x00, -0x4a,0x00,0x00,0x00,0x43,0x01,0x00,0x00,0x31,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0x44,0x01,0x00,0x00,0x43,0x01,0x00,0x00,0x41,0x00,0x06,0x00, -0x57,0x00,0x00,0x00,0x45,0x01,0x00,0x00,0x9d,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x42,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0x45,0x01,0x00,0x00,0x44,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0x4e,0x01,0x00,0x00,0x58,0x00,0x00,0x00, -0x73,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0x4f,0x01,0x00,0x00, -0x4e,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0x51,0x01,0x00,0x00,0x5d,0x00,0x00,0x00,0x73,0x00,0x04,0x00, -0x4a,0x00,0x00,0x00,0x52,0x01,0x00,0x00,0x51,0x01,0x00,0x00, -0x3d,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x54,0x01,0x00,0x00, -0x65,0x00,0x00,0x00,0xc2,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0x55,0x01,0x00,0x00,0x54,0x01,0x00,0x00,0x7a,0x00,0x00,0x00, -0xc4,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0x56,0x01,0x00,0x00, -0x55,0x01,0x00,0x00,0x69,0x00,0x00,0x00,0xc7,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0x57,0x01,0x00,0x00,0x56,0x01,0x00,0x00, -0x4f,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x58,0x01,0x00,0x00,0x57,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x5a,0x01,0x00,0x00,0x65,0x00,0x00,0x00, -0xc2,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0x5c,0x01,0x00,0x00, -0x5a,0x01,0x00,0x00,0x80,0x04,0x00,0x00,0xc7,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0x5d,0x01,0x00,0x00,0x5c,0x01,0x00,0x00, -0x4f,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x5e,0x01,0x00,0x00,0x5d,0x01,0x00,0x00,0x41,0x00,0x08,0x00, -0x7c,0x00,0x00,0x00,0x60,0x01,0x00,0x00,0x55,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x7a,0x00,0x00,0x00, -0x7a,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x4e,0x00,0x00,0x00, -0x61,0x01,0x00,0x00,0x60,0x01,0x00,0x00,0x71,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x62,0x01,0x00,0x00,0x61,0x01,0x00,0x00, -0xc7,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0x63,0x01,0x00,0x00, -0x62,0x01,0x00,0x00,0x84,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x65,0x01,0x00,0x00,0x58,0x01,0x00,0x00, -0xc5,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0x66,0x01,0x00,0x00, -0x63,0x01,0x00,0x00,0x65,0x01,0x00,0x00,0x70,0x00,0x04,0x00, -0x4a,0x00,0x00,0x00,0x67,0x01,0x00,0x00,0x66,0x01,0x00,0x00, -0xc2,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0x68,0x01,0x00,0x00, -0x62,0x01,0x00,0x00,0x69,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x6a,0x01,0x00,0x00,0x5e,0x01,0x00,0x00, -0xc5,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0x6b,0x01,0x00,0x00, -0x68,0x01,0x00,0x00,0x6a,0x01,0x00,0x00,0x70,0x00,0x04,0x00, -0x4a,0x00,0x00,0x00,0x6c,0x01,0x00,0x00,0x6b,0x01,0x00,0x00, -0x50,0x00,0x05,0x00,0x80,0x00,0x00,0x00,0x6d,0x01,0x00,0x00, -0x67,0x01,0x00,0x00,0x6c,0x01,0x00,0x00,0x8e,0x00,0x05,0x00, -0x80,0x00,0x00,0x00,0x6e,0x01,0x00,0x00,0x6d,0x01,0x00,0x00, -0x4f,0x01,0x00,0x00,0x50,0x00,0x05,0x00,0x80,0x00,0x00,0x00, -0x6f,0x01,0x00,0x00,0x52,0x01,0x00,0x00,0x52,0x01,0x00,0x00, -0x81,0x00,0x05,0x00,0x80,0x00,0x00,0x00,0x70,0x01,0x00,0x00, -0x6e,0x01,0x00,0x00,0x6f,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x76,0x01,0x00,0x00,0xa4,0x00,0x00,0x00, -0x7a,0x00,0x00,0x00,0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00, -0x78,0x01,0x00,0x00,0x70,0x01,0x00,0x00,0x00,0x00,0x00,0x00, -0x73,0x00,0x04,0x00,0x4d,0x00,0x00,0x00,0x79,0x01,0x00,0x00, -0x78,0x01,0x00,0x00,0x41,0x00,0x06,0x00,0x57,0x00,0x00,0x00, -0x7a,0x01,0x00,0x00,0x9d,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x76,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x7a,0x01,0x00,0x00, -0x79,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x81,0x01,0x00,0x00,0xa4,0x00,0x00,0x00,0x81,0x04,0x00,0x00, -0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00,0x82,0x01,0x00,0x00, -0x70,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x73,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0x83,0x01,0x00,0x00,0x82,0x01,0x00,0x00, -0x41,0x00,0x06,0x00,0x57,0x00,0x00,0x00,0x84,0x01,0x00,0x00, -0x9d,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x81,0x01,0x00,0x00, -0x3e,0x00,0x03,0x00,0x84,0x01,0x00,0x00,0x83,0x01,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00,0x8d,0x01,0x00,0x00, -0x58,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0x8e,0x01,0x00,0x00,0x8d,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0x90,0x01,0x00,0x00,0x5d,0x00,0x00,0x00, -0x73,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0x91,0x01,0x00,0x00, -0x90,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x93,0x01,0x00,0x00,0x65,0x00,0x00,0x00,0xc2,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0x94,0x01,0x00,0x00,0x93,0x01,0x00,0x00, -0x69,0x00,0x00,0x00,0xc4,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0x95,0x01,0x00,0x00,0x94,0x01,0x00,0x00,0x69,0x00,0x00,0x00, -0xc7,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0x96,0x01,0x00,0x00, -0x95,0x01,0x00,0x00,0x4f,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x97,0x01,0x00,0x00,0x96,0x01,0x00,0x00, -0x3d,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x99,0x01,0x00,0x00, -0x65,0x00,0x00,0x00,0xc2,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0x9b,0x01,0x00,0x00,0x99,0x01,0x00,0x00,0x48,0x00,0x00,0x00, -0xc7,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0x9c,0x01,0x00,0x00, -0x9b,0x01,0x00,0x00,0x4f,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x9d,0x01,0x00,0x00,0x9c,0x01,0x00,0x00, -0x41,0x00,0x08,0x00,0x7c,0x00,0x00,0x00,0x9f,0x01,0x00,0x00, -0x55,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x7a,0x00,0x00,0x00,0x69,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4e,0x00,0x00,0x00,0xa0,0x01,0x00,0x00,0x9f,0x01,0x00,0x00, -0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0xa1,0x01,0x00,0x00, -0xa0,0x01,0x00,0x00,0xc7,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0xa2,0x01,0x00,0x00,0xa1,0x01,0x00,0x00,0x84,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0xa4,0x01,0x00,0x00, -0x97,0x01,0x00,0x00,0xc5,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0xa5,0x01,0x00,0x00,0xa2,0x01,0x00,0x00,0xa4,0x01,0x00,0x00, -0x70,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0xa6,0x01,0x00,0x00, -0xa5,0x01,0x00,0x00,0xc2,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0xa7,0x01,0x00,0x00,0xa1,0x01,0x00,0x00,0x69,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0xa9,0x01,0x00,0x00, -0x9d,0x01,0x00,0x00,0xc5,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0xaa,0x01,0x00,0x00,0xa7,0x01,0x00,0x00,0xa9,0x01,0x00,0x00, -0x70,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0xab,0x01,0x00,0x00, -0xaa,0x01,0x00,0x00,0x50,0x00,0x05,0x00,0x80,0x00,0x00,0x00, -0xac,0x01,0x00,0x00,0xa6,0x01,0x00,0x00,0xab,0x01,0x00,0x00, -0x8e,0x00,0x05,0x00,0x80,0x00,0x00,0x00,0xad,0x01,0x00,0x00, -0xac,0x01,0x00,0x00,0x8e,0x01,0x00,0x00,0x50,0x00,0x05,0x00, -0x80,0x00,0x00,0x00,0xae,0x01,0x00,0x00,0x91,0x01,0x00,0x00, -0x91,0x01,0x00,0x00,0x81,0x00,0x05,0x00,0x80,0x00,0x00,0x00, -0xaf,0x01,0x00,0x00,0xad,0x01,0x00,0x00,0xae,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xb5,0x01,0x00,0x00, -0xa4,0x00,0x00,0x00,0x69,0x00,0x00,0x00,0x51,0x00,0x05,0x00, -0x4a,0x00,0x00,0x00,0xb7,0x01,0x00,0x00,0xaf,0x01,0x00,0x00, -0x00,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0xb8,0x01,0x00,0x00,0xb7,0x01,0x00,0x00,0x41,0x00,0x06,0x00, -0x57,0x00,0x00,0x00,0xb9,0x01,0x00,0x00,0x9d,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0xb5,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0xb9,0x01,0x00,0x00,0xb8,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xc0,0x01,0x00,0x00,0xa4,0x00,0x00,0x00, -0x82,0x04,0x00,0x00,0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00, -0xc1,0x01,0x00,0x00,0xaf,0x01,0x00,0x00,0x01,0x00,0x00,0x00, -0x73,0x00,0x04,0x00,0x4d,0x00,0x00,0x00,0xc2,0x01,0x00,0x00, -0xc1,0x01,0x00,0x00,0x41,0x00,0x06,0x00,0x57,0x00,0x00,0x00, -0xc3,0x01,0x00,0x00,0x9d,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0xc0,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0xc3,0x01,0x00,0x00, -0xc2,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0xcc,0x01,0x00,0x00,0x58,0x00,0x00,0x00,0x73,0x00,0x04,0x00, -0x4a,0x00,0x00,0x00,0xcd,0x01,0x00,0x00,0xcc,0x01,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00,0xcf,0x01,0x00,0x00, -0x5d,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0xd0,0x01,0x00,0x00,0xcf,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0xd2,0x01,0x00,0x00,0x65,0x00,0x00,0x00, -0xc2,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0xd3,0x01,0x00,0x00, -0xd2,0x01,0x00,0x00,0x83,0x04,0x00,0x00,0xc4,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0xd4,0x01,0x00,0x00,0xd3,0x01,0x00,0x00, -0x69,0x00,0x00,0x00,0xc7,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0xd5,0x01,0x00,0x00,0xd4,0x01,0x00,0x00,0x4f,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xd6,0x01,0x00,0x00, -0xd5,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0xd8,0x01,0x00,0x00,0x65,0x00,0x00,0x00,0xc2,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0xda,0x01,0x00,0x00,0xd8,0x01,0x00,0x00, -0x7d,0x04,0x00,0x00,0xc7,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0xdb,0x01,0x00,0x00,0xda,0x01,0x00,0x00,0x4f,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xdc,0x01,0x00,0x00, -0xdb,0x01,0x00,0x00,0x41,0x00,0x08,0x00,0x7c,0x00,0x00,0x00, -0xde,0x01,0x00,0x00,0x55,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0x7a,0x00,0x00,0x00,0x83,0x04,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4e,0x00,0x00,0x00,0xdf,0x01,0x00,0x00, -0xde,0x01,0x00,0x00,0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0xe0,0x01,0x00,0x00,0xdf,0x01,0x00,0x00,0xc7,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0xe1,0x01,0x00,0x00,0xe0,0x01,0x00,0x00, -0x84,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0xe3,0x01,0x00,0x00,0xd6,0x01,0x00,0x00,0xc5,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0xe4,0x01,0x00,0x00,0xe1,0x01,0x00,0x00, -0xe3,0x01,0x00,0x00,0x70,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0xe5,0x01,0x00,0x00,0xe4,0x01,0x00,0x00,0xc2,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0xe6,0x01,0x00,0x00,0xe0,0x01,0x00,0x00, -0x69,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0xe8,0x01,0x00,0x00,0xdc,0x01,0x00,0x00,0xc5,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0xe9,0x01,0x00,0x00,0xe6,0x01,0x00,0x00, -0xe8,0x01,0x00,0x00,0x70,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0xea,0x01,0x00,0x00,0xe9,0x01,0x00,0x00,0x50,0x00,0x05,0x00, -0x80,0x00,0x00,0x00,0xeb,0x01,0x00,0x00,0xe5,0x01,0x00,0x00, -0xea,0x01,0x00,0x00,0x8e,0x00,0x05,0x00,0x80,0x00,0x00,0x00, -0xec,0x01,0x00,0x00,0xeb,0x01,0x00,0x00,0xcd,0x01,0x00,0x00, -0x50,0x00,0x05,0x00,0x80,0x00,0x00,0x00,0xed,0x01,0x00,0x00, -0xd0,0x01,0x00,0x00,0xd0,0x01,0x00,0x00,0x81,0x00,0x05,0x00, -0x80,0x00,0x00,0x00,0xee,0x01,0x00,0x00,0xec,0x01,0x00,0x00, -0xed,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xf4,0x01,0x00,0x00,0xa4,0x00,0x00,0x00,0x83,0x04,0x00,0x00, -0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00,0xf6,0x01,0x00,0x00, -0xee,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x73,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0xf7,0x01,0x00,0x00,0xf6,0x01,0x00,0x00, -0x41,0x00,0x06,0x00,0x57,0x00,0x00,0x00,0xf8,0x01,0x00,0x00, -0x9d,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0xf4,0x01,0x00,0x00, -0x3e,0x00,0x03,0x00,0xf8,0x01,0x00,0x00,0xf7,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xff,0x01,0x00,0x00, -0xa4,0x00,0x00,0x00,0x84,0x04,0x00,0x00,0x51,0x00,0x05,0x00, -0x4a,0x00,0x00,0x00,0x00,0x02,0x00,0x00,0xee,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0x01,0x02,0x00,0x00,0x00,0x02,0x00,0x00,0x41,0x00,0x06,0x00, -0x57,0x00,0x00,0x00,0x02,0x02,0x00,0x00,0x9d,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0xff,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0x02,0x02,0x00,0x00,0x01,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0x0b,0x02,0x00,0x00,0x58,0x00,0x00,0x00, -0x73,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0x0c,0x02,0x00,0x00, -0x0b,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0x0e,0x02,0x00,0x00,0x5d,0x00,0x00,0x00,0x73,0x00,0x04,0x00, -0x4a,0x00,0x00,0x00,0x0f,0x02,0x00,0x00,0x0e,0x02,0x00,0x00, -0x3d,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x11,0x02,0x00,0x00, -0x65,0x00,0x00,0x00,0xc2,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0x12,0x02,0x00,0x00,0x11,0x02,0x00,0x00,0x85,0x04,0x00,0x00, -0xc4,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0x13,0x02,0x00,0x00, -0x12,0x02,0x00,0x00,0x69,0x00,0x00,0x00,0xc7,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0x14,0x02,0x00,0x00,0x13,0x02,0x00,0x00, -0x4f,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x15,0x02,0x00,0x00,0x14,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x17,0x02,0x00,0x00,0x65,0x00,0x00,0x00, -0xc2,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0x19,0x02,0x00,0x00, -0x17,0x02,0x00,0x00,0x7f,0x04,0x00,0x00,0xc7,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0x1a,0x02,0x00,0x00,0x19,0x02,0x00,0x00, -0x4f,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x1b,0x02,0x00,0x00,0x1a,0x02,0x00,0x00,0x41,0x00,0x08,0x00, -0x7c,0x00,0x00,0x00,0x1d,0x02,0x00,0x00,0x55,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x7a,0x00,0x00,0x00, -0x85,0x04,0x00,0x00,0x3d,0x00,0x04,0x00,0x4e,0x00,0x00,0x00, -0x1e,0x02,0x00,0x00,0x1d,0x02,0x00,0x00,0x71,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x1f,0x02,0x00,0x00,0x1e,0x02,0x00,0x00, -0xc7,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0x20,0x02,0x00,0x00, -0x1f,0x02,0x00,0x00,0x84,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x22,0x02,0x00,0x00,0x15,0x02,0x00,0x00, -0xc5,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0x23,0x02,0x00,0x00, -0x20,0x02,0x00,0x00,0x22,0x02,0x00,0x00,0x70,0x00,0x04,0x00, -0x4a,0x00,0x00,0x00,0x24,0x02,0x00,0x00,0x23,0x02,0x00,0x00, -0xc2,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0x25,0x02,0x00,0x00, -0x1f,0x02,0x00,0x00,0x69,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x27,0x02,0x00,0x00,0x1b,0x02,0x00,0x00, -0xc5,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0x28,0x02,0x00,0x00, -0x25,0x02,0x00,0x00,0x27,0x02,0x00,0x00,0x70,0x00,0x04,0x00, -0x4a,0x00,0x00,0x00,0x29,0x02,0x00,0x00,0x28,0x02,0x00,0x00, -0x50,0x00,0x05,0x00,0x80,0x00,0x00,0x00,0x2a,0x02,0x00,0x00, -0x24,0x02,0x00,0x00,0x29,0x02,0x00,0x00,0x8e,0x00,0x05,0x00, -0x80,0x00,0x00,0x00,0x2b,0x02,0x00,0x00,0x2a,0x02,0x00,0x00, -0x0c,0x02,0x00,0x00,0x50,0x00,0x05,0x00,0x80,0x00,0x00,0x00, -0x2c,0x02,0x00,0x00,0x0f,0x02,0x00,0x00,0x0f,0x02,0x00,0x00, -0x81,0x00,0x05,0x00,0x80,0x00,0x00,0x00,0x2d,0x02,0x00,0x00, -0x2b,0x02,0x00,0x00,0x2c,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x33,0x02,0x00,0x00,0xa4,0x00,0x00,0x00, -0x85,0x04,0x00,0x00,0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00, -0x35,0x02,0x00,0x00,0x2d,0x02,0x00,0x00,0x00,0x00,0x00,0x00, -0x73,0x00,0x04,0x00,0x4d,0x00,0x00,0x00,0x36,0x02,0x00,0x00, -0x35,0x02,0x00,0x00,0x41,0x00,0x06,0x00,0x57,0x00,0x00,0x00, -0x37,0x02,0x00,0x00,0x9d,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x33,0x02,0x00,0x00,0x3e,0x00,0x03,0x00,0x37,0x02,0x00,0x00, -0x36,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x3e,0x02,0x00,0x00,0xa4,0x00,0x00,0x00,0x86,0x04,0x00,0x00, -0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00,0x3f,0x02,0x00,0x00, -0x2d,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0x73,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0x40,0x02,0x00,0x00,0x3f,0x02,0x00,0x00, -0x41,0x00,0x06,0x00,0x57,0x00,0x00,0x00,0x41,0x02,0x00,0x00, -0x9d,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x3e,0x02,0x00,0x00, -0x3e,0x00,0x03,0x00,0x41,0x02,0x00,0x00,0x40,0x02,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00,0x4a,0x02,0x00,0x00, -0x58,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0x4b,0x02,0x00,0x00,0x4a,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0x4d,0x02,0x00,0x00,0x5d,0x00,0x00,0x00, -0x73,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0x4e,0x02,0x00,0x00, -0x4d,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x50,0x02,0x00,0x00,0x65,0x00,0x00,0x00,0xc2,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0x51,0x02,0x00,0x00,0x50,0x02,0x00,0x00, -0x87,0x04,0x00,0x00,0xc4,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0x52,0x02,0x00,0x00,0x51,0x02,0x00,0x00,0x69,0x00,0x00,0x00, -0xc7,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0x53,0x02,0x00,0x00, -0x52,0x02,0x00,0x00,0x4f,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x54,0x02,0x00,0x00,0x53,0x02,0x00,0x00, -0x3d,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x56,0x02,0x00,0x00, -0x65,0x00,0x00,0x00,0xc2,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0x58,0x02,0x00,0x00,0x56,0x02,0x00,0x00,0x81,0x04,0x00,0x00, -0xc7,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0x59,0x02,0x00,0x00, -0x58,0x02,0x00,0x00,0x4f,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x5a,0x02,0x00,0x00,0x59,0x02,0x00,0x00, -0x41,0x00,0x08,0x00,0x7c,0x00,0x00,0x00,0x5c,0x02,0x00,0x00, -0x55,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x7a,0x00,0x00,0x00,0x87,0x04,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4e,0x00,0x00,0x00,0x5d,0x02,0x00,0x00,0x5c,0x02,0x00,0x00, -0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x5e,0x02,0x00,0x00, -0x5d,0x02,0x00,0x00,0xc7,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0x5f,0x02,0x00,0x00,0x5e,0x02,0x00,0x00,0x84,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x61,0x02,0x00,0x00, -0x54,0x02,0x00,0x00,0xc5,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0x62,0x02,0x00,0x00,0x5f,0x02,0x00,0x00,0x61,0x02,0x00,0x00, -0x70,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0x63,0x02,0x00,0x00, -0x62,0x02,0x00,0x00,0xc2,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0x64,0x02,0x00,0x00,0x5e,0x02,0x00,0x00,0x69,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x66,0x02,0x00,0x00, -0x5a,0x02,0x00,0x00,0xc5,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0x67,0x02,0x00,0x00,0x64,0x02,0x00,0x00,0x66,0x02,0x00,0x00, -0x70,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0x68,0x02,0x00,0x00, -0x67,0x02,0x00,0x00,0x50,0x00,0x05,0x00,0x80,0x00,0x00,0x00, -0x69,0x02,0x00,0x00,0x63,0x02,0x00,0x00,0x68,0x02,0x00,0x00, -0x8e,0x00,0x05,0x00,0x80,0x00,0x00,0x00,0x6a,0x02,0x00,0x00, -0x69,0x02,0x00,0x00,0x4b,0x02,0x00,0x00,0x50,0x00,0x05,0x00, -0x80,0x00,0x00,0x00,0x6b,0x02,0x00,0x00,0x4e,0x02,0x00,0x00, -0x4e,0x02,0x00,0x00,0x81,0x00,0x05,0x00,0x80,0x00,0x00,0x00, -0x6c,0x02,0x00,0x00,0x6a,0x02,0x00,0x00,0x6b,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x72,0x02,0x00,0x00, -0xa4,0x00,0x00,0x00,0x87,0x04,0x00,0x00,0x51,0x00,0x05,0x00, -0x4a,0x00,0x00,0x00,0x74,0x02,0x00,0x00,0x6c,0x02,0x00,0x00, -0x00,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0x75,0x02,0x00,0x00,0x74,0x02,0x00,0x00,0x41,0x00,0x06,0x00, -0x57,0x00,0x00,0x00,0x76,0x02,0x00,0x00,0x9d,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x72,0x02,0x00,0x00,0x3e,0x00,0x03,0x00, -0x76,0x02,0x00,0x00,0x75,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x7d,0x02,0x00,0x00,0xa4,0x00,0x00,0x00, -0x88,0x04,0x00,0x00,0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00, -0x7e,0x02,0x00,0x00,0x6c,0x02,0x00,0x00,0x01,0x00,0x00,0x00, -0x73,0x00,0x04,0x00,0x4d,0x00,0x00,0x00,0x7f,0x02,0x00,0x00, -0x7e,0x02,0x00,0x00,0x41,0x00,0x06,0x00,0x57,0x00,0x00,0x00, -0x80,0x02,0x00,0x00,0x9d,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x7d,0x02,0x00,0x00,0x3e,0x00,0x03,0x00,0x80,0x02,0x00,0x00, -0x7f,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0x89,0x02,0x00,0x00,0x58,0x00,0x00,0x00,0x73,0x00,0x04,0x00, -0x4a,0x00,0x00,0x00,0x8a,0x02,0x00,0x00,0x89,0x02,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00,0x8c,0x02,0x00,0x00, -0x5d,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0x8d,0x02,0x00,0x00,0x8c,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x8f,0x02,0x00,0x00,0x65,0x00,0x00,0x00, -0xc2,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0x90,0x02,0x00,0x00, -0x8f,0x02,0x00,0x00,0x89,0x04,0x00,0x00,0xc4,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0x91,0x02,0x00,0x00,0x90,0x02,0x00,0x00, -0x69,0x00,0x00,0x00,0xc7,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0x92,0x02,0x00,0x00,0x91,0x02,0x00,0x00,0x4f,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x93,0x02,0x00,0x00, -0x92,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x95,0x02,0x00,0x00,0x65,0x00,0x00,0x00,0xc2,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0x97,0x02,0x00,0x00,0x95,0x02,0x00,0x00, -0x82,0x04,0x00,0x00,0xc7,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0x98,0x02,0x00,0x00,0x97,0x02,0x00,0x00,0x4f,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x99,0x02,0x00,0x00, -0x98,0x02,0x00,0x00,0x41,0x00,0x08,0x00,0x7c,0x00,0x00,0x00, -0x9b,0x02,0x00,0x00,0x55,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0x7a,0x00,0x00,0x00,0x89,0x04,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4e,0x00,0x00,0x00,0x9c,0x02,0x00,0x00, -0x9b,0x02,0x00,0x00,0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x9d,0x02,0x00,0x00,0x9c,0x02,0x00,0x00,0xc7,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0x9e,0x02,0x00,0x00,0x9d,0x02,0x00,0x00, -0x84,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0xa0,0x02,0x00,0x00,0x93,0x02,0x00,0x00,0xc5,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0xa1,0x02,0x00,0x00,0x9e,0x02,0x00,0x00, -0xa0,0x02,0x00,0x00,0x70,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0xa2,0x02,0x00,0x00,0xa1,0x02,0x00,0x00,0xc2,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0xa3,0x02,0x00,0x00,0x9d,0x02,0x00,0x00, -0x69,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0xa5,0x02,0x00,0x00,0x99,0x02,0x00,0x00,0xc5,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0xa6,0x02,0x00,0x00,0xa3,0x02,0x00,0x00, -0xa5,0x02,0x00,0x00,0x70,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0xa7,0x02,0x00,0x00,0xa6,0x02,0x00,0x00,0x50,0x00,0x05,0x00, -0x80,0x00,0x00,0x00,0xa8,0x02,0x00,0x00,0xa2,0x02,0x00,0x00, -0xa7,0x02,0x00,0x00,0x8e,0x00,0x05,0x00,0x80,0x00,0x00,0x00, -0xa9,0x02,0x00,0x00,0xa8,0x02,0x00,0x00,0x8a,0x02,0x00,0x00, -0x50,0x00,0x05,0x00,0x80,0x00,0x00,0x00,0xaa,0x02,0x00,0x00, -0x8d,0x02,0x00,0x00,0x8d,0x02,0x00,0x00,0x81,0x00,0x05,0x00, -0x80,0x00,0x00,0x00,0xab,0x02,0x00,0x00,0xa9,0x02,0x00,0x00, -0xaa,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xb1,0x02,0x00,0x00,0xa4,0x00,0x00,0x00,0x89,0x04,0x00,0x00, -0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00,0xb3,0x02,0x00,0x00, -0xab,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x73,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0xb4,0x02,0x00,0x00,0xb3,0x02,0x00,0x00, -0x41,0x00,0x06,0x00,0x57,0x00,0x00,0x00,0xb5,0x02,0x00,0x00, -0x9d,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0xb1,0x02,0x00,0x00, -0x3e,0x00,0x03,0x00,0xb5,0x02,0x00,0x00,0xb4,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xbc,0x02,0x00,0x00, -0xa4,0x00,0x00,0x00,0x8a,0x04,0x00,0x00,0x51,0x00,0x05,0x00, -0x4a,0x00,0x00,0x00,0xbd,0x02,0x00,0x00,0xab,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0xbe,0x02,0x00,0x00,0xbd,0x02,0x00,0x00,0x41,0x00,0x06,0x00, -0x57,0x00,0x00,0x00,0xbf,0x02,0x00,0x00,0x9d,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0xbc,0x02,0x00,0x00,0x3e,0x00,0x03,0x00, -0xbf,0x02,0x00,0x00,0xbe,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0xc8,0x02,0x00,0x00,0x58,0x00,0x00,0x00, -0x73,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0xc9,0x02,0x00,0x00, -0xc8,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0xcb,0x02,0x00,0x00,0x5d,0x00,0x00,0x00,0x73,0x00,0x04,0x00, -0x4a,0x00,0x00,0x00,0xcc,0x02,0x00,0x00,0xcb,0x02,0x00,0x00, -0x3d,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0xce,0x02,0x00,0x00, -0x65,0x00,0x00,0x00,0xc2,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0xcf,0x02,0x00,0x00,0xce,0x02,0x00,0x00,0x8b,0x04,0x00,0x00, -0xc4,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0xd0,0x02,0x00,0x00, -0xcf,0x02,0x00,0x00,0x69,0x00,0x00,0x00,0xc7,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0xd1,0x02,0x00,0x00,0xd0,0x02,0x00,0x00, -0x4f,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xd2,0x02,0x00,0x00,0xd1,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0xd4,0x02,0x00,0x00,0x65,0x00,0x00,0x00, -0xc2,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0xd6,0x02,0x00,0x00, -0xd4,0x02,0x00,0x00,0x84,0x04,0x00,0x00,0xc7,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0xd7,0x02,0x00,0x00,0xd6,0x02,0x00,0x00, -0x4f,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xd8,0x02,0x00,0x00,0xd7,0x02,0x00,0x00,0x41,0x00,0x08,0x00, -0x7c,0x00,0x00,0x00,0xda,0x02,0x00,0x00,0x55,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x7a,0x00,0x00,0x00, -0x8b,0x04,0x00,0x00,0x3d,0x00,0x04,0x00,0x4e,0x00,0x00,0x00, -0xdb,0x02,0x00,0x00,0xda,0x02,0x00,0x00,0x71,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0xdc,0x02,0x00,0x00,0xdb,0x02,0x00,0x00, -0xc7,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0xdd,0x02,0x00,0x00, -0xdc,0x02,0x00,0x00,0x84,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0xdf,0x02,0x00,0x00,0xd2,0x02,0x00,0x00, -0xc5,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0xe0,0x02,0x00,0x00, -0xdd,0x02,0x00,0x00,0xdf,0x02,0x00,0x00,0x70,0x00,0x04,0x00, -0x4a,0x00,0x00,0x00,0xe1,0x02,0x00,0x00,0xe0,0x02,0x00,0x00, -0xc2,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0xe2,0x02,0x00,0x00, -0xdc,0x02,0x00,0x00,0x69,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0xe4,0x02,0x00,0x00,0xd8,0x02,0x00,0x00, -0xc5,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0xe5,0x02,0x00,0x00, -0xe2,0x02,0x00,0x00,0xe4,0x02,0x00,0x00,0x70,0x00,0x04,0x00, -0x4a,0x00,0x00,0x00,0xe6,0x02,0x00,0x00,0xe5,0x02,0x00,0x00, -0x50,0x00,0x05,0x00,0x80,0x00,0x00,0x00,0xe7,0x02,0x00,0x00, -0xe1,0x02,0x00,0x00,0xe6,0x02,0x00,0x00,0x8e,0x00,0x05,0x00, -0x80,0x00,0x00,0x00,0xe8,0x02,0x00,0x00,0xe7,0x02,0x00,0x00, -0xc9,0x02,0x00,0x00,0x50,0x00,0x05,0x00,0x80,0x00,0x00,0x00, -0xe9,0x02,0x00,0x00,0xcc,0x02,0x00,0x00,0xcc,0x02,0x00,0x00, -0x81,0x00,0x05,0x00,0x80,0x00,0x00,0x00,0xea,0x02,0x00,0x00, -0xe8,0x02,0x00,0x00,0xe9,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xf0,0x02,0x00,0x00,0xa4,0x00,0x00,0x00, -0x8b,0x04,0x00,0x00,0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00, -0xf2,0x02,0x00,0x00,0xea,0x02,0x00,0x00,0x00,0x00,0x00,0x00, -0x73,0x00,0x04,0x00,0x4d,0x00,0x00,0x00,0xf3,0x02,0x00,0x00, -0xf2,0x02,0x00,0x00,0x41,0x00,0x06,0x00,0x57,0x00,0x00,0x00, -0xf4,0x02,0x00,0x00,0x9d,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0xf0,0x02,0x00,0x00,0x3e,0x00,0x03,0x00,0xf4,0x02,0x00,0x00, -0xf3,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xfb,0x02,0x00,0x00,0xa4,0x00,0x00,0x00,0x8c,0x04,0x00,0x00, -0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00,0xfc,0x02,0x00,0x00, -0xea,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0x73,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0xfd,0x02,0x00,0x00,0xfc,0x02,0x00,0x00, -0x41,0x00,0x06,0x00,0x57,0x00,0x00,0x00,0xfe,0x02,0x00,0x00, -0x9d,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0xfb,0x02,0x00,0x00, -0x3e,0x00,0x03,0x00,0xfe,0x02,0x00,0x00,0xfd,0x02,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00,0x07,0x03,0x00,0x00, -0x58,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0x08,0x03,0x00,0x00,0x07,0x03,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0x0a,0x03,0x00,0x00,0x5d,0x00,0x00,0x00, -0x73,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0x0b,0x03,0x00,0x00, -0x0a,0x03,0x00,0x00,0x3d,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x0d,0x03,0x00,0x00,0x65,0x00,0x00,0x00,0xc2,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0x0e,0x03,0x00,0x00,0x0d,0x03,0x00,0x00, -0x8d,0x04,0x00,0x00,0xc4,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0x0f,0x03,0x00,0x00,0x0e,0x03,0x00,0x00,0x69,0x00,0x00,0x00, -0xc7,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0x10,0x03,0x00,0x00, -0x0f,0x03,0x00,0x00,0x4f,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x11,0x03,0x00,0x00,0x10,0x03,0x00,0x00, -0x3d,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x13,0x03,0x00,0x00, -0x65,0x00,0x00,0x00,0xc2,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0x15,0x03,0x00,0x00,0x13,0x03,0x00,0x00,0x86,0x04,0x00,0x00, -0xc7,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0x16,0x03,0x00,0x00, -0x15,0x03,0x00,0x00,0x4f,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x17,0x03,0x00,0x00,0x16,0x03,0x00,0x00, -0x41,0x00,0x08,0x00,0x7c,0x00,0x00,0x00,0x19,0x03,0x00,0x00, -0x55,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x7a,0x00,0x00,0x00,0x8d,0x04,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4e,0x00,0x00,0x00,0x1a,0x03,0x00,0x00,0x19,0x03,0x00,0x00, -0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x1b,0x03,0x00,0x00, -0x1a,0x03,0x00,0x00,0xc7,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0x1c,0x03,0x00,0x00,0x1b,0x03,0x00,0x00,0x84,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x1e,0x03,0x00,0x00, -0x11,0x03,0x00,0x00,0xc5,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0x1f,0x03,0x00,0x00,0x1c,0x03,0x00,0x00,0x1e,0x03,0x00,0x00, -0x70,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0x20,0x03,0x00,0x00, -0x1f,0x03,0x00,0x00,0xc2,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0x21,0x03,0x00,0x00,0x1b,0x03,0x00,0x00,0x69,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x23,0x03,0x00,0x00, -0x17,0x03,0x00,0x00,0xc5,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0x24,0x03,0x00,0x00,0x21,0x03,0x00,0x00,0x23,0x03,0x00,0x00, -0x70,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0x25,0x03,0x00,0x00, -0x24,0x03,0x00,0x00,0x50,0x00,0x05,0x00,0x80,0x00,0x00,0x00, -0x26,0x03,0x00,0x00,0x20,0x03,0x00,0x00,0x25,0x03,0x00,0x00, -0x8e,0x00,0x05,0x00,0x80,0x00,0x00,0x00,0x27,0x03,0x00,0x00, -0x26,0x03,0x00,0x00,0x08,0x03,0x00,0x00,0x50,0x00,0x05,0x00, -0x80,0x00,0x00,0x00,0x28,0x03,0x00,0x00,0x0b,0x03,0x00,0x00, -0x0b,0x03,0x00,0x00,0x81,0x00,0x05,0x00,0x80,0x00,0x00,0x00, -0x29,0x03,0x00,0x00,0x27,0x03,0x00,0x00,0x28,0x03,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x2f,0x03,0x00,0x00, -0xa4,0x00,0x00,0x00,0x8d,0x04,0x00,0x00,0x51,0x00,0x05,0x00, -0x4a,0x00,0x00,0x00,0x31,0x03,0x00,0x00,0x29,0x03,0x00,0x00, -0x00,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0x32,0x03,0x00,0x00,0x31,0x03,0x00,0x00,0x41,0x00,0x06,0x00, -0x57,0x00,0x00,0x00,0x33,0x03,0x00,0x00,0x9d,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x2f,0x03,0x00,0x00,0x3e,0x00,0x03,0x00, -0x33,0x03,0x00,0x00,0x32,0x03,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x3a,0x03,0x00,0x00,0xa4,0x00,0x00,0x00, -0x8e,0x04,0x00,0x00,0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00, -0x3b,0x03,0x00,0x00,0x29,0x03,0x00,0x00,0x01,0x00,0x00,0x00, -0x73,0x00,0x04,0x00,0x4d,0x00,0x00,0x00,0x3c,0x03,0x00,0x00, -0x3b,0x03,0x00,0x00,0x41,0x00,0x06,0x00,0x57,0x00,0x00,0x00, -0x3d,0x03,0x00,0x00,0x9d,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x3a,0x03,0x00,0x00,0x3e,0x00,0x03,0x00,0x3d,0x03,0x00,0x00, -0x3c,0x03,0x00,0x00,0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0x46,0x03,0x00,0x00,0x58,0x00,0x00,0x00,0x73,0x00,0x04,0x00, -0x4a,0x00,0x00,0x00,0x47,0x03,0x00,0x00,0x46,0x03,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00,0x49,0x03,0x00,0x00, -0x5d,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0x4a,0x03,0x00,0x00,0x49,0x03,0x00,0x00,0x3d,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x4c,0x03,0x00,0x00,0x65,0x00,0x00,0x00, -0xc2,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0x4d,0x03,0x00,0x00, -0x4c,0x03,0x00,0x00,0x8f,0x04,0x00,0x00,0xc4,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0x4e,0x03,0x00,0x00,0x4d,0x03,0x00,0x00, -0x69,0x00,0x00,0x00,0xc7,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0x4f,0x03,0x00,0x00,0x4e,0x03,0x00,0x00,0x4f,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x50,0x03,0x00,0x00, -0x4f,0x03,0x00,0x00,0x3d,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x52,0x03,0x00,0x00,0x65,0x00,0x00,0x00,0xc2,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0x54,0x03,0x00,0x00,0x52,0x03,0x00,0x00, -0x88,0x04,0x00,0x00,0xc7,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0x55,0x03,0x00,0x00,0x54,0x03,0x00,0x00,0x4f,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x56,0x03,0x00,0x00, -0x55,0x03,0x00,0x00,0x41,0x00,0x08,0x00,0x7c,0x00,0x00,0x00, -0x58,0x03,0x00,0x00,0x55,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0x7a,0x00,0x00,0x00,0x8f,0x04,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4e,0x00,0x00,0x00,0x59,0x03,0x00,0x00, -0x58,0x03,0x00,0x00,0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x5a,0x03,0x00,0x00,0x59,0x03,0x00,0x00,0xc7,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0x5b,0x03,0x00,0x00,0x5a,0x03,0x00,0x00, -0x84,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x5d,0x03,0x00,0x00,0x50,0x03,0x00,0x00,0xc5,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0x5e,0x03,0x00,0x00,0x5b,0x03,0x00,0x00, -0x5d,0x03,0x00,0x00,0x70,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0x5f,0x03,0x00,0x00,0x5e,0x03,0x00,0x00,0xc2,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0x60,0x03,0x00,0x00,0x5a,0x03,0x00,0x00, -0x69,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x62,0x03,0x00,0x00,0x56,0x03,0x00,0x00,0xc5,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0x63,0x03,0x00,0x00,0x60,0x03,0x00,0x00, -0x62,0x03,0x00,0x00,0x70,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0x64,0x03,0x00,0x00,0x63,0x03,0x00,0x00,0x50,0x00,0x05,0x00, -0x80,0x00,0x00,0x00,0x65,0x03,0x00,0x00,0x5f,0x03,0x00,0x00, -0x64,0x03,0x00,0x00,0x8e,0x00,0x05,0x00,0x80,0x00,0x00,0x00, -0x66,0x03,0x00,0x00,0x65,0x03,0x00,0x00,0x47,0x03,0x00,0x00, -0x50,0x00,0x05,0x00,0x80,0x00,0x00,0x00,0x67,0x03,0x00,0x00, -0x4a,0x03,0x00,0x00,0x4a,0x03,0x00,0x00,0x81,0x00,0x05,0x00, -0x80,0x00,0x00,0x00,0x68,0x03,0x00,0x00,0x66,0x03,0x00,0x00, -0x67,0x03,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x6e,0x03,0x00,0x00,0xa4,0x00,0x00,0x00,0x8f,0x04,0x00,0x00, -0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00,0x70,0x03,0x00,0x00, -0x68,0x03,0x00,0x00,0x00,0x00,0x00,0x00,0x73,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0x71,0x03,0x00,0x00,0x70,0x03,0x00,0x00, -0x41,0x00,0x06,0x00,0x57,0x00,0x00,0x00,0x72,0x03,0x00,0x00, -0x9d,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x6e,0x03,0x00,0x00, -0x3e,0x00,0x03,0x00,0x72,0x03,0x00,0x00,0x71,0x03,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x79,0x03,0x00,0x00, -0xa4,0x00,0x00,0x00,0x90,0x04,0x00,0x00,0x51,0x00,0x05,0x00, -0x4a,0x00,0x00,0x00,0x7a,0x03,0x00,0x00,0x68,0x03,0x00,0x00, -0x01,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0x7b,0x03,0x00,0x00,0x7a,0x03,0x00,0x00,0x41,0x00,0x06,0x00, -0x57,0x00,0x00,0x00,0x7c,0x03,0x00,0x00,0x9d,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x79,0x03,0x00,0x00,0x3e,0x00,0x03,0x00, -0x7c,0x03,0x00,0x00,0x7b,0x03,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0x85,0x03,0x00,0x00,0x58,0x00,0x00,0x00, -0x73,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0x86,0x03,0x00,0x00, -0x85,0x03,0x00,0x00,0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0x88,0x03,0x00,0x00,0x5d,0x00,0x00,0x00,0x73,0x00,0x04,0x00, -0x4a,0x00,0x00,0x00,0x89,0x03,0x00,0x00,0x88,0x03,0x00,0x00, -0x3d,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x8b,0x03,0x00,0x00, -0x65,0x00,0x00,0x00,0xc2,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0x8c,0x03,0x00,0x00,0x8b,0x03,0x00,0x00,0x71,0x00,0x00,0x00, -0xc4,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0x8d,0x03,0x00,0x00, -0x8c,0x03,0x00,0x00,0x69,0x00,0x00,0x00,0xc7,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0x8e,0x03,0x00,0x00,0x8d,0x03,0x00,0x00, -0x4f,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x8f,0x03,0x00,0x00,0x8e,0x03,0x00,0x00,0x3d,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x91,0x03,0x00,0x00,0x65,0x00,0x00,0x00, -0xc2,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0x93,0x03,0x00,0x00, -0x91,0x03,0x00,0x00,0x8a,0x04,0x00,0x00,0xc7,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0x94,0x03,0x00,0x00,0x93,0x03,0x00,0x00, -0x4f,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x95,0x03,0x00,0x00,0x94,0x03,0x00,0x00,0x41,0x00,0x08,0x00, -0x7c,0x00,0x00,0x00,0x97,0x03,0x00,0x00,0x55,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x7a,0x00,0x00,0x00, -0x71,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x4e,0x00,0x00,0x00, -0x98,0x03,0x00,0x00,0x97,0x03,0x00,0x00,0x71,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x99,0x03,0x00,0x00,0x98,0x03,0x00,0x00, -0xc7,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0x9a,0x03,0x00,0x00, -0x99,0x03,0x00,0x00,0x84,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x9c,0x03,0x00,0x00,0x8f,0x03,0x00,0x00, -0xc5,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0x9d,0x03,0x00,0x00, -0x9a,0x03,0x00,0x00,0x9c,0x03,0x00,0x00,0x70,0x00,0x04,0x00, -0x4a,0x00,0x00,0x00,0x9e,0x03,0x00,0x00,0x9d,0x03,0x00,0x00, -0xc2,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0x9f,0x03,0x00,0x00, -0x99,0x03,0x00,0x00,0x69,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0xa1,0x03,0x00,0x00,0x95,0x03,0x00,0x00, -0xc5,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0xa2,0x03,0x00,0x00, -0x9f,0x03,0x00,0x00,0xa1,0x03,0x00,0x00,0x70,0x00,0x04,0x00, -0x4a,0x00,0x00,0x00,0xa3,0x03,0x00,0x00,0xa2,0x03,0x00,0x00, -0x50,0x00,0x05,0x00,0x80,0x00,0x00,0x00,0xa4,0x03,0x00,0x00, -0x9e,0x03,0x00,0x00,0xa3,0x03,0x00,0x00,0x8e,0x00,0x05,0x00, -0x80,0x00,0x00,0x00,0xa5,0x03,0x00,0x00,0xa4,0x03,0x00,0x00, -0x86,0x03,0x00,0x00,0x50,0x00,0x05,0x00,0x80,0x00,0x00,0x00, -0xa6,0x03,0x00,0x00,0x89,0x03,0x00,0x00,0x89,0x03,0x00,0x00, -0x81,0x00,0x05,0x00,0x80,0x00,0x00,0x00,0xa7,0x03,0x00,0x00, -0xa5,0x03,0x00,0x00,0xa6,0x03,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xad,0x03,0x00,0x00,0xa4,0x00,0x00,0x00, -0x71,0x00,0x00,0x00,0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00, -0xaf,0x03,0x00,0x00,0xa7,0x03,0x00,0x00,0x00,0x00,0x00,0x00, -0x73,0x00,0x04,0x00,0x4d,0x00,0x00,0x00,0xb0,0x03,0x00,0x00, -0xaf,0x03,0x00,0x00,0x41,0x00,0x06,0x00,0x57,0x00,0x00,0x00, -0xb1,0x03,0x00,0x00,0x9d,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0xad,0x03,0x00,0x00,0x3e,0x00,0x03,0x00,0xb1,0x03,0x00,0x00, -0xb0,0x03,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xb8,0x03,0x00,0x00,0xa4,0x00,0x00,0x00,0x91,0x04,0x00,0x00, -0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00,0xb9,0x03,0x00,0x00, -0xa7,0x03,0x00,0x00,0x01,0x00,0x00,0x00,0x73,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0xba,0x03,0x00,0x00,0xb9,0x03,0x00,0x00, -0x41,0x00,0x06,0x00,0x57,0x00,0x00,0x00,0xbb,0x03,0x00,0x00, -0x9d,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0xb8,0x03,0x00,0x00, -0x3e,0x00,0x03,0x00,0xbb,0x03,0x00,0x00,0xba,0x03,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00,0xc4,0x03,0x00,0x00, -0x58,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0xc5,0x03,0x00,0x00,0xc4,0x03,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0xc7,0x03,0x00,0x00,0x5d,0x00,0x00,0x00, -0x73,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0xc8,0x03,0x00,0x00, -0xc7,0x03,0x00,0x00,0x3d,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0xca,0x03,0x00,0x00,0x65,0x00,0x00,0x00,0xc2,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0xcb,0x03,0x00,0x00,0xca,0x03,0x00,0x00, -0x7c,0x04,0x00,0x00,0xc4,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0xcc,0x03,0x00,0x00,0xcb,0x03,0x00,0x00,0x69,0x00,0x00,0x00, -0xc7,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0xcd,0x03,0x00,0x00, -0xcc,0x03,0x00,0x00,0x4f,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xce,0x03,0x00,0x00,0xcd,0x03,0x00,0x00, -0x3d,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0xd0,0x03,0x00,0x00, -0x65,0x00,0x00,0x00,0xc2,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0xd2,0x03,0x00,0x00,0xd0,0x03,0x00,0x00,0x8c,0x04,0x00,0x00, -0xc7,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0xd3,0x03,0x00,0x00, -0xd2,0x03,0x00,0x00,0x4f,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xd4,0x03,0x00,0x00,0xd3,0x03,0x00,0x00, -0x41,0x00,0x08,0x00,0x7c,0x00,0x00,0x00,0xd6,0x03,0x00,0x00, -0x55,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x7a,0x00,0x00,0x00,0x7c,0x04,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4e,0x00,0x00,0x00,0xd7,0x03,0x00,0x00,0xd6,0x03,0x00,0x00, -0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0xd8,0x03,0x00,0x00, -0xd7,0x03,0x00,0x00,0xc7,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0xd9,0x03,0x00,0x00,0xd8,0x03,0x00,0x00,0x84,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0xdb,0x03,0x00,0x00, -0xce,0x03,0x00,0x00,0xc5,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0xdc,0x03,0x00,0x00,0xd9,0x03,0x00,0x00,0xdb,0x03,0x00,0x00, -0x70,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0xdd,0x03,0x00,0x00, -0xdc,0x03,0x00,0x00,0xc2,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0xde,0x03,0x00,0x00,0xd8,0x03,0x00,0x00,0x69,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0xe0,0x03,0x00,0x00, -0xd4,0x03,0x00,0x00,0xc5,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0xe1,0x03,0x00,0x00,0xde,0x03,0x00,0x00,0xe0,0x03,0x00,0x00, -0x70,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0xe2,0x03,0x00,0x00, -0xe1,0x03,0x00,0x00,0x50,0x00,0x05,0x00,0x80,0x00,0x00,0x00, -0xe3,0x03,0x00,0x00,0xdd,0x03,0x00,0x00,0xe2,0x03,0x00,0x00, -0x8e,0x00,0x05,0x00,0x80,0x00,0x00,0x00,0xe4,0x03,0x00,0x00, -0xe3,0x03,0x00,0x00,0xc5,0x03,0x00,0x00,0x50,0x00,0x05,0x00, -0x80,0x00,0x00,0x00,0xe5,0x03,0x00,0x00,0xc8,0x03,0x00,0x00, -0xc8,0x03,0x00,0x00,0x81,0x00,0x05,0x00,0x80,0x00,0x00,0x00, -0xe6,0x03,0x00,0x00,0xe4,0x03,0x00,0x00,0xe5,0x03,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xec,0x03,0x00,0x00, -0xa4,0x00,0x00,0x00,0x7c,0x04,0x00,0x00,0x51,0x00,0x05,0x00, -0x4a,0x00,0x00,0x00,0xee,0x03,0x00,0x00,0xe6,0x03,0x00,0x00, -0x00,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0xef,0x03,0x00,0x00,0xee,0x03,0x00,0x00,0x41,0x00,0x06,0x00, -0x57,0x00,0x00,0x00,0xf0,0x03,0x00,0x00,0x9d,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0xec,0x03,0x00,0x00,0x3e,0x00,0x03,0x00, -0xf0,0x03,0x00,0x00,0xef,0x03,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xf7,0x03,0x00,0x00,0xa4,0x00,0x00,0x00, -0x92,0x04,0x00,0x00,0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00, -0xf8,0x03,0x00,0x00,0xe6,0x03,0x00,0x00,0x01,0x00,0x00,0x00, -0x73,0x00,0x04,0x00,0x4d,0x00,0x00,0x00,0xf9,0x03,0x00,0x00, -0xf8,0x03,0x00,0x00,0x41,0x00,0x06,0x00,0x57,0x00,0x00,0x00, -0xfa,0x03,0x00,0x00,0x9d,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0xf7,0x03,0x00,0x00,0x3e,0x00,0x03,0x00,0xfa,0x03,0x00,0x00, -0xf9,0x03,0x00,0x00,0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0x03,0x04,0x00,0x00,0x58,0x00,0x00,0x00,0x73,0x00,0x04,0x00, -0x4a,0x00,0x00,0x00,0x04,0x04,0x00,0x00,0x03,0x04,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00,0x06,0x04,0x00,0x00, -0x5d,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0x07,0x04,0x00,0x00,0x06,0x04,0x00,0x00,0x3d,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x09,0x04,0x00,0x00,0x65,0x00,0x00,0x00, -0xc2,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0x0a,0x04,0x00,0x00, -0x09,0x04,0x00,0x00,0x7e,0x04,0x00,0x00,0xc4,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0x0b,0x04,0x00,0x00,0x0a,0x04,0x00,0x00, -0x69,0x00,0x00,0x00,0xc7,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0x0c,0x04,0x00,0x00,0x0b,0x04,0x00,0x00,0x4f,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x0d,0x04,0x00,0x00, -0x0c,0x04,0x00,0x00,0x3d,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x0f,0x04,0x00,0x00,0x65,0x00,0x00,0x00,0xc2,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0x11,0x04,0x00,0x00,0x0f,0x04,0x00,0x00, -0x8e,0x04,0x00,0x00,0xc7,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0x12,0x04,0x00,0x00,0x11,0x04,0x00,0x00,0x4f,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x13,0x04,0x00,0x00, -0x12,0x04,0x00,0x00,0x41,0x00,0x08,0x00,0x7c,0x00,0x00,0x00, -0x15,0x04,0x00,0x00,0x55,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0x7a,0x00,0x00,0x00,0x7e,0x04,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4e,0x00,0x00,0x00,0x16,0x04,0x00,0x00, -0x15,0x04,0x00,0x00,0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x17,0x04,0x00,0x00,0x16,0x04,0x00,0x00,0xc7,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0x18,0x04,0x00,0x00,0x17,0x04,0x00,0x00, -0x84,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x1a,0x04,0x00,0x00,0x0d,0x04,0x00,0x00,0xc5,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0x1b,0x04,0x00,0x00,0x18,0x04,0x00,0x00, -0x1a,0x04,0x00,0x00,0x70,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0x1c,0x04,0x00,0x00,0x1b,0x04,0x00,0x00,0xc2,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0x1d,0x04,0x00,0x00,0x17,0x04,0x00,0x00, -0x69,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x1f,0x04,0x00,0x00,0x13,0x04,0x00,0x00,0xc5,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0x20,0x04,0x00,0x00,0x1d,0x04,0x00,0x00, -0x1f,0x04,0x00,0x00,0x70,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, -0x21,0x04,0x00,0x00,0x20,0x04,0x00,0x00,0x50,0x00,0x05,0x00, -0x80,0x00,0x00,0x00,0x22,0x04,0x00,0x00,0x1c,0x04,0x00,0x00, -0x21,0x04,0x00,0x00,0x8e,0x00,0x05,0x00,0x80,0x00,0x00,0x00, -0x23,0x04,0x00,0x00,0x22,0x04,0x00,0x00,0x04,0x04,0x00,0x00, -0x50,0x00,0x05,0x00,0x80,0x00,0x00,0x00,0x24,0x04,0x00,0x00, -0x07,0x04,0x00,0x00,0x07,0x04,0x00,0x00,0x81,0x00,0x05,0x00, -0x80,0x00,0x00,0x00,0x25,0x04,0x00,0x00,0x23,0x04,0x00,0x00, -0x24,0x04,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x2b,0x04,0x00,0x00,0xa4,0x00,0x00,0x00,0x7e,0x04,0x00,0x00, -0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00,0x2d,0x04,0x00,0x00, -0x25,0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x73,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0x2e,0x04,0x00,0x00,0x2d,0x04,0x00,0x00, -0x41,0x00,0x06,0x00,0x57,0x00,0x00,0x00,0x2f,0x04,0x00,0x00, -0x9d,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x2b,0x04,0x00,0x00, -0x3e,0x00,0x03,0x00,0x2f,0x04,0x00,0x00,0x2e,0x04,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x36,0x04,0x00,0x00, -0xa4,0x00,0x00,0x00,0x93,0x04,0x00,0x00,0x51,0x00,0x05,0x00, -0x4a,0x00,0x00,0x00,0x37,0x04,0x00,0x00,0x25,0x04,0x00,0x00, -0x01,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0x38,0x04,0x00,0x00,0x37,0x04,0x00,0x00,0x41,0x00,0x06,0x00, -0x57,0x00,0x00,0x00,0x39,0x04,0x00,0x00,0x9d,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x36,0x04,0x00,0x00,0x3e,0x00,0x03,0x00, -0x39,0x04,0x00,0x00,0x38,0x04,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0x42,0x04,0x00,0x00,0x58,0x00,0x00,0x00, -0x73,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0x43,0x04,0x00,0x00, -0x42,0x04,0x00,0x00,0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0x45,0x04,0x00,0x00,0x5d,0x00,0x00,0x00,0x73,0x00,0x04,0x00, -0x4a,0x00,0x00,0x00,0x46,0x04,0x00,0x00,0x45,0x04,0x00,0x00, -0x3d,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x48,0x04,0x00,0x00, -0x65,0x00,0x00,0x00,0xc2,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0x49,0x04,0x00,0x00,0x48,0x04,0x00,0x00,0x80,0x04,0x00,0x00, -0xc4,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0x4a,0x04,0x00,0x00, -0x49,0x04,0x00,0x00,0x69,0x00,0x00,0x00,0xc7,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0x4b,0x04,0x00,0x00,0x4a,0x04,0x00,0x00, -0x4f,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x4c,0x04,0x00,0x00,0x4b,0x04,0x00,0x00,0x3d,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x4e,0x04,0x00,0x00,0x65,0x00,0x00,0x00, -0xc2,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0x50,0x04,0x00,0x00, -0x4e,0x04,0x00,0x00,0x90,0x04,0x00,0x00,0xc7,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0x51,0x04,0x00,0x00,0x50,0x04,0x00,0x00, -0x4f,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x52,0x04,0x00,0x00,0x51,0x04,0x00,0x00,0x41,0x00,0x08,0x00, -0x7c,0x00,0x00,0x00,0x54,0x04,0x00,0x00,0x55,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x7a,0x00,0x00,0x00, -0x80,0x04,0x00,0x00,0x3d,0x00,0x04,0x00,0x4e,0x00,0x00,0x00, -0x55,0x04,0x00,0x00,0x54,0x04,0x00,0x00,0x71,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x56,0x04,0x00,0x00,0x55,0x04,0x00,0x00, -0xc7,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0x57,0x04,0x00,0x00, -0x56,0x04,0x00,0x00,0x84,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x59,0x04,0x00,0x00,0x4c,0x04,0x00,0x00, -0xc5,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0x5a,0x04,0x00,0x00, -0x57,0x04,0x00,0x00,0x59,0x04,0x00,0x00,0x70,0x00,0x04,0x00, -0x4a,0x00,0x00,0x00,0x5b,0x04,0x00,0x00,0x5a,0x04,0x00,0x00, -0xc2,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0x5c,0x04,0x00,0x00, -0x56,0x04,0x00,0x00,0x69,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x5e,0x04,0x00,0x00,0x52,0x04,0x00,0x00, -0xc5,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0x5f,0x04,0x00,0x00, -0x5c,0x04,0x00,0x00,0x5e,0x04,0x00,0x00,0x70,0x00,0x04,0x00, -0x4a,0x00,0x00,0x00,0x60,0x04,0x00,0x00,0x5f,0x04,0x00,0x00, -0x50,0x00,0x05,0x00,0x80,0x00,0x00,0x00,0x61,0x04,0x00,0x00, -0x5b,0x04,0x00,0x00,0x60,0x04,0x00,0x00,0x8e,0x00,0x05,0x00, -0x80,0x00,0x00,0x00,0x62,0x04,0x00,0x00,0x61,0x04,0x00,0x00, -0x43,0x04,0x00,0x00,0x50,0x00,0x05,0x00,0x80,0x00,0x00,0x00, -0x63,0x04,0x00,0x00,0x46,0x04,0x00,0x00,0x46,0x04,0x00,0x00, -0x81,0x00,0x05,0x00,0x80,0x00,0x00,0x00,0x64,0x04,0x00,0x00, -0x62,0x04,0x00,0x00,0x63,0x04,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x6a,0x04,0x00,0x00,0xa4,0x00,0x00,0x00, -0x80,0x04,0x00,0x00,0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00, -0x6c,0x04,0x00,0x00,0x64,0x04,0x00,0x00,0x00,0x00,0x00,0x00, -0x73,0x00,0x04,0x00,0x4d,0x00,0x00,0x00,0x6d,0x04,0x00,0x00, -0x6c,0x04,0x00,0x00,0x41,0x00,0x06,0x00,0x57,0x00,0x00,0x00, -0x6e,0x04,0x00,0x00,0x9d,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x6a,0x04,0x00,0x00,0x3e,0x00,0x03,0x00,0x6e,0x04,0x00,0x00, -0x6d,0x04,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x75,0x04,0x00,0x00,0xa4,0x00,0x00,0x00,0x94,0x04,0x00,0x00, -0x51,0x00,0x05,0x00,0x4a,0x00,0x00,0x00,0x76,0x04,0x00,0x00, -0x64,0x04,0x00,0x00,0x01,0x00,0x00,0x00,0x73,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0x77,0x04,0x00,0x00,0x76,0x04,0x00,0x00, -0x41,0x00,0x06,0x00,0x57,0x00,0x00,0x00,0x78,0x04,0x00,0x00, -0x9d,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x75,0x04,0x00,0x00, -0x3e,0x00,0x03,0x00,0x78,0x04,0x00,0x00,0x77,0x04,0x00,0x00, -0xf9,0x00,0x02,0x00,0xbe,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xbe,0x00,0x00,0x00,0xfd,0x00,0x01,0x00,0x38,0x00,0x01,0x00, - +0xa8,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xfb,0x00,0x03,0x00, +0x0c,0x00,0x00,0x00,0xa9,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xa9,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00, +0x0e,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, +0x0e,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x11,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x10,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00,0x13,0x00,0x00,0x00, +0x12,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x13,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x16,0x00,0x00,0x00, +0x14,0x00,0x00,0x00,0x15,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x17,0x00,0x00,0x00,0x11,0x00,0x00,0x00, +0x16,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x1b,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x15,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x1f,0x00,0x00,0x00, +0x1b,0x00,0x00,0x00,0x1e,0x00,0x00,0x00,0x89,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x1b,0x00,0x00,0x00, +0x1e,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x25,0x00,0x00,0x00,0x1e,0x00,0x00,0x00,0x17,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x27,0x00,0x00,0x00, +0x25,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x2e,0x00,0x00,0x00,0x2f,0x00,0x00,0x00,0x2b,0x00,0x00,0x00, +0x2d,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x30,0x00,0x00,0x00,0x2f,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x31,0x00,0x00,0x00,0x30,0x00,0x00,0x00, +0x1e,0x00,0x00,0x00,0xae,0x00,0x05,0x00,0x32,0x00,0x00,0x00, +0x33,0x00,0x00,0x00,0x27,0x00,0x00,0x00,0x31,0x00,0x00,0x00, +0xf7,0x00,0x03,0x00,0x35,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x33,0x00,0x00,0x00,0x34,0x00,0x00,0x00, +0x35,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x34,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xa8,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0x35,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x3a,0x00,0x00,0x00,0x38,0x00,0x00,0x00,0x17,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3c,0x00,0x00,0x00, +0x1e,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x3d,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, +0x3c,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x40,0x00,0x00,0x00,0x3e,0x00,0x00,0x00,0x1f,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x41,0x00,0x00,0x00, +0x3d,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x41,0x00,0x07,0x00, +0x50,0x00,0x00,0x00,0x51,0x00,0x00,0x00,0x4d,0x00,0x00,0x00, +0x4e,0x00,0x00,0x00,0x27,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x45,0x00,0x00,0x00,0x52,0x00,0x00,0x00, +0x51,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x42,0x00,0x00,0x00, +0x53,0x00,0x00,0x00,0x52,0x00,0x00,0x00,0x41,0x00,0x07,0x00, +0x50,0x00,0x00,0x00,0x57,0x00,0x00,0x00,0x4d,0x00,0x00,0x00, +0x4e,0x00,0x00,0x00,0x27,0x00,0x00,0x00,0x56,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x45,0x00,0x00,0x00,0x58,0x00,0x00,0x00, +0x57,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x42,0x00,0x00,0x00, +0x59,0x00,0x00,0x00,0x58,0x00,0x00,0x00,0x41,0x00,0x07,0x00, +0x5d,0x00,0x00,0x00,0x5e,0x00,0x00,0x00,0x4d,0x00,0x00,0x00, +0x4e,0x00,0x00,0x00,0x27,0x00,0x00,0x00,0x5c,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x5f,0x00,0x00,0x00, +0x5e,0x00,0x00,0x00,0x41,0x00,0x08,0x00,0x73,0x00,0x00,0x00, +0x74,0x00,0x00,0x00,0x4d,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, +0x27,0x00,0x00,0x00,0x71,0x00,0x00,0x00,0x40,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x46,0x00,0x00,0x00,0x75,0x00,0x00,0x00, +0x74,0x00,0x00,0x00,0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x76,0x00,0x00,0x00,0x75,0x00,0x00,0x00,0xc7,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x82,0x00,0x00,0x00,0x76,0x00,0x00,0x00, +0x81,0x00,0x00,0x00,0xc2,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x85,0x00,0x00,0x00,0x5f,0x00,0x00,0x00,0x40,0x00,0x00,0x00, +0xc4,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0x85,0x00,0x00,0x00,0x2d,0x00,0x00,0x00,0xc7,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x87,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0x47,0x00,0x00,0x00,0xc5,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x88,0x00,0x00,0x00,0x82,0x00,0x00,0x00,0x87,0x00,0x00,0x00, +0x70,0x00,0x04,0x00,0x42,0x00,0x00,0x00,0x89,0x00,0x00,0x00, +0x88,0x00,0x00,0x00,0x0c,0x00,0x08,0x00,0x42,0x00,0x00,0x00, +0x8c,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00, +0x53,0x00,0x00,0x00,0x89,0x00,0x00,0x00,0x59,0x00,0x00,0x00, +0x73,0x00,0x04,0x00,0x45,0x00,0x00,0x00,0x8d,0x00,0x00,0x00, +0x8c,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x50,0x00,0x00,0x00, +0x8e,0x00,0x00,0x00,0x7a,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, +0x41,0x00,0x00,0x00,0x3e,0x00,0x03,0x00,0x8e,0x00,0x00,0x00, +0x8d,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x92,0x00,0x00,0x00,0x41,0x00,0x00,0x00,0x47,0x00,0x00,0x00, +0xc2,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x95,0x00,0x00,0x00, +0x76,0x00,0x00,0x00,0x2d,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x99,0x00,0x00,0x00,0x40,0x00,0x00,0x00, +0x98,0x00,0x00,0x00,0xc2,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x9a,0x00,0x00,0x00,0x5f,0x00,0x00,0x00,0x99,0x00,0x00,0x00, +0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9b,0x00,0x00,0x00, +0x9a,0x00,0x00,0x00,0x47,0x00,0x00,0x00,0xc5,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x9c,0x00,0x00,0x00,0x95,0x00,0x00,0x00, +0x9b,0x00,0x00,0x00,0x70,0x00,0x04,0x00,0x42,0x00,0x00,0x00, +0x9d,0x00,0x00,0x00,0x9c,0x00,0x00,0x00,0x0c,0x00,0x08,0x00, +0x42,0x00,0x00,0x00,0xa0,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x32,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x9d,0x00,0x00,0x00, +0x59,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x45,0x00,0x00,0x00, +0xa1,0x00,0x00,0x00,0xa0,0x00,0x00,0x00,0x41,0x00,0x06,0x00, +0x50,0x00,0x00,0x00,0xa2,0x00,0x00,0x00,0x7a,0x00,0x00,0x00, +0x4e,0x00,0x00,0x00,0x92,0x00,0x00,0x00,0x3e,0x00,0x03,0x00, +0xa2,0x00,0x00,0x00,0xa1,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xb5,0x00,0x00,0x00,0x40,0x00,0x00,0x00, +0xa6,0x00,0x00,0x00,0x41,0x00,0x08,0x00,0x73,0x00,0x00,0x00, +0xb6,0x00,0x00,0x00,0x4d,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, +0x27,0x00,0x00,0x00,0x71,0x00,0x00,0x00,0xb5,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x46,0x00,0x00,0x00,0xb7,0x00,0x00,0x00, +0xb6,0x00,0x00,0x00,0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xb8,0x00,0x00,0x00,0xb7,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xb9,0x00,0x00,0x00,0x41,0x00,0x00,0x00, +0xa6,0x00,0x00,0x00,0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xbb,0x00,0x00,0x00,0xb8,0x00,0x00,0x00,0x81,0x00,0x00,0x00, +0xc2,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xbc,0x00,0x00,0x00, +0x5f,0x00,0x00,0x00,0xb5,0x00,0x00,0x00,0xc4,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xbd,0x00,0x00,0x00,0xbc,0x00,0x00,0x00, +0x2d,0x00,0x00,0x00,0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xbe,0x00,0x00,0x00,0xbd,0x00,0x00,0x00,0x47,0x00,0x00,0x00, +0xc5,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xbf,0x00,0x00,0x00, +0xbb,0x00,0x00,0x00,0xbe,0x00,0x00,0x00,0x70,0x00,0x04,0x00, +0x42,0x00,0x00,0x00,0xc0,0x00,0x00,0x00,0xbf,0x00,0x00,0x00, +0x0c,0x00,0x08,0x00,0x42,0x00,0x00,0x00,0xc2,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x53,0x00,0x00,0x00, +0xc0,0x00,0x00,0x00,0x59,0x00,0x00,0x00,0x73,0x00,0x04,0x00, +0x45,0x00,0x00,0x00,0xc3,0x00,0x00,0x00,0xc2,0x00,0x00,0x00, +0x41,0x00,0x06,0x00,0x50,0x00,0x00,0x00,0xc4,0x00,0x00,0x00, +0x7a,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0xb9,0x00,0x00,0x00, +0x3e,0x00,0x03,0x00,0xc4,0x00,0x00,0x00,0xc3,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xc6,0x00,0x00,0x00, +0x41,0x00,0x00,0x00,0xa6,0x01,0x00,0x00,0xc2,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xc7,0x00,0x00,0x00,0xb8,0x00,0x00,0x00, +0x2d,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xc8,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0xa7,0x01,0x00,0x00, +0xc2,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xc9,0x00,0x00,0x00, +0x5f,0x00,0x00,0x00,0xc8,0x00,0x00,0x00,0xc7,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xca,0x00,0x00,0x00,0xc9,0x00,0x00,0x00, +0x47,0x00,0x00,0x00,0xc5,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xcb,0x00,0x00,0x00,0xc7,0x00,0x00,0x00,0xca,0x00,0x00,0x00, +0x70,0x00,0x04,0x00,0x42,0x00,0x00,0x00,0xcc,0x00,0x00,0x00, +0xcb,0x00,0x00,0x00,0x0c,0x00,0x08,0x00,0x42,0x00,0x00,0x00, +0xce,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00, +0x53,0x00,0x00,0x00,0xcc,0x00,0x00,0x00,0x59,0x00,0x00,0x00, +0x73,0x00,0x04,0x00,0x45,0x00,0x00,0x00,0xcf,0x00,0x00,0x00, +0xce,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x50,0x00,0x00,0x00, +0xd0,0x00,0x00,0x00,0x7a,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, +0xc6,0x00,0x00,0x00,0x3e,0x00,0x03,0x00,0xd0,0x00,0x00,0x00, +0xcf,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xd8,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0xa8,0x01,0x00,0x00, +0x41,0x00,0x08,0x00,0x73,0x00,0x00,0x00,0xd9,0x00,0x00,0x00, +0x4d,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x27,0x00,0x00,0x00, +0x71,0x00,0x00,0x00,0xd8,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x46,0x00,0x00,0x00,0xda,0x00,0x00,0x00,0xd9,0x00,0x00,0x00, +0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xdb,0x00,0x00,0x00, +0xda,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xdc,0x00,0x00,0x00,0x41,0x00,0x00,0x00,0xa8,0x01,0x00,0x00, +0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xde,0x00,0x00,0x00, +0xdb,0x00,0x00,0x00,0x81,0x00,0x00,0x00,0xc2,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xdf,0x00,0x00,0x00,0x5f,0x00,0x00,0x00, +0xd8,0x00,0x00,0x00,0xc4,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xe0,0x00,0x00,0x00,0xdf,0x00,0x00,0x00,0x2d,0x00,0x00,0x00, +0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xe1,0x00,0x00,0x00, +0xe0,0x00,0x00,0x00,0x47,0x00,0x00,0x00,0xc5,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xe2,0x00,0x00,0x00,0xde,0x00,0x00,0x00, +0xe1,0x00,0x00,0x00,0x70,0x00,0x04,0x00,0x42,0x00,0x00,0x00, +0xe3,0x00,0x00,0x00,0xe2,0x00,0x00,0x00,0x0c,0x00,0x08,0x00, +0x42,0x00,0x00,0x00,0xe5,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x32,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0xe3,0x00,0x00,0x00, +0x59,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x45,0x00,0x00,0x00, +0xe6,0x00,0x00,0x00,0xe5,0x00,0x00,0x00,0x41,0x00,0x06,0x00, +0x50,0x00,0x00,0x00,0xe7,0x00,0x00,0x00,0x7a,0x00,0x00,0x00, +0x4e,0x00,0x00,0x00,0xdc,0x00,0x00,0x00,0x3e,0x00,0x03,0x00, +0xe7,0x00,0x00,0x00,0xe6,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xe9,0x00,0x00,0x00,0x41,0x00,0x00,0x00, +0xa9,0x01,0x00,0x00,0xc2,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xea,0x00,0x00,0x00,0xdb,0x00,0x00,0x00,0x2d,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xeb,0x00,0x00,0x00, +0x40,0x00,0x00,0x00,0xaa,0x01,0x00,0x00,0xc2,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xec,0x00,0x00,0x00,0x5f,0x00,0x00,0x00, +0xeb,0x00,0x00,0x00,0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xed,0x00,0x00,0x00,0xec,0x00,0x00,0x00,0x47,0x00,0x00,0x00, +0xc5,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xee,0x00,0x00,0x00, +0xea,0x00,0x00,0x00,0xed,0x00,0x00,0x00,0x70,0x00,0x04,0x00, +0x42,0x00,0x00,0x00,0xef,0x00,0x00,0x00,0xee,0x00,0x00,0x00, +0x0c,0x00,0x08,0x00,0x42,0x00,0x00,0x00,0xf1,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x53,0x00,0x00,0x00, +0xef,0x00,0x00,0x00,0x59,0x00,0x00,0x00,0x73,0x00,0x04,0x00, +0x45,0x00,0x00,0x00,0xf2,0x00,0x00,0x00,0xf1,0x00,0x00,0x00, +0x41,0x00,0x06,0x00,0x50,0x00,0x00,0x00,0xf3,0x00,0x00,0x00, +0x7a,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0xe9,0x00,0x00,0x00, +0x3e,0x00,0x03,0x00,0xf3,0x00,0x00,0x00,0xf2,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xfb,0x00,0x00,0x00, +0x40,0x00,0x00,0x00,0xab,0x01,0x00,0x00,0x41,0x00,0x08,0x00, +0x73,0x00,0x00,0x00,0xfc,0x00,0x00,0x00,0x4d,0x00,0x00,0x00, +0x4e,0x00,0x00,0x00,0x27,0x00,0x00,0x00,0x71,0x00,0x00,0x00, +0xfb,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x46,0x00,0x00,0x00, +0xfd,0x00,0x00,0x00,0xfc,0x00,0x00,0x00,0x71,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xfe,0x00,0x00,0x00,0xfd,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xff,0x00,0x00,0x00, +0x41,0x00,0x00,0x00,0xab,0x01,0x00,0x00,0xc7,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x01,0x01,0x00,0x00,0xfe,0x00,0x00,0x00, +0x81,0x00,0x00,0x00,0xc2,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x02,0x01,0x00,0x00,0x5f,0x00,0x00,0x00,0xfb,0x00,0x00,0x00, +0xc4,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x03,0x01,0x00,0x00, +0x02,0x01,0x00,0x00,0x2d,0x00,0x00,0x00,0xc7,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x04,0x01,0x00,0x00,0x03,0x01,0x00,0x00, +0x47,0x00,0x00,0x00,0xc5,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x05,0x01,0x00,0x00,0x01,0x01,0x00,0x00,0x04,0x01,0x00,0x00, +0x70,0x00,0x04,0x00,0x42,0x00,0x00,0x00,0x06,0x01,0x00,0x00, +0x05,0x01,0x00,0x00,0x0c,0x00,0x08,0x00,0x42,0x00,0x00,0x00, +0x08,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00, +0x53,0x00,0x00,0x00,0x06,0x01,0x00,0x00,0x59,0x00,0x00,0x00, +0x73,0x00,0x04,0x00,0x45,0x00,0x00,0x00,0x09,0x01,0x00,0x00, +0x08,0x01,0x00,0x00,0x41,0x00,0x06,0x00,0x50,0x00,0x00,0x00, +0x0a,0x01,0x00,0x00,0x7a,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, +0xff,0x00,0x00,0x00,0x3e,0x00,0x03,0x00,0x0a,0x01,0x00,0x00, +0x09,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x0c,0x01,0x00,0x00,0x41,0x00,0x00,0x00,0xac,0x01,0x00,0x00, +0xc2,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x0d,0x01,0x00,0x00, +0xfe,0x00,0x00,0x00,0x2d,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x0e,0x01,0x00,0x00,0x40,0x00,0x00,0x00, +0x81,0x00,0x00,0x00,0xc2,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x0f,0x01,0x00,0x00,0x5f,0x00,0x00,0x00,0x0e,0x01,0x00,0x00, +0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x10,0x01,0x00,0x00, +0x0f,0x01,0x00,0x00,0x47,0x00,0x00,0x00,0xc5,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x11,0x01,0x00,0x00,0x0d,0x01,0x00,0x00, +0x10,0x01,0x00,0x00,0x70,0x00,0x04,0x00,0x42,0x00,0x00,0x00, +0x12,0x01,0x00,0x00,0x11,0x01,0x00,0x00,0x0c,0x00,0x08,0x00, +0x42,0x00,0x00,0x00,0x14,0x01,0x00,0x00,0x01,0x00,0x00,0x00, +0x32,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x12,0x01,0x00,0x00, +0x59,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x45,0x00,0x00,0x00, +0x15,0x01,0x00,0x00,0x14,0x01,0x00,0x00,0x41,0x00,0x06,0x00, +0x50,0x00,0x00,0x00,0x16,0x01,0x00,0x00,0x7a,0x00,0x00,0x00, +0x4e,0x00,0x00,0x00,0x0c,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, +0x16,0x01,0x00,0x00,0x15,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x1e,0x01,0x00,0x00,0x40,0x00,0x00,0x00, +0x10,0x00,0x00,0x00,0x41,0x00,0x08,0x00,0x73,0x00,0x00,0x00, +0x1f,0x01,0x00,0x00,0x4d,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, +0x27,0x00,0x00,0x00,0x71,0x00,0x00,0x00,0x1e,0x01,0x00,0x00, +0x3d,0x00,0x04,0x00,0x46,0x00,0x00,0x00,0x20,0x01,0x00,0x00, +0x1f,0x01,0x00,0x00,0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x21,0x01,0x00,0x00,0x20,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x22,0x01,0x00,0x00,0x41,0x00,0x00,0x00, +0x10,0x00,0x00,0x00,0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x24,0x01,0x00,0x00,0x21,0x01,0x00,0x00,0x81,0x00,0x00,0x00, +0xc2,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x25,0x01,0x00,0x00, +0x5f,0x00,0x00,0x00,0x1e,0x01,0x00,0x00,0xc4,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x26,0x01,0x00,0x00,0x25,0x01,0x00,0x00, +0x2d,0x00,0x00,0x00,0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x27,0x01,0x00,0x00,0x26,0x01,0x00,0x00,0x47,0x00,0x00,0x00, +0xc5,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x28,0x01,0x00,0x00, +0x24,0x01,0x00,0x00,0x27,0x01,0x00,0x00,0x70,0x00,0x04,0x00, +0x42,0x00,0x00,0x00,0x29,0x01,0x00,0x00,0x28,0x01,0x00,0x00, +0x0c,0x00,0x08,0x00,0x42,0x00,0x00,0x00,0x2b,0x01,0x00,0x00, +0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x53,0x00,0x00,0x00, +0x29,0x01,0x00,0x00,0x59,0x00,0x00,0x00,0x73,0x00,0x04,0x00, +0x45,0x00,0x00,0x00,0x2c,0x01,0x00,0x00,0x2b,0x01,0x00,0x00, +0x41,0x00,0x06,0x00,0x50,0x00,0x00,0x00,0x2d,0x01,0x00,0x00, +0x7a,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x22,0x01,0x00,0x00, +0x3e,0x00,0x03,0x00,0x2d,0x01,0x00,0x00,0x2c,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x2f,0x01,0x00,0x00, +0x41,0x00,0x00,0x00,0xad,0x01,0x00,0x00,0xc2,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x30,0x01,0x00,0x00,0x21,0x01,0x00,0x00, +0x2d,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x31,0x01,0x00,0x00,0x40,0x00,0x00,0x00,0x47,0x00,0x00,0x00, +0xc2,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x32,0x01,0x00,0x00, +0x5f,0x00,0x00,0x00,0x31,0x01,0x00,0x00,0xc7,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x33,0x01,0x00,0x00,0x32,0x01,0x00,0x00, +0x47,0x00,0x00,0x00,0xc5,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x34,0x01,0x00,0x00,0x30,0x01,0x00,0x00,0x33,0x01,0x00,0x00, +0x70,0x00,0x04,0x00,0x42,0x00,0x00,0x00,0x35,0x01,0x00,0x00, +0x34,0x01,0x00,0x00,0x0c,0x00,0x08,0x00,0x42,0x00,0x00,0x00, +0x37,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00, +0x53,0x00,0x00,0x00,0x35,0x01,0x00,0x00,0x59,0x00,0x00,0x00, +0x73,0x00,0x04,0x00,0x45,0x00,0x00,0x00,0x38,0x01,0x00,0x00, +0x37,0x01,0x00,0x00,0x41,0x00,0x06,0x00,0x50,0x00,0x00,0x00, +0x39,0x01,0x00,0x00,0x7a,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, +0x2f,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x39,0x01,0x00,0x00, +0x38,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x41,0x01,0x00,0x00,0x40,0x00,0x00,0x00,0xae,0x01,0x00,0x00, +0x41,0x00,0x08,0x00,0x73,0x00,0x00,0x00,0x42,0x01,0x00,0x00, +0x4d,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x27,0x00,0x00,0x00, +0x71,0x00,0x00,0x00,0x41,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, +0x46,0x00,0x00,0x00,0x43,0x01,0x00,0x00,0x42,0x01,0x00,0x00, +0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x44,0x01,0x00,0x00, +0x43,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x45,0x01,0x00,0x00,0x41,0x00,0x00,0x00,0xae,0x01,0x00,0x00, +0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x47,0x01,0x00,0x00, +0x44,0x01,0x00,0x00,0x81,0x00,0x00,0x00,0xc2,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x48,0x01,0x00,0x00,0x5f,0x00,0x00,0x00, +0x41,0x01,0x00,0x00,0xc4,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x49,0x01,0x00,0x00,0x48,0x01,0x00,0x00,0x2d,0x00,0x00,0x00, +0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x4a,0x01,0x00,0x00, +0x49,0x01,0x00,0x00,0x47,0x00,0x00,0x00,0xc5,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x4b,0x01,0x00,0x00,0x47,0x01,0x00,0x00, +0x4a,0x01,0x00,0x00,0x70,0x00,0x04,0x00,0x42,0x00,0x00,0x00, +0x4c,0x01,0x00,0x00,0x4b,0x01,0x00,0x00,0x0c,0x00,0x08,0x00, +0x42,0x00,0x00,0x00,0x4e,0x01,0x00,0x00,0x01,0x00,0x00,0x00, +0x32,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x4c,0x01,0x00,0x00, +0x59,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x45,0x00,0x00,0x00, +0x4f,0x01,0x00,0x00,0x4e,0x01,0x00,0x00,0x41,0x00,0x06,0x00, +0x50,0x00,0x00,0x00,0x50,0x01,0x00,0x00,0x7a,0x00,0x00,0x00, +0x4e,0x00,0x00,0x00,0x45,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, +0x50,0x01,0x00,0x00,0x4f,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x52,0x01,0x00,0x00,0x41,0x00,0x00,0x00, +0xaf,0x01,0x00,0x00,0xc2,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x53,0x01,0x00,0x00,0x44,0x01,0x00,0x00,0x2d,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x54,0x01,0x00,0x00, +0x40,0x00,0x00,0x00,0xa6,0x01,0x00,0x00,0xc2,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x55,0x01,0x00,0x00,0x5f,0x00,0x00,0x00, +0x54,0x01,0x00,0x00,0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x56,0x01,0x00,0x00,0x55,0x01,0x00,0x00,0x47,0x00,0x00,0x00, +0xc5,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x57,0x01,0x00,0x00, +0x53,0x01,0x00,0x00,0x56,0x01,0x00,0x00,0x70,0x00,0x04,0x00, +0x42,0x00,0x00,0x00,0x58,0x01,0x00,0x00,0x57,0x01,0x00,0x00, +0x0c,0x00,0x08,0x00,0x42,0x00,0x00,0x00,0x5a,0x01,0x00,0x00, +0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x53,0x00,0x00,0x00, +0x58,0x01,0x00,0x00,0x59,0x00,0x00,0x00,0x73,0x00,0x04,0x00, +0x45,0x00,0x00,0x00,0x5b,0x01,0x00,0x00,0x5a,0x01,0x00,0x00, +0x41,0x00,0x06,0x00,0x50,0x00,0x00,0x00,0x5c,0x01,0x00,0x00, +0x7a,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x52,0x01,0x00,0x00, +0x3e,0x00,0x03,0x00,0x5c,0x01,0x00,0x00,0x5b,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x64,0x01,0x00,0x00, +0x40,0x00,0x00,0x00,0xb0,0x01,0x00,0x00,0x41,0x00,0x08,0x00, +0x73,0x00,0x00,0x00,0x65,0x01,0x00,0x00,0x4d,0x00,0x00,0x00, +0x4e,0x00,0x00,0x00,0x27,0x00,0x00,0x00,0x71,0x00,0x00,0x00, +0x64,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0x46,0x00,0x00,0x00, +0x66,0x01,0x00,0x00,0x65,0x01,0x00,0x00,0x71,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x67,0x01,0x00,0x00,0x66,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x68,0x01,0x00,0x00, +0x41,0x00,0x00,0x00,0xb0,0x01,0x00,0x00,0xc7,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x6a,0x01,0x00,0x00,0x67,0x01,0x00,0x00, +0x81,0x00,0x00,0x00,0xc2,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x6b,0x01,0x00,0x00,0x5f,0x00,0x00,0x00,0x64,0x01,0x00,0x00, +0xc4,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x6c,0x01,0x00,0x00, +0x6b,0x01,0x00,0x00,0x2d,0x00,0x00,0x00,0xc7,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x6d,0x01,0x00,0x00,0x6c,0x01,0x00,0x00, +0x47,0x00,0x00,0x00,0xc5,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x6e,0x01,0x00,0x00,0x6a,0x01,0x00,0x00,0x6d,0x01,0x00,0x00, +0x70,0x00,0x04,0x00,0x42,0x00,0x00,0x00,0x6f,0x01,0x00,0x00, +0x6e,0x01,0x00,0x00,0x0c,0x00,0x08,0x00,0x42,0x00,0x00,0x00, +0x71,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00, +0x53,0x00,0x00,0x00,0x6f,0x01,0x00,0x00,0x59,0x00,0x00,0x00, +0x73,0x00,0x04,0x00,0x45,0x00,0x00,0x00,0x72,0x01,0x00,0x00, +0x71,0x01,0x00,0x00,0x41,0x00,0x06,0x00,0x50,0x00,0x00,0x00, +0x73,0x01,0x00,0x00,0x7a,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, +0x68,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x73,0x01,0x00,0x00, +0x72,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x75,0x01,0x00,0x00,0x41,0x00,0x00,0x00,0xb1,0x01,0x00,0x00, +0xc2,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x76,0x01,0x00,0x00, +0x67,0x01,0x00,0x00,0x2d,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x77,0x01,0x00,0x00,0x40,0x00,0x00,0x00, +0xa9,0x01,0x00,0x00,0xc2,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x78,0x01,0x00,0x00,0x5f,0x00,0x00,0x00,0x77,0x01,0x00,0x00, +0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x79,0x01,0x00,0x00, +0x78,0x01,0x00,0x00,0x47,0x00,0x00,0x00,0xc5,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x7a,0x01,0x00,0x00,0x76,0x01,0x00,0x00, +0x79,0x01,0x00,0x00,0x70,0x00,0x04,0x00,0x42,0x00,0x00,0x00, +0x7b,0x01,0x00,0x00,0x7a,0x01,0x00,0x00,0x0c,0x00,0x08,0x00, +0x42,0x00,0x00,0x00,0x7d,0x01,0x00,0x00,0x01,0x00,0x00,0x00, +0x32,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x7b,0x01,0x00,0x00, +0x59,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x45,0x00,0x00,0x00, +0x7e,0x01,0x00,0x00,0x7d,0x01,0x00,0x00,0x41,0x00,0x06,0x00, +0x50,0x00,0x00,0x00,0x7f,0x01,0x00,0x00,0x7a,0x00,0x00,0x00, +0x4e,0x00,0x00,0x00,0x75,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, +0x7f,0x01,0x00,0x00,0x7e,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x87,0x01,0x00,0x00,0x40,0x00,0x00,0x00, +0xb2,0x01,0x00,0x00,0x41,0x00,0x08,0x00,0x73,0x00,0x00,0x00, +0x88,0x01,0x00,0x00,0x4d,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, +0x27,0x00,0x00,0x00,0x71,0x00,0x00,0x00,0x87,0x01,0x00,0x00, +0x3d,0x00,0x04,0x00,0x46,0x00,0x00,0x00,0x89,0x01,0x00,0x00, +0x88,0x01,0x00,0x00,0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x8a,0x01,0x00,0x00,0x89,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x8b,0x01,0x00,0x00,0x41,0x00,0x00,0x00, +0xb2,0x01,0x00,0x00,0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x8d,0x01,0x00,0x00,0x8a,0x01,0x00,0x00,0x81,0x00,0x00,0x00, +0xc2,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8e,0x01,0x00,0x00, +0x5f,0x00,0x00,0x00,0x87,0x01,0x00,0x00,0xc4,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x8f,0x01,0x00,0x00,0x8e,0x01,0x00,0x00, +0x2d,0x00,0x00,0x00,0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x90,0x01,0x00,0x00,0x8f,0x01,0x00,0x00,0x47,0x00,0x00,0x00, +0xc5,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x91,0x01,0x00,0x00, +0x8d,0x01,0x00,0x00,0x90,0x01,0x00,0x00,0x70,0x00,0x04,0x00, +0x42,0x00,0x00,0x00,0x92,0x01,0x00,0x00,0x91,0x01,0x00,0x00, +0x0c,0x00,0x08,0x00,0x42,0x00,0x00,0x00,0x94,0x01,0x00,0x00, +0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x53,0x00,0x00,0x00, +0x92,0x01,0x00,0x00,0x59,0x00,0x00,0x00,0x73,0x00,0x04,0x00, +0x45,0x00,0x00,0x00,0x95,0x01,0x00,0x00,0x94,0x01,0x00,0x00, +0x41,0x00,0x06,0x00,0x50,0x00,0x00,0x00,0x96,0x01,0x00,0x00, +0x7a,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x8b,0x01,0x00,0x00, +0x3e,0x00,0x03,0x00,0x96,0x01,0x00,0x00,0x95,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x98,0x01,0x00,0x00, +0x41,0x00,0x00,0x00,0xb3,0x01,0x00,0x00,0xc2,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x99,0x01,0x00,0x00,0x8a,0x01,0x00,0x00, +0x2d,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x9a,0x01,0x00,0x00,0x40,0x00,0x00,0x00,0xac,0x01,0x00,0x00, +0xc2,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9b,0x01,0x00,0x00, +0x5f,0x00,0x00,0x00,0x9a,0x01,0x00,0x00,0xc7,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x9c,0x01,0x00,0x00,0x9b,0x01,0x00,0x00, +0x47,0x00,0x00,0x00,0xc5,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x9d,0x01,0x00,0x00,0x99,0x01,0x00,0x00,0x9c,0x01,0x00,0x00, +0x70,0x00,0x04,0x00,0x42,0x00,0x00,0x00,0x9e,0x01,0x00,0x00, +0x9d,0x01,0x00,0x00,0x0c,0x00,0x08,0x00,0x42,0x00,0x00,0x00, +0xa0,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00, +0x53,0x00,0x00,0x00,0x9e,0x01,0x00,0x00,0x59,0x00,0x00,0x00, +0x73,0x00,0x04,0x00,0x45,0x00,0x00,0x00,0xa1,0x01,0x00,0x00, +0xa0,0x01,0x00,0x00,0x41,0x00,0x06,0x00,0x50,0x00,0x00,0x00, +0xa2,0x01,0x00,0x00,0x7a,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, +0x98,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0xa2,0x01,0x00,0x00, +0xa1,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xa8,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xa8,0x00,0x00,0x00,0xfd,0x00,0x01,0x00, +0x38,0x00,0x01,0x00, }; -const uint64_t dequant_q5_1_len = 13548; +const uint64_t dequant_q5_1_len = 6412; unsigned char dequant_q5_K_data[] = { 0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, -0xa0,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, +0x9e,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, 0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x27,0x00,0x00,0x00, 0x11,0x00,0x02,0x00,0x51,0x11,0x00,0x00,0x11,0x00,0x02,0x00, 0x60,0x11,0x00,0x00,0x0b,0x00,0x06,0x00,0x01,0x00,0x00,0x00, @@ -6161,1606 +5304,1262 @@ unsigned char dequant_q5_K_data[] = { 0x00,0x00,0x00,0x00,0x0e,0x00,0x03,0x00,0x00,0x00,0x00,0x00, 0x01,0x00,0x00,0x00,0x0f,0x00,0x0a,0x00,0x05,0x00,0x00,0x00, 0x04,0x00,0x00,0x00,0x6d,0x61,0x69,0x6e,0x00,0x00,0x00,0x00, -0x17,0x00,0x00,0x00,0x25,0x00,0x00,0x00,0x33,0x00,0x00,0x00, -0x52,0x00,0x00,0x00,0x15,0x01,0x00,0x00,0x10,0x00,0x06,0x00, +0x16,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x30,0x00,0x00,0x00, +0x4e,0x00,0x00,0x00,0x11,0x01,0x00,0x00,0x10,0x00,0x06,0x00, 0x04,0x00,0x00,0x00,0x11,0x00,0x00,0x00,0x40,0x00,0x00,0x00, 0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x17,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x1a,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x16,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x1a,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x1e,0x00,0x00,0x00,0x00,0x00,0x00,0x00, 0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x23,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x23,0x00,0x00,0x00, +0x1e,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x1e,0x00,0x00,0x00, 0x02,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x08,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x23,0x00,0x00,0x00,0x03,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x47,0x00,0x03,0x00, -0x23,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x33,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x1b,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x49,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x4b,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x4e,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x4e,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x4e,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x10,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x4e,0x00,0x00,0x00,0x03,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x30,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x4f,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0xb0,0x00,0x00,0x00, -0x48,0x00,0x04,0x00,0x50,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x50,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x03,0x00,0x50,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x52,0x00,0x00,0x00,0x22,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x52,0x00,0x00,0x00, -0x21,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x12,0x01,0x00,0x00,0x06,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x48,0x00,0x04,0x00,0x13,0x01,0x00,0x00,0x00,0x00,0x00,0x00, -0x19,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x13,0x01,0x00,0x00, -0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x03,0x00,0x13,0x01,0x00,0x00,0x02,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x15,0x01,0x00,0x00,0x22,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x15,0x01,0x00,0x00, -0x21,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x89,0x01,0x00,0x00,0x0b,0x00,0x00,0x00,0x19,0x00,0x00,0x00, -0x13,0x00,0x02,0x00,0x02,0x00,0x00,0x00,0x21,0x00,0x03,0x00, -0x03,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x15,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x10,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x14,0x00,0x02,0x00, -0x11,0x00,0x00,0x00,0x15,0x00,0x04,0x00,0x14,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x17,0x00,0x04,0x00, -0x15,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x03,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x16,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x15,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x16,0x00,0x00,0x00, -0x17,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x14,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x19,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x14,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x14,0x00,0x00,0x00, -0x1c,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x1e,0x00,0x06,0x00, -0x23,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x24,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x24,0x00,0x00,0x00,0x25,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x26,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x29,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x16,0x00,0x00,0x00,0x33,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x3f,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x16,0x00,0x03,0x00,0x42,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x16,0x00,0x03,0x00,0x45,0x00,0x00,0x00,0x10,0x00,0x00,0x00, -0x17,0x00,0x04,0x00,0x46,0x00,0x00,0x00,0x45,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x15,0x00,0x04,0x00,0x47,0x00,0x00,0x00, -0x08,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x14,0x00,0x00,0x00,0x48,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x1c,0x00,0x04,0x00,0x49,0x00,0x00,0x00,0x47,0x00,0x00,0x00, -0x48,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x14,0x00,0x00,0x00, -0x4a,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x1c,0x00,0x04,0x00, -0x4b,0x00,0x00,0x00,0x47,0x00,0x00,0x00,0x4a,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0x4c,0x00,0x00,0x00, -0x80,0x00,0x00,0x00,0x1c,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0x47,0x00,0x00,0x00,0x4c,0x00,0x00,0x00,0x1e,0x00,0x06,0x00, -0x4e,0x00,0x00,0x00,0x46,0x00,0x00,0x00,0x49,0x00,0x00,0x00, -0x4b,0x00,0x00,0x00,0x4d,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, -0x4f,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, -0x50,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x51,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x50,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x51,0x00,0x00,0x00,0x52,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x54,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x45,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x14,0x00,0x00,0x00,0x5a,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x61,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x69,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x73,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x7b,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x47,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x80,0x00,0x00,0x00,0x3f,0x00,0x00,0x00,0x15,0x00,0x04,0x00, -0x82,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x98,0x00,0x00,0x00, -0x0f,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x9f,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xce,0x00,0x00,0x00,0x05,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xe2,0x00,0x00,0x00, -0x03,0x00,0x00,0x00,0x1d,0x00,0x03,0x00,0x12,0x01,0x00,0x00, -0x45,0x00,0x00,0x00,0x1e,0x00,0x03,0x00,0x13,0x01,0x00,0x00, -0x12,0x01,0x00,0x00,0x20,0x00,0x04,0x00,0x14,0x01,0x00,0x00, -0x0c,0x00,0x00,0x00,0x13,0x01,0x00,0x00,0x3b,0x00,0x04,0x00, -0x14,0x01,0x00,0x00,0x15,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x69,0x01,0x00,0x00, -0x21,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x14,0x00,0x00,0x00, -0x88,0x01,0x00,0x00,0x40,0x00,0x00,0x00,0x2c,0x00,0x06,0x00, -0x15,0x00,0x00,0x00,0x89,0x01,0x00,0x00,0x88,0x01,0x00,0x00, -0x5a,0x00,0x00,0x00,0x5a,0x00,0x00,0x00,0x2a,0x00,0x03,0x00, -0x11,0x00,0x00,0x00,0x8c,0x01,0x00,0x00,0x29,0x00,0x03,0x00, -0x11,0x00,0x00,0x00,0x8f,0x01,0x00,0x00,0x36,0x00,0x05,0x00, -0x02,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x03,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x05,0x00,0x00,0x00, -0xf7,0x00,0x03,0x00,0x8a,0x01,0x00,0x00,0x00,0x00,0x00,0x00, -0xfb,0x00,0x03,0x00,0x18,0x00,0x00,0x00,0x8b,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x8b,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0x0a,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x0a,0x00,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x92,0x01,0x00,0x00, -0x09,0x00,0x00,0x00,0x8b,0x01,0x00,0x00,0x87,0x01,0x00,0x00, -0x0d,0x00,0x00,0x00,0xb1,0x00,0x05,0x00,0x11,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x92,0x01,0x00,0x00,0x10,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0x0c,0x00,0x00,0x00,0x0d,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x12,0x00,0x00,0x00, -0x0b,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x0b,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x19,0x00,0x00,0x00, -0x1a,0x00,0x00,0x00,0x17,0x00,0x00,0x00,0x18,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0x1b,0x00,0x00,0x00, -0x1a,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x14,0x00,0x00,0x00, -0x1d,0x00,0x00,0x00,0x1b,0x00,0x00,0x00,0x1c,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0x1f,0x00,0x00,0x00, -0x92,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x14,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x1d,0x00,0x00,0x00,0x1f,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x26,0x00,0x00,0x00, -0x27,0x00,0x00,0x00,0x25,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x28,0x00,0x00,0x00, -0x27,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x26,0x00,0x00,0x00, -0x2a,0x00,0x00,0x00,0x25,0x00,0x00,0x00,0x29,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x2b,0x00,0x00,0x00, -0x2a,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x2c,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0x2b,0x00,0x00,0x00, -0x87,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x2d,0x00,0x00,0x00, -0x2c,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0xaf,0x00,0x05,0x00, -0x11,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x2d,0x00,0x00,0x00,0xf7,0x00,0x03,0x00,0x30,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x2e,0x00,0x00,0x00, -0x2f,0x00,0x00,0x00,0x30,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x2f,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x0c,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0x30,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x19,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0x33,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x14,0x00,0x00,0x00, -0x35,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x36,0x00,0x00,0x00,0x35,0x00,0x00,0x00, -0x87,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, -0x36,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x8b,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x3d,0x00,0x00,0x00,0x36,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x41,0x00,0x00,0x00,0x3f,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, -0x41,0x00,0x08,0x00,0x54,0x00,0x00,0x00,0x55,0x00,0x00,0x00, -0x52,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x45,0x00,0x00,0x00,0x56,0x00,0x00,0x00,0x55,0x00,0x00,0x00, -0x73,0x00,0x04,0x00,0x42,0x00,0x00,0x00,0x57,0x00,0x00,0x00, -0x56,0x00,0x00,0x00,0x41,0x00,0x08,0x00,0x54,0x00,0x00,0x00, -0x5b,0x00,0x00,0x00,0x52,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x21,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x5a,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x45,0x00,0x00,0x00,0x5c,0x00,0x00,0x00, -0x5b,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x42,0x00,0x00,0x00, -0x5d,0x00,0x00,0x00,0x5c,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x10,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x63,0x00,0x00,0x00,0x61,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x64,0x00,0x00,0x00, -0x60,0x00,0x00,0x00,0x63,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x66,0x00,0x00,0x00,0x3f,0x00,0x00,0x00, -0x3d,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x67,0x00,0x00,0x00,0x64,0x00,0x00,0x00,0x66,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x6b,0x00,0x00,0x00, -0x69,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x6e,0x00,0x00,0x00,0x6b,0x00,0x00,0x00, -0x66,0x00,0x00,0x00,0xb1,0x00,0x05,0x00,0x11,0x00,0x00,0x00, -0x74,0x00,0x00,0x00,0x41,0x00,0x00,0x00,0x73,0x00,0x00,0x00, -0xf7,0x00,0x03,0x00,0x76,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x74,0x00,0x00,0x00,0x75,0x00,0x00,0x00, -0x90,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x75,0x00,0x00,0x00, -0x41,0x00,0x08,0x00,0x7b,0x00,0x00,0x00,0x7c,0x00,0x00,0x00, -0x52,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x29,0x00,0x00,0x00,0x41,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x47,0x00,0x00,0x00,0x7d,0x00,0x00,0x00,0x7c,0x00,0x00,0x00, -0x71,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0x7e,0x00,0x00,0x00, -0x7d,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x7f,0x00,0x00,0x00,0x7e,0x00,0x00,0x00,0xc7,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x81,0x00,0x00,0x00,0x7f,0x00,0x00,0x00, -0x80,0x00,0x00,0x00,0x72,0x00,0x04,0x00,0x82,0x00,0x00,0x00, -0x83,0x00,0x00,0x00,0x81,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x47,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x83,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x88,0x00,0x00,0x00, -0x41,0x00,0x00,0x00,0x73,0x00,0x00,0x00,0x41,0x00,0x08,0x00, -0x7b,0x00,0x00,0x00,0x89,0x00,0x00,0x00,0x52,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x29,0x00,0x00,0x00, -0x88,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x47,0x00,0x00,0x00, -0x8a,0x00,0x00,0x00,0x89,0x00,0x00,0x00,0x71,0x00,0x04,0x00, -0x14,0x00,0x00,0x00,0x8b,0x00,0x00,0x00,0x8a,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x8c,0x00,0x00,0x00, -0x8b,0x00,0x00,0x00,0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x8d,0x00,0x00,0x00,0x8c,0x00,0x00,0x00,0x80,0x00,0x00,0x00, -0x72,0x00,0x04,0x00,0x82,0x00,0x00,0x00,0x8e,0x00,0x00,0x00, -0x8d,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x47,0x00,0x00,0x00, -0x8f,0x00,0x00,0x00,0x8e,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x76,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x90,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x93,0x00,0x00,0x00, -0x41,0x00,0x00,0x00,0x73,0x00,0x00,0x00,0x41,0x00,0x08,0x00, -0x7b,0x00,0x00,0x00,0x94,0x00,0x00,0x00,0x52,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x29,0x00,0x00,0x00, -0x93,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x47,0x00,0x00,0x00, -0x95,0x00,0x00,0x00,0x94,0x00,0x00,0x00,0x71,0x00,0x04,0x00, -0x14,0x00,0x00,0x00,0x96,0x00,0x00,0x00,0x95,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x97,0x00,0x00,0x00, -0x96,0x00,0x00,0x00,0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x99,0x00,0x00,0x00,0x97,0x00,0x00,0x00,0x98,0x00,0x00,0x00, -0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9c,0x00,0x00,0x00, -0x41,0x00,0x00,0x00,0x73,0x00,0x00,0x00,0x41,0x00,0x08,0x00, -0x7b,0x00,0x00,0x00,0x9d,0x00,0x00,0x00,0x52,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x29,0x00,0x00,0x00, -0x9c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x47,0x00,0x00,0x00, -0x9e,0x00,0x00,0x00,0x9d,0x00,0x00,0x00,0xc2,0x00,0x05,0x00, -0x47,0x00,0x00,0x00,0xa0,0x00,0x00,0x00,0x9e,0x00,0x00,0x00, -0x9f,0x00,0x00,0x00,0xc4,0x00,0x05,0x00,0x47,0x00,0x00,0x00, -0xa1,0x00,0x00,0x00,0xa0,0x00,0x00,0x00,0x73,0x00,0x00,0x00, -0x71,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0xa2,0x00,0x00,0x00, -0xa1,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xa3,0x00,0x00,0x00,0xa2,0x00,0x00,0x00,0xc5,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xa4,0x00,0x00,0x00,0x99,0x00,0x00,0x00, -0xa3,0x00,0x00,0x00,0x72,0x00,0x04,0x00,0x82,0x00,0x00,0x00, -0xa5,0x00,0x00,0x00,0xa4,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x47,0x00,0x00,0x00,0xa6,0x00,0x00,0x00,0xa5,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x47,0x00,0x00,0x00,0xab,0x00,0x00,0x00, -0x94,0x00,0x00,0x00,0xc2,0x00,0x05,0x00,0x47,0x00,0x00,0x00, -0xac,0x00,0x00,0x00,0xab,0x00,0x00,0x00,0x73,0x00,0x00,0x00, -0x41,0x00,0x08,0x00,0x7b,0x00,0x00,0x00,0xaf,0x00,0x00,0x00, -0x52,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x29,0x00,0x00,0x00,0x41,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x47,0x00,0x00,0x00,0xb0,0x00,0x00,0x00,0xaf,0x00,0x00,0x00, -0xc2,0x00,0x05,0x00,0x47,0x00,0x00,0x00,0xb1,0x00,0x00,0x00, -0xb0,0x00,0x00,0x00,0x9f,0x00,0x00,0x00,0xc4,0x00,0x05,0x00, -0x47,0x00,0x00,0x00,0xb2,0x00,0x00,0x00,0xb1,0x00,0x00,0x00, -0x73,0x00,0x00,0x00,0xc5,0x00,0x05,0x00,0x47,0x00,0x00,0x00, -0xb3,0x00,0x00,0x00,0xac,0x00,0x00,0x00,0xb2,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x76,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x76,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x47,0x00,0x00,0x00, -0x94,0x01,0x00,0x00,0x8f,0x00,0x00,0x00,0x75,0x00,0x00,0x00, -0xb3,0x00,0x00,0x00,0x90,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, -0x47,0x00,0x00,0x00,0x93,0x01,0x00,0x00,0x84,0x00,0x00,0x00, -0x75,0x00,0x00,0x00,0xa6,0x00,0x00,0x00,0x90,0x00,0x00,0x00, -0x70,0x00,0x04,0x00,0x42,0x00,0x00,0x00,0xb7,0x00,0x00,0x00, -0x93,0x01,0x00,0x00,0x85,0x00,0x05,0x00,0x42,0x00,0x00,0x00, -0xb8,0x00,0x00,0x00,0x57,0x00,0x00,0x00,0xb7,0x00,0x00,0x00, -0x70,0x00,0x04,0x00,0x42,0x00,0x00,0x00,0xbc,0x00,0x00,0x00, -0x94,0x01,0x00,0x00,0x85,0x00,0x05,0x00,0x42,0x00,0x00,0x00, -0xbd,0x00,0x00,0x00,0x5d,0x00,0x00,0x00,0xbc,0x00,0x00,0x00, -0xf7,0x00,0x03,0x00,0xc1,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x74,0x00,0x00,0x00,0xc0,0x00,0x00,0x00, -0xd7,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xc0,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xc4,0x00,0x00,0x00, -0x41,0x00,0x00,0x00,0x29,0x00,0x00,0x00,0x41,0x00,0x08,0x00, -0x7b,0x00,0x00,0x00,0xc5,0x00,0x00,0x00,0x52,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x29,0x00,0x00,0x00, -0xc4,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x47,0x00,0x00,0x00, -0xc6,0x00,0x00,0x00,0xc5,0x00,0x00,0x00,0x71,0x00,0x04,0x00, -0x14,0x00,0x00,0x00,0xc7,0x00,0x00,0x00,0xc6,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xc8,0x00,0x00,0x00, -0xc7,0x00,0x00,0x00,0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xc9,0x00,0x00,0x00,0xc8,0x00,0x00,0x00,0x80,0x00,0x00,0x00, -0x72,0x00,0x04,0x00,0x82,0x00,0x00,0x00,0xca,0x00,0x00,0x00, -0xc9,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x47,0x00,0x00,0x00, -0xcb,0x00,0x00,0x00,0xca,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xcf,0x00,0x00,0x00,0x41,0x00,0x00,0x00, -0xce,0x00,0x00,0x00,0x41,0x00,0x08,0x00,0x7b,0x00,0x00,0x00, -0xd0,0x00,0x00,0x00,0x52,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x21,0x00,0x00,0x00,0x29,0x00,0x00,0x00,0xcf,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x47,0x00,0x00,0x00,0xd1,0x00,0x00,0x00, -0xd0,0x00,0x00,0x00,0x71,0x00,0x04,0x00,0x14,0x00,0x00,0x00, -0xd2,0x00,0x00,0x00,0xd1,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xd3,0x00,0x00,0x00,0xd2,0x00,0x00,0x00, -0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xd4,0x00,0x00,0x00, -0xd3,0x00,0x00,0x00,0x80,0x00,0x00,0x00,0x72,0x00,0x04,0x00, -0x82,0x00,0x00,0x00,0xd5,0x00,0x00,0x00,0xd4,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x47,0x00,0x00,0x00,0xd6,0x00,0x00,0x00, -0xd5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xc1,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xd7,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xda,0x00,0x00,0x00,0x41,0x00,0x00,0x00, -0xce,0x00,0x00,0x00,0x41,0x00,0x08,0x00,0x7b,0x00,0x00,0x00, -0xdb,0x00,0x00,0x00,0x52,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x21,0x00,0x00,0x00,0x29,0x00,0x00,0x00,0xda,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x47,0x00,0x00,0x00,0xdc,0x00,0x00,0x00, -0xdb,0x00,0x00,0x00,0x71,0x00,0x04,0x00,0x14,0x00,0x00,0x00, -0xdd,0x00,0x00,0x00,0xdc,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xde,0x00,0x00,0x00,0xdd,0x00,0x00,0x00, -0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xdf,0x00,0x00,0x00, -0xde,0x00,0x00,0x00,0x98,0x00,0x00,0x00,0x82,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xe3,0x00,0x00,0x00,0x41,0x00,0x00,0x00, -0xe2,0x00,0x00,0x00,0x41,0x00,0x08,0x00,0x7b,0x00,0x00,0x00, -0xe4,0x00,0x00,0x00,0x52,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x21,0x00,0x00,0x00,0x29,0x00,0x00,0x00,0xe3,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x47,0x00,0x00,0x00,0xe5,0x00,0x00,0x00, -0xe4,0x00,0x00,0x00,0xc2,0x00,0x05,0x00,0x47,0x00,0x00,0x00, -0xe6,0x00,0x00,0x00,0xe5,0x00,0x00,0x00,0x9f,0x00,0x00,0x00, -0xc4,0x00,0x05,0x00,0x47,0x00,0x00,0x00,0xe7,0x00,0x00,0x00, -0xe6,0x00,0x00,0x00,0x73,0x00,0x00,0x00,0x71,0x00,0x04,0x00, -0x14,0x00,0x00,0x00,0xe8,0x00,0x00,0x00,0xe7,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xe9,0x00,0x00,0x00, -0xe8,0x00,0x00,0x00,0xc5,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xea,0x00,0x00,0x00,0xdf,0x00,0x00,0x00,0xe9,0x00,0x00,0x00, -0x72,0x00,0x04,0x00,0x82,0x00,0x00,0x00,0xeb,0x00,0x00,0x00, -0xea,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x47,0x00,0x00,0x00, -0xec,0x00,0x00,0x00,0xeb,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x47,0x00,0x00,0x00,0xf1,0x00,0x00,0x00,0xdb,0x00,0x00,0x00, -0xc2,0x00,0x05,0x00,0x47,0x00,0x00,0x00,0xf2,0x00,0x00,0x00, -0xf1,0x00,0x00,0x00,0x73,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xf5,0x00,0x00,0x00,0x41,0x00,0x00,0x00, -0x29,0x00,0x00,0x00,0x41,0x00,0x08,0x00,0x7b,0x00,0x00,0x00, -0xf6,0x00,0x00,0x00,0x52,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x21,0x00,0x00,0x00,0x29,0x00,0x00,0x00,0xf5,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x47,0x00,0x00,0x00,0xf7,0x00,0x00,0x00, -0xf6,0x00,0x00,0x00,0xc2,0x00,0x05,0x00,0x47,0x00,0x00,0x00, -0xf8,0x00,0x00,0x00,0xf7,0x00,0x00,0x00,0x9f,0x00,0x00,0x00, -0xc4,0x00,0x05,0x00,0x47,0x00,0x00,0x00,0xf9,0x00,0x00,0x00, -0xf8,0x00,0x00,0x00,0x73,0x00,0x00,0x00,0xc5,0x00,0x05,0x00, -0x47,0x00,0x00,0x00,0xfa,0x00,0x00,0x00,0xf2,0x00,0x00,0x00, -0xf9,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xc1,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xc1,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, -0x47,0x00,0x00,0x00,0x96,0x01,0x00,0x00,0xd6,0x00,0x00,0x00, -0xc0,0x00,0x00,0x00,0xfa,0x00,0x00,0x00,0xd7,0x00,0x00,0x00, -0xf5,0x00,0x07,0x00,0x47,0x00,0x00,0x00,0x95,0x01,0x00,0x00, -0xcb,0x00,0x00,0x00,0xc0,0x00,0x00,0x00,0xec,0x00,0x00,0x00, -0xd7,0x00,0x00,0x00,0x70,0x00,0x04,0x00,0x42,0x00,0x00,0x00, -0xfe,0x00,0x00,0x00,0x95,0x01,0x00,0x00,0x85,0x00,0x05,0x00, -0x42,0x00,0x00,0x00,0xff,0x00,0x00,0x00,0x57,0x00,0x00,0x00, -0xfe,0x00,0x00,0x00,0x70,0x00,0x04,0x00,0x42,0x00,0x00,0x00, -0x03,0x01,0x00,0x00,0x96,0x01,0x00,0x00,0x85,0x00,0x05,0x00, -0x42,0x00,0x00,0x00,0x04,0x01,0x00,0x00,0x5d,0x00,0x00,0x00, -0x03,0x01,0x00,0x00,0xc4,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x08,0x01,0x00,0x00,0x29,0x00,0x00,0x00,0x41,0x00,0x00,0x00, -0x72,0x00,0x04,0x00,0x82,0x00,0x00,0x00,0x09,0x01,0x00,0x00, -0x08,0x01,0x00,0x00,0x7c,0x00,0x04,0x00,0x47,0x00,0x00,0x00, -0x0a,0x01,0x00,0x00,0x09,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x0e,0x01,0x00,0x00,0x41,0x00,0x00,0x00, -0x29,0x00,0x00,0x00,0xc4,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x0f,0x01,0x00,0x00,0x29,0x00,0x00,0x00,0x0e,0x01,0x00,0x00, -0x72,0x00,0x04,0x00,0x82,0x00,0x00,0x00,0x10,0x01,0x00,0x00, -0x0f,0x01,0x00,0x00,0x7c,0x00,0x04,0x00,0x47,0x00,0x00,0x00, -0x11,0x01,0x00,0x00,0x10,0x01,0x00,0x00,0x41,0x00,0x08,0x00, -0x7b,0x00,0x00,0x00,0x1a,0x01,0x00,0x00,0x52,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0xe2,0x00,0x00,0x00, -0x6e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x47,0x00,0x00,0x00, -0x1b,0x01,0x00,0x00,0x1a,0x01,0x00,0x00,0x71,0x00,0x04,0x00, -0x14,0x00,0x00,0x00,0x1c,0x01,0x00,0x00,0x1b,0x01,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x1d,0x01,0x00,0x00, -0x1c,0x01,0x00,0x00,0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x1e,0x01,0x00,0x00,0x1d,0x01,0x00,0x00,0x98,0x00,0x00,0x00, -0x41,0x00,0x08,0x00,0x7b,0x00,0x00,0x00,0x21,0x01,0x00,0x00, -0x52,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x3f,0x00,0x00,0x00,0x66,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x47,0x00,0x00,0x00,0x22,0x01,0x00,0x00,0x21,0x01,0x00,0x00, -0xc7,0x00,0x05,0x00,0x47,0x00,0x00,0x00,0x24,0x01,0x00,0x00, -0x22,0x01,0x00,0x00,0x0a,0x01,0x00,0x00,0x71,0x00,0x04,0x00, -0x14,0x00,0x00,0x00,0x25,0x01,0x00,0x00,0x24,0x01,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x26,0x01,0x00,0x00, -0x25,0x01,0x00,0x00,0xab,0x00,0x05,0x00,0x11,0x00,0x00,0x00, -0x27,0x01,0x00,0x00,0x26,0x01,0x00,0x00,0x09,0x00,0x00,0x00, -0xa9,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x28,0x01,0x00,0x00, -0x27,0x01,0x00,0x00,0x39,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x29,0x01,0x00,0x00, -0x1e,0x01,0x00,0x00,0x28,0x01,0x00,0x00,0x6f,0x00,0x04,0x00, -0x42,0x00,0x00,0x00,0x2a,0x01,0x00,0x00,0x29,0x01,0x00,0x00, -0x7f,0x00,0x04,0x00,0x42,0x00,0x00,0x00,0x9d,0x01,0x00,0x00, -0xbd,0x00,0x00,0x00,0x0c,0x00,0x08,0x00,0x42,0x00,0x00,0x00, -0x2d,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00, -0xb8,0x00,0x00,0x00,0x2a,0x01,0x00,0x00,0x9d,0x01,0x00,0x00, -0x73,0x00,0x04,0x00,0x45,0x00,0x00,0x00,0x2e,0x01,0x00,0x00, -0x2d,0x01,0x00,0x00,0x41,0x00,0x06,0x00,0x54,0x00,0x00,0x00, -0x2f,0x01,0x00,0x00,0x15,0x01,0x00,0x00,0x09,0x00,0x00,0x00, -0x67,0x00,0x00,0x00,0x3e,0x00,0x03,0x00,0x2f,0x01,0x00,0x00, -0x2e,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x31,0x01,0x00,0x00,0x67,0x00,0x00,0x00,0x29,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x35,0x01,0x00,0x00, -0x6e,0x00,0x00,0x00,0x29,0x00,0x00,0x00,0x41,0x00,0x08,0x00, -0x7b,0x00,0x00,0x00,0x36,0x01,0x00,0x00,0x52,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0xe2,0x00,0x00,0x00, -0x35,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0x47,0x00,0x00,0x00, -0x37,0x01,0x00,0x00,0x36,0x01,0x00,0x00,0x71,0x00,0x04,0x00, -0x14,0x00,0x00,0x00,0x38,0x01,0x00,0x00,0x37,0x01,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x39,0x01,0x00,0x00, -0x38,0x01,0x00,0x00,0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x3a,0x01,0x00,0x00,0x39,0x01,0x00,0x00,0x98,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3d,0x01,0x00,0x00, -0x66,0x00,0x00,0x00,0x29,0x00,0x00,0x00,0x41,0x00,0x08,0x00, -0x7b,0x00,0x00,0x00,0x3e,0x01,0x00,0x00,0x52,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x3f,0x00,0x00,0x00, -0x3d,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0x47,0x00,0x00,0x00, -0x3f,0x01,0x00,0x00,0x3e,0x01,0x00,0x00,0xc7,0x00,0x05,0x00, -0x47,0x00,0x00,0x00,0x41,0x01,0x00,0x00,0x3f,0x01,0x00,0x00, -0x0a,0x01,0x00,0x00,0x71,0x00,0x04,0x00,0x14,0x00,0x00,0x00, -0x42,0x01,0x00,0x00,0x41,0x01,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x43,0x01,0x00,0x00,0x42,0x01,0x00,0x00, -0xab,0x00,0x05,0x00,0x11,0x00,0x00,0x00,0x44,0x01,0x00,0x00, -0x43,0x01,0x00,0x00,0x09,0x00,0x00,0x00,0xa9,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x45,0x01,0x00,0x00,0x44,0x01,0x00,0x00, -0x39,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x46,0x01,0x00,0x00,0x3a,0x01,0x00,0x00, -0x45,0x01,0x00,0x00,0x6f,0x00,0x04,0x00,0x42,0x00,0x00,0x00, -0x47,0x01,0x00,0x00,0x46,0x01,0x00,0x00,0x0c,0x00,0x08,0x00, -0x42,0x00,0x00,0x00,0x4a,0x01,0x00,0x00,0x01,0x00,0x00,0x00, -0x32,0x00,0x00,0x00,0xb8,0x00,0x00,0x00,0x47,0x01,0x00,0x00, -0x9d,0x01,0x00,0x00,0x73,0x00,0x04,0x00,0x45,0x00,0x00,0x00, -0x4b,0x01,0x00,0x00,0x4a,0x01,0x00,0x00,0x41,0x00,0x06,0x00, -0x54,0x00,0x00,0x00,0x4c,0x01,0x00,0x00,0x15,0x01,0x00,0x00, -0x09,0x00,0x00,0x00,0x31,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0x4c,0x01,0x00,0x00,0x4b,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x4e,0x01,0x00,0x00,0x67,0x00,0x00,0x00, -0x69,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x47,0x00,0x00,0x00, -0x53,0x01,0x00,0x00,0x1a,0x01,0x00,0x00,0xc2,0x00,0x05,0x00, -0x47,0x00,0x00,0x00,0x54,0x01,0x00,0x00,0x53,0x01,0x00,0x00, -0x73,0x00,0x00,0x00,0x71,0x00,0x04,0x00,0x14,0x00,0x00,0x00, -0x55,0x01,0x00,0x00,0x54,0x01,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x56,0x01,0x00,0x00,0x55,0x01,0x00,0x00, -0x3d,0x00,0x04,0x00,0x47,0x00,0x00,0x00,0x5a,0x01,0x00,0x00, -0x21,0x01,0x00,0x00,0xc7,0x00,0x05,0x00,0x47,0x00,0x00,0x00, -0x5c,0x01,0x00,0x00,0x5a,0x01,0x00,0x00,0x11,0x01,0x00,0x00, -0x71,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0x5d,0x01,0x00,0x00, -0x5c,0x01,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x5e,0x01,0x00,0x00,0x5d,0x01,0x00,0x00,0xab,0x00,0x05,0x00, -0x11,0x00,0x00,0x00,0x5f,0x01,0x00,0x00,0x5e,0x01,0x00,0x00, -0x09,0x00,0x00,0x00,0xa9,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x60,0x01,0x00,0x00,0x5f,0x01,0x00,0x00,0x39,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x61,0x01,0x00,0x00,0x56,0x01,0x00,0x00,0x60,0x01,0x00,0x00, -0x6f,0x00,0x04,0x00,0x42,0x00,0x00,0x00,0x62,0x01,0x00,0x00, -0x61,0x01,0x00,0x00,0x7f,0x00,0x04,0x00,0x42,0x00,0x00,0x00, -0x9f,0x01,0x00,0x00,0x04,0x01,0x00,0x00,0x0c,0x00,0x08,0x00, -0x42,0x00,0x00,0x00,0x65,0x01,0x00,0x00,0x01,0x00,0x00,0x00, -0x32,0x00,0x00,0x00,0xff,0x00,0x00,0x00,0x62,0x01,0x00,0x00, -0x9f,0x01,0x00,0x00,0x73,0x00,0x04,0x00,0x45,0x00,0x00,0x00, -0x66,0x01,0x00,0x00,0x65,0x01,0x00,0x00,0x41,0x00,0x06,0x00, -0x54,0x00,0x00,0x00,0x67,0x01,0x00,0x00,0x15,0x01,0x00,0x00, -0x09,0x00,0x00,0x00,0x4e,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0x67,0x01,0x00,0x00,0x66,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x6a,0x01,0x00,0x00,0x67,0x00,0x00,0x00, -0x69,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0x47,0x00,0x00,0x00, -0x70,0x01,0x00,0x00,0x36,0x01,0x00,0x00,0xc2,0x00,0x05,0x00, -0x47,0x00,0x00,0x00,0x71,0x01,0x00,0x00,0x70,0x01,0x00,0x00, -0x73,0x00,0x00,0x00,0x71,0x00,0x04,0x00,0x14,0x00,0x00,0x00, -0x72,0x01,0x00,0x00,0x71,0x01,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x73,0x01,0x00,0x00,0x72,0x01,0x00,0x00, -0x3d,0x00,0x04,0x00,0x47,0x00,0x00,0x00,0x78,0x01,0x00,0x00, -0x3e,0x01,0x00,0x00,0xc7,0x00,0x05,0x00,0x47,0x00,0x00,0x00, -0x7a,0x01,0x00,0x00,0x78,0x01,0x00,0x00,0x11,0x01,0x00,0x00, -0x71,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0x7b,0x01,0x00,0x00, -0x7a,0x01,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x7c,0x01,0x00,0x00,0x7b,0x01,0x00,0x00,0xab,0x00,0x05,0x00, -0x11,0x00,0x00,0x00,0x7d,0x01,0x00,0x00,0x7c,0x01,0x00,0x00, -0x09,0x00,0x00,0x00,0xa9,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x7e,0x01,0x00,0x00,0x7d,0x01,0x00,0x00,0x39,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x7f,0x01,0x00,0x00,0x73,0x01,0x00,0x00,0x7e,0x01,0x00,0x00, -0x6f,0x00,0x04,0x00,0x42,0x00,0x00,0x00,0x80,0x01,0x00,0x00, -0x7f,0x01,0x00,0x00,0x0c,0x00,0x08,0x00,0x42,0x00,0x00,0x00, -0x83,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00, -0xff,0x00,0x00,0x00,0x80,0x01,0x00,0x00,0x9f,0x01,0x00,0x00, -0x73,0x00,0x04,0x00,0x45,0x00,0x00,0x00,0x84,0x01,0x00,0x00, -0x83,0x01,0x00,0x00,0x41,0x00,0x06,0x00,0x54,0x00,0x00,0x00, -0x85,0x01,0x00,0x00,0x15,0x01,0x00,0x00,0x09,0x00,0x00,0x00, -0x6a,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x85,0x01,0x00,0x00, -0x84,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x0d,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0x0d,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x87,0x01,0x00,0x00,0x92,0x01,0x00,0x00, -0x29,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x0a,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0x0c,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, -0x11,0x00,0x00,0x00,0x99,0x01,0x00,0x00,0x8c,0x01,0x00,0x00, -0x0a,0x00,0x00,0x00,0x8f,0x01,0x00,0x00,0x2f,0x00,0x00,0x00, -0xf7,0x00,0x03,0x00,0x90,0x01,0x00,0x00,0x00,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x99,0x01,0x00,0x00,0x8a,0x01,0x00,0x00, -0x90,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x90,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0x8a,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x8a,0x01,0x00,0x00,0xfd,0x00,0x01,0x00,0x38,0x00,0x01,0x00, - -}; -const uint64_t dequant_q5_K_len = 5988; - -unsigned char dequant_q6_K_data[] = { -0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, -0x10,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, -0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x27,0x00,0x00,0x00, -0x11,0x00,0x02,0x00,0x51,0x11,0x00,0x00,0x11,0x00,0x02,0x00, -0x60,0x11,0x00,0x00,0x0b,0x00,0x06,0x00,0x01,0x00,0x00,0x00, -0x47,0x4c,0x53,0x4c,0x2e,0x73,0x74,0x64,0x2e,0x34,0x35,0x30, -0x00,0x00,0x00,0x00,0x0e,0x00,0x03,0x00,0x00,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x0f,0x00,0x0a,0x00,0x05,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x6d,0x61,0x69,0x6e,0x00,0x00,0x00,0x00, -0x17,0x00,0x00,0x00,0x25,0x00,0x00,0x00,0x33,0x00,0x00,0x00, -0x66,0x00,0x00,0x00,0x7b,0x00,0x00,0x00,0x10,0x00,0x06,0x00, -0x04,0x00,0x00,0x00,0x11,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x17,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x1a,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x23,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x23,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x08,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x23,0x00,0x00,0x00,0x03,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x47,0x00,0x03,0x00, -0x23,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x33,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x1b,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x5b,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x5d,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x60,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x62,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x62,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x80,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x62,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0xc0,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x62,0x00,0x00,0x00,0x03,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0xd0,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x63,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0xd2,0x00,0x00,0x00, -0x48,0x00,0x04,0x00,0x64,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x64,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x03,0x00,0x64,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x66,0x00,0x00,0x00,0x22,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x66,0x00,0x00,0x00, -0x21,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x78,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x48,0x00,0x04,0x00,0x79,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x19,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x79,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x03,0x00,0x79,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x7b,0x00,0x00,0x00,0x22,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x7b,0x00,0x00,0x00, -0x21,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x05,0x01,0x00,0x00,0x0b,0x00,0x00,0x00,0x19,0x00,0x00,0x00, -0x13,0x00,0x02,0x00,0x02,0x00,0x00,0x00,0x21,0x00,0x03,0x00, -0x03,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x15,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x10,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x14,0x00,0x02,0x00, -0x11,0x00,0x00,0x00,0x15,0x00,0x04,0x00,0x14,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x17,0x00,0x04,0x00, -0x15,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x03,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x16,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x15,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x16,0x00,0x00,0x00, -0x17,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x14,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x19,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x14,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x14,0x00,0x00,0x00, -0x1c,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x1e,0x00,0x06,0x00, -0x23,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x24,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x24,0x00,0x00,0x00,0x25,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x26,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x29,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x16,0x00,0x00,0x00,0x33,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x41,0x00,0x00,0x00,0x08,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x45,0x00,0x00,0x00, -0x10,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x4b,0x00,0x00,0x00,0x80,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x52,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x15,0x00,0x04,0x00,0x57,0x00,0x00,0x00,0x08,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x14,0x00,0x00,0x00, -0x5a,0x00,0x00,0x00,0x80,0x00,0x00,0x00,0x1c,0x00,0x04,0x00, -0x5b,0x00,0x00,0x00,0x57,0x00,0x00,0x00,0x5a,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0x5c,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0x1c,0x00,0x04,0x00,0x5d,0x00,0x00,0x00, -0x57,0x00,0x00,0x00,0x5c,0x00,0x00,0x00,0x15,0x00,0x04,0x00, -0x5e,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0x5f,0x00,0x00,0x00, -0x10,0x00,0x00,0x00,0x1c,0x00,0x04,0x00,0x60,0x00,0x00,0x00, -0x5e,0x00,0x00,0x00,0x5f,0x00,0x00,0x00,0x16,0x00,0x03,0x00, -0x61,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x1e,0x00,0x06,0x00, -0x62,0x00,0x00,0x00,0x5b,0x00,0x00,0x00,0x5d,0x00,0x00,0x00, -0x60,0x00,0x00,0x00,0x61,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, -0x63,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, -0x64,0x00,0x00,0x00,0x63,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x65,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x64,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x65,0x00,0x00,0x00,0x66,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x6c,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x57,0x00,0x00,0x00,0x16,0x00,0x03,0x00, -0x6f,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x73,0x00,0x00,0x00,0x03,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x74,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x61,0x00,0x00,0x00,0x1d,0x00,0x03,0x00,0x78,0x00,0x00,0x00, -0x61,0x00,0x00,0x00,0x1e,0x00,0x03,0x00,0x79,0x00,0x00,0x00, -0x78,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x7a,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x79,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x7a,0x00,0x00,0x00,0x7b,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x80,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x83,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x5e,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x8e,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x95,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xe1,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xe6,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0x04,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0x2c,0x00,0x06,0x00,0x15,0x00,0x00,0x00, -0x05,0x01,0x00,0x00,0x5c,0x00,0x00,0x00,0x04,0x01,0x00,0x00, -0x04,0x01,0x00,0x00,0x2a,0x00,0x03,0x00,0x11,0x00,0x00,0x00, -0x08,0x01,0x00,0x00,0x29,0x00,0x03,0x00,0x11,0x00,0x00,0x00, -0x0b,0x01,0x00,0x00,0x36,0x00,0x05,0x00,0x02,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x03,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0x05,0x00,0x00,0x00,0xf7,0x00,0x03,0x00, -0x06,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0xfb,0x00,0x03,0x00, -0x18,0x00,0x00,0x00,0x07,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x07,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x0a,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0x0a,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0x0e,0x01,0x00,0x00,0x09,0x00,0x00,0x00, -0x07,0x01,0x00,0x00,0x03,0x01,0x00,0x00,0x0d,0x00,0x00,0x00, -0xb1,0x00,0x05,0x00,0x11,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x0e,0x01,0x00,0x00,0x10,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0x0c,0x00,0x00,0x00,0x0d,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x12,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x0b,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x19,0x00,0x00,0x00,0x1a,0x00,0x00,0x00, -0x17,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x14,0x00,0x00,0x00,0x1b,0x00,0x00,0x00,0x1a,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x14,0x00,0x00,0x00,0x1d,0x00,0x00,0x00, -0x1b,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x14,0x00,0x00,0x00,0x1f,0x00,0x00,0x00,0x0e,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x14,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x1d,0x00,0x00,0x00,0x1f,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x26,0x00,0x00,0x00,0x27,0x00,0x00,0x00, -0x25,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0x27,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x26,0x00,0x00,0x00,0x2a,0x00,0x00,0x00, -0x25,0x00,0x00,0x00,0x29,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x2b,0x00,0x00,0x00,0x2a,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x2c,0x00,0x00,0x00, -0x28,0x00,0x00,0x00,0x2b,0x00,0x00,0x00,0x87,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x2d,0x00,0x00,0x00,0x2c,0x00,0x00,0x00, -0x10,0x00,0x00,0x00,0xaf,0x00,0x05,0x00,0x11,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x2d,0x00,0x00,0x00, -0xf7,0x00,0x03,0x00,0x30,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x2e,0x00,0x00,0x00,0x2f,0x00,0x00,0x00, -0x30,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x2f,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x0c,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x30,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x19,0x00,0x00,0x00, -0x34,0x00,0x00,0x00,0x33,0x00,0x00,0x00,0x18,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0x35,0x00,0x00,0x00, -0x34,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x36,0x00,0x00,0x00,0x35,0x00,0x00,0x00,0x87,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x36,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x3e,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, -0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3f,0x00,0x00,0x00, -0x36,0x00,0x00,0x00,0x3e,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x43,0x00,0x00,0x00,0x41,0x00,0x00,0x00, -0x3a,0x00,0x00,0x00,0x87,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x46,0x00,0x00,0x00,0x3f,0x00,0x00,0x00,0x45,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x47,0x00,0x00,0x00, -0x43,0x00,0x00,0x00,0x46,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x4a,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x10,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x4d,0x00,0x00,0x00,0x4b,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, -0x4a,0x00,0x00,0x00,0x4d,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x50,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, -0x3f,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x54,0x00,0x00,0x00,0x52,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x56,0x00,0x00,0x00, -0x54,0x00,0x00,0x00,0x3f,0x00,0x00,0x00,0x41,0x00,0x08,0x00, -0x6c,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x66,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x29,0x00,0x00,0x00, -0x36,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x57,0x00,0x00,0x00, -0x6e,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x41,0x00,0x07,0x00, -0x74,0x00,0x00,0x00,0x75,0x00,0x00,0x00,0x66,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x73,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x61,0x00,0x00,0x00,0x76,0x00,0x00,0x00, -0x75,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x6f,0x00,0x00,0x00, -0x77,0x00,0x00,0x00,0x76,0x00,0x00,0x00,0x41,0x00,0x08,0x00, -0x83,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x66,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x80,0x00,0x00,0x00, -0x47,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x5e,0x00,0x00,0x00, -0x85,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x72,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x85,0x00,0x00,0x00, -0x41,0x00,0x08,0x00,0x6c,0x00,0x00,0x00,0x8a,0x00,0x00,0x00, -0x66,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x56,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x57,0x00,0x00,0x00,0x8b,0x00,0x00,0x00,0x8a,0x00,0x00,0x00, -0x71,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0x8c,0x00,0x00,0x00, -0x8b,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x8d,0x00,0x00,0x00,0x8c,0x00,0x00,0x00,0xc7,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x8f,0x00,0x00,0x00,0x8d,0x00,0x00,0x00, -0x8e,0x00,0x00,0x00,0xc2,0x00,0x05,0x00,0x57,0x00,0x00,0x00, -0x91,0x00,0x00,0x00,0x6e,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x71,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0x92,0x00,0x00,0x00, -0x91,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x93,0x00,0x00,0x00,0x92,0x00,0x00,0x00,0xc7,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x94,0x00,0x00,0x00,0x93,0x00,0x00,0x00, -0x73,0x00,0x00,0x00,0xc4,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x96,0x00,0x00,0x00,0x94,0x00,0x00,0x00,0x95,0x00,0x00,0x00, -0xc5,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x97,0x00,0x00,0x00, -0x8f,0x00,0x00,0x00,0x96,0x00,0x00,0x00,0x72,0x00,0x04,0x00, -0x5e,0x00,0x00,0x00,0x98,0x00,0x00,0x00,0x97,0x00,0x00,0x00, -0x72,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x99,0x00,0x00,0x00, -0x98,0x00,0x00,0x00,0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x9a,0x00,0x00,0x00,0x99,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9b,0x00,0x00,0x00, -0x86,0x00,0x00,0x00,0x9a,0x00,0x00,0x00,0x6f,0x00,0x04,0x00, -0x6f,0x00,0x00,0x00,0x9c,0x00,0x00,0x00,0x9b,0x00,0x00,0x00, -0x85,0x00,0x05,0x00,0x6f,0x00,0x00,0x00,0x9d,0x00,0x00,0x00, -0x77,0x00,0x00,0x00,0x9c,0x00,0x00,0x00,0x73,0x00,0x04,0x00, -0x61,0x00,0x00,0x00,0x9e,0x00,0x00,0x00,0x9d,0x00,0x00,0x00, -0x41,0x00,0x06,0x00,0x74,0x00,0x00,0x00,0x9f,0x00,0x00,0x00, -0x7b,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x50,0x00,0x00,0x00, -0x3e,0x00,0x03,0x00,0x9f,0x00,0x00,0x00,0x9e,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa1,0x00,0x00,0x00, -0x50,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xa5,0x00,0x00,0x00,0x47,0x00,0x00,0x00, -0x80,0x00,0x00,0x00,0x41,0x00,0x08,0x00,0x83,0x00,0x00,0x00, -0xa6,0x00,0x00,0x00,0x66,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x21,0x00,0x00,0x00,0x80,0x00,0x00,0x00,0xa5,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x5e,0x00,0x00,0x00,0xa7,0x00,0x00,0x00, -0xa6,0x00,0x00,0x00,0x72,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xa8,0x00,0x00,0x00,0xa7,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xab,0x00,0x00,0x00,0x56,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x41,0x00,0x08,0x00,0x6c,0x00,0x00,0x00, -0xac,0x00,0x00,0x00,0x66,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x21,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0xab,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x57,0x00,0x00,0x00,0xad,0x00,0x00,0x00, -0xac,0x00,0x00,0x00,0x71,0x00,0x04,0x00,0x14,0x00,0x00,0x00, -0xae,0x00,0x00,0x00,0xad,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xaf,0x00,0x00,0x00,0xae,0x00,0x00,0x00, -0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xb0,0x00,0x00,0x00, -0xaf,0x00,0x00,0x00,0x8e,0x00,0x00,0x00,0xc2,0x00,0x05,0x00, -0x57,0x00,0x00,0x00,0xb2,0x00,0x00,0x00,0x6e,0x00,0x00,0x00, -0x80,0x00,0x00,0x00,0x71,0x00,0x04,0x00,0x14,0x00,0x00,0x00, -0xb3,0x00,0x00,0x00,0xb2,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xb4,0x00,0x00,0x00,0xb3,0x00,0x00,0x00, -0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xb5,0x00,0x00,0x00, -0xb4,0x00,0x00,0x00,0x73,0x00,0x00,0x00,0xc4,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xb6,0x00,0x00,0x00,0xb5,0x00,0x00,0x00, -0x95,0x00,0x00,0x00,0xc5,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xb7,0x00,0x00,0x00,0xb0,0x00,0x00,0x00,0xb6,0x00,0x00,0x00, -0x72,0x00,0x04,0x00,0x5e,0x00,0x00,0x00,0xb8,0x00,0x00,0x00, -0xb7,0x00,0x00,0x00,0x72,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xb9,0x00,0x00,0x00,0xb8,0x00,0x00,0x00,0x82,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xba,0x00,0x00,0x00,0xb9,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xbb,0x00,0x00,0x00,0xa8,0x00,0x00,0x00,0xba,0x00,0x00,0x00, -0x6f,0x00,0x04,0x00,0x6f,0x00,0x00,0x00,0xbc,0x00,0x00,0x00, -0xbb,0x00,0x00,0x00,0x85,0x00,0x05,0x00,0x6f,0x00,0x00,0x00, -0xbd,0x00,0x00,0x00,0x77,0x00,0x00,0x00,0xbc,0x00,0x00,0x00, -0x73,0x00,0x04,0x00,0x61,0x00,0x00,0x00,0xbe,0x00,0x00,0x00, -0xbd,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x74,0x00,0x00,0x00, -0xbf,0x00,0x00,0x00,0x7b,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0xa1,0x00,0x00,0x00,0x3e,0x00,0x03,0x00,0xbf,0x00,0x00,0x00, -0xbe,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xc1,0x00,0x00,0x00,0x50,0x00,0x00,0x00,0x52,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xc5,0x00,0x00,0x00, -0x47,0x00,0x00,0x00,0x95,0x00,0x00,0x00,0x41,0x00,0x08,0x00, -0x83,0x00,0x00,0x00,0xc6,0x00,0x00,0x00,0x66,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x80,0x00,0x00,0x00, -0xc5,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x5e,0x00,0x00,0x00, -0xc7,0x00,0x00,0x00,0xc6,0x00,0x00,0x00,0x72,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xc8,0x00,0x00,0x00,0xc7,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x57,0x00,0x00,0x00,0xcd,0x00,0x00,0x00, -0x8a,0x00,0x00,0x00,0xc2,0x00,0x05,0x00,0x57,0x00,0x00,0x00, -0xce,0x00,0x00,0x00,0xcd,0x00,0x00,0x00,0x95,0x00,0x00,0x00, -0x71,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0xcf,0x00,0x00,0x00, -0xce,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xd0,0x00,0x00,0x00,0xcf,0x00,0x00,0x00,0xc2,0x00,0x05,0x00, -0x57,0x00,0x00,0x00,0xd2,0x00,0x00,0x00,0x6e,0x00,0x00,0x00, -0x95,0x00,0x00,0x00,0x71,0x00,0x04,0x00,0x14,0x00,0x00,0x00, -0xd3,0x00,0x00,0x00,0xd2,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xd4,0x00,0x00,0x00,0xd3,0x00,0x00,0x00, -0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xd5,0x00,0x00,0x00, -0xd4,0x00,0x00,0x00,0x73,0x00,0x00,0x00,0xc4,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xd6,0x00,0x00,0x00,0xd5,0x00,0x00,0x00, -0x95,0x00,0x00,0x00,0xc5,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xd7,0x00,0x00,0x00,0xd0,0x00,0x00,0x00,0xd6,0x00,0x00,0x00, -0x72,0x00,0x04,0x00,0x5e,0x00,0x00,0x00,0xd8,0x00,0x00,0x00, -0xd7,0x00,0x00,0x00,0x72,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xd9,0x00,0x00,0x00,0xd8,0x00,0x00,0x00,0x82,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xda,0x00,0x00,0x00,0xd9,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xdb,0x00,0x00,0x00,0xc8,0x00,0x00,0x00,0xda,0x00,0x00,0x00, -0x6f,0x00,0x04,0x00,0x6f,0x00,0x00,0x00,0xdc,0x00,0x00,0x00, -0xdb,0x00,0x00,0x00,0x85,0x00,0x05,0x00,0x6f,0x00,0x00,0x00, -0xdd,0x00,0x00,0x00,0x77,0x00,0x00,0x00,0xdc,0x00,0x00,0x00, -0x73,0x00,0x04,0x00,0x61,0x00,0x00,0x00,0xde,0x00,0x00,0x00, -0xdd,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x74,0x00,0x00,0x00, -0xdf,0x00,0x00,0x00,0x7b,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0xc1,0x00,0x00,0x00,0x3e,0x00,0x03,0x00,0xdf,0x00,0x00,0x00, -0xde,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xe2,0x00,0x00,0x00,0x50,0x00,0x00,0x00,0xe1,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xe7,0x00,0x00,0x00, -0x47,0x00,0x00,0x00,0xe6,0x00,0x00,0x00,0x41,0x00,0x08,0x00, -0x83,0x00,0x00,0x00,0xe8,0x00,0x00,0x00,0x66,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x80,0x00,0x00,0x00, -0xe7,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x5e,0x00,0x00,0x00, -0xe9,0x00,0x00,0x00,0xe8,0x00,0x00,0x00,0x72,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xea,0x00,0x00,0x00,0xe9,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x57,0x00,0x00,0x00,0xef,0x00,0x00,0x00, -0xac,0x00,0x00,0x00,0xc2,0x00,0x05,0x00,0x57,0x00,0x00,0x00, -0xf0,0x00,0x00,0x00,0xef,0x00,0x00,0x00,0x95,0x00,0x00,0x00, -0x71,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0xf1,0x00,0x00,0x00, -0xf0,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xf2,0x00,0x00,0x00,0xf1,0x00,0x00,0x00,0xc2,0x00,0x05,0x00, -0x57,0x00,0x00,0x00,0xf4,0x00,0x00,0x00,0x6e,0x00,0x00,0x00, -0xe6,0x00,0x00,0x00,0x71,0x00,0x04,0x00,0x14,0x00,0x00,0x00, -0xf5,0x00,0x00,0x00,0xf4,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xf6,0x00,0x00,0x00,0xf5,0x00,0x00,0x00, -0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xf7,0x00,0x00,0x00, -0xf6,0x00,0x00,0x00,0x73,0x00,0x00,0x00,0xc4,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xf8,0x00,0x00,0x00,0xf7,0x00,0x00,0x00, -0x95,0x00,0x00,0x00,0xc5,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xf9,0x00,0x00,0x00,0xf2,0x00,0x00,0x00,0xf8,0x00,0x00,0x00, -0x72,0x00,0x04,0x00,0x5e,0x00,0x00,0x00,0xfa,0x00,0x00,0x00, -0xf9,0x00,0x00,0x00,0x72,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xfb,0x00,0x00,0x00,0xfa,0x00,0x00,0x00,0x82,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xfc,0x00,0x00,0x00,0xfb,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xfd,0x00,0x00,0x00,0xea,0x00,0x00,0x00,0xfc,0x00,0x00,0x00, -0x6f,0x00,0x04,0x00,0x6f,0x00,0x00,0x00,0xfe,0x00,0x00,0x00, -0xfd,0x00,0x00,0x00,0x85,0x00,0x05,0x00,0x6f,0x00,0x00,0x00, -0xff,0x00,0x00,0x00,0x77,0x00,0x00,0x00,0xfe,0x00,0x00,0x00, -0x73,0x00,0x04,0x00,0x61,0x00,0x00,0x00,0x00,0x01,0x00,0x00, -0xff,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x74,0x00,0x00,0x00, -0x01,0x01,0x00,0x00,0x7b,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0xe2,0x00,0x00,0x00,0x3e,0x00,0x03,0x00,0x01,0x01,0x00,0x00, -0x00,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x0d,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0x0d,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x03,0x01,0x00,0x00,0x0e,0x01,0x00,0x00, -0x29,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x0a,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0x0c,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, -0x11,0x00,0x00,0x00,0x0f,0x01,0x00,0x00,0x08,0x01,0x00,0x00, -0x0a,0x00,0x00,0x00,0x0b,0x01,0x00,0x00,0x2f,0x00,0x00,0x00, -0xf7,0x00,0x03,0x00,0x0c,0x01,0x00,0x00,0x00,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x0f,0x01,0x00,0x00,0x06,0x01,0x00,0x00, -0x0c,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x0c,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0x06,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x06,0x01,0x00,0x00,0xfd,0x00,0x01,0x00,0x38,0x00,0x01,0x00, - -}; -const uint64_t dequant_q6_K_len = 4296; - -unsigned char dequant_q8_0_data[] = { -0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, -0x23,0x03,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, -0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x51,0x11,0x00,0x00, -0x11,0x00,0x02,0x00,0x60,0x11,0x00,0x00,0x0b,0x00,0x06,0x00, -0x01,0x00,0x00,0x00,0x47,0x4c,0x53,0x4c,0x2e,0x73,0x74,0x64, -0x2e,0x34,0x35,0x30,0x00,0x00,0x00,0x00,0x0e,0x00,0x03,0x00, -0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x0f,0x00,0x09,0x00, -0x05,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x6d,0x61,0x69,0x6e, -0x00,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0x54,0x00,0x00,0x00,0x72,0x00,0x00,0x00,0x10,0x00,0x06,0x00, -0x04,0x00,0x00,0x00,0x11,0x00,0x00,0x00,0x00,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x0c,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x1c,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x14,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x14,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x14,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x08,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x14,0x00,0x00,0x00,0x03,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x47,0x00,0x03,0x00, -0x14,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x4f,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x50,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x50,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x51,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x48,0x00,0x04,0x00, -0x52,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x52,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x1e,0x00,0x00,0x00,0x03,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x1e,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x10,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x1e,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x30,0x00,0x00,0x00, +0x0b,0x00,0x00,0x00,0x1b,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x45,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x47,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x49,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x4a,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x4a,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x4a,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x4a,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x30,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x4b,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0xb0,0x00,0x00,0x00,0x48,0x00,0x04,0x00, +0x4c,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x4c,0x00,0x00,0x00,0x00,0x00,0x00,0x00, 0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, -0x52,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x54,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x54,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x6f,0x00,0x00,0x00, +0x4c,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x4e,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x4e,0x00,0x00,0x00,0x21,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x0e,0x01,0x00,0x00, 0x06,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x48,0x00,0x04,0x00, -0x70,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x19,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x70,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x0f,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x19,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x0f,0x01,0x00,0x00,0x00,0x00,0x00,0x00, 0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, -0x70,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x72,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x72,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x94,0x00,0x00,0x00, +0x0f,0x01,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x11,0x01,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x11,0x01,0x00,0x00,0x21,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x87,0x01,0x00,0x00, 0x0b,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x13,0x00,0x02,0x00, 0x02,0x00,0x00,0x00,0x21,0x00,0x03,0x00,0x03,0x00,0x00,0x00, 0x02,0x00,0x00,0x00,0x15,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x15,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x17,0x00,0x04,0x00,0x0a,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x03,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x0b,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x0a,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x0b,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x0d,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x0e,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x1e,0x00,0x06,0x00, -0x14,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x10,0x00,0x00,0x00, +0x00,0x01,0x00,0x00,0x14,0x00,0x02,0x00,0x11,0x00,0x00,0x00, +0x17,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x03,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x15,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0x15,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x17,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x1e,0x00,0x07,0x00,0x1e,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, 0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x15,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x14,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x15,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x17,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x18,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x1b,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x14,0x00,0x02,0x00,0x24,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, +0x1f,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x1e,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0x1f,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0x09,0x00,0x00,0x00,0x15,0x00,0x04,0x00,0x21,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x21,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x23,0x00,0x00,0x00,0x09,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x21,0x00,0x00,0x00, +0x26,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0x15,0x00,0x00,0x00,0x30,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x35,0x00,0x00,0x00, +0x10,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x3b,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x16,0x00,0x03,0x00, +0x3e,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x16,0x00,0x03,0x00, +0x41,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x17,0x00,0x04,0x00, +0x42,0x00,0x00,0x00,0x41,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x15,0x00,0x04,0x00,0x43,0x00,0x00,0x00,0x08,0x00,0x00,0x00, 0x00,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x37,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x16,0x00,0x03,0x00, -0x49,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x16,0x00,0x03,0x00, -0x4c,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x15,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x1c,0x00,0x04,0x00,0x4f,0x00,0x00,0x00, -0x4d,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x1e,0x00,0x04,0x00, -0x50,0x00,0x00,0x00,0x4c,0x00,0x00,0x00,0x4f,0x00,0x00,0x00, -0x1d,0x00,0x03,0x00,0x51,0x00,0x00,0x00,0x50,0x00,0x00,0x00, -0x1e,0x00,0x03,0x00,0x52,0x00,0x00,0x00,0x51,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x53,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x52,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x53,0x00,0x00,0x00, -0x54,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x56,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x4c,0x00,0x00,0x00, -0x17,0x00,0x04,0x00,0x5a,0x00,0x00,0x00,0x49,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x5f,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x4d,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, -0x6f,0x00,0x00,0x00,0x4c,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, -0x70,0x00,0x00,0x00,0x6f,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x71,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x70,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x71,0x00,0x00,0x00,0x72,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x74,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x8c,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x93,0x00,0x00,0x00, -0x00,0x01,0x00,0x00,0x2c,0x00,0x06,0x00,0x0a,0x00,0x00,0x00, -0x94,0x00,0x00,0x00,0x93,0x00,0x00,0x00,0x8c,0x00,0x00,0x00, -0x8c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x07,0x03,0x00,0x00,0x04,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x08,0x03,0x00,0x00,0x05,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x09,0x03,0x00,0x00, -0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x0a,0x03,0x00,0x00,0x07,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x0b,0x03,0x00,0x00,0x08,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x0c,0x03,0x00,0x00, -0x09,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x0d,0x03,0x00,0x00,0x0a,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x0e,0x03,0x00,0x00,0x0b,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x0f,0x03,0x00,0x00, -0x0c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x10,0x03,0x00,0x00,0x0d,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x11,0x03,0x00,0x00,0x0e,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x12,0x03,0x00,0x00, -0x0f,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x13,0x03,0x00,0x00,0x10,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x14,0x03,0x00,0x00,0x11,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x15,0x03,0x00,0x00, -0x12,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x16,0x03,0x00,0x00,0x13,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x17,0x03,0x00,0x00,0x14,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x18,0x03,0x00,0x00, -0x15,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x19,0x03,0x00,0x00,0x16,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x1a,0x03,0x00,0x00,0x17,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x1b,0x03,0x00,0x00, -0x18,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x1c,0x03,0x00,0x00,0x19,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x1d,0x03,0x00,0x00,0x1a,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x1e,0x03,0x00,0x00, -0x1b,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x1f,0x03,0x00,0x00,0x1c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x20,0x03,0x00,0x00,0x1d,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x21,0x03,0x00,0x00, -0x1e,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x22,0x03,0x00,0x00,0x1f,0x00,0x00,0x00,0x36,0x00,0x05,0x00, -0x02,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x03,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x05,0x00,0x00,0x00, -0xf7,0x00,0x03,0x00,0x95,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0xfb,0x00,0x03,0x00,0x0d,0x00,0x00,0x00,0x96,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0x96,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x0e,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x0d,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x10,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x11,0x00,0x00,0x00,0x10,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x18,0x00,0x00,0x00,0x19,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x17,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x44,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x1c,0x00,0x04,0x00, +0x45,0x00,0x00,0x00,0x43,0x00,0x00,0x00,0x44,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x46,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0x1c,0x00,0x04,0x00,0x47,0x00,0x00,0x00, +0x43,0x00,0x00,0x00,0x46,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x48,0x00,0x00,0x00,0x80,0x00,0x00,0x00, +0x1c,0x00,0x04,0x00,0x49,0x00,0x00,0x00,0x43,0x00,0x00,0x00, +0x48,0x00,0x00,0x00,0x1e,0x00,0x06,0x00,0x4a,0x00,0x00,0x00, +0x42,0x00,0x00,0x00,0x45,0x00,0x00,0x00,0x47,0x00,0x00,0x00, +0x49,0x00,0x00,0x00,0x1d,0x00,0x03,0x00,0x4b,0x00,0x00,0x00, +0x4a,0x00,0x00,0x00,0x1e,0x00,0x03,0x00,0x4c,0x00,0x00,0x00, +0x4b,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x4c,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0x4d,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x50,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x41,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x56,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x5d,0x00,0x00,0x00,0x40,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x6e,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x76,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x43,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x21,0x00,0x00,0x00,0x7b,0x00,0x00,0x00,0x3f,0x00,0x00,0x00, +0x15,0x00,0x04,0x00,0x7d,0x00,0x00,0x00,0x08,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x21,0x00,0x00,0x00, +0x93,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x21,0x00,0x00,0x00,0x9a,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x21,0x00,0x00,0x00,0x9c,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xca,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xde,0x00,0x00,0x00,0x03,0x00,0x00,0x00, +0x1d,0x00,0x03,0x00,0x0e,0x01,0x00,0x00,0x41,0x00,0x00,0x00, +0x1e,0x00,0x03,0x00,0x0f,0x01,0x00,0x00,0x0e,0x01,0x00,0x00, +0x20,0x00,0x04,0x00,0x10,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, +0x0f,0x01,0x00,0x00,0x3b,0x00,0x04,0x00,0x10,0x01,0x00,0x00, +0x11,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x21,0x00,0x00,0x00,0x15,0x01,0x00,0x00,0x03,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x21,0x00,0x00,0x00,0x1d,0x01,0x00,0x00, +0x02,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x21,0x00,0x00,0x00, +0x26,0x01,0x00,0x00,0x10,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x68,0x01,0x00,0x00,0x21,0x00,0x00,0x00, +0x2c,0x00,0x06,0x00,0x14,0x00,0x00,0x00,0x87,0x01,0x00,0x00, +0x5d,0x00,0x00,0x00,0x56,0x00,0x00,0x00,0x56,0x00,0x00,0x00, +0x2a,0x00,0x03,0x00,0x11,0x00,0x00,0x00,0x8a,0x01,0x00,0x00, +0x29,0x00,0x03,0x00,0x11,0x00,0x00,0x00,0x8d,0x01,0x00,0x00, +0x36,0x00,0x05,0x00,0x02,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0x05,0x00,0x00,0x00,0xf7,0x00,0x03,0x00,0x88,0x01,0x00,0x00, +0x00,0x00,0x00,0x00,0xfb,0x00,0x03,0x00,0x09,0x00,0x00,0x00, +0x89,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x89,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0x0a,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0x0a,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0x90,0x01,0x00,0x00,0x09,0x00,0x00,0x00,0x89,0x01,0x00,0x00, +0x86,0x01,0x00,0x00,0x0d,0x00,0x00,0x00,0xb0,0x00,0x05,0x00, +0x11,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x90,0x01,0x00,0x00, +0x10,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x0c,0x00,0x00,0x00, +0x0d,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x12,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0x0b,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x17,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x16,0x00,0x00,0x00, +0x09,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x19,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x84,0x00,0x05,0x00, 0x06,0x00,0x00,0x00,0x1a,0x00,0x00,0x00,0x19,0x00,0x00,0x00, -0x87,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x1c,0x00,0x00,0x00, -0x1a,0x00,0x00,0x00,0x1b,0x00,0x00,0x00,0x8b,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x1d,0x00,0x00,0x00,0x11,0x00,0x00,0x00, -0x1c,0x00,0x00,0x00,0x87,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x11,0x00,0x00,0x00,0x1c,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x26,0x00,0x00,0x00, -0x1d,0x00,0x00,0x00,0x1b,0x00,0x00,0x00,0xaf,0x00,0x05,0x00, -0x24,0x00,0x00,0x00,0x29,0x00,0x00,0x00,0x26,0x00,0x00,0x00, -0x1a,0x00,0x00,0x00,0xa8,0x00,0x04,0x00,0x24,0x00,0x00,0x00, -0x2a,0x00,0x00,0x00,0x29,0x00,0x00,0x00,0xf7,0x00,0x03,0x00, -0x2c,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x2a,0x00,0x00,0x00,0x2b,0x00,0x00,0x00,0x2c,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0x2b,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x18,0x00,0x00,0x00,0x2f,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x30,0x00,0x00,0x00,0x2f,0x00,0x00,0x00,0xaf,0x00,0x05,0x00, -0x24,0x00,0x00,0x00,0x31,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x30,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x2c,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0x2c,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, -0x24,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x29,0x00,0x00,0x00, -0x96,0x00,0x00,0x00,0x31,0x00,0x00,0x00,0x2b,0x00,0x00,0x00, -0xf7,0x00,0x03,0x00,0x34,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x32,0x00,0x00,0x00,0x33,0x00,0x00,0x00, -0x34,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x33,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x95,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x34,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x18,0x00,0x00,0x00, -0x38,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x37,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x38,0x00,0x00,0x00,0x87,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x3a,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x1b,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3e,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x3e,0x00,0x00,0x00, -0x1d,0x00,0x00,0x00,0x41,0x00,0x07,0x00,0x56,0x00,0x00,0x00, -0x57,0x00,0x00,0x00,0x54,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4c,0x00,0x00,0x00,0x58,0x00,0x00,0x00,0x57,0x00,0x00,0x00, -0x73,0x00,0x04,0x00,0x49,0x00,0x00,0x00,0x59,0x00,0x00,0x00, -0x58,0x00,0x00,0x00,0x41,0x00,0x08,0x00,0x5f,0x00,0x00,0x00, -0x60,0x00,0x00,0x00,0x54,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0x17,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00,0x61,0x00,0x00,0x00, -0x60,0x00,0x00,0x00,0x72,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x61,0x00,0x00,0x00,0x6f,0x00,0x04,0x00, -0x49,0x00,0x00,0x00,0x63,0x00,0x00,0x00,0x62,0x00,0x00,0x00, -0x41,0x00,0x08,0x00,0x5f,0x00,0x00,0x00,0x67,0x00,0x00,0x00, -0x54,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x17,0x00,0x00,0x00,0x17,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0x68,0x00,0x00,0x00,0x67,0x00,0x00,0x00, -0x72,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x69,0x00,0x00,0x00, -0x68,0x00,0x00,0x00,0x6f,0x00,0x04,0x00,0x49,0x00,0x00,0x00, -0x6a,0x00,0x00,0x00,0x69,0x00,0x00,0x00,0x50,0x00,0x05,0x00, -0x5a,0x00,0x00,0x00,0x6b,0x00,0x00,0x00,0x63,0x00,0x00,0x00, -0x6a,0x00,0x00,0x00,0x8e,0x00,0x05,0x00,0x5a,0x00,0x00,0x00, -0x6e,0x00,0x00,0x00,0x6b,0x00,0x00,0x00,0x59,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x18,0x00,0x00,0x00,0x75,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x74,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x76,0x00,0x00,0x00,0x75,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x77,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x76,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x7a,0x00,0x00,0x00,0x77,0x00,0x00,0x00, -0x26,0x00,0x00,0x00,0x51,0x00,0x05,0x00,0x49,0x00,0x00,0x00, -0x7f,0x00,0x00,0x00,0x6e,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x73,0x00,0x04,0x00,0x4c,0x00,0x00,0x00,0x80,0x00,0x00,0x00, -0x7f,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x56,0x00,0x00,0x00, -0x81,0x00,0x00,0x00,0x72,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x7a,0x00,0x00,0x00,0x3e,0x00,0x03,0x00,0x81,0x00,0x00,0x00, -0x80,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x8b,0x00,0x00,0x00,0x7a,0x00,0x00,0x00,0x17,0x00,0x00,0x00, -0x51,0x00,0x05,0x00,0x49,0x00,0x00,0x00,0x8e,0x00,0x00,0x00, -0x6e,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x73,0x00,0x04,0x00, -0x4c,0x00,0x00,0x00,0x8f,0x00,0x00,0x00,0x8e,0x00,0x00,0x00, -0x41,0x00,0x06,0x00,0x56,0x00,0x00,0x00,0x90,0x00,0x00,0x00, -0x72,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x8b,0x00,0x00,0x00, -0x3e,0x00,0x03,0x00,0x90,0x00,0x00,0x00,0x8f,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4c,0x00,0x00,0x00,0xa5,0x00,0x00,0x00, -0x57,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x49,0x00,0x00,0x00, -0xa6,0x00,0x00,0x00,0xa5,0x00,0x00,0x00,0x41,0x00,0x08,0x00, -0x5f,0x00,0x00,0x00,0xa7,0x00,0x00,0x00,0x54,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x17,0x00,0x00,0x00, -0x37,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0xa8,0x00,0x00,0x00,0xa7,0x00,0x00,0x00,0x72,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xa9,0x00,0x00,0x00,0xa8,0x00,0x00,0x00, -0x6f,0x00,0x04,0x00,0x49,0x00,0x00,0x00,0xaa,0x00,0x00,0x00, -0xa9,0x00,0x00,0x00,0x41,0x00,0x08,0x00,0x5f,0x00,0x00,0x00, -0xac,0x00,0x00,0x00,0x54,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0x17,0x00,0x00,0x00,0x74,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00,0xad,0x00,0x00,0x00, -0xac,0x00,0x00,0x00,0x72,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xae,0x00,0x00,0x00,0xad,0x00,0x00,0x00,0x6f,0x00,0x04,0x00, -0x49,0x00,0x00,0x00,0xaf,0x00,0x00,0x00,0xae,0x00,0x00,0x00, -0x50,0x00,0x05,0x00,0x5a,0x00,0x00,0x00,0xb0,0x00,0x00,0x00, -0xaa,0x00,0x00,0x00,0xaf,0x00,0x00,0x00,0x8e,0x00,0x05,0x00, -0x5a,0x00,0x00,0x00,0xb1,0x00,0x00,0x00,0xb0,0x00,0x00,0x00, -0xa6,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xb7,0x00,0x00,0x00,0x7a,0x00,0x00,0x00,0x37,0x00,0x00,0x00, -0x51,0x00,0x05,0x00,0x49,0x00,0x00,0x00,0xb9,0x00,0x00,0x00, -0xb1,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x73,0x00,0x04,0x00, -0x4c,0x00,0x00,0x00,0xba,0x00,0x00,0x00,0xb9,0x00,0x00,0x00, -0x41,0x00,0x06,0x00,0x56,0x00,0x00,0x00,0xbb,0x00,0x00,0x00, -0x72,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0xb7,0x00,0x00,0x00, -0x3e,0x00,0x03,0x00,0xbb,0x00,0x00,0x00,0xba,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xc2,0x00,0x00,0x00, -0x7a,0x00,0x00,0x00,0x74,0x00,0x00,0x00,0x51,0x00,0x05,0x00, -0x49,0x00,0x00,0x00,0xc3,0x00,0x00,0x00,0xb1,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x4c,0x00,0x00,0x00, -0xc4,0x00,0x00,0x00,0xc3,0x00,0x00,0x00,0x41,0x00,0x06,0x00, -0x56,0x00,0x00,0x00,0xc5,0x00,0x00,0x00,0x72,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0xc2,0x00,0x00,0x00,0x3e,0x00,0x03,0x00, -0xc5,0x00,0x00,0x00,0xc4,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4c,0x00,0x00,0x00,0xce,0x00,0x00,0x00,0x57,0x00,0x00,0x00, -0x73,0x00,0x04,0x00,0x49,0x00,0x00,0x00,0xcf,0x00,0x00,0x00, -0xce,0x00,0x00,0x00,0x41,0x00,0x08,0x00,0x5f,0x00,0x00,0x00, -0xd0,0x00,0x00,0x00,0x54,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0x17,0x00,0x00,0x00,0x07,0x03,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00,0xd1,0x00,0x00,0x00, -0xd0,0x00,0x00,0x00,0x72,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xd2,0x00,0x00,0x00,0xd1,0x00,0x00,0x00,0x6f,0x00,0x04,0x00, -0x49,0x00,0x00,0x00,0xd3,0x00,0x00,0x00,0xd2,0x00,0x00,0x00, -0x41,0x00,0x08,0x00,0x5f,0x00,0x00,0x00,0xd5,0x00,0x00,0x00, -0x54,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x17,0x00,0x00,0x00,0x08,0x03,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0xd6,0x00,0x00,0x00,0xd5,0x00,0x00,0x00, -0x72,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xd7,0x00,0x00,0x00, -0xd6,0x00,0x00,0x00,0x6f,0x00,0x04,0x00,0x49,0x00,0x00,0x00, -0xd8,0x00,0x00,0x00,0xd7,0x00,0x00,0x00,0x50,0x00,0x05,0x00, -0x5a,0x00,0x00,0x00,0xd9,0x00,0x00,0x00,0xd3,0x00,0x00,0x00, -0xd8,0x00,0x00,0x00,0x8e,0x00,0x05,0x00,0x5a,0x00,0x00,0x00, -0xda,0x00,0x00,0x00,0xd9,0x00,0x00,0x00,0xcf,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xe0,0x00,0x00,0x00, -0x7a,0x00,0x00,0x00,0x07,0x03,0x00,0x00,0x51,0x00,0x05,0x00, -0x49,0x00,0x00,0x00,0xe2,0x00,0x00,0x00,0xda,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x4c,0x00,0x00,0x00, -0xe3,0x00,0x00,0x00,0xe2,0x00,0x00,0x00,0x41,0x00,0x06,0x00, -0x56,0x00,0x00,0x00,0xe4,0x00,0x00,0x00,0x72,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0xe0,0x00,0x00,0x00,0x3e,0x00,0x03,0x00, -0xe4,0x00,0x00,0x00,0xe3,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xeb,0x00,0x00,0x00,0x7a,0x00,0x00,0x00, -0x08,0x03,0x00,0x00,0x51,0x00,0x05,0x00,0x49,0x00,0x00,0x00, -0xec,0x00,0x00,0x00,0xda,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x73,0x00,0x04,0x00,0x4c,0x00,0x00,0x00,0xed,0x00,0x00,0x00, -0xec,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x56,0x00,0x00,0x00, -0xee,0x00,0x00,0x00,0x72,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0xeb,0x00,0x00,0x00,0x3e,0x00,0x03,0x00,0xee,0x00,0x00,0x00, -0xed,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x4c,0x00,0x00,0x00, -0xf7,0x00,0x00,0x00,0x57,0x00,0x00,0x00,0x73,0x00,0x04,0x00, -0x49,0x00,0x00,0x00,0xf8,0x00,0x00,0x00,0xf7,0x00,0x00,0x00, -0x41,0x00,0x08,0x00,0x5f,0x00,0x00,0x00,0xf9,0x00,0x00,0x00, -0x54,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x17,0x00,0x00,0x00,0x09,0x03,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0xfa,0x00,0x00,0x00,0xf9,0x00,0x00,0x00, -0x72,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xfb,0x00,0x00,0x00, -0xfa,0x00,0x00,0x00,0x6f,0x00,0x04,0x00,0x49,0x00,0x00,0x00, -0xfc,0x00,0x00,0x00,0xfb,0x00,0x00,0x00,0x41,0x00,0x08,0x00, -0x5f,0x00,0x00,0x00,0xfe,0x00,0x00,0x00,0x54,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x17,0x00,0x00,0x00, -0x0a,0x03,0x00,0x00,0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0xff,0x00,0x00,0x00,0xfe,0x00,0x00,0x00,0x72,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0xff,0x00,0x00,0x00, -0x6f,0x00,0x04,0x00,0x49,0x00,0x00,0x00,0x01,0x01,0x00,0x00, -0x00,0x01,0x00,0x00,0x50,0x00,0x05,0x00,0x5a,0x00,0x00,0x00, -0x02,0x01,0x00,0x00,0xfc,0x00,0x00,0x00,0x01,0x01,0x00,0x00, -0x8e,0x00,0x05,0x00,0x5a,0x00,0x00,0x00,0x03,0x01,0x00,0x00, -0x02,0x01,0x00,0x00,0xf8,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x09,0x01,0x00,0x00,0x7a,0x00,0x00,0x00, -0x09,0x03,0x00,0x00,0x51,0x00,0x05,0x00,0x49,0x00,0x00,0x00, -0x0b,0x01,0x00,0x00,0x03,0x01,0x00,0x00,0x00,0x00,0x00,0x00, -0x73,0x00,0x04,0x00,0x4c,0x00,0x00,0x00,0x0c,0x01,0x00,0x00, -0x0b,0x01,0x00,0x00,0x41,0x00,0x06,0x00,0x56,0x00,0x00,0x00, -0x0d,0x01,0x00,0x00,0x72,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x09,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x0d,0x01,0x00,0x00, -0x0c,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x14,0x01,0x00,0x00,0x7a,0x00,0x00,0x00,0x0a,0x03,0x00,0x00, -0x51,0x00,0x05,0x00,0x49,0x00,0x00,0x00,0x15,0x01,0x00,0x00, -0x03,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x73,0x00,0x04,0x00, -0x4c,0x00,0x00,0x00,0x16,0x01,0x00,0x00,0x15,0x01,0x00,0x00, -0x41,0x00,0x06,0x00,0x56,0x00,0x00,0x00,0x17,0x01,0x00,0x00, -0x72,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x14,0x01,0x00,0x00, -0x3e,0x00,0x03,0x00,0x17,0x01,0x00,0x00,0x16,0x01,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4c,0x00,0x00,0x00,0x20,0x01,0x00,0x00, -0x57,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x49,0x00,0x00,0x00, -0x21,0x01,0x00,0x00,0x20,0x01,0x00,0x00,0x41,0x00,0x08,0x00, -0x5f,0x00,0x00,0x00,0x22,0x01,0x00,0x00,0x54,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x17,0x00,0x00,0x00, -0x0b,0x03,0x00,0x00,0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0x23,0x01,0x00,0x00,0x22,0x01,0x00,0x00,0x72,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x24,0x01,0x00,0x00,0x23,0x01,0x00,0x00, -0x6f,0x00,0x04,0x00,0x49,0x00,0x00,0x00,0x25,0x01,0x00,0x00, -0x24,0x01,0x00,0x00,0x41,0x00,0x08,0x00,0x5f,0x00,0x00,0x00, -0x27,0x01,0x00,0x00,0x54,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0x17,0x00,0x00,0x00,0x0c,0x03,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00,0x28,0x01,0x00,0x00, -0x27,0x01,0x00,0x00,0x72,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x29,0x01,0x00,0x00,0x28,0x01,0x00,0x00,0x6f,0x00,0x04,0x00, -0x49,0x00,0x00,0x00,0x2a,0x01,0x00,0x00,0x29,0x01,0x00,0x00, -0x50,0x00,0x05,0x00,0x5a,0x00,0x00,0x00,0x2b,0x01,0x00,0x00, -0x25,0x01,0x00,0x00,0x2a,0x01,0x00,0x00,0x8e,0x00,0x05,0x00, -0x5a,0x00,0x00,0x00,0x2c,0x01,0x00,0x00,0x2b,0x01,0x00,0x00, -0x21,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x32,0x01,0x00,0x00,0x7a,0x00,0x00,0x00,0x0b,0x03,0x00,0x00, -0x51,0x00,0x05,0x00,0x49,0x00,0x00,0x00,0x34,0x01,0x00,0x00, -0x2c,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x73,0x00,0x04,0x00, -0x4c,0x00,0x00,0x00,0x35,0x01,0x00,0x00,0x34,0x01,0x00,0x00, -0x41,0x00,0x06,0x00,0x56,0x00,0x00,0x00,0x36,0x01,0x00,0x00, -0x72,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x32,0x01,0x00,0x00, -0x3e,0x00,0x03,0x00,0x36,0x01,0x00,0x00,0x35,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3d,0x01,0x00,0x00, -0x7a,0x00,0x00,0x00,0x0c,0x03,0x00,0x00,0x51,0x00,0x05,0x00, -0x49,0x00,0x00,0x00,0x3e,0x01,0x00,0x00,0x2c,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x4c,0x00,0x00,0x00, -0x3f,0x01,0x00,0x00,0x3e,0x01,0x00,0x00,0x41,0x00,0x06,0x00, -0x56,0x00,0x00,0x00,0x40,0x01,0x00,0x00,0x72,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x3d,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0x40,0x01,0x00,0x00,0x3f,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4c,0x00,0x00,0x00,0x49,0x01,0x00,0x00,0x57,0x00,0x00,0x00, -0x73,0x00,0x04,0x00,0x49,0x00,0x00,0x00,0x4a,0x01,0x00,0x00, -0x49,0x01,0x00,0x00,0x41,0x00,0x08,0x00,0x5f,0x00,0x00,0x00, -0x4b,0x01,0x00,0x00,0x54,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0x17,0x00,0x00,0x00,0x0d,0x03,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00,0x4c,0x01,0x00,0x00, -0x4b,0x01,0x00,0x00,0x72,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x4d,0x01,0x00,0x00,0x4c,0x01,0x00,0x00,0x6f,0x00,0x04,0x00, -0x49,0x00,0x00,0x00,0x4e,0x01,0x00,0x00,0x4d,0x01,0x00,0x00, -0x41,0x00,0x08,0x00,0x5f,0x00,0x00,0x00,0x50,0x01,0x00,0x00, -0x54,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x17,0x00,0x00,0x00,0x0e,0x03,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0x51,0x01,0x00,0x00,0x50,0x01,0x00,0x00, -0x72,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x52,0x01,0x00,0x00, -0x51,0x01,0x00,0x00,0x6f,0x00,0x04,0x00,0x49,0x00,0x00,0x00, -0x53,0x01,0x00,0x00,0x52,0x01,0x00,0x00,0x50,0x00,0x05,0x00, -0x5a,0x00,0x00,0x00,0x54,0x01,0x00,0x00,0x4e,0x01,0x00,0x00, -0x53,0x01,0x00,0x00,0x8e,0x00,0x05,0x00,0x5a,0x00,0x00,0x00, -0x55,0x01,0x00,0x00,0x54,0x01,0x00,0x00,0x4a,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x5b,0x01,0x00,0x00, -0x7a,0x00,0x00,0x00,0x0d,0x03,0x00,0x00,0x51,0x00,0x05,0x00, -0x49,0x00,0x00,0x00,0x5d,0x01,0x00,0x00,0x55,0x01,0x00,0x00, -0x00,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x4c,0x00,0x00,0x00, -0x5e,0x01,0x00,0x00,0x5d,0x01,0x00,0x00,0x41,0x00,0x06,0x00, -0x56,0x00,0x00,0x00,0x5f,0x01,0x00,0x00,0x72,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x5b,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0x5f,0x01,0x00,0x00,0x5e,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x66,0x01,0x00,0x00,0x7a,0x00,0x00,0x00, -0x0e,0x03,0x00,0x00,0x51,0x00,0x05,0x00,0x49,0x00,0x00,0x00, -0x67,0x01,0x00,0x00,0x55,0x01,0x00,0x00,0x01,0x00,0x00,0x00, -0x73,0x00,0x04,0x00,0x4c,0x00,0x00,0x00,0x68,0x01,0x00,0x00, -0x67,0x01,0x00,0x00,0x41,0x00,0x06,0x00,0x56,0x00,0x00,0x00, -0x69,0x01,0x00,0x00,0x72,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x66,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x69,0x01,0x00,0x00, -0x68,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0x4c,0x00,0x00,0x00, -0x72,0x01,0x00,0x00,0x57,0x00,0x00,0x00,0x73,0x00,0x04,0x00, -0x49,0x00,0x00,0x00,0x73,0x01,0x00,0x00,0x72,0x01,0x00,0x00, -0x41,0x00,0x08,0x00,0x5f,0x00,0x00,0x00,0x74,0x01,0x00,0x00, -0x54,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x17,0x00,0x00,0x00,0x0f,0x03,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0x75,0x01,0x00,0x00,0x74,0x01,0x00,0x00, -0x72,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x76,0x01,0x00,0x00, -0x75,0x01,0x00,0x00,0x6f,0x00,0x04,0x00,0x49,0x00,0x00,0x00, -0x77,0x01,0x00,0x00,0x76,0x01,0x00,0x00,0x41,0x00,0x08,0x00, -0x5f,0x00,0x00,0x00,0x79,0x01,0x00,0x00,0x54,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x17,0x00,0x00,0x00, -0x10,0x03,0x00,0x00,0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0x7a,0x01,0x00,0x00,0x79,0x01,0x00,0x00,0x72,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x7b,0x01,0x00,0x00,0x7a,0x01,0x00,0x00, -0x6f,0x00,0x04,0x00,0x49,0x00,0x00,0x00,0x7c,0x01,0x00,0x00, -0x7b,0x01,0x00,0x00,0x50,0x00,0x05,0x00,0x5a,0x00,0x00,0x00, -0x7d,0x01,0x00,0x00,0x77,0x01,0x00,0x00,0x7c,0x01,0x00,0x00, -0x8e,0x00,0x05,0x00,0x5a,0x00,0x00,0x00,0x7e,0x01,0x00,0x00, -0x7d,0x01,0x00,0x00,0x73,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x84,0x01,0x00,0x00,0x7a,0x00,0x00,0x00, -0x0f,0x03,0x00,0x00,0x51,0x00,0x05,0x00,0x49,0x00,0x00,0x00, -0x86,0x01,0x00,0x00,0x7e,0x01,0x00,0x00,0x00,0x00,0x00,0x00, -0x73,0x00,0x04,0x00,0x4c,0x00,0x00,0x00,0x87,0x01,0x00,0x00, -0x86,0x01,0x00,0x00,0x41,0x00,0x06,0x00,0x56,0x00,0x00,0x00, -0x88,0x01,0x00,0x00,0x72,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x84,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x88,0x01,0x00,0x00, -0x87,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x8f,0x01,0x00,0x00,0x7a,0x00,0x00,0x00,0x10,0x03,0x00,0x00, -0x51,0x00,0x05,0x00,0x49,0x00,0x00,0x00,0x90,0x01,0x00,0x00, -0x7e,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x73,0x00,0x04,0x00, -0x4c,0x00,0x00,0x00,0x91,0x01,0x00,0x00,0x90,0x01,0x00,0x00, -0x41,0x00,0x06,0x00,0x56,0x00,0x00,0x00,0x92,0x01,0x00,0x00, -0x72,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x8f,0x01,0x00,0x00, -0x3e,0x00,0x03,0x00,0x92,0x01,0x00,0x00,0x91,0x01,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4c,0x00,0x00,0x00,0x9b,0x01,0x00,0x00, -0x57,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x49,0x00,0x00,0x00, -0x9c,0x01,0x00,0x00,0x9b,0x01,0x00,0x00,0x41,0x00,0x08,0x00, -0x5f,0x00,0x00,0x00,0x9d,0x01,0x00,0x00,0x54,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x17,0x00,0x00,0x00, -0x11,0x03,0x00,0x00,0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0x9e,0x01,0x00,0x00,0x9d,0x01,0x00,0x00,0x72,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x9f,0x01,0x00,0x00,0x9e,0x01,0x00,0x00, -0x6f,0x00,0x04,0x00,0x49,0x00,0x00,0x00,0xa0,0x01,0x00,0x00, -0x9f,0x01,0x00,0x00,0x41,0x00,0x08,0x00,0x5f,0x00,0x00,0x00, -0xa2,0x01,0x00,0x00,0x54,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0x17,0x00,0x00,0x00,0x12,0x03,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00,0xa3,0x01,0x00,0x00, -0xa2,0x01,0x00,0x00,0x72,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xa4,0x01,0x00,0x00,0xa3,0x01,0x00,0x00,0x6f,0x00,0x04,0x00, -0x49,0x00,0x00,0x00,0xa5,0x01,0x00,0x00,0xa4,0x01,0x00,0x00, -0x50,0x00,0x05,0x00,0x5a,0x00,0x00,0x00,0xa6,0x01,0x00,0x00, -0xa0,0x01,0x00,0x00,0xa5,0x01,0x00,0x00,0x8e,0x00,0x05,0x00, -0x5a,0x00,0x00,0x00,0xa7,0x01,0x00,0x00,0xa6,0x01,0x00,0x00, -0x9c,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xad,0x01,0x00,0x00,0x7a,0x00,0x00,0x00,0x11,0x03,0x00,0x00, -0x51,0x00,0x05,0x00,0x49,0x00,0x00,0x00,0xaf,0x01,0x00,0x00, -0xa7,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x73,0x00,0x04,0x00, -0x4c,0x00,0x00,0x00,0xb0,0x01,0x00,0x00,0xaf,0x01,0x00,0x00, -0x41,0x00,0x06,0x00,0x56,0x00,0x00,0x00,0xb1,0x01,0x00,0x00, -0x72,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0xad,0x01,0x00,0x00, -0x3e,0x00,0x03,0x00,0xb1,0x01,0x00,0x00,0xb0,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xb8,0x01,0x00,0x00, -0x7a,0x00,0x00,0x00,0x12,0x03,0x00,0x00,0x51,0x00,0x05,0x00, -0x49,0x00,0x00,0x00,0xb9,0x01,0x00,0x00,0xa7,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x4c,0x00,0x00,0x00, -0xba,0x01,0x00,0x00,0xb9,0x01,0x00,0x00,0x41,0x00,0x06,0x00, -0x56,0x00,0x00,0x00,0xbb,0x01,0x00,0x00,0x72,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0xb8,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0xbb,0x01,0x00,0x00,0xba,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4c,0x00,0x00,0x00,0xc4,0x01,0x00,0x00,0x57,0x00,0x00,0x00, -0x73,0x00,0x04,0x00,0x49,0x00,0x00,0x00,0xc5,0x01,0x00,0x00, -0xc4,0x01,0x00,0x00,0x41,0x00,0x08,0x00,0x5f,0x00,0x00,0x00, -0xc6,0x01,0x00,0x00,0x54,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0x17,0x00,0x00,0x00,0x13,0x03,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00,0xc7,0x01,0x00,0x00, -0xc6,0x01,0x00,0x00,0x72,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xc8,0x01,0x00,0x00,0xc7,0x01,0x00,0x00,0x6f,0x00,0x04,0x00, -0x49,0x00,0x00,0x00,0xc9,0x01,0x00,0x00,0xc8,0x01,0x00,0x00, -0x41,0x00,0x08,0x00,0x5f,0x00,0x00,0x00,0xcb,0x01,0x00,0x00, -0x54,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x17,0x00,0x00,0x00,0x14,0x03,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0xcc,0x01,0x00,0x00,0xcb,0x01,0x00,0x00, -0x72,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xcd,0x01,0x00,0x00, -0xcc,0x01,0x00,0x00,0x6f,0x00,0x04,0x00,0x49,0x00,0x00,0x00, -0xce,0x01,0x00,0x00,0xcd,0x01,0x00,0x00,0x50,0x00,0x05,0x00, -0x5a,0x00,0x00,0x00,0xcf,0x01,0x00,0x00,0xc9,0x01,0x00,0x00, -0xce,0x01,0x00,0x00,0x8e,0x00,0x05,0x00,0x5a,0x00,0x00,0x00, -0xd0,0x01,0x00,0x00,0xcf,0x01,0x00,0x00,0xc5,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xd6,0x01,0x00,0x00, -0x7a,0x00,0x00,0x00,0x13,0x03,0x00,0x00,0x51,0x00,0x05,0x00, -0x49,0x00,0x00,0x00,0xd8,0x01,0x00,0x00,0xd0,0x01,0x00,0x00, -0x00,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x4c,0x00,0x00,0x00, -0xd9,0x01,0x00,0x00,0xd8,0x01,0x00,0x00,0x41,0x00,0x06,0x00, -0x56,0x00,0x00,0x00,0xda,0x01,0x00,0x00,0x72,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0xd6,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0xda,0x01,0x00,0x00,0xd9,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xe1,0x01,0x00,0x00,0x7a,0x00,0x00,0x00, -0x14,0x03,0x00,0x00,0x51,0x00,0x05,0x00,0x49,0x00,0x00,0x00, -0xe2,0x01,0x00,0x00,0xd0,0x01,0x00,0x00,0x01,0x00,0x00,0x00, -0x73,0x00,0x04,0x00,0x4c,0x00,0x00,0x00,0xe3,0x01,0x00,0x00, -0xe2,0x01,0x00,0x00,0x41,0x00,0x06,0x00,0x56,0x00,0x00,0x00, -0xe4,0x01,0x00,0x00,0x72,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0xe1,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0xe4,0x01,0x00,0x00, -0xe3,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0x4c,0x00,0x00,0x00, -0xed,0x01,0x00,0x00,0x57,0x00,0x00,0x00,0x73,0x00,0x04,0x00, -0x49,0x00,0x00,0x00,0xee,0x01,0x00,0x00,0xed,0x01,0x00,0x00, -0x41,0x00,0x08,0x00,0x5f,0x00,0x00,0x00,0xef,0x01,0x00,0x00, -0x54,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x17,0x00,0x00,0x00,0x15,0x03,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0xf0,0x01,0x00,0x00,0xef,0x01,0x00,0x00, -0x72,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xf1,0x01,0x00,0x00, -0xf0,0x01,0x00,0x00,0x6f,0x00,0x04,0x00,0x49,0x00,0x00,0x00, -0xf2,0x01,0x00,0x00,0xf1,0x01,0x00,0x00,0x41,0x00,0x08,0x00, -0x5f,0x00,0x00,0x00,0xf4,0x01,0x00,0x00,0x54,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x17,0x00,0x00,0x00, -0x16,0x03,0x00,0x00,0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0xf5,0x01,0x00,0x00,0xf4,0x01,0x00,0x00,0x72,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xf6,0x01,0x00,0x00,0xf5,0x01,0x00,0x00, -0x6f,0x00,0x04,0x00,0x49,0x00,0x00,0x00,0xf7,0x01,0x00,0x00, -0xf6,0x01,0x00,0x00,0x50,0x00,0x05,0x00,0x5a,0x00,0x00,0x00, -0xf8,0x01,0x00,0x00,0xf2,0x01,0x00,0x00,0xf7,0x01,0x00,0x00, -0x8e,0x00,0x05,0x00,0x5a,0x00,0x00,0x00,0xf9,0x01,0x00,0x00, -0xf8,0x01,0x00,0x00,0xee,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xff,0x01,0x00,0x00,0x7a,0x00,0x00,0x00, -0x15,0x03,0x00,0x00,0x51,0x00,0x05,0x00,0x49,0x00,0x00,0x00, -0x01,0x02,0x00,0x00,0xf9,0x01,0x00,0x00,0x00,0x00,0x00,0x00, -0x73,0x00,0x04,0x00,0x4c,0x00,0x00,0x00,0x02,0x02,0x00,0x00, -0x01,0x02,0x00,0x00,0x41,0x00,0x06,0x00,0x56,0x00,0x00,0x00, -0x03,0x02,0x00,0x00,0x72,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0xff,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x03,0x02,0x00,0x00, -0x02,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x0a,0x02,0x00,0x00,0x7a,0x00,0x00,0x00,0x16,0x03,0x00,0x00, -0x51,0x00,0x05,0x00,0x49,0x00,0x00,0x00,0x0b,0x02,0x00,0x00, -0xf9,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x73,0x00,0x04,0x00, -0x4c,0x00,0x00,0x00,0x0c,0x02,0x00,0x00,0x0b,0x02,0x00,0x00, -0x41,0x00,0x06,0x00,0x56,0x00,0x00,0x00,0x0d,0x02,0x00,0x00, -0x72,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x0a,0x02,0x00,0x00, -0x3e,0x00,0x03,0x00,0x0d,0x02,0x00,0x00,0x0c,0x02,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4c,0x00,0x00,0x00,0x16,0x02,0x00,0x00, -0x57,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x49,0x00,0x00,0x00, -0x17,0x02,0x00,0x00,0x16,0x02,0x00,0x00,0x41,0x00,0x08,0x00, -0x5f,0x00,0x00,0x00,0x18,0x02,0x00,0x00,0x54,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x17,0x00,0x00,0x00, -0x17,0x03,0x00,0x00,0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0x19,0x02,0x00,0x00,0x18,0x02,0x00,0x00,0x72,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x1a,0x02,0x00,0x00,0x19,0x02,0x00,0x00, -0x6f,0x00,0x04,0x00,0x49,0x00,0x00,0x00,0x1b,0x02,0x00,0x00, -0x1a,0x02,0x00,0x00,0x41,0x00,0x08,0x00,0x5f,0x00,0x00,0x00, -0x1d,0x02,0x00,0x00,0x54,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0x17,0x00,0x00,0x00,0x18,0x03,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00,0x1e,0x02,0x00,0x00, -0x1d,0x02,0x00,0x00,0x72,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x1f,0x02,0x00,0x00,0x1e,0x02,0x00,0x00,0x6f,0x00,0x04,0x00, -0x49,0x00,0x00,0x00,0x20,0x02,0x00,0x00,0x1f,0x02,0x00,0x00, -0x50,0x00,0x05,0x00,0x5a,0x00,0x00,0x00,0x21,0x02,0x00,0x00, -0x1b,0x02,0x00,0x00,0x20,0x02,0x00,0x00,0x8e,0x00,0x05,0x00, -0x5a,0x00,0x00,0x00,0x22,0x02,0x00,0x00,0x21,0x02,0x00,0x00, -0x17,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x28,0x02,0x00,0x00,0x7a,0x00,0x00,0x00,0x17,0x03,0x00,0x00, -0x51,0x00,0x05,0x00,0x49,0x00,0x00,0x00,0x2a,0x02,0x00,0x00, -0x22,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x73,0x00,0x04,0x00, -0x4c,0x00,0x00,0x00,0x2b,0x02,0x00,0x00,0x2a,0x02,0x00,0x00, -0x41,0x00,0x06,0x00,0x56,0x00,0x00,0x00,0x2c,0x02,0x00,0x00, -0x72,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x28,0x02,0x00,0x00, -0x3e,0x00,0x03,0x00,0x2c,0x02,0x00,0x00,0x2b,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x33,0x02,0x00,0x00, -0x7a,0x00,0x00,0x00,0x18,0x03,0x00,0x00,0x51,0x00,0x05,0x00, -0x49,0x00,0x00,0x00,0x34,0x02,0x00,0x00,0x22,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x4c,0x00,0x00,0x00, -0x35,0x02,0x00,0x00,0x34,0x02,0x00,0x00,0x41,0x00,0x06,0x00, -0x56,0x00,0x00,0x00,0x36,0x02,0x00,0x00,0x72,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x33,0x02,0x00,0x00,0x3e,0x00,0x03,0x00, -0x36,0x02,0x00,0x00,0x35,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4c,0x00,0x00,0x00,0x3f,0x02,0x00,0x00,0x57,0x00,0x00,0x00, -0x73,0x00,0x04,0x00,0x49,0x00,0x00,0x00,0x40,0x02,0x00,0x00, -0x3f,0x02,0x00,0x00,0x41,0x00,0x08,0x00,0x5f,0x00,0x00,0x00, -0x41,0x02,0x00,0x00,0x54,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0x17,0x00,0x00,0x00,0x19,0x03,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00,0x42,0x02,0x00,0x00, -0x41,0x02,0x00,0x00,0x72,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x43,0x02,0x00,0x00,0x42,0x02,0x00,0x00,0x6f,0x00,0x04,0x00, -0x49,0x00,0x00,0x00,0x44,0x02,0x00,0x00,0x43,0x02,0x00,0x00, -0x41,0x00,0x08,0x00,0x5f,0x00,0x00,0x00,0x46,0x02,0x00,0x00, -0x54,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x17,0x00,0x00,0x00,0x1a,0x03,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0x47,0x02,0x00,0x00,0x46,0x02,0x00,0x00, -0x72,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x48,0x02,0x00,0x00, -0x47,0x02,0x00,0x00,0x6f,0x00,0x04,0x00,0x49,0x00,0x00,0x00, -0x49,0x02,0x00,0x00,0x48,0x02,0x00,0x00,0x50,0x00,0x05,0x00, -0x5a,0x00,0x00,0x00,0x4a,0x02,0x00,0x00,0x44,0x02,0x00,0x00, -0x49,0x02,0x00,0x00,0x8e,0x00,0x05,0x00,0x5a,0x00,0x00,0x00, -0x4b,0x02,0x00,0x00,0x4a,0x02,0x00,0x00,0x40,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x51,0x02,0x00,0x00, -0x7a,0x00,0x00,0x00,0x19,0x03,0x00,0x00,0x51,0x00,0x05,0x00, -0x49,0x00,0x00,0x00,0x53,0x02,0x00,0x00,0x4b,0x02,0x00,0x00, -0x00,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x4c,0x00,0x00,0x00, -0x54,0x02,0x00,0x00,0x53,0x02,0x00,0x00,0x41,0x00,0x06,0x00, -0x56,0x00,0x00,0x00,0x55,0x02,0x00,0x00,0x72,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x51,0x02,0x00,0x00,0x3e,0x00,0x03,0x00, -0x55,0x02,0x00,0x00,0x54,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x5c,0x02,0x00,0x00,0x7a,0x00,0x00,0x00, -0x1a,0x03,0x00,0x00,0x51,0x00,0x05,0x00,0x49,0x00,0x00,0x00, -0x5d,0x02,0x00,0x00,0x4b,0x02,0x00,0x00,0x01,0x00,0x00,0x00, -0x73,0x00,0x04,0x00,0x4c,0x00,0x00,0x00,0x5e,0x02,0x00,0x00, -0x5d,0x02,0x00,0x00,0x41,0x00,0x06,0x00,0x56,0x00,0x00,0x00, -0x5f,0x02,0x00,0x00,0x72,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x5c,0x02,0x00,0x00,0x3e,0x00,0x03,0x00,0x5f,0x02,0x00,0x00, -0x5e,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0x4c,0x00,0x00,0x00, -0x68,0x02,0x00,0x00,0x57,0x00,0x00,0x00,0x73,0x00,0x04,0x00, -0x49,0x00,0x00,0x00,0x69,0x02,0x00,0x00,0x68,0x02,0x00,0x00, -0x41,0x00,0x08,0x00,0x5f,0x00,0x00,0x00,0x6a,0x02,0x00,0x00, -0x54,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x17,0x00,0x00,0x00,0x1b,0x03,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0x6b,0x02,0x00,0x00,0x6a,0x02,0x00,0x00, -0x72,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x6c,0x02,0x00,0x00, -0x6b,0x02,0x00,0x00,0x6f,0x00,0x04,0x00,0x49,0x00,0x00,0x00, -0x6d,0x02,0x00,0x00,0x6c,0x02,0x00,0x00,0x41,0x00,0x08,0x00, -0x5f,0x00,0x00,0x00,0x6f,0x02,0x00,0x00,0x54,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x17,0x00,0x00,0x00, -0x1c,0x03,0x00,0x00,0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0x70,0x02,0x00,0x00,0x6f,0x02,0x00,0x00,0x72,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x71,0x02,0x00,0x00,0x70,0x02,0x00,0x00, -0x6f,0x00,0x04,0x00,0x49,0x00,0x00,0x00,0x72,0x02,0x00,0x00, -0x71,0x02,0x00,0x00,0x50,0x00,0x05,0x00,0x5a,0x00,0x00,0x00, -0x73,0x02,0x00,0x00,0x6d,0x02,0x00,0x00,0x72,0x02,0x00,0x00, -0x8e,0x00,0x05,0x00,0x5a,0x00,0x00,0x00,0x74,0x02,0x00,0x00, -0x73,0x02,0x00,0x00,0x69,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x7a,0x02,0x00,0x00,0x7a,0x00,0x00,0x00, -0x1b,0x03,0x00,0x00,0x51,0x00,0x05,0x00,0x49,0x00,0x00,0x00, -0x7c,0x02,0x00,0x00,0x74,0x02,0x00,0x00,0x00,0x00,0x00,0x00, -0x73,0x00,0x04,0x00,0x4c,0x00,0x00,0x00,0x7d,0x02,0x00,0x00, -0x7c,0x02,0x00,0x00,0x41,0x00,0x06,0x00,0x56,0x00,0x00,0x00, -0x7e,0x02,0x00,0x00,0x72,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x7a,0x02,0x00,0x00,0x3e,0x00,0x03,0x00,0x7e,0x02,0x00,0x00, -0x7d,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x85,0x02,0x00,0x00,0x7a,0x00,0x00,0x00,0x1c,0x03,0x00,0x00, -0x51,0x00,0x05,0x00,0x49,0x00,0x00,0x00,0x86,0x02,0x00,0x00, -0x74,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0x73,0x00,0x04,0x00, -0x4c,0x00,0x00,0x00,0x87,0x02,0x00,0x00,0x86,0x02,0x00,0x00, -0x41,0x00,0x06,0x00,0x56,0x00,0x00,0x00,0x88,0x02,0x00,0x00, -0x72,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x85,0x02,0x00,0x00, -0x3e,0x00,0x03,0x00,0x88,0x02,0x00,0x00,0x87,0x02,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4c,0x00,0x00,0x00,0x91,0x02,0x00,0x00, -0x57,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x49,0x00,0x00,0x00, -0x92,0x02,0x00,0x00,0x91,0x02,0x00,0x00,0x41,0x00,0x08,0x00, -0x5f,0x00,0x00,0x00,0x93,0x02,0x00,0x00,0x54,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x17,0x00,0x00,0x00, -0x1d,0x03,0x00,0x00,0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0x94,0x02,0x00,0x00,0x93,0x02,0x00,0x00,0x72,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x95,0x02,0x00,0x00,0x94,0x02,0x00,0x00, -0x6f,0x00,0x04,0x00,0x49,0x00,0x00,0x00,0x96,0x02,0x00,0x00, -0x95,0x02,0x00,0x00,0x41,0x00,0x08,0x00,0x5f,0x00,0x00,0x00, -0x98,0x02,0x00,0x00,0x54,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0x17,0x00,0x00,0x00,0x1e,0x03,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00,0x99,0x02,0x00,0x00, -0x98,0x02,0x00,0x00,0x72,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x9a,0x02,0x00,0x00,0x99,0x02,0x00,0x00,0x6f,0x00,0x04,0x00, -0x49,0x00,0x00,0x00,0x9b,0x02,0x00,0x00,0x9a,0x02,0x00,0x00, -0x50,0x00,0x05,0x00,0x5a,0x00,0x00,0x00,0x9c,0x02,0x00,0x00, -0x96,0x02,0x00,0x00,0x9b,0x02,0x00,0x00,0x8e,0x00,0x05,0x00, -0x5a,0x00,0x00,0x00,0x9d,0x02,0x00,0x00,0x9c,0x02,0x00,0x00, -0x92,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xa3,0x02,0x00,0x00,0x7a,0x00,0x00,0x00,0x1d,0x03,0x00,0x00, -0x51,0x00,0x05,0x00,0x49,0x00,0x00,0x00,0xa5,0x02,0x00,0x00, -0x9d,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x73,0x00,0x04,0x00, -0x4c,0x00,0x00,0x00,0xa6,0x02,0x00,0x00,0xa5,0x02,0x00,0x00, -0x41,0x00,0x06,0x00,0x56,0x00,0x00,0x00,0xa7,0x02,0x00,0x00, -0x72,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0xa3,0x02,0x00,0x00, -0x3e,0x00,0x03,0x00,0xa7,0x02,0x00,0x00,0xa6,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xae,0x02,0x00,0x00, -0x7a,0x00,0x00,0x00,0x1e,0x03,0x00,0x00,0x51,0x00,0x05,0x00, -0x49,0x00,0x00,0x00,0xaf,0x02,0x00,0x00,0x9d,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x4c,0x00,0x00,0x00, -0xb0,0x02,0x00,0x00,0xaf,0x02,0x00,0x00,0x41,0x00,0x06,0x00, -0x56,0x00,0x00,0x00,0xb1,0x02,0x00,0x00,0x72,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0xae,0x02,0x00,0x00,0x3e,0x00,0x03,0x00, -0xb1,0x02,0x00,0x00,0xb0,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4c,0x00,0x00,0x00,0xba,0x02,0x00,0x00,0x57,0x00,0x00,0x00, -0x73,0x00,0x04,0x00,0x49,0x00,0x00,0x00,0xbb,0x02,0x00,0x00, -0xba,0x02,0x00,0x00,0x41,0x00,0x08,0x00,0x5f,0x00,0x00,0x00, -0xbc,0x02,0x00,0x00,0x54,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0x17,0x00,0x00,0x00,0x1f,0x03,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00,0xbd,0x02,0x00,0x00, -0xbc,0x02,0x00,0x00,0x72,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xbe,0x02,0x00,0x00,0xbd,0x02,0x00,0x00,0x6f,0x00,0x04,0x00, -0x49,0x00,0x00,0x00,0xbf,0x02,0x00,0x00,0xbe,0x02,0x00,0x00, -0x41,0x00,0x08,0x00,0x5f,0x00,0x00,0x00,0xc1,0x02,0x00,0x00, -0x54,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x17,0x00,0x00,0x00,0x20,0x03,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0xc2,0x02,0x00,0x00,0xc1,0x02,0x00,0x00, -0x72,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xc3,0x02,0x00,0x00, -0xc2,0x02,0x00,0x00,0x6f,0x00,0x04,0x00,0x49,0x00,0x00,0x00, -0xc4,0x02,0x00,0x00,0xc3,0x02,0x00,0x00,0x50,0x00,0x05,0x00, -0x5a,0x00,0x00,0x00,0xc5,0x02,0x00,0x00,0xbf,0x02,0x00,0x00, -0xc4,0x02,0x00,0x00,0x8e,0x00,0x05,0x00,0x5a,0x00,0x00,0x00, -0xc6,0x02,0x00,0x00,0xc5,0x02,0x00,0x00,0xbb,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xcc,0x02,0x00,0x00, -0x7a,0x00,0x00,0x00,0x1f,0x03,0x00,0x00,0x51,0x00,0x05,0x00, -0x49,0x00,0x00,0x00,0xce,0x02,0x00,0x00,0xc6,0x02,0x00,0x00, -0x00,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x4c,0x00,0x00,0x00, -0xcf,0x02,0x00,0x00,0xce,0x02,0x00,0x00,0x41,0x00,0x06,0x00, -0x56,0x00,0x00,0x00,0xd0,0x02,0x00,0x00,0x72,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0xcc,0x02,0x00,0x00,0x3e,0x00,0x03,0x00, -0xd0,0x02,0x00,0x00,0xcf,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xd7,0x02,0x00,0x00,0x7a,0x00,0x00,0x00, -0x20,0x03,0x00,0x00,0x51,0x00,0x05,0x00,0x49,0x00,0x00,0x00, -0xd8,0x02,0x00,0x00,0xc6,0x02,0x00,0x00,0x01,0x00,0x00,0x00, -0x73,0x00,0x04,0x00,0x4c,0x00,0x00,0x00,0xd9,0x02,0x00,0x00, -0xd8,0x02,0x00,0x00,0x41,0x00,0x06,0x00,0x56,0x00,0x00,0x00, -0xda,0x02,0x00,0x00,0x72,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0xd7,0x02,0x00,0x00,0x3e,0x00,0x03,0x00,0xda,0x02,0x00,0x00, -0xd9,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0x4c,0x00,0x00,0x00, -0xe3,0x02,0x00,0x00,0x57,0x00,0x00,0x00,0x73,0x00,0x04,0x00, -0x49,0x00,0x00,0x00,0xe4,0x02,0x00,0x00,0xe3,0x02,0x00,0x00, -0x41,0x00,0x08,0x00,0x5f,0x00,0x00,0x00,0xe5,0x02,0x00,0x00, -0x54,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x17,0x00,0x00,0x00,0x21,0x03,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0xe6,0x02,0x00,0x00,0xe5,0x02,0x00,0x00, -0x72,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xe7,0x02,0x00,0x00, -0xe6,0x02,0x00,0x00,0x6f,0x00,0x04,0x00,0x49,0x00,0x00,0x00, -0xe8,0x02,0x00,0x00,0xe7,0x02,0x00,0x00,0x41,0x00,0x08,0x00, -0x5f,0x00,0x00,0x00,0xea,0x02,0x00,0x00,0x54,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x17,0x00,0x00,0x00, -0x22,0x03,0x00,0x00,0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0xeb,0x02,0x00,0x00,0xea,0x02,0x00,0x00,0x72,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xec,0x02,0x00,0x00,0xeb,0x02,0x00,0x00, -0x6f,0x00,0x04,0x00,0x49,0x00,0x00,0x00,0xed,0x02,0x00,0x00, -0xec,0x02,0x00,0x00,0x50,0x00,0x05,0x00,0x5a,0x00,0x00,0x00, -0xee,0x02,0x00,0x00,0xe8,0x02,0x00,0x00,0xed,0x02,0x00,0x00, -0x8e,0x00,0x05,0x00,0x5a,0x00,0x00,0x00,0xef,0x02,0x00,0x00, -0xee,0x02,0x00,0x00,0xe4,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xf5,0x02,0x00,0x00,0x7a,0x00,0x00,0x00, -0x21,0x03,0x00,0x00,0x51,0x00,0x05,0x00,0x49,0x00,0x00,0x00, -0xf7,0x02,0x00,0x00,0xef,0x02,0x00,0x00,0x00,0x00,0x00,0x00, -0x73,0x00,0x04,0x00,0x4c,0x00,0x00,0x00,0xf8,0x02,0x00,0x00, -0xf7,0x02,0x00,0x00,0x41,0x00,0x06,0x00,0x56,0x00,0x00,0x00, -0xf9,0x02,0x00,0x00,0x72,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0xf5,0x02,0x00,0x00,0x3e,0x00,0x03,0x00,0xf9,0x02,0x00,0x00, -0xf8,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x00,0x03,0x00,0x00,0x7a,0x00,0x00,0x00,0x22,0x03,0x00,0x00, -0x51,0x00,0x05,0x00,0x49,0x00,0x00,0x00,0x01,0x03,0x00,0x00, -0xef,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0x73,0x00,0x04,0x00, -0x4c,0x00,0x00,0x00,0x02,0x03,0x00,0x00,0x01,0x03,0x00,0x00, -0x41,0x00,0x06,0x00,0x56,0x00,0x00,0x00,0x03,0x03,0x00,0x00, -0x72,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x00,0x03,0x00,0x00, -0x3e,0x00,0x03,0x00,0x03,0x03,0x00,0x00,0x02,0x03,0x00,0x00, -0xf9,0x00,0x02,0x00,0x95,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x95,0x00,0x00,0x00,0xfd,0x00,0x01,0x00,0x38,0x00,0x01,0x00, +0x10,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x1c,0x00,0x00,0x00,0x1a,0x00,0x00,0x00,0x90,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0x23,0x00,0x00,0x00,0x24,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x25,0x00,0x00,0x00,0x24,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x23,0x00,0x00,0x00,0x27,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0x26,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0x27,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x29,0x00,0x00,0x00, +0x25,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x2a,0x00,0x00,0x00,0x29,0x00,0x00,0x00, +0x10,0x00,0x00,0x00,0xae,0x00,0x05,0x00,0x11,0x00,0x00,0x00, +0x2b,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x2a,0x00,0x00,0x00, +0xf7,0x00,0x03,0x00,0x2d,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x2b,0x00,0x00,0x00,0x2c,0x00,0x00,0x00, +0x2d,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x2c,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x0c,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0x2d,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00, +0x31,0x00,0x00,0x00,0x30,0x00,0x00,0x00,0x09,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x32,0x00,0x00,0x00, +0x31,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x36,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x35,0x00,0x00,0x00, +0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0x32,0x00,0x00,0x00,0x35,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x3d,0x00,0x00,0x00,0x3b,0x00,0x00,0x00, +0x36,0x00,0x00,0x00,0x41,0x00,0x08,0x00,0x50,0x00,0x00,0x00, +0x51,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x22,0x00,0x00,0x00, +0x1c,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x09,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x41,0x00,0x00,0x00,0x52,0x00,0x00,0x00, +0x51,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x3e,0x00,0x00,0x00, +0x53,0x00,0x00,0x00,0x52,0x00,0x00,0x00,0x41,0x00,0x08,0x00, +0x50,0x00,0x00,0x00,0x57,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, +0x22,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x22,0x00,0x00,0x00, +0x56,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x41,0x00,0x00,0x00, +0x58,0x00,0x00,0x00,0x57,0x00,0x00,0x00,0x73,0x00,0x04,0x00, +0x3e,0x00,0x00,0x00,0x59,0x00,0x00,0x00,0x58,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x5c,0x00,0x00,0x00, +0x1c,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x5f,0x00,0x00,0x00,0x5d,0x00,0x00,0x00, +0x36,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x60,0x00,0x00,0x00,0x5c,0x00,0x00,0x00,0x5f,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x62,0x00,0x00,0x00, +0x3b,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x63,0x00,0x00,0x00,0x60,0x00,0x00,0x00, +0x62,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x66,0x00,0x00,0x00,0x46,0x00,0x00,0x00,0x36,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x69,0x00,0x00,0x00, +0x66,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0xb0,0x00,0x05,0x00, +0x11,0x00,0x00,0x00,0x6f,0x00,0x00,0x00,0x3d,0x00,0x00,0x00, +0x6e,0x00,0x00,0x00,0xf7,0x00,0x03,0x00,0x71,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x6f,0x00,0x00,0x00, +0x70,0x00,0x00,0x00,0x8b,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0x70,0x00,0x00,0x00,0x41,0x00,0x08,0x00,0x76,0x00,0x00,0x00, +0x77,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x22,0x00,0x00,0x00, +0x1c,0x00,0x00,0x00,0x26,0x00,0x00,0x00,0x3d,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x43,0x00,0x00,0x00,0x78,0x00,0x00,0x00, +0x77,0x00,0x00,0x00,0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x79,0x00,0x00,0x00,0x78,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, +0x21,0x00,0x00,0x00,0x7a,0x00,0x00,0x00,0x79,0x00,0x00,0x00, +0xc7,0x00,0x05,0x00,0x21,0x00,0x00,0x00,0x7c,0x00,0x00,0x00, +0x7a,0x00,0x00,0x00,0x7b,0x00,0x00,0x00,0x72,0x00,0x04,0x00, +0x7d,0x00,0x00,0x00,0x7e,0x00,0x00,0x00,0x7c,0x00,0x00,0x00, +0x7c,0x00,0x04,0x00,0x43,0x00,0x00,0x00,0x7f,0x00,0x00,0x00, +0x7e,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x83,0x00,0x00,0x00,0x3d,0x00,0x00,0x00,0x6e,0x00,0x00,0x00, +0x41,0x00,0x08,0x00,0x76,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0x4e,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x1c,0x00,0x00,0x00, +0x26,0x00,0x00,0x00,0x83,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x43,0x00,0x00,0x00,0x85,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0x85,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x21,0x00,0x00,0x00, +0x87,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0xc7,0x00,0x05,0x00, +0x21,0x00,0x00,0x00,0x88,0x00,0x00,0x00,0x87,0x00,0x00,0x00, +0x7b,0x00,0x00,0x00,0x72,0x00,0x04,0x00,0x7d,0x00,0x00,0x00, +0x89,0x00,0x00,0x00,0x88,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, +0x43,0x00,0x00,0x00,0x8a,0x00,0x00,0x00,0x89,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x71,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0x8b,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x8e,0x00,0x00,0x00,0x3d,0x00,0x00,0x00,0x6e,0x00,0x00,0x00, +0x41,0x00,0x08,0x00,0x76,0x00,0x00,0x00,0x8f,0x00,0x00,0x00, +0x4e,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x1c,0x00,0x00,0x00, +0x26,0x00,0x00,0x00,0x8e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x43,0x00,0x00,0x00,0x90,0x00,0x00,0x00,0x8f,0x00,0x00,0x00, +0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x91,0x00,0x00,0x00, +0x90,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x21,0x00,0x00,0x00, +0x92,0x00,0x00,0x00,0x91,0x00,0x00,0x00,0xc7,0x00,0x05,0x00, +0x21,0x00,0x00,0x00,0x94,0x00,0x00,0x00,0x92,0x00,0x00,0x00, +0x93,0x00,0x00,0x00,0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x97,0x00,0x00,0x00,0x3d,0x00,0x00,0x00,0x6e,0x00,0x00,0x00, +0x41,0x00,0x08,0x00,0x76,0x00,0x00,0x00,0x98,0x00,0x00,0x00, +0x4e,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x1c,0x00,0x00,0x00, +0x26,0x00,0x00,0x00,0x97,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x43,0x00,0x00,0x00,0x99,0x00,0x00,0x00,0x98,0x00,0x00,0x00, +0xc2,0x00,0x05,0x00,0x43,0x00,0x00,0x00,0x9b,0x00,0x00,0x00, +0x99,0x00,0x00,0x00,0x9a,0x00,0x00,0x00,0xc4,0x00,0x05,0x00, +0x43,0x00,0x00,0x00,0x9d,0x00,0x00,0x00,0x9b,0x00,0x00,0x00, +0x9c,0x00,0x00,0x00,0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x9e,0x00,0x00,0x00,0x9d,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, +0x21,0x00,0x00,0x00,0x9f,0x00,0x00,0x00,0x9e,0x00,0x00,0x00, +0xc5,0x00,0x05,0x00,0x21,0x00,0x00,0x00,0xa0,0x00,0x00,0x00, +0x94,0x00,0x00,0x00,0x9f,0x00,0x00,0x00,0x72,0x00,0x04,0x00, +0x7d,0x00,0x00,0x00,0xa1,0x00,0x00,0x00,0xa0,0x00,0x00,0x00, +0x7c,0x00,0x04,0x00,0x43,0x00,0x00,0x00,0xa2,0x00,0x00,0x00, +0xa1,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x43,0x00,0x00,0x00, +0xa7,0x00,0x00,0x00,0x8f,0x00,0x00,0x00,0xc2,0x00,0x05,0x00, +0x43,0x00,0x00,0x00,0xa8,0x00,0x00,0x00,0xa7,0x00,0x00,0x00, +0x9c,0x00,0x00,0x00,0x41,0x00,0x08,0x00,0x76,0x00,0x00,0x00, +0xab,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x22,0x00,0x00,0x00, +0x1c,0x00,0x00,0x00,0x26,0x00,0x00,0x00,0x3d,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x43,0x00,0x00,0x00,0xac,0x00,0x00,0x00, +0xab,0x00,0x00,0x00,0xc2,0x00,0x05,0x00,0x43,0x00,0x00,0x00, +0xad,0x00,0x00,0x00,0xac,0x00,0x00,0x00,0x9a,0x00,0x00,0x00, +0xc4,0x00,0x05,0x00,0x43,0x00,0x00,0x00,0xae,0x00,0x00,0x00, +0xad,0x00,0x00,0x00,0x9c,0x00,0x00,0x00,0xc5,0x00,0x05,0x00, +0x43,0x00,0x00,0x00,0xaf,0x00,0x00,0x00,0xa8,0x00,0x00,0x00, +0xae,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x71,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0x71,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, +0x43,0x00,0x00,0x00,0x92,0x01,0x00,0x00,0x8a,0x00,0x00,0x00, +0x70,0x00,0x00,0x00,0xaf,0x00,0x00,0x00,0x8b,0x00,0x00,0x00, +0xf5,0x00,0x07,0x00,0x43,0x00,0x00,0x00,0x91,0x01,0x00,0x00, +0x7f,0x00,0x00,0x00,0x70,0x00,0x00,0x00,0xa2,0x00,0x00,0x00, +0x8b,0x00,0x00,0x00,0x70,0x00,0x04,0x00,0x3e,0x00,0x00,0x00, +0xb3,0x00,0x00,0x00,0x91,0x01,0x00,0x00,0x85,0x00,0x05,0x00, +0x3e,0x00,0x00,0x00,0xb4,0x00,0x00,0x00,0x53,0x00,0x00,0x00, +0xb3,0x00,0x00,0x00,0x70,0x00,0x04,0x00,0x3e,0x00,0x00,0x00, +0xb8,0x00,0x00,0x00,0x92,0x01,0x00,0x00,0x85,0x00,0x05,0x00, +0x3e,0x00,0x00,0x00,0xb9,0x00,0x00,0x00,0x59,0x00,0x00,0x00, +0xb8,0x00,0x00,0x00,0xf7,0x00,0x03,0x00,0xbd,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x6f,0x00,0x00,0x00, +0xbc,0x00,0x00,0x00,0xd3,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xbc,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xc0,0x00,0x00,0x00,0x3d,0x00,0x00,0x00,0x56,0x00,0x00,0x00, +0x41,0x00,0x08,0x00,0x76,0x00,0x00,0x00,0xc1,0x00,0x00,0x00, +0x4e,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x1c,0x00,0x00,0x00, +0x26,0x00,0x00,0x00,0xc0,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x43,0x00,0x00,0x00,0xc2,0x00,0x00,0x00,0xc1,0x00,0x00,0x00, +0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xc3,0x00,0x00,0x00, +0xc2,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x21,0x00,0x00,0x00, +0xc4,0x00,0x00,0x00,0xc3,0x00,0x00,0x00,0xc7,0x00,0x05,0x00, +0x21,0x00,0x00,0x00,0xc5,0x00,0x00,0x00,0xc4,0x00,0x00,0x00, +0x7b,0x00,0x00,0x00,0x72,0x00,0x04,0x00,0x7d,0x00,0x00,0x00, +0xc6,0x00,0x00,0x00,0xc5,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, +0x43,0x00,0x00,0x00,0xc7,0x00,0x00,0x00,0xc6,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xcb,0x00,0x00,0x00, +0x3d,0x00,0x00,0x00,0xca,0x00,0x00,0x00,0x41,0x00,0x08,0x00, +0x76,0x00,0x00,0x00,0xcc,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, +0x22,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x26,0x00,0x00,0x00, +0xcb,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x43,0x00,0x00,0x00, +0xcd,0x00,0x00,0x00,0xcc,0x00,0x00,0x00,0x71,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xce,0x00,0x00,0x00,0xcd,0x00,0x00,0x00, +0x7c,0x00,0x04,0x00,0x21,0x00,0x00,0x00,0xcf,0x00,0x00,0x00, +0xce,0x00,0x00,0x00,0xc7,0x00,0x05,0x00,0x21,0x00,0x00,0x00, +0xd0,0x00,0x00,0x00,0xcf,0x00,0x00,0x00,0x7b,0x00,0x00,0x00, +0x72,0x00,0x04,0x00,0x7d,0x00,0x00,0x00,0xd1,0x00,0x00,0x00, +0xd0,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x43,0x00,0x00,0x00, +0xd2,0x00,0x00,0x00,0xd1,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xbd,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xd3,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xd6,0x00,0x00,0x00, +0x3d,0x00,0x00,0x00,0xca,0x00,0x00,0x00,0x41,0x00,0x08,0x00, +0x76,0x00,0x00,0x00,0xd7,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, +0x22,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x26,0x00,0x00,0x00, +0xd6,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x43,0x00,0x00,0x00, +0xd8,0x00,0x00,0x00,0xd7,0x00,0x00,0x00,0x71,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xd9,0x00,0x00,0x00,0xd8,0x00,0x00,0x00, +0x7c,0x00,0x04,0x00,0x21,0x00,0x00,0x00,0xda,0x00,0x00,0x00, +0xd9,0x00,0x00,0x00,0xc7,0x00,0x05,0x00,0x21,0x00,0x00,0x00, +0xdb,0x00,0x00,0x00,0xda,0x00,0x00,0x00,0x93,0x00,0x00,0x00, +0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xdf,0x00,0x00,0x00, +0x3d,0x00,0x00,0x00,0xde,0x00,0x00,0x00,0x41,0x00,0x08,0x00, +0x76,0x00,0x00,0x00,0xe0,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, +0x22,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x26,0x00,0x00,0x00, +0xdf,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x43,0x00,0x00,0x00, +0xe1,0x00,0x00,0x00,0xe0,0x00,0x00,0x00,0xc2,0x00,0x05,0x00, +0x43,0x00,0x00,0x00,0xe2,0x00,0x00,0x00,0xe1,0x00,0x00,0x00, +0x9a,0x00,0x00,0x00,0xc4,0x00,0x05,0x00,0x43,0x00,0x00,0x00, +0xe3,0x00,0x00,0x00,0xe2,0x00,0x00,0x00,0x9c,0x00,0x00,0x00, +0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xe4,0x00,0x00,0x00, +0xe3,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x21,0x00,0x00,0x00, +0xe5,0x00,0x00,0x00,0xe4,0x00,0x00,0x00,0xc5,0x00,0x05,0x00, +0x21,0x00,0x00,0x00,0xe6,0x00,0x00,0x00,0xdb,0x00,0x00,0x00, +0xe5,0x00,0x00,0x00,0x72,0x00,0x04,0x00,0x7d,0x00,0x00,0x00, +0xe7,0x00,0x00,0x00,0xe6,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, +0x43,0x00,0x00,0x00,0xe8,0x00,0x00,0x00,0xe7,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x43,0x00,0x00,0x00,0xed,0x00,0x00,0x00, +0xd7,0x00,0x00,0x00,0xc2,0x00,0x05,0x00,0x43,0x00,0x00,0x00, +0xee,0x00,0x00,0x00,0xed,0x00,0x00,0x00,0x9c,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xf1,0x00,0x00,0x00, +0x3d,0x00,0x00,0x00,0x56,0x00,0x00,0x00,0x41,0x00,0x08,0x00, +0x76,0x00,0x00,0x00,0xf2,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, +0x22,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x26,0x00,0x00,0x00, +0xf1,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x43,0x00,0x00,0x00, +0xf3,0x00,0x00,0x00,0xf2,0x00,0x00,0x00,0xc2,0x00,0x05,0x00, +0x43,0x00,0x00,0x00,0xf4,0x00,0x00,0x00,0xf3,0x00,0x00,0x00, +0x9a,0x00,0x00,0x00,0xc4,0x00,0x05,0x00,0x43,0x00,0x00,0x00, +0xf5,0x00,0x00,0x00,0xf4,0x00,0x00,0x00,0x9c,0x00,0x00,0x00, +0xc5,0x00,0x05,0x00,0x43,0x00,0x00,0x00,0xf6,0x00,0x00,0x00, +0xee,0x00,0x00,0x00,0xf5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xbd,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xbd,0x00,0x00,0x00, +0xf5,0x00,0x07,0x00,0x43,0x00,0x00,0x00,0x94,0x01,0x00,0x00, +0xd2,0x00,0x00,0x00,0xbc,0x00,0x00,0x00,0xf6,0x00,0x00,0x00, +0xd3,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x43,0x00,0x00,0x00, +0x93,0x01,0x00,0x00,0xc7,0x00,0x00,0x00,0xbc,0x00,0x00,0x00, +0xe8,0x00,0x00,0x00,0xd3,0x00,0x00,0x00,0x70,0x00,0x04,0x00, +0x3e,0x00,0x00,0x00,0xfa,0x00,0x00,0x00,0x93,0x01,0x00,0x00, +0x85,0x00,0x05,0x00,0x3e,0x00,0x00,0x00,0xfb,0x00,0x00,0x00, +0x53,0x00,0x00,0x00,0xfa,0x00,0x00,0x00,0x70,0x00,0x04,0x00, +0x3e,0x00,0x00,0x00,0xff,0x00,0x00,0x00,0x94,0x01,0x00,0x00, +0x85,0x00,0x05,0x00,0x3e,0x00,0x00,0x00,0x00,0x01,0x00,0x00, +0x59,0x00,0x00,0x00,0xff,0x00,0x00,0x00,0xc4,0x00,0x05,0x00, +0x21,0x00,0x00,0x00,0x04,0x01,0x00,0x00,0x26,0x00,0x00,0x00, +0x3d,0x00,0x00,0x00,0x72,0x00,0x04,0x00,0x7d,0x00,0x00,0x00, +0x05,0x01,0x00,0x00,0x04,0x01,0x00,0x00,0x7c,0x00,0x04,0x00, +0x43,0x00,0x00,0x00,0x06,0x01,0x00,0x00,0x05,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x0a,0x01,0x00,0x00, +0x3d,0x00,0x00,0x00,0x56,0x00,0x00,0x00,0xc4,0x00,0x05,0x00, +0x21,0x00,0x00,0x00,0x0b,0x01,0x00,0x00,0x26,0x00,0x00,0x00, +0x0a,0x01,0x00,0x00,0x72,0x00,0x04,0x00,0x7d,0x00,0x00,0x00, +0x0c,0x01,0x00,0x00,0x0b,0x01,0x00,0x00,0x7c,0x00,0x04,0x00, +0x43,0x00,0x00,0x00,0x0d,0x01,0x00,0x00,0x0c,0x01,0x00,0x00, +0x41,0x00,0x08,0x00,0x76,0x00,0x00,0x00,0x17,0x01,0x00,0x00, +0x4e,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x1c,0x00,0x00,0x00, +0x15,0x01,0x00,0x00,0x69,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x43,0x00,0x00,0x00,0x18,0x01,0x00,0x00,0x17,0x01,0x00,0x00, +0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x19,0x01,0x00,0x00, +0x18,0x01,0x00,0x00,0x7c,0x00,0x04,0x00,0x21,0x00,0x00,0x00, +0x1a,0x01,0x00,0x00,0x19,0x01,0x00,0x00,0xc7,0x00,0x05,0x00, +0x21,0x00,0x00,0x00,0x1b,0x01,0x00,0x00,0x1a,0x01,0x00,0x00, +0x93,0x00,0x00,0x00,0x41,0x00,0x08,0x00,0x76,0x00,0x00,0x00, +0x1f,0x01,0x00,0x00,0x4e,0x00,0x00,0x00,0x22,0x00,0x00,0x00, +0x1c,0x00,0x00,0x00,0x1d,0x01,0x00,0x00,0x62,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x43,0x00,0x00,0x00,0x20,0x01,0x00,0x00, +0x1f,0x01,0x00,0x00,0xc7,0x00,0x05,0x00,0x43,0x00,0x00,0x00, +0x22,0x01,0x00,0x00,0x20,0x01,0x00,0x00,0x06,0x01,0x00,0x00, +0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x23,0x01,0x00,0x00, +0x22,0x01,0x00,0x00,0x7c,0x00,0x04,0x00,0x21,0x00,0x00,0x00, +0x24,0x01,0x00,0x00,0x23,0x01,0x00,0x00,0xab,0x00,0x05,0x00, +0x11,0x00,0x00,0x00,0x25,0x01,0x00,0x00,0x24,0x01,0x00,0x00, +0x22,0x00,0x00,0x00,0xa9,0x00,0x06,0x00,0x21,0x00,0x00,0x00, +0x27,0x01,0x00,0x00,0x25,0x01,0x00,0x00,0x26,0x01,0x00,0x00, +0x22,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x21,0x00,0x00,0x00, +0x28,0x01,0x00,0x00,0x1b,0x01,0x00,0x00,0x27,0x01,0x00,0x00, +0x6f,0x00,0x04,0x00,0x3e,0x00,0x00,0x00,0x29,0x01,0x00,0x00, +0x28,0x01,0x00,0x00,0x7f,0x00,0x04,0x00,0x3e,0x00,0x00,0x00, +0x9b,0x01,0x00,0x00,0xb9,0x00,0x00,0x00,0x0c,0x00,0x08,0x00, +0x3e,0x00,0x00,0x00,0x2c,0x01,0x00,0x00,0x01,0x00,0x00,0x00, +0x32,0x00,0x00,0x00,0xb4,0x00,0x00,0x00,0x29,0x01,0x00,0x00, +0x9b,0x01,0x00,0x00,0x73,0x00,0x04,0x00,0x41,0x00,0x00,0x00, +0x2d,0x01,0x00,0x00,0x2c,0x01,0x00,0x00,0x41,0x00,0x06,0x00, +0x50,0x00,0x00,0x00,0x2e,0x01,0x00,0x00,0x11,0x01,0x00,0x00, +0x22,0x00,0x00,0x00,0x63,0x00,0x00,0x00,0x3e,0x00,0x03,0x00, +0x2e,0x01,0x00,0x00,0x2d,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x30,0x01,0x00,0x00,0x63,0x00,0x00,0x00, +0x56,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x34,0x01,0x00,0x00,0x69,0x00,0x00,0x00,0x56,0x00,0x00,0x00, +0x41,0x00,0x08,0x00,0x76,0x00,0x00,0x00,0x35,0x01,0x00,0x00, +0x4e,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x1c,0x00,0x00,0x00, +0x15,0x01,0x00,0x00,0x34,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, +0x43,0x00,0x00,0x00,0x36,0x01,0x00,0x00,0x35,0x01,0x00,0x00, +0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x37,0x01,0x00,0x00, +0x36,0x01,0x00,0x00,0x7c,0x00,0x04,0x00,0x21,0x00,0x00,0x00, +0x38,0x01,0x00,0x00,0x37,0x01,0x00,0x00,0xc7,0x00,0x05,0x00, +0x21,0x00,0x00,0x00,0x39,0x01,0x00,0x00,0x38,0x01,0x00,0x00, +0x93,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x3c,0x01,0x00,0x00,0x62,0x00,0x00,0x00,0x56,0x00,0x00,0x00, +0x41,0x00,0x08,0x00,0x76,0x00,0x00,0x00,0x3d,0x01,0x00,0x00, +0x4e,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x1c,0x00,0x00,0x00, +0x1d,0x01,0x00,0x00,0x3c,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, +0x43,0x00,0x00,0x00,0x3e,0x01,0x00,0x00,0x3d,0x01,0x00,0x00, +0xc7,0x00,0x05,0x00,0x43,0x00,0x00,0x00,0x40,0x01,0x00,0x00, +0x3e,0x01,0x00,0x00,0x06,0x01,0x00,0x00,0x71,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x41,0x01,0x00,0x00,0x40,0x01,0x00,0x00, +0x7c,0x00,0x04,0x00,0x21,0x00,0x00,0x00,0x42,0x01,0x00,0x00, +0x41,0x01,0x00,0x00,0xab,0x00,0x05,0x00,0x11,0x00,0x00,0x00, +0x43,0x01,0x00,0x00,0x42,0x01,0x00,0x00,0x22,0x00,0x00,0x00, +0xa9,0x00,0x06,0x00,0x21,0x00,0x00,0x00,0x44,0x01,0x00,0x00, +0x43,0x01,0x00,0x00,0x26,0x01,0x00,0x00,0x22,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x21,0x00,0x00,0x00,0x45,0x01,0x00,0x00, +0x39,0x01,0x00,0x00,0x44,0x01,0x00,0x00,0x6f,0x00,0x04,0x00, +0x3e,0x00,0x00,0x00,0x46,0x01,0x00,0x00,0x45,0x01,0x00,0x00, +0x0c,0x00,0x08,0x00,0x3e,0x00,0x00,0x00,0x49,0x01,0x00,0x00, +0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0xb4,0x00,0x00,0x00, +0x46,0x01,0x00,0x00,0x9b,0x01,0x00,0x00,0x73,0x00,0x04,0x00, +0x41,0x00,0x00,0x00,0x4a,0x01,0x00,0x00,0x49,0x01,0x00,0x00, +0x41,0x00,0x06,0x00,0x50,0x00,0x00,0x00,0x4b,0x01,0x00,0x00, +0x11,0x01,0x00,0x00,0x22,0x00,0x00,0x00,0x30,0x01,0x00,0x00, +0x3e,0x00,0x03,0x00,0x4b,0x01,0x00,0x00,0x4a,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x4d,0x01,0x00,0x00, +0x63,0x00,0x00,0x00,0x46,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x43,0x00,0x00,0x00,0x52,0x01,0x00,0x00,0x17,0x01,0x00,0x00, +0xc2,0x00,0x05,0x00,0x43,0x00,0x00,0x00,0x53,0x01,0x00,0x00, +0x52,0x01,0x00,0x00,0x9c,0x00,0x00,0x00,0x71,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x54,0x01,0x00,0x00,0x53,0x01,0x00,0x00, +0x7c,0x00,0x04,0x00,0x21,0x00,0x00,0x00,0x55,0x01,0x00,0x00, +0x54,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0x43,0x00,0x00,0x00, +0x59,0x01,0x00,0x00,0x1f,0x01,0x00,0x00,0xc7,0x00,0x05,0x00, +0x43,0x00,0x00,0x00,0x5b,0x01,0x00,0x00,0x59,0x01,0x00,0x00, +0x0d,0x01,0x00,0x00,0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x5c,0x01,0x00,0x00,0x5b,0x01,0x00,0x00,0x7c,0x00,0x04,0x00, +0x21,0x00,0x00,0x00,0x5d,0x01,0x00,0x00,0x5c,0x01,0x00,0x00, +0xab,0x00,0x05,0x00,0x11,0x00,0x00,0x00,0x5e,0x01,0x00,0x00, +0x5d,0x01,0x00,0x00,0x22,0x00,0x00,0x00,0xa9,0x00,0x06,0x00, +0x21,0x00,0x00,0x00,0x5f,0x01,0x00,0x00,0x5e,0x01,0x00,0x00, +0x26,0x01,0x00,0x00,0x22,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x21,0x00,0x00,0x00,0x60,0x01,0x00,0x00,0x55,0x01,0x00,0x00, +0x5f,0x01,0x00,0x00,0x6f,0x00,0x04,0x00,0x3e,0x00,0x00,0x00, +0x61,0x01,0x00,0x00,0x60,0x01,0x00,0x00,0x7f,0x00,0x04,0x00, +0x3e,0x00,0x00,0x00,0x9d,0x01,0x00,0x00,0x00,0x01,0x00,0x00, +0x0c,0x00,0x08,0x00,0x3e,0x00,0x00,0x00,0x64,0x01,0x00,0x00, +0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0xfb,0x00,0x00,0x00, +0x61,0x01,0x00,0x00,0x9d,0x01,0x00,0x00,0x73,0x00,0x04,0x00, +0x41,0x00,0x00,0x00,0x65,0x01,0x00,0x00,0x64,0x01,0x00,0x00, +0x41,0x00,0x06,0x00,0x50,0x00,0x00,0x00,0x66,0x01,0x00,0x00, +0x11,0x01,0x00,0x00,0x22,0x00,0x00,0x00,0x4d,0x01,0x00,0x00, +0x3e,0x00,0x03,0x00,0x66,0x01,0x00,0x00,0x65,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x69,0x01,0x00,0x00, +0x63,0x00,0x00,0x00,0x68,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, +0x43,0x00,0x00,0x00,0x6f,0x01,0x00,0x00,0x35,0x01,0x00,0x00, +0xc2,0x00,0x05,0x00,0x43,0x00,0x00,0x00,0x70,0x01,0x00,0x00, +0x6f,0x01,0x00,0x00,0x9c,0x00,0x00,0x00,0x71,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x71,0x01,0x00,0x00,0x70,0x01,0x00,0x00, +0x7c,0x00,0x04,0x00,0x21,0x00,0x00,0x00,0x72,0x01,0x00,0x00, +0x71,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0x43,0x00,0x00,0x00, +0x77,0x01,0x00,0x00,0x3d,0x01,0x00,0x00,0xc7,0x00,0x05,0x00, +0x43,0x00,0x00,0x00,0x79,0x01,0x00,0x00,0x77,0x01,0x00,0x00, +0x0d,0x01,0x00,0x00,0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x7a,0x01,0x00,0x00,0x79,0x01,0x00,0x00,0x7c,0x00,0x04,0x00, +0x21,0x00,0x00,0x00,0x7b,0x01,0x00,0x00,0x7a,0x01,0x00,0x00, +0xab,0x00,0x05,0x00,0x11,0x00,0x00,0x00,0x7c,0x01,0x00,0x00, +0x7b,0x01,0x00,0x00,0x22,0x00,0x00,0x00,0xa9,0x00,0x06,0x00, +0x21,0x00,0x00,0x00,0x7d,0x01,0x00,0x00,0x7c,0x01,0x00,0x00, +0x26,0x01,0x00,0x00,0x22,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x21,0x00,0x00,0x00,0x7e,0x01,0x00,0x00,0x72,0x01,0x00,0x00, +0x7d,0x01,0x00,0x00,0x6f,0x00,0x04,0x00,0x3e,0x00,0x00,0x00, +0x7f,0x01,0x00,0x00,0x7e,0x01,0x00,0x00,0x0c,0x00,0x08,0x00, +0x3e,0x00,0x00,0x00,0x82,0x01,0x00,0x00,0x01,0x00,0x00,0x00, +0x32,0x00,0x00,0x00,0xfb,0x00,0x00,0x00,0x7f,0x01,0x00,0x00, +0x9d,0x01,0x00,0x00,0x73,0x00,0x04,0x00,0x41,0x00,0x00,0x00, +0x83,0x01,0x00,0x00,0x82,0x01,0x00,0x00,0x41,0x00,0x06,0x00, +0x50,0x00,0x00,0x00,0x84,0x01,0x00,0x00,0x11,0x01,0x00,0x00, +0x22,0x00,0x00,0x00,0x69,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, +0x84,0x01,0x00,0x00,0x83,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0x0d,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x0d,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x86,0x01,0x00,0x00, +0x90,0x01,0x00,0x00,0x26,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x0a,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x0c,0x00,0x00,0x00, +0xf5,0x00,0x07,0x00,0x11,0x00,0x00,0x00,0x97,0x01,0x00,0x00, +0x8a,0x01,0x00,0x00,0x0a,0x00,0x00,0x00,0x8d,0x01,0x00,0x00, +0x2c,0x00,0x00,0x00,0xf7,0x00,0x03,0x00,0x8e,0x01,0x00,0x00, +0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x97,0x01,0x00,0x00, +0x88,0x01,0x00,0x00,0x8e,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x8e,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x88,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x88,0x01,0x00,0x00,0xfd,0x00,0x01,0x00, +0x38,0x00,0x01,0x00, +}; +const uint64_t dequant_q5_K_len = 5980; + +unsigned char dequant_q6_K_data[] = { +0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, +0x0d,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, +0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x27,0x00,0x00,0x00, +0x11,0x00,0x02,0x00,0x51,0x11,0x00,0x00,0x11,0x00,0x02,0x00, +0x60,0x11,0x00,0x00,0x0b,0x00,0x06,0x00,0x01,0x00,0x00,0x00, +0x47,0x4c,0x53,0x4c,0x2e,0x73,0x74,0x64,0x2e,0x34,0x35,0x30, +0x00,0x00,0x00,0x00,0x0e,0x00,0x03,0x00,0x00,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x0f,0x00,0x0a,0x00,0x05,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x6d,0x61,0x69,0x6e,0x00,0x00,0x00,0x00, +0x16,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x30,0x00,0x00,0x00, +0x5f,0x00,0x00,0x00,0x74,0x00,0x00,0x00,0x10,0x00,0x06,0x00, +0x04,0x00,0x00,0x00,0x11,0x00,0x00,0x00,0x40,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x16,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x1a,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x1e,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x1e,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x1e,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x08,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x1e,0x00,0x00,0x00,0x03,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x1e,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x10,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x1e,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x30,0x00,0x00,0x00, +0x0b,0x00,0x00,0x00,0x1b,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x56,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x57,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x59,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x5b,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x5b,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x80,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x5b,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0xc0,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x5b,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0xd0,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x5c,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0xd2,0x00,0x00,0x00,0x48,0x00,0x04,0x00, +0x5d,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x5d,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, +0x5d,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x5f,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x5f,0x00,0x00,0x00,0x21,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x71,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x48,0x00,0x04,0x00, +0x72,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x19,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x72,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, +0x72,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x74,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x74,0x00,0x00,0x00,0x21,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x02,0x01,0x00,0x00, +0x0b,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x13,0x00,0x02,0x00, +0x02,0x00,0x00,0x00,0x21,0x00,0x03,0x00,0x03,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x15,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x10,0x00,0x00,0x00, +0x00,0x01,0x00,0x00,0x14,0x00,0x02,0x00,0x11,0x00,0x00,0x00, +0x17,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x03,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x15,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0x15,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x17,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x1e,0x00,0x07,0x00,0x1e,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x1f,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x1e,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0x1f,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0x09,0x00,0x00,0x00,0x15,0x00,0x04,0x00,0x21,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x21,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x23,0x00,0x00,0x00,0x09,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x21,0x00,0x00,0x00, +0x26,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0x15,0x00,0x00,0x00,0x30,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x35,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x3d,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x41,0x00,0x00,0x00,0x10,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x47,0x00,0x00,0x00, +0x80,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x4e,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x15,0x00,0x04,0x00, +0x53,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x1c,0x00,0x04,0x00,0x56,0x00,0x00,0x00,0x53,0x00,0x00,0x00, +0x47,0x00,0x00,0x00,0x1c,0x00,0x04,0x00,0x57,0x00,0x00,0x00, +0x53,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x15,0x00,0x04,0x00, +0x58,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x1c,0x00,0x04,0x00,0x59,0x00,0x00,0x00,0x58,0x00,0x00,0x00, +0x41,0x00,0x00,0x00,0x16,0x00,0x03,0x00,0x5a,0x00,0x00,0x00, +0x10,0x00,0x00,0x00,0x1e,0x00,0x06,0x00,0x5b,0x00,0x00,0x00, +0x56,0x00,0x00,0x00,0x57,0x00,0x00,0x00,0x59,0x00,0x00,0x00, +0x5a,0x00,0x00,0x00,0x1d,0x00,0x03,0x00,0x5c,0x00,0x00,0x00, +0x5b,0x00,0x00,0x00,0x1e,0x00,0x03,0x00,0x5d,0x00,0x00,0x00, +0x5c,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x5e,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x5d,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0x5e,0x00,0x00,0x00,0x5f,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x65,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x53,0x00,0x00,0x00,0x16,0x00,0x03,0x00,0x68,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x21,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x6d,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x5a,0x00,0x00,0x00, +0x1d,0x00,0x03,0x00,0x71,0x00,0x00,0x00,0x5a,0x00,0x00,0x00, +0x1e,0x00,0x03,0x00,0x72,0x00,0x00,0x00,0x71,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x73,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x72,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x73,0x00,0x00,0x00, +0x74,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x21,0x00,0x00,0x00,0x79,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x7c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x58,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x21,0x00,0x00,0x00, +0x87,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x21,0x00,0x00,0x00,0x8e,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x21,0x00,0x00,0x00,0x93,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x9f,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xc0,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xdd,0x00,0x00,0x00, +0x60,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xe2,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x21,0x00,0x00,0x00,0xf0,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x01,0x01,0x00,0x00, +0x01,0x00,0x00,0x00,0x2c,0x00,0x06,0x00,0x14,0x00,0x00,0x00, +0x02,0x01,0x00,0x00,0x4e,0x00,0x00,0x00,0x01,0x01,0x00,0x00, +0x01,0x01,0x00,0x00,0x2a,0x00,0x03,0x00,0x11,0x00,0x00,0x00, +0x05,0x01,0x00,0x00,0x29,0x00,0x03,0x00,0x11,0x00,0x00,0x00, +0x08,0x01,0x00,0x00,0x36,0x00,0x05,0x00,0x02,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x03,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0x05,0x00,0x00,0x00,0xf7,0x00,0x03,0x00, +0x03,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0xfb,0x00,0x03,0x00, +0x09,0x00,0x00,0x00,0x04,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x04,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x0a,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0x0a,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0x0b,0x01,0x00,0x00,0x09,0x00,0x00,0x00, +0x04,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x0d,0x00,0x00,0x00, +0xb0,0x00,0x05,0x00,0x11,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0x0b,0x01,0x00,0x00,0x10,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0x0c,0x00,0x00,0x00,0x0d,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x12,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x0b,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00,0x18,0x00,0x00,0x00, +0x16,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x18,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x1a,0x00,0x00,0x00, +0x19,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x1a,0x00,0x00,0x00, +0x0b,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0x23,0x00,0x00,0x00, +0x24,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x22,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x25,0x00,0x00,0x00, +0x24,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x23,0x00,0x00,0x00, +0x27,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x26,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x28,0x00,0x00,0x00, +0x27,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x29,0x00,0x00,0x00,0x25,0x00,0x00,0x00,0x28,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x2a,0x00,0x00,0x00, +0x29,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0xae,0x00,0x05,0x00, +0x11,0x00,0x00,0x00,0x2b,0x00,0x00,0x00,0x1c,0x00,0x00,0x00, +0x2a,0x00,0x00,0x00,0xf7,0x00,0x03,0x00,0x2d,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x2b,0x00,0x00,0x00, +0x2c,0x00,0x00,0x00,0x2d,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0x2c,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x0c,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0x2d,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x17,0x00,0x00,0x00,0x31,0x00,0x00,0x00,0x30,0x00,0x00,0x00, +0x09,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x32,0x00,0x00,0x00,0x31,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x36,0x00,0x00,0x00,0x32,0x00,0x00,0x00, +0x35,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x3a,0x00,0x00,0x00,0x35,0x00,0x00,0x00,0x36,0x00,0x00,0x00, +0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3b,0x00,0x00,0x00, +0x32,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x3f,0x00,0x00,0x00,0x3d,0x00,0x00,0x00, +0x36,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x42,0x00,0x00,0x00,0x3b,0x00,0x00,0x00,0x41,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x43,0x00,0x00,0x00, +0x3f,0x00,0x00,0x00,0x42,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x46,0x00,0x00,0x00,0x1c,0x00,0x00,0x00, +0x10,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x49,0x00,0x00,0x00,0x47,0x00,0x00,0x00,0x36,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x4a,0x00,0x00,0x00, +0x46,0x00,0x00,0x00,0x49,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x4c,0x00,0x00,0x00,0x4a,0x00,0x00,0x00, +0x3b,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x50,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x36,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x52,0x00,0x00,0x00, +0x50,0x00,0x00,0x00,0x3b,0x00,0x00,0x00,0x41,0x00,0x08,0x00, +0x65,0x00,0x00,0x00,0x66,0x00,0x00,0x00,0x5f,0x00,0x00,0x00, +0x22,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x26,0x00,0x00,0x00, +0x32,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x53,0x00,0x00,0x00, +0x67,0x00,0x00,0x00,0x66,0x00,0x00,0x00,0x41,0x00,0x07,0x00, +0x6d,0x00,0x00,0x00,0x6e,0x00,0x00,0x00,0x5f,0x00,0x00,0x00, +0x22,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x5a,0x00,0x00,0x00,0x6f,0x00,0x00,0x00, +0x6e,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x68,0x00,0x00,0x00, +0x70,0x00,0x00,0x00,0x6f,0x00,0x00,0x00,0x41,0x00,0x08,0x00, +0x7c,0x00,0x00,0x00,0x7d,0x00,0x00,0x00,0x5f,0x00,0x00,0x00, +0x22,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x79,0x00,0x00,0x00, +0x43,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x58,0x00,0x00,0x00, +0x7e,0x00,0x00,0x00,0x7d,0x00,0x00,0x00,0x72,0x00,0x04,0x00, +0x21,0x00,0x00,0x00,0x7f,0x00,0x00,0x00,0x7e,0x00,0x00,0x00, +0x41,0x00,0x08,0x00,0x65,0x00,0x00,0x00,0x83,0x00,0x00,0x00, +0x5f,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x1c,0x00,0x00,0x00, +0x22,0x00,0x00,0x00,0x52,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x53,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x83,0x00,0x00,0x00, +0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x85,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x21,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0x85,0x00,0x00,0x00,0xc7,0x00,0x05,0x00, +0x21,0x00,0x00,0x00,0x88,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0x87,0x00,0x00,0x00,0xc2,0x00,0x05,0x00,0x53,0x00,0x00,0x00, +0x8a,0x00,0x00,0x00,0x67,0x00,0x00,0x00,0x22,0x00,0x00,0x00, +0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x8b,0x00,0x00,0x00, +0x8a,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x21,0x00,0x00,0x00, +0x8c,0x00,0x00,0x00,0x8b,0x00,0x00,0x00,0xc7,0x00,0x05,0x00, +0x21,0x00,0x00,0x00,0x8d,0x00,0x00,0x00,0x8c,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0xc4,0x00,0x05,0x00,0x21,0x00,0x00,0x00, +0x8f,0x00,0x00,0x00,0x8d,0x00,0x00,0x00,0x8e,0x00,0x00,0x00, +0xc5,0x00,0x05,0x00,0x21,0x00,0x00,0x00,0x90,0x00,0x00,0x00, +0x88,0x00,0x00,0x00,0x8f,0x00,0x00,0x00,0x72,0x00,0x04,0x00, +0x58,0x00,0x00,0x00,0x91,0x00,0x00,0x00,0x90,0x00,0x00,0x00, +0x72,0x00,0x04,0x00,0x21,0x00,0x00,0x00,0x92,0x00,0x00,0x00, +0x91,0x00,0x00,0x00,0x82,0x00,0x05,0x00,0x21,0x00,0x00,0x00, +0x94,0x00,0x00,0x00,0x92,0x00,0x00,0x00,0x93,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x21,0x00,0x00,0x00,0x95,0x00,0x00,0x00, +0x7f,0x00,0x00,0x00,0x94,0x00,0x00,0x00,0x6f,0x00,0x04,0x00, +0x68,0x00,0x00,0x00,0x96,0x00,0x00,0x00,0x95,0x00,0x00,0x00, +0x85,0x00,0x05,0x00,0x68,0x00,0x00,0x00,0x97,0x00,0x00,0x00, +0x70,0x00,0x00,0x00,0x96,0x00,0x00,0x00,0x73,0x00,0x04,0x00, +0x5a,0x00,0x00,0x00,0x98,0x00,0x00,0x00,0x97,0x00,0x00,0x00, +0x41,0x00,0x06,0x00,0x6d,0x00,0x00,0x00,0x99,0x00,0x00,0x00, +0x74,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x4c,0x00,0x00,0x00, +0x3e,0x00,0x03,0x00,0x99,0x00,0x00,0x00,0x98,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9b,0x00,0x00,0x00, +0x4c,0x00,0x00,0x00,0x35,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xa0,0x00,0x00,0x00,0x43,0x00,0x00,0x00, +0x9f,0x00,0x00,0x00,0x41,0x00,0x08,0x00,0x7c,0x00,0x00,0x00, +0xa1,0x00,0x00,0x00,0x5f,0x00,0x00,0x00,0x22,0x00,0x00,0x00, +0x1c,0x00,0x00,0x00,0x79,0x00,0x00,0x00,0xa0,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x58,0x00,0x00,0x00,0xa2,0x00,0x00,0x00, +0xa1,0x00,0x00,0x00,0x72,0x00,0x04,0x00,0x21,0x00,0x00,0x00, +0xa3,0x00,0x00,0x00,0xa2,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xa6,0x00,0x00,0x00,0x52,0x00,0x00,0x00, +0x35,0x00,0x00,0x00,0x41,0x00,0x08,0x00,0x65,0x00,0x00,0x00, +0xa7,0x00,0x00,0x00,0x5f,0x00,0x00,0x00,0x22,0x00,0x00,0x00, +0x1c,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0xa6,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x53,0x00,0x00,0x00,0xa8,0x00,0x00,0x00, +0xa7,0x00,0x00,0x00,0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xa9,0x00,0x00,0x00,0xa8,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, +0x21,0x00,0x00,0x00,0xaa,0x00,0x00,0x00,0xa9,0x00,0x00,0x00, +0xc7,0x00,0x05,0x00,0x21,0x00,0x00,0x00,0xab,0x00,0x00,0x00, +0xaa,0x00,0x00,0x00,0x87,0x00,0x00,0x00,0xc2,0x00,0x05,0x00, +0x53,0x00,0x00,0x00,0xad,0x00,0x00,0x00,0x67,0x00,0x00,0x00, +0x79,0x00,0x00,0x00,0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xae,0x00,0x00,0x00,0xad,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, +0x21,0x00,0x00,0x00,0xaf,0x00,0x00,0x00,0xae,0x00,0x00,0x00, +0xc7,0x00,0x05,0x00,0x21,0x00,0x00,0x00,0xb0,0x00,0x00,0x00, +0xaf,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0xc4,0x00,0x05,0x00, +0x21,0x00,0x00,0x00,0xb1,0x00,0x00,0x00,0xb0,0x00,0x00,0x00, +0x8e,0x00,0x00,0x00,0xc5,0x00,0x05,0x00,0x21,0x00,0x00,0x00, +0xb2,0x00,0x00,0x00,0xab,0x00,0x00,0x00,0xb1,0x00,0x00,0x00, +0x72,0x00,0x04,0x00,0x58,0x00,0x00,0x00,0xb3,0x00,0x00,0x00, +0xb2,0x00,0x00,0x00,0x72,0x00,0x04,0x00,0x21,0x00,0x00,0x00, +0xb4,0x00,0x00,0x00,0xb3,0x00,0x00,0x00,0x82,0x00,0x05,0x00, +0x21,0x00,0x00,0x00,0xb5,0x00,0x00,0x00,0xb4,0x00,0x00,0x00, +0x93,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x21,0x00,0x00,0x00, +0xb6,0x00,0x00,0x00,0xa3,0x00,0x00,0x00,0xb5,0x00,0x00,0x00, +0x6f,0x00,0x04,0x00,0x68,0x00,0x00,0x00,0xb7,0x00,0x00,0x00, +0xb6,0x00,0x00,0x00,0x85,0x00,0x05,0x00,0x68,0x00,0x00,0x00, +0xb8,0x00,0x00,0x00,0x70,0x00,0x00,0x00,0xb7,0x00,0x00,0x00, +0x73,0x00,0x04,0x00,0x5a,0x00,0x00,0x00,0xb9,0x00,0x00,0x00, +0xb8,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x6d,0x00,0x00,0x00, +0xba,0x00,0x00,0x00,0x74,0x00,0x00,0x00,0x22,0x00,0x00,0x00, +0x9b,0x00,0x00,0x00,0x3e,0x00,0x03,0x00,0xba,0x00,0x00,0x00, +0xb9,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xbc,0x00,0x00,0x00,0x4c,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xc1,0x00,0x00,0x00, +0x43,0x00,0x00,0x00,0xc0,0x00,0x00,0x00,0x41,0x00,0x08,0x00, +0x7c,0x00,0x00,0x00,0xc2,0x00,0x00,0x00,0x5f,0x00,0x00,0x00, +0x22,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x79,0x00,0x00,0x00, +0xc1,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x58,0x00,0x00,0x00, +0xc3,0x00,0x00,0x00,0xc2,0x00,0x00,0x00,0x72,0x00,0x04,0x00, +0x21,0x00,0x00,0x00,0xc4,0x00,0x00,0x00,0xc3,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x53,0x00,0x00,0x00,0xc9,0x00,0x00,0x00, +0x83,0x00,0x00,0x00,0xc2,0x00,0x05,0x00,0x53,0x00,0x00,0x00, +0xca,0x00,0x00,0x00,0xc9,0x00,0x00,0x00,0x8e,0x00,0x00,0x00, +0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xcb,0x00,0x00,0x00, +0xca,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x21,0x00,0x00,0x00, +0xcc,0x00,0x00,0x00,0xcb,0x00,0x00,0x00,0xc2,0x00,0x05,0x00, +0x53,0x00,0x00,0x00,0xce,0x00,0x00,0x00,0x67,0x00,0x00,0x00, +0x8e,0x00,0x00,0x00,0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xcf,0x00,0x00,0x00,0xce,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, +0x21,0x00,0x00,0x00,0xd0,0x00,0x00,0x00,0xcf,0x00,0x00,0x00, +0xc7,0x00,0x05,0x00,0x21,0x00,0x00,0x00,0xd1,0x00,0x00,0x00, +0xd0,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0xc4,0x00,0x05,0x00, +0x21,0x00,0x00,0x00,0xd2,0x00,0x00,0x00,0xd1,0x00,0x00,0x00, +0x8e,0x00,0x00,0x00,0xc5,0x00,0x05,0x00,0x21,0x00,0x00,0x00, +0xd3,0x00,0x00,0x00,0xcc,0x00,0x00,0x00,0xd2,0x00,0x00,0x00, +0x72,0x00,0x04,0x00,0x58,0x00,0x00,0x00,0xd4,0x00,0x00,0x00, +0xd3,0x00,0x00,0x00,0x72,0x00,0x04,0x00,0x21,0x00,0x00,0x00, +0xd5,0x00,0x00,0x00,0xd4,0x00,0x00,0x00,0x82,0x00,0x05,0x00, +0x21,0x00,0x00,0x00,0xd6,0x00,0x00,0x00,0xd5,0x00,0x00,0x00, +0x93,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x21,0x00,0x00,0x00, +0xd7,0x00,0x00,0x00,0xc4,0x00,0x00,0x00,0xd6,0x00,0x00,0x00, +0x6f,0x00,0x04,0x00,0x68,0x00,0x00,0x00,0xd8,0x00,0x00,0x00, +0xd7,0x00,0x00,0x00,0x85,0x00,0x05,0x00,0x68,0x00,0x00,0x00, +0xd9,0x00,0x00,0x00,0x70,0x00,0x00,0x00,0xd8,0x00,0x00,0x00, +0x73,0x00,0x04,0x00,0x5a,0x00,0x00,0x00,0xda,0x00,0x00,0x00, +0xd9,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x6d,0x00,0x00,0x00, +0xdb,0x00,0x00,0x00,0x74,0x00,0x00,0x00,0x22,0x00,0x00,0x00, +0xbc,0x00,0x00,0x00,0x3e,0x00,0x03,0x00,0xdb,0x00,0x00,0x00, +0xda,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xde,0x00,0x00,0x00,0x4c,0x00,0x00,0x00,0xdd,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xe3,0x00,0x00,0x00, +0x43,0x00,0x00,0x00,0xe2,0x00,0x00,0x00,0x41,0x00,0x08,0x00, +0x7c,0x00,0x00,0x00,0xe4,0x00,0x00,0x00,0x5f,0x00,0x00,0x00, +0x22,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x79,0x00,0x00,0x00, +0xe3,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x58,0x00,0x00,0x00, +0xe5,0x00,0x00,0x00,0xe4,0x00,0x00,0x00,0x72,0x00,0x04,0x00, +0x21,0x00,0x00,0x00,0xe6,0x00,0x00,0x00,0xe5,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x53,0x00,0x00,0x00,0xeb,0x00,0x00,0x00, +0xa7,0x00,0x00,0x00,0xc2,0x00,0x05,0x00,0x53,0x00,0x00,0x00, +0xec,0x00,0x00,0x00,0xeb,0x00,0x00,0x00,0x8e,0x00,0x00,0x00, +0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xed,0x00,0x00,0x00, +0xec,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x21,0x00,0x00,0x00, +0xee,0x00,0x00,0x00,0xed,0x00,0x00,0x00,0xc2,0x00,0x05,0x00, +0x53,0x00,0x00,0x00,0xf1,0x00,0x00,0x00,0x67,0x00,0x00,0x00, +0xf0,0x00,0x00,0x00,0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xf2,0x00,0x00,0x00,0xf1,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, +0x21,0x00,0x00,0x00,0xf3,0x00,0x00,0x00,0xf2,0x00,0x00,0x00, +0xc7,0x00,0x05,0x00,0x21,0x00,0x00,0x00,0xf4,0x00,0x00,0x00, +0xf3,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0xc4,0x00,0x05,0x00, +0x21,0x00,0x00,0x00,0xf5,0x00,0x00,0x00,0xf4,0x00,0x00,0x00, +0x8e,0x00,0x00,0x00,0xc5,0x00,0x05,0x00,0x21,0x00,0x00,0x00, +0xf6,0x00,0x00,0x00,0xee,0x00,0x00,0x00,0xf5,0x00,0x00,0x00, +0x72,0x00,0x04,0x00,0x58,0x00,0x00,0x00,0xf7,0x00,0x00,0x00, +0xf6,0x00,0x00,0x00,0x72,0x00,0x04,0x00,0x21,0x00,0x00,0x00, +0xf8,0x00,0x00,0x00,0xf7,0x00,0x00,0x00,0x82,0x00,0x05,0x00, +0x21,0x00,0x00,0x00,0xf9,0x00,0x00,0x00,0xf8,0x00,0x00,0x00, +0x93,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x21,0x00,0x00,0x00, +0xfa,0x00,0x00,0x00,0xe6,0x00,0x00,0x00,0xf9,0x00,0x00,0x00, +0x6f,0x00,0x04,0x00,0x68,0x00,0x00,0x00,0xfb,0x00,0x00,0x00, +0xfa,0x00,0x00,0x00,0x85,0x00,0x05,0x00,0x68,0x00,0x00,0x00, +0xfc,0x00,0x00,0x00,0x70,0x00,0x00,0x00,0xfb,0x00,0x00,0x00, +0x73,0x00,0x04,0x00,0x5a,0x00,0x00,0x00,0xfd,0x00,0x00,0x00, +0xfc,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x6d,0x00,0x00,0x00, +0xfe,0x00,0x00,0x00,0x74,0x00,0x00,0x00,0x22,0x00,0x00,0x00, +0xde,0x00,0x00,0x00,0x3e,0x00,0x03,0x00,0xfe,0x00,0x00,0x00, +0xfd,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x0d,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0x0d,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x0b,0x01,0x00,0x00, +0x26,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x0a,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0x0c,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, +0x11,0x00,0x00,0x00,0x0c,0x01,0x00,0x00,0x05,0x01,0x00,0x00, +0x0a,0x00,0x00,0x00,0x08,0x01,0x00,0x00,0x2c,0x00,0x00,0x00, +0xf7,0x00,0x03,0x00,0x09,0x01,0x00,0x00,0x00,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x0c,0x01,0x00,0x00,0x03,0x01,0x00,0x00, +0x09,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x09,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0x03,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x03,0x01,0x00,0x00,0xfd,0x00,0x01,0x00,0x38,0x00,0x01,0x00, }; -const uint64_t dequant_q8_0_len = 8868; +const uint64_t dequant_q6_K_len = 4272; + +unsigned char dequant_q8_0_data[] = { +0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, +0x4c,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, +0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x27,0x00,0x00,0x00, +0x11,0x00,0x02,0x00,0x51,0x11,0x00,0x00,0x11,0x00,0x02,0x00, +0x60,0x11,0x00,0x00,0x0b,0x00,0x06,0x00,0x01,0x00,0x00,0x00, +0x47,0x4c,0x53,0x4c,0x2e,0x73,0x74,0x64,0x2e,0x34,0x35,0x30, +0x00,0x00,0x00,0x00,0x0e,0x00,0x03,0x00,0x00,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x0f,0x00,0x0a,0x00,0x05,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x6d,0x61,0x69,0x6e,0x00,0x00,0x00,0x00, +0x0b,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x2b,0x00,0x00,0x00, +0x4c,0x00,0x00,0x00,0x61,0x00,0x00,0x00,0x10,0x00,0x06,0x00, +0x04,0x00,0x00,0x00,0x11,0x00,0x00,0x00,0x00,0x01,0x00,0x00, +0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x0b,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x1a,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x12,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, +0x1b,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x29,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x29,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x29,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x08,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x29,0x00,0x00,0x00, +0x03,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x29,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x47,0x00,0x03,0x00, +0x29,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x47,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x48,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x48,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x49,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x48,0x00,0x04,0x00, +0x4a,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x4a,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, +0x4a,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x4c,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x4c,0x00,0x00,0x00,0x21,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x5e,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x48,0x00,0x04,0x00, +0x5f,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x19,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x5f,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, +0x5f,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x61,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x61,0x00,0x00,0x00,0x21,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x87,0x00,0x00,0x00, +0x0b,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x13,0x00,0x02,0x00, +0x02,0x00,0x00,0x00,0x21,0x00,0x03,0x00,0x03,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x15,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x17,0x00,0x04,0x00, +0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x03,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x0a,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x09,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00, +0x0b,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x0d,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x10,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0x0a,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x15,0x00,0x00,0x00, +0x40,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x1e,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x1e,0x00,0x07,0x00, +0x29,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x2a,0x00,0x00,0x00,0x09,0x00,0x00,0x00, +0x29,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x2a,0x00,0x00,0x00, +0x2b,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x15,0x00,0x04,0x00, +0x2c,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x2c,0x00,0x00,0x00,0x2d,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x2e,0x00,0x00,0x00, +0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x14,0x00,0x02,0x00, +0x32,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x38,0x00,0x00,0x00,0x00,0x04,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x3e,0x00,0x00,0x00,0x10,0x00,0x00,0x00, +0x16,0x00,0x03,0x00,0x42,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0x16,0x00,0x03,0x00,0x45,0x00,0x00,0x00,0x10,0x00,0x00,0x00, +0x15,0x00,0x04,0x00,0x46,0x00,0x00,0x00,0x08,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x1c,0x00,0x04,0x00,0x47,0x00,0x00,0x00, +0x46,0x00,0x00,0x00,0x1e,0x00,0x00,0x00,0x1e,0x00,0x04,0x00, +0x48,0x00,0x00,0x00,0x45,0x00,0x00,0x00,0x47,0x00,0x00,0x00, +0x1d,0x00,0x03,0x00,0x49,0x00,0x00,0x00,0x48,0x00,0x00,0x00, +0x1e,0x00,0x03,0x00,0x4a,0x00,0x00,0x00,0x49,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x4b,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x4a,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x4b,0x00,0x00,0x00, +0x4c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x2c,0x00,0x00,0x00,0x4d,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x4f,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x45,0x00,0x00,0x00,0x1d,0x00,0x03,0x00,0x5e,0x00,0x00,0x00, +0x45,0x00,0x00,0x00,0x1e,0x00,0x03,0x00,0x5f,0x00,0x00,0x00, +0x5e,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x60,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x5f,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0x60,0x00,0x00,0x00,0x61,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x2c,0x00,0x00,0x00,0x67,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x6b,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x46,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x75,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x83,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x2c,0x00,0x06,0x00, +0x09,0x00,0x00,0x00,0x87,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0x75,0x00,0x00,0x00,0x75,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x40,0x01,0x00,0x00,0x03,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x41,0x01,0x00,0x00, +0x05,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x42,0x01,0x00,0x00,0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x43,0x01,0x00,0x00,0x07,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x44,0x01,0x00,0x00, +0x08,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x45,0x01,0x00,0x00,0x09,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x46,0x01,0x00,0x00,0x0a,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x47,0x01,0x00,0x00, +0x0b,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x48,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x49,0x01,0x00,0x00,0x0d,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x4a,0x01,0x00,0x00, +0x0e,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x4b,0x01,0x00,0x00,0x0f,0x00,0x00,0x00,0x36,0x00,0x05,0x00, +0x02,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x03,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x05,0x00,0x00,0x00, +0xf7,0x00,0x03,0x00,0x88,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0xfb,0x00,0x03,0x00,0x0c,0x00,0x00,0x00,0x89,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0x89,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x0d,0x00,0x00,0x00,0x0e,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x0f,0x00,0x00,0x00,0x0e,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x11,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, +0x10,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00, +0x13,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x14,0x00,0x00,0x00, +0x13,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x16,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x15,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x17,0x00,0x00,0x00, +0x11,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x89,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x1b,0x00,0x00,0x00,0x14,0x00,0x00,0x00, +0x15,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x1f,0x00,0x00,0x00,0x1b,0x00,0x00,0x00,0x1e,0x00,0x00,0x00, +0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x22,0x00,0x00,0x00, +0x1b,0x00,0x00,0x00,0x1e,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x25,0x00,0x00,0x00,0x1e,0x00,0x00,0x00, +0x17,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x27,0x00,0x00,0x00,0x25,0x00,0x00,0x00,0x22,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x2e,0x00,0x00,0x00,0x2f,0x00,0x00,0x00, +0x2b,0x00,0x00,0x00,0x2d,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x30,0x00,0x00,0x00,0x2f,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x31,0x00,0x00,0x00, +0x30,0x00,0x00,0x00,0x1e,0x00,0x00,0x00,0xae,0x00,0x05,0x00, +0x32,0x00,0x00,0x00,0x33,0x00,0x00,0x00,0x27,0x00,0x00,0x00, +0x31,0x00,0x00,0x00,0xf7,0x00,0x03,0x00,0x35,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x33,0x00,0x00,0x00, +0x34,0x00,0x00,0x00,0x35,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0x34,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x88,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0x35,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x38,0x00,0x00,0x00, +0x17,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x3c,0x00,0x00,0x00,0x1e,0x00,0x00,0x00,0x22,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3d,0x00,0x00,0x00, +0x3a,0x00,0x00,0x00,0x3c,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x3e,0x00,0x00,0x00, +0x1f,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x41,0x00,0x00,0x00,0x3d,0x00,0x00,0x00,0x40,0x00,0x00,0x00, +0x41,0x00,0x07,0x00,0x4f,0x00,0x00,0x00,0x50,0x00,0x00,0x00, +0x4c,0x00,0x00,0x00,0x4d,0x00,0x00,0x00,0x27,0x00,0x00,0x00, +0x4d,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x45,0x00,0x00,0x00, +0x51,0x00,0x00,0x00,0x50,0x00,0x00,0x00,0x73,0x00,0x04,0x00, +0x42,0x00,0x00,0x00,0x52,0x00,0x00,0x00,0x51,0x00,0x00,0x00, +0x41,0x00,0x08,0x00,0x6b,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, +0x4c,0x00,0x00,0x00,0x4d,0x00,0x00,0x00,0x27,0x00,0x00,0x00, +0x67,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x46,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, +0x6f,0x00,0x04,0x00,0x42,0x00,0x00,0x00,0x6e,0x00,0x00,0x00, +0x6d,0x00,0x00,0x00,0x85,0x00,0x05,0x00,0x42,0x00,0x00,0x00, +0x6f,0x00,0x00,0x00,0x52,0x00,0x00,0x00,0x6e,0x00,0x00,0x00, +0x73,0x00,0x04,0x00,0x45,0x00,0x00,0x00,0x70,0x00,0x00,0x00, +0x6f,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x4f,0x00,0x00,0x00, +0x71,0x00,0x00,0x00,0x61,0x00,0x00,0x00,0x4d,0x00,0x00,0x00, +0x41,0x00,0x00,0x00,0x3e,0x00,0x03,0x00,0x71,0x00,0x00,0x00, +0x70,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x76,0x00,0x00,0x00,0x41,0x00,0x00,0x00,0x75,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x7c,0x00,0x00,0x00, +0x40,0x00,0x00,0x00,0x75,0x00,0x00,0x00,0x41,0x00,0x08,0x00, +0x6b,0x00,0x00,0x00,0x7d,0x00,0x00,0x00,0x4c,0x00,0x00,0x00, +0x4d,0x00,0x00,0x00,0x27,0x00,0x00,0x00,0x67,0x00,0x00,0x00, +0x7c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x46,0x00,0x00,0x00, +0x7e,0x00,0x00,0x00,0x7d,0x00,0x00,0x00,0x6f,0x00,0x04,0x00, +0x42,0x00,0x00,0x00,0x7f,0x00,0x00,0x00,0x7e,0x00,0x00,0x00, +0x85,0x00,0x05,0x00,0x42,0x00,0x00,0x00,0x80,0x00,0x00,0x00, +0x52,0x00,0x00,0x00,0x7f,0x00,0x00,0x00,0x73,0x00,0x04,0x00, +0x45,0x00,0x00,0x00,0x81,0x00,0x00,0x00,0x80,0x00,0x00,0x00, +0x41,0x00,0x06,0x00,0x4f,0x00,0x00,0x00,0x82,0x00,0x00,0x00, +0x61,0x00,0x00,0x00,0x4d,0x00,0x00,0x00,0x76,0x00,0x00,0x00, +0x3e,0x00,0x03,0x00,0x82,0x00,0x00,0x00,0x81,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x95,0x00,0x00,0x00, +0x41,0x00,0x00,0x00,0x83,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x96,0x00,0x00,0x00,0x40,0x00,0x00,0x00, +0x83,0x00,0x00,0x00,0x41,0x00,0x08,0x00,0x6b,0x00,0x00,0x00, +0x97,0x00,0x00,0x00,0x4c,0x00,0x00,0x00,0x4d,0x00,0x00,0x00, +0x27,0x00,0x00,0x00,0x67,0x00,0x00,0x00,0x96,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x46,0x00,0x00,0x00,0x98,0x00,0x00,0x00, +0x97,0x00,0x00,0x00,0x6f,0x00,0x04,0x00,0x42,0x00,0x00,0x00, +0x99,0x00,0x00,0x00,0x98,0x00,0x00,0x00,0x85,0x00,0x05,0x00, +0x42,0x00,0x00,0x00,0x9a,0x00,0x00,0x00,0x52,0x00,0x00,0x00, +0x99,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x45,0x00,0x00,0x00, +0x9b,0x00,0x00,0x00,0x9a,0x00,0x00,0x00,0x41,0x00,0x06,0x00, +0x4f,0x00,0x00,0x00,0x9c,0x00,0x00,0x00,0x61,0x00,0x00,0x00, +0x4d,0x00,0x00,0x00,0x95,0x00,0x00,0x00,0x3e,0x00,0x03,0x00, +0x9c,0x00,0x00,0x00,0x9b,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x9e,0x00,0x00,0x00,0x41,0x00,0x00,0x00, +0x40,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xa0,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x40,0x01,0x00,0x00, +0x41,0x00,0x08,0x00,0x6b,0x00,0x00,0x00,0xa1,0x00,0x00,0x00, +0x4c,0x00,0x00,0x00,0x4d,0x00,0x00,0x00,0x27,0x00,0x00,0x00, +0x67,0x00,0x00,0x00,0xa0,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x46,0x00,0x00,0x00,0xa2,0x00,0x00,0x00,0xa1,0x00,0x00,0x00, +0x6f,0x00,0x04,0x00,0x42,0x00,0x00,0x00,0xa3,0x00,0x00,0x00, +0xa2,0x00,0x00,0x00,0x85,0x00,0x05,0x00,0x42,0x00,0x00,0x00, +0xa4,0x00,0x00,0x00,0x52,0x00,0x00,0x00,0xa3,0x00,0x00,0x00, +0x73,0x00,0x04,0x00,0x45,0x00,0x00,0x00,0xa5,0x00,0x00,0x00, +0xa4,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x4f,0x00,0x00,0x00, +0xa6,0x00,0x00,0x00,0x61,0x00,0x00,0x00,0x4d,0x00,0x00,0x00, +0x9e,0x00,0x00,0x00,0x3e,0x00,0x03,0x00,0xa6,0x00,0x00,0x00, +0xa5,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xae,0x00,0x00,0x00,0x41,0x00,0x00,0x00,0x10,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xaf,0x00,0x00,0x00, +0x40,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x41,0x00,0x08,0x00, +0x6b,0x00,0x00,0x00,0xb0,0x00,0x00,0x00,0x4c,0x00,0x00,0x00, +0x4d,0x00,0x00,0x00,0x27,0x00,0x00,0x00,0x67,0x00,0x00,0x00, +0xaf,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x46,0x00,0x00,0x00, +0xb1,0x00,0x00,0x00,0xb0,0x00,0x00,0x00,0x6f,0x00,0x04,0x00, +0x42,0x00,0x00,0x00,0xb2,0x00,0x00,0x00,0xb1,0x00,0x00,0x00, +0x85,0x00,0x05,0x00,0x42,0x00,0x00,0x00,0xb3,0x00,0x00,0x00, +0x52,0x00,0x00,0x00,0xb2,0x00,0x00,0x00,0x73,0x00,0x04,0x00, +0x45,0x00,0x00,0x00,0xb4,0x00,0x00,0x00,0xb3,0x00,0x00,0x00, +0x41,0x00,0x06,0x00,0x4f,0x00,0x00,0x00,0xb5,0x00,0x00,0x00, +0x61,0x00,0x00,0x00,0x4d,0x00,0x00,0x00,0xae,0x00,0x00,0x00, +0x3e,0x00,0x03,0x00,0xb5,0x00,0x00,0x00,0xb4,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xb7,0x00,0x00,0x00, +0x41,0x00,0x00,0x00,0x41,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xb9,0x00,0x00,0x00,0x40,0x00,0x00,0x00, +0x41,0x01,0x00,0x00,0x41,0x00,0x08,0x00,0x6b,0x00,0x00,0x00, +0xba,0x00,0x00,0x00,0x4c,0x00,0x00,0x00,0x4d,0x00,0x00,0x00, +0x27,0x00,0x00,0x00,0x67,0x00,0x00,0x00,0xb9,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x46,0x00,0x00,0x00,0xbb,0x00,0x00,0x00, +0xba,0x00,0x00,0x00,0x6f,0x00,0x04,0x00,0x42,0x00,0x00,0x00, +0xbc,0x00,0x00,0x00,0xbb,0x00,0x00,0x00,0x85,0x00,0x05,0x00, +0x42,0x00,0x00,0x00,0xbd,0x00,0x00,0x00,0x52,0x00,0x00,0x00, +0xbc,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x45,0x00,0x00,0x00, +0xbe,0x00,0x00,0x00,0xbd,0x00,0x00,0x00,0x41,0x00,0x06,0x00, +0x4f,0x00,0x00,0x00,0xbf,0x00,0x00,0x00,0x61,0x00,0x00,0x00, +0x4d,0x00,0x00,0x00,0xb7,0x00,0x00,0x00,0x3e,0x00,0x03,0x00, +0xbf,0x00,0x00,0x00,0xbe,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xc7,0x00,0x00,0x00,0x41,0x00,0x00,0x00, +0x42,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xc8,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x42,0x01,0x00,0x00, +0x41,0x00,0x08,0x00,0x6b,0x00,0x00,0x00,0xc9,0x00,0x00,0x00, +0x4c,0x00,0x00,0x00,0x4d,0x00,0x00,0x00,0x27,0x00,0x00,0x00, +0x67,0x00,0x00,0x00,0xc8,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x46,0x00,0x00,0x00,0xca,0x00,0x00,0x00,0xc9,0x00,0x00,0x00, +0x6f,0x00,0x04,0x00,0x42,0x00,0x00,0x00,0xcb,0x00,0x00,0x00, +0xca,0x00,0x00,0x00,0x85,0x00,0x05,0x00,0x42,0x00,0x00,0x00, +0xcc,0x00,0x00,0x00,0x52,0x00,0x00,0x00,0xcb,0x00,0x00,0x00, +0x73,0x00,0x04,0x00,0x45,0x00,0x00,0x00,0xcd,0x00,0x00,0x00, +0xcc,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x4f,0x00,0x00,0x00, +0xce,0x00,0x00,0x00,0x61,0x00,0x00,0x00,0x4d,0x00,0x00,0x00, +0xc7,0x00,0x00,0x00,0x3e,0x00,0x03,0x00,0xce,0x00,0x00,0x00, +0xcd,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xd0,0x00,0x00,0x00,0x41,0x00,0x00,0x00,0x43,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xd2,0x00,0x00,0x00, +0x40,0x00,0x00,0x00,0x43,0x01,0x00,0x00,0x41,0x00,0x08,0x00, +0x6b,0x00,0x00,0x00,0xd3,0x00,0x00,0x00,0x4c,0x00,0x00,0x00, +0x4d,0x00,0x00,0x00,0x27,0x00,0x00,0x00,0x67,0x00,0x00,0x00, +0xd2,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x46,0x00,0x00,0x00, +0xd4,0x00,0x00,0x00,0xd3,0x00,0x00,0x00,0x6f,0x00,0x04,0x00, +0x42,0x00,0x00,0x00,0xd5,0x00,0x00,0x00,0xd4,0x00,0x00,0x00, +0x85,0x00,0x05,0x00,0x42,0x00,0x00,0x00,0xd6,0x00,0x00,0x00, +0x52,0x00,0x00,0x00,0xd5,0x00,0x00,0x00,0x73,0x00,0x04,0x00, +0x45,0x00,0x00,0x00,0xd7,0x00,0x00,0x00,0xd6,0x00,0x00,0x00, +0x41,0x00,0x06,0x00,0x4f,0x00,0x00,0x00,0xd8,0x00,0x00,0x00, +0x61,0x00,0x00,0x00,0x4d,0x00,0x00,0x00,0xd0,0x00,0x00,0x00, +0x3e,0x00,0x03,0x00,0xd8,0x00,0x00,0x00,0xd7,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xe0,0x00,0x00,0x00, +0x41,0x00,0x00,0x00,0x44,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xe1,0x00,0x00,0x00,0x40,0x00,0x00,0x00, +0x44,0x01,0x00,0x00,0x41,0x00,0x08,0x00,0x6b,0x00,0x00,0x00, +0xe2,0x00,0x00,0x00,0x4c,0x00,0x00,0x00,0x4d,0x00,0x00,0x00, +0x27,0x00,0x00,0x00,0x67,0x00,0x00,0x00,0xe1,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x46,0x00,0x00,0x00,0xe3,0x00,0x00,0x00, +0xe2,0x00,0x00,0x00,0x6f,0x00,0x04,0x00,0x42,0x00,0x00,0x00, +0xe4,0x00,0x00,0x00,0xe3,0x00,0x00,0x00,0x85,0x00,0x05,0x00, +0x42,0x00,0x00,0x00,0xe5,0x00,0x00,0x00,0x52,0x00,0x00,0x00, +0xe4,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x45,0x00,0x00,0x00, +0xe6,0x00,0x00,0x00,0xe5,0x00,0x00,0x00,0x41,0x00,0x06,0x00, +0x4f,0x00,0x00,0x00,0xe7,0x00,0x00,0x00,0x61,0x00,0x00,0x00, +0x4d,0x00,0x00,0x00,0xe0,0x00,0x00,0x00,0x3e,0x00,0x03,0x00, +0xe7,0x00,0x00,0x00,0xe6,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xe9,0x00,0x00,0x00,0x41,0x00,0x00,0x00, +0x45,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xeb,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x45,0x01,0x00,0x00, +0x41,0x00,0x08,0x00,0x6b,0x00,0x00,0x00,0xec,0x00,0x00,0x00, +0x4c,0x00,0x00,0x00,0x4d,0x00,0x00,0x00,0x27,0x00,0x00,0x00, +0x67,0x00,0x00,0x00,0xeb,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x46,0x00,0x00,0x00,0xed,0x00,0x00,0x00,0xec,0x00,0x00,0x00, +0x6f,0x00,0x04,0x00,0x42,0x00,0x00,0x00,0xee,0x00,0x00,0x00, +0xed,0x00,0x00,0x00,0x85,0x00,0x05,0x00,0x42,0x00,0x00,0x00, +0xef,0x00,0x00,0x00,0x52,0x00,0x00,0x00,0xee,0x00,0x00,0x00, +0x73,0x00,0x04,0x00,0x45,0x00,0x00,0x00,0xf0,0x00,0x00,0x00, +0xef,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x4f,0x00,0x00,0x00, +0xf1,0x00,0x00,0x00,0x61,0x00,0x00,0x00,0x4d,0x00,0x00,0x00, +0xe9,0x00,0x00,0x00,0x3e,0x00,0x03,0x00,0xf1,0x00,0x00,0x00, +0xf0,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xf9,0x00,0x00,0x00,0x41,0x00,0x00,0x00,0x46,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xfa,0x00,0x00,0x00, +0x40,0x00,0x00,0x00,0x46,0x01,0x00,0x00,0x41,0x00,0x08,0x00, +0x6b,0x00,0x00,0x00,0xfb,0x00,0x00,0x00,0x4c,0x00,0x00,0x00, +0x4d,0x00,0x00,0x00,0x27,0x00,0x00,0x00,0x67,0x00,0x00,0x00, +0xfa,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x46,0x00,0x00,0x00, +0xfc,0x00,0x00,0x00,0xfb,0x00,0x00,0x00,0x6f,0x00,0x04,0x00, +0x42,0x00,0x00,0x00,0xfd,0x00,0x00,0x00,0xfc,0x00,0x00,0x00, +0x85,0x00,0x05,0x00,0x42,0x00,0x00,0x00,0xfe,0x00,0x00,0x00, +0x52,0x00,0x00,0x00,0xfd,0x00,0x00,0x00,0x73,0x00,0x04,0x00, +0x45,0x00,0x00,0x00,0xff,0x00,0x00,0x00,0xfe,0x00,0x00,0x00, +0x41,0x00,0x06,0x00,0x4f,0x00,0x00,0x00,0x00,0x01,0x00,0x00, +0x61,0x00,0x00,0x00,0x4d,0x00,0x00,0x00,0xf9,0x00,0x00,0x00, +0x3e,0x00,0x03,0x00,0x00,0x01,0x00,0x00,0xff,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x02,0x01,0x00,0x00, +0x41,0x00,0x00,0x00,0x47,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x04,0x01,0x00,0x00,0x40,0x00,0x00,0x00, +0x47,0x01,0x00,0x00,0x41,0x00,0x08,0x00,0x6b,0x00,0x00,0x00, +0x05,0x01,0x00,0x00,0x4c,0x00,0x00,0x00,0x4d,0x00,0x00,0x00, +0x27,0x00,0x00,0x00,0x67,0x00,0x00,0x00,0x04,0x01,0x00,0x00, +0x3d,0x00,0x04,0x00,0x46,0x00,0x00,0x00,0x06,0x01,0x00,0x00, +0x05,0x01,0x00,0x00,0x6f,0x00,0x04,0x00,0x42,0x00,0x00,0x00, +0x07,0x01,0x00,0x00,0x06,0x01,0x00,0x00,0x85,0x00,0x05,0x00, +0x42,0x00,0x00,0x00,0x08,0x01,0x00,0x00,0x52,0x00,0x00,0x00, +0x07,0x01,0x00,0x00,0x73,0x00,0x04,0x00,0x45,0x00,0x00,0x00, +0x09,0x01,0x00,0x00,0x08,0x01,0x00,0x00,0x41,0x00,0x06,0x00, +0x4f,0x00,0x00,0x00,0x0a,0x01,0x00,0x00,0x61,0x00,0x00,0x00, +0x4d,0x00,0x00,0x00,0x02,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, +0x0a,0x01,0x00,0x00,0x09,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x12,0x01,0x00,0x00,0x41,0x00,0x00,0x00, +0x48,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x13,0x01,0x00,0x00,0x40,0x00,0x00,0x00,0x48,0x01,0x00,0x00, +0x41,0x00,0x08,0x00,0x6b,0x00,0x00,0x00,0x14,0x01,0x00,0x00, +0x4c,0x00,0x00,0x00,0x4d,0x00,0x00,0x00,0x27,0x00,0x00,0x00, +0x67,0x00,0x00,0x00,0x13,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, +0x46,0x00,0x00,0x00,0x15,0x01,0x00,0x00,0x14,0x01,0x00,0x00, +0x6f,0x00,0x04,0x00,0x42,0x00,0x00,0x00,0x16,0x01,0x00,0x00, +0x15,0x01,0x00,0x00,0x85,0x00,0x05,0x00,0x42,0x00,0x00,0x00, +0x17,0x01,0x00,0x00,0x52,0x00,0x00,0x00,0x16,0x01,0x00,0x00, +0x73,0x00,0x04,0x00,0x45,0x00,0x00,0x00,0x18,0x01,0x00,0x00, +0x17,0x01,0x00,0x00,0x41,0x00,0x06,0x00,0x4f,0x00,0x00,0x00, +0x19,0x01,0x00,0x00,0x61,0x00,0x00,0x00,0x4d,0x00,0x00,0x00, +0x12,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x19,0x01,0x00,0x00, +0x18,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x1b,0x01,0x00,0x00,0x41,0x00,0x00,0x00,0x49,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x1d,0x01,0x00,0x00, +0x40,0x00,0x00,0x00,0x49,0x01,0x00,0x00,0x41,0x00,0x08,0x00, +0x6b,0x00,0x00,0x00,0x1e,0x01,0x00,0x00,0x4c,0x00,0x00,0x00, +0x4d,0x00,0x00,0x00,0x27,0x00,0x00,0x00,0x67,0x00,0x00,0x00, +0x1d,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0x46,0x00,0x00,0x00, +0x1f,0x01,0x00,0x00,0x1e,0x01,0x00,0x00,0x6f,0x00,0x04,0x00, +0x42,0x00,0x00,0x00,0x20,0x01,0x00,0x00,0x1f,0x01,0x00,0x00, +0x85,0x00,0x05,0x00,0x42,0x00,0x00,0x00,0x21,0x01,0x00,0x00, +0x52,0x00,0x00,0x00,0x20,0x01,0x00,0x00,0x73,0x00,0x04,0x00, +0x45,0x00,0x00,0x00,0x22,0x01,0x00,0x00,0x21,0x01,0x00,0x00, +0x41,0x00,0x06,0x00,0x4f,0x00,0x00,0x00,0x23,0x01,0x00,0x00, +0x61,0x00,0x00,0x00,0x4d,0x00,0x00,0x00,0x1b,0x01,0x00,0x00, +0x3e,0x00,0x03,0x00,0x23,0x01,0x00,0x00,0x22,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x2b,0x01,0x00,0x00, +0x41,0x00,0x00,0x00,0x4a,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x2c,0x01,0x00,0x00,0x40,0x00,0x00,0x00, +0x4a,0x01,0x00,0x00,0x41,0x00,0x08,0x00,0x6b,0x00,0x00,0x00, +0x2d,0x01,0x00,0x00,0x4c,0x00,0x00,0x00,0x4d,0x00,0x00,0x00, +0x27,0x00,0x00,0x00,0x67,0x00,0x00,0x00,0x2c,0x01,0x00,0x00, +0x3d,0x00,0x04,0x00,0x46,0x00,0x00,0x00,0x2e,0x01,0x00,0x00, +0x2d,0x01,0x00,0x00,0x6f,0x00,0x04,0x00,0x42,0x00,0x00,0x00, +0x2f,0x01,0x00,0x00,0x2e,0x01,0x00,0x00,0x85,0x00,0x05,0x00, +0x42,0x00,0x00,0x00,0x30,0x01,0x00,0x00,0x52,0x00,0x00,0x00, +0x2f,0x01,0x00,0x00,0x73,0x00,0x04,0x00,0x45,0x00,0x00,0x00, +0x31,0x01,0x00,0x00,0x30,0x01,0x00,0x00,0x41,0x00,0x06,0x00, +0x4f,0x00,0x00,0x00,0x32,0x01,0x00,0x00,0x61,0x00,0x00,0x00, +0x4d,0x00,0x00,0x00,0x2b,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, +0x32,0x01,0x00,0x00,0x31,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x34,0x01,0x00,0x00,0x41,0x00,0x00,0x00, +0x4b,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x36,0x01,0x00,0x00,0x40,0x00,0x00,0x00,0x4b,0x01,0x00,0x00, +0x41,0x00,0x08,0x00,0x6b,0x00,0x00,0x00,0x37,0x01,0x00,0x00, +0x4c,0x00,0x00,0x00,0x4d,0x00,0x00,0x00,0x27,0x00,0x00,0x00, +0x67,0x00,0x00,0x00,0x36,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, +0x46,0x00,0x00,0x00,0x38,0x01,0x00,0x00,0x37,0x01,0x00,0x00, +0x6f,0x00,0x04,0x00,0x42,0x00,0x00,0x00,0x39,0x01,0x00,0x00, +0x38,0x01,0x00,0x00,0x85,0x00,0x05,0x00,0x42,0x00,0x00,0x00, +0x3a,0x01,0x00,0x00,0x52,0x00,0x00,0x00,0x39,0x01,0x00,0x00, +0x73,0x00,0x04,0x00,0x45,0x00,0x00,0x00,0x3b,0x01,0x00,0x00, +0x3a,0x01,0x00,0x00,0x41,0x00,0x06,0x00,0x4f,0x00,0x00,0x00, +0x3c,0x01,0x00,0x00,0x61,0x00,0x00,0x00,0x4d,0x00,0x00,0x00, +0x34,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x3c,0x01,0x00,0x00, +0x3b,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x88,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0x88,0x00,0x00,0x00,0xfd,0x00,0x01,0x00, +0x38,0x00,0x01,0x00, +}; +const uint64_t dequant_q8_0_len = 4780; unsigned char diag_mask_inf_f32_data[] = { 0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, @@ -7890,144 +6689,6 @@ unsigned char diag_mask_inf_f32_data[] = { }; const uint64_t diag_mask_inf_f32_len = 1480; -unsigned char f32_to_f16_data[] = { -0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, -0x53,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, -0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x51,0x11,0x00,0x00, -0x0b,0x00,0x06,0x00,0x01,0x00,0x00,0x00,0x47,0x4c,0x53,0x4c, -0x2e,0x73,0x74,0x64,0x2e,0x34,0x35,0x30,0x00,0x00,0x00,0x00, -0x0e,0x00,0x03,0x00,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x0f,0x00,0x09,0x00,0x05,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x6d,0x61,0x69,0x6e,0x00,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x13,0x00,0x00,0x00,0x36,0x00,0x00,0x00,0x42,0x00,0x00,0x00, -0x10,0x00,0x06,0x00,0x04,0x00,0x00,0x00,0x11,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x0c,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x1c,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x11,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x11,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x11,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x08,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x11,0x00,0x00,0x00, -0x03,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x47,0x00,0x03,0x00,0x11,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x33,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0x34,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x34,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x34,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x36,0x00,0x00,0x00, -0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x36,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x3f,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0x40,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x40,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x40,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x42,0x00,0x00,0x00, -0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x42,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x52,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x19,0x00,0x00,0x00,0x13,0x00,0x02,0x00,0x02,0x00,0x00,0x00, -0x21,0x00,0x03,0x00,0x03,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x15,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x15,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x17,0x00,0x04,0x00, -0x0a,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x03,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x0b,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x0a,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x0b,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x0e,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x1e,0x00,0x06,0x00,0x11,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x12,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x11,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x12,0x00,0x00,0x00,0x13,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x14,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x15,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x14,0x00,0x02,0x00, -0x23,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x2b,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x16,0x00,0x03,0x00, -0x32,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, -0x33,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, -0x34,0x00,0x00,0x00,0x33,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x35,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x34,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x35,0x00,0x00,0x00,0x36,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x38,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x16,0x00,0x03,0x00, -0x3e,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, -0x3f,0x00,0x00,0x00,0x3e,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, -0x40,0x00,0x00,0x00,0x3f,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x41,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x41,0x00,0x00,0x00,0x42,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x44,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x4a,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x3e,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x4e,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x32,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x50,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x51,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x2c,0x00,0x06,0x00,0x0a,0x00,0x00,0x00,0x52,0x00,0x00,0x00, -0x50,0x00,0x00,0x00,0x51,0x00,0x00,0x00,0x51,0x00,0x00,0x00, -0x36,0x00,0x05,0x00,0x02,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x05,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x0e,0x00,0x00,0x00, -0x0f,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x0d,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x10,0x00,0x00,0x00, -0x0f,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x13,0x00,0x00,0x00,0x14,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x17,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x17,0x00,0x00,0x00,0x89,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x10,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x1a,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x10,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x22,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0xb1,0x00,0x05,0x00, -0x23,0x00,0x00,0x00,0x27,0x00,0x00,0x00,0x1a,0x00,0x00,0x00, -0x17,0x00,0x00,0x00,0xf7,0x00,0x03,0x00,0x29,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x27,0x00,0x00,0x00, -0x28,0x00,0x00,0x00,0x29,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x28,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0x2c,0x00,0x00,0x00,0x13,0x00,0x00,0x00,0x2b,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x2d,0x00,0x00,0x00, -0x2c,0x00,0x00,0x00,0xb1,0x00,0x05,0x00,0x23,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x2d,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x29,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x29,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x23,0x00,0x00,0x00, -0x2f,0x00,0x00,0x00,0x27,0x00,0x00,0x00,0x05,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0xf7,0x00,0x03,0x00, -0x31,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x2f,0x00,0x00,0x00,0x30,0x00,0x00,0x00,0x31,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0x30,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x13,0x00,0x00,0x00, -0x38,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x3a,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x3b,0x00,0x00,0x00,0x22,0x00,0x00,0x00, -0x3a,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x3d,0x00,0x00,0x00,0x3b,0x00,0x00,0x00,0x1a,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x45,0x00,0x00,0x00, -0x13,0x00,0x00,0x00,0x44,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x46,0x00,0x00,0x00,0x45,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x47,0x00,0x00,0x00, -0x22,0x00,0x00,0x00,0x46,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x49,0x00,0x00,0x00,0x47,0x00,0x00,0x00, -0x1a,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x4a,0x00,0x00,0x00, -0x4b,0x00,0x00,0x00,0x42,0x00,0x00,0x00,0x2b,0x00,0x00,0x00, -0x49,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x3e,0x00,0x00,0x00, -0x4c,0x00,0x00,0x00,0x4b,0x00,0x00,0x00,0x73,0x00,0x04,0x00, -0x32,0x00,0x00,0x00,0x4d,0x00,0x00,0x00,0x4c,0x00,0x00,0x00, -0x41,0x00,0x06,0x00,0x4e,0x00,0x00,0x00,0x4f,0x00,0x00,0x00, -0x36,0x00,0x00,0x00,0x2b,0x00,0x00,0x00,0x3d,0x00,0x00,0x00, -0x3e,0x00,0x03,0x00,0x4f,0x00,0x00,0x00,0x4d,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x31,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x31,0x00,0x00,0x00,0xfd,0x00,0x01,0x00,0x38,0x00,0x01,0x00, - -}; -const uint64_t f32_to_f16_len = 1596; - unsigned char gelu_f32_data[] = { 0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, 0x4b,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, @@ -9897,9 +8558,9 @@ unsigned char get_rows_q5_1_data[] = { 0x58,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x59,0x00,0x00,0x00, 0x5a,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00, 0x5c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x52,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x69,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x67,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, 0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x10,0x00,0x00,0x00, -0x6e,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x70,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, 0x06,0x00,0x00,0x00,0x76,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, 0x2b,0x00,0x04,0x00,0x10,0x00,0x00,0x00,0x7e,0x00,0x00,0x00, 0x03,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x80,0x00,0x00,0x00, @@ -9971,68 +8632,66 @@ unsigned char get_rows_q5_1_data[] = { 0x20,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x52,0x00,0x00,0x00, 0x63,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x73,0x00,0x04,0x00, 0x1c,0x00,0x00,0x00,0x64,0x00,0x00,0x00,0x63,0x00,0x00,0x00, -0x41,0x00,0x07,0x00,0x69,0x00,0x00,0x00,0x6a,0x00,0x00,0x00, +0x41,0x00,0x07,0x00,0x67,0x00,0x00,0x00,0x68,0x00,0x00,0x00, 0x5a,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x45,0x00,0x00,0x00, 0x12,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x6b,0x00,0x00,0x00,0x6a,0x00,0x00,0x00,0xc2,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x6b,0x00,0x00,0x00, +0x69,0x00,0x00,0x00,0x68,0x00,0x00,0x00,0xc2,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x6f,0x00,0x00,0x00,0x69,0x00,0x00,0x00, 0x4a,0x00,0x00,0x00,0xc4,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x6f,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x6e,0x00,0x00,0x00, -0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x70,0x00,0x00,0x00, -0x6f,0x00,0x00,0x00,0x54,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x10,0x00,0x00,0x00,0x71,0x00,0x00,0x00,0x70,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x74,0x00,0x00,0x00, -0x6a,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x77,0x00,0x00,0x00,0x4a,0x00,0x00,0x00,0x76,0x00,0x00,0x00, -0xc2,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x78,0x00,0x00,0x00, -0x74,0x00,0x00,0x00,0x77,0x00,0x00,0x00,0xc7,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x79,0x00,0x00,0x00,0x78,0x00,0x00,0x00, -0x54,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x10,0x00,0x00,0x00, -0x7a,0x00,0x00,0x00,0x79,0x00,0x00,0x00,0x41,0x00,0x08,0x00, -0x80,0x00,0x00,0x00,0x81,0x00,0x00,0x00,0x5a,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x45,0x00,0x00,0x00,0x7e,0x00,0x00,0x00, -0x4a,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x53,0x00,0x00,0x00, -0x82,0x00,0x00,0x00,0x81,0x00,0x00,0x00,0x71,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x83,0x00,0x00,0x00,0x82,0x00,0x00,0x00, -0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x89,0x00,0x00,0x00, -0x83,0x00,0x00,0x00,0x88,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x8d,0x00,0x00,0x00,0x71,0x00,0x00,0x00, -0xc5,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8e,0x00,0x00,0x00, -0x89,0x00,0x00,0x00,0x8d,0x00,0x00,0x00,0x70,0x00,0x04,0x00, -0x1c,0x00,0x00,0x00,0x8f,0x00,0x00,0x00,0x8e,0x00,0x00,0x00, -0xc2,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x91,0x00,0x00,0x00, -0x83,0x00,0x00,0x00,0x6e,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x94,0x00,0x00,0x00,0x7a,0x00,0x00,0x00, -0xc5,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x95,0x00,0x00,0x00, -0x91,0x00,0x00,0x00,0x94,0x00,0x00,0x00,0x70,0x00,0x04,0x00, -0x1c,0x00,0x00,0x00,0x96,0x00,0x00,0x00,0x95,0x00,0x00,0x00, -0x50,0x00,0x05,0x00,0x84,0x00,0x00,0x00,0x97,0x00,0x00,0x00, -0x8f,0x00,0x00,0x00,0x96,0x00,0x00,0x00,0x8e,0x00,0x05,0x00, -0x84,0x00,0x00,0x00,0x9a,0x00,0x00,0x00,0x97,0x00,0x00,0x00, -0x5f,0x00,0x00,0x00,0x50,0x00,0x05,0x00,0x84,0x00,0x00,0x00, -0x9c,0x00,0x00,0x00,0x64,0x00,0x00,0x00,0x64,0x00,0x00,0x00, -0x81,0x00,0x05,0x00,0x84,0x00,0x00,0x00,0x9d,0x00,0x00,0x00, -0x9a,0x00,0x00,0x00,0x9c,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xa4,0x00,0x00,0x00,0x4f,0x00,0x00,0x00, -0x4a,0x00,0x00,0x00,0x51,0x00,0x05,0x00,0x1c,0x00,0x00,0x00, -0xa7,0x00,0x00,0x00,0x9d,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x73,0x00,0x04,0x00,0x52,0x00,0x00,0x00,0xa8,0x00,0x00,0x00, -0xa7,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x5c,0x00,0x00,0x00, -0xa9,0x00,0x00,0x00,0xa1,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0xa4,0x00,0x00,0x00,0x3e,0x00,0x03,0x00,0xa9,0x00,0x00,0x00, -0xa8,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xad,0x00,0x00,0x00,0xa4,0x00,0x00,0x00,0x54,0x00,0x00,0x00, -0x51,0x00,0x05,0x00,0x1c,0x00,0x00,0x00,0xaf,0x00,0x00,0x00, -0x9d,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x73,0x00,0x04,0x00, -0x52,0x00,0x00,0x00,0xb0,0x00,0x00,0x00,0xaf,0x00,0x00,0x00, -0x41,0x00,0x06,0x00,0x5c,0x00,0x00,0x00,0xb1,0x00,0x00,0x00, -0xa1,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0xad,0x00,0x00,0x00, -0x3e,0x00,0x03,0x00,0xb1,0x00,0x00,0x00,0xb0,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xb4,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xb4,0x00,0x00,0x00,0xfd,0x00,0x01,0x00,0x38,0x00,0x01,0x00, - +0x71,0x00,0x00,0x00,0x6f,0x00,0x00,0x00,0x70,0x00,0x00,0x00, +0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x72,0x00,0x00,0x00, +0x71,0x00,0x00,0x00,0x54,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, +0x10,0x00,0x00,0x00,0x73,0x00,0x00,0x00,0x72,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x77,0x00,0x00,0x00, +0x4a,0x00,0x00,0x00,0x76,0x00,0x00,0x00,0xc2,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x78,0x00,0x00,0x00,0x69,0x00,0x00,0x00, +0x77,0x00,0x00,0x00,0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x79,0x00,0x00,0x00,0x78,0x00,0x00,0x00,0x54,0x00,0x00,0x00, +0x7c,0x00,0x04,0x00,0x10,0x00,0x00,0x00,0x7a,0x00,0x00,0x00, +0x79,0x00,0x00,0x00,0x41,0x00,0x08,0x00,0x80,0x00,0x00,0x00, +0x81,0x00,0x00,0x00,0x5a,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, +0x45,0x00,0x00,0x00,0x7e,0x00,0x00,0x00,0x4a,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x53,0x00,0x00,0x00,0x82,0x00,0x00,0x00, +0x81,0x00,0x00,0x00,0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x83,0x00,0x00,0x00,0x82,0x00,0x00,0x00,0xc7,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x89,0x00,0x00,0x00,0x83,0x00,0x00,0x00, +0x88,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x8d,0x00,0x00,0x00,0x73,0x00,0x00,0x00,0xc5,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x8e,0x00,0x00,0x00,0x89,0x00,0x00,0x00, +0x8d,0x00,0x00,0x00,0x70,0x00,0x04,0x00,0x1c,0x00,0x00,0x00, +0x8f,0x00,0x00,0x00,0x8e,0x00,0x00,0x00,0xc2,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x91,0x00,0x00,0x00,0x83,0x00,0x00,0x00, +0x70,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x94,0x00,0x00,0x00,0x7a,0x00,0x00,0x00,0xc5,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x95,0x00,0x00,0x00,0x91,0x00,0x00,0x00, +0x94,0x00,0x00,0x00,0x70,0x00,0x04,0x00,0x1c,0x00,0x00,0x00, +0x96,0x00,0x00,0x00,0x95,0x00,0x00,0x00,0x50,0x00,0x05,0x00, +0x84,0x00,0x00,0x00,0x97,0x00,0x00,0x00,0x8f,0x00,0x00,0x00, +0x96,0x00,0x00,0x00,0x8e,0x00,0x05,0x00,0x84,0x00,0x00,0x00, +0x9a,0x00,0x00,0x00,0x97,0x00,0x00,0x00,0x5f,0x00,0x00,0x00, +0x50,0x00,0x05,0x00,0x84,0x00,0x00,0x00,0x9c,0x00,0x00,0x00, +0x64,0x00,0x00,0x00,0x64,0x00,0x00,0x00,0x81,0x00,0x05,0x00, +0x84,0x00,0x00,0x00,0x9d,0x00,0x00,0x00,0x9a,0x00,0x00,0x00, +0x9c,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xa4,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x4a,0x00,0x00,0x00, +0x51,0x00,0x05,0x00,0x1c,0x00,0x00,0x00,0xa7,0x00,0x00,0x00, +0x9d,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x73,0x00,0x04,0x00, +0x52,0x00,0x00,0x00,0xa8,0x00,0x00,0x00,0xa7,0x00,0x00,0x00, +0x41,0x00,0x06,0x00,0x5c,0x00,0x00,0x00,0xa9,0x00,0x00,0x00, +0xa1,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0xa4,0x00,0x00,0x00, +0x3e,0x00,0x03,0x00,0xa9,0x00,0x00,0x00,0xa8,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xad,0x00,0x00,0x00, +0xa4,0x00,0x00,0x00,0x54,0x00,0x00,0x00,0x51,0x00,0x05,0x00, +0x1c,0x00,0x00,0x00,0xaf,0x00,0x00,0x00,0x9d,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x52,0x00,0x00,0x00, +0xb0,0x00,0x00,0x00,0xaf,0x00,0x00,0x00,0x41,0x00,0x06,0x00, +0x5c,0x00,0x00,0x00,0xb1,0x00,0x00,0x00,0xa1,0x00,0x00,0x00, +0x2e,0x00,0x00,0x00,0xad,0x00,0x00,0x00,0x3e,0x00,0x03,0x00, +0xb1,0x00,0x00,0x00,0xb0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xb4,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xb4,0x00,0x00,0x00, +0xfd,0x00,0x01,0x00,0x38,0x00,0x01,0x00, }; -const uint64_t get_rows_q5_1_len = 2796; +const uint64_t get_rows_q5_1_len = 2780; unsigned char get_rows_q5_1_f32_data[] = { 0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, @@ -10135,9 +8794,9 @@ unsigned char get_rows_q5_1_f32_data[] = { 0x58,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x59,0x00,0x00,0x00, 0x5a,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00, 0x5c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x52,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x69,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x67,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, 0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x10,0x00,0x00,0x00, -0x6e,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x70,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, 0x06,0x00,0x00,0x00,0x76,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, 0x2b,0x00,0x04,0x00,0x10,0x00,0x00,0x00,0x7e,0x00,0x00,0x00, 0x03,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x80,0x00,0x00,0x00, @@ -10210,65 +8869,64 @@ unsigned char get_rows_q5_1_f32_data[] = { 0x45,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, 0x52,0x00,0x00,0x00,0x63,0x00,0x00,0x00,0x62,0x00,0x00,0x00, 0x73,0x00,0x04,0x00,0x1c,0x00,0x00,0x00,0x64,0x00,0x00,0x00, -0x63,0x00,0x00,0x00,0x41,0x00,0x07,0x00,0x69,0x00,0x00,0x00, -0x6a,0x00,0x00,0x00,0x5a,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, +0x63,0x00,0x00,0x00,0x41,0x00,0x07,0x00,0x67,0x00,0x00,0x00, +0x68,0x00,0x00,0x00,0x5a,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, 0x45,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x6b,0x00,0x00,0x00,0x6a,0x00,0x00,0x00, -0xc2,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, -0x6b,0x00,0x00,0x00,0x4a,0x00,0x00,0x00,0xc4,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x6f,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, -0x6e,0x00,0x00,0x00,0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x70,0x00,0x00,0x00,0x6f,0x00,0x00,0x00,0x54,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x10,0x00,0x00,0x00,0x71,0x00,0x00,0x00, -0x70,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x74,0x00,0x00,0x00,0x6a,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x77,0x00,0x00,0x00,0x4a,0x00,0x00,0x00, -0x76,0x00,0x00,0x00,0xc2,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x78,0x00,0x00,0x00,0x74,0x00,0x00,0x00,0x77,0x00,0x00,0x00, -0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x79,0x00,0x00,0x00, -0x78,0x00,0x00,0x00,0x54,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x10,0x00,0x00,0x00,0x7a,0x00,0x00,0x00,0x79,0x00,0x00,0x00, -0x41,0x00,0x08,0x00,0x80,0x00,0x00,0x00,0x81,0x00,0x00,0x00, -0x5a,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x45,0x00,0x00,0x00, -0x7e,0x00,0x00,0x00,0x4a,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x53,0x00,0x00,0x00,0x82,0x00,0x00,0x00,0x81,0x00,0x00,0x00, -0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x83,0x00,0x00,0x00, -0x82,0x00,0x00,0x00,0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x89,0x00,0x00,0x00,0x83,0x00,0x00,0x00,0x88,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x8d,0x00,0x00,0x00, -0x71,0x00,0x00,0x00,0xc5,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x8e,0x00,0x00,0x00,0x89,0x00,0x00,0x00,0x8d,0x00,0x00,0x00, -0x70,0x00,0x04,0x00,0x1c,0x00,0x00,0x00,0x8f,0x00,0x00,0x00, -0x8e,0x00,0x00,0x00,0xc2,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x91,0x00,0x00,0x00,0x83,0x00,0x00,0x00,0x6e,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x94,0x00,0x00,0x00, -0x7a,0x00,0x00,0x00,0xc5,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x95,0x00,0x00,0x00,0x91,0x00,0x00,0x00,0x94,0x00,0x00,0x00, -0x70,0x00,0x04,0x00,0x1c,0x00,0x00,0x00,0x96,0x00,0x00,0x00, -0x95,0x00,0x00,0x00,0x50,0x00,0x05,0x00,0x84,0x00,0x00,0x00, -0x97,0x00,0x00,0x00,0x8f,0x00,0x00,0x00,0x96,0x00,0x00,0x00, -0x8e,0x00,0x05,0x00,0x84,0x00,0x00,0x00,0x9a,0x00,0x00,0x00, -0x97,0x00,0x00,0x00,0x5f,0x00,0x00,0x00,0x50,0x00,0x05,0x00, -0x84,0x00,0x00,0x00,0x9c,0x00,0x00,0x00,0x64,0x00,0x00,0x00, -0x64,0x00,0x00,0x00,0x81,0x00,0x05,0x00,0x84,0x00,0x00,0x00, -0x9d,0x00,0x00,0x00,0x9a,0x00,0x00,0x00,0x9c,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa4,0x00,0x00,0x00, -0x4f,0x00,0x00,0x00,0x4a,0x00,0x00,0x00,0x51,0x00,0x05,0x00, -0x1c,0x00,0x00,0x00,0xa7,0x00,0x00,0x00,0x9d,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0xa8,0x00,0x00,0x00, -0xa9,0x00,0x00,0x00,0xa1,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0xa4,0x00,0x00,0x00,0x3e,0x00,0x03,0x00,0xa9,0x00,0x00,0x00, -0xa7,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xad,0x00,0x00,0x00,0xa4,0x00,0x00,0x00,0x54,0x00,0x00,0x00, -0x51,0x00,0x05,0x00,0x1c,0x00,0x00,0x00,0xaf,0x00,0x00,0x00, -0x9d,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x41,0x00,0x06,0x00, -0xa8,0x00,0x00,0x00,0xb0,0x00,0x00,0x00,0xa1,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0xad,0x00,0x00,0x00,0x3e,0x00,0x03,0x00, -0xb0,0x00,0x00,0x00,0xaf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xb3,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xb3,0x00,0x00,0x00, -0xfd,0x00,0x01,0x00,0x38,0x00,0x01,0x00, +0x06,0x00,0x00,0x00,0x69,0x00,0x00,0x00,0x68,0x00,0x00,0x00, +0xc2,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x6f,0x00,0x00,0x00, +0x69,0x00,0x00,0x00,0x4a,0x00,0x00,0x00,0xc4,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x71,0x00,0x00,0x00,0x6f,0x00,0x00,0x00, +0x70,0x00,0x00,0x00,0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x72,0x00,0x00,0x00,0x71,0x00,0x00,0x00,0x54,0x00,0x00,0x00, +0x7c,0x00,0x04,0x00,0x10,0x00,0x00,0x00,0x73,0x00,0x00,0x00, +0x72,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x77,0x00,0x00,0x00,0x4a,0x00,0x00,0x00,0x76,0x00,0x00,0x00, +0xc2,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x78,0x00,0x00,0x00, +0x69,0x00,0x00,0x00,0x77,0x00,0x00,0x00,0xc7,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x79,0x00,0x00,0x00,0x78,0x00,0x00,0x00, +0x54,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x10,0x00,0x00,0x00, +0x7a,0x00,0x00,0x00,0x79,0x00,0x00,0x00,0x41,0x00,0x08,0x00, +0x80,0x00,0x00,0x00,0x81,0x00,0x00,0x00,0x5a,0x00,0x00,0x00, +0x2e,0x00,0x00,0x00,0x45,0x00,0x00,0x00,0x7e,0x00,0x00,0x00, +0x4a,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x53,0x00,0x00,0x00, +0x82,0x00,0x00,0x00,0x81,0x00,0x00,0x00,0x71,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x83,0x00,0x00,0x00,0x82,0x00,0x00,0x00, +0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x89,0x00,0x00,0x00, +0x83,0x00,0x00,0x00,0x88,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x8d,0x00,0x00,0x00,0x73,0x00,0x00,0x00, +0xc5,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8e,0x00,0x00,0x00, +0x89,0x00,0x00,0x00,0x8d,0x00,0x00,0x00,0x70,0x00,0x04,0x00, +0x1c,0x00,0x00,0x00,0x8f,0x00,0x00,0x00,0x8e,0x00,0x00,0x00, +0xc2,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x91,0x00,0x00,0x00, +0x83,0x00,0x00,0x00,0x70,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x94,0x00,0x00,0x00,0x7a,0x00,0x00,0x00, +0xc5,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x95,0x00,0x00,0x00, +0x91,0x00,0x00,0x00,0x94,0x00,0x00,0x00,0x70,0x00,0x04,0x00, +0x1c,0x00,0x00,0x00,0x96,0x00,0x00,0x00,0x95,0x00,0x00,0x00, +0x50,0x00,0x05,0x00,0x84,0x00,0x00,0x00,0x97,0x00,0x00,0x00, +0x8f,0x00,0x00,0x00,0x96,0x00,0x00,0x00,0x8e,0x00,0x05,0x00, +0x84,0x00,0x00,0x00,0x9a,0x00,0x00,0x00,0x97,0x00,0x00,0x00, +0x5f,0x00,0x00,0x00,0x50,0x00,0x05,0x00,0x84,0x00,0x00,0x00, +0x9c,0x00,0x00,0x00,0x64,0x00,0x00,0x00,0x64,0x00,0x00,0x00, +0x81,0x00,0x05,0x00,0x84,0x00,0x00,0x00,0x9d,0x00,0x00,0x00, +0x9a,0x00,0x00,0x00,0x9c,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xa4,0x00,0x00,0x00,0x4f,0x00,0x00,0x00, +0x4a,0x00,0x00,0x00,0x51,0x00,0x05,0x00,0x1c,0x00,0x00,0x00, +0xa7,0x00,0x00,0x00,0x9d,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x41,0x00,0x06,0x00,0xa8,0x00,0x00,0x00,0xa9,0x00,0x00,0x00, +0xa1,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0xa4,0x00,0x00,0x00, +0x3e,0x00,0x03,0x00,0xa9,0x00,0x00,0x00,0xa7,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xad,0x00,0x00,0x00, +0xa4,0x00,0x00,0x00,0x54,0x00,0x00,0x00,0x51,0x00,0x05,0x00, +0x1c,0x00,0x00,0x00,0xaf,0x00,0x00,0x00,0x9d,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0xa8,0x00,0x00,0x00, +0xb0,0x00,0x00,0x00,0xa1,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, +0xad,0x00,0x00,0x00,0x3e,0x00,0x03,0x00,0xb0,0x00,0x00,0x00, +0xaf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xb3,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xb3,0x00,0x00,0x00,0xfd,0x00,0x01,0x00, +0x38,0x00,0x01,0x00, }; -const uint64_t get_rows_q5_1_f32_len = 2780; +const uint64_t get_rows_q5_1_f32_len = 2764; unsigned char get_rows_q8_0_data[] = { 0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, @@ -10661,9 +9319,9 @@ unsigned char get_rows_q8_0_f32_data[] = { }; const uint64_t get_rows_q8_0_f32_len = 2280; -unsigned char matmul_f16_aligned_l_data[] = { +unsigned char matmul_f16_data[] = { 0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, -0x5b,0x03,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, +0xd9,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, 0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x09,0x00,0x00,0x00, 0x11,0x00,0x02,0x00,0x51,0x11,0x00,0x00,0x0b,0x00,0x06,0x00, 0x01,0x00,0x00,0x00,0x47,0x4c,0x53,0x4c,0x2e,0x73,0x74,0x64, @@ -10671,9 +9329,9 @@ unsigned char matmul_f16_aligned_l_data[] = { 0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x0f,0x00,0x0f,0x00, 0x05,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x6d,0x61,0x69,0x6e, 0x00,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x3d,0x00,0x00,0x00,0x4c,0x00,0x00,0x00,0xeb,0x00,0x00,0x00, -0xfa,0x00,0x00,0x00,0x80,0x01,0x00,0x00,0x8d,0x01,0x00,0x00, -0xbb,0x02,0x00,0x00,0x04,0x03,0x00,0x00,0x10,0x00,0x06,0x00, +0x3d,0x00,0x00,0x00,0x4c,0x00,0x00,0x00,0xfb,0x00,0x00,0x00, +0x06,0x01,0x00,0x00,0x46,0x01,0x00,0x00,0x51,0x01,0x00,0x00, +0x39,0x02,0x00,0x00,0x82,0x02,0x00,0x00,0x10,0x00,0x06,0x00, 0x04,0x00,0x00,0x00,0x11,0x00,0x00,0x00,0x01,0x00,0x00,0x00, 0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00, 0x0b,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x1c,0x00,0x00,0x00, @@ -10712,1924 +9370,38 @@ unsigned char matmul_f16_aligned_l_data[] = { 0x06,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x62,0x00,0x00,0x00, 0x01,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0x47,0x00,0x04,0x00, 0x6c,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x03,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x9d,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xaf,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0xa6,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xb8,0x00,0x00,0x00, 0x01,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0xb2,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x08,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0xf7,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x10,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0xf8,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0x48,0x00,0x04,0x00, -0xf8,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0xf8,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0xf8,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x07,0x00,0x00,0x00, -0x08,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0xf8,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xfa,0x00,0x00,0x00, -0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0xfa,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x61,0x01,0x00,0x00,0x01,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x62,0x01,0x00,0x00, -0x0b,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x8a,0x01,0x00,0x00,0x06,0x00,0x00,0x00,0x10,0x00,0x00,0x00, -0x48,0x00,0x04,0x00,0x8b,0x01,0x00,0x00,0x00,0x00,0x00,0x00, -0x05,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0x8b,0x01,0x00,0x00, +0xbb,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x08,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x03,0x01,0x00,0x00,0x06,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0x04,0x01,0x00,0x00, 0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x8b,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x8b,0x01,0x00,0x00, -0x00,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0x08,0x00,0x00,0x00, -0x47,0x00,0x03,0x00,0x8b,0x01,0x00,0x00,0x02,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x8d,0x01,0x00,0x00,0x22,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x8d,0x01,0x00,0x00, -0x21,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0xbb,0x02,0x00,0x00,0x0b,0x00,0x00,0x00,0x18,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x01,0x03,0x00,0x00,0x06,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0x02,0x03,0x00,0x00, -0x00,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x02,0x03,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x02,0x03,0x00,0x00, -0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x04,0x03,0x00,0x00, +0x04,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x04,0x01,0x00,0x00, +0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x06,0x01,0x00,0x00, 0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x04,0x03,0x00,0x00,0x21,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x13,0x00,0x02,0x00,0x02,0x00,0x00,0x00,0x21,0x00,0x03,0x00, -0x03,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x15,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x17,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x03,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x0a,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x0a,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x0d,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x1e,0x00,0x10,0x00, -0x10,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x11,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x10,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x11,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x15,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x14,0x00,0x00,0x00, -0x08,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x15,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x0a,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x27,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, -0x2d,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x37,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x0a,0x00,0x00,0x00,0x3d,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x3e,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00, -0x4c,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x53,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x54,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x37,0x00,0x00,0x00, -0x53,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x58,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x37,0x00,0x00,0x00, -0x53,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x60,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x61,0x00,0x00,0x00,0x86,0x00,0x00,0x00, -0x53,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x32,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x63,0x00,0x00,0x00, -0x86,0x00,0x00,0x00,0x61,0x00,0x00,0x00,0x62,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x67,0x00,0x00,0x00, -0x86,0x00,0x00,0x00,0x61,0x00,0x00,0x00,0x62,0x00,0x00,0x00, -0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x10,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x6d,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x6e,0x00,0x00,0x00,0x86,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x73,0x00,0x00,0x00,0x86,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0x77,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x7c,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, -0x87,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0x8d,0x00,0x00,0x00,0x03,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x98,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x9d,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0x9f,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xae,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00, -0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xaf,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xb0,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x53,0x00,0x00,0x00, -0xaf,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xb1,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x4f,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xb2,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xb3,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0xb1,0x00,0x00,0x00,0xb2,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xb4,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0xb3,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xb5,0x00,0x00,0x00,0x86,0x00,0x00,0x00, -0xb0,0x00,0x00,0x00,0xb4,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xb6,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0xae,0x00,0x00,0x00,0xb5,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xb7,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0xb6,0x00,0x00,0x00,0xb2,0x00,0x00,0x00,0x14,0x00,0x02,0x00, -0xb8,0x00,0x00,0x00,0x16,0x00,0x03,0x00,0xba,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xbb,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xbc,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0xbb,0x00,0x00,0x00, -0xb5,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xbd,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0xbc,0x00,0x00,0x00, -0xb2,0x00,0x00,0x00,0x1c,0x00,0x04,0x00,0xbe,0x00,0x00,0x00, -0xba,0x00,0x00,0x00,0xbd,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0xbf,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0xbe,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0xba,0x00,0x00,0x00,0xc2,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0xc3,0x00,0x00,0x00, -0x07,0x00,0x00,0x00,0xba,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0xc6,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x16,0x00,0x03,0x00,0xe6,0x00,0x00,0x00,0x10,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xe7,0x00,0x00,0x00, -0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xe8,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0xe7,0x00,0x00,0x00, -0x1c,0x00,0x04,0x00,0xe9,0x00,0x00,0x00,0xe6,0x00,0x00,0x00, -0xe8,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0xea,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0xe9,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0xea,0x00,0x00,0x00,0xeb,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xef,0x00,0x00,0x00, -0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x17,0x00,0x04,0x00,0xf5,0x00,0x00,0x00,0xe6,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x18,0x00,0x04,0x00,0xf6,0x00,0x00,0x00, -0xf5,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, -0xf7,0x00,0x00,0x00,0xf6,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, -0xf8,0x00,0x00,0x00,0xf7,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0xf9,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0xf8,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0xf9,0x00,0x00,0x00,0xfa,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0xfc,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0xe6,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0xff,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0xe6,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x04,0x01,0x00,0x00, -0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x11,0x01,0x00,0x00, -0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x1e,0x01,0x00,0x00, -0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x23,0x01,0x00,0x00, -0x03,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x2c,0x01,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x31,0x01,0x00,0x00,0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x3a,0x01,0x00,0x00,0x80,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x3f,0x01,0x00,0x00,0x05,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x48,0x01,0x00,0x00, -0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x4d,0x01,0x00,0x00, -0x06,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x56,0x01,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x5b,0x01,0x00,0x00,0x07,0x00,0x00,0x00,0x32,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x61,0x01,0x00,0x00,0x01,0x00,0x00,0x00, -0x33,0x00,0x06,0x00,0x09,0x00,0x00,0x00,0x62,0x01,0x00,0x00, -0x61,0x01,0x00,0x00,0x39,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x63,0x01,0x00,0x00, -0x51,0x00,0x00,0x00,0x62,0x01,0x00,0x00,0x00,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x64,0x01,0x00,0x00, -0x84,0x00,0x00,0x00,0x63,0x01,0x00,0x00,0x6d,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x65,0x01,0x00,0x00, -0x86,0x00,0x00,0x00,0x64,0x01,0x00,0x00,0x6c,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x7c,0x01,0x00,0x00, -0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x7d,0x01,0x00,0x00, -0x84,0x00,0x00,0x00,0x9d,0x00,0x00,0x00,0x7c,0x01,0x00,0x00, -0x1c,0x00,0x04,0x00,0x7e,0x01,0x00,0x00,0xe6,0x00,0x00,0x00, -0x7d,0x01,0x00,0x00,0x20,0x00,0x04,0x00,0x7f,0x01,0x00,0x00, -0x04,0x00,0x00,0x00,0x7e,0x01,0x00,0x00,0x3b,0x00,0x04,0x00, -0x7f,0x01,0x00,0x00,0x80,0x01,0x00,0x00,0x04,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x84,0x01,0x00,0x00, -0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x1d,0x00,0x03,0x00,0x8a,0x01,0x00,0x00,0xf6,0x00,0x00,0x00, -0x1e,0x00,0x03,0x00,0x8b,0x01,0x00,0x00,0x8a,0x01,0x00,0x00, -0x20,0x00,0x04,0x00,0x8c,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, -0x8b,0x01,0x00,0x00,0x3b,0x00,0x04,0x00,0x8c,0x01,0x00,0x00, -0x8d,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x95,0x01,0x00,0x00,0x80,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xa2,0x01,0x00,0x00,0x80,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xaf,0x01,0x00,0x00,0x80,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xbc,0x01,0x00,0x00,0x80,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xc9,0x01,0x00,0x00,0x80,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xd6,0x01,0x00,0x00,0x80,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xe3,0x01,0x00,0x00,0x80,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xef,0x01,0x00,0x00,0x08,0x01,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xf0,0x01,0x00,0x00, -0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xf3,0x01,0x00,0x00, -0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x0e,0x02,0x00,0x00, -0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00, -0x1c,0x00,0x04,0x00,0x0f,0x02,0x00,0x00,0xe6,0x00,0x00,0x00, -0x0e,0x02,0x00,0x00,0x20,0x00,0x04,0x00,0x10,0x02,0x00,0x00, -0x07,0x00,0x00,0x00,0x0f,0x02,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x20,0x02,0x00,0x00,0x80,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x26,0x02,0x00,0x00,0x07,0x00,0x00,0x00,0xe6,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x3c,0x02,0x00,0x00, -0x84,0x00,0x00,0x00,0xb5,0x00,0x00,0x00,0xb2,0x00,0x00,0x00, -0x1c,0x00,0x04,0x00,0x3d,0x02,0x00,0x00,0xe6,0x00,0x00,0x00, -0x3c,0x02,0x00,0x00,0x20,0x00,0x04,0x00,0x3e,0x02,0x00,0x00, -0x07,0x00,0x00,0x00,0x3d,0x02,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x47,0x02,0x00,0x00,0x86,0x00,0x00,0x00, -0xaf,0x00,0x00,0x00,0xb5,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x4f,0x02,0x00,0x00,0x80,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x7e,0x02,0x00,0x00,0x84,0x00,0x00,0x00, -0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0xb3,0x02,0x00,0x00,0x0d,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00,0xbb,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0x1d,0x00,0x03,0x00,0x01,0x03,0x00,0x00, -0xba,0x00,0x00,0x00,0x1e,0x00,0x03,0x00,0x02,0x03,0x00,0x00, -0x01,0x03,0x00,0x00,0x20,0x00,0x04,0x00,0x03,0x03,0x00,0x00, -0x0c,0x00,0x00,0x00,0x02,0x03,0x00,0x00,0x3b,0x00,0x04,0x00, -0x03,0x03,0x00,0x00,0x04,0x03,0x00,0x00,0x0c,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x09,0x03,0x00,0x00, -0x05,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x16,0x03,0x00,0x00,0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x1f,0x03,0x00,0x00, -0x0c,0x00,0x00,0x00,0xba,0x00,0x00,0x00,0x36,0x00,0x05,0x00, -0x02,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x03,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x05,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0xbf,0x00,0x00,0x00,0xc0,0x00,0x00,0x00, -0x07,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x10,0x02,0x00,0x00, -0x11,0x02,0x00,0x00,0x07,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x3e,0x02,0x00,0x00,0x3f,0x02,0x00,0x00,0x07,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00,0x0e,0x00,0x00,0x00, -0x0b,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x0e,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x17,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x18,0x00,0x00,0x00, -0x0f,0x00,0x00,0x00,0x17,0x00,0x00,0x00,0x89,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x1e,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, -0x17,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0x22,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x22,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x24,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x28,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x27,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x29,0x00,0x00,0x00,0x28,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x2a,0x00,0x00,0x00, -0x1e,0x00,0x00,0x00,0x29,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x2d,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x2f,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x30,0x00,0x00,0x00,0x24,0x00,0x00,0x00, -0x2f,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x32,0x00,0x00,0x00,0x30,0x00,0x00,0x00,0x2a,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x35,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x36,0x00,0x00,0x00,0x35,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x38,0x00,0x00,0x00, -0x36,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x82,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x38,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x3b,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x37,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00,0x3f,0x00,0x00,0x00, -0x3d,0x00,0x00,0x00,0x3e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x3f,0x00,0x00,0x00, -0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x42,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0x3b,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x47,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x3b,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00, -0x49,0x00,0x00,0x00,0x3d,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x4a,0x00,0x00,0x00, -0x49,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00, -0x4d,0x00,0x00,0x00,0x4c,0x00,0x00,0x00,0x3e,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, -0x4d,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x50,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x4f,0x00,0x00,0x00, -0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x55,0x00,0x00,0x00, -0x50,0x00,0x00,0x00,0x54,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x59,0x00,0x00,0x00,0x50,0x00,0x00,0x00, -0x58,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x5d,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x4f,0x00,0x00,0x00, -0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x64,0x00,0x00,0x00, -0x5d,0x00,0x00,0x00,0x63,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x68,0x00,0x00,0x00,0x5d,0x00,0x00,0x00, -0x67,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x6f,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x6e,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x74,0x00,0x00,0x00, -0x4e,0x00,0x00,0x00,0x73,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x78,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x77,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x79,0x00,0x00,0x00,0x78,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x7a,0x00,0x00,0x00,0x47,0x00,0x00,0x00, -0x79,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0x7d,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x7c,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x7e,0x00,0x00,0x00, -0x7d,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x80,0x00,0x00,0x00,0x47,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x83,0x00,0x00,0x00, -0x80,0x00,0x00,0x00,0x79,0x00,0x00,0x00,0x0c,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x26,0x00,0x00,0x00,0x7e,0x00,0x00,0x00,0x83,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x88,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x87,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x89,0x00,0x00,0x00,0x88,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8a,0x00,0x00,0x00, -0x32,0x00,0x00,0x00,0x89,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x8c,0x00,0x00,0x00,0x42,0x00,0x00,0x00, -0x37,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0x8e,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x8d,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x8f,0x00,0x00,0x00, -0x8e,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x90,0x00,0x00,0x00,0x8c,0x00,0x00,0x00,0x8f,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x91,0x00,0x00,0x00, -0x8a,0x00,0x00,0x00,0x90,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x93,0x00,0x00,0x00,0x91,0x00,0x00,0x00, -0x7a,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x94,0x00,0x00,0x00,0x93,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x99,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x98,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x9a,0x00,0x00,0x00,0x99,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9b,0x00,0x00,0x00, -0x0f,0x00,0x00,0x00,0x9a,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x9e,0x00,0x00,0x00,0x4a,0x00,0x00,0x00, -0x9d,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0xa0,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x9f,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xa1,0x00,0x00,0x00, -0xa0,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xa2,0x00,0x00,0x00,0x9e,0x00,0x00,0x00,0xa1,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa3,0x00,0x00,0x00, -0x9b,0x00,0x00,0x00,0xa2,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xa5,0x00,0x00,0x00,0xa3,0x00,0x00,0x00, -0x7a,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xa6,0x00,0x00,0x00,0xa5,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xa8,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xa8,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0x29,0x03,0x00,0x00,0x3e,0x00,0x00,0x00,0x05,0x00,0x00,0x00, -0xc7,0x00,0x00,0x00,0xa9,0x00,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb8,0x00,0x00,0x00,0xb9,0x00,0x00,0x00,0x29,0x03,0x00,0x00, -0xb7,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xaa,0x00,0x00,0x00, -0xa9,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0xb9,0x00,0x00,0x00,0xa9,0x00,0x00,0x00,0xaa,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xa9,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0xc3,0x00,0x00,0x00,0xc4,0x00,0x00,0x00,0xc0,0x00,0x00,0x00, -0x29,0x03,0x00,0x00,0x3e,0x00,0x03,0x00,0xc4,0x00,0x00,0x00, -0xc2,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xc7,0x00,0x00,0x00,0x29,0x03,0x00,0x00,0xc6,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xa8,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xaa,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xca,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xca,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0x42,0x03,0x00,0x00,0xa6,0x00,0x00,0x00, -0xaa,0x00,0x00,0x00,0xf5,0x01,0x00,0x00,0xcd,0x00,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x3e,0x03,0x00,0x00, -0x94,0x00,0x00,0x00,0xaa,0x00,0x00,0x00,0xf2,0x01,0x00,0x00, -0xcd,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0x2a,0x03,0x00,0x00,0x7a,0x00,0x00,0x00,0xaa,0x00,0x00,0x00, -0xa3,0x02,0x00,0x00,0xcd,0x00,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb8,0x00,0x00,0x00,0xd1,0x00,0x00,0x00,0x2a,0x03,0x00,0x00, -0x84,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xcc,0x00,0x00,0x00, -0xcd,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0xd1,0x00,0x00,0x00,0xcb,0x00,0x00,0x00,0xcc,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xcb,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xd3,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xd3,0x00,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x3a,0x03,0x00,0x00, -0x3e,0x00,0x00,0x00,0xcb,0x00,0x00,0x00,0x67,0x01,0x00,0x00, -0xd4,0x00,0x00,0x00,0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00, -0xd9,0x00,0x00,0x00,0x3a,0x03,0x00,0x00,0x37,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0xd5,0x00,0x00,0x00,0xd4,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xd9,0x00,0x00,0x00, -0xd4,0x00,0x00,0x00,0xd5,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xd4,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xde,0x00,0x00,0x00,0x74,0x00,0x00,0x00,0x3a,0x03,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xe1,0x00,0x00,0x00, -0xde,0x00,0x00,0x00,0x8f,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xe2,0x00,0x00,0x00,0xe1,0x00,0x00,0x00, -0x6d,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xe3,0x00,0x00,0x00,0x3e,0x03,0x00,0x00,0xe2,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xe5,0x00,0x00,0x00, -0xe3,0x00,0x00,0x00,0x6f,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xf0,0x00,0x00,0x00,0xde,0x00,0x00,0x00, -0xef,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xf2,0x00,0x00,0x00,0x6f,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xf3,0x00,0x00,0x00, -0xf0,0x00,0x00,0x00,0xf2,0x00,0x00,0x00,0x41,0x00,0x08,0x00, -0xfc,0x00,0x00,0x00,0xfd,0x00,0x00,0x00,0xfa,0x00,0x00,0x00, -0x34,0x00,0x00,0x00,0xe5,0x00,0x00,0x00,0x34,0x00,0x00,0x00, -0x3e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0xe6,0x00,0x00,0x00, -0xfe,0x00,0x00,0x00,0xfd,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0xff,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0xeb,0x00,0x00,0x00, -0xf3,0x00,0x00,0x00,0x3e,0x00,0x03,0x00,0x00,0x01,0x00,0x00, -0xfe,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x05,0x01,0x00,0x00,0xde,0x00,0x00,0x00,0x04,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x08,0x01,0x00,0x00, -0x05,0x01,0x00,0x00,0xf2,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x09,0x01,0x00,0x00,0x08,0x01,0x00,0x00, -0x39,0x00,0x00,0x00,0x41,0x00,0x08,0x00,0xfc,0x00,0x00,0x00, -0x0b,0x01,0x00,0x00,0xfa,0x00,0x00,0x00,0x34,0x00,0x00,0x00, -0xe5,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0xe6,0x00,0x00,0x00,0x0c,0x01,0x00,0x00, -0x0b,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0xff,0x00,0x00,0x00, -0x0d,0x01,0x00,0x00,0xeb,0x00,0x00,0x00,0x09,0x01,0x00,0x00, -0x3e,0x00,0x03,0x00,0x0d,0x01,0x00,0x00,0x0c,0x01,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x12,0x01,0x00,0x00, -0xde,0x00,0x00,0x00,0x11,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x15,0x01,0x00,0x00,0x12,0x01,0x00,0x00, -0xf2,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x16,0x01,0x00,0x00,0x15,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, -0x41,0x00,0x08,0x00,0xfc,0x00,0x00,0x00,0x18,0x01,0x00,0x00, -0xfa,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0xe5,0x00,0x00,0x00, -0x34,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0xe6,0x00,0x00,0x00,0x19,0x01,0x00,0x00,0x18,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0xff,0x00,0x00,0x00,0x1a,0x01,0x00,0x00, -0xeb,0x00,0x00,0x00,0x16,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0x1a,0x01,0x00,0x00,0x19,0x01,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x1f,0x01,0x00,0x00,0xde,0x00,0x00,0x00, -0x1e,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x22,0x01,0x00,0x00,0x1f,0x01,0x00,0x00,0xf2,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x24,0x01,0x00,0x00, -0x22,0x01,0x00,0x00,0x23,0x01,0x00,0x00,0x41,0x00,0x08,0x00, -0xfc,0x00,0x00,0x00,0x26,0x01,0x00,0x00,0xfa,0x00,0x00,0x00, -0x34,0x00,0x00,0x00,0xe5,0x00,0x00,0x00,0x34,0x00,0x00,0x00, -0x23,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0xe6,0x00,0x00,0x00, -0x27,0x01,0x00,0x00,0x26,0x01,0x00,0x00,0x41,0x00,0x05,0x00, -0xff,0x00,0x00,0x00,0x28,0x01,0x00,0x00,0xeb,0x00,0x00,0x00, -0x24,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x28,0x01,0x00,0x00, -0x27,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x2d,0x01,0x00,0x00,0xde,0x00,0x00,0x00,0x2c,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x30,0x01,0x00,0x00, -0x2d,0x01,0x00,0x00,0xf2,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x32,0x01,0x00,0x00,0x30,0x01,0x00,0x00, -0x31,0x01,0x00,0x00,0x41,0x00,0x08,0x00,0xfc,0x00,0x00,0x00, -0x34,0x01,0x00,0x00,0xfa,0x00,0x00,0x00,0x34,0x00,0x00,0x00, -0xe5,0x00,0x00,0x00,0xc6,0x00,0x00,0x00,0x3e,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0xe6,0x00,0x00,0x00,0x35,0x01,0x00,0x00, -0x34,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0xff,0x00,0x00,0x00, -0x36,0x01,0x00,0x00,0xeb,0x00,0x00,0x00,0x32,0x01,0x00,0x00, -0x3e,0x00,0x03,0x00,0x36,0x01,0x00,0x00,0x35,0x01,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3b,0x01,0x00,0x00, -0xde,0x00,0x00,0x00,0x3a,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x3e,0x01,0x00,0x00,0x3b,0x01,0x00,0x00, -0xf2,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x40,0x01,0x00,0x00,0x3e,0x01,0x00,0x00,0x3f,0x01,0x00,0x00, -0x41,0x00,0x08,0x00,0xfc,0x00,0x00,0x00,0x42,0x01,0x00,0x00, -0xfa,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0xe5,0x00,0x00,0x00, -0xc6,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0xe6,0x00,0x00,0x00,0x43,0x01,0x00,0x00,0x42,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0xff,0x00,0x00,0x00,0x44,0x01,0x00,0x00, -0xeb,0x00,0x00,0x00,0x40,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0x44,0x01,0x00,0x00,0x43,0x01,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x49,0x01,0x00,0x00,0xde,0x00,0x00,0x00, -0x48,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x4c,0x01,0x00,0x00,0x49,0x01,0x00,0x00,0xf2,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x4e,0x01,0x00,0x00, -0x4c,0x01,0x00,0x00,0x4d,0x01,0x00,0x00,0x41,0x00,0x08,0x00, -0xfc,0x00,0x00,0x00,0x50,0x01,0x00,0x00,0xfa,0x00,0x00,0x00, -0x34,0x00,0x00,0x00,0xe5,0x00,0x00,0x00,0xc6,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0xe6,0x00,0x00,0x00, -0x51,0x01,0x00,0x00,0x50,0x01,0x00,0x00,0x41,0x00,0x05,0x00, -0xff,0x00,0x00,0x00,0x52,0x01,0x00,0x00,0xeb,0x00,0x00,0x00, -0x4e,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x52,0x01,0x00,0x00, -0x51,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x57,0x01,0x00,0x00,0xde,0x00,0x00,0x00,0x56,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x5a,0x01,0x00,0x00, -0x57,0x01,0x00,0x00,0xf2,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x5c,0x01,0x00,0x00,0x5a,0x01,0x00,0x00, -0x5b,0x01,0x00,0x00,0x41,0x00,0x08,0x00,0xfc,0x00,0x00,0x00, -0x5e,0x01,0x00,0x00,0xfa,0x00,0x00,0x00,0x34,0x00,0x00,0x00, -0xe5,0x00,0x00,0x00,0xc6,0x00,0x00,0x00,0x23,0x01,0x00,0x00, -0x3d,0x00,0x04,0x00,0xe6,0x00,0x00,0x00,0x5f,0x01,0x00,0x00, -0x5e,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0xff,0x00,0x00,0x00, -0x60,0x01,0x00,0x00,0xeb,0x00,0x00,0x00,0x5c,0x01,0x00,0x00, -0x3e,0x00,0x03,0x00,0x60,0x01,0x00,0x00,0x5f,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x67,0x01,0x00,0x00, -0x3a,0x03,0x00,0x00,0x65,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0xd3,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xd5,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x69,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x69,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0x3b,0x03,0x00,0x00,0x3e,0x00,0x00,0x00,0xd5,0x00,0x00,0x00, -0xee,0x01,0x00,0x00,0x6a,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb8,0x00,0x00,0x00,0x6f,0x01,0x00,0x00,0x3b,0x03,0x00,0x00, -0x9d,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x6b,0x01,0x00,0x00, -0x6a,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x6f,0x01,0x00,0x00,0x6a,0x01,0x00,0x00,0x6b,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x6a,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x74,0x01,0x00,0x00,0x74,0x00,0x00,0x00, -0x3b,0x03,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x77,0x01,0x00,0x00,0x74,0x01,0x00,0x00,0xa1,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x78,0x01,0x00,0x00, -0x77,0x01,0x00,0x00,0x6d,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x79,0x01,0x00,0x00,0x42,0x03,0x00,0x00, -0x78,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x7b,0x01,0x00,0x00,0x79,0x01,0x00,0x00,0x6f,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x85,0x01,0x00,0x00, -0x74,0x01,0x00,0x00,0x84,0x01,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x87,0x01,0x00,0x00,0x6f,0x00,0x00,0x00, -0x6d,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x88,0x01,0x00,0x00,0x85,0x01,0x00,0x00,0x87,0x01,0x00,0x00, -0x41,0x00,0x08,0x00,0xfc,0x00,0x00,0x00,0x8f,0x01,0x00,0x00, -0x8d,0x01,0x00,0x00,0x34,0x00,0x00,0x00,0x7b,0x01,0x00,0x00, -0x34,0x00,0x00,0x00,0x3e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0xe6,0x00,0x00,0x00,0x90,0x01,0x00,0x00,0x8f,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0xff,0x00,0x00,0x00,0x91,0x01,0x00,0x00, -0x80,0x01,0x00,0x00,0x88,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0x91,0x01,0x00,0x00,0x90,0x01,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x96,0x01,0x00,0x00,0x74,0x01,0x00,0x00, -0x95,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x99,0x01,0x00,0x00,0x96,0x01,0x00,0x00,0x87,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9a,0x01,0x00,0x00, -0x99,0x01,0x00,0x00,0x39,0x00,0x00,0x00,0x41,0x00,0x08,0x00, -0xfc,0x00,0x00,0x00,0x9c,0x01,0x00,0x00,0x8d,0x01,0x00,0x00, -0x34,0x00,0x00,0x00,0x7b,0x01,0x00,0x00,0x34,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0xe6,0x00,0x00,0x00, -0x9d,0x01,0x00,0x00,0x9c,0x01,0x00,0x00,0x41,0x00,0x05,0x00, -0xff,0x00,0x00,0x00,0x9e,0x01,0x00,0x00,0x80,0x01,0x00,0x00, -0x9a,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x9e,0x01,0x00,0x00, -0x9d,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xa3,0x01,0x00,0x00,0x74,0x01,0x00,0x00,0xa2,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa6,0x01,0x00,0x00, -0xa3,0x01,0x00,0x00,0x87,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xa7,0x01,0x00,0x00,0xa6,0x01,0x00,0x00, -0x0c,0x00,0x00,0x00,0x41,0x00,0x08,0x00,0xfc,0x00,0x00,0x00, -0xa9,0x01,0x00,0x00,0x8d,0x01,0x00,0x00,0x34,0x00,0x00,0x00, -0x7b,0x01,0x00,0x00,0x34,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0xe6,0x00,0x00,0x00,0xaa,0x01,0x00,0x00, -0xa9,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0xff,0x00,0x00,0x00, -0xab,0x01,0x00,0x00,0x80,0x01,0x00,0x00,0xa7,0x01,0x00,0x00, -0x3e,0x00,0x03,0x00,0xab,0x01,0x00,0x00,0xaa,0x01,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xb0,0x01,0x00,0x00, -0x74,0x01,0x00,0x00,0xaf,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xb3,0x01,0x00,0x00,0xb0,0x01,0x00,0x00, -0x87,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xb4,0x01,0x00,0x00,0xb3,0x01,0x00,0x00,0x23,0x01,0x00,0x00, -0x41,0x00,0x08,0x00,0xfc,0x00,0x00,0x00,0xb6,0x01,0x00,0x00, -0x8d,0x01,0x00,0x00,0x34,0x00,0x00,0x00,0x7b,0x01,0x00,0x00, -0x34,0x00,0x00,0x00,0x23,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, -0xe6,0x00,0x00,0x00,0xb7,0x01,0x00,0x00,0xb6,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0xff,0x00,0x00,0x00,0xb8,0x01,0x00,0x00, -0x80,0x01,0x00,0x00,0xb4,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0xb8,0x01,0x00,0x00,0xb7,0x01,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xbd,0x01,0x00,0x00,0x74,0x01,0x00,0x00, -0xbc,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xc0,0x01,0x00,0x00,0xbd,0x01,0x00,0x00,0x87,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xc1,0x01,0x00,0x00, -0xc0,0x01,0x00,0x00,0x31,0x01,0x00,0x00,0x41,0x00,0x08,0x00, -0xfc,0x00,0x00,0x00,0xc3,0x01,0x00,0x00,0x8d,0x01,0x00,0x00, -0x34,0x00,0x00,0x00,0x7b,0x01,0x00,0x00,0xc6,0x00,0x00,0x00, -0x3e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0xe6,0x00,0x00,0x00, -0xc4,0x01,0x00,0x00,0xc3,0x01,0x00,0x00,0x41,0x00,0x05,0x00, -0xff,0x00,0x00,0x00,0xc5,0x01,0x00,0x00,0x80,0x01,0x00,0x00, -0xc1,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0xc5,0x01,0x00,0x00, -0xc4,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xca,0x01,0x00,0x00,0x74,0x01,0x00,0x00,0xc9,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xcd,0x01,0x00,0x00, -0xca,0x01,0x00,0x00,0x87,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xce,0x01,0x00,0x00,0xcd,0x01,0x00,0x00, -0x3f,0x01,0x00,0x00,0x41,0x00,0x08,0x00,0xfc,0x00,0x00,0x00, -0xd0,0x01,0x00,0x00,0x8d,0x01,0x00,0x00,0x34,0x00,0x00,0x00, -0x7b,0x01,0x00,0x00,0xc6,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0xe6,0x00,0x00,0x00,0xd1,0x01,0x00,0x00, -0xd0,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0xff,0x00,0x00,0x00, -0xd2,0x01,0x00,0x00,0x80,0x01,0x00,0x00,0xce,0x01,0x00,0x00, -0x3e,0x00,0x03,0x00,0xd2,0x01,0x00,0x00,0xd1,0x01,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xd7,0x01,0x00,0x00, -0x74,0x01,0x00,0x00,0xd6,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xda,0x01,0x00,0x00,0xd7,0x01,0x00,0x00, -0x87,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xdb,0x01,0x00,0x00,0xda,0x01,0x00,0x00,0x4d,0x01,0x00,0x00, -0x41,0x00,0x08,0x00,0xfc,0x00,0x00,0x00,0xdd,0x01,0x00,0x00, -0x8d,0x01,0x00,0x00,0x34,0x00,0x00,0x00,0x7b,0x01,0x00,0x00, -0xc6,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0xe6,0x00,0x00,0x00,0xde,0x01,0x00,0x00,0xdd,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0xff,0x00,0x00,0x00,0xdf,0x01,0x00,0x00, -0x80,0x01,0x00,0x00,0xdb,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0xdf,0x01,0x00,0x00,0xde,0x01,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xe4,0x01,0x00,0x00,0x74,0x01,0x00,0x00, -0xe3,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xe7,0x01,0x00,0x00,0xe4,0x01,0x00,0x00,0x87,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xe8,0x01,0x00,0x00, -0xe7,0x01,0x00,0x00,0x5b,0x01,0x00,0x00,0x41,0x00,0x08,0x00, -0xfc,0x00,0x00,0x00,0xea,0x01,0x00,0x00,0x8d,0x01,0x00,0x00, -0x34,0x00,0x00,0x00,0x7b,0x01,0x00,0x00,0xc6,0x00,0x00,0x00, -0x23,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0xe6,0x00,0x00,0x00, -0xeb,0x01,0x00,0x00,0xea,0x01,0x00,0x00,0x41,0x00,0x05,0x00, -0xff,0x00,0x00,0x00,0xec,0x01,0x00,0x00,0x80,0x01,0x00,0x00, -0xe8,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0xec,0x01,0x00,0x00, -0xeb,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xee,0x01,0x00,0x00,0x3b,0x03,0x00,0x00,0x65,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0x69,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x6b,0x01,0x00,0x00,0xe0,0x00,0x04,0x00,0x0c,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0xef,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xf2,0x01,0x00,0x00,0x3e,0x03,0x00,0x00, -0xf0,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xf5,0x01,0x00,0x00,0x42,0x03,0x00,0x00,0xf3,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0xf7,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xf7,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0x44,0x03,0x00,0x00,0x3e,0x00,0x00,0x00,0x6b,0x01,0x00,0x00, -0xa1,0x02,0x00,0x00,0xfa,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb8,0x00,0x00,0x00,0xfd,0x01,0x00,0x00,0x44,0x03,0x00,0x00, -0x6c,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xf9,0x01,0x00,0x00, -0xfa,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0xfd,0x01,0x00,0x00,0xf8,0x01,0x00,0x00,0xf9,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xf8,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0xff,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xff,0x01,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x48,0x03,0x00,0x00, -0x3e,0x00,0x00,0x00,0xf8,0x01,0x00,0x00,0x2b,0x02,0x00,0x00, -0x02,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00, -0x05,0x02,0x00,0x00,0x48,0x03,0x00,0x00,0x60,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0x01,0x02,0x00,0x00,0x02,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x05,0x02,0x00,0x00, -0x00,0x02,0x00,0x00,0x01,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x00,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x07,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x07,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0x5a,0x03,0x00,0x00,0x3e,0x00,0x00,0x00, -0x00,0x02,0x00,0x00,0x29,0x02,0x00,0x00,0x08,0x02,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00,0x0d,0x02,0x00,0x00, -0x5a,0x03,0x00,0x00,0x62,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0x09,0x02,0x00,0x00,0x08,0x02,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x0d,0x02,0x00,0x00,0x08,0x02,0x00,0x00, -0x09,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x08,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x13,0x02,0x00,0x00, -0x48,0x03,0x00,0x00,0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x15,0x02,0x00,0x00,0x13,0x02,0x00,0x00, -0x5a,0x03,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x17,0x02,0x00,0x00,0x55,0x00,0x00,0x00,0x53,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x19,0x02,0x00,0x00, -0x48,0x03,0x00,0x00,0x61,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x1a,0x02,0x00,0x00,0x17,0x02,0x00,0x00, -0x19,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x1c,0x02,0x00,0x00,0x64,0x00,0x00,0x00,0x62,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x1d,0x02,0x00,0x00, -0x1a,0x02,0x00,0x00,0x1c,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x1f,0x02,0x00,0x00,0x1d,0x02,0x00,0x00, -0x5a,0x03,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x21,0x02,0x00,0x00,0x1f,0x02,0x00,0x00,0x20,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x23,0x02,0x00,0x00, -0x21,0x02,0x00,0x00,0x44,0x03,0x00,0x00,0x41,0x00,0x05,0x00, -0xff,0x00,0x00,0x00,0x24,0x02,0x00,0x00,0xeb,0x00,0x00,0x00, -0x23,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0xe6,0x00,0x00,0x00, -0x25,0x02,0x00,0x00,0x24,0x02,0x00,0x00,0x41,0x00,0x05,0x00, -0x26,0x02,0x00,0x00,0x27,0x02,0x00,0x00,0x11,0x02,0x00,0x00, -0x15,0x02,0x00,0x00,0x3e,0x00,0x03,0x00,0x27,0x02,0x00,0x00, -0x25,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x29,0x02,0x00,0x00,0x5a,0x03,0x00,0x00,0xc6,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x07,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x09,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x02,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x02,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x2b,0x02,0x00,0x00,0x48,0x03,0x00,0x00, -0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xff,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x01,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x2d,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x2d,0x02,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x49,0x03,0x00,0x00, -0x3e,0x00,0x00,0x00,0x01,0x02,0x00,0x00,0x59,0x02,0x00,0x00, -0x30,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00, -0x33,0x02,0x00,0x00,0x49,0x03,0x00,0x00,0xb5,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0x2f,0x02,0x00,0x00,0x30,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x33,0x02,0x00,0x00, -0x2e,0x02,0x00,0x00,0x2f,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x2e,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x35,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x35,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0x57,0x03,0x00,0x00,0x3e,0x00,0x00,0x00, -0x2e,0x02,0x00,0x00,0x57,0x02,0x00,0x00,0x36,0x02,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00,0x3b,0x02,0x00,0x00, -0x57,0x03,0x00,0x00,0xb2,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0x37,0x02,0x00,0x00,0x36,0x02,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x3b,0x02,0x00,0x00,0x36,0x02,0x00,0x00, -0x37,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x36,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x41,0x02,0x00,0x00, -0x49,0x03,0x00,0x00,0xb2,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x43,0x02,0x00,0x00,0x41,0x02,0x00,0x00, -0x57,0x03,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x45,0x02,0x00,0x00,0x59,0x00,0x00,0x00,0xaf,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x48,0x02,0x00,0x00, -0x49,0x03,0x00,0x00,0x47,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x49,0x02,0x00,0x00,0x45,0x02,0x00,0x00, -0x48,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x4b,0x02,0x00,0x00,0x68,0x00,0x00,0x00,0xb2,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x4c,0x02,0x00,0x00, -0x49,0x02,0x00,0x00,0x4b,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x4e,0x02,0x00,0x00,0x4c,0x02,0x00,0x00, -0x57,0x03,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x50,0x02,0x00,0x00,0x4e,0x02,0x00,0x00,0x4f,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x52,0x02,0x00,0x00, -0x50,0x02,0x00,0x00,0x44,0x03,0x00,0x00,0x41,0x00,0x05,0x00, -0xff,0x00,0x00,0x00,0x53,0x02,0x00,0x00,0x80,0x01,0x00,0x00, -0x52,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0xe6,0x00,0x00,0x00, -0x54,0x02,0x00,0x00,0x53,0x02,0x00,0x00,0x41,0x00,0x05,0x00, -0x26,0x02,0x00,0x00,0x55,0x02,0x00,0x00,0x3f,0x02,0x00,0x00, -0x43,0x02,0x00,0x00,0x3e,0x00,0x03,0x00,0x55,0x02,0x00,0x00, -0x54,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x57,0x02,0x00,0x00,0x57,0x03,0x00,0x00,0xc6,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x35,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x37,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x30,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x30,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x59,0x02,0x00,0x00,0x49,0x03,0x00,0x00, -0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x2d,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x2f,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x5b,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x5b,0x02,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x4a,0x03,0x00,0x00, -0x3e,0x00,0x00,0x00,0x2f,0x02,0x00,0x00,0x9f,0x02,0x00,0x00, -0x5e,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00, -0x61,0x02,0x00,0x00,0x4a,0x03,0x00,0x00,0xb5,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0x5d,0x02,0x00,0x00,0x5e,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x61,0x02,0x00,0x00, -0x5c,0x02,0x00,0x00,0x5d,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x5c,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x63,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x63,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0x4e,0x03,0x00,0x00,0x3e,0x00,0x00,0x00, -0x5c,0x02,0x00,0x00,0x9d,0x02,0x00,0x00,0x66,0x02,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00,0x69,0x02,0x00,0x00, -0x4e,0x03,0x00,0x00,0x60,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0x65,0x02,0x00,0x00,0x66,0x02,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x69,0x02,0x00,0x00,0x64,0x02,0x00,0x00, -0x65,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x64,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0x6b,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x6b,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0x50,0x03,0x00,0x00,0x3e,0x00,0x00,0x00,0x64,0x02,0x00,0x00, -0x9b,0x02,0x00,0x00,0x6e,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb8,0x00,0x00,0x00,0x71,0x02,0x00,0x00,0x50,0x03,0x00,0x00, -0xb2,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x6d,0x02,0x00,0x00, -0x6e,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x71,0x02,0x00,0x00,0x6c,0x02,0x00,0x00,0x6d,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x6c,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x73,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x73,0x02,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x52,0x03,0x00,0x00, -0x3e,0x00,0x00,0x00,0x6c,0x02,0x00,0x00,0x99,0x02,0x00,0x00, -0x74,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00, -0x79,0x02,0x00,0x00,0x52,0x03,0x00,0x00,0x62,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0x75,0x02,0x00,0x00,0x74,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x79,0x02,0x00,0x00, -0x74,0x02,0x00,0x00,0x75,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x74,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x7b,0x02,0x00,0x00,0x4a,0x03,0x00,0x00,0xb2,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x7d,0x02,0x00,0x00, -0x7b,0x02,0x00,0x00,0x50,0x03,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x7f,0x02,0x00,0x00,0x7d,0x02,0x00,0x00, -0x7e,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x81,0x02,0x00,0x00,0x4e,0x03,0x00,0x00,0x62,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x82,0x02,0x00,0x00, -0x7f,0x02,0x00,0x00,0x81,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x84,0x02,0x00,0x00,0x82,0x02,0x00,0x00, -0x52,0x03,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x88,0x02,0x00,0x00,0x81,0x02,0x00,0x00,0x52,0x03,0x00,0x00, -0x41,0x00,0x05,0x00,0x26,0x02,0x00,0x00,0x89,0x02,0x00,0x00, -0x11,0x02,0x00,0x00,0x88,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, -0xe6,0x00,0x00,0x00,0x8a,0x02,0x00,0x00,0x89,0x02,0x00,0x00, -0x73,0x00,0x04,0x00,0xba,0x00,0x00,0x00,0x8b,0x02,0x00,0x00, -0x8a,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x26,0x02,0x00,0x00, -0x90,0x02,0x00,0x00,0x3f,0x02,0x00,0x00,0x7d,0x02,0x00,0x00, -0x3d,0x00,0x04,0x00,0xe6,0x00,0x00,0x00,0x91,0x02,0x00,0x00, -0x90,0x02,0x00,0x00,0x73,0x00,0x04,0x00,0xba,0x00,0x00,0x00, -0x92,0x02,0x00,0x00,0x91,0x02,0x00,0x00,0x41,0x00,0x05,0x00, -0xc3,0x00,0x00,0x00,0x94,0x02,0x00,0x00,0xc0,0x00,0x00,0x00, -0x84,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0xba,0x00,0x00,0x00, -0x95,0x02,0x00,0x00,0x94,0x02,0x00,0x00,0x0c,0x00,0x08,0x00, -0xba,0x00,0x00,0x00,0x96,0x02,0x00,0x00,0x01,0x00,0x00,0x00, -0x32,0x00,0x00,0x00,0x8b,0x02,0x00,0x00,0x92,0x02,0x00,0x00, -0x95,0x02,0x00,0x00,0x3e,0x00,0x03,0x00,0x94,0x02,0x00,0x00, -0x96,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x99,0x02,0x00,0x00,0x52,0x03,0x00,0x00,0xc6,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x73,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x75,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x6e,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x6e,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x9b,0x02,0x00,0x00,0x50,0x03,0x00,0x00, -0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x6b,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x6d,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x66,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x66,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9d,0x02,0x00,0x00, -0x4e,0x03,0x00,0x00,0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x63,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x65,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0x5e,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x5e,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x9f,0x02,0x00,0x00,0x4a,0x03,0x00,0x00,0xc6,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x5b,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x5d,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0xfa,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xfa,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xa1,0x02,0x00,0x00,0x44,0x03,0x00,0x00, -0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xf7,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xf9,0x01,0x00,0x00,0xe0,0x00,0x04,0x00, -0x0c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0xef,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0xcd,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xcd,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xa3,0x02,0x00,0x00,0x2a,0x03,0x00,0x00,0x6c,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xca,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xcc,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xa8,0x02,0x00,0x00,0x55,0x00,0x00,0x00,0x53,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa9,0x02,0x00,0x00, -0x8c,0x00,0x00,0x00,0xa8,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xae,0x02,0x00,0x00,0x59,0x00,0x00,0x00, -0xaf,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xaf,0x02,0x00,0x00,0x9e,0x00,0x00,0x00,0xae,0x02,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0xb4,0x02,0x00,0x00, -0x12,0x00,0x00,0x00,0xb3,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xb5,0x02,0x00,0x00,0xb4,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xb6,0x02,0x00,0x00, -0x0f,0x00,0x00,0x00,0xb5,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xba,0x02,0x00,0x00,0x47,0x00,0x00,0x00, -0xb5,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00, -0xbc,0x02,0x00,0x00,0xbb,0x02,0x00,0x00,0x0c,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xbd,0x02,0x00,0x00, -0xbc,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xbe,0x02,0x00,0x00,0xba,0x02,0x00,0x00,0xbd,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xbf,0x02,0x00,0x00, -0xb6,0x02,0x00,0x00,0xbe,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0xc1,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0xc1,0x02,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x2b,0x03,0x00,0x00, -0x3e,0x00,0x00,0x00,0xcc,0x00,0x00,0x00,0x28,0x03,0x00,0x00, -0xc4,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00, -0xc7,0x02,0x00,0x00,0x2b,0x03,0x00,0x00,0xb5,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0xc3,0x02,0x00,0x00,0xc4,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xc7,0x02,0x00,0x00, -0xc2,0x02,0x00,0x00,0xc3,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0xc2,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0xc9,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0xc9,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0x2c,0x03,0x00,0x00,0x3e,0x00,0x00,0x00, -0xc2,0x02,0x00,0x00,0x26,0x03,0x00,0x00,0xcc,0x02,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00,0xcf,0x02,0x00,0x00, -0x2c,0x03,0x00,0x00,0x60,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0xcb,0x02,0x00,0x00,0xcc,0x02,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xcf,0x02,0x00,0x00,0xca,0x02,0x00,0x00, -0xcb,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0xca,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xd3,0x02,0x00,0x00, -0x2c,0x03,0x00,0x00,0x61,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xd4,0x02,0x00,0x00,0xa9,0x02,0x00,0x00, -0xd3,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xd6,0x02,0x00,0x00,0x64,0x00,0x00,0x00,0x62,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xd7,0x02,0x00,0x00, -0xd4,0x02,0x00,0x00,0xd6,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xdb,0x02,0x00,0x00,0x2b,0x03,0x00,0x00, -0x47,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xdc,0x02,0x00,0x00,0xaf,0x02,0x00,0x00,0xdb,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xde,0x02,0x00,0x00, -0x68,0x00,0x00,0x00,0xb2,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xdf,0x02,0x00,0x00,0xdc,0x02,0x00,0x00, -0xde,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0xe1,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0xe1,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0x2e,0x03,0x00,0x00,0x3e,0x00,0x00,0x00, -0xca,0x02,0x00,0x00,0x24,0x03,0x00,0x00,0xe4,0x02,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00,0xe7,0x02,0x00,0x00, -0x2e,0x03,0x00,0x00,0xb2,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0xe3,0x02,0x00,0x00,0xe4,0x02,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xe7,0x02,0x00,0x00,0xe2,0x02,0x00,0x00, -0xe3,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0xe2,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0xe9,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0xe9,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0x30,0x03,0x00,0x00,0x3e,0x00,0x00,0x00,0xe2,0x02,0x00,0x00, -0x22,0x03,0x00,0x00,0xec,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb8,0x00,0x00,0x00,0xef,0x02,0x00,0x00,0x30,0x03,0x00,0x00, -0x62,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xeb,0x02,0x00,0x00, -0xec,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0xef,0x02,0x00,0x00,0xea,0x02,0x00,0x00,0xeb,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0xea,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xf2,0x02,0x00,0x00,0xd7,0x02,0x00,0x00, -0x30,0x03,0x00,0x00,0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00, -0xf5,0x02,0x00,0x00,0xf2,0x02,0x00,0x00,0x36,0x00,0x00,0x00, -0xf7,0x00,0x03,0x00,0xf7,0x02,0x00,0x00,0x00,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xf5,0x02,0x00,0x00,0xf6,0x02,0x00,0x00, -0xf7,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0xf6,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xfa,0x02,0x00,0x00, -0xdf,0x02,0x00,0x00,0x2e,0x03,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0xfb,0x02,0x00,0x00,0x12,0x00,0x00,0x00, -0xc6,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xfc,0x02,0x00,0x00,0xfb,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb8,0x00,0x00,0x00,0xfd,0x02,0x00,0x00,0xfa,0x02,0x00,0x00, -0xfc,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0xf7,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0xf7,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, -0xb8,0x00,0x00,0x00,0xfe,0x02,0x00,0x00,0xf5,0x02,0x00,0x00, -0xea,0x02,0x00,0x00,0xfd,0x02,0x00,0x00,0xf6,0x02,0x00,0x00, -0xf7,0x00,0x03,0x00,0x00,0x03,0x00,0x00,0x00,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xfe,0x02,0x00,0x00,0xff,0x02,0x00,0x00, -0x00,0x03,0x00,0x00,0xf8,0x00,0x02,0x00,0xff,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x08,0x03,0x00,0x00, -0xdf,0x02,0x00,0x00,0x2e,0x03,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x0a,0x03,0x00,0x00,0x12,0x00,0x00,0x00, -0x09,0x03,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x0b,0x03,0x00,0x00,0x0a,0x03,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x0c,0x03,0x00,0x00,0x08,0x03,0x00,0x00, -0x0b,0x03,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x0d,0x03,0x00,0x00,0xbf,0x02,0x00,0x00,0x0c,0x03,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x0f,0x03,0x00,0x00, -0x0d,0x03,0x00,0x00,0xd7,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x11,0x03,0x00,0x00,0x0f,0x03,0x00,0x00, -0x30,0x03,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x13,0x03,0x00,0x00,0x2b,0x03,0x00,0x00,0xb2,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x15,0x03,0x00,0x00, -0x13,0x03,0x00,0x00,0x2e,0x03,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x17,0x03,0x00,0x00,0x15,0x03,0x00,0x00, -0x16,0x03,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x19,0x03,0x00,0x00,0x2c,0x03,0x00,0x00,0x62,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x1a,0x03,0x00,0x00, -0x17,0x03,0x00,0x00,0x19,0x03,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x1c,0x03,0x00,0x00,0x1a,0x03,0x00,0x00, -0x30,0x03,0x00,0x00,0x41,0x00,0x05,0x00,0xc3,0x00,0x00,0x00, -0x1d,0x03,0x00,0x00,0xc0,0x00,0x00,0x00,0x1c,0x03,0x00,0x00, -0x3d,0x00,0x04,0x00,0xba,0x00,0x00,0x00,0x1e,0x03,0x00,0x00, -0x1d,0x03,0x00,0x00,0x41,0x00,0x06,0x00,0x1f,0x03,0x00,0x00, -0x20,0x03,0x00,0x00,0x04,0x03,0x00,0x00,0x34,0x00,0x00,0x00, -0x11,0x03,0x00,0x00,0x3e,0x00,0x03,0x00,0x20,0x03,0x00,0x00, -0x1e,0x03,0x00,0x00,0xf9,0x00,0x02,0x00,0x00,0x03,0x00,0x00, -0xf8,0x00,0x02,0x00,0x00,0x03,0x00,0x00,0xf9,0x00,0x02,0x00, -0xec,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0xec,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x22,0x03,0x00,0x00, -0x30,0x03,0x00,0x00,0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xe9,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0xeb,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0xe4,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0xe4,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x24,0x03,0x00,0x00,0x2e,0x03,0x00,0x00,0xc6,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xe1,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0xe3,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0xcc,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0xcc,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x26,0x03,0x00,0x00,0x2c,0x03,0x00,0x00, -0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xc9,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0xcb,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0xc4,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0xc4,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x28,0x03,0x00,0x00, -0x2b,0x03,0x00,0x00,0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xc1,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0xc3,0x02,0x00,0x00, -0xfd,0x00,0x01,0x00,0x38,0x00,0x01,0x00, -}; -const uint64_t matmul_f16_aligned_l_len = 11936; - -unsigned char matmul_f16_aligned_l_fp32_data[] = { -0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, -0xf3,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, -0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x51,0x11,0x00,0x00, -0x0b,0x00,0x06,0x00,0x01,0x00,0x00,0x00,0x47,0x4c,0x53,0x4c, -0x2e,0x73,0x74,0x64,0x2e,0x34,0x35,0x30,0x00,0x00,0x00,0x00, -0x0e,0x00,0x03,0x00,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x0f,0x00,0x0f,0x00,0x05,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x6d,0x61,0x69,0x6e,0x00,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x3d,0x00,0x00,0x00,0x4c,0x00,0x00,0x00, -0xea,0x00,0x00,0x00,0xf9,0x00,0x00,0x00,0x4b,0x01,0x00,0x00, -0x58,0x01,0x00,0x00,0x53,0x02,0x00,0x00,0x9c,0x02,0x00,0x00, -0x10,0x00,0x06,0x00,0x04,0x00,0x00,0x00,0x11,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x0b,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x1c,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, +0x06,0x01,0x00,0x00,0x21,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x20,0x01,0x00,0x00,0x01,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x21,0x01,0x00,0x00, +0x0b,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x4e,0x01,0x00,0x00,0x06,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x48,0x00,0x04,0x00,0x4f,0x01,0x00,0x00,0x00,0x00,0x00,0x00, +0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x4f,0x01,0x00,0x00, 0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x10,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x08,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x03,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x10,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x14,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x18,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x07,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x10,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x24,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x0a,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x10,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x2c,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x30,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x0d,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0x47,0x00,0x03,0x00, -0x10,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x37,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x3d,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x1a,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x4c,0x00,0x00,0x00, -0x0b,0x00,0x00,0x00,0x1b,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x4f,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x53,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x60,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x62,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x07,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x6c,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x03,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x9d,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0xaf,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x05,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0xb2,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x08,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xf6,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x48,0x00,0x04,0x00, -0xf7,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0xf7,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, -0xf7,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0xf9,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0xf9,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x2c,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x2d,0x01,0x00,0x00,0x0b,0x00,0x00,0x00,0x19,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x55,0x01,0x00,0x00,0x06,0x00,0x00,0x00, -0x08,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0x56,0x01,0x00,0x00, -0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x56,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x56,0x01,0x00,0x00, -0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x58,0x01,0x00,0x00, -0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x58,0x01,0x00,0x00,0x21,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x53,0x02,0x00,0x00,0x0b,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x99,0x02,0x00,0x00, -0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00, -0x9a,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x19,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x9a,0x02,0x00,0x00,0x00,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, -0x9a,0x02,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x9c,0x02,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x9c,0x02,0x00,0x00,0x21,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x13,0x00,0x02,0x00,0x02,0x00,0x00,0x00, -0x21,0x00,0x03,0x00,0x03,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x15,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x17,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x0a,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x0d,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x1e,0x00,0x10,0x00,0x10,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x11,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x11,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x15,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, -0x14,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x15,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x0a,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, -0x27,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0x2d,0x00,0x00,0x00,0x07,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x34,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x37,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00,0x3d,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x3e,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x0a,0x00,0x00,0x00,0x4c,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x4f,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x53,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x54,0x00,0x00,0x00,0x86,0x00,0x00,0x00, -0x37,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x58,0x00,0x00,0x00,0x86,0x00,0x00,0x00, -0x37,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x32,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x61,0x00,0x00,0x00, -0x86,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x60,0x00,0x00,0x00, -0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x62,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x63,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x61,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x67,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x61,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x6e,0x00,0x00,0x00, -0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x73,0x00,0x00,0x00, -0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x77,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, -0x7c,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0x87,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x8d,0x00,0x00,0x00, -0x03,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, -0x98,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x32,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x9d,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x9f,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xae,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xaf,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xb0,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0x53,0x00,0x00,0x00,0xaf,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xb1,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0x4f,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x32,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xb2,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xb3,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0xb1,0x00,0x00,0x00,0xb2,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xb4,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0xb3,0x00,0x00,0x00,0x60,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xb5,0x00,0x00,0x00, -0x86,0x00,0x00,0x00,0xb0,0x00,0x00,0x00,0xb4,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xb6,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0xae,0x00,0x00,0x00,0xb5,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xb7,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0xb6,0x00,0x00,0x00,0xb2,0x00,0x00,0x00, -0x14,0x00,0x02,0x00,0xb8,0x00,0x00,0x00,0x16,0x00,0x03,0x00, -0xba,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xbb,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xbc,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0xbb,0x00,0x00,0x00,0xb5,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xbd,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0xbc,0x00,0x00,0x00,0xb2,0x00,0x00,0x00,0x1c,0x00,0x04,0x00, -0xbe,0x00,0x00,0x00,0xba,0x00,0x00,0x00,0xbd,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0xbf,0x00,0x00,0x00,0x07,0x00,0x00,0x00, -0xbe,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0xba,0x00,0x00,0x00, -0xc2,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0xc3,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0xba,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0xc6,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xe6,0x00,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xe7,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x37,0x00,0x00,0x00, -0xe6,0x00,0x00,0x00,0x1c,0x00,0x04,0x00,0xe8,0x00,0x00,0x00, -0xba,0x00,0x00,0x00,0xe7,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0xe9,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0xe8,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0xe9,0x00,0x00,0x00,0xea,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xee,0x00,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x16,0x00,0x03,0x00,0xf4,0x00,0x00,0x00, -0x10,0x00,0x00,0x00,0x17,0x00,0x04,0x00,0xf5,0x00,0x00,0x00, -0xf4,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, -0xf6,0x00,0x00,0x00,0xf5,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, -0xf7,0x00,0x00,0x00,0xf6,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0xf8,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0xf7,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0xf8,0x00,0x00,0x00,0xf9,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0xfb,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0xf4,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0xff,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0xba,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x04,0x01,0x00,0x00, -0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x12,0x01,0x00,0x00, -0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x20,0x01,0x00,0x00, -0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x25,0x01,0x00,0x00, -0x03,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x2c,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x33,0x00,0x06,0x00, -0x09,0x00,0x00,0x00,0x2d,0x01,0x00,0x00,0x2c,0x01,0x00,0x00, -0x39,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x2e,0x01,0x00,0x00,0x51,0x00,0x00,0x00, -0x2d,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x2f,0x01,0x00,0x00,0x84,0x00,0x00,0x00, -0x2e,0x01,0x00,0x00,0x6d,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x30,0x01,0x00,0x00,0x86,0x00,0x00,0x00, -0x2f,0x01,0x00,0x00,0x6c,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x47,0x01,0x00,0x00,0x80,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x48,0x01,0x00,0x00,0x84,0x00,0x00,0x00, -0x9d,0x00,0x00,0x00,0x47,0x01,0x00,0x00,0x1c,0x00,0x04,0x00, -0x49,0x01,0x00,0x00,0xba,0x00,0x00,0x00,0x48,0x01,0x00,0x00, -0x20,0x00,0x04,0x00,0x4a,0x01,0x00,0x00,0x04,0x00,0x00,0x00, -0x49,0x01,0x00,0x00,0x3b,0x00,0x04,0x00,0x4a,0x01,0x00,0x00, -0x4b,0x01,0x00,0x00,0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x4f,0x01,0x00,0x00,0x80,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, -0x55,0x01,0x00,0x00,0xf5,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, -0x56,0x01,0x00,0x00,0x55,0x01,0x00,0x00,0x20,0x00,0x04,0x00, -0x57,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x56,0x01,0x00,0x00, -0x3b,0x00,0x04,0x00,0x57,0x01,0x00,0x00,0x58,0x01,0x00,0x00, -0x0c,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x61,0x01,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x6f,0x01,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x7d,0x01,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x8a,0x01,0x00,0x00,0x08,0x01,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x8b,0x01,0x00,0x00,0x86,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x8e,0x01,0x00,0x00,0x86,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xa9,0x01,0x00,0x00,0x84,0x00,0x00,0x00, -0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x1c,0x00,0x04,0x00, -0xaa,0x01,0x00,0x00,0xba,0x00,0x00,0x00,0xa9,0x01,0x00,0x00, -0x20,0x00,0x04,0x00,0xab,0x01,0x00,0x00,0x07,0x00,0x00,0x00, -0xaa,0x01,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xbb,0x01,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xd6,0x01,0x00,0x00,0x84,0x00,0x00,0x00,0xb5,0x00,0x00,0x00, -0xb2,0x00,0x00,0x00,0x1c,0x00,0x04,0x00,0xd7,0x01,0x00,0x00, -0xba,0x00,0x00,0x00,0xd6,0x01,0x00,0x00,0x20,0x00,0x04,0x00, -0xd8,0x01,0x00,0x00,0x07,0x00,0x00,0x00,0xd7,0x01,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xe1,0x01,0x00,0x00, -0x86,0x00,0x00,0x00,0xaf,0x00,0x00,0x00,0xb5,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xe9,0x01,0x00,0x00, -0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x18,0x02,0x00,0x00, -0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x4b,0x02,0x00,0x00, -0x0d,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00, -0x53,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, -0x99,0x02,0x00,0x00,0xba,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, -0x9a,0x02,0x00,0x00,0x99,0x02,0x00,0x00,0x20,0x00,0x04,0x00, -0x9b,0x02,0x00,0x00,0x0c,0x00,0x00,0x00,0x9a,0x02,0x00,0x00, -0x3b,0x00,0x04,0x00,0x9b,0x02,0x00,0x00,0x9c,0x02,0x00,0x00, -0x0c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, -0xa1,0x02,0x00,0x00,0x05,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xae,0x02,0x00,0x00,0x84,0x00,0x00,0x00, -0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0xb7,0x02,0x00,0x00,0x0c,0x00,0x00,0x00,0xba,0x00,0x00,0x00, -0x36,0x00,0x05,0x00,0x02,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x05,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0xbf,0x00,0x00,0x00, -0xc0,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0xab,0x01,0x00,0x00,0xac,0x01,0x00,0x00,0x07,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0xd8,0x01,0x00,0x00,0xd9,0x01,0x00,0x00, -0x07,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00, -0x0e,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, -0x0e,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x14,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x17,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x17,0x00,0x00,0x00, -0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x1e,0x00,0x00,0x00, -0x0f,0x00,0x00,0x00,0x17,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x21,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x24,0x00,0x00,0x00,0x18,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0x28,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x27,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x29,0x00,0x00,0x00, -0x28,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x2a,0x00,0x00,0x00,0x1e,0x00,0x00,0x00,0x29,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x2d,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x2f,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x30,0x00,0x00,0x00, -0x24,0x00,0x00,0x00,0x2f,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x30,0x00,0x00,0x00, -0x2a,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0x35,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x34,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x36,0x00,0x00,0x00, -0x35,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x38,0x00,0x00,0x00,0x36,0x00,0x00,0x00,0x37,0x00,0x00,0x00, -0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, -0x38,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x3b,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, -0x37,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00, -0x3f,0x00,0x00,0x00,0x3d,0x00,0x00,0x00,0x3e,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x3f,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x42,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x3b,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x47,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0x3b,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x0d,0x00,0x00,0x00,0x49,0x00,0x00,0x00,0x3d,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x4a,0x00,0x00,0x00,0x49,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x0d,0x00,0x00,0x00,0x4d,0x00,0x00,0x00,0x4c,0x00,0x00,0x00, -0x3e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x4e,0x00,0x00,0x00,0x4d,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x50,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, -0x4f,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x55,0x00,0x00,0x00,0x50,0x00,0x00,0x00,0x54,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x59,0x00,0x00,0x00, -0x50,0x00,0x00,0x00,0x58,0x00,0x00,0x00,0x89,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x5d,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, -0x4f,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x64,0x00,0x00,0x00,0x5d,0x00,0x00,0x00,0x63,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x68,0x00,0x00,0x00, -0x5d,0x00,0x00,0x00,0x67,0x00,0x00,0x00,0x89,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x6f,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, -0x6e,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x74,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x73,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x78,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x77,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x79,0x00,0x00,0x00,0x78,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x7a,0x00,0x00,0x00, -0x47,0x00,0x00,0x00,0x79,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x7d,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x7c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x7e,0x00,0x00,0x00,0x7d,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x80,0x00,0x00,0x00,0x47,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x83,0x00,0x00,0x00,0x80,0x00,0x00,0x00,0x79,0x00,0x00,0x00, -0x0c,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x26,0x00,0x00,0x00,0x7e,0x00,0x00,0x00, -0x83,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0x88,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x87,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x89,0x00,0x00,0x00, -0x88,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x8a,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x89,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8c,0x00,0x00,0x00, -0x42,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x8e,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x8d,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x8f,0x00,0x00,0x00,0x8e,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x90,0x00,0x00,0x00,0x8c,0x00,0x00,0x00, -0x8f,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x91,0x00,0x00,0x00,0x8a,0x00,0x00,0x00,0x90,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x93,0x00,0x00,0x00, -0x91,0x00,0x00,0x00,0x7a,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x94,0x00,0x00,0x00,0x93,0x00,0x00,0x00, -0x6d,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0x99,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x98,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x9a,0x00,0x00,0x00, -0x99,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x9b,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x9a,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9e,0x00,0x00,0x00, -0x4a,0x00,0x00,0x00,0x9d,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0xa0,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x9f,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xa1,0x00,0x00,0x00,0xa0,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xa2,0x00,0x00,0x00,0x9e,0x00,0x00,0x00, -0xa1,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xa3,0x00,0x00,0x00,0x9b,0x00,0x00,0x00,0xa2,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa5,0x00,0x00,0x00, -0xa3,0x00,0x00,0x00,0x7a,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xa6,0x00,0x00,0x00,0xa5,0x00,0x00,0x00, -0x6d,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xa8,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xa8,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xc1,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, -0x05,0x00,0x00,0x00,0xc7,0x00,0x00,0x00,0xa9,0x00,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00,0xb9,0x00,0x00,0x00, -0xc1,0x02,0x00,0x00,0xb7,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0xaa,0x00,0x00,0x00,0xa9,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xb9,0x00,0x00,0x00,0xa9,0x00,0x00,0x00, -0xaa,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xa9,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0xc3,0x00,0x00,0x00,0xc4,0x00,0x00,0x00, -0xc0,0x00,0x00,0x00,0xc1,0x02,0x00,0x00,0x3e,0x00,0x03,0x00, -0xc4,0x00,0x00,0x00,0xc2,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xc7,0x00,0x00,0x00,0xc1,0x02,0x00,0x00, -0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xa8,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xaa,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xca,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xca,0x00,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xda,0x02,0x00,0x00, -0xa6,0x00,0x00,0x00,0xaa,0x00,0x00,0x00,0x90,0x01,0x00,0x00, -0xcd,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xd6,0x02,0x00,0x00,0x94,0x00,0x00,0x00,0xaa,0x00,0x00,0x00, -0x8d,0x01,0x00,0x00,0xcd,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xc2,0x02,0x00,0x00,0x7a,0x00,0x00,0x00, -0xaa,0x00,0x00,0x00,0x3b,0x02,0x00,0x00,0xcd,0x00,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00,0xd1,0x00,0x00,0x00, -0xc2,0x02,0x00,0x00,0x84,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0xcc,0x00,0x00,0x00,0xcd,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xd1,0x00,0x00,0x00,0xcb,0x00,0x00,0x00, -0xcc,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xcb,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xd3,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xd3,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xd2,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0xcb,0x00,0x00,0x00, -0x32,0x01,0x00,0x00,0xd4,0x00,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb8,0x00,0x00,0x00,0xd9,0x00,0x00,0x00,0xd2,0x02,0x00,0x00, -0x37,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xd5,0x00,0x00,0x00, -0xd4,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0xd9,0x00,0x00,0x00,0xd4,0x00,0x00,0x00,0xd5,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xd4,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xde,0x00,0x00,0x00,0x74,0x00,0x00,0x00, -0xd2,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xe1,0x00,0x00,0x00,0xde,0x00,0x00,0x00,0x8f,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xe2,0x00,0x00,0x00, -0xe1,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xe3,0x00,0x00,0x00,0xd6,0x02,0x00,0x00, -0xe2,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xe5,0x00,0x00,0x00,0xe3,0x00,0x00,0x00,0x6f,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xef,0x00,0x00,0x00, -0xde,0x00,0x00,0x00,0xee,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xf1,0x00,0x00,0x00,0x6f,0x00,0x00,0x00, -0x6d,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xf2,0x00,0x00,0x00,0xef,0x00,0x00,0x00,0xf1,0x00,0x00,0x00, -0x41,0x00,0x07,0x00,0xfb,0x00,0x00,0x00,0xfc,0x00,0x00,0x00, -0xf9,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0xe5,0x00,0x00,0x00, -0x3e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0xf4,0x00,0x00,0x00, -0xfd,0x00,0x00,0x00,0xfc,0x00,0x00,0x00,0x73,0x00,0x04,0x00, -0xba,0x00,0x00,0x00,0xfe,0x00,0x00,0x00,0xfd,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0xff,0x00,0x00,0x00,0x00,0x01,0x00,0x00, -0xea,0x00,0x00,0x00,0xf2,0x00,0x00,0x00,0x3e,0x00,0x03,0x00, -0x00,0x01,0x00,0x00,0xfe,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x05,0x01,0x00,0x00,0xde,0x00,0x00,0x00, -0x04,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x08,0x01,0x00,0x00,0x05,0x01,0x00,0x00,0xf1,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x09,0x01,0x00,0x00, -0x08,0x01,0x00,0x00,0x39,0x00,0x00,0x00,0x41,0x00,0x07,0x00, -0xfb,0x00,0x00,0x00,0x0b,0x01,0x00,0x00,0xf9,0x00,0x00,0x00, -0x34,0x00,0x00,0x00,0xe5,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0xf4,0x00,0x00,0x00,0x0c,0x01,0x00,0x00, -0x0b,0x01,0x00,0x00,0x73,0x00,0x04,0x00,0xba,0x00,0x00,0x00, -0x0d,0x01,0x00,0x00,0x0c,0x01,0x00,0x00,0x41,0x00,0x05,0x00, -0xff,0x00,0x00,0x00,0x0e,0x01,0x00,0x00,0xea,0x00,0x00,0x00, -0x09,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x0e,0x01,0x00,0x00, -0x0d,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x13,0x01,0x00,0x00,0xde,0x00,0x00,0x00,0x12,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x16,0x01,0x00,0x00, -0x13,0x01,0x00,0x00,0xf1,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x17,0x01,0x00,0x00,0x16,0x01,0x00,0x00, -0x0c,0x00,0x00,0x00,0x41,0x00,0x07,0x00,0xfb,0x00,0x00,0x00, -0x19,0x01,0x00,0x00,0xf9,0x00,0x00,0x00,0x34,0x00,0x00,0x00, -0xe5,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0xf4,0x00,0x00,0x00,0x1a,0x01,0x00,0x00,0x19,0x01,0x00,0x00, -0x73,0x00,0x04,0x00,0xba,0x00,0x00,0x00,0x1b,0x01,0x00,0x00, -0x1a,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0xff,0x00,0x00,0x00, -0x1c,0x01,0x00,0x00,0xea,0x00,0x00,0x00,0x17,0x01,0x00,0x00, -0x3e,0x00,0x03,0x00,0x1c,0x01,0x00,0x00,0x1b,0x01,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x21,0x01,0x00,0x00, -0xde,0x00,0x00,0x00,0x20,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x24,0x01,0x00,0x00,0x21,0x01,0x00,0x00, -0xf1,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x26,0x01,0x00,0x00,0x24,0x01,0x00,0x00,0x25,0x01,0x00,0x00, -0x41,0x00,0x07,0x00,0xfb,0x00,0x00,0x00,0x28,0x01,0x00,0x00, -0xf9,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0xe5,0x00,0x00,0x00, -0x25,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0xf4,0x00,0x00,0x00, -0x29,0x01,0x00,0x00,0x28,0x01,0x00,0x00,0x73,0x00,0x04,0x00, -0xba,0x00,0x00,0x00,0x2a,0x01,0x00,0x00,0x29,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0xff,0x00,0x00,0x00,0x2b,0x01,0x00,0x00, -0xea,0x00,0x00,0x00,0x26,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0x2b,0x01,0x00,0x00,0x2a,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x32,0x01,0x00,0x00,0xd2,0x02,0x00,0x00, -0x30,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xd3,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xd5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x34,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x34,0x01,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xd3,0x02,0x00,0x00, -0x3e,0x00,0x00,0x00,0xd5,0x00,0x00,0x00,0x89,0x01,0x00,0x00, -0x35,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00, -0x3a,0x01,0x00,0x00,0xd3,0x02,0x00,0x00,0x9d,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0x36,0x01,0x00,0x00,0x35,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x3a,0x01,0x00,0x00, -0x35,0x01,0x00,0x00,0x36,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x35,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x3f,0x01,0x00,0x00,0x74,0x00,0x00,0x00,0xd3,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x42,0x01,0x00,0x00, -0x3f,0x01,0x00,0x00,0xa1,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x43,0x01,0x00,0x00,0x42,0x01,0x00,0x00, -0x6d,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x44,0x01,0x00,0x00,0xda,0x02,0x00,0x00,0x43,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x46,0x01,0x00,0x00, -0x44,0x01,0x00,0x00,0x6f,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x50,0x01,0x00,0x00,0x3f,0x01,0x00,0x00, -0x4f,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x52,0x01,0x00,0x00,0x6f,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x53,0x01,0x00,0x00, -0x50,0x01,0x00,0x00,0x52,0x01,0x00,0x00,0x41,0x00,0x07,0x00, -0xfb,0x00,0x00,0x00,0x5a,0x01,0x00,0x00,0x58,0x01,0x00,0x00, -0x34,0x00,0x00,0x00,0x46,0x01,0x00,0x00,0x3e,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0xf4,0x00,0x00,0x00,0x5b,0x01,0x00,0x00, -0x5a,0x01,0x00,0x00,0x73,0x00,0x04,0x00,0xba,0x00,0x00,0x00, -0x5c,0x01,0x00,0x00,0x5b,0x01,0x00,0x00,0x41,0x00,0x05,0x00, -0xff,0x00,0x00,0x00,0x5d,0x01,0x00,0x00,0x4b,0x01,0x00,0x00, -0x53,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x5d,0x01,0x00,0x00, -0x5c,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x62,0x01,0x00,0x00,0x3f,0x01,0x00,0x00,0x61,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x65,0x01,0x00,0x00, -0x62,0x01,0x00,0x00,0x52,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x66,0x01,0x00,0x00,0x65,0x01,0x00,0x00, -0x39,0x00,0x00,0x00,0x41,0x00,0x07,0x00,0xfb,0x00,0x00,0x00, -0x68,0x01,0x00,0x00,0x58,0x01,0x00,0x00,0x34,0x00,0x00,0x00, -0x46,0x01,0x00,0x00,0x39,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0xf4,0x00,0x00,0x00,0x69,0x01,0x00,0x00,0x68,0x01,0x00,0x00, -0x73,0x00,0x04,0x00,0xba,0x00,0x00,0x00,0x6a,0x01,0x00,0x00, -0x69,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0xff,0x00,0x00,0x00, -0x6b,0x01,0x00,0x00,0x4b,0x01,0x00,0x00,0x66,0x01,0x00,0x00, -0x3e,0x00,0x03,0x00,0x6b,0x01,0x00,0x00,0x6a,0x01,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x70,0x01,0x00,0x00, -0x3f,0x01,0x00,0x00,0x6f,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x73,0x01,0x00,0x00,0x70,0x01,0x00,0x00, -0x52,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x74,0x01,0x00,0x00,0x73,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, -0x41,0x00,0x07,0x00,0xfb,0x00,0x00,0x00,0x76,0x01,0x00,0x00, -0x58,0x01,0x00,0x00,0x34,0x00,0x00,0x00,0x46,0x01,0x00,0x00, -0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0xf4,0x00,0x00,0x00, -0x77,0x01,0x00,0x00,0x76,0x01,0x00,0x00,0x73,0x00,0x04,0x00, -0xba,0x00,0x00,0x00,0x78,0x01,0x00,0x00,0x77,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0xff,0x00,0x00,0x00,0x79,0x01,0x00,0x00, -0x4b,0x01,0x00,0x00,0x74,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0x79,0x01,0x00,0x00,0x78,0x01,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x7e,0x01,0x00,0x00,0x3f,0x01,0x00,0x00, -0x7d,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x81,0x01,0x00,0x00,0x7e,0x01,0x00,0x00,0x52,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x82,0x01,0x00,0x00, -0x81,0x01,0x00,0x00,0x25,0x01,0x00,0x00,0x41,0x00,0x07,0x00, -0xfb,0x00,0x00,0x00,0x84,0x01,0x00,0x00,0x58,0x01,0x00,0x00, -0x34,0x00,0x00,0x00,0x46,0x01,0x00,0x00,0x25,0x01,0x00,0x00, -0x3d,0x00,0x04,0x00,0xf4,0x00,0x00,0x00,0x85,0x01,0x00,0x00, -0x84,0x01,0x00,0x00,0x73,0x00,0x04,0x00,0xba,0x00,0x00,0x00, -0x86,0x01,0x00,0x00,0x85,0x01,0x00,0x00,0x41,0x00,0x05,0x00, -0xff,0x00,0x00,0x00,0x87,0x01,0x00,0x00,0x4b,0x01,0x00,0x00, -0x82,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x87,0x01,0x00,0x00, -0x86,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x89,0x01,0x00,0x00,0xd3,0x02,0x00,0x00,0x30,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0x34,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x36,0x01,0x00,0x00,0xe0,0x00,0x04,0x00,0x0c,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x8a,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x8d,0x01,0x00,0x00,0xd6,0x02,0x00,0x00, -0x8b,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x90,0x01,0x00,0x00,0xda,0x02,0x00,0x00,0x8e,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0x92,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x92,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xdc,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0x36,0x01,0x00,0x00, -0x39,0x02,0x00,0x00,0x95,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb8,0x00,0x00,0x00,0x98,0x01,0x00,0x00,0xdc,0x02,0x00,0x00, -0x6c,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x94,0x01,0x00,0x00, -0x95,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x98,0x01,0x00,0x00,0x93,0x01,0x00,0x00,0x94,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x93,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0x9a,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x9a,0x01,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xe0,0x02,0x00,0x00, -0x3e,0x00,0x00,0x00,0x93,0x01,0x00,0x00,0xc5,0x01,0x00,0x00, -0x9d,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00, -0xa0,0x01,0x00,0x00,0xe0,0x02,0x00,0x00,0x60,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0x9c,0x01,0x00,0x00,0x9d,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xa0,0x01,0x00,0x00, -0x9b,0x01,0x00,0x00,0x9c,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x9b,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xa2,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xa2,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xf2,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, -0x9b,0x01,0x00,0x00,0xc3,0x01,0x00,0x00,0xa3,0x01,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00,0xa8,0x01,0x00,0x00, -0xf2,0x02,0x00,0x00,0x62,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0xa4,0x01,0x00,0x00,0xa3,0x01,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xa8,0x01,0x00,0x00,0xa3,0x01,0x00,0x00, -0xa4,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xa3,0x01,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xae,0x01,0x00,0x00, -0xe0,0x02,0x00,0x00,0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xb0,0x01,0x00,0x00,0xae,0x01,0x00,0x00, -0xf2,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xb2,0x01,0x00,0x00,0x55,0x00,0x00,0x00,0x53,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xb4,0x01,0x00,0x00, -0xe0,0x02,0x00,0x00,0x61,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xb5,0x01,0x00,0x00,0xb2,0x01,0x00,0x00, -0xb4,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xb7,0x01,0x00,0x00,0x64,0x00,0x00,0x00,0x62,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xb8,0x01,0x00,0x00, -0xb5,0x01,0x00,0x00,0xb7,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xba,0x01,0x00,0x00,0xb8,0x01,0x00,0x00, -0xf2,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xbc,0x01,0x00,0x00,0xba,0x01,0x00,0x00,0xbb,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xbe,0x01,0x00,0x00, -0xbc,0x01,0x00,0x00,0xdc,0x02,0x00,0x00,0x41,0x00,0x05,0x00, -0xff,0x00,0x00,0x00,0xbf,0x01,0x00,0x00,0xea,0x00,0x00,0x00, -0xbe,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0xba,0x00,0x00,0x00, -0xc0,0x01,0x00,0x00,0xbf,0x01,0x00,0x00,0x41,0x00,0x05,0x00, -0xc3,0x00,0x00,0x00,0xc1,0x01,0x00,0x00,0xac,0x01,0x00,0x00, -0xb0,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0xc1,0x01,0x00,0x00, -0xc0,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xc3,0x01,0x00,0x00,0xf2,0x02,0x00,0x00,0xc6,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xa2,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xa4,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x9d,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x9d,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xc5,0x01,0x00,0x00,0xe0,0x02,0x00,0x00, -0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x9a,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x9c,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0xc7,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xc7,0x01,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xe1,0x02,0x00,0x00, -0x3e,0x00,0x00,0x00,0x9c,0x01,0x00,0x00,0xf3,0x01,0x00,0x00, -0xca,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00, -0xcd,0x01,0x00,0x00,0xe1,0x02,0x00,0x00,0xb5,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0xc9,0x01,0x00,0x00,0xca,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xcd,0x01,0x00,0x00, -0xc8,0x01,0x00,0x00,0xc9,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xc8,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xcf,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xcf,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xef,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, -0xc8,0x01,0x00,0x00,0xf1,0x01,0x00,0x00,0xd0,0x01,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00,0xd5,0x01,0x00,0x00, -0xef,0x02,0x00,0x00,0xb2,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0xd1,0x01,0x00,0x00,0xd0,0x01,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xd5,0x01,0x00,0x00,0xd0,0x01,0x00,0x00, -0xd1,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xd0,0x01,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xdb,0x01,0x00,0x00, -0xe1,0x02,0x00,0x00,0xb2,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xdd,0x01,0x00,0x00,0xdb,0x01,0x00,0x00, -0xef,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xdf,0x01,0x00,0x00,0x59,0x00,0x00,0x00,0xaf,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xe2,0x01,0x00,0x00, -0xe1,0x02,0x00,0x00,0xe1,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xe3,0x01,0x00,0x00,0xdf,0x01,0x00,0x00, -0xe2,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xe5,0x01,0x00,0x00,0x68,0x00,0x00,0x00,0xb2,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xe6,0x01,0x00,0x00, -0xe3,0x01,0x00,0x00,0xe5,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xe8,0x01,0x00,0x00,0xe6,0x01,0x00,0x00, -0xef,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xea,0x01,0x00,0x00,0xe8,0x01,0x00,0x00,0xe9,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xec,0x01,0x00,0x00, -0xea,0x01,0x00,0x00,0xdc,0x02,0x00,0x00,0x41,0x00,0x05,0x00, -0xff,0x00,0x00,0x00,0xed,0x01,0x00,0x00,0x4b,0x01,0x00,0x00, -0xec,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0xba,0x00,0x00,0x00, -0xee,0x01,0x00,0x00,0xed,0x01,0x00,0x00,0x41,0x00,0x05,0x00, -0xc3,0x00,0x00,0x00,0xef,0x01,0x00,0x00,0xd9,0x01,0x00,0x00, -0xdd,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0xef,0x01,0x00,0x00, -0xee,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xf1,0x01,0x00,0x00,0xef,0x02,0x00,0x00,0xc6,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xcf,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xd1,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xca,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xca,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xf3,0x01,0x00,0x00,0xe1,0x02,0x00,0x00, -0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xc7,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xc9,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0xf5,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xf5,0x01,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xe2,0x02,0x00,0x00, -0x3e,0x00,0x00,0x00,0xc9,0x01,0x00,0x00,0x37,0x02,0x00,0x00, -0xf8,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00, -0xfb,0x01,0x00,0x00,0xe2,0x02,0x00,0x00,0xb5,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0xf7,0x01,0x00,0x00,0xf8,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xfb,0x01,0x00,0x00, -0xf6,0x01,0x00,0x00,0xf7,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xf6,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xfd,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xfd,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xe6,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, -0xf6,0x01,0x00,0x00,0x35,0x02,0x00,0x00,0x00,0x02,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00,0x03,0x02,0x00,0x00, -0xe6,0x02,0x00,0x00,0x60,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0xff,0x01,0x00,0x00,0x00,0x02,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x03,0x02,0x00,0x00,0xfe,0x01,0x00,0x00, -0xff,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xfe,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0x05,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x05,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xe8,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0xfe,0x01,0x00,0x00, -0x33,0x02,0x00,0x00,0x08,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb8,0x00,0x00,0x00,0x0b,0x02,0x00,0x00,0xe8,0x02,0x00,0x00, -0xb2,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x07,0x02,0x00,0x00, -0x08,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x0b,0x02,0x00,0x00,0x06,0x02,0x00,0x00,0x07,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x06,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x0d,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x0d,0x02,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xea,0x02,0x00,0x00, -0x3e,0x00,0x00,0x00,0x06,0x02,0x00,0x00,0x31,0x02,0x00,0x00, -0x0e,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00, -0x13,0x02,0x00,0x00,0xea,0x02,0x00,0x00,0x62,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0x0f,0x02,0x00,0x00,0x0e,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x13,0x02,0x00,0x00, -0x0e,0x02,0x00,0x00,0x0f,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x0e,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x15,0x02,0x00,0x00,0xe2,0x02,0x00,0x00,0xb2,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x17,0x02,0x00,0x00, -0x15,0x02,0x00,0x00,0xe8,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x19,0x02,0x00,0x00,0x17,0x02,0x00,0x00, -0x18,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x1b,0x02,0x00,0x00,0xe6,0x02,0x00,0x00,0x62,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x1c,0x02,0x00,0x00, -0x19,0x02,0x00,0x00,0x1b,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x1e,0x02,0x00,0x00,0x1c,0x02,0x00,0x00, -0xea,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x22,0x02,0x00,0x00,0x1b,0x02,0x00,0x00,0xea,0x02,0x00,0x00, -0x41,0x00,0x05,0x00,0xc3,0x00,0x00,0x00,0x23,0x02,0x00,0x00, -0xac,0x01,0x00,0x00,0x22,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, -0xba,0x00,0x00,0x00,0x24,0x02,0x00,0x00,0x23,0x02,0x00,0x00, -0x41,0x00,0x05,0x00,0xc3,0x00,0x00,0x00,0x29,0x02,0x00,0x00, -0xd9,0x01,0x00,0x00,0x17,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, -0xba,0x00,0x00,0x00,0x2a,0x02,0x00,0x00,0x29,0x02,0x00,0x00, -0x41,0x00,0x05,0x00,0xc3,0x00,0x00,0x00,0x2c,0x02,0x00,0x00, -0xc0,0x00,0x00,0x00,0x1e,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, -0xba,0x00,0x00,0x00,0x2d,0x02,0x00,0x00,0x2c,0x02,0x00,0x00, -0x0c,0x00,0x08,0x00,0xba,0x00,0x00,0x00,0x2e,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x24,0x02,0x00,0x00, -0x2a,0x02,0x00,0x00,0x2d,0x02,0x00,0x00,0x3e,0x00,0x03,0x00, -0x2c,0x02,0x00,0x00,0x2e,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x31,0x02,0x00,0x00,0xea,0x02,0x00,0x00, -0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x0d,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x0f,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x08,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x08,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x33,0x02,0x00,0x00, -0xe8,0x02,0x00,0x00,0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x05,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x07,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0x00,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x00,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x35,0x02,0x00,0x00,0xe6,0x02,0x00,0x00,0xc6,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xfd,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xff,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xf8,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xf8,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x37,0x02,0x00,0x00,0xe2,0x02,0x00,0x00, -0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xf5,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xf7,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0x95,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x95,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x39,0x02,0x00,0x00, -0xdc,0x02,0x00,0x00,0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x92,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x94,0x01,0x00,0x00, -0xe0,0x00,0x04,0x00,0x0c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x8a,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xcd,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xcd,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x3b,0x02,0x00,0x00,0xc2,0x02,0x00,0x00, -0x6c,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xca,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xcc,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x40,0x02,0x00,0x00,0x55,0x00,0x00,0x00, -0x53,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x41,0x02,0x00,0x00,0x8c,0x00,0x00,0x00,0x40,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x46,0x02,0x00,0x00, -0x59,0x00,0x00,0x00,0xaf,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x47,0x02,0x00,0x00,0x9e,0x00,0x00,0x00, -0x46,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0x4c,0x02,0x00,0x00,0x12,0x00,0x00,0x00,0x4b,0x02,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x4d,0x02,0x00,0x00, -0x4c,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x4e,0x02,0x00,0x00,0x0f,0x00,0x00,0x00,0x4d,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x52,0x02,0x00,0x00, -0x47,0x00,0x00,0x00,0x4d,0x02,0x00,0x00,0x41,0x00,0x05,0x00, -0x0d,0x00,0x00,0x00,0x54,0x02,0x00,0x00,0x53,0x02,0x00,0x00, -0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x55,0x02,0x00,0x00,0x54,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x56,0x02,0x00,0x00,0x52,0x02,0x00,0x00, -0x55,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x57,0x02,0x00,0x00,0x4e,0x02,0x00,0x00,0x56,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0x59,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x59,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xc3,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0xcc,0x00,0x00,0x00, -0xc0,0x02,0x00,0x00,0x5c,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb8,0x00,0x00,0x00,0x5f,0x02,0x00,0x00,0xc3,0x02,0x00,0x00, -0xb5,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x5b,0x02,0x00,0x00, -0x5c,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x5f,0x02,0x00,0x00,0x5a,0x02,0x00,0x00,0x5b,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x5a,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x61,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x61,0x02,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xc4,0x02,0x00,0x00, -0x3e,0x00,0x00,0x00,0x5a,0x02,0x00,0x00,0xbe,0x02,0x00,0x00, -0x64,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00, -0x67,0x02,0x00,0x00,0xc4,0x02,0x00,0x00,0x60,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0x63,0x02,0x00,0x00,0x64,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x67,0x02,0x00,0x00, -0x62,0x02,0x00,0x00,0x63,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x62,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x6b,0x02,0x00,0x00,0xc4,0x02,0x00,0x00,0x61,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x6c,0x02,0x00,0x00, -0x41,0x02,0x00,0x00,0x6b,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x6e,0x02,0x00,0x00,0x64,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x6f,0x02,0x00,0x00,0x6c,0x02,0x00,0x00,0x6e,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x73,0x02,0x00,0x00, -0xc3,0x02,0x00,0x00,0xe1,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x74,0x02,0x00,0x00,0x47,0x02,0x00,0x00, -0x73,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x76,0x02,0x00,0x00,0x68,0x00,0x00,0x00,0xb2,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x77,0x02,0x00,0x00, -0x74,0x02,0x00,0x00,0x76,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x79,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x79,0x02,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xc6,0x02,0x00,0x00, -0x3e,0x00,0x00,0x00,0x62,0x02,0x00,0x00,0xbc,0x02,0x00,0x00, -0x7c,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00, -0x7f,0x02,0x00,0x00,0xc6,0x02,0x00,0x00,0xb2,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0x7b,0x02,0x00,0x00,0x7c,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x7f,0x02,0x00,0x00, -0x7a,0x02,0x00,0x00,0x7b,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x7a,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x81,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x81,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xc8,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, -0x7a,0x02,0x00,0x00,0xba,0x02,0x00,0x00,0x84,0x02,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00,0x87,0x02,0x00,0x00, -0xc8,0x02,0x00,0x00,0x62,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0x83,0x02,0x00,0x00,0x84,0x02,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x87,0x02,0x00,0x00,0x82,0x02,0x00,0x00, -0x83,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x82,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8a,0x02,0x00,0x00, -0x6f,0x02,0x00,0x00,0xc8,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb8,0x00,0x00,0x00,0x8d,0x02,0x00,0x00,0x8a,0x02,0x00,0x00, -0x36,0x00,0x00,0x00,0xf7,0x00,0x03,0x00,0x8f,0x02,0x00,0x00, -0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x8d,0x02,0x00,0x00, -0x8e,0x02,0x00,0x00,0x8f,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x8e,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x92,0x02,0x00,0x00,0x77,0x02,0x00,0x00,0xc6,0x02,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x93,0x02,0x00,0x00, -0x12,0x00,0x00,0x00,0xc6,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x94,0x02,0x00,0x00,0x93,0x02,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00,0x95,0x02,0x00,0x00, -0x92,0x02,0x00,0x00,0x94,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x8f,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x8f,0x02,0x00,0x00, -0xf5,0x00,0x07,0x00,0xb8,0x00,0x00,0x00,0x96,0x02,0x00,0x00, -0x8d,0x02,0x00,0x00,0x82,0x02,0x00,0x00,0x95,0x02,0x00,0x00, -0x8e,0x02,0x00,0x00,0xf7,0x00,0x03,0x00,0x98,0x02,0x00,0x00, -0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x96,0x02,0x00,0x00, -0x97,0x02,0x00,0x00,0x98,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x97,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xa0,0x02,0x00,0x00,0x77,0x02,0x00,0x00,0xc6,0x02,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0xa2,0x02,0x00,0x00, -0x12,0x00,0x00,0x00,0xa1,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xa3,0x02,0x00,0x00,0xa2,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa4,0x02,0x00,0x00, -0xa0,0x02,0x00,0x00,0xa3,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xa5,0x02,0x00,0x00,0x57,0x02,0x00,0x00, -0xa4,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xa7,0x02,0x00,0x00,0xa5,0x02,0x00,0x00,0x6f,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa9,0x02,0x00,0x00, -0xa7,0x02,0x00,0x00,0xc8,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xab,0x02,0x00,0x00,0xc3,0x02,0x00,0x00, -0xb2,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xad,0x02,0x00,0x00,0xab,0x02,0x00,0x00,0xc6,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xaf,0x02,0x00,0x00, -0xad,0x02,0x00,0x00,0xae,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xb1,0x02,0x00,0x00,0xc4,0x02,0x00,0x00, -0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xb2,0x02,0x00,0x00,0xaf,0x02,0x00,0x00,0xb1,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xb4,0x02,0x00,0x00, -0xb2,0x02,0x00,0x00,0xc8,0x02,0x00,0x00,0x41,0x00,0x05,0x00, -0xc3,0x00,0x00,0x00,0xb5,0x02,0x00,0x00,0xc0,0x00,0x00,0x00, -0xb4,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0xba,0x00,0x00,0x00, -0xb6,0x02,0x00,0x00,0xb5,0x02,0x00,0x00,0x41,0x00,0x06,0x00, -0xb7,0x02,0x00,0x00,0xb8,0x02,0x00,0x00,0x9c,0x02,0x00,0x00, -0x34,0x00,0x00,0x00,0xa9,0x02,0x00,0x00,0x3e,0x00,0x03,0x00, -0xb8,0x02,0x00,0x00,0xb6,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x98,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x98,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0x84,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x84,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xba,0x02,0x00,0x00,0xc8,0x02,0x00,0x00,0xc6,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x81,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x83,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x7c,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x7c,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xbc,0x02,0x00,0x00,0xc6,0x02,0x00,0x00, -0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x79,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x7b,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x64,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x64,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xbe,0x02,0x00,0x00, -0xc4,0x02,0x00,0x00,0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x61,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x63,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0x5c,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x5c,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xc0,0x02,0x00,0x00,0xc3,0x02,0x00,0x00,0xc6,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x59,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x5b,0x02,0x00,0x00,0xfd,0x00,0x01,0x00,0x38,0x00,0x01,0x00, - -}; -const uint64_t matmul_f16_aligned_l_fp32_len = 10512; - -unsigned char matmul_f16_aligned_m_data[] = { -0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, -0x5b,0x03,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, -0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x09,0x00,0x00,0x00, -0x11,0x00,0x02,0x00,0x51,0x11,0x00,0x00,0x0b,0x00,0x06,0x00, -0x01,0x00,0x00,0x00,0x47,0x4c,0x53,0x4c,0x2e,0x73,0x74,0x64, -0x2e,0x34,0x35,0x30,0x00,0x00,0x00,0x00,0x0e,0x00,0x03,0x00, -0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x0f,0x00,0x0f,0x00, -0x05,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x6d,0x61,0x69,0x6e, -0x00,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x3d,0x00,0x00,0x00,0x4c,0x00,0x00,0x00,0xeb,0x00,0x00,0x00, -0xfa,0x00,0x00,0x00,0x80,0x01,0x00,0x00,0x8d,0x01,0x00,0x00, -0xbb,0x02,0x00,0x00,0x04,0x03,0x00,0x00,0x10,0x00,0x06,0x00, -0x04,0x00,0x00,0x00,0x11,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x0b,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x1c,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x10,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x08,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x03,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x10,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x10,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x05,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x14,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x10,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x1c,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x08,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x24,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x10,0x00,0x00,0x00,0x0a,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x28,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x0b,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x2c,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x30,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x10,0x00,0x00,0x00,0x0d,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x34,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x10,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x37,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x3d,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x1a,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x4c,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x1b,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x4f,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x53,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x60,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x62,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x6c,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x03,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x9d,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xaf,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0xb2,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x08,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0xf7,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x10,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0xf8,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0x48,0x00,0x04,0x00, -0xf8,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0xf8,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0xf8,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x07,0x00,0x00,0x00, -0x08,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0xf8,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xfa,0x00,0x00,0x00, -0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0xfa,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x61,0x01,0x00,0x00,0x01,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x62,0x01,0x00,0x00, -0x0b,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x8a,0x01,0x00,0x00,0x06,0x00,0x00,0x00,0x10,0x00,0x00,0x00, -0x48,0x00,0x04,0x00,0x8b,0x01,0x00,0x00,0x00,0x00,0x00,0x00, -0x05,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0x8b,0x01,0x00,0x00, -0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x8b,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x8b,0x01,0x00,0x00, -0x00,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0x08,0x00,0x00,0x00, -0x47,0x00,0x03,0x00,0x8b,0x01,0x00,0x00,0x02,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x8d,0x01,0x00,0x00,0x22,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x8d,0x01,0x00,0x00, +0x47,0x00,0x03,0x00,0x4f,0x01,0x00,0x00,0x02,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x51,0x01,0x00,0x00,0x22,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x51,0x01,0x00,0x00, 0x21,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0xbb,0x02,0x00,0x00,0x0b,0x00,0x00,0x00,0x18,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x01,0x03,0x00,0x00,0x06,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0x02,0x03,0x00,0x00, +0x39,0x02,0x00,0x00,0x0b,0x00,0x00,0x00,0x18,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x7f,0x02,0x00,0x00,0x06,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0x80,0x02,0x00,0x00, 0x00,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x02,0x03,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x02,0x03,0x00,0x00, -0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x04,0x03,0x00,0x00, +0x80,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x80,0x02,0x00,0x00, +0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x82,0x02,0x00,0x00, 0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x04,0x03,0x00,0x00,0x21,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x82,0x02,0x00,0x00,0x21,0x00,0x00,0x00,0x02,0x00,0x00,0x00, 0x13,0x00,0x02,0x00,0x02,0x00,0x00,0x00,0x21,0x00,0x03,0x00, 0x03,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x15,0x00,0x04,0x00, 0x06,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x00,0x00,0x00,0x00, @@ -12680,3939 +9452,153 @@ unsigned char matmul_f16_aligned_m_data[] = { 0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x67,0x00,0x00,0x00, 0x86,0x00,0x00,0x00,0x61,0x00,0x00,0x00,0x62,0x00,0x00,0x00, 0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x10,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x6d,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x6e,0x00,0x00,0x00,0x86,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x73,0x00,0x00,0x00,0x86,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0x77,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x7c,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, -0x87,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0x8d,0x00,0x00,0x00,0x03,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x98,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x9d,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0x9f,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xae,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00, -0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xaf,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xb0,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x53,0x00,0x00,0x00, -0xaf,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xb1,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x4f,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xb2,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xb3,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0xb1,0x00,0x00,0x00,0xb2,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xb4,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0xb3,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xb5,0x00,0x00,0x00,0x86,0x00,0x00,0x00, -0xb0,0x00,0x00,0x00,0xb4,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xb6,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0xae,0x00,0x00,0x00,0xb5,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xb7,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0xb6,0x00,0x00,0x00,0xb2,0x00,0x00,0x00,0x14,0x00,0x02,0x00, -0xb8,0x00,0x00,0x00,0x16,0x00,0x03,0x00,0xba,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xbb,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xbc,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0xbb,0x00,0x00,0x00, -0xb5,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xbd,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0xbc,0x00,0x00,0x00, -0xb2,0x00,0x00,0x00,0x1c,0x00,0x04,0x00,0xbe,0x00,0x00,0x00, -0xba,0x00,0x00,0x00,0xbd,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0xbf,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0xbe,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0xba,0x00,0x00,0x00,0xc2,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0xc3,0x00,0x00,0x00, -0x07,0x00,0x00,0x00,0xba,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0xc6,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x16,0x00,0x03,0x00,0xe6,0x00,0x00,0x00,0x10,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xe7,0x00,0x00,0x00, -0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xe8,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0xe7,0x00,0x00,0x00, -0x1c,0x00,0x04,0x00,0xe9,0x00,0x00,0x00,0xe6,0x00,0x00,0x00, -0xe8,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0xea,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0xe9,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0xea,0x00,0x00,0x00,0xeb,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xef,0x00,0x00,0x00, -0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x17,0x00,0x04,0x00,0xf5,0x00,0x00,0x00,0xe6,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x18,0x00,0x04,0x00,0xf6,0x00,0x00,0x00, -0xf5,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, -0xf7,0x00,0x00,0x00,0xf6,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, -0xf8,0x00,0x00,0x00,0xf7,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0xf9,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0xf8,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0xf9,0x00,0x00,0x00,0xfa,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0xfc,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0xe6,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0xff,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0xe6,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x04,0x01,0x00,0x00, -0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x11,0x01,0x00,0x00, -0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x1e,0x01,0x00,0x00, -0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x23,0x01,0x00,0x00, -0x03,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x2c,0x01,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x31,0x01,0x00,0x00,0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x3a,0x01,0x00,0x00,0x80,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x3f,0x01,0x00,0x00,0x05,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x48,0x01,0x00,0x00, -0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x4d,0x01,0x00,0x00, -0x06,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x56,0x01,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x5b,0x01,0x00,0x00,0x07,0x00,0x00,0x00,0x32,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x61,0x01,0x00,0x00,0x01,0x00,0x00,0x00, -0x33,0x00,0x06,0x00,0x09,0x00,0x00,0x00,0x62,0x01,0x00,0x00, -0x61,0x01,0x00,0x00,0x39,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x63,0x01,0x00,0x00, -0x51,0x00,0x00,0x00,0x62,0x01,0x00,0x00,0x00,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x64,0x01,0x00,0x00, -0x84,0x00,0x00,0x00,0x63,0x01,0x00,0x00,0x6d,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x65,0x01,0x00,0x00, -0x86,0x00,0x00,0x00,0x64,0x01,0x00,0x00,0x6c,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x7c,0x01,0x00,0x00, -0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x7d,0x01,0x00,0x00, -0x84,0x00,0x00,0x00,0x9d,0x00,0x00,0x00,0x7c,0x01,0x00,0x00, -0x1c,0x00,0x04,0x00,0x7e,0x01,0x00,0x00,0xe6,0x00,0x00,0x00, -0x7d,0x01,0x00,0x00,0x20,0x00,0x04,0x00,0x7f,0x01,0x00,0x00, -0x04,0x00,0x00,0x00,0x7e,0x01,0x00,0x00,0x3b,0x00,0x04,0x00, -0x7f,0x01,0x00,0x00,0x80,0x01,0x00,0x00,0x04,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x84,0x01,0x00,0x00, -0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x1d,0x00,0x03,0x00,0x8a,0x01,0x00,0x00,0xf6,0x00,0x00,0x00, -0x1e,0x00,0x03,0x00,0x8b,0x01,0x00,0x00,0x8a,0x01,0x00,0x00, -0x20,0x00,0x04,0x00,0x8c,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, -0x8b,0x01,0x00,0x00,0x3b,0x00,0x04,0x00,0x8c,0x01,0x00,0x00, -0x8d,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x95,0x01,0x00,0x00,0x80,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xa2,0x01,0x00,0x00,0x80,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xaf,0x01,0x00,0x00,0x80,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xbc,0x01,0x00,0x00,0x80,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xc9,0x01,0x00,0x00,0x80,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xd6,0x01,0x00,0x00,0x80,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xe3,0x01,0x00,0x00,0x80,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xef,0x01,0x00,0x00,0x08,0x01,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xf0,0x01,0x00,0x00, -0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xf3,0x01,0x00,0x00, -0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x0e,0x02,0x00,0x00, -0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00, -0x1c,0x00,0x04,0x00,0x0f,0x02,0x00,0x00,0xe6,0x00,0x00,0x00, -0x0e,0x02,0x00,0x00,0x20,0x00,0x04,0x00,0x10,0x02,0x00,0x00, -0x07,0x00,0x00,0x00,0x0f,0x02,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x20,0x02,0x00,0x00,0x80,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x26,0x02,0x00,0x00,0x07,0x00,0x00,0x00,0xe6,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x3c,0x02,0x00,0x00, -0x84,0x00,0x00,0x00,0xb5,0x00,0x00,0x00,0xb2,0x00,0x00,0x00, -0x1c,0x00,0x04,0x00,0x3d,0x02,0x00,0x00,0xe6,0x00,0x00,0x00, -0x3c,0x02,0x00,0x00,0x20,0x00,0x04,0x00,0x3e,0x02,0x00,0x00, -0x07,0x00,0x00,0x00,0x3d,0x02,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x47,0x02,0x00,0x00,0x86,0x00,0x00,0x00, -0xaf,0x00,0x00,0x00,0xb5,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x4f,0x02,0x00,0x00,0x80,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x7e,0x02,0x00,0x00,0x84,0x00,0x00,0x00, -0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0xb3,0x02,0x00,0x00,0x0d,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00,0xbb,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0x1d,0x00,0x03,0x00,0x01,0x03,0x00,0x00, -0xba,0x00,0x00,0x00,0x1e,0x00,0x03,0x00,0x02,0x03,0x00,0x00, -0x01,0x03,0x00,0x00,0x20,0x00,0x04,0x00,0x03,0x03,0x00,0x00, -0x0c,0x00,0x00,0x00,0x02,0x03,0x00,0x00,0x3b,0x00,0x04,0x00, -0x03,0x03,0x00,0x00,0x04,0x03,0x00,0x00,0x0c,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x09,0x03,0x00,0x00, -0x05,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x16,0x03,0x00,0x00,0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x1f,0x03,0x00,0x00, -0x0c,0x00,0x00,0x00,0xba,0x00,0x00,0x00,0x36,0x00,0x05,0x00, -0x02,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x03,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x05,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0xbf,0x00,0x00,0x00,0xc0,0x00,0x00,0x00, -0x07,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x10,0x02,0x00,0x00, -0x11,0x02,0x00,0x00,0x07,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x3e,0x02,0x00,0x00,0x3f,0x02,0x00,0x00,0x07,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00,0x0e,0x00,0x00,0x00, -0x0b,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x0e,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x17,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x18,0x00,0x00,0x00, -0x0f,0x00,0x00,0x00,0x17,0x00,0x00,0x00,0x89,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x1e,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, -0x17,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0x22,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x22,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x24,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x28,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x27,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x29,0x00,0x00,0x00,0x28,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x2a,0x00,0x00,0x00, -0x1e,0x00,0x00,0x00,0x29,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x2d,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x2f,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x30,0x00,0x00,0x00,0x24,0x00,0x00,0x00, -0x2f,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x32,0x00,0x00,0x00,0x30,0x00,0x00,0x00,0x2a,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x35,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x36,0x00,0x00,0x00,0x35,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x38,0x00,0x00,0x00, -0x36,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x82,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x38,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x3b,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x37,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00,0x3f,0x00,0x00,0x00, -0x3d,0x00,0x00,0x00,0x3e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x3f,0x00,0x00,0x00, -0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x42,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0x3b,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x47,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x3b,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00, -0x49,0x00,0x00,0x00,0x3d,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x4a,0x00,0x00,0x00, -0x49,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00, -0x4d,0x00,0x00,0x00,0x4c,0x00,0x00,0x00,0x3e,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, -0x4d,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x50,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x4f,0x00,0x00,0x00, -0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x55,0x00,0x00,0x00, -0x50,0x00,0x00,0x00,0x54,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x59,0x00,0x00,0x00,0x50,0x00,0x00,0x00, -0x58,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x5d,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x4f,0x00,0x00,0x00, -0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x64,0x00,0x00,0x00, -0x5d,0x00,0x00,0x00,0x63,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x68,0x00,0x00,0x00,0x5d,0x00,0x00,0x00, -0x67,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x6f,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x6e,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x74,0x00,0x00,0x00, -0x4e,0x00,0x00,0x00,0x73,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x78,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x77,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x79,0x00,0x00,0x00,0x78,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x7a,0x00,0x00,0x00,0x47,0x00,0x00,0x00, -0x79,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0x7d,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x7c,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x7e,0x00,0x00,0x00, -0x7d,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x80,0x00,0x00,0x00,0x47,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x83,0x00,0x00,0x00, -0x80,0x00,0x00,0x00,0x79,0x00,0x00,0x00,0x0c,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x26,0x00,0x00,0x00,0x7e,0x00,0x00,0x00,0x83,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x88,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x87,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x89,0x00,0x00,0x00,0x88,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8a,0x00,0x00,0x00, -0x32,0x00,0x00,0x00,0x89,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x8c,0x00,0x00,0x00,0x42,0x00,0x00,0x00, -0x37,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0x8e,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x8d,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x8f,0x00,0x00,0x00, -0x8e,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x90,0x00,0x00,0x00,0x8c,0x00,0x00,0x00,0x8f,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x91,0x00,0x00,0x00, -0x8a,0x00,0x00,0x00,0x90,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x93,0x00,0x00,0x00,0x91,0x00,0x00,0x00, -0x7a,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x94,0x00,0x00,0x00,0x93,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x99,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x98,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x9a,0x00,0x00,0x00,0x99,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9b,0x00,0x00,0x00, -0x0f,0x00,0x00,0x00,0x9a,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x9e,0x00,0x00,0x00,0x4a,0x00,0x00,0x00, -0x9d,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0xa0,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x9f,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xa1,0x00,0x00,0x00, -0xa0,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xa2,0x00,0x00,0x00,0x9e,0x00,0x00,0x00,0xa1,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa3,0x00,0x00,0x00, -0x9b,0x00,0x00,0x00,0xa2,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xa5,0x00,0x00,0x00,0xa3,0x00,0x00,0x00, -0x7a,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xa6,0x00,0x00,0x00,0xa5,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xa8,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xa8,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0x29,0x03,0x00,0x00,0x3e,0x00,0x00,0x00,0x05,0x00,0x00,0x00, -0xc7,0x00,0x00,0x00,0xa9,0x00,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb8,0x00,0x00,0x00,0xb9,0x00,0x00,0x00,0x29,0x03,0x00,0x00, -0xb7,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xaa,0x00,0x00,0x00, -0xa9,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0xb9,0x00,0x00,0x00,0xa9,0x00,0x00,0x00,0xaa,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xa9,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0xc3,0x00,0x00,0x00,0xc4,0x00,0x00,0x00,0xc0,0x00,0x00,0x00, -0x29,0x03,0x00,0x00,0x3e,0x00,0x03,0x00,0xc4,0x00,0x00,0x00, -0xc2,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xc7,0x00,0x00,0x00,0x29,0x03,0x00,0x00,0xc6,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xa8,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xaa,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xca,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xca,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0x42,0x03,0x00,0x00,0xa6,0x00,0x00,0x00, -0xaa,0x00,0x00,0x00,0xf5,0x01,0x00,0x00,0xcd,0x00,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x3e,0x03,0x00,0x00, -0x94,0x00,0x00,0x00,0xaa,0x00,0x00,0x00,0xf2,0x01,0x00,0x00, -0xcd,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0x2a,0x03,0x00,0x00,0x7a,0x00,0x00,0x00,0xaa,0x00,0x00,0x00, -0xa3,0x02,0x00,0x00,0xcd,0x00,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb8,0x00,0x00,0x00,0xd1,0x00,0x00,0x00,0x2a,0x03,0x00,0x00, -0x84,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xcc,0x00,0x00,0x00, -0xcd,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0xd1,0x00,0x00,0x00,0xcb,0x00,0x00,0x00,0xcc,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xcb,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xd3,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xd3,0x00,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x3a,0x03,0x00,0x00, -0x3e,0x00,0x00,0x00,0xcb,0x00,0x00,0x00,0x67,0x01,0x00,0x00, -0xd4,0x00,0x00,0x00,0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00, -0xd9,0x00,0x00,0x00,0x3a,0x03,0x00,0x00,0x37,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0xd5,0x00,0x00,0x00,0xd4,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xd9,0x00,0x00,0x00, -0xd4,0x00,0x00,0x00,0xd5,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xd4,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xde,0x00,0x00,0x00,0x74,0x00,0x00,0x00,0x3a,0x03,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xe1,0x00,0x00,0x00, -0xde,0x00,0x00,0x00,0x8f,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xe2,0x00,0x00,0x00,0xe1,0x00,0x00,0x00, -0x6d,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xe3,0x00,0x00,0x00,0x3e,0x03,0x00,0x00,0xe2,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xe5,0x00,0x00,0x00, -0xe3,0x00,0x00,0x00,0x6f,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xf0,0x00,0x00,0x00,0xde,0x00,0x00,0x00, -0xef,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xf2,0x00,0x00,0x00,0x6f,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xf3,0x00,0x00,0x00, -0xf0,0x00,0x00,0x00,0xf2,0x00,0x00,0x00,0x41,0x00,0x08,0x00, -0xfc,0x00,0x00,0x00,0xfd,0x00,0x00,0x00,0xfa,0x00,0x00,0x00, -0x34,0x00,0x00,0x00,0xe5,0x00,0x00,0x00,0x34,0x00,0x00,0x00, -0x3e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0xe6,0x00,0x00,0x00, -0xfe,0x00,0x00,0x00,0xfd,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0xff,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0xeb,0x00,0x00,0x00, -0xf3,0x00,0x00,0x00,0x3e,0x00,0x03,0x00,0x00,0x01,0x00,0x00, -0xfe,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x05,0x01,0x00,0x00,0xde,0x00,0x00,0x00,0x04,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x08,0x01,0x00,0x00, -0x05,0x01,0x00,0x00,0xf2,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x09,0x01,0x00,0x00,0x08,0x01,0x00,0x00, -0x39,0x00,0x00,0x00,0x41,0x00,0x08,0x00,0xfc,0x00,0x00,0x00, -0x0b,0x01,0x00,0x00,0xfa,0x00,0x00,0x00,0x34,0x00,0x00,0x00, -0xe5,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0xe6,0x00,0x00,0x00,0x0c,0x01,0x00,0x00, -0x0b,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0xff,0x00,0x00,0x00, -0x0d,0x01,0x00,0x00,0xeb,0x00,0x00,0x00,0x09,0x01,0x00,0x00, -0x3e,0x00,0x03,0x00,0x0d,0x01,0x00,0x00,0x0c,0x01,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x12,0x01,0x00,0x00, -0xde,0x00,0x00,0x00,0x11,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x15,0x01,0x00,0x00,0x12,0x01,0x00,0x00, -0xf2,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x16,0x01,0x00,0x00,0x15,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, -0x41,0x00,0x08,0x00,0xfc,0x00,0x00,0x00,0x18,0x01,0x00,0x00, -0xfa,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0xe5,0x00,0x00,0x00, -0x34,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0xe6,0x00,0x00,0x00,0x19,0x01,0x00,0x00,0x18,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0xff,0x00,0x00,0x00,0x1a,0x01,0x00,0x00, -0xeb,0x00,0x00,0x00,0x16,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0x1a,0x01,0x00,0x00,0x19,0x01,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x1f,0x01,0x00,0x00,0xde,0x00,0x00,0x00, -0x1e,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x22,0x01,0x00,0x00,0x1f,0x01,0x00,0x00,0xf2,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x24,0x01,0x00,0x00, -0x22,0x01,0x00,0x00,0x23,0x01,0x00,0x00,0x41,0x00,0x08,0x00, -0xfc,0x00,0x00,0x00,0x26,0x01,0x00,0x00,0xfa,0x00,0x00,0x00, -0x34,0x00,0x00,0x00,0xe5,0x00,0x00,0x00,0x34,0x00,0x00,0x00, -0x23,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0xe6,0x00,0x00,0x00, -0x27,0x01,0x00,0x00,0x26,0x01,0x00,0x00,0x41,0x00,0x05,0x00, -0xff,0x00,0x00,0x00,0x28,0x01,0x00,0x00,0xeb,0x00,0x00,0x00, -0x24,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x28,0x01,0x00,0x00, -0x27,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x2d,0x01,0x00,0x00,0xde,0x00,0x00,0x00,0x2c,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x30,0x01,0x00,0x00, -0x2d,0x01,0x00,0x00,0xf2,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x32,0x01,0x00,0x00,0x30,0x01,0x00,0x00, -0x31,0x01,0x00,0x00,0x41,0x00,0x08,0x00,0xfc,0x00,0x00,0x00, -0x34,0x01,0x00,0x00,0xfa,0x00,0x00,0x00,0x34,0x00,0x00,0x00, -0xe5,0x00,0x00,0x00,0xc6,0x00,0x00,0x00,0x3e,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0xe6,0x00,0x00,0x00,0x35,0x01,0x00,0x00, -0x34,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0xff,0x00,0x00,0x00, -0x36,0x01,0x00,0x00,0xeb,0x00,0x00,0x00,0x32,0x01,0x00,0x00, -0x3e,0x00,0x03,0x00,0x36,0x01,0x00,0x00,0x35,0x01,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3b,0x01,0x00,0x00, -0xde,0x00,0x00,0x00,0x3a,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x3e,0x01,0x00,0x00,0x3b,0x01,0x00,0x00, -0xf2,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x40,0x01,0x00,0x00,0x3e,0x01,0x00,0x00,0x3f,0x01,0x00,0x00, -0x41,0x00,0x08,0x00,0xfc,0x00,0x00,0x00,0x42,0x01,0x00,0x00, -0xfa,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0xe5,0x00,0x00,0x00, -0xc6,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0xe6,0x00,0x00,0x00,0x43,0x01,0x00,0x00,0x42,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0xff,0x00,0x00,0x00,0x44,0x01,0x00,0x00, -0xeb,0x00,0x00,0x00,0x40,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0x44,0x01,0x00,0x00,0x43,0x01,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x49,0x01,0x00,0x00,0xde,0x00,0x00,0x00, -0x48,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x4c,0x01,0x00,0x00,0x49,0x01,0x00,0x00,0xf2,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x4e,0x01,0x00,0x00, -0x4c,0x01,0x00,0x00,0x4d,0x01,0x00,0x00,0x41,0x00,0x08,0x00, -0xfc,0x00,0x00,0x00,0x50,0x01,0x00,0x00,0xfa,0x00,0x00,0x00, -0x34,0x00,0x00,0x00,0xe5,0x00,0x00,0x00,0xc6,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0xe6,0x00,0x00,0x00, -0x51,0x01,0x00,0x00,0x50,0x01,0x00,0x00,0x41,0x00,0x05,0x00, -0xff,0x00,0x00,0x00,0x52,0x01,0x00,0x00,0xeb,0x00,0x00,0x00, -0x4e,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x52,0x01,0x00,0x00, -0x51,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x57,0x01,0x00,0x00,0xde,0x00,0x00,0x00,0x56,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x5a,0x01,0x00,0x00, -0x57,0x01,0x00,0x00,0xf2,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x5c,0x01,0x00,0x00,0x5a,0x01,0x00,0x00, -0x5b,0x01,0x00,0x00,0x41,0x00,0x08,0x00,0xfc,0x00,0x00,0x00, -0x5e,0x01,0x00,0x00,0xfa,0x00,0x00,0x00,0x34,0x00,0x00,0x00, -0xe5,0x00,0x00,0x00,0xc6,0x00,0x00,0x00,0x23,0x01,0x00,0x00, -0x3d,0x00,0x04,0x00,0xe6,0x00,0x00,0x00,0x5f,0x01,0x00,0x00, -0x5e,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0xff,0x00,0x00,0x00, -0x60,0x01,0x00,0x00,0xeb,0x00,0x00,0x00,0x5c,0x01,0x00,0x00, -0x3e,0x00,0x03,0x00,0x60,0x01,0x00,0x00,0x5f,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x67,0x01,0x00,0x00, -0x3a,0x03,0x00,0x00,0x65,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0xd3,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xd5,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x69,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x69,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0x3b,0x03,0x00,0x00,0x3e,0x00,0x00,0x00,0xd5,0x00,0x00,0x00, -0xee,0x01,0x00,0x00,0x6a,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb8,0x00,0x00,0x00,0x6f,0x01,0x00,0x00,0x3b,0x03,0x00,0x00, -0x9d,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x6b,0x01,0x00,0x00, -0x6a,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x6f,0x01,0x00,0x00,0x6a,0x01,0x00,0x00,0x6b,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x6a,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x74,0x01,0x00,0x00,0x74,0x00,0x00,0x00, -0x3b,0x03,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x77,0x01,0x00,0x00,0x74,0x01,0x00,0x00,0xa1,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x78,0x01,0x00,0x00, -0x77,0x01,0x00,0x00,0x6d,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x79,0x01,0x00,0x00,0x42,0x03,0x00,0x00, -0x78,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x7b,0x01,0x00,0x00,0x79,0x01,0x00,0x00,0x6f,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x85,0x01,0x00,0x00, -0x74,0x01,0x00,0x00,0x84,0x01,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x87,0x01,0x00,0x00,0x6f,0x00,0x00,0x00, -0x6d,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x88,0x01,0x00,0x00,0x85,0x01,0x00,0x00,0x87,0x01,0x00,0x00, -0x41,0x00,0x08,0x00,0xfc,0x00,0x00,0x00,0x8f,0x01,0x00,0x00, -0x8d,0x01,0x00,0x00,0x34,0x00,0x00,0x00,0x7b,0x01,0x00,0x00, -0x34,0x00,0x00,0x00,0x3e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0xe6,0x00,0x00,0x00,0x90,0x01,0x00,0x00,0x8f,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0xff,0x00,0x00,0x00,0x91,0x01,0x00,0x00, -0x80,0x01,0x00,0x00,0x88,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0x91,0x01,0x00,0x00,0x90,0x01,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x96,0x01,0x00,0x00,0x74,0x01,0x00,0x00, -0x95,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x99,0x01,0x00,0x00,0x96,0x01,0x00,0x00,0x87,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9a,0x01,0x00,0x00, -0x99,0x01,0x00,0x00,0x39,0x00,0x00,0x00,0x41,0x00,0x08,0x00, -0xfc,0x00,0x00,0x00,0x9c,0x01,0x00,0x00,0x8d,0x01,0x00,0x00, -0x34,0x00,0x00,0x00,0x7b,0x01,0x00,0x00,0x34,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0xe6,0x00,0x00,0x00, -0x9d,0x01,0x00,0x00,0x9c,0x01,0x00,0x00,0x41,0x00,0x05,0x00, -0xff,0x00,0x00,0x00,0x9e,0x01,0x00,0x00,0x80,0x01,0x00,0x00, -0x9a,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x9e,0x01,0x00,0x00, -0x9d,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xa3,0x01,0x00,0x00,0x74,0x01,0x00,0x00,0xa2,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa6,0x01,0x00,0x00, -0xa3,0x01,0x00,0x00,0x87,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xa7,0x01,0x00,0x00,0xa6,0x01,0x00,0x00, -0x0c,0x00,0x00,0x00,0x41,0x00,0x08,0x00,0xfc,0x00,0x00,0x00, -0xa9,0x01,0x00,0x00,0x8d,0x01,0x00,0x00,0x34,0x00,0x00,0x00, -0x7b,0x01,0x00,0x00,0x34,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0xe6,0x00,0x00,0x00,0xaa,0x01,0x00,0x00, -0xa9,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0xff,0x00,0x00,0x00, -0xab,0x01,0x00,0x00,0x80,0x01,0x00,0x00,0xa7,0x01,0x00,0x00, -0x3e,0x00,0x03,0x00,0xab,0x01,0x00,0x00,0xaa,0x01,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xb0,0x01,0x00,0x00, -0x74,0x01,0x00,0x00,0xaf,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xb3,0x01,0x00,0x00,0xb0,0x01,0x00,0x00, -0x87,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xb4,0x01,0x00,0x00,0xb3,0x01,0x00,0x00,0x23,0x01,0x00,0x00, -0x41,0x00,0x08,0x00,0xfc,0x00,0x00,0x00,0xb6,0x01,0x00,0x00, -0x8d,0x01,0x00,0x00,0x34,0x00,0x00,0x00,0x7b,0x01,0x00,0x00, -0x34,0x00,0x00,0x00,0x23,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, -0xe6,0x00,0x00,0x00,0xb7,0x01,0x00,0x00,0xb6,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0xff,0x00,0x00,0x00,0xb8,0x01,0x00,0x00, -0x80,0x01,0x00,0x00,0xb4,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0xb8,0x01,0x00,0x00,0xb7,0x01,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xbd,0x01,0x00,0x00,0x74,0x01,0x00,0x00, -0xbc,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xc0,0x01,0x00,0x00,0xbd,0x01,0x00,0x00,0x87,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xc1,0x01,0x00,0x00, -0xc0,0x01,0x00,0x00,0x31,0x01,0x00,0x00,0x41,0x00,0x08,0x00, -0xfc,0x00,0x00,0x00,0xc3,0x01,0x00,0x00,0x8d,0x01,0x00,0x00, -0x34,0x00,0x00,0x00,0x7b,0x01,0x00,0x00,0xc6,0x00,0x00,0x00, -0x3e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0xe6,0x00,0x00,0x00, -0xc4,0x01,0x00,0x00,0xc3,0x01,0x00,0x00,0x41,0x00,0x05,0x00, -0xff,0x00,0x00,0x00,0xc5,0x01,0x00,0x00,0x80,0x01,0x00,0x00, -0xc1,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0xc5,0x01,0x00,0x00, -0xc4,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xca,0x01,0x00,0x00,0x74,0x01,0x00,0x00,0xc9,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xcd,0x01,0x00,0x00, -0xca,0x01,0x00,0x00,0x87,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xce,0x01,0x00,0x00,0xcd,0x01,0x00,0x00, -0x3f,0x01,0x00,0x00,0x41,0x00,0x08,0x00,0xfc,0x00,0x00,0x00, -0xd0,0x01,0x00,0x00,0x8d,0x01,0x00,0x00,0x34,0x00,0x00,0x00, -0x7b,0x01,0x00,0x00,0xc6,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0xe6,0x00,0x00,0x00,0xd1,0x01,0x00,0x00, -0xd0,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0xff,0x00,0x00,0x00, -0xd2,0x01,0x00,0x00,0x80,0x01,0x00,0x00,0xce,0x01,0x00,0x00, -0x3e,0x00,0x03,0x00,0xd2,0x01,0x00,0x00,0xd1,0x01,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xd7,0x01,0x00,0x00, -0x74,0x01,0x00,0x00,0xd6,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xda,0x01,0x00,0x00,0xd7,0x01,0x00,0x00, -0x87,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xdb,0x01,0x00,0x00,0xda,0x01,0x00,0x00,0x4d,0x01,0x00,0x00, -0x41,0x00,0x08,0x00,0xfc,0x00,0x00,0x00,0xdd,0x01,0x00,0x00, -0x8d,0x01,0x00,0x00,0x34,0x00,0x00,0x00,0x7b,0x01,0x00,0x00, -0xc6,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0xe6,0x00,0x00,0x00,0xde,0x01,0x00,0x00,0xdd,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0xff,0x00,0x00,0x00,0xdf,0x01,0x00,0x00, -0x80,0x01,0x00,0x00,0xdb,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0xdf,0x01,0x00,0x00,0xde,0x01,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xe4,0x01,0x00,0x00,0x74,0x01,0x00,0x00, -0xe3,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xe7,0x01,0x00,0x00,0xe4,0x01,0x00,0x00,0x87,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xe8,0x01,0x00,0x00, -0xe7,0x01,0x00,0x00,0x5b,0x01,0x00,0x00,0x41,0x00,0x08,0x00, -0xfc,0x00,0x00,0x00,0xea,0x01,0x00,0x00,0x8d,0x01,0x00,0x00, -0x34,0x00,0x00,0x00,0x7b,0x01,0x00,0x00,0xc6,0x00,0x00,0x00, -0x23,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0xe6,0x00,0x00,0x00, -0xeb,0x01,0x00,0x00,0xea,0x01,0x00,0x00,0x41,0x00,0x05,0x00, -0xff,0x00,0x00,0x00,0xec,0x01,0x00,0x00,0x80,0x01,0x00,0x00, -0xe8,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0xec,0x01,0x00,0x00, -0xeb,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xee,0x01,0x00,0x00,0x3b,0x03,0x00,0x00,0x65,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0x69,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x6b,0x01,0x00,0x00,0xe0,0x00,0x04,0x00,0x0c,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0xef,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xf2,0x01,0x00,0x00,0x3e,0x03,0x00,0x00, -0xf0,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xf5,0x01,0x00,0x00,0x42,0x03,0x00,0x00,0xf3,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0xf7,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xf7,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0x44,0x03,0x00,0x00,0x3e,0x00,0x00,0x00,0x6b,0x01,0x00,0x00, -0xa1,0x02,0x00,0x00,0xfa,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb8,0x00,0x00,0x00,0xfd,0x01,0x00,0x00,0x44,0x03,0x00,0x00, -0x6c,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xf9,0x01,0x00,0x00, -0xfa,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0xfd,0x01,0x00,0x00,0xf8,0x01,0x00,0x00,0xf9,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xf8,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0xff,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xff,0x01,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x48,0x03,0x00,0x00, -0x3e,0x00,0x00,0x00,0xf8,0x01,0x00,0x00,0x2b,0x02,0x00,0x00, -0x02,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00, -0x05,0x02,0x00,0x00,0x48,0x03,0x00,0x00,0x60,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0x01,0x02,0x00,0x00,0x02,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x05,0x02,0x00,0x00, -0x00,0x02,0x00,0x00,0x01,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x00,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x07,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x07,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0x5a,0x03,0x00,0x00,0x3e,0x00,0x00,0x00, -0x00,0x02,0x00,0x00,0x29,0x02,0x00,0x00,0x08,0x02,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00,0x0d,0x02,0x00,0x00, -0x5a,0x03,0x00,0x00,0x62,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0x09,0x02,0x00,0x00,0x08,0x02,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x0d,0x02,0x00,0x00,0x08,0x02,0x00,0x00, -0x09,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x08,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x13,0x02,0x00,0x00, -0x48,0x03,0x00,0x00,0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x15,0x02,0x00,0x00,0x13,0x02,0x00,0x00, -0x5a,0x03,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x17,0x02,0x00,0x00,0x55,0x00,0x00,0x00,0x53,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x19,0x02,0x00,0x00, -0x48,0x03,0x00,0x00,0x61,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x1a,0x02,0x00,0x00,0x17,0x02,0x00,0x00, -0x19,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x1c,0x02,0x00,0x00,0x64,0x00,0x00,0x00,0x62,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x1d,0x02,0x00,0x00, -0x1a,0x02,0x00,0x00,0x1c,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x1f,0x02,0x00,0x00,0x1d,0x02,0x00,0x00, -0x5a,0x03,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x21,0x02,0x00,0x00,0x1f,0x02,0x00,0x00,0x20,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x23,0x02,0x00,0x00, -0x21,0x02,0x00,0x00,0x44,0x03,0x00,0x00,0x41,0x00,0x05,0x00, -0xff,0x00,0x00,0x00,0x24,0x02,0x00,0x00,0xeb,0x00,0x00,0x00, -0x23,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0xe6,0x00,0x00,0x00, -0x25,0x02,0x00,0x00,0x24,0x02,0x00,0x00,0x41,0x00,0x05,0x00, -0x26,0x02,0x00,0x00,0x27,0x02,0x00,0x00,0x11,0x02,0x00,0x00, -0x15,0x02,0x00,0x00,0x3e,0x00,0x03,0x00,0x27,0x02,0x00,0x00, -0x25,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x29,0x02,0x00,0x00,0x5a,0x03,0x00,0x00,0xc6,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x07,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x09,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x02,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x02,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x2b,0x02,0x00,0x00,0x48,0x03,0x00,0x00, -0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xff,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x01,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x2d,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x2d,0x02,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x49,0x03,0x00,0x00, -0x3e,0x00,0x00,0x00,0x01,0x02,0x00,0x00,0x59,0x02,0x00,0x00, -0x30,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00, -0x33,0x02,0x00,0x00,0x49,0x03,0x00,0x00,0xb5,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0x2f,0x02,0x00,0x00,0x30,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x33,0x02,0x00,0x00, -0x2e,0x02,0x00,0x00,0x2f,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x2e,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x35,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x35,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0x57,0x03,0x00,0x00,0x3e,0x00,0x00,0x00, -0x2e,0x02,0x00,0x00,0x57,0x02,0x00,0x00,0x36,0x02,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00,0x3b,0x02,0x00,0x00, -0x57,0x03,0x00,0x00,0xb2,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0x37,0x02,0x00,0x00,0x36,0x02,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x3b,0x02,0x00,0x00,0x36,0x02,0x00,0x00, -0x37,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x36,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x41,0x02,0x00,0x00, -0x49,0x03,0x00,0x00,0xb2,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x43,0x02,0x00,0x00,0x41,0x02,0x00,0x00, -0x57,0x03,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x45,0x02,0x00,0x00,0x59,0x00,0x00,0x00,0xaf,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x48,0x02,0x00,0x00, -0x49,0x03,0x00,0x00,0x47,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x49,0x02,0x00,0x00,0x45,0x02,0x00,0x00, -0x48,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x4b,0x02,0x00,0x00,0x68,0x00,0x00,0x00,0xb2,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x4c,0x02,0x00,0x00, -0x49,0x02,0x00,0x00,0x4b,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x4e,0x02,0x00,0x00,0x4c,0x02,0x00,0x00, -0x57,0x03,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x50,0x02,0x00,0x00,0x4e,0x02,0x00,0x00,0x4f,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x52,0x02,0x00,0x00, -0x50,0x02,0x00,0x00,0x44,0x03,0x00,0x00,0x41,0x00,0x05,0x00, -0xff,0x00,0x00,0x00,0x53,0x02,0x00,0x00,0x80,0x01,0x00,0x00, -0x52,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0xe6,0x00,0x00,0x00, -0x54,0x02,0x00,0x00,0x53,0x02,0x00,0x00,0x41,0x00,0x05,0x00, -0x26,0x02,0x00,0x00,0x55,0x02,0x00,0x00,0x3f,0x02,0x00,0x00, -0x43,0x02,0x00,0x00,0x3e,0x00,0x03,0x00,0x55,0x02,0x00,0x00, -0x54,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x57,0x02,0x00,0x00,0x57,0x03,0x00,0x00,0xc6,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x35,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x37,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x30,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x30,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x59,0x02,0x00,0x00,0x49,0x03,0x00,0x00, -0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x2d,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x2f,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x5b,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x5b,0x02,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x4a,0x03,0x00,0x00, -0x3e,0x00,0x00,0x00,0x2f,0x02,0x00,0x00,0x9f,0x02,0x00,0x00, -0x5e,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00, -0x61,0x02,0x00,0x00,0x4a,0x03,0x00,0x00,0xb5,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0x5d,0x02,0x00,0x00,0x5e,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x61,0x02,0x00,0x00, -0x5c,0x02,0x00,0x00,0x5d,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x5c,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x63,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x63,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0x4e,0x03,0x00,0x00,0x3e,0x00,0x00,0x00, -0x5c,0x02,0x00,0x00,0x9d,0x02,0x00,0x00,0x66,0x02,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00,0x69,0x02,0x00,0x00, -0x4e,0x03,0x00,0x00,0x60,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0x65,0x02,0x00,0x00,0x66,0x02,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x69,0x02,0x00,0x00,0x64,0x02,0x00,0x00, -0x65,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x64,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0x6b,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x6b,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0x50,0x03,0x00,0x00,0x3e,0x00,0x00,0x00,0x64,0x02,0x00,0x00, -0x9b,0x02,0x00,0x00,0x6e,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb8,0x00,0x00,0x00,0x71,0x02,0x00,0x00,0x50,0x03,0x00,0x00, -0xb2,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x6d,0x02,0x00,0x00, -0x6e,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x71,0x02,0x00,0x00,0x6c,0x02,0x00,0x00,0x6d,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x6c,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x73,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x73,0x02,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x52,0x03,0x00,0x00, -0x3e,0x00,0x00,0x00,0x6c,0x02,0x00,0x00,0x99,0x02,0x00,0x00, -0x74,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00, -0x79,0x02,0x00,0x00,0x52,0x03,0x00,0x00,0x62,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0x75,0x02,0x00,0x00,0x74,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x79,0x02,0x00,0x00, -0x74,0x02,0x00,0x00,0x75,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x74,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x7b,0x02,0x00,0x00,0x4a,0x03,0x00,0x00,0xb2,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x7d,0x02,0x00,0x00, -0x7b,0x02,0x00,0x00,0x50,0x03,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x7f,0x02,0x00,0x00,0x7d,0x02,0x00,0x00, -0x7e,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x81,0x02,0x00,0x00,0x4e,0x03,0x00,0x00,0x62,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x82,0x02,0x00,0x00, -0x7f,0x02,0x00,0x00,0x81,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x84,0x02,0x00,0x00,0x82,0x02,0x00,0x00, -0x52,0x03,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x88,0x02,0x00,0x00,0x81,0x02,0x00,0x00,0x52,0x03,0x00,0x00, -0x41,0x00,0x05,0x00,0x26,0x02,0x00,0x00,0x89,0x02,0x00,0x00, -0x11,0x02,0x00,0x00,0x88,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, -0xe6,0x00,0x00,0x00,0x8a,0x02,0x00,0x00,0x89,0x02,0x00,0x00, -0x73,0x00,0x04,0x00,0xba,0x00,0x00,0x00,0x8b,0x02,0x00,0x00, -0x8a,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x26,0x02,0x00,0x00, -0x90,0x02,0x00,0x00,0x3f,0x02,0x00,0x00,0x7d,0x02,0x00,0x00, -0x3d,0x00,0x04,0x00,0xe6,0x00,0x00,0x00,0x91,0x02,0x00,0x00, -0x90,0x02,0x00,0x00,0x73,0x00,0x04,0x00,0xba,0x00,0x00,0x00, -0x92,0x02,0x00,0x00,0x91,0x02,0x00,0x00,0x41,0x00,0x05,0x00, -0xc3,0x00,0x00,0x00,0x94,0x02,0x00,0x00,0xc0,0x00,0x00,0x00, -0x84,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0xba,0x00,0x00,0x00, -0x95,0x02,0x00,0x00,0x94,0x02,0x00,0x00,0x0c,0x00,0x08,0x00, -0xba,0x00,0x00,0x00,0x96,0x02,0x00,0x00,0x01,0x00,0x00,0x00, -0x32,0x00,0x00,0x00,0x8b,0x02,0x00,0x00,0x92,0x02,0x00,0x00, -0x95,0x02,0x00,0x00,0x3e,0x00,0x03,0x00,0x94,0x02,0x00,0x00, -0x96,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x99,0x02,0x00,0x00,0x52,0x03,0x00,0x00,0xc6,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x73,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x75,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x6e,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x6e,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x9b,0x02,0x00,0x00,0x50,0x03,0x00,0x00, -0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x6b,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x6d,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x66,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x66,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9d,0x02,0x00,0x00, -0x4e,0x03,0x00,0x00,0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x63,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x65,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0x5e,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x5e,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x9f,0x02,0x00,0x00,0x4a,0x03,0x00,0x00,0xc6,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x5b,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x5d,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0xfa,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xfa,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xa1,0x02,0x00,0x00,0x44,0x03,0x00,0x00, -0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xf7,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xf9,0x01,0x00,0x00,0xe0,0x00,0x04,0x00, -0x0c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0xef,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0xcd,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xcd,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xa3,0x02,0x00,0x00,0x2a,0x03,0x00,0x00,0x6c,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xca,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xcc,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xa8,0x02,0x00,0x00,0x55,0x00,0x00,0x00,0x53,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa9,0x02,0x00,0x00, -0x8c,0x00,0x00,0x00,0xa8,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xae,0x02,0x00,0x00,0x59,0x00,0x00,0x00, -0xaf,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xaf,0x02,0x00,0x00,0x9e,0x00,0x00,0x00,0xae,0x02,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0xb4,0x02,0x00,0x00, -0x12,0x00,0x00,0x00,0xb3,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xb5,0x02,0x00,0x00,0xb4,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xb6,0x02,0x00,0x00, -0x0f,0x00,0x00,0x00,0xb5,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xba,0x02,0x00,0x00,0x47,0x00,0x00,0x00, -0xb5,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00, -0xbc,0x02,0x00,0x00,0xbb,0x02,0x00,0x00,0x0c,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xbd,0x02,0x00,0x00, -0xbc,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xbe,0x02,0x00,0x00,0xba,0x02,0x00,0x00,0xbd,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xbf,0x02,0x00,0x00, -0xb6,0x02,0x00,0x00,0xbe,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0xc1,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0xc1,0x02,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x2b,0x03,0x00,0x00, -0x3e,0x00,0x00,0x00,0xcc,0x00,0x00,0x00,0x28,0x03,0x00,0x00, -0xc4,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00, -0xc7,0x02,0x00,0x00,0x2b,0x03,0x00,0x00,0xb5,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0xc3,0x02,0x00,0x00,0xc4,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xc7,0x02,0x00,0x00, -0xc2,0x02,0x00,0x00,0xc3,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0xc2,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0xc9,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0xc9,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0x2c,0x03,0x00,0x00,0x3e,0x00,0x00,0x00, -0xc2,0x02,0x00,0x00,0x26,0x03,0x00,0x00,0xcc,0x02,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00,0xcf,0x02,0x00,0x00, -0x2c,0x03,0x00,0x00,0x60,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0xcb,0x02,0x00,0x00,0xcc,0x02,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xcf,0x02,0x00,0x00,0xca,0x02,0x00,0x00, -0xcb,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0xca,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xd3,0x02,0x00,0x00, -0x2c,0x03,0x00,0x00,0x61,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xd4,0x02,0x00,0x00,0xa9,0x02,0x00,0x00, -0xd3,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xd6,0x02,0x00,0x00,0x64,0x00,0x00,0x00,0x62,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xd7,0x02,0x00,0x00, -0xd4,0x02,0x00,0x00,0xd6,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xdb,0x02,0x00,0x00,0x2b,0x03,0x00,0x00, -0x47,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xdc,0x02,0x00,0x00,0xaf,0x02,0x00,0x00,0xdb,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xde,0x02,0x00,0x00, -0x68,0x00,0x00,0x00,0xb2,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xdf,0x02,0x00,0x00,0xdc,0x02,0x00,0x00, -0xde,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0xe1,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0xe1,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0x2e,0x03,0x00,0x00,0x3e,0x00,0x00,0x00, -0xca,0x02,0x00,0x00,0x24,0x03,0x00,0x00,0xe4,0x02,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00,0xe7,0x02,0x00,0x00, -0x2e,0x03,0x00,0x00,0xb2,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0xe3,0x02,0x00,0x00,0xe4,0x02,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xe7,0x02,0x00,0x00,0xe2,0x02,0x00,0x00, -0xe3,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0xe2,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0xe9,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0xe9,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0x30,0x03,0x00,0x00,0x3e,0x00,0x00,0x00,0xe2,0x02,0x00,0x00, -0x22,0x03,0x00,0x00,0xec,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb8,0x00,0x00,0x00,0xef,0x02,0x00,0x00,0x30,0x03,0x00,0x00, -0x62,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xeb,0x02,0x00,0x00, -0xec,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0xef,0x02,0x00,0x00,0xea,0x02,0x00,0x00,0xeb,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0xea,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xf2,0x02,0x00,0x00,0xd7,0x02,0x00,0x00, -0x30,0x03,0x00,0x00,0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00, -0xf5,0x02,0x00,0x00,0xf2,0x02,0x00,0x00,0x36,0x00,0x00,0x00, -0xf7,0x00,0x03,0x00,0xf7,0x02,0x00,0x00,0x00,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xf5,0x02,0x00,0x00,0xf6,0x02,0x00,0x00, -0xf7,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0xf6,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xfa,0x02,0x00,0x00, -0xdf,0x02,0x00,0x00,0x2e,0x03,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0xfb,0x02,0x00,0x00,0x12,0x00,0x00,0x00, -0xc6,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xfc,0x02,0x00,0x00,0xfb,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb8,0x00,0x00,0x00,0xfd,0x02,0x00,0x00,0xfa,0x02,0x00,0x00, -0xfc,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0xf7,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0xf7,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, -0xb8,0x00,0x00,0x00,0xfe,0x02,0x00,0x00,0xf5,0x02,0x00,0x00, -0xea,0x02,0x00,0x00,0xfd,0x02,0x00,0x00,0xf6,0x02,0x00,0x00, -0xf7,0x00,0x03,0x00,0x00,0x03,0x00,0x00,0x00,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xfe,0x02,0x00,0x00,0xff,0x02,0x00,0x00, -0x00,0x03,0x00,0x00,0xf8,0x00,0x02,0x00,0xff,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x08,0x03,0x00,0x00, -0xdf,0x02,0x00,0x00,0x2e,0x03,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x0a,0x03,0x00,0x00,0x12,0x00,0x00,0x00, -0x09,0x03,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x0b,0x03,0x00,0x00,0x0a,0x03,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x0c,0x03,0x00,0x00,0x08,0x03,0x00,0x00, -0x0b,0x03,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x0d,0x03,0x00,0x00,0xbf,0x02,0x00,0x00,0x0c,0x03,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x0f,0x03,0x00,0x00, -0x0d,0x03,0x00,0x00,0xd7,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x11,0x03,0x00,0x00,0x0f,0x03,0x00,0x00, -0x30,0x03,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x13,0x03,0x00,0x00,0x2b,0x03,0x00,0x00,0xb2,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x15,0x03,0x00,0x00, -0x13,0x03,0x00,0x00,0x2e,0x03,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x17,0x03,0x00,0x00,0x15,0x03,0x00,0x00, -0x16,0x03,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x19,0x03,0x00,0x00,0x2c,0x03,0x00,0x00,0x62,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x1a,0x03,0x00,0x00, -0x17,0x03,0x00,0x00,0x19,0x03,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x1c,0x03,0x00,0x00,0x1a,0x03,0x00,0x00, -0x30,0x03,0x00,0x00,0x41,0x00,0x05,0x00,0xc3,0x00,0x00,0x00, -0x1d,0x03,0x00,0x00,0xc0,0x00,0x00,0x00,0x1c,0x03,0x00,0x00, -0x3d,0x00,0x04,0x00,0xba,0x00,0x00,0x00,0x1e,0x03,0x00,0x00, -0x1d,0x03,0x00,0x00,0x41,0x00,0x06,0x00,0x1f,0x03,0x00,0x00, -0x20,0x03,0x00,0x00,0x04,0x03,0x00,0x00,0x34,0x00,0x00,0x00, -0x11,0x03,0x00,0x00,0x3e,0x00,0x03,0x00,0x20,0x03,0x00,0x00, -0x1e,0x03,0x00,0x00,0xf9,0x00,0x02,0x00,0x00,0x03,0x00,0x00, -0xf8,0x00,0x02,0x00,0x00,0x03,0x00,0x00,0xf9,0x00,0x02,0x00, -0xec,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0xec,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x22,0x03,0x00,0x00, -0x30,0x03,0x00,0x00,0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xe9,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0xeb,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0xe4,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0xe4,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x24,0x03,0x00,0x00,0x2e,0x03,0x00,0x00,0xc6,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xe1,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0xe3,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0xcc,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0xcc,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x26,0x03,0x00,0x00,0x2c,0x03,0x00,0x00, -0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xc9,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0xcb,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0xc4,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0xc4,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x28,0x03,0x00,0x00, -0x2b,0x03,0x00,0x00,0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xc1,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0xc3,0x02,0x00,0x00, -0xfd,0x00,0x01,0x00,0x38,0x00,0x01,0x00, -}; -const uint64_t matmul_f16_aligned_m_len = 11936; - -unsigned char matmul_f16_aligned_m_fp32_data[] = { -0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, -0xf3,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, -0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x51,0x11,0x00,0x00, -0x0b,0x00,0x06,0x00,0x01,0x00,0x00,0x00,0x47,0x4c,0x53,0x4c, -0x2e,0x73,0x74,0x64,0x2e,0x34,0x35,0x30,0x00,0x00,0x00,0x00, -0x0e,0x00,0x03,0x00,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x0f,0x00,0x0f,0x00,0x05,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x6d,0x61,0x69,0x6e,0x00,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x3d,0x00,0x00,0x00,0x4c,0x00,0x00,0x00, -0xea,0x00,0x00,0x00,0xf9,0x00,0x00,0x00,0x4b,0x01,0x00,0x00, -0x58,0x01,0x00,0x00,0x53,0x02,0x00,0x00,0x9c,0x02,0x00,0x00, -0x10,0x00,0x06,0x00,0x04,0x00,0x00,0x00,0x11,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x0b,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x1c,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x10,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x08,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x03,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x10,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x14,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x18,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x07,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x10,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x24,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x0a,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x10,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x2c,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x30,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x0d,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0x47,0x00,0x03,0x00, -0x10,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x37,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x3d,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x1a,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x4c,0x00,0x00,0x00, -0x0b,0x00,0x00,0x00,0x1b,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x4f,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x53,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x60,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x62,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x07,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x6c,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x03,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x9d,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0xaf,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x05,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0xb2,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x08,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xf6,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x48,0x00,0x04,0x00, -0xf7,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0xf7,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, -0xf7,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0xf9,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0xf9,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x2c,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x2d,0x01,0x00,0x00,0x0b,0x00,0x00,0x00,0x19,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x55,0x01,0x00,0x00,0x06,0x00,0x00,0x00, -0x08,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0x56,0x01,0x00,0x00, -0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x56,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x56,0x01,0x00,0x00, -0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x58,0x01,0x00,0x00, -0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x58,0x01,0x00,0x00,0x21,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x53,0x02,0x00,0x00,0x0b,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x99,0x02,0x00,0x00, -0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00, -0x9a,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x19,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x9a,0x02,0x00,0x00,0x00,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, -0x9a,0x02,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x9c,0x02,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x9c,0x02,0x00,0x00,0x21,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x13,0x00,0x02,0x00,0x02,0x00,0x00,0x00, -0x21,0x00,0x03,0x00,0x03,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x15,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x17,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x0a,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x0d,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x1e,0x00,0x10,0x00,0x10,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x11,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x11,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x15,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, -0x14,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x15,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x0a,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, -0x27,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0x2d,0x00,0x00,0x00,0x07,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x34,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x37,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00,0x3d,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x3e,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x0a,0x00,0x00,0x00,0x4c,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x4f,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x53,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x54,0x00,0x00,0x00,0x86,0x00,0x00,0x00, -0x37,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x58,0x00,0x00,0x00,0x86,0x00,0x00,0x00, -0x37,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x32,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x61,0x00,0x00,0x00, -0x86,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x60,0x00,0x00,0x00, -0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x62,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x63,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x61,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x67,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x61,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x6e,0x00,0x00,0x00, -0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x73,0x00,0x00,0x00, -0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x77,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, -0x7c,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0x87,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x8d,0x00,0x00,0x00, -0x03,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, -0x98,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x32,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x9d,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x9f,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xae,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xaf,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xb0,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0x53,0x00,0x00,0x00,0xaf,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xb1,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0x4f,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x32,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xb2,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xb3,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0xb1,0x00,0x00,0x00,0xb2,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xb4,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0xb3,0x00,0x00,0x00,0x60,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xb5,0x00,0x00,0x00, -0x86,0x00,0x00,0x00,0xb0,0x00,0x00,0x00,0xb4,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xb6,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0xae,0x00,0x00,0x00,0xb5,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xb7,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0xb6,0x00,0x00,0x00,0xb2,0x00,0x00,0x00, -0x14,0x00,0x02,0x00,0xb8,0x00,0x00,0x00,0x16,0x00,0x03,0x00, -0xba,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xbb,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xbc,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0xbb,0x00,0x00,0x00,0xb5,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xbd,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0xbc,0x00,0x00,0x00,0xb2,0x00,0x00,0x00,0x1c,0x00,0x04,0x00, -0xbe,0x00,0x00,0x00,0xba,0x00,0x00,0x00,0xbd,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0xbf,0x00,0x00,0x00,0x07,0x00,0x00,0x00, -0xbe,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0xba,0x00,0x00,0x00, -0xc2,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0xc3,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0xba,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0xc6,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xe6,0x00,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, +0x10,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x6d,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, 0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xe7,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x37,0x00,0x00,0x00, -0xe6,0x00,0x00,0x00,0x1c,0x00,0x04,0x00,0xe8,0x00,0x00,0x00, -0xba,0x00,0x00,0x00,0xe7,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0xe9,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0xe8,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0xe9,0x00,0x00,0x00,0xea,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xee,0x00,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x16,0x00,0x03,0x00,0xf4,0x00,0x00,0x00, -0x10,0x00,0x00,0x00,0x17,0x00,0x04,0x00,0xf5,0x00,0x00,0x00, -0xf4,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, -0xf6,0x00,0x00,0x00,0xf5,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, -0xf7,0x00,0x00,0x00,0xf6,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0xf8,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0xf7,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0xf8,0x00,0x00,0x00,0xf9,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0xfb,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0xf4,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0xff,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0xba,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x04,0x01,0x00,0x00, -0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x12,0x01,0x00,0x00, -0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x20,0x01,0x00,0x00, -0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x25,0x01,0x00,0x00, -0x03,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x2c,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x33,0x00,0x06,0x00, -0x09,0x00,0x00,0x00,0x2d,0x01,0x00,0x00,0x2c,0x01,0x00,0x00, -0x39,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x2e,0x01,0x00,0x00,0x51,0x00,0x00,0x00, -0x2d,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x2f,0x01,0x00,0x00,0x84,0x00,0x00,0x00, -0x2e,0x01,0x00,0x00,0x6d,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x30,0x01,0x00,0x00,0x86,0x00,0x00,0x00, -0x2f,0x01,0x00,0x00,0x6c,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x47,0x01,0x00,0x00,0x80,0x00,0x00,0x00, +0x72,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, +0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x77,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, +0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x7c,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, +0x39,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, +0x80,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0x85,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x90,0x00,0x00,0x00, +0x0b,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, +0x96,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0xa1,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xa6,0x00,0x00,0x00, +0x40,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, +0xa8,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xb7,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x32,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xb8,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xb9,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0xb8,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xba,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x62,0x00,0x00,0x00, +0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xbb,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xbc,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0xba,0x00,0x00,0x00, +0xbb,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xbd,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0xbc,0x00,0x00,0x00, +0x60,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xbe,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0xb9,0x00,0x00,0x00, +0xbd,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xbf,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0xb7,0x00,0x00,0x00, +0xbe,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xc0,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0xbf,0x00,0x00,0x00, +0xbb,0x00,0x00,0x00,0x14,0x00,0x02,0x00,0xc1,0x00,0x00,0x00, +0x16,0x00,0x03,0x00,0xc3,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xc4,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xc5,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0xc4,0x00,0x00,0x00,0xbe,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xc6,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0xc5,0x00,0x00,0x00,0xbb,0x00,0x00,0x00, +0x1c,0x00,0x04,0x00,0xc7,0x00,0x00,0x00,0xc3,0x00,0x00,0x00, +0xc6,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0xc8,0x00,0x00,0x00, +0x07,0x00,0x00,0x00,0xc7,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0xc3,0x00,0x00,0x00,0xcb,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0xcc,0x00,0x00,0x00,0x07,0x00,0x00,0x00, +0xc3,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, +0xcf,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x16,0x00,0x03,0x00, +0xf6,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xf7,0x00,0x00,0x00,0x80,0x00,0x00,0x00, 0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x48,0x01,0x00,0x00,0x84,0x00,0x00,0x00, -0x9d,0x00,0x00,0x00,0x47,0x01,0x00,0x00,0x1c,0x00,0x04,0x00, -0x49,0x01,0x00,0x00,0xba,0x00,0x00,0x00,0x48,0x01,0x00,0x00, -0x20,0x00,0x04,0x00,0x4a,0x01,0x00,0x00,0x04,0x00,0x00,0x00, -0x49,0x01,0x00,0x00,0x3b,0x00,0x04,0x00,0x4a,0x01,0x00,0x00, -0x4b,0x01,0x00,0x00,0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x4f,0x01,0x00,0x00,0x80,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0xf8,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0x37,0x00,0x00,0x00,0xf7,0x00,0x00,0x00,0x1c,0x00,0x04,0x00, +0xf9,0x00,0x00,0x00,0xf6,0x00,0x00,0x00,0xf8,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0xfa,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0xf9,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0xfa,0x00,0x00,0x00, +0xfb,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xff,0x00,0x00,0x00,0x80,0x00,0x00,0x00, 0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, -0x55,0x01,0x00,0x00,0xf5,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, -0x56,0x01,0x00,0x00,0x55,0x01,0x00,0x00,0x20,0x00,0x04,0x00, -0x57,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x56,0x01,0x00,0x00, -0x3b,0x00,0x04,0x00,0x57,0x01,0x00,0x00,0x58,0x01,0x00,0x00, -0x0c,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x61,0x01,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x6f,0x01,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x7d,0x01,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x8a,0x01,0x00,0x00,0x08,0x01,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x8b,0x01,0x00,0x00,0x86,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x8e,0x01,0x00,0x00,0x86,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xa9,0x01,0x00,0x00,0x84,0x00,0x00,0x00, -0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x1c,0x00,0x04,0x00, -0xaa,0x01,0x00,0x00,0xba,0x00,0x00,0x00,0xa9,0x01,0x00,0x00, -0x20,0x00,0x04,0x00,0xab,0x01,0x00,0x00,0x07,0x00,0x00,0x00, -0xaa,0x01,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xbb,0x01,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xd6,0x01,0x00,0x00,0x84,0x00,0x00,0x00,0xb5,0x00,0x00,0x00, -0xb2,0x00,0x00,0x00,0x1c,0x00,0x04,0x00,0xd7,0x01,0x00,0x00, -0xba,0x00,0x00,0x00,0xd6,0x01,0x00,0x00,0x20,0x00,0x04,0x00, -0xd8,0x01,0x00,0x00,0x07,0x00,0x00,0x00,0xd7,0x01,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xe1,0x01,0x00,0x00, -0x86,0x00,0x00,0x00,0xaf,0x00,0x00,0x00,0xb5,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xe9,0x01,0x00,0x00, +0x03,0x01,0x00,0x00,0xf6,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, +0x04,0x01,0x00,0x00,0x03,0x01,0x00,0x00,0x20,0x00,0x04,0x00, +0x05,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x04,0x01,0x00,0x00, +0x3b,0x00,0x04,0x00,0x05,0x01,0x00,0x00,0x06,0x01,0x00,0x00, +0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x11,0x01,0x00,0x00, +0x0c,0x00,0x00,0x00,0xf6,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x14,0x01,0x00,0x00,0x04,0x00,0x00,0x00,0xf6,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x1a,0x01,0x00,0x00, 0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x18,0x02,0x00,0x00, -0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x4b,0x02,0x00,0x00, -0x0d,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00, -0x53,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, -0x99,0x02,0x00,0x00,0xba,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, -0x9a,0x02,0x00,0x00,0x99,0x02,0x00,0x00,0x20,0x00,0x04,0x00, -0x9b,0x02,0x00,0x00,0x0c,0x00,0x00,0x00,0x9a,0x02,0x00,0x00, -0x3b,0x00,0x04,0x00,0x9b,0x02,0x00,0x00,0x9c,0x02,0x00,0x00, -0x0c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, -0xa1,0x02,0x00,0x00,0x05,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xae,0x02,0x00,0x00,0x84,0x00,0x00,0x00, -0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0xb7,0x02,0x00,0x00,0x0c,0x00,0x00,0x00,0xba,0x00,0x00,0x00, -0x36,0x00,0x05,0x00,0x02,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x05,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0xbf,0x00,0x00,0x00, -0xc0,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0xab,0x01,0x00,0x00,0xac,0x01,0x00,0x00,0x07,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0xd8,0x01,0x00,0x00,0xd9,0x01,0x00,0x00, -0x07,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00, -0x0e,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, -0x0e,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x14,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x17,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x17,0x00,0x00,0x00, -0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x1e,0x00,0x00,0x00, -0x0f,0x00,0x00,0x00,0x17,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x21,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x24,0x00,0x00,0x00,0x18,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0x28,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x27,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x29,0x00,0x00,0x00, -0x28,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x2a,0x00,0x00,0x00,0x1e,0x00,0x00,0x00,0x29,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x2d,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x2f,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x30,0x00,0x00,0x00, -0x24,0x00,0x00,0x00,0x2f,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x30,0x00,0x00,0x00, -0x2a,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0x35,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x34,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x36,0x00,0x00,0x00, -0x35,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x38,0x00,0x00,0x00,0x36,0x00,0x00,0x00,0x37,0x00,0x00,0x00, -0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, -0x38,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x3b,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, -0x37,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00, -0x3f,0x00,0x00,0x00,0x3d,0x00,0x00,0x00,0x3e,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x3f,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x42,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x3b,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x47,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0x3b,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x0d,0x00,0x00,0x00,0x49,0x00,0x00,0x00,0x3d,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x4a,0x00,0x00,0x00,0x49,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x0d,0x00,0x00,0x00,0x4d,0x00,0x00,0x00,0x4c,0x00,0x00,0x00, -0x3e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x4e,0x00,0x00,0x00,0x4d,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x50,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, -0x4f,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x55,0x00,0x00,0x00,0x50,0x00,0x00,0x00,0x54,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x59,0x00,0x00,0x00, -0x50,0x00,0x00,0x00,0x58,0x00,0x00,0x00,0x89,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x5d,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, -0x4f,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x64,0x00,0x00,0x00,0x5d,0x00,0x00,0x00,0x63,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x68,0x00,0x00,0x00, -0x5d,0x00,0x00,0x00,0x67,0x00,0x00,0x00,0x89,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x6f,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, -0x6e,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x74,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x73,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x78,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x77,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x79,0x00,0x00,0x00,0x78,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x7a,0x00,0x00,0x00, -0x47,0x00,0x00,0x00,0x79,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x7d,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x7c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x7e,0x00,0x00,0x00,0x7d,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x80,0x00,0x00,0x00,0x47,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x83,0x00,0x00,0x00,0x80,0x00,0x00,0x00,0x79,0x00,0x00,0x00, -0x0c,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x26,0x00,0x00,0x00,0x7e,0x00,0x00,0x00, -0x83,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0x88,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x87,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x89,0x00,0x00,0x00, -0x88,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x8a,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x89,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8c,0x00,0x00,0x00, -0x42,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x8e,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x8d,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x8f,0x00,0x00,0x00,0x8e,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x90,0x00,0x00,0x00,0x8c,0x00,0x00,0x00, -0x8f,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x91,0x00,0x00,0x00,0x8a,0x00,0x00,0x00,0x90,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x93,0x00,0x00,0x00, -0x91,0x00,0x00,0x00,0x7a,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x94,0x00,0x00,0x00,0x93,0x00,0x00,0x00, -0x6d,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0x99,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x98,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x9a,0x00,0x00,0x00, -0x99,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x9b,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x9a,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9e,0x00,0x00,0x00, -0x4a,0x00,0x00,0x00,0x9d,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0xa0,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x9f,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xa1,0x00,0x00,0x00,0xa0,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xa2,0x00,0x00,0x00,0x9e,0x00,0x00,0x00, -0xa1,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xa3,0x00,0x00,0x00,0x9b,0x00,0x00,0x00,0xa2,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa5,0x00,0x00,0x00, -0xa3,0x00,0x00,0x00,0x7a,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xa6,0x00,0x00,0x00,0xa5,0x00,0x00,0x00, -0x6d,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xa8,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xa8,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xc1,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, -0x05,0x00,0x00,0x00,0xc7,0x00,0x00,0x00,0xa9,0x00,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00,0xb9,0x00,0x00,0x00, -0xc1,0x02,0x00,0x00,0xb7,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0xaa,0x00,0x00,0x00,0xa9,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xb9,0x00,0x00,0x00,0xa9,0x00,0x00,0x00, -0xaa,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xa9,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0xc3,0x00,0x00,0x00,0xc4,0x00,0x00,0x00, -0xc0,0x00,0x00,0x00,0xc1,0x02,0x00,0x00,0x3e,0x00,0x03,0x00, -0xc4,0x00,0x00,0x00,0xc2,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xc7,0x00,0x00,0x00,0xc1,0x02,0x00,0x00, -0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xa8,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xaa,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xca,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xca,0x00,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xda,0x02,0x00,0x00, -0xa6,0x00,0x00,0x00,0xaa,0x00,0x00,0x00,0x90,0x01,0x00,0x00, -0xcd,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xd6,0x02,0x00,0x00,0x94,0x00,0x00,0x00,0xaa,0x00,0x00,0x00, -0x8d,0x01,0x00,0x00,0xcd,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xc2,0x02,0x00,0x00,0x7a,0x00,0x00,0x00, -0xaa,0x00,0x00,0x00,0x3b,0x02,0x00,0x00,0xcd,0x00,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00,0xd1,0x00,0x00,0x00, -0xc2,0x02,0x00,0x00,0x84,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0xcc,0x00,0x00,0x00,0xcd,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xd1,0x00,0x00,0x00,0xcb,0x00,0x00,0x00, -0xcc,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xcb,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xd3,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xd3,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xd2,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0xcb,0x00,0x00,0x00, -0x32,0x01,0x00,0x00,0xd4,0x00,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb8,0x00,0x00,0x00,0xd9,0x00,0x00,0x00,0xd2,0x02,0x00,0x00, -0x37,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xd5,0x00,0x00,0x00, -0xd4,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0xd9,0x00,0x00,0x00,0xd4,0x00,0x00,0x00,0xd5,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xd4,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xde,0x00,0x00,0x00,0x74,0x00,0x00,0x00, -0xd2,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xe1,0x00,0x00,0x00,0xde,0x00,0x00,0x00,0x8f,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xe2,0x00,0x00,0x00, -0xe1,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xe3,0x00,0x00,0x00,0xd6,0x02,0x00,0x00, -0xe2,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xe5,0x00,0x00,0x00,0xe3,0x00,0x00,0x00,0x6f,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xef,0x00,0x00,0x00, -0xde,0x00,0x00,0x00,0xee,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xf1,0x00,0x00,0x00,0x6f,0x00,0x00,0x00, -0x6d,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xf2,0x00,0x00,0x00,0xef,0x00,0x00,0x00,0xf1,0x00,0x00,0x00, -0x41,0x00,0x07,0x00,0xfb,0x00,0x00,0x00,0xfc,0x00,0x00,0x00, -0xf9,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0xe5,0x00,0x00,0x00, -0x3e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0xf4,0x00,0x00,0x00, -0xfd,0x00,0x00,0x00,0xfc,0x00,0x00,0x00,0x73,0x00,0x04,0x00, -0xba,0x00,0x00,0x00,0xfe,0x00,0x00,0x00,0xfd,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0xff,0x00,0x00,0x00,0x00,0x01,0x00,0x00, -0xea,0x00,0x00,0x00,0xf2,0x00,0x00,0x00,0x3e,0x00,0x03,0x00, -0x00,0x01,0x00,0x00,0xfe,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x05,0x01,0x00,0x00,0xde,0x00,0x00,0x00, -0x04,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x08,0x01,0x00,0x00,0x05,0x01,0x00,0x00,0xf1,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x09,0x01,0x00,0x00, -0x08,0x01,0x00,0x00,0x39,0x00,0x00,0x00,0x41,0x00,0x07,0x00, -0xfb,0x00,0x00,0x00,0x0b,0x01,0x00,0x00,0xf9,0x00,0x00,0x00, -0x34,0x00,0x00,0x00,0xe5,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0xf4,0x00,0x00,0x00,0x0c,0x01,0x00,0x00, -0x0b,0x01,0x00,0x00,0x73,0x00,0x04,0x00,0xba,0x00,0x00,0x00, -0x0d,0x01,0x00,0x00,0x0c,0x01,0x00,0x00,0x41,0x00,0x05,0x00, -0xff,0x00,0x00,0x00,0x0e,0x01,0x00,0x00,0xea,0x00,0x00,0x00, -0x09,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x0e,0x01,0x00,0x00, -0x0d,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x13,0x01,0x00,0x00,0xde,0x00,0x00,0x00,0x12,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x16,0x01,0x00,0x00, -0x13,0x01,0x00,0x00,0xf1,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x17,0x01,0x00,0x00,0x16,0x01,0x00,0x00, -0x0c,0x00,0x00,0x00,0x41,0x00,0x07,0x00,0xfb,0x00,0x00,0x00, -0x19,0x01,0x00,0x00,0xf9,0x00,0x00,0x00,0x34,0x00,0x00,0x00, -0xe5,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0xf4,0x00,0x00,0x00,0x1a,0x01,0x00,0x00,0x19,0x01,0x00,0x00, -0x73,0x00,0x04,0x00,0xba,0x00,0x00,0x00,0x1b,0x01,0x00,0x00, -0x1a,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0xff,0x00,0x00,0x00, -0x1c,0x01,0x00,0x00,0xea,0x00,0x00,0x00,0x17,0x01,0x00,0x00, -0x3e,0x00,0x03,0x00,0x1c,0x01,0x00,0x00,0x1b,0x01,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x21,0x01,0x00,0x00, -0xde,0x00,0x00,0x00,0x20,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x24,0x01,0x00,0x00,0x21,0x01,0x00,0x00, -0xf1,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x26,0x01,0x00,0x00,0x24,0x01,0x00,0x00,0x25,0x01,0x00,0x00, -0x41,0x00,0x07,0x00,0xfb,0x00,0x00,0x00,0x28,0x01,0x00,0x00, -0xf9,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0xe5,0x00,0x00,0x00, -0x25,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0xf4,0x00,0x00,0x00, -0x29,0x01,0x00,0x00,0x28,0x01,0x00,0x00,0x73,0x00,0x04,0x00, -0xba,0x00,0x00,0x00,0x2a,0x01,0x00,0x00,0x29,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0xff,0x00,0x00,0x00,0x2b,0x01,0x00,0x00, -0xea,0x00,0x00,0x00,0x26,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0x2b,0x01,0x00,0x00,0x2a,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x32,0x01,0x00,0x00,0xd2,0x02,0x00,0x00, -0x30,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xd3,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xd5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x34,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x34,0x01,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xd3,0x02,0x00,0x00, -0x3e,0x00,0x00,0x00,0xd5,0x00,0x00,0x00,0x89,0x01,0x00,0x00, -0x35,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00, -0x3a,0x01,0x00,0x00,0xd3,0x02,0x00,0x00,0x9d,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0x36,0x01,0x00,0x00,0x35,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x3a,0x01,0x00,0x00, -0x35,0x01,0x00,0x00,0x36,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x35,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x3f,0x01,0x00,0x00,0x74,0x00,0x00,0x00,0xd3,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x42,0x01,0x00,0x00, -0x3f,0x01,0x00,0x00,0xa1,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x43,0x01,0x00,0x00,0x42,0x01,0x00,0x00, -0x6d,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x44,0x01,0x00,0x00,0xda,0x02,0x00,0x00,0x43,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x46,0x01,0x00,0x00, -0x44,0x01,0x00,0x00,0x6f,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x50,0x01,0x00,0x00,0x3f,0x01,0x00,0x00, -0x4f,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x52,0x01,0x00,0x00,0x6f,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x53,0x01,0x00,0x00, -0x50,0x01,0x00,0x00,0x52,0x01,0x00,0x00,0x41,0x00,0x07,0x00, -0xfb,0x00,0x00,0x00,0x5a,0x01,0x00,0x00,0x58,0x01,0x00,0x00, -0x34,0x00,0x00,0x00,0x46,0x01,0x00,0x00,0x3e,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0xf4,0x00,0x00,0x00,0x5b,0x01,0x00,0x00, -0x5a,0x01,0x00,0x00,0x73,0x00,0x04,0x00,0xba,0x00,0x00,0x00, -0x5c,0x01,0x00,0x00,0x5b,0x01,0x00,0x00,0x41,0x00,0x05,0x00, -0xff,0x00,0x00,0x00,0x5d,0x01,0x00,0x00,0x4b,0x01,0x00,0x00, -0x53,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x5d,0x01,0x00,0x00, -0x5c,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x62,0x01,0x00,0x00,0x3f,0x01,0x00,0x00,0x61,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x65,0x01,0x00,0x00, -0x62,0x01,0x00,0x00,0x52,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x66,0x01,0x00,0x00,0x65,0x01,0x00,0x00, -0x39,0x00,0x00,0x00,0x41,0x00,0x07,0x00,0xfb,0x00,0x00,0x00, -0x68,0x01,0x00,0x00,0x58,0x01,0x00,0x00,0x34,0x00,0x00,0x00, -0x46,0x01,0x00,0x00,0x39,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0xf4,0x00,0x00,0x00,0x69,0x01,0x00,0x00,0x68,0x01,0x00,0x00, -0x73,0x00,0x04,0x00,0xba,0x00,0x00,0x00,0x6a,0x01,0x00,0x00, -0x69,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0xff,0x00,0x00,0x00, -0x6b,0x01,0x00,0x00,0x4b,0x01,0x00,0x00,0x66,0x01,0x00,0x00, -0x3e,0x00,0x03,0x00,0x6b,0x01,0x00,0x00,0x6a,0x01,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x70,0x01,0x00,0x00, -0x3f,0x01,0x00,0x00,0x6f,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x73,0x01,0x00,0x00,0x70,0x01,0x00,0x00, -0x52,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x74,0x01,0x00,0x00,0x73,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, -0x41,0x00,0x07,0x00,0xfb,0x00,0x00,0x00,0x76,0x01,0x00,0x00, -0x58,0x01,0x00,0x00,0x34,0x00,0x00,0x00,0x46,0x01,0x00,0x00, -0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0xf4,0x00,0x00,0x00, -0x77,0x01,0x00,0x00,0x76,0x01,0x00,0x00,0x73,0x00,0x04,0x00, -0xba,0x00,0x00,0x00,0x78,0x01,0x00,0x00,0x77,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0xff,0x00,0x00,0x00,0x79,0x01,0x00,0x00, -0x4b,0x01,0x00,0x00,0x74,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0x79,0x01,0x00,0x00,0x78,0x01,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x7e,0x01,0x00,0x00,0x3f,0x01,0x00,0x00, -0x7d,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x81,0x01,0x00,0x00,0x7e,0x01,0x00,0x00,0x52,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x82,0x01,0x00,0x00, -0x81,0x01,0x00,0x00,0x25,0x01,0x00,0x00,0x41,0x00,0x07,0x00, -0xfb,0x00,0x00,0x00,0x84,0x01,0x00,0x00,0x58,0x01,0x00,0x00, -0x34,0x00,0x00,0x00,0x46,0x01,0x00,0x00,0x25,0x01,0x00,0x00, -0x3d,0x00,0x04,0x00,0xf4,0x00,0x00,0x00,0x85,0x01,0x00,0x00, -0x84,0x01,0x00,0x00,0x73,0x00,0x04,0x00,0xba,0x00,0x00,0x00, -0x86,0x01,0x00,0x00,0x85,0x01,0x00,0x00,0x41,0x00,0x05,0x00, -0xff,0x00,0x00,0x00,0x87,0x01,0x00,0x00,0x4b,0x01,0x00,0x00, -0x82,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x87,0x01,0x00,0x00, -0x86,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x89,0x01,0x00,0x00,0xd3,0x02,0x00,0x00,0x30,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0x34,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x36,0x01,0x00,0x00,0xe0,0x00,0x04,0x00,0x0c,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x8a,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x8d,0x01,0x00,0x00,0xd6,0x02,0x00,0x00, -0x8b,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x90,0x01,0x00,0x00,0xda,0x02,0x00,0x00,0x8e,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0x92,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x92,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xdc,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0x36,0x01,0x00,0x00, -0x39,0x02,0x00,0x00,0x95,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb8,0x00,0x00,0x00,0x98,0x01,0x00,0x00,0xdc,0x02,0x00,0x00, -0x6c,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x94,0x01,0x00,0x00, -0x95,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x98,0x01,0x00,0x00,0x93,0x01,0x00,0x00,0x94,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x93,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0x9a,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x9a,0x01,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xe0,0x02,0x00,0x00, -0x3e,0x00,0x00,0x00,0x93,0x01,0x00,0x00,0xc5,0x01,0x00,0x00, -0x9d,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00, -0xa0,0x01,0x00,0x00,0xe0,0x02,0x00,0x00,0x60,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0x9c,0x01,0x00,0x00,0x9d,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xa0,0x01,0x00,0x00, -0x9b,0x01,0x00,0x00,0x9c,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x9b,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xa2,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xa2,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xf2,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, -0x9b,0x01,0x00,0x00,0xc3,0x01,0x00,0x00,0xa3,0x01,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00,0xa8,0x01,0x00,0x00, -0xf2,0x02,0x00,0x00,0x62,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0xa4,0x01,0x00,0x00,0xa3,0x01,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xa8,0x01,0x00,0x00,0xa3,0x01,0x00,0x00, -0xa4,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xa3,0x01,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xae,0x01,0x00,0x00, -0xe0,0x02,0x00,0x00,0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xb0,0x01,0x00,0x00,0xae,0x01,0x00,0x00, -0xf2,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xb2,0x01,0x00,0x00,0x55,0x00,0x00,0x00,0x53,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xb4,0x01,0x00,0x00, -0xe0,0x02,0x00,0x00,0x61,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xb5,0x01,0x00,0x00,0xb2,0x01,0x00,0x00, -0xb4,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xb7,0x01,0x00,0x00,0x64,0x00,0x00,0x00,0x62,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xb8,0x01,0x00,0x00, -0xb5,0x01,0x00,0x00,0xb7,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xba,0x01,0x00,0x00,0xb8,0x01,0x00,0x00, -0xf2,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xbc,0x01,0x00,0x00,0xba,0x01,0x00,0x00,0xbb,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xbe,0x01,0x00,0x00, -0xbc,0x01,0x00,0x00,0xdc,0x02,0x00,0x00,0x41,0x00,0x05,0x00, -0xff,0x00,0x00,0x00,0xbf,0x01,0x00,0x00,0xea,0x00,0x00,0x00, -0xbe,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0xba,0x00,0x00,0x00, -0xc0,0x01,0x00,0x00,0xbf,0x01,0x00,0x00,0x41,0x00,0x05,0x00, -0xc3,0x00,0x00,0x00,0xc1,0x01,0x00,0x00,0xac,0x01,0x00,0x00, -0xb0,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0xc1,0x01,0x00,0x00, -0xc0,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xc3,0x01,0x00,0x00,0xf2,0x02,0x00,0x00,0xc6,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xa2,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xa4,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x9d,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x9d,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xc5,0x01,0x00,0x00,0xe0,0x02,0x00,0x00, -0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x9a,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x9c,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0xc7,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xc7,0x01,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xe1,0x02,0x00,0x00, -0x3e,0x00,0x00,0x00,0x9c,0x01,0x00,0x00,0xf3,0x01,0x00,0x00, -0xca,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00, -0xcd,0x01,0x00,0x00,0xe1,0x02,0x00,0x00,0xb5,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0xc9,0x01,0x00,0x00,0xca,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xcd,0x01,0x00,0x00, -0xc8,0x01,0x00,0x00,0xc9,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xc8,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xcf,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xcf,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xef,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, -0xc8,0x01,0x00,0x00,0xf1,0x01,0x00,0x00,0xd0,0x01,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00,0xd5,0x01,0x00,0x00, -0xef,0x02,0x00,0x00,0xb2,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0xd1,0x01,0x00,0x00,0xd0,0x01,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xd5,0x01,0x00,0x00,0xd0,0x01,0x00,0x00, -0xd1,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xd0,0x01,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xdb,0x01,0x00,0x00, -0xe1,0x02,0x00,0x00,0xb2,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xdd,0x01,0x00,0x00,0xdb,0x01,0x00,0x00, -0xef,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xdf,0x01,0x00,0x00,0x59,0x00,0x00,0x00,0xaf,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xe2,0x01,0x00,0x00, -0xe1,0x02,0x00,0x00,0xe1,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xe3,0x01,0x00,0x00,0xdf,0x01,0x00,0x00, -0xe2,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xe5,0x01,0x00,0x00,0x68,0x00,0x00,0x00,0xb2,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xe6,0x01,0x00,0x00, -0xe3,0x01,0x00,0x00,0xe5,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xe8,0x01,0x00,0x00,0xe6,0x01,0x00,0x00, -0xef,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xea,0x01,0x00,0x00,0xe8,0x01,0x00,0x00,0xe9,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xec,0x01,0x00,0x00, -0xea,0x01,0x00,0x00,0xdc,0x02,0x00,0x00,0x41,0x00,0x05,0x00, -0xff,0x00,0x00,0x00,0xed,0x01,0x00,0x00,0x4b,0x01,0x00,0x00, -0xec,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0xba,0x00,0x00,0x00, -0xee,0x01,0x00,0x00,0xed,0x01,0x00,0x00,0x41,0x00,0x05,0x00, -0xc3,0x00,0x00,0x00,0xef,0x01,0x00,0x00,0xd9,0x01,0x00,0x00, -0xdd,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0xef,0x01,0x00,0x00, -0xee,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xf1,0x01,0x00,0x00,0xef,0x02,0x00,0x00,0xc6,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xcf,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xd1,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xca,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xca,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xf3,0x01,0x00,0x00,0xe1,0x02,0x00,0x00, -0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xc7,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xc9,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0xf5,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xf5,0x01,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xe2,0x02,0x00,0x00, -0x3e,0x00,0x00,0x00,0xc9,0x01,0x00,0x00,0x37,0x02,0x00,0x00, -0xf8,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00, -0xfb,0x01,0x00,0x00,0xe2,0x02,0x00,0x00,0xb5,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0xf7,0x01,0x00,0x00,0xf8,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xfb,0x01,0x00,0x00, -0xf6,0x01,0x00,0x00,0xf7,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xf6,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xfd,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xfd,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xe6,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, -0xf6,0x01,0x00,0x00,0x35,0x02,0x00,0x00,0x00,0x02,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00,0x03,0x02,0x00,0x00, -0xe6,0x02,0x00,0x00,0x60,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0xff,0x01,0x00,0x00,0x00,0x02,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x03,0x02,0x00,0x00,0xfe,0x01,0x00,0x00, -0xff,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xfe,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0x05,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x05,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xe8,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0xfe,0x01,0x00,0x00, -0x33,0x02,0x00,0x00,0x08,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb8,0x00,0x00,0x00,0x0b,0x02,0x00,0x00,0xe8,0x02,0x00,0x00, -0xb2,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x07,0x02,0x00,0x00, -0x08,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x0b,0x02,0x00,0x00,0x06,0x02,0x00,0x00,0x07,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x06,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x0d,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x0d,0x02,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xea,0x02,0x00,0x00, -0x3e,0x00,0x00,0x00,0x06,0x02,0x00,0x00,0x31,0x02,0x00,0x00, -0x0e,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00, -0x13,0x02,0x00,0x00,0xea,0x02,0x00,0x00,0x62,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0x0f,0x02,0x00,0x00,0x0e,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x13,0x02,0x00,0x00, -0x0e,0x02,0x00,0x00,0x0f,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x0e,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x15,0x02,0x00,0x00,0xe2,0x02,0x00,0x00,0xb2,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x17,0x02,0x00,0x00, -0x15,0x02,0x00,0x00,0xe8,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x19,0x02,0x00,0x00,0x17,0x02,0x00,0x00, -0x18,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x1b,0x02,0x00,0x00,0xe6,0x02,0x00,0x00,0x62,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x1c,0x02,0x00,0x00, -0x19,0x02,0x00,0x00,0x1b,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x1e,0x02,0x00,0x00,0x1c,0x02,0x00,0x00, -0xea,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x22,0x02,0x00,0x00,0x1b,0x02,0x00,0x00,0xea,0x02,0x00,0x00, -0x41,0x00,0x05,0x00,0xc3,0x00,0x00,0x00,0x23,0x02,0x00,0x00, -0xac,0x01,0x00,0x00,0x22,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, -0xba,0x00,0x00,0x00,0x24,0x02,0x00,0x00,0x23,0x02,0x00,0x00, -0x41,0x00,0x05,0x00,0xc3,0x00,0x00,0x00,0x29,0x02,0x00,0x00, -0xd9,0x01,0x00,0x00,0x17,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, -0xba,0x00,0x00,0x00,0x2a,0x02,0x00,0x00,0x29,0x02,0x00,0x00, -0x41,0x00,0x05,0x00,0xc3,0x00,0x00,0x00,0x2c,0x02,0x00,0x00, -0xc0,0x00,0x00,0x00,0x1e,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, -0xba,0x00,0x00,0x00,0x2d,0x02,0x00,0x00,0x2c,0x02,0x00,0x00, -0x0c,0x00,0x08,0x00,0xba,0x00,0x00,0x00,0x2e,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x24,0x02,0x00,0x00, -0x2a,0x02,0x00,0x00,0x2d,0x02,0x00,0x00,0x3e,0x00,0x03,0x00, -0x2c,0x02,0x00,0x00,0x2e,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x31,0x02,0x00,0x00,0xea,0x02,0x00,0x00, -0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x0d,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x0f,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x08,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x08,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x33,0x02,0x00,0x00, -0xe8,0x02,0x00,0x00,0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x05,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x07,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0x00,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x00,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x35,0x02,0x00,0x00,0xe6,0x02,0x00,0x00,0xc6,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xfd,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xff,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xf8,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xf8,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x37,0x02,0x00,0x00,0xe2,0x02,0x00,0x00, -0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xf5,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xf7,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0x95,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x95,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x39,0x02,0x00,0x00, -0xdc,0x02,0x00,0x00,0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x92,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x94,0x01,0x00,0x00, -0xe0,0x00,0x04,0x00,0x0c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x8a,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xcd,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xcd,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x3b,0x02,0x00,0x00,0xc2,0x02,0x00,0x00, -0x6c,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xca,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xcc,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x40,0x02,0x00,0x00,0x55,0x00,0x00,0x00, -0x53,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x41,0x02,0x00,0x00,0x8c,0x00,0x00,0x00,0x40,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x46,0x02,0x00,0x00, -0x59,0x00,0x00,0x00,0xaf,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x47,0x02,0x00,0x00,0x9e,0x00,0x00,0x00, -0x46,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0x4c,0x02,0x00,0x00,0x12,0x00,0x00,0x00,0x4b,0x02,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x4d,0x02,0x00,0x00, -0x4c,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x4e,0x02,0x00,0x00,0x0f,0x00,0x00,0x00,0x4d,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x52,0x02,0x00,0x00, -0x47,0x00,0x00,0x00,0x4d,0x02,0x00,0x00,0x41,0x00,0x05,0x00, -0x0d,0x00,0x00,0x00,0x54,0x02,0x00,0x00,0x53,0x02,0x00,0x00, -0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x55,0x02,0x00,0x00,0x54,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x56,0x02,0x00,0x00,0x52,0x02,0x00,0x00, -0x55,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x57,0x02,0x00,0x00,0x4e,0x02,0x00,0x00,0x56,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0x59,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x59,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xc3,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0xcc,0x00,0x00,0x00, -0xc0,0x02,0x00,0x00,0x5c,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb8,0x00,0x00,0x00,0x5f,0x02,0x00,0x00,0xc3,0x02,0x00,0x00, -0xb5,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x5b,0x02,0x00,0x00, -0x5c,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x5f,0x02,0x00,0x00,0x5a,0x02,0x00,0x00,0x5b,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x5a,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x61,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x61,0x02,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xc4,0x02,0x00,0x00, -0x3e,0x00,0x00,0x00,0x5a,0x02,0x00,0x00,0xbe,0x02,0x00,0x00, -0x64,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00, -0x67,0x02,0x00,0x00,0xc4,0x02,0x00,0x00,0x60,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0x63,0x02,0x00,0x00,0x64,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x67,0x02,0x00,0x00, -0x62,0x02,0x00,0x00,0x63,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x62,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x6b,0x02,0x00,0x00,0xc4,0x02,0x00,0x00,0x61,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x6c,0x02,0x00,0x00, -0x41,0x02,0x00,0x00,0x6b,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x6e,0x02,0x00,0x00,0x64,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x6f,0x02,0x00,0x00,0x6c,0x02,0x00,0x00,0x6e,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x73,0x02,0x00,0x00, -0xc3,0x02,0x00,0x00,0xe1,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x74,0x02,0x00,0x00,0x47,0x02,0x00,0x00, -0x73,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x76,0x02,0x00,0x00,0x68,0x00,0x00,0x00,0xb2,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x77,0x02,0x00,0x00, -0x74,0x02,0x00,0x00,0x76,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x79,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x79,0x02,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xc6,0x02,0x00,0x00, -0x3e,0x00,0x00,0x00,0x62,0x02,0x00,0x00,0xbc,0x02,0x00,0x00, -0x7c,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00, -0x7f,0x02,0x00,0x00,0xc6,0x02,0x00,0x00,0xb2,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0x7b,0x02,0x00,0x00,0x7c,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x7f,0x02,0x00,0x00, -0x7a,0x02,0x00,0x00,0x7b,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x7a,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x81,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x81,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xc8,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, -0x7a,0x02,0x00,0x00,0xba,0x02,0x00,0x00,0x84,0x02,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00,0x87,0x02,0x00,0x00, -0xc8,0x02,0x00,0x00,0x62,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0x83,0x02,0x00,0x00,0x84,0x02,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x87,0x02,0x00,0x00,0x82,0x02,0x00,0x00, -0x83,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x82,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8a,0x02,0x00,0x00, -0x6f,0x02,0x00,0x00,0xc8,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb8,0x00,0x00,0x00,0x8d,0x02,0x00,0x00,0x8a,0x02,0x00,0x00, -0x36,0x00,0x00,0x00,0xf7,0x00,0x03,0x00,0x8f,0x02,0x00,0x00, -0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x8d,0x02,0x00,0x00, -0x8e,0x02,0x00,0x00,0x8f,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x8e,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x92,0x02,0x00,0x00,0x77,0x02,0x00,0x00,0xc6,0x02,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x93,0x02,0x00,0x00, -0x12,0x00,0x00,0x00,0xc6,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x94,0x02,0x00,0x00,0x93,0x02,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00,0x95,0x02,0x00,0x00, -0x92,0x02,0x00,0x00,0x94,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x8f,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x8f,0x02,0x00,0x00, -0xf5,0x00,0x07,0x00,0xb8,0x00,0x00,0x00,0x96,0x02,0x00,0x00, -0x8d,0x02,0x00,0x00,0x82,0x02,0x00,0x00,0x95,0x02,0x00,0x00, -0x8e,0x02,0x00,0x00,0xf7,0x00,0x03,0x00,0x98,0x02,0x00,0x00, -0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x96,0x02,0x00,0x00, -0x97,0x02,0x00,0x00,0x98,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x97,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xa0,0x02,0x00,0x00,0x77,0x02,0x00,0x00,0xc6,0x02,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0xa2,0x02,0x00,0x00, -0x12,0x00,0x00,0x00,0xa1,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xa3,0x02,0x00,0x00,0xa2,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa4,0x02,0x00,0x00, -0xa0,0x02,0x00,0x00,0xa3,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xa5,0x02,0x00,0x00,0x57,0x02,0x00,0x00, -0xa4,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xa7,0x02,0x00,0x00,0xa5,0x02,0x00,0x00,0x6f,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa9,0x02,0x00,0x00, -0xa7,0x02,0x00,0x00,0xc8,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xab,0x02,0x00,0x00,0xc3,0x02,0x00,0x00, -0xb2,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xad,0x02,0x00,0x00,0xab,0x02,0x00,0x00,0xc6,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xaf,0x02,0x00,0x00, -0xad,0x02,0x00,0x00,0xae,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xb1,0x02,0x00,0x00,0xc4,0x02,0x00,0x00, -0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xb2,0x02,0x00,0x00,0xaf,0x02,0x00,0x00,0xb1,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xb4,0x02,0x00,0x00, -0xb2,0x02,0x00,0x00,0xc8,0x02,0x00,0x00,0x41,0x00,0x05,0x00, -0xc3,0x00,0x00,0x00,0xb5,0x02,0x00,0x00,0xc0,0x00,0x00,0x00, -0xb4,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0xba,0x00,0x00,0x00, -0xb6,0x02,0x00,0x00,0xb5,0x02,0x00,0x00,0x41,0x00,0x06,0x00, -0xb7,0x02,0x00,0x00,0xb8,0x02,0x00,0x00,0x9c,0x02,0x00,0x00, -0x34,0x00,0x00,0x00,0xa9,0x02,0x00,0x00,0x3e,0x00,0x03,0x00, -0xb8,0x02,0x00,0x00,0xb6,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x98,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x98,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0x84,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x84,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xba,0x02,0x00,0x00,0xc8,0x02,0x00,0x00,0xc6,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x81,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x83,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x7c,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x7c,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xbc,0x02,0x00,0x00,0xc6,0x02,0x00,0x00, -0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x79,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x7b,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x64,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x64,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xbe,0x02,0x00,0x00, -0xc4,0x02,0x00,0x00,0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x61,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x63,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0x5c,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x5c,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xc0,0x02,0x00,0x00,0xc3,0x02,0x00,0x00,0xc6,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x59,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x5b,0x02,0x00,0x00,0xfd,0x00,0x01,0x00,0x38,0x00,0x01,0x00, - -}; -const uint64_t matmul_f16_aligned_m_fp32_len = 10512; - -unsigned char matmul_f16_aligned_s_data[] = { -0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, -0x5b,0x03,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, -0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x09,0x00,0x00,0x00, -0x11,0x00,0x02,0x00,0x51,0x11,0x00,0x00,0x0b,0x00,0x06,0x00, -0x01,0x00,0x00,0x00,0x47,0x4c,0x53,0x4c,0x2e,0x73,0x74,0x64, -0x2e,0x34,0x35,0x30,0x00,0x00,0x00,0x00,0x0e,0x00,0x03,0x00, -0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x0f,0x00,0x0f,0x00, -0x05,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x6d,0x61,0x69,0x6e, -0x00,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x3d,0x00,0x00,0x00,0x4c,0x00,0x00,0x00,0xeb,0x00,0x00,0x00, -0xfa,0x00,0x00,0x00,0x80,0x01,0x00,0x00,0x8d,0x01,0x00,0x00, -0xbb,0x02,0x00,0x00,0x04,0x03,0x00,0x00,0x10,0x00,0x06,0x00, -0x04,0x00,0x00,0x00,0x11,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x0b,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x1c,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x10,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x08,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x03,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x10,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x10,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x05,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x14,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x10,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x1c,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x08,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x24,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x10,0x00,0x00,0x00,0x0a,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x28,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x0b,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x2c,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x30,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x10,0x00,0x00,0x00,0x0d,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x34,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x10,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x37,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x3d,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x1a,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x4c,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x1b,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x4f,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x53,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x60,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x62,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x6c,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x03,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x9d,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xaf,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0xb2,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x08,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0xf7,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x10,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0xf8,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0x48,0x00,0x04,0x00, -0xf8,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0xf8,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0xf8,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x07,0x00,0x00,0x00, -0x08,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0xf8,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xfa,0x00,0x00,0x00, -0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0xfa,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x61,0x01,0x00,0x00,0x01,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x62,0x01,0x00,0x00, -0x0b,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x8a,0x01,0x00,0x00,0x06,0x00,0x00,0x00,0x10,0x00,0x00,0x00, -0x48,0x00,0x04,0x00,0x8b,0x01,0x00,0x00,0x00,0x00,0x00,0x00, -0x05,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0x8b,0x01,0x00,0x00, -0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x8b,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x8b,0x01,0x00,0x00, -0x00,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0x08,0x00,0x00,0x00, -0x47,0x00,0x03,0x00,0x8b,0x01,0x00,0x00,0x02,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x8d,0x01,0x00,0x00,0x22,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x8d,0x01,0x00,0x00, -0x21,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0xbb,0x02,0x00,0x00,0x0b,0x00,0x00,0x00,0x18,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x01,0x03,0x00,0x00,0x06,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0x02,0x03,0x00,0x00, -0x00,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x02,0x03,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x02,0x03,0x00,0x00, -0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x04,0x03,0x00,0x00, -0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x04,0x03,0x00,0x00,0x21,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x13,0x00,0x02,0x00,0x02,0x00,0x00,0x00,0x21,0x00,0x03,0x00, -0x03,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x15,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x17,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x03,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x0a,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x0a,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x0d,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x1e,0x00,0x10,0x00, -0x10,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x11,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x10,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x11,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x15,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x14,0x00,0x00,0x00, -0x08,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x15,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x0a,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x27,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, -0x2d,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x37,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x0a,0x00,0x00,0x00,0x3d,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x3e,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00, -0x4c,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x53,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x54,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x37,0x00,0x00,0x00, -0x53,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x58,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x37,0x00,0x00,0x00, -0x53,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x60,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x61,0x00,0x00,0x00,0x86,0x00,0x00,0x00, -0x53,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x32,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x63,0x00,0x00,0x00, -0x86,0x00,0x00,0x00,0x61,0x00,0x00,0x00,0x62,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x67,0x00,0x00,0x00, -0x86,0x00,0x00,0x00,0x61,0x00,0x00,0x00,0x62,0x00,0x00,0x00, -0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x10,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x6d,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x6e,0x00,0x00,0x00,0x86,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x73,0x00,0x00,0x00,0x86,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0x77,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x7c,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, -0x87,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0x8d,0x00,0x00,0x00,0x03,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x98,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x9d,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0x9f,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xae,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00, -0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xaf,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xb0,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x53,0x00,0x00,0x00, -0xaf,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xb1,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x4f,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xb2,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xb3,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0xb1,0x00,0x00,0x00,0xb2,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xb4,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0xb3,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xb5,0x00,0x00,0x00,0x86,0x00,0x00,0x00, -0xb0,0x00,0x00,0x00,0xb4,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xb6,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0xae,0x00,0x00,0x00,0xb5,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xb7,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0xb6,0x00,0x00,0x00,0xb2,0x00,0x00,0x00,0x14,0x00,0x02,0x00, -0xb8,0x00,0x00,0x00,0x16,0x00,0x03,0x00,0xba,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xbb,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xbc,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0xbb,0x00,0x00,0x00, -0xb5,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xbd,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0xbc,0x00,0x00,0x00, -0xb2,0x00,0x00,0x00,0x1c,0x00,0x04,0x00,0xbe,0x00,0x00,0x00, -0xba,0x00,0x00,0x00,0xbd,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0xbf,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0xbe,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0xba,0x00,0x00,0x00,0xc2,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0xc3,0x00,0x00,0x00, -0x07,0x00,0x00,0x00,0xba,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0xc6,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x16,0x00,0x03,0x00,0xe6,0x00,0x00,0x00,0x10,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xe7,0x00,0x00,0x00, -0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xe8,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0xe7,0x00,0x00,0x00, -0x1c,0x00,0x04,0x00,0xe9,0x00,0x00,0x00,0xe6,0x00,0x00,0x00, -0xe8,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0xea,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0xe9,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0xea,0x00,0x00,0x00,0xeb,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xef,0x00,0x00,0x00, -0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x17,0x00,0x04,0x00,0xf5,0x00,0x00,0x00,0xe6,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x18,0x00,0x04,0x00,0xf6,0x00,0x00,0x00, -0xf5,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, -0xf7,0x00,0x00,0x00,0xf6,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, -0xf8,0x00,0x00,0x00,0xf7,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0xf9,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0xf8,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0xf9,0x00,0x00,0x00,0xfa,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0xfc,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0xe6,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0xff,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0xe6,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x04,0x01,0x00,0x00, -0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x11,0x01,0x00,0x00, -0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x1e,0x01,0x00,0x00, -0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x23,0x01,0x00,0x00, -0x03,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x2c,0x01,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x31,0x01,0x00,0x00,0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x3a,0x01,0x00,0x00,0x80,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x3f,0x01,0x00,0x00,0x05,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x48,0x01,0x00,0x00, -0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x4d,0x01,0x00,0x00, -0x06,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x56,0x01,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x5b,0x01,0x00,0x00,0x07,0x00,0x00,0x00,0x32,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x61,0x01,0x00,0x00,0x01,0x00,0x00,0x00, -0x33,0x00,0x06,0x00,0x09,0x00,0x00,0x00,0x62,0x01,0x00,0x00, -0x61,0x01,0x00,0x00,0x39,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x63,0x01,0x00,0x00, -0x51,0x00,0x00,0x00,0x62,0x01,0x00,0x00,0x00,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x64,0x01,0x00,0x00, -0x84,0x00,0x00,0x00,0x63,0x01,0x00,0x00,0x6d,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x65,0x01,0x00,0x00, -0x86,0x00,0x00,0x00,0x64,0x01,0x00,0x00,0x6c,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x7c,0x01,0x00,0x00, -0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x7d,0x01,0x00,0x00, -0x84,0x00,0x00,0x00,0x9d,0x00,0x00,0x00,0x7c,0x01,0x00,0x00, -0x1c,0x00,0x04,0x00,0x7e,0x01,0x00,0x00,0xe6,0x00,0x00,0x00, -0x7d,0x01,0x00,0x00,0x20,0x00,0x04,0x00,0x7f,0x01,0x00,0x00, -0x04,0x00,0x00,0x00,0x7e,0x01,0x00,0x00,0x3b,0x00,0x04,0x00, -0x7f,0x01,0x00,0x00,0x80,0x01,0x00,0x00,0x04,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x84,0x01,0x00,0x00, -0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x1d,0x00,0x03,0x00,0x8a,0x01,0x00,0x00,0xf6,0x00,0x00,0x00, -0x1e,0x00,0x03,0x00,0x8b,0x01,0x00,0x00,0x8a,0x01,0x00,0x00, -0x20,0x00,0x04,0x00,0x8c,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, -0x8b,0x01,0x00,0x00,0x3b,0x00,0x04,0x00,0x8c,0x01,0x00,0x00, -0x8d,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x95,0x01,0x00,0x00,0x80,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xa2,0x01,0x00,0x00,0x80,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xaf,0x01,0x00,0x00,0x80,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xbc,0x01,0x00,0x00,0x80,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xc9,0x01,0x00,0x00,0x80,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xd6,0x01,0x00,0x00,0x80,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xe3,0x01,0x00,0x00,0x80,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xef,0x01,0x00,0x00,0x08,0x01,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xf0,0x01,0x00,0x00, -0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xf3,0x01,0x00,0x00, -0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x0e,0x02,0x00,0x00, -0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00, -0x1c,0x00,0x04,0x00,0x0f,0x02,0x00,0x00,0xe6,0x00,0x00,0x00, -0x0e,0x02,0x00,0x00,0x20,0x00,0x04,0x00,0x10,0x02,0x00,0x00, -0x07,0x00,0x00,0x00,0x0f,0x02,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x20,0x02,0x00,0x00,0x80,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x26,0x02,0x00,0x00,0x07,0x00,0x00,0x00,0xe6,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x3c,0x02,0x00,0x00, -0x84,0x00,0x00,0x00,0xb5,0x00,0x00,0x00,0xb2,0x00,0x00,0x00, -0x1c,0x00,0x04,0x00,0x3d,0x02,0x00,0x00,0xe6,0x00,0x00,0x00, -0x3c,0x02,0x00,0x00,0x20,0x00,0x04,0x00,0x3e,0x02,0x00,0x00, -0x07,0x00,0x00,0x00,0x3d,0x02,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x47,0x02,0x00,0x00,0x86,0x00,0x00,0x00, -0xaf,0x00,0x00,0x00,0xb5,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x4f,0x02,0x00,0x00,0x80,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x7e,0x02,0x00,0x00,0x84,0x00,0x00,0x00, -0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0xb3,0x02,0x00,0x00,0x0d,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00,0xbb,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0x1d,0x00,0x03,0x00,0x01,0x03,0x00,0x00, -0xba,0x00,0x00,0x00,0x1e,0x00,0x03,0x00,0x02,0x03,0x00,0x00, -0x01,0x03,0x00,0x00,0x20,0x00,0x04,0x00,0x03,0x03,0x00,0x00, -0x0c,0x00,0x00,0x00,0x02,0x03,0x00,0x00,0x3b,0x00,0x04,0x00, -0x03,0x03,0x00,0x00,0x04,0x03,0x00,0x00,0x0c,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x09,0x03,0x00,0x00, -0x05,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x16,0x03,0x00,0x00,0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x1f,0x03,0x00,0x00, -0x0c,0x00,0x00,0x00,0xba,0x00,0x00,0x00,0x36,0x00,0x05,0x00, -0x02,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x03,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x05,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0xbf,0x00,0x00,0x00,0xc0,0x00,0x00,0x00, -0x07,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x10,0x02,0x00,0x00, -0x11,0x02,0x00,0x00,0x07,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x3e,0x02,0x00,0x00,0x3f,0x02,0x00,0x00,0x07,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00,0x0e,0x00,0x00,0x00, -0x0b,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x0e,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x17,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x18,0x00,0x00,0x00, -0x0f,0x00,0x00,0x00,0x17,0x00,0x00,0x00,0x89,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x1e,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, -0x17,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0x22,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x22,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x24,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x28,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x27,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x29,0x00,0x00,0x00,0x28,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x2a,0x00,0x00,0x00, -0x1e,0x00,0x00,0x00,0x29,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x2d,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x2f,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x30,0x00,0x00,0x00,0x24,0x00,0x00,0x00, -0x2f,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x32,0x00,0x00,0x00,0x30,0x00,0x00,0x00,0x2a,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x35,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x36,0x00,0x00,0x00,0x35,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x38,0x00,0x00,0x00, -0x36,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x82,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x38,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x3b,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x37,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00,0x3f,0x00,0x00,0x00, -0x3d,0x00,0x00,0x00,0x3e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x3f,0x00,0x00,0x00, -0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x42,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0x3b,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x47,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x3b,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00, -0x49,0x00,0x00,0x00,0x3d,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x4a,0x00,0x00,0x00, -0x49,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00, -0x4d,0x00,0x00,0x00,0x4c,0x00,0x00,0x00,0x3e,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, -0x4d,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x50,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x4f,0x00,0x00,0x00, -0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x55,0x00,0x00,0x00, -0x50,0x00,0x00,0x00,0x54,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x59,0x00,0x00,0x00,0x50,0x00,0x00,0x00, -0x58,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x5d,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x4f,0x00,0x00,0x00, -0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x64,0x00,0x00,0x00, -0x5d,0x00,0x00,0x00,0x63,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x68,0x00,0x00,0x00,0x5d,0x00,0x00,0x00, -0x67,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x6f,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x6e,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x74,0x00,0x00,0x00, -0x4e,0x00,0x00,0x00,0x73,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x78,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x77,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x79,0x00,0x00,0x00,0x78,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x7a,0x00,0x00,0x00,0x47,0x00,0x00,0x00, -0x79,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0x7d,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x7c,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x7e,0x00,0x00,0x00, -0x7d,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x80,0x00,0x00,0x00,0x47,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x83,0x00,0x00,0x00, -0x80,0x00,0x00,0x00,0x79,0x00,0x00,0x00,0x0c,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x26,0x00,0x00,0x00,0x7e,0x00,0x00,0x00,0x83,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x88,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x87,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x89,0x00,0x00,0x00,0x88,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8a,0x00,0x00,0x00, -0x32,0x00,0x00,0x00,0x89,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x8c,0x00,0x00,0x00,0x42,0x00,0x00,0x00, -0x37,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0x8e,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x8d,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x8f,0x00,0x00,0x00, -0x8e,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x90,0x00,0x00,0x00,0x8c,0x00,0x00,0x00,0x8f,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x91,0x00,0x00,0x00, -0x8a,0x00,0x00,0x00,0x90,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x93,0x00,0x00,0x00,0x91,0x00,0x00,0x00, -0x7a,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x94,0x00,0x00,0x00,0x93,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x99,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x98,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x9a,0x00,0x00,0x00,0x99,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9b,0x00,0x00,0x00, -0x0f,0x00,0x00,0x00,0x9a,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x9e,0x00,0x00,0x00,0x4a,0x00,0x00,0x00, -0x9d,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0xa0,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x9f,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xa1,0x00,0x00,0x00, -0xa0,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xa2,0x00,0x00,0x00,0x9e,0x00,0x00,0x00,0xa1,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa3,0x00,0x00,0x00, -0x9b,0x00,0x00,0x00,0xa2,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xa5,0x00,0x00,0x00,0xa3,0x00,0x00,0x00, -0x7a,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xa6,0x00,0x00,0x00,0xa5,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xa8,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xa8,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0x29,0x03,0x00,0x00,0x3e,0x00,0x00,0x00,0x05,0x00,0x00,0x00, -0xc7,0x00,0x00,0x00,0xa9,0x00,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb8,0x00,0x00,0x00,0xb9,0x00,0x00,0x00,0x29,0x03,0x00,0x00, -0xb7,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xaa,0x00,0x00,0x00, -0xa9,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0xb9,0x00,0x00,0x00,0xa9,0x00,0x00,0x00,0xaa,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xa9,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0xc3,0x00,0x00,0x00,0xc4,0x00,0x00,0x00,0xc0,0x00,0x00,0x00, -0x29,0x03,0x00,0x00,0x3e,0x00,0x03,0x00,0xc4,0x00,0x00,0x00, -0xc2,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xc7,0x00,0x00,0x00,0x29,0x03,0x00,0x00,0xc6,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xa8,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xaa,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xca,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xca,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0x42,0x03,0x00,0x00,0xa6,0x00,0x00,0x00, -0xaa,0x00,0x00,0x00,0xf5,0x01,0x00,0x00,0xcd,0x00,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x3e,0x03,0x00,0x00, -0x94,0x00,0x00,0x00,0xaa,0x00,0x00,0x00,0xf2,0x01,0x00,0x00, -0xcd,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0x2a,0x03,0x00,0x00,0x7a,0x00,0x00,0x00,0xaa,0x00,0x00,0x00, -0xa3,0x02,0x00,0x00,0xcd,0x00,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb8,0x00,0x00,0x00,0xd1,0x00,0x00,0x00,0x2a,0x03,0x00,0x00, -0x84,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xcc,0x00,0x00,0x00, -0xcd,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0xd1,0x00,0x00,0x00,0xcb,0x00,0x00,0x00,0xcc,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xcb,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xd3,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xd3,0x00,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x3a,0x03,0x00,0x00, -0x3e,0x00,0x00,0x00,0xcb,0x00,0x00,0x00,0x67,0x01,0x00,0x00, -0xd4,0x00,0x00,0x00,0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00, -0xd9,0x00,0x00,0x00,0x3a,0x03,0x00,0x00,0x37,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0xd5,0x00,0x00,0x00,0xd4,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xd9,0x00,0x00,0x00, -0xd4,0x00,0x00,0x00,0xd5,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xd4,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xde,0x00,0x00,0x00,0x74,0x00,0x00,0x00,0x3a,0x03,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xe1,0x00,0x00,0x00, -0xde,0x00,0x00,0x00,0x8f,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xe2,0x00,0x00,0x00,0xe1,0x00,0x00,0x00, -0x6d,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xe3,0x00,0x00,0x00,0x3e,0x03,0x00,0x00,0xe2,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xe5,0x00,0x00,0x00, -0xe3,0x00,0x00,0x00,0x6f,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xf0,0x00,0x00,0x00,0xde,0x00,0x00,0x00, -0xef,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xf2,0x00,0x00,0x00,0x6f,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xf3,0x00,0x00,0x00, -0xf0,0x00,0x00,0x00,0xf2,0x00,0x00,0x00,0x41,0x00,0x08,0x00, -0xfc,0x00,0x00,0x00,0xfd,0x00,0x00,0x00,0xfa,0x00,0x00,0x00, -0x34,0x00,0x00,0x00,0xe5,0x00,0x00,0x00,0x34,0x00,0x00,0x00, -0x3e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0xe6,0x00,0x00,0x00, -0xfe,0x00,0x00,0x00,0xfd,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0xff,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0xeb,0x00,0x00,0x00, -0xf3,0x00,0x00,0x00,0x3e,0x00,0x03,0x00,0x00,0x01,0x00,0x00, -0xfe,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x05,0x01,0x00,0x00,0xde,0x00,0x00,0x00,0x04,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x08,0x01,0x00,0x00, -0x05,0x01,0x00,0x00,0xf2,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x09,0x01,0x00,0x00,0x08,0x01,0x00,0x00, -0x39,0x00,0x00,0x00,0x41,0x00,0x08,0x00,0xfc,0x00,0x00,0x00, -0x0b,0x01,0x00,0x00,0xfa,0x00,0x00,0x00,0x34,0x00,0x00,0x00, -0xe5,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0xe6,0x00,0x00,0x00,0x0c,0x01,0x00,0x00, -0x0b,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0xff,0x00,0x00,0x00, -0x0d,0x01,0x00,0x00,0xeb,0x00,0x00,0x00,0x09,0x01,0x00,0x00, -0x3e,0x00,0x03,0x00,0x0d,0x01,0x00,0x00,0x0c,0x01,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x12,0x01,0x00,0x00, -0xde,0x00,0x00,0x00,0x11,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x15,0x01,0x00,0x00,0x12,0x01,0x00,0x00, -0xf2,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x16,0x01,0x00,0x00,0x15,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, -0x41,0x00,0x08,0x00,0xfc,0x00,0x00,0x00,0x18,0x01,0x00,0x00, -0xfa,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0xe5,0x00,0x00,0x00, -0x34,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0xe6,0x00,0x00,0x00,0x19,0x01,0x00,0x00,0x18,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0xff,0x00,0x00,0x00,0x1a,0x01,0x00,0x00, -0xeb,0x00,0x00,0x00,0x16,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0x1a,0x01,0x00,0x00,0x19,0x01,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x1f,0x01,0x00,0x00,0xde,0x00,0x00,0x00, -0x1e,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x22,0x01,0x00,0x00,0x1f,0x01,0x00,0x00,0xf2,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x24,0x01,0x00,0x00, -0x22,0x01,0x00,0x00,0x23,0x01,0x00,0x00,0x41,0x00,0x08,0x00, -0xfc,0x00,0x00,0x00,0x26,0x01,0x00,0x00,0xfa,0x00,0x00,0x00, -0x34,0x00,0x00,0x00,0xe5,0x00,0x00,0x00,0x34,0x00,0x00,0x00, -0x23,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0xe6,0x00,0x00,0x00, -0x27,0x01,0x00,0x00,0x26,0x01,0x00,0x00,0x41,0x00,0x05,0x00, -0xff,0x00,0x00,0x00,0x28,0x01,0x00,0x00,0xeb,0x00,0x00,0x00, -0x24,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x28,0x01,0x00,0x00, -0x27,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x2d,0x01,0x00,0x00,0xde,0x00,0x00,0x00,0x2c,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x30,0x01,0x00,0x00, -0x2d,0x01,0x00,0x00,0xf2,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x32,0x01,0x00,0x00,0x30,0x01,0x00,0x00, -0x31,0x01,0x00,0x00,0x41,0x00,0x08,0x00,0xfc,0x00,0x00,0x00, -0x34,0x01,0x00,0x00,0xfa,0x00,0x00,0x00,0x34,0x00,0x00,0x00, -0xe5,0x00,0x00,0x00,0xc6,0x00,0x00,0x00,0x3e,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0xe6,0x00,0x00,0x00,0x35,0x01,0x00,0x00, -0x34,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0xff,0x00,0x00,0x00, -0x36,0x01,0x00,0x00,0xeb,0x00,0x00,0x00,0x32,0x01,0x00,0x00, -0x3e,0x00,0x03,0x00,0x36,0x01,0x00,0x00,0x35,0x01,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3b,0x01,0x00,0x00, -0xde,0x00,0x00,0x00,0x3a,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x3e,0x01,0x00,0x00,0x3b,0x01,0x00,0x00, -0xf2,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x40,0x01,0x00,0x00,0x3e,0x01,0x00,0x00,0x3f,0x01,0x00,0x00, -0x41,0x00,0x08,0x00,0xfc,0x00,0x00,0x00,0x42,0x01,0x00,0x00, -0xfa,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0xe5,0x00,0x00,0x00, -0xc6,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0xe6,0x00,0x00,0x00,0x43,0x01,0x00,0x00,0x42,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0xff,0x00,0x00,0x00,0x44,0x01,0x00,0x00, -0xeb,0x00,0x00,0x00,0x40,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0x44,0x01,0x00,0x00,0x43,0x01,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x49,0x01,0x00,0x00,0xde,0x00,0x00,0x00, -0x48,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x4c,0x01,0x00,0x00,0x49,0x01,0x00,0x00,0xf2,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x4e,0x01,0x00,0x00, -0x4c,0x01,0x00,0x00,0x4d,0x01,0x00,0x00,0x41,0x00,0x08,0x00, -0xfc,0x00,0x00,0x00,0x50,0x01,0x00,0x00,0xfa,0x00,0x00,0x00, -0x34,0x00,0x00,0x00,0xe5,0x00,0x00,0x00,0xc6,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0xe6,0x00,0x00,0x00, -0x51,0x01,0x00,0x00,0x50,0x01,0x00,0x00,0x41,0x00,0x05,0x00, -0xff,0x00,0x00,0x00,0x52,0x01,0x00,0x00,0xeb,0x00,0x00,0x00, -0x4e,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x52,0x01,0x00,0x00, -0x51,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x57,0x01,0x00,0x00,0xde,0x00,0x00,0x00,0x56,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x5a,0x01,0x00,0x00, -0x57,0x01,0x00,0x00,0xf2,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x5c,0x01,0x00,0x00,0x5a,0x01,0x00,0x00, -0x5b,0x01,0x00,0x00,0x41,0x00,0x08,0x00,0xfc,0x00,0x00,0x00, -0x5e,0x01,0x00,0x00,0xfa,0x00,0x00,0x00,0x34,0x00,0x00,0x00, -0xe5,0x00,0x00,0x00,0xc6,0x00,0x00,0x00,0x23,0x01,0x00,0x00, -0x3d,0x00,0x04,0x00,0xe6,0x00,0x00,0x00,0x5f,0x01,0x00,0x00, -0x5e,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0xff,0x00,0x00,0x00, -0x60,0x01,0x00,0x00,0xeb,0x00,0x00,0x00,0x5c,0x01,0x00,0x00, -0x3e,0x00,0x03,0x00,0x60,0x01,0x00,0x00,0x5f,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x67,0x01,0x00,0x00, -0x3a,0x03,0x00,0x00,0x65,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0xd3,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xd5,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x69,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x69,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0x3b,0x03,0x00,0x00,0x3e,0x00,0x00,0x00,0xd5,0x00,0x00,0x00, -0xee,0x01,0x00,0x00,0x6a,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb8,0x00,0x00,0x00,0x6f,0x01,0x00,0x00,0x3b,0x03,0x00,0x00, -0x9d,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x6b,0x01,0x00,0x00, -0x6a,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x6f,0x01,0x00,0x00,0x6a,0x01,0x00,0x00,0x6b,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x6a,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x74,0x01,0x00,0x00,0x74,0x00,0x00,0x00, -0x3b,0x03,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x77,0x01,0x00,0x00,0x74,0x01,0x00,0x00,0xa1,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x78,0x01,0x00,0x00, -0x77,0x01,0x00,0x00,0x6d,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x79,0x01,0x00,0x00,0x42,0x03,0x00,0x00, -0x78,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x7b,0x01,0x00,0x00,0x79,0x01,0x00,0x00,0x6f,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x85,0x01,0x00,0x00, -0x74,0x01,0x00,0x00,0x84,0x01,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x87,0x01,0x00,0x00,0x6f,0x00,0x00,0x00, -0x6d,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x88,0x01,0x00,0x00,0x85,0x01,0x00,0x00,0x87,0x01,0x00,0x00, -0x41,0x00,0x08,0x00,0xfc,0x00,0x00,0x00,0x8f,0x01,0x00,0x00, -0x8d,0x01,0x00,0x00,0x34,0x00,0x00,0x00,0x7b,0x01,0x00,0x00, -0x34,0x00,0x00,0x00,0x3e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0xe6,0x00,0x00,0x00,0x90,0x01,0x00,0x00,0x8f,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0xff,0x00,0x00,0x00,0x91,0x01,0x00,0x00, -0x80,0x01,0x00,0x00,0x88,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0x91,0x01,0x00,0x00,0x90,0x01,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x96,0x01,0x00,0x00,0x74,0x01,0x00,0x00, -0x95,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x99,0x01,0x00,0x00,0x96,0x01,0x00,0x00,0x87,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9a,0x01,0x00,0x00, -0x99,0x01,0x00,0x00,0x39,0x00,0x00,0x00,0x41,0x00,0x08,0x00, -0xfc,0x00,0x00,0x00,0x9c,0x01,0x00,0x00,0x8d,0x01,0x00,0x00, -0x34,0x00,0x00,0x00,0x7b,0x01,0x00,0x00,0x34,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0xe6,0x00,0x00,0x00, -0x9d,0x01,0x00,0x00,0x9c,0x01,0x00,0x00,0x41,0x00,0x05,0x00, -0xff,0x00,0x00,0x00,0x9e,0x01,0x00,0x00,0x80,0x01,0x00,0x00, -0x9a,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x9e,0x01,0x00,0x00, -0x9d,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xa3,0x01,0x00,0x00,0x74,0x01,0x00,0x00,0xa2,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa6,0x01,0x00,0x00, -0xa3,0x01,0x00,0x00,0x87,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xa7,0x01,0x00,0x00,0xa6,0x01,0x00,0x00, -0x0c,0x00,0x00,0x00,0x41,0x00,0x08,0x00,0xfc,0x00,0x00,0x00, -0xa9,0x01,0x00,0x00,0x8d,0x01,0x00,0x00,0x34,0x00,0x00,0x00, -0x7b,0x01,0x00,0x00,0x34,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0xe6,0x00,0x00,0x00,0xaa,0x01,0x00,0x00, -0xa9,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0xff,0x00,0x00,0x00, -0xab,0x01,0x00,0x00,0x80,0x01,0x00,0x00,0xa7,0x01,0x00,0x00, -0x3e,0x00,0x03,0x00,0xab,0x01,0x00,0x00,0xaa,0x01,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xb0,0x01,0x00,0x00, -0x74,0x01,0x00,0x00,0xaf,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xb3,0x01,0x00,0x00,0xb0,0x01,0x00,0x00, -0x87,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xb4,0x01,0x00,0x00,0xb3,0x01,0x00,0x00,0x23,0x01,0x00,0x00, -0x41,0x00,0x08,0x00,0xfc,0x00,0x00,0x00,0xb6,0x01,0x00,0x00, -0x8d,0x01,0x00,0x00,0x34,0x00,0x00,0x00,0x7b,0x01,0x00,0x00, -0x34,0x00,0x00,0x00,0x23,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, -0xe6,0x00,0x00,0x00,0xb7,0x01,0x00,0x00,0xb6,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0xff,0x00,0x00,0x00,0xb8,0x01,0x00,0x00, -0x80,0x01,0x00,0x00,0xb4,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0xb8,0x01,0x00,0x00,0xb7,0x01,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xbd,0x01,0x00,0x00,0x74,0x01,0x00,0x00, -0xbc,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xc0,0x01,0x00,0x00,0xbd,0x01,0x00,0x00,0x87,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xc1,0x01,0x00,0x00, -0xc0,0x01,0x00,0x00,0x31,0x01,0x00,0x00,0x41,0x00,0x08,0x00, -0xfc,0x00,0x00,0x00,0xc3,0x01,0x00,0x00,0x8d,0x01,0x00,0x00, -0x34,0x00,0x00,0x00,0x7b,0x01,0x00,0x00,0xc6,0x00,0x00,0x00, -0x3e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0xe6,0x00,0x00,0x00, -0xc4,0x01,0x00,0x00,0xc3,0x01,0x00,0x00,0x41,0x00,0x05,0x00, -0xff,0x00,0x00,0x00,0xc5,0x01,0x00,0x00,0x80,0x01,0x00,0x00, -0xc1,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0xc5,0x01,0x00,0x00, -0xc4,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xca,0x01,0x00,0x00,0x74,0x01,0x00,0x00,0xc9,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xcd,0x01,0x00,0x00, -0xca,0x01,0x00,0x00,0x87,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xce,0x01,0x00,0x00,0xcd,0x01,0x00,0x00, -0x3f,0x01,0x00,0x00,0x41,0x00,0x08,0x00,0xfc,0x00,0x00,0x00, -0xd0,0x01,0x00,0x00,0x8d,0x01,0x00,0x00,0x34,0x00,0x00,0x00, -0x7b,0x01,0x00,0x00,0xc6,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0xe6,0x00,0x00,0x00,0xd1,0x01,0x00,0x00, -0xd0,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0xff,0x00,0x00,0x00, -0xd2,0x01,0x00,0x00,0x80,0x01,0x00,0x00,0xce,0x01,0x00,0x00, -0x3e,0x00,0x03,0x00,0xd2,0x01,0x00,0x00,0xd1,0x01,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xd7,0x01,0x00,0x00, -0x74,0x01,0x00,0x00,0xd6,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xda,0x01,0x00,0x00,0xd7,0x01,0x00,0x00, -0x87,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xdb,0x01,0x00,0x00,0xda,0x01,0x00,0x00,0x4d,0x01,0x00,0x00, -0x41,0x00,0x08,0x00,0xfc,0x00,0x00,0x00,0xdd,0x01,0x00,0x00, -0x8d,0x01,0x00,0x00,0x34,0x00,0x00,0x00,0x7b,0x01,0x00,0x00, -0xc6,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0xe6,0x00,0x00,0x00,0xde,0x01,0x00,0x00,0xdd,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0xff,0x00,0x00,0x00,0xdf,0x01,0x00,0x00, -0x80,0x01,0x00,0x00,0xdb,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0xdf,0x01,0x00,0x00,0xde,0x01,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xe4,0x01,0x00,0x00,0x74,0x01,0x00,0x00, -0xe3,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xe7,0x01,0x00,0x00,0xe4,0x01,0x00,0x00,0x87,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xe8,0x01,0x00,0x00, -0xe7,0x01,0x00,0x00,0x5b,0x01,0x00,0x00,0x41,0x00,0x08,0x00, -0xfc,0x00,0x00,0x00,0xea,0x01,0x00,0x00,0x8d,0x01,0x00,0x00, -0x34,0x00,0x00,0x00,0x7b,0x01,0x00,0x00,0xc6,0x00,0x00,0x00, -0x23,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0xe6,0x00,0x00,0x00, -0xeb,0x01,0x00,0x00,0xea,0x01,0x00,0x00,0x41,0x00,0x05,0x00, -0xff,0x00,0x00,0x00,0xec,0x01,0x00,0x00,0x80,0x01,0x00,0x00, -0xe8,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0xec,0x01,0x00,0x00, -0xeb,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xee,0x01,0x00,0x00,0x3b,0x03,0x00,0x00,0x65,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0x69,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x6b,0x01,0x00,0x00,0xe0,0x00,0x04,0x00,0x0c,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0xef,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xf2,0x01,0x00,0x00,0x3e,0x03,0x00,0x00, -0xf0,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xf5,0x01,0x00,0x00,0x42,0x03,0x00,0x00,0xf3,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0xf7,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xf7,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0x44,0x03,0x00,0x00,0x3e,0x00,0x00,0x00,0x6b,0x01,0x00,0x00, -0xa1,0x02,0x00,0x00,0xfa,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb8,0x00,0x00,0x00,0xfd,0x01,0x00,0x00,0x44,0x03,0x00,0x00, -0x6c,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xf9,0x01,0x00,0x00, -0xfa,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0xfd,0x01,0x00,0x00,0xf8,0x01,0x00,0x00,0xf9,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xf8,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0xff,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xff,0x01,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x48,0x03,0x00,0x00, -0x3e,0x00,0x00,0x00,0xf8,0x01,0x00,0x00,0x2b,0x02,0x00,0x00, -0x02,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00, -0x05,0x02,0x00,0x00,0x48,0x03,0x00,0x00,0x60,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0x01,0x02,0x00,0x00,0x02,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x05,0x02,0x00,0x00, -0x00,0x02,0x00,0x00,0x01,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x00,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x07,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x07,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0x5a,0x03,0x00,0x00,0x3e,0x00,0x00,0x00, -0x00,0x02,0x00,0x00,0x29,0x02,0x00,0x00,0x08,0x02,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00,0x0d,0x02,0x00,0x00, -0x5a,0x03,0x00,0x00,0x62,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0x09,0x02,0x00,0x00,0x08,0x02,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x0d,0x02,0x00,0x00,0x08,0x02,0x00,0x00, -0x09,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x08,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x13,0x02,0x00,0x00, -0x48,0x03,0x00,0x00,0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x15,0x02,0x00,0x00,0x13,0x02,0x00,0x00, -0x5a,0x03,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x17,0x02,0x00,0x00,0x55,0x00,0x00,0x00,0x53,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x19,0x02,0x00,0x00, -0x48,0x03,0x00,0x00,0x61,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x1a,0x02,0x00,0x00,0x17,0x02,0x00,0x00, -0x19,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x1c,0x02,0x00,0x00,0x64,0x00,0x00,0x00,0x62,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x1d,0x02,0x00,0x00, -0x1a,0x02,0x00,0x00,0x1c,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x1f,0x02,0x00,0x00,0x1d,0x02,0x00,0x00, -0x5a,0x03,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x21,0x02,0x00,0x00,0x1f,0x02,0x00,0x00,0x20,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x23,0x02,0x00,0x00, -0x21,0x02,0x00,0x00,0x44,0x03,0x00,0x00,0x41,0x00,0x05,0x00, -0xff,0x00,0x00,0x00,0x24,0x02,0x00,0x00,0xeb,0x00,0x00,0x00, -0x23,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0xe6,0x00,0x00,0x00, -0x25,0x02,0x00,0x00,0x24,0x02,0x00,0x00,0x41,0x00,0x05,0x00, -0x26,0x02,0x00,0x00,0x27,0x02,0x00,0x00,0x11,0x02,0x00,0x00, -0x15,0x02,0x00,0x00,0x3e,0x00,0x03,0x00,0x27,0x02,0x00,0x00, -0x25,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x29,0x02,0x00,0x00,0x5a,0x03,0x00,0x00,0xc6,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x07,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x09,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x02,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x02,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x2b,0x02,0x00,0x00,0x48,0x03,0x00,0x00, -0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xff,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x01,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x2d,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x2d,0x02,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x49,0x03,0x00,0x00, -0x3e,0x00,0x00,0x00,0x01,0x02,0x00,0x00,0x59,0x02,0x00,0x00, -0x30,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00, -0x33,0x02,0x00,0x00,0x49,0x03,0x00,0x00,0xb5,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0x2f,0x02,0x00,0x00,0x30,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x33,0x02,0x00,0x00, -0x2e,0x02,0x00,0x00,0x2f,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x2e,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x35,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x35,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0x57,0x03,0x00,0x00,0x3e,0x00,0x00,0x00, -0x2e,0x02,0x00,0x00,0x57,0x02,0x00,0x00,0x36,0x02,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00,0x3b,0x02,0x00,0x00, -0x57,0x03,0x00,0x00,0xb2,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0x37,0x02,0x00,0x00,0x36,0x02,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x3b,0x02,0x00,0x00,0x36,0x02,0x00,0x00, -0x37,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x36,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x41,0x02,0x00,0x00, -0x49,0x03,0x00,0x00,0xb2,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x43,0x02,0x00,0x00,0x41,0x02,0x00,0x00, -0x57,0x03,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x45,0x02,0x00,0x00,0x59,0x00,0x00,0x00,0xaf,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x48,0x02,0x00,0x00, -0x49,0x03,0x00,0x00,0x47,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x49,0x02,0x00,0x00,0x45,0x02,0x00,0x00, -0x48,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x4b,0x02,0x00,0x00,0x68,0x00,0x00,0x00,0xb2,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x4c,0x02,0x00,0x00, -0x49,0x02,0x00,0x00,0x4b,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x4e,0x02,0x00,0x00,0x4c,0x02,0x00,0x00, -0x57,0x03,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x50,0x02,0x00,0x00,0x4e,0x02,0x00,0x00,0x4f,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x52,0x02,0x00,0x00, -0x50,0x02,0x00,0x00,0x44,0x03,0x00,0x00,0x41,0x00,0x05,0x00, -0xff,0x00,0x00,0x00,0x53,0x02,0x00,0x00,0x80,0x01,0x00,0x00, -0x52,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0xe6,0x00,0x00,0x00, -0x54,0x02,0x00,0x00,0x53,0x02,0x00,0x00,0x41,0x00,0x05,0x00, -0x26,0x02,0x00,0x00,0x55,0x02,0x00,0x00,0x3f,0x02,0x00,0x00, -0x43,0x02,0x00,0x00,0x3e,0x00,0x03,0x00,0x55,0x02,0x00,0x00, -0x54,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x57,0x02,0x00,0x00,0x57,0x03,0x00,0x00,0xc6,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x35,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x37,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x30,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x30,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x59,0x02,0x00,0x00,0x49,0x03,0x00,0x00, -0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x2d,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x2f,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x5b,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x5b,0x02,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x4a,0x03,0x00,0x00, -0x3e,0x00,0x00,0x00,0x2f,0x02,0x00,0x00,0x9f,0x02,0x00,0x00, -0x5e,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00, -0x61,0x02,0x00,0x00,0x4a,0x03,0x00,0x00,0xb5,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0x5d,0x02,0x00,0x00,0x5e,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x61,0x02,0x00,0x00, -0x5c,0x02,0x00,0x00,0x5d,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x5c,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x63,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x63,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0x4e,0x03,0x00,0x00,0x3e,0x00,0x00,0x00, -0x5c,0x02,0x00,0x00,0x9d,0x02,0x00,0x00,0x66,0x02,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00,0x69,0x02,0x00,0x00, -0x4e,0x03,0x00,0x00,0x60,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0x65,0x02,0x00,0x00,0x66,0x02,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x69,0x02,0x00,0x00,0x64,0x02,0x00,0x00, -0x65,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x64,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0x6b,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x6b,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0x50,0x03,0x00,0x00,0x3e,0x00,0x00,0x00,0x64,0x02,0x00,0x00, -0x9b,0x02,0x00,0x00,0x6e,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb8,0x00,0x00,0x00,0x71,0x02,0x00,0x00,0x50,0x03,0x00,0x00, -0xb2,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x6d,0x02,0x00,0x00, -0x6e,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x71,0x02,0x00,0x00,0x6c,0x02,0x00,0x00,0x6d,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x6c,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x73,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x73,0x02,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x52,0x03,0x00,0x00, -0x3e,0x00,0x00,0x00,0x6c,0x02,0x00,0x00,0x99,0x02,0x00,0x00, -0x74,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00, -0x79,0x02,0x00,0x00,0x52,0x03,0x00,0x00,0x62,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0x75,0x02,0x00,0x00,0x74,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x79,0x02,0x00,0x00, -0x74,0x02,0x00,0x00,0x75,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x74,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x7b,0x02,0x00,0x00,0x4a,0x03,0x00,0x00,0xb2,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x7d,0x02,0x00,0x00, -0x7b,0x02,0x00,0x00,0x50,0x03,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x7f,0x02,0x00,0x00,0x7d,0x02,0x00,0x00, -0x7e,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x81,0x02,0x00,0x00,0x4e,0x03,0x00,0x00,0x62,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x82,0x02,0x00,0x00, -0x7f,0x02,0x00,0x00,0x81,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x84,0x02,0x00,0x00,0x82,0x02,0x00,0x00, -0x52,0x03,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x88,0x02,0x00,0x00,0x81,0x02,0x00,0x00,0x52,0x03,0x00,0x00, -0x41,0x00,0x05,0x00,0x26,0x02,0x00,0x00,0x89,0x02,0x00,0x00, -0x11,0x02,0x00,0x00,0x88,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, -0xe6,0x00,0x00,0x00,0x8a,0x02,0x00,0x00,0x89,0x02,0x00,0x00, -0x73,0x00,0x04,0x00,0xba,0x00,0x00,0x00,0x8b,0x02,0x00,0x00, -0x8a,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x26,0x02,0x00,0x00, -0x90,0x02,0x00,0x00,0x3f,0x02,0x00,0x00,0x7d,0x02,0x00,0x00, -0x3d,0x00,0x04,0x00,0xe6,0x00,0x00,0x00,0x91,0x02,0x00,0x00, -0x90,0x02,0x00,0x00,0x73,0x00,0x04,0x00,0xba,0x00,0x00,0x00, -0x92,0x02,0x00,0x00,0x91,0x02,0x00,0x00,0x41,0x00,0x05,0x00, -0xc3,0x00,0x00,0x00,0x94,0x02,0x00,0x00,0xc0,0x00,0x00,0x00, -0x84,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0xba,0x00,0x00,0x00, -0x95,0x02,0x00,0x00,0x94,0x02,0x00,0x00,0x0c,0x00,0x08,0x00, -0xba,0x00,0x00,0x00,0x96,0x02,0x00,0x00,0x01,0x00,0x00,0x00, -0x32,0x00,0x00,0x00,0x8b,0x02,0x00,0x00,0x92,0x02,0x00,0x00, -0x95,0x02,0x00,0x00,0x3e,0x00,0x03,0x00,0x94,0x02,0x00,0x00, -0x96,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x99,0x02,0x00,0x00,0x52,0x03,0x00,0x00,0xc6,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x73,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x75,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x6e,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x6e,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x9b,0x02,0x00,0x00,0x50,0x03,0x00,0x00, -0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x6b,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x6d,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x66,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x66,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9d,0x02,0x00,0x00, -0x4e,0x03,0x00,0x00,0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x63,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x65,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0x5e,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x5e,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x9f,0x02,0x00,0x00,0x4a,0x03,0x00,0x00,0xc6,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x5b,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x5d,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0xfa,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xfa,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xa1,0x02,0x00,0x00,0x44,0x03,0x00,0x00, -0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xf7,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xf9,0x01,0x00,0x00,0xe0,0x00,0x04,0x00, -0x0c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0xef,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0xcd,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xcd,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xa3,0x02,0x00,0x00,0x2a,0x03,0x00,0x00,0x6c,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xca,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xcc,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xa8,0x02,0x00,0x00,0x55,0x00,0x00,0x00,0x53,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa9,0x02,0x00,0x00, -0x8c,0x00,0x00,0x00,0xa8,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xae,0x02,0x00,0x00,0x59,0x00,0x00,0x00, -0xaf,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xaf,0x02,0x00,0x00,0x9e,0x00,0x00,0x00,0xae,0x02,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0xb4,0x02,0x00,0x00, -0x12,0x00,0x00,0x00,0xb3,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xb5,0x02,0x00,0x00,0xb4,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xb6,0x02,0x00,0x00, -0x0f,0x00,0x00,0x00,0xb5,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xba,0x02,0x00,0x00,0x47,0x00,0x00,0x00, -0xb5,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00, -0xbc,0x02,0x00,0x00,0xbb,0x02,0x00,0x00,0x0c,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xbd,0x02,0x00,0x00, -0xbc,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xbe,0x02,0x00,0x00,0xba,0x02,0x00,0x00,0xbd,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xbf,0x02,0x00,0x00, -0xb6,0x02,0x00,0x00,0xbe,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0xc1,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0xc1,0x02,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x2b,0x03,0x00,0x00, -0x3e,0x00,0x00,0x00,0xcc,0x00,0x00,0x00,0x28,0x03,0x00,0x00, -0xc4,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00, -0xc7,0x02,0x00,0x00,0x2b,0x03,0x00,0x00,0xb5,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0xc3,0x02,0x00,0x00,0xc4,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xc7,0x02,0x00,0x00, -0xc2,0x02,0x00,0x00,0xc3,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0xc2,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0xc9,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0xc9,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0x2c,0x03,0x00,0x00,0x3e,0x00,0x00,0x00, -0xc2,0x02,0x00,0x00,0x26,0x03,0x00,0x00,0xcc,0x02,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00,0xcf,0x02,0x00,0x00, -0x2c,0x03,0x00,0x00,0x60,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0xcb,0x02,0x00,0x00,0xcc,0x02,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xcf,0x02,0x00,0x00,0xca,0x02,0x00,0x00, -0xcb,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0xca,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xd3,0x02,0x00,0x00, -0x2c,0x03,0x00,0x00,0x61,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xd4,0x02,0x00,0x00,0xa9,0x02,0x00,0x00, -0xd3,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xd6,0x02,0x00,0x00,0x64,0x00,0x00,0x00,0x62,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xd7,0x02,0x00,0x00, -0xd4,0x02,0x00,0x00,0xd6,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xdb,0x02,0x00,0x00,0x2b,0x03,0x00,0x00, -0x47,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xdc,0x02,0x00,0x00,0xaf,0x02,0x00,0x00,0xdb,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xde,0x02,0x00,0x00, -0x68,0x00,0x00,0x00,0xb2,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xdf,0x02,0x00,0x00,0xdc,0x02,0x00,0x00, -0xde,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0xe1,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0xe1,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0x2e,0x03,0x00,0x00,0x3e,0x00,0x00,0x00, -0xca,0x02,0x00,0x00,0x24,0x03,0x00,0x00,0xe4,0x02,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00,0xe7,0x02,0x00,0x00, -0x2e,0x03,0x00,0x00,0xb2,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0xe3,0x02,0x00,0x00,0xe4,0x02,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xe7,0x02,0x00,0x00,0xe2,0x02,0x00,0x00, -0xe3,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0xe2,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0xe9,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0xe9,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0x30,0x03,0x00,0x00,0x3e,0x00,0x00,0x00,0xe2,0x02,0x00,0x00, -0x22,0x03,0x00,0x00,0xec,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb8,0x00,0x00,0x00,0xef,0x02,0x00,0x00,0x30,0x03,0x00,0x00, -0x62,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xeb,0x02,0x00,0x00, -0xec,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0xef,0x02,0x00,0x00,0xea,0x02,0x00,0x00,0xeb,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0xea,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xf2,0x02,0x00,0x00,0xd7,0x02,0x00,0x00, -0x30,0x03,0x00,0x00,0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00, -0xf5,0x02,0x00,0x00,0xf2,0x02,0x00,0x00,0x36,0x00,0x00,0x00, -0xf7,0x00,0x03,0x00,0xf7,0x02,0x00,0x00,0x00,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xf5,0x02,0x00,0x00,0xf6,0x02,0x00,0x00, -0xf7,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0xf6,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xfa,0x02,0x00,0x00, -0xdf,0x02,0x00,0x00,0x2e,0x03,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0xfb,0x02,0x00,0x00,0x12,0x00,0x00,0x00, -0xc6,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xfc,0x02,0x00,0x00,0xfb,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb8,0x00,0x00,0x00,0xfd,0x02,0x00,0x00,0xfa,0x02,0x00,0x00, -0xfc,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0xf7,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0xf7,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, -0xb8,0x00,0x00,0x00,0xfe,0x02,0x00,0x00,0xf5,0x02,0x00,0x00, -0xea,0x02,0x00,0x00,0xfd,0x02,0x00,0x00,0xf6,0x02,0x00,0x00, -0xf7,0x00,0x03,0x00,0x00,0x03,0x00,0x00,0x00,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xfe,0x02,0x00,0x00,0xff,0x02,0x00,0x00, -0x00,0x03,0x00,0x00,0xf8,0x00,0x02,0x00,0xff,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x08,0x03,0x00,0x00, -0xdf,0x02,0x00,0x00,0x2e,0x03,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x0a,0x03,0x00,0x00,0x12,0x00,0x00,0x00, -0x09,0x03,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x0b,0x03,0x00,0x00,0x0a,0x03,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x0c,0x03,0x00,0x00,0x08,0x03,0x00,0x00, -0x0b,0x03,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x0d,0x03,0x00,0x00,0xbf,0x02,0x00,0x00,0x0c,0x03,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x0f,0x03,0x00,0x00, -0x0d,0x03,0x00,0x00,0xd7,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x11,0x03,0x00,0x00,0x0f,0x03,0x00,0x00, -0x30,0x03,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x13,0x03,0x00,0x00,0x2b,0x03,0x00,0x00,0xb2,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x15,0x03,0x00,0x00, -0x13,0x03,0x00,0x00,0x2e,0x03,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x17,0x03,0x00,0x00,0x15,0x03,0x00,0x00, -0x16,0x03,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x19,0x03,0x00,0x00,0x2c,0x03,0x00,0x00,0x62,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x1a,0x03,0x00,0x00, -0x17,0x03,0x00,0x00,0x19,0x03,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x1c,0x03,0x00,0x00,0x1a,0x03,0x00,0x00, -0x30,0x03,0x00,0x00,0x41,0x00,0x05,0x00,0xc3,0x00,0x00,0x00, -0x1d,0x03,0x00,0x00,0xc0,0x00,0x00,0x00,0x1c,0x03,0x00,0x00, -0x3d,0x00,0x04,0x00,0xba,0x00,0x00,0x00,0x1e,0x03,0x00,0x00, -0x1d,0x03,0x00,0x00,0x41,0x00,0x06,0x00,0x1f,0x03,0x00,0x00, -0x20,0x03,0x00,0x00,0x04,0x03,0x00,0x00,0x34,0x00,0x00,0x00, -0x11,0x03,0x00,0x00,0x3e,0x00,0x03,0x00,0x20,0x03,0x00,0x00, -0x1e,0x03,0x00,0x00,0xf9,0x00,0x02,0x00,0x00,0x03,0x00,0x00, -0xf8,0x00,0x02,0x00,0x00,0x03,0x00,0x00,0xf9,0x00,0x02,0x00, -0xec,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0xec,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x22,0x03,0x00,0x00, -0x30,0x03,0x00,0x00,0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xe9,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0xeb,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0xe4,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0xe4,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x24,0x03,0x00,0x00,0x2e,0x03,0x00,0x00,0xc6,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xe1,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0xe3,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0xcc,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0xcc,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x26,0x03,0x00,0x00,0x2c,0x03,0x00,0x00, -0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xc9,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0xcb,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0xc4,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0xc4,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x28,0x03,0x00,0x00, -0x2b,0x03,0x00,0x00,0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xc1,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0xc3,0x02,0x00,0x00, -0xfd,0x00,0x01,0x00,0x38,0x00,0x01,0x00, -}; -const uint64_t matmul_f16_aligned_s_len = 11936; - -unsigned char matmul_f16_aligned_s_fp32_data[] = { -0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, -0xf3,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, -0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x51,0x11,0x00,0x00, -0x0b,0x00,0x06,0x00,0x01,0x00,0x00,0x00,0x47,0x4c,0x53,0x4c, -0x2e,0x73,0x74,0x64,0x2e,0x34,0x35,0x30,0x00,0x00,0x00,0x00, -0x0e,0x00,0x03,0x00,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x0f,0x00,0x0f,0x00,0x05,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x6d,0x61,0x69,0x6e,0x00,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x3d,0x00,0x00,0x00,0x4c,0x00,0x00,0x00, -0xea,0x00,0x00,0x00,0xf9,0x00,0x00,0x00,0x4b,0x01,0x00,0x00, -0x58,0x01,0x00,0x00,0x53,0x02,0x00,0x00,0x9c,0x02,0x00,0x00, -0x10,0x00,0x06,0x00,0x04,0x00,0x00,0x00,0x11,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x0b,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x1c,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x10,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x08,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x03,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x10,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x14,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x18,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x07,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x10,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x24,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x0a,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x10,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x2c,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x30,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x0d,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0x47,0x00,0x03,0x00, -0x10,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x37,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x3d,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x1a,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x4c,0x00,0x00,0x00, -0x0b,0x00,0x00,0x00,0x1b,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x4f,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x53,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x60,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x62,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x07,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x6c,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x03,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x9d,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0xaf,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x05,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0xb2,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x08,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xf6,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x48,0x00,0x04,0x00, -0xf7,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0xf7,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, -0xf7,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0xf9,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0xf9,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x2c,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x2d,0x01,0x00,0x00,0x0b,0x00,0x00,0x00,0x19,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x55,0x01,0x00,0x00,0x06,0x00,0x00,0x00, -0x08,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0x56,0x01,0x00,0x00, -0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x56,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x56,0x01,0x00,0x00, -0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x58,0x01,0x00,0x00, -0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x58,0x01,0x00,0x00,0x21,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x53,0x02,0x00,0x00,0x0b,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x99,0x02,0x00,0x00, -0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00, -0x9a,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x19,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x9a,0x02,0x00,0x00,0x00,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, -0x9a,0x02,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x9c,0x02,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x9c,0x02,0x00,0x00,0x21,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x13,0x00,0x02,0x00,0x02,0x00,0x00,0x00, -0x21,0x00,0x03,0x00,0x03,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x15,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x17,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x0a,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x0d,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x1e,0x00,0x10,0x00,0x10,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x11,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x11,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x15,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, -0x14,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x15,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x0a,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, -0x27,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0x2d,0x00,0x00,0x00,0x07,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x34,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0xf6,0x00,0x00,0x00,0x1e,0x01,0x00,0x00, 0x00,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x37,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00,0x3d,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x3e,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x0a,0x00,0x00,0x00,0x4c,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x4f,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x53,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x54,0x00,0x00,0x00,0x86,0x00,0x00,0x00, -0x37,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x58,0x00,0x00,0x00,0x86,0x00,0x00,0x00, -0x37,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x32,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x61,0x00,0x00,0x00, -0x86,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x60,0x00,0x00,0x00, -0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x62,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x63,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x61,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x67,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x61,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x6e,0x00,0x00,0x00, -0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x73,0x00,0x00,0x00, -0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x77,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, -0x7c,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0x87,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x8d,0x00,0x00,0x00, -0x03,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, -0x98,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x32,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x9d,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x9f,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xae,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xaf,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xb0,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0x53,0x00,0x00,0x00,0xaf,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xb1,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0x4f,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x32,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xb2,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xb3,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0xb1,0x00,0x00,0x00,0xb2,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xb4,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0xb3,0x00,0x00,0x00,0x60,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xb5,0x00,0x00,0x00, -0x86,0x00,0x00,0x00,0xb0,0x00,0x00,0x00,0xb4,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xb6,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0xae,0x00,0x00,0x00,0xb5,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xb7,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0xb6,0x00,0x00,0x00,0xb2,0x00,0x00,0x00, -0x14,0x00,0x02,0x00,0xb8,0x00,0x00,0x00,0x16,0x00,0x03,0x00, -0xba,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xbb,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xbc,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0xbb,0x00,0x00,0x00,0xb5,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xbd,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0xbc,0x00,0x00,0x00,0xb2,0x00,0x00,0x00,0x1c,0x00,0x04,0x00, -0xbe,0x00,0x00,0x00,0xba,0x00,0x00,0x00,0xbd,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0xbf,0x00,0x00,0x00,0x07,0x00,0x00,0x00, -0xbe,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0xba,0x00,0x00,0x00, -0xc2,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0xc3,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0xba,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0xc6,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xe6,0x00,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xe7,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x37,0x00,0x00,0x00, -0xe6,0x00,0x00,0x00,0x1c,0x00,0x04,0x00,0xe8,0x00,0x00,0x00, -0xba,0x00,0x00,0x00,0xe7,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0xe9,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0xe8,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0xe9,0x00,0x00,0x00,0xea,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xee,0x00,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x16,0x00,0x03,0x00,0xf4,0x00,0x00,0x00, -0x10,0x00,0x00,0x00,0x17,0x00,0x04,0x00,0xf5,0x00,0x00,0x00, -0xf4,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, -0xf6,0x00,0x00,0x00,0xf5,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, -0xf7,0x00,0x00,0x00,0xf6,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0xf8,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0xf7,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0xf8,0x00,0x00,0x00,0xf9,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0xfb,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0xf4,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0xff,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0xba,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x04,0x01,0x00,0x00, -0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x12,0x01,0x00,0x00, -0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x20,0x01,0x00,0x00, -0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x25,0x01,0x00,0x00, -0x03,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x2c,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x33,0x00,0x06,0x00, -0x09,0x00,0x00,0x00,0x2d,0x01,0x00,0x00,0x2c,0x01,0x00,0x00, +0x20,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x33,0x00,0x06,0x00, +0x09,0x00,0x00,0x00,0x21,0x01,0x00,0x00,0x20,0x01,0x00,0x00, 0x39,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x2e,0x01,0x00,0x00,0x51,0x00,0x00,0x00, -0x2d,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x2f,0x01,0x00,0x00,0x84,0x00,0x00,0x00, -0x2e,0x01,0x00,0x00,0x6d,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x30,0x01,0x00,0x00,0x86,0x00,0x00,0x00, -0x2f,0x01,0x00,0x00,0x6c,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x47,0x01,0x00,0x00,0x80,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x22,0x01,0x00,0x00,0x51,0x00,0x00,0x00, +0x21,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x23,0x01,0x00,0x00,0x84,0x00,0x00,0x00, +0x22,0x01,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x24,0x01,0x00,0x00,0x86,0x00,0x00,0x00, +0x23,0x01,0x00,0x00,0x6c,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x42,0x01,0x00,0x00,0x80,0x00,0x00,0x00, 0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x48,0x01,0x00,0x00,0x84,0x00,0x00,0x00, -0x9d,0x00,0x00,0x00,0x47,0x01,0x00,0x00,0x1c,0x00,0x04,0x00, -0x49,0x01,0x00,0x00,0xba,0x00,0x00,0x00,0x48,0x01,0x00,0x00, -0x20,0x00,0x04,0x00,0x4a,0x01,0x00,0x00,0x04,0x00,0x00,0x00, -0x49,0x01,0x00,0x00,0x3b,0x00,0x04,0x00,0x4a,0x01,0x00,0x00, -0x4b,0x01,0x00,0x00,0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x4f,0x01,0x00,0x00,0x80,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x43,0x01,0x00,0x00,0x84,0x00,0x00,0x00, +0xa6,0x00,0x00,0x00,0x42,0x01,0x00,0x00,0x1c,0x00,0x04,0x00, +0x44,0x01,0x00,0x00,0xf6,0x00,0x00,0x00,0x43,0x01,0x00,0x00, +0x20,0x00,0x04,0x00,0x45,0x01,0x00,0x00,0x04,0x00,0x00,0x00, +0x44,0x01,0x00,0x00,0x3b,0x00,0x04,0x00,0x45,0x01,0x00,0x00, +0x46,0x01,0x00,0x00,0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x4a,0x01,0x00,0x00,0x80,0x00,0x00,0x00, 0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, -0x55,0x01,0x00,0x00,0xf5,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, -0x56,0x01,0x00,0x00,0x55,0x01,0x00,0x00,0x20,0x00,0x04,0x00, -0x57,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x56,0x01,0x00,0x00, -0x3b,0x00,0x04,0x00,0x57,0x01,0x00,0x00,0x58,0x01,0x00,0x00, +0x4e,0x01,0x00,0x00,0xf6,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, +0x4f,0x01,0x00,0x00,0x4e,0x01,0x00,0x00,0x20,0x00,0x04,0x00, +0x50,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x4f,0x01,0x00,0x00, +0x3b,0x00,0x04,0x00,0x50,0x01,0x00,0x00,0x51,0x01,0x00,0x00, 0x0c,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x61,0x01,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, +0x63,0x01,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, 0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x6f,0x01,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, +0x68,0x01,0x00,0x00,0x51,0x00,0x00,0x00,0x21,0x01,0x00,0x00, +0x00,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x69,0x01,0x00,0x00,0x84,0x00,0x00,0x00,0x68,0x01,0x00,0x00, 0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x7d,0x01,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x8a,0x01,0x00,0x00,0x08,0x01,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x8b,0x01,0x00,0x00,0x86,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x8e,0x01,0x00,0x00,0x86,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xa9,0x01,0x00,0x00,0x84,0x00,0x00,0x00, +0x6a,0x01,0x00,0x00,0x86,0x00,0x00,0x00,0x69,0x01,0x00,0x00, +0x6c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x6d,0x01,0x00,0x00,0x08,0x01,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x6e,0x01,0x00,0x00,0x86,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x71,0x01,0x00,0x00,0x86,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x8c,0x01,0x00,0x00,0x84,0x00,0x00,0x00, 0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x1c,0x00,0x04,0x00, -0xaa,0x01,0x00,0x00,0xba,0x00,0x00,0x00,0xa9,0x01,0x00,0x00, -0x20,0x00,0x04,0x00,0xab,0x01,0x00,0x00,0x07,0x00,0x00,0x00, -0xaa,0x01,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xbb,0x01,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, +0x8d,0x01,0x00,0x00,0xf6,0x00,0x00,0x00,0x8c,0x01,0x00,0x00, +0x20,0x00,0x04,0x00,0x8e,0x01,0x00,0x00,0x07,0x00,0x00,0x00, +0x8d,0x01,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x9e,0x01,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, +0x39,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0xa4,0x01,0x00,0x00, +0x07,0x00,0x00,0x00,0xf6,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xba,0x01,0x00,0x00,0x84,0x00,0x00,0x00, +0xbe,0x00,0x00,0x00,0xbb,0x00,0x00,0x00,0x1c,0x00,0x04,0x00, +0xbb,0x01,0x00,0x00,0xf6,0x00,0x00,0x00,0xba,0x01,0x00,0x00, +0x20,0x00,0x04,0x00,0xbc,0x01,0x00,0x00,0x07,0x00,0x00,0x00, +0xbb,0x01,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xc5,0x01,0x00,0x00,0x86,0x00,0x00,0x00,0xb8,0x00,0x00,0x00, +0xbe,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xcd,0x01,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, 0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xd6,0x01,0x00,0x00,0x84,0x00,0x00,0x00,0xb5,0x00,0x00,0x00, -0xb2,0x00,0x00,0x00,0x1c,0x00,0x04,0x00,0xd7,0x01,0x00,0x00, -0xba,0x00,0x00,0x00,0xd6,0x01,0x00,0x00,0x20,0x00,0x04,0x00, -0xd8,0x01,0x00,0x00,0x07,0x00,0x00,0x00,0xd7,0x01,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xe1,0x01,0x00,0x00, -0x86,0x00,0x00,0x00,0xaf,0x00,0x00,0x00,0xb5,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xe9,0x01,0x00,0x00, -0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x18,0x02,0x00,0x00, +0xfc,0x01,0x00,0x00,0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00, +0x62,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, +0x31,0x02,0x00,0x00,0x0d,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0x0a,0x00,0x00,0x00,0x39,0x02,0x00,0x00,0x01,0x00,0x00,0x00, +0x1d,0x00,0x03,0x00,0x7f,0x02,0x00,0x00,0xc3,0x00,0x00,0x00, +0x1e,0x00,0x03,0x00,0x80,0x02,0x00,0x00,0x7f,0x02,0x00,0x00, +0x20,0x00,0x04,0x00,0x81,0x02,0x00,0x00,0x0c,0x00,0x00,0x00, +0x80,0x02,0x00,0x00,0x3b,0x00,0x04,0x00,0x81,0x02,0x00,0x00, +0x82,0x02,0x00,0x00,0x0c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0x87,0x02,0x00,0x00,0x05,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x94,0x02,0x00,0x00, 0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x4b,0x02,0x00,0x00, -0x0d,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00, -0x53,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, -0x99,0x02,0x00,0x00,0xba,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, -0x9a,0x02,0x00,0x00,0x99,0x02,0x00,0x00,0x20,0x00,0x04,0x00, -0x9b,0x02,0x00,0x00,0x0c,0x00,0x00,0x00,0x9a,0x02,0x00,0x00, -0x3b,0x00,0x04,0x00,0x9b,0x02,0x00,0x00,0x9c,0x02,0x00,0x00, -0x0c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, -0xa1,0x02,0x00,0x00,0x05,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xae,0x02,0x00,0x00,0x84,0x00,0x00,0x00, -0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0xb7,0x02,0x00,0x00,0x0c,0x00,0x00,0x00,0xba,0x00,0x00,0x00, -0x36,0x00,0x05,0x00,0x02,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x05,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0xbf,0x00,0x00,0x00, -0xc0,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0xab,0x01,0x00,0x00,0xac,0x01,0x00,0x00,0x07,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0xd8,0x01,0x00,0x00,0xd9,0x01,0x00,0x00, -0x07,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00, -0x0e,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, -0x0e,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x14,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x17,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x17,0x00,0x00,0x00, -0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x1e,0x00,0x00,0x00, -0x0f,0x00,0x00,0x00,0x17,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x21,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x24,0x00,0x00,0x00,0x18,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0x28,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x27,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x29,0x00,0x00,0x00, -0x28,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x2a,0x00,0x00,0x00,0x1e,0x00,0x00,0x00,0x29,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x2d,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x2f,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x30,0x00,0x00,0x00, -0x24,0x00,0x00,0x00,0x2f,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x30,0x00,0x00,0x00, -0x2a,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0x35,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x34,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x36,0x00,0x00,0x00, -0x35,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x38,0x00,0x00,0x00,0x36,0x00,0x00,0x00,0x37,0x00,0x00,0x00, -0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, -0x38,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x3b,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, -0x37,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00, -0x3f,0x00,0x00,0x00,0x3d,0x00,0x00,0x00,0x3e,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x3f,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x42,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x3b,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x47,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0x3b,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x0d,0x00,0x00,0x00,0x49,0x00,0x00,0x00,0x3d,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x4a,0x00,0x00,0x00,0x49,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x0d,0x00,0x00,0x00,0x4d,0x00,0x00,0x00,0x4c,0x00,0x00,0x00, -0x3e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x4e,0x00,0x00,0x00,0x4d,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x50,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, -0x4f,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x55,0x00,0x00,0x00,0x50,0x00,0x00,0x00,0x54,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x59,0x00,0x00,0x00, -0x50,0x00,0x00,0x00,0x58,0x00,0x00,0x00,0x89,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x5d,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, -0x4f,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x64,0x00,0x00,0x00,0x5d,0x00,0x00,0x00,0x63,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x68,0x00,0x00,0x00, -0x5d,0x00,0x00,0x00,0x67,0x00,0x00,0x00,0x89,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x6f,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, -0x6e,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x74,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x73,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x78,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x77,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x79,0x00,0x00,0x00,0x78,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x7a,0x00,0x00,0x00, -0x47,0x00,0x00,0x00,0x79,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x7d,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x7c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x7e,0x00,0x00,0x00,0x7d,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x80,0x00,0x00,0x00,0x47,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x83,0x00,0x00,0x00,0x80,0x00,0x00,0x00,0x79,0x00,0x00,0x00, -0x0c,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x26,0x00,0x00,0x00,0x7e,0x00,0x00,0x00, -0x83,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0x88,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x87,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x89,0x00,0x00,0x00, -0x88,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x8a,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x89,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8c,0x00,0x00,0x00, -0x42,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x8e,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x8d,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x8f,0x00,0x00,0x00,0x8e,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x90,0x00,0x00,0x00,0x8c,0x00,0x00,0x00, -0x8f,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x91,0x00,0x00,0x00,0x8a,0x00,0x00,0x00,0x90,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x93,0x00,0x00,0x00, -0x91,0x00,0x00,0x00,0x7a,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x94,0x00,0x00,0x00,0x93,0x00,0x00,0x00, -0x6d,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0x99,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x98,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x9a,0x00,0x00,0x00, -0x99,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x9b,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x9a,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9e,0x00,0x00,0x00, -0x4a,0x00,0x00,0x00,0x9d,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0xa0,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x9f,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xa1,0x00,0x00,0x00,0xa0,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xa2,0x00,0x00,0x00,0x9e,0x00,0x00,0x00, -0xa1,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xa3,0x00,0x00,0x00,0x9b,0x00,0x00,0x00,0xa2,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa5,0x00,0x00,0x00, -0xa3,0x00,0x00,0x00,0x7a,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xa6,0x00,0x00,0x00,0xa5,0x00,0x00,0x00, -0x6d,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xa8,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xa8,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xc1,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, -0x05,0x00,0x00,0x00,0xc7,0x00,0x00,0x00,0xa9,0x00,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00,0xb9,0x00,0x00,0x00, -0xc1,0x02,0x00,0x00,0xb7,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0xaa,0x00,0x00,0x00,0xa9,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xb9,0x00,0x00,0x00,0xa9,0x00,0x00,0x00, -0xaa,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xa9,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0xc3,0x00,0x00,0x00,0xc4,0x00,0x00,0x00, -0xc0,0x00,0x00,0x00,0xc1,0x02,0x00,0x00,0x3e,0x00,0x03,0x00, -0xc4,0x00,0x00,0x00,0xc2,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xc7,0x00,0x00,0x00,0xc1,0x02,0x00,0x00, -0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xa8,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xaa,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xca,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xca,0x00,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xda,0x02,0x00,0x00, -0xa6,0x00,0x00,0x00,0xaa,0x00,0x00,0x00,0x90,0x01,0x00,0x00, -0xcd,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xd6,0x02,0x00,0x00,0x94,0x00,0x00,0x00,0xaa,0x00,0x00,0x00, -0x8d,0x01,0x00,0x00,0xcd,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xc2,0x02,0x00,0x00,0x7a,0x00,0x00,0x00, -0xaa,0x00,0x00,0x00,0x3b,0x02,0x00,0x00,0xcd,0x00,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00,0xd1,0x00,0x00,0x00, -0xc2,0x02,0x00,0x00,0x84,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0xcc,0x00,0x00,0x00,0xcd,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xd1,0x00,0x00,0x00,0xcb,0x00,0x00,0x00, -0xcc,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xcb,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xd3,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xd3,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xd2,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0xcb,0x00,0x00,0x00, -0x32,0x01,0x00,0x00,0xd4,0x00,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb8,0x00,0x00,0x00,0xd9,0x00,0x00,0x00,0xd2,0x02,0x00,0x00, -0x37,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xd5,0x00,0x00,0x00, -0xd4,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0xd9,0x00,0x00,0x00,0xd4,0x00,0x00,0x00,0xd5,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xd4,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xde,0x00,0x00,0x00,0x74,0x00,0x00,0x00, -0xd2,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xe1,0x00,0x00,0x00,0xde,0x00,0x00,0x00,0x8f,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xe2,0x00,0x00,0x00, -0xe1,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xe3,0x00,0x00,0x00,0xd6,0x02,0x00,0x00, -0xe2,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xe5,0x00,0x00,0x00,0xe3,0x00,0x00,0x00,0x6f,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xef,0x00,0x00,0x00, -0xde,0x00,0x00,0x00,0xee,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xf1,0x00,0x00,0x00,0x6f,0x00,0x00,0x00, -0x6d,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xf2,0x00,0x00,0x00,0xef,0x00,0x00,0x00,0xf1,0x00,0x00,0x00, -0x41,0x00,0x07,0x00,0xfb,0x00,0x00,0x00,0xfc,0x00,0x00,0x00, -0xf9,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0xe5,0x00,0x00,0x00, -0x3e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0xf4,0x00,0x00,0x00, -0xfd,0x00,0x00,0x00,0xfc,0x00,0x00,0x00,0x73,0x00,0x04,0x00, -0xba,0x00,0x00,0x00,0xfe,0x00,0x00,0x00,0xfd,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0xff,0x00,0x00,0x00,0x00,0x01,0x00,0x00, -0xea,0x00,0x00,0x00,0xf2,0x00,0x00,0x00,0x3e,0x00,0x03,0x00, -0x00,0x01,0x00,0x00,0xfe,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x05,0x01,0x00,0x00,0xde,0x00,0x00,0x00, -0x04,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x08,0x01,0x00,0x00,0x05,0x01,0x00,0x00,0xf1,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x09,0x01,0x00,0x00, -0x08,0x01,0x00,0x00,0x39,0x00,0x00,0x00,0x41,0x00,0x07,0x00, -0xfb,0x00,0x00,0x00,0x0b,0x01,0x00,0x00,0xf9,0x00,0x00,0x00, -0x34,0x00,0x00,0x00,0xe5,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0xf4,0x00,0x00,0x00,0x0c,0x01,0x00,0x00, -0x0b,0x01,0x00,0x00,0x73,0x00,0x04,0x00,0xba,0x00,0x00,0x00, -0x0d,0x01,0x00,0x00,0x0c,0x01,0x00,0x00,0x41,0x00,0x05,0x00, -0xff,0x00,0x00,0x00,0x0e,0x01,0x00,0x00,0xea,0x00,0x00,0x00, -0x09,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x0e,0x01,0x00,0x00, -0x0d,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x13,0x01,0x00,0x00,0xde,0x00,0x00,0x00,0x12,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x16,0x01,0x00,0x00, -0x13,0x01,0x00,0x00,0xf1,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x17,0x01,0x00,0x00,0x16,0x01,0x00,0x00, -0x0c,0x00,0x00,0x00,0x41,0x00,0x07,0x00,0xfb,0x00,0x00,0x00, -0x19,0x01,0x00,0x00,0xf9,0x00,0x00,0x00,0x34,0x00,0x00,0x00, -0xe5,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0xf4,0x00,0x00,0x00,0x1a,0x01,0x00,0x00,0x19,0x01,0x00,0x00, -0x73,0x00,0x04,0x00,0xba,0x00,0x00,0x00,0x1b,0x01,0x00,0x00, -0x1a,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0xff,0x00,0x00,0x00, -0x1c,0x01,0x00,0x00,0xea,0x00,0x00,0x00,0x17,0x01,0x00,0x00, -0x3e,0x00,0x03,0x00,0x1c,0x01,0x00,0x00,0x1b,0x01,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x21,0x01,0x00,0x00, -0xde,0x00,0x00,0x00,0x20,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x24,0x01,0x00,0x00,0x21,0x01,0x00,0x00, -0xf1,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x26,0x01,0x00,0x00,0x24,0x01,0x00,0x00,0x25,0x01,0x00,0x00, -0x41,0x00,0x07,0x00,0xfb,0x00,0x00,0x00,0x28,0x01,0x00,0x00, -0xf9,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0xe5,0x00,0x00,0x00, -0x25,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0xf4,0x00,0x00,0x00, -0x29,0x01,0x00,0x00,0x28,0x01,0x00,0x00,0x73,0x00,0x04,0x00, -0xba,0x00,0x00,0x00,0x2a,0x01,0x00,0x00,0x29,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0xff,0x00,0x00,0x00,0x2b,0x01,0x00,0x00, -0xea,0x00,0x00,0x00,0x26,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0x2b,0x01,0x00,0x00,0x2a,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x32,0x01,0x00,0x00,0xd2,0x02,0x00,0x00, -0x30,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xd3,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xd5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x34,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x34,0x01,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xd3,0x02,0x00,0x00, -0x3e,0x00,0x00,0x00,0xd5,0x00,0x00,0x00,0x89,0x01,0x00,0x00, -0x35,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00, -0x3a,0x01,0x00,0x00,0xd3,0x02,0x00,0x00,0x9d,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0x36,0x01,0x00,0x00,0x35,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x3a,0x01,0x00,0x00, -0x35,0x01,0x00,0x00,0x36,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x35,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x3f,0x01,0x00,0x00,0x74,0x00,0x00,0x00,0xd3,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x42,0x01,0x00,0x00, -0x3f,0x01,0x00,0x00,0xa1,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x43,0x01,0x00,0x00,0x42,0x01,0x00,0x00, -0x6d,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x44,0x01,0x00,0x00,0xda,0x02,0x00,0x00,0x43,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x46,0x01,0x00,0x00, -0x44,0x01,0x00,0x00,0x6f,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x50,0x01,0x00,0x00,0x3f,0x01,0x00,0x00, -0x4f,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x52,0x01,0x00,0x00,0x6f,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x53,0x01,0x00,0x00, -0x50,0x01,0x00,0x00,0x52,0x01,0x00,0x00,0x41,0x00,0x07,0x00, -0xfb,0x00,0x00,0x00,0x5a,0x01,0x00,0x00,0x58,0x01,0x00,0x00, -0x34,0x00,0x00,0x00,0x46,0x01,0x00,0x00,0x3e,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0xf4,0x00,0x00,0x00,0x5b,0x01,0x00,0x00, -0x5a,0x01,0x00,0x00,0x73,0x00,0x04,0x00,0xba,0x00,0x00,0x00, -0x5c,0x01,0x00,0x00,0x5b,0x01,0x00,0x00,0x41,0x00,0x05,0x00, -0xff,0x00,0x00,0x00,0x5d,0x01,0x00,0x00,0x4b,0x01,0x00,0x00, -0x53,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x5d,0x01,0x00,0x00, -0x5c,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x62,0x01,0x00,0x00,0x3f,0x01,0x00,0x00,0x61,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x65,0x01,0x00,0x00, -0x62,0x01,0x00,0x00,0x52,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x66,0x01,0x00,0x00,0x65,0x01,0x00,0x00, -0x39,0x00,0x00,0x00,0x41,0x00,0x07,0x00,0xfb,0x00,0x00,0x00, -0x68,0x01,0x00,0x00,0x58,0x01,0x00,0x00,0x34,0x00,0x00,0x00, -0x46,0x01,0x00,0x00,0x39,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0xf4,0x00,0x00,0x00,0x69,0x01,0x00,0x00,0x68,0x01,0x00,0x00, -0x73,0x00,0x04,0x00,0xba,0x00,0x00,0x00,0x6a,0x01,0x00,0x00, -0x69,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0xff,0x00,0x00,0x00, -0x6b,0x01,0x00,0x00,0x4b,0x01,0x00,0x00,0x66,0x01,0x00,0x00, -0x3e,0x00,0x03,0x00,0x6b,0x01,0x00,0x00,0x6a,0x01,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x70,0x01,0x00,0x00, -0x3f,0x01,0x00,0x00,0x6f,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x73,0x01,0x00,0x00,0x70,0x01,0x00,0x00, -0x52,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x74,0x01,0x00,0x00,0x73,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, -0x41,0x00,0x07,0x00,0xfb,0x00,0x00,0x00,0x76,0x01,0x00,0x00, -0x58,0x01,0x00,0x00,0x34,0x00,0x00,0x00,0x46,0x01,0x00,0x00, -0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0xf4,0x00,0x00,0x00, -0x77,0x01,0x00,0x00,0x76,0x01,0x00,0x00,0x73,0x00,0x04,0x00, -0xba,0x00,0x00,0x00,0x78,0x01,0x00,0x00,0x77,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0xff,0x00,0x00,0x00,0x79,0x01,0x00,0x00, -0x4b,0x01,0x00,0x00,0x74,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0x79,0x01,0x00,0x00,0x78,0x01,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x7e,0x01,0x00,0x00,0x3f,0x01,0x00,0x00, -0x7d,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x81,0x01,0x00,0x00,0x7e,0x01,0x00,0x00,0x52,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x82,0x01,0x00,0x00, -0x81,0x01,0x00,0x00,0x25,0x01,0x00,0x00,0x41,0x00,0x07,0x00, -0xfb,0x00,0x00,0x00,0x84,0x01,0x00,0x00,0x58,0x01,0x00,0x00, -0x34,0x00,0x00,0x00,0x46,0x01,0x00,0x00,0x25,0x01,0x00,0x00, -0x3d,0x00,0x04,0x00,0xf4,0x00,0x00,0x00,0x85,0x01,0x00,0x00, -0x84,0x01,0x00,0x00,0x73,0x00,0x04,0x00,0xba,0x00,0x00,0x00, -0x86,0x01,0x00,0x00,0x85,0x01,0x00,0x00,0x41,0x00,0x05,0x00, -0xff,0x00,0x00,0x00,0x87,0x01,0x00,0x00,0x4b,0x01,0x00,0x00, -0x82,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x87,0x01,0x00,0x00, -0x86,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x89,0x01,0x00,0x00,0xd3,0x02,0x00,0x00,0x30,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0x34,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x36,0x01,0x00,0x00,0xe0,0x00,0x04,0x00,0x0c,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x8a,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x8d,0x01,0x00,0x00,0xd6,0x02,0x00,0x00, -0x8b,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x90,0x01,0x00,0x00,0xda,0x02,0x00,0x00,0x8e,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0x92,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x92,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xdc,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0x36,0x01,0x00,0x00, -0x39,0x02,0x00,0x00,0x95,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb8,0x00,0x00,0x00,0x98,0x01,0x00,0x00,0xdc,0x02,0x00,0x00, -0x6c,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x94,0x01,0x00,0x00, -0x95,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x98,0x01,0x00,0x00,0x93,0x01,0x00,0x00,0x94,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x93,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0x9a,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x9a,0x01,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xe0,0x02,0x00,0x00, -0x3e,0x00,0x00,0x00,0x93,0x01,0x00,0x00,0xc5,0x01,0x00,0x00, -0x9d,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00, -0xa0,0x01,0x00,0x00,0xe0,0x02,0x00,0x00,0x60,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0x9c,0x01,0x00,0x00,0x9d,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xa0,0x01,0x00,0x00, -0x9b,0x01,0x00,0x00,0x9c,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x9b,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xa2,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xa2,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xf2,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, -0x9b,0x01,0x00,0x00,0xc3,0x01,0x00,0x00,0xa3,0x01,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00,0xa8,0x01,0x00,0x00, -0xf2,0x02,0x00,0x00,0x62,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0xa4,0x01,0x00,0x00,0xa3,0x01,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xa8,0x01,0x00,0x00,0xa3,0x01,0x00,0x00, -0xa4,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xa3,0x01,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xae,0x01,0x00,0x00, -0xe0,0x02,0x00,0x00,0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xb0,0x01,0x00,0x00,0xae,0x01,0x00,0x00, -0xf2,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xb2,0x01,0x00,0x00,0x55,0x00,0x00,0x00,0x53,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xb4,0x01,0x00,0x00, -0xe0,0x02,0x00,0x00,0x61,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xb5,0x01,0x00,0x00,0xb2,0x01,0x00,0x00, -0xb4,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xb7,0x01,0x00,0x00,0x64,0x00,0x00,0x00,0x62,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xb8,0x01,0x00,0x00, -0xb5,0x01,0x00,0x00,0xb7,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xba,0x01,0x00,0x00,0xb8,0x01,0x00,0x00, -0xf2,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xbc,0x01,0x00,0x00,0xba,0x01,0x00,0x00,0xbb,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xbe,0x01,0x00,0x00, -0xbc,0x01,0x00,0x00,0xdc,0x02,0x00,0x00,0x41,0x00,0x05,0x00, -0xff,0x00,0x00,0x00,0xbf,0x01,0x00,0x00,0xea,0x00,0x00,0x00, -0xbe,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0xba,0x00,0x00,0x00, -0xc0,0x01,0x00,0x00,0xbf,0x01,0x00,0x00,0x41,0x00,0x05,0x00, -0xc3,0x00,0x00,0x00,0xc1,0x01,0x00,0x00,0xac,0x01,0x00,0x00, -0xb0,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0xc1,0x01,0x00,0x00, -0xc0,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xc3,0x01,0x00,0x00,0xf2,0x02,0x00,0x00,0xc6,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xa2,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xa4,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x9d,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x9d,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xc5,0x01,0x00,0x00,0xe0,0x02,0x00,0x00, -0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x9a,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x9c,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0xc7,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xc7,0x01,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xe1,0x02,0x00,0x00, -0x3e,0x00,0x00,0x00,0x9c,0x01,0x00,0x00,0xf3,0x01,0x00,0x00, -0xca,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00, -0xcd,0x01,0x00,0x00,0xe1,0x02,0x00,0x00,0xb5,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0xc9,0x01,0x00,0x00,0xca,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xcd,0x01,0x00,0x00, -0xc8,0x01,0x00,0x00,0xc9,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xc8,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xcf,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xcf,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xef,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, -0xc8,0x01,0x00,0x00,0xf1,0x01,0x00,0x00,0xd0,0x01,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00,0xd5,0x01,0x00,0x00, -0xef,0x02,0x00,0x00,0xb2,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0xd1,0x01,0x00,0x00,0xd0,0x01,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xd5,0x01,0x00,0x00,0xd0,0x01,0x00,0x00, -0xd1,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xd0,0x01,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xdb,0x01,0x00,0x00, -0xe1,0x02,0x00,0x00,0xb2,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xdd,0x01,0x00,0x00,0xdb,0x01,0x00,0x00, -0xef,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xdf,0x01,0x00,0x00,0x59,0x00,0x00,0x00,0xaf,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xe2,0x01,0x00,0x00, -0xe1,0x02,0x00,0x00,0xe1,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xe3,0x01,0x00,0x00,0xdf,0x01,0x00,0x00, -0xe2,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xe5,0x01,0x00,0x00,0x68,0x00,0x00,0x00,0xb2,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xe6,0x01,0x00,0x00, -0xe3,0x01,0x00,0x00,0xe5,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xe8,0x01,0x00,0x00,0xe6,0x01,0x00,0x00, -0xef,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xea,0x01,0x00,0x00,0xe8,0x01,0x00,0x00,0xe9,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xec,0x01,0x00,0x00, -0xea,0x01,0x00,0x00,0xdc,0x02,0x00,0x00,0x41,0x00,0x05,0x00, -0xff,0x00,0x00,0x00,0xed,0x01,0x00,0x00,0x4b,0x01,0x00,0x00, -0xec,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0xba,0x00,0x00,0x00, -0xee,0x01,0x00,0x00,0xed,0x01,0x00,0x00,0x41,0x00,0x05,0x00, -0xc3,0x00,0x00,0x00,0xef,0x01,0x00,0x00,0xd9,0x01,0x00,0x00, -0xdd,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0xef,0x01,0x00,0x00, -0xee,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xf1,0x01,0x00,0x00,0xef,0x02,0x00,0x00,0xc6,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xcf,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xd1,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xca,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xca,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xf3,0x01,0x00,0x00,0xe1,0x02,0x00,0x00, -0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xc7,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xc9,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0xf5,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xf5,0x01,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xe2,0x02,0x00,0x00, -0x3e,0x00,0x00,0x00,0xc9,0x01,0x00,0x00,0x37,0x02,0x00,0x00, -0xf8,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00, -0xfb,0x01,0x00,0x00,0xe2,0x02,0x00,0x00,0xb5,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0xf7,0x01,0x00,0x00,0xf8,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xfb,0x01,0x00,0x00, -0xf6,0x01,0x00,0x00,0xf7,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xf6,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xfd,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xfd,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xe6,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, -0xf6,0x01,0x00,0x00,0x35,0x02,0x00,0x00,0x00,0x02,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00,0x03,0x02,0x00,0x00, -0xe6,0x02,0x00,0x00,0x60,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0xff,0x01,0x00,0x00,0x00,0x02,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x03,0x02,0x00,0x00,0xfe,0x01,0x00,0x00, -0xff,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xfe,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0x05,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x05,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xe8,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0xfe,0x01,0x00,0x00, -0x33,0x02,0x00,0x00,0x08,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb8,0x00,0x00,0x00,0x0b,0x02,0x00,0x00,0xe8,0x02,0x00,0x00, -0xb2,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x07,0x02,0x00,0x00, -0x08,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x0b,0x02,0x00,0x00,0x06,0x02,0x00,0x00,0x07,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x06,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x0d,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x0d,0x02,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xea,0x02,0x00,0x00, -0x3e,0x00,0x00,0x00,0x06,0x02,0x00,0x00,0x31,0x02,0x00,0x00, -0x0e,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00, -0x13,0x02,0x00,0x00,0xea,0x02,0x00,0x00,0x62,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0x0f,0x02,0x00,0x00,0x0e,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x13,0x02,0x00,0x00, -0x0e,0x02,0x00,0x00,0x0f,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x0e,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x15,0x02,0x00,0x00,0xe2,0x02,0x00,0x00,0xb2,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x17,0x02,0x00,0x00, -0x15,0x02,0x00,0x00,0xe8,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x19,0x02,0x00,0x00,0x17,0x02,0x00,0x00, -0x18,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x1b,0x02,0x00,0x00,0xe6,0x02,0x00,0x00,0x62,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x1c,0x02,0x00,0x00, -0x19,0x02,0x00,0x00,0x1b,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x1e,0x02,0x00,0x00,0x1c,0x02,0x00,0x00, -0xea,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x22,0x02,0x00,0x00,0x1b,0x02,0x00,0x00,0xea,0x02,0x00,0x00, -0x41,0x00,0x05,0x00,0xc3,0x00,0x00,0x00,0x23,0x02,0x00,0x00, -0xac,0x01,0x00,0x00,0x22,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, -0xba,0x00,0x00,0x00,0x24,0x02,0x00,0x00,0x23,0x02,0x00,0x00, -0x41,0x00,0x05,0x00,0xc3,0x00,0x00,0x00,0x29,0x02,0x00,0x00, -0xd9,0x01,0x00,0x00,0x17,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, -0xba,0x00,0x00,0x00,0x2a,0x02,0x00,0x00,0x29,0x02,0x00,0x00, -0x41,0x00,0x05,0x00,0xc3,0x00,0x00,0x00,0x2c,0x02,0x00,0x00, -0xc0,0x00,0x00,0x00,0x1e,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, -0xba,0x00,0x00,0x00,0x2d,0x02,0x00,0x00,0x2c,0x02,0x00,0x00, -0x0c,0x00,0x08,0x00,0xba,0x00,0x00,0x00,0x2e,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x24,0x02,0x00,0x00, -0x2a,0x02,0x00,0x00,0x2d,0x02,0x00,0x00,0x3e,0x00,0x03,0x00, -0x2c,0x02,0x00,0x00,0x2e,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x31,0x02,0x00,0x00,0xea,0x02,0x00,0x00, -0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x0d,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x0f,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x08,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x08,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x33,0x02,0x00,0x00, -0xe8,0x02,0x00,0x00,0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x05,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x07,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0x00,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x00,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x35,0x02,0x00,0x00,0xe6,0x02,0x00,0x00,0xc6,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xfd,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xff,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xf8,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xf8,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x37,0x02,0x00,0x00,0xe2,0x02,0x00,0x00, -0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xf5,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xf7,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0x95,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x95,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x39,0x02,0x00,0x00, -0xdc,0x02,0x00,0x00,0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x92,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x94,0x01,0x00,0x00, -0xe0,0x00,0x04,0x00,0x0c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x8a,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xcd,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xcd,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x3b,0x02,0x00,0x00,0xc2,0x02,0x00,0x00, -0x6c,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xca,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xcc,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x40,0x02,0x00,0x00,0x55,0x00,0x00,0x00, -0x53,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x41,0x02,0x00,0x00,0x8c,0x00,0x00,0x00,0x40,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x46,0x02,0x00,0x00, -0x59,0x00,0x00,0x00,0xaf,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x47,0x02,0x00,0x00,0x9e,0x00,0x00,0x00, -0x46,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0x4c,0x02,0x00,0x00,0x12,0x00,0x00,0x00,0x4b,0x02,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x4d,0x02,0x00,0x00, -0x4c,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x4e,0x02,0x00,0x00,0x0f,0x00,0x00,0x00,0x4d,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x52,0x02,0x00,0x00, -0x47,0x00,0x00,0x00,0x4d,0x02,0x00,0x00,0x41,0x00,0x05,0x00, -0x0d,0x00,0x00,0x00,0x54,0x02,0x00,0x00,0x53,0x02,0x00,0x00, -0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x55,0x02,0x00,0x00,0x54,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x56,0x02,0x00,0x00,0x52,0x02,0x00,0x00, -0x55,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x57,0x02,0x00,0x00,0x4e,0x02,0x00,0x00,0x56,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0x59,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x59,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xc3,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0xcc,0x00,0x00,0x00, -0xc0,0x02,0x00,0x00,0x5c,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb8,0x00,0x00,0x00,0x5f,0x02,0x00,0x00,0xc3,0x02,0x00,0x00, -0xb5,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x5b,0x02,0x00,0x00, -0x5c,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x5f,0x02,0x00,0x00,0x5a,0x02,0x00,0x00,0x5b,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x5a,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x61,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x61,0x02,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xc4,0x02,0x00,0x00, -0x3e,0x00,0x00,0x00,0x5a,0x02,0x00,0x00,0xbe,0x02,0x00,0x00, -0x64,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00, -0x67,0x02,0x00,0x00,0xc4,0x02,0x00,0x00,0x60,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0x63,0x02,0x00,0x00,0x64,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x67,0x02,0x00,0x00, -0x62,0x02,0x00,0x00,0x63,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x62,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x6b,0x02,0x00,0x00,0xc4,0x02,0x00,0x00,0x61,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x6c,0x02,0x00,0x00, -0x41,0x02,0x00,0x00,0x6b,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x6e,0x02,0x00,0x00,0x64,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x6f,0x02,0x00,0x00,0x6c,0x02,0x00,0x00,0x6e,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x73,0x02,0x00,0x00, -0xc3,0x02,0x00,0x00,0xe1,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x74,0x02,0x00,0x00,0x47,0x02,0x00,0x00, -0x73,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x76,0x02,0x00,0x00,0x68,0x00,0x00,0x00,0xb2,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x77,0x02,0x00,0x00, -0x74,0x02,0x00,0x00,0x76,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x79,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x79,0x02,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xc6,0x02,0x00,0x00, -0x3e,0x00,0x00,0x00,0x62,0x02,0x00,0x00,0xbc,0x02,0x00,0x00, -0x7c,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00, -0x7f,0x02,0x00,0x00,0xc6,0x02,0x00,0x00,0xb2,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0x7b,0x02,0x00,0x00,0x7c,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x7f,0x02,0x00,0x00, -0x7a,0x02,0x00,0x00,0x7b,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x7a,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x81,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x81,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xc8,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, -0x7a,0x02,0x00,0x00,0xba,0x02,0x00,0x00,0x84,0x02,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00,0x87,0x02,0x00,0x00, -0xc8,0x02,0x00,0x00,0x62,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0x83,0x02,0x00,0x00,0x84,0x02,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x87,0x02,0x00,0x00,0x82,0x02,0x00,0x00, -0x83,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x82,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8a,0x02,0x00,0x00, -0x6f,0x02,0x00,0x00,0xc8,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb8,0x00,0x00,0x00,0x8d,0x02,0x00,0x00,0x8a,0x02,0x00,0x00, -0x36,0x00,0x00,0x00,0xf7,0x00,0x03,0x00,0x8f,0x02,0x00,0x00, -0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x8d,0x02,0x00,0x00, -0x8e,0x02,0x00,0x00,0x8f,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x8e,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x92,0x02,0x00,0x00,0x77,0x02,0x00,0x00,0xc6,0x02,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x93,0x02,0x00,0x00, -0x12,0x00,0x00,0x00,0xc6,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x94,0x02,0x00,0x00,0x93,0x02,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00,0x95,0x02,0x00,0x00, -0x92,0x02,0x00,0x00,0x94,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x8f,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x8f,0x02,0x00,0x00, -0xf5,0x00,0x07,0x00,0xb8,0x00,0x00,0x00,0x96,0x02,0x00,0x00, -0x8d,0x02,0x00,0x00,0x82,0x02,0x00,0x00,0x95,0x02,0x00,0x00, -0x8e,0x02,0x00,0x00,0xf7,0x00,0x03,0x00,0x98,0x02,0x00,0x00, -0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x96,0x02,0x00,0x00, -0x97,0x02,0x00,0x00,0x98,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x97,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xa0,0x02,0x00,0x00,0x77,0x02,0x00,0x00,0xc6,0x02,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0xa2,0x02,0x00,0x00, -0x12,0x00,0x00,0x00,0xa1,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xa3,0x02,0x00,0x00,0xa2,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa4,0x02,0x00,0x00, -0xa0,0x02,0x00,0x00,0xa3,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xa5,0x02,0x00,0x00,0x57,0x02,0x00,0x00, -0xa4,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xa7,0x02,0x00,0x00,0xa5,0x02,0x00,0x00,0x6f,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa9,0x02,0x00,0x00, -0xa7,0x02,0x00,0x00,0xc8,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xab,0x02,0x00,0x00,0xc3,0x02,0x00,0x00, -0xb2,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xad,0x02,0x00,0x00,0xab,0x02,0x00,0x00,0xc6,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xaf,0x02,0x00,0x00, -0xad,0x02,0x00,0x00,0xae,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xb1,0x02,0x00,0x00,0xc4,0x02,0x00,0x00, -0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xb2,0x02,0x00,0x00,0xaf,0x02,0x00,0x00,0xb1,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xb4,0x02,0x00,0x00, -0xb2,0x02,0x00,0x00,0xc8,0x02,0x00,0x00,0x41,0x00,0x05,0x00, -0xc3,0x00,0x00,0x00,0xb5,0x02,0x00,0x00,0xc0,0x00,0x00,0x00, -0xb4,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0xba,0x00,0x00,0x00, -0xb6,0x02,0x00,0x00,0xb5,0x02,0x00,0x00,0x41,0x00,0x06,0x00, -0xb7,0x02,0x00,0x00,0xb8,0x02,0x00,0x00,0x9c,0x02,0x00,0x00, -0x34,0x00,0x00,0x00,0xa9,0x02,0x00,0x00,0x3e,0x00,0x03,0x00, -0xb8,0x02,0x00,0x00,0xb6,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x98,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x98,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0x84,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x84,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xba,0x02,0x00,0x00,0xc8,0x02,0x00,0x00,0xc6,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x81,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x83,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x7c,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x7c,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xbc,0x02,0x00,0x00,0xc6,0x02,0x00,0x00, -0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x79,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x7b,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x64,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x64,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xbe,0x02,0x00,0x00, -0xc4,0x02,0x00,0x00,0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x61,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x63,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0x5c,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x5c,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xc0,0x02,0x00,0x00,0xc3,0x02,0x00,0x00,0xc6,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x59,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x5b,0x02,0x00,0x00,0xfd,0x00,0x01,0x00,0x38,0x00,0x01,0x00, - -}; -const uint64_t matmul_f16_aligned_s_fp32_len = 10512; - -unsigned char matmul_f16_f32_aligned_l_data[] = { -0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, -0x65,0x03,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, -0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x09,0x00,0x00,0x00, -0x11,0x00,0x02,0x00,0x51,0x11,0x00,0x00,0x0b,0x00,0x06,0x00, -0x01,0x00,0x00,0x00,0x47,0x4c,0x53,0x4c,0x2e,0x73,0x74,0x64, -0x2e,0x34,0x35,0x30,0x00,0x00,0x00,0x00,0x0e,0x00,0x03,0x00, -0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x0f,0x00,0x0f,0x00, -0x05,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x6d,0x61,0x69,0x6e, -0x00,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x3d,0x00,0x00,0x00,0x4c,0x00,0x00,0x00,0xeb,0x00,0x00,0x00, -0xfa,0x00,0x00,0x00,0x80,0x01,0x00,0x00,0x8f,0x01,0x00,0x00, -0xc6,0x02,0x00,0x00,0x0f,0x03,0x00,0x00,0x10,0x00,0x06,0x00, -0x04,0x00,0x00,0x00,0x11,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x0b,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x1c,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x10,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x08,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x03,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x10,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x10,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x05,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x14,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x10,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x1c,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x08,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x24,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x10,0x00,0x00,0x00,0x0a,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x28,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x0b,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x2c,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x30,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x10,0x00,0x00,0x00,0x0d,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x34,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x10,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x37,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x3d,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x1a,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x4c,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x1b,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x4f,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x53,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x60,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x62,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x6c,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x03,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x9d,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xaf,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0xb2,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x08,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0xf7,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x10,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0xf8,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0x48,0x00,0x04,0x00, -0xf8,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0xf8,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0xf8,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x07,0x00,0x00,0x00, -0x08,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0xf8,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xfa,0x00,0x00,0x00, -0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0xfa,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x61,0x01,0x00,0x00,0x01,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x62,0x01,0x00,0x00, -0x0b,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x8c,0x01,0x00,0x00,0x06,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x48,0x00,0x04,0x00,0x8d,0x01,0x00,0x00,0x00,0x00,0x00,0x00, -0x05,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0x8d,0x01,0x00,0x00, -0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x8d,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x8d,0x01,0x00,0x00, -0x00,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0x10,0x00,0x00,0x00, -0x47,0x00,0x03,0x00,0x8d,0x01,0x00,0x00,0x02,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x8f,0x01,0x00,0x00,0x22,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x8f,0x01,0x00,0x00, -0x21,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0xc6,0x02,0x00,0x00,0x0b,0x00,0x00,0x00,0x18,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x0c,0x03,0x00,0x00,0x06,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0x0d,0x03,0x00,0x00, -0x00,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x0d,0x03,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x0d,0x03,0x00,0x00, -0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x0f,0x03,0x00,0x00, -0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x0f,0x03,0x00,0x00,0x21,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x13,0x00,0x02,0x00,0x02,0x00,0x00,0x00,0x21,0x00,0x03,0x00, -0x03,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x15,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x17,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x03,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x0a,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x0a,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x0d,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x1e,0x00,0x10,0x00, -0x10,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x11,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x10,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x11,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x15,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x14,0x00,0x00,0x00, -0x08,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x15,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x0a,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x27,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, -0x2d,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x37,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x0a,0x00,0x00,0x00,0x3d,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x3e,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00, -0x4c,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x53,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x54,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x37,0x00,0x00,0x00, -0x53,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x58,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x37,0x00,0x00,0x00, -0x53,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x60,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x61,0x00,0x00,0x00,0x86,0x00,0x00,0x00, -0x53,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x32,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x63,0x00,0x00,0x00, -0x86,0x00,0x00,0x00,0x61,0x00,0x00,0x00,0x62,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x67,0x00,0x00,0x00, -0x86,0x00,0x00,0x00,0x61,0x00,0x00,0x00,0x62,0x00,0x00,0x00, -0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x10,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x6d,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x6e,0x00,0x00,0x00,0x86,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x73,0x00,0x00,0x00,0x86,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0x77,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x7c,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, -0x87,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0x8d,0x00,0x00,0x00,0x03,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x98,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x9d,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0x9f,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xae,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00, -0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xaf,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xb0,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x53,0x00,0x00,0x00, -0xaf,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xb1,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x4f,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xb2,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xb3,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0xb1,0x00,0x00,0x00,0xb2,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xb4,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0xb3,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xb5,0x00,0x00,0x00,0x86,0x00,0x00,0x00, -0xb0,0x00,0x00,0x00,0xb4,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xb6,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0xae,0x00,0x00,0x00,0xb5,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xb7,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0xb6,0x00,0x00,0x00,0xb2,0x00,0x00,0x00,0x14,0x00,0x02,0x00, -0xb8,0x00,0x00,0x00,0x16,0x00,0x03,0x00,0xba,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xbb,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xbc,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0xbb,0x00,0x00,0x00, -0xb5,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xbd,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0xbc,0x00,0x00,0x00, -0xb2,0x00,0x00,0x00,0x1c,0x00,0x04,0x00,0xbe,0x00,0x00,0x00, -0xba,0x00,0x00,0x00,0xbd,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0xbf,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0xbe,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0xba,0x00,0x00,0x00,0xc2,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0xc3,0x00,0x00,0x00, -0x07,0x00,0x00,0x00,0xba,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0xc6,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x16,0x00,0x03,0x00,0xe6,0x00,0x00,0x00,0x10,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xe7,0x00,0x00,0x00, -0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xe8,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0xe7,0x00,0x00,0x00, -0x1c,0x00,0x04,0x00,0xe9,0x00,0x00,0x00,0xe6,0x00,0x00,0x00, -0xe8,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0xea,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0xe9,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0xea,0x00,0x00,0x00,0xeb,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xef,0x00,0x00,0x00, -0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x17,0x00,0x04,0x00,0xf5,0x00,0x00,0x00,0xe6,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x18,0x00,0x04,0x00,0xf6,0x00,0x00,0x00, -0xf5,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, -0xf7,0x00,0x00,0x00,0xf6,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, -0xf8,0x00,0x00,0x00,0xf7,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0xf9,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0xf8,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0xf9,0x00,0x00,0x00,0xfa,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0xfc,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0xe6,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0xff,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0xe6,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x04,0x01,0x00,0x00, -0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x11,0x01,0x00,0x00, -0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x1e,0x01,0x00,0x00, -0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x23,0x01,0x00,0x00, -0x03,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x2c,0x01,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x31,0x01,0x00,0x00,0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x3a,0x01,0x00,0x00,0x80,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x3f,0x01,0x00,0x00,0x05,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x48,0x01,0x00,0x00, -0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x4d,0x01,0x00,0x00, -0x06,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x56,0x01,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x5b,0x01,0x00,0x00,0x07,0x00,0x00,0x00,0x32,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x61,0x01,0x00,0x00,0x01,0x00,0x00,0x00, -0x33,0x00,0x06,0x00,0x09,0x00,0x00,0x00,0x62,0x01,0x00,0x00, -0x61,0x01,0x00,0x00,0x39,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x63,0x01,0x00,0x00, -0x51,0x00,0x00,0x00,0x62,0x01,0x00,0x00,0x00,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x64,0x01,0x00,0x00, -0x84,0x00,0x00,0x00,0x63,0x01,0x00,0x00,0x6d,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x65,0x01,0x00,0x00, -0x86,0x00,0x00,0x00,0x64,0x01,0x00,0x00,0x6c,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x7c,0x01,0x00,0x00, -0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x7d,0x01,0x00,0x00, -0x84,0x00,0x00,0x00,0x9d,0x00,0x00,0x00,0x7c,0x01,0x00,0x00, -0x1c,0x00,0x04,0x00,0x7e,0x01,0x00,0x00,0xe6,0x00,0x00,0x00, -0x7d,0x01,0x00,0x00,0x20,0x00,0x04,0x00,0x7f,0x01,0x00,0x00, -0x04,0x00,0x00,0x00,0x7e,0x01,0x00,0x00,0x3b,0x00,0x04,0x00, -0x7f,0x01,0x00,0x00,0x80,0x01,0x00,0x00,0x04,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x84,0x01,0x00,0x00, -0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x17,0x00,0x04,0x00,0x8a,0x01,0x00,0x00,0xba,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x18,0x00,0x04,0x00,0x8b,0x01,0x00,0x00, -0x8a,0x01,0x00,0x00,0x02,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, -0x8c,0x01,0x00,0x00,0x8b,0x01,0x00,0x00,0x1e,0x00,0x03,0x00, -0x8d,0x01,0x00,0x00,0x8c,0x01,0x00,0x00,0x20,0x00,0x04,0x00, -0x8e,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x8d,0x01,0x00,0x00, +0x20,0x00,0x04,0x00,0x9d,0x02,0x00,0x00,0x0c,0x00,0x00,0x00, +0xc3,0x00,0x00,0x00,0x36,0x00,0x05,0x00,0x02,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x03,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0x05,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0xc8,0x00,0x00,0x00,0xc9,0x00,0x00,0x00,0x07,0x00,0x00,0x00, 0x3b,0x00,0x04,0x00,0x8e,0x01,0x00,0x00,0x8f,0x01,0x00,0x00, -0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x91,0x01,0x00,0x00, -0x0c,0x00,0x00,0x00,0xba,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x99,0x01,0x00,0x00,0x80,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xa7,0x01,0x00,0x00,0x80,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xb5,0x01,0x00,0x00,0x80,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xc3,0x01,0x00,0x00,0x80,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xd1,0x01,0x00,0x00,0x80,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xdf,0x01,0x00,0x00,0x80,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xed,0x01,0x00,0x00,0x80,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xfa,0x01,0x00,0x00,0x08,0x01,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xfb,0x01,0x00,0x00, -0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xfe,0x01,0x00,0x00, -0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x19,0x02,0x00,0x00, -0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00, -0x1c,0x00,0x04,0x00,0x1a,0x02,0x00,0x00,0xe6,0x00,0x00,0x00, -0x19,0x02,0x00,0x00,0x20,0x00,0x04,0x00,0x1b,0x02,0x00,0x00, -0x07,0x00,0x00,0x00,0x1a,0x02,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x2b,0x02,0x00,0x00,0x80,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x31,0x02,0x00,0x00,0x07,0x00,0x00,0x00,0xe6,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x47,0x02,0x00,0x00, -0x84,0x00,0x00,0x00,0xb5,0x00,0x00,0x00,0xb2,0x00,0x00,0x00, -0x1c,0x00,0x04,0x00,0x48,0x02,0x00,0x00,0xe6,0x00,0x00,0x00, -0x47,0x02,0x00,0x00,0x20,0x00,0x04,0x00,0x49,0x02,0x00,0x00, -0x07,0x00,0x00,0x00,0x48,0x02,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x52,0x02,0x00,0x00,0x86,0x00,0x00,0x00, -0xaf,0x00,0x00,0x00,0xb5,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x5a,0x02,0x00,0x00,0x80,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x89,0x02,0x00,0x00,0x84,0x00,0x00,0x00, -0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0xbe,0x02,0x00,0x00,0x0d,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00,0xc6,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0x1d,0x00,0x03,0x00,0x0c,0x03,0x00,0x00, -0xba,0x00,0x00,0x00,0x1e,0x00,0x03,0x00,0x0d,0x03,0x00,0x00, -0x0c,0x03,0x00,0x00,0x20,0x00,0x04,0x00,0x0e,0x03,0x00,0x00, -0x0c,0x00,0x00,0x00,0x0d,0x03,0x00,0x00,0x3b,0x00,0x04,0x00, -0x0e,0x03,0x00,0x00,0x0f,0x03,0x00,0x00,0x0c,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x14,0x03,0x00,0x00, -0x05,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x21,0x03,0x00,0x00,0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x36,0x00,0x05,0x00,0x02,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x03,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0x05,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0xbf,0x00,0x00,0x00,0xc0,0x00,0x00,0x00,0x07,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x1b,0x02,0x00,0x00,0x1c,0x02,0x00,0x00, -0x07,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x49,0x02,0x00,0x00, -0x4a,0x02,0x00,0x00,0x07,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x07,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0xbc,0x01,0x00,0x00, +0xbd,0x01,0x00,0x00,0x07,0x00,0x00,0x00,0x41,0x00,0x05,0x00, 0x0d,0x00,0x00,0x00,0x0e,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, 0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, 0x0f,0x00,0x00,0x00,0x0e,0x00,0x00,0x00,0x41,0x00,0x05,0x00, @@ -16668,1532 +9654,538 @@ unsigned char matmul_f16_f32_aligned_l_data[] = { 0x06,0x00,0x00,0x00,0x64,0x00,0x00,0x00,0x5d,0x00,0x00,0x00, 0x63,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, 0x68,0x00,0x00,0x00,0x5d,0x00,0x00,0x00,0x67,0x00,0x00,0x00, -0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x6f,0x00,0x00,0x00, -0x4e,0x00,0x00,0x00,0x6e,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x74,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, -0x73,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0x78,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x77,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x79,0x00,0x00,0x00, -0x78,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x7a,0x00,0x00,0x00,0x47,0x00,0x00,0x00,0x79,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x7d,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x7c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x7e,0x00,0x00,0x00,0x7d,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x80,0x00,0x00,0x00, -0x47,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x83,0x00,0x00,0x00,0x80,0x00,0x00,0x00, -0x79,0x00,0x00,0x00,0x0c,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x26,0x00,0x00,0x00, -0x7e,0x00,0x00,0x00,0x83,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x88,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x87,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x89,0x00,0x00,0x00,0x88,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x8a,0x00,0x00,0x00,0x32,0x00,0x00,0x00, -0x89,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x8c,0x00,0x00,0x00,0x42,0x00,0x00,0x00,0x37,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x8e,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x8d,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x8f,0x00,0x00,0x00,0x8e,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x90,0x00,0x00,0x00, -0x8c,0x00,0x00,0x00,0x8f,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x91,0x00,0x00,0x00,0x8a,0x00,0x00,0x00, -0x90,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x93,0x00,0x00,0x00,0x91,0x00,0x00,0x00,0x7a,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x94,0x00,0x00,0x00, -0x93,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x99,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x98,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x9a,0x00,0x00,0x00,0x99,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x9b,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, -0x9a,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x9e,0x00,0x00,0x00,0x4a,0x00,0x00,0x00,0x9d,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0xa0,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x9f,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xa1,0x00,0x00,0x00,0xa0,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa2,0x00,0x00,0x00, -0x9e,0x00,0x00,0x00,0xa1,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xa3,0x00,0x00,0x00,0x9b,0x00,0x00,0x00, -0xa2,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xa5,0x00,0x00,0x00,0xa3,0x00,0x00,0x00,0x7a,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa6,0x00,0x00,0x00, -0xa5,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xa8,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xa8,0x00,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x33,0x03,0x00,0x00, -0x3e,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0xc7,0x00,0x00,0x00, -0xa9,0x00,0x00,0x00,0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00, -0xb9,0x00,0x00,0x00,0x33,0x03,0x00,0x00,0xb7,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0xaa,0x00,0x00,0x00,0xa9,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xb9,0x00,0x00,0x00, -0xa9,0x00,0x00,0x00,0xaa,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xa9,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0xc3,0x00,0x00,0x00, -0xc4,0x00,0x00,0x00,0xc0,0x00,0x00,0x00,0x33,0x03,0x00,0x00, -0x3e,0x00,0x03,0x00,0xc4,0x00,0x00,0x00,0xc2,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xc7,0x00,0x00,0x00, -0x33,0x03,0x00,0x00,0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xa8,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xaa,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xca,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xca,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0x4c,0x03,0x00,0x00,0xa6,0x00,0x00,0x00,0xaa,0x00,0x00,0x00, -0x00,0x02,0x00,0x00,0xcd,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0x48,0x03,0x00,0x00,0x94,0x00,0x00,0x00, -0xaa,0x00,0x00,0x00,0xfd,0x01,0x00,0x00,0xcd,0x00,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x34,0x03,0x00,0x00, -0x7a,0x00,0x00,0x00,0xaa,0x00,0x00,0x00,0xae,0x02,0x00,0x00, -0xcd,0x00,0x00,0x00,0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00, -0xd1,0x00,0x00,0x00,0x34,0x03,0x00,0x00,0x84,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0xcc,0x00,0x00,0x00,0xcd,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xd1,0x00,0x00,0x00, -0xcb,0x00,0x00,0x00,0xcc,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xcb,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xd3,0x00,0x00,0x00, +0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x6e,0x00,0x00,0x00, +0x4e,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x73,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, +0x72,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x78,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x77,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x7d,0x00,0x00,0x00, +0x4e,0x00,0x00,0x00,0x7c,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x15,0x00,0x00,0x00,0x81,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0x80,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x82,0x00,0x00,0x00,0x81,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x83,0x00,0x00,0x00,0x47,0x00,0x00,0x00, +0x82,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x85,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x87,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x89,0x00,0x00,0x00,0x47,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8c,0x00,0x00,0x00, +0x89,0x00,0x00,0x00,0x82,0x00,0x00,0x00,0x0c,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0x8d,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x26,0x00,0x00,0x00,0x87,0x00,0x00,0x00,0x8c,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x91,0x00,0x00,0x00, +0x12,0x00,0x00,0x00,0x90,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x92,0x00,0x00,0x00,0x91,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x93,0x00,0x00,0x00, +0x32,0x00,0x00,0x00,0x92,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x95,0x00,0x00,0x00,0x42,0x00,0x00,0x00, +0x37,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, +0x97,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x96,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x98,0x00,0x00,0x00, +0x97,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x99,0x00,0x00,0x00,0x95,0x00,0x00,0x00,0x98,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9a,0x00,0x00,0x00, +0x93,0x00,0x00,0x00,0x99,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x9c,0x00,0x00,0x00,0x9a,0x00,0x00,0x00, +0x83,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x9d,0x00,0x00,0x00,0x9c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0xa2,0x00,0x00,0x00, +0x12,0x00,0x00,0x00,0xa1,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xa3,0x00,0x00,0x00,0xa2,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa4,0x00,0x00,0x00, +0x0f,0x00,0x00,0x00,0xa3,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xa7,0x00,0x00,0x00,0x4a,0x00,0x00,0x00, +0xa6,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, +0xa9,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0xa8,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xaa,0x00,0x00,0x00, +0xa9,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xab,0x00,0x00,0x00,0xa7,0x00,0x00,0x00,0xaa,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xac,0x00,0x00,0x00, +0xa4,0x00,0x00,0x00,0xab,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xae,0x00,0x00,0x00,0xac,0x00,0x00,0x00, +0x83,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xaf,0x00,0x00,0x00,0xae,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xb1,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xb1,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xa7,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0x05,0x00,0x00,0x00, +0xd0,0x00,0x00,0x00,0xb2,0x00,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc1,0x00,0x00,0x00,0xc2,0x00,0x00,0x00,0xa7,0x02,0x00,0x00, +0xc0,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xb3,0x00,0x00,0x00, +0xb2,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0xc2,0x00,0x00,0x00,0xb2,0x00,0x00,0x00,0xb3,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xb2,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0xcc,0x00,0x00,0x00,0xcd,0x00,0x00,0x00,0xc9,0x00,0x00,0x00, +0xa7,0x02,0x00,0x00,0x3e,0x00,0x03,0x00,0xcd,0x00,0x00,0x00, +0xcb,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xd0,0x00,0x00,0x00,0xa7,0x02,0x00,0x00,0xcf,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xb1,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xb3,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xd3,0x00,0x00,0x00, 0xf8,0x00,0x02,0x00,0xd3,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0x44,0x03,0x00,0x00,0x3e,0x00,0x00,0x00, -0xcb,0x00,0x00,0x00,0x67,0x01,0x00,0x00,0xd4,0x00,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00,0xd9,0x00,0x00,0x00, -0x44,0x03,0x00,0x00,0x37,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0xd5,0x00,0x00,0x00,0xd4,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xd9,0x00,0x00,0x00,0xd4,0x00,0x00,0x00, -0xd5,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xd4,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xde,0x00,0x00,0x00, -0x74,0x00,0x00,0x00,0x44,0x03,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xe1,0x00,0x00,0x00,0xde,0x00,0x00,0x00, -0x8f,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xe2,0x00,0x00,0x00,0xe1,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xe3,0x00,0x00,0x00, -0x48,0x03,0x00,0x00,0xe2,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xe5,0x00,0x00,0x00,0xe3,0x00,0x00,0x00, -0x6f,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xf0,0x00,0x00,0x00,0xde,0x00,0x00,0x00,0xef,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xf2,0x00,0x00,0x00, -0x6f,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xf3,0x00,0x00,0x00,0xf0,0x00,0x00,0x00, -0xf2,0x00,0x00,0x00,0x41,0x00,0x08,0x00,0xfc,0x00,0x00,0x00, -0xfd,0x00,0x00,0x00,0xfa,0x00,0x00,0x00,0x34,0x00,0x00,0x00, -0xe5,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0x3e,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0xe6,0x00,0x00,0x00,0xfe,0x00,0x00,0x00, -0xfd,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0xff,0x00,0x00,0x00, -0x00,0x01,0x00,0x00,0xeb,0x00,0x00,0x00,0xf3,0x00,0x00,0x00, -0x3e,0x00,0x03,0x00,0x00,0x01,0x00,0x00,0xfe,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x05,0x01,0x00,0x00, -0xde,0x00,0x00,0x00,0x04,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x08,0x01,0x00,0x00,0x05,0x01,0x00,0x00, -0xf2,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x09,0x01,0x00,0x00,0x08,0x01,0x00,0x00,0x39,0x00,0x00,0x00, -0x41,0x00,0x08,0x00,0xfc,0x00,0x00,0x00,0x0b,0x01,0x00,0x00, -0xfa,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0xe5,0x00,0x00,0x00, -0x34,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0xe6,0x00,0x00,0x00,0x0c,0x01,0x00,0x00,0x0b,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0xff,0x00,0x00,0x00,0x0d,0x01,0x00,0x00, -0xeb,0x00,0x00,0x00,0x09,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0x0d,0x01,0x00,0x00,0x0c,0x01,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x12,0x01,0x00,0x00,0xde,0x00,0x00,0x00, -0x11,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x15,0x01,0x00,0x00,0x12,0x01,0x00,0x00,0xf2,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x16,0x01,0x00,0x00, -0x15,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x41,0x00,0x08,0x00, -0xfc,0x00,0x00,0x00,0x18,0x01,0x00,0x00,0xfa,0x00,0x00,0x00, -0x34,0x00,0x00,0x00,0xe5,0x00,0x00,0x00,0x34,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0xe6,0x00,0x00,0x00, -0x19,0x01,0x00,0x00,0x18,0x01,0x00,0x00,0x41,0x00,0x05,0x00, -0xff,0x00,0x00,0x00,0x1a,0x01,0x00,0x00,0xeb,0x00,0x00,0x00, -0x16,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x1a,0x01,0x00,0x00, -0x19,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x1f,0x01,0x00,0x00,0xde,0x00,0x00,0x00,0x1e,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x22,0x01,0x00,0x00, -0x1f,0x01,0x00,0x00,0xf2,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x24,0x01,0x00,0x00,0x22,0x01,0x00,0x00, -0x23,0x01,0x00,0x00,0x41,0x00,0x08,0x00,0xfc,0x00,0x00,0x00, -0x26,0x01,0x00,0x00,0xfa,0x00,0x00,0x00,0x34,0x00,0x00,0x00, -0xe5,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0x23,0x01,0x00,0x00, -0x3d,0x00,0x04,0x00,0xe6,0x00,0x00,0x00,0x27,0x01,0x00,0x00, -0x26,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0xff,0x00,0x00,0x00, -0x28,0x01,0x00,0x00,0xeb,0x00,0x00,0x00,0x24,0x01,0x00,0x00, -0x3e,0x00,0x03,0x00,0x28,0x01,0x00,0x00,0x27,0x01,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x2d,0x01,0x00,0x00, -0xde,0x00,0x00,0x00,0x2c,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x30,0x01,0x00,0x00,0x2d,0x01,0x00,0x00, -0xf2,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x32,0x01,0x00,0x00,0x30,0x01,0x00,0x00,0x31,0x01,0x00,0x00, -0x41,0x00,0x08,0x00,0xfc,0x00,0x00,0x00,0x34,0x01,0x00,0x00, -0xfa,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0xe5,0x00,0x00,0x00, -0xc6,0x00,0x00,0x00,0x3e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0xe6,0x00,0x00,0x00,0x35,0x01,0x00,0x00,0x34,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0xff,0x00,0x00,0x00,0x36,0x01,0x00,0x00, -0xeb,0x00,0x00,0x00,0x32,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0x36,0x01,0x00,0x00,0x35,0x01,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x3b,0x01,0x00,0x00,0xde,0x00,0x00,0x00, -0x3a,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x3e,0x01,0x00,0x00,0x3b,0x01,0x00,0x00,0xf2,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x40,0x01,0x00,0x00, -0x3e,0x01,0x00,0x00,0x3f,0x01,0x00,0x00,0x41,0x00,0x08,0x00, -0xfc,0x00,0x00,0x00,0x42,0x01,0x00,0x00,0xfa,0x00,0x00,0x00, -0x34,0x00,0x00,0x00,0xe5,0x00,0x00,0x00,0xc6,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0xe6,0x00,0x00,0x00, -0x43,0x01,0x00,0x00,0x42,0x01,0x00,0x00,0x41,0x00,0x05,0x00, -0xff,0x00,0x00,0x00,0x44,0x01,0x00,0x00,0xeb,0x00,0x00,0x00, -0x40,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x44,0x01,0x00,0x00, -0x43,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x49,0x01,0x00,0x00,0xde,0x00,0x00,0x00,0x48,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x4c,0x01,0x00,0x00, -0x49,0x01,0x00,0x00,0xf2,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x4e,0x01,0x00,0x00,0x4c,0x01,0x00,0x00, -0x4d,0x01,0x00,0x00,0x41,0x00,0x08,0x00,0xfc,0x00,0x00,0x00, -0x50,0x01,0x00,0x00,0xfa,0x00,0x00,0x00,0x34,0x00,0x00,0x00, -0xe5,0x00,0x00,0x00,0xc6,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0xe6,0x00,0x00,0x00,0x51,0x01,0x00,0x00, -0x50,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0xff,0x00,0x00,0x00, -0x52,0x01,0x00,0x00,0xeb,0x00,0x00,0x00,0x4e,0x01,0x00,0x00, -0x3e,0x00,0x03,0x00,0x52,0x01,0x00,0x00,0x51,0x01,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x57,0x01,0x00,0x00, -0xde,0x00,0x00,0x00,0x56,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x5a,0x01,0x00,0x00,0x57,0x01,0x00,0x00, -0xf2,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x5c,0x01,0x00,0x00,0x5a,0x01,0x00,0x00,0x5b,0x01,0x00,0x00, -0x41,0x00,0x08,0x00,0xfc,0x00,0x00,0x00,0x5e,0x01,0x00,0x00, -0xfa,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0xe5,0x00,0x00,0x00, -0xc6,0x00,0x00,0x00,0x23,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, -0xe6,0x00,0x00,0x00,0x5f,0x01,0x00,0x00,0x5e,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0xff,0x00,0x00,0x00,0x60,0x01,0x00,0x00, -0xeb,0x00,0x00,0x00,0x5c,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0x60,0x01,0x00,0x00,0x5f,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x67,0x01,0x00,0x00,0x44,0x03,0x00,0x00, -0x65,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xd3,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xd5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x69,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x69,0x01,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x45,0x03,0x00,0x00, -0x3e,0x00,0x00,0x00,0xd5,0x00,0x00,0x00,0xf9,0x01,0x00,0x00, -0x6a,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00, -0x6f,0x01,0x00,0x00,0x45,0x03,0x00,0x00,0x9d,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0x6b,0x01,0x00,0x00,0x6a,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x6f,0x01,0x00,0x00, -0x6a,0x01,0x00,0x00,0x6b,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x6a,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x74,0x01,0x00,0x00,0x74,0x00,0x00,0x00,0x45,0x03,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x77,0x01,0x00,0x00, -0x74,0x01,0x00,0x00,0xa1,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x78,0x01,0x00,0x00,0x77,0x01,0x00,0x00, -0x6d,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x79,0x01,0x00,0x00,0x4c,0x03,0x00,0x00,0x78,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x7b,0x01,0x00,0x00, -0x79,0x01,0x00,0x00,0x6f,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x85,0x01,0x00,0x00,0x74,0x01,0x00,0x00, -0x84,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x87,0x01,0x00,0x00,0x6f,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x88,0x01,0x00,0x00, -0x85,0x01,0x00,0x00,0x87,0x01,0x00,0x00,0x41,0x00,0x08,0x00, -0x91,0x01,0x00,0x00,0x92,0x01,0x00,0x00,0x8f,0x01,0x00,0x00, -0x34,0x00,0x00,0x00,0x7b,0x01,0x00,0x00,0x34,0x00,0x00,0x00, -0x3e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0xba,0x00,0x00,0x00, -0x93,0x01,0x00,0x00,0x92,0x01,0x00,0x00,0x73,0x00,0x04,0x00, -0xe6,0x00,0x00,0x00,0x94,0x01,0x00,0x00,0x93,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0xff,0x00,0x00,0x00,0x95,0x01,0x00,0x00, -0x80,0x01,0x00,0x00,0x88,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0x95,0x01,0x00,0x00,0x94,0x01,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x9a,0x01,0x00,0x00,0x74,0x01,0x00,0x00, -0x99,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x9d,0x01,0x00,0x00,0x9a,0x01,0x00,0x00,0x87,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9e,0x01,0x00,0x00, -0x9d,0x01,0x00,0x00,0x39,0x00,0x00,0x00,0x41,0x00,0x08,0x00, -0x91,0x01,0x00,0x00,0xa0,0x01,0x00,0x00,0x8f,0x01,0x00,0x00, -0x34,0x00,0x00,0x00,0x7b,0x01,0x00,0x00,0x34,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0xba,0x00,0x00,0x00, -0xa1,0x01,0x00,0x00,0xa0,0x01,0x00,0x00,0x73,0x00,0x04,0x00, -0xe6,0x00,0x00,0x00,0xa2,0x01,0x00,0x00,0xa1,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0xff,0x00,0x00,0x00,0xa3,0x01,0x00,0x00, -0x80,0x01,0x00,0x00,0x9e,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0xa3,0x01,0x00,0x00,0xa2,0x01,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xa8,0x01,0x00,0x00,0x74,0x01,0x00,0x00, -0xa7,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xab,0x01,0x00,0x00,0xa8,0x01,0x00,0x00,0x87,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xac,0x01,0x00,0x00, -0xab,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x41,0x00,0x08,0x00, -0x91,0x01,0x00,0x00,0xae,0x01,0x00,0x00,0x8f,0x01,0x00,0x00, -0x34,0x00,0x00,0x00,0x7b,0x01,0x00,0x00,0x34,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0xba,0x00,0x00,0x00, -0xaf,0x01,0x00,0x00,0xae,0x01,0x00,0x00,0x73,0x00,0x04,0x00, -0xe6,0x00,0x00,0x00,0xb0,0x01,0x00,0x00,0xaf,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0xff,0x00,0x00,0x00,0xb1,0x01,0x00,0x00, -0x80,0x01,0x00,0x00,0xac,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0xb1,0x01,0x00,0x00,0xb0,0x01,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xb6,0x01,0x00,0x00,0x74,0x01,0x00,0x00, -0xb5,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xb9,0x01,0x00,0x00,0xb6,0x01,0x00,0x00,0x87,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xba,0x01,0x00,0x00, -0xb9,0x01,0x00,0x00,0x23,0x01,0x00,0x00,0x41,0x00,0x08,0x00, -0x91,0x01,0x00,0x00,0xbc,0x01,0x00,0x00,0x8f,0x01,0x00,0x00, -0x34,0x00,0x00,0x00,0x7b,0x01,0x00,0x00,0x34,0x00,0x00,0x00, -0x23,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0xba,0x00,0x00,0x00, -0xbd,0x01,0x00,0x00,0xbc,0x01,0x00,0x00,0x73,0x00,0x04,0x00, -0xe6,0x00,0x00,0x00,0xbe,0x01,0x00,0x00,0xbd,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0xff,0x00,0x00,0x00,0xbf,0x01,0x00,0x00, -0x80,0x01,0x00,0x00,0xba,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0xbf,0x01,0x00,0x00,0xbe,0x01,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xc4,0x01,0x00,0x00,0x74,0x01,0x00,0x00, -0xc3,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xc7,0x01,0x00,0x00,0xc4,0x01,0x00,0x00,0x87,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xc8,0x01,0x00,0x00, -0xc7,0x01,0x00,0x00,0x31,0x01,0x00,0x00,0x41,0x00,0x08,0x00, -0x91,0x01,0x00,0x00,0xca,0x01,0x00,0x00,0x8f,0x01,0x00,0x00, -0x34,0x00,0x00,0x00,0x7b,0x01,0x00,0x00,0xc6,0x00,0x00,0x00, -0x3e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0xba,0x00,0x00,0x00, -0xcb,0x01,0x00,0x00,0xca,0x01,0x00,0x00,0x73,0x00,0x04,0x00, -0xe6,0x00,0x00,0x00,0xcc,0x01,0x00,0x00,0xcb,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0xff,0x00,0x00,0x00,0xcd,0x01,0x00,0x00, -0x80,0x01,0x00,0x00,0xc8,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0xcd,0x01,0x00,0x00,0xcc,0x01,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xd2,0x01,0x00,0x00,0x74,0x01,0x00,0x00, -0xd1,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xd5,0x01,0x00,0x00,0xd2,0x01,0x00,0x00,0x87,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xd6,0x01,0x00,0x00, -0xd5,0x01,0x00,0x00,0x3f,0x01,0x00,0x00,0x41,0x00,0x08,0x00, -0x91,0x01,0x00,0x00,0xd8,0x01,0x00,0x00,0x8f,0x01,0x00,0x00, -0x34,0x00,0x00,0x00,0x7b,0x01,0x00,0x00,0xc6,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0xba,0x00,0x00,0x00, -0xd9,0x01,0x00,0x00,0xd8,0x01,0x00,0x00,0x73,0x00,0x04,0x00, -0xe6,0x00,0x00,0x00,0xda,0x01,0x00,0x00,0xd9,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0xff,0x00,0x00,0x00,0xdb,0x01,0x00,0x00, -0x80,0x01,0x00,0x00,0xd6,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0xdb,0x01,0x00,0x00,0xda,0x01,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xe0,0x01,0x00,0x00,0x74,0x01,0x00,0x00, -0xdf,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xe3,0x01,0x00,0x00,0xe0,0x01,0x00,0x00,0x87,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xe4,0x01,0x00,0x00, -0xe3,0x01,0x00,0x00,0x4d,0x01,0x00,0x00,0x41,0x00,0x08,0x00, -0x91,0x01,0x00,0x00,0xe6,0x01,0x00,0x00,0x8f,0x01,0x00,0x00, -0x34,0x00,0x00,0x00,0x7b,0x01,0x00,0x00,0xc6,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0xba,0x00,0x00,0x00, -0xe7,0x01,0x00,0x00,0xe6,0x01,0x00,0x00,0x73,0x00,0x04,0x00, -0xe6,0x00,0x00,0x00,0xe8,0x01,0x00,0x00,0xe7,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0xff,0x00,0x00,0x00,0xe9,0x01,0x00,0x00, -0x80,0x01,0x00,0x00,0xe4,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0xe9,0x01,0x00,0x00,0xe8,0x01,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xee,0x01,0x00,0x00,0x74,0x01,0x00,0x00, -0xed,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xf1,0x01,0x00,0x00,0xee,0x01,0x00,0x00,0x87,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xf2,0x01,0x00,0x00, -0xf1,0x01,0x00,0x00,0x5b,0x01,0x00,0x00,0x41,0x00,0x08,0x00, -0x91,0x01,0x00,0x00,0xf4,0x01,0x00,0x00,0x8f,0x01,0x00,0x00, -0x34,0x00,0x00,0x00,0x7b,0x01,0x00,0x00,0xc6,0x00,0x00,0x00, -0x23,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0xba,0x00,0x00,0x00, -0xf5,0x01,0x00,0x00,0xf4,0x01,0x00,0x00,0x73,0x00,0x04,0x00, -0xe6,0x00,0x00,0x00,0xf6,0x01,0x00,0x00,0xf5,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0xff,0x00,0x00,0x00,0xf7,0x01,0x00,0x00, -0x80,0x01,0x00,0x00,0xf2,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0xf7,0x01,0x00,0x00,0xf6,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xf9,0x01,0x00,0x00,0x45,0x03,0x00,0x00, -0x65,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x69,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x6b,0x01,0x00,0x00,0xe0,0x00,0x04,0x00, -0x0c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0xfa,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xfd,0x01,0x00,0x00, -0x48,0x03,0x00,0x00,0xfb,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x00,0x02,0x00,0x00,0x4c,0x03,0x00,0x00, -0xfe,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x02,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x02,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0x4e,0x03,0x00,0x00,0x3e,0x00,0x00,0x00, -0x6b,0x01,0x00,0x00,0xac,0x02,0x00,0x00,0x05,0x02,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00,0x08,0x02,0x00,0x00, -0x4e,0x03,0x00,0x00,0x6c,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0x04,0x02,0x00,0x00,0x05,0x02,0x00,0x00,0x00,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x08,0x02,0x00,0x00,0x03,0x02,0x00,0x00, -0x04,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x03,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0x0a,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x0a,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0x52,0x03,0x00,0x00,0x3e,0x00,0x00,0x00,0x03,0x02,0x00,0x00, -0x36,0x02,0x00,0x00,0x0d,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb8,0x00,0x00,0x00,0x10,0x02,0x00,0x00,0x52,0x03,0x00,0x00, -0x60,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x0c,0x02,0x00,0x00, -0x0d,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x10,0x02,0x00,0x00,0x0b,0x02,0x00,0x00,0x0c,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x0b,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x12,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x12,0x02,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x64,0x03,0x00,0x00, -0x3e,0x00,0x00,0x00,0x0b,0x02,0x00,0x00,0x34,0x02,0x00,0x00, -0x13,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00, -0x18,0x02,0x00,0x00,0x64,0x03,0x00,0x00,0x62,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0x14,0x02,0x00,0x00,0x13,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x18,0x02,0x00,0x00, -0x13,0x02,0x00,0x00,0x14,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x13,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x1e,0x02,0x00,0x00,0x52,0x03,0x00,0x00,0x62,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x20,0x02,0x00,0x00, -0x1e,0x02,0x00,0x00,0x64,0x03,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x22,0x02,0x00,0x00,0x55,0x00,0x00,0x00, -0x53,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x24,0x02,0x00,0x00,0x52,0x03,0x00,0x00,0x61,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x25,0x02,0x00,0x00, -0x22,0x02,0x00,0x00,0x24,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x27,0x02,0x00,0x00,0x64,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x28,0x02,0x00,0x00,0x25,0x02,0x00,0x00,0x27,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x2a,0x02,0x00,0x00, -0x28,0x02,0x00,0x00,0x64,0x03,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x2c,0x02,0x00,0x00,0x2a,0x02,0x00,0x00, -0x2b,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x2e,0x02,0x00,0x00,0x2c,0x02,0x00,0x00,0x4e,0x03,0x00,0x00, -0x41,0x00,0x05,0x00,0xff,0x00,0x00,0x00,0x2f,0x02,0x00,0x00, -0xeb,0x00,0x00,0x00,0x2e,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, -0xe6,0x00,0x00,0x00,0x30,0x02,0x00,0x00,0x2f,0x02,0x00,0x00, -0x41,0x00,0x05,0x00,0x31,0x02,0x00,0x00,0x32,0x02,0x00,0x00, -0x1c,0x02,0x00,0x00,0x20,0x02,0x00,0x00,0x3e,0x00,0x03,0x00, -0x32,0x02,0x00,0x00,0x30,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x34,0x02,0x00,0x00,0x64,0x03,0x00,0x00, -0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x12,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x14,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x0d,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x0d,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x36,0x02,0x00,0x00, -0x52,0x03,0x00,0x00,0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x0a,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x0c,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0x38,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x38,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0x53,0x03,0x00,0x00,0x3e,0x00,0x00,0x00,0x0c,0x02,0x00,0x00, -0x64,0x02,0x00,0x00,0x3b,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb8,0x00,0x00,0x00,0x3e,0x02,0x00,0x00,0x53,0x03,0x00,0x00, -0xb5,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x3a,0x02,0x00,0x00, -0x3b,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x3e,0x02,0x00,0x00,0x39,0x02,0x00,0x00,0x3a,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x39,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x40,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x40,0x02,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x61,0x03,0x00,0x00, -0x3e,0x00,0x00,0x00,0x39,0x02,0x00,0x00,0x62,0x02,0x00,0x00, -0x41,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00, -0x46,0x02,0x00,0x00,0x61,0x03,0x00,0x00,0xb2,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0x42,0x02,0x00,0x00,0x41,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x46,0x02,0x00,0x00, -0x41,0x02,0x00,0x00,0x42,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x41,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x4c,0x02,0x00,0x00,0x53,0x03,0x00,0x00,0xb2,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x4e,0x02,0x00,0x00, -0x4c,0x02,0x00,0x00,0x61,0x03,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x50,0x02,0x00,0x00,0x59,0x00,0x00,0x00, -0xaf,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x53,0x02,0x00,0x00,0x53,0x03,0x00,0x00,0x52,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x54,0x02,0x00,0x00, -0x50,0x02,0x00,0x00,0x53,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x56,0x02,0x00,0x00,0x68,0x00,0x00,0x00, -0xb2,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x57,0x02,0x00,0x00,0x54,0x02,0x00,0x00,0x56,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x59,0x02,0x00,0x00, -0x57,0x02,0x00,0x00,0x61,0x03,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x5b,0x02,0x00,0x00,0x59,0x02,0x00,0x00, -0x5a,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x5d,0x02,0x00,0x00,0x5b,0x02,0x00,0x00,0x4e,0x03,0x00,0x00, -0x41,0x00,0x05,0x00,0xff,0x00,0x00,0x00,0x5e,0x02,0x00,0x00, -0x80,0x01,0x00,0x00,0x5d,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, -0xe6,0x00,0x00,0x00,0x5f,0x02,0x00,0x00,0x5e,0x02,0x00,0x00, -0x41,0x00,0x05,0x00,0x31,0x02,0x00,0x00,0x60,0x02,0x00,0x00, -0x4a,0x02,0x00,0x00,0x4e,0x02,0x00,0x00,0x3e,0x00,0x03,0x00, -0x60,0x02,0x00,0x00,0x5f,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x62,0x02,0x00,0x00,0x61,0x03,0x00,0x00, -0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x40,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x42,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x3b,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x3b,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x64,0x02,0x00,0x00, -0x53,0x03,0x00,0x00,0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x38,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x3a,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0x66,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x66,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0x54,0x03,0x00,0x00,0x3e,0x00,0x00,0x00,0x3a,0x02,0x00,0x00, -0xaa,0x02,0x00,0x00,0x69,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb8,0x00,0x00,0x00,0x6c,0x02,0x00,0x00,0x54,0x03,0x00,0x00, -0xb5,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x68,0x02,0x00,0x00, -0x69,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x6c,0x02,0x00,0x00,0x67,0x02,0x00,0x00,0x68,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x67,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x6e,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x6e,0x02,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x58,0x03,0x00,0x00, -0x3e,0x00,0x00,0x00,0x67,0x02,0x00,0x00,0xa8,0x02,0x00,0x00, -0x71,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00, -0x74,0x02,0x00,0x00,0x58,0x03,0x00,0x00,0x60,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0x70,0x02,0x00,0x00,0x71,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x74,0x02,0x00,0x00, -0x6f,0x02,0x00,0x00,0x70,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x6f,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x76,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x76,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0x5a,0x03,0x00,0x00,0x3e,0x00,0x00,0x00, -0x6f,0x02,0x00,0x00,0xa6,0x02,0x00,0x00,0x79,0x02,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00,0x7c,0x02,0x00,0x00, -0x5a,0x03,0x00,0x00,0xb2,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0x78,0x02,0x00,0x00,0x79,0x02,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x7c,0x02,0x00,0x00,0x77,0x02,0x00,0x00, -0x78,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x77,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0x7e,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x7e,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0x5c,0x03,0x00,0x00,0x3e,0x00,0x00,0x00,0x77,0x02,0x00,0x00, -0xa4,0x02,0x00,0x00,0x7f,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb8,0x00,0x00,0x00,0x84,0x02,0x00,0x00,0x5c,0x03,0x00,0x00, -0x62,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x80,0x02,0x00,0x00, -0x7f,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x84,0x02,0x00,0x00,0x7f,0x02,0x00,0x00,0x80,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x7f,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x86,0x02,0x00,0x00,0x54,0x03,0x00,0x00, -0xb2,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x88,0x02,0x00,0x00,0x86,0x02,0x00,0x00,0x5a,0x03,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8a,0x02,0x00,0x00, -0x88,0x02,0x00,0x00,0x89,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x8c,0x02,0x00,0x00,0x58,0x03,0x00,0x00, -0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x8d,0x02,0x00,0x00,0x8a,0x02,0x00,0x00,0x8c,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8f,0x02,0x00,0x00, -0x8d,0x02,0x00,0x00,0x5c,0x03,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x93,0x02,0x00,0x00,0x8c,0x02,0x00,0x00, -0x5c,0x03,0x00,0x00,0x41,0x00,0x05,0x00,0x31,0x02,0x00,0x00, -0x94,0x02,0x00,0x00,0x1c,0x02,0x00,0x00,0x93,0x02,0x00,0x00, -0x3d,0x00,0x04,0x00,0xe6,0x00,0x00,0x00,0x95,0x02,0x00,0x00, -0x94,0x02,0x00,0x00,0x73,0x00,0x04,0x00,0xba,0x00,0x00,0x00, -0x96,0x02,0x00,0x00,0x95,0x02,0x00,0x00,0x41,0x00,0x05,0x00, -0x31,0x02,0x00,0x00,0x9b,0x02,0x00,0x00,0x4a,0x02,0x00,0x00, -0x88,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0xe6,0x00,0x00,0x00, -0x9c,0x02,0x00,0x00,0x9b,0x02,0x00,0x00,0x73,0x00,0x04,0x00, -0xba,0x00,0x00,0x00,0x9d,0x02,0x00,0x00,0x9c,0x02,0x00,0x00, -0x41,0x00,0x05,0x00,0xc3,0x00,0x00,0x00,0x9f,0x02,0x00,0x00, -0xc0,0x00,0x00,0x00,0x8f,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, -0xba,0x00,0x00,0x00,0xa0,0x02,0x00,0x00,0x9f,0x02,0x00,0x00, -0x0c,0x00,0x08,0x00,0xba,0x00,0x00,0x00,0xa1,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x96,0x02,0x00,0x00, -0x9d,0x02,0x00,0x00,0xa0,0x02,0x00,0x00,0x3e,0x00,0x03,0x00, -0x9f,0x02,0x00,0x00,0xa1,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xa4,0x02,0x00,0x00,0x5c,0x03,0x00,0x00, -0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x7e,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x80,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x79,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x79,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa6,0x02,0x00,0x00, -0x5a,0x03,0x00,0x00,0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x76,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x78,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0x71,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x71,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xa8,0x02,0x00,0x00,0x58,0x03,0x00,0x00,0xc6,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x6e,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x70,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x69,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x69,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xaa,0x02,0x00,0x00,0x54,0x03,0x00,0x00, -0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x66,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x68,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x05,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x05,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xac,0x02,0x00,0x00, -0x4e,0x03,0x00,0x00,0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x02,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x04,0x02,0x00,0x00, -0xe0,0x00,0x04,0x00,0x0c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0xfa,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xcd,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xcd,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xae,0x02,0x00,0x00,0x34,0x03,0x00,0x00, -0x6c,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xca,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xcc,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xb3,0x02,0x00,0x00,0x55,0x00,0x00,0x00, -0x53,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xb4,0x02,0x00,0x00,0x8c,0x00,0x00,0x00,0xb3,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xb9,0x02,0x00,0x00, -0x59,0x00,0x00,0x00,0xaf,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xba,0x02,0x00,0x00,0x9e,0x00,0x00,0x00, -0xb9,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0xbf,0x02,0x00,0x00,0x12,0x00,0x00,0x00,0xbe,0x02,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xc0,0x02,0x00,0x00, -0xbf,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xc1,0x02,0x00,0x00,0x0f,0x00,0x00,0x00,0xc0,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xc5,0x02,0x00,0x00, -0x47,0x00,0x00,0x00,0xc0,0x02,0x00,0x00,0x41,0x00,0x05,0x00, -0x0d,0x00,0x00,0x00,0xc7,0x02,0x00,0x00,0xc6,0x02,0x00,0x00, -0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xc8,0x02,0x00,0x00,0xc7,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xc9,0x02,0x00,0x00,0xc5,0x02,0x00,0x00, -0xc8,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xca,0x02,0x00,0x00,0xc1,0x02,0x00,0x00,0xc9,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0xcc,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0xcc,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0x35,0x03,0x00,0x00,0x3e,0x00,0x00,0x00,0xcc,0x00,0x00,0x00, -0x32,0x03,0x00,0x00,0xcf,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb8,0x00,0x00,0x00,0xd2,0x02,0x00,0x00,0x35,0x03,0x00,0x00, -0xb5,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xce,0x02,0x00,0x00, -0xcf,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0xd2,0x02,0x00,0x00,0xcd,0x02,0x00,0x00,0xce,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0xcd,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0xd4,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0xd4,0x02,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x36,0x03,0x00,0x00, -0x3e,0x00,0x00,0x00,0xcd,0x02,0x00,0x00,0x30,0x03,0x00,0x00, -0xd7,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00, -0xda,0x02,0x00,0x00,0x36,0x03,0x00,0x00,0x60,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0xd6,0x02,0x00,0x00,0xd7,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xda,0x02,0x00,0x00, -0xd5,0x02,0x00,0x00,0xd6,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x06,0x00,0x00,0x00,0xc0,0x02,0x00,0x00,0xaf,0x00,0x00,0x00, +0xb3,0x00,0x00,0x00,0x73,0x01,0x00,0x00,0xd6,0x00,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xbc,0x02,0x00,0x00, +0x9d,0x00,0x00,0x00,0xb3,0x00,0x00,0x00,0x70,0x01,0x00,0x00, +0xd6,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xa8,0x02,0x00,0x00,0x83,0x00,0x00,0x00,0xb3,0x00,0x00,0x00, +0x21,0x02,0x00,0x00,0xd6,0x00,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc1,0x00,0x00,0x00,0xda,0x00,0x00,0x00,0xa8,0x02,0x00,0x00, +0x8d,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xd5,0x00,0x00,0x00, +0xd6,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0xda,0x00,0x00,0x00,0xd4,0x00,0x00,0x00,0xd5,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xd4,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xdc,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xdc,0x00,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xb8,0x02,0x00,0x00, +0x3e,0x00,0x00,0x00,0xd4,0x00,0x00,0x00,0x26,0x01,0x00,0x00, +0xdf,0x00,0x00,0x00,0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00, +0xe2,0x00,0x00,0x00,0xb8,0x02,0x00,0x00,0x37,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0xde,0x00,0x00,0x00,0xdf,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xe2,0x00,0x00,0x00, +0xdd,0x00,0x00,0x00,0xde,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xdd,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xe6,0x00,0x00,0x00,0x95,0x00,0x00,0x00,0x73,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xe8,0x00,0x00,0x00, +0xe6,0x00,0x00,0x00,0xb8,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc1,0x00,0x00,0x00,0xeb,0x00,0x00,0x00,0xe8,0x00,0x00,0x00, +0x36,0x00,0x00,0x00,0xf7,0x00,0x03,0x00,0xed,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xeb,0x00,0x00,0x00, +0xec,0x00,0x00,0x00,0xed,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xec,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xf0,0x00,0x00,0x00,0xa8,0x02,0x00,0x00,0x6e,0x00,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00,0xf2,0x00,0x00,0x00, +0xf0,0x00,0x00,0x00,0x8d,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xed,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xed,0x00,0x00,0x00, +0xf5,0x00,0x07,0x00,0xc1,0x00,0x00,0x00,0xf3,0x00,0x00,0x00, +0xeb,0x00,0x00,0x00,0xdd,0x00,0x00,0x00,0xf2,0x00,0x00,0x00, +0xec,0x00,0x00,0x00,0xf7,0x00,0x03,0x00,0xf5,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xf3,0x00,0x00,0x00, +0xf4,0x00,0x00,0x00,0x16,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xf4,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xfe,0x00,0x00,0x00,0x73,0x00,0x00,0x00,0xb8,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x00,0x01,0x00,0x00, +0xfe,0x00,0x00,0x00,0xff,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x02,0x01,0x00,0x00,0x00,0x01,0x00,0x00, +0x6e,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x0d,0x01,0x00,0x00,0xfe,0x00,0x00,0x00,0x98,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x0e,0x01,0x00,0x00, +0xbc,0x02,0x00,0x00,0x0d,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x10,0x01,0x00,0x00,0x0e,0x01,0x00,0x00, +0x6e,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x11,0x01,0x00,0x00, +0x12,0x01,0x00,0x00,0x06,0x01,0x00,0x00,0x34,0x00,0x00,0x00, +0x10,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0xf6,0x00,0x00,0x00, +0x13,0x01,0x00,0x00,0x12,0x01,0x00,0x00,0x41,0x00,0x05,0x00, +0x14,0x01,0x00,0x00,0x15,0x01,0x00,0x00,0xfb,0x00,0x00,0x00, +0x02,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x15,0x01,0x00,0x00, +0x13,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xf5,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0x16,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x19,0x01,0x00,0x00,0x73,0x00,0x00,0x00, +0xb8,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x1b,0x01,0x00,0x00,0x19,0x01,0x00,0x00,0x1a,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x1d,0x01,0x00,0x00, +0x1b,0x01,0x00,0x00,0x6e,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x14,0x01,0x00,0x00,0x1f,0x01,0x00,0x00,0xfb,0x00,0x00,0x00, +0x1d,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x1f,0x01,0x00,0x00, +0x1e,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xf5,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xf5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xdf,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xdf,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x26,0x01,0x00,0x00, +0xb8,0x02,0x00,0x00,0x24,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0xdc,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xde,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x28,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x28,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xb9,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0xde,0x00,0x00,0x00, +0x6c,0x01,0x00,0x00,0x2b,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc1,0x00,0x00,0x00,0x2e,0x01,0x00,0x00,0xb9,0x02,0x00,0x00, +0xa6,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x2a,0x01,0x00,0x00, +0x2b,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x2e,0x01,0x00,0x00,0x29,0x01,0x00,0x00,0x2a,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x29,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x32,0x01,0x00,0x00,0xa7,0x00,0x00,0x00, +0x7d,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x34,0x01,0x00,0x00,0x32,0x01,0x00,0x00,0xb9,0x02,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x35,0x01,0x00,0x00, +0x12,0x00,0x00,0x00,0xcf,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x36,0x01,0x00,0x00,0x35,0x01,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00,0x37,0x01,0x00,0x00, +0x34,0x01,0x00,0x00,0x36,0x01,0x00,0x00,0xf7,0x00,0x03,0x00, +0x39,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x37,0x01,0x00,0x00,0x38,0x01,0x00,0x00,0x39,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x38,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x3c,0x01,0x00,0x00,0xa8,0x02,0x00,0x00, +0x78,0x00,0x00,0x00,0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00, +0x3e,0x01,0x00,0x00,0x3c,0x01,0x00,0x00,0x8d,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x39,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x39,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0xc1,0x00,0x00,0x00, +0x3f,0x01,0x00,0x00,0x37,0x01,0x00,0x00,0x29,0x01,0x00,0x00, +0x3e,0x01,0x00,0x00,0x38,0x01,0x00,0x00,0xf7,0x00,0x03,0x00, +0x41,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x3f,0x01,0x00,0x00,0x40,0x01,0x00,0x00,0x5f,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x40,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x49,0x01,0x00,0x00,0x7d,0x00,0x00,0x00, +0xb9,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x4b,0x01,0x00,0x00,0x49,0x01,0x00,0x00,0x4a,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x4d,0x01,0x00,0x00, +0x4b,0x01,0x00,0x00,0x78,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x58,0x01,0x00,0x00,0x49,0x01,0x00,0x00, +0xaa,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x59,0x01,0x00,0x00,0xc0,0x02,0x00,0x00,0x58,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x5b,0x01,0x00,0x00, +0x59,0x01,0x00,0x00,0x78,0x00,0x00,0x00,0x41,0x00,0x06,0x00, +0x11,0x01,0x00,0x00,0x5c,0x01,0x00,0x00,0x51,0x01,0x00,0x00, +0x34,0x00,0x00,0x00,0x5b,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, +0xf6,0x00,0x00,0x00,0x5d,0x01,0x00,0x00,0x5c,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0x14,0x01,0x00,0x00,0x5e,0x01,0x00,0x00, +0x46,0x01,0x00,0x00,0x4d,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, +0x5e,0x01,0x00,0x00,0x5d,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0x41,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x5f,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x62,0x01,0x00,0x00, +0x7d,0x00,0x00,0x00,0xb9,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x64,0x01,0x00,0x00,0x62,0x01,0x00,0x00, +0x63,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x66,0x01,0x00,0x00,0x64,0x01,0x00,0x00,0x78,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x14,0x01,0x00,0x00,0x67,0x01,0x00,0x00, +0x46,0x01,0x00,0x00,0x66,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, +0x67,0x01,0x00,0x00,0x1e,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0x41,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x41,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0x2b,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x2b,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x6c,0x01,0x00,0x00,0xb9,0x02,0x00,0x00,0x6a,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0x28,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x2a,0x01,0x00,0x00,0xe0,0x00,0x04,0x00,0x0c,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x6d,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x70,0x01,0x00,0x00,0xbc,0x02,0x00,0x00, +0x6e,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x73,0x01,0x00,0x00,0xc0,0x02,0x00,0x00,0x71,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0x75,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x75,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xc2,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0x2a,0x01,0x00,0x00, +0x1f,0x02,0x00,0x00,0x78,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc1,0x00,0x00,0x00,0x7b,0x01,0x00,0x00,0xc2,0x02,0x00,0x00, +0x6c,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x77,0x01,0x00,0x00, +0x78,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x7b,0x01,0x00,0x00,0x76,0x01,0x00,0x00,0x77,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x76,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0x7d,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x7d,0x01,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xc6,0x02,0x00,0x00, +0x3e,0x00,0x00,0x00,0x76,0x01,0x00,0x00,0xa9,0x01,0x00,0x00, +0x80,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00, +0x83,0x01,0x00,0x00,0xc6,0x02,0x00,0x00,0x60,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0x7f,0x01,0x00,0x00,0x80,0x01,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x83,0x01,0x00,0x00, +0x7e,0x01,0x00,0x00,0x7f,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x7e,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x85,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x85,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xd8,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, +0x7e,0x01,0x00,0x00,0xa7,0x01,0x00,0x00,0x86,0x01,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00,0x8b,0x01,0x00,0x00, +0xd8,0x02,0x00,0x00,0x62,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0x87,0x01,0x00,0x00,0x86,0x01,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x8b,0x01,0x00,0x00,0x86,0x01,0x00,0x00, +0x87,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x86,0x01,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x91,0x01,0x00,0x00, +0xc6,0x02,0x00,0x00,0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x93,0x01,0x00,0x00,0x91,0x01,0x00,0x00, +0xd8,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x95,0x01,0x00,0x00,0x55,0x00,0x00,0x00,0x53,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x97,0x01,0x00,0x00, +0xc6,0x02,0x00,0x00,0x61,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x98,0x01,0x00,0x00,0x95,0x01,0x00,0x00, +0x97,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x9a,0x01,0x00,0x00,0x64,0x00,0x00,0x00,0x62,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9b,0x01,0x00,0x00, +0x98,0x01,0x00,0x00,0x9a,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x9d,0x01,0x00,0x00,0x9b,0x01,0x00,0x00, +0xd8,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x9f,0x01,0x00,0x00,0x9d,0x01,0x00,0x00,0x9e,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa1,0x01,0x00,0x00, +0x9f,0x01,0x00,0x00,0xc2,0x02,0x00,0x00,0x41,0x00,0x05,0x00, +0x14,0x01,0x00,0x00,0xa2,0x01,0x00,0x00,0xfb,0x00,0x00,0x00, +0xa1,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0xf6,0x00,0x00,0x00, +0xa3,0x01,0x00,0x00,0xa2,0x01,0x00,0x00,0x41,0x00,0x05,0x00, +0xa4,0x01,0x00,0x00,0xa5,0x01,0x00,0x00,0x8f,0x01,0x00,0x00, +0x93,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0xa5,0x01,0x00,0x00, +0xa3,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xa7,0x01,0x00,0x00,0xd8,0x02,0x00,0x00,0xcf,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x85,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x87,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x80,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x80,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xa9,0x01,0x00,0x00,0xc6,0x02,0x00,0x00, +0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x7d,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x7f,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0xab,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xab,0x01,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xc7,0x02,0x00,0x00, +0x3e,0x00,0x00,0x00,0x7f,0x01,0x00,0x00,0xd7,0x01,0x00,0x00, +0xae,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00, +0xb1,0x01,0x00,0x00,0xc7,0x02,0x00,0x00,0xbe,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0xad,0x01,0x00,0x00,0xae,0x01,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xb1,0x01,0x00,0x00, +0xac,0x01,0x00,0x00,0xad,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xac,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xb3,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xb3,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xd5,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, +0xac,0x01,0x00,0x00,0xd5,0x01,0x00,0x00,0xb4,0x01,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00,0xb9,0x01,0x00,0x00, +0xd5,0x02,0x00,0x00,0xbb,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0xb5,0x01,0x00,0x00,0xb4,0x01,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0xb9,0x01,0x00,0x00,0xb4,0x01,0x00,0x00, +0xb5,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xb4,0x01,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xbf,0x01,0x00,0x00, +0xc7,0x02,0x00,0x00,0xbb,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xc1,0x01,0x00,0x00,0xbf,0x01,0x00,0x00, 0xd5,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xde,0x02,0x00,0x00,0x36,0x03,0x00,0x00,0x61,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xdf,0x02,0x00,0x00, -0xb4,0x02,0x00,0x00,0xde,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xe1,0x02,0x00,0x00,0x64,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xe2,0x02,0x00,0x00,0xdf,0x02,0x00,0x00,0xe1,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xe6,0x02,0x00,0x00, -0x35,0x03,0x00,0x00,0x52,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xe7,0x02,0x00,0x00,0xba,0x02,0x00,0x00, -0xe6,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xe9,0x02,0x00,0x00,0x68,0x00,0x00,0x00,0xb2,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xea,0x02,0x00,0x00, -0xe7,0x02,0x00,0x00,0xe9,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0xec,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0xec,0x02,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x38,0x03,0x00,0x00, -0x3e,0x00,0x00,0x00,0xd5,0x02,0x00,0x00,0x2e,0x03,0x00,0x00, -0xef,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00, -0xf2,0x02,0x00,0x00,0x38,0x03,0x00,0x00,0xb2,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0xee,0x02,0x00,0x00,0xef,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xf2,0x02,0x00,0x00, -0xed,0x02,0x00,0x00,0xee,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0xed,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0xf4,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0xf4,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0x3a,0x03,0x00,0x00,0x3e,0x00,0x00,0x00, -0xed,0x02,0x00,0x00,0x2c,0x03,0x00,0x00,0xf7,0x02,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00,0xfa,0x02,0x00,0x00, -0x3a,0x03,0x00,0x00,0x62,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0xf6,0x02,0x00,0x00,0xf7,0x02,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xfa,0x02,0x00,0x00,0xf5,0x02,0x00,0x00, -0xf6,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0xf5,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xfd,0x02,0x00,0x00, -0xe2,0x02,0x00,0x00,0x3a,0x03,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb8,0x00,0x00,0x00,0x00,0x03,0x00,0x00,0xfd,0x02,0x00,0x00, -0x36,0x00,0x00,0x00,0xf7,0x00,0x03,0x00,0x02,0x03,0x00,0x00, -0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x00,0x03,0x00,0x00, -0x01,0x03,0x00,0x00,0x02,0x03,0x00,0x00,0xf8,0x00,0x02,0x00, -0x01,0x03,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x05,0x03,0x00,0x00,0xea,0x02,0x00,0x00,0x38,0x03,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x06,0x03,0x00,0x00, -0x12,0x00,0x00,0x00,0xc6,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x07,0x03,0x00,0x00,0x06,0x03,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00,0x08,0x03,0x00,0x00, -0x05,0x03,0x00,0x00,0x07,0x03,0x00,0x00,0xf9,0x00,0x02,0x00, -0x02,0x03,0x00,0x00,0xf8,0x00,0x02,0x00,0x02,0x03,0x00,0x00, -0xf5,0x00,0x07,0x00,0xb8,0x00,0x00,0x00,0x09,0x03,0x00,0x00, -0x00,0x03,0x00,0x00,0xf5,0x02,0x00,0x00,0x08,0x03,0x00,0x00, -0x01,0x03,0x00,0x00,0xf7,0x00,0x03,0x00,0x0b,0x03,0x00,0x00, -0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x09,0x03,0x00,0x00, -0x0a,0x03,0x00,0x00,0x0b,0x03,0x00,0x00,0xf8,0x00,0x02,0x00, -0x0a,0x03,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x13,0x03,0x00,0x00,0xea,0x02,0x00,0x00,0x38,0x03,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x15,0x03,0x00,0x00, -0x12,0x00,0x00,0x00,0x14,0x03,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x16,0x03,0x00,0x00,0x15,0x03,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x17,0x03,0x00,0x00, -0x13,0x03,0x00,0x00,0x16,0x03,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x18,0x03,0x00,0x00,0xca,0x02,0x00,0x00, -0x17,0x03,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x1a,0x03,0x00,0x00,0x18,0x03,0x00,0x00,0xe2,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x1c,0x03,0x00,0x00, -0x1a,0x03,0x00,0x00,0x3a,0x03,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x1e,0x03,0x00,0x00,0x35,0x03,0x00,0x00, -0xb2,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x20,0x03,0x00,0x00,0x1e,0x03,0x00,0x00,0x38,0x03,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x22,0x03,0x00,0x00, -0x20,0x03,0x00,0x00,0x21,0x03,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x24,0x03,0x00,0x00,0x36,0x03,0x00,0x00, -0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x25,0x03,0x00,0x00,0x22,0x03,0x00,0x00,0x24,0x03,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x27,0x03,0x00,0x00, -0x25,0x03,0x00,0x00,0x3a,0x03,0x00,0x00,0x41,0x00,0x05,0x00, -0xc3,0x00,0x00,0x00,0x28,0x03,0x00,0x00,0xc0,0x00,0x00,0x00, -0x27,0x03,0x00,0x00,0x3d,0x00,0x04,0x00,0xba,0x00,0x00,0x00, -0x29,0x03,0x00,0x00,0x28,0x03,0x00,0x00,0x41,0x00,0x06,0x00, -0x91,0x01,0x00,0x00,0x2a,0x03,0x00,0x00,0x0f,0x03,0x00,0x00, -0x34,0x00,0x00,0x00,0x1c,0x03,0x00,0x00,0x3e,0x00,0x03,0x00, -0x2a,0x03,0x00,0x00,0x29,0x03,0x00,0x00,0xf9,0x00,0x02,0x00, -0x0b,0x03,0x00,0x00,0xf8,0x00,0x02,0x00,0x0b,0x03,0x00,0x00, -0xf9,0x00,0x02,0x00,0xf7,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0xf7,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x2c,0x03,0x00,0x00,0x3a,0x03,0x00,0x00,0xc6,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xf4,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0xf6,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0xef,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0xef,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x2e,0x03,0x00,0x00,0x38,0x03,0x00,0x00, -0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xec,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0xee,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0xd7,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0xd7,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x30,0x03,0x00,0x00, -0x36,0x03,0x00,0x00,0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xd4,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0xd6,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0xcf,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0xcf,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x32,0x03,0x00,0x00,0x35,0x03,0x00,0x00,0xc6,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xcc,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0xce,0x02,0x00,0x00,0xfd,0x00,0x01,0x00,0x38,0x00,0x01,0x00, - +0xc3,0x01,0x00,0x00,0x59,0x00,0x00,0x00,0xb8,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xc6,0x01,0x00,0x00, +0xc7,0x02,0x00,0x00,0xc5,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xc7,0x01,0x00,0x00,0xc3,0x01,0x00,0x00, +0xc6,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xc9,0x01,0x00,0x00,0x68,0x00,0x00,0x00,0xbb,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xca,0x01,0x00,0x00, +0xc7,0x01,0x00,0x00,0xc9,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xcc,0x01,0x00,0x00,0xca,0x01,0x00,0x00, +0xd5,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xce,0x01,0x00,0x00,0xcc,0x01,0x00,0x00,0xcd,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xd0,0x01,0x00,0x00, +0xce,0x01,0x00,0x00,0xc2,0x02,0x00,0x00,0x41,0x00,0x05,0x00, +0x14,0x01,0x00,0x00,0xd1,0x01,0x00,0x00,0x46,0x01,0x00,0x00, +0xd0,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0xf6,0x00,0x00,0x00, +0xd2,0x01,0x00,0x00,0xd1,0x01,0x00,0x00,0x41,0x00,0x05,0x00, +0xa4,0x01,0x00,0x00,0xd3,0x01,0x00,0x00,0xbd,0x01,0x00,0x00, +0xc1,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0xd3,0x01,0x00,0x00, +0xd2,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xd5,0x01,0x00,0x00,0xd5,0x02,0x00,0x00,0xcf,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xb3,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xb5,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xae,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xae,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xd7,0x01,0x00,0x00,0xc7,0x02,0x00,0x00, +0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xab,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xad,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0xd9,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xd9,0x01,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xc8,0x02,0x00,0x00, +0x3e,0x00,0x00,0x00,0xad,0x01,0x00,0x00,0x1d,0x02,0x00,0x00, +0xdc,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00, +0xdf,0x01,0x00,0x00,0xc8,0x02,0x00,0x00,0xbe,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0xdb,0x01,0x00,0x00,0xdc,0x01,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xdf,0x01,0x00,0x00, +0xda,0x01,0x00,0x00,0xdb,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xda,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xe1,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xe1,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xcc,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, +0xda,0x01,0x00,0x00,0x1b,0x02,0x00,0x00,0xe4,0x01,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00,0xe7,0x01,0x00,0x00, +0xcc,0x02,0x00,0x00,0x60,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0xe3,0x01,0x00,0x00,0xe4,0x01,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0xe7,0x01,0x00,0x00,0xe2,0x01,0x00,0x00, +0xe3,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xe2,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0xe9,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xe9,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xce,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0xe2,0x01,0x00,0x00, +0x19,0x02,0x00,0x00,0xec,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc1,0x00,0x00,0x00,0xef,0x01,0x00,0x00,0xce,0x02,0x00,0x00, +0xbb,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xeb,0x01,0x00,0x00, +0xec,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0xef,0x01,0x00,0x00,0xea,0x01,0x00,0x00,0xeb,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xea,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0xf1,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xf1,0x01,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xd0,0x02,0x00,0x00, +0x3e,0x00,0x00,0x00,0xea,0x01,0x00,0x00,0x17,0x02,0x00,0x00, +0xf2,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00, +0xf7,0x01,0x00,0x00,0xd0,0x02,0x00,0x00,0x62,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0xf3,0x01,0x00,0x00,0xf2,0x01,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xf7,0x01,0x00,0x00, +0xf2,0x01,0x00,0x00,0xf3,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xf2,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xf9,0x01,0x00,0x00,0xc8,0x02,0x00,0x00,0xbb,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xfb,0x01,0x00,0x00, +0xf9,0x01,0x00,0x00,0xce,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xfd,0x01,0x00,0x00,0xfb,0x01,0x00,0x00, +0xfc,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xff,0x01,0x00,0x00,0xcc,0x02,0x00,0x00,0x62,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x00,0x02,0x00,0x00, +0xfd,0x01,0x00,0x00,0xff,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x02,0x02,0x00,0x00,0x00,0x02,0x00,0x00, +0xd0,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x06,0x02,0x00,0x00,0xff,0x01,0x00,0x00,0xd0,0x02,0x00,0x00, +0x41,0x00,0x05,0x00,0xa4,0x01,0x00,0x00,0x07,0x02,0x00,0x00, +0x8f,0x01,0x00,0x00,0x06,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, +0xf6,0x00,0x00,0x00,0x08,0x02,0x00,0x00,0x07,0x02,0x00,0x00, +0x73,0x00,0x04,0x00,0xc3,0x00,0x00,0x00,0x09,0x02,0x00,0x00, +0x08,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0xa4,0x01,0x00,0x00, +0x0e,0x02,0x00,0x00,0xbd,0x01,0x00,0x00,0xfb,0x01,0x00,0x00, +0x3d,0x00,0x04,0x00,0xf6,0x00,0x00,0x00,0x0f,0x02,0x00,0x00, +0x0e,0x02,0x00,0x00,0x73,0x00,0x04,0x00,0xc3,0x00,0x00,0x00, +0x10,0x02,0x00,0x00,0x0f,0x02,0x00,0x00,0x41,0x00,0x05,0x00, +0xcc,0x00,0x00,0x00,0x12,0x02,0x00,0x00,0xc9,0x00,0x00,0x00, +0x02,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0xc3,0x00,0x00,0x00, +0x13,0x02,0x00,0x00,0x12,0x02,0x00,0x00,0x0c,0x00,0x08,0x00, +0xc3,0x00,0x00,0x00,0x14,0x02,0x00,0x00,0x01,0x00,0x00,0x00, +0x32,0x00,0x00,0x00,0x09,0x02,0x00,0x00,0x10,0x02,0x00,0x00, +0x13,0x02,0x00,0x00,0x3e,0x00,0x03,0x00,0x12,0x02,0x00,0x00, +0x14,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x17,0x02,0x00,0x00,0xd0,0x02,0x00,0x00,0xcf,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xf1,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xf3,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xec,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xec,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x19,0x02,0x00,0x00,0xce,0x02,0x00,0x00, +0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xe9,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xeb,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0xe4,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xe4,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x1b,0x02,0x00,0x00, +0xcc,0x02,0x00,0x00,0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xe1,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xe3,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0xdc,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xdc,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x1d,0x02,0x00,0x00,0xc8,0x02,0x00,0x00,0xcf,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xd9,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xdb,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x78,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x78,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x1f,0x02,0x00,0x00,0xc2,0x02,0x00,0x00, +0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x75,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x77,0x01,0x00,0x00,0xe0,0x00,0x04,0x00, +0x0c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x6d,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0xd6,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xd6,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x21,0x02,0x00,0x00,0xa8,0x02,0x00,0x00,0x6c,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xd3,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xd5,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x26,0x02,0x00,0x00,0x55,0x00,0x00,0x00,0x53,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x27,0x02,0x00,0x00, +0x95,0x00,0x00,0x00,0x26,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x2c,0x02,0x00,0x00,0x59,0x00,0x00,0x00, +0xb8,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x2d,0x02,0x00,0x00,0xa7,0x00,0x00,0x00,0x2c,0x02,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x32,0x02,0x00,0x00, +0x12,0x00,0x00,0x00,0x31,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x33,0x02,0x00,0x00,0x32,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x34,0x02,0x00,0x00, +0x0f,0x00,0x00,0x00,0x33,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x38,0x02,0x00,0x00,0x47,0x00,0x00,0x00, +0x33,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00, +0x3a,0x02,0x00,0x00,0x39,0x02,0x00,0x00,0x0c,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x3b,0x02,0x00,0x00, +0x3a,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x3c,0x02,0x00,0x00,0x38,0x02,0x00,0x00,0x3b,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3d,0x02,0x00,0x00, +0x34,0x02,0x00,0x00,0x3c,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0x3f,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x3f,0x02,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xa9,0x02,0x00,0x00, +0x3e,0x00,0x00,0x00,0xd5,0x00,0x00,0x00,0xa6,0x02,0x00,0x00, +0x42,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00, +0x45,0x02,0x00,0x00,0xa9,0x02,0x00,0x00,0xbe,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0x41,0x02,0x00,0x00,0x42,0x02,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x45,0x02,0x00,0x00, +0x40,0x02,0x00,0x00,0x41,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x40,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x47,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x47,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xaa,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, +0x40,0x02,0x00,0x00,0xa4,0x02,0x00,0x00,0x4a,0x02,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00,0x4d,0x02,0x00,0x00, +0xaa,0x02,0x00,0x00,0x60,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0x49,0x02,0x00,0x00,0x4a,0x02,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x4d,0x02,0x00,0x00,0x48,0x02,0x00,0x00, +0x49,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x48,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x51,0x02,0x00,0x00, +0xaa,0x02,0x00,0x00,0x61,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x52,0x02,0x00,0x00,0x27,0x02,0x00,0x00, +0x51,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x54,0x02,0x00,0x00,0x64,0x00,0x00,0x00,0x62,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x55,0x02,0x00,0x00, +0x52,0x02,0x00,0x00,0x54,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x59,0x02,0x00,0x00,0xa9,0x02,0x00,0x00, +0xc5,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x5a,0x02,0x00,0x00,0x2d,0x02,0x00,0x00,0x59,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x5c,0x02,0x00,0x00, +0x68,0x00,0x00,0x00,0xbb,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x5d,0x02,0x00,0x00,0x5a,0x02,0x00,0x00, +0x5c,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x5f,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x5f,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xac,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, +0x48,0x02,0x00,0x00,0xa2,0x02,0x00,0x00,0x62,0x02,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00,0x65,0x02,0x00,0x00, +0xac,0x02,0x00,0x00,0xbb,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0x61,0x02,0x00,0x00,0x62,0x02,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x65,0x02,0x00,0x00,0x60,0x02,0x00,0x00, +0x61,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x60,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0x67,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x67,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xae,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0x60,0x02,0x00,0x00, +0xa0,0x02,0x00,0x00,0x6a,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc1,0x00,0x00,0x00,0x6d,0x02,0x00,0x00,0xae,0x02,0x00,0x00, +0x62,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x69,0x02,0x00,0x00, +0x6a,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x6d,0x02,0x00,0x00,0x68,0x02,0x00,0x00,0x69,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x68,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x70,0x02,0x00,0x00,0x55,0x02,0x00,0x00, +0xae,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00, +0x73,0x02,0x00,0x00,0x70,0x02,0x00,0x00,0x36,0x00,0x00,0x00, +0xf7,0x00,0x03,0x00,0x75,0x02,0x00,0x00,0x00,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x73,0x02,0x00,0x00,0x74,0x02,0x00,0x00, +0x75,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x74,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x78,0x02,0x00,0x00, +0x5d,0x02,0x00,0x00,0xac,0x02,0x00,0x00,0x41,0x00,0x05,0x00, +0x15,0x00,0x00,0x00,0x79,0x02,0x00,0x00,0x12,0x00,0x00,0x00, +0xcf,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x7a,0x02,0x00,0x00,0x79,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc1,0x00,0x00,0x00,0x7b,0x02,0x00,0x00,0x78,0x02,0x00,0x00, +0x7a,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x75,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x75,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, +0xc1,0x00,0x00,0x00,0x7c,0x02,0x00,0x00,0x73,0x02,0x00,0x00, +0x68,0x02,0x00,0x00,0x7b,0x02,0x00,0x00,0x74,0x02,0x00,0x00, +0xf7,0x00,0x03,0x00,0x7e,0x02,0x00,0x00,0x00,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x7c,0x02,0x00,0x00,0x7d,0x02,0x00,0x00, +0x7e,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x7d,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x86,0x02,0x00,0x00, +0x5d,0x02,0x00,0x00,0xac,0x02,0x00,0x00,0x41,0x00,0x05,0x00, +0x15,0x00,0x00,0x00,0x88,0x02,0x00,0x00,0x12,0x00,0x00,0x00, +0x87,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x89,0x02,0x00,0x00,0x88,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x8a,0x02,0x00,0x00,0x86,0x02,0x00,0x00, +0x89,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x8b,0x02,0x00,0x00,0x3d,0x02,0x00,0x00,0x8a,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8d,0x02,0x00,0x00, +0x8b,0x02,0x00,0x00,0x55,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x8f,0x02,0x00,0x00,0x8d,0x02,0x00,0x00, +0xae,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x91,0x02,0x00,0x00,0xa9,0x02,0x00,0x00,0xbb,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x93,0x02,0x00,0x00, +0x91,0x02,0x00,0x00,0xac,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x95,0x02,0x00,0x00,0x93,0x02,0x00,0x00, +0x94,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x97,0x02,0x00,0x00,0xaa,0x02,0x00,0x00,0x62,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x98,0x02,0x00,0x00, +0x95,0x02,0x00,0x00,0x97,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x9a,0x02,0x00,0x00,0x98,0x02,0x00,0x00, +0xae,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0xcc,0x00,0x00,0x00, +0x9b,0x02,0x00,0x00,0xc9,0x00,0x00,0x00,0x9a,0x02,0x00,0x00, +0x3d,0x00,0x04,0x00,0xc3,0x00,0x00,0x00,0x9c,0x02,0x00,0x00, +0x9b,0x02,0x00,0x00,0x41,0x00,0x06,0x00,0x9d,0x02,0x00,0x00, +0x9e,0x02,0x00,0x00,0x82,0x02,0x00,0x00,0x34,0x00,0x00,0x00, +0x8f,0x02,0x00,0x00,0x3e,0x00,0x03,0x00,0x9e,0x02,0x00,0x00, +0x9c,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x7e,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x7e,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0x6a,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x6a,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa0,0x02,0x00,0x00, +0xae,0x02,0x00,0x00,0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x67,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x69,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0x62,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x62,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xa2,0x02,0x00,0x00,0xac,0x02,0x00,0x00,0xcf,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x5f,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x61,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x4a,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x4a,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xa4,0x02,0x00,0x00,0xaa,0x02,0x00,0x00, +0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x47,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x49,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0x42,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x42,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa6,0x02,0x00,0x00, +0xa9,0x02,0x00,0x00,0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x3f,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x41,0x02,0x00,0x00, +0xfd,0x00,0x01,0x00,0x38,0x00,0x01,0x00, }; -const uint64_t matmul_f16_f32_aligned_l_len = 12096; +const uint64_t matmul_f16_len = 10316; -unsigned char matmul_f16_f32_aligned_l_fp32_data[] = { +unsigned char matmul_f16_aligned_data[] = { 0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, -0xf0,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, -0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x51,0x11,0x00,0x00, -0x0b,0x00,0x06,0x00,0x01,0x00,0x00,0x00,0x47,0x4c,0x53,0x4c, -0x2e,0x73,0x74,0x64,0x2e,0x34,0x35,0x30,0x00,0x00,0x00,0x00, -0x0e,0x00,0x03,0x00,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x0f,0x00,0x0f,0x00,0x05,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x6d,0x61,0x69,0x6e,0x00,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x3d,0x00,0x00,0x00,0x4c,0x00,0x00,0x00, -0xea,0x00,0x00,0x00,0xf9,0x00,0x00,0x00,0x4b,0x01,0x00,0x00, -0x59,0x01,0x00,0x00,0x51,0x02,0x00,0x00,0x9a,0x02,0x00,0x00, -0x10,0x00,0x06,0x00,0x04,0x00,0x00,0x00,0x11,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x0b,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x1c,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x10,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x08,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x03,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x10,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x14,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x18,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x07,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x10,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x24,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x0a,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x10,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x2c,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x30,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x0d,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0x47,0x00,0x03,0x00, -0x10,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x37,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x3d,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x1a,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x4c,0x00,0x00,0x00, -0x0b,0x00,0x00,0x00,0x1b,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x4f,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x53,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x60,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x62,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x07,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x6c,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x03,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x9d,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0xaf,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x05,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0xb2,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x08,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xf6,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x48,0x00,0x04,0x00, -0xf7,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0xf7,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, -0xf7,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0xf9,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0xf9,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x2c,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x2d,0x01,0x00,0x00,0x0b,0x00,0x00,0x00,0x19,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x56,0x01,0x00,0x00,0x06,0x00,0x00,0x00, -0x10,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0x57,0x01,0x00,0x00, -0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x57,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x57,0x01,0x00,0x00, -0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x59,0x01,0x00,0x00, -0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x59,0x01,0x00,0x00,0x21,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x51,0x02,0x00,0x00,0x0b,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x97,0x02,0x00,0x00, -0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00, -0x98,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x19,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x98,0x02,0x00,0x00,0x00,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, -0x98,0x02,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x9a,0x02,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x9a,0x02,0x00,0x00,0x21,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x13,0x00,0x02,0x00,0x02,0x00,0x00,0x00, -0x21,0x00,0x03,0x00,0x03,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x15,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x17,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x0a,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x0d,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x1e,0x00,0x10,0x00,0x10,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x11,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x11,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x15,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, -0x14,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x15,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x0a,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, -0x27,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0x2d,0x00,0x00,0x00,0x07,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x34,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x37,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00,0x3d,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x3e,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x0a,0x00,0x00,0x00,0x4c,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x4f,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x53,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x54,0x00,0x00,0x00,0x86,0x00,0x00,0x00, -0x37,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x58,0x00,0x00,0x00,0x86,0x00,0x00,0x00, -0x37,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x32,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x61,0x00,0x00,0x00, -0x86,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x60,0x00,0x00,0x00, -0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x62,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x63,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x61,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x67,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x61,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x6e,0x00,0x00,0x00, -0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x73,0x00,0x00,0x00, -0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x77,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, -0x7c,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0x87,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x8d,0x00,0x00,0x00, -0x03,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, -0x98,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x32,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x9d,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x9f,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xae,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xaf,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xb0,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0x53,0x00,0x00,0x00,0xaf,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xb1,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0x4f,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x32,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xb2,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xb3,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0xb1,0x00,0x00,0x00,0xb2,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xb4,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0xb3,0x00,0x00,0x00,0x60,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xb5,0x00,0x00,0x00, -0x86,0x00,0x00,0x00,0xb0,0x00,0x00,0x00,0xb4,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xb6,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0xae,0x00,0x00,0x00,0xb5,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xb7,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0xb6,0x00,0x00,0x00,0xb2,0x00,0x00,0x00, -0x14,0x00,0x02,0x00,0xb8,0x00,0x00,0x00,0x16,0x00,0x03,0x00, -0xba,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xbb,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xbc,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0xbb,0x00,0x00,0x00,0xb5,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xbd,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0xbc,0x00,0x00,0x00,0xb2,0x00,0x00,0x00,0x1c,0x00,0x04,0x00, -0xbe,0x00,0x00,0x00,0xba,0x00,0x00,0x00,0xbd,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0xbf,0x00,0x00,0x00,0x07,0x00,0x00,0x00, -0xbe,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0xba,0x00,0x00,0x00, -0xc2,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0xc3,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0xba,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0xc6,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xe6,0x00,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xe7,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x37,0x00,0x00,0x00, -0xe6,0x00,0x00,0x00,0x1c,0x00,0x04,0x00,0xe8,0x00,0x00,0x00, -0xba,0x00,0x00,0x00,0xe7,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0xe9,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0xe8,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0xe9,0x00,0x00,0x00,0xea,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xee,0x00,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x16,0x00,0x03,0x00,0xf4,0x00,0x00,0x00, -0x10,0x00,0x00,0x00,0x17,0x00,0x04,0x00,0xf5,0x00,0x00,0x00, -0xf4,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, -0xf6,0x00,0x00,0x00,0xf5,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, -0xf7,0x00,0x00,0x00,0xf6,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0xf8,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0xf7,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0xf8,0x00,0x00,0x00,0xf9,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0xfb,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0xf4,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0xff,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0xba,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x04,0x01,0x00,0x00, -0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x12,0x01,0x00,0x00, -0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x20,0x01,0x00,0x00, -0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x25,0x01,0x00,0x00, -0x03,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x2c,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x33,0x00,0x06,0x00, -0x09,0x00,0x00,0x00,0x2d,0x01,0x00,0x00,0x2c,0x01,0x00,0x00, -0x39,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x2e,0x01,0x00,0x00,0x51,0x00,0x00,0x00, -0x2d,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x2f,0x01,0x00,0x00,0x84,0x00,0x00,0x00, -0x2e,0x01,0x00,0x00,0x6d,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x30,0x01,0x00,0x00,0x86,0x00,0x00,0x00, -0x2f,0x01,0x00,0x00,0x6c,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x47,0x01,0x00,0x00,0x80,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x48,0x01,0x00,0x00,0x84,0x00,0x00,0x00, -0x9d,0x00,0x00,0x00,0x47,0x01,0x00,0x00,0x1c,0x00,0x04,0x00, -0x49,0x01,0x00,0x00,0xba,0x00,0x00,0x00,0x48,0x01,0x00,0x00, -0x20,0x00,0x04,0x00,0x4a,0x01,0x00,0x00,0x04,0x00,0x00,0x00, -0x49,0x01,0x00,0x00,0x3b,0x00,0x04,0x00,0x4a,0x01,0x00,0x00, -0x4b,0x01,0x00,0x00,0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x4f,0x01,0x00,0x00,0x80,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x17,0x00,0x04,0x00, -0x55,0x01,0x00,0x00,0xba,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x1d,0x00,0x03,0x00,0x56,0x01,0x00,0x00,0x55,0x01,0x00,0x00, -0x1e,0x00,0x03,0x00,0x57,0x01,0x00,0x00,0x56,0x01,0x00,0x00, -0x20,0x00,0x04,0x00,0x58,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, -0x57,0x01,0x00,0x00,0x3b,0x00,0x04,0x00,0x58,0x01,0x00,0x00, -0x59,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x5b,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0xba,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x62,0x01,0x00,0x00, -0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x6f,0x01,0x00,0x00, -0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x7c,0x01,0x00,0x00, -0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x88,0x01,0x00,0x00, -0x08,0x01,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x89,0x01,0x00,0x00,0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x6d,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x8c,0x01,0x00,0x00,0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x6d,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xa7,0x01,0x00,0x00,0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x1c,0x00,0x04,0x00,0xa8,0x01,0x00,0x00, -0xba,0x00,0x00,0x00,0xa7,0x01,0x00,0x00,0x20,0x00,0x04,0x00, -0xa9,0x01,0x00,0x00,0x07,0x00,0x00,0x00,0xa8,0x01,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xb9,0x01,0x00,0x00, -0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xd4,0x01,0x00,0x00, -0x84,0x00,0x00,0x00,0xb5,0x00,0x00,0x00,0xb2,0x00,0x00,0x00, -0x1c,0x00,0x04,0x00,0xd5,0x01,0x00,0x00,0xba,0x00,0x00,0x00, -0xd4,0x01,0x00,0x00,0x20,0x00,0x04,0x00,0xd6,0x01,0x00,0x00, -0x07,0x00,0x00,0x00,0xd5,0x01,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xdf,0x01,0x00,0x00,0x86,0x00,0x00,0x00, -0xaf,0x00,0x00,0x00,0xb5,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xe7,0x01,0x00,0x00,0x80,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x16,0x02,0x00,0x00,0x84,0x00,0x00,0x00, -0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0x49,0x02,0x00,0x00,0x0d,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00,0x51,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0x1d,0x00,0x03,0x00,0x97,0x02,0x00,0x00, -0xba,0x00,0x00,0x00,0x1e,0x00,0x03,0x00,0x98,0x02,0x00,0x00, -0x97,0x02,0x00,0x00,0x20,0x00,0x04,0x00,0x99,0x02,0x00,0x00, -0x0c,0x00,0x00,0x00,0x98,0x02,0x00,0x00,0x3b,0x00,0x04,0x00, -0x99,0x02,0x00,0x00,0x9a,0x02,0x00,0x00,0x0c,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x9f,0x02,0x00,0x00, -0x05,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xac,0x02,0x00,0x00,0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x36,0x00,0x05,0x00,0x02,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x03,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0x05,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0xbf,0x00,0x00,0x00,0xc0,0x00,0x00,0x00,0x07,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0xa9,0x01,0x00,0x00,0xaa,0x01,0x00,0x00, -0x07,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0xd6,0x01,0x00,0x00, -0xd7,0x01,0x00,0x00,0x07,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x0d,0x00,0x00,0x00,0x0e,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x0f,0x00,0x00,0x00,0x0e,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x14,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x17,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, -0x17,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x1e,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x17,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x22,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x22,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x24,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x27,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x29,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x2a,0x00,0x00,0x00,0x1e,0x00,0x00,0x00, -0x29,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x2d,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x2f,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x30,0x00,0x00,0x00,0x24,0x00,0x00,0x00,0x2f,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x32,0x00,0x00,0x00, -0x30,0x00,0x00,0x00,0x2a,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x35,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x34,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x36,0x00,0x00,0x00,0x35,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x38,0x00,0x00,0x00,0x36,0x00,0x00,0x00, -0x37,0x00,0x00,0x00,0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x3a,0x00,0x00,0x00,0x38,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3b,0x00,0x00,0x00, -0x3a,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x0d,0x00,0x00,0x00,0x3f,0x00,0x00,0x00,0x3d,0x00,0x00,0x00, -0x3e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0x3f,0x00,0x00,0x00,0x89,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x42,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x3b,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x47,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x3b,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00,0x49,0x00,0x00,0x00, -0x3d,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x4a,0x00,0x00,0x00,0x49,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00,0x4d,0x00,0x00,0x00, -0x4c,0x00,0x00,0x00,0x3e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x4d,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x50,0x00,0x00,0x00, -0x4e,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x89,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x55,0x00,0x00,0x00,0x50,0x00,0x00,0x00, -0x54,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x59,0x00,0x00,0x00,0x50,0x00,0x00,0x00,0x58,0x00,0x00,0x00, -0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x5d,0x00,0x00,0x00, -0x4e,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x89,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x64,0x00,0x00,0x00,0x5d,0x00,0x00,0x00, -0x63,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x68,0x00,0x00,0x00,0x5d,0x00,0x00,0x00,0x67,0x00,0x00,0x00, -0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x6f,0x00,0x00,0x00, -0x4e,0x00,0x00,0x00,0x6e,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x74,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, -0x73,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0x78,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x77,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x79,0x00,0x00,0x00, -0x78,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x7a,0x00,0x00,0x00,0x47,0x00,0x00,0x00,0x79,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x7d,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x7c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x7e,0x00,0x00,0x00,0x7d,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x80,0x00,0x00,0x00, -0x47,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x83,0x00,0x00,0x00,0x80,0x00,0x00,0x00, -0x79,0x00,0x00,0x00,0x0c,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x26,0x00,0x00,0x00, -0x7e,0x00,0x00,0x00,0x83,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x88,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x87,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x89,0x00,0x00,0x00,0x88,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x8a,0x00,0x00,0x00,0x32,0x00,0x00,0x00, -0x89,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x8c,0x00,0x00,0x00,0x42,0x00,0x00,0x00,0x37,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x8e,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x8d,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x8f,0x00,0x00,0x00,0x8e,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x90,0x00,0x00,0x00, -0x8c,0x00,0x00,0x00,0x8f,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x91,0x00,0x00,0x00,0x8a,0x00,0x00,0x00, -0x90,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x93,0x00,0x00,0x00,0x91,0x00,0x00,0x00,0x7a,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x94,0x00,0x00,0x00, -0x93,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x99,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x98,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x9a,0x00,0x00,0x00,0x99,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x9b,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, -0x9a,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x9e,0x00,0x00,0x00,0x4a,0x00,0x00,0x00,0x9d,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0xa0,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x9f,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xa1,0x00,0x00,0x00,0xa0,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa2,0x00,0x00,0x00, -0x9e,0x00,0x00,0x00,0xa1,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xa3,0x00,0x00,0x00,0x9b,0x00,0x00,0x00, -0xa2,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xa5,0x00,0x00,0x00,0xa3,0x00,0x00,0x00,0x7a,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa6,0x00,0x00,0x00, -0xa5,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xa8,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xa8,0x00,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xbe,0x02,0x00,0x00, -0x3e,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0xc7,0x00,0x00,0x00, -0xa9,0x00,0x00,0x00,0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00, -0xb9,0x00,0x00,0x00,0xbe,0x02,0x00,0x00,0xb7,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0xaa,0x00,0x00,0x00,0xa9,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xb9,0x00,0x00,0x00, -0xa9,0x00,0x00,0x00,0xaa,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xa9,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0xc3,0x00,0x00,0x00, -0xc4,0x00,0x00,0x00,0xc0,0x00,0x00,0x00,0xbe,0x02,0x00,0x00, -0x3e,0x00,0x03,0x00,0xc4,0x00,0x00,0x00,0xc2,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xc7,0x00,0x00,0x00, -0xbe,0x02,0x00,0x00,0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xa8,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xaa,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xca,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xca,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xd7,0x02,0x00,0x00,0xa6,0x00,0x00,0x00,0xaa,0x00,0x00,0x00, -0x8e,0x01,0x00,0x00,0xcd,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xd3,0x02,0x00,0x00,0x94,0x00,0x00,0x00, -0xaa,0x00,0x00,0x00,0x8b,0x01,0x00,0x00,0xcd,0x00,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xbf,0x02,0x00,0x00, -0x7a,0x00,0x00,0x00,0xaa,0x00,0x00,0x00,0x39,0x02,0x00,0x00, -0xcd,0x00,0x00,0x00,0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00, -0xd1,0x00,0x00,0x00,0xbf,0x02,0x00,0x00,0x84,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0xcc,0x00,0x00,0x00,0xcd,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xd1,0x00,0x00,0x00, -0xcb,0x00,0x00,0x00,0xcc,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xcb,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xd3,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xd3,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xcf,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, -0xcb,0x00,0x00,0x00,0x32,0x01,0x00,0x00,0xd4,0x00,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00,0xd9,0x00,0x00,0x00, -0xcf,0x02,0x00,0x00,0x37,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0xd5,0x00,0x00,0x00,0xd4,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xd9,0x00,0x00,0x00,0xd4,0x00,0x00,0x00, -0xd5,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xd4,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xde,0x00,0x00,0x00, -0x74,0x00,0x00,0x00,0xcf,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xe1,0x00,0x00,0x00,0xde,0x00,0x00,0x00, -0x8f,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xe2,0x00,0x00,0x00,0xe1,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xe3,0x00,0x00,0x00, -0xd3,0x02,0x00,0x00,0xe2,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xe5,0x00,0x00,0x00,0xe3,0x00,0x00,0x00, -0x6f,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xef,0x00,0x00,0x00,0xde,0x00,0x00,0x00,0xee,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xf1,0x00,0x00,0x00, -0x6f,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xf2,0x00,0x00,0x00,0xef,0x00,0x00,0x00, -0xf1,0x00,0x00,0x00,0x41,0x00,0x07,0x00,0xfb,0x00,0x00,0x00, -0xfc,0x00,0x00,0x00,0xf9,0x00,0x00,0x00,0x34,0x00,0x00,0x00, -0xe5,0x00,0x00,0x00,0x3e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0xf4,0x00,0x00,0x00,0xfd,0x00,0x00,0x00,0xfc,0x00,0x00,0x00, -0x73,0x00,0x04,0x00,0xba,0x00,0x00,0x00,0xfe,0x00,0x00,0x00, -0xfd,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0xff,0x00,0x00,0x00, -0x00,0x01,0x00,0x00,0xea,0x00,0x00,0x00,0xf2,0x00,0x00,0x00, -0x3e,0x00,0x03,0x00,0x00,0x01,0x00,0x00,0xfe,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x05,0x01,0x00,0x00, -0xde,0x00,0x00,0x00,0x04,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x08,0x01,0x00,0x00,0x05,0x01,0x00,0x00, -0xf1,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x09,0x01,0x00,0x00,0x08,0x01,0x00,0x00,0x39,0x00,0x00,0x00, -0x41,0x00,0x07,0x00,0xfb,0x00,0x00,0x00,0x0b,0x01,0x00,0x00, -0xf9,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0xe5,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0xf4,0x00,0x00,0x00, -0x0c,0x01,0x00,0x00,0x0b,0x01,0x00,0x00,0x73,0x00,0x04,0x00, -0xba,0x00,0x00,0x00,0x0d,0x01,0x00,0x00,0x0c,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0xff,0x00,0x00,0x00,0x0e,0x01,0x00,0x00, -0xea,0x00,0x00,0x00,0x09,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0x0e,0x01,0x00,0x00,0x0d,0x01,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x13,0x01,0x00,0x00,0xde,0x00,0x00,0x00, -0x12,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x16,0x01,0x00,0x00,0x13,0x01,0x00,0x00,0xf1,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x17,0x01,0x00,0x00, -0x16,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x41,0x00,0x07,0x00, -0xfb,0x00,0x00,0x00,0x19,0x01,0x00,0x00,0xf9,0x00,0x00,0x00, -0x34,0x00,0x00,0x00,0xe5,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0xf4,0x00,0x00,0x00,0x1a,0x01,0x00,0x00, -0x19,0x01,0x00,0x00,0x73,0x00,0x04,0x00,0xba,0x00,0x00,0x00, -0x1b,0x01,0x00,0x00,0x1a,0x01,0x00,0x00,0x41,0x00,0x05,0x00, -0xff,0x00,0x00,0x00,0x1c,0x01,0x00,0x00,0xea,0x00,0x00,0x00, -0x17,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x1c,0x01,0x00,0x00, -0x1b,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x21,0x01,0x00,0x00,0xde,0x00,0x00,0x00,0x20,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x24,0x01,0x00,0x00, -0x21,0x01,0x00,0x00,0xf1,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x26,0x01,0x00,0x00,0x24,0x01,0x00,0x00, -0x25,0x01,0x00,0x00,0x41,0x00,0x07,0x00,0xfb,0x00,0x00,0x00, -0x28,0x01,0x00,0x00,0xf9,0x00,0x00,0x00,0x34,0x00,0x00,0x00, -0xe5,0x00,0x00,0x00,0x25,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, -0xf4,0x00,0x00,0x00,0x29,0x01,0x00,0x00,0x28,0x01,0x00,0x00, -0x73,0x00,0x04,0x00,0xba,0x00,0x00,0x00,0x2a,0x01,0x00,0x00, -0x29,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0xff,0x00,0x00,0x00, -0x2b,0x01,0x00,0x00,0xea,0x00,0x00,0x00,0x26,0x01,0x00,0x00, -0x3e,0x00,0x03,0x00,0x2b,0x01,0x00,0x00,0x2a,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x32,0x01,0x00,0x00, -0xcf,0x02,0x00,0x00,0x30,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0xd3,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xd5,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x34,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x34,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xd0,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0xd5,0x00,0x00,0x00, -0x87,0x01,0x00,0x00,0x35,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb8,0x00,0x00,0x00,0x3a,0x01,0x00,0x00,0xd0,0x02,0x00,0x00, -0x9d,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x36,0x01,0x00,0x00, -0x35,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x3a,0x01,0x00,0x00,0x35,0x01,0x00,0x00,0x36,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x35,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x3f,0x01,0x00,0x00,0x74,0x00,0x00,0x00, -0xd0,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x42,0x01,0x00,0x00,0x3f,0x01,0x00,0x00,0xa1,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x43,0x01,0x00,0x00, -0x42,0x01,0x00,0x00,0x6d,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x44,0x01,0x00,0x00,0xd7,0x02,0x00,0x00, -0x43,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x46,0x01,0x00,0x00,0x44,0x01,0x00,0x00,0x6f,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x50,0x01,0x00,0x00, -0x3f,0x01,0x00,0x00,0x4f,0x01,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x52,0x01,0x00,0x00,0x6f,0x00,0x00,0x00, -0x6d,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x53,0x01,0x00,0x00,0x50,0x01,0x00,0x00,0x52,0x01,0x00,0x00, -0x41,0x00,0x07,0x00,0x5b,0x01,0x00,0x00,0x5c,0x01,0x00,0x00, -0x59,0x01,0x00,0x00,0x34,0x00,0x00,0x00,0x46,0x01,0x00,0x00, -0x3e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0xba,0x00,0x00,0x00, -0x5d,0x01,0x00,0x00,0x5c,0x01,0x00,0x00,0x41,0x00,0x05,0x00, -0xff,0x00,0x00,0x00,0x5e,0x01,0x00,0x00,0x4b,0x01,0x00,0x00, -0x53,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x5e,0x01,0x00,0x00, -0x5d,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x63,0x01,0x00,0x00,0x3f,0x01,0x00,0x00,0x62,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x66,0x01,0x00,0x00, -0x63,0x01,0x00,0x00,0x52,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x67,0x01,0x00,0x00,0x66,0x01,0x00,0x00, -0x39,0x00,0x00,0x00,0x41,0x00,0x07,0x00,0x5b,0x01,0x00,0x00, -0x69,0x01,0x00,0x00,0x59,0x01,0x00,0x00,0x34,0x00,0x00,0x00, -0x46,0x01,0x00,0x00,0x39,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0xba,0x00,0x00,0x00,0x6a,0x01,0x00,0x00,0x69,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0xff,0x00,0x00,0x00,0x6b,0x01,0x00,0x00, -0x4b,0x01,0x00,0x00,0x67,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0x6b,0x01,0x00,0x00,0x6a,0x01,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x70,0x01,0x00,0x00,0x3f,0x01,0x00,0x00, -0x6f,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x73,0x01,0x00,0x00,0x70,0x01,0x00,0x00,0x52,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x74,0x01,0x00,0x00, -0x73,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x41,0x00,0x07,0x00, -0x5b,0x01,0x00,0x00,0x76,0x01,0x00,0x00,0x59,0x01,0x00,0x00, -0x34,0x00,0x00,0x00,0x46,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0xba,0x00,0x00,0x00,0x77,0x01,0x00,0x00, -0x76,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0xff,0x00,0x00,0x00, -0x78,0x01,0x00,0x00,0x4b,0x01,0x00,0x00,0x74,0x01,0x00,0x00, -0x3e,0x00,0x03,0x00,0x78,0x01,0x00,0x00,0x77,0x01,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x7d,0x01,0x00,0x00, -0x3f,0x01,0x00,0x00,0x7c,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x80,0x01,0x00,0x00,0x7d,0x01,0x00,0x00, -0x52,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x81,0x01,0x00,0x00,0x80,0x01,0x00,0x00,0x25,0x01,0x00,0x00, -0x41,0x00,0x07,0x00,0x5b,0x01,0x00,0x00,0x83,0x01,0x00,0x00, -0x59,0x01,0x00,0x00,0x34,0x00,0x00,0x00,0x46,0x01,0x00,0x00, -0x25,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0xba,0x00,0x00,0x00, -0x84,0x01,0x00,0x00,0x83,0x01,0x00,0x00,0x41,0x00,0x05,0x00, -0xff,0x00,0x00,0x00,0x85,0x01,0x00,0x00,0x4b,0x01,0x00,0x00, -0x81,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x85,0x01,0x00,0x00, -0x84,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x87,0x01,0x00,0x00,0xd0,0x02,0x00,0x00,0x30,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0x34,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x36,0x01,0x00,0x00,0xe0,0x00,0x04,0x00,0x0c,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x88,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x8b,0x01,0x00,0x00,0xd3,0x02,0x00,0x00, -0x89,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x8e,0x01,0x00,0x00,0xd7,0x02,0x00,0x00,0x8c,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0x90,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x90,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xd9,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0x36,0x01,0x00,0x00, -0x37,0x02,0x00,0x00,0x93,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb8,0x00,0x00,0x00,0x96,0x01,0x00,0x00,0xd9,0x02,0x00,0x00, -0x6c,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x92,0x01,0x00,0x00, -0x93,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x96,0x01,0x00,0x00,0x91,0x01,0x00,0x00,0x92,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x91,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0x98,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x98,0x01,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xdd,0x02,0x00,0x00, -0x3e,0x00,0x00,0x00,0x91,0x01,0x00,0x00,0xc3,0x01,0x00,0x00, -0x9b,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00, -0x9e,0x01,0x00,0x00,0xdd,0x02,0x00,0x00,0x60,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0x9a,0x01,0x00,0x00,0x9b,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x9e,0x01,0x00,0x00, -0x99,0x01,0x00,0x00,0x9a,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x99,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xa0,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xa0,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xef,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, -0x99,0x01,0x00,0x00,0xc1,0x01,0x00,0x00,0xa1,0x01,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00,0xa6,0x01,0x00,0x00, -0xef,0x02,0x00,0x00,0x62,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0xa2,0x01,0x00,0x00,0xa1,0x01,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xa6,0x01,0x00,0x00,0xa1,0x01,0x00,0x00, -0xa2,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xa1,0x01,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xac,0x01,0x00,0x00, -0xdd,0x02,0x00,0x00,0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xae,0x01,0x00,0x00,0xac,0x01,0x00,0x00, -0xef,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xb0,0x01,0x00,0x00,0x55,0x00,0x00,0x00,0x53,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xb2,0x01,0x00,0x00, -0xdd,0x02,0x00,0x00,0x61,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xb3,0x01,0x00,0x00,0xb0,0x01,0x00,0x00, -0xb2,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xb5,0x01,0x00,0x00,0x64,0x00,0x00,0x00,0x62,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xb6,0x01,0x00,0x00, -0xb3,0x01,0x00,0x00,0xb5,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xb8,0x01,0x00,0x00,0xb6,0x01,0x00,0x00, -0xef,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xba,0x01,0x00,0x00,0xb8,0x01,0x00,0x00,0xb9,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xbc,0x01,0x00,0x00, -0xba,0x01,0x00,0x00,0xd9,0x02,0x00,0x00,0x41,0x00,0x05,0x00, -0xff,0x00,0x00,0x00,0xbd,0x01,0x00,0x00,0xea,0x00,0x00,0x00, -0xbc,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0xba,0x00,0x00,0x00, -0xbe,0x01,0x00,0x00,0xbd,0x01,0x00,0x00,0x41,0x00,0x05,0x00, -0xc3,0x00,0x00,0x00,0xbf,0x01,0x00,0x00,0xaa,0x01,0x00,0x00, -0xae,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0xbf,0x01,0x00,0x00, -0xbe,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xc1,0x01,0x00,0x00,0xef,0x02,0x00,0x00,0xc6,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xa0,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xa2,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x9b,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x9b,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xc3,0x01,0x00,0x00,0xdd,0x02,0x00,0x00, -0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x98,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x9a,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0xc5,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xc5,0x01,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xde,0x02,0x00,0x00, -0x3e,0x00,0x00,0x00,0x9a,0x01,0x00,0x00,0xf1,0x01,0x00,0x00, -0xc8,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00, -0xcb,0x01,0x00,0x00,0xde,0x02,0x00,0x00,0xb5,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0xc7,0x01,0x00,0x00,0xc8,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xcb,0x01,0x00,0x00, -0xc6,0x01,0x00,0x00,0xc7,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xc6,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xcd,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xcd,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xec,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, -0xc6,0x01,0x00,0x00,0xef,0x01,0x00,0x00,0xce,0x01,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00,0xd3,0x01,0x00,0x00, -0xec,0x02,0x00,0x00,0xb2,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0xcf,0x01,0x00,0x00,0xce,0x01,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xd3,0x01,0x00,0x00,0xce,0x01,0x00,0x00, -0xcf,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xce,0x01,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xd9,0x01,0x00,0x00, -0xde,0x02,0x00,0x00,0xb2,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xdb,0x01,0x00,0x00,0xd9,0x01,0x00,0x00, -0xec,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xdd,0x01,0x00,0x00,0x59,0x00,0x00,0x00,0xaf,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xe0,0x01,0x00,0x00, -0xde,0x02,0x00,0x00,0xdf,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xe1,0x01,0x00,0x00,0xdd,0x01,0x00,0x00, -0xe0,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xe3,0x01,0x00,0x00,0x68,0x00,0x00,0x00,0xb2,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xe4,0x01,0x00,0x00, -0xe1,0x01,0x00,0x00,0xe3,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xe6,0x01,0x00,0x00,0xe4,0x01,0x00,0x00, -0xec,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xe8,0x01,0x00,0x00,0xe6,0x01,0x00,0x00,0xe7,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xea,0x01,0x00,0x00, -0xe8,0x01,0x00,0x00,0xd9,0x02,0x00,0x00,0x41,0x00,0x05,0x00, -0xff,0x00,0x00,0x00,0xeb,0x01,0x00,0x00,0x4b,0x01,0x00,0x00, -0xea,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0xba,0x00,0x00,0x00, -0xec,0x01,0x00,0x00,0xeb,0x01,0x00,0x00,0x41,0x00,0x05,0x00, -0xc3,0x00,0x00,0x00,0xed,0x01,0x00,0x00,0xd7,0x01,0x00,0x00, -0xdb,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0xed,0x01,0x00,0x00, -0xec,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xef,0x01,0x00,0x00,0xec,0x02,0x00,0x00,0xc6,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xcd,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xcf,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xc8,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xc8,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xf1,0x01,0x00,0x00,0xde,0x02,0x00,0x00, -0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xc5,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xc7,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0xf3,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xf3,0x01,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xdf,0x02,0x00,0x00, -0x3e,0x00,0x00,0x00,0xc7,0x01,0x00,0x00,0x35,0x02,0x00,0x00, -0xf6,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00, -0xf9,0x01,0x00,0x00,0xdf,0x02,0x00,0x00,0xb5,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0xf5,0x01,0x00,0x00,0xf6,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xf9,0x01,0x00,0x00, -0xf4,0x01,0x00,0x00,0xf5,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xf4,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xfb,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xfb,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xe3,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, -0xf4,0x01,0x00,0x00,0x33,0x02,0x00,0x00,0xfe,0x01,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00,0x01,0x02,0x00,0x00, -0xe3,0x02,0x00,0x00,0x60,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0xfd,0x01,0x00,0x00,0xfe,0x01,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x01,0x02,0x00,0x00,0xfc,0x01,0x00,0x00, -0xfd,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xfc,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0x03,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x03,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xe5,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0xfc,0x01,0x00,0x00, -0x31,0x02,0x00,0x00,0x06,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb8,0x00,0x00,0x00,0x09,0x02,0x00,0x00,0xe5,0x02,0x00,0x00, -0xb2,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x05,0x02,0x00,0x00, -0x06,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x09,0x02,0x00,0x00,0x04,0x02,0x00,0x00,0x05,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x04,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x0b,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x0b,0x02,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xe7,0x02,0x00,0x00, -0x3e,0x00,0x00,0x00,0x04,0x02,0x00,0x00,0x2f,0x02,0x00,0x00, -0x0c,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00, -0x11,0x02,0x00,0x00,0xe7,0x02,0x00,0x00,0x62,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0x0d,0x02,0x00,0x00,0x0c,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x11,0x02,0x00,0x00, -0x0c,0x02,0x00,0x00,0x0d,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x0c,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x13,0x02,0x00,0x00,0xdf,0x02,0x00,0x00,0xb2,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x15,0x02,0x00,0x00, -0x13,0x02,0x00,0x00,0xe5,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x17,0x02,0x00,0x00,0x15,0x02,0x00,0x00, -0x16,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x19,0x02,0x00,0x00,0xe3,0x02,0x00,0x00,0x62,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x1a,0x02,0x00,0x00, -0x17,0x02,0x00,0x00,0x19,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x1c,0x02,0x00,0x00,0x1a,0x02,0x00,0x00, -0xe7,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x20,0x02,0x00,0x00,0x19,0x02,0x00,0x00,0xe7,0x02,0x00,0x00, -0x41,0x00,0x05,0x00,0xc3,0x00,0x00,0x00,0x21,0x02,0x00,0x00, -0xaa,0x01,0x00,0x00,0x20,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, -0xba,0x00,0x00,0x00,0x22,0x02,0x00,0x00,0x21,0x02,0x00,0x00, -0x41,0x00,0x05,0x00,0xc3,0x00,0x00,0x00,0x27,0x02,0x00,0x00, -0xd7,0x01,0x00,0x00,0x15,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, -0xba,0x00,0x00,0x00,0x28,0x02,0x00,0x00,0x27,0x02,0x00,0x00, -0x41,0x00,0x05,0x00,0xc3,0x00,0x00,0x00,0x2a,0x02,0x00,0x00, -0xc0,0x00,0x00,0x00,0x1c,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, -0xba,0x00,0x00,0x00,0x2b,0x02,0x00,0x00,0x2a,0x02,0x00,0x00, -0x0c,0x00,0x08,0x00,0xba,0x00,0x00,0x00,0x2c,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x22,0x02,0x00,0x00, -0x28,0x02,0x00,0x00,0x2b,0x02,0x00,0x00,0x3e,0x00,0x03,0x00, -0x2a,0x02,0x00,0x00,0x2c,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x2f,0x02,0x00,0x00,0xe7,0x02,0x00,0x00, -0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x0b,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x0d,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x06,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x06,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x31,0x02,0x00,0x00, -0xe5,0x02,0x00,0x00,0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x03,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x05,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0xfe,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xfe,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x33,0x02,0x00,0x00,0xe3,0x02,0x00,0x00,0xc6,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xfb,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xfd,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xf6,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xf6,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x35,0x02,0x00,0x00,0xdf,0x02,0x00,0x00, -0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xf3,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xf5,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0x93,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x93,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x37,0x02,0x00,0x00, -0xd9,0x02,0x00,0x00,0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x90,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x92,0x01,0x00,0x00, -0xe0,0x00,0x04,0x00,0x0c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x88,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xcd,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xcd,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x39,0x02,0x00,0x00,0xbf,0x02,0x00,0x00, -0x6c,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xca,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xcc,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x3e,0x02,0x00,0x00,0x55,0x00,0x00,0x00, -0x53,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x3f,0x02,0x00,0x00,0x8c,0x00,0x00,0x00,0x3e,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x44,0x02,0x00,0x00, -0x59,0x00,0x00,0x00,0xaf,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x45,0x02,0x00,0x00,0x9e,0x00,0x00,0x00, -0x44,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0x4a,0x02,0x00,0x00,0x12,0x00,0x00,0x00,0x49,0x02,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x4b,0x02,0x00,0x00, -0x4a,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x4c,0x02,0x00,0x00,0x0f,0x00,0x00,0x00,0x4b,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x50,0x02,0x00,0x00, -0x47,0x00,0x00,0x00,0x4b,0x02,0x00,0x00,0x41,0x00,0x05,0x00, -0x0d,0x00,0x00,0x00,0x52,0x02,0x00,0x00,0x51,0x02,0x00,0x00, -0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x53,0x02,0x00,0x00,0x52,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x54,0x02,0x00,0x00,0x50,0x02,0x00,0x00, -0x53,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x55,0x02,0x00,0x00,0x4c,0x02,0x00,0x00,0x54,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0x57,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x57,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xc0,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0xcc,0x00,0x00,0x00, -0xbd,0x02,0x00,0x00,0x5a,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb8,0x00,0x00,0x00,0x5d,0x02,0x00,0x00,0xc0,0x02,0x00,0x00, -0xb5,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x59,0x02,0x00,0x00, -0x5a,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x5d,0x02,0x00,0x00,0x58,0x02,0x00,0x00,0x59,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x58,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x5f,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x5f,0x02,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xc1,0x02,0x00,0x00, -0x3e,0x00,0x00,0x00,0x58,0x02,0x00,0x00,0xbb,0x02,0x00,0x00, -0x62,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00, -0x65,0x02,0x00,0x00,0xc1,0x02,0x00,0x00,0x60,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0x61,0x02,0x00,0x00,0x62,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x65,0x02,0x00,0x00, -0x60,0x02,0x00,0x00,0x61,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x60,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x69,0x02,0x00,0x00,0xc1,0x02,0x00,0x00,0x61,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x6a,0x02,0x00,0x00, -0x3f,0x02,0x00,0x00,0x69,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x6c,0x02,0x00,0x00,0x64,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x6d,0x02,0x00,0x00,0x6a,0x02,0x00,0x00,0x6c,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x71,0x02,0x00,0x00, -0xc0,0x02,0x00,0x00,0xdf,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x72,0x02,0x00,0x00,0x45,0x02,0x00,0x00, -0x71,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x74,0x02,0x00,0x00,0x68,0x00,0x00,0x00,0xb2,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x75,0x02,0x00,0x00, -0x72,0x02,0x00,0x00,0x74,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x77,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x77,0x02,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xc3,0x02,0x00,0x00, -0x3e,0x00,0x00,0x00,0x60,0x02,0x00,0x00,0xb9,0x02,0x00,0x00, -0x7a,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00, -0x7d,0x02,0x00,0x00,0xc3,0x02,0x00,0x00,0xb2,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0x79,0x02,0x00,0x00,0x7a,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x7d,0x02,0x00,0x00, -0x78,0x02,0x00,0x00,0x79,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x78,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x7f,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x7f,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xc5,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, -0x78,0x02,0x00,0x00,0xb7,0x02,0x00,0x00,0x82,0x02,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00,0x85,0x02,0x00,0x00, -0xc5,0x02,0x00,0x00,0x62,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0x81,0x02,0x00,0x00,0x82,0x02,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x85,0x02,0x00,0x00,0x80,0x02,0x00,0x00, -0x81,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x80,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x88,0x02,0x00,0x00, -0x6d,0x02,0x00,0x00,0xc5,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb8,0x00,0x00,0x00,0x8b,0x02,0x00,0x00,0x88,0x02,0x00,0x00, -0x36,0x00,0x00,0x00,0xf7,0x00,0x03,0x00,0x8d,0x02,0x00,0x00, -0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x8b,0x02,0x00,0x00, -0x8c,0x02,0x00,0x00,0x8d,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x8c,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x90,0x02,0x00,0x00,0x75,0x02,0x00,0x00,0xc3,0x02,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x91,0x02,0x00,0x00, -0x12,0x00,0x00,0x00,0xc6,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x92,0x02,0x00,0x00,0x91,0x02,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00,0x93,0x02,0x00,0x00, -0x90,0x02,0x00,0x00,0x92,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x8d,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x8d,0x02,0x00,0x00, -0xf5,0x00,0x07,0x00,0xb8,0x00,0x00,0x00,0x94,0x02,0x00,0x00, -0x8b,0x02,0x00,0x00,0x80,0x02,0x00,0x00,0x93,0x02,0x00,0x00, -0x8c,0x02,0x00,0x00,0xf7,0x00,0x03,0x00,0x96,0x02,0x00,0x00, -0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x94,0x02,0x00,0x00, -0x95,0x02,0x00,0x00,0x96,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x95,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x9e,0x02,0x00,0x00,0x75,0x02,0x00,0x00,0xc3,0x02,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0xa0,0x02,0x00,0x00, -0x12,0x00,0x00,0x00,0x9f,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xa1,0x02,0x00,0x00,0xa0,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa2,0x02,0x00,0x00, -0x9e,0x02,0x00,0x00,0xa1,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xa3,0x02,0x00,0x00,0x55,0x02,0x00,0x00, -0xa2,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xa5,0x02,0x00,0x00,0xa3,0x02,0x00,0x00,0x6d,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa7,0x02,0x00,0x00, -0xa5,0x02,0x00,0x00,0xc5,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xa9,0x02,0x00,0x00,0xc0,0x02,0x00,0x00, -0xb2,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xab,0x02,0x00,0x00,0xa9,0x02,0x00,0x00,0xc3,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xad,0x02,0x00,0x00, -0xab,0x02,0x00,0x00,0xac,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xaf,0x02,0x00,0x00,0xc1,0x02,0x00,0x00, -0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xb0,0x02,0x00,0x00,0xad,0x02,0x00,0x00,0xaf,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xb2,0x02,0x00,0x00, -0xb0,0x02,0x00,0x00,0xc5,0x02,0x00,0x00,0x41,0x00,0x05,0x00, -0xc3,0x00,0x00,0x00,0xb3,0x02,0x00,0x00,0xc0,0x00,0x00,0x00, -0xb2,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0xba,0x00,0x00,0x00, -0xb4,0x02,0x00,0x00,0xb3,0x02,0x00,0x00,0x41,0x00,0x06,0x00, -0x5b,0x01,0x00,0x00,0xb5,0x02,0x00,0x00,0x9a,0x02,0x00,0x00, -0x34,0x00,0x00,0x00,0xa7,0x02,0x00,0x00,0x3e,0x00,0x03,0x00, -0xb5,0x02,0x00,0x00,0xb4,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x96,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x96,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0x82,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x82,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xb7,0x02,0x00,0x00,0xc5,0x02,0x00,0x00,0xc6,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x7f,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x81,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x7a,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x7a,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xb9,0x02,0x00,0x00,0xc3,0x02,0x00,0x00, -0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x77,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x79,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x62,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x62,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xbb,0x02,0x00,0x00, -0xc1,0x02,0x00,0x00,0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x5f,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x61,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0x5a,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x5a,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xbd,0x02,0x00,0x00,0xc0,0x02,0x00,0x00,0xc6,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x57,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x59,0x02,0x00,0x00,0xfd,0x00,0x01,0x00,0x38,0x00,0x01,0x00, - -}; -const uint64_t matmul_f16_f32_aligned_l_fp32_len = 10464; - -unsigned char matmul_f16_f32_aligned_m_data[] = { -0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, -0x65,0x03,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, +0x09,0x03,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, 0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x09,0x00,0x00,0x00, 0x11,0x00,0x02,0x00,0x51,0x11,0x00,0x00,0x0b,0x00,0x06,0x00, 0x01,0x00,0x00,0x00,0x47,0x4c,0x53,0x4c,0x2e,0x73,0x74,0x64, @@ -18201,9 +10193,9 @@ unsigned char matmul_f16_f32_aligned_m_data[] = { 0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x0f,0x00,0x0f,0x00, 0x05,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x6d,0x61,0x69,0x6e, 0x00,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x3d,0x00,0x00,0x00,0x4c,0x00,0x00,0x00,0xeb,0x00,0x00,0x00, -0xfa,0x00,0x00,0x00,0x80,0x01,0x00,0x00,0x8f,0x01,0x00,0x00, -0xc6,0x02,0x00,0x00,0x0f,0x03,0x00,0x00,0x10,0x00,0x06,0x00, +0x3d,0x00,0x00,0x00,0x4c,0x00,0x00,0x00,0xfe,0x00,0x00,0x00, +0x05,0x01,0x00,0x00,0x63,0x01,0x00,0x00,0x69,0x01,0x00,0x00, +0x69,0x02,0x00,0x00,0xb2,0x02,0x00,0x00,0x10,0x00,0x06,0x00, 0x04,0x00,0x00,0x00,0x11,0x00,0x00,0x00,0x01,0x00,0x00,0x00, 0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00, 0x0b,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x1c,0x00,0x00,0x00, @@ -18242,14014 +10234,7104 @@ unsigned char matmul_f16_f32_aligned_m_data[] = { 0x06,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x62,0x00,0x00,0x00, 0x01,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0x47,0x00,0x04,0x00, 0x6c,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x03,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x9d,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xaf,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0xa7,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xb9,0x00,0x00,0x00, 0x01,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0xb2,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x08,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0xf7,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x10,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0xf8,0x00,0x00,0x00, +0xbc,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x08,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x02,0x01,0x00,0x00,0x06,0x00,0x00,0x00, +0x10,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0x03,0x01,0x00,0x00, 0x00,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0x48,0x00,0x04,0x00, -0xf8,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0xf8,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x03,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x03,0x01,0x00,0x00,0x00,0x00,0x00,0x00, 0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0xf8,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x07,0x00,0x00,0x00, -0x08,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0xf8,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xfa,0x00,0x00,0x00, +0x03,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x07,0x00,0x00,0x00, +0x08,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x03,0x01,0x00,0x00, +0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x05,0x01,0x00,0x00, 0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0xfa,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x61,0x01,0x00,0x00,0x01,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x62,0x01,0x00,0x00, +0x05,0x01,0x00,0x00,0x21,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x3b,0x01,0x00,0x00,0x01,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x3c,0x01,0x00,0x00, 0x0b,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x8c,0x01,0x00,0x00,0x06,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x48,0x00,0x04,0x00,0x8d,0x01,0x00,0x00,0x00,0x00,0x00,0x00, -0x05,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0x8d,0x01,0x00,0x00, +0x66,0x01,0x00,0x00,0x06,0x00,0x00,0x00,0x10,0x00,0x00,0x00, +0x48,0x00,0x04,0x00,0x67,0x01,0x00,0x00,0x00,0x00,0x00,0x00, +0x05,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0x67,0x01,0x00,0x00, 0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x8d,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x8d,0x01,0x00,0x00, -0x00,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0x10,0x00,0x00,0x00, -0x47,0x00,0x03,0x00,0x8d,0x01,0x00,0x00,0x02,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x8f,0x01,0x00,0x00,0x22,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x8f,0x01,0x00,0x00, -0x21,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0xc6,0x02,0x00,0x00,0x0b,0x00,0x00,0x00,0x18,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x0c,0x03,0x00,0x00,0x06,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0x0d,0x03,0x00,0x00, -0x00,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x0d,0x03,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x0d,0x03,0x00,0x00, -0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x0f,0x03,0x00,0x00, -0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x0f,0x03,0x00,0x00,0x21,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x13,0x00,0x02,0x00,0x02,0x00,0x00,0x00,0x21,0x00,0x03,0x00, -0x03,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x15,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x17,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x03,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x0a,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x0a,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x0d,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x1e,0x00,0x10,0x00, -0x10,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x11,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x10,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x11,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x15,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x14,0x00,0x00,0x00, -0x08,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x15,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x0a,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x27,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, -0x2d,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x37,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x0a,0x00,0x00,0x00,0x3d,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x3e,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00, -0x4c,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x53,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x54,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x37,0x00,0x00,0x00, -0x53,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x58,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x37,0x00,0x00,0x00, -0x53,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x60,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x61,0x00,0x00,0x00,0x86,0x00,0x00,0x00, -0x53,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x32,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x63,0x00,0x00,0x00, -0x86,0x00,0x00,0x00,0x61,0x00,0x00,0x00,0x62,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x67,0x00,0x00,0x00, -0x86,0x00,0x00,0x00,0x61,0x00,0x00,0x00,0x62,0x00,0x00,0x00, -0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x10,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x6d,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x6e,0x00,0x00,0x00,0x86,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x73,0x00,0x00,0x00,0x86,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0x77,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x7c,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, -0x87,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0x8d,0x00,0x00,0x00,0x03,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x98,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x9d,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0x9f,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xae,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00, -0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xaf,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xb0,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x53,0x00,0x00,0x00, -0xaf,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xb1,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x4f,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xb2,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xb3,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0xb1,0x00,0x00,0x00,0xb2,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xb4,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0xb3,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xb5,0x00,0x00,0x00,0x86,0x00,0x00,0x00, -0xb0,0x00,0x00,0x00,0xb4,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xb6,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0xae,0x00,0x00,0x00,0xb5,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xb7,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0xb6,0x00,0x00,0x00,0xb2,0x00,0x00,0x00,0x14,0x00,0x02,0x00, -0xb8,0x00,0x00,0x00,0x16,0x00,0x03,0x00,0xba,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xbb,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xbc,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0xbb,0x00,0x00,0x00, -0xb5,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xbd,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0xbc,0x00,0x00,0x00, -0xb2,0x00,0x00,0x00,0x1c,0x00,0x04,0x00,0xbe,0x00,0x00,0x00, -0xba,0x00,0x00,0x00,0xbd,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0xbf,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0xbe,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0xba,0x00,0x00,0x00,0xc2,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0xc3,0x00,0x00,0x00, -0x07,0x00,0x00,0x00,0xba,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0xc6,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x16,0x00,0x03,0x00,0xe6,0x00,0x00,0x00,0x10,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xe7,0x00,0x00,0x00, -0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xe8,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0xe7,0x00,0x00,0x00, -0x1c,0x00,0x04,0x00,0xe9,0x00,0x00,0x00,0xe6,0x00,0x00,0x00, -0xe8,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0xea,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0xe9,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0xea,0x00,0x00,0x00,0xeb,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xef,0x00,0x00,0x00, -0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x17,0x00,0x04,0x00,0xf5,0x00,0x00,0x00,0xe6,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x18,0x00,0x04,0x00,0xf6,0x00,0x00,0x00, -0xf5,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, -0xf7,0x00,0x00,0x00,0xf6,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, -0xf8,0x00,0x00,0x00,0xf7,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0xf9,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0xf8,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0xf9,0x00,0x00,0x00,0xfa,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0xfc,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0xe6,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0xff,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0xe6,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x04,0x01,0x00,0x00, -0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x11,0x01,0x00,0x00, -0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x1e,0x01,0x00,0x00, -0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x23,0x01,0x00,0x00, -0x03,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x2c,0x01,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x31,0x01,0x00,0x00,0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x3a,0x01,0x00,0x00,0x80,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x3f,0x01,0x00,0x00,0x05,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x48,0x01,0x00,0x00, -0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x4d,0x01,0x00,0x00, -0x06,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x56,0x01,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x5b,0x01,0x00,0x00,0x07,0x00,0x00,0x00,0x32,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x61,0x01,0x00,0x00,0x01,0x00,0x00,0x00, -0x33,0x00,0x06,0x00,0x09,0x00,0x00,0x00,0x62,0x01,0x00,0x00, -0x61,0x01,0x00,0x00,0x39,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x63,0x01,0x00,0x00, -0x51,0x00,0x00,0x00,0x62,0x01,0x00,0x00,0x00,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x64,0x01,0x00,0x00, -0x84,0x00,0x00,0x00,0x63,0x01,0x00,0x00,0x6d,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x65,0x01,0x00,0x00, -0x86,0x00,0x00,0x00,0x64,0x01,0x00,0x00,0x6c,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x7c,0x01,0x00,0x00, -0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x7d,0x01,0x00,0x00, -0x84,0x00,0x00,0x00,0x9d,0x00,0x00,0x00,0x7c,0x01,0x00,0x00, -0x1c,0x00,0x04,0x00,0x7e,0x01,0x00,0x00,0xe6,0x00,0x00,0x00, -0x7d,0x01,0x00,0x00,0x20,0x00,0x04,0x00,0x7f,0x01,0x00,0x00, -0x04,0x00,0x00,0x00,0x7e,0x01,0x00,0x00,0x3b,0x00,0x04,0x00, -0x7f,0x01,0x00,0x00,0x80,0x01,0x00,0x00,0x04,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x84,0x01,0x00,0x00, -0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x17,0x00,0x04,0x00,0x8a,0x01,0x00,0x00,0xba,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x18,0x00,0x04,0x00,0x8b,0x01,0x00,0x00, -0x8a,0x01,0x00,0x00,0x02,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, -0x8c,0x01,0x00,0x00,0x8b,0x01,0x00,0x00,0x1e,0x00,0x03,0x00, -0x8d,0x01,0x00,0x00,0x8c,0x01,0x00,0x00,0x20,0x00,0x04,0x00, -0x8e,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x8d,0x01,0x00,0x00, -0x3b,0x00,0x04,0x00,0x8e,0x01,0x00,0x00,0x8f,0x01,0x00,0x00, -0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x91,0x01,0x00,0x00, -0x0c,0x00,0x00,0x00,0xba,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x99,0x01,0x00,0x00,0x80,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xa7,0x01,0x00,0x00,0x80,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xb5,0x01,0x00,0x00,0x80,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xc3,0x01,0x00,0x00,0x80,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xd1,0x01,0x00,0x00,0x80,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xdf,0x01,0x00,0x00,0x80,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xed,0x01,0x00,0x00,0x80,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xfa,0x01,0x00,0x00,0x08,0x01,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xfb,0x01,0x00,0x00, -0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xfe,0x01,0x00,0x00, -0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x19,0x02,0x00,0x00, -0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00, -0x1c,0x00,0x04,0x00,0x1a,0x02,0x00,0x00,0xe6,0x00,0x00,0x00, -0x19,0x02,0x00,0x00,0x20,0x00,0x04,0x00,0x1b,0x02,0x00,0x00, -0x07,0x00,0x00,0x00,0x1a,0x02,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x2b,0x02,0x00,0x00,0x80,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x31,0x02,0x00,0x00,0x07,0x00,0x00,0x00,0xe6,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x47,0x02,0x00,0x00, -0x84,0x00,0x00,0x00,0xb5,0x00,0x00,0x00,0xb2,0x00,0x00,0x00, -0x1c,0x00,0x04,0x00,0x48,0x02,0x00,0x00,0xe6,0x00,0x00,0x00, -0x47,0x02,0x00,0x00,0x20,0x00,0x04,0x00,0x49,0x02,0x00,0x00, -0x07,0x00,0x00,0x00,0x48,0x02,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x52,0x02,0x00,0x00,0x86,0x00,0x00,0x00, -0xaf,0x00,0x00,0x00,0xb5,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x5a,0x02,0x00,0x00,0x80,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x89,0x02,0x00,0x00,0x84,0x00,0x00,0x00, -0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0xbe,0x02,0x00,0x00,0x0d,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00,0xc6,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0x1d,0x00,0x03,0x00,0x0c,0x03,0x00,0x00, -0xba,0x00,0x00,0x00,0x1e,0x00,0x03,0x00,0x0d,0x03,0x00,0x00, -0x0c,0x03,0x00,0x00,0x20,0x00,0x04,0x00,0x0e,0x03,0x00,0x00, -0x0c,0x00,0x00,0x00,0x0d,0x03,0x00,0x00,0x3b,0x00,0x04,0x00, -0x0e,0x03,0x00,0x00,0x0f,0x03,0x00,0x00,0x0c,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x14,0x03,0x00,0x00, -0x05,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x21,0x03,0x00,0x00,0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x36,0x00,0x05,0x00,0x02,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x03,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0x05,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0xbf,0x00,0x00,0x00,0xc0,0x00,0x00,0x00,0x07,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x1b,0x02,0x00,0x00,0x1c,0x02,0x00,0x00, -0x07,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x49,0x02,0x00,0x00, -0x4a,0x02,0x00,0x00,0x07,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x0d,0x00,0x00,0x00,0x0e,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x0f,0x00,0x00,0x00,0x0e,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x14,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x17,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, -0x17,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x1e,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x17,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x22,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x22,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x24,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x27,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x29,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x2a,0x00,0x00,0x00,0x1e,0x00,0x00,0x00, -0x29,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x2d,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x2f,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x30,0x00,0x00,0x00,0x24,0x00,0x00,0x00,0x2f,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x32,0x00,0x00,0x00, -0x30,0x00,0x00,0x00,0x2a,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x35,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x34,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x36,0x00,0x00,0x00,0x35,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x38,0x00,0x00,0x00,0x36,0x00,0x00,0x00, -0x37,0x00,0x00,0x00,0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x3a,0x00,0x00,0x00,0x38,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3b,0x00,0x00,0x00, -0x3a,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x0d,0x00,0x00,0x00,0x3f,0x00,0x00,0x00,0x3d,0x00,0x00,0x00, -0x3e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0x3f,0x00,0x00,0x00,0x89,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x42,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x3b,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x47,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x3b,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00,0x49,0x00,0x00,0x00, -0x3d,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x4a,0x00,0x00,0x00,0x49,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00,0x4d,0x00,0x00,0x00, -0x4c,0x00,0x00,0x00,0x3e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x4d,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x50,0x00,0x00,0x00, -0x4e,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x89,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x55,0x00,0x00,0x00,0x50,0x00,0x00,0x00, -0x54,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x59,0x00,0x00,0x00,0x50,0x00,0x00,0x00,0x58,0x00,0x00,0x00, -0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x5d,0x00,0x00,0x00, -0x4e,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x89,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x64,0x00,0x00,0x00,0x5d,0x00,0x00,0x00, -0x63,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x68,0x00,0x00,0x00,0x5d,0x00,0x00,0x00,0x67,0x00,0x00,0x00, -0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x6f,0x00,0x00,0x00, -0x4e,0x00,0x00,0x00,0x6e,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x74,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, -0x73,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0x78,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x77,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x79,0x00,0x00,0x00, -0x78,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x7a,0x00,0x00,0x00,0x47,0x00,0x00,0x00,0x79,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x7d,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x7c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x7e,0x00,0x00,0x00,0x7d,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x80,0x00,0x00,0x00, -0x47,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x83,0x00,0x00,0x00,0x80,0x00,0x00,0x00, -0x79,0x00,0x00,0x00,0x0c,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x26,0x00,0x00,0x00, -0x7e,0x00,0x00,0x00,0x83,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x88,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x87,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x89,0x00,0x00,0x00,0x88,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x8a,0x00,0x00,0x00,0x32,0x00,0x00,0x00, -0x89,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x8c,0x00,0x00,0x00,0x42,0x00,0x00,0x00,0x37,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x8e,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x8d,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x8f,0x00,0x00,0x00,0x8e,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x90,0x00,0x00,0x00, -0x8c,0x00,0x00,0x00,0x8f,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x91,0x00,0x00,0x00,0x8a,0x00,0x00,0x00, -0x90,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x93,0x00,0x00,0x00,0x91,0x00,0x00,0x00,0x7a,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x94,0x00,0x00,0x00, -0x93,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x99,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x98,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x9a,0x00,0x00,0x00,0x99,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x9b,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, -0x9a,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x9e,0x00,0x00,0x00,0x4a,0x00,0x00,0x00,0x9d,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0xa0,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x9f,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xa1,0x00,0x00,0x00,0xa0,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa2,0x00,0x00,0x00, -0x9e,0x00,0x00,0x00,0xa1,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xa3,0x00,0x00,0x00,0x9b,0x00,0x00,0x00, -0xa2,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xa5,0x00,0x00,0x00,0xa3,0x00,0x00,0x00,0x7a,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa6,0x00,0x00,0x00, -0xa5,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xa8,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xa8,0x00,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x33,0x03,0x00,0x00, -0x3e,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0xc7,0x00,0x00,0x00, -0xa9,0x00,0x00,0x00,0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00, -0xb9,0x00,0x00,0x00,0x33,0x03,0x00,0x00,0xb7,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0xaa,0x00,0x00,0x00,0xa9,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xb9,0x00,0x00,0x00, -0xa9,0x00,0x00,0x00,0xaa,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xa9,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0xc3,0x00,0x00,0x00, -0xc4,0x00,0x00,0x00,0xc0,0x00,0x00,0x00,0x33,0x03,0x00,0x00, -0x3e,0x00,0x03,0x00,0xc4,0x00,0x00,0x00,0xc2,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xc7,0x00,0x00,0x00, -0x33,0x03,0x00,0x00,0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xa8,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xaa,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xca,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xca,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0x4c,0x03,0x00,0x00,0xa6,0x00,0x00,0x00,0xaa,0x00,0x00,0x00, -0x00,0x02,0x00,0x00,0xcd,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0x48,0x03,0x00,0x00,0x94,0x00,0x00,0x00, -0xaa,0x00,0x00,0x00,0xfd,0x01,0x00,0x00,0xcd,0x00,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x34,0x03,0x00,0x00, -0x7a,0x00,0x00,0x00,0xaa,0x00,0x00,0x00,0xae,0x02,0x00,0x00, -0xcd,0x00,0x00,0x00,0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00, -0xd1,0x00,0x00,0x00,0x34,0x03,0x00,0x00,0x84,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0xcc,0x00,0x00,0x00,0xcd,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xd1,0x00,0x00,0x00, -0xcb,0x00,0x00,0x00,0xcc,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xcb,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xd3,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xd3,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0x44,0x03,0x00,0x00,0x3e,0x00,0x00,0x00, -0xcb,0x00,0x00,0x00,0x67,0x01,0x00,0x00,0xd4,0x00,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00,0xd9,0x00,0x00,0x00, -0x44,0x03,0x00,0x00,0x37,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0xd5,0x00,0x00,0x00,0xd4,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xd9,0x00,0x00,0x00,0xd4,0x00,0x00,0x00, -0xd5,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xd4,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xde,0x00,0x00,0x00, -0x74,0x00,0x00,0x00,0x44,0x03,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xe1,0x00,0x00,0x00,0xde,0x00,0x00,0x00, -0x8f,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xe2,0x00,0x00,0x00,0xe1,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xe3,0x00,0x00,0x00, -0x48,0x03,0x00,0x00,0xe2,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xe5,0x00,0x00,0x00,0xe3,0x00,0x00,0x00, -0x6f,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xf0,0x00,0x00,0x00,0xde,0x00,0x00,0x00,0xef,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xf2,0x00,0x00,0x00, -0x6f,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xf3,0x00,0x00,0x00,0xf0,0x00,0x00,0x00, -0xf2,0x00,0x00,0x00,0x41,0x00,0x08,0x00,0xfc,0x00,0x00,0x00, -0xfd,0x00,0x00,0x00,0xfa,0x00,0x00,0x00,0x34,0x00,0x00,0x00, -0xe5,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0x3e,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0xe6,0x00,0x00,0x00,0xfe,0x00,0x00,0x00, -0xfd,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0xff,0x00,0x00,0x00, -0x00,0x01,0x00,0x00,0xeb,0x00,0x00,0x00,0xf3,0x00,0x00,0x00, -0x3e,0x00,0x03,0x00,0x00,0x01,0x00,0x00,0xfe,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x05,0x01,0x00,0x00, -0xde,0x00,0x00,0x00,0x04,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x08,0x01,0x00,0x00,0x05,0x01,0x00,0x00, -0xf2,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x09,0x01,0x00,0x00,0x08,0x01,0x00,0x00,0x39,0x00,0x00,0x00, -0x41,0x00,0x08,0x00,0xfc,0x00,0x00,0x00,0x0b,0x01,0x00,0x00, -0xfa,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0xe5,0x00,0x00,0x00, -0x34,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0xe6,0x00,0x00,0x00,0x0c,0x01,0x00,0x00,0x0b,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0xff,0x00,0x00,0x00,0x0d,0x01,0x00,0x00, -0xeb,0x00,0x00,0x00,0x09,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0x0d,0x01,0x00,0x00,0x0c,0x01,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x12,0x01,0x00,0x00,0xde,0x00,0x00,0x00, -0x11,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x15,0x01,0x00,0x00,0x12,0x01,0x00,0x00,0xf2,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x16,0x01,0x00,0x00, -0x15,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x41,0x00,0x08,0x00, -0xfc,0x00,0x00,0x00,0x18,0x01,0x00,0x00,0xfa,0x00,0x00,0x00, -0x34,0x00,0x00,0x00,0xe5,0x00,0x00,0x00,0x34,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0xe6,0x00,0x00,0x00, -0x19,0x01,0x00,0x00,0x18,0x01,0x00,0x00,0x41,0x00,0x05,0x00, -0xff,0x00,0x00,0x00,0x1a,0x01,0x00,0x00,0xeb,0x00,0x00,0x00, -0x16,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x1a,0x01,0x00,0x00, -0x19,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x1f,0x01,0x00,0x00,0xde,0x00,0x00,0x00,0x1e,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x22,0x01,0x00,0x00, -0x1f,0x01,0x00,0x00,0xf2,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x24,0x01,0x00,0x00,0x22,0x01,0x00,0x00, -0x23,0x01,0x00,0x00,0x41,0x00,0x08,0x00,0xfc,0x00,0x00,0x00, -0x26,0x01,0x00,0x00,0xfa,0x00,0x00,0x00,0x34,0x00,0x00,0x00, -0xe5,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0x23,0x01,0x00,0x00, -0x3d,0x00,0x04,0x00,0xe6,0x00,0x00,0x00,0x27,0x01,0x00,0x00, -0x26,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0xff,0x00,0x00,0x00, -0x28,0x01,0x00,0x00,0xeb,0x00,0x00,0x00,0x24,0x01,0x00,0x00, -0x3e,0x00,0x03,0x00,0x28,0x01,0x00,0x00,0x27,0x01,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x2d,0x01,0x00,0x00, -0xde,0x00,0x00,0x00,0x2c,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x30,0x01,0x00,0x00,0x2d,0x01,0x00,0x00, -0xf2,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x32,0x01,0x00,0x00,0x30,0x01,0x00,0x00,0x31,0x01,0x00,0x00, -0x41,0x00,0x08,0x00,0xfc,0x00,0x00,0x00,0x34,0x01,0x00,0x00, -0xfa,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0xe5,0x00,0x00,0x00, -0xc6,0x00,0x00,0x00,0x3e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0xe6,0x00,0x00,0x00,0x35,0x01,0x00,0x00,0x34,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0xff,0x00,0x00,0x00,0x36,0x01,0x00,0x00, -0xeb,0x00,0x00,0x00,0x32,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0x36,0x01,0x00,0x00,0x35,0x01,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x3b,0x01,0x00,0x00,0xde,0x00,0x00,0x00, -0x3a,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x3e,0x01,0x00,0x00,0x3b,0x01,0x00,0x00,0xf2,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x40,0x01,0x00,0x00, -0x3e,0x01,0x00,0x00,0x3f,0x01,0x00,0x00,0x41,0x00,0x08,0x00, -0xfc,0x00,0x00,0x00,0x42,0x01,0x00,0x00,0xfa,0x00,0x00,0x00, -0x34,0x00,0x00,0x00,0xe5,0x00,0x00,0x00,0xc6,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0xe6,0x00,0x00,0x00, -0x43,0x01,0x00,0x00,0x42,0x01,0x00,0x00,0x41,0x00,0x05,0x00, -0xff,0x00,0x00,0x00,0x44,0x01,0x00,0x00,0xeb,0x00,0x00,0x00, -0x40,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x44,0x01,0x00,0x00, -0x43,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x49,0x01,0x00,0x00,0xde,0x00,0x00,0x00,0x48,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x4c,0x01,0x00,0x00, -0x49,0x01,0x00,0x00,0xf2,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x4e,0x01,0x00,0x00,0x4c,0x01,0x00,0x00, -0x4d,0x01,0x00,0x00,0x41,0x00,0x08,0x00,0xfc,0x00,0x00,0x00, -0x50,0x01,0x00,0x00,0xfa,0x00,0x00,0x00,0x34,0x00,0x00,0x00, -0xe5,0x00,0x00,0x00,0xc6,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0xe6,0x00,0x00,0x00,0x51,0x01,0x00,0x00, -0x50,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0xff,0x00,0x00,0x00, -0x52,0x01,0x00,0x00,0xeb,0x00,0x00,0x00,0x4e,0x01,0x00,0x00, -0x3e,0x00,0x03,0x00,0x52,0x01,0x00,0x00,0x51,0x01,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x57,0x01,0x00,0x00, -0xde,0x00,0x00,0x00,0x56,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x5a,0x01,0x00,0x00,0x57,0x01,0x00,0x00, -0xf2,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x5c,0x01,0x00,0x00,0x5a,0x01,0x00,0x00,0x5b,0x01,0x00,0x00, -0x41,0x00,0x08,0x00,0xfc,0x00,0x00,0x00,0x5e,0x01,0x00,0x00, -0xfa,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0xe5,0x00,0x00,0x00, -0xc6,0x00,0x00,0x00,0x23,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, -0xe6,0x00,0x00,0x00,0x5f,0x01,0x00,0x00,0x5e,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0xff,0x00,0x00,0x00,0x60,0x01,0x00,0x00, -0xeb,0x00,0x00,0x00,0x5c,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0x60,0x01,0x00,0x00,0x5f,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x67,0x01,0x00,0x00,0x44,0x03,0x00,0x00, -0x65,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xd3,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xd5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x69,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x69,0x01,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x45,0x03,0x00,0x00, -0x3e,0x00,0x00,0x00,0xd5,0x00,0x00,0x00,0xf9,0x01,0x00,0x00, -0x6a,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00, -0x6f,0x01,0x00,0x00,0x45,0x03,0x00,0x00,0x9d,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0x6b,0x01,0x00,0x00,0x6a,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x6f,0x01,0x00,0x00, -0x6a,0x01,0x00,0x00,0x6b,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x6a,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x74,0x01,0x00,0x00,0x74,0x00,0x00,0x00,0x45,0x03,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x77,0x01,0x00,0x00, -0x74,0x01,0x00,0x00,0xa1,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x78,0x01,0x00,0x00,0x77,0x01,0x00,0x00, -0x6d,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x79,0x01,0x00,0x00,0x4c,0x03,0x00,0x00,0x78,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x7b,0x01,0x00,0x00, -0x79,0x01,0x00,0x00,0x6f,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x85,0x01,0x00,0x00,0x74,0x01,0x00,0x00, -0x84,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x87,0x01,0x00,0x00,0x6f,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x88,0x01,0x00,0x00, -0x85,0x01,0x00,0x00,0x87,0x01,0x00,0x00,0x41,0x00,0x08,0x00, -0x91,0x01,0x00,0x00,0x92,0x01,0x00,0x00,0x8f,0x01,0x00,0x00, -0x34,0x00,0x00,0x00,0x7b,0x01,0x00,0x00,0x34,0x00,0x00,0x00, -0x3e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0xba,0x00,0x00,0x00, -0x93,0x01,0x00,0x00,0x92,0x01,0x00,0x00,0x73,0x00,0x04,0x00, -0xe6,0x00,0x00,0x00,0x94,0x01,0x00,0x00,0x93,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0xff,0x00,0x00,0x00,0x95,0x01,0x00,0x00, -0x80,0x01,0x00,0x00,0x88,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0x95,0x01,0x00,0x00,0x94,0x01,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x9a,0x01,0x00,0x00,0x74,0x01,0x00,0x00, -0x99,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x9d,0x01,0x00,0x00,0x9a,0x01,0x00,0x00,0x87,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9e,0x01,0x00,0x00, -0x9d,0x01,0x00,0x00,0x39,0x00,0x00,0x00,0x41,0x00,0x08,0x00, -0x91,0x01,0x00,0x00,0xa0,0x01,0x00,0x00,0x8f,0x01,0x00,0x00, -0x34,0x00,0x00,0x00,0x7b,0x01,0x00,0x00,0x34,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0xba,0x00,0x00,0x00, -0xa1,0x01,0x00,0x00,0xa0,0x01,0x00,0x00,0x73,0x00,0x04,0x00, -0xe6,0x00,0x00,0x00,0xa2,0x01,0x00,0x00,0xa1,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0xff,0x00,0x00,0x00,0xa3,0x01,0x00,0x00, -0x80,0x01,0x00,0x00,0x9e,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0xa3,0x01,0x00,0x00,0xa2,0x01,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xa8,0x01,0x00,0x00,0x74,0x01,0x00,0x00, -0xa7,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xab,0x01,0x00,0x00,0xa8,0x01,0x00,0x00,0x87,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xac,0x01,0x00,0x00, -0xab,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x41,0x00,0x08,0x00, -0x91,0x01,0x00,0x00,0xae,0x01,0x00,0x00,0x8f,0x01,0x00,0x00, -0x34,0x00,0x00,0x00,0x7b,0x01,0x00,0x00,0x34,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0xba,0x00,0x00,0x00, -0xaf,0x01,0x00,0x00,0xae,0x01,0x00,0x00,0x73,0x00,0x04,0x00, -0xe6,0x00,0x00,0x00,0xb0,0x01,0x00,0x00,0xaf,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0xff,0x00,0x00,0x00,0xb1,0x01,0x00,0x00, -0x80,0x01,0x00,0x00,0xac,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0xb1,0x01,0x00,0x00,0xb0,0x01,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xb6,0x01,0x00,0x00,0x74,0x01,0x00,0x00, -0xb5,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xb9,0x01,0x00,0x00,0xb6,0x01,0x00,0x00,0x87,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xba,0x01,0x00,0x00, -0xb9,0x01,0x00,0x00,0x23,0x01,0x00,0x00,0x41,0x00,0x08,0x00, -0x91,0x01,0x00,0x00,0xbc,0x01,0x00,0x00,0x8f,0x01,0x00,0x00, -0x34,0x00,0x00,0x00,0x7b,0x01,0x00,0x00,0x34,0x00,0x00,0x00, -0x23,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0xba,0x00,0x00,0x00, -0xbd,0x01,0x00,0x00,0xbc,0x01,0x00,0x00,0x73,0x00,0x04,0x00, -0xe6,0x00,0x00,0x00,0xbe,0x01,0x00,0x00,0xbd,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0xff,0x00,0x00,0x00,0xbf,0x01,0x00,0x00, -0x80,0x01,0x00,0x00,0xba,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0xbf,0x01,0x00,0x00,0xbe,0x01,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xc4,0x01,0x00,0x00,0x74,0x01,0x00,0x00, -0xc3,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xc7,0x01,0x00,0x00,0xc4,0x01,0x00,0x00,0x87,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xc8,0x01,0x00,0x00, -0xc7,0x01,0x00,0x00,0x31,0x01,0x00,0x00,0x41,0x00,0x08,0x00, -0x91,0x01,0x00,0x00,0xca,0x01,0x00,0x00,0x8f,0x01,0x00,0x00, -0x34,0x00,0x00,0x00,0x7b,0x01,0x00,0x00,0xc6,0x00,0x00,0x00, -0x3e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0xba,0x00,0x00,0x00, -0xcb,0x01,0x00,0x00,0xca,0x01,0x00,0x00,0x73,0x00,0x04,0x00, -0xe6,0x00,0x00,0x00,0xcc,0x01,0x00,0x00,0xcb,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0xff,0x00,0x00,0x00,0xcd,0x01,0x00,0x00, -0x80,0x01,0x00,0x00,0xc8,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0xcd,0x01,0x00,0x00,0xcc,0x01,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xd2,0x01,0x00,0x00,0x74,0x01,0x00,0x00, -0xd1,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xd5,0x01,0x00,0x00,0xd2,0x01,0x00,0x00,0x87,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xd6,0x01,0x00,0x00, -0xd5,0x01,0x00,0x00,0x3f,0x01,0x00,0x00,0x41,0x00,0x08,0x00, -0x91,0x01,0x00,0x00,0xd8,0x01,0x00,0x00,0x8f,0x01,0x00,0x00, -0x34,0x00,0x00,0x00,0x7b,0x01,0x00,0x00,0xc6,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0xba,0x00,0x00,0x00, -0xd9,0x01,0x00,0x00,0xd8,0x01,0x00,0x00,0x73,0x00,0x04,0x00, -0xe6,0x00,0x00,0x00,0xda,0x01,0x00,0x00,0xd9,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0xff,0x00,0x00,0x00,0xdb,0x01,0x00,0x00, -0x80,0x01,0x00,0x00,0xd6,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0xdb,0x01,0x00,0x00,0xda,0x01,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xe0,0x01,0x00,0x00,0x74,0x01,0x00,0x00, -0xdf,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xe3,0x01,0x00,0x00,0xe0,0x01,0x00,0x00,0x87,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xe4,0x01,0x00,0x00, -0xe3,0x01,0x00,0x00,0x4d,0x01,0x00,0x00,0x41,0x00,0x08,0x00, -0x91,0x01,0x00,0x00,0xe6,0x01,0x00,0x00,0x8f,0x01,0x00,0x00, -0x34,0x00,0x00,0x00,0x7b,0x01,0x00,0x00,0xc6,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0xba,0x00,0x00,0x00, -0xe7,0x01,0x00,0x00,0xe6,0x01,0x00,0x00,0x73,0x00,0x04,0x00, -0xe6,0x00,0x00,0x00,0xe8,0x01,0x00,0x00,0xe7,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0xff,0x00,0x00,0x00,0xe9,0x01,0x00,0x00, -0x80,0x01,0x00,0x00,0xe4,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0xe9,0x01,0x00,0x00,0xe8,0x01,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xee,0x01,0x00,0x00,0x74,0x01,0x00,0x00, -0xed,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xf1,0x01,0x00,0x00,0xee,0x01,0x00,0x00,0x87,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xf2,0x01,0x00,0x00, -0xf1,0x01,0x00,0x00,0x5b,0x01,0x00,0x00,0x41,0x00,0x08,0x00, -0x91,0x01,0x00,0x00,0xf4,0x01,0x00,0x00,0x8f,0x01,0x00,0x00, -0x34,0x00,0x00,0x00,0x7b,0x01,0x00,0x00,0xc6,0x00,0x00,0x00, -0x23,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0xba,0x00,0x00,0x00, -0xf5,0x01,0x00,0x00,0xf4,0x01,0x00,0x00,0x73,0x00,0x04,0x00, -0xe6,0x00,0x00,0x00,0xf6,0x01,0x00,0x00,0xf5,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0xff,0x00,0x00,0x00,0xf7,0x01,0x00,0x00, -0x80,0x01,0x00,0x00,0xf2,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0xf7,0x01,0x00,0x00,0xf6,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xf9,0x01,0x00,0x00,0x45,0x03,0x00,0x00, -0x65,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x69,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x6b,0x01,0x00,0x00,0xe0,0x00,0x04,0x00, -0x0c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0xfa,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xfd,0x01,0x00,0x00, -0x48,0x03,0x00,0x00,0xfb,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x00,0x02,0x00,0x00,0x4c,0x03,0x00,0x00, -0xfe,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x02,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x02,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0x4e,0x03,0x00,0x00,0x3e,0x00,0x00,0x00, -0x6b,0x01,0x00,0x00,0xac,0x02,0x00,0x00,0x05,0x02,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00,0x08,0x02,0x00,0x00, -0x4e,0x03,0x00,0x00,0x6c,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0x04,0x02,0x00,0x00,0x05,0x02,0x00,0x00,0x00,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x08,0x02,0x00,0x00,0x03,0x02,0x00,0x00, -0x04,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x03,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0x0a,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x0a,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0x52,0x03,0x00,0x00,0x3e,0x00,0x00,0x00,0x03,0x02,0x00,0x00, -0x36,0x02,0x00,0x00,0x0d,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb8,0x00,0x00,0x00,0x10,0x02,0x00,0x00,0x52,0x03,0x00,0x00, -0x60,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x0c,0x02,0x00,0x00, -0x0d,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x10,0x02,0x00,0x00,0x0b,0x02,0x00,0x00,0x0c,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x0b,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x12,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x12,0x02,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x64,0x03,0x00,0x00, -0x3e,0x00,0x00,0x00,0x0b,0x02,0x00,0x00,0x34,0x02,0x00,0x00, -0x13,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00, -0x18,0x02,0x00,0x00,0x64,0x03,0x00,0x00,0x62,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0x14,0x02,0x00,0x00,0x13,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x18,0x02,0x00,0x00, -0x13,0x02,0x00,0x00,0x14,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x13,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x1e,0x02,0x00,0x00,0x52,0x03,0x00,0x00,0x62,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x20,0x02,0x00,0x00, -0x1e,0x02,0x00,0x00,0x64,0x03,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x22,0x02,0x00,0x00,0x55,0x00,0x00,0x00, -0x53,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x24,0x02,0x00,0x00,0x52,0x03,0x00,0x00,0x61,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x25,0x02,0x00,0x00, -0x22,0x02,0x00,0x00,0x24,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x27,0x02,0x00,0x00,0x64,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x28,0x02,0x00,0x00,0x25,0x02,0x00,0x00,0x27,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x2a,0x02,0x00,0x00, -0x28,0x02,0x00,0x00,0x64,0x03,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x2c,0x02,0x00,0x00,0x2a,0x02,0x00,0x00, -0x2b,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x2e,0x02,0x00,0x00,0x2c,0x02,0x00,0x00,0x4e,0x03,0x00,0x00, -0x41,0x00,0x05,0x00,0xff,0x00,0x00,0x00,0x2f,0x02,0x00,0x00, -0xeb,0x00,0x00,0x00,0x2e,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, -0xe6,0x00,0x00,0x00,0x30,0x02,0x00,0x00,0x2f,0x02,0x00,0x00, -0x41,0x00,0x05,0x00,0x31,0x02,0x00,0x00,0x32,0x02,0x00,0x00, -0x1c,0x02,0x00,0x00,0x20,0x02,0x00,0x00,0x3e,0x00,0x03,0x00, -0x32,0x02,0x00,0x00,0x30,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x34,0x02,0x00,0x00,0x64,0x03,0x00,0x00, -0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x12,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x14,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x0d,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x0d,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x36,0x02,0x00,0x00, -0x52,0x03,0x00,0x00,0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x0a,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x0c,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0x38,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x38,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0x53,0x03,0x00,0x00,0x3e,0x00,0x00,0x00,0x0c,0x02,0x00,0x00, -0x64,0x02,0x00,0x00,0x3b,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb8,0x00,0x00,0x00,0x3e,0x02,0x00,0x00,0x53,0x03,0x00,0x00, -0xb5,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x3a,0x02,0x00,0x00, -0x3b,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x3e,0x02,0x00,0x00,0x39,0x02,0x00,0x00,0x3a,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x39,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x40,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x40,0x02,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x61,0x03,0x00,0x00, -0x3e,0x00,0x00,0x00,0x39,0x02,0x00,0x00,0x62,0x02,0x00,0x00, -0x41,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00, -0x46,0x02,0x00,0x00,0x61,0x03,0x00,0x00,0xb2,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0x42,0x02,0x00,0x00,0x41,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x46,0x02,0x00,0x00, -0x41,0x02,0x00,0x00,0x42,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x41,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x4c,0x02,0x00,0x00,0x53,0x03,0x00,0x00,0xb2,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x4e,0x02,0x00,0x00, -0x4c,0x02,0x00,0x00,0x61,0x03,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x50,0x02,0x00,0x00,0x59,0x00,0x00,0x00, -0xaf,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x53,0x02,0x00,0x00,0x53,0x03,0x00,0x00,0x52,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x54,0x02,0x00,0x00, -0x50,0x02,0x00,0x00,0x53,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x56,0x02,0x00,0x00,0x68,0x00,0x00,0x00, -0xb2,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x57,0x02,0x00,0x00,0x54,0x02,0x00,0x00,0x56,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x59,0x02,0x00,0x00, -0x57,0x02,0x00,0x00,0x61,0x03,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x5b,0x02,0x00,0x00,0x59,0x02,0x00,0x00, -0x5a,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x5d,0x02,0x00,0x00,0x5b,0x02,0x00,0x00,0x4e,0x03,0x00,0x00, -0x41,0x00,0x05,0x00,0xff,0x00,0x00,0x00,0x5e,0x02,0x00,0x00, -0x80,0x01,0x00,0x00,0x5d,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, -0xe6,0x00,0x00,0x00,0x5f,0x02,0x00,0x00,0x5e,0x02,0x00,0x00, -0x41,0x00,0x05,0x00,0x31,0x02,0x00,0x00,0x60,0x02,0x00,0x00, -0x4a,0x02,0x00,0x00,0x4e,0x02,0x00,0x00,0x3e,0x00,0x03,0x00, -0x60,0x02,0x00,0x00,0x5f,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x62,0x02,0x00,0x00,0x61,0x03,0x00,0x00, -0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x40,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x42,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x3b,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x3b,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x64,0x02,0x00,0x00, -0x53,0x03,0x00,0x00,0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x38,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x3a,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0x66,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x66,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0x54,0x03,0x00,0x00,0x3e,0x00,0x00,0x00,0x3a,0x02,0x00,0x00, -0xaa,0x02,0x00,0x00,0x69,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb8,0x00,0x00,0x00,0x6c,0x02,0x00,0x00,0x54,0x03,0x00,0x00, -0xb5,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x68,0x02,0x00,0x00, -0x69,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x6c,0x02,0x00,0x00,0x67,0x02,0x00,0x00,0x68,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x67,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x6e,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x6e,0x02,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x58,0x03,0x00,0x00, -0x3e,0x00,0x00,0x00,0x67,0x02,0x00,0x00,0xa8,0x02,0x00,0x00, -0x71,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00, -0x74,0x02,0x00,0x00,0x58,0x03,0x00,0x00,0x60,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0x70,0x02,0x00,0x00,0x71,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x74,0x02,0x00,0x00, -0x6f,0x02,0x00,0x00,0x70,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x6f,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x76,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x76,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0x5a,0x03,0x00,0x00,0x3e,0x00,0x00,0x00, -0x6f,0x02,0x00,0x00,0xa6,0x02,0x00,0x00,0x79,0x02,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00,0x7c,0x02,0x00,0x00, -0x5a,0x03,0x00,0x00,0xb2,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0x78,0x02,0x00,0x00,0x79,0x02,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x7c,0x02,0x00,0x00,0x77,0x02,0x00,0x00, -0x78,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x77,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0x7e,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x7e,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0x5c,0x03,0x00,0x00,0x3e,0x00,0x00,0x00,0x77,0x02,0x00,0x00, -0xa4,0x02,0x00,0x00,0x7f,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb8,0x00,0x00,0x00,0x84,0x02,0x00,0x00,0x5c,0x03,0x00,0x00, -0x62,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x80,0x02,0x00,0x00, -0x7f,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x84,0x02,0x00,0x00,0x7f,0x02,0x00,0x00,0x80,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x7f,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x86,0x02,0x00,0x00,0x54,0x03,0x00,0x00, -0xb2,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x88,0x02,0x00,0x00,0x86,0x02,0x00,0x00,0x5a,0x03,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8a,0x02,0x00,0x00, -0x88,0x02,0x00,0x00,0x89,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x8c,0x02,0x00,0x00,0x58,0x03,0x00,0x00, -0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x8d,0x02,0x00,0x00,0x8a,0x02,0x00,0x00,0x8c,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8f,0x02,0x00,0x00, -0x8d,0x02,0x00,0x00,0x5c,0x03,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x93,0x02,0x00,0x00,0x8c,0x02,0x00,0x00, -0x5c,0x03,0x00,0x00,0x41,0x00,0x05,0x00,0x31,0x02,0x00,0x00, -0x94,0x02,0x00,0x00,0x1c,0x02,0x00,0x00,0x93,0x02,0x00,0x00, -0x3d,0x00,0x04,0x00,0xe6,0x00,0x00,0x00,0x95,0x02,0x00,0x00, -0x94,0x02,0x00,0x00,0x73,0x00,0x04,0x00,0xba,0x00,0x00,0x00, -0x96,0x02,0x00,0x00,0x95,0x02,0x00,0x00,0x41,0x00,0x05,0x00, -0x31,0x02,0x00,0x00,0x9b,0x02,0x00,0x00,0x4a,0x02,0x00,0x00, -0x88,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0xe6,0x00,0x00,0x00, -0x9c,0x02,0x00,0x00,0x9b,0x02,0x00,0x00,0x73,0x00,0x04,0x00, -0xba,0x00,0x00,0x00,0x9d,0x02,0x00,0x00,0x9c,0x02,0x00,0x00, -0x41,0x00,0x05,0x00,0xc3,0x00,0x00,0x00,0x9f,0x02,0x00,0x00, -0xc0,0x00,0x00,0x00,0x8f,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, -0xba,0x00,0x00,0x00,0xa0,0x02,0x00,0x00,0x9f,0x02,0x00,0x00, -0x0c,0x00,0x08,0x00,0xba,0x00,0x00,0x00,0xa1,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x96,0x02,0x00,0x00, -0x9d,0x02,0x00,0x00,0xa0,0x02,0x00,0x00,0x3e,0x00,0x03,0x00, -0x9f,0x02,0x00,0x00,0xa1,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xa4,0x02,0x00,0x00,0x5c,0x03,0x00,0x00, -0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x7e,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x80,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x79,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x79,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa6,0x02,0x00,0x00, -0x5a,0x03,0x00,0x00,0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x76,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x78,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0x71,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x71,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xa8,0x02,0x00,0x00,0x58,0x03,0x00,0x00,0xc6,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x6e,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x70,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x69,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x69,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xaa,0x02,0x00,0x00,0x54,0x03,0x00,0x00, -0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x66,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x68,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x05,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x05,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xac,0x02,0x00,0x00, -0x4e,0x03,0x00,0x00,0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x02,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x04,0x02,0x00,0x00, -0xe0,0x00,0x04,0x00,0x0c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0xfa,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xcd,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xcd,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xae,0x02,0x00,0x00,0x34,0x03,0x00,0x00, -0x6c,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xca,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xcc,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xb3,0x02,0x00,0x00,0x55,0x00,0x00,0x00, -0x53,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xb4,0x02,0x00,0x00,0x8c,0x00,0x00,0x00,0xb3,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xb9,0x02,0x00,0x00, -0x59,0x00,0x00,0x00,0xaf,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xba,0x02,0x00,0x00,0x9e,0x00,0x00,0x00, -0xb9,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0xbf,0x02,0x00,0x00,0x12,0x00,0x00,0x00,0xbe,0x02,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xc0,0x02,0x00,0x00, -0xbf,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xc1,0x02,0x00,0x00,0x0f,0x00,0x00,0x00,0xc0,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xc5,0x02,0x00,0x00, -0x47,0x00,0x00,0x00,0xc0,0x02,0x00,0x00,0x41,0x00,0x05,0x00, -0x0d,0x00,0x00,0x00,0xc7,0x02,0x00,0x00,0xc6,0x02,0x00,0x00, -0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xc8,0x02,0x00,0x00,0xc7,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xc9,0x02,0x00,0x00,0xc5,0x02,0x00,0x00, -0xc8,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xca,0x02,0x00,0x00,0xc1,0x02,0x00,0x00,0xc9,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0xcc,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0xcc,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0x35,0x03,0x00,0x00,0x3e,0x00,0x00,0x00,0xcc,0x00,0x00,0x00, -0x32,0x03,0x00,0x00,0xcf,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb8,0x00,0x00,0x00,0xd2,0x02,0x00,0x00,0x35,0x03,0x00,0x00, -0xb5,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xce,0x02,0x00,0x00, -0xcf,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0xd2,0x02,0x00,0x00,0xcd,0x02,0x00,0x00,0xce,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0xcd,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0xd4,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0xd4,0x02,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x36,0x03,0x00,0x00, -0x3e,0x00,0x00,0x00,0xcd,0x02,0x00,0x00,0x30,0x03,0x00,0x00, -0xd7,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00, -0xda,0x02,0x00,0x00,0x36,0x03,0x00,0x00,0x60,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0xd6,0x02,0x00,0x00,0xd7,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xda,0x02,0x00,0x00, -0xd5,0x02,0x00,0x00,0xd6,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0xd5,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xde,0x02,0x00,0x00,0x36,0x03,0x00,0x00,0x61,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xdf,0x02,0x00,0x00, -0xb4,0x02,0x00,0x00,0xde,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xe1,0x02,0x00,0x00,0x64,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xe2,0x02,0x00,0x00,0xdf,0x02,0x00,0x00,0xe1,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xe6,0x02,0x00,0x00, -0x35,0x03,0x00,0x00,0x52,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xe7,0x02,0x00,0x00,0xba,0x02,0x00,0x00, -0xe6,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xe9,0x02,0x00,0x00,0x68,0x00,0x00,0x00,0xb2,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xea,0x02,0x00,0x00, -0xe7,0x02,0x00,0x00,0xe9,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0xec,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0xec,0x02,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x38,0x03,0x00,0x00, -0x3e,0x00,0x00,0x00,0xd5,0x02,0x00,0x00,0x2e,0x03,0x00,0x00, -0xef,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00, -0xf2,0x02,0x00,0x00,0x38,0x03,0x00,0x00,0xb2,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0xee,0x02,0x00,0x00,0xef,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xf2,0x02,0x00,0x00, -0xed,0x02,0x00,0x00,0xee,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0xed,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0xf4,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0xf4,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0x3a,0x03,0x00,0x00,0x3e,0x00,0x00,0x00, -0xed,0x02,0x00,0x00,0x2c,0x03,0x00,0x00,0xf7,0x02,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00,0xfa,0x02,0x00,0x00, -0x3a,0x03,0x00,0x00,0x62,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0xf6,0x02,0x00,0x00,0xf7,0x02,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xfa,0x02,0x00,0x00,0xf5,0x02,0x00,0x00, -0xf6,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0xf5,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xfd,0x02,0x00,0x00, -0xe2,0x02,0x00,0x00,0x3a,0x03,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb8,0x00,0x00,0x00,0x00,0x03,0x00,0x00,0xfd,0x02,0x00,0x00, -0x36,0x00,0x00,0x00,0xf7,0x00,0x03,0x00,0x02,0x03,0x00,0x00, -0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x00,0x03,0x00,0x00, -0x01,0x03,0x00,0x00,0x02,0x03,0x00,0x00,0xf8,0x00,0x02,0x00, -0x01,0x03,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x05,0x03,0x00,0x00,0xea,0x02,0x00,0x00,0x38,0x03,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x06,0x03,0x00,0x00, -0x12,0x00,0x00,0x00,0xc6,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x07,0x03,0x00,0x00,0x06,0x03,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00,0x08,0x03,0x00,0x00, -0x05,0x03,0x00,0x00,0x07,0x03,0x00,0x00,0xf9,0x00,0x02,0x00, -0x02,0x03,0x00,0x00,0xf8,0x00,0x02,0x00,0x02,0x03,0x00,0x00, -0xf5,0x00,0x07,0x00,0xb8,0x00,0x00,0x00,0x09,0x03,0x00,0x00, -0x00,0x03,0x00,0x00,0xf5,0x02,0x00,0x00,0x08,0x03,0x00,0x00, -0x01,0x03,0x00,0x00,0xf7,0x00,0x03,0x00,0x0b,0x03,0x00,0x00, -0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x09,0x03,0x00,0x00, -0x0a,0x03,0x00,0x00,0x0b,0x03,0x00,0x00,0xf8,0x00,0x02,0x00, -0x0a,0x03,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x13,0x03,0x00,0x00,0xea,0x02,0x00,0x00,0x38,0x03,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x15,0x03,0x00,0x00, -0x12,0x00,0x00,0x00,0x14,0x03,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x16,0x03,0x00,0x00,0x15,0x03,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x17,0x03,0x00,0x00, -0x13,0x03,0x00,0x00,0x16,0x03,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x18,0x03,0x00,0x00,0xca,0x02,0x00,0x00, -0x17,0x03,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x1a,0x03,0x00,0x00,0x18,0x03,0x00,0x00,0xe2,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x1c,0x03,0x00,0x00, -0x1a,0x03,0x00,0x00,0x3a,0x03,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x1e,0x03,0x00,0x00,0x35,0x03,0x00,0x00, -0xb2,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x20,0x03,0x00,0x00,0x1e,0x03,0x00,0x00,0x38,0x03,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x22,0x03,0x00,0x00, -0x20,0x03,0x00,0x00,0x21,0x03,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x24,0x03,0x00,0x00,0x36,0x03,0x00,0x00, -0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x25,0x03,0x00,0x00,0x22,0x03,0x00,0x00,0x24,0x03,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x27,0x03,0x00,0x00, -0x25,0x03,0x00,0x00,0x3a,0x03,0x00,0x00,0x41,0x00,0x05,0x00, -0xc3,0x00,0x00,0x00,0x28,0x03,0x00,0x00,0xc0,0x00,0x00,0x00, -0x27,0x03,0x00,0x00,0x3d,0x00,0x04,0x00,0xba,0x00,0x00,0x00, -0x29,0x03,0x00,0x00,0x28,0x03,0x00,0x00,0x41,0x00,0x06,0x00, -0x91,0x01,0x00,0x00,0x2a,0x03,0x00,0x00,0x0f,0x03,0x00,0x00, -0x34,0x00,0x00,0x00,0x1c,0x03,0x00,0x00,0x3e,0x00,0x03,0x00, -0x2a,0x03,0x00,0x00,0x29,0x03,0x00,0x00,0xf9,0x00,0x02,0x00, -0x0b,0x03,0x00,0x00,0xf8,0x00,0x02,0x00,0x0b,0x03,0x00,0x00, -0xf9,0x00,0x02,0x00,0xf7,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0xf7,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x2c,0x03,0x00,0x00,0x3a,0x03,0x00,0x00,0xc6,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xf4,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0xf6,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0xef,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0xef,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x2e,0x03,0x00,0x00,0x38,0x03,0x00,0x00, -0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xec,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0xee,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0xd7,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0xd7,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x30,0x03,0x00,0x00, -0x36,0x03,0x00,0x00,0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xd4,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0xd6,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0xcf,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0xcf,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x32,0x03,0x00,0x00,0x35,0x03,0x00,0x00,0xc6,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xcc,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0xce,0x02,0x00,0x00,0xfd,0x00,0x01,0x00,0x38,0x00,0x01,0x00, - -}; -const uint64_t matmul_f16_f32_aligned_m_len = 12096; - -unsigned char matmul_f16_f32_aligned_m_fp32_data[] = { -0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, -0xf0,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, -0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x51,0x11,0x00,0x00, -0x0b,0x00,0x06,0x00,0x01,0x00,0x00,0x00,0x47,0x4c,0x53,0x4c, -0x2e,0x73,0x74,0x64,0x2e,0x34,0x35,0x30,0x00,0x00,0x00,0x00, -0x0e,0x00,0x03,0x00,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x0f,0x00,0x0f,0x00,0x05,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x6d,0x61,0x69,0x6e,0x00,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x3d,0x00,0x00,0x00,0x4c,0x00,0x00,0x00, -0xea,0x00,0x00,0x00,0xf9,0x00,0x00,0x00,0x4b,0x01,0x00,0x00, -0x59,0x01,0x00,0x00,0x51,0x02,0x00,0x00,0x9a,0x02,0x00,0x00, -0x10,0x00,0x06,0x00,0x04,0x00,0x00,0x00,0x11,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x0b,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x1c,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x10,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x08,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x03,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x10,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x14,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x18,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x07,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x10,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x24,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x0a,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x10,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x2c,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x30,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x0d,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0x47,0x00,0x03,0x00, -0x10,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x37,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x3d,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x1a,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x4c,0x00,0x00,0x00, -0x0b,0x00,0x00,0x00,0x1b,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x4f,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x53,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x60,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x62,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x07,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x6c,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x03,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x9d,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0xaf,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x05,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0xb2,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x08,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xf6,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x48,0x00,0x04,0x00, -0xf7,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0xf7,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, -0xf7,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0xf9,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0xf9,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x2c,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x2d,0x01,0x00,0x00,0x0b,0x00,0x00,0x00,0x19,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x56,0x01,0x00,0x00,0x06,0x00,0x00,0x00, -0x10,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0x57,0x01,0x00,0x00, -0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x57,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x57,0x01,0x00,0x00, -0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x59,0x01,0x00,0x00, -0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x59,0x01,0x00,0x00,0x21,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x51,0x02,0x00,0x00,0x0b,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x97,0x02,0x00,0x00, -0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00, -0x98,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x19,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x98,0x02,0x00,0x00,0x00,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, -0x98,0x02,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x9a,0x02,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x9a,0x02,0x00,0x00,0x21,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x13,0x00,0x02,0x00,0x02,0x00,0x00,0x00, -0x21,0x00,0x03,0x00,0x03,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x15,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x17,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x0a,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x0d,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x1e,0x00,0x10,0x00,0x10,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x11,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x11,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x15,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, -0x14,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x15,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x0a,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, -0x27,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0x2d,0x00,0x00,0x00,0x07,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x34,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x37,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00,0x3d,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x3e,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x0a,0x00,0x00,0x00,0x4c,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x4f,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x53,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x54,0x00,0x00,0x00,0x86,0x00,0x00,0x00, -0x37,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x58,0x00,0x00,0x00,0x86,0x00,0x00,0x00, -0x37,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x32,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x61,0x00,0x00,0x00, -0x86,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x60,0x00,0x00,0x00, -0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x62,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x63,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x61,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x67,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x61,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x6e,0x00,0x00,0x00, -0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x73,0x00,0x00,0x00, -0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x77,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, -0x7c,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0x87,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x8d,0x00,0x00,0x00, -0x03,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, -0x98,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x32,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x9d,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x9f,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xae,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xaf,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xb0,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0x53,0x00,0x00,0x00,0xaf,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xb1,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0x4f,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x32,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xb2,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xb3,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0xb1,0x00,0x00,0x00,0xb2,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xb4,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0xb3,0x00,0x00,0x00,0x60,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xb5,0x00,0x00,0x00, -0x86,0x00,0x00,0x00,0xb0,0x00,0x00,0x00,0xb4,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xb6,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0xae,0x00,0x00,0x00,0xb5,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xb7,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0xb6,0x00,0x00,0x00,0xb2,0x00,0x00,0x00, -0x14,0x00,0x02,0x00,0xb8,0x00,0x00,0x00,0x16,0x00,0x03,0x00, -0xba,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xbb,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xbc,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0xbb,0x00,0x00,0x00,0xb5,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xbd,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0xbc,0x00,0x00,0x00,0xb2,0x00,0x00,0x00,0x1c,0x00,0x04,0x00, -0xbe,0x00,0x00,0x00,0xba,0x00,0x00,0x00,0xbd,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0xbf,0x00,0x00,0x00,0x07,0x00,0x00,0x00, -0xbe,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0xba,0x00,0x00,0x00, -0xc2,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0xc3,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0xba,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0xc6,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xe6,0x00,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xe7,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x37,0x00,0x00,0x00, -0xe6,0x00,0x00,0x00,0x1c,0x00,0x04,0x00,0xe8,0x00,0x00,0x00, -0xba,0x00,0x00,0x00,0xe7,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0xe9,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0xe8,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0xe9,0x00,0x00,0x00,0xea,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xee,0x00,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x16,0x00,0x03,0x00,0xf4,0x00,0x00,0x00, -0x10,0x00,0x00,0x00,0x17,0x00,0x04,0x00,0xf5,0x00,0x00,0x00, -0xf4,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, -0xf6,0x00,0x00,0x00,0xf5,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, -0xf7,0x00,0x00,0x00,0xf6,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0xf8,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0xf7,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0xf8,0x00,0x00,0x00,0xf9,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0xfb,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0xf4,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0xff,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0xba,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x04,0x01,0x00,0x00, -0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x12,0x01,0x00,0x00, -0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x20,0x01,0x00,0x00, -0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x25,0x01,0x00,0x00, -0x03,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x2c,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x33,0x00,0x06,0x00, -0x09,0x00,0x00,0x00,0x2d,0x01,0x00,0x00,0x2c,0x01,0x00,0x00, -0x39,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x2e,0x01,0x00,0x00,0x51,0x00,0x00,0x00, -0x2d,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x2f,0x01,0x00,0x00,0x84,0x00,0x00,0x00, -0x2e,0x01,0x00,0x00,0x6d,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x30,0x01,0x00,0x00,0x86,0x00,0x00,0x00, -0x2f,0x01,0x00,0x00,0x6c,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x47,0x01,0x00,0x00,0x80,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x48,0x01,0x00,0x00,0x84,0x00,0x00,0x00, -0x9d,0x00,0x00,0x00,0x47,0x01,0x00,0x00,0x1c,0x00,0x04,0x00, -0x49,0x01,0x00,0x00,0xba,0x00,0x00,0x00,0x48,0x01,0x00,0x00, -0x20,0x00,0x04,0x00,0x4a,0x01,0x00,0x00,0x04,0x00,0x00,0x00, -0x49,0x01,0x00,0x00,0x3b,0x00,0x04,0x00,0x4a,0x01,0x00,0x00, -0x4b,0x01,0x00,0x00,0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x4f,0x01,0x00,0x00,0x80,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x17,0x00,0x04,0x00, -0x55,0x01,0x00,0x00,0xba,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x1d,0x00,0x03,0x00,0x56,0x01,0x00,0x00,0x55,0x01,0x00,0x00, -0x1e,0x00,0x03,0x00,0x57,0x01,0x00,0x00,0x56,0x01,0x00,0x00, -0x20,0x00,0x04,0x00,0x58,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, -0x57,0x01,0x00,0x00,0x3b,0x00,0x04,0x00,0x58,0x01,0x00,0x00, -0x59,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x5b,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0xba,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x62,0x01,0x00,0x00, -0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x6f,0x01,0x00,0x00, -0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x7c,0x01,0x00,0x00, -0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x88,0x01,0x00,0x00, -0x08,0x01,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x89,0x01,0x00,0x00,0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x6d,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x8c,0x01,0x00,0x00,0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x6d,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xa7,0x01,0x00,0x00,0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x1c,0x00,0x04,0x00,0xa8,0x01,0x00,0x00, -0xba,0x00,0x00,0x00,0xa7,0x01,0x00,0x00,0x20,0x00,0x04,0x00, -0xa9,0x01,0x00,0x00,0x07,0x00,0x00,0x00,0xa8,0x01,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xb9,0x01,0x00,0x00, -0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xd4,0x01,0x00,0x00, -0x84,0x00,0x00,0x00,0xb5,0x00,0x00,0x00,0xb2,0x00,0x00,0x00, -0x1c,0x00,0x04,0x00,0xd5,0x01,0x00,0x00,0xba,0x00,0x00,0x00, -0xd4,0x01,0x00,0x00,0x20,0x00,0x04,0x00,0xd6,0x01,0x00,0x00, -0x07,0x00,0x00,0x00,0xd5,0x01,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xdf,0x01,0x00,0x00,0x86,0x00,0x00,0x00, -0xaf,0x00,0x00,0x00,0xb5,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xe7,0x01,0x00,0x00,0x80,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x16,0x02,0x00,0x00,0x84,0x00,0x00,0x00, -0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0x49,0x02,0x00,0x00,0x0d,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00,0x51,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0x1d,0x00,0x03,0x00,0x97,0x02,0x00,0x00, -0xba,0x00,0x00,0x00,0x1e,0x00,0x03,0x00,0x98,0x02,0x00,0x00, -0x97,0x02,0x00,0x00,0x20,0x00,0x04,0x00,0x99,0x02,0x00,0x00, -0x0c,0x00,0x00,0x00,0x98,0x02,0x00,0x00,0x3b,0x00,0x04,0x00, -0x99,0x02,0x00,0x00,0x9a,0x02,0x00,0x00,0x0c,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x9f,0x02,0x00,0x00, -0x05,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xac,0x02,0x00,0x00,0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x36,0x00,0x05,0x00,0x02,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x03,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0x05,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0xbf,0x00,0x00,0x00,0xc0,0x00,0x00,0x00,0x07,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0xa9,0x01,0x00,0x00,0xaa,0x01,0x00,0x00, -0x07,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0xd6,0x01,0x00,0x00, -0xd7,0x01,0x00,0x00,0x07,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x0d,0x00,0x00,0x00,0x0e,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x0f,0x00,0x00,0x00,0x0e,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x14,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x17,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, -0x17,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x1e,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x17,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x22,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x22,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x24,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x27,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x29,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x2a,0x00,0x00,0x00,0x1e,0x00,0x00,0x00, -0x29,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x2d,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x2f,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x30,0x00,0x00,0x00,0x24,0x00,0x00,0x00,0x2f,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x32,0x00,0x00,0x00, -0x30,0x00,0x00,0x00,0x2a,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x35,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x34,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x36,0x00,0x00,0x00,0x35,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x38,0x00,0x00,0x00,0x36,0x00,0x00,0x00, -0x37,0x00,0x00,0x00,0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x3a,0x00,0x00,0x00,0x38,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3b,0x00,0x00,0x00, -0x3a,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x0d,0x00,0x00,0x00,0x3f,0x00,0x00,0x00,0x3d,0x00,0x00,0x00, -0x3e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0x3f,0x00,0x00,0x00,0x89,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x42,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x3b,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x47,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x3b,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00,0x49,0x00,0x00,0x00, -0x3d,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x4a,0x00,0x00,0x00,0x49,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00,0x4d,0x00,0x00,0x00, -0x4c,0x00,0x00,0x00,0x3e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x4d,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x50,0x00,0x00,0x00, -0x4e,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x89,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x55,0x00,0x00,0x00,0x50,0x00,0x00,0x00, -0x54,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x59,0x00,0x00,0x00,0x50,0x00,0x00,0x00,0x58,0x00,0x00,0x00, -0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x5d,0x00,0x00,0x00, -0x4e,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x89,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x64,0x00,0x00,0x00,0x5d,0x00,0x00,0x00, -0x63,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x68,0x00,0x00,0x00,0x5d,0x00,0x00,0x00,0x67,0x00,0x00,0x00, -0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x6f,0x00,0x00,0x00, -0x4e,0x00,0x00,0x00,0x6e,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x74,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, -0x73,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0x78,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x77,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x79,0x00,0x00,0x00, -0x78,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x7a,0x00,0x00,0x00,0x47,0x00,0x00,0x00,0x79,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x7d,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x7c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x7e,0x00,0x00,0x00,0x7d,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x80,0x00,0x00,0x00, -0x47,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x83,0x00,0x00,0x00,0x80,0x00,0x00,0x00, -0x79,0x00,0x00,0x00,0x0c,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x26,0x00,0x00,0x00, -0x7e,0x00,0x00,0x00,0x83,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x88,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x87,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x89,0x00,0x00,0x00,0x88,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x8a,0x00,0x00,0x00,0x32,0x00,0x00,0x00, -0x89,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x8c,0x00,0x00,0x00,0x42,0x00,0x00,0x00,0x37,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x8e,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x8d,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x8f,0x00,0x00,0x00,0x8e,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x90,0x00,0x00,0x00, -0x8c,0x00,0x00,0x00,0x8f,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x91,0x00,0x00,0x00,0x8a,0x00,0x00,0x00, -0x90,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x93,0x00,0x00,0x00,0x91,0x00,0x00,0x00,0x7a,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x94,0x00,0x00,0x00, -0x93,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x99,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x98,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x9a,0x00,0x00,0x00,0x99,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x9b,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, -0x9a,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x9e,0x00,0x00,0x00,0x4a,0x00,0x00,0x00,0x9d,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0xa0,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x9f,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xa1,0x00,0x00,0x00,0xa0,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa2,0x00,0x00,0x00, -0x9e,0x00,0x00,0x00,0xa1,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xa3,0x00,0x00,0x00,0x9b,0x00,0x00,0x00, -0xa2,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xa5,0x00,0x00,0x00,0xa3,0x00,0x00,0x00,0x7a,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa6,0x00,0x00,0x00, -0xa5,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xa8,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xa8,0x00,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xbe,0x02,0x00,0x00, -0x3e,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0xc7,0x00,0x00,0x00, -0xa9,0x00,0x00,0x00,0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00, -0xb9,0x00,0x00,0x00,0xbe,0x02,0x00,0x00,0xb7,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0xaa,0x00,0x00,0x00,0xa9,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xb9,0x00,0x00,0x00, -0xa9,0x00,0x00,0x00,0xaa,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xa9,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0xc3,0x00,0x00,0x00, -0xc4,0x00,0x00,0x00,0xc0,0x00,0x00,0x00,0xbe,0x02,0x00,0x00, -0x3e,0x00,0x03,0x00,0xc4,0x00,0x00,0x00,0xc2,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xc7,0x00,0x00,0x00, -0xbe,0x02,0x00,0x00,0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xa8,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xaa,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xca,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xca,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xd7,0x02,0x00,0x00,0xa6,0x00,0x00,0x00,0xaa,0x00,0x00,0x00, -0x8e,0x01,0x00,0x00,0xcd,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xd3,0x02,0x00,0x00,0x94,0x00,0x00,0x00, -0xaa,0x00,0x00,0x00,0x8b,0x01,0x00,0x00,0xcd,0x00,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xbf,0x02,0x00,0x00, -0x7a,0x00,0x00,0x00,0xaa,0x00,0x00,0x00,0x39,0x02,0x00,0x00, -0xcd,0x00,0x00,0x00,0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00, -0xd1,0x00,0x00,0x00,0xbf,0x02,0x00,0x00,0x84,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0xcc,0x00,0x00,0x00,0xcd,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xd1,0x00,0x00,0x00, -0xcb,0x00,0x00,0x00,0xcc,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xcb,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xd3,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xd3,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xcf,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, -0xcb,0x00,0x00,0x00,0x32,0x01,0x00,0x00,0xd4,0x00,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00,0xd9,0x00,0x00,0x00, -0xcf,0x02,0x00,0x00,0x37,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0xd5,0x00,0x00,0x00,0xd4,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xd9,0x00,0x00,0x00,0xd4,0x00,0x00,0x00, -0xd5,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xd4,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xde,0x00,0x00,0x00, -0x74,0x00,0x00,0x00,0xcf,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xe1,0x00,0x00,0x00,0xde,0x00,0x00,0x00, -0x8f,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xe2,0x00,0x00,0x00,0xe1,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xe3,0x00,0x00,0x00, -0xd3,0x02,0x00,0x00,0xe2,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xe5,0x00,0x00,0x00,0xe3,0x00,0x00,0x00, -0x6f,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xef,0x00,0x00,0x00,0xde,0x00,0x00,0x00,0xee,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xf1,0x00,0x00,0x00, -0x6f,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xf2,0x00,0x00,0x00,0xef,0x00,0x00,0x00, -0xf1,0x00,0x00,0x00,0x41,0x00,0x07,0x00,0xfb,0x00,0x00,0x00, -0xfc,0x00,0x00,0x00,0xf9,0x00,0x00,0x00,0x34,0x00,0x00,0x00, -0xe5,0x00,0x00,0x00,0x3e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0xf4,0x00,0x00,0x00,0xfd,0x00,0x00,0x00,0xfc,0x00,0x00,0x00, -0x73,0x00,0x04,0x00,0xba,0x00,0x00,0x00,0xfe,0x00,0x00,0x00, -0xfd,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0xff,0x00,0x00,0x00, -0x00,0x01,0x00,0x00,0xea,0x00,0x00,0x00,0xf2,0x00,0x00,0x00, -0x3e,0x00,0x03,0x00,0x00,0x01,0x00,0x00,0xfe,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x05,0x01,0x00,0x00, -0xde,0x00,0x00,0x00,0x04,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x08,0x01,0x00,0x00,0x05,0x01,0x00,0x00, -0xf1,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x09,0x01,0x00,0x00,0x08,0x01,0x00,0x00,0x39,0x00,0x00,0x00, -0x41,0x00,0x07,0x00,0xfb,0x00,0x00,0x00,0x0b,0x01,0x00,0x00, -0xf9,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0xe5,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0xf4,0x00,0x00,0x00, -0x0c,0x01,0x00,0x00,0x0b,0x01,0x00,0x00,0x73,0x00,0x04,0x00, -0xba,0x00,0x00,0x00,0x0d,0x01,0x00,0x00,0x0c,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0xff,0x00,0x00,0x00,0x0e,0x01,0x00,0x00, -0xea,0x00,0x00,0x00,0x09,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0x0e,0x01,0x00,0x00,0x0d,0x01,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x13,0x01,0x00,0x00,0xde,0x00,0x00,0x00, -0x12,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x16,0x01,0x00,0x00,0x13,0x01,0x00,0x00,0xf1,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x17,0x01,0x00,0x00, -0x16,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x41,0x00,0x07,0x00, -0xfb,0x00,0x00,0x00,0x19,0x01,0x00,0x00,0xf9,0x00,0x00,0x00, -0x34,0x00,0x00,0x00,0xe5,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0xf4,0x00,0x00,0x00,0x1a,0x01,0x00,0x00, -0x19,0x01,0x00,0x00,0x73,0x00,0x04,0x00,0xba,0x00,0x00,0x00, -0x1b,0x01,0x00,0x00,0x1a,0x01,0x00,0x00,0x41,0x00,0x05,0x00, -0xff,0x00,0x00,0x00,0x1c,0x01,0x00,0x00,0xea,0x00,0x00,0x00, -0x17,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x1c,0x01,0x00,0x00, -0x1b,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x21,0x01,0x00,0x00,0xde,0x00,0x00,0x00,0x20,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x24,0x01,0x00,0x00, -0x21,0x01,0x00,0x00,0xf1,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x26,0x01,0x00,0x00,0x24,0x01,0x00,0x00, -0x25,0x01,0x00,0x00,0x41,0x00,0x07,0x00,0xfb,0x00,0x00,0x00, -0x28,0x01,0x00,0x00,0xf9,0x00,0x00,0x00,0x34,0x00,0x00,0x00, -0xe5,0x00,0x00,0x00,0x25,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, -0xf4,0x00,0x00,0x00,0x29,0x01,0x00,0x00,0x28,0x01,0x00,0x00, -0x73,0x00,0x04,0x00,0xba,0x00,0x00,0x00,0x2a,0x01,0x00,0x00, -0x29,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0xff,0x00,0x00,0x00, -0x2b,0x01,0x00,0x00,0xea,0x00,0x00,0x00,0x26,0x01,0x00,0x00, -0x3e,0x00,0x03,0x00,0x2b,0x01,0x00,0x00,0x2a,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x32,0x01,0x00,0x00, -0xcf,0x02,0x00,0x00,0x30,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0xd3,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xd5,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x34,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x34,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xd0,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0xd5,0x00,0x00,0x00, -0x87,0x01,0x00,0x00,0x35,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb8,0x00,0x00,0x00,0x3a,0x01,0x00,0x00,0xd0,0x02,0x00,0x00, -0x9d,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x36,0x01,0x00,0x00, -0x35,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x3a,0x01,0x00,0x00,0x35,0x01,0x00,0x00,0x36,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x35,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x3f,0x01,0x00,0x00,0x74,0x00,0x00,0x00, -0xd0,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x42,0x01,0x00,0x00,0x3f,0x01,0x00,0x00,0xa1,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x43,0x01,0x00,0x00, -0x42,0x01,0x00,0x00,0x6d,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x44,0x01,0x00,0x00,0xd7,0x02,0x00,0x00, -0x43,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x46,0x01,0x00,0x00,0x44,0x01,0x00,0x00,0x6f,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x50,0x01,0x00,0x00, -0x3f,0x01,0x00,0x00,0x4f,0x01,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x52,0x01,0x00,0x00,0x6f,0x00,0x00,0x00, -0x6d,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x53,0x01,0x00,0x00,0x50,0x01,0x00,0x00,0x52,0x01,0x00,0x00, -0x41,0x00,0x07,0x00,0x5b,0x01,0x00,0x00,0x5c,0x01,0x00,0x00, -0x59,0x01,0x00,0x00,0x34,0x00,0x00,0x00,0x46,0x01,0x00,0x00, -0x3e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0xba,0x00,0x00,0x00, -0x5d,0x01,0x00,0x00,0x5c,0x01,0x00,0x00,0x41,0x00,0x05,0x00, -0xff,0x00,0x00,0x00,0x5e,0x01,0x00,0x00,0x4b,0x01,0x00,0x00, -0x53,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x5e,0x01,0x00,0x00, -0x5d,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x63,0x01,0x00,0x00,0x3f,0x01,0x00,0x00,0x62,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x66,0x01,0x00,0x00, -0x63,0x01,0x00,0x00,0x52,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x67,0x01,0x00,0x00,0x66,0x01,0x00,0x00, -0x39,0x00,0x00,0x00,0x41,0x00,0x07,0x00,0x5b,0x01,0x00,0x00, -0x69,0x01,0x00,0x00,0x59,0x01,0x00,0x00,0x34,0x00,0x00,0x00, -0x46,0x01,0x00,0x00,0x39,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0xba,0x00,0x00,0x00,0x6a,0x01,0x00,0x00,0x69,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0xff,0x00,0x00,0x00,0x6b,0x01,0x00,0x00, -0x4b,0x01,0x00,0x00,0x67,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0x6b,0x01,0x00,0x00,0x6a,0x01,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x70,0x01,0x00,0x00,0x3f,0x01,0x00,0x00, -0x6f,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x73,0x01,0x00,0x00,0x70,0x01,0x00,0x00,0x52,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x74,0x01,0x00,0x00, -0x73,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x41,0x00,0x07,0x00, -0x5b,0x01,0x00,0x00,0x76,0x01,0x00,0x00,0x59,0x01,0x00,0x00, -0x34,0x00,0x00,0x00,0x46,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0xba,0x00,0x00,0x00,0x77,0x01,0x00,0x00, -0x76,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0xff,0x00,0x00,0x00, -0x78,0x01,0x00,0x00,0x4b,0x01,0x00,0x00,0x74,0x01,0x00,0x00, -0x3e,0x00,0x03,0x00,0x78,0x01,0x00,0x00,0x77,0x01,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x7d,0x01,0x00,0x00, -0x3f,0x01,0x00,0x00,0x7c,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x80,0x01,0x00,0x00,0x7d,0x01,0x00,0x00, -0x52,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x81,0x01,0x00,0x00,0x80,0x01,0x00,0x00,0x25,0x01,0x00,0x00, -0x41,0x00,0x07,0x00,0x5b,0x01,0x00,0x00,0x83,0x01,0x00,0x00, -0x59,0x01,0x00,0x00,0x34,0x00,0x00,0x00,0x46,0x01,0x00,0x00, -0x25,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0xba,0x00,0x00,0x00, -0x84,0x01,0x00,0x00,0x83,0x01,0x00,0x00,0x41,0x00,0x05,0x00, -0xff,0x00,0x00,0x00,0x85,0x01,0x00,0x00,0x4b,0x01,0x00,0x00, -0x81,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x85,0x01,0x00,0x00, -0x84,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x87,0x01,0x00,0x00,0xd0,0x02,0x00,0x00,0x30,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0x34,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x36,0x01,0x00,0x00,0xe0,0x00,0x04,0x00,0x0c,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x88,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x8b,0x01,0x00,0x00,0xd3,0x02,0x00,0x00, -0x89,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x8e,0x01,0x00,0x00,0xd7,0x02,0x00,0x00,0x8c,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0x90,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x90,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xd9,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0x36,0x01,0x00,0x00, -0x37,0x02,0x00,0x00,0x93,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb8,0x00,0x00,0x00,0x96,0x01,0x00,0x00,0xd9,0x02,0x00,0x00, -0x6c,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x92,0x01,0x00,0x00, -0x93,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x96,0x01,0x00,0x00,0x91,0x01,0x00,0x00,0x92,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x91,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0x98,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x98,0x01,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xdd,0x02,0x00,0x00, -0x3e,0x00,0x00,0x00,0x91,0x01,0x00,0x00,0xc3,0x01,0x00,0x00, -0x9b,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00, -0x9e,0x01,0x00,0x00,0xdd,0x02,0x00,0x00,0x60,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0x9a,0x01,0x00,0x00,0x9b,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x9e,0x01,0x00,0x00, -0x99,0x01,0x00,0x00,0x9a,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x99,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xa0,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xa0,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xef,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, -0x99,0x01,0x00,0x00,0xc1,0x01,0x00,0x00,0xa1,0x01,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00,0xa6,0x01,0x00,0x00, -0xef,0x02,0x00,0x00,0x62,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0xa2,0x01,0x00,0x00,0xa1,0x01,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xa6,0x01,0x00,0x00,0xa1,0x01,0x00,0x00, -0xa2,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xa1,0x01,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xac,0x01,0x00,0x00, -0xdd,0x02,0x00,0x00,0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xae,0x01,0x00,0x00,0xac,0x01,0x00,0x00, -0xef,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xb0,0x01,0x00,0x00,0x55,0x00,0x00,0x00,0x53,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xb2,0x01,0x00,0x00, -0xdd,0x02,0x00,0x00,0x61,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xb3,0x01,0x00,0x00,0xb0,0x01,0x00,0x00, -0xb2,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xb5,0x01,0x00,0x00,0x64,0x00,0x00,0x00,0x62,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xb6,0x01,0x00,0x00, -0xb3,0x01,0x00,0x00,0xb5,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xb8,0x01,0x00,0x00,0xb6,0x01,0x00,0x00, -0xef,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xba,0x01,0x00,0x00,0xb8,0x01,0x00,0x00,0xb9,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xbc,0x01,0x00,0x00, -0xba,0x01,0x00,0x00,0xd9,0x02,0x00,0x00,0x41,0x00,0x05,0x00, -0xff,0x00,0x00,0x00,0xbd,0x01,0x00,0x00,0xea,0x00,0x00,0x00, -0xbc,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0xba,0x00,0x00,0x00, -0xbe,0x01,0x00,0x00,0xbd,0x01,0x00,0x00,0x41,0x00,0x05,0x00, -0xc3,0x00,0x00,0x00,0xbf,0x01,0x00,0x00,0xaa,0x01,0x00,0x00, -0xae,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0xbf,0x01,0x00,0x00, -0xbe,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xc1,0x01,0x00,0x00,0xef,0x02,0x00,0x00,0xc6,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xa0,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xa2,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x9b,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x9b,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xc3,0x01,0x00,0x00,0xdd,0x02,0x00,0x00, -0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x98,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x9a,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0xc5,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xc5,0x01,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xde,0x02,0x00,0x00, -0x3e,0x00,0x00,0x00,0x9a,0x01,0x00,0x00,0xf1,0x01,0x00,0x00, -0xc8,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00, -0xcb,0x01,0x00,0x00,0xde,0x02,0x00,0x00,0xb5,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0xc7,0x01,0x00,0x00,0xc8,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xcb,0x01,0x00,0x00, -0xc6,0x01,0x00,0x00,0xc7,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xc6,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xcd,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xcd,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xec,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, -0xc6,0x01,0x00,0x00,0xef,0x01,0x00,0x00,0xce,0x01,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00,0xd3,0x01,0x00,0x00, -0xec,0x02,0x00,0x00,0xb2,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0xcf,0x01,0x00,0x00,0xce,0x01,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xd3,0x01,0x00,0x00,0xce,0x01,0x00,0x00, -0xcf,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xce,0x01,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xd9,0x01,0x00,0x00, -0xde,0x02,0x00,0x00,0xb2,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xdb,0x01,0x00,0x00,0xd9,0x01,0x00,0x00, -0xec,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xdd,0x01,0x00,0x00,0x59,0x00,0x00,0x00,0xaf,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xe0,0x01,0x00,0x00, -0xde,0x02,0x00,0x00,0xdf,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xe1,0x01,0x00,0x00,0xdd,0x01,0x00,0x00, -0xe0,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xe3,0x01,0x00,0x00,0x68,0x00,0x00,0x00,0xb2,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xe4,0x01,0x00,0x00, -0xe1,0x01,0x00,0x00,0xe3,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xe6,0x01,0x00,0x00,0xe4,0x01,0x00,0x00, -0xec,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xe8,0x01,0x00,0x00,0xe6,0x01,0x00,0x00,0xe7,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xea,0x01,0x00,0x00, -0xe8,0x01,0x00,0x00,0xd9,0x02,0x00,0x00,0x41,0x00,0x05,0x00, -0xff,0x00,0x00,0x00,0xeb,0x01,0x00,0x00,0x4b,0x01,0x00,0x00, -0xea,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0xba,0x00,0x00,0x00, -0xec,0x01,0x00,0x00,0xeb,0x01,0x00,0x00,0x41,0x00,0x05,0x00, -0xc3,0x00,0x00,0x00,0xed,0x01,0x00,0x00,0xd7,0x01,0x00,0x00, -0xdb,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0xed,0x01,0x00,0x00, -0xec,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xef,0x01,0x00,0x00,0xec,0x02,0x00,0x00,0xc6,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xcd,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xcf,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xc8,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xc8,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xf1,0x01,0x00,0x00,0xde,0x02,0x00,0x00, -0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xc5,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xc7,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0xf3,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xf3,0x01,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xdf,0x02,0x00,0x00, -0x3e,0x00,0x00,0x00,0xc7,0x01,0x00,0x00,0x35,0x02,0x00,0x00, -0xf6,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00, -0xf9,0x01,0x00,0x00,0xdf,0x02,0x00,0x00,0xb5,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0xf5,0x01,0x00,0x00,0xf6,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xf9,0x01,0x00,0x00, -0xf4,0x01,0x00,0x00,0xf5,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xf4,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xfb,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xfb,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xe3,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, -0xf4,0x01,0x00,0x00,0x33,0x02,0x00,0x00,0xfe,0x01,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00,0x01,0x02,0x00,0x00, -0xe3,0x02,0x00,0x00,0x60,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0xfd,0x01,0x00,0x00,0xfe,0x01,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x01,0x02,0x00,0x00,0xfc,0x01,0x00,0x00, -0xfd,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xfc,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0x03,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x03,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xe5,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0xfc,0x01,0x00,0x00, -0x31,0x02,0x00,0x00,0x06,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb8,0x00,0x00,0x00,0x09,0x02,0x00,0x00,0xe5,0x02,0x00,0x00, -0xb2,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x05,0x02,0x00,0x00, -0x06,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x09,0x02,0x00,0x00,0x04,0x02,0x00,0x00,0x05,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x04,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x0b,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x0b,0x02,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xe7,0x02,0x00,0x00, -0x3e,0x00,0x00,0x00,0x04,0x02,0x00,0x00,0x2f,0x02,0x00,0x00, -0x0c,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00, -0x11,0x02,0x00,0x00,0xe7,0x02,0x00,0x00,0x62,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0x0d,0x02,0x00,0x00,0x0c,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x11,0x02,0x00,0x00, -0x0c,0x02,0x00,0x00,0x0d,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x0c,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x13,0x02,0x00,0x00,0xdf,0x02,0x00,0x00,0xb2,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x15,0x02,0x00,0x00, -0x13,0x02,0x00,0x00,0xe5,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x17,0x02,0x00,0x00,0x15,0x02,0x00,0x00, -0x16,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x19,0x02,0x00,0x00,0xe3,0x02,0x00,0x00,0x62,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x1a,0x02,0x00,0x00, -0x17,0x02,0x00,0x00,0x19,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x1c,0x02,0x00,0x00,0x1a,0x02,0x00,0x00, -0xe7,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x20,0x02,0x00,0x00,0x19,0x02,0x00,0x00,0xe7,0x02,0x00,0x00, -0x41,0x00,0x05,0x00,0xc3,0x00,0x00,0x00,0x21,0x02,0x00,0x00, -0xaa,0x01,0x00,0x00,0x20,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, -0xba,0x00,0x00,0x00,0x22,0x02,0x00,0x00,0x21,0x02,0x00,0x00, -0x41,0x00,0x05,0x00,0xc3,0x00,0x00,0x00,0x27,0x02,0x00,0x00, -0xd7,0x01,0x00,0x00,0x15,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, -0xba,0x00,0x00,0x00,0x28,0x02,0x00,0x00,0x27,0x02,0x00,0x00, -0x41,0x00,0x05,0x00,0xc3,0x00,0x00,0x00,0x2a,0x02,0x00,0x00, -0xc0,0x00,0x00,0x00,0x1c,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, -0xba,0x00,0x00,0x00,0x2b,0x02,0x00,0x00,0x2a,0x02,0x00,0x00, -0x0c,0x00,0x08,0x00,0xba,0x00,0x00,0x00,0x2c,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x22,0x02,0x00,0x00, -0x28,0x02,0x00,0x00,0x2b,0x02,0x00,0x00,0x3e,0x00,0x03,0x00, -0x2a,0x02,0x00,0x00,0x2c,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x2f,0x02,0x00,0x00,0xe7,0x02,0x00,0x00, -0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x0b,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x0d,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x06,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x06,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x31,0x02,0x00,0x00, -0xe5,0x02,0x00,0x00,0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x03,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x05,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0xfe,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xfe,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x33,0x02,0x00,0x00,0xe3,0x02,0x00,0x00,0xc6,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xfb,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xfd,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xf6,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xf6,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x35,0x02,0x00,0x00,0xdf,0x02,0x00,0x00, -0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xf3,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xf5,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0x93,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x93,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x37,0x02,0x00,0x00, -0xd9,0x02,0x00,0x00,0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x90,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x92,0x01,0x00,0x00, -0xe0,0x00,0x04,0x00,0x0c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x88,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xcd,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xcd,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x39,0x02,0x00,0x00,0xbf,0x02,0x00,0x00, -0x6c,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xca,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xcc,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x3e,0x02,0x00,0x00,0x55,0x00,0x00,0x00, -0x53,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x3f,0x02,0x00,0x00,0x8c,0x00,0x00,0x00,0x3e,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x44,0x02,0x00,0x00, -0x59,0x00,0x00,0x00,0xaf,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x45,0x02,0x00,0x00,0x9e,0x00,0x00,0x00, -0x44,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0x4a,0x02,0x00,0x00,0x12,0x00,0x00,0x00,0x49,0x02,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x4b,0x02,0x00,0x00, -0x4a,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x4c,0x02,0x00,0x00,0x0f,0x00,0x00,0x00,0x4b,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x50,0x02,0x00,0x00, -0x47,0x00,0x00,0x00,0x4b,0x02,0x00,0x00,0x41,0x00,0x05,0x00, -0x0d,0x00,0x00,0x00,0x52,0x02,0x00,0x00,0x51,0x02,0x00,0x00, -0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x53,0x02,0x00,0x00,0x52,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x54,0x02,0x00,0x00,0x50,0x02,0x00,0x00, -0x53,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x55,0x02,0x00,0x00,0x4c,0x02,0x00,0x00,0x54,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0x57,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x57,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xc0,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0xcc,0x00,0x00,0x00, -0xbd,0x02,0x00,0x00,0x5a,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb8,0x00,0x00,0x00,0x5d,0x02,0x00,0x00,0xc0,0x02,0x00,0x00, -0xb5,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x59,0x02,0x00,0x00, -0x5a,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x5d,0x02,0x00,0x00,0x58,0x02,0x00,0x00,0x59,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x58,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x5f,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x5f,0x02,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xc1,0x02,0x00,0x00, -0x3e,0x00,0x00,0x00,0x58,0x02,0x00,0x00,0xbb,0x02,0x00,0x00, -0x62,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00, -0x65,0x02,0x00,0x00,0xc1,0x02,0x00,0x00,0x60,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0x61,0x02,0x00,0x00,0x62,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x65,0x02,0x00,0x00, -0x60,0x02,0x00,0x00,0x61,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x60,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x69,0x02,0x00,0x00,0xc1,0x02,0x00,0x00,0x61,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x6a,0x02,0x00,0x00, -0x3f,0x02,0x00,0x00,0x69,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x6c,0x02,0x00,0x00,0x64,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x6d,0x02,0x00,0x00,0x6a,0x02,0x00,0x00,0x6c,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x71,0x02,0x00,0x00, -0xc0,0x02,0x00,0x00,0xdf,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x72,0x02,0x00,0x00,0x45,0x02,0x00,0x00, -0x71,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x74,0x02,0x00,0x00,0x68,0x00,0x00,0x00,0xb2,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x75,0x02,0x00,0x00, -0x72,0x02,0x00,0x00,0x74,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x77,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x77,0x02,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xc3,0x02,0x00,0x00, -0x3e,0x00,0x00,0x00,0x60,0x02,0x00,0x00,0xb9,0x02,0x00,0x00, -0x7a,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00, -0x7d,0x02,0x00,0x00,0xc3,0x02,0x00,0x00,0xb2,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0x79,0x02,0x00,0x00,0x7a,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x7d,0x02,0x00,0x00, -0x78,0x02,0x00,0x00,0x79,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x78,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x7f,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x7f,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xc5,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, -0x78,0x02,0x00,0x00,0xb7,0x02,0x00,0x00,0x82,0x02,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00,0x85,0x02,0x00,0x00, -0xc5,0x02,0x00,0x00,0x62,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0x81,0x02,0x00,0x00,0x82,0x02,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x85,0x02,0x00,0x00,0x80,0x02,0x00,0x00, -0x81,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x80,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x88,0x02,0x00,0x00, -0x6d,0x02,0x00,0x00,0xc5,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb8,0x00,0x00,0x00,0x8b,0x02,0x00,0x00,0x88,0x02,0x00,0x00, -0x36,0x00,0x00,0x00,0xf7,0x00,0x03,0x00,0x8d,0x02,0x00,0x00, -0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x8b,0x02,0x00,0x00, -0x8c,0x02,0x00,0x00,0x8d,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x8c,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x90,0x02,0x00,0x00,0x75,0x02,0x00,0x00,0xc3,0x02,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x91,0x02,0x00,0x00, -0x12,0x00,0x00,0x00,0xc6,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x92,0x02,0x00,0x00,0x91,0x02,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00,0x93,0x02,0x00,0x00, -0x90,0x02,0x00,0x00,0x92,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x8d,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x8d,0x02,0x00,0x00, -0xf5,0x00,0x07,0x00,0xb8,0x00,0x00,0x00,0x94,0x02,0x00,0x00, -0x8b,0x02,0x00,0x00,0x80,0x02,0x00,0x00,0x93,0x02,0x00,0x00, -0x8c,0x02,0x00,0x00,0xf7,0x00,0x03,0x00,0x96,0x02,0x00,0x00, -0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x94,0x02,0x00,0x00, -0x95,0x02,0x00,0x00,0x96,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x95,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x9e,0x02,0x00,0x00,0x75,0x02,0x00,0x00,0xc3,0x02,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0xa0,0x02,0x00,0x00, -0x12,0x00,0x00,0x00,0x9f,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xa1,0x02,0x00,0x00,0xa0,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa2,0x02,0x00,0x00, -0x9e,0x02,0x00,0x00,0xa1,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xa3,0x02,0x00,0x00,0x55,0x02,0x00,0x00, -0xa2,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xa5,0x02,0x00,0x00,0xa3,0x02,0x00,0x00,0x6d,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa7,0x02,0x00,0x00, -0xa5,0x02,0x00,0x00,0xc5,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xa9,0x02,0x00,0x00,0xc0,0x02,0x00,0x00, -0xb2,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xab,0x02,0x00,0x00,0xa9,0x02,0x00,0x00,0xc3,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xad,0x02,0x00,0x00, -0xab,0x02,0x00,0x00,0xac,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xaf,0x02,0x00,0x00,0xc1,0x02,0x00,0x00, -0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xb0,0x02,0x00,0x00,0xad,0x02,0x00,0x00,0xaf,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xb2,0x02,0x00,0x00, -0xb0,0x02,0x00,0x00,0xc5,0x02,0x00,0x00,0x41,0x00,0x05,0x00, -0xc3,0x00,0x00,0x00,0xb3,0x02,0x00,0x00,0xc0,0x00,0x00,0x00, -0xb2,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0xba,0x00,0x00,0x00, -0xb4,0x02,0x00,0x00,0xb3,0x02,0x00,0x00,0x41,0x00,0x06,0x00, -0x5b,0x01,0x00,0x00,0xb5,0x02,0x00,0x00,0x9a,0x02,0x00,0x00, -0x34,0x00,0x00,0x00,0xa7,0x02,0x00,0x00,0x3e,0x00,0x03,0x00, -0xb5,0x02,0x00,0x00,0xb4,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x96,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x96,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0x82,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x82,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xb7,0x02,0x00,0x00,0xc5,0x02,0x00,0x00,0xc6,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x7f,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x81,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x7a,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x7a,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xb9,0x02,0x00,0x00,0xc3,0x02,0x00,0x00, -0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x77,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x79,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x62,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x62,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xbb,0x02,0x00,0x00, -0xc1,0x02,0x00,0x00,0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x5f,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x61,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0x5a,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x5a,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xbd,0x02,0x00,0x00,0xc0,0x02,0x00,0x00,0xc6,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x57,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x59,0x02,0x00,0x00,0xfd,0x00,0x01,0x00,0x38,0x00,0x01,0x00, - -}; -const uint64_t matmul_f16_f32_aligned_m_fp32_len = 10464; - -unsigned char matmul_f16_f32_aligned_s_data[] = { -0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, -0x65,0x03,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, -0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x09,0x00,0x00,0x00, -0x11,0x00,0x02,0x00,0x51,0x11,0x00,0x00,0x0b,0x00,0x06,0x00, -0x01,0x00,0x00,0x00,0x47,0x4c,0x53,0x4c,0x2e,0x73,0x74,0x64, -0x2e,0x34,0x35,0x30,0x00,0x00,0x00,0x00,0x0e,0x00,0x03,0x00, -0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x0f,0x00,0x0f,0x00, -0x05,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x6d,0x61,0x69,0x6e, -0x00,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x3d,0x00,0x00,0x00,0x4c,0x00,0x00,0x00,0xeb,0x00,0x00,0x00, -0xfa,0x00,0x00,0x00,0x80,0x01,0x00,0x00,0x8f,0x01,0x00,0x00, -0xc6,0x02,0x00,0x00,0x0f,0x03,0x00,0x00,0x10,0x00,0x06,0x00, -0x04,0x00,0x00,0x00,0x11,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x0b,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x1c,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x10,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x08,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x03,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x10,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x10,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x05,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x14,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x10,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x1c,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x08,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x24,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x10,0x00,0x00,0x00,0x0a,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x28,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x0b,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x2c,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x30,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x10,0x00,0x00,0x00,0x0d,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x34,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x10,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x37,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x3d,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x1a,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x4c,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x1b,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x4f,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x53,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x60,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x62,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x6c,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x03,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x9d,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xaf,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0xb2,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x08,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0xf7,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x10,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0xf8,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0x48,0x00,0x04,0x00, -0xf8,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0xf8,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0xf8,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x07,0x00,0x00,0x00, -0x08,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0xf8,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xfa,0x00,0x00,0x00, -0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0xfa,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x61,0x01,0x00,0x00,0x01,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x62,0x01,0x00,0x00, -0x0b,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x8c,0x01,0x00,0x00,0x06,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x48,0x00,0x04,0x00,0x8d,0x01,0x00,0x00,0x00,0x00,0x00,0x00, -0x05,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0x8d,0x01,0x00,0x00, -0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x8d,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x8d,0x01,0x00,0x00, -0x00,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0x10,0x00,0x00,0x00, -0x47,0x00,0x03,0x00,0x8d,0x01,0x00,0x00,0x02,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x8f,0x01,0x00,0x00,0x22,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x8f,0x01,0x00,0x00, -0x21,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0xc6,0x02,0x00,0x00,0x0b,0x00,0x00,0x00,0x18,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x0c,0x03,0x00,0x00,0x06,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0x0d,0x03,0x00,0x00, -0x00,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x0d,0x03,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x0d,0x03,0x00,0x00, -0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x0f,0x03,0x00,0x00, -0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x0f,0x03,0x00,0x00,0x21,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x13,0x00,0x02,0x00,0x02,0x00,0x00,0x00,0x21,0x00,0x03,0x00, -0x03,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x15,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x17,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x03,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x0a,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x0a,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x0d,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x1e,0x00,0x10,0x00, -0x10,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x11,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x10,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x11,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x15,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x14,0x00,0x00,0x00, -0x08,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x15,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x0a,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x27,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, -0x2d,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x37,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x0a,0x00,0x00,0x00,0x3d,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x3e,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00, -0x4c,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x53,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x54,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x37,0x00,0x00,0x00, -0x53,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x58,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x37,0x00,0x00,0x00, -0x53,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x60,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x61,0x00,0x00,0x00,0x86,0x00,0x00,0x00, -0x53,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x32,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x63,0x00,0x00,0x00, -0x86,0x00,0x00,0x00,0x61,0x00,0x00,0x00,0x62,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x67,0x00,0x00,0x00, -0x86,0x00,0x00,0x00,0x61,0x00,0x00,0x00,0x62,0x00,0x00,0x00, -0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x10,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x6d,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x6e,0x00,0x00,0x00,0x86,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x73,0x00,0x00,0x00,0x86,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0x77,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x7c,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, -0x87,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0x8d,0x00,0x00,0x00,0x03,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x98,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x9d,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0x9f,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xae,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00, -0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xaf,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xb0,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x53,0x00,0x00,0x00, -0xaf,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xb1,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x4f,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xb2,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xb3,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0xb1,0x00,0x00,0x00,0xb2,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xb4,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0xb3,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xb5,0x00,0x00,0x00,0x86,0x00,0x00,0x00, -0xb0,0x00,0x00,0x00,0xb4,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xb6,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0xae,0x00,0x00,0x00,0xb5,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xb7,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0xb6,0x00,0x00,0x00,0xb2,0x00,0x00,0x00,0x14,0x00,0x02,0x00, -0xb8,0x00,0x00,0x00,0x16,0x00,0x03,0x00,0xba,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xbb,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xbc,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0xbb,0x00,0x00,0x00, -0xb5,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xbd,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0xbc,0x00,0x00,0x00, -0xb2,0x00,0x00,0x00,0x1c,0x00,0x04,0x00,0xbe,0x00,0x00,0x00, -0xba,0x00,0x00,0x00,0xbd,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0xbf,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0xbe,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0xba,0x00,0x00,0x00,0xc2,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0xc3,0x00,0x00,0x00, -0x07,0x00,0x00,0x00,0xba,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0xc6,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x16,0x00,0x03,0x00,0xe6,0x00,0x00,0x00,0x10,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xe7,0x00,0x00,0x00, -0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xe8,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0xe7,0x00,0x00,0x00, -0x1c,0x00,0x04,0x00,0xe9,0x00,0x00,0x00,0xe6,0x00,0x00,0x00, -0xe8,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0xea,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0xe9,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0xea,0x00,0x00,0x00,0xeb,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xef,0x00,0x00,0x00, -0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x17,0x00,0x04,0x00,0xf5,0x00,0x00,0x00,0xe6,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x18,0x00,0x04,0x00,0xf6,0x00,0x00,0x00, -0xf5,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, -0xf7,0x00,0x00,0x00,0xf6,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, -0xf8,0x00,0x00,0x00,0xf7,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0xf9,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0xf8,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0xf9,0x00,0x00,0x00,0xfa,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0xfc,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0xe6,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0xff,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0xe6,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x04,0x01,0x00,0x00, -0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x11,0x01,0x00,0x00, -0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x1e,0x01,0x00,0x00, -0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x23,0x01,0x00,0x00, -0x03,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x2c,0x01,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x31,0x01,0x00,0x00,0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x3a,0x01,0x00,0x00,0x80,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x3f,0x01,0x00,0x00,0x05,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x48,0x01,0x00,0x00, -0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x4d,0x01,0x00,0x00, -0x06,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x56,0x01,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x5b,0x01,0x00,0x00,0x07,0x00,0x00,0x00,0x32,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x61,0x01,0x00,0x00,0x01,0x00,0x00,0x00, -0x33,0x00,0x06,0x00,0x09,0x00,0x00,0x00,0x62,0x01,0x00,0x00, -0x61,0x01,0x00,0x00,0x39,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x63,0x01,0x00,0x00, -0x51,0x00,0x00,0x00,0x62,0x01,0x00,0x00,0x00,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x64,0x01,0x00,0x00, -0x84,0x00,0x00,0x00,0x63,0x01,0x00,0x00,0x6d,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x65,0x01,0x00,0x00, -0x86,0x00,0x00,0x00,0x64,0x01,0x00,0x00,0x6c,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x7c,0x01,0x00,0x00, -0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x7d,0x01,0x00,0x00, -0x84,0x00,0x00,0x00,0x9d,0x00,0x00,0x00,0x7c,0x01,0x00,0x00, -0x1c,0x00,0x04,0x00,0x7e,0x01,0x00,0x00,0xe6,0x00,0x00,0x00, -0x7d,0x01,0x00,0x00,0x20,0x00,0x04,0x00,0x7f,0x01,0x00,0x00, -0x04,0x00,0x00,0x00,0x7e,0x01,0x00,0x00,0x3b,0x00,0x04,0x00, -0x7f,0x01,0x00,0x00,0x80,0x01,0x00,0x00,0x04,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x84,0x01,0x00,0x00, -0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x17,0x00,0x04,0x00,0x8a,0x01,0x00,0x00,0xba,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x18,0x00,0x04,0x00,0x8b,0x01,0x00,0x00, -0x8a,0x01,0x00,0x00,0x02,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, -0x8c,0x01,0x00,0x00,0x8b,0x01,0x00,0x00,0x1e,0x00,0x03,0x00, -0x8d,0x01,0x00,0x00,0x8c,0x01,0x00,0x00,0x20,0x00,0x04,0x00, -0x8e,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x8d,0x01,0x00,0x00, -0x3b,0x00,0x04,0x00,0x8e,0x01,0x00,0x00,0x8f,0x01,0x00,0x00, -0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x91,0x01,0x00,0x00, -0x0c,0x00,0x00,0x00,0xba,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x99,0x01,0x00,0x00,0x80,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xa7,0x01,0x00,0x00,0x80,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xb5,0x01,0x00,0x00,0x80,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xc3,0x01,0x00,0x00,0x80,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xd1,0x01,0x00,0x00,0x80,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xdf,0x01,0x00,0x00,0x80,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xed,0x01,0x00,0x00,0x80,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xfa,0x01,0x00,0x00,0x08,0x01,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xfb,0x01,0x00,0x00, -0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xfe,0x01,0x00,0x00, -0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x19,0x02,0x00,0x00, -0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00, -0x1c,0x00,0x04,0x00,0x1a,0x02,0x00,0x00,0xe6,0x00,0x00,0x00, -0x19,0x02,0x00,0x00,0x20,0x00,0x04,0x00,0x1b,0x02,0x00,0x00, -0x07,0x00,0x00,0x00,0x1a,0x02,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x2b,0x02,0x00,0x00,0x80,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x31,0x02,0x00,0x00,0x07,0x00,0x00,0x00,0xe6,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x47,0x02,0x00,0x00, -0x84,0x00,0x00,0x00,0xb5,0x00,0x00,0x00,0xb2,0x00,0x00,0x00, -0x1c,0x00,0x04,0x00,0x48,0x02,0x00,0x00,0xe6,0x00,0x00,0x00, -0x47,0x02,0x00,0x00,0x20,0x00,0x04,0x00,0x49,0x02,0x00,0x00, -0x07,0x00,0x00,0x00,0x48,0x02,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x52,0x02,0x00,0x00,0x86,0x00,0x00,0x00, -0xaf,0x00,0x00,0x00,0xb5,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x5a,0x02,0x00,0x00,0x80,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x89,0x02,0x00,0x00,0x84,0x00,0x00,0x00, -0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0xbe,0x02,0x00,0x00,0x0d,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00,0xc6,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0x1d,0x00,0x03,0x00,0x0c,0x03,0x00,0x00, -0xba,0x00,0x00,0x00,0x1e,0x00,0x03,0x00,0x0d,0x03,0x00,0x00, -0x0c,0x03,0x00,0x00,0x20,0x00,0x04,0x00,0x0e,0x03,0x00,0x00, -0x0c,0x00,0x00,0x00,0x0d,0x03,0x00,0x00,0x3b,0x00,0x04,0x00, -0x0e,0x03,0x00,0x00,0x0f,0x03,0x00,0x00,0x0c,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x14,0x03,0x00,0x00, -0x05,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x21,0x03,0x00,0x00,0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x36,0x00,0x05,0x00,0x02,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x03,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0x05,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0xbf,0x00,0x00,0x00,0xc0,0x00,0x00,0x00,0x07,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x1b,0x02,0x00,0x00,0x1c,0x02,0x00,0x00, -0x07,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x49,0x02,0x00,0x00, -0x4a,0x02,0x00,0x00,0x07,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x0d,0x00,0x00,0x00,0x0e,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x0f,0x00,0x00,0x00,0x0e,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x14,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x17,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, -0x17,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x1e,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x17,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x22,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x22,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x24,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x27,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x29,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x2a,0x00,0x00,0x00,0x1e,0x00,0x00,0x00, -0x29,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x2d,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x2f,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x30,0x00,0x00,0x00,0x24,0x00,0x00,0x00,0x2f,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x32,0x00,0x00,0x00, -0x30,0x00,0x00,0x00,0x2a,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x35,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x34,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x36,0x00,0x00,0x00,0x35,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x38,0x00,0x00,0x00,0x36,0x00,0x00,0x00, -0x37,0x00,0x00,0x00,0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x3a,0x00,0x00,0x00,0x38,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3b,0x00,0x00,0x00, -0x3a,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x0d,0x00,0x00,0x00,0x3f,0x00,0x00,0x00,0x3d,0x00,0x00,0x00, -0x3e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0x3f,0x00,0x00,0x00,0x89,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x42,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x3b,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x47,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x3b,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00,0x49,0x00,0x00,0x00, -0x3d,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x4a,0x00,0x00,0x00,0x49,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00,0x4d,0x00,0x00,0x00, -0x4c,0x00,0x00,0x00,0x3e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x4d,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x50,0x00,0x00,0x00, -0x4e,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x89,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x55,0x00,0x00,0x00,0x50,0x00,0x00,0x00, -0x54,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x59,0x00,0x00,0x00,0x50,0x00,0x00,0x00,0x58,0x00,0x00,0x00, -0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x5d,0x00,0x00,0x00, -0x4e,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x89,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x64,0x00,0x00,0x00,0x5d,0x00,0x00,0x00, -0x63,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x68,0x00,0x00,0x00,0x5d,0x00,0x00,0x00,0x67,0x00,0x00,0x00, -0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x6f,0x00,0x00,0x00, -0x4e,0x00,0x00,0x00,0x6e,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x74,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, -0x73,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0x78,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x77,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x79,0x00,0x00,0x00, -0x78,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x7a,0x00,0x00,0x00,0x47,0x00,0x00,0x00,0x79,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x7d,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x7c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x7e,0x00,0x00,0x00,0x7d,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x80,0x00,0x00,0x00, -0x47,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x83,0x00,0x00,0x00,0x80,0x00,0x00,0x00, -0x79,0x00,0x00,0x00,0x0c,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x26,0x00,0x00,0x00, -0x7e,0x00,0x00,0x00,0x83,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x88,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x87,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x89,0x00,0x00,0x00,0x88,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x8a,0x00,0x00,0x00,0x32,0x00,0x00,0x00, -0x89,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x8c,0x00,0x00,0x00,0x42,0x00,0x00,0x00,0x37,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x8e,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x8d,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x8f,0x00,0x00,0x00,0x8e,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x90,0x00,0x00,0x00, -0x8c,0x00,0x00,0x00,0x8f,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x91,0x00,0x00,0x00,0x8a,0x00,0x00,0x00, -0x90,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x93,0x00,0x00,0x00,0x91,0x00,0x00,0x00,0x7a,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x94,0x00,0x00,0x00, -0x93,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x99,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x98,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x9a,0x00,0x00,0x00,0x99,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x9b,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, -0x9a,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x9e,0x00,0x00,0x00,0x4a,0x00,0x00,0x00,0x9d,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0xa0,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x9f,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xa1,0x00,0x00,0x00,0xa0,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa2,0x00,0x00,0x00, -0x9e,0x00,0x00,0x00,0xa1,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xa3,0x00,0x00,0x00,0x9b,0x00,0x00,0x00, -0xa2,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xa5,0x00,0x00,0x00,0xa3,0x00,0x00,0x00,0x7a,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa6,0x00,0x00,0x00, -0xa5,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xa8,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xa8,0x00,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x33,0x03,0x00,0x00, -0x3e,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0xc7,0x00,0x00,0x00, -0xa9,0x00,0x00,0x00,0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00, -0xb9,0x00,0x00,0x00,0x33,0x03,0x00,0x00,0xb7,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0xaa,0x00,0x00,0x00,0xa9,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xb9,0x00,0x00,0x00, -0xa9,0x00,0x00,0x00,0xaa,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xa9,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0xc3,0x00,0x00,0x00, -0xc4,0x00,0x00,0x00,0xc0,0x00,0x00,0x00,0x33,0x03,0x00,0x00, -0x3e,0x00,0x03,0x00,0xc4,0x00,0x00,0x00,0xc2,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xc7,0x00,0x00,0x00, -0x33,0x03,0x00,0x00,0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xa8,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xaa,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xca,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xca,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0x4c,0x03,0x00,0x00,0xa6,0x00,0x00,0x00,0xaa,0x00,0x00,0x00, -0x00,0x02,0x00,0x00,0xcd,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0x48,0x03,0x00,0x00,0x94,0x00,0x00,0x00, -0xaa,0x00,0x00,0x00,0xfd,0x01,0x00,0x00,0xcd,0x00,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x34,0x03,0x00,0x00, -0x7a,0x00,0x00,0x00,0xaa,0x00,0x00,0x00,0xae,0x02,0x00,0x00, -0xcd,0x00,0x00,0x00,0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00, -0xd1,0x00,0x00,0x00,0x34,0x03,0x00,0x00,0x84,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0xcc,0x00,0x00,0x00,0xcd,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xd1,0x00,0x00,0x00, -0xcb,0x00,0x00,0x00,0xcc,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xcb,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xd3,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xd3,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0x44,0x03,0x00,0x00,0x3e,0x00,0x00,0x00, -0xcb,0x00,0x00,0x00,0x67,0x01,0x00,0x00,0xd4,0x00,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00,0xd9,0x00,0x00,0x00, -0x44,0x03,0x00,0x00,0x37,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0xd5,0x00,0x00,0x00,0xd4,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xd9,0x00,0x00,0x00,0xd4,0x00,0x00,0x00, -0xd5,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xd4,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xde,0x00,0x00,0x00, -0x74,0x00,0x00,0x00,0x44,0x03,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xe1,0x00,0x00,0x00,0xde,0x00,0x00,0x00, -0x8f,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xe2,0x00,0x00,0x00,0xe1,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xe3,0x00,0x00,0x00, -0x48,0x03,0x00,0x00,0xe2,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xe5,0x00,0x00,0x00,0xe3,0x00,0x00,0x00, -0x6f,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xf0,0x00,0x00,0x00,0xde,0x00,0x00,0x00,0xef,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xf2,0x00,0x00,0x00, -0x6f,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xf3,0x00,0x00,0x00,0xf0,0x00,0x00,0x00, -0xf2,0x00,0x00,0x00,0x41,0x00,0x08,0x00,0xfc,0x00,0x00,0x00, -0xfd,0x00,0x00,0x00,0xfa,0x00,0x00,0x00,0x34,0x00,0x00,0x00, -0xe5,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0x3e,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0xe6,0x00,0x00,0x00,0xfe,0x00,0x00,0x00, -0xfd,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0xff,0x00,0x00,0x00, -0x00,0x01,0x00,0x00,0xeb,0x00,0x00,0x00,0xf3,0x00,0x00,0x00, -0x3e,0x00,0x03,0x00,0x00,0x01,0x00,0x00,0xfe,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x05,0x01,0x00,0x00, -0xde,0x00,0x00,0x00,0x04,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x08,0x01,0x00,0x00,0x05,0x01,0x00,0x00, -0xf2,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x09,0x01,0x00,0x00,0x08,0x01,0x00,0x00,0x39,0x00,0x00,0x00, -0x41,0x00,0x08,0x00,0xfc,0x00,0x00,0x00,0x0b,0x01,0x00,0x00, -0xfa,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0xe5,0x00,0x00,0x00, -0x34,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0xe6,0x00,0x00,0x00,0x0c,0x01,0x00,0x00,0x0b,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0xff,0x00,0x00,0x00,0x0d,0x01,0x00,0x00, -0xeb,0x00,0x00,0x00,0x09,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0x0d,0x01,0x00,0x00,0x0c,0x01,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x12,0x01,0x00,0x00,0xde,0x00,0x00,0x00, -0x11,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x15,0x01,0x00,0x00,0x12,0x01,0x00,0x00,0xf2,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x16,0x01,0x00,0x00, -0x15,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x41,0x00,0x08,0x00, -0xfc,0x00,0x00,0x00,0x18,0x01,0x00,0x00,0xfa,0x00,0x00,0x00, -0x34,0x00,0x00,0x00,0xe5,0x00,0x00,0x00,0x34,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0xe6,0x00,0x00,0x00, -0x19,0x01,0x00,0x00,0x18,0x01,0x00,0x00,0x41,0x00,0x05,0x00, -0xff,0x00,0x00,0x00,0x1a,0x01,0x00,0x00,0xeb,0x00,0x00,0x00, -0x16,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x1a,0x01,0x00,0x00, -0x19,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x1f,0x01,0x00,0x00,0xde,0x00,0x00,0x00,0x1e,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x22,0x01,0x00,0x00, -0x1f,0x01,0x00,0x00,0xf2,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x24,0x01,0x00,0x00,0x22,0x01,0x00,0x00, -0x23,0x01,0x00,0x00,0x41,0x00,0x08,0x00,0xfc,0x00,0x00,0x00, -0x26,0x01,0x00,0x00,0xfa,0x00,0x00,0x00,0x34,0x00,0x00,0x00, -0xe5,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0x23,0x01,0x00,0x00, -0x3d,0x00,0x04,0x00,0xe6,0x00,0x00,0x00,0x27,0x01,0x00,0x00, -0x26,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0xff,0x00,0x00,0x00, -0x28,0x01,0x00,0x00,0xeb,0x00,0x00,0x00,0x24,0x01,0x00,0x00, -0x3e,0x00,0x03,0x00,0x28,0x01,0x00,0x00,0x27,0x01,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x2d,0x01,0x00,0x00, -0xde,0x00,0x00,0x00,0x2c,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x30,0x01,0x00,0x00,0x2d,0x01,0x00,0x00, -0xf2,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x32,0x01,0x00,0x00,0x30,0x01,0x00,0x00,0x31,0x01,0x00,0x00, -0x41,0x00,0x08,0x00,0xfc,0x00,0x00,0x00,0x34,0x01,0x00,0x00, -0xfa,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0xe5,0x00,0x00,0x00, -0xc6,0x00,0x00,0x00,0x3e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0xe6,0x00,0x00,0x00,0x35,0x01,0x00,0x00,0x34,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0xff,0x00,0x00,0x00,0x36,0x01,0x00,0x00, -0xeb,0x00,0x00,0x00,0x32,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0x36,0x01,0x00,0x00,0x35,0x01,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x3b,0x01,0x00,0x00,0xde,0x00,0x00,0x00, -0x3a,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x3e,0x01,0x00,0x00,0x3b,0x01,0x00,0x00,0xf2,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x40,0x01,0x00,0x00, -0x3e,0x01,0x00,0x00,0x3f,0x01,0x00,0x00,0x41,0x00,0x08,0x00, -0xfc,0x00,0x00,0x00,0x42,0x01,0x00,0x00,0xfa,0x00,0x00,0x00, -0x34,0x00,0x00,0x00,0xe5,0x00,0x00,0x00,0xc6,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0xe6,0x00,0x00,0x00, -0x43,0x01,0x00,0x00,0x42,0x01,0x00,0x00,0x41,0x00,0x05,0x00, -0xff,0x00,0x00,0x00,0x44,0x01,0x00,0x00,0xeb,0x00,0x00,0x00, -0x40,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x44,0x01,0x00,0x00, -0x43,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x49,0x01,0x00,0x00,0xde,0x00,0x00,0x00,0x48,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x4c,0x01,0x00,0x00, -0x49,0x01,0x00,0x00,0xf2,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x4e,0x01,0x00,0x00,0x4c,0x01,0x00,0x00, -0x4d,0x01,0x00,0x00,0x41,0x00,0x08,0x00,0xfc,0x00,0x00,0x00, -0x50,0x01,0x00,0x00,0xfa,0x00,0x00,0x00,0x34,0x00,0x00,0x00, -0xe5,0x00,0x00,0x00,0xc6,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0xe6,0x00,0x00,0x00,0x51,0x01,0x00,0x00, -0x50,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0xff,0x00,0x00,0x00, -0x52,0x01,0x00,0x00,0xeb,0x00,0x00,0x00,0x4e,0x01,0x00,0x00, -0x3e,0x00,0x03,0x00,0x52,0x01,0x00,0x00,0x51,0x01,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x57,0x01,0x00,0x00, -0xde,0x00,0x00,0x00,0x56,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x5a,0x01,0x00,0x00,0x57,0x01,0x00,0x00, -0xf2,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x5c,0x01,0x00,0x00,0x5a,0x01,0x00,0x00,0x5b,0x01,0x00,0x00, -0x41,0x00,0x08,0x00,0xfc,0x00,0x00,0x00,0x5e,0x01,0x00,0x00, -0xfa,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0xe5,0x00,0x00,0x00, -0xc6,0x00,0x00,0x00,0x23,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, -0xe6,0x00,0x00,0x00,0x5f,0x01,0x00,0x00,0x5e,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0xff,0x00,0x00,0x00,0x60,0x01,0x00,0x00, -0xeb,0x00,0x00,0x00,0x5c,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0x60,0x01,0x00,0x00,0x5f,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x67,0x01,0x00,0x00,0x44,0x03,0x00,0x00, -0x65,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xd3,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xd5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x69,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x69,0x01,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x45,0x03,0x00,0x00, -0x3e,0x00,0x00,0x00,0xd5,0x00,0x00,0x00,0xf9,0x01,0x00,0x00, -0x6a,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00, -0x6f,0x01,0x00,0x00,0x45,0x03,0x00,0x00,0x9d,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0x6b,0x01,0x00,0x00,0x6a,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x6f,0x01,0x00,0x00, -0x6a,0x01,0x00,0x00,0x6b,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x6a,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x74,0x01,0x00,0x00,0x74,0x00,0x00,0x00,0x45,0x03,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x77,0x01,0x00,0x00, -0x74,0x01,0x00,0x00,0xa1,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x78,0x01,0x00,0x00,0x77,0x01,0x00,0x00, -0x6d,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x79,0x01,0x00,0x00,0x4c,0x03,0x00,0x00,0x78,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x7b,0x01,0x00,0x00, -0x79,0x01,0x00,0x00,0x6f,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x85,0x01,0x00,0x00,0x74,0x01,0x00,0x00, -0x84,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x87,0x01,0x00,0x00,0x6f,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x88,0x01,0x00,0x00, -0x85,0x01,0x00,0x00,0x87,0x01,0x00,0x00,0x41,0x00,0x08,0x00, -0x91,0x01,0x00,0x00,0x92,0x01,0x00,0x00,0x8f,0x01,0x00,0x00, -0x34,0x00,0x00,0x00,0x7b,0x01,0x00,0x00,0x34,0x00,0x00,0x00, -0x3e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0xba,0x00,0x00,0x00, -0x93,0x01,0x00,0x00,0x92,0x01,0x00,0x00,0x73,0x00,0x04,0x00, -0xe6,0x00,0x00,0x00,0x94,0x01,0x00,0x00,0x93,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0xff,0x00,0x00,0x00,0x95,0x01,0x00,0x00, -0x80,0x01,0x00,0x00,0x88,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0x95,0x01,0x00,0x00,0x94,0x01,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x9a,0x01,0x00,0x00,0x74,0x01,0x00,0x00, -0x99,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x9d,0x01,0x00,0x00,0x9a,0x01,0x00,0x00,0x87,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9e,0x01,0x00,0x00, -0x9d,0x01,0x00,0x00,0x39,0x00,0x00,0x00,0x41,0x00,0x08,0x00, -0x91,0x01,0x00,0x00,0xa0,0x01,0x00,0x00,0x8f,0x01,0x00,0x00, -0x34,0x00,0x00,0x00,0x7b,0x01,0x00,0x00,0x34,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0xba,0x00,0x00,0x00, -0xa1,0x01,0x00,0x00,0xa0,0x01,0x00,0x00,0x73,0x00,0x04,0x00, -0xe6,0x00,0x00,0x00,0xa2,0x01,0x00,0x00,0xa1,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0xff,0x00,0x00,0x00,0xa3,0x01,0x00,0x00, -0x80,0x01,0x00,0x00,0x9e,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0xa3,0x01,0x00,0x00,0xa2,0x01,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xa8,0x01,0x00,0x00,0x74,0x01,0x00,0x00, -0xa7,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xab,0x01,0x00,0x00,0xa8,0x01,0x00,0x00,0x87,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xac,0x01,0x00,0x00, -0xab,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x41,0x00,0x08,0x00, -0x91,0x01,0x00,0x00,0xae,0x01,0x00,0x00,0x8f,0x01,0x00,0x00, -0x34,0x00,0x00,0x00,0x7b,0x01,0x00,0x00,0x34,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0xba,0x00,0x00,0x00, -0xaf,0x01,0x00,0x00,0xae,0x01,0x00,0x00,0x73,0x00,0x04,0x00, -0xe6,0x00,0x00,0x00,0xb0,0x01,0x00,0x00,0xaf,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0xff,0x00,0x00,0x00,0xb1,0x01,0x00,0x00, -0x80,0x01,0x00,0x00,0xac,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0xb1,0x01,0x00,0x00,0xb0,0x01,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xb6,0x01,0x00,0x00,0x74,0x01,0x00,0x00, -0xb5,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xb9,0x01,0x00,0x00,0xb6,0x01,0x00,0x00,0x87,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xba,0x01,0x00,0x00, -0xb9,0x01,0x00,0x00,0x23,0x01,0x00,0x00,0x41,0x00,0x08,0x00, -0x91,0x01,0x00,0x00,0xbc,0x01,0x00,0x00,0x8f,0x01,0x00,0x00, -0x34,0x00,0x00,0x00,0x7b,0x01,0x00,0x00,0x34,0x00,0x00,0x00, -0x23,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0xba,0x00,0x00,0x00, -0xbd,0x01,0x00,0x00,0xbc,0x01,0x00,0x00,0x73,0x00,0x04,0x00, -0xe6,0x00,0x00,0x00,0xbe,0x01,0x00,0x00,0xbd,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0xff,0x00,0x00,0x00,0xbf,0x01,0x00,0x00, -0x80,0x01,0x00,0x00,0xba,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0xbf,0x01,0x00,0x00,0xbe,0x01,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xc4,0x01,0x00,0x00,0x74,0x01,0x00,0x00, -0xc3,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xc7,0x01,0x00,0x00,0xc4,0x01,0x00,0x00,0x87,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xc8,0x01,0x00,0x00, -0xc7,0x01,0x00,0x00,0x31,0x01,0x00,0x00,0x41,0x00,0x08,0x00, -0x91,0x01,0x00,0x00,0xca,0x01,0x00,0x00,0x8f,0x01,0x00,0x00, -0x34,0x00,0x00,0x00,0x7b,0x01,0x00,0x00,0xc6,0x00,0x00,0x00, -0x3e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0xba,0x00,0x00,0x00, -0xcb,0x01,0x00,0x00,0xca,0x01,0x00,0x00,0x73,0x00,0x04,0x00, -0xe6,0x00,0x00,0x00,0xcc,0x01,0x00,0x00,0xcb,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0xff,0x00,0x00,0x00,0xcd,0x01,0x00,0x00, -0x80,0x01,0x00,0x00,0xc8,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0xcd,0x01,0x00,0x00,0xcc,0x01,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xd2,0x01,0x00,0x00,0x74,0x01,0x00,0x00, -0xd1,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xd5,0x01,0x00,0x00,0xd2,0x01,0x00,0x00,0x87,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xd6,0x01,0x00,0x00, -0xd5,0x01,0x00,0x00,0x3f,0x01,0x00,0x00,0x41,0x00,0x08,0x00, -0x91,0x01,0x00,0x00,0xd8,0x01,0x00,0x00,0x8f,0x01,0x00,0x00, -0x34,0x00,0x00,0x00,0x7b,0x01,0x00,0x00,0xc6,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0xba,0x00,0x00,0x00, -0xd9,0x01,0x00,0x00,0xd8,0x01,0x00,0x00,0x73,0x00,0x04,0x00, -0xe6,0x00,0x00,0x00,0xda,0x01,0x00,0x00,0xd9,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0xff,0x00,0x00,0x00,0xdb,0x01,0x00,0x00, -0x80,0x01,0x00,0x00,0xd6,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0xdb,0x01,0x00,0x00,0xda,0x01,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xe0,0x01,0x00,0x00,0x74,0x01,0x00,0x00, -0xdf,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xe3,0x01,0x00,0x00,0xe0,0x01,0x00,0x00,0x87,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xe4,0x01,0x00,0x00, -0xe3,0x01,0x00,0x00,0x4d,0x01,0x00,0x00,0x41,0x00,0x08,0x00, -0x91,0x01,0x00,0x00,0xe6,0x01,0x00,0x00,0x8f,0x01,0x00,0x00, -0x34,0x00,0x00,0x00,0x7b,0x01,0x00,0x00,0xc6,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0xba,0x00,0x00,0x00, -0xe7,0x01,0x00,0x00,0xe6,0x01,0x00,0x00,0x73,0x00,0x04,0x00, -0xe6,0x00,0x00,0x00,0xe8,0x01,0x00,0x00,0xe7,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0xff,0x00,0x00,0x00,0xe9,0x01,0x00,0x00, -0x80,0x01,0x00,0x00,0xe4,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0xe9,0x01,0x00,0x00,0xe8,0x01,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xee,0x01,0x00,0x00,0x74,0x01,0x00,0x00, -0xed,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xf1,0x01,0x00,0x00,0xee,0x01,0x00,0x00,0x87,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xf2,0x01,0x00,0x00, -0xf1,0x01,0x00,0x00,0x5b,0x01,0x00,0x00,0x41,0x00,0x08,0x00, -0x91,0x01,0x00,0x00,0xf4,0x01,0x00,0x00,0x8f,0x01,0x00,0x00, -0x34,0x00,0x00,0x00,0x7b,0x01,0x00,0x00,0xc6,0x00,0x00,0x00, -0x23,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0xba,0x00,0x00,0x00, -0xf5,0x01,0x00,0x00,0xf4,0x01,0x00,0x00,0x73,0x00,0x04,0x00, -0xe6,0x00,0x00,0x00,0xf6,0x01,0x00,0x00,0xf5,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0xff,0x00,0x00,0x00,0xf7,0x01,0x00,0x00, -0x80,0x01,0x00,0x00,0xf2,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0xf7,0x01,0x00,0x00,0xf6,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xf9,0x01,0x00,0x00,0x45,0x03,0x00,0x00, -0x65,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x69,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x6b,0x01,0x00,0x00,0xe0,0x00,0x04,0x00, -0x0c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0xfa,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xfd,0x01,0x00,0x00, -0x48,0x03,0x00,0x00,0xfb,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x00,0x02,0x00,0x00,0x4c,0x03,0x00,0x00, -0xfe,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x02,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x02,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0x4e,0x03,0x00,0x00,0x3e,0x00,0x00,0x00, -0x6b,0x01,0x00,0x00,0xac,0x02,0x00,0x00,0x05,0x02,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00,0x08,0x02,0x00,0x00, -0x4e,0x03,0x00,0x00,0x6c,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0x04,0x02,0x00,0x00,0x05,0x02,0x00,0x00,0x00,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x08,0x02,0x00,0x00,0x03,0x02,0x00,0x00, -0x04,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x03,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0x0a,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x0a,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0x52,0x03,0x00,0x00,0x3e,0x00,0x00,0x00,0x03,0x02,0x00,0x00, -0x36,0x02,0x00,0x00,0x0d,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb8,0x00,0x00,0x00,0x10,0x02,0x00,0x00,0x52,0x03,0x00,0x00, -0x60,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x0c,0x02,0x00,0x00, -0x0d,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x10,0x02,0x00,0x00,0x0b,0x02,0x00,0x00,0x0c,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x0b,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x12,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x12,0x02,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x64,0x03,0x00,0x00, -0x3e,0x00,0x00,0x00,0x0b,0x02,0x00,0x00,0x34,0x02,0x00,0x00, -0x13,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00, -0x18,0x02,0x00,0x00,0x64,0x03,0x00,0x00,0x62,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0x14,0x02,0x00,0x00,0x13,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x18,0x02,0x00,0x00, -0x13,0x02,0x00,0x00,0x14,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x13,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x1e,0x02,0x00,0x00,0x52,0x03,0x00,0x00,0x62,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x20,0x02,0x00,0x00, -0x1e,0x02,0x00,0x00,0x64,0x03,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x22,0x02,0x00,0x00,0x55,0x00,0x00,0x00, -0x53,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x24,0x02,0x00,0x00,0x52,0x03,0x00,0x00,0x61,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x25,0x02,0x00,0x00, -0x22,0x02,0x00,0x00,0x24,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x27,0x02,0x00,0x00,0x64,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x28,0x02,0x00,0x00,0x25,0x02,0x00,0x00,0x27,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x2a,0x02,0x00,0x00, -0x28,0x02,0x00,0x00,0x64,0x03,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x2c,0x02,0x00,0x00,0x2a,0x02,0x00,0x00, -0x2b,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x2e,0x02,0x00,0x00,0x2c,0x02,0x00,0x00,0x4e,0x03,0x00,0x00, -0x41,0x00,0x05,0x00,0xff,0x00,0x00,0x00,0x2f,0x02,0x00,0x00, -0xeb,0x00,0x00,0x00,0x2e,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, -0xe6,0x00,0x00,0x00,0x30,0x02,0x00,0x00,0x2f,0x02,0x00,0x00, -0x41,0x00,0x05,0x00,0x31,0x02,0x00,0x00,0x32,0x02,0x00,0x00, -0x1c,0x02,0x00,0x00,0x20,0x02,0x00,0x00,0x3e,0x00,0x03,0x00, -0x32,0x02,0x00,0x00,0x30,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x34,0x02,0x00,0x00,0x64,0x03,0x00,0x00, -0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x12,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x14,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x0d,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x0d,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x36,0x02,0x00,0x00, -0x52,0x03,0x00,0x00,0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x0a,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x0c,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0x38,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x38,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0x53,0x03,0x00,0x00,0x3e,0x00,0x00,0x00,0x0c,0x02,0x00,0x00, -0x64,0x02,0x00,0x00,0x3b,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb8,0x00,0x00,0x00,0x3e,0x02,0x00,0x00,0x53,0x03,0x00,0x00, -0xb5,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x3a,0x02,0x00,0x00, -0x3b,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x3e,0x02,0x00,0x00,0x39,0x02,0x00,0x00,0x3a,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x39,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x40,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x40,0x02,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x61,0x03,0x00,0x00, -0x3e,0x00,0x00,0x00,0x39,0x02,0x00,0x00,0x62,0x02,0x00,0x00, -0x41,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00, -0x46,0x02,0x00,0x00,0x61,0x03,0x00,0x00,0xb2,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0x42,0x02,0x00,0x00,0x41,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x46,0x02,0x00,0x00, -0x41,0x02,0x00,0x00,0x42,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x41,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x4c,0x02,0x00,0x00,0x53,0x03,0x00,0x00,0xb2,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x4e,0x02,0x00,0x00, -0x4c,0x02,0x00,0x00,0x61,0x03,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x50,0x02,0x00,0x00,0x59,0x00,0x00,0x00, -0xaf,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x53,0x02,0x00,0x00,0x53,0x03,0x00,0x00,0x52,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x54,0x02,0x00,0x00, -0x50,0x02,0x00,0x00,0x53,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x56,0x02,0x00,0x00,0x68,0x00,0x00,0x00, -0xb2,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x57,0x02,0x00,0x00,0x54,0x02,0x00,0x00,0x56,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x59,0x02,0x00,0x00, -0x57,0x02,0x00,0x00,0x61,0x03,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x5b,0x02,0x00,0x00,0x59,0x02,0x00,0x00, -0x5a,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x5d,0x02,0x00,0x00,0x5b,0x02,0x00,0x00,0x4e,0x03,0x00,0x00, -0x41,0x00,0x05,0x00,0xff,0x00,0x00,0x00,0x5e,0x02,0x00,0x00, -0x80,0x01,0x00,0x00,0x5d,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, -0xe6,0x00,0x00,0x00,0x5f,0x02,0x00,0x00,0x5e,0x02,0x00,0x00, -0x41,0x00,0x05,0x00,0x31,0x02,0x00,0x00,0x60,0x02,0x00,0x00, -0x4a,0x02,0x00,0x00,0x4e,0x02,0x00,0x00,0x3e,0x00,0x03,0x00, -0x60,0x02,0x00,0x00,0x5f,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x62,0x02,0x00,0x00,0x61,0x03,0x00,0x00, -0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x40,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x42,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x3b,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x3b,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x64,0x02,0x00,0x00, -0x53,0x03,0x00,0x00,0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x38,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x3a,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0x66,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x66,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0x54,0x03,0x00,0x00,0x3e,0x00,0x00,0x00,0x3a,0x02,0x00,0x00, -0xaa,0x02,0x00,0x00,0x69,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb8,0x00,0x00,0x00,0x6c,0x02,0x00,0x00,0x54,0x03,0x00,0x00, -0xb5,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x68,0x02,0x00,0x00, -0x69,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x6c,0x02,0x00,0x00,0x67,0x02,0x00,0x00,0x68,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x67,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x6e,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x6e,0x02,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x58,0x03,0x00,0x00, -0x3e,0x00,0x00,0x00,0x67,0x02,0x00,0x00,0xa8,0x02,0x00,0x00, -0x71,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00, -0x74,0x02,0x00,0x00,0x58,0x03,0x00,0x00,0x60,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0x70,0x02,0x00,0x00,0x71,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x74,0x02,0x00,0x00, -0x6f,0x02,0x00,0x00,0x70,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x6f,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x76,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x76,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0x5a,0x03,0x00,0x00,0x3e,0x00,0x00,0x00, -0x6f,0x02,0x00,0x00,0xa6,0x02,0x00,0x00,0x79,0x02,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00,0x7c,0x02,0x00,0x00, -0x5a,0x03,0x00,0x00,0xb2,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0x78,0x02,0x00,0x00,0x79,0x02,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x7c,0x02,0x00,0x00,0x77,0x02,0x00,0x00, -0x78,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x77,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0x7e,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x7e,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0x5c,0x03,0x00,0x00,0x3e,0x00,0x00,0x00,0x77,0x02,0x00,0x00, -0xa4,0x02,0x00,0x00,0x7f,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb8,0x00,0x00,0x00,0x84,0x02,0x00,0x00,0x5c,0x03,0x00,0x00, -0x62,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x80,0x02,0x00,0x00, -0x7f,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x84,0x02,0x00,0x00,0x7f,0x02,0x00,0x00,0x80,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x7f,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x86,0x02,0x00,0x00,0x54,0x03,0x00,0x00, -0xb2,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x88,0x02,0x00,0x00,0x86,0x02,0x00,0x00,0x5a,0x03,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8a,0x02,0x00,0x00, -0x88,0x02,0x00,0x00,0x89,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x8c,0x02,0x00,0x00,0x58,0x03,0x00,0x00, -0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x8d,0x02,0x00,0x00,0x8a,0x02,0x00,0x00,0x8c,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8f,0x02,0x00,0x00, -0x8d,0x02,0x00,0x00,0x5c,0x03,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x93,0x02,0x00,0x00,0x8c,0x02,0x00,0x00, -0x5c,0x03,0x00,0x00,0x41,0x00,0x05,0x00,0x31,0x02,0x00,0x00, -0x94,0x02,0x00,0x00,0x1c,0x02,0x00,0x00,0x93,0x02,0x00,0x00, -0x3d,0x00,0x04,0x00,0xe6,0x00,0x00,0x00,0x95,0x02,0x00,0x00, -0x94,0x02,0x00,0x00,0x73,0x00,0x04,0x00,0xba,0x00,0x00,0x00, -0x96,0x02,0x00,0x00,0x95,0x02,0x00,0x00,0x41,0x00,0x05,0x00, -0x31,0x02,0x00,0x00,0x9b,0x02,0x00,0x00,0x4a,0x02,0x00,0x00, -0x88,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0xe6,0x00,0x00,0x00, -0x9c,0x02,0x00,0x00,0x9b,0x02,0x00,0x00,0x73,0x00,0x04,0x00, -0xba,0x00,0x00,0x00,0x9d,0x02,0x00,0x00,0x9c,0x02,0x00,0x00, -0x41,0x00,0x05,0x00,0xc3,0x00,0x00,0x00,0x9f,0x02,0x00,0x00, -0xc0,0x00,0x00,0x00,0x8f,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, -0xba,0x00,0x00,0x00,0xa0,0x02,0x00,0x00,0x9f,0x02,0x00,0x00, -0x0c,0x00,0x08,0x00,0xba,0x00,0x00,0x00,0xa1,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x96,0x02,0x00,0x00, -0x9d,0x02,0x00,0x00,0xa0,0x02,0x00,0x00,0x3e,0x00,0x03,0x00, -0x9f,0x02,0x00,0x00,0xa1,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xa4,0x02,0x00,0x00,0x5c,0x03,0x00,0x00, -0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x7e,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x80,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x79,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x79,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa6,0x02,0x00,0x00, -0x5a,0x03,0x00,0x00,0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x76,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x78,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0x71,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x71,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xa8,0x02,0x00,0x00,0x58,0x03,0x00,0x00,0xc6,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x6e,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x70,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x69,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x69,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xaa,0x02,0x00,0x00,0x54,0x03,0x00,0x00, -0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x66,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x68,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x05,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x05,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xac,0x02,0x00,0x00, -0x4e,0x03,0x00,0x00,0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x02,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x04,0x02,0x00,0x00, -0xe0,0x00,0x04,0x00,0x0c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0xfa,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xcd,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xcd,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xae,0x02,0x00,0x00,0x34,0x03,0x00,0x00, -0x6c,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xca,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xcc,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xb3,0x02,0x00,0x00,0x55,0x00,0x00,0x00, -0x53,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xb4,0x02,0x00,0x00,0x8c,0x00,0x00,0x00,0xb3,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xb9,0x02,0x00,0x00, -0x59,0x00,0x00,0x00,0xaf,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xba,0x02,0x00,0x00,0x9e,0x00,0x00,0x00, -0xb9,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0xbf,0x02,0x00,0x00,0x12,0x00,0x00,0x00,0xbe,0x02,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xc0,0x02,0x00,0x00, -0xbf,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xc1,0x02,0x00,0x00,0x0f,0x00,0x00,0x00,0xc0,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xc5,0x02,0x00,0x00, -0x47,0x00,0x00,0x00,0xc0,0x02,0x00,0x00,0x41,0x00,0x05,0x00, -0x0d,0x00,0x00,0x00,0xc7,0x02,0x00,0x00,0xc6,0x02,0x00,0x00, -0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xc8,0x02,0x00,0x00,0xc7,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xc9,0x02,0x00,0x00,0xc5,0x02,0x00,0x00, -0xc8,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xca,0x02,0x00,0x00,0xc1,0x02,0x00,0x00,0xc9,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0xcc,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0xcc,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0x35,0x03,0x00,0x00,0x3e,0x00,0x00,0x00,0xcc,0x00,0x00,0x00, -0x32,0x03,0x00,0x00,0xcf,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb8,0x00,0x00,0x00,0xd2,0x02,0x00,0x00,0x35,0x03,0x00,0x00, -0xb5,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xce,0x02,0x00,0x00, -0xcf,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0xd2,0x02,0x00,0x00,0xcd,0x02,0x00,0x00,0xce,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0xcd,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0xd4,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0xd4,0x02,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x36,0x03,0x00,0x00, -0x3e,0x00,0x00,0x00,0xcd,0x02,0x00,0x00,0x30,0x03,0x00,0x00, -0xd7,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00, -0xda,0x02,0x00,0x00,0x36,0x03,0x00,0x00,0x60,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0xd6,0x02,0x00,0x00,0xd7,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xda,0x02,0x00,0x00, -0xd5,0x02,0x00,0x00,0xd6,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0xd5,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xde,0x02,0x00,0x00,0x36,0x03,0x00,0x00,0x61,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xdf,0x02,0x00,0x00, -0xb4,0x02,0x00,0x00,0xde,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xe1,0x02,0x00,0x00,0x64,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xe2,0x02,0x00,0x00,0xdf,0x02,0x00,0x00,0xe1,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xe6,0x02,0x00,0x00, -0x35,0x03,0x00,0x00,0x52,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xe7,0x02,0x00,0x00,0xba,0x02,0x00,0x00, -0xe6,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xe9,0x02,0x00,0x00,0x68,0x00,0x00,0x00,0xb2,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xea,0x02,0x00,0x00, -0xe7,0x02,0x00,0x00,0xe9,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0xec,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0xec,0x02,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x38,0x03,0x00,0x00, -0x3e,0x00,0x00,0x00,0xd5,0x02,0x00,0x00,0x2e,0x03,0x00,0x00, -0xef,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00, -0xf2,0x02,0x00,0x00,0x38,0x03,0x00,0x00,0xb2,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0xee,0x02,0x00,0x00,0xef,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xf2,0x02,0x00,0x00, -0xed,0x02,0x00,0x00,0xee,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0xed,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0xf4,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0xf4,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0x3a,0x03,0x00,0x00,0x3e,0x00,0x00,0x00, -0xed,0x02,0x00,0x00,0x2c,0x03,0x00,0x00,0xf7,0x02,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00,0xfa,0x02,0x00,0x00, -0x3a,0x03,0x00,0x00,0x62,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0xf6,0x02,0x00,0x00,0xf7,0x02,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xfa,0x02,0x00,0x00,0xf5,0x02,0x00,0x00, -0xf6,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0xf5,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xfd,0x02,0x00,0x00, -0xe2,0x02,0x00,0x00,0x3a,0x03,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb8,0x00,0x00,0x00,0x00,0x03,0x00,0x00,0xfd,0x02,0x00,0x00, -0x36,0x00,0x00,0x00,0xf7,0x00,0x03,0x00,0x02,0x03,0x00,0x00, -0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x00,0x03,0x00,0x00, -0x01,0x03,0x00,0x00,0x02,0x03,0x00,0x00,0xf8,0x00,0x02,0x00, -0x01,0x03,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x05,0x03,0x00,0x00,0xea,0x02,0x00,0x00,0x38,0x03,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x06,0x03,0x00,0x00, -0x12,0x00,0x00,0x00,0xc6,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x07,0x03,0x00,0x00,0x06,0x03,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00,0x08,0x03,0x00,0x00, -0x05,0x03,0x00,0x00,0x07,0x03,0x00,0x00,0xf9,0x00,0x02,0x00, -0x02,0x03,0x00,0x00,0xf8,0x00,0x02,0x00,0x02,0x03,0x00,0x00, -0xf5,0x00,0x07,0x00,0xb8,0x00,0x00,0x00,0x09,0x03,0x00,0x00, -0x00,0x03,0x00,0x00,0xf5,0x02,0x00,0x00,0x08,0x03,0x00,0x00, -0x01,0x03,0x00,0x00,0xf7,0x00,0x03,0x00,0x0b,0x03,0x00,0x00, -0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x09,0x03,0x00,0x00, -0x0a,0x03,0x00,0x00,0x0b,0x03,0x00,0x00,0xf8,0x00,0x02,0x00, -0x0a,0x03,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x13,0x03,0x00,0x00,0xea,0x02,0x00,0x00,0x38,0x03,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x15,0x03,0x00,0x00, -0x12,0x00,0x00,0x00,0x14,0x03,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x16,0x03,0x00,0x00,0x15,0x03,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x17,0x03,0x00,0x00, -0x13,0x03,0x00,0x00,0x16,0x03,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x18,0x03,0x00,0x00,0xca,0x02,0x00,0x00, -0x17,0x03,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x1a,0x03,0x00,0x00,0x18,0x03,0x00,0x00,0xe2,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x1c,0x03,0x00,0x00, -0x1a,0x03,0x00,0x00,0x3a,0x03,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x1e,0x03,0x00,0x00,0x35,0x03,0x00,0x00, -0xb2,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x20,0x03,0x00,0x00,0x1e,0x03,0x00,0x00,0x38,0x03,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x22,0x03,0x00,0x00, -0x20,0x03,0x00,0x00,0x21,0x03,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x24,0x03,0x00,0x00,0x36,0x03,0x00,0x00, -0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x25,0x03,0x00,0x00,0x22,0x03,0x00,0x00,0x24,0x03,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x27,0x03,0x00,0x00, -0x25,0x03,0x00,0x00,0x3a,0x03,0x00,0x00,0x41,0x00,0x05,0x00, -0xc3,0x00,0x00,0x00,0x28,0x03,0x00,0x00,0xc0,0x00,0x00,0x00, -0x27,0x03,0x00,0x00,0x3d,0x00,0x04,0x00,0xba,0x00,0x00,0x00, -0x29,0x03,0x00,0x00,0x28,0x03,0x00,0x00,0x41,0x00,0x06,0x00, -0x91,0x01,0x00,0x00,0x2a,0x03,0x00,0x00,0x0f,0x03,0x00,0x00, -0x34,0x00,0x00,0x00,0x1c,0x03,0x00,0x00,0x3e,0x00,0x03,0x00, -0x2a,0x03,0x00,0x00,0x29,0x03,0x00,0x00,0xf9,0x00,0x02,0x00, -0x0b,0x03,0x00,0x00,0xf8,0x00,0x02,0x00,0x0b,0x03,0x00,0x00, -0xf9,0x00,0x02,0x00,0xf7,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0xf7,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x2c,0x03,0x00,0x00,0x3a,0x03,0x00,0x00,0xc6,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xf4,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0xf6,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0xef,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0xef,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x2e,0x03,0x00,0x00,0x38,0x03,0x00,0x00, -0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xec,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0xee,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0xd7,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0xd7,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x30,0x03,0x00,0x00, -0x36,0x03,0x00,0x00,0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xd4,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0xd6,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0xcf,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0xcf,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x32,0x03,0x00,0x00,0x35,0x03,0x00,0x00,0xc6,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xcc,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0xce,0x02,0x00,0x00,0xfd,0x00,0x01,0x00,0x38,0x00,0x01,0x00, - -}; -const uint64_t matmul_f16_f32_aligned_s_len = 12096; - -unsigned char matmul_f16_f32_aligned_s_fp32_data[] = { -0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, -0xf0,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, -0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x51,0x11,0x00,0x00, -0x0b,0x00,0x06,0x00,0x01,0x00,0x00,0x00,0x47,0x4c,0x53,0x4c, -0x2e,0x73,0x74,0x64,0x2e,0x34,0x35,0x30,0x00,0x00,0x00,0x00, -0x0e,0x00,0x03,0x00,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x0f,0x00,0x0f,0x00,0x05,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x6d,0x61,0x69,0x6e,0x00,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x3d,0x00,0x00,0x00,0x4c,0x00,0x00,0x00, -0xea,0x00,0x00,0x00,0xf9,0x00,0x00,0x00,0x4b,0x01,0x00,0x00, -0x59,0x01,0x00,0x00,0x51,0x02,0x00,0x00,0x9a,0x02,0x00,0x00, -0x10,0x00,0x06,0x00,0x04,0x00,0x00,0x00,0x11,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x0b,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x1c,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x10,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x08,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x03,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x10,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x14,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x18,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x07,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x10,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x24,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x0a,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x10,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x2c,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x30,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x0d,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0x47,0x00,0x03,0x00, -0x10,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x37,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x3d,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x1a,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x4c,0x00,0x00,0x00, -0x0b,0x00,0x00,0x00,0x1b,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x4f,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x53,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x60,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x62,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x07,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x6c,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x03,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x9d,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0xaf,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x05,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0xb2,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x08,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xf6,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x48,0x00,0x04,0x00, -0xf7,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0xf7,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, -0xf7,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0xf9,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0xf9,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x2c,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x2d,0x01,0x00,0x00,0x0b,0x00,0x00,0x00,0x19,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x56,0x01,0x00,0x00,0x06,0x00,0x00,0x00, -0x10,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0x57,0x01,0x00,0x00, -0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x57,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x57,0x01,0x00,0x00, -0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x59,0x01,0x00,0x00, -0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x59,0x01,0x00,0x00,0x21,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x51,0x02,0x00,0x00,0x0b,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x97,0x02,0x00,0x00, -0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00, -0x98,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x19,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x98,0x02,0x00,0x00,0x00,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, -0x98,0x02,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x9a,0x02,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x9a,0x02,0x00,0x00,0x21,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x13,0x00,0x02,0x00,0x02,0x00,0x00,0x00, -0x21,0x00,0x03,0x00,0x03,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x15,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x17,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x0a,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x0d,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x1e,0x00,0x10,0x00,0x10,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x11,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x11,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x15,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, -0x14,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x15,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x0a,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, -0x27,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0x2d,0x00,0x00,0x00,0x07,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x34,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x37,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00,0x3d,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x3e,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x0a,0x00,0x00,0x00,0x4c,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x4f,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x53,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x54,0x00,0x00,0x00,0x86,0x00,0x00,0x00, -0x37,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x58,0x00,0x00,0x00,0x86,0x00,0x00,0x00, -0x37,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x32,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x61,0x00,0x00,0x00, -0x86,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x60,0x00,0x00,0x00, -0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x62,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x63,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x61,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x67,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x61,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x6e,0x00,0x00,0x00, -0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x73,0x00,0x00,0x00, -0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x77,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, -0x7c,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0x87,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x8d,0x00,0x00,0x00, -0x03,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, -0x98,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x32,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x9d,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x9f,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xae,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xaf,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xb0,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0x53,0x00,0x00,0x00,0xaf,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xb1,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0x4f,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x32,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xb2,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xb3,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0xb1,0x00,0x00,0x00,0xb2,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xb4,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0xb3,0x00,0x00,0x00,0x60,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xb5,0x00,0x00,0x00, -0x86,0x00,0x00,0x00,0xb0,0x00,0x00,0x00,0xb4,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xb6,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0xae,0x00,0x00,0x00,0xb5,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xb7,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0xb6,0x00,0x00,0x00,0xb2,0x00,0x00,0x00, -0x14,0x00,0x02,0x00,0xb8,0x00,0x00,0x00,0x16,0x00,0x03,0x00, -0xba,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xbb,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xbc,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0xbb,0x00,0x00,0x00,0xb5,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xbd,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0xbc,0x00,0x00,0x00,0xb2,0x00,0x00,0x00,0x1c,0x00,0x04,0x00, -0xbe,0x00,0x00,0x00,0xba,0x00,0x00,0x00,0xbd,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0xbf,0x00,0x00,0x00,0x07,0x00,0x00,0x00, -0xbe,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0xba,0x00,0x00,0x00, -0xc2,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0xc3,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0xba,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0xc6,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xe6,0x00,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xe7,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x37,0x00,0x00,0x00, -0xe6,0x00,0x00,0x00,0x1c,0x00,0x04,0x00,0xe8,0x00,0x00,0x00, -0xba,0x00,0x00,0x00,0xe7,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0xe9,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0xe8,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0xe9,0x00,0x00,0x00,0xea,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xee,0x00,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x16,0x00,0x03,0x00,0xf4,0x00,0x00,0x00, -0x10,0x00,0x00,0x00,0x17,0x00,0x04,0x00,0xf5,0x00,0x00,0x00, -0xf4,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, -0xf6,0x00,0x00,0x00,0xf5,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, -0xf7,0x00,0x00,0x00,0xf6,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0xf8,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0xf7,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0xf8,0x00,0x00,0x00,0xf9,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0xfb,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0xf4,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0xff,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0xba,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x04,0x01,0x00,0x00, -0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x12,0x01,0x00,0x00, -0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x20,0x01,0x00,0x00, -0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x25,0x01,0x00,0x00, -0x03,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x2c,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x33,0x00,0x06,0x00, -0x09,0x00,0x00,0x00,0x2d,0x01,0x00,0x00,0x2c,0x01,0x00,0x00, -0x39,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x2e,0x01,0x00,0x00,0x51,0x00,0x00,0x00, -0x2d,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x2f,0x01,0x00,0x00,0x84,0x00,0x00,0x00, -0x2e,0x01,0x00,0x00,0x6d,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x30,0x01,0x00,0x00,0x86,0x00,0x00,0x00, -0x2f,0x01,0x00,0x00,0x6c,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x47,0x01,0x00,0x00,0x80,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x48,0x01,0x00,0x00,0x84,0x00,0x00,0x00, -0x9d,0x00,0x00,0x00,0x47,0x01,0x00,0x00,0x1c,0x00,0x04,0x00, -0x49,0x01,0x00,0x00,0xba,0x00,0x00,0x00,0x48,0x01,0x00,0x00, -0x20,0x00,0x04,0x00,0x4a,0x01,0x00,0x00,0x04,0x00,0x00,0x00, -0x49,0x01,0x00,0x00,0x3b,0x00,0x04,0x00,0x4a,0x01,0x00,0x00, -0x4b,0x01,0x00,0x00,0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x4f,0x01,0x00,0x00,0x80,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x17,0x00,0x04,0x00, -0x55,0x01,0x00,0x00,0xba,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x1d,0x00,0x03,0x00,0x56,0x01,0x00,0x00,0x55,0x01,0x00,0x00, -0x1e,0x00,0x03,0x00,0x57,0x01,0x00,0x00,0x56,0x01,0x00,0x00, -0x20,0x00,0x04,0x00,0x58,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, -0x57,0x01,0x00,0x00,0x3b,0x00,0x04,0x00,0x58,0x01,0x00,0x00, -0x59,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x5b,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0xba,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x62,0x01,0x00,0x00, -0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x6f,0x01,0x00,0x00, -0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x7c,0x01,0x00,0x00, -0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x88,0x01,0x00,0x00, -0x08,0x01,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x89,0x01,0x00,0x00,0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x6d,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x8c,0x01,0x00,0x00,0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x6d,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xa7,0x01,0x00,0x00,0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x1c,0x00,0x04,0x00,0xa8,0x01,0x00,0x00, -0xba,0x00,0x00,0x00,0xa7,0x01,0x00,0x00,0x20,0x00,0x04,0x00, -0xa9,0x01,0x00,0x00,0x07,0x00,0x00,0x00,0xa8,0x01,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xb9,0x01,0x00,0x00, -0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xd4,0x01,0x00,0x00, -0x84,0x00,0x00,0x00,0xb5,0x00,0x00,0x00,0xb2,0x00,0x00,0x00, -0x1c,0x00,0x04,0x00,0xd5,0x01,0x00,0x00,0xba,0x00,0x00,0x00, -0xd4,0x01,0x00,0x00,0x20,0x00,0x04,0x00,0xd6,0x01,0x00,0x00, -0x07,0x00,0x00,0x00,0xd5,0x01,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xdf,0x01,0x00,0x00,0x86,0x00,0x00,0x00, -0xaf,0x00,0x00,0x00,0xb5,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xe7,0x01,0x00,0x00,0x80,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x16,0x02,0x00,0x00,0x84,0x00,0x00,0x00, -0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0x49,0x02,0x00,0x00,0x0d,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00,0x51,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0x1d,0x00,0x03,0x00,0x97,0x02,0x00,0x00, -0xba,0x00,0x00,0x00,0x1e,0x00,0x03,0x00,0x98,0x02,0x00,0x00, -0x97,0x02,0x00,0x00,0x20,0x00,0x04,0x00,0x99,0x02,0x00,0x00, -0x0c,0x00,0x00,0x00,0x98,0x02,0x00,0x00,0x3b,0x00,0x04,0x00, -0x99,0x02,0x00,0x00,0x9a,0x02,0x00,0x00,0x0c,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x9f,0x02,0x00,0x00, -0x05,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xac,0x02,0x00,0x00,0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x36,0x00,0x05,0x00,0x02,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x03,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0x05,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0xbf,0x00,0x00,0x00,0xc0,0x00,0x00,0x00,0x07,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0xa9,0x01,0x00,0x00,0xaa,0x01,0x00,0x00, -0x07,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0xd6,0x01,0x00,0x00, -0xd7,0x01,0x00,0x00,0x07,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x0d,0x00,0x00,0x00,0x0e,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x0f,0x00,0x00,0x00,0x0e,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x14,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x17,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, -0x17,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x1e,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x17,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x22,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x22,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x24,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x27,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x29,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x2a,0x00,0x00,0x00,0x1e,0x00,0x00,0x00, -0x29,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x2d,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x2f,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x30,0x00,0x00,0x00,0x24,0x00,0x00,0x00,0x2f,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x32,0x00,0x00,0x00, -0x30,0x00,0x00,0x00,0x2a,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x35,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x34,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x36,0x00,0x00,0x00,0x35,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x38,0x00,0x00,0x00,0x36,0x00,0x00,0x00, -0x37,0x00,0x00,0x00,0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x3a,0x00,0x00,0x00,0x38,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3b,0x00,0x00,0x00, -0x3a,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x0d,0x00,0x00,0x00,0x3f,0x00,0x00,0x00,0x3d,0x00,0x00,0x00, -0x3e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0x3f,0x00,0x00,0x00,0x89,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x42,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x3b,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x47,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x3b,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00,0x49,0x00,0x00,0x00, -0x3d,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x4a,0x00,0x00,0x00,0x49,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00,0x4d,0x00,0x00,0x00, -0x4c,0x00,0x00,0x00,0x3e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x4d,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x50,0x00,0x00,0x00, -0x4e,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x89,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x55,0x00,0x00,0x00,0x50,0x00,0x00,0x00, -0x54,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x59,0x00,0x00,0x00,0x50,0x00,0x00,0x00,0x58,0x00,0x00,0x00, -0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x5d,0x00,0x00,0x00, -0x4e,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x89,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x64,0x00,0x00,0x00,0x5d,0x00,0x00,0x00, -0x63,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x68,0x00,0x00,0x00,0x5d,0x00,0x00,0x00,0x67,0x00,0x00,0x00, -0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x6f,0x00,0x00,0x00, -0x4e,0x00,0x00,0x00,0x6e,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x74,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, -0x73,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0x78,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x77,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x79,0x00,0x00,0x00, -0x78,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x7a,0x00,0x00,0x00,0x47,0x00,0x00,0x00,0x79,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x7d,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x7c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x7e,0x00,0x00,0x00,0x7d,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x80,0x00,0x00,0x00, -0x47,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x83,0x00,0x00,0x00,0x80,0x00,0x00,0x00, -0x79,0x00,0x00,0x00,0x0c,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x26,0x00,0x00,0x00, -0x7e,0x00,0x00,0x00,0x83,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x88,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x87,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x89,0x00,0x00,0x00,0x88,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x8a,0x00,0x00,0x00,0x32,0x00,0x00,0x00, -0x89,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x8c,0x00,0x00,0x00,0x42,0x00,0x00,0x00,0x37,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x8e,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x8d,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x8f,0x00,0x00,0x00,0x8e,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x90,0x00,0x00,0x00, -0x8c,0x00,0x00,0x00,0x8f,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x91,0x00,0x00,0x00,0x8a,0x00,0x00,0x00, -0x90,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x93,0x00,0x00,0x00,0x91,0x00,0x00,0x00,0x7a,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x94,0x00,0x00,0x00, -0x93,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x99,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x98,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x9a,0x00,0x00,0x00,0x99,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x9b,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, -0x9a,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x9e,0x00,0x00,0x00,0x4a,0x00,0x00,0x00,0x9d,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0xa0,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x9f,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xa1,0x00,0x00,0x00,0xa0,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa2,0x00,0x00,0x00, -0x9e,0x00,0x00,0x00,0xa1,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xa3,0x00,0x00,0x00,0x9b,0x00,0x00,0x00, -0xa2,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xa5,0x00,0x00,0x00,0xa3,0x00,0x00,0x00,0x7a,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa6,0x00,0x00,0x00, -0xa5,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xa8,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xa8,0x00,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xbe,0x02,0x00,0x00, -0x3e,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0xc7,0x00,0x00,0x00, -0xa9,0x00,0x00,0x00,0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00, -0xb9,0x00,0x00,0x00,0xbe,0x02,0x00,0x00,0xb7,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0xaa,0x00,0x00,0x00,0xa9,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xb9,0x00,0x00,0x00, -0xa9,0x00,0x00,0x00,0xaa,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xa9,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0xc3,0x00,0x00,0x00, -0xc4,0x00,0x00,0x00,0xc0,0x00,0x00,0x00,0xbe,0x02,0x00,0x00, -0x3e,0x00,0x03,0x00,0xc4,0x00,0x00,0x00,0xc2,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xc7,0x00,0x00,0x00, -0xbe,0x02,0x00,0x00,0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xa8,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xaa,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xca,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xca,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xd7,0x02,0x00,0x00,0xa6,0x00,0x00,0x00,0xaa,0x00,0x00,0x00, -0x8e,0x01,0x00,0x00,0xcd,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xd3,0x02,0x00,0x00,0x94,0x00,0x00,0x00, -0xaa,0x00,0x00,0x00,0x8b,0x01,0x00,0x00,0xcd,0x00,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xbf,0x02,0x00,0x00, -0x7a,0x00,0x00,0x00,0xaa,0x00,0x00,0x00,0x39,0x02,0x00,0x00, -0xcd,0x00,0x00,0x00,0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00, -0xd1,0x00,0x00,0x00,0xbf,0x02,0x00,0x00,0x84,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0xcc,0x00,0x00,0x00,0xcd,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xd1,0x00,0x00,0x00, -0xcb,0x00,0x00,0x00,0xcc,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xcb,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xd3,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xd3,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xcf,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, -0xcb,0x00,0x00,0x00,0x32,0x01,0x00,0x00,0xd4,0x00,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00,0xd9,0x00,0x00,0x00, -0xcf,0x02,0x00,0x00,0x37,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0xd5,0x00,0x00,0x00,0xd4,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xd9,0x00,0x00,0x00,0xd4,0x00,0x00,0x00, -0xd5,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xd4,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xde,0x00,0x00,0x00, -0x74,0x00,0x00,0x00,0xcf,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xe1,0x00,0x00,0x00,0xde,0x00,0x00,0x00, -0x8f,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xe2,0x00,0x00,0x00,0xe1,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xe3,0x00,0x00,0x00, -0xd3,0x02,0x00,0x00,0xe2,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xe5,0x00,0x00,0x00,0xe3,0x00,0x00,0x00, -0x6f,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xef,0x00,0x00,0x00,0xde,0x00,0x00,0x00,0xee,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xf1,0x00,0x00,0x00, -0x6f,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xf2,0x00,0x00,0x00,0xef,0x00,0x00,0x00, -0xf1,0x00,0x00,0x00,0x41,0x00,0x07,0x00,0xfb,0x00,0x00,0x00, -0xfc,0x00,0x00,0x00,0xf9,0x00,0x00,0x00,0x34,0x00,0x00,0x00, -0xe5,0x00,0x00,0x00,0x3e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0xf4,0x00,0x00,0x00,0xfd,0x00,0x00,0x00,0xfc,0x00,0x00,0x00, -0x73,0x00,0x04,0x00,0xba,0x00,0x00,0x00,0xfe,0x00,0x00,0x00, -0xfd,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0xff,0x00,0x00,0x00, -0x00,0x01,0x00,0x00,0xea,0x00,0x00,0x00,0xf2,0x00,0x00,0x00, -0x3e,0x00,0x03,0x00,0x00,0x01,0x00,0x00,0xfe,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x05,0x01,0x00,0x00, -0xde,0x00,0x00,0x00,0x04,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x08,0x01,0x00,0x00,0x05,0x01,0x00,0x00, -0xf1,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x09,0x01,0x00,0x00,0x08,0x01,0x00,0x00,0x39,0x00,0x00,0x00, -0x41,0x00,0x07,0x00,0xfb,0x00,0x00,0x00,0x0b,0x01,0x00,0x00, -0xf9,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0xe5,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0xf4,0x00,0x00,0x00, -0x0c,0x01,0x00,0x00,0x0b,0x01,0x00,0x00,0x73,0x00,0x04,0x00, -0xba,0x00,0x00,0x00,0x0d,0x01,0x00,0x00,0x0c,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0xff,0x00,0x00,0x00,0x0e,0x01,0x00,0x00, -0xea,0x00,0x00,0x00,0x09,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0x0e,0x01,0x00,0x00,0x0d,0x01,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x13,0x01,0x00,0x00,0xde,0x00,0x00,0x00, -0x12,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x16,0x01,0x00,0x00,0x13,0x01,0x00,0x00,0xf1,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x17,0x01,0x00,0x00, -0x16,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x41,0x00,0x07,0x00, -0xfb,0x00,0x00,0x00,0x19,0x01,0x00,0x00,0xf9,0x00,0x00,0x00, -0x34,0x00,0x00,0x00,0xe5,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0xf4,0x00,0x00,0x00,0x1a,0x01,0x00,0x00, -0x19,0x01,0x00,0x00,0x73,0x00,0x04,0x00,0xba,0x00,0x00,0x00, -0x1b,0x01,0x00,0x00,0x1a,0x01,0x00,0x00,0x41,0x00,0x05,0x00, -0xff,0x00,0x00,0x00,0x1c,0x01,0x00,0x00,0xea,0x00,0x00,0x00, -0x17,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x1c,0x01,0x00,0x00, -0x1b,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x21,0x01,0x00,0x00,0xde,0x00,0x00,0x00,0x20,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x24,0x01,0x00,0x00, -0x21,0x01,0x00,0x00,0xf1,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x26,0x01,0x00,0x00,0x24,0x01,0x00,0x00, -0x25,0x01,0x00,0x00,0x41,0x00,0x07,0x00,0xfb,0x00,0x00,0x00, -0x28,0x01,0x00,0x00,0xf9,0x00,0x00,0x00,0x34,0x00,0x00,0x00, -0xe5,0x00,0x00,0x00,0x25,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, -0xf4,0x00,0x00,0x00,0x29,0x01,0x00,0x00,0x28,0x01,0x00,0x00, -0x73,0x00,0x04,0x00,0xba,0x00,0x00,0x00,0x2a,0x01,0x00,0x00, -0x29,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0xff,0x00,0x00,0x00, -0x2b,0x01,0x00,0x00,0xea,0x00,0x00,0x00,0x26,0x01,0x00,0x00, -0x3e,0x00,0x03,0x00,0x2b,0x01,0x00,0x00,0x2a,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x32,0x01,0x00,0x00, -0xcf,0x02,0x00,0x00,0x30,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0xd3,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xd5,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x34,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x34,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xd0,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0xd5,0x00,0x00,0x00, -0x87,0x01,0x00,0x00,0x35,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb8,0x00,0x00,0x00,0x3a,0x01,0x00,0x00,0xd0,0x02,0x00,0x00, -0x9d,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x36,0x01,0x00,0x00, -0x35,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x3a,0x01,0x00,0x00,0x35,0x01,0x00,0x00,0x36,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x35,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x3f,0x01,0x00,0x00,0x74,0x00,0x00,0x00, -0xd0,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x42,0x01,0x00,0x00,0x3f,0x01,0x00,0x00,0xa1,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x43,0x01,0x00,0x00, -0x42,0x01,0x00,0x00,0x6d,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x44,0x01,0x00,0x00,0xd7,0x02,0x00,0x00, -0x43,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x46,0x01,0x00,0x00,0x44,0x01,0x00,0x00,0x6f,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x50,0x01,0x00,0x00, -0x3f,0x01,0x00,0x00,0x4f,0x01,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x52,0x01,0x00,0x00,0x6f,0x00,0x00,0x00, -0x6d,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x53,0x01,0x00,0x00,0x50,0x01,0x00,0x00,0x52,0x01,0x00,0x00, -0x41,0x00,0x07,0x00,0x5b,0x01,0x00,0x00,0x5c,0x01,0x00,0x00, -0x59,0x01,0x00,0x00,0x34,0x00,0x00,0x00,0x46,0x01,0x00,0x00, -0x3e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0xba,0x00,0x00,0x00, -0x5d,0x01,0x00,0x00,0x5c,0x01,0x00,0x00,0x41,0x00,0x05,0x00, -0xff,0x00,0x00,0x00,0x5e,0x01,0x00,0x00,0x4b,0x01,0x00,0x00, -0x53,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x5e,0x01,0x00,0x00, -0x5d,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x63,0x01,0x00,0x00,0x3f,0x01,0x00,0x00,0x62,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x66,0x01,0x00,0x00, -0x63,0x01,0x00,0x00,0x52,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x67,0x01,0x00,0x00,0x66,0x01,0x00,0x00, -0x39,0x00,0x00,0x00,0x41,0x00,0x07,0x00,0x5b,0x01,0x00,0x00, -0x69,0x01,0x00,0x00,0x59,0x01,0x00,0x00,0x34,0x00,0x00,0x00, -0x46,0x01,0x00,0x00,0x39,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0xba,0x00,0x00,0x00,0x6a,0x01,0x00,0x00,0x69,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0xff,0x00,0x00,0x00,0x6b,0x01,0x00,0x00, -0x4b,0x01,0x00,0x00,0x67,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0x6b,0x01,0x00,0x00,0x6a,0x01,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x70,0x01,0x00,0x00,0x3f,0x01,0x00,0x00, -0x6f,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x73,0x01,0x00,0x00,0x70,0x01,0x00,0x00,0x52,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x74,0x01,0x00,0x00, -0x73,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x41,0x00,0x07,0x00, -0x5b,0x01,0x00,0x00,0x76,0x01,0x00,0x00,0x59,0x01,0x00,0x00, -0x34,0x00,0x00,0x00,0x46,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0xba,0x00,0x00,0x00,0x77,0x01,0x00,0x00, -0x76,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0xff,0x00,0x00,0x00, -0x78,0x01,0x00,0x00,0x4b,0x01,0x00,0x00,0x74,0x01,0x00,0x00, -0x3e,0x00,0x03,0x00,0x78,0x01,0x00,0x00,0x77,0x01,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x7d,0x01,0x00,0x00, -0x3f,0x01,0x00,0x00,0x7c,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x80,0x01,0x00,0x00,0x7d,0x01,0x00,0x00, -0x52,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x81,0x01,0x00,0x00,0x80,0x01,0x00,0x00,0x25,0x01,0x00,0x00, -0x41,0x00,0x07,0x00,0x5b,0x01,0x00,0x00,0x83,0x01,0x00,0x00, -0x59,0x01,0x00,0x00,0x34,0x00,0x00,0x00,0x46,0x01,0x00,0x00, -0x25,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0xba,0x00,0x00,0x00, -0x84,0x01,0x00,0x00,0x83,0x01,0x00,0x00,0x41,0x00,0x05,0x00, -0xff,0x00,0x00,0x00,0x85,0x01,0x00,0x00,0x4b,0x01,0x00,0x00, -0x81,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x85,0x01,0x00,0x00, -0x84,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x87,0x01,0x00,0x00,0xd0,0x02,0x00,0x00,0x30,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0x34,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x36,0x01,0x00,0x00,0xe0,0x00,0x04,0x00,0x0c,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x88,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x8b,0x01,0x00,0x00,0xd3,0x02,0x00,0x00, -0x89,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x8e,0x01,0x00,0x00,0xd7,0x02,0x00,0x00,0x8c,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0x90,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x90,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xd9,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0x36,0x01,0x00,0x00, -0x37,0x02,0x00,0x00,0x93,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb8,0x00,0x00,0x00,0x96,0x01,0x00,0x00,0xd9,0x02,0x00,0x00, -0x6c,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x92,0x01,0x00,0x00, -0x93,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x96,0x01,0x00,0x00,0x91,0x01,0x00,0x00,0x92,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x91,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0x98,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x98,0x01,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xdd,0x02,0x00,0x00, -0x3e,0x00,0x00,0x00,0x91,0x01,0x00,0x00,0xc3,0x01,0x00,0x00, -0x9b,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00, -0x9e,0x01,0x00,0x00,0xdd,0x02,0x00,0x00,0x60,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0x9a,0x01,0x00,0x00,0x9b,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x9e,0x01,0x00,0x00, -0x99,0x01,0x00,0x00,0x9a,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x99,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xa0,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xa0,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xef,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, -0x99,0x01,0x00,0x00,0xc1,0x01,0x00,0x00,0xa1,0x01,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00,0xa6,0x01,0x00,0x00, -0xef,0x02,0x00,0x00,0x62,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0xa2,0x01,0x00,0x00,0xa1,0x01,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xa6,0x01,0x00,0x00,0xa1,0x01,0x00,0x00, -0xa2,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xa1,0x01,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xac,0x01,0x00,0x00, -0xdd,0x02,0x00,0x00,0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xae,0x01,0x00,0x00,0xac,0x01,0x00,0x00, -0xef,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xb0,0x01,0x00,0x00,0x55,0x00,0x00,0x00,0x53,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xb2,0x01,0x00,0x00, -0xdd,0x02,0x00,0x00,0x61,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xb3,0x01,0x00,0x00,0xb0,0x01,0x00,0x00, -0xb2,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xb5,0x01,0x00,0x00,0x64,0x00,0x00,0x00,0x62,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xb6,0x01,0x00,0x00, -0xb3,0x01,0x00,0x00,0xb5,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xb8,0x01,0x00,0x00,0xb6,0x01,0x00,0x00, -0xef,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xba,0x01,0x00,0x00,0xb8,0x01,0x00,0x00,0xb9,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xbc,0x01,0x00,0x00, -0xba,0x01,0x00,0x00,0xd9,0x02,0x00,0x00,0x41,0x00,0x05,0x00, -0xff,0x00,0x00,0x00,0xbd,0x01,0x00,0x00,0xea,0x00,0x00,0x00, -0xbc,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0xba,0x00,0x00,0x00, -0xbe,0x01,0x00,0x00,0xbd,0x01,0x00,0x00,0x41,0x00,0x05,0x00, -0xc3,0x00,0x00,0x00,0xbf,0x01,0x00,0x00,0xaa,0x01,0x00,0x00, -0xae,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0xbf,0x01,0x00,0x00, -0xbe,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xc1,0x01,0x00,0x00,0xef,0x02,0x00,0x00,0xc6,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xa0,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xa2,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x9b,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x9b,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xc3,0x01,0x00,0x00,0xdd,0x02,0x00,0x00, -0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x98,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x9a,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0xc5,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xc5,0x01,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xde,0x02,0x00,0x00, -0x3e,0x00,0x00,0x00,0x9a,0x01,0x00,0x00,0xf1,0x01,0x00,0x00, -0xc8,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00, -0xcb,0x01,0x00,0x00,0xde,0x02,0x00,0x00,0xb5,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0xc7,0x01,0x00,0x00,0xc8,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xcb,0x01,0x00,0x00, -0xc6,0x01,0x00,0x00,0xc7,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xc6,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xcd,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xcd,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xec,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, -0xc6,0x01,0x00,0x00,0xef,0x01,0x00,0x00,0xce,0x01,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00,0xd3,0x01,0x00,0x00, -0xec,0x02,0x00,0x00,0xb2,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0xcf,0x01,0x00,0x00,0xce,0x01,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xd3,0x01,0x00,0x00,0xce,0x01,0x00,0x00, -0xcf,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xce,0x01,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xd9,0x01,0x00,0x00, -0xde,0x02,0x00,0x00,0xb2,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xdb,0x01,0x00,0x00,0xd9,0x01,0x00,0x00, -0xec,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xdd,0x01,0x00,0x00,0x59,0x00,0x00,0x00,0xaf,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xe0,0x01,0x00,0x00, -0xde,0x02,0x00,0x00,0xdf,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xe1,0x01,0x00,0x00,0xdd,0x01,0x00,0x00, -0xe0,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xe3,0x01,0x00,0x00,0x68,0x00,0x00,0x00,0xb2,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xe4,0x01,0x00,0x00, -0xe1,0x01,0x00,0x00,0xe3,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xe6,0x01,0x00,0x00,0xe4,0x01,0x00,0x00, -0xec,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xe8,0x01,0x00,0x00,0xe6,0x01,0x00,0x00,0xe7,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xea,0x01,0x00,0x00, -0xe8,0x01,0x00,0x00,0xd9,0x02,0x00,0x00,0x41,0x00,0x05,0x00, -0xff,0x00,0x00,0x00,0xeb,0x01,0x00,0x00,0x4b,0x01,0x00,0x00, -0xea,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0xba,0x00,0x00,0x00, -0xec,0x01,0x00,0x00,0xeb,0x01,0x00,0x00,0x41,0x00,0x05,0x00, -0xc3,0x00,0x00,0x00,0xed,0x01,0x00,0x00,0xd7,0x01,0x00,0x00, -0xdb,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0xed,0x01,0x00,0x00, -0xec,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xef,0x01,0x00,0x00,0xec,0x02,0x00,0x00,0xc6,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xcd,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xcf,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xc8,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xc8,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xf1,0x01,0x00,0x00,0xde,0x02,0x00,0x00, -0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xc5,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xc7,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0xf3,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xf3,0x01,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xdf,0x02,0x00,0x00, -0x3e,0x00,0x00,0x00,0xc7,0x01,0x00,0x00,0x35,0x02,0x00,0x00, -0xf6,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00, -0xf9,0x01,0x00,0x00,0xdf,0x02,0x00,0x00,0xb5,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0xf5,0x01,0x00,0x00,0xf6,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xf9,0x01,0x00,0x00, -0xf4,0x01,0x00,0x00,0xf5,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xf4,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xfb,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xfb,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xe3,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, -0xf4,0x01,0x00,0x00,0x33,0x02,0x00,0x00,0xfe,0x01,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00,0x01,0x02,0x00,0x00, -0xe3,0x02,0x00,0x00,0x60,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0xfd,0x01,0x00,0x00,0xfe,0x01,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x01,0x02,0x00,0x00,0xfc,0x01,0x00,0x00, -0xfd,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xfc,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0x03,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x03,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xe5,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0xfc,0x01,0x00,0x00, -0x31,0x02,0x00,0x00,0x06,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb8,0x00,0x00,0x00,0x09,0x02,0x00,0x00,0xe5,0x02,0x00,0x00, -0xb2,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x05,0x02,0x00,0x00, -0x06,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x09,0x02,0x00,0x00,0x04,0x02,0x00,0x00,0x05,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x04,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x0b,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x0b,0x02,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xe7,0x02,0x00,0x00, -0x3e,0x00,0x00,0x00,0x04,0x02,0x00,0x00,0x2f,0x02,0x00,0x00, -0x0c,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00, -0x11,0x02,0x00,0x00,0xe7,0x02,0x00,0x00,0x62,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0x0d,0x02,0x00,0x00,0x0c,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x11,0x02,0x00,0x00, -0x0c,0x02,0x00,0x00,0x0d,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x0c,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x13,0x02,0x00,0x00,0xdf,0x02,0x00,0x00,0xb2,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x15,0x02,0x00,0x00, -0x13,0x02,0x00,0x00,0xe5,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x17,0x02,0x00,0x00,0x15,0x02,0x00,0x00, -0x16,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x19,0x02,0x00,0x00,0xe3,0x02,0x00,0x00,0x62,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x1a,0x02,0x00,0x00, -0x17,0x02,0x00,0x00,0x19,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x1c,0x02,0x00,0x00,0x1a,0x02,0x00,0x00, -0xe7,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x20,0x02,0x00,0x00,0x19,0x02,0x00,0x00,0xe7,0x02,0x00,0x00, -0x41,0x00,0x05,0x00,0xc3,0x00,0x00,0x00,0x21,0x02,0x00,0x00, -0xaa,0x01,0x00,0x00,0x20,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, -0xba,0x00,0x00,0x00,0x22,0x02,0x00,0x00,0x21,0x02,0x00,0x00, -0x41,0x00,0x05,0x00,0xc3,0x00,0x00,0x00,0x27,0x02,0x00,0x00, -0xd7,0x01,0x00,0x00,0x15,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, -0xba,0x00,0x00,0x00,0x28,0x02,0x00,0x00,0x27,0x02,0x00,0x00, -0x41,0x00,0x05,0x00,0xc3,0x00,0x00,0x00,0x2a,0x02,0x00,0x00, -0xc0,0x00,0x00,0x00,0x1c,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, -0xba,0x00,0x00,0x00,0x2b,0x02,0x00,0x00,0x2a,0x02,0x00,0x00, -0x0c,0x00,0x08,0x00,0xba,0x00,0x00,0x00,0x2c,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x22,0x02,0x00,0x00, -0x28,0x02,0x00,0x00,0x2b,0x02,0x00,0x00,0x3e,0x00,0x03,0x00, -0x2a,0x02,0x00,0x00,0x2c,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x2f,0x02,0x00,0x00,0xe7,0x02,0x00,0x00, -0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x0b,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x0d,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x06,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x06,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x31,0x02,0x00,0x00, -0xe5,0x02,0x00,0x00,0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x03,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x05,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0xfe,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xfe,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x33,0x02,0x00,0x00,0xe3,0x02,0x00,0x00,0xc6,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xfb,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xfd,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xf6,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xf6,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x35,0x02,0x00,0x00,0xdf,0x02,0x00,0x00, -0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xf3,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xf5,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0x93,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x93,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x37,0x02,0x00,0x00, -0xd9,0x02,0x00,0x00,0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x90,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x92,0x01,0x00,0x00, -0xe0,0x00,0x04,0x00,0x0c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x88,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xcd,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xcd,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x39,0x02,0x00,0x00,0xbf,0x02,0x00,0x00, -0x6c,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xca,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xcc,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x3e,0x02,0x00,0x00,0x55,0x00,0x00,0x00, -0x53,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x3f,0x02,0x00,0x00,0x8c,0x00,0x00,0x00,0x3e,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x44,0x02,0x00,0x00, -0x59,0x00,0x00,0x00,0xaf,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x45,0x02,0x00,0x00,0x9e,0x00,0x00,0x00, -0x44,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0x4a,0x02,0x00,0x00,0x12,0x00,0x00,0x00,0x49,0x02,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x4b,0x02,0x00,0x00, -0x4a,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x4c,0x02,0x00,0x00,0x0f,0x00,0x00,0x00,0x4b,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x50,0x02,0x00,0x00, -0x47,0x00,0x00,0x00,0x4b,0x02,0x00,0x00,0x41,0x00,0x05,0x00, -0x0d,0x00,0x00,0x00,0x52,0x02,0x00,0x00,0x51,0x02,0x00,0x00, -0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x53,0x02,0x00,0x00,0x52,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x54,0x02,0x00,0x00,0x50,0x02,0x00,0x00, -0x53,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x55,0x02,0x00,0x00,0x4c,0x02,0x00,0x00,0x54,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0x57,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x57,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xc0,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0xcc,0x00,0x00,0x00, -0xbd,0x02,0x00,0x00,0x5a,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb8,0x00,0x00,0x00,0x5d,0x02,0x00,0x00,0xc0,0x02,0x00,0x00, -0xb5,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x59,0x02,0x00,0x00, -0x5a,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x5d,0x02,0x00,0x00,0x58,0x02,0x00,0x00,0x59,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x58,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x5f,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x5f,0x02,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xc1,0x02,0x00,0x00, -0x3e,0x00,0x00,0x00,0x58,0x02,0x00,0x00,0xbb,0x02,0x00,0x00, -0x62,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00, -0x65,0x02,0x00,0x00,0xc1,0x02,0x00,0x00,0x60,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0x61,0x02,0x00,0x00,0x62,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x65,0x02,0x00,0x00, -0x60,0x02,0x00,0x00,0x61,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x60,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x69,0x02,0x00,0x00,0xc1,0x02,0x00,0x00,0x61,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x6a,0x02,0x00,0x00, -0x3f,0x02,0x00,0x00,0x69,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x6c,0x02,0x00,0x00,0x64,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x6d,0x02,0x00,0x00,0x6a,0x02,0x00,0x00,0x6c,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x71,0x02,0x00,0x00, -0xc0,0x02,0x00,0x00,0xdf,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x72,0x02,0x00,0x00,0x45,0x02,0x00,0x00, -0x71,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x74,0x02,0x00,0x00,0x68,0x00,0x00,0x00,0xb2,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x75,0x02,0x00,0x00, -0x72,0x02,0x00,0x00,0x74,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x77,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x77,0x02,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xc3,0x02,0x00,0x00, -0x3e,0x00,0x00,0x00,0x60,0x02,0x00,0x00,0xb9,0x02,0x00,0x00, -0x7a,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00, -0x7d,0x02,0x00,0x00,0xc3,0x02,0x00,0x00,0xb2,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0x79,0x02,0x00,0x00,0x7a,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x7d,0x02,0x00,0x00, -0x78,0x02,0x00,0x00,0x79,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x78,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x7f,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x7f,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xc5,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, -0x78,0x02,0x00,0x00,0xb7,0x02,0x00,0x00,0x82,0x02,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00,0x85,0x02,0x00,0x00, -0xc5,0x02,0x00,0x00,0x62,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0x81,0x02,0x00,0x00,0x82,0x02,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x85,0x02,0x00,0x00,0x80,0x02,0x00,0x00, -0x81,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x80,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x88,0x02,0x00,0x00, -0x6d,0x02,0x00,0x00,0xc5,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb8,0x00,0x00,0x00,0x8b,0x02,0x00,0x00,0x88,0x02,0x00,0x00, -0x36,0x00,0x00,0x00,0xf7,0x00,0x03,0x00,0x8d,0x02,0x00,0x00, -0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x8b,0x02,0x00,0x00, -0x8c,0x02,0x00,0x00,0x8d,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x8c,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x90,0x02,0x00,0x00,0x75,0x02,0x00,0x00,0xc3,0x02,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x91,0x02,0x00,0x00, -0x12,0x00,0x00,0x00,0xc6,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x92,0x02,0x00,0x00,0x91,0x02,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00,0x93,0x02,0x00,0x00, -0x90,0x02,0x00,0x00,0x92,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x8d,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x8d,0x02,0x00,0x00, -0xf5,0x00,0x07,0x00,0xb8,0x00,0x00,0x00,0x94,0x02,0x00,0x00, -0x8b,0x02,0x00,0x00,0x80,0x02,0x00,0x00,0x93,0x02,0x00,0x00, -0x8c,0x02,0x00,0x00,0xf7,0x00,0x03,0x00,0x96,0x02,0x00,0x00, -0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x94,0x02,0x00,0x00, -0x95,0x02,0x00,0x00,0x96,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x95,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x9e,0x02,0x00,0x00,0x75,0x02,0x00,0x00,0xc3,0x02,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0xa0,0x02,0x00,0x00, -0x12,0x00,0x00,0x00,0x9f,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xa1,0x02,0x00,0x00,0xa0,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa2,0x02,0x00,0x00, -0x9e,0x02,0x00,0x00,0xa1,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xa3,0x02,0x00,0x00,0x55,0x02,0x00,0x00, -0xa2,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xa5,0x02,0x00,0x00,0xa3,0x02,0x00,0x00,0x6d,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa7,0x02,0x00,0x00, -0xa5,0x02,0x00,0x00,0xc5,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xa9,0x02,0x00,0x00,0xc0,0x02,0x00,0x00, -0xb2,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xab,0x02,0x00,0x00,0xa9,0x02,0x00,0x00,0xc3,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xad,0x02,0x00,0x00, -0xab,0x02,0x00,0x00,0xac,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xaf,0x02,0x00,0x00,0xc1,0x02,0x00,0x00, -0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xb0,0x02,0x00,0x00,0xad,0x02,0x00,0x00,0xaf,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xb2,0x02,0x00,0x00, -0xb0,0x02,0x00,0x00,0xc5,0x02,0x00,0x00,0x41,0x00,0x05,0x00, -0xc3,0x00,0x00,0x00,0xb3,0x02,0x00,0x00,0xc0,0x00,0x00,0x00, -0xb2,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0xba,0x00,0x00,0x00, -0xb4,0x02,0x00,0x00,0xb3,0x02,0x00,0x00,0x41,0x00,0x06,0x00, -0x5b,0x01,0x00,0x00,0xb5,0x02,0x00,0x00,0x9a,0x02,0x00,0x00, -0x34,0x00,0x00,0x00,0xa7,0x02,0x00,0x00,0x3e,0x00,0x03,0x00, -0xb5,0x02,0x00,0x00,0xb4,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x96,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x96,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0x82,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x82,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xb7,0x02,0x00,0x00,0xc5,0x02,0x00,0x00,0xc6,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x7f,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x81,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x7a,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x7a,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xb9,0x02,0x00,0x00,0xc3,0x02,0x00,0x00, -0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x77,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x79,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x62,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x62,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xbb,0x02,0x00,0x00, -0xc1,0x02,0x00,0x00,0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x5f,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x61,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0x5a,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x5a,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xbd,0x02,0x00,0x00,0xc0,0x02,0x00,0x00,0xc6,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x57,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x59,0x02,0x00,0x00,0xfd,0x00,0x01,0x00,0x38,0x00,0x01,0x00, - -}; -const uint64_t matmul_f16_f32_aligned_s_fp32_len = 10464; - -unsigned char matmul_f16_f32_l_data[] = { -0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, -0xcd,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, -0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x09,0x00,0x00,0x00, -0x11,0x00,0x02,0x00,0x51,0x11,0x00,0x00,0x0b,0x00,0x06,0x00, -0x01,0x00,0x00,0x00,0x47,0x4c,0x53,0x4c,0x2e,0x73,0x74,0x64, -0x2e,0x34,0x35,0x30,0x00,0x00,0x00,0x00,0x0e,0x00,0x03,0x00, -0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x0f,0x00,0x0f,0x00, -0x05,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x6d,0x61,0x69,0x6e, -0x00,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x3d,0x00,0x00,0x00,0x4c,0x00,0x00,0x00,0xf1,0x00,0x00,0x00, -0xfc,0x00,0x00,0x00,0x3c,0x01,0x00,0x00,0x47,0x01,0x00,0x00, -0x2e,0x02,0x00,0x00,0x77,0x02,0x00,0x00,0x10,0x00,0x06,0x00, -0x04,0x00,0x00,0x00,0x11,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x0b,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x1c,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x10,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x08,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x03,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x10,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x10,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x05,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x14,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x10,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x1c,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x08,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x24,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x10,0x00,0x00,0x00,0x0a,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x28,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x0b,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x2c,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x30,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x10,0x00,0x00,0x00,0x0d,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x34,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x10,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x37,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x3d,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x1a,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x4c,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x1b,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x4f,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x53,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x60,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x62,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x6c,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x03,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x9c,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xae,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0xb1,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x08,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0xf9,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0xfa,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0xfa,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0xfa,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xfc,0x00,0x00,0x00, -0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0xfc,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x16,0x01,0x00,0x00,0x01,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x17,0x01,0x00,0x00, -0x0b,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x44,0x01,0x00,0x00,0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x48,0x00,0x04,0x00,0x45,0x01,0x00,0x00,0x00,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x45,0x01,0x00,0x00, -0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x03,0x00,0x45,0x01,0x00,0x00,0x02,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x47,0x01,0x00,0x00,0x22,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x47,0x01,0x00,0x00, -0x21,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x2e,0x02,0x00,0x00,0x0b,0x00,0x00,0x00,0x18,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x74,0x02,0x00,0x00,0x06,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0x75,0x02,0x00,0x00, -0x00,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x75,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x75,0x02,0x00,0x00, -0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x77,0x02,0x00,0x00, -0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x77,0x02,0x00,0x00,0x21,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x13,0x00,0x02,0x00,0x02,0x00,0x00,0x00,0x21,0x00,0x03,0x00, -0x03,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x15,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x17,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x03,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x0a,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x0a,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x0d,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x1e,0x00,0x10,0x00, -0x10,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x11,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x10,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x11,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x15,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x14,0x00,0x00,0x00, -0x08,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x15,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x0a,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x27,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, -0x2d,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x37,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x0a,0x00,0x00,0x00,0x3d,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x3e,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00, -0x4c,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x53,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x54,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x37,0x00,0x00,0x00, -0x53,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x58,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x37,0x00,0x00,0x00, -0x53,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x60,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x61,0x00,0x00,0x00,0x86,0x00,0x00,0x00, -0x53,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x32,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x63,0x00,0x00,0x00, -0x86,0x00,0x00,0x00,0x61,0x00,0x00,0x00,0x62,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x67,0x00,0x00,0x00, -0x86,0x00,0x00,0x00,0x61,0x00,0x00,0x00,0x62,0x00,0x00,0x00, -0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x10,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x6d,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x72,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, -0x76,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0x7b,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x86,0x00,0x00,0x00, -0x0b,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, -0x8c,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0x97,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x9c,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, -0x9e,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xad,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x32,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xae,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xaf,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0xae,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xb0,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x62,0x00,0x00,0x00, -0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xb1,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xb2,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0xb0,0x00,0x00,0x00, -0xb1,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xb3,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0xb2,0x00,0x00,0x00, -0x60,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xb4,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0xaf,0x00,0x00,0x00, -0xb3,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xb5,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0xad,0x00,0x00,0x00, -0xb4,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xb6,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0xb5,0x00,0x00,0x00, -0xb1,0x00,0x00,0x00,0x14,0x00,0x02,0x00,0xb7,0x00,0x00,0x00, -0x16,0x00,0x03,0x00,0xb9,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xba,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xbb,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0xba,0x00,0x00,0x00,0xb4,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xbc,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0xbb,0x00,0x00,0x00,0xb1,0x00,0x00,0x00, -0x1c,0x00,0x04,0x00,0xbd,0x00,0x00,0x00,0xb9,0x00,0x00,0x00, -0xbc,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0xbe,0x00,0x00,0x00, -0x07,0x00,0x00,0x00,0xbd,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0xb9,0x00,0x00,0x00,0xc1,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0xc2,0x00,0x00,0x00,0x07,0x00,0x00,0x00, -0xb9,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, -0xc5,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x16,0x00,0x03,0x00, -0xec,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xed,0x00,0x00,0x00,0x80,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xee,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0x37,0x00,0x00,0x00,0xed,0x00,0x00,0x00,0x1c,0x00,0x04,0x00, -0xef,0x00,0x00,0x00,0xec,0x00,0x00,0x00,0xee,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0xf0,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0xef,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0xf0,0x00,0x00,0x00, -0xf1,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xf5,0x00,0x00,0x00,0x80,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, -0xf9,0x00,0x00,0x00,0xec,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, -0xfa,0x00,0x00,0x00,0xf9,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0xfb,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0xfa,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0xfb,0x00,0x00,0x00,0xfc,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x07,0x01,0x00,0x00, -0x0c,0x00,0x00,0x00,0xec,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x0a,0x01,0x00,0x00,0x04,0x00,0x00,0x00,0xec,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x10,0x01,0x00,0x00, -0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0xec,0x00,0x00,0x00,0x14,0x01,0x00,0x00, -0x00,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x16,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x33,0x00,0x06,0x00, -0x09,0x00,0x00,0x00,0x17,0x01,0x00,0x00,0x16,0x01,0x00,0x00, -0x39,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x18,0x01,0x00,0x00,0x51,0x00,0x00,0x00, -0x17,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x19,0x01,0x00,0x00,0x84,0x00,0x00,0x00, -0x18,0x01,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x1a,0x01,0x00,0x00,0x86,0x00,0x00,0x00, -0x19,0x01,0x00,0x00,0x6c,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x38,0x01,0x00,0x00,0x80,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x39,0x01,0x00,0x00,0x84,0x00,0x00,0x00, -0x9c,0x00,0x00,0x00,0x38,0x01,0x00,0x00,0x1c,0x00,0x04,0x00, -0x3a,0x01,0x00,0x00,0xec,0x00,0x00,0x00,0x39,0x01,0x00,0x00, -0x20,0x00,0x04,0x00,0x3b,0x01,0x00,0x00,0x04,0x00,0x00,0x00, -0x3a,0x01,0x00,0x00,0x3b,0x00,0x04,0x00,0x3b,0x01,0x00,0x00, -0x3c,0x01,0x00,0x00,0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x40,0x01,0x00,0x00,0x80,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, -0x44,0x01,0x00,0x00,0xb9,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, -0x45,0x01,0x00,0x00,0x44,0x01,0x00,0x00,0x20,0x00,0x04,0x00, -0x46,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x45,0x01,0x00,0x00, -0x3b,0x00,0x04,0x00,0x46,0x01,0x00,0x00,0x47,0x01,0x00,0x00, -0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x52,0x01,0x00,0x00, -0x0c,0x00,0x00,0x00,0xb9,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x5b,0x01,0x00,0x00,0x80,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x62,0x01,0x00,0x00,0x08,0x01,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x63,0x01,0x00,0x00, -0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x66,0x01,0x00,0x00, -0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x81,0x01,0x00,0x00, -0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00, -0x1c,0x00,0x04,0x00,0x82,0x01,0x00,0x00,0xec,0x00,0x00,0x00, -0x81,0x01,0x00,0x00,0x20,0x00,0x04,0x00,0x83,0x01,0x00,0x00, -0x07,0x00,0x00,0x00,0x82,0x01,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x93,0x01,0x00,0x00,0x80,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x99,0x01,0x00,0x00,0x07,0x00,0x00,0x00,0xec,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xaf,0x01,0x00,0x00, -0x84,0x00,0x00,0x00,0xb4,0x00,0x00,0x00,0xb1,0x00,0x00,0x00, -0x1c,0x00,0x04,0x00,0xb0,0x01,0x00,0x00,0xec,0x00,0x00,0x00, -0xaf,0x01,0x00,0x00,0x20,0x00,0x04,0x00,0xb1,0x01,0x00,0x00, -0x07,0x00,0x00,0x00,0xb0,0x01,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xba,0x01,0x00,0x00,0x86,0x00,0x00,0x00, -0xae,0x00,0x00,0x00,0xb4,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xc2,0x01,0x00,0x00,0x80,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xf1,0x01,0x00,0x00,0x84,0x00,0x00,0x00, -0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0x26,0x02,0x00,0x00,0x0d,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00,0x2e,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0x1d,0x00,0x03,0x00,0x74,0x02,0x00,0x00, -0xb9,0x00,0x00,0x00,0x1e,0x00,0x03,0x00,0x75,0x02,0x00,0x00, -0x74,0x02,0x00,0x00,0x20,0x00,0x04,0x00,0x76,0x02,0x00,0x00, -0x0c,0x00,0x00,0x00,0x75,0x02,0x00,0x00,0x3b,0x00,0x04,0x00, -0x76,0x02,0x00,0x00,0x77,0x02,0x00,0x00,0x0c,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x7c,0x02,0x00,0x00, -0x05,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x89,0x02,0x00,0x00,0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x36,0x00,0x05,0x00,0x02,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x03,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0x05,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0xbe,0x00,0x00,0x00,0xbf,0x00,0x00,0x00,0x07,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x83,0x01,0x00,0x00,0x84,0x01,0x00,0x00, -0x07,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0xb1,0x01,0x00,0x00, -0xb2,0x01,0x00,0x00,0x07,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x0d,0x00,0x00,0x00,0x0e,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x0f,0x00,0x00,0x00,0x0e,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x14,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x17,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, -0x17,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x1e,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x17,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x22,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x22,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x24,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x27,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x29,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x2a,0x00,0x00,0x00,0x1e,0x00,0x00,0x00, -0x29,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x2d,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x2f,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x30,0x00,0x00,0x00,0x24,0x00,0x00,0x00,0x2f,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x32,0x00,0x00,0x00, -0x30,0x00,0x00,0x00,0x2a,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x35,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x34,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x36,0x00,0x00,0x00,0x35,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x38,0x00,0x00,0x00,0x36,0x00,0x00,0x00, -0x37,0x00,0x00,0x00,0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x3a,0x00,0x00,0x00,0x38,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3b,0x00,0x00,0x00, -0x3a,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x0d,0x00,0x00,0x00,0x3f,0x00,0x00,0x00,0x3d,0x00,0x00,0x00, -0x3e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0x3f,0x00,0x00,0x00,0x89,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x42,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x3b,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x47,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x3b,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00,0x49,0x00,0x00,0x00, -0x3d,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x4a,0x00,0x00,0x00,0x49,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00,0x4d,0x00,0x00,0x00, -0x4c,0x00,0x00,0x00,0x3e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x4d,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x50,0x00,0x00,0x00, -0x4e,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x89,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x55,0x00,0x00,0x00,0x50,0x00,0x00,0x00, -0x54,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x59,0x00,0x00,0x00,0x50,0x00,0x00,0x00,0x58,0x00,0x00,0x00, -0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x5d,0x00,0x00,0x00, -0x4e,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x89,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x64,0x00,0x00,0x00,0x5d,0x00,0x00,0x00, -0x63,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x68,0x00,0x00,0x00,0x5d,0x00,0x00,0x00,0x67,0x00,0x00,0x00, -0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x6e,0x00,0x00,0x00, -0x4e,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x73,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, -0x72,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0x77,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x76,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x78,0x00,0x00,0x00, -0x77,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x79,0x00,0x00,0x00,0x47,0x00,0x00,0x00,0x78,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x7c,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x7b,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x7d,0x00,0x00,0x00,0x7c,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x7f,0x00,0x00,0x00, -0x47,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x82,0x00,0x00,0x00,0x7f,0x00,0x00,0x00, -0x78,0x00,0x00,0x00,0x0c,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0x83,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x26,0x00,0x00,0x00, -0x7d,0x00,0x00,0x00,0x82,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x87,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x86,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x88,0x00,0x00,0x00,0x87,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x89,0x00,0x00,0x00,0x32,0x00,0x00,0x00, -0x88,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x8b,0x00,0x00,0x00,0x42,0x00,0x00,0x00,0x37,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x8d,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x8c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x8e,0x00,0x00,0x00,0x8d,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8f,0x00,0x00,0x00, -0x8b,0x00,0x00,0x00,0x8e,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x90,0x00,0x00,0x00,0x89,0x00,0x00,0x00, -0x8f,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x92,0x00,0x00,0x00,0x90,0x00,0x00,0x00,0x79,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x93,0x00,0x00,0x00, -0x92,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x98,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x97,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x99,0x00,0x00,0x00,0x98,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x9a,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, -0x99,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x9d,0x00,0x00,0x00,0x4a,0x00,0x00,0x00,0x9c,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x9f,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x9e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xa0,0x00,0x00,0x00,0x9f,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa1,0x00,0x00,0x00, -0x9d,0x00,0x00,0x00,0xa0,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xa2,0x00,0x00,0x00,0x9a,0x00,0x00,0x00, -0xa1,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xa4,0x00,0x00,0x00,0xa2,0x00,0x00,0x00,0x79,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa5,0x00,0x00,0x00, -0xa4,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xa7,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xa7,0x00,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x9b,0x02,0x00,0x00, -0x3e,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0xc6,0x00,0x00,0x00, -0xa8,0x00,0x00,0x00,0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00, -0xb8,0x00,0x00,0x00,0x9b,0x02,0x00,0x00,0xb6,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0xa9,0x00,0x00,0x00,0xa8,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xb8,0x00,0x00,0x00, -0xa8,0x00,0x00,0x00,0xa9,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xa8,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, -0xc3,0x00,0x00,0x00,0xbf,0x00,0x00,0x00,0x9b,0x02,0x00,0x00, -0x3e,0x00,0x03,0x00,0xc3,0x00,0x00,0x00,0xc1,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xc6,0x00,0x00,0x00, -0x9b,0x02,0x00,0x00,0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xa7,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xa9,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xc9,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xc9,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xb4,0x02,0x00,0x00,0xa5,0x00,0x00,0x00,0xa9,0x00,0x00,0x00, -0x68,0x01,0x00,0x00,0xcc,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xb0,0x02,0x00,0x00,0x93,0x00,0x00,0x00, -0xa9,0x00,0x00,0x00,0x65,0x01,0x00,0x00,0xcc,0x00,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x9c,0x02,0x00,0x00, -0x79,0x00,0x00,0x00,0xa9,0x00,0x00,0x00,0x16,0x02,0x00,0x00, -0xcc,0x00,0x00,0x00,0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00, -0xd0,0x00,0x00,0x00,0x9c,0x02,0x00,0x00,0x83,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0xcb,0x00,0x00,0x00,0xcc,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xd0,0x00,0x00,0x00, -0xca,0x00,0x00,0x00,0xcb,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xca,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xd2,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xd2,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xac,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, -0xca,0x00,0x00,0x00,0x1c,0x01,0x00,0x00,0xd5,0x00,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00,0xd8,0x00,0x00,0x00, -0xac,0x02,0x00,0x00,0x37,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0xd4,0x00,0x00,0x00,0xd5,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xd8,0x00,0x00,0x00,0xd3,0x00,0x00,0x00, -0xd4,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xd3,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xdc,0x00,0x00,0x00, -0x8b,0x00,0x00,0x00,0x73,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xde,0x00,0x00,0x00,0xdc,0x00,0x00,0x00, -0xac,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00, -0xe1,0x00,0x00,0x00,0xde,0x00,0x00,0x00,0x36,0x00,0x00,0x00, -0xf7,0x00,0x03,0x00,0xe3,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xe1,0x00,0x00,0x00,0xe2,0x00,0x00,0x00, -0xe3,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xe2,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xe6,0x00,0x00,0x00, -0x9c,0x02,0x00,0x00,0x6e,0x00,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb7,0x00,0x00,0x00,0xe8,0x00,0x00,0x00,0xe6,0x00,0x00,0x00, -0x83,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xe3,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xe3,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, -0xb7,0x00,0x00,0x00,0xe9,0x00,0x00,0x00,0xe1,0x00,0x00,0x00, -0xd3,0x00,0x00,0x00,0xe8,0x00,0x00,0x00,0xe2,0x00,0x00,0x00, -0xf7,0x00,0x03,0x00,0xeb,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xe9,0x00,0x00,0x00,0xea,0x00,0x00,0x00, -0x0c,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xea,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xf4,0x00,0x00,0x00, -0x73,0x00,0x00,0x00,0xac,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xf6,0x00,0x00,0x00,0xf4,0x00,0x00,0x00, -0xf5,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xf8,0x00,0x00,0x00,0xf6,0x00,0x00,0x00,0x6e,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x03,0x01,0x00,0x00, -0xf4,0x00,0x00,0x00,0x8e,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x04,0x01,0x00,0x00,0xb0,0x02,0x00,0x00, -0x03,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x06,0x01,0x00,0x00,0x04,0x01,0x00,0x00,0x6e,0x00,0x00,0x00, -0x41,0x00,0x06,0x00,0x07,0x01,0x00,0x00,0x08,0x01,0x00,0x00, -0xfc,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0x06,0x01,0x00,0x00, -0x3d,0x00,0x04,0x00,0xec,0x00,0x00,0x00,0x09,0x01,0x00,0x00, -0x08,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0x0a,0x01,0x00,0x00, -0x0b,0x01,0x00,0x00,0xf1,0x00,0x00,0x00,0xf8,0x00,0x00,0x00, -0x3e,0x00,0x03,0x00,0x0b,0x01,0x00,0x00,0x09,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0xeb,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x0c,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x0f,0x01,0x00,0x00,0x73,0x00,0x00,0x00,0xac,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x11,0x01,0x00,0x00, -0x0f,0x01,0x00,0x00,0x10,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x13,0x01,0x00,0x00,0x11,0x01,0x00,0x00, -0x6e,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x0a,0x01,0x00,0x00, -0x15,0x01,0x00,0x00,0xf1,0x00,0x00,0x00,0x13,0x01,0x00,0x00, -0x3e,0x00,0x03,0x00,0x15,0x01,0x00,0x00,0x14,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0xeb,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xeb,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xd5,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xd5,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x1c,0x01,0x00,0x00,0xac,0x02,0x00,0x00, -0x1a,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xd2,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xd4,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x1e,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x1e,0x01,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xad,0x02,0x00,0x00, -0x3e,0x00,0x00,0x00,0xd4,0x00,0x00,0x00,0x61,0x01,0x00,0x00, -0x21,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00, -0x24,0x01,0x00,0x00,0xad,0x02,0x00,0x00,0x9c,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0x20,0x01,0x00,0x00,0x21,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x24,0x01,0x00,0x00, -0x1f,0x01,0x00,0x00,0x20,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x1f,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x28,0x01,0x00,0x00,0x9d,0x00,0x00,0x00,0x73,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x2a,0x01,0x00,0x00, -0x28,0x01,0x00,0x00,0xad,0x02,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x2b,0x01,0x00,0x00,0x12,0x00,0x00,0x00, -0xc5,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x2c,0x01,0x00,0x00,0x2b,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb7,0x00,0x00,0x00,0x2d,0x01,0x00,0x00,0x2a,0x01,0x00,0x00, -0x2c,0x01,0x00,0x00,0xf7,0x00,0x03,0x00,0x2f,0x01,0x00,0x00, -0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x2d,0x01,0x00,0x00, -0x2e,0x01,0x00,0x00,0x2f,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x2e,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x32,0x01,0x00,0x00,0x9c,0x02,0x00,0x00,0x6e,0x00,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00,0x34,0x01,0x00,0x00, -0x32,0x01,0x00,0x00,0x83,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x2f,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x2f,0x01,0x00,0x00, -0xf5,0x00,0x07,0x00,0xb7,0x00,0x00,0x00,0x35,0x01,0x00,0x00, -0x2d,0x01,0x00,0x00,0x1f,0x01,0x00,0x00,0x34,0x01,0x00,0x00, -0x2e,0x01,0x00,0x00,0xf7,0x00,0x03,0x00,0x37,0x01,0x00,0x00, -0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x35,0x01,0x00,0x00, -0x36,0x01,0x00,0x00,0x57,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x36,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x3f,0x01,0x00,0x00,0x73,0x00,0x00,0x00,0xad,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x41,0x01,0x00,0x00, -0x3f,0x01,0x00,0x00,0x40,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x43,0x01,0x00,0x00,0x41,0x01,0x00,0x00, -0x6e,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x4e,0x01,0x00,0x00,0x3f,0x01,0x00,0x00,0xa0,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x4f,0x01,0x00,0x00, -0xb4,0x02,0x00,0x00,0x4e,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x51,0x01,0x00,0x00,0x4f,0x01,0x00,0x00, -0x6e,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x52,0x01,0x00,0x00, -0x53,0x01,0x00,0x00,0x47,0x01,0x00,0x00,0x34,0x00,0x00,0x00, -0x51,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0xb9,0x00,0x00,0x00, -0x54,0x01,0x00,0x00,0x53,0x01,0x00,0x00,0x73,0x00,0x04,0x00, -0xec,0x00,0x00,0x00,0x55,0x01,0x00,0x00,0x54,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0x0a,0x01,0x00,0x00,0x56,0x01,0x00,0x00, -0x3c,0x01,0x00,0x00,0x43,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0x56,0x01,0x00,0x00,0x55,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0x37,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x57,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x5a,0x01,0x00,0x00, -0x73,0x00,0x00,0x00,0xad,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x5c,0x01,0x00,0x00,0x5a,0x01,0x00,0x00, -0x5b,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x5e,0x01,0x00,0x00,0x5c,0x01,0x00,0x00,0x6e,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x0a,0x01,0x00,0x00,0x5f,0x01,0x00,0x00, -0x3c,0x01,0x00,0x00,0x5e,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0x5f,0x01,0x00,0x00,0x14,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0x37,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x37,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0x21,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x21,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x61,0x01,0x00,0x00,0xad,0x02,0x00,0x00,0x1a,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0x1e,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x20,0x01,0x00,0x00,0xe0,0x00,0x04,0x00,0x0c,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x62,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x65,0x01,0x00,0x00,0xb0,0x02,0x00,0x00, -0x63,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x68,0x01,0x00,0x00,0xb4,0x02,0x00,0x00,0x66,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0x6a,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x6a,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xb6,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0x20,0x01,0x00,0x00, -0x14,0x02,0x00,0x00,0x6d,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb7,0x00,0x00,0x00,0x70,0x01,0x00,0x00,0xb6,0x02,0x00,0x00, -0x6c,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x6c,0x01,0x00,0x00, -0x6d,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x70,0x01,0x00,0x00,0x6b,0x01,0x00,0x00,0x6c,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x6b,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0x72,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x72,0x01,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xba,0x02,0x00,0x00, -0x3e,0x00,0x00,0x00,0x6b,0x01,0x00,0x00,0x9e,0x01,0x00,0x00, -0x75,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00, -0x78,0x01,0x00,0x00,0xba,0x02,0x00,0x00,0x60,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0x74,0x01,0x00,0x00,0x75,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x78,0x01,0x00,0x00, -0x73,0x01,0x00,0x00,0x74,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x73,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x7a,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x7a,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xcc,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, -0x73,0x01,0x00,0x00,0x9c,0x01,0x00,0x00,0x7b,0x01,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00,0x80,0x01,0x00,0x00, -0xcc,0x02,0x00,0x00,0x62,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0x7c,0x01,0x00,0x00,0x7b,0x01,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x80,0x01,0x00,0x00,0x7b,0x01,0x00,0x00, -0x7c,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x7b,0x01,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x86,0x01,0x00,0x00, -0xba,0x02,0x00,0x00,0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x88,0x01,0x00,0x00,0x86,0x01,0x00,0x00, -0xcc,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x8a,0x01,0x00,0x00,0x55,0x00,0x00,0x00,0x53,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8c,0x01,0x00,0x00, -0xba,0x02,0x00,0x00,0x61,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x8d,0x01,0x00,0x00,0x8a,0x01,0x00,0x00, -0x8c,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x8f,0x01,0x00,0x00,0x64,0x00,0x00,0x00,0x62,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x90,0x01,0x00,0x00, -0x8d,0x01,0x00,0x00,0x8f,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x92,0x01,0x00,0x00,0x90,0x01,0x00,0x00, -0xcc,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x94,0x01,0x00,0x00,0x92,0x01,0x00,0x00,0x93,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x96,0x01,0x00,0x00, -0x94,0x01,0x00,0x00,0xb6,0x02,0x00,0x00,0x41,0x00,0x05,0x00, -0x0a,0x01,0x00,0x00,0x97,0x01,0x00,0x00,0xf1,0x00,0x00,0x00, -0x96,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0xec,0x00,0x00,0x00, -0x98,0x01,0x00,0x00,0x97,0x01,0x00,0x00,0x41,0x00,0x05,0x00, -0x99,0x01,0x00,0x00,0x9a,0x01,0x00,0x00,0x84,0x01,0x00,0x00, -0x88,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x9a,0x01,0x00,0x00, -0x98,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x9c,0x01,0x00,0x00,0xcc,0x02,0x00,0x00,0xc5,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x7a,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x7c,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x75,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x75,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x9e,0x01,0x00,0x00,0xba,0x02,0x00,0x00, -0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x72,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x74,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0xa0,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xa0,0x01,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xbb,0x02,0x00,0x00, -0x3e,0x00,0x00,0x00,0x74,0x01,0x00,0x00,0xcc,0x01,0x00,0x00, -0xa3,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00, -0xa6,0x01,0x00,0x00,0xbb,0x02,0x00,0x00,0xb4,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0xa2,0x01,0x00,0x00,0xa3,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xa6,0x01,0x00,0x00, -0xa1,0x01,0x00,0x00,0xa2,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xa1,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xa8,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xa8,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xc9,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, -0xa1,0x01,0x00,0x00,0xca,0x01,0x00,0x00,0xa9,0x01,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00,0xae,0x01,0x00,0x00, -0xc9,0x02,0x00,0x00,0xb1,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0xaa,0x01,0x00,0x00,0xa9,0x01,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xae,0x01,0x00,0x00,0xa9,0x01,0x00,0x00, -0xaa,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xa9,0x01,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xb4,0x01,0x00,0x00, -0xbb,0x02,0x00,0x00,0xb1,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xb6,0x01,0x00,0x00,0xb4,0x01,0x00,0x00, -0xc9,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xb8,0x01,0x00,0x00,0x59,0x00,0x00,0x00,0xae,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xbb,0x01,0x00,0x00, -0xbb,0x02,0x00,0x00,0xba,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xbc,0x01,0x00,0x00,0xb8,0x01,0x00,0x00, -0xbb,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xbe,0x01,0x00,0x00,0x68,0x00,0x00,0x00,0xb1,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xbf,0x01,0x00,0x00, -0xbc,0x01,0x00,0x00,0xbe,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xc1,0x01,0x00,0x00,0xbf,0x01,0x00,0x00, -0xc9,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xc3,0x01,0x00,0x00,0xc1,0x01,0x00,0x00,0xc2,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xc5,0x01,0x00,0x00, -0xc3,0x01,0x00,0x00,0xb6,0x02,0x00,0x00,0x41,0x00,0x05,0x00, -0x0a,0x01,0x00,0x00,0xc6,0x01,0x00,0x00,0x3c,0x01,0x00,0x00, -0xc5,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0xec,0x00,0x00,0x00, -0xc7,0x01,0x00,0x00,0xc6,0x01,0x00,0x00,0x41,0x00,0x05,0x00, -0x99,0x01,0x00,0x00,0xc8,0x01,0x00,0x00,0xb2,0x01,0x00,0x00, -0xb6,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0xc8,0x01,0x00,0x00, -0xc7,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xca,0x01,0x00,0x00,0xc9,0x02,0x00,0x00,0xc5,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xa8,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xaa,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xa3,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xa3,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xcc,0x01,0x00,0x00,0xbb,0x02,0x00,0x00, -0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xa0,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xa2,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0xce,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xce,0x01,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xbc,0x02,0x00,0x00, -0x3e,0x00,0x00,0x00,0xa2,0x01,0x00,0x00,0x12,0x02,0x00,0x00, -0xd1,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00, -0xd4,0x01,0x00,0x00,0xbc,0x02,0x00,0x00,0xb4,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0xd0,0x01,0x00,0x00,0xd1,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xd4,0x01,0x00,0x00, -0xcf,0x01,0x00,0x00,0xd0,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xcf,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xd6,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xd6,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xc0,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, -0xcf,0x01,0x00,0x00,0x10,0x02,0x00,0x00,0xd9,0x01,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00,0xdc,0x01,0x00,0x00, -0xc0,0x02,0x00,0x00,0x60,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0xd8,0x01,0x00,0x00,0xd9,0x01,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xdc,0x01,0x00,0x00,0xd7,0x01,0x00,0x00, -0xd8,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xd7,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0xde,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xde,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xc2,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0xd7,0x01,0x00,0x00, -0x0e,0x02,0x00,0x00,0xe1,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb7,0x00,0x00,0x00,0xe4,0x01,0x00,0x00,0xc2,0x02,0x00,0x00, -0xb1,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xe0,0x01,0x00,0x00, -0xe1,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0xe4,0x01,0x00,0x00,0xdf,0x01,0x00,0x00,0xe0,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xdf,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0xe6,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xe6,0x01,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xc4,0x02,0x00,0x00, -0x3e,0x00,0x00,0x00,0xdf,0x01,0x00,0x00,0x0c,0x02,0x00,0x00, -0xe7,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00, -0xec,0x01,0x00,0x00,0xc4,0x02,0x00,0x00,0x62,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0xe8,0x01,0x00,0x00,0xe7,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xec,0x01,0x00,0x00, -0xe7,0x01,0x00,0x00,0xe8,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xe7,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xee,0x01,0x00,0x00,0xbc,0x02,0x00,0x00,0xb1,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xf0,0x01,0x00,0x00, -0xee,0x01,0x00,0x00,0xc2,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xf2,0x01,0x00,0x00,0xf0,0x01,0x00,0x00, -0xf1,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xf4,0x01,0x00,0x00,0xc0,0x02,0x00,0x00,0x62,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xf5,0x01,0x00,0x00, -0xf2,0x01,0x00,0x00,0xf4,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xf7,0x01,0x00,0x00,0xf5,0x01,0x00,0x00, -0xc4,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xfb,0x01,0x00,0x00,0xf4,0x01,0x00,0x00,0xc4,0x02,0x00,0x00, -0x41,0x00,0x05,0x00,0x99,0x01,0x00,0x00,0xfc,0x01,0x00,0x00, -0x84,0x01,0x00,0x00,0xfb,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, -0xec,0x00,0x00,0x00,0xfd,0x01,0x00,0x00,0xfc,0x01,0x00,0x00, -0x73,0x00,0x04,0x00,0xb9,0x00,0x00,0x00,0xfe,0x01,0x00,0x00, -0xfd,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0x99,0x01,0x00,0x00, -0x03,0x02,0x00,0x00,0xb2,0x01,0x00,0x00,0xf0,0x01,0x00,0x00, -0x3d,0x00,0x04,0x00,0xec,0x00,0x00,0x00,0x04,0x02,0x00,0x00, -0x03,0x02,0x00,0x00,0x73,0x00,0x04,0x00,0xb9,0x00,0x00,0x00, -0x05,0x02,0x00,0x00,0x04,0x02,0x00,0x00,0x41,0x00,0x05,0x00, -0xc2,0x00,0x00,0x00,0x07,0x02,0x00,0x00,0xbf,0x00,0x00,0x00, -0xf7,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0xb9,0x00,0x00,0x00, -0x08,0x02,0x00,0x00,0x07,0x02,0x00,0x00,0x0c,0x00,0x08,0x00, -0xb9,0x00,0x00,0x00,0x09,0x02,0x00,0x00,0x01,0x00,0x00,0x00, -0x32,0x00,0x00,0x00,0xfe,0x01,0x00,0x00,0x05,0x02,0x00,0x00, -0x08,0x02,0x00,0x00,0x3e,0x00,0x03,0x00,0x07,0x02,0x00,0x00, -0x09,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x0c,0x02,0x00,0x00,0xc4,0x02,0x00,0x00,0xc5,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xe6,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xe8,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xe1,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xe1,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x0e,0x02,0x00,0x00,0xc2,0x02,0x00,0x00, -0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xde,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xe0,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0xd9,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xd9,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x10,0x02,0x00,0x00, -0xc0,0x02,0x00,0x00,0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xd6,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xd8,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0xd1,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xd1,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x12,0x02,0x00,0x00,0xbc,0x02,0x00,0x00,0xc5,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xce,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xd0,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x6d,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x6d,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x14,0x02,0x00,0x00,0xb6,0x02,0x00,0x00, -0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x6a,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x6c,0x01,0x00,0x00,0xe0,0x00,0x04,0x00, -0x0c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x62,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0xcc,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xcc,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x16,0x02,0x00,0x00,0x9c,0x02,0x00,0x00,0x6c,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xc9,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xcb,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x1b,0x02,0x00,0x00,0x55,0x00,0x00,0x00,0x53,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x1c,0x02,0x00,0x00, -0x8b,0x00,0x00,0x00,0x1b,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x21,0x02,0x00,0x00,0x59,0x00,0x00,0x00, -0xae,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x22,0x02,0x00,0x00,0x9d,0x00,0x00,0x00,0x21,0x02,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x27,0x02,0x00,0x00, -0x12,0x00,0x00,0x00,0x26,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x28,0x02,0x00,0x00,0x27,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x29,0x02,0x00,0x00, -0x0f,0x00,0x00,0x00,0x28,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x2d,0x02,0x00,0x00,0x47,0x00,0x00,0x00, -0x28,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00, -0x2f,0x02,0x00,0x00,0x2e,0x02,0x00,0x00,0x0c,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x30,0x02,0x00,0x00, -0x2f,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x31,0x02,0x00,0x00,0x2d,0x02,0x00,0x00,0x30,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x32,0x02,0x00,0x00, -0x29,0x02,0x00,0x00,0x31,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x34,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x34,0x02,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x9d,0x02,0x00,0x00, -0x3e,0x00,0x00,0x00,0xcb,0x00,0x00,0x00,0x9a,0x02,0x00,0x00, -0x37,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00, -0x3a,0x02,0x00,0x00,0x9d,0x02,0x00,0x00,0xb4,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0x36,0x02,0x00,0x00,0x37,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x3a,0x02,0x00,0x00, -0x35,0x02,0x00,0x00,0x36,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x35,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x3c,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x3c,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0x9e,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, -0x35,0x02,0x00,0x00,0x98,0x02,0x00,0x00,0x3f,0x02,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00,0x42,0x02,0x00,0x00, -0x9e,0x02,0x00,0x00,0x60,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0x3e,0x02,0x00,0x00,0x3f,0x02,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x42,0x02,0x00,0x00,0x3d,0x02,0x00,0x00, -0x3e,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x3d,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x46,0x02,0x00,0x00, -0x9e,0x02,0x00,0x00,0x61,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x47,0x02,0x00,0x00,0x1c,0x02,0x00,0x00, -0x46,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x49,0x02,0x00,0x00,0x64,0x00,0x00,0x00,0x62,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x4a,0x02,0x00,0x00, -0x47,0x02,0x00,0x00,0x49,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x4e,0x02,0x00,0x00,0x9d,0x02,0x00,0x00, -0xba,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x4f,0x02,0x00,0x00,0x22,0x02,0x00,0x00,0x4e,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x51,0x02,0x00,0x00, -0x68,0x00,0x00,0x00,0xb1,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x52,0x02,0x00,0x00,0x4f,0x02,0x00,0x00, -0x51,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x54,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x54,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xa0,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, -0x3d,0x02,0x00,0x00,0x96,0x02,0x00,0x00,0x57,0x02,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00,0x5a,0x02,0x00,0x00, -0xa0,0x02,0x00,0x00,0xb1,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0x56,0x02,0x00,0x00,0x57,0x02,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x5a,0x02,0x00,0x00,0x55,0x02,0x00,0x00, -0x56,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x55,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0x5c,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x5c,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xa2,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0x55,0x02,0x00,0x00, -0x94,0x02,0x00,0x00,0x5f,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb7,0x00,0x00,0x00,0x62,0x02,0x00,0x00,0xa2,0x02,0x00,0x00, -0x62,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x5e,0x02,0x00,0x00, -0x5f,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x62,0x02,0x00,0x00,0x5d,0x02,0x00,0x00,0x5e,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x5d,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x65,0x02,0x00,0x00,0x4a,0x02,0x00,0x00, -0xa2,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00, -0x68,0x02,0x00,0x00,0x65,0x02,0x00,0x00,0x36,0x00,0x00,0x00, -0xf7,0x00,0x03,0x00,0x6a,0x02,0x00,0x00,0x00,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x68,0x02,0x00,0x00,0x69,0x02,0x00,0x00, -0x6a,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x69,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x6d,0x02,0x00,0x00, -0x52,0x02,0x00,0x00,0xa0,0x02,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x6e,0x02,0x00,0x00,0x12,0x00,0x00,0x00, -0xc5,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x6f,0x02,0x00,0x00,0x6e,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb7,0x00,0x00,0x00,0x70,0x02,0x00,0x00,0x6d,0x02,0x00,0x00, -0x6f,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x6a,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x6a,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, -0xb7,0x00,0x00,0x00,0x71,0x02,0x00,0x00,0x68,0x02,0x00,0x00, -0x5d,0x02,0x00,0x00,0x70,0x02,0x00,0x00,0x69,0x02,0x00,0x00, -0xf7,0x00,0x03,0x00,0x73,0x02,0x00,0x00,0x00,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x71,0x02,0x00,0x00,0x72,0x02,0x00,0x00, -0x73,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x72,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x7b,0x02,0x00,0x00, -0x52,0x02,0x00,0x00,0xa0,0x02,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x7d,0x02,0x00,0x00,0x12,0x00,0x00,0x00, -0x7c,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x7e,0x02,0x00,0x00,0x7d,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x7f,0x02,0x00,0x00,0x7b,0x02,0x00,0x00, -0x7e,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x80,0x02,0x00,0x00,0x32,0x02,0x00,0x00,0x7f,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x82,0x02,0x00,0x00, -0x80,0x02,0x00,0x00,0x4a,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x84,0x02,0x00,0x00,0x82,0x02,0x00,0x00, -0xa2,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x86,0x02,0x00,0x00,0x9d,0x02,0x00,0x00,0xb1,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x88,0x02,0x00,0x00, -0x86,0x02,0x00,0x00,0xa0,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x8a,0x02,0x00,0x00,0x88,0x02,0x00,0x00, -0x89,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x8c,0x02,0x00,0x00,0x9e,0x02,0x00,0x00,0x62,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8d,0x02,0x00,0x00, -0x8a,0x02,0x00,0x00,0x8c,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x8f,0x02,0x00,0x00,0x8d,0x02,0x00,0x00, -0xa2,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, -0x90,0x02,0x00,0x00,0xbf,0x00,0x00,0x00,0x8f,0x02,0x00,0x00, -0x3d,0x00,0x04,0x00,0xb9,0x00,0x00,0x00,0x91,0x02,0x00,0x00, -0x90,0x02,0x00,0x00,0x41,0x00,0x06,0x00,0x52,0x01,0x00,0x00, -0x92,0x02,0x00,0x00,0x77,0x02,0x00,0x00,0x34,0x00,0x00,0x00, -0x84,0x02,0x00,0x00,0x3e,0x00,0x03,0x00,0x92,0x02,0x00,0x00, -0x91,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x73,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x73,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x5f,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x5f,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x94,0x02,0x00,0x00, -0xa2,0x02,0x00,0x00,0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x5c,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x5e,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0x57,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x57,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x96,0x02,0x00,0x00,0xa0,0x02,0x00,0x00,0xc5,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x54,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x56,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x3f,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x3f,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x98,0x02,0x00,0x00,0x9e,0x02,0x00,0x00, -0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x3c,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x3e,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x37,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x37,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9a,0x02,0x00,0x00, -0x9d,0x02,0x00,0x00,0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x34,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x36,0x02,0x00,0x00, -0xfd,0x00,0x01,0x00,0x38,0x00,0x01,0x00, -}; -const uint64_t matmul_f16_f32_l_len = 10172; - -unsigned char matmul_f16_f32_l_fp32_data[] = { -0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, -0xc9,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, -0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x51,0x11,0x00,0x00, -0x0b,0x00,0x06,0x00,0x01,0x00,0x00,0x00,0x47,0x4c,0x53,0x4c, -0x2e,0x73,0x74,0x64,0x2e,0x34,0x35,0x30,0x00,0x00,0x00,0x00, -0x0e,0x00,0x03,0x00,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x0f,0x00,0x0f,0x00,0x05,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x6d,0x61,0x69,0x6e,0x00,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x3d,0x00,0x00,0x00,0x4c,0x00,0x00,0x00, -0xf0,0x00,0x00,0x00,0xfc,0x00,0x00,0x00,0x3c,0x01,0x00,0x00, -0x47,0x01,0x00,0x00,0x2a,0x02,0x00,0x00,0x73,0x02,0x00,0x00, -0x10,0x00,0x06,0x00,0x04,0x00,0x00,0x00,0x11,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x0b,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x1c,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x10,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x08,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x03,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x10,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x14,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x18,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x07,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x10,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x24,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x0a,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x10,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x2c,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x30,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x0d,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0x47,0x00,0x03,0x00, -0x10,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x37,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x3d,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x1a,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x4c,0x00,0x00,0x00, -0x0b,0x00,0x00,0x00,0x1b,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x4f,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x53,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x60,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x62,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x07,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x6c,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x03,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x9c,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0xae,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x05,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0xb1,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x08,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xf9,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x48,0x00,0x04,0x00, -0xfa,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0xfa,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, -0xfa,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0xfc,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0xfc,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x16,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x17,0x01,0x00,0x00,0x0b,0x00,0x00,0x00,0x19,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x44,0x01,0x00,0x00,0x06,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0x45,0x01,0x00,0x00, -0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x45,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x45,0x01,0x00,0x00, -0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x47,0x01,0x00,0x00, -0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x47,0x01,0x00,0x00,0x21,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x2a,0x02,0x00,0x00,0x0b,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x70,0x02,0x00,0x00, -0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00, -0x71,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x19,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x71,0x02,0x00,0x00,0x00,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, -0x71,0x02,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x73,0x02,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x73,0x02,0x00,0x00,0x21,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x13,0x00,0x02,0x00,0x02,0x00,0x00,0x00, -0x21,0x00,0x03,0x00,0x03,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x15,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x17,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x0a,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x0d,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x1e,0x00,0x10,0x00,0x10,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x11,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x11,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x15,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, -0x14,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x15,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x0a,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, -0x27,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0x2d,0x00,0x00,0x00,0x07,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x34,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x37,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00,0x3d,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x3e,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x0a,0x00,0x00,0x00,0x4c,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x4f,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x53,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x54,0x00,0x00,0x00,0x86,0x00,0x00,0x00, -0x37,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x58,0x00,0x00,0x00,0x86,0x00,0x00,0x00, -0x37,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x32,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x61,0x00,0x00,0x00, -0x86,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x60,0x00,0x00,0x00, -0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x62,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x63,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x61,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x67,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x61,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x86,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x72,0x00,0x00,0x00,0x86,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0x76,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x7b,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, -0x86,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0x8c,0x00,0x00,0x00,0x03,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x97,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x9c,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0x9e,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xad,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00, -0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xae,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xaf,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x53,0x00,0x00,0x00, -0xae,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xb0,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x4f,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xb1,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xb2,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0xb0,0x00,0x00,0x00,0xb1,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xb3,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0xb2,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xb4,0x00,0x00,0x00,0x86,0x00,0x00,0x00, -0xaf,0x00,0x00,0x00,0xb3,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xb5,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0xad,0x00,0x00,0x00,0xb4,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xb6,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0xb5,0x00,0x00,0x00,0xb1,0x00,0x00,0x00,0x14,0x00,0x02,0x00, -0xb7,0x00,0x00,0x00,0x16,0x00,0x03,0x00,0xb9,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xba,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xbb,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0xba,0x00,0x00,0x00, -0xb4,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xbc,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0xbb,0x00,0x00,0x00, -0xb1,0x00,0x00,0x00,0x1c,0x00,0x04,0x00,0xbd,0x00,0x00,0x00, -0xb9,0x00,0x00,0x00,0xbc,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0xbe,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0xbd,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0xb9,0x00,0x00,0x00,0xc1,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0xc2,0x00,0x00,0x00, -0x07,0x00,0x00,0x00,0xb9,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0xc5,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xec,0x00,0x00,0x00, -0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xed,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0xec,0x00,0x00,0x00, -0x1c,0x00,0x04,0x00,0xee,0x00,0x00,0x00,0xb9,0x00,0x00,0x00, -0xed,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0xef,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0xee,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0xef,0x00,0x00,0x00,0xf0,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xf4,0x00,0x00,0x00, -0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x16,0x00,0x03,0x00,0xf8,0x00,0x00,0x00,0x10,0x00,0x00,0x00, -0x1d,0x00,0x03,0x00,0xf9,0x00,0x00,0x00,0xf8,0x00,0x00,0x00, -0x1e,0x00,0x03,0x00,0xfa,0x00,0x00,0x00,0xf9,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0xfb,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0xfa,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0xfb,0x00,0x00,0x00, -0xfc,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x07,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0xf8,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x0b,0x01,0x00,0x00,0x04,0x00,0x00,0x00, -0xb9,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x11,0x01,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x16,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x33,0x00,0x06,0x00, -0x09,0x00,0x00,0x00,0x17,0x01,0x00,0x00,0x16,0x01,0x00,0x00, -0x39,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x18,0x01,0x00,0x00,0x51,0x00,0x00,0x00, -0x17,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x19,0x01,0x00,0x00,0x84,0x00,0x00,0x00, -0x18,0x01,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x1a,0x01,0x00,0x00,0x86,0x00,0x00,0x00, -0x19,0x01,0x00,0x00,0x6c,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x38,0x01,0x00,0x00,0x80,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x39,0x01,0x00,0x00,0x84,0x00,0x00,0x00, -0x9c,0x00,0x00,0x00,0x38,0x01,0x00,0x00,0x1c,0x00,0x04,0x00, -0x3a,0x01,0x00,0x00,0xb9,0x00,0x00,0x00,0x39,0x01,0x00,0x00, -0x20,0x00,0x04,0x00,0x3b,0x01,0x00,0x00,0x04,0x00,0x00,0x00, -0x3a,0x01,0x00,0x00,0x3b,0x00,0x04,0x00,0x3b,0x01,0x00,0x00, -0x3c,0x01,0x00,0x00,0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x40,0x01,0x00,0x00,0x80,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, -0x44,0x01,0x00,0x00,0xb9,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, -0x45,0x01,0x00,0x00,0x44,0x01,0x00,0x00,0x20,0x00,0x04,0x00, -0x46,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x45,0x01,0x00,0x00, -0x3b,0x00,0x04,0x00,0x46,0x01,0x00,0x00,0x47,0x01,0x00,0x00, -0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x52,0x01,0x00,0x00, -0x0c,0x00,0x00,0x00,0xb9,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x5a,0x01,0x00,0x00,0x80,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x61,0x01,0x00,0x00,0x08,0x01,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x62,0x01,0x00,0x00, -0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x65,0x01,0x00,0x00, -0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x80,0x01,0x00,0x00, -0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00, -0x1c,0x00,0x04,0x00,0x81,0x01,0x00,0x00,0xb9,0x00,0x00,0x00, -0x80,0x01,0x00,0x00,0x20,0x00,0x04,0x00,0x82,0x01,0x00,0x00, -0x07,0x00,0x00,0x00,0x81,0x01,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x92,0x01,0x00,0x00,0x80,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xad,0x01,0x00,0x00,0x84,0x00,0x00,0x00, -0xb4,0x00,0x00,0x00,0xb1,0x00,0x00,0x00,0x1c,0x00,0x04,0x00, -0xae,0x01,0x00,0x00,0xb9,0x00,0x00,0x00,0xad,0x01,0x00,0x00, -0x20,0x00,0x04,0x00,0xaf,0x01,0x00,0x00,0x07,0x00,0x00,0x00, -0xae,0x01,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xb8,0x01,0x00,0x00,0x86,0x00,0x00,0x00,0xae,0x00,0x00,0x00, -0xb4,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xc0,0x01,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xef,0x01,0x00,0x00,0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, -0x22,0x02,0x00,0x00,0x0d,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x0a,0x00,0x00,0x00,0x2a,0x02,0x00,0x00,0x01,0x00,0x00,0x00, -0x1d,0x00,0x03,0x00,0x70,0x02,0x00,0x00,0xb9,0x00,0x00,0x00, -0x1e,0x00,0x03,0x00,0x71,0x02,0x00,0x00,0x70,0x02,0x00,0x00, -0x20,0x00,0x04,0x00,0x72,0x02,0x00,0x00,0x0c,0x00,0x00,0x00, -0x71,0x02,0x00,0x00,0x3b,0x00,0x04,0x00,0x72,0x02,0x00,0x00, -0x73,0x02,0x00,0x00,0x0c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0x78,0x02,0x00,0x00,0x05,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x85,0x02,0x00,0x00, -0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00, -0x36,0x00,0x05,0x00,0x02,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x05,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0xbe,0x00,0x00,0x00, -0xbf,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x82,0x01,0x00,0x00,0x83,0x01,0x00,0x00,0x07,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0xaf,0x01,0x00,0x00,0xb0,0x01,0x00,0x00, -0x07,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00, -0x0e,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, -0x0e,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x14,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x17,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x17,0x00,0x00,0x00, -0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x1e,0x00,0x00,0x00, -0x0f,0x00,0x00,0x00,0x17,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x21,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x24,0x00,0x00,0x00,0x18,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0x28,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x27,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x29,0x00,0x00,0x00, -0x28,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x2a,0x00,0x00,0x00,0x1e,0x00,0x00,0x00,0x29,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x2d,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x2f,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x30,0x00,0x00,0x00, -0x24,0x00,0x00,0x00,0x2f,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x30,0x00,0x00,0x00, -0x2a,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0x35,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x34,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x36,0x00,0x00,0x00, -0x35,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x38,0x00,0x00,0x00,0x36,0x00,0x00,0x00,0x37,0x00,0x00,0x00, -0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, -0x38,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x3b,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, -0x37,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00, -0x3f,0x00,0x00,0x00,0x3d,0x00,0x00,0x00,0x3e,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x3f,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x42,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x3b,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x47,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0x3b,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x0d,0x00,0x00,0x00,0x49,0x00,0x00,0x00,0x3d,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x4a,0x00,0x00,0x00,0x49,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x0d,0x00,0x00,0x00,0x4d,0x00,0x00,0x00,0x4c,0x00,0x00,0x00, -0x3e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x4e,0x00,0x00,0x00,0x4d,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x50,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, -0x4f,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x55,0x00,0x00,0x00,0x50,0x00,0x00,0x00,0x54,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x59,0x00,0x00,0x00, -0x50,0x00,0x00,0x00,0x58,0x00,0x00,0x00,0x89,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x5d,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, -0x4f,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x64,0x00,0x00,0x00,0x5d,0x00,0x00,0x00,0x63,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x68,0x00,0x00,0x00, -0x5d,0x00,0x00,0x00,0x67,0x00,0x00,0x00,0x89,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x6e,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, -0x6d,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x73,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x72,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x77,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x76,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x78,0x00,0x00,0x00,0x77,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x79,0x00,0x00,0x00, -0x47,0x00,0x00,0x00,0x78,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x7c,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x7b,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x7d,0x00,0x00,0x00,0x7c,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x7f,0x00,0x00,0x00,0x47,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x82,0x00,0x00,0x00,0x7f,0x00,0x00,0x00,0x78,0x00,0x00,0x00, -0x0c,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x83,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x26,0x00,0x00,0x00,0x7d,0x00,0x00,0x00, -0x82,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0x87,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x86,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x88,0x00,0x00,0x00, -0x87,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x89,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x88,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8b,0x00,0x00,0x00, -0x42,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x8d,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x8c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x8e,0x00,0x00,0x00,0x8d,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x8f,0x00,0x00,0x00,0x8b,0x00,0x00,0x00, -0x8e,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x90,0x00,0x00,0x00,0x89,0x00,0x00,0x00,0x8f,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x92,0x00,0x00,0x00, -0x90,0x00,0x00,0x00,0x79,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x93,0x00,0x00,0x00,0x92,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0x98,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x97,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x99,0x00,0x00,0x00, -0x98,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x9a,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x99,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9d,0x00,0x00,0x00, -0x4a,0x00,0x00,0x00,0x9c,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x9f,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x9e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xa0,0x00,0x00,0x00,0x9f,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xa1,0x00,0x00,0x00,0x9d,0x00,0x00,0x00, -0xa0,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xa2,0x00,0x00,0x00,0x9a,0x00,0x00,0x00,0xa1,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa4,0x00,0x00,0x00, -0xa2,0x00,0x00,0x00,0x79,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xa5,0x00,0x00,0x00,0xa4,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xa7,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xa7,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0x97,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, -0x05,0x00,0x00,0x00,0xc6,0x00,0x00,0x00,0xa8,0x00,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00,0xb8,0x00,0x00,0x00, -0x97,0x02,0x00,0x00,0xb6,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0xa9,0x00,0x00,0x00,0xa8,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xb8,0x00,0x00,0x00,0xa8,0x00,0x00,0x00, -0xa9,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xa8,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0xc3,0x00,0x00,0x00, -0xbf,0x00,0x00,0x00,0x97,0x02,0x00,0x00,0x3e,0x00,0x03,0x00, -0xc3,0x00,0x00,0x00,0xc1,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xc6,0x00,0x00,0x00,0x97,0x02,0x00,0x00, -0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xa7,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xa9,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xc9,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xc9,0x00,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xb0,0x02,0x00,0x00, -0xa5,0x00,0x00,0x00,0xa9,0x00,0x00,0x00,0x67,0x01,0x00,0x00, -0xcc,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xac,0x02,0x00,0x00,0x93,0x00,0x00,0x00,0xa9,0x00,0x00,0x00, -0x64,0x01,0x00,0x00,0xcc,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0x98,0x02,0x00,0x00,0x79,0x00,0x00,0x00, -0xa9,0x00,0x00,0x00,0x12,0x02,0x00,0x00,0xcc,0x00,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00,0xd0,0x00,0x00,0x00, -0x98,0x02,0x00,0x00,0x83,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0xcb,0x00,0x00,0x00,0xcc,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xd0,0x00,0x00,0x00,0xca,0x00,0x00,0x00, -0xcb,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xca,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xd2,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xd2,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xa8,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0xca,0x00,0x00,0x00, -0x1c,0x01,0x00,0x00,0xd5,0x00,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb7,0x00,0x00,0x00,0xd8,0x00,0x00,0x00,0xa8,0x02,0x00,0x00, -0x37,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xd4,0x00,0x00,0x00, -0xd5,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0xd8,0x00,0x00,0x00,0xd3,0x00,0x00,0x00,0xd4,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xd3,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xdc,0x00,0x00,0x00,0x8b,0x00,0x00,0x00, -0x73,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xde,0x00,0x00,0x00,0xdc,0x00,0x00,0x00,0xa8,0x02,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00,0xe1,0x00,0x00,0x00, -0xde,0x00,0x00,0x00,0x36,0x00,0x00,0x00,0xf7,0x00,0x03,0x00, -0xe3,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0xe1,0x00,0x00,0x00,0xe2,0x00,0x00,0x00,0xe3,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xe2,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xe6,0x00,0x00,0x00,0x98,0x02,0x00,0x00, -0x6e,0x00,0x00,0x00,0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00, -0xe8,0x00,0x00,0x00,0xe6,0x00,0x00,0x00,0x83,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xe3,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xe3,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0xb7,0x00,0x00,0x00, -0xe9,0x00,0x00,0x00,0xe1,0x00,0x00,0x00,0xd3,0x00,0x00,0x00, -0xe8,0x00,0x00,0x00,0xe2,0x00,0x00,0x00,0xf7,0x00,0x03,0x00, -0xeb,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0xe9,0x00,0x00,0x00,0xea,0x00,0x00,0x00,0x0d,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xea,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xf3,0x00,0x00,0x00,0x73,0x00,0x00,0x00, -0xa8,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xf5,0x00,0x00,0x00,0xf3,0x00,0x00,0x00,0xf4,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xf7,0x00,0x00,0x00, -0xf5,0x00,0x00,0x00,0x6e,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x03,0x01,0x00,0x00,0xf3,0x00,0x00,0x00, -0x8e,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x04,0x01,0x00,0x00,0xac,0x02,0x00,0x00,0x03,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x06,0x01,0x00,0x00, -0x04,0x01,0x00,0x00,0x6e,0x00,0x00,0x00,0x41,0x00,0x06,0x00, -0x07,0x01,0x00,0x00,0x08,0x01,0x00,0x00,0xfc,0x00,0x00,0x00, -0x34,0x00,0x00,0x00,0x06,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, -0xf8,0x00,0x00,0x00,0x09,0x01,0x00,0x00,0x08,0x01,0x00,0x00, -0x73,0x00,0x04,0x00,0xb9,0x00,0x00,0x00,0x0a,0x01,0x00,0x00, -0x09,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0x0b,0x01,0x00,0x00, -0x0c,0x01,0x00,0x00,0xf0,0x00,0x00,0x00,0xf7,0x00,0x00,0x00, -0x3e,0x00,0x03,0x00,0x0c,0x01,0x00,0x00,0x0a,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0xeb,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x0d,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x10,0x01,0x00,0x00,0x73,0x00,0x00,0x00,0xa8,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x12,0x01,0x00,0x00, -0x10,0x01,0x00,0x00,0x11,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x14,0x01,0x00,0x00,0x12,0x01,0x00,0x00, -0x6e,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x0b,0x01,0x00,0x00, -0x15,0x01,0x00,0x00,0xf0,0x00,0x00,0x00,0x14,0x01,0x00,0x00, -0x3e,0x00,0x03,0x00,0x15,0x01,0x00,0x00,0xc1,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xeb,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xeb,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xd5,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xd5,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x1c,0x01,0x00,0x00,0xa8,0x02,0x00,0x00, -0x1a,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xd2,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xd4,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x1e,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x1e,0x01,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xa9,0x02,0x00,0x00, -0x3e,0x00,0x00,0x00,0xd4,0x00,0x00,0x00,0x60,0x01,0x00,0x00, -0x21,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00, -0x24,0x01,0x00,0x00,0xa9,0x02,0x00,0x00,0x9c,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0x20,0x01,0x00,0x00,0x21,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x24,0x01,0x00,0x00, -0x1f,0x01,0x00,0x00,0x20,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x1f,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x28,0x01,0x00,0x00,0x9d,0x00,0x00,0x00,0x73,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x2a,0x01,0x00,0x00, -0x28,0x01,0x00,0x00,0xa9,0x02,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x2b,0x01,0x00,0x00,0x12,0x00,0x00,0x00, -0xc5,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x2c,0x01,0x00,0x00,0x2b,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb7,0x00,0x00,0x00,0x2d,0x01,0x00,0x00,0x2a,0x01,0x00,0x00, -0x2c,0x01,0x00,0x00,0xf7,0x00,0x03,0x00,0x2f,0x01,0x00,0x00, -0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x2d,0x01,0x00,0x00, -0x2e,0x01,0x00,0x00,0x2f,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x2e,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x32,0x01,0x00,0x00,0x98,0x02,0x00,0x00,0x6e,0x00,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00,0x34,0x01,0x00,0x00, -0x32,0x01,0x00,0x00,0x83,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x2f,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x2f,0x01,0x00,0x00, -0xf5,0x00,0x07,0x00,0xb7,0x00,0x00,0x00,0x35,0x01,0x00,0x00, -0x2d,0x01,0x00,0x00,0x1f,0x01,0x00,0x00,0x34,0x01,0x00,0x00, -0x2e,0x01,0x00,0x00,0xf7,0x00,0x03,0x00,0x37,0x01,0x00,0x00, -0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x35,0x01,0x00,0x00, -0x36,0x01,0x00,0x00,0x56,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x36,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x3f,0x01,0x00,0x00,0x73,0x00,0x00,0x00,0xa9,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x41,0x01,0x00,0x00, -0x3f,0x01,0x00,0x00,0x40,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x43,0x01,0x00,0x00,0x41,0x01,0x00,0x00, -0x6e,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x4e,0x01,0x00,0x00,0x3f,0x01,0x00,0x00,0xa0,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x4f,0x01,0x00,0x00, -0xb0,0x02,0x00,0x00,0x4e,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x51,0x01,0x00,0x00,0x4f,0x01,0x00,0x00, -0x6e,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x52,0x01,0x00,0x00, -0x53,0x01,0x00,0x00,0x47,0x01,0x00,0x00,0x34,0x00,0x00,0x00, -0x51,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0xb9,0x00,0x00,0x00, -0x54,0x01,0x00,0x00,0x53,0x01,0x00,0x00,0x41,0x00,0x05,0x00, -0x0b,0x01,0x00,0x00,0x55,0x01,0x00,0x00,0x3c,0x01,0x00,0x00, -0x43,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x55,0x01,0x00,0x00, -0x54,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x37,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x56,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x59,0x01,0x00,0x00,0x73,0x00,0x00,0x00, -0xa9,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x5b,0x01,0x00,0x00,0x59,0x01,0x00,0x00,0x5a,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x5d,0x01,0x00,0x00, -0x5b,0x01,0x00,0x00,0x6e,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x0b,0x01,0x00,0x00,0x5e,0x01,0x00,0x00,0x3c,0x01,0x00,0x00, -0x5d,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x5e,0x01,0x00,0x00, -0xc1,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x37,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x37,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0x21,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x21,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x60,0x01,0x00,0x00, -0xa9,0x02,0x00,0x00,0x1a,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0x1e,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x20,0x01,0x00,0x00, -0xe0,0x00,0x04,0x00,0x0c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x61,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x64,0x01,0x00,0x00,0xac,0x02,0x00,0x00,0x62,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x67,0x01,0x00,0x00, -0xb0,0x02,0x00,0x00,0x65,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0x69,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x69,0x01,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xb2,0x02,0x00,0x00, -0x3e,0x00,0x00,0x00,0x20,0x01,0x00,0x00,0x10,0x02,0x00,0x00, -0x6c,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00, -0x6f,0x01,0x00,0x00,0xb2,0x02,0x00,0x00,0x6c,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0x6b,0x01,0x00,0x00,0x6c,0x01,0x00,0x00, -0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x6f,0x01,0x00,0x00, -0x6a,0x01,0x00,0x00,0x6b,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x6a,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x71,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x71,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xb6,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, -0x6a,0x01,0x00,0x00,0x9c,0x01,0x00,0x00,0x74,0x01,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00,0x77,0x01,0x00,0x00, -0xb6,0x02,0x00,0x00,0x60,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0x73,0x01,0x00,0x00,0x74,0x01,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x77,0x01,0x00,0x00,0x72,0x01,0x00,0x00, -0x73,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x72,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0x79,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x79,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xc8,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0x72,0x01,0x00,0x00, -0x9a,0x01,0x00,0x00,0x7a,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb7,0x00,0x00,0x00,0x7f,0x01,0x00,0x00,0xc8,0x02,0x00,0x00, -0x62,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x7b,0x01,0x00,0x00, -0x7a,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x7f,0x01,0x00,0x00,0x7a,0x01,0x00,0x00,0x7b,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x7a,0x01,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x85,0x01,0x00,0x00,0xb6,0x02,0x00,0x00, -0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x87,0x01,0x00,0x00,0x85,0x01,0x00,0x00,0xc8,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x89,0x01,0x00,0x00, -0x55,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x8b,0x01,0x00,0x00,0xb6,0x02,0x00,0x00, -0x61,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x8c,0x01,0x00,0x00,0x89,0x01,0x00,0x00,0x8b,0x01,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8e,0x01,0x00,0x00, -0x64,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x8f,0x01,0x00,0x00,0x8c,0x01,0x00,0x00, -0x8e,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x91,0x01,0x00,0x00,0x8f,0x01,0x00,0x00,0xc8,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x93,0x01,0x00,0x00, -0x91,0x01,0x00,0x00,0x92,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x95,0x01,0x00,0x00,0x93,0x01,0x00,0x00, -0xb2,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x0b,0x01,0x00,0x00, -0x96,0x01,0x00,0x00,0xf0,0x00,0x00,0x00,0x95,0x01,0x00,0x00, -0x3d,0x00,0x04,0x00,0xb9,0x00,0x00,0x00,0x97,0x01,0x00,0x00, -0x96,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, -0x98,0x01,0x00,0x00,0x83,0x01,0x00,0x00,0x87,0x01,0x00,0x00, -0x3e,0x00,0x03,0x00,0x98,0x01,0x00,0x00,0x97,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9a,0x01,0x00,0x00, -0xc8,0x02,0x00,0x00,0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x79,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x7b,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0x74,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x74,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x9c,0x01,0x00,0x00,0xb6,0x02,0x00,0x00,0xc5,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x71,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x73,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x9e,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x9e,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xb7,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, -0x73,0x01,0x00,0x00,0xca,0x01,0x00,0x00,0xa1,0x01,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00,0xa4,0x01,0x00,0x00, -0xb7,0x02,0x00,0x00,0xb4,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0xa0,0x01,0x00,0x00,0xa1,0x01,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xa4,0x01,0x00,0x00,0x9f,0x01,0x00,0x00, -0xa0,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x9f,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0xa6,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xa6,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xc5,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0x9f,0x01,0x00,0x00, -0xc8,0x01,0x00,0x00,0xa7,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb7,0x00,0x00,0x00,0xac,0x01,0x00,0x00,0xc5,0x02,0x00,0x00, -0xb1,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xa8,0x01,0x00,0x00, -0xa7,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0xac,0x01,0x00,0x00,0xa7,0x01,0x00,0x00,0xa8,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xa7,0x01,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xb2,0x01,0x00,0x00,0xb7,0x02,0x00,0x00, -0xb1,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xb4,0x01,0x00,0x00,0xb2,0x01,0x00,0x00,0xc5,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xb6,0x01,0x00,0x00, -0x59,0x00,0x00,0x00,0xae,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xb9,0x01,0x00,0x00,0xb7,0x02,0x00,0x00, -0xb8,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xba,0x01,0x00,0x00,0xb6,0x01,0x00,0x00,0xb9,0x01,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xbc,0x01,0x00,0x00, -0x68,0x00,0x00,0x00,0xb1,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xbd,0x01,0x00,0x00,0xba,0x01,0x00,0x00, -0xbc,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xbf,0x01,0x00,0x00,0xbd,0x01,0x00,0x00,0xc5,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xc1,0x01,0x00,0x00, -0xbf,0x01,0x00,0x00,0xc0,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xc3,0x01,0x00,0x00,0xc1,0x01,0x00,0x00, -0xb2,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x0b,0x01,0x00,0x00, -0xc4,0x01,0x00,0x00,0x3c,0x01,0x00,0x00,0xc3,0x01,0x00,0x00, -0x3d,0x00,0x04,0x00,0xb9,0x00,0x00,0x00,0xc5,0x01,0x00,0x00, -0xc4,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, -0xc6,0x01,0x00,0x00,0xb0,0x01,0x00,0x00,0xb4,0x01,0x00,0x00, -0x3e,0x00,0x03,0x00,0xc6,0x01,0x00,0x00,0xc5,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xc8,0x01,0x00,0x00, -0xc5,0x02,0x00,0x00,0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xa6,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xa8,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0xa1,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xa1,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xca,0x01,0x00,0x00,0xb7,0x02,0x00,0x00,0xc5,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x9e,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xa0,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xcc,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xcc,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xb8,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, -0xa0,0x01,0x00,0x00,0x0e,0x02,0x00,0x00,0xcf,0x01,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00,0xd2,0x01,0x00,0x00, -0xb8,0x02,0x00,0x00,0xb4,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0xce,0x01,0x00,0x00,0xcf,0x01,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xd2,0x01,0x00,0x00,0xcd,0x01,0x00,0x00, -0xce,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xcd,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0xd4,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xd4,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xbc,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0xcd,0x01,0x00,0x00, -0x0c,0x02,0x00,0x00,0xd7,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb7,0x00,0x00,0x00,0xda,0x01,0x00,0x00,0xbc,0x02,0x00,0x00, -0x60,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xd6,0x01,0x00,0x00, -0xd7,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0xda,0x01,0x00,0x00,0xd5,0x01,0x00,0x00,0xd6,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xd5,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0xdc,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xdc,0x01,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xbe,0x02,0x00,0x00, -0x3e,0x00,0x00,0x00,0xd5,0x01,0x00,0x00,0x0a,0x02,0x00,0x00, -0xdf,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00, -0xe2,0x01,0x00,0x00,0xbe,0x02,0x00,0x00,0xb1,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0xde,0x01,0x00,0x00,0xdf,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xe2,0x01,0x00,0x00, -0xdd,0x01,0x00,0x00,0xde,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xdd,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xe4,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xe4,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xc0,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, -0xdd,0x01,0x00,0x00,0x08,0x02,0x00,0x00,0xe5,0x01,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00,0xea,0x01,0x00,0x00, -0xc0,0x02,0x00,0x00,0x62,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0xe6,0x01,0x00,0x00,0xe5,0x01,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xea,0x01,0x00,0x00,0xe5,0x01,0x00,0x00, -0xe6,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xe5,0x01,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xec,0x01,0x00,0x00, -0xb8,0x02,0x00,0x00,0xb1,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xee,0x01,0x00,0x00,0xec,0x01,0x00,0x00, -0xbe,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xf0,0x01,0x00,0x00,0xee,0x01,0x00,0x00,0xef,0x01,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xf2,0x01,0x00,0x00, -0xbc,0x02,0x00,0x00,0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xf3,0x01,0x00,0x00,0xf0,0x01,0x00,0x00, -0xf2,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xf5,0x01,0x00,0x00,0xf3,0x01,0x00,0x00,0xc0,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xf9,0x01,0x00,0x00, -0xf2,0x01,0x00,0x00,0xc0,0x02,0x00,0x00,0x41,0x00,0x05,0x00, -0xc2,0x00,0x00,0x00,0xfa,0x01,0x00,0x00,0x83,0x01,0x00,0x00, -0xf9,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0xb9,0x00,0x00,0x00, -0xfb,0x01,0x00,0x00,0xfa,0x01,0x00,0x00,0x41,0x00,0x05,0x00, -0xc2,0x00,0x00,0x00,0x00,0x02,0x00,0x00,0xb0,0x01,0x00,0x00, -0xee,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0xb9,0x00,0x00,0x00, -0x01,0x02,0x00,0x00,0x00,0x02,0x00,0x00,0x41,0x00,0x05,0x00, -0xc2,0x00,0x00,0x00,0x03,0x02,0x00,0x00,0xbf,0x00,0x00,0x00, -0xf5,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0xb9,0x00,0x00,0x00, -0x04,0x02,0x00,0x00,0x03,0x02,0x00,0x00,0x0c,0x00,0x08,0x00, -0xb9,0x00,0x00,0x00,0x05,0x02,0x00,0x00,0x01,0x00,0x00,0x00, -0x32,0x00,0x00,0x00,0xfb,0x01,0x00,0x00,0x01,0x02,0x00,0x00, -0x04,0x02,0x00,0x00,0x3e,0x00,0x03,0x00,0x03,0x02,0x00,0x00, -0x05,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x08,0x02,0x00,0x00,0xc0,0x02,0x00,0x00,0xc5,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xe4,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xe6,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xdf,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xdf,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x0a,0x02,0x00,0x00,0xbe,0x02,0x00,0x00, -0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xdc,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xde,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0xd7,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xd7,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x0c,0x02,0x00,0x00, -0xbc,0x02,0x00,0x00,0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xd4,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xd6,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0xcf,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xcf,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x0e,0x02,0x00,0x00,0xb8,0x02,0x00,0x00,0xc5,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xcc,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xce,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x6c,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x6c,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x10,0x02,0x00,0x00,0xb2,0x02,0x00,0x00, -0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x69,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x6b,0x01,0x00,0x00,0xe0,0x00,0x04,0x00, -0x0c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x61,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0xcc,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xcc,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x12,0x02,0x00,0x00,0x98,0x02,0x00,0x00,0x6c,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xc9,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xcb,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x17,0x02,0x00,0x00,0x55,0x00,0x00,0x00,0x53,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x18,0x02,0x00,0x00, -0x8b,0x00,0x00,0x00,0x17,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x1d,0x02,0x00,0x00,0x59,0x00,0x00,0x00, -0xae,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x1e,0x02,0x00,0x00,0x9d,0x00,0x00,0x00,0x1d,0x02,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x23,0x02,0x00,0x00, -0x12,0x00,0x00,0x00,0x22,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x24,0x02,0x00,0x00,0x23,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x25,0x02,0x00,0x00, -0x0f,0x00,0x00,0x00,0x24,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x29,0x02,0x00,0x00,0x47,0x00,0x00,0x00, -0x24,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00, -0x2b,0x02,0x00,0x00,0x2a,0x02,0x00,0x00,0x0c,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x2c,0x02,0x00,0x00, -0x2b,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x2d,0x02,0x00,0x00,0x29,0x02,0x00,0x00,0x2c,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x2e,0x02,0x00,0x00, -0x25,0x02,0x00,0x00,0x2d,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x30,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x30,0x02,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x99,0x02,0x00,0x00, -0x3e,0x00,0x00,0x00,0xcb,0x00,0x00,0x00,0x96,0x02,0x00,0x00, -0x33,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00, -0x36,0x02,0x00,0x00,0x99,0x02,0x00,0x00,0xb4,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0x32,0x02,0x00,0x00,0x33,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x36,0x02,0x00,0x00, -0x31,0x02,0x00,0x00,0x32,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x31,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x38,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x38,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0x9a,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, -0x31,0x02,0x00,0x00,0x94,0x02,0x00,0x00,0x3b,0x02,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00,0x3e,0x02,0x00,0x00, -0x9a,0x02,0x00,0x00,0x60,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0x3a,0x02,0x00,0x00,0x3b,0x02,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x3e,0x02,0x00,0x00,0x39,0x02,0x00,0x00, -0x3a,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x39,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x42,0x02,0x00,0x00, -0x9a,0x02,0x00,0x00,0x61,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x43,0x02,0x00,0x00,0x18,0x02,0x00,0x00, -0x42,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x45,0x02,0x00,0x00,0x64,0x00,0x00,0x00,0x62,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x46,0x02,0x00,0x00, -0x43,0x02,0x00,0x00,0x45,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x4a,0x02,0x00,0x00,0x99,0x02,0x00,0x00, -0xb8,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x4b,0x02,0x00,0x00,0x1e,0x02,0x00,0x00,0x4a,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x4d,0x02,0x00,0x00, -0x68,0x00,0x00,0x00,0xb1,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x4e,0x02,0x00,0x00,0x4b,0x02,0x00,0x00, -0x4d,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x50,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x50,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0x9c,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, -0x39,0x02,0x00,0x00,0x92,0x02,0x00,0x00,0x53,0x02,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00,0x56,0x02,0x00,0x00, -0x9c,0x02,0x00,0x00,0xb1,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0x52,0x02,0x00,0x00,0x53,0x02,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x56,0x02,0x00,0x00,0x51,0x02,0x00,0x00, -0x52,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x51,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0x58,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x58,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0x9e,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0x51,0x02,0x00,0x00, -0x90,0x02,0x00,0x00,0x5b,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb7,0x00,0x00,0x00,0x5e,0x02,0x00,0x00,0x9e,0x02,0x00,0x00, -0x62,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x5a,0x02,0x00,0x00, -0x5b,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x5e,0x02,0x00,0x00,0x59,0x02,0x00,0x00,0x5a,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x59,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x61,0x02,0x00,0x00,0x46,0x02,0x00,0x00, -0x9e,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00, -0x64,0x02,0x00,0x00,0x61,0x02,0x00,0x00,0x36,0x00,0x00,0x00, -0xf7,0x00,0x03,0x00,0x66,0x02,0x00,0x00,0x00,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x64,0x02,0x00,0x00,0x65,0x02,0x00,0x00, -0x66,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x65,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x69,0x02,0x00,0x00, -0x4e,0x02,0x00,0x00,0x9c,0x02,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x6a,0x02,0x00,0x00,0x12,0x00,0x00,0x00, -0xc5,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x6b,0x02,0x00,0x00,0x6a,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb7,0x00,0x00,0x00,0x6c,0x02,0x00,0x00,0x69,0x02,0x00,0x00, -0x6b,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x66,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x66,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, -0xb7,0x00,0x00,0x00,0x6d,0x02,0x00,0x00,0x64,0x02,0x00,0x00, -0x59,0x02,0x00,0x00,0x6c,0x02,0x00,0x00,0x65,0x02,0x00,0x00, -0xf7,0x00,0x03,0x00,0x6f,0x02,0x00,0x00,0x00,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x6d,0x02,0x00,0x00,0x6e,0x02,0x00,0x00, -0x6f,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x6e,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x77,0x02,0x00,0x00, -0x4e,0x02,0x00,0x00,0x9c,0x02,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x79,0x02,0x00,0x00,0x12,0x00,0x00,0x00, -0x78,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x7a,0x02,0x00,0x00,0x79,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x7b,0x02,0x00,0x00,0x77,0x02,0x00,0x00, -0x7a,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x7c,0x02,0x00,0x00,0x2e,0x02,0x00,0x00,0x7b,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x7e,0x02,0x00,0x00, -0x7c,0x02,0x00,0x00,0x46,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x80,0x02,0x00,0x00,0x7e,0x02,0x00,0x00, -0x9e,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x82,0x02,0x00,0x00,0x99,0x02,0x00,0x00,0xb1,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x84,0x02,0x00,0x00, -0x82,0x02,0x00,0x00,0x9c,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x86,0x02,0x00,0x00,0x84,0x02,0x00,0x00, -0x85,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x88,0x02,0x00,0x00,0x9a,0x02,0x00,0x00,0x62,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x89,0x02,0x00,0x00, -0x86,0x02,0x00,0x00,0x88,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x8b,0x02,0x00,0x00,0x89,0x02,0x00,0x00, -0x9e,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, -0x8c,0x02,0x00,0x00,0xbf,0x00,0x00,0x00,0x8b,0x02,0x00,0x00, -0x3d,0x00,0x04,0x00,0xb9,0x00,0x00,0x00,0x8d,0x02,0x00,0x00, -0x8c,0x02,0x00,0x00,0x41,0x00,0x06,0x00,0x52,0x01,0x00,0x00, -0x8e,0x02,0x00,0x00,0x73,0x02,0x00,0x00,0x34,0x00,0x00,0x00, -0x80,0x02,0x00,0x00,0x3e,0x00,0x03,0x00,0x8e,0x02,0x00,0x00, -0x8d,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x6f,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x6f,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x5b,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x5b,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x90,0x02,0x00,0x00, -0x9e,0x02,0x00,0x00,0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x58,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x5a,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0x53,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x53,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x92,0x02,0x00,0x00,0x9c,0x02,0x00,0x00,0xc5,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x50,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x52,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x3b,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x3b,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x94,0x02,0x00,0x00,0x9a,0x02,0x00,0x00, -0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x38,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x3a,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x33,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x33,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x96,0x02,0x00,0x00, -0x99,0x02,0x00,0x00,0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x30,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x32,0x02,0x00,0x00, -0xfd,0x00,0x01,0x00,0x38,0x00,0x01,0x00, -}; -const uint64_t matmul_f16_f32_l_fp32_len = 10100; - -unsigned char matmul_f16_f32_m_data[] = { -0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, -0xcd,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, -0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x09,0x00,0x00,0x00, -0x11,0x00,0x02,0x00,0x51,0x11,0x00,0x00,0x0b,0x00,0x06,0x00, -0x01,0x00,0x00,0x00,0x47,0x4c,0x53,0x4c,0x2e,0x73,0x74,0x64, -0x2e,0x34,0x35,0x30,0x00,0x00,0x00,0x00,0x0e,0x00,0x03,0x00, -0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x0f,0x00,0x0f,0x00, -0x05,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x6d,0x61,0x69,0x6e, -0x00,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x3d,0x00,0x00,0x00,0x4c,0x00,0x00,0x00,0xf1,0x00,0x00,0x00, -0xfc,0x00,0x00,0x00,0x3c,0x01,0x00,0x00,0x47,0x01,0x00,0x00, -0x2e,0x02,0x00,0x00,0x77,0x02,0x00,0x00,0x10,0x00,0x06,0x00, -0x04,0x00,0x00,0x00,0x11,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x0b,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x1c,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x10,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x08,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x03,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x10,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x10,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x05,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x14,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x10,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x1c,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x08,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x24,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x10,0x00,0x00,0x00,0x0a,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x28,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x0b,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x2c,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x30,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x10,0x00,0x00,0x00,0x0d,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x34,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x10,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x37,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x3d,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x1a,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x4c,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x1b,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x4f,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x53,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x60,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x62,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x6c,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x03,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x9c,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xae,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0xb1,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x08,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0xf9,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0xfa,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0xfa,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0xfa,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xfc,0x00,0x00,0x00, -0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0xfc,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x16,0x01,0x00,0x00,0x01,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x17,0x01,0x00,0x00, -0x0b,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x44,0x01,0x00,0x00,0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x48,0x00,0x04,0x00,0x45,0x01,0x00,0x00,0x00,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x45,0x01,0x00,0x00, -0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x03,0x00,0x45,0x01,0x00,0x00,0x02,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x47,0x01,0x00,0x00,0x22,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x47,0x01,0x00,0x00, -0x21,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x2e,0x02,0x00,0x00,0x0b,0x00,0x00,0x00,0x18,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x74,0x02,0x00,0x00,0x06,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0x75,0x02,0x00,0x00, -0x00,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x75,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x75,0x02,0x00,0x00, -0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x77,0x02,0x00,0x00, -0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x77,0x02,0x00,0x00,0x21,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x13,0x00,0x02,0x00,0x02,0x00,0x00,0x00,0x21,0x00,0x03,0x00, -0x03,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x15,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x17,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x03,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x0a,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x0a,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x0d,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x1e,0x00,0x10,0x00, -0x10,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x11,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x10,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x11,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x15,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x14,0x00,0x00,0x00, -0x08,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x15,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x0a,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x27,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, -0x2d,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x37,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x0a,0x00,0x00,0x00,0x3d,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x3e,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00, -0x4c,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x53,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x54,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x37,0x00,0x00,0x00, -0x53,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x58,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x37,0x00,0x00,0x00, -0x53,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x60,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x61,0x00,0x00,0x00,0x86,0x00,0x00,0x00, -0x53,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x32,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x63,0x00,0x00,0x00, -0x86,0x00,0x00,0x00,0x61,0x00,0x00,0x00,0x62,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x67,0x00,0x00,0x00, -0x86,0x00,0x00,0x00,0x61,0x00,0x00,0x00,0x62,0x00,0x00,0x00, -0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x10,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x6d,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x72,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, -0x76,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0x7b,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x86,0x00,0x00,0x00, -0x0b,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, -0x8c,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0x97,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x9c,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, -0x9e,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xad,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x32,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xae,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xaf,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0xae,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xb0,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x62,0x00,0x00,0x00, -0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xb1,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xb2,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0xb0,0x00,0x00,0x00, -0xb1,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xb3,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0xb2,0x00,0x00,0x00, -0x60,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xb4,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0xaf,0x00,0x00,0x00, -0xb3,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xb5,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0xad,0x00,0x00,0x00, -0xb4,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xb6,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0xb5,0x00,0x00,0x00, -0xb1,0x00,0x00,0x00,0x14,0x00,0x02,0x00,0xb7,0x00,0x00,0x00, -0x16,0x00,0x03,0x00,0xb9,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xba,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xbb,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0xba,0x00,0x00,0x00,0xb4,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xbc,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0xbb,0x00,0x00,0x00,0xb1,0x00,0x00,0x00, -0x1c,0x00,0x04,0x00,0xbd,0x00,0x00,0x00,0xb9,0x00,0x00,0x00, -0xbc,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0xbe,0x00,0x00,0x00, -0x07,0x00,0x00,0x00,0xbd,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0xb9,0x00,0x00,0x00,0xc1,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0xc2,0x00,0x00,0x00,0x07,0x00,0x00,0x00, -0xb9,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, -0xc5,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x16,0x00,0x03,0x00, -0xec,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xed,0x00,0x00,0x00,0x80,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xee,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0x37,0x00,0x00,0x00,0xed,0x00,0x00,0x00,0x1c,0x00,0x04,0x00, -0xef,0x00,0x00,0x00,0xec,0x00,0x00,0x00,0xee,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0xf0,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0xef,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0xf0,0x00,0x00,0x00, -0xf1,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xf5,0x00,0x00,0x00,0x80,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, -0xf9,0x00,0x00,0x00,0xec,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, -0xfa,0x00,0x00,0x00,0xf9,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0xfb,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0xfa,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0xfb,0x00,0x00,0x00,0xfc,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x07,0x01,0x00,0x00, -0x0c,0x00,0x00,0x00,0xec,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x0a,0x01,0x00,0x00,0x04,0x00,0x00,0x00,0xec,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x10,0x01,0x00,0x00, -0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0xec,0x00,0x00,0x00,0x14,0x01,0x00,0x00, -0x00,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x16,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x33,0x00,0x06,0x00, -0x09,0x00,0x00,0x00,0x17,0x01,0x00,0x00,0x16,0x01,0x00,0x00, -0x39,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x18,0x01,0x00,0x00,0x51,0x00,0x00,0x00, -0x17,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x19,0x01,0x00,0x00,0x84,0x00,0x00,0x00, -0x18,0x01,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x1a,0x01,0x00,0x00,0x86,0x00,0x00,0x00, -0x19,0x01,0x00,0x00,0x6c,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x38,0x01,0x00,0x00,0x80,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x39,0x01,0x00,0x00,0x84,0x00,0x00,0x00, -0x9c,0x00,0x00,0x00,0x38,0x01,0x00,0x00,0x1c,0x00,0x04,0x00, -0x3a,0x01,0x00,0x00,0xec,0x00,0x00,0x00,0x39,0x01,0x00,0x00, -0x20,0x00,0x04,0x00,0x3b,0x01,0x00,0x00,0x04,0x00,0x00,0x00, -0x3a,0x01,0x00,0x00,0x3b,0x00,0x04,0x00,0x3b,0x01,0x00,0x00, -0x3c,0x01,0x00,0x00,0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x40,0x01,0x00,0x00,0x80,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, -0x44,0x01,0x00,0x00,0xb9,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, -0x45,0x01,0x00,0x00,0x44,0x01,0x00,0x00,0x20,0x00,0x04,0x00, -0x46,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x45,0x01,0x00,0x00, -0x3b,0x00,0x04,0x00,0x46,0x01,0x00,0x00,0x47,0x01,0x00,0x00, -0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x52,0x01,0x00,0x00, -0x0c,0x00,0x00,0x00,0xb9,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x5b,0x01,0x00,0x00,0x80,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x62,0x01,0x00,0x00,0x08,0x01,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x63,0x01,0x00,0x00, -0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x66,0x01,0x00,0x00, -0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x81,0x01,0x00,0x00, -0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00, -0x1c,0x00,0x04,0x00,0x82,0x01,0x00,0x00,0xec,0x00,0x00,0x00, -0x81,0x01,0x00,0x00,0x20,0x00,0x04,0x00,0x83,0x01,0x00,0x00, -0x07,0x00,0x00,0x00,0x82,0x01,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x93,0x01,0x00,0x00,0x80,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x99,0x01,0x00,0x00,0x07,0x00,0x00,0x00,0xec,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xaf,0x01,0x00,0x00, -0x84,0x00,0x00,0x00,0xb4,0x00,0x00,0x00,0xb1,0x00,0x00,0x00, -0x1c,0x00,0x04,0x00,0xb0,0x01,0x00,0x00,0xec,0x00,0x00,0x00, -0xaf,0x01,0x00,0x00,0x20,0x00,0x04,0x00,0xb1,0x01,0x00,0x00, -0x07,0x00,0x00,0x00,0xb0,0x01,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xba,0x01,0x00,0x00,0x86,0x00,0x00,0x00, -0xae,0x00,0x00,0x00,0xb4,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xc2,0x01,0x00,0x00,0x80,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xf1,0x01,0x00,0x00,0x84,0x00,0x00,0x00, -0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0x26,0x02,0x00,0x00,0x0d,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00,0x2e,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0x1d,0x00,0x03,0x00,0x74,0x02,0x00,0x00, -0xb9,0x00,0x00,0x00,0x1e,0x00,0x03,0x00,0x75,0x02,0x00,0x00, -0x74,0x02,0x00,0x00,0x20,0x00,0x04,0x00,0x76,0x02,0x00,0x00, -0x0c,0x00,0x00,0x00,0x75,0x02,0x00,0x00,0x3b,0x00,0x04,0x00, -0x76,0x02,0x00,0x00,0x77,0x02,0x00,0x00,0x0c,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x7c,0x02,0x00,0x00, -0x05,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x89,0x02,0x00,0x00,0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x36,0x00,0x05,0x00,0x02,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x03,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0x05,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0xbe,0x00,0x00,0x00,0xbf,0x00,0x00,0x00,0x07,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x83,0x01,0x00,0x00,0x84,0x01,0x00,0x00, -0x07,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0xb1,0x01,0x00,0x00, -0xb2,0x01,0x00,0x00,0x07,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x0d,0x00,0x00,0x00,0x0e,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x0f,0x00,0x00,0x00,0x0e,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x14,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x17,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, -0x17,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x1e,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x17,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x22,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x22,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x24,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x27,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x29,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x2a,0x00,0x00,0x00,0x1e,0x00,0x00,0x00, -0x29,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x2d,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x2f,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x30,0x00,0x00,0x00,0x24,0x00,0x00,0x00,0x2f,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x32,0x00,0x00,0x00, -0x30,0x00,0x00,0x00,0x2a,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x35,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x34,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x36,0x00,0x00,0x00,0x35,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x38,0x00,0x00,0x00,0x36,0x00,0x00,0x00, -0x37,0x00,0x00,0x00,0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x3a,0x00,0x00,0x00,0x38,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3b,0x00,0x00,0x00, -0x3a,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x0d,0x00,0x00,0x00,0x3f,0x00,0x00,0x00,0x3d,0x00,0x00,0x00, -0x3e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0x3f,0x00,0x00,0x00,0x89,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x42,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x3b,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x47,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x3b,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00,0x49,0x00,0x00,0x00, -0x3d,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x4a,0x00,0x00,0x00,0x49,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00,0x4d,0x00,0x00,0x00, -0x4c,0x00,0x00,0x00,0x3e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x4d,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x50,0x00,0x00,0x00, -0x4e,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x89,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x55,0x00,0x00,0x00,0x50,0x00,0x00,0x00, -0x54,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x59,0x00,0x00,0x00,0x50,0x00,0x00,0x00,0x58,0x00,0x00,0x00, -0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x5d,0x00,0x00,0x00, -0x4e,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x89,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x64,0x00,0x00,0x00,0x5d,0x00,0x00,0x00, -0x63,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x68,0x00,0x00,0x00,0x5d,0x00,0x00,0x00,0x67,0x00,0x00,0x00, -0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x6e,0x00,0x00,0x00, -0x4e,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x73,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, -0x72,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0x77,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x76,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x78,0x00,0x00,0x00, -0x77,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x79,0x00,0x00,0x00,0x47,0x00,0x00,0x00,0x78,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x7c,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x7b,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x7d,0x00,0x00,0x00,0x7c,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x7f,0x00,0x00,0x00, -0x47,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x82,0x00,0x00,0x00,0x7f,0x00,0x00,0x00, -0x78,0x00,0x00,0x00,0x0c,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0x83,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x26,0x00,0x00,0x00, -0x7d,0x00,0x00,0x00,0x82,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x87,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x86,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x88,0x00,0x00,0x00,0x87,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x89,0x00,0x00,0x00,0x32,0x00,0x00,0x00, -0x88,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x8b,0x00,0x00,0x00,0x42,0x00,0x00,0x00,0x37,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x8d,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x8c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x8e,0x00,0x00,0x00,0x8d,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8f,0x00,0x00,0x00, -0x8b,0x00,0x00,0x00,0x8e,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x90,0x00,0x00,0x00,0x89,0x00,0x00,0x00, -0x8f,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x92,0x00,0x00,0x00,0x90,0x00,0x00,0x00,0x79,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x93,0x00,0x00,0x00, -0x92,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x98,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x97,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x99,0x00,0x00,0x00,0x98,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x9a,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, -0x99,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x9d,0x00,0x00,0x00,0x4a,0x00,0x00,0x00,0x9c,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x9f,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x9e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xa0,0x00,0x00,0x00,0x9f,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa1,0x00,0x00,0x00, -0x9d,0x00,0x00,0x00,0xa0,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xa2,0x00,0x00,0x00,0x9a,0x00,0x00,0x00, -0xa1,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xa4,0x00,0x00,0x00,0xa2,0x00,0x00,0x00,0x79,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa5,0x00,0x00,0x00, -0xa4,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xa7,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xa7,0x00,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x9b,0x02,0x00,0x00, -0x3e,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0xc6,0x00,0x00,0x00, -0xa8,0x00,0x00,0x00,0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00, -0xb8,0x00,0x00,0x00,0x9b,0x02,0x00,0x00,0xb6,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0xa9,0x00,0x00,0x00,0xa8,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xb8,0x00,0x00,0x00, -0xa8,0x00,0x00,0x00,0xa9,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xa8,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, -0xc3,0x00,0x00,0x00,0xbf,0x00,0x00,0x00,0x9b,0x02,0x00,0x00, -0x3e,0x00,0x03,0x00,0xc3,0x00,0x00,0x00,0xc1,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xc6,0x00,0x00,0x00, -0x9b,0x02,0x00,0x00,0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xa7,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xa9,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xc9,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xc9,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xb4,0x02,0x00,0x00,0xa5,0x00,0x00,0x00,0xa9,0x00,0x00,0x00, -0x68,0x01,0x00,0x00,0xcc,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xb0,0x02,0x00,0x00,0x93,0x00,0x00,0x00, -0xa9,0x00,0x00,0x00,0x65,0x01,0x00,0x00,0xcc,0x00,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x9c,0x02,0x00,0x00, -0x79,0x00,0x00,0x00,0xa9,0x00,0x00,0x00,0x16,0x02,0x00,0x00, -0xcc,0x00,0x00,0x00,0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00, -0xd0,0x00,0x00,0x00,0x9c,0x02,0x00,0x00,0x83,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0xcb,0x00,0x00,0x00,0xcc,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xd0,0x00,0x00,0x00, -0xca,0x00,0x00,0x00,0xcb,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xca,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xd2,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xd2,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xac,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, -0xca,0x00,0x00,0x00,0x1c,0x01,0x00,0x00,0xd5,0x00,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00,0xd8,0x00,0x00,0x00, -0xac,0x02,0x00,0x00,0x37,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0xd4,0x00,0x00,0x00,0xd5,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xd8,0x00,0x00,0x00,0xd3,0x00,0x00,0x00, -0xd4,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xd3,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xdc,0x00,0x00,0x00, -0x8b,0x00,0x00,0x00,0x73,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xde,0x00,0x00,0x00,0xdc,0x00,0x00,0x00, -0xac,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00, -0xe1,0x00,0x00,0x00,0xde,0x00,0x00,0x00,0x36,0x00,0x00,0x00, -0xf7,0x00,0x03,0x00,0xe3,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xe1,0x00,0x00,0x00,0xe2,0x00,0x00,0x00, -0xe3,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xe2,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xe6,0x00,0x00,0x00, -0x9c,0x02,0x00,0x00,0x6e,0x00,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb7,0x00,0x00,0x00,0xe8,0x00,0x00,0x00,0xe6,0x00,0x00,0x00, -0x83,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xe3,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xe3,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, -0xb7,0x00,0x00,0x00,0xe9,0x00,0x00,0x00,0xe1,0x00,0x00,0x00, -0xd3,0x00,0x00,0x00,0xe8,0x00,0x00,0x00,0xe2,0x00,0x00,0x00, -0xf7,0x00,0x03,0x00,0xeb,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xe9,0x00,0x00,0x00,0xea,0x00,0x00,0x00, -0x0c,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xea,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xf4,0x00,0x00,0x00, -0x73,0x00,0x00,0x00,0xac,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xf6,0x00,0x00,0x00,0xf4,0x00,0x00,0x00, -0xf5,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xf8,0x00,0x00,0x00,0xf6,0x00,0x00,0x00,0x6e,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x03,0x01,0x00,0x00, -0xf4,0x00,0x00,0x00,0x8e,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x04,0x01,0x00,0x00,0xb0,0x02,0x00,0x00, -0x03,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x06,0x01,0x00,0x00,0x04,0x01,0x00,0x00,0x6e,0x00,0x00,0x00, -0x41,0x00,0x06,0x00,0x07,0x01,0x00,0x00,0x08,0x01,0x00,0x00, -0xfc,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0x06,0x01,0x00,0x00, -0x3d,0x00,0x04,0x00,0xec,0x00,0x00,0x00,0x09,0x01,0x00,0x00, -0x08,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0x0a,0x01,0x00,0x00, -0x0b,0x01,0x00,0x00,0xf1,0x00,0x00,0x00,0xf8,0x00,0x00,0x00, -0x3e,0x00,0x03,0x00,0x0b,0x01,0x00,0x00,0x09,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0xeb,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x0c,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x0f,0x01,0x00,0x00,0x73,0x00,0x00,0x00,0xac,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x11,0x01,0x00,0x00, -0x0f,0x01,0x00,0x00,0x10,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x13,0x01,0x00,0x00,0x11,0x01,0x00,0x00, -0x6e,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x0a,0x01,0x00,0x00, -0x15,0x01,0x00,0x00,0xf1,0x00,0x00,0x00,0x13,0x01,0x00,0x00, -0x3e,0x00,0x03,0x00,0x15,0x01,0x00,0x00,0x14,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0xeb,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xeb,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xd5,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xd5,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x1c,0x01,0x00,0x00,0xac,0x02,0x00,0x00, -0x1a,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xd2,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xd4,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x1e,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x1e,0x01,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xad,0x02,0x00,0x00, -0x3e,0x00,0x00,0x00,0xd4,0x00,0x00,0x00,0x61,0x01,0x00,0x00, -0x21,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00, -0x24,0x01,0x00,0x00,0xad,0x02,0x00,0x00,0x9c,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0x20,0x01,0x00,0x00,0x21,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x24,0x01,0x00,0x00, -0x1f,0x01,0x00,0x00,0x20,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x1f,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x28,0x01,0x00,0x00,0x9d,0x00,0x00,0x00,0x73,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x2a,0x01,0x00,0x00, -0x28,0x01,0x00,0x00,0xad,0x02,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x2b,0x01,0x00,0x00,0x12,0x00,0x00,0x00, -0xc5,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x2c,0x01,0x00,0x00,0x2b,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb7,0x00,0x00,0x00,0x2d,0x01,0x00,0x00,0x2a,0x01,0x00,0x00, -0x2c,0x01,0x00,0x00,0xf7,0x00,0x03,0x00,0x2f,0x01,0x00,0x00, -0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x2d,0x01,0x00,0x00, -0x2e,0x01,0x00,0x00,0x2f,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x2e,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x32,0x01,0x00,0x00,0x9c,0x02,0x00,0x00,0x6e,0x00,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00,0x34,0x01,0x00,0x00, -0x32,0x01,0x00,0x00,0x83,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x2f,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x2f,0x01,0x00,0x00, -0xf5,0x00,0x07,0x00,0xb7,0x00,0x00,0x00,0x35,0x01,0x00,0x00, -0x2d,0x01,0x00,0x00,0x1f,0x01,0x00,0x00,0x34,0x01,0x00,0x00, -0x2e,0x01,0x00,0x00,0xf7,0x00,0x03,0x00,0x37,0x01,0x00,0x00, -0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x35,0x01,0x00,0x00, -0x36,0x01,0x00,0x00,0x57,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x36,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x3f,0x01,0x00,0x00,0x73,0x00,0x00,0x00,0xad,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x41,0x01,0x00,0x00, -0x3f,0x01,0x00,0x00,0x40,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x43,0x01,0x00,0x00,0x41,0x01,0x00,0x00, -0x6e,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x4e,0x01,0x00,0x00,0x3f,0x01,0x00,0x00,0xa0,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x4f,0x01,0x00,0x00, -0xb4,0x02,0x00,0x00,0x4e,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x51,0x01,0x00,0x00,0x4f,0x01,0x00,0x00, -0x6e,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x52,0x01,0x00,0x00, -0x53,0x01,0x00,0x00,0x47,0x01,0x00,0x00,0x34,0x00,0x00,0x00, -0x51,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0xb9,0x00,0x00,0x00, -0x54,0x01,0x00,0x00,0x53,0x01,0x00,0x00,0x73,0x00,0x04,0x00, -0xec,0x00,0x00,0x00,0x55,0x01,0x00,0x00,0x54,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0x0a,0x01,0x00,0x00,0x56,0x01,0x00,0x00, -0x3c,0x01,0x00,0x00,0x43,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0x56,0x01,0x00,0x00,0x55,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0x37,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x57,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x5a,0x01,0x00,0x00, -0x73,0x00,0x00,0x00,0xad,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x5c,0x01,0x00,0x00,0x5a,0x01,0x00,0x00, -0x5b,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x5e,0x01,0x00,0x00,0x5c,0x01,0x00,0x00,0x6e,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x0a,0x01,0x00,0x00,0x5f,0x01,0x00,0x00, -0x3c,0x01,0x00,0x00,0x5e,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0x5f,0x01,0x00,0x00,0x14,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0x37,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x37,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0x21,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x21,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x61,0x01,0x00,0x00,0xad,0x02,0x00,0x00,0x1a,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0x1e,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x20,0x01,0x00,0x00,0xe0,0x00,0x04,0x00,0x0c,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x62,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x65,0x01,0x00,0x00,0xb0,0x02,0x00,0x00, -0x63,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x68,0x01,0x00,0x00,0xb4,0x02,0x00,0x00,0x66,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0x6a,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x6a,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xb6,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0x20,0x01,0x00,0x00, -0x14,0x02,0x00,0x00,0x6d,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb7,0x00,0x00,0x00,0x70,0x01,0x00,0x00,0xb6,0x02,0x00,0x00, -0x6c,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x6c,0x01,0x00,0x00, -0x6d,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x70,0x01,0x00,0x00,0x6b,0x01,0x00,0x00,0x6c,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x6b,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0x72,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x72,0x01,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xba,0x02,0x00,0x00, -0x3e,0x00,0x00,0x00,0x6b,0x01,0x00,0x00,0x9e,0x01,0x00,0x00, -0x75,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00, -0x78,0x01,0x00,0x00,0xba,0x02,0x00,0x00,0x60,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0x74,0x01,0x00,0x00,0x75,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x78,0x01,0x00,0x00, -0x73,0x01,0x00,0x00,0x74,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x73,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x7a,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x7a,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xcc,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, -0x73,0x01,0x00,0x00,0x9c,0x01,0x00,0x00,0x7b,0x01,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00,0x80,0x01,0x00,0x00, -0xcc,0x02,0x00,0x00,0x62,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0x7c,0x01,0x00,0x00,0x7b,0x01,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x80,0x01,0x00,0x00,0x7b,0x01,0x00,0x00, -0x7c,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x7b,0x01,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x86,0x01,0x00,0x00, -0xba,0x02,0x00,0x00,0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x88,0x01,0x00,0x00,0x86,0x01,0x00,0x00, -0xcc,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x8a,0x01,0x00,0x00,0x55,0x00,0x00,0x00,0x53,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8c,0x01,0x00,0x00, -0xba,0x02,0x00,0x00,0x61,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x8d,0x01,0x00,0x00,0x8a,0x01,0x00,0x00, -0x8c,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x8f,0x01,0x00,0x00,0x64,0x00,0x00,0x00,0x62,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x90,0x01,0x00,0x00, -0x8d,0x01,0x00,0x00,0x8f,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x92,0x01,0x00,0x00,0x90,0x01,0x00,0x00, -0xcc,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x94,0x01,0x00,0x00,0x92,0x01,0x00,0x00,0x93,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x96,0x01,0x00,0x00, -0x94,0x01,0x00,0x00,0xb6,0x02,0x00,0x00,0x41,0x00,0x05,0x00, -0x0a,0x01,0x00,0x00,0x97,0x01,0x00,0x00,0xf1,0x00,0x00,0x00, -0x96,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0xec,0x00,0x00,0x00, -0x98,0x01,0x00,0x00,0x97,0x01,0x00,0x00,0x41,0x00,0x05,0x00, -0x99,0x01,0x00,0x00,0x9a,0x01,0x00,0x00,0x84,0x01,0x00,0x00, -0x88,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x9a,0x01,0x00,0x00, -0x98,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x9c,0x01,0x00,0x00,0xcc,0x02,0x00,0x00,0xc5,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x7a,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x7c,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x75,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x75,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x9e,0x01,0x00,0x00,0xba,0x02,0x00,0x00, -0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x72,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x74,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0xa0,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xa0,0x01,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xbb,0x02,0x00,0x00, -0x3e,0x00,0x00,0x00,0x74,0x01,0x00,0x00,0xcc,0x01,0x00,0x00, -0xa3,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00, -0xa6,0x01,0x00,0x00,0xbb,0x02,0x00,0x00,0xb4,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0xa2,0x01,0x00,0x00,0xa3,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xa6,0x01,0x00,0x00, -0xa1,0x01,0x00,0x00,0xa2,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xa1,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xa8,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xa8,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xc9,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, -0xa1,0x01,0x00,0x00,0xca,0x01,0x00,0x00,0xa9,0x01,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00,0xae,0x01,0x00,0x00, -0xc9,0x02,0x00,0x00,0xb1,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0xaa,0x01,0x00,0x00,0xa9,0x01,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xae,0x01,0x00,0x00,0xa9,0x01,0x00,0x00, -0xaa,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xa9,0x01,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xb4,0x01,0x00,0x00, -0xbb,0x02,0x00,0x00,0xb1,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xb6,0x01,0x00,0x00,0xb4,0x01,0x00,0x00, -0xc9,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xb8,0x01,0x00,0x00,0x59,0x00,0x00,0x00,0xae,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xbb,0x01,0x00,0x00, -0xbb,0x02,0x00,0x00,0xba,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xbc,0x01,0x00,0x00,0xb8,0x01,0x00,0x00, -0xbb,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xbe,0x01,0x00,0x00,0x68,0x00,0x00,0x00,0xb1,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xbf,0x01,0x00,0x00, -0xbc,0x01,0x00,0x00,0xbe,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xc1,0x01,0x00,0x00,0xbf,0x01,0x00,0x00, -0xc9,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xc3,0x01,0x00,0x00,0xc1,0x01,0x00,0x00,0xc2,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xc5,0x01,0x00,0x00, -0xc3,0x01,0x00,0x00,0xb6,0x02,0x00,0x00,0x41,0x00,0x05,0x00, -0x0a,0x01,0x00,0x00,0xc6,0x01,0x00,0x00,0x3c,0x01,0x00,0x00, -0xc5,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0xec,0x00,0x00,0x00, -0xc7,0x01,0x00,0x00,0xc6,0x01,0x00,0x00,0x41,0x00,0x05,0x00, -0x99,0x01,0x00,0x00,0xc8,0x01,0x00,0x00,0xb2,0x01,0x00,0x00, -0xb6,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0xc8,0x01,0x00,0x00, -0xc7,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xca,0x01,0x00,0x00,0xc9,0x02,0x00,0x00,0xc5,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xa8,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xaa,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xa3,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xa3,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xcc,0x01,0x00,0x00,0xbb,0x02,0x00,0x00, -0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xa0,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xa2,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0xce,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xce,0x01,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xbc,0x02,0x00,0x00, -0x3e,0x00,0x00,0x00,0xa2,0x01,0x00,0x00,0x12,0x02,0x00,0x00, -0xd1,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00, -0xd4,0x01,0x00,0x00,0xbc,0x02,0x00,0x00,0xb4,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0xd0,0x01,0x00,0x00,0xd1,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xd4,0x01,0x00,0x00, -0xcf,0x01,0x00,0x00,0xd0,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xcf,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xd6,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xd6,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xc0,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, -0xcf,0x01,0x00,0x00,0x10,0x02,0x00,0x00,0xd9,0x01,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00,0xdc,0x01,0x00,0x00, -0xc0,0x02,0x00,0x00,0x60,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0xd8,0x01,0x00,0x00,0xd9,0x01,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xdc,0x01,0x00,0x00,0xd7,0x01,0x00,0x00, -0xd8,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xd7,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0xde,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xde,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xc2,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0xd7,0x01,0x00,0x00, -0x0e,0x02,0x00,0x00,0xe1,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb7,0x00,0x00,0x00,0xe4,0x01,0x00,0x00,0xc2,0x02,0x00,0x00, -0xb1,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xe0,0x01,0x00,0x00, -0xe1,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0xe4,0x01,0x00,0x00,0xdf,0x01,0x00,0x00,0xe0,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xdf,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0xe6,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xe6,0x01,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xc4,0x02,0x00,0x00, -0x3e,0x00,0x00,0x00,0xdf,0x01,0x00,0x00,0x0c,0x02,0x00,0x00, -0xe7,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00, -0xec,0x01,0x00,0x00,0xc4,0x02,0x00,0x00,0x62,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0xe8,0x01,0x00,0x00,0xe7,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xec,0x01,0x00,0x00, -0xe7,0x01,0x00,0x00,0xe8,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xe7,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xee,0x01,0x00,0x00,0xbc,0x02,0x00,0x00,0xb1,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xf0,0x01,0x00,0x00, -0xee,0x01,0x00,0x00,0xc2,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xf2,0x01,0x00,0x00,0xf0,0x01,0x00,0x00, -0xf1,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xf4,0x01,0x00,0x00,0xc0,0x02,0x00,0x00,0x62,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xf5,0x01,0x00,0x00, -0xf2,0x01,0x00,0x00,0xf4,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xf7,0x01,0x00,0x00,0xf5,0x01,0x00,0x00, -0xc4,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xfb,0x01,0x00,0x00,0xf4,0x01,0x00,0x00,0xc4,0x02,0x00,0x00, -0x41,0x00,0x05,0x00,0x99,0x01,0x00,0x00,0xfc,0x01,0x00,0x00, -0x84,0x01,0x00,0x00,0xfb,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, -0xec,0x00,0x00,0x00,0xfd,0x01,0x00,0x00,0xfc,0x01,0x00,0x00, -0x73,0x00,0x04,0x00,0xb9,0x00,0x00,0x00,0xfe,0x01,0x00,0x00, -0xfd,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0x99,0x01,0x00,0x00, -0x03,0x02,0x00,0x00,0xb2,0x01,0x00,0x00,0xf0,0x01,0x00,0x00, -0x3d,0x00,0x04,0x00,0xec,0x00,0x00,0x00,0x04,0x02,0x00,0x00, -0x03,0x02,0x00,0x00,0x73,0x00,0x04,0x00,0xb9,0x00,0x00,0x00, -0x05,0x02,0x00,0x00,0x04,0x02,0x00,0x00,0x41,0x00,0x05,0x00, -0xc2,0x00,0x00,0x00,0x07,0x02,0x00,0x00,0xbf,0x00,0x00,0x00, -0xf7,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0xb9,0x00,0x00,0x00, -0x08,0x02,0x00,0x00,0x07,0x02,0x00,0x00,0x0c,0x00,0x08,0x00, -0xb9,0x00,0x00,0x00,0x09,0x02,0x00,0x00,0x01,0x00,0x00,0x00, -0x32,0x00,0x00,0x00,0xfe,0x01,0x00,0x00,0x05,0x02,0x00,0x00, -0x08,0x02,0x00,0x00,0x3e,0x00,0x03,0x00,0x07,0x02,0x00,0x00, -0x09,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x0c,0x02,0x00,0x00,0xc4,0x02,0x00,0x00,0xc5,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xe6,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xe8,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xe1,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xe1,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x0e,0x02,0x00,0x00,0xc2,0x02,0x00,0x00, -0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xde,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xe0,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0xd9,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xd9,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x10,0x02,0x00,0x00, -0xc0,0x02,0x00,0x00,0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xd6,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xd8,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0xd1,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xd1,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x12,0x02,0x00,0x00,0xbc,0x02,0x00,0x00,0xc5,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xce,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xd0,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x6d,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x6d,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x14,0x02,0x00,0x00,0xb6,0x02,0x00,0x00, -0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x6a,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x6c,0x01,0x00,0x00,0xe0,0x00,0x04,0x00, -0x0c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x62,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0xcc,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xcc,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x16,0x02,0x00,0x00,0x9c,0x02,0x00,0x00,0x6c,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xc9,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xcb,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x1b,0x02,0x00,0x00,0x55,0x00,0x00,0x00,0x53,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x1c,0x02,0x00,0x00, -0x8b,0x00,0x00,0x00,0x1b,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x21,0x02,0x00,0x00,0x59,0x00,0x00,0x00, -0xae,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x22,0x02,0x00,0x00,0x9d,0x00,0x00,0x00,0x21,0x02,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x27,0x02,0x00,0x00, -0x12,0x00,0x00,0x00,0x26,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x28,0x02,0x00,0x00,0x27,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x29,0x02,0x00,0x00, -0x0f,0x00,0x00,0x00,0x28,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x2d,0x02,0x00,0x00,0x47,0x00,0x00,0x00, -0x28,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00, -0x2f,0x02,0x00,0x00,0x2e,0x02,0x00,0x00,0x0c,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x30,0x02,0x00,0x00, -0x2f,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x31,0x02,0x00,0x00,0x2d,0x02,0x00,0x00,0x30,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x32,0x02,0x00,0x00, -0x29,0x02,0x00,0x00,0x31,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x34,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x34,0x02,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x9d,0x02,0x00,0x00, -0x3e,0x00,0x00,0x00,0xcb,0x00,0x00,0x00,0x9a,0x02,0x00,0x00, -0x37,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00, -0x3a,0x02,0x00,0x00,0x9d,0x02,0x00,0x00,0xb4,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0x36,0x02,0x00,0x00,0x37,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x3a,0x02,0x00,0x00, -0x35,0x02,0x00,0x00,0x36,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x35,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x3c,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x3c,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0x9e,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, -0x35,0x02,0x00,0x00,0x98,0x02,0x00,0x00,0x3f,0x02,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00,0x42,0x02,0x00,0x00, -0x9e,0x02,0x00,0x00,0x60,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0x3e,0x02,0x00,0x00,0x3f,0x02,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x42,0x02,0x00,0x00,0x3d,0x02,0x00,0x00, -0x3e,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x3d,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x46,0x02,0x00,0x00, -0x9e,0x02,0x00,0x00,0x61,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x47,0x02,0x00,0x00,0x1c,0x02,0x00,0x00, -0x46,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x49,0x02,0x00,0x00,0x64,0x00,0x00,0x00,0x62,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x4a,0x02,0x00,0x00, -0x47,0x02,0x00,0x00,0x49,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x4e,0x02,0x00,0x00,0x9d,0x02,0x00,0x00, -0xba,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x4f,0x02,0x00,0x00,0x22,0x02,0x00,0x00,0x4e,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x51,0x02,0x00,0x00, -0x68,0x00,0x00,0x00,0xb1,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x52,0x02,0x00,0x00,0x4f,0x02,0x00,0x00, -0x51,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x54,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x54,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xa0,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, -0x3d,0x02,0x00,0x00,0x96,0x02,0x00,0x00,0x57,0x02,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00,0x5a,0x02,0x00,0x00, -0xa0,0x02,0x00,0x00,0xb1,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0x56,0x02,0x00,0x00,0x57,0x02,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x5a,0x02,0x00,0x00,0x55,0x02,0x00,0x00, -0x56,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x55,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0x5c,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x5c,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xa2,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0x55,0x02,0x00,0x00, -0x94,0x02,0x00,0x00,0x5f,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb7,0x00,0x00,0x00,0x62,0x02,0x00,0x00,0xa2,0x02,0x00,0x00, -0x62,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x5e,0x02,0x00,0x00, -0x5f,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x62,0x02,0x00,0x00,0x5d,0x02,0x00,0x00,0x5e,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x5d,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x65,0x02,0x00,0x00,0x4a,0x02,0x00,0x00, -0xa2,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00, -0x68,0x02,0x00,0x00,0x65,0x02,0x00,0x00,0x36,0x00,0x00,0x00, -0xf7,0x00,0x03,0x00,0x6a,0x02,0x00,0x00,0x00,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x68,0x02,0x00,0x00,0x69,0x02,0x00,0x00, -0x6a,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x69,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x6d,0x02,0x00,0x00, -0x52,0x02,0x00,0x00,0xa0,0x02,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x6e,0x02,0x00,0x00,0x12,0x00,0x00,0x00, -0xc5,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x6f,0x02,0x00,0x00,0x6e,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb7,0x00,0x00,0x00,0x70,0x02,0x00,0x00,0x6d,0x02,0x00,0x00, -0x6f,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x6a,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x6a,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, -0xb7,0x00,0x00,0x00,0x71,0x02,0x00,0x00,0x68,0x02,0x00,0x00, -0x5d,0x02,0x00,0x00,0x70,0x02,0x00,0x00,0x69,0x02,0x00,0x00, -0xf7,0x00,0x03,0x00,0x73,0x02,0x00,0x00,0x00,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x71,0x02,0x00,0x00,0x72,0x02,0x00,0x00, -0x73,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x72,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x7b,0x02,0x00,0x00, -0x52,0x02,0x00,0x00,0xa0,0x02,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x7d,0x02,0x00,0x00,0x12,0x00,0x00,0x00, -0x7c,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x7e,0x02,0x00,0x00,0x7d,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x7f,0x02,0x00,0x00,0x7b,0x02,0x00,0x00, -0x7e,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x80,0x02,0x00,0x00,0x32,0x02,0x00,0x00,0x7f,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x82,0x02,0x00,0x00, -0x80,0x02,0x00,0x00,0x4a,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x84,0x02,0x00,0x00,0x82,0x02,0x00,0x00, -0xa2,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x86,0x02,0x00,0x00,0x9d,0x02,0x00,0x00,0xb1,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x88,0x02,0x00,0x00, -0x86,0x02,0x00,0x00,0xa0,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x8a,0x02,0x00,0x00,0x88,0x02,0x00,0x00, -0x89,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x8c,0x02,0x00,0x00,0x9e,0x02,0x00,0x00,0x62,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8d,0x02,0x00,0x00, -0x8a,0x02,0x00,0x00,0x8c,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x8f,0x02,0x00,0x00,0x8d,0x02,0x00,0x00, -0xa2,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, -0x90,0x02,0x00,0x00,0xbf,0x00,0x00,0x00,0x8f,0x02,0x00,0x00, -0x3d,0x00,0x04,0x00,0xb9,0x00,0x00,0x00,0x91,0x02,0x00,0x00, -0x90,0x02,0x00,0x00,0x41,0x00,0x06,0x00,0x52,0x01,0x00,0x00, -0x92,0x02,0x00,0x00,0x77,0x02,0x00,0x00,0x34,0x00,0x00,0x00, -0x84,0x02,0x00,0x00,0x3e,0x00,0x03,0x00,0x92,0x02,0x00,0x00, -0x91,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x73,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x73,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x5f,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x5f,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x94,0x02,0x00,0x00, -0xa2,0x02,0x00,0x00,0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x5c,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x5e,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0x57,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x57,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x96,0x02,0x00,0x00,0xa0,0x02,0x00,0x00,0xc5,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x54,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x56,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x3f,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x3f,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x98,0x02,0x00,0x00,0x9e,0x02,0x00,0x00, -0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x3c,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x3e,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x37,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x37,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9a,0x02,0x00,0x00, -0x9d,0x02,0x00,0x00,0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x34,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x36,0x02,0x00,0x00, -0xfd,0x00,0x01,0x00,0x38,0x00,0x01,0x00, -}; -const uint64_t matmul_f16_f32_m_len = 10172; - -unsigned char matmul_f16_f32_m_fp32_data[] = { -0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, -0xc9,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, -0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x51,0x11,0x00,0x00, -0x0b,0x00,0x06,0x00,0x01,0x00,0x00,0x00,0x47,0x4c,0x53,0x4c, -0x2e,0x73,0x74,0x64,0x2e,0x34,0x35,0x30,0x00,0x00,0x00,0x00, -0x0e,0x00,0x03,0x00,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x0f,0x00,0x0f,0x00,0x05,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x6d,0x61,0x69,0x6e,0x00,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x3d,0x00,0x00,0x00,0x4c,0x00,0x00,0x00, -0xf0,0x00,0x00,0x00,0xfc,0x00,0x00,0x00,0x3c,0x01,0x00,0x00, -0x47,0x01,0x00,0x00,0x2a,0x02,0x00,0x00,0x73,0x02,0x00,0x00, -0x10,0x00,0x06,0x00,0x04,0x00,0x00,0x00,0x11,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x0b,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x1c,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x10,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x08,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x03,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x10,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x14,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x18,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x07,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x10,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x24,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x0a,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x10,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x2c,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x30,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x0d,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0x47,0x00,0x03,0x00, -0x10,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x37,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x3d,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x1a,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x4c,0x00,0x00,0x00, -0x0b,0x00,0x00,0x00,0x1b,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x4f,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x53,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x60,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x62,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x07,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x6c,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x03,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x9c,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0xae,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x05,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0xb1,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x08,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xf9,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x48,0x00,0x04,0x00, -0xfa,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0xfa,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, -0xfa,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0xfc,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0xfc,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x16,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x17,0x01,0x00,0x00,0x0b,0x00,0x00,0x00,0x19,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x44,0x01,0x00,0x00,0x06,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0x45,0x01,0x00,0x00, -0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x45,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x45,0x01,0x00,0x00, -0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x47,0x01,0x00,0x00, -0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x47,0x01,0x00,0x00,0x21,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x2a,0x02,0x00,0x00,0x0b,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x70,0x02,0x00,0x00, -0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00, -0x71,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x19,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x71,0x02,0x00,0x00,0x00,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, -0x71,0x02,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x73,0x02,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x73,0x02,0x00,0x00,0x21,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x13,0x00,0x02,0x00,0x02,0x00,0x00,0x00, -0x21,0x00,0x03,0x00,0x03,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x15,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x17,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x0a,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x0d,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x1e,0x00,0x10,0x00,0x10,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x11,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x11,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x15,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, -0x14,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x15,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x0a,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, -0x27,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0x2d,0x00,0x00,0x00,0x07,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x34,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x37,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00,0x3d,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x3e,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x0a,0x00,0x00,0x00,0x4c,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x4f,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x53,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x54,0x00,0x00,0x00,0x86,0x00,0x00,0x00, -0x37,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x58,0x00,0x00,0x00,0x86,0x00,0x00,0x00, -0x37,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x32,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x61,0x00,0x00,0x00, -0x86,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x60,0x00,0x00,0x00, -0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x62,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x63,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x61,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x67,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x61,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x86,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x72,0x00,0x00,0x00,0x86,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0x76,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x7b,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, -0x86,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0x8c,0x00,0x00,0x00,0x03,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x97,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x9c,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0x9e,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xad,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00, -0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xae,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xaf,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x53,0x00,0x00,0x00, -0xae,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xb0,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x4f,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xb1,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xb2,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0xb0,0x00,0x00,0x00,0xb1,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xb3,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0xb2,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xb4,0x00,0x00,0x00,0x86,0x00,0x00,0x00, -0xaf,0x00,0x00,0x00,0xb3,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xb5,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0xad,0x00,0x00,0x00,0xb4,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xb6,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0xb5,0x00,0x00,0x00,0xb1,0x00,0x00,0x00,0x14,0x00,0x02,0x00, -0xb7,0x00,0x00,0x00,0x16,0x00,0x03,0x00,0xb9,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xba,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xbb,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0xba,0x00,0x00,0x00, -0xb4,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xbc,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0xbb,0x00,0x00,0x00, -0xb1,0x00,0x00,0x00,0x1c,0x00,0x04,0x00,0xbd,0x00,0x00,0x00, -0xb9,0x00,0x00,0x00,0xbc,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0xbe,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0xbd,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0xb9,0x00,0x00,0x00,0xc1,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0xc2,0x00,0x00,0x00, -0x07,0x00,0x00,0x00,0xb9,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0xc5,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xec,0x00,0x00,0x00, -0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xed,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0xec,0x00,0x00,0x00, -0x1c,0x00,0x04,0x00,0xee,0x00,0x00,0x00,0xb9,0x00,0x00,0x00, -0xed,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0xef,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0xee,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0xef,0x00,0x00,0x00,0xf0,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xf4,0x00,0x00,0x00, -0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x16,0x00,0x03,0x00,0xf8,0x00,0x00,0x00,0x10,0x00,0x00,0x00, -0x1d,0x00,0x03,0x00,0xf9,0x00,0x00,0x00,0xf8,0x00,0x00,0x00, -0x1e,0x00,0x03,0x00,0xfa,0x00,0x00,0x00,0xf9,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0xfb,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0xfa,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0xfb,0x00,0x00,0x00, -0xfc,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x07,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0xf8,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x0b,0x01,0x00,0x00,0x04,0x00,0x00,0x00, -0xb9,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x11,0x01,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x16,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x33,0x00,0x06,0x00, -0x09,0x00,0x00,0x00,0x17,0x01,0x00,0x00,0x16,0x01,0x00,0x00, -0x39,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x18,0x01,0x00,0x00,0x51,0x00,0x00,0x00, -0x17,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x19,0x01,0x00,0x00,0x84,0x00,0x00,0x00, -0x18,0x01,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x1a,0x01,0x00,0x00,0x86,0x00,0x00,0x00, -0x19,0x01,0x00,0x00,0x6c,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x38,0x01,0x00,0x00,0x80,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x39,0x01,0x00,0x00,0x84,0x00,0x00,0x00, -0x9c,0x00,0x00,0x00,0x38,0x01,0x00,0x00,0x1c,0x00,0x04,0x00, -0x3a,0x01,0x00,0x00,0xb9,0x00,0x00,0x00,0x39,0x01,0x00,0x00, -0x20,0x00,0x04,0x00,0x3b,0x01,0x00,0x00,0x04,0x00,0x00,0x00, -0x3a,0x01,0x00,0x00,0x3b,0x00,0x04,0x00,0x3b,0x01,0x00,0x00, -0x3c,0x01,0x00,0x00,0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x40,0x01,0x00,0x00,0x80,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, -0x44,0x01,0x00,0x00,0xb9,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, -0x45,0x01,0x00,0x00,0x44,0x01,0x00,0x00,0x20,0x00,0x04,0x00, -0x46,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x45,0x01,0x00,0x00, -0x3b,0x00,0x04,0x00,0x46,0x01,0x00,0x00,0x47,0x01,0x00,0x00, -0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x52,0x01,0x00,0x00, -0x0c,0x00,0x00,0x00,0xb9,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x5a,0x01,0x00,0x00,0x80,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x61,0x01,0x00,0x00,0x08,0x01,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x62,0x01,0x00,0x00, -0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x65,0x01,0x00,0x00, -0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x80,0x01,0x00,0x00, -0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00, -0x1c,0x00,0x04,0x00,0x81,0x01,0x00,0x00,0xb9,0x00,0x00,0x00, -0x80,0x01,0x00,0x00,0x20,0x00,0x04,0x00,0x82,0x01,0x00,0x00, -0x07,0x00,0x00,0x00,0x81,0x01,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x92,0x01,0x00,0x00,0x80,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xad,0x01,0x00,0x00,0x84,0x00,0x00,0x00, -0xb4,0x00,0x00,0x00,0xb1,0x00,0x00,0x00,0x1c,0x00,0x04,0x00, -0xae,0x01,0x00,0x00,0xb9,0x00,0x00,0x00,0xad,0x01,0x00,0x00, -0x20,0x00,0x04,0x00,0xaf,0x01,0x00,0x00,0x07,0x00,0x00,0x00, -0xae,0x01,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xb8,0x01,0x00,0x00,0x86,0x00,0x00,0x00,0xae,0x00,0x00,0x00, -0xb4,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xc0,0x01,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xef,0x01,0x00,0x00,0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, -0x22,0x02,0x00,0x00,0x0d,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x0a,0x00,0x00,0x00,0x2a,0x02,0x00,0x00,0x01,0x00,0x00,0x00, -0x1d,0x00,0x03,0x00,0x70,0x02,0x00,0x00,0xb9,0x00,0x00,0x00, -0x1e,0x00,0x03,0x00,0x71,0x02,0x00,0x00,0x70,0x02,0x00,0x00, -0x20,0x00,0x04,0x00,0x72,0x02,0x00,0x00,0x0c,0x00,0x00,0x00, -0x71,0x02,0x00,0x00,0x3b,0x00,0x04,0x00,0x72,0x02,0x00,0x00, -0x73,0x02,0x00,0x00,0x0c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0x78,0x02,0x00,0x00,0x05,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x85,0x02,0x00,0x00, -0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00, -0x36,0x00,0x05,0x00,0x02,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x05,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0xbe,0x00,0x00,0x00, -0xbf,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x82,0x01,0x00,0x00,0x83,0x01,0x00,0x00,0x07,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0xaf,0x01,0x00,0x00,0xb0,0x01,0x00,0x00, -0x07,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00, -0x0e,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, -0x0e,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x14,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x17,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x17,0x00,0x00,0x00, -0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x1e,0x00,0x00,0x00, -0x0f,0x00,0x00,0x00,0x17,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x21,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x24,0x00,0x00,0x00,0x18,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0x28,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x27,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x29,0x00,0x00,0x00, -0x28,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x2a,0x00,0x00,0x00,0x1e,0x00,0x00,0x00,0x29,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x2d,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x2f,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x30,0x00,0x00,0x00, -0x24,0x00,0x00,0x00,0x2f,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x30,0x00,0x00,0x00, -0x2a,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0x35,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x34,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x36,0x00,0x00,0x00, -0x35,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x38,0x00,0x00,0x00,0x36,0x00,0x00,0x00,0x37,0x00,0x00,0x00, -0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, -0x38,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x3b,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, -0x37,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00, -0x3f,0x00,0x00,0x00,0x3d,0x00,0x00,0x00,0x3e,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x3f,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x42,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x3b,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x47,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0x3b,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x0d,0x00,0x00,0x00,0x49,0x00,0x00,0x00,0x3d,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x4a,0x00,0x00,0x00,0x49,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x0d,0x00,0x00,0x00,0x4d,0x00,0x00,0x00,0x4c,0x00,0x00,0x00, -0x3e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x4e,0x00,0x00,0x00,0x4d,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x50,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, -0x4f,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x55,0x00,0x00,0x00,0x50,0x00,0x00,0x00,0x54,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x59,0x00,0x00,0x00, -0x50,0x00,0x00,0x00,0x58,0x00,0x00,0x00,0x89,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x5d,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, -0x4f,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x64,0x00,0x00,0x00,0x5d,0x00,0x00,0x00,0x63,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x68,0x00,0x00,0x00, -0x5d,0x00,0x00,0x00,0x67,0x00,0x00,0x00,0x89,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x6e,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, -0x6d,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x73,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x72,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x77,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x76,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x78,0x00,0x00,0x00,0x77,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x79,0x00,0x00,0x00, -0x47,0x00,0x00,0x00,0x78,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x7c,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x7b,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x7d,0x00,0x00,0x00,0x7c,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x7f,0x00,0x00,0x00,0x47,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x82,0x00,0x00,0x00,0x7f,0x00,0x00,0x00,0x78,0x00,0x00,0x00, -0x0c,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x83,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x26,0x00,0x00,0x00,0x7d,0x00,0x00,0x00, -0x82,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0x87,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x86,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x88,0x00,0x00,0x00, -0x87,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x89,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x88,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8b,0x00,0x00,0x00, -0x42,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x8d,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x8c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x8e,0x00,0x00,0x00,0x8d,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x8f,0x00,0x00,0x00,0x8b,0x00,0x00,0x00, -0x8e,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x90,0x00,0x00,0x00,0x89,0x00,0x00,0x00,0x8f,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x92,0x00,0x00,0x00, -0x90,0x00,0x00,0x00,0x79,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x93,0x00,0x00,0x00,0x92,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0x98,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x97,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x99,0x00,0x00,0x00, -0x98,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x9a,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x99,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9d,0x00,0x00,0x00, -0x4a,0x00,0x00,0x00,0x9c,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x9f,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x9e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xa0,0x00,0x00,0x00,0x9f,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xa1,0x00,0x00,0x00,0x9d,0x00,0x00,0x00, -0xa0,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xa2,0x00,0x00,0x00,0x9a,0x00,0x00,0x00,0xa1,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa4,0x00,0x00,0x00, -0xa2,0x00,0x00,0x00,0x79,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xa5,0x00,0x00,0x00,0xa4,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xa7,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xa7,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0x97,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, -0x05,0x00,0x00,0x00,0xc6,0x00,0x00,0x00,0xa8,0x00,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00,0xb8,0x00,0x00,0x00, -0x97,0x02,0x00,0x00,0xb6,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0xa9,0x00,0x00,0x00,0xa8,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xb8,0x00,0x00,0x00,0xa8,0x00,0x00,0x00, -0xa9,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xa8,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0xc3,0x00,0x00,0x00, -0xbf,0x00,0x00,0x00,0x97,0x02,0x00,0x00,0x3e,0x00,0x03,0x00, -0xc3,0x00,0x00,0x00,0xc1,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xc6,0x00,0x00,0x00,0x97,0x02,0x00,0x00, -0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xa7,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xa9,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xc9,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xc9,0x00,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xb0,0x02,0x00,0x00, -0xa5,0x00,0x00,0x00,0xa9,0x00,0x00,0x00,0x67,0x01,0x00,0x00, -0xcc,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xac,0x02,0x00,0x00,0x93,0x00,0x00,0x00,0xa9,0x00,0x00,0x00, -0x64,0x01,0x00,0x00,0xcc,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0x98,0x02,0x00,0x00,0x79,0x00,0x00,0x00, -0xa9,0x00,0x00,0x00,0x12,0x02,0x00,0x00,0xcc,0x00,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00,0xd0,0x00,0x00,0x00, -0x98,0x02,0x00,0x00,0x83,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0xcb,0x00,0x00,0x00,0xcc,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xd0,0x00,0x00,0x00,0xca,0x00,0x00,0x00, -0xcb,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xca,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xd2,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xd2,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xa8,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0xca,0x00,0x00,0x00, -0x1c,0x01,0x00,0x00,0xd5,0x00,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb7,0x00,0x00,0x00,0xd8,0x00,0x00,0x00,0xa8,0x02,0x00,0x00, -0x37,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xd4,0x00,0x00,0x00, -0xd5,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0xd8,0x00,0x00,0x00,0xd3,0x00,0x00,0x00,0xd4,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xd3,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xdc,0x00,0x00,0x00,0x8b,0x00,0x00,0x00, -0x73,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xde,0x00,0x00,0x00,0xdc,0x00,0x00,0x00,0xa8,0x02,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00,0xe1,0x00,0x00,0x00, -0xde,0x00,0x00,0x00,0x36,0x00,0x00,0x00,0xf7,0x00,0x03,0x00, -0xe3,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0xe1,0x00,0x00,0x00,0xe2,0x00,0x00,0x00,0xe3,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xe2,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xe6,0x00,0x00,0x00,0x98,0x02,0x00,0x00, -0x6e,0x00,0x00,0x00,0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00, -0xe8,0x00,0x00,0x00,0xe6,0x00,0x00,0x00,0x83,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xe3,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xe3,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0xb7,0x00,0x00,0x00, -0xe9,0x00,0x00,0x00,0xe1,0x00,0x00,0x00,0xd3,0x00,0x00,0x00, -0xe8,0x00,0x00,0x00,0xe2,0x00,0x00,0x00,0xf7,0x00,0x03,0x00, -0xeb,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0xe9,0x00,0x00,0x00,0xea,0x00,0x00,0x00,0x0d,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xea,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xf3,0x00,0x00,0x00,0x73,0x00,0x00,0x00, -0xa8,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xf5,0x00,0x00,0x00,0xf3,0x00,0x00,0x00,0xf4,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xf7,0x00,0x00,0x00, -0xf5,0x00,0x00,0x00,0x6e,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x03,0x01,0x00,0x00,0xf3,0x00,0x00,0x00, -0x8e,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x04,0x01,0x00,0x00,0xac,0x02,0x00,0x00,0x03,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x06,0x01,0x00,0x00, -0x04,0x01,0x00,0x00,0x6e,0x00,0x00,0x00,0x41,0x00,0x06,0x00, -0x07,0x01,0x00,0x00,0x08,0x01,0x00,0x00,0xfc,0x00,0x00,0x00, -0x34,0x00,0x00,0x00,0x06,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, -0xf8,0x00,0x00,0x00,0x09,0x01,0x00,0x00,0x08,0x01,0x00,0x00, -0x73,0x00,0x04,0x00,0xb9,0x00,0x00,0x00,0x0a,0x01,0x00,0x00, -0x09,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0x0b,0x01,0x00,0x00, -0x0c,0x01,0x00,0x00,0xf0,0x00,0x00,0x00,0xf7,0x00,0x00,0x00, -0x3e,0x00,0x03,0x00,0x0c,0x01,0x00,0x00,0x0a,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0xeb,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x0d,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x10,0x01,0x00,0x00,0x73,0x00,0x00,0x00,0xa8,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x12,0x01,0x00,0x00, -0x10,0x01,0x00,0x00,0x11,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x14,0x01,0x00,0x00,0x12,0x01,0x00,0x00, -0x6e,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x0b,0x01,0x00,0x00, -0x15,0x01,0x00,0x00,0xf0,0x00,0x00,0x00,0x14,0x01,0x00,0x00, -0x3e,0x00,0x03,0x00,0x15,0x01,0x00,0x00,0xc1,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xeb,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xeb,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xd5,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xd5,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x1c,0x01,0x00,0x00,0xa8,0x02,0x00,0x00, -0x1a,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xd2,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xd4,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x1e,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x1e,0x01,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xa9,0x02,0x00,0x00, -0x3e,0x00,0x00,0x00,0xd4,0x00,0x00,0x00,0x60,0x01,0x00,0x00, -0x21,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00, -0x24,0x01,0x00,0x00,0xa9,0x02,0x00,0x00,0x9c,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0x20,0x01,0x00,0x00,0x21,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x24,0x01,0x00,0x00, -0x1f,0x01,0x00,0x00,0x20,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x1f,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x28,0x01,0x00,0x00,0x9d,0x00,0x00,0x00,0x73,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x2a,0x01,0x00,0x00, -0x28,0x01,0x00,0x00,0xa9,0x02,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x2b,0x01,0x00,0x00,0x12,0x00,0x00,0x00, -0xc5,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x2c,0x01,0x00,0x00,0x2b,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb7,0x00,0x00,0x00,0x2d,0x01,0x00,0x00,0x2a,0x01,0x00,0x00, -0x2c,0x01,0x00,0x00,0xf7,0x00,0x03,0x00,0x2f,0x01,0x00,0x00, -0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x2d,0x01,0x00,0x00, -0x2e,0x01,0x00,0x00,0x2f,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x2e,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x32,0x01,0x00,0x00,0x98,0x02,0x00,0x00,0x6e,0x00,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00,0x34,0x01,0x00,0x00, -0x32,0x01,0x00,0x00,0x83,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x2f,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x2f,0x01,0x00,0x00, -0xf5,0x00,0x07,0x00,0xb7,0x00,0x00,0x00,0x35,0x01,0x00,0x00, -0x2d,0x01,0x00,0x00,0x1f,0x01,0x00,0x00,0x34,0x01,0x00,0x00, -0x2e,0x01,0x00,0x00,0xf7,0x00,0x03,0x00,0x37,0x01,0x00,0x00, -0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x35,0x01,0x00,0x00, -0x36,0x01,0x00,0x00,0x56,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x36,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x3f,0x01,0x00,0x00,0x73,0x00,0x00,0x00,0xa9,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x41,0x01,0x00,0x00, -0x3f,0x01,0x00,0x00,0x40,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x43,0x01,0x00,0x00,0x41,0x01,0x00,0x00, -0x6e,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x4e,0x01,0x00,0x00,0x3f,0x01,0x00,0x00,0xa0,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x4f,0x01,0x00,0x00, -0xb0,0x02,0x00,0x00,0x4e,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x51,0x01,0x00,0x00,0x4f,0x01,0x00,0x00, -0x6e,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x52,0x01,0x00,0x00, -0x53,0x01,0x00,0x00,0x47,0x01,0x00,0x00,0x34,0x00,0x00,0x00, -0x51,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0xb9,0x00,0x00,0x00, -0x54,0x01,0x00,0x00,0x53,0x01,0x00,0x00,0x41,0x00,0x05,0x00, -0x0b,0x01,0x00,0x00,0x55,0x01,0x00,0x00,0x3c,0x01,0x00,0x00, -0x43,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x55,0x01,0x00,0x00, -0x54,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x37,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x56,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x59,0x01,0x00,0x00,0x73,0x00,0x00,0x00, -0xa9,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x5b,0x01,0x00,0x00,0x59,0x01,0x00,0x00,0x5a,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x5d,0x01,0x00,0x00, -0x5b,0x01,0x00,0x00,0x6e,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x0b,0x01,0x00,0x00,0x5e,0x01,0x00,0x00,0x3c,0x01,0x00,0x00, -0x5d,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x5e,0x01,0x00,0x00, -0xc1,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x37,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x37,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0x21,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x21,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x60,0x01,0x00,0x00, -0xa9,0x02,0x00,0x00,0x1a,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0x1e,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x20,0x01,0x00,0x00, -0xe0,0x00,0x04,0x00,0x0c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x61,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x64,0x01,0x00,0x00,0xac,0x02,0x00,0x00,0x62,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x67,0x01,0x00,0x00, -0xb0,0x02,0x00,0x00,0x65,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0x69,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x69,0x01,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xb2,0x02,0x00,0x00, -0x3e,0x00,0x00,0x00,0x20,0x01,0x00,0x00,0x10,0x02,0x00,0x00, -0x6c,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00, -0x6f,0x01,0x00,0x00,0xb2,0x02,0x00,0x00,0x6c,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0x6b,0x01,0x00,0x00,0x6c,0x01,0x00,0x00, -0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x6f,0x01,0x00,0x00, -0x6a,0x01,0x00,0x00,0x6b,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x6a,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x71,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x71,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xb6,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, -0x6a,0x01,0x00,0x00,0x9c,0x01,0x00,0x00,0x74,0x01,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00,0x77,0x01,0x00,0x00, -0xb6,0x02,0x00,0x00,0x60,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0x73,0x01,0x00,0x00,0x74,0x01,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x77,0x01,0x00,0x00,0x72,0x01,0x00,0x00, -0x73,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x72,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0x79,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x79,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xc8,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0x72,0x01,0x00,0x00, -0x9a,0x01,0x00,0x00,0x7a,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb7,0x00,0x00,0x00,0x7f,0x01,0x00,0x00,0xc8,0x02,0x00,0x00, -0x62,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x7b,0x01,0x00,0x00, -0x7a,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x7f,0x01,0x00,0x00,0x7a,0x01,0x00,0x00,0x7b,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x7a,0x01,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x85,0x01,0x00,0x00,0xb6,0x02,0x00,0x00, -0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x87,0x01,0x00,0x00,0x85,0x01,0x00,0x00,0xc8,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x89,0x01,0x00,0x00, -0x55,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x8b,0x01,0x00,0x00,0xb6,0x02,0x00,0x00, -0x61,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x8c,0x01,0x00,0x00,0x89,0x01,0x00,0x00,0x8b,0x01,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8e,0x01,0x00,0x00, -0x64,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x8f,0x01,0x00,0x00,0x8c,0x01,0x00,0x00, -0x8e,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x91,0x01,0x00,0x00,0x8f,0x01,0x00,0x00,0xc8,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x93,0x01,0x00,0x00, -0x91,0x01,0x00,0x00,0x92,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x95,0x01,0x00,0x00,0x93,0x01,0x00,0x00, -0xb2,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x0b,0x01,0x00,0x00, -0x96,0x01,0x00,0x00,0xf0,0x00,0x00,0x00,0x95,0x01,0x00,0x00, -0x3d,0x00,0x04,0x00,0xb9,0x00,0x00,0x00,0x97,0x01,0x00,0x00, -0x96,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, -0x98,0x01,0x00,0x00,0x83,0x01,0x00,0x00,0x87,0x01,0x00,0x00, -0x3e,0x00,0x03,0x00,0x98,0x01,0x00,0x00,0x97,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9a,0x01,0x00,0x00, -0xc8,0x02,0x00,0x00,0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x79,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x7b,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0x74,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x74,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x9c,0x01,0x00,0x00,0xb6,0x02,0x00,0x00,0xc5,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x71,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x73,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x9e,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x9e,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xb7,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, -0x73,0x01,0x00,0x00,0xca,0x01,0x00,0x00,0xa1,0x01,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00,0xa4,0x01,0x00,0x00, -0xb7,0x02,0x00,0x00,0xb4,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0xa0,0x01,0x00,0x00,0xa1,0x01,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xa4,0x01,0x00,0x00,0x9f,0x01,0x00,0x00, -0xa0,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x9f,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0xa6,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xa6,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xc5,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0x9f,0x01,0x00,0x00, -0xc8,0x01,0x00,0x00,0xa7,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb7,0x00,0x00,0x00,0xac,0x01,0x00,0x00,0xc5,0x02,0x00,0x00, -0xb1,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xa8,0x01,0x00,0x00, -0xa7,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0xac,0x01,0x00,0x00,0xa7,0x01,0x00,0x00,0xa8,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xa7,0x01,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xb2,0x01,0x00,0x00,0xb7,0x02,0x00,0x00, -0xb1,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xb4,0x01,0x00,0x00,0xb2,0x01,0x00,0x00,0xc5,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xb6,0x01,0x00,0x00, -0x59,0x00,0x00,0x00,0xae,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xb9,0x01,0x00,0x00,0xb7,0x02,0x00,0x00, -0xb8,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xba,0x01,0x00,0x00,0xb6,0x01,0x00,0x00,0xb9,0x01,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xbc,0x01,0x00,0x00, -0x68,0x00,0x00,0x00,0xb1,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xbd,0x01,0x00,0x00,0xba,0x01,0x00,0x00, -0xbc,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xbf,0x01,0x00,0x00,0xbd,0x01,0x00,0x00,0xc5,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xc1,0x01,0x00,0x00, -0xbf,0x01,0x00,0x00,0xc0,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xc3,0x01,0x00,0x00,0xc1,0x01,0x00,0x00, -0xb2,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x0b,0x01,0x00,0x00, -0xc4,0x01,0x00,0x00,0x3c,0x01,0x00,0x00,0xc3,0x01,0x00,0x00, -0x3d,0x00,0x04,0x00,0xb9,0x00,0x00,0x00,0xc5,0x01,0x00,0x00, -0xc4,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, -0xc6,0x01,0x00,0x00,0xb0,0x01,0x00,0x00,0xb4,0x01,0x00,0x00, -0x3e,0x00,0x03,0x00,0xc6,0x01,0x00,0x00,0xc5,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xc8,0x01,0x00,0x00, -0xc5,0x02,0x00,0x00,0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xa6,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xa8,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0xa1,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xa1,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xca,0x01,0x00,0x00,0xb7,0x02,0x00,0x00,0xc5,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x9e,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xa0,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xcc,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xcc,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xb8,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, -0xa0,0x01,0x00,0x00,0x0e,0x02,0x00,0x00,0xcf,0x01,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00,0xd2,0x01,0x00,0x00, -0xb8,0x02,0x00,0x00,0xb4,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0xce,0x01,0x00,0x00,0xcf,0x01,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xd2,0x01,0x00,0x00,0xcd,0x01,0x00,0x00, -0xce,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xcd,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0xd4,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xd4,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xbc,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0xcd,0x01,0x00,0x00, -0x0c,0x02,0x00,0x00,0xd7,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb7,0x00,0x00,0x00,0xda,0x01,0x00,0x00,0xbc,0x02,0x00,0x00, -0x60,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xd6,0x01,0x00,0x00, -0xd7,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0xda,0x01,0x00,0x00,0xd5,0x01,0x00,0x00,0xd6,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xd5,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0xdc,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xdc,0x01,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xbe,0x02,0x00,0x00, -0x3e,0x00,0x00,0x00,0xd5,0x01,0x00,0x00,0x0a,0x02,0x00,0x00, -0xdf,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00, -0xe2,0x01,0x00,0x00,0xbe,0x02,0x00,0x00,0xb1,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0xde,0x01,0x00,0x00,0xdf,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xe2,0x01,0x00,0x00, -0xdd,0x01,0x00,0x00,0xde,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xdd,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xe4,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xe4,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xc0,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, -0xdd,0x01,0x00,0x00,0x08,0x02,0x00,0x00,0xe5,0x01,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00,0xea,0x01,0x00,0x00, -0xc0,0x02,0x00,0x00,0x62,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0xe6,0x01,0x00,0x00,0xe5,0x01,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xea,0x01,0x00,0x00,0xe5,0x01,0x00,0x00, -0xe6,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xe5,0x01,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xec,0x01,0x00,0x00, -0xb8,0x02,0x00,0x00,0xb1,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xee,0x01,0x00,0x00,0xec,0x01,0x00,0x00, -0xbe,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xf0,0x01,0x00,0x00,0xee,0x01,0x00,0x00,0xef,0x01,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xf2,0x01,0x00,0x00, -0xbc,0x02,0x00,0x00,0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xf3,0x01,0x00,0x00,0xf0,0x01,0x00,0x00, -0xf2,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xf5,0x01,0x00,0x00,0xf3,0x01,0x00,0x00,0xc0,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xf9,0x01,0x00,0x00, -0xf2,0x01,0x00,0x00,0xc0,0x02,0x00,0x00,0x41,0x00,0x05,0x00, -0xc2,0x00,0x00,0x00,0xfa,0x01,0x00,0x00,0x83,0x01,0x00,0x00, -0xf9,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0xb9,0x00,0x00,0x00, -0xfb,0x01,0x00,0x00,0xfa,0x01,0x00,0x00,0x41,0x00,0x05,0x00, -0xc2,0x00,0x00,0x00,0x00,0x02,0x00,0x00,0xb0,0x01,0x00,0x00, -0xee,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0xb9,0x00,0x00,0x00, -0x01,0x02,0x00,0x00,0x00,0x02,0x00,0x00,0x41,0x00,0x05,0x00, -0xc2,0x00,0x00,0x00,0x03,0x02,0x00,0x00,0xbf,0x00,0x00,0x00, -0xf5,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0xb9,0x00,0x00,0x00, -0x04,0x02,0x00,0x00,0x03,0x02,0x00,0x00,0x0c,0x00,0x08,0x00, -0xb9,0x00,0x00,0x00,0x05,0x02,0x00,0x00,0x01,0x00,0x00,0x00, -0x32,0x00,0x00,0x00,0xfb,0x01,0x00,0x00,0x01,0x02,0x00,0x00, -0x04,0x02,0x00,0x00,0x3e,0x00,0x03,0x00,0x03,0x02,0x00,0x00, -0x05,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x08,0x02,0x00,0x00,0xc0,0x02,0x00,0x00,0xc5,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xe4,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xe6,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xdf,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xdf,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x0a,0x02,0x00,0x00,0xbe,0x02,0x00,0x00, -0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xdc,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xde,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0xd7,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xd7,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x0c,0x02,0x00,0x00, -0xbc,0x02,0x00,0x00,0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xd4,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xd6,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0xcf,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xcf,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x0e,0x02,0x00,0x00,0xb8,0x02,0x00,0x00,0xc5,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xcc,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xce,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x6c,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x6c,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x10,0x02,0x00,0x00,0xb2,0x02,0x00,0x00, -0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x69,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x6b,0x01,0x00,0x00,0xe0,0x00,0x04,0x00, -0x0c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x61,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0xcc,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xcc,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x12,0x02,0x00,0x00,0x98,0x02,0x00,0x00,0x6c,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xc9,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xcb,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x17,0x02,0x00,0x00,0x55,0x00,0x00,0x00,0x53,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x18,0x02,0x00,0x00, -0x8b,0x00,0x00,0x00,0x17,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x1d,0x02,0x00,0x00,0x59,0x00,0x00,0x00, -0xae,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x1e,0x02,0x00,0x00,0x9d,0x00,0x00,0x00,0x1d,0x02,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x23,0x02,0x00,0x00, -0x12,0x00,0x00,0x00,0x22,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x24,0x02,0x00,0x00,0x23,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x25,0x02,0x00,0x00, -0x0f,0x00,0x00,0x00,0x24,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x29,0x02,0x00,0x00,0x47,0x00,0x00,0x00, -0x24,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00, -0x2b,0x02,0x00,0x00,0x2a,0x02,0x00,0x00,0x0c,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x2c,0x02,0x00,0x00, -0x2b,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x2d,0x02,0x00,0x00,0x29,0x02,0x00,0x00,0x2c,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x2e,0x02,0x00,0x00, -0x25,0x02,0x00,0x00,0x2d,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x30,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x30,0x02,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x99,0x02,0x00,0x00, -0x3e,0x00,0x00,0x00,0xcb,0x00,0x00,0x00,0x96,0x02,0x00,0x00, -0x33,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00, -0x36,0x02,0x00,0x00,0x99,0x02,0x00,0x00,0xb4,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0x32,0x02,0x00,0x00,0x33,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x36,0x02,0x00,0x00, -0x31,0x02,0x00,0x00,0x32,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x31,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x38,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x38,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0x9a,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, -0x31,0x02,0x00,0x00,0x94,0x02,0x00,0x00,0x3b,0x02,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00,0x3e,0x02,0x00,0x00, -0x9a,0x02,0x00,0x00,0x60,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0x3a,0x02,0x00,0x00,0x3b,0x02,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x3e,0x02,0x00,0x00,0x39,0x02,0x00,0x00, -0x3a,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x39,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x42,0x02,0x00,0x00, -0x9a,0x02,0x00,0x00,0x61,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x43,0x02,0x00,0x00,0x18,0x02,0x00,0x00, -0x42,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x45,0x02,0x00,0x00,0x64,0x00,0x00,0x00,0x62,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x46,0x02,0x00,0x00, -0x43,0x02,0x00,0x00,0x45,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x4a,0x02,0x00,0x00,0x99,0x02,0x00,0x00, -0xb8,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x4b,0x02,0x00,0x00,0x1e,0x02,0x00,0x00,0x4a,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x4d,0x02,0x00,0x00, -0x68,0x00,0x00,0x00,0xb1,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x4e,0x02,0x00,0x00,0x4b,0x02,0x00,0x00, -0x4d,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x50,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x50,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0x9c,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, -0x39,0x02,0x00,0x00,0x92,0x02,0x00,0x00,0x53,0x02,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00,0x56,0x02,0x00,0x00, -0x9c,0x02,0x00,0x00,0xb1,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0x52,0x02,0x00,0x00,0x53,0x02,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x56,0x02,0x00,0x00,0x51,0x02,0x00,0x00, -0x52,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x51,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0x58,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x58,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0x9e,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0x51,0x02,0x00,0x00, -0x90,0x02,0x00,0x00,0x5b,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb7,0x00,0x00,0x00,0x5e,0x02,0x00,0x00,0x9e,0x02,0x00,0x00, -0x62,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x5a,0x02,0x00,0x00, -0x5b,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x5e,0x02,0x00,0x00,0x59,0x02,0x00,0x00,0x5a,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x59,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x61,0x02,0x00,0x00,0x46,0x02,0x00,0x00, -0x9e,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00, -0x64,0x02,0x00,0x00,0x61,0x02,0x00,0x00,0x36,0x00,0x00,0x00, -0xf7,0x00,0x03,0x00,0x66,0x02,0x00,0x00,0x00,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x64,0x02,0x00,0x00,0x65,0x02,0x00,0x00, -0x66,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x65,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x69,0x02,0x00,0x00, -0x4e,0x02,0x00,0x00,0x9c,0x02,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x6a,0x02,0x00,0x00,0x12,0x00,0x00,0x00, -0xc5,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x6b,0x02,0x00,0x00,0x6a,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb7,0x00,0x00,0x00,0x6c,0x02,0x00,0x00,0x69,0x02,0x00,0x00, -0x6b,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x66,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x66,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, -0xb7,0x00,0x00,0x00,0x6d,0x02,0x00,0x00,0x64,0x02,0x00,0x00, -0x59,0x02,0x00,0x00,0x6c,0x02,0x00,0x00,0x65,0x02,0x00,0x00, -0xf7,0x00,0x03,0x00,0x6f,0x02,0x00,0x00,0x00,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x6d,0x02,0x00,0x00,0x6e,0x02,0x00,0x00, -0x6f,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x6e,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x77,0x02,0x00,0x00, -0x4e,0x02,0x00,0x00,0x9c,0x02,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x79,0x02,0x00,0x00,0x12,0x00,0x00,0x00, -0x78,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x7a,0x02,0x00,0x00,0x79,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x7b,0x02,0x00,0x00,0x77,0x02,0x00,0x00, -0x7a,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x7c,0x02,0x00,0x00,0x2e,0x02,0x00,0x00,0x7b,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x7e,0x02,0x00,0x00, -0x7c,0x02,0x00,0x00,0x46,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x80,0x02,0x00,0x00,0x7e,0x02,0x00,0x00, -0x9e,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x82,0x02,0x00,0x00,0x99,0x02,0x00,0x00,0xb1,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x84,0x02,0x00,0x00, -0x82,0x02,0x00,0x00,0x9c,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x86,0x02,0x00,0x00,0x84,0x02,0x00,0x00, -0x85,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x88,0x02,0x00,0x00,0x9a,0x02,0x00,0x00,0x62,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x89,0x02,0x00,0x00, -0x86,0x02,0x00,0x00,0x88,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x8b,0x02,0x00,0x00,0x89,0x02,0x00,0x00, -0x9e,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, -0x8c,0x02,0x00,0x00,0xbf,0x00,0x00,0x00,0x8b,0x02,0x00,0x00, -0x3d,0x00,0x04,0x00,0xb9,0x00,0x00,0x00,0x8d,0x02,0x00,0x00, -0x8c,0x02,0x00,0x00,0x41,0x00,0x06,0x00,0x52,0x01,0x00,0x00, -0x8e,0x02,0x00,0x00,0x73,0x02,0x00,0x00,0x34,0x00,0x00,0x00, -0x80,0x02,0x00,0x00,0x3e,0x00,0x03,0x00,0x8e,0x02,0x00,0x00, -0x8d,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x6f,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x6f,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x5b,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x5b,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x90,0x02,0x00,0x00, -0x9e,0x02,0x00,0x00,0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x58,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x5a,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0x53,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x53,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x92,0x02,0x00,0x00,0x9c,0x02,0x00,0x00,0xc5,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x50,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x52,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x3b,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x3b,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x94,0x02,0x00,0x00,0x9a,0x02,0x00,0x00, -0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x38,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x3a,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x33,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x33,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x96,0x02,0x00,0x00, -0x99,0x02,0x00,0x00,0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x30,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x32,0x02,0x00,0x00, -0xfd,0x00,0x01,0x00,0x38,0x00,0x01,0x00, -}; -const uint64_t matmul_f16_f32_m_fp32_len = 10100; - -unsigned char matmul_f16_f32_s_data[] = { -0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, -0xcd,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, -0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x09,0x00,0x00,0x00, -0x11,0x00,0x02,0x00,0x51,0x11,0x00,0x00,0x0b,0x00,0x06,0x00, -0x01,0x00,0x00,0x00,0x47,0x4c,0x53,0x4c,0x2e,0x73,0x74,0x64, -0x2e,0x34,0x35,0x30,0x00,0x00,0x00,0x00,0x0e,0x00,0x03,0x00, -0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x0f,0x00,0x0f,0x00, -0x05,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x6d,0x61,0x69,0x6e, -0x00,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x3d,0x00,0x00,0x00,0x4c,0x00,0x00,0x00,0xf1,0x00,0x00,0x00, -0xfc,0x00,0x00,0x00,0x3c,0x01,0x00,0x00,0x47,0x01,0x00,0x00, -0x2e,0x02,0x00,0x00,0x77,0x02,0x00,0x00,0x10,0x00,0x06,0x00, -0x04,0x00,0x00,0x00,0x11,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x0b,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x1c,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x10,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x08,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x03,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x10,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x10,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x05,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x14,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x10,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x1c,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x08,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x24,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x10,0x00,0x00,0x00,0x0a,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x28,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x0b,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x2c,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x30,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x10,0x00,0x00,0x00,0x0d,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x34,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x10,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x37,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x3d,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x1a,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x4c,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x1b,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x4f,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x53,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x60,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x62,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x6c,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x03,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x9c,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xae,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0xb1,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x08,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0xf9,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0xfa,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0xfa,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0xfa,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xfc,0x00,0x00,0x00, -0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0xfc,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x16,0x01,0x00,0x00,0x01,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x17,0x01,0x00,0x00, -0x0b,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x44,0x01,0x00,0x00,0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x48,0x00,0x04,0x00,0x45,0x01,0x00,0x00,0x00,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x45,0x01,0x00,0x00, -0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x03,0x00,0x45,0x01,0x00,0x00,0x02,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x47,0x01,0x00,0x00,0x22,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x47,0x01,0x00,0x00, -0x21,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x2e,0x02,0x00,0x00,0x0b,0x00,0x00,0x00,0x18,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x74,0x02,0x00,0x00,0x06,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0x75,0x02,0x00,0x00, -0x00,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x75,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x75,0x02,0x00,0x00, -0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x77,0x02,0x00,0x00, -0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x77,0x02,0x00,0x00,0x21,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x13,0x00,0x02,0x00,0x02,0x00,0x00,0x00,0x21,0x00,0x03,0x00, -0x03,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x15,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x17,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x03,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x0a,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x0a,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x0d,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x1e,0x00,0x10,0x00, -0x10,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x11,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x10,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x11,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x15,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x14,0x00,0x00,0x00, -0x08,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x15,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x0a,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x27,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, -0x2d,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x37,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x0a,0x00,0x00,0x00,0x3d,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x3e,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00, -0x4c,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x53,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x54,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x37,0x00,0x00,0x00, -0x53,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x58,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x37,0x00,0x00,0x00, -0x53,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x60,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x61,0x00,0x00,0x00,0x86,0x00,0x00,0x00, -0x53,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x32,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x63,0x00,0x00,0x00, -0x86,0x00,0x00,0x00,0x61,0x00,0x00,0x00,0x62,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x67,0x00,0x00,0x00, -0x86,0x00,0x00,0x00,0x61,0x00,0x00,0x00,0x62,0x00,0x00,0x00, -0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x10,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x6d,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x72,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, -0x76,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0x7b,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x86,0x00,0x00,0x00, -0x0b,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, -0x8c,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0x97,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x9c,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, -0x9e,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xad,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x32,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xae,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xaf,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0xae,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xb0,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x62,0x00,0x00,0x00, -0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xb1,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xb2,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0xb0,0x00,0x00,0x00, -0xb1,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xb3,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0xb2,0x00,0x00,0x00, -0x60,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xb4,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0xaf,0x00,0x00,0x00, -0xb3,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xb5,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0xad,0x00,0x00,0x00, -0xb4,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xb6,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0xb5,0x00,0x00,0x00, -0xb1,0x00,0x00,0x00,0x14,0x00,0x02,0x00,0xb7,0x00,0x00,0x00, -0x16,0x00,0x03,0x00,0xb9,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xba,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xbb,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0xba,0x00,0x00,0x00,0xb4,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xbc,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0xbb,0x00,0x00,0x00,0xb1,0x00,0x00,0x00, -0x1c,0x00,0x04,0x00,0xbd,0x00,0x00,0x00,0xb9,0x00,0x00,0x00, -0xbc,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0xbe,0x00,0x00,0x00, -0x07,0x00,0x00,0x00,0xbd,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0xb9,0x00,0x00,0x00,0xc1,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0xc2,0x00,0x00,0x00,0x07,0x00,0x00,0x00, -0xb9,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, -0xc5,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x16,0x00,0x03,0x00, -0xec,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xed,0x00,0x00,0x00,0x80,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xee,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0x37,0x00,0x00,0x00,0xed,0x00,0x00,0x00,0x1c,0x00,0x04,0x00, -0xef,0x00,0x00,0x00,0xec,0x00,0x00,0x00,0xee,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0xf0,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0xef,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0xf0,0x00,0x00,0x00, -0xf1,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xf5,0x00,0x00,0x00,0x80,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, -0xf9,0x00,0x00,0x00,0xec,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, -0xfa,0x00,0x00,0x00,0xf9,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0xfb,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0xfa,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0xfb,0x00,0x00,0x00,0xfc,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x07,0x01,0x00,0x00, -0x0c,0x00,0x00,0x00,0xec,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x0a,0x01,0x00,0x00,0x04,0x00,0x00,0x00,0xec,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x10,0x01,0x00,0x00, -0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0xec,0x00,0x00,0x00,0x14,0x01,0x00,0x00, -0x00,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x16,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x33,0x00,0x06,0x00, -0x09,0x00,0x00,0x00,0x17,0x01,0x00,0x00,0x16,0x01,0x00,0x00, -0x39,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x18,0x01,0x00,0x00,0x51,0x00,0x00,0x00, -0x17,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x19,0x01,0x00,0x00,0x84,0x00,0x00,0x00, -0x18,0x01,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x1a,0x01,0x00,0x00,0x86,0x00,0x00,0x00, -0x19,0x01,0x00,0x00,0x6c,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x38,0x01,0x00,0x00,0x80,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x39,0x01,0x00,0x00,0x84,0x00,0x00,0x00, -0x9c,0x00,0x00,0x00,0x38,0x01,0x00,0x00,0x1c,0x00,0x04,0x00, -0x3a,0x01,0x00,0x00,0xec,0x00,0x00,0x00,0x39,0x01,0x00,0x00, -0x20,0x00,0x04,0x00,0x3b,0x01,0x00,0x00,0x04,0x00,0x00,0x00, -0x3a,0x01,0x00,0x00,0x3b,0x00,0x04,0x00,0x3b,0x01,0x00,0x00, -0x3c,0x01,0x00,0x00,0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x40,0x01,0x00,0x00,0x80,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, -0x44,0x01,0x00,0x00,0xb9,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, -0x45,0x01,0x00,0x00,0x44,0x01,0x00,0x00,0x20,0x00,0x04,0x00, -0x46,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x45,0x01,0x00,0x00, -0x3b,0x00,0x04,0x00,0x46,0x01,0x00,0x00,0x47,0x01,0x00,0x00, -0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x52,0x01,0x00,0x00, -0x0c,0x00,0x00,0x00,0xb9,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x5b,0x01,0x00,0x00,0x80,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x62,0x01,0x00,0x00,0x08,0x01,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x63,0x01,0x00,0x00, -0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x66,0x01,0x00,0x00, -0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x81,0x01,0x00,0x00, -0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00, -0x1c,0x00,0x04,0x00,0x82,0x01,0x00,0x00,0xec,0x00,0x00,0x00, -0x81,0x01,0x00,0x00,0x20,0x00,0x04,0x00,0x83,0x01,0x00,0x00, -0x07,0x00,0x00,0x00,0x82,0x01,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x93,0x01,0x00,0x00,0x80,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x99,0x01,0x00,0x00,0x07,0x00,0x00,0x00,0xec,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xaf,0x01,0x00,0x00, -0x84,0x00,0x00,0x00,0xb4,0x00,0x00,0x00,0xb1,0x00,0x00,0x00, -0x1c,0x00,0x04,0x00,0xb0,0x01,0x00,0x00,0xec,0x00,0x00,0x00, -0xaf,0x01,0x00,0x00,0x20,0x00,0x04,0x00,0xb1,0x01,0x00,0x00, -0x07,0x00,0x00,0x00,0xb0,0x01,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xba,0x01,0x00,0x00,0x86,0x00,0x00,0x00, -0xae,0x00,0x00,0x00,0xb4,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xc2,0x01,0x00,0x00,0x80,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xf1,0x01,0x00,0x00,0x84,0x00,0x00,0x00, -0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0x26,0x02,0x00,0x00,0x0d,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00,0x2e,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0x1d,0x00,0x03,0x00,0x74,0x02,0x00,0x00, -0xb9,0x00,0x00,0x00,0x1e,0x00,0x03,0x00,0x75,0x02,0x00,0x00, -0x74,0x02,0x00,0x00,0x20,0x00,0x04,0x00,0x76,0x02,0x00,0x00, -0x0c,0x00,0x00,0x00,0x75,0x02,0x00,0x00,0x3b,0x00,0x04,0x00, -0x76,0x02,0x00,0x00,0x77,0x02,0x00,0x00,0x0c,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x7c,0x02,0x00,0x00, -0x05,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x89,0x02,0x00,0x00,0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x36,0x00,0x05,0x00,0x02,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x03,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0x05,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0xbe,0x00,0x00,0x00,0xbf,0x00,0x00,0x00,0x07,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x83,0x01,0x00,0x00,0x84,0x01,0x00,0x00, -0x07,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0xb1,0x01,0x00,0x00, -0xb2,0x01,0x00,0x00,0x07,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x0d,0x00,0x00,0x00,0x0e,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x0f,0x00,0x00,0x00,0x0e,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x14,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x17,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, -0x17,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x1e,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x17,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x22,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x22,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x24,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x27,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x29,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x2a,0x00,0x00,0x00,0x1e,0x00,0x00,0x00, -0x29,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x2d,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x2f,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x30,0x00,0x00,0x00,0x24,0x00,0x00,0x00,0x2f,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x32,0x00,0x00,0x00, -0x30,0x00,0x00,0x00,0x2a,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x35,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x34,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x36,0x00,0x00,0x00,0x35,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x38,0x00,0x00,0x00,0x36,0x00,0x00,0x00, -0x37,0x00,0x00,0x00,0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x3a,0x00,0x00,0x00,0x38,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3b,0x00,0x00,0x00, -0x3a,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x0d,0x00,0x00,0x00,0x3f,0x00,0x00,0x00,0x3d,0x00,0x00,0x00, -0x3e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0x3f,0x00,0x00,0x00,0x89,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x42,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x3b,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x47,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x3b,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00,0x49,0x00,0x00,0x00, -0x3d,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x4a,0x00,0x00,0x00,0x49,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00,0x4d,0x00,0x00,0x00, -0x4c,0x00,0x00,0x00,0x3e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x4d,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x50,0x00,0x00,0x00, -0x4e,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x89,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x55,0x00,0x00,0x00,0x50,0x00,0x00,0x00, -0x54,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x59,0x00,0x00,0x00,0x50,0x00,0x00,0x00,0x58,0x00,0x00,0x00, -0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x5d,0x00,0x00,0x00, -0x4e,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x89,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x64,0x00,0x00,0x00,0x5d,0x00,0x00,0x00, -0x63,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x68,0x00,0x00,0x00,0x5d,0x00,0x00,0x00,0x67,0x00,0x00,0x00, -0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x6e,0x00,0x00,0x00, -0x4e,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x73,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, -0x72,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0x77,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x76,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x78,0x00,0x00,0x00, -0x77,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x79,0x00,0x00,0x00,0x47,0x00,0x00,0x00,0x78,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x7c,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x7b,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x7d,0x00,0x00,0x00,0x7c,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x7f,0x00,0x00,0x00, -0x47,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x82,0x00,0x00,0x00,0x7f,0x00,0x00,0x00, -0x78,0x00,0x00,0x00,0x0c,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0x83,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x26,0x00,0x00,0x00, -0x7d,0x00,0x00,0x00,0x82,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x87,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x86,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x88,0x00,0x00,0x00,0x87,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x89,0x00,0x00,0x00,0x32,0x00,0x00,0x00, -0x88,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x8b,0x00,0x00,0x00,0x42,0x00,0x00,0x00,0x37,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x8d,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x8c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x8e,0x00,0x00,0x00,0x8d,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8f,0x00,0x00,0x00, -0x8b,0x00,0x00,0x00,0x8e,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x90,0x00,0x00,0x00,0x89,0x00,0x00,0x00, -0x8f,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x92,0x00,0x00,0x00,0x90,0x00,0x00,0x00,0x79,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x93,0x00,0x00,0x00, -0x92,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x98,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x97,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x99,0x00,0x00,0x00,0x98,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x9a,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, -0x99,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x9d,0x00,0x00,0x00,0x4a,0x00,0x00,0x00,0x9c,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x9f,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x9e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xa0,0x00,0x00,0x00,0x9f,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa1,0x00,0x00,0x00, -0x9d,0x00,0x00,0x00,0xa0,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xa2,0x00,0x00,0x00,0x9a,0x00,0x00,0x00, -0xa1,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xa4,0x00,0x00,0x00,0xa2,0x00,0x00,0x00,0x79,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa5,0x00,0x00,0x00, -0xa4,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xa7,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xa7,0x00,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x9b,0x02,0x00,0x00, -0x3e,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0xc6,0x00,0x00,0x00, -0xa8,0x00,0x00,0x00,0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00, -0xb8,0x00,0x00,0x00,0x9b,0x02,0x00,0x00,0xb6,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0xa9,0x00,0x00,0x00,0xa8,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xb8,0x00,0x00,0x00, -0xa8,0x00,0x00,0x00,0xa9,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xa8,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, -0xc3,0x00,0x00,0x00,0xbf,0x00,0x00,0x00,0x9b,0x02,0x00,0x00, -0x3e,0x00,0x03,0x00,0xc3,0x00,0x00,0x00,0xc1,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xc6,0x00,0x00,0x00, -0x9b,0x02,0x00,0x00,0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xa7,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xa9,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xc9,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xc9,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xb4,0x02,0x00,0x00,0xa5,0x00,0x00,0x00,0xa9,0x00,0x00,0x00, -0x68,0x01,0x00,0x00,0xcc,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xb0,0x02,0x00,0x00,0x93,0x00,0x00,0x00, -0xa9,0x00,0x00,0x00,0x65,0x01,0x00,0x00,0xcc,0x00,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x9c,0x02,0x00,0x00, -0x79,0x00,0x00,0x00,0xa9,0x00,0x00,0x00,0x16,0x02,0x00,0x00, -0xcc,0x00,0x00,0x00,0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00, -0xd0,0x00,0x00,0x00,0x9c,0x02,0x00,0x00,0x83,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0xcb,0x00,0x00,0x00,0xcc,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xd0,0x00,0x00,0x00, -0xca,0x00,0x00,0x00,0xcb,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xca,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xd2,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xd2,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xac,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, -0xca,0x00,0x00,0x00,0x1c,0x01,0x00,0x00,0xd5,0x00,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00,0xd8,0x00,0x00,0x00, -0xac,0x02,0x00,0x00,0x37,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0xd4,0x00,0x00,0x00,0xd5,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xd8,0x00,0x00,0x00,0xd3,0x00,0x00,0x00, -0xd4,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xd3,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xdc,0x00,0x00,0x00, -0x8b,0x00,0x00,0x00,0x73,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xde,0x00,0x00,0x00,0xdc,0x00,0x00,0x00, -0xac,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00, -0xe1,0x00,0x00,0x00,0xde,0x00,0x00,0x00,0x36,0x00,0x00,0x00, -0xf7,0x00,0x03,0x00,0xe3,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xe1,0x00,0x00,0x00,0xe2,0x00,0x00,0x00, -0xe3,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xe2,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xe6,0x00,0x00,0x00, -0x9c,0x02,0x00,0x00,0x6e,0x00,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb7,0x00,0x00,0x00,0xe8,0x00,0x00,0x00,0xe6,0x00,0x00,0x00, -0x83,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xe3,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xe3,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, -0xb7,0x00,0x00,0x00,0xe9,0x00,0x00,0x00,0xe1,0x00,0x00,0x00, -0xd3,0x00,0x00,0x00,0xe8,0x00,0x00,0x00,0xe2,0x00,0x00,0x00, -0xf7,0x00,0x03,0x00,0xeb,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xe9,0x00,0x00,0x00,0xea,0x00,0x00,0x00, -0x0c,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xea,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xf4,0x00,0x00,0x00, -0x73,0x00,0x00,0x00,0xac,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xf6,0x00,0x00,0x00,0xf4,0x00,0x00,0x00, -0xf5,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xf8,0x00,0x00,0x00,0xf6,0x00,0x00,0x00,0x6e,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x03,0x01,0x00,0x00, -0xf4,0x00,0x00,0x00,0x8e,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x04,0x01,0x00,0x00,0xb0,0x02,0x00,0x00, -0x03,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x06,0x01,0x00,0x00,0x04,0x01,0x00,0x00,0x6e,0x00,0x00,0x00, -0x41,0x00,0x06,0x00,0x07,0x01,0x00,0x00,0x08,0x01,0x00,0x00, -0xfc,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0x06,0x01,0x00,0x00, -0x3d,0x00,0x04,0x00,0xec,0x00,0x00,0x00,0x09,0x01,0x00,0x00, -0x08,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0x0a,0x01,0x00,0x00, -0x0b,0x01,0x00,0x00,0xf1,0x00,0x00,0x00,0xf8,0x00,0x00,0x00, -0x3e,0x00,0x03,0x00,0x0b,0x01,0x00,0x00,0x09,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0xeb,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x0c,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x0f,0x01,0x00,0x00,0x73,0x00,0x00,0x00,0xac,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x11,0x01,0x00,0x00, -0x0f,0x01,0x00,0x00,0x10,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x13,0x01,0x00,0x00,0x11,0x01,0x00,0x00, -0x6e,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x0a,0x01,0x00,0x00, -0x15,0x01,0x00,0x00,0xf1,0x00,0x00,0x00,0x13,0x01,0x00,0x00, -0x3e,0x00,0x03,0x00,0x15,0x01,0x00,0x00,0x14,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0xeb,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xeb,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xd5,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xd5,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x1c,0x01,0x00,0x00,0xac,0x02,0x00,0x00, -0x1a,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xd2,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xd4,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x1e,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x1e,0x01,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xad,0x02,0x00,0x00, -0x3e,0x00,0x00,0x00,0xd4,0x00,0x00,0x00,0x61,0x01,0x00,0x00, -0x21,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00, -0x24,0x01,0x00,0x00,0xad,0x02,0x00,0x00,0x9c,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0x20,0x01,0x00,0x00,0x21,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x24,0x01,0x00,0x00, -0x1f,0x01,0x00,0x00,0x20,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x1f,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x28,0x01,0x00,0x00,0x9d,0x00,0x00,0x00,0x73,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x2a,0x01,0x00,0x00, -0x28,0x01,0x00,0x00,0xad,0x02,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x2b,0x01,0x00,0x00,0x12,0x00,0x00,0x00, -0xc5,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x2c,0x01,0x00,0x00,0x2b,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb7,0x00,0x00,0x00,0x2d,0x01,0x00,0x00,0x2a,0x01,0x00,0x00, -0x2c,0x01,0x00,0x00,0xf7,0x00,0x03,0x00,0x2f,0x01,0x00,0x00, -0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x2d,0x01,0x00,0x00, -0x2e,0x01,0x00,0x00,0x2f,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x2e,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x32,0x01,0x00,0x00,0x9c,0x02,0x00,0x00,0x6e,0x00,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00,0x34,0x01,0x00,0x00, -0x32,0x01,0x00,0x00,0x83,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x2f,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x2f,0x01,0x00,0x00, -0xf5,0x00,0x07,0x00,0xb7,0x00,0x00,0x00,0x35,0x01,0x00,0x00, -0x2d,0x01,0x00,0x00,0x1f,0x01,0x00,0x00,0x34,0x01,0x00,0x00, -0x2e,0x01,0x00,0x00,0xf7,0x00,0x03,0x00,0x37,0x01,0x00,0x00, -0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x35,0x01,0x00,0x00, -0x36,0x01,0x00,0x00,0x57,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x36,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x3f,0x01,0x00,0x00,0x73,0x00,0x00,0x00,0xad,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x41,0x01,0x00,0x00, -0x3f,0x01,0x00,0x00,0x40,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x43,0x01,0x00,0x00,0x41,0x01,0x00,0x00, -0x6e,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x4e,0x01,0x00,0x00,0x3f,0x01,0x00,0x00,0xa0,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x4f,0x01,0x00,0x00, -0xb4,0x02,0x00,0x00,0x4e,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x51,0x01,0x00,0x00,0x4f,0x01,0x00,0x00, -0x6e,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x52,0x01,0x00,0x00, -0x53,0x01,0x00,0x00,0x47,0x01,0x00,0x00,0x34,0x00,0x00,0x00, -0x51,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0xb9,0x00,0x00,0x00, -0x54,0x01,0x00,0x00,0x53,0x01,0x00,0x00,0x73,0x00,0x04,0x00, -0xec,0x00,0x00,0x00,0x55,0x01,0x00,0x00,0x54,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0x0a,0x01,0x00,0x00,0x56,0x01,0x00,0x00, -0x3c,0x01,0x00,0x00,0x43,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0x56,0x01,0x00,0x00,0x55,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0x37,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x57,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x5a,0x01,0x00,0x00, -0x73,0x00,0x00,0x00,0xad,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x5c,0x01,0x00,0x00,0x5a,0x01,0x00,0x00, -0x5b,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x5e,0x01,0x00,0x00,0x5c,0x01,0x00,0x00,0x6e,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x0a,0x01,0x00,0x00,0x5f,0x01,0x00,0x00, -0x3c,0x01,0x00,0x00,0x5e,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0x5f,0x01,0x00,0x00,0x14,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0x37,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x37,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0x21,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x21,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x61,0x01,0x00,0x00,0xad,0x02,0x00,0x00,0x1a,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0x1e,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x20,0x01,0x00,0x00,0xe0,0x00,0x04,0x00,0x0c,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x62,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x65,0x01,0x00,0x00,0xb0,0x02,0x00,0x00, -0x63,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x68,0x01,0x00,0x00,0xb4,0x02,0x00,0x00,0x66,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0x6a,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x6a,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xb6,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0x20,0x01,0x00,0x00, -0x14,0x02,0x00,0x00,0x6d,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb7,0x00,0x00,0x00,0x70,0x01,0x00,0x00,0xb6,0x02,0x00,0x00, -0x6c,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x6c,0x01,0x00,0x00, -0x6d,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x70,0x01,0x00,0x00,0x6b,0x01,0x00,0x00,0x6c,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x6b,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0x72,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x72,0x01,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xba,0x02,0x00,0x00, -0x3e,0x00,0x00,0x00,0x6b,0x01,0x00,0x00,0x9e,0x01,0x00,0x00, -0x75,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00, -0x78,0x01,0x00,0x00,0xba,0x02,0x00,0x00,0x60,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0x74,0x01,0x00,0x00,0x75,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x78,0x01,0x00,0x00, -0x73,0x01,0x00,0x00,0x74,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x73,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x7a,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x7a,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xcc,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, -0x73,0x01,0x00,0x00,0x9c,0x01,0x00,0x00,0x7b,0x01,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00,0x80,0x01,0x00,0x00, -0xcc,0x02,0x00,0x00,0x62,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0x7c,0x01,0x00,0x00,0x7b,0x01,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x80,0x01,0x00,0x00,0x7b,0x01,0x00,0x00, -0x7c,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x7b,0x01,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x86,0x01,0x00,0x00, -0xba,0x02,0x00,0x00,0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x88,0x01,0x00,0x00,0x86,0x01,0x00,0x00, -0xcc,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x8a,0x01,0x00,0x00,0x55,0x00,0x00,0x00,0x53,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8c,0x01,0x00,0x00, -0xba,0x02,0x00,0x00,0x61,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x8d,0x01,0x00,0x00,0x8a,0x01,0x00,0x00, -0x8c,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x8f,0x01,0x00,0x00,0x64,0x00,0x00,0x00,0x62,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x90,0x01,0x00,0x00, -0x8d,0x01,0x00,0x00,0x8f,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x92,0x01,0x00,0x00,0x90,0x01,0x00,0x00, -0xcc,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x94,0x01,0x00,0x00,0x92,0x01,0x00,0x00,0x93,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x96,0x01,0x00,0x00, -0x94,0x01,0x00,0x00,0xb6,0x02,0x00,0x00,0x41,0x00,0x05,0x00, -0x0a,0x01,0x00,0x00,0x97,0x01,0x00,0x00,0xf1,0x00,0x00,0x00, -0x96,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0xec,0x00,0x00,0x00, -0x98,0x01,0x00,0x00,0x97,0x01,0x00,0x00,0x41,0x00,0x05,0x00, -0x99,0x01,0x00,0x00,0x9a,0x01,0x00,0x00,0x84,0x01,0x00,0x00, -0x88,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x9a,0x01,0x00,0x00, -0x98,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x9c,0x01,0x00,0x00,0xcc,0x02,0x00,0x00,0xc5,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x7a,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x7c,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x75,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x75,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x9e,0x01,0x00,0x00,0xba,0x02,0x00,0x00, -0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x72,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x74,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0xa0,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xa0,0x01,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xbb,0x02,0x00,0x00, -0x3e,0x00,0x00,0x00,0x74,0x01,0x00,0x00,0xcc,0x01,0x00,0x00, -0xa3,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00, -0xa6,0x01,0x00,0x00,0xbb,0x02,0x00,0x00,0xb4,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0xa2,0x01,0x00,0x00,0xa3,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xa6,0x01,0x00,0x00, -0xa1,0x01,0x00,0x00,0xa2,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xa1,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xa8,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xa8,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xc9,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, -0xa1,0x01,0x00,0x00,0xca,0x01,0x00,0x00,0xa9,0x01,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00,0xae,0x01,0x00,0x00, -0xc9,0x02,0x00,0x00,0xb1,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0xaa,0x01,0x00,0x00,0xa9,0x01,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xae,0x01,0x00,0x00,0xa9,0x01,0x00,0x00, -0xaa,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xa9,0x01,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xb4,0x01,0x00,0x00, -0xbb,0x02,0x00,0x00,0xb1,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xb6,0x01,0x00,0x00,0xb4,0x01,0x00,0x00, -0xc9,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xb8,0x01,0x00,0x00,0x59,0x00,0x00,0x00,0xae,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xbb,0x01,0x00,0x00, -0xbb,0x02,0x00,0x00,0xba,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xbc,0x01,0x00,0x00,0xb8,0x01,0x00,0x00, -0xbb,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xbe,0x01,0x00,0x00,0x68,0x00,0x00,0x00,0xb1,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xbf,0x01,0x00,0x00, -0xbc,0x01,0x00,0x00,0xbe,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xc1,0x01,0x00,0x00,0xbf,0x01,0x00,0x00, -0xc9,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xc3,0x01,0x00,0x00,0xc1,0x01,0x00,0x00,0xc2,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xc5,0x01,0x00,0x00, -0xc3,0x01,0x00,0x00,0xb6,0x02,0x00,0x00,0x41,0x00,0x05,0x00, -0x0a,0x01,0x00,0x00,0xc6,0x01,0x00,0x00,0x3c,0x01,0x00,0x00, -0xc5,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0xec,0x00,0x00,0x00, -0xc7,0x01,0x00,0x00,0xc6,0x01,0x00,0x00,0x41,0x00,0x05,0x00, -0x99,0x01,0x00,0x00,0xc8,0x01,0x00,0x00,0xb2,0x01,0x00,0x00, -0xb6,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0xc8,0x01,0x00,0x00, -0xc7,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xca,0x01,0x00,0x00,0xc9,0x02,0x00,0x00,0xc5,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xa8,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xaa,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xa3,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xa3,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xcc,0x01,0x00,0x00,0xbb,0x02,0x00,0x00, -0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xa0,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xa2,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0xce,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xce,0x01,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xbc,0x02,0x00,0x00, -0x3e,0x00,0x00,0x00,0xa2,0x01,0x00,0x00,0x12,0x02,0x00,0x00, -0xd1,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00, -0xd4,0x01,0x00,0x00,0xbc,0x02,0x00,0x00,0xb4,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0xd0,0x01,0x00,0x00,0xd1,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xd4,0x01,0x00,0x00, -0xcf,0x01,0x00,0x00,0xd0,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xcf,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xd6,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xd6,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xc0,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, -0xcf,0x01,0x00,0x00,0x10,0x02,0x00,0x00,0xd9,0x01,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00,0xdc,0x01,0x00,0x00, -0xc0,0x02,0x00,0x00,0x60,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0xd8,0x01,0x00,0x00,0xd9,0x01,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xdc,0x01,0x00,0x00,0xd7,0x01,0x00,0x00, -0xd8,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xd7,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0xde,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xde,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xc2,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0xd7,0x01,0x00,0x00, -0x0e,0x02,0x00,0x00,0xe1,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb7,0x00,0x00,0x00,0xe4,0x01,0x00,0x00,0xc2,0x02,0x00,0x00, -0xb1,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xe0,0x01,0x00,0x00, -0xe1,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0xe4,0x01,0x00,0x00,0xdf,0x01,0x00,0x00,0xe0,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xdf,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0xe6,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xe6,0x01,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xc4,0x02,0x00,0x00, -0x3e,0x00,0x00,0x00,0xdf,0x01,0x00,0x00,0x0c,0x02,0x00,0x00, -0xe7,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00, -0xec,0x01,0x00,0x00,0xc4,0x02,0x00,0x00,0x62,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0xe8,0x01,0x00,0x00,0xe7,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xec,0x01,0x00,0x00, -0xe7,0x01,0x00,0x00,0xe8,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xe7,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xee,0x01,0x00,0x00,0xbc,0x02,0x00,0x00,0xb1,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xf0,0x01,0x00,0x00, -0xee,0x01,0x00,0x00,0xc2,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xf2,0x01,0x00,0x00,0xf0,0x01,0x00,0x00, -0xf1,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xf4,0x01,0x00,0x00,0xc0,0x02,0x00,0x00,0x62,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xf5,0x01,0x00,0x00, -0xf2,0x01,0x00,0x00,0xf4,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xf7,0x01,0x00,0x00,0xf5,0x01,0x00,0x00, -0xc4,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xfb,0x01,0x00,0x00,0xf4,0x01,0x00,0x00,0xc4,0x02,0x00,0x00, -0x41,0x00,0x05,0x00,0x99,0x01,0x00,0x00,0xfc,0x01,0x00,0x00, -0x84,0x01,0x00,0x00,0xfb,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, -0xec,0x00,0x00,0x00,0xfd,0x01,0x00,0x00,0xfc,0x01,0x00,0x00, -0x73,0x00,0x04,0x00,0xb9,0x00,0x00,0x00,0xfe,0x01,0x00,0x00, -0xfd,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0x99,0x01,0x00,0x00, -0x03,0x02,0x00,0x00,0xb2,0x01,0x00,0x00,0xf0,0x01,0x00,0x00, -0x3d,0x00,0x04,0x00,0xec,0x00,0x00,0x00,0x04,0x02,0x00,0x00, -0x03,0x02,0x00,0x00,0x73,0x00,0x04,0x00,0xb9,0x00,0x00,0x00, -0x05,0x02,0x00,0x00,0x04,0x02,0x00,0x00,0x41,0x00,0x05,0x00, -0xc2,0x00,0x00,0x00,0x07,0x02,0x00,0x00,0xbf,0x00,0x00,0x00, -0xf7,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0xb9,0x00,0x00,0x00, -0x08,0x02,0x00,0x00,0x07,0x02,0x00,0x00,0x0c,0x00,0x08,0x00, -0xb9,0x00,0x00,0x00,0x09,0x02,0x00,0x00,0x01,0x00,0x00,0x00, -0x32,0x00,0x00,0x00,0xfe,0x01,0x00,0x00,0x05,0x02,0x00,0x00, -0x08,0x02,0x00,0x00,0x3e,0x00,0x03,0x00,0x07,0x02,0x00,0x00, -0x09,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x0c,0x02,0x00,0x00,0xc4,0x02,0x00,0x00,0xc5,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xe6,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xe8,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xe1,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xe1,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x0e,0x02,0x00,0x00,0xc2,0x02,0x00,0x00, -0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xde,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xe0,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0xd9,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xd9,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x10,0x02,0x00,0x00, -0xc0,0x02,0x00,0x00,0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xd6,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xd8,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0xd1,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xd1,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x12,0x02,0x00,0x00,0xbc,0x02,0x00,0x00,0xc5,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xce,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xd0,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x6d,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x6d,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x14,0x02,0x00,0x00,0xb6,0x02,0x00,0x00, -0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x6a,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x6c,0x01,0x00,0x00,0xe0,0x00,0x04,0x00, -0x0c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x62,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0xcc,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xcc,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x16,0x02,0x00,0x00,0x9c,0x02,0x00,0x00,0x6c,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xc9,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xcb,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x1b,0x02,0x00,0x00,0x55,0x00,0x00,0x00,0x53,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x1c,0x02,0x00,0x00, -0x8b,0x00,0x00,0x00,0x1b,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x21,0x02,0x00,0x00,0x59,0x00,0x00,0x00, -0xae,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x22,0x02,0x00,0x00,0x9d,0x00,0x00,0x00,0x21,0x02,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x27,0x02,0x00,0x00, -0x12,0x00,0x00,0x00,0x26,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x28,0x02,0x00,0x00,0x27,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x29,0x02,0x00,0x00, -0x0f,0x00,0x00,0x00,0x28,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x2d,0x02,0x00,0x00,0x47,0x00,0x00,0x00, -0x28,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00, -0x2f,0x02,0x00,0x00,0x2e,0x02,0x00,0x00,0x0c,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x30,0x02,0x00,0x00, -0x2f,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x31,0x02,0x00,0x00,0x2d,0x02,0x00,0x00,0x30,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x32,0x02,0x00,0x00, -0x29,0x02,0x00,0x00,0x31,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x34,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x34,0x02,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x9d,0x02,0x00,0x00, -0x3e,0x00,0x00,0x00,0xcb,0x00,0x00,0x00,0x9a,0x02,0x00,0x00, -0x37,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00, -0x3a,0x02,0x00,0x00,0x9d,0x02,0x00,0x00,0xb4,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0x36,0x02,0x00,0x00,0x37,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x3a,0x02,0x00,0x00, -0x35,0x02,0x00,0x00,0x36,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x35,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x3c,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x3c,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0x9e,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, -0x35,0x02,0x00,0x00,0x98,0x02,0x00,0x00,0x3f,0x02,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00,0x42,0x02,0x00,0x00, -0x9e,0x02,0x00,0x00,0x60,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0x3e,0x02,0x00,0x00,0x3f,0x02,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x42,0x02,0x00,0x00,0x3d,0x02,0x00,0x00, -0x3e,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x3d,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x46,0x02,0x00,0x00, -0x9e,0x02,0x00,0x00,0x61,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x47,0x02,0x00,0x00,0x1c,0x02,0x00,0x00, -0x46,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x49,0x02,0x00,0x00,0x64,0x00,0x00,0x00,0x62,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x4a,0x02,0x00,0x00, -0x47,0x02,0x00,0x00,0x49,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x4e,0x02,0x00,0x00,0x9d,0x02,0x00,0x00, -0xba,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x4f,0x02,0x00,0x00,0x22,0x02,0x00,0x00,0x4e,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x51,0x02,0x00,0x00, -0x68,0x00,0x00,0x00,0xb1,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x52,0x02,0x00,0x00,0x4f,0x02,0x00,0x00, -0x51,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x54,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x54,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xa0,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, -0x3d,0x02,0x00,0x00,0x96,0x02,0x00,0x00,0x57,0x02,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00,0x5a,0x02,0x00,0x00, -0xa0,0x02,0x00,0x00,0xb1,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0x56,0x02,0x00,0x00,0x57,0x02,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x5a,0x02,0x00,0x00,0x55,0x02,0x00,0x00, -0x56,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x55,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0x5c,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x5c,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xa2,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0x55,0x02,0x00,0x00, -0x94,0x02,0x00,0x00,0x5f,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb7,0x00,0x00,0x00,0x62,0x02,0x00,0x00,0xa2,0x02,0x00,0x00, -0x62,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x5e,0x02,0x00,0x00, -0x5f,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x62,0x02,0x00,0x00,0x5d,0x02,0x00,0x00,0x5e,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x5d,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x65,0x02,0x00,0x00,0x4a,0x02,0x00,0x00, -0xa2,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00, -0x68,0x02,0x00,0x00,0x65,0x02,0x00,0x00,0x36,0x00,0x00,0x00, -0xf7,0x00,0x03,0x00,0x6a,0x02,0x00,0x00,0x00,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x68,0x02,0x00,0x00,0x69,0x02,0x00,0x00, -0x6a,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x69,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x6d,0x02,0x00,0x00, -0x52,0x02,0x00,0x00,0xa0,0x02,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x6e,0x02,0x00,0x00,0x12,0x00,0x00,0x00, -0xc5,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x6f,0x02,0x00,0x00,0x6e,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb7,0x00,0x00,0x00,0x70,0x02,0x00,0x00,0x6d,0x02,0x00,0x00, -0x6f,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x6a,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x6a,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, -0xb7,0x00,0x00,0x00,0x71,0x02,0x00,0x00,0x68,0x02,0x00,0x00, -0x5d,0x02,0x00,0x00,0x70,0x02,0x00,0x00,0x69,0x02,0x00,0x00, -0xf7,0x00,0x03,0x00,0x73,0x02,0x00,0x00,0x00,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x71,0x02,0x00,0x00,0x72,0x02,0x00,0x00, -0x73,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x72,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x7b,0x02,0x00,0x00, -0x52,0x02,0x00,0x00,0xa0,0x02,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x7d,0x02,0x00,0x00,0x12,0x00,0x00,0x00, -0x7c,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x7e,0x02,0x00,0x00,0x7d,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x7f,0x02,0x00,0x00,0x7b,0x02,0x00,0x00, -0x7e,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x80,0x02,0x00,0x00,0x32,0x02,0x00,0x00,0x7f,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x82,0x02,0x00,0x00, -0x80,0x02,0x00,0x00,0x4a,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x84,0x02,0x00,0x00,0x82,0x02,0x00,0x00, -0xa2,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x86,0x02,0x00,0x00,0x9d,0x02,0x00,0x00,0xb1,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x88,0x02,0x00,0x00, -0x86,0x02,0x00,0x00,0xa0,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x8a,0x02,0x00,0x00,0x88,0x02,0x00,0x00, -0x89,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x8c,0x02,0x00,0x00,0x9e,0x02,0x00,0x00,0x62,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8d,0x02,0x00,0x00, -0x8a,0x02,0x00,0x00,0x8c,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x8f,0x02,0x00,0x00,0x8d,0x02,0x00,0x00, -0xa2,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, -0x90,0x02,0x00,0x00,0xbf,0x00,0x00,0x00,0x8f,0x02,0x00,0x00, -0x3d,0x00,0x04,0x00,0xb9,0x00,0x00,0x00,0x91,0x02,0x00,0x00, -0x90,0x02,0x00,0x00,0x41,0x00,0x06,0x00,0x52,0x01,0x00,0x00, -0x92,0x02,0x00,0x00,0x77,0x02,0x00,0x00,0x34,0x00,0x00,0x00, -0x84,0x02,0x00,0x00,0x3e,0x00,0x03,0x00,0x92,0x02,0x00,0x00, -0x91,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x73,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x73,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x5f,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x5f,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x94,0x02,0x00,0x00, -0xa2,0x02,0x00,0x00,0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x5c,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x5e,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0x57,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x57,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x96,0x02,0x00,0x00,0xa0,0x02,0x00,0x00,0xc5,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x54,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x56,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x3f,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x3f,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x98,0x02,0x00,0x00,0x9e,0x02,0x00,0x00, -0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x3c,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x3e,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x37,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x37,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9a,0x02,0x00,0x00, -0x9d,0x02,0x00,0x00,0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x34,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x36,0x02,0x00,0x00, -0xfd,0x00,0x01,0x00,0x38,0x00,0x01,0x00, -}; -const uint64_t matmul_f16_f32_s_len = 10172; - -unsigned char matmul_f16_f32_s_fp32_data[] = { -0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, -0xc9,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, -0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x51,0x11,0x00,0x00, -0x0b,0x00,0x06,0x00,0x01,0x00,0x00,0x00,0x47,0x4c,0x53,0x4c, -0x2e,0x73,0x74,0x64,0x2e,0x34,0x35,0x30,0x00,0x00,0x00,0x00, -0x0e,0x00,0x03,0x00,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x0f,0x00,0x0f,0x00,0x05,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x6d,0x61,0x69,0x6e,0x00,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x3d,0x00,0x00,0x00,0x4c,0x00,0x00,0x00, -0xf0,0x00,0x00,0x00,0xfc,0x00,0x00,0x00,0x3c,0x01,0x00,0x00, -0x47,0x01,0x00,0x00,0x2a,0x02,0x00,0x00,0x73,0x02,0x00,0x00, -0x10,0x00,0x06,0x00,0x04,0x00,0x00,0x00,0x11,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x0b,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x1c,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x10,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x08,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x03,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x10,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x14,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x18,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x07,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x10,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x24,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x0a,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x10,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x2c,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x30,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x0d,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0x47,0x00,0x03,0x00, -0x10,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x37,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x3d,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x1a,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x4c,0x00,0x00,0x00, -0x0b,0x00,0x00,0x00,0x1b,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x4f,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x53,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x60,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x62,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x07,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x6c,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x03,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x9c,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0xae,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x05,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0xb1,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x08,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xf9,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x48,0x00,0x04,0x00, -0xfa,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0xfa,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, -0xfa,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0xfc,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0xfc,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x16,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x17,0x01,0x00,0x00,0x0b,0x00,0x00,0x00,0x19,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x44,0x01,0x00,0x00,0x06,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0x45,0x01,0x00,0x00, -0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x45,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x45,0x01,0x00,0x00, -0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x47,0x01,0x00,0x00, -0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x47,0x01,0x00,0x00,0x21,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x2a,0x02,0x00,0x00,0x0b,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x70,0x02,0x00,0x00, -0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00, -0x71,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x19,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x71,0x02,0x00,0x00,0x00,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, -0x71,0x02,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x73,0x02,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x73,0x02,0x00,0x00,0x21,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x13,0x00,0x02,0x00,0x02,0x00,0x00,0x00, -0x21,0x00,0x03,0x00,0x03,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x15,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x17,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x0a,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x0d,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x1e,0x00,0x10,0x00,0x10,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x11,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x11,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x15,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, -0x14,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x15,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x0a,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, -0x27,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0x2d,0x00,0x00,0x00,0x07,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x34,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x37,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00,0x3d,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x3e,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x0a,0x00,0x00,0x00,0x4c,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x4f,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x53,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x54,0x00,0x00,0x00,0x86,0x00,0x00,0x00, -0x37,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x58,0x00,0x00,0x00,0x86,0x00,0x00,0x00, -0x37,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x32,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x61,0x00,0x00,0x00, -0x86,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x60,0x00,0x00,0x00, -0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x62,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x63,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x61,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x67,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x61,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x86,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x72,0x00,0x00,0x00,0x86,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0x76,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x7b,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, -0x86,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0x8c,0x00,0x00,0x00,0x03,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x97,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x9c,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0x9e,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xad,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00, -0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xae,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xaf,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x53,0x00,0x00,0x00, -0xae,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xb0,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x4f,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xb1,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xb2,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0xb0,0x00,0x00,0x00,0xb1,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xb3,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0xb2,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xb4,0x00,0x00,0x00,0x86,0x00,0x00,0x00, -0xaf,0x00,0x00,0x00,0xb3,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xb5,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0xad,0x00,0x00,0x00,0xb4,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xb6,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0xb5,0x00,0x00,0x00,0xb1,0x00,0x00,0x00,0x14,0x00,0x02,0x00, -0xb7,0x00,0x00,0x00,0x16,0x00,0x03,0x00,0xb9,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xba,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xbb,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0xba,0x00,0x00,0x00, -0xb4,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xbc,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0xbb,0x00,0x00,0x00, -0xb1,0x00,0x00,0x00,0x1c,0x00,0x04,0x00,0xbd,0x00,0x00,0x00, -0xb9,0x00,0x00,0x00,0xbc,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0xbe,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0xbd,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0xb9,0x00,0x00,0x00,0xc1,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0xc2,0x00,0x00,0x00, -0x07,0x00,0x00,0x00,0xb9,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0xc5,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xec,0x00,0x00,0x00, -0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xed,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0xec,0x00,0x00,0x00, -0x1c,0x00,0x04,0x00,0xee,0x00,0x00,0x00,0xb9,0x00,0x00,0x00, -0xed,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0xef,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0xee,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0xef,0x00,0x00,0x00,0xf0,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xf4,0x00,0x00,0x00, -0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x16,0x00,0x03,0x00,0xf8,0x00,0x00,0x00,0x10,0x00,0x00,0x00, -0x1d,0x00,0x03,0x00,0xf9,0x00,0x00,0x00,0xf8,0x00,0x00,0x00, -0x1e,0x00,0x03,0x00,0xfa,0x00,0x00,0x00,0xf9,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0xfb,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0xfa,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0xfb,0x00,0x00,0x00, -0xfc,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x07,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0xf8,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x0b,0x01,0x00,0x00,0x04,0x00,0x00,0x00, -0xb9,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x11,0x01,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x16,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x33,0x00,0x06,0x00, -0x09,0x00,0x00,0x00,0x17,0x01,0x00,0x00,0x16,0x01,0x00,0x00, -0x39,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x18,0x01,0x00,0x00,0x51,0x00,0x00,0x00, -0x17,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x19,0x01,0x00,0x00,0x84,0x00,0x00,0x00, -0x18,0x01,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x1a,0x01,0x00,0x00,0x86,0x00,0x00,0x00, -0x19,0x01,0x00,0x00,0x6c,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x38,0x01,0x00,0x00,0x80,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x39,0x01,0x00,0x00,0x84,0x00,0x00,0x00, -0x9c,0x00,0x00,0x00,0x38,0x01,0x00,0x00,0x1c,0x00,0x04,0x00, -0x3a,0x01,0x00,0x00,0xb9,0x00,0x00,0x00,0x39,0x01,0x00,0x00, -0x20,0x00,0x04,0x00,0x3b,0x01,0x00,0x00,0x04,0x00,0x00,0x00, -0x3a,0x01,0x00,0x00,0x3b,0x00,0x04,0x00,0x3b,0x01,0x00,0x00, -0x3c,0x01,0x00,0x00,0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x40,0x01,0x00,0x00,0x80,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, -0x44,0x01,0x00,0x00,0xb9,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, -0x45,0x01,0x00,0x00,0x44,0x01,0x00,0x00,0x20,0x00,0x04,0x00, -0x46,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x45,0x01,0x00,0x00, -0x3b,0x00,0x04,0x00,0x46,0x01,0x00,0x00,0x47,0x01,0x00,0x00, -0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x52,0x01,0x00,0x00, -0x0c,0x00,0x00,0x00,0xb9,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x5a,0x01,0x00,0x00,0x80,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x61,0x01,0x00,0x00,0x08,0x01,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x62,0x01,0x00,0x00, -0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x65,0x01,0x00,0x00, -0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x80,0x01,0x00,0x00, -0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00, -0x1c,0x00,0x04,0x00,0x81,0x01,0x00,0x00,0xb9,0x00,0x00,0x00, -0x80,0x01,0x00,0x00,0x20,0x00,0x04,0x00,0x82,0x01,0x00,0x00, -0x07,0x00,0x00,0x00,0x81,0x01,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x92,0x01,0x00,0x00,0x80,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xad,0x01,0x00,0x00,0x84,0x00,0x00,0x00, -0xb4,0x00,0x00,0x00,0xb1,0x00,0x00,0x00,0x1c,0x00,0x04,0x00, -0xae,0x01,0x00,0x00,0xb9,0x00,0x00,0x00,0xad,0x01,0x00,0x00, -0x20,0x00,0x04,0x00,0xaf,0x01,0x00,0x00,0x07,0x00,0x00,0x00, -0xae,0x01,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xb8,0x01,0x00,0x00,0x86,0x00,0x00,0x00,0xae,0x00,0x00,0x00, -0xb4,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xc0,0x01,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xef,0x01,0x00,0x00,0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, -0x22,0x02,0x00,0x00,0x0d,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x0a,0x00,0x00,0x00,0x2a,0x02,0x00,0x00,0x01,0x00,0x00,0x00, -0x1d,0x00,0x03,0x00,0x70,0x02,0x00,0x00,0xb9,0x00,0x00,0x00, -0x1e,0x00,0x03,0x00,0x71,0x02,0x00,0x00,0x70,0x02,0x00,0x00, -0x20,0x00,0x04,0x00,0x72,0x02,0x00,0x00,0x0c,0x00,0x00,0x00, -0x71,0x02,0x00,0x00,0x3b,0x00,0x04,0x00,0x72,0x02,0x00,0x00, -0x73,0x02,0x00,0x00,0x0c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0x78,0x02,0x00,0x00,0x05,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x85,0x02,0x00,0x00, -0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00, -0x36,0x00,0x05,0x00,0x02,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x05,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0xbe,0x00,0x00,0x00, -0xbf,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x82,0x01,0x00,0x00,0x83,0x01,0x00,0x00,0x07,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0xaf,0x01,0x00,0x00,0xb0,0x01,0x00,0x00, -0x07,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00, -0x0e,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, -0x0e,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x14,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x17,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x17,0x00,0x00,0x00, -0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x1e,0x00,0x00,0x00, -0x0f,0x00,0x00,0x00,0x17,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x21,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x24,0x00,0x00,0x00,0x18,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0x28,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x27,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x29,0x00,0x00,0x00, -0x28,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x2a,0x00,0x00,0x00,0x1e,0x00,0x00,0x00,0x29,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x2d,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x2f,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x30,0x00,0x00,0x00, -0x24,0x00,0x00,0x00,0x2f,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x30,0x00,0x00,0x00, -0x2a,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0x35,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x34,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x36,0x00,0x00,0x00, -0x35,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x38,0x00,0x00,0x00,0x36,0x00,0x00,0x00,0x37,0x00,0x00,0x00, -0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, -0x38,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x3b,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, -0x37,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00, -0x3f,0x00,0x00,0x00,0x3d,0x00,0x00,0x00,0x3e,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x3f,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x42,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x3b,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x47,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0x3b,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x0d,0x00,0x00,0x00,0x49,0x00,0x00,0x00,0x3d,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x4a,0x00,0x00,0x00,0x49,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x0d,0x00,0x00,0x00,0x4d,0x00,0x00,0x00,0x4c,0x00,0x00,0x00, -0x3e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x4e,0x00,0x00,0x00,0x4d,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x50,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, -0x4f,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x55,0x00,0x00,0x00,0x50,0x00,0x00,0x00,0x54,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x59,0x00,0x00,0x00, -0x50,0x00,0x00,0x00,0x58,0x00,0x00,0x00,0x89,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x5d,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, -0x4f,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x64,0x00,0x00,0x00,0x5d,0x00,0x00,0x00,0x63,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x68,0x00,0x00,0x00, -0x5d,0x00,0x00,0x00,0x67,0x00,0x00,0x00,0x89,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x6e,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, -0x6d,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x73,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x72,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x77,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x76,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x78,0x00,0x00,0x00,0x77,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x79,0x00,0x00,0x00, -0x47,0x00,0x00,0x00,0x78,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x7c,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x7b,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x7d,0x00,0x00,0x00,0x7c,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x7f,0x00,0x00,0x00,0x47,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x82,0x00,0x00,0x00,0x7f,0x00,0x00,0x00,0x78,0x00,0x00,0x00, -0x0c,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x83,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x26,0x00,0x00,0x00,0x7d,0x00,0x00,0x00, -0x82,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0x87,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x86,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x88,0x00,0x00,0x00, -0x87,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x89,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x88,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8b,0x00,0x00,0x00, -0x42,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x8d,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x8c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x8e,0x00,0x00,0x00,0x8d,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x8f,0x00,0x00,0x00,0x8b,0x00,0x00,0x00, -0x8e,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x90,0x00,0x00,0x00,0x89,0x00,0x00,0x00,0x8f,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x92,0x00,0x00,0x00, -0x90,0x00,0x00,0x00,0x79,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x93,0x00,0x00,0x00,0x92,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0x98,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x97,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x99,0x00,0x00,0x00, -0x98,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x9a,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x99,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9d,0x00,0x00,0x00, -0x4a,0x00,0x00,0x00,0x9c,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x9f,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x9e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xa0,0x00,0x00,0x00,0x9f,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xa1,0x00,0x00,0x00,0x9d,0x00,0x00,0x00, -0xa0,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xa2,0x00,0x00,0x00,0x9a,0x00,0x00,0x00,0xa1,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa4,0x00,0x00,0x00, -0xa2,0x00,0x00,0x00,0x79,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xa5,0x00,0x00,0x00,0xa4,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xa7,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xa7,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0x97,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, -0x05,0x00,0x00,0x00,0xc6,0x00,0x00,0x00,0xa8,0x00,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00,0xb8,0x00,0x00,0x00, -0x97,0x02,0x00,0x00,0xb6,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0xa9,0x00,0x00,0x00,0xa8,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xb8,0x00,0x00,0x00,0xa8,0x00,0x00,0x00, -0xa9,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xa8,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0xc3,0x00,0x00,0x00, -0xbf,0x00,0x00,0x00,0x97,0x02,0x00,0x00,0x3e,0x00,0x03,0x00, -0xc3,0x00,0x00,0x00,0xc1,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xc6,0x00,0x00,0x00,0x97,0x02,0x00,0x00, -0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xa7,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xa9,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xc9,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xc9,0x00,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xb0,0x02,0x00,0x00, -0xa5,0x00,0x00,0x00,0xa9,0x00,0x00,0x00,0x67,0x01,0x00,0x00, -0xcc,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xac,0x02,0x00,0x00,0x93,0x00,0x00,0x00,0xa9,0x00,0x00,0x00, -0x64,0x01,0x00,0x00,0xcc,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0x98,0x02,0x00,0x00,0x79,0x00,0x00,0x00, -0xa9,0x00,0x00,0x00,0x12,0x02,0x00,0x00,0xcc,0x00,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00,0xd0,0x00,0x00,0x00, -0x98,0x02,0x00,0x00,0x83,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0xcb,0x00,0x00,0x00,0xcc,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xd0,0x00,0x00,0x00,0xca,0x00,0x00,0x00, -0xcb,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xca,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xd2,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xd2,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xa8,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0xca,0x00,0x00,0x00, -0x1c,0x01,0x00,0x00,0xd5,0x00,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb7,0x00,0x00,0x00,0xd8,0x00,0x00,0x00,0xa8,0x02,0x00,0x00, -0x37,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xd4,0x00,0x00,0x00, -0xd5,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0xd8,0x00,0x00,0x00,0xd3,0x00,0x00,0x00,0xd4,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xd3,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xdc,0x00,0x00,0x00,0x8b,0x00,0x00,0x00, -0x73,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xde,0x00,0x00,0x00,0xdc,0x00,0x00,0x00,0xa8,0x02,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00,0xe1,0x00,0x00,0x00, -0xde,0x00,0x00,0x00,0x36,0x00,0x00,0x00,0xf7,0x00,0x03,0x00, -0xe3,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0xe1,0x00,0x00,0x00,0xe2,0x00,0x00,0x00,0xe3,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xe2,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xe6,0x00,0x00,0x00,0x98,0x02,0x00,0x00, -0x6e,0x00,0x00,0x00,0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00, -0xe8,0x00,0x00,0x00,0xe6,0x00,0x00,0x00,0x83,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xe3,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xe3,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0xb7,0x00,0x00,0x00, -0xe9,0x00,0x00,0x00,0xe1,0x00,0x00,0x00,0xd3,0x00,0x00,0x00, -0xe8,0x00,0x00,0x00,0xe2,0x00,0x00,0x00,0xf7,0x00,0x03,0x00, -0xeb,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0xe9,0x00,0x00,0x00,0xea,0x00,0x00,0x00,0x0d,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xea,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xf3,0x00,0x00,0x00,0x73,0x00,0x00,0x00, -0xa8,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xf5,0x00,0x00,0x00,0xf3,0x00,0x00,0x00,0xf4,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xf7,0x00,0x00,0x00, -0xf5,0x00,0x00,0x00,0x6e,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x03,0x01,0x00,0x00,0xf3,0x00,0x00,0x00, -0x8e,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x04,0x01,0x00,0x00,0xac,0x02,0x00,0x00,0x03,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x06,0x01,0x00,0x00, -0x04,0x01,0x00,0x00,0x6e,0x00,0x00,0x00,0x41,0x00,0x06,0x00, -0x07,0x01,0x00,0x00,0x08,0x01,0x00,0x00,0xfc,0x00,0x00,0x00, -0x34,0x00,0x00,0x00,0x06,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, -0xf8,0x00,0x00,0x00,0x09,0x01,0x00,0x00,0x08,0x01,0x00,0x00, -0x73,0x00,0x04,0x00,0xb9,0x00,0x00,0x00,0x0a,0x01,0x00,0x00, -0x09,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0x0b,0x01,0x00,0x00, -0x0c,0x01,0x00,0x00,0xf0,0x00,0x00,0x00,0xf7,0x00,0x00,0x00, -0x3e,0x00,0x03,0x00,0x0c,0x01,0x00,0x00,0x0a,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0xeb,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x0d,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x10,0x01,0x00,0x00,0x73,0x00,0x00,0x00,0xa8,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x12,0x01,0x00,0x00, -0x10,0x01,0x00,0x00,0x11,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x14,0x01,0x00,0x00,0x12,0x01,0x00,0x00, -0x6e,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x0b,0x01,0x00,0x00, -0x15,0x01,0x00,0x00,0xf0,0x00,0x00,0x00,0x14,0x01,0x00,0x00, -0x3e,0x00,0x03,0x00,0x15,0x01,0x00,0x00,0xc1,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xeb,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xeb,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xd5,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xd5,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x1c,0x01,0x00,0x00,0xa8,0x02,0x00,0x00, -0x1a,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xd2,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xd4,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x1e,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x1e,0x01,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xa9,0x02,0x00,0x00, -0x3e,0x00,0x00,0x00,0xd4,0x00,0x00,0x00,0x60,0x01,0x00,0x00, -0x21,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00, -0x24,0x01,0x00,0x00,0xa9,0x02,0x00,0x00,0x9c,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0x20,0x01,0x00,0x00,0x21,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x24,0x01,0x00,0x00, -0x1f,0x01,0x00,0x00,0x20,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x1f,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x28,0x01,0x00,0x00,0x9d,0x00,0x00,0x00,0x73,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x2a,0x01,0x00,0x00, -0x28,0x01,0x00,0x00,0xa9,0x02,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x2b,0x01,0x00,0x00,0x12,0x00,0x00,0x00, -0xc5,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x2c,0x01,0x00,0x00,0x2b,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb7,0x00,0x00,0x00,0x2d,0x01,0x00,0x00,0x2a,0x01,0x00,0x00, -0x2c,0x01,0x00,0x00,0xf7,0x00,0x03,0x00,0x2f,0x01,0x00,0x00, -0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x2d,0x01,0x00,0x00, -0x2e,0x01,0x00,0x00,0x2f,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x2e,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x32,0x01,0x00,0x00,0x98,0x02,0x00,0x00,0x6e,0x00,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00,0x34,0x01,0x00,0x00, -0x32,0x01,0x00,0x00,0x83,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x2f,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x2f,0x01,0x00,0x00, -0xf5,0x00,0x07,0x00,0xb7,0x00,0x00,0x00,0x35,0x01,0x00,0x00, -0x2d,0x01,0x00,0x00,0x1f,0x01,0x00,0x00,0x34,0x01,0x00,0x00, -0x2e,0x01,0x00,0x00,0xf7,0x00,0x03,0x00,0x37,0x01,0x00,0x00, -0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x35,0x01,0x00,0x00, -0x36,0x01,0x00,0x00,0x56,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x36,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x3f,0x01,0x00,0x00,0x73,0x00,0x00,0x00,0xa9,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x41,0x01,0x00,0x00, -0x3f,0x01,0x00,0x00,0x40,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x43,0x01,0x00,0x00,0x41,0x01,0x00,0x00, -0x6e,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x4e,0x01,0x00,0x00,0x3f,0x01,0x00,0x00,0xa0,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x4f,0x01,0x00,0x00, -0xb0,0x02,0x00,0x00,0x4e,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x51,0x01,0x00,0x00,0x4f,0x01,0x00,0x00, -0x6e,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x52,0x01,0x00,0x00, -0x53,0x01,0x00,0x00,0x47,0x01,0x00,0x00,0x34,0x00,0x00,0x00, -0x51,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0xb9,0x00,0x00,0x00, -0x54,0x01,0x00,0x00,0x53,0x01,0x00,0x00,0x41,0x00,0x05,0x00, -0x0b,0x01,0x00,0x00,0x55,0x01,0x00,0x00,0x3c,0x01,0x00,0x00, -0x43,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x55,0x01,0x00,0x00, -0x54,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x37,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x56,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x59,0x01,0x00,0x00,0x73,0x00,0x00,0x00, -0xa9,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x5b,0x01,0x00,0x00,0x59,0x01,0x00,0x00,0x5a,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x5d,0x01,0x00,0x00, -0x5b,0x01,0x00,0x00,0x6e,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x0b,0x01,0x00,0x00,0x5e,0x01,0x00,0x00,0x3c,0x01,0x00,0x00, -0x5d,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x5e,0x01,0x00,0x00, -0xc1,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x37,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x37,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0x21,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x21,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x60,0x01,0x00,0x00, -0xa9,0x02,0x00,0x00,0x1a,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0x1e,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x20,0x01,0x00,0x00, -0xe0,0x00,0x04,0x00,0x0c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x61,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x64,0x01,0x00,0x00,0xac,0x02,0x00,0x00,0x62,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x67,0x01,0x00,0x00, -0xb0,0x02,0x00,0x00,0x65,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0x69,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x69,0x01,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xb2,0x02,0x00,0x00, -0x3e,0x00,0x00,0x00,0x20,0x01,0x00,0x00,0x10,0x02,0x00,0x00, -0x6c,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00, -0x6f,0x01,0x00,0x00,0xb2,0x02,0x00,0x00,0x6c,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0x6b,0x01,0x00,0x00,0x6c,0x01,0x00,0x00, -0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x6f,0x01,0x00,0x00, -0x6a,0x01,0x00,0x00,0x6b,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x6a,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x71,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x71,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xb6,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, -0x6a,0x01,0x00,0x00,0x9c,0x01,0x00,0x00,0x74,0x01,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00,0x77,0x01,0x00,0x00, -0xb6,0x02,0x00,0x00,0x60,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0x73,0x01,0x00,0x00,0x74,0x01,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x77,0x01,0x00,0x00,0x72,0x01,0x00,0x00, -0x73,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x72,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0x79,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x79,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xc8,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0x72,0x01,0x00,0x00, -0x9a,0x01,0x00,0x00,0x7a,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb7,0x00,0x00,0x00,0x7f,0x01,0x00,0x00,0xc8,0x02,0x00,0x00, -0x62,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x7b,0x01,0x00,0x00, -0x7a,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x7f,0x01,0x00,0x00,0x7a,0x01,0x00,0x00,0x7b,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x7a,0x01,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x85,0x01,0x00,0x00,0xb6,0x02,0x00,0x00, -0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x87,0x01,0x00,0x00,0x85,0x01,0x00,0x00,0xc8,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x89,0x01,0x00,0x00, -0x55,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x8b,0x01,0x00,0x00,0xb6,0x02,0x00,0x00, -0x61,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x8c,0x01,0x00,0x00,0x89,0x01,0x00,0x00,0x8b,0x01,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8e,0x01,0x00,0x00, -0x64,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x8f,0x01,0x00,0x00,0x8c,0x01,0x00,0x00, -0x8e,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x91,0x01,0x00,0x00,0x8f,0x01,0x00,0x00,0xc8,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x93,0x01,0x00,0x00, -0x91,0x01,0x00,0x00,0x92,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x95,0x01,0x00,0x00,0x93,0x01,0x00,0x00, -0xb2,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x0b,0x01,0x00,0x00, -0x96,0x01,0x00,0x00,0xf0,0x00,0x00,0x00,0x95,0x01,0x00,0x00, -0x3d,0x00,0x04,0x00,0xb9,0x00,0x00,0x00,0x97,0x01,0x00,0x00, -0x96,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, -0x98,0x01,0x00,0x00,0x83,0x01,0x00,0x00,0x87,0x01,0x00,0x00, -0x3e,0x00,0x03,0x00,0x98,0x01,0x00,0x00,0x97,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9a,0x01,0x00,0x00, -0xc8,0x02,0x00,0x00,0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x79,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x7b,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0x74,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x74,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x9c,0x01,0x00,0x00,0xb6,0x02,0x00,0x00,0xc5,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x71,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x73,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x9e,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x9e,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xb7,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, -0x73,0x01,0x00,0x00,0xca,0x01,0x00,0x00,0xa1,0x01,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00,0xa4,0x01,0x00,0x00, -0xb7,0x02,0x00,0x00,0xb4,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0xa0,0x01,0x00,0x00,0xa1,0x01,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xa4,0x01,0x00,0x00,0x9f,0x01,0x00,0x00, -0xa0,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x9f,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0xa6,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xa6,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xc5,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0x9f,0x01,0x00,0x00, -0xc8,0x01,0x00,0x00,0xa7,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb7,0x00,0x00,0x00,0xac,0x01,0x00,0x00,0xc5,0x02,0x00,0x00, -0xb1,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xa8,0x01,0x00,0x00, -0xa7,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0xac,0x01,0x00,0x00,0xa7,0x01,0x00,0x00,0xa8,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xa7,0x01,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xb2,0x01,0x00,0x00,0xb7,0x02,0x00,0x00, -0xb1,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xb4,0x01,0x00,0x00,0xb2,0x01,0x00,0x00,0xc5,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xb6,0x01,0x00,0x00, -0x59,0x00,0x00,0x00,0xae,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xb9,0x01,0x00,0x00,0xb7,0x02,0x00,0x00, -0xb8,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xba,0x01,0x00,0x00,0xb6,0x01,0x00,0x00,0xb9,0x01,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xbc,0x01,0x00,0x00, -0x68,0x00,0x00,0x00,0xb1,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xbd,0x01,0x00,0x00,0xba,0x01,0x00,0x00, -0xbc,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xbf,0x01,0x00,0x00,0xbd,0x01,0x00,0x00,0xc5,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xc1,0x01,0x00,0x00, -0xbf,0x01,0x00,0x00,0xc0,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xc3,0x01,0x00,0x00,0xc1,0x01,0x00,0x00, -0xb2,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x0b,0x01,0x00,0x00, -0xc4,0x01,0x00,0x00,0x3c,0x01,0x00,0x00,0xc3,0x01,0x00,0x00, -0x3d,0x00,0x04,0x00,0xb9,0x00,0x00,0x00,0xc5,0x01,0x00,0x00, -0xc4,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, -0xc6,0x01,0x00,0x00,0xb0,0x01,0x00,0x00,0xb4,0x01,0x00,0x00, -0x3e,0x00,0x03,0x00,0xc6,0x01,0x00,0x00,0xc5,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xc8,0x01,0x00,0x00, -0xc5,0x02,0x00,0x00,0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xa6,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xa8,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0xa1,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xa1,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xca,0x01,0x00,0x00,0xb7,0x02,0x00,0x00,0xc5,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x9e,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xa0,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xcc,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xcc,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xb8,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, -0xa0,0x01,0x00,0x00,0x0e,0x02,0x00,0x00,0xcf,0x01,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00,0xd2,0x01,0x00,0x00, -0xb8,0x02,0x00,0x00,0xb4,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0xce,0x01,0x00,0x00,0xcf,0x01,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xd2,0x01,0x00,0x00,0xcd,0x01,0x00,0x00, -0xce,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xcd,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0xd4,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xd4,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xbc,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0xcd,0x01,0x00,0x00, -0x0c,0x02,0x00,0x00,0xd7,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb7,0x00,0x00,0x00,0xda,0x01,0x00,0x00,0xbc,0x02,0x00,0x00, -0x60,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xd6,0x01,0x00,0x00, -0xd7,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0xda,0x01,0x00,0x00,0xd5,0x01,0x00,0x00,0xd6,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xd5,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0xdc,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xdc,0x01,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xbe,0x02,0x00,0x00, -0x3e,0x00,0x00,0x00,0xd5,0x01,0x00,0x00,0x0a,0x02,0x00,0x00, -0xdf,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00, -0xe2,0x01,0x00,0x00,0xbe,0x02,0x00,0x00,0xb1,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0xde,0x01,0x00,0x00,0xdf,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xe2,0x01,0x00,0x00, -0xdd,0x01,0x00,0x00,0xde,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xdd,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xe4,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xe4,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xc0,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, -0xdd,0x01,0x00,0x00,0x08,0x02,0x00,0x00,0xe5,0x01,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00,0xea,0x01,0x00,0x00, -0xc0,0x02,0x00,0x00,0x62,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0xe6,0x01,0x00,0x00,0xe5,0x01,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xea,0x01,0x00,0x00,0xe5,0x01,0x00,0x00, -0xe6,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xe5,0x01,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xec,0x01,0x00,0x00, -0xb8,0x02,0x00,0x00,0xb1,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xee,0x01,0x00,0x00,0xec,0x01,0x00,0x00, -0xbe,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xf0,0x01,0x00,0x00,0xee,0x01,0x00,0x00,0xef,0x01,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xf2,0x01,0x00,0x00, -0xbc,0x02,0x00,0x00,0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xf3,0x01,0x00,0x00,0xf0,0x01,0x00,0x00, -0xf2,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xf5,0x01,0x00,0x00,0xf3,0x01,0x00,0x00,0xc0,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xf9,0x01,0x00,0x00, -0xf2,0x01,0x00,0x00,0xc0,0x02,0x00,0x00,0x41,0x00,0x05,0x00, -0xc2,0x00,0x00,0x00,0xfa,0x01,0x00,0x00,0x83,0x01,0x00,0x00, -0xf9,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0xb9,0x00,0x00,0x00, -0xfb,0x01,0x00,0x00,0xfa,0x01,0x00,0x00,0x41,0x00,0x05,0x00, -0xc2,0x00,0x00,0x00,0x00,0x02,0x00,0x00,0xb0,0x01,0x00,0x00, -0xee,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0xb9,0x00,0x00,0x00, -0x01,0x02,0x00,0x00,0x00,0x02,0x00,0x00,0x41,0x00,0x05,0x00, -0xc2,0x00,0x00,0x00,0x03,0x02,0x00,0x00,0xbf,0x00,0x00,0x00, -0xf5,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0xb9,0x00,0x00,0x00, -0x04,0x02,0x00,0x00,0x03,0x02,0x00,0x00,0x0c,0x00,0x08,0x00, -0xb9,0x00,0x00,0x00,0x05,0x02,0x00,0x00,0x01,0x00,0x00,0x00, -0x32,0x00,0x00,0x00,0xfb,0x01,0x00,0x00,0x01,0x02,0x00,0x00, -0x04,0x02,0x00,0x00,0x3e,0x00,0x03,0x00,0x03,0x02,0x00,0x00, -0x05,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x08,0x02,0x00,0x00,0xc0,0x02,0x00,0x00,0xc5,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xe4,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xe6,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xdf,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xdf,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x0a,0x02,0x00,0x00,0xbe,0x02,0x00,0x00, -0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xdc,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xde,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0xd7,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xd7,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x0c,0x02,0x00,0x00, -0xbc,0x02,0x00,0x00,0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xd4,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xd6,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0xcf,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xcf,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x0e,0x02,0x00,0x00,0xb8,0x02,0x00,0x00,0xc5,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xcc,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xce,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x6c,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x6c,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x10,0x02,0x00,0x00,0xb2,0x02,0x00,0x00, -0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x69,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x6b,0x01,0x00,0x00,0xe0,0x00,0x04,0x00, -0x0c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x61,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0xcc,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xcc,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x12,0x02,0x00,0x00,0x98,0x02,0x00,0x00,0x6c,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xc9,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xcb,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x17,0x02,0x00,0x00,0x55,0x00,0x00,0x00,0x53,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x18,0x02,0x00,0x00, -0x8b,0x00,0x00,0x00,0x17,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x1d,0x02,0x00,0x00,0x59,0x00,0x00,0x00, -0xae,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x1e,0x02,0x00,0x00,0x9d,0x00,0x00,0x00,0x1d,0x02,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x23,0x02,0x00,0x00, -0x12,0x00,0x00,0x00,0x22,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x24,0x02,0x00,0x00,0x23,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x25,0x02,0x00,0x00, -0x0f,0x00,0x00,0x00,0x24,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x29,0x02,0x00,0x00,0x47,0x00,0x00,0x00, -0x24,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00, -0x2b,0x02,0x00,0x00,0x2a,0x02,0x00,0x00,0x0c,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x2c,0x02,0x00,0x00, -0x2b,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x2d,0x02,0x00,0x00,0x29,0x02,0x00,0x00,0x2c,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x2e,0x02,0x00,0x00, -0x25,0x02,0x00,0x00,0x2d,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x30,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x30,0x02,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x99,0x02,0x00,0x00, -0x3e,0x00,0x00,0x00,0xcb,0x00,0x00,0x00,0x96,0x02,0x00,0x00, -0x33,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00, -0x36,0x02,0x00,0x00,0x99,0x02,0x00,0x00,0xb4,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0x32,0x02,0x00,0x00,0x33,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x36,0x02,0x00,0x00, -0x31,0x02,0x00,0x00,0x32,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x31,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x38,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x38,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0x9a,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, -0x31,0x02,0x00,0x00,0x94,0x02,0x00,0x00,0x3b,0x02,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00,0x3e,0x02,0x00,0x00, -0x9a,0x02,0x00,0x00,0x60,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0x3a,0x02,0x00,0x00,0x3b,0x02,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x3e,0x02,0x00,0x00,0x39,0x02,0x00,0x00, -0x3a,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x39,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x42,0x02,0x00,0x00, -0x9a,0x02,0x00,0x00,0x61,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x43,0x02,0x00,0x00,0x18,0x02,0x00,0x00, -0x42,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x45,0x02,0x00,0x00,0x64,0x00,0x00,0x00,0x62,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x46,0x02,0x00,0x00, -0x43,0x02,0x00,0x00,0x45,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x4a,0x02,0x00,0x00,0x99,0x02,0x00,0x00, -0xb8,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x4b,0x02,0x00,0x00,0x1e,0x02,0x00,0x00,0x4a,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x4d,0x02,0x00,0x00, -0x68,0x00,0x00,0x00,0xb1,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x4e,0x02,0x00,0x00,0x4b,0x02,0x00,0x00, -0x4d,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x50,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x50,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0x9c,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, -0x39,0x02,0x00,0x00,0x92,0x02,0x00,0x00,0x53,0x02,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00,0x56,0x02,0x00,0x00, -0x9c,0x02,0x00,0x00,0xb1,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0x52,0x02,0x00,0x00,0x53,0x02,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x56,0x02,0x00,0x00,0x51,0x02,0x00,0x00, -0x52,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x51,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0x58,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x58,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0x9e,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0x51,0x02,0x00,0x00, -0x90,0x02,0x00,0x00,0x5b,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb7,0x00,0x00,0x00,0x5e,0x02,0x00,0x00,0x9e,0x02,0x00,0x00, -0x62,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x5a,0x02,0x00,0x00, -0x5b,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x5e,0x02,0x00,0x00,0x59,0x02,0x00,0x00,0x5a,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x59,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x61,0x02,0x00,0x00,0x46,0x02,0x00,0x00, -0x9e,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00, -0x64,0x02,0x00,0x00,0x61,0x02,0x00,0x00,0x36,0x00,0x00,0x00, -0xf7,0x00,0x03,0x00,0x66,0x02,0x00,0x00,0x00,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x64,0x02,0x00,0x00,0x65,0x02,0x00,0x00, -0x66,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x65,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x69,0x02,0x00,0x00, -0x4e,0x02,0x00,0x00,0x9c,0x02,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x6a,0x02,0x00,0x00,0x12,0x00,0x00,0x00, -0xc5,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x6b,0x02,0x00,0x00,0x6a,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb7,0x00,0x00,0x00,0x6c,0x02,0x00,0x00,0x69,0x02,0x00,0x00, -0x6b,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x66,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x66,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, -0xb7,0x00,0x00,0x00,0x6d,0x02,0x00,0x00,0x64,0x02,0x00,0x00, -0x59,0x02,0x00,0x00,0x6c,0x02,0x00,0x00,0x65,0x02,0x00,0x00, -0xf7,0x00,0x03,0x00,0x6f,0x02,0x00,0x00,0x00,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x6d,0x02,0x00,0x00,0x6e,0x02,0x00,0x00, -0x6f,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x6e,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x77,0x02,0x00,0x00, -0x4e,0x02,0x00,0x00,0x9c,0x02,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x79,0x02,0x00,0x00,0x12,0x00,0x00,0x00, -0x78,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x7a,0x02,0x00,0x00,0x79,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x7b,0x02,0x00,0x00,0x77,0x02,0x00,0x00, -0x7a,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x7c,0x02,0x00,0x00,0x2e,0x02,0x00,0x00,0x7b,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x7e,0x02,0x00,0x00, -0x7c,0x02,0x00,0x00,0x46,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x80,0x02,0x00,0x00,0x7e,0x02,0x00,0x00, -0x9e,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x82,0x02,0x00,0x00,0x99,0x02,0x00,0x00,0xb1,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x84,0x02,0x00,0x00, -0x82,0x02,0x00,0x00,0x9c,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x86,0x02,0x00,0x00,0x84,0x02,0x00,0x00, -0x85,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x88,0x02,0x00,0x00,0x9a,0x02,0x00,0x00,0x62,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x89,0x02,0x00,0x00, -0x86,0x02,0x00,0x00,0x88,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x8b,0x02,0x00,0x00,0x89,0x02,0x00,0x00, -0x9e,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, -0x8c,0x02,0x00,0x00,0xbf,0x00,0x00,0x00,0x8b,0x02,0x00,0x00, -0x3d,0x00,0x04,0x00,0xb9,0x00,0x00,0x00,0x8d,0x02,0x00,0x00, -0x8c,0x02,0x00,0x00,0x41,0x00,0x06,0x00,0x52,0x01,0x00,0x00, -0x8e,0x02,0x00,0x00,0x73,0x02,0x00,0x00,0x34,0x00,0x00,0x00, -0x80,0x02,0x00,0x00,0x3e,0x00,0x03,0x00,0x8e,0x02,0x00,0x00, -0x8d,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x6f,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x6f,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x5b,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x5b,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x90,0x02,0x00,0x00, -0x9e,0x02,0x00,0x00,0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x58,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x5a,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0x53,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x53,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x92,0x02,0x00,0x00,0x9c,0x02,0x00,0x00,0xc5,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x50,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x52,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x3b,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x3b,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x94,0x02,0x00,0x00,0x9a,0x02,0x00,0x00, -0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x38,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x3a,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x33,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x33,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x96,0x02,0x00,0x00, -0x99,0x02,0x00,0x00,0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x30,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x32,0x02,0x00,0x00, -0xfd,0x00,0x01,0x00,0x38,0x00,0x01,0x00, -}; -const uint64_t matmul_f16_f32_s_fp32_len = 10100; - -unsigned char matmul_f16_l_data[] = { -0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, -0xcc,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, -0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x09,0x00,0x00,0x00, -0x11,0x00,0x02,0x00,0x51,0x11,0x00,0x00,0x0b,0x00,0x06,0x00, -0x01,0x00,0x00,0x00,0x47,0x4c,0x53,0x4c,0x2e,0x73,0x74,0x64, -0x2e,0x34,0x35,0x30,0x00,0x00,0x00,0x00,0x0e,0x00,0x03,0x00, -0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x0f,0x00,0x0f,0x00, -0x05,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x6d,0x61,0x69,0x6e, -0x00,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x3d,0x00,0x00,0x00,0x4c,0x00,0x00,0x00,0xf1,0x00,0x00,0x00, -0xfc,0x00,0x00,0x00,0x3c,0x01,0x00,0x00,0x47,0x01,0x00,0x00, -0x2c,0x02,0x00,0x00,0x75,0x02,0x00,0x00,0x10,0x00,0x06,0x00, -0x04,0x00,0x00,0x00,0x11,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x0b,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x1c,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x10,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x08,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x03,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x10,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x10,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x05,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x14,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x10,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x1c,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x08,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x24,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x10,0x00,0x00,0x00,0x0a,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x28,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x0b,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x2c,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x30,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x10,0x00,0x00,0x00,0x0d,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x34,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x10,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x37,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x3d,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x1a,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x4c,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x1b,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x4f,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x53,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x60,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x62,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x6c,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x03,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x9c,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xae,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0xb1,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x08,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0xf9,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0xfa,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0xfa,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0xfa,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xfc,0x00,0x00,0x00, -0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0xfc,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x16,0x01,0x00,0x00,0x01,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x17,0x01,0x00,0x00, -0x0b,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x44,0x01,0x00,0x00,0x06,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x48,0x00,0x04,0x00,0x45,0x01,0x00,0x00,0x00,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x45,0x01,0x00,0x00, -0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x03,0x00,0x45,0x01,0x00,0x00,0x02,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x47,0x01,0x00,0x00,0x22,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x47,0x01,0x00,0x00, -0x21,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x2c,0x02,0x00,0x00,0x0b,0x00,0x00,0x00,0x18,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x72,0x02,0x00,0x00,0x06,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0x73,0x02,0x00,0x00, -0x00,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x73,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x73,0x02,0x00,0x00, -0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x75,0x02,0x00,0x00, -0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x75,0x02,0x00,0x00,0x21,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x13,0x00,0x02,0x00,0x02,0x00,0x00,0x00,0x21,0x00,0x03,0x00, -0x03,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x15,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x17,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x03,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x0a,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x0a,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x0d,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x1e,0x00,0x10,0x00, -0x10,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x11,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x10,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x11,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x15,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x14,0x00,0x00,0x00, -0x08,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x15,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x0a,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x27,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, -0x2d,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x37,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x0a,0x00,0x00,0x00,0x3d,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x3e,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00, -0x4c,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x53,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x54,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x37,0x00,0x00,0x00, -0x53,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x58,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x37,0x00,0x00,0x00, -0x53,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x60,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x61,0x00,0x00,0x00,0x86,0x00,0x00,0x00, -0x53,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x32,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x63,0x00,0x00,0x00, -0x86,0x00,0x00,0x00,0x61,0x00,0x00,0x00,0x62,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x67,0x00,0x00,0x00, -0x86,0x00,0x00,0x00,0x61,0x00,0x00,0x00,0x62,0x00,0x00,0x00, -0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x10,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x6d,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x72,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, -0x76,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0x7b,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x86,0x00,0x00,0x00, -0x0b,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, -0x8c,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0x97,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x9c,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, -0x9e,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xad,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x32,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xae,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xaf,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0xae,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xb0,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x62,0x00,0x00,0x00, -0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xb1,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xb2,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0xb0,0x00,0x00,0x00, -0xb1,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xb3,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0xb2,0x00,0x00,0x00, -0x60,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xb4,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0xaf,0x00,0x00,0x00, -0xb3,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xb5,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0xad,0x00,0x00,0x00, -0xb4,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xb6,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0xb5,0x00,0x00,0x00, -0xb1,0x00,0x00,0x00,0x14,0x00,0x02,0x00,0xb7,0x00,0x00,0x00, -0x16,0x00,0x03,0x00,0xb9,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xba,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xbb,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0xba,0x00,0x00,0x00,0xb4,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xbc,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0xbb,0x00,0x00,0x00,0xb1,0x00,0x00,0x00, -0x1c,0x00,0x04,0x00,0xbd,0x00,0x00,0x00,0xb9,0x00,0x00,0x00, -0xbc,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0xbe,0x00,0x00,0x00, -0x07,0x00,0x00,0x00,0xbd,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0xb9,0x00,0x00,0x00,0xc1,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0xc2,0x00,0x00,0x00,0x07,0x00,0x00,0x00, -0xb9,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, -0xc5,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x16,0x00,0x03,0x00, -0xec,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xed,0x00,0x00,0x00,0x80,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xee,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0x37,0x00,0x00,0x00,0xed,0x00,0x00,0x00,0x1c,0x00,0x04,0x00, -0xef,0x00,0x00,0x00,0xec,0x00,0x00,0x00,0xee,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0xf0,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0xef,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0xf0,0x00,0x00,0x00, -0xf1,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xf5,0x00,0x00,0x00,0x80,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, -0xf9,0x00,0x00,0x00,0xec,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, -0xfa,0x00,0x00,0x00,0xf9,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0xfb,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0xfa,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0xfb,0x00,0x00,0x00,0xfc,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x07,0x01,0x00,0x00, -0x0c,0x00,0x00,0x00,0xec,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x0a,0x01,0x00,0x00,0x04,0x00,0x00,0x00,0xec,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x10,0x01,0x00,0x00, -0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0xec,0x00,0x00,0x00,0x14,0x01,0x00,0x00, -0x00,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x16,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x33,0x00,0x06,0x00, -0x09,0x00,0x00,0x00,0x17,0x01,0x00,0x00,0x16,0x01,0x00,0x00, -0x39,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x18,0x01,0x00,0x00,0x51,0x00,0x00,0x00, -0x17,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x19,0x01,0x00,0x00,0x84,0x00,0x00,0x00, -0x18,0x01,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x1a,0x01,0x00,0x00,0x86,0x00,0x00,0x00, -0x19,0x01,0x00,0x00,0x6c,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x38,0x01,0x00,0x00,0x80,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x39,0x01,0x00,0x00,0x84,0x00,0x00,0x00, -0x9c,0x00,0x00,0x00,0x38,0x01,0x00,0x00,0x1c,0x00,0x04,0x00, -0x3a,0x01,0x00,0x00,0xec,0x00,0x00,0x00,0x39,0x01,0x00,0x00, -0x20,0x00,0x04,0x00,0x3b,0x01,0x00,0x00,0x04,0x00,0x00,0x00, -0x3a,0x01,0x00,0x00,0x3b,0x00,0x04,0x00,0x3b,0x01,0x00,0x00, -0x3c,0x01,0x00,0x00,0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x40,0x01,0x00,0x00,0x80,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, -0x44,0x01,0x00,0x00,0xec,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, -0x45,0x01,0x00,0x00,0x44,0x01,0x00,0x00,0x20,0x00,0x04,0x00, -0x46,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x45,0x01,0x00,0x00, -0x3b,0x00,0x04,0x00,0x46,0x01,0x00,0x00,0x47,0x01,0x00,0x00, -0x0c,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x59,0x01,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x60,0x01,0x00,0x00,0x08,0x01,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x61,0x01,0x00,0x00,0x86,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x64,0x01,0x00,0x00,0x86,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x7f,0x01,0x00,0x00,0x84,0x00,0x00,0x00, -0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x1c,0x00,0x04,0x00, -0x80,0x01,0x00,0x00,0xec,0x00,0x00,0x00,0x7f,0x01,0x00,0x00, -0x20,0x00,0x04,0x00,0x81,0x01,0x00,0x00,0x07,0x00,0x00,0x00, -0x80,0x01,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x91,0x01,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x97,0x01,0x00,0x00, -0x07,0x00,0x00,0x00,0xec,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xad,0x01,0x00,0x00,0x84,0x00,0x00,0x00, -0xb4,0x00,0x00,0x00,0xb1,0x00,0x00,0x00,0x1c,0x00,0x04,0x00, -0xae,0x01,0x00,0x00,0xec,0x00,0x00,0x00,0xad,0x01,0x00,0x00, -0x20,0x00,0x04,0x00,0xaf,0x01,0x00,0x00,0x07,0x00,0x00,0x00, -0xae,0x01,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xb8,0x01,0x00,0x00,0x86,0x00,0x00,0x00,0xae,0x00,0x00,0x00, -0xb4,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xc0,0x01,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xef,0x01,0x00,0x00,0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, -0x24,0x02,0x00,0x00,0x0d,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x0a,0x00,0x00,0x00,0x2c,0x02,0x00,0x00,0x01,0x00,0x00,0x00, -0x1d,0x00,0x03,0x00,0x72,0x02,0x00,0x00,0xb9,0x00,0x00,0x00, -0x1e,0x00,0x03,0x00,0x73,0x02,0x00,0x00,0x72,0x02,0x00,0x00, -0x20,0x00,0x04,0x00,0x74,0x02,0x00,0x00,0x0c,0x00,0x00,0x00, -0x73,0x02,0x00,0x00,0x3b,0x00,0x04,0x00,0x74,0x02,0x00,0x00, -0x75,0x02,0x00,0x00,0x0c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0x7a,0x02,0x00,0x00,0x05,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x87,0x02,0x00,0x00, -0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x90,0x02,0x00,0x00,0x0c,0x00,0x00,0x00, -0xb9,0x00,0x00,0x00,0x36,0x00,0x05,0x00,0x02,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x03,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0x05,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0xbe,0x00,0x00,0x00,0xbf,0x00,0x00,0x00,0x07,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x81,0x01,0x00,0x00,0x82,0x01,0x00,0x00, -0x07,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0xaf,0x01,0x00,0x00, -0xb0,0x01,0x00,0x00,0x07,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x0d,0x00,0x00,0x00,0x0e,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x0f,0x00,0x00,0x00,0x0e,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x14,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x17,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, -0x17,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x1e,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x17,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x22,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x22,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x24,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x27,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x29,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x2a,0x00,0x00,0x00,0x1e,0x00,0x00,0x00, -0x29,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x2d,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x2f,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x30,0x00,0x00,0x00,0x24,0x00,0x00,0x00,0x2f,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x32,0x00,0x00,0x00, -0x30,0x00,0x00,0x00,0x2a,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x35,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x34,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x36,0x00,0x00,0x00,0x35,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x38,0x00,0x00,0x00,0x36,0x00,0x00,0x00, -0x37,0x00,0x00,0x00,0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x3a,0x00,0x00,0x00,0x38,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3b,0x00,0x00,0x00, -0x3a,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x0d,0x00,0x00,0x00,0x3f,0x00,0x00,0x00,0x3d,0x00,0x00,0x00, -0x3e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0x3f,0x00,0x00,0x00,0x89,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x42,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x3b,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x47,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x3b,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00,0x49,0x00,0x00,0x00, -0x3d,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x4a,0x00,0x00,0x00,0x49,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00,0x4d,0x00,0x00,0x00, -0x4c,0x00,0x00,0x00,0x3e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x4d,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x50,0x00,0x00,0x00, -0x4e,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x89,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x55,0x00,0x00,0x00,0x50,0x00,0x00,0x00, -0x54,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x59,0x00,0x00,0x00,0x50,0x00,0x00,0x00,0x58,0x00,0x00,0x00, -0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x5d,0x00,0x00,0x00, -0x4e,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x89,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x64,0x00,0x00,0x00,0x5d,0x00,0x00,0x00, -0x63,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x68,0x00,0x00,0x00,0x5d,0x00,0x00,0x00,0x67,0x00,0x00,0x00, -0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x6e,0x00,0x00,0x00, -0x4e,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x73,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, -0x72,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0x77,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x76,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x78,0x00,0x00,0x00, -0x77,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x79,0x00,0x00,0x00,0x47,0x00,0x00,0x00,0x78,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x7c,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x7b,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x7d,0x00,0x00,0x00,0x7c,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x7f,0x00,0x00,0x00, -0x47,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x82,0x00,0x00,0x00,0x7f,0x00,0x00,0x00, -0x78,0x00,0x00,0x00,0x0c,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0x83,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x26,0x00,0x00,0x00, -0x7d,0x00,0x00,0x00,0x82,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x87,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x86,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x88,0x00,0x00,0x00,0x87,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x89,0x00,0x00,0x00,0x32,0x00,0x00,0x00, -0x88,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x8b,0x00,0x00,0x00,0x42,0x00,0x00,0x00,0x37,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x8d,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x8c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x8e,0x00,0x00,0x00,0x8d,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8f,0x00,0x00,0x00, -0x8b,0x00,0x00,0x00,0x8e,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x90,0x00,0x00,0x00,0x89,0x00,0x00,0x00, -0x8f,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x92,0x00,0x00,0x00,0x90,0x00,0x00,0x00,0x79,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x93,0x00,0x00,0x00, -0x92,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x98,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x97,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x99,0x00,0x00,0x00,0x98,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x9a,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, -0x99,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x9d,0x00,0x00,0x00,0x4a,0x00,0x00,0x00,0x9c,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x9f,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x9e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xa0,0x00,0x00,0x00,0x9f,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa1,0x00,0x00,0x00, -0x9d,0x00,0x00,0x00,0xa0,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xa2,0x00,0x00,0x00,0x9a,0x00,0x00,0x00, -0xa1,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xa4,0x00,0x00,0x00,0xa2,0x00,0x00,0x00,0x79,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa5,0x00,0x00,0x00, -0xa4,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xa7,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xa7,0x00,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x9a,0x02,0x00,0x00, -0x3e,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0xc6,0x00,0x00,0x00, -0xa8,0x00,0x00,0x00,0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00, -0xb8,0x00,0x00,0x00,0x9a,0x02,0x00,0x00,0xb6,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0xa9,0x00,0x00,0x00,0xa8,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xb8,0x00,0x00,0x00, -0xa8,0x00,0x00,0x00,0xa9,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xa8,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, -0xc3,0x00,0x00,0x00,0xbf,0x00,0x00,0x00,0x9a,0x02,0x00,0x00, -0x3e,0x00,0x03,0x00,0xc3,0x00,0x00,0x00,0xc1,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xc6,0x00,0x00,0x00, -0x9a,0x02,0x00,0x00,0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xa7,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xa9,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xc9,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xc9,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xb3,0x02,0x00,0x00,0xa5,0x00,0x00,0x00,0xa9,0x00,0x00,0x00, -0x66,0x01,0x00,0x00,0xcc,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xaf,0x02,0x00,0x00,0x93,0x00,0x00,0x00, -0xa9,0x00,0x00,0x00,0x63,0x01,0x00,0x00,0xcc,0x00,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x9b,0x02,0x00,0x00, -0x79,0x00,0x00,0x00,0xa9,0x00,0x00,0x00,0x14,0x02,0x00,0x00, -0xcc,0x00,0x00,0x00,0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00, -0xd0,0x00,0x00,0x00,0x9b,0x02,0x00,0x00,0x83,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0xcb,0x00,0x00,0x00,0xcc,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xd0,0x00,0x00,0x00, -0xca,0x00,0x00,0x00,0xcb,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xca,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xd2,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xd2,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xab,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, -0xca,0x00,0x00,0x00,0x1c,0x01,0x00,0x00,0xd5,0x00,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00,0xd8,0x00,0x00,0x00, -0xab,0x02,0x00,0x00,0x37,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0xd4,0x00,0x00,0x00,0xd5,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xd8,0x00,0x00,0x00,0xd3,0x00,0x00,0x00, -0xd4,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xd3,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xdc,0x00,0x00,0x00, -0x8b,0x00,0x00,0x00,0x73,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xde,0x00,0x00,0x00,0xdc,0x00,0x00,0x00, -0xab,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00, -0xe1,0x00,0x00,0x00,0xde,0x00,0x00,0x00,0x36,0x00,0x00,0x00, -0xf7,0x00,0x03,0x00,0xe3,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xe1,0x00,0x00,0x00,0xe2,0x00,0x00,0x00, -0xe3,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xe2,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xe6,0x00,0x00,0x00, -0x9b,0x02,0x00,0x00,0x6e,0x00,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb7,0x00,0x00,0x00,0xe8,0x00,0x00,0x00,0xe6,0x00,0x00,0x00, -0x83,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xe3,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xe3,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, -0xb7,0x00,0x00,0x00,0xe9,0x00,0x00,0x00,0xe1,0x00,0x00,0x00, -0xd3,0x00,0x00,0x00,0xe8,0x00,0x00,0x00,0xe2,0x00,0x00,0x00, -0xf7,0x00,0x03,0x00,0xeb,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xe9,0x00,0x00,0x00,0xea,0x00,0x00,0x00, -0x0c,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xea,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xf4,0x00,0x00,0x00, -0x73,0x00,0x00,0x00,0xab,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xf6,0x00,0x00,0x00,0xf4,0x00,0x00,0x00, -0xf5,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xf8,0x00,0x00,0x00,0xf6,0x00,0x00,0x00,0x6e,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x03,0x01,0x00,0x00, -0xf4,0x00,0x00,0x00,0x8e,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x04,0x01,0x00,0x00,0xaf,0x02,0x00,0x00, -0x03,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x06,0x01,0x00,0x00,0x04,0x01,0x00,0x00,0x6e,0x00,0x00,0x00, -0x41,0x00,0x06,0x00,0x07,0x01,0x00,0x00,0x08,0x01,0x00,0x00, -0xfc,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0x06,0x01,0x00,0x00, -0x3d,0x00,0x04,0x00,0xec,0x00,0x00,0x00,0x09,0x01,0x00,0x00, -0x08,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0x0a,0x01,0x00,0x00, -0x0b,0x01,0x00,0x00,0xf1,0x00,0x00,0x00,0xf8,0x00,0x00,0x00, -0x3e,0x00,0x03,0x00,0x0b,0x01,0x00,0x00,0x09,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0xeb,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x0c,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x0f,0x01,0x00,0x00,0x73,0x00,0x00,0x00,0xab,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x11,0x01,0x00,0x00, -0x0f,0x01,0x00,0x00,0x10,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x13,0x01,0x00,0x00,0x11,0x01,0x00,0x00, -0x6e,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x0a,0x01,0x00,0x00, -0x15,0x01,0x00,0x00,0xf1,0x00,0x00,0x00,0x13,0x01,0x00,0x00, -0x3e,0x00,0x03,0x00,0x15,0x01,0x00,0x00,0x14,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0xeb,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xeb,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xd5,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xd5,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x1c,0x01,0x00,0x00,0xab,0x02,0x00,0x00, -0x1a,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xd2,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xd4,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x1e,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x1e,0x01,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xac,0x02,0x00,0x00, -0x3e,0x00,0x00,0x00,0xd4,0x00,0x00,0x00,0x5f,0x01,0x00,0x00, -0x21,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00, -0x24,0x01,0x00,0x00,0xac,0x02,0x00,0x00,0x9c,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0x20,0x01,0x00,0x00,0x21,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x24,0x01,0x00,0x00, -0x1f,0x01,0x00,0x00,0x20,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x1f,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x28,0x01,0x00,0x00,0x9d,0x00,0x00,0x00,0x73,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x2a,0x01,0x00,0x00, -0x28,0x01,0x00,0x00,0xac,0x02,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x2b,0x01,0x00,0x00,0x12,0x00,0x00,0x00, -0xc5,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x2c,0x01,0x00,0x00,0x2b,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb7,0x00,0x00,0x00,0x2d,0x01,0x00,0x00,0x2a,0x01,0x00,0x00, -0x2c,0x01,0x00,0x00,0xf7,0x00,0x03,0x00,0x2f,0x01,0x00,0x00, -0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x2d,0x01,0x00,0x00, -0x2e,0x01,0x00,0x00,0x2f,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x2e,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x32,0x01,0x00,0x00,0x9b,0x02,0x00,0x00,0x6e,0x00,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00,0x34,0x01,0x00,0x00, -0x32,0x01,0x00,0x00,0x83,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x2f,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x2f,0x01,0x00,0x00, -0xf5,0x00,0x07,0x00,0xb7,0x00,0x00,0x00,0x35,0x01,0x00,0x00, -0x2d,0x01,0x00,0x00,0x1f,0x01,0x00,0x00,0x34,0x01,0x00,0x00, -0x2e,0x01,0x00,0x00,0xf7,0x00,0x03,0x00,0x37,0x01,0x00,0x00, -0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x35,0x01,0x00,0x00, -0x36,0x01,0x00,0x00,0x55,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x36,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x3f,0x01,0x00,0x00,0x73,0x00,0x00,0x00,0xac,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x41,0x01,0x00,0x00, -0x3f,0x01,0x00,0x00,0x40,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x43,0x01,0x00,0x00,0x41,0x01,0x00,0x00, -0x6e,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x4e,0x01,0x00,0x00,0x3f,0x01,0x00,0x00,0xa0,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x4f,0x01,0x00,0x00, -0xb3,0x02,0x00,0x00,0x4e,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x51,0x01,0x00,0x00,0x4f,0x01,0x00,0x00, -0x6e,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x07,0x01,0x00,0x00, -0x52,0x01,0x00,0x00,0x47,0x01,0x00,0x00,0x34,0x00,0x00,0x00, -0x51,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0xec,0x00,0x00,0x00, -0x53,0x01,0x00,0x00,0x52,0x01,0x00,0x00,0x41,0x00,0x05,0x00, -0x0a,0x01,0x00,0x00,0x54,0x01,0x00,0x00,0x3c,0x01,0x00,0x00, -0x43,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x54,0x01,0x00,0x00, -0x53,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x37,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x55,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x58,0x01,0x00,0x00,0x73,0x00,0x00,0x00, -0xac,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x5a,0x01,0x00,0x00,0x58,0x01,0x00,0x00,0x59,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x5c,0x01,0x00,0x00, -0x5a,0x01,0x00,0x00,0x6e,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x0a,0x01,0x00,0x00,0x5d,0x01,0x00,0x00,0x3c,0x01,0x00,0x00, -0x5c,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x5d,0x01,0x00,0x00, -0x14,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x37,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x37,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0x21,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x21,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x5f,0x01,0x00,0x00, -0xac,0x02,0x00,0x00,0x1a,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0x1e,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x20,0x01,0x00,0x00, -0xe0,0x00,0x04,0x00,0x0c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x60,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x63,0x01,0x00,0x00,0xaf,0x02,0x00,0x00,0x61,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x66,0x01,0x00,0x00, -0xb3,0x02,0x00,0x00,0x64,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0x68,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x68,0x01,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xb5,0x02,0x00,0x00, -0x3e,0x00,0x00,0x00,0x20,0x01,0x00,0x00,0x12,0x02,0x00,0x00, -0x6b,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00, -0x6e,0x01,0x00,0x00,0xb5,0x02,0x00,0x00,0x6c,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0x6a,0x01,0x00,0x00,0x6b,0x01,0x00,0x00, -0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x6e,0x01,0x00,0x00, -0x69,0x01,0x00,0x00,0x6a,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x69,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x70,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x70,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xb9,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, -0x69,0x01,0x00,0x00,0x9c,0x01,0x00,0x00,0x73,0x01,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00,0x76,0x01,0x00,0x00, -0xb9,0x02,0x00,0x00,0x60,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0x72,0x01,0x00,0x00,0x73,0x01,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x76,0x01,0x00,0x00,0x71,0x01,0x00,0x00, -0x72,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x71,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0x78,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x78,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xcb,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0x71,0x01,0x00,0x00, -0x9a,0x01,0x00,0x00,0x79,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb7,0x00,0x00,0x00,0x7e,0x01,0x00,0x00,0xcb,0x02,0x00,0x00, -0x62,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x7a,0x01,0x00,0x00, -0x79,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x7e,0x01,0x00,0x00,0x79,0x01,0x00,0x00,0x7a,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x79,0x01,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x84,0x01,0x00,0x00,0xb9,0x02,0x00,0x00, -0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x86,0x01,0x00,0x00,0x84,0x01,0x00,0x00,0xcb,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x88,0x01,0x00,0x00, -0x55,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x8a,0x01,0x00,0x00,0xb9,0x02,0x00,0x00, -0x61,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x8b,0x01,0x00,0x00,0x88,0x01,0x00,0x00,0x8a,0x01,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8d,0x01,0x00,0x00, -0x64,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x8e,0x01,0x00,0x00,0x8b,0x01,0x00,0x00, -0x8d,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x90,0x01,0x00,0x00,0x8e,0x01,0x00,0x00,0xcb,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x92,0x01,0x00,0x00, -0x90,0x01,0x00,0x00,0x91,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x94,0x01,0x00,0x00,0x92,0x01,0x00,0x00, -0xb5,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x0a,0x01,0x00,0x00, -0x95,0x01,0x00,0x00,0xf1,0x00,0x00,0x00,0x94,0x01,0x00,0x00, -0x3d,0x00,0x04,0x00,0xec,0x00,0x00,0x00,0x96,0x01,0x00,0x00, -0x95,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0x97,0x01,0x00,0x00, -0x98,0x01,0x00,0x00,0x82,0x01,0x00,0x00,0x86,0x01,0x00,0x00, -0x3e,0x00,0x03,0x00,0x98,0x01,0x00,0x00,0x96,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9a,0x01,0x00,0x00, -0xcb,0x02,0x00,0x00,0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x78,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x7a,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0x73,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x73,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x9c,0x01,0x00,0x00,0xb9,0x02,0x00,0x00,0xc5,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x70,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x72,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x9e,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x9e,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xba,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, -0x72,0x01,0x00,0x00,0xca,0x01,0x00,0x00,0xa1,0x01,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00,0xa4,0x01,0x00,0x00, -0xba,0x02,0x00,0x00,0xb4,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0xa0,0x01,0x00,0x00,0xa1,0x01,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xa4,0x01,0x00,0x00,0x9f,0x01,0x00,0x00, -0xa0,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x9f,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0xa6,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xa6,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xc8,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0x9f,0x01,0x00,0x00, -0xc8,0x01,0x00,0x00,0xa7,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb7,0x00,0x00,0x00,0xac,0x01,0x00,0x00,0xc8,0x02,0x00,0x00, -0xb1,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xa8,0x01,0x00,0x00, -0xa7,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0xac,0x01,0x00,0x00,0xa7,0x01,0x00,0x00,0xa8,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xa7,0x01,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xb2,0x01,0x00,0x00,0xba,0x02,0x00,0x00, -0xb1,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xb4,0x01,0x00,0x00,0xb2,0x01,0x00,0x00,0xc8,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xb6,0x01,0x00,0x00, -0x59,0x00,0x00,0x00,0xae,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xb9,0x01,0x00,0x00,0xba,0x02,0x00,0x00, -0xb8,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xba,0x01,0x00,0x00,0xb6,0x01,0x00,0x00,0xb9,0x01,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xbc,0x01,0x00,0x00, -0x68,0x00,0x00,0x00,0xb1,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xbd,0x01,0x00,0x00,0xba,0x01,0x00,0x00, -0xbc,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xbf,0x01,0x00,0x00,0xbd,0x01,0x00,0x00,0xc8,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xc1,0x01,0x00,0x00, -0xbf,0x01,0x00,0x00,0xc0,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xc3,0x01,0x00,0x00,0xc1,0x01,0x00,0x00, -0xb5,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x0a,0x01,0x00,0x00, -0xc4,0x01,0x00,0x00,0x3c,0x01,0x00,0x00,0xc3,0x01,0x00,0x00, -0x3d,0x00,0x04,0x00,0xec,0x00,0x00,0x00,0xc5,0x01,0x00,0x00, -0xc4,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0x97,0x01,0x00,0x00, -0xc6,0x01,0x00,0x00,0xb0,0x01,0x00,0x00,0xb4,0x01,0x00,0x00, -0x3e,0x00,0x03,0x00,0xc6,0x01,0x00,0x00,0xc5,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xc8,0x01,0x00,0x00, -0xc8,0x02,0x00,0x00,0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xa6,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xa8,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0xa1,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xa1,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xca,0x01,0x00,0x00,0xba,0x02,0x00,0x00,0xc5,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x9e,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xa0,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xcc,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xcc,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xbb,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, -0xa0,0x01,0x00,0x00,0x10,0x02,0x00,0x00,0xcf,0x01,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00,0xd2,0x01,0x00,0x00, -0xbb,0x02,0x00,0x00,0xb4,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0xce,0x01,0x00,0x00,0xcf,0x01,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xd2,0x01,0x00,0x00,0xcd,0x01,0x00,0x00, -0xce,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xcd,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0xd4,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xd4,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xbf,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0xcd,0x01,0x00,0x00, -0x0e,0x02,0x00,0x00,0xd7,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb7,0x00,0x00,0x00,0xda,0x01,0x00,0x00,0xbf,0x02,0x00,0x00, -0x60,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xd6,0x01,0x00,0x00, -0xd7,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0xda,0x01,0x00,0x00,0xd5,0x01,0x00,0x00,0xd6,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xd5,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0xdc,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xdc,0x01,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xc1,0x02,0x00,0x00, -0x3e,0x00,0x00,0x00,0xd5,0x01,0x00,0x00,0x0c,0x02,0x00,0x00, -0xdf,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00, -0xe2,0x01,0x00,0x00,0xc1,0x02,0x00,0x00,0xb1,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0xde,0x01,0x00,0x00,0xdf,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xe2,0x01,0x00,0x00, -0xdd,0x01,0x00,0x00,0xde,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xdd,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xe4,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xe4,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xc3,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, -0xdd,0x01,0x00,0x00,0x0a,0x02,0x00,0x00,0xe5,0x01,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00,0xea,0x01,0x00,0x00, -0xc3,0x02,0x00,0x00,0x62,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0xe6,0x01,0x00,0x00,0xe5,0x01,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xea,0x01,0x00,0x00,0xe5,0x01,0x00,0x00, -0xe6,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xe5,0x01,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xec,0x01,0x00,0x00, -0xbb,0x02,0x00,0x00,0xb1,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xee,0x01,0x00,0x00,0xec,0x01,0x00,0x00, -0xc1,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xf0,0x01,0x00,0x00,0xee,0x01,0x00,0x00,0xef,0x01,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xf2,0x01,0x00,0x00, -0xbf,0x02,0x00,0x00,0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xf3,0x01,0x00,0x00,0xf0,0x01,0x00,0x00, -0xf2,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xf5,0x01,0x00,0x00,0xf3,0x01,0x00,0x00,0xc3,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xf9,0x01,0x00,0x00, -0xf2,0x01,0x00,0x00,0xc3,0x02,0x00,0x00,0x41,0x00,0x05,0x00, -0x97,0x01,0x00,0x00,0xfa,0x01,0x00,0x00,0x82,0x01,0x00,0x00, -0xf9,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0xec,0x00,0x00,0x00, -0xfb,0x01,0x00,0x00,0xfa,0x01,0x00,0x00,0x73,0x00,0x04,0x00, -0xb9,0x00,0x00,0x00,0xfc,0x01,0x00,0x00,0xfb,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0x97,0x01,0x00,0x00,0x01,0x02,0x00,0x00, -0xb0,0x01,0x00,0x00,0xee,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, -0xec,0x00,0x00,0x00,0x02,0x02,0x00,0x00,0x01,0x02,0x00,0x00, -0x73,0x00,0x04,0x00,0xb9,0x00,0x00,0x00,0x03,0x02,0x00,0x00, -0x02,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, -0x05,0x02,0x00,0x00,0xbf,0x00,0x00,0x00,0xf5,0x01,0x00,0x00, -0x3d,0x00,0x04,0x00,0xb9,0x00,0x00,0x00,0x06,0x02,0x00,0x00, -0x05,0x02,0x00,0x00,0x0c,0x00,0x08,0x00,0xb9,0x00,0x00,0x00, -0x07,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00, -0xfc,0x01,0x00,0x00,0x03,0x02,0x00,0x00,0x06,0x02,0x00,0x00, -0x3e,0x00,0x03,0x00,0x05,0x02,0x00,0x00,0x07,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x0a,0x02,0x00,0x00, -0xc3,0x02,0x00,0x00,0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xe4,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xe6,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0xdf,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xdf,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x0c,0x02,0x00,0x00,0xc1,0x02,0x00,0x00,0xc5,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xdc,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xde,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xd7,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xd7,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x0e,0x02,0x00,0x00,0xbf,0x02,0x00,0x00, -0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xd4,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xd6,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0xcf,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xcf,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x10,0x02,0x00,0x00, -0xbb,0x02,0x00,0x00,0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xcc,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xce,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0x6b,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x6b,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x12,0x02,0x00,0x00,0xb5,0x02,0x00,0x00,0xc5,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x68,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x6a,0x01,0x00,0x00,0xe0,0x00,0x04,0x00,0x0c,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x60,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0xcc,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xcc,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x14,0x02,0x00,0x00, -0x9b,0x02,0x00,0x00,0x6c,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xc9,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xcb,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x19,0x02,0x00,0x00, -0x55,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x1a,0x02,0x00,0x00,0x8b,0x00,0x00,0x00, -0x19,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x1f,0x02,0x00,0x00,0x59,0x00,0x00,0x00,0xae,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x20,0x02,0x00,0x00, -0x9d,0x00,0x00,0x00,0x1f,0x02,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x25,0x02,0x00,0x00,0x12,0x00,0x00,0x00, -0x24,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x26,0x02,0x00,0x00,0x25,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x27,0x02,0x00,0x00,0x0f,0x00,0x00,0x00, -0x26,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x2b,0x02,0x00,0x00,0x47,0x00,0x00,0x00,0x26,0x02,0x00,0x00, -0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00,0x2d,0x02,0x00,0x00, -0x2c,0x02,0x00,0x00,0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x2e,0x02,0x00,0x00,0x2d,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x2f,0x02,0x00,0x00, -0x2b,0x02,0x00,0x00,0x2e,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x30,0x02,0x00,0x00,0x27,0x02,0x00,0x00, -0x2f,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x32,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x32,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0x9c,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, -0xcb,0x00,0x00,0x00,0x99,0x02,0x00,0x00,0x35,0x02,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00,0x38,0x02,0x00,0x00, -0x9c,0x02,0x00,0x00,0xb4,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0x34,0x02,0x00,0x00,0x35,0x02,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x38,0x02,0x00,0x00,0x33,0x02,0x00,0x00, -0x34,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x33,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0x3a,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x3a,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0x9d,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0x33,0x02,0x00,0x00, -0x97,0x02,0x00,0x00,0x3d,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb7,0x00,0x00,0x00,0x40,0x02,0x00,0x00,0x9d,0x02,0x00,0x00, -0x60,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x3c,0x02,0x00,0x00, -0x3d,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x40,0x02,0x00,0x00,0x3b,0x02,0x00,0x00,0x3c,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x3b,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x44,0x02,0x00,0x00,0x9d,0x02,0x00,0x00, -0x61,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x45,0x02,0x00,0x00,0x1a,0x02,0x00,0x00,0x44,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x47,0x02,0x00,0x00, -0x64,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x48,0x02,0x00,0x00,0x45,0x02,0x00,0x00, -0x47,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x4c,0x02,0x00,0x00,0x9c,0x02,0x00,0x00,0xb8,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x4d,0x02,0x00,0x00, -0x20,0x02,0x00,0x00,0x4c,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x4f,0x02,0x00,0x00,0x68,0x00,0x00,0x00, -0xb1,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x50,0x02,0x00,0x00,0x4d,0x02,0x00,0x00,0x4f,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0x52,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x52,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0x9f,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0x3b,0x02,0x00,0x00, -0x95,0x02,0x00,0x00,0x55,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb7,0x00,0x00,0x00,0x58,0x02,0x00,0x00,0x9f,0x02,0x00,0x00, -0xb1,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x54,0x02,0x00,0x00, -0x55,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x58,0x02,0x00,0x00,0x53,0x02,0x00,0x00,0x54,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x53,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x5a,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x5a,0x02,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xa1,0x02,0x00,0x00, -0x3e,0x00,0x00,0x00,0x53,0x02,0x00,0x00,0x93,0x02,0x00,0x00, -0x5d,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00, -0x60,0x02,0x00,0x00,0xa1,0x02,0x00,0x00,0x62,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0x5c,0x02,0x00,0x00,0x5d,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x60,0x02,0x00,0x00, -0x5b,0x02,0x00,0x00,0x5c,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x5b,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x63,0x02,0x00,0x00,0x48,0x02,0x00,0x00,0xa1,0x02,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00,0x66,0x02,0x00,0x00, -0x63,0x02,0x00,0x00,0x36,0x00,0x00,0x00,0xf7,0x00,0x03,0x00, -0x68,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x66,0x02,0x00,0x00,0x67,0x02,0x00,0x00,0x68,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x67,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x6b,0x02,0x00,0x00,0x50,0x02,0x00,0x00, -0x9f,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0x6c,0x02,0x00,0x00,0x12,0x00,0x00,0x00,0xc5,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x6d,0x02,0x00,0x00, -0x6c,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00, -0x6e,0x02,0x00,0x00,0x6b,0x02,0x00,0x00,0x6d,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0x68,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x68,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0xb7,0x00,0x00,0x00, -0x6f,0x02,0x00,0x00,0x66,0x02,0x00,0x00,0x5b,0x02,0x00,0x00, -0x6e,0x02,0x00,0x00,0x67,0x02,0x00,0x00,0xf7,0x00,0x03,0x00, -0x71,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x6f,0x02,0x00,0x00,0x70,0x02,0x00,0x00,0x71,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x70,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x79,0x02,0x00,0x00,0x50,0x02,0x00,0x00, -0x9f,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0x7b,0x02,0x00,0x00,0x12,0x00,0x00,0x00,0x7a,0x02,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x7c,0x02,0x00,0x00, -0x7b,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x7d,0x02,0x00,0x00,0x79,0x02,0x00,0x00,0x7c,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x7e,0x02,0x00,0x00, -0x30,0x02,0x00,0x00,0x7d,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x80,0x02,0x00,0x00,0x7e,0x02,0x00,0x00, -0x48,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x82,0x02,0x00,0x00,0x80,0x02,0x00,0x00,0xa1,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x84,0x02,0x00,0x00, -0x9c,0x02,0x00,0x00,0xb1,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x86,0x02,0x00,0x00,0x84,0x02,0x00,0x00, -0x9f,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x88,0x02,0x00,0x00,0x86,0x02,0x00,0x00,0x87,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8a,0x02,0x00,0x00, -0x9d,0x02,0x00,0x00,0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x8b,0x02,0x00,0x00,0x88,0x02,0x00,0x00, -0x8a,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x8d,0x02,0x00,0x00,0x8b,0x02,0x00,0x00,0xa1,0x02,0x00,0x00, -0x41,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0x8e,0x02,0x00,0x00, -0xbf,0x00,0x00,0x00,0x8d,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, -0xb9,0x00,0x00,0x00,0x8f,0x02,0x00,0x00,0x8e,0x02,0x00,0x00, -0x41,0x00,0x06,0x00,0x90,0x02,0x00,0x00,0x91,0x02,0x00,0x00, -0x75,0x02,0x00,0x00,0x34,0x00,0x00,0x00,0x82,0x02,0x00,0x00, -0x3e,0x00,0x03,0x00,0x91,0x02,0x00,0x00,0x8f,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0x71,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x71,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x5d,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x5d,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x93,0x02,0x00,0x00,0xa1,0x02,0x00,0x00, -0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x5a,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x5c,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x55,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x55,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x95,0x02,0x00,0x00, -0x9f,0x02,0x00,0x00,0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x52,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x54,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0x3d,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x3d,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x97,0x02,0x00,0x00,0x9d,0x02,0x00,0x00,0xc5,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x3a,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x3c,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x35,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x35,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x99,0x02,0x00,0x00,0x9c,0x02,0x00,0x00, -0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x32,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x34,0x02,0x00,0x00,0xfd,0x00,0x01,0x00, -0x38,0x00,0x01,0x00, -}; -const uint64_t matmul_f16_l_len = 10156; - -unsigned char matmul_f16_l_fp32_data[] = { -0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, -0xca,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, -0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x51,0x11,0x00,0x00, -0x0b,0x00,0x06,0x00,0x01,0x00,0x00,0x00,0x47,0x4c,0x53,0x4c, -0x2e,0x73,0x74,0x64,0x2e,0x34,0x35,0x30,0x00,0x00,0x00,0x00, -0x0e,0x00,0x03,0x00,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x0f,0x00,0x0f,0x00,0x05,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x6d,0x61,0x69,0x6e,0x00,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x3d,0x00,0x00,0x00,0x4c,0x00,0x00,0x00, -0xf0,0x00,0x00,0x00,0xfc,0x00,0x00,0x00,0x3c,0x01,0x00,0x00, -0x47,0x01,0x00,0x00,0x2a,0x02,0x00,0x00,0x73,0x02,0x00,0x00, -0x10,0x00,0x06,0x00,0x04,0x00,0x00,0x00,0x11,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x0b,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x1c,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x10,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x08,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x03,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x10,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x14,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x18,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x07,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x10,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x24,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x0a,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x10,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x2c,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x30,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x0d,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0x47,0x00,0x03,0x00, -0x10,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x37,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x3d,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x1a,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x4c,0x00,0x00,0x00, -0x0b,0x00,0x00,0x00,0x1b,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x4f,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x53,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x60,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x62,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x07,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x6c,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x03,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x9c,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0xae,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x05,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0xb1,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x08,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xf9,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x48,0x00,0x04,0x00, -0xfa,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0xfa,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, -0xfa,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0xfc,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0xfc,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x16,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x17,0x01,0x00,0x00,0x0b,0x00,0x00,0x00,0x19,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x44,0x01,0x00,0x00,0x06,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0x45,0x01,0x00,0x00, -0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x45,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x45,0x01,0x00,0x00, -0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x47,0x01,0x00,0x00, -0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x47,0x01,0x00,0x00,0x21,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x2a,0x02,0x00,0x00,0x0b,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x70,0x02,0x00,0x00, -0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00, -0x71,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x19,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x71,0x02,0x00,0x00,0x00,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, -0x71,0x02,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x73,0x02,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x73,0x02,0x00,0x00,0x21,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x13,0x00,0x02,0x00,0x02,0x00,0x00,0x00, -0x21,0x00,0x03,0x00,0x03,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x15,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x17,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x0a,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x0d,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x1e,0x00,0x10,0x00,0x10,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x11,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x11,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x15,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, -0x14,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x15,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x0a,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, -0x27,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0x2d,0x00,0x00,0x00,0x07,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x34,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x37,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00,0x3d,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x3e,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x0a,0x00,0x00,0x00,0x4c,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x4f,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x53,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x54,0x00,0x00,0x00,0x86,0x00,0x00,0x00, -0x37,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x58,0x00,0x00,0x00,0x86,0x00,0x00,0x00, -0x37,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x32,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x61,0x00,0x00,0x00, -0x86,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x60,0x00,0x00,0x00, -0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x62,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x63,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x61,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x67,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x61,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x86,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x72,0x00,0x00,0x00,0x86,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0x76,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x7b,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, -0x86,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0x8c,0x00,0x00,0x00,0x03,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x97,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x9c,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0x9e,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xad,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00, -0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xae,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xaf,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x53,0x00,0x00,0x00, -0xae,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xb0,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x4f,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xb1,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xb2,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0xb0,0x00,0x00,0x00,0xb1,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xb3,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0xb2,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xb4,0x00,0x00,0x00,0x86,0x00,0x00,0x00, -0xaf,0x00,0x00,0x00,0xb3,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xb5,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0xad,0x00,0x00,0x00,0xb4,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xb6,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0xb5,0x00,0x00,0x00,0xb1,0x00,0x00,0x00,0x14,0x00,0x02,0x00, -0xb7,0x00,0x00,0x00,0x16,0x00,0x03,0x00,0xb9,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xba,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xbb,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0xba,0x00,0x00,0x00, -0xb4,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xbc,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0xbb,0x00,0x00,0x00, -0xb1,0x00,0x00,0x00,0x1c,0x00,0x04,0x00,0xbd,0x00,0x00,0x00, -0xb9,0x00,0x00,0x00,0xbc,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0xbe,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0xbd,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0xb9,0x00,0x00,0x00,0xc1,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0xc2,0x00,0x00,0x00, -0x07,0x00,0x00,0x00,0xb9,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0xc5,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xec,0x00,0x00,0x00, -0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xed,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0xec,0x00,0x00,0x00, -0x1c,0x00,0x04,0x00,0xee,0x00,0x00,0x00,0xb9,0x00,0x00,0x00, -0xed,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0xef,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0xee,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0xef,0x00,0x00,0x00,0xf0,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xf4,0x00,0x00,0x00, -0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x16,0x00,0x03,0x00,0xf8,0x00,0x00,0x00,0x10,0x00,0x00,0x00, -0x1d,0x00,0x03,0x00,0xf9,0x00,0x00,0x00,0xf8,0x00,0x00,0x00, -0x1e,0x00,0x03,0x00,0xfa,0x00,0x00,0x00,0xf9,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0xfb,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0xfa,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0xfb,0x00,0x00,0x00, -0xfc,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x07,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0xf8,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x0b,0x01,0x00,0x00,0x04,0x00,0x00,0x00, -0xb9,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x11,0x01,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x16,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x33,0x00,0x06,0x00, -0x09,0x00,0x00,0x00,0x17,0x01,0x00,0x00,0x16,0x01,0x00,0x00, -0x39,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x18,0x01,0x00,0x00,0x51,0x00,0x00,0x00, -0x17,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x19,0x01,0x00,0x00,0x84,0x00,0x00,0x00, -0x18,0x01,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x1a,0x01,0x00,0x00,0x86,0x00,0x00,0x00, -0x19,0x01,0x00,0x00,0x6c,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x38,0x01,0x00,0x00,0x80,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x39,0x01,0x00,0x00,0x84,0x00,0x00,0x00, -0x9c,0x00,0x00,0x00,0x38,0x01,0x00,0x00,0x1c,0x00,0x04,0x00, -0x3a,0x01,0x00,0x00,0xb9,0x00,0x00,0x00,0x39,0x01,0x00,0x00, -0x20,0x00,0x04,0x00,0x3b,0x01,0x00,0x00,0x04,0x00,0x00,0x00, -0x3a,0x01,0x00,0x00,0x3b,0x00,0x04,0x00,0x3b,0x01,0x00,0x00, -0x3c,0x01,0x00,0x00,0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x40,0x01,0x00,0x00,0x80,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, -0x44,0x01,0x00,0x00,0xf8,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, -0x45,0x01,0x00,0x00,0x44,0x01,0x00,0x00,0x20,0x00,0x04,0x00, -0x46,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x45,0x01,0x00,0x00, -0x3b,0x00,0x04,0x00,0x46,0x01,0x00,0x00,0x47,0x01,0x00,0x00, -0x0c,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x5a,0x01,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x61,0x01,0x00,0x00,0x08,0x01,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x62,0x01,0x00,0x00,0x86,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x65,0x01,0x00,0x00,0x86,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x80,0x01,0x00,0x00,0x84,0x00,0x00,0x00, -0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x1c,0x00,0x04,0x00, -0x81,0x01,0x00,0x00,0xb9,0x00,0x00,0x00,0x80,0x01,0x00,0x00, -0x20,0x00,0x04,0x00,0x82,0x01,0x00,0x00,0x07,0x00,0x00,0x00, -0x81,0x01,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x92,0x01,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xad,0x01,0x00,0x00,0x84,0x00,0x00,0x00,0xb4,0x00,0x00,0x00, -0xb1,0x00,0x00,0x00,0x1c,0x00,0x04,0x00,0xae,0x01,0x00,0x00, -0xb9,0x00,0x00,0x00,0xad,0x01,0x00,0x00,0x20,0x00,0x04,0x00, -0xaf,0x01,0x00,0x00,0x07,0x00,0x00,0x00,0xae,0x01,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xb8,0x01,0x00,0x00, -0x86,0x00,0x00,0x00,0xae,0x00,0x00,0x00,0xb4,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xc0,0x01,0x00,0x00, -0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xef,0x01,0x00,0x00, -0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x22,0x02,0x00,0x00, -0x0d,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00, -0x2a,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, -0x70,0x02,0x00,0x00,0xb9,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, -0x71,0x02,0x00,0x00,0x70,0x02,0x00,0x00,0x20,0x00,0x04,0x00, -0x72,0x02,0x00,0x00,0x0c,0x00,0x00,0x00,0x71,0x02,0x00,0x00, -0x3b,0x00,0x04,0x00,0x72,0x02,0x00,0x00,0x73,0x02,0x00,0x00, -0x0c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, -0x78,0x02,0x00,0x00,0x05,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x85,0x02,0x00,0x00,0x84,0x00,0x00,0x00, -0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x8e,0x02,0x00,0x00,0x0c,0x00,0x00,0x00,0xb9,0x00,0x00,0x00, -0x36,0x00,0x05,0x00,0x02,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x05,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0xbe,0x00,0x00,0x00, -0xbf,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x82,0x01,0x00,0x00,0x83,0x01,0x00,0x00,0x07,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0xaf,0x01,0x00,0x00,0xb0,0x01,0x00,0x00, -0x07,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00, -0x0e,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, -0x0e,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x14,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x17,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x17,0x00,0x00,0x00, -0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x1e,0x00,0x00,0x00, -0x0f,0x00,0x00,0x00,0x17,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x21,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x24,0x00,0x00,0x00,0x18,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0x28,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x27,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x29,0x00,0x00,0x00, -0x28,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x2a,0x00,0x00,0x00,0x1e,0x00,0x00,0x00,0x29,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x2d,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x2f,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x30,0x00,0x00,0x00, -0x24,0x00,0x00,0x00,0x2f,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x30,0x00,0x00,0x00, -0x2a,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0x35,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x34,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x36,0x00,0x00,0x00, -0x35,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x38,0x00,0x00,0x00,0x36,0x00,0x00,0x00,0x37,0x00,0x00,0x00, -0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, -0x38,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x3b,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, -0x37,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00, -0x3f,0x00,0x00,0x00,0x3d,0x00,0x00,0x00,0x3e,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x3f,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x42,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x3b,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x47,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0x3b,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x0d,0x00,0x00,0x00,0x49,0x00,0x00,0x00,0x3d,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x4a,0x00,0x00,0x00,0x49,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x0d,0x00,0x00,0x00,0x4d,0x00,0x00,0x00,0x4c,0x00,0x00,0x00, -0x3e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x4e,0x00,0x00,0x00,0x4d,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x50,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, -0x4f,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x55,0x00,0x00,0x00,0x50,0x00,0x00,0x00,0x54,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x59,0x00,0x00,0x00, -0x50,0x00,0x00,0x00,0x58,0x00,0x00,0x00,0x89,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x5d,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, -0x4f,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x64,0x00,0x00,0x00,0x5d,0x00,0x00,0x00,0x63,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x68,0x00,0x00,0x00, -0x5d,0x00,0x00,0x00,0x67,0x00,0x00,0x00,0x89,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x6e,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, -0x6d,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x73,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x72,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x77,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x76,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x78,0x00,0x00,0x00,0x77,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x79,0x00,0x00,0x00, -0x47,0x00,0x00,0x00,0x78,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x7c,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x7b,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x7d,0x00,0x00,0x00,0x7c,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x7f,0x00,0x00,0x00,0x47,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x82,0x00,0x00,0x00,0x7f,0x00,0x00,0x00,0x78,0x00,0x00,0x00, -0x0c,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x83,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x26,0x00,0x00,0x00,0x7d,0x00,0x00,0x00, -0x82,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0x87,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x86,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x88,0x00,0x00,0x00, -0x87,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x89,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x88,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8b,0x00,0x00,0x00, -0x42,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x8d,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x8c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x8e,0x00,0x00,0x00,0x8d,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x8f,0x00,0x00,0x00,0x8b,0x00,0x00,0x00, -0x8e,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x90,0x00,0x00,0x00,0x89,0x00,0x00,0x00,0x8f,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x92,0x00,0x00,0x00, -0x90,0x00,0x00,0x00,0x79,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x93,0x00,0x00,0x00,0x92,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0x98,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x97,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x99,0x00,0x00,0x00, -0x98,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x9a,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x99,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9d,0x00,0x00,0x00, -0x4a,0x00,0x00,0x00,0x9c,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x9f,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x9e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xa0,0x00,0x00,0x00,0x9f,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xa1,0x00,0x00,0x00,0x9d,0x00,0x00,0x00, -0xa0,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xa2,0x00,0x00,0x00,0x9a,0x00,0x00,0x00,0xa1,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa4,0x00,0x00,0x00, -0xa2,0x00,0x00,0x00,0x79,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xa5,0x00,0x00,0x00,0xa4,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xa7,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xa7,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0x98,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, -0x05,0x00,0x00,0x00,0xc6,0x00,0x00,0x00,0xa8,0x00,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00,0xb8,0x00,0x00,0x00, -0x98,0x02,0x00,0x00,0xb6,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0xa9,0x00,0x00,0x00,0xa8,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xb8,0x00,0x00,0x00,0xa8,0x00,0x00,0x00, -0xa9,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xa8,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0xc3,0x00,0x00,0x00, -0xbf,0x00,0x00,0x00,0x98,0x02,0x00,0x00,0x3e,0x00,0x03,0x00, -0xc3,0x00,0x00,0x00,0xc1,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xc6,0x00,0x00,0x00,0x98,0x02,0x00,0x00, -0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xa7,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xa9,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xc9,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xc9,0x00,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xb1,0x02,0x00,0x00, -0xa5,0x00,0x00,0x00,0xa9,0x00,0x00,0x00,0x67,0x01,0x00,0x00, -0xcc,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xad,0x02,0x00,0x00,0x93,0x00,0x00,0x00,0xa9,0x00,0x00,0x00, -0x64,0x01,0x00,0x00,0xcc,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0x99,0x02,0x00,0x00,0x79,0x00,0x00,0x00, -0xa9,0x00,0x00,0x00,0x12,0x02,0x00,0x00,0xcc,0x00,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00,0xd0,0x00,0x00,0x00, -0x99,0x02,0x00,0x00,0x83,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0xcb,0x00,0x00,0x00,0xcc,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xd0,0x00,0x00,0x00,0xca,0x00,0x00,0x00, -0xcb,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xca,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xd2,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xd2,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xa9,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0xca,0x00,0x00,0x00, -0x1c,0x01,0x00,0x00,0xd5,0x00,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb7,0x00,0x00,0x00,0xd8,0x00,0x00,0x00,0xa9,0x02,0x00,0x00, -0x37,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xd4,0x00,0x00,0x00, -0xd5,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0xd8,0x00,0x00,0x00,0xd3,0x00,0x00,0x00,0xd4,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xd3,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xdc,0x00,0x00,0x00,0x8b,0x00,0x00,0x00, -0x73,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xde,0x00,0x00,0x00,0xdc,0x00,0x00,0x00,0xa9,0x02,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00,0xe1,0x00,0x00,0x00, -0xde,0x00,0x00,0x00,0x36,0x00,0x00,0x00,0xf7,0x00,0x03,0x00, -0xe3,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0xe1,0x00,0x00,0x00,0xe2,0x00,0x00,0x00,0xe3,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xe2,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xe6,0x00,0x00,0x00,0x99,0x02,0x00,0x00, -0x6e,0x00,0x00,0x00,0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00, -0xe8,0x00,0x00,0x00,0xe6,0x00,0x00,0x00,0x83,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xe3,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xe3,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0xb7,0x00,0x00,0x00, -0xe9,0x00,0x00,0x00,0xe1,0x00,0x00,0x00,0xd3,0x00,0x00,0x00, -0xe8,0x00,0x00,0x00,0xe2,0x00,0x00,0x00,0xf7,0x00,0x03,0x00, -0xeb,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0xe9,0x00,0x00,0x00,0xea,0x00,0x00,0x00,0x0d,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xea,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xf3,0x00,0x00,0x00,0x73,0x00,0x00,0x00, -0xa9,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xf5,0x00,0x00,0x00,0xf3,0x00,0x00,0x00,0xf4,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xf7,0x00,0x00,0x00, -0xf5,0x00,0x00,0x00,0x6e,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x03,0x01,0x00,0x00,0xf3,0x00,0x00,0x00, -0x8e,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x04,0x01,0x00,0x00,0xad,0x02,0x00,0x00,0x03,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x06,0x01,0x00,0x00, -0x04,0x01,0x00,0x00,0x6e,0x00,0x00,0x00,0x41,0x00,0x06,0x00, -0x07,0x01,0x00,0x00,0x08,0x01,0x00,0x00,0xfc,0x00,0x00,0x00, -0x34,0x00,0x00,0x00,0x06,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, -0xf8,0x00,0x00,0x00,0x09,0x01,0x00,0x00,0x08,0x01,0x00,0x00, -0x73,0x00,0x04,0x00,0xb9,0x00,0x00,0x00,0x0a,0x01,0x00,0x00, -0x09,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0x0b,0x01,0x00,0x00, -0x0c,0x01,0x00,0x00,0xf0,0x00,0x00,0x00,0xf7,0x00,0x00,0x00, -0x3e,0x00,0x03,0x00,0x0c,0x01,0x00,0x00,0x0a,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0xeb,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x0d,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x10,0x01,0x00,0x00,0x73,0x00,0x00,0x00,0xa9,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x12,0x01,0x00,0x00, -0x10,0x01,0x00,0x00,0x11,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x14,0x01,0x00,0x00,0x12,0x01,0x00,0x00, -0x6e,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x0b,0x01,0x00,0x00, -0x15,0x01,0x00,0x00,0xf0,0x00,0x00,0x00,0x14,0x01,0x00,0x00, -0x3e,0x00,0x03,0x00,0x15,0x01,0x00,0x00,0xc1,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xeb,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xeb,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xd5,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xd5,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x1c,0x01,0x00,0x00,0xa9,0x02,0x00,0x00, -0x1a,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xd2,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xd4,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x1e,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x1e,0x01,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xaa,0x02,0x00,0x00, -0x3e,0x00,0x00,0x00,0xd4,0x00,0x00,0x00,0x60,0x01,0x00,0x00, -0x21,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00, -0x24,0x01,0x00,0x00,0xaa,0x02,0x00,0x00,0x9c,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0x20,0x01,0x00,0x00,0x21,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x24,0x01,0x00,0x00, -0x1f,0x01,0x00,0x00,0x20,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x1f,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x28,0x01,0x00,0x00,0x9d,0x00,0x00,0x00,0x73,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x2a,0x01,0x00,0x00, -0x28,0x01,0x00,0x00,0xaa,0x02,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x2b,0x01,0x00,0x00,0x12,0x00,0x00,0x00, -0xc5,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x2c,0x01,0x00,0x00,0x2b,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb7,0x00,0x00,0x00,0x2d,0x01,0x00,0x00,0x2a,0x01,0x00,0x00, -0x2c,0x01,0x00,0x00,0xf7,0x00,0x03,0x00,0x2f,0x01,0x00,0x00, -0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x2d,0x01,0x00,0x00, -0x2e,0x01,0x00,0x00,0x2f,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x2e,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x32,0x01,0x00,0x00,0x99,0x02,0x00,0x00,0x6e,0x00,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00,0x34,0x01,0x00,0x00, -0x32,0x01,0x00,0x00,0x83,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x2f,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x2f,0x01,0x00,0x00, -0xf5,0x00,0x07,0x00,0xb7,0x00,0x00,0x00,0x35,0x01,0x00,0x00, -0x2d,0x01,0x00,0x00,0x1f,0x01,0x00,0x00,0x34,0x01,0x00,0x00, -0x2e,0x01,0x00,0x00,0xf7,0x00,0x03,0x00,0x37,0x01,0x00,0x00, -0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x35,0x01,0x00,0x00, -0x36,0x01,0x00,0x00,0x56,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x36,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x3f,0x01,0x00,0x00,0x73,0x00,0x00,0x00,0xaa,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x41,0x01,0x00,0x00, -0x3f,0x01,0x00,0x00,0x40,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x43,0x01,0x00,0x00,0x41,0x01,0x00,0x00, -0x6e,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x4e,0x01,0x00,0x00,0x3f,0x01,0x00,0x00,0xa0,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x4f,0x01,0x00,0x00, -0xb1,0x02,0x00,0x00,0x4e,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x51,0x01,0x00,0x00,0x4f,0x01,0x00,0x00, -0x6e,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x07,0x01,0x00,0x00, -0x52,0x01,0x00,0x00,0x47,0x01,0x00,0x00,0x34,0x00,0x00,0x00, -0x51,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0xf8,0x00,0x00,0x00, -0x53,0x01,0x00,0x00,0x52,0x01,0x00,0x00,0x73,0x00,0x04,0x00, -0xb9,0x00,0x00,0x00,0x54,0x01,0x00,0x00,0x53,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0x0b,0x01,0x00,0x00,0x55,0x01,0x00,0x00, -0x3c,0x01,0x00,0x00,0x43,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0x55,0x01,0x00,0x00,0x54,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0x37,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x56,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x59,0x01,0x00,0x00, -0x73,0x00,0x00,0x00,0xaa,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x5b,0x01,0x00,0x00,0x59,0x01,0x00,0x00, -0x5a,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x5d,0x01,0x00,0x00,0x5b,0x01,0x00,0x00,0x6e,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x0b,0x01,0x00,0x00,0x5e,0x01,0x00,0x00, -0x3c,0x01,0x00,0x00,0x5d,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0x5e,0x01,0x00,0x00,0xc1,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x37,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x37,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0x21,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x21,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x60,0x01,0x00,0x00,0xaa,0x02,0x00,0x00,0x1a,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0x1e,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x20,0x01,0x00,0x00,0xe0,0x00,0x04,0x00,0x0c,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x61,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x64,0x01,0x00,0x00,0xad,0x02,0x00,0x00, -0x62,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x67,0x01,0x00,0x00,0xb1,0x02,0x00,0x00,0x65,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0x69,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x69,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xb3,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0x20,0x01,0x00,0x00, -0x10,0x02,0x00,0x00,0x6c,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb7,0x00,0x00,0x00,0x6f,0x01,0x00,0x00,0xb3,0x02,0x00,0x00, -0x6c,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x6b,0x01,0x00,0x00, -0x6c,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x6f,0x01,0x00,0x00,0x6a,0x01,0x00,0x00,0x6b,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x6a,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0x71,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x71,0x01,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xb7,0x02,0x00,0x00, -0x3e,0x00,0x00,0x00,0x6a,0x01,0x00,0x00,0x9c,0x01,0x00,0x00, -0x74,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00, -0x77,0x01,0x00,0x00,0xb7,0x02,0x00,0x00,0x60,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0x73,0x01,0x00,0x00,0x74,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x77,0x01,0x00,0x00, -0x72,0x01,0x00,0x00,0x73,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x72,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x79,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x79,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xc9,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, -0x72,0x01,0x00,0x00,0x9a,0x01,0x00,0x00,0x7a,0x01,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00,0x7f,0x01,0x00,0x00, -0xc9,0x02,0x00,0x00,0x62,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0x7b,0x01,0x00,0x00,0x7a,0x01,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x7f,0x01,0x00,0x00,0x7a,0x01,0x00,0x00, -0x7b,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x7a,0x01,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x85,0x01,0x00,0x00, -0xb7,0x02,0x00,0x00,0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x87,0x01,0x00,0x00,0x85,0x01,0x00,0x00, -0xc9,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x89,0x01,0x00,0x00,0x55,0x00,0x00,0x00,0x53,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8b,0x01,0x00,0x00, -0xb7,0x02,0x00,0x00,0x61,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x8c,0x01,0x00,0x00,0x89,0x01,0x00,0x00, -0x8b,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x8e,0x01,0x00,0x00,0x64,0x00,0x00,0x00,0x62,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8f,0x01,0x00,0x00, -0x8c,0x01,0x00,0x00,0x8e,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x91,0x01,0x00,0x00,0x8f,0x01,0x00,0x00, -0xc9,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x93,0x01,0x00,0x00,0x91,0x01,0x00,0x00,0x92,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x95,0x01,0x00,0x00, -0x93,0x01,0x00,0x00,0xb3,0x02,0x00,0x00,0x41,0x00,0x05,0x00, -0x0b,0x01,0x00,0x00,0x96,0x01,0x00,0x00,0xf0,0x00,0x00,0x00, -0x95,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0xb9,0x00,0x00,0x00, -0x97,0x01,0x00,0x00,0x96,0x01,0x00,0x00,0x41,0x00,0x05,0x00, -0xc2,0x00,0x00,0x00,0x98,0x01,0x00,0x00,0x83,0x01,0x00,0x00, -0x87,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x98,0x01,0x00,0x00, -0x97,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x9a,0x01,0x00,0x00,0xc9,0x02,0x00,0x00,0xc5,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x79,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x7b,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x74,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x74,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x9c,0x01,0x00,0x00,0xb7,0x02,0x00,0x00, -0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x71,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x73,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0x9e,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x9e,0x01,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xb8,0x02,0x00,0x00, -0x3e,0x00,0x00,0x00,0x73,0x01,0x00,0x00,0xca,0x01,0x00,0x00, -0xa1,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00, -0xa4,0x01,0x00,0x00,0xb8,0x02,0x00,0x00,0xb4,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0xa0,0x01,0x00,0x00,0xa1,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xa4,0x01,0x00,0x00, -0x9f,0x01,0x00,0x00,0xa0,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x9f,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xa6,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xa6,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xc6,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, -0x9f,0x01,0x00,0x00,0xc8,0x01,0x00,0x00,0xa7,0x01,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00,0xac,0x01,0x00,0x00, -0xc6,0x02,0x00,0x00,0xb1,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0xa8,0x01,0x00,0x00,0xa7,0x01,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xac,0x01,0x00,0x00,0xa7,0x01,0x00,0x00, -0xa8,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xa7,0x01,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xb2,0x01,0x00,0x00, -0xb8,0x02,0x00,0x00,0xb1,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xb4,0x01,0x00,0x00,0xb2,0x01,0x00,0x00, -0xc6,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xb6,0x01,0x00,0x00,0x59,0x00,0x00,0x00,0xae,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xb9,0x01,0x00,0x00, -0xb8,0x02,0x00,0x00,0xb8,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xba,0x01,0x00,0x00,0xb6,0x01,0x00,0x00, -0xb9,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xbc,0x01,0x00,0x00,0x68,0x00,0x00,0x00,0xb1,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xbd,0x01,0x00,0x00, -0xba,0x01,0x00,0x00,0xbc,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xbf,0x01,0x00,0x00,0xbd,0x01,0x00,0x00, -0xc6,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xc1,0x01,0x00,0x00,0xbf,0x01,0x00,0x00,0xc0,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xc3,0x01,0x00,0x00, -0xc1,0x01,0x00,0x00,0xb3,0x02,0x00,0x00,0x41,0x00,0x05,0x00, -0x0b,0x01,0x00,0x00,0xc4,0x01,0x00,0x00,0x3c,0x01,0x00,0x00, -0xc3,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0xb9,0x00,0x00,0x00, -0xc5,0x01,0x00,0x00,0xc4,0x01,0x00,0x00,0x41,0x00,0x05,0x00, -0xc2,0x00,0x00,0x00,0xc6,0x01,0x00,0x00,0xb0,0x01,0x00,0x00, -0xb4,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0xc6,0x01,0x00,0x00, -0xc5,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xc8,0x01,0x00,0x00,0xc6,0x02,0x00,0x00,0xc5,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xa6,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xa8,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xa1,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xa1,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xca,0x01,0x00,0x00,0xb8,0x02,0x00,0x00, -0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x9e,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xa0,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0xcc,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xcc,0x01,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xb9,0x02,0x00,0x00, -0x3e,0x00,0x00,0x00,0xa0,0x01,0x00,0x00,0x0e,0x02,0x00,0x00, -0xcf,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00, -0xd2,0x01,0x00,0x00,0xb9,0x02,0x00,0x00,0xb4,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0xce,0x01,0x00,0x00,0xcf,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xd2,0x01,0x00,0x00, -0xcd,0x01,0x00,0x00,0xce,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xcd,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xd4,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xd4,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xbd,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, -0xcd,0x01,0x00,0x00,0x0c,0x02,0x00,0x00,0xd7,0x01,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00,0xda,0x01,0x00,0x00, -0xbd,0x02,0x00,0x00,0x60,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0xd6,0x01,0x00,0x00,0xd7,0x01,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xda,0x01,0x00,0x00,0xd5,0x01,0x00,0x00, -0xd6,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xd5,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0xdc,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xdc,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xbf,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0xd5,0x01,0x00,0x00, -0x0a,0x02,0x00,0x00,0xdf,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb7,0x00,0x00,0x00,0xe2,0x01,0x00,0x00,0xbf,0x02,0x00,0x00, -0xb1,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xde,0x01,0x00,0x00, -0xdf,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0xe2,0x01,0x00,0x00,0xdd,0x01,0x00,0x00,0xde,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xdd,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0xe4,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xe4,0x01,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xc1,0x02,0x00,0x00, -0x3e,0x00,0x00,0x00,0xdd,0x01,0x00,0x00,0x08,0x02,0x00,0x00, -0xe5,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00, -0xea,0x01,0x00,0x00,0xc1,0x02,0x00,0x00,0x62,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0xe6,0x01,0x00,0x00,0xe5,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xea,0x01,0x00,0x00, -0xe5,0x01,0x00,0x00,0xe6,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xe5,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xec,0x01,0x00,0x00,0xb9,0x02,0x00,0x00,0xb1,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xee,0x01,0x00,0x00, -0xec,0x01,0x00,0x00,0xbf,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xf0,0x01,0x00,0x00,0xee,0x01,0x00,0x00, -0xef,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xf2,0x01,0x00,0x00,0xbd,0x02,0x00,0x00,0x62,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xf3,0x01,0x00,0x00, -0xf0,0x01,0x00,0x00,0xf2,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xf5,0x01,0x00,0x00,0xf3,0x01,0x00,0x00, -0xc1,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xf9,0x01,0x00,0x00,0xf2,0x01,0x00,0x00,0xc1,0x02,0x00,0x00, -0x41,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0xfa,0x01,0x00,0x00, -0x83,0x01,0x00,0x00,0xf9,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, -0xb9,0x00,0x00,0x00,0xfb,0x01,0x00,0x00,0xfa,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0x00,0x02,0x00,0x00, -0xb0,0x01,0x00,0x00,0xee,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, -0xb9,0x00,0x00,0x00,0x01,0x02,0x00,0x00,0x00,0x02,0x00,0x00, -0x41,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0x03,0x02,0x00,0x00, -0xbf,0x00,0x00,0x00,0xf5,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, -0xb9,0x00,0x00,0x00,0x04,0x02,0x00,0x00,0x03,0x02,0x00,0x00, -0x0c,0x00,0x08,0x00,0xb9,0x00,0x00,0x00,0x05,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0xfb,0x01,0x00,0x00, -0x01,0x02,0x00,0x00,0x04,0x02,0x00,0x00,0x3e,0x00,0x03,0x00, -0x03,0x02,0x00,0x00,0x05,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x08,0x02,0x00,0x00,0xc1,0x02,0x00,0x00, -0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xe4,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xe6,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0xdf,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xdf,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x0a,0x02,0x00,0x00, -0xbf,0x02,0x00,0x00,0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xdc,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xde,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0xd7,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xd7,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x0c,0x02,0x00,0x00,0xbd,0x02,0x00,0x00,0xc5,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xd4,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xd6,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xcf,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xcf,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x0e,0x02,0x00,0x00,0xb9,0x02,0x00,0x00, -0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xcc,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xce,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0x6c,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x6c,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x10,0x02,0x00,0x00, -0xb3,0x02,0x00,0x00,0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x69,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x6b,0x01,0x00,0x00, -0xe0,0x00,0x04,0x00,0x0c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x61,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xcc,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xcc,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x12,0x02,0x00,0x00,0x99,0x02,0x00,0x00, -0x6c,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xc9,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xcb,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x17,0x02,0x00,0x00,0x55,0x00,0x00,0x00, -0x53,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x18,0x02,0x00,0x00,0x8b,0x00,0x00,0x00,0x17,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x1d,0x02,0x00,0x00, -0x59,0x00,0x00,0x00,0xae,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x1e,0x02,0x00,0x00,0x9d,0x00,0x00,0x00, -0x1d,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0x23,0x02,0x00,0x00,0x12,0x00,0x00,0x00,0x22,0x02,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x24,0x02,0x00,0x00, -0x23,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x25,0x02,0x00,0x00,0x0f,0x00,0x00,0x00,0x24,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x29,0x02,0x00,0x00, -0x47,0x00,0x00,0x00,0x24,0x02,0x00,0x00,0x41,0x00,0x05,0x00, -0x0d,0x00,0x00,0x00,0x2b,0x02,0x00,0x00,0x2a,0x02,0x00,0x00, -0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x2c,0x02,0x00,0x00,0x2b,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x2d,0x02,0x00,0x00,0x29,0x02,0x00,0x00, -0x2c,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x2e,0x02,0x00,0x00,0x25,0x02,0x00,0x00,0x2d,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0x30,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x30,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0x9a,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0xcb,0x00,0x00,0x00, -0x97,0x02,0x00,0x00,0x33,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb7,0x00,0x00,0x00,0x36,0x02,0x00,0x00,0x9a,0x02,0x00,0x00, -0xb4,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x32,0x02,0x00,0x00, -0x33,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x36,0x02,0x00,0x00,0x31,0x02,0x00,0x00,0x32,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x31,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x38,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x38,0x02,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x9b,0x02,0x00,0x00, -0x3e,0x00,0x00,0x00,0x31,0x02,0x00,0x00,0x95,0x02,0x00,0x00, -0x3b,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00, -0x3e,0x02,0x00,0x00,0x9b,0x02,0x00,0x00,0x60,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0x3a,0x02,0x00,0x00,0x3b,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x3e,0x02,0x00,0x00, -0x39,0x02,0x00,0x00,0x3a,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x39,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x42,0x02,0x00,0x00,0x9b,0x02,0x00,0x00,0x61,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x43,0x02,0x00,0x00, -0x18,0x02,0x00,0x00,0x42,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x45,0x02,0x00,0x00,0x64,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x46,0x02,0x00,0x00,0x43,0x02,0x00,0x00,0x45,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x4a,0x02,0x00,0x00, -0x9a,0x02,0x00,0x00,0xb8,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x4b,0x02,0x00,0x00,0x1e,0x02,0x00,0x00, -0x4a,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x4d,0x02,0x00,0x00,0x68,0x00,0x00,0x00,0xb1,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x4e,0x02,0x00,0x00, -0x4b,0x02,0x00,0x00,0x4d,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x50,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x50,0x02,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x9d,0x02,0x00,0x00, -0x3e,0x00,0x00,0x00,0x39,0x02,0x00,0x00,0x93,0x02,0x00,0x00, -0x53,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00, -0x56,0x02,0x00,0x00,0x9d,0x02,0x00,0x00,0xb1,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0x52,0x02,0x00,0x00,0x53,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x56,0x02,0x00,0x00, -0x51,0x02,0x00,0x00,0x52,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x51,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x58,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x58,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0x9f,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, -0x51,0x02,0x00,0x00,0x91,0x02,0x00,0x00,0x5b,0x02,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00,0x5e,0x02,0x00,0x00, -0x9f,0x02,0x00,0x00,0x62,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0x5a,0x02,0x00,0x00,0x5b,0x02,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x5e,0x02,0x00,0x00,0x59,0x02,0x00,0x00, -0x5a,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x59,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x61,0x02,0x00,0x00, -0x46,0x02,0x00,0x00,0x9f,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb7,0x00,0x00,0x00,0x64,0x02,0x00,0x00,0x61,0x02,0x00,0x00, -0x36,0x00,0x00,0x00,0xf7,0x00,0x03,0x00,0x66,0x02,0x00,0x00, -0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x64,0x02,0x00,0x00, -0x65,0x02,0x00,0x00,0x66,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x65,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x69,0x02,0x00,0x00,0x4e,0x02,0x00,0x00,0x9d,0x02,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x6a,0x02,0x00,0x00, -0x12,0x00,0x00,0x00,0xc5,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x6b,0x02,0x00,0x00,0x6a,0x02,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00,0x6c,0x02,0x00,0x00, -0x69,0x02,0x00,0x00,0x6b,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x66,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x66,0x02,0x00,0x00, -0xf5,0x00,0x07,0x00,0xb7,0x00,0x00,0x00,0x6d,0x02,0x00,0x00, -0x64,0x02,0x00,0x00,0x59,0x02,0x00,0x00,0x6c,0x02,0x00,0x00, -0x65,0x02,0x00,0x00,0xf7,0x00,0x03,0x00,0x6f,0x02,0x00,0x00, -0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x6d,0x02,0x00,0x00, -0x6e,0x02,0x00,0x00,0x6f,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x6e,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x77,0x02,0x00,0x00,0x4e,0x02,0x00,0x00,0x9d,0x02,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x79,0x02,0x00,0x00, -0x12,0x00,0x00,0x00,0x78,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x7a,0x02,0x00,0x00,0x79,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x7b,0x02,0x00,0x00, -0x77,0x02,0x00,0x00,0x7a,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x7c,0x02,0x00,0x00,0x2e,0x02,0x00,0x00, -0x7b,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x7e,0x02,0x00,0x00,0x7c,0x02,0x00,0x00,0x46,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x80,0x02,0x00,0x00, -0x7e,0x02,0x00,0x00,0x9f,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x82,0x02,0x00,0x00,0x9a,0x02,0x00,0x00, -0xb1,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x84,0x02,0x00,0x00,0x82,0x02,0x00,0x00,0x9d,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x86,0x02,0x00,0x00, -0x84,0x02,0x00,0x00,0x85,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x88,0x02,0x00,0x00,0x9b,0x02,0x00,0x00, -0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x89,0x02,0x00,0x00,0x86,0x02,0x00,0x00,0x88,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8b,0x02,0x00,0x00, -0x89,0x02,0x00,0x00,0x9f,0x02,0x00,0x00,0x41,0x00,0x05,0x00, -0xc2,0x00,0x00,0x00,0x8c,0x02,0x00,0x00,0xbf,0x00,0x00,0x00, -0x8b,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0xb9,0x00,0x00,0x00, -0x8d,0x02,0x00,0x00,0x8c,0x02,0x00,0x00,0x41,0x00,0x06,0x00, -0x8e,0x02,0x00,0x00,0x8f,0x02,0x00,0x00,0x73,0x02,0x00,0x00, -0x34,0x00,0x00,0x00,0x80,0x02,0x00,0x00,0x3e,0x00,0x03,0x00, -0x8f,0x02,0x00,0x00,0x8d,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x6f,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x6f,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0x5b,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x5b,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x91,0x02,0x00,0x00,0x9f,0x02,0x00,0x00,0xc5,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x58,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x5a,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x53,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x53,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x93,0x02,0x00,0x00,0x9d,0x02,0x00,0x00, -0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x50,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x52,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x3b,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x3b,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x95,0x02,0x00,0x00, -0x9b,0x02,0x00,0x00,0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x38,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x3a,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0x33,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x33,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x97,0x02,0x00,0x00,0x9a,0x02,0x00,0x00,0xc5,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x30,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x32,0x02,0x00,0x00,0xfd,0x00,0x01,0x00,0x38,0x00,0x01,0x00, - -}; -const uint64_t matmul_f16_l_fp32_len = 10116; - -unsigned char matmul_f16_m_data[] = { -0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, -0xcc,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, -0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x09,0x00,0x00,0x00, -0x11,0x00,0x02,0x00,0x51,0x11,0x00,0x00,0x0b,0x00,0x06,0x00, -0x01,0x00,0x00,0x00,0x47,0x4c,0x53,0x4c,0x2e,0x73,0x74,0x64, -0x2e,0x34,0x35,0x30,0x00,0x00,0x00,0x00,0x0e,0x00,0x03,0x00, -0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x0f,0x00,0x0f,0x00, -0x05,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x6d,0x61,0x69,0x6e, -0x00,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x3d,0x00,0x00,0x00,0x4c,0x00,0x00,0x00,0xf1,0x00,0x00,0x00, -0xfc,0x00,0x00,0x00,0x3c,0x01,0x00,0x00,0x47,0x01,0x00,0x00, -0x2c,0x02,0x00,0x00,0x75,0x02,0x00,0x00,0x10,0x00,0x06,0x00, -0x04,0x00,0x00,0x00,0x11,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x0b,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x1c,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x10,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x08,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x03,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x10,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x10,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x05,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x14,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x10,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x1c,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x08,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x24,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x10,0x00,0x00,0x00,0x0a,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x28,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x0b,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x2c,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x30,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x10,0x00,0x00,0x00,0x0d,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x34,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x10,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x37,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x3d,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x1a,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x4c,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x1b,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x4f,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x53,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x60,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x62,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x6c,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x03,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x9c,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xae,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0xb1,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x08,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0xf9,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0xfa,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0xfa,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0xfa,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xfc,0x00,0x00,0x00, -0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0xfc,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x16,0x01,0x00,0x00,0x01,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x17,0x01,0x00,0x00, -0x0b,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x44,0x01,0x00,0x00,0x06,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x48,0x00,0x04,0x00,0x45,0x01,0x00,0x00,0x00,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x45,0x01,0x00,0x00, -0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x03,0x00,0x45,0x01,0x00,0x00,0x02,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x47,0x01,0x00,0x00,0x22,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x47,0x01,0x00,0x00, -0x21,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x2c,0x02,0x00,0x00,0x0b,0x00,0x00,0x00,0x18,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x72,0x02,0x00,0x00,0x06,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0x73,0x02,0x00,0x00, -0x00,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x73,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x73,0x02,0x00,0x00, -0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x75,0x02,0x00,0x00, -0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x75,0x02,0x00,0x00,0x21,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x13,0x00,0x02,0x00,0x02,0x00,0x00,0x00,0x21,0x00,0x03,0x00, -0x03,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x15,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x17,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x03,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x0a,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x0a,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x0d,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x1e,0x00,0x10,0x00, -0x10,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x11,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x10,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x11,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x15,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x14,0x00,0x00,0x00, -0x08,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x15,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x0a,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x27,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, -0x2d,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x37,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x0a,0x00,0x00,0x00,0x3d,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x3e,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00, -0x4c,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x53,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x54,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x37,0x00,0x00,0x00, -0x53,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x58,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x37,0x00,0x00,0x00, -0x53,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x60,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x61,0x00,0x00,0x00,0x86,0x00,0x00,0x00, -0x53,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x32,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x63,0x00,0x00,0x00, -0x86,0x00,0x00,0x00,0x61,0x00,0x00,0x00,0x62,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x67,0x00,0x00,0x00, -0x86,0x00,0x00,0x00,0x61,0x00,0x00,0x00,0x62,0x00,0x00,0x00, -0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x10,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x6d,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x72,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, -0x76,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0x7b,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x86,0x00,0x00,0x00, -0x0b,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, -0x8c,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0x97,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x9c,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, -0x9e,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xad,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x32,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xae,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xaf,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0xae,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xb0,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x62,0x00,0x00,0x00, -0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xb1,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xb2,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0xb0,0x00,0x00,0x00, -0xb1,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xb3,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0xb2,0x00,0x00,0x00, -0x60,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xb4,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0xaf,0x00,0x00,0x00, -0xb3,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xb5,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0xad,0x00,0x00,0x00, -0xb4,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xb6,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0xb5,0x00,0x00,0x00, -0xb1,0x00,0x00,0x00,0x14,0x00,0x02,0x00,0xb7,0x00,0x00,0x00, -0x16,0x00,0x03,0x00,0xb9,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xba,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xbb,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0xba,0x00,0x00,0x00,0xb4,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xbc,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0xbb,0x00,0x00,0x00,0xb1,0x00,0x00,0x00, -0x1c,0x00,0x04,0x00,0xbd,0x00,0x00,0x00,0xb9,0x00,0x00,0x00, -0xbc,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0xbe,0x00,0x00,0x00, -0x07,0x00,0x00,0x00,0xbd,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0xb9,0x00,0x00,0x00,0xc1,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0xc2,0x00,0x00,0x00,0x07,0x00,0x00,0x00, -0xb9,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, -0xc5,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x16,0x00,0x03,0x00, -0xec,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xed,0x00,0x00,0x00,0x80,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xee,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0x37,0x00,0x00,0x00,0xed,0x00,0x00,0x00,0x1c,0x00,0x04,0x00, -0xef,0x00,0x00,0x00,0xec,0x00,0x00,0x00,0xee,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0xf0,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0xef,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0xf0,0x00,0x00,0x00, -0xf1,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xf5,0x00,0x00,0x00,0x80,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, -0xf9,0x00,0x00,0x00,0xec,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, -0xfa,0x00,0x00,0x00,0xf9,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0xfb,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0xfa,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0xfb,0x00,0x00,0x00,0xfc,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x07,0x01,0x00,0x00, -0x0c,0x00,0x00,0x00,0xec,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x0a,0x01,0x00,0x00,0x04,0x00,0x00,0x00,0xec,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x10,0x01,0x00,0x00, -0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0xec,0x00,0x00,0x00,0x14,0x01,0x00,0x00, -0x00,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x16,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x33,0x00,0x06,0x00, -0x09,0x00,0x00,0x00,0x17,0x01,0x00,0x00,0x16,0x01,0x00,0x00, -0x39,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x18,0x01,0x00,0x00,0x51,0x00,0x00,0x00, -0x17,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x19,0x01,0x00,0x00,0x84,0x00,0x00,0x00, -0x18,0x01,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x1a,0x01,0x00,0x00,0x86,0x00,0x00,0x00, -0x19,0x01,0x00,0x00,0x6c,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x38,0x01,0x00,0x00,0x80,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x39,0x01,0x00,0x00,0x84,0x00,0x00,0x00, -0x9c,0x00,0x00,0x00,0x38,0x01,0x00,0x00,0x1c,0x00,0x04,0x00, -0x3a,0x01,0x00,0x00,0xec,0x00,0x00,0x00,0x39,0x01,0x00,0x00, -0x20,0x00,0x04,0x00,0x3b,0x01,0x00,0x00,0x04,0x00,0x00,0x00, -0x3a,0x01,0x00,0x00,0x3b,0x00,0x04,0x00,0x3b,0x01,0x00,0x00, -0x3c,0x01,0x00,0x00,0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x40,0x01,0x00,0x00,0x80,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, -0x44,0x01,0x00,0x00,0xec,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, -0x45,0x01,0x00,0x00,0x44,0x01,0x00,0x00,0x20,0x00,0x04,0x00, -0x46,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x45,0x01,0x00,0x00, -0x3b,0x00,0x04,0x00,0x46,0x01,0x00,0x00,0x47,0x01,0x00,0x00, -0x0c,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x59,0x01,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x60,0x01,0x00,0x00,0x08,0x01,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x61,0x01,0x00,0x00,0x86,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x64,0x01,0x00,0x00,0x86,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x7f,0x01,0x00,0x00,0x84,0x00,0x00,0x00, -0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x1c,0x00,0x04,0x00, -0x80,0x01,0x00,0x00,0xec,0x00,0x00,0x00,0x7f,0x01,0x00,0x00, -0x20,0x00,0x04,0x00,0x81,0x01,0x00,0x00,0x07,0x00,0x00,0x00, -0x80,0x01,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x91,0x01,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x97,0x01,0x00,0x00, -0x07,0x00,0x00,0x00,0xec,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xad,0x01,0x00,0x00,0x84,0x00,0x00,0x00, -0xb4,0x00,0x00,0x00,0xb1,0x00,0x00,0x00,0x1c,0x00,0x04,0x00, -0xae,0x01,0x00,0x00,0xec,0x00,0x00,0x00,0xad,0x01,0x00,0x00, -0x20,0x00,0x04,0x00,0xaf,0x01,0x00,0x00,0x07,0x00,0x00,0x00, -0xae,0x01,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xb8,0x01,0x00,0x00,0x86,0x00,0x00,0x00,0xae,0x00,0x00,0x00, -0xb4,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xc0,0x01,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xef,0x01,0x00,0x00,0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, -0x24,0x02,0x00,0x00,0x0d,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x0a,0x00,0x00,0x00,0x2c,0x02,0x00,0x00,0x01,0x00,0x00,0x00, -0x1d,0x00,0x03,0x00,0x72,0x02,0x00,0x00,0xb9,0x00,0x00,0x00, -0x1e,0x00,0x03,0x00,0x73,0x02,0x00,0x00,0x72,0x02,0x00,0x00, -0x20,0x00,0x04,0x00,0x74,0x02,0x00,0x00,0x0c,0x00,0x00,0x00, -0x73,0x02,0x00,0x00,0x3b,0x00,0x04,0x00,0x74,0x02,0x00,0x00, -0x75,0x02,0x00,0x00,0x0c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0x7a,0x02,0x00,0x00,0x05,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x87,0x02,0x00,0x00, -0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x90,0x02,0x00,0x00,0x0c,0x00,0x00,0x00, -0xb9,0x00,0x00,0x00,0x36,0x00,0x05,0x00,0x02,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x03,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0x05,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0xbe,0x00,0x00,0x00,0xbf,0x00,0x00,0x00,0x07,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x81,0x01,0x00,0x00,0x82,0x01,0x00,0x00, -0x07,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0xaf,0x01,0x00,0x00, -0xb0,0x01,0x00,0x00,0x07,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x0d,0x00,0x00,0x00,0x0e,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x0f,0x00,0x00,0x00,0x0e,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x14,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x17,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, -0x17,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x1e,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x17,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x22,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x22,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x24,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x27,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x29,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x2a,0x00,0x00,0x00,0x1e,0x00,0x00,0x00, -0x29,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x2d,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x2f,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x30,0x00,0x00,0x00,0x24,0x00,0x00,0x00,0x2f,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x32,0x00,0x00,0x00, -0x30,0x00,0x00,0x00,0x2a,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x35,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x34,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x36,0x00,0x00,0x00,0x35,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x38,0x00,0x00,0x00,0x36,0x00,0x00,0x00, -0x37,0x00,0x00,0x00,0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x3a,0x00,0x00,0x00,0x38,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3b,0x00,0x00,0x00, -0x3a,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x0d,0x00,0x00,0x00,0x3f,0x00,0x00,0x00,0x3d,0x00,0x00,0x00, -0x3e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0x3f,0x00,0x00,0x00,0x89,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x42,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x3b,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x47,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x3b,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00,0x49,0x00,0x00,0x00, -0x3d,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x4a,0x00,0x00,0x00,0x49,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00,0x4d,0x00,0x00,0x00, -0x4c,0x00,0x00,0x00,0x3e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x4d,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x50,0x00,0x00,0x00, -0x4e,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x89,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x55,0x00,0x00,0x00,0x50,0x00,0x00,0x00, -0x54,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x59,0x00,0x00,0x00,0x50,0x00,0x00,0x00,0x58,0x00,0x00,0x00, -0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x5d,0x00,0x00,0x00, -0x4e,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x89,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x64,0x00,0x00,0x00,0x5d,0x00,0x00,0x00, -0x63,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x68,0x00,0x00,0x00,0x5d,0x00,0x00,0x00,0x67,0x00,0x00,0x00, -0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x6e,0x00,0x00,0x00, -0x4e,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x73,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, -0x72,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0x77,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x76,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x78,0x00,0x00,0x00, -0x77,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x79,0x00,0x00,0x00,0x47,0x00,0x00,0x00,0x78,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x7c,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x7b,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x7d,0x00,0x00,0x00,0x7c,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x7f,0x00,0x00,0x00, -0x47,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x82,0x00,0x00,0x00,0x7f,0x00,0x00,0x00, -0x78,0x00,0x00,0x00,0x0c,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0x83,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x26,0x00,0x00,0x00, -0x7d,0x00,0x00,0x00,0x82,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x87,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x86,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x88,0x00,0x00,0x00,0x87,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x89,0x00,0x00,0x00,0x32,0x00,0x00,0x00, -0x88,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x8b,0x00,0x00,0x00,0x42,0x00,0x00,0x00,0x37,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x8d,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x8c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x8e,0x00,0x00,0x00,0x8d,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8f,0x00,0x00,0x00, -0x8b,0x00,0x00,0x00,0x8e,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x90,0x00,0x00,0x00,0x89,0x00,0x00,0x00, -0x8f,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x92,0x00,0x00,0x00,0x90,0x00,0x00,0x00,0x79,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x93,0x00,0x00,0x00, -0x92,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x98,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x97,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x99,0x00,0x00,0x00,0x98,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x9a,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, -0x99,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x9d,0x00,0x00,0x00,0x4a,0x00,0x00,0x00,0x9c,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x9f,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x9e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xa0,0x00,0x00,0x00,0x9f,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa1,0x00,0x00,0x00, -0x9d,0x00,0x00,0x00,0xa0,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xa2,0x00,0x00,0x00,0x9a,0x00,0x00,0x00, -0xa1,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xa4,0x00,0x00,0x00,0xa2,0x00,0x00,0x00,0x79,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa5,0x00,0x00,0x00, -0xa4,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xa7,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xa7,0x00,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x9a,0x02,0x00,0x00, -0x3e,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0xc6,0x00,0x00,0x00, -0xa8,0x00,0x00,0x00,0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00, -0xb8,0x00,0x00,0x00,0x9a,0x02,0x00,0x00,0xb6,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0xa9,0x00,0x00,0x00,0xa8,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xb8,0x00,0x00,0x00, -0xa8,0x00,0x00,0x00,0xa9,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xa8,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, -0xc3,0x00,0x00,0x00,0xbf,0x00,0x00,0x00,0x9a,0x02,0x00,0x00, -0x3e,0x00,0x03,0x00,0xc3,0x00,0x00,0x00,0xc1,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xc6,0x00,0x00,0x00, -0x9a,0x02,0x00,0x00,0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xa7,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xa9,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xc9,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xc9,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xb3,0x02,0x00,0x00,0xa5,0x00,0x00,0x00,0xa9,0x00,0x00,0x00, -0x66,0x01,0x00,0x00,0xcc,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xaf,0x02,0x00,0x00,0x93,0x00,0x00,0x00, -0xa9,0x00,0x00,0x00,0x63,0x01,0x00,0x00,0xcc,0x00,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x9b,0x02,0x00,0x00, -0x79,0x00,0x00,0x00,0xa9,0x00,0x00,0x00,0x14,0x02,0x00,0x00, -0xcc,0x00,0x00,0x00,0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00, -0xd0,0x00,0x00,0x00,0x9b,0x02,0x00,0x00,0x83,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0xcb,0x00,0x00,0x00,0xcc,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xd0,0x00,0x00,0x00, -0xca,0x00,0x00,0x00,0xcb,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xca,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xd2,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xd2,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xab,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, -0xca,0x00,0x00,0x00,0x1c,0x01,0x00,0x00,0xd5,0x00,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00,0xd8,0x00,0x00,0x00, -0xab,0x02,0x00,0x00,0x37,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0xd4,0x00,0x00,0x00,0xd5,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xd8,0x00,0x00,0x00,0xd3,0x00,0x00,0x00, -0xd4,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xd3,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xdc,0x00,0x00,0x00, -0x8b,0x00,0x00,0x00,0x73,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xde,0x00,0x00,0x00,0xdc,0x00,0x00,0x00, -0xab,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00, -0xe1,0x00,0x00,0x00,0xde,0x00,0x00,0x00,0x36,0x00,0x00,0x00, -0xf7,0x00,0x03,0x00,0xe3,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xe1,0x00,0x00,0x00,0xe2,0x00,0x00,0x00, -0xe3,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xe2,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xe6,0x00,0x00,0x00, -0x9b,0x02,0x00,0x00,0x6e,0x00,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb7,0x00,0x00,0x00,0xe8,0x00,0x00,0x00,0xe6,0x00,0x00,0x00, -0x83,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xe3,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xe3,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, -0xb7,0x00,0x00,0x00,0xe9,0x00,0x00,0x00,0xe1,0x00,0x00,0x00, -0xd3,0x00,0x00,0x00,0xe8,0x00,0x00,0x00,0xe2,0x00,0x00,0x00, -0xf7,0x00,0x03,0x00,0xeb,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xe9,0x00,0x00,0x00,0xea,0x00,0x00,0x00, -0x0c,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xea,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xf4,0x00,0x00,0x00, -0x73,0x00,0x00,0x00,0xab,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xf6,0x00,0x00,0x00,0xf4,0x00,0x00,0x00, -0xf5,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xf8,0x00,0x00,0x00,0xf6,0x00,0x00,0x00,0x6e,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x03,0x01,0x00,0x00, -0xf4,0x00,0x00,0x00,0x8e,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x04,0x01,0x00,0x00,0xaf,0x02,0x00,0x00, -0x03,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x06,0x01,0x00,0x00,0x04,0x01,0x00,0x00,0x6e,0x00,0x00,0x00, -0x41,0x00,0x06,0x00,0x07,0x01,0x00,0x00,0x08,0x01,0x00,0x00, -0xfc,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0x06,0x01,0x00,0x00, -0x3d,0x00,0x04,0x00,0xec,0x00,0x00,0x00,0x09,0x01,0x00,0x00, -0x08,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0x0a,0x01,0x00,0x00, -0x0b,0x01,0x00,0x00,0xf1,0x00,0x00,0x00,0xf8,0x00,0x00,0x00, -0x3e,0x00,0x03,0x00,0x0b,0x01,0x00,0x00,0x09,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0xeb,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x0c,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x0f,0x01,0x00,0x00,0x73,0x00,0x00,0x00,0xab,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x11,0x01,0x00,0x00, -0x0f,0x01,0x00,0x00,0x10,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x13,0x01,0x00,0x00,0x11,0x01,0x00,0x00, -0x6e,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x0a,0x01,0x00,0x00, -0x15,0x01,0x00,0x00,0xf1,0x00,0x00,0x00,0x13,0x01,0x00,0x00, -0x3e,0x00,0x03,0x00,0x15,0x01,0x00,0x00,0x14,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0xeb,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xeb,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xd5,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xd5,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x1c,0x01,0x00,0x00,0xab,0x02,0x00,0x00, -0x1a,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xd2,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xd4,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x1e,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x1e,0x01,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xac,0x02,0x00,0x00, -0x3e,0x00,0x00,0x00,0xd4,0x00,0x00,0x00,0x5f,0x01,0x00,0x00, -0x21,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00, -0x24,0x01,0x00,0x00,0xac,0x02,0x00,0x00,0x9c,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0x20,0x01,0x00,0x00,0x21,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x24,0x01,0x00,0x00, -0x1f,0x01,0x00,0x00,0x20,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x1f,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x28,0x01,0x00,0x00,0x9d,0x00,0x00,0x00,0x73,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x2a,0x01,0x00,0x00, -0x28,0x01,0x00,0x00,0xac,0x02,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x2b,0x01,0x00,0x00,0x12,0x00,0x00,0x00, -0xc5,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x2c,0x01,0x00,0x00,0x2b,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb7,0x00,0x00,0x00,0x2d,0x01,0x00,0x00,0x2a,0x01,0x00,0x00, -0x2c,0x01,0x00,0x00,0xf7,0x00,0x03,0x00,0x2f,0x01,0x00,0x00, -0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x2d,0x01,0x00,0x00, -0x2e,0x01,0x00,0x00,0x2f,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x2e,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x32,0x01,0x00,0x00,0x9b,0x02,0x00,0x00,0x6e,0x00,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00,0x34,0x01,0x00,0x00, -0x32,0x01,0x00,0x00,0x83,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x2f,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x2f,0x01,0x00,0x00, -0xf5,0x00,0x07,0x00,0xb7,0x00,0x00,0x00,0x35,0x01,0x00,0x00, -0x2d,0x01,0x00,0x00,0x1f,0x01,0x00,0x00,0x34,0x01,0x00,0x00, -0x2e,0x01,0x00,0x00,0xf7,0x00,0x03,0x00,0x37,0x01,0x00,0x00, -0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x35,0x01,0x00,0x00, -0x36,0x01,0x00,0x00,0x55,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x36,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x3f,0x01,0x00,0x00,0x73,0x00,0x00,0x00,0xac,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x41,0x01,0x00,0x00, -0x3f,0x01,0x00,0x00,0x40,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x43,0x01,0x00,0x00,0x41,0x01,0x00,0x00, -0x6e,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x4e,0x01,0x00,0x00,0x3f,0x01,0x00,0x00,0xa0,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x4f,0x01,0x00,0x00, -0xb3,0x02,0x00,0x00,0x4e,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x51,0x01,0x00,0x00,0x4f,0x01,0x00,0x00, -0x6e,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x07,0x01,0x00,0x00, -0x52,0x01,0x00,0x00,0x47,0x01,0x00,0x00,0x34,0x00,0x00,0x00, -0x51,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0xec,0x00,0x00,0x00, -0x53,0x01,0x00,0x00,0x52,0x01,0x00,0x00,0x41,0x00,0x05,0x00, -0x0a,0x01,0x00,0x00,0x54,0x01,0x00,0x00,0x3c,0x01,0x00,0x00, -0x43,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x54,0x01,0x00,0x00, -0x53,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x37,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x55,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x58,0x01,0x00,0x00,0x73,0x00,0x00,0x00, -0xac,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x5a,0x01,0x00,0x00,0x58,0x01,0x00,0x00,0x59,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x5c,0x01,0x00,0x00, -0x5a,0x01,0x00,0x00,0x6e,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x0a,0x01,0x00,0x00,0x5d,0x01,0x00,0x00,0x3c,0x01,0x00,0x00, -0x5c,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x5d,0x01,0x00,0x00, -0x14,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x37,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x37,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0x21,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x21,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x5f,0x01,0x00,0x00, -0xac,0x02,0x00,0x00,0x1a,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0x1e,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x20,0x01,0x00,0x00, -0xe0,0x00,0x04,0x00,0x0c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x60,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x63,0x01,0x00,0x00,0xaf,0x02,0x00,0x00,0x61,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x66,0x01,0x00,0x00, -0xb3,0x02,0x00,0x00,0x64,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0x68,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x68,0x01,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xb5,0x02,0x00,0x00, -0x3e,0x00,0x00,0x00,0x20,0x01,0x00,0x00,0x12,0x02,0x00,0x00, -0x6b,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00, -0x6e,0x01,0x00,0x00,0xb5,0x02,0x00,0x00,0x6c,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0x6a,0x01,0x00,0x00,0x6b,0x01,0x00,0x00, -0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x6e,0x01,0x00,0x00, -0x69,0x01,0x00,0x00,0x6a,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x69,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x70,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x70,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xb9,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, -0x69,0x01,0x00,0x00,0x9c,0x01,0x00,0x00,0x73,0x01,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00,0x76,0x01,0x00,0x00, -0xb9,0x02,0x00,0x00,0x60,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0x72,0x01,0x00,0x00,0x73,0x01,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x76,0x01,0x00,0x00,0x71,0x01,0x00,0x00, -0x72,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x71,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0x78,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x78,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xcb,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0x71,0x01,0x00,0x00, -0x9a,0x01,0x00,0x00,0x79,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb7,0x00,0x00,0x00,0x7e,0x01,0x00,0x00,0xcb,0x02,0x00,0x00, -0x62,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x7a,0x01,0x00,0x00, -0x79,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x7e,0x01,0x00,0x00,0x79,0x01,0x00,0x00,0x7a,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x79,0x01,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x84,0x01,0x00,0x00,0xb9,0x02,0x00,0x00, -0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x86,0x01,0x00,0x00,0x84,0x01,0x00,0x00,0xcb,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x88,0x01,0x00,0x00, -0x55,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x8a,0x01,0x00,0x00,0xb9,0x02,0x00,0x00, -0x61,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x8b,0x01,0x00,0x00,0x88,0x01,0x00,0x00,0x8a,0x01,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8d,0x01,0x00,0x00, -0x64,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x8e,0x01,0x00,0x00,0x8b,0x01,0x00,0x00, -0x8d,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x90,0x01,0x00,0x00,0x8e,0x01,0x00,0x00,0xcb,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x92,0x01,0x00,0x00, -0x90,0x01,0x00,0x00,0x91,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x94,0x01,0x00,0x00,0x92,0x01,0x00,0x00, -0xb5,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x0a,0x01,0x00,0x00, -0x95,0x01,0x00,0x00,0xf1,0x00,0x00,0x00,0x94,0x01,0x00,0x00, -0x3d,0x00,0x04,0x00,0xec,0x00,0x00,0x00,0x96,0x01,0x00,0x00, -0x95,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0x97,0x01,0x00,0x00, -0x98,0x01,0x00,0x00,0x82,0x01,0x00,0x00,0x86,0x01,0x00,0x00, -0x3e,0x00,0x03,0x00,0x98,0x01,0x00,0x00,0x96,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9a,0x01,0x00,0x00, -0xcb,0x02,0x00,0x00,0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x78,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x7a,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0x73,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x73,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x9c,0x01,0x00,0x00,0xb9,0x02,0x00,0x00,0xc5,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x70,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x72,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x9e,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x9e,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xba,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, -0x72,0x01,0x00,0x00,0xca,0x01,0x00,0x00,0xa1,0x01,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00,0xa4,0x01,0x00,0x00, -0xba,0x02,0x00,0x00,0xb4,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0xa0,0x01,0x00,0x00,0xa1,0x01,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xa4,0x01,0x00,0x00,0x9f,0x01,0x00,0x00, -0xa0,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x9f,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0xa6,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xa6,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xc8,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0x9f,0x01,0x00,0x00, -0xc8,0x01,0x00,0x00,0xa7,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb7,0x00,0x00,0x00,0xac,0x01,0x00,0x00,0xc8,0x02,0x00,0x00, -0xb1,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xa8,0x01,0x00,0x00, -0xa7,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0xac,0x01,0x00,0x00,0xa7,0x01,0x00,0x00,0xa8,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xa7,0x01,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xb2,0x01,0x00,0x00,0xba,0x02,0x00,0x00, -0xb1,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xb4,0x01,0x00,0x00,0xb2,0x01,0x00,0x00,0xc8,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xb6,0x01,0x00,0x00, -0x59,0x00,0x00,0x00,0xae,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xb9,0x01,0x00,0x00,0xba,0x02,0x00,0x00, -0xb8,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xba,0x01,0x00,0x00,0xb6,0x01,0x00,0x00,0xb9,0x01,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xbc,0x01,0x00,0x00, -0x68,0x00,0x00,0x00,0xb1,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xbd,0x01,0x00,0x00,0xba,0x01,0x00,0x00, -0xbc,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xbf,0x01,0x00,0x00,0xbd,0x01,0x00,0x00,0xc8,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xc1,0x01,0x00,0x00, -0xbf,0x01,0x00,0x00,0xc0,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xc3,0x01,0x00,0x00,0xc1,0x01,0x00,0x00, -0xb5,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x0a,0x01,0x00,0x00, -0xc4,0x01,0x00,0x00,0x3c,0x01,0x00,0x00,0xc3,0x01,0x00,0x00, -0x3d,0x00,0x04,0x00,0xec,0x00,0x00,0x00,0xc5,0x01,0x00,0x00, -0xc4,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0x97,0x01,0x00,0x00, -0xc6,0x01,0x00,0x00,0xb0,0x01,0x00,0x00,0xb4,0x01,0x00,0x00, -0x3e,0x00,0x03,0x00,0xc6,0x01,0x00,0x00,0xc5,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xc8,0x01,0x00,0x00, -0xc8,0x02,0x00,0x00,0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xa6,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xa8,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0xa1,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xa1,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xca,0x01,0x00,0x00,0xba,0x02,0x00,0x00,0xc5,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x9e,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xa0,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xcc,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xcc,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xbb,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, -0xa0,0x01,0x00,0x00,0x10,0x02,0x00,0x00,0xcf,0x01,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00,0xd2,0x01,0x00,0x00, -0xbb,0x02,0x00,0x00,0xb4,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0xce,0x01,0x00,0x00,0xcf,0x01,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xd2,0x01,0x00,0x00,0xcd,0x01,0x00,0x00, -0xce,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xcd,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0xd4,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xd4,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xbf,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0xcd,0x01,0x00,0x00, -0x0e,0x02,0x00,0x00,0xd7,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb7,0x00,0x00,0x00,0xda,0x01,0x00,0x00,0xbf,0x02,0x00,0x00, -0x60,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xd6,0x01,0x00,0x00, -0xd7,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0xda,0x01,0x00,0x00,0xd5,0x01,0x00,0x00,0xd6,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xd5,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0xdc,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xdc,0x01,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xc1,0x02,0x00,0x00, -0x3e,0x00,0x00,0x00,0xd5,0x01,0x00,0x00,0x0c,0x02,0x00,0x00, -0xdf,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00, -0xe2,0x01,0x00,0x00,0xc1,0x02,0x00,0x00,0xb1,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0xde,0x01,0x00,0x00,0xdf,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xe2,0x01,0x00,0x00, -0xdd,0x01,0x00,0x00,0xde,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xdd,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xe4,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xe4,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xc3,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, -0xdd,0x01,0x00,0x00,0x0a,0x02,0x00,0x00,0xe5,0x01,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00,0xea,0x01,0x00,0x00, -0xc3,0x02,0x00,0x00,0x62,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0xe6,0x01,0x00,0x00,0xe5,0x01,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xea,0x01,0x00,0x00,0xe5,0x01,0x00,0x00, -0xe6,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xe5,0x01,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xec,0x01,0x00,0x00, -0xbb,0x02,0x00,0x00,0xb1,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xee,0x01,0x00,0x00,0xec,0x01,0x00,0x00, -0xc1,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xf0,0x01,0x00,0x00,0xee,0x01,0x00,0x00,0xef,0x01,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xf2,0x01,0x00,0x00, -0xbf,0x02,0x00,0x00,0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xf3,0x01,0x00,0x00,0xf0,0x01,0x00,0x00, -0xf2,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xf5,0x01,0x00,0x00,0xf3,0x01,0x00,0x00,0xc3,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xf9,0x01,0x00,0x00, -0xf2,0x01,0x00,0x00,0xc3,0x02,0x00,0x00,0x41,0x00,0x05,0x00, -0x97,0x01,0x00,0x00,0xfa,0x01,0x00,0x00,0x82,0x01,0x00,0x00, -0xf9,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0xec,0x00,0x00,0x00, -0xfb,0x01,0x00,0x00,0xfa,0x01,0x00,0x00,0x73,0x00,0x04,0x00, -0xb9,0x00,0x00,0x00,0xfc,0x01,0x00,0x00,0xfb,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0x97,0x01,0x00,0x00,0x01,0x02,0x00,0x00, -0xb0,0x01,0x00,0x00,0xee,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, -0xec,0x00,0x00,0x00,0x02,0x02,0x00,0x00,0x01,0x02,0x00,0x00, -0x73,0x00,0x04,0x00,0xb9,0x00,0x00,0x00,0x03,0x02,0x00,0x00, -0x02,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, -0x05,0x02,0x00,0x00,0xbf,0x00,0x00,0x00,0xf5,0x01,0x00,0x00, -0x3d,0x00,0x04,0x00,0xb9,0x00,0x00,0x00,0x06,0x02,0x00,0x00, -0x05,0x02,0x00,0x00,0x0c,0x00,0x08,0x00,0xb9,0x00,0x00,0x00, -0x07,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00, -0xfc,0x01,0x00,0x00,0x03,0x02,0x00,0x00,0x06,0x02,0x00,0x00, -0x3e,0x00,0x03,0x00,0x05,0x02,0x00,0x00,0x07,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x0a,0x02,0x00,0x00, -0xc3,0x02,0x00,0x00,0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xe4,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xe6,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0xdf,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xdf,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x0c,0x02,0x00,0x00,0xc1,0x02,0x00,0x00,0xc5,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xdc,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xde,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xd7,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xd7,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x0e,0x02,0x00,0x00,0xbf,0x02,0x00,0x00, -0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xd4,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xd6,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0xcf,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xcf,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x10,0x02,0x00,0x00, -0xbb,0x02,0x00,0x00,0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xcc,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xce,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0x6b,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x6b,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x12,0x02,0x00,0x00,0xb5,0x02,0x00,0x00,0xc5,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x68,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x6a,0x01,0x00,0x00,0xe0,0x00,0x04,0x00,0x0c,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x60,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0xcc,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xcc,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x14,0x02,0x00,0x00, -0x9b,0x02,0x00,0x00,0x6c,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xc9,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xcb,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x19,0x02,0x00,0x00, -0x55,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x1a,0x02,0x00,0x00,0x8b,0x00,0x00,0x00, -0x19,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x1f,0x02,0x00,0x00,0x59,0x00,0x00,0x00,0xae,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x20,0x02,0x00,0x00, -0x9d,0x00,0x00,0x00,0x1f,0x02,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x25,0x02,0x00,0x00,0x12,0x00,0x00,0x00, -0x24,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x26,0x02,0x00,0x00,0x25,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x27,0x02,0x00,0x00,0x0f,0x00,0x00,0x00, -0x26,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x2b,0x02,0x00,0x00,0x47,0x00,0x00,0x00,0x26,0x02,0x00,0x00, -0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00,0x2d,0x02,0x00,0x00, -0x2c,0x02,0x00,0x00,0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x2e,0x02,0x00,0x00,0x2d,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x2f,0x02,0x00,0x00, -0x2b,0x02,0x00,0x00,0x2e,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x30,0x02,0x00,0x00,0x27,0x02,0x00,0x00, -0x2f,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x32,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x32,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0x9c,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, -0xcb,0x00,0x00,0x00,0x99,0x02,0x00,0x00,0x35,0x02,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00,0x38,0x02,0x00,0x00, -0x9c,0x02,0x00,0x00,0xb4,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0x34,0x02,0x00,0x00,0x35,0x02,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x38,0x02,0x00,0x00,0x33,0x02,0x00,0x00, -0x34,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x33,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0x3a,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x3a,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0x9d,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0x33,0x02,0x00,0x00, -0x97,0x02,0x00,0x00,0x3d,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb7,0x00,0x00,0x00,0x40,0x02,0x00,0x00,0x9d,0x02,0x00,0x00, -0x60,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x3c,0x02,0x00,0x00, -0x3d,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x40,0x02,0x00,0x00,0x3b,0x02,0x00,0x00,0x3c,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x3b,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x44,0x02,0x00,0x00,0x9d,0x02,0x00,0x00, -0x61,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x45,0x02,0x00,0x00,0x1a,0x02,0x00,0x00,0x44,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x47,0x02,0x00,0x00, -0x64,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x48,0x02,0x00,0x00,0x45,0x02,0x00,0x00, -0x47,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x4c,0x02,0x00,0x00,0x9c,0x02,0x00,0x00,0xb8,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x4d,0x02,0x00,0x00, -0x20,0x02,0x00,0x00,0x4c,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x4f,0x02,0x00,0x00,0x68,0x00,0x00,0x00, -0xb1,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x50,0x02,0x00,0x00,0x4d,0x02,0x00,0x00,0x4f,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0x52,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x52,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0x9f,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0x3b,0x02,0x00,0x00, -0x95,0x02,0x00,0x00,0x55,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb7,0x00,0x00,0x00,0x58,0x02,0x00,0x00,0x9f,0x02,0x00,0x00, -0xb1,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x54,0x02,0x00,0x00, -0x55,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x58,0x02,0x00,0x00,0x53,0x02,0x00,0x00,0x54,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x53,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x5a,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x5a,0x02,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xa1,0x02,0x00,0x00, -0x3e,0x00,0x00,0x00,0x53,0x02,0x00,0x00,0x93,0x02,0x00,0x00, -0x5d,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00, -0x60,0x02,0x00,0x00,0xa1,0x02,0x00,0x00,0x62,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0x5c,0x02,0x00,0x00,0x5d,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x60,0x02,0x00,0x00, -0x5b,0x02,0x00,0x00,0x5c,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x5b,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x63,0x02,0x00,0x00,0x48,0x02,0x00,0x00,0xa1,0x02,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00,0x66,0x02,0x00,0x00, -0x63,0x02,0x00,0x00,0x36,0x00,0x00,0x00,0xf7,0x00,0x03,0x00, -0x68,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x66,0x02,0x00,0x00,0x67,0x02,0x00,0x00,0x68,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x67,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x6b,0x02,0x00,0x00,0x50,0x02,0x00,0x00, -0x9f,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0x6c,0x02,0x00,0x00,0x12,0x00,0x00,0x00,0xc5,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x6d,0x02,0x00,0x00, -0x6c,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00, -0x6e,0x02,0x00,0x00,0x6b,0x02,0x00,0x00,0x6d,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0x68,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x68,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0xb7,0x00,0x00,0x00, -0x6f,0x02,0x00,0x00,0x66,0x02,0x00,0x00,0x5b,0x02,0x00,0x00, -0x6e,0x02,0x00,0x00,0x67,0x02,0x00,0x00,0xf7,0x00,0x03,0x00, -0x71,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x6f,0x02,0x00,0x00,0x70,0x02,0x00,0x00,0x71,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x70,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x79,0x02,0x00,0x00,0x50,0x02,0x00,0x00, -0x9f,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0x7b,0x02,0x00,0x00,0x12,0x00,0x00,0x00,0x7a,0x02,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x7c,0x02,0x00,0x00, -0x7b,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x7d,0x02,0x00,0x00,0x79,0x02,0x00,0x00,0x7c,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x7e,0x02,0x00,0x00, -0x30,0x02,0x00,0x00,0x7d,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x80,0x02,0x00,0x00,0x7e,0x02,0x00,0x00, -0x48,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x82,0x02,0x00,0x00,0x80,0x02,0x00,0x00,0xa1,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x84,0x02,0x00,0x00, -0x9c,0x02,0x00,0x00,0xb1,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x86,0x02,0x00,0x00,0x84,0x02,0x00,0x00, -0x9f,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x88,0x02,0x00,0x00,0x86,0x02,0x00,0x00,0x87,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8a,0x02,0x00,0x00, -0x9d,0x02,0x00,0x00,0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x8b,0x02,0x00,0x00,0x88,0x02,0x00,0x00, -0x8a,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x8d,0x02,0x00,0x00,0x8b,0x02,0x00,0x00,0xa1,0x02,0x00,0x00, -0x41,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0x8e,0x02,0x00,0x00, -0xbf,0x00,0x00,0x00,0x8d,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, -0xb9,0x00,0x00,0x00,0x8f,0x02,0x00,0x00,0x8e,0x02,0x00,0x00, -0x41,0x00,0x06,0x00,0x90,0x02,0x00,0x00,0x91,0x02,0x00,0x00, -0x75,0x02,0x00,0x00,0x34,0x00,0x00,0x00,0x82,0x02,0x00,0x00, -0x3e,0x00,0x03,0x00,0x91,0x02,0x00,0x00,0x8f,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0x71,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x71,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x5d,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x5d,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x93,0x02,0x00,0x00,0xa1,0x02,0x00,0x00, -0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x5a,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x5c,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x55,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x55,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x95,0x02,0x00,0x00, -0x9f,0x02,0x00,0x00,0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x52,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x54,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0x3d,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x3d,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x97,0x02,0x00,0x00,0x9d,0x02,0x00,0x00,0xc5,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x3a,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x3c,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x35,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x35,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x99,0x02,0x00,0x00,0x9c,0x02,0x00,0x00, -0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x32,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x34,0x02,0x00,0x00,0xfd,0x00,0x01,0x00, -0x38,0x00,0x01,0x00, -}; -const uint64_t matmul_f16_m_len = 10156; - -unsigned char matmul_f16_m_fp32_data[] = { -0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, -0xca,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, -0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x51,0x11,0x00,0x00, -0x0b,0x00,0x06,0x00,0x01,0x00,0x00,0x00,0x47,0x4c,0x53,0x4c, -0x2e,0x73,0x74,0x64,0x2e,0x34,0x35,0x30,0x00,0x00,0x00,0x00, -0x0e,0x00,0x03,0x00,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x0f,0x00,0x0f,0x00,0x05,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x6d,0x61,0x69,0x6e,0x00,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x3d,0x00,0x00,0x00,0x4c,0x00,0x00,0x00, -0xf0,0x00,0x00,0x00,0xfc,0x00,0x00,0x00,0x3c,0x01,0x00,0x00, -0x47,0x01,0x00,0x00,0x2a,0x02,0x00,0x00,0x73,0x02,0x00,0x00, -0x10,0x00,0x06,0x00,0x04,0x00,0x00,0x00,0x11,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x0b,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x1c,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x10,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x08,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x03,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x10,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x14,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x18,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x07,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x10,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x24,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x0a,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x10,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x2c,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x30,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x0d,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0x47,0x00,0x03,0x00, -0x10,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x37,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x3d,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x1a,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x4c,0x00,0x00,0x00, -0x0b,0x00,0x00,0x00,0x1b,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x4f,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x53,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x60,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x62,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x07,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x6c,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x03,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x9c,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0xae,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x05,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0xb1,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x08,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xf9,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x48,0x00,0x04,0x00, -0xfa,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0xfa,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, -0xfa,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0xfc,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0xfc,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x16,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x17,0x01,0x00,0x00,0x0b,0x00,0x00,0x00,0x19,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x44,0x01,0x00,0x00,0x06,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0x45,0x01,0x00,0x00, -0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x45,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x45,0x01,0x00,0x00, -0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x47,0x01,0x00,0x00, -0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x47,0x01,0x00,0x00,0x21,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x2a,0x02,0x00,0x00,0x0b,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x70,0x02,0x00,0x00, -0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00, -0x71,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x19,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x71,0x02,0x00,0x00,0x00,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, -0x71,0x02,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x73,0x02,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x73,0x02,0x00,0x00,0x21,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x13,0x00,0x02,0x00,0x02,0x00,0x00,0x00, -0x21,0x00,0x03,0x00,0x03,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x15,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x17,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x0a,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x0d,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x1e,0x00,0x10,0x00,0x10,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x11,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x11,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x15,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, -0x14,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x15,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x0a,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, -0x27,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0x2d,0x00,0x00,0x00,0x07,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x34,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x37,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00,0x3d,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x3e,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x0a,0x00,0x00,0x00,0x4c,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x4f,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x53,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x54,0x00,0x00,0x00,0x86,0x00,0x00,0x00, -0x37,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x58,0x00,0x00,0x00,0x86,0x00,0x00,0x00, -0x37,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x32,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x61,0x00,0x00,0x00, -0x86,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x60,0x00,0x00,0x00, -0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x62,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x63,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x61,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x67,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x61,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x86,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x72,0x00,0x00,0x00,0x86,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0x76,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x7b,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, -0x86,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0x8c,0x00,0x00,0x00,0x03,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x97,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x9c,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0x9e,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xad,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00, -0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xae,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xaf,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x53,0x00,0x00,0x00, -0xae,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xb0,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x4f,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xb1,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xb2,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0xb0,0x00,0x00,0x00,0xb1,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xb3,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0xb2,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xb4,0x00,0x00,0x00,0x86,0x00,0x00,0x00, -0xaf,0x00,0x00,0x00,0xb3,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xb5,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0xad,0x00,0x00,0x00,0xb4,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xb6,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0xb5,0x00,0x00,0x00,0xb1,0x00,0x00,0x00,0x14,0x00,0x02,0x00, -0xb7,0x00,0x00,0x00,0x16,0x00,0x03,0x00,0xb9,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xba,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xbb,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0xba,0x00,0x00,0x00, -0xb4,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xbc,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0xbb,0x00,0x00,0x00, -0xb1,0x00,0x00,0x00,0x1c,0x00,0x04,0x00,0xbd,0x00,0x00,0x00, -0xb9,0x00,0x00,0x00,0xbc,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0xbe,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0xbd,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0xb9,0x00,0x00,0x00,0xc1,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0xc2,0x00,0x00,0x00, -0x07,0x00,0x00,0x00,0xb9,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0xc5,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xec,0x00,0x00,0x00, -0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xed,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0xec,0x00,0x00,0x00, -0x1c,0x00,0x04,0x00,0xee,0x00,0x00,0x00,0xb9,0x00,0x00,0x00, -0xed,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0xef,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0xee,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0xef,0x00,0x00,0x00,0xf0,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xf4,0x00,0x00,0x00, -0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x16,0x00,0x03,0x00,0xf8,0x00,0x00,0x00,0x10,0x00,0x00,0x00, -0x1d,0x00,0x03,0x00,0xf9,0x00,0x00,0x00,0xf8,0x00,0x00,0x00, -0x1e,0x00,0x03,0x00,0xfa,0x00,0x00,0x00,0xf9,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0xfb,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0xfa,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0xfb,0x00,0x00,0x00, -0xfc,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x07,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0xf8,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x0b,0x01,0x00,0x00,0x04,0x00,0x00,0x00, -0xb9,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x11,0x01,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x16,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x33,0x00,0x06,0x00, -0x09,0x00,0x00,0x00,0x17,0x01,0x00,0x00,0x16,0x01,0x00,0x00, -0x39,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x18,0x01,0x00,0x00,0x51,0x00,0x00,0x00, -0x17,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x19,0x01,0x00,0x00,0x84,0x00,0x00,0x00, -0x18,0x01,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x1a,0x01,0x00,0x00,0x86,0x00,0x00,0x00, -0x19,0x01,0x00,0x00,0x6c,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x38,0x01,0x00,0x00,0x80,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x39,0x01,0x00,0x00,0x84,0x00,0x00,0x00, -0x9c,0x00,0x00,0x00,0x38,0x01,0x00,0x00,0x1c,0x00,0x04,0x00, -0x3a,0x01,0x00,0x00,0xb9,0x00,0x00,0x00,0x39,0x01,0x00,0x00, -0x20,0x00,0x04,0x00,0x3b,0x01,0x00,0x00,0x04,0x00,0x00,0x00, -0x3a,0x01,0x00,0x00,0x3b,0x00,0x04,0x00,0x3b,0x01,0x00,0x00, -0x3c,0x01,0x00,0x00,0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x40,0x01,0x00,0x00,0x80,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, -0x44,0x01,0x00,0x00,0xf8,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, -0x45,0x01,0x00,0x00,0x44,0x01,0x00,0x00,0x20,0x00,0x04,0x00, -0x46,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x45,0x01,0x00,0x00, -0x3b,0x00,0x04,0x00,0x46,0x01,0x00,0x00,0x47,0x01,0x00,0x00, -0x0c,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x5a,0x01,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x61,0x01,0x00,0x00,0x08,0x01,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x62,0x01,0x00,0x00,0x86,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x65,0x01,0x00,0x00,0x86,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x80,0x01,0x00,0x00,0x84,0x00,0x00,0x00, -0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x1c,0x00,0x04,0x00, -0x81,0x01,0x00,0x00,0xb9,0x00,0x00,0x00,0x80,0x01,0x00,0x00, -0x20,0x00,0x04,0x00,0x82,0x01,0x00,0x00,0x07,0x00,0x00,0x00, -0x81,0x01,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x92,0x01,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xad,0x01,0x00,0x00,0x84,0x00,0x00,0x00,0xb4,0x00,0x00,0x00, -0xb1,0x00,0x00,0x00,0x1c,0x00,0x04,0x00,0xae,0x01,0x00,0x00, -0xb9,0x00,0x00,0x00,0xad,0x01,0x00,0x00,0x20,0x00,0x04,0x00, -0xaf,0x01,0x00,0x00,0x07,0x00,0x00,0x00,0xae,0x01,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xb8,0x01,0x00,0x00, -0x86,0x00,0x00,0x00,0xae,0x00,0x00,0x00,0xb4,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xc0,0x01,0x00,0x00, -0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xef,0x01,0x00,0x00, -0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x22,0x02,0x00,0x00, -0x0d,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00, -0x2a,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, -0x70,0x02,0x00,0x00,0xb9,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, -0x71,0x02,0x00,0x00,0x70,0x02,0x00,0x00,0x20,0x00,0x04,0x00, -0x72,0x02,0x00,0x00,0x0c,0x00,0x00,0x00,0x71,0x02,0x00,0x00, -0x3b,0x00,0x04,0x00,0x72,0x02,0x00,0x00,0x73,0x02,0x00,0x00, -0x0c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, -0x78,0x02,0x00,0x00,0x05,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x85,0x02,0x00,0x00,0x84,0x00,0x00,0x00, -0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x8e,0x02,0x00,0x00,0x0c,0x00,0x00,0x00,0xb9,0x00,0x00,0x00, -0x36,0x00,0x05,0x00,0x02,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x05,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0xbe,0x00,0x00,0x00, -0xbf,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x82,0x01,0x00,0x00,0x83,0x01,0x00,0x00,0x07,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0xaf,0x01,0x00,0x00,0xb0,0x01,0x00,0x00, -0x07,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00, -0x0e,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, -0x0e,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x14,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x17,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x17,0x00,0x00,0x00, -0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x1e,0x00,0x00,0x00, -0x0f,0x00,0x00,0x00,0x17,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x21,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x24,0x00,0x00,0x00,0x18,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0x28,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x27,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x29,0x00,0x00,0x00, -0x28,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x2a,0x00,0x00,0x00,0x1e,0x00,0x00,0x00,0x29,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x2d,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x2f,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x30,0x00,0x00,0x00, -0x24,0x00,0x00,0x00,0x2f,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x30,0x00,0x00,0x00, -0x2a,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0x35,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x34,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x36,0x00,0x00,0x00, -0x35,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x38,0x00,0x00,0x00,0x36,0x00,0x00,0x00,0x37,0x00,0x00,0x00, -0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, -0x38,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x3b,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, -0x37,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00, -0x3f,0x00,0x00,0x00,0x3d,0x00,0x00,0x00,0x3e,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x3f,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x42,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x3b,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x47,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0x3b,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x0d,0x00,0x00,0x00,0x49,0x00,0x00,0x00,0x3d,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x4a,0x00,0x00,0x00,0x49,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x0d,0x00,0x00,0x00,0x4d,0x00,0x00,0x00,0x4c,0x00,0x00,0x00, -0x3e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x4e,0x00,0x00,0x00,0x4d,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x50,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, -0x4f,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x55,0x00,0x00,0x00,0x50,0x00,0x00,0x00,0x54,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x59,0x00,0x00,0x00, -0x50,0x00,0x00,0x00,0x58,0x00,0x00,0x00,0x89,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x5d,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, -0x4f,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x64,0x00,0x00,0x00,0x5d,0x00,0x00,0x00,0x63,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x68,0x00,0x00,0x00, -0x5d,0x00,0x00,0x00,0x67,0x00,0x00,0x00,0x89,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x6e,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, -0x6d,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x73,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x72,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x77,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x76,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x78,0x00,0x00,0x00,0x77,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x79,0x00,0x00,0x00, -0x47,0x00,0x00,0x00,0x78,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x7c,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x7b,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x7d,0x00,0x00,0x00,0x7c,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x7f,0x00,0x00,0x00,0x47,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x82,0x00,0x00,0x00,0x7f,0x00,0x00,0x00,0x78,0x00,0x00,0x00, -0x0c,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x83,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x26,0x00,0x00,0x00,0x7d,0x00,0x00,0x00, -0x82,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0x87,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x86,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x88,0x00,0x00,0x00, -0x87,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x89,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x88,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8b,0x00,0x00,0x00, -0x42,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x8d,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x8c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x8e,0x00,0x00,0x00,0x8d,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x8f,0x00,0x00,0x00,0x8b,0x00,0x00,0x00, -0x8e,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x90,0x00,0x00,0x00,0x89,0x00,0x00,0x00,0x8f,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x92,0x00,0x00,0x00, -0x90,0x00,0x00,0x00,0x79,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x93,0x00,0x00,0x00,0x92,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0x98,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x97,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x99,0x00,0x00,0x00, -0x98,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x9a,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x99,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9d,0x00,0x00,0x00, -0x4a,0x00,0x00,0x00,0x9c,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x9f,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x9e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xa0,0x00,0x00,0x00,0x9f,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xa1,0x00,0x00,0x00,0x9d,0x00,0x00,0x00, -0xa0,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xa2,0x00,0x00,0x00,0x9a,0x00,0x00,0x00,0xa1,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa4,0x00,0x00,0x00, -0xa2,0x00,0x00,0x00,0x79,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xa5,0x00,0x00,0x00,0xa4,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xa7,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xa7,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0x98,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, -0x05,0x00,0x00,0x00,0xc6,0x00,0x00,0x00,0xa8,0x00,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00,0xb8,0x00,0x00,0x00, -0x98,0x02,0x00,0x00,0xb6,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0xa9,0x00,0x00,0x00,0xa8,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xb8,0x00,0x00,0x00,0xa8,0x00,0x00,0x00, -0xa9,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xa8,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0xc3,0x00,0x00,0x00, -0xbf,0x00,0x00,0x00,0x98,0x02,0x00,0x00,0x3e,0x00,0x03,0x00, -0xc3,0x00,0x00,0x00,0xc1,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xc6,0x00,0x00,0x00,0x98,0x02,0x00,0x00, -0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xa7,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xa9,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xc9,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xc9,0x00,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xb1,0x02,0x00,0x00, -0xa5,0x00,0x00,0x00,0xa9,0x00,0x00,0x00,0x67,0x01,0x00,0x00, -0xcc,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xad,0x02,0x00,0x00,0x93,0x00,0x00,0x00,0xa9,0x00,0x00,0x00, -0x64,0x01,0x00,0x00,0xcc,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0x99,0x02,0x00,0x00,0x79,0x00,0x00,0x00, -0xa9,0x00,0x00,0x00,0x12,0x02,0x00,0x00,0xcc,0x00,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00,0xd0,0x00,0x00,0x00, -0x99,0x02,0x00,0x00,0x83,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0xcb,0x00,0x00,0x00,0xcc,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xd0,0x00,0x00,0x00,0xca,0x00,0x00,0x00, -0xcb,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xca,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xd2,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xd2,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xa9,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0xca,0x00,0x00,0x00, -0x1c,0x01,0x00,0x00,0xd5,0x00,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb7,0x00,0x00,0x00,0xd8,0x00,0x00,0x00,0xa9,0x02,0x00,0x00, -0x37,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xd4,0x00,0x00,0x00, -0xd5,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0xd8,0x00,0x00,0x00,0xd3,0x00,0x00,0x00,0xd4,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xd3,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xdc,0x00,0x00,0x00,0x8b,0x00,0x00,0x00, -0x73,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xde,0x00,0x00,0x00,0xdc,0x00,0x00,0x00,0xa9,0x02,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00,0xe1,0x00,0x00,0x00, -0xde,0x00,0x00,0x00,0x36,0x00,0x00,0x00,0xf7,0x00,0x03,0x00, -0xe3,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0xe1,0x00,0x00,0x00,0xe2,0x00,0x00,0x00,0xe3,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xe2,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xe6,0x00,0x00,0x00,0x99,0x02,0x00,0x00, -0x6e,0x00,0x00,0x00,0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00, -0xe8,0x00,0x00,0x00,0xe6,0x00,0x00,0x00,0x83,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xe3,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xe3,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0xb7,0x00,0x00,0x00, -0xe9,0x00,0x00,0x00,0xe1,0x00,0x00,0x00,0xd3,0x00,0x00,0x00, -0xe8,0x00,0x00,0x00,0xe2,0x00,0x00,0x00,0xf7,0x00,0x03,0x00, -0xeb,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0xe9,0x00,0x00,0x00,0xea,0x00,0x00,0x00,0x0d,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xea,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xf3,0x00,0x00,0x00,0x73,0x00,0x00,0x00, -0xa9,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xf5,0x00,0x00,0x00,0xf3,0x00,0x00,0x00,0xf4,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xf7,0x00,0x00,0x00, -0xf5,0x00,0x00,0x00,0x6e,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x03,0x01,0x00,0x00,0xf3,0x00,0x00,0x00, -0x8e,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x04,0x01,0x00,0x00,0xad,0x02,0x00,0x00,0x03,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x06,0x01,0x00,0x00, -0x04,0x01,0x00,0x00,0x6e,0x00,0x00,0x00,0x41,0x00,0x06,0x00, -0x07,0x01,0x00,0x00,0x08,0x01,0x00,0x00,0xfc,0x00,0x00,0x00, -0x34,0x00,0x00,0x00,0x06,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, -0xf8,0x00,0x00,0x00,0x09,0x01,0x00,0x00,0x08,0x01,0x00,0x00, -0x73,0x00,0x04,0x00,0xb9,0x00,0x00,0x00,0x0a,0x01,0x00,0x00, -0x09,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0x0b,0x01,0x00,0x00, -0x0c,0x01,0x00,0x00,0xf0,0x00,0x00,0x00,0xf7,0x00,0x00,0x00, -0x3e,0x00,0x03,0x00,0x0c,0x01,0x00,0x00,0x0a,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0xeb,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x0d,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x10,0x01,0x00,0x00,0x73,0x00,0x00,0x00,0xa9,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x12,0x01,0x00,0x00, -0x10,0x01,0x00,0x00,0x11,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x14,0x01,0x00,0x00,0x12,0x01,0x00,0x00, -0x6e,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x0b,0x01,0x00,0x00, -0x15,0x01,0x00,0x00,0xf0,0x00,0x00,0x00,0x14,0x01,0x00,0x00, -0x3e,0x00,0x03,0x00,0x15,0x01,0x00,0x00,0xc1,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xeb,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xeb,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xd5,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xd5,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x1c,0x01,0x00,0x00,0xa9,0x02,0x00,0x00, -0x1a,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xd2,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xd4,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x1e,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x1e,0x01,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xaa,0x02,0x00,0x00, -0x3e,0x00,0x00,0x00,0xd4,0x00,0x00,0x00,0x60,0x01,0x00,0x00, -0x21,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00, -0x24,0x01,0x00,0x00,0xaa,0x02,0x00,0x00,0x9c,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0x20,0x01,0x00,0x00,0x21,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x24,0x01,0x00,0x00, -0x1f,0x01,0x00,0x00,0x20,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x1f,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x28,0x01,0x00,0x00,0x9d,0x00,0x00,0x00,0x73,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x2a,0x01,0x00,0x00, -0x28,0x01,0x00,0x00,0xaa,0x02,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x2b,0x01,0x00,0x00,0x12,0x00,0x00,0x00, -0xc5,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x2c,0x01,0x00,0x00,0x2b,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb7,0x00,0x00,0x00,0x2d,0x01,0x00,0x00,0x2a,0x01,0x00,0x00, -0x2c,0x01,0x00,0x00,0xf7,0x00,0x03,0x00,0x2f,0x01,0x00,0x00, -0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x2d,0x01,0x00,0x00, -0x2e,0x01,0x00,0x00,0x2f,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x2e,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x32,0x01,0x00,0x00,0x99,0x02,0x00,0x00,0x6e,0x00,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00,0x34,0x01,0x00,0x00, -0x32,0x01,0x00,0x00,0x83,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x2f,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x2f,0x01,0x00,0x00, -0xf5,0x00,0x07,0x00,0xb7,0x00,0x00,0x00,0x35,0x01,0x00,0x00, -0x2d,0x01,0x00,0x00,0x1f,0x01,0x00,0x00,0x34,0x01,0x00,0x00, -0x2e,0x01,0x00,0x00,0xf7,0x00,0x03,0x00,0x37,0x01,0x00,0x00, -0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x35,0x01,0x00,0x00, -0x36,0x01,0x00,0x00,0x56,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x36,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x3f,0x01,0x00,0x00,0x73,0x00,0x00,0x00,0xaa,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x41,0x01,0x00,0x00, -0x3f,0x01,0x00,0x00,0x40,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x43,0x01,0x00,0x00,0x41,0x01,0x00,0x00, -0x6e,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x4e,0x01,0x00,0x00,0x3f,0x01,0x00,0x00,0xa0,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x4f,0x01,0x00,0x00, -0xb1,0x02,0x00,0x00,0x4e,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x51,0x01,0x00,0x00,0x4f,0x01,0x00,0x00, -0x6e,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x07,0x01,0x00,0x00, -0x52,0x01,0x00,0x00,0x47,0x01,0x00,0x00,0x34,0x00,0x00,0x00, -0x51,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0xf8,0x00,0x00,0x00, -0x53,0x01,0x00,0x00,0x52,0x01,0x00,0x00,0x73,0x00,0x04,0x00, -0xb9,0x00,0x00,0x00,0x54,0x01,0x00,0x00,0x53,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0x0b,0x01,0x00,0x00,0x55,0x01,0x00,0x00, -0x3c,0x01,0x00,0x00,0x43,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0x55,0x01,0x00,0x00,0x54,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0x37,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x56,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x59,0x01,0x00,0x00, -0x73,0x00,0x00,0x00,0xaa,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x5b,0x01,0x00,0x00,0x59,0x01,0x00,0x00, -0x5a,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x5d,0x01,0x00,0x00,0x5b,0x01,0x00,0x00,0x6e,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x0b,0x01,0x00,0x00,0x5e,0x01,0x00,0x00, -0x3c,0x01,0x00,0x00,0x5d,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0x5e,0x01,0x00,0x00,0xc1,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x37,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x37,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0x21,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x21,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x60,0x01,0x00,0x00,0xaa,0x02,0x00,0x00,0x1a,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0x1e,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x20,0x01,0x00,0x00,0xe0,0x00,0x04,0x00,0x0c,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x61,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x64,0x01,0x00,0x00,0xad,0x02,0x00,0x00, -0x62,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x67,0x01,0x00,0x00,0xb1,0x02,0x00,0x00,0x65,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0x69,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x69,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xb3,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0x20,0x01,0x00,0x00, -0x10,0x02,0x00,0x00,0x6c,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb7,0x00,0x00,0x00,0x6f,0x01,0x00,0x00,0xb3,0x02,0x00,0x00, -0x6c,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x6b,0x01,0x00,0x00, -0x6c,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x6f,0x01,0x00,0x00,0x6a,0x01,0x00,0x00,0x6b,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x6a,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0x71,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x71,0x01,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xb7,0x02,0x00,0x00, -0x3e,0x00,0x00,0x00,0x6a,0x01,0x00,0x00,0x9c,0x01,0x00,0x00, -0x74,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00, -0x77,0x01,0x00,0x00,0xb7,0x02,0x00,0x00,0x60,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0x73,0x01,0x00,0x00,0x74,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x77,0x01,0x00,0x00, -0x72,0x01,0x00,0x00,0x73,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x72,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x79,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x79,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xc9,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, -0x72,0x01,0x00,0x00,0x9a,0x01,0x00,0x00,0x7a,0x01,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00,0x7f,0x01,0x00,0x00, -0xc9,0x02,0x00,0x00,0x62,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0x7b,0x01,0x00,0x00,0x7a,0x01,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x7f,0x01,0x00,0x00,0x7a,0x01,0x00,0x00, -0x7b,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x7a,0x01,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x85,0x01,0x00,0x00, -0xb7,0x02,0x00,0x00,0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x87,0x01,0x00,0x00,0x85,0x01,0x00,0x00, -0xc9,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x89,0x01,0x00,0x00,0x55,0x00,0x00,0x00,0x53,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8b,0x01,0x00,0x00, -0xb7,0x02,0x00,0x00,0x61,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x8c,0x01,0x00,0x00,0x89,0x01,0x00,0x00, -0x8b,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x8e,0x01,0x00,0x00,0x64,0x00,0x00,0x00,0x62,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8f,0x01,0x00,0x00, -0x8c,0x01,0x00,0x00,0x8e,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x91,0x01,0x00,0x00,0x8f,0x01,0x00,0x00, -0xc9,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x93,0x01,0x00,0x00,0x91,0x01,0x00,0x00,0x92,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x95,0x01,0x00,0x00, -0x93,0x01,0x00,0x00,0xb3,0x02,0x00,0x00,0x41,0x00,0x05,0x00, -0x0b,0x01,0x00,0x00,0x96,0x01,0x00,0x00,0xf0,0x00,0x00,0x00, -0x95,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0xb9,0x00,0x00,0x00, -0x97,0x01,0x00,0x00,0x96,0x01,0x00,0x00,0x41,0x00,0x05,0x00, -0xc2,0x00,0x00,0x00,0x98,0x01,0x00,0x00,0x83,0x01,0x00,0x00, -0x87,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x98,0x01,0x00,0x00, -0x97,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x9a,0x01,0x00,0x00,0xc9,0x02,0x00,0x00,0xc5,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x79,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x7b,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x74,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x74,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x9c,0x01,0x00,0x00,0xb7,0x02,0x00,0x00, -0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x71,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x73,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0x9e,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x9e,0x01,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xb8,0x02,0x00,0x00, -0x3e,0x00,0x00,0x00,0x73,0x01,0x00,0x00,0xca,0x01,0x00,0x00, -0xa1,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00, -0xa4,0x01,0x00,0x00,0xb8,0x02,0x00,0x00,0xb4,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0xa0,0x01,0x00,0x00,0xa1,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xa4,0x01,0x00,0x00, -0x9f,0x01,0x00,0x00,0xa0,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x9f,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xa6,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xa6,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xc6,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, -0x9f,0x01,0x00,0x00,0xc8,0x01,0x00,0x00,0xa7,0x01,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00,0xac,0x01,0x00,0x00, -0xc6,0x02,0x00,0x00,0xb1,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0xa8,0x01,0x00,0x00,0xa7,0x01,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xac,0x01,0x00,0x00,0xa7,0x01,0x00,0x00, -0xa8,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xa7,0x01,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xb2,0x01,0x00,0x00, -0xb8,0x02,0x00,0x00,0xb1,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xb4,0x01,0x00,0x00,0xb2,0x01,0x00,0x00, -0xc6,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xb6,0x01,0x00,0x00,0x59,0x00,0x00,0x00,0xae,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xb9,0x01,0x00,0x00, -0xb8,0x02,0x00,0x00,0xb8,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xba,0x01,0x00,0x00,0xb6,0x01,0x00,0x00, -0xb9,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xbc,0x01,0x00,0x00,0x68,0x00,0x00,0x00,0xb1,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xbd,0x01,0x00,0x00, -0xba,0x01,0x00,0x00,0xbc,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xbf,0x01,0x00,0x00,0xbd,0x01,0x00,0x00, -0xc6,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xc1,0x01,0x00,0x00,0xbf,0x01,0x00,0x00,0xc0,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xc3,0x01,0x00,0x00, -0xc1,0x01,0x00,0x00,0xb3,0x02,0x00,0x00,0x41,0x00,0x05,0x00, -0x0b,0x01,0x00,0x00,0xc4,0x01,0x00,0x00,0x3c,0x01,0x00,0x00, -0xc3,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0xb9,0x00,0x00,0x00, -0xc5,0x01,0x00,0x00,0xc4,0x01,0x00,0x00,0x41,0x00,0x05,0x00, -0xc2,0x00,0x00,0x00,0xc6,0x01,0x00,0x00,0xb0,0x01,0x00,0x00, -0xb4,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0xc6,0x01,0x00,0x00, -0xc5,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xc8,0x01,0x00,0x00,0xc6,0x02,0x00,0x00,0xc5,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xa6,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xa8,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xa1,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xa1,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xca,0x01,0x00,0x00,0xb8,0x02,0x00,0x00, -0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x9e,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xa0,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0xcc,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xcc,0x01,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xb9,0x02,0x00,0x00, -0x3e,0x00,0x00,0x00,0xa0,0x01,0x00,0x00,0x0e,0x02,0x00,0x00, -0xcf,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00, -0xd2,0x01,0x00,0x00,0xb9,0x02,0x00,0x00,0xb4,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0xce,0x01,0x00,0x00,0xcf,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xd2,0x01,0x00,0x00, -0xcd,0x01,0x00,0x00,0xce,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xcd,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xd4,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xd4,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xbd,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, -0xcd,0x01,0x00,0x00,0x0c,0x02,0x00,0x00,0xd7,0x01,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00,0xda,0x01,0x00,0x00, -0xbd,0x02,0x00,0x00,0x60,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0xd6,0x01,0x00,0x00,0xd7,0x01,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xda,0x01,0x00,0x00,0xd5,0x01,0x00,0x00, -0xd6,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xd5,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0xdc,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xdc,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xbf,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0xd5,0x01,0x00,0x00, -0x0a,0x02,0x00,0x00,0xdf,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb7,0x00,0x00,0x00,0xe2,0x01,0x00,0x00,0xbf,0x02,0x00,0x00, -0xb1,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xde,0x01,0x00,0x00, -0xdf,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0xe2,0x01,0x00,0x00,0xdd,0x01,0x00,0x00,0xde,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xdd,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0xe4,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xe4,0x01,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xc1,0x02,0x00,0x00, -0x3e,0x00,0x00,0x00,0xdd,0x01,0x00,0x00,0x08,0x02,0x00,0x00, -0xe5,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00, -0xea,0x01,0x00,0x00,0xc1,0x02,0x00,0x00,0x62,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0xe6,0x01,0x00,0x00,0xe5,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xea,0x01,0x00,0x00, -0xe5,0x01,0x00,0x00,0xe6,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xe5,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xec,0x01,0x00,0x00,0xb9,0x02,0x00,0x00,0xb1,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xee,0x01,0x00,0x00, -0xec,0x01,0x00,0x00,0xbf,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xf0,0x01,0x00,0x00,0xee,0x01,0x00,0x00, -0xef,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xf2,0x01,0x00,0x00,0xbd,0x02,0x00,0x00,0x62,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xf3,0x01,0x00,0x00, -0xf0,0x01,0x00,0x00,0xf2,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xf5,0x01,0x00,0x00,0xf3,0x01,0x00,0x00, -0xc1,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xf9,0x01,0x00,0x00,0xf2,0x01,0x00,0x00,0xc1,0x02,0x00,0x00, -0x41,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0xfa,0x01,0x00,0x00, -0x83,0x01,0x00,0x00,0xf9,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, -0xb9,0x00,0x00,0x00,0xfb,0x01,0x00,0x00,0xfa,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0x00,0x02,0x00,0x00, -0xb0,0x01,0x00,0x00,0xee,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, -0xb9,0x00,0x00,0x00,0x01,0x02,0x00,0x00,0x00,0x02,0x00,0x00, -0x41,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0x03,0x02,0x00,0x00, -0xbf,0x00,0x00,0x00,0xf5,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, -0xb9,0x00,0x00,0x00,0x04,0x02,0x00,0x00,0x03,0x02,0x00,0x00, -0x0c,0x00,0x08,0x00,0xb9,0x00,0x00,0x00,0x05,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0xfb,0x01,0x00,0x00, -0x01,0x02,0x00,0x00,0x04,0x02,0x00,0x00,0x3e,0x00,0x03,0x00, -0x03,0x02,0x00,0x00,0x05,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x08,0x02,0x00,0x00,0xc1,0x02,0x00,0x00, -0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xe4,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xe6,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0xdf,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xdf,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x0a,0x02,0x00,0x00, -0xbf,0x02,0x00,0x00,0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xdc,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xde,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0xd7,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xd7,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x0c,0x02,0x00,0x00,0xbd,0x02,0x00,0x00,0xc5,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xd4,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xd6,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xcf,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xcf,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x0e,0x02,0x00,0x00,0xb9,0x02,0x00,0x00, -0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xcc,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xce,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0x6c,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x6c,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x10,0x02,0x00,0x00, -0xb3,0x02,0x00,0x00,0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x69,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x6b,0x01,0x00,0x00, -0xe0,0x00,0x04,0x00,0x0c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x61,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xcc,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xcc,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x12,0x02,0x00,0x00,0x99,0x02,0x00,0x00, -0x6c,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xc9,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xcb,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x17,0x02,0x00,0x00,0x55,0x00,0x00,0x00, -0x53,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x18,0x02,0x00,0x00,0x8b,0x00,0x00,0x00,0x17,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x1d,0x02,0x00,0x00, -0x59,0x00,0x00,0x00,0xae,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x1e,0x02,0x00,0x00,0x9d,0x00,0x00,0x00, -0x1d,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0x23,0x02,0x00,0x00,0x12,0x00,0x00,0x00,0x22,0x02,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x24,0x02,0x00,0x00, -0x23,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x25,0x02,0x00,0x00,0x0f,0x00,0x00,0x00,0x24,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x29,0x02,0x00,0x00, -0x47,0x00,0x00,0x00,0x24,0x02,0x00,0x00,0x41,0x00,0x05,0x00, -0x0d,0x00,0x00,0x00,0x2b,0x02,0x00,0x00,0x2a,0x02,0x00,0x00, -0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x2c,0x02,0x00,0x00,0x2b,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x2d,0x02,0x00,0x00,0x29,0x02,0x00,0x00, -0x2c,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x2e,0x02,0x00,0x00,0x25,0x02,0x00,0x00,0x2d,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0x30,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x30,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0x9a,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0xcb,0x00,0x00,0x00, -0x97,0x02,0x00,0x00,0x33,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb7,0x00,0x00,0x00,0x36,0x02,0x00,0x00,0x9a,0x02,0x00,0x00, -0xb4,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x32,0x02,0x00,0x00, -0x33,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x36,0x02,0x00,0x00,0x31,0x02,0x00,0x00,0x32,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x31,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x38,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x38,0x02,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x9b,0x02,0x00,0x00, -0x3e,0x00,0x00,0x00,0x31,0x02,0x00,0x00,0x95,0x02,0x00,0x00, -0x3b,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00, -0x3e,0x02,0x00,0x00,0x9b,0x02,0x00,0x00,0x60,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0x3a,0x02,0x00,0x00,0x3b,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x3e,0x02,0x00,0x00, -0x39,0x02,0x00,0x00,0x3a,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x39,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x42,0x02,0x00,0x00,0x9b,0x02,0x00,0x00,0x61,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x43,0x02,0x00,0x00, -0x18,0x02,0x00,0x00,0x42,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x45,0x02,0x00,0x00,0x64,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x46,0x02,0x00,0x00,0x43,0x02,0x00,0x00,0x45,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x4a,0x02,0x00,0x00, -0x9a,0x02,0x00,0x00,0xb8,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x4b,0x02,0x00,0x00,0x1e,0x02,0x00,0x00, -0x4a,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x4d,0x02,0x00,0x00,0x68,0x00,0x00,0x00,0xb1,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x4e,0x02,0x00,0x00, -0x4b,0x02,0x00,0x00,0x4d,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x50,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x50,0x02,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x9d,0x02,0x00,0x00, -0x3e,0x00,0x00,0x00,0x39,0x02,0x00,0x00,0x93,0x02,0x00,0x00, -0x53,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00, -0x56,0x02,0x00,0x00,0x9d,0x02,0x00,0x00,0xb1,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0x52,0x02,0x00,0x00,0x53,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x56,0x02,0x00,0x00, -0x51,0x02,0x00,0x00,0x52,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x51,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x58,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x58,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0x9f,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, -0x51,0x02,0x00,0x00,0x91,0x02,0x00,0x00,0x5b,0x02,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00,0x5e,0x02,0x00,0x00, -0x9f,0x02,0x00,0x00,0x62,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0x5a,0x02,0x00,0x00,0x5b,0x02,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x5e,0x02,0x00,0x00,0x59,0x02,0x00,0x00, -0x5a,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x59,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x61,0x02,0x00,0x00, -0x46,0x02,0x00,0x00,0x9f,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb7,0x00,0x00,0x00,0x64,0x02,0x00,0x00,0x61,0x02,0x00,0x00, -0x36,0x00,0x00,0x00,0xf7,0x00,0x03,0x00,0x66,0x02,0x00,0x00, -0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x64,0x02,0x00,0x00, -0x65,0x02,0x00,0x00,0x66,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x65,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x69,0x02,0x00,0x00,0x4e,0x02,0x00,0x00,0x9d,0x02,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x6a,0x02,0x00,0x00, -0x12,0x00,0x00,0x00,0xc5,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x6b,0x02,0x00,0x00,0x6a,0x02,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00,0x6c,0x02,0x00,0x00, -0x69,0x02,0x00,0x00,0x6b,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x66,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x66,0x02,0x00,0x00, -0xf5,0x00,0x07,0x00,0xb7,0x00,0x00,0x00,0x6d,0x02,0x00,0x00, -0x64,0x02,0x00,0x00,0x59,0x02,0x00,0x00,0x6c,0x02,0x00,0x00, -0x65,0x02,0x00,0x00,0xf7,0x00,0x03,0x00,0x6f,0x02,0x00,0x00, -0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x6d,0x02,0x00,0x00, -0x6e,0x02,0x00,0x00,0x6f,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x6e,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x77,0x02,0x00,0x00,0x4e,0x02,0x00,0x00,0x9d,0x02,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x79,0x02,0x00,0x00, -0x12,0x00,0x00,0x00,0x78,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x7a,0x02,0x00,0x00,0x79,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x7b,0x02,0x00,0x00, -0x77,0x02,0x00,0x00,0x7a,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x7c,0x02,0x00,0x00,0x2e,0x02,0x00,0x00, -0x7b,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x7e,0x02,0x00,0x00,0x7c,0x02,0x00,0x00,0x46,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x80,0x02,0x00,0x00, -0x7e,0x02,0x00,0x00,0x9f,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x82,0x02,0x00,0x00,0x9a,0x02,0x00,0x00, -0xb1,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x84,0x02,0x00,0x00,0x82,0x02,0x00,0x00,0x9d,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x86,0x02,0x00,0x00, -0x84,0x02,0x00,0x00,0x85,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x88,0x02,0x00,0x00,0x9b,0x02,0x00,0x00, -0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x89,0x02,0x00,0x00,0x86,0x02,0x00,0x00,0x88,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8b,0x02,0x00,0x00, -0x89,0x02,0x00,0x00,0x9f,0x02,0x00,0x00,0x41,0x00,0x05,0x00, -0xc2,0x00,0x00,0x00,0x8c,0x02,0x00,0x00,0xbf,0x00,0x00,0x00, -0x8b,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0xb9,0x00,0x00,0x00, -0x8d,0x02,0x00,0x00,0x8c,0x02,0x00,0x00,0x41,0x00,0x06,0x00, -0x8e,0x02,0x00,0x00,0x8f,0x02,0x00,0x00,0x73,0x02,0x00,0x00, -0x34,0x00,0x00,0x00,0x80,0x02,0x00,0x00,0x3e,0x00,0x03,0x00, -0x8f,0x02,0x00,0x00,0x8d,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x6f,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x6f,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0x5b,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x5b,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x91,0x02,0x00,0x00,0x9f,0x02,0x00,0x00,0xc5,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x58,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x5a,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x53,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x53,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x93,0x02,0x00,0x00,0x9d,0x02,0x00,0x00, -0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x50,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x52,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x3b,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x3b,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x95,0x02,0x00,0x00, -0x9b,0x02,0x00,0x00,0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x38,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x3a,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0x33,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x33,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x97,0x02,0x00,0x00,0x9a,0x02,0x00,0x00,0xc5,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x30,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x32,0x02,0x00,0x00,0xfd,0x00,0x01,0x00,0x38,0x00,0x01,0x00, - -}; -const uint64_t matmul_f16_m_fp32_len = 10116; - -unsigned char matmul_f16_s_data[] = { -0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, -0xcc,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, -0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x09,0x00,0x00,0x00, -0x11,0x00,0x02,0x00,0x51,0x11,0x00,0x00,0x0b,0x00,0x06,0x00, -0x01,0x00,0x00,0x00,0x47,0x4c,0x53,0x4c,0x2e,0x73,0x74,0x64, -0x2e,0x34,0x35,0x30,0x00,0x00,0x00,0x00,0x0e,0x00,0x03,0x00, -0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x0f,0x00,0x0f,0x00, -0x05,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x6d,0x61,0x69,0x6e, -0x00,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x3d,0x00,0x00,0x00,0x4c,0x00,0x00,0x00,0xf1,0x00,0x00,0x00, -0xfc,0x00,0x00,0x00,0x3c,0x01,0x00,0x00,0x47,0x01,0x00,0x00, -0x2c,0x02,0x00,0x00,0x75,0x02,0x00,0x00,0x10,0x00,0x06,0x00, -0x04,0x00,0x00,0x00,0x11,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x0b,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x1c,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x10,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x08,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x03,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x10,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x10,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x05,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x14,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x10,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x1c,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x08,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x24,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x10,0x00,0x00,0x00,0x0a,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x28,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x0b,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x2c,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x30,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x10,0x00,0x00,0x00,0x0d,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x34,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x10,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x37,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x3d,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x1a,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x4c,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x1b,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x4f,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x53,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x60,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x62,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x6c,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x03,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x9c,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xae,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0xb1,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x08,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0xf9,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0xfa,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0xfa,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0xfa,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xfc,0x00,0x00,0x00, -0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0xfc,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x16,0x01,0x00,0x00,0x01,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x17,0x01,0x00,0x00, -0x0b,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x44,0x01,0x00,0x00,0x06,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x48,0x00,0x04,0x00,0x45,0x01,0x00,0x00,0x00,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x45,0x01,0x00,0x00, -0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x03,0x00,0x45,0x01,0x00,0x00,0x02,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x47,0x01,0x00,0x00,0x22,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x47,0x01,0x00,0x00, -0x21,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x2c,0x02,0x00,0x00,0x0b,0x00,0x00,0x00,0x18,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x72,0x02,0x00,0x00,0x06,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0x73,0x02,0x00,0x00, -0x00,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x73,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x73,0x02,0x00,0x00, -0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x75,0x02,0x00,0x00, -0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x75,0x02,0x00,0x00,0x21,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x13,0x00,0x02,0x00,0x02,0x00,0x00,0x00,0x21,0x00,0x03,0x00, -0x03,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x15,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x17,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x03,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x0a,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x0a,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x0d,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x1e,0x00,0x10,0x00, -0x10,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x11,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x10,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x11,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x15,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x14,0x00,0x00,0x00, -0x08,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x15,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x0a,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x27,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, -0x2d,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x37,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x0a,0x00,0x00,0x00,0x3d,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x3e,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00, -0x4c,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x53,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x54,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x37,0x00,0x00,0x00, -0x53,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x58,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x37,0x00,0x00,0x00, -0x53,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x60,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x61,0x00,0x00,0x00,0x86,0x00,0x00,0x00, -0x53,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x32,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x63,0x00,0x00,0x00, -0x86,0x00,0x00,0x00,0x61,0x00,0x00,0x00,0x62,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x67,0x00,0x00,0x00, -0x86,0x00,0x00,0x00,0x61,0x00,0x00,0x00,0x62,0x00,0x00,0x00, -0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x10,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x6d,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x72,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, -0x76,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0x7b,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x86,0x00,0x00,0x00, -0x0b,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, -0x8c,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0x97,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x9c,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, -0x9e,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xad,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x32,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xae,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xaf,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0xae,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xb0,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x62,0x00,0x00,0x00, -0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xb1,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xb2,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0xb0,0x00,0x00,0x00, -0xb1,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xb3,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0xb2,0x00,0x00,0x00, -0x60,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xb4,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0xaf,0x00,0x00,0x00, -0xb3,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xb5,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0xad,0x00,0x00,0x00, -0xb4,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xb6,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0xb5,0x00,0x00,0x00, -0xb1,0x00,0x00,0x00,0x14,0x00,0x02,0x00,0xb7,0x00,0x00,0x00, -0x16,0x00,0x03,0x00,0xb9,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xba,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xbb,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0xba,0x00,0x00,0x00,0xb4,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xbc,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0xbb,0x00,0x00,0x00,0xb1,0x00,0x00,0x00, -0x1c,0x00,0x04,0x00,0xbd,0x00,0x00,0x00,0xb9,0x00,0x00,0x00, -0xbc,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0xbe,0x00,0x00,0x00, -0x07,0x00,0x00,0x00,0xbd,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0xb9,0x00,0x00,0x00,0xc1,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0xc2,0x00,0x00,0x00,0x07,0x00,0x00,0x00, -0xb9,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, -0xc5,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x16,0x00,0x03,0x00, -0xec,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xed,0x00,0x00,0x00,0x80,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xee,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0x37,0x00,0x00,0x00,0xed,0x00,0x00,0x00,0x1c,0x00,0x04,0x00, -0xef,0x00,0x00,0x00,0xec,0x00,0x00,0x00,0xee,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0xf0,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0xef,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0xf0,0x00,0x00,0x00, -0xf1,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xf5,0x00,0x00,0x00,0x80,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, -0xf9,0x00,0x00,0x00,0xec,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, -0xfa,0x00,0x00,0x00,0xf9,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0xfb,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0xfa,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0xfb,0x00,0x00,0x00,0xfc,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x07,0x01,0x00,0x00, -0x0c,0x00,0x00,0x00,0xec,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x0a,0x01,0x00,0x00,0x04,0x00,0x00,0x00,0xec,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x10,0x01,0x00,0x00, -0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0xec,0x00,0x00,0x00,0x14,0x01,0x00,0x00, -0x00,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x16,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x33,0x00,0x06,0x00, -0x09,0x00,0x00,0x00,0x17,0x01,0x00,0x00,0x16,0x01,0x00,0x00, -0x39,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x18,0x01,0x00,0x00,0x51,0x00,0x00,0x00, -0x17,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x19,0x01,0x00,0x00,0x84,0x00,0x00,0x00, -0x18,0x01,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x1a,0x01,0x00,0x00,0x86,0x00,0x00,0x00, -0x19,0x01,0x00,0x00,0x6c,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x38,0x01,0x00,0x00,0x80,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x39,0x01,0x00,0x00,0x84,0x00,0x00,0x00, -0x9c,0x00,0x00,0x00,0x38,0x01,0x00,0x00,0x1c,0x00,0x04,0x00, -0x3a,0x01,0x00,0x00,0xec,0x00,0x00,0x00,0x39,0x01,0x00,0x00, -0x20,0x00,0x04,0x00,0x3b,0x01,0x00,0x00,0x04,0x00,0x00,0x00, -0x3a,0x01,0x00,0x00,0x3b,0x00,0x04,0x00,0x3b,0x01,0x00,0x00, -0x3c,0x01,0x00,0x00,0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x40,0x01,0x00,0x00,0x80,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, -0x44,0x01,0x00,0x00,0xec,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, -0x45,0x01,0x00,0x00,0x44,0x01,0x00,0x00,0x20,0x00,0x04,0x00, -0x46,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x45,0x01,0x00,0x00, -0x3b,0x00,0x04,0x00,0x46,0x01,0x00,0x00,0x47,0x01,0x00,0x00, -0x0c,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x59,0x01,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x60,0x01,0x00,0x00,0x08,0x01,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x61,0x01,0x00,0x00,0x86,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x64,0x01,0x00,0x00,0x86,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x7f,0x01,0x00,0x00,0x84,0x00,0x00,0x00, -0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x1c,0x00,0x04,0x00, -0x80,0x01,0x00,0x00,0xec,0x00,0x00,0x00,0x7f,0x01,0x00,0x00, -0x20,0x00,0x04,0x00,0x81,0x01,0x00,0x00,0x07,0x00,0x00,0x00, -0x80,0x01,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x91,0x01,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x97,0x01,0x00,0x00, -0x07,0x00,0x00,0x00,0xec,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xad,0x01,0x00,0x00,0x84,0x00,0x00,0x00, -0xb4,0x00,0x00,0x00,0xb1,0x00,0x00,0x00,0x1c,0x00,0x04,0x00, -0xae,0x01,0x00,0x00,0xec,0x00,0x00,0x00,0xad,0x01,0x00,0x00, -0x20,0x00,0x04,0x00,0xaf,0x01,0x00,0x00,0x07,0x00,0x00,0x00, -0xae,0x01,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xb8,0x01,0x00,0x00,0x86,0x00,0x00,0x00,0xae,0x00,0x00,0x00, -0xb4,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xc0,0x01,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xef,0x01,0x00,0x00,0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, -0x24,0x02,0x00,0x00,0x0d,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x0a,0x00,0x00,0x00,0x2c,0x02,0x00,0x00,0x01,0x00,0x00,0x00, -0x1d,0x00,0x03,0x00,0x72,0x02,0x00,0x00,0xb9,0x00,0x00,0x00, -0x1e,0x00,0x03,0x00,0x73,0x02,0x00,0x00,0x72,0x02,0x00,0x00, -0x20,0x00,0x04,0x00,0x74,0x02,0x00,0x00,0x0c,0x00,0x00,0x00, -0x73,0x02,0x00,0x00,0x3b,0x00,0x04,0x00,0x74,0x02,0x00,0x00, -0x75,0x02,0x00,0x00,0x0c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0x7a,0x02,0x00,0x00,0x05,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x87,0x02,0x00,0x00, -0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x90,0x02,0x00,0x00,0x0c,0x00,0x00,0x00, -0xb9,0x00,0x00,0x00,0x36,0x00,0x05,0x00,0x02,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x03,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0x05,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0xbe,0x00,0x00,0x00,0xbf,0x00,0x00,0x00,0x07,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x81,0x01,0x00,0x00,0x82,0x01,0x00,0x00, -0x07,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0xaf,0x01,0x00,0x00, -0xb0,0x01,0x00,0x00,0x07,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x0d,0x00,0x00,0x00,0x0e,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x0f,0x00,0x00,0x00,0x0e,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x14,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x17,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, -0x17,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x1e,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x17,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x22,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x22,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x24,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x27,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x29,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x2a,0x00,0x00,0x00,0x1e,0x00,0x00,0x00, -0x29,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x2d,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x2f,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x30,0x00,0x00,0x00,0x24,0x00,0x00,0x00,0x2f,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x32,0x00,0x00,0x00, -0x30,0x00,0x00,0x00,0x2a,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x35,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x34,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x36,0x00,0x00,0x00,0x35,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x38,0x00,0x00,0x00,0x36,0x00,0x00,0x00, -0x37,0x00,0x00,0x00,0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x3a,0x00,0x00,0x00,0x38,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3b,0x00,0x00,0x00, -0x3a,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x0d,0x00,0x00,0x00,0x3f,0x00,0x00,0x00,0x3d,0x00,0x00,0x00, -0x3e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0x3f,0x00,0x00,0x00,0x89,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x42,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x3b,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x47,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x3b,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00,0x49,0x00,0x00,0x00, -0x3d,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x4a,0x00,0x00,0x00,0x49,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00,0x4d,0x00,0x00,0x00, -0x4c,0x00,0x00,0x00,0x3e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x4d,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x50,0x00,0x00,0x00, -0x4e,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x89,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x55,0x00,0x00,0x00,0x50,0x00,0x00,0x00, -0x54,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x59,0x00,0x00,0x00,0x50,0x00,0x00,0x00,0x58,0x00,0x00,0x00, -0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x5d,0x00,0x00,0x00, -0x4e,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x89,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x64,0x00,0x00,0x00,0x5d,0x00,0x00,0x00, -0x63,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x68,0x00,0x00,0x00,0x5d,0x00,0x00,0x00,0x67,0x00,0x00,0x00, -0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x6e,0x00,0x00,0x00, -0x4e,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x73,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, -0x72,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0x77,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x76,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x78,0x00,0x00,0x00, -0x77,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x79,0x00,0x00,0x00,0x47,0x00,0x00,0x00,0x78,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x7c,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x7b,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x7d,0x00,0x00,0x00,0x7c,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x7f,0x00,0x00,0x00, -0x47,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x82,0x00,0x00,0x00,0x7f,0x00,0x00,0x00, -0x78,0x00,0x00,0x00,0x0c,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0x83,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x26,0x00,0x00,0x00, -0x7d,0x00,0x00,0x00,0x82,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x87,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x86,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x88,0x00,0x00,0x00,0x87,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x89,0x00,0x00,0x00,0x32,0x00,0x00,0x00, -0x88,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x8b,0x00,0x00,0x00,0x42,0x00,0x00,0x00,0x37,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x8d,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x8c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x8e,0x00,0x00,0x00,0x8d,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8f,0x00,0x00,0x00, -0x8b,0x00,0x00,0x00,0x8e,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x90,0x00,0x00,0x00,0x89,0x00,0x00,0x00, -0x8f,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x92,0x00,0x00,0x00,0x90,0x00,0x00,0x00,0x79,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x93,0x00,0x00,0x00, -0x92,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x98,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x97,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x99,0x00,0x00,0x00,0x98,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x9a,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, -0x99,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x9d,0x00,0x00,0x00,0x4a,0x00,0x00,0x00,0x9c,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x9f,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x9e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xa0,0x00,0x00,0x00,0x9f,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa1,0x00,0x00,0x00, -0x9d,0x00,0x00,0x00,0xa0,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xa2,0x00,0x00,0x00,0x9a,0x00,0x00,0x00, -0xa1,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xa4,0x00,0x00,0x00,0xa2,0x00,0x00,0x00,0x79,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa5,0x00,0x00,0x00, -0xa4,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xa7,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xa7,0x00,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x9a,0x02,0x00,0x00, -0x3e,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0xc6,0x00,0x00,0x00, -0xa8,0x00,0x00,0x00,0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00, -0xb8,0x00,0x00,0x00,0x9a,0x02,0x00,0x00,0xb6,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0xa9,0x00,0x00,0x00,0xa8,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xb8,0x00,0x00,0x00, -0xa8,0x00,0x00,0x00,0xa9,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xa8,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, -0xc3,0x00,0x00,0x00,0xbf,0x00,0x00,0x00,0x9a,0x02,0x00,0x00, -0x3e,0x00,0x03,0x00,0xc3,0x00,0x00,0x00,0xc1,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xc6,0x00,0x00,0x00, -0x9a,0x02,0x00,0x00,0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xa7,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xa9,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xc9,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xc9,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xb3,0x02,0x00,0x00,0xa5,0x00,0x00,0x00,0xa9,0x00,0x00,0x00, -0x66,0x01,0x00,0x00,0xcc,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xaf,0x02,0x00,0x00,0x93,0x00,0x00,0x00, -0xa9,0x00,0x00,0x00,0x63,0x01,0x00,0x00,0xcc,0x00,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x9b,0x02,0x00,0x00, -0x79,0x00,0x00,0x00,0xa9,0x00,0x00,0x00,0x14,0x02,0x00,0x00, -0xcc,0x00,0x00,0x00,0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00, -0xd0,0x00,0x00,0x00,0x9b,0x02,0x00,0x00,0x83,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0xcb,0x00,0x00,0x00,0xcc,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xd0,0x00,0x00,0x00, -0xca,0x00,0x00,0x00,0xcb,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xca,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xd2,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xd2,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xab,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, -0xca,0x00,0x00,0x00,0x1c,0x01,0x00,0x00,0xd5,0x00,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00,0xd8,0x00,0x00,0x00, -0xab,0x02,0x00,0x00,0x37,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0xd4,0x00,0x00,0x00,0xd5,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xd8,0x00,0x00,0x00,0xd3,0x00,0x00,0x00, -0xd4,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xd3,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xdc,0x00,0x00,0x00, -0x8b,0x00,0x00,0x00,0x73,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xde,0x00,0x00,0x00,0xdc,0x00,0x00,0x00, -0xab,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00, -0xe1,0x00,0x00,0x00,0xde,0x00,0x00,0x00,0x36,0x00,0x00,0x00, -0xf7,0x00,0x03,0x00,0xe3,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xe1,0x00,0x00,0x00,0xe2,0x00,0x00,0x00, -0xe3,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xe2,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xe6,0x00,0x00,0x00, -0x9b,0x02,0x00,0x00,0x6e,0x00,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb7,0x00,0x00,0x00,0xe8,0x00,0x00,0x00,0xe6,0x00,0x00,0x00, -0x83,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xe3,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xe3,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, -0xb7,0x00,0x00,0x00,0xe9,0x00,0x00,0x00,0xe1,0x00,0x00,0x00, -0xd3,0x00,0x00,0x00,0xe8,0x00,0x00,0x00,0xe2,0x00,0x00,0x00, -0xf7,0x00,0x03,0x00,0xeb,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xe9,0x00,0x00,0x00,0xea,0x00,0x00,0x00, -0x0c,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xea,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xf4,0x00,0x00,0x00, -0x73,0x00,0x00,0x00,0xab,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xf6,0x00,0x00,0x00,0xf4,0x00,0x00,0x00, -0xf5,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xf8,0x00,0x00,0x00,0xf6,0x00,0x00,0x00,0x6e,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x03,0x01,0x00,0x00, -0xf4,0x00,0x00,0x00,0x8e,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x04,0x01,0x00,0x00,0xaf,0x02,0x00,0x00, -0x03,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x06,0x01,0x00,0x00,0x04,0x01,0x00,0x00,0x6e,0x00,0x00,0x00, -0x41,0x00,0x06,0x00,0x07,0x01,0x00,0x00,0x08,0x01,0x00,0x00, -0xfc,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0x06,0x01,0x00,0x00, -0x3d,0x00,0x04,0x00,0xec,0x00,0x00,0x00,0x09,0x01,0x00,0x00, -0x08,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0x0a,0x01,0x00,0x00, -0x0b,0x01,0x00,0x00,0xf1,0x00,0x00,0x00,0xf8,0x00,0x00,0x00, -0x3e,0x00,0x03,0x00,0x0b,0x01,0x00,0x00,0x09,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0xeb,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x0c,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x0f,0x01,0x00,0x00,0x73,0x00,0x00,0x00,0xab,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x11,0x01,0x00,0x00, -0x0f,0x01,0x00,0x00,0x10,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x13,0x01,0x00,0x00,0x11,0x01,0x00,0x00, -0x6e,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x0a,0x01,0x00,0x00, -0x15,0x01,0x00,0x00,0xf1,0x00,0x00,0x00,0x13,0x01,0x00,0x00, -0x3e,0x00,0x03,0x00,0x15,0x01,0x00,0x00,0x14,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0xeb,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xeb,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xd5,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xd5,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x1c,0x01,0x00,0x00,0xab,0x02,0x00,0x00, -0x1a,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xd2,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xd4,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x1e,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x1e,0x01,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xac,0x02,0x00,0x00, -0x3e,0x00,0x00,0x00,0xd4,0x00,0x00,0x00,0x5f,0x01,0x00,0x00, -0x21,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00, -0x24,0x01,0x00,0x00,0xac,0x02,0x00,0x00,0x9c,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0x20,0x01,0x00,0x00,0x21,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x24,0x01,0x00,0x00, -0x1f,0x01,0x00,0x00,0x20,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x1f,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x28,0x01,0x00,0x00,0x9d,0x00,0x00,0x00,0x73,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x2a,0x01,0x00,0x00, -0x28,0x01,0x00,0x00,0xac,0x02,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x2b,0x01,0x00,0x00,0x12,0x00,0x00,0x00, -0xc5,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x2c,0x01,0x00,0x00,0x2b,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb7,0x00,0x00,0x00,0x2d,0x01,0x00,0x00,0x2a,0x01,0x00,0x00, -0x2c,0x01,0x00,0x00,0xf7,0x00,0x03,0x00,0x2f,0x01,0x00,0x00, -0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x2d,0x01,0x00,0x00, -0x2e,0x01,0x00,0x00,0x2f,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x2e,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x32,0x01,0x00,0x00,0x9b,0x02,0x00,0x00,0x6e,0x00,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00,0x34,0x01,0x00,0x00, -0x32,0x01,0x00,0x00,0x83,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x2f,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x2f,0x01,0x00,0x00, -0xf5,0x00,0x07,0x00,0xb7,0x00,0x00,0x00,0x35,0x01,0x00,0x00, -0x2d,0x01,0x00,0x00,0x1f,0x01,0x00,0x00,0x34,0x01,0x00,0x00, -0x2e,0x01,0x00,0x00,0xf7,0x00,0x03,0x00,0x37,0x01,0x00,0x00, -0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x35,0x01,0x00,0x00, -0x36,0x01,0x00,0x00,0x55,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x36,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x3f,0x01,0x00,0x00,0x73,0x00,0x00,0x00,0xac,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x41,0x01,0x00,0x00, -0x3f,0x01,0x00,0x00,0x40,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x43,0x01,0x00,0x00,0x41,0x01,0x00,0x00, -0x6e,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x4e,0x01,0x00,0x00,0x3f,0x01,0x00,0x00,0xa0,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x4f,0x01,0x00,0x00, -0xb3,0x02,0x00,0x00,0x4e,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x51,0x01,0x00,0x00,0x4f,0x01,0x00,0x00, -0x6e,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x07,0x01,0x00,0x00, -0x52,0x01,0x00,0x00,0x47,0x01,0x00,0x00,0x34,0x00,0x00,0x00, -0x51,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0xec,0x00,0x00,0x00, -0x53,0x01,0x00,0x00,0x52,0x01,0x00,0x00,0x41,0x00,0x05,0x00, -0x0a,0x01,0x00,0x00,0x54,0x01,0x00,0x00,0x3c,0x01,0x00,0x00, -0x43,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x54,0x01,0x00,0x00, -0x53,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x37,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x55,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x58,0x01,0x00,0x00,0x73,0x00,0x00,0x00, -0xac,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x5a,0x01,0x00,0x00,0x58,0x01,0x00,0x00,0x59,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x5c,0x01,0x00,0x00, -0x5a,0x01,0x00,0x00,0x6e,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x0a,0x01,0x00,0x00,0x5d,0x01,0x00,0x00,0x3c,0x01,0x00,0x00, -0x5c,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x5d,0x01,0x00,0x00, -0x14,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x37,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x37,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0x21,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x21,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x5f,0x01,0x00,0x00, -0xac,0x02,0x00,0x00,0x1a,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0x1e,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x20,0x01,0x00,0x00, -0xe0,0x00,0x04,0x00,0x0c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x60,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x63,0x01,0x00,0x00,0xaf,0x02,0x00,0x00,0x61,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x66,0x01,0x00,0x00, -0xb3,0x02,0x00,0x00,0x64,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0x68,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x68,0x01,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xb5,0x02,0x00,0x00, -0x3e,0x00,0x00,0x00,0x20,0x01,0x00,0x00,0x12,0x02,0x00,0x00, -0x6b,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00, -0x6e,0x01,0x00,0x00,0xb5,0x02,0x00,0x00,0x6c,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0x6a,0x01,0x00,0x00,0x6b,0x01,0x00,0x00, -0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x6e,0x01,0x00,0x00, -0x69,0x01,0x00,0x00,0x6a,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x69,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x70,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x70,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xb9,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, -0x69,0x01,0x00,0x00,0x9c,0x01,0x00,0x00,0x73,0x01,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00,0x76,0x01,0x00,0x00, -0xb9,0x02,0x00,0x00,0x60,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0x72,0x01,0x00,0x00,0x73,0x01,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x76,0x01,0x00,0x00,0x71,0x01,0x00,0x00, -0x72,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x71,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0x78,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x78,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xcb,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0x71,0x01,0x00,0x00, -0x9a,0x01,0x00,0x00,0x79,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb7,0x00,0x00,0x00,0x7e,0x01,0x00,0x00,0xcb,0x02,0x00,0x00, -0x62,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x7a,0x01,0x00,0x00, -0x79,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x7e,0x01,0x00,0x00,0x79,0x01,0x00,0x00,0x7a,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x79,0x01,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x84,0x01,0x00,0x00,0xb9,0x02,0x00,0x00, -0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x86,0x01,0x00,0x00,0x84,0x01,0x00,0x00,0xcb,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x88,0x01,0x00,0x00, -0x55,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x8a,0x01,0x00,0x00,0xb9,0x02,0x00,0x00, -0x61,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x8b,0x01,0x00,0x00,0x88,0x01,0x00,0x00,0x8a,0x01,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8d,0x01,0x00,0x00, -0x64,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x8e,0x01,0x00,0x00,0x8b,0x01,0x00,0x00, -0x8d,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x90,0x01,0x00,0x00,0x8e,0x01,0x00,0x00,0xcb,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x92,0x01,0x00,0x00, -0x90,0x01,0x00,0x00,0x91,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x94,0x01,0x00,0x00,0x92,0x01,0x00,0x00, -0xb5,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x0a,0x01,0x00,0x00, -0x95,0x01,0x00,0x00,0xf1,0x00,0x00,0x00,0x94,0x01,0x00,0x00, -0x3d,0x00,0x04,0x00,0xec,0x00,0x00,0x00,0x96,0x01,0x00,0x00, -0x95,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0x97,0x01,0x00,0x00, -0x98,0x01,0x00,0x00,0x82,0x01,0x00,0x00,0x86,0x01,0x00,0x00, -0x3e,0x00,0x03,0x00,0x98,0x01,0x00,0x00,0x96,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9a,0x01,0x00,0x00, -0xcb,0x02,0x00,0x00,0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x78,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x7a,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0x73,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x73,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x9c,0x01,0x00,0x00,0xb9,0x02,0x00,0x00,0xc5,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x70,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x72,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x9e,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x9e,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xba,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, -0x72,0x01,0x00,0x00,0xca,0x01,0x00,0x00,0xa1,0x01,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00,0xa4,0x01,0x00,0x00, -0xba,0x02,0x00,0x00,0xb4,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0xa0,0x01,0x00,0x00,0xa1,0x01,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xa4,0x01,0x00,0x00,0x9f,0x01,0x00,0x00, -0xa0,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x9f,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0xa6,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xa6,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xc8,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0x9f,0x01,0x00,0x00, -0xc8,0x01,0x00,0x00,0xa7,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb7,0x00,0x00,0x00,0xac,0x01,0x00,0x00,0xc8,0x02,0x00,0x00, -0xb1,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xa8,0x01,0x00,0x00, -0xa7,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0xac,0x01,0x00,0x00,0xa7,0x01,0x00,0x00,0xa8,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xa7,0x01,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xb2,0x01,0x00,0x00,0xba,0x02,0x00,0x00, -0xb1,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xb4,0x01,0x00,0x00,0xb2,0x01,0x00,0x00,0xc8,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xb6,0x01,0x00,0x00, -0x59,0x00,0x00,0x00,0xae,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xb9,0x01,0x00,0x00,0xba,0x02,0x00,0x00, -0xb8,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xba,0x01,0x00,0x00,0xb6,0x01,0x00,0x00,0xb9,0x01,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xbc,0x01,0x00,0x00, -0x68,0x00,0x00,0x00,0xb1,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xbd,0x01,0x00,0x00,0xba,0x01,0x00,0x00, -0xbc,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xbf,0x01,0x00,0x00,0xbd,0x01,0x00,0x00,0xc8,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xc1,0x01,0x00,0x00, -0xbf,0x01,0x00,0x00,0xc0,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xc3,0x01,0x00,0x00,0xc1,0x01,0x00,0x00, -0xb5,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x0a,0x01,0x00,0x00, -0xc4,0x01,0x00,0x00,0x3c,0x01,0x00,0x00,0xc3,0x01,0x00,0x00, -0x3d,0x00,0x04,0x00,0xec,0x00,0x00,0x00,0xc5,0x01,0x00,0x00, -0xc4,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0x97,0x01,0x00,0x00, -0xc6,0x01,0x00,0x00,0xb0,0x01,0x00,0x00,0xb4,0x01,0x00,0x00, -0x3e,0x00,0x03,0x00,0xc6,0x01,0x00,0x00,0xc5,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xc8,0x01,0x00,0x00, -0xc8,0x02,0x00,0x00,0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xa6,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xa8,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0xa1,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xa1,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xca,0x01,0x00,0x00,0xba,0x02,0x00,0x00,0xc5,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x9e,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xa0,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xcc,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xcc,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xbb,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, -0xa0,0x01,0x00,0x00,0x10,0x02,0x00,0x00,0xcf,0x01,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00,0xd2,0x01,0x00,0x00, -0xbb,0x02,0x00,0x00,0xb4,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0xce,0x01,0x00,0x00,0xcf,0x01,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xd2,0x01,0x00,0x00,0xcd,0x01,0x00,0x00, -0xce,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xcd,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0xd4,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xd4,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xbf,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0xcd,0x01,0x00,0x00, -0x0e,0x02,0x00,0x00,0xd7,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb7,0x00,0x00,0x00,0xda,0x01,0x00,0x00,0xbf,0x02,0x00,0x00, -0x60,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xd6,0x01,0x00,0x00, -0xd7,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0xda,0x01,0x00,0x00,0xd5,0x01,0x00,0x00,0xd6,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xd5,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0xdc,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xdc,0x01,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xc1,0x02,0x00,0x00, -0x3e,0x00,0x00,0x00,0xd5,0x01,0x00,0x00,0x0c,0x02,0x00,0x00, -0xdf,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00, -0xe2,0x01,0x00,0x00,0xc1,0x02,0x00,0x00,0xb1,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0xde,0x01,0x00,0x00,0xdf,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xe2,0x01,0x00,0x00, -0xdd,0x01,0x00,0x00,0xde,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xdd,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xe4,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xe4,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xc3,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, -0xdd,0x01,0x00,0x00,0x0a,0x02,0x00,0x00,0xe5,0x01,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00,0xea,0x01,0x00,0x00, -0xc3,0x02,0x00,0x00,0x62,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0xe6,0x01,0x00,0x00,0xe5,0x01,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xea,0x01,0x00,0x00,0xe5,0x01,0x00,0x00, -0xe6,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xe5,0x01,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xec,0x01,0x00,0x00, -0xbb,0x02,0x00,0x00,0xb1,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xee,0x01,0x00,0x00,0xec,0x01,0x00,0x00, -0xc1,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xf0,0x01,0x00,0x00,0xee,0x01,0x00,0x00,0xef,0x01,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xf2,0x01,0x00,0x00, -0xbf,0x02,0x00,0x00,0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xf3,0x01,0x00,0x00,0xf0,0x01,0x00,0x00, -0xf2,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xf5,0x01,0x00,0x00,0xf3,0x01,0x00,0x00,0xc3,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xf9,0x01,0x00,0x00, -0xf2,0x01,0x00,0x00,0xc3,0x02,0x00,0x00,0x41,0x00,0x05,0x00, -0x97,0x01,0x00,0x00,0xfa,0x01,0x00,0x00,0x82,0x01,0x00,0x00, -0xf9,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0xec,0x00,0x00,0x00, -0xfb,0x01,0x00,0x00,0xfa,0x01,0x00,0x00,0x73,0x00,0x04,0x00, -0xb9,0x00,0x00,0x00,0xfc,0x01,0x00,0x00,0xfb,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0x97,0x01,0x00,0x00,0x01,0x02,0x00,0x00, -0xb0,0x01,0x00,0x00,0xee,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, -0xec,0x00,0x00,0x00,0x02,0x02,0x00,0x00,0x01,0x02,0x00,0x00, -0x73,0x00,0x04,0x00,0xb9,0x00,0x00,0x00,0x03,0x02,0x00,0x00, -0x02,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, -0x05,0x02,0x00,0x00,0xbf,0x00,0x00,0x00,0xf5,0x01,0x00,0x00, -0x3d,0x00,0x04,0x00,0xb9,0x00,0x00,0x00,0x06,0x02,0x00,0x00, -0x05,0x02,0x00,0x00,0x0c,0x00,0x08,0x00,0xb9,0x00,0x00,0x00, -0x07,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00, -0xfc,0x01,0x00,0x00,0x03,0x02,0x00,0x00,0x06,0x02,0x00,0x00, -0x3e,0x00,0x03,0x00,0x05,0x02,0x00,0x00,0x07,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x0a,0x02,0x00,0x00, -0xc3,0x02,0x00,0x00,0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xe4,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xe6,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0xdf,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xdf,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x0c,0x02,0x00,0x00,0xc1,0x02,0x00,0x00,0xc5,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xdc,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xde,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xd7,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xd7,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x0e,0x02,0x00,0x00,0xbf,0x02,0x00,0x00, -0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xd4,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xd6,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0xcf,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xcf,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x10,0x02,0x00,0x00, -0xbb,0x02,0x00,0x00,0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xcc,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xce,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0x6b,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x6b,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x12,0x02,0x00,0x00,0xb5,0x02,0x00,0x00,0xc5,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x68,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x6a,0x01,0x00,0x00,0xe0,0x00,0x04,0x00,0x0c,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x60,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0xcc,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xcc,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x14,0x02,0x00,0x00, -0x9b,0x02,0x00,0x00,0x6c,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xc9,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xcb,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x19,0x02,0x00,0x00, -0x55,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x1a,0x02,0x00,0x00,0x8b,0x00,0x00,0x00, -0x19,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x1f,0x02,0x00,0x00,0x59,0x00,0x00,0x00,0xae,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x20,0x02,0x00,0x00, -0x9d,0x00,0x00,0x00,0x1f,0x02,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x25,0x02,0x00,0x00,0x12,0x00,0x00,0x00, -0x24,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x26,0x02,0x00,0x00,0x25,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x27,0x02,0x00,0x00,0x0f,0x00,0x00,0x00, -0x26,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x2b,0x02,0x00,0x00,0x47,0x00,0x00,0x00,0x26,0x02,0x00,0x00, -0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00,0x2d,0x02,0x00,0x00, -0x2c,0x02,0x00,0x00,0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x2e,0x02,0x00,0x00,0x2d,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x2f,0x02,0x00,0x00, -0x2b,0x02,0x00,0x00,0x2e,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x30,0x02,0x00,0x00,0x27,0x02,0x00,0x00, -0x2f,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x32,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x32,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0x9c,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, -0xcb,0x00,0x00,0x00,0x99,0x02,0x00,0x00,0x35,0x02,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00,0x38,0x02,0x00,0x00, -0x9c,0x02,0x00,0x00,0xb4,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0x34,0x02,0x00,0x00,0x35,0x02,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x38,0x02,0x00,0x00,0x33,0x02,0x00,0x00, -0x34,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x33,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0x3a,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x3a,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0x9d,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0x33,0x02,0x00,0x00, -0x97,0x02,0x00,0x00,0x3d,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb7,0x00,0x00,0x00,0x40,0x02,0x00,0x00,0x9d,0x02,0x00,0x00, -0x60,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x3c,0x02,0x00,0x00, -0x3d,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x40,0x02,0x00,0x00,0x3b,0x02,0x00,0x00,0x3c,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x3b,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x44,0x02,0x00,0x00,0x9d,0x02,0x00,0x00, -0x61,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x45,0x02,0x00,0x00,0x1a,0x02,0x00,0x00,0x44,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x47,0x02,0x00,0x00, -0x64,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x48,0x02,0x00,0x00,0x45,0x02,0x00,0x00, -0x47,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x4c,0x02,0x00,0x00,0x9c,0x02,0x00,0x00,0xb8,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x4d,0x02,0x00,0x00, -0x20,0x02,0x00,0x00,0x4c,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x4f,0x02,0x00,0x00,0x68,0x00,0x00,0x00, -0xb1,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x50,0x02,0x00,0x00,0x4d,0x02,0x00,0x00,0x4f,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0x52,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x52,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0x9f,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0x3b,0x02,0x00,0x00, -0x95,0x02,0x00,0x00,0x55,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb7,0x00,0x00,0x00,0x58,0x02,0x00,0x00,0x9f,0x02,0x00,0x00, -0xb1,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x54,0x02,0x00,0x00, -0x55,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x58,0x02,0x00,0x00,0x53,0x02,0x00,0x00,0x54,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x53,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x5a,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x5a,0x02,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xa1,0x02,0x00,0x00, -0x3e,0x00,0x00,0x00,0x53,0x02,0x00,0x00,0x93,0x02,0x00,0x00, -0x5d,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00, -0x60,0x02,0x00,0x00,0xa1,0x02,0x00,0x00,0x62,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0x5c,0x02,0x00,0x00,0x5d,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x60,0x02,0x00,0x00, -0x5b,0x02,0x00,0x00,0x5c,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x5b,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x63,0x02,0x00,0x00,0x48,0x02,0x00,0x00,0xa1,0x02,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00,0x66,0x02,0x00,0x00, -0x63,0x02,0x00,0x00,0x36,0x00,0x00,0x00,0xf7,0x00,0x03,0x00, -0x68,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x66,0x02,0x00,0x00,0x67,0x02,0x00,0x00,0x68,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x67,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x6b,0x02,0x00,0x00,0x50,0x02,0x00,0x00, -0x9f,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0x6c,0x02,0x00,0x00,0x12,0x00,0x00,0x00,0xc5,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x6d,0x02,0x00,0x00, -0x6c,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00, -0x6e,0x02,0x00,0x00,0x6b,0x02,0x00,0x00,0x6d,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0x68,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x68,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0xb7,0x00,0x00,0x00, -0x6f,0x02,0x00,0x00,0x66,0x02,0x00,0x00,0x5b,0x02,0x00,0x00, -0x6e,0x02,0x00,0x00,0x67,0x02,0x00,0x00,0xf7,0x00,0x03,0x00, -0x71,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x6f,0x02,0x00,0x00,0x70,0x02,0x00,0x00,0x71,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x70,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x79,0x02,0x00,0x00,0x50,0x02,0x00,0x00, -0x9f,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0x7b,0x02,0x00,0x00,0x12,0x00,0x00,0x00,0x7a,0x02,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x7c,0x02,0x00,0x00, -0x7b,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x7d,0x02,0x00,0x00,0x79,0x02,0x00,0x00,0x7c,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x7e,0x02,0x00,0x00, -0x30,0x02,0x00,0x00,0x7d,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x80,0x02,0x00,0x00,0x7e,0x02,0x00,0x00, -0x48,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x82,0x02,0x00,0x00,0x80,0x02,0x00,0x00,0xa1,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x84,0x02,0x00,0x00, -0x9c,0x02,0x00,0x00,0xb1,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x86,0x02,0x00,0x00,0x84,0x02,0x00,0x00, -0x9f,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x88,0x02,0x00,0x00,0x86,0x02,0x00,0x00,0x87,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8a,0x02,0x00,0x00, -0x9d,0x02,0x00,0x00,0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x8b,0x02,0x00,0x00,0x88,0x02,0x00,0x00, -0x8a,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x8d,0x02,0x00,0x00,0x8b,0x02,0x00,0x00,0xa1,0x02,0x00,0x00, -0x41,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0x8e,0x02,0x00,0x00, -0xbf,0x00,0x00,0x00,0x8d,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, -0xb9,0x00,0x00,0x00,0x8f,0x02,0x00,0x00,0x8e,0x02,0x00,0x00, -0x41,0x00,0x06,0x00,0x90,0x02,0x00,0x00,0x91,0x02,0x00,0x00, -0x75,0x02,0x00,0x00,0x34,0x00,0x00,0x00,0x82,0x02,0x00,0x00, -0x3e,0x00,0x03,0x00,0x91,0x02,0x00,0x00,0x8f,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0x71,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x71,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x5d,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x5d,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x93,0x02,0x00,0x00,0xa1,0x02,0x00,0x00, -0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x5a,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x5c,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x55,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x55,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x95,0x02,0x00,0x00, -0x9f,0x02,0x00,0x00,0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x52,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x54,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0x3d,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x3d,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x97,0x02,0x00,0x00,0x9d,0x02,0x00,0x00,0xc5,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x3a,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x3c,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x35,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x35,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x99,0x02,0x00,0x00,0x9c,0x02,0x00,0x00, -0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x32,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x34,0x02,0x00,0x00,0xfd,0x00,0x01,0x00, -0x38,0x00,0x01,0x00, -}; -const uint64_t matmul_f16_s_len = 10156; - -unsigned char matmul_f16_s_fp32_data[] = { -0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, -0xca,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, -0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x51,0x11,0x00,0x00, -0x0b,0x00,0x06,0x00,0x01,0x00,0x00,0x00,0x47,0x4c,0x53,0x4c, -0x2e,0x73,0x74,0x64,0x2e,0x34,0x35,0x30,0x00,0x00,0x00,0x00, -0x0e,0x00,0x03,0x00,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x0f,0x00,0x0f,0x00,0x05,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x6d,0x61,0x69,0x6e,0x00,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x3d,0x00,0x00,0x00,0x4c,0x00,0x00,0x00, -0xf0,0x00,0x00,0x00,0xfc,0x00,0x00,0x00,0x3c,0x01,0x00,0x00, -0x47,0x01,0x00,0x00,0x2a,0x02,0x00,0x00,0x73,0x02,0x00,0x00, -0x10,0x00,0x06,0x00,0x04,0x00,0x00,0x00,0x11,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x0b,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x1c,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x10,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x08,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x03,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x10,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x14,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x18,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x07,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x10,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x24,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x0a,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x10,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x2c,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x30,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x0d,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0x47,0x00,0x03,0x00, -0x10,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x37,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x3d,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x1a,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x4c,0x00,0x00,0x00, -0x0b,0x00,0x00,0x00,0x1b,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x4f,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x53,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x60,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x62,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x07,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x6c,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x03,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x9c,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0xae,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x05,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0xb1,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x08,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xf9,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x48,0x00,0x04,0x00, -0xfa,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0xfa,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, -0xfa,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0xfc,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0xfc,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x16,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x17,0x01,0x00,0x00,0x0b,0x00,0x00,0x00,0x19,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x44,0x01,0x00,0x00,0x06,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0x45,0x01,0x00,0x00, -0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x45,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x45,0x01,0x00,0x00, -0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x47,0x01,0x00,0x00, -0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x47,0x01,0x00,0x00,0x21,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x2a,0x02,0x00,0x00,0x0b,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x70,0x02,0x00,0x00, -0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00, -0x71,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x19,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x71,0x02,0x00,0x00,0x00,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, -0x71,0x02,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x73,0x02,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x73,0x02,0x00,0x00,0x21,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x13,0x00,0x02,0x00,0x02,0x00,0x00,0x00, -0x21,0x00,0x03,0x00,0x03,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x15,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x17,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x0a,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x0d,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x1e,0x00,0x10,0x00,0x10,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x11,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x11,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x15,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, -0x14,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x15,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x0a,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, -0x27,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0x2d,0x00,0x00,0x00,0x07,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x34,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x37,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00,0x3d,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x3e,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x0a,0x00,0x00,0x00,0x4c,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x4f,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x53,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x54,0x00,0x00,0x00,0x86,0x00,0x00,0x00, -0x37,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x58,0x00,0x00,0x00,0x86,0x00,0x00,0x00, -0x37,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x32,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x61,0x00,0x00,0x00, -0x86,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x60,0x00,0x00,0x00, -0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x62,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x63,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x61,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x67,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x61,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x86,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x72,0x00,0x00,0x00,0x86,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0x76,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x7b,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, -0x86,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0x8c,0x00,0x00,0x00,0x03,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x97,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x9c,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0x9e,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xad,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00, -0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xae,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xaf,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x53,0x00,0x00,0x00, -0xae,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xb0,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x4f,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xb1,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xb2,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0xb0,0x00,0x00,0x00,0xb1,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xb3,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0xb2,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xb4,0x00,0x00,0x00,0x86,0x00,0x00,0x00, -0xaf,0x00,0x00,0x00,0xb3,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xb5,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0xad,0x00,0x00,0x00,0xb4,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xb6,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0xb5,0x00,0x00,0x00,0xb1,0x00,0x00,0x00,0x14,0x00,0x02,0x00, -0xb7,0x00,0x00,0x00,0x16,0x00,0x03,0x00,0xb9,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xba,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xbb,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0xba,0x00,0x00,0x00, -0xb4,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xbc,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0xbb,0x00,0x00,0x00, -0xb1,0x00,0x00,0x00,0x1c,0x00,0x04,0x00,0xbd,0x00,0x00,0x00, -0xb9,0x00,0x00,0x00,0xbc,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0xbe,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0xbd,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0xb9,0x00,0x00,0x00,0xc1,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0xc2,0x00,0x00,0x00, -0x07,0x00,0x00,0x00,0xb9,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0xc5,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xec,0x00,0x00,0x00, -0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xed,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0xec,0x00,0x00,0x00, -0x1c,0x00,0x04,0x00,0xee,0x00,0x00,0x00,0xb9,0x00,0x00,0x00, -0xed,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0xef,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0xee,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0xef,0x00,0x00,0x00,0xf0,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xf4,0x00,0x00,0x00, -0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x16,0x00,0x03,0x00,0xf8,0x00,0x00,0x00,0x10,0x00,0x00,0x00, -0x1d,0x00,0x03,0x00,0xf9,0x00,0x00,0x00,0xf8,0x00,0x00,0x00, -0x1e,0x00,0x03,0x00,0xfa,0x00,0x00,0x00,0xf9,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0xfb,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0xfa,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0xfb,0x00,0x00,0x00, -0xfc,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x07,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0xf8,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x0b,0x01,0x00,0x00,0x04,0x00,0x00,0x00, -0xb9,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x11,0x01,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x16,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x33,0x00,0x06,0x00, -0x09,0x00,0x00,0x00,0x17,0x01,0x00,0x00,0x16,0x01,0x00,0x00, -0x39,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x18,0x01,0x00,0x00,0x51,0x00,0x00,0x00, -0x17,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x19,0x01,0x00,0x00,0x84,0x00,0x00,0x00, -0x18,0x01,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x1a,0x01,0x00,0x00,0x86,0x00,0x00,0x00, -0x19,0x01,0x00,0x00,0x6c,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x38,0x01,0x00,0x00,0x80,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x39,0x01,0x00,0x00,0x84,0x00,0x00,0x00, -0x9c,0x00,0x00,0x00,0x38,0x01,0x00,0x00,0x1c,0x00,0x04,0x00, -0x3a,0x01,0x00,0x00,0xb9,0x00,0x00,0x00,0x39,0x01,0x00,0x00, -0x20,0x00,0x04,0x00,0x3b,0x01,0x00,0x00,0x04,0x00,0x00,0x00, -0x3a,0x01,0x00,0x00,0x3b,0x00,0x04,0x00,0x3b,0x01,0x00,0x00, -0x3c,0x01,0x00,0x00,0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x40,0x01,0x00,0x00,0x80,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, -0x44,0x01,0x00,0x00,0xf8,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, -0x45,0x01,0x00,0x00,0x44,0x01,0x00,0x00,0x20,0x00,0x04,0x00, -0x46,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x45,0x01,0x00,0x00, -0x3b,0x00,0x04,0x00,0x46,0x01,0x00,0x00,0x47,0x01,0x00,0x00, -0x0c,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x5a,0x01,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x61,0x01,0x00,0x00,0x08,0x01,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x62,0x01,0x00,0x00,0x86,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x65,0x01,0x00,0x00,0x86,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x80,0x01,0x00,0x00,0x84,0x00,0x00,0x00, -0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x1c,0x00,0x04,0x00, -0x81,0x01,0x00,0x00,0xb9,0x00,0x00,0x00,0x80,0x01,0x00,0x00, -0x20,0x00,0x04,0x00,0x82,0x01,0x00,0x00,0x07,0x00,0x00,0x00, -0x81,0x01,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x92,0x01,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xad,0x01,0x00,0x00,0x84,0x00,0x00,0x00,0xb4,0x00,0x00,0x00, -0xb1,0x00,0x00,0x00,0x1c,0x00,0x04,0x00,0xae,0x01,0x00,0x00, -0xb9,0x00,0x00,0x00,0xad,0x01,0x00,0x00,0x20,0x00,0x04,0x00, -0xaf,0x01,0x00,0x00,0x07,0x00,0x00,0x00,0xae,0x01,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xb8,0x01,0x00,0x00, -0x86,0x00,0x00,0x00,0xae,0x00,0x00,0x00,0xb4,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xc0,0x01,0x00,0x00, -0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xef,0x01,0x00,0x00, -0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x22,0x02,0x00,0x00, -0x0d,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00, -0x2a,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, -0x70,0x02,0x00,0x00,0xb9,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, -0x71,0x02,0x00,0x00,0x70,0x02,0x00,0x00,0x20,0x00,0x04,0x00, -0x72,0x02,0x00,0x00,0x0c,0x00,0x00,0x00,0x71,0x02,0x00,0x00, -0x3b,0x00,0x04,0x00,0x72,0x02,0x00,0x00,0x73,0x02,0x00,0x00, -0x0c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, -0x78,0x02,0x00,0x00,0x05,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x85,0x02,0x00,0x00,0x84,0x00,0x00,0x00, -0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x8e,0x02,0x00,0x00,0x0c,0x00,0x00,0x00,0xb9,0x00,0x00,0x00, -0x36,0x00,0x05,0x00,0x02,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x05,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0xbe,0x00,0x00,0x00, -0xbf,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x82,0x01,0x00,0x00,0x83,0x01,0x00,0x00,0x07,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0xaf,0x01,0x00,0x00,0xb0,0x01,0x00,0x00, -0x07,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00, -0x0e,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, -0x0e,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x14,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x17,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x17,0x00,0x00,0x00, -0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x1e,0x00,0x00,0x00, -0x0f,0x00,0x00,0x00,0x17,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x21,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x24,0x00,0x00,0x00,0x18,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0x28,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x27,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x29,0x00,0x00,0x00, -0x28,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x2a,0x00,0x00,0x00,0x1e,0x00,0x00,0x00,0x29,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x2d,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x2f,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x30,0x00,0x00,0x00, -0x24,0x00,0x00,0x00,0x2f,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x30,0x00,0x00,0x00, -0x2a,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0x35,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x34,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x36,0x00,0x00,0x00, -0x35,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x38,0x00,0x00,0x00,0x36,0x00,0x00,0x00,0x37,0x00,0x00,0x00, -0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, -0x38,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x3b,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, -0x37,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00, -0x3f,0x00,0x00,0x00,0x3d,0x00,0x00,0x00,0x3e,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x3f,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x42,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x3b,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x47,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0x3b,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x0d,0x00,0x00,0x00,0x49,0x00,0x00,0x00,0x3d,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x4a,0x00,0x00,0x00,0x49,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x0d,0x00,0x00,0x00,0x4d,0x00,0x00,0x00,0x4c,0x00,0x00,0x00, -0x3e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x4e,0x00,0x00,0x00,0x4d,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x50,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, -0x4f,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x55,0x00,0x00,0x00,0x50,0x00,0x00,0x00,0x54,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x59,0x00,0x00,0x00, -0x50,0x00,0x00,0x00,0x58,0x00,0x00,0x00,0x89,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x5d,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, -0x4f,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x64,0x00,0x00,0x00,0x5d,0x00,0x00,0x00,0x63,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x68,0x00,0x00,0x00, -0x5d,0x00,0x00,0x00,0x67,0x00,0x00,0x00,0x89,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x6e,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, -0x6d,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x73,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x72,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x77,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x76,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x78,0x00,0x00,0x00,0x77,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x79,0x00,0x00,0x00, -0x47,0x00,0x00,0x00,0x78,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x7c,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x7b,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x7d,0x00,0x00,0x00,0x7c,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x7f,0x00,0x00,0x00,0x47,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x82,0x00,0x00,0x00,0x7f,0x00,0x00,0x00,0x78,0x00,0x00,0x00, -0x0c,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x83,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x26,0x00,0x00,0x00,0x7d,0x00,0x00,0x00, -0x82,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0x87,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x86,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x88,0x00,0x00,0x00, -0x87,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x89,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x88,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8b,0x00,0x00,0x00, -0x42,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x8d,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x8c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x8e,0x00,0x00,0x00,0x8d,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x8f,0x00,0x00,0x00,0x8b,0x00,0x00,0x00, -0x8e,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x90,0x00,0x00,0x00,0x89,0x00,0x00,0x00,0x8f,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x92,0x00,0x00,0x00, -0x90,0x00,0x00,0x00,0x79,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x93,0x00,0x00,0x00,0x92,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0x98,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x97,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x99,0x00,0x00,0x00, -0x98,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x9a,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x99,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9d,0x00,0x00,0x00, -0x4a,0x00,0x00,0x00,0x9c,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x9f,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x9e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xa0,0x00,0x00,0x00,0x9f,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xa1,0x00,0x00,0x00,0x9d,0x00,0x00,0x00, -0xa0,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xa2,0x00,0x00,0x00,0x9a,0x00,0x00,0x00,0xa1,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa4,0x00,0x00,0x00, -0xa2,0x00,0x00,0x00,0x79,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xa5,0x00,0x00,0x00,0xa4,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xa7,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xa7,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0x98,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, -0x05,0x00,0x00,0x00,0xc6,0x00,0x00,0x00,0xa8,0x00,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00,0xb8,0x00,0x00,0x00, -0x98,0x02,0x00,0x00,0xb6,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0xa9,0x00,0x00,0x00,0xa8,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xb8,0x00,0x00,0x00,0xa8,0x00,0x00,0x00, -0xa9,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xa8,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0xc3,0x00,0x00,0x00, -0xbf,0x00,0x00,0x00,0x98,0x02,0x00,0x00,0x3e,0x00,0x03,0x00, -0xc3,0x00,0x00,0x00,0xc1,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xc6,0x00,0x00,0x00,0x98,0x02,0x00,0x00, -0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xa7,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xa9,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xc9,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xc9,0x00,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xb1,0x02,0x00,0x00, -0xa5,0x00,0x00,0x00,0xa9,0x00,0x00,0x00,0x67,0x01,0x00,0x00, -0xcc,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xad,0x02,0x00,0x00,0x93,0x00,0x00,0x00,0xa9,0x00,0x00,0x00, -0x64,0x01,0x00,0x00,0xcc,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0x99,0x02,0x00,0x00,0x79,0x00,0x00,0x00, -0xa9,0x00,0x00,0x00,0x12,0x02,0x00,0x00,0xcc,0x00,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00,0xd0,0x00,0x00,0x00, -0x99,0x02,0x00,0x00,0x83,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0xcb,0x00,0x00,0x00,0xcc,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xd0,0x00,0x00,0x00,0xca,0x00,0x00,0x00, -0xcb,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xca,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xd2,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xd2,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xa9,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0xca,0x00,0x00,0x00, -0x1c,0x01,0x00,0x00,0xd5,0x00,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb7,0x00,0x00,0x00,0xd8,0x00,0x00,0x00,0xa9,0x02,0x00,0x00, -0x37,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xd4,0x00,0x00,0x00, -0xd5,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0xd8,0x00,0x00,0x00,0xd3,0x00,0x00,0x00,0xd4,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xd3,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xdc,0x00,0x00,0x00,0x8b,0x00,0x00,0x00, -0x73,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xde,0x00,0x00,0x00,0xdc,0x00,0x00,0x00,0xa9,0x02,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00,0xe1,0x00,0x00,0x00, -0xde,0x00,0x00,0x00,0x36,0x00,0x00,0x00,0xf7,0x00,0x03,0x00, -0xe3,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0xe1,0x00,0x00,0x00,0xe2,0x00,0x00,0x00,0xe3,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xe2,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xe6,0x00,0x00,0x00,0x99,0x02,0x00,0x00, -0x6e,0x00,0x00,0x00,0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00, -0xe8,0x00,0x00,0x00,0xe6,0x00,0x00,0x00,0x83,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xe3,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xe3,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0xb7,0x00,0x00,0x00, -0xe9,0x00,0x00,0x00,0xe1,0x00,0x00,0x00,0xd3,0x00,0x00,0x00, -0xe8,0x00,0x00,0x00,0xe2,0x00,0x00,0x00,0xf7,0x00,0x03,0x00, -0xeb,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0xe9,0x00,0x00,0x00,0xea,0x00,0x00,0x00,0x0d,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xea,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xf3,0x00,0x00,0x00,0x73,0x00,0x00,0x00, -0xa9,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xf5,0x00,0x00,0x00,0xf3,0x00,0x00,0x00,0xf4,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xf7,0x00,0x00,0x00, -0xf5,0x00,0x00,0x00,0x6e,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x03,0x01,0x00,0x00,0xf3,0x00,0x00,0x00, -0x8e,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x04,0x01,0x00,0x00,0xad,0x02,0x00,0x00,0x03,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x06,0x01,0x00,0x00, -0x04,0x01,0x00,0x00,0x6e,0x00,0x00,0x00,0x41,0x00,0x06,0x00, -0x07,0x01,0x00,0x00,0x08,0x01,0x00,0x00,0xfc,0x00,0x00,0x00, -0x34,0x00,0x00,0x00,0x06,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, -0xf8,0x00,0x00,0x00,0x09,0x01,0x00,0x00,0x08,0x01,0x00,0x00, -0x73,0x00,0x04,0x00,0xb9,0x00,0x00,0x00,0x0a,0x01,0x00,0x00, -0x09,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0x0b,0x01,0x00,0x00, -0x0c,0x01,0x00,0x00,0xf0,0x00,0x00,0x00,0xf7,0x00,0x00,0x00, -0x3e,0x00,0x03,0x00,0x0c,0x01,0x00,0x00,0x0a,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0xeb,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x0d,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x10,0x01,0x00,0x00,0x73,0x00,0x00,0x00,0xa9,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x12,0x01,0x00,0x00, -0x10,0x01,0x00,0x00,0x11,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x14,0x01,0x00,0x00,0x12,0x01,0x00,0x00, -0x6e,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x0b,0x01,0x00,0x00, -0x15,0x01,0x00,0x00,0xf0,0x00,0x00,0x00,0x14,0x01,0x00,0x00, -0x3e,0x00,0x03,0x00,0x15,0x01,0x00,0x00,0xc1,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xeb,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xeb,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xd5,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xd5,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x1c,0x01,0x00,0x00,0xa9,0x02,0x00,0x00, -0x1a,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xd2,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xd4,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x1e,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x1e,0x01,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xaa,0x02,0x00,0x00, -0x3e,0x00,0x00,0x00,0xd4,0x00,0x00,0x00,0x60,0x01,0x00,0x00, -0x21,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00, -0x24,0x01,0x00,0x00,0xaa,0x02,0x00,0x00,0x9c,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0x20,0x01,0x00,0x00,0x21,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x24,0x01,0x00,0x00, -0x1f,0x01,0x00,0x00,0x20,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x1f,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x28,0x01,0x00,0x00,0x9d,0x00,0x00,0x00,0x73,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x2a,0x01,0x00,0x00, -0x28,0x01,0x00,0x00,0xaa,0x02,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x2b,0x01,0x00,0x00,0x12,0x00,0x00,0x00, -0xc5,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x2c,0x01,0x00,0x00,0x2b,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb7,0x00,0x00,0x00,0x2d,0x01,0x00,0x00,0x2a,0x01,0x00,0x00, -0x2c,0x01,0x00,0x00,0xf7,0x00,0x03,0x00,0x2f,0x01,0x00,0x00, -0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x2d,0x01,0x00,0x00, -0x2e,0x01,0x00,0x00,0x2f,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x2e,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x32,0x01,0x00,0x00,0x99,0x02,0x00,0x00,0x6e,0x00,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00,0x34,0x01,0x00,0x00, -0x32,0x01,0x00,0x00,0x83,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x2f,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x2f,0x01,0x00,0x00, -0xf5,0x00,0x07,0x00,0xb7,0x00,0x00,0x00,0x35,0x01,0x00,0x00, -0x2d,0x01,0x00,0x00,0x1f,0x01,0x00,0x00,0x34,0x01,0x00,0x00, -0x2e,0x01,0x00,0x00,0xf7,0x00,0x03,0x00,0x37,0x01,0x00,0x00, -0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x35,0x01,0x00,0x00, -0x36,0x01,0x00,0x00,0x56,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x36,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x3f,0x01,0x00,0x00,0x73,0x00,0x00,0x00,0xaa,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x41,0x01,0x00,0x00, -0x3f,0x01,0x00,0x00,0x40,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x43,0x01,0x00,0x00,0x41,0x01,0x00,0x00, -0x6e,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x4e,0x01,0x00,0x00,0x3f,0x01,0x00,0x00,0xa0,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x4f,0x01,0x00,0x00, -0xb1,0x02,0x00,0x00,0x4e,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x51,0x01,0x00,0x00,0x4f,0x01,0x00,0x00, -0x6e,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x07,0x01,0x00,0x00, -0x52,0x01,0x00,0x00,0x47,0x01,0x00,0x00,0x34,0x00,0x00,0x00, -0x51,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0xf8,0x00,0x00,0x00, -0x53,0x01,0x00,0x00,0x52,0x01,0x00,0x00,0x73,0x00,0x04,0x00, -0xb9,0x00,0x00,0x00,0x54,0x01,0x00,0x00,0x53,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0x0b,0x01,0x00,0x00,0x55,0x01,0x00,0x00, -0x3c,0x01,0x00,0x00,0x43,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0x55,0x01,0x00,0x00,0x54,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0x37,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x56,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x59,0x01,0x00,0x00, -0x73,0x00,0x00,0x00,0xaa,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x5b,0x01,0x00,0x00,0x59,0x01,0x00,0x00, -0x5a,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x5d,0x01,0x00,0x00,0x5b,0x01,0x00,0x00,0x6e,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x0b,0x01,0x00,0x00,0x5e,0x01,0x00,0x00, -0x3c,0x01,0x00,0x00,0x5d,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0x5e,0x01,0x00,0x00,0xc1,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x37,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x37,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0x21,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x21,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x60,0x01,0x00,0x00,0xaa,0x02,0x00,0x00,0x1a,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0x1e,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x20,0x01,0x00,0x00,0xe0,0x00,0x04,0x00,0x0c,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x61,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x64,0x01,0x00,0x00,0xad,0x02,0x00,0x00, -0x62,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x67,0x01,0x00,0x00,0xb1,0x02,0x00,0x00,0x65,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0x69,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x69,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xb3,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0x20,0x01,0x00,0x00, -0x10,0x02,0x00,0x00,0x6c,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb7,0x00,0x00,0x00,0x6f,0x01,0x00,0x00,0xb3,0x02,0x00,0x00, -0x6c,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x6b,0x01,0x00,0x00, -0x6c,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x6f,0x01,0x00,0x00,0x6a,0x01,0x00,0x00,0x6b,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x6a,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0x71,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x71,0x01,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xb7,0x02,0x00,0x00, -0x3e,0x00,0x00,0x00,0x6a,0x01,0x00,0x00,0x9c,0x01,0x00,0x00, -0x74,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00, -0x77,0x01,0x00,0x00,0xb7,0x02,0x00,0x00,0x60,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0x73,0x01,0x00,0x00,0x74,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x77,0x01,0x00,0x00, -0x72,0x01,0x00,0x00,0x73,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x72,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x79,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x79,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xc9,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, -0x72,0x01,0x00,0x00,0x9a,0x01,0x00,0x00,0x7a,0x01,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00,0x7f,0x01,0x00,0x00, -0xc9,0x02,0x00,0x00,0x62,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0x7b,0x01,0x00,0x00,0x7a,0x01,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x7f,0x01,0x00,0x00,0x7a,0x01,0x00,0x00, -0x7b,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x7a,0x01,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x85,0x01,0x00,0x00, -0xb7,0x02,0x00,0x00,0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x87,0x01,0x00,0x00,0x85,0x01,0x00,0x00, -0xc9,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x89,0x01,0x00,0x00,0x55,0x00,0x00,0x00,0x53,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8b,0x01,0x00,0x00, -0xb7,0x02,0x00,0x00,0x61,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x8c,0x01,0x00,0x00,0x89,0x01,0x00,0x00, -0x8b,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x8e,0x01,0x00,0x00,0x64,0x00,0x00,0x00,0x62,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8f,0x01,0x00,0x00, -0x8c,0x01,0x00,0x00,0x8e,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x91,0x01,0x00,0x00,0x8f,0x01,0x00,0x00, -0xc9,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x93,0x01,0x00,0x00,0x91,0x01,0x00,0x00,0x92,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x95,0x01,0x00,0x00, -0x93,0x01,0x00,0x00,0xb3,0x02,0x00,0x00,0x41,0x00,0x05,0x00, -0x0b,0x01,0x00,0x00,0x96,0x01,0x00,0x00,0xf0,0x00,0x00,0x00, -0x95,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0xb9,0x00,0x00,0x00, -0x97,0x01,0x00,0x00,0x96,0x01,0x00,0x00,0x41,0x00,0x05,0x00, -0xc2,0x00,0x00,0x00,0x98,0x01,0x00,0x00,0x83,0x01,0x00,0x00, -0x87,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x98,0x01,0x00,0x00, -0x97,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x9a,0x01,0x00,0x00,0xc9,0x02,0x00,0x00,0xc5,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x79,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x7b,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x74,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x74,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x9c,0x01,0x00,0x00,0xb7,0x02,0x00,0x00, -0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x71,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x73,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0x9e,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x9e,0x01,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xb8,0x02,0x00,0x00, -0x3e,0x00,0x00,0x00,0x73,0x01,0x00,0x00,0xca,0x01,0x00,0x00, -0xa1,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00, -0xa4,0x01,0x00,0x00,0xb8,0x02,0x00,0x00,0xb4,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0xa0,0x01,0x00,0x00,0xa1,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xa4,0x01,0x00,0x00, -0x9f,0x01,0x00,0x00,0xa0,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x9f,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xa6,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xa6,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xc6,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, -0x9f,0x01,0x00,0x00,0xc8,0x01,0x00,0x00,0xa7,0x01,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00,0xac,0x01,0x00,0x00, -0xc6,0x02,0x00,0x00,0xb1,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0xa8,0x01,0x00,0x00,0xa7,0x01,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xac,0x01,0x00,0x00,0xa7,0x01,0x00,0x00, -0xa8,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xa7,0x01,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xb2,0x01,0x00,0x00, -0xb8,0x02,0x00,0x00,0xb1,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xb4,0x01,0x00,0x00,0xb2,0x01,0x00,0x00, -0xc6,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xb6,0x01,0x00,0x00,0x59,0x00,0x00,0x00,0xae,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xb9,0x01,0x00,0x00, -0xb8,0x02,0x00,0x00,0xb8,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xba,0x01,0x00,0x00,0xb6,0x01,0x00,0x00, -0xb9,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xbc,0x01,0x00,0x00,0x68,0x00,0x00,0x00,0xb1,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xbd,0x01,0x00,0x00, -0xba,0x01,0x00,0x00,0xbc,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xbf,0x01,0x00,0x00,0xbd,0x01,0x00,0x00, -0xc6,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xc1,0x01,0x00,0x00,0xbf,0x01,0x00,0x00,0xc0,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xc3,0x01,0x00,0x00, -0xc1,0x01,0x00,0x00,0xb3,0x02,0x00,0x00,0x41,0x00,0x05,0x00, -0x0b,0x01,0x00,0x00,0xc4,0x01,0x00,0x00,0x3c,0x01,0x00,0x00, -0xc3,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0xb9,0x00,0x00,0x00, -0xc5,0x01,0x00,0x00,0xc4,0x01,0x00,0x00,0x41,0x00,0x05,0x00, -0xc2,0x00,0x00,0x00,0xc6,0x01,0x00,0x00,0xb0,0x01,0x00,0x00, -0xb4,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0xc6,0x01,0x00,0x00, -0xc5,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xc8,0x01,0x00,0x00,0xc6,0x02,0x00,0x00,0xc5,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xa6,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xa8,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xa1,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xa1,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xca,0x01,0x00,0x00,0xb8,0x02,0x00,0x00, -0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x9e,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xa0,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0xcc,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xcc,0x01,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xb9,0x02,0x00,0x00, -0x3e,0x00,0x00,0x00,0xa0,0x01,0x00,0x00,0x0e,0x02,0x00,0x00, -0xcf,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00, -0xd2,0x01,0x00,0x00,0xb9,0x02,0x00,0x00,0xb4,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0xce,0x01,0x00,0x00,0xcf,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xd2,0x01,0x00,0x00, -0xcd,0x01,0x00,0x00,0xce,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xcd,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xd4,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xd4,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xbd,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, -0xcd,0x01,0x00,0x00,0x0c,0x02,0x00,0x00,0xd7,0x01,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00,0xda,0x01,0x00,0x00, -0xbd,0x02,0x00,0x00,0x60,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0xd6,0x01,0x00,0x00,0xd7,0x01,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xda,0x01,0x00,0x00,0xd5,0x01,0x00,0x00, -0xd6,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xd5,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0xdc,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xdc,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xbf,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0xd5,0x01,0x00,0x00, -0x0a,0x02,0x00,0x00,0xdf,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb7,0x00,0x00,0x00,0xe2,0x01,0x00,0x00,0xbf,0x02,0x00,0x00, -0xb1,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xde,0x01,0x00,0x00, -0xdf,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0xe2,0x01,0x00,0x00,0xdd,0x01,0x00,0x00,0xde,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xdd,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0xe4,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xe4,0x01,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xc1,0x02,0x00,0x00, -0x3e,0x00,0x00,0x00,0xdd,0x01,0x00,0x00,0x08,0x02,0x00,0x00, -0xe5,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00, -0xea,0x01,0x00,0x00,0xc1,0x02,0x00,0x00,0x62,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0xe6,0x01,0x00,0x00,0xe5,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xea,0x01,0x00,0x00, -0xe5,0x01,0x00,0x00,0xe6,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xe5,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xec,0x01,0x00,0x00,0xb9,0x02,0x00,0x00,0xb1,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xee,0x01,0x00,0x00, -0xec,0x01,0x00,0x00,0xbf,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xf0,0x01,0x00,0x00,0xee,0x01,0x00,0x00, -0xef,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xf2,0x01,0x00,0x00,0xbd,0x02,0x00,0x00,0x62,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xf3,0x01,0x00,0x00, -0xf0,0x01,0x00,0x00,0xf2,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xf5,0x01,0x00,0x00,0xf3,0x01,0x00,0x00, -0xc1,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xf9,0x01,0x00,0x00,0xf2,0x01,0x00,0x00,0xc1,0x02,0x00,0x00, -0x41,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0xfa,0x01,0x00,0x00, -0x83,0x01,0x00,0x00,0xf9,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, -0xb9,0x00,0x00,0x00,0xfb,0x01,0x00,0x00,0xfa,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0x00,0x02,0x00,0x00, -0xb0,0x01,0x00,0x00,0xee,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, -0xb9,0x00,0x00,0x00,0x01,0x02,0x00,0x00,0x00,0x02,0x00,0x00, -0x41,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0x03,0x02,0x00,0x00, -0xbf,0x00,0x00,0x00,0xf5,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, -0xb9,0x00,0x00,0x00,0x04,0x02,0x00,0x00,0x03,0x02,0x00,0x00, -0x0c,0x00,0x08,0x00,0xb9,0x00,0x00,0x00,0x05,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0xfb,0x01,0x00,0x00, -0x01,0x02,0x00,0x00,0x04,0x02,0x00,0x00,0x3e,0x00,0x03,0x00, -0x03,0x02,0x00,0x00,0x05,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x08,0x02,0x00,0x00,0xc1,0x02,0x00,0x00, -0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xe4,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xe6,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0xdf,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xdf,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x0a,0x02,0x00,0x00, -0xbf,0x02,0x00,0x00,0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xdc,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xde,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0xd7,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xd7,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x0c,0x02,0x00,0x00,0xbd,0x02,0x00,0x00,0xc5,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xd4,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xd6,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xcf,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xcf,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x0e,0x02,0x00,0x00,0xb9,0x02,0x00,0x00, -0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xcc,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xce,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0x6c,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x6c,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x10,0x02,0x00,0x00, -0xb3,0x02,0x00,0x00,0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x69,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x6b,0x01,0x00,0x00, -0xe0,0x00,0x04,0x00,0x0c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x61,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xcc,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xcc,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x12,0x02,0x00,0x00,0x99,0x02,0x00,0x00, -0x6c,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xc9,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xcb,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x17,0x02,0x00,0x00,0x55,0x00,0x00,0x00, -0x53,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x18,0x02,0x00,0x00,0x8b,0x00,0x00,0x00,0x17,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x1d,0x02,0x00,0x00, -0x59,0x00,0x00,0x00,0xae,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x1e,0x02,0x00,0x00,0x9d,0x00,0x00,0x00, -0x1d,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0x23,0x02,0x00,0x00,0x12,0x00,0x00,0x00,0x22,0x02,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x24,0x02,0x00,0x00, -0x23,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x25,0x02,0x00,0x00,0x0f,0x00,0x00,0x00,0x24,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x29,0x02,0x00,0x00, -0x47,0x00,0x00,0x00,0x24,0x02,0x00,0x00,0x41,0x00,0x05,0x00, -0x0d,0x00,0x00,0x00,0x2b,0x02,0x00,0x00,0x2a,0x02,0x00,0x00, -0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x2c,0x02,0x00,0x00,0x2b,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x2d,0x02,0x00,0x00,0x29,0x02,0x00,0x00, -0x2c,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x2e,0x02,0x00,0x00,0x25,0x02,0x00,0x00,0x2d,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0x30,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x30,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0x9a,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0xcb,0x00,0x00,0x00, -0x97,0x02,0x00,0x00,0x33,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb7,0x00,0x00,0x00,0x36,0x02,0x00,0x00,0x9a,0x02,0x00,0x00, -0xb4,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x32,0x02,0x00,0x00, -0x33,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x36,0x02,0x00,0x00,0x31,0x02,0x00,0x00,0x32,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x31,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x38,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x38,0x02,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x9b,0x02,0x00,0x00, -0x3e,0x00,0x00,0x00,0x31,0x02,0x00,0x00,0x95,0x02,0x00,0x00, -0x3b,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00, -0x3e,0x02,0x00,0x00,0x9b,0x02,0x00,0x00,0x60,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0x3a,0x02,0x00,0x00,0x3b,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x3e,0x02,0x00,0x00, -0x39,0x02,0x00,0x00,0x3a,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x39,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x42,0x02,0x00,0x00,0x9b,0x02,0x00,0x00,0x61,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x43,0x02,0x00,0x00, -0x18,0x02,0x00,0x00,0x42,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x45,0x02,0x00,0x00,0x64,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x46,0x02,0x00,0x00,0x43,0x02,0x00,0x00,0x45,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x4a,0x02,0x00,0x00, -0x9a,0x02,0x00,0x00,0xb8,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x4b,0x02,0x00,0x00,0x1e,0x02,0x00,0x00, -0x4a,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x4d,0x02,0x00,0x00,0x68,0x00,0x00,0x00,0xb1,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x4e,0x02,0x00,0x00, -0x4b,0x02,0x00,0x00,0x4d,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x50,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x50,0x02,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x9d,0x02,0x00,0x00, -0x3e,0x00,0x00,0x00,0x39,0x02,0x00,0x00,0x93,0x02,0x00,0x00, -0x53,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00, -0x56,0x02,0x00,0x00,0x9d,0x02,0x00,0x00,0xb1,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0x52,0x02,0x00,0x00,0x53,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x56,0x02,0x00,0x00, -0x51,0x02,0x00,0x00,0x52,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x51,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x58,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x58,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0x9f,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, -0x51,0x02,0x00,0x00,0x91,0x02,0x00,0x00,0x5b,0x02,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00,0x5e,0x02,0x00,0x00, -0x9f,0x02,0x00,0x00,0x62,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0x5a,0x02,0x00,0x00,0x5b,0x02,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x5e,0x02,0x00,0x00,0x59,0x02,0x00,0x00, -0x5a,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x59,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x61,0x02,0x00,0x00, -0x46,0x02,0x00,0x00,0x9f,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb7,0x00,0x00,0x00,0x64,0x02,0x00,0x00,0x61,0x02,0x00,0x00, -0x36,0x00,0x00,0x00,0xf7,0x00,0x03,0x00,0x66,0x02,0x00,0x00, -0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x64,0x02,0x00,0x00, -0x65,0x02,0x00,0x00,0x66,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x65,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x69,0x02,0x00,0x00,0x4e,0x02,0x00,0x00,0x9d,0x02,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x6a,0x02,0x00,0x00, -0x12,0x00,0x00,0x00,0xc5,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x6b,0x02,0x00,0x00,0x6a,0x02,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00,0x6c,0x02,0x00,0x00, -0x69,0x02,0x00,0x00,0x6b,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x66,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x66,0x02,0x00,0x00, -0xf5,0x00,0x07,0x00,0xb7,0x00,0x00,0x00,0x6d,0x02,0x00,0x00, -0x64,0x02,0x00,0x00,0x59,0x02,0x00,0x00,0x6c,0x02,0x00,0x00, -0x65,0x02,0x00,0x00,0xf7,0x00,0x03,0x00,0x6f,0x02,0x00,0x00, -0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x6d,0x02,0x00,0x00, -0x6e,0x02,0x00,0x00,0x6f,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x6e,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x77,0x02,0x00,0x00,0x4e,0x02,0x00,0x00,0x9d,0x02,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x79,0x02,0x00,0x00, -0x12,0x00,0x00,0x00,0x78,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x7a,0x02,0x00,0x00,0x79,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x7b,0x02,0x00,0x00, -0x77,0x02,0x00,0x00,0x7a,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x7c,0x02,0x00,0x00,0x2e,0x02,0x00,0x00, -0x7b,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x7e,0x02,0x00,0x00,0x7c,0x02,0x00,0x00,0x46,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x80,0x02,0x00,0x00, -0x7e,0x02,0x00,0x00,0x9f,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x82,0x02,0x00,0x00,0x9a,0x02,0x00,0x00, -0xb1,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x84,0x02,0x00,0x00,0x82,0x02,0x00,0x00,0x9d,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x86,0x02,0x00,0x00, -0x84,0x02,0x00,0x00,0x85,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x88,0x02,0x00,0x00,0x9b,0x02,0x00,0x00, -0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x89,0x02,0x00,0x00,0x86,0x02,0x00,0x00,0x88,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8b,0x02,0x00,0x00, -0x89,0x02,0x00,0x00,0x9f,0x02,0x00,0x00,0x41,0x00,0x05,0x00, -0xc2,0x00,0x00,0x00,0x8c,0x02,0x00,0x00,0xbf,0x00,0x00,0x00, -0x8b,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0xb9,0x00,0x00,0x00, -0x8d,0x02,0x00,0x00,0x8c,0x02,0x00,0x00,0x41,0x00,0x06,0x00, -0x8e,0x02,0x00,0x00,0x8f,0x02,0x00,0x00,0x73,0x02,0x00,0x00, -0x34,0x00,0x00,0x00,0x80,0x02,0x00,0x00,0x3e,0x00,0x03,0x00, -0x8f,0x02,0x00,0x00,0x8d,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x6f,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x6f,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0x5b,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x5b,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x91,0x02,0x00,0x00,0x9f,0x02,0x00,0x00,0xc5,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x58,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x5a,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x53,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x53,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x93,0x02,0x00,0x00,0x9d,0x02,0x00,0x00, -0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x50,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x52,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x3b,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x3b,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x95,0x02,0x00,0x00, -0x9b,0x02,0x00,0x00,0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x38,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x3a,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0x33,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x33,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x97,0x02,0x00,0x00,0x9a,0x02,0x00,0x00,0xc5,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x30,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x32,0x02,0x00,0x00,0xfd,0x00,0x01,0x00,0x38,0x00,0x01,0x00, - -}; -const uint64_t matmul_f16_s_fp32_len = 10116; - -unsigned char matmul_f32_aligned_l_data[] = { -0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, -0x6a,0x03,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, -0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x09,0x00,0x00,0x00, -0x0b,0x00,0x06,0x00,0x01,0x00,0x00,0x00,0x47,0x4c,0x53,0x4c, -0x2e,0x73,0x74,0x64,0x2e,0x34,0x35,0x30,0x00,0x00,0x00,0x00, -0x0e,0x00,0x03,0x00,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x0f,0x00,0x0f,0x00,0x05,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x6d,0x61,0x69,0x6e,0x00,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x3d,0x00,0x00,0x00,0x4c,0x00,0x00,0x00, -0xeb,0x00,0x00,0x00,0xfa,0x00,0x00,0x00,0x88,0x01,0x00,0x00, -0x95,0x01,0x00,0x00,0xcb,0x02,0x00,0x00,0x14,0x03,0x00,0x00, -0x10,0x00,0x06,0x00,0x04,0x00,0x00,0x00,0x11,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x0b,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x1c,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x10,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x08,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x03,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x10,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x14,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x18,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x07,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x10,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x24,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x0a,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x10,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x2c,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x30,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x0d,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0x47,0x00,0x03,0x00, -0x10,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x37,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x3d,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x1a,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x4c,0x00,0x00,0x00, -0x0b,0x00,0x00,0x00,0x1b,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x4f,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x53,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x60,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x62,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x07,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x6c,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x03,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x9d,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0xaf,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x05,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0xb2,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x08,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xf7,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x48,0x00,0x04,0x00, -0xf8,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x05,0x00,0x00,0x00, -0x48,0x00,0x04,0x00,0xf8,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0xf8,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0xf8,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x07,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x47,0x00,0x03,0x00, -0xf8,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0xfa,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0xfa,0x00,0x00,0x00,0x21,0x00,0x00,0x00, +0x67,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x67,0x01,0x00,0x00, +0x00,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0x08,0x00,0x00,0x00, +0x47,0x00,0x03,0x00,0x67,0x01,0x00,0x00,0x02,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x69,0x01,0x00,0x00,0x22,0x00,0x00,0x00, 0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x69,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x6a,0x01,0x00,0x00,0x0b,0x00,0x00,0x00,0x19,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x92,0x01,0x00,0x00,0x06,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0x93,0x01,0x00,0x00, -0x00,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0x48,0x00,0x04,0x00, -0x93,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x93,0x01,0x00,0x00,0x00,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x93,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x07,0x00,0x00,0x00, -0x10,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x93,0x01,0x00,0x00, -0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x95,0x01,0x00,0x00, +0x21,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x69,0x02,0x00,0x00,0x0b,0x00,0x00,0x00,0x18,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0xaf,0x02,0x00,0x00,0x06,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0xb0,0x02,0x00,0x00, +0x00,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0xb0,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0xb0,0x02,0x00,0x00, +0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xb2,0x02,0x00,0x00, 0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x95,0x01,0x00,0x00,0x21,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0xcb,0x02,0x00,0x00,0x0b,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x11,0x03,0x00,0x00, -0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00, -0x12,0x03,0x00,0x00,0x00,0x00,0x00,0x00,0x19,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x12,0x03,0x00,0x00,0x00,0x00,0x00,0x00, +0xb2,0x02,0x00,0x00,0x21,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x13,0x00,0x02,0x00,0x02,0x00,0x00,0x00,0x21,0x00,0x03,0x00, +0x03,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x15,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x17,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x03,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x0a,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0x0a,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x0d,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x1e,0x00,0x10,0x00, +0x10,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x11,0x00,0x00,0x00,0x09,0x00,0x00,0x00, +0x10,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x11,0x00,0x00,0x00, +0x12,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x15,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x14,0x00,0x00,0x00, +0x08,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x15,0x00,0x00,0x00, +0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x0a,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x27,0x00,0x00,0x00, +0x09,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, +0x2d,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x37,0x00,0x00,0x00, +0x40,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x39,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0x0a,0x00,0x00,0x00,0x3d,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x3e,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00, +0x4c,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x53,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x54,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x37,0x00,0x00,0x00, +0x53,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x58,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x37,0x00,0x00,0x00, +0x53,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x60,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x61,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0x53,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x32,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x63,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0x61,0x00,0x00,0x00,0x62,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x67,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0x61,0x00,0x00,0x00,0x62,0x00,0x00,0x00, +0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, +0x10,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x6d,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x6e,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x73,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x78,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x7d,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0x81,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, +0x91,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0x97,0x00,0x00,0x00,0x03,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0xa2,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xa7,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0xa9,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xb8,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00, +0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xb9,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xba,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x53,0x00,0x00,0x00, +0xb9,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xbb,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x4f,0x00,0x00,0x00, +0x62,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xbc,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xbd,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0xbb,0x00,0x00,0x00,0xbc,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xbe,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0xbd,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xbf,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0xba,0x00,0x00,0x00,0xbe,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xc0,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0xb8,0x00,0x00,0x00,0xbf,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xc1,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0xc0,0x00,0x00,0x00,0xbc,0x00,0x00,0x00,0x14,0x00,0x02,0x00, +0xc2,0x00,0x00,0x00,0x16,0x00,0x03,0x00,0xc4,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xc5,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00, +0x62,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xc6,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0xc5,0x00,0x00,0x00, +0xbf,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xc7,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0xc6,0x00,0x00,0x00, +0xbc,0x00,0x00,0x00,0x1c,0x00,0x04,0x00,0xc8,0x00,0x00,0x00, +0xc4,0x00,0x00,0x00,0xc7,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0xc9,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0xc8,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0xc4,0x00,0x00,0x00,0xcc,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0xcd,0x00,0x00,0x00, +0x07,0x00,0x00,0x00,0xc4,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0xd0,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xf4,0x00,0x00,0x00, +0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0x16,0x00,0x03,0x00,0xf9,0x00,0x00,0x00,0x10,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xfa,0x00,0x00,0x00, +0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xfb,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0xfa,0x00,0x00,0x00, +0x1c,0x00,0x04,0x00,0xfc,0x00,0x00,0x00,0xf9,0x00,0x00,0x00, +0xfb,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0xfd,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0xfc,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0xfd,0x00,0x00,0x00,0xfe,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x17,0x00,0x04,0x00,0x00,0x01,0x00,0x00,0xf9,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x18,0x00,0x04,0x00,0x01,0x01,0x00,0x00, +0x00,0x01,0x00,0x00,0x02,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, +0x02,0x01,0x00,0x00,0x01,0x01,0x00,0x00,0x1e,0x00,0x03,0x00, +0x03,0x01,0x00,0x00,0x02,0x01,0x00,0x00,0x20,0x00,0x04,0x00, +0x04,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x03,0x01,0x00,0x00, +0x3b,0x00,0x04,0x00,0x04,0x01,0x00,0x00,0x05,0x01,0x00,0x00, +0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x07,0x01,0x00,0x00, +0x0c,0x00,0x00,0x00,0xf9,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x0a,0x01,0x00,0x00,0x04,0x00,0x00,0x00,0xf9,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x19,0x01,0x00,0x00, +0x03,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x20,0x01,0x00,0x00,0x04,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x27,0x01,0x00,0x00,0x05,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x2e,0x01,0x00,0x00, +0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x35,0x01,0x00,0x00,0x07,0x00,0x00,0x00,0x32,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x3b,0x01,0x00,0x00,0x01,0x00,0x00,0x00, +0x33,0x00,0x06,0x00,0x09,0x00,0x00,0x00,0x3c,0x01,0x00,0x00, +0x3b,0x01,0x00,0x00,0x39,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x3d,0x01,0x00,0x00, +0x51,0x00,0x00,0x00,0x3c,0x01,0x00,0x00,0x00,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x3e,0x01,0x00,0x00, +0x84,0x00,0x00,0x00,0x3d,0x01,0x00,0x00,0x6d,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x3f,0x01,0x00,0x00, +0x86,0x00,0x00,0x00,0x3e,0x01,0x00,0x00,0x6c,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x5a,0x01,0x00,0x00, +0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x5f,0x01,0x00,0x00, +0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x60,0x01,0x00,0x00, +0x84,0x00,0x00,0x00,0xa7,0x00,0x00,0x00,0x5f,0x01,0x00,0x00, +0x1c,0x00,0x04,0x00,0x61,0x01,0x00,0x00,0xf9,0x00,0x00,0x00, +0x60,0x01,0x00,0x00,0x20,0x00,0x04,0x00,0x62,0x01,0x00,0x00, +0x04,0x00,0x00,0x00,0x61,0x01,0x00,0x00,0x3b,0x00,0x04,0x00, +0x62,0x01,0x00,0x00,0x63,0x01,0x00,0x00,0x04,0x00,0x00,0x00, +0x1d,0x00,0x03,0x00,0x66,0x01,0x00,0x00,0x01,0x01,0x00,0x00, +0x1e,0x00,0x03,0x00,0x67,0x01,0x00,0x00,0x66,0x01,0x00,0x00, +0x20,0x00,0x04,0x00,0x68,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, +0x67,0x01,0x00,0x00,0x3b,0x00,0x04,0x00,0x68,0x01,0x00,0x00, +0x69,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x98,0x01,0x00,0x00,0x51,0x00,0x00,0x00, +0x3c,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x99,0x01,0x00,0x00,0x84,0x00,0x00,0x00, +0x98,0x01,0x00,0x00,0x6d,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x9a,0x01,0x00,0x00,0x86,0x00,0x00,0x00, +0x99,0x01,0x00,0x00,0x6c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x9d,0x01,0x00,0x00,0x08,0x01,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x9e,0x01,0x00,0x00, +0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xa1,0x01,0x00,0x00, +0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xbc,0x01,0x00,0x00, +0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00, +0x1c,0x00,0x04,0x00,0xbd,0x01,0x00,0x00,0xf9,0x00,0x00,0x00, +0xbc,0x01,0x00,0x00,0x20,0x00,0x04,0x00,0xbe,0x01,0x00,0x00, +0x07,0x00,0x00,0x00,0xbd,0x01,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xce,0x01,0x00,0x00,0x80,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0xd4,0x01,0x00,0x00,0x07,0x00,0x00,0x00,0xf9,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xea,0x01,0x00,0x00, +0x84,0x00,0x00,0x00,0xbf,0x00,0x00,0x00,0xbc,0x00,0x00,0x00, +0x1c,0x00,0x04,0x00,0xeb,0x01,0x00,0x00,0xf9,0x00,0x00,0x00, +0xea,0x01,0x00,0x00,0x20,0x00,0x04,0x00,0xec,0x01,0x00,0x00, +0x07,0x00,0x00,0x00,0xeb,0x01,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xf5,0x01,0x00,0x00,0x86,0x00,0x00,0x00, +0xb9,0x00,0x00,0x00,0xbf,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xfd,0x01,0x00,0x00,0x80,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x2c,0x02,0x00,0x00,0x84,0x00,0x00,0x00, +0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0x61,0x02,0x00,0x00,0x0d,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00,0x69,0x02,0x00,0x00, +0x01,0x00,0x00,0x00,0x1d,0x00,0x03,0x00,0xaf,0x02,0x00,0x00, +0xc4,0x00,0x00,0x00,0x1e,0x00,0x03,0x00,0xb0,0x02,0x00,0x00, +0xaf,0x02,0x00,0x00,0x20,0x00,0x04,0x00,0xb1,0x02,0x00,0x00, +0x0c,0x00,0x00,0x00,0xb0,0x02,0x00,0x00,0x3b,0x00,0x04,0x00, +0xb1,0x02,0x00,0x00,0xb2,0x02,0x00,0x00,0x0c,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0xb7,0x02,0x00,0x00, +0x05,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xc4,0x02,0x00,0x00,0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00, +0x62,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0xcd,0x02,0x00,0x00, +0x0c,0x00,0x00,0x00,0xc4,0x00,0x00,0x00,0x36,0x00,0x05,0x00, +0x02,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x03,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x05,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0xc9,0x00,0x00,0x00,0xca,0x00,0x00,0x00, +0x07,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0xbe,0x01,0x00,0x00, +0xbf,0x01,0x00,0x00,0x07,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0xec,0x01,0x00,0x00,0xed,0x01,0x00,0x00,0x07,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00,0x0e,0x00,0x00,0x00, +0x0b,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x0e,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x16,0x00,0x00,0x00, +0x12,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x17,0x00,0x00,0x00,0x16,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x18,0x00,0x00,0x00, +0x0f,0x00,0x00,0x00,0x17,0x00,0x00,0x00,0x89,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x1e,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, +0x17,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, +0x22,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x21,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x22,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x24,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x28,0x00,0x00,0x00, +0x12,0x00,0x00,0x00,0x27,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x29,0x00,0x00,0x00,0x28,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x2a,0x00,0x00,0x00, +0x1e,0x00,0x00,0x00,0x29,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x15,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0x2d,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x2f,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x30,0x00,0x00,0x00,0x24,0x00,0x00,0x00, +0x2f,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x32,0x00,0x00,0x00,0x30,0x00,0x00,0x00,0x2a,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x35,0x00,0x00,0x00, +0x12,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x36,0x00,0x00,0x00,0x35,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x38,0x00,0x00,0x00, +0x36,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x82,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x38,0x00,0x00,0x00, +0x39,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x3b,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x37,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00,0x3f,0x00,0x00,0x00, +0x3d,0x00,0x00,0x00,0x3e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x3f,0x00,0x00,0x00, +0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x42,0x00,0x00,0x00, +0x40,0x00,0x00,0x00,0x3b,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x47,0x00,0x00,0x00,0x40,0x00,0x00,0x00, +0x3b,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00, +0x49,0x00,0x00,0x00,0x3d,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x4a,0x00,0x00,0x00, +0x49,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00, +0x4d,0x00,0x00,0x00,0x4c,0x00,0x00,0x00,0x3e,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, +0x4d,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x50,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x4f,0x00,0x00,0x00, +0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x55,0x00,0x00,0x00, +0x50,0x00,0x00,0x00,0x54,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x59,0x00,0x00,0x00,0x50,0x00,0x00,0x00, +0x58,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x5d,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x4f,0x00,0x00,0x00, +0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x64,0x00,0x00,0x00, +0x5d,0x00,0x00,0x00,0x63,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x68,0x00,0x00,0x00,0x5d,0x00,0x00,0x00, +0x67,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x6f,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x6e,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x74,0x00,0x00,0x00, +0x4e,0x00,0x00,0x00,0x73,0x00,0x00,0x00,0x89,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x79,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, +0x78,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x7e,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x7d,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x82,0x00,0x00,0x00, +0x12,0x00,0x00,0x00,0x81,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x83,0x00,0x00,0x00,0x82,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0x47,0x00,0x00,0x00,0x83,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x15,0x00,0x00,0x00,0x87,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x88,0x00,0x00,0x00,0x87,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x8a,0x00,0x00,0x00,0x47,0x00,0x00,0x00, +0x39,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x8d,0x00,0x00,0x00,0x8a,0x00,0x00,0x00,0x83,0x00,0x00,0x00, +0x0c,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x8e,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x26,0x00,0x00,0x00,0x88,0x00,0x00,0x00, +0x8d,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, +0x92,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x91,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x93,0x00,0x00,0x00, +0x92,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x94,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x93,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x96,0x00,0x00,0x00, +0x42,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x15,0x00,0x00,0x00,0x98,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0x97,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x99,0x00,0x00,0x00,0x98,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x9a,0x00,0x00,0x00,0x96,0x00,0x00,0x00, +0x99,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x9b,0x00,0x00,0x00,0x94,0x00,0x00,0x00,0x9a,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9d,0x00,0x00,0x00, +0x9b,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x9e,0x00,0x00,0x00,0x9d,0x00,0x00,0x00, +0x6d,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, +0xa3,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0xa2,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xa4,0x00,0x00,0x00, +0xa3,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xa5,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0xa4,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa8,0x00,0x00,0x00, +0x4a,0x00,0x00,0x00,0xa7,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x15,0x00,0x00,0x00,0xaa,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0xa9,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xab,0x00,0x00,0x00,0xaa,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xac,0x00,0x00,0x00,0xa8,0x00,0x00,0x00, +0xab,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xad,0x00,0x00,0x00,0xa5,0x00,0x00,0x00,0xac,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xaf,0x00,0x00,0x00, +0xad,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xb0,0x00,0x00,0x00,0xaf,0x00,0x00,0x00, +0x6d,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xb2,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xb2,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xd7,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, +0x05,0x00,0x00,0x00,0xd1,0x00,0x00,0x00,0xb3,0x00,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0xc3,0x00,0x00,0x00, +0xd7,0x02,0x00,0x00,0xc1,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0xb4,0x00,0x00,0x00,0xb3,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0xc3,0x00,0x00,0x00,0xb3,0x00,0x00,0x00, +0xb4,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xb3,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0xcd,0x00,0x00,0x00,0xce,0x00,0x00,0x00, +0xca,0x00,0x00,0x00,0xd7,0x02,0x00,0x00,0x3e,0x00,0x03,0x00, +0xce,0x00,0x00,0x00,0xcc,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xd1,0x00,0x00,0x00,0xd7,0x02,0x00,0x00, +0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xb2,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xb4,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xd4,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xd4,0x00,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xf0,0x02,0x00,0x00, +0xb0,0x00,0x00,0x00,0xb4,0x00,0x00,0x00,0xa3,0x01,0x00,0x00, +0xd7,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xec,0x02,0x00,0x00,0x9e,0x00,0x00,0x00,0xb4,0x00,0x00,0x00, +0xa0,0x01,0x00,0x00,0xd7,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xd8,0x02,0x00,0x00,0x84,0x00,0x00,0x00, +0xb4,0x00,0x00,0x00,0x51,0x02,0x00,0x00,0xd7,0x00,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0xdb,0x00,0x00,0x00, +0xd8,0x02,0x00,0x00,0x8e,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0xd6,0x00,0x00,0x00,0xd7,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0xdb,0x00,0x00,0x00,0xd5,0x00,0x00,0x00, +0xd6,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xd5,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xdd,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xdd,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xe8,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0xd5,0x00,0x00,0x00, +0x41,0x01,0x00,0x00,0xde,0x00,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc2,0x00,0x00,0x00,0xe3,0x00,0x00,0x00,0xe8,0x02,0x00,0x00, +0x37,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xdf,0x00,0x00,0x00, +0xde,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0xe3,0x00,0x00,0x00,0xde,0x00,0x00,0x00,0xdf,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xde,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xe8,0x00,0x00,0x00,0x74,0x00,0x00,0x00, +0xe8,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xeb,0x00,0x00,0x00,0xe8,0x00,0x00,0x00,0x99,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xec,0x00,0x00,0x00, +0xeb,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xed,0x00,0x00,0x00,0xec,0x02,0x00,0x00, +0xec,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xef,0x00,0x00,0x00,0xed,0x00,0x00,0x00,0x6f,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xf5,0x00,0x00,0x00, +0xe8,0x00,0x00,0x00,0xf4,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xf7,0x00,0x00,0x00,0x6f,0x00,0x00,0x00, +0x6d,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xf8,0x00,0x00,0x00,0xf5,0x00,0x00,0x00,0xf7,0x00,0x00,0x00, +0x41,0x00,0x08,0x00,0x07,0x01,0x00,0x00,0x08,0x01,0x00,0x00, +0x05,0x01,0x00,0x00,0x34,0x00,0x00,0x00,0xef,0x00,0x00,0x00, +0x34,0x00,0x00,0x00,0x3e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0xf9,0x00,0x00,0x00,0x09,0x01,0x00,0x00,0x08,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0x0a,0x01,0x00,0x00,0x0b,0x01,0x00,0x00, +0xfe,0x00,0x00,0x00,0xf8,0x00,0x00,0x00,0x3e,0x00,0x03,0x00, +0x0b,0x01,0x00,0x00,0x09,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x0d,0x01,0x00,0x00,0xf8,0x00,0x00,0x00, +0x39,0x00,0x00,0x00,0x41,0x00,0x08,0x00,0x07,0x01,0x00,0x00, +0x0f,0x01,0x00,0x00,0x05,0x01,0x00,0x00,0x34,0x00,0x00,0x00, +0xef,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0xf9,0x00,0x00,0x00,0x10,0x01,0x00,0x00, +0x0f,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0x0a,0x01,0x00,0x00, +0x11,0x01,0x00,0x00,0xfe,0x00,0x00,0x00,0x0d,0x01,0x00,0x00, +0x3e,0x00,0x03,0x00,0x11,0x01,0x00,0x00,0x10,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x13,0x01,0x00,0x00, +0xf8,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x41,0x00,0x08,0x00, +0x07,0x01,0x00,0x00,0x15,0x01,0x00,0x00,0x05,0x01,0x00,0x00, +0x34,0x00,0x00,0x00,0xef,0x00,0x00,0x00,0x34,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0xf9,0x00,0x00,0x00, +0x16,0x01,0x00,0x00,0x15,0x01,0x00,0x00,0x41,0x00,0x05,0x00, +0x0a,0x01,0x00,0x00,0x17,0x01,0x00,0x00,0xfe,0x00,0x00,0x00, +0x13,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x17,0x01,0x00,0x00, +0x16,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x1a,0x01,0x00,0x00,0xf8,0x00,0x00,0x00,0x19,0x01,0x00,0x00, +0x41,0x00,0x08,0x00,0x07,0x01,0x00,0x00,0x1c,0x01,0x00,0x00, +0x05,0x01,0x00,0x00,0x34,0x00,0x00,0x00,0xef,0x00,0x00,0x00, +0x34,0x00,0x00,0x00,0x19,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, +0xf9,0x00,0x00,0x00,0x1d,0x01,0x00,0x00,0x1c,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0x0a,0x01,0x00,0x00,0x1e,0x01,0x00,0x00, +0xfe,0x00,0x00,0x00,0x1a,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, +0x1e,0x01,0x00,0x00,0x1d,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x21,0x01,0x00,0x00,0xf8,0x00,0x00,0x00, +0x20,0x01,0x00,0x00,0x41,0x00,0x08,0x00,0x07,0x01,0x00,0x00, +0x23,0x01,0x00,0x00,0x05,0x01,0x00,0x00,0x34,0x00,0x00,0x00, +0xef,0x00,0x00,0x00,0xd0,0x00,0x00,0x00,0x3e,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0xf9,0x00,0x00,0x00,0x24,0x01,0x00,0x00, +0x23,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0x0a,0x01,0x00,0x00, +0x25,0x01,0x00,0x00,0xfe,0x00,0x00,0x00,0x21,0x01,0x00,0x00, +0x3e,0x00,0x03,0x00,0x25,0x01,0x00,0x00,0x24,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x28,0x01,0x00,0x00, +0xf8,0x00,0x00,0x00,0x27,0x01,0x00,0x00,0x41,0x00,0x08,0x00, +0x07,0x01,0x00,0x00,0x2a,0x01,0x00,0x00,0x05,0x01,0x00,0x00, +0x34,0x00,0x00,0x00,0xef,0x00,0x00,0x00,0xd0,0x00,0x00,0x00, +0x39,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0xf9,0x00,0x00,0x00, +0x2b,0x01,0x00,0x00,0x2a,0x01,0x00,0x00,0x41,0x00,0x05,0x00, +0x0a,0x01,0x00,0x00,0x2c,0x01,0x00,0x00,0xfe,0x00,0x00,0x00, +0x28,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x2c,0x01,0x00,0x00, +0x2b,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x2f,0x01,0x00,0x00,0xf8,0x00,0x00,0x00,0x2e,0x01,0x00,0x00, +0x41,0x00,0x08,0x00,0x07,0x01,0x00,0x00,0x31,0x01,0x00,0x00, +0x05,0x01,0x00,0x00,0x34,0x00,0x00,0x00,0xef,0x00,0x00,0x00, +0xd0,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0xf9,0x00,0x00,0x00,0x32,0x01,0x00,0x00,0x31,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0x0a,0x01,0x00,0x00,0x33,0x01,0x00,0x00, +0xfe,0x00,0x00,0x00,0x2f,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, +0x33,0x01,0x00,0x00,0x32,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x36,0x01,0x00,0x00,0xf8,0x00,0x00,0x00, +0x35,0x01,0x00,0x00,0x41,0x00,0x08,0x00,0x07,0x01,0x00,0x00, +0x38,0x01,0x00,0x00,0x05,0x01,0x00,0x00,0x34,0x00,0x00,0x00, +0xef,0x00,0x00,0x00,0xd0,0x00,0x00,0x00,0x19,0x01,0x00,0x00, +0x3d,0x00,0x04,0x00,0xf9,0x00,0x00,0x00,0x39,0x01,0x00,0x00, +0x38,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0x0a,0x01,0x00,0x00, +0x3a,0x01,0x00,0x00,0xfe,0x00,0x00,0x00,0x36,0x01,0x00,0x00, +0x3e,0x00,0x03,0x00,0x3a,0x01,0x00,0x00,0x39,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x41,0x01,0x00,0x00, +0xe8,0x02,0x00,0x00,0x3f,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0xdd,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xdf,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x43,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x43,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xe9,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0xdf,0x00,0x00,0x00, +0x9c,0x01,0x00,0x00,0x44,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc2,0x00,0x00,0x00,0x49,0x01,0x00,0x00,0xe9,0x02,0x00,0x00, +0xa7,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x45,0x01,0x00,0x00, +0x44,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x49,0x01,0x00,0x00,0x44,0x01,0x00,0x00,0x45,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x44,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x4e,0x01,0x00,0x00,0x7e,0x00,0x00,0x00, +0xe9,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x51,0x01,0x00,0x00,0x4e,0x01,0x00,0x00,0xab,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x52,0x01,0x00,0x00, +0x51,0x01,0x00,0x00,0x6d,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x53,0x01,0x00,0x00,0xf0,0x02,0x00,0x00, +0x52,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x55,0x01,0x00,0x00,0x53,0x01,0x00,0x00,0x79,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x5b,0x01,0x00,0x00, +0x4e,0x01,0x00,0x00,0x5a,0x01,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x5d,0x01,0x00,0x00,0x79,0x00,0x00,0x00, +0x6d,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x5e,0x01,0x00,0x00,0x5b,0x01,0x00,0x00,0x5d,0x01,0x00,0x00, +0x41,0x00,0x08,0x00,0x07,0x01,0x00,0x00,0x6b,0x01,0x00,0x00, +0x69,0x01,0x00,0x00,0x34,0x00,0x00,0x00,0x55,0x01,0x00,0x00, +0x34,0x00,0x00,0x00,0x3e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0xf9,0x00,0x00,0x00,0x6c,0x01,0x00,0x00,0x6b,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0x0a,0x01,0x00,0x00,0x6d,0x01,0x00,0x00, +0x63,0x01,0x00,0x00,0x5e,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, +0x6d,0x01,0x00,0x00,0x6c,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x6f,0x01,0x00,0x00,0x5e,0x01,0x00,0x00, +0x39,0x00,0x00,0x00,0x41,0x00,0x08,0x00,0x07,0x01,0x00,0x00, +0x71,0x01,0x00,0x00,0x69,0x01,0x00,0x00,0x34,0x00,0x00,0x00, +0x55,0x01,0x00,0x00,0x34,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0xf9,0x00,0x00,0x00,0x72,0x01,0x00,0x00, +0x71,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0x0a,0x01,0x00,0x00, +0x73,0x01,0x00,0x00,0x63,0x01,0x00,0x00,0x6f,0x01,0x00,0x00, +0x3e,0x00,0x03,0x00,0x73,0x01,0x00,0x00,0x72,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x75,0x01,0x00,0x00, +0x5e,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x41,0x00,0x08,0x00, +0x07,0x01,0x00,0x00,0x77,0x01,0x00,0x00,0x69,0x01,0x00,0x00, +0x34,0x00,0x00,0x00,0x55,0x01,0x00,0x00,0x34,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0xf9,0x00,0x00,0x00, +0x78,0x01,0x00,0x00,0x77,0x01,0x00,0x00,0x41,0x00,0x05,0x00, +0x0a,0x01,0x00,0x00,0x79,0x01,0x00,0x00,0x63,0x01,0x00,0x00, +0x75,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x79,0x01,0x00,0x00, +0x78,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x7b,0x01,0x00,0x00,0x5e,0x01,0x00,0x00,0x19,0x01,0x00,0x00, +0x41,0x00,0x08,0x00,0x07,0x01,0x00,0x00,0x7d,0x01,0x00,0x00, +0x69,0x01,0x00,0x00,0x34,0x00,0x00,0x00,0x55,0x01,0x00,0x00, +0x34,0x00,0x00,0x00,0x19,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, +0xf9,0x00,0x00,0x00,0x7e,0x01,0x00,0x00,0x7d,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0x0a,0x01,0x00,0x00,0x7f,0x01,0x00,0x00, +0x63,0x01,0x00,0x00,0x7b,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, +0x7f,0x01,0x00,0x00,0x7e,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x81,0x01,0x00,0x00,0x5e,0x01,0x00,0x00, +0x20,0x01,0x00,0x00,0x41,0x00,0x08,0x00,0x07,0x01,0x00,0x00, +0x83,0x01,0x00,0x00,0x69,0x01,0x00,0x00,0x34,0x00,0x00,0x00, +0x55,0x01,0x00,0x00,0xd0,0x00,0x00,0x00,0x3e,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0xf9,0x00,0x00,0x00,0x84,0x01,0x00,0x00, +0x83,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0x0a,0x01,0x00,0x00, +0x85,0x01,0x00,0x00,0x63,0x01,0x00,0x00,0x81,0x01,0x00,0x00, +0x3e,0x00,0x03,0x00,0x85,0x01,0x00,0x00,0x84,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x87,0x01,0x00,0x00, +0x5e,0x01,0x00,0x00,0x27,0x01,0x00,0x00,0x41,0x00,0x08,0x00, +0x07,0x01,0x00,0x00,0x89,0x01,0x00,0x00,0x69,0x01,0x00,0x00, +0x34,0x00,0x00,0x00,0x55,0x01,0x00,0x00,0xd0,0x00,0x00,0x00, +0x39,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0xf9,0x00,0x00,0x00, +0x8a,0x01,0x00,0x00,0x89,0x01,0x00,0x00,0x41,0x00,0x05,0x00, +0x0a,0x01,0x00,0x00,0x8b,0x01,0x00,0x00,0x63,0x01,0x00,0x00, +0x87,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x8b,0x01,0x00,0x00, +0x8a,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x8d,0x01,0x00,0x00,0x5e,0x01,0x00,0x00,0x2e,0x01,0x00,0x00, +0x41,0x00,0x08,0x00,0x07,0x01,0x00,0x00,0x8f,0x01,0x00,0x00, +0x69,0x01,0x00,0x00,0x34,0x00,0x00,0x00,0x55,0x01,0x00,0x00, +0xd0,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0xf9,0x00,0x00,0x00,0x90,0x01,0x00,0x00,0x8f,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0x0a,0x01,0x00,0x00,0x91,0x01,0x00,0x00, +0x63,0x01,0x00,0x00,0x8d,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, +0x91,0x01,0x00,0x00,0x90,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x93,0x01,0x00,0x00,0x5e,0x01,0x00,0x00, +0x35,0x01,0x00,0x00,0x41,0x00,0x08,0x00,0x07,0x01,0x00,0x00, +0x95,0x01,0x00,0x00,0x69,0x01,0x00,0x00,0x34,0x00,0x00,0x00, +0x55,0x01,0x00,0x00,0xd0,0x00,0x00,0x00,0x19,0x01,0x00,0x00, +0x3d,0x00,0x04,0x00,0xf9,0x00,0x00,0x00,0x96,0x01,0x00,0x00, +0x95,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0x0a,0x01,0x00,0x00, +0x97,0x01,0x00,0x00,0x63,0x01,0x00,0x00,0x93,0x01,0x00,0x00, +0x3e,0x00,0x03,0x00,0x97,0x01,0x00,0x00,0x96,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9c,0x01,0x00,0x00, +0xe9,0x02,0x00,0x00,0x9a,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0x43,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x45,0x01,0x00,0x00, +0xe0,0x00,0x04,0x00,0x0c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x9d,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xa0,0x01,0x00,0x00,0xec,0x02,0x00,0x00,0x9e,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa3,0x01,0x00,0x00, +0xf0,0x02,0x00,0x00,0xa1,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0xa5,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xa5,0x01,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xf2,0x02,0x00,0x00, +0x3e,0x00,0x00,0x00,0x45,0x01,0x00,0x00,0x4f,0x02,0x00,0x00, +0xa8,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, +0xab,0x01,0x00,0x00,0xf2,0x02,0x00,0x00,0x6c,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0xa7,0x01,0x00,0x00,0xa8,0x01,0x00,0x00, +0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xab,0x01,0x00,0x00, +0xa6,0x01,0x00,0x00,0xa7,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xa6,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xad,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xad,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xf6,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, +0xa6,0x01,0x00,0x00,0xd9,0x01,0x00,0x00,0xb0,0x01,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0xb3,0x01,0x00,0x00, +0xf6,0x02,0x00,0x00,0x60,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0xaf,0x01,0x00,0x00,0xb0,0x01,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0xb3,0x01,0x00,0x00,0xae,0x01,0x00,0x00, +0xaf,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xae,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0xb5,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xb5,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0x08,0x03,0x00,0x00,0x3e,0x00,0x00,0x00,0xae,0x01,0x00,0x00, +0xd7,0x01,0x00,0x00,0xb6,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc2,0x00,0x00,0x00,0xbb,0x01,0x00,0x00,0x08,0x03,0x00,0x00, +0x62,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xb7,0x01,0x00,0x00, +0xb6,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0xbb,0x01,0x00,0x00,0xb6,0x01,0x00,0x00,0xb7,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xb6,0x01,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xc1,0x01,0x00,0x00,0xf6,0x02,0x00,0x00, +0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xc3,0x01,0x00,0x00,0xc1,0x01,0x00,0x00,0x08,0x03,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xc5,0x01,0x00,0x00, +0x55,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xc7,0x01,0x00,0x00,0xf6,0x02,0x00,0x00, +0x61,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xc8,0x01,0x00,0x00,0xc5,0x01,0x00,0x00,0xc7,0x01,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xca,0x01,0x00,0x00, +0x64,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xcb,0x01,0x00,0x00,0xc8,0x01,0x00,0x00, +0xca,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xcd,0x01,0x00,0x00,0xcb,0x01,0x00,0x00,0x08,0x03,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xcf,0x01,0x00,0x00, +0xcd,0x01,0x00,0x00,0xce,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xd1,0x01,0x00,0x00,0xcf,0x01,0x00,0x00, +0xf2,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x0a,0x01,0x00,0x00, +0xd2,0x01,0x00,0x00,0xfe,0x00,0x00,0x00,0xd1,0x01,0x00,0x00, +0x3d,0x00,0x04,0x00,0xf9,0x00,0x00,0x00,0xd3,0x01,0x00,0x00, +0xd2,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0xd4,0x01,0x00,0x00, +0xd5,0x01,0x00,0x00,0xbf,0x01,0x00,0x00,0xc3,0x01,0x00,0x00, +0x3e,0x00,0x03,0x00,0xd5,0x01,0x00,0x00,0xd3,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xd7,0x01,0x00,0x00, +0x08,0x03,0x00,0x00,0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xb5,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xb7,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0xb0,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xb0,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xd9,0x01,0x00,0x00,0xf6,0x02,0x00,0x00,0xd0,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xad,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xaf,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xdb,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xdb,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xf7,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, +0xaf,0x01,0x00,0x00,0x07,0x02,0x00,0x00,0xde,0x01,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0xe1,0x01,0x00,0x00, +0xf7,0x02,0x00,0x00,0xbf,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0xdd,0x01,0x00,0x00,0xde,0x01,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0xe1,0x01,0x00,0x00,0xdc,0x01,0x00,0x00, +0xdd,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xdc,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0xe3,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xe3,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0x05,0x03,0x00,0x00,0x3e,0x00,0x00,0x00,0xdc,0x01,0x00,0x00, +0x05,0x02,0x00,0x00,0xe4,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc2,0x00,0x00,0x00,0xe9,0x01,0x00,0x00,0x05,0x03,0x00,0x00, +0xbc,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xe5,0x01,0x00,0x00, +0xe4,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0xe9,0x01,0x00,0x00,0xe4,0x01,0x00,0x00,0xe5,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xe4,0x01,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xef,0x01,0x00,0x00,0xf7,0x02,0x00,0x00, +0xbc,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xf1,0x01,0x00,0x00,0xef,0x01,0x00,0x00,0x05,0x03,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xf3,0x01,0x00,0x00, +0x59,0x00,0x00,0x00,0xb9,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xf6,0x01,0x00,0x00,0xf7,0x02,0x00,0x00, +0xf5,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xf7,0x01,0x00,0x00,0xf3,0x01,0x00,0x00,0xf6,0x01,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xf9,0x01,0x00,0x00, +0x68,0x00,0x00,0x00,0xbc,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xfa,0x01,0x00,0x00,0xf7,0x01,0x00,0x00, +0xf9,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xfc,0x01,0x00,0x00,0xfa,0x01,0x00,0x00,0x05,0x03,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xfe,0x01,0x00,0x00, +0xfc,0x01,0x00,0x00,0xfd,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x00,0x02,0x00,0x00,0xfe,0x01,0x00,0x00, +0xf2,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x0a,0x01,0x00,0x00, +0x01,0x02,0x00,0x00,0x63,0x01,0x00,0x00,0x00,0x02,0x00,0x00, +0x3d,0x00,0x04,0x00,0xf9,0x00,0x00,0x00,0x02,0x02,0x00,0x00, +0x01,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0xd4,0x01,0x00,0x00, +0x03,0x02,0x00,0x00,0xed,0x01,0x00,0x00,0xf1,0x01,0x00,0x00, +0x3e,0x00,0x03,0x00,0x03,0x02,0x00,0x00,0x02,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x05,0x02,0x00,0x00, +0x05,0x03,0x00,0x00,0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xe3,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xe5,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0xde,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xde,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x07,0x02,0x00,0x00,0xf7,0x02,0x00,0x00,0xd0,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xdb,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xdd,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x09,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x09,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xf8,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, +0xdd,0x01,0x00,0x00,0x4d,0x02,0x00,0x00,0x0c,0x02,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0x0f,0x02,0x00,0x00, +0xf8,0x02,0x00,0x00,0xbf,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0x0b,0x02,0x00,0x00,0x0c,0x02,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x0f,0x02,0x00,0x00,0x0a,0x02,0x00,0x00, +0x0b,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x0a,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0x11,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x11,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xfc,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0x0a,0x02,0x00,0x00, +0x4b,0x02,0x00,0x00,0x14,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc2,0x00,0x00,0x00,0x17,0x02,0x00,0x00,0xfc,0x02,0x00,0x00, +0x60,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x13,0x02,0x00,0x00, +0x14,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x17,0x02,0x00,0x00,0x12,0x02,0x00,0x00,0x13,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x12,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0x19,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x19,0x02,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xfe,0x02,0x00,0x00, +0x3e,0x00,0x00,0x00,0x12,0x02,0x00,0x00,0x49,0x02,0x00,0x00, +0x1c,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, +0x1f,0x02,0x00,0x00,0xfe,0x02,0x00,0x00,0xbc,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0x1b,0x02,0x00,0x00,0x1c,0x02,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x1f,0x02,0x00,0x00, +0x1a,0x02,0x00,0x00,0x1b,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x1a,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x21,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x21,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0x00,0x03,0x00,0x00,0x3e,0x00,0x00,0x00, +0x1a,0x02,0x00,0x00,0x47,0x02,0x00,0x00,0x22,0x02,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0x27,0x02,0x00,0x00, +0x00,0x03,0x00,0x00,0x62,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0x23,0x02,0x00,0x00,0x22,0x02,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x27,0x02,0x00,0x00,0x22,0x02,0x00,0x00, +0x23,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x22,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x29,0x02,0x00,0x00, +0xf8,0x02,0x00,0x00,0xbc,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x2b,0x02,0x00,0x00,0x29,0x02,0x00,0x00, +0xfe,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x2d,0x02,0x00,0x00,0x2b,0x02,0x00,0x00,0x2c,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x2f,0x02,0x00,0x00, +0xfc,0x02,0x00,0x00,0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x30,0x02,0x00,0x00,0x2d,0x02,0x00,0x00, +0x2f,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x32,0x02,0x00,0x00,0x30,0x02,0x00,0x00,0x00,0x03,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x36,0x02,0x00,0x00, +0x2f,0x02,0x00,0x00,0x00,0x03,0x00,0x00,0x41,0x00,0x05,0x00, +0xd4,0x01,0x00,0x00,0x37,0x02,0x00,0x00,0xbf,0x01,0x00,0x00, +0x36,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0xf9,0x00,0x00,0x00, +0x38,0x02,0x00,0x00,0x37,0x02,0x00,0x00,0x73,0x00,0x04,0x00, +0xc4,0x00,0x00,0x00,0x39,0x02,0x00,0x00,0x38,0x02,0x00,0x00, +0x41,0x00,0x05,0x00,0xd4,0x01,0x00,0x00,0x3e,0x02,0x00,0x00, +0xed,0x01,0x00,0x00,0x2b,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, +0xf9,0x00,0x00,0x00,0x3f,0x02,0x00,0x00,0x3e,0x02,0x00,0x00, +0x73,0x00,0x04,0x00,0xc4,0x00,0x00,0x00,0x40,0x02,0x00,0x00, +0x3f,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0xcd,0x00,0x00,0x00, +0x42,0x02,0x00,0x00,0xca,0x00,0x00,0x00,0x32,0x02,0x00,0x00, +0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00,0x43,0x02,0x00,0x00, +0x42,0x02,0x00,0x00,0x0c,0x00,0x08,0x00,0xc4,0x00,0x00,0x00, +0x44,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00, +0x39,0x02,0x00,0x00,0x40,0x02,0x00,0x00,0x43,0x02,0x00,0x00, +0x3e,0x00,0x03,0x00,0x42,0x02,0x00,0x00,0x44,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x47,0x02,0x00,0x00, +0x00,0x03,0x00,0x00,0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x21,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x23,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0x1c,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x1c,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x49,0x02,0x00,0x00,0xfe,0x02,0x00,0x00,0xd0,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x19,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x1b,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x14,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x14,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x4b,0x02,0x00,0x00,0xfc,0x02,0x00,0x00, +0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x11,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x13,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0x0c,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x0c,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x4d,0x02,0x00,0x00, +0xf8,0x02,0x00,0x00,0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x09,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x0b,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0xa8,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xa8,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x4f,0x02,0x00,0x00,0xf2,0x02,0x00,0x00,0xd0,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xa5,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xa7,0x01,0x00,0x00,0xe0,0x00,0x04,0x00,0x0c,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x9d,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0xd7,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xd7,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x51,0x02,0x00,0x00, +0xd8,0x02,0x00,0x00,0x6c,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xd4,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xd6,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x56,0x02,0x00,0x00, +0x55,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x57,0x02,0x00,0x00,0x96,0x00,0x00,0x00, +0x56,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x5c,0x02,0x00,0x00,0x59,0x00,0x00,0x00,0xb9,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x5d,0x02,0x00,0x00, +0xa8,0x00,0x00,0x00,0x5c,0x02,0x00,0x00,0x41,0x00,0x05,0x00, +0x15,0x00,0x00,0x00,0x62,0x02,0x00,0x00,0x12,0x00,0x00,0x00, +0x61,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x63,0x02,0x00,0x00,0x62,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x64,0x02,0x00,0x00,0x0f,0x00,0x00,0x00, +0x63,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x68,0x02,0x00,0x00,0x47,0x00,0x00,0x00,0x63,0x02,0x00,0x00, +0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00,0x6a,0x02,0x00,0x00, +0x69,0x02,0x00,0x00,0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x6b,0x02,0x00,0x00,0x6a,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x6c,0x02,0x00,0x00, +0x68,0x02,0x00,0x00,0x6b,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x6d,0x02,0x00,0x00,0x64,0x02,0x00,0x00, +0x6c,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x6f,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x6f,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xd9,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, +0xd6,0x00,0x00,0x00,0xd6,0x02,0x00,0x00,0x72,0x02,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0x75,0x02,0x00,0x00, +0xd9,0x02,0x00,0x00,0xbf,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0x71,0x02,0x00,0x00,0x72,0x02,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x75,0x02,0x00,0x00,0x70,0x02,0x00,0x00, +0x71,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x70,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0x77,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x77,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xda,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0x70,0x02,0x00,0x00, +0xd4,0x02,0x00,0x00,0x7a,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc2,0x00,0x00,0x00,0x7d,0x02,0x00,0x00,0xda,0x02,0x00,0x00, +0x60,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x79,0x02,0x00,0x00, +0x7a,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x7d,0x02,0x00,0x00,0x78,0x02,0x00,0x00,0x79,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x78,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x81,0x02,0x00,0x00,0xda,0x02,0x00,0x00, +0x61,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x82,0x02,0x00,0x00,0x57,0x02,0x00,0x00,0x81,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x84,0x02,0x00,0x00, +0x64,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x85,0x02,0x00,0x00,0x82,0x02,0x00,0x00, +0x84,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x89,0x02,0x00,0x00,0xd9,0x02,0x00,0x00,0xf5,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8a,0x02,0x00,0x00, +0x5d,0x02,0x00,0x00,0x89,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x8c,0x02,0x00,0x00,0x68,0x00,0x00,0x00, +0xbc,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x8d,0x02,0x00,0x00,0x8a,0x02,0x00,0x00,0x8c,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0x8f,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x8f,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xdc,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0x78,0x02,0x00,0x00, +0xd2,0x02,0x00,0x00,0x92,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc2,0x00,0x00,0x00,0x95,0x02,0x00,0x00,0xdc,0x02,0x00,0x00, +0xbc,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x91,0x02,0x00,0x00, +0x92,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x95,0x02,0x00,0x00,0x90,0x02,0x00,0x00,0x91,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x90,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0x97,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x97,0x02,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xde,0x02,0x00,0x00, +0x3e,0x00,0x00,0x00,0x90,0x02,0x00,0x00,0xd0,0x02,0x00,0x00, +0x9a,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, +0x9d,0x02,0x00,0x00,0xde,0x02,0x00,0x00,0x62,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0x99,0x02,0x00,0x00,0x9a,0x02,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x9d,0x02,0x00,0x00, +0x98,0x02,0x00,0x00,0x99,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x98,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xa0,0x02,0x00,0x00,0x85,0x02,0x00,0x00,0xde,0x02,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0xa3,0x02,0x00,0x00, +0xa0,0x02,0x00,0x00,0x36,0x00,0x00,0x00,0xf7,0x00,0x03,0x00, +0xa5,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0xa3,0x02,0x00,0x00,0xa4,0x02,0x00,0x00,0xa5,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0xa4,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xa8,0x02,0x00,0x00,0x8d,0x02,0x00,0x00, +0xdc,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, +0xa9,0x02,0x00,0x00,0x12,0x00,0x00,0x00,0xd0,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xaa,0x02,0x00,0x00, +0xa9,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, +0xab,0x02,0x00,0x00,0xa8,0x02,0x00,0x00,0xaa,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0xa5,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0xa5,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0xc2,0x00,0x00,0x00, +0xac,0x02,0x00,0x00,0xa3,0x02,0x00,0x00,0x98,0x02,0x00,0x00, +0xab,0x02,0x00,0x00,0xa4,0x02,0x00,0x00,0xf7,0x00,0x03,0x00, +0xae,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0xac,0x02,0x00,0x00,0xad,0x02,0x00,0x00,0xae,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0xad,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xb6,0x02,0x00,0x00,0x8d,0x02,0x00,0x00, +0xdc,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, +0xb8,0x02,0x00,0x00,0x12,0x00,0x00,0x00,0xb7,0x02,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xb9,0x02,0x00,0x00, +0xb8,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xba,0x02,0x00,0x00,0xb6,0x02,0x00,0x00,0xb9,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xbb,0x02,0x00,0x00, +0x6d,0x02,0x00,0x00,0xba,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xbd,0x02,0x00,0x00,0xbb,0x02,0x00,0x00, +0x85,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xbf,0x02,0x00,0x00,0xbd,0x02,0x00,0x00,0xde,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xc1,0x02,0x00,0x00, +0xd9,0x02,0x00,0x00,0xbc,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xc3,0x02,0x00,0x00,0xc1,0x02,0x00,0x00, +0xdc,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xc5,0x02,0x00,0x00,0xc3,0x02,0x00,0x00,0xc4,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xc7,0x02,0x00,0x00, +0xda,0x02,0x00,0x00,0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xc8,0x02,0x00,0x00,0xc5,0x02,0x00,0x00, +0xc7,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xca,0x02,0x00,0x00,0xc8,0x02,0x00,0x00,0xde,0x02,0x00,0x00, +0x41,0x00,0x05,0x00,0xcd,0x00,0x00,0x00,0xcb,0x02,0x00,0x00, +0xca,0x00,0x00,0x00,0xca,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, +0xc4,0x00,0x00,0x00,0xcc,0x02,0x00,0x00,0xcb,0x02,0x00,0x00, +0x41,0x00,0x06,0x00,0xcd,0x02,0x00,0x00,0xce,0x02,0x00,0x00, +0xb2,0x02,0x00,0x00,0x34,0x00,0x00,0x00,0xbf,0x02,0x00,0x00, +0x3e,0x00,0x03,0x00,0xce,0x02,0x00,0x00,0xcc,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0xae,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0xae,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x9a,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x9a,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xd0,0x02,0x00,0x00,0xde,0x02,0x00,0x00, +0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x97,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x99,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0x92,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x92,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xd2,0x02,0x00,0x00, +0xdc,0x02,0x00,0x00,0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x8f,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x91,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0x7a,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x7a,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xd4,0x02,0x00,0x00,0xda,0x02,0x00,0x00,0xd0,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x77,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x79,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x72,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x72,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xd6,0x02,0x00,0x00,0xd9,0x02,0x00,0x00, +0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x6f,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x71,0x02,0x00,0x00,0xfd,0x00,0x01,0x00, +0x38,0x00,0x01,0x00, +}; +const uint64_t matmul_f16_aligned_len = 11200; + +unsigned char matmul_f16_aligned_fp32_data[] = { +0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, +0xd9,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, +0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x51,0x11,0x00,0x00, +0x0b,0x00,0x06,0x00,0x01,0x00,0x00,0x00,0x47,0x4c,0x53,0x4c, +0x2e,0x73,0x74,0x64,0x2e,0x34,0x35,0x30,0x00,0x00,0x00,0x00, +0x0e,0x00,0x03,0x00,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x0f,0x00,0x0f,0x00,0x05,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x6d,0x61,0x69,0x6e,0x00,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, +0x12,0x00,0x00,0x00,0x3d,0x00,0x00,0x00,0x4c,0x00,0x00,0x00, +0xfd,0x00,0x00,0x00,0x04,0x01,0x00,0x00,0x4a,0x01,0x00,0x00, +0x50,0x01,0x00,0x00,0x39,0x02,0x00,0x00,0x82,0x02,0x00,0x00, +0x10,0x00,0x06,0x00,0x04,0x00,0x00,0x00,0x11,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x0b,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, +0x1c,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x10,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x08,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, +0x03,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x10,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x14,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x18,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x07,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x10,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, +0x09,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x24,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x0a,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x10,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x2c,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x30,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x0d,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0x47,0x00,0x03,0x00, +0x10,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x37,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x3d,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, +0x1a,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x4c,0x00,0x00,0x00, +0x0b,0x00,0x00,0x00,0x1b,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x4f,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x09,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x53,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x60,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x62,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x07,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x6c,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x03,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xa7,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0xb9,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x05,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0xbc,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x08,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x01,0x01,0x00,0x00, +0x06,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x48,0x00,0x04,0x00, +0x02,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x02,0x01,0x00,0x00,0x00,0x00,0x00,0x00, 0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, -0x12,0x03,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x14,0x03,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x14,0x03,0x00,0x00,0x21,0x00,0x00,0x00, +0x02,0x01,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x04,0x01,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x04,0x01,0x00,0x00,0x21,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x22,0x01,0x00,0x00, +0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x23,0x01,0x00,0x00,0x0b,0x00,0x00,0x00,0x19,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x4d,0x01,0x00,0x00,0x06,0x00,0x00,0x00, +0x08,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0x4e,0x01,0x00,0x00, +0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x4e,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x4e,0x01,0x00,0x00, +0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x50,0x01,0x00,0x00, +0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x50,0x01,0x00,0x00,0x21,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x39,0x02,0x00,0x00,0x0b,0x00,0x00,0x00, +0x18,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x7f,0x02,0x00,0x00, +0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00, +0x80,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x19,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x80,0x02,0x00,0x00,0x00,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, +0x80,0x02,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x82,0x02,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x82,0x02,0x00,0x00,0x21,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x13,0x00,0x02,0x00,0x02,0x00,0x00,0x00, +0x21,0x00,0x03,0x00,0x03,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x15,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x17,0x00,0x04,0x00,0x09,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x0a,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x09,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x0d,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x1e,0x00,0x10,0x00,0x10,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x11,0x00,0x00,0x00, +0x09,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0x11,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x09,0x00,0x00,0x00, +0x15,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, +0x14,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x15,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x21,0x00,0x00,0x00, +0x0a,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, +0x27,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0x2d,0x00,0x00,0x00,0x07,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x34,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x37,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00,0x3d,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x3e,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0x0a,0x00,0x00,0x00,0x4c,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x4f,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x53,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x54,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0x37,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x58,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0x37,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x32,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x61,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x60,0x00,0x00,0x00, +0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x62,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x63,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x61,0x00,0x00,0x00, +0x62,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x67,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x61,0x00,0x00,0x00, +0x62,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x6e,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x73,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x78,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x7d,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x81,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0x91,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x97,0x00,0x00,0x00, +0x03,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, +0xa2,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x32,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xa7,0x00,0x00,0x00,0x40,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0xa9,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xb8,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00, +0x62,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xb9,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xba,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0x53,0x00,0x00,0x00,0xb9,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xbb,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0x4f,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x32,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xbc,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xbd,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0xbb,0x00,0x00,0x00,0xbc,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xbe,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0xbd,0x00,0x00,0x00,0x60,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xbf,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0xba,0x00,0x00,0x00,0xbe,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xc0,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0xb8,0x00,0x00,0x00,0xbf,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xc1,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0xc0,0x00,0x00,0x00,0xbc,0x00,0x00,0x00, +0x14,0x00,0x02,0x00,0xc2,0x00,0x00,0x00,0x16,0x00,0x03,0x00, +0xc4,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xc5,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xc6,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0xc5,0x00,0x00,0x00,0xbf,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xc7,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0xc6,0x00,0x00,0x00,0xbc,0x00,0x00,0x00,0x1c,0x00,0x04,0x00, +0xc8,0x00,0x00,0x00,0xc4,0x00,0x00,0x00,0xc7,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0xc9,0x00,0x00,0x00,0x07,0x00,0x00,0x00, +0xc8,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0xc4,0x00,0x00,0x00, +0xcc,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0xcd,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0xc4,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0xd0,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xf4,0x00,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, +0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xf9,0x00,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, +0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xfa,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x37,0x00,0x00,0x00, +0xf9,0x00,0x00,0x00,0x1c,0x00,0x04,0x00,0xfb,0x00,0x00,0x00, +0xc4,0x00,0x00,0x00,0xfa,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0xfc,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0xfb,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0xfc,0x00,0x00,0x00,0xfd,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x16,0x00,0x03,0x00,0xff,0x00,0x00,0x00, +0x10,0x00,0x00,0x00,0x17,0x00,0x04,0x00,0x00,0x01,0x00,0x00, +0xff,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, +0x01,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x1e,0x00,0x03,0x00, +0x02,0x01,0x00,0x00,0x01,0x01,0x00,0x00,0x20,0x00,0x04,0x00, +0x03,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x02,0x01,0x00,0x00, +0x3b,0x00,0x04,0x00,0x03,0x01,0x00,0x00,0x04,0x01,0x00,0x00, +0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x06,0x01,0x00,0x00, +0x0c,0x00,0x00,0x00,0xff,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x0a,0x01,0x00,0x00,0x04,0x00,0x00,0x00,0xc4,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x1b,0x01,0x00,0x00, +0x03,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x22,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x33,0x00,0x06,0x00, +0x09,0x00,0x00,0x00,0x23,0x01,0x00,0x00,0x22,0x01,0x00,0x00, +0x39,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x24,0x01,0x00,0x00,0x51,0x00,0x00,0x00, +0x23,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x25,0x01,0x00,0x00,0x84,0x00,0x00,0x00, +0x24,0x01,0x00,0x00,0x6d,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x26,0x01,0x00,0x00,0x86,0x00,0x00,0x00, +0x25,0x01,0x00,0x00,0x6c,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x41,0x01,0x00,0x00,0x80,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x46,0x01,0x00,0x00,0x80,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x47,0x01,0x00,0x00,0x84,0x00,0x00,0x00, +0xa7,0x00,0x00,0x00,0x46,0x01,0x00,0x00,0x1c,0x00,0x04,0x00, +0x48,0x01,0x00,0x00,0xc4,0x00,0x00,0x00,0x47,0x01,0x00,0x00, +0x20,0x00,0x04,0x00,0x49,0x01,0x00,0x00,0x04,0x00,0x00,0x00, +0x48,0x01,0x00,0x00,0x3b,0x00,0x04,0x00,0x49,0x01,0x00,0x00, +0x4a,0x01,0x00,0x00,0x04,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, +0x4d,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x1e,0x00,0x03,0x00, +0x4e,0x01,0x00,0x00,0x4d,0x01,0x00,0x00,0x20,0x00,0x04,0x00, +0x4f,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x4e,0x01,0x00,0x00, +0x3b,0x00,0x04,0x00,0x4f,0x01,0x00,0x00,0x50,0x01,0x00,0x00, +0x0c,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x6b,0x01,0x00,0x00,0x51,0x00,0x00,0x00,0x23,0x01,0x00,0x00, +0x00,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x6c,0x01,0x00,0x00,0x84,0x00,0x00,0x00,0x6b,0x01,0x00,0x00, +0x6d,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x6d,0x01,0x00,0x00,0x86,0x00,0x00,0x00,0x6c,0x01,0x00,0x00, +0x6c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x70,0x01,0x00,0x00,0x08,0x01,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x71,0x01,0x00,0x00,0x86,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x74,0x01,0x00,0x00,0x86,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x8f,0x01,0x00,0x00,0x84,0x00,0x00,0x00, +0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x1c,0x00,0x04,0x00, +0x90,0x01,0x00,0x00,0xc4,0x00,0x00,0x00,0x8f,0x01,0x00,0x00, +0x20,0x00,0x04,0x00,0x91,0x01,0x00,0x00,0x07,0x00,0x00,0x00, +0x90,0x01,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xa1,0x01,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, +0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xbc,0x01,0x00,0x00,0x84,0x00,0x00,0x00,0xbf,0x00,0x00,0x00, +0xbc,0x00,0x00,0x00,0x1c,0x00,0x04,0x00,0xbd,0x01,0x00,0x00, +0xc4,0x00,0x00,0x00,0xbc,0x01,0x00,0x00,0x20,0x00,0x04,0x00, +0xbe,0x01,0x00,0x00,0x07,0x00,0x00,0x00,0xbd,0x01,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xc7,0x01,0x00,0x00, +0x86,0x00,0x00,0x00,0xb9,0x00,0x00,0x00,0xbf,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xcf,0x01,0x00,0x00, +0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xfe,0x01,0x00,0x00, +0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x31,0x02,0x00,0x00, +0x0d,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00, +0x39,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, +0x7f,0x02,0x00,0x00,0xc4,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, +0x80,0x02,0x00,0x00,0x7f,0x02,0x00,0x00,0x20,0x00,0x04,0x00, +0x81,0x02,0x00,0x00,0x0c,0x00,0x00,0x00,0x80,0x02,0x00,0x00, +0x3b,0x00,0x04,0x00,0x81,0x02,0x00,0x00,0x82,0x02,0x00,0x00, +0x0c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, +0x87,0x02,0x00,0x00,0x05,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x94,0x02,0x00,0x00,0x84,0x00,0x00,0x00, +0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x9d,0x02,0x00,0x00,0x0c,0x00,0x00,0x00,0xc4,0x00,0x00,0x00, +0x36,0x00,0x05,0x00,0x02,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0x05,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0xc9,0x00,0x00,0x00, +0xca,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0x91,0x01,0x00,0x00,0x92,0x01,0x00,0x00,0x07,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0xbe,0x01,0x00,0x00,0xbf,0x01,0x00,0x00, +0x07,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00, +0x0e,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, +0x0e,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, +0x16,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x14,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x17,0x00,0x00,0x00, +0x16,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x18,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x17,0x00,0x00,0x00, +0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x1e,0x00,0x00,0x00, +0x0f,0x00,0x00,0x00,0x17,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x15,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0x21,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x24,0x00,0x00,0x00,0x18,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, +0x28,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x27,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x29,0x00,0x00,0x00, +0x28,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x2a,0x00,0x00,0x00,0x1e,0x00,0x00,0x00,0x29,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, +0x12,0x00,0x00,0x00,0x2d,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x2f,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x30,0x00,0x00,0x00, +0x24,0x00,0x00,0x00,0x2f,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x30,0x00,0x00,0x00, +0x2a,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, +0x35,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x34,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x36,0x00,0x00,0x00, +0x35,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x38,0x00,0x00,0x00,0x36,0x00,0x00,0x00,0x37,0x00,0x00,0x00, +0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, +0x38,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x3b,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, +0x37,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00, +0x3f,0x00,0x00,0x00,0x3d,0x00,0x00,0x00,0x3e,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x40,0x00,0x00,0x00, +0x3f,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x42,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x3b,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x47,0x00,0x00,0x00, +0x40,0x00,0x00,0x00,0x3b,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x0d,0x00,0x00,0x00,0x49,0x00,0x00,0x00,0x3d,0x00,0x00,0x00, +0x39,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x4a,0x00,0x00,0x00,0x49,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x0d,0x00,0x00,0x00,0x4d,0x00,0x00,0x00,0x4c,0x00,0x00,0x00, +0x3e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x4e,0x00,0x00,0x00,0x4d,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x50,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, +0x4f,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x55,0x00,0x00,0x00,0x50,0x00,0x00,0x00,0x54,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x59,0x00,0x00,0x00, +0x50,0x00,0x00,0x00,0x58,0x00,0x00,0x00,0x89,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x5d,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, +0x4f,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x64,0x00,0x00,0x00,0x5d,0x00,0x00,0x00,0x63,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x68,0x00,0x00,0x00, +0x5d,0x00,0x00,0x00,0x67,0x00,0x00,0x00,0x89,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x6f,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, +0x6e,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x74,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x73,0x00,0x00,0x00, +0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x79,0x00,0x00,0x00, +0x4e,0x00,0x00,0x00,0x78,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x7e,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, +0x7d,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, +0x82,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x81,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x83,0x00,0x00,0x00, +0x82,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0x47,0x00,0x00,0x00,0x83,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x87,0x00,0x00,0x00, +0x12,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x88,0x00,0x00,0x00,0x87,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8a,0x00,0x00,0x00, +0x47,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x8d,0x00,0x00,0x00,0x8a,0x00,0x00,0x00, +0x83,0x00,0x00,0x00,0x0c,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0x8e,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x26,0x00,0x00,0x00, +0x88,0x00,0x00,0x00,0x8d,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x15,0x00,0x00,0x00,0x92,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0x91,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x93,0x00,0x00,0x00,0x92,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x94,0x00,0x00,0x00,0x32,0x00,0x00,0x00, +0x93,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x96,0x00,0x00,0x00,0x42,0x00,0x00,0x00,0x37,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x98,0x00,0x00,0x00, +0x12,0x00,0x00,0x00,0x97,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x99,0x00,0x00,0x00,0x98,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9a,0x00,0x00,0x00, +0x96,0x00,0x00,0x00,0x99,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x9b,0x00,0x00,0x00,0x94,0x00,0x00,0x00, +0x9a,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x9d,0x00,0x00,0x00,0x9b,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9e,0x00,0x00,0x00, +0x9d,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x15,0x00,0x00,0x00,0xa3,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0xa2,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xa4,0x00,0x00,0x00,0xa3,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xa5,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, +0xa4,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xa8,0x00,0x00,0x00,0x4a,0x00,0x00,0x00,0xa7,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0xaa,0x00,0x00,0x00, +0x12,0x00,0x00,0x00,0xa9,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xab,0x00,0x00,0x00,0xaa,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xac,0x00,0x00,0x00, +0xa8,0x00,0x00,0x00,0xab,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xad,0x00,0x00,0x00,0xa5,0x00,0x00,0x00, +0xac,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xaf,0x00,0x00,0x00,0xad,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xb0,0x00,0x00,0x00, +0xaf,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xb2,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xb2,0x00,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xa7,0x02,0x00,0x00, +0x3e,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0xd1,0x00,0x00,0x00, +0xb3,0x00,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, +0xc3,0x00,0x00,0x00,0xa7,0x02,0x00,0x00,0xc1,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0xb4,0x00,0x00,0x00,0xb3,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xc3,0x00,0x00,0x00, +0xb3,0x00,0x00,0x00,0xb4,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xb3,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0xcd,0x00,0x00,0x00, +0xce,0x00,0x00,0x00,0xca,0x00,0x00,0x00,0xa7,0x02,0x00,0x00, +0x3e,0x00,0x03,0x00,0xce,0x00,0x00,0x00,0xcc,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xd1,0x00,0x00,0x00, +0xa7,0x02,0x00,0x00,0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xb2,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xb4,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xd4,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xd4,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xc0,0x02,0x00,0x00,0xb0,0x00,0x00,0x00,0xb4,0x00,0x00,0x00, +0x76,0x01,0x00,0x00,0xd7,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xbc,0x02,0x00,0x00,0x9e,0x00,0x00,0x00, +0xb4,0x00,0x00,0x00,0x73,0x01,0x00,0x00,0xd7,0x00,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xa8,0x02,0x00,0x00, +0x84,0x00,0x00,0x00,0xb4,0x00,0x00,0x00,0x21,0x02,0x00,0x00, +0xd7,0x00,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, +0xdb,0x00,0x00,0x00,0xa8,0x02,0x00,0x00,0x8e,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0xd6,0x00,0x00,0x00,0xd7,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xdb,0x00,0x00,0x00, +0xd5,0x00,0x00,0x00,0xd6,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xd5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xdd,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xdd,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xb8,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, +0xd5,0x00,0x00,0x00,0x28,0x01,0x00,0x00,0xde,0x00,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0xe3,0x00,0x00,0x00, +0xb8,0x02,0x00,0x00,0x37,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0xdf,0x00,0x00,0x00,0xde,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0xe3,0x00,0x00,0x00,0xde,0x00,0x00,0x00, +0xdf,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xde,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xe8,0x00,0x00,0x00, +0x74,0x00,0x00,0x00,0xb8,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xeb,0x00,0x00,0x00,0xe8,0x00,0x00,0x00, +0x99,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xec,0x00,0x00,0x00,0xeb,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xed,0x00,0x00,0x00, +0xbc,0x02,0x00,0x00,0xec,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xef,0x00,0x00,0x00,0xed,0x00,0x00,0x00, +0x6f,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xf5,0x00,0x00,0x00,0xe8,0x00,0x00,0x00,0xf4,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xf7,0x00,0x00,0x00, +0x6f,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xf8,0x00,0x00,0x00,0xf5,0x00,0x00,0x00, +0xf7,0x00,0x00,0x00,0x41,0x00,0x07,0x00,0x06,0x01,0x00,0x00, +0x07,0x01,0x00,0x00,0x04,0x01,0x00,0x00,0x34,0x00,0x00,0x00, +0xef,0x00,0x00,0x00,0x3e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0xff,0x00,0x00,0x00,0x08,0x01,0x00,0x00,0x07,0x01,0x00,0x00, +0x73,0x00,0x04,0x00,0xc4,0x00,0x00,0x00,0x09,0x01,0x00,0x00, +0x08,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0x0a,0x01,0x00,0x00, +0x0b,0x01,0x00,0x00,0xfd,0x00,0x00,0x00,0xf8,0x00,0x00,0x00, +0x3e,0x00,0x03,0x00,0x0b,0x01,0x00,0x00,0x09,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x0d,0x01,0x00,0x00, +0xf8,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x41,0x00,0x07,0x00, +0x06,0x01,0x00,0x00,0x0f,0x01,0x00,0x00,0x04,0x01,0x00,0x00, +0x34,0x00,0x00,0x00,0xef,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0xff,0x00,0x00,0x00,0x10,0x01,0x00,0x00, +0x0f,0x01,0x00,0x00,0x73,0x00,0x04,0x00,0xc4,0x00,0x00,0x00, +0x11,0x01,0x00,0x00,0x10,0x01,0x00,0x00,0x41,0x00,0x05,0x00, +0x0a,0x01,0x00,0x00,0x12,0x01,0x00,0x00,0xfd,0x00,0x00,0x00, +0x0d,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x12,0x01,0x00,0x00, +0x11,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x14,0x01,0x00,0x00,0xf8,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x41,0x00,0x07,0x00,0x06,0x01,0x00,0x00,0x16,0x01,0x00,0x00, +0x04,0x01,0x00,0x00,0x34,0x00,0x00,0x00,0xef,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0xff,0x00,0x00,0x00, +0x17,0x01,0x00,0x00,0x16,0x01,0x00,0x00,0x73,0x00,0x04,0x00, +0xc4,0x00,0x00,0x00,0x18,0x01,0x00,0x00,0x17,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0x0a,0x01,0x00,0x00,0x19,0x01,0x00,0x00, +0xfd,0x00,0x00,0x00,0x14,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, +0x19,0x01,0x00,0x00,0x18,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x1c,0x01,0x00,0x00,0xf8,0x00,0x00,0x00, +0x1b,0x01,0x00,0x00,0x41,0x00,0x07,0x00,0x06,0x01,0x00,0x00, +0x1e,0x01,0x00,0x00,0x04,0x01,0x00,0x00,0x34,0x00,0x00,0x00, +0xef,0x00,0x00,0x00,0x1b,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, +0xff,0x00,0x00,0x00,0x1f,0x01,0x00,0x00,0x1e,0x01,0x00,0x00, +0x73,0x00,0x04,0x00,0xc4,0x00,0x00,0x00,0x20,0x01,0x00,0x00, +0x1f,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0x0a,0x01,0x00,0x00, +0x21,0x01,0x00,0x00,0xfd,0x00,0x00,0x00,0x1c,0x01,0x00,0x00, +0x3e,0x00,0x03,0x00,0x21,0x01,0x00,0x00,0x20,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x28,0x01,0x00,0x00, +0xb8,0x02,0x00,0x00,0x26,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0xdd,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xdf,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x2a,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x2a,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xb9,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0xdf,0x00,0x00,0x00, +0x6f,0x01,0x00,0x00,0x2b,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc2,0x00,0x00,0x00,0x30,0x01,0x00,0x00,0xb9,0x02,0x00,0x00, +0xa7,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x2c,0x01,0x00,0x00, +0x2b,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x30,0x01,0x00,0x00,0x2b,0x01,0x00,0x00,0x2c,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x2b,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x35,0x01,0x00,0x00,0x7e,0x00,0x00,0x00, +0xb9,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x38,0x01,0x00,0x00,0x35,0x01,0x00,0x00,0xab,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x39,0x01,0x00,0x00, +0x38,0x01,0x00,0x00,0x6d,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x3a,0x01,0x00,0x00,0xc0,0x02,0x00,0x00, +0x39,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x3c,0x01,0x00,0x00,0x3a,0x01,0x00,0x00,0x79,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x42,0x01,0x00,0x00, +0x35,0x01,0x00,0x00,0x41,0x01,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x44,0x01,0x00,0x00,0x79,0x00,0x00,0x00, +0x6d,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x45,0x01,0x00,0x00,0x42,0x01,0x00,0x00,0x44,0x01,0x00,0x00, +0x41,0x00,0x07,0x00,0x06,0x01,0x00,0x00,0x52,0x01,0x00,0x00, +0x50,0x01,0x00,0x00,0x34,0x00,0x00,0x00,0x3c,0x01,0x00,0x00, +0x3e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0xff,0x00,0x00,0x00, +0x53,0x01,0x00,0x00,0x52,0x01,0x00,0x00,0x73,0x00,0x04,0x00, +0xc4,0x00,0x00,0x00,0x54,0x01,0x00,0x00,0x53,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0x0a,0x01,0x00,0x00,0x55,0x01,0x00,0x00, +0x4a,0x01,0x00,0x00,0x45,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, +0x55,0x01,0x00,0x00,0x54,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x57,0x01,0x00,0x00,0x45,0x01,0x00,0x00, +0x39,0x00,0x00,0x00,0x41,0x00,0x07,0x00,0x06,0x01,0x00,0x00, +0x59,0x01,0x00,0x00,0x50,0x01,0x00,0x00,0x34,0x00,0x00,0x00, +0x3c,0x01,0x00,0x00,0x39,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0xff,0x00,0x00,0x00,0x5a,0x01,0x00,0x00,0x59,0x01,0x00,0x00, +0x73,0x00,0x04,0x00,0xc4,0x00,0x00,0x00,0x5b,0x01,0x00,0x00, +0x5a,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0x0a,0x01,0x00,0x00, +0x5c,0x01,0x00,0x00,0x4a,0x01,0x00,0x00,0x57,0x01,0x00,0x00, +0x3e,0x00,0x03,0x00,0x5c,0x01,0x00,0x00,0x5b,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x5e,0x01,0x00,0x00, +0x45,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x41,0x00,0x07,0x00, +0x06,0x01,0x00,0x00,0x60,0x01,0x00,0x00,0x50,0x01,0x00,0x00, +0x34,0x00,0x00,0x00,0x3c,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0xff,0x00,0x00,0x00,0x61,0x01,0x00,0x00, +0x60,0x01,0x00,0x00,0x73,0x00,0x04,0x00,0xc4,0x00,0x00,0x00, +0x62,0x01,0x00,0x00,0x61,0x01,0x00,0x00,0x41,0x00,0x05,0x00, +0x0a,0x01,0x00,0x00,0x63,0x01,0x00,0x00,0x4a,0x01,0x00,0x00, +0x5e,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x63,0x01,0x00,0x00, +0x62,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x65,0x01,0x00,0x00,0x45,0x01,0x00,0x00,0x1b,0x01,0x00,0x00, +0x41,0x00,0x07,0x00,0x06,0x01,0x00,0x00,0x67,0x01,0x00,0x00, +0x50,0x01,0x00,0x00,0x34,0x00,0x00,0x00,0x3c,0x01,0x00,0x00, +0x1b,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0xff,0x00,0x00,0x00, +0x68,0x01,0x00,0x00,0x67,0x01,0x00,0x00,0x73,0x00,0x04,0x00, +0xc4,0x00,0x00,0x00,0x69,0x01,0x00,0x00,0x68,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0x0a,0x01,0x00,0x00,0x6a,0x01,0x00,0x00, +0x4a,0x01,0x00,0x00,0x65,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, +0x6a,0x01,0x00,0x00,0x69,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x6f,0x01,0x00,0x00,0xb9,0x02,0x00,0x00, +0x6d,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x2a,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x2c,0x01,0x00,0x00,0xe0,0x00,0x04,0x00, +0x0c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x70,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x73,0x01,0x00,0x00, +0xbc,0x02,0x00,0x00,0x71,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x76,0x01,0x00,0x00,0xc0,0x02,0x00,0x00, +0x74,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x78,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x78,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xc2,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, +0x2c,0x01,0x00,0x00,0x1f,0x02,0x00,0x00,0x7b,0x01,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0x7e,0x01,0x00,0x00, +0xc2,0x02,0x00,0x00,0x6c,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0x7a,0x01,0x00,0x00,0x7b,0x01,0x00,0x00,0x00,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x7e,0x01,0x00,0x00,0x79,0x01,0x00,0x00, +0x7a,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x79,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0x80,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x80,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xc6,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0x79,0x01,0x00,0x00, +0xab,0x01,0x00,0x00,0x83,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc2,0x00,0x00,0x00,0x86,0x01,0x00,0x00,0xc6,0x02,0x00,0x00, +0x60,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x82,0x01,0x00,0x00, +0x83,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x86,0x01,0x00,0x00,0x81,0x01,0x00,0x00,0x82,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x81,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0x88,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x88,0x01,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xd8,0x02,0x00,0x00, +0x3e,0x00,0x00,0x00,0x81,0x01,0x00,0x00,0xa9,0x01,0x00,0x00, +0x89,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, +0x8e,0x01,0x00,0x00,0xd8,0x02,0x00,0x00,0x62,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0x8a,0x01,0x00,0x00,0x89,0x01,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x8e,0x01,0x00,0x00, +0x89,0x01,0x00,0x00,0x8a,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x89,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x94,0x01,0x00,0x00,0xc6,0x02,0x00,0x00,0x62,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x96,0x01,0x00,0x00, +0x94,0x01,0x00,0x00,0xd8,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x98,0x01,0x00,0x00,0x55,0x00,0x00,0x00, +0x53,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x9a,0x01,0x00,0x00,0xc6,0x02,0x00,0x00,0x61,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9b,0x01,0x00,0x00, +0x98,0x01,0x00,0x00,0x9a,0x01,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x9d,0x01,0x00,0x00,0x64,0x00,0x00,0x00, +0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x9e,0x01,0x00,0x00,0x9b,0x01,0x00,0x00,0x9d,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa0,0x01,0x00,0x00, +0x9e,0x01,0x00,0x00,0xd8,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xa2,0x01,0x00,0x00,0xa0,0x01,0x00,0x00, +0xa1,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xa4,0x01,0x00,0x00,0xa2,0x01,0x00,0x00,0xc2,0x02,0x00,0x00, +0x41,0x00,0x05,0x00,0x0a,0x01,0x00,0x00,0xa5,0x01,0x00,0x00, +0xfd,0x00,0x00,0x00,0xa4,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, +0xc4,0x00,0x00,0x00,0xa6,0x01,0x00,0x00,0xa5,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0xcd,0x00,0x00,0x00,0xa7,0x01,0x00,0x00, +0x92,0x01,0x00,0x00,0x96,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, +0xa7,0x01,0x00,0x00,0xa6,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xa9,0x01,0x00,0x00,0xd8,0x02,0x00,0x00, +0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x88,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x8a,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0x83,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x83,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xab,0x01,0x00,0x00, +0xc6,0x02,0x00,0x00,0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x80,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x82,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0xad,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xad,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xc7,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0x82,0x01,0x00,0x00, +0xd9,0x01,0x00,0x00,0xb0,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc2,0x00,0x00,0x00,0xb3,0x01,0x00,0x00,0xc7,0x02,0x00,0x00, +0xbf,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xaf,0x01,0x00,0x00, +0xb0,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0xb3,0x01,0x00,0x00,0xae,0x01,0x00,0x00,0xaf,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xae,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0xb5,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xb5,0x01,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xd5,0x02,0x00,0x00, +0x3e,0x00,0x00,0x00,0xae,0x01,0x00,0x00,0xd7,0x01,0x00,0x00, +0xb6,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, +0xbb,0x01,0x00,0x00,0xd5,0x02,0x00,0x00,0xbc,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0xb7,0x01,0x00,0x00,0xb6,0x01,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xbb,0x01,0x00,0x00, +0xb6,0x01,0x00,0x00,0xb7,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xb6,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xc1,0x01,0x00,0x00,0xc7,0x02,0x00,0x00,0xbc,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xc3,0x01,0x00,0x00, +0xc1,0x01,0x00,0x00,0xd5,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xc5,0x01,0x00,0x00,0x59,0x00,0x00,0x00, +0xb9,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xc8,0x01,0x00,0x00,0xc7,0x02,0x00,0x00,0xc7,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xc9,0x01,0x00,0x00, +0xc5,0x01,0x00,0x00,0xc8,0x01,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xcb,0x01,0x00,0x00,0x68,0x00,0x00,0x00, +0xbc,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xcc,0x01,0x00,0x00,0xc9,0x01,0x00,0x00,0xcb,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xce,0x01,0x00,0x00, +0xcc,0x01,0x00,0x00,0xd5,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xd0,0x01,0x00,0x00,0xce,0x01,0x00,0x00, +0xcf,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xd2,0x01,0x00,0x00,0xd0,0x01,0x00,0x00,0xc2,0x02,0x00,0x00, +0x41,0x00,0x05,0x00,0x0a,0x01,0x00,0x00,0xd3,0x01,0x00,0x00, +0x4a,0x01,0x00,0x00,0xd2,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, +0xc4,0x00,0x00,0x00,0xd4,0x01,0x00,0x00,0xd3,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0xcd,0x00,0x00,0x00,0xd5,0x01,0x00,0x00, +0xbf,0x01,0x00,0x00,0xc3,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, +0xd5,0x01,0x00,0x00,0xd4,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xd7,0x01,0x00,0x00,0xd5,0x02,0x00,0x00, +0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xb5,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xb7,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0xb0,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xb0,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xd9,0x01,0x00,0x00, +0xc7,0x02,0x00,0x00,0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xad,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xaf,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0xdb,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xdb,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xc8,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0xaf,0x01,0x00,0x00, +0x1d,0x02,0x00,0x00,0xde,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc2,0x00,0x00,0x00,0xe1,0x01,0x00,0x00,0xc8,0x02,0x00,0x00, +0xbf,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xdd,0x01,0x00,0x00, +0xde,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0xe1,0x01,0x00,0x00,0xdc,0x01,0x00,0x00,0xdd,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xdc,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0xe3,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xe3,0x01,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xcc,0x02,0x00,0x00, +0x3e,0x00,0x00,0x00,0xdc,0x01,0x00,0x00,0x1b,0x02,0x00,0x00, +0xe6,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, +0xe9,0x01,0x00,0x00,0xcc,0x02,0x00,0x00,0x60,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0xe5,0x01,0x00,0x00,0xe6,0x01,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xe9,0x01,0x00,0x00, +0xe4,0x01,0x00,0x00,0xe5,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xe4,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xeb,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xeb,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xce,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, +0xe4,0x01,0x00,0x00,0x19,0x02,0x00,0x00,0xee,0x01,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0xf1,0x01,0x00,0x00, +0xce,0x02,0x00,0x00,0xbc,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0xed,0x01,0x00,0x00,0xee,0x01,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0xf1,0x01,0x00,0x00,0xec,0x01,0x00,0x00, +0xed,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xec,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0xf3,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xf3,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xd0,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0xec,0x01,0x00,0x00, +0x17,0x02,0x00,0x00,0xf4,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc2,0x00,0x00,0x00,0xf9,0x01,0x00,0x00,0xd0,0x02,0x00,0x00, +0x62,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xf5,0x01,0x00,0x00, +0xf4,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0xf9,0x01,0x00,0x00,0xf4,0x01,0x00,0x00,0xf5,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xf4,0x01,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xfb,0x01,0x00,0x00,0xc8,0x02,0x00,0x00, +0xbc,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xfd,0x01,0x00,0x00,0xfb,0x01,0x00,0x00,0xce,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xff,0x01,0x00,0x00, +0xfd,0x01,0x00,0x00,0xfe,0x01,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x01,0x02,0x00,0x00,0xcc,0x02,0x00,0x00, +0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x02,0x02,0x00,0x00,0xff,0x01,0x00,0x00,0x01,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x04,0x02,0x00,0x00, +0x02,0x02,0x00,0x00,0xd0,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x08,0x02,0x00,0x00,0x01,0x02,0x00,0x00, +0xd0,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0xcd,0x00,0x00,0x00, +0x09,0x02,0x00,0x00,0x92,0x01,0x00,0x00,0x08,0x02,0x00,0x00, +0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00,0x0a,0x02,0x00,0x00, +0x09,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0xcd,0x00,0x00,0x00, +0x0f,0x02,0x00,0x00,0xbf,0x01,0x00,0x00,0xfd,0x01,0x00,0x00, +0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00,0x10,0x02,0x00,0x00, +0x0f,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0xcd,0x00,0x00,0x00, +0x12,0x02,0x00,0x00,0xca,0x00,0x00,0x00,0x04,0x02,0x00,0x00, +0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00,0x13,0x02,0x00,0x00, +0x12,0x02,0x00,0x00,0x0c,0x00,0x08,0x00,0xc4,0x00,0x00,0x00, +0x14,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00, +0x0a,0x02,0x00,0x00,0x10,0x02,0x00,0x00,0x13,0x02,0x00,0x00, +0x3e,0x00,0x03,0x00,0x12,0x02,0x00,0x00,0x14,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x17,0x02,0x00,0x00, +0xd0,0x02,0x00,0x00,0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xf3,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xf5,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0xee,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xee,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x19,0x02,0x00,0x00,0xce,0x02,0x00,0x00,0xd0,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xeb,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xed,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xe6,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xe6,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x1b,0x02,0x00,0x00,0xcc,0x02,0x00,0x00, +0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xe3,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xe5,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0xde,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xde,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x1d,0x02,0x00,0x00, +0xc8,0x02,0x00,0x00,0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xdb,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xdd,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0x7b,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x7b,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x1f,0x02,0x00,0x00,0xc2,0x02,0x00,0x00,0xd0,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x78,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x7a,0x01,0x00,0x00,0xe0,0x00,0x04,0x00,0x0c,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x70,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0xd7,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xd7,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x21,0x02,0x00,0x00, +0xa8,0x02,0x00,0x00,0x6c,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xd4,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xd6,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x26,0x02,0x00,0x00, +0x55,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x27,0x02,0x00,0x00,0x96,0x00,0x00,0x00, +0x26,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x2c,0x02,0x00,0x00,0x59,0x00,0x00,0x00,0xb9,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x2d,0x02,0x00,0x00, +0xa8,0x00,0x00,0x00,0x2c,0x02,0x00,0x00,0x41,0x00,0x05,0x00, +0x15,0x00,0x00,0x00,0x32,0x02,0x00,0x00,0x12,0x00,0x00,0x00, +0x31,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x33,0x02,0x00,0x00,0x32,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x34,0x02,0x00,0x00,0x0f,0x00,0x00,0x00, +0x33,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x38,0x02,0x00,0x00,0x47,0x00,0x00,0x00,0x33,0x02,0x00,0x00, +0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00,0x3a,0x02,0x00,0x00, +0x39,0x02,0x00,0x00,0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x3b,0x02,0x00,0x00,0x3a,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3c,0x02,0x00,0x00, +0x38,0x02,0x00,0x00,0x3b,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x3d,0x02,0x00,0x00,0x34,0x02,0x00,0x00, +0x3c,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x3f,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x3f,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xa9,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, +0xd6,0x00,0x00,0x00,0xa6,0x02,0x00,0x00,0x42,0x02,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0x45,0x02,0x00,0x00, +0xa9,0x02,0x00,0x00,0xbf,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0x41,0x02,0x00,0x00,0x42,0x02,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x45,0x02,0x00,0x00,0x40,0x02,0x00,0x00, +0x41,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x40,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0x47,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x47,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xaa,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0x40,0x02,0x00,0x00, +0xa4,0x02,0x00,0x00,0x4a,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc2,0x00,0x00,0x00,0x4d,0x02,0x00,0x00,0xaa,0x02,0x00,0x00, +0x60,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x49,0x02,0x00,0x00, +0x4a,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x4d,0x02,0x00,0x00,0x48,0x02,0x00,0x00,0x49,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x48,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x51,0x02,0x00,0x00,0xaa,0x02,0x00,0x00, +0x61,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x52,0x02,0x00,0x00,0x27,0x02,0x00,0x00,0x51,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x54,0x02,0x00,0x00, +0x64,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x55,0x02,0x00,0x00,0x52,0x02,0x00,0x00, +0x54,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x59,0x02,0x00,0x00,0xa9,0x02,0x00,0x00,0xc7,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x5a,0x02,0x00,0x00, +0x2d,0x02,0x00,0x00,0x59,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x5c,0x02,0x00,0x00,0x68,0x00,0x00,0x00, +0xbc,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x5d,0x02,0x00,0x00,0x5a,0x02,0x00,0x00,0x5c,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0x5f,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x5f,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xac,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0x48,0x02,0x00,0x00, +0xa2,0x02,0x00,0x00,0x62,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc2,0x00,0x00,0x00,0x65,0x02,0x00,0x00,0xac,0x02,0x00,0x00, +0xbc,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x61,0x02,0x00,0x00, +0x62,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x65,0x02,0x00,0x00,0x60,0x02,0x00,0x00,0x61,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x60,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0x67,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x67,0x02,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xae,0x02,0x00,0x00, +0x3e,0x00,0x00,0x00,0x60,0x02,0x00,0x00,0xa0,0x02,0x00,0x00, +0x6a,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, +0x6d,0x02,0x00,0x00,0xae,0x02,0x00,0x00,0x62,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0x69,0x02,0x00,0x00,0x6a,0x02,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x6d,0x02,0x00,0x00, +0x68,0x02,0x00,0x00,0x69,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x68,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x70,0x02,0x00,0x00,0x55,0x02,0x00,0x00,0xae,0x02,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0x73,0x02,0x00,0x00, +0x70,0x02,0x00,0x00,0x36,0x00,0x00,0x00,0xf7,0x00,0x03,0x00, +0x75,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x73,0x02,0x00,0x00,0x74,0x02,0x00,0x00,0x75,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x74,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x78,0x02,0x00,0x00,0x5d,0x02,0x00,0x00, +0xac,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, +0x79,0x02,0x00,0x00,0x12,0x00,0x00,0x00,0xd0,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x7a,0x02,0x00,0x00, +0x79,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, +0x7b,0x02,0x00,0x00,0x78,0x02,0x00,0x00,0x7a,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0x75,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x75,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0xc2,0x00,0x00,0x00, +0x7c,0x02,0x00,0x00,0x73,0x02,0x00,0x00,0x68,0x02,0x00,0x00, +0x7b,0x02,0x00,0x00,0x74,0x02,0x00,0x00,0xf7,0x00,0x03,0x00, +0x7e,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x7c,0x02,0x00,0x00,0x7d,0x02,0x00,0x00,0x7e,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x7d,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x86,0x02,0x00,0x00,0x5d,0x02,0x00,0x00, +0xac,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, +0x88,0x02,0x00,0x00,0x12,0x00,0x00,0x00,0x87,0x02,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x89,0x02,0x00,0x00, +0x88,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x8a,0x02,0x00,0x00,0x86,0x02,0x00,0x00,0x89,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8b,0x02,0x00,0x00, +0x3d,0x02,0x00,0x00,0x8a,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x8d,0x02,0x00,0x00,0x8b,0x02,0x00,0x00, +0x55,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x8f,0x02,0x00,0x00,0x8d,0x02,0x00,0x00,0xae,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x91,0x02,0x00,0x00, +0xa9,0x02,0x00,0x00,0xbc,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x93,0x02,0x00,0x00,0x91,0x02,0x00,0x00, +0xac,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x95,0x02,0x00,0x00,0x93,0x02,0x00,0x00,0x94,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x97,0x02,0x00,0x00, +0xaa,0x02,0x00,0x00,0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x98,0x02,0x00,0x00,0x95,0x02,0x00,0x00, +0x97,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x9a,0x02,0x00,0x00,0x98,0x02,0x00,0x00,0xae,0x02,0x00,0x00, +0x41,0x00,0x05,0x00,0xcd,0x00,0x00,0x00,0x9b,0x02,0x00,0x00, +0xca,0x00,0x00,0x00,0x9a,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, +0xc4,0x00,0x00,0x00,0x9c,0x02,0x00,0x00,0x9b,0x02,0x00,0x00, +0x41,0x00,0x06,0x00,0x9d,0x02,0x00,0x00,0x9e,0x02,0x00,0x00, +0x82,0x02,0x00,0x00,0x34,0x00,0x00,0x00,0x8f,0x02,0x00,0x00, +0x3e,0x00,0x03,0x00,0x9e,0x02,0x00,0x00,0x9c,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0x7e,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x7e,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x6a,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x6a,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xa0,0x02,0x00,0x00,0xae,0x02,0x00,0x00, +0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x67,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x69,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0x62,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x62,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa2,0x02,0x00,0x00, +0xac,0x02,0x00,0x00,0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x5f,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x61,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0x4a,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x4a,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xa4,0x02,0x00,0x00,0xaa,0x02,0x00,0x00,0xd0,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x47,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x49,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x42,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x42,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xa6,0x02,0x00,0x00,0xa9,0x02,0x00,0x00, +0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x3f,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x41,0x02,0x00,0x00,0xfd,0x00,0x01,0x00, +0x38,0x00,0x01,0x00, +}; +const uint64_t matmul_f16_aligned_fp32_len = 10288; + +unsigned char matmul_f16_f32_data[] = { +0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, +0xda,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, +0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x09,0x00,0x00,0x00, +0x11,0x00,0x02,0x00,0x51,0x11,0x00,0x00,0x0b,0x00,0x06,0x00, +0x01,0x00,0x00,0x00,0x47,0x4c,0x53,0x4c,0x2e,0x73,0x74,0x64, +0x2e,0x34,0x35,0x30,0x00,0x00,0x00,0x00,0x0e,0x00,0x03,0x00, +0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x0f,0x00,0x0f,0x00, +0x05,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x6d,0x61,0x69,0x6e, +0x00,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0x3d,0x00,0x00,0x00,0x4c,0x00,0x00,0x00,0xfb,0x00,0x00,0x00, +0x06,0x01,0x00,0x00,0x46,0x01,0x00,0x00,0x51,0x01,0x00,0x00, +0x3b,0x02,0x00,0x00,0x84,0x02,0x00,0x00,0x10,0x00,0x06,0x00, +0x04,0x00,0x00,0x00,0x11,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x0b,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x1c,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x10,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x08,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x03,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x10,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x10,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, +0x05,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x14,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x10,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x1c,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, +0x08,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x09,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x24,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x10,0x00,0x00,0x00,0x0a,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x28,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, +0x0b,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x2c,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x30,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x10,0x00,0x00,0x00,0x0d,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x34,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x10,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x37,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x3d,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x1a,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x4c,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, +0x1b,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x4f,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x53,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x60,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x62,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x6c,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x03,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0xa6,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xb8,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0xbb,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x08,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x03,0x01,0x00,0x00,0x06,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0x04,0x01,0x00,0x00, +0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x04,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x04,0x01,0x00,0x00, +0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x06,0x01,0x00,0x00, +0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x06,0x01,0x00,0x00,0x21,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x20,0x01,0x00,0x00,0x01,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x21,0x01,0x00,0x00, +0x0b,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x4e,0x01,0x00,0x00,0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x48,0x00,0x04,0x00,0x4f,0x01,0x00,0x00,0x00,0x00,0x00,0x00, +0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x4f,0x01,0x00,0x00, +0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x03,0x00,0x4f,0x01,0x00,0x00,0x02,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x51,0x01,0x00,0x00,0x22,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x51,0x01,0x00,0x00, +0x21,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x3b,0x02,0x00,0x00,0x0b,0x00,0x00,0x00,0x18,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x81,0x02,0x00,0x00,0x06,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0x82,0x02,0x00,0x00, +0x00,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x82,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x82,0x02,0x00,0x00, +0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x84,0x02,0x00,0x00, +0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x84,0x02,0x00,0x00,0x21,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x13,0x00,0x02,0x00,0x02,0x00,0x00,0x00,0x21,0x00,0x03,0x00, +0x03,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x15,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x17,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x03,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x0a,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0x0a,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x0d,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x1e,0x00,0x10,0x00, +0x10,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x11,0x00,0x00,0x00,0x09,0x00,0x00,0x00, +0x10,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x11,0x00,0x00,0x00, +0x12,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x15,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x14,0x00,0x00,0x00, +0x08,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x15,0x00,0x00,0x00, +0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x0a,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x27,0x00,0x00,0x00, +0x09,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, +0x2d,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x37,0x00,0x00,0x00, +0x40,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x39,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0x0a,0x00,0x00,0x00,0x3d,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x3e,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00, +0x4c,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x53,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x54,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x37,0x00,0x00,0x00, +0x53,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x58,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x37,0x00,0x00,0x00, +0x53,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x60,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x61,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0x53,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x32,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x63,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0x61,0x00,0x00,0x00,0x62,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x67,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0x61,0x00,0x00,0x00,0x62,0x00,0x00,0x00, +0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, +0x10,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x6d,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, +0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x72,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, +0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x77,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, +0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x7c,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, +0x39,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, +0x80,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0x85,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x90,0x00,0x00,0x00, +0x0b,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, +0x96,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0xa1,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xa6,0x00,0x00,0x00, +0x40,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, +0xa8,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xb7,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x32,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xb8,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xb9,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0xb8,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xba,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x62,0x00,0x00,0x00, +0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xbb,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xbc,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0xba,0x00,0x00,0x00, +0xbb,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xbd,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0xbc,0x00,0x00,0x00, +0x60,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xbe,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0xb9,0x00,0x00,0x00, +0xbd,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xbf,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0xb7,0x00,0x00,0x00, +0xbe,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xc0,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0xbf,0x00,0x00,0x00, +0xbb,0x00,0x00,0x00,0x14,0x00,0x02,0x00,0xc1,0x00,0x00,0x00, +0x16,0x00,0x03,0x00,0xc3,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xc4,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xc5,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0xc4,0x00,0x00,0x00,0xbe,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xc6,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0xc5,0x00,0x00,0x00,0xbb,0x00,0x00,0x00, +0x1c,0x00,0x04,0x00,0xc7,0x00,0x00,0x00,0xc3,0x00,0x00,0x00, +0xc6,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0xc8,0x00,0x00,0x00, +0x07,0x00,0x00,0x00,0xc7,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0xc3,0x00,0x00,0x00,0xcb,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0xcc,0x00,0x00,0x00,0x07,0x00,0x00,0x00, +0xc3,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, +0xcf,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x16,0x00,0x03,0x00, +0xf6,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xf7,0x00,0x00,0x00,0x80,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xf8,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0x37,0x00,0x00,0x00,0xf7,0x00,0x00,0x00,0x1c,0x00,0x04,0x00, +0xf9,0x00,0x00,0x00,0xf6,0x00,0x00,0x00,0xf8,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0xfa,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0xf9,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0xfa,0x00,0x00,0x00, +0xfb,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xff,0x00,0x00,0x00,0x80,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, +0x03,0x01,0x00,0x00,0xf6,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, +0x04,0x01,0x00,0x00,0x03,0x01,0x00,0x00,0x20,0x00,0x04,0x00, +0x05,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x04,0x01,0x00,0x00, +0x3b,0x00,0x04,0x00,0x05,0x01,0x00,0x00,0x06,0x01,0x00,0x00, +0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x11,0x01,0x00,0x00, +0x0c,0x00,0x00,0x00,0xf6,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x14,0x01,0x00,0x00,0x04,0x00,0x00,0x00,0xf6,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x1a,0x01,0x00,0x00, +0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0xf6,0x00,0x00,0x00,0x1e,0x01,0x00,0x00, +0x00,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x20,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x33,0x00,0x06,0x00, +0x09,0x00,0x00,0x00,0x21,0x01,0x00,0x00,0x20,0x01,0x00,0x00, +0x39,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x22,0x01,0x00,0x00,0x51,0x00,0x00,0x00, +0x21,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x23,0x01,0x00,0x00,0x84,0x00,0x00,0x00, +0x22,0x01,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x24,0x01,0x00,0x00,0x86,0x00,0x00,0x00, +0x23,0x01,0x00,0x00,0x6c,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x42,0x01,0x00,0x00,0x80,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x43,0x01,0x00,0x00,0x84,0x00,0x00,0x00, +0xa6,0x00,0x00,0x00,0x42,0x01,0x00,0x00,0x1c,0x00,0x04,0x00, +0x44,0x01,0x00,0x00,0xf6,0x00,0x00,0x00,0x43,0x01,0x00,0x00, +0x20,0x00,0x04,0x00,0x45,0x01,0x00,0x00,0x04,0x00,0x00,0x00, +0x44,0x01,0x00,0x00,0x3b,0x00,0x04,0x00,0x45,0x01,0x00,0x00, +0x46,0x01,0x00,0x00,0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x4a,0x01,0x00,0x00,0x80,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, +0x4e,0x01,0x00,0x00,0xc3,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, +0x4f,0x01,0x00,0x00,0x4e,0x01,0x00,0x00,0x20,0x00,0x04,0x00, +0x50,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x4f,0x01,0x00,0x00, +0x3b,0x00,0x04,0x00,0x50,0x01,0x00,0x00,0x51,0x01,0x00,0x00, +0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x5c,0x01,0x00,0x00, +0x0c,0x00,0x00,0x00,0xc3,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x65,0x01,0x00,0x00,0x80,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x6a,0x01,0x00,0x00,0x51,0x00,0x00,0x00, +0x21,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x6b,0x01,0x00,0x00,0x84,0x00,0x00,0x00, +0x6a,0x01,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x6c,0x01,0x00,0x00,0x86,0x00,0x00,0x00, +0x6b,0x01,0x00,0x00,0x6c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x6f,0x01,0x00,0x00,0x08,0x01,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x70,0x01,0x00,0x00, +0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x73,0x01,0x00,0x00, +0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x8e,0x01,0x00,0x00, +0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00, +0x1c,0x00,0x04,0x00,0x8f,0x01,0x00,0x00,0xf6,0x00,0x00,0x00, +0x8e,0x01,0x00,0x00,0x20,0x00,0x04,0x00,0x90,0x01,0x00,0x00, +0x07,0x00,0x00,0x00,0x8f,0x01,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xa0,0x01,0x00,0x00,0x80,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0xa6,0x01,0x00,0x00,0x07,0x00,0x00,0x00,0xf6,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xbc,0x01,0x00,0x00, +0x84,0x00,0x00,0x00,0xbe,0x00,0x00,0x00,0xbb,0x00,0x00,0x00, +0x1c,0x00,0x04,0x00,0xbd,0x01,0x00,0x00,0xf6,0x00,0x00,0x00, +0xbc,0x01,0x00,0x00,0x20,0x00,0x04,0x00,0xbe,0x01,0x00,0x00, +0x07,0x00,0x00,0x00,0xbd,0x01,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xc7,0x01,0x00,0x00,0x86,0x00,0x00,0x00, +0xb8,0x00,0x00,0x00,0xbe,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xcf,0x01,0x00,0x00,0x80,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xfe,0x01,0x00,0x00,0x84,0x00,0x00,0x00, +0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0x33,0x02,0x00,0x00,0x0d,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00,0x3b,0x02,0x00,0x00, +0x01,0x00,0x00,0x00,0x1d,0x00,0x03,0x00,0x81,0x02,0x00,0x00, +0xc3,0x00,0x00,0x00,0x1e,0x00,0x03,0x00,0x82,0x02,0x00,0x00, +0x81,0x02,0x00,0x00,0x20,0x00,0x04,0x00,0x83,0x02,0x00,0x00, +0x0c,0x00,0x00,0x00,0x82,0x02,0x00,0x00,0x3b,0x00,0x04,0x00, +0x83,0x02,0x00,0x00,0x84,0x02,0x00,0x00,0x0c,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x89,0x02,0x00,0x00, +0x05,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x96,0x02,0x00,0x00,0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00, +0x62,0x00,0x00,0x00,0x36,0x00,0x05,0x00,0x02,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x03,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0x05,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0xc8,0x00,0x00,0x00,0xc9,0x00,0x00,0x00,0x07,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0x90,0x01,0x00,0x00,0x91,0x01,0x00,0x00, +0x07,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0xbe,0x01,0x00,0x00, +0xbf,0x01,0x00,0x00,0x07,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x0d,0x00,0x00,0x00,0x0e,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x0f,0x00,0x00,0x00,0x0e,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x15,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0x14,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x17,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, +0x17,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x1e,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x17,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x22,0x00,0x00,0x00, +0x12,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x22,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x24,0x00,0x00,0x00, +0x18,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x15,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0x27,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x29,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x2a,0x00,0x00,0x00,0x1e,0x00,0x00,0x00, +0x29,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, +0x2e,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x2d,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x2f,0x00,0x00,0x00, +0x2e,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x30,0x00,0x00,0x00,0x24,0x00,0x00,0x00,0x2f,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x32,0x00,0x00,0x00, +0x30,0x00,0x00,0x00,0x2a,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x15,0x00,0x00,0x00,0x35,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0x34,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x36,0x00,0x00,0x00,0x35,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x38,0x00,0x00,0x00,0x36,0x00,0x00,0x00, +0x37,0x00,0x00,0x00,0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x3a,0x00,0x00,0x00,0x38,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3b,0x00,0x00,0x00, +0x3a,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x0d,0x00,0x00,0x00,0x3f,0x00,0x00,0x00,0x3d,0x00,0x00,0x00, +0x3e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x40,0x00,0x00,0x00,0x3f,0x00,0x00,0x00,0x89,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x42,0x00,0x00,0x00,0x40,0x00,0x00,0x00, +0x3b,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x47,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x3b,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00,0x49,0x00,0x00,0x00, +0x3d,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x4a,0x00,0x00,0x00,0x49,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00,0x4d,0x00,0x00,0x00, +0x4c,0x00,0x00,0x00,0x3e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x4d,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x50,0x00,0x00,0x00, +0x4e,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x89,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x55,0x00,0x00,0x00,0x50,0x00,0x00,0x00, +0x54,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x59,0x00,0x00,0x00,0x50,0x00,0x00,0x00,0x58,0x00,0x00,0x00, +0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x5d,0x00,0x00,0x00, +0x4e,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x89,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x64,0x00,0x00,0x00,0x5d,0x00,0x00,0x00, +0x63,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x68,0x00,0x00,0x00,0x5d,0x00,0x00,0x00,0x67,0x00,0x00,0x00, +0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x6e,0x00,0x00,0x00, +0x4e,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x73,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, +0x72,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x78,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x77,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x7d,0x00,0x00,0x00, +0x4e,0x00,0x00,0x00,0x7c,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x15,0x00,0x00,0x00,0x81,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0x80,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x82,0x00,0x00,0x00,0x81,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x83,0x00,0x00,0x00,0x47,0x00,0x00,0x00, +0x82,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x85,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x87,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x89,0x00,0x00,0x00,0x47,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8c,0x00,0x00,0x00, +0x89,0x00,0x00,0x00,0x82,0x00,0x00,0x00,0x0c,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0x8d,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x26,0x00,0x00,0x00,0x87,0x00,0x00,0x00,0x8c,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x91,0x00,0x00,0x00, +0x12,0x00,0x00,0x00,0x90,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x92,0x00,0x00,0x00,0x91,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x93,0x00,0x00,0x00, +0x32,0x00,0x00,0x00,0x92,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x95,0x00,0x00,0x00,0x42,0x00,0x00,0x00, +0x37,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, +0x97,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x96,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x98,0x00,0x00,0x00, +0x97,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x99,0x00,0x00,0x00,0x95,0x00,0x00,0x00,0x98,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9a,0x00,0x00,0x00, +0x93,0x00,0x00,0x00,0x99,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x9c,0x00,0x00,0x00,0x9a,0x00,0x00,0x00, +0x83,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x9d,0x00,0x00,0x00,0x9c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0xa2,0x00,0x00,0x00, +0x12,0x00,0x00,0x00,0xa1,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xa3,0x00,0x00,0x00,0xa2,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa4,0x00,0x00,0x00, +0x0f,0x00,0x00,0x00,0xa3,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xa7,0x00,0x00,0x00,0x4a,0x00,0x00,0x00, +0xa6,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, +0xa9,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0xa8,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xaa,0x00,0x00,0x00, +0xa9,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xab,0x00,0x00,0x00,0xa7,0x00,0x00,0x00,0xaa,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xac,0x00,0x00,0x00, +0xa4,0x00,0x00,0x00,0xab,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xae,0x00,0x00,0x00,0xac,0x00,0x00,0x00, +0x83,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xaf,0x00,0x00,0x00,0xae,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xb1,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xb1,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xa8,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0x05,0x00,0x00,0x00, +0xd0,0x00,0x00,0x00,0xb2,0x00,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc1,0x00,0x00,0x00,0xc2,0x00,0x00,0x00,0xa8,0x02,0x00,0x00, +0xc0,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xb3,0x00,0x00,0x00, +0xb2,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0xc2,0x00,0x00,0x00,0xb2,0x00,0x00,0x00,0xb3,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xb2,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0xcc,0x00,0x00,0x00,0xcd,0x00,0x00,0x00,0xc9,0x00,0x00,0x00, +0xa8,0x02,0x00,0x00,0x3e,0x00,0x03,0x00,0xcd,0x00,0x00,0x00, +0xcb,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xd0,0x00,0x00,0x00,0xa8,0x02,0x00,0x00,0xcf,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xb1,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xb3,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xd3,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xd3,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xc1,0x02,0x00,0x00,0xaf,0x00,0x00,0x00, +0xb3,0x00,0x00,0x00,0x75,0x01,0x00,0x00,0xd6,0x00,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xbd,0x02,0x00,0x00, +0x9d,0x00,0x00,0x00,0xb3,0x00,0x00,0x00,0x72,0x01,0x00,0x00, +0xd6,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xa9,0x02,0x00,0x00,0x83,0x00,0x00,0x00,0xb3,0x00,0x00,0x00, +0x23,0x02,0x00,0x00,0xd6,0x00,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc1,0x00,0x00,0x00,0xda,0x00,0x00,0x00,0xa9,0x02,0x00,0x00, +0x8d,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xd5,0x00,0x00,0x00, +0xd6,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0xda,0x00,0x00,0x00,0xd4,0x00,0x00,0x00,0xd5,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xd4,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xdc,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xdc,0x00,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xb9,0x02,0x00,0x00, +0x3e,0x00,0x00,0x00,0xd4,0x00,0x00,0x00,0x26,0x01,0x00,0x00, +0xdf,0x00,0x00,0x00,0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00, +0xe2,0x00,0x00,0x00,0xb9,0x02,0x00,0x00,0x37,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0xde,0x00,0x00,0x00,0xdf,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xe2,0x00,0x00,0x00, +0xdd,0x00,0x00,0x00,0xde,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xdd,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xe6,0x00,0x00,0x00,0x95,0x00,0x00,0x00,0x73,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xe8,0x00,0x00,0x00, +0xe6,0x00,0x00,0x00,0xb9,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc1,0x00,0x00,0x00,0xeb,0x00,0x00,0x00,0xe8,0x00,0x00,0x00, +0x36,0x00,0x00,0x00,0xf7,0x00,0x03,0x00,0xed,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xeb,0x00,0x00,0x00, +0xec,0x00,0x00,0x00,0xed,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xec,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xf0,0x00,0x00,0x00,0xa9,0x02,0x00,0x00,0x6e,0x00,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00,0xf2,0x00,0x00,0x00, +0xf0,0x00,0x00,0x00,0x8d,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xed,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xed,0x00,0x00,0x00, +0xf5,0x00,0x07,0x00,0xc1,0x00,0x00,0x00,0xf3,0x00,0x00,0x00, +0xeb,0x00,0x00,0x00,0xdd,0x00,0x00,0x00,0xf2,0x00,0x00,0x00, +0xec,0x00,0x00,0x00,0xf7,0x00,0x03,0x00,0xf5,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xf3,0x00,0x00,0x00, +0xf4,0x00,0x00,0x00,0x16,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xf4,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xfe,0x00,0x00,0x00,0x73,0x00,0x00,0x00,0xb9,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x00,0x01,0x00,0x00, +0xfe,0x00,0x00,0x00,0xff,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x02,0x01,0x00,0x00,0x00,0x01,0x00,0x00, +0x6e,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x0d,0x01,0x00,0x00,0xfe,0x00,0x00,0x00,0x98,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x0e,0x01,0x00,0x00, +0xbd,0x02,0x00,0x00,0x0d,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x10,0x01,0x00,0x00,0x0e,0x01,0x00,0x00, +0x6e,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x11,0x01,0x00,0x00, +0x12,0x01,0x00,0x00,0x06,0x01,0x00,0x00,0x34,0x00,0x00,0x00, +0x10,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0xf6,0x00,0x00,0x00, +0x13,0x01,0x00,0x00,0x12,0x01,0x00,0x00,0x41,0x00,0x05,0x00, +0x14,0x01,0x00,0x00,0x15,0x01,0x00,0x00,0xfb,0x00,0x00,0x00, +0x02,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x15,0x01,0x00,0x00, +0x13,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xf5,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0x16,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x19,0x01,0x00,0x00,0x73,0x00,0x00,0x00, +0xb9,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x1b,0x01,0x00,0x00,0x19,0x01,0x00,0x00,0x1a,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x1d,0x01,0x00,0x00, +0x1b,0x01,0x00,0x00,0x6e,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x14,0x01,0x00,0x00,0x1f,0x01,0x00,0x00,0xfb,0x00,0x00,0x00, +0x1d,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x1f,0x01,0x00,0x00, +0x1e,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xf5,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xf5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xdf,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xdf,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x26,0x01,0x00,0x00, +0xb9,0x02,0x00,0x00,0x24,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0xdc,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xde,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x28,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x28,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xba,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0xde,0x00,0x00,0x00, +0x6e,0x01,0x00,0x00,0x2b,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc1,0x00,0x00,0x00,0x2e,0x01,0x00,0x00,0xba,0x02,0x00,0x00, +0xa6,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x2a,0x01,0x00,0x00, +0x2b,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x2e,0x01,0x00,0x00,0x29,0x01,0x00,0x00,0x2a,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x29,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x32,0x01,0x00,0x00,0xa7,0x00,0x00,0x00, +0x7d,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x34,0x01,0x00,0x00,0x32,0x01,0x00,0x00,0xba,0x02,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x35,0x01,0x00,0x00, +0x12,0x00,0x00,0x00,0xcf,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x36,0x01,0x00,0x00,0x35,0x01,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00,0x37,0x01,0x00,0x00, +0x34,0x01,0x00,0x00,0x36,0x01,0x00,0x00,0xf7,0x00,0x03,0x00, +0x39,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x37,0x01,0x00,0x00,0x38,0x01,0x00,0x00,0x39,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x38,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x3c,0x01,0x00,0x00,0xa9,0x02,0x00,0x00, +0x78,0x00,0x00,0x00,0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00, +0x3e,0x01,0x00,0x00,0x3c,0x01,0x00,0x00,0x8d,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x39,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x39,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0xc1,0x00,0x00,0x00, +0x3f,0x01,0x00,0x00,0x37,0x01,0x00,0x00,0x29,0x01,0x00,0x00, +0x3e,0x01,0x00,0x00,0x38,0x01,0x00,0x00,0xf7,0x00,0x03,0x00, +0x41,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x3f,0x01,0x00,0x00,0x40,0x01,0x00,0x00,0x61,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x40,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x49,0x01,0x00,0x00,0x7d,0x00,0x00,0x00, +0xba,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x4b,0x01,0x00,0x00,0x49,0x01,0x00,0x00,0x4a,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x4d,0x01,0x00,0x00, +0x4b,0x01,0x00,0x00,0x78,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x58,0x01,0x00,0x00,0x49,0x01,0x00,0x00, +0xaa,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x59,0x01,0x00,0x00,0xc1,0x02,0x00,0x00,0x58,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x5b,0x01,0x00,0x00, +0x59,0x01,0x00,0x00,0x78,0x00,0x00,0x00,0x41,0x00,0x06,0x00, +0x5c,0x01,0x00,0x00,0x5d,0x01,0x00,0x00,0x51,0x01,0x00,0x00, +0x34,0x00,0x00,0x00,0x5b,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, +0xc3,0x00,0x00,0x00,0x5e,0x01,0x00,0x00,0x5d,0x01,0x00,0x00, +0x73,0x00,0x04,0x00,0xf6,0x00,0x00,0x00,0x5f,0x01,0x00,0x00, +0x5e,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0x14,0x01,0x00,0x00, +0x60,0x01,0x00,0x00,0x46,0x01,0x00,0x00,0x4d,0x01,0x00,0x00, +0x3e,0x00,0x03,0x00,0x60,0x01,0x00,0x00,0x5f,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0x41,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x61,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x64,0x01,0x00,0x00,0x7d,0x00,0x00,0x00,0xba,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x66,0x01,0x00,0x00, +0x64,0x01,0x00,0x00,0x65,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x68,0x01,0x00,0x00,0x66,0x01,0x00,0x00, +0x78,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x14,0x01,0x00,0x00, +0x69,0x01,0x00,0x00,0x46,0x01,0x00,0x00,0x68,0x01,0x00,0x00, +0x3e,0x00,0x03,0x00,0x69,0x01,0x00,0x00,0x1e,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0x41,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x41,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x2b,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x2b,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x6e,0x01,0x00,0x00,0xba,0x02,0x00,0x00, +0x6c,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x28,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x2a,0x01,0x00,0x00,0xe0,0x00,0x04,0x00, +0x0c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x6f,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x72,0x01,0x00,0x00, +0xbd,0x02,0x00,0x00,0x70,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x75,0x01,0x00,0x00,0xc1,0x02,0x00,0x00, +0x73,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x77,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x77,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xc3,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, +0x2a,0x01,0x00,0x00,0x21,0x02,0x00,0x00,0x7a,0x01,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00,0x7d,0x01,0x00,0x00, +0xc3,0x02,0x00,0x00,0x6c,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0x79,0x01,0x00,0x00,0x7a,0x01,0x00,0x00,0x00,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x7d,0x01,0x00,0x00,0x78,0x01,0x00,0x00, +0x79,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x78,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0x7f,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x7f,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xc7,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0x78,0x01,0x00,0x00, +0xab,0x01,0x00,0x00,0x82,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc1,0x00,0x00,0x00,0x85,0x01,0x00,0x00,0xc7,0x02,0x00,0x00, +0x60,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x81,0x01,0x00,0x00, +0x82,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x85,0x01,0x00,0x00,0x80,0x01,0x00,0x00,0x81,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x80,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0x87,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x87,0x01,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xd9,0x02,0x00,0x00, +0x3e,0x00,0x00,0x00,0x80,0x01,0x00,0x00,0xa9,0x01,0x00,0x00, +0x88,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00, +0x8d,0x01,0x00,0x00,0xd9,0x02,0x00,0x00,0x62,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0x89,0x01,0x00,0x00,0x88,0x01,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x8d,0x01,0x00,0x00, +0x88,0x01,0x00,0x00,0x89,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x88,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x93,0x01,0x00,0x00,0xc7,0x02,0x00,0x00,0x62,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x95,0x01,0x00,0x00, +0x93,0x01,0x00,0x00,0xd9,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x97,0x01,0x00,0x00,0x55,0x00,0x00,0x00, +0x53,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x99,0x01,0x00,0x00,0xc7,0x02,0x00,0x00,0x61,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9a,0x01,0x00,0x00, +0x97,0x01,0x00,0x00,0x99,0x01,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x9c,0x01,0x00,0x00,0x64,0x00,0x00,0x00, +0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x9d,0x01,0x00,0x00,0x9a,0x01,0x00,0x00,0x9c,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9f,0x01,0x00,0x00, +0x9d,0x01,0x00,0x00,0xd9,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xa1,0x01,0x00,0x00,0x9f,0x01,0x00,0x00, +0xa0,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xa3,0x01,0x00,0x00,0xa1,0x01,0x00,0x00,0xc3,0x02,0x00,0x00, +0x41,0x00,0x05,0x00,0x14,0x01,0x00,0x00,0xa4,0x01,0x00,0x00, +0xfb,0x00,0x00,0x00,0xa3,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, +0xf6,0x00,0x00,0x00,0xa5,0x01,0x00,0x00,0xa4,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0xa6,0x01,0x00,0x00,0xa7,0x01,0x00,0x00, +0x91,0x01,0x00,0x00,0x95,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, +0xa7,0x01,0x00,0x00,0xa5,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xa9,0x01,0x00,0x00,0xd9,0x02,0x00,0x00, +0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x87,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x89,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0x82,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x82,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xab,0x01,0x00,0x00, +0xc7,0x02,0x00,0x00,0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x7f,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x81,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0xad,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xad,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xc8,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0x81,0x01,0x00,0x00, +0xd9,0x01,0x00,0x00,0xb0,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc1,0x00,0x00,0x00,0xb3,0x01,0x00,0x00,0xc8,0x02,0x00,0x00, +0xbe,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xaf,0x01,0x00,0x00, +0xb0,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0xb3,0x01,0x00,0x00,0xae,0x01,0x00,0x00,0xaf,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xae,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0xb5,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xb5,0x01,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xd6,0x02,0x00,0x00, +0x3e,0x00,0x00,0x00,0xae,0x01,0x00,0x00,0xd7,0x01,0x00,0x00, +0xb6,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00, +0xbb,0x01,0x00,0x00,0xd6,0x02,0x00,0x00,0xbb,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0xb7,0x01,0x00,0x00,0xb6,0x01,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xbb,0x01,0x00,0x00, +0xb6,0x01,0x00,0x00,0xb7,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xb6,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xc1,0x01,0x00,0x00,0xc8,0x02,0x00,0x00,0xbb,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xc3,0x01,0x00,0x00, +0xc1,0x01,0x00,0x00,0xd6,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xc5,0x01,0x00,0x00,0x59,0x00,0x00,0x00, +0xb8,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xc8,0x01,0x00,0x00,0xc8,0x02,0x00,0x00,0xc7,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xc9,0x01,0x00,0x00, +0xc5,0x01,0x00,0x00,0xc8,0x01,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xcb,0x01,0x00,0x00,0x68,0x00,0x00,0x00, +0xbb,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xcc,0x01,0x00,0x00,0xc9,0x01,0x00,0x00,0xcb,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xce,0x01,0x00,0x00, +0xcc,0x01,0x00,0x00,0xd6,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xd0,0x01,0x00,0x00,0xce,0x01,0x00,0x00, +0xcf,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xd2,0x01,0x00,0x00,0xd0,0x01,0x00,0x00,0xc3,0x02,0x00,0x00, +0x41,0x00,0x05,0x00,0x14,0x01,0x00,0x00,0xd3,0x01,0x00,0x00, +0x46,0x01,0x00,0x00,0xd2,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, +0xf6,0x00,0x00,0x00,0xd4,0x01,0x00,0x00,0xd3,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0xa6,0x01,0x00,0x00,0xd5,0x01,0x00,0x00, +0xbf,0x01,0x00,0x00,0xc3,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, +0xd5,0x01,0x00,0x00,0xd4,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xd7,0x01,0x00,0x00,0xd6,0x02,0x00,0x00, +0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xb5,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xb7,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0xb0,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xb0,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xd9,0x01,0x00,0x00, +0xc8,0x02,0x00,0x00,0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xad,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xaf,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0xdb,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xdb,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xc9,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0xaf,0x01,0x00,0x00, +0x1f,0x02,0x00,0x00,0xde,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc1,0x00,0x00,0x00,0xe1,0x01,0x00,0x00,0xc9,0x02,0x00,0x00, +0xbe,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xdd,0x01,0x00,0x00, +0xde,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0xe1,0x01,0x00,0x00,0xdc,0x01,0x00,0x00,0xdd,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xdc,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0xe3,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xe3,0x01,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xcd,0x02,0x00,0x00, +0x3e,0x00,0x00,0x00,0xdc,0x01,0x00,0x00,0x1d,0x02,0x00,0x00, +0xe6,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00, +0xe9,0x01,0x00,0x00,0xcd,0x02,0x00,0x00,0x60,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0xe5,0x01,0x00,0x00,0xe6,0x01,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xe9,0x01,0x00,0x00, +0xe4,0x01,0x00,0x00,0xe5,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xe4,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xeb,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xeb,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xcf,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, +0xe4,0x01,0x00,0x00,0x1b,0x02,0x00,0x00,0xee,0x01,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00,0xf1,0x01,0x00,0x00, +0xcf,0x02,0x00,0x00,0xbb,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0xed,0x01,0x00,0x00,0xee,0x01,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0xf1,0x01,0x00,0x00,0xec,0x01,0x00,0x00, +0xed,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xec,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0xf3,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xf3,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xd1,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0xec,0x01,0x00,0x00, +0x19,0x02,0x00,0x00,0xf4,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc1,0x00,0x00,0x00,0xf9,0x01,0x00,0x00,0xd1,0x02,0x00,0x00, +0x62,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xf5,0x01,0x00,0x00, +0xf4,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0xf9,0x01,0x00,0x00,0xf4,0x01,0x00,0x00,0xf5,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xf4,0x01,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xfb,0x01,0x00,0x00,0xc9,0x02,0x00,0x00, +0xbb,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xfd,0x01,0x00,0x00,0xfb,0x01,0x00,0x00,0xcf,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xff,0x01,0x00,0x00, +0xfd,0x01,0x00,0x00,0xfe,0x01,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x01,0x02,0x00,0x00,0xcd,0x02,0x00,0x00, +0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x02,0x02,0x00,0x00,0xff,0x01,0x00,0x00,0x01,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x04,0x02,0x00,0x00, +0x02,0x02,0x00,0x00,0xd1,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x08,0x02,0x00,0x00,0x01,0x02,0x00,0x00, +0xd1,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0xa6,0x01,0x00,0x00, +0x09,0x02,0x00,0x00,0x91,0x01,0x00,0x00,0x08,0x02,0x00,0x00, +0x3d,0x00,0x04,0x00,0xf6,0x00,0x00,0x00,0x0a,0x02,0x00,0x00, +0x09,0x02,0x00,0x00,0x73,0x00,0x04,0x00,0xc3,0x00,0x00,0x00, +0x0b,0x02,0x00,0x00,0x0a,0x02,0x00,0x00,0x41,0x00,0x05,0x00, +0xa6,0x01,0x00,0x00,0x10,0x02,0x00,0x00,0xbf,0x01,0x00,0x00, +0xfd,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0xf6,0x00,0x00,0x00, +0x11,0x02,0x00,0x00,0x10,0x02,0x00,0x00,0x73,0x00,0x04,0x00, +0xc3,0x00,0x00,0x00,0x12,0x02,0x00,0x00,0x11,0x02,0x00,0x00, +0x41,0x00,0x05,0x00,0xcc,0x00,0x00,0x00,0x14,0x02,0x00,0x00, +0xc9,0x00,0x00,0x00,0x04,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, +0xc3,0x00,0x00,0x00,0x15,0x02,0x00,0x00,0x14,0x02,0x00,0x00, +0x0c,0x00,0x08,0x00,0xc3,0x00,0x00,0x00,0x16,0x02,0x00,0x00, +0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x0b,0x02,0x00,0x00, +0x12,0x02,0x00,0x00,0x15,0x02,0x00,0x00,0x3e,0x00,0x03,0x00, +0x14,0x02,0x00,0x00,0x16,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x19,0x02,0x00,0x00,0xd1,0x02,0x00,0x00, +0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xf3,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xf5,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0xee,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xee,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x1b,0x02,0x00,0x00, +0xcf,0x02,0x00,0x00,0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xeb,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xed,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0xe6,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xe6,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x1d,0x02,0x00,0x00,0xcd,0x02,0x00,0x00,0xcf,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xe3,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xe5,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xde,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xde,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x1f,0x02,0x00,0x00,0xc9,0x02,0x00,0x00, +0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xdb,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xdd,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0x7a,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x7a,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x21,0x02,0x00,0x00, +0xc3,0x02,0x00,0x00,0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x77,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x79,0x01,0x00,0x00, +0xe0,0x00,0x04,0x00,0x0c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x6f,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xd6,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xd6,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x23,0x02,0x00,0x00,0xa9,0x02,0x00,0x00, +0x6c,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xd3,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xd5,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x28,0x02,0x00,0x00,0x55,0x00,0x00,0x00, +0x53,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x29,0x02,0x00,0x00,0x95,0x00,0x00,0x00,0x28,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x2e,0x02,0x00,0x00, +0x59,0x00,0x00,0x00,0xb8,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x2f,0x02,0x00,0x00,0xa7,0x00,0x00,0x00, +0x2e,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, +0x34,0x02,0x00,0x00,0x12,0x00,0x00,0x00,0x33,0x02,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x35,0x02,0x00,0x00, +0x34,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x36,0x02,0x00,0x00,0x0f,0x00,0x00,0x00,0x35,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3a,0x02,0x00,0x00, +0x47,0x00,0x00,0x00,0x35,0x02,0x00,0x00,0x41,0x00,0x05,0x00, +0x0d,0x00,0x00,0x00,0x3c,0x02,0x00,0x00,0x3b,0x02,0x00,0x00, +0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x3d,0x02,0x00,0x00,0x3c,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x3e,0x02,0x00,0x00,0x3a,0x02,0x00,0x00, +0x3d,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x3f,0x02,0x00,0x00,0x36,0x02,0x00,0x00,0x3e,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0x41,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x41,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xaa,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0xd5,0x00,0x00,0x00, +0xa7,0x02,0x00,0x00,0x44,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc1,0x00,0x00,0x00,0x47,0x02,0x00,0x00,0xaa,0x02,0x00,0x00, +0xbe,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x43,0x02,0x00,0x00, +0x44,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x47,0x02,0x00,0x00,0x42,0x02,0x00,0x00,0x43,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x42,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0x49,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x49,0x02,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xab,0x02,0x00,0x00, +0x3e,0x00,0x00,0x00,0x42,0x02,0x00,0x00,0xa5,0x02,0x00,0x00, +0x4c,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00, +0x4f,0x02,0x00,0x00,0xab,0x02,0x00,0x00,0x60,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0x4b,0x02,0x00,0x00,0x4c,0x02,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x4f,0x02,0x00,0x00, +0x4a,0x02,0x00,0x00,0x4b,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x4a,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x53,0x02,0x00,0x00,0xab,0x02,0x00,0x00,0x61,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x54,0x02,0x00,0x00, +0x29,0x02,0x00,0x00,0x53,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x56,0x02,0x00,0x00,0x64,0x00,0x00,0x00, +0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x57,0x02,0x00,0x00,0x54,0x02,0x00,0x00,0x56,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x5b,0x02,0x00,0x00, +0xaa,0x02,0x00,0x00,0xc7,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x5c,0x02,0x00,0x00,0x2f,0x02,0x00,0x00, +0x5b,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x5e,0x02,0x00,0x00,0x68,0x00,0x00,0x00,0xbb,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x5f,0x02,0x00,0x00, +0x5c,0x02,0x00,0x00,0x5e,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0x61,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x61,0x02,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xad,0x02,0x00,0x00, +0x3e,0x00,0x00,0x00,0x4a,0x02,0x00,0x00,0xa3,0x02,0x00,0x00, +0x64,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00, +0x67,0x02,0x00,0x00,0xad,0x02,0x00,0x00,0xbb,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0x63,0x02,0x00,0x00,0x64,0x02,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x67,0x02,0x00,0x00, +0x62,0x02,0x00,0x00,0x63,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x62,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x69,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x69,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xaf,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, +0x62,0x02,0x00,0x00,0xa1,0x02,0x00,0x00,0x6c,0x02,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00,0x6f,0x02,0x00,0x00, +0xaf,0x02,0x00,0x00,0x62,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0x6b,0x02,0x00,0x00,0x6c,0x02,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x6f,0x02,0x00,0x00,0x6a,0x02,0x00,0x00, +0x6b,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x6a,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x72,0x02,0x00,0x00, +0x57,0x02,0x00,0x00,0xaf,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc1,0x00,0x00,0x00,0x75,0x02,0x00,0x00,0x72,0x02,0x00,0x00, +0x36,0x00,0x00,0x00,0xf7,0x00,0x03,0x00,0x77,0x02,0x00,0x00, +0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x75,0x02,0x00,0x00, +0x76,0x02,0x00,0x00,0x77,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x76,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x7a,0x02,0x00,0x00,0x5f,0x02,0x00,0x00,0xad,0x02,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x7b,0x02,0x00,0x00, +0x12,0x00,0x00,0x00,0xcf,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x7c,0x02,0x00,0x00,0x7b,0x02,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00,0x7d,0x02,0x00,0x00, +0x7a,0x02,0x00,0x00,0x7c,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0x77,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x77,0x02,0x00,0x00, +0xf5,0x00,0x07,0x00,0xc1,0x00,0x00,0x00,0x7e,0x02,0x00,0x00, +0x75,0x02,0x00,0x00,0x6a,0x02,0x00,0x00,0x7d,0x02,0x00,0x00, +0x76,0x02,0x00,0x00,0xf7,0x00,0x03,0x00,0x80,0x02,0x00,0x00, +0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x7e,0x02,0x00,0x00, +0x7f,0x02,0x00,0x00,0x80,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x7f,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x88,0x02,0x00,0x00,0x5f,0x02,0x00,0x00,0xad,0x02,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x8a,0x02,0x00,0x00, +0x12,0x00,0x00,0x00,0x89,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x8b,0x02,0x00,0x00,0x8a,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8c,0x02,0x00,0x00, +0x88,0x02,0x00,0x00,0x8b,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x8d,0x02,0x00,0x00,0x3f,0x02,0x00,0x00, +0x8c,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x8f,0x02,0x00,0x00,0x8d,0x02,0x00,0x00,0x57,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x91,0x02,0x00,0x00, +0x8f,0x02,0x00,0x00,0xaf,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x93,0x02,0x00,0x00,0xaa,0x02,0x00,0x00, +0xbb,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x95,0x02,0x00,0x00,0x93,0x02,0x00,0x00,0xad,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x97,0x02,0x00,0x00, +0x95,0x02,0x00,0x00,0x96,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x99,0x02,0x00,0x00,0xab,0x02,0x00,0x00, +0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x9a,0x02,0x00,0x00,0x97,0x02,0x00,0x00,0x99,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9c,0x02,0x00,0x00, +0x9a,0x02,0x00,0x00,0xaf,0x02,0x00,0x00,0x41,0x00,0x05,0x00, +0xcc,0x00,0x00,0x00,0x9d,0x02,0x00,0x00,0xc9,0x00,0x00,0x00, +0x9c,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0xc3,0x00,0x00,0x00, +0x9e,0x02,0x00,0x00,0x9d,0x02,0x00,0x00,0x41,0x00,0x06,0x00, +0x5c,0x01,0x00,0x00,0x9f,0x02,0x00,0x00,0x84,0x02,0x00,0x00, +0x34,0x00,0x00,0x00,0x91,0x02,0x00,0x00,0x3e,0x00,0x03,0x00, +0x9f,0x02,0x00,0x00,0x9e,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0x80,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x80,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0x6c,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x6c,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xa1,0x02,0x00,0x00,0xaf,0x02,0x00,0x00,0xcf,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x69,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x6b,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x64,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x64,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xa3,0x02,0x00,0x00,0xad,0x02,0x00,0x00, +0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x61,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x63,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0x4c,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x4c,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa5,0x02,0x00,0x00, +0xab,0x02,0x00,0x00,0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x49,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x4b,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0x44,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x44,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xa7,0x02,0x00,0x00,0xaa,0x02,0x00,0x00,0xcf,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x41,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x43,0x02,0x00,0x00,0xfd,0x00,0x01,0x00,0x38,0x00,0x01,0x00, + +}; +const uint64_t matmul_f16_f32_len = 10332; + +unsigned char matmul_f16_f32_aligned_data[] = { +0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, +0x13,0x03,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, +0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x09,0x00,0x00,0x00, +0x11,0x00,0x02,0x00,0x51,0x11,0x00,0x00,0x0b,0x00,0x06,0x00, +0x01,0x00,0x00,0x00,0x47,0x4c,0x53,0x4c,0x2e,0x73,0x74,0x64, +0x2e,0x34,0x35,0x30,0x00,0x00,0x00,0x00,0x0e,0x00,0x03,0x00, +0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x0f,0x00,0x0f,0x00, +0x05,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x6d,0x61,0x69,0x6e, +0x00,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0x3d,0x00,0x00,0x00,0x4c,0x00,0x00,0x00,0xfe,0x00,0x00,0x00, +0x05,0x01,0x00,0x00,0x63,0x01,0x00,0x00,0x6b,0x01,0x00,0x00, +0x74,0x02,0x00,0x00,0xbd,0x02,0x00,0x00,0x10,0x00,0x06,0x00, +0x04,0x00,0x00,0x00,0x11,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x0b,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x1c,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x10,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x08,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x03,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x10,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x10,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, +0x05,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x14,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x10,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x1c,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, +0x08,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x09,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x24,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x10,0x00,0x00,0x00,0x0a,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x28,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, +0x0b,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x2c,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x30,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x10,0x00,0x00,0x00,0x0d,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x34,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x10,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x37,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x3d,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x1a,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x4c,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, +0x1b,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x4f,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x53,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x60,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x62,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x6c,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x03,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0xa7,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xb9,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0xbc,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x08,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x02,0x01,0x00,0x00,0x06,0x00,0x00,0x00, +0x10,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0x03,0x01,0x00,0x00, +0x00,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0x48,0x00,0x04,0x00, +0x03,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x03,0x01,0x00,0x00,0x00,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x03,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x07,0x00,0x00,0x00, +0x08,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x03,0x01,0x00,0x00, +0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x05,0x01,0x00,0x00, +0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x05,0x01,0x00,0x00,0x21,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x3b,0x01,0x00,0x00,0x01,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x3c,0x01,0x00,0x00, +0x0b,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x68,0x01,0x00,0x00,0x06,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0x48,0x00,0x04,0x00,0x69,0x01,0x00,0x00,0x00,0x00,0x00,0x00, +0x05,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0x69,0x01,0x00,0x00, +0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x69,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x69,0x01,0x00,0x00, +0x00,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0x10,0x00,0x00,0x00, +0x47,0x00,0x03,0x00,0x69,0x01,0x00,0x00,0x02,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x6b,0x01,0x00,0x00,0x22,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x6b,0x01,0x00,0x00, +0x21,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x74,0x02,0x00,0x00,0x0b,0x00,0x00,0x00,0x18,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0xba,0x02,0x00,0x00,0x06,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0xbb,0x02,0x00,0x00, +0x00,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0xbb,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0xbb,0x02,0x00,0x00, +0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xbd,0x02,0x00,0x00, +0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0xbd,0x02,0x00,0x00,0x21,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x13,0x00,0x02,0x00,0x02,0x00,0x00,0x00,0x21,0x00,0x03,0x00, +0x03,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x15,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x17,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x03,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x0a,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0x0a,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x0d,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x1e,0x00,0x10,0x00, +0x10,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x11,0x00,0x00,0x00,0x09,0x00,0x00,0x00, +0x10,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x11,0x00,0x00,0x00, +0x12,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x15,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x14,0x00,0x00,0x00, +0x08,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x15,0x00,0x00,0x00, +0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x0a,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x27,0x00,0x00,0x00, +0x09,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, +0x2d,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x37,0x00,0x00,0x00, +0x40,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x39,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0x0a,0x00,0x00,0x00,0x3d,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x3e,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00, +0x4c,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x53,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x54,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x37,0x00,0x00,0x00, +0x53,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x58,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x37,0x00,0x00,0x00, +0x53,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x60,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x61,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0x53,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x32,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x63,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0x61,0x00,0x00,0x00,0x62,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x67,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0x61,0x00,0x00,0x00,0x62,0x00,0x00,0x00, +0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, +0x10,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x6d,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x6e,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x73,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x78,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x7d,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0x81,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, +0x91,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0x97,0x00,0x00,0x00,0x03,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0xa2,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xa7,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0xa9,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xb8,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00, +0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xb9,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xba,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x53,0x00,0x00,0x00, +0xb9,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xbb,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x4f,0x00,0x00,0x00, +0x62,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xbc,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xbd,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0xbb,0x00,0x00,0x00,0xbc,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xbe,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0xbd,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xbf,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0xba,0x00,0x00,0x00,0xbe,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xc0,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0xb8,0x00,0x00,0x00,0xbf,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xc1,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0xc0,0x00,0x00,0x00,0xbc,0x00,0x00,0x00,0x14,0x00,0x02,0x00, +0xc2,0x00,0x00,0x00,0x16,0x00,0x03,0x00,0xc4,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xc5,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00, +0x62,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xc6,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0xc5,0x00,0x00,0x00, +0xbf,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xc7,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0xc6,0x00,0x00,0x00, +0xbc,0x00,0x00,0x00,0x1c,0x00,0x04,0x00,0xc8,0x00,0x00,0x00, +0xc4,0x00,0x00,0x00,0xc7,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0xc9,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0xc8,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0xc4,0x00,0x00,0x00,0xcc,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0xcd,0x00,0x00,0x00, +0x07,0x00,0x00,0x00,0xc4,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0xd0,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xf4,0x00,0x00,0x00, +0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0x16,0x00,0x03,0x00,0xf9,0x00,0x00,0x00,0x10,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xfa,0x00,0x00,0x00, +0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xfb,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0xfa,0x00,0x00,0x00, +0x1c,0x00,0x04,0x00,0xfc,0x00,0x00,0x00,0xf9,0x00,0x00,0x00, +0xfb,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0xfd,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0xfc,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0xfd,0x00,0x00,0x00,0xfe,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x17,0x00,0x04,0x00,0x00,0x01,0x00,0x00,0xf9,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x18,0x00,0x04,0x00,0x01,0x01,0x00,0x00, +0x00,0x01,0x00,0x00,0x02,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, +0x02,0x01,0x00,0x00,0x01,0x01,0x00,0x00,0x1e,0x00,0x03,0x00, +0x03,0x01,0x00,0x00,0x02,0x01,0x00,0x00,0x20,0x00,0x04,0x00, +0x04,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x03,0x01,0x00,0x00, +0x3b,0x00,0x04,0x00,0x04,0x01,0x00,0x00,0x05,0x01,0x00,0x00, +0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x07,0x01,0x00,0x00, +0x0c,0x00,0x00,0x00,0xf9,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x0a,0x01,0x00,0x00,0x04,0x00,0x00,0x00,0xf9,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x19,0x01,0x00,0x00, +0x03,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x20,0x01,0x00,0x00,0x04,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x27,0x01,0x00,0x00,0x05,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x2e,0x01,0x00,0x00, +0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x35,0x01,0x00,0x00,0x07,0x00,0x00,0x00,0x32,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x3b,0x01,0x00,0x00,0x01,0x00,0x00,0x00, +0x33,0x00,0x06,0x00,0x09,0x00,0x00,0x00,0x3c,0x01,0x00,0x00, +0x3b,0x01,0x00,0x00,0x39,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x3d,0x01,0x00,0x00, +0x51,0x00,0x00,0x00,0x3c,0x01,0x00,0x00,0x00,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x3e,0x01,0x00,0x00, +0x84,0x00,0x00,0x00,0x3d,0x01,0x00,0x00,0x6d,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x3f,0x01,0x00,0x00, +0x86,0x00,0x00,0x00,0x3e,0x01,0x00,0x00,0x6c,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x5a,0x01,0x00,0x00, +0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x5f,0x01,0x00,0x00, +0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x60,0x01,0x00,0x00, +0x84,0x00,0x00,0x00,0xa7,0x00,0x00,0x00,0x5f,0x01,0x00,0x00, +0x1c,0x00,0x04,0x00,0x61,0x01,0x00,0x00,0xf9,0x00,0x00,0x00, +0x60,0x01,0x00,0x00,0x20,0x00,0x04,0x00,0x62,0x01,0x00,0x00, +0x04,0x00,0x00,0x00,0x61,0x01,0x00,0x00,0x3b,0x00,0x04,0x00, +0x62,0x01,0x00,0x00,0x63,0x01,0x00,0x00,0x04,0x00,0x00,0x00, +0x17,0x00,0x04,0x00,0x66,0x01,0x00,0x00,0xc4,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x18,0x00,0x04,0x00,0x67,0x01,0x00,0x00, +0x66,0x01,0x00,0x00,0x02,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, +0x68,0x01,0x00,0x00,0x67,0x01,0x00,0x00,0x1e,0x00,0x03,0x00, +0x69,0x01,0x00,0x00,0x68,0x01,0x00,0x00,0x20,0x00,0x04,0x00, +0x6a,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x69,0x01,0x00,0x00, +0x3b,0x00,0x04,0x00,0x6a,0x01,0x00,0x00,0x6b,0x01,0x00,0x00, +0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x6d,0x01,0x00,0x00, +0x0c,0x00,0x00,0x00,0xc4,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xa3,0x01,0x00,0x00,0x51,0x00,0x00,0x00, +0x3c,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xa4,0x01,0x00,0x00,0x84,0x00,0x00,0x00, +0xa3,0x01,0x00,0x00,0x6d,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xa5,0x01,0x00,0x00,0x86,0x00,0x00,0x00, +0xa4,0x01,0x00,0x00,0x6c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xa8,0x01,0x00,0x00,0x08,0x01,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xa9,0x01,0x00,0x00, +0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xac,0x01,0x00,0x00, +0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xc7,0x01,0x00,0x00, +0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00, +0x1c,0x00,0x04,0x00,0xc8,0x01,0x00,0x00,0xf9,0x00,0x00,0x00, +0xc7,0x01,0x00,0x00,0x20,0x00,0x04,0x00,0xc9,0x01,0x00,0x00, +0x07,0x00,0x00,0x00,0xc8,0x01,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xd9,0x01,0x00,0x00,0x80,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0xdf,0x01,0x00,0x00,0x07,0x00,0x00,0x00,0xf9,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xf5,0x01,0x00,0x00, +0x84,0x00,0x00,0x00,0xbf,0x00,0x00,0x00,0xbc,0x00,0x00,0x00, +0x1c,0x00,0x04,0x00,0xf6,0x01,0x00,0x00,0xf9,0x00,0x00,0x00, +0xf5,0x01,0x00,0x00,0x20,0x00,0x04,0x00,0xf7,0x01,0x00,0x00, +0x07,0x00,0x00,0x00,0xf6,0x01,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x00,0x02,0x00,0x00,0x86,0x00,0x00,0x00, +0xb9,0x00,0x00,0x00,0xbf,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x08,0x02,0x00,0x00,0x80,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x37,0x02,0x00,0x00,0x84,0x00,0x00,0x00, +0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0x6c,0x02,0x00,0x00,0x0d,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00,0x74,0x02,0x00,0x00, +0x01,0x00,0x00,0x00,0x1d,0x00,0x03,0x00,0xba,0x02,0x00,0x00, +0xc4,0x00,0x00,0x00,0x1e,0x00,0x03,0x00,0xbb,0x02,0x00,0x00, +0xba,0x02,0x00,0x00,0x20,0x00,0x04,0x00,0xbc,0x02,0x00,0x00, +0x0c,0x00,0x00,0x00,0xbb,0x02,0x00,0x00,0x3b,0x00,0x04,0x00, +0xbc,0x02,0x00,0x00,0xbd,0x02,0x00,0x00,0x0c,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0xc2,0x02,0x00,0x00, +0x05,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xcf,0x02,0x00,0x00,0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00, +0x62,0x00,0x00,0x00,0x36,0x00,0x05,0x00,0x02,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x03,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0x05,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0xc9,0x00,0x00,0x00,0xca,0x00,0x00,0x00,0x07,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0xc9,0x01,0x00,0x00,0xca,0x01,0x00,0x00, +0x07,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0xf7,0x01,0x00,0x00, +0xf8,0x01,0x00,0x00,0x07,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x0d,0x00,0x00,0x00,0x0e,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x0f,0x00,0x00,0x00,0x0e,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x15,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0x14,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x17,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, +0x17,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x1e,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x17,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x22,0x00,0x00,0x00, +0x12,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x22,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x24,0x00,0x00,0x00, +0x18,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x15,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0x27,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x29,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x2a,0x00,0x00,0x00,0x1e,0x00,0x00,0x00, +0x29,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, +0x2e,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x2d,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x2f,0x00,0x00,0x00, +0x2e,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x30,0x00,0x00,0x00,0x24,0x00,0x00,0x00,0x2f,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x32,0x00,0x00,0x00, +0x30,0x00,0x00,0x00,0x2a,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x15,0x00,0x00,0x00,0x35,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0x34,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x36,0x00,0x00,0x00,0x35,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x38,0x00,0x00,0x00,0x36,0x00,0x00,0x00, +0x37,0x00,0x00,0x00,0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x3a,0x00,0x00,0x00,0x38,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3b,0x00,0x00,0x00, +0x3a,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x0d,0x00,0x00,0x00,0x3f,0x00,0x00,0x00,0x3d,0x00,0x00,0x00, +0x3e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x40,0x00,0x00,0x00,0x3f,0x00,0x00,0x00,0x89,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x42,0x00,0x00,0x00,0x40,0x00,0x00,0x00, +0x3b,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x47,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x3b,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00,0x49,0x00,0x00,0x00, +0x3d,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x4a,0x00,0x00,0x00,0x49,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00,0x4d,0x00,0x00,0x00, +0x4c,0x00,0x00,0x00,0x3e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x4d,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x50,0x00,0x00,0x00, +0x4e,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x89,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x55,0x00,0x00,0x00,0x50,0x00,0x00,0x00, +0x54,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x59,0x00,0x00,0x00,0x50,0x00,0x00,0x00,0x58,0x00,0x00,0x00, +0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x5d,0x00,0x00,0x00, +0x4e,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x89,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x64,0x00,0x00,0x00,0x5d,0x00,0x00,0x00, +0x63,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x68,0x00,0x00,0x00,0x5d,0x00,0x00,0x00,0x67,0x00,0x00,0x00, +0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x6f,0x00,0x00,0x00, +0x4e,0x00,0x00,0x00,0x6e,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x74,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, +0x73,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x79,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x78,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x7e,0x00,0x00,0x00, +0x4e,0x00,0x00,0x00,0x7d,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x15,0x00,0x00,0x00,0x82,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0x81,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x83,0x00,0x00,0x00,0x82,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x47,0x00,0x00,0x00, +0x83,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, +0x87,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x88,0x00,0x00,0x00, +0x87,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x8a,0x00,0x00,0x00,0x47,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8d,0x00,0x00,0x00, +0x8a,0x00,0x00,0x00,0x83,0x00,0x00,0x00,0x0c,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0x8e,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x26,0x00,0x00,0x00,0x88,0x00,0x00,0x00,0x8d,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x92,0x00,0x00,0x00, +0x12,0x00,0x00,0x00,0x91,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x93,0x00,0x00,0x00,0x92,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x94,0x00,0x00,0x00, +0x32,0x00,0x00,0x00,0x93,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x96,0x00,0x00,0x00,0x42,0x00,0x00,0x00, +0x37,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, +0x98,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x97,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x99,0x00,0x00,0x00, +0x98,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x9a,0x00,0x00,0x00,0x96,0x00,0x00,0x00,0x99,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9b,0x00,0x00,0x00, +0x94,0x00,0x00,0x00,0x9a,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x9d,0x00,0x00,0x00,0x9b,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x9e,0x00,0x00,0x00,0x9d,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0xa3,0x00,0x00,0x00, +0x12,0x00,0x00,0x00,0xa2,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xa4,0x00,0x00,0x00,0xa3,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa5,0x00,0x00,0x00, +0x0f,0x00,0x00,0x00,0xa4,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xa8,0x00,0x00,0x00,0x4a,0x00,0x00,0x00, +0xa7,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, +0xaa,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0xa9,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xab,0x00,0x00,0x00, +0xaa,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xac,0x00,0x00,0x00,0xa8,0x00,0x00,0x00,0xab,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xad,0x00,0x00,0x00, +0xa5,0x00,0x00,0x00,0xac,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xaf,0x00,0x00,0x00,0xad,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xb0,0x00,0x00,0x00,0xaf,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xb2,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xb2,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xe1,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0x05,0x00,0x00,0x00, +0xd1,0x00,0x00,0x00,0xb3,0x00,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc2,0x00,0x00,0x00,0xc3,0x00,0x00,0x00,0xe1,0x02,0x00,0x00, +0xc1,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xb4,0x00,0x00,0x00, +0xb3,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0xc3,0x00,0x00,0x00,0xb3,0x00,0x00,0x00,0xb4,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xb3,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0xcd,0x00,0x00,0x00,0xce,0x00,0x00,0x00,0xca,0x00,0x00,0x00, +0xe1,0x02,0x00,0x00,0x3e,0x00,0x03,0x00,0xce,0x00,0x00,0x00, +0xcc,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xd1,0x00,0x00,0x00,0xe1,0x02,0x00,0x00,0xd0,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xb2,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xb4,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xd4,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xd4,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xfa,0x02,0x00,0x00,0xb0,0x00,0x00,0x00, +0xb4,0x00,0x00,0x00,0xae,0x01,0x00,0x00,0xd7,0x00,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xf6,0x02,0x00,0x00, +0x9e,0x00,0x00,0x00,0xb4,0x00,0x00,0x00,0xab,0x01,0x00,0x00, +0xd7,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xe2,0x02,0x00,0x00,0x84,0x00,0x00,0x00,0xb4,0x00,0x00,0x00, +0x5c,0x02,0x00,0x00,0xd7,0x00,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc2,0x00,0x00,0x00,0xdb,0x00,0x00,0x00,0xe2,0x02,0x00,0x00, +0x8e,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xd6,0x00,0x00,0x00, +0xd7,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0xdb,0x00,0x00,0x00,0xd5,0x00,0x00,0x00,0xd6,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xd5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xdd,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xdd,0x00,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xf2,0x02,0x00,0x00, +0x3e,0x00,0x00,0x00,0xd5,0x00,0x00,0x00,0x41,0x01,0x00,0x00, +0xde,0x00,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, +0xe3,0x00,0x00,0x00,0xf2,0x02,0x00,0x00,0x37,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0xdf,0x00,0x00,0x00,0xde,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xe3,0x00,0x00,0x00, +0xde,0x00,0x00,0x00,0xdf,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xde,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xe8,0x00,0x00,0x00,0x74,0x00,0x00,0x00,0xf2,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xeb,0x00,0x00,0x00, +0xe8,0x00,0x00,0x00,0x99,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xec,0x00,0x00,0x00,0xeb,0x00,0x00,0x00, +0x6d,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xed,0x00,0x00,0x00,0xf6,0x02,0x00,0x00,0xec,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xef,0x00,0x00,0x00, +0xed,0x00,0x00,0x00,0x6f,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xf5,0x00,0x00,0x00,0xe8,0x00,0x00,0x00, +0xf4,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xf7,0x00,0x00,0x00,0x6f,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xf8,0x00,0x00,0x00, +0xf5,0x00,0x00,0x00,0xf7,0x00,0x00,0x00,0x41,0x00,0x08,0x00, +0x07,0x01,0x00,0x00,0x08,0x01,0x00,0x00,0x05,0x01,0x00,0x00, +0x34,0x00,0x00,0x00,0xef,0x00,0x00,0x00,0x34,0x00,0x00,0x00, +0x3e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0xf9,0x00,0x00,0x00, +0x09,0x01,0x00,0x00,0x08,0x01,0x00,0x00,0x41,0x00,0x05,0x00, +0x0a,0x01,0x00,0x00,0x0b,0x01,0x00,0x00,0xfe,0x00,0x00,0x00, +0xf8,0x00,0x00,0x00,0x3e,0x00,0x03,0x00,0x0b,0x01,0x00,0x00, +0x09,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x0d,0x01,0x00,0x00,0xf8,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0x41,0x00,0x08,0x00,0x07,0x01,0x00,0x00,0x0f,0x01,0x00,0x00, +0x05,0x01,0x00,0x00,0x34,0x00,0x00,0x00,0xef,0x00,0x00,0x00, +0x34,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0xf9,0x00,0x00,0x00,0x10,0x01,0x00,0x00,0x0f,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0x0a,0x01,0x00,0x00,0x11,0x01,0x00,0x00, +0xfe,0x00,0x00,0x00,0x0d,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, +0x11,0x01,0x00,0x00,0x10,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x13,0x01,0x00,0x00,0xf8,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x41,0x00,0x08,0x00,0x07,0x01,0x00,0x00, +0x15,0x01,0x00,0x00,0x05,0x01,0x00,0x00,0x34,0x00,0x00,0x00, +0xef,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0xf9,0x00,0x00,0x00,0x16,0x01,0x00,0x00, +0x15,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0x0a,0x01,0x00,0x00, +0x17,0x01,0x00,0x00,0xfe,0x00,0x00,0x00,0x13,0x01,0x00,0x00, +0x3e,0x00,0x03,0x00,0x17,0x01,0x00,0x00,0x16,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x1a,0x01,0x00,0x00, +0xf8,0x00,0x00,0x00,0x19,0x01,0x00,0x00,0x41,0x00,0x08,0x00, +0x07,0x01,0x00,0x00,0x1c,0x01,0x00,0x00,0x05,0x01,0x00,0x00, +0x34,0x00,0x00,0x00,0xef,0x00,0x00,0x00,0x34,0x00,0x00,0x00, +0x19,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0xf9,0x00,0x00,0x00, +0x1d,0x01,0x00,0x00,0x1c,0x01,0x00,0x00,0x41,0x00,0x05,0x00, +0x0a,0x01,0x00,0x00,0x1e,0x01,0x00,0x00,0xfe,0x00,0x00,0x00, +0x1a,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x1e,0x01,0x00,0x00, +0x1d,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x21,0x01,0x00,0x00,0xf8,0x00,0x00,0x00,0x20,0x01,0x00,0x00, +0x41,0x00,0x08,0x00,0x07,0x01,0x00,0x00,0x23,0x01,0x00,0x00, +0x05,0x01,0x00,0x00,0x34,0x00,0x00,0x00,0xef,0x00,0x00,0x00, +0xd0,0x00,0x00,0x00,0x3e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0xf9,0x00,0x00,0x00,0x24,0x01,0x00,0x00,0x23,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0x0a,0x01,0x00,0x00,0x25,0x01,0x00,0x00, +0xfe,0x00,0x00,0x00,0x21,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, +0x25,0x01,0x00,0x00,0x24,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x28,0x01,0x00,0x00,0xf8,0x00,0x00,0x00, +0x27,0x01,0x00,0x00,0x41,0x00,0x08,0x00,0x07,0x01,0x00,0x00, +0x2a,0x01,0x00,0x00,0x05,0x01,0x00,0x00,0x34,0x00,0x00,0x00, +0xef,0x00,0x00,0x00,0xd0,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0xf9,0x00,0x00,0x00,0x2b,0x01,0x00,0x00, +0x2a,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0x0a,0x01,0x00,0x00, +0x2c,0x01,0x00,0x00,0xfe,0x00,0x00,0x00,0x28,0x01,0x00,0x00, +0x3e,0x00,0x03,0x00,0x2c,0x01,0x00,0x00,0x2b,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x2f,0x01,0x00,0x00, +0xf8,0x00,0x00,0x00,0x2e,0x01,0x00,0x00,0x41,0x00,0x08,0x00, +0x07,0x01,0x00,0x00,0x31,0x01,0x00,0x00,0x05,0x01,0x00,0x00, +0x34,0x00,0x00,0x00,0xef,0x00,0x00,0x00,0xd0,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0xf9,0x00,0x00,0x00, +0x32,0x01,0x00,0x00,0x31,0x01,0x00,0x00,0x41,0x00,0x05,0x00, +0x0a,0x01,0x00,0x00,0x33,0x01,0x00,0x00,0xfe,0x00,0x00,0x00, +0x2f,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x33,0x01,0x00,0x00, +0x32,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x36,0x01,0x00,0x00,0xf8,0x00,0x00,0x00,0x35,0x01,0x00,0x00, +0x41,0x00,0x08,0x00,0x07,0x01,0x00,0x00,0x38,0x01,0x00,0x00, +0x05,0x01,0x00,0x00,0x34,0x00,0x00,0x00,0xef,0x00,0x00,0x00, +0xd0,0x00,0x00,0x00,0x19,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, +0xf9,0x00,0x00,0x00,0x39,0x01,0x00,0x00,0x38,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0x0a,0x01,0x00,0x00,0x3a,0x01,0x00,0x00, +0xfe,0x00,0x00,0x00,0x36,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, +0x3a,0x01,0x00,0x00,0x39,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x41,0x01,0x00,0x00,0xf2,0x02,0x00,0x00, +0x3f,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xdd,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xdf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x43,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x43,0x01,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xf3,0x02,0x00,0x00, +0x3e,0x00,0x00,0x00,0xdf,0x00,0x00,0x00,0xa7,0x01,0x00,0x00, +0x44,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, +0x49,0x01,0x00,0x00,0xf3,0x02,0x00,0x00,0xa7,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0x45,0x01,0x00,0x00,0x44,0x01,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x49,0x01,0x00,0x00, +0x44,0x01,0x00,0x00,0x45,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x44,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x4e,0x01,0x00,0x00,0x7e,0x00,0x00,0x00,0xf3,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x51,0x01,0x00,0x00, +0x4e,0x01,0x00,0x00,0xab,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x52,0x01,0x00,0x00,0x51,0x01,0x00,0x00, +0x6d,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x53,0x01,0x00,0x00,0xfa,0x02,0x00,0x00,0x52,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x55,0x01,0x00,0x00, +0x53,0x01,0x00,0x00,0x79,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x5b,0x01,0x00,0x00,0x4e,0x01,0x00,0x00, +0x5a,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x5d,0x01,0x00,0x00,0x79,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x5e,0x01,0x00,0x00, +0x5b,0x01,0x00,0x00,0x5d,0x01,0x00,0x00,0x41,0x00,0x08,0x00, +0x6d,0x01,0x00,0x00,0x6e,0x01,0x00,0x00,0x6b,0x01,0x00,0x00, +0x34,0x00,0x00,0x00,0x55,0x01,0x00,0x00,0x34,0x00,0x00,0x00, +0x3e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00, +0x6f,0x01,0x00,0x00,0x6e,0x01,0x00,0x00,0x73,0x00,0x04,0x00, +0xf9,0x00,0x00,0x00,0x70,0x01,0x00,0x00,0x6f,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0x0a,0x01,0x00,0x00,0x71,0x01,0x00,0x00, +0x63,0x01,0x00,0x00,0x5e,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, +0x71,0x01,0x00,0x00,0x70,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x73,0x01,0x00,0x00,0x5e,0x01,0x00,0x00, +0x39,0x00,0x00,0x00,0x41,0x00,0x08,0x00,0x6d,0x01,0x00,0x00, +0x75,0x01,0x00,0x00,0x6b,0x01,0x00,0x00,0x34,0x00,0x00,0x00, +0x55,0x01,0x00,0x00,0x34,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00,0x76,0x01,0x00,0x00, +0x75,0x01,0x00,0x00,0x73,0x00,0x04,0x00,0xf9,0x00,0x00,0x00, +0x77,0x01,0x00,0x00,0x76,0x01,0x00,0x00,0x41,0x00,0x05,0x00, +0x0a,0x01,0x00,0x00,0x78,0x01,0x00,0x00,0x63,0x01,0x00,0x00, +0x73,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x78,0x01,0x00,0x00, +0x77,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x7a,0x01,0x00,0x00,0x5e,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, +0x41,0x00,0x08,0x00,0x6d,0x01,0x00,0x00,0x7c,0x01,0x00,0x00, +0x6b,0x01,0x00,0x00,0x34,0x00,0x00,0x00,0x55,0x01,0x00,0x00, +0x34,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0xc4,0x00,0x00,0x00,0x7d,0x01,0x00,0x00,0x7c,0x01,0x00,0x00, +0x73,0x00,0x04,0x00,0xf9,0x00,0x00,0x00,0x7e,0x01,0x00,0x00, +0x7d,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0x0a,0x01,0x00,0x00, +0x7f,0x01,0x00,0x00,0x63,0x01,0x00,0x00,0x7a,0x01,0x00,0x00, +0x3e,0x00,0x03,0x00,0x7f,0x01,0x00,0x00,0x7e,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x81,0x01,0x00,0x00, +0x5e,0x01,0x00,0x00,0x19,0x01,0x00,0x00,0x41,0x00,0x08,0x00, +0x6d,0x01,0x00,0x00,0x83,0x01,0x00,0x00,0x6b,0x01,0x00,0x00, +0x34,0x00,0x00,0x00,0x55,0x01,0x00,0x00,0x34,0x00,0x00,0x00, +0x19,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00, +0x84,0x01,0x00,0x00,0x83,0x01,0x00,0x00,0x73,0x00,0x04,0x00, +0xf9,0x00,0x00,0x00,0x85,0x01,0x00,0x00,0x84,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0x0a,0x01,0x00,0x00,0x86,0x01,0x00,0x00, +0x63,0x01,0x00,0x00,0x81,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, +0x86,0x01,0x00,0x00,0x85,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x88,0x01,0x00,0x00,0x5e,0x01,0x00,0x00, +0x20,0x01,0x00,0x00,0x41,0x00,0x08,0x00,0x6d,0x01,0x00,0x00, +0x8a,0x01,0x00,0x00,0x6b,0x01,0x00,0x00,0x34,0x00,0x00,0x00, +0x55,0x01,0x00,0x00,0xd0,0x00,0x00,0x00,0x3e,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00,0x8b,0x01,0x00,0x00, +0x8a,0x01,0x00,0x00,0x73,0x00,0x04,0x00,0xf9,0x00,0x00,0x00, +0x8c,0x01,0x00,0x00,0x8b,0x01,0x00,0x00,0x41,0x00,0x05,0x00, +0x0a,0x01,0x00,0x00,0x8d,0x01,0x00,0x00,0x63,0x01,0x00,0x00, +0x88,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x8d,0x01,0x00,0x00, +0x8c,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x8f,0x01,0x00,0x00,0x5e,0x01,0x00,0x00,0x27,0x01,0x00,0x00, +0x41,0x00,0x08,0x00,0x6d,0x01,0x00,0x00,0x91,0x01,0x00,0x00, +0x6b,0x01,0x00,0x00,0x34,0x00,0x00,0x00,0x55,0x01,0x00,0x00, +0xd0,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0xc4,0x00,0x00,0x00,0x92,0x01,0x00,0x00,0x91,0x01,0x00,0x00, +0x73,0x00,0x04,0x00,0xf9,0x00,0x00,0x00,0x93,0x01,0x00,0x00, +0x92,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0x0a,0x01,0x00,0x00, +0x94,0x01,0x00,0x00,0x63,0x01,0x00,0x00,0x8f,0x01,0x00,0x00, +0x3e,0x00,0x03,0x00,0x94,0x01,0x00,0x00,0x93,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x96,0x01,0x00,0x00, +0x5e,0x01,0x00,0x00,0x2e,0x01,0x00,0x00,0x41,0x00,0x08,0x00, +0x6d,0x01,0x00,0x00,0x98,0x01,0x00,0x00,0x6b,0x01,0x00,0x00, +0x34,0x00,0x00,0x00,0x55,0x01,0x00,0x00,0xd0,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00, +0x99,0x01,0x00,0x00,0x98,0x01,0x00,0x00,0x73,0x00,0x04,0x00, +0xf9,0x00,0x00,0x00,0x9a,0x01,0x00,0x00,0x99,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0x0a,0x01,0x00,0x00,0x9b,0x01,0x00,0x00, +0x63,0x01,0x00,0x00,0x96,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, +0x9b,0x01,0x00,0x00,0x9a,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x9d,0x01,0x00,0x00,0x5e,0x01,0x00,0x00, +0x35,0x01,0x00,0x00,0x41,0x00,0x08,0x00,0x6d,0x01,0x00,0x00, +0x9f,0x01,0x00,0x00,0x6b,0x01,0x00,0x00,0x34,0x00,0x00,0x00, +0x55,0x01,0x00,0x00,0xd0,0x00,0x00,0x00,0x19,0x01,0x00,0x00, +0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00,0xa0,0x01,0x00,0x00, +0x9f,0x01,0x00,0x00,0x73,0x00,0x04,0x00,0xf9,0x00,0x00,0x00, +0xa1,0x01,0x00,0x00,0xa0,0x01,0x00,0x00,0x41,0x00,0x05,0x00, +0x0a,0x01,0x00,0x00,0xa2,0x01,0x00,0x00,0x63,0x01,0x00,0x00, +0x9d,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0xa2,0x01,0x00,0x00, +0xa1,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xa7,0x01,0x00,0x00,0xf3,0x02,0x00,0x00,0xa5,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0x43,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x45,0x01,0x00,0x00,0xe0,0x00,0x04,0x00,0x0c,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0xa8,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xab,0x01,0x00,0x00,0xf6,0x02,0x00,0x00, +0xa9,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xae,0x01,0x00,0x00,0xfa,0x02,0x00,0x00,0xac,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0xb0,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xb0,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xfc,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0x45,0x01,0x00,0x00, +0x5a,0x02,0x00,0x00,0xb3,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc2,0x00,0x00,0x00,0xb6,0x01,0x00,0x00,0xfc,0x02,0x00,0x00, +0x6c,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xb2,0x01,0x00,0x00, +0xb3,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0xb6,0x01,0x00,0x00,0xb1,0x01,0x00,0x00,0xb2,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xb1,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0xb8,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xb8,0x01,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x00,0x03,0x00,0x00, +0x3e,0x00,0x00,0x00,0xb1,0x01,0x00,0x00,0xe4,0x01,0x00,0x00, +0xbb,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, +0xbe,0x01,0x00,0x00,0x00,0x03,0x00,0x00,0x60,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0xba,0x01,0x00,0x00,0xbb,0x01,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xbe,0x01,0x00,0x00, +0xb9,0x01,0x00,0x00,0xba,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xb9,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xc0,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xc0,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0x12,0x03,0x00,0x00,0x3e,0x00,0x00,0x00, +0xb9,0x01,0x00,0x00,0xe2,0x01,0x00,0x00,0xc1,0x01,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0xc6,0x01,0x00,0x00, +0x12,0x03,0x00,0x00,0x62,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0xc2,0x01,0x00,0x00,0xc1,0x01,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0xc6,0x01,0x00,0x00,0xc1,0x01,0x00,0x00, +0xc2,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xc1,0x01,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xcc,0x01,0x00,0x00, +0x00,0x03,0x00,0x00,0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xce,0x01,0x00,0x00,0xcc,0x01,0x00,0x00, +0x12,0x03,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xd0,0x01,0x00,0x00,0x55,0x00,0x00,0x00,0x53,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xd2,0x01,0x00,0x00, +0x00,0x03,0x00,0x00,0x61,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xd3,0x01,0x00,0x00,0xd0,0x01,0x00,0x00, +0xd2,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xd5,0x01,0x00,0x00,0x64,0x00,0x00,0x00,0x62,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xd6,0x01,0x00,0x00, +0xd3,0x01,0x00,0x00,0xd5,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xd8,0x01,0x00,0x00,0xd6,0x01,0x00,0x00, +0x12,0x03,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xda,0x01,0x00,0x00,0xd8,0x01,0x00,0x00,0xd9,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xdc,0x01,0x00,0x00, +0xda,0x01,0x00,0x00,0xfc,0x02,0x00,0x00,0x41,0x00,0x05,0x00, +0x0a,0x01,0x00,0x00,0xdd,0x01,0x00,0x00,0xfe,0x00,0x00,0x00, +0xdc,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0xf9,0x00,0x00,0x00, +0xde,0x01,0x00,0x00,0xdd,0x01,0x00,0x00,0x41,0x00,0x05,0x00, +0xdf,0x01,0x00,0x00,0xe0,0x01,0x00,0x00,0xca,0x01,0x00,0x00, +0xce,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0xe0,0x01,0x00,0x00, +0xde,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xe2,0x01,0x00,0x00,0x12,0x03,0x00,0x00,0xd0,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xc0,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xc2,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xbb,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xbb,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xe4,0x01,0x00,0x00,0x00,0x03,0x00,0x00, +0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xb8,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xba,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0xe6,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xe6,0x01,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x01,0x03,0x00,0x00, +0x3e,0x00,0x00,0x00,0xba,0x01,0x00,0x00,0x12,0x02,0x00,0x00, +0xe9,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, +0xec,0x01,0x00,0x00,0x01,0x03,0x00,0x00,0xbf,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0xe8,0x01,0x00,0x00,0xe9,0x01,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xec,0x01,0x00,0x00, +0xe7,0x01,0x00,0x00,0xe8,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xe7,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xee,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xee,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0x0f,0x03,0x00,0x00,0x3e,0x00,0x00,0x00, +0xe7,0x01,0x00,0x00,0x10,0x02,0x00,0x00,0xef,0x01,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0xf4,0x01,0x00,0x00, +0x0f,0x03,0x00,0x00,0xbc,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0xf0,0x01,0x00,0x00,0xef,0x01,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0xf4,0x01,0x00,0x00,0xef,0x01,0x00,0x00, +0xf0,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xef,0x01,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xfa,0x01,0x00,0x00, +0x01,0x03,0x00,0x00,0xbc,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xfc,0x01,0x00,0x00,0xfa,0x01,0x00,0x00, +0x0f,0x03,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xfe,0x01,0x00,0x00,0x59,0x00,0x00,0x00,0xb9,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x01,0x02,0x00,0x00, +0x01,0x03,0x00,0x00,0x00,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x02,0x02,0x00,0x00,0xfe,0x01,0x00,0x00, +0x01,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x04,0x02,0x00,0x00,0x68,0x00,0x00,0x00,0xbc,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x05,0x02,0x00,0x00, +0x02,0x02,0x00,0x00,0x04,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x07,0x02,0x00,0x00,0x05,0x02,0x00,0x00, +0x0f,0x03,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x09,0x02,0x00,0x00,0x07,0x02,0x00,0x00,0x08,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x0b,0x02,0x00,0x00, +0x09,0x02,0x00,0x00,0xfc,0x02,0x00,0x00,0x41,0x00,0x05,0x00, +0x0a,0x01,0x00,0x00,0x0c,0x02,0x00,0x00,0x63,0x01,0x00,0x00, +0x0b,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0xf9,0x00,0x00,0x00, +0x0d,0x02,0x00,0x00,0x0c,0x02,0x00,0x00,0x41,0x00,0x05,0x00, +0xdf,0x01,0x00,0x00,0x0e,0x02,0x00,0x00,0xf8,0x01,0x00,0x00, +0xfc,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x0e,0x02,0x00,0x00, +0x0d,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x10,0x02,0x00,0x00,0x0f,0x03,0x00,0x00,0xd0,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xee,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xf0,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xe9,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xe9,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x12,0x02,0x00,0x00,0x01,0x03,0x00,0x00, +0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xe6,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xe8,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0x14,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x14,0x02,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x02,0x03,0x00,0x00, +0x3e,0x00,0x00,0x00,0xe8,0x01,0x00,0x00,0x58,0x02,0x00,0x00, +0x17,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, +0x1a,0x02,0x00,0x00,0x02,0x03,0x00,0x00,0xbf,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0x16,0x02,0x00,0x00,0x17,0x02,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x1a,0x02,0x00,0x00, +0x15,0x02,0x00,0x00,0x16,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x15,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x1c,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x1c,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0x06,0x03,0x00,0x00,0x3e,0x00,0x00,0x00, +0x15,0x02,0x00,0x00,0x56,0x02,0x00,0x00,0x1f,0x02,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0x22,0x02,0x00,0x00, +0x06,0x03,0x00,0x00,0x60,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0x1e,0x02,0x00,0x00,0x1f,0x02,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x22,0x02,0x00,0x00,0x1d,0x02,0x00,0x00, +0x1e,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x1d,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0x24,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x24,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0x08,0x03,0x00,0x00,0x3e,0x00,0x00,0x00,0x1d,0x02,0x00,0x00, +0x54,0x02,0x00,0x00,0x27,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc2,0x00,0x00,0x00,0x2a,0x02,0x00,0x00,0x08,0x03,0x00,0x00, +0xbc,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x26,0x02,0x00,0x00, +0x27,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x2a,0x02,0x00,0x00,0x25,0x02,0x00,0x00,0x26,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x25,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0x2c,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x2c,0x02,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x0a,0x03,0x00,0x00, +0x3e,0x00,0x00,0x00,0x25,0x02,0x00,0x00,0x52,0x02,0x00,0x00, +0x2d,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, +0x32,0x02,0x00,0x00,0x0a,0x03,0x00,0x00,0x62,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0x2e,0x02,0x00,0x00,0x2d,0x02,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x32,0x02,0x00,0x00, +0x2d,0x02,0x00,0x00,0x2e,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x2d,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x34,0x02,0x00,0x00,0x02,0x03,0x00,0x00,0xbc,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x36,0x02,0x00,0x00, +0x34,0x02,0x00,0x00,0x08,0x03,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x38,0x02,0x00,0x00,0x36,0x02,0x00,0x00, +0x37,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x3a,0x02,0x00,0x00,0x06,0x03,0x00,0x00,0x62,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3b,0x02,0x00,0x00, +0x38,0x02,0x00,0x00,0x3a,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x3d,0x02,0x00,0x00,0x3b,0x02,0x00,0x00, +0x0a,0x03,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x41,0x02,0x00,0x00,0x3a,0x02,0x00,0x00,0x0a,0x03,0x00,0x00, +0x41,0x00,0x05,0x00,0xdf,0x01,0x00,0x00,0x42,0x02,0x00,0x00, +0xca,0x01,0x00,0x00,0x41,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, +0xf9,0x00,0x00,0x00,0x43,0x02,0x00,0x00,0x42,0x02,0x00,0x00, +0x73,0x00,0x04,0x00,0xc4,0x00,0x00,0x00,0x44,0x02,0x00,0x00, +0x43,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0xdf,0x01,0x00,0x00, +0x49,0x02,0x00,0x00,0xf8,0x01,0x00,0x00,0x36,0x02,0x00,0x00, +0x3d,0x00,0x04,0x00,0xf9,0x00,0x00,0x00,0x4a,0x02,0x00,0x00, +0x49,0x02,0x00,0x00,0x73,0x00,0x04,0x00,0xc4,0x00,0x00,0x00, +0x4b,0x02,0x00,0x00,0x4a,0x02,0x00,0x00,0x41,0x00,0x05,0x00, +0xcd,0x00,0x00,0x00,0x4d,0x02,0x00,0x00,0xca,0x00,0x00,0x00, +0x3d,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00, +0x4e,0x02,0x00,0x00,0x4d,0x02,0x00,0x00,0x0c,0x00,0x08,0x00, +0xc4,0x00,0x00,0x00,0x4f,0x02,0x00,0x00,0x01,0x00,0x00,0x00, +0x32,0x00,0x00,0x00,0x44,0x02,0x00,0x00,0x4b,0x02,0x00,0x00, +0x4e,0x02,0x00,0x00,0x3e,0x00,0x03,0x00,0x4d,0x02,0x00,0x00, +0x4f,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x52,0x02,0x00,0x00,0x0a,0x03,0x00,0x00,0xd0,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x2c,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x2e,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x27,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x27,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x54,0x02,0x00,0x00,0x08,0x03,0x00,0x00, +0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x24,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x26,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0x1f,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x1f,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x56,0x02,0x00,0x00, +0x06,0x03,0x00,0x00,0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x1c,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x1e,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0x17,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x17,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x58,0x02,0x00,0x00,0x02,0x03,0x00,0x00,0xd0,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x14,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x16,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0xb3,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xb3,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x5a,0x02,0x00,0x00,0xfc,0x02,0x00,0x00, +0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xb0,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xb2,0x01,0x00,0x00,0xe0,0x00,0x04,0x00, +0x0c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0xa8,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0xd7,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xd7,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x5c,0x02,0x00,0x00,0xe2,0x02,0x00,0x00,0x6c,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xd4,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xd6,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x61,0x02,0x00,0x00,0x55,0x00,0x00,0x00,0x53,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x62,0x02,0x00,0x00, +0x96,0x00,0x00,0x00,0x61,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x67,0x02,0x00,0x00,0x59,0x00,0x00,0x00, +0xb9,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x68,0x02,0x00,0x00,0xa8,0x00,0x00,0x00,0x67,0x02,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x6d,0x02,0x00,0x00, +0x12,0x00,0x00,0x00,0x6c,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x6e,0x02,0x00,0x00,0x6d,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x6f,0x02,0x00,0x00, +0x0f,0x00,0x00,0x00,0x6e,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x73,0x02,0x00,0x00,0x47,0x00,0x00,0x00, +0x6e,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00, +0x75,0x02,0x00,0x00,0x74,0x02,0x00,0x00,0x0c,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x76,0x02,0x00,0x00, +0x75,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x77,0x02,0x00,0x00,0x73,0x02,0x00,0x00,0x76,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x78,0x02,0x00,0x00, +0x6f,0x02,0x00,0x00,0x77,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0x7a,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x7a,0x02,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xe3,0x02,0x00,0x00, +0x3e,0x00,0x00,0x00,0xd6,0x00,0x00,0x00,0xe0,0x02,0x00,0x00, +0x7d,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, +0x80,0x02,0x00,0x00,0xe3,0x02,0x00,0x00,0xbf,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0x7c,0x02,0x00,0x00,0x7d,0x02,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x80,0x02,0x00,0x00, +0x7b,0x02,0x00,0x00,0x7c,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x7b,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x82,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x82,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xe4,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, +0x7b,0x02,0x00,0x00,0xde,0x02,0x00,0x00,0x85,0x02,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0x88,0x02,0x00,0x00, +0xe4,0x02,0x00,0x00,0x60,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0x84,0x02,0x00,0x00,0x85,0x02,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x88,0x02,0x00,0x00,0x83,0x02,0x00,0x00, +0x84,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x83,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8c,0x02,0x00,0x00, +0xe4,0x02,0x00,0x00,0x61,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x8d,0x02,0x00,0x00,0x62,0x02,0x00,0x00, +0x8c,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x8f,0x02,0x00,0x00,0x64,0x00,0x00,0x00,0x62,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x90,0x02,0x00,0x00, +0x8d,0x02,0x00,0x00,0x8f,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x94,0x02,0x00,0x00,0xe3,0x02,0x00,0x00, +0x00,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x95,0x02,0x00,0x00,0x68,0x02,0x00,0x00,0x94,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x97,0x02,0x00,0x00, +0x68,0x00,0x00,0x00,0xbc,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x98,0x02,0x00,0x00,0x95,0x02,0x00,0x00, +0x97,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x9a,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x9a,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xe6,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, +0x83,0x02,0x00,0x00,0xdc,0x02,0x00,0x00,0x9d,0x02,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0xa0,0x02,0x00,0x00, +0xe6,0x02,0x00,0x00,0xbc,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0x9c,0x02,0x00,0x00,0x9d,0x02,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0xa0,0x02,0x00,0x00,0x9b,0x02,0x00,0x00, +0x9c,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x9b,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0xa2,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0xa2,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xe8,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0x9b,0x02,0x00,0x00, +0xda,0x02,0x00,0x00,0xa5,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc2,0x00,0x00,0x00,0xa8,0x02,0x00,0x00,0xe8,0x02,0x00,0x00, +0x62,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xa4,0x02,0x00,0x00, +0xa5,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0xa8,0x02,0x00,0x00,0xa3,0x02,0x00,0x00,0xa4,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0xa3,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xab,0x02,0x00,0x00,0x90,0x02,0x00,0x00, +0xe8,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, +0xae,0x02,0x00,0x00,0xab,0x02,0x00,0x00,0x36,0x00,0x00,0x00, +0xf7,0x00,0x03,0x00,0xb0,0x02,0x00,0x00,0x00,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0xae,0x02,0x00,0x00,0xaf,0x02,0x00,0x00, +0xb0,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0xaf,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xb3,0x02,0x00,0x00, +0x98,0x02,0x00,0x00,0xe6,0x02,0x00,0x00,0x41,0x00,0x05,0x00, +0x15,0x00,0x00,0x00,0xb4,0x02,0x00,0x00,0x12,0x00,0x00,0x00, +0xd0,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xb5,0x02,0x00,0x00,0xb4,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc2,0x00,0x00,0x00,0xb6,0x02,0x00,0x00,0xb3,0x02,0x00,0x00, +0xb5,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0xb0,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0xb0,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, +0xc2,0x00,0x00,0x00,0xb7,0x02,0x00,0x00,0xae,0x02,0x00,0x00, +0xa3,0x02,0x00,0x00,0xb6,0x02,0x00,0x00,0xaf,0x02,0x00,0x00, +0xf7,0x00,0x03,0x00,0xb9,0x02,0x00,0x00,0x00,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0xb7,0x02,0x00,0x00,0xb8,0x02,0x00,0x00, +0xb9,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0xb8,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xc1,0x02,0x00,0x00, +0x98,0x02,0x00,0x00,0xe6,0x02,0x00,0x00,0x41,0x00,0x05,0x00, +0x15,0x00,0x00,0x00,0xc3,0x02,0x00,0x00,0x12,0x00,0x00,0x00, +0xc2,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xc4,0x02,0x00,0x00,0xc3,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xc5,0x02,0x00,0x00,0xc1,0x02,0x00,0x00, +0xc4,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xc6,0x02,0x00,0x00,0x78,0x02,0x00,0x00,0xc5,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xc8,0x02,0x00,0x00, +0xc6,0x02,0x00,0x00,0x90,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xca,0x02,0x00,0x00,0xc8,0x02,0x00,0x00, +0xe8,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xcc,0x02,0x00,0x00,0xe3,0x02,0x00,0x00,0xbc,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xce,0x02,0x00,0x00, +0xcc,0x02,0x00,0x00,0xe6,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xd0,0x02,0x00,0x00,0xce,0x02,0x00,0x00, +0xcf,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xd2,0x02,0x00,0x00,0xe4,0x02,0x00,0x00,0x62,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xd3,0x02,0x00,0x00, +0xd0,0x02,0x00,0x00,0xd2,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xd5,0x02,0x00,0x00,0xd3,0x02,0x00,0x00, +0xe8,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0xcd,0x00,0x00,0x00, +0xd6,0x02,0x00,0x00,0xca,0x00,0x00,0x00,0xd5,0x02,0x00,0x00, +0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00,0xd7,0x02,0x00,0x00, +0xd6,0x02,0x00,0x00,0x41,0x00,0x06,0x00,0x6d,0x01,0x00,0x00, +0xd8,0x02,0x00,0x00,0xbd,0x02,0x00,0x00,0x34,0x00,0x00,0x00, +0xca,0x02,0x00,0x00,0x3e,0x00,0x03,0x00,0xd8,0x02,0x00,0x00, +0xd7,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0xb9,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0xb9,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0xa5,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0xa5,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xda,0x02,0x00,0x00, +0xe8,0x02,0x00,0x00,0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xa2,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0xa4,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0x9d,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x9d,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xdc,0x02,0x00,0x00,0xe6,0x02,0x00,0x00,0xd0,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x9a,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x9c,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x85,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x85,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xde,0x02,0x00,0x00,0xe4,0x02,0x00,0x00, +0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x82,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x84,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0x7d,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x7d,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xe0,0x02,0x00,0x00, +0xe3,0x02,0x00,0x00,0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x7a,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x7c,0x02,0x00,0x00, +0xfd,0x00,0x01,0x00,0x38,0x00,0x01,0x00, +}; +const uint64_t matmul_f16_f32_aligned_len = 11360; + +unsigned char matmul_f16_f32_aligned_fp32_data[] = { +0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, +0xd6,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, +0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x51,0x11,0x00,0x00, +0x0b,0x00,0x06,0x00,0x01,0x00,0x00,0x00,0x47,0x4c,0x53,0x4c, +0x2e,0x73,0x74,0x64,0x2e,0x34,0x35,0x30,0x00,0x00,0x00,0x00, +0x0e,0x00,0x03,0x00,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x0f,0x00,0x0f,0x00,0x05,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x6d,0x61,0x69,0x6e,0x00,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, +0x12,0x00,0x00,0x00,0x3d,0x00,0x00,0x00,0x4c,0x00,0x00,0x00, +0xfd,0x00,0x00,0x00,0x04,0x01,0x00,0x00,0x4a,0x01,0x00,0x00, +0x51,0x01,0x00,0x00,0x37,0x02,0x00,0x00,0x80,0x02,0x00,0x00, +0x10,0x00,0x06,0x00,0x04,0x00,0x00,0x00,0x11,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x0b,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, +0x1c,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x10,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x08,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, +0x03,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x10,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x14,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x18,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x07,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x10,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, +0x09,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x24,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x0a,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x10,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x2c,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x30,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x0d,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0x47,0x00,0x03,0x00, +0x10,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x37,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x3d,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, +0x1a,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x4c,0x00,0x00,0x00, +0x0b,0x00,0x00,0x00,0x1b,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x4f,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x09,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x53,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x60,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x62,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x07,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x6c,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x03,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xa7,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0xb9,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x05,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0xbc,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x08,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x01,0x01,0x00,0x00, +0x06,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x48,0x00,0x04,0x00, +0x02,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x02,0x01,0x00,0x00,0x00,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, +0x02,0x01,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x04,0x01,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x04,0x01,0x00,0x00,0x21,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x22,0x01,0x00,0x00, +0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x23,0x01,0x00,0x00,0x0b,0x00,0x00,0x00,0x19,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x4e,0x01,0x00,0x00,0x06,0x00,0x00,0x00, +0x10,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0x4f,0x01,0x00,0x00, +0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x4f,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x4f,0x01,0x00,0x00, +0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x51,0x01,0x00,0x00, +0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x51,0x01,0x00,0x00,0x21,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x37,0x02,0x00,0x00,0x0b,0x00,0x00,0x00, +0x18,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x7d,0x02,0x00,0x00, +0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00, +0x7e,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x19,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x7e,0x02,0x00,0x00,0x00,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, +0x7e,0x02,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x80,0x02,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x80,0x02,0x00,0x00,0x21,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x13,0x00,0x02,0x00,0x02,0x00,0x00,0x00, +0x21,0x00,0x03,0x00,0x03,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x15,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x17,0x00,0x04,0x00,0x09,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x0a,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x09,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x0d,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x1e,0x00,0x10,0x00,0x10,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x11,0x00,0x00,0x00, +0x09,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0x11,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x09,0x00,0x00,0x00, +0x15,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, +0x14,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x15,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x21,0x00,0x00,0x00, +0x0a,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, +0x27,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0x2d,0x00,0x00,0x00,0x07,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x34,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x37,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00,0x3d,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x3e,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0x0a,0x00,0x00,0x00,0x4c,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x4f,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x53,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x54,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0x37,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x58,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0x37,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x32,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x61,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x60,0x00,0x00,0x00, +0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x62,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x63,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x61,0x00,0x00,0x00, +0x62,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x67,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x61,0x00,0x00,0x00, +0x62,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x6e,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x73,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x78,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x7d,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x81,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0x91,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x97,0x00,0x00,0x00, +0x03,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, +0xa2,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x32,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xa7,0x00,0x00,0x00,0x40,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0xa9,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xb8,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00, +0x62,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xb9,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xba,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0x53,0x00,0x00,0x00,0xb9,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xbb,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0x4f,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x32,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xbc,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xbd,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0xbb,0x00,0x00,0x00,0xbc,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xbe,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0xbd,0x00,0x00,0x00,0x60,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xbf,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0xba,0x00,0x00,0x00,0xbe,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xc0,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0xb8,0x00,0x00,0x00,0xbf,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xc1,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0xc0,0x00,0x00,0x00,0xbc,0x00,0x00,0x00, +0x14,0x00,0x02,0x00,0xc2,0x00,0x00,0x00,0x16,0x00,0x03,0x00, +0xc4,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xc5,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xc6,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0xc5,0x00,0x00,0x00,0xbf,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xc7,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0xc6,0x00,0x00,0x00,0xbc,0x00,0x00,0x00,0x1c,0x00,0x04,0x00, +0xc8,0x00,0x00,0x00,0xc4,0x00,0x00,0x00,0xc7,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0xc9,0x00,0x00,0x00,0x07,0x00,0x00,0x00, +0xc8,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0xc4,0x00,0x00,0x00, +0xcc,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0xcd,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0xc4,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0xd0,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xf4,0x00,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, +0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xf9,0x00,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, +0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xfa,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x37,0x00,0x00,0x00, +0xf9,0x00,0x00,0x00,0x1c,0x00,0x04,0x00,0xfb,0x00,0x00,0x00, +0xc4,0x00,0x00,0x00,0xfa,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0xfc,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0xfb,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0xfc,0x00,0x00,0x00,0xfd,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x16,0x00,0x03,0x00,0xff,0x00,0x00,0x00, +0x10,0x00,0x00,0x00,0x17,0x00,0x04,0x00,0x00,0x01,0x00,0x00, +0xff,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, +0x01,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x1e,0x00,0x03,0x00, +0x02,0x01,0x00,0x00,0x01,0x01,0x00,0x00,0x20,0x00,0x04,0x00, +0x03,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x02,0x01,0x00,0x00, +0x3b,0x00,0x04,0x00,0x03,0x01,0x00,0x00,0x04,0x01,0x00,0x00, +0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x06,0x01,0x00,0x00, +0x0c,0x00,0x00,0x00,0xff,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x0a,0x01,0x00,0x00,0x04,0x00,0x00,0x00,0xc4,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x1b,0x01,0x00,0x00, +0x03,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x22,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x33,0x00,0x06,0x00, +0x09,0x00,0x00,0x00,0x23,0x01,0x00,0x00,0x22,0x01,0x00,0x00, +0x39,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x24,0x01,0x00,0x00,0x51,0x00,0x00,0x00, +0x23,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x25,0x01,0x00,0x00,0x84,0x00,0x00,0x00, +0x24,0x01,0x00,0x00,0x6d,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x26,0x01,0x00,0x00,0x86,0x00,0x00,0x00, +0x25,0x01,0x00,0x00,0x6c,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x41,0x01,0x00,0x00,0x80,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x46,0x01,0x00,0x00,0x80,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x47,0x01,0x00,0x00,0x84,0x00,0x00,0x00, +0xa7,0x00,0x00,0x00,0x46,0x01,0x00,0x00,0x1c,0x00,0x04,0x00, +0x48,0x01,0x00,0x00,0xc4,0x00,0x00,0x00,0x47,0x01,0x00,0x00, +0x20,0x00,0x04,0x00,0x49,0x01,0x00,0x00,0x04,0x00,0x00,0x00, +0x48,0x01,0x00,0x00,0x3b,0x00,0x04,0x00,0x49,0x01,0x00,0x00, +0x4a,0x01,0x00,0x00,0x04,0x00,0x00,0x00,0x17,0x00,0x04,0x00, +0x4d,0x01,0x00,0x00,0xc4,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x1d,0x00,0x03,0x00,0x4e,0x01,0x00,0x00,0x4d,0x01,0x00,0x00, +0x1e,0x00,0x03,0x00,0x4f,0x01,0x00,0x00,0x4e,0x01,0x00,0x00, +0x20,0x00,0x04,0x00,0x50,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, +0x4f,0x01,0x00,0x00,0x3b,0x00,0x04,0x00,0x50,0x01,0x00,0x00, +0x51,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x53,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0xc4,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x69,0x01,0x00,0x00, +0x51,0x00,0x00,0x00,0x23,0x01,0x00,0x00,0x00,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x6a,0x01,0x00,0x00, +0x84,0x00,0x00,0x00,0x69,0x01,0x00,0x00,0x6d,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x6b,0x01,0x00,0x00, +0x86,0x00,0x00,0x00,0x6a,0x01,0x00,0x00,0x6c,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x6e,0x01,0x00,0x00, +0x08,0x01,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x6f,0x01,0x00,0x00,0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, +0x6d,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x72,0x01,0x00,0x00,0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, +0x6d,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x8d,0x01,0x00,0x00,0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00, +0x62,0x00,0x00,0x00,0x1c,0x00,0x04,0x00,0x8e,0x01,0x00,0x00, +0xc4,0x00,0x00,0x00,0x8d,0x01,0x00,0x00,0x20,0x00,0x04,0x00, +0x8f,0x01,0x00,0x00,0x07,0x00,0x00,0x00,0x8e,0x01,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x9f,0x01,0x00,0x00, +0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xba,0x01,0x00,0x00, +0x84,0x00,0x00,0x00,0xbf,0x00,0x00,0x00,0xbc,0x00,0x00,0x00, +0x1c,0x00,0x04,0x00,0xbb,0x01,0x00,0x00,0xc4,0x00,0x00,0x00, +0xba,0x01,0x00,0x00,0x20,0x00,0x04,0x00,0xbc,0x01,0x00,0x00, +0x07,0x00,0x00,0x00,0xbb,0x01,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xc5,0x01,0x00,0x00,0x86,0x00,0x00,0x00, +0xb9,0x00,0x00,0x00,0xbf,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xcd,0x01,0x00,0x00,0x80,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xfc,0x01,0x00,0x00,0x84,0x00,0x00,0x00, +0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0x2f,0x02,0x00,0x00,0x0d,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00,0x37,0x02,0x00,0x00, +0x01,0x00,0x00,0x00,0x1d,0x00,0x03,0x00,0x7d,0x02,0x00,0x00, +0xc4,0x00,0x00,0x00,0x1e,0x00,0x03,0x00,0x7e,0x02,0x00,0x00, +0x7d,0x02,0x00,0x00,0x20,0x00,0x04,0x00,0x7f,0x02,0x00,0x00, +0x0c,0x00,0x00,0x00,0x7e,0x02,0x00,0x00,0x3b,0x00,0x04,0x00, +0x7f,0x02,0x00,0x00,0x80,0x02,0x00,0x00,0x0c,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x85,0x02,0x00,0x00, +0x05,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x92,0x02,0x00,0x00,0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00, +0x62,0x00,0x00,0x00,0x36,0x00,0x05,0x00,0x02,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x03,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0x05,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0xc9,0x00,0x00,0x00,0xca,0x00,0x00,0x00,0x07,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0x8f,0x01,0x00,0x00,0x90,0x01,0x00,0x00, +0x07,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0xbc,0x01,0x00,0x00, +0xbd,0x01,0x00,0x00,0x07,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x0d,0x00,0x00,0x00,0x0e,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x0f,0x00,0x00,0x00,0x0e,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x15,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0x14,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x17,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, +0x17,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x1e,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x17,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x22,0x00,0x00,0x00, +0x12,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x22,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x24,0x00,0x00,0x00, +0x18,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x15,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0x27,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x29,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x2a,0x00,0x00,0x00,0x1e,0x00,0x00,0x00, +0x29,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, +0x2e,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x2d,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x2f,0x00,0x00,0x00, +0x2e,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x30,0x00,0x00,0x00,0x24,0x00,0x00,0x00,0x2f,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x32,0x00,0x00,0x00, +0x30,0x00,0x00,0x00,0x2a,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x15,0x00,0x00,0x00,0x35,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0x34,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x36,0x00,0x00,0x00,0x35,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x38,0x00,0x00,0x00,0x36,0x00,0x00,0x00, +0x37,0x00,0x00,0x00,0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x3a,0x00,0x00,0x00,0x38,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3b,0x00,0x00,0x00, +0x3a,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x0d,0x00,0x00,0x00,0x3f,0x00,0x00,0x00,0x3d,0x00,0x00,0x00, +0x3e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x40,0x00,0x00,0x00,0x3f,0x00,0x00,0x00,0x89,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x42,0x00,0x00,0x00,0x40,0x00,0x00,0x00, +0x3b,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x47,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x3b,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00,0x49,0x00,0x00,0x00, +0x3d,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x4a,0x00,0x00,0x00,0x49,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00,0x4d,0x00,0x00,0x00, +0x4c,0x00,0x00,0x00,0x3e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x4d,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x50,0x00,0x00,0x00, +0x4e,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x89,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x55,0x00,0x00,0x00,0x50,0x00,0x00,0x00, +0x54,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x59,0x00,0x00,0x00,0x50,0x00,0x00,0x00,0x58,0x00,0x00,0x00, +0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x5d,0x00,0x00,0x00, +0x4e,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x89,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x64,0x00,0x00,0x00,0x5d,0x00,0x00,0x00, +0x63,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x68,0x00,0x00,0x00,0x5d,0x00,0x00,0x00,0x67,0x00,0x00,0x00, +0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x6f,0x00,0x00,0x00, +0x4e,0x00,0x00,0x00,0x6e,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x74,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, +0x73,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x79,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x78,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x7e,0x00,0x00,0x00, +0x4e,0x00,0x00,0x00,0x7d,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x15,0x00,0x00,0x00,0x82,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0x81,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x83,0x00,0x00,0x00,0x82,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x47,0x00,0x00,0x00, +0x83,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, +0x87,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x88,0x00,0x00,0x00, +0x87,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x8a,0x00,0x00,0x00,0x47,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8d,0x00,0x00,0x00, +0x8a,0x00,0x00,0x00,0x83,0x00,0x00,0x00,0x0c,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0x8e,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x26,0x00,0x00,0x00,0x88,0x00,0x00,0x00,0x8d,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x92,0x00,0x00,0x00, +0x12,0x00,0x00,0x00,0x91,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x93,0x00,0x00,0x00,0x92,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x94,0x00,0x00,0x00, +0x32,0x00,0x00,0x00,0x93,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x96,0x00,0x00,0x00,0x42,0x00,0x00,0x00, +0x37,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, +0x98,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x97,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x99,0x00,0x00,0x00, +0x98,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x9a,0x00,0x00,0x00,0x96,0x00,0x00,0x00,0x99,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9b,0x00,0x00,0x00, +0x94,0x00,0x00,0x00,0x9a,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x9d,0x00,0x00,0x00,0x9b,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x9e,0x00,0x00,0x00,0x9d,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0xa3,0x00,0x00,0x00, +0x12,0x00,0x00,0x00,0xa2,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xa4,0x00,0x00,0x00,0xa3,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa5,0x00,0x00,0x00, +0x0f,0x00,0x00,0x00,0xa4,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xa8,0x00,0x00,0x00,0x4a,0x00,0x00,0x00, +0xa7,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, +0xaa,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0xa9,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xab,0x00,0x00,0x00, +0xaa,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xac,0x00,0x00,0x00,0xa8,0x00,0x00,0x00,0xab,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xad,0x00,0x00,0x00, +0xa5,0x00,0x00,0x00,0xac,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xaf,0x00,0x00,0x00,0xad,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xb0,0x00,0x00,0x00,0xaf,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xb2,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xb2,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xa4,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0x05,0x00,0x00,0x00, +0xd1,0x00,0x00,0x00,0xb3,0x00,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc2,0x00,0x00,0x00,0xc3,0x00,0x00,0x00,0xa4,0x02,0x00,0x00, +0xc1,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xb4,0x00,0x00,0x00, +0xb3,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0xc3,0x00,0x00,0x00,0xb3,0x00,0x00,0x00,0xb4,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xb3,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0xcd,0x00,0x00,0x00,0xce,0x00,0x00,0x00,0xca,0x00,0x00,0x00, +0xa4,0x02,0x00,0x00,0x3e,0x00,0x03,0x00,0xce,0x00,0x00,0x00, +0xcc,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xd1,0x00,0x00,0x00,0xa4,0x02,0x00,0x00,0xd0,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xb2,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xb4,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xd4,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xd4,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xbd,0x02,0x00,0x00,0xb0,0x00,0x00,0x00, +0xb4,0x00,0x00,0x00,0x74,0x01,0x00,0x00,0xd7,0x00,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xb9,0x02,0x00,0x00, +0x9e,0x00,0x00,0x00,0xb4,0x00,0x00,0x00,0x71,0x01,0x00,0x00, +0xd7,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xa5,0x02,0x00,0x00,0x84,0x00,0x00,0x00,0xb4,0x00,0x00,0x00, +0x1f,0x02,0x00,0x00,0xd7,0x00,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc2,0x00,0x00,0x00,0xdb,0x00,0x00,0x00,0xa5,0x02,0x00,0x00, +0x8e,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xd6,0x00,0x00,0x00, +0xd7,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0xdb,0x00,0x00,0x00,0xd5,0x00,0x00,0x00,0xd6,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xd5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xdd,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xdd,0x00,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xb5,0x02,0x00,0x00, +0x3e,0x00,0x00,0x00,0xd5,0x00,0x00,0x00,0x28,0x01,0x00,0x00, +0xde,0x00,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, +0xe3,0x00,0x00,0x00,0xb5,0x02,0x00,0x00,0x37,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0xdf,0x00,0x00,0x00,0xde,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xe3,0x00,0x00,0x00, +0xde,0x00,0x00,0x00,0xdf,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xde,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xe8,0x00,0x00,0x00,0x74,0x00,0x00,0x00,0xb5,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xeb,0x00,0x00,0x00, +0xe8,0x00,0x00,0x00,0x99,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xec,0x00,0x00,0x00,0xeb,0x00,0x00,0x00, +0x6d,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xed,0x00,0x00,0x00,0xb9,0x02,0x00,0x00,0xec,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xef,0x00,0x00,0x00, +0xed,0x00,0x00,0x00,0x6f,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xf5,0x00,0x00,0x00,0xe8,0x00,0x00,0x00, +0xf4,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xf7,0x00,0x00,0x00,0x6f,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xf8,0x00,0x00,0x00, +0xf5,0x00,0x00,0x00,0xf7,0x00,0x00,0x00,0x41,0x00,0x07,0x00, +0x06,0x01,0x00,0x00,0x07,0x01,0x00,0x00,0x04,0x01,0x00,0x00, +0x34,0x00,0x00,0x00,0xef,0x00,0x00,0x00,0x3e,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0xff,0x00,0x00,0x00,0x08,0x01,0x00,0x00, +0x07,0x01,0x00,0x00,0x73,0x00,0x04,0x00,0xc4,0x00,0x00,0x00, +0x09,0x01,0x00,0x00,0x08,0x01,0x00,0x00,0x41,0x00,0x05,0x00, +0x0a,0x01,0x00,0x00,0x0b,0x01,0x00,0x00,0xfd,0x00,0x00,0x00, +0xf8,0x00,0x00,0x00,0x3e,0x00,0x03,0x00,0x0b,0x01,0x00,0x00, +0x09,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x0d,0x01,0x00,0x00,0xf8,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0x41,0x00,0x07,0x00,0x06,0x01,0x00,0x00,0x0f,0x01,0x00,0x00, +0x04,0x01,0x00,0x00,0x34,0x00,0x00,0x00,0xef,0x00,0x00,0x00, +0x39,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0xff,0x00,0x00,0x00, +0x10,0x01,0x00,0x00,0x0f,0x01,0x00,0x00,0x73,0x00,0x04,0x00, +0xc4,0x00,0x00,0x00,0x11,0x01,0x00,0x00,0x10,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0x0a,0x01,0x00,0x00,0x12,0x01,0x00,0x00, +0xfd,0x00,0x00,0x00,0x0d,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, +0x12,0x01,0x00,0x00,0x11,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x14,0x01,0x00,0x00,0xf8,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x41,0x00,0x07,0x00,0x06,0x01,0x00,0x00, +0x16,0x01,0x00,0x00,0x04,0x01,0x00,0x00,0x34,0x00,0x00,0x00, +0xef,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0xff,0x00,0x00,0x00,0x17,0x01,0x00,0x00,0x16,0x01,0x00,0x00, +0x73,0x00,0x04,0x00,0xc4,0x00,0x00,0x00,0x18,0x01,0x00,0x00, +0x17,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0x0a,0x01,0x00,0x00, +0x19,0x01,0x00,0x00,0xfd,0x00,0x00,0x00,0x14,0x01,0x00,0x00, +0x3e,0x00,0x03,0x00,0x19,0x01,0x00,0x00,0x18,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x1c,0x01,0x00,0x00, +0xf8,0x00,0x00,0x00,0x1b,0x01,0x00,0x00,0x41,0x00,0x07,0x00, +0x06,0x01,0x00,0x00,0x1e,0x01,0x00,0x00,0x04,0x01,0x00,0x00, +0x34,0x00,0x00,0x00,0xef,0x00,0x00,0x00,0x1b,0x01,0x00,0x00, +0x3d,0x00,0x04,0x00,0xff,0x00,0x00,0x00,0x1f,0x01,0x00,0x00, +0x1e,0x01,0x00,0x00,0x73,0x00,0x04,0x00,0xc4,0x00,0x00,0x00, +0x20,0x01,0x00,0x00,0x1f,0x01,0x00,0x00,0x41,0x00,0x05,0x00, +0x0a,0x01,0x00,0x00,0x21,0x01,0x00,0x00,0xfd,0x00,0x00,0x00, +0x1c,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x21,0x01,0x00,0x00, +0x20,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x28,0x01,0x00,0x00,0xb5,0x02,0x00,0x00,0x26,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0xdd,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xdf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x2a,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x2a,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xb6,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, +0xdf,0x00,0x00,0x00,0x6d,0x01,0x00,0x00,0x2b,0x01,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0x30,0x01,0x00,0x00, +0xb6,0x02,0x00,0x00,0xa7,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0x2c,0x01,0x00,0x00,0x2b,0x01,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x30,0x01,0x00,0x00,0x2b,0x01,0x00,0x00, +0x2c,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x2b,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x35,0x01,0x00,0x00, +0x7e,0x00,0x00,0x00,0xb6,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x38,0x01,0x00,0x00,0x35,0x01,0x00,0x00, +0xab,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x39,0x01,0x00,0x00,0x38,0x01,0x00,0x00,0x6d,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3a,0x01,0x00,0x00, +0xbd,0x02,0x00,0x00,0x39,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x3c,0x01,0x00,0x00,0x3a,0x01,0x00,0x00, +0x79,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x42,0x01,0x00,0x00,0x35,0x01,0x00,0x00,0x41,0x01,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x44,0x01,0x00,0x00, +0x79,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x45,0x01,0x00,0x00,0x42,0x01,0x00,0x00, +0x44,0x01,0x00,0x00,0x41,0x00,0x07,0x00,0x53,0x01,0x00,0x00, +0x54,0x01,0x00,0x00,0x51,0x01,0x00,0x00,0x34,0x00,0x00,0x00, +0x3c,0x01,0x00,0x00,0x3e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0xc4,0x00,0x00,0x00,0x55,0x01,0x00,0x00,0x54,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0x0a,0x01,0x00,0x00,0x56,0x01,0x00,0x00, +0x4a,0x01,0x00,0x00,0x45,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, +0x56,0x01,0x00,0x00,0x55,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x58,0x01,0x00,0x00,0x45,0x01,0x00,0x00, +0x39,0x00,0x00,0x00,0x41,0x00,0x07,0x00,0x53,0x01,0x00,0x00, +0x5a,0x01,0x00,0x00,0x51,0x01,0x00,0x00,0x34,0x00,0x00,0x00, +0x3c,0x01,0x00,0x00,0x39,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0xc4,0x00,0x00,0x00,0x5b,0x01,0x00,0x00,0x5a,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0x0a,0x01,0x00,0x00,0x5c,0x01,0x00,0x00, +0x4a,0x01,0x00,0x00,0x58,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, +0x5c,0x01,0x00,0x00,0x5b,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x5e,0x01,0x00,0x00,0x45,0x01,0x00,0x00, +0x0c,0x00,0x00,0x00,0x41,0x00,0x07,0x00,0x53,0x01,0x00,0x00, +0x60,0x01,0x00,0x00,0x51,0x01,0x00,0x00,0x34,0x00,0x00,0x00, +0x3c,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0xc4,0x00,0x00,0x00,0x61,0x01,0x00,0x00,0x60,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0x0a,0x01,0x00,0x00,0x62,0x01,0x00,0x00, +0x4a,0x01,0x00,0x00,0x5e,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, +0x62,0x01,0x00,0x00,0x61,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x64,0x01,0x00,0x00,0x45,0x01,0x00,0x00, +0x1b,0x01,0x00,0x00,0x41,0x00,0x07,0x00,0x53,0x01,0x00,0x00, +0x66,0x01,0x00,0x00,0x51,0x01,0x00,0x00,0x34,0x00,0x00,0x00, +0x3c,0x01,0x00,0x00,0x1b,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, +0xc4,0x00,0x00,0x00,0x67,0x01,0x00,0x00,0x66,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0x0a,0x01,0x00,0x00,0x68,0x01,0x00,0x00, +0x4a,0x01,0x00,0x00,0x64,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, +0x68,0x01,0x00,0x00,0x67,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x6d,0x01,0x00,0x00,0xb6,0x02,0x00,0x00, +0x6b,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x2a,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x2c,0x01,0x00,0x00,0xe0,0x00,0x04,0x00, +0x0c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x6e,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x71,0x01,0x00,0x00, +0xb9,0x02,0x00,0x00,0x6f,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x74,0x01,0x00,0x00,0xbd,0x02,0x00,0x00, +0x72,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x76,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x76,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xbf,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, +0x2c,0x01,0x00,0x00,0x1d,0x02,0x00,0x00,0x79,0x01,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0x7c,0x01,0x00,0x00, +0xbf,0x02,0x00,0x00,0x6c,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0x78,0x01,0x00,0x00,0x79,0x01,0x00,0x00,0x00,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x7c,0x01,0x00,0x00,0x77,0x01,0x00,0x00, +0x78,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x77,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0x7e,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x7e,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xc3,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0x77,0x01,0x00,0x00, +0xa9,0x01,0x00,0x00,0x81,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc2,0x00,0x00,0x00,0x84,0x01,0x00,0x00,0xc3,0x02,0x00,0x00, +0x60,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x80,0x01,0x00,0x00, +0x81,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x84,0x01,0x00,0x00,0x7f,0x01,0x00,0x00,0x80,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x7f,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0x86,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x86,0x01,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xd5,0x02,0x00,0x00, +0x3e,0x00,0x00,0x00,0x7f,0x01,0x00,0x00,0xa7,0x01,0x00,0x00, +0x87,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, +0x8c,0x01,0x00,0x00,0xd5,0x02,0x00,0x00,0x62,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0x88,0x01,0x00,0x00,0x87,0x01,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x8c,0x01,0x00,0x00, +0x87,0x01,0x00,0x00,0x88,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x87,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x92,0x01,0x00,0x00,0xc3,0x02,0x00,0x00,0x62,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x94,0x01,0x00,0x00, +0x92,0x01,0x00,0x00,0xd5,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x96,0x01,0x00,0x00,0x55,0x00,0x00,0x00, +0x53,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x98,0x01,0x00,0x00,0xc3,0x02,0x00,0x00,0x61,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x99,0x01,0x00,0x00, +0x96,0x01,0x00,0x00,0x98,0x01,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x9b,0x01,0x00,0x00,0x64,0x00,0x00,0x00, +0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x9c,0x01,0x00,0x00,0x99,0x01,0x00,0x00,0x9b,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9e,0x01,0x00,0x00, +0x9c,0x01,0x00,0x00,0xd5,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xa0,0x01,0x00,0x00,0x9e,0x01,0x00,0x00, +0x9f,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xa2,0x01,0x00,0x00,0xa0,0x01,0x00,0x00,0xbf,0x02,0x00,0x00, +0x41,0x00,0x05,0x00,0x0a,0x01,0x00,0x00,0xa3,0x01,0x00,0x00, +0xfd,0x00,0x00,0x00,0xa2,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, +0xc4,0x00,0x00,0x00,0xa4,0x01,0x00,0x00,0xa3,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0xcd,0x00,0x00,0x00,0xa5,0x01,0x00,0x00, +0x90,0x01,0x00,0x00,0x94,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, +0xa5,0x01,0x00,0x00,0xa4,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xa7,0x01,0x00,0x00,0xd5,0x02,0x00,0x00, +0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x86,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x88,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0x81,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x81,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa9,0x01,0x00,0x00, +0xc3,0x02,0x00,0x00,0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x7e,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x80,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0xab,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xab,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xc4,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0x80,0x01,0x00,0x00, +0xd7,0x01,0x00,0x00,0xae,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc2,0x00,0x00,0x00,0xb1,0x01,0x00,0x00,0xc4,0x02,0x00,0x00, +0xbf,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xad,0x01,0x00,0x00, +0xae,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0xb1,0x01,0x00,0x00,0xac,0x01,0x00,0x00,0xad,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xac,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0xb3,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xb3,0x01,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xd2,0x02,0x00,0x00, +0x3e,0x00,0x00,0x00,0xac,0x01,0x00,0x00,0xd5,0x01,0x00,0x00, +0xb4,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, +0xb9,0x01,0x00,0x00,0xd2,0x02,0x00,0x00,0xbc,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0xb5,0x01,0x00,0x00,0xb4,0x01,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xb9,0x01,0x00,0x00, +0xb4,0x01,0x00,0x00,0xb5,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xb4,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xbf,0x01,0x00,0x00,0xc4,0x02,0x00,0x00,0xbc,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xc1,0x01,0x00,0x00, +0xbf,0x01,0x00,0x00,0xd2,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xc3,0x01,0x00,0x00,0x59,0x00,0x00,0x00, +0xb9,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xc6,0x01,0x00,0x00,0xc4,0x02,0x00,0x00,0xc5,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xc7,0x01,0x00,0x00, +0xc3,0x01,0x00,0x00,0xc6,0x01,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xc9,0x01,0x00,0x00,0x68,0x00,0x00,0x00, +0xbc,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xca,0x01,0x00,0x00,0xc7,0x01,0x00,0x00,0xc9,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xcc,0x01,0x00,0x00, +0xca,0x01,0x00,0x00,0xd2,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xce,0x01,0x00,0x00,0xcc,0x01,0x00,0x00, +0xcd,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xd0,0x01,0x00,0x00,0xce,0x01,0x00,0x00,0xbf,0x02,0x00,0x00, +0x41,0x00,0x05,0x00,0x0a,0x01,0x00,0x00,0xd1,0x01,0x00,0x00, +0x4a,0x01,0x00,0x00,0xd0,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, +0xc4,0x00,0x00,0x00,0xd2,0x01,0x00,0x00,0xd1,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0xcd,0x00,0x00,0x00,0xd3,0x01,0x00,0x00, +0xbd,0x01,0x00,0x00,0xc1,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, +0xd3,0x01,0x00,0x00,0xd2,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xd5,0x01,0x00,0x00,0xd2,0x02,0x00,0x00, +0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xb3,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xb5,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0xae,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xae,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xd7,0x01,0x00,0x00, +0xc4,0x02,0x00,0x00,0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xab,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xad,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0xd9,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xd9,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xc5,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0xad,0x01,0x00,0x00, +0x1b,0x02,0x00,0x00,0xdc,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc2,0x00,0x00,0x00,0xdf,0x01,0x00,0x00,0xc5,0x02,0x00,0x00, +0xbf,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xdb,0x01,0x00,0x00, +0xdc,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0xdf,0x01,0x00,0x00,0xda,0x01,0x00,0x00,0xdb,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xda,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0xe1,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xe1,0x01,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xc9,0x02,0x00,0x00, +0x3e,0x00,0x00,0x00,0xda,0x01,0x00,0x00,0x19,0x02,0x00,0x00, +0xe4,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, +0xe7,0x01,0x00,0x00,0xc9,0x02,0x00,0x00,0x60,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0xe3,0x01,0x00,0x00,0xe4,0x01,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xe7,0x01,0x00,0x00, +0xe2,0x01,0x00,0x00,0xe3,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xe2,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xe9,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xe9,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xcb,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, +0xe2,0x01,0x00,0x00,0x17,0x02,0x00,0x00,0xec,0x01,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0xef,0x01,0x00,0x00, +0xcb,0x02,0x00,0x00,0xbc,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0xeb,0x01,0x00,0x00,0xec,0x01,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0xef,0x01,0x00,0x00,0xea,0x01,0x00,0x00, +0xeb,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xea,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0xf1,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xf1,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xcd,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0xea,0x01,0x00,0x00, +0x15,0x02,0x00,0x00,0xf2,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc2,0x00,0x00,0x00,0xf7,0x01,0x00,0x00,0xcd,0x02,0x00,0x00, +0x62,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xf3,0x01,0x00,0x00, +0xf2,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0xf7,0x01,0x00,0x00,0xf2,0x01,0x00,0x00,0xf3,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xf2,0x01,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xf9,0x01,0x00,0x00,0xc5,0x02,0x00,0x00, +0xbc,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xfb,0x01,0x00,0x00,0xf9,0x01,0x00,0x00,0xcb,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xfd,0x01,0x00,0x00, +0xfb,0x01,0x00,0x00,0xfc,0x01,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xff,0x01,0x00,0x00,0xc9,0x02,0x00,0x00, +0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x00,0x02,0x00,0x00,0xfd,0x01,0x00,0x00,0xff,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x02,0x02,0x00,0x00, +0x00,0x02,0x00,0x00,0xcd,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x06,0x02,0x00,0x00,0xff,0x01,0x00,0x00, +0xcd,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0xcd,0x00,0x00,0x00, +0x07,0x02,0x00,0x00,0x90,0x01,0x00,0x00,0x06,0x02,0x00,0x00, +0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00,0x08,0x02,0x00,0x00, +0x07,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0xcd,0x00,0x00,0x00, +0x0d,0x02,0x00,0x00,0xbd,0x01,0x00,0x00,0xfb,0x01,0x00,0x00, +0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00,0x0e,0x02,0x00,0x00, +0x0d,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0xcd,0x00,0x00,0x00, +0x10,0x02,0x00,0x00,0xca,0x00,0x00,0x00,0x02,0x02,0x00,0x00, +0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00,0x11,0x02,0x00,0x00, +0x10,0x02,0x00,0x00,0x0c,0x00,0x08,0x00,0xc4,0x00,0x00,0x00, +0x12,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00, +0x08,0x02,0x00,0x00,0x0e,0x02,0x00,0x00,0x11,0x02,0x00,0x00, +0x3e,0x00,0x03,0x00,0x10,0x02,0x00,0x00,0x12,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x15,0x02,0x00,0x00, +0xcd,0x02,0x00,0x00,0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xf1,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xf3,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0xec,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xec,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x17,0x02,0x00,0x00,0xcb,0x02,0x00,0x00,0xd0,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xe9,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xeb,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xe4,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xe4,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x19,0x02,0x00,0x00,0xc9,0x02,0x00,0x00, +0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xe1,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xe3,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0xdc,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xdc,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x1b,0x02,0x00,0x00, +0xc5,0x02,0x00,0x00,0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xd9,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xdb,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0x79,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x79,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x1d,0x02,0x00,0x00,0xbf,0x02,0x00,0x00,0xd0,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x76,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x78,0x01,0x00,0x00,0xe0,0x00,0x04,0x00,0x0c,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x6e,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0xd7,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xd7,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x1f,0x02,0x00,0x00, +0xa5,0x02,0x00,0x00,0x6c,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xd4,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xd6,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x24,0x02,0x00,0x00, +0x55,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x25,0x02,0x00,0x00,0x96,0x00,0x00,0x00, +0x24,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x2a,0x02,0x00,0x00,0x59,0x00,0x00,0x00,0xb9,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x2b,0x02,0x00,0x00, +0xa8,0x00,0x00,0x00,0x2a,0x02,0x00,0x00,0x41,0x00,0x05,0x00, +0x15,0x00,0x00,0x00,0x30,0x02,0x00,0x00,0x12,0x00,0x00,0x00, +0x2f,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x31,0x02,0x00,0x00,0x30,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x32,0x02,0x00,0x00,0x0f,0x00,0x00,0x00, +0x31,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x36,0x02,0x00,0x00,0x47,0x00,0x00,0x00,0x31,0x02,0x00,0x00, +0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00,0x38,0x02,0x00,0x00, +0x37,0x02,0x00,0x00,0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x39,0x02,0x00,0x00,0x38,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3a,0x02,0x00,0x00, +0x36,0x02,0x00,0x00,0x39,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x3b,0x02,0x00,0x00,0x32,0x02,0x00,0x00, +0x3a,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x3d,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x3d,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xa6,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, +0xd6,0x00,0x00,0x00,0xa3,0x02,0x00,0x00,0x40,0x02,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0x43,0x02,0x00,0x00, +0xa6,0x02,0x00,0x00,0xbf,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0x3f,0x02,0x00,0x00,0x40,0x02,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x43,0x02,0x00,0x00,0x3e,0x02,0x00,0x00, +0x3f,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x3e,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0x45,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x45,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xa7,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0x3e,0x02,0x00,0x00, +0xa1,0x02,0x00,0x00,0x48,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc2,0x00,0x00,0x00,0x4b,0x02,0x00,0x00,0xa7,0x02,0x00,0x00, +0x60,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x47,0x02,0x00,0x00, +0x48,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x4b,0x02,0x00,0x00,0x46,0x02,0x00,0x00,0x47,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x46,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x4f,0x02,0x00,0x00,0xa7,0x02,0x00,0x00, +0x61,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x50,0x02,0x00,0x00,0x25,0x02,0x00,0x00,0x4f,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x52,0x02,0x00,0x00, +0x64,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x53,0x02,0x00,0x00,0x50,0x02,0x00,0x00, +0x52,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x57,0x02,0x00,0x00,0xa6,0x02,0x00,0x00,0xc5,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x58,0x02,0x00,0x00, +0x2b,0x02,0x00,0x00,0x57,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x5a,0x02,0x00,0x00,0x68,0x00,0x00,0x00, +0xbc,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x5b,0x02,0x00,0x00,0x58,0x02,0x00,0x00,0x5a,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0x5d,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x5d,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xa9,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0x46,0x02,0x00,0x00, +0x9f,0x02,0x00,0x00,0x60,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc2,0x00,0x00,0x00,0x63,0x02,0x00,0x00,0xa9,0x02,0x00,0x00, +0xbc,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x5f,0x02,0x00,0x00, +0x60,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x63,0x02,0x00,0x00,0x5e,0x02,0x00,0x00,0x5f,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x5e,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0x65,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x65,0x02,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xab,0x02,0x00,0x00, +0x3e,0x00,0x00,0x00,0x5e,0x02,0x00,0x00,0x9d,0x02,0x00,0x00, +0x68,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, +0x6b,0x02,0x00,0x00,0xab,0x02,0x00,0x00,0x62,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0x67,0x02,0x00,0x00,0x68,0x02,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x6b,0x02,0x00,0x00, +0x66,0x02,0x00,0x00,0x67,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x66,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x6e,0x02,0x00,0x00,0x53,0x02,0x00,0x00,0xab,0x02,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0x71,0x02,0x00,0x00, +0x6e,0x02,0x00,0x00,0x36,0x00,0x00,0x00,0xf7,0x00,0x03,0x00, +0x73,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x71,0x02,0x00,0x00,0x72,0x02,0x00,0x00,0x73,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x72,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x76,0x02,0x00,0x00,0x5b,0x02,0x00,0x00, +0xa9,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, +0x77,0x02,0x00,0x00,0x12,0x00,0x00,0x00,0xd0,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x78,0x02,0x00,0x00, +0x77,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, +0x79,0x02,0x00,0x00,0x76,0x02,0x00,0x00,0x78,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0x73,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x73,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0xc2,0x00,0x00,0x00, +0x7a,0x02,0x00,0x00,0x71,0x02,0x00,0x00,0x66,0x02,0x00,0x00, +0x79,0x02,0x00,0x00,0x72,0x02,0x00,0x00,0xf7,0x00,0x03,0x00, +0x7c,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x7a,0x02,0x00,0x00,0x7b,0x02,0x00,0x00,0x7c,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x7b,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x84,0x02,0x00,0x00,0x5b,0x02,0x00,0x00, +0xa9,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, +0x86,0x02,0x00,0x00,0x12,0x00,0x00,0x00,0x85,0x02,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x87,0x02,0x00,0x00, +0x86,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x88,0x02,0x00,0x00,0x84,0x02,0x00,0x00,0x87,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x89,0x02,0x00,0x00, +0x3b,0x02,0x00,0x00,0x88,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x8b,0x02,0x00,0x00,0x89,0x02,0x00,0x00, +0x53,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x8d,0x02,0x00,0x00,0x8b,0x02,0x00,0x00,0xab,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8f,0x02,0x00,0x00, +0xa6,0x02,0x00,0x00,0xbc,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x91,0x02,0x00,0x00,0x8f,0x02,0x00,0x00, +0xa9,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x93,0x02,0x00,0x00,0x91,0x02,0x00,0x00,0x92,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x95,0x02,0x00,0x00, +0xa7,0x02,0x00,0x00,0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x96,0x02,0x00,0x00,0x93,0x02,0x00,0x00, +0x95,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x98,0x02,0x00,0x00,0x96,0x02,0x00,0x00,0xab,0x02,0x00,0x00, +0x41,0x00,0x05,0x00,0xcd,0x00,0x00,0x00,0x99,0x02,0x00,0x00, +0xca,0x00,0x00,0x00,0x98,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, +0xc4,0x00,0x00,0x00,0x9a,0x02,0x00,0x00,0x99,0x02,0x00,0x00, +0x41,0x00,0x06,0x00,0x53,0x01,0x00,0x00,0x9b,0x02,0x00,0x00, +0x80,0x02,0x00,0x00,0x34,0x00,0x00,0x00,0x8d,0x02,0x00,0x00, +0x3e,0x00,0x03,0x00,0x9b,0x02,0x00,0x00,0x9a,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0x7c,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x7c,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x68,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x68,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x9d,0x02,0x00,0x00,0xab,0x02,0x00,0x00, +0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x65,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x67,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0x60,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x60,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9f,0x02,0x00,0x00, +0xa9,0x02,0x00,0x00,0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x5d,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x5f,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0x48,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x48,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xa1,0x02,0x00,0x00,0xa7,0x02,0x00,0x00,0xd0,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x45,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x47,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x40,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x40,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xa3,0x02,0x00,0x00,0xa6,0x02,0x00,0x00, +0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x3d,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x3f,0x02,0x00,0x00,0xfd,0x00,0x01,0x00, +0x38,0x00,0x01,0x00, +}; +const uint64_t matmul_f16_f32_aligned_fp32_len = 10240; + +unsigned char matmul_f16_f32_fp32_data[] = { +0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, +0xd6,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, +0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x51,0x11,0x00,0x00, +0x0b,0x00,0x06,0x00,0x01,0x00,0x00,0x00,0x47,0x4c,0x53,0x4c, +0x2e,0x73,0x74,0x64,0x2e,0x34,0x35,0x30,0x00,0x00,0x00,0x00, +0x0e,0x00,0x03,0x00,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x0f,0x00,0x0f,0x00,0x05,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x6d,0x61,0x69,0x6e,0x00,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, +0x12,0x00,0x00,0x00,0x3d,0x00,0x00,0x00,0x4c,0x00,0x00,0x00, +0xfa,0x00,0x00,0x00,0x06,0x01,0x00,0x00,0x46,0x01,0x00,0x00, +0x51,0x01,0x00,0x00,0x37,0x02,0x00,0x00,0x80,0x02,0x00,0x00, +0x10,0x00,0x06,0x00,0x04,0x00,0x00,0x00,0x11,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x0b,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, +0x1c,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x10,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x08,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, +0x03,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x10,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x14,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x18,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x07,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x10,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, +0x09,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x24,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x0a,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x10,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x2c,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x30,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x0d,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0x47,0x00,0x03,0x00, +0x10,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x37,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x3d,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, +0x1a,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x4c,0x00,0x00,0x00, +0x0b,0x00,0x00,0x00,0x1b,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x4f,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x09,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x53,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x60,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x62,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x07,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x6c,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x03,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xa6,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0xb8,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x05,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0xbb,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x08,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x03,0x01,0x00,0x00, +0x06,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x48,0x00,0x04,0x00, +0x04,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x04,0x01,0x00,0x00,0x00,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, +0x04,0x01,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x06,0x01,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x06,0x01,0x00,0x00,0x21,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x20,0x01,0x00,0x00, +0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x21,0x01,0x00,0x00,0x0b,0x00,0x00,0x00,0x19,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x4e,0x01,0x00,0x00,0x06,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0x4f,0x01,0x00,0x00, +0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x4f,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x4f,0x01,0x00,0x00, +0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x51,0x01,0x00,0x00, +0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x51,0x01,0x00,0x00,0x21,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x37,0x02,0x00,0x00,0x0b,0x00,0x00,0x00, +0x18,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x7d,0x02,0x00,0x00, +0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00, +0x7e,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x19,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x7e,0x02,0x00,0x00,0x00,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, +0x7e,0x02,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x80,0x02,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x80,0x02,0x00,0x00,0x21,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x13,0x00,0x02,0x00,0x02,0x00,0x00,0x00, +0x21,0x00,0x03,0x00,0x03,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x15,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x17,0x00,0x04,0x00,0x09,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x0a,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x09,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x0d,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x1e,0x00,0x10,0x00,0x10,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x11,0x00,0x00,0x00, +0x09,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0x11,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x09,0x00,0x00,0x00, +0x15,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, +0x14,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x15,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x21,0x00,0x00,0x00, +0x0a,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, +0x27,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0x2d,0x00,0x00,0x00,0x07,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x34,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x37,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00,0x3d,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x3e,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0x0a,0x00,0x00,0x00,0x4c,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x4f,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x53,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x54,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0x37,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x58,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0x37,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x32,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x61,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x60,0x00,0x00,0x00, +0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x62,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x63,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x61,0x00,0x00,0x00, +0x62,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x67,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x61,0x00,0x00,0x00, +0x62,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x72,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x77,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x7c,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0x80,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x85,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, +0x90,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0x96,0x00,0x00,0x00,0x03,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0xa1,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xa6,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0xa8,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xb7,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00, +0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xb8,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xb9,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x53,0x00,0x00,0x00, +0xb8,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xba,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x4f,0x00,0x00,0x00, +0x62,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xbb,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xbc,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0xba,0x00,0x00,0x00,0xbb,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xbd,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0xbc,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xbe,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0xb9,0x00,0x00,0x00,0xbd,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xbf,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0xb7,0x00,0x00,0x00,0xbe,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xc0,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0xbf,0x00,0x00,0x00,0xbb,0x00,0x00,0x00,0x14,0x00,0x02,0x00, +0xc1,0x00,0x00,0x00,0x16,0x00,0x03,0x00,0xc3,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xc4,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00, +0x62,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xc5,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0xc4,0x00,0x00,0x00, +0xbe,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xc6,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0xc5,0x00,0x00,0x00, +0xbb,0x00,0x00,0x00,0x1c,0x00,0x04,0x00,0xc7,0x00,0x00,0x00, +0xc3,0x00,0x00,0x00,0xc6,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0xc8,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0xc7,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0xc3,0x00,0x00,0x00,0xcb,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0xcc,0x00,0x00,0x00, +0x07,0x00,0x00,0x00,0xc3,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0xcf,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xf6,0x00,0x00,0x00, +0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xf7,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0xf6,0x00,0x00,0x00, +0x1c,0x00,0x04,0x00,0xf8,0x00,0x00,0x00,0xc3,0x00,0x00,0x00, +0xf7,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0xf9,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0xf8,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0xf9,0x00,0x00,0x00,0xfa,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xfe,0x00,0x00,0x00, +0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0x16,0x00,0x03,0x00,0x02,0x01,0x00,0x00,0x10,0x00,0x00,0x00, +0x1d,0x00,0x03,0x00,0x03,0x01,0x00,0x00,0x02,0x01,0x00,0x00, +0x1e,0x00,0x03,0x00,0x04,0x01,0x00,0x00,0x03,0x01,0x00,0x00, +0x20,0x00,0x04,0x00,0x05,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, +0x04,0x01,0x00,0x00,0x3b,0x00,0x04,0x00,0x05,0x01,0x00,0x00, +0x06,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x11,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x02,0x01,0x00,0x00, +0x20,0x00,0x04,0x00,0x15,0x01,0x00,0x00,0x04,0x00,0x00,0x00, +0xc3,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x1b,0x01,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, +0x39,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x20,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x33,0x00,0x06,0x00, +0x09,0x00,0x00,0x00,0x21,0x01,0x00,0x00,0x20,0x01,0x00,0x00, +0x39,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x22,0x01,0x00,0x00,0x51,0x00,0x00,0x00, +0x21,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x23,0x01,0x00,0x00,0x84,0x00,0x00,0x00, +0x22,0x01,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x24,0x01,0x00,0x00,0x86,0x00,0x00,0x00, +0x23,0x01,0x00,0x00,0x6c,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x42,0x01,0x00,0x00,0x80,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x43,0x01,0x00,0x00,0x84,0x00,0x00,0x00, +0xa6,0x00,0x00,0x00,0x42,0x01,0x00,0x00,0x1c,0x00,0x04,0x00, +0x44,0x01,0x00,0x00,0xc3,0x00,0x00,0x00,0x43,0x01,0x00,0x00, +0x20,0x00,0x04,0x00,0x45,0x01,0x00,0x00,0x04,0x00,0x00,0x00, +0x44,0x01,0x00,0x00,0x3b,0x00,0x04,0x00,0x45,0x01,0x00,0x00, +0x46,0x01,0x00,0x00,0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x4a,0x01,0x00,0x00,0x80,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, +0x4e,0x01,0x00,0x00,0xc3,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, +0x4f,0x01,0x00,0x00,0x4e,0x01,0x00,0x00,0x20,0x00,0x04,0x00, +0x50,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x4f,0x01,0x00,0x00, +0x3b,0x00,0x04,0x00,0x50,0x01,0x00,0x00,0x51,0x01,0x00,0x00, +0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x5c,0x01,0x00,0x00, +0x0c,0x00,0x00,0x00,0xc3,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x64,0x01,0x00,0x00,0x80,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x69,0x01,0x00,0x00,0x51,0x00,0x00,0x00, +0x21,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x6a,0x01,0x00,0x00,0x84,0x00,0x00,0x00, +0x69,0x01,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x6b,0x01,0x00,0x00,0x86,0x00,0x00,0x00, +0x6a,0x01,0x00,0x00,0x6c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x6e,0x01,0x00,0x00,0x08,0x01,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x6f,0x01,0x00,0x00, +0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x72,0x01,0x00,0x00, +0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x8d,0x01,0x00,0x00, +0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00, +0x1c,0x00,0x04,0x00,0x8e,0x01,0x00,0x00,0xc3,0x00,0x00,0x00, +0x8d,0x01,0x00,0x00,0x20,0x00,0x04,0x00,0x8f,0x01,0x00,0x00, +0x07,0x00,0x00,0x00,0x8e,0x01,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x9f,0x01,0x00,0x00,0x80,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xba,0x01,0x00,0x00,0x84,0x00,0x00,0x00, +0xbe,0x00,0x00,0x00,0xbb,0x00,0x00,0x00,0x1c,0x00,0x04,0x00, +0xbb,0x01,0x00,0x00,0xc3,0x00,0x00,0x00,0xba,0x01,0x00,0x00, +0x20,0x00,0x04,0x00,0xbc,0x01,0x00,0x00,0x07,0x00,0x00,0x00, +0xbb,0x01,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xc5,0x01,0x00,0x00,0x86,0x00,0x00,0x00,0xb8,0x00,0x00,0x00, +0xbe,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xcd,0x01,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, +0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xfc,0x01,0x00,0x00,0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00, +0x62,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, +0x2f,0x02,0x00,0x00,0x0d,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0x0a,0x00,0x00,0x00,0x37,0x02,0x00,0x00,0x01,0x00,0x00,0x00, +0x1d,0x00,0x03,0x00,0x7d,0x02,0x00,0x00,0xc3,0x00,0x00,0x00, +0x1e,0x00,0x03,0x00,0x7e,0x02,0x00,0x00,0x7d,0x02,0x00,0x00, +0x20,0x00,0x04,0x00,0x7f,0x02,0x00,0x00,0x0c,0x00,0x00,0x00, +0x7e,0x02,0x00,0x00,0x3b,0x00,0x04,0x00,0x7f,0x02,0x00,0x00, +0x80,0x02,0x00,0x00,0x0c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0x85,0x02,0x00,0x00,0x05,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x92,0x02,0x00,0x00, +0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00, +0x36,0x00,0x05,0x00,0x02,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0x05,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0xc8,0x00,0x00,0x00, +0xc9,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0x8f,0x01,0x00,0x00,0x90,0x01,0x00,0x00,0x07,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0xbc,0x01,0x00,0x00,0xbd,0x01,0x00,0x00, +0x07,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00, +0x0e,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, +0x0e,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, +0x16,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x14,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x17,0x00,0x00,0x00, +0x16,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x18,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x17,0x00,0x00,0x00, +0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x1e,0x00,0x00,0x00, +0x0f,0x00,0x00,0x00,0x17,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x15,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0x21,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x24,0x00,0x00,0x00,0x18,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, +0x28,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x27,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x29,0x00,0x00,0x00, +0x28,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x2a,0x00,0x00,0x00,0x1e,0x00,0x00,0x00,0x29,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, +0x12,0x00,0x00,0x00,0x2d,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x2f,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x30,0x00,0x00,0x00, +0x24,0x00,0x00,0x00,0x2f,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x30,0x00,0x00,0x00, +0x2a,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, +0x35,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x34,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x36,0x00,0x00,0x00, +0x35,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x38,0x00,0x00,0x00,0x36,0x00,0x00,0x00,0x37,0x00,0x00,0x00, +0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, +0x38,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x3b,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, +0x37,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00, +0x3f,0x00,0x00,0x00,0x3d,0x00,0x00,0x00,0x3e,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x40,0x00,0x00,0x00, +0x3f,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x42,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x3b,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x47,0x00,0x00,0x00, +0x40,0x00,0x00,0x00,0x3b,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x0d,0x00,0x00,0x00,0x49,0x00,0x00,0x00,0x3d,0x00,0x00,0x00, +0x39,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x4a,0x00,0x00,0x00,0x49,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x0d,0x00,0x00,0x00,0x4d,0x00,0x00,0x00,0x4c,0x00,0x00,0x00, +0x3e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x4e,0x00,0x00,0x00,0x4d,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x50,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, +0x4f,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x55,0x00,0x00,0x00,0x50,0x00,0x00,0x00,0x54,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x59,0x00,0x00,0x00, +0x50,0x00,0x00,0x00,0x58,0x00,0x00,0x00,0x89,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x5d,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, +0x4f,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x64,0x00,0x00,0x00,0x5d,0x00,0x00,0x00,0x63,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x68,0x00,0x00,0x00, +0x5d,0x00,0x00,0x00,0x67,0x00,0x00,0x00,0x89,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x6e,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, +0x6d,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x73,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x72,0x00,0x00,0x00, +0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x78,0x00,0x00,0x00, +0x4e,0x00,0x00,0x00,0x77,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x7d,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, +0x7c,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, +0x81,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x80,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x82,0x00,0x00,0x00, +0x81,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x83,0x00,0x00,0x00,0x47,0x00,0x00,0x00,0x82,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0x12,0x00,0x00,0x00,0x85,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x87,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x89,0x00,0x00,0x00, +0x47,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x8c,0x00,0x00,0x00,0x89,0x00,0x00,0x00, +0x82,0x00,0x00,0x00,0x0c,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0x8d,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x26,0x00,0x00,0x00, +0x87,0x00,0x00,0x00,0x8c,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x15,0x00,0x00,0x00,0x91,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0x90,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x92,0x00,0x00,0x00,0x91,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x93,0x00,0x00,0x00,0x32,0x00,0x00,0x00, +0x92,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x95,0x00,0x00,0x00,0x42,0x00,0x00,0x00,0x37,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x97,0x00,0x00,0x00, +0x12,0x00,0x00,0x00,0x96,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x98,0x00,0x00,0x00,0x97,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x99,0x00,0x00,0x00, +0x95,0x00,0x00,0x00,0x98,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x9a,0x00,0x00,0x00,0x93,0x00,0x00,0x00, +0x99,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x9c,0x00,0x00,0x00,0x9a,0x00,0x00,0x00,0x83,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9d,0x00,0x00,0x00, +0x9c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x15,0x00,0x00,0x00,0xa2,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0xa1,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xa3,0x00,0x00,0x00,0xa2,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xa4,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, +0xa3,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xa7,0x00,0x00,0x00,0x4a,0x00,0x00,0x00,0xa6,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0xa9,0x00,0x00,0x00, +0x12,0x00,0x00,0x00,0xa8,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xaa,0x00,0x00,0x00,0xa9,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xab,0x00,0x00,0x00, +0xa7,0x00,0x00,0x00,0xaa,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xac,0x00,0x00,0x00,0xa4,0x00,0x00,0x00, +0xab,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xae,0x00,0x00,0x00,0xac,0x00,0x00,0x00,0x83,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xaf,0x00,0x00,0x00, +0xae,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xb1,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xb1,0x00,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xa4,0x02,0x00,0x00, +0x3e,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0xd0,0x00,0x00,0x00, +0xb2,0x00,0x00,0x00,0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00, +0xc2,0x00,0x00,0x00,0xa4,0x02,0x00,0x00,0xc0,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0xb3,0x00,0x00,0x00,0xb2,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xc2,0x00,0x00,0x00, +0xb2,0x00,0x00,0x00,0xb3,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xb2,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0xcc,0x00,0x00,0x00, +0xcd,0x00,0x00,0x00,0xc9,0x00,0x00,0x00,0xa4,0x02,0x00,0x00, +0x3e,0x00,0x03,0x00,0xcd,0x00,0x00,0x00,0xcb,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xd0,0x00,0x00,0x00, +0xa4,0x02,0x00,0x00,0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xb1,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xb3,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xd3,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xd3,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xbd,0x02,0x00,0x00,0xaf,0x00,0x00,0x00,0xb3,0x00,0x00,0x00, +0x74,0x01,0x00,0x00,0xd6,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xb9,0x02,0x00,0x00,0x9d,0x00,0x00,0x00, +0xb3,0x00,0x00,0x00,0x71,0x01,0x00,0x00,0xd6,0x00,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xa5,0x02,0x00,0x00, +0x83,0x00,0x00,0x00,0xb3,0x00,0x00,0x00,0x1f,0x02,0x00,0x00, +0xd6,0x00,0x00,0x00,0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00, +0xda,0x00,0x00,0x00,0xa5,0x02,0x00,0x00,0x8d,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0xd5,0x00,0x00,0x00,0xd6,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xda,0x00,0x00,0x00, +0xd4,0x00,0x00,0x00,0xd5,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xd4,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xdc,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xdc,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xb5,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, +0xd4,0x00,0x00,0x00,0x26,0x01,0x00,0x00,0xdf,0x00,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00,0xe2,0x00,0x00,0x00, +0xb5,0x02,0x00,0x00,0x37,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0xde,0x00,0x00,0x00,0xdf,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0xe2,0x00,0x00,0x00,0xdd,0x00,0x00,0x00, +0xde,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xdd,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xe6,0x00,0x00,0x00, +0x95,0x00,0x00,0x00,0x73,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xe8,0x00,0x00,0x00,0xe6,0x00,0x00,0x00, +0xb5,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00, +0xeb,0x00,0x00,0x00,0xe8,0x00,0x00,0x00,0x36,0x00,0x00,0x00, +0xf7,0x00,0x03,0x00,0xed,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0xeb,0x00,0x00,0x00,0xec,0x00,0x00,0x00, +0xed,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xec,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xf0,0x00,0x00,0x00, +0xa5,0x02,0x00,0x00,0x6e,0x00,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc1,0x00,0x00,0x00,0xf2,0x00,0x00,0x00,0xf0,0x00,0x00,0x00, +0x8d,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xed,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xed,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, +0xc1,0x00,0x00,0x00,0xf3,0x00,0x00,0x00,0xeb,0x00,0x00,0x00, +0xdd,0x00,0x00,0x00,0xf2,0x00,0x00,0x00,0xec,0x00,0x00,0x00, +0xf7,0x00,0x03,0x00,0xf5,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0xf3,0x00,0x00,0x00,0xf4,0x00,0x00,0x00, +0x17,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xf4,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xfd,0x00,0x00,0x00, +0x73,0x00,0x00,0x00,0xb5,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xff,0x00,0x00,0x00,0xfd,0x00,0x00,0x00, +0xfe,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x01,0x01,0x00,0x00,0xff,0x00,0x00,0x00,0x6e,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x0d,0x01,0x00,0x00, +0xfd,0x00,0x00,0x00,0x98,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x0e,0x01,0x00,0x00,0xb9,0x02,0x00,0x00, +0x0d,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x10,0x01,0x00,0x00,0x0e,0x01,0x00,0x00,0x6e,0x00,0x00,0x00, +0x41,0x00,0x06,0x00,0x11,0x01,0x00,0x00,0x12,0x01,0x00,0x00, +0x06,0x01,0x00,0x00,0x34,0x00,0x00,0x00,0x10,0x01,0x00,0x00, +0x3d,0x00,0x04,0x00,0x02,0x01,0x00,0x00,0x13,0x01,0x00,0x00, +0x12,0x01,0x00,0x00,0x73,0x00,0x04,0x00,0xc3,0x00,0x00,0x00, +0x14,0x01,0x00,0x00,0x13,0x01,0x00,0x00,0x41,0x00,0x05,0x00, +0x15,0x01,0x00,0x00,0x16,0x01,0x00,0x00,0xfa,0x00,0x00,0x00, +0x01,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x16,0x01,0x00,0x00, +0x14,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xf5,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0x17,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x1a,0x01,0x00,0x00,0x73,0x00,0x00,0x00, +0xb5,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x1c,0x01,0x00,0x00,0x1a,0x01,0x00,0x00,0x1b,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x1e,0x01,0x00,0x00, +0x1c,0x01,0x00,0x00,0x6e,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x15,0x01,0x00,0x00,0x1f,0x01,0x00,0x00,0xfa,0x00,0x00,0x00, +0x1e,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x1f,0x01,0x00,0x00, +0xcb,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xf5,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xf5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xdf,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xdf,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x26,0x01,0x00,0x00, +0xb5,0x02,0x00,0x00,0x24,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0xdc,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xde,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x28,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x28,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xb6,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0xde,0x00,0x00,0x00, +0x6d,0x01,0x00,0x00,0x2b,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc1,0x00,0x00,0x00,0x2e,0x01,0x00,0x00,0xb6,0x02,0x00,0x00, +0xa6,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x2a,0x01,0x00,0x00, +0x2b,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x2e,0x01,0x00,0x00,0x29,0x01,0x00,0x00,0x2a,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x29,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x32,0x01,0x00,0x00,0xa7,0x00,0x00,0x00, +0x7d,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x34,0x01,0x00,0x00,0x32,0x01,0x00,0x00,0xb6,0x02,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x35,0x01,0x00,0x00, +0x12,0x00,0x00,0x00,0xcf,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x36,0x01,0x00,0x00,0x35,0x01,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00,0x37,0x01,0x00,0x00, +0x34,0x01,0x00,0x00,0x36,0x01,0x00,0x00,0xf7,0x00,0x03,0x00, +0x39,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x37,0x01,0x00,0x00,0x38,0x01,0x00,0x00,0x39,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x38,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x3c,0x01,0x00,0x00,0xa5,0x02,0x00,0x00, +0x78,0x00,0x00,0x00,0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00, +0x3e,0x01,0x00,0x00,0x3c,0x01,0x00,0x00,0x8d,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x39,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x39,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0xc1,0x00,0x00,0x00, +0x3f,0x01,0x00,0x00,0x37,0x01,0x00,0x00,0x29,0x01,0x00,0x00, +0x3e,0x01,0x00,0x00,0x38,0x01,0x00,0x00,0xf7,0x00,0x03,0x00, +0x41,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x3f,0x01,0x00,0x00,0x40,0x01,0x00,0x00,0x60,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x40,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x49,0x01,0x00,0x00,0x7d,0x00,0x00,0x00, +0xb6,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x4b,0x01,0x00,0x00,0x49,0x01,0x00,0x00,0x4a,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x4d,0x01,0x00,0x00, +0x4b,0x01,0x00,0x00,0x78,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x58,0x01,0x00,0x00,0x49,0x01,0x00,0x00, +0xaa,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x59,0x01,0x00,0x00,0xbd,0x02,0x00,0x00,0x58,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x5b,0x01,0x00,0x00, +0x59,0x01,0x00,0x00,0x78,0x00,0x00,0x00,0x41,0x00,0x06,0x00, +0x5c,0x01,0x00,0x00,0x5d,0x01,0x00,0x00,0x51,0x01,0x00,0x00, +0x34,0x00,0x00,0x00,0x5b,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, +0xc3,0x00,0x00,0x00,0x5e,0x01,0x00,0x00,0x5d,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x01,0x00,0x00,0x5f,0x01,0x00,0x00, +0x46,0x01,0x00,0x00,0x4d,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, +0x5f,0x01,0x00,0x00,0x5e,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0x41,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x60,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x63,0x01,0x00,0x00, +0x7d,0x00,0x00,0x00,0xb6,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x65,0x01,0x00,0x00,0x63,0x01,0x00,0x00, +0x64,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x67,0x01,0x00,0x00,0x65,0x01,0x00,0x00,0x78,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x01,0x00,0x00,0x68,0x01,0x00,0x00, +0x46,0x01,0x00,0x00,0x67,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, +0x68,0x01,0x00,0x00,0xcb,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x41,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x41,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0x2b,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x2b,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x6d,0x01,0x00,0x00,0xb6,0x02,0x00,0x00,0x6b,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0x28,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x2a,0x01,0x00,0x00,0xe0,0x00,0x04,0x00,0x0c,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x6e,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x71,0x01,0x00,0x00,0xb9,0x02,0x00,0x00, +0x6f,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x74,0x01,0x00,0x00,0xbd,0x02,0x00,0x00,0x72,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0x76,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x76,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xbf,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0x2a,0x01,0x00,0x00, +0x1d,0x02,0x00,0x00,0x79,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc1,0x00,0x00,0x00,0x7c,0x01,0x00,0x00,0xbf,0x02,0x00,0x00, +0x6c,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x78,0x01,0x00,0x00, +0x79,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x7c,0x01,0x00,0x00,0x77,0x01,0x00,0x00,0x78,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x77,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0x7e,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x7e,0x01,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xc3,0x02,0x00,0x00, +0x3e,0x00,0x00,0x00,0x77,0x01,0x00,0x00,0xa9,0x01,0x00,0x00, +0x81,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00, +0x84,0x01,0x00,0x00,0xc3,0x02,0x00,0x00,0x60,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0x80,0x01,0x00,0x00,0x81,0x01,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x84,0x01,0x00,0x00, +0x7f,0x01,0x00,0x00,0x80,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x7f,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x86,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x86,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xd5,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, +0x7f,0x01,0x00,0x00,0xa7,0x01,0x00,0x00,0x87,0x01,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00,0x8c,0x01,0x00,0x00, +0xd5,0x02,0x00,0x00,0x62,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0x88,0x01,0x00,0x00,0x87,0x01,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x8c,0x01,0x00,0x00,0x87,0x01,0x00,0x00, +0x88,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x87,0x01,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x92,0x01,0x00,0x00, +0xc3,0x02,0x00,0x00,0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x94,0x01,0x00,0x00,0x92,0x01,0x00,0x00, +0xd5,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x96,0x01,0x00,0x00,0x55,0x00,0x00,0x00,0x53,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x98,0x01,0x00,0x00, +0xc3,0x02,0x00,0x00,0x61,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x99,0x01,0x00,0x00,0x96,0x01,0x00,0x00, +0x98,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x9b,0x01,0x00,0x00,0x64,0x00,0x00,0x00,0x62,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9c,0x01,0x00,0x00, +0x99,0x01,0x00,0x00,0x9b,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x9e,0x01,0x00,0x00,0x9c,0x01,0x00,0x00, +0xd5,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xa0,0x01,0x00,0x00,0x9e,0x01,0x00,0x00,0x9f,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa2,0x01,0x00,0x00, +0xa0,0x01,0x00,0x00,0xbf,0x02,0x00,0x00,0x41,0x00,0x05,0x00, +0x15,0x01,0x00,0x00,0xa3,0x01,0x00,0x00,0xfa,0x00,0x00,0x00, +0xa2,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0xc3,0x00,0x00,0x00, +0xa4,0x01,0x00,0x00,0xa3,0x01,0x00,0x00,0x41,0x00,0x05,0x00, +0xcc,0x00,0x00,0x00,0xa5,0x01,0x00,0x00,0x90,0x01,0x00,0x00, +0x94,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0xa5,0x01,0x00,0x00, +0xa4,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xa7,0x01,0x00,0x00,0xd5,0x02,0x00,0x00,0xcf,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x86,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x88,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x81,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x81,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xa9,0x01,0x00,0x00,0xc3,0x02,0x00,0x00, +0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x7e,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x80,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0xab,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xab,0x01,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xc4,0x02,0x00,0x00, +0x3e,0x00,0x00,0x00,0x80,0x01,0x00,0x00,0xd7,0x01,0x00,0x00, +0xae,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00, +0xb1,0x01,0x00,0x00,0xc4,0x02,0x00,0x00,0xbe,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0xad,0x01,0x00,0x00,0xae,0x01,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xb1,0x01,0x00,0x00, +0xac,0x01,0x00,0x00,0xad,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xac,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xb3,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xb3,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xd2,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, +0xac,0x01,0x00,0x00,0xd5,0x01,0x00,0x00,0xb4,0x01,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00,0xb9,0x01,0x00,0x00, +0xd2,0x02,0x00,0x00,0xbb,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0xb5,0x01,0x00,0x00,0xb4,0x01,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0xb9,0x01,0x00,0x00,0xb4,0x01,0x00,0x00, +0xb5,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xb4,0x01,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xbf,0x01,0x00,0x00, +0xc4,0x02,0x00,0x00,0xbb,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xc1,0x01,0x00,0x00,0xbf,0x01,0x00,0x00, +0xd2,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xc3,0x01,0x00,0x00,0x59,0x00,0x00,0x00,0xb8,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xc6,0x01,0x00,0x00, +0xc4,0x02,0x00,0x00,0xc5,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xc7,0x01,0x00,0x00,0xc3,0x01,0x00,0x00, +0xc6,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xc9,0x01,0x00,0x00,0x68,0x00,0x00,0x00,0xbb,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xca,0x01,0x00,0x00, +0xc7,0x01,0x00,0x00,0xc9,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xcc,0x01,0x00,0x00,0xca,0x01,0x00,0x00, +0xd2,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xce,0x01,0x00,0x00,0xcc,0x01,0x00,0x00,0xcd,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xd0,0x01,0x00,0x00, +0xce,0x01,0x00,0x00,0xbf,0x02,0x00,0x00,0x41,0x00,0x05,0x00, +0x15,0x01,0x00,0x00,0xd1,0x01,0x00,0x00,0x46,0x01,0x00,0x00, +0xd0,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0xc3,0x00,0x00,0x00, +0xd2,0x01,0x00,0x00,0xd1,0x01,0x00,0x00,0x41,0x00,0x05,0x00, +0xcc,0x00,0x00,0x00,0xd3,0x01,0x00,0x00,0xbd,0x01,0x00,0x00, +0xc1,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0xd3,0x01,0x00,0x00, +0xd2,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xd5,0x01,0x00,0x00,0xd2,0x02,0x00,0x00,0xcf,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xb3,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xb5,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xae,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xae,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xd7,0x01,0x00,0x00,0xc4,0x02,0x00,0x00, +0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xab,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xad,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0xd9,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xd9,0x01,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xc5,0x02,0x00,0x00, +0x3e,0x00,0x00,0x00,0xad,0x01,0x00,0x00,0x1b,0x02,0x00,0x00, +0xdc,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00, +0xdf,0x01,0x00,0x00,0xc5,0x02,0x00,0x00,0xbe,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0xdb,0x01,0x00,0x00,0xdc,0x01,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xdf,0x01,0x00,0x00, +0xda,0x01,0x00,0x00,0xdb,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xda,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xe1,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xe1,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xc9,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, +0xda,0x01,0x00,0x00,0x19,0x02,0x00,0x00,0xe4,0x01,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00,0xe7,0x01,0x00,0x00, +0xc9,0x02,0x00,0x00,0x60,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0xe3,0x01,0x00,0x00,0xe4,0x01,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0xe7,0x01,0x00,0x00,0xe2,0x01,0x00,0x00, +0xe3,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xe2,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0xe9,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xe9,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xcb,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0xe2,0x01,0x00,0x00, +0x17,0x02,0x00,0x00,0xec,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc1,0x00,0x00,0x00,0xef,0x01,0x00,0x00,0xcb,0x02,0x00,0x00, +0xbb,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xeb,0x01,0x00,0x00, +0xec,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0xef,0x01,0x00,0x00,0xea,0x01,0x00,0x00,0xeb,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xea,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0xf1,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xf1,0x01,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xcd,0x02,0x00,0x00, +0x3e,0x00,0x00,0x00,0xea,0x01,0x00,0x00,0x15,0x02,0x00,0x00, +0xf2,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00, +0xf7,0x01,0x00,0x00,0xcd,0x02,0x00,0x00,0x62,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0xf3,0x01,0x00,0x00,0xf2,0x01,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xf7,0x01,0x00,0x00, +0xf2,0x01,0x00,0x00,0xf3,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xf2,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xf9,0x01,0x00,0x00,0xc5,0x02,0x00,0x00,0xbb,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xfb,0x01,0x00,0x00, +0xf9,0x01,0x00,0x00,0xcb,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xfd,0x01,0x00,0x00,0xfb,0x01,0x00,0x00, +0xfc,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xff,0x01,0x00,0x00,0xc9,0x02,0x00,0x00,0x62,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x00,0x02,0x00,0x00, +0xfd,0x01,0x00,0x00,0xff,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x02,0x02,0x00,0x00,0x00,0x02,0x00,0x00, +0xcd,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x06,0x02,0x00,0x00,0xff,0x01,0x00,0x00,0xcd,0x02,0x00,0x00, +0x41,0x00,0x05,0x00,0xcc,0x00,0x00,0x00,0x07,0x02,0x00,0x00, +0x90,0x01,0x00,0x00,0x06,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, +0xc3,0x00,0x00,0x00,0x08,0x02,0x00,0x00,0x07,0x02,0x00,0x00, +0x41,0x00,0x05,0x00,0xcc,0x00,0x00,0x00,0x0d,0x02,0x00,0x00, +0xbd,0x01,0x00,0x00,0xfb,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, +0xc3,0x00,0x00,0x00,0x0e,0x02,0x00,0x00,0x0d,0x02,0x00,0x00, +0x41,0x00,0x05,0x00,0xcc,0x00,0x00,0x00,0x10,0x02,0x00,0x00, +0xc9,0x00,0x00,0x00,0x02,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, +0xc3,0x00,0x00,0x00,0x11,0x02,0x00,0x00,0x10,0x02,0x00,0x00, +0x0c,0x00,0x08,0x00,0xc3,0x00,0x00,0x00,0x12,0x02,0x00,0x00, +0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x08,0x02,0x00,0x00, +0x0e,0x02,0x00,0x00,0x11,0x02,0x00,0x00,0x3e,0x00,0x03,0x00, +0x10,0x02,0x00,0x00,0x12,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x15,0x02,0x00,0x00,0xcd,0x02,0x00,0x00, +0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xf1,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xf3,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0xec,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xec,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x17,0x02,0x00,0x00, +0xcb,0x02,0x00,0x00,0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xe9,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xeb,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0xe4,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xe4,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x19,0x02,0x00,0x00,0xc9,0x02,0x00,0x00,0xcf,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xe1,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xe3,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xdc,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xdc,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x1b,0x02,0x00,0x00,0xc5,0x02,0x00,0x00, +0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xd9,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xdb,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0x79,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x79,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x1d,0x02,0x00,0x00, +0xbf,0x02,0x00,0x00,0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x76,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x78,0x01,0x00,0x00, +0xe0,0x00,0x04,0x00,0x0c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x6e,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xd6,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xd6,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x1f,0x02,0x00,0x00,0xa5,0x02,0x00,0x00, +0x6c,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xd3,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xd5,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x24,0x02,0x00,0x00,0x55,0x00,0x00,0x00, +0x53,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x25,0x02,0x00,0x00,0x95,0x00,0x00,0x00,0x24,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x2a,0x02,0x00,0x00, +0x59,0x00,0x00,0x00,0xb8,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x2b,0x02,0x00,0x00,0xa7,0x00,0x00,0x00, +0x2a,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, +0x30,0x02,0x00,0x00,0x12,0x00,0x00,0x00,0x2f,0x02,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x31,0x02,0x00,0x00, +0x30,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x32,0x02,0x00,0x00,0x0f,0x00,0x00,0x00,0x31,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x36,0x02,0x00,0x00, +0x47,0x00,0x00,0x00,0x31,0x02,0x00,0x00,0x41,0x00,0x05,0x00, +0x0d,0x00,0x00,0x00,0x38,0x02,0x00,0x00,0x37,0x02,0x00,0x00, +0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x39,0x02,0x00,0x00,0x38,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x3a,0x02,0x00,0x00,0x36,0x02,0x00,0x00, +0x39,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x3b,0x02,0x00,0x00,0x32,0x02,0x00,0x00,0x3a,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0x3d,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x3d,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xa6,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0xd5,0x00,0x00,0x00, +0xa3,0x02,0x00,0x00,0x40,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc1,0x00,0x00,0x00,0x43,0x02,0x00,0x00,0xa6,0x02,0x00,0x00, +0xbe,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x3f,0x02,0x00,0x00, +0x40,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x43,0x02,0x00,0x00,0x3e,0x02,0x00,0x00,0x3f,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x3e,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0x45,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x45,0x02,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xa7,0x02,0x00,0x00, +0x3e,0x00,0x00,0x00,0x3e,0x02,0x00,0x00,0xa1,0x02,0x00,0x00, +0x48,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00, +0x4b,0x02,0x00,0x00,0xa7,0x02,0x00,0x00,0x60,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0x47,0x02,0x00,0x00,0x48,0x02,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x4b,0x02,0x00,0x00, +0x46,0x02,0x00,0x00,0x47,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x46,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x4f,0x02,0x00,0x00,0xa7,0x02,0x00,0x00,0x61,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x50,0x02,0x00,0x00, +0x25,0x02,0x00,0x00,0x4f,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x52,0x02,0x00,0x00,0x64,0x00,0x00,0x00, +0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x53,0x02,0x00,0x00,0x50,0x02,0x00,0x00,0x52,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x57,0x02,0x00,0x00, +0xa6,0x02,0x00,0x00,0xc5,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x58,0x02,0x00,0x00,0x2b,0x02,0x00,0x00, +0x57,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x5a,0x02,0x00,0x00,0x68,0x00,0x00,0x00,0xbb,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x5b,0x02,0x00,0x00, +0x58,0x02,0x00,0x00,0x5a,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0x5d,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x5d,0x02,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xa9,0x02,0x00,0x00, +0x3e,0x00,0x00,0x00,0x46,0x02,0x00,0x00,0x9f,0x02,0x00,0x00, +0x60,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00, +0x63,0x02,0x00,0x00,0xa9,0x02,0x00,0x00,0xbb,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0x5f,0x02,0x00,0x00,0x60,0x02,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x63,0x02,0x00,0x00, +0x5e,0x02,0x00,0x00,0x5f,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x5e,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x65,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x65,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xab,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, +0x5e,0x02,0x00,0x00,0x9d,0x02,0x00,0x00,0x68,0x02,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00,0x6b,0x02,0x00,0x00, +0xab,0x02,0x00,0x00,0x62,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0x67,0x02,0x00,0x00,0x68,0x02,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x6b,0x02,0x00,0x00,0x66,0x02,0x00,0x00, +0x67,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x66,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x6e,0x02,0x00,0x00, +0x53,0x02,0x00,0x00,0xab,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc1,0x00,0x00,0x00,0x71,0x02,0x00,0x00,0x6e,0x02,0x00,0x00, +0x36,0x00,0x00,0x00,0xf7,0x00,0x03,0x00,0x73,0x02,0x00,0x00, +0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x71,0x02,0x00,0x00, +0x72,0x02,0x00,0x00,0x73,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x72,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x76,0x02,0x00,0x00,0x5b,0x02,0x00,0x00,0xa9,0x02,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x77,0x02,0x00,0x00, +0x12,0x00,0x00,0x00,0xcf,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x78,0x02,0x00,0x00,0x77,0x02,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00,0x79,0x02,0x00,0x00, +0x76,0x02,0x00,0x00,0x78,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0x73,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x73,0x02,0x00,0x00, +0xf5,0x00,0x07,0x00,0xc1,0x00,0x00,0x00,0x7a,0x02,0x00,0x00, +0x71,0x02,0x00,0x00,0x66,0x02,0x00,0x00,0x79,0x02,0x00,0x00, +0x72,0x02,0x00,0x00,0xf7,0x00,0x03,0x00,0x7c,0x02,0x00,0x00, +0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x7a,0x02,0x00,0x00, +0x7b,0x02,0x00,0x00,0x7c,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x7b,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x84,0x02,0x00,0x00,0x5b,0x02,0x00,0x00,0xa9,0x02,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x86,0x02,0x00,0x00, +0x12,0x00,0x00,0x00,0x85,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x87,0x02,0x00,0x00,0x86,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x88,0x02,0x00,0x00, +0x84,0x02,0x00,0x00,0x87,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x89,0x02,0x00,0x00,0x3b,0x02,0x00,0x00, +0x88,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x8b,0x02,0x00,0x00,0x89,0x02,0x00,0x00,0x53,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8d,0x02,0x00,0x00, +0x8b,0x02,0x00,0x00,0xab,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x8f,0x02,0x00,0x00,0xa6,0x02,0x00,0x00, +0xbb,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x91,0x02,0x00,0x00,0x8f,0x02,0x00,0x00,0xa9,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x93,0x02,0x00,0x00, +0x91,0x02,0x00,0x00,0x92,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x95,0x02,0x00,0x00,0xa7,0x02,0x00,0x00, +0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x96,0x02,0x00,0x00,0x93,0x02,0x00,0x00,0x95,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x98,0x02,0x00,0x00, +0x96,0x02,0x00,0x00,0xab,0x02,0x00,0x00,0x41,0x00,0x05,0x00, +0xcc,0x00,0x00,0x00,0x99,0x02,0x00,0x00,0xc9,0x00,0x00,0x00, +0x98,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0xc3,0x00,0x00,0x00, +0x9a,0x02,0x00,0x00,0x99,0x02,0x00,0x00,0x41,0x00,0x06,0x00, +0x5c,0x01,0x00,0x00,0x9b,0x02,0x00,0x00,0x80,0x02,0x00,0x00, +0x34,0x00,0x00,0x00,0x8d,0x02,0x00,0x00,0x3e,0x00,0x03,0x00, +0x9b,0x02,0x00,0x00,0x9a,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0x7c,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x7c,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0x68,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x68,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x9d,0x02,0x00,0x00,0xab,0x02,0x00,0x00,0xcf,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x65,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x67,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x60,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x60,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x9f,0x02,0x00,0x00,0xa9,0x02,0x00,0x00, +0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x5d,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x5f,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0x48,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x48,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa1,0x02,0x00,0x00, +0xa7,0x02,0x00,0x00,0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x45,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x47,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0x40,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x40,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xa3,0x02,0x00,0x00,0xa6,0x02,0x00,0x00,0xcf,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x3d,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x3f,0x02,0x00,0x00,0xfd,0x00,0x01,0x00,0x38,0x00,0x01,0x00, + +}; +const uint64_t matmul_f16_f32_fp32_len = 10260; + +unsigned char matmul_f16_fp32_data[] = { +0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, +0xd7,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, +0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x51,0x11,0x00,0x00, +0x0b,0x00,0x06,0x00,0x01,0x00,0x00,0x00,0x47,0x4c,0x53,0x4c, +0x2e,0x73,0x74,0x64,0x2e,0x34,0x35,0x30,0x00,0x00,0x00,0x00, +0x0e,0x00,0x03,0x00,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x0f,0x00,0x0f,0x00,0x05,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x6d,0x61,0x69,0x6e,0x00,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, +0x12,0x00,0x00,0x00,0x3d,0x00,0x00,0x00,0x4c,0x00,0x00,0x00, +0xfa,0x00,0x00,0x00,0x06,0x01,0x00,0x00,0x46,0x01,0x00,0x00, +0x51,0x01,0x00,0x00,0x37,0x02,0x00,0x00,0x80,0x02,0x00,0x00, +0x10,0x00,0x06,0x00,0x04,0x00,0x00,0x00,0x11,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x0b,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, +0x1c,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x10,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x08,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, +0x03,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x10,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x14,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x18,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x07,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x10,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, +0x09,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x24,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x0a,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x10,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x2c,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x30,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x0d,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0x47,0x00,0x03,0x00, +0x10,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x37,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x3d,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, +0x1a,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x4c,0x00,0x00,0x00, +0x0b,0x00,0x00,0x00,0x1b,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x4f,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x09,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x53,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x60,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x62,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x07,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x6c,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x03,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xa6,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0xb8,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x05,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0xbb,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x08,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x03,0x01,0x00,0x00, +0x06,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x48,0x00,0x04,0x00, +0x04,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x04,0x01,0x00,0x00,0x00,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, +0x04,0x01,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x06,0x01,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x06,0x01,0x00,0x00,0x21,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x20,0x01,0x00,0x00, +0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x21,0x01,0x00,0x00,0x0b,0x00,0x00,0x00,0x19,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x4e,0x01,0x00,0x00,0x06,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0x4f,0x01,0x00,0x00, +0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x4f,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x4f,0x01,0x00,0x00, +0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x51,0x01,0x00,0x00, +0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x51,0x01,0x00,0x00,0x21,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x37,0x02,0x00,0x00,0x0b,0x00,0x00,0x00, +0x18,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x7d,0x02,0x00,0x00, +0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00, +0x7e,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x19,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x7e,0x02,0x00,0x00,0x00,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, +0x7e,0x02,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x80,0x02,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x80,0x02,0x00,0x00,0x21,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x13,0x00,0x02,0x00,0x02,0x00,0x00,0x00, +0x21,0x00,0x03,0x00,0x03,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x15,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x17,0x00,0x04,0x00,0x09,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x0a,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x09,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x0d,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x1e,0x00,0x10,0x00,0x10,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x11,0x00,0x00,0x00, +0x09,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0x11,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x09,0x00,0x00,0x00, +0x15,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, +0x14,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x15,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x21,0x00,0x00,0x00, +0x0a,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, +0x27,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0x2d,0x00,0x00,0x00,0x07,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x34,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x37,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00,0x3d,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x3e,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0x0a,0x00,0x00,0x00,0x4c,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x4f,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x53,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x54,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0x37,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x58,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0x37,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x32,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x61,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x60,0x00,0x00,0x00, +0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x62,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x63,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x61,0x00,0x00,0x00, +0x62,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x67,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x61,0x00,0x00,0x00, +0x62,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x72,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x77,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x7c,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0x80,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x85,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, +0x90,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0x96,0x00,0x00,0x00,0x03,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0xa1,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xa6,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0xa8,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xb7,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00, +0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xb8,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xb9,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x53,0x00,0x00,0x00, +0xb8,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xba,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x4f,0x00,0x00,0x00, +0x62,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xbb,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xbc,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0xba,0x00,0x00,0x00,0xbb,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xbd,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0xbc,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xbe,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0xb9,0x00,0x00,0x00,0xbd,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xbf,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0xb7,0x00,0x00,0x00,0xbe,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xc0,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0xbf,0x00,0x00,0x00,0xbb,0x00,0x00,0x00,0x14,0x00,0x02,0x00, +0xc1,0x00,0x00,0x00,0x16,0x00,0x03,0x00,0xc3,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xc4,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00, +0x62,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xc5,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0xc4,0x00,0x00,0x00, +0xbe,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xc6,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0xc5,0x00,0x00,0x00, +0xbb,0x00,0x00,0x00,0x1c,0x00,0x04,0x00,0xc7,0x00,0x00,0x00, +0xc3,0x00,0x00,0x00,0xc6,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0xc8,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0xc7,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0xc3,0x00,0x00,0x00,0xcb,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0xcc,0x00,0x00,0x00, +0x07,0x00,0x00,0x00,0xc3,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0xcf,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xf6,0x00,0x00,0x00, +0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xf7,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0xf6,0x00,0x00,0x00, +0x1c,0x00,0x04,0x00,0xf8,0x00,0x00,0x00,0xc3,0x00,0x00,0x00, +0xf7,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0xf9,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0xf8,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0xf9,0x00,0x00,0x00,0xfa,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xfe,0x00,0x00,0x00, +0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0x16,0x00,0x03,0x00,0x02,0x01,0x00,0x00,0x10,0x00,0x00,0x00, +0x1d,0x00,0x03,0x00,0x03,0x01,0x00,0x00,0x02,0x01,0x00,0x00, +0x1e,0x00,0x03,0x00,0x04,0x01,0x00,0x00,0x03,0x01,0x00,0x00, +0x20,0x00,0x04,0x00,0x05,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, +0x04,0x01,0x00,0x00,0x3b,0x00,0x04,0x00,0x05,0x01,0x00,0x00, +0x06,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x11,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x02,0x01,0x00,0x00, +0x20,0x00,0x04,0x00,0x15,0x01,0x00,0x00,0x04,0x00,0x00,0x00, +0xc3,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x1b,0x01,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, +0x39,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x20,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x33,0x00,0x06,0x00, +0x09,0x00,0x00,0x00,0x21,0x01,0x00,0x00,0x20,0x01,0x00,0x00, +0x39,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x22,0x01,0x00,0x00,0x51,0x00,0x00,0x00, +0x21,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x23,0x01,0x00,0x00,0x84,0x00,0x00,0x00, +0x22,0x01,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x24,0x01,0x00,0x00,0x86,0x00,0x00,0x00, +0x23,0x01,0x00,0x00,0x6c,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x42,0x01,0x00,0x00,0x80,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x43,0x01,0x00,0x00,0x84,0x00,0x00,0x00, +0xa6,0x00,0x00,0x00,0x42,0x01,0x00,0x00,0x1c,0x00,0x04,0x00, +0x44,0x01,0x00,0x00,0xc3,0x00,0x00,0x00,0x43,0x01,0x00,0x00, +0x20,0x00,0x04,0x00,0x45,0x01,0x00,0x00,0x04,0x00,0x00,0x00, +0x44,0x01,0x00,0x00,0x3b,0x00,0x04,0x00,0x45,0x01,0x00,0x00, +0x46,0x01,0x00,0x00,0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x4a,0x01,0x00,0x00,0x80,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, +0x4e,0x01,0x00,0x00,0x02,0x01,0x00,0x00,0x1e,0x00,0x03,0x00, +0x4f,0x01,0x00,0x00,0x4e,0x01,0x00,0x00,0x20,0x00,0x04,0x00, +0x50,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x4f,0x01,0x00,0x00, +0x3b,0x00,0x04,0x00,0x50,0x01,0x00,0x00,0x51,0x01,0x00,0x00, +0x0c,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x64,0x01,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, +0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x69,0x01,0x00,0x00,0x51,0x00,0x00,0x00,0x21,0x01,0x00,0x00, +0x00,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x6a,0x01,0x00,0x00,0x84,0x00,0x00,0x00,0x69,0x01,0x00,0x00, +0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x6b,0x01,0x00,0x00,0x86,0x00,0x00,0x00,0x6a,0x01,0x00,0x00, +0x6c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x6e,0x01,0x00,0x00,0x08,0x01,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x6f,0x01,0x00,0x00,0x86,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x72,0x01,0x00,0x00,0x86,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x8d,0x01,0x00,0x00,0x84,0x00,0x00,0x00, +0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x1c,0x00,0x04,0x00, +0x8e,0x01,0x00,0x00,0xc3,0x00,0x00,0x00,0x8d,0x01,0x00,0x00, +0x20,0x00,0x04,0x00,0x8f,0x01,0x00,0x00,0x07,0x00,0x00,0x00, +0x8e,0x01,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x9f,0x01,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, +0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xba,0x01,0x00,0x00,0x84,0x00,0x00,0x00,0xbe,0x00,0x00,0x00, +0xbb,0x00,0x00,0x00,0x1c,0x00,0x04,0x00,0xbb,0x01,0x00,0x00, +0xc3,0x00,0x00,0x00,0xba,0x01,0x00,0x00,0x20,0x00,0x04,0x00, +0xbc,0x01,0x00,0x00,0x07,0x00,0x00,0x00,0xbb,0x01,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xc5,0x01,0x00,0x00, +0x86,0x00,0x00,0x00,0xb8,0x00,0x00,0x00,0xbe,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xcd,0x01,0x00,0x00, +0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xfc,0x01,0x00,0x00, +0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x2f,0x02,0x00,0x00, +0x0d,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00, +0x37,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, +0x7d,0x02,0x00,0x00,0xc3,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, +0x7e,0x02,0x00,0x00,0x7d,0x02,0x00,0x00,0x20,0x00,0x04,0x00, +0x7f,0x02,0x00,0x00,0x0c,0x00,0x00,0x00,0x7e,0x02,0x00,0x00, +0x3b,0x00,0x04,0x00,0x7f,0x02,0x00,0x00,0x80,0x02,0x00,0x00, +0x0c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, +0x85,0x02,0x00,0x00,0x05,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x92,0x02,0x00,0x00,0x84,0x00,0x00,0x00, +0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x9b,0x02,0x00,0x00,0x0c,0x00,0x00,0x00,0xc3,0x00,0x00,0x00, +0x36,0x00,0x05,0x00,0x02,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0x05,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0xc8,0x00,0x00,0x00, +0xc9,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0x8f,0x01,0x00,0x00,0x90,0x01,0x00,0x00,0x07,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0xbc,0x01,0x00,0x00,0xbd,0x01,0x00,0x00, +0x07,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00, +0x0e,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, +0x0e,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, +0x16,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x14,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x17,0x00,0x00,0x00, +0x16,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x18,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x17,0x00,0x00,0x00, +0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x1e,0x00,0x00,0x00, +0x0f,0x00,0x00,0x00,0x17,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x15,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0x21,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x24,0x00,0x00,0x00,0x18,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, +0x28,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x27,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x29,0x00,0x00,0x00, +0x28,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x2a,0x00,0x00,0x00,0x1e,0x00,0x00,0x00,0x29,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, +0x12,0x00,0x00,0x00,0x2d,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x2f,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x30,0x00,0x00,0x00, +0x24,0x00,0x00,0x00,0x2f,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x30,0x00,0x00,0x00, +0x2a,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, +0x35,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x34,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x36,0x00,0x00,0x00, +0x35,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x38,0x00,0x00,0x00,0x36,0x00,0x00,0x00,0x37,0x00,0x00,0x00, +0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, +0x38,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x3b,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, +0x37,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00, +0x3f,0x00,0x00,0x00,0x3d,0x00,0x00,0x00,0x3e,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x40,0x00,0x00,0x00, +0x3f,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x42,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x3b,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x47,0x00,0x00,0x00, +0x40,0x00,0x00,0x00,0x3b,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x0d,0x00,0x00,0x00,0x49,0x00,0x00,0x00,0x3d,0x00,0x00,0x00, +0x39,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x4a,0x00,0x00,0x00,0x49,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x0d,0x00,0x00,0x00,0x4d,0x00,0x00,0x00,0x4c,0x00,0x00,0x00, +0x3e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x4e,0x00,0x00,0x00,0x4d,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x50,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, +0x4f,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x55,0x00,0x00,0x00,0x50,0x00,0x00,0x00,0x54,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x59,0x00,0x00,0x00, +0x50,0x00,0x00,0x00,0x58,0x00,0x00,0x00,0x89,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x5d,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, +0x4f,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x64,0x00,0x00,0x00,0x5d,0x00,0x00,0x00,0x63,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x68,0x00,0x00,0x00, +0x5d,0x00,0x00,0x00,0x67,0x00,0x00,0x00,0x89,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x6e,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, +0x6d,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x73,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x72,0x00,0x00,0x00, +0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x78,0x00,0x00,0x00, +0x4e,0x00,0x00,0x00,0x77,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x7d,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, +0x7c,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, +0x81,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x80,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x82,0x00,0x00,0x00, +0x81,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x83,0x00,0x00,0x00,0x47,0x00,0x00,0x00,0x82,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0x12,0x00,0x00,0x00,0x85,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x87,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x89,0x00,0x00,0x00, +0x47,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x8c,0x00,0x00,0x00,0x89,0x00,0x00,0x00, +0x82,0x00,0x00,0x00,0x0c,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0x8d,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x26,0x00,0x00,0x00, +0x87,0x00,0x00,0x00,0x8c,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x15,0x00,0x00,0x00,0x91,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0x90,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x92,0x00,0x00,0x00,0x91,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x93,0x00,0x00,0x00,0x32,0x00,0x00,0x00, +0x92,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x95,0x00,0x00,0x00,0x42,0x00,0x00,0x00,0x37,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x97,0x00,0x00,0x00, +0x12,0x00,0x00,0x00,0x96,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x98,0x00,0x00,0x00,0x97,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x99,0x00,0x00,0x00, +0x95,0x00,0x00,0x00,0x98,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x9a,0x00,0x00,0x00,0x93,0x00,0x00,0x00, +0x99,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x9c,0x00,0x00,0x00,0x9a,0x00,0x00,0x00,0x83,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9d,0x00,0x00,0x00, +0x9c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x15,0x00,0x00,0x00,0xa2,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0xa1,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xa3,0x00,0x00,0x00,0xa2,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xa4,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, +0xa3,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xa7,0x00,0x00,0x00,0x4a,0x00,0x00,0x00,0xa6,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0xa9,0x00,0x00,0x00, +0x12,0x00,0x00,0x00,0xa8,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xaa,0x00,0x00,0x00,0xa9,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xab,0x00,0x00,0x00, +0xa7,0x00,0x00,0x00,0xaa,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xac,0x00,0x00,0x00,0xa4,0x00,0x00,0x00, +0xab,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xae,0x00,0x00,0x00,0xac,0x00,0x00,0x00,0x83,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xaf,0x00,0x00,0x00, +0xae,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xb1,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xb1,0x00,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xa5,0x02,0x00,0x00, +0x3e,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0xd0,0x00,0x00,0x00, +0xb2,0x00,0x00,0x00,0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00, +0xc2,0x00,0x00,0x00,0xa5,0x02,0x00,0x00,0xc0,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0xb3,0x00,0x00,0x00,0xb2,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xc2,0x00,0x00,0x00, +0xb2,0x00,0x00,0x00,0xb3,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xb2,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0xcc,0x00,0x00,0x00, +0xcd,0x00,0x00,0x00,0xc9,0x00,0x00,0x00,0xa5,0x02,0x00,0x00, +0x3e,0x00,0x03,0x00,0xcd,0x00,0x00,0x00,0xcb,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xd0,0x00,0x00,0x00, +0xa5,0x02,0x00,0x00,0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xb1,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xb3,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xd3,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xd3,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xbe,0x02,0x00,0x00,0xaf,0x00,0x00,0x00,0xb3,0x00,0x00,0x00, +0x74,0x01,0x00,0x00,0xd6,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xba,0x02,0x00,0x00,0x9d,0x00,0x00,0x00, +0xb3,0x00,0x00,0x00,0x71,0x01,0x00,0x00,0xd6,0x00,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xa6,0x02,0x00,0x00, +0x83,0x00,0x00,0x00,0xb3,0x00,0x00,0x00,0x1f,0x02,0x00,0x00, +0xd6,0x00,0x00,0x00,0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00, +0xda,0x00,0x00,0x00,0xa6,0x02,0x00,0x00,0x8d,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0xd5,0x00,0x00,0x00,0xd6,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xda,0x00,0x00,0x00, +0xd4,0x00,0x00,0x00,0xd5,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xd4,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xdc,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xdc,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xb6,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, +0xd4,0x00,0x00,0x00,0x26,0x01,0x00,0x00,0xdf,0x00,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00,0xe2,0x00,0x00,0x00, +0xb6,0x02,0x00,0x00,0x37,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0xde,0x00,0x00,0x00,0xdf,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0xe2,0x00,0x00,0x00,0xdd,0x00,0x00,0x00, +0xde,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xdd,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xe6,0x00,0x00,0x00, +0x95,0x00,0x00,0x00,0x73,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xe8,0x00,0x00,0x00,0xe6,0x00,0x00,0x00, +0xb6,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00, +0xeb,0x00,0x00,0x00,0xe8,0x00,0x00,0x00,0x36,0x00,0x00,0x00, +0xf7,0x00,0x03,0x00,0xed,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0xeb,0x00,0x00,0x00,0xec,0x00,0x00,0x00, +0xed,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xec,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xf0,0x00,0x00,0x00, +0xa6,0x02,0x00,0x00,0x6e,0x00,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc1,0x00,0x00,0x00,0xf2,0x00,0x00,0x00,0xf0,0x00,0x00,0x00, +0x8d,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xed,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xed,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, +0xc1,0x00,0x00,0x00,0xf3,0x00,0x00,0x00,0xeb,0x00,0x00,0x00, +0xdd,0x00,0x00,0x00,0xf2,0x00,0x00,0x00,0xec,0x00,0x00,0x00, +0xf7,0x00,0x03,0x00,0xf5,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0xf3,0x00,0x00,0x00,0xf4,0x00,0x00,0x00, +0x17,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xf4,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xfd,0x00,0x00,0x00, +0x73,0x00,0x00,0x00,0xb6,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xff,0x00,0x00,0x00,0xfd,0x00,0x00,0x00, +0xfe,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x01,0x01,0x00,0x00,0xff,0x00,0x00,0x00,0x6e,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x0d,0x01,0x00,0x00, +0xfd,0x00,0x00,0x00,0x98,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x0e,0x01,0x00,0x00,0xba,0x02,0x00,0x00, +0x0d,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x10,0x01,0x00,0x00,0x0e,0x01,0x00,0x00,0x6e,0x00,0x00,0x00, +0x41,0x00,0x06,0x00,0x11,0x01,0x00,0x00,0x12,0x01,0x00,0x00, +0x06,0x01,0x00,0x00,0x34,0x00,0x00,0x00,0x10,0x01,0x00,0x00, +0x3d,0x00,0x04,0x00,0x02,0x01,0x00,0x00,0x13,0x01,0x00,0x00, +0x12,0x01,0x00,0x00,0x73,0x00,0x04,0x00,0xc3,0x00,0x00,0x00, +0x14,0x01,0x00,0x00,0x13,0x01,0x00,0x00,0x41,0x00,0x05,0x00, +0x15,0x01,0x00,0x00,0x16,0x01,0x00,0x00,0xfa,0x00,0x00,0x00, +0x01,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x16,0x01,0x00,0x00, +0x14,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xf5,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0x17,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x1a,0x01,0x00,0x00,0x73,0x00,0x00,0x00, +0xb6,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x1c,0x01,0x00,0x00,0x1a,0x01,0x00,0x00,0x1b,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x1e,0x01,0x00,0x00, +0x1c,0x01,0x00,0x00,0x6e,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x15,0x01,0x00,0x00,0x1f,0x01,0x00,0x00,0xfa,0x00,0x00,0x00, +0x1e,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x1f,0x01,0x00,0x00, +0xcb,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xf5,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xf5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xdf,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xdf,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x26,0x01,0x00,0x00, +0xb6,0x02,0x00,0x00,0x24,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0xdc,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xde,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x28,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x28,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xb7,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0xde,0x00,0x00,0x00, +0x6d,0x01,0x00,0x00,0x2b,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc1,0x00,0x00,0x00,0x2e,0x01,0x00,0x00,0xb7,0x02,0x00,0x00, +0xa6,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x2a,0x01,0x00,0x00, +0x2b,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x2e,0x01,0x00,0x00,0x29,0x01,0x00,0x00,0x2a,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x29,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x32,0x01,0x00,0x00,0xa7,0x00,0x00,0x00, +0x7d,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x34,0x01,0x00,0x00,0x32,0x01,0x00,0x00,0xb7,0x02,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x35,0x01,0x00,0x00, +0x12,0x00,0x00,0x00,0xcf,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x36,0x01,0x00,0x00,0x35,0x01,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00,0x37,0x01,0x00,0x00, +0x34,0x01,0x00,0x00,0x36,0x01,0x00,0x00,0xf7,0x00,0x03,0x00, +0x39,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x37,0x01,0x00,0x00,0x38,0x01,0x00,0x00,0x39,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x38,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x3c,0x01,0x00,0x00,0xa6,0x02,0x00,0x00, +0x78,0x00,0x00,0x00,0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00, +0x3e,0x01,0x00,0x00,0x3c,0x01,0x00,0x00,0x8d,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x39,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x39,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0xc1,0x00,0x00,0x00, +0x3f,0x01,0x00,0x00,0x37,0x01,0x00,0x00,0x29,0x01,0x00,0x00, +0x3e,0x01,0x00,0x00,0x38,0x01,0x00,0x00,0xf7,0x00,0x03,0x00, +0x41,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x3f,0x01,0x00,0x00,0x40,0x01,0x00,0x00,0x60,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x40,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x49,0x01,0x00,0x00,0x7d,0x00,0x00,0x00, +0xb7,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x4b,0x01,0x00,0x00,0x49,0x01,0x00,0x00,0x4a,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x4d,0x01,0x00,0x00, +0x4b,0x01,0x00,0x00,0x78,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x58,0x01,0x00,0x00,0x49,0x01,0x00,0x00, +0xaa,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x59,0x01,0x00,0x00,0xbe,0x02,0x00,0x00,0x58,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x5b,0x01,0x00,0x00, +0x59,0x01,0x00,0x00,0x78,0x00,0x00,0x00,0x41,0x00,0x06,0x00, +0x11,0x01,0x00,0x00,0x5c,0x01,0x00,0x00,0x51,0x01,0x00,0x00, +0x34,0x00,0x00,0x00,0x5b,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, +0x02,0x01,0x00,0x00,0x5d,0x01,0x00,0x00,0x5c,0x01,0x00,0x00, +0x73,0x00,0x04,0x00,0xc3,0x00,0x00,0x00,0x5e,0x01,0x00,0x00, +0x5d,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x01,0x00,0x00, +0x5f,0x01,0x00,0x00,0x46,0x01,0x00,0x00,0x4d,0x01,0x00,0x00, +0x3e,0x00,0x03,0x00,0x5f,0x01,0x00,0x00,0x5e,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0x41,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x60,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x63,0x01,0x00,0x00,0x7d,0x00,0x00,0x00,0xb7,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x65,0x01,0x00,0x00, +0x63,0x01,0x00,0x00,0x64,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x67,0x01,0x00,0x00,0x65,0x01,0x00,0x00, +0x78,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x01,0x00,0x00, +0x68,0x01,0x00,0x00,0x46,0x01,0x00,0x00,0x67,0x01,0x00,0x00, +0x3e,0x00,0x03,0x00,0x68,0x01,0x00,0x00,0xcb,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x41,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x41,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x2b,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x2b,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x6d,0x01,0x00,0x00,0xb7,0x02,0x00,0x00, +0x6b,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x28,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x2a,0x01,0x00,0x00,0xe0,0x00,0x04,0x00, +0x0c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x6e,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x71,0x01,0x00,0x00, +0xba,0x02,0x00,0x00,0x6f,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x74,0x01,0x00,0x00,0xbe,0x02,0x00,0x00, +0x72,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x76,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x76,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xc0,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, +0x2a,0x01,0x00,0x00,0x1d,0x02,0x00,0x00,0x79,0x01,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00,0x7c,0x01,0x00,0x00, +0xc0,0x02,0x00,0x00,0x6c,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0x78,0x01,0x00,0x00,0x79,0x01,0x00,0x00,0x00,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x7c,0x01,0x00,0x00,0x77,0x01,0x00,0x00, +0x78,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x77,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0x7e,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x7e,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xc4,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0x77,0x01,0x00,0x00, +0xa9,0x01,0x00,0x00,0x81,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc1,0x00,0x00,0x00,0x84,0x01,0x00,0x00,0xc4,0x02,0x00,0x00, +0x60,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x80,0x01,0x00,0x00, +0x81,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x84,0x01,0x00,0x00,0x7f,0x01,0x00,0x00,0x80,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x7f,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0x86,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x86,0x01,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xd6,0x02,0x00,0x00, +0x3e,0x00,0x00,0x00,0x7f,0x01,0x00,0x00,0xa7,0x01,0x00,0x00, +0x87,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00, +0x8c,0x01,0x00,0x00,0xd6,0x02,0x00,0x00,0x62,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0x88,0x01,0x00,0x00,0x87,0x01,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x8c,0x01,0x00,0x00, +0x87,0x01,0x00,0x00,0x88,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x87,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x92,0x01,0x00,0x00,0xc4,0x02,0x00,0x00,0x62,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x94,0x01,0x00,0x00, +0x92,0x01,0x00,0x00,0xd6,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x96,0x01,0x00,0x00,0x55,0x00,0x00,0x00, +0x53,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x98,0x01,0x00,0x00,0xc4,0x02,0x00,0x00,0x61,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x99,0x01,0x00,0x00, +0x96,0x01,0x00,0x00,0x98,0x01,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x9b,0x01,0x00,0x00,0x64,0x00,0x00,0x00, +0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x9c,0x01,0x00,0x00,0x99,0x01,0x00,0x00,0x9b,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9e,0x01,0x00,0x00, +0x9c,0x01,0x00,0x00,0xd6,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xa0,0x01,0x00,0x00,0x9e,0x01,0x00,0x00, +0x9f,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xa2,0x01,0x00,0x00,0xa0,0x01,0x00,0x00,0xc0,0x02,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x01,0x00,0x00,0xa3,0x01,0x00,0x00, +0xfa,0x00,0x00,0x00,0xa2,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, +0xc3,0x00,0x00,0x00,0xa4,0x01,0x00,0x00,0xa3,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0xcc,0x00,0x00,0x00,0xa5,0x01,0x00,0x00, +0x90,0x01,0x00,0x00,0x94,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, +0xa5,0x01,0x00,0x00,0xa4,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xa7,0x01,0x00,0x00,0xd6,0x02,0x00,0x00, +0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x86,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x88,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0x81,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x81,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa9,0x01,0x00,0x00, +0xc4,0x02,0x00,0x00,0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x7e,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x80,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0xab,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xab,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xc5,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0x80,0x01,0x00,0x00, +0xd7,0x01,0x00,0x00,0xae,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc1,0x00,0x00,0x00,0xb1,0x01,0x00,0x00,0xc5,0x02,0x00,0x00, +0xbe,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xad,0x01,0x00,0x00, +0xae,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0xb1,0x01,0x00,0x00,0xac,0x01,0x00,0x00,0xad,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xac,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0xb3,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xb3,0x01,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xd3,0x02,0x00,0x00, +0x3e,0x00,0x00,0x00,0xac,0x01,0x00,0x00,0xd5,0x01,0x00,0x00, +0xb4,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00, +0xb9,0x01,0x00,0x00,0xd3,0x02,0x00,0x00,0xbb,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0xb5,0x01,0x00,0x00,0xb4,0x01,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xb9,0x01,0x00,0x00, +0xb4,0x01,0x00,0x00,0xb5,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xb4,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xbf,0x01,0x00,0x00,0xc5,0x02,0x00,0x00,0xbb,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xc1,0x01,0x00,0x00, +0xbf,0x01,0x00,0x00,0xd3,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xc3,0x01,0x00,0x00,0x59,0x00,0x00,0x00, +0xb8,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xc6,0x01,0x00,0x00,0xc5,0x02,0x00,0x00,0xc5,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xc7,0x01,0x00,0x00, +0xc3,0x01,0x00,0x00,0xc6,0x01,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xc9,0x01,0x00,0x00,0x68,0x00,0x00,0x00, +0xbb,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xca,0x01,0x00,0x00,0xc7,0x01,0x00,0x00,0xc9,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xcc,0x01,0x00,0x00, +0xca,0x01,0x00,0x00,0xd3,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xce,0x01,0x00,0x00,0xcc,0x01,0x00,0x00, +0xcd,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xd0,0x01,0x00,0x00,0xce,0x01,0x00,0x00,0xc0,0x02,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x01,0x00,0x00,0xd1,0x01,0x00,0x00, +0x46,0x01,0x00,0x00,0xd0,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, +0xc3,0x00,0x00,0x00,0xd2,0x01,0x00,0x00,0xd1,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0xcc,0x00,0x00,0x00,0xd3,0x01,0x00,0x00, +0xbd,0x01,0x00,0x00,0xc1,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, +0xd3,0x01,0x00,0x00,0xd2,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xd5,0x01,0x00,0x00,0xd3,0x02,0x00,0x00, +0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xb3,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xb5,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0xae,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xae,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xd7,0x01,0x00,0x00, +0xc5,0x02,0x00,0x00,0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xab,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xad,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0xd9,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xd9,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xc6,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0xad,0x01,0x00,0x00, +0x1b,0x02,0x00,0x00,0xdc,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc1,0x00,0x00,0x00,0xdf,0x01,0x00,0x00,0xc6,0x02,0x00,0x00, +0xbe,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xdb,0x01,0x00,0x00, +0xdc,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0xdf,0x01,0x00,0x00,0xda,0x01,0x00,0x00,0xdb,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xda,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0xe1,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xe1,0x01,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xca,0x02,0x00,0x00, +0x3e,0x00,0x00,0x00,0xda,0x01,0x00,0x00,0x19,0x02,0x00,0x00, +0xe4,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00, +0xe7,0x01,0x00,0x00,0xca,0x02,0x00,0x00,0x60,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0xe3,0x01,0x00,0x00,0xe4,0x01,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xe7,0x01,0x00,0x00, +0xe2,0x01,0x00,0x00,0xe3,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xe2,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xe9,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xe9,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xcc,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, +0xe2,0x01,0x00,0x00,0x17,0x02,0x00,0x00,0xec,0x01,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00,0xef,0x01,0x00,0x00, +0xcc,0x02,0x00,0x00,0xbb,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0xeb,0x01,0x00,0x00,0xec,0x01,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0xef,0x01,0x00,0x00,0xea,0x01,0x00,0x00, +0xeb,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xea,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0xf1,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xf1,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xce,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0xea,0x01,0x00,0x00, +0x15,0x02,0x00,0x00,0xf2,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc1,0x00,0x00,0x00,0xf7,0x01,0x00,0x00,0xce,0x02,0x00,0x00, +0x62,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xf3,0x01,0x00,0x00, +0xf2,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0xf7,0x01,0x00,0x00,0xf2,0x01,0x00,0x00,0xf3,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xf2,0x01,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xf9,0x01,0x00,0x00,0xc6,0x02,0x00,0x00, +0xbb,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xfb,0x01,0x00,0x00,0xf9,0x01,0x00,0x00,0xcc,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xfd,0x01,0x00,0x00, +0xfb,0x01,0x00,0x00,0xfc,0x01,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xff,0x01,0x00,0x00,0xca,0x02,0x00,0x00, +0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x00,0x02,0x00,0x00,0xfd,0x01,0x00,0x00,0xff,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x02,0x02,0x00,0x00, +0x00,0x02,0x00,0x00,0xce,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x06,0x02,0x00,0x00,0xff,0x01,0x00,0x00, +0xce,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0xcc,0x00,0x00,0x00, +0x07,0x02,0x00,0x00,0x90,0x01,0x00,0x00,0x06,0x02,0x00,0x00, +0x3d,0x00,0x04,0x00,0xc3,0x00,0x00,0x00,0x08,0x02,0x00,0x00, +0x07,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0xcc,0x00,0x00,0x00, +0x0d,0x02,0x00,0x00,0xbd,0x01,0x00,0x00,0xfb,0x01,0x00,0x00, +0x3d,0x00,0x04,0x00,0xc3,0x00,0x00,0x00,0x0e,0x02,0x00,0x00, +0x0d,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0xcc,0x00,0x00,0x00, +0x10,0x02,0x00,0x00,0xc9,0x00,0x00,0x00,0x02,0x02,0x00,0x00, +0x3d,0x00,0x04,0x00,0xc3,0x00,0x00,0x00,0x11,0x02,0x00,0x00, +0x10,0x02,0x00,0x00,0x0c,0x00,0x08,0x00,0xc3,0x00,0x00,0x00, +0x12,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00, +0x08,0x02,0x00,0x00,0x0e,0x02,0x00,0x00,0x11,0x02,0x00,0x00, +0x3e,0x00,0x03,0x00,0x10,0x02,0x00,0x00,0x12,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x15,0x02,0x00,0x00, +0xce,0x02,0x00,0x00,0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xf1,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xf3,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0xec,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xec,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x17,0x02,0x00,0x00,0xcc,0x02,0x00,0x00,0xcf,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xe9,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xeb,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xe4,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xe4,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x19,0x02,0x00,0x00,0xca,0x02,0x00,0x00, +0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xe1,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xe3,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0xdc,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xdc,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x1b,0x02,0x00,0x00, +0xc6,0x02,0x00,0x00,0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xd9,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xdb,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0x79,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x79,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x1d,0x02,0x00,0x00,0xc0,0x02,0x00,0x00,0xcf,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x76,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x78,0x01,0x00,0x00,0xe0,0x00,0x04,0x00,0x0c,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x6e,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0xd6,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xd6,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x1f,0x02,0x00,0x00, +0xa6,0x02,0x00,0x00,0x6c,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xd3,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xd5,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x24,0x02,0x00,0x00, +0x55,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x25,0x02,0x00,0x00,0x95,0x00,0x00,0x00, +0x24,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x2a,0x02,0x00,0x00,0x59,0x00,0x00,0x00,0xb8,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x2b,0x02,0x00,0x00, +0xa7,0x00,0x00,0x00,0x2a,0x02,0x00,0x00,0x41,0x00,0x05,0x00, +0x15,0x00,0x00,0x00,0x30,0x02,0x00,0x00,0x12,0x00,0x00,0x00, +0x2f,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x31,0x02,0x00,0x00,0x30,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x32,0x02,0x00,0x00,0x0f,0x00,0x00,0x00, +0x31,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x36,0x02,0x00,0x00,0x47,0x00,0x00,0x00,0x31,0x02,0x00,0x00, +0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00,0x38,0x02,0x00,0x00, +0x37,0x02,0x00,0x00,0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x39,0x02,0x00,0x00,0x38,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3a,0x02,0x00,0x00, +0x36,0x02,0x00,0x00,0x39,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x3b,0x02,0x00,0x00,0x32,0x02,0x00,0x00, +0x3a,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x3d,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x3d,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xa7,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, +0xd5,0x00,0x00,0x00,0xa4,0x02,0x00,0x00,0x40,0x02,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00,0x43,0x02,0x00,0x00, +0xa7,0x02,0x00,0x00,0xbe,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0x3f,0x02,0x00,0x00,0x40,0x02,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x43,0x02,0x00,0x00,0x3e,0x02,0x00,0x00, +0x3f,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x3e,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0x45,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x45,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xa8,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0x3e,0x02,0x00,0x00, +0xa2,0x02,0x00,0x00,0x48,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc1,0x00,0x00,0x00,0x4b,0x02,0x00,0x00,0xa8,0x02,0x00,0x00, +0x60,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x47,0x02,0x00,0x00, +0x48,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x4b,0x02,0x00,0x00,0x46,0x02,0x00,0x00,0x47,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x46,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x4f,0x02,0x00,0x00,0xa8,0x02,0x00,0x00, +0x61,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x50,0x02,0x00,0x00,0x25,0x02,0x00,0x00,0x4f,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x52,0x02,0x00,0x00, +0x64,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x53,0x02,0x00,0x00,0x50,0x02,0x00,0x00, +0x52,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x57,0x02,0x00,0x00,0xa7,0x02,0x00,0x00,0xc5,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x58,0x02,0x00,0x00, +0x2b,0x02,0x00,0x00,0x57,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x5a,0x02,0x00,0x00,0x68,0x00,0x00,0x00, +0xbb,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x5b,0x02,0x00,0x00,0x58,0x02,0x00,0x00,0x5a,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0x5d,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x5d,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xaa,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0x46,0x02,0x00,0x00, +0xa0,0x02,0x00,0x00,0x60,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc1,0x00,0x00,0x00,0x63,0x02,0x00,0x00,0xaa,0x02,0x00,0x00, +0xbb,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x5f,0x02,0x00,0x00, +0x60,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x63,0x02,0x00,0x00,0x5e,0x02,0x00,0x00,0x5f,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x5e,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0x65,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x65,0x02,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xac,0x02,0x00,0x00, +0x3e,0x00,0x00,0x00,0x5e,0x02,0x00,0x00,0x9e,0x02,0x00,0x00, +0x68,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00, +0x6b,0x02,0x00,0x00,0xac,0x02,0x00,0x00,0x62,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0x67,0x02,0x00,0x00,0x68,0x02,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x6b,0x02,0x00,0x00, +0x66,0x02,0x00,0x00,0x67,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x66,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x6e,0x02,0x00,0x00,0x53,0x02,0x00,0x00,0xac,0x02,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00,0x71,0x02,0x00,0x00, +0x6e,0x02,0x00,0x00,0x36,0x00,0x00,0x00,0xf7,0x00,0x03,0x00, +0x73,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x71,0x02,0x00,0x00,0x72,0x02,0x00,0x00,0x73,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x72,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x76,0x02,0x00,0x00,0x5b,0x02,0x00,0x00, +0xaa,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, +0x77,0x02,0x00,0x00,0x12,0x00,0x00,0x00,0xcf,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x78,0x02,0x00,0x00, +0x77,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00, +0x79,0x02,0x00,0x00,0x76,0x02,0x00,0x00,0x78,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0x73,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x73,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0xc1,0x00,0x00,0x00, +0x7a,0x02,0x00,0x00,0x71,0x02,0x00,0x00,0x66,0x02,0x00,0x00, +0x79,0x02,0x00,0x00,0x72,0x02,0x00,0x00,0xf7,0x00,0x03,0x00, +0x7c,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x7a,0x02,0x00,0x00,0x7b,0x02,0x00,0x00,0x7c,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x7b,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x84,0x02,0x00,0x00,0x5b,0x02,0x00,0x00, +0xaa,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, +0x86,0x02,0x00,0x00,0x12,0x00,0x00,0x00,0x85,0x02,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x87,0x02,0x00,0x00, +0x86,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x88,0x02,0x00,0x00,0x84,0x02,0x00,0x00,0x87,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x89,0x02,0x00,0x00, +0x3b,0x02,0x00,0x00,0x88,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x8b,0x02,0x00,0x00,0x89,0x02,0x00,0x00, +0x53,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x8d,0x02,0x00,0x00,0x8b,0x02,0x00,0x00,0xac,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8f,0x02,0x00,0x00, +0xa7,0x02,0x00,0x00,0xbb,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x91,0x02,0x00,0x00,0x8f,0x02,0x00,0x00, +0xaa,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x93,0x02,0x00,0x00,0x91,0x02,0x00,0x00,0x92,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x95,0x02,0x00,0x00, +0xa8,0x02,0x00,0x00,0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x96,0x02,0x00,0x00,0x93,0x02,0x00,0x00, +0x95,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x98,0x02,0x00,0x00,0x96,0x02,0x00,0x00,0xac,0x02,0x00,0x00, +0x41,0x00,0x05,0x00,0xcc,0x00,0x00,0x00,0x99,0x02,0x00,0x00, +0xc9,0x00,0x00,0x00,0x98,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, +0xc3,0x00,0x00,0x00,0x9a,0x02,0x00,0x00,0x99,0x02,0x00,0x00, +0x41,0x00,0x06,0x00,0x9b,0x02,0x00,0x00,0x9c,0x02,0x00,0x00, +0x80,0x02,0x00,0x00,0x34,0x00,0x00,0x00,0x8d,0x02,0x00,0x00, +0x3e,0x00,0x03,0x00,0x9c,0x02,0x00,0x00,0x9a,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0x7c,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x7c,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x68,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x68,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x9e,0x02,0x00,0x00,0xac,0x02,0x00,0x00, +0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x65,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x67,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0x60,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x60,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa0,0x02,0x00,0x00, +0xaa,0x02,0x00,0x00,0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x5d,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x5f,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0x48,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x48,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xa2,0x02,0x00,0x00,0xa8,0x02,0x00,0x00,0xcf,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x45,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x47,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x40,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x40,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xa4,0x02,0x00,0x00,0xa7,0x02,0x00,0x00, +0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x3d,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x3f,0x02,0x00,0x00,0xfd,0x00,0x01,0x00, +0x38,0x00,0x01,0x00, +}; +const uint64_t matmul_f16_fp32_len = 10276; + +unsigned char matmul_f32_data[] = { +0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, +0xda,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, +0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x09,0x00,0x00,0x00, +0x0b,0x00,0x06,0x00,0x01,0x00,0x00,0x00,0x47,0x4c,0x53,0x4c, +0x2e,0x73,0x74,0x64,0x2e,0x34,0x35,0x30,0x00,0x00,0x00,0x00, +0x0e,0x00,0x03,0x00,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x0f,0x00,0x0f,0x00,0x05,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x6d,0x61,0x69,0x6e,0x00,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, +0x12,0x00,0x00,0x00,0x3d,0x00,0x00,0x00,0x4c,0x00,0x00,0x00, +0xfb,0x00,0x00,0x00,0x06,0x01,0x00,0x00,0x47,0x01,0x00,0x00, +0x52,0x01,0x00,0x00,0x3b,0x02,0x00,0x00,0x84,0x02,0x00,0x00, +0x10,0x00,0x06,0x00,0x04,0x00,0x00,0x00,0x11,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x0b,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, +0x1c,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x10,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x08,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, +0x03,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x10,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x14,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x18,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x07,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x10,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, +0x09,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x24,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x0a,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x10,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x2c,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x30,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x0d,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0x47,0x00,0x03,0x00, +0x10,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x37,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x3d,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, +0x1a,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x4c,0x00,0x00,0x00, +0x0b,0x00,0x00,0x00,0x1b,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x4f,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x09,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x53,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x60,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x62,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x07,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x6c,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x03,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xa6,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0xb8,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x05,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0xbb,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x08,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x03,0x01,0x00,0x00, +0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00, +0x04,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x04,0x01,0x00,0x00,0x00,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, +0x04,0x01,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x06,0x01,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x06,0x01,0x00,0x00,0x21,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x21,0x01,0x00,0x00, +0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x22,0x01,0x00,0x00,0x0b,0x00,0x00,0x00,0x19,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x4f,0x01,0x00,0x00,0x06,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0x50,0x01,0x00,0x00, +0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x50,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x50,0x01,0x00,0x00, +0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x52,0x01,0x00,0x00, +0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x52,0x01,0x00,0x00,0x21,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x3b,0x02,0x00,0x00,0x0b,0x00,0x00,0x00, +0x18,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x81,0x02,0x00,0x00, +0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00, +0x82,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x19,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x82,0x02,0x00,0x00,0x00,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, +0x82,0x02,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x84,0x02,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x84,0x02,0x00,0x00,0x21,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x13,0x00,0x02,0x00,0x02,0x00,0x00,0x00, +0x21,0x00,0x03,0x00,0x03,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x15,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x17,0x00,0x04,0x00,0x09,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x0a,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x09,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x0d,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x1e,0x00,0x10,0x00,0x10,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x11,0x00,0x00,0x00, +0x09,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0x11,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x09,0x00,0x00,0x00, +0x15,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, +0x14,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x15,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x21,0x00,0x00,0x00, +0x0a,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, +0x27,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0x2d,0x00,0x00,0x00,0x07,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x34,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x37,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00,0x3d,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x3e,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0x0a,0x00,0x00,0x00,0x4c,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x4f,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x53,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x54,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0x37,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x58,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0x37,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x32,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x61,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x60,0x00,0x00,0x00, +0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x62,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x63,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x61,0x00,0x00,0x00, +0x62,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x67,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x61,0x00,0x00,0x00, +0x62,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x72,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x77,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x7c,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0x80,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x85,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, +0x90,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0x96,0x00,0x00,0x00,0x03,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0xa1,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xa6,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0xa8,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xb7,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00, +0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xb8,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xb9,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x53,0x00,0x00,0x00, +0xb8,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xba,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x4f,0x00,0x00,0x00, +0x62,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xbb,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xbc,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0xba,0x00,0x00,0x00,0xbb,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xbd,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0xbc,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xbe,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0xb9,0x00,0x00,0x00,0xbd,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xbf,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0xb7,0x00,0x00,0x00,0xbe,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xc0,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0xbf,0x00,0x00,0x00,0xbb,0x00,0x00,0x00,0x14,0x00,0x02,0x00, +0xc1,0x00,0x00,0x00,0x16,0x00,0x03,0x00,0xc3,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xc4,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00, +0x62,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xc5,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0xc4,0x00,0x00,0x00, +0xbe,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xc6,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0xc5,0x00,0x00,0x00, +0xbb,0x00,0x00,0x00,0x1c,0x00,0x04,0x00,0xc7,0x00,0x00,0x00, +0xc3,0x00,0x00,0x00,0xc6,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0xc8,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0xc7,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0xc3,0x00,0x00,0x00,0xcb,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0xcc,0x00,0x00,0x00, +0x07,0x00,0x00,0x00,0xc3,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0xcf,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x16,0x00,0x03,0x00,0xf6,0x00,0x00,0x00,0x10,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xf7,0x00,0x00,0x00, +0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xf8,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0xf7,0x00,0x00,0x00, +0x1c,0x00,0x04,0x00,0xf9,0x00,0x00,0x00,0xf6,0x00,0x00,0x00, +0xf8,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0xfa,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0xf9,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0xfa,0x00,0x00,0x00,0xfb,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xff,0x00,0x00,0x00, +0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0x1d,0x00,0x03,0x00,0x03,0x01,0x00,0x00,0xc3,0x00,0x00,0x00, +0x1e,0x00,0x03,0x00,0x04,0x01,0x00,0x00,0x03,0x01,0x00,0x00, +0x20,0x00,0x04,0x00,0x05,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, +0x04,0x01,0x00,0x00,0x3b,0x00,0x04,0x00,0x05,0x01,0x00,0x00, +0x06,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x11,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0xc3,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x15,0x01,0x00,0x00,0x04,0x00,0x00,0x00, +0xf6,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x1b,0x01,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, +0x39,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0xf6,0x00,0x00,0x00, +0x1f,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x32,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x21,0x01,0x00,0x00,0x01,0x00,0x00,0x00, +0x33,0x00,0x06,0x00,0x09,0x00,0x00,0x00,0x22,0x01,0x00,0x00, +0x21,0x01,0x00,0x00,0x39,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x23,0x01,0x00,0x00, +0x51,0x00,0x00,0x00,0x22,0x01,0x00,0x00,0x00,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x24,0x01,0x00,0x00, +0x84,0x00,0x00,0x00,0x23,0x01,0x00,0x00,0x39,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x25,0x01,0x00,0x00, +0x86,0x00,0x00,0x00,0x24,0x01,0x00,0x00,0x6c,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x43,0x01,0x00,0x00, +0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x44,0x01,0x00,0x00, +0x84,0x00,0x00,0x00,0xa6,0x00,0x00,0x00,0x43,0x01,0x00,0x00, +0x1c,0x00,0x04,0x00,0x45,0x01,0x00,0x00,0xf6,0x00,0x00,0x00, +0x44,0x01,0x00,0x00,0x20,0x00,0x04,0x00,0x46,0x01,0x00,0x00, +0x04,0x00,0x00,0x00,0x45,0x01,0x00,0x00,0x3b,0x00,0x04,0x00, +0x46,0x01,0x00,0x00,0x47,0x01,0x00,0x00,0x04,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x4b,0x01,0x00,0x00, +0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0x1d,0x00,0x03,0x00,0x4f,0x01,0x00,0x00,0xc3,0x00,0x00,0x00, +0x1e,0x00,0x03,0x00,0x50,0x01,0x00,0x00,0x4f,0x01,0x00,0x00, +0x20,0x00,0x04,0x00,0x51,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, +0x50,0x01,0x00,0x00,0x3b,0x00,0x04,0x00,0x51,0x01,0x00,0x00, +0x52,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x65,0x01,0x00,0x00,0x80,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x6a,0x01,0x00,0x00,0x51,0x00,0x00,0x00, +0x22,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x6b,0x01,0x00,0x00,0x84,0x00,0x00,0x00, +0x6a,0x01,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x6c,0x01,0x00,0x00,0x86,0x00,0x00,0x00, +0x6b,0x01,0x00,0x00,0x6c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x6f,0x01,0x00,0x00,0x08,0x01,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x70,0x01,0x00,0x00, +0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x73,0x01,0x00,0x00, +0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x8e,0x01,0x00,0x00, +0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00, +0x1c,0x00,0x04,0x00,0x8f,0x01,0x00,0x00,0xf6,0x00,0x00,0x00, +0x8e,0x01,0x00,0x00,0x20,0x00,0x04,0x00,0x90,0x01,0x00,0x00, +0x07,0x00,0x00,0x00,0x8f,0x01,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xa0,0x01,0x00,0x00,0x80,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0xa6,0x01,0x00,0x00,0x07,0x00,0x00,0x00,0xf6,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xbc,0x01,0x00,0x00, +0x84,0x00,0x00,0x00,0xbe,0x00,0x00,0x00,0xbb,0x00,0x00,0x00, +0x1c,0x00,0x04,0x00,0xbd,0x01,0x00,0x00,0xf6,0x00,0x00,0x00, +0xbc,0x01,0x00,0x00,0x20,0x00,0x04,0x00,0xbe,0x01,0x00,0x00, +0x07,0x00,0x00,0x00,0xbd,0x01,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xc7,0x01,0x00,0x00,0x86,0x00,0x00,0x00, +0xb8,0x00,0x00,0x00,0xbe,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xcf,0x01,0x00,0x00,0x80,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xfe,0x01,0x00,0x00,0x84,0x00,0x00,0x00, +0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0x33,0x02,0x00,0x00,0x0d,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00,0x3b,0x02,0x00,0x00, +0x01,0x00,0x00,0x00,0x1d,0x00,0x03,0x00,0x81,0x02,0x00,0x00, +0xc3,0x00,0x00,0x00,0x1e,0x00,0x03,0x00,0x82,0x02,0x00,0x00, +0x81,0x02,0x00,0x00,0x20,0x00,0x04,0x00,0x83,0x02,0x00,0x00, +0x0c,0x00,0x00,0x00,0x82,0x02,0x00,0x00,0x3b,0x00,0x04,0x00, +0x83,0x02,0x00,0x00,0x84,0x02,0x00,0x00,0x0c,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x89,0x02,0x00,0x00, +0x05,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x96,0x02,0x00,0x00,0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00, +0x62,0x00,0x00,0x00,0x36,0x00,0x05,0x00,0x02,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x03,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0x05,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0xc8,0x00,0x00,0x00,0xc9,0x00,0x00,0x00,0x07,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0x90,0x01,0x00,0x00,0x91,0x01,0x00,0x00, +0x07,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0xbe,0x01,0x00,0x00, +0xbf,0x01,0x00,0x00,0x07,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x0d,0x00,0x00,0x00,0x0e,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x0f,0x00,0x00,0x00,0x0e,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x15,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0x14,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x17,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, +0x17,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x1e,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x17,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x22,0x00,0x00,0x00, +0x12,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x22,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x24,0x00,0x00,0x00, +0x18,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x15,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0x27,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x29,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x2a,0x00,0x00,0x00,0x1e,0x00,0x00,0x00, +0x29,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, +0x2e,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x2d,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x2f,0x00,0x00,0x00, +0x2e,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x30,0x00,0x00,0x00,0x24,0x00,0x00,0x00,0x2f,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x32,0x00,0x00,0x00, +0x30,0x00,0x00,0x00,0x2a,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x15,0x00,0x00,0x00,0x35,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0x34,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x36,0x00,0x00,0x00,0x35,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x38,0x00,0x00,0x00,0x36,0x00,0x00,0x00, +0x37,0x00,0x00,0x00,0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x3a,0x00,0x00,0x00,0x38,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3b,0x00,0x00,0x00, +0x3a,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x0d,0x00,0x00,0x00,0x3f,0x00,0x00,0x00,0x3d,0x00,0x00,0x00, +0x3e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x40,0x00,0x00,0x00,0x3f,0x00,0x00,0x00,0x89,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x42,0x00,0x00,0x00,0x40,0x00,0x00,0x00, +0x3b,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x47,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x3b,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00,0x49,0x00,0x00,0x00, +0x3d,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x4a,0x00,0x00,0x00,0x49,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00,0x4d,0x00,0x00,0x00, +0x4c,0x00,0x00,0x00,0x3e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x4d,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x50,0x00,0x00,0x00, +0x4e,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x89,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x55,0x00,0x00,0x00,0x50,0x00,0x00,0x00, +0x54,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x59,0x00,0x00,0x00,0x50,0x00,0x00,0x00,0x58,0x00,0x00,0x00, +0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x5d,0x00,0x00,0x00, +0x4e,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x89,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x64,0x00,0x00,0x00,0x5d,0x00,0x00,0x00, +0x63,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x68,0x00,0x00,0x00,0x5d,0x00,0x00,0x00,0x67,0x00,0x00,0x00, +0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x6e,0x00,0x00,0x00, +0x4e,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x73,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, +0x72,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x78,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x77,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x7d,0x00,0x00,0x00, +0x4e,0x00,0x00,0x00,0x7c,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x15,0x00,0x00,0x00,0x81,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0x80,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x82,0x00,0x00,0x00,0x81,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x83,0x00,0x00,0x00,0x47,0x00,0x00,0x00, +0x82,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x85,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x87,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x89,0x00,0x00,0x00,0x47,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8c,0x00,0x00,0x00, +0x89,0x00,0x00,0x00,0x82,0x00,0x00,0x00,0x0c,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0x8d,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x26,0x00,0x00,0x00,0x87,0x00,0x00,0x00,0x8c,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x91,0x00,0x00,0x00, +0x12,0x00,0x00,0x00,0x90,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x92,0x00,0x00,0x00,0x91,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x93,0x00,0x00,0x00, +0x32,0x00,0x00,0x00,0x92,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x95,0x00,0x00,0x00,0x42,0x00,0x00,0x00, +0x37,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, +0x97,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x96,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x98,0x00,0x00,0x00, +0x97,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x99,0x00,0x00,0x00,0x95,0x00,0x00,0x00,0x98,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9a,0x00,0x00,0x00, +0x93,0x00,0x00,0x00,0x99,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x9c,0x00,0x00,0x00,0x9a,0x00,0x00,0x00, +0x83,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x9d,0x00,0x00,0x00,0x9c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0xa2,0x00,0x00,0x00, +0x12,0x00,0x00,0x00,0xa1,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xa3,0x00,0x00,0x00,0xa2,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa4,0x00,0x00,0x00, +0x0f,0x00,0x00,0x00,0xa3,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xa7,0x00,0x00,0x00,0x4a,0x00,0x00,0x00, +0xa6,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, +0xa9,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0xa8,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xaa,0x00,0x00,0x00, +0xa9,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xab,0x00,0x00,0x00,0xa7,0x00,0x00,0x00,0xaa,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xac,0x00,0x00,0x00, +0xa4,0x00,0x00,0x00,0xab,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xae,0x00,0x00,0x00,0xac,0x00,0x00,0x00, +0x83,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xaf,0x00,0x00,0x00,0xae,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xb1,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xb1,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xa8,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0x05,0x00,0x00,0x00, +0xd0,0x00,0x00,0x00,0xb2,0x00,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc1,0x00,0x00,0x00,0xc2,0x00,0x00,0x00,0xa8,0x02,0x00,0x00, +0xc0,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xb3,0x00,0x00,0x00, +0xb2,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0xc2,0x00,0x00,0x00,0xb2,0x00,0x00,0x00,0xb3,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xb2,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0xcc,0x00,0x00,0x00,0xcd,0x00,0x00,0x00,0xc9,0x00,0x00,0x00, +0xa8,0x02,0x00,0x00,0x3e,0x00,0x03,0x00,0xcd,0x00,0x00,0x00, +0xcb,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xd0,0x00,0x00,0x00,0xa8,0x02,0x00,0x00,0xcf,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xb1,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xb3,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xd3,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xd3,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xc1,0x02,0x00,0x00,0xaf,0x00,0x00,0x00, +0xb3,0x00,0x00,0x00,0x75,0x01,0x00,0x00,0xd6,0x00,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xbd,0x02,0x00,0x00, +0x9d,0x00,0x00,0x00,0xb3,0x00,0x00,0x00,0x72,0x01,0x00,0x00, +0xd6,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xa9,0x02,0x00,0x00,0x83,0x00,0x00,0x00,0xb3,0x00,0x00,0x00, +0x23,0x02,0x00,0x00,0xd6,0x00,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc1,0x00,0x00,0x00,0xda,0x00,0x00,0x00,0xa9,0x02,0x00,0x00, +0x8d,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xd5,0x00,0x00,0x00, +0xd6,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0xda,0x00,0x00,0x00,0xd4,0x00,0x00,0x00,0xd5,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xd4,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xdc,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xdc,0x00,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xb9,0x02,0x00,0x00, +0x3e,0x00,0x00,0x00,0xd4,0x00,0x00,0x00,0x27,0x01,0x00,0x00, +0xdf,0x00,0x00,0x00,0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00, +0xe2,0x00,0x00,0x00,0xb9,0x02,0x00,0x00,0x37,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0xde,0x00,0x00,0x00,0xdf,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xe2,0x00,0x00,0x00, +0xdd,0x00,0x00,0x00,0xde,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xdd,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xe6,0x00,0x00,0x00,0x95,0x00,0x00,0x00,0x73,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xe8,0x00,0x00,0x00, +0xe6,0x00,0x00,0x00,0xb9,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc1,0x00,0x00,0x00,0xeb,0x00,0x00,0x00,0xe8,0x00,0x00,0x00, +0x36,0x00,0x00,0x00,0xf7,0x00,0x03,0x00,0xed,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xeb,0x00,0x00,0x00, +0xec,0x00,0x00,0x00,0xed,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xec,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xf0,0x00,0x00,0x00,0xa9,0x02,0x00,0x00,0x6e,0x00,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00,0xf2,0x00,0x00,0x00, +0xf0,0x00,0x00,0x00,0x8d,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xed,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xed,0x00,0x00,0x00, +0xf5,0x00,0x07,0x00,0xc1,0x00,0x00,0x00,0xf3,0x00,0x00,0x00, +0xeb,0x00,0x00,0x00,0xdd,0x00,0x00,0x00,0xf2,0x00,0x00,0x00, +0xec,0x00,0x00,0x00,0xf7,0x00,0x03,0x00,0xf5,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xf3,0x00,0x00,0x00, +0xf4,0x00,0x00,0x00,0x17,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xf4,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xfe,0x00,0x00,0x00,0x73,0x00,0x00,0x00,0xb9,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x00,0x01,0x00,0x00, +0xfe,0x00,0x00,0x00,0xff,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x02,0x01,0x00,0x00,0x00,0x01,0x00,0x00, +0x6e,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x0d,0x01,0x00,0x00,0xfe,0x00,0x00,0x00,0x98,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x0e,0x01,0x00,0x00, +0xbd,0x02,0x00,0x00,0x0d,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x10,0x01,0x00,0x00,0x0e,0x01,0x00,0x00, +0x6e,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x11,0x01,0x00,0x00, +0x12,0x01,0x00,0x00,0x06,0x01,0x00,0x00,0x34,0x00,0x00,0x00, +0x10,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0xc3,0x00,0x00,0x00, +0x13,0x01,0x00,0x00,0x12,0x01,0x00,0x00,0x73,0x00,0x04,0x00, +0xf6,0x00,0x00,0x00,0x14,0x01,0x00,0x00,0x13,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x01,0x00,0x00,0x16,0x01,0x00,0x00, +0xfb,0x00,0x00,0x00,0x02,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, +0x16,0x01,0x00,0x00,0x14,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0xf5,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x17,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x1a,0x01,0x00,0x00, +0x73,0x00,0x00,0x00,0xb9,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x1c,0x01,0x00,0x00,0x1a,0x01,0x00,0x00, +0x1b,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x1e,0x01,0x00,0x00,0x1c,0x01,0x00,0x00,0x6e,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x01,0x00,0x00,0x20,0x01,0x00,0x00, +0xfb,0x00,0x00,0x00,0x1e,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, +0x20,0x01,0x00,0x00,0x1f,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0xf5,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xf5,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xdf,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xdf,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x27,0x01,0x00,0x00,0xb9,0x02,0x00,0x00,0x25,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0xdc,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xde,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x29,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x29,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xba,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, +0xde,0x00,0x00,0x00,0x6e,0x01,0x00,0x00,0x2c,0x01,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00,0x2f,0x01,0x00,0x00, +0xba,0x02,0x00,0x00,0xa6,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0x2b,0x01,0x00,0x00,0x2c,0x01,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x2f,0x01,0x00,0x00,0x2a,0x01,0x00,0x00, +0x2b,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x2a,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x33,0x01,0x00,0x00, +0xa7,0x00,0x00,0x00,0x7d,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x35,0x01,0x00,0x00,0x33,0x01,0x00,0x00, +0xba,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, +0x36,0x01,0x00,0x00,0x12,0x00,0x00,0x00,0xcf,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x37,0x01,0x00,0x00, +0x36,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00, +0x38,0x01,0x00,0x00,0x35,0x01,0x00,0x00,0x37,0x01,0x00,0x00, +0xf7,0x00,0x03,0x00,0x3a,0x01,0x00,0x00,0x00,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x38,0x01,0x00,0x00,0x39,0x01,0x00,0x00, +0x3a,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x39,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3d,0x01,0x00,0x00, +0xa9,0x02,0x00,0x00,0x78,0x00,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc1,0x00,0x00,0x00,0x3f,0x01,0x00,0x00,0x3d,0x01,0x00,0x00, +0x8d,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x3a,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x3a,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, +0xc1,0x00,0x00,0x00,0x40,0x01,0x00,0x00,0x38,0x01,0x00,0x00, +0x2a,0x01,0x00,0x00,0x3f,0x01,0x00,0x00,0x39,0x01,0x00,0x00, +0xf7,0x00,0x03,0x00,0x42,0x01,0x00,0x00,0x00,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x40,0x01,0x00,0x00,0x41,0x01,0x00,0x00, +0x61,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x41,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x4a,0x01,0x00,0x00, +0x7d,0x00,0x00,0x00,0xba,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x4c,0x01,0x00,0x00,0x4a,0x01,0x00,0x00, +0x4b,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x4e,0x01,0x00,0x00,0x4c,0x01,0x00,0x00,0x78,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x59,0x01,0x00,0x00, +0x4a,0x01,0x00,0x00,0xaa,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x5a,0x01,0x00,0x00,0xc1,0x02,0x00,0x00, +0x59,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x5c,0x01,0x00,0x00,0x5a,0x01,0x00,0x00,0x78,0x00,0x00,0x00, +0x41,0x00,0x06,0x00,0x11,0x01,0x00,0x00,0x5d,0x01,0x00,0x00, +0x52,0x01,0x00,0x00,0x34,0x00,0x00,0x00,0x5c,0x01,0x00,0x00, +0x3d,0x00,0x04,0x00,0xc3,0x00,0x00,0x00,0x5e,0x01,0x00,0x00, +0x5d,0x01,0x00,0x00,0x73,0x00,0x04,0x00,0xf6,0x00,0x00,0x00, +0x5f,0x01,0x00,0x00,0x5e,0x01,0x00,0x00,0x41,0x00,0x05,0x00, +0x15,0x01,0x00,0x00,0x60,0x01,0x00,0x00,0x47,0x01,0x00,0x00, +0x4e,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x60,0x01,0x00,0x00, +0x5f,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x42,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x61,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x64,0x01,0x00,0x00,0x7d,0x00,0x00,0x00, +0xba,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x66,0x01,0x00,0x00,0x64,0x01,0x00,0x00,0x65,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x68,0x01,0x00,0x00, +0x66,0x01,0x00,0x00,0x78,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x15,0x01,0x00,0x00,0x69,0x01,0x00,0x00,0x47,0x01,0x00,0x00, +0x68,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x69,0x01,0x00,0x00, +0x1f,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x42,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x42,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0x2c,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x2c,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x6e,0x01,0x00,0x00, +0xba,0x02,0x00,0x00,0x6c,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0x29,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x2b,0x01,0x00,0x00, +0xe0,0x00,0x04,0x00,0x0c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x6f,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x72,0x01,0x00,0x00,0xbd,0x02,0x00,0x00,0x70,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x75,0x01,0x00,0x00, +0xc1,0x02,0x00,0x00,0x73,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0x77,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x77,0x01,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xc3,0x02,0x00,0x00, +0x3e,0x00,0x00,0x00,0x2b,0x01,0x00,0x00,0x21,0x02,0x00,0x00, +0x7a,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00, +0x7d,0x01,0x00,0x00,0xc3,0x02,0x00,0x00,0x6c,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0x79,0x01,0x00,0x00,0x7a,0x01,0x00,0x00, +0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x7d,0x01,0x00,0x00, +0x78,0x01,0x00,0x00,0x79,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x78,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x7f,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x7f,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xc7,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, +0x78,0x01,0x00,0x00,0xab,0x01,0x00,0x00,0x82,0x01,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00,0x85,0x01,0x00,0x00, +0xc7,0x02,0x00,0x00,0x60,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0x81,0x01,0x00,0x00,0x82,0x01,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x85,0x01,0x00,0x00,0x80,0x01,0x00,0x00, +0x81,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x80,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0x87,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x87,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xd9,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0x80,0x01,0x00,0x00, +0xa9,0x01,0x00,0x00,0x88,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc1,0x00,0x00,0x00,0x8d,0x01,0x00,0x00,0xd9,0x02,0x00,0x00, +0x62,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x89,0x01,0x00,0x00, +0x88,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x8d,0x01,0x00,0x00,0x88,0x01,0x00,0x00,0x89,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x88,0x01,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x93,0x01,0x00,0x00,0xc7,0x02,0x00,0x00, +0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x95,0x01,0x00,0x00,0x93,0x01,0x00,0x00,0xd9,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x97,0x01,0x00,0x00, +0x55,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x99,0x01,0x00,0x00,0xc7,0x02,0x00,0x00, +0x61,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x9a,0x01,0x00,0x00,0x97,0x01,0x00,0x00,0x99,0x01,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9c,0x01,0x00,0x00, +0x64,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x9d,0x01,0x00,0x00,0x9a,0x01,0x00,0x00, +0x9c,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x9f,0x01,0x00,0x00,0x9d,0x01,0x00,0x00,0xd9,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa1,0x01,0x00,0x00, +0x9f,0x01,0x00,0x00,0xa0,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xa3,0x01,0x00,0x00,0xa1,0x01,0x00,0x00, +0xc3,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x01,0x00,0x00, +0xa4,0x01,0x00,0x00,0xfb,0x00,0x00,0x00,0xa3,0x01,0x00,0x00, +0x3d,0x00,0x04,0x00,0xf6,0x00,0x00,0x00,0xa5,0x01,0x00,0x00, +0xa4,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0xa6,0x01,0x00,0x00, +0xa7,0x01,0x00,0x00,0x91,0x01,0x00,0x00,0x95,0x01,0x00,0x00, +0x3e,0x00,0x03,0x00,0xa7,0x01,0x00,0x00,0xa5,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa9,0x01,0x00,0x00, +0xd9,0x02,0x00,0x00,0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x87,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x89,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0x82,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x82,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xab,0x01,0x00,0x00,0xc7,0x02,0x00,0x00,0xcf,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x7f,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x81,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xad,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xad,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xc8,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, +0x81,0x01,0x00,0x00,0xd9,0x01,0x00,0x00,0xb0,0x01,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00,0xb3,0x01,0x00,0x00, +0xc8,0x02,0x00,0x00,0xbe,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0xaf,0x01,0x00,0x00,0xb0,0x01,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0xb3,0x01,0x00,0x00,0xae,0x01,0x00,0x00, +0xaf,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xae,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0xb5,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xb5,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xd6,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0xae,0x01,0x00,0x00, +0xd7,0x01,0x00,0x00,0xb6,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc1,0x00,0x00,0x00,0xbb,0x01,0x00,0x00,0xd6,0x02,0x00,0x00, +0xbb,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xb7,0x01,0x00,0x00, +0xb6,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0xbb,0x01,0x00,0x00,0xb6,0x01,0x00,0x00,0xb7,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xb6,0x01,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xc1,0x01,0x00,0x00,0xc8,0x02,0x00,0x00, +0xbb,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xc3,0x01,0x00,0x00,0xc1,0x01,0x00,0x00,0xd6,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xc5,0x01,0x00,0x00, +0x59,0x00,0x00,0x00,0xb8,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xc8,0x01,0x00,0x00,0xc8,0x02,0x00,0x00, +0xc7,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xc9,0x01,0x00,0x00,0xc5,0x01,0x00,0x00,0xc8,0x01,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xcb,0x01,0x00,0x00, +0x68,0x00,0x00,0x00,0xbb,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xcc,0x01,0x00,0x00,0xc9,0x01,0x00,0x00, +0xcb,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xce,0x01,0x00,0x00,0xcc,0x01,0x00,0x00,0xd6,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xd0,0x01,0x00,0x00, +0xce,0x01,0x00,0x00,0xcf,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xd2,0x01,0x00,0x00,0xd0,0x01,0x00,0x00, +0xc3,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x01,0x00,0x00, +0xd3,0x01,0x00,0x00,0x47,0x01,0x00,0x00,0xd2,0x01,0x00,0x00, +0x3d,0x00,0x04,0x00,0xf6,0x00,0x00,0x00,0xd4,0x01,0x00,0x00, +0xd3,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0xa6,0x01,0x00,0x00, +0xd5,0x01,0x00,0x00,0xbf,0x01,0x00,0x00,0xc3,0x01,0x00,0x00, +0x3e,0x00,0x03,0x00,0xd5,0x01,0x00,0x00,0xd4,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xd7,0x01,0x00,0x00, +0xd6,0x02,0x00,0x00,0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xb5,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xb7,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0xb0,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xb0,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xd9,0x01,0x00,0x00,0xc8,0x02,0x00,0x00,0xcf,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xad,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xaf,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xdb,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xdb,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xc9,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, +0xaf,0x01,0x00,0x00,0x1f,0x02,0x00,0x00,0xde,0x01,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00,0xe1,0x01,0x00,0x00, +0xc9,0x02,0x00,0x00,0xbe,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0xdd,0x01,0x00,0x00,0xde,0x01,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0xe1,0x01,0x00,0x00,0xdc,0x01,0x00,0x00, +0xdd,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xdc,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0xe3,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xe3,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xcd,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0xdc,0x01,0x00,0x00, +0x1d,0x02,0x00,0x00,0xe6,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc1,0x00,0x00,0x00,0xe9,0x01,0x00,0x00,0xcd,0x02,0x00,0x00, +0x60,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xe5,0x01,0x00,0x00, +0xe6,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0xe9,0x01,0x00,0x00,0xe4,0x01,0x00,0x00,0xe5,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xe4,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0xeb,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xeb,0x01,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xcf,0x02,0x00,0x00, +0x3e,0x00,0x00,0x00,0xe4,0x01,0x00,0x00,0x1b,0x02,0x00,0x00, +0xee,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00, +0xf1,0x01,0x00,0x00,0xcf,0x02,0x00,0x00,0xbb,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0xed,0x01,0x00,0x00,0xee,0x01,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xf1,0x01,0x00,0x00, +0xec,0x01,0x00,0x00,0xed,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xec,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xf3,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xf3,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xd1,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, +0xec,0x01,0x00,0x00,0x19,0x02,0x00,0x00,0xf4,0x01,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00,0xf9,0x01,0x00,0x00, +0xd1,0x02,0x00,0x00,0x62,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0xf5,0x01,0x00,0x00,0xf4,0x01,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0xf9,0x01,0x00,0x00,0xf4,0x01,0x00,0x00, +0xf5,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xf4,0x01,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xfb,0x01,0x00,0x00, +0xc9,0x02,0x00,0x00,0xbb,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xfd,0x01,0x00,0x00,0xfb,0x01,0x00,0x00, +0xcf,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xff,0x01,0x00,0x00,0xfd,0x01,0x00,0x00,0xfe,0x01,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x01,0x02,0x00,0x00, +0xcd,0x02,0x00,0x00,0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x02,0x02,0x00,0x00,0xff,0x01,0x00,0x00, +0x01,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x04,0x02,0x00,0x00,0x02,0x02,0x00,0x00,0xd1,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x08,0x02,0x00,0x00, +0x01,0x02,0x00,0x00,0xd1,0x02,0x00,0x00,0x41,0x00,0x05,0x00, +0xa6,0x01,0x00,0x00,0x09,0x02,0x00,0x00,0x91,0x01,0x00,0x00, +0x08,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0xf6,0x00,0x00,0x00, +0x0a,0x02,0x00,0x00,0x09,0x02,0x00,0x00,0x73,0x00,0x04,0x00, +0xc3,0x00,0x00,0x00,0x0b,0x02,0x00,0x00,0x0a,0x02,0x00,0x00, +0x41,0x00,0x05,0x00,0xa6,0x01,0x00,0x00,0x10,0x02,0x00,0x00, +0xbf,0x01,0x00,0x00,0xfd,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, +0xf6,0x00,0x00,0x00,0x11,0x02,0x00,0x00,0x10,0x02,0x00,0x00, +0x73,0x00,0x04,0x00,0xc3,0x00,0x00,0x00,0x12,0x02,0x00,0x00, +0x11,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0xcc,0x00,0x00,0x00, +0x14,0x02,0x00,0x00,0xc9,0x00,0x00,0x00,0x04,0x02,0x00,0x00, +0x3d,0x00,0x04,0x00,0xc3,0x00,0x00,0x00,0x15,0x02,0x00,0x00, +0x14,0x02,0x00,0x00,0x0c,0x00,0x08,0x00,0xc3,0x00,0x00,0x00, +0x16,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00, +0x0b,0x02,0x00,0x00,0x12,0x02,0x00,0x00,0x15,0x02,0x00,0x00, +0x3e,0x00,0x03,0x00,0x14,0x02,0x00,0x00,0x16,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x19,0x02,0x00,0x00, +0xd1,0x02,0x00,0x00,0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xf3,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xf5,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0xee,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xee,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x1b,0x02,0x00,0x00,0xcf,0x02,0x00,0x00,0xcf,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xeb,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xed,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xe6,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xe6,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x1d,0x02,0x00,0x00,0xcd,0x02,0x00,0x00, +0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xe3,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xe5,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0xde,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xde,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x1f,0x02,0x00,0x00, +0xc9,0x02,0x00,0x00,0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xdb,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xdd,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0x7a,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x7a,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x21,0x02,0x00,0x00,0xc3,0x02,0x00,0x00,0xcf,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x77,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x79,0x01,0x00,0x00,0xe0,0x00,0x04,0x00,0x0c,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x6f,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0xd6,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xd6,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x23,0x02,0x00,0x00, +0xa9,0x02,0x00,0x00,0x6c,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xd3,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xd5,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x28,0x02,0x00,0x00, +0x55,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x29,0x02,0x00,0x00,0x95,0x00,0x00,0x00, +0x28,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x2e,0x02,0x00,0x00,0x59,0x00,0x00,0x00,0xb8,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x2f,0x02,0x00,0x00, +0xa7,0x00,0x00,0x00,0x2e,0x02,0x00,0x00,0x41,0x00,0x05,0x00, +0x15,0x00,0x00,0x00,0x34,0x02,0x00,0x00,0x12,0x00,0x00,0x00, +0x33,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x35,0x02,0x00,0x00,0x34,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x36,0x02,0x00,0x00,0x0f,0x00,0x00,0x00, +0x35,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x3a,0x02,0x00,0x00,0x47,0x00,0x00,0x00,0x35,0x02,0x00,0x00, +0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00,0x3c,0x02,0x00,0x00, +0x3b,0x02,0x00,0x00,0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x3d,0x02,0x00,0x00,0x3c,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3e,0x02,0x00,0x00, +0x3a,0x02,0x00,0x00,0x3d,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x3f,0x02,0x00,0x00,0x36,0x02,0x00,0x00, +0x3e,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x41,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x41,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xaa,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, +0xd5,0x00,0x00,0x00,0xa7,0x02,0x00,0x00,0x44,0x02,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00,0x47,0x02,0x00,0x00, +0xaa,0x02,0x00,0x00,0xbe,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0x43,0x02,0x00,0x00,0x44,0x02,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x47,0x02,0x00,0x00,0x42,0x02,0x00,0x00, +0x43,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x42,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0x49,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x49,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xab,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0x42,0x02,0x00,0x00, +0xa5,0x02,0x00,0x00,0x4c,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc1,0x00,0x00,0x00,0x4f,0x02,0x00,0x00,0xab,0x02,0x00,0x00, +0x60,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x4b,0x02,0x00,0x00, +0x4c,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x4f,0x02,0x00,0x00,0x4a,0x02,0x00,0x00,0x4b,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x4a,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x53,0x02,0x00,0x00,0xab,0x02,0x00,0x00, +0x61,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x54,0x02,0x00,0x00,0x29,0x02,0x00,0x00,0x53,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x56,0x02,0x00,0x00, +0x64,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x57,0x02,0x00,0x00,0x54,0x02,0x00,0x00, +0x56,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x5b,0x02,0x00,0x00,0xaa,0x02,0x00,0x00,0xc7,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x5c,0x02,0x00,0x00, +0x2f,0x02,0x00,0x00,0x5b,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x5e,0x02,0x00,0x00,0x68,0x00,0x00,0x00, +0xbb,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x5f,0x02,0x00,0x00,0x5c,0x02,0x00,0x00,0x5e,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0x61,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x61,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xad,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0x4a,0x02,0x00,0x00, +0xa3,0x02,0x00,0x00,0x64,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc1,0x00,0x00,0x00,0x67,0x02,0x00,0x00,0xad,0x02,0x00,0x00, +0xbb,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x63,0x02,0x00,0x00, +0x64,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x67,0x02,0x00,0x00,0x62,0x02,0x00,0x00,0x63,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x62,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0x69,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x69,0x02,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xaf,0x02,0x00,0x00, +0x3e,0x00,0x00,0x00,0x62,0x02,0x00,0x00,0xa1,0x02,0x00,0x00, +0x6c,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00, +0x6f,0x02,0x00,0x00,0xaf,0x02,0x00,0x00,0x62,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0x6b,0x02,0x00,0x00,0x6c,0x02,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x6f,0x02,0x00,0x00, +0x6a,0x02,0x00,0x00,0x6b,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x6a,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x72,0x02,0x00,0x00,0x57,0x02,0x00,0x00,0xaf,0x02,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00,0x75,0x02,0x00,0x00, +0x72,0x02,0x00,0x00,0x36,0x00,0x00,0x00,0xf7,0x00,0x03,0x00, +0x77,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x75,0x02,0x00,0x00,0x76,0x02,0x00,0x00,0x77,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x76,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x7a,0x02,0x00,0x00,0x5f,0x02,0x00,0x00, +0xad,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, +0x7b,0x02,0x00,0x00,0x12,0x00,0x00,0x00,0xcf,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x7c,0x02,0x00,0x00, +0x7b,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00, +0x7d,0x02,0x00,0x00,0x7a,0x02,0x00,0x00,0x7c,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0x77,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x77,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0xc1,0x00,0x00,0x00, +0x7e,0x02,0x00,0x00,0x75,0x02,0x00,0x00,0x6a,0x02,0x00,0x00, +0x7d,0x02,0x00,0x00,0x76,0x02,0x00,0x00,0xf7,0x00,0x03,0x00, +0x80,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x7e,0x02,0x00,0x00,0x7f,0x02,0x00,0x00,0x80,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x7f,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x88,0x02,0x00,0x00,0x5f,0x02,0x00,0x00, +0xad,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, +0x8a,0x02,0x00,0x00,0x12,0x00,0x00,0x00,0x89,0x02,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x8b,0x02,0x00,0x00, +0x8a,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x8c,0x02,0x00,0x00,0x88,0x02,0x00,0x00,0x8b,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8d,0x02,0x00,0x00, +0x3f,0x02,0x00,0x00,0x8c,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x8f,0x02,0x00,0x00,0x8d,0x02,0x00,0x00, +0x57,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x91,0x02,0x00,0x00,0x8f,0x02,0x00,0x00,0xaf,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x93,0x02,0x00,0x00, +0xaa,0x02,0x00,0x00,0xbb,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x95,0x02,0x00,0x00,0x93,0x02,0x00,0x00, +0xad,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x97,0x02,0x00,0x00,0x95,0x02,0x00,0x00,0x96,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x99,0x02,0x00,0x00, +0xab,0x02,0x00,0x00,0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x9a,0x02,0x00,0x00,0x97,0x02,0x00,0x00, +0x99,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x9c,0x02,0x00,0x00,0x9a,0x02,0x00,0x00,0xaf,0x02,0x00,0x00, +0x41,0x00,0x05,0x00,0xcc,0x00,0x00,0x00,0x9d,0x02,0x00,0x00, +0xc9,0x00,0x00,0x00,0x9c,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, +0xc3,0x00,0x00,0x00,0x9e,0x02,0x00,0x00,0x9d,0x02,0x00,0x00, +0x41,0x00,0x06,0x00,0x11,0x01,0x00,0x00,0x9f,0x02,0x00,0x00, +0x84,0x02,0x00,0x00,0x34,0x00,0x00,0x00,0x91,0x02,0x00,0x00, +0x3e,0x00,0x03,0x00,0x9f,0x02,0x00,0x00,0x9e,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0x80,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x80,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x6c,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x6c,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xa1,0x02,0x00,0x00,0xaf,0x02,0x00,0x00, +0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x69,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x6b,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0x64,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x64,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa3,0x02,0x00,0x00, +0xad,0x02,0x00,0x00,0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x61,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x63,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0x4c,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x4c,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xa5,0x02,0x00,0x00,0xab,0x02,0x00,0x00,0xcf,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x49,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x4b,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x44,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x44,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xa7,0x02,0x00,0x00,0xaa,0x02,0x00,0x00, +0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x41,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x43,0x02,0x00,0x00,0xfd,0x00,0x01,0x00, +0x38,0x00,0x01,0x00, +}; +const uint64_t matmul_f32_len = 10324; + +unsigned char matmul_f32_aligned_data[] = { +0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, +0x18,0x03,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, +0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x09,0x00,0x00,0x00, +0x0b,0x00,0x06,0x00,0x01,0x00,0x00,0x00,0x47,0x4c,0x53,0x4c, +0x2e,0x73,0x74,0x64,0x2e,0x34,0x35,0x30,0x00,0x00,0x00,0x00, +0x0e,0x00,0x03,0x00,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x0f,0x00,0x0f,0x00,0x05,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x6d,0x61,0x69,0x6e,0x00,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, +0x12,0x00,0x00,0x00,0x3d,0x00,0x00,0x00,0x4c,0x00,0x00,0x00, +0xfe,0x00,0x00,0x00,0x05,0x01,0x00,0x00,0x6b,0x01,0x00,0x00, +0x71,0x01,0x00,0x00,0x79,0x02,0x00,0x00,0xc2,0x02,0x00,0x00, +0x10,0x00,0x06,0x00,0x04,0x00,0x00,0x00,0x11,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x0b,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, +0x1c,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x10,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x08,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, +0x03,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x10,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x14,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x18,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x07,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x10,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, +0x09,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x24,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x0a,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x10,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x2c,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x30,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x0d,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0x47,0x00,0x03,0x00, +0x10,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x37,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x3d,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, +0x1a,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x4c,0x00,0x00,0x00, +0x0b,0x00,0x00,0x00,0x1b,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x4f,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x09,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x53,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x60,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x62,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x07,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x6c,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x03,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xa7,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0xb9,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x05,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0xbc,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x08,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x02,0x01,0x00,0x00, +0x06,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x48,0x00,0x04,0x00, +0x03,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x05,0x00,0x00,0x00, +0x48,0x00,0x04,0x00,0x03,0x01,0x00,0x00,0x00,0x00,0x00,0x00, +0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x03,0x01,0x00,0x00, +0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x03,0x01,0x00,0x00,0x00,0x00,0x00,0x00, +0x07,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x47,0x00,0x03,0x00, +0x03,0x01,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x05,0x01,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x05,0x01,0x00,0x00,0x21,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x43,0x01,0x00,0x00, +0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x44,0x01,0x00,0x00,0x0b,0x00,0x00,0x00,0x19,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x6e,0x01,0x00,0x00,0x06,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0x6f,0x01,0x00,0x00, +0x00,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0x48,0x00,0x04,0x00, +0x6f,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x6f,0x01,0x00,0x00,0x00,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x6f,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x07,0x00,0x00,0x00, +0x10,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x6f,0x01,0x00,0x00, +0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x71,0x01,0x00,0x00, +0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x71,0x01,0x00,0x00,0x21,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x79,0x02,0x00,0x00,0x0b,0x00,0x00,0x00, +0x18,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xbf,0x02,0x00,0x00, +0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00, +0xc0,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x19,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0xc0,0x02,0x00,0x00,0x00,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, +0xc0,0x02,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0xc2,0x02,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0xc2,0x02,0x00,0x00,0x21,0x00,0x00,0x00, 0x02,0x00,0x00,0x00,0x13,0x00,0x02,0x00,0x02,0x00,0x00,0x00, 0x21,0x00,0x03,0x00,0x03,0x00,0x00,0x00,0x02,0x00,0x00,0x00, 0x15,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x20,0x00,0x00,0x00, @@ -32306,169 +17388,151 @@ unsigned char matmul_f32_aligned_l_data[] = { 0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, 0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x73,0x00,0x00,0x00, 0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x77,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x78,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x7d,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x81,0x00,0x00,0x00, 0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, -0x7c,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0x87,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x8d,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0x91,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x97,0x00,0x00,0x00, 0x03,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, -0x98,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x32,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x9d,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x9f,0x00,0x00,0x00, +0xa2,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x32,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xa7,0x00,0x00,0x00,0x40,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0xa9,0x00,0x00,0x00, 0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xae,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00, +0xb8,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00, 0x62,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xaf,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xb0,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0x53,0x00,0x00,0x00,0xaf,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xb1,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0x4f,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x32,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xb2,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xb3,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0xb1,0x00,0x00,0x00,0xb2,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xb4,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0xb3,0x00,0x00,0x00,0x60,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xb5,0x00,0x00,0x00, -0x86,0x00,0x00,0x00,0xb0,0x00,0x00,0x00,0xb4,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xb6,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0xae,0x00,0x00,0x00,0xb5,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xb7,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0xb6,0x00,0x00,0x00,0xb2,0x00,0x00,0x00, -0x14,0x00,0x02,0x00,0xb8,0x00,0x00,0x00,0x16,0x00,0x03,0x00, -0xba,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0xb9,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xba,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0x53,0x00,0x00,0x00,0xb9,0x00,0x00,0x00,0x34,0x00,0x06,0x00, 0x06,0x00,0x00,0x00,0xbb,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0x4f,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x32,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xbc,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xbd,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0xbb,0x00,0x00,0x00,0xbc,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xbe,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0xbd,0x00,0x00,0x00,0x60,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xbf,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0xba,0x00,0x00,0x00,0xbe,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xc0,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0xb8,0x00,0x00,0x00,0xbf,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xc1,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0xc0,0x00,0x00,0x00,0xbc,0x00,0x00,0x00, +0x14,0x00,0x02,0x00,0xc2,0x00,0x00,0x00,0x16,0x00,0x03,0x00, +0xc4,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xc5,0x00,0x00,0x00,0x84,0x00,0x00,0x00, 0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xbc,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0xbb,0x00,0x00,0x00,0xb5,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xbd,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0xbc,0x00,0x00,0x00,0xb2,0x00,0x00,0x00,0x1c,0x00,0x04,0x00, -0xbe,0x00,0x00,0x00,0xba,0x00,0x00,0x00,0xbd,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0xbf,0x00,0x00,0x00,0x07,0x00,0x00,0x00, -0xbe,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0xba,0x00,0x00,0x00, -0xc2,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0xc3,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0xba,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0xc6,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x16,0x00,0x03,0x00,0xe6,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0xc6,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0xc5,0x00,0x00,0x00,0xbf,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xc7,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0xc6,0x00,0x00,0x00,0xbc,0x00,0x00,0x00,0x1c,0x00,0x04,0x00, +0xc8,0x00,0x00,0x00,0xc4,0x00,0x00,0x00,0xc7,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0xc9,0x00,0x00,0x00,0x07,0x00,0x00,0x00, +0xc8,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0xc4,0x00,0x00,0x00, +0xcc,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0xcd,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0xc4,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0xd0,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xf4,0x00,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, +0x39,0x00,0x00,0x00,0x16,0x00,0x03,0x00,0xf9,0x00,0x00,0x00, 0x10,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xe7,0x00,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, +0xfa,0x00,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, 0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xe8,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x37,0x00,0x00,0x00, -0xe7,0x00,0x00,0x00,0x1c,0x00,0x04,0x00,0xe9,0x00,0x00,0x00, -0xe6,0x00,0x00,0x00,0xe8,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0xea,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0xe9,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0xea,0x00,0x00,0x00,0xeb,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xef,0x00,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x17,0x00,0x04,0x00,0xf5,0x00,0x00,0x00, -0xba,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x18,0x00,0x04,0x00, -0xf6,0x00,0x00,0x00,0xf5,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x1d,0x00,0x03,0x00,0xf7,0x00,0x00,0x00,0xf6,0x00,0x00,0x00, -0x1e,0x00,0x03,0x00,0xf8,0x00,0x00,0x00,0xf7,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0xf9,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0xf8,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0xf9,0x00,0x00,0x00, -0xfa,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0xfc,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0xba,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x00,0x01,0x00,0x00,0x04,0x00,0x00,0x00, -0xe6,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x05,0x01,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x13,0x01,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x21,0x01,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x26,0x01,0x00,0x00,0x03,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x30,0x01,0x00,0x00,0x80,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x35,0x01,0x00,0x00,0x04,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x3f,0x01,0x00,0x00, -0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x44,0x01,0x00,0x00, -0x05,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x4e,0x01,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x53,0x01,0x00,0x00,0x06,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x5d,0x01,0x00,0x00,0x80,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x62,0x01,0x00,0x00,0x07,0x00,0x00,0x00, -0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x69,0x01,0x00,0x00, +0xfb,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x37,0x00,0x00,0x00, +0xfa,0x00,0x00,0x00,0x1c,0x00,0x04,0x00,0xfc,0x00,0x00,0x00, +0xf9,0x00,0x00,0x00,0xfb,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0xfd,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0xfc,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0xfd,0x00,0x00,0x00,0xfe,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x17,0x00,0x04,0x00,0x00,0x01,0x00,0x00, +0xc4,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x18,0x00,0x04,0x00, +0x01,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x02,0x00,0x00,0x00, +0x1d,0x00,0x03,0x00,0x02,0x01,0x00,0x00,0x01,0x01,0x00,0x00, +0x1e,0x00,0x03,0x00,0x03,0x01,0x00,0x00,0x02,0x01,0x00,0x00, +0x20,0x00,0x04,0x00,0x04,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, +0x03,0x01,0x00,0x00,0x3b,0x00,0x04,0x00,0x04,0x01,0x00,0x00, +0x05,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x07,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0xc4,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x0b,0x01,0x00,0x00,0x04,0x00,0x00,0x00, +0xf9,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x1c,0x01,0x00,0x00,0x03,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x24,0x01,0x00,0x00,0x04,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x2c,0x01,0x00,0x00, +0x05,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x34,0x01,0x00,0x00,0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x3c,0x01,0x00,0x00,0x07,0x00,0x00,0x00, +0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x43,0x01,0x00,0x00, 0x01,0x00,0x00,0x00,0x33,0x00,0x06,0x00,0x09,0x00,0x00,0x00, -0x6a,0x01,0x00,0x00,0x69,0x01,0x00,0x00,0x39,0x00,0x00,0x00, +0x44,0x01,0x00,0x00,0x43,0x01,0x00,0x00,0x39,0x00,0x00,0x00, 0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x6b,0x01,0x00,0x00,0x51,0x00,0x00,0x00,0x6a,0x01,0x00,0x00, +0x45,0x01,0x00,0x00,0x51,0x00,0x00,0x00,0x44,0x01,0x00,0x00, 0x00,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x6c,0x01,0x00,0x00,0x84,0x00,0x00,0x00,0x6b,0x01,0x00,0x00, +0x46,0x01,0x00,0x00,0x84,0x00,0x00,0x00,0x45,0x01,0x00,0x00, 0x6d,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x6d,0x01,0x00,0x00,0x86,0x00,0x00,0x00,0x6c,0x01,0x00,0x00, +0x47,0x01,0x00,0x00,0x86,0x00,0x00,0x00,0x46,0x01,0x00,0x00, 0x6c,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x84,0x01,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, +0x62,0x01,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, 0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x85,0x01,0x00,0x00,0x84,0x00,0x00,0x00,0x9d,0x00,0x00,0x00, -0x84,0x01,0x00,0x00,0x1c,0x00,0x04,0x00,0x86,0x01,0x00,0x00, -0xe6,0x00,0x00,0x00,0x85,0x01,0x00,0x00,0x20,0x00,0x04,0x00, -0x87,0x01,0x00,0x00,0x04,0x00,0x00,0x00,0x86,0x01,0x00,0x00, -0x3b,0x00,0x04,0x00,0x87,0x01,0x00,0x00,0x88,0x01,0x00,0x00, -0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x8c,0x01,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x1d,0x00,0x03,0x00,0x92,0x01,0x00,0x00, -0xf6,0x00,0x00,0x00,0x1e,0x00,0x03,0x00,0x93,0x01,0x00,0x00, -0x92,0x01,0x00,0x00,0x20,0x00,0x04,0x00,0x94,0x01,0x00,0x00, -0x0c,0x00,0x00,0x00,0x93,0x01,0x00,0x00,0x3b,0x00,0x04,0x00, -0x94,0x01,0x00,0x00,0x95,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x9e,0x01,0x00,0x00, -0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xac,0x01,0x00,0x00, -0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xba,0x01,0x00,0x00, -0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xc8,0x01,0x00,0x00, -0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xd6,0x01,0x00,0x00, -0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xe4,0x01,0x00,0x00, -0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xf2,0x01,0x00,0x00, -0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xff,0x01,0x00,0x00, +0x67,0x01,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, +0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x68,0x01,0x00,0x00,0x84,0x00,0x00,0x00,0xa7,0x00,0x00,0x00, +0x67,0x01,0x00,0x00,0x1c,0x00,0x04,0x00,0x69,0x01,0x00,0x00, +0xf9,0x00,0x00,0x00,0x68,0x01,0x00,0x00,0x20,0x00,0x04,0x00, +0x6a,0x01,0x00,0x00,0x04,0x00,0x00,0x00,0x69,0x01,0x00,0x00, +0x3b,0x00,0x04,0x00,0x6a,0x01,0x00,0x00,0x6b,0x01,0x00,0x00, +0x04,0x00,0x00,0x00,0x1d,0x00,0x03,0x00,0x6e,0x01,0x00,0x00, +0x01,0x01,0x00,0x00,0x1e,0x00,0x03,0x00,0x6f,0x01,0x00,0x00, +0x6e,0x01,0x00,0x00,0x20,0x00,0x04,0x00,0x70,0x01,0x00,0x00, +0x0c,0x00,0x00,0x00,0x6f,0x01,0x00,0x00,0x3b,0x00,0x04,0x00, +0x70,0x01,0x00,0x00,0x71,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xa8,0x01,0x00,0x00, +0x51,0x00,0x00,0x00,0x44,0x01,0x00,0x00,0x00,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xa9,0x01,0x00,0x00, +0x84,0x00,0x00,0x00,0xa8,0x01,0x00,0x00,0x6d,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xaa,0x01,0x00,0x00, +0x86,0x00,0x00,0x00,0xa9,0x01,0x00,0x00,0x6c,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xad,0x01,0x00,0x00, 0x08,0x01,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x00,0x02,0x00,0x00,0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, +0xae,0x01,0x00,0x00,0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, 0x6d,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x03,0x02,0x00,0x00,0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, +0xb1,0x01,0x00,0x00,0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, 0x6d,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x1e,0x02,0x00,0x00,0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x1c,0x00,0x04,0x00,0x1f,0x02,0x00,0x00, -0xe6,0x00,0x00,0x00,0x1e,0x02,0x00,0x00,0x20,0x00,0x04,0x00, -0x20,0x02,0x00,0x00,0x07,0x00,0x00,0x00,0x1f,0x02,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x30,0x02,0x00,0x00, +0xcc,0x01,0x00,0x00,0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00, +0x62,0x00,0x00,0x00,0x1c,0x00,0x04,0x00,0xcd,0x01,0x00,0x00, +0xf9,0x00,0x00,0x00,0xcc,0x01,0x00,0x00,0x20,0x00,0x04,0x00, +0xce,0x01,0x00,0x00,0x07,0x00,0x00,0x00,0xcd,0x01,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xde,0x01,0x00,0x00, 0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x36,0x02,0x00,0x00,0x07,0x00,0x00,0x00, -0xe6,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x4c,0x02,0x00,0x00,0x84,0x00,0x00,0x00,0xb5,0x00,0x00,0x00, -0xb2,0x00,0x00,0x00,0x1c,0x00,0x04,0x00,0x4d,0x02,0x00,0x00, -0xe6,0x00,0x00,0x00,0x4c,0x02,0x00,0x00,0x20,0x00,0x04,0x00, -0x4e,0x02,0x00,0x00,0x07,0x00,0x00,0x00,0x4d,0x02,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x57,0x02,0x00,0x00, -0x86,0x00,0x00,0x00,0xaf,0x00,0x00,0x00,0xb5,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x5f,0x02,0x00,0x00, +0x20,0x00,0x04,0x00,0xe4,0x01,0x00,0x00,0x07,0x00,0x00,0x00, +0xf9,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xfa,0x01,0x00,0x00,0x84,0x00,0x00,0x00,0xbf,0x00,0x00,0x00, +0xbc,0x00,0x00,0x00,0x1c,0x00,0x04,0x00,0xfb,0x01,0x00,0x00, +0xf9,0x00,0x00,0x00,0xfa,0x01,0x00,0x00,0x20,0x00,0x04,0x00, +0xfc,0x01,0x00,0x00,0x07,0x00,0x00,0x00,0xfb,0x01,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x05,0x02,0x00,0x00, +0x86,0x00,0x00,0x00,0xb9,0x00,0x00,0x00,0xbf,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x0d,0x02,0x00,0x00, 0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x8e,0x02,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x3c,0x02,0x00,0x00, 0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0xc3,0x02,0x00,0x00, +0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x71,0x02,0x00,0x00, 0x0d,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00, -0xcb,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, -0x11,0x03,0x00,0x00,0xba,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, -0x12,0x03,0x00,0x00,0x11,0x03,0x00,0x00,0x20,0x00,0x04,0x00, -0x13,0x03,0x00,0x00,0x0c,0x00,0x00,0x00,0x12,0x03,0x00,0x00, -0x3b,0x00,0x04,0x00,0x13,0x03,0x00,0x00,0x14,0x03,0x00,0x00, +0x79,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, +0xbf,0x02,0x00,0x00,0xc4,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, +0xc0,0x02,0x00,0x00,0xbf,0x02,0x00,0x00,0x20,0x00,0x04,0x00, +0xc1,0x02,0x00,0x00,0x0c,0x00,0x00,0x00,0xc0,0x02,0x00,0x00, +0x3b,0x00,0x04,0x00,0xc1,0x02,0x00,0x00,0xc2,0x02,0x00,0x00, 0x0c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, -0x19,0x03,0x00,0x00,0x05,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x26,0x03,0x00,0x00,0x84,0x00,0x00,0x00, +0xc7,0x02,0x00,0x00,0x05,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xd4,0x02,0x00,0x00,0x84,0x00,0x00,0x00, 0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x36,0x00,0x05,0x00, 0x02,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x00, 0x03,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x05,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0xbf,0x00,0x00,0x00,0xc0,0x00,0x00,0x00, -0x07,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x20,0x02,0x00,0x00, -0x21,0x02,0x00,0x00,0x07,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x4e,0x02,0x00,0x00,0x4f,0x02,0x00,0x00,0x07,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0xc9,0x00,0x00,0x00,0xca,0x00,0x00,0x00, +0x07,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0xce,0x01,0x00,0x00, +0xcf,0x01,0x00,0x00,0x07,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0xfc,0x01,0x00,0x00,0xfd,0x01,0x00,0x00,0x07,0x00,0x00,0x00, 0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00,0x0e,0x00,0x00,0x00, 0x0b,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, 0x06,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x0e,0x00,0x00,0x00, @@ -32527,672 +17591,628 @@ unsigned char matmul_f32_aligned_l_data[] = { 0x67,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, 0x6f,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x6e,0x00,0x00,0x00, 0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x74,0x00,0x00,0x00, -0x4e,0x00,0x00,0x00,0x73,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x78,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x77,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x79,0x00,0x00,0x00,0x78,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x7a,0x00,0x00,0x00,0x47,0x00,0x00,0x00, -0x79,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0x7d,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x7c,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x7e,0x00,0x00,0x00, -0x7d,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x80,0x00,0x00,0x00,0x47,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x83,0x00,0x00,0x00, -0x80,0x00,0x00,0x00,0x79,0x00,0x00,0x00,0x0c,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x26,0x00,0x00,0x00,0x7e,0x00,0x00,0x00,0x83,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x88,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x87,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x89,0x00,0x00,0x00,0x88,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8a,0x00,0x00,0x00, -0x32,0x00,0x00,0x00,0x89,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x8c,0x00,0x00,0x00,0x42,0x00,0x00,0x00, -0x37,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0x8e,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x8d,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x8f,0x00,0x00,0x00, -0x8e,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x90,0x00,0x00,0x00,0x8c,0x00,0x00,0x00,0x8f,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x91,0x00,0x00,0x00, -0x8a,0x00,0x00,0x00,0x90,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x93,0x00,0x00,0x00,0x91,0x00,0x00,0x00, -0x7a,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x94,0x00,0x00,0x00,0x93,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x99,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x98,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x9a,0x00,0x00,0x00,0x99,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9b,0x00,0x00,0x00, -0x0f,0x00,0x00,0x00,0x9a,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x9e,0x00,0x00,0x00,0x4a,0x00,0x00,0x00, -0x9d,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0xa0,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x9f,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xa1,0x00,0x00,0x00, -0xa0,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xa2,0x00,0x00,0x00,0x9e,0x00,0x00,0x00,0xa1,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa3,0x00,0x00,0x00, -0x9b,0x00,0x00,0x00,0xa2,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xa5,0x00,0x00,0x00,0xa3,0x00,0x00,0x00, -0x7a,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xa6,0x00,0x00,0x00,0xa5,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xa8,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xa8,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0x38,0x03,0x00,0x00,0x3e,0x00,0x00,0x00,0x05,0x00,0x00,0x00, -0xc7,0x00,0x00,0x00,0xa9,0x00,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb8,0x00,0x00,0x00,0xb9,0x00,0x00,0x00,0x38,0x03,0x00,0x00, -0xb7,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xaa,0x00,0x00,0x00, -0xa9,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0xb9,0x00,0x00,0x00,0xa9,0x00,0x00,0x00,0xaa,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xa9,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0xc3,0x00,0x00,0x00,0xc4,0x00,0x00,0x00,0xc0,0x00,0x00,0x00, -0x38,0x03,0x00,0x00,0x3e,0x00,0x03,0x00,0xc4,0x00,0x00,0x00, -0xc2,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xc7,0x00,0x00,0x00,0x38,0x03,0x00,0x00,0xc6,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xa8,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xaa,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xca,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xca,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0x51,0x03,0x00,0x00,0xa6,0x00,0x00,0x00, -0xaa,0x00,0x00,0x00,0x05,0x02,0x00,0x00,0xcd,0x00,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x4d,0x03,0x00,0x00, -0x94,0x00,0x00,0x00,0xaa,0x00,0x00,0x00,0x02,0x02,0x00,0x00, -0xcd,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0x39,0x03,0x00,0x00,0x7a,0x00,0x00,0x00,0xaa,0x00,0x00,0x00, -0xb3,0x02,0x00,0x00,0xcd,0x00,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb8,0x00,0x00,0x00,0xd1,0x00,0x00,0x00,0x39,0x03,0x00,0x00, -0x84,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xcc,0x00,0x00,0x00, -0xcd,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0xd1,0x00,0x00,0x00,0xcb,0x00,0x00,0x00,0xcc,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xcb,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xd3,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xd3,0x00,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x49,0x03,0x00,0x00, -0x3e,0x00,0x00,0x00,0xcb,0x00,0x00,0x00,0x6f,0x01,0x00,0x00, -0xd4,0x00,0x00,0x00,0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00, -0xd9,0x00,0x00,0x00,0x49,0x03,0x00,0x00,0x37,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0xd5,0x00,0x00,0x00,0xd4,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xd9,0x00,0x00,0x00, -0xd4,0x00,0x00,0x00,0xd5,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xd4,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xde,0x00,0x00,0x00,0x74,0x00,0x00,0x00,0x49,0x03,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xe1,0x00,0x00,0x00, -0xde,0x00,0x00,0x00,0x8f,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xe2,0x00,0x00,0x00,0xe1,0x00,0x00,0x00, +0x4e,0x00,0x00,0x00,0x73,0x00,0x00,0x00,0x89,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x79,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, +0x78,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x7e,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x7d,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x82,0x00,0x00,0x00, +0x12,0x00,0x00,0x00,0x81,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x83,0x00,0x00,0x00,0x82,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0x47,0x00,0x00,0x00,0x83,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x15,0x00,0x00,0x00,0x87,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x88,0x00,0x00,0x00,0x87,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x8a,0x00,0x00,0x00,0x47,0x00,0x00,0x00, +0x39,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x8d,0x00,0x00,0x00,0x8a,0x00,0x00,0x00,0x83,0x00,0x00,0x00, +0x0c,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x8e,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x26,0x00,0x00,0x00,0x88,0x00,0x00,0x00, +0x8d,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, +0x92,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x91,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x93,0x00,0x00,0x00, +0x92,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x94,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x93,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x96,0x00,0x00,0x00, +0x42,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x15,0x00,0x00,0x00,0x98,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0x97,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x99,0x00,0x00,0x00,0x98,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x9a,0x00,0x00,0x00,0x96,0x00,0x00,0x00, +0x99,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x9b,0x00,0x00,0x00,0x94,0x00,0x00,0x00,0x9a,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9d,0x00,0x00,0x00, +0x9b,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x9e,0x00,0x00,0x00,0x9d,0x00,0x00,0x00, +0x6d,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, +0xa3,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0xa2,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xa4,0x00,0x00,0x00, +0xa3,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xa5,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0xa4,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa8,0x00,0x00,0x00, +0x4a,0x00,0x00,0x00,0xa7,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x15,0x00,0x00,0x00,0xaa,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0xa9,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xab,0x00,0x00,0x00,0xaa,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xac,0x00,0x00,0x00,0xa8,0x00,0x00,0x00, +0xab,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xad,0x00,0x00,0x00,0xa5,0x00,0x00,0x00,0xac,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xaf,0x00,0x00,0x00, +0xad,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xb0,0x00,0x00,0x00,0xaf,0x00,0x00,0x00, +0x6d,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xb2,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xb2,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xe6,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, +0x05,0x00,0x00,0x00,0xd1,0x00,0x00,0x00,0xb3,0x00,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0xc3,0x00,0x00,0x00, +0xe6,0x02,0x00,0x00,0xc1,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0xb4,0x00,0x00,0x00,0xb3,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0xc3,0x00,0x00,0x00,0xb3,0x00,0x00,0x00, +0xb4,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xb3,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0xcd,0x00,0x00,0x00,0xce,0x00,0x00,0x00, +0xca,0x00,0x00,0x00,0xe6,0x02,0x00,0x00,0x3e,0x00,0x03,0x00, +0xce,0x00,0x00,0x00,0xcc,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xd1,0x00,0x00,0x00,0xe6,0x02,0x00,0x00, +0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xb2,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xb4,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xd4,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xd4,0x00,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xff,0x02,0x00,0x00, +0xb0,0x00,0x00,0x00,0xb4,0x00,0x00,0x00,0xb3,0x01,0x00,0x00, +0xd7,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xfb,0x02,0x00,0x00,0x9e,0x00,0x00,0x00,0xb4,0x00,0x00,0x00, +0xb0,0x01,0x00,0x00,0xd7,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xe7,0x02,0x00,0x00,0x84,0x00,0x00,0x00, +0xb4,0x00,0x00,0x00,0x61,0x02,0x00,0x00,0xd7,0x00,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0xdb,0x00,0x00,0x00, +0xe7,0x02,0x00,0x00,0x8e,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0xd6,0x00,0x00,0x00,0xd7,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0xdb,0x00,0x00,0x00,0xd5,0x00,0x00,0x00, +0xd6,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xd5,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xdd,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xdd,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xf7,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0xd5,0x00,0x00,0x00, +0x49,0x01,0x00,0x00,0xde,0x00,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc2,0x00,0x00,0x00,0xe3,0x00,0x00,0x00,0xf7,0x02,0x00,0x00, +0x37,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xdf,0x00,0x00,0x00, +0xde,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0xe3,0x00,0x00,0x00,0xde,0x00,0x00,0x00,0xdf,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xde,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xe8,0x00,0x00,0x00,0x74,0x00,0x00,0x00, +0xf7,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xeb,0x00,0x00,0x00,0xe8,0x00,0x00,0x00,0x99,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xec,0x00,0x00,0x00, +0xeb,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xed,0x00,0x00,0x00,0xfb,0x02,0x00,0x00, +0xec,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xef,0x00,0x00,0x00,0xed,0x00,0x00,0x00,0x6f,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xf5,0x00,0x00,0x00, +0xe8,0x00,0x00,0x00,0xf4,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xf7,0x00,0x00,0x00,0x6f,0x00,0x00,0x00, 0x6d,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xe3,0x00,0x00,0x00,0x4d,0x03,0x00,0x00,0xe2,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xe5,0x00,0x00,0x00, -0xe3,0x00,0x00,0x00,0x6f,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xf0,0x00,0x00,0x00,0xde,0x00,0x00,0x00, -0xef,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xf2,0x00,0x00,0x00,0x6f,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xf3,0x00,0x00,0x00, -0xf0,0x00,0x00,0x00,0xf2,0x00,0x00,0x00,0x41,0x00,0x08,0x00, -0xfc,0x00,0x00,0x00,0xfd,0x00,0x00,0x00,0xfa,0x00,0x00,0x00, -0x34,0x00,0x00,0x00,0xe5,0x00,0x00,0x00,0x34,0x00,0x00,0x00, -0x3e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0xba,0x00,0x00,0x00, -0xfe,0x00,0x00,0x00,0xfd,0x00,0x00,0x00,0x73,0x00,0x04,0x00, -0xe6,0x00,0x00,0x00,0xff,0x00,0x00,0x00,0xfe,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x00,0x01,0x00,0x00,0x01,0x01,0x00,0x00, -0xeb,0x00,0x00,0x00,0xf3,0x00,0x00,0x00,0x3e,0x00,0x03,0x00, -0x01,0x01,0x00,0x00,0xff,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x06,0x01,0x00,0x00,0xde,0x00,0x00,0x00, -0x05,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x09,0x01,0x00,0x00,0x06,0x01,0x00,0x00,0xf2,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x0a,0x01,0x00,0x00, -0x09,0x01,0x00,0x00,0x39,0x00,0x00,0x00,0x41,0x00,0x08,0x00, -0xfc,0x00,0x00,0x00,0x0c,0x01,0x00,0x00,0xfa,0x00,0x00,0x00, -0x34,0x00,0x00,0x00,0xe5,0x00,0x00,0x00,0x34,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0xba,0x00,0x00,0x00, -0x0d,0x01,0x00,0x00,0x0c,0x01,0x00,0x00,0x73,0x00,0x04,0x00, -0xe6,0x00,0x00,0x00,0x0e,0x01,0x00,0x00,0x0d,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0x00,0x01,0x00,0x00,0x0f,0x01,0x00,0x00, -0xeb,0x00,0x00,0x00,0x0a,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0x0f,0x01,0x00,0x00,0x0e,0x01,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x14,0x01,0x00,0x00,0xde,0x00,0x00,0x00, -0x13,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x17,0x01,0x00,0x00,0x14,0x01,0x00,0x00,0xf2,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x18,0x01,0x00,0x00, -0x17,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x41,0x00,0x08,0x00, -0xfc,0x00,0x00,0x00,0x1a,0x01,0x00,0x00,0xfa,0x00,0x00,0x00, -0x34,0x00,0x00,0x00,0xe5,0x00,0x00,0x00,0x34,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0xba,0x00,0x00,0x00, -0x1b,0x01,0x00,0x00,0x1a,0x01,0x00,0x00,0x73,0x00,0x04,0x00, -0xe6,0x00,0x00,0x00,0x1c,0x01,0x00,0x00,0x1b,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0x00,0x01,0x00,0x00,0x1d,0x01,0x00,0x00, -0xeb,0x00,0x00,0x00,0x18,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0x1d,0x01,0x00,0x00,0x1c,0x01,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x22,0x01,0x00,0x00,0xde,0x00,0x00,0x00, -0x21,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x25,0x01,0x00,0x00,0x22,0x01,0x00,0x00,0xf2,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x27,0x01,0x00,0x00, -0x25,0x01,0x00,0x00,0x26,0x01,0x00,0x00,0x41,0x00,0x08,0x00, -0xfc,0x00,0x00,0x00,0x29,0x01,0x00,0x00,0xfa,0x00,0x00,0x00, -0x34,0x00,0x00,0x00,0xe5,0x00,0x00,0x00,0x34,0x00,0x00,0x00, -0x26,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0xba,0x00,0x00,0x00, -0x2a,0x01,0x00,0x00,0x29,0x01,0x00,0x00,0x73,0x00,0x04,0x00, -0xe6,0x00,0x00,0x00,0x2b,0x01,0x00,0x00,0x2a,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0x00,0x01,0x00,0x00,0x2c,0x01,0x00,0x00, -0xeb,0x00,0x00,0x00,0x27,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0x2c,0x01,0x00,0x00,0x2b,0x01,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x31,0x01,0x00,0x00,0xde,0x00,0x00,0x00, -0x30,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x34,0x01,0x00,0x00,0x31,0x01,0x00,0x00,0xf2,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x36,0x01,0x00,0x00, -0x34,0x01,0x00,0x00,0x35,0x01,0x00,0x00,0x41,0x00,0x08,0x00, -0xfc,0x00,0x00,0x00,0x38,0x01,0x00,0x00,0xfa,0x00,0x00,0x00, -0x34,0x00,0x00,0x00,0xe5,0x00,0x00,0x00,0xc6,0x00,0x00,0x00, -0x3e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0xba,0x00,0x00,0x00, -0x39,0x01,0x00,0x00,0x38,0x01,0x00,0x00,0x73,0x00,0x04,0x00, -0xe6,0x00,0x00,0x00,0x3a,0x01,0x00,0x00,0x39,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0x00,0x01,0x00,0x00,0x3b,0x01,0x00,0x00, -0xeb,0x00,0x00,0x00,0x36,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0x3b,0x01,0x00,0x00,0x3a,0x01,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x40,0x01,0x00,0x00,0xde,0x00,0x00,0x00, -0x3f,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x43,0x01,0x00,0x00,0x40,0x01,0x00,0x00,0xf2,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x45,0x01,0x00,0x00, -0x43,0x01,0x00,0x00,0x44,0x01,0x00,0x00,0x41,0x00,0x08,0x00, -0xfc,0x00,0x00,0x00,0x47,0x01,0x00,0x00,0xfa,0x00,0x00,0x00, -0x34,0x00,0x00,0x00,0xe5,0x00,0x00,0x00,0xc6,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0xba,0x00,0x00,0x00, -0x48,0x01,0x00,0x00,0x47,0x01,0x00,0x00,0x73,0x00,0x04,0x00, -0xe6,0x00,0x00,0x00,0x49,0x01,0x00,0x00,0x48,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0x00,0x01,0x00,0x00,0x4a,0x01,0x00,0x00, -0xeb,0x00,0x00,0x00,0x45,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0x4a,0x01,0x00,0x00,0x49,0x01,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x4f,0x01,0x00,0x00,0xde,0x00,0x00,0x00, -0x4e,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x52,0x01,0x00,0x00,0x4f,0x01,0x00,0x00,0xf2,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x54,0x01,0x00,0x00, -0x52,0x01,0x00,0x00,0x53,0x01,0x00,0x00,0x41,0x00,0x08,0x00, -0xfc,0x00,0x00,0x00,0x56,0x01,0x00,0x00,0xfa,0x00,0x00,0x00, -0x34,0x00,0x00,0x00,0xe5,0x00,0x00,0x00,0xc6,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0xba,0x00,0x00,0x00, -0x57,0x01,0x00,0x00,0x56,0x01,0x00,0x00,0x73,0x00,0x04,0x00, -0xe6,0x00,0x00,0x00,0x58,0x01,0x00,0x00,0x57,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0x00,0x01,0x00,0x00,0x59,0x01,0x00,0x00, -0xeb,0x00,0x00,0x00,0x54,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0x59,0x01,0x00,0x00,0x58,0x01,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x5e,0x01,0x00,0x00,0xde,0x00,0x00,0x00, -0x5d,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x61,0x01,0x00,0x00,0x5e,0x01,0x00,0x00,0xf2,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x63,0x01,0x00,0x00, -0x61,0x01,0x00,0x00,0x62,0x01,0x00,0x00,0x41,0x00,0x08,0x00, -0xfc,0x00,0x00,0x00,0x65,0x01,0x00,0x00,0xfa,0x00,0x00,0x00, -0x34,0x00,0x00,0x00,0xe5,0x00,0x00,0x00,0xc6,0x00,0x00,0x00, -0x26,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0xba,0x00,0x00,0x00, -0x66,0x01,0x00,0x00,0x65,0x01,0x00,0x00,0x73,0x00,0x04,0x00, -0xe6,0x00,0x00,0x00,0x67,0x01,0x00,0x00,0x66,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0x00,0x01,0x00,0x00,0x68,0x01,0x00,0x00, -0xeb,0x00,0x00,0x00,0x63,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0x68,0x01,0x00,0x00,0x67,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x6f,0x01,0x00,0x00,0x49,0x03,0x00,0x00, -0x6d,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xd3,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xd5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x71,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x71,0x01,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x4a,0x03,0x00,0x00, -0x3e,0x00,0x00,0x00,0xd5,0x00,0x00,0x00,0xfe,0x01,0x00,0x00, -0x72,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00, -0x77,0x01,0x00,0x00,0x4a,0x03,0x00,0x00,0x9d,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0x73,0x01,0x00,0x00,0x72,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x77,0x01,0x00,0x00, -0x72,0x01,0x00,0x00,0x73,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x72,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x7c,0x01,0x00,0x00,0x74,0x00,0x00,0x00,0x4a,0x03,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x7f,0x01,0x00,0x00, -0x7c,0x01,0x00,0x00,0xa1,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x80,0x01,0x00,0x00,0x7f,0x01,0x00,0x00, +0xf8,0x00,0x00,0x00,0xf5,0x00,0x00,0x00,0xf7,0x00,0x00,0x00, +0x41,0x00,0x08,0x00,0x07,0x01,0x00,0x00,0x08,0x01,0x00,0x00, +0x05,0x01,0x00,0x00,0x34,0x00,0x00,0x00,0xef,0x00,0x00,0x00, +0x34,0x00,0x00,0x00,0x3e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0xc4,0x00,0x00,0x00,0x09,0x01,0x00,0x00,0x08,0x01,0x00,0x00, +0x73,0x00,0x04,0x00,0xf9,0x00,0x00,0x00,0x0a,0x01,0x00,0x00, +0x09,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0x0b,0x01,0x00,0x00, +0x0c,0x01,0x00,0x00,0xfe,0x00,0x00,0x00,0xf8,0x00,0x00,0x00, +0x3e,0x00,0x03,0x00,0x0c,0x01,0x00,0x00,0x0a,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x0e,0x01,0x00,0x00, +0xf8,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x41,0x00,0x08,0x00, +0x07,0x01,0x00,0x00,0x10,0x01,0x00,0x00,0x05,0x01,0x00,0x00, +0x34,0x00,0x00,0x00,0xef,0x00,0x00,0x00,0x34,0x00,0x00,0x00, +0x39,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00, +0x11,0x01,0x00,0x00,0x10,0x01,0x00,0x00,0x73,0x00,0x04,0x00, +0xf9,0x00,0x00,0x00,0x12,0x01,0x00,0x00,0x11,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0x0b,0x01,0x00,0x00,0x13,0x01,0x00,0x00, +0xfe,0x00,0x00,0x00,0x0e,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, +0x13,0x01,0x00,0x00,0x12,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x15,0x01,0x00,0x00,0xf8,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x41,0x00,0x08,0x00,0x07,0x01,0x00,0x00, +0x17,0x01,0x00,0x00,0x05,0x01,0x00,0x00,0x34,0x00,0x00,0x00, +0xef,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00,0x18,0x01,0x00,0x00, +0x17,0x01,0x00,0x00,0x73,0x00,0x04,0x00,0xf9,0x00,0x00,0x00, +0x19,0x01,0x00,0x00,0x18,0x01,0x00,0x00,0x41,0x00,0x05,0x00, +0x0b,0x01,0x00,0x00,0x1a,0x01,0x00,0x00,0xfe,0x00,0x00,0x00, +0x15,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x1a,0x01,0x00,0x00, +0x19,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x1d,0x01,0x00,0x00,0xf8,0x00,0x00,0x00,0x1c,0x01,0x00,0x00, +0x41,0x00,0x08,0x00,0x07,0x01,0x00,0x00,0x1f,0x01,0x00,0x00, +0x05,0x01,0x00,0x00,0x34,0x00,0x00,0x00,0xef,0x00,0x00,0x00, +0x34,0x00,0x00,0x00,0x1c,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, +0xc4,0x00,0x00,0x00,0x20,0x01,0x00,0x00,0x1f,0x01,0x00,0x00, +0x73,0x00,0x04,0x00,0xf9,0x00,0x00,0x00,0x21,0x01,0x00,0x00, +0x20,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0x0b,0x01,0x00,0x00, +0x22,0x01,0x00,0x00,0xfe,0x00,0x00,0x00,0x1d,0x01,0x00,0x00, +0x3e,0x00,0x03,0x00,0x22,0x01,0x00,0x00,0x21,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x25,0x01,0x00,0x00, +0xf8,0x00,0x00,0x00,0x24,0x01,0x00,0x00,0x41,0x00,0x08,0x00, +0x07,0x01,0x00,0x00,0x27,0x01,0x00,0x00,0x05,0x01,0x00,0x00, +0x34,0x00,0x00,0x00,0xef,0x00,0x00,0x00,0xd0,0x00,0x00,0x00, +0x3e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00, +0x28,0x01,0x00,0x00,0x27,0x01,0x00,0x00,0x73,0x00,0x04,0x00, +0xf9,0x00,0x00,0x00,0x29,0x01,0x00,0x00,0x28,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0x0b,0x01,0x00,0x00,0x2a,0x01,0x00,0x00, +0xfe,0x00,0x00,0x00,0x25,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, +0x2a,0x01,0x00,0x00,0x29,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x2d,0x01,0x00,0x00,0xf8,0x00,0x00,0x00, +0x2c,0x01,0x00,0x00,0x41,0x00,0x08,0x00,0x07,0x01,0x00,0x00, +0x2f,0x01,0x00,0x00,0x05,0x01,0x00,0x00,0x34,0x00,0x00,0x00, +0xef,0x00,0x00,0x00,0xd0,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00,0x30,0x01,0x00,0x00, +0x2f,0x01,0x00,0x00,0x73,0x00,0x04,0x00,0xf9,0x00,0x00,0x00, +0x31,0x01,0x00,0x00,0x30,0x01,0x00,0x00,0x41,0x00,0x05,0x00, +0x0b,0x01,0x00,0x00,0x32,0x01,0x00,0x00,0xfe,0x00,0x00,0x00, +0x2d,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x32,0x01,0x00,0x00, +0x31,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x35,0x01,0x00,0x00,0xf8,0x00,0x00,0x00,0x34,0x01,0x00,0x00, +0x41,0x00,0x08,0x00,0x07,0x01,0x00,0x00,0x37,0x01,0x00,0x00, +0x05,0x01,0x00,0x00,0x34,0x00,0x00,0x00,0xef,0x00,0x00,0x00, +0xd0,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0xc4,0x00,0x00,0x00,0x38,0x01,0x00,0x00,0x37,0x01,0x00,0x00, +0x73,0x00,0x04,0x00,0xf9,0x00,0x00,0x00,0x39,0x01,0x00,0x00, +0x38,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0x0b,0x01,0x00,0x00, +0x3a,0x01,0x00,0x00,0xfe,0x00,0x00,0x00,0x35,0x01,0x00,0x00, +0x3e,0x00,0x03,0x00,0x3a,0x01,0x00,0x00,0x39,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3d,0x01,0x00,0x00, +0xf8,0x00,0x00,0x00,0x3c,0x01,0x00,0x00,0x41,0x00,0x08,0x00, +0x07,0x01,0x00,0x00,0x3f,0x01,0x00,0x00,0x05,0x01,0x00,0x00, +0x34,0x00,0x00,0x00,0xef,0x00,0x00,0x00,0xd0,0x00,0x00,0x00, +0x1c,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00, +0x40,0x01,0x00,0x00,0x3f,0x01,0x00,0x00,0x73,0x00,0x04,0x00, +0xf9,0x00,0x00,0x00,0x41,0x01,0x00,0x00,0x40,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0x0b,0x01,0x00,0x00,0x42,0x01,0x00,0x00, +0xfe,0x00,0x00,0x00,0x3d,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, +0x42,0x01,0x00,0x00,0x41,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x49,0x01,0x00,0x00,0xf7,0x02,0x00,0x00, +0x47,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xdd,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xdf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x4b,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x4b,0x01,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xf8,0x02,0x00,0x00, +0x3e,0x00,0x00,0x00,0xdf,0x00,0x00,0x00,0xac,0x01,0x00,0x00, +0x4c,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, +0x51,0x01,0x00,0x00,0xf8,0x02,0x00,0x00,0xa7,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0x4d,0x01,0x00,0x00,0x4c,0x01,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x51,0x01,0x00,0x00, +0x4c,0x01,0x00,0x00,0x4d,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x4c,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x56,0x01,0x00,0x00,0x7e,0x00,0x00,0x00,0xf8,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x59,0x01,0x00,0x00, +0x56,0x01,0x00,0x00,0xab,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x5a,0x01,0x00,0x00,0x59,0x01,0x00,0x00, 0x6d,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x81,0x01,0x00,0x00,0x51,0x03,0x00,0x00,0x80,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x83,0x01,0x00,0x00, -0x81,0x01,0x00,0x00,0x6f,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x8d,0x01,0x00,0x00,0x7c,0x01,0x00,0x00, -0x8c,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x8f,0x01,0x00,0x00,0x6f,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x90,0x01,0x00,0x00, -0x8d,0x01,0x00,0x00,0x8f,0x01,0x00,0x00,0x41,0x00,0x08,0x00, -0xfc,0x00,0x00,0x00,0x97,0x01,0x00,0x00,0x95,0x01,0x00,0x00, -0x34,0x00,0x00,0x00,0x83,0x01,0x00,0x00,0x34,0x00,0x00,0x00, -0x3e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0xba,0x00,0x00,0x00, -0x98,0x01,0x00,0x00,0x97,0x01,0x00,0x00,0x73,0x00,0x04,0x00, -0xe6,0x00,0x00,0x00,0x99,0x01,0x00,0x00,0x98,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0x00,0x01,0x00,0x00,0x9a,0x01,0x00,0x00, -0x88,0x01,0x00,0x00,0x90,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0x9a,0x01,0x00,0x00,0x99,0x01,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x9f,0x01,0x00,0x00,0x7c,0x01,0x00,0x00, -0x9e,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xa2,0x01,0x00,0x00,0x9f,0x01,0x00,0x00,0x8f,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa3,0x01,0x00,0x00, -0xa2,0x01,0x00,0x00,0x39,0x00,0x00,0x00,0x41,0x00,0x08,0x00, -0xfc,0x00,0x00,0x00,0xa5,0x01,0x00,0x00,0x95,0x01,0x00,0x00, -0x34,0x00,0x00,0x00,0x83,0x01,0x00,0x00,0x34,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0xba,0x00,0x00,0x00, -0xa6,0x01,0x00,0x00,0xa5,0x01,0x00,0x00,0x73,0x00,0x04,0x00, -0xe6,0x00,0x00,0x00,0xa7,0x01,0x00,0x00,0xa6,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0x00,0x01,0x00,0x00,0xa8,0x01,0x00,0x00, -0x88,0x01,0x00,0x00,0xa3,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0xa8,0x01,0x00,0x00,0xa7,0x01,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xad,0x01,0x00,0x00,0x7c,0x01,0x00,0x00, -0xac,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xb0,0x01,0x00,0x00,0xad,0x01,0x00,0x00,0x8f,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xb1,0x01,0x00,0x00, -0xb0,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x41,0x00,0x08,0x00, -0xfc,0x00,0x00,0x00,0xb3,0x01,0x00,0x00,0x95,0x01,0x00,0x00, -0x34,0x00,0x00,0x00,0x83,0x01,0x00,0x00,0x34,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0xba,0x00,0x00,0x00, -0xb4,0x01,0x00,0x00,0xb3,0x01,0x00,0x00,0x73,0x00,0x04,0x00, -0xe6,0x00,0x00,0x00,0xb5,0x01,0x00,0x00,0xb4,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0x00,0x01,0x00,0x00,0xb6,0x01,0x00,0x00, -0x88,0x01,0x00,0x00,0xb1,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0xb6,0x01,0x00,0x00,0xb5,0x01,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xbb,0x01,0x00,0x00,0x7c,0x01,0x00,0x00, -0xba,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xbe,0x01,0x00,0x00,0xbb,0x01,0x00,0x00,0x8f,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xbf,0x01,0x00,0x00, -0xbe,0x01,0x00,0x00,0x26,0x01,0x00,0x00,0x41,0x00,0x08,0x00, -0xfc,0x00,0x00,0x00,0xc1,0x01,0x00,0x00,0x95,0x01,0x00,0x00, -0x34,0x00,0x00,0x00,0x83,0x01,0x00,0x00,0x34,0x00,0x00,0x00, -0x26,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0xba,0x00,0x00,0x00, -0xc2,0x01,0x00,0x00,0xc1,0x01,0x00,0x00,0x73,0x00,0x04,0x00, -0xe6,0x00,0x00,0x00,0xc3,0x01,0x00,0x00,0xc2,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0x00,0x01,0x00,0x00,0xc4,0x01,0x00,0x00, -0x88,0x01,0x00,0x00,0xbf,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0xc4,0x01,0x00,0x00,0xc3,0x01,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xc9,0x01,0x00,0x00,0x7c,0x01,0x00,0x00, -0xc8,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xcc,0x01,0x00,0x00,0xc9,0x01,0x00,0x00,0x8f,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xcd,0x01,0x00,0x00, -0xcc,0x01,0x00,0x00,0x35,0x01,0x00,0x00,0x41,0x00,0x08,0x00, -0xfc,0x00,0x00,0x00,0xcf,0x01,0x00,0x00,0x95,0x01,0x00,0x00, -0x34,0x00,0x00,0x00,0x83,0x01,0x00,0x00,0xc6,0x00,0x00,0x00, -0x3e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0xba,0x00,0x00,0x00, -0xd0,0x01,0x00,0x00,0xcf,0x01,0x00,0x00,0x73,0x00,0x04,0x00, -0xe6,0x00,0x00,0x00,0xd1,0x01,0x00,0x00,0xd0,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0x00,0x01,0x00,0x00,0xd2,0x01,0x00,0x00, -0x88,0x01,0x00,0x00,0xcd,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0xd2,0x01,0x00,0x00,0xd1,0x01,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xd7,0x01,0x00,0x00,0x7c,0x01,0x00,0x00, -0xd6,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xda,0x01,0x00,0x00,0xd7,0x01,0x00,0x00,0x8f,0x01,0x00,0x00, +0x5b,0x01,0x00,0x00,0xff,0x02,0x00,0x00,0x5a,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x5d,0x01,0x00,0x00, +0x5b,0x01,0x00,0x00,0x79,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x63,0x01,0x00,0x00,0x56,0x01,0x00,0x00, +0x62,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x65,0x01,0x00,0x00,0x79,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x66,0x01,0x00,0x00, +0x63,0x01,0x00,0x00,0x65,0x01,0x00,0x00,0x41,0x00,0x08,0x00, +0x07,0x01,0x00,0x00,0x73,0x01,0x00,0x00,0x71,0x01,0x00,0x00, +0x34,0x00,0x00,0x00,0x5d,0x01,0x00,0x00,0x34,0x00,0x00,0x00, +0x3e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00, +0x74,0x01,0x00,0x00,0x73,0x01,0x00,0x00,0x73,0x00,0x04,0x00, +0xf9,0x00,0x00,0x00,0x75,0x01,0x00,0x00,0x74,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0x0b,0x01,0x00,0x00,0x76,0x01,0x00,0x00, +0x6b,0x01,0x00,0x00,0x66,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, +0x76,0x01,0x00,0x00,0x75,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x78,0x01,0x00,0x00,0x66,0x01,0x00,0x00, +0x39,0x00,0x00,0x00,0x41,0x00,0x08,0x00,0x07,0x01,0x00,0x00, +0x7a,0x01,0x00,0x00,0x71,0x01,0x00,0x00,0x34,0x00,0x00,0x00, +0x5d,0x01,0x00,0x00,0x34,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00,0x7b,0x01,0x00,0x00, +0x7a,0x01,0x00,0x00,0x73,0x00,0x04,0x00,0xf9,0x00,0x00,0x00, +0x7c,0x01,0x00,0x00,0x7b,0x01,0x00,0x00,0x41,0x00,0x05,0x00, +0x0b,0x01,0x00,0x00,0x7d,0x01,0x00,0x00,0x6b,0x01,0x00,0x00, +0x78,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x7d,0x01,0x00,0x00, +0x7c,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x7f,0x01,0x00,0x00,0x66,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, +0x41,0x00,0x08,0x00,0x07,0x01,0x00,0x00,0x81,0x01,0x00,0x00, +0x71,0x01,0x00,0x00,0x34,0x00,0x00,0x00,0x5d,0x01,0x00,0x00, +0x34,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0xc4,0x00,0x00,0x00,0x82,0x01,0x00,0x00,0x81,0x01,0x00,0x00, +0x73,0x00,0x04,0x00,0xf9,0x00,0x00,0x00,0x83,0x01,0x00,0x00, +0x82,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0x0b,0x01,0x00,0x00, +0x84,0x01,0x00,0x00,0x6b,0x01,0x00,0x00,0x7f,0x01,0x00,0x00, +0x3e,0x00,0x03,0x00,0x84,0x01,0x00,0x00,0x83,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x86,0x01,0x00,0x00, +0x66,0x01,0x00,0x00,0x1c,0x01,0x00,0x00,0x41,0x00,0x08,0x00, +0x07,0x01,0x00,0x00,0x88,0x01,0x00,0x00,0x71,0x01,0x00,0x00, +0x34,0x00,0x00,0x00,0x5d,0x01,0x00,0x00,0x34,0x00,0x00,0x00, +0x1c,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00, +0x89,0x01,0x00,0x00,0x88,0x01,0x00,0x00,0x73,0x00,0x04,0x00, +0xf9,0x00,0x00,0x00,0x8a,0x01,0x00,0x00,0x89,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0x0b,0x01,0x00,0x00,0x8b,0x01,0x00,0x00, +0x6b,0x01,0x00,0x00,0x86,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, +0x8b,0x01,0x00,0x00,0x8a,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x8d,0x01,0x00,0x00,0x66,0x01,0x00,0x00, +0x24,0x01,0x00,0x00,0x41,0x00,0x08,0x00,0x07,0x01,0x00,0x00, +0x8f,0x01,0x00,0x00,0x71,0x01,0x00,0x00,0x34,0x00,0x00,0x00, +0x5d,0x01,0x00,0x00,0xd0,0x00,0x00,0x00,0x3e,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00,0x90,0x01,0x00,0x00, +0x8f,0x01,0x00,0x00,0x73,0x00,0x04,0x00,0xf9,0x00,0x00,0x00, +0x91,0x01,0x00,0x00,0x90,0x01,0x00,0x00,0x41,0x00,0x05,0x00, +0x0b,0x01,0x00,0x00,0x92,0x01,0x00,0x00,0x6b,0x01,0x00,0x00, +0x8d,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x92,0x01,0x00,0x00, +0x91,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x94,0x01,0x00,0x00,0x66,0x01,0x00,0x00,0x2c,0x01,0x00,0x00, +0x41,0x00,0x08,0x00,0x07,0x01,0x00,0x00,0x96,0x01,0x00,0x00, +0x71,0x01,0x00,0x00,0x34,0x00,0x00,0x00,0x5d,0x01,0x00,0x00, +0xd0,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0xc4,0x00,0x00,0x00,0x97,0x01,0x00,0x00,0x96,0x01,0x00,0x00, +0x73,0x00,0x04,0x00,0xf9,0x00,0x00,0x00,0x98,0x01,0x00,0x00, +0x97,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0x0b,0x01,0x00,0x00, +0x99,0x01,0x00,0x00,0x6b,0x01,0x00,0x00,0x94,0x01,0x00,0x00, +0x3e,0x00,0x03,0x00,0x99,0x01,0x00,0x00,0x98,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9b,0x01,0x00,0x00, +0x66,0x01,0x00,0x00,0x34,0x01,0x00,0x00,0x41,0x00,0x08,0x00, +0x07,0x01,0x00,0x00,0x9d,0x01,0x00,0x00,0x71,0x01,0x00,0x00, +0x34,0x00,0x00,0x00,0x5d,0x01,0x00,0x00,0xd0,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00, +0x9e,0x01,0x00,0x00,0x9d,0x01,0x00,0x00,0x73,0x00,0x04,0x00, +0xf9,0x00,0x00,0x00,0x9f,0x01,0x00,0x00,0x9e,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0x0b,0x01,0x00,0x00,0xa0,0x01,0x00,0x00, +0x6b,0x01,0x00,0x00,0x9b,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, +0xa0,0x01,0x00,0x00,0x9f,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xa2,0x01,0x00,0x00,0x66,0x01,0x00,0x00, +0x3c,0x01,0x00,0x00,0x41,0x00,0x08,0x00,0x07,0x01,0x00,0x00, +0xa4,0x01,0x00,0x00,0x71,0x01,0x00,0x00,0x34,0x00,0x00,0x00, +0x5d,0x01,0x00,0x00,0xd0,0x00,0x00,0x00,0x1c,0x01,0x00,0x00, +0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00,0xa5,0x01,0x00,0x00, +0xa4,0x01,0x00,0x00,0x73,0x00,0x04,0x00,0xf9,0x00,0x00,0x00, +0xa6,0x01,0x00,0x00,0xa5,0x01,0x00,0x00,0x41,0x00,0x05,0x00, +0x0b,0x01,0x00,0x00,0xa7,0x01,0x00,0x00,0x6b,0x01,0x00,0x00, +0xa2,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0xa7,0x01,0x00,0x00, +0xa6,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xac,0x01,0x00,0x00,0xf8,0x02,0x00,0x00,0xaa,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0x4b,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x4d,0x01,0x00,0x00,0xe0,0x00,0x04,0x00,0x0c,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0xad,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xb0,0x01,0x00,0x00,0xfb,0x02,0x00,0x00, +0xae,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xb3,0x01,0x00,0x00,0xff,0x02,0x00,0x00,0xb1,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0xb5,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xb5,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0x01,0x03,0x00,0x00,0x3e,0x00,0x00,0x00,0x4d,0x01,0x00,0x00, +0x5f,0x02,0x00,0x00,0xb8,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc2,0x00,0x00,0x00,0xbb,0x01,0x00,0x00,0x01,0x03,0x00,0x00, +0x6c,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xb7,0x01,0x00,0x00, +0xb8,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0xbb,0x01,0x00,0x00,0xb6,0x01,0x00,0x00,0xb7,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xb6,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0xbd,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xbd,0x01,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x05,0x03,0x00,0x00, +0x3e,0x00,0x00,0x00,0xb6,0x01,0x00,0x00,0xe9,0x01,0x00,0x00, +0xc0,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, +0xc3,0x01,0x00,0x00,0x05,0x03,0x00,0x00,0x60,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0xbf,0x01,0x00,0x00,0xc0,0x01,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xc3,0x01,0x00,0x00, +0xbe,0x01,0x00,0x00,0xbf,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xbe,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xc5,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xc5,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0x17,0x03,0x00,0x00,0x3e,0x00,0x00,0x00, +0xbe,0x01,0x00,0x00,0xe7,0x01,0x00,0x00,0xc6,0x01,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0xcb,0x01,0x00,0x00, +0x17,0x03,0x00,0x00,0x62,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0xc7,0x01,0x00,0x00,0xc6,0x01,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0xcb,0x01,0x00,0x00,0xc6,0x01,0x00,0x00, +0xc7,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xc6,0x01,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xd1,0x01,0x00,0x00, +0x05,0x03,0x00,0x00,0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xd3,0x01,0x00,0x00,0xd1,0x01,0x00,0x00, +0x17,0x03,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xd5,0x01,0x00,0x00,0x55,0x00,0x00,0x00,0x53,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xd7,0x01,0x00,0x00, +0x05,0x03,0x00,0x00,0x61,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xd8,0x01,0x00,0x00,0xd5,0x01,0x00,0x00, +0xd7,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xda,0x01,0x00,0x00,0x64,0x00,0x00,0x00,0x62,0x00,0x00,0x00, 0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xdb,0x01,0x00,0x00, -0xda,0x01,0x00,0x00,0x44,0x01,0x00,0x00,0x41,0x00,0x08,0x00, -0xfc,0x00,0x00,0x00,0xdd,0x01,0x00,0x00,0x95,0x01,0x00,0x00, -0x34,0x00,0x00,0x00,0x83,0x01,0x00,0x00,0xc6,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0xba,0x00,0x00,0x00, -0xde,0x01,0x00,0x00,0xdd,0x01,0x00,0x00,0x73,0x00,0x04,0x00, -0xe6,0x00,0x00,0x00,0xdf,0x01,0x00,0x00,0xde,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0x00,0x01,0x00,0x00,0xe0,0x01,0x00,0x00, -0x88,0x01,0x00,0x00,0xdb,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0xe0,0x01,0x00,0x00,0xdf,0x01,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xe5,0x01,0x00,0x00,0x7c,0x01,0x00,0x00, -0xe4,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xe8,0x01,0x00,0x00,0xe5,0x01,0x00,0x00,0x8f,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xe9,0x01,0x00,0x00, -0xe8,0x01,0x00,0x00,0x53,0x01,0x00,0x00,0x41,0x00,0x08,0x00, -0xfc,0x00,0x00,0x00,0xeb,0x01,0x00,0x00,0x95,0x01,0x00,0x00, -0x34,0x00,0x00,0x00,0x83,0x01,0x00,0x00,0xc6,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0xba,0x00,0x00,0x00, -0xec,0x01,0x00,0x00,0xeb,0x01,0x00,0x00,0x73,0x00,0x04,0x00, -0xe6,0x00,0x00,0x00,0xed,0x01,0x00,0x00,0xec,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0x00,0x01,0x00,0x00,0xee,0x01,0x00,0x00, -0x88,0x01,0x00,0x00,0xe9,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0xee,0x01,0x00,0x00,0xed,0x01,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xf3,0x01,0x00,0x00,0x7c,0x01,0x00,0x00, -0xf2,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xf6,0x01,0x00,0x00,0xf3,0x01,0x00,0x00,0x8f,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xf7,0x01,0x00,0x00, -0xf6,0x01,0x00,0x00,0x62,0x01,0x00,0x00,0x41,0x00,0x08,0x00, -0xfc,0x00,0x00,0x00,0xf9,0x01,0x00,0x00,0x95,0x01,0x00,0x00, -0x34,0x00,0x00,0x00,0x83,0x01,0x00,0x00,0xc6,0x00,0x00,0x00, -0x26,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0xba,0x00,0x00,0x00, -0xfa,0x01,0x00,0x00,0xf9,0x01,0x00,0x00,0x73,0x00,0x04,0x00, -0xe6,0x00,0x00,0x00,0xfb,0x01,0x00,0x00,0xfa,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0x00,0x01,0x00,0x00,0xfc,0x01,0x00,0x00, -0x88,0x01,0x00,0x00,0xf7,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0xfc,0x01,0x00,0x00,0xfb,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xfe,0x01,0x00,0x00,0x4a,0x03,0x00,0x00, -0x6d,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x71,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x73,0x01,0x00,0x00,0xe0,0x00,0x04,0x00, -0x0c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0xff,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x02,0x02,0x00,0x00, -0x4d,0x03,0x00,0x00,0x00,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x05,0x02,0x00,0x00,0x51,0x03,0x00,0x00, -0x03,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x07,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x07,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0x53,0x03,0x00,0x00,0x3e,0x00,0x00,0x00, -0x73,0x01,0x00,0x00,0xb1,0x02,0x00,0x00,0x0a,0x02,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00,0x0d,0x02,0x00,0x00, -0x53,0x03,0x00,0x00,0x6c,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0x09,0x02,0x00,0x00,0x0a,0x02,0x00,0x00,0x00,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x0d,0x02,0x00,0x00,0x08,0x02,0x00,0x00, -0x09,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x08,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0x0f,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x0f,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0x57,0x03,0x00,0x00,0x3e,0x00,0x00,0x00,0x08,0x02,0x00,0x00, -0x3b,0x02,0x00,0x00,0x12,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb8,0x00,0x00,0x00,0x15,0x02,0x00,0x00,0x57,0x03,0x00,0x00, -0x60,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x11,0x02,0x00,0x00, -0x12,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x15,0x02,0x00,0x00,0x10,0x02,0x00,0x00,0x11,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x10,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x17,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x17,0x02,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x69,0x03,0x00,0x00, -0x3e,0x00,0x00,0x00,0x10,0x02,0x00,0x00,0x39,0x02,0x00,0x00, -0x18,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00, -0x1d,0x02,0x00,0x00,0x69,0x03,0x00,0x00,0x62,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0x19,0x02,0x00,0x00,0x18,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x1d,0x02,0x00,0x00, -0x18,0x02,0x00,0x00,0x19,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x18,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x23,0x02,0x00,0x00,0x57,0x03,0x00,0x00,0x62,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x25,0x02,0x00,0x00, -0x23,0x02,0x00,0x00,0x69,0x03,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x27,0x02,0x00,0x00,0x55,0x00,0x00,0x00, -0x53,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x29,0x02,0x00,0x00,0x57,0x03,0x00,0x00,0x61,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x2a,0x02,0x00,0x00, -0x27,0x02,0x00,0x00,0x29,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x2c,0x02,0x00,0x00,0x64,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x2d,0x02,0x00,0x00,0x2a,0x02,0x00,0x00,0x2c,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x2f,0x02,0x00,0x00, -0x2d,0x02,0x00,0x00,0x69,0x03,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x31,0x02,0x00,0x00,0x2f,0x02,0x00,0x00, -0x30,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x33,0x02,0x00,0x00,0x31,0x02,0x00,0x00,0x53,0x03,0x00,0x00, -0x41,0x00,0x05,0x00,0x00,0x01,0x00,0x00,0x34,0x02,0x00,0x00, -0xeb,0x00,0x00,0x00,0x33,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, -0xe6,0x00,0x00,0x00,0x35,0x02,0x00,0x00,0x34,0x02,0x00,0x00, -0x41,0x00,0x05,0x00,0x36,0x02,0x00,0x00,0x37,0x02,0x00,0x00, -0x21,0x02,0x00,0x00,0x25,0x02,0x00,0x00,0x3e,0x00,0x03,0x00, -0x37,0x02,0x00,0x00,0x35,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x39,0x02,0x00,0x00,0x69,0x03,0x00,0x00, -0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x17,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x19,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x12,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x12,0x02,0x00,0x00, +0xd8,0x01,0x00,0x00,0xda,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xdd,0x01,0x00,0x00,0xdb,0x01,0x00,0x00, +0x17,0x03,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xdf,0x01,0x00,0x00,0xdd,0x01,0x00,0x00,0xde,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xe1,0x01,0x00,0x00, +0xdf,0x01,0x00,0x00,0x01,0x03,0x00,0x00,0x41,0x00,0x05,0x00, +0x0b,0x01,0x00,0x00,0xe2,0x01,0x00,0x00,0xfe,0x00,0x00,0x00, +0xe1,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0xf9,0x00,0x00,0x00, +0xe3,0x01,0x00,0x00,0xe2,0x01,0x00,0x00,0x41,0x00,0x05,0x00, +0xe4,0x01,0x00,0x00,0xe5,0x01,0x00,0x00,0xcf,0x01,0x00,0x00, +0xd3,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0xe5,0x01,0x00,0x00, +0xe3,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xe7,0x01,0x00,0x00,0x17,0x03,0x00,0x00,0xd0,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xc5,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xc7,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xc0,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xc0,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xe9,0x01,0x00,0x00,0x05,0x03,0x00,0x00, +0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xbd,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xbf,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0xeb,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xeb,0x01,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x06,0x03,0x00,0x00, +0x3e,0x00,0x00,0x00,0xbf,0x01,0x00,0x00,0x17,0x02,0x00,0x00, +0xee,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, +0xf1,0x01,0x00,0x00,0x06,0x03,0x00,0x00,0xbf,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0xed,0x01,0x00,0x00,0xee,0x01,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xf1,0x01,0x00,0x00, +0xec,0x01,0x00,0x00,0xed,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xec,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xf3,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xf3,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0x14,0x03,0x00,0x00,0x3e,0x00,0x00,0x00, +0xec,0x01,0x00,0x00,0x15,0x02,0x00,0x00,0xf4,0x01,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0xf9,0x01,0x00,0x00, +0x14,0x03,0x00,0x00,0xbc,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0xf5,0x01,0x00,0x00,0xf4,0x01,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0xf9,0x01,0x00,0x00,0xf4,0x01,0x00,0x00, +0xf5,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xf4,0x01,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xff,0x01,0x00,0x00, +0x06,0x03,0x00,0x00,0xbc,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x01,0x02,0x00,0x00,0xff,0x01,0x00,0x00, +0x14,0x03,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x03,0x02,0x00,0x00,0x59,0x00,0x00,0x00,0xb9,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x06,0x02,0x00,0x00, +0x06,0x03,0x00,0x00,0x05,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x07,0x02,0x00,0x00,0x03,0x02,0x00,0x00, +0x06,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x09,0x02,0x00,0x00,0x68,0x00,0x00,0x00,0xbc,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x0a,0x02,0x00,0x00, +0x07,0x02,0x00,0x00,0x09,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x0c,0x02,0x00,0x00,0x0a,0x02,0x00,0x00, +0x14,0x03,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x0e,0x02,0x00,0x00,0x0c,0x02,0x00,0x00,0x0d,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x10,0x02,0x00,0x00, +0x0e,0x02,0x00,0x00,0x01,0x03,0x00,0x00,0x41,0x00,0x05,0x00, +0x0b,0x01,0x00,0x00,0x11,0x02,0x00,0x00,0x6b,0x01,0x00,0x00, +0x10,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0xf9,0x00,0x00,0x00, +0x12,0x02,0x00,0x00,0x11,0x02,0x00,0x00,0x41,0x00,0x05,0x00, +0xe4,0x01,0x00,0x00,0x13,0x02,0x00,0x00,0xfd,0x01,0x00,0x00, +0x01,0x02,0x00,0x00,0x3e,0x00,0x03,0x00,0x13,0x02,0x00,0x00, +0x12,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x15,0x02,0x00,0x00,0x14,0x03,0x00,0x00,0xd0,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xf3,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xf5,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xee,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xee,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x17,0x02,0x00,0x00,0x06,0x03,0x00,0x00, +0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xeb,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xed,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0x19,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x19,0x02,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x07,0x03,0x00,0x00, +0x3e,0x00,0x00,0x00,0xed,0x01,0x00,0x00,0x5d,0x02,0x00,0x00, +0x1c,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, +0x1f,0x02,0x00,0x00,0x07,0x03,0x00,0x00,0xbf,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0x1b,0x02,0x00,0x00,0x1c,0x02,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x1f,0x02,0x00,0x00, +0x1a,0x02,0x00,0x00,0x1b,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x1a,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x21,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x21,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0x0b,0x03,0x00,0x00,0x3e,0x00,0x00,0x00, +0x1a,0x02,0x00,0x00,0x5b,0x02,0x00,0x00,0x24,0x02,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0x27,0x02,0x00,0x00, +0x0b,0x03,0x00,0x00,0x60,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0x23,0x02,0x00,0x00,0x24,0x02,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x27,0x02,0x00,0x00,0x22,0x02,0x00,0x00, +0x23,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x22,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0x29,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x29,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0x0d,0x03,0x00,0x00,0x3e,0x00,0x00,0x00,0x22,0x02,0x00,0x00, +0x59,0x02,0x00,0x00,0x2c,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc2,0x00,0x00,0x00,0x2f,0x02,0x00,0x00,0x0d,0x03,0x00,0x00, +0xbc,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x2b,0x02,0x00,0x00, +0x2c,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x2f,0x02,0x00,0x00,0x2a,0x02,0x00,0x00,0x2b,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x2a,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0x31,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x31,0x02,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x0f,0x03,0x00,0x00, +0x3e,0x00,0x00,0x00,0x2a,0x02,0x00,0x00,0x57,0x02,0x00,0x00, +0x32,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, +0x37,0x02,0x00,0x00,0x0f,0x03,0x00,0x00,0x62,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0x33,0x02,0x00,0x00,0x32,0x02,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x37,0x02,0x00,0x00, +0x32,0x02,0x00,0x00,0x33,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x32,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x39,0x02,0x00,0x00,0x07,0x03,0x00,0x00,0xbc,0x00,0x00,0x00, 0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3b,0x02,0x00,0x00, -0x57,0x03,0x00,0x00,0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x0f,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x11,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0x3d,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x3d,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0x58,0x03,0x00,0x00,0x3e,0x00,0x00,0x00,0x11,0x02,0x00,0x00, -0x69,0x02,0x00,0x00,0x40,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb8,0x00,0x00,0x00,0x43,0x02,0x00,0x00,0x58,0x03,0x00,0x00, -0xb5,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x3f,0x02,0x00,0x00, -0x40,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x43,0x02,0x00,0x00,0x3e,0x02,0x00,0x00,0x3f,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x3e,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x45,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x45,0x02,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x66,0x03,0x00,0x00, -0x3e,0x00,0x00,0x00,0x3e,0x02,0x00,0x00,0x67,0x02,0x00,0x00, -0x46,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00, -0x4b,0x02,0x00,0x00,0x66,0x03,0x00,0x00,0xb2,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0x47,0x02,0x00,0x00,0x46,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x4b,0x02,0x00,0x00, -0x46,0x02,0x00,0x00,0x47,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x46,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x51,0x02,0x00,0x00,0x58,0x03,0x00,0x00,0xb2,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x53,0x02,0x00,0x00, -0x51,0x02,0x00,0x00,0x66,0x03,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x55,0x02,0x00,0x00,0x59,0x00,0x00,0x00, -0xaf,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x58,0x02,0x00,0x00,0x58,0x03,0x00,0x00,0x57,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x59,0x02,0x00,0x00, -0x55,0x02,0x00,0x00,0x58,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x5b,0x02,0x00,0x00,0x68,0x00,0x00,0x00, -0xb2,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x5c,0x02,0x00,0x00,0x59,0x02,0x00,0x00,0x5b,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x5e,0x02,0x00,0x00, -0x5c,0x02,0x00,0x00,0x66,0x03,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x60,0x02,0x00,0x00,0x5e,0x02,0x00,0x00, -0x5f,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x62,0x02,0x00,0x00,0x60,0x02,0x00,0x00,0x53,0x03,0x00,0x00, -0x41,0x00,0x05,0x00,0x00,0x01,0x00,0x00,0x63,0x02,0x00,0x00, -0x88,0x01,0x00,0x00,0x62,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, -0xe6,0x00,0x00,0x00,0x64,0x02,0x00,0x00,0x63,0x02,0x00,0x00, -0x41,0x00,0x05,0x00,0x36,0x02,0x00,0x00,0x65,0x02,0x00,0x00, -0x4f,0x02,0x00,0x00,0x53,0x02,0x00,0x00,0x3e,0x00,0x03,0x00, -0x65,0x02,0x00,0x00,0x64,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x67,0x02,0x00,0x00,0x66,0x03,0x00,0x00, -0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x45,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x47,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x40,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x40,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x69,0x02,0x00,0x00, -0x58,0x03,0x00,0x00,0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x3d,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x3f,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0x6b,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x6b,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0x59,0x03,0x00,0x00,0x3e,0x00,0x00,0x00,0x3f,0x02,0x00,0x00, -0xaf,0x02,0x00,0x00,0x6e,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb8,0x00,0x00,0x00,0x71,0x02,0x00,0x00,0x59,0x03,0x00,0x00, -0xb5,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x6d,0x02,0x00,0x00, -0x6e,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x71,0x02,0x00,0x00,0x6c,0x02,0x00,0x00,0x6d,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x6c,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x73,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x73,0x02,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x5d,0x03,0x00,0x00, -0x3e,0x00,0x00,0x00,0x6c,0x02,0x00,0x00,0xad,0x02,0x00,0x00, -0x76,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00, -0x79,0x02,0x00,0x00,0x5d,0x03,0x00,0x00,0x60,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0x75,0x02,0x00,0x00,0x76,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x79,0x02,0x00,0x00, -0x74,0x02,0x00,0x00,0x75,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x74,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x7b,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x7b,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0x5f,0x03,0x00,0x00,0x3e,0x00,0x00,0x00, -0x74,0x02,0x00,0x00,0xab,0x02,0x00,0x00,0x7e,0x02,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00,0x81,0x02,0x00,0x00, -0x5f,0x03,0x00,0x00,0xb2,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0x7d,0x02,0x00,0x00,0x7e,0x02,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x81,0x02,0x00,0x00,0x7c,0x02,0x00,0x00, -0x7d,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x7c,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0x83,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x83,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0x61,0x03,0x00,0x00,0x3e,0x00,0x00,0x00,0x7c,0x02,0x00,0x00, -0xa9,0x02,0x00,0x00,0x84,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb8,0x00,0x00,0x00,0x89,0x02,0x00,0x00,0x61,0x03,0x00,0x00, -0x62,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x85,0x02,0x00,0x00, -0x84,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x89,0x02,0x00,0x00,0x84,0x02,0x00,0x00,0x85,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x84,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x8b,0x02,0x00,0x00,0x59,0x03,0x00,0x00, -0xb2,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x8d,0x02,0x00,0x00,0x8b,0x02,0x00,0x00,0x5f,0x03,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8f,0x02,0x00,0x00, -0x8d,0x02,0x00,0x00,0x8e,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x91,0x02,0x00,0x00,0x5d,0x03,0x00,0x00, -0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x92,0x02,0x00,0x00,0x8f,0x02,0x00,0x00,0x91,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x94,0x02,0x00,0x00, -0x92,0x02,0x00,0x00,0x61,0x03,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x98,0x02,0x00,0x00,0x91,0x02,0x00,0x00, -0x61,0x03,0x00,0x00,0x41,0x00,0x05,0x00,0x36,0x02,0x00,0x00, -0x99,0x02,0x00,0x00,0x21,0x02,0x00,0x00,0x98,0x02,0x00,0x00, -0x3d,0x00,0x04,0x00,0xe6,0x00,0x00,0x00,0x9a,0x02,0x00,0x00, -0x99,0x02,0x00,0x00,0x73,0x00,0x04,0x00,0xba,0x00,0x00,0x00, -0x9b,0x02,0x00,0x00,0x9a,0x02,0x00,0x00,0x41,0x00,0x05,0x00, -0x36,0x02,0x00,0x00,0xa0,0x02,0x00,0x00,0x4f,0x02,0x00,0x00, -0x8d,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0xe6,0x00,0x00,0x00, -0xa1,0x02,0x00,0x00,0xa0,0x02,0x00,0x00,0x73,0x00,0x04,0x00, -0xba,0x00,0x00,0x00,0xa2,0x02,0x00,0x00,0xa1,0x02,0x00,0x00, -0x41,0x00,0x05,0x00,0xc3,0x00,0x00,0x00,0xa4,0x02,0x00,0x00, -0xc0,0x00,0x00,0x00,0x94,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, -0xba,0x00,0x00,0x00,0xa5,0x02,0x00,0x00,0xa4,0x02,0x00,0x00, -0x0c,0x00,0x08,0x00,0xba,0x00,0x00,0x00,0xa6,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x9b,0x02,0x00,0x00, -0xa2,0x02,0x00,0x00,0xa5,0x02,0x00,0x00,0x3e,0x00,0x03,0x00, -0xa4,0x02,0x00,0x00,0xa6,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xa9,0x02,0x00,0x00,0x61,0x03,0x00,0x00, -0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x83,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x85,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x7e,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x7e,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xab,0x02,0x00,0x00, -0x5f,0x03,0x00,0x00,0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x7b,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x7d,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0x76,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x76,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xad,0x02,0x00,0x00,0x5d,0x03,0x00,0x00,0xc6,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x73,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x75,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x6e,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x6e,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xaf,0x02,0x00,0x00,0x59,0x03,0x00,0x00, -0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x6b,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x6d,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x0a,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x0a,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xb1,0x02,0x00,0x00, -0x53,0x03,0x00,0x00,0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x07,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x09,0x02,0x00,0x00, -0xe0,0x00,0x04,0x00,0x0c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0xff,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xcd,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xcd,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xb3,0x02,0x00,0x00,0x39,0x03,0x00,0x00, -0x6c,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xca,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xcc,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xb8,0x02,0x00,0x00,0x55,0x00,0x00,0x00, -0x53,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xb9,0x02,0x00,0x00,0x8c,0x00,0x00,0x00,0xb8,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xbe,0x02,0x00,0x00, -0x59,0x00,0x00,0x00,0xaf,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xbf,0x02,0x00,0x00,0x9e,0x00,0x00,0x00, -0xbe,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0xc4,0x02,0x00,0x00,0x12,0x00,0x00,0x00,0xc3,0x02,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xc5,0x02,0x00,0x00, -0xc4,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xc6,0x02,0x00,0x00,0x0f,0x00,0x00,0x00,0xc5,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xca,0x02,0x00,0x00, -0x47,0x00,0x00,0x00,0xc5,0x02,0x00,0x00,0x41,0x00,0x05,0x00, -0x0d,0x00,0x00,0x00,0xcc,0x02,0x00,0x00,0xcb,0x02,0x00,0x00, -0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xcd,0x02,0x00,0x00,0xcc,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xce,0x02,0x00,0x00,0xca,0x02,0x00,0x00, -0xcd,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xcf,0x02,0x00,0x00,0xc6,0x02,0x00,0x00,0xce,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0xd1,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0xd1,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0x3a,0x03,0x00,0x00,0x3e,0x00,0x00,0x00,0xcc,0x00,0x00,0x00, -0x37,0x03,0x00,0x00,0xd4,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb8,0x00,0x00,0x00,0xd7,0x02,0x00,0x00,0x3a,0x03,0x00,0x00, -0xb5,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xd3,0x02,0x00,0x00, -0xd4,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0xd7,0x02,0x00,0x00,0xd2,0x02,0x00,0x00,0xd3,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0xd2,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0xd9,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0xd9,0x02,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x3b,0x03,0x00,0x00, -0x3e,0x00,0x00,0x00,0xd2,0x02,0x00,0x00,0x35,0x03,0x00,0x00, -0xdc,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00, -0xdf,0x02,0x00,0x00,0x3b,0x03,0x00,0x00,0x60,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0xdb,0x02,0x00,0x00,0xdc,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xdf,0x02,0x00,0x00, -0xda,0x02,0x00,0x00,0xdb,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0xda,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xe3,0x02,0x00,0x00,0x3b,0x03,0x00,0x00,0x61,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xe4,0x02,0x00,0x00, -0xb9,0x02,0x00,0x00,0xe3,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xe6,0x02,0x00,0x00,0x64,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xe7,0x02,0x00,0x00,0xe4,0x02,0x00,0x00,0xe6,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xeb,0x02,0x00,0x00, -0x3a,0x03,0x00,0x00,0x57,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xec,0x02,0x00,0x00,0xbf,0x02,0x00,0x00, -0xeb,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xee,0x02,0x00,0x00,0x68,0x00,0x00,0x00,0xb2,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xef,0x02,0x00,0x00, -0xec,0x02,0x00,0x00,0xee,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0xf1,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0xf1,0x02,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x3d,0x03,0x00,0x00, -0x3e,0x00,0x00,0x00,0xda,0x02,0x00,0x00,0x33,0x03,0x00,0x00, -0xf4,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00, -0xf7,0x02,0x00,0x00,0x3d,0x03,0x00,0x00,0xb2,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0xf3,0x02,0x00,0x00,0xf4,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xf7,0x02,0x00,0x00, -0xf2,0x02,0x00,0x00,0xf3,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0xf2,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0xf9,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0xf9,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0x3f,0x03,0x00,0x00,0x3e,0x00,0x00,0x00, -0xf2,0x02,0x00,0x00,0x31,0x03,0x00,0x00,0xfc,0x02,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00,0xff,0x02,0x00,0x00, -0x3f,0x03,0x00,0x00,0x62,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0xfb,0x02,0x00,0x00,0xfc,0x02,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xff,0x02,0x00,0x00,0xfa,0x02,0x00,0x00, -0xfb,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0xfa,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x02,0x03,0x00,0x00, -0xe7,0x02,0x00,0x00,0x3f,0x03,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb8,0x00,0x00,0x00,0x05,0x03,0x00,0x00,0x02,0x03,0x00,0x00, -0x36,0x00,0x00,0x00,0xf7,0x00,0x03,0x00,0x07,0x03,0x00,0x00, -0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x05,0x03,0x00,0x00, -0x06,0x03,0x00,0x00,0x07,0x03,0x00,0x00,0xf8,0x00,0x02,0x00, -0x06,0x03,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x0a,0x03,0x00,0x00,0xef,0x02,0x00,0x00,0x3d,0x03,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x0b,0x03,0x00,0x00, -0x12,0x00,0x00,0x00,0xc6,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x0c,0x03,0x00,0x00,0x0b,0x03,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00,0x0d,0x03,0x00,0x00, -0x0a,0x03,0x00,0x00,0x0c,0x03,0x00,0x00,0xf9,0x00,0x02,0x00, -0x07,0x03,0x00,0x00,0xf8,0x00,0x02,0x00,0x07,0x03,0x00,0x00, -0xf5,0x00,0x07,0x00,0xb8,0x00,0x00,0x00,0x0e,0x03,0x00,0x00, -0x05,0x03,0x00,0x00,0xfa,0x02,0x00,0x00,0x0d,0x03,0x00,0x00, -0x06,0x03,0x00,0x00,0xf7,0x00,0x03,0x00,0x10,0x03,0x00,0x00, -0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x0e,0x03,0x00,0x00, -0x0f,0x03,0x00,0x00,0x10,0x03,0x00,0x00,0xf8,0x00,0x02,0x00, +0x39,0x02,0x00,0x00,0x0d,0x03,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x3d,0x02,0x00,0x00,0x3b,0x02,0x00,0x00, +0x3c,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x3f,0x02,0x00,0x00,0x0b,0x03,0x00,0x00,0x62,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x40,0x02,0x00,0x00, +0x3d,0x02,0x00,0x00,0x3f,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x42,0x02,0x00,0x00,0x40,0x02,0x00,0x00, 0x0f,0x03,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x18,0x03,0x00,0x00,0xef,0x02,0x00,0x00,0x3d,0x03,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x1a,0x03,0x00,0x00, -0x12,0x00,0x00,0x00,0x19,0x03,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x1b,0x03,0x00,0x00,0x1a,0x03,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x1c,0x03,0x00,0x00, -0x18,0x03,0x00,0x00,0x1b,0x03,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x1d,0x03,0x00,0x00,0xcf,0x02,0x00,0x00, -0x1c,0x03,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x1f,0x03,0x00,0x00,0x1d,0x03,0x00,0x00,0xe7,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x21,0x03,0x00,0x00, -0x1f,0x03,0x00,0x00,0x3f,0x03,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x23,0x03,0x00,0x00,0x3a,0x03,0x00,0x00, -0xb2,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x25,0x03,0x00,0x00,0x23,0x03,0x00,0x00,0x3d,0x03,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x27,0x03,0x00,0x00, -0x25,0x03,0x00,0x00,0x26,0x03,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x29,0x03,0x00,0x00,0x3b,0x03,0x00,0x00, -0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x2a,0x03,0x00,0x00,0x27,0x03,0x00,0x00,0x29,0x03,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x2c,0x03,0x00,0x00, -0x2a,0x03,0x00,0x00,0x3f,0x03,0x00,0x00,0x41,0x00,0x05,0x00, -0xc3,0x00,0x00,0x00,0x2d,0x03,0x00,0x00,0xc0,0x00,0x00,0x00, -0x2c,0x03,0x00,0x00,0x3d,0x00,0x04,0x00,0xba,0x00,0x00,0x00, -0x2e,0x03,0x00,0x00,0x2d,0x03,0x00,0x00,0x41,0x00,0x06,0x00, -0xfc,0x00,0x00,0x00,0x2f,0x03,0x00,0x00,0x14,0x03,0x00,0x00, -0x34,0x00,0x00,0x00,0x21,0x03,0x00,0x00,0x3e,0x00,0x03,0x00, -0x2f,0x03,0x00,0x00,0x2e,0x03,0x00,0x00,0xf9,0x00,0x02,0x00, -0x10,0x03,0x00,0x00,0xf8,0x00,0x02,0x00,0x10,0x03,0x00,0x00, -0xf9,0x00,0x02,0x00,0xfc,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0xfc,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x31,0x03,0x00,0x00,0x3f,0x03,0x00,0x00,0xc6,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xf9,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0xfb,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0xf4,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0xf4,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x33,0x03,0x00,0x00,0x3d,0x03,0x00,0x00, -0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xf1,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0xf3,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0xdc,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0xdc,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x35,0x03,0x00,0x00, -0x3b,0x03,0x00,0x00,0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xd9,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0xdb,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0xd4,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0xd4,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x37,0x03,0x00,0x00,0x3a,0x03,0x00,0x00,0xc6,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xd1,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0xd3,0x02,0x00,0x00,0xfd,0x00,0x01,0x00,0x38,0x00,0x01,0x00, - +0x46,0x02,0x00,0x00,0x3f,0x02,0x00,0x00,0x0f,0x03,0x00,0x00, +0x41,0x00,0x05,0x00,0xe4,0x01,0x00,0x00,0x47,0x02,0x00,0x00, +0xcf,0x01,0x00,0x00,0x46,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, +0xf9,0x00,0x00,0x00,0x48,0x02,0x00,0x00,0x47,0x02,0x00,0x00, +0x73,0x00,0x04,0x00,0xc4,0x00,0x00,0x00,0x49,0x02,0x00,0x00, +0x48,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0xe4,0x01,0x00,0x00, +0x4e,0x02,0x00,0x00,0xfd,0x01,0x00,0x00,0x3b,0x02,0x00,0x00, +0x3d,0x00,0x04,0x00,0xf9,0x00,0x00,0x00,0x4f,0x02,0x00,0x00, +0x4e,0x02,0x00,0x00,0x73,0x00,0x04,0x00,0xc4,0x00,0x00,0x00, +0x50,0x02,0x00,0x00,0x4f,0x02,0x00,0x00,0x41,0x00,0x05,0x00, +0xcd,0x00,0x00,0x00,0x52,0x02,0x00,0x00,0xca,0x00,0x00,0x00, +0x42,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00, +0x53,0x02,0x00,0x00,0x52,0x02,0x00,0x00,0x0c,0x00,0x08,0x00, +0xc4,0x00,0x00,0x00,0x54,0x02,0x00,0x00,0x01,0x00,0x00,0x00, +0x32,0x00,0x00,0x00,0x49,0x02,0x00,0x00,0x50,0x02,0x00,0x00, +0x53,0x02,0x00,0x00,0x3e,0x00,0x03,0x00,0x52,0x02,0x00,0x00, +0x54,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x57,0x02,0x00,0x00,0x0f,0x03,0x00,0x00,0xd0,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x31,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x33,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x2c,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x2c,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x59,0x02,0x00,0x00,0x0d,0x03,0x00,0x00, +0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x29,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x2b,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0x24,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x24,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x5b,0x02,0x00,0x00, +0x0b,0x03,0x00,0x00,0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x21,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x23,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0x1c,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x1c,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x5d,0x02,0x00,0x00,0x07,0x03,0x00,0x00,0xd0,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x19,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x1b,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0xb8,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xb8,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x5f,0x02,0x00,0x00,0x01,0x03,0x00,0x00, +0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xb5,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xb7,0x01,0x00,0x00,0xe0,0x00,0x04,0x00, +0x0c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0xad,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0xd7,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xd7,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x61,0x02,0x00,0x00,0xe7,0x02,0x00,0x00,0x6c,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xd4,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xd6,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x66,0x02,0x00,0x00,0x55,0x00,0x00,0x00,0x53,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x67,0x02,0x00,0x00, +0x96,0x00,0x00,0x00,0x66,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x6c,0x02,0x00,0x00,0x59,0x00,0x00,0x00, +0xb9,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x6d,0x02,0x00,0x00,0xa8,0x00,0x00,0x00,0x6c,0x02,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x72,0x02,0x00,0x00, +0x12,0x00,0x00,0x00,0x71,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x73,0x02,0x00,0x00,0x72,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x74,0x02,0x00,0x00, +0x0f,0x00,0x00,0x00,0x73,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x78,0x02,0x00,0x00,0x47,0x00,0x00,0x00, +0x73,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00, +0x7a,0x02,0x00,0x00,0x79,0x02,0x00,0x00,0x0c,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x7b,0x02,0x00,0x00, +0x7a,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x7c,0x02,0x00,0x00,0x78,0x02,0x00,0x00,0x7b,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x7d,0x02,0x00,0x00, +0x74,0x02,0x00,0x00,0x7c,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0x7f,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x7f,0x02,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xe8,0x02,0x00,0x00, +0x3e,0x00,0x00,0x00,0xd6,0x00,0x00,0x00,0xe5,0x02,0x00,0x00, +0x82,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, +0x85,0x02,0x00,0x00,0xe8,0x02,0x00,0x00,0xbf,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0x81,0x02,0x00,0x00,0x82,0x02,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x85,0x02,0x00,0x00, +0x80,0x02,0x00,0x00,0x81,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x80,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x87,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x87,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xe9,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, +0x80,0x02,0x00,0x00,0xe3,0x02,0x00,0x00,0x8a,0x02,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0x8d,0x02,0x00,0x00, +0xe9,0x02,0x00,0x00,0x60,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0x89,0x02,0x00,0x00,0x8a,0x02,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x8d,0x02,0x00,0x00,0x88,0x02,0x00,0x00, +0x89,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x88,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x91,0x02,0x00,0x00, +0xe9,0x02,0x00,0x00,0x61,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x92,0x02,0x00,0x00,0x67,0x02,0x00,0x00, +0x91,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x94,0x02,0x00,0x00,0x64,0x00,0x00,0x00,0x62,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x95,0x02,0x00,0x00, +0x92,0x02,0x00,0x00,0x94,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x99,0x02,0x00,0x00,0xe8,0x02,0x00,0x00, +0x05,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x9a,0x02,0x00,0x00,0x6d,0x02,0x00,0x00,0x99,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9c,0x02,0x00,0x00, +0x68,0x00,0x00,0x00,0xbc,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x9d,0x02,0x00,0x00,0x9a,0x02,0x00,0x00, +0x9c,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x9f,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x9f,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xeb,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, +0x88,0x02,0x00,0x00,0xe1,0x02,0x00,0x00,0xa2,0x02,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0xa5,0x02,0x00,0x00, +0xeb,0x02,0x00,0x00,0xbc,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0xa1,0x02,0x00,0x00,0xa2,0x02,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0xa5,0x02,0x00,0x00,0xa0,0x02,0x00,0x00, +0xa1,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0xa0,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0xa7,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0xa7,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xed,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0xa0,0x02,0x00,0x00, +0xdf,0x02,0x00,0x00,0xaa,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc2,0x00,0x00,0x00,0xad,0x02,0x00,0x00,0xed,0x02,0x00,0x00, +0x62,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xa9,0x02,0x00,0x00, +0xaa,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0xad,0x02,0x00,0x00,0xa8,0x02,0x00,0x00,0xa9,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0xa8,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xb0,0x02,0x00,0x00,0x95,0x02,0x00,0x00, +0xed,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, +0xb3,0x02,0x00,0x00,0xb0,0x02,0x00,0x00,0x36,0x00,0x00,0x00, +0xf7,0x00,0x03,0x00,0xb5,0x02,0x00,0x00,0x00,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0xb3,0x02,0x00,0x00,0xb4,0x02,0x00,0x00, +0xb5,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0xb4,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xb8,0x02,0x00,0x00, +0x9d,0x02,0x00,0x00,0xeb,0x02,0x00,0x00,0x41,0x00,0x05,0x00, +0x15,0x00,0x00,0x00,0xb9,0x02,0x00,0x00,0x12,0x00,0x00,0x00, +0xd0,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xba,0x02,0x00,0x00,0xb9,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc2,0x00,0x00,0x00,0xbb,0x02,0x00,0x00,0xb8,0x02,0x00,0x00, +0xba,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0xb5,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0xb5,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, +0xc2,0x00,0x00,0x00,0xbc,0x02,0x00,0x00,0xb3,0x02,0x00,0x00, +0xa8,0x02,0x00,0x00,0xbb,0x02,0x00,0x00,0xb4,0x02,0x00,0x00, +0xf7,0x00,0x03,0x00,0xbe,0x02,0x00,0x00,0x00,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0xbc,0x02,0x00,0x00,0xbd,0x02,0x00,0x00, +0xbe,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0xbd,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xc6,0x02,0x00,0x00, +0x9d,0x02,0x00,0x00,0xeb,0x02,0x00,0x00,0x41,0x00,0x05,0x00, +0x15,0x00,0x00,0x00,0xc8,0x02,0x00,0x00,0x12,0x00,0x00,0x00, +0xc7,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xc9,0x02,0x00,0x00,0xc8,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xca,0x02,0x00,0x00,0xc6,0x02,0x00,0x00, +0xc9,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xcb,0x02,0x00,0x00,0x7d,0x02,0x00,0x00,0xca,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xcd,0x02,0x00,0x00, +0xcb,0x02,0x00,0x00,0x95,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xcf,0x02,0x00,0x00,0xcd,0x02,0x00,0x00, +0xed,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xd1,0x02,0x00,0x00,0xe8,0x02,0x00,0x00,0xbc,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xd3,0x02,0x00,0x00, +0xd1,0x02,0x00,0x00,0xeb,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xd5,0x02,0x00,0x00,0xd3,0x02,0x00,0x00, +0xd4,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xd7,0x02,0x00,0x00,0xe9,0x02,0x00,0x00,0x62,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xd8,0x02,0x00,0x00, +0xd5,0x02,0x00,0x00,0xd7,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xda,0x02,0x00,0x00,0xd8,0x02,0x00,0x00, +0xed,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0xcd,0x00,0x00,0x00, +0xdb,0x02,0x00,0x00,0xca,0x00,0x00,0x00,0xda,0x02,0x00,0x00, +0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00,0xdc,0x02,0x00,0x00, +0xdb,0x02,0x00,0x00,0x41,0x00,0x06,0x00,0x07,0x01,0x00,0x00, +0xdd,0x02,0x00,0x00,0xc2,0x02,0x00,0x00,0x34,0x00,0x00,0x00, +0xcf,0x02,0x00,0x00,0x3e,0x00,0x03,0x00,0xdd,0x02,0x00,0x00, +0xdc,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0xbe,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0xbe,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0xaa,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0xaa,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xdf,0x02,0x00,0x00, +0xed,0x02,0x00,0x00,0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xa7,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0xa9,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0xa2,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0xa2,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xe1,0x02,0x00,0x00,0xeb,0x02,0x00,0x00,0xd0,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x9f,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0xa1,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x8a,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x8a,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xe3,0x02,0x00,0x00,0xe9,0x02,0x00,0x00, +0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x87,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x89,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0x82,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x82,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xe5,0x02,0x00,0x00, +0xe8,0x02,0x00,0x00,0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x7f,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x81,0x02,0x00,0x00, +0xfd,0x00,0x01,0x00,0x38,0x00,0x01,0x00, }; -const uint64_t matmul_f32_aligned_l_len = 12168; +const uint64_t matmul_f32_aligned_len = 11432; -unsigned char matmul_f32_aligned_l_fp32_data[] = { +unsigned char matmul_f32_aligned_fp32_data[] = { 0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, -0xe9,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, +0xcf,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, 0x01,0x00,0x00,0x00,0x0b,0x00,0x06,0x00,0x01,0x00,0x00,0x00, 0x47,0x4c,0x53,0x4c,0x2e,0x73,0x74,0x64,0x2e,0x34,0x35,0x30, 0x00,0x00,0x00,0x00,0x0e,0x00,0x03,0x00,0x00,0x00,0x00,0x00, 0x01,0x00,0x00,0x00,0x0f,0x00,0x0f,0x00,0x05,0x00,0x00,0x00, 0x04,0x00,0x00,0x00,0x6d,0x61,0x69,0x6e,0x00,0x00,0x00,0x00, 0x0b,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x3d,0x00,0x00,0x00, -0x4c,0x00,0x00,0x00,0xea,0x00,0x00,0x00,0xf8,0x00,0x00,0x00, -0x46,0x01,0x00,0x00,0x53,0x01,0x00,0x00,0x4a,0x02,0x00,0x00, -0x93,0x02,0x00,0x00,0x10,0x00,0x06,0x00,0x04,0x00,0x00,0x00, +0x4c,0x00,0x00,0x00,0xfd,0x00,0x00,0x00,0x03,0x01,0x00,0x00, +0x45,0x01,0x00,0x00,0x4b,0x01,0x00,0x00,0x30,0x02,0x00,0x00, +0x79,0x02,0x00,0x00,0x10,0x00,0x06,0x00,0x04,0x00,0x00,0x00, 0x11,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00, 0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x0b,0x00,0x00,0x00, 0x0b,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x48,0x00,0x05,0x00, @@ -33231,28 +18251,18156 @@ unsigned char matmul_f32_aligned_l_fp32_data[] = { 0x47,0x00,0x04,0x00,0x62,0x00,0x00,0x00,0x01,0x00,0x00,0x00, 0x07,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x6c,0x00,0x00,0x00, 0x01,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x9d,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0xaf,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x05,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xb2,0x00,0x00,0x00, +0xa7,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0xb9,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x05,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xbc,0x00,0x00,0x00, 0x01,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0xf5,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x10,0x00,0x00,0x00, -0x48,0x00,0x04,0x00,0xf6,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0xf6,0x00,0x00,0x00, +0x00,0x01,0x00,0x00,0x06,0x00,0x00,0x00,0x10,0x00,0x00,0x00, +0x48,0x00,0x04,0x00,0x01,0x01,0x00,0x00,0x00,0x00,0x00,0x00, +0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x01,0x01,0x00,0x00, 0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x03,0x00,0xf6,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0xf8,0x00,0x00,0x00,0x22,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xf8,0x00,0x00,0x00, +0x47,0x00,0x03,0x00,0x01,0x01,0x00,0x00,0x02,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x03,0x01,0x00,0x00,0x22,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x03,0x01,0x00,0x00, 0x21,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x27,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x28,0x01,0x00,0x00,0x0b,0x00,0x00,0x00, -0x19,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x50,0x01,0x00,0x00, +0x1d,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x1e,0x01,0x00,0x00,0x0b,0x00,0x00,0x00, +0x19,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x48,0x01,0x00,0x00, 0x06,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x48,0x00,0x04,0x00, -0x51,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x51,0x01,0x00,0x00,0x00,0x00,0x00,0x00, +0x49,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x49,0x01,0x00,0x00,0x00,0x00,0x00,0x00, 0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, -0x51,0x01,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x53,0x01,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x53,0x01,0x00,0x00,0x21,0x00,0x00,0x00, +0x49,0x01,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x4b,0x01,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x4b,0x01,0x00,0x00,0x21,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x30,0x02,0x00,0x00, +0x0b,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x76,0x02,0x00,0x00,0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x48,0x00,0x04,0x00,0x77,0x02,0x00,0x00,0x00,0x00,0x00,0x00, +0x19,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x77,0x02,0x00,0x00, +0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x03,0x00,0x77,0x02,0x00,0x00,0x02,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x79,0x02,0x00,0x00,0x22,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x79,0x02,0x00,0x00, +0x21,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x13,0x00,0x02,0x00, +0x02,0x00,0x00,0x00,0x21,0x00,0x03,0x00,0x03,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x15,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x17,0x00,0x04,0x00, +0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x03,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x0a,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x09,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00, +0x0b,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x0d,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x1e,0x00,0x10,0x00,0x10,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x11,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x10,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0x11,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0x09,0x00,0x00,0x00,0x15,0x00,0x04,0x00,0x13,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x08,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x15,0x00,0x00,0x00,0x09,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, +0x21,0x00,0x00,0x00,0x0a,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0x27,0x00,0x00,0x00,0x09,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x2d,0x00,0x00,0x00, +0x07,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, +0x34,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x32,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x40,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00, +0x3d,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x3e,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00,0x4c,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x4f,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x32,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x54,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x53,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x58,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x53,0x00,0x00,0x00, +0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x60,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x61,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x53,0x00,0x00,0x00, +0x60,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x62,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x63,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0x61,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x67,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0x61,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x32,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x10,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x6e,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, +0x6d,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x73,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, +0x6d,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x78,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, +0x6d,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x7d,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, +0x6d,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, +0x81,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x91,0x00,0x00,0x00, +0x0b,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, +0x97,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0xa2,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xa7,0x00,0x00,0x00, +0x40,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, +0xa9,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xb8,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x32,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xb9,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xba,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0xb9,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xbb,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x62,0x00,0x00,0x00, +0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xbc,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xbd,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0xbb,0x00,0x00,0x00, +0xbc,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xbe,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0xbd,0x00,0x00,0x00, +0x60,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xbf,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0xba,0x00,0x00,0x00, +0xbe,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xc0,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0xb8,0x00,0x00,0x00, +0xbf,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xc1,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0xc0,0x00,0x00,0x00, +0xbc,0x00,0x00,0x00,0x14,0x00,0x02,0x00,0xc2,0x00,0x00,0x00, +0x16,0x00,0x03,0x00,0xc4,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xc5,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xc6,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0xc5,0x00,0x00,0x00,0xbf,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xc7,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0xc6,0x00,0x00,0x00,0xbc,0x00,0x00,0x00, +0x1c,0x00,0x04,0x00,0xc8,0x00,0x00,0x00,0xc4,0x00,0x00,0x00, +0xc7,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0xc9,0x00,0x00,0x00, +0x07,0x00,0x00,0x00,0xc8,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0xc4,0x00,0x00,0x00,0xcc,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0xcd,0x00,0x00,0x00,0x07,0x00,0x00,0x00, +0xc4,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, +0xd0,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xf4,0x00,0x00,0x00,0x80,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xf9,0x00,0x00,0x00,0x80,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xfa,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0x37,0x00,0x00,0x00,0xf9,0x00,0x00,0x00,0x1c,0x00,0x04,0x00, +0xfb,0x00,0x00,0x00,0xc4,0x00,0x00,0x00,0xfa,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0xfc,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0xfb,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0xfc,0x00,0x00,0x00, +0xfd,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x17,0x00,0x04,0x00, +0xff,0x00,0x00,0x00,0xc4,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x1d,0x00,0x03,0x00,0x00,0x01,0x00,0x00,0xff,0x00,0x00,0x00, +0x1e,0x00,0x03,0x00,0x01,0x01,0x00,0x00,0x00,0x01,0x00,0x00, +0x20,0x00,0x04,0x00,0x02,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, +0x01,0x01,0x00,0x00,0x3b,0x00,0x04,0x00,0x02,0x01,0x00,0x00, +0x03,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x05,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0xc4,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x08,0x01,0x00,0x00,0x04,0x00,0x00,0x00, +0xc4,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x17,0x01,0x00,0x00,0x03,0x00,0x00,0x00,0x32,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x1d,0x01,0x00,0x00,0x01,0x00,0x00,0x00, +0x33,0x00,0x06,0x00,0x09,0x00,0x00,0x00,0x1e,0x01,0x00,0x00, +0x1d,0x01,0x00,0x00,0x39,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x1f,0x01,0x00,0x00, +0x51,0x00,0x00,0x00,0x1e,0x01,0x00,0x00,0x00,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x20,0x01,0x00,0x00, +0x84,0x00,0x00,0x00,0x1f,0x01,0x00,0x00,0x6d,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x21,0x01,0x00,0x00, +0x86,0x00,0x00,0x00,0x20,0x01,0x00,0x00,0x6c,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x3c,0x01,0x00,0x00, +0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x41,0x01,0x00,0x00, +0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x42,0x01,0x00,0x00, +0x84,0x00,0x00,0x00,0xa7,0x00,0x00,0x00,0x41,0x01,0x00,0x00, +0x1c,0x00,0x04,0x00,0x43,0x01,0x00,0x00,0xc4,0x00,0x00,0x00, +0x42,0x01,0x00,0x00,0x20,0x00,0x04,0x00,0x44,0x01,0x00,0x00, +0x04,0x00,0x00,0x00,0x43,0x01,0x00,0x00,0x3b,0x00,0x04,0x00, +0x44,0x01,0x00,0x00,0x45,0x01,0x00,0x00,0x04,0x00,0x00,0x00, +0x1d,0x00,0x03,0x00,0x48,0x01,0x00,0x00,0xff,0x00,0x00,0x00, +0x1e,0x00,0x03,0x00,0x49,0x01,0x00,0x00,0x48,0x01,0x00,0x00, +0x20,0x00,0x04,0x00,0x4a,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, +0x49,0x01,0x00,0x00,0x3b,0x00,0x04,0x00,0x4a,0x01,0x00,0x00, +0x4b,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x62,0x01,0x00,0x00,0x51,0x00,0x00,0x00, +0x1e,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x63,0x01,0x00,0x00,0x84,0x00,0x00,0x00, +0x62,0x01,0x00,0x00,0x6d,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x64,0x01,0x00,0x00,0x86,0x00,0x00,0x00, +0x63,0x01,0x00,0x00,0x6c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x67,0x01,0x00,0x00,0x08,0x01,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x68,0x01,0x00,0x00, +0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x6b,0x01,0x00,0x00, +0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x86,0x01,0x00,0x00, +0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00, +0x1c,0x00,0x04,0x00,0x87,0x01,0x00,0x00,0xc4,0x00,0x00,0x00, +0x86,0x01,0x00,0x00,0x20,0x00,0x04,0x00,0x88,0x01,0x00,0x00, +0x07,0x00,0x00,0x00,0x87,0x01,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x98,0x01,0x00,0x00,0x80,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xb3,0x01,0x00,0x00,0x84,0x00,0x00,0x00, +0xbf,0x00,0x00,0x00,0xbc,0x00,0x00,0x00,0x1c,0x00,0x04,0x00, +0xb4,0x01,0x00,0x00,0xc4,0x00,0x00,0x00,0xb3,0x01,0x00,0x00, +0x20,0x00,0x04,0x00,0xb5,0x01,0x00,0x00,0x07,0x00,0x00,0x00, +0xb4,0x01,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xbe,0x01,0x00,0x00,0x86,0x00,0x00,0x00,0xb9,0x00,0x00,0x00, +0xbf,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xc6,0x01,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, +0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xf5,0x01,0x00,0x00,0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00, +0x62,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, +0x28,0x02,0x00,0x00,0x0d,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0x0a,0x00,0x00,0x00,0x30,0x02,0x00,0x00,0x01,0x00,0x00,0x00, +0x1d,0x00,0x03,0x00,0x76,0x02,0x00,0x00,0xc4,0x00,0x00,0x00, +0x1e,0x00,0x03,0x00,0x77,0x02,0x00,0x00,0x76,0x02,0x00,0x00, +0x20,0x00,0x04,0x00,0x78,0x02,0x00,0x00,0x0c,0x00,0x00,0x00, +0x77,0x02,0x00,0x00,0x3b,0x00,0x04,0x00,0x78,0x02,0x00,0x00, +0x79,0x02,0x00,0x00,0x0c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0x7e,0x02,0x00,0x00,0x05,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x8b,0x02,0x00,0x00, +0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00, +0x36,0x00,0x05,0x00,0x02,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0x05,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0xc9,0x00,0x00,0x00, +0xca,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0x88,0x01,0x00,0x00,0x89,0x01,0x00,0x00,0x07,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0xb5,0x01,0x00,0x00,0xb6,0x01,0x00,0x00, +0x07,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00, +0x0e,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, +0x0e,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, +0x16,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x14,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x17,0x00,0x00,0x00, +0x16,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x18,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x17,0x00,0x00,0x00, +0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x1e,0x00,0x00,0x00, +0x0f,0x00,0x00,0x00,0x17,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x15,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0x21,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x24,0x00,0x00,0x00,0x18,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, +0x28,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x27,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x29,0x00,0x00,0x00, +0x28,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x2a,0x00,0x00,0x00,0x1e,0x00,0x00,0x00,0x29,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, +0x12,0x00,0x00,0x00,0x2d,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x2f,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x30,0x00,0x00,0x00, +0x24,0x00,0x00,0x00,0x2f,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x30,0x00,0x00,0x00, +0x2a,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, +0x35,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x34,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x36,0x00,0x00,0x00, +0x35,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x38,0x00,0x00,0x00,0x36,0x00,0x00,0x00,0x37,0x00,0x00,0x00, +0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, +0x38,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x3b,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, +0x37,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00, +0x3f,0x00,0x00,0x00,0x3d,0x00,0x00,0x00,0x3e,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x40,0x00,0x00,0x00, +0x3f,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x42,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x3b,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x47,0x00,0x00,0x00, +0x40,0x00,0x00,0x00,0x3b,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x0d,0x00,0x00,0x00,0x49,0x00,0x00,0x00,0x3d,0x00,0x00,0x00, +0x39,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x4a,0x00,0x00,0x00,0x49,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x0d,0x00,0x00,0x00,0x4d,0x00,0x00,0x00,0x4c,0x00,0x00,0x00, +0x3e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x4e,0x00,0x00,0x00,0x4d,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x50,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, +0x4f,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x55,0x00,0x00,0x00,0x50,0x00,0x00,0x00,0x54,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x59,0x00,0x00,0x00, +0x50,0x00,0x00,0x00,0x58,0x00,0x00,0x00,0x89,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x5d,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, +0x4f,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x64,0x00,0x00,0x00,0x5d,0x00,0x00,0x00,0x63,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x68,0x00,0x00,0x00, +0x5d,0x00,0x00,0x00,0x67,0x00,0x00,0x00,0x89,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x6f,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, +0x6e,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x74,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x73,0x00,0x00,0x00, +0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x79,0x00,0x00,0x00, +0x4e,0x00,0x00,0x00,0x78,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x7e,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, +0x7d,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, +0x82,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x81,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x83,0x00,0x00,0x00, +0x82,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0x47,0x00,0x00,0x00,0x83,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x87,0x00,0x00,0x00, +0x12,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x88,0x00,0x00,0x00,0x87,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8a,0x00,0x00,0x00, +0x47,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x8d,0x00,0x00,0x00,0x8a,0x00,0x00,0x00, +0x83,0x00,0x00,0x00,0x0c,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0x8e,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x26,0x00,0x00,0x00, +0x88,0x00,0x00,0x00,0x8d,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x15,0x00,0x00,0x00,0x92,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0x91,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x93,0x00,0x00,0x00,0x92,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x94,0x00,0x00,0x00,0x32,0x00,0x00,0x00, +0x93,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x96,0x00,0x00,0x00,0x42,0x00,0x00,0x00,0x37,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x98,0x00,0x00,0x00, +0x12,0x00,0x00,0x00,0x97,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x99,0x00,0x00,0x00,0x98,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9a,0x00,0x00,0x00, +0x96,0x00,0x00,0x00,0x99,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x9b,0x00,0x00,0x00,0x94,0x00,0x00,0x00, +0x9a,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x9d,0x00,0x00,0x00,0x9b,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9e,0x00,0x00,0x00, +0x9d,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x15,0x00,0x00,0x00,0xa3,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0xa2,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xa4,0x00,0x00,0x00,0xa3,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xa5,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, +0xa4,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xa8,0x00,0x00,0x00,0x4a,0x00,0x00,0x00,0xa7,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0xaa,0x00,0x00,0x00, +0x12,0x00,0x00,0x00,0xa9,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xab,0x00,0x00,0x00,0xaa,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xac,0x00,0x00,0x00, +0xa8,0x00,0x00,0x00,0xab,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xad,0x00,0x00,0x00,0xa5,0x00,0x00,0x00, +0xac,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xaf,0x00,0x00,0x00,0xad,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xb0,0x00,0x00,0x00, +0xaf,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xb2,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xb2,0x00,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x9d,0x02,0x00,0x00, +0x3e,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0xd1,0x00,0x00,0x00, +0xb3,0x00,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, +0xc3,0x00,0x00,0x00,0x9d,0x02,0x00,0x00,0xc1,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0xb4,0x00,0x00,0x00,0xb3,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xc3,0x00,0x00,0x00, +0xb3,0x00,0x00,0x00,0xb4,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xb3,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0xcd,0x00,0x00,0x00, +0xce,0x00,0x00,0x00,0xca,0x00,0x00,0x00,0x9d,0x02,0x00,0x00, +0x3e,0x00,0x03,0x00,0xce,0x00,0x00,0x00,0xcc,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xd1,0x00,0x00,0x00, +0x9d,0x02,0x00,0x00,0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xb2,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xb4,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xd4,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xd4,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xb6,0x02,0x00,0x00,0xb0,0x00,0x00,0x00,0xb4,0x00,0x00,0x00, +0x6d,0x01,0x00,0x00,0xd7,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xb2,0x02,0x00,0x00,0x9e,0x00,0x00,0x00, +0xb4,0x00,0x00,0x00,0x6a,0x01,0x00,0x00,0xd7,0x00,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x9e,0x02,0x00,0x00, +0x84,0x00,0x00,0x00,0xb4,0x00,0x00,0x00,0x18,0x02,0x00,0x00, +0xd7,0x00,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, +0xdb,0x00,0x00,0x00,0x9e,0x02,0x00,0x00,0x8e,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0xd6,0x00,0x00,0x00,0xd7,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xdb,0x00,0x00,0x00, +0xd5,0x00,0x00,0x00,0xd6,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xd5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xdd,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xdd,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xae,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, +0xd5,0x00,0x00,0x00,0x23,0x01,0x00,0x00,0xde,0x00,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0xe3,0x00,0x00,0x00, +0xae,0x02,0x00,0x00,0x37,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0xdf,0x00,0x00,0x00,0xde,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0xe3,0x00,0x00,0x00,0xde,0x00,0x00,0x00, +0xdf,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xde,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xe8,0x00,0x00,0x00, +0x74,0x00,0x00,0x00,0xae,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xeb,0x00,0x00,0x00,0xe8,0x00,0x00,0x00, +0x99,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xec,0x00,0x00,0x00,0xeb,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xed,0x00,0x00,0x00, +0xb2,0x02,0x00,0x00,0xec,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xef,0x00,0x00,0x00,0xed,0x00,0x00,0x00, +0x6f,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xf5,0x00,0x00,0x00,0xe8,0x00,0x00,0x00,0xf4,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xf7,0x00,0x00,0x00, +0x6f,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xf8,0x00,0x00,0x00,0xf5,0x00,0x00,0x00, +0xf7,0x00,0x00,0x00,0x41,0x00,0x07,0x00,0x05,0x01,0x00,0x00, +0x06,0x01,0x00,0x00,0x03,0x01,0x00,0x00,0x34,0x00,0x00,0x00, +0xef,0x00,0x00,0x00,0x3e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0xc4,0x00,0x00,0x00,0x07,0x01,0x00,0x00,0x06,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0x08,0x01,0x00,0x00,0x09,0x01,0x00,0x00, +0xfd,0x00,0x00,0x00,0xf8,0x00,0x00,0x00,0x3e,0x00,0x03,0x00, +0x09,0x01,0x00,0x00,0x07,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x0b,0x01,0x00,0x00,0xf8,0x00,0x00,0x00, +0x39,0x00,0x00,0x00,0x41,0x00,0x07,0x00,0x05,0x01,0x00,0x00, +0x0d,0x01,0x00,0x00,0x03,0x01,0x00,0x00,0x34,0x00,0x00,0x00, +0xef,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0xc4,0x00,0x00,0x00,0x0e,0x01,0x00,0x00,0x0d,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0x08,0x01,0x00,0x00,0x0f,0x01,0x00,0x00, +0xfd,0x00,0x00,0x00,0x0b,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, +0x0f,0x01,0x00,0x00,0x0e,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x11,0x01,0x00,0x00,0xf8,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x41,0x00,0x07,0x00,0x05,0x01,0x00,0x00, +0x13,0x01,0x00,0x00,0x03,0x01,0x00,0x00,0x34,0x00,0x00,0x00, +0xef,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0xc4,0x00,0x00,0x00,0x14,0x01,0x00,0x00,0x13,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0x08,0x01,0x00,0x00,0x15,0x01,0x00,0x00, +0xfd,0x00,0x00,0x00,0x11,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, +0x15,0x01,0x00,0x00,0x14,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x18,0x01,0x00,0x00,0xf8,0x00,0x00,0x00, +0x17,0x01,0x00,0x00,0x41,0x00,0x07,0x00,0x05,0x01,0x00,0x00, +0x1a,0x01,0x00,0x00,0x03,0x01,0x00,0x00,0x34,0x00,0x00,0x00, +0xef,0x00,0x00,0x00,0x17,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, +0xc4,0x00,0x00,0x00,0x1b,0x01,0x00,0x00,0x1a,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0x08,0x01,0x00,0x00,0x1c,0x01,0x00,0x00, +0xfd,0x00,0x00,0x00,0x18,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, +0x1c,0x01,0x00,0x00,0x1b,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x23,0x01,0x00,0x00,0xae,0x02,0x00,0x00, +0x21,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xdd,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xdf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x25,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x25,0x01,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xaf,0x02,0x00,0x00, +0x3e,0x00,0x00,0x00,0xdf,0x00,0x00,0x00,0x66,0x01,0x00,0x00, +0x26,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, +0x2b,0x01,0x00,0x00,0xaf,0x02,0x00,0x00,0xa7,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0x27,0x01,0x00,0x00,0x26,0x01,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x2b,0x01,0x00,0x00, +0x26,0x01,0x00,0x00,0x27,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x26,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x30,0x01,0x00,0x00,0x7e,0x00,0x00,0x00,0xaf,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x33,0x01,0x00,0x00, +0x30,0x01,0x00,0x00,0xab,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x34,0x01,0x00,0x00,0x33,0x01,0x00,0x00, +0x6d,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x35,0x01,0x00,0x00,0xb6,0x02,0x00,0x00,0x34,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x37,0x01,0x00,0x00, +0x35,0x01,0x00,0x00,0x79,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x3d,0x01,0x00,0x00,0x30,0x01,0x00,0x00, +0x3c,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x3f,0x01,0x00,0x00,0x79,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x40,0x01,0x00,0x00, +0x3d,0x01,0x00,0x00,0x3f,0x01,0x00,0x00,0x41,0x00,0x07,0x00, +0x05,0x01,0x00,0x00,0x4d,0x01,0x00,0x00,0x4b,0x01,0x00,0x00, +0x34,0x00,0x00,0x00,0x37,0x01,0x00,0x00,0x3e,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00,0x4e,0x01,0x00,0x00, +0x4d,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0x08,0x01,0x00,0x00, +0x4f,0x01,0x00,0x00,0x45,0x01,0x00,0x00,0x40,0x01,0x00,0x00, +0x3e,0x00,0x03,0x00,0x4f,0x01,0x00,0x00,0x4e,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x51,0x01,0x00,0x00, +0x40,0x01,0x00,0x00,0x39,0x00,0x00,0x00,0x41,0x00,0x07,0x00, +0x05,0x01,0x00,0x00,0x53,0x01,0x00,0x00,0x4b,0x01,0x00,0x00, +0x34,0x00,0x00,0x00,0x37,0x01,0x00,0x00,0x39,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00,0x54,0x01,0x00,0x00, +0x53,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0x08,0x01,0x00,0x00, +0x55,0x01,0x00,0x00,0x45,0x01,0x00,0x00,0x51,0x01,0x00,0x00, +0x3e,0x00,0x03,0x00,0x55,0x01,0x00,0x00,0x54,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x57,0x01,0x00,0x00, +0x40,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x41,0x00,0x07,0x00, +0x05,0x01,0x00,0x00,0x59,0x01,0x00,0x00,0x4b,0x01,0x00,0x00, +0x34,0x00,0x00,0x00,0x37,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00,0x5a,0x01,0x00,0x00, +0x59,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0x08,0x01,0x00,0x00, +0x5b,0x01,0x00,0x00,0x45,0x01,0x00,0x00,0x57,0x01,0x00,0x00, +0x3e,0x00,0x03,0x00,0x5b,0x01,0x00,0x00,0x5a,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x5d,0x01,0x00,0x00, +0x40,0x01,0x00,0x00,0x17,0x01,0x00,0x00,0x41,0x00,0x07,0x00, +0x05,0x01,0x00,0x00,0x5f,0x01,0x00,0x00,0x4b,0x01,0x00,0x00, +0x34,0x00,0x00,0x00,0x37,0x01,0x00,0x00,0x17,0x01,0x00,0x00, +0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00,0x60,0x01,0x00,0x00, +0x5f,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0x08,0x01,0x00,0x00, +0x61,0x01,0x00,0x00,0x45,0x01,0x00,0x00,0x5d,0x01,0x00,0x00, +0x3e,0x00,0x03,0x00,0x61,0x01,0x00,0x00,0x60,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x66,0x01,0x00,0x00, +0xaf,0x02,0x00,0x00,0x64,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0x25,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x27,0x01,0x00,0x00, +0xe0,0x00,0x04,0x00,0x0c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x67,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x6a,0x01,0x00,0x00,0xb2,0x02,0x00,0x00,0x68,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x6d,0x01,0x00,0x00, +0xb6,0x02,0x00,0x00,0x6b,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0x6f,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x6f,0x01,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xb8,0x02,0x00,0x00, +0x3e,0x00,0x00,0x00,0x27,0x01,0x00,0x00,0x16,0x02,0x00,0x00, +0x72,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, +0x75,0x01,0x00,0x00,0xb8,0x02,0x00,0x00,0x6c,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0x71,0x01,0x00,0x00,0x72,0x01,0x00,0x00, +0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x75,0x01,0x00,0x00, +0x70,0x01,0x00,0x00,0x71,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x70,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x77,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x77,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xbc,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, +0x70,0x01,0x00,0x00,0xa2,0x01,0x00,0x00,0x7a,0x01,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0x7d,0x01,0x00,0x00, +0xbc,0x02,0x00,0x00,0x60,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0x79,0x01,0x00,0x00,0x7a,0x01,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x7d,0x01,0x00,0x00,0x78,0x01,0x00,0x00, +0x79,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x78,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0x7f,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x7f,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xce,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0x78,0x01,0x00,0x00, +0xa0,0x01,0x00,0x00,0x80,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc2,0x00,0x00,0x00,0x85,0x01,0x00,0x00,0xce,0x02,0x00,0x00, +0x62,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x81,0x01,0x00,0x00, +0x80,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x85,0x01,0x00,0x00,0x80,0x01,0x00,0x00,0x81,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x80,0x01,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x8b,0x01,0x00,0x00,0xbc,0x02,0x00,0x00, +0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x8d,0x01,0x00,0x00,0x8b,0x01,0x00,0x00,0xce,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8f,0x01,0x00,0x00, +0x55,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x91,0x01,0x00,0x00,0xbc,0x02,0x00,0x00, +0x61,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x92,0x01,0x00,0x00,0x8f,0x01,0x00,0x00,0x91,0x01,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x94,0x01,0x00,0x00, +0x64,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x95,0x01,0x00,0x00,0x92,0x01,0x00,0x00, +0x94,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x97,0x01,0x00,0x00,0x95,0x01,0x00,0x00,0xce,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x99,0x01,0x00,0x00, +0x97,0x01,0x00,0x00,0x98,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x9b,0x01,0x00,0x00,0x99,0x01,0x00,0x00, +0xb8,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x08,0x01,0x00,0x00, +0x9c,0x01,0x00,0x00,0xfd,0x00,0x00,0x00,0x9b,0x01,0x00,0x00, +0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00,0x9d,0x01,0x00,0x00, +0x9c,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0xcd,0x00,0x00,0x00, +0x9e,0x01,0x00,0x00,0x89,0x01,0x00,0x00,0x8d,0x01,0x00,0x00, +0x3e,0x00,0x03,0x00,0x9e,0x01,0x00,0x00,0x9d,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa0,0x01,0x00,0x00, +0xce,0x02,0x00,0x00,0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x7f,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x81,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0x7a,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x7a,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xa2,0x01,0x00,0x00,0xbc,0x02,0x00,0x00,0xd0,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x77,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x79,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xa4,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xa4,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xbd,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, +0x79,0x01,0x00,0x00,0xd0,0x01,0x00,0x00,0xa7,0x01,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0xaa,0x01,0x00,0x00, +0xbd,0x02,0x00,0x00,0xbf,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0xa6,0x01,0x00,0x00,0xa7,0x01,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0xaa,0x01,0x00,0x00,0xa5,0x01,0x00,0x00, +0xa6,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xa5,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0xac,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xac,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xcb,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0xa5,0x01,0x00,0x00, +0xce,0x01,0x00,0x00,0xad,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc2,0x00,0x00,0x00,0xb2,0x01,0x00,0x00,0xcb,0x02,0x00,0x00, +0xbc,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xae,0x01,0x00,0x00, +0xad,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0xb2,0x01,0x00,0x00,0xad,0x01,0x00,0x00,0xae,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xad,0x01,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xb8,0x01,0x00,0x00,0xbd,0x02,0x00,0x00, +0xbc,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xba,0x01,0x00,0x00,0xb8,0x01,0x00,0x00,0xcb,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xbc,0x01,0x00,0x00, +0x59,0x00,0x00,0x00,0xb9,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xbf,0x01,0x00,0x00,0xbd,0x02,0x00,0x00, +0xbe,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xc0,0x01,0x00,0x00,0xbc,0x01,0x00,0x00,0xbf,0x01,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xc2,0x01,0x00,0x00, +0x68,0x00,0x00,0x00,0xbc,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xc3,0x01,0x00,0x00,0xc0,0x01,0x00,0x00, +0xc2,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xc5,0x01,0x00,0x00,0xc3,0x01,0x00,0x00,0xcb,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xc7,0x01,0x00,0x00, +0xc5,0x01,0x00,0x00,0xc6,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xc9,0x01,0x00,0x00,0xc7,0x01,0x00,0x00, +0xb8,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x08,0x01,0x00,0x00, +0xca,0x01,0x00,0x00,0x45,0x01,0x00,0x00,0xc9,0x01,0x00,0x00, +0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00,0xcb,0x01,0x00,0x00, +0xca,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0xcd,0x00,0x00,0x00, +0xcc,0x01,0x00,0x00,0xb6,0x01,0x00,0x00,0xba,0x01,0x00,0x00, +0x3e,0x00,0x03,0x00,0xcc,0x01,0x00,0x00,0xcb,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xce,0x01,0x00,0x00, +0xcb,0x02,0x00,0x00,0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xac,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xae,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0xa7,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xa7,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xd0,0x01,0x00,0x00,0xbd,0x02,0x00,0x00,0xd0,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xa4,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xa6,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xd2,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xd2,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xbe,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, +0xa6,0x01,0x00,0x00,0x14,0x02,0x00,0x00,0xd5,0x01,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0xd8,0x01,0x00,0x00, +0xbe,0x02,0x00,0x00,0xbf,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0xd4,0x01,0x00,0x00,0xd5,0x01,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0xd8,0x01,0x00,0x00,0xd3,0x01,0x00,0x00, +0xd4,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xd3,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0xda,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xda,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xc2,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0xd3,0x01,0x00,0x00, +0x12,0x02,0x00,0x00,0xdd,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc2,0x00,0x00,0x00,0xe0,0x01,0x00,0x00,0xc2,0x02,0x00,0x00, +0x60,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xdc,0x01,0x00,0x00, +0xdd,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0xe0,0x01,0x00,0x00,0xdb,0x01,0x00,0x00,0xdc,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xdb,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0xe2,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xe2,0x01,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xc4,0x02,0x00,0x00, +0x3e,0x00,0x00,0x00,0xdb,0x01,0x00,0x00,0x10,0x02,0x00,0x00, +0xe5,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, +0xe8,0x01,0x00,0x00,0xc4,0x02,0x00,0x00,0xbc,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0xe4,0x01,0x00,0x00,0xe5,0x01,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xe8,0x01,0x00,0x00, +0xe3,0x01,0x00,0x00,0xe4,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xe3,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xea,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xea,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xc6,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, +0xe3,0x01,0x00,0x00,0x0e,0x02,0x00,0x00,0xeb,0x01,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0xf0,0x01,0x00,0x00, +0xc6,0x02,0x00,0x00,0x62,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0xec,0x01,0x00,0x00,0xeb,0x01,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0xf0,0x01,0x00,0x00,0xeb,0x01,0x00,0x00, +0xec,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xeb,0x01,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xf2,0x01,0x00,0x00, +0xbe,0x02,0x00,0x00,0xbc,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xf4,0x01,0x00,0x00,0xf2,0x01,0x00,0x00, +0xc4,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xf6,0x01,0x00,0x00,0xf4,0x01,0x00,0x00,0xf5,0x01,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xf8,0x01,0x00,0x00, +0xc2,0x02,0x00,0x00,0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xf9,0x01,0x00,0x00,0xf6,0x01,0x00,0x00, +0xf8,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xfb,0x01,0x00,0x00,0xf9,0x01,0x00,0x00,0xc6,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xff,0x01,0x00,0x00, +0xf8,0x01,0x00,0x00,0xc6,0x02,0x00,0x00,0x41,0x00,0x05,0x00, +0xcd,0x00,0x00,0x00,0x00,0x02,0x00,0x00,0x89,0x01,0x00,0x00, +0xff,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00, +0x01,0x02,0x00,0x00,0x00,0x02,0x00,0x00,0x41,0x00,0x05,0x00, +0xcd,0x00,0x00,0x00,0x06,0x02,0x00,0x00,0xb6,0x01,0x00,0x00, +0xf4,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00, +0x07,0x02,0x00,0x00,0x06,0x02,0x00,0x00,0x41,0x00,0x05,0x00, +0xcd,0x00,0x00,0x00,0x09,0x02,0x00,0x00,0xca,0x00,0x00,0x00, +0xfb,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00, +0x0a,0x02,0x00,0x00,0x09,0x02,0x00,0x00,0x0c,0x00,0x08,0x00, +0xc4,0x00,0x00,0x00,0x0b,0x02,0x00,0x00,0x01,0x00,0x00,0x00, +0x32,0x00,0x00,0x00,0x01,0x02,0x00,0x00,0x07,0x02,0x00,0x00, +0x0a,0x02,0x00,0x00,0x3e,0x00,0x03,0x00,0x09,0x02,0x00,0x00, +0x0b,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x0e,0x02,0x00,0x00,0xc6,0x02,0x00,0x00,0xd0,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xea,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xec,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xe5,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xe5,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x10,0x02,0x00,0x00,0xc4,0x02,0x00,0x00, +0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xe2,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xe4,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0xdd,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xdd,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x12,0x02,0x00,0x00, +0xc2,0x02,0x00,0x00,0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xda,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xdc,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0xd5,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xd5,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x14,0x02,0x00,0x00,0xbe,0x02,0x00,0x00,0xd0,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xd2,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xd4,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x72,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x72,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x16,0x02,0x00,0x00,0xb8,0x02,0x00,0x00, +0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x6f,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x71,0x01,0x00,0x00,0xe0,0x00,0x04,0x00, +0x0c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x67,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0xd7,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xd7,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x18,0x02,0x00,0x00,0x9e,0x02,0x00,0x00,0x6c,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xd4,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xd6,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x1d,0x02,0x00,0x00,0x55,0x00,0x00,0x00,0x53,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x1e,0x02,0x00,0x00, +0x96,0x00,0x00,0x00,0x1d,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x23,0x02,0x00,0x00,0x59,0x00,0x00,0x00, +0xb9,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x24,0x02,0x00,0x00,0xa8,0x00,0x00,0x00,0x23,0x02,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x29,0x02,0x00,0x00, +0x12,0x00,0x00,0x00,0x28,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x2a,0x02,0x00,0x00,0x29,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x2b,0x02,0x00,0x00, +0x0f,0x00,0x00,0x00,0x2a,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x2f,0x02,0x00,0x00,0x47,0x00,0x00,0x00, +0x2a,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00, +0x31,0x02,0x00,0x00,0x30,0x02,0x00,0x00,0x0c,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x32,0x02,0x00,0x00, +0x31,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x33,0x02,0x00,0x00,0x2f,0x02,0x00,0x00,0x32,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x34,0x02,0x00,0x00, +0x2b,0x02,0x00,0x00,0x33,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0x36,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x36,0x02,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x9f,0x02,0x00,0x00, +0x3e,0x00,0x00,0x00,0xd6,0x00,0x00,0x00,0x9c,0x02,0x00,0x00, +0x39,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, +0x3c,0x02,0x00,0x00,0x9f,0x02,0x00,0x00,0xbf,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0x38,0x02,0x00,0x00,0x39,0x02,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x3c,0x02,0x00,0x00, +0x37,0x02,0x00,0x00,0x38,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x37,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x3e,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x3e,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xa0,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, +0x37,0x02,0x00,0x00,0x9a,0x02,0x00,0x00,0x41,0x02,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0x44,0x02,0x00,0x00, +0xa0,0x02,0x00,0x00,0x60,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0x40,0x02,0x00,0x00,0x41,0x02,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x44,0x02,0x00,0x00,0x3f,0x02,0x00,0x00, +0x40,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x3f,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x48,0x02,0x00,0x00, +0xa0,0x02,0x00,0x00,0x61,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x49,0x02,0x00,0x00,0x1e,0x02,0x00,0x00, +0x48,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x4b,0x02,0x00,0x00,0x64,0x00,0x00,0x00,0x62,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x4c,0x02,0x00,0x00, +0x49,0x02,0x00,0x00,0x4b,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x50,0x02,0x00,0x00,0x9f,0x02,0x00,0x00, +0xbe,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x51,0x02,0x00,0x00,0x24,0x02,0x00,0x00,0x50,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x53,0x02,0x00,0x00, +0x68,0x00,0x00,0x00,0xbc,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x54,0x02,0x00,0x00,0x51,0x02,0x00,0x00, +0x53,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x56,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x56,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xa2,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, +0x3f,0x02,0x00,0x00,0x98,0x02,0x00,0x00,0x59,0x02,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0x5c,0x02,0x00,0x00, +0xa2,0x02,0x00,0x00,0xbc,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0x58,0x02,0x00,0x00,0x59,0x02,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x5c,0x02,0x00,0x00,0x57,0x02,0x00,0x00, +0x58,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x57,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0x5e,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x5e,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xa4,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0x57,0x02,0x00,0x00, +0x96,0x02,0x00,0x00,0x61,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc2,0x00,0x00,0x00,0x64,0x02,0x00,0x00,0xa4,0x02,0x00,0x00, +0x62,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x60,0x02,0x00,0x00, +0x61,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x64,0x02,0x00,0x00,0x5f,0x02,0x00,0x00,0x60,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x5f,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x67,0x02,0x00,0x00,0x4c,0x02,0x00,0x00, +0xa4,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, +0x6a,0x02,0x00,0x00,0x67,0x02,0x00,0x00,0x36,0x00,0x00,0x00, +0xf7,0x00,0x03,0x00,0x6c,0x02,0x00,0x00,0x00,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x6a,0x02,0x00,0x00,0x6b,0x02,0x00,0x00, +0x6c,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x6b,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x6f,0x02,0x00,0x00, +0x54,0x02,0x00,0x00,0xa2,0x02,0x00,0x00,0x41,0x00,0x05,0x00, +0x15,0x00,0x00,0x00,0x70,0x02,0x00,0x00,0x12,0x00,0x00,0x00, +0xd0,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x71,0x02,0x00,0x00,0x70,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc2,0x00,0x00,0x00,0x72,0x02,0x00,0x00,0x6f,0x02,0x00,0x00, +0x71,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x6c,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x6c,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, +0xc2,0x00,0x00,0x00,0x73,0x02,0x00,0x00,0x6a,0x02,0x00,0x00, +0x5f,0x02,0x00,0x00,0x72,0x02,0x00,0x00,0x6b,0x02,0x00,0x00, +0xf7,0x00,0x03,0x00,0x75,0x02,0x00,0x00,0x00,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x73,0x02,0x00,0x00,0x74,0x02,0x00,0x00, +0x75,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x74,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x7d,0x02,0x00,0x00, +0x54,0x02,0x00,0x00,0xa2,0x02,0x00,0x00,0x41,0x00,0x05,0x00, +0x15,0x00,0x00,0x00,0x7f,0x02,0x00,0x00,0x12,0x00,0x00,0x00, +0x7e,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x80,0x02,0x00,0x00,0x7f,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x81,0x02,0x00,0x00,0x7d,0x02,0x00,0x00, +0x80,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x82,0x02,0x00,0x00,0x34,0x02,0x00,0x00,0x81,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x84,0x02,0x00,0x00, +0x82,0x02,0x00,0x00,0x4c,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x86,0x02,0x00,0x00,0x84,0x02,0x00,0x00, +0xa4,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x88,0x02,0x00,0x00,0x9f,0x02,0x00,0x00,0xbc,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8a,0x02,0x00,0x00, +0x88,0x02,0x00,0x00,0xa2,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x8c,0x02,0x00,0x00,0x8a,0x02,0x00,0x00, +0x8b,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x8e,0x02,0x00,0x00,0xa0,0x02,0x00,0x00,0x62,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8f,0x02,0x00,0x00, +0x8c,0x02,0x00,0x00,0x8e,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x91,0x02,0x00,0x00,0x8f,0x02,0x00,0x00, +0xa4,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0xcd,0x00,0x00,0x00, +0x92,0x02,0x00,0x00,0xca,0x00,0x00,0x00,0x91,0x02,0x00,0x00, +0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00,0x93,0x02,0x00,0x00, +0x92,0x02,0x00,0x00,0x41,0x00,0x06,0x00,0x05,0x01,0x00,0x00, +0x94,0x02,0x00,0x00,0x79,0x02,0x00,0x00,0x34,0x00,0x00,0x00, +0x86,0x02,0x00,0x00,0x3e,0x00,0x03,0x00,0x94,0x02,0x00,0x00, +0x93,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x75,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x75,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0x61,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x61,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x96,0x02,0x00,0x00, +0xa4,0x02,0x00,0x00,0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x5e,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x60,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0x59,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x59,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x98,0x02,0x00,0x00,0xa2,0x02,0x00,0x00,0xd0,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x56,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x58,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x41,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x41,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x9a,0x02,0x00,0x00,0xa0,0x02,0x00,0x00, +0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x3e,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x40,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0x39,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x39,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9c,0x02,0x00,0x00, +0x9f,0x02,0x00,0x00,0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x36,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x38,0x02,0x00,0x00, +0xfd,0x00,0x01,0x00,0x38,0x00,0x01,0x00, +}; +const uint64_t matmul_f32_aligned_fp32_len = 10124; + +unsigned char matmul_f32_fp32_data[] = { +0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, +0xd3,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, +0x01,0x00,0x00,0x00,0x0b,0x00,0x06,0x00,0x01,0x00,0x00,0x00, +0x47,0x4c,0x53,0x4c,0x2e,0x73,0x74,0x64,0x2e,0x34,0x35,0x30, +0x00,0x00,0x00,0x00,0x0e,0x00,0x03,0x00,0x00,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x0f,0x00,0x0f,0x00,0x05,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x6d,0x61,0x69,0x6e,0x00,0x00,0x00,0x00, +0x0b,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x3d,0x00,0x00,0x00, +0x4c,0x00,0x00,0x00,0xfa,0x00,0x00,0x00,0x05,0x01,0x00,0x00, +0x44,0x01,0x00,0x00,0x4f,0x01,0x00,0x00,0x34,0x02,0x00,0x00, +0x7d,0x02,0x00,0x00,0x10,0x00,0x06,0x00,0x04,0x00,0x00,0x00, +0x11,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x0b,0x00,0x00,0x00, +0x0b,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x10,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x10,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x10,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x05,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x10,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, +0x07,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x1c,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x08,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x10,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x24,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, +0x0a,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x28,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x2c,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x10,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x30,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, +0x0d,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x34,0x00,0x00,0x00, +0x47,0x00,0x03,0x00,0x10,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x37,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x3d,0x00,0x00,0x00, +0x0b,0x00,0x00,0x00,0x1a,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x4c,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x1b,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x4f,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x09,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x53,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x60,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x62,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x07,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x6c,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0xa6,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0xb8,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x05,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xbb,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x02,0x01,0x00,0x00,0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x48,0x00,0x04,0x00,0x03,0x01,0x00,0x00,0x00,0x00,0x00,0x00, +0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x03,0x01,0x00,0x00, +0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x03,0x00,0x03,0x01,0x00,0x00,0x02,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x05,0x01,0x00,0x00,0x22,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x05,0x01,0x00,0x00, +0x21,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x1e,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x1f,0x01,0x00,0x00,0x0b,0x00,0x00,0x00, +0x19,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x4c,0x01,0x00,0x00, +0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00, +0x4d,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x4d,0x01,0x00,0x00,0x00,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, +0x4d,0x01,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x4f,0x01,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x4f,0x01,0x00,0x00,0x21,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x34,0x02,0x00,0x00, +0x0b,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x7a,0x02,0x00,0x00,0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x48,0x00,0x04,0x00,0x7b,0x02,0x00,0x00,0x00,0x00,0x00,0x00, +0x19,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x7b,0x02,0x00,0x00, +0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x03,0x00,0x7b,0x02,0x00,0x00,0x02,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x7d,0x02,0x00,0x00,0x22,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x7d,0x02,0x00,0x00, +0x21,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x13,0x00,0x02,0x00, +0x02,0x00,0x00,0x00,0x21,0x00,0x03,0x00,0x03,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x15,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x17,0x00,0x04,0x00, +0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x03,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x0a,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x09,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00, +0x0b,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x0d,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x1e,0x00,0x10,0x00,0x10,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x11,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x10,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0x11,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0x09,0x00,0x00,0x00,0x15,0x00,0x04,0x00,0x13,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x08,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x15,0x00,0x00,0x00,0x09,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, +0x21,0x00,0x00,0x00,0x0a,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0x27,0x00,0x00,0x00,0x09,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x2d,0x00,0x00,0x00, +0x07,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, +0x34,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x32,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x40,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00, +0x3d,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x3e,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00,0x4c,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x4f,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x32,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x54,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x53,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x58,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x53,0x00,0x00,0x00, +0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x60,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x61,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x53,0x00,0x00,0x00, +0x60,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x62,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x63,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0x61,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x67,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0x61,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x32,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x10,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x72,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x77,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x7c,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x80,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, +0x85,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0x90,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x96,0x00,0x00,0x00, +0x03,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, +0xa1,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x32,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xa6,0x00,0x00,0x00,0x40,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0xa8,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xb7,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00, +0x62,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xb8,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xb9,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0x53,0x00,0x00,0x00,0xb8,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xba,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0x4f,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x32,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xbb,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xbc,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0xba,0x00,0x00,0x00,0xbb,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xbd,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0xbc,0x00,0x00,0x00,0x60,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xbe,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0xb9,0x00,0x00,0x00,0xbd,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xbf,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0xb7,0x00,0x00,0x00,0xbe,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xc0,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0xbf,0x00,0x00,0x00,0xbb,0x00,0x00,0x00, +0x14,0x00,0x02,0x00,0xc1,0x00,0x00,0x00,0x16,0x00,0x03,0x00, +0xc3,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xc4,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xc5,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0xc4,0x00,0x00,0x00,0xbe,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xc6,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0xc5,0x00,0x00,0x00,0xbb,0x00,0x00,0x00,0x1c,0x00,0x04,0x00, +0xc7,0x00,0x00,0x00,0xc3,0x00,0x00,0x00,0xc6,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0xc8,0x00,0x00,0x00,0x07,0x00,0x00,0x00, +0xc7,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0xc3,0x00,0x00,0x00, +0xcb,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0xcc,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0xc3,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0xcf,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xf6,0x00,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, +0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xf7,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x37,0x00,0x00,0x00, +0xf6,0x00,0x00,0x00,0x1c,0x00,0x04,0x00,0xf8,0x00,0x00,0x00, +0xc3,0x00,0x00,0x00,0xf7,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0xf9,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0xf8,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0xf9,0x00,0x00,0x00,0xfa,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xfe,0x00,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, +0x39,0x00,0x00,0x00,0x1d,0x00,0x03,0x00,0x02,0x01,0x00,0x00, +0xc3,0x00,0x00,0x00,0x1e,0x00,0x03,0x00,0x03,0x01,0x00,0x00, +0x02,0x01,0x00,0x00,0x20,0x00,0x04,0x00,0x04,0x01,0x00,0x00, +0x0c,0x00,0x00,0x00,0x03,0x01,0x00,0x00,0x3b,0x00,0x04,0x00, +0x04,0x01,0x00,0x00,0x05,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x10,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, +0xc3,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x13,0x01,0x00,0x00, +0x04,0x00,0x00,0x00,0xc3,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x19,0x01,0x00,0x00,0x80,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x32,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x1e,0x01,0x00,0x00,0x01,0x00,0x00,0x00, +0x33,0x00,0x06,0x00,0x09,0x00,0x00,0x00,0x1f,0x01,0x00,0x00, +0x1e,0x01,0x00,0x00,0x39,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x20,0x01,0x00,0x00, +0x51,0x00,0x00,0x00,0x1f,0x01,0x00,0x00,0x00,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x21,0x01,0x00,0x00, +0x84,0x00,0x00,0x00,0x20,0x01,0x00,0x00,0x39,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x22,0x01,0x00,0x00, +0x86,0x00,0x00,0x00,0x21,0x01,0x00,0x00,0x6c,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x40,0x01,0x00,0x00, +0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x41,0x01,0x00,0x00, +0x84,0x00,0x00,0x00,0xa6,0x00,0x00,0x00,0x40,0x01,0x00,0x00, +0x1c,0x00,0x04,0x00,0x42,0x01,0x00,0x00,0xc3,0x00,0x00,0x00, +0x41,0x01,0x00,0x00,0x20,0x00,0x04,0x00,0x43,0x01,0x00,0x00, +0x04,0x00,0x00,0x00,0x42,0x01,0x00,0x00,0x3b,0x00,0x04,0x00, +0x43,0x01,0x00,0x00,0x44,0x01,0x00,0x00,0x04,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x48,0x01,0x00,0x00, +0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0x1d,0x00,0x03,0x00,0x4c,0x01,0x00,0x00,0xc3,0x00,0x00,0x00, +0x1e,0x00,0x03,0x00,0x4d,0x01,0x00,0x00,0x4c,0x01,0x00,0x00, +0x20,0x00,0x04,0x00,0x4e,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, +0x4d,0x01,0x00,0x00,0x3b,0x00,0x04,0x00,0x4e,0x01,0x00,0x00, +0x4f,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x61,0x01,0x00,0x00,0x80,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x66,0x01,0x00,0x00,0x51,0x00,0x00,0x00, +0x1f,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x67,0x01,0x00,0x00,0x84,0x00,0x00,0x00, +0x66,0x01,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x68,0x01,0x00,0x00,0x86,0x00,0x00,0x00, +0x67,0x01,0x00,0x00,0x6c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x6b,0x01,0x00,0x00,0x08,0x01,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x6c,0x01,0x00,0x00, +0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x6f,0x01,0x00,0x00, +0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x8a,0x01,0x00,0x00, +0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00, +0x1c,0x00,0x04,0x00,0x8b,0x01,0x00,0x00,0xc3,0x00,0x00,0x00, +0x8a,0x01,0x00,0x00,0x20,0x00,0x04,0x00,0x8c,0x01,0x00,0x00, +0x07,0x00,0x00,0x00,0x8b,0x01,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x9c,0x01,0x00,0x00,0x80,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xb7,0x01,0x00,0x00,0x84,0x00,0x00,0x00, +0xbe,0x00,0x00,0x00,0xbb,0x00,0x00,0x00,0x1c,0x00,0x04,0x00, +0xb8,0x01,0x00,0x00,0xc3,0x00,0x00,0x00,0xb7,0x01,0x00,0x00, +0x20,0x00,0x04,0x00,0xb9,0x01,0x00,0x00,0x07,0x00,0x00,0x00, +0xb8,0x01,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xc2,0x01,0x00,0x00,0x86,0x00,0x00,0x00,0xb8,0x00,0x00,0x00, +0xbe,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xca,0x01,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, +0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xf9,0x01,0x00,0x00,0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00, +0x62,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, +0x2c,0x02,0x00,0x00,0x0d,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0x0a,0x00,0x00,0x00,0x34,0x02,0x00,0x00,0x01,0x00,0x00,0x00, +0x1d,0x00,0x03,0x00,0x7a,0x02,0x00,0x00,0xc3,0x00,0x00,0x00, +0x1e,0x00,0x03,0x00,0x7b,0x02,0x00,0x00,0x7a,0x02,0x00,0x00, +0x20,0x00,0x04,0x00,0x7c,0x02,0x00,0x00,0x0c,0x00,0x00,0x00, +0x7b,0x02,0x00,0x00,0x3b,0x00,0x04,0x00,0x7c,0x02,0x00,0x00, +0x7d,0x02,0x00,0x00,0x0c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0x82,0x02,0x00,0x00,0x05,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x8f,0x02,0x00,0x00, +0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00, +0x36,0x00,0x05,0x00,0x02,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0x05,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0xc8,0x00,0x00,0x00, +0xc9,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0x8c,0x01,0x00,0x00,0x8d,0x01,0x00,0x00,0x07,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0xb9,0x01,0x00,0x00,0xba,0x01,0x00,0x00, +0x07,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00, +0x0e,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, +0x0e,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, +0x16,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x14,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x17,0x00,0x00,0x00, +0x16,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x18,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x17,0x00,0x00,0x00, +0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x1e,0x00,0x00,0x00, +0x0f,0x00,0x00,0x00,0x17,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x15,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0x21,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x24,0x00,0x00,0x00,0x18,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, +0x28,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x27,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x29,0x00,0x00,0x00, +0x28,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x2a,0x00,0x00,0x00,0x1e,0x00,0x00,0x00,0x29,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, +0x12,0x00,0x00,0x00,0x2d,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x2f,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x30,0x00,0x00,0x00, +0x24,0x00,0x00,0x00,0x2f,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x30,0x00,0x00,0x00, +0x2a,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, +0x35,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x34,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x36,0x00,0x00,0x00, +0x35,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x38,0x00,0x00,0x00,0x36,0x00,0x00,0x00,0x37,0x00,0x00,0x00, +0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, +0x38,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x3b,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, +0x37,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00, +0x3f,0x00,0x00,0x00,0x3d,0x00,0x00,0x00,0x3e,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x40,0x00,0x00,0x00, +0x3f,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x42,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x3b,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x47,0x00,0x00,0x00, +0x40,0x00,0x00,0x00,0x3b,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x0d,0x00,0x00,0x00,0x49,0x00,0x00,0x00,0x3d,0x00,0x00,0x00, +0x39,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x4a,0x00,0x00,0x00,0x49,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x0d,0x00,0x00,0x00,0x4d,0x00,0x00,0x00,0x4c,0x00,0x00,0x00, +0x3e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x4e,0x00,0x00,0x00,0x4d,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x50,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, +0x4f,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x55,0x00,0x00,0x00,0x50,0x00,0x00,0x00,0x54,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x59,0x00,0x00,0x00, +0x50,0x00,0x00,0x00,0x58,0x00,0x00,0x00,0x89,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x5d,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, +0x4f,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x64,0x00,0x00,0x00,0x5d,0x00,0x00,0x00,0x63,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x68,0x00,0x00,0x00, +0x5d,0x00,0x00,0x00,0x67,0x00,0x00,0x00,0x89,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x6e,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, +0x6d,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x73,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x72,0x00,0x00,0x00, +0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x78,0x00,0x00,0x00, +0x4e,0x00,0x00,0x00,0x77,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x7d,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, +0x7c,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, +0x81,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x80,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x82,0x00,0x00,0x00, +0x81,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x83,0x00,0x00,0x00,0x47,0x00,0x00,0x00,0x82,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0x12,0x00,0x00,0x00,0x85,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x87,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x89,0x00,0x00,0x00, +0x47,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x8c,0x00,0x00,0x00,0x89,0x00,0x00,0x00, +0x82,0x00,0x00,0x00,0x0c,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0x8d,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x26,0x00,0x00,0x00, +0x87,0x00,0x00,0x00,0x8c,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x15,0x00,0x00,0x00,0x91,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0x90,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x92,0x00,0x00,0x00,0x91,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x93,0x00,0x00,0x00,0x32,0x00,0x00,0x00, +0x92,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x95,0x00,0x00,0x00,0x42,0x00,0x00,0x00,0x37,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x97,0x00,0x00,0x00, +0x12,0x00,0x00,0x00,0x96,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x98,0x00,0x00,0x00,0x97,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x99,0x00,0x00,0x00, +0x95,0x00,0x00,0x00,0x98,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x9a,0x00,0x00,0x00,0x93,0x00,0x00,0x00, +0x99,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x9c,0x00,0x00,0x00,0x9a,0x00,0x00,0x00,0x83,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9d,0x00,0x00,0x00, +0x9c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x15,0x00,0x00,0x00,0xa2,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0xa1,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xa3,0x00,0x00,0x00,0xa2,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xa4,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, +0xa3,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xa7,0x00,0x00,0x00,0x4a,0x00,0x00,0x00,0xa6,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0xa9,0x00,0x00,0x00, +0x12,0x00,0x00,0x00,0xa8,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xaa,0x00,0x00,0x00,0xa9,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xab,0x00,0x00,0x00, +0xa7,0x00,0x00,0x00,0xaa,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xac,0x00,0x00,0x00,0xa4,0x00,0x00,0x00, +0xab,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xae,0x00,0x00,0x00,0xac,0x00,0x00,0x00,0x83,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xaf,0x00,0x00,0x00, +0xae,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xb1,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xb1,0x00,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xa1,0x02,0x00,0x00, +0x3e,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0xd0,0x00,0x00,0x00, +0xb2,0x00,0x00,0x00,0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00, +0xc2,0x00,0x00,0x00,0xa1,0x02,0x00,0x00,0xc0,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0xb3,0x00,0x00,0x00,0xb2,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xc2,0x00,0x00,0x00, +0xb2,0x00,0x00,0x00,0xb3,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xb2,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0xcc,0x00,0x00,0x00, +0xcd,0x00,0x00,0x00,0xc9,0x00,0x00,0x00,0xa1,0x02,0x00,0x00, +0x3e,0x00,0x03,0x00,0xcd,0x00,0x00,0x00,0xcb,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xd0,0x00,0x00,0x00, +0xa1,0x02,0x00,0x00,0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xb1,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xb3,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xd3,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xd3,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xba,0x02,0x00,0x00,0xaf,0x00,0x00,0x00,0xb3,0x00,0x00,0x00, +0x71,0x01,0x00,0x00,0xd6,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xb6,0x02,0x00,0x00,0x9d,0x00,0x00,0x00, +0xb3,0x00,0x00,0x00,0x6e,0x01,0x00,0x00,0xd6,0x00,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xa2,0x02,0x00,0x00, +0x83,0x00,0x00,0x00,0xb3,0x00,0x00,0x00,0x1c,0x02,0x00,0x00, +0xd6,0x00,0x00,0x00,0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00, +0xda,0x00,0x00,0x00,0xa2,0x02,0x00,0x00,0x8d,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0xd5,0x00,0x00,0x00,0xd6,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xda,0x00,0x00,0x00, +0xd4,0x00,0x00,0x00,0xd5,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xd4,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xdc,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xdc,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xb2,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, +0xd4,0x00,0x00,0x00,0x24,0x01,0x00,0x00,0xdf,0x00,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00,0xe2,0x00,0x00,0x00, +0xb2,0x02,0x00,0x00,0x37,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0xde,0x00,0x00,0x00,0xdf,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0xe2,0x00,0x00,0x00,0xdd,0x00,0x00,0x00, +0xde,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xdd,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xe6,0x00,0x00,0x00, +0x95,0x00,0x00,0x00,0x73,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xe8,0x00,0x00,0x00,0xe6,0x00,0x00,0x00, +0xb2,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00, +0xeb,0x00,0x00,0x00,0xe8,0x00,0x00,0x00,0x36,0x00,0x00,0x00, +0xf7,0x00,0x03,0x00,0xed,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0xeb,0x00,0x00,0x00,0xec,0x00,0x00,0x00, +0xed,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xec,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xf0,0x00,0x00,0x00, +0xa2,0x02,0x00,0x00,0x6e,0x00,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc1,0x00,0x00,0x00,0xf2,0x00,0x00,0x00,0xf0,0x00,0x00,0x00, +0x8d,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xed,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xed,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, +0xc1,0x00,0x00,0x00,0xf3,0x00,0x00,0x00,0xeb,0x00,0x00,0x00, +0xdd,0x00,0x00,0x00,0xf2,0x00,0x00,0x00,0xec,0x00,0x00,0x00, +0xf7,0x00,0x03,0x00,0xf5,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0xf3,0x00,0x00,0x00,0xf4,0x00,0x00,0x00, +0x15,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xf4,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xfd,0x00,0x00,0x00, +0x73,0x00,0x00,0x00,0xb2,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xff,0x00,0x00,0x00,0xfd,0x00,0x00,0x00, +0xfe,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x01,0x01,0x00,0x00,0xff,0x00,0x00,0x00,0x6e,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x0c,0x01,0x00,0x00, +0xfd,0x00,0x00,0x00,0x98,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x0d,0x01,0x00,0x00,0xb6,0x02,0x00,0x00, +0x0c,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x0f,0x01,0x00,0x00,0x0d,0x01,0x00,0x00,0x6e,0x00,0x00,0x00, +0x41,0x00,0x06,0x00,0x10,0x01,0x00,0x00,0x11,0x01,0x00,0x00, +0x05,0x01,0x00,0x00,0x34,0x00,0x00,0x00,0x0f,0x01,0x00,0x00, +0x3d,0x00,0x04,0x00,0xc3,0x00,0x00,0x00,0x12,0x01,0x00,0x00, +0x11,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0x13,0x01,0x00,0x00, +0x14,0x01,0x00,0x00,0xfa,0x00,0x00,0x00,0x01,0x01,0x00,0x00, +0x3e,0x00,0x03,0x00,0x14,0x01,0x00,0x00,0x12,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0xf5,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0x15,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x18,0x01,0x00,0x00,0x73,0x00,0x00,0x00,0xb2,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x1a,0x01,0x00,0x00, +0x18,0x01,0x00,0x00,0x19,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x1c,0x01,0x00,0x00,0x1a,0x01,0x00,0x00, +0x6e,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x13,0x01,0x00,0x00, +0x1d,0x01,0x00,0x00,0xfa,0x00,0x00,0x00,0x1c,0x01,0x00,0x00, +0x3e,0x00,0x03,0x00,0x1d,0x01,0x00,0x00,0xcb,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xf5,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xf5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xdf,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xdf,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x24,0x01,0x00,0x00,0xb2,0x02,0x00,0x00, +0x22,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xdc,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xde,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x26,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x26,0x01,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xb3,0x02,0x00,0x00, +0x3e,0x00,0x00,0x00,0xde,0x00,0x00,0x00,0x6a,0x01,0x00,0x00, +0x29,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00, +0x2c,0x01,0x00,0x00,0xb3,0x02,0x00,0x00,0xa6,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0x28,0x01,0x00,0x00,0x29,0x01,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x2c,0x01,0x00,0x00, +0x27,0x01,0x00,0x00,0x28,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x27,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x30,0x01,0x00,0x00,0xa7,0x00,0x00,0x00,0x7d,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x32,0x01,0x00,0x00, +0x30,0x01,0x00,0x00,0xb3,0x02,0x00,0x00,0x41,0x00,0x05,0x00, +0x15,0x00,0x00,0x00,0x33,0x01,0x00,0x00,0x12,0x00,0x00,0x00, +0xcf,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x34,0x01,0x00,0x00,0x33,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc1,0x00,0x00,0x00,0x35,0x01,0x00,0x00,0x32,0x01,0x00,0x00, +0x34,0x01,0x00,0x00,0xf7,0x00,0x03,0x00,0x37,0x01,0x00,0x00, +0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x35,0x01,0x00,0x00, +0x36,0x01,0x00,0x00,0x37,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x36,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x3a,0x01,0x00,0x00,0xa2,0x02,0x00,0x00,0x78,0x00,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00,0x3c,0x01,0x00,0x00, +0x3a,0x01,0x00,0x00,0x8d,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x37,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x37,0x01,0x00,0x00, +0xf5,0x00,0x07,0x00,0xc1,0x00,0x00,0x00,0x3d,0x01,0x00,0x00, +0x35,0x01,0x00,0x00,0x27,0x01,0x00,0x00,0x3c,0x01,0x00,0x00, +0x36,0x01,0x00,0x00,0xf7,0x00,0x03,0x00,0x3f,0x01,0x00,0x00, +0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x3d,0x01,0x00,0x00, +0x3e,0x01,0x00,0x00,0x5d,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x3e,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x47,0x01,0x00,0x00,0x7d,0x00,0x00,0x00,0xb3,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x49,0x01,0x00,0x00, +0x47,0x01,0x00,0x00,0x48,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x4b,0x01,0x00,0x00,0x49,0x01,0x00,0x00, +0x78,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x56,0x01,0x00,0x00,0x47,0x01,0x00,0x00,0xaa,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x57,0x01,0x00,0x00, +0xba,0x02,0x00,0x00,0x56,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x59,0x01,0x00,0x00,0x57,0x01,0x00,0x00, +0x78,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x10,0x01,0x00,0x00, +0x5a,0x01,0x00,0x00,0x4f,0x01,0x00,0x00,0x34,0x00,0x00,0x00, +0x59,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0xc3,0x00,0x00,0x00, +0x5b,0x01,0x00,0x00,0x5a,0x01,0x00,0x00,0x41,0x00,0x05,0x00, +0x13,0x01,0x00,0x00,0x5c,0x01,0x00,0x00,0x44,0x01,0x00,0x00, +0x4b,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x5c,0x01,0x00,0x00, +0x5b,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x3f,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x5d,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x60,0x01,0x00,0x00,0x7d,0x00,0x00,0x00, +0xb3,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x62,0x01,0x00,0x00,0x60,0x01,0x00,0x00,0x61,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x64,0x01,0x00,0x00, +0x62,0x01,0x00,0x00,0x78,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x13,0x01,0x00,0x00,0x65,0x01,0x00,0x00,0x44,0x01,0x00,0x00, +0x64,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x65,0x01,0x00,0x00, +0xcb,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x3f,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x3f,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0x29,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x29,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x6a,0x01,0x00,0x00, +0xb3,0x02,0x00,0x00,0x68,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0x26,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x28,0x01,0x00,0x00, +0xe0,0x00,0x04,0x00,0x0c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x6b,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x6e,0x01,0x00,0x00,0xb6,0x02,0x00,0x00,0x6c,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x71,0x01,0x00,0x00, +0xba,0x02,0x00,0x00,0x6f,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0x73,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x73,0x01,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xbc,0x02,0x00,0x00, +0x3e,0x00,0x00,0x00,0x28,0x01,0x00,0x00,0x1a,0x02,0x00,0x00, +0x76,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00, +0x79,0x01,0x00,0x00,0xbc,0x02,0x00,0x00,0x6c,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0x75,0x01,0x00,0x00,0x76,0x01,0x00,0x00, +0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x79,0x01,0x00,0x00, +0x74,0x01,0x00,0x00,0x75,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x74,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x7b,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x7b,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xc0,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, +0x74,0x01,0x00,0x00,0xa6,0x01,0x00,0x00,0x7e,0x01,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00,0x81,0x01,0x00,0x00, +0xc0,0x02,0x00,0x00,0x60,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0x7d,0x01,0x00,0x00,0x7e,0x01,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x81,0x01,0x00,0x00,0x7c,0x01,0x00,0x00, +0x7d,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x7c,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0x83,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x83,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xd2,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0x7c,0x01,0x00,0x00, +0xa4,0x01,0x00,0x00,0x84,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc1,0x00,0x00,0x00,0x89,0x01,0x00,0x00,0xd2,0x02,0x00,0x00, +0x62,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x85,0x01,0x00,0x00, +0x84,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x89,0x01,0x00,0x00,0x84,0x01,0x00,0x00,0x85,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x84,0x01,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x8f,0x01,0x00,0x00,0xc0,0x02,0x00,0x00, +0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x91,0x01,0x00,0x00,0x8f,0x01,0x00,0x00,0xd2,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x93,0x01,0x00,0x00, +0x55,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x95,0x01,0x00,0x00,0xc0,0x02,0x00,0x00, +0x61,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x96,0x01,0x00,0x00,0x93,0x01,0x00,0x00,0x95,0x01,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x98,0x01,0x00,0x00, +0x64,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x99,0x01,0x00,0x00,0x96,0x01,0x00,0x00, +0x98,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x9b,0x01,0x00,0x00,0x99,0x01,0x00,0x00,0xd2,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9d,0x01,0x00,0x00, +0x9b,0x01,0x00,0x00,0x9c,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x9f,0x01,0x00,0x00,0x9d,0x01,0x00,0x00, +0xbc,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x13,0x01,0x00,0x00, +0xa0,0x01,0x00,0x00,0xfa,0x00,0x00,0x00,0x9f,0x01,0x00,0x00, +0x3d,0x00,0x04,0x00,0xc3,0x00,0x00,0x00,0xa1,0x01,0x00,0x00, +0xa0,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0xcc,0x00,0x00,0x00, +0xa2,0x01,0x00,0x00,0x8d,0x01,0x00,0x00,0x91,0x01,0x00,0x00, +0x3e,0x00,0x03,0x00,0xa2,0x01,0x00,0x00,0xa1,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa4,0x01,0x00,0x00, +0xd2,0x02,0x00,0x00,0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x83,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x85,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0x7e,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x7e,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xa6,0x01,0x00,0x00,0xc0,0x02,0x00,0x00,0xcf,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x7b,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x7d,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xa8,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xa8,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xc1,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, +0x7d,0x01,0x00,0x00,0xd4,0x01,0x00,0x00,0xab,0x01,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00,0xae,0x01,0x00,0x00, +0xc1,0x02,0x00,0x00,0xbe,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0xaa,0x01,0x00,0x00,0xab,0x01,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0xae,0x01,0x00,0x00,0xa9,0x01,0x00,0x00, +0xaa,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xa9,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0xb0,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xb0,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xcf,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0xa9,0x01,0x00,0x00, +0xd2,0x01,0x00,0x00,0xb1,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc1,0x00,0x00,0x00,0xb6,0x01,0x00,0x00,0xcf,0x02,0x00,0x00, +0xbb,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xb2,0x01,0x00,0x00, +0xb1,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0xb6,0x01,0x00,0x00,0xb1,0x01,0x00,0x00,0xb2,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xb1,0x01,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xbc,0x01,0x00,0x00,0xc1,0x02,0x00,0x00, +0xbb,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xbe,0x01,0x00,0x00,0xbc,0x01,0x00,0x00,0xcf,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xc0,0x01,0x00,0x00, +0x59,0x00,0x00,0x00,0xb8,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xc3,0x01,0x00,0x00,0xc1,0x02,0x00,0x00, +0xc2,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xc4,0x01,0x00,0x00,0xc0,0x01,0x00,0x00,0xc3,0x01,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xc6,0x01,0x00,0x00, +0x68,0x00,0x00,0x00,0xbb,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xc7,0x01,0x00,0x00,0xc4,0x01,0x00,0x00, +0xc6,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xc9,0x01,0x00,0x00,0xc7,0x01,0x00,0x00,0xcf,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xcb,0x01,0x00,0x00, +0xc9,0x01,0x00,0x00,0xca,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xcd,0x01,0x00,0x00,0xcb,0x01,0x00,0x00, +0xbc,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x13,0x01,0x00,0x00, +0xce,0x01,0x00,0x00,0x44,0x01,0x00,0x00,0xcd,0x01,0x00,0x00, +0x3d,0x00,0x04,0x00,0xc3,0x00,0x00,0x00,0xcf,0x01,0x00,0x00, +0xce,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0xcc,0x00,0x00,0x00, +0xd0,0x01,0x00,0x00,0xba,0x01,0x00,0x00,0xbe,0x01,0x00,0x00, +0x3e,0x00,0x03,0x00,0xd0,0x01,0x00,0x00,0xcf,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xd2,0x01,0x00,0x00, +0xcf,0x02,0x00,0x00,0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xb0,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xb2,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0xab,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xab,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xd4,0x01,0x00,0x00,0xc1,0x02,0x00,0x00,0xcf,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xa8,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xaa,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xd6,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xd6,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xc2,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, +0xaa,0x01,0x00,0x00,0x18,0x02,0x00,0x00,0xd9,0x01,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00,0xdc,0x01,0x00,0x00, +0xc2,0x02,0x00,0x00,0xbe,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0xd8,0x01,0x00,0x00,0xd9,0x01,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0xdc,0x01,0x00,0x00,0xd7,0x01,0x00,0x00, +0xd8,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xd7,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0xde,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xde,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xc6,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0xd7,0x01,0x00,0x00, +0x16,0x02,0x00,0x00,0xe1,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc1,0x00,0x00,0x00,0xe4,0x01,0x00,0x00,0xc6,0x02,0x00,0x00, +0x60,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xe0,0x01,0x00,0x00, +0xe1,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0xe4,0x01,0x00,0x00,0xdf,0x01,0x00,0x00,0xe0,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xdf,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0xe6,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xe6,0x01,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xc8,0x02,0x00,0x00, +0x3e,0x00,0x00,0x00,0xdf,0x01,0x00,0x00,0x14,0x02,0x00,0x00, +0xe9,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00, +0xec,0x01,0x00,0x00,0xc8,0x02,0x00,0x00,0xbb,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0xe8,0x01,0x00,0x00,0xe9,0x01,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xec,0x01,0x00,0x00, +0xe7,0x01,0x00,0x00,0xe8,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xe7,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xee,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xee,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xca,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, +0xe7,0x01,0x00,0x00,0x12,0x02,0x00,0x00,0xef,0x01,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00,0xf4,0x01,0x00,0x00, +0xca,0x02,0x00,0x00,0x62,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0xf0,0x01,0x00,0x00,0xef,0x01,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0xf4,0x01,0x00,0x00,0xef,0x01,0x00,0x00, +0xf0,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xef,0x01,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xf6,0x01,0x00,0x00, +0xc2,0x02,0x00,0x00,0xbb,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xf8,0x01,0x00,0x00,0xf6,0x01,0x00,0x00, +0xc8,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xfa,0x01,0x00,0x00,0xf8,0x01,0x00,0x00,0xf9,0x01,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xfc,0x01,0x00,0x00, +0xc6,0x02,0x00,0x00,0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xfd,0x01,0x00,0x00,0xfa,0x01,0x00,0x00, +0xfc,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xff,0x01,0x00,0x00,0xfd,0x01,0x00,0x00,0xca,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x03,0x02,0x00,0x00, +0xfc,0x01,0x00,0x00,0xca,0x02,0x00,0x00,0x41,0x00,0x05,0x00, +0xcc,0x00,0x00,0x00,0x04,0x02,0x00,0x00,0x8d,0x01,0x00,0x00, +0x03,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0xc3,0x00,0x00,0x00, +0x05,0x02,0x00,0x00,0x04,0x02,0x00,0x00,0x41,0x00,0x05,0x00, +0xcc,0x00,0x00,0x00,0x0a,0x02,0x00,0x00,0xba,0x01,0x00,0x00, +0xf8,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0xc3,0x00,0x00,0x00, +0x0b,0x02,0x00,0x00,0x0a,0x02,0x00,0x00,0x41,0x00,0x05,0x00, +0xcc,0x00,0x00,0x00,0x0d,0x02,0x00,0x00,0xc9,0x00,0x00,0x00, +0xff,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0xc3,0x00,0x00,0x00, +0x0e,0x02,0x00,0x00,0x0d,0x02,0x00,0x00,0x0c,0x00,0x08,0x00, +0xc3,0x00,0x00,0x00,0x0f,0x02,0x00,0x00,0x01,0x00,0x00,0x00, +0x32,0x00,0x00,0x00,0x05,0x02,0x00,0x00,0x0b,0x02,0x00,0x00, +0x0e,0x02,0x00,0x00,0x3e,0x00,0x03,0x00,0x0d,0x02,0x00,0x00, +0x0f,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x12,0x02,0x00,0x00,0xca,0x02,0x00,0x00,0xcf,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xee,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xf0,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xe9,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xe9,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x14,0x02,0x00,0x00,0xc8,0x02,0x00,0x00, +0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xe6,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xe8,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0xe1,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xe1,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x16,0x02,0x00,0x00, +0xc6,0x02,0x00,0x00,0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xde,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xe0,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0xd9,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xd9,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x18,0x02,0x00,0x00,0xc2,0x02,0x00,0x00,0xcf,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xd6,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xd8,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x76,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x76,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x1a,0x02,0x00,0x00,0xbc,0x02,0x00,0x00, +0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x73,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x75,0x01,0x00,0x00,0xe0,0x00,0x04,0x00, +0x0c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x6b,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0xd6,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xd6,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x1c,0x02,0x00,0x00,0xa2,0x02,0x00,0x00,0x6c,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xd3,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xd5,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x21,0x02,0x00,0x00,0x55,0x00,0x00,0x00,0x53,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x22,0x02,0x00,0x00, +0x95,0x00,0x00,0x00,0x21,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x27,0x02,0x00,0x00,0x59,0x00,0x00,0x00, +0xb8,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x28,0x02,0x00,0x00,0xa7,0x00,0x00,0x00,0x27,0x02,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x2d,0x02,0x00,0x00, +0x12,0x00,0x00,0x00,0x2c,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x2e,0x02,0x00,0x00,0x2d,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x2f,0x02,0x00,0x00, +0x0f,0x00,0x00,0x00,0x2e,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x33,0x02,0x00,0x00,0x47,0x00,0x00,0x00, +0x2e,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00, +0x35,0x02,0x00,0x00,0x34,0x02,0x00,0x00,0x0c,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x36,0x02,0x00,0x00, +0x35,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x37,0x02,0x00,0x00,0x33,0x02,0x00,0x00,0x36,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x38,0x02,0x00,0x00, +0x2f,0x02,0x00,0x00,0x37,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0x3a,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x3a,0x02,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xa3,0x02,0x00,0x00, +0x3e,0x00,0x00,0x00,0xd5,0x00,0x00,0x00,0xa0,0x02,0x00,0x00, +0x3d,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00, +0x40,0x02,0x00,0x00,0xa3,0x02,0x00,0x00,0xbe,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0x3c,0x02,0x00,0x00,0x3d,0x02,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x40,0x02,0x00,0x00, +0x3b,0x02,0x00,0x00,0x3c,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x3b,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x42,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x42,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xa4,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, +0x3b,0x02,0x00,0x00,0x9e,0x02,0x00,0x00,0x45,0x02,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00,0x48,0x02,0x00,0x00, +0xa4,0x02,0x00,0x00,0x60,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0x44,0x02,0x00,0x00,0x45,0x02,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x48,0x02,0x00,0x00,0x43,0x02,0x00,0x00, +0x44,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x43,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x4c,0x02,0x00,0x00, +0xa4,0x02,0x00,0x00,0x61,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x4d,0x02,0x00,0x00,0x22,0x02,0x00,0x00, +0x4c,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x4f,0x02,0x00,0x00,0x64,0x00,0x00,0x00,0x62,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x50,0x02,0x00,0x00, +0x4d,0x02,0x00,0x00,0x4f,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x54,0x02,0x00,0x00,0xa3,0x02,0x00,0x00, +0xc2,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x55,0x02,0x00,0x00,0x28,0x02,0x00,0x00,0x54,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x57,0x02,0x00,0x00, +0x68,0x00,0x00,0x00,0xbb,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x58,0x02,0x00,0x00,0x55,0x02,0x00,0x00, +0x57,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x5a,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x5a,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xa6,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, +0x43,0x02,0x00,0x00,0x9c,0x02,0x00,0x00,0x5d,0x02,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00,0x60,0x02,0x00,0x00, +0xa6,0x02,0x00,0x00,0xbb,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0x5c,0x02,0x00,0x00,0x5d,0x02,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x60,0x02,0x00,0x00,0x5b,0x02,0x00,0x00, +0x5c,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x5b,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0x62,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x62,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xa8,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0x5b,0x02,0x00,0x00, +0x9a,0x02,0x00,0x00,0x65,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc1,0x00,0x00,0x00,0x68,0x02,0x00,0x00,0xa8,0x02,0x00,0x00, +0x62,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x64,0x02,0x00,0x00, +0x65,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x68,0x02,0x00,0x00,0x63,0x02,0x00,0x00,0x64,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x63,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x6b,0x02,0x00,0x00,0x50,0x02,0x00,0x00, +0xa8,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00, +0x6e,0x02,0x00,0x00,0x6b,0x02,0x00,0x00,0x36,0x00,0x00,0x00, +0xf7,0x00,0x03,0x00,0x70,0x02,0x00,0x00,0x00,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x6e,0x02,0x00,0x00,0x6f,0x02,0x00,0x00, +0x70,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x6f,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x73,0x02,0x00,0x00, +0x58,0x02,0x00,0x00,0xa6,0x02,0x00,0x00,0x41,0x00,0x05,0x00, +0x15,0x00,0x00,0x00,0x74,0x02,0x00,0x00,0x12,0x00,0x00,0x00, +0xcf,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x75,0x02,0x00,0x00,0x74,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc1,0x00,0x00,0x00,0x76,0x02,0x00,0x00,0x73,0x02,0x00,0x00, +0x75,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x70,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x70,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, +0xc1,0x00,0x00,0x00,0x77,0x02,0x00,0x00,0x6e,0x02,0x00,0x00, +0x63,0x02,0x00,0x00,0x76,0x02,0x00,0x00,0x6f,0x02,0x00,0x00, +0xf7,0x00,0x03,0x00,0x79,0x02,0x00,0x00,0x00,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x77,0x02,0x00,0x00,0x78,0x02,0x00,0x00, +0x79,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x78,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x81,0x02,0x00,0x00, +0x58,0x02,0x00,0x00,0xa6,0x02,0x00,0x00,0x41,0x00,0x05,0x00, +0x15,0x00,0x00,0x00,0x83,0x02,0x00,0x00,0x12,0x00,0x00,0x00, +0x82,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x84,0x02,0x00,0x00,0x83,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x85,0x02,0x00,0x00,0x81,0x02,0x00,0x00, +0x84,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x86,0x02,0x00,0x00,0x38,0x02,0x00,0x00,0x85,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x88,0x02,0x00,0x00, +0x86,0x02,0x00,0x00,0x50,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x8a,0x02,0x00,0x00,0x88,0x02,0x00,0x00, +0xa8,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x8c,0x02,0x00,0x00,0xa3,0x02,0x00,0x00,0xbb,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8e,0x02,0x00,0x00, +0x8c,0x02,0x00,0x00,0xa6,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x90,0x02,0x00,0x00,0x8e,0x02,0x00,0x00, +0x8f,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x92,0x02,0x00,0x00,0xa4,0x02,0x00,0x00,0x62,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x93,0x02,0x00,0x00, +0x90,0x02,0x00,0x00,0x92,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x95,0x02,0x00,0x00,0x93,0x02,0x00,0x00, +0xa8,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0xcc,0x00,0x00,0x00, +0x96,0x02,0x00,0x00,0xc9,0x00,0x00,0x00,0x95,0x02,0x00,0x00, +0x3d,0x00,0x04,0x00,0xc3,0x00,0x00,0x00,0x97,0x02,0x00,0x00, +0x96,0x02,0x00,0x00,0x41,0x00,0x06,0x00,0x10,0x01,0x00,0x00, +0x98,0x02,0x00,0x00,0x7d,0x02,0x00,0x00,0x34,0x00,0x00,0x00, +0x8a,0x02,0x00,0x00,0x3e,0x00,0x03,0x00,0x98,0x02,0x00,0x00, +0x97,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x79,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x79,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0x65,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x65,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9a,0x02,0x00,0x00, +0xa8,0x02,0x00,0x00,0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x62,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x64,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0x5d,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x5d,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x9c,0x02,0x00,0x00,0xa6,0x02,0x00,0x00,0xcf,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x5a,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x5c,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x45,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x45,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x9e,0x02,0x00,0x00,0xa4,0x02,0x00,0x00, +0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x42,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x44,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0x3d,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x3d,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa0,0x02,0x00,0x00, +0xa3,0x02,0x00,0x00,0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x3a,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x3c,0x02,0x00,0x00, +0xfd,0x00,0x01,0x00,0x38,0x00,0x01,0x00, +}; +const uint64_t matmul_f32_fp32_len = 10208; + +unsigned char matmul_q4_0_f32_data[] = { +0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, +0x1c,0x03,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, +0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x09,0x00,0x00,0x00, +0x11,0x00,0x02,0x00,0x51,0x11,0x00,0x00,0x11,0x00,0x02,0x00, +0x60,0x11,0x00,0x00,0x0b,0x00,0x06,0x00,0x01,0x00,0x00,0x00, +0x47,0x4c,0x53,0x4c,0x2e,0x73,0x74,0x64,0x2e,0x34,0x35,0x30, +0x00,0x00,0x00,0x00,0x0e,0x00,0x03,0x00,0x00,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x0f,0x00,0x0f,0x00,0x05,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x6d,0x61,0x69,0x6e,0x00,0x00,0x00,0x00, +0x0b,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x3d,0x00,0x00,0x00, +0x4c,0x00,0x00,0x00,0x07,0x01,0x00,0x00,0x27,0x01,0x00,0x00, +0x5a,0x01,0x00,0x00,0x65,0x01,0x00,0x00,0x50,0x02,0x00,0x00, +0x99,0x02,0x00,0x00,0x10,0x00,0x06,0x00,0x04,0x00,0x00,0x00, +0x11,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x0b,0x00,0x00,0x00, +0x0b,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x10,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x10,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x10,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x05,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x10,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, +0x07,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x1c,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x08,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x10,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x24,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, +0x0a,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x28,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x2c,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x10,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x30,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, +0x0d,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x34,0x00,0x00,0x00, +0x47,0x00,0x03,0x00,0x10,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x37,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x3d,0x00,0x00,0x00, +0x0b,0x00,0x00,0x00,0x1a,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x4c,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x1b,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x4f,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x09,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x53,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x60,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x62,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x07,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x6c,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0xa6,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0xb8,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x05,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xbb,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x02,0x01,0x00,0x00,0x06,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x03,0x01,0x00,0x00,0x00,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x03,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x04,0x01,0x00,0x00, +0x06,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x48,0x00,0x04,0x00, +0x05,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x05,0x01,0x00,0x00,0x00,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, +0x05,0x01,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x07,0x01,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x07,0x01,0x00,0x00,0x21,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x34,0x01,0x00,0x00, +0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x35,0x01,0x00,0x00,0x0b,0x00,0x00,0x00,0x19,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x62,0x01,0x00,0x00,0x06,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0x63,0x01,0x00,0x00, +0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x63,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x63,0x01,0x00,0x00, +0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x65,0x01,0x00,0x00, +0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x65,0x01,0x00,0x00,0x21,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x50,0x02,0x00,0x00,0x0b,0x00,0x00,0x00, +0x18,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x96,0x02,0x00,0x00, +0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00, +0x97,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x19,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x97,0x02,0x00,0x00,0x00,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, +0x97,0x02,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x99,0x02,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x99,0x02,0x00,0x00,0x21,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x13,0x00,0x02,0x00,0x02,0x00,0x00,0x00, +0x21,0x00,0x03,0x00,0x03,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x15,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x17,0x00,0x04,0x00,0x09,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x0a,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x09,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x0d,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x1e,0x00,0x10,0x00,0x10,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x11,0x00,0x00,0x00, +0x09,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0x11,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x09,0x00,0x00,0x00, +0x15,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, +0x14,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x15,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x21,0x00,0x00,0x00, +0x0a,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, +0x27,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0x2d,0x00,0x00,0x00,0x07,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x34,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x37,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00,0x3d,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x3e,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0x0a,0x00,0x00,0x00,0x4c,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x4f,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x53,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x54,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0x37,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x58,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0x37,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x32,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x61,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x60,0x00,0x00,0x00, +0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x62,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x63,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x61,0x00,0x00,0x00, +0x62,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x67,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x61,0x00,0x00,0x00, +0x62,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x72,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x77,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x7c,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0x80,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x85,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, +0x90,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0x96,0x00,0x00,0x00,0x03,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0xa1,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xa6,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0xa8,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xb7,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00, +0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xb8,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xb9,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x53,0x00,0x00,0x00, +0xb8,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xba,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x4f,0x00,0x00,0x00, +0x62,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xbb,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xbc,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0xba,0x00,0x00,0x00,0xbb,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xbd,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0xbc,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xbe,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0xb9,0x00,0x00,0x00,0xbd,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xbf,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0xb7,0x00,0x00,0x00,0xbe,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xc0,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0xbf,0x00,0x00,0x00,0xbb,0x00,0x00,0x00,0x14,0x00,0x02,0x00, +0xc1,0x00,0x00,0x00,0x16,0x00,0x03,0x00,0xc3,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xc4,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00, +0x62,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xc5,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0xc4,0x00,0x00,0x00, +0xbe,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xc6,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0xc5,0x00,0x00,0x00, +0xbb,0x00,0x00,0x00,0x1c,0x00,0x04,0x00,0xc7,0x00,0x00,0x00, +0xc3,0x00,0x00,0x00,0xc6,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0xc8,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0xc7,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0xc3,0x00,0x00,0x00,0xcb,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0xcc,0x00,0x00,0x00, +0x07,0x00,0x00,0x00,0xc3,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0xcf,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xf3,0x00,0x00,0x00, +0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xf9,0x00,0x00,0x00, +0x10,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xfd,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x16,0x00,0x03,0x00, +0x00,0x01,0x00,0x00,0x10,0x00,0x00,0x00,0x15,0x00,0x04,0x00, +0x01,0x01,0x00,0x00,0x08,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x1c,0x00,0x04,0x00,0x02,0x01,0x00,0x00,0x01,0x01,0x00,0x00, +0xf9,0x00,0x00,0x00,0x1e,0x00,0x04,0x00,0x03,0x01,0x00,0x00, +0x00,0x01,0x00,0x00,0x02,0x01,0x00,0x00,0x1d,0x00,0x03,0x00, +0x04,0x01,0x00,0x00,0x03,0x01,0x00,0x00,0x1e,0x00,0x03,0x00, +0x05,0x01,0x00,0x00,0x04,0x01,0x00,0x00,0x20,0x00,0x04,0x00, +0x06,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x05,0x01,0x00,0x00, +0x3b,0x00,0x04,0x00,0x06,0x01,0x00,0x00,0x07,0x01,0x00,0x00, +0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x09,0x01,0x00,0x00, +0x0c,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x20,0x00,0x04,0x00, +0x10,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x01,0x01,0x00,0x00, +0x17,0x00,0x04,0x00,0x14,0x01,0x00,0x00,0xc3,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0xc3,0x00,0x00,0x00, +0x1e,0x01,0x00,0x00,0x00,0x00,0x00,0x41,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x23,0x01,0x00,0x00,0x80,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x24,0x01,0x00,0x00,0x84,0x00,0x00,0x00, +0x37,0x00,0x00,0x00,0x23,0x01,0x00,0x00,0x1c,0x00,0x04,0x00, +0x25,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x24,0x01,0x00,0x00, +0x20,0x00,0x04,0x00,0x26,0x01,0x00,0x00,0x04,0x00,0x00,0x00, +0x25,0x01,0x00,0x00,0x3b,0x00,0x04,0x00,0x26,0x01,0x00,0x00, +0x27,0x01,0x00,0x00,0x04,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x2c,0x01,0x00,0x00,0x04,0x00,0x00,0x00,0x00,0x01,0x00,0x00, +0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x34,0x01,0x00,0x00, +0x01,0x00,0x00,0x00,0x33,0x00,0x06,0x00,0x09,0x00,0x00,0x00, +0x35,0x01,0x00,0x00,0x34,0x01,0x00,0x00,0x39,0x00,0x00,0x00, +0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x36,0x01,0x00,0x00,0x51,0x00,0x00,0x00,0x35,0x01,0x00,0x00, +0x00,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x37,0x01,0x00,0x00,0x84,0x00,0x00,0x00,0x36,0x01,0x00,0x00, +0x0c,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x38,0x01,0x00,0x00,0x86,0x00,0x00,0x00,0x37,0x01,0x00,0x00, +0x6c,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x56,0x01,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, +0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x57,0x01,0x00,0x00,0x84,0x00,0x00,0x00,0xa6,0x00,0x00,0x00, +0x56,0x01,0x00,0x00,0x1c,0x00,0x04,0x00,0x58,0x01,0x00,0x00, +0x00,0x01,0x00,0x00,0x57,0x01,0x00,0x00,0x20,0x00,0x04,0x00, +0x59,0x01,0x00,0x00,0x04,0x00,0x00,0x00,0x58,0x01,0x00,0x00, +0x3b,0x00,0x04,0x00,0x59,0x01,0x00,0x00,0x5a,0x01,0x00,0x00, +0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x5e,0x01,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, +0x39,0x00,0x00,0x00,0x1d,0x00,0x03,0x00,0x62,0x01,0x00,0x00, +0xc3,0x00,0x00,0x00,0x1e,0x00,0x03,0x00,0x63,0x01,0x00,0x00, +0x62,0x01,0x00,0x00,0x20,0x00,0x04,0x00,0x64,0x01,0x00,0x00, +0x0c,0x00,0x00,0x00,0x63,0x01,0x00,0x00,0x3b,0x00,0x04,0x00, +0x64,0x01,0x00,0x00,0x65,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x70,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, +0xc3,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x79,0x01,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, +0x39,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x00,0x01,0x00,0x00, +0x7d,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x7f,0x01,0x00,0x00,0x51,0x00,0x00,0x00, +0x35,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x80,0x01,0x00,0x00,0x84,0x00,0x00,0x00, +0x7f,0x01,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x81,0x01,0x00,0x00,0x86,0x00,0x00,0x00, +0x80,0x01,0x00,0x00,0x6c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x84,0x01,0x00,0x00,0x08,0x01,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x85,0x01,0x00,0x00, +0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x88,0x01,0x00,0x00, +0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xa3,0x01,0x00,0x00, +0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00, +0x1c,0x00,0x04,0x00,0xa4,0x01,0x00,0x00,0x00,0x01,0x00,0x00, +0xa3,0x01,0x00,0x00,0x20,0x00,0x04,0x00,0xa5,0x01,0x00,0x00, +0x07,0x00,0x00,0x00,0xa4,0x01,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xb5,0x01,0x00,0x00,0x80,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0xbb,0x01,0x00,0x00,0x07,0x00,0x00,0x00,0x00,0x01,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xd1,0x01,0x00,0x00, +0x84,0x00,0x00,0x00,0xbe,0x00,0x00,0x00,0xbb,0x00,0x00,0x00, +0x1c,0x00,0x04,0x00,0xd2,0x01,0x00,0x00,0x00,0x01,0x00,0x00, +0xd1,0x01,0x00,0x00,0x20,0x00,0x04,0x00,0xd3,0x01,0x00,0x00, +0x07,0x00,0x00,0x00,0xd2,0x01,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xdc,0x01,0x00,0x00,0x86,0x00,0x00,0x00, +0xb8,0x00,0x00,0x00,0xbe,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xe4,0x01,0x00,0x00,0x80,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x13,0x02,0x00,0x00,0x84,0x00,0x00,0x00, +0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0x48,0x02,0x00,0x00,0x0d,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00,0x50,0x02,0x00,0x00, +0x01,0x00,0x00,0x00,0x1d,0x00,0x03,0x00,0x96,0x02,0x00,0x00, +0xc3,0x00,0x00,0x00,0x1e,0x00,0x03,0x00,0x97,0x02,0x00,0x00, +0x96,0x02,0x00,0x00,0x20,0x00,0x04,0x00,0x98,0x02,0x00,0x00, +0x0c,0x00,0x00,0x00,0x97,0x02,0x00,0x00,0x3b,0x00,0x04,0x00, +0x98,0x02,0x00,0x00,0x99,0x02,0x00,0x00,0x0c,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x9e,0x02,0x00,0x00, +0x05,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xab,0x02,0x00,0x00,0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00, +0x62,0x00,0x00,0x00,0x2c,0x00,0x05,0x00,0x14,0x01,0x00,0x00, +0x1b,0x03,0x00,0x00,0x1e,0x01,0x00,0x00,0x1e,0x01,0x00,0x00, +0x36,0x00,0x05,0x00,0x02,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0x05,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0xc8,0x00,0x00,0x00, +0xc9,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0xa5,0x01,0x00,0x00,0xa6,0x01,0x00,0x00,0x07,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0xd3,0x01,0x00,0x00,0xd4,0x01,0x00,0x00, +0x07,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00, +0x0e,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, +0x0e,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, +0x16,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x14,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x17,0x00,0x00,0x00, +0x16,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x18,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x17,0x00,0x00,0x00, +0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x1e,0x00,0x00,0x00, +0x0f,0x00,0x00,0x00,0x17,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x15,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0x21,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x24,0x00,0x00,0x00,0x18,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, +0x28,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x27,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x29,0x00,0x00,0x00, +0x28,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x2a,0x00,0x00,0x00,0x1e,0x00,0x00,0x00,0x29,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, +0x12,0x00,0x00,0x00,0x2d,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x2f,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x30,0x00,0x00,0x00, +0x24,0x00,0x00,0x00,0x2f,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x30,0x00,0x00,0x00, +0x2a,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, +0x35,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x34,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x36,0x00,0x00,0x00, +0x35,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x38,0x00,0x00,0x00,0x36,0x00,0x00,0x00,0x37,0x00,0x00,0x00, +0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, +0x38,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x3b,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, +0x37,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00, +0x3f,0x00,0x00,0x00,0x3d,0x00,0x00,0x00,0x3e,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x40,0x00,0x00,0x00, +0x3f,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x42,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x3b,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x47,0x00,0x00,0x00, +0x40,0x00,0x00,0x00,0x3b,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x0d,0x00,0x00,0x00,0x49,0x00,0x00,0x00,0x3d,0x00,0x00,0x00, +0x39,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x4a,0x00,0x00,0x00,0x49,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x0d,0x00,0x00,0x00,0x4d,0x00,0x00,0x00,0x4c,0x00,0x00,0x00, +0x3e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x4e,0x00,0x00,0x00,0x4d,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x50,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, +0x4f,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x55,0x00,0x00,0x00,0x50,0x00,0x00,0x00,0x54,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x59,0x00,0x00,0x00, +0x50,0x00,0x00,0x00,0x58,0x00,0x00,0x00,0x89,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x5d,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, +0x4f,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x64,0x00,0x00,0x00,0x5d,0x00,0x00,0x00,0x63,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x68,0x00,0x00,0x00, +0x5d,0x00,0x00,0x00,0x67,0x00,0x00,0x00,0x89,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x6e,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, +0x6d,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x73,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x72,0x00,0x00,0x00, +0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x78,0x00,0x00,0x00, +0x4e,0x00,0x00,0x00,0x77,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x7d,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, +0x7c,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, +0x81,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x80,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x82,0x00,0x00,0x00, +0x81,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x83,0x00,0x00,0x00,0x47,0x00,0x00,0x00,0x82,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0x12,0x00,0x00,0x00,0x85,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x87,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x89,0x00,0x00,0x00, +0x47,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x8c,0x00,0x00,0x00,0x89,0x00,0x00,0x00, +0x82,0x00,0x00,0x00,0x0c,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0x8d,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x26,0x00,0x00,0x00, +0x87,0x00,0x00,0x00,0x8c,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x15,0x00,0x00,0x00,0x91,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0x90,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x92,0x00,0x00,0x00,0x91,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x93,0x00,0x00,0x00,0x32,0x00,0x00,0x00, +0x92,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x95,0x00,0x00,0x00,0x42,0x00,0x00,0x00,0x37,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x97,0x00,0x00,0x00, +0x12,0x00,0x00,0x00,0x96,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x98,0x00,0x00,0x00,0x97,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x99,0x00,0x00,0x00, +0x95,0x00,0x00,0x00,0x98,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x9a,0x00,0x00,0x00,0x93,0x00,0x00,0x00, +0x99,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x9c,0x00,0x00,0x00,0x9a,0x00,0x00,0x00,0x83,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9d,0x00,0x00,0x00, +0x9c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x15,0x00,0x00,0x00,0xa2,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0xa1,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xa3,0x00,0x00,0x00,0xa2,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xa4,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, +0xa3,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xa7,0x00,0x00,0x00,0x4a,0x00,0x00,0x00,0xa6,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0xa9,0x00,0x00,0x00, +0x12,0x00,0x00,0x00,0xa8,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xaa,0x00,0x00,0x00,0xa9,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xab,0x00,0x00,0x00, +0xa7,0x00,0x00,0x00,0xaa,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xac,0x00,0x00,0x00,0xa4,0x00,0x00,0x00, +0xab,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xae,0x00,0x00,0x00,0xac,0x00,0x00,0x00,0x83,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xaf,0x00,0x00,0x00, +0xae,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xb1,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xb1,0x00,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xbf,0x02,0x00,0x00, +0x3e,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0xd0,0x00,0x00,0x00, +0xb2,0x00,0x00,0x00,0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00, +0xc2,0x00,0x00,0x00,0xbf,0x02,0x00,0x00,0xc0,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0xb3,0x00,0x00,0x00,0xb2,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xc2,0x00,0x00,0x00, +0xb2,0x00,0x00,0x00,0xb3,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xb2,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0xcc,0x00,0x00,0x00, +0xcd,0x00,0x00,0x00,0xc9,0x00,0x00,0x00,0xbf,0x02,0x00,0x00, +0x3e,0x00,0x03,0x00,0xcd,0x00,0x00,0x00,0xcb,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xd0,0x00,0x00,0x00, +0xbf,0x02,0x00,0x00,0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xb1,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xb3,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xd3,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xd3,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xd8,0x02,0x00,0x00,0xaf,0x00,0x00,0x00,0xb3,0x00,0x00,0x00, +0x8a,0x01,0x00,0x00,0xd6,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xd4,0x02,0x00,0x00,0x9d,0x00,0x00,0x00, +0xb3,0x00,0x00,0x00,0x87,0x01,0x00,0x00,0xd6,0x00,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xc0,0x02,0x00,0x00, +0x83,0x00,0x00,0x00,0xb3,0x00,0x00,0x00,0x38,0x02,0x00,0x00, +0xd6,0x00,0x00,0x00,0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00, +0xda,0x00,0x00,0x00,0xc0,0x02,0x00,0x00,0x8d,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0xd5,0x00,0x00,0x00,0xd6,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xda,0x00,0x00,0x00, +0xd4,0x00,0x00,0x00,0xd5,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xd4,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xdc,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xdc,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xd0,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, +0xd4,0x00,0x00,0x00,0x3a,0x01,0x00,0x00,0xdd,0x00,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00,0xe2,0x00,0x00,0x00, +0xd0,0x02,0x00,0x00,0x37,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0xde,0x00,0x00,0x00,0xdd,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0xe2,0x00,0x00,0x00,0xdd,0x00,0x00,0x00, +0xde,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xdd,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xe7,0x00,0x00,0x00, +0x73,0x00,0x00,0x00,0xd0,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xea,0x00,0x00,0x00,0xe7,0x00,0x00,0x00, +0x98,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xeb,0x00,0x00,0x00,0xea,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xec,0x00,0x00,0x00, +0xd4,0x02,0x00,0x00,0xeb,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xee,0x00,0x00,0x00,0xec,0x00,0x00,0x00, +0x6e,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xf4,0x00,0x00,0x00,0xe7,0x00,0x00,0x00,0xf3,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xf6,0x00,0x00,0x00, +0xf4,0x00,0x00,0x00,0x6e,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xfa,0x00,0x00,0x00,0xee,0x00,0x00,0x00, +0xf9,0x00,0x00,0x00,0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xfe,0x00,0x00,0x00,0xee,0x00,0x00,0x00,0xfd,0x00,0x00,0x00, +0x41,0x00,0x07,0x00,0x09,0x01,0x00,0x00,0x0a,0x01,0x00,0x00, +0x07,0x01,0x00,0x00,0x34,0x00,0x00,0x00,0xfa,0x00,0x00,0x00, +0x34,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x00,0x01,0x00,0x00, +0x0b,0x01,0x00,0x00,0x0a,0x01,0x00,0x00,0x73,0x00,0x04,0x00, +0xc3,0x00,0x00,0x00,0x0c,0x01,0x00,0x00,0x0b,0x01,0x00,0x00, +0x41,0x00,0x08,0x00,0x10,0x01,0x00,0x00,0x11,0x01,0x00,0x00, +0x07,0x01,0x00,0x00,0x34,0x00,0x00,0x00,0xfa,0x00,0x00,0x00, +0xcf,0x00,0x00,0x00,0xfe,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x01,0x01,0x00,0x00,0x12,0x01,0x00,0x00,0x11,0x01,0x00,0x00, +0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x13,0x01,0x00,0x00, +0x12,0x01,0x00,0x00,0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x18,0x01,0x00,0x00,0x13,0x01,0x00,0x00,0xfd,0x00,0x00,0x00, +0x70,0x00,0x04,0x00,0xc3,0x00,0x00,0x00,0x19,0x01,0x00,0x00, +0x18,0x01,0x00,0x00,0xc2,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x1b,0x01,0x00,0x00,0x13,0x01,0x00,0x00,0xa8,0x00,0x00,0x00, +0x70,0x00,0x04,0x00,0xc3,0x00,0x00,0x00,0x1c,0x01,0x00,0x00, +0x1b,0x01,0x00,0x00,0x50,0x00,0x05,0x00,0x14,0x01,0x00,0x00, +0x1d,0x01,0x00,0x00,0x19,0x01,0x00,0x00,0x1c,0x01,0x00,0x00, +0x83,0x00,0x05,0x00,0x14,0x01,0x00,0x00,0x20,0x01,0x00,0x00, +0x1d,0x01,0x00,0x00,0x1b,0x03,0x00,0x00,0x8e,0x00,0x05,0x00, +0x14,0x01,0x00,0x00,0x22,0x01,0x00,0x00,0x20,0x01,0x00,0x00, +0x0c,0x01,0x00,0x00,0x51,0x00,0x05,0x00,0xc3,0x00,0x00,0x00, +0x2a,0x01,0x00,0x00,0x22,0x01,0x00,0x00,0x00,0x00,0x00,0x00, +0x73,0x00,0x04,0x00,0x00,0x01,0x00,0x00,0x2b,0x01,0x00,0x00, +0x2a,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0x2c,0x01,0x00,0x00, +0x2d,0x01,0x00,0x00,0x27,0x01,0x00,0x00,0xf6,0x00,0x00,0x00, +0x3e,0x00,0x03,0x00,0x2d,0x01,0x00,0x00,0x2b,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x2f,0x01,0x00,0x00, +0xf6,0x00,0x00,0x00,0xf9,0x00,0x00,0x00,0x51,0x00,0x05,0x00, +0xc3,0x00,0x00,0x00,0x31,0x01,0x00,0x00,0x22,0x01,0x00,0x00, +0x01,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x00,0x01,0x00,0x00, +0x32,0x01,0x00,0x00,0x31,0x01,0x00,0x00,0x41,0x00,0x05,0x00, +0x2c,0x01,0x00,0x00,0x33,0x01,0x00,0x00,0x27,0x01,0x00,0x00, +0x2f,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x33,0x01,0x00,0x00, +0x32,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x3a,0x01,0x00,0x00,0xd0,0x02,0x00,0x00,0x38,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0xdc,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xde,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x3c,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x3c,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xd1,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, +0xde,0x00,0x00,0x00,0x83,0x01,0x00,0x00,0x3f,0x01,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00,0x42,0x01,0x00,0x00, +0xd1,0x02,0x00,0x00,0xa6,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0x3e,0x01,0x00,0x00,0x3f,0x01,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x42,0x01,0x00,0x00,0x3d,0x01,0x00,0x00, +0x3e,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x3d,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x46,0x01,0x00,0x00, +0xa7,0x00,0x00,0x00,0x7d,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x48,0x01,0x00,0x00,0x46,0x01,0x00,0x00, +0xd1,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, +0x49,0x01,0x00,0x00,0x12,0x00,0x00,0x00,0xcf,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x4a,0x01,0x00,0x00, +0x49,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00, +0x4b,0x01,0x00,0x00,0x48,0x01,0x00,0x00,0x4a,0x01,0x00,0x00, +0xf7,0x00,0x03,0x00,0x4d,0x01,0x00,0x00,0x00,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x4b,0x01,0x00,0x00,0x4c,0x01,0x00,0x00, +0x4d,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x4c,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x50,0x01,0x00,0x00, +0xc0,0x02,0x00,0x00,0x78,0x00,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc1,0x00,0x00,0x00,0x52,0x01,0x00,0x00,0x50,0x01,0x00,0x00, +0x8d,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x4d,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x4d,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, +0xc1,0x00,0x00,0x00,0x53,0x01,0x00,0x00,0x4b,0x01,0x00,0x00, +0x3d,0x01,0x00,0x00,0x52,0x01,0x00,0x00,0x4c,0x01,0x00,0x00, +0xf7,0x00,0x03,0x00,0x55,0x01,0x00,0x00,0x00,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x53,0x01,0x00,0x00,0x54,0x01,0x00,0x00, +0x75,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x54,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x5d,0x01,0x00,0x00, +0x7d,0x00,0x00,0x00,0xd1,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x5f,0x01,0x00,0x00,0x5d,0x01,0x00,0x00, +0x5e,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x61,0x01,0x00,0x00,0x5f,0x01,0x00,0x00,0x78,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x6c,0x01,0x00,0x00, +0x5d,0x01,0x00,0x00,0xaa,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x6d,0x01,0x00,0x00,0xd8,0x02,0x00,0x00, +0x6c,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x6f,0x01,0x00,0x00,0x6d,0x01,0x00,0x00,0x78,0x00,0x00,0x00, +0x41,0x00,0x06,0x00,0x70,0x01,0x00,0x00,0x71,0x01,0x00,0x00, +0x65,0x01,0x00,0x00,0x34,0x00,0x00,0x00,0x6f,0x01,0x00,0x00, +0x3d,0x00,0x04,0x00,0xc3,0x00,0x00,0x00,0x72,0x01,0x00,0x00, +0x71,0x01,0x00,0x00,0x73,0x00,0x04,0x00,0x00,0x01,0x00,0x00, +0x73,0x01,0x00,0x00,0x72,0x01,0x00,0x00,0x41,0x00,0x05,0x00, +0x2c,0x01,0x00,0x00,0x74,0x01,0x00,0x00,0x5a,0x01,0x00,0x00, +0x61,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x74,0x01,0x00,0x00, +0x73,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x55,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x75,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x78,0x01,0x00,0x00,0x7d,0x00,0x00,0x00, +0xd1,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x7a,0x01,0x00,0x00,0x78,0x01,0x00,0x00,0x79,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x7c,0x01,0x00,0x00, +0x7a,0x01,0x00,0x00,0x78,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x2c,0x01,0x00,0x00,0x7e,0x01,0x00,0x00,0x5a,0x01,0x00,0x00, +0x7c,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x7e,0x01,0x00,0x00, +0x7d,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x55,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x55,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0x3f,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x3f,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x83,0x01,0x00,0x00, +0xd1,0x02,0x00,0x00,0x81,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0x3c,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x3e,0x01,0x00,0x00, +0xe0,0x00,0x04,0x00,0x0c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x84,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x87,0x01,0x00,0x00,0xd4,0x02,0x00,0x00,0x85,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8a,0x01,0x00,0x00, +0xd8,0x02,0x00,0x00,0x88,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0x8c,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x8c,0x01,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xda,0x02,0x00,0x00, +0x3e,0x00,0x00,0x00,0x3e,0x01,0x00,0x00,0x36,0x02,0x00,0x00, +0x8f,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00, +0x92,0x01,0x00,0x00,0xda,0x02,0x00,0x00,0x6c,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0x8e,0x01,0x00,0x00,0x8f,0x01,0x00,0x00, +0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x92,0x01,0x00,0x00, +0x8d,0x01,0x00,0x00,0x8e,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x8d,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x94,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x94,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xde,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, +0x8d,0x01,0x00,0x00,0xc0,0x01,0x00,0x00,0x97,0x01,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00,0x9a,0x01,0x00,0x00, +0xde,0x02,0x00,0x00,0x60,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0x96,0x01,0x00,0x00,0x97,0x01,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x9a,0x01,0x00,0x00,0x95,0x01,0x00,0x00, +0x96,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x95,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0x9c,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x9c,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xf0,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0x95,0x01,0x00,0x00, +0xbe,0x01,0x00,0x00,0x9d,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc1,0x00,0x00,0x00,0xa2,0x01,0x00,0x00,0xf0,0x02,0x00,0x00, +0x62,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x9e,0x01,0x00,0x00, +0x9d,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0xa2,0x01,0x00,0x00,0x9d,0x01,0x00,0x00,0x9e,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x9d,0x01,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xa8,0x01,0x00,0x00,0xde,0x02,0x00,0x00, +0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xaa,0x01,0x00,0x00,0xa8,0x01,0x00,0x00,0xf0,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xac,0x01,0x00,0x00, +0x55,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xae,0x01,0x00,0x00,0xde,0x02,0x00,0x00, +0x61,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xaf,0x01,0x00,0x00,0xac,0x01,0x00,0x00,0xae,0x01,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xb1,0x01,0x00,0x00, +0x64,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xb2,0x01,0x00,0x00,0xaf,0x01,0x00,0x00, +0xb1,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xb4,0x01,0x00,0x00,0xb2,0x01,0x00,0x00,0xf0,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xb6,0x01,0x00,0x00, +0xb4,0x01,0x00,0x00,0xb5,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xb8,0x01,0x00,0x00,0xb6,0x01,0x00,0x00, +0xda,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x2c,0x01,0x00,0x00, +0xb9,0x01,0x00,0x00,0x27,0x01,0x00,0x00,0xb8,0x01,0x00,0x00, +0x3d,0x00,0x04,0x00,0x00,0x01,0x00,0x00,0xba,0x01,0x00,0x00, +0xb9,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0xbb,0x01,0x00,0x00, +0xbc,0x01,0x00,0x00,0xa6,0x01,0x00,0x00,0xaa,0x01,0x00,0x00, +0x3e,0x00,0x03,0x00,0xbc,0x01,0x00,0x00,0xba,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xbe,0x01,0x00,0x00, +0xf0,0x02,0x00,0x00,0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x9c,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x9e,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0x97,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x97,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xc0,0x01,0x00,0x00,0xde,0x02,0x00,0x00,0xcf,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x94,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x96,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xc2,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xc2,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xdf,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, +0x96,0x01,0x00,0x00,0xee,0x01,0x00,0x00,0xc5,0x01,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00,0xc8,0x01,0x00,0x00, +0xdf,0x02,0x00,0x00,0xbe,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0xc4,0x01,0x00,0x00,0xc5,0x01,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0xc8,0x01,0x00,0x00,0xc3,0x01,0x00,0x00, +0xc4,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xc3,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0xca,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xca,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xed,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0xc3,0x01,0x00,0x00, +0xec,0x01,0x00,0x00,0xcb,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc1,0x00,0x00,0x00,0xd0,0x01,0x00,0x00,0xed,0x02,0x00,0x00, +0xbb,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xcc,0x01,0x00,0x00, +0xcb,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0xd0,0x01,0x00,0x00,0xcb,0x01,0x00,0x00,0xcc,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xcb,0x01,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xd6,0x01,0x00,0x00,0xdf,0x02,0x00,0x00, +0xbb,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xd8,0x01,0x00,0x00,0xd6,0x01,0x00,0x00,0xed,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xda,0x01,0x00,0x00, +0x59,0x00,0x00,0x00,0xb8,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xdd,0x01,0x00,0x00,0xdf,0x02,0x00,0x00, +0xdc,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xde,0x01,0x00,0x00,0xda,0x01,0x00,0x00,0xdd,0x01,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xe0,0x01,0x00,0x00, +0x68,0x00,0x00,0x00,0xbb,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xe1,0x01,0x00,0x00,0xde,0x01,0x00,0x00, +0xe0,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xe3,0x01,0x00,0x00,0xe1,0x01,0x00,0x00,0xed,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xe5,0x01,0x00,0x00, +0xe3,0x01,0x00,0x00,0xe4,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xe7,0x01,0x00,0x00,0xe5,0x01,0x00,0x00, +0xda,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x2c,0x01,0x00,0x00, +0xe8,0x01,0x00,0x00,0x5a,0x01,0x00,0x00,0xe7,0x01,0x00,0x00, +0x3d,0x00,0x04,0x00,0x00,0x01,0x00,0x00,0xe9,0x01,0x00,0x00, +0xe8,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0xbb,0x01,0x00,0x00, +0xea,0x01,0x00,0x00,0xd4,0x01,0x00,0x00,0xd8,0x01,0x00,0x00, +0x3e,0x00,0x03,0x00,0xea,0x01,0x00,0x00,0xe9,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xec,0x01,0x00,0x00, +0xed,0x02,0x00,0x00,0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xca,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xcc,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0xc5,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xc5,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xee,0x01,0x00,0x00,0xdf,0x02,0x00,0x00,0xcf,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xc2,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xc4,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xf0,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xf0,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xe0,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, +0xc4,0x01,0x00,0x00,0x34,0x02,0x00,0x00,0xf3,0x01,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00,0xf6,0x01,0x00,0x00, +0xe0,0x02,0x00,0x00,0xbe,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0xf2,0x01,0x00,0x00,0xf3,0x01,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0xf6,0x01,0x00,0x00,0xf1,0x01,0x00,0x00, +0xf2,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xf1,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0xf8,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xf8,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xe4,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0xf1,0x01,0x00,0x00, +0x32,0x02,0x00,0x00,0xfb,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc1,0x00,0x00,0x00,0xfe,0x01,0x00,0x00,0xe4,0x02,0x00,0x00, +0x60,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xfa,0x01,0x00,0x00, +0xfb,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0xfe,0x01,0x00,0x00,0xf9,0x01,0x00,0x00,0xfa,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xf9,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0x00,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x00,0x02,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xe6,0x02,0x00,0x00, +0x3e,0x00,0x00,0x00,0xf9,0x01,0x00,0x00,0x30,0x02,0x00,0x00, +0x03,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00, +0x06,0x02,0x00,0x00,0xe6,0x02,0x00,0x00,0xbb,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0x02,0x02,0x00,0x00,0x03,0x02,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x06,0x02,0x00,0x00, +0x01,0x02,0x00,0x00,0x02,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x01,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x08,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x08,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xe8,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, +0x01,0x02,0x00,0x00,0x2e,0x02,0x00,0x00,0x09,0x02,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00,0x0e,0x02,0x00,0x00, +0xe8,0x02,0x00,0x00,0x62,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0x0a,0x02,0x00,0x00,0x09,0x02,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x0e,0x02,0x00,0x00,0x09,0x02,0x00,0x00, +0x0a,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x09,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x10,0x02,0x00,0x00, +0xe0,0x02,0x00,0x00,0xbb,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x12,0x02,0x00,0x00,0x10,0x02,0x00,0x00, +0xe6,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x14,0x02,0x00,0x00,0x12,0x02,0x00,0x00,0x13,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x16,0x02,0x00,0x00, +0xe4,0x02,0x00,0x00,0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x17,0x02,0x00,0x00,0x14,0x02,0x00,0x00, +0x16,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x19,0x02,0x00,0x00,0x17,0x02,0x00,0x00,0xe8,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x1d,0x02,0x00,0x00, +0x16,0x02,0x00,0x00,0xe8,0x02,0x00,0x00,0x41,0x00,0x05,0x00, +0xbb,0x01,0x00,0x00,0x1e,0x02,0x00,0x00,0xa6,0x01,0x00,0x00, +0x1d,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0x00,0x01,0x00,0x00, +0x1f,0x02,0x00,0x00,0x1e,0x02,0x00,0x00,0x73,0x00,0x04,0x00, +0xc3,0x00,0x00,0x00,0x20,0x02,0x00,0x00,0x1f,0x02,0x00,0x00, +0x41,0x00,0x05,0x00,0xbb,0x01,0x00,0x00,0x25,0x02,0x00,0x00, +0xd4,0x01,0x00,0x00,0x12,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, +0x00,0x01,0x00,0x00,0x26,0x02,0x00,0x00,0x25,0x02,0x00,0x00, +0x73,0x00,0x04,0x00,0xc3,0x00,0x00,0x00,0x27,0x02,0x00,0x00, +0x26,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0xcc,0x00,0x00,0x00, +0x29,0x02,0x00,0x00,0xc9,0x00,0x00,0x00,0x19,0x02,0x00,0x00, +0x3d,0x00,0x04,0x00,0xc3,0x00,0x00,0x00,0x2a,0x02,0x00,0x00, +0x29,0x02,0x00,0x00,0x0c,0x00,0x08,0x00,0xc3,0x00,0x00,0x00, +0x2b,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00, +0x20,0x02,0x00,0x00,0x27,0x02,0x00,0x00,0x2a,0x02,0x00,0x00, +0x3e,0x00,0x03,0x00,0x29,0x02,0x00,0x00,0x2b,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x2e,0x02,0x00,0x00, +0xe8,0x02,0x00,0x00,0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x08,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x0a,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0x03,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x03,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x30,0x02,0x00,0x00,0xe6,0x02,0x00,0x00,0xcf,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x00,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x02,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0xfb,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xfb,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x32,0x02,0x00,0x00,0xe4,0x02,0x00,0x00, +0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xf8,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xfa,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0xf3,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xf3,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x34,0x02,0x00,0x00, +0xe0,0x02,0x00,0x00,0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xf0,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xf2,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0x8f,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x8f,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x36,0x02,0x00,0x00,0xda,0x02,0x00,0x00,0xcf,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x8c,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x8e,0x01,0x00,0x00,0xe0,0x00,0x04,0x00,0x0c,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x84,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0xd6,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xd6,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x38,0x02,0x00,0x00, +0xc0,0x02,0x00,0x00,0x6c,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xd3,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xd5,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3d,0x02,0x00,0x00, +0x55,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x3e,0x02,0x00,0x00,0x95,0x00,0x00,0x00, +0x3d,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x43,0x02,0x00,0x00,0x59,0x00,0x00,0x00,0xb8,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x44,0x02,0x00,0x00, +0xa7,0x00,0x00,0x00,0x43,0x02,0x00,0x00,0x41,0x00,0x05,0x00, +0x15,0x00,0x00,0x00,0x49,0x02,0x00,0x00,0x12,0x00,0x00,0x00, +0x48,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x4a,0x02,0x00,0x00,0x49,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x4b,0x02,0x00,0x00,0x0f,0x00,0x00,0x00, +0x4a,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x4f,0x02,0x00,0x00,0x47,0x00,0x00,0x00,0x4a,0x02,0x00,0x00, +0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00,0x51,0x02,0x00,0x00, +0x50,0x02,0x00,0x00,0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x52,0x02,0x00,0x00,0x51,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x53,0x02,0x00,0x00, +0x4f,0x02,0x00,0x00,0x52,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x54,0x02,0x00,0x00,0x4b,0x02,0x00,0x00, +0x53,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x56,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x56,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xc1,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, +0xd5,0x00,0x00,0x00,0xbc,0x02,0x00,0x00,0x59,0x02,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00,0x5c,0x02,0x00,0x00, +0xc1,0x02,0x00,0x00,0xbe,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0x58,0x02,0x00,0x00,0x59,0x02,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x5c,0x02,0x00,0x00,0x57,0x02,0x00,0x00, +0x58,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x57,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0x5e,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x5e,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xc2,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0x57,0x02,0x00,0x00, +0xba,0x02,0x00,0x00,0x61,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc1,0x00,0x00,0x00,0x64,0x02,0x00,0x00,0xc2,0x02,0x00,0x00, +0x60,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x60,0x02,0x00,0x00, +0x61,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x64,0x02,0x00,0x00,0x5f,0x02,0x00,0x00,0x60,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x5f,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x68,0x02,0x00,0x00,0xc2,0x02,0x00,0x00, +0x61,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x69,0x02,0x00,0x00,0x3e,0x02,0x00,0x00,0x68,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x6b,0x02,0x00,0x00, +0x64,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x6c,0x02,0x00,0x00,0x69,0x02,0x00,0x00, +0x6b,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x70,0x02,0x00,0x00,0xc1,0x02,0x00,0x00,0xdc,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x71,0x02,0x00,0x00, +0x44,0x02,0x00,0x00,0x70,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x73,0x02,0x00,0x00,0x68,0x00,0x00,0x00, +0xbb,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x74,0x02,0x00,0x00,0x71,0x02,0x00,0x00,0x73,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0x76,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x76,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xc4,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0x5f,0x02,0x00,0x00, +0xb8,0x02,0x00,0x00,0x79,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc1,0x00,0x00,0x00,0x7c,0x02,0x00,0x00,0xc4,0x02,0x00,0x00, +0xbb,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x78,0x02,0x00,0x00, +0x79,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x7c,0x02,0x00,0x00,0x77,0x02,0x00,0x00,0x78,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x77,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0x7e,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x7e,0x02,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xc6,0x02,0x00,0x00, +0x3e,0x00,0x00,0x00,0x77,0x02,0x00,0x00,0xb6,0x02,0x00,0x00, +0x81,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00, +0x84,0x02,0x00,0x00,0xc6,0x02,0x00,0x00,0x62,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0x80,0x02,0x00,0x00,0x81,0x02,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x84,0x02,0x00,0x00, +0x7f,0x02,0x00,0x00,0x80,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x7f,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x87,0x02,0x00,0x00,0x6c,0x02,0x00,0x00,0xc6,0x02,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00,0x8a,0x02,0x00,0x00, +0x87,0x02,0x00,0x00,0x36,0x00,0x00,0x00,0xf7,0x00,0x03,0x00, +0x8c,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x8a,0x02,0x00,0x00,0x8b,0x02,0x00,0x00,0x8c,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x8b,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x8f,0x02,0x00,0x00,0x74,0x02,0x00,0x00, +0xc4,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, +0x90,0x02,0x00,0x00,0x12,0x00,0x00,0x00,0xcf,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x91,0x02,0x00,0x00, +0x90,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00, +0x92,0x02,0x00,0x00,0x8f,0x02,0x00,0x00,0x91,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0x8c,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x8c,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0xc1,0x00,0x00,0x00, +0x93,0x02,0x00,0x00,0x8a,0x02,0x00,0x00,0x7f,0x02,0x00,0x00, +0x92,0x02,0x00,0x00,0x8b,0x02,0x00,0x00,0xf7,0x00,0x03,0x00, +0x95,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x93,0x02,0x00,0x00,0x94,0x02,0x00,0x00,0x95,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x94,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x9d,0x02,0x00,0x00,0x74,0x02,0x00,0x00, +0xc4,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, +0x9f,0x02,0x00,0x00,0x12,0x00,0x00,0x00,0x9e,0x02,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xa0,0x02,0x00,0x00, +0x9f,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xa1,0x02,0x00,0x00,0x9d,0x02,0x00,0x00,0xa0,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa2,0x02,0x00,0x00, +0x54,0x02,0x00,0x00,0xa1,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xa4,0x02,0x00,0x00,0xa2,0x02,0x00,0x00, +0x6c,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xa6,0x02,0x00,0x00,0xa4,0x02,0x00,0x00,0xc6,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa8,0x02,0x00,0x00, +0xc1,0x02,0x00,0x00,0xbb,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xaa,0x02,0x00,0x00,0xa8,0x02,0x00,0x00, +0xc4,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xac,0x02,0x00,0x00,0xaa,0x02,0x00,0x00,0xab,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xae,0x02,0x00,0x00, +0xc2,0x02,0x00,0x00,0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xaf,0x02,0x00,0x00,0xac,0x02,0x00,0x00, +0xae,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xb1,0x02,0x00,0x00,0xaf,0x02,0x00,0x00,0xc6,0x02,0x00,0x00, +0x41,0x00,0x05,0x00,0xcc,0x00,0x00,0x00,0xb2,0x02,0x00,0x00, +0xc9,0x00,0x00,0x00,0xb1,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, +0xc3,0x00,0x00,0x00,0xb3,0x02,0x00,0x00,0xb2,0x02,0x00,0x00, +0x41,0x00,0x06,0x00,0x70,0x01,0x00,0x00,0xb4,0x02,0x00,0x00, +0x99,0x02,0x00,0x00,0x34,0x00,0x00,0x00,0xa6,0x02,0x00,0x00, +0x3e,0x00,0x03,0x00,0xb4,0x02,0x00,0x00,0xb3,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0x95,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x95,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x81,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x81,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xb6,0x02,0x00,0x00,0xc6,0x02,0x00,0x00, +0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x7e,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x80,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0x79,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x79,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xb8,0x02,0x00,0x00, +0xc4,0x02,0x00,0x00,0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x76,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x78,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0x61,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x61,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xba,0x02,0x00,0x00,0xc2,0x02,0x00,0x00,0xcf,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x5e,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x60,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x59,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x59,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xbc,0x02,0x00,0x00,0xc1,0x02,0x00,0x00, +0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x56,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x58,0x02,0x00,0x00,0xfd,0x00,0x01,0x00, +0x38,0x00,0x01,0x00, +}; +const uint64_t matmul_q4_0_f32_len = 10564; + +unsigned char matmul_q4_0_f32_aligned_data[] = { +0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, +0x37,0x03,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, +0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x09,0x00,0x00,0x00, +0x11,0x00,0x02,0x00,0x51,0x11,0x00,0x00,0x11,0x00,0x02,0x00, +0x60,0x11,0x00,0x00,0x0b,0x00,0x06,0x00,0x01,0x00,0x00,0x00, +0x47,0x4c,0x53,0x4c,0x2e,0x73,0x74,0x64,0x2e,0x34,0x35,0x30, +0x00,0x00,0x00,0x00,0x0e,0x00,0x03,0x00,0x00,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x0f,0x00,0x0f,0x00,0x05,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x6d,0x61,0x69,0x6e,0x00,0x00,0x00,0x00, +0x0b,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x3d,0x00,0x00,0x00, +0x4c,0x00,0x00,0x00,0x08,0x01,0x00,0x00,0x28,0x01,0x00,0x00, +0x5d,0x01,0x00,0x00,0x65,0x01,0x00,0x00,0x73,0x02,0x00,0x00, +0xbc,0x02,0x00,0x00,0x10,0x00,0x06,0x00,0x04,0x00,0x00,0x00, +0x11,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x0b,0x00,0x00,0x00, +0x0b,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x10,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x10,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x10,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x05,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x10,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, +0x07,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x1c,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x08,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x10,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x24,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, +0x0a,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x28,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x2c,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x10,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x30,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, +0x0d,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x34,0x00,0x00,0x00, +0x47,0x00,0x03,0x00,0x10,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x37,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x3d,0x00,0x00,0x00, +0x0b,0x00,0x00,0x00,0x1a,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x4c,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x1b,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x4f,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x09,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x53,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x60,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x62,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x07,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x6c,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0xa7,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0xb9,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x05,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xbc,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x03,0x01,0x00,0x00,0x06,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x04,0x01,0x00,0x00,0x00,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x04,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x05,0x01,0x00,0x00, +0x06,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x48,0x00,0x04,0x00, +0x06,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x06,0x01,0x00,0x00,0x00,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, +0x06,0x01,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x08,0x01,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x08,0x01,0x00,0x00,0x21,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x35,0x01,0x00,0x00, +0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x36,0x01,0x00,0x00,0x0b,0x00,0x00,0x00,0x19,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x62,0x01,0x00,0x00,0x06,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0x63,0x01,0x00,0x00, +0x00,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0x48,0x00,0x04,0x00, +0x63,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x63,0x01,0x00,0x00,0x00,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x63,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x07,0x00,0x00,0x00, +0x10,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x63,0x01,0x00,0x00, +0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x65,0x01,0x00,0x00, +0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x65,0x01,0x00,0x00,0x21,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x73,0x02,0x00,0x00,0x0b,0x00,0x00,0x00, +0x18,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xb9,0x02,0x00,0x00, +0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00, +0xba,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x19,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0xba,0x02,0x00,0x00,0x00,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, +0xba,0x02,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0xbc,0x02,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0xbc,0x02,0x00,0x00,0x21,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x13,0x00,0x02,0x00,0x02,0x00,0x00,0x00, +0x21,0x00,0x03,0x00,0x03,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x15,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x17,0x00,0x04,0x00,0x09,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x0a,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x09,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x0d,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x1e,0x00,0x10,0x00,0x10,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x11,0x00,0x00,0x00, +0x09,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0x11,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x09,0x00,0x00,0x00, +0x15,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, +0x14,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x15,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x21,0x00,0x00,0x00, +0x0a,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, +0x27,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0x2d,0x00,0x00,0x00,0x07,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x34,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x37,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00,0x3d,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x3e,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0x0a,0x00,0x00,0x00,0x4c,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x4f,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x53,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x54,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0x37,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x58,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0x37,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x32,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x61,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x60,0x00,0x00,0x00, +0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x62,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x63,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x61,0x00,0x00,0x00, +0x62,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x67,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x61,0x00,0x00,0x00, +0x62,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x72,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x77,0x00,0x00,0x00,0x08,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x78,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x77,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x7d,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x77,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x81,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0x91,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x97,0x00,0x00,0x00, +0x03,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, +0xa2,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x32,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xa7,0x00,0x00,0x00,0x40,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0xa9,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xb8,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00, +0x62,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xb9,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xba,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0x53,0x00,0x00,0x00,0xb9,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xbb,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0x4f,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x32,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xbc,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xbd,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0xbb,0x00,0x00,0x00,0xbc,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xbe,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0xbd,0x00,0x00,0x00,0x60,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xbf,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0xba,0x00,0x00,0x00,0xbe,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xc0,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0xb8,0x00,0x00,0x00,0xbf,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xc1,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0xc0,0x00,0x00,0x00,0xbc,0x00,0x00,0x00, +0x14,0x00,0x02,0x00,0xc2,0x00,0x00,0x00,0x16,0x00,0x03,0x00, +0xc4,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xc5,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xc6,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0xc5,0x00,0x00,0x00,0xbf,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xc7,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0xc6,0x00,0x00,0x00,0xbc,0x00,0x00,0x00,0x1c,0x00,0x04,0x00, +0xc8,0x00,0x00,0x00,0xc4,0x00,0x00,0x00,0xc7,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0xc9,0x00,0x00,0x00,0x07,0x00,0x00,0x00, +0xc8,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0xc4,0x00,0x00,0x00, +0xcc,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0xcd,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0xc4,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0xd0,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xf4,0x00,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, +0x39,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xfa,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xfe,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, +0x16,0x00,0x03,0x00,0x01,0x01,0x00,0x00,0x10,0x00,0x00,0x00, +0x15,0x00,0x04,0x00,0x02,0x01,0x00,0x00,0x08,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x1c,0x00,0x04,0x00,0x03,0x01,0x00,0x00, +0x02,0x01,0x00,0x00,0xfa,0x00,0x00,0x00,0x1e,0x00,0x04,0x00, +0x04,0x01,0x00,0x00,0x01,0x01,0x00,0x00,0x03,0x01,0x00,0x00, +0x1d,0x00,0x03,0x00,0x05,0x01,0x00,0x00,0x04,0x01,0x00,0x00, +0x1e,0x00,0x03,0x00,0x06,0x01,0x00,0x00,0x05,0x01,0x00,0x00, +0x20,0x00,0x04,0x00,0x07,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, +0x06,0x01,0x00,0x00,0x3b,0x00,0x04,0x00,0x07,0x01,0x00,0x00, +0x08,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x0a,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x01,0x01,0x00,0x00, +0x20,0x00,0x04,0x00,0x11,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, +0x02,0x01,0x00,0x00,0x17,0x00,0x04,0x00,0x15,0x01,0x00,0x00, +0xc4,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0xc4,0x00,0x00,0x00,0x1f,0x01,0x00,0x00,0x00,0x00,0x00,0x41, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x24,0x01,0x00,0x00, +0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x25,0x01,0x00,0x00, +0x84,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x24,0x01,0x00,0x00, +0x1c,0x00,0x04,0x00,0x26,0x01,0x00,0x00,0x01,0x01,0x00,0x00, +0x25,0x01,0x00,0x00,0x20,0x00,0x04,0x00,0x27,0x01,0x00,0x00, +0x04,0x00,0x00,0x00,0x26,0x01,0x00,0x00,0x3b,0x00,0x04,0x00, +0x27,0x01,0x00,0x00,0x28,0x01,0x00,0x00,0x04,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x2d,0x01,0x00,0x00,0x04,0x00,0x00,0x00, +0x01,0x01,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x35,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x33,0x00,0x06,0x00, +0x09,0x00,0x00,0x00,0x36,0x01,0x00,0x00,0x35,0x01,0x00,0x00, +0x39,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x37,0x01,0x00,0x00,0x51,0x00,0x00,0x00, +0x36,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x38,0x01,0x00,0x00,0x84,0x00,0x00,0x00, +0x37,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x39,0x01,0x00,0x00,0x86,0x00,0x00,0x00, +0x38,0x01,0x00,0x00,0x6c,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x54,0x01,0x00,0x00,0x80,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x59,0x01,0x00,0x00,0x80,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x5a,0x01,0x00,0x00,0x84,0x00,0x00,0x00, +0xa7,0x00,0x00,0x00,0x59,0x01,0x00,0x00,0x1c,0x00,0x04,0x00, +0x5b,0x01,0x00,0x00,0x01,0x01,0x00,0x00,0x5a,0x01,0x00,0x00, +0x20,0x00,0x04,0x00,0x5c,0x01,0x00,0x00,0x04,0x00,0x00,0x00, +0x5b,0x01,0x00,0x00,0x3b,0x00,0x04,0x00,0x5c,0x01,0x00,0x00, +0x5d,0x01,0x00,0x00,0x04,0x00,0x00,0x00,0x17,0x00,0x04,0x00, +0x60,0x01,0x00,0x00,0xc4,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x18,0x00,0x04,0x00,0x61,0x01,0x00,0x00,0x60,0x01,0x00,0x00, +0x02,0x00,0x00,0x00,0x1d,0x00,0x03,0x00,0x62,0x01,0x00,0x00, +0x61,0x01,0x00,0x00,0x1e,0x00,0x03,0x00,0x63,0x01,0x00,0x00, +0x62,0x01,0x00,0x00,0x20,0x00,0x04,0x00,0x64,0x01,0x00,0x00, +0x0c,0x00,0x00,0x00,0x63,0x01,0x00,0x00,0x3b,0x00,0x04,0x00, +0x64,0x01,0x00,0x00,0x65,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x67,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, +0xc4,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x7b,0x01,0x00,0x00,0x03,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x83,0x01,0x00,0x00,0x04,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x8b,0x01,0x00,0x00, +0x05,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x93,0x01,0x00,0x00,0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x9b,0x01,0x00,0x00,0x07,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xa2,0x01,0x00,0x00, +0x51,0x00,0x00,0x00,0x36,0x01,0x00,0x00,0x00,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xa3,0x01,0x00,0x00, +0x84,0x00,0x00,0x00,0xa2,0x01,0x00,0x00,0x77,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xa4,0x01,0x00,0x00, +0x86,0x00,0x00,0x00,0xa3,0x01,0x00,0x00,0x6c,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xa7,0x01,0x00,0x00, +0x08,0x01,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xa8,0x01,0x00,0x00,0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xab,0x01,0x00,0x00,0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, +0x77,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xc6,0x01,0x00,0x00,0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00, +0x62,0x00,0x00,0x00,0x1c,0x00,0x04,0x00,0xc7,0x01,0x00,0x00, +0x01,0x01,0x00,0x00,0xc6,0x01,0x00,0x00,0x20,0x00,0x04,0x00, +0xc8,0x01,0x00,0x00,0x07,0x00,0x00,0x00,0xc7,0x01,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xd8,0x01,0x00,0x00, +0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0xde,0x01,0x00,0x00,0x07,0x00,0x00,0x00, +0x01,0x01,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xf4,0x01,0x00,0x00,0x84,0x00,0x00,0x00,0xbf,0x00,0x00,0x00, +0xbc,0x00,0x00,0x00,0x1c,0x00,0x04,0x00,0xf5,0x01,0x00,0x00, +0x01,0x01,0x00,0x00,0xf4,0x01,0x00,0x00,0x20,0x00,0x04,0x00, +0xf6,0x01,0x00,0x00,0x07,0x00,0x00,0x00,0xf5,0x01,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xff,0x01,0x00,0x00, +0x86,0x00,0x00,0x00,0xb9,0x00,0x00,0x00,0xbf,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x07,0x02,0x00,0x00, +0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x36,0x02,0x00,0x00, +0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x6b,0x02,0x00,0x00, +0x0d,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00, +0x73,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, +0xb9,0x02,0x00,0x00,0xc4,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, +0xba,0x02,0x00,0x00,0xb9,0x02,0x00,0x00,0x20,0x00,0x04,0x00, +0xbb,0x02,0x00,0x00,0x0c,0x00,0x00,0x00,0xba,0x02,0x00,0x00, +0x3b,0x00,0x04,0x00,0xbb,0x02,0x00,0x00,0xbc,0x02,0x00,0x00, +0x0c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, +0xc1,0x02,0x00,0x00,0x05,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xce,0x02,0x00,0x00,0x84,0x00,0x00,0x00, +0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x2c,0x00,0x05,0x00, +0x15,0x01,0x00,0x00,0x36,0x03,0x00,0x00,0x1f,0x01,0x00,0x00, +0x1f,0x01,0x00,0x00,0x36,0x00,0x05,0x00,0x02,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x03,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0x05,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0xc9,0x00,0x00,0x00,0xca,0x00,0x00,0x00,0x07,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0xc8,0x01,0x00,0x00,0xc9,0x01,0x00,0x00, +0x07,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0xf6,0x01,0x00,0x00, +0xf7,0x01,0x00,0x00,0x07,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x0d,0x00,0x00,0x00,0x0e,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x0f,0x00,0x00,0x00,0x0e,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x15,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0x14,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x17,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, +0x17,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x1e,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x17,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x22,0x00,0x00,0x00, +0x12,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x22,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x24,0x00,0x00,0x00, +0x18,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x15,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0x27,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x29,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x2a,0x00,0x00,0x00,0x1e,0x00,0x00,0x00, +0x29,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, +0x2e,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x2d,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x2f,0x00,0x00,0x00, +0x2e,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x30,0x00,0x00,0x00,0x24,0x00,0x00,0x00,0x2f,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x32,0x00,0x00,0x00, +0x30,0x00,0x00,0x00,0x2a,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x15,0x00,0x00,0x00,0x35,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0x34,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x36,0x00,0x00,0x00,0x35,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x38,0x00,0x00,0x00,0x36,0x00,0x00,0x00, +0x37,0x00,0x00,0x00,0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x3a,0x00,0x00,0x00,0x38,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3b,0x00,0x00,0x00, +0x3a,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x0d,0x00,0x00,0x00,0x3f,0x00,0x00,0x00,0x3d,0x00,0x00,0x00, +0x3e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x40,0x00,0x00,0x00,0x3f,0x00,0x00,0x00,0x89,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x42,0x00,0x00,0x00,0x40,0x00,0x00,0x00, +0x3b,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x47,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x3b,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00,0x49,0x00,0x00,0x00, +0x3d,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x4a,0x00,0x00,0x00,0x49,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00,0x4d,0x00,0x00,0x00, +0x4c,0x00,0x00,0x00,0x3e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x4d,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x50,0x00,0x00,0x00, +0x4e,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x89,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x55,0x00,0x00,0x00,0x50,0x00,0x00,0x00, +0x54,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x59,0x00,0x00,0x00,0x50,0x00,0x00,0x00,0x58,0x00,0x00,0x00, +0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x5d,0x00,0x00,0x00, +0x4e,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x89,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x64,0x00,0x00,0x00,0x5d,0x00,0x00,0x00, +0x63,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x68,0x00,0x00,0x00,0x5d,0x00,0x00,0x00,0x67,0x00,0x00,0x00, +0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x6e,0x00,0x00,0x00, +0x4e,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x73,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, +0x72,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x79,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x78,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x7e,0x00,0x00,0x00, +0x4e,0x00,0x00,0x00,0x7d,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x15,0x00,0x00,0x00,0x82,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0x81,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x83,0x00,0x00,0x00,0x82,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x47,0x00,0x00,0x00, +0x83,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, +0x87,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x88,0x00,0x00,0x00, +0x87,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x8a,0x00,0x00,0x00,0x47,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8d,0x00,0x00,0x00, +0x8a,0x00,0x00,0x00,0x83,0x00,0x00,0x00,0x0c,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0x8e,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x26,0x00,0x00,0x00,0x88,0x00,0x00,0x00,0x8d,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x92,0x00,0x00,0x00, +0x12,0x00,0x00,0x00,0x91,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x93,0x00,0x00,0x00,0x92,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x94,0x00,0x00,0x00, +0x32,0x00,0x00,0x00,0x93,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x96,0x00,0x00,0x00,0x42,0x00,0x00,0x00, +0x37,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, +0x98,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x97,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x99,0x00,0x00,0x00, +0x98,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x9a,0x00,0x00,0x00,0x96,0x00,0x00,0x00,0x99,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9b,0x00,0x00,0x00, +0x94,0x00,0x00,0x00,0x9a,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x9d,0x00,0x00,0x00,0x9b,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x9e,0x00,0x00,0x00,0x9d,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0xa3,0x00,0x00,0x00, +0x12,0x00,0x00,0x00,0xa2,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xa4,0x00,0x00,0x00,0xa3,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa5,0x00,0x00,0x00, +0x0f,0x00,0x00,0x00,0xa4,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xa8,0x00,0x00,0x00,0x4a,0x00,0x00,0x00, +0xa7,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, +0xaa,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0xa9,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xab,0x00,0x00,0x00, +0xaa,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xac,0x00,0x00,0x00,0xa8,0x00,0x00,0x00,0xab,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xad,0x00,0x00,0x00, +0xa5,0x00,0x00,0x00,0xac,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xaf,0x00,0x00,0x00,0xad,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xb0,0x00,0x00,0x00,0xaf,0x00,0x00,0x00,0x77,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xb2,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xb2,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xe2,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0x05,0x00,0x00,0x00, +0xd1,0x00,0x00,0x00,0xb3,0x00,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc2,0x00,0x00,0x00,0xc3,0x00,0x00,0x00,0xe2,0x02,0x00,0x00, +0xc1,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xb4,0x00,0x00,0x00, +0xb3,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0xc3,0x00,0x00,0x00,0xb3,0x00,0x00,0x00,0xb4,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xb3,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0xcd,0x00,0x00,0x00,0xce,0x00,0x00,0x00,0xca,0x00,0x00,0x00, +0xe2,0x02,0x00,0x00,0x3e,0x00,0x03,0x00,0xce,0x00,0x00,0x00, +0xcc,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xd1,0x00,0x00,0x00,0xe2,0x02,0x00,0x00,0xd0,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xb2,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xb4,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xd4,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xd4,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xfb,0x02,0x00,0x00,0xb0,0x00,0x00,0x00, +0xb4,0x00,0x00,0x00,0xad,0x01,0x00,0x00,0xd7,0x00,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xf7,0x02,0x00,0x00, +0x9e,0x00,0x00,0x00,0xb4,0x00,0x00,0x00,0xaa,0x01,0x00,0x00, +0xd7,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xe3,0x02,0x00,0x00,0x84,0x00,0x00,0x00,0xb4,0x00,0x00,0x00, +0x5b,0x02,0x00,0x00,0xd7,0x00,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc2,0x00,0x00,0x00,0xdb,0x00,0x00,0x00,0xe3,0x02,0x00,0x00, +0x8e,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xd6,0x00,0x00,0x00, +0xd7,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0xdb,0x00,0x00,0x00,0xd5,0x00,0x00,0x00,0xd6,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xd5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xdd,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xdd,0x00,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xf3,0x02,0x00,0x00, +0x3e,0x00,0x00,0x00,0xd5,0x00,0x00,0x00,0x3b,0x01,0x00,0x00, +0xde,0x00,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, +0xe3,0x00,0x00,0x00,0xf3,0x02,0x00,0x00,0x37,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0xdf,0x00,0x00,0x00,0xde,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xe3,0x00,0x00,0x00, +0xde,0x00,0x00,0x00,0xdf,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xde,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xe8,0x00,0x00,0x00,0x73,0x00,0x00,0x00,0xf3,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xeb,0x00,0x00,0x00, +0xe8,0x00,0x00,0x00,0x99,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xec,0x00,0x00,0x00,0xeb,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xed,0x00,0x00,0x00,0xf7,0x02,0x00,0x00,0xec,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xef,0x00,0x00,0x00, +0xed,0x00,0x00,0x00,0x6e,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xf5,0x00,0x00,0x00,0xe8,0x00,0x00,0x00, +0xf4,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xf7,0x00,0x00,0x00,0xf5,0x00,0x00,0x00,0x6e,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xfb,0x00,0x00,0x00, +0xef,0x00,0x00,0x00,0xfa,0x00,0x00,0x00,0xc7,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xff,0x00,0x00,0x00,0xef,0x00,0x00,0x00, +0xfe,0x00,0x00,0x00,0x41,0x00,0x07,0x00,0x0a,0x01,0x00,0x00, +0x0b,0x01,0x00,0x00,0x08,0x01,0x00,0x00,0x34,0x00,0x00,0x00, +0xfb,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x01,0x01,0x00,0x00,0x0c,0x01,0x00,0x00,0x0b,0x01,0x00,0x00, +0x73,0x00,0x04,0x00,0xc4,0x00,0x00,0x00,0x0d,0x01,0x00,0x00, +0x0c,0x01,0x00,0x00,0x41,0x00,0x08,0x00,0x11,0x01,0x00,0x00, +0x12,0x01,0x00,0x00,0x08,0x01,0x00,0x00,0x34,0x00,0x00,0x00, +0xfb,0x00,0x00,0x00,0xd0,0x00,0x00,0x00,0xff,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x02,0x01,0x00,0x00,0x13,0x01,0x00,0x00, +0x12,0x01,0x00,0x00,0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x14,0x01,0x00,0x00,0x13,0x01,0x00,0x00,0xc7,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x19,0x01,0x00,0x00,0x14,0x01,0x00,0x00, +0xfe,0x00,0x00,0x00,0x70,0x00,0x04,0x00,0xc4,0x00,0x00,0x00, +0x1a,0x01,0x00,0x00,0x19,0x01,0x00,0x00,0xc2,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x1c,0x01,0x00,0x00,0x14,0x01,0x00,0x00, +0xa9,0x00,0x00,0x00,0x70,0x00,0x04,0x00,0xc4,0x00,0x00,0x00, +0x1d,0x01,0x00,0x00,0x1c,0x01,0x00,0x00,0x50,0x00,0x05,0x00, +0x15,0x01,0x00,0x00,0x1e,0x01,0x00,0x00,0x1a,0x01,0x00,0x00, +0x1d,0x01,0x00,0x00,0x83,0x00,0x05,0x00,0x15,0x01,0x00,0x00, +0x21,0x01,0x00,0x00,0x1e,0x01,0x00,0x00,0x36,0x03,0x00,0x00, +0x8e,0x00,0x05,0x00,0x15,0x01,0x00,0x00,0x23,0x01,0x00,0x00, +0x21,0x01,0x00,0x00,0x0d,0x01,0x00,0x00,0x51,0x00,0x05,0x00, +0xc4,0x00,0x00,0x00,0x2b,0x01,0x00,0x00,0x23,0x01,0x00,0x00, +0x00,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x01,0x01,0x00,0x00, +0x2c,0x01,0x00,0x00,0x2b,0x01,0x00,0x00,0x41,0x00,0x05,0x00, +0x2d,0x01,0x00,0x00,0x2e,0x01,0x00,0x00,0x28,0x01,0x00,0x00, +0xf7,0x00,0x00,0x00,0x3e,0x00,0x03,0x00,0x2e,0x01,0x00,0x00, +0x2c,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x30,0x01,0x00,0x00,0xf7,0x00,0x00,0x00,0xfa,0x00,0x00,0x00, +0x51,0x00,0x05,0x00,0xc4,0x00,0x00,0x00,0x32,0x01,0x00,0x00, +0x23,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x73,0x00,0x04,0x00, +0x01,0x01,0x00,0x00,0x33,0x01,0x00,0x00,0x32,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0x2d,0x01,0x00,0x00,0x34,0x01,0x00,0x00, +0x28,0x01,0x00,0x00,0x30,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, +0x34,0x01,0x00,0x00,0x33,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x3b,0x01,0x00,0x00,0xf3,0x02,0x00,0x00, +0x39,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xdd,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xdf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x3d,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x3d,0x01,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xf4,0x02,0x00,0x00, +0x3e,0x00,0x00,0x00,0xdf,0x00,0x00,0x00,0xa6,0x01,0x00,0x00, +0x3e,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, +0x43,0x01,0x00,0x00,0xf4,0x02,0x00,0x00,0xa7,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0x3f,0x01,0x00,0x00,0x3e,0x01,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x43,0x01,0x00,0x00, +0x3e,0x01,0x00,0x00,0x3f,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x3e,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x48,0x01,0x00,0x00,0x7e,0x00,0x00,0x00,0xf4,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x4b,0x01,0x00,0x00, +0x48,0x01,0x00,0x00,0xab,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x4c,0x01,0x00,0x00,0x4b,0x01,0x00,0x00, +0x77,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x4d,0x01,0x00,0x00,0xfb,0x02,0x00,0x00,0x4c,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x4f,0x01,0x00,0x00, +0x4d,0x01,0x00,0x00,0x79,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x55,0x01,0x00,0x00,0x48,0x01,0x00,0x00, +0x54,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x57,0x01,0x00,0x00,0x79,0x00,0x00,0x00,0x77,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x58,0x01,0x00,0x00, +0x55,0x01,0x00,0x00,0x57,0x01,0x00,0x00,0x41,0x00,0x08,0x00, +0x67,0x01,0x00,0x00,0x68,0x01,0x00,0x00,0x65,0x01,0x00,0x00, +0x34,0x00,0x00,0x00,0x4f,0x01,0x00,0x00,0x34,0x00,0x00,0x00, +0x3e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00, +0x69,0x01,0x00,0x00,0x68,0x01,0x00,0x00,0x73,0x00,0x04,0x00, +0x01,0x01,0x00,0x00,0x6a,0x01,0x00,0x00,0x69,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0x2d,0x01,0x00,0x00,0x6b,0x01,0x00,0x00, +0x5d,0x01,0x00,0x00,0x58,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, +0x6b,0x01,0x00,0x00,0x6a,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x6d,0x01,0x00,0x00,0x58,0x01,0x00,0x00, +0x39,0x00,0x00,0x00,0x41,0x00,0x08,0x00,0x67,0x01,0x00,0x00, +0x6f,0x01,0x00,0x00,0x65,0x01,0x00,0x00,0x34,0x00,0x00,0x00, +0x4f,0x01,0x00,0x00,0x34,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00,0x70,0x01,0x00,0x00, +0x6f,0x01,0x00,0x00,0x73,0x00,0x04,0x00,0x01,0x01,0x00,0x00, +0x71,0x01,0x00,0x00,0x70,0x01,0x00,0x00,0x41,0x00,0x05,0x00, +0x2d,0x01,0x00,0x00,0x72,0x01,0x00,0x00,0x5d,0x01,0x00,0x00, +0x6d,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x72,0x01,0x00,0x00, +0x71,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x74,0x01,0x00,0x00,0x58,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, +0x41,0x00,0x08,0x00,0x67,0x01,0x00,0x00,0x76,0x01,0x00,0x00, +0x65,0x01,0x00,0x00,0x34,0x00,0x00,0x00,0x4f,0x01,0x00,0x00, +0x34,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0xc4,0x00,0x00,0x00,0x77,0x01,0x00,0x00,0x76,0x01,0x00,0x00, +0x73,0x00,0x04,0x00,0x01,0x01,0x00,0x00,0x78,0x01,0x00,0x00, +0x77,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0x2d,0x01,0x00,0x00, +0x79,0x01,0x00,0x00,0x5d,0x01,0x00,0x00,0x74,0x01,0x00,0x00, +0x3e,0x00,0x03,0x00,0x79,0x01,0x00,0x00,0x78,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x7c,0x01,0x00,0x00, +0x58,0x01,0x00,0x00,0x7b,0x01,0x00,0x00,0x41,0x00,0x08,0x00, +0x67,0x01,0x00,0x00,0x7e,0x01,0x00,0x00,0x65,0x01,0x00,0x00, +0x34,0x00,0x00,0x00,0x4f,0x01,0x00,0x00,0x34,0x00,0x00,0x00, +0x7b,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00, +0x7f,0x01,0x00,0x00,0x7e,0x01,0x00,0x00,0x73,0x00,0x04,0x00, +0x01,0x01,0x00,0x00,0x80,0x01,0x00,0x00,0x7f,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0x2d,0x01,0x00,0x00,0x81,0x01,0x00,0x00, +0x5d,0x01,0x00,0x00,0x7c,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, +0x81,0x01,0x00,0x00,0x80,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x84,0x01,0x00,0x00,0x58,0x01,0x00,0x00, +0x83,0x01,0x00,0x00,0x41,0x00,0x08,0x00,0x67,0x01,0x00,0x00, +0x86,0x01,0x00,0x00,0x65,0x01,0x00,0x00,0x34,0x00,0x00,0x00, +0x4f,0x01,0x00,0x00,0xd0,0x00,0x00,0x00,0x3e,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00,0x87,0x01,0x00,0x00, +0x86,0x01,0x00,0x00,0x73,0x00,0x04,0x00,0x01,0x01,0x00,0x00, +0x88,0x01,0x00,0x00,0x87,0x01,0x00,0x00,0x41,0x00,0x05,0x00, +0x2d,0x01,0x00,0x00,0x89,0x01,0x00,0x00,0x5d,0x01,0x00,0x00, +0x84,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x89,0x01,0x00,0x00, +0x88,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x8c,0x01,0x00,0x00,0x58,0x01,0x00,0x00,0x8b,0x01,0x00,0x00, +0x41,0x00,0x08,0x00,0x67,0x01,0x00,0x00,0x8e,0x01,0x00,0x00, +0x65,0x01,0x00,0x00,0x34,0x00,0x00,0x00,0x4f,0x01,0x00,0x00, +0xd0,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0xc4,0x00,0x00,0x00,0x8f,0x01,0x00,0x00,0x8e,0x01,0x00,0x00, +0x73,0x00,0x04,0x00,0x01,0x01,0x00,0x00,0x90,0x01,0x00,0x00, +0x8f,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0x2d,0x01,0x00,0x00, +0x91,0x01,0x00,0x00,0x5d,0x01,0x00,0x00,0x8c,0x01,0x00,0x00, +0x3e,0x00,0x03,0x00,0x91,0x01,0x00,0x00,0x90,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x94,0x01,0x00,0x00, +0x58,0x01,0x00,0x00,0x93,0x01,0x00,0x00,0x41,0x00,0x08,0x00, +0x67,0x01,0x00,0x00,0x96,0x01,0x00,0x00,0x65,0x01,0x00,0x00, +0x34,0x00,0x00,0x00,0x4f,0x01,0x00,0x00,0xd0,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00, +0x97,0x01,0x00,0x00,0x96,0x01,0x00,0x00,0x73,0x00,0x04,0x00, +0x01,0x01,0x00,0x00,0x98,0x01,0x00,0x00,0x97,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0x2d,0x01,0x00,0x00,0x99,0x01,0x00,0x00, +0x5d,0x01,0x00,0x00,0x94,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, +0x99,0x01,0x00,0x00,0x98,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x9c,0x01,0x00,0x00,0x58,0x01,0x00,0x00, +0x9b,0x01,0x00,0x00,0x41,0x00,0x08,0x00,0x67,0x01,0x00,0x00, +0x9e,0x01,0x00,0x00,0x65,0x01,0x00,0x00,0x34,0x00,0x00,0x00, +0x4f,0x01,0x00,0x00,0xd0,0x00,0x00,0x00,0x7b,0x01,0x00,0x00, +0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00,0x9f,0x01,0x00,0x00, +0x9e,0x01,0x00,0x00,0x73,0x00,0x04,0x00,0x01,0x01,0x00,0x00, +0xa0,0x01,0x00,0x00,0x9f,0x01,0x00,0x00,0x41,0x00,0x05,0x00, +0x2d,0x01,0x00,0x00,0xa1,0x01,0x00,0x00,0x5d,0x01,0x00,0x00, +0x9c,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0xa1,0x01,0x00,0x00, +0xa0,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xa6,0x01,0x00,0x00,0xf4,0x02,0x00,0x00,0xa4,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0x3d,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x3f,0x01,0x00,0x00,0xe0,0x00,0x04,0x00,0x0c,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0xa7,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xaa,0x01,0x00,0x00,0xf7,0x02,0x00,0x00, +0xa8,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xad,0x01,0x00,0x00,0xfb,0x02,0x00,0x00,0xab,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0xaf,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xaf,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xfd,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0x3f,0x01,0x00,0x00, +0x59,0x02,0x00,0x00,0xb2,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc2,0x00,0x00,0x00,0xb5,0x01,0x00,0x00,0xfd,0x02,0x00,0x00, +0x6c,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xb1,0x01,0x00,0x00, +0xb2,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0xb5,0x01,0x00,0x00,0xb0,0x01,0x00,0x00,0xb1,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xb0,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0xb7,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xb7,0x01,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x01,0x03,0x00,0x00, +0x3e,0x00,0x00,0x00,0xb0,0x01,0x00,0x00,0xe3,0x01,0x00,0x00, +0xba,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, +0xbd,0x01,0x00,0x00,0x01,0x03,0x00,0x00,0x60,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0xb9,0x01,0x00,0x00,0xba,0x01,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xbd,0x01,0x00,0x00, +0xb8,0x01,0x00,0x00,0xb9,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xb8,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xbf,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xbf,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0x13,0x03,0x00,0x00,0x3e,0x00,0x00,0x00, +0xb8,0x01,0x00,0x00,0xe1,0x01,0x00,0x00,0xc0,0x01,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0xc5,0x01,0x00,0x00, +0x13,0x03,0x00,0x00,0x62,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0xc1,0x01,0x00,0x00,0xc0,0x01,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0xc5,0x01,0x00,0x00,0xc0,0x01,0x00,0x00, +0xc1,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xc0,0x01,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xcb,0x01,0x00,0x00, +0x01,0x03,0x00,0x00,0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xcd,0x01,0x00,0x00,0xcb,0x01,0x00,0x00, +0x13,0x03,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xcf,0x01,0x00,0x00,0x55,0x00,0x00,0x00,0x53,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xd1,0x01,0x00,0x00, +0x01,0x03,0x00,0x00,0x61,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xd2,0x01,0x00,0x00,0xcf,0x01,0x00,0x00, +0xd1,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xd4,0x01,0x00,0x00,0x64,0x00,0x00,0x00,0x62,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xd5,0x01,0x00,0x00, +0xd2,0x01,0x00,0x00,0xd4,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xd7,0x01,0x00,0x00,0xd5,0x01,0x00,0x00, +0x13,0x03,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xd9,0x01,0x00,0x00,0xd7,0x01,0x00,0x00,0xd8,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xdb,0x01,0x00,0x00, +0xd9,0x01,0x00,0x00,0xfd,0x02,0x00,0x00,0x41,0x00,0x05,0x00, +0x2d,0x01,0x00,0x00,0xdc,0x01,0x00,0x00,0x28,0x01,0x00,0x00, +0xdb,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0x01,0x01,0x00,0x00, +0xdd,0x01,0x00,0x00,0xdc,0x01,0x00,0x00,0x41,0x00,0x05,0x00, +0xde,0x01,0x00,0x00,0xdf,0x01,0x00,0x00,0xc9,0x01,0x00,0x00, +0xcd,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0xdf,0x01,0x00,0x00, +0xdd,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xe1,0x01,0x00,0x00,0x13,0x03,0x00,0x00,0xd0,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xbf,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xc1,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xba,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xba,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xe3,0x01,0x00,0x00,0x01,0x03,0x00,0x00, +0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xb7,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xb9,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0xe5,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xe5,0x01,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x02,0x03,0x00,0x00, +0x3e,0x00,0x00,0x00,0xb9,0x01,0x00,0x00,0x11,0x02,0x00,0x00, +0xe8,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, +0xeb,0x01,0x00,0x00,0x02,0x03,0x00,0x00,0xbf,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0xe7,0x01,0x00,0x00,0xe8,0x01,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xeb,0x01,0x00,0x00, +0xe6,0x01,0x00,0x00,0xe7,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xe6,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xed,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xed,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0x10,0x03,0x00,0x00,0x3e,0x00,0x00,0x00, +0xe6,0x01,0x00,0x00,0x0f,0x02,0x00,0x00,0xee,0x01,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0xf3,0x01,0x00,0x00, +0x10,0x03,0x00,0x00,0xbc,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0xef,0x01,0x00,0x00,0xee,0x01,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0xf3,0x01,0x00,0x00,0xee,0x01,0x00,0x00, +0xef,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xee,0x01,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xf9,0x01,0x00,0x00, +0x02,0x03,0x00,0x00,0xbc,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xfb,0x01,0x00,0x00,0xf9,0x01,0x00,0x00, +0x10,0x03,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xfd,0x01,0x00,0x00,0x59,0x00,0x00,0x00,0xb9,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x00,0x02,0x00,0x00, +0x02,0x03,0x00,0x00,0xff,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x01,0x02,0x00,0x00,0xfd,0x01,0x00,0x00, +0x00,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x03,0x02,0x00,0x00,0x68,0x00,0x00,0x00,0xbc,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x04,0x02,0x00,0x00, +0x01,0x02,0x00,0x00,0x03,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x06,0x02,0x00,0x00,0x04,0x02,0x00,0x00, +0x10,0x03,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x08,0x02,0x00,0x00,0x06,0x02,0x00,0x00,0x07,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x0a,0x02,0x00,0x00, +0x08,0x02,0x00,0x00,0xfd,0x02,0x00,0x00,0x41,0x00,0x05,0x00, +0x2d,0x01,0x00,0x00,0x0b,0x02,0x00,0x00,0x5d,0x01,0x00,0x00, +0x0a,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0x01,0x01,0x00,0x00, +0x0c,0x02,0x00,0x00,0x0b,0x02,0x00,0x00,0x41,0x00,0x05,0x00, +0xde,0x01,0x00,0x00,0x0d,0x02,0x00,0x00,0xf7,0x01,0x00,0x00, +0xfb,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x0d,0x02,0x00,0x00, +0x0c,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x0f,0x02,0x00,0x00,0x10,0x03,0x00,0x00,0xd0,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xed,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xef,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xe8,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xe8,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x11,0x02,0x00,0x00,0x02,0x03,0x00,0x00, +0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xe5,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xe7,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0x13,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x13,0x02,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x03,0x03,0x00,0x00, +0x3e,0x00,0x00,0x00,0xe7,0x01,0x00,0x00,0x57,0x02,0x00,0x00, +0x16,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, +0x19,0x02,0x00,0x00,0x03,0x03,0x00,0x00,0xbf,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0x15,0x02,0x00,0x00,0x16,0x02,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x19,0x02,0x00,0x00, +0x14,0x02,0x00,0x00,0x15,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x14,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x1b,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x1b,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0x07,0x03,0x00,0x00,0x3e,0x00,0x00,0x00, +0x14,0x02,0x00,0x00,0x55,0x02,0x00,0x00,0x1e,0x02,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0x21,0x02,0x00,0x00, +0x07,0x03,0x00,0x00,0x60,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0x1d,0x02,0x00,0x00,0x1e,0x02,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x21,0x02,0x00,0x00,0x1c,0x02,0x00,0x00, +0x1d,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x1c,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0x23,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x23,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0x09,0x03,0x00,0x00,0x3e,0x00,0x00,0x00,0x1c,0x02,0x00,0x00, +0x53,0x02,0x00,0x00,0x26,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc2,0x00,0x00,0x00,0x29,0x02,0x00,0x00,0x09,0x03,0x00,0x00, +0xbc,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x25,0x02,0x00,0x00, +0x26,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x29,0x02,0x00,0x00,0x24,0x02,0x00,0x00,0x25,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x24,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0x2b,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x2b,0x02,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x0b,0x03,0x00,0x00, +0x3e,0x00,0x00,0x00,0x24,0x02,0x00,0x00,0x51,0x02,0x00,0x00, +0x2c,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, +0x31,0x02,0x00,0x00,0x0b,0x03,0x00,0x00,0x62,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0x2d,0x02,0x00,0x00,0x2c,0x02,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x31,0x02,0x00,0x00, +0x2c,0x02,0x00,0x00,0x2d,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x2c,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x33,0x02,0x00,0x00,0x03,0x03,0x00,0x00,0xbc,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x35,0x02,0x00,0x00, +0x33,0x02,0x00,0x00,0x09,0x03,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x37,0x02,0x00,0x00,0x35,0x02,0x00,0x00, +0x36,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x39,0x02,0x00,0x00,0x07,0x03,0x00,0x00,0x62,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3a,0x02,0x00,0x00, +0x37,0x02,0x00,0x00,0x39,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x3c,0x02,0x00,0x00,0x3a,0x02,0x00,0x00, +0x0b,0x03,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x40,0x02,0x00,0x00,0x39,0x02,0x00,0x00,0x0b,0x03,0x00,0x00, +0x41,0x00,0x05,0x00,0xde,0x01,0x00,0x00,0x41,0x02,0x00,0x00, +0xc9,0x01,0x00,0x00,0x40,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, +0x01,0x01,0x00,0x00,0x42,0x02,0x00,0x00,0x41,0x02,0x00,0x00, +0x73,0x00,0x04,0x00,0xc4,0x00,0x00,0x00,0x43,0x02,0x00,0x00, +0x42,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0xde,0x01,0x00,0x00, +0x48,0x02,0x00,0x00,0xf7,0x01,0x00,0x00,0x35,0x02,0x00,0x00, +0x3d,0x00,0x04,0x00,0x01,0x01,0x00,0x00,0x49,0x02,0x00,0x00, +0x48,0x02,0x00,0x00,0x73,0x00,0x04,0x00,0xc4,0x00,0x00,0x00, +0x4a,0x02,0x00,0x00,0x49,0x02,0x00,0x00,0x41,0x00,0x05,0x00, +0xcd,0x00,0x00,0x00,0x4c,0x02,0x00,0x00,0xca,0x00,0x00,0x00, +0x3c,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00, +0x4d,0x02,0x00,0x00,0x4c,0x02,0x00,0x00,0x0c,0x00,0x08,0x00, +0xc4,0x00,0x00,0x00,0x4e,0x02,0x00,0x00,0x01,0x00,0x00,0x00, +0x32,0x00,0x00,0x00,0x43,0x02,0x00,0x00,0x4a,0x02,0x00,0x00, +0x4d,0x02,0x00,0x00,0x3e,0x00,0x03,0x00,0x4c,0x02,0x00,0x00, +0x4e,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x51,0x02,0x00,0x00,0x0b,0x03,0x00,0x00,0xd0,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x2b,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x2d,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x26,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x26,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x53,0x02,0x00,0x00,0x09,0x03,0x00,0x00, +0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x23,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x25,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0x1e,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x1e,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x55,0x02,0x00,0x00, +0x07,0x03,0x00,0x00,0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x1b,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x1d,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0x16,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x16,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x57,0x02,0x00,0x00,0x03,0x03,0x00,0x00,0xd0,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x13,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x15,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0xb2,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xb2,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x59,0x02,0x00,0x00,0xfd,0x02,0x00,0x00, +0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xaf,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xb1,0x01,0x00,0x00,0xe0,0x00,0x04,0x00, +0x0c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0xa7,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0xd7,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xd7,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x5b,0x02,0x00,0x00,0xe3,0x02,0x00,0x00,0x6c,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xd4,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xd6,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x60,0x02,0x00,0x00,0x55,0x00,0x00,0x00,0x53,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x61,0x02,0x00,0x00, +0x96,0x00,0x00,0x00,0x60,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x66,0x02,0x00,0x00,0x59,0x00,0x00,0x00, +0xb9,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x67,0x02,0x00,0x00,0xa8,0x00,0x00,0x00,0x66,0x02,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x6c,0x02,0x00,0x00, +0x12,0x00,0x00,0x00,0x6b,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x6d,0x02,0x00,0x00,0x6c,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x6e,0x02,0x00,0x00, +0x0f,0x00,0x00,0x00,0x6d,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x72,0x02,0x00,0x00,0x47,0x00,0x00,0x00, +0x6d,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00, +0x74,0x02,0x00,0x00,0x73,0x02,0x00,0x00,0x0c,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x75,0x02,0x00,0x00, +0x74,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x76,0x02,0x00,0x00,0x72,0x02,0x00,0x00,0x75,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x77,0x02,0x00,0x00, +0x6e,0x02,0x00,0x00,0x76,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0x79,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x79,0x02,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xe4,0x02,0x00,0x00, +0x3e,0x00,0x00,0x00,0xd6,0x00,0x00,0x00,0xdf,0x02,0x00,0x00, +0x7c,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, +0x7f,0x02,0x00,0x00,0xe4,0x02,0x00,0x00,0xbf,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0x7b,0x02,0x00,0x00,0x7c,0x02,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x7f,0x02,0x00,0x00, +0x7a,0x02,0x00,0x00,0x7b,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x7a,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x81,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x81,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xe5,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, +0x7a,0x02,0x00,0x00,0xdd,0x02,0x00,0x00,0x84,0x02,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0x87,0x02,0x00,0x00, +0xe5,0x02,0x00,0x00,0x60,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0x83,0x02,0x00,0x00,0x84,0x02,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x87,0x02,0x00,0x00,0x82,0x02,0x00,0x00, +0x83,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x82,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8b,0x02,0x00,0x00, +0xe5,0x02,0x00,0x00,0x61,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x8c,0x02,0x00,0x00,0x61,0x02,0x00,0x00, +0x8b,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x8e,0x02,0x00,0x00,0x64,0x00,0x00,0x00,0x62,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8f,0x02,0x00,0x00, +0x8c,0x02,0x00,0x00,0x8e,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x93,0x02,0x00,0x00,0xe4,0x02,0x00,0x00, +0xff,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x94,0x02,0x00,0x00,0x67,0x02,0x00,0x00,0x93,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x96,0x02,0x00,0x00, +0x68,0x00,0x00,0x00,0xbc,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x97,0x02,0x00,0x00,0x94,0x02,0x00,0x00, +0x96,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x99,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x99,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xe7,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, +0x82,0x02,0x00,0x00,0xdb,0x02,0x00,0x00,0x9c,0x02,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0x9f,0x02,0x00,0x00, +0xe7,0x02,0x00,0x00,0xbc,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0x9b,0x02,0x00,0x00,0x9c,0x02,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x9f,0x02,0x00,0x00,0x9a,0x02,0x00,0x00, +0x9b,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x9a,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0xa1,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0xa1,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xe9,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0x9a,0x02,0x00,0x00, +0xd9,0x02,0x00,0x00,0xa4,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc2,0x00,0x00,0x00,0xa7,0x02,0x00,0x00,0xe9,0x02,0x00,0x00, +0x62,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xa3,0x02,0x00,0x00, +0xa4,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0xa7,0x02,0x00,0x00,0xa2,0x02,0x00,0x00,0xa3,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0xa2,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xaa,0x02,0x00,0x00,0x8f,0x02,0x00,0x00, +0xe9,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, +0xad,0x02,0x00,0x00,0xaa,0x02,0x00,0x00,0x36,0x00,0x00,0x00, +0xf7,0x00,0x03,0x00,0xaf,0x02,0x00,0x00,0x00,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0xad,0x02,0x00,0x00,0xae,0x02,0x00,0x00, +0xaf,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0xae,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xb2,0x02,0x00,0x00, +0x97,0x02,0x00,0x00,0xe7,0x02,0x00,0x00,0x41,0x00,0x05,0x00, +0x15,0x00,0x00,0x00,0xb3,0x02,0x00,0x00,0x12,0x00,0x00,0x00, +0xd0,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xb4,0x02,0x00,0x00,0xb3,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc2,0x00,0x00,0x00,0xb5,0x02,0x00,0x00,0xb2,0x02,0x00,0x00, +0xb4,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0xaf,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0xaf,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, +0xc2,0x00,0x00,0x00,0xb6,0x02,0x00,0x00,0xad,0x02,0x00,0x00, +0xa2,0x02,0x00,0x00,0xb5,0x02,0x00,0x00,0xae,0x02,0x00,0x00, +0xf7,0x00,0x03,0x00,0xb8,0x02,0x00,0x00,0x00,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0xb6,0x02,0x00,0x00,0xb7,0x02,0x00,0x00, +0xb8,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0xb7,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xc0,0x02,0x00,0x00, +0x97,0x02,0x00,0x00,0xe7,0x02,0x00,0x00,0x41,0x00,0x05,0x00, +0x15,0x00,0x00,0x00,0xc2,0x02,0x00,0x00,0x12,0x00,0x00,0x00, +0xc1,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xc3,0x02,0x00,0x00,0xc2,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xc4,0x02,0x00,0x00,0xc0,0x02,0x00,0x00, +0xc3,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xc5,0x02,0x00,0x00,0x77,0x02,0x00,0x00,0xc4,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xc7,0x02,0x00,0x00, +0xc5,0x02,0x00,0x00,0x8f,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xc9,0x02,0x00,0x00,0xc7,0x02,0x00,0x00, +0xe9,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xcb,0x02,0x00,0x00,0xe4,0x02,0x00,0x00,0xbc,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xcd,0x02,0x00,0x00, +0xcb,0x02,0x00,0x00,0xe7,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xcf,0x02,0x00,0x00,0xcd,0x02,0x00,0x00, +0xce,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xd1,0x02,0x00,0x00,0xe5,0x02,0x00,0x00,0x62,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xd2,0x02,0x00,0x00, +0xcf,0x02,0x00,0x00,0xd1,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xd4,0x02,0x00,0x00,0xd2,0x02,0x00,0x00, +0xe9,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0xcd,0x00,0x00,0x00, +0xd5,0x02,0x00,0x00,0xca,0x00,0x00,0x00,0xd4,0x02,0x00,0x00, +0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00,0xd6,0x02,0x00,0x00, +0xd5,0x02,0x00,0x00,0x41,0x00,0x06,0x00,0x67,0x01,0x00,0x00, +0xd7,0x02,0x00,0x00,0xbc,0x02,0x00,0x00,0x34,0x00,0x00,0x00, +0xc9,0x02,0x00,0x00,0x3e,0x00,0x03,0x00,0xd7,0x02,0x00,0x00, +0xd6,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0xb8,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0xb8,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0xa4,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0xa4,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xd9,0x02,0x00,0x00, +0xe9,0x02,0x00,0x00,0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xa1,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0xa3,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0x9c,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x9c,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xdb,0x02,0x00,0x00,0xe7,0x02,0x00,0x00,0xd0,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x99,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x9b,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x84,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x84,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xdd,0x02,0x00,0x00,0xe5,0x02,0x00,0x00, +0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x81,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x83,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0x7c,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x7c,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xdf,0x02,0x00,0x00, +0xe4,0x02,0x00,0x00,0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x79,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x7b,0x02,0x00,0x00, +0xfd,0x00,0x01,0x00,0x38,0x00,0x01,0x00, +}; +const uint64_t matmul_q4_0_f32_aligned_len = 11156; + +unsigned char matmul_q4_0_f32_aligned_fp32_data[] = { +0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, +0x0d,0x03,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, +0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x51,0x11,0x00,0x00, +0x11,0x00,0x02,0x00,0x60,0x11,0x00,0x00,0x0b,0x00,0x06,0x00, +0x01,0x00,0x00,0x00,0x47,0x4c,0x53,0x4c,0x2e,0x73,0x74,0x64, +0x2e,0x34,0x35,0x30,0x00,0x00,0x00,0x00,0x0e,0x00,0x03,0x00, +0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x0f,0x00,0x0f,0x00, +0x05,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x6d,0x61,0x69,0x6e, +0x00,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0x3d,0x00,0x00,0x00,0x4c,0x00,0x00,0x00,0x08,0x01,0x00,0x00, +0x28,0x01,0x00,0x00,0x5b,0x01,0x00,0x00,0x62,0x01,0x00,0x00, +0x49,0x02,0x00,0x00,0x92,0x02,0x00,0x00,0x10,0x00,0x06,0x00, +0x04,0x00,0x00,0x00,0x11,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x0b,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x1c,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x10,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x08,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x03,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x10,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x10,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, +0x05,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x14,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x10,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x1c,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, +0x08,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x09,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x24,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x10,0x00,0x00,0x00,0x0a,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x28,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, +0x0b,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x2c,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x30,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x10,0x00,0x00,0x00,0x0d,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x34,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x10,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x37,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x3d,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x1a,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x4c,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, +0x1b,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x4f,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x53,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x60,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x62,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x6c,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x03,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0xa7,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xb9,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0xbc,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x08,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x03,0x01,0x00,0x00,0x06,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x04,0x01,0x00,0x00, +0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x04,0x01,0x00,0x00,0x01,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x05,0x01,0x00,0x00,0x06,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0x48,0x00,0x04,0x00,0x06,0x01,0x00,0x00,0x00,0x00,0x00,0x00, +0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x06,0x01,0x00,0x00, +0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x03,0x00,0x06,0x01,0x00,0x00,0x02,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x08,0x01,0x00,0x00,0x22,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x08,0x01,0x00,0x00, +0x21,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x33,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x34,0x01,0x00,0x00,0x0b,0x00,0x00,0x00, +0x19,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x5f,0x01,0x00,0x00, +0x06,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x48,0x00,0x04,0x00, +0x60,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x60,0x01,0x00,0x00,0x00,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, +0x60,0x01,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x62,0x01,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x62,0x01,0x00,0x00,0x21,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x49,0x02,0x00,0x00, +0x0b,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x8f,0x02,0x00,0x00,0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x48,0x00,0x04,0x00,0x90,0x02,0x00,0x00,0x00,0x00,0x00,0x00, +0x19,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x90,0x02,0x00,0x00, +0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x03,0x00,0x90,0x02,0x00,0x00,0x02,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x92,0x02,0x00,0x00,0x22,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x92,0x02,0x00,0x00, +0x21,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x13,0x00,0x02,0x00, +0x02,0x00,0x00,0x00,0x21,0x00,0x03,0x00,0x03,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x15,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x17,0x00,0x04,0x00, +0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x03,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x0a,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x09,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00, +0x0b,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x0d,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x1e,0x00,0x10,0x00,0x10,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x11,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x10,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0x11,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0x09,0x00,0x00,0x00,0x15,0x00,0x04,0x00,0x13,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x08,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x15,0x00,0x00,0x00,0x09,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, +0x21,0x00,0x00,0x00,0x0a,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0x27,0x00,0x00,0x00,0x09,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x2d,0x00,0x00,0x00, +0x07,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, +0x34,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x32,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x40,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00, +0x3d,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x3e,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00,0x4c,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x4f,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x32,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x54,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x53,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x58,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x53,0x00,0x00,0x00, +0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x60,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x61,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x53,0x00,0x00,0x00, +0x60,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x62,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x63,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0x61,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x67,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0x61,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x32,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x10,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x72,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x77,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x78,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, +0x77,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x7d,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, +0x77,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, +0x81,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x91,0x00,0x00,0x00, +0x0b,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, +0x97,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0xa2,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xa7,0x00,0x00,0x00, +0x40,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, +0xa9,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xb8,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x32,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xb9,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xba,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0xb9,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xbb,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x62,0x00,0x00,0x00, +0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xbc,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xbd,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0xbb,0x00,0x00,0x00, +0xbc,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xbe,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0xbd,0x00,0x00,0x00, +0x60,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xbf,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0xba,0x00,0x00,0x00, +0xbe,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xc0,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0xb8,0x00,0x00,0x00, +0xbf,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xc1,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0xc0,0x00,0x00,0x00, +0xbc,0x00,0x00,0x00,0x14,0x00,0x02,0x00,0xc2,0x00,0x00,0x00, +0x16,0x00,0x03,0x00,0xc4,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xc5,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xc6,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0xc5,0x00,0x00,0x00,0xbf,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xc7,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0xc6,0x00,0x00,0x00,0xbc,0x00,0x00,0x00, +0x1c,0x00,0x04,0x00,0xc8,0x00,0x00,0x00,0xc4,0x00,0x00,0x00, +0xc7,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0xc9,0x00,0x00,0x00, +0x07,0x00,0x00,0x00,0xc8,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0xc4,0x00,0x00,0x00,0xcc,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0xcd,0x00,0x00,0x00,0x07,0x00,0x00,0x00, +0xc4,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, +0xd0,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xf4,0x00,0x00,0x00,0x80,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xfa,0x00,0x00,0x00,0x10,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xfe,0x00,0x00,0x00, +0x0f,0x00,0x00,0x00,0x16,0x00,0x03,0x00,0x01,0x01,0x00,0x00, +0x10,0x00,0x00,0x00,0x15,0x00,0x04,0x00,0x02,0x01,0x00,0x00, +0x08,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x1c,0x00,0x04,0x00, +0x03,0x01,0x00,0x00,0x02,0x01,0x00,0x00,0xfa,0x00,0x00,0x00, +0x1e,0x00,0x04,0x00,0x04,0x01,0x00,0x00,0x01,0x01,0x00,0x00, +0x03,0x01,0x00,0x00,0x1d,0x00,0x03,0x00,0x05,0x01,0x00,0x00, +0x04,0x01,0x00,0x00,0x1e,0x00,0x03,0x00,0x06,0x01,0x00,0x00, +0x05,0x01,0x00,0x00,0x20,0x00,0x04,0x00,0x07,0x01,0x00,0x00, +0x0c,0x00,0x00,0x00,0x06,0x01,0x00,0x00,0x3b,0x00,0x04,0x00, +0x07,0x01,0x00,0x00,0x08,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x0a,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, +0x01,0x01,0x00,0x00,0x20,0x00,0x04,0x00,0x11,0x01,0x00,0x00, +0x0c,0x00,0x00,0x00,0x02,0x01,0x00,0x00,0x17,0x00,0x04,0x00, +0x15,0x01,0x00,0x00,0xc4,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0xc4,0x00,0x00,0x00,0x1f,0x01,0x00,0x00, +0x00,0x00,0x00,0x41,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x24,0x01,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, +0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x25,0x01,0x00,0x00,0x84,0x00,0x00,0x00,0x37,0x00,0x00,0x00, +0x24,0x01,0x00,0x00,0x1c,0x00,0x04,0x00,0x26,0x01,0x00,0x00, +0xc4,0x00,0x00,0x00,0x25,0x01,0x00,0x00,0x20,0x00,0x04,0x00, +0x27,0x01,0x00,0x00,0x04,0x00,0x00,0x00,0x26,0x01,0x00,0x00, +0x3b,0x00,0x04,0x00,0x27,0x01,0x00,0x00,0x28,0x01,0x00,0x00, +0x04,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x2c,0x01,0x00,0x00, +0x04,0x00,0x00,0x00,0xc4,0x00,0x00,0x00,0x32,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x33,0x01,0x00,0x00,0x01,0x00,0x00,0x00, +0x33,0x00,0x06,0x00,0x09,0x00,0x00,0x00,0x34,0x01,0x00,0x00, +0x33,0x01,0x00,0x00,0x39,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x35,0x01,0x00,0x00, +0x51,0x00,0x00,0x00,0x34,0x01,0x00,0x00,0x00,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x36,0x01,0x00,0x00, +0x84,0x00,0x00,0x00,0x35,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x37,0x01,0x00,0x00, +0x86,0x00,0x00,0x00,0x36,0x01,0x00,0x00,0x6c,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x52,0x01,0x00,0x00, +0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x57,0x01,0x00,0x00, +0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x58,0x01,0x00,0x00, +0x84,0x00,0x00,0x00,0xa7,0x00,0x00,0x00,0x57,0x01,0x00,0x00, +0x1c,0x00,0x04,0x00,0x59,0x01,0x00,0x00,0xc4,0x00,0x00,0x00, +0x58,0x01,0x00,0x00,0x20,0x00,0x04,0x00,0x5a,0x01,0x00,0x00, +0x04,0x00,0x00,0x00,0x59,0x01,0x00,0x00,0x3b,0x00,0x04,0x00, +0x5a,0x01,0x00,0x00,0x5b,0x01,0x00,0x00,0x04,0x00,0x00,0x00, +0x17,0x00,0x04,0x00,0x5e,0x01,0x00,0x00,0xc4,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x1d,0x00,0x03,0x00,0x5f,0x01,0x00,0x00, +0x5e,0x01,0x00,0x00,0x1e,0x00,0x03,0x00,0x60,0x01,0x00,0x00, +0x5f,0x01,0x00,0x00,0x20,0x00,0x04,0x00,0x61,0x01,0x00,0x00, +0x0c,0x00,0x00,0x00,0x60,0x01,0x00,0x00,0x3b,0x00,0x04,0x00, +0x61,0x01,0x00,0x00,0x62,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x64,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, +0xc4,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x75,0x01,0x00,0x00,0x03,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x7b,0x01,0x00,0x00,0x51,0x00,0x00,0x00, +0x34,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x7c,0x01,0x00,0x00,0x84,0x00,0x00,0x00, +0x7b,0x01,0x00,0x00,0x77,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x7d,0x01,0x00,0x00,0x86,0x00,0x00,0x00, +0x7c,0x01,0x00,0x00,0x6c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x80,0x01,0x00,0x00,0x08,0x01,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x81,0x01,0x00,0x00, +0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x84,0x01,0x00,0x00, +0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x77,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x9f,0x01,0x00,0x00, +0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00, +0x1c,0x00,0x04,0x00,0xa0,0x01,0x00,0x00,0xc4,0x00,0x00,0x00, +0x9f,0x01,0x00,0x00,0x20,0x00,0x04,0x00,0xa1,0x01,0x00,0x00, +0x07,0x00,0x00,0x00,0xa0,0x01,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xb1,0x01,0x00,0x00,0x80,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xcc,0x01,0x00,0x00,0x84,0x00,0x00,0x00, +0xbf,0x00,0x00,0x00,0xbc,0x00,0x00,0x00,0x1c,0x00,0x04,0x00, +0xcd,0x01,0x00,0x00,0xc4,0x00,0x00,0x00,0xcc,0x01,0x00,0x00, +0x20,0x00,0x04,0x00,0xce,0x01,0x00,0x00,0x07,0x00,0x00,0x00, +0xcd,0x01,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xd7,0x01,0x00,0x00,0x86,0x00,0x00,0x00,0xb9,0x00,0x00,0x00, +0xbf,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xdf,0x01,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, +0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x0e,0x02,0x00,0x00,0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00, +0x62,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, +0x41,0x02,0x00,0x00,0x0d,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0x0a,0x00,0x00,0x00,0x49,0x02,0x00,0x00,0x01,0x00,0x00,0x00, +0x1d,0x00,0x03,0x00,0x8f,0x02,0x00,0x00,0xc4,0x00,0x00,0x00, +0x1e,0x00,0x03,0x00,0x90,0x02,0x00,0x00,0x8f,0x02,0x00,0x00, +0x20,0x00,0x04,0x00,0x91,0x02,0x00,0x00,0x0c,0x00,0x00,0x00, +0x90,0x02,0x00,0x00,0x3b,0x00,0x04,0x00,0x91,0x02,0x00,0x00, +0x92,0x02,0x00,0x00,0x0c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0x97,0x02,0x00,0x00,0x05,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xa4,0x02,0x00,0x00, +0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00, +0x2c,0x00,0x05,0x00,0x15,0x01,0x00,0x00,0x0c,0x03,0x00,0x00, +0x1f,0x01,0x00,0x00,0x1f,0x01,0x00,0x00,0x36,0x00,0x05,0x00, +0x02,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x03,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x05,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0xc9,0x00,0x00,0x00,0xca,0x00,0x00,0x00, +0x07,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0xa1,0x01,0x00,0x00, +0xa2,0x01,0x00,0x00,0x07,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0xce,0x01,0x00,0x00,0xcf,0x01,0x00,0x00,0x07,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00,0x0e,0x00,0x00,0x00, +0x0b,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x0e,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x16,0x00,0x00,0x00, +0x12,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x17,0x00,0x00,0x00,0x16,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x18,0x00,0x00,0x00, +0x0f,0x00,0x00,0x00,0x17,0x00,0x00,0x00,0x89,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x1e,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, +0x17,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, +0x22,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x21,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x22,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x24,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x28,0x00,0x00,0x00, +0x12,0x00,0x00,0x00,0x27,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x29,0x00,0x00,0x00,0x28,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x2a,0x00,0x00,0x00, +0x1e,0x00,0x00,0x00,0x29,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x15,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0x2d,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x2f,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x30,0x00,0x00,0x00,0x24,0x00,0x00,0x00, +0x2f,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x32,0x00,0x00,0x00,0x30,0x00,0x00,0x00,0x2a,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x35,0x00,0x00,0x00, +0x12,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x36,0x00,0x00,0x00,0x35,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x38,0x00,0x00,0x00, +0x36,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x82,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x38,0x00,0x00,0x00, +0x39,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x3b,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x37,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00,0x3f,0x00,0x00,0x00, +0x3d,0x00,0x00,0x00,0x3e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x3f,0x00,0x00,0x00, +0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x42,0x00,0x00,0x00, +0x40,0x00,0x00,0x00,0x3b,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x47,0x00,0x00,0x00,0x40,0x00,0x00,0x00, +0x3b,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00, +0x49,0x00,0x00,0x00,0x3d,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x4a,0x00,0x00,0x00, +0x49,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00, +0x4d,0x00,0x00,0x00,0x4c,0x00,0x00,0x00,0x3e,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, +0x4d,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x50,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x4f,0x00,0x00,0x00, +0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x55,0x00,0x00,0x00, +0x50,0x00,0x00,0x00,0x54,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x59,0x00,0x00,0x00,0x50,0x00,0x00,0x00, +0x58,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x5d,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x4f,0x00,0x00,0x00, +0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x64,0x00,0x00,0x00, +0x5d,0x00,0x00,0x00,0x63,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x68,0x00,0x00,0x00,0x5d,0x00,0x00,0x00, +0x67,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x6e,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x73,0x00,0x00,0x00, +0x4e,0x00,0x00,0x00,0x72,0x00,0x00,0x00,0x89,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x79,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, +0x78,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x7e,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x7d,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x82,0x00,0x00,0x00, +0x12,0x00,0x00,0x00,0x81,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x83,0x00,0x00,0x00,0x82,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0x47,0x00,0x00,0x00,0x83,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x15,0x00,0x00,0x00,0x87,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x88,0x00,0x00,0x00,0x87,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x8a,0x00,0x00,0x00,0x47,0x00,0x00,0x00, +0x39,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x8d,0x00,0x00,0x00,0x8a,0x00,0x00,0x00,0x83,0x00,0x00,0x00, +0x0c,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x8e,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x26,0x00,0x00,0x00,0x88,0x00,0x00,0x00, +0x8d,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, +0x92,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x91,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x93,0x00,0x00,0x00, +0x92,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x94,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x93,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x96,0x00,0x00,0x00, +0x42,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x15,0x00,0x00,0x00,0x98,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0x97,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x99,0x00,0x00,0x00,0x98,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x9a,0x00,0x00,0x00,0x96,0x00,0x00,0x00, +0x99,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x9b,0x00,0x00,0x00,0x94,0x00,0x00,0x00,0x9a,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9d,0x00,0x00,0x00, +0x9b,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x9e,0x00,0x00,0x00,0x9d,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, +0xa3,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0xa2,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xa4,0x00,0x00,0x00, +0xa3,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xa5,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0xa4,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa8,0x00,0x00,0x00, +0x4a,0x00,0x00,0x00,0xa7,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x15,0x00,0x00,0x00,0xaa,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0xa9,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xab,0x00,0x00,0x00,0xaa,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xac,0x00,0x00,0x00,0xa8,0x00,0x00,0x00, +0xab,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xad,0x00,0x00,0x00,0xa5,0x00,0x00,0x00,0xac,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xaf,0x00,0x00,0x00, +0xad,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xb0,0x00,0x00,0x00,0xaf,0x00,0x00,0x00, +0x77,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xb2,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xb2,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xb8,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, +0x05,0x00,0x00,0x00,0xd1,0x00,0x00,0x00,0xb3,0x00,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0xc3,0x00,0x00,0x00, +0xb8,0x02,0x00,0x00,0xc1,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0xb4,0x00,0x00,0x00,0xb3,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0xc3,0x00,0x00,0x00,0xb3,0x00,0x00,0x00, +0xb4,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xb3,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0xcd,0x00,0x00,0x00,0xce,0x00,0x00,0x00, +0xca,0x00,0x00,0x00,0xb8,0x02,0x00,0x00,0x3e,0x00,0x03,0x00, +0xce,0x00,0x00,0x00,0xcc,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xd1,0x00,0x00,0x00,0xb8,0x02,0x00,0x00, +0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xb2,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xb4,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xd4,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xd4,0x00,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xd1,0x02,0x00,0x00, +0xb0,0x00,0x00,0x00,0xb4,0x00,0x00,0x00,0x86,0x01,0x00,0x00, +0xd7,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xcd,0x02,0x00,0x00,0x9e,0x00,0x00,0x00,0xb4,0x00,0x00,0x00, +0x83,0x01,0x00,0x00,0xd7,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xb9,0x02,0x00,0x00,0x84,0x00,0x00,0x00, +0xb4,0x00,0x00,0x00,0x31,0x02,0x00,0x00,0xd7,0x00,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0xdb,0x00,0x00,0x00, +0xb9,0x02,0x00,0x00,0x8e,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0xd6,0x00,0x00,0x00,0xd7,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0xdb,0x00,0x00,0x00,0xd5,0x00,0x00,0x00, +0xd6,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xd5,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xdd,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xdd,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xc9,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0xd5,0x00,0x00,0x00, +0x39,0x01,0x00,0x00,0xde,0x00,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc2,0x00,0x00,0x00,0xe3,0x00,0x00,0x00,0xc9,0x02,0x00,0x00, +0x37,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xdf,0x00,0x00,0x00, +0xde,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0xe3,0x00,0x00,0x00,0xde,0x00,0x00,0x00,0xdf,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xde,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xe8,0x00,0x00,0x00,0x73,0x00,0x00,0x00, +0xc9,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xeb,0x00,0x00,0x00,0xe8,0x00,0x00,0x00,0x99,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xec,0x00,0x00,0x00, +0xeb,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xed,0x00,0x00,0x00,0xcd,0x02,0x00,0x00, +0xec,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xef,0x00,0x00,0x00,0xed,0x00,0x00,0x00,0x6e,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xf5,0x00,0x00,0x00, +0xe8,0x00,0x00,0x00,0xf4,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xf7,0x00,0x00,0x00,0xf5,0x00,0x00,0x00, +0x6e,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xfb,0x00,0x00,0x00,0xef,0x00,0x00,0x00,0xfa,0x00,0x00,0x00, +0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xff,0x00,0x00,0x00, +0xef,0x00,0x00,0x00,0xfe,0x00,0x00,0x00,0x41,0x00,0x07,0x00, +0x0a,0x01,0x00,0x00,0x0b,0x01,0x00,0x00,0x08,0x01,0x00,0x00, +0x34,0x00,0x00,0x00,0xfb,0x00,0x00,0x00,0x34,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x01,0x01,0x00,0x00,0x0c,0x01,0x00,0x00, +0x0b,0x01,0x00,0x00,0x73,0x00,0x04,0x00,0xc4,0x00,0x00,0x00, +0x0d,0x01,0x00,0x00,0x0c,0x01,0x00,0x00,0x41,0x00,0x08,0x00, +0x11,0x01,0x00,0x00,0x12,0x01,0x00,0x00,0x08,0x01,0x00,0x00, +0x34,0x00,0x00,0x00,0xfb,0x00,0x00,0x00,0xd0,0x00,0x00,0x00, +0xff,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x02,0x01,0x00,0x00, +0x13,0x01,0x00,0x00,0x12,0x01,0x00,0x00,0x71,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x14,0x01,0x00,0x00,0x13,0x01,0x00,0x00, +0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x19,0x01,0x00,0x00, +0x14,0x01,0x00,0x00,0xfe,0x00,0x00,0x00,0x70,0x00,0x04,0x00, +0xc4,0x00,0x00,0x00,0x1a,0x01,0x00,0x00,0x19,0x01,0x00,0x00, +0xc2,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x1c,0x01,0x00,0x00, +0x14,0x01,0x00,0x00,0xa9,0x00,0x00,0x00,0x70,0x00,0x04,0x00, +0xc4,0x00,0x00,0x00,0x1d,0x01,0x00,0x00,0x1c,0x01,0x00,0x00, +0x50,0x00,0x05,0x00,0x15,0x01,0x00,0x00,0x1e,0x01,0x00,0x00, +0x1a,0x01,0x00,0x00,0x1d,0x01,0x00,0x00,0x83,0x00,0x05,0x00, +0x15,0x01,0x00,0x00,0x21,0x01,0x00,0x00,0x1e,0x01,0x00,0x00, +0x0c,0x03,0x00,0x00,0x8e,0x00,0x05,0x00,0x15,0x01,0x00,0x00, +0x23,0x01,0x00,0x00,0x21,0x01,0x00,0x00,0x0d,0x01,0x00,0x00, +0x51,0x00,0x05,0x00,0xc4,0x00,0x00,0x00,0x2b,0x01,0x00,0x00, +0x23,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x2c,0x01,0x00,0x00,0x2d,0x01,0x00,0x00,0x28,0x01,0x00,0x00, +0xf7,0x00,0x00,0x00,0x3e,0x00,0x03,0x00,0x2d,0x01,0x00,0x00, +0x2b,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x2f,0x01,0x00,0x00,0xf7,0x00,0x00,0x00,0xfa,0x00,0x00,0x00, +0x51,0x00,0x05,0x00,0xc4,0x00,0x00,0x00,0x31,0x01,0x00,0x00, +0x23,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x2c,0x01,0x00,0x00,0x32,0x01,0x00,0x00,0x28,0x01,0x00,0x00, +0x2f,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x32,0x01,0x00,0x00, +0x31,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x39,0x01,0x00,0x00,0xc9,0x02,0x00,0x00,0x37,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0xdd,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xdf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x3b,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x3b,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xca,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, +0xdf,0x00,0x00,0x00,0x7f,0x01,0x00,0x00,0x3c,0x01,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0x41,0x01,0x00,0x00, +0xca,0x02,0x00,0x00,0xa7,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0x3d,0x01,0x00,0x00,0x3c,0x01,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x41,0x01,0x00,0x00,0x3c,0x01,0x00,0x00, +0x3d,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x3c,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x46,0x01,0x00,0x00, +0x7e,0x00,0x00,0x00,0xca,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x49,0x01,0x00,0x00,0x46,0x01,0x00,0x00, +0xab,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x4a,0x01,0x00,0x00,0x49,0x01,0x00,0x00,0x77,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x4b,0x01,0x00,0x00, +0xd1,0x02,0x00,0x00,0x4a,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x4d,0x01,0x00,0x00,0x4b,0x01,0x00,0x00, +0x79,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x53,0x01,0x00,0x00,0x46,0x01,0x00,0x00,0x52,0x01,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x55,0x01,0x00,0x00, +0x79,0x00,0x00,0x00,0x77,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x56,0x01,0x00,0x00,0x53,0x01,0x00,0x00, +0x55,0x01,0x00,0x00,0x41,0x00,0x07,0x00,0x64,0x01,0x00,0x00, +0x65,0x01,0x00,0x00,0x62,0x01,0x00,0x00,0x34,0x00,0x00,0x00, +0x4d,0x01,0x00,0x00,0x3e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0xc4,0x00,0x00,0x00,0x66,0x01,0x00,0x00,0x65,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0x2c,0x01,0x00,0x00,0x67,0x01,0x00,0x00, +0x5b,0x01,0x00,0x00,0x56,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, +0x67,0x01,0x00,0x00,0x66,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x69,0x01,0x00,0x00,0x56,0x01,0x00,0x00, +0x39,0x00,0x00,0x00,0x41,0x00,0x07,0x00,0x64,0x01,0x00,0x00, +0x6b,0x01,0x00,0x00,0x62,0x01,0x00,0x00,0x34,0x00,0x00,0x00, +0x4d,0x01,0x00,0x00,0x39,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0xc4,0x00,0x00,0x00,0x6c,0x01,0x00,0x00,0x6b,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0x2c,0x01,0x00,0x00,0x6d,0x01,0x00,0x00, +0x5b,0x01,0x00,0x00,0x69,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, +0x6d,0x01,0x00,0x00,0x6c,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x6f,0x01,0x00,0x00,0x56,0x01,0x00,0x00, +0x0c,0x00,0x00,0x00,0x41,0x00,0x07,0x00,0x64,0x01,0x00,0x00, +0x71,0x01,0x00,0x00,0x62,0x01,0x00,0x00,0x34,0x00,0x00,0x00, +0x4d,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0xc4,0x00,0x00,0x00,0x72,0x01,0x00,0x00,0x71,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0x2c,0x01,0x00,0x00,0x73,0x01,0x00,0x00, +0x5b,0x01,0x00,0x00,0x6f,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, +0x73,0x01,0x00,0x00,0x72,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x76,0x01,0x00,0x00,0x56,0x01,0x00,0x00, +0x75,0x01,0x00,0x00,0x41,0x00,0x07,0x00,0x64,0x01,0x00,0x00, +0x78,0x01,0x00,0x00,0x62,0x01,0x00,0x00,0x34,0x00,0x00,0x00, +0x4d,0x01,0x00,0x00,0x75,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, +0xc4,0x00,0x00,0x00,0x79,0x01,0x00,0x00,0x78,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0x2c,0x01,0x00,0x00,0x7a,0x01,0x00,0x00, +0x5b,0x01,0x00,0x00,0x76,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, +0x7a,0x01,0x00,0x00,0x79,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x7f,0x01,0x00,0x00,0xca,0x02,0x00,0x00, +0x7d,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x3b,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x3d,0x01,0x00,0x00,0xe0,0x00,0x04,0x00, +0x0c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x80,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x83,0x01,0x00,0x00, +0xcd,0x02,0x00,0x00,0x81,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x86,0x01,0x00,0x00,0xd1,0x02,0x00,0x00, +0x84,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x88,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x88,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xd3,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, +0x3d,0x01,0x00,0x00,0x2f,0x02,0x00,0x00,0x8b,0x01,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0x8e,0x01,0x00,0x00, +0xd3,0x02,0x00,0x00,0x6c,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0x8a,0x01,0x00,0x00,0x8b,0x01,0x00,0x00,0x00,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x8e,0x01,0x00,0x00,0x89,0x01,0x00,0x00, +0x8a,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x89,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0x90,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x90,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xd7,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0x89,0x01,0x00,0x00, +0xbb,0x01,0x00,0x00,0x93,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc2,0x00,0x00,0x00,0x96,0x01,0x00,0x00,0xd7,0x02,0x00,0x00, +0x60,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x92,0x01,0x00,0x00, +0x93,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x96,0x01,0x00,0x00,0x91,0x01,0x00,0x00,0x92,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x91,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0x98,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x98,0x01,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xe9,0x02,0x00,0x00, +0x3e,0x00,0x00,0x00,0x91,0x01,0x00,0x00,0xb9,0x01,0x00,0x00, +0x99,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, +0x9e,0x01,0x00,0x00,0xe9,0x02,0x00,0x00,0x62,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0x9a,0x01,0x00,0x00,0x99,0x01,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x9e,0x01,0x00,0x00, +0x99,0x01,0x00,0x00,0x9a,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x99,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xa4,0x01,0x00,0x00,0xd7,0x02,0x00,0x00,0x62,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa6,0x01,0x00,0x00, +0xa4,0x01,0x00,0x00,0xe9,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xa8,0x01,0x00,0x00,0x55,0x00,0x00,0x00, +0x53,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xaa,0x01,0x00,0x00,0xd7,0x02,0x00,0x00,0x61,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xab,0x01,0x00,0x00, +0xa8,0x01,0x00,0x00,0xaa,0x01,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xad,0x01,0x00,0x00,0x64,0x00,0x00,0x00, +0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xae,0x01,0x00,0x00,0xab,0x01,0x00,0x00,0xad,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xb0,0x01,0x00,0x00, +0xae,0x01,0x00,0x00,0xe9,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xb2,0x01,0x00,0x00,0xb0,0x01,0x00,0x00, +0xb1,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xb4,0x01,0x00,0x00,0xb2,0x01,0x00,0x00,0xd3,0x02,0x00,0x00, +0x41,0x00,0x05,0x00,0x2c,0x01,0x00,0x00,0xb5,0x01,0x00,0x00, +0x28,0x01,0x00,0x00,0xb4,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, +0xc4,0x00,0x00,0x00,0xb6,0x01,0x00,0x00,0xb5,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0xcd,0x00,0x00,0x00,0xb7,0x01,0x00,0x00, +0xa2,0x01,0x00,0x00,0xa6,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, +0xb7,0x01,0x00,0x00,0xb6,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xb9,0x01,0x00,0x00,0xe9,0x02,0x00,0x00, +0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x98,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x9a,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0x93,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x93,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xbb,0x01,0x00,0x00, +0xd7,0x02,0x00,0x00,0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x90,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x92,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0xbd,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xbd,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xd8,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0x92,0x01,0x00,0x00, +0xe9,0x01,0x00,0x00,0xc0,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc2,0x00,0x00,0x00,0xc3,0x01,0x00,0x00,0xd8,0x02,0x00,0x00, +0xbf,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xbf,0x01,0x00,0x00, +0xc0,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0xc3,0x01,0x00,0x00,0xbe,0x01,0x00,0x00,0xbf,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xbe,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0xc5,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xc5,0x01,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xe6,0x02,0x00,0x00, +0x3e,0x00,0x00,0x00,0xbe,0x01,0x00,0x00,0xe7,0x01,0x00,0x00, +0xc6,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, +0xcb,0x01,0x00,0x00,0xe6,0x02,0x00,0x00,0xbc,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0xc7,0x01,0x00,0x00,0xc6,0x01,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xcb,0x01,0x00,0x00, +0xc6,0x01,0x00,0x00,0xc7,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xc6,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xd1,0x01,0x00,0x00,0xd8,0x02,0x00,0x00,0xbc,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xd3,0x01,0x00,0x00, +0xd1,0x01,0x00,0x00,0xe6,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xd5,0x01,0x00,0x00,0x59,0x00,0x00,0x00, +0xb9,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xd8,0x01,0x00,0x00,0xd8,0x02,0x00,0x00,0xd7,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xd9,0x01,0x00,0x00, +0xd5,0x01,0x00,0x00,0xd8,0x01,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xdb,0x01,0x00,0x00,0x68,0x00,0x00,0x00, +0xbc,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xdc,0x01,0x00,0x00,0xd9,0x01,0x00,0x00,0xdb,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xde,0x01,0x00,0x00, +0xdc,0x01,0x00,0x00,0xe6,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xe0,0x01,0x00,0x00,0xde,0x01,0x00,0x00, +0xdf,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xe2,0x01,0x00,0x00,0xe0,0x01,0x00,0x00,0xd3,0x02,0x00,0x00, +0x41,0x00,0x05,0x00,0x2c,0x01,0x00,0x00,0xe3,0x01,0x00,0x00, +0x5b,0x01,0x00,0x00,0xe2,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, +0xc4,0x00,0x00,0x00,0xe4,0x01,0x00,0x00,0xe3,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0xcd,0x00,0x00,0x00,0xe5,0x01,0x00,0x00, +0xcf,0x01,0x00,0x00,0xd3,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, +0xe5,0x01,0x00,0x00,0xe4,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xe7,0x01,0x00,0x00,0xe6,0x02,0x00,0x00, +0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xc5,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xc7,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0xc0,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xc0,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xe9,0x01,0x00,0x00, +0xd8,0x02,0x00,0x00,0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xbd,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xbf,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0xeb,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xeb,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xd9,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0xbf,0x01,0x00,0x00, +0x2d,0x02,0x00,0x00,0xee,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc2,0x00,0x00,0x00,0xf1,0x01,0x00,0x00,0xd9,0x02,0x00,0x00, +0xbf,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xed,0x01,0x00,0x00, +0xee,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0xf1,0x01,0x00,0x00,0xec,0x01,0x00,0x00,0xed,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xec,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0xf3,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xf3,0x01,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xdd,0x02,0x00,0x00, +0x3e,0x00,0x00,0x00,0xec,0x01,0x00,0x00,0x2b,0x02,0x00,0x00, +0xf6,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, +0xf9,0x01,0x00,0x00,0xdd,0x02,0x00,0x00,0x60,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0xf5,0x01,0x00,0x00,0xf6,0x01,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xf9,0x01,0x00,0x00, +0xf4,0x01,0x00,0x00,0xf5,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xf4,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xfb,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xfb,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xdf,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, +0xf4,0x01,0x00,0x00,0x29,0x02,0x00,0x00,0xfe,0x01,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0x01,0x02,0x00,0x00, +0xdf,0x02,0x00,0x00,0xbc,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0xfd,0x01,0x00,0x00,0xfe,0x01,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x01,0x02,0x00,0x00,0xfc,0x01,0x00,0x00, +0xfd,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xfc,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0x03,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x03,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xe1,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0xfc,0x01,0x00,0x00, +0x27,0x02,0x00,0x00,0x04,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc2,0x00,0x00,0x00,0x09,0x02,0x00,0x00,0xe1,0x02,0x00,0x00, +0x62,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x05,0x02,0x00,0x00, +0x04,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x09,0x02,0x00,0x00,0x04,0x02,0x00,0x00,0x05,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x04,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x0b,0x02,0x00,0x00,0xd9,0x02,0x00,0x00, +0xbc,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x0d,0x02,0x00,0x00,0x0b,0x02,0x00,0x00,0xdf,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x0f,0x02,0x00,0x00, +0x0d,0x02,0x00,0x00,0x0e,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x11,0x02,0x00,0x00,0xdd,0x02,0x00,0x00, +0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x12,0x02,0x00,0x00,0x0f,0x02,0x00,0x00,0x11,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x14,0x02,0x00,0x00, +0x12,0x02,0x00,0x00,0xe1,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x18,0x02,0x00,0x00,0x11,0x02,0x00,0x00, +0xe1,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0xcd,0x00,0x00,0x00, +0x19,0x02,0x00,0x00,0xa2,0x01,0x00,0x00,0x18,0x02,0x00,0x00, +0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00,0x1a,0x02,0x00,0x00, +0x19,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0xcd,0x00,0x00,0x00, +0x1f,0x02,0x00,0x00,0xcf,0x01,0x00,0x00,0x0d,0x02,0x00,0x00, +0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00,0x20,0x02,0x00,0x00, +0x1f,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0xcd,0x00,0x00,0x00, +0x22,0x02,0x00,0x00,0xca,0x00,0x00,0x00,0x14,0x02,0x00,0x00, +0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00,0x23,0x02,0x00,0x00, +0x22,0x02,0x00,0x00,0x0c,0x00,0x08,0x00,0xc4,0x00,0x00,0x00, +0x24,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00, +0x1a,0x02,0x00,0x00,0x20,0x02,0x00,0x00,0x23,0x02,0x00,0x00, +0x3e,0x00,0x03,0x00,0x22,0x02,0x00,0x00,0x24,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x27,0x02,0x00,0x00, +0xe1,0x02,0x00,0x00,0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x03,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x05,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0xfe,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xfe,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x29,0x02,0x00,0x00,0xdf,0x02,0x00,0x00,0xd0,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xfb,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xfd,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xf6,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xf6,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x2b,0x02,0x00,0x00,0xdd,0x02,0x00,0x00, +0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xf3,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xf5,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0xee,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xee,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x2d,0x02,0x00,0x00, +0xd9,0x02,0x00,0x00,0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xeb,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xed,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0x8b,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x8b,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x2f,0x02,0x00,0x00,0xd3,0x02,0x00,0x00,0xd0,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x88,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x8a,0x01,0x00,0x00,0xe0,0x00,0x04,0x00,0x0c,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x80,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0xd7,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xd7,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x31,0x02,0x00,0x00, +0xb9,0x02,0x00,0x00,0x6c,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xd4,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xd6,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x36,0x02,0x00,0x00, +0x55,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x37,0x02,0x00,0x00,0x96,0x00,0x00,0x00, +0x36,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x3c,0x02,0x00,0x00,0x59,0x00,0x00,0x00,0xb9,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3d,0x02,0x00,0x00, +0xa8,0x00,0x00,0x00,0x3c,0x02,0x00,0x00,0x41,0x00,0x05,0x00, +0x15,0x00,0x00,0x00,0x42,0x02,0x00,0x00,0x12,0x00,0x00,0x00, +0x41,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x43,0x02,0x00,0x00,0x42,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x44,0x02,0x00,0x00,0x0f,0x00,0x00,0x00, +0x43,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x48,0x02,0x00,0x00,0x47,0x00,0x00,0x00,0x43,0x02,0x00,0x00, +0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00,0x4a,0x02,0x00,0x00, +0x49,0x02,0x00,0x00,0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x4b,0x02,0x00,0x00,0x4a,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x4c,0x02,0x00,0x00, +0x48,0x02,0x00,0x00,0x4b,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x4d,0x02,0x00,0x00,0x44,0x02,0x00,0x00, +0x4c,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x4f,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x4f,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xba,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, +0xd6,0x00,0x00,0x00,0xb5,0x02,0x00,0x00,0x52,0x02,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0x55,0x02,0x00,0x00, +0xba,0x02,0x00,0x00,0xbf,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0x51,0x02,0x00,0x00,0x52,0x02,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x55,0x02,0x00,0x00,0x50,0x02,0x00,0x00, +0x51,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x50,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0x57,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x57,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xbb,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0x50,0x02,0x00,0x00, +0xb3,0x02,0x00,0x00,0x5a,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc2,0x00,0x00,0x00,0x5d,0x02,0x00,0x00,0xbb,0x02,0x00,0x00, +0x60,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x59,0x02,0x00,0x00, +0x5a,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x5d,0x02,0x00,0x00,0x58,0x02,0x00,0x00,0x59,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x58,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x61,0x02,0x00,0x00,0xbb,0x02,0x00,0x00, +0x61,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x62,0x02,0x00,0x00,0x37,0x02,0x00,0x00,0x61,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x64,0x02,0x00,0x00, +0x64,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x65,0x02,0x00,0x00,0x62,0x02,0x00,0x00, +0x64,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x69,0x02,0x00,0x00,0xba,0x02,0x00,0x00,0xd7,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x6a,0x02,0x00,0x00, +0x3d,0x02,0x00,0x00,0x69,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x6c,0x02,0x00,0x00,0x68,0x00,0x00,0x00, +0xbc,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x6d,0x02,0x00,0x00,0x6a,0x02,0x00,0x00,0x6c,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0x6f,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x6f,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xbd,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0x58,0x02,0x00,0x00, +0xb1,0x02,0x00,0x00,0x72,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc2,0x00,0x00,0x00,0x75,0x02,0x00,0x00,0xbd,0x02,0x00,0x00, +0xbc,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x71,0x02,0x00,0x00, +0x72,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x75,0x02,0x00,0x00,0x70,0x02,0x00,0x00,0x71,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x70,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0x77,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x77,0x02,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xbf,0x02,0x00,0x00, +0x3e,0x00,0x00,0x00,0x70,0x02,0x00,0x00,0xaf,0x02,0x00,0x00, +0x7a,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, +0x7d,0x02,0x00,0x00,0xbf,0x02,0x00,0x00,0x62,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0x79,0x02,0x00,0x00,0x7a,0x02,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x7d,0x02,0x00,0x00, +0x78,0x02,0x00,0x00,0x79,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x78,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x80,0x02,0x00,0x00,0x65,0x02,0x00,0x00,0xbf,0x02,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0x83,0x02,0x00,0x00, +0x80,0x02,0x00,0x00,0x36,0x00,0x00,0x00,0xf7,0x00,0x03,0x00, +0x85,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x83,0x02,0x00,0x00,0x84,0x02,0x00,0x00,0x85,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x84,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x88,0x02,0x00,0x00,0x6d,0x02,0x00,0x00, +0xbd,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, +0x89,0x02,0x00,0x00,0x12,0x00,0x00,0x00,0xd0,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x8a,0x02,0x00,0x00, +0x89,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, +0x8b,0x02,0x00,0x00,0x88,0x02,0x00,0x00,0x8a,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0x85,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x85,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0xc2,0x00,0x00,0x00, +0x8c,0x02,0x00,0x00,0x83,0x02,0x00,0x00,0x78,0x02,0x00,0x00, +0x8b,0x02,0x00,0x00,0x84,0x02,0x00,0x00,0xf7,0x00,0x03,0x00, +0x8e,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x8c,0x02,0x00,0x00,0x8d,0x02,0x00,0x00,0x8e,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x8d,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x96,0x02,0x00,0x00,0x6d,0x02,0x00,0x00, +0xbd,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, +0x98,0x02,0x00,0x00,0x12,0x00,0x00,0x00,0x97,0x02,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x99,0x02,0x00,0x00, +0x98,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x9a,0x02,0x00,0x00,0x96,0x02,0x00,0x00,0x99,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9b,0x02,0x00,0x00, +0x4d,0x02,0x00,0x00,0x9a,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x9d,0x02,0x00,0x00,0x9b,0x02,0x00,0x00, +0x65,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x9f,0x02,0x00,0x00,0x9d,0x02,0x00,0x00,0xbf,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa1,0x02,0x00,0x00, +0xba,0x02,0x00,0x00,0xbc,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xa3,0x02,0x00,0x00,0xa1,0x02,0x00,0x00, +0xbd,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xa5,0x02,0x00,0x00,0xa3,0x02,0x00,0x00,0xa4,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa7,0x02,0x00,0x00, +0xbb,0x02,0x00,0x00,0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xa8,0x02,0x00,0x00,0xa5,0x02,0x00,0x00, +0xa7,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xaa,0x02,0x00,0x00,0xa8,0x02,0x00,0x00,0xbf,0x02,0x00,0x00, +0x41,0x00,0x05,0x00,0xcd,0x00,0x00,0x00,0xab,0x02,0x00,0x00, +0xca,0x00,0x00,0x00,0xaa,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, +0xc4,0x00,0x00,0x00,0xac,0x02,0x00,0x00,0xab,0x02,0x00,0x00, +0x41,0x00,0x06,0x00,0x64,0x01,0x00,0x00,0xad,0x02,0x00,0x00, +0x92,0x02,0x00,0x00,0x34,0x00,0x00,0x00,0x9f,0x02,0x00,0x00, +0x3e,0x00,0x03,0x00,0xad,0x02,0x00,0x00,0xac,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0x8e,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x8e,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x7a,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x7a,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xaf,0x02,0x00,0x00,0xbf,0x02,0x00,0x00, +0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x77,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x79,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0x72,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x72,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xb1,0x02,0x00,0x00, +0xbd,0x02,0x00,0x00,0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x6f,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x71,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0x5a,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x5a,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xb3,0x02,0x00,0x00,0xbb,0x02,0x00,0x00,0xd0,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x57,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x59,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x52,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x52,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xb5,0x02,0x00,0x00,0xba,0x02,0x00,0x00, +0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x4f,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x51,0x02,0x00,0x00,0xfd,0x00,0x01,0x00, +0x38,0x00,0x01,0x00, +}; +const uint64_t matmul_q4_0_f32_aligned_fp32_len = 10408; + +unsigned char matmul_q4_0_f32_fp32_data[] = { +0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, +0x15,0x03,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, +0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x51,0x11,0x00,0x00, +0x11,0x00,0x02,0x00,0x60,0x11,0x00,0x00,0x0b,0x00,0x06,0x00, +0x01,0x00,0x00,0x00,0x47,0x4c,0x53,0x4c,0x2e,0x73,0x74,0x64, +0x2e,0x34,0x35,0x30,0x00,0x00,0x00,0x00,0x0e,0x00,0x03,0x00, +0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x0f,0x00,0x0f,0x00, +0x05,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x6d,0x61,0x69,0x6e, +0x00,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0x3d,0x00,0x00,0x00,0x4c,0x00,0x00,0x00,0x07,0x01,0x00,0x00, +0x27,0x01,0x00,0x00,0x58,0x01,0x00,0x00,0x63,0x01,0x00,0x00, +0x49,0x02,0x00,0x00,0x92,0x02,0x00,0x00,0x10,0x00,0x06,0x00, +0x04,0x00,0x00,0x00,0x11,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x0b,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x1c,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x10,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x08,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x03,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x10,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x10,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, +0x05,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x14,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x10,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x1c,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, +0x08,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x09,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x24,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x10,0x00,0x00,0x00,0x0a,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x28,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, +0x0b,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x2c,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x30,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x10,0x00,0x00,0x00,0x0d,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x34,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x10,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x37,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x3d,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x1a,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x4c,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, +0x1b,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x4f,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x53,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x60,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x62,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x6c,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x03,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0xa6,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xb8,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0xbb,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x08,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x02,0x01,0x00,0x00,0x06,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x03,0x01,0x00,0x00, +0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x03,0x01,0x00,0x00,0x01,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x04,0x01,0x00,0x00,0x06,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0x48,0x00,0x04,0x00,0x05,0x01,0x00,0x00,0x00,0x00,0x00,0x00, +0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x05,0x01,0x00,0x00, +0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x03,0x00,0x05,0x01,0x00,0x00,0x02,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x07,0x01,0x00,0x00,0x22,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x07,0x01,0x00,0x00, +0x21,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x32,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x33,0x01,0x00,0x00,0x0b,0x00,0x00,0x00, +0x19,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x60,0x01,0x00,0x00, +0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00, +0x61,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x61,0x01,0x00,0x00,0x00,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, +0x61,0x01,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x63,0x01,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x63,0x01,0x00,0x00,0x21,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x49,0x02,0x00,0x00, +0x0b,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x8f,0x02,0x00,0x00,0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x48,0x00,0x04,0x00,0x90,0x02,0x00,0x00,0x00,0x00,0x00,0x00, +0x19,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x90,0x02,0x00,0x00, +0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x03,0x00,0x90,0x02,0x00,0x00,0x02,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x92,0x02,0x00,0x00,0x22,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x92,0x02,0x00,0x00, +0x21,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x13,0x00,0x02,0x00, +0x02,0x00,0x00,0x00,0x21,0x00,0x03,0x00,0x03,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x15,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x17,0x00,0x04,0x00, +0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x03,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x0a,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x09,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00, +0x0b,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x0d,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x1e,0x00,0x10,0x00,0x10,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x11,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x10,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0x11,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0x09,0x00,0x00,0x00,0x15,0x00,0x04,0x00,0x13,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x08,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x15,0x00,0x00,0x00,0x09,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, +0x21,0x00,0x00,0x00,0x0a,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0x27,0x00,0x00,0x00,0x09,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x2d,0x00,0x00,0x00, +0x07,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, +0x34,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x32,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x40,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00, +0x3d,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x3e,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00,0x4c,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x4f,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x32,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x54,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x53,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x58,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x53,0x00,0x00,0x00, +0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x60,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x61,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x53,0x00,0x00,0x00, +0x60,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x62,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x63,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0x61,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x67,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0x61,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x32,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x10,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x72,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x77,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x7c,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x80,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, +0x85,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0x90,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x96,0x00,0x00,0x00, +0x03,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, +0xa1,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x32,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xa6,0x00,0x00,0x00,0x40,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0xa8,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xb7,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00, +0x62,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xb8,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xb9,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0x53,0x00,0x00,0x00,0xb8,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xba,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0x4f,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x32,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xbb,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xbc,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0xba,0x00,0x00,0x00,0xbb,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xbd,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0xbc,0x00,0x00,0x00,0x60,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xbe,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0xb9,0x00,0x00,0x00,0xbd,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xbf,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0xb7,0x00,0x00,0x00,0xbe,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xc0,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0xbf,0x00,0x00,0x00,0xbb,0x00,0x00,0x00, +0x14,0x00,0x02,0x00,0xc1,0x00,0x00,0x00,0x16,0x00,0x03,0x00, +0xc3,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xc4,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xc5,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0xc4,0x00,0x00,0x00,0xbe,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xc6,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0xc5,0x00,0x00,0x00,0xbb,0x00,0x00,0x00,0x1c,0x00,0x04,0x00, +0xc7,0x00,0x00,0x00,0xc3,0x00,0x00,0x00,0xc6,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0xc8,0x00,0x00,0x00,0x07,0x00,0x00,0x00, +0xc7,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0xc3,0x00,0x00,0x00, +0xcb,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0xcc,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0xc3,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0xcf,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xf3,0x00,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, +0x39,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xf9,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xfd,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, +0x16,0x00,0x03,0x00,0x00,0x01,0x00,0x00,0x10,0x00,0x00,0x00, +0x15,0x00,0x04,0x00,0x01,0x01,0x00,0x00,0x08,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x1c,0x00,0x04,0x00,0x02,0x01,0x00,0x00, +0x01,0x01,0x00,0x00,0xf9,0x00,0x00,0x00,0x1e,0x00,0x04,0x00, +0x03,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x02,0x01,0x00,0x00, +0x1d,0x00,0x03,0x00,0x04,0x01,0x00,0x00,0x03,0x01,0x00,0x00, +0x1e,0x00,0x03,0x00,0x05,0x01,0x00,0x00,0x04,0x01,0x00,0x00, +0x20,0x00,0x04,0x00,0x06,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, +0x05,0x01,0x00,0x00,0x3b,0x00,0x04,0x00,0x06,0x01,0x00,0x00, +0x07,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x09,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,0x01,0x00,0x00, +0x20,0x00,0x04,0x00,0x10,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, +0x01,0x01,0x00,0x00,0x17,0x00,0x04,0x00,0x14,0x01,0x00,0x00, +0xc3,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0xc3,0x00,0x00,0x00,0x1e,0x01,0x00,0x00,0x00,0x00,0x00,0x41, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x23,0x01,0x00,0x00, +0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x24,0x01,0x00,0x00, +0x84,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x23,0x01,0x00,0x00, +0x1c,0x00,0x04,0x00,0x25,0x01,0x00,0x00,0xc3,0x00,0x00,0x00, +0x24,0x01,0x00,0x00,0x20,0x00,0x04,0x00,0x26,0x01,0x00,0x00, +0x04,0x00,0x00,0x00,0x25,0x01,0x00,0x00,0x3b,0x00,0x04,0x00, +0x26,0x01,0x00,0x00,0x27,0x01,0x00,0x00,0x04,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x2b,0x01,0x00,0x00,0x04,0x00,0x00,0x00, +0xc3,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x32,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x33,0x00,0x06,0x00, +0x09,0x00,0x00,0x00,0x33,0x01,0x00,0x00,0x32,0x01,0x00,0x00, +0x39,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x34,0x01,0x00,0x00,0x51,0x00,0x00,0x00, +0x33,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x35,0x01,0x00,0x00,0x84,0x00,0x00,0x00, +0x34,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x36,0x01,0x00,0x00,0x86,0x00,0x00,0x00, +0x35,0x01,0x00,0x00,0x6c,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x54,0x01,0x00,0x00,0x80,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x55,0x01,0x00,0x00,0x84,0x00,0x00,0x00, +0xa6,0x00,0x00,0x00,0x54,0x01,0x00,0x00,0x1c,0x00,0x04,0x00, +0x56,0x01,0x00,0x00,0xc3,0x00,0x00,0x00,0x55,0x01,0x00,0x00, +0x20,0x00,0x04,0x00,0x57,0x01,0x00,0x00,0x04,0x00,0x00,0x00, +0x56,0x01,0x00,0x00,0x3b,0x00,0x04,0x00,0x57,0x01,0x00,0x00, +0x58,0x01,0x00,0x00,0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x5c,0x01,0x00,0x00,0x80,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, +0x60,0x01,0x00,0x00,0xc3,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, +0x61,0x01,0x00,0x00,0x60,0x01,0x00,0x00,0x20,0x00,0x04,0x00, +0x62,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x61,0x01,0x00,0x00, +0x3b,0x00,0x04,0x00,0x62,0x01,0x00,0x00,0x63,0x01,0x00,0x00, +0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x6e,0x01,0x00,0x00, +0x0c,0x00,0x00,0x00,0xc3,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x76,0x01,0x00,0x00,0x80,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x7b,0x01,0x00,0x00,0x51,0x00,0x00,0x00, +0x33,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x7c,0x01,0x00,0x00,0x84,0x00,0x00,0x00, +0x7b,0x01,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x7d,0x01,0x00,0x00,0x86,0x00,0x00,0x00, +0x7c,0x01,0x00,0x00,0x6c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x80,0x01,0x00,0x00,0x08,0x01,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x81,0x01,0x00,0x00, +0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x84,0x01,0x00,0x00, +0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x9f,0x01,0x00,0x00, +0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00, +0x1c,0x00,0x04,0x00,0xa0,0x01,0x00,0x00,0xc3,0x00,0x00,0x00, +0x9f,0x01,0x00,0x00,0x20,0x00,0x04,0x00,0xa1,0x01,0x00,0x00, +0x07,0x00,0x00,0x00,0xa0,0x01,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xb1,0x01,0x00,0x00,0x80,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xcc,0x01,0x00,0x00,0x84,0x00,0x00,0x00, +0xbe,0x00,0x00,0x00,0xbb,0x00,0x00,0x00,0x1c,0x00,0x04,0x00, +0xcd,0x01,0x00,0x00,0xc3,0x00,0x00,0x00,0xcc,0x01,0x00,0x00, +0x20,0x00,0x04,0x00,0xce,0x01,0x00,0x00,0x07,0x00,0x00,0x00, +0xcd,0x01,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xd7,0x01,0x00,0x00,0x86,0x00,0x00,0x00,0xb8,0x00,0x00,0x00, +0xbe,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xdf,0x01,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, +0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x0e,0x02,0x00,0x00,0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00, +0x62,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, +0x41,0x02,0x00,0x00,0x0d,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0x0a,0x00,0x00,0x00,0x49,0x02,0x00,0x00,0x01,0x00,0x00,0x00, +0x1d,0x00,0x03,0x00,0x8f,0x02,0x00,0x00,0xc3,0x00,0x00,0x00, +0x1e,0x00,0x03,0x00,0x90,0x02,0x00,0x00,0x8f,0x02,0x00,0x00, +0x20,0x00,0x04,0x00,0x91,0x02,0x00,0x00,0x0c,0x00,0x00,0x00, +0x90,0x02,0x00,0x00,0x3b,0x00,0x04,0x00,0x91,0x02,0x00,0x00, +0x92,0x02,0x00,0x00,0x0c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0x97,0x02,0x00,0x00,0x05,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xa4,0x02,0x00,0x00, +0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00, +0x2c,0x00,0x05,0x00,0x14,0x01,0x00,0x00,0x14,0x03,0x00,0x00, +0x1e,0x01,0x00,0x00,0x1e,0x01,0x00,0x00,0x36,0x00,0x05,0x00, +0x02,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x03,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x05,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0xc8,0x00,0x00,0x00,0xc9,0x00,0x00,0x00, +0x07,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0xa1,0x01,0x00,0x00, +0xa2,0x01,0x00,0x00,0x07,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0xce,0x01,0x00,0x00,0xcf,0x01,0x00,0x00,0x07,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00,0x0e,0x00,0x00,0x00, +0x0b,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x0e,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x16,0x00,0x00,0x00, +0x12,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x17,0x00,0x00,0x00,0x16,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x18,0x00,0x00,0x00, +0x0f,0x00,0x00,0x00,0x17,0x00,0x00,0x00,0x89,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x1e,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, +0x17,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, +0x22,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x21,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x22,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x24,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x28,0x00,0x00,0x00, +0x12,0x00,0x00,0x00,0x27,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x29,0x00,0x00,0x00,0x28,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x2a,0x00,0x00,0x00, +0x1e,0x00,0x00,0x00,0x29,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x15,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0x2d,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x2f,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x30,0x00,0x00,0x00,0x24,0x00,0x00,0x00, +0x2f,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x32,0x00,0x00,0x00,0x30,0x00,0x00,0x00,0x2a,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x35,0x00,0x00,0x00, +0x12,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x36,0x00,0x00,0x00,0x35,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x38,0x00,0x00,0x00, +0x36,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x82,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x38,0x00,0x00,0x00, +0x39,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x3b,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x37,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00,0x3f,0x00,0x00,0x00, +0x3d,0x00,0x00,0x00,0x3e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x3f,0x00,0x00,0x00, +0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x42,0x00,0x00,0x00, +0x40,0x00,0x00,0x00,0x3b,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x47,0x00,0x00,0x00,0x40,0x00,0x00,0x00, +0x3b,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00, +0x49,0x00,0x00,0x00,0x3d,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x4a,0x00,0x00,0x00, +0x49,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00, +0x4d,0x00,0x00,0x00,0x4c,0x00,0x00,0x00,0x3e,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, +0x4d,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x50,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x4f,0x00,0x00,0x00, +0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x55,0x00,0x00,0x00, +0x50,0x00,0x00,0x00,0x54,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x59,0x00,0x00,0x00,0x50,0x00,0x00,0x00, +0x58,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x5d,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x4f,0x00,0x00,0x00, +0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x64,0x00,0x00,0x00, +0x5d,0x00,0x00,0x00,0x63,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x68,0x00,0x00,0x00,0x5d,0x00,0x00,0x00, +0x67,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x6e,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x73,0x00,0x00,0x00, +0x4e,0x00,0x00,0x00,0x72,0x00,0x00,0x00,0x89,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x78,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, +0x77,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x7d,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x7c,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x81,0x00,0x00,0x00, +0x12,0x00,0x00,0x00,0x80,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x82,0x00,0x00,0x00,0x81,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x83,0x00,0x00,0x00, +0x47,0x00,0x00,0x00,0x82,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x15,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0x85,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x87,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x89,0x00,0x00,0x00,0x47,0x00,0x00,0x00, +0x39,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x8c,0x00,0x00,0x00,0x89,0x00,0x00,0x00,0x82,0x00,0x00,0x00, +0x0c,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x8d,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x26,0x00,0x00,0x00,0x87,0x00,0x00,0x00, +0x8c,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, +0x91,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x90,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x92,0x00,0x00,0x00, +0x91,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x93,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x92,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x95,0x00,0x00,0x00, +0x42,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x15,0x00,0x00,0x00,0x97,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0x96,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x98,0x00,0x00,0x00,0x97,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x99,0x00,0x00,0x00,0x95,0x00,0x00,0x00, +0x98,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x9a,0x00,0x00,0x00,0x93,0x00,0x00,0x00,0x99,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9c,0x00,0x00,0x00, +0x9a,0x00,0x00,0x00,0x83,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x9d,0x00,0x00,0x00,0x9c,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, +0xa2,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0xa1,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xa3,0x00,0x00,0x00, +0xa2,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xa4,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0xa3,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa7,0x00,0x00,0x00, +0x4a,0x00,0x00,0x00,0xa6,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x15,0x00,0x00,0x00,0xa9,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0xa8,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xaa,0x00,0x00,0x00,0xa9,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xab,0x00,0x00,0x00,0xa7,0x00,0x00,0x00, +0xaa,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xac,0x00,0x00,0x00,0xa4,0x00,0x00,0x00,0xab,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xae,0x00,0x00,0x00, +0xac,0x00,0x00,0x00,0x83,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xaf,0x00,0x00,0x00,0xae,0x00,0x00,0x00, +0x39,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xb1,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xb1,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xb8,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, +0x05,0x00,0x00,0x00,0xd0,0x00,0x00,0x00,0xb2,0x00,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00,0xc2,0x00,0x00,0x00, +0xb8,0x02,0x00,0x00,0xc0,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0xb3,0x00,0x00,0x00,0xb2,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0xc2,0x00,0x00,0x00,0xb2,0x00,0x00,0x00, +0xb3,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xb2,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0xcc,0x00,0x00,0x00,0xcd,0x00,0x00,0x00, +0xc9,0x00,0x00,0x00,0xb8,0x02,0x00,0x00,0x3e,0x00,0x03,0x00, +0xcd,0x00,0x00,0x00,0xcb,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xd0,0x00,0x00,0x00,0xb8,0x02,0x00,0x00, +0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xb1,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xb3,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xd3,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xd3,0x00,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xd1,0x02,0x00,0x00, +0xaf,0x00,0x00,0x00,0xb3,0x00,0x00,0x00,0x86,0x01,0x00,0x00, +0xd6,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xcd,0x02,0x00,0x00,0x9d,0x00,0x00,0x00,0xb3,0x00,0x00,0x00, +0x83,0x01,0x00,0x00,0xd6,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xb9,0x02,0x00,0x00,0x83,0x00,0x00,0x00, +0xb3,0x00,0x00,0x00,0x31,0x02,0x00,0x00,0xd6,0x00,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00,0xda,0x00,0x00,0x00, +0xb9,0x02,0x00,0x00,0x8d,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0xd5,0x00,0x00,0x00,0xd6,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0xda,0x00,0x00,0x00,0xd4,0x00,0x00,0x00, +0xd5,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xd4,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xdc,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xdc,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xc9,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0xd4,0x00,0x00,0x00, +0x38,0x01,0x00,0x00,0xdd,0x00,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc1,0x00,0x00,0x00,0xe2,0x00,0x00,0x00,0xc9,0x02,0x00,0x00, +0x37,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xde,0x00,0x00,0x00, +0xdd,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0xe2,0x00,0x00,0x00,0xdd,0x00,0x00,0x00,0xde,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xdd,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xe7,0x00,0x00,0x00,0x73,0x00,0x00,0x00, +0xc9,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xea,0x00,0x00,0x00,0xe7,0x00,0x00,0x00,0x98,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xeb,0x00,0x00,0x00, +0xea,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xec,0x00,0x00,0x00,0xcd,0x02,0x00,0x00, +0xeb,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xee,0x00,0x00,0x00,0xec,0x00,0x00,0x00,0x6e,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xf4,0x00,0x00,0x00, +0xe7,0x00,0x00,0x00,0xf3,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xf6,0x00,0x00,0x00,0xf4,0x00,0x00,0x00, +0x6e,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xfa,0x00,0x00,0x00,0xee,0x00,0x00,0x00,0xf9,0x00,0x00,0x00, +0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xfe,0x00,0x00,0x00, +0xee,0x00,0x00,0x00,0xfd,0x00,0x00,0x00,0x41,0x00,0x07,0x00, +0x09,0x01,0x00,0x00,0x0a,0x01,0x00,0x00,0x07,0x01,0x00,0x00, +0x34,0x00,0x00,0x00,0xfa,0x00,0x00,0x00,0x34,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x00,0x01,0x00,0x00,0x0b,0x01,0x00,0x00, +0x0a,0x01,0x00,0x00,0x73,0x00,0x04,0x00,0xc3,0x00,0x00,0x00, +0x0c,0x01,0x00,0x00,0x0b,0x01,0x00,0x00,0x41,0x00,0x08,0x00, +0x10,0x01,0x00,0x00,0x11,0x01,0x00,0x00,0x07,0x01,0x00,0x00, +0x34,0x00,0x00,0x00,0xfa,0x00,0x00,0x00,0xcf,0x00,0x00,0x00, +0xfe,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x01,0x01,0x00,0x00, +0x12,0x01,0x00,0x00,0x11,0x01,0x00,0x00,0x71,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x13,0x01,0x00,0x00,0x12,0x01,0x00,0x00, +0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x18,0x01,0x00,0x00, +0x13,0x01,0x00,0x00,0xfd,0x00,0x00,0x00,0x70,0x00,0x04,0x00, +0xc3,0x00,0x00,0x00,0x19,0x01,0x00,0x00,0x18,0x01,0x00,0x00, +0xc2,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x1b,0x01,0x00,0x00, +0x13,0x01,0x00,0x00,0xa8,0x00,0x00,0x00,0x70,0x00,0x04,0x00, +0xc3,0x00,0x00,0x00,0x1c,0x01,0x00,0x00,0x1b,0x01,0x00,0x00, +0x50,0x00,0x05,0x00,0x14,0x01,0x00,0x00,0x1d,0x01,0x00,0x00, +0x19,0x01,0x00,0x00,0x1c,0x01,0x00,0x00,0x83,0x00,0x05,0x00, +0x14,0x01,0x00,0x00,0x20,0x01,0x00,0x00,0x1d,0x01,0x00,0x00, +0x14,0x03,0x00,0x00,0x8e,0x00,0x05,0x00,0x14,0x01,0x00,0x00, +0x22,0x01,0x00,0x00,0x20,0x01,0x00,0x00,0x0c,0x01,0x00,0x00, +0x51,0x00,0x05,0x00,0xc3,0x00,0x00,0x00,0x2a,0x01,0x00,0x00, +0x22,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x2b,0x01,0x00,0x00,0x2c,0x01,0x00,0x00,0x27,0x01,0x00,0x00, +0xf6,0x00,0x00,0x00,0x3e,0x00,0x03,0x00,0x2c,0x01,0x00,0x00, +0x2a,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x2e,0x01,0x00,0x00,0xf6,0x00,0x00,0x00,0xf9,0x00,0x00,0x00, +0x51,0x00,0x05,0x00,0xc3,0x00,0x00,0x00,0x30,0x01,0x00,0x00, +0x22,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x2b,0x01,0x00,0x00,0x31,0x01,0x00,0x00,0x27,0x01,0x00,0x00, +0x2e,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x31,0x01,0x00,0x00, +0x30,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x38,0x01,0x00,0x00,0xc9,0x02,0x00,0x00,0x36,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0xdc,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xde,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x3a,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x3a,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xca,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, +0xde,0x00,0x00,0x00,0x7f,0x01,0x00,0x00,0x3d,0x01,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00,0x40,0x01,0x00,0x00, +0xca,0x02,0x00,0x00,0xa6,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0x3c,0x01,0x00,0x00,0x3d,0x01,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x40,0x01,0x00,0x00,0x3b,0x01,0x00,0x00, +0x3c,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x3b,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x44,0x01,0x00,0x00, +0xa7,0x00,0x00,0x00,0x7d,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x46,0x01,0x00,0x00,0x44,0x01,0x00,0x00, +0xca,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, +0x47,0x01,0x00,0x00,0x12,0x00,0x00,0x00,0xcf,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x48,0x01,0x00,0x00, +0x47,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00, +0x49,0x01,0x00,0x00,0x46,0x01,0x00,0x00,0x48,0x01,0x00,0x00, +0xf7,0x00,0x03,0x00,0x4b,0x01,0x00,0x00,0x00,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x49,0x01,0x00,0x00,0x4a,0x01,0x00,0x00, +0x4b,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x4a,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x4e,0x01,0x00,0x00, +0xb9,0x02,0x00,0x00,0x78,0x00,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc1,0x00,0x00,0x00,0x50,0x01,0x00,0x00,0x4e,0x01,0x00,0x00, +0x8d,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x4b,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x4b,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, +0xc1,0x00,0x00,0x00,0x51,0x01,0x00,0x00,0x49,0x01,0x00,0x00, +0x3b,0x01,0x00,0x00,0x50,0x01,0x00,0x00,0x4a,0x01,0x00,0x00, +0xf7,0x00,0x03,0x00,0x53,0x01,0x00,0x00,0x00,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x51,0x01,0x00,0x00,0x52,0x01,0x00,0x00, +0x72,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x52,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x5b,0x01,0x00,0x00, +0x7d,0x00,0x00,0x00,0xca,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x5d,0x01,0x00,0x00,0x5b,0x01,0x00,0x00, +0x5c,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x5f,0x01,0x00,0x00,0x5d,0x01,0x00,0x00,0x78,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x6a,0x01,0x00,0x00, +0x5b,0x01,0x00,0x00,0xaa,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x6b,0x01,0x00,0x00,0xd1,0x02,0x00,0x00, +0x6a,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x6d,0x01,0x00,0x00,0x6b,0x01,0x00,0x00,0x78,0x00,0x00,0x00, +0x41,0x00,0x06,0x00,0x6e,0x01,0x00,0x00,0x6f,0x01,0x00,0x00, +0x63,0x01,0x00,0x00,0x34,0x00,0x00,0x00,0x6d,0x01,0x00,0x00, +0x3d,0x00,0x04,0x00,0xc3,0x00,0x00,0x00,0x70,0x01,0x00,0x00, +0x6f,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0x2b,0x01,0x00,0x00, +0x71,0x01,0x00,0x00,0x58,0x01,0x00,0x00,0x5f,0x01,0x00,0x00, +0x3e,0x00,0x03,0x00,0x71,0x01,0x00,0x00,0x70,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0x53,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x72,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x75,0x01,0x00,0x00,0x7d,0x00,0x00,0x00,0xca,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x77,0x01,0x00,0x00, +0x75,0x01,0x00,0x00,0x76,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x79,0x01,0x00,0x00,0x77,0x01,0x00,0x00, +0x78,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x2b,0x01,0x00,0x00, +0x7a,0x01,0x00,0x00,0x58,0x01,0x00,0x00,0x79,0x01,0x00,0x00, +0x3e,0x00,0x03,0x00,0x7a,0x01,0x00,0x00,0xcb,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x53,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x53,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x3d,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x3d,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x7f,0x01,0x00,0x00,0xca,0x02,0x00,0x00, +0x7d,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x3a,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x3c,0x01,0x00,0x00,0xe0,0x00,0x04,0x00, +0x0c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x80,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x83,0x01,0x00,0x00, +0xcd,0x02,0x00,0x00,0x81,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x86,0x01,0x00,0x00,0xd1,0x02,0x00,0x00, +0x84,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x88,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x88,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xd3,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, +0x3c,0x01,0x00,0x00,0x2f,0x02,0x00,0x00,0x8b,0x01,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00,0x8e,0x01,0x00,0x00, +0xd3,0x02,0x00,0x00,0x6c,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0x8a,0x01,0x00,0x00,0x8b,0x01,0x00,0x00,0x00,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x8e,0x01,0x00,0x00,0x89,0x01,0x00,0x00, +0x8a,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x89,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0x90,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x90,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xd7,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0x89,0x01,0x00,0x00, +0xbb,0x01,0x00,0x00,0x93,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc1,0x00,0x00,0x00,0x96,0x01,0x00,0x00,0xd7,0x02,0x00,0x00, +0x60,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x92,0x01,0x00,0x00, +0x93,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x96,0x01,0x00,0x00,0x91,0x01,0x00,0x00,0x92,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x91,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0x98,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x98,0x01,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xe9,0x02,0x00,0x00, +0x3e,0x00,0x00,0x00,0x91,0x01,0x00,0x00,0xb9,0x01,0x00,0x00, +0x99,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00, +0x9e,0x01,0x00,0x00,0xe9,0x02,0x00,0x00,0x62,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0x9a,0x01,0x00,0x00,0x99,0x01,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x9e,0x01,0x00,0x00, +0x99,0x01,0x00,0x00,0x9a,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x99,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xa4,0x01,0x00,0x00,0xd7,0x02,0x00,0x00,0x62,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa6,0x01,0x00,0x00, +0xa4,0x01,0x00,0x00,0xe9,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xa8,0x01,0x00,0x00,0x55,0x00,0x00,0x00, +0x53,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xaa,0x01,0x00,0x00,0xd7,0x02,0x00,0x00,0x61,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xab,0x01,0x00,0x00, +0xa8,0x01,0x00,0x00,0xaa,0x01,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xad,0x01,0x00,0x00,0x64,0x00,0x00,0x00, +0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xae,0x01,0x00,0x00,0xab,0x01,0x00,0x00,0xad,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xb0,0x01,0x00,0x00, +0xae,0x01,0x00,0x00,0xe9,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xb2,0x01,0x00,0x00,0xb0,0x01,0x00,0x00, +0xb1,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xb4,0x01,0x00,0x00,0xb2,0x01,0x00,0x00,0xd3,0x02,0x00,0x00, +0x41,0x00,0x05,0x00,0x2b,0x01,0x00,0x00,0xb5,0x01,0x00,0x00, +0x27,0x01,0x00,0x00,0xb4,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, +0xc3,0x00,0x00,0x00,0xb6,0x01,0x00,0x00,0xb5,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0xcc,0x00,0x00,0x00,0xb7,0x01,0x00,0x00, +0xa2,0x01,0x00,0x00,0xa6,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, +0xb7,0x01,0x00,0x00,0xb6,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xb9,0x01,0x00,0x00,0xe9,0x02,0x00,0x00, +0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x98,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x9a,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0x93,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x93,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xbb,0x01,0x00,0x00, +0xd7,0x02,0x00,0x00,0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x90,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x92,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0xbd,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xbd,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xd8,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0x92,0x01,0x00,0x00, +0xe9,0x01,0x00,0x00,0xc0,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc1,0x00,0x00,0x00,0xc3,0x01,0x00,0x00,0xd8,0x02,0x00,0x00, +0xbe,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xbf,0x01,0x00,0x00, +0xc0,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0xc3,0x01,0x00,0x00,0xbe,0x01,0x00,0x00,0xbf,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xbe,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0xc5,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xc5,0x01,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xe6,0x02,0x00,0x00, +0x3e,0x00,0x00,0x00,0xbe,0x01,0x00,0x00,0xe7,0x01,0x00,0x00, +0xc6,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00, +0xcb,0x01,0x00,0x00,0xe6,0x02,0x00,0x00,0xbb,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0xc7,0x01,0x00,0x00,0xc6,0x01,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xcb,0x01,0x00,0x00, +0xc6,0x01,0x00,0x00,0xc7,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xc6,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xd1,0x01,0x00,0x00,0xd8,0x02,0x00,0x00,0xbb,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xd3,0x01,0x00,0x00, +0xd1,0x01,0x00,0x00,0xe6,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xd5,0x01,0x00,0x00,0x59,0x00,0x00,0x00, +0xb8,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xd8,0x01,0x00,0x00,0xd8,0x02,0x00,0x00,0xd7,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xd9,0x01,0x00,0x00, +0xd5,0x01,0x00,0x00,0xd8,0x01,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xdb,0x01,0x00,0x00,0x68,0x00,0x00,0x00, +0xbb,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xdc,0x01,0x00,0x00,0xd9,0x01,0x00,0x00,0xdb,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xde,0x01,0x00,0x00, +0xdc,0x01,0x00,0x00,0xe6,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xe0,0x01,0x00,0x00,0xde,0x01,0x00,0x00, +0xdf,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xe2,0x01,0x00,0x00,0xe0,0x01,0x00,0x00,0xd3,0x02,0x00,0x00, +0x41,0x00,0x05,0x00,0x2b,0x01,0x00,0x00,0xe3,0x01,0x00,0x00, +0x58,0x01,0x00,0x00,0xe2,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, +0xc3,0x00,0x00,0x00,0xe4,0x01,0x00,0x00,0xe3,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0xcc,0x00,0x00,0x00,0xe5,0x01,0x00,0x00, +0xcf,0x01,0x00,0x00,0xd3,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, +0xe5,0x01,0x00,0x00,0xe4,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xe7,0x01,0x00,0x00,0xe6,0x02,0x00,0x00, +0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xc5,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xc7,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0xc0,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xc0,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xe9,0x01,0x00,0x00, +0xd8,0x02,0x00,0x00,0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xbd,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xbf,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0xeb,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xeb,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xd9,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0xbf,0x01,0x00,0x00, +0x2d,0x02,0x00,0x00,0xee,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc1,0x00,0x00,0x00,0xf1,0x01,0x00,0x00,0xd9,0x02,0x00,0x00, +0xbe,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xed,0x01,0x00,0x00, +0xee,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0xf1,0x01,0x00,0x00,0xec,0x01,0x00,0x00,0xed,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xec,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0xf3,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xf3,0x01,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xdd,0x02,0x00,0x00, +0x3e,0x00,0x00,0x00,0xec,0x01,0x00,0x00,0x2b,0x02,0x00,0x00, +0xf6,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00, +0xf9,0x01,0x00,0x00,0xdd,0x02,0x00,0x00,0x60,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0xf5,0x01,0x00,0x00,0xf6,0x01,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xf9,0x01,0x00,0x00, +0xf4,0x01,0x00,0x00,0xf5,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xf4,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xfb,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xfb,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xdf,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, +0xf4,0x01,0x00,0x00,0x29,0x02,0x00,0x00,0xfe,0x01,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00,0x01,0x02,0x00,0x00, +0xdf,0x02,0x00,0x00,0xbb,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0xfd,0x01,0x00,0x00,0xfe,0x01,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x01,0x02,0x00,0x00,0xfc,0x01,0x00,0x00, +0xfd,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xfc,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0x03,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x03,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xe1,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0xfc,0x01,0x00,0x00, +0x27,0x02,0x00,0x00,0x04,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc1,0x00,0x00,0x00,0x09,0x02,0x00,0x00,0xe1,0x02,0x00,0x00, +0x62,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x05,0x02,0x00,0x00, +0x04,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x09,0x02,0x00,0x00,0x04,0x02,0x00,0x00,0x05,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x04,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x0b,0x02,0x00,0x00,0xd9,0x02,0x00,0x00, +0xbb,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x0d,0x02,0x00,0x00,0x0b,0x02,0x00,0x00,0xdf,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x0f,0x02,0x00,0x00, +0x0d,0x02,0x00,0x00,0x0e,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x11,0x02,0x00,0x00,0xdd,0x02,0x00,0x00, +0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x12,0x02,0x00,0x00,0x0f,0x02,0x00,0x00,0x11,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x14,0x02,0x00,0x00, +0x12,0x02,0x00,0x00,0xe1,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x18,0x02,0x00,0x00,0x11,0x02,0x00,0x00, +0xe1,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0xcc,0x00,0x00,0x00, +0x19,0x02,0x00,0x00,0xa2,0x01,0x00,0x00,0x18,0x02,0x00,0x00, +0x3d,0x00,0x04,0x00,0xc3,0x00,0x00,0x00,0x1a,0x02,0x00,0x00, +0x19,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0xcc,0x00,0x00,0x00, +0x1f,0x02,0x00,0x00,0xcf,0x01,0x00,0x00,0x0d,0x02,0x00,0x00, +0x3d,0x00,0x04,0x00,0xc3,0x00,0x00,0x00,0x20,0x02,0x00,0x00, +0x1f,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0xcc,0x00,0x00,0x00, +0x22,0x02,0x00,0x00,0xc9,0x00,0x00,0x00,0x14,0x02,0x00,0x00, +0x3d,0x00,0x04,0x00,0xc3,0x00,0x00,0x00,0x23,0x02,0x00,0x00, +0x22,0x02,0x00,0x00,0x0c,0x00,0x08,0x00,0xc3,0x00,0x00,0x00, +0x24,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00, +0x1a,0x02,0x00,0x00,0x20,0x02,0x00,0x00,0x23,0x02,0x00,0x00, +0x3e,0x00,0x03,0x00,0x22,0x02,0x00,0x00,0x24,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x27,0x02,0x00,0x00, +0xe1,0x02,0x00,0x00,0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x03,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x05,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0xfe,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xfe,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x29,0x02,0x00,0x00,0xdf,0x02,0x00,0x00,0xcf,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xfb,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xfd,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xf6,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xf6,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x2b,0x02,0x00,0x00,0xdd,0x02,0x00,0x00, +0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xf3,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xf5,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0xee,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xee,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x2d,0x02,0x00,0x00, +0xd9,0x02,0x00,0x00,0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xeb,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xed,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0x8b,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x8b,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x2f,0x02,0x00,0x00,0xd3,0x02,0x00,0x00,0xcf,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x88,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x8a,0x01,0x00,0x00,0xe0,0x00,0x04,0x00,0x0c,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x80,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0xd6,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xd6,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x31,0x02,0x00,0x00, +0xb9,0x02,0x00,0x00,0x6c,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xd3,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xd5,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x36,0x02,0x00,0x00, +0x55,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x37,0x02,0x00,0x00,0x95,0x00,0x00,0x00, +0x36,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x3c,0x02,0x00,0x00,0x59,0x00,0x00,0x00,0xb8,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3d,0x02,0x00,0x00, +0xa7,0x00,0x00,0x00,0x3c,0x02,0x00,0x00,0x41,0x00,0x05,0x00, +0x15,0x00,0x00,0x00,0x42,0x02,0x00,0x00,0x12,0x00,0x00,0x00, +0x41,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x43,0x02,0x00,0x00,0x42,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x44,0x02,0x00,0x00,0x0f,0x00,0x00,0x00, +0x43,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x48,0x02,0x00,0x00,0x47,0x00,0x00,0x00,0x43,0x02,0x00,0x00, +0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00,0x4a,0x02,0x00,0x00, +0x49,0x02,0x00,0x00,0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x4b,0x02,0x00,0x00,0x4a,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x4c,0x02,0x00,0x00, +0x48,0x02,0x00,0x00,0x4b,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x4d,0x02,0x00,0x00,0x44,0x02,0x00,0x00, +0x4c,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x4f,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x4f,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xba,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, +0xd5,0x00,0x00,0x00,0xb5,0x02,0x00,0x00,0x52,0x02,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00,0x55,0x02,0x00,0x00, +0xba,0x02,0x00,0x00,0xbe,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0x51,0x02,0x00,0x00,0x52,0x02,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x55,0x02,0x00,0x00,0x50,0x02,0x00,0x00, +0x51,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x50,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0x57,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x57,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xbb,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0x50,0x02,0x00,0x00, +0xb3,0x02,0x00,0x00,0x5a,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc1,0x00,0x00,0x00,0x5d,0x02,0x00,0x00,0xbb,0x02,0x00,0x00, +0x60,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x59,0x02,0x00,0x00, +0x5a,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x5d,0x02,0x00,0x00,0x58,0x02,0x00,0x00,0x59,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x58,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x61,0x02,0x00,0x00,0xbb,0x02,0x00,0x00, +0x61,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x62,0x02,0x00,0x00,0x37,0x02,0x00,0x00,0x61,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x64,0x02,0x00,0x00, +0x64,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x65,0x02,0x00,0x00,0x62,0x02,0x00,0x00, +0x64,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x69,0x02,0x00,0x00,0xba,0x02,0x00,0x00,0xd7,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x6a,0x02,0x00,0x00, +0x3d,0x02,0x00,0x00,0x69,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x6c,0x02,0x00,0x00,0x68,0x00,0x00,0x00, +0xbb,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x6d,0x02,0x00,0x00,0x6a,0x02,0x00,0x00,0x6c,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0x6f,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x6f,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xbd,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0x58,0x02,0x00,0x00, +0xb1,0x02,0x00,0x00,0x72,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc1,0x00,0x00,0x00,0x75,0x02,0x00,0x00,0xbd,0x02,0x00,0x00, +0xbb,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x71,0x02,0x00,0x00, +0x72,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x75,0x02,0x00,0x00,0x70,0x02,0x00,0x00,0x71,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x70,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0x77,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x77,0x02,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xbf,0x02,0x00,0x00, +0x3e,0x00,0x00,0x00,0x70,0x02,0x00,0x00,0xaf,0x02,0x00,0x00, +0x7a,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00, +0x7d,0x02,0x00,0x00,0xbf,0x02,0x00,0x00,0x62,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0x79,0x02,0x00,0x00,0x7a,0x02,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x7d,0x02,0x00,0x00, +0x78,0x02,0x00,0x00,0x79,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x78,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x80,0x02,0x00,0x00,0x65,0x02,0x00,0x00,0xbf,0x02,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00,0x83,0x02,0x00,0x00, +0x80,0x02,0x00,0x00,0x36,0x00,0x00,0x00,0xf7,0x00,0x03,0x00, +0x85,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x83,0x02,0x00,0x00,0x84,0x02,0x00,0x00,0x85,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x84,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x88,0x02,0x00,0x00,0x6d,0x02,0x00,0x00, +0xbd,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, +0x89,0x02,0x00,0x00,0x12,0x00,0x00,0x00,0xcf,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x8a,0x02,0x00,0x00, +0x89,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00, +0x8b,0x02,0x00,0x00,0x88,0x02,0x00,0x00,0x8a,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0x85,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x85,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0xc1,0x00,0x00,0x00, +0x8c,0x02,0x00,0x00,0x83,0x02,0x00,0x00,0x78,0x02,0x00,0x00, +0x8b,0x02,0x00,0x00,0x84,0x02,0x00,0x00,0xf7,0x00,0x03,0x00, +0x8e,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x8c,0x02,0x00,0x00,0x8d,0x02,0x00,0x00,0x8e,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x8d,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x96,0x02,0x00,0x00,0x6d,0x02,0x00,0x00, +0xbd,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, +0x98,0x02,0x00,0x00,0x12,0x00,0x00,0x00,0x97,0x02,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x99,0x02,0x00,0x00, +0x98,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x9a,0x02,0x00,0x00,0x96,0x02,0x00,0x00,0x99,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9b,0x02,0x00,0x00, +0x4d,0x02,0x00,0x00,0x9a,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x9d,0x02,0x00,0x00,0x9b,0x02,0x00,0x00, +0x65,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x9f,0x02,0x00,0x00,0x9d,0x02,0x00,0x00,0xbf,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa1,0x02,0x00,0x00, +0xba,0x02,0x00,0x00,0xbb,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xa3,0x02,0x00,0x00,0xa1,0x02,0x00,0x00, +0xbd,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xa5,0x02,0x00,0x00,0xa3,0x02,0x00,0x00,0xa4,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa7,0x02,0x00,0x00, +0xbb,0x02,0x00,0x00,0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xa8,0x02,0x00,0x00,0xa5,0x02,0x00,0x00, +0xa7,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xaa,0x02,0x00,0x00,0xa8,0x02,0x00,0x00,0xbf,0x02,0x00,0x00, +0x41,0x00,0x05,0x00,0xcc,0x00,0x00,0x00,0xab,0x02,0x00,0x00, +0xc9,0x00,0x00,0x00,0xaa,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, +0xc3,0x00,0x00,0x00,0xac,0x02,0x00,0x00,0xab,0x02,0x00,0x00, +0x41,0x00,0x06,0x00,0x6e,0x01,0x00,0x00,0xad,0x02,0x00,0x00, +0x92,0x02,0x00,0x00,0x34,0x00,0x00,0x00,0x9f,0x02,0x00,0x00, +0x3e,0x00,0x03,0x00,0xad,0x02,0x00,0x00,0xac,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0x8e,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x8e,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x7a,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x7a,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xaf,0x02,0x00,0x00,0xbf,0x02,0x00,0x00, +0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x77,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x79,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0x72,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x72,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xb1,0x02,0x00,0x00, +0xbd,0x02,0x00,0x00,0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x6f,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x71,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0x5a,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x5a,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xb3,0x02,0x00,0x00,0xbb,0x02,0x00,0x00,0xcf,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x57,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x59,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x52,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x52,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xb5,0x02,0x00,0x00,0xba,0x02,0x00,0x00, +0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x4f,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x51,0x02,0x00,0x00,0xfd,0x00,0x01,0x00, +0x38,0x00,0x01,0x00, +}; +const uint64_t matmul_q4_0_f32_fp32_len = 10444; + +unsigned char matmul_q4_1_f32_data[] = { +0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, +0xf6,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, +0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x09,0x00,0x00,0x00, +0x11,0x00,0x02,0x00,0x51,0x11,0x00,0x00,0x11,0x00,0x02,0x00, +0x60,0x11,0x00,0x00,0x0b,0x00,0x06,0x00,0x01,0x00,0x00,0x00, +0x47,0x4c,0x53,0x4c,0x2e,0x73,0x74,0x64,0x2e,0x34,0x35,0x30, +0x00,0x00,0x00,0x00,0x0e,0x00,0x03,0x00,0x00,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x0f,0x00,0x0f,0x00,0x05,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x6d,0x61,0x69,0x6e,0x00,0x00,0x00,0x00, +0x0b,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x3d,0x00,0x00,0x00, +0x4c,0x00,0x00,0x00,0x07,0x01,0x00,0x00,0x2c,0x01,0x00,0x00, +0x5f,0x01,0x00,0x00,0x6a,0x01,0x00,0x00,0x55,0x02,0x00,0x00, +0x9e,0x02,0x00,0x00,0x10,0x00,0x06,0x00,0x04,0x00,0x00,0x00, +0x11,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x0b,0x00,0x00,0x00, +0x0b,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x10,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x10,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x10,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x05,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x10,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, +0x07,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x1c,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x08,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x10,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x24,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, +0x0a,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x28,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x2c,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x10,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x30,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, +0x0d,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x34,0x00,0x00,0x00, +0x47,0x00,0x03,0x00,0x10,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x37,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x3d,0x00,0x00,0x00, +0x0b,0x00,0x00,0x00,0x1a,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x4c,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x1b,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x4f,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x09,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x53,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x60,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x62,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x07,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x6c,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0xa6,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0xb8,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x05,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xbb,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x02,0x01,0x00,0x00,0x06,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x03,0x01,0x00,0x00,0x00,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x03,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x03,0x01,0x00,0x00, +0x02,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x04,0x01,0x00,0x00,0x06,0x00,0x00,0x00, +0x14,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0x05,0x01,0x00,0x00, +0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x05,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x05,0x01,0x00,0x00, +0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x07,0x01,0x00,0x00, +0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x07,0x01,0x00,0x00,0x21,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x39,0x01,0x00,0x00,0x01,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x3a,0x01,0x00,0x00, +0x0b,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x67,0x01,0x00,0x00,0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x48,0x00,0x04,0x00,0x68,0x01,0x00,0x00,0x00,0x00,0x00,0x00, +0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x68,0x01,0x00,0x00, +0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x03,0x00,0x68,0x01,0x00,0x00,0x02,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x6a,0x01,0x00,0x00,0x22,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x6a,0x01,0x00,0x00, +0x21,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x55,0x02,0x00,0x00,0x0b,0x00,0x00,0x00,0x18,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x9b,0x02,0x00,0x00,0x06,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0x9c,0x02,0x00,0x00, +0x00,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x9c,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x9c,0x02,0x00,0x00, +0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x9e,0x02,0x00,0x00, +0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x9e,0x02,0x00,0x00,0x21,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x13,0x00,0x02,0x00,0x02,0x00,0x00,0x00,0x21,0x00,0x03,0x00, +0x03,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x15,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x17,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x03,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x0a,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0x0a,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x0d,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x1e,0x00,0x10,0x00, +0x10,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x11,0x00,0x00,0x00,0x09,0x00,0x00,0x00, +0x10,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x11,0x00,0x00,0x00, +0x12,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x15,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x14,0x00,0x00,0x00, +0x08,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x15,0x00,0x00,0x00, +0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x0a,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x27,0x00,0x00,0x00, +0x09,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, +0x2d,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x37,0x00,0x00,0x00, +0x40,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x39,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0x0a,0x00,0x00,0x00,0x3d,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x3e,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00, +0x4c,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x53,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x54,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x37,0x00,0x00,0x00, +0x53,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x58,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x37,0x00,0x00,0x00, +0x53,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x60,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x61,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0x53,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x32,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x63,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0x61,0x00,0x00,0x00,0x62,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x67,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0x61,0x00,0x00,0x00,0x62,0x00,0x00,0x00, +0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, +0x10,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x6d,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x72,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x77,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, +0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x7c,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, +0x39,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, +0x80,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0x85,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x90,0x00,0x00,0x00, +0x0b,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, +0x96,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0xa1,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xa6,0x00,0x00,0x00, +0x40,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, +0xa8,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xb7,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x32,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xb8,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xb9,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0xb8,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xba,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x62,0x00,0x00,0x00, +0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xbb,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xbc,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0xba,0x00,0x00,0x00, +0xbb,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xbd,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0xbc,0x00,0x00,0x00, +0x60,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xbe,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0xb9,0x00,0x00,0x00, +0xbd,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xbf,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0xb7,0x00,0x00,0x00, +0xbe,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xc0,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0xbf,0x00,0x00,0x00, +0xbb,0x00,0x00,0x00,0x14,0x00,0x02,0x00,0xc1,0x00,0x00,0x00, +0x16,0x00,0x03,0x00,0xc3,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xc4,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xc5,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0xc4,0x00,0x00,0x00,0xbe,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xc6,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0xc5,0x00,0x00,0x00,0xbb,0x00,0x00,0x00, +0x1c,0x00,0x04,0x00,0xc7,0x00,0x00,0x00,0xc3,0x00,0x00,0x00, +0xc6,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0xc8,0x00,0x00,0x00, +0x07,0x00,0x00,0x00,0xc7,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0xc3,0x00,0x00,0x00,0xcb,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0xcc,0x00,0x00,0x00,0x07,0x00,0x00,0x00, +0xc3,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, +0xcf,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xf3,0x00,0x00,0x00,0x80,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xf9,0x00,0x00,0x00,0x10,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xfd,0x00,0x00,0x00, +0x0f,0x00,0x00,0x00,0x16,0x00,0x03,0x00,0x00,0x01,0x00,0x00, +0x10,0x00,0x00,0x00,0x15,0x00,0x04,0x00,0x01,0x01,0x00,0x00, +0x08,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x1c,0x00,0x04,0x00, +0x02,0x01,0x00,0x00,0x01,0x01,0x00,0x00,0xf9,0x00,0x00,0x00, +0x1e,0x00,0x05,0x00,0x03,0x01,0x00,0x00,0x00,0x01,0x00,0x00, +0x00,0x01,0x00,0x00,0x02,0x01,0x00,0x00,0x1d,0x00,0x03,0x00, +0x04,0x01,0x00,0x00,0x03,0x01,0x00,0x00,0x1e,0x00,0x03,0x00, +0x05,0x01,0x00,0x00,0x04,0x01,0x00,0x00,0x20,0x00,0x04,0x00, +0x06,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x05,0x01,0x00,0x00, +0x3b,0x00,0x04,0x00,0x06,0x01,0x00,0x00,0x07,0x01,0x00,0x00, +0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x09,0x01,0x00,0x00, +0x0c,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x20,0x00,0x04,0x00, +0x15,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x01,0x01,0x00,0x00, +0x17,0x00,0x04,0x00,0x19,0x01,0x00,0x00,0xc3,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x28,0x01,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, +0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x29,0x01,0x00,0x00,0x84,0x00,0x00,0x00,0x37,0x00,0x00,0x00, +0x28,0x01,0x00,0x00,0x1c,0x00,0x04,0x00,0x2a,0x01,0x00,0x00, +0x00,0x01,0x00,0x00,0x29,0x01,0x00,0x00,0x20,0x00,0x04,0x00, +0x2b,0x01,0x00,0x00,0x04,0x00,0x00,0x00,0x2a,0x01,0x00,0x00, +0x3b,0x00,0x04,0x00,0x2b,0x01,0x00,0x00,0x2c,0x01,0x00,0x00, +0x04,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x31,0x01,0x00,0x00, +0x04,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x32,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x39,0x01,0x00,0x00,0x01,0x00,0x00,0x00, +0x33,0x00,0x06,0x00,0x09,0x00,0x00,0x00,0x3a,0x01,0x00,0x00, +0x39,0x01,0x00,0x00,0x39,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x3b,0x01,0x00,0x00, +0x51,0x00,0x00,0x00,0x3a,0x01,0x00,0x00,0x00,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x3c,0x01,0x00,0x00, +0x84,0x00,0x00,0x00,0x3b,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x3d,0x01,0x00,0x00, +0x86,0x00,0x00,0x00,0x3c,0x01,0x00,0x00,0x6c,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x5b,0x01,0x00,0x00, +0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x5c,0x01,0x00,0x00, +0x84,0x00,0x00,0x00,0xa6,0x00,0x00,0x00,0x5b,0x01,0x00,0x00, +0x1c,0x00,0x04,0x00,0x5d,0x01,0x00,0x00,0x00,0x01,0x00,0x00, +0x5c,0x01,0x00,0x00,0x20,0x00,0x04,0x00,0x5e,0x01,0x00,0x00, +0x04,0x00,0x00,0x00,0x5d,0x01,0x00,0x00,0x3b,0x00,0x04,0x00, +0x5e,0x01,0x00,0x00,0x5f,0x01,0x00,0x00,0x04,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x63,0x01,0x00,0x00, +0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0x1d,0x00,0x03,0x00,0x67,0x01,0x00,0x00,0xc3,0x00,0x00,0x00, +0x1e,0x00,0x03,0x00,0x68,0x01,0x00,0x00,0x67,0x01,0x00,0x00, +0x20,0x00,0x04,0x00,0x69,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, +0x68,0x01,0x00,0x00,0x3b,0x00,0x04,0x00,0x69,0x01,0x00,0x00, +0x6a,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x75,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0xc3,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x7e,0x01,0x00,0x00, +0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x00,0x01,0x00,0x00,0x82,0x01,0x00,0x00, +0x00,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x84,0x01,0x00,0x00,0x51,0x00,0x00,0x00,0x3a,0x01,0x00,0x00, +0x00,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x85,0x01,0x00,0x00,0x84,0x00,0x00,0x00,0x84,0x01,0x00,0x00, +0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x86,0x01,0x00,0x00,0x86,0x00,0x00,0x00,0x85,0x01,0x00,0x00, +0x6c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x89,0x01,0x00,0x00,0x08,0x01,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x8a,0x01,0x00,0x00,0x86,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x8d,0x01,0x00,0x00,0x86,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xa8,0x01,0x00,0x00,0x84,0x00,0x00,0x00, +0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x1c,0x00,0x04,0x00, +0xa9,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0xa8,0x01,0x00,0x00, +0x20,0x00,0x04,0x00,0xaa,0x01,0x00,0x00,0x07,0x00,0x00,0x00, +0xa9,0x01,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xba,0x01,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, +0x39,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0xc0,0x01,0x00,0x00, +0x07,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xd6,0x01,0x00,0x00,0x84,0x00,0x00,0x00, +0xbe,0x00,0x00,0x00,0xbb,0x00,0x00,0x00,0x1c,0x00,0x04,0x00, +0xd7,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0xd6,0x01,0x00,0x00, +0x20,0x00,0x04,0x00,0xd8,0x01,0x00,0x00,0x07,0x00,0x00,0x00, +0xd7,0x01,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xe1,0x01,0x00,0x00,0x86,0x00,0x00,0x00,0xb8,0x00,0x00,0x00, +0xbe,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xe9,0x01,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, +0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x18,0x02,0x00,0x00,0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00, +0x62,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, +0x4d,0x02,0x00,0x00,0x0d,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0x0a,0x00,0x00,0x00,0x55,0x02,0x00,0x00,0x01,0x00,0x00,0x00, +0x1d,0x00,0x03,0x00,0x9b,0x02,0x00,0x00,0xc3,0x00,0x00,0x00, +0x1e,0x00,0x03,0x00,0x9c,0x02,0x00,0x00,0x9b,0x02,0x00,0x00, +0x20,0x00,0x04,0x00,0x9d,0x02,0x00,0x00,0x0c,0x00,0x00,0x00, +0x9c,0x02,0x00,0x00,0x3b,0x00,0x04,0x00,0x9d,0x02,0x00,0x00, +0x9e,0x02,0x00,0x00,0x0c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0xa3,0x02,0x00,0x00,0x05,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xb0,0x02,0x00,0x00, +0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00, +0x36,0x00,0x05,0x00,0x02,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0x05,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0xc8,0x00,0x00,0x00, +0xc9,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0xaa,0x01,0x00,0x00,0xab,0x01,0x00,0x00,0x07,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0xd8,0x01,0x00,0x00,0xd9,0x01,0x00,0x00, +0x07,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00, +0x0e,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, +0x0e,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, +0x16,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x14,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x17,0x00,0x00,0x00, +0x16,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x18,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x17,0x00,0x00,0x00, +0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x1e,0x00,0x00,0x00, +0x0f,0x00,0x00,0x00,0x17,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x15,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0x21,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x24,0x00,0x00,0x00,0x18,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, +0x28,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x27,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x29,0x00,0x00,0x00, +0x28,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x2a,0x00,0x00,0x00,0x1e,0x00,0x00,0x00,0x29,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, +0x12,0x00,0x00,0x00,0x2d,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x2f,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x30,0x00,0x00,0x00, +0x24,0x00,0x00,0x00,0x2f,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x30,0x00,0x00,0x00, +0x2a,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, +0x35,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x34,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x36,0x00,0x00,0x00, +0x35,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x38,0x00,0x00,0x00,0x36,0x00,0x00,0x00,0x37,0x00,0x00,0x00, +0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, +0x38,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x3b,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, +0x37,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00, +0x3f,0x00,0x00,0x00,0x3d,0x00,0x00,0x00,0x3e,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x40,0x00,0x00,0x00, +0x3f,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x42,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x3b,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x47,0x00,0x00,0x00, +0x40,0x00,0x00,0x00,0x3b,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x0d,0x00,0x00,0x00,0x49,0x00,0x00,0x00,0x3d,0x00,0x00,0x00, +0x39,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x4a,0x00,0x00,0x00,0x49,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x0d,0x00,0x00,0x00,0x4d,0x00,0x00,0x00,0x4c,0x00,0x00,0x00, +0x3e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x4e,0x00,0x00,0x00,0x4d,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x50,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, +0x4f,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x55,0x00,0x00,0x00,0x50,0x00,0x00,0x00,0x54,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x59,0x00,0x00,0x00, +0x50,0x00,0x00,0x00,0x58,0x00,0x00,0x00,0x89,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x5d,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, +0x4f,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x64,0x00,0x00,0x00,0x5d,0x00,0x00,0x00,0x63,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x68,0x00,0x00,0x00, +0x5d,0x00,0x00,0x00,0x67,0x00,0x00,0x00,0x89,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x6e,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, +0x6d,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x73,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x72,0x00,0x00,0x00, +0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x78,0x00,0x00,0x00, +0x4e,0x00,0x00,0x00,0x77,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x7d,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, +0x7c,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, +0x81,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x80,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x82,0x00,0x00,0x00, +0x81,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x83,0x00,0x00,0x00,0x47,0x00,0x00,0x00,0x82,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0x12,0x00,0x00,0x00,0x85,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x87,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x89,0x00,0x00,0x00, +0x47,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x8c,0x00,0x00,0x00,0x89,0x00,0x00,0x00, +0x82,0x00,0x00,0x00,0x0c,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0x8d,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x26,0x00,0x00,0x00, +0x87,0x00,0x00,0x00,0x8c,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x15,0x00,0x00,0x00,0x91,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0x90,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x92,0x00,0x00,0x00,0x91,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x93,0x00,0x00,0x00,0x32,0x00,0x00,0x00, +0x92,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x95,0x00,0x00,0x00,0x42,0x00,0x00,0x00,0x37,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x97,0x00,0x00,0x00, +0x12,0x00,0x00,0x00,0x96,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x98,0x00,0x00,0x00,0x97,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x99,0x00,0x00,0x00, +0x95,0x00,0x00,0x00,0x98,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x9a,0x00,0x00,0x00,0x93,0x00,0x00,0x00, +0x99,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x9c,0x00,0x00,0x00,0x9a,0x00,0x00,0x00,0x83,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9d,0x00,0x00,0x00, +0x9c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x15,0x00,0x00,0x00,0xa2,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0xa1,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xa3,0x00,0x00,0x00,0xa2,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xa4,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, +0xa3,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xa7,0x00,0x00,0x00,0x4a,0x00,0x00,0x00,0xa6,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0xa9,0x00,0x00,0x00, +0x12,0x00,0x00,0x00,0xa8,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xaa,0x00,0x00,0x00,0xa9,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xab,0x00,0x00,0x00, +0xa7,0x00,0x00,0x00,0xaa,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xac,0x00,0x00,0x00,0xa4,0x00,0x00,0x00, +0xab,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xae,0x00,0x00,0x00,0xac,0x00,0x00,0x00,0x83,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xaf,0x00,0x00,0x00, +0xae,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xb1,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xb1,0x00,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xc4,0x02,0x00,0x00, +0x3e,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0xd0,0x00,0x00,0x00, +0xb2,0x00,0x00,0x00,0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00, +0xc2,0x00,0x00,0x00,0xc4,0x02,0x00,0x00,0xc0,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0xb3,0x00,0x00,0x00,0xb2,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xc2,0x00,0x00,0x00, +0xb2,0x00,0x00,0x00,0xb3,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xb2,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0xcc,0x00,0x00,0x00, +0xcd,0x00,0x00,0x00,0xc9,0x00,0x00,0x00,0xc4,0x02,0x00,0x00, +0x3e,0x00,0x03,0x00,0xcd,0x00,0x00,0x00,0xcb,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xd0,0x00,0x00,0x00, +0xc4,0x02,0x00,0x00,0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xb1,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xb3,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xd3,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xd3,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xdd,0x02,0x00,0x00,0xaf,0x00,0x00,0x00,0xb3,0x00,0x00,0x00, +0x8f,0x01,0x00,0x00,0xd6,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xd9,0x02,0x00,0x00,0x9d,0x00,0x00,0x00, +0xb3,0x00,0x00,0x00,0x8c,0x01,0x00,0x00,0xd6,0x00,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xc5,0x02,0x00,0x00, +0x83,0x00,0x00,0x00,0xb3,0x00,0x00,0x00,0x3d,0x02,0x00,0x00, +0xd6,0x00,0x00,0x00,0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00, +0xda,0x00,0x00,0x00,0xc5,0x02,0x00,0x00,0x8d,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0xd5,0x00,0x00,0x00,0xd6,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xda,0x00,0x00,0x00, +0xd4,0x00,0x00,0x00,0xd5,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xd4,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xdc,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xdc,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xd5,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, +0xd4,0x00,0x00,0x00,0x3f,0x01,0x00,0x00,0xdd,0x00,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00,0xe2,0x00,0x00,0x00, +0xd5,0x02,0x00,0x00,0x37,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0xde,0x00,0x00,0x00,0xdd,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0xe2,0x00,0x00,0x00,0xdd,0x00,0x00,0x00, +0xde,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xdd,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xe7,0x00,0x00,0x00, +0x73,0x00,0x00,0x00,0xd5,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xea,0x00,0x00,0x00,0xe7,0x00,0x00,0x00, +0x98,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xeb,0x00,0x00,0x00,0xea,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xec,0x00,0x00,0x00, +0xd9,0x02,0x00,0x00,0xeb,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xee,0x00,0x00,0x00,0xec,0x00,0x00,0x00, +0x6e,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xf4,0x00,0x00,0x00,0xe7,0x00,0x00,0x00,0xf3,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xf6,0x00,0x00,0x00, +0xf4,0x00,0x00,0x00,0x6e,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xfa,0x00,0x00,0x00,0xee,0x00,0x00,0x00, +0xf9,0x00,0x00,0x00,0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xfe,0x00,0x00,0x00,0xee,0x00,0x00,0x00,0xfd,0x00,0x00,0x00, +0x41,0x00,0x07,0x00,0x09,0x01,0x00,0x00,0x0a,0x01,0x00,0x00, +0x07,0x01,0x00,0x00,0x34,0x00,0x00,0x00,0xfa,0x00,0x00,0x00, +0x34,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x00,0x01,0x00,0x00, +0x0b,0x01,0x00,0x00,0x0a,0x01,0x00,0x00,0x73,0x00,0x04,0x00, +0xc3,0x00,0x00,0x00,0x0c,0x01,0x00,0x00,0x0b,0x01,0x00,0x00, +0x41,0x00,0x07,0x00,0x09,0x01,0x00,0x00,0x0f,0x01,0x00,0x00, +0x07,0x01,0x00,0x00,0x34,0x00,0x00,0x00,0xfa,0x00,0x00,0x00, +0xcf,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x00,0x01,0x00,0x00, +0x10,0x01,0x00,0x00,0x0f,0x01,0x00,0x00,0x73,0x00,0x04,0x00, +0xc3,0x00,0x00,0x00,0x11,0x01,0x00,0x00,0x10,0x01,0x00,0x00, +0x41,0x00,0x08,0x00,0x15,0x01,0x00,0x00,0x16,0x01,0x00,0x00, +0x07,0x01,0x00,0x00,0x34,0x00,0x00,0x00,0xfa,0x00,0x00,0x00, +0x85,0x00,0x00,0x00,0xfe,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x01,0x01,0x00,0x00,0x17,0x01,0x00,0x00,0x16,0x01,0x00,0x00, +0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x18,0x01,0x00,0x00, +0x17,0x01,0x00,0x00,0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x1d,0x01,0x00,0x00,0x18,0x01,0x00,0x00,0xfd,0x00,0x00,0x00, +0x70,0x00,0x04,0x00,0xc3,0x00,0x00,0x00,0x1e,0x01,0x00,0x00, +0x1d,0x01,0x00,0x00,0xc2,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x20,0x01,0x00,0x00,0x18,0x01,0x00,0x00,0xa8,0x00,0x00,0x00, +0x70,0x00,0x04,0x00,0xc3,0x00,0x00,0x00,0x21,0x01,0x00,0x00, +0x20,0x01,0x00,0x00,0x50,0x00,0x05,0x00,0x19,0x01,0x00,0x00, +0x22,0x01,0x00,0x00,0x1e,0x01,0x00,0x00,0x21,0x01,0x00,0x00, +0x8e,0x00,0x05,0x00,0x19,0x01,0x00,0x00,0x24,0x01,0x00,0x00, +0x22,0x01,0x00,0x00,0x0c,0x01,0x00,0x00,0x50,0x00,0x05,0x00, +0x19,0x01,0x00,0x00,0x26,0x01,0x00,0x00,0x11,0x01,0x00,0x00, +0x11,0x01,0x00,0x00,0x81,0x00,0x05,0x00,0x19,0x01,0x00,0x00, +0x27,0x01,0x00,0x00,0x24,0x01,0x00,0x00,0x26,0x01,0x00,0x00, +0x51,0x00,0x05,0x00,0xc3,0x00,0x00,0x00,0x2f,0x01,0x00,0x00, +0x27,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x73,0x00,0x04,0x00, +0x00,0x01,0x00,0x00,0x30,0x01,0x00,0x00,0x2f,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0x31,0x01,0x00,0x00,0x32,0x01,0x00,0x00, +0x2c,0x01,0x00,0x00,0xf6,0x00,0x00,0x00,0x3e,0x00,0x03,0x00, +0x32,0x01,0x00,0x00,0x30,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x34,0x01,0x00,0x00,0xf6,0x00,0x00,0x00, +0xf9,0x00,0x00,0x00,0x51,0x00,0x05,0x00,0xc3,0x00,0x00,0x00, +0x36,0x01,0x00,0x00,0x27,0x01,0x00,0x00,0x01,0x00,0x00,0x00, +0x73,0x00,0x04,0x00,0x00,0x01,0x00,0x00,0x37,0x01,0x00,0x00, +0x36,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0x31,0x01,0x00,0x00, +0x38,0x01,0x00,0x00,0x2c,0x01,0x00,0x00,0x34,0x01,0x00,0x00, +0x3e,0x00,0x03,0x00,0x38,0x01,0x00,0x00,0x37,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3f,0x01,0x00,0x00, +0xd5,0x02,0x00,0x00,0x3d,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0xdc,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xde,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x41,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x41,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xd6,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0xde,0x00,0x00,0x00, +0x88,0x01,0x00,0x00,0x44,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc1,0x00,0x00,0x00,0x47,0x01,0x00,0x00,0xd6,0x02,0x00,0x00, +0xa6,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x43,0x01,0x00,0x00, +0x44,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x47,0x01,0x00,0x00,0x42,0x01,0x00,0x00,0x43,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x42,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x4b,0x01,0x00,0x00,0xa7,0x00,0x00,0x00, +0x7d,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x4d,0x01,0x00,0x00,0x4b,0x01,0x00,0x00,0xd6,0x02,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x4e,0x01,0x00,0x00, +0x12,0x00,0x00,0x00,0xcf,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x4f,0x01,0x00,0x00,0x4e,0x01,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00,0x50,0x01,0x00,0x00, +0x4d,0x01,0x00,0x00,0x4f,0x01,0x00,0x00,0xf7,0x00,0x03,0x00, +0x52,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x50,0x01,0x00,0x00,0x51,0x01,0x00,0x00,0x52,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x51,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x55,0x01,0x00,0x00,0xc5,0x02,0x00,0x00, +0x78,0x00,0x00,0x00,0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00, +0x57,0x01,0x00,0x00,0x55,0x01,0x00,0x00,0x8d,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x52,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x52,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0xc1,0x00,0x00,0x00, +0x58,0x01,0x00,0x00,0x50,0x01,0x00,0x00,0x42,0x01,0x00,0x00, +0x57,0x01,0x00,0x00,0x51,0x01,0x00,0x00,0xf7,0x00,0x03,0x00, +0x5a,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x58,0x01,0x00,0x00,0x59,0x01,0x00,0x00,0x7a,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x59,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x62,0x01,0x00,0x00,0x7d,0x00,0x00,0x00, +0xd6,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x64,0x01,0x00,0x00,0x62,0x01,0x00,0x00,0x63,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x66,0x01,0x00,0x00, +0x64,0x01,0x00,0x00,0x78,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x71,0x01,0x00,0x00,0x62,0x01,0x00,0x00, +0xaa,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x72,0x01,0x00,0x00,0xdd,0x02,0x00,0x00,0x71,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x74,0x01,0x00,0x00, +0x72,0x01,0x00,0x00,0x78,0x00,0x00,0x00,0x41,0x00,0x06,0x00, +0x75,0x01,0x00,0x00,0x76,0x01,0x00,0x00,0x6a,0x01,0x00,0x00, +0x34,0x00,0x00,0x00,0x74,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, +0xc3,0x00,0x00,0x00,0x77,0x01,0x00,0x00,0x76,0x01,0x00,0x00, +0x73,0x00,0x04,0x00,0x00,0x01,0x00,0x00,0x78,0x01,0x00,0x00, +0x77,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0x31,0x01,0x00,0x00, +0x79,0x01,0x00,0x00,0x5f,0x01,0x00,0x00,0x66,0x01,0x00,0x00, +0x3e,0x00,0x03,0x00,0x79,0x01,0x00,0x00,0x78,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0x5a,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x7a,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x7d,0x01,0x00,0x00,0x7d,0x00,0x00,0x00,0xd6,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x7f,0x01,0x00,0x00, +0x7d,0x01,0x00,0x00,0x7e,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x81,0x01,0x00,0x00,0x7f,0x01,0x00,0x00, +0x78,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x31,0x01,0x00,0x00, +0x83,0x01,0x00,0x00,0x5f,0x01,0x00,0x00,0x81,0x01,0x00,0x00, +0x3e,0x00,0x03,0x00,0x83,0x01,0x00,0x00,0x82,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0x5a,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x5a,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x44,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x44,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x88,0x01,0x00,0x00,0xd6,0x02,0x00,0x00, +0x86,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x41,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x43,0x01,0x00,0x00,0xe0,0x00,0x04,0x00, +0x0c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x89,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8c,0x01,0x00,0x00, +0xd9,0x02,0x00,0x00,0x8a,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x8f,0x01,0x00,0x00,0xdd,0x02,0x00,0x00, +0x8d,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x91,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x91,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xdf,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, +0x43,0x01,0x00,0x00,0x3b,0x02,0x00,0x00,0x94,0x01,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00,0x97,0x01,0x00,0x00, +0xdf,0x02,0x00,0x00,0x6c,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0x93,0x01,0x00,0x00,0x94,0x01,0x00,0x00,0x00,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x97,0x01,0x00,0x00,0x92,0x01,0x00,0x00, +0x93,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x92,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0x99,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x99,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xe3,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0x92,0x01,0x00,0x00, +0xc5,0x01,0x00,0x00,0x9c,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc1,0x00,0x00,0x00,0x9f,0x01,0x00,0x00,0xe3,0x02,0x00,0x00, +0x60,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x9b,0x01,0x00,0x00, +0x9c,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x9f,0x01,0x00,0x00,0x9a,0x01,0x00,0x00,0x9b,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x9a,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0xa1,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xa1,0x01,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xf5,0x02,0x00,0x00, +0x3e,0x00,0x00,0x00,0x9a,0x01,0x00,0x00,0xc3,0x01,0x00,0x00, +0xa2,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00, +0xa7,0x01,0x00,0x00,0xf5,0x02,0x00,0x00,0x62,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0xa3,0x01,0x00,0x00,0xa2,0x01,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xa7,0x01,0x00,0x00, +0xa2,0x01,0x00,0x00,0xa3,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xa2,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xad,0x01,0x00,0x00,0xe3,0x02,0x00,0x00,0x62,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xaf,0x01,0x00,0x00, +0xad,0x01,0x00,0x00,0xf5,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xb1,0x01,0x00,0x00,0x55,0x00,0x00,0x00, +0x53,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xb3,0x01,0x00,0x00,0xe3,0x02,0x00,0x00,0x61,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xb4,0x01,0x00,0x00, +0xb1,0x01,0x00,0x00,0xb3,0x01,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xb6,0x01,0x00,0x00,0x64,0x00,0x00,0x00, +0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xb7,0x01,0x00,0x00,0xb4,0x01,0x00,0x00,0xb6,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xb9,0x01,0x00,0x00, +0xb7,0x01,0x00,0x00,0xf5,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xbb,0x01,0x00,0x00,0xb9,0x01,0x00,0x00, +0xba,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xbd,0x01,0x00,0x00,0xbb,0x01,0x00,0x00,0xdf,0x02,0x00,0x00, +0x41,0x00,0x05,0x00,0x31,0x01,0x00,0x00,0xbe,0x01,0x00,0x00, +0x2c,0x01,0x00,0x00,0xbd,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, +0x00,0x01,0x00,0x00,0xbf,0x01,0x00,0x00,0xbe,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0xc0,0x01,0x00,0x00,0xc1,0x01,0x00,0x00, +0xab,0x01,0x00,0x00,0xaf,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, +0xc1,0x01,0x00,0x00,0xbf,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xc3,0x01,0x00,0x00,0xf5,0x02,0x00,0x00, +0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xa1,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xa3,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0x9c,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x9c,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xc5,0x01,0x00,0x00, +0xe3,0x02,0x00,0x00,0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x99,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x9b,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0xc7,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xc7,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xe4,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0x9b,0x01,0x00,0x00, +0xf3,0x01,0x00,0x00,0xca,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc1,0x00,0x00,0x00,0xcd,0x01,0x00,0x00,0xe4,0x02,0x00,0x00, +0xbe,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xc9,0x01,0x00,0x00, +0xca,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0xcd,0x01,0x00,0x00,0xc8,0x01,0x00,0x00,0xc9,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xc8,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0xcf,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xcf,0x01,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xf2,0x02,0x00,0x00, +0x3e,0x00,0x00,0x00,0xc8,0x01,0x00,0x00,0xf1,0x01,0x00,0x00, +0xd0,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00, +0xd5,0x01,0x00,0x00,0xf2,0x02,0x00,0x00,0xbb,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0xd1,0x01,0x00,0x00,0xd0,0x01,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xd5,0x01,0x00,0x00, +0xd0,0x01,0x00,0x00,0xd1,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xd0,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xdb,0x01,0x00,0x00,0xe4,0x02,0x00,0x00,0xbb,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xdd,0x01,0x00,0x00, +0xdb,0x01,0x00,0x00,0xf2,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xdf,0x01,0x00,0x00,0x59,0x00,0x00,0x00, +0xb8,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xe2,0x01,0x00,0x00,0xe4,0x02,0x00,0x00,0xe1,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xe3,0x01,0x00,0x00, +0xdf,0x01,0x00,0x00,0xe2,0x01,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xe5,0x01,0x00,0x00,0x68,0x00,0x00,0x00, +0xbb,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xe6,0x01,0x00,0x00,0xe3,0x01,0x00,0x00,0xe5,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xe8,0x01,0x00,0x00, +0xe6,0x01,0x00,0x00,0xf2,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xea,0x01,0x00,0x00,0xe8,0x01,0x00,0x00, +0xe9,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xec,0x01,0x00,0x00,0xea,0x01,0x00,0x00,0xdf,0x02,0x00,0x00, +0x41,0x00,0x05,0x00,0x31,0x01,0x00,0x00,0xed,0x01,0x00,0x00, +0x5f,0x01,0x00,0x00,0xec,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, +0x00,0x01,0x00,0x00,0xee,0x01,0x00,0x00,0xed,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0xc0,0x01,0x00,0x00,0xef,0x01,0x00,0x00, +0xd9,0x01,0x00,0x00,0xdd,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, +0xef,0x01,0x00,0x00,0xee,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xf1,0x01,0x00,0x00,0xf2,0x02,0x00,0x00, +0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xcf,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xd1,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0xca,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xca,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xf3,0x01,0x00,0x00, +0xe4,0x02,0x00,0x00,0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xc7,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xc9,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0xf5,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xf5,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xe5,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0xc9,0x01,0x00,0x00, +0x39,0x02,0x00,0x00,0xf8,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc1,0x00,0x00,0x00,0xfb,0x01,0x00,0x00,0xe5,0x02,0x00,0x00, +0xbe,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xf7,0x01,0x00,0x00, +0xf8,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0xfb,0x01,0x00,0x00,0xf6,0x01,0x00,0x00,0xf7,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xf6,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0xfd,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xfd,0x01,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xe9,0x02,0x00,0x00, +0x3e,0x00,0x00,0x00,0xf6,0x01,0x00,0x00,0x37,0x02,0x00,0x00, +0x00,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00, +0x03,0x02,0x00,0x00,0xe9,0x02,0x00,0x00,0x60,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0xff,0x01,0x00,0x00,0x00,0x02,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x03,0x02,0x00,0x00, +0xfe,0x01,0x00,0x00,0xff,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xfe,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x05,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x05,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xeb,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, +0xfe,0x01,0x00,0x00,0x35,0x02,0x00,0x00,0x08,0x02,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00,0x0b,0x02,0x00,0x00, +0xeb,0x02,0x00,0x00,0xbb,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0x07,0x02,0x00,0x00,0x08,0x02,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x0b,0x02,0x00,0x00,0x06,0x02,0x00,0x00, +0x07,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x06,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0x0d,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x0d,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xed,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0x06,0x02,0x00,0x00, +0x33,0x02,0x00,0x00,0x0e,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc1,0x00,0x00,0x00,0x13,0x02,0x00,0x00,0xed,0x02,0x00,0x00, +0x62,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x0f,0x02,0x00,0x00, +0x0e,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x13,0x02,0x00,0x00,0x0e,0x02,0x00,0x00,0x0f,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x0e,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x15,0x02,0x00,0x00,0xe5,0x02,0x00,0x00, +0xbb,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x17,0x02,0x00,0x00,0x15,0x02,0x00,0x00,0xeb,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x19,0x02,0x00,0x00, +0x17,0x02,0x00,0x00,0x18,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x1b,0x02,0x00,0x00,0xe9,0x02,0x00,0x00, +0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x1c,0x02,0x00,0x00,0x19,0x02,0x00,0x00,0x1b,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x1e,0x02,0x00,0x00, +0x1c,0x02,0x00,0x00,0xed,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x22,0x02,0x00,0x00,0x1b,0x02,0x00,0x00, +0xed,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0xc0,0x01,0x00,0x00, +0x23,0x02,0x00,0x00,0xab,0x01,0x00,0x00,0x22,0x02,0x00,0x00, +0x3d,0x00,0x04,0x00,0x00,0x01,0x00,0x00,0x24,0x02,0x00,0x00, +0x23,0x02,0x00,0x00,0x73,0x00,0x04,0x00,0xc3,0x00,0x00,0x00, +0x25,0x02,0x00,0x00,0x24,0x02,0x00,0x00,0x41,0x00,0x05,0x00, +0xc0,0x01,0x00,0x00,0x2a,0x02,0x00,0x00,0xd9,0x01,0x00,0x00, +0x17,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0x00,0x01,0x00,0x00, +0x2b,0x02,0x00,0x00,0x2a,0x02,0x00,0x00,0x73,0x00,0x04,0x00, +0xc3,0x00,0x00,0x00,0x2c,0x02,0x00,0x00,0x2b,0x02,0x00,0x00, +0x41,0x00,0x05,0x00,0xcc,0x00,0x00,0x00,0x2e,0x02,0x00,0x00, +0xc9,0x00,0x00,0x00,0x1e,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, +0xc3,0x00,0x00,0x00,0x2f,0x02,0x00,0x00,0x2e,0x02,0x00,0x00, +0x0c,0x00,0x08,0x00,0xc3,0x00,0x00,0x00,0x30,0x02,0x00,0x00, +0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x25,0x02,0x00,0x00, +0x2c,0x02,0x00,0x00,0x2f,0x02,0x00,0x00,0x3e,0x00,0x03,0x00, +0x2e,0x02,0x00,0x00,0x30,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x33,0x02,0x00,0x00,0xed,0x02,0x00,0x00, +0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x0d,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x0f,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0x08,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x08,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x35,0x02,0x00,0x00, +0xeb,0x02,0x00,0x00,0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x05,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x07,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0x00,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x00,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x37,0x02,0x00,0x00,0xe9,0x02,0x00,0x00,0xcf,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xfd,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xff,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xf8,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xf8,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x39,0x02,0x00,0x00,0xe5,0x02,0x00,0x00, +0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xf5,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xf7,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0x94,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x94,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3b,0x02,0x00,0x00, +0xdf,0x02,0x00,0x00,0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x91,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x93,0x01,0x00,0x00, +0xe0,0x00,0x04,0x00,0x0c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x89,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xd6,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xd6,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x3d,0x02,0x00,0x00,0xc5,0x02,0x00,0x00, +0x6c,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xd3,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xd5,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x42,0x02,0x00,0x00,0x55,0x00,0x00,0x00, +0x53,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x43,0x02,0x00,0x00,0x95,0x00,0x00,0x00,0x42,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x48,0x02,0x00,0x00, +0x59,0x00,0x00,0x00,0xb8,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x49,0x02,0x00,0x00,0xa7,0x00,0x00,0x00, +0x48,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, +0x4e,0x02,0x00,0x00,0x12,0x00,0x00,0x00,0x4d,0x02,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x4f,0x02,0x00,0x00, +0x4e,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x50,0x02,0x00,0x00,0x0f,0x00,0x00,0x00,0x4f,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x54,0x02,0x00,0x00, +0x47,0x00,0x00,0x00,0x4f,0x02,0x00,0x00,0x41,0x00,0x05,0x00, +0x0d,0x00,0x00,0x00,0x56,0x02,0x00,0x00,0x55,0x02,0x00,0x00, +0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x57,0x02,0x00,0x00,0x56,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x58,0x02,0x00,0x00,0x54,0x02,0x00,0x00, +0x57,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x59,0x02,0x00,0x00,0x50,0x02,0x00,0x00,0x58,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0x5b,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x5b,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xc6,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0xd5,0x00,0x00,0x00, +0xc1,0x02,0x00,0x00,0x5e,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc1,0x00,0x00,0x00,0x61,0x02,0x00,0x00,0xc6,0x02,0x00,0x00, +0xbe,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x5d,0x02,0x00,0x00, +0x5e,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x61,0x02,0x00,0x00,0x5c,0x02,0x00,0x00,0x5d,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x5c,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0x63,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x63,0x02,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xc7,0x02,0x00,0x00, +0x3e,0x00,0x00,0x00,0x5c,0x02,0x00,0x00,0xbf,0x02,0x00,0x00, +0x66,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00, +0x69,0x02,0x00,0x00,0xc7,0x02,0x00,0x00,0x60,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0x65,0x02,0x00,0x00,0x66,0x02,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x69,0x02,0x00,0x00, +0x64,0x02,0x00,0x00,0x65,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x64,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x6d,0x02,0x00,0x00,0xc7,0x02,0x00,0x00,0x61,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x6e,0x02,0x00,0x00, +0x43,0x02,0x00,0x00,0x6d,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x70,0x02,0x00,0x00,0x64,0x00,0x00,0x00, +0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x71,0x02,0x00,0x00,0x6e,0x02,0x00,0x00,0x70,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x75,0x02,0x00,0x00, +0xc6,0x02,0x00,0x00,0xe1,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x76,0x02,0x00,0x00,0x49,0x02,0x00,0x00, +0x75,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x78,0x02,0x00,0x00,0x68,0x00,0x00,0x00,0xbb,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x79,0x02,0x00,0x00, +0x76,0x02,0x00,0x00,0x78,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0x7b,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x7b,0x02,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xc9,0x02,0x00,0x00, +0x3e,0x00,0x00,0x00,0x64,0x02,0x00,0x00,0xbd,0x02,0x00,0x00, +0x7e,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00, +0x81,0x02,0x00,0x00,0xc9,0x02,0x00,0x00,0xbb,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0x7d,0x02,0x00,0x00,0x7e,0x02,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x81,0x02,0x00,0x00, +0x7c,0x02,0x00,0x00,0x7d,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x7c,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x83,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x83,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xcb,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, +0x7c,0x02,0x00,0x00,0xbb,0x02,0x00,0x00,0x86,0x02,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00,0x89,0x02,0x00,0x00, +0xcb,0x02,0x00,0x00,0x62,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0x85,0x02,0x00,0x00,0x86,0x02,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x89,0x02,0x00,0x00,0x84,0x02,0x00,0x00, +0x85,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x84,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8c,0x02,0x00,0x00, +0x71,0x02,0x00,0x00,0xcb,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc1,0x00,0x00,0x00,0x8f,0x02,0x00,0x00,0x8c,0x02,0x00,0x00, +0x36,0x00,0x00,0x00,0xf7,0x00,0x03,0x00,0x91,0x02,0x00,0x00, +0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x8f,0x02,0x00,0x00, +0x90,0x02,0x00,0x00,0x91,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x90,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x94,0x02,0x00,0x00,0x79,0x02,0x00,0x00,0xc9,0x02,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x95,0x02,0x00,0x00, +0x12,0x00,0x00,0x00,0xcf,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x96,0x02,0x00,0x00,0x95,0x02,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00,0x97,0x02,0x00,0x00, +0x94,0x02,0x00,0x00,0x96,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0x91,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x91,0x02,0x00,0x00, +0xf5,0x00,0x07,0x00,0xc1,0x00,0x00,0x00,0x98,0x02,0x00,0x00, +0x8f,0x02,0x00,0x00,0x84,0x02,0x00,0x00,0x97,0x02,0x00,0x00, +0x90,0x02,0x00,0x00,0xf7,0x00,0x03,0x00,0x9a,0x02,0x00,0x00, +0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x98,0x02,0x00,0x00, +0x99,0x02,0x00,0x00,0x9a,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x99,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xa2,0x02,0x00,0x00,0x79,0x02,0x00,0x00,0xc9,0x02,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0xa4,0x02,0x00,0x00, +0x12,0x00,0x00,0x00,0xa3,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xa5,0x02,0x00,0x00,0xa4,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa6,0x02,0x00,0x00, +0xa2,0x02,0x00,0x00,0xa5,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xa7,0x02,0x00,0x00,0x59,0x02,0x00,0x00, +0xa6,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xa9,0x02,0x00,0x00,0xa7,0x02,0x00,0x00,0x71,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xab,0x02,0x00,0x00, +0xa9,0x02,0x00,0x00,0xcb,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xad,0x02,0x00,0x00,0xc6,0x02,0x00,0x00, +0xbb,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xaf,0x02,0x00,0x00,0xad,0x02,0x00,0x00,0xc9,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xb1,0x02,0x00,0x00, +0xaf,0x02,0x00,0x00,0xb0,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xb3,0x02,0x00,0x00,0xc7,0x02,0x00,0x00, +0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xb4,0x02,0x00,0x00,0xb1,0x02,0x00,0x00,0xb3,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xb6,0x02,0x00,0x00, +0xb4,0x02,0x00,0x00,0xcb,0x02,0x00,0x00,0x41,0x00,0x05,0x00, +0xcc,0x00,0x00,0x00,0xb7,0x02,0x00,0x00,0xc9,0x00,0x00,0x00, +0xb6,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0xc3,0x00,0x00,0x00, +0xb8,0x02,0x00,0x00,0xb7,0x02,0x00,0x00,0x41,0x00,0x06,0x00, +0x75,0x01,0x00,0x00,0xb9,0x02,0x00,0x00,0x9e,0x02,0x00,0x00, +0x34,0x00,0x00,0x00,0xab,0x02,0x00,0x00,0x3e,0x00,0x03,0x00, +0xb9,0x02,0x00,0x00,0xb8,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0x9a,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x9a,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0x86,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x86,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xbb,0x02,0x00,0x00,0xcb,0x02,0x00,0x00,0xcf,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x83,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x85,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x7e,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x7e,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xbd,0x02,0x00,0x00,0xc9,0x02,0x00,0x00, +0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x7b,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x7d,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0x66,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x66,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xbf,0x02,0x00,0x00, +0xc7,0x02,0x00,0x00,0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x63,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x65,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0x5e,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x5e,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xc1,0x02,0x00,0x00,0xc6,0x02,0x00,0x00,0xcf,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x5b,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x5d,0x02,0x00,0x00,0xfd,0x00,0x01,0x00,0x38,0x00,0x01,0x00, + +}; +const uint64_t matmul_q4_1_f32_len = 10632; + +unsigned char matmul_q4_1_f32_aligned_data[] = { +0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, +0x19,0x03,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, +0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x09,0x00,0x00,0x00, +0x11,0x00,0x02,0x00,0x51,0x11,0x00,0x00,0x11,0x00,0x02,0x00, +0x60,0x11,0x00,0x00,0x0b,0x00,0x06,0x00,0x01,0x00,0x00,0x00, +0x47,0x4c,0x53,0x4c,0x2e,0x73,0x74,0x64,0x2e,0x34,0x35,0x30, +0x00,0x00,0x00,0x00,0x0e,0x00,0x03,0x00,0x00,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x0f,0x00,0x0f,0x00,0x05,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x6d,0x61,0x69,0x6e,0x00,0x00,0x00,0x00, +0x0b,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x3d,0x00,0x00,0x00, +0x4c,0x00,0x00,0x00,0x08,0x01,0x00,0x00,0x2d,0x01,0x00,0x00, +0x62,0x01,0x00,0x00,0x6a,0x01,0x00,0x00,0x78,0x02,0x00,0x00, +0xc1,0x02,0x00,0x00,0x10,0x00,0x06,0x00,0x04,0x00,0x00,0x00, +0x11,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x0b,0x00,0x00,0x00, +0x0b,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x10,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x10,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x10,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x05,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x10,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, +0x07,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x1c,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x08,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x10,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x24,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, +0x0a,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x28,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x2c,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x10,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x30,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, +0x0d,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x34,0x00,0x00,0x00, +0x47,0x00,0x03,0x00,0x10,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x37,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x3d,0x00,0x00,0x00, +0x0b,0x00,0x00,0x00,0x1a,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x4c,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x1b,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x4f,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x09,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x53,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x60,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x62,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x07,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x6c,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0xa7,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0xb9,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x05,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xbc,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x03,0x01,0x00,0x00,0x06,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x04,0x01,0x00,0x00,0x00,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x04,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x04,0x01,0x00,0x00, +0x02,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x05,0x01,0x00,0x00,0x06,0x00,0x00,0x00, +0x14,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0x06,0x01,0x00,0x00, +0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x06,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x06,0x01,0x00,0x00, +0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x08,0x01,0x00,0x00, +0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x08,0x01,0x00,0x00,0x21,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x3a,0x01,0x00,0x00,0x01,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x3b,0x01,0x00,0x00, +0x0b,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x67,0x01,0x00,0x00,0x06,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0x48,0x00,0x04,0x00,0x68,0x01,0x00,0x00,0x00,0x00,0x00,0x00, +0x05,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0x68,0x01,0x00,0x00, +0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x68,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x68,0x01,0x00,0x00, +0x00,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0x10,0x00,0x00,0x00, +0x47,0x00,0x03,0x00,0x68,0x01,0x00,0x00,0x02,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x6a,0x01,0x00,0x00,0x22,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x6a,0x01,0x00,0x00, +0x21,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x78,0x02,0x00,0x00,0x0b,0x00,0x00,0x00,0x18,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0xbe,0x02,0x00,0x00,0x06,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0xbf,0x02,0x00,0x00, +0x00,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0xbf,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0xbf,0x02,0x00,0x00, +0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xc1,0x02,0x00,0x00, +0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0xc1,0x02,0x00,0x00,0x21,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x13,0x00,0x02,0x00,0x02,0x00,0x00,0x00,0x21,0x00,0x03,0x00, +0x03,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x15,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x17,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x03,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x0a,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0x0a,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x0d,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x1e,0x00,0x10,0x00, +0x10,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x11,0x00,0x00,0x00,0x09,0x00,0x00,0x00, +0x10,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x11,0x00,0x00,0x00, +0x12,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x15,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x14,0x00,0x00,0x00, +0x08,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x15,0x00,0x00,0x00, +0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x0a,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x27,0x00,0x00,0x00, +0x09,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, +0x2d,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x37,0x00,0x00,0x00, +0x40,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x39,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0x0a,0x00,0x00,0x00,0x3d,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x3e,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00, +0x4c,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x53,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x54,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x37,0x00,0x00,0x00, +0x53,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x58,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x37,0x00,0x00,0x00, +0x53,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x60,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x61,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0x53,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x32,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x63,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0x61,0x00,0x00,0x00,0x62,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x67,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0x61,0x00,0x00,0x00,0x62,0x00,0x00,0x00, +0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, +0x10,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x6d,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x72,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x77,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x78,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x77,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x7d,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x77,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0x81,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, +0x91,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0x97,0x00,0x00,0x00,0x03,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0xa2,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xa7,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0xa9,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xb8,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00, +0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xb9,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xba,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x53,0x00,0x00,0x00, +0xb9,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xbb,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x4f,0x00,0x00,0x00, +0x62,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xbc,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xbd,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0xbb,0x00,0x00,0x00,0xbc,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xbe,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0xbd,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xbf,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0xba,0x00,0x00,0x00,0xbe,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xc0,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0xb8,0x00,0x00,0x00,0xbf,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xc1,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0xc0,0x00,0x00,0x00,0xbc,0x00,0x00,0x00,0x14,0x00,0x02,0x00, +0xc2,0x00,0x00,0x00,0x16,0x00,0x03,0x00,0xc4,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xc5,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00, +0x62,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xc6,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0xc5,0x00,0x00,0x00, +0xbf,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xc7,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0xc6,0x00,0x00,0x00, +0xbc,0x00,0x00,0x00,0x1c,0x00,0x04,0x00,0xc8,0x00,0x00,0x00, +0xc4,0x00,0x00,0x00,0xc7,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0xc9,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0xc8,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0xc4,0x00,0x00,0x00,0xcc,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0xcd,0x00,0x00,0x00, +0x07,0x00,0x00,0x00,0xc4,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0xd0,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xf4,0x00,0x00,0x00, +0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xfa,0x00,0x00,0x00, +0x10,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xfe,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x16,0x00,0x03,0x00, +0x01,0x01,0x00,0x00,0x10,0x00,0x00,0x00,0x15,0x00,0x04,0x00, +0x02,0x01,0x00,0x00,0x08,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x1c,0x00,0x04,0x00,0x03,0x01,0x00,0x00,0x02,0x01,0x00,0x00, +0xfa,0x00,0x00,0x00,0x1e,0x00,0x05,0x00,0x04,0x01,0x00,0x00, +0x01,0x01,0x00,0x00,0x01,0x01,0x00,0x00,0x03,0x01,0x00,0x00, +0x1d,0x00,0x03,0x00,0x05,0x01,0x00,0x00,0x04,0x01,0x00,0x00, +0x1e,0x00,0x03,0x00,0x06,0x01,0x00,0x00,0x05,0x01,0x00,0x00, +0x20,0x00,0x04,0x00,0x07,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, +0x06,0x01,0x00,0x00,0x3b,0x00,0x04,0x00,0x07,0x01,0x00,0x00, +0x08,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x0a,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x01,0x01,0x00,0x00, +0x20,0x00,0x04,0x00,0x16,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, +0x02,0x01,0x00,0x00,0x17,0x00,0x04,0x00,0x1a,0x01,0x00,0x00, +0xc4,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x29,0x01,0x00,0x00,0x80,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x2a,0x01,0x00,0x00,0x84,0x00,0x00,0x00, +0x37,0x00,0x00,0x00,0x29,0x01,0x00,0x00,0x1c,0x00,0x04,0x00, +0x2b,0x01,0x00,0x00,0x01,0x01,0x00,0x00,0x2a,0x01,0x00,0x00, +0x20,0x00,0x04,0x00,0x2c,0x01,0x00,0x00,0x04,0x00,0x00,0x00, +0x2b,0x01,0x00,0x00,0x3b,0x00,0x04,0x00,0x2c,0x01,0x00,0x00, +0x2d,0x01,0x00,0x00,0x04,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x32,0x01,0x00,0x00,0x04,0x00,0x00,0x00,0x01,0x01,0x00,0x00, +0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x3a,0x01,0x00,0x00, +0x01,0x00,0x00,0x00,0x33,0x00,0x06,0x00,0x09,0x00,0x00,0x00, +0x3b,0x01,0x00,0x00,0x3a,0x01,0x00,0x00,0x39,0x00,0x00,0x00, +0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x3c,0x01,0x00,0x00,0x51,0x00,0x00,0x00,0x3b,0x01,0x00,0x00, +0x00,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x3d,0x01,0x00,0x00,0x84,0x00,0x00,0x00,0x3c,0x01,0x00,0x00, +0x0c,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x3e,0x01,0x00,0x00,0x86,0x00,0x00,0x00,0x3d,0x01,0x00,0x00, +0x6c,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x59,0x01,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, +0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x5e,0x01,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, +0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x5f,0x01,0x00,0x00,0x84,0x00,0x00,0x00,0xa7,0x00,0x00,0x00, +0x5e,0x01,0x00,0x00,0x1c,0x00,0x04,0x00,0x60,0x01,0x00,0x00, +0x01,0x01,0x00,0x00,0x5f,0x01,0x00,0x00,0x20,0x00,0x04,0x00, +0x61,0x01,0x00,0x00,0x04,0x00,0x00,0x00,0x60,0x01,0x00,0x00, +0x3b,0x00,0x04,0x00,0x61,0x01,0x00,0x00,0x62,0x01,0x00,0x00, +0x04,0x00,0x00,0x00,0x17,0x00,0x04,0x00,0x65,0x01,0x00,0x00, +0xc4,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x18,0x00,0x04,0x00, +0x66,0x01,0x00,0x00,0x65,0x01,0x00,0x00,0x02,0x00,0x00,0x00, +0x1d,0x00,0x03,0x00,0x67,0x01,0x00,0x00,0x66,0x01,0x00,0x00, +0x1e,0x00,0x03,0x00,0x68,0x01,0x00,0x00,0x67,0x01,0x00,0x00, +0x20,0x00,0x04,0x00,0x69,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, +0x68,0x01,0x00,0x00,0x3b,0x00,0x04,0x00,0x69,0x01,0x00,0x00, +0x6a,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x6c,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0xc4,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x80,0x01,0x00,0x00, +0x03,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x88,0x01,0x00,0x00,0x04,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x90,0x01,0x00,0x00,0x05,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x98,0x01,0x00,0x00, +0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xa0,0x01,0x00,0x00,0x07,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xa7,0x01,0x00,0x00,0x51,0x00,0x00,0x00, +0x3b,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xa8,0x01,0x00,0x00,0x84,0x00,0x00,0x00, +0xa7,0x01,0x00,0x00,0x77,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xa9,0x01,0x00,0x00,0x86,0x00,0x00,0x00, +0xa8,0x01,0x00,0x00,0x6c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xac,0x01,0x00,0x00,0x08,0x01,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xad,0x01,0x00,0x00, +0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xb0,0x01,0x00,0x00, +0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x77,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xcb,0x01,0x00,0x00, +0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00, +0x1c,0x00,0x04,0x00,0xcc,0x01,0x00,0x00,0x01,0x01,0x00,0x00, +0xcb,0x01,0x00,0x00,0x20,0x00,0x04,0x00,0xcd,0x01,0x00,0x00, +0x07,0x00,0x00,0x00,0xcc,0x01,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xdd,0x01,0x00,0x00,0x80,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0xe3,0x01,0x00,0x00,0x07,0x00,0x00,0x00,0x01,0x01,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xf9,0x01,0x00,0x00, +0x84,0x00,0x00,0x00,0xbf,0x00,0x00,0x00,0xbc,0x00,0x00,0x00, +0x1c,0x00,0x04,0x00,0xfa,0x01,0x00,0x00,0x01,0x01,0x00,0x00, +0xf9,0x01,0x00,0x00,0x20,0x00,0x04,0x00,0xfb,0x01,0x00,0x00, +0x07,0x00,0x00,0x00,0xfa,0x01,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x04,0x02,0x00,0x00,0x86,0x00,0x00,0x00, +0xb9,0x00,0x00,0x00,0xbf,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x0c,0x02,0x00,0x00,0x80,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x3b,0x02,0x00,0x00,0x84,0x00,0x00,0x00, +0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0x70,0x02,0x00,0x00,0x0d,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00,0x78,0x02,0x00,0x00, +0x01,0x00,0x00,0x00,0x1d,0x00,0x03,0x00,0xbe,0x02,0x00,0x00, +0xc4,0x00,0x00,0x00,0x1e,0x00,0x03,0x00,0xbf,0x02,0x00,0x00, +0xbe,0x02,0x00,0x00,0x20,0x00,0x04,0x00,0xc0,0x02,0x00,0x00, +0x0c,0x00,0x00,0x00,0xbf,0x02,0x00,0x00,0x3b,0x00,0x04,0x00, +0xc0,0x02,0x00,0x00,0xc1,0x02,0x00,0x00,0x0c,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0xc6,0x02,0x00,0x00, +0x05,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xd3,0x02,0x00,0x00,0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00, +0x62,0x00,0x00,0x00,0x36,0x00,0x05,0x00,0x02,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x03,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0x05,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0xc9,0x00,0x00,0x00,0xca,0x00,0x00,0x00,0x07,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0xcd,0x01,0x00,0x00,0xce,0x01,0x00,0x00, +0x07,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0xfb,0x01,0x00,0x00, +0xfc,0x01,0x00,0x00,0x07,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x0d,0x00,0x00,0x00,0x0e,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x0f,0x00,0x00,0x00,0x0e,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x15,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0x14,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x17,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, +0x17,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x1e,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x17,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x22,0x00,0x00,0x00, +0x12,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x22,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x24,0x00,0x00,0x00, +0x18,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x15,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0x27,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x29,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x2a,0x00,0x00,0x00,0x1e,0x00,0x00,0x00, +0x29,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, +0x2e,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x2d,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x2f,0x00,0x00,0x00, +0x2e,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x30,0x00,0x00,0x00,0x24,0x00,0x00,0x00,0x2f,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x32,0x00,0x00,0x00, +0x30,0x00,0x00,0x00,0x2a,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x15,0x00,0x00,0x00,0x35,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0x34,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x36,0x00,0x00,0x00,0x35,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x38,0x00,0x00,0x00,0x36,0x00,0x00,0x00, +0x37,0x00,0x00,0x00,0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x3a,0x00,0x00,0x00,0x38,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3b,0x00,0x00,0x00, +0x3a,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x0d,0x00,0x00,0x00,0x3f,0x00,0x00,0x00,0x3d,0x00,0x00,0x00, +0x3e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x40,0x00,0x00,0x00,0x3f,0x00,0x00,0x00,0x89,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x42,0x00,0x00,0x00,0x40,0x00,0x00,0x00, +0x3b,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x47,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x3b,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00,0x49,0x00,0x00,0x00, +0x3d,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x4a,0x00,0x00,0x00,0x49,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00,0x4d,0x00,0x00,0x00, +0x4c,0x00,0x00,0x00,0x3e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x4d,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x50,0x00,0x00,0x00, +0x4e,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x89,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x55,0x00,0x00,0x00,0x50,0x00,0x00,0x00, +0x54,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x59,0x00,0x00,0x00,0x50,0x00,0x00,0x00,0x58,0x00,0x00,0x00, +0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x5d,0x00,0x00,0x00, +0x4e,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x89,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x64,0x00,0x00,0x00,0x5d,0x00,0x00,0x00, +0x63,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x68,0x00,0x00,0x00,0x5d,0x00,0x00,0x00,0x67,0x00,0x00,0x00, +0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x6e,0x00,0x00,0x00, +0x4e,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x73,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, +0x72,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x79,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x78,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x7e,0x00,0x00,0x00, +0x4e,0x00,0x00,0x00,0x7d,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x15,0x00,0x00,0x00,0x82,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0x81,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x83,0x00,0x00,0x00,0x82,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x47,0x00,0x00,0x00, +0x83,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, +0x87,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x88,0x00,0x00,0x00, +0x87,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x8a,0x00,0x00,0x00,0x47,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8d,0x00,0x00,0x00, +0x8a,0x00,0x00,0x00,0x83,0x00,0x00,0x00,0x0c,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0x8e,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x26,0x00,0x00,0x00,0x88,0x00,0x00,0x00,0x8d,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x92,0x00,0x00,0x00, +0x12,0x00,0x00,0x00,0x91,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x93,0x00,0x00,0x00,0x92,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x94,0x00,0x00,0x00, +0x32,0x00,0x00,0x00,0x93,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x96,0x00,0x00,0x00,0x42,0x00,0x00,0x00, +0x37,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, +0x98,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x97,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x99,0x00,0x00,0x00, +0x98,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x9a,0x00,0x00,0x00,0x96,0x00,0x00,0x00,0x99,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9b,0x00,0x00,0x00, +0x94,0x00,0x00,0x00,0x9a,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x9d,0x00,0x00,0x00,0x9b,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x9e,0x00,0x00,0x00,0x9d,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0xa3,0x00,0x00,0x00, +0x12,0x00,0x00,0x00,0xa2,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xa4,0x00,0x00,0x00,0xa3,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa5,0x00,0x00,0x00, +0x0f,0x00,0x00,0x00,0xa4,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xa8,0x00,0x00,0x00,0x4a,0x00,0x00,0x00, +0xa7,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, +0xaa,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0xa9,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xab,0x00,0x00,0x00, +0xaa,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xac,0x00,0x00,0x00,0xa8,0x00,0x00,0x00,0xab,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xad,0x00,0x00,0x00, +0xa5,0x00,0x00,0x00,0xac,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xaf,0x00,0x00,0x00,0xad,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xb0,0x00,0x00,0x00,0xaf,0x00,0x00,0x00,0x77,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xb2,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xb2,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xe7,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0x05,0x00,0x00,0x00, +0xd1,0x00,0x00,0x00,0xb3,0x00,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc2,0x00,0x00,0x00,0xc3,0x00,0x00,0x00,0xe7,0x02,0x00,0x00, +0xc1,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xb4,0x00,0x00,0x00, +0xb3,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0xc3,0x00,0x00,0x00,0xb3,0x00,0x00,0x00,0xb4,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xb3,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0xcd,0x00,0x00,0x00,0xce,0x00,0x00,0x00,0xca,0x00,0x00,0x00, +0xe7,0x02,0x00,0x00,0x3e,0x00,0x03,0x00,0xce,0x00,0x00,0x00, +0xcc,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xd1,0x00,0x00,0x00,0xe7,0x02,0x00,0x00,0xd0,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xb2,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xb4,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xd4,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xd4,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0x00,0x03,0x00,0x00,0xb0,0x00,0x00,0x00, +0xb4,0x00,0x00,0x00,0xb2,0x01,0x00,0x00,0xd7,0x00,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xfc,0x02,0x00,0x00, +0x9e,0x00,0x00,0x00,0xb4,0x00,0x00,0x00,0xaf,0x01,0x00,0x00, +0xd7,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xe8,0x02,0x00,0x00,0x84,0x00,0x00,0x00,0xb4,0x00,0x00,0x00, +0x60,0x02,0x00,0x00,0xd7,0x00,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc2,0x00,0x00,0x00,0xdb,0x00,0x00,0x00,0xe8,0x02,0x00,0x00, +0x8e,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xd6,0x00,0x00,0x00, +0xd7,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0xdb,0x00,0x00,0x00,0xd5,0x00,0x00,0x00,0xd6,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xd5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xdd,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xdd,0x00,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xf8,0x02,0x00,0x00, +0x3e,0x00,0x00,0x00,0xd5,0x00,0x00,0x00,0x40,0x01,0x00,0x00, +0xde,0x00,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, +0xe3,0x00,0x00,0x00,0xf8,0x02,0x00,0x00,0x37,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0xdf,0x00,0x00,0x00,0xde,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xe3,0x00,0x00,0x00, +0xde,0x00,0x00,0x00,0xdf,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xde,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xe8,0x00,0x00,0x00,0x73,0x00,0x00,0x00,0xf8,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xeb,0x00,0x00,0x00, +0xe8,0x00,0x00,0x00,0x99,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xec,0x00,0x00,0x00,0xeb,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xed,0x00,0x00,0x00,0xfc,0x02,0x00,0x00,0xec,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xef,0x00,0x00,0x00, +0xed,0x00,0x00,0x00,0x6e,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xf5,0x00,0x00,0x00,0xe8,0x00,0x00,0x00, +0xf4,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xf7,0x00,0x00,0x00,0xf5,0x00,0x00,0x00,0x6e,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xfb,0x00,0x00,0x00, +0xef,0x00,0x00,0x00,0xfa,0x00,0x00,0x00,0xc7,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xff,0x00,0x00,0x00,0xef,0x00,0x00,0x00, +0xfe,0x00,0x00,0x00,0x41,0x00,0x07,0x00,0x0a,0x01,0x00,0x00, +0x0b,0x01,0x00,0x00,0x08,0x01,0x00,0x00,0x34,0x00,0x00,0x00, +0xfb,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x01,0x01,0x00,0x00,0x0c,0x01,0x00,0x00,0x0b,0x01,0x00,0x00, +0x73,0x00,0x04,0x00,0xc4,0x00,0x00,0x00,0x0d,0x01,0x00,0x00, +0x0c,0x01,0x00,0x00,0x41,0x00,0x07,0x00,0x0a,0x01,0x00,0x00, +0x10,0x01,0x00,0x00,0x08,0x01,0x00,0x00,0x34,0x00,0x00,0x00, +0xfb,0x00,0x00,0x00,0xd0,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x01,0x01,0x00,0x00,0x11,0x01,0x00,0x00,0x10,0x01,0x00,0x00, +0x73,0x00,0x04,0x00,0xc4,0x00,0x00,0x00,0x12,0x01,0x00,0x00, +0x11,0x01,0x00,0x00,0x41,0x00,0x08,0x00,0x16,0x01,0x00,0x00, +0x17,0x01,0x00,0x00,0x08,0x01,0x00,0x00,0x34,0x00,0x00,0x00, +0xfb,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0xff,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x02,0x01,0x00,0x00,0x18,0x01,0x00,0x00, +0x17,0x01,0x00,0x00,0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x19,0x01,0x00,0x00,0x18,0x01,0x00,0x00,0xc7,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x1e,0x01,0x00,0x00,0x19,0x01,0x00,0x00, +0xfe,0x00,0x00,0x00,0x70,0x00,0x04,0x00,0xc4,0x00,0x00,0x00, +0x1f,0x01,0x00,0x00,0x1e,0x01,0x00,0x00,0xc2,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x21,0x01,0x00,0x00,0x19,0x01,0x00,0x00, +0xa9,0x00,0x00,0x00,0x70,0x00,0x04,0x00,0xc4,0x00,0x00,0x00, +0x22,0x01,0x00,0x00,0x21,0x01,0x00,0x00,0x50,0x00,0x05,0x00, +0x1a,0x01,0x00,0x00,0x23,0x01,0x00,0x00,0x1f,0x01,0x00,0x00, +0x22,0x01,0x00,0x00,0x8e,0x00,0x05,0x00,0x1a,0x01,0x00,0x00, +0x25,0x01,0x00,0x00,0x23,0x01,0x00,0x00,0x0d,0x01,0x00,0x00, +0x50,0x00,0x05,0x00,0x1a,0x01,0x00,0x00,0x27,0x01,0x00,0x00, +0x12,0x01,0x00,0x00,0x12,0x01,0x00,0x00,0x81,0x00,0x05,0x00, +0x1a,0x01,0x00,0x00,0x28,0x01,0x00,0x00,0x25,0x01,0x00,0x00, +0x27,0x01,0x00,0x00,0x51,0x00,0x05,0x00,0xc4,0x00,0x00,0x00, +0x30,0x01,0x00,0x00,0x28,0x01,0x00,0x00,0x00,0x00,0x00,0x00, +0x73,0x00,0x04,0x00,0x01,0x01,0x00,0x00,0x31,0x01,0x00,0x00, +0x30,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0x32,0x01,0x00,0x00, +0x33,0x01,0x00,0x00,0x2d,0x01,0x00,0x00,0xf7,0x00,0x00,0x00, +0x3e,0x00,0x03,0x00,0x33,0x01,0x00,0x00,0x31,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x35,0x01,0x00,0x00, +0xf7,0x00,0x00,0x00,0xfa,0x00,0x00,0x00,0x51,0x00,0x05,0x00, +0xc4,0x00,0x00,0x00,0x37,0x01,0x00,0x00,0x28,0x01,0x00,0x00, +0x01,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x01,0x01,0x00,0x00, +0x38,0x01,0x00,0x00,0x37,0x01,0x00,0x00,0x41,0x00,0x05,0x00, +0x32,0x01,0x00,0x00,0x39,0x01,0x00,0x00,0x2d,0x01,0x00,0x00, +0x35,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x39,0x01,0x00,0x00, +0x38,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x40,0x01,0x00,0x00,0xf8,0x02,0x00,0x00,0x3e,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0xdd,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xdf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x42,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x42,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xf9,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, +0xdf,0x00,0x00,0x00,0xab,0x01,0x00,0x00,0x43,0x01,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0x48,0x01,0x00,0x00, +0xf9,0x02,0x00,0x00,0xa7,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0x44,0x01,0x00,0x00,0x43,0x01,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x48,0x01,0x00,0x00,0x43,0x01,0x00,0x00, +0x44,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x43,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x4d,0x01,0x00,0x00, +0x7e,0x00,0x00,0x00,0xf9,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x50,0x01,0x00,0x00,0x4d,0x01,0x00,0x00, +0xab,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x51,0x01,0x00,0x00,0x50,0x01,0x00,0x00,0x77,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x52,0x01,0x00,0x00, +0x00,0x03,0x00,0x00,0x51,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x54,0x01,0x00,0x00,0x52,0x01,0x00,0x00, +0x79,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x5a,0x01,0x00,0x00,0x4d,0x01,0x00,0x00,0x59,0x01,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x5c,0x01,0x00,0x00, +0x79,0x00,0x00,0x00,0x77,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x5d,0x01,0x00,0x00,0x5a,0x01,0x00,0x00, +0x5c,0x01,0x00,0x00,0x41,0x00,0x08,0x00,0x6c,0x01,0x00,0x00, +0x6d,0x01,0x00,0x00,0x6a,0x01,0x00,0x00,0x34,0x00,0x00,0x00, +0x54,0x01,0x00,0x00,0x34,0x00,0x00,0x00,0x3e,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00,0x6e,0x01,0x00,0x00, +0x6d,0x01,0x00,0x00,0x73,0x00,0x04,0x00,0x01,0x01,0x00,0x00, +0x6f,0x01,0x00,0x00,0x6e,0x01,0x00,0x00,0x41,0x00,0x05,0x00, +0x32,0x01,0x00,0x00,0x70,0x01,0x00,0x00,0x62,0x01,0x00,0x00, +0x5d,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x70,0x01,0x00,0x00, +0x6f,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x72,0x01,0x00,0x00,0x5d,0x01,0x00,0x00,0x39,0x00,0x00,0x00, +0x41,0x00,0x08,0x00,0x6c,0x01,0x00,0x00,0x74,0x01,0x00,0x00, +0x6a,0x01,0x00,0x00,0x34,0x00,0x00,0x00,0x54,0x01,0x00,0x00, +0x34,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0xc4,0x00,0x00,0x00,0x75,0x01,0x00,0x00,0x74,0x01,0x00,0x00, +0x73,0x00,0x04,0x00,0x01,0x01,0x00,0x00,0x76,0x01,0x00,0x00, +0x75,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0x32,0x01,0x00,0x00, +0x77,0x01,0x00,0x00,0x62,0x01,0x00,0x00,0x72,0x01,0x00,0x00, +0x3e,0x00,0x03,0x00,0x77,0x01,0x00,0x00,0x76,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x79,0x01,0x00,0x00, +0x5d,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x41,0x00,0x08,0x00, +0x6c,0x01,0x00,0x00,0x7b,0x01,0x00,0x00,0x6a,0x01,0x00,0x00, +0x34,0x00,0x00,0x00,0x54,0x01,0x00,0x00,0x34,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00, +0x7c,0x01,0x00,0x00,0x7b,0x01,0x00,0x00,0x73,0x00,0x04,0x00, +0x01,0x01,0x00,0x00,0x7d,0x01,0x00,0x00,0x7c,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0x32,0x01,0x00,0x00,0x7e,0x01,0x00,0x00, +0x62,0x01,0x00,0x00,0x79,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, +0x7e,0x01,0x00,0x00,0x7d,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x81,0x01,0x00,0x00,0x5d,0x01,0x00,0x00, +0x80,0x01,0x00,0x00,0x41,0x00,0x08,0x00,0x6c,0x01,0x00,0x00, +0x83,0x01,0x00,0x00,0x6a,0x01,0x00,0x00,0x34,0x00,0x00,0x00, +0x54,0x01,0x00,0x00,0x34,0x00,0x00,0x00,0x80,0x01,0x00,0x00, +0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00,0x84,0x01,0x00,0x00, +0x83,0x01,0x00,0x00,0x73,0x00,0x04,0x00,0x01,0x01,0x00,0x00, +0x85,0x01,0x00,0x00,0x84,0x01,0x00,0x00,0x41,0x00,0x05,0x00, +0x32,0x01,0x00,0x00,0x86,0x01,0x00,0x00,0x62,0x01,0x00,0x00, +0x81,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x86,0x01,0x00,0x00, +0x85,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x89,0x01,0x00,0x00,0x5d,0x01,0x00,0x00,0x88,0x01,0x00,0x00, +0x41,0x00,0x08,0x00,0x6c,0x01,0x00,0x00,0x8b,0x01,0x00,0x00, +0x6a,0x01,0x00,0x00,0x34,0x00,0x00,0x00,0x54,0x01,0x00,0x00, +0xd0,0x00,0x00,0x00,0x3e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0xc4,0x00,0x00,0x00,0x8c,0x01,0x00,0x00,0x8b,0x01,0x00,0x00, +0x73,0x00,0x04,0x00,0x01,0x01,0x00,0x00,0x8d,0x01,0x00,0x00, +0x8c,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0x32,0x01,0x00,0x00, +0x8e,0x01,0x00,0x00,0x62,0x01,0x00,0x00,0x89,0x01,0x00,0x00, +0x3e,0x00,0x03,0x00,0x8e,0x01,0x00,0x00,0x8d,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x91,0x01,0x00,0x00, +0x5d,0x01,0x00,0x00,0x90,0x01,0x00,0x00,0x41,0x00,0x08,0x00, +0x6c,0x01,0x00,0x00,0x93,0x01,0x00,0x00,0x6a,0x01,0x00,0x00, +0x34,0x00,0x00,0x00,0x54,0x01,0x00,0x00,0xd0,0x00,0x00,0x00, +0x39,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00, +0x94,0x01,0x00,0x00,0x93,0x01,0x00,0x00,0x73,0x00,0x04,0x00, +0x01,0x01,0x00,0x00,0x95,0x01,0x00,0x00,0x94,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0x32,0x01,0x00,0x00,0x96,0x01,0x00,0x00, +0x62,0x01,0x00,0x00,0x91,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, +0x96,0x01,0x00,0x00,0x95,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x99,0x01,0x00,0x00,0x5d,0x01,0x00,0x00, +0x98,0x01,0x00,0x00,0x41,0x00,0x08,0x00,0x6c,0x01,0x00,0x00, +0x9b,0x01,0x00,0x00,0x6a,0x01,0x00,0x00,0x34,0x00,0x00,0x00, +0x54,0x01,0x00,0x00,0xd0,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00,0x9c,0x01,0x00,0x00, +0x9b,0x01,0x00,0x00,0x73,0x00,0x04,0x00,0x01,0x01,0x00,0x00, +0x9d,0x01,0x00,0x00,0x9c,0x01,0x00,0x00,0x41,0x00,0x05,0x00, +0x32,0x01,0x00,0x00,0x9e,0x01,0x00,0x00,0x62,0x01,0x00,0x00, +0x99,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x9e,0x01,0x00,0x00, +0x9d,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xa1,0x01,0x00,0x00,0x5d,0x01,0x00,0x00,0xa0,0x01,0x00,0x00, +0x41,0x00,0x08,0x00,0x6c,0x01,0x00,0x00,0xa3,0x01,0x00,0x00, +0x6a,0x01,0x00,0x00,0x34,0x00,0x00,0x00,0x54,0x01,0x00,0x00, +0xd0,0x00,0x00,0x00,0x80,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, +0xc4,0x00,0x00,0x00,0xa4,0x01,0x00,0x00,0xa3,0x01,0x00,0x00, +0x73,0x00,0x04,0x00,0x01,0x01,0x00,0x00,0xa5,0x01,0x00,0x00, +0xa4,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0x32,0x01,0x00,0x00, +0xa6,0x01,0x00,0x00,0x62,0x01,0x00,0x00,0xa1,0x01,0x00,0x00, +0x3e,0x00,0x03,0x00,0xa6,0x01,0x00,0x00,0xa5,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xab,0x01,0x00,0x00, +0xf9,0x02,0x00,0x00,0xa9,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0x42,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x44,0x01,0x00,0x00, +0xe0,0x00,0x04,0x00,0x0c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0xac,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xaf,0x01,0x00,0x00,0xfc,0x02,0x00,0x00,0xad,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xb2,0x01,0x00,0x00, +0x00,0x03,0x00,0x00,0xb0,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0xb4,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xb4,0x01,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x02,0x03,0x00,0x00, +0x3e,0x00,0x00,0x00,0x44,0x01,0x00,0x00,0x5e,0x02,0x00,0x00, +0xb7,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, +0xba,0x01,0x00,0x00,0x02,0x03,0x00,0x00,0x6c,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0xb6,0x01,0x00,0x00,0xb7,0x01,0x00,0x00, +0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xba,0x01,0x00,0x00, +0xb5,0x01,0x00,0x00,0xb6,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xb5,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xbc,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xbc,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0x06,0x03,0x00,0x00,0x3e,0x00,0x00,0x00, +0xb5,0x01,0x00,0x00,0xe8,0x01,0x00,0x00,0xbf,0x01,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0xc2,0x01,0x00,0x00, +0x06,0x03,0x00,0x00,0x60,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0xbe,0x01,0x00,0x00,0xbf,0x01,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0xc2,0x01,0x00,0x00,0xbd,0x01,0x00,0x00, +0xbe,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xbd,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0xc4,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xc4,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0x18,0x03,0x00,0x00,0x3e,0x00,0x00,0x00,0xbd,0x01,0x00,0x00, +0xe6,0x01,0x00,0x00,0xc5,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc2,0x00,0x00,0x00,0xca,0x01,0x00,0x00,0x18,0x03,0x00,0x00, +0x62,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xc6,0x01,0x00,0x00, +0xc5,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0xca,0x01,0x00,0x00,0xc5,0x01,0x00,0x00,0xc6,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xc5,0x01,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xd0,0x01,0x00,0x00,0x06,0x03,0x00,0x00, +0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xd2,0x01,0x00,0x00,0xd0,0x01,0x00,0x00,0x18,0x03,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xd4,0x01,0x00,0x00, +0x55,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xd6,0x01,0x00,0x00,0x06,0x03,0x00,0x00, +0x61,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xd7,0x01,0x00,0x00,0xd4,0x01,0x00,0x00,0xd6,0x01,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xd9,0x01,0x00,0x00, +0x64,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xda,0x01,0x00,0x00,0xd7,0x01,0x00,0x00, +0xd9,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xdc,0x01,0x00,0x00,0xda,0x01,0x00,0x00,0x18,0x03,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xde,0x01,0x00,0x00, +0xdc,0x01,0x00,0x00,0xdd,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xe0,0x01,0x00,0x00,0xde,0x01,0x00,0x00, +0x02,0x03,0x00,0x00,0x41,0x00,0x05,0x00,0x32,0x01,0x00,0x00, +0xe1,0x01,0x00,0x00,0x2d,0x01,0x00,0x00,0xe0,0x01,0x00,0x00, +0x3d,0x00,0x04,0x00,0x01,0x01,0x00,0x00,0xe2,0x01,0x00,0x00, +0xe1,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0xe3,0x01,0x00,0x00, +0xe4,0x01,0x00,0x00,0xce,0x01,0x00,0x00,0xd2,0x01,0x00,0x00, +0x3e,0x00,0x03,0x00,0xe4,0x01,0x00,0x00,0xe2,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xe6,0x01,0x00,0x00, +0x18,0x03,0x00,0x00,0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xc4,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xc6,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0xbf,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xbf,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xe8,0x01,0x00,0x00,0x06,0x03,0x00,0x00,0xd0,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xbc,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xbe,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xea,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xea,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0x07,0x03,0x00,0x00,0x3e,0x00,0x00,0x00, +0xbe,0x01,0x00,0x00,0x16,0x02,0x00,0x00,0xed,0x01,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0xf0,0x01,0x00,0x00, +0x07,0x03,0x00,0x00,0xbf,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0xec,0x01,0x00,0x00,0xed,0x01,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0xf0,0x01,0x00,0x00,0xeb,0x01,0x00,0x00, +0xec,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xeb,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0xf2,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xf2,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0x15,0x03,0x00,0x00,0x3e,0x00,0x00,0x00,0xeb,0x01,0x00,0x00, +0x14,0x02,0x00,0x00,0xf3,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc2,0x00,0x00,0x00,0xf8,0x01,0x00,0x00,0x15,0x03,0x00,0x00, +0xbc,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xf4,0x01,0x00,0x00, +0xf3,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0xf8,0x01,0x00,0x00,0xf3,0x01,0x00,0x00,0xf4,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xf3,0x01,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xfe,0x01,0x00,0x00,0x07,0x03,0x00,0x00, +0xbc,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x00,0x02,0x00,0x00,0xfe,0x01,0x00,0x00,0x15,0x03,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x02,0x02,0x00,0x00, +0x59,0x00,0x00,0x00,0xb9,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x05,0x02,0x00,0x00,0x07,0x03,0x00,0x00, +0x04,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x06,0x02,0x00,0x00,0x02,0x02,0x00,0x00,0x05,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x08,0x02,0x00,0x00, +0x68,0x00,0x00,0x00,0xbc,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x09,0x02,0x00,0x00,0x06,0x02,0x00,0x00, +0x08,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x0b,0x02,0x00,0x00,0x09,0x02,0x00,0x00,0x15,0x03,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x0d,0x02,0x00,0x00, +0x0b,0x02,0x00,0x00,0x0c,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x0f,0x02,0x00,0x00,0x0d,0x02,0x00,0x00, +0x02,0x03,0x00,0x00,0x41,0x00,0x05,0x00,0x32,0x01,0x00,0x00, +0x10,0x02,0x00,0x00,0x62,0x01,0x00,0x00,0x0f,0x02,0x00,0x00, +0x3d,0x00,0x04,0x00,0x01,0x01,0x00,0x00,0x11,0x02,0x00,0x00, +0x10,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0xe3,0x01,0x00,0x00, +0x12,0x02,0x00,0x00,0xfc,0x01,0x00,0x00,0x00,0x02,0x00,0x00, +0x3e,0x00,0x03,0x00,0x12,0x02,0x00,0x00,0x11,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x14,0x02,0x00,0x00, +0x15,0x03,0x00,0x00,0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xf2,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xf4,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0xed,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xed,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x16,0x02,0x00,0x00,0x07,0x03,0x00,0x00,0xd0,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xea,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xec,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x18,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x18,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0x08,0x03,0x00,0x00,0x3e,0x00,0x00,0x00, +0xec,0x01,0x00,0x00,0x5c,0x02,0x00,0x00,0x1b,0x02,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0x1e,0x02,0x00,0x00, +0x08,0x03,0x00,0x00,0xbf,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0x1a,0x02,0x00,0x00,0x1b,0x02,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x1e,0x02,0x00,0x00,0x19,0x02,0x00,0x00, +0x1a,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x19,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0x20,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x20,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0x0c,0x03,0x00,0x00,0x3e,0x00,0x00,0x00,0x19,0x02,0x00,0x00, +0x5a,0x02,0x00,0x00,0x23,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc2,0x00,0x00,0x00,0x26,0x02,0x00,0x00,0x0c,0x03,0x00,0x00, +0x60,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x22,0x02,0x00,0x00, +0x23,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x26,0x02,0x00,0x00,0x21,0x02,0x00,0x00,0x22,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x21,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0x28,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x28,0x02,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x0e,0x03,0x00,0x00, +0x3e,0x00,0x00,0x00,0x21,0x02,0x00,0x00,0x58,0x02,0x00,0x00, +0x2b,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, +0x2e,0x02,0x00,0x00,0x0e,0x03,0x00,0x00,0xbc,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0x2a,0x02,0x00,0x00,0x2b,0x02,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x2e,0x02,0x00,0x00, +0x29,0x02,0x00,0x00,0x2a,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x29,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x30,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x30,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0x10,0x03,0x00,0x00,0x3e,0x00,0x00,0x00, +0x29,0x02,0x00,0x00,0x56,0x02,0x00,0x00,0x31,0x02,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0x36,0x02,0x00,0x00, +0x10,0x03,0x00,0x00,0x62,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0x32,0x02,0x00,0x00,0x31,0x02,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x36,0x02,0x00,0x00,0x31,0x02,0x00,0x00, +0x32,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x31,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x38,0x02,0x00,0x00, +0x08,0x03,0x00,0x00,0xbc,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x3a,0x02,0x00,0x00,0x38,0x02,0x00,0x00, +0x0e,0x03,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x3c,0x02,0x00,0x00,0x3a,0x02,0x00,0x00,0x3b,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3e,0x02,0x00,0x00, +0x0c,0x03,0x00,0x00,0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x3f,0x02,0x00,0x00,0x3c,0x02,0x00,0x00, +0x3e,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x41,0x02,0x00,0x00,0x3f,0x02,0x00,0x00,0x10,0x03,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x45,0x02,0x00,0x00, +0x3e,0x02,0x00,0x00,0x10,0x03,0x00,0x00,0x41,0x00,0x05,0x00, +0xe3,0x01,0x00,0x00,0x46,0x02,0x00,0x00,0xce,0x01,0x00,0x00, +0x45,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0x01,0x01,0x00,0x00, +0x47,0x02,0x00,0x00,0x46,0x02,0x00,0x00,0x73,0x00,0x04,0x00, +0xc4,0x00,0x00,0x00,0x48,0x02,0x00,0x00,0x47,0x02,0x00,0x00, +0x41,0x00,0x05,0x00,0xe3,0x01,0x00,0x00,0x4d,0x02,0x00,0x00, +0xfc,0x01,0x00,0x00,0x3a,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, +0x01,0x01,0x00,0x00,0x4e,0x02,0x00,0x00,0x4d,0x02,0x00,0x00, +0x73,0x00,0x04,0x00,0xc4,0x00,0x00,0x00,0x4f,0x02,0x00,0x00, +0x4e,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0xcd,0x00,0x00,0x00, +0x51,0x02,0x00,0x00,0xca,0x00,0x00,0x00,0x41,0x02,0x00,0x00, +0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00,0x52,0x02,0x00,0x00, +0x51,0x02,0x00,0x00,0x0c,0x00,0x08,0x00,0xc4,0x00,0x00,0x00, +0x53,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00, +0x48,0x02,0x00,0x00,0x4f,0x02,0x00,0x00,0x52,0x02,0x00,0x00, +0x3e,0x00,0x03,0x00,0x51,0x02,0x00,0x00,0x53,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x56,0x02,0x00,0x00, +0x10,0x03,0x00,0x00,0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x30,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x32,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0x2b,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x2b,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x58,0x02,0x00,0x00,0x0e,0x03,0x00,0x00,0xd0,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x28,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x2a,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x23,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x23,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x5a,0x02,0x00,0x00,0x0c,0x03,0x00,0x00, +0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x20,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x22,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0x1b,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x1b,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x5c,0x02,0x00,0x00, +0x08,0x03,0x00,0x00,0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x18,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x1a,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0xb7,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xb7,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x5e,0x02,0x00,0x00,0x02,0x03,0x00,0x00,0xd0,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xb4,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xb6,0x01,0x00,0x00,0xe0,0x00,0x04,0x00,0x0c,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0xac,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0xd7,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xd7,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x60,0x02,0x00,0x00, +0xe8,0x02,0x00,0x00,0x6c,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xd4,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xd6,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x65,0x02,0x00,0x00, +0x55,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x66,0x02,0x00,0x00,0x96,0x00,0x00,0x00, +0x65,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x6b,0x02,0x00,0x00,0x59,0x00,0x00,0x00,0xb9,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x6c,0x02,0x00,0x00, +0xa8,0x00,0x00,0x00,0x6b,0x02,0x00,0x00,0x41,0x00,0x05,0x00, +0x15,0x00,0x00,0x00,0x71,0x02,0x00,0x00,0x12,0x00,0x00,0x00, +0x70,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x72,0x02,0x00,0x00,0x71,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x73,0x02,0x00,0x00,0x0f,0x00,0x00,0x00, +0x72,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x77,0x02,0x00,0x00,0x47,0x00,0x00,0x00,0x72,0x02,0x00,0x00, +0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00,0x79,0x02,0x00,0x00, +0x78,0x02,0x00,0x00,0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x7a,0x02,0x00,0x00,0x79,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x7b,0x02,0x00,0x00, +0x77,0x02,0x00,0x00,0x7a,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x7c,0x02,0x00,0x00,0x73,0x02,0x00,0x00, +0x7b,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x7e,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x7e,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xe9,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, +0xd6,0x00,0x00,0x00,0xe4,0x02,0x00,0x00,0x81,0x02,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0x84,0x02,0x00,0x00, +0xe9,0x02,0x00,0x00,0xbf,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0x80,0x02,0x00,0x00,0x81,0x02,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x84,0x02,0x00,0x00,0x7f,0x02,0x00,0x00, +0x80,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x7f,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0x86,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x86,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xea,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0x7f,0x02,0x00,0x00, +0xe2,0x02,0x00,0x00,0x89,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc2,0x00,0x00,0x00,0x8c,0x02,0x00,0x00,0xea,0x02,0x00,0x00, +0x60,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x88,0x02,0x00,0x00, +0x89,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x8c,0x02,0x00,0x00,0x87,0x02,0x00,0x00,0x88,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x87,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x90,0x02,0x00,0x00,0xea,0x02,0x00,0x00, +0x61,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x91,0x02,0x00,0x00,0x66,0x02,0x00,0x00,0x90,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x93,0x02,0x00,0x00, +0x64,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x94,0x02,0x00,0x00,0x91,0x02,0x00,0x00, +0x93,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x98,0x02,0x00,0x00,0xe9,0x02,0x00,0x00,0x04,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x99,0x02,0x00,0x00, +0x6c,0x02,0x00,0x00,0x98,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x9b,0x02,0x00,0x00,0x68,0x00,0x00,0x00, +0xbc,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x9c,0x02,0x00,0x00,0x99,0x02,0x00,0x00,0x9b,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0x9e,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x9e,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xec,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0x87,0x02,0x00,0x00, +0xe0,0x02,0x00,0x00,0xa1,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc2,0x00,0x00,0x00,0xa4,0x02,0x00,0x00,0xec,0x02,0x00,0x00, +0xbc,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xa0,0x02,0x00,0x00, +0xa1,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0xa4,0x02,0x00,0x00,0x9f,0x02,0x00,0x00,0xa0,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x9f,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0xa6,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0xa6,0x02,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xee,0x02,0x00,0x00, +0x3e,0x00,0x00,0x00,0x9f,0x02,0x00,0x00,0xde,0x02,0x00,0x00, +0xa9,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, +0xac,0x02,0x00,0x00,0xee,0x02,0x00,0x00,0x62,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0xa8,0x02,0x00,0x00,0xa9,0x02,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xac,0x02,0x00,0x00, +0xa7,0x02,0x00,0x00,0xa8,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0xa7,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xaf,0x02,0x00,0x00,0x94,0x02,0x00,0x00,0xee,0x02,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0xb2,0x02,0x00,0x00, +0xaf,0x02,0x00,0x00,0x36,0x00,0x00,0x00,0xf7,0x00,0x03,0x00, +0xb4,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0xb2,0x02,0x00,0x00,0xb3,0x02,0x00,0x00,0xb4,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0xb3,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xb7,0x02,0x00,0x00,0x9c,0x02,0x00,0x00, +0xec,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, +0xb8,0x02,0x00,0x00,0x12,0x00,0x00,0x00,0xd0,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xb9,0x02,0x00,0x00, +0xb8,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, +0xba,0x02,0x00,0x00,0xb7,0x02,0x00,0x00,0xb9,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0xb4,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0xb4,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0xc2,0x00,0x00,0x00, +0xbb,0x02,0x00,0x00,0xb2,0x02,0x00,0x00,0xa7,0x02,0x00,0x00, +0xba,0x02,0x00,0x00,0xb3,0x02,0x00,0x00,0xf7,0x00,0x03,0x00, +0xbd,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0xbb,0x02,0x00,0x00,0xbc,0x02,0x00,0x00,0xbd,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0xbc,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xc5,0x02,0x00,0x00,0x9c,0x02,0x00,0x00, +0xec,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, +0xc7,0x02,0x00,0x00,0x12,0x00,0x00,0x00,0xc6,0x02,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xc8,0x02,0x00,0x00, +0xc7,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xc9,0x02,0x00,0x00,0xc5,0x02,0x00,0x00,0xc8,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xca,0x02,0x00,0x00, +0x7c,0x02,0x00,0x00,0xc9,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xcc,0x02,0x00,0x00,0xca,0x02,0x00,0x00, +0x94,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xce,0x02,0x00,0x00,0xcc,0x02,0x00,0x00,0xee,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xd0,0x02,0x00,0x00, +0xe9,0x02,0x00,0x00,0xbc,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xd2,0x02,0x00,0x00,0xd0,0x02,0x00,0x00, +0xec,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xd4,0x02,0x00,0x00,0xd2,0x02,0x00,0x00,0xd3,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xd6,0x02,0x00,0x00, +0xea,0x02,0x00,0x00,0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xd7,0x02,0x00,0x00,0xd4,0x02,0x00,0x00, +0xd6,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xd9,0x02,0x00,0x00,0xd7,0x02,0x00,0x00,0xee,0x02,0x00,0x00, +0x41,0x00,0x05,0x00,0xcd,0x00,0x00,0x00,0xda,0x02,0x00,0x00, +0xca,0x00,0x00,0x00,0xd9,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, +0xc4,0x00,0x00,0x00,0xdb,0x02,0x00,0x00,0xda,0x02,0x00,0x00, +0x41,0x00,0x06,0x00,0x6c,0x01,0x00,0x00,0xdc,0x02,0x00,0x00, +0xc1,0x02,0x00,0x00,0x34,0x00,0x00,0x00,0xce,0x02,0x00,0x00, +0x3e,0x00,0x03,0x00,0xdc,0x02,0x00,0x00,0xdb,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0xbd,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0xbd,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0xa9,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0xa9,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xde,0x02,0x00,0x00,0xee,0x02,0x00,0x00, +0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xa6,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0xa8,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0xa1,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0xa1,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xe0,0x02,0x00,0x00, +0xec,0x02,0x00,0x00,0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x9e,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0xa0,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0x89,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x89,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xe2,0x02,0x00,0x00,0xea,0x02,0x00,0x00,0xd0,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x86,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x88,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x81,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x81,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xe4,0x02,0x00,0x00,0xe9,0x02,0x00,0x00, +0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x7e,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x80,0x02,0x00,0x00,0xfd,0x00,0x01,0x00, +0x38,0x00,0x01,0x00, +}; +const uint64_t matmul_q4_1_f32_aligned_len = 11224; + +unsigned char matmul_q4_1_f32_aligned_fp32_data[] = { +0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, +0xef,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, +0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x51,0x11,0x00,0x00, +0x11,0x00,0x02,0x00,0x60,0x11,0x00,0x00,0x0b,0x00,0x06,0x00, +0x01,0x00,0x00,0x00,0x47,0x4c,0x53,0x4c,0x2e,0x73,0x74,0x64, +0x2e,0x34,0x35,0x30,0x00,0x00,0x00,0x00,0x0e,0x00,0x03,0x00, +0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x0f,0x00,0x0f,0x00, +0x05,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x6d,0x61,0x69,0x6e, +0x00,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0x3d,0x00,0x00,0x00,0x4c,0x00,0x00,0x00,0x08,0x01,0x00,0x00, +0x2d,0x01,0x00,0x00,0x60,0x01,0x00,0x00,0x67,0x01,0x00,0x00, +0x4e,0x02,0x00,0x00,0x97,0x02,0x00,0x00,0x10,0x00,0x06,0x00, +0x04,0x00,0x00,0x00,0x11,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x0b,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x1c,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x10,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x08,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x03,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x10,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x10,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, +0x05,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x14,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x10,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x1c,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, +0x08,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x09,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x24,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x10,0x00,0x00,0x00,0x0a,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x28,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, +0x0b,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x2c,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x30,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x10,0x00,0x00,0x00,0x0d,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x34,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x10,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x37,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x3d,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x1a,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x4c,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, +0x1b,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x4f,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x53,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x60,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x62,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x6c,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x03,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0xa7,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xb9,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0xbc,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x08,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x03,0x01,0x00,0x00,0x06,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x04,0x01,0x00,0x00, +0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x04,0x01,0x00,0x00,0x01,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x04,0x01,0x00,0x00,0x02,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x05,0x01,0x00,0x00, +0x06,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x48,0x00,0x04,0x00, +0x06,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x06,0x01,0x00,0x00,0x00,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, +0x06,0x01,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x08,0x01,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x08,0x01,0x00,0x00,0x21,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x38,0x01,0x00,0x00, +0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x39,0x01,0x00,0x00,0x0b,0x00,0x00,0x00,0x19,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x64,0x01,0x00,0x00,0x06,0x00,0x00,0x00, +0x10,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0x65,0x01,0x00,0x00, +0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x65,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x65,0x01,0x00,0x00, +0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x67,0x01,0x00,0x00, +0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x67,0x01,0x00,0x00,0x21,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x4e,0x02,0x00,0x00,0x0b,0x00,0x00,0x00, +0x18,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x94,0x02,0x00,0x00, +0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00, +0x95,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x19,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x95,0x02,0x00,0x00,0x00,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, +0x95,0x02,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x97,0x02,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x97,0x02,0x00,0x00,0x21,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x13,0x00,0x02,0x00,0x02,0x00,0x00,0x00, +0x21,0x00,0x03,0x00,0x03,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x15,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x17,0x00,0x04,0x00,0x09,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x0a,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x09,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x0d,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x1e,0x00,0x10,0x00,0x10,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x11,0x00,0x00,0x00, +0x09,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0x11,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x09,0x00,0x00,0x00, +0x15,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, +0x14,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x15,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x21,0x00,0x00,0x00, +0x0a,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, +0x27,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0x2d,0x00,0x00,0x00,0x07,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x34,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x37,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00,0x3d,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x3e,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0x0a,0x00,0x00,0x00,0x4c,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x4f,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x53,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x54,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0x37,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x58,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0x37,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x32,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x61,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x60,0x00,0x00,0x00, +0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x62,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x63,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x61,0x00,0x00,0x00, +0x62,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x67,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x61,0x00,0x00,0x00, +0x62,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x72,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x77,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x78,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x77,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x7d,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x77,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x81,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0x91,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x97,0x00,0x00,0x00, +0x03,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, +0xa2,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x32,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xa7,0x00,0x00,0x00,0x40,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0xa9,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xb8,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00, +0x62,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xb9,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xba,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0x53,0x00,0x00,0x00,0xb9,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xbb,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0x4f,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x32,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xbc,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xbd,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0xbb,0x00,0x00,0x00,0xbc,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xbe,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0xbd,0x00,0x00,0x00,0x60,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xbf,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0xba,0x00,0x00,0x00,0xbe,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xc0,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0xb8,0x00,0x00,0x00,0xbf,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xc1,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0xc0,0x00,0x00,0x00,0xbc,0x00,0x00,0x00, +0x14,0x00,0x02,0x00,0xc2,0x00,0x00,0x00,0x16,0x00,0x03,0x00, +0xc4,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xc5,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xc6,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0xc5,0x00,0x00,0x00,0xbf,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xc7,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0xc6,0x00,0x00,0x00,0xbc,0x00,0x00,0x00,0x1c,0x00,0x04,0x00, +0xc8,0x00,0x00,0x00,0xc4,0x00,0x00,0x00,0xc7,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0xc9,0x00,0x00,0x00,0x07,0x00,0x00,0x00, +0xc8,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0xc4,0x00,0x00,0x00, +0xcc,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0xcd,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0xc4,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0xd0,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xf4,0x00,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, +0x39,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xfa,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xfe,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, +0x16,0x00,0x03,0x00,0x01,0x01,0x00,0x00,0x10,0x00,0x00,0x00, +0x15,0x00,0x04,0x00,0x02,0x01,0x00,0x00,0x08,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x1c,0x00,0x04,0x00,0x03,0x01,0x00,0x00, +0x02,0x01,0x00,0x00,0xfa,0x00,0x00,0x00,0x1e,0x00,0x05,0x00, +0x04,0x01,0x00,0x00,0x01,0x01,0x00,0x00,0x01,0x01,0x00,0x00, +0x03,0x01,0x00,0x00,0x1d,0x00,0x03,0x00,0x05,0x01,0x00,0x00, +0x04,0x01,0x00,0x00,0x1e,0x00,0x03,0x00,0x06,0x01,0x00,0x00, +0x05,0x01,0x00,0x00,0x20,0x00,0x04,0x00,0x07,0x01,0x00,0x00, +0x0c,0x00,0x00,0x00,0x06,0x01,0x00,0x00,0x3b,0x00,0x04,0x00, +0x07,0x01,0x00,0x00,0x08,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x0a,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, +0x01,0x01,0x00,0x00,0x20,0x00,0x04,0x00,0x16,0x01,0x00,0x00, +0x0c,0x00,0x00,0x00,0x02,0x01,0x00,0x00,0x17,0x00,0x04,0x00, +0x1a,0x01,0x00,0x00,0xc4,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x29,0x01,0x00,0x00, +0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x2a,0x01,0x00,0x00, +0x84,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x29,0x01,0x00,0x00, +0x1c,0x00,0x04,0x00,0x2b,0x01,0x00,0x00,0xc4,0x00,0x00,0x00, +0x2a,0x01,0x00,0x00,0x20,0x00,0x04,0x00,0x2c,0x01,0x00,0x00, +0x04,0x00,0x00,0x00,0x2b,0x01,0x00,0x00,0x3b,0x00,0x04,0x00, +0x2c,0x01,0x00,0x00,0x2d,0x01,0x00,0x00,0x04,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x31,0x01,0x00,0x00,0x04,0x00,0x00,0x00, +0xc4,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x38,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x33,0x00,0x06,0x00, +0x09,0x00,0x00,0x00,0x39,0x01,0x00,0x00,0x38,0x01,0x00,0x00, +0x39,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x3a,0x01,0x00,0x00,0x51,0x00,0x00,0x00, +0x39,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x3b,0x01,0x00,0x00,0x84,0x00,0x00,0x00, +0x3a,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x3c,0x01,0x00,0x00,0x86,0x00,0x00,0x00, +0x3b,0x01,0x00,0x00,0x6c,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x57,0x01,0x00,0x00,0x80,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x5c,0x01,0x00,0x00,0x80,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x5d,0x01,0x00,0x00,0x84,0x00,0x00,0x00, +0xa7,0x00,0x00,0x00,0x5c,0x01,0x00,0x00,0x1c,0x00,0x04,0x00, +0x5e,0x01,0x00,0x00,0xc4,0x00,0x00,0x00,0x5d,0x01,0x00,0x00, +0x20,0x00,0x04,0x00,0x5f,0x01,0x00,0x00,0x04,0x00,0x00,0x00, +0x5e,0x01,0x00,0x00,0x3b,0x00,0x04,0x00,0x5f,0x01,0x00,0x00, +0x60,0x01,0x00,0x00,0x04,0x00,0x00,0x00,0x17,0x00,0x04,0x00, +0x63,0x01,0x00,0x00,0xc4,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x1d,0x00,0x03,0x00,0x64,0x01,0x00,0x00,0x63,0x01,0x00,0x00, +0x1e,0x00,0x03,0x00,0x65,0x01,0x00,0x00,0x64,0x01,0x00,0x00, +0x20,0x00,0x04,0x00,0x66,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, +0x65,0x01,0x00,0x00,0x3b,0x00,0x04,0x00,0x66,0x01,0x00,0x00, +0x67,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x69,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0xc4,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x7a,0x01,0x00,0x00, +0x03,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x80,0x01,0x00,0x00,0x51,0x00,0x00,0x00,0x39,0x01,0x00,0x00, +0x00,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x81,0x01,0x00,0x00,0x84,0x00,0x00,0x00,0x80,0x01,0x00,0x00, +0x77,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x82,0x01,0x00,0x00,0x86,0x00,0x00,0x00,0x81,0x01,0x00,0x00, +0x6c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x85,0x01,0x00,0x00,0x08,0x01,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x86,0x01,0x00,0x00,0x86,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x89,0x01,0x00,0x00,0x86,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x77,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xa4,0x01,0x00,0x00,0x84,0x00,0x00,0x00, +0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x1c,0x00,0x04,0x00, +0xa5,0x01,0x00,0x00,0xc4,0x00,0x00,0x00,0xa4,0x01,0x00,0x00, +0x20,0x00,0x04,0x00,0xa6,0x01,0x00,0x00,0x07,0x00,0x00,0x00, +0xa5,0x01,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xb6,0x01,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, +0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xd1,0x01,0x00,0x00,0x84,0x00,0x00,0x00,0xbf,0x00,0x00,0x00, +0xbc,0x00,0x00,0x00,0x1c,0x00,0x04,0x00,0xd2,0x01,0x00,0x00, +0xc4,0x00,0x00,0x00,0xd1,0x01,0x00,0x00,0x20,0x00,0x04,0x00, +0xd3,0x01,0x00,0x00,0x07,0x00,0x00,0x00,0xd2,0x01,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xdc,0x01,0x00,0x00, +0x86,0x00,0x00,0x00,0xb9,0x00,0x00,0x00,0xbf,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xe4,0x01,0x00,0x00, +0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x13,0x02,0x00,0x00, +0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x46,0x02,0x00,0x00, +0x0d,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00, +0x4e,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, +0x94,0x02,0x00,0x00,0xc4,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, +0x95,0x02,0x00,0x00,0x94,0x02,0x00,0x00,0x20,0x00,0x04,0x00, +0x96,0x02,0x00,0x00,0x0c,0x00,0x00,0x00,0x95,0x02,0x00,0x00, +0x3b,0x00,0x04,0x00,0x96,0x02,0x00,0x00,0x97,0x02,0x00,0x00, +0x0c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, +0x9c,0x02,0x00,0x00,0x05,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xa9,0x02,0x00,0x00,0x84,0x00,0x00,0x00, +0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x36,0x00,0x05,0x00, +0x02,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x03,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x05,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0xc9,0x00,0x00,0x00,0xca,0x00,0x00,0x00, +0x07,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0xa6,0x01,0x00,0x00, +0xa7,0x01,0x00,0x00,0x07,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0xd3,0x01,0x00,0x00,0xd4,0x01,0x00,0x00,0x07,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00,0x0e,0x00,0x00,0x00, +0x0b,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x0e,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x16,0x00,0x00,0x00, +0x12,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x17,0x00,0x00,0x00,0x16,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x18,0x00,0x00,0x00, +0x0f,0x00,0x00,0x00,0x17,0x00,0x00,0x00,0x89,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x1e,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, +0x17,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, +0x22,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x21,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x22,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x24,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x28,0x00,0x00,0x00, +0x12,0x00,0x00,0x00,0x27,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x29,0x00,0x00,0x00,0x28,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x2a,0x00,0x00,0x00, +0x1e,0x00,0x00,0x00,0x29,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x15,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0x2d,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x2f,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x30,0x00,0x00,0x00,0x24,0x00,0x00,0x00, +0x2f,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x32,0x00,0x00,0x00,0x30,0x00,0x00,0x00,0x2a,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x35,0x00,0x00,0x00, +0x12,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x36,0x00,0x00,0x00,0x35,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x38,0x00,0x00,0x00, +0x36,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x82,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x38,0x00,0x00,0x00, +0x39,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x3b,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x37,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00,0x3f,0x00,0x00,0x00, +0x3d,0x00,0x00,0x00,0x3e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x3f,0x00,0x00,0x00, +0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x42,0x00,0x00,0x00, +0x40,0x00,0x00,0x00,0x3b,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x47,0x00,0x00,0x00,0x40,0x00,0x00,0x00, +0x3b,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00, +0x49,0x00,0x00,0x00,0x3d,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x4a,0x00,0x00,0x00, +0x49,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00, +0x4d,0x00,0x00,0x00,0x4c,0x00,0x00,0x00,0x3e,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, +0x4d,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x50,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x4f,0x00,0x00,0x00, +0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x55,0x00,0x00,0x00, +0x50,0x00,0x00,0x00,0x54,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x59,0x00,0x00,0x00,0x50,0x00,0x00,0x00, +0x58,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x5d,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x4f,0x00,0x00,0x00, +0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x64,0x00,0x00,0x00, +0x5d,0x00,0x00,0x00,0x63,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x68,0x00,0x00,0x00,0x5d,0x00,0x00,0x00, +0x67,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x6e,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x73,0x00,0x00,0x00, +0x4e,0x00,0x00,0x00,0x72,0x00,0x00,0x00,0x89,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x79,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, +0x78,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x7e,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x7d,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x82,0x00,0x00,0x00, +0x12,0x00,0x00,0x00,0x81,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x83,0x00,0x00,0x00,0x82,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0x47,0x00,0x00,0x00,0x83,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x15,0x00,0x00,0x00,0x87,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x88,0x00,0x00,0x00,0x87,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x8a,0x00,0x00,0x00,0x47,0x00,0x00,0x00, +0x39,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x8d,0x00,0x00,0x00,0x8a,0x00,0x00,0x00,0x83,0x00,0x00,0x00, +0x0c,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x8e,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x26,0x00,0x00,0x00,0x88,0x00,0x00,0x00, +0x8d,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, +0x92,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x91,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x93,0x00,0x00,0x00, +0x92,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x94,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x93,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x96,0x00,0x00,0x00, +0x42,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x15,0x00,0x00,0x00,0x98,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0x97,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x99,0x00,0x00,0x00,0x98,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x9a,0x00,0x00,0x00,0x96,0x00,0x00,0x00, +0x99,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x9b,0x00,0x00,0x00,0x94,0x00,0x00,0x00,0x9a,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9d,0x00,0x00,0x00, +0x9b,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x9e,0x00,0x00,0x00,0x9d,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, +0xa3,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0xa2,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xa4,0x00,0x00,0x00, +0xa3,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xa5,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0xa4,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa8,0x00,0x00,0x00, +0x4a,0x00,0x00,0x00,0xa7,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x15,0x00,0x00,0x00,0xaa,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0xa9,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xab,0x00,0x00,0x00,0xaa,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xac,0x00,0x00,0x00,0xa8,0x00,0x00,0x00, +0xab,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xad,0x00,0x00,0x00,0xa5,0x00,0x00,0x00,0xac,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xaf,0x00,0x00,0x00, +0xad,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xb0,0x00,0x00,0x00,0xaf,0x00,0x00,0x00, +0x77,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xb2,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xb2,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xbd,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, +0x05,0x00,0x00,0x00,0xd1,0x00,0x00,0x00,0xb3,0x00,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0xc3,0x00,0x00,0x00, +0xbd,0x02,0x00,0x00,0xc1,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0xb4,0x00,0x00,0x00,0xb3,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0xc3,0x00,0x00,0x00,0xb3,0x00,0x00,0x00, +0xb4,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xb3,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0xcd,0x00,0x00,0x00,0xce,0x00,0x00,0x00, +0xca,0x00,0x00,0x00,0xbd,0x02,0x00,0x00,0x3e,0x00,0x03,0x00, +0xce,0x00,0x00,0x00,0xcc,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xd1,0x00,0x00,0x00,0xbd,0x02,0x00,0x00, +0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xb2,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xb4,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xd4,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xd4,0x00,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xd6,0x02,0x00,0x00, +0xb0,0x00,0x00,0x00,0xb4,0x00,0x00,0x00,0x8b,0x01,0x00,0x00, +0xd7,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xd2,0x02,0x00,0x00,0x9e,0x00,0x00,0x00,0xb4,0x00,0x00,0x00, +0x88,0x01,0x00,0x00,0xd7,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xbe,0x02,0x00,0x00,0x84,0x00,0x00,0x00, +0xb4,0x00,0x00,0x00,0x36,0x02,0x00,0x00,0xd7,0x00,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0xdb,0x00,0x00,0x00, +0xbe,0x02,0x00,0x00,0x8e,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0xd6,0x00,0x00,0x00,0xd7,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0xdb,0x00,0x00,0x00,0xd5,0x00,0x00,0x00, +0xd6,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xd5,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xdd,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xdd,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xce,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0xd5,0x00,0x00,0x00, +0x3e,0x01,0x00,0x00,0xde,0x00,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc2,0x00,0x00,0x00,0xe3,0x00,0x00,0x00,0xce,0x02,0x00,0x00, +0x37,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xdf,0x00,0x00,0x00, +0xde,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0xe3,0x00,0x00,0x00,0xde,0x00,0x00,0x00,0xdf,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xde,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xe8,0x00,0x00,0x00,0x73,0x00,0x00,0x00, +0xce,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xeb,0x00,0x00,0x00,0xe8,0x00,0x00,0x00,0x99,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xec,0x00,0x00,0x00, +0xeb,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xed,0x00,0x00,0x00,0xd2,0x02,0x00,0x00, +0xec,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xef,0x00,0x00,0x00,0xed,0x00,0x00,0x00,0x6e,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xf5,0x00,0x00,0x00, +0xe8,0x00,0x00,0x00,0xf4,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xf7,0x00,0x00,0x00,0xf5,0x00,0x00,0x00, +0x6e,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xfb,0x00,0x00,0x00,0xef,0x00,0x00,0x00,0xfa,0x00,0x00,0x00, +0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xff,0x00,0x00,0x00, +0xef,0x00,0x00,0x00,0xfe,0x00,0x00,0x00,0x41,0x00,0x07,0x00, +0x0a,0x01,0x00,0x00,0x0b,0x01,0x00,0x00,0x08,0x01,0x00,0x00, +0x34,0x00,0x00,0x00,0xfb,0x00,0x00,0x00,0x34,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x01,0x01,0x00,0x00,0x0c,0x01,0x00,0x00, +0x0b,0x01,0x00,0x00,0x73,0x00,0x04,0x00,0xc4,0x00,0x00,0x00, +0x0d,0x01,0x00,0x00,0x0c,0x01,0x00,0x00,0x41,0x00,0x07,0x00, +0x0a,0x01,0x00,0x00,0x10,0x01,0x00,0x00,0x08,0x01,0x00,0x00, +0x34,0x00,0x00,0x00,0xfb,0x00,0x00,0x00,0xd0,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x01,0x01,0x00,0x00,0x11,0x01,0x00,0x00, +0x10,0x01,0x00,0x00,0x73,0x00,0x04,0x00,0xc4,0x00,0x00,0x00, +0x12,0x01,0x00,0x00,0x11,0x01,0x00,0x00,0x41,0x00,0x08,0x00, +0x16,0x01,0x00,0x00,0x17,0x01,0x00,0x00,0x08,0x01,0x00,0x00, +0x34,0x00,0x00,0x00,0xfb,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0xff,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x02,0x01,0x00,0x00, +0x18,0x01,0x00,0x00,0x17,0x01,0x00,0x00,0x71,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x19,0x01,0x00,0x00,0x18,0x01,0x00,0x00, +0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x1e,0x01,0x00,0x00, +0x19,0x01,0x00,0x00,0xfe,0x00,0x00,0x00,0x70,0x00,0x04,0x00, +0xc4,0x00,0x00,0x00,0x1f,0x01,0x00,0x00,0x1e,0x01,0x00,0x00, +0xc2,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x21,0x01,0x00,0x00, +0x19,0x01,0x00,0x00,0xa9,0x00,0x00,0x00,0x70,0x00,0x04,0x00, +0xc4,0x00,0x00,0x00,0x22,0x01,0x00,0x00,0x21,0x01,0x00,0x00, +0x50,0x00,0x05,0x00,0x1a,0x01,0x00,0x00,0x23,0x01,0x00,0x00, +0x1f,0x01,0x00,0x00,0x22,0x01,0x00,0x00,0x8e,0x00,0x05,0x00, +0x1a,0x01,0x00,0x00,0x25,0x01,0x00,0x00,0x23,0x01,0x00,0x00, +0x0d,0x01,0x00,0x00,0x50,0x00,0x05,0x00,0x1a,0x01,0x00,0x00, +0x27,0x01,0x00,0x00,0x12,0x01,0x00,0x00,0x12,0x01,0x00,0x00, +0x81,0x00,0x05,0x00,0x1a,0x01,0x00,0x00,0x28,0x01,0x00,0x00, +0x25,0x01,0x00,0x00,0x27,0x01,0x00,0x00,0x51,0x00,0x05,0x00, +0xc4,0x00,0x00,0x00,0x30,0x01,0x00,0x00,0x28,0x01,0x00,0x00, +0x00,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x31,0x01,0x00,0x00, +0x32,0x01,0x00,0x00,0x2d,0x01,0x00,0x00,0xf7,0x00,0x00,0x00, +0x3e,0x00,0x03,0x00,0x32,0x01,0x00,0x00,0x30,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x34,0x01,0x00,0x00, +0xf7,0x00,0x00,0x00,0xfa,0x00,0x00,0x00,0x51,0x00,0x05,0x00, +0xc4,0x00,0x00,0x00,0x36,0x01,0x00,0x00,0x28,0x01,0x00,0x00, +0x01,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x31,0x01,0x00,0x00, +0x37,0x01,0x00,0x00,0x2d,0x01,0x00,0x00,0x34,0x01,0x00,0x00, +0x3e,0x00,0x03,0x00,0x37,0x01,0x00,0x00,0x36,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3e,0x01,0x00,0x00, +0xce,0x02,0x00,0x00,0x3c,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0xdd,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xdf,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x40,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x40,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xcf,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0xdf,0x00,0x00,0x00, +0x84,0x01,0x00,0x00,0x41,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc2,0x00,0x00,0x00,0x46,0x01,0x00,0x00,0xcf,0x02,0x00,0x00, +0xa7,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x42,0x01,0x00,0x00, +0x41,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x46,0x01,0x00,0x00,0x41,0x01,0x00,0x00,0x42,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x41,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x4b,0x01,0x00,0x00,0x7e,0x00,0x00,0x00, +0xcf,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x4e,0x01,0x00,0x00,0x4b,0x01,0x00,0x00,0xab,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x4f,0x01,0x00,0x00, +0x4e,0x01,0x00,0x00,0x77,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x50,0x01,0x00,0x00,0xd6,0x02,0x00,0x00, +0x4f,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x52,0x01,0x00,0x00,0x50,0x01,0x00,0x00,0x79,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x58,0x01,0x00,0x00, +0x4b,0x01,0x00,0x00,0x57,0x01,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x5a,0x01,0x00,0x00,0x79,0x00,0x00,0x00, +0x77,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x5b,0x01,0x00,0x00,0x58,0x01,0x00,0x00,0x5a,0x01,0x00,0x00, +0x41,0x00,0x07,0x00,0x69,0x01,0x00,0x00,0x6a,0x01,0x00,0x00, +0x67,0x01,0x00,0x00,0x34,0x00,0x00,0x00,0x52,0x01,0x00,0x00, +0x3e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00, +0x6b,0x01,0x00,0x00,0x6a,0x01,0x00,0x00,0x41,0x00,0x05,0x00, +0x31,0x01,0x00,0x00,0x6c,0x01,0x00,0x00,0x60,0x01,0x00,0x00, +0x5b,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x6c,0x01,0x00,0x00, +0x6b,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x6e,0x01,0x00,0x00,0x5b,0x01,0x00,0x00,0x39,0x00,0x00,0x00, +0x41,0x00,0x07,0x00,0x69,0x01,0x00,0x00,0x70,0x01,0x00,0x00, +0x67,0x01,0x00,0x00,0x34,0x00,0x00,0x00,0x52,0x01,0x00,0x00, +0x39,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00, +0x71,0x01,0x00,0x00,0x70,0x01,0x00,0x00,0x41,0x00,0x05,0x00, +0x31,0x01,0x00,0x00,0x72,0x01,0x00,0x00,0x60,0x01,0x00,0x00, +0x6e,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x72,0x01,0x00,0x00, +0x71,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x74,0x01,0x00,0x00,0x5b,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, +0x41,0x00,0x07,0x00,0x69,0x01,0x00,0x00,0x76,0x01,0x00,0x00, +0x67,0x01,0x00,0x00,0x34,0x00,0x00,0x00,0x52,0x01,0x00,0x00, +0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00, +0x77,0x01,0x00,0x00,0x76,0x01,0x00,0x00,0x41,0x00,0x05,0x00, +0x31,0x01,0x00,0x00,0x78,0x01,0x00,0x00,0x60,0x01,0x00,0x00, +0x74,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x78,0x01,0x00,0x00, +0x77,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x7b,0x01,0x00,0x00,0x5b,0x01,0x00,0x00,0x7a,0x01,0x00,0x00, +0x41,0x00,0x07,0x00,0x69,0x01,0x00,0x00,0x7d,0x01,0x00,0x00, +0x67,0x01,0x00,0x00,0x34,0x00,0x00,0x00,0x52,0x01,0x00,0x00, +0x7a,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00, +0x7e,0x01,0x00,0x00,0x7d,0x01,0x00,0x00,0x41,0x00,0x05,0x00, +0x31,0x01,0x00,0x00,0x7f,0x01,0x00,0x00,0x60,0x01,0x00,0x00, +0x7b,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x7f,0x01,0x00,0x00, +0x7e,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x84,0x01,0x00,0x00,0xcf,0x02,0x00,0x00,0x82,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0x40,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x42,0x01,0x00,0x00,0xe0,0x00,0x04,0x00,0x0c,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x85,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x88,0x01,0x00,0x00,0xd2,0x02,0x00,0x00, +0x86,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x8b,0x01,0x00,0x00,0xd6,0x02,0x00,0x00,0x89,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0x8d,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x8d,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xd8,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0x42,0x01,0x00,0x00, +0x34,0x02,0x00,0x00,0x90,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc2,0x00,0x00,0x00,0x93,0x01,0x00,0x00,0xd8,0x02,0x00,0x00, +0x6c,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x8f,0x01,0x00,0x00, +0x90,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x93,0x01,0x00,0x00,0x8e,0x01,0x00,0x00,0x8f,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x8e,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0x95,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x95,0x01,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xdc,0x02,0x00,0x00, +0x3e,0x00,0x00,0x00,0x8e,0x01,0x00,0x00,0xc0,0x01,0x00,0x00, +0x98,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, +0x9b,0x01,0x00,0x00,0xdc,0x02,0x00,0x00,0x60,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0x97,0x01,0x00,0x00,0x98,0x01,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x9b,0x01,0x00,0x00, +0x96,0x01,0x00,0x00,0x97,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x96,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x9d,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x9d,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xee,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, +0x96,0x01,0x00,0x00,0xbe,0x01,0x00,0x00,0x9e,0x01,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0xa3,0x01,0x00,0x00, +0xee,0x02,0x00,0x00,0x62,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0x9f,0x01,0x00,0x00,0x9e,0x01,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0xa3,0x01,0x00,0x00,0x9e,0x01,0x00,0x00, +0x9f,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x9e,0x01,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa9,0x01,0x00,0x00, +0xdc,0x02,0x00,0x00,0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xab,0x01,0x00,0x00,0xa9,0x01,0x00,0x00, +0xee,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xad,0x01,0x00,0x00,0x55,0x00,0x00,0x00,0x53,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xaf,0x01,0x00,0x00, +0xdc,0x02,0x00,0x00,0x61,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xb0,0x01,0x00,0x00,0xad,0x01,0x00,0x00, +0xaf,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xb2,0x01,0x00,0x00,0x64,0x00,0x00,0x00,0x62,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xb3,0x01,0x00,0x00, +0xb0,0x01,0x00,0x00,0xb2,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xb5,0x01,0x00,0x00,0xb3,0x01,0x00,0x00, +0xee,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xb7,0x01,0x00,0x00,0xb5,0x01,0x00,0x00,0xb6,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xb9,0x01,0x00,0x00, +0xb7,0x01,0x00,0x00,0xd8,0x02,0x00,0x00,0x41,0x00,0x05,0x00, +0x31,0x01,0x00,0x00,0xba,0x01,0x00,0x00,0x2d,0x01,0x00,0x00, +0xb9,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00, +0xbb,0x01,0x00,0x00,0xba,0x01,0x00,0x00,0x41,0x00,0x05,0x00, +0xcd,0x00,0x00,0x00,0xbc,0x01,0x00,0x00,0xa7,0x01,0x00,0x00, +0xab,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0xbc,0x01,0x00,0x00, +0xbb,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xbe,0x01,0x00,0x00,0xee,0x02,0x00,0x00,0xd0,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x9d,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x9f,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x98,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x98,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xc0,0x01,0x00,0x00,0xdc,0x02,0x00,0x00, +0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x95,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x97,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0xc2,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xc2,0x01,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xdd,0x02,0x00,0x00, +0x3e,0x00,0x00,0x00,0x97,0x01,0x00,0x00,0xee,0x01,0x00,0x00, +0xc5,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, +0xc8,0x01,0x00,0x00,0xdd,0x02,0x00,0x00,0xbf,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0xc4,0x01,0x00,0x00,0xc5,0x01,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xc8,0x01,0x00,0x00, +0xc3,0x01,0x00,0x00,0xc4,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xc3,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xca,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xca,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xeb,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, +0xc3,0x01,0x00,0x00,0xec,0x01,0x00,0x00,0xcb,0x01,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0xd0,0x01,0x00,0x00, +0xeb,0x02,0x00,0x00,0xbc,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0xcc,0x01,0x00,0x00,0xcb,0x01,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0xd0,0x01,0x00,0x00,0xcb,0x01,0x00,0x00, +0xcc,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xcb,0x01,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xd6,0x01,0x00,0x00, +0xdd,0x02,0x00,0x00,0xbc,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xd8,0x01,0x00,0x00,0xd6,0x01,0x00,0x00, +0xeb,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xda,0x01,0x00,0x00,0x59,0x00,0x00,0x00,0xb9,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xdd,0x01,0x00,0x00, +0xdd,0x02,0x00,0x00,0xdc,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xde,0x01,0x00,0x00,0xda,0x01,0x00,0x00, +0xdd,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xe0,0x01,0x00,0x00,0x68,0x00,0x00,0x00,0xbc,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xe1,0x01,0x00,0x00, +0xde,0x01,0x00,0x00,0xe0,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xe3,0x01,0x00,0x00,0xe1,0x01,0x00,0x00, +0xeb,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xe5,0x01,0x00,0x00,0xe3,0x01,0x00,0x00,0xe4,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xe7,0x01,0x00,0x00, +0xe5,0x01,0x00,0x00,0xd8,0x02,0x00,0x00,0x41,0x00,0x05,0x00, +0x31,0x01,0x00,0x00,0xe8,0x01,0x00,0x00,0x60,0x01,0x00,0x00, +0xe7,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00, +0xe9,0x01,0x00,0x00,0xe8,0x01,0x00,0x00,0x41,0x00,0x05,0x00, +0xcd,0x00,0x00,0x00,0xea,0x01,0x00,0x00,0xd4,0x01,0x00,0x00, +0xd8,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0xea,0x01,0x00,0x00, +0xe9,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xec,0x01,0x00,0x00,0xeb,0x02,0x00,0x00,0xd0,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xca,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xcc,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xc5,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xc5,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xee,0x01,0x00,0x00,0xdd,0x02,0x00,0x00, +0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xc2,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xc4,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0xf0,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xf0,0x01,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xde,0x02,0x00,0x00, +0x3e,0x00,0x00,0x00,0xc4,0x01,0x00,0x00,0x32,0x02,0x00,0x00, +0xf3,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, +0xf6,0x01,0x00,0x00,0xde,0x02,0x00,0x00,0xbf,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0xf2,0x01,0x00,0x00,0xf3,0x01,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xf6,0x01,0x00,0x00, +0xf1,0x01,0x00,0x00,0xf2,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xf1,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xf8,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xf8,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xe2,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, +0xf1,0x01,0x00,0x00,0x30,0x02,0x00,0x00,0xfb,0x01,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0xfe,0x01,0x00,0x00, +0xe2,0x02,0x00,0x00,0x60,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0xfa,0x01,0x00,0x00,0xfb,0x01,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0xfe,0x01,0x00,0x00,0xf9,0x01,0x00,0x00, +0xfa,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xf9,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0x00,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x00,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xe4,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0xf9,0x01,0x00,0x00, +0x2e,0x02,0x00,0x00,0x03,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc2,0x00,0x00,0x00,0x06,0x02,0x00,0x00,0xe4,0x02,0x00,0x00, +0xbc,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x02,0x02,0x00,0x00, +0x03,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x06,0x02,0x00,0x00,0x01,0x02,0x00,0x00,0x02,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x01,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0x08,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x08,0x02,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xe6,0x02,0x00,0x00, +0x3e,0x00,0x00,0x00,0x01,0x02,0x00,0x00,0x2c,0x02,0x00,0x00, +0x09,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, +0x0e,0x02,0x00,0x00,0xe6,0x02,0x00,0x00,0x62,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0x0a,0x02,0x00,0x00,0x09,0x02,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x0e,0x02,0x00,0x00, +0x09,0x02,0x00,0x00,0x0a,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x09,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x10,0x02,0x00,0x00,0xde,0x02,0x00,0x00,0xbc,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x12,0x02,0x00,0x00, +0x10,0x02,0x00,0x00,0xe4,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x14,0x02,0x00,0x00,0x12,0x02,0x00,0x00, +0x13,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x16,0x02,0x00,0x00,0xe2,0x02,0x00,0x00,0x62,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x17,0x02,0x00,0x00, +0x14,0x02,0x00,0x00,0x16,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x19,0x02,0x00,0x00,0x17,0x02,0x00,0x00, +0xe6,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x1d,0x02,0x00,0x00,0x16,0x02,0x00,0x00,0xe6,0x02,0x00,0x00, +0x41,0x00,0x05,0x00,0xcd,0x00,0x00,0x00,0x1e,0x02,0x00,0x00, +0xa7,0x01,0x00,0x00,0x1d,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, +0xc4,0x00,0x00,0x00,0x1f,0x02,0x00,0x00,0x1e,0x02,0x00,0x00, +0x41,0x00,0x05,0x00,0xcd,0x00,0x00,0x00,0x24,0x02,0x00,0x00, +0xd4,0x01,0x00,0x00,0x12,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, +0xc4,0x00,0x00,0x00,0x25,0x02,0x00,0x00,0x24,0x02,0x00,0x00, +0x41,0x00,0x05,0x00,0xcd,0x00,0x00,0x00,0x27,0x02,0x00,0x00, +0xca,0x00,0x00,0x00,0x19,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, +0xc4,0x00,0x00,0x00,0x28,0x02,0x00,0x00,0x27,0x02,0x00,0x00, +0x0c,0x00,0x08,0x00,0xc4,0x00,0x00,0x00,0x29,0x02,0x00,0x00, +0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x1f,0x02,0x00,0x00, +0x25,0x02,0x00,0x00,0x28,0x02,0x00,0x00,0x3e,0x00,0x03,0x00, +0x27,0x02,0x00,0x00,0x29,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x2c,0x02,0x00,0x00,0xe6,0x02,0x00,0x00, +0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x08,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x0a,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0x03,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x03,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x2e,0x02,0x00,0x00, +0xe4,0x02,0x00,0x00,0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x00,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x02,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0xfb,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xfb,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x30,0x02,0x00,0x00,0xe2,0x02,0x00,0x00,0xd0,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xf8,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xfa,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xf3,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xf3,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x32,0x02,0x00,0x00,0xde,0x02,0x00,0x00, +0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xf0,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xf2,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0x90,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x90,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x34,0x02,0x00,0x00, +0xd8,0x02,0x00,0x00,0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x8d,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x8f,0x01,0x00,0x00, +0xe0,0x00,0x04,0x00,0x0c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x85,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xd7,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xd7,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x36,0x02,0x00,0x00,0xbe,0x02,0x00,0x00, +0x6c,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xd4,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xd6,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x3b,0x02,0x00,0x00,0x55,0x00,0x00,0x00, +0x53,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x3c,0x02,0x00,0x00,0x96,0x00,0x00,0x00,0x3b,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x41,0x02,0x00,0x00, +0x59,0x00,0x00,0x00,0xb9,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x42,0x02,0x00,0x00,0xa8,0x00,0x00,0x00, +0x41,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, +0x47,0x02,0x00,0x00,0x12,0x00,0x00,0x00,0x46,0x02,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x48,0x02,0x00,0x00, +0x47,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x49,0x02,0x00,0x00,0x0f,0x00,0x00,0x00,0x48,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x4d,0x02,0x00,0x00, +0x47,0x00,0x00,0x00,0x48,0x02,0x00,0x00,0x41,0x00,0x05,0x00, +0x0d,0x00,0x00,0x00,0x4f,0x02,0x00,0x00,0x4e,0x02,0x00,0x00, +0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x50,0x02,0x00,0x00,0x4f,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x51,0x02,0x00,0x00,0x4d,0x02,0x00,0x00, +0x50,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x52,0x02,0x00,0x00,0x49,0x02,0x00,0x00,0x51,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0x54,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x54,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xbf,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0xd6,0x00,0x00,0x00, +0xba,0x02,0x00,0x00,0x57,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc2,0x00,0x00,0x00,0x5a,0x02,0x00,0x00,0xbf,0x02,0x00,0x00, +0xbf,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x56,0x02,0x00,0x00, +0x57,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x5a,0x02,0x00,0x00,0x55,0x02,0x00,0x00,0x56,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x55,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0x5c,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x5c,0x02,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xc0,0x02,0x00,0x00, +0x3e,0x00,0x00,0x00,0x55,0x02,0x00,0x00,0xb8,0x02,0x00,0x00, +0x5f,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, +0x62,0x02,0x00,0x00,0xc0,0x02,0x00,0x00,0x60,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0x5e,0x02,0x00,0x00,0x5f,0x02,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x62,0x02,0x00,0x00, +0x5d,0x02,0x00,0x00,0x5e,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x5d,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x66,0x02,0x00,0x00,0xc0,0x02,0x00,0x00,0x61,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x67,0x02,0x00,0x00, +0x3c,0x02,0x00,0x00,0x66,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x69,0x02,0x00,0x00,0x64,0x00,0x00,0x00, +0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x6a,0x02,0x00,0x00,0x67,0x02,0x00,0x00,0x69,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x6e,0x02,0x00,0x00, +0xbf,0x02,0x00,0x00,0xdc,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x6f,0x02,0x00,0x00,0x42,0x02,0x00,0x00, +0x6e,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x71,0x02,0x00,0x00,0x68,0x00,0x00,0x00,0xbc,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x72,0x02,0x00,0x00, +0x6f,0x02,0x00,0x00,0x71,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0x74,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x74,0x02,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xc2,0x02,0x00,0x00, +0x3e,0x00,0x00,0x00,0x5d,0x02,0x00,0x00,0xb6,0x02,0x00,0x00, +0x77,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, +0x7a,0x02,0x00,0x00,0xc2,0x02,0x00,0x00,0xbc,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0x76,0x02,0x00,0x00,0x77,0x02,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x7a,0x02,0x00,0x00, +0x75,0x02,0x00,0x00,0x76,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x75,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x7c,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x7c,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xc4,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, +0x75,0x02,0x00,0x00,0xb4,0x02,0x00,0x00,0x7f,0x02,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0x82,0x02,0x00,0x00, +0xc4,0x02,0x00,0x00,0x62,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0x7e,0x02,0x00,0x00,0x7f,0x02,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x82,0x02,0x00,0x00,0x7d,0x02,0x00,0x00, +0x7e,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x7d,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x85,0x02,0x00,0x00, +0x6a,0x02,0x00,0x00,0xc4,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc2,0x00,0x00,0x00,0x88,0x02,0x00,0x00,0x85,0x02,0x00,0x00, +0x36,0x00,0x00,0x00,0xf7,0x00,0x03,0x00,0x8a,0x02,0x00,0x00, +0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x88,0x02,0x00,0x00, +0x89,0x02,0x00,0x00,0x8a,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x89,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x8d,0x02,0x00,0x00,0x72,0x02,0x00,0x00,0xc2,0x02,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x8e,0x02,0x00,0x00, +0x12,0x00,0x00,0x00,0xd0,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x8f,0x02,0x00,0x00,0x8e,0x02,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0x90,0x02,0x00,0x00, +0x8d,0x02,0x00,0x00,0x8f,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0x8a,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x8a,0x02,0x00,0x00, +0xf5,0x00,0x07,0x00,0xc2,0x00,0x00,0x00,0x91,0x02,0x00,0x00, +0x88,0x02,0x00,0x00,0x7d,0x02,0x00,0x00,0x90,0x02,0x00,0x00, +0x89,0x02,0x00,0x00,0xf7,0x00,0x03,0x00,0x93,0x02,0x00,0x00, +0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x91,0x02,0x00,0x00, +0x92,0x02,0x00,0x00,0x93,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x92,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x9b,0x02,0x00,0x00,0x72,0x02,0x00,0x00,0xc2,0x02,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x9d,0x02,0x00,0x00, +0x12,0x00,0x00,0x00,0x9c,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x9e,0x02,0x00,0x00,0x9d,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9f,0x02,0x00,0x00, +0x9b,0x02,0x00,0x00,0x9e,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xa0,0x02,0x00,0x00,0x52,0x02,0x00,0x00, +0x9f,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xa2,0x02,0x00,0x00,0xa0,0x02,0x00,0x00,0x6a,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa4,0x02,0x00,0x00, +0xa2,0x02,0x00,0x00,0xc4,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xa6,0x02,0x00,0x00,0xbf,0x02,0x00,0x00, +0xbc,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xa8,0x02,0x00,0x00,0xa6,0x02,0x00,0x00,0xc2,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xaa,0x02,0x00,0x00, +0xa8,0x02,0x00,0x00,0xa9,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xac,0x02,0x00,0x00,0xc0,0x02,0x00,0x00, +0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xad,0x02,0x00,0x00,0xaa,0x02,0x00,0x00,0xac,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xaf,0x02,0x00,0x00, +0xad,0x02,0x00,0x00,0xc4,0x02,0x00,0x00,0x41,0x00,0x05,0x00, +0xcd,0x00,0x00,0x00,0xb0,0x02,0x00,0x00,0xca,0x00,0x00,0x00, +0xaf,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00, +0xb1,0x02,0x00,0x00,0xb0,0x02,0x00,0x00,0x41,0x00,0x06,0x00, +0x69,0x01,0x00,0x00,0xb2,0x02,0x00,0x00,0x97,0x02,0x00,0x00, +0x34,0x00,0x00,0x00,0xa4,0x02,0x00,0x00,0x3e,0x00,0x03,0x00, +0xb2,0x02,0x00,0x00,0xb1,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0x93,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x93,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0x7f,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x7f,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xb4,0x02,0x00,0x00,0xc4,0x02,0x00,0x00,0xd0,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x7c,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x7e,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x77,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x77,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xb6,0x02,0x00,0x00,0xc2,0x02,0x00,0x00, +0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x74,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x76,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0x5f,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x5f,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xb8,0x02,0x00,0x00, +0xc0,0x02,0x00,0x00,0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x5c,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x5e,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0x57,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x57,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xba,0x02,0x00,0x00,0xbf,0x02,0x00,0x00,0xd0,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x54,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x56,0x02,0x00,0x00,0xfd,0x00,0x01,0x00,0x38,0x00,0x01,0x00, + +}; +const uint64_t matmul_q4_1_f32_aligned_fp32_len = 10476; + +unsigned char matmul_q4_1_f32_fp32_data[] = { +0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, +0xef,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, +0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x51,0x11,0x00,0x00, +0x11,0x00,0x02,0x00,0x60,0x11,0x00,0x00,0x0b,0x00,0x06,0x00, +0x01,0x00,0x00,0x00,0x47,0x4c,0x53,0x4c,0x2e,0x73,0x74,0x64, +0x2e,0x34,0x35,0x30,0x00,0x00,0x00,0x00,0x0e,0x00,0x03,0x00, +0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x0f,0x00,0x0f,0x00, +0x05,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x6d,0x61,0x69,0x6e, +0x00,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0x3d,0x00,0x00,0x00,0x4c,0x00,0x00,0x00,0x07,0x01,0x00,0x00, +0x2c,0x01,0x00,0x00,0x5d,0x01,0x00,0x00,0x68,0x01,0x00,0x00, +0x4e,0x02,0x00,0x00,0x97,0x02,0x00,0x00,0x10,0x00,0x06,0x00, +0x04,0x00,0x00,0x00,0x11,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x0b,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x1c,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x10,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x08,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x03,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x10,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x10,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, +0x05,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x14,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x10,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x1c,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, +0x08,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x09,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x24,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x10,0x00,0x00,0x00,0x0a,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x28,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, +0x0b,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x2c,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x30,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x10,0x00,0x00,0x00,0x0d,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x34,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x10,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x37,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x3d,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x1a,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x4c,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, +0x1b,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x4f,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x53,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x60,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x62,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x6c,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x03,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0xa6,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xb8,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0xbb,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x08,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x02,0x01,0x00,0x00,0x06,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x03,0x01,0x00,0x00, +0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x03,0x01,0x00,0x00,0x01,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x03,0x01,0x00,0x00,0x02,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x04,0x01,0x00,0x00, +0x06,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x48,0x00,0x04,0x00, +0x05,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x05,0x01,0x00,0x00,0x00,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, +0x05,0x01,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x07,0x01,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x07,0x01,0x00,0x00,0x21,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x37,0x01,0x00,0x00, +0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x38,0x01,0x00,0x00,0x0b,0x00,0x00,0x00,0x19,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x65,0x01,0x00,0x00,0x06,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0x66,0x01,0x00,0x00, +0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x66,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x66,0x01,0x00,0x00, +0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x68,0x01,0x00,0x00, +0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x68,0x01,0x00,0x00,0x21,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x4e,0x02,0x00,0x00,0x0b,0x00,0x00,0x00, +0x18,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x94,0x02,0x00,0x00, +0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00, +0x95,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x19,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x95,0x02,0x00,0x00,0x00,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, +0x95,0x02,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x97,0x02,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x97,0x02,0x00,0x00,0x21,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x13,0x00,0x02,0x00,0x02,0x00,0x00,0x00, +0x21,0x00,0x03,0x00,0x03,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x15,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x17,0x00,0x04,0x00,0x09,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x0a,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x09,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x0d,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x1e,0x00,0x10,0x00,0x10,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x11,0x00,0x00,0x00, +0x09,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0x11,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x09,0x00,0x00,0x00, +0x15,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, +0x14,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x15,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x21,0x00,0x00,0x00, +0x0a,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, +0x27,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0x2d,0x00,0x00,0x00,0x07,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x34,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x37,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00,0x3d,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x3e,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0x0a,0x00,0x00,0x00,0x4c,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x4f,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x53,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x54,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0x37,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x58,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0x37,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x32,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x61,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x60,0x00,0x00,0x00, +0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x62,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x63,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x61,0x00,0x00,0x00, +0x62,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x67,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x61,0x00,0x00,0x00, +0x62,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x72,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x77,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x7c,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0x80,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x85,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, +0x90,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0x96,0x00,0x00,0x00,0x03,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0xa1,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xa6,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0xa8,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xb7,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00, +0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xb8,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xb9,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x53,0x00,0x00,0x00, +0xb8,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xba,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x4f,0x00,0x00,0x00, +0x62,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xbb,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xbc,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0xba,0x00,0x00,0x00,0xbb,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xbd,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0xbc,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xbe,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0xb9,0x00,0x00,0x00,0xbd,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xbf,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0xb7,0x00,0x00,0x00,0xbe,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xc0,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0xbf,0x00,0x00,0x00,0xbb,0x00,0x00,0x00,0x14,0x00,0x02,0x00, +0xc1,0x00,0x00,0x00,0x16,0x00,0x03,0x00,0xc3,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xc4,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00, +0x62,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xc5,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0xc4,0x00,0x00,0x00, +0xbe,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xc6,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0xc5,0x00,0x00,0x00, +0xbb,0x00,0x00,0x00,0x1c,0x00,0x04,0x00,0xc7,0x00,0x00,0x00, +0xc3,0x00,0x00,0x00,0xc6,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0xc8,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0xc7,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0xc3,0x00,0x00,0x00,0xcb,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0xcc,0x00,0x00,0x00, +0x07,0x00,0x00,0x00,0xc3,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0xcf,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xf3,0x00,0x00,0x00, +0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xf9,0x00,0x00,0x00, +0x10,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xfd,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x16,0x00,0x03,0x00, +0x00,0x01,0x00,0x00,0x10,0x00,0x00,0x00,0x15,0x00,0x04,0x00, +0x01,0x01,0x00,0x00,0x08,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x1c,0x00,0x04,0x00,0x02,0x01,0x00,0x00,0x01,0x01,0x00,0x00, +0xf9,0x00,0x00,0x00,0x1e,0x00,0x05,0x00,0x03,0x01,0x00,0x00, +0x00,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x02,0x01,0x00,0x00, +0x1d,0x00,0x03,0x00,0x04,0x01,0x00,0x00,0x03,0x01,0x00,0x00, +0x1e,0x00,0x03,0x00,0x05,0x01,0x00,0x00,0x04,0x01,0x00,0x00, +0x20,0x00,0x04,0x00,0x06,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, +0x05,0x01,0x00,0x00,0x3b,0x00,0x04,0x00,0x06,0x01,0x00,0x00, +0x07,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x09,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,0x01,0x00,0x00, +0x20,0x00,0x04,0x00,0x15,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, +0x01,0x01,0x00,0x00,0x17,0x00,0x04,0x00,0x19,0x01,0x00,0x00, +0xc3,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x28,0x01,0x00,0x00,0x80,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x29,0x01,0x00,0x00,0x84,0x00,0x00,0x00, +0x37,0x00,0x00,0x00,0x28,0x01,0x00,0x00,0x1c,0x00,0x04,0x00, +0x2a,0x01,0x00,0x00,0xc3,0x00,0x00,0x00,0x29,0x01,0x00,0x00, +0x20,0x00,0x04,0x00,0x2b,0x01,0x00,0x00,0x04,0x00,0x00,0x00, +0x2a,0x01,0x00,0x00,0x3b,0x00,0x04,0x00,0x2b,0x01,0x00,0x00, +0x2c,0x01,0x00,0x00,0x04,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x30,0x01,0x00,0x00,0x04,0x00,0x00,0x00,0xc3,0x00,0x00,0x00, +0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x37,0x01,0x00,0x00, +0x01,0x00,0x00,0x00,0x33,0x00,0x06,0x00,0x09,0x00,0x00,0x00, +0x38,0x01,0x00,0x00,0x37,0x01,0x00,0x00,0x39,0x00,0x00,0x00, +0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x39,0x01,0x00,0x00,0x51,0x00,0x00,0x00,0x38,0x01,0x00,0x00, +0x00,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x3a,0x01,0x00,0x00,0x84,0x00,0x00,0x00,0x39,0x01,0x00,0x00, +0x0c,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x3b,0x01,0x00,0x00,0x86,0x00,0x00,0x00,0x3a,0x01,0x00,0x00, +0x6c,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x59,0x01,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, +0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x5a,0x01,0x00,0x00,0x84,0x00,0x00,0x00,0xa6,0x00,0x00,0x00, +0x59,0x01,0x00,0x00,0x1c,0x00,0x04,0x00,0x5b,0x01,0x00,0x00, +0xc3,0x00,0x00,0x00,0x5a,0x01,0x00,0x00,0x20,0x00,0x04,0x00, +0x5c,0x01,0x00,0x00,0x04,0x00,0x00,0x00,0x5b,0x01,0x00,0x00, +0x3b,0x00,0x04,0x00,0x5c,0x01,0x00,0x00,0x5d,0x01,0x00,0x00, +0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x61,0x01,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, +0x39,0x00,0x00,0x00,0x1d,0x00,0x03,0x00,0x65,0x01,0x00,0x00, +0xc3,0x00,0x00,0x00,0x1e,0x00,0x03,0x00,0x66,0x01,0x00,0x00, +0x65,0x01,0x00,0x00,0x20,0x00,0x04,0x00,0x67,0x01,0x00,0x00, +0x0c,0x00,0x00,0x00,0x66,0x01,0x00,0x00,0x3b,0x00,0x04,0x00, +0x67,0x01,0x00,0x00,0x68,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x73,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, +0xc3,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x7b,0x01,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, +0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x80,0x01,0x00,0x00,0x51,0x00,0x00,0x00,0x38,0x01,0x00,0x00, +0x00,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x81,0x01,0x00,0x00,0x84,0x00,0x00,0x00,0x80,0x01,0x00,0x00, +0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x82,0x01,0x00,0x00,0x86,0x00,0x00,0x00,0x81,0x01,0x00,0x00, +0x6c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x85,0x01,0x00,0x00,0x08,0x01,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x86,0x01,0x00,0x00,0x86,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x89,0x01,0x00,0x00,0x86,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xa4,0x01,0x00,0x00,0x84,0x00,0x00,0x00, +0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x1c,0x00,0x04,0x00, +0xa5,0x01,0x00,0x00,0xc3,0x00,0x00,0x00,0xa4,0x01,0x00,0x00, +0x20,0x00,0x04,0x00,0xa6,0x01,0x00,0x00,0x07,0x00,0x00,0x00, +0xa5,0x01,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xb6,0x01,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, +0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xd1,0x01,0x00,0x00,0x84,0x00,0x00,0x00,0xbe,0x00,0x00,0x00, +0xbb,0x00,0x00,0x00,0x1c,0x00,0x04,0x00,0xd2,0x01,0x00,0x00, +0xc3,0x00,0x00,0x00,0xd1,0x01,0x00,0x00,0x20,0x00,0x04,0x00, +0xd3,0x01,0x00,0x00,0x07,0x00,0x00,0x00,0xd2,0x01,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xdc,0x01,0x00,0x00, +0x86,0x00,0x00,0x00,0xb8,0x00,0x00,0x00,0xbe,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xe4,0x01,0x00,0x00, +0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x13,0x02,0x00,0x00, +0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x46,0x02,0x00,0x00, +0x0d,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00, +0x4e,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, +0x94,0x02,0x00,0x00,0xc3,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, +0x95,0x02,0x00,0x00,0x94,0x02,0x00,0x00,0x20,0x00,0x04,0x00, +0x96,0x02,0x00,0x00,0x0c,0x00,0x00,0x00,0x95,0x02,0x00,0x00, +0x3b,0x00,0x04,0x00,0x96,0x02,0x00,0x00,0x97,0x02,0x00,0x00, +0x0c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, +0x9c,0x02,0x00,0x00,0x05,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xa9,0x02,0x00,0x00,0x84,0x00,0x00,0x00, +0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x36,0x00,0x05,0x00, +0x02,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x03,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x05,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0xc8,0x00,0x00,0x00,0xc9,0x00,0x00,0x00, +0x07,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0xa6,0x01,0x00,0x00, +0xa7,0x01,0x00,0x00,0x07,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0xd3,0x01,0x00,0x00,0xd4,0x01,0x00,0x00,0x07,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00,0x0e,0x00,0x00,0x00, +0x0b,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x0e,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x16,0x00,0x00,0x00, +0x12,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x17,0x00,0x00,0x00,0x16,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x18,0x00,0x00,0x00, +0x0f,0x00,0x00,0x00,0x17,0x00,0x00,0x00,0x89,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x1e,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, +0x17,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, +0x22,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x21,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x22,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x24,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x28,0x00,0x00,0x00, +0x12,0x00,0x00,0x00,0x27,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x29,0x00,0x00,0x00,0x28,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x2a,0x00,0x00,0x00, +0x1e,0x00,0x00,0x00,0x29,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x15,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0x2d,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x2f,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x30,0x00,0x00,0x00,0x24,0x00,0x00,0x00, +0x2f,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x32,0x00,0x00,0x00,0x30,0x00,0x00,0x00,0x2a,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x35,0x00,0x00,0x00, +0x12,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x36,0x00,0x00,0x00,0x35,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x38,0x00,0x00,0x00, +0x36,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x82,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x38,0x00,0x00,0x00, +0x39,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x3b,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x37,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00,0x3f,0x00,0x00,0x00, +0x3d,0x00,0x00,0x00,0x3e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x3f,0x00,0x00,0x00, +0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x42,0x00,0x00,0x00, +0x40,0x00,0x00,0x00,0x3b,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x47,0x00,0x00,0x00,0x40,0x00,0x00,0x00, +0x3b,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00, +0x49,0x00,0x00,0x00,0x3d,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x4a,0x00,0x00,0x00, +0x49,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00, +0x4d,0x00,0x00,0x00,0x4c,0x00,0x00,0x00,0x3e,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, +0x4d,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x50,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x4f,0x00,0x00,0x00, +0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x55,0x00,0x00,0x00, +0x50,0x00,0x00,0x00,0x54,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x59,0x00,0x00,0x00,0x50,0x00,0x00,0x00, +0x58,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x5d,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x4f,0x00,0x00,0x00, +0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x64,0x00,0x00,0x00, +0x5d,0x00,0x00,0x00,0x63,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x68,0x00,0x00,0x00,0x5d,0x00,0x00,0x00, +0x67,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x6e,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x73,0x00,0x00,0x00, +0x4e,0x00,0x00,0x00,0x72,0x00,0x00,0x00,0x89,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x78,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, +0x77,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x7d,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x7c,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x81,0x00,0x00,0x00, +0x12,0x00,0x00,0x00,0x80,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x82,0x00,0x00,0x00,0x81,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x83,0x00,0x00,0x00, +0x47,0x00,0x00,0x00,0x82,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x15,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0x85,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x87,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x89,0x00,0x00,0x00,0x47,0x00,0x00,0x00, +0x39,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x8c,0x00,0x00,0x00,0x89,0x00,0x00,0x00,0x82,0x00,0x00,0x00, +0x0c,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x8d,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x26,0x00,0x00,0x00,0x87,0x00,0x00,0x00, +0x8c,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, +0x91,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x90,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x92,0x00,0x00,0x00, +0x91,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x93,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x92,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x95,0x00,0x00,0x00, +0x42,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x15,0x00,0x00,0x00,0x97,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0x96,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x98,0x00,0x00,0x00,0x97,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x99,0x00,0x00,0x00,0x95,0x00,0x00,0x00, +0x98,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x9a,0x00,0x00,0x00,0x93,0x00,0x00,0x00,0x99,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9c,0x00,0x00,0x00, +0x9a,0x00,0x00,0x00,0x83,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x9d,0x00,0x00,0x00,0x9c,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, +0xa2,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0xa1,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xa3,0x00,0x00,0x00, +0xa2,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xa4,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0xa3,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa7,0x00,0x00,0x00, +0x4a,0x00,0x00,0x00,0xa6,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x15,0x00,0x00,0x00,0xa9,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0xa8,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xaa,0x00,0x00,0x00,0xa9,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xab,0x00,0x00,0x00,0xa7,0x00,0x00,0x00, +0xaa,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xac,0x00,0x00,0x00,0xa4,0x00,0x00,0x00,0xab,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xae,0x00,0x00,0x00, +0xac,0x00,0x00,0x00,0x83,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xaf,0x00,0x00,0x00,0xae,0x00,0x00,0x00, +0x39,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xb1,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xb1,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xbd,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, +0x05,0x00,0x00,0x00,0xd0,0x00,0x00,0x00,0xb2,0x00,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00,0xc2,0x00,0x00,0x00, +0xbd,0x02,0x00,0x00,0xc0,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0xb3,0x00,0x00,0x00,0xb2,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0xc2,0x00,0x00,0x00,0xb2,0x00,0x00,0x00, +0xb3,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xb2,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0xcc,0x00,0x00,0x00,0xcd,0x00,0x00,0x00, +0xc9,0x00,0x00,0x00,0xbd,0x02,0x00,0x00,0x3e,0x00,0x03,0x00, +0xcd,0x00,0x00,0x00,0xcb,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xd0,0x00,0x00,0x00,0xbd,0x02,0x00,0x00, +0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xb1,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xb3,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xd3,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xd3,0x00,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xd6,0x02,0x00,0x00, +0xaf,0x00,0x00,0x00,0xb3,0x00,0x00,0x00,0x8b,0x01,0x00,0x00, +0xd6,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xd2,0x02,0x00,0x00,0x9d,0x00,0x00,0x00,0xb3,0x00,0x00,0x00, +0x88,0x01,0x00,0x00,0xd6,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xbe,0x02,0x00,0x00,0x83,0x00,0x00,0x00, +0xb3,0x00,0x00,0x00,0x36,0x02,0x00,0x00,0xd6,0x00,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00,0xda,0x00,0x00,0x00, +0xbe,0x02,0x00,0x00,0x8d,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0xd5,0x00,0x00,0x00,0xd6,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0xda,0x00,0x00,0x00,0xd4,0x00,0x00,0x00, +0xd5,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xd4,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xdc,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xdc,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xce,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0xd4,0x00,0x00,0x00, +0x3d,0x01,0x00,0x00,0xdd,0x00,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc1,0x00,0x00,0x00,0xe2,0x00,0x00,0x00,0xce,0x02,0x00,0x00, +0x37,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xde,0x00,0x00,0x00, +0xdd,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0xe2,0x00,0x00,0x00,0xdd,0x00,0x00,0x00,0xde,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xdd,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xe7,0x00,0x00,0x00,0x73,0x00,0x00,0x00, +0xce,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xea,0x00,0x00,0x00,0xe7,0x00,0x00,0x00,0x98,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xeb,0x00,0x00,0x00, +0xea,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xec,0x00,0x00,0x00,0xd2,0x02,0x00,0x00, +0xeb,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xee,0x00,0x00,0x00,0xec,0x00,0x00,0x00,0x6e,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xf4,0x00,0x00,0x00, +0xe7,0x00,0x00,0x00,0xf3,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xf6,0x00,0x00,0x00,0xf4,0x00,0x00,0x00, +0x6e,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xfa,0x00,0x00,0x00,0xee,0x00,0x00,0x00,0xf9,0x00,0x00,0x00, +0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xfe,0x00,0x00,0x00, +0xee,0x00,0x00,0x00,0xfd,0x00,0x00,0x00,0x41,0x00,0x07,0x00, +0x09,0x01,0x00,0x00,0x0a,0x01,0x00,0x00,0x07,0x01,0x00,0x00, +0x34,0x00,0x00,0x00,0xfa,0x00,0x00,0x00,0x34,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x00,0x01,0x00,0x00,0x0b,0x01,0x00,0x00, +0x0a,0x01,0x00,0x00,0x73,0x00,0x04,0x00,0xc3,0x00,0x00,0x00, +0x0c,0x01,0x00,0x00,0x0b,0x01,0x00,0x00,0x41,0x00,0x07,0x00, +0x09,0x01,0x00,0x00,0x0f,0x01,0x00,0x00,0x07,0x01,0x00,0x00, +0x34,0x00,0x00,0x00,0xfa,0x00,0x00,0x00,0xcf,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x00,0x01,0x00,0x00,0x10,0x01,0x00,0x00, +0x0f,0x01,0x00,0x00,0x73,0x00,0x04,0x00,0xc3,0x00,0x00,0x00, +0x11,0x01,0x00,0x00,0x10,0x01,0x00,0x00,0x41,0x00,0x08,0x00, +0x15,0x01,0x00,0x00,0x16,0x01,0x00,0x00,0x07,0x01,0x00,0x00, +0x34,0x00,0x00,0x00,0xfa,0x00,0x00,0x00,0x85,0x00,0x00,0x00, +0xfe,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x01,0x01,0x00,0x00, +0x17,0x01,0x00,0x00,0x16,0x01,0x00,0x00,0x71,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x18,0x01,0x00,0x00,0x17,0x01,0x00,0x00, +0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x1d,0x01,0x00,0x00, +0x18,0x01,0x00,0x00,0xfd,0x00,0x00,0x00,0x70,0x00,0x04,0x00, +0xc3,0x00,0x00,0x00,0x1e,0x01,0x00,0x00,0x1d,0x01,0x00,0x00, +0xc2,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x20,0x01,0x00,0x00, +0x18,0x01,0x00,0x00,0xa8,0x00,0x00,0x00,0x70,0x00,0x04,0x00, +0xc3,0x00,0x00,0x00,0x21,0x01,0x00,0x00,0x20,0x01,0x00,0x00, +0x50,0x00,0x05,0x00,0x19,0x01,0x00,0x00,0x22,0x01,0x00,0x00, +0x1e,0x01,0x00,0x00,0x21,0x01,0x00,0x00,0x8e,0x00,0x05,0x00, +0x19,0x01,0x00,0x00,0x24,0x01,0x00,0x00,0x22,0x01,0x00,0x00, +0x0c,0x01,0x00,0x00,0x50,0x00,0x05,0x00,0x19,0x01,0x00,0x00, +0x26,0x01,0x00,0x00,0x11,0x01,0x00,0x00,0x11,0x01,0x00,0x00, +0x81,0x00,0x05,0x00,0x19,0x01,0x00,0x00,0x27,0x01,0x00,0x00, +0x24,0x01,0x00,0x00,0x26,0x01,0x00,0x00,0x51,0x00,0x05,0x00, +0xc3,0x00,0x00,0x00,0x2f,0x01,0x00,0x00,0x27,0x01,0x00,0x00, +0x00,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x30,0x01,0x00,0x00, +0x31,0x01,0x00,0x00,0x2c,0x01,0x00,0x00,0xf6,0x00,0x00,0x00, +0x3e,0x00,0x03,0x00,0x31,0x01,0x00,0x00,0x2f,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x33,0x01,0x00,0x00, +0xf6,0x00,0x00,0x00,0xf9,0x00,0x00,0x00,0x51,0x00,0x05,0x00, +0xc3,0x00,0x00,0x00,0x35,0x01,0x00,0x00,0x27,0x01,0x00,0x00, +0x01,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x30,0x01,0x00,0x00, +0x36,0x01,0x00,0x00,0x2c,0x01,0x00,0x00,0x33,0x01,0x00,0x00, +0x3e,0x00,0x03,0x00,0x36,0x01,0x00,0x00,0x35,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3d,0x01,0x00,0x00, +0xce,0x02,0x00,0x00,0x3b,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0xdc,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xde,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x3f,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x3f,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xcf,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0xde,0x00,0x00,0x00, +0x84,0x01,0x00,0x00,0x42,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc1,0x00,0x00,0x00,0x45,0x01,0x00,0x00,0xcf,0x02,0x00,0x00, +0xa6,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x41,0x01,0x00,0x00, +0x42,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x45,0x01,0x00,0x00,0x40,0x01,0x00,0x00,0x41,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x40,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x49,0x01,0x00,0x00,0xa7,0x00,0x00,0x00, +0x7d,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x4b,0x01,0x00,0x00,0x49,0x01,0x00,0x00,0xcf,0x02,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x4c,0x01,0x00,0x00, +0x12,0x00,0x00,0x00,0xcf,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x4d,0x01,0x00,0x00,0x4c,0x01,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00,0x4e,0x01,0x00,0x00, +0x4b,0x01,0x00,0x00,0x4d,0x01,0x00,0x00,0xf7,0x00,0x03,0x00, +0x50,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x4e,0x01,0x00,0x00,0x4f,0x01,0x00,0x00,0x50,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x4f,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x53,0x01,0x00,0x00,0xbe,0x02,0x00,0x00, +0x78,0x00,0x00,0x00,0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00, +0x55,0x01,0x00,0x00,0x53,0x01,0x00,0x00,0x8d,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x50,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x50,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0xc1,0x00,0x00,0x00, +0x56,0x01,0x00,0x00,0x4e,0x01,0x00,0x00,0x40,0x01,0x00,0x00, +0x55,0x01,0x00,0x00,0x4f,0x01,0x00,0x00,0xf7,0x00,0x03,0x00, +0x58,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x56,0x01,0x00,0x00,0x57,0x01,0x00,0x00,0x77,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x57,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x60,0x01,0x00,0x00,0x7d,0x00,0x00,0x00, +0xcf,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x62,0x01,0x00,0x00,0x60,0x01,0x00,0x00,0x61,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x64,0x01,0x00,0x00, +0x62,0x01,0x00,0x00,0x78,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x6f,0x01,0x00,0x00,0x60,0x01,0x00,0x00, +0xaa,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x70,0x01,0x00,0x00,0xd6,0x02,0x00,0x00,0x6f,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x72,0x01,0x00,0x00, +0x70,0x01,0x00,0x00,0x78,0x00,0x00,0x00,0x41,0x00,0x06,0x00, +0x73,0x01,0x00,0x00,0x74,0x01,0x00,0x00,0x68,0x01,0x00,0x00, +0x34,0x00,0x00,0x00,0x72,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, +0xc3,0x00,0x00,0x00,0x75,0x01,0x00,0x00,0x74,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0x30,0x01,0x00,0x00,0x76,0x01,0x00,0x00, +0x5d,0x01,0x00,0x00,0x64,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, +0x76,0x01,0x00,0x00,0x75,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0x58,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x77,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x7a,0x01,0x00,0x00, +0x7d,0x00,0x00,0x00,0xcf,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x7c,0x01,0x00,0x00,0x7a,0x01,0x00,0x00, +0x7b,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x7e,0x01,0x00,0x00,0x7c,0x01,0x00,0x00,0x78,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x30,0x01,0x00,0x00,0x7f,0x01,0x00,0x00, +0x5d,0x01,0x00,0x00,0x7e,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, +0x7f,0x01,0x00,0x00,0xcb,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x58,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x58,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0x42,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x42,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x84,0x01,0x00,0x00,0xcf,0x02,0x00,0x00,0x82,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0x3f,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x41,0x01,0x00,0x00,0xe0,0x00,0x04,0x00,0x0c,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x85,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x88,0x01,0x00,0x00,0xd2,0x02,0x00,0x00, +0x86,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x8b,0x01,0x00,0x00,0xd6,0x02,0x00,0x00,0x89,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0x8d,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x8d,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xd8,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0x41,0x01,0x00,0x00, +0x34,0x02,0x00,0x00,0x90,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc1,0x00,0x00,0x00,0x93,0x01,0x00,0x00,0xd8,0x02,0x00,0x00, +0x6c,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x8f,0x01,0x00,0x00, +0x90,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x93,0x01,0x00,0x00,0x8e,0x01,0x00,0x00,0x8f,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x8e,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0x95,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x95,0x01,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xdc,0x02,0x00,0x00, +0x3e,0x00,0x00,0x00,0x8e,0x01,0x00,0x00,0xc0,0x01,0x00,0x00, +0x98,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00, +0x9b,0x01,0x00,0x00,0xdc,0x02,0x00,0x00,0x60,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0x97,0x01,0x00,0x00,0x98,0x01,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x9b,0x01,0x00,0x00, +0x96,0x01,0x00,0x00,0x97,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x96,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x9d,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x9d,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xee,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, +0x96,0x01,0x00,0x00,0xbe,0x01,0x00,0x00,0x9e,0x01,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00,0xa3,0x01,0x00,0x00, +0xee,0x02,0x00,0x00,0x62,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0x9f,0x01,0x00,0x00,0x9e,0x01,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0xa3,0x01,0x00,0x00,0x9e,0x01,0x00,0x00, +0x9f,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x9e,0x01,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa9,0x01,0x00,0x00, +0xdc,0x02,0x00,0x00,0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xab,0x01,0x00,0x00,0xa9,0x01,0x00,0x00, +0xee,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xad,0x01,0x00,0x00,0x55,0x00,0x00,0x00,0x53,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xaf,0x01,0x00,0x00, +0xdc,0x02,0x00,0x00,0x61,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xb0,0x01,0x00,0x00,0xad,0x01,0x00,0x00, +0xaf,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xb2,0x01,0x00,0x00,0x64,0x00,0x00,0x00,0x62,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xb3,0x01,0x00,0x00, +0xb0,0x01,0x00,0x00,0xb2,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xb5,0x01,0x00,0x00,0xb3,0x01,0x00,0x00, +0xee,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xb7,0x01,0x00,0x00,0xb5,0x01,0x00,0x00,0xb6,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xb9,0x01,0x00,0x00, +0xb7,0x01,0x00,0x00,0xd8,0x02,0x00,0x00,0x41,0x00,0x05,0x00, +0x30,0x01,0x00,0x00,0xba,0x01,0x00,0x00,0x2c,0x01,0x00,0x00, +0xb9,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0xc3,0x00,0x00,0x00, +0xbb,0x01,0x00,0x00,0xba,0x01,0x00,0x00,0x41,0x00,0x05,0x00, +0xcc,0x00,0x00,0x00,0xbc,0x01,0x00,0x00,0xa7,0x01,0x00,0x00, +0xab,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0xbc,0x01,0x00,0x00, +0xbb,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xbe,0x01,0x00,0x00,0xee,0x02,0x00,0x00,0xcf,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x9d,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x9f,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x98,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x98,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xc0,0x01,0x00,0x00,0xdc,0x02,0x00,0x00, +0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x95,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x97,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0xc2,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xc2,0x01,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xdd,0x02,0x00,0x00, +0x3e,0x00,0x00,0x00,0x97,0x01,0x00,0x00,0xee,0x01,0x00,0x00, +0xc5,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00, +0xc8,0x01,0x00,0x00,0xdd,0x02,0x00,0x00,0xbe,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0xc4,0x01,0x00,0x00,0xc5,0x01,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xc8,0x01,0x00,0x00, +0xc3,0x01,0x00,0x00,0xc4,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xc3,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xca,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xca,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xeb,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, +0xc3,0x01,0x00,0x00,0xec,0x01,0x00,0x00,0xcb,0x01,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00,0xd0,0x01,0x00,0x00, +0xeb,0x02,0x00,0x00,0xbb,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0xcc,0x01,0x00,0x00,0xcb,0x01,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0xd0,0x01,0x00,0x00,0xcb,0x01,0x00,0x00, +0xcc,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xcb,0x01,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xd6,0x01,0x00,0x00, +0xdd,0x02,0x00,0x00,0xbb,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xd8,0x01,0x00,0x00,0xd6,0x01,0x00,0x00, +0xeb,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xda,0x01,0x00,0x00,0x59,0x00,0x00,0x00,0xb8,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xdd,0x01,0x00,0x00, +0xdd,0x02,0x00,0x00,0xdc,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xde,0x01,0x00,0x00,0xda,0x01,0x00,0x00, +0xdd,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xe0,0x01,0x00,0x00,0x68,0x00,0x00,0x00,0xbb,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xe1,0x01,0x00,0x00, +0xde,0x01,0x00,0x00,0xe0,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xe3,0x01,0x00,0x00,0xe1,0x01,0x00,0x00, +0xeb,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xe5,0x01,0x00,0x00,0xe3,0x01,0x00,0x00,0xe4,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xe7,0x01,0x00,0x00, +0xe5,0x01,0x00,0x00,0xd8,0x02,0x00,0x00,0x41,0x00,0x05,0x00, +0x30,0x01,0x00,0x00,0xe8,0x01,0x00,0x00,0x5d,0x01,0x00,0x00, +0xe7,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0xc3,0x00,0x00,0x00, +0xe9,0x01,0x00,0x00,0xe8,0x01,0x00,0x00,0x41,0x00,0x05,0x00, +0xcc,0x00,0x00,0x00,0xea,0x01,0x00,0x00,0xd4,0x01,0x00,0x00, +0xd8,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0xea,0x01,0x00,0x00, +0xe9,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xec,0x01,0x00,0x00,0xeb,0x02,0x00,0x00,0xcf,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xca,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xcc,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xc5,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xc5,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xee,0x01,0x00,0x00,0xdd,0x02,0x00,0x00, +0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xc2,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xc4,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0xf0,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xf0,0x01,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xde,0x02,0x00,0x00, +0x3e,0x00,0x00,0x00,0xc4,0x01,0x00,0x00,0x32,0x02,0x00,0x00, +0xf3,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00, +0xf6,0x01,0x00,0x00,0xde,0x02,0x00,0x00,0xbe,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0xf2,0x01,0x00,0x00,0xf3,0x01,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xf6,0x01,0x00,0x00, +0xf1,0x01,0x00,0x00,0xf2,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xf1,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xf8,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xf8,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xe2,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, +0xf1,0x01,0x00,0x00,0x30,0x02,0x00,0x00,0xfb,0x01,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00,0xfe,0x01,0x00,0x00, +0xe2,0x02,0x00,0x00,0x60,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0xfa,0x01,0x00,0x00,0xfb,0x01,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0xfe,0x01,0x00,0x00,0xf9,0x01,0x00,0x00, +0xfa,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xf9,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0x00,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x00,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xe4,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0xf9,0x01,0x00,0x00, +0x2e,0x02,0x00,0x00,0x03,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc1,0x00,0x00,0x00,0x06,0x02,0x00,0x00,0xe4,0x02,0x00,0x00, +0xbb,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x02,0x02,0x00,0x00, +0x03,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x06,0x02,0x00,0x00,0x01,0x02,0x00,0x00,0x02,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x01,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0x08,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x08,0x02,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xe6,0x02,0x00,0x00, +0x3e,0x00,0x00,0x00,0x01,0x02,0x00,0x00,0x2c,0x02,0x00,0x00, +0x09,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00, +0x0e,0x02,0x00,0x00,0xe6,0x02,0x00,0x00,0x62,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0x0a,0x02,0x00,0x00,0x09,0x02,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x0e,0x02,0x00,0x00, +0x09,0x02,0x00,0x00,0x0a,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x09,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x10,0x02,0x00,0x00,0xde,0x02,0x00,0x00,0xbb,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x12,0x02,0x00,0x00, +0x10,0x02,0x00,0x00,0xe4,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x14,0x02,0x00,0x00,0x12,0x02,0x00,0x00, +0x13,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x16,0x02,0x00,0x00,0xe2,0x02,0x00,0x00,0x62,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x17,0x02,0x00,0x00, +0x14,0x02,0x00,0x00,0x16,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x19,0x02,0x00,0x00,0x17,0x02,0x00,0x00, +0xe6,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x1d,0x02,0x00,0x00,0x16,0x02,0x00,0x00,0xe6,0x02,0x00,0x00, +0x41,0x00,0x05,0x00,0xcc,0x00,0x00,0x00,0x1e,0x02,0x00,0x00, +0xa7,0x01,0x00,0x00,0x1d,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, +0xc3,0x00,0x00,0x00,0x1f,0x02,0x00,0x00,0x1e,0x02,0x00,0x00, +0x41,0x00,0x05,0x00,0xcc,0x00,0x00,0x00,0x24,0x02,0x00,0x00, +0xd4,0x01,0x00,0x00,0x12,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, +0xc3,0x00,0x00,0x00,0x25,0x02,0x00,0x00,0x24,0x02,0x00,0x00, +0x41,0x00,0x05,0x00,0xcc,0x00,0x00,0x00,0x27,0x02,0x00,0x00, +0xc9,0x00,0x00,0x00,0x19,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, +0xc3,0x00,0x00,0x00,0x28,0x02,0x00,0x00,0x27,0x02,0x00,0x00, +0x0c,0x00,0x08,0x00,0xc3,0x00,0x00,0x00,0x29,0x02,0x00,0x00, +0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x1f,0x02,0x00,0x00, +0x25,0x02,0x00,0x00,0x28,0x02,0x00,0x00,0x3e,0x00,0x03,0x00, +0x27,0x02,0x00,0x00,0x29,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x2c,0x02,0x00,0x00,0xe6,0x02,0x00,0x00, +0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x08,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x0a,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0x03,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x03,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x2e,0x02,0x00,0x00, +0xe4,0x02,0x00,0x00,0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x00,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x02,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0xfb,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xfb,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x30,0x02,0x00,0x00,0xe2,0x02,0x00,0x00,0xcf,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xf8,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xfa,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xf3,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xf3,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x32,0x02,0x00,0x00,0xde,0x02,0x00,0x00, +0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xf0,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xf2,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0x90,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x90,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x34,0x02,0x00,0x00, +0xd8,0x02,0x00,0x00,0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x8d,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x8f,0x01,0x00,0x00, +0xe0,0x00,0x04,0x00,0x0c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x85,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xd6,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xd6,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x36,0x02,0x00,0x00,0xbe,0x02,0x00,0x00, +0x6c,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xd3,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xd5,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x3b,0x02,0x00,0x00,0x55,0x00,0x00,0x00, +0x53,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x3c,0x02,0x00,0x00,0x95,0x00,0x00,0x00,0x3b,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x41,0x02,0x00,0x00, +0x59,0x00,0x00,0x00,0xb8,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x42,0x02,0x00,0x00,0xa7,0x00,0x00,0x00, +0x41,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, +0x47,0x02,0x00,0x00,0x12,0x00,0x00,0x00,0x46,0x02,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x48,0x02,0x00,0x00, +0x47,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x49,0x02,0x00,0x00,0x0f,0x00,0x00,0x00,0x48,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x4d,0x02,0x00,0x00, +0x47,0x00,0x00,0x00,0x48,0x02,0x00,0x00,0x41,0x00,0x05,0x00, +0x0d,0x00,0x00,0x00,0x4f,0x02,0x00,0x00,0x4e,0x02,0x00,0x00, +0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x50,0x02,0x00,0x00,0x4f,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x51,0x02,0x00,0x00,0x4d,0x02,0x00,0x00, +0x50,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x52,0x02,0x00,0x00,0x49,0x02,0x00,0x00,0x51,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0x54,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x54,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xbf,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0xd5,0x00,0x00,0x00, +0xba,0x02,0x00,0x00,0x57,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc1,0x00,0x00,0x00,0x5a,0x02,0x00,0x00,0xbf,0x02,0x00,0x00, +0xbe,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x56,0x02,0x00,0x00, +0x57,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x5a,0x02,0x00,0x00,0x55,0x02,0x00,0x00,0x56,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x55,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0x5c,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x5c,0x02,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xc0,0x02,0x00,0x00, +0x3e,0x00,0x00,0x00,0x55,0x02,0x00,0x00,0xb8,0x02,0x00,0x00, +0x5f,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00, +0x62,0x02,0x00,0x00,0xc0,0x02,0x00,0x00,0x60,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0x5e,0x02,0x00,0x00,0x5f,0x02,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x62,0x02,0x00,0x00, +0x5d,0x02,0x00,0x00,0x5e,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x5d,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x66,0x02,0x00,0x00,0xc0,0x02,0x00,0x00,0x61,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x67,0x02,0x00,0x00, +0x3c,0x02,0x00,0x00,0x66,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x69,0x02,0x00,0x00,0x64,0x00,0x00,0x00, +0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x6a,0x02,0x00,0x00,0x67,0x02,0x00,0x00,0x69,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x6e,0x02,0x00,0x00, +0xbf,0x02,0x00,0x00,0xdc,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x6f,0x02,0x00,0x00,0x42,0x02,0x00,0x00, +0x6e,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x71,0x02,0x00,0x00,0x68,0x00,0x00,0x00,0xbb,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x72,0x02,0x00,0x00, +0x6f,0x02,0x00,0x00,0x71,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0x74,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x74,0x02,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xc2,0x02,0x00,0x00, +0x3e,0x00,0x00,0x00,0x5d,0x02,0x00,0x00,0xb6,0x02,0x00,0x00, +0x77,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00, +0x7a,0x02,0x00,0x00,0xc2,0x02,0x00,0x00,0xbb,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0x76,0x02,0x00,0x00,0x77,0x02,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x7a,0x02,0x00,0x00, +0x75,0x02,0x00,0x00,0x76,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x75,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x7c,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x7c,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xc4,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, +0x75,0x02,0x00,0x00,0xb4,0x02,0x00,0x00,0x7f,0x02,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00,0x82,0x02,0x00,0x00, +0xc4,0x02,0x00,0x00,0x62,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0x7e,0x02,0x00,0x00,0x7f,0x02,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x82,0x02,0x00,0x00,0x7d,0x02,0x00,0x00, +0x7e,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x7d,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x85,0x02,0x00,0x00, +0x6a,0x02,0x00,0x00,0xc4,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc1,0x00,0x00,0x00,0x88,0x02,0x00,0x00,0x85,0x02,0x00,0x00, +0x36,0x00,0x00,0x00,0xf7,0x00,0x03,0x00,0x8a,0x02,0x00,0x00, +0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x88,0x02,0x00,0x00, +0x89,0x02,0x00,0x00,0x8a,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x89,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x8d,0x02,0x00,0x00,0x72,0x02,0x00,0x00,0xc2,0x02,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x8e,0x02,0x00,0x00, +0x12,0x00,0x00,0x00,0xcf,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x8f,0x02,0x00,0x00,0x8e,0x02,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00,0x90,0x02,0x00,0x00, +0x8d,0x02,0x00,0x00,0x8f,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0x8a,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x8a,0x02,0x00,0x00, +0xf5,0x00,0x07,0x00,0xc1,0x00,0x00,0x00,0x91,0x02,0x00,0x00, +0x88,0x02,0x00,0x00,0x7d,0x02,0x00,0x00,0x90,0x02,0x00,0x00, +0x89,0x02,0x00,0x00,0xf7,0x00,0x03,0x00,0x93,0x02,0x00,0x00, +0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x91,0x02,0x00,0x00, +0x92,0x02,0x00,0x00,0x93,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x92,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x9b,0x02,0x00,0x00,0x72,0x02,0x00,0x00,0xc2,0x02,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x9d,0x02,0x00,0x00, +0x12,0x00,0x00,0x00,0x9c,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x9e,0x02,0x00,0x00,0x9d,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9f,0x02,0x00,0x00, +0x9b,0x02,0x00,0x00,0x9e,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xa0,0x02,0x00,0x00,0x52,0x02,0x00,0x00, +0x9f,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xa2,0x02,0x00,0x00,0xa0,0x02,0x00,0x00,0x6a,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa4,0x02,0x00,0x00, +0xa2,0x02,0x00,0x00,0xc4,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xa6,0x02,0x00,0x00,0xbf,0x02,0x00,0x00, +0xbb,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xa8,0x02,0x00,0x00,0xa6,0x02,0x00,0x00,0xc2,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xaa,0x02,0x00,0x00, +0xa8,0x02,0x00,0x00,0xa9,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xac,0x02,0x00,0x00,0xc0,0x02,0x00,0x00, +0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xad,0x02,0x00,0x00,0xaa,0x02,0x00,0x00,0xac,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xaf,0x02,0x00,0x00, +0xad,0x02,0x00,0x00,0xc4,0x02,0x00,0x00,0x41,0x00,0x05,0x00, +0xcc,0x00,0x00,0x00,0xb0,0x02,0x00,0x00,0xc9,0x00,0x00,0x00, +0xaf,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0xc3,0x00,0x00,0x00, +0xb1,0x02,0x00,0x00,0xb0,0x02,0x00,0x00,0x41,0x00,0x06,0x00, +0x73,0x01,0x00,0x00,0xb2,0x02,0x00,0x00,0x97,0x02,0x00,0x00, +0x34,0x00,0x00,0x00,0xa4,0x02,0x00,0x00,0x3e,0x00,0x03,0x00, +0xb2,0x02,0x00,0x00,0xb1,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0x93,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x93,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0x7f,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x7f,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xb4,0x02,0x00,0x00,0xc4,0x02,0x00,0x00,0xcf,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x7c,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x7e,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x77,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x77,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xb6,0x02,0x00,0x00,0xc2,0x02,0x00,0x00, +0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x74,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x76,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0x5f,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x5f,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xb8,0x02,0x00,0x00, +0xc0,0x02,0x00,0x00,0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x5c,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x5e,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0x57,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x57,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xba,0x02,0x00,0x00,0xbf,0x02,0x00,0x00,0xcf,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x54,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x56,0x02,0x00,0x00,0xfd,0x00,0x01,0x00,0x38,0x00,0x01,0x00, + +}; +const uint64_t matmul_q4_1_f32_fp32_len = 10512; + +unsigned char matmul_q5_0_f32_data[] = { +0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, +0x47,0x03,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, +0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x09,0x00,0x00,0x00, +0x11,0x00,0x02,0x00,0x51,0x11,0x00,0x00,0x11,0x00,0x02,0x00, +0x60,0x11,0x00,0x00,0x0b,0x00,0x06,0x00,0x01,0x00,0x00,0x00, +0x47,0x4c,0x53,0x4c,0x2e,0x73,0x74,0x64,0x2e,0x34,0x35,0x30, +0x00,0x00,0x00,0x00,0x0e,0x00,0x03,0x00,0x00,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x0f,0x00,0x0f,0x00,0x05,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x6d,0x61,0x69,0x6e,0x00,0x00,0x00,0x00, +0x0b,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x3d,0x00,0x00,0x00, +0x4c,0x00,0x00,0x00,0x09,0x01,0x00,0x00,0x50,0x01,0x00,0x00, +0x83,0x01,0x00,0x00,0x8e,0x01,0x00,0x00,0x79,0x02,0x00,0x00, +0xc2,0x02,0x00,0x00,0x10,0x00,0x06,0x00,0x04,0x00,0x00,0x00, +0x11,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x0b,0x00,0x00,0x00, +0x0b,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x10,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x10,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x10,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x05,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x10,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, +0x07,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x1c,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x08,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x10,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x24,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, +0x0a,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x28,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x2c,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x10,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x30,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, +0x0d,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x34,0x00,0x00,0x00, +0x47,0x00,0x03,0x00,0x10,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x37,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x3d,0x00,0x00,0x00, +0x0b,0x00,0x00,0x00,0x1a,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x4c,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x1b,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x4f,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x09,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x53,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x60,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x62,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x07,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x6c,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0xa6,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0xb8,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x05,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xbb,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x02,0x01,0x00,0x00,0x06,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x04,0x01,0x00,0x00,0x06,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x05,0x01,0x00,0x00, +0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x05,0x01,0x00,0x00,0x01,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x05,0x01,0x00,0x00,0x02,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x06,0x01,0x00,0x00, +0x06,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x48,0x00,0x04,0x00, +0x07,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x07,0x01,0x00,0x00,0x00,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, +0x07,0x01,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x09,0x01,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x09,0x01,0x00,0x00,0x21,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x5d,0x01,0x00,0x00, +0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x5e,0x01,0x00,0x00,0x0b,0x00,0x00,0x00,0x19,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x8b,0x01,0x00,0x00,0x06,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0x8c,0x01,0x00,0x00, +0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x8c,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x8c,0x01,0x00,0x00, +0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x8e,0x01,0x00,0x00, +0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x8e,0x01,0x00,0x00,0x21,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x79,0x02,0x00,0x00,0x0b,0x00,0x00,0x00, +0x18,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xbf,0x02,0x00,0x00, +0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00, +0xc0,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x19,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0xc0,0x02,0x00,0x00,0x00,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, +0xc0,0x02,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0xc2,0x02,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0xc2,0x02,0x00,0x00,0x21,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x13,0x00,0x02,0x00,0x02,0x00,0x00,0x00, +0x21,0x00,0x03,0x00,0x03,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x15,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x17,0x00,0x04,0x00,0x09,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x0a,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x09,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x0d,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x1e,0x00,0x10,0x00,0x10,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x11,0x00,0x00,0x00, +0x09,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0x11,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x09,0x00,0x00,0x00, +0x15,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, +0x14,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x15,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x21,0x00,0x00,0x00, +0x0a,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, +0x27,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0x2d,0x00,0x00,0x00,0x07,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x34,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x37,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00,0x3d,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x3e,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0x0a,0x00,0x00,0x00,0x4c,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x4f,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x53,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x54,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0x37,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x58,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0x37,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x32,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x61,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x60,0x00,0x00,0x00, +0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x62,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x63,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x61,0x00,0x00,0x00, +0x62,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x67,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x61,0x00,0x00,0x00, +0x62,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x72,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x77,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x7c,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0x80,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x85,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, +0x90,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0x96,0x00,0x00,0x00,0x03,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0xa1,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xa6,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0xa8,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xb7,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00, +0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xb8,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xb9,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x53,0x00,0x00,0x00, +0xb8,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xba,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x4f,0x00,0x00,0x00, +0x62,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xbb,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xbc,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0xba,0x00,0x00,0x00,0xbb,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xbd,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0xbc,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xbe,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0xb9,0x00,0x00,0x00,0xbd,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xbf,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0xb7,0x00,0x00,0x00,0xbe,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xc0,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0xbf,0x00,0x00,0x00,0xbb,0x00,0x00,0x00,0x14,0x00,0x02,0x00, +0xc1,0x00,0x00,0x00,0x16,0x00,0x03,0x00,0xc3,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xc4,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00, +0x62,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xc5,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0xc4,0x00,0x00,0x00, +0xbe,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xc6,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0xc5,0x00,0x00,0x00, +0xbb,0x00,0x00,0x00,0x1c,0x00,0x04,0x00,0xc7,0x00,0x00,0x00, +0xc3,0x00,0x00,0x00,0xc6,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0xc8,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0xc7,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0xc3,0x00,0x00,0x00,0xcb,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0xcc,0x00,0x00,0x00, +0x07,0x00,0x00,0x00,0xc3,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0xcf,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xf3,0x00,0x00,0x00, +0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xf9,0x00,0x00,0x00, +0x10,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xfd,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x16,0x00,0x03,0x00, +0x00,0x01,0x00,0x00,0x10,0x00,0x00,0x00,0x15,0x00,0x04,0x00, +0x01,0x01,0x00,0x00,0x10,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x1c,0x00,0x04,0x00,0x02,0x01,0x00,0x00,0x01,0x01,0x00,0x00, +0x0c,0x00,0x00,0x00,0x15,0x00,0x04,0x00,0x03,0x01,0x00,0x00, +0x08,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x1c,0x00,0x04,0x00, +0x04,0x01,0x00,0x00,0x03,0x01,0x00,0x00,0xf9,0x00,0x00,0x00, +0x1e,0x00,0x05,0x00,0x05,0x01,0x00,0x00,0x00,0x01,0x00,0x00, +0x02,0x01,0x00,0x00,0x04,0x01,0x00,0x00,0x1d,0x00,0x03,0x00, +0x06,0x01,0x00,0x00,0x05,0x01,0x00,0x00,0x1e,0x00,0x03,0x00, +0x07,0x01,0x00,0x00,0x06,0x01,0x00,0x00,0x20,0x00,0x04,0x00, +0x08,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x07,0x01,0x00,0x00, +0x3b,0x00,0x04,0x00,0x08,0x01,0x00,0x00,0x09,0x01,0x00,0x00, +0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x0b,0x01,0x00,0x00, +0x0c,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x20,0x00,0x04,0x00, +0x11,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x01,0x01,0x00,0x00, +0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x15,0x01,0x00,0x00, +0x10,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x27,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x30,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x03,0x01,0x00,0x00, +0x17,0x00,0x04,0x00,0x34,0x01,0x00,0x00,0xc3,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0xc3,0x00,0x00,0x00, +0x47,0x01,0x00,0x00,0x00,0x00,0x80,0x41,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x4c,0x01,0x00,0x00,0x80,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x4d,0x01,0x00,0x00,0x84,0x00,0x00,0x00, +0x37,0x00,0x00,0x00,0x4c,0x01,0x00,0x00,0x1c,0x00,0x04,0x00, +0x4e,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x4d,0x01,0x00,0x00, +0x20,0x00,0x04,0x00,0x4f,0x01,0x00,0x00,0x04,0x00,0x00,0x00, +0x4e,0x01,0x00,0x00,0x3b,0x00,0x04,0x00,0x4f,0x01,0x00,0x00, +0x50,0x01,0x00,0x00,0x04,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x55,0x01,0x00,0x00,0x04,0x00,0x00,0x00,0x00,0x01,0x00,0x00, +0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x5d,0x01,0x00,0x00, +0x01,0x00,0x00,0x00,0x33,0x00,0x06,0x00,0x09,0x00,0x00,0x00, +0x5e,0x01,0x00,0x00,0x5d,0x01,0x00,0x00,0x39,0x00,0x00,0x00, +0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x5f,0x01,0x00,0x00,0x51,0x00,0x00,0x00,0x5e,0x01,0x00,0x00, +0x00,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x60,0x01,0x00,0x00,0x84,0x00,0x00,0x00,0x5f,0x01,0x00,0x00, +0x0c,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x61,0x01,0x00,0x00,0x86,0x00,0x00,0x00,0x60,0x01,0x00,0x00, +0x6c,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x7f,0x01,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, +0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x80,0x01,0x00,0x00,0x84,0x00,0x00,0x00,0xa6,0x00,0x00,0x00, +0x7f,0x01,0x00,0x00,0x1c,0x00,0x04,0x00,0x81,0x01,0x00,0x00, +0x00,0x01,0x00,0x00,0x80,0x01,0x00,0x00,0x20,0x00,0x04,0x00, +0x82,0x01,0x00,0x00,0x04,0x00,0x00,0x00,0x81,0x01,0x00,0x00, +0x3b,0x00,0x04,0x00,0x82,0x01,0x00,0x00,0x83,0x01,0x00,0x00, +0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x87,0x01,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, +0x39,0x00,0x00,0x00,0x1d,0x00,0x03,0x00,0x8b,0x01,0x00,0x00, +0xc3,0x00,0x00,0x00,0x1e,0x00,0x03,0x00,0x8c,0x01,0x00,0x00, +0x8b,0x01,0x00,0x00,0x20,0x00,0x04,0x00,0x8d,0x01,0x00,0x00, +0x0c,0x00,0x00,0x00,0x8c,0x01,0x00,0x00,0x3b,0x00,0x04,0x00, +0x8d,0x01,0x00,0x00,0x8e,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x99,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, +0xc3,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xa2,0x01,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, +0x39,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x00,0x01,0x00,0x00, +0xa6,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xa8,0x01,0x00,0x00,0x51,0x00,0x00,0x00, +0x5e,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xa9,0x01,0x00,0x00,0x84,0x00,0x00,0x00, +0xa8,0x01,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xaa,0x01,0x00,0x00,0x86,0x00,0x00,0x00, +0xa9,0x01,0x00,0x00,0x6c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xad,0x01,0x00,0x00,0x08,0x01,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xae,0x01,0x00,0x00, +0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xb1,0x01,0x00,0x00, +0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xcc,0x01,0x00,0x00, +0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00, +0x1c,0x00,0x04,0x00,0xcd,0x01,0x00,0x00,0x00,0x01,0x00,0x00, +0xcc,0x01,0x00,0x00,0x20,0x00,0x04,0x00,0xce,0x01,0x00,0x00, +0x07,0x00,0x00,0x00,0xcd,0x01,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xde,0x01,0x00,0x00,0x80,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0xe4,0x01,0x00,0x00,0x07,0x00,0x00,0x00,0x00,0x01,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xfa,0x01,0x00,0x00, +0x84,0x00,0x00,0x00,0xbe,0x00,0x00,0x00,0xbb,0x00,0x00,0x00, +0x1c,0x00,0x04,0x00,0xfb,0x01,0x00,0x00,0x00,0x01,0x00,0x00, +0xfa,0x01,0x00,0x00,0x20,0x00,0x04,0x00,0xfc,0x01,0x00,0x00, +0x07,0x00,0x00,0x00,0xfb,0x01,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x05,0x02,0x00,0x00,0x86,0x00,0x00,0x00, +0xb8,0x00,0x00,0x00,0xbe,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x0d,0x02,0x00,0x00,0x80,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x3c,0x02,0x00,0x00,0x84,0x00,0x00,0x00, +0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0x71,0x02,0x00,0x00,0x0d,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00,0x79,0x02,0x00,0x00, +0x01,0x00,0x00,0x00,0x1d,0x00,0x03,0x00,0xbf,0x02,0x00,0x00, +0xc3,0x00,0x00,0x00,0x1e,0x00,0x03,0x00,0xc0,0x02,0x00,0x00, +0xbf,0x02,0x00,0x00,0x20,0x00,0x04,0x00,0xc1,0x02,0x00,0x00, +0x0c,0x00,0x00,0x00,0xc0,0x02,0x00,0x00,0x3b,0x00,0x04,0x00, +0xc1,0x02,0x00,0x00,0xc2,0x02,0x00,0x00,0x0c,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0xc7,0x02,0x00,0x00, +0x05,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xd4,0x02,0x00,0x00,0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00, +0x62,0x00,0x00,0x00,0x2c,0x00,0x05,0x00,0x34,0x01,0x00,0x00, +0x46,0x03,0x00,0x00,0x47,0x01,0x00,0x00,0x47,0x01,0x00,0x00, +0x36,0x00,0x05,0x00,0x02,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0x05,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0xc8,0x00,0x00,0x00, +0xc9,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0xce,0x01,0x00,0x00,0xcf,0x01,0x00,0x00,0x07,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0xfc,0x01,0x00,0x00,0xfd,0x01,0x00,0x00, +0x07,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00, +0x0e,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, +0x0e,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, +0x16,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x14,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x17,0x00,0x00,0x00, +0x16,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x18,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x17,0x00,0x00,0x00, +0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x1e,0x00,0x00,0x00, +0x0f,0x00,0x00,0x00,0x17,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x15,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0x21,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x24,0x00,0x00,0x00,0x18,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, +0x28,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x27,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x29,0x00,0x00,0x00, +0x28,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x2a,0x00,0x00,0x00,0x1e,0x00,0x00,0x00,0x29,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, +0x12,0x00,0x00,0x00,0x2d,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x2f,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x30,0x00,0x00,0x00, +0x24,0x00,0x00,0x00,0x2f,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x30,0x00,0x00,0x00, +0x2a,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, +0x35,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x34,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x36,0x00,0x00,0x00, +0x35,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x38,0x00,0x00,0x00,0x36,0x00,0x00,0x00,0x37,0x00,0x00,0x00, +0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, +0x38,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x3b,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, +0x37,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00, +0x3f,0x00,0x00,0x00,0x3d,0x00,0x00,0x00,0x3e,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x40,0x00,0x00,0x00, +0x3f,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x42,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x3b,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x47,0x00,0x00,0x00, +0x40,0x00,0x00,0x00,0x3b,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x0d,0x00,0x00,0x00,0x49,0x00,0x00,0x00,0x3d,0x00,0x00,0x00, +0x39,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x4a,0x00,0x00,0x00,0x49,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x0d,0x00,0x00,0x00,0x4d,0x00,0x00,0x00,0x4c,0x00,0x00,0x00, +0x3e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x4e,0x00,0x00,0x00,0x4d,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x50,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, +0x4f,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x55,0x00,0x00,0x00,0x50,0x00,0x00,0x00,0x54,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x59,0x00,0x00,0x00, +0x50,0x00,0x00,0x00,0x58,0x00,0x00,0x00,0x89,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x5d,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, +0x4f,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x64,0x00,0x00,0x00,0x5d,0x00,0x00,0x00,0x63,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x68,0x00,0x00,0x00, +0x5d,0x00,0x00,0x00,0x67,0x00,0x00,0x00,0x89,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x6e,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, +0x6d,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x73,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x72,0x00,0x00,0x00, +0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x78,0x00,0x00,0x00, +0x4e,0x00,0x00,0x00,0x77,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x7d,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, +0x7c,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, +0x81,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x80,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x82,0x00,0x00,0x00, +0x81,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x83,0x00,0x00,0x00,0x47,0x00,0x00,0x00,0x82,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0x12,0x00,0x00,0x00,0x85,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x87,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x89,0x00,0x00,0x00, +0x47,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x8c,0x00,0x00,0x00,0x89,0x00,0x00,0x00, +0x82,0x00,0x00,0x00,0x0c,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0x8d,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x26,0x00,0x00,0x00, +0x87,0x00,0x00,0x00,0x8c,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x15,0x00,0x00,0x00,0x91,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0x90,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x92,0x00,0x00,0x00,0x91,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x93,0x00,0x00,0x00,0x32,0x00,0x00,0x00, +0x92,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x95,0x00,0x00,0x00,0x42,0x00,0x00,0x00,0x37,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x97,0x00,0x00,0x00, +0x12,0x00,0x00,0x00,0x96,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x98,0x00,0x00,0x00,0x97,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x99,0x00,0x00,0x00, +0x95,0x00,0x00,0x00,0x98,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x9a,0x00,0x00,0x00,0x93,0x00,0x00,0x00, +0x99,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x9c,0x00,0x00,0x00,0x9a,0x00,0x00,0x00,0x83,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9d,0x00,0x00,0x00, +0x9c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x15,0x00,0x00,0x00,0xa2,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0xa1,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xa3,0x00,0x00,0x00,0xa2,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xa4,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, +0xa3,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xa7,0x00,0x00,0x00,0x4a,0x00,0x00,0x00,0xa6,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0xa9,0x00,0x00,0x00, +0x12,0x00,0x00,0x00,0xa8,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xaa,0x00,0x00,0x00,0xa9,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xab,0x00,0x00,0x00, +0xa7,0x00,0x00,0x00,0xaa,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xac,0x00,0x00,0x00,0xa4,0x00,0x00,0x00, +0xab,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xae,0x00,0x00,0x00,0xac,0x00,0x00,0x00,0x83,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xaf,0x00,0x00,0x00, +0xae,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xb1,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xb1,0x00,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xea,0x02,0x00,0x00, +0x3e,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0xd0,0x00,0x00,0x00, +0xb2,0x00,0x00,0x00,0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00, +0xc2,0x00,0x00,0x00,0xea,0x02,0x00,0x00,0xc0,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0xb3,0x00,0x00,0x00,0xb2,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xc2,0x00,0x00,0x00, +0xb2,0x00,0x00,0x00,0xb3,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xb2,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0xcc,0x00,0x00,0x00, +0xcd,0x00,0x00,0x00,0xc9,0x00,0x00,0x00,0xea,0x02,0x00,0x00, +0x3e,0x00,0x03,0x00,0xcd,0x00,0x00,0x00,0xcb,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xd0,0x00,0x00,0x00, +0xea,0x02,0x00,0x00,0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xb1,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xb3,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xd3,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xd3,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0x03,0x03,0x00,0x00,0xaf,0x00,0x00,0x00,0xb3,0x00,0x00,0x00, +0xb3,0x01,0x00,0x00,0xd6,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xff,0x02,0x00,0x00,0x9d,0x00,0x00,0x00, +0xb3,0x00,0x00,0x00,0xb0,0x01,0x00,0x00,0xd6,0x00,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xeb,0x02,0x00,0x00, +0x83,0x00,0x00,0x00,0xb3,0x00,0x00,0x00,0x61,0x02,0x00,0x00, +0xd6,0x00,0x00,0x00,0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00, +0xda,0x00,0x00,0x00,0xeb,0x02,0x00,0x00,0x8d,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0xd5,0x00,0x00,0x00,0xd6,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xda,0x00,0x00,0x00, +0xd4,0x00,0x00,0x00,0xd5,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xd4,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xdc,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xdc,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xfb,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, +0xd4,0x00,0x00,0x00,0x63,0x01,0x00,0x00,0xdd,0x00,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00,0xe2,0x00,0x00,0x00, +0xfb,0x02,0x00,0x00,0x37,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0xde,0x00,0x00,0x00,0xdd,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0xe2,0x00,0x00,0x00,0xdd,0x00,0x00,0x00, +0xde,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xdd,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xe7,0x00,0x00,0x00, +0x73,0x00,0x00,0x00,0xfb,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xea,0x00,0x00,0x00,0xe7,0x00,0x00,0x00, +0x98,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xeb,0x00,0x00,0x00,0xea,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xec,0x00,0x00,0x00, +0xff,0x02,0x00,0x00,0xeb,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xee,0x00,0x00,0x00,0xec,0x00,0x00,0x00, +0x6e,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xf4,0x00,0x00,0x00,0xe7,0x00,0x00,0x00,0xf3,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xf6,0x00,0x00,0x00, +0xf4,0x00,0x00,0x00,0x6e,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xfa,0x00,0x00,0x00,0xee,0x00,0x00,0x00, +0xf9,0x00,0x00,0x00,0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xfe,0x00,0x00,0x00,0xee,0x00,0x00,0x00,0xfd,0x00,0x00,0x00, +0x41,0x00,0x07,0x00,0x0b,0x01,0x00,0x00,0x0c,0x01,0x00,0x00, +0x09,0x01,0x00,0x00,0x34,0x00,0x00,0x00,0xfa,0x00,0x00,0x00, +0x34,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x00,0x01,0x00,0x00, +0x0d,0x01,0x00,0x00,0x0c,0x01,0x00,0x00,0x73,0x00,0x04,0x00, +0xc3,0x00,0x00,0x00,0x0e,0x01,0x00,0x00,0x0d,0x01,0x00,0x00, +0x41,0x00,0x08,0x00,0x11,0x01,0x00,0x00,0x12,0x01,0x00,0x00, +0x09,0x01,0x00,0x00,0x34,0x00,0x00,0x00,0xfa,0x00,0x00,0x00, +0xcf,0x00,0x00,0x00,0xcf,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x01,0x01,0x00,0x00,0x13,0x01,0x00,0x00,0x12,0x01,0x00,0x00, +0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x14,0x01,0x00,0x00, +0x13,0x01,0x00,0x00,0xc4,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x16,0x01,0x00,0x00,0x14,0x01,0x00,0x00,0x15,0x01,0x00,0x00, +0x41,0x00,0x08,0x00,0x11,0x01,0x00,0x00,0x18,0x01,0x00,0x00, +0x09,0x01,0x00,0x00,0x34,0x00,0x00,0x00,0xfa,0x00,0x00,0x00, +0xcf,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x01,0x01,0x00,0x00,0x19,0x01,0x00,0x00,0x18,0x01,0x00,0x00, +0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x1a,0x01,0x00,0x00, +0x19,0x01,0x00,0x00,0xc5,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x1b,0x01,0x00,0x00,0x16,0x01,0x00,0x00,0x1a,0x01,0x00,0x00, +0xc2,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x21,0x01,0x00,0x00, +0x1b,0x01,0x00,0x00,0xfe,0x00,0x00,0x00,0xc4,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x22,0x01,0x00,0x00,0x21,0x01,0x00,0x00, +0xa8,0x00,0x00,0x00,0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x23,0x01,0x00,0x00,0x22,0x01,0x00,0x00,0xf9,0x00,0x00,0x00, +0x7c,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x24,0x01,0x00,0x00, +0x23,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x28,0x01,0x00,0x00,0xfe,0x00,0x00,0x00,0x27,0x01,0x00,0x00, +0xc2,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x29,0x01,0x00,0x00, +0x1b,0x01,0x00,0x00,0x28,0x01,0x00,0x00,0xc7,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x2a,0x01,0x00,0x00,0x29,0x01,0x00,0x00, +0xf9,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x13,0x00,0x00,0x00, +0x2b,0x01,0x00,0x00,0x2a,0x01,0x00,0x00,0x41,0x00,0x08,0x00, +0x30,0x01,0x00,0x00,0x31,0x01,0x00,0x00,0x09,0x01,0x00,0x00, +0x34,0x00,0x00,0x00,0xfa,0x00,0x00,0x00,0x85,0x00,0x00,0x00, +0xfe,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x03,0x01,0x00,0x00, +0x32,0x01,0x00,0x00,0x31,0x01,0x00,0x00,0x71,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x33,0x01,0x00,0x00,0x32,0x01,0x00,0x00, +0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x38,0x01,0x00,0x00, +0x33,0x01,0x00,0x00,0xfd,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x3c,0x01,0x00,0x00,0x24,0x01,0x00,0x00, +0xc5,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3d,0x01,0x00,0x00, +0x38,0x01,0x00,0x00,0x3c,0x01,0x00,0x00,0x70,0x00,0x04,0x00, +0xc3,0x00,0x00,0x00,0x3e,0x01,0x00,0x00,0x3d,0x01,0x00,0x00, +0xc2,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x40,0x01,0x00,0x00, +0x33,0x01,0x00,0x00,0xa8,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x43,0x01,0x00,0x00,0x2b,0x01,0x00,0x00, +0xc5,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x44,0x01,0x00,0x00, +0x40,0x01,0x00,0x00,0x43,0x01,0x00,0x00,0x70,0x00,0x04,0x00, +0xc3,0x00,0x00,0x00,0x45,0x01,0x00,0x00,0x44,0x01,0x00,0x00, +0x50,0x00,0x05,0x00,0x34,0x01,0x00,0x00,0x46,0x01,0x00,0x00, +0x3e,0x01,0x00,0x00,0x45,0x01,0x00,0x00,0x83,0x00,0x05,0x00, +0x34,0x01,0x00,0x00,0x49,0x01,0x00,0x00,0x46,0x01,0x00,0x00, +0x46,0x03,0x00,0x00,0x8e,0x00,0x05,0x00,0x34,0x01,0x00,0x00, +0x4b,0x01,0x00,0x00,0x49,0x01,0x00,0x00,0x0e,0x01,0x00,0x00, +0x51,0x00,0x05,0x00,0xc3,0x00,0x00,0x00,0x53,0x01,0x00,0x00, +0x4b,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x73,0x00,0x04,0x00, +0x00,0x01,0x00,0x00,0x54,0x01,0x00,0x00,0x53,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0x55,0x01,0x00,0x00,0x56,0x01,0x00,0x00, +0x50,0x01,0x00,0x00,0xf6,0x00,0x00,0x00,0x3e,0x00,0x03,0x00, +0x56,0x01,0x00,0x00,0x54,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x58,0x01,0x00,0x00,0xf6,0x00,0x00,0x00, +0xf9,0x00,0x00,0x00,0x51,0x00,0x05,0x00,0xc3,0x00,0x00,0x00, +0x5a,0x01,0x00,0x00,0x4b,0x01,0x00,0x00,0x01,0x00,0x00,0x00, +0x73,0x00,0x04,0x00,0x00,0x01,0x00,0x00,0x5b,0x01,0x00,0x00, +0x5a,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0x55,0x01,0x00,0x00, +0x5c,0x01,0x00,0x00,0x50,0x01,0x00,0x00,0x58,0x01,0x00,0x00, +0x3e,0x00,0x03,0x00,0x5c,0x01,0x00,0x00,0x5b,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x63,0x01,0x00,0x00, +0xfb,0x02,0x00,0x00,0x61,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0xdc,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xde,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x65,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x65,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xfc,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0xde,0x00,0x00,0x00, +0xac,0x01,0x00,0x00,0x68,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc1,0x00,0x00,0x00,0x6b,0x01,0x00,0x00,0xfc,0x02,0x00,0x00, +0xa6,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x67,0x01,0x00,0x00, +0x68,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x6b,0x01,0x00,0x00,0x66,0x01,0x00,0x00,0x67,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x66,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x6f,0x01,0x00,0x00,0xa7,0x00,0x00,0x00, +0x7d,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x71,0x01,0x00,0x00,0x6f,0x01,0x00,0x00,0xfc,0x02,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x72,0x01,0x00,0x00, +0x12,0x00,0x00,0x00,0xcf,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x73,0x01,0x00,0x00,0x72,0x01,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00,0x74,0x01,0x00,0x00, +0x71,0x01,0x00,0x00,0x73,0x01,0x00,0x00,0xf7,0x00,0x03,0x00, +0x76,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x74,0x01,0x00,0x00,0x75,0x01,0x00,0x00,0x76,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x75,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x79,0x01,0x00,0x00,0xeb,0x02,0x00,0x00, +0x78,0x00,0x00,0x00,0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00, +0x7b,0x01,0x00,0x00,0x79,0x01,0x00,0x00,0x8d,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x76,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x76,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0xc1,0x00,0x00,0x00, +0x7c,0x01,0x00,0x00,0x74,0x01,0x00,0x00,0x66,0x01,0x00,0x00, +0x7b,0x01,0x00,0x00,0x75,0x01,0x00,0x00,0xf7,0x00,0x03,0x00, +0x7e,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x7c,0x01,0x00,0x00,0x7d,0x01,0x00,0x00,0x9e,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x7d,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x86,0x01,0x00,0x00,0x7d,0x00,0x00,0x00, +0xfc,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x88,0x01,0x00,0x00,0x86,0x01,0x00,0x00,0x87,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8a,0x01,0x00,0x00, +0x88,0x01,0x00,0x00,0x78,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x95,0x01,0x00,0x00,0x86,0x01,0x00,0x00, +0xaa,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x96,0x01,0x00,0x00,0x03,0x03,0x00,0x00,0x95,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x98,0x01,0x00,0x00, +0x96,0x01,0x00,0x00,0x78,0x00,0x00,0x00,0x41,0x00,0x06,0x00, +0x99,0x01,0x00,0x00,0x9a,0x01,0x00,0x00,0x8e,0x01,0x00,0x00, +0x34,0x00,0x00,0x00,0x98,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, +0xc3,0x00,0x00,0x00,0x9b,0x01,0x00,0x00,0x9a,0x01,0x00,0x00, +0x73,0x00,0x04,0x00,0x00,0x01,0x00,0x00,0x9c,0x01,0x00,0x00, +0x9b,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0x55,0x01,0x00,0x00, +0x9d,0x01,0x00,0x00,0x83,0x01,0x00,0x00,0x8a,0x01,0x00,0x00, +0x3e,0x00,0x03,0x00,0x9d,0x01,0x00,0x00,0x9c,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0x7e,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x9e,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xa1,0x01,0x00,0x00,0x7d,0x00,0x00,0x00,0xfc,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa3,0x01,0x00,0x00, +0xa1,0x01,0x00,0x00,0xa2,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xa5,0x01,0x00,0x00,0xa3,0x01,0x00,0x00, +0x78,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x55,0x01,0x00,0x00, +0xa7,0x01,0x00,0x00,0x83,0x01,0x00,0x00,0xa5,0x01,0x00,0x00, +0x3e,0x00,0x03,0x00,0xa7,0x01,0x00,0x00,0xa6,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0x7e,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x7e,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x68,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x68,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xac,0x01,0x00,0x00,0xfc,0x02,0x00,0x00, +0xaa,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x65,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x67,0x01,0x00,0x00,0xe0,0x00,0x04,0x00, +0x0c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0xad,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xb0,0x01,0x00,0x00, +0xff,0x02,0x00,0x00,0xae,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xb3,0x01,0x00,0x00,0x03,0x03,0x00,0x00, +0xb1,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xb5,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xb5,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0x05,0x03,0x00,0x00,0x3e,0x00,0x00,0x00, +0x67,0x01,0x00,0x00,0x5f,0x02,0x00,0x00,0xb8,0x01,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00,0xbb,0x01,0x00,0x00, +0x05,0x03,0x00,0x00,0x6c,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0xb7,0x01,0x00,0x00,0xb8,0x01,0x00,0x00,0x00,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0xbb,0x01,0x00,0x00,0xb6,0x01,0x00,0x00, +0xb7,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xb6,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0xbd,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xbd,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0x09,0x03,0x00,0x00,0x3e,0x00,0x00,0x00,0xb6,0x01,0x00,0x00, +0xe9,0x01,0x00,0x00,0xc0,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc1,0x00,0x00,0x00,0xc3,0x01,0x00,0x00,0x09,0x03,0x00,0x00, +0x60,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xbf,0x01,0x00,0x00, +0xc0,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0xc3,0x01,0x00,0x00,0xbe,0x01,0x00,0x00,0xbf,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xbe,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0xc5,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xc5,0x01,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x1b,0x03,0x00,0x00, +0x3e,0x00,0x00,0x00,0xbe,0x01,0x00,0x00,0xe7,0x01,0x00,0x00, +0xc6,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00, +0xcb,0x01,0x00,0x00,0x1b,0x03,0x00,0x00,0x62,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0xc7,0x01,0x00,0x00,0xc6,0x01,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xcb,0x01,0x00,0x00, +0xc6,0x01,0x00,0x00,0xc7,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xc6,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xd1,0x01,0x00,0x00,0x09,0x03,0x00,0x00,0x62,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xd3,0x01,0x00,0x00, +0xd1,0x01,0x00,0x00,0x1b,0x03,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xd5,0x01,0x00,0x00,0x55,0x00,0x00,0x00, +0x53,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xd7,0x01,0x00,0x00,0x09,0x03,0x00,0x00,0x61,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xd8,0x01,0x00,0x00, +0xd5,0x01,0x00,0x00,0xd7,0x01,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xda,0x01,0x00,0x00,0x64,0x00,0x00,0x00, +0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xdb,0x01,0x00,0x00,0xd8,0x01,0x00,0x00,0xda,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xdd,0x01,0x00,0x00, +0xdb,0x01,0x00,0x00,0x1b,0x03,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xdf,0x01,0x00,0x00,0xdd,0x01,0x00,0x00, +0xde,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xe1,0x01,0x00,0x00,0xdf,0x01,0x00,0x00,0x05,0x03,0x00,0x00, +0x41,0x00,0x05,0x00,0x55,0x01,0x00,0x00,0xe2,0x01,0x00,0x00, +0x50,0x01,0x00,0x00,0xe1,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, +0x00,0x01,0x00,0x00,0xe3,0x01,0x00,0x00,0xe2,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0xe4,0x01,0x00,0x00,0xe5,0x01,0x00,0x00, +0xcf,0x01,0x00,0x00,0xd3,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, +0xe5,0x01,0x00,0x00,0xe3,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xe7,0x01,0x00,0x00,0x1b,0x03,0x00,0x00, +0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xc5,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xc7,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0xc0,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xc0,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xe9,0x01,0x00,0x00, +0x09,0x03,0x00,0x00,0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xbd,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xbf,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0xeb,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xeb,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0x0a,0x03,0x00,0x00,0x3e,0x00,0x00,0x00,0xbf,0x01,0x00,0x00, +0x17,0x02,0x00,0x00,0xee,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc1,0x00,0x00,0x00,0xf1,0x01,0x00,0x00,0x0a,0x03,0x00,0x00, +0xbe,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xed,0x01,0x00,0x00, +0xee,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0xf1,0x01,0x00,0x00,0xec,0x01,0x00,0x00,0xed,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xec,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0xf3,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xf3,0x01,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x18,0x03,0x00,0x00, +0x3e,0x00,0x00,0x00,0xec,0x01,0x00,0x00,0x15,0x02,0x00,0x00, +0xf4,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00, +0xf9,0x01,0x00,0x00,0x18,0x03,0x00,0x00,0xbb,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0xf5,0x01,0x00,0x00,0xf4,0x01,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xf9,0x01,0x00,0x00, +0xf4,0x01,0x00,0x00,0xf5,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xf4,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xff,0x01,0x00,0x00,0x0a,0x03,0x00,0x00,0xbb,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x01,0x02,0x00,0x00, +0xff,0x01,0x00,0x00,0x18,0x03,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x03,0x02,0x00,0x00,0x59,0x00,0x00,0x00, +0xb8,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x06,0x02,0x00,0x00,0x0a,0x03,0x00,0x00,0x05,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x07,0x02,0x00,0x00, +0x03,0x02,0x00,0x00,0x06,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x09,0x02,0x00,0x00,0x68,0x00,0x00,0x00, +0xbb,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x0a,0x02,0x00,0x00,0x07,0x02,0x00,0x00,0x09,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x0c,0x02,0x00,0x00, +0x0a,0x02,0x00,0x00,0x18,0x03,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x0e,0x02,0x00,0x00,0x0c,0x02,0x00,0x00, +0x0d,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x10,0x02,0x00,0x00,0x0e,0x02,0x00,0x00,0x05,0x03,0x00,0x00, +0x41,0x00,0x05,0x00,0x55,0x01,0x00,0x00,0x11,0x02,0x00,0x00, +0x83,0x01,0x00,0x00,0x10,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, +0x00,0x01,0x00,0x00,0x12,0x02,0x00,0x00,0x11,0x02,0x00,0x00, +0x41,0x00,0x05,0x00,0xe4,0x01,0x00,0x00,0x13,0x02,0x00,0x00, +0xfd,0x01,0x00,0x00,0x01,0x02,0x00,0x00,0x3e,0x00,0x03,0x00, +0x13,0x02,0x00,0x00,0x12,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x15,0x02,0x00,0x00,0x18,0x03,0x00,0x00, +0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xf3,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xf5,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0xee,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xee,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x17,0x02,0x00,0x00, +0x0a,0x03,0x00,0x00,0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xeb,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xed,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0x19,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x19,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0x0b,0x03,0x00,0x00,0x3e,0x00,0x00,0x00,0xed,0x01,0x00,0x00, +0x5d,0x02,0x00,0x00,0x1c,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc1,0x00,0x00,0x00,0x1f,0x02,0x00,0x00,0x0b,0x03,0x00,0x00, +0xbe,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x1b,0x02,0x00,0x00, +0x1c,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x1f,0x02,0x00,0x00,0x1a,0x02,0x00,0x00,0x1b,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x1a,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0x21,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x21,0x02,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x0f,0x03,0x00,0x00, +0x3e,0x00,0x00,0x00,0x1a,0x02,0x00,0x00,0x5b,0x02,0x00,0x00, +0x24,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00, +0x27,0x02,0x00,0x00,0x0f,0x03,0x00,0x00,0x60,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0x23,0x02,0x00,0x00,0x24,0x02,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x27,0x02,0x00,0x00, +0x22,0x02,0x00,0x00,0x23,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x22,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x29,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x29,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0x11,0x03,0x00,0x00,0x3e,0x00,0x00,0x00, +0x22,0x02,0x00,0x00,0x59,0x02,0x00,0x00,0x2c,0x02,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00,0x2f,0x02,0x00,0x00, +0x11,0x03,0x00,0x00,0xbb,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0x2b,0x02,0x00,0x00,0x2c,0x02,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x2f,0x02,0x00,0x00,0x2a,0x02,0x00,0x00, +0x2b,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x2a,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0x31,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x31,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0x13,0x03,0x00,0x00,0x3e,0x00,0x00,0x00,0x2a,0x02,0x00,0x00, +0x57,0x02,0x00,0x00,0x32,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc1,0x00,0x00,0x00,0x37,0x02,0x00,0x00,0x13,0x03,0x00,0x00, +0x62,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x33,0x02,0x00,0x00, +0x32,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x37,0x02,0x00,0x00,0x32,0x02,0x00,0x00,0x33,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x32,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x39,0x02,0x00,0x00,0x0b,0x03,0x00,0x00, +0xbb,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x3b,0x02,0x00,0x00,0x39,0x02,0x00,0x00,0x11,0x03,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3d,0x02,0x00,0x00, +0x3b,0x02,0x00,0x00,0x3c,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x3f,0x02,0x00,0x00,0x0f,0x03,0x00,0x00, +0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x40,0x02,0x00,0x00,0x3d,0x02,0x00,0x00,0x3f,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x42,0x02,0x00,0x00, +0x40,0x02,0x00,0x00,0x13,0x03,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x46,0x02,0x00,0x00,0x3f,0x02,0x00,0x00, +0x13,0x03,0x00,0x00,0x41,0x00,0x05,0x00,0xe4,0x01,0x00,0x00, +0x47,0x02,0x00,0x00,0xcf,0x01,0x00,0x00,0x46,0x02,0x00,0x00, +0x3d,0x00,0x04,0x00,0x00,0x01,0x00,0x00,0x48,0x02,0x00,0x00, +0x47,0x02,0x00,0x00,0x73,0x00,0x04,0x00,0xc3,0x00,0x00,0x00, +0x49,0x02,0x00,0x00,0x48,0x02,0x00,0x00,0x41,0x00,0x05,0x00, +0xe4,0x01,0x00,0x00,0x4e,0x02,0x00,0x00,0xfd,0x01,0x00,0x00, +0x3b,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0x00,0x01,0x00,0x00, +0x4f,0x02,0x00,0x00,0x4e,0x02,0x00,0x00,0x73,0x00,0x04,0x00, +0xc3,0x00,0x00,0x00,0x50,0x02,0x00,0x00,0x4f,0x02,0x00,0x00, +0x41,0x00,0x05,0x00,0xcc,0x00,0x00,0x00,0x52,0x02,0x00,0x00, +0xc9,0x00,0x00,0x00,0x42,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, +0xc3,0x00,0x00,0x00,0x53,0x02,0x00,0x00,0x52,0x02,0x00,0x00, +0x0c,0x00,0x08,0x00,0xc3,0x00,0x00,0x00,0x54,0x02,0x00,0x00, +0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x49,0x02,0x00,0x00, +0x50,0x02,0x00,0x00,0x53,0x02,0x00,0x00,0x3e,0x00,0x03,0x00, +0x52,0x02,0x00,0x00,0x54,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x57,0x02,0x00,0x00,0x13,0x03,0x00,0x00, +0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x31,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x33,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0x2c,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x2c,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x59,0x02,0x00,0x00, +0x11,0x03,0x00,0x00,0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x29,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x2b,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0x24,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x24,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x5b,0x02,0x00,0x00,0x0f,0x03,0x00,0x00,0xcf,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x21,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x23,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x1c,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x1c,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x5d,0x02,0x00,0x00,0x0b,0x03,0x00,0x00, +0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x19,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x1b,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0xb8,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xb8,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x5f,0x02,0x00,0x00, +0x05,0x03,0x00,0x00,0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xb5,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xb7,0x01,0x00,0x00, +0xe0,0x00,0x04,0x00,0x0c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0xad,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xd6,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xd6,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x61,0x02,0x00,0x00,0xeb,0x02,0x00,0x00, +0x6c,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xd3,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xd5,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x66,0x02,0x00,0x00,0x55,0x00,0x00,0x00, +0x53,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x67,0x02,0x00,0x00,0x95,0x00,0x00,0x00,0x66,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x6c,0x02,0x00,0x00, +0x59,0x00,0x00,0x00,0xb8,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x6d,0x02,0x00,0x00,0xa7,0x00,0x00,0x00, +0x6c,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, +0x72,0x02,0x00,0x00,0x12,0x00,0x00,0x00,0x71,0x02,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x73,0x02,0x00,0x00, +0x72,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x74,0x02,0x00,0x00,0x0f,0x00,0x00,0x00,0x73,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x78,0x02,0x00,0x00, +0x47,0x00,0x00,0x00,0x73,0x02,0x00,0x00,0x41,0x00,0x05,0x00, +0x0d,0x00,0x00,0x00,0x7a,0x02,0x00,0x00,0x79,0x02,0x00,0x00, +0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x7b,0x02,0x00,0x00,0x7a,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x7c,0x02,0x00,0x00,0x78,0x02,0x00,0x00, +0x7b,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x7d,0x02,0x00,0x00,0x74,0x02,0x00,0x00,0x7c,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0x7f,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x7f,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xec,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0xd5,0x00,0x00,0x00, +0xe5,0x02,0x00,0x00,0x82,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc1,0x00,0x00,0x00,0x85,0x02,0x00,0x00,0xec,0x02,0x00,0x00, +0xbe,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x81,0x02,0x00,0x00, +0x82,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x85,0x02,0x00,0x00,0x80,0x02,0x00,0x00,0x81,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x80,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0x87,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x87,0x02,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xed,0x02,0x00,0x00, +0x3e,0x00,0x00,0x00,0x80,0x02,0x00,0x00,0xe3,0x02,0x00,0x00, +0x8a,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00, +0x8d,0x02,0x00,0x00,0xed,0x02,0x00,0x00,0x60,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0x89,0x02,0x00,0x00,0x8a,0x02,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x8d,0x02,0x00,0x00, +0x88,0x02,0x00,0x00,0x89,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x88,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x91,0x02,0x00,0x00,0xed,0x02,0x00,0x00,0x61,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x92,0x02,0x00,0x00, +0x67,0x02,0x00,0x00,0x91,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x94,0x02,0x00,0x00,0x64,0x00,0x00,0x00, +0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x95,0x02,0x00,0x00,0x92,0x02,0x00,0x00,0x94,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x99,0x02,0x00,0x00, +0xec,0x02,0x00,0x00,0x05,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x9a,0x02,0x00,0x00,0x6d,0x02,0x00,0x00, +0x99,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x9c,0x02,0x00,0x00,0x68,0x00,0x00,0x00,0xbb,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9d,0x02,0x00,0x00, +0x9a,0x02,0x00,0x00,0x9c,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0x9f,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x9f,0x02,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xef,0x02,0x00,0x00, +0x3e,0x00,0x00,0x00,0x88,0x02,0x00,0x00,0xe1,0x02,0x00,0x00, +0xa2,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00, +0xa5,0x02,0x00,0x00,0xef,0x02,0x00,0x00,0xbb,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0xa1,0x02,0x00,0x00,0xa2,0x02,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xa5,0x02,0x00,0x00, +0xa0,0x02,0x00,0x00,0xa1,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0xa0,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0xa7,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0xa7,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xf1,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, +0xa0,0x02,0x00,0x00,0xdf,0x02,0x00,0x00,0xaa,0x02,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00,0xad,0x02,0x00,0x00, +0xf1,0x02,0x00,0x00,0x62,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0xa9,0x02,0x00,0x00,0xaa,0x02,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0xad,0x02,0x00,0x00,0xa8,0x02,0x00,0x00, +0xa9,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0xa8,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xb0,0x02,0x00,0x00, +0x95,0x02,0x00,0x00,0xf1,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc1,0x00,0x00,0x00,0xb3,0x02,0x00,0x00,0xb0,0x02,0x00,0x00, +0x36,0x00,0x00,0x00,0xf7,0x00,0x03,0x00,0xb5,0x02,0x00,0x00, +0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xb3,0x02,0x00,0x00, +0xb4,0x02,0x00,0x00,0xb5,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0xb4,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xb8,0x02,0x00,0x00,0x9d,0x02,0x00,0x00,0xef,0x02,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0xb9,0x02,0x00,0x00, +0x12,0x00,0x00,0x00,0xcf,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xba,0x02,0x00,0x00,0xb9,0x02,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00,0xbb,0x02,0x00,0x00, +0xb8,0x02,0x00,0x00,0xba,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0xb5,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0xb5,0x02,0x00,0x00, +0xf5,0x00,0x07,0x00,0xc1,0x00,0x00,0x00,0xbc,0x02,0x00,0x00, +0xb3,0x02,0x00,0x00,0xa8,0x02,0x00,0x00,0xbb,0x02,0x00,0x00, +0xb4,0x02,0x00,0x00,0xf7,0x00,0x03,0x00,0xbe,0x02,0x00,0x00, +0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xbc,0x02,0x00,0x00, +0xbd,0x02,0x00,0x00,0xbe,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0xbd,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xc6,0x02,0x00,0x00,0x9d,0x02,0x00,0x00,0xef,0x02,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0xc8,0x02,0x00,0x00, +0x12,0x00,0x00,0x00,0xc7,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xc9,0x02,0x00,0x00,0xc8,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xca,0x02,0x00,0x00, +0xc6,0x02,0x00,0x00,0xc9,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xcb,0x02,0x00,0x00,0x7d,0x02,0x00,0x00, +0xca,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xcd,0x02,0x00,0x00,0xcb,0x02,0x00,0x00,0x95,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xcf,0x02,0x00,0x00, +0xcd,0x02,0x00,0x00,0xf1,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xd1,0x02,0x00,0x00,0xec,0x02,0x00,0x00, +0xbb,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xd3,0x02,0x00,0x00,0xd1,0x02,0x00,0x00,0xef,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xd5,0x02,0x00,0x00, +0xd3,0x02,0x00,0x00,0xd4,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xd7,0x02,0x00,0x00,0xed,0x02,0x00,0x00, +0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xd8,0x02,0x00,0x00,0xd5,0x02,0x00,0x00,0xd7,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xda,0x02,0x00,0x00, +0xd8,0x02,0x00,0x00,0xf1,0x02,0x00,0x00,0x41,0x00,0x05,0x00, +0xcc,0x00,0x00,0x00,0xdb,0x02,0x00,0x00,0xc9,0x00,0x00,0x00, +0xda,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0xc3,0x00,0x00,0x00, +0xdc,0x02,0x00,0x00,0xdb,0x02,0x00,0x00,0x41,0x00,0x06,0x00, +0x99,0x01,0x00,0x00,0xdd,0x02,0x00,0x00,0xc2,0x02,0x00,0x00, +0x34,0x00,0x00,0x00,0xcf,0x02,0x00,0x00,0x3e,0x00,0x03,0x00, +0xdd,0x02,0x00,0x00,0xdc,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0xbe,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0xbe,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0xaa,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0xaa,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xdf,0x02,0x00,0x00,0xf1,0x02,0x00,0x00,0xcf,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xa7,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0xa9,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0xa2,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0xa2,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xe1,0x02,0x00,0x00,0xef,0x02,0x00,0x00, +0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x9f,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0xa1,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0x8a,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x8a,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xe3,0x02,0x00,0x00, +0xed,0x02,0x00,0x00,0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x87,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x89,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0x82,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x82,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xe5,0x02,0x00,0x00,0xec,0x02,0x00,0x00,0xcf,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x7f,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x81,0x02,0x00,0x00,0xfd,0x00,0x01,0x00,0x38,0x00,0x01,0x00, + +}; +const uint64_t matmul_q5_0_f32_len = 11076; + +unsigned char matmul_q5_0_f32_aligned_data[] = { +0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, +0x62,0x03,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, +0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x09,0x00,0x00,0x00, +0x11,0x00,0x02,0x00,0x51,0x11,0x00,0x00,0x11,0x00,0x02,0x00, +0x60,0x11,0x00,0x00,0x0b,0x00,0x06,0x00,0x01,0x00,0x00,0x00, +0x47,0x4c,0x53,0x4c,0x2e,0x73,0x74,0x64,0x2e,0x34,0x35,0x30, +0x00,0x00,0x00,0x00,0x0e,0x00,0x03,0x00,0x00,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x0f,0x00,0x0f,0x00,0x05,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x6d,0x61,0x69,0x6e,0x00,0x00,0x00,0x00, +0x0b,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x3d,0x00,0x00,0x00, +0x4c,0x00,0x00,0x00,0x0a,0x01,0x00,0x00,0x51,0x01,0x00,0x00, +0x86,0x01,0x00,0x00,0x8e,0x01,0x00,0x00,0x9c,0x02,0x00,0x00, +0xe5,0x02,0x00,0x00,0x10,0x00,0x06,0x00,0x04,0x00,0x00,0x00, +0x11,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x0b,0x00,0x00,0x00, +0x0b,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x10,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x10,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x10,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x05,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x10,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, +0x07,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x1c,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x08,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x10,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x24,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, +0x0a,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x28,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x2c,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x10,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x30,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, +0x0d,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x34,0x00,0x00,0x00, +0x47,0x00,0x03,0x00,0x10,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x37,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x3d,0x00,0x00,0x00, +0x0b,0x00,0x00,0x00,0x1a,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x4c,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x1b,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x4f,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x09,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x53,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x60,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x62,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x07,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x6c,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0xa7,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0xb9,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x05,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xbc,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x03,0x01,0x00,0x00,0x06,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x05,0x01,0x00,0x00,0x06,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x06,0x01,0x00,0x00, +0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x06,0x01,0x00,0x00,0x01,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x06,0x01,0x00,0x00,0x02,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x07,0x01,0x00,0x00, +0x06,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x48,0x00,0x04,0x00, +0x08,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x08,0x01,0x00,0x00,0x00,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, +0x08,0x01,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x0a,0x01,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x0a,0x01,0x00,0x00,0x21,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x5e,0x01,0x00,0x00, +0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x5f,0x01,0x00,0x00,0x0b,0x00,0x00,0x00,0x19,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x8b,0x01,0x00,0x00,0x06,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0x8c,0x01,0x00,0x00, +0x00,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0x48,0x00,0x04,0x00, +0x8c,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x8c,0x01,0x00,0x00,0x00,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x8c,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x07,0x00,0x00,0x00, +0x10,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x8c,0x01,0x00,0x00, +0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x8e,0x01,0x00,0x00, +0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x8e,0x01,0x00,0x00,0x21,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x9c,0x02,0x00,0x00,0x0b,0x00,0x00,0x00, +0x18,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xe2,0x02,0x00,0x00, +0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00, +0xe3,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x19,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0xe3,0x02,0x00,0x00,0x00,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, +0xe3,0x02,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0xe5,0x02,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0xe5,0x02,0x00,0x00,0x21,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x13,0x00,0x02,0x00,0x02,0x00,0x00,0x00, +0x21,0x00,0x03,0x00,0x03,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x15,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x17,0x00,0x04,0x00,0x09,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x0a,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x09,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x0d,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x1e,0x00,0x10,0x00,0x10,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x11,0x00,0x00,0x00, +0x09,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0x11,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x09,0x00,0x00,0x00, +0x15,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, +0x14,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x15,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x21,0x00,0x00,0x00, +0x0a,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, +0x27,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0x2d,0x00,0x00,0x00,0x07,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x34,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x37,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00,0x3d,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x3e,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0x0a,0x00,0x00,0x00,0x4c,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x4f,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x53,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x54,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0x37,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x58,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0x37,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x32,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x61,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x60,0x00,0x00,0x00, +0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x62,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x63,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x61,0x00,0x00,0x00, +0x62,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x67,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x61,0x00,0x00,0x00, +0x62,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x72,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x77,0x00,0x00,0x00,0x08,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x78,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x77,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x7d,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x77,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x81,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0x91,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x97,0x00,0x00,0x00, +0x03,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, +0xa2,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x32,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xa7,0x00,0x00,0x00,0x40,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0xa9,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xb8,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00, +0x62,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xb9,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xba,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0x53,0x00,0x00,0x00,0xb9,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xbb,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0x4f,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x32,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xbc,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xbd,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0xbb,0x00,0x00,0x00,0xbc,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xbe,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0xbd,0x00,0x00,0x00,0x60,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xbf,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0xba,0x00,0x00,0x00,0xbe,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xc0,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0xb8,0x00,0x00,0x00,0xbf,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xc1,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0xc0,0x00,0x00,0x00,0xbc,0x00,0x00,0x00, +0x14,0x00,0x02,0x00,0xc2,0x00,0x00,0x00,0x16,0x00,0x03,0x00, +0xc4,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xc5,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xc6,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0xc5,0x00,0x00,0x00,0xbf,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xc7,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0xc6,0x00,0x00,0x00,0xbc,0x00,0x00,0x00,0x1c,0x00,0x04,0x00, +0xc8,0x00,0x00,0x00,0xc4,0x00,0x00,0x00,0xc7,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0xc9,0x00,0x00,0x00,0x07,0x00,0x00,0x00, +0xc8,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0xc4,0x00,0x00,0x00, +0xcc,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0xcd,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0xc4,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0xd0,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xf4,0x00,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, +0x39,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xfa,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xfe,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, +0x16,0x00,0x03,0x00,0x01,0x01,0x00,0x00,0x10,0x00,0x00,0x00, +0x15,0x00,0x04,0x00,0x02,0x01,0x00,0x00,0x10,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x1c,0x00,0x04,0x00,0x03,0x01,0x00,0x00, +0x02,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x15,0x00,0x04,0x00, +0x04,0x01,0x00,0x00,0x08,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x1c,0x00,0x04,0x00,0x05,0x01,0x00,0x00,0x04,0x01,0x00,0x00, +0xfa,0x00,0x00,0x00,0x1e,0x00,0x05,0x00,0x06,0x01,0x00,0x00, +0x01,0x01,0x00,0x00,0x03,0x01,0x00,0x00,0x05,0x01,0x00,0x00, +0x1d,0x00,0x03,0x00,0x07,0x01,0x00,0x00,0x06,0x01,0x00,0x00, +0x1e,0x00,0x03,0x00,0x08,0x01,0x00,0x00,0x07,0x01,0x00,0x00, +0x20,0x00,0x04,0x00,0x09,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, +0x08,0x01,0x00,0x00,0x3b,0x00,0x04,0x00,0x09,0x01,0x00,0x00, +0x0a,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x0c,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x01,0x01,0x00,0x00, +0x20,0x00,0x04,0x00,0x12,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, +0x02,0x01,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, +0x16,0x01,0x00,0x00,0x10,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x28,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x31,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, +0x04,0x01,0x00,0x00,0x17,0x00,0x04,0x00,0x35,0x01,0x00,0x00, +0xc4,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0xc4,0x00,0x00,0x00,0x48,0x01,0x00,0x00,0x00,0x00,0x80,0x41, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x4d,0x01,0x00,0x00, +0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x4e,0x01,0x00,0x00, +0x84,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x4d,0x01,0x00,0x00, +0x1c,0x00,0x04,0x00,0x4f,0x01,0x00,0x00,0x01,0x01,0x00,0x00, +0x4e,0x01,0x00,0x00,0x20,0x00,0x04,0x00,0x50,0x01,0x00,0x00, +0x04,0x00,0x00,0x00,0x4f,0x01,0x00,0x00,0x3b,0x00,0x04,0x00, +0x50,0x01,0x00,0x00,0x51,0x01,0x00,0x00,0x04,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x56,0x01,0x00,0x00,0x04,0x00,0x00,0x00, +0x01,0x01,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x5e,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x33,0x00,0x06,0x00, +0x09,0x00,0x00,0x00,0x5f,0x01,0x00,0x00,0x5e,0x01,0x00,0x00, +0x39,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x60,0x01,0x00,0x00,0x51,0x00,0x00,0x00, +0x5f,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x61,0x01,0x00,0x00,0x84,0x00,0x00,0x00, +0x60,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x62,0x01,0x00,0x00,0x86,0x00,0x00,0x00, +0x61,0x01,0x00,0x00,0x6c,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x7d,0x01,0x00,0x00,0x80,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x82,0x01,0x00,0x00,0x80,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x83,0x01,0x00,0x00,0x84,0x00,0x00,0x00, +0xa7,0x00,0x00,0x00,0x82,0x01,0x00,0x00,0x1c,0x00,0x04,0x00, +0x84,0x01,0x00,0x00,0x01,0x01,0x00,0x00,0x83,0x01,0x00,0x00, +0x20,0x00,0x04,0x00,0x85,0x01,0x00,0x00,0x04,0x00,0x00,0x00, +0x84,0x01,0x00,0x00,0x3b,0x00,0x04,0x00,0x85,0x01,0x00,0x00, +0x86,0x01,0x00,0x00,0x04,0x00,0x00,0x00,0x17,0x00,0x04,0x00, +0x89,0x01,0x00,0x00,0xc4,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x18,0x00,0x04,0x00,0x8a,0x01,0x00,0x00,0x89,0x01,0x00,0x00, +0x02,0x00,0x00,0x00,0x1d,0x00,0x03,0x00,0x8b,0x01,0x00,0x00, +0x8a,0x01,0x00,0x00,0x1e,0x00,0x03,0x00,0x8c,0x01,0x00,0x00, +0x8b,0x01,0x00,0x00,0x20,0x00,0x04,0x00,0x8d,0x01,0x00,0x00, +0x0c,0x00,0x00,0x00,0x8c,0x01,0x00,0x00,0x3b,0x00,0x04,0x00, +0x8d,0x01,0x00,0x00,0x8e,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x90,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, +0xc4,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xa4,0x01,0x00,0x00,0x03,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xac,0x01,0x00,0x00,0x04,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xb4,0x01,0x00,0x00, +0x05,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xbc,0x01,0x00,0x00,0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xc4,0x01,0x00,0x00,0x07,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xcb,0x01,0x00,0x00, +0x51,0x00,0x00,0x00,0x5f,0x01,0x00,0x00,0x00,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xcc,0x01,0x00,0x00, +0x84,0x00,0x00,0x00,0xcb,0x01,0x00,0x00,0x77,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xcd,0x01,0x00,0x00, +0x86,0x00,0x00,0x00,0xcc,0x01,0x00,0x00,0x6c,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xd0,0x01,0x00,0x00, +0x08,0x01,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xd1,0x01,0x00,0x00,0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xd4,0x01,0x00,0x00,0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, +0x77,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xef,0x01,0x00,0x00,0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00, +0x62,0x00,0x00,0x00,0x1c,0x00,0x04,0x00,0xf0,0x01,0x00,0x00, +0x01,0x01,0x00,0x00,0xef,0x01,0x00,0x00,0x20,0x00,0x04,0x00, +0xf1,0x01,0x00,0x00,0x07,0x00,0x00,0x00,0xf0,0x01,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x01,0x02,0x00,0x00, +0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x07,0x02,0x00,0x00,0x07,0x00,0x00,0x00, +0x01,0x01,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x1d,0x02,0x00,0x00,0x84,0x00,0x00,0x00,0xbf,0x00,0x00,0x00, +0xbc,0x00,0x00,0x00,0x1c,0x00,0x04,0x00,0x1e,0x02,0x00,0x00, +0x01,0x01,0x00,0x00,0x1d,0x02,0x00,0x00,0x20,0x00,0x04,0x00, +0x1f,0x02,0x00,0x00,0x07,0x00,0x00,0x00,0x1e,0x02,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x28,0x02,0x00,0x00, +0x86,0x00,0x00,0x00,0xb9,0x00,0x00,0x00,0xbf,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x30,0x02,0x00,0x00, +0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x5f,0x02,0x00,0x00, +0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x94,0x02,0x00,0x00, +0x0d,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00, +0x9c,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, +0xe2,0x02,0x00,0x00,0xc4,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, +0xe3,0x02,0x00,0x00,0xe2,0x02,0x00,0x00,0x20,0x00,0x04,0x00, +0xe4,0x02,0x00,0x00,0x0c,0x00,0x00,0x00,0xe3,0x02,0x00,0x00, +0x3b,0x00,0x04,0x00,0xe4,0x02,0x00,0x00,0xe5,0x02,0x00,0x00, +0x0c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, +0xea,0x02,0x00,0x00,0x05,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xf7,0x02,0x00,0x00,0x84,0x00,0x00,0x00, +0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x2c,0x00,0x05,0x00, +0x35,0x01,0x00,0x00,0x61,0x03,0x00,0x00,0x48,0x01,0x00,0x00, +0x48,0x01,0x00,0x00,0x36,0x00,0x05,0x00,0x02,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x03,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0x05,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0xc9,0x00,0x00,0x00,0xca,0x00,0x00,0x00,0x07,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0xf1,0x01,0x00,0x00,0xf2,0x01,0x00,0x00, +0x07,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x1f,0x02,0x00,0x00, +0x20,0x02,0x00,0x00,0x07,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x0d,0x00,0x00,0x00,0x0e,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x0f,0x00,0x00,0x00,0x0e,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x15,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0x14,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x17,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, +0x17,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x1e,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x17,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x22,0x00,0x00,0x00, +0x12,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x22,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x24,0x00,0x00,0x00, +0x18,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x15,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0x27,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x29,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x2a,0x00,0x00,0x00,0x1e,0x00,0x00,0x00, +0x29,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, +0x2e,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x2d,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x2f,0x00,0x00,0x00, +0x2e,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x30,0x00,0x00,0x00,0x24,0x00,0x00,0x00,0x2f,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x32,0x00,0x00,0x00, +0x30,0x00,0x00,0x00,0x2a,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x15,0x00,0x00,0x00,0x35,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0x34,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x36,0x00,0x00,0x00,0x35,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x38,0x00,0x00,0x00,0x36,0x00,0x00,0x00, +0x37,0x00,0x00,0x00,0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x3a,0x00,0x00,0x00,0x38,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3b,0x00,0x00,0x00, +0x3a,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x0d,0x00,0x00,0x00,0x3f,0x00,0x00,0x00,0x3d,0x00,0x00,0x00, +0x3e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x40,0x00,0x00,0x00,0x3f,0x00,0x00,0x00,0x89,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x42,0x00,0x00,0x00,0x40,0x00,0x00,0x00, +0x3b,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x47,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x3b,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00,0x49,0x00,0x00,0x00, +0x3d,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x4a,0x00,0x00,0x00,0x49,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00,0x4d,0x00,0x00,0x00, +0x4c,0x00,0x00,0x00,0x3e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x4d,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x50,0x00,0x00,0x00, +0x4e,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x89,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x55,0x00,0x00,0x00,0x50,0x00,0x00,0x00, +0x54,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x59,0x00,0x00,0x00,0x50,0x00,0x00,0x00,0x58,0x00,0x00,0x00, +0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x5d,0x00,0x00,0x00, +0x4e,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x89,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x64,0x00,0x00,0x00,0x5d,0x00,0x00,0x00, +0x63,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x68,0x00,0x00,0x00,0x5d,0x00,0x00,0x00,0x67,0x00,0x00,0x00, +0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x6e,0x00,0x00,0x00, +0x4e,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x73,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, +0x72,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x79,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x78,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x7e,0x00,0x00,0x00, +0x4e,0x00,0x00,0x00,0x7d,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x15,0x00,0x00,0x00,0x82,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0x81,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x83,0x00,0x00,0x00,0x82,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x47,0x00,0x00,0x00, +0x83,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, +0x87,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x88,0x00,0x00,0x00, +0x87,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x8a,0x00,0x00,0x00,0x47,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8d,0x00,0x00,0x00, +0x8a,0x00,0x00,0x00,0x83,0x00,0x00,0x00,0x0c,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0x8e,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x26,0x00,0x00,0x00,0x88,0x00,0x00,0x00,0x8d,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x92,0x00,0x00,0x00, +0x12,0x00,0x00,0x00,0x91,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x93,0x00,0x00,0x00,0x92,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x94,0x00,0x00,0x00, +0x32,0x00,0x00,0x00,0x93,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x96,0x00,0x00,0x00,0x42,0x00,0x00,0x00, +0x37,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, +0x98,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x97,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x99,0x00,0x00,0x00, +0x98,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x9a,0x00,0x00,0x00,0x96,0x00,0x00,0x00,0x99,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9b,0x00,0x00,0x00, +0x94,0x00,0x00,0x00,0x9a,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x9d,0x00,0x00,0x00,0x9b,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x9e,0x00,0x00,0x00,0x9d,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0xa3,0x00,0x00,0x00, +0x12,0x00,0x00,0x00,0xa2,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xa4,0x00,0x00,0x00,0xa3,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa5,0x00,0x00,0x00, +0x0f,0x00,0x00,0x00,0xa4,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xa8,0x00,0x00,0x00,0x4a,0x00,0x00,0x00, +0xa7,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, +0xaa,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0xa9,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xab,0x00,0x00,0x00, +0xaa,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xac,0x00,0x00,0x00,0xa8,0x00,0x00,0x00,0xab,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xad,0x00,0x00,0x00, +0xa5,0x00,0x00,0x00,0xac,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xaf,0x00,0x00,0x00,0xad,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xb0,0x00,0x00,0x00,0xaf,0x00,0x00,0x00,0x77,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xb2,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xb2,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0x0d,0x03,0x00,0x00,0x3e,0x00,0x00,0x00,0x05,0x00,0x00,0x00, +0xd1,0x00,0x00,0x00,0xb3,0x00,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc2,0x00,0x00,0x00,0xc3,0x00,0x00,0x00,0x0d,0x03,0x00,0x00, +0xc1,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xb4,0x00,0x00,0x00, +0xb3,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0xc3,0x00,0x00,0x00,0xb3,0x00,0x00,0x00,0xb4,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xb3,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0xcd,0x00,0x00,0x00,0xce,0x00,0x00,0x00,0xca,0x00,0x00,0x00, +0x0d,0x03,0x00,0x00,0x3e,0x00,0x03,0x00,0xce,0x00,0x00,0x00, +0xcc,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xd1,0x00,0x00,0x00,0x0d,0x03,0x00,0x00,0xd0,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xb2,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xb4,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xd4,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xd4,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0x26,0x03,0x00,0x00,0xb0,0x00,0x00,0x00, +0xb4,0x00,0x00,0x00,0xd6,0x01,0x00,0x00,0xd7,0x00,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x22,0x03,0x00,0x00, +0x9e,0x00,0x00,0x00,0xb4,0x00,0x00,0x00,0xd3,0x01,0x00,0x00, +0xd7,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0x0e,0x03,0x00,0x00,0x84,0x00,0x00,0x00,0xb4,0x00,0x00,0x00, +0x84,0x02,0x00,0x00,0xd7,0x00,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc2,0x00,0x00,0x00,0xdb,0x00,0x00,0x00,0x0e,0x03,0x00,0x00, +0x8e,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xd6,0x00,0x00,0x00, +0xd7,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0xdb,0x00,0x00,0x00,0xd5,0x00,0x00,0x00,0xd6,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xd5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xdd,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xdd,0x00,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x1e,0x03,0x00,0x00, +0x3e,0x00,0x00,0x00,0xd5,0x00,0x00,0x00,0x64,0x01,0x00,0x00, +0xde,0x00,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, +0xe3,0x00,0x00,0x00,0x1e,0x03,0x00,0x00,0x37,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0xdf,0x00,0x00,0x00,0xde,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xe3,0x00,0x00,0x00, +0xde,0x00,0x00,0x00,0xdf,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xde,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xe8,0x00,0x00,0x00,0x73,0x00,0x00,0x00,0x1e,0x03,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xeb,0x00,0x00,0x00, +0xe8,0x00,0x00,0x00,0x99,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xec,0x00,0x00,0x00,0xeb,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xed,0x00,0x00,0x00,0x22,0x03,0x00,0x00,0xec,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xef,0x00,0x00,0x00, +0xed,0x00,0x00,0x00,0x6e,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xf5,0x00,0x00,0x00,0xe8,0x00,0x00,0x00, +0xf4,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xf7,0x00,0x00,0x00,0xf5,0x00,0x00,0x00,0x6e,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xfb,0x00,0x00,0x00, +0xef,0x00,0x00,0x00,0xfa,0x00,0x00,0x00,0xc7,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xff,0x00,0x00,0x00,0xef,0x00,0x00,0x00, +0xfe,0x00,0x00,0x00,0x41,0x00,0x07,0x00,0x0c,0x01,0x00,0x00, +0x0d,0x01,0x00,0x00,0x0a,0x01,0x00,0x00,0x34,0x00,0x00,0x00, +0xfb,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x01,0x01,0x00,0x00,0x0e,0x01,0x00,0x00,0x0d,0x01,0x00,0x00, +0x73,0x00,0x04,0x00,0xc4,0x00,0x00,0x00,0x0f,0x01,0x00,0x00, +0x0e,0x01,0x00,0x00,0x41,0x00,0x08,0x00,0x12,0x01,0x00,0x00, +0x13,0x01,0x00,0x00,0x0a,0x01,0x00,0x00,0x34,0x00,0x00,0x00, +0xfb,0x00,0x00,0x00,0xd0,0x00,0x00,0x00,0xd0,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x02,0x01,0x00,0x00,0x14,0x01,0x00,0x00, +0x13,0x01,0x00,0x00,0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x15,0x01,0x00,0x00,0x14,0x01,0x00,0x00,0xc4,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x17,0x01,0x00,0x00,0x15,0x01,0x00,0x00, +0x16,0x01,0x00,0x00,0x41,0x00,0x08,0x00,0x12,0x01,0x00,0x00, +0x19,0x01,0x00,0x00,0x0a,0x01,0x00,0x00,0x34,0x00,0x00,0x00, +0xfb,0x00,0x00,0x00,0xd0,0x00,0x00,0x00,0x34,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x02,0x01,0x00,0x00,0x1a,0x01,0x00,0x00, +0x19,0x01,0x00,0x00,0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x1b,0x01,0x00,0x00,0x1a,0x01,0x00,0x00,0xc5,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x1c,0x01,0x00,0x00,0x17,0x01,0x00,0x00, +0x1b,0x01,0x00,0x00,0xc2,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x22,0x01,0x00,0x00,0x1c,0x01,0x00,0x00,0xff,0x00,0x00,0x00, +0xc4,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x23,0x01,0x00,0x00, +0x22,0x01,0x00,0x00,0xa9,0x00,0x00,0x00,0xc7,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x24,0x01,0x00,0x00,0x23,0x01,0x00,0x00, +0xfa,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x13,0x00,0x00,0x00, +0x25,0x01,0x00,0x00,0x24,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x29,0x01,0x00,0x00,0xff,0x00,0x00,0x00, +0x28,0x01,0x00,0x00,0xc2,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x2a,0x01,0x00,0x00,0x1c,0x01,0x00,0x00,0x29,0x01,0x00,0x00, +0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x2b,0x01,0x00,0x00, +0x2a,0x01,0x00,0x00,0xfa,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0x2c,0x01,0x00,0x00,0x2b,0x01,0x00,0x00, +0x41,0x00,0x08,0x00,0x31,0x01,0x00,0x00,0x32,0x01,0x00,0x00, +0x0a,0x01,0x00,0x00,0x34,0x00,0x00,0x00,0xfb,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0xff,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x04,0x01,0x00,0x00,0x33,0x01,0x00,0x00,0x32,0x01,0x00,0x00, +0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x34,0x01,0x00,0x00, +0x33,0x01,0x00,0x00,0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x39,0x01,0x00,0x00,0x34,0x01,0x00,0x00,0xfe,0x00,0x00,0x00, +0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x3d,0x01,0x00,0x00, +0x25,0x01,0x00,0x00,0xc5,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x3e,0x01,0x00,0x00,0x39,0x01,0x00,0x00,0x3d,0x01,0x00,0x00, +0x70,0x00,0x04,0x00,0xc4,0x00,0x00,0x00,0x3f,0x01,0x00,0x00, +0x3e,0x01,0x00,0x00,0xc2,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x41,0x01,0x00,0x00,0x34,0x01,0x00,0x00,0xa9,0x00,0x00,0x00, +0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x44,0x01,0x00,0x00, +0x2c,0x01,0x00,0x00,0xc5,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x45,0x01,0x00,0x00,0x41,0x01,0x00,0x00,0x44,0x01,0x00,0x00, +0x70,0x00,0x04,0x00,0xc4,0x00,0x00,0x00,0x46,0x01,0x00,0x00, +0x45,0x01,0x00,0x00,0x50,0x00,0x05,0x00,0x35,0x01,0x00,0x00, +0x47,0x01,0x00,0x00,0x3f,0x01,0x00,0x00,0x46,0x01,0x00,0x00, +0x83,0x00,0x05,0x00,0x35,0x01,0x00,0x00,0x4a,0x01,0x00,0x00, +0x47,0x01,0x00,0x00,0x61,0x03,0x00,0x00,0x8e,0x00,0x05,0x00, +0x35,0x01,0x00,0x00,0x4c,0x01,0x00,0x00,0x4a,0x01,0x00,0x00, +0x0f,0x01,0x00,0x00,0x51,0x00,0x05,0x00,0xc4,0x00,0x00,0x00, +0x54,0x01,0x00,0x00,0x4c,0x01,0x00,0x00,0x00,0x00,0x00,0x00, +0x73,0x00,0x04,0x00,0x01,0x01,0x00,0x00,0x55,0x01,0x00,0x00, +0x54,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0x56,0x01,0x00,0x00, +0x57,0x01,0x00,0x00,0x51,0x01,0x00,0x00,0xf7,0x00,0x00,0x00, +0x3e,0x00,0x03,0x00,0x57,0x01,0x00,0x00,0x55,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x59,0x01,0x00,0x00, +0xf7,0x00,0x00,0x00,0xfa,0x00,0x00,0x00,0x51,0x00,0x05,0x00, +0xc4,0x00,0x00,0x00,0x5b,0x01,0x00,0x00,0x4c,0x01,0x00,0x00, +0x01,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x01,0x01,0x00,0x00, +0x5c,0x01,0x00,0x00,0x5b,0x01,0x00,0x00,0x41,0x00,0x05,0x00, +0x56,0x01,0x00,0x00,0x5d,0x01,0x00,0x00,0x51,0x01,0x00,0x00, +0x59,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x5d,0x01,0x00,0x00, +0x5c,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x64,0x01,0x00,0x00,0x1e,0x03,0x00,0x00,0x62,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0xdd,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xdf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x66,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x66,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0x1f,0x03,0x00,0x00,0x3e,0x00,0x00,0x00, +0xdf,0x00,0x00,0x00,0xcf,0x01,0x00,0x00,0x67,0x01,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0x6c,0x01,0x00,0x00, +0x1f,0x03,0x00,0x00,0xa7,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0x68,0x01,0x00,0x00,0x67,0x01,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x6c,0x01,0x00,0x00,0x67,0x01,0x00,0x00, +0x68,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x67,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x71,0x01,0x00,0x00, +0x7e,0x00,0x00,0x00,0x1f,0x03,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x74,0x01,0x00,0x00,0x71,0x01,0x00,0x00, +0xab,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x75,0x01,0x00,0x00,0x74,0x01,0x00,0x00,0x77,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x76,0x01,0x00,0x00, +0x26,0x03,0x00,0x00,0x75,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x78,0x01,0x00,0x00,0x76,0x01,0x00,0x00, +0x79,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x7e,0x01,0x00,0x00,0x71,0x01,0x00,0x00,0x7d,0x01,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x80,0x01,0x00,0x00, +0x79,0x00,0x00,0x00,0x77,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x81,0x01,0x00,0x00,0x7e,0x01,0x00,0x00, +0x80,0x01,0x00,0x00,0x41,0x00,0x08,0x00,0x90,0x01,0x00,0x00, +0x91,0x01,0x00,0x00,0x8e,0x01,0x00,0x00,0x34,0x00,0x00,0x00, +0x78,0x01,0x00,0x00,0x34,0x00,0x00,0x00,0x3e,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00,0x92,0x01,0x00,0x00, +0x91,0x01,0x00,0x00,0x73,0x00,0x04,0x00,0x01,0x01,0x00,0x00, +0x93,0x01,0x00,0x00,0x92,0x01,0x00,0x00,0x41,0x00,0x05,0x00, +0x56,0x01,0x00,0x00,0x94,0x01,0x00,0x00,0x86,0x01,0x00,0x00, +0x81,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x94,0x01,0x00,0x00, +0x93,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x96,0x01,0x00,0x00,0x81,0x01,0x00,0x00,0x39,0x00,0x00,0x00, +0x41,0x00,0x08,0x00,0x90,0x01,0x00,0x00,0x98,0x01,0x00,0x00, +0x8e,0x01,0x00,0x00,0x34,0x00,0x00,0x00,0x78,0x01,0x00,0x00, +0x34,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0xc4,0x00,0x00,0x00,0x99,0x01,0x00,0x00,0x98,0x01,0x00,0x00, +0x73,0x00,0x04,0x00,0x01,0x01,0x00,0x00,0x9a,0x01,0x00,0x00, +0x99,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0x56,0x01,0x00,0x00, +0x9b,0x01,0x00,0x00,0x86,0x01,0x00,0x00,0x96,0x01,0x00,0x00, +0x3e,0x00,0x03,0x00,0x9b,0x01,0x00,0x00,0x9a,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9d,0x01,0x00,0x00, +0x81,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x41,0x00,0x08,0x00, +0x90,0x01,0x00,0x00,0x9f,0x01,0x00,0x00,0x8e,0x01,0x00,0x00, +0x34,0x00,0x00,0x00,0x78,0x01,0x00,0x00,0x34,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00, +0xa0,0x01,0x00,0x00,0x9f,0x01,0x00,0x00,0x73,0x00,0x04,0x00, +0x01,0x01,0x00,0x00,0xa1,0x01,0x00,0x00,0xa0,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0x56,0x01,0x00,0x00,0xa2,0x01,0x00,0x00, +0x86,0x01,0x00,0x00,0x9d,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, +0xa2,0x01,0x00,0x00,0xa1,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xa5,0x01,0x00,0x00,0x81,0x01,0x00,0x00, +0xa4,0x01,0x00,0x00,0x41,0x00,0x08,0x00,0x90,0x01,0x00,0x00, +0xa7,0x01,0x00,0x00,0x8e,0x01,0x00,0x00,0x34,0x00,0x00,0x00, +0x78,0x01,0x00,0x00,0x34,0x00,0x00,0x00,0xa4,0x01,0x00,0x00, +0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00,0xa8,0x01,0x00,0x00, +0xa7,0x01,0x00,0x00,0x73,0x00,0x04,0x00,0x01,0x01,0x00,0x00, +0xa9,0x01,0x00,0x00,0xa8,0x01,0x00,0x00,0x41,0x00,0x05,0x00, +0x56,0x01,0x00,0x00,0xaa,0x01,0x00,0x00,0x86,0x01,0x00,0x00, +0xa5,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0xaa,0x01,0x00,0x00, +0xa9,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xad,0x01,0x00,0x00,0x81,0x01,0x00,0x00,0xac,0x01,0x00,0x00, +0x41,0x00,0x08,0x00,0x90,0x01,0x00,0x00,0xaf,0x01,0x00,0x00, +0x8e,0x01,0x00,0x00,0x34,0x00,0x00,0x00,0x78,0x01,0x00,0x00, +0xd0,0x00,0x00,0x00,0x3e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0xc4,0x00,0x00,0x00,0xb0,0x01,0x00,0x00,0xaf,0x01,0x00,0x00, +0x73,0x00,0x04,0x00,0x01,0x01,0x00,0x00,0xb1,0x01,0x00,0x00, +0xb0,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0x56,0x01,0x00,0x00, +0xb2,0x01,0x00,0x00,0x86,0x01,0x00,0x00,0xad,0x01,0x00,0x00, +0x3e,0x00,0x03,0x00,0xb2,0x01,0x00,0x00,0xb1,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xb5,0x01,0x00,0x00, +0x81,0x01,0x00,0x00,0xb4,0x01,0x00,0x00,0x41,0x00,0x08,0x00, +0x90,0x01,0x00,0x00,0xb7,0x01,0x00,0x00,0x8e,0x01,0x00,0x00, +0x34,0x00,0x00,0x00,0x78,0x01,0x00,0x00,0xd0,0x00,0x00,0x00, +0x39,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00, +0xb8,0x01,0x00,0x00,0xb7,0x01,0x00,0x00,0x73,0x00,0x04,0x00, +0x01,0x01,0x00,0x00,0xb9,0x01,0x00,0x00,0xb8,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0x56,0x01,0x00,0x00,0xba,0x01,0x00,0x00, +0x86,0x01,0x00,0x00,0xb5,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, +0xba,0x01,0x00,0x00,0xb9,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xbd,0x01,0x00,0x00,0x81,0x01,0x00,0x00, +0xbc,0x01,0x00,0x00,0x41,0x00,0x08,0x00,0x90,0x01,0x00,0x00, +0xbf,0x01,0x00,0x00,0x8e,0x01,0x00,0x00,0x34,0x00,0x00,0x00, +0x78,0x01,0x00,0x00,0xd0,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00,0xc0,0x01,0x00,0x00, +0xbf,0x01,0x00,0x00,0x73,0x00,0x04,0x00,0x01,0x01,0x00,0x00, +0xc1,0x01,0x00,0x00,0xc0,0x01,0x00,0x00,0x41,0x00,0x05,0x00, +0x56,0x01,0x00,0x00,0xc2,0x01,0x00,0x00,0x86,0x01,0x00,0x00, +0xbd,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0xc2,0x01,0x00,0x00, +0xc1,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xc5,0x01,0x00,0x00,0x81,0x01,0x00,0x00,0xc4,0x01,0x00,0x00, +0x41,0x00,0x08,0x00,0x90,0x01,0x00,0x00,0xc7,0x01,0x00,0x00, +0x8e,0x01,0x00,0x00,0x34,0x00,0x00,0x00,0x78,0x01,0x00,0x00, +0xd0,0x00,0x00,0x00,0xa4,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, +0xc4,0x00,0x00,0x00,0xc8,0x01,0x00,0x00,0xc7,0x01,0x00,0x00, +0x73,0x00,0x04,0x00,0x01,0x01,0x00,0x00,0xc9,0x01,0x00,0x00, +0xc8,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0x56,0x01,0x00,0x00, +0xca,0x01,0x00,0x00,0x86,0x01,0x00,0x00,0xc5,0x01,0x00,0x00, +0x3e,0x00,0x03,0x00,0xca,0x01,0x00,0x00,0xc9,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xcf,0x01,0x00,0x00, +0x1f,0x03,0x00,0x00,0xcd,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0x66,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x68,0x01,0x00,0x00, +0xe0,0x00,0x04,0x00,0x0c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0xd0,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xd3,0x01,0x00,0x00,0x22,0x03,0x00,0x00,0xd1,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xd6,0x01,0x00,0x00, +0x26,0x03,0x00,0x00,0xd4,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0xd8,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xd8,0x01,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x28,0x03,0x00,0x00, +0x3e,0x00,0x00,0x00,0x68,0x01,0x00,0x00,0x82,0x02,0x00,0x00, +0xdb,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, +0xde,0x01,0x00,0x00,0x28,0x03,0x00,0x00,0x6c,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0xda,0x01,0x00,0x00,0xdb,0x01,0x00,0x00, +0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xde,0x01,0x00,0x00, +0xd9,0x01,0x00,0x00,0xda,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xd9,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xe0,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xe0,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0x2c,0x03,0x00,0x00,0x3e,0x00,0x00,0x00, +0xd9,0x01,0x00,0x00,0x0c,0x02,0x00,0x00,0xe3,0x01,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0xe6,0x01,0x00,0x00, +0x2c,0x03,0x00,0x00,0x60,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0xe2,0x01,0x00,0x00,0xe3,0x01,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0xe6,0x01,0x00,0x00,0xe1,0x01,0x00,0x00, +0xe2,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xe1,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0xe8,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xe8,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0x3e,0x03,0x00,0x00,0x3e,0x00,0x00,0x00,0xe1,0x01,0x00,0x00, +0x0a,0x02,0x00,0x00,0xe9,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc2,0x00,0x00,0x00,0xee,0x01,0x00,0x00,0x3e,0x03,0x00,0x00, +0x62,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xea,0x01,0x00,0x00, +0xe9,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0xee,0x01,0x00,0x00,0xe9,0x01,0x00,0x00,0xea,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xe9,0x01,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xf4,0x01,0x00,0x00,0x2c,0x03,0x00,0x00, +0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xf6,0x01,0x00,0x00,0xf4,0x01,0x00,0x00,0x3e,0x03,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xf8,0x01,0x00,0x00, +0x55,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xfa,0x01,0x00,0x00,0x2c,0x03,0x00,0x00, +0x61,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xfb,0x01,0x00,0x00,0xf8,0x01,0x00,0x00,0xfa,0x01,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xfd,0x01,0x00,0x00, +0x64,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xfe,0x01,0x00,0x00,0xfb,0x01,0x00,0x00, +0xfd,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x00,0x02,0x00,0x00,0xfe,0x01,0x00,0x00,0x3e,0x03,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x02,0x02,0x00,0x00, +0x00,0x02,0x00,0x00,0x01,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x04,0x02,0x00,0x00,0x02,0x02,0x00,0x00, +0x28,0x03,0x00,0x00,0x41,0x00,0x05,0x00,0x56,0x01,0x00,0x00, +0x05,0x02,0x00,0x00,0x51,0x01,0x00,0x00,0x04,0x02,0x00,0x00, +0x3d,0x00,0x04,0x00,0x01,0x01,0x00,0x00,0x06,0x02,0x00,0x00, +0x05,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x07,0x02,0x00,0x00, +0x08,0x02,0x00,0x00,0xf2,0x01,0x00,0x00,0xf6,0x01,0x00,0x00, +0x3e,0x00,0x03,0x00,0x08,0x02,0x00,0x00,0x06,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x0a,0x02,0x00,0x00, +0x3e,0x03,0x00,0x00,0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xe8,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xea,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0xe3,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xe3,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x0c,0x02,0x00,0x00,0x2c,0x03,0x00,0x00,0xd0,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xe0,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xe2,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x0e,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x0e,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0x2d,0x03,0x00,0x00,0x3e,0x00,0x00,0x00, +0xe2,0x01,0x00,0x00,0x3a,0x02,0x00,0x00,0x11,0x02,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0x14,0x02,0x00,0x00, +0x2d,0x03,0x00,0x00,0xbf,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0x10,0x02,0x00,0x00,0x11,0x02,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x14,0x02,0x00,0x00,0x0f,0x02,0x00,0x00, +0x10,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x0f,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0x16,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x16,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0x3b,0x03,0x00,0x00,0x3e,0x00,0x00,0x00,0x0f,0x02,0x00,0x00, +0x38,0x02,0x00,0x00,0x17,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc2,0x00,0x00,0x00,0x1c,0x02,0x00,0x00,0x3b,0x03,0x00,0x00, +0xbc,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x18,0x02,0x00,0x00, +0x17,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x1c,0x02,0x00,0x00,0x17,0x02,0x00,0x00,0x18,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x17,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x22,0x02,0x00,0x00,0x2d,0x03,0x00,0x00, +0xbc,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x24,0x02,0x00,0x00,0x22,0x02,0x00,0x00,0x3b,0x03,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x26,0x02,0x00,0x00, +0x59,0x00,0x00,0x00,0xb9,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x29,0x02,0x00,0x00,0x2d,0x03,0x00,0x00, +0x28,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x2a,0x02,0x00,0x00,0x26,0x02,0x00,0x00,0x29,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x2c,0x02,0x00,0x00, +0x68,0x00,0x00,0x00,0xbc,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x2d,0x02,0x00,0x00,0x2a,0x02,0x00,0x00, +0x2c,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x2f,0x02,0x00,0x00,0x2d,0x02,0x00,0x00,0x3b,0x03,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x31,0x02,0x00,0x00, +0x2f,0x02,0x00,0x00,0x30,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x33,0x02,0x00,0x00,0x31,0x02,0x00,0x00, +0x28,0x03,0x00,0x00,0x41,0x00,0x05,0x00,0x56,0x01,0x00,0x00, +0x34,0x02,0x00,0x00,0x86,0x01,0x00,0x00,0x33,0x02,0x00,0x00, +0x3d,0x00,0x04,0x00,0x01,0x01,0x00,0x00,0x35,0x02,0x00,0x00, +0x34,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x07,0x02,0x00,0x00, +0x36,0x02,0x00,0x00,0x20,0x02,0x00,0x00,0x24,0x02,0x00,0x00, +0x3e,0x00,0x03,0x00,0x36,0x02,0x00,0x00,0x35,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x38,0x02,0x00,0x00, +0x3b,0x03,0x00,0x00,0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x16,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x18,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0x11,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x11,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x3a,0x02,0x00,0x00,0x2d,0x03,0x00,0x00,0xd0,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x0e,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x10,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x3c,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x3c,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0x2e,0x03,0x00,0x00,0x3e,0x00,0x00,0x00, +0x10,0x02,0x00,0x00,0x80,0x02,0x00,0x00,0x3f,0x02,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0x42,0x02,0x00,0x00, +0x2e,0x03,0x00,0x00,0xbf,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0x3e,0x02,0x00,0x00,0x3f,0x02,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x42,0x02,0x00,0x00,0x3d,0x02,0x00,0x00, +0x3e,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x3d,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0x44,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x44,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0x32,0x03,0x00,0x00,0x3e,0x00,0x00,0x00,0x3d,0x02,0x00,0x00, +0x7e,0x02,0x00,0x00,0x47,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc2,0x00,0x00,0x00,0x4a,0x02,0x00,0x00,0x32,0x03,0x00,0x00, +0x60,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x46,0x02,0x00,0x00, +0x47,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x4a,0x02,0x00,0x00,0x45,0x02,0x00,0x00,0x46,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x45,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0x4c,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x4c,0x02,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x34,0x03,0x00,0x00, +0x3e,0x00,0x00,0x00,0x45,0x02,0x00,0x00,0x7c,0x02,0x00,0x00, +0x4f,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, +0x52,0x02,0x00,0x00,0x34,0x03,0x00,0x00,0xbc,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0x4e,0x02,0x00,0x00,0x4f,0x02,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x52,0x02,0x00,0x00, +0x4d,0x02,0x00,0x00,0x4e,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x4d,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x54,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x54,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0x36,0x03,0x00,0x00,0x3e,0x00,0x00,0x00, +0x4d,0x02,0x00,0x00,0x7a,0x02,0x00,0x00,0x55,0x02,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0x5a,0x02,0x00,0x00, +0x36,0x03,0x00,0x00,0x62,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0x56,0x02,0x00,0x00,0x55,0x02,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x5a,0x02,0x00,0x00,0x55,0x02,0x00,0x00, +0x56,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x55,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x5c,0x02,0x00,0x00, +0x2e,0x03,0x00,0x00,0xbc,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x5e,0x02,0x00,0x00,0x5c,0x02,0x00,0x00, +0x34,0x03,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x60,0x02,0x00,0x00,0x5e,0x02,0x00,0x00,0x5f,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x62,0x02,0x00,0x00, +0x32,0x03,0x00,0x00,0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x63,0x02,0x00,0x00,0x60,0x02,0x00,0x00, +0x62,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x65,0x02,0x00,0x00,0x63,0x02,0x00,0x00,0x36,0x03,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x69,0x02,0x00,0x00, +0x62,0x02,0x00,0x00,0x36,0x03,0x00,0x00,0x41,0x00,0x05,0x00, +0x07,0x02,0x00,0x00,0x6a,0x02,0x00,0x00,0xf2,0x01,0x00,0x00, +0x69,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0x01,0x01,0x00,0x00, +0x6b,0x02,0x00,0x00,0x6a,0x02,0x00,0x00,0x73,0x00,0x04,0x00, +0xc4,0x00,0x00,0x00,0x6c,0x02,0x00,0x00,0x6b,0x02,0x00,0x00, +0x41,0x00,0x05,0x00,0x07,0x02,0x00,0x00,0x71,0x02,0x00,0x00, +0x20,0x02,0x00,0x00,0x5e,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, +0x01,0x01,0x00,0x00,0x72,0x02,0x00,0x00,0x71,0x02,0x00,0x00, +0x73,0x00,0x04,0x00,0xc4,0x00,0x00,0x00,0x73,0x02,0x00,0x00, +0x72,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0xcd,0x00,0x00,0x00, +0x75,0x02,0x00,0x00,0xca,0x00,0x00,0x00,0x65,0x02,0x00,0x00, +0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00,0x76,0x02,0x00,0x00, +0x75,0x02,0x00,0x00,0x0c,0x00,0x08,0x00,0xc4,0x00,0x00,0x00, +0x77,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00, +0x6c,0x02,0x00,0x00,0x73,0x02,0x00,0x00,0x76,0x02,0x00,0x00, +0x3e,0x00,0x03,0x00,0x75,0x02,0x00,0x00,0x77,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x7a,0x02,0x00,0x00, +0x36,0x03,0x00,0x00,0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x54,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x56,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0x4f,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x4f,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x7c,0x02,0x00,0x00,0x34,0x03,0x00,0x00,0xd0,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x4c,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x4e,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x47,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x47,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x7e,0x02,0x00,0x00,0x32,0x03,0x00,0x00, +0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x44,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x46,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0x3f,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x3f,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x80,0x02,0x00,0x00, +0x2e,0x03,0x00,0x00,0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x3c,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x3e,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0xdb,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xdb,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x82,0x02,0x00,0x00,0x28,0x03,0x00,0x00,0xd0,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xd8,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xda,0x01,0x00,0x00,0xe0,0x00,0x04,0x00,0x0c,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0xd0,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0xd7,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xd7,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x84,0x02,0x00,0x00, +0x0e,0x03,0x00,0x00,0x6c,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xd4,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xd6,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x89,0x02,0x00,0x00, +0x55,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x8a,0x02,0x00,0x00,0x96,0x00,0x00,0x00, +0x89,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x8f,0x02,0x00,0x00,0x59,0x00,0x00,0x00,0xb9,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x90,0x02,0x00,0x00, +0xa8,0x00,0x00,0x00,0x8f,0x02,0x00,0x00,0x41,0x00,0x05,0x00, +0x15,0x00,0x00,0x00,0x95,0x02,0x00,0x00,0x12,0x00,0x00,0x00, +0x94,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x96,0x02,0x00,0x00,0x95,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x97,0x02,0x00,0x00,0x0f,0x00,0x00,0x00, +0x96,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x9b,0x02,0x00,0x00,0x47,0x00,0x00,0x00,0x96,0x02,0x00,0x00, +0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00,0x9d,0x02,0x00,0x00, +0x9c,0x02,0x00,0x00,0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x9e,0x02,0x00,0x00,0x9d,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9f,0x02,0x00,0x00, +0x9b,0x02,0x00,0x00,0x9e,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xa0,0x02,0x00,0x00,0x97,0x02,0x00,0x00, +0x9f,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0xa2,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0xa2,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0x0f,0x03,0x00,0x00,0x3e,0x00,0x00,0x00, +0xd6,0x00,0x00,0x00,0x08,0x03,0x00,0x00,0xa5,0x02,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0xa8,0x02,0x00,0x00, +0x0f,0x03,0x00,0x00,0xbf,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0xa4,0x02,0x00,0x00,0xa5,0x02,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0xa8,0x02,0x00,0x00,0xa3,0x02,0x00,0x00, +0xa4,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0xa3,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0xaa,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0xaa,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0x10,0x03,0x00,0x00,0x3e,0x00,0x00,0x00,0xa3,0x02,0x00,0x00, +0x06,0x03,0x00,0x00,0xad,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc2,0x00,0x00,0x00,0xb0,0x02,0x00,0x00,0x10,0x03,0x00,0x00, +0x60,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xac,0x02,0x00,0x00, +0xad,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0xb0,0x02,0x00,0x00,0xab,0x02,0x00,0x00,0xac,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0xab,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xb4,0x02,0x00,0x00,0x10,0x03,0x00,0x00, +0x61,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xb5,0x02,0x00,0x00,0x8a,0x02,0x00,0x00,0xb4,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xb7,0x02,0x00,0x00, +0x64,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xb8,0x02,0x00,0x00,0xb5,0x02,0x00,0x00, +0xb7,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xbc,0x02,0x00,0x00,0x0f,0x03,0x00,0x00,0x28,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xbd,0x02,0x00,0x00, +0x90,0x02,0x00,0x00,0xbc,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xbf,0x02,0x00,0x00,0x68,0x00,0x00,0x00, +0xbc,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xc0,0x02,0x00,0x00,0xbd,0x02,0x00,0x00,0xbf,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0xc2,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0xc2,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0x12,0x03,0x00,0x00,0x3e,0x00,0x00,0x00,0xab,0x02,0x00,0x00, +0x04,0x03,0x00,0x00,0xc5,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc2,0x00,0x00,0x00,0xc8,0x02,0x00,0x00,0x12,0x03,0x00,0x00, +0xbc,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xc4,0x02,0x00,0x00, +0xc5,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0xc8,0x02,0x00,0x00,0xc3,0x02,0x00,0x00,0xc4,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0xc3,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0xca,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0xca,0x02,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x14,0x03,0x00,0x00, +0x3e,0x00,0x00,0x00,0xc3,0x02,0x00,0x00,0x02,0x03,0x00,0x00, +0xcd,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, +0xd0,0x02,0x00,0x00,0x14,0x03,0x00,0x00,0x62,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0xcc,0x02,0x00,0x00,0xcd,0x02,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xd0,0x02,0x00,0x00, +0xcb,0x02,0x00,0x00,0xcc,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0xcb,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xd3,0x02,0x00,0x00,0xb8,0x02,0x00,0x00,0x14,0x03,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0xd6,0x02,0x00,0x00, +0xd3,0x02,0x00,0x00,0x36,0x00,0x00,0x00,0xf7,0x00,0x03,0x00, +0xd8,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0xd6,0x02,0x00,0x00,0xd7,0x02,0x00,0x00,0xd8,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0xd7,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xdb,0x02,0x00,0x00,0xc0,0x02,0x00,0x00, +0x12,0x03,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, +0xdc,0x02,0x00,0x00,0x12,0x00,0x00,0x00,0xd0,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xdd,0x02,0x00,0x00, +0xdc,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, +0xde,0x02,0x00,0x00,0xdb,0x02,0x00,0x00,0xdd,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0xd8,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0xd8,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0xc2,0x00,0x00,0x00, +0xdf,0x02,0x00,0x00,0xd6,0x02,0x00,0x00,0xcb,0x02,0x00,0x00, +0xde,0x02,0x00,0x00,0xd7,0x02,0x00,0x00,0xf7,0x00,0x03,0x00, +0xe1,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0xdf,0x02,0x00,0x00,0xe0,0x02,0x00,0x00,0xe1,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0xe0,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xe9,0x02,0x00,0x00,0xc0,0x02,0x00,0x00, +0x12,0x03,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, +0xeb,0x02,0x00,0x00,0x12,0x00,0x00,0x00,0xea,0x02,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xec,0x02,0x00,0x00, +0xeb,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xed,0x02,0x00,0x00,0xe9,0x02,0x00,0x00,0xec,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xee,0x02,0x00,0x00, +0xa0,0x02,0x00,0x00,0xed,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xf0,0x02,0x00,0x00,0xee,0x02,0x00,0x00, +0xb8,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xf2,0x02,0x00,0x00,0xf0,0x02,0x00,0x00,0x14,0x03,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xf4,0x02,0x00,0x00, +0x0f,0x03,0x00,0x00,0xbc,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xf6,0x02,0x00,0x00,0xf4,0x02,0x00,0x00, +0x12,0x03,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xf8,0x02,0x00,0x00,0xf6,0x02,0x00,0x00,0xf7,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xfa,0x02,0x00,0x00, +0x10,0x03,0x00,0x00,0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xfb,0x02,0x00,0x00,0xf8,0x02,0x00,0x00, +0xfa,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xfd,0x02,0x00,0x00,0xfb,0x02,0x00,0x00,0x14,0x03,0x00,0x00, +0x41,0x00,0x05,0x00,0xcd,0x00,0x00,0x00,0xfe,0x02,0x00,0x00, +0xca,0x00,0x00,0x00,0xfd,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, +0xc4,0x00,0x00,0x00,0xff,0x02,0x00,0x00,0xfe,0x02,0x00,0x00, +0x41,0x00,0x06,0x00,0x90,0x01,0x00,0x00,0x00,0x03,0x00,0x00, +0xe5,0x02,0x00,0x00,0x34,0x00,0x00,0x00,0xf2,0x02,0x00,0x00, +0x3e,0x00,0x03,0x00,0x00,0x03,0x00,0x00,0xff,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0xe1,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0xe1,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0xcd,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0xcd,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x02,0x03,0x00,0x00,0x14,0x03,0x00,0x00, +0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xca,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0xcc,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0xc5,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0xc5,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x04,0x03,0x00,0x00, +0x12,0x03,0x00,0x00,0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xc2,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0xc4,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0xad,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0xad,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x06,0x03,0x00,0x00,0x10,0x03,0x00,0x00,0xd0,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xaa,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0xac,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0xa5,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0xa5,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x08,0x03,0x00,0x00,0x0f,0x03,0x00,0x00, +0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xa2,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0xa4,0x02,0x00,0x00,0xfd,0x00,0x01,0x00, +0x38,0x00,0x01,0x00, +}; +const uint64_t matmul_q5_0_f32_aligned_len = 11668; + +unsigned char matmul_q5_0_f32_aligned_fp32_data[] = { +0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, +0x38,0x03,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, +0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x51,0x11,0x00,0x00, +0x11,0x00,0x02,0x00,0x60,0x11,0x00,0x00,0x0b,0x00,0x06,0x00, +0x01,0x00,0x00,0x00,0x47,0x4c,0x53,0x4c,0x2e,0x73,0x74,0x64, +0x2e,0x34,0x35,0x30,0x00,0x00,0x00,0x00,0x0e,0x00,0x03,0x00, +0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x0f,0x00,0x0f,0x00, +0x05,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x6d,0x61,0x69,0x6e, +0x00,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0x3d,0x00,0x00,0x00,0x4c,0x00,0x00,0x00,0x0a,0x01,0x00,0x00, +0x51,0x01,0x00,0x00,0x84,0x01,0x00,0x00,0x8b,0x01,0x00,0x00, +0x72,0x02,0x00,0x00,0xbb,0x02,0x00,0x00,0x10,0x00,0x06,0x00, +0x04,0x00,0x00,0x00,0x11,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x0b,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x1c,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x10,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x08,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x03,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x10,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x10,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, +0x05,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x14,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x10,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x1c,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, +0x08,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x09,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x24,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x10,0x00,0x00,0x00,0x0a,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x28,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, +0x0b,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x2c,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x30,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x10,0x00,0x00,0x00,0x0d,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x34,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x10,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x37,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x3d,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x1a,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x4c,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, +0x1b,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x4f,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x53,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x60,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x62,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x6c,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x03,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0xa7,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xb9,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0xbc,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x08,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x03,0x01,0x00,0x00,0x06,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x05,0x01,0x00,0x00, +0x06,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x06,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x06,0x01,0x00,0x00, +0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x06,0x01,0x00,0x00,0x02,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x07,0x01,0x00,0x00,0x06,0x00,0x00,0x00,0x16,0x00,0x00,0x00, +0x48,0x00,0x04,0x00,0x08,0x01,0x00,0x00,0x00,0x00,0x00,0x00, +0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x08,0x01,0x00,0x00, +0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x03,0x00,0x08,0x01,0x00,0x00,0x02,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x0a,0x01,0x00,0x00,0x22,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x0a,0x01,0x00,0x00, +0x21,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x5c,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x5d,0x01,0x00,0x00,0x0b,0x00,0x00,0x00, +0x19,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x88,0x01,0x00,0x00, +0x06,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x48,0x00,0x04,0x00, +0x89,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x89,0x01,0x00,0x00,0x00,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, +0x89,0x01,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x8b,0x01,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x8b,0x01,0x00,0x00,0x21,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x72,0x02,0x00,0x00, +0x0b,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0xb8,0x02,0x00,0x00,0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x48,0x00,0x04,0x00,0xb9,0x02,0x00,0x00,0x00,0x00,0x00,0x00, +0x19,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0xb9,0x02,0x00,0x00, +0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x03,0x00,0xb9,0x02,0x00,0x00,0x02,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0xbb,0x02,0x00,0x00,0x22,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xbb,0x02,0x00,0x00, +0x21,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x13,0x00,0x02,0x00, +0x02,0x00,0x00,0x00,0x21,0x00,0x03,0x00,0x03,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x15,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x17,0x00,0x04,0x00, +0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x03,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x0a,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x09,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00, +0x0b,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x0d,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x1e,0x00,0x10,0x00,0x10,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x11,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x10,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0x11,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0x09,0x00,0x00,0x00,0x15,0x00,0x04,0x00,0x13,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x08,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x15,0x00,0x00,0x00,0x09,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, +0x21,0x00,0x00,0x00,0x0a,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0x27,0x00,0x00,0x00,0x09,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x2d,0x00,0x00,0x00, +0x07,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, +0x34,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x32,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x40,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00, +0x3d,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x3e,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00,0x4c,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x4f,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x32,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x54,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x53,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x58,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x53,0x00,0x00,0x00, +0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x60,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x61,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x53,0x00,0x00,0x00, +0x60,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x62,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x63,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0x61,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x67,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0x61,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x32,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x10,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x72,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x77,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x78,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, +0x77,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x7d,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, +0x77,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, +0x81,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x91,0x00,0x00,0x00, +0x0b,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, +0x97,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0xa2,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xa7,0x00,0x00,0x00, +0x40,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, +0xa9,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xb8,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x32,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xb9,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xba,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0xb9,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xbb,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x62,0x00,0x00,0x00, +0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xbc,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xbd,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0xbb,0x00,0x00,0x00, +0xbc,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xbe,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0xbd,0x00,0x00,0x00, +0x60,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xbf,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0xba,0x00,0x00,0x00, +0xbe,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xc0,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0xb8,0x00,0x00,0x00, +0xbf,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xc1,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0xc0,0x00,0x00,0x00, +0xbc,0x00,0x00,0x00,0x14,0x00,0x02,0x00,0xc2,0x00,0x00,0x00, +0x16,0x00,0x03,0x00,0xc4,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xc5,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xc6,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0xc5,0x00,0x00,0x00,0xbf,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xc7,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0xc6,0x00,0x00,0x00,0xbc,0x00,0x00,0x00, +0x1c,0x00,0x04,0x00,0xc8,0x00,0x00,0x00,0xc4,0x00,0x00,0x00, +0xc7,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0xc9,0x00,0x00,0x00, +0x07,0x00,0x00,0x00,0xc8,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0xc4,0x00,0x00,0x00,0xcc,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0xcd,0x00,0x00,0x00,0x07,0x00,0x00,0x00, +0xc4,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, +0xd0,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xf4,0x00,0x00,0x00,0x80,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xfa,0x00,0x00,0x00,0x10,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xfe,0x00,0x00,0x00, +0x0f,0x00,0x00,0x00,0x16,0x00,0x03,0x00,0x01,0x01,0x00,0x00, +0x10,0x00,0x00,0x00,0x15,0x00,0x04,0x00,0x02,0x01,0x00,0x00, +0x10,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x1c,0x00,0x04,0x00, +0x03,0x01,0x00,0x00,0x02,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, +0x15,0x00,0x04,0x00,0x04,0x01,0x00,0x00,0x08,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x1c,0x00,0x04,0x00,0x05,0x01,0x00,0x00, +0x04,0x01,0x00,0x00,0xfa,0x00,0x00,0x00,0x1e,0x00,0x05,0x00, +0x06,0x01,0x00,0x00,0x01,0x01,0x00,0x00,0x03,0x01,0x00,0x00, +0x05,0x01,0x00,0x00,0x1d,0x00,0x03,0x00,0x07,0x01,0x00,0x00, +0x06,0x01,0x00,0x00,0x1e,0x00,0x03,0x00,0x08,0x01,0x00,0x00, +0x07,0x01,0x00,0x00,0x20,0x00,0x04,0x00,0x09,0x01,0x00,0x00, +0x0c,0x00,0x00,0x00,0x08,0x01,0x00,0x00,0x3b,0x00,0x04,0x00, +0x09,0x01,0x00,0x00,0x0a,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x0c,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, +0x01,0x01,0x00,0x00,0x20,0x00,0x04,0x00,0x12,0x01,0x00,0x00, +0x0c,0x00,0x00,0x00,0x02,0x01,0x00,0x00,0x2b,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0x16,0x01,0x00,0x00,0x10,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x28,0x01,0x00,0x00, +0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x31,0x01,0x00,0x00, +0x0c,0x00,0x00,0x00,0x04,0x01,0x00,0x00,0x17,0x00,0x04,0x00, +0x35,0x01,0x00,0x00,0xc4,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0xc4,0x00,0x00,0x00,0x48,0x01,0x00,0x00, +0x00,0x00,0x80,0x41,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x4d,0x01,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, +0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x4e,0x01,0x00,0x00,0x84,0x00,0x00,0x00,0x37,0x00,0x00,0x00, +0x4d,0x01,0x00,0x00,0x1c,0x00,0x04,0x00,0x4f,0x01,0x00,0x00, +0xc4,0x00,0x00,0x00,0x4e,0x01,0x00,0x00,0x20,0x00,0x04,0x00, +0x50,0x01,0x00,0x00,0x04,0x00,0x00,0x00,0x4f,0x01,0x00,0x00, +0x3b,0x00,0x04,0x00,0x50,0x01,0x00,0x00,0x51,0x01,0x00,0x00, +0x04,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x55,0x01,0x00,0x00, +0x04,0x00,0x00,0x00,0xc4,0x00,0x00,0x00,0x32,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x5c,0x01,0x00,0x00,0x01,0x00,0x00,0x00, +0x33,0x00,0x06,0x00,0x09,0x00,0x00,0x00,0x5d,0x01,0x00,0x00, +0x5c,0x01,0x00,0x00,0x39,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x5e,0x01,0x00,0x00, +0x51,0x00,0x00,0x00,0x5d,0x01,0x00,0x00,0x00,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x5f,0x01,0x00,0x00, +0x84,0x00,0x00,0x00,0x5e,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x60,0x01,0x00,0x00, +0x86,0x00,0x00,0x00,0x5f,0x01,0x00,0x00,0x6c,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x7b,0x01,0x00,0x00, +0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x80,0x01,0x00,0x00, +0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x81,0x01,0x00,0x00, +0x84,0x00,0x00,0x00,0xa7,0x00,0x00,0x00,0x80,0x01,0x00,0x00, +0x1c,0x00,0x04,0x00,0x82,0x01,0x00,0x00,0xc4,0x00,0x00,0x00, +0x81,0x01,0x00,0x00,0x20,0x00,0x04,0x00,0x83,0x01,0x00,0x00, +0x04,0x00,0x00,0x00,0x82,0x01,0x00,0x00,0x3b,0x00,0x04,0x00, +0x83,0x01,0x00,0x00,0x84,0x01,0x00,0x00,0x04,0x00,0x00,0x00, +0x17,0x00,0x04,0x00,0x87,0x01,0x00,0x00,0xc4,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x1d,0x00,0x03,0x00,0x88,0x01,0x00,0x00, +0x87,0x01,0x00,0x00,0x1e,0x00,0x03,0x00,0x89,0x01,0x00,0x00, +0x88,0x01,0x00,0x00,0x20,0x00,0x04,0x00,0x8a,0x01,0x00,0x00, +0x0c,0x00,0x00,0x00,0x89,0x01,0x00,0x00,0x3b,0x00,0x04,0x00, +0x8a,0x01,0x00,0x00,0x8b,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x8d,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, +0xc4,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x9e,0x01,0x00,0x00,0x03,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xa4,0x01,0x00,0x00,0x51,0x00,0x00,0x00, +0x5d,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xa5,0x01,0x00,0x00,0x84,0x00,0x00,0x00, +0xa4,0x01,0x00,0x00,0x77,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xa6,0x01,0x00,0x00,0x86,0x00,0x00,0x00, +0xa5,0x01,0x00,0x00,0x6c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xa9,0x01,0x00,0x00,0x08,0x01,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xaa,0x01,0x00,0x00, +0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xad,0x01,0x00,0x00, +0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x77,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xc8,0x01,0x00,0x00, +0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00, +0x1c,0x00,0x04,0x00,0xc9,0x01,0x00,0x00,0xc4,0x00,0x00,0x00, +0xc8,0x01,0x00,0x00,0x20,0x00,0x04,0x00,0xca,0x01,0x00,0x00, +0x07,0x00,0x00,0x00,0xc9,0x01,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xda,0x01,0x00,0x00,0x80,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xf5,0x01,0x00,0x00,0x84,0x00,0x00,0x00, +0xbf,0x00,0x00,0x00,0xbc,0x00,0x00,0x00,0x1c,0x00,0x04,0x00, +0xf6,0x01,0x00,0x00,0xc4,0x00,0x00,0x00,0xf5,0x01,0x00,0x00, +0x20,0x00,0x04,0x00,0xf7,0x01,0x00,0x00,0x07,0x00,0x00,0x00, +0xf6,0x01,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x00,0x02,0x00,0x00,0x86,0x00,0x00,0x00,0xb9,0x00,0x00,0x00, +0xbf,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x08,0x02,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, +0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x37,0x02,0x00,0x00,0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00, +0x62,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, +0x6a,0x02,0x00,0x00,0x0d,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0x0a,0x00,0x00,0x00,0x72,0x02,0x00,0x00,0x01,0x00,0x00,0x00, +0x1d,0x00,0x03,0x00,0xb8,0x02,0x00,0x00,0xc4,0x00,0x00,0x00, +0x1e,0x00,0x03,0x00,0xb9,0x02,0x00,0x00,0xb8,0x02,0x00,0x00, +0x20,0x00,0x04,0x00,0xba,0x02,0x00,0x00,0x0c,0x00,0x00,0x00, +0xb9,0x02,0x00,0x00,0x3b,0x00,0x04,0x00,0xba,0x02,0x00,0x00, +0xbb,0x02,0x00,0x00,0x0c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0xc0,0x02,0x00,0x00,0x05,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xcd,0x02,0x00,0x00, +0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00, +0x2c,0x00,0x05,0x00,0x35,0x01,0x00,0x00,0x37,0x03,0x00,0x00, +0x48,0x01,0x00,0x00,0x48,0x01,0x00,0x00,0x36,0x00,0x05,0x00, +0x02,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x03,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x05,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0xc9,0x00,0x00,0x00,0xca,0x00,0x00,0x00, +0x07,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0xca,0x01,0x00,0x00, +0xcb,0x01,0x00,0x00,0x07,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0xf7,0x01,0x00,0x00,0xf8,0x01,0x00,0x00,0x07,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00,0x0e,0x00,0x00,0x00, +0x0b,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x0e,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x16,0x00,0x00,0x00, +0x12,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x17,0x00,0x00,0x00,0x16,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x18,0x00,0x00,0x00, +0x0f,0x00,0x00,0x00,0x17,0x00,0x00,0x00,0x89,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x1e,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, +0x17,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, +0x22,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x21,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x22,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x24,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x28,0x00,0x00,0x00, +0x12,0x00,0x00,0x00,0x27,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x29,0x00,0x00,0x00,0x28,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x2a,0x00,0x00,0x00, +0x1e,0x00,0x00,0x00,0x29,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x15,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0x2d,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x2f,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x30,0x00,0x00,0x00,0x24,0x00,0x00,0x00, +0x2f,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x32,0x00,0x00,0x00,0x30,0x00,0x00,0x00,0x2a,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x35,0x00,0x00,0x00, +0x12,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x36,0x00,0x00,0x00,0x35,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x38,0x00,0x00,0x00, +0x36,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x82,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x38,0x00,0x00,0x00, +0x39,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x3b,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x37,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00,0x3f,0x00,0x00,0x00, +0x3d,0x00,0x00,0x00,0x3e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x3f,0x00,0x00,0x00, +0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x42,0x00,0x00,0x00, +0x40,0x00,0x00,0x00,0x3b,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x47,0x00,0x00,0x00,0x40,0x00,0x00,0x00, +0x3b,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00, +0x49,0x00,0x00,0x00,0x3d,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x4a,0x00,0x00,0x00, +0x49,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00, +0x4d,0x00,0x00,0x00,0x4c,0x00,0x00,0x00,0x3e,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, +0x4d,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x50,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x4f,0x00,0x00,0x00, +0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x55,0x00,0x00,0x00, +0x50,0x00,0x00,0x00,0x54,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x59,0x00,0x00,0x00,0x50,0x00,0x00,0x00, +0x58,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x5d,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x4f,0x00,0x00,0x00, +0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x64,0x00,0x00,0x00, +0x5d,0x00,0x00,0x00,0x63,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x68,0x00,0x00,0x00,0x5d,0x00,0x00,0x00, +0x67,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x6e,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x73,0x00,0x00,0x00, +0x4e,0x00,0x00,0x00,0x72,0x00,0x00,0x00,0x89,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x79,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, +0x78,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x7e,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x7d,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x82,0x00,0x00,0x00, +0x12,0x00,0x00,0x00,0x81,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x83,0x00,0x00,0x00,0x82,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0x47,0x00,0x00,0x00,0x83,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x15,0x00,0x00,0x00,0x87,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x88,0x00,0x00,0x00,0x87,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x8a,0x00,0x00,0x00,0x47,0x00,0x00,0x00, +0x39,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x8d,0x00,0x00,0x00,0x8a,0x00,0x00,0x00,0x83,0x00,0x00,0x00, +0x0c,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x8e,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x26,0x00,0x00,0x00,0x88,0x00,0x00,0x00, +0x8d,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, +0x92,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x91,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x93,0x00,0x00,0x00, +0x92,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x94,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x93,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x96,0x00,0x00,0x00, +0x42,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x15,0x00,0x00,0x00,0x98,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0x97,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x99,0x00,0x00,0x00,0x98,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x9a,0x00,0x00,0x00,0x96,0x00,0x00,0x00, +0x99,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x9b,0x00,0x00,0x00,0x94,0x00,0x00,0x00,0x9a,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9d,0x00,0x00,0x00, +0x9b,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x9e,0x00,0x00,0x00,0x9d,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, +0xa3,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0xa2,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xa4,0x00,0x00,0x00, +0xa3,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xa5,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0xa4,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa8,0x00,0x00,0x00, +0x4a,0x00,0x00,0x00,0xa7,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x15,0x00,0x00,0x00,0xaa,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0xa9,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xab,0x00,0x00,0x00,0xaa,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xac,0x00,0x00,0x00,0xa8,0x00,0x00,0x00, +0xab,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xad,0x00,0x00,0x00,0xa5,0x00,0x00,0x00,0xac,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xaf,0x00,0x00,0x00, +0xad,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xb0,0x00,0x00,0x00,0xaf,0x00,0x00,0x00, +0x77,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xb2,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xb2,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xe3,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, +0x05,0x00,0x00,0x00,0xd1,0x00,0x00,0x00,0xb3,0x00,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0xc3,0x00,0x00,0x00, +0xe3,0x02,0x00,0x00,0xc1,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0xb4,0x00,0x00,0x00,0xb3,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0xc3,0x00,0x00,0x00,0xb3,0x00,0x00,0x00, +0xb4,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xb3,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0xcd,0x00,0x00,0x00,0xce,0x00,0x00,0x00, +0xca,0x00,0x00,0x00,0xe3,0x02,0x00,0x00,0x3e,0x00,0x03,0x00, +0xce,0x00,0x00,0x00,0xcc,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xd1,0x00,0x00,0x00,0xe3,0x02,0x00,0x00, +0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xb2,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xb4,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xd4,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xd4,0x00,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xfc,0x02,0x00,0x00, +0xb0,0x00,0x00,0x00,0xb4,0x00,0x00,0x00,0xaf,0x01,0x00,0x00, +0xd7,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xf8,0x02,0x00,0x00,0x9e,0x00,0x00,0x00,0xb4,0x00,0x00,0x00, +0xac,0x01,0x00,0x00,0xd7,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xe4,0x02,0x00,0x00,0x84,0x00,0x00,0x00, +0xb4,0x00,0x00,0x00,0x5a,0x02,0x00,0x00,0xd7,0x00,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0xdb,0x00,0x00,0x00, +0xe4,0x02,0x00,0x00,0x8e,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0xd6,0x00,0x00,0x00,0xd7,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0xdb,0x00,0x00,0x00,0xd5,0x00,0x00,0x00, +0xd6,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xd5,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xdd,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xdd,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xf4,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0xd5,0x00,0x00,0x00, +0x62,0x01,0x00,0x00,0xde,0x00,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc2,0x00,0x00,0x00,0xe3,0x00,0x00,0x00,0xf4,0x02,0x00,0x00, +0x37,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xdf,0x00,0x00,0x00, +0xde,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0xe3,0x00,0x00,0x00,0xde,0x00,0x00,0x00,0xdf,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xde,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xe8,0x00,0x00,0x00,0x73,0x00,0x00,0x00, +0xf4,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xeb,0x00,0x00,0x00,0xe8,0x00,0x00,0x00,0x99,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xec,0x00,0x00,0x00, +0xeb,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xed,0x00,0x00,0x00,0xf8,0x02,0x00,0x00, +0xec,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xef,0x00,0x00,0x00,0xed,0x00,0x00,0x00,0x6e,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xf5,0x00,0x00,0x00, +0xe8,0x00,0x00,0x00,0xf4,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xf7,0x00,0x00,0x00,0xf5,0x00,0x00,0x00, +0x6e,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xfb,0x00,0x00,0x00,0xef,0x00,0x00,0x00,0xfa,0x00,0x00,0x00, +0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xff,0x00,0x00,0x00, +0xef,0x00,0x00,0x00,0xfe,0x00,0x00,0x00,0x41,0x00,0x07,0x00, +0x0c,0x01,0x00,0x00,0x0d,0x01,0x00,0x00,0x0a,0x01,0x00,0x00, +0x34,0x00,0x00,0x00,0xfb,0x00,0x00,0x00,0x34,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x01,0x01,0x00,0x00,0x0e,0x01,0x00,0x00, +0x0d,0x01,0x00,0x00,0x73,0x00,0x04,0x00,0xc4,0x00,0x00,0x00, +0x0f,0x01,0x00,0x00,0x0e,0x01,0x00,0x00,0x41,0x00,0x08,0x00, +0x12,0x01,0x00,0x00,0x13,0x01,0x00,0x00,0x0a,0x01,0x00,0x00, +0x34,0x00,0x00,0x00,0xfb,0x00,0x00,0x00,0xd0,0x00,0x00,0x00, +0xd0,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x02,0x01,0x00,0x00, +0x14,0x01,0x00,0x00,0x13,0x01,0x00,0x00,0x71,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x15,0x01,0x00,0x00,0x14,0x01,0x00,0x00, +0xc4,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x17,0x01,0x00,0x00, +0x15,0x01,0x00,0x00,0x16,0x01,0x00,0x00,0x41,0x00,0x08,0x00, +0x12,0x01,0x00,0x00,0x19,0x01,0x00,0x00,0x0a,0x01,0x00,0x00, +0x34,0x00,0x00,0x00,0xfb,0x00,0x00,0x00,0xd0,0x00,0x00,0x00, +0x34,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x02,0x01,0x00,0x00, +0x1a,0x01,0x00,0x00,0x19,0x01,0x00,0x00,0x71,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x1b,0x01,0x00,0x00,0x1a,0x01,0x00,0x00, +0xc5,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x1c,0x01,0x00,0x00, +0x17,0x01,0x00,0x00,0x1b,0x01,0x00,0x00,0xc2,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x22,0x01,0x00,0x00,0x1c,0x01,0x00,0x00, +0xff,0x00,0x00,0x00,0xc4,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x23,0x01,0x00,0x00,0x22,0x01,0x00,0x00,0xa9,0x00,0x00,0x00, +0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x24,0x01,0x00,0x00, +0x23,0x01,0x00,0x00,0xfa,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0x25,0x01,0x00,0x00,0x24,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x29,0x01,0x00,0x00, +0xff,0x00,0x00,0x00,0x28,0x01,0x00,0x00,0xc2,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x2a,0x01,0x00,0x00,0x1c,0x01,0x00,0x00, +0x29,0x01,0x00,0x00,0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x2b,0x01,0x00,0x00,0x2a,0x01,0x00,0x00,0xfa,0x00,0x00,0x00, +0x7c,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x2c,0x01,0x00,0x00, +0x2b,0x01,0x00,0x00,0x41,0x00,0x08,0x00,0x31,0x01,0x00,0x00, +0x32,0x01,0x00,0x00,0x0a,0x01,0x00,0x00,0x34,0x00,0x00,0x00, +0xfb,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0xff,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x04,0x01,0x00,0x00,0x33,0x01,0x00,0x00, +0x32,0x01,0x00,0x00,0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x34,0x01,0x00,0x00,0x33,0x01,0x00,0x00,0xc7,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x39,0x01,0x00,0x00,0x34,0x01,0x00,0x00, +0xfe,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x3d,0x01,0x00,0x00,0x25,0x01,0x00,0x00,0xc5,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x3e,0x01,0x00,0x00,0x39,0x01,0x00,0x00, +0x3d,0x01,0x00,0x00,0x70,0x00,0x04,0x00,0xc4,0x00,0x00,0x00, +0x3f,0x01,0x00,0x00,0x3e,0x01,0x00,0x00,0xc2,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x41,0x01,0x00,0x00,0x34,0x01,0x00,0x00, +0xa9,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x44,0x01,0x00,0x00,0x2c,0x01,0x00,0x00,0xc5,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x45,0x01,0x00,0x00,0x41,0x01,0x00,0x00, +0x44,0x01,0x00,0x00,0x70,0x00,0x04,0x00,0xc4,0x00,0x00,0x00, +0x46,0x01,0x00,0x00,0x45,0x01,0x00,0x00,0x50,0x00,0x05,0x00, +0x35,0x01,0x00,0x00,0x47,0x01,0x00,0x00,0x3f,0x01,0x00,0x00, +0x46,0x01,0x00,0x00,0x83,0x00,0x05,0x00,0x35,0x01,0x00,0x00, +0x4a,0x01,0x00,0x00,0x47,0x01,0x00,0x00,0x37,0x03,0x00,0x00, +0x8e,0x00,0x05,0x00,0x35,0x01,0x00,0x00,0x4c,0x01,0x00,0x00, +0x4a,0x01,0x00,0x00,0x0f,0x01,0x00,0x00,0x51,0x00,0x05,0x00, +0xc4,0x00,0x00,0x00,0x54,0x01,0x00,0x00,0x4c,0x01,0x00,0x00, +0x00,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x55,0x01,0x00,0x00, +0x56,0x01,0x00,0x00,0x51,0x01,0x00,0x00,0xf7,0x00,0x00,0x00, +0x3e,0x00,0x03,0x00,0x56,0x01,0x00,0x00,0x54,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x58,0x01,0x00,0x00, +0xf7,0x00,0x00,0x00,0xfa,0x00,0x00,0x00,0x51,0x00,0x05,0x00, +0xc4,0x00,0x00,0x00,0x5a,0x01,0x00,0x00,0x4c,0x01,0x00,0x00, +0x01,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x55,0x01,0x00,0x00, +0x5b,0x01,0x00,0x00,0x51,0x01,0x00,0x00,0x58,0x01,0x00,0x00, +0x3e,0x00,0x03,0x00,0x5b,0x01,0x00,0x00,0x5a,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x62,0x01,0x00,0x00, +0xf4,0x02,0x00,0x00,0x60,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0xdd,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xdf,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x64,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x64,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xf5,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0xdf,0x00,0x00,0x00, +0xa8,0x01,0x00,0x00,0x65,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc2,0x00,0x00,0x00,0x6a,0x01,0x00,0x00,0xf5,0x02,0x00,0x00, +0xa7,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x66,0x01,0x00,0x00, +0x65,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x6a,0x01,0x00,0x00,0x65,0x01,0x00,0x00,0x66,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x65,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x6f,0x01,0x00,0x00,0x7e,0x00,0x00,0x00, +0xf5,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x72,0x01,0x00,0x00,0x6f,0x01,0x00,0x00,0xab,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x73,0x01,0x00,0x00, +0x72,0x01,0x00,0x00,0x77,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x74,0x01,0x00,0x00,0xfc,0x02,0x00,0x00, +0x73,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x76,0x01,0x00,0x00,0x74,0x01,0x00,0x00,0x79,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x7c,0x01,0x00,0x00, +0x6f,0x01,0x00,0x00,0x7b,0x01,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x7e,0x01,0x00,0x00,0x79,0x00,0x00,0x00, +0x77,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x7f,0x01,0x00,0x00,0x7c,0x01,0x00,0x00,0x7e,0x01,0x00,0x00, +0x41,0x00,0x07,0x00,0x8d,0x01,0x00,0x00,0x8e,0x01,0x00,0x00, +0x8b,0x01,0x00,0x00,0x34,0x00,0x00,0x00,0x76,0x01,0x00,0x00, +0x3e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00, +0x8f,0x01,0x00,0x00,0x8e,0x01,0x00,0x00,0x41,0x00,0x05,0x00, +0x55,0x01,0x00,0x00,0x90,0x01,0x00,0x00,0x84,0x01,0x00,0x00, +0x7f,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x90,0x01,0x00,0x00, +0x8f,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x92,0x01,0x00,0x00,0x7f,0x01,0x00,0x00,0x39,0x00,0x00,0x00, +0x41,0x00,0x07,0x00,0x8d,0x01,0x00,0x00,0x94,0x01,0x00,0x00, +0x8b,0x01,0x00,0x00,0x34,0x00,0x00,0x00,0x76,0x01,0x00,0x00, +0x39,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00, +0x95,0x01,0x00,0x00,0x94,0x01,0x00,0x00,0x41,0x00,0x05,0x00, +0x55,0x01,0x00,0x00,0x96,0x01,0x00,0x00,0x84,0x01,0x00,0x00, +0x92,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x96,0x01,0x00,0x00, +0x95,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x98,0x01,0x00,0x00,0x7f,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, +0x41,0x00,0x07,0x00,0x8d,0x01,0x00,0x00,0x9a,0x01,0x00,0x00, +0x8b,0x01,0x00,0x00,0x34,0x00,0x00,0x00,0x76,0x01,0x00,0x00, +0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00, +0x9b,0x01,0x00,0x00,0x9a,0x01,0x00,0x00,0x41,0x00,0x05,0x00, +0x55,0x01,0x00,0x00,0x9c,0x01,0x00,0x00,0x84,0x01,0x00,0x00, +0x98,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x9c,0x01,0x00,0x00, +0x9b,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x9f,0x01,0x00,0x00,0x7f,0x01,0x00,0x00,0x9e,0x01,0x00,0x00, +0x41,0x00,0x07,0x00,0x8d,0x01,0x00,0x00,0xa1,0x01,0x00,0x00, +0x8b,0x01,0x00,0x00,0x34,0x00,0x00,0x00,0x76,0x01,0x00,0x00, +0x9e,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00, +0xa2,0x01,0x00,0x00,0xa1,0x01,0x00,0x00,0x41,0x00,0x05,0x00, +0x55,0x01,0x00,0x00,0xa3,0x01,0x00,0x00,0x84,0x01,0x00,0x00, +0x9f,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0xa3,0x01,0x00,0x00, +0xa2,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xa8,0x01,0x00,0x00,0xf5,0x02,0x00,0x00,0xa6,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0x64,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x66,0x01,0x00,0x00,0xe0,0x00,0x04,0x00,0x0c,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0xa9,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xac,0x01,0x00,0x00,0xf8,0x02,0x00,0x00, +0xaa,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xaf,0x01,0x00,0x00,0xfc,0x02,0x00,0x00,0xad,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0xb1,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xb1,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xfe,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0x66,0x01,0x00,0x00, +0x58,0x02,0x00,0x00,0xb4,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc2,0x00,0x00,0x00,0xb7,0x01,0x00,0x00,0xfe,0x02,0x00,0x00, +0x6c,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xb3,0x01,0x00,0x00, +0xb4,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0xb7,0x01,0x00,0x00,0xb2,0x01,0x00,0x00,0xb3,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xb2,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0xb9,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xb9,0x01,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x02,0x03,0x00,0x00, +0x3e,0x00,0x00,0x00,0xb2,0x01,0x00,0x00,0xe4,0x01,0x00,0x00, +0xbc,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, +0xbf,0x01,0x00,0x00,0x02,0x03,0x00,0x00,0x60,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0xbb,0x01,0x00,0x00,0xbc,0x01,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xbf,0x01,0x00,0x00, +0xba,0x01,0x00,0x00,0xbb,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xba,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xc1,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xc1,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0x14,0x03,0x00,0x00,0x3e,0x00,0x00,0x00, +0xba,0x01,0x00,0x00,0xe2,0x01,0x00,0x00,0xc2,0x01,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0xc7,0x01,0x00,0x00, +0x14,0x03,0x00,0x00,0x62,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0xc3,0x01,0x00,0x00,0xc2,0x01,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0xc7,0x01,0x00,0x00,0xc2,0x01,0x00,0x00, +0xc3,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xc2,0x01,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xcd,0x01,0x00,0x00, +0x02,0x03,0x00,0x00,0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xcf,0x01,0x00,0x00,0xcd,0x01,0x00,0x00, +0x14,0x03,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xd1,0x01,0x00,0x00,0x55,0x00,0x00,0x00,0x53,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xd3,0x01,0x00,0x00, +0x02,0x03,0x00,0x00,0x61,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xd4,0x01,0x00,0x00,0xd1,0x01,0x00,0x00, +0xd3,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xd6,0x01,0x00,0x00,0x64,0x00,0x00,0x00,0x62,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xd7,0x01,0x00,0x00, +0xd4,0x01,0x00,0x00,0xd6,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xd9,0x01,0x00,0x00,0xd7,0x01,0x00,0x00, +0x14,0x03,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xdb,0x01,0x00,0x00,0xd9,0x01,0x00,0x00,0xda,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xdd,0x01,0x00,0x00, +0xdb,0x01,0x00,0x00,0xfe,0x02,0x00,0x00,0x41,0x00,0x05,0x00, +0x55,0x01,0x00,0x00,0xde,0x01,0x00,0x00,0x51,0x01,0x00,0x00, +0xdd,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00, +0xdf,0x01,0x00,0x00,0xde,0x01,0x00,0x00,0x41,0x00,0x05,0x00, +0xcd,0x00,0x00,0x00,0xe0,0x01,0x00,0x00,0xcb,0x01,0x00,0x00, +0xcf,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0xe0,0x01,0x00,0x00, +0xdf,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xe2,0x01,0x00,0x00,0x14,0x03,0x00,0x00,0xd0,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xc1,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xc3,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xbc,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xbc,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xe4,0x01,0x00,0x00,0x02,0x03,0x00,0x00, +0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xb9,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xbb,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0xe6,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xe6,0x01,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x03,0x03,0x00,0x00, +0x3e,0x00,0x00,0x00,0xbb,0x01,0x00,0x00,0x12,0x02,0x00,0x00, +0xe9,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, +0xec,0x01,0x00,0x00,0x03,0x03,0x00,0x00,0xbf,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0xe8,0x01,0x00,0x00,0xe9,0x01,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xec,0x01,0x00,0x00, +0xe7,0x01,0x00,0x00,0xe8,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xe7,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xee,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xee,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0x11,0x03,0x00,0x00,0x3e,0x00,0x00,0x00, +0xe7,0x01,0x00,0x00,0x10,0x02,0x00,0x00,0xef,0x01,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0xf4,0x01,0x00,0x00, +0x11,0x03,0x00,0x00,0xbc,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0xf0,0x01,0x00,0x00,0xef,0x01,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0xf4,0x01,0x00,0x00,0xef,0x01,0x00,0x00, +0xf0,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xef,0x01,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xfa,0x01,0x00,0x00, +0x03,0x03,0x00,0x00,0xbc,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xfc,0x01,0x00,0x00,0xfa,0x01,0x00,0x00, +0x11,0x03,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xfe,0x01,0x00,0x00,0x59,0x00,0x00,0x00,0xb9,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x01,0x02,0x00,0x00, +0x03,0x03,0x00,0x00,0x00,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x02,0x02,0x00,0x00,0xfe,0x01,0x00,0x00, +0x01,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x04,0x02,0x00,0x00,0x68,0x00,0x00,0x00,0xbc,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x05,0x02,0x00,0x00, +0x02,0x02,0x00,0x00,0x04,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x07,0x02,0x00,0x00,0x05,0x02,0x00,0x00, +0x11,0x03,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x09,0x02,0x00,0x00,0x07,0x02,0x00,0x00,0x08,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x0b,0x02,0x00,0x00, +0x09,0x02,0x00,0x00,0xfe,0x02,0x00,0x00,0x41,0x00,0x05,0x00, +0x55,0x01,0x00,0x00,0x0c,0x02,0x00,0x00,0x84,0x01,0x00,0x00, +0x0b,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00, +0x0d,0x02,0x00,0x00,0x0c,0x02,0x00,0x00,0x41,0x00,0x05,0x00, +0xcd,0x00,0x00,0x00,0x0e,0x02,0x00,0x00,0xf8,0x01,0x00,0x00, +0xfc,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x0e,0x02,0x00,0x00, +0x0d,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x10,0x02,0x00,0x00,0x11,0x03,0x00,0x00,0xd0,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xee,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xf0,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xe9,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xe9,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x12,0x02,0x00,0x00,0x03,0x03,0x00,0x00, +0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xe6,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xe8,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0x14,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x14,0x02,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x04,0x03,0x00,0x00, +0x3e,0x00,0x00,0x00,0xe8,0x01,0x00,0x00,0x56,0x02,0x00,0x00, +0x17,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, +0x1a,0x02,0x00,0x00,0x04,0x03,0x00,0x00,0xbf,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0x16,0x02,0x00,0x00,0x17,0x02,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x1a,0x02,0x00,0x00, +0x15,0x02,0x00,0x00,0x16,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x15,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x1c,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x1c,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0x08,0x03,0x00,0x00,0x3e,0x00,0x00,0x00, +0x15,0x02,0x00,0x00,0x54,0x02,0x00,0x00,0x1f,0x02,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0x22,0x02,0x00,0x00, +0x08,0x03,0x00,0x00,0x60,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0x1e,0x02,0x00,0x00,0x1f,0x02,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x22,0x02,0x00,0x00,0x1d,0x02,0x00,0x00, +0x1e,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x1d,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0x24,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x24,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0x0a,0x03,0x00,0x00,0x3e,0x00,0x00,0x00,0x1d,0x02,0x00,0x00, +0x52,0x02,0x00,0x00,0x27,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc2,0x00,0x00,0x00,0x2a,0x02,0x00,0x00,0x0a,0x03,0x00,0x00, +0xbc,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x26,0x02,0x00,0x00, +0x27,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x2a,0x02,0x00,0x00,0x25,0x02,0x00,0x00,0x26,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x25,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0x2c,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x2c,0x02,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x0c,0x03,0x00,0x00, +0x3e,0x00,0x00,0x00,0x25,0x02,0x00,0x00,0x50,0x02,0x00,0x00, +0x2d,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, +0x32,0x02,0x00,0x00,0x0c,0x03,0x00,0x00,0x62,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0x2e,0x02,0x00,0x00,0x2d,0x02,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x32,0x02,0x00,0x00, +0x2d,0x02,0x00,0x00,0x2e,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x2d,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x34,0x02,0x00,0x00,0x04,0x03,0x00,0x00,0xbc,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x36,0x02,0x00,0x00, +0x34,0x02,0x00,0x00,0x0a,0x03,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x38,0x02,0x00,0x00,0x36,0x02,0x00,0x00, +0x37,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x3a,0x02,0x00,0x00,0x08,0x03,0x00,0x00,0x62,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3b,0x02,0x00,0x00, +0x38,0x02,0x00,0x00,0x3a,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x3d,0x02,0x00,0x00,0x3b,0x02,0x00,0x00, +0x0c,0x03,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x41,0x02,0x00,0x00,0x3a,0x02,0x00,0x00,0x0c,0x03,0x00,0x00, +0x41,0x00,0x05,0x00,0xcd,0x00,0x00,0x00,0x42,0x02,0x00,0x00, +0xcb,0x01,0x00,0x00,0x41,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, +0xc4,0x00,0x00,0x00,0x43,0x02,0x00,0x00,0x42,0x02,0x00,0x00, +0x41,0x00,0x05,0x00,0xcd,0x00,0x00,0x00,0x48,0x02,0x00,0x00, +0xf8,0x01,0x00,0x00,0x36,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, +0xc4,0x00,0x00,0x00,0x49,0x02,0x00,0x00,0x48,0x02,0x00,0x00, +0x41,0x00,0x05,0x00,0xcd,0x00,0x00,0x00,0x4b,0x02,0x00,0x00, +0xca,0x00,0x00,0x00,0x3d,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, +0xc4,0x00,0x00,0x00,0x4c,0x02,0x00,0x00,0x4b,0x02,0x00,0x00, +0x0c,0x00,0x08,0x00,0xc4,0x00,0x00,0x00,0x4d,0x02,0x00,0x00, +0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x43,0x02,0x00,0x00, +0x49,0x02,0x00,0x00,0x4c,0x02,0x00,0x00,0x3e,0x00,0x03,0x00, +0x4b,0x02,0x00,0x00,0x4d,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x50,0x02,0x00,0x00,0x0c,0x03,0x00,0x00, +0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x2c,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x2e,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0x27,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x27,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x52,0x02,0x00,0x00, +0x0a,0x03,0x00,0x00,0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x24,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x26,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0x1f,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x1f,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x54,0x02,0x00,0x00,0x08,0x03,0x00,0x00,0xd0,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x1c,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x1e,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x17,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x17,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x56,0x02,0x00,0x00,0x04,0x03,0x00,0x00, +0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x14,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x16,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0xb4,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xb4,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x58,0x02,0x00,0x00, +0xfe,0x02,0x00,0x00,0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xb1,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xb3,0x01,0x00,0x00, +0xe0,0x00,0x04,0x00,0x0c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0xa9,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xd7,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xd7,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x5a,0x02,0x00,0x00,0xe4,0x02,0x00,0x00, +0x6c,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xd4,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xd6,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x5f,0x02,0x00,0x00,0x55,0x00,0x00,0x00, +0x53,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x60,0x02,0x00,0x00,0x96,0x00,0x00,0x00,0x5f,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x65,0x02,0x00,0x00, +0x59,0x00,0x00,0x00,0xb9,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x66,0x02,0x00,0x00,0xa8,0x00,0x00,0x00, +0x65,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, +0x6b,0x02,0x00,0x00,0x12,0x00,0x00,0x00,0x6a,0x02,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x6c,0x02,0x00,0x00, +0x6b,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x6d,0x02,0x00,0x00,0x0f,0x00,0x00,0x00,0x6c,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x71,0x02,0x00,0x00, +0x47,0x00,0x00,0x00,0x6c,0x02,0x00,0x00,0x41,0x00,0x05,0x00, +0x0d,0x00,0x00,0x00,0x73,0x02,0x00,0x00,0x72,0x02,0x00,0x00, +0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x74,0x02,0x00,0x00,0x73,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x75,0x02,0x00,0x00,0x71,0x02,0x00,0x00, +0x74,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x76,0x02,0x00,0x00,0x6d,0x02,0x00,0x00,0x75,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0x78,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x78,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xe5,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0xd6,0x00,0x00,0x00, +0xde,0x02,0x00,0x00,0x7b,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc2,0x00,0x00,0x00,0x7e,0x02,0x00,0x00,0xe5,0x02,0x00,0x00, +0xbf,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x7a,0x02,0x00,0x00, +0x7b,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x7e,0x02,0x00,0x00,0x79,0x02,0x00,0x00,0x7a,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x79,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0x80,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x80,0x02,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xe6,0x02,0x00,0x00, +0x3e,0x00,0x00,0x00,0x79,0x02,0x00,0x00,0xdc,0x02,0x00,0x00, +0x83,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, +0x86,0x02,0x00,0x00,0xe6,0x02,0x00,0x00,0x60,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0x82,0x02,0x00,0x00,0x83,0x02,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x86,0x02,0x00,0x00, +0x81,0x02,0x00,0x00,0x82,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x81,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x8a,0x02,0x00,0x00,0xe6,0x02,0x00,0x00,0x61,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8b,0x02,0x00,0x00, +0x60,0x02,0x00,0x00,0x8a,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x8d,0x02,0x00,0x00,0x64,0x00,0x00,0x00, +0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x8e,0x02,0x00,0x00,0x8b,0x02,0x00,0x00,0x8d,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x92,0x02,0x00,0x00, +0xe5,0x02,0x00,0x00,0x00,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x93,0x02,0x00,0x00,0x66,0x02,0x00,0x00, +0x92,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x95,0x02,0x00,0x00,0x68,0x00,0x00,0x00,0xbc,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x96,0x02,0x00,0x00, +0x93,0x02,0x00,0x00,0x95,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0x98,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x98,0x02,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xe8,0x02,0x00,0x00, +0x3e,0x00,0x00,0x00,0x81,0x02,0x00,0x00,0xda,0x02,0x00,0x00, +0x9b,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, +0x9e,0x02,0x00,0x00,0xe8,0x02,0x00,0x00,0xbc,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0x9a,0x02,0x00,0x00,0x9b,0x02,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x9e,0x02,0x00,0x00, +0x99,0x02,0x00,0x00,0x9a,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x99,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0xa0,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0xa0,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xea,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, +0x99,0x02,0x00,0x00,0xd8,0x02,0x00,0x00,0xa3,0x02,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0xa6,0x02,0x00,0x00, +0xea,0x02,0x00,0x00,0x62,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0xa2,0x02,0x00,0x00,0xa3,0x02,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0xa6,0x02,0x00,0x00,0xa1,0x02,0x00,0x00, +0xa2,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0xa1,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa9,0x02,0x00,0x00, +0x8e,0x02,0x00,0x00,0xea,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc2,0x00,0x00,0x00,0xac,0x02,0x00,0x00,0xa9,0x02,0x00,0x00, +0x36,0x00,0x00,0x00,0xf7,0x00,0x03,0x00,0xae,0x02,0x00,0x00, +0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xac,0x02,0x00,0x00, +0xad,0x02,0x00,0x00,0xae,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0xad,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xb1,0x02,0x00,0x00,0x96,0x02,0x00,0x00,0xe8,0x02,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0xb2,0x02,0x00,0x00, +0x12,0x00,0x00,0x00,0xd0,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xb3,0x02,0x00,0x00,0xb2,0x02,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0xb4,0x02,0x00,0x00, +0xb1,0x02,0x00,0x00,0xb3,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0xae,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0xae,0x02,0x00,0x00, +0xf5,0x00,0x07,0x00,0xc2,0x00,0x00,0x00,0xb5,0x02,0x00,0x00, +0xac,0x02,0x00,0x00,0xa1,0x02,0x00,0x00,0xb4,0x02,0x00,0x00, +0xad,0x02,0x00,0x00,0xf7,0x00,0x03,0x00,0xb7,0x02,0x00,0x00, +0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xb5,0x02,0x00,0x00, +0xb6,0x02,0x00,0x00,0xb7,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0xb6,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xbf,0x02,0x00,0x00,0x96,0x02,0x00,0x00,0xe8,0x02,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0xc1,0x02,0x00,0x00, +0x12,0x00,0x00,0x00,0xc0,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xc2,0x02,0x00,0x00,0xc1,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xc3,0x02,0x00,0x00, +0xbf,0x02,0x00,0x00,0xc2,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xc4,0x02,0x00,0x00,0x76,0x02,0x00,0x00, +0xc3,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xc6,0x02,0x00,0x00,0xc4,0x02,0x00,0x00,0x8e,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xc8,0x02,0x00,0x00, +0xc6,0x02,0x00,0x00,0xea,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xca,0x02,0x00,0x00,0xe5,0x02,0x00,0x00, +0xbc,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xcc,0x02,0x00,0x00,0xca,0x02,0x00,0x00,0xe8,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xce,0x02,0x00,0x00, +0xcc,0x02,0x00,0x00,0xcd,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xd0,0x02,0x00,0x00,0xe6,0x02,0x00,0x00, +0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xd1,0x02,0x00,0x00,0xce,0x02,0x00,0x00,0xd0,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xd3,0x02,0x00,0x00, +0xd1,0x02,0x00,0x00,0xea,0x02,0x00,0x00,0x41,0x00,0x05,0x00, +0xcd,0x00,0x00,0x00,0xd4,0x02,0x00,0x00,0xca,0x00,0x00,0x00, +0xd3,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00, +0xd5,0x02,0x00,0x00,0xd4,0x02,0x00,0x00,0x41,0x00,0x06,0x00, +0x8d,0x01,0x00,0x00,0xd6,0x02,0x00,0x00,0xbb,0x02,0x00,0x00, +0x34,0x00,0x00,0x00,0xc8,0x02,0x00,0x00,0x3e,0x00,0x03,0x00, +0xd6,0x02,0x00,0x00,0xd5,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0xb7,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0xb7,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0xa3,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0xa3,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xd8,0x02,0x00,0x00,0xea,0x02,0x00,0x00,0xd0,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xa0,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0xa2,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x9b,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x9b,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xda,0x02,0x00,0x00,0xe8,0x02,0x00,0x00, +0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x98,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x9a,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0x83,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x83,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xdc,0x02,0x00,0x00, +0xe6,0x02,0x00,0x00,0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x80,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x82,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0x7b,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x7b,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xde,0x02,0x00,0x00,0xe5,0x02,0x00,0x00,0xd0,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x78,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x7a,0x02,0x00,0x00,0xfd,0x00,0x01,0x00,0x38,0x00,0x01,0x00, + +}; +const uint64_t matmul_q5_0_f32_aligned_fp32_len = 10920; + +unsigned char matmul_q5_0_f32_fp32_data[] = { +0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, +0x40,0x03,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, +0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x51,0x11,0x00,0x00, +0x11,0x00,0x02,0x00,0x60,0x11,0x00,0x00,0x0b,0x00,0x06,0x00, +0x01,0x00,0x00,0x00,0x47,0x4c,0x53,0x4c,0x2e,0x73,0x74,0x64, +0x2e,0x34,0x35,0x30,0x00,0x00,0x00,0x00,0x0e,0x00,0x03,0x00, +0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x0f,0x00,0x0f,0x00, +0x05,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x6d,0x61,0x69,0x6e, +0x00,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0x3d,0x00,0x00,0x00,0x4c,0x00,0x00,0x00,0x09,0x01,0x00,0x00, +0x50,0x01,0x00,0x00,0x81,0x01,0x00,0x00,0x8c,0x01,0x00,0x00, +0x72,0x02,0x00,0x00,0xbb,0x02,0x00,0x00,0x10,0x00,0x06,0x00, +0x04,0x00,0x00,0x00,0x11,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x0b,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x1c,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x10,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x08,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x03,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x10,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x10,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, +0x05,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x14,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x10,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x1c,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, +0x08,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x09,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x24,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x10,0x00,0x00,0x00,0x0a,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x28,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, +0x0b,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x2c,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x30,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x10,0x00,0x00,0x00,0x0d,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x34,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x10,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x37,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x3d,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x1a,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x4c,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, +0x1b,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x4f,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x53,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x60,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x62,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x6c,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x03,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0xa6,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xb8,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0xbb,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x08,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x02,0x01,0x00,0x00,0x06,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x04,0x01,0x00,0x00, +0x06,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x05,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x05,0x01,0x00,0x00, +0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x05,0x01,0x00,0x00,0x02,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x06,0x01,0x00,0x00,0x06,0x00,0x00,0x00,0x16,0x00,0x00,0x00, +0x48,0x00,0x04,0x00,0x07,0x01,0x00,0x00,0x00,0x00,0x00,0x00, +0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x07,0x01,0x00,0x00, +0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x03,0x00,0x07,0x01,0x00,0x00,0x02,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x09,0x01,0x00,0x00,0x22,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x09,0x01,0x00,0x00, +0x21,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x5b,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x5c,0x01,0x00,0x00,0x0b,0x00,0x00,0x00, +0x19,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x89,0x01,0x00,0x00, +0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00, +0x8a,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x8a,0x01,0x00,0x00,0x00,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, +0x8a,0x01,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x8c,0x01,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x8c,0x01,0x00,0x00,0x21,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x72,0x02,0x00,0x00, +0x0b,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0xb8,0x02,0x00,0x00,0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x48,0x00,0x04,0x00,0xb9,0x02,0x00,0x00,0x00,0x00,0x00,0x00, +0x19,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0xb9,0x02,0x00,0x00, +0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x03,0x00,0xb9,0x02,0x00,0x00,0x02,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0xbb,0x02,0x00,0x00,0x22,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xbb,0x02,0x00,0x00, +0x21,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x13,0x00,0x02,0x00, +0x02,0x00,0x00,0x00,0x21,0x00,0x03,0x00,0x03,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x15,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x17,0x00,0x04,0x00, +0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x03,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x0a,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x09,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00, +0x0b,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x0d,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x1e,0x00,0x10,0x00,0x10,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x11,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x10,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0x11,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0x09,0x00,0x00,0x00,0x15,0x00,0x04,0x00,0x13,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x08,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x15,0x00,0x00,0x00,0x09,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, +0x21,0x00,0x00,0x00,0x0a,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0x27,0x00,0x00,0x00,0x09,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x2d,0x00,0x00,0x00, +0x07,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, +0x34,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x32,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x40,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00, +0x3d,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x3e,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00,0x4c,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x4f,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x32,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x54,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x53,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x58,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x53,0x00,0x00,0x00, +0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x60,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x61,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x53,0x00,0x00,0x00, +0x60,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x62,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x63,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0x61,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x67,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0x61,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x32,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x10,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x72,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x77,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x7c,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x80,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, +0x85,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0x90,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x96,0x00,0x00,0x00, +0x03,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, +0xa1,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x32,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xa6,0x00,0x00,0x00,0x40,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0xa8,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xb7,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00, +0x62,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xb8,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xb9,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0x53,0x00,0x00,0x00,0xb8,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xba,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0x4f,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x32,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xbb,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xbc,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0xba,0x00,0x00,0x00,0xbb,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xbd,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0xbc,0x00,0x00,0x00,0x60,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xbe,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0xb9,0x00,0x00,0x00,0xbd,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xbf,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0xb7,0x00,0x00,0x00,0xbe,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xc0,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0xbf,0x00,0x00,0x00,0xbb,0x00,0x00,0x00, +0x14,0x00,0x02,0x00,0xc1,0x00,0x00,0x00,0x16,0x00,0x03,0x00, +0xc3,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xc4,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xc5,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0xc4,0x00,0x00,0x00,0xbe,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xc6,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0xc5,0x00,0x00,0x00,0xbb,0x00,0x00,0x00,0x1c,0x00,0x04,0x00, +0xc7,0x00,0x00,0x00,0xc3,0x00,0x00,0x00,0xc6,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0xc8,0x00,0x00,0x00,0x07,0x00,0x00,0x00, +0xc7,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0xc3,0x00,0x00,0x00, +0xcb,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0xcc,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0xc3,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0xcf,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xf3,0x00,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, +0x39,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xf9,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xfd,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, +0x16,0x00,0x03,0x00,0x00,0x01,0x00,0x00,0x10,0x00,0x00,0x00, +0x15,0x00,0x04,0x00,0x01,0x01,0x00,0x00,0x10,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x1c,0x00,0x04,0x00,0x02,0x01,0x00,0x00, +0x01,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x15,0x00,0x04,0x00, +0x03,0x01,0x00,0x00,0x08,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x1c,0x00,0x04,0x00,0x04,0x01,0x00,0x00,0x03,0x01,0x00,0x00, +0xf9,0x00,0x00,0x00,0x1e,0x00,0x05,0x00,0x05,0x01,0x00,0x00, +0x00,0x01,0x00,0x00,0x02,0x01,0x00,0x00,0x04,0x01,0x00,0x00, +0x1d,0x00,0x03,0x00,0x06,0x01,0x00,0x00,0x05,0x01,0x00,0x00, +0x1e,0x00,0x03,0x00,0x07,0x01,0x00,0x00,0x06,0x01,0x00,0x00, +0x20,0x00,0x04,0x00,0x08,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, +0x07,0x01,0x00,0x00,0x3b,0x00,0x04,0x00,0x08,0x01,0x00,0x00, +0x09,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x0b,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,0x01,0x00,0x00, +0x20,0x00,0x04,0x00,0x11,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, +0x01,0x01,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, +0x15,0x01,0x00,0x00,0x10,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x27,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x30,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, +0x03,0x01,0x00,0x00,0x17,0x00,0x04,0x00,0x34,0x01,0x00,0x00, +0xc3,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0xc3,0x00,0x00,0x00,0x47,0x01,0x00,0x00,0x00,0x00,0x80,0x41, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x4c,0x01,0x00,0x00, +0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x4d,0x01,0x00,0x00, +0x84,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x4c,0x01,0x00,0x00, +0x1c,0x00,0x04,0x00,0x4e,0x01,0x00,0x00,0xc3,0x00,0x00,0x00, +0x4d,0x01,0x00,0x00,0x20,0x00,0x04,0x00,0x4f,0x01,0x00,0x00, +0x04,0x00,0x00,0x00,0x4e,0x01,0x00,0x00,0x3b,0x00,0x04,0x00, +0x4f,0x01,0x00,0x00,0x50,0x01,0x00,0x00,0x04,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x54,0x01,0x00,0x00,0x04,0x00,0x00,0x00, +0xc3,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x5b,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x33,0x00,0x06,0x00, +0x09,0x00,0x00,0x00,0x5c,0x01,0x00,0x00,0x5b,0x01,0x00,0x00, +0x39,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x5d,0x01,0x00,0x00,0x51,0x00,0x00,0x00, +0x5c,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x5e,0x01,0x00,0x00,0x84,0x00,0x00,0x00, +0x5d,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x5f,0x01,0x00,0x00,0x86,0x00,0x00,0x00, +0x5e,0x01,0x00,0x00,0x6c,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x7d,0x01,0x00,0x00,0x80,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x7e,0x01,0x00,0x00,0x84,0x00,0x00,0x00, +0xa6,0x00,0x00,0x00,0x7d,0x01,0x00,0x00,0x1c,0x00,0x04,0x00, +0x7f,0x01,0x00,0x00,0xc3,0x00,0x00,0x00,0x7e,0x01,0x00,0x00, +0x20,0x00,0x04,0x00,0x80,0x01,0x00,0x00,0x04,0x00,0x00,0x00, +0x7f,0x01,0x00,0x00,0x3b,0x00,0x04,0x00,0x80,0x01,0x00,0x00, +0x81,0x01,0x00,0x00,0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x85,0x01,0x00,0x00,0x80,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, +0x89,0x01,0x00,0x00,0xc3,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, +0x8a,0x01,0x00,0x00,0x89,0x01,0x00,0x00,0x20,0x00,0x04,0x00, +0x8b,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x8a,0x01,0x00,0x00, +0x3b,0x00,0x04,0x00,0x8b,0x01,0x00,0x00,0x8c,0x01,0x00,0x00, +0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x97,0x01,0x00,0x00, +0x0c,0x00,0x00,0x00,0xc3,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x9f,0x01,0x00,0x00,0x80,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xa4,0x01,0x00,0x00,0x51,0x00,0x00,0x00, +0x5c,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xa5,0x01,0x00,0x00,0x84,0x00,0x00,0x00, +0xa4,0x01,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xa6,0x01,0x00,0x00,0x86,0x00,0x00,0x00, +0xa5,0x01,0x00,0x00,0x6c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xa9,0x01,0x00,0x00,0x08,0x01,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xaa,0x01,0x00,0x00, +0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xad,0x01,0x00,0x00, +0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xc8,0x01,0x00,0x00, +0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00, +0x1c,0x00,0x04,0x00,0xc9,0x01,0x00,0x00,0xc3,0x00,0x00,0x00, +0xc8,0x01,0x00,0x00,0x20,0x00,0x04,0x00,0xca,0x01,0x00,0x00, +0x07,0x00,0x00,0x00,0xc9,0x01,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xda,0x01,0x00,0x00,0x80,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xf5,0x01,0x00,0x00,0x84,0x00,0x00,0x00, +0xbe,0x00,0x00,0x00,0xbb,0x00,0x00,0x00,0x1c,0x00,0x04,0x00, +0xf6,0x01,0x00,0x00,0xc3,0x00,0x00,0x00,0xf5,0x01,0x00,0x00, +0x20,0x00,0x04,0x00,0xf7,0x01,0x00,0x00,0x07,0x00,0x00,0x00, +0xf6,0x01,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x00,0x02,0x00,0x00,0x86,0x00,0x00,0x00,0xb8,0x00,0x00,0x00, +0xbe,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x08,0x02,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, +0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x37,0x02,0x00,0x00,0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00, +0x62,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, +0x6a,0x02,0x00,0x00,0x0d,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0x0a,0x00,0x00,0x00,0x72,0x02,0x00,0x00,0x01,0x00,0x00,0x00, +0x1d,0x00,0x03,0x00,0xb8,0x02,0x00,0x00,0xc3,0x00,0x00,0x00, +0x1e,0x00,0x03,0x00,0xb9,0x02,0x00,0x00,0xb8,0x02,0x00,0x00, +0x20,0x00,0x04,0x00,0xba,0x02,0x00,0x00,0x0c,0x00,0x00,0x00, +0xb9,0x02,0x00,0x00,0x3b,0x00,0x04,0x00,0xba,0x02,0x00,0x00, +0xbb,0x02,0x00,0x00,0x0c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0xc0,0x02,0x00,0x00,0x05,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xcd,0x02,0x00,0x00, +0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00, +0x2c,0x00,0x05,0x00,0x34,0x01,0x00,0x00,0x3f,0x03,0x00,0x00, +0x47,0x01,0x00,0x00,0x47,0x01,0x00,0x00,0x36,0x00,0x05,0x00, +0x02,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x03,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x05,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0xc8,0x00,0x00,0x00,0xc9,0x00,0x00,0x00, +0x07,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0xca,0x01,0x00,0x00, +0xcb,0x01,0x00,0x00,0x07,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0xf7,0x01,0x00,0x00,0xf8,0x01,0x00,0x00,0x07,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00,0x0e,0x00,0x00,0x00, +0x0b,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x0e,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x16,0x00,0x00,0x00, +0x12,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x17,0x00,0x00,0x00,0x16,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x18,0x00,0x00,0x00, +0x0f,0x00,0x00,0x00,0x17,0x00,0x00,0x00,0x89,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x1e,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, +0x17,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, +0x22,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x21,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x22,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x24,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x28,0x00,0x00,0x00, +0x12,0x00,0x00,0x00,0x27,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x29,0x00,0x00,0x00,0x28,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x2a,0x00,0x00,0x00, +0x1e,0x00,0x00,0x00,0x29,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x15,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0x2d,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x2f,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x30,0x00,0x00,0x00,0x24,0x00,0x00,0x00, +0x2f,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x32,0x00,0x00,0x00,0x30,0x00,0x00,0x00,0x2a,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x35,0x00,0x00,0x00, +0x12,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x36,0x00,0x00,0x00,0x35,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x38,0x00,0x00,0x00, +0x36,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x82,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x38,0x00,0x00,0x00, +0x39,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x3b,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x37,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00,0x3f,0x00,0x00,0x00, +0x3d,0x00,0x00,0x00,0x3e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x3f,0x00,0x00,0x00, +0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x42,0x00,0x00,0x00, +0x40,0x00,0x00,0x00,0x3b,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x47,0x00,0x00,0x00,0x40,0x00,0x00,0x00, +0x3b,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00, +0x49,0x00,0x00,0x00,0x3d,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x4a,0x00,0x00,0x00, +0x49,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00, +0x4d,0x00,0x00,0x00,0x4c,0x00,0x00,0x00,0x3e,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, +0x4d,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x50,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x4f,0x00,0x00,0x00, +0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x55,0x00,0x00,0x00, +0x50,0x00,0x00,0x00,0x54,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x59,0x00,0x00,0x00,0x50,0x00,0x00,0x00, +0x58,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x5d,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x4f,0x00,0x00,0x00, +0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x64,0x00,0x00,0x00, +0x5d,0x00,0x00,0x00,0x63,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x68,0x00,0x00,0x00,0x5d,0x00,0x00,0x00, +0x67,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x6e,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x73,0x00,0x00,0x00, +0x4e,0x00,0x00,0x00,0x72,0x00,0x00,0x00,0x89,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x78,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, +0x77,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x7d,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x7c,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x81,0x00,0x00,0x00, +0x12,0x00,0x00,0x00,0x80,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x82,0x00,0x00,0x00,0x81,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x83,0x00,0x00,0x00, +0x47,0x00,0x00,0x00,0x82,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x15,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0x85,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x87,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x89,0x00,0x00,0x00,0x47,0x00,0x00,0x00, +0x39,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x8c,0x00,0x00,0x00,0x89,0x00,0x00,0x00,0x82,0x00,0x00,0x00, +0x0c,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x8d,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x26,0x00,0x00,0x00,0x87,0x00,0x00,0x00, +0x8c,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, +0x91,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x90,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x92,0x00,0x00,0x00, +0x91,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x93,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x92,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x95,0x00,0x00,0x00, +0x42,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x15,0x00,0x00,0x00,0x97,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0x96,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x98,0x00,0x00,0x00,0x97,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x99,0x00,0x00,0x00,0x95,0x00,0x00,0x00, +0x98,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x9a,0x00,0x00,0x00,0x93,0x00,0x00,0x00,0x99,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9c,0x00,0x00,0x00, +0x9a,0x00,0x00,0x00,0x83,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x9d,0x00,0x00,0x00,0x9c,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, +0xa2,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0xa1,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xa3,0x00,0x00,0x00, +0xa2,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xa4,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0xa3,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa7,0x00,0x00,0x00, +0x4a,0x00,0x00,0x00,0xa6,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x15,0x00,0x00,0x00,0xa9,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0xa8,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xaa,0x00,0x00,0x00,0xa9,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xab,0x00,0x00,0x00,0xa7,0x00,0x00,0x00, +0xaa,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xac,0x00,0x00,0x00,0xa4,0x00,0x00,0x00,0xab,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xae,0x00,0x00,0x00, +0xac,0x00,0x00,0x00,0x83,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xaf,0x00,0x00,0x00,0xae,0x00,0x00,0x00, +0x39,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xb1,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xb1,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xe3,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, +0x05,0x00,0x00,0x00,0xd0,0x00,0x00,0x00,0xb2,0x00,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00,0xc2,0x00,0x00,0x00, +0xe3,0x02,0x00,0x00,0xc0,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0xb3,0x00,0x00,0x00,0xb2,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0xc2,0x00,0x00,0x00,0xb2,0x00,0x00,0x00, +0xb3,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xb2,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0xcc,0x00,0x00,0x00,0xcd,0x00,0x00,0x00, +0xc9,0x00,0x00,0x00,0xe3,0x02,0x00,0x00,0x3e,0x00,0x03,0x00, +0xcd,0x00,0x00,0x00,0xcb,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xd0,0x00,0x00,0x00,0xe3,0x02,0x00,0x00, +0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xb1,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xb3,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xd3,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xd3,0x00,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xfc,0x02,0x00,0x00, +0xaf,0x00,0x00,0x00,0xb3,0x00,0x00,0x00,0xaf,0x01,0x00,0x00, +0xd6,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xf8,0x02,0x00,0x00,0x9d,0x00,0x00,0x00,0xb3,0x00,0x00,0x00, +0xac,0x01,0x00,0x00,0xd6,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xe4,0x02,0x00,0x00,0x83,0x00,0x00,0x00, +0xb3,0x00,0x00,0x00,0x5a,0x02,0x00,0x00,0xd6,0x00,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00,0xda,0x00,0x00,0x00, +0xe4,0x02,0x00,0x00,0x8d,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0xd5,0x00,0x00,0x00,0xd6,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0xda,0x00,0x00,0x00,0xd4,0x00,0x00,0x00, +0xd5,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xd4,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xdc,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xdc,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xf4,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0xd4,0x00,0x00,0x00, +0x61,0x01,0x00,0x00,0xdd,0x00,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc1,0x00,0x00,0x00,0xe2,0x00,0x00,0x00,0xf4,0x02,0x00,0x00, +0x37,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xde,0x00,0x00,0x00, +0xdd,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0xe2,0x00,0x00,0x00,0xdd,0x00,0x00,0x00,0xde,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xdd,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xe7,0x00,0x00,0x00,0x73,0x00,0x00,0x00, +0xf4,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xea,0x00,0x00,0x00,0xe7,0x00,0x00,0x00,0x98,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xeb,0x00,0x00,0x00, +0xea,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xec,0x00,0x00,0x00,0xf8,0x02,0x00,0x00, +0xeb,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xee,0x00,0x00,0x00,0xec,0x00,0x00,0x00,0x6e,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xf4,0x00,0x00,0x00, +0xe7,0x00,0x00,0x00,0xf3,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xf6,0x00,0x00,0x00,0xf4,0x00,0x00,0x00, +0x6e,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xfa,0x00,0x00,0x00,0xee,0x00,0x00,0x00,0xf9,0x00,0x00,0x00, +0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xfe,0x00,0x00,0x00, +0xee,0x00,0x00,0x00,0xfd,0x00,0x00,0x00,0x41,0x00,0x07,0x00, +0x0b,0x01,0x00,0x00,0x0c,0x01,0x00,0x00,0x09,0x01,0x00,0x00, +0x34,0x00,0x00,0x00,0xfa,0x00,0x00,0x00,0x34,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x00,0x01,0x00,0x00,0x0d,0x01,0x00,0x00, +0x0c,0x01,0x00,0x00,0x73,0x00,0x04,0x00,0xc3,0x00,0x00,0x00, +0x0e,0x01,0x00,0x00,0x0d,0x01,0x00,0x00,0x41,0x00,0x08,0x00, +0x11,0x01,0x00,0x00,0x12,0x01,0x00,0x00,0x09,0x01,0x00,0x00, +0x34,0x00,0x00,0x00,0xfa,0x00,0x00,0x00,0xcf,0x00,0x00,0x00, +0xcf,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x01,0x01,0x00,0x00, +0x13,0x01,0x00,0x00,0x12,0x01,0x00,0x00,0x71,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x14,0x01,0x00,0x00,0x13,0x01,0x00,0x00, +0xc4,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x16,0x01,0x00,0x00, +0x14,0x01,0x00,0x00,0x15,0x01,0x00,0x00,0x41,0x00,0x08,0x00, +0x11,0x01,0x00,0x00,0x18,0x01,0x00,0x00,0x09,0x01,0x00,0x00, +0x34,0x00,0x00,0x00,0xfa,0x00,0x00,0x00,0xcf,0x00,0x00,0x00, +0x34,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x01,0x01,0x00,0x00, +0x19,0x01,0x00,0x00,0x18,0x01,0x00,0x00,0x71,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x1a,0x01,0x00,0x00,0x19,0x01,0x00,0x00, +0xc5,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x1b,0x01,0x00,0x00, +0x16,0x01,0x00,0x00,0x1a,0x01,0x00,0x00,0xc2,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x21,0x01,0x00,0x00,0x1b,0x01,0x00,0x00, +0xfe,0x00,0x00,0x00,0xc4,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x22,0x01,0x00,0x00,0x21,0x01,0x00,0x00,0xa8,0x00,0x00,0x00, +0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x23,0x01,0x00,0x00, +0x22,0x01,0x00,0x00,0xf9,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0x24,0x01,0x00,0x00,0x23,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x28,0x01,0x00,0x00, +0xfe,0x00,0x00,0x00,0x27,0x01,0x00,0x00,0xc2,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x29,0x01,0x00,0x00,0x1b,0x01,0x00,0x00, +0x28,0x01,0x00,0x00,0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x2a,0x01,0x00,0x00,0x29,0x01,0x00,0x00,0xf9,0x00,0x00,0x00, +0x7c,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x2b,0x01,0x00,0x00, +0x2a,0x01,0x00,0x00,0x41,0x00,0x08,0x00,0x30,0x01,0x00,0x00, +0x31,0x01,0x00,0x00,0x09,0x01,0x00,0x00,0x34,0x00,0x00,0x00, +0xfa,0x00,0x00,0x00,0x85,0x00,0x00,0x00,0xfe,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x03,0x01,0x00,0x00,0x32,0x01,0x00,0x00, +0x31,0x01,0x00,0x00,0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x33,0x01,0x00,0x00,0x32,0x01,0x00,0x00,0xc7,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x38,0x01,0x00,0x00,0x33,0x01,0x00,0x00, +0xfd,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x3c,0x01,0x00,0x00,0x24,0x01,0x00,0x00,0xc5,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x3d,0x01,0x00,0x00,0x38,0x01,0x00,0x00, +0x3c,0x01,0x00,0x00,0x70,0x00,0x04,0x00,0xc3,0x00,0x00,0x00, +0x3e,0x01,0x00,0x00,0x3d,0x01,0x00,0x00,0xc2,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x40,0x01,0x00,0x00,0x33,0x01,0x00,0x00, +0xa8,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x43,0x01,0x00,0x00,0x2b,0x01,0x00,0x00,0xc5,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x44,0x01,0x00,0x00,0x40,0x01,0x00,0x00, +0x43,0x01,0x00,0x00,0x70,0x00,0x04,0x00,0xc3,0x00,0x00,0x00, +0x45,0x01,0x00,0x00,0x44,0x01,0x00,0x00,0x50,0x00,0x05,0x00, +0x34,0x01,0x00,0x00,0x46,0x01,0x00,0x00,0x3e,0x01,0x00,0x00, +0x45,0x01,0x00,0x00,0x83,0x00,0x05,0x00,0x34,0x01,0x00,0x00, +0x49,0x01,0x00,0x00,0x46,0x01,0x00,0x00,0x3f,0x03,0x00,0x00, +0x8e,0x00,0x05,0x00,0x34,0x01,0x00,0x00,0x4b,0x01,0x00,0x00, +0x49,0x01,0x00,0x00,0x0e,0x01,0x00,0x00,0x51,0x00,0x05,0x00, +0xc3,0x00,0x00,0x00,0x53,0x01,0x00,0x00,0x4b,0x01,0x00,0x00, +0x00,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x54,0x01,0x00,0x00, +0x55,0x01,0x00,0x00,0x50,0x01,0x00,0x00,0xf6,0x00,0x00,0x00, +0x3e,0x00,0x03,0x00,0x55,0x01,0x00,0x00,0x53,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x57,0x01,0x00,0x00, +0xf6,0x00,0x00,0x00,0xf9,0x00,0x00,0x00,0x51,0x00,0x05,0x00, +0xc3,0x00,0x00,0x00,0x59,0x01,0x00,0x00,0x4b,0x01,0x00,0x00, +0x01,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x54,0x01,0x00,0x00, +0x5a,0x01,0x00,0x00,0x50,0x01,0x00,0x00,0x57,0x01,0x00,0x00, +0x3e,0x00,0x03,0x00,0x5a,0x01,0x00,0x00,0x59,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x61,0x01,0x00,0x00, +0xf4,0x02,0x00,0x00,0x5f,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0xdc,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xde,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x63,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x63,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xf5,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0xde,0x00,0x00,0x00, +0xa8,0x01,0x00,0x00,0x66,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc1,0x00,0x00,0x00,0x69,0x01,0x00,0x00,0xf5,0x02,0x00,0x00, +0xa6,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x65,0x01,0x00,0x00, +0x66,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x69,0x01,0x00,0x00,0x64,0x01,0x00,0x00,0x65,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x64,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x6d,0x01,0x00,0x00,0xa7,0x00,0x00,0x00, +0x7d,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x6f,0x01,0x00,0x00,0x6d,0x01,0x00,0x00,0xf5,0x02,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x70,0x01,0x00,0x00, +0x12,0x00,0x00,0x00,0xcf,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x71,0x01,0x00,0x00,0x70,0x01,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00,0x72,0x01,0x00,0x00, +0x6f,0x01,0x00,0x00,0x71,0x01,0x00,0x00,0xf7,0x00,0x03,0x00, +0x74,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x72,0x01,0x00,0x00,0x73,0x01,0x00,0x00,0x74,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x73,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x77,0x01,0x00,0x00,0xe4,0x02,0x00,0x00, +0x78,0x00,0x00,0x00,0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00, +0x79,0x01,0x00,0x00,0x77,0x01,0x00,0x00,0x8d,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x74,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x74,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0xc1,0x00,0x00,0x00, +0x7a,0x01,0x00,0x00,0x72,0x01,0x00,0x00,0x64,0x01,0x00,0x00, +0x79,0x01,0x00,0x00,0x73,0x01,0x00,0x00,0xf7,0x00,0x03,0x00, +0x7c,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x7a,0x01,0x00,0x00,0x7b,0x01,0x00,0x00,0x9b,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x7b,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x84,0x01,0x00,0x00,0x7d,0x00,0x00,0x00, +0xf5,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x86,0x01,0x00,0x00,0x84,0x01,0x00,0x00,0x85,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x88,0x01,0x00,0x00, +0x86,0x01,0x00,0x00,0x78,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x93,0x01,0x00,0x00,0x84,0x01,0x00,0x00, +0xaa,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x94,0x01,0x00,0x00,0xfc,0x02,0x00,0x00,0x93,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x96,0x01,0x00,0x00, +0x94,0x01,0x00,0x00,0x78,0x00,0x00,0x00,0x41,0x00,0x06,0x00, +0x97,0x01,0x00,0x00,0x98,0x01,0x00,0x00,0x8c,0x01,0x00,0x00, +0x34,0x00,0x00,0x00,0x96,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, +0xc3,0x00,0x00,0x00,0x99,0x01,0x00,0x00,0x98,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0x54,0x01,0x00,0x00,0x9a,0x01,0x00,0x00, +0x81,0x01,0x00,0x00,0x88,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, +0x9a,0x01,0x00,0x00,0x99,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0x7c,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x9b,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9e,0x01,0x00,0x00, +0x7d,0x00,0x00,0x00,0xf5,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xa0,0x01,0x00,0x00,0x9e,0x01,0x00,0x00, +0x9f,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xa2,0x01,0x00,0x00,0xa0,0x01,0x00,0x00,0x78,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x54,0x01,0x00,0x00,0xa3,0x01,0x00,0x00, +0x81,0x01,0x00,0x00,0xa2,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, +0xa3,0x01,0x00,0x00,0xcb,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x7c,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x7c,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0x66,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x66,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xa8,0x01,0x00,0x00,0xf5,0x02,0x00,0x00,0xa6,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0x63,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x65,0x01,0x00,0x00,0xe0,0x00,0x04,0x00,0x0c,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0xa9,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xac,0x01,0x00,0x00,0xf8,0x02,0x00,0x00, +0xaa,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xaf,0x01,0x00,0x00,0xfc,0x02,0x00,0x00,0xad,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0xb1,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xb1,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xfe,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0x65,0x01,0x00,0x00, +0x58,0x02,0x00,0x00,0xb4,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc1,0x00,0x00,0x00,0xb7,0x01,0x00,0x00,0xfe,0x02,0x00,0x00, +0x6c,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xb3,0x01,0x00,0x00, +0xb4,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0xb7,0x01,0x00,0x00,0xb2,0x01,0x00,0x00,0xb3,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xb2,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0xb9,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xb9,0x01,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x02,0x03,0x00,0x00, +0x3e,0x00,0x00,0x00,0xb2,0x01,0x00,0x00,0xe4,0x01,0x00,0x00, +0xbc,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00, +0xbf,0x01,0x00,0x00,0x02,0x03,0x00,0x00,0x60,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0xbb,0x01,0x00,0x00,0xbc,0x01,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xbf,0x01,0x00,0x00, +0xba,0x01,0x00,0x00,0xbb,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xba,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xc1,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xc1,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0x14,0x03,0x00,0x00,0x3e,0x00,0x00,0x00, +0xba,0x01,0x00,0x00,0xe2,0x01,0x00,0x00,0xc2,0x01,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00,0xc7,0x01,0x00,0x00, +0x14,0x03,0x00,0x00,0x62,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0xc3,0x01,0x00,0x00,0xc2,0x01,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0xc7,0x01,0x00,0x00,0xc2,0x01,0x00,0x00, +0xc3,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xc2,0x01,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xcd,0x01,0x00,0x00, +0x02,0x03,0x00,0x00,0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xcf,0x01,0x00,0x00,0xcd,0x01,0x00,0x00, +0x14,0x03,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xd1,0x01,0x00,0x00,0x55,0x00,0x00,0x00,0x53,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xd3,0x01,0x00,0x00, +0x02,0x03,0x00,0x00,0x61,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xd4,0x01,0x00,0x00,0xd1,0x01,0x00,0x00, +0xd3,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xd6,0x01,0x00,0x00,0x64,0x00,0x00,0x00,0x62,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xd7,0x01,0x00,0x00, +0xd4,0x01,0x00,0x00,0xd6,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xd9,0x01,0x00,0x00,0xd7,0x01,0x00,0x00, +0x14,0x03,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xdb,0x01,0x00,0x00,0xd9,0x01,0x00,0x00,0xda,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xdd,0x01,0x00,0x00, +0xdb,0x01,0x00,0x00,0xfe,0x02,0x00,0x00,0x41,0x00,0x05,0x00, +0x54,0x01,0x00,0x00,0xde,0x01,0x00,0x00,0x50,0x01,0x00,0x00, +0xdd,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0xc3,0x00,0x00,0x00, +0xdf,0x01,0x00,0x00,0xde,0x01,0x00,0x00,0x41,0x00,0x05,0x00, +0xcc,0x00,0x00,0x00,0xe0,0x01,0x00,0x00,0xcb,0x01,0x00,0x00, +0xcf,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0xe0,0x01,0x00,0x00, +0xdf,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xe2,0x01,0x00,0x00,0x14,0x03,0x00,0x00,0xcf,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xc1,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xc3,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xbc,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xbc,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xe4,0x01,0x00,0x00,0x02,0x03,0x00,0x00, +0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xb9,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xbb,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0xe6,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xe6,0x01,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x03,0x03,0x00,0x00, +0x3e,0x00,0x00,0x00,0xbb,0x01,0x00,0x00,0x12,0x02,0x00,0x00, +0xe9,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00, +0xec,0x01,0x00,0x00,0x03,0x03,0x00,0x00,0xbe,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0xe8,0x01,0x00,0x00,0xe9,0x01,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xec,0x01,0x00,0x00, +0xe7,0x01,0x00,0x00,0xe8,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xe7,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xee,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xee,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0x11,0x03,0x00,0x00,0x3e,0x00,0x00,0x00, +0xe7,0x01,0x00,0x00,0x10,0x02,0x00,0x00,0xef,0x01,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00,0xf4,0x01,0x00,0x00, +0x11,0x03,0x00,0x00,0xbb,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0xf0,0x01,0x00,0x00,0xef,0x01,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0xf4,0x01,0x00,0x00,0xef,0x01,0x00,0x00, +0xf0,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xef,0x01,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xfa,0x01,0x00,0x00, +0x03,0x03,0x00,0x00,0xbb,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xfc,0x01,0x00,0x00,0xfa,0x01,0x00,0x00, +0x11,0x03,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xfe,0x01,0x00,0x00,0x59,0x00,0x00,0x00,0xb8,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x01,0x02,0x00,0x00, +0x03,0x03,0x00,0x00,0x00,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x02,0x02,0x00,0x00,0xfe,0x01,0x00,0x00, +0x01,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x04,0x02,0x00,0x00,0x68,0x00,0x00,0x00,0xbb,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x05,0x02,0x00,0x00, +0x02,0x02,0x00,0x00,0x04,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x07,0x02,0x00,0x00,0x05,0x02,0x00,0x00, +0x11,0x03,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x09,0x02,0x00,0x00,0x07,0x02,0x00,0x00,0x08,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x0b,0x02,0x00,0x00, +0x09,0x02,0x00,0x00,0xfe,0x02,0x00,0x00,0x41,0x00,0x05,0x00, +0x54,0x01,0x00,0x00,0x0c,0x02,0x00,0x00,0x81,0x01,0x00,0x00, +0x0b,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0xc3,0x00,0x00,0x00, +0x0d,0x02,0x00,0x00,0x0c,0x02,0x00,0x00,0x41,0x00,0x05,0x00, +0xcc,0x00,0x00,0x00,0x0e,0x02,0x00,0x00,0xf8,0x01,0x00,0x00, +0xfc,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x0e,0x02,0x00,0x00, +0x0d,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x10,0x02,0x00,0x00,0x11,0x03,0x00,0x00,0xcf,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xee,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xf0,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xe9,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xe9,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x12,0x02,0x00,0x00,0x03,0x03,0x00,0x00, +0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xe6,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xe8,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0x14,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x14,0x02,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x04,0x03,0x00,0x00, +0x3e,0x00,0x00,0x00,0xe8,0x01,0x00,0x00,0x56,0x02,0x00,0x00, +0x17,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00, +0x1a,0x02,0x00,0x00,0x04,0x03,0x00,0x00,0xbe,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0x16,0x02,0x00,0x00,0x17,0x02,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x1a,0x02,0x00,0x00, +0x15,0x02,0x00,0x00,0x16,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x15,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x1c,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x1c,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0x08,0x03,0x00,0x00,0x3e,0x00,0x00,0x00, +0x15,0x02,0x00,0x00,0x54,0x02,0x00,0x00,0x1f,0x02,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00,0x22,0x02,0x00,0x00, +0x08,0x03,0x00,0x00,0x60,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0x1e,0x02,0x00,0x00,0x1f,0x02,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x22,0x02,0x00,0x00,0x1d,0x02,0x00,0x00, +0x1e,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x1d,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0x24,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x24,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0x0a,0x03,0x00,0x00,0x3e,0x00,0x00,0x00,0x1d,0x02,0x00,0x00, +0x52,0x02,0x00,0x00,0x27,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc1,0x00,0x00,0x00,0x2a,0x02,0x00,0x00,0x0a,0x03,0x00,0x00, +0xbb,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x26,0x02,0x00,0x00, +0x27,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x2a,0x02,0x00,0x00,0x25,0x02,0x00,0x00,0x26,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x25,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0x2c,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x2c,0x02,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x0c,0x03,0x00,0x00, +0x3e,0x00,0x00,0x00,0x25,0x02,0x00,0x00,0x50,0x02,0x00,0x00, +0x2d,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00, +0x32,0x02,0x00,0x00,0x0c,0x03,0x00,0x00,0x62,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0x2e,0x02,0x00,0x00,0x2d,0x02,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x32,0x02,0x00,0x00, +0x2d,0x02,0x00,0x00,0x2e,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x2d,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x34,0x02,0x00,0x00,0x04,0x03,0x00,0x00,0xbb,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x36,0x02,0x00,0x00, +0x34,0x02,0x00,0x00,0x0a,0x03,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x38,0x02,0x00,0x00,0x36,0x02,0x00,0x00, +0x37,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x3a,0x02,0x00,0x00,0x08,0x03,0x00,0x00,0x62,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3b,0x02,0x00,0x00, +0x38,0x02,0x00,0x00,0x3a,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x3d,0x02,0x00,0x00,0x3b,0x02,0x00,0x00, +0x0c,0x03,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x41,0x02,0x00,0x00,0x3a,0x02,0x00,0x00,0x0c,0x03,0x00,0x00, +0x41,0x00,0x05,0x00,0xcc,0x00,0x00,0x00,0x42,0x02,0x00,0x00, +0xcb,0x01,0x00,0x00,0x41,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, +0xc3,0x00,0x00,0x00,0x43,0x02,0x00,0x00,0x42,0x02,0x00,0x00, +0x41,0x00,0x05,0x00,0xcc,0x00,0x00,0x00,0x48,0x02,0x00,0x00, +0xf8,0x01,0x00,0x00,0x36,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, +0xc3,0x00,0x00,0x00,0x49,0x02,0x00,0x00,0x48,0x02,0x00,0x00, +0x41,0x00,0x05,0x00,0xcc,0x00,0x00,0x00,0x4b,0x02,0x00,0x00, +0xc9,0x00,0x00,0x00,0x3d,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, +0xc3,0x00,0x00,0x00,0x4c,0x02,0x00,0x00,0x4b,0x02,0x00,0x00, +0x0c,0x00,0x08,0x00,0xc3,0x00,0x00,0x00,0x4d,0x02,0x00,0x00, +0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x43,0x02,0x00,0x00, +0x49,0x02,0x00,0x00,0x4c,0x02,0x00,0x00,0x3e,0x00,0x03,0x00, +0x4b,0x02,0x00,0x00,0x4d,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x50,0x02,0x00,0x00,0x0c,0x03,0x00,0x00, +0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x2c,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x2e,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0x27,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x27,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x52,0x02,0x00,0x00, +0x0a,0x03,0x00,0x00,0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x24,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x26,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0x1f,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x1f,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x54,0x02,0x00,0x00,0x08,0x03,0x00,0x00,0xcf,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x1c,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x1e,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x17,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x17,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x56,0x02,0x00,0x00,0x04,0x03,0x00,0x00, +0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x14,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x16,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0xb4,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xb4,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x58,0x02,0x00,0x00, +0xfe,0x02,0x00,0x00,0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xb1,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xb3,0x01,0x00,0x00, +0xe0,0x00,0x04,0x00,0x0c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0xa9,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xd6,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xd6,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x5a,0x02,0x00,0x00,0xe4,0x02,0x00,0x00, +0x6c,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xd3,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xd5,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x5f,0x02,0x00,0x00,0x55,0x00,0x00,0x00, +0x53,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x60,0x02,0x00,0x00,0x95,0x00,0x00,0x00,0x5f,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x65,0x02,0x00,0x00, +0x59,0x00,0x00,0x00,0xb8,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x66,0x02,0x00,0x00,0xa7,0x00,0x00,0x00, +0x65,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, +0x6b,0x02,0x00,0x00,0x12,0x00,0x00,0x00,0x6a,0x02,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x6c,0x02,0x00,0x00, +0x6b,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x6d,0x02,0x00,0x00,0x0f,0x00,0x00,0x00,0x6c,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x71,0x02,0x00,0x00, +0x47,0x00,0x00,0x00,0x6c,0x02,0x00,0x00,0x41,0x00,0x05,0x00, +0x0d,0x00,0x00,0x00,0x73,0x02,0x00,0x00,0x72,0x02,0x00,0x00, +0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x74,0x02,0x00,0x00,0x73,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x75,0x02,0x00,0x00,0x71,0x02,0x00,0x00, +0x74,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x76,0x02,0x00,0x00,0x6d,0x02,0x00,0x00,0x75,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0x78,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x78,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xe5,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0xd5,0x00,0x00,0x00, +0xde,0x02,0x00,0x00,0x7b,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc1,0x00,0x00,0x00,0x7e,0x02,0x00,0x00,0xe5,0x02,0x00,0x00, +0xbe,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x7a,0x02,0x00,0x00, +0x7b,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x7e,0x02,0x00,0x00,0x79,0x02,0x00,0x00,0x7a,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x79,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0x80,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x80,0x02,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xe6,0x02,0x00,0x00, +0x3e,0x00,0x00,0x00,0x79,0x02,0x00,0x00,0xdc,0x02,0x00,0x00, +0x83,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00, +0x86,0x02,0x00,0x00,0xe6,0x02,0x00,0x00,0x60,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0x82,0x02,0x00,0x00,0x83,0x02,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x86,0x02,0x00,0x00, +0x81,0x02,0x00,0x00,0x82,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x81,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x8a,0x02,0x00,0x00,0xe6,0x02,0x00,0x00,0x61,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8b,0x02,0x00,0x00, +0x60,0x02,0x00,0x00,0x8a,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x8d,0x02,0x00,0x00,0x64,0x00,0x00,0x00, +0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x8e,0x02,0x00,0x00,0x8b,0x02,0x00,0x00,0x8d,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x92,0x02,0x00,0x00, +0xe5,0x02,0x00,0x00,0x00,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x93,0x02,0x00,0x00,0x66,0x02,0x00,0x00, +0x92,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x95,0x02,0x00,0x00,0x68,0x00,0x00,0x00,0xbb,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x96,0x02,0x00,0x00, +0x93,0x02,0x00,0x00,0x95,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0x98,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x98,0x02,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xe8,0x02,0x00,0x00, +0x3e,0x00,0x00,0x00,0x81,0x02,0x00,0x00,0xda,0x02,0x00,0x00, +0x9b,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00, +0x9e,0x02,0x00,0x00,0xe8,0x02,0x00,0x00,0xbb,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0x9a,0x02,0x00,0x00,0x9b,0x02,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x9e,0x02,0x00,0x00, +0x99,0x02,0x00,0x00,0x9a,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x99,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0xa0,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0xa0,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xea,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, +0x99,0x02,0x00,0x00,0xd8,0x02,0x00,0x00,0xa3,0x02,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00,0xa6,0x02,0x00,0x00, +0xea,0x02,0x00,0x00,0x62,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0xa2,0x02,0x00,0x00,0xa3,0x02,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0xa6,0x02,0x00,0x00,0xa1,0x02,0x00,0x00, +0xa2,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0xa1,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa9,0x02,0x00,0x00, +0x8e,0x02,0x00,0x00,0xea,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc1,0x00,0x00,0x00,0xac,0x02,0x00,0x00,0xa9,0x02,0x00,0x00, +0x36,0x00,0x00,0x00,0xf7,0x00,0x03,0x00,0xae,0x02,0x00,0x00, +0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xac,0x02,0x00,0x00, +0xad,0x02,0x00,0x00,0xae,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0xad,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xb1,0x02,0x00,0x00,0x96,0x02,0x00,0x00,0xe8,0x02,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0xb2,0x02,0x00,0x00, +0x12,0x00,0x00,0x00,0xcf,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xb3,0x02,0x00,0x00,0xb2,0x02,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00,0xb4,0x02,0x00,0x00, +0xb1,0x02,0x00,0x00,0xb3,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0xae,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0xae,0x02,0x00,0x00, +0xf5,0x00,0x07,0x00,0xc1,0x00,0x00,0x00,0xb5,0x02,0x00,0x00, +0xac,0x02,0x00,0x00,0xa1,0x02,0x00,0x00,0xb4,0x02,0x00,0x00, +0xad,0x02,0x00,0x00,0xf7,0x00,0x03,0x00,0xb7,0x02,0x00,0x00, +0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xb5,0x02,0x00,0x00, +0xb6,0x02,0x00,0x00,0xb7,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0xb6,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xbf,0x02,0x00,0x00,0x96,0x02,0x00,0x00,0xe8,0x02,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0xc1,0x02,0x00,0x00, +0x12,0x00,0x00,0x00,0xc0,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xc2,0x02,0x00,0x00,0xc1,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xc3,0x02,0x00,0x00, +0xbf,0x02,0x00,0x00,0xc2,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xc4,0x02,0x00,0x00,0x76,0x02,0x00,0x00, +0xc3,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xc6,0x02,0x00,0x00,0xc4,0x02,0x00,0x00,0x8e,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xc8,0x02,0x00,0x00, +0xc6,0x02,0x00,0x00,0xea,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xca,0x02,0x00,0x00,0xe5,0x02,0x00,0x00, +0xbb,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xcc,0x02,0x00,0x00,0xca,0x02,0x00,0x00,0xe8,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xce,0x02,0x00,0x00, +0xcc,0x02,0x00,0x00,0xcd,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xd0,0x02,0x00,0x00,0xe6,0x02,0x00,0x00, +0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xd1,0x02,0x00,0x00,0xce,0x02,0x00,0x00,0xd0,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xd3,0x02,0x00,0x00, +0xd1,0x02,0x00,0x00,0xea,0x02,0x00,0x00,0x41,0x00,0x05,0x00, +0xcc,0x00,0x00,0x00,0xd4,0x02,0x00,0x00,0xc9,0x00,0x00,0x00, +0xd3,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0xc3,0x00,0x00,0x00, +0xd5,0x02,0x00,0x00,0xd4,0x02,0x00,0x00,0x41,0x00,0x06,0x00, +0x97,0x01,0x00,0x00,0xd6,0x02,0x00,0x00,0xbb,0x02,0x00,0x00, +0x34,0x00,0x00,0x00,0xc8,0x02,0x00,0x00,0x3e,0x00,0x03,0x00, +0xd6,0x02,0x00,0x00,0xd5,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0xb7,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0xb7,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0xa3,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0xa3,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xd8,0x02,0x00,0x00,0xea,0x02,0x00,0x00,0xcf,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xa0,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0xa2,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x9b,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x9b,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xda,0x02,0x00,0x00,0xe8,0x02,0x00,0x00, +0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x98,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x9a,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0x83,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x83,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xdc,0x02,0x00,0x00, +0xe6,0x02,0x00,0x00,0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x80,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x82,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0x7b,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x7b,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xde,0x02,0x00,0x00,0xe5,0x02,0x00,0x00,0xcf,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x78,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x7a,0x02,0x00,0x00,0xfd,0x00,0x01,0x00,0x38,0x00,0x01,0x00, + +}; +const uint64_t matmul_q5_0_f32_fp32_len = 10956; + +unsigned char matmul_q5_1_f32_data[] = { +0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, +0x17,0x03,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, +0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x09,0x00,0x00,0x00, +0x11,0x00,0x02,0x00,0x51,0x11,0x00,0x00,0x11,0x00,0x02,0x00, +0x60,0x11,0x00,0x00,0x0b,0x00,0x06,0x00,0x01,0x00,0x00,0x00, +0x47,0x4c,0x53,0x4c,0x2e,0x73,0x74,0x64,0x2e,0x34,0x35,0x30, +0x00,0x00,0x00,0x00,0x0e,0x00,0x03,0x00,0x00,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x0f,0x00,0x0f,0x00,0x05,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x6d,0x61,0x69,0x6e,0x00,0x00,0x00,0x00, +0x0b,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x3d,0x00,0x00,0x00, +0x4c,0x00,0x00,0x00,0x07,0x01,0x00,0x00,0x4b,0x01,0x00,0x00, +0x7e,0x01,0x00,0x00,0x89,0x01,0x00,0x00,0x74,0x02,0x00,0x00, +0xbd,0x02,0x00,0x00,0x10,0x00,0x06,0x00,0x04,0x00,0x00,0x00, +0x11,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x0b,0x00,0x00,0x00, +0x0b,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x10,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x10,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x10,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x05,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x10,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, +0x07,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x1c,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x08,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x10,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x24,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, +0x0a,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x28,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x2c,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x10,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x30,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, +0x0d,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x34,0x00,0x00,0x00, +0x47,0x00,0x03,0x00,0x10,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x37,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x3d,0x00,0x00,0x00, +0x0b,0x00,0x00,0x00,0x1a,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x4c,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x1b,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x4f,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x09,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x53,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x60,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x62,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x07,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x6c,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0xa6,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0xb8,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x05,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xbb,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x02,0x01,0x00,0x00,0x06,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x03,0x01,0x00,0x00,0x00,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x03,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x03,0x01,0x00,0x00, +0x02,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x03,0x01,0x00,0x00,0x03,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x04,0x01,0x00,0x00,0x06,0x00,0x00,0x00,0x18,0x00,0x00,0x00, +0x48,0x00,0x04,0x00,0x05,0x01,0x00,0x00,0x00,0x00,0x00,0x00, +0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x05,0x01,0x00,0x00, +0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x03,0x00,0x05,0x01,0x00,0x00,0x02,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x07,0x01,0x00,0x00,0x22,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x07,0x01,0x00,0x00, +0x21,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x58,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x59,0x01,0x00,0x00,0x0b,0x00,0x00,0x00, +0x19,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x86,0x01,0x00,0x00, +0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00, +0x87,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x87,0x01,0x00,0x00,0x00,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, +0x87,0x01,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x89,0x01,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x89,0x01,0x00,0x00,0x21,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x74,0x02,0x00,0x00, +0x0b,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0xba,0x02,0x00,0x00,0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x48,0x00,0x04,0x00,0xbb,0x02,0x00,0x00,0x00,0x00,0x00,0x00, +0x19,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0xbb,0x02,0x00,0x00, +0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x03,0x00,0xbb,0x02,0x00,0x00,0x02,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0xbd,0x02,0x00,0x00,0x22,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xbd,0x02,0x00,0x00, +0x21,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x13,0x00,0x02,0x00, +0x02,0x00,0x00,0x00,0x21,0x00,0x03,0x00,0x03,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x15,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x17,0x00,0x04,0x00, +0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x03,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x0a,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x09,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00, +0x0b,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x0d,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x1e,0x00,0x10,0x00,0x10,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x11,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x10,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0x11,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0x09,0x00,0x00,0x00,0x15,0x00,0x04,0x00,0x13,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x08,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x15,0x00,0x00,0x00,0x09,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, +0x21,0x00,0x00,0x00,0x0a,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0x27,0x00,0x00,0x00,0x09,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x2d,0x00,0x00,0x00, +0x07,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, +0x34,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x32,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x40,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00, +0x3d,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x3e,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00,0x4c,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x4f,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x32,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x54,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x53,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x58,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x53,0x00,0x00,0x00, +0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x60,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x61,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x53,0x00,0x00,0x00, +0x60,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x62,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x63,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0x61,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x67,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0x61,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x32,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x10,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x72,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x77,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x7c,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x80,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, +0x85,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0x90,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x96,0x00,0x00,0x00, +0x03,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, +0xa1,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x32,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xa6,0x00,0x00,0x00,0x40,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0xa8,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xb7,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00, +0x62,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xb8,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xb9,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0x53,0x00,0x00,0x00,0xb8,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xba,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0x4f,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x32,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xbb,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xbc,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0xba,0x00,0x00,0x00,0xbb,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xbd,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0xbc,0x00,0x00,0x00,0x60,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xbe,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0xb9,0x00,0x00,0x00,0xbd,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xbf,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0xb7,0x00,0x00,0x00,0xbe,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xc0,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0xbf,0x00,0x00,0x00,0xbb,0x00,0x00,0x00, +0x14,0x00,0x02,0x00,0xc1,0x00,0x00,0x00,0x16,0x00,0x03,0x00, +0xc3,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xc4,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xc5,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0xc4,0x00,0x00,0x00,0xbe,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xc6,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0xc5,0x00,0x00,0x00,0xbb,0x00,0x00,0x00,0x1c,0x00,0x04,0x00, +0xc7,0x00,0x00,0x00,0xc3,0x00,0x00,0x00,0xc6,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0xc8,0x00,0x00,0x00,0x07,0x00,0x00,0x00, +0xc7,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0xc3,0x00,0x00,0x00, +0xcb,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0xcc,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0xc3,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0xcf,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xf3,0x00,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, +0x39,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xf9,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xfd,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, +0x16,0x00,0x03,0x00,0x00,0x01,0x00,0x00,0x10,0x00,0x00,0x00, +0x15,0x00,0x04,0x00,0x01,0x01,0x00,0x00,0x08,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x1c,0x00,0x04,0x00,0x02,0x01,0x00,0x00, +0x01,0x01,0x00,0x00,0xf9,0x00,0x00,0x00,0x1e,0x00,0x06,0x00, +0x03,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x01,0x00,0x00, +0x06,0x00,0x00,0x00,0x02,0x01,0x00,0x00,0x1d,0x00,0x03,0x00, +0x04,0x01,0x00,0x00,0x03,0x01,0x00,0x00,0x1e,0x00,0x03,0x00, +0x05,0x01,0x00,0x00,0x04,0x01,0x00,0x00,0x20,0x00,0x04,0x00, +0x06,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x05,0x01,0x00,0x00, +0x3b,0x00,0x04,0x00,0x06,0x01,0x00,0x00,0x07,0x01,0x00,0x00, +0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x09,0x01,0x00,0x00, +0x0c,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x20,0x00,0x04,0x00, +0x14,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x22,0x01,0x00,0x00, +0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x2b,0x01,0x00,0x00, +0x0c,0x00,0x00,0x00,0x01,0x01,0x00,0x00,0x17,0x00,0x04,0x00, +0x2f,0x01,0x00,0x00,0xc3,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x47,0x01,0x00,0x00, +0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x48,0x01,0x00,0x00, +0x84,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x47,0x01,0x00,0x00, +0x1c,0x00,0x04,0x00,0x49,0x01,0x00,0x00,0x00,0x01,0x00,0x00, +0x48,0x01,0x00,0x00,0x20,0x00,0x04,0x00,0x4a,0x01,0x00,0x00, +0x04,0x00,0x00,0x00,0x49,0x01,0x00,0x00,0x3b,0x00,0x04,0x00, +0x4a,0x01,0x00,0x00,0x4b,0x01,0x00,0x00,0x04,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x50,0x01,0x00,0x00,0x04,0x00,0x00,0x00, +0x00,0x01,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x58,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x33,0x00,0x06,0x00, +0x09,0x00,0x00,0x00,0x59,0x01,0x00,0x00,0x58,0x01,0x00,0x00, +0x39,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x5a,0x01,0x00,0x00,0x51,0x00,0x00,0x00, +0x59,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x5b,0x01,0x00,0x00,0x84,0x00,0x00,0x00, +0x5a,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x5c,0x01,0x00,0x00,0x86,0x00,0x00,0x00, +0x5b,0x01,0x00,0x00,0x6c,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x7a,0x01,0x00,0x00,0x80,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x7b,0x01,0x00,0x00,0x84,0x00,0x00,0x00, +0xa6,0x00,0x00,0x00,0x7a,0x01,0x00,0x00,0x1c,0x00,0x04,0x00, +0x7c,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x7b,0x01,0x00,0x00, +0x20,0x00,0x04,0x00,0x7d,0x01,0x00,0x00,0x04,0x00,0x00,0x00, +0x7c,0x01,0x00,0x00,0x3b,0x00,0x04,0x00,0x7d,0x01,0x00,0x00, +0x7e,0x01,0x00,0x00,0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x82,0x01,0x00,0x00,0x80,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, +0x86,0x01,0x00,0x00,0xc3,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, +0x87,0x01,0x00,0x00,0x86,0x01,0x00,0x00,0x20,0x00,0x04,0x00, +0x88,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x87,0x01,0x00,0x00, +0x3b,0x00,0x04,0x00,0x88,0x01,0x00,0x00,0x89,0x01,0x00,0x00, +0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x94,0x01,0x00,0x00, +0x0c,0x00,0x00,0x00,0xc3,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x9d,0x01,0x00,0x00,0x80,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x00,0x01,0x00,0x00,0xa1,0x01,0x00,0x00,0x00,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xa3,0x01,0x00,0x00, +0x51,0x00,0x00,0x00,0x59,0x01,0x00,0x00,0x00,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xa4,0x01,0x00,0x00, +0x84,0x00,0x00,0x00,0xa3,0x01,0x00,0x00,0x39,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xa5,0x01,0x00,0x00, +0x86,0x00,0x00,0x00,0xa4,0x01,0x00,0x00,0x6c,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xa8,0x01,0x00,0x00, +0x08,0x01,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xa9,0x01,0x00,0x00,0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xac,0x01,0x00,0x00,0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, +0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xc7,0x01,0x00,0x00,0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00, +0x62,0x00,0x00,0x00,0x1c,0x00,0x04,0x00,0xc8,0x01,0x00,0x00, +0x00,0x01,0x00,0x00,0xc7,0x01,0x00,0x00,0x20,0x00,0x04,0x00, +0xc9,0x01,0x00,0x00,0x07,0x00,0x00,0x00,0xc8,0x01,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xd9,0x01,0x00,0x00, +0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0xdf,0x01,0x00,0x00,0x07,0x00,0x00,0x00, +0x00,0x01,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xf5,0x01,0x00,0x00,0x84,0x00,0x00,0x00,0xbe,0x00,0x00,0x00, +0xbb,0x00,0x00,0x00,0x1c,0x00,0x04,0x00,0xf6,0x01,0x00,0x00, +0x00,0x01,0x00,0x00,0xf5,0x01,0x00,0x00,0x20,0x00,0x04,0x00, +0xf7,0x01,0x00,0x00,0x07,0x00,0x00,0x00,0xf6,0x01,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x00,0x02,0x00,0x00, +0x86,0x00,0x00,0x00,0xb8,0x00,0x00,0x00,0xbe,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x08,0x02,0x00,0x00, +0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x37,0x02,0x00,0x00, +0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x6c,0x02,0x00,0x00, +0x0d,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00, +0x74,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, +0xba,0x02,0x00,0x00,0xc3,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, +0xbb,0x02,0x00,0x00,0xba,0x02,0x00,0x00,0x20,0x00,0x04,0x00, +0xbc,0x02,0x00,0x00,0x0c,0x00,0x00,0x00,0xbb,0x02,0x00,0x00, +0x3b,0x00,0x04,0x00,0xbc,0x02,0x00,0x00,0xbd,0x02,0x00,0x00, +0x0c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, +0xc2,0x02,0x00,0x00,0x05,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xcf,0x02,0x00,0x00,0x84,0x00,0x00,0x00, +0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x36,0x00,0x05,0x00, +0x02,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x03,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x05,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0xc8,0x00,0x00,0x00,0xc9,0x00,0x00,0x00, +0x07,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0xc9,0x01,0x00,0x00, +0xca,0x01,0x00,0x00,0x07,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0xf7,0x01,0x00,0x00,0xf8,0x01,0x00,0x00,0x07,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00,0x0e,0x00,0x00,0x00, +0x0b,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x0e,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x16,0x00,0x00,0x00, +0x12,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x17,0x00,0x00,0x00,0x16,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x18,0x00,0x00,0x00, +0x0f,0x00,0x00,0x00,0x17,0x00,0x00,0x00,0x89,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x1e,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, +0x17,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, +0x22,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x21,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x22,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x24,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x28,0x00,0x00,0x00, +0x12,0x00,0x00,0x00,0x27,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x29,0x00,0x00,0x00,0x28,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x2a,0x00,0x00,0x00, +0x1e,0x00,0x00,0x00,0x29,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x15,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0x2d,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x2f,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x30,0x00,0x00,0x00,0x24,0x00,0x00,0x00, +0x2f,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x32,0x00,0x00,0x00,0x30,0x00,0x00,0x00,0x2a,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x35,0x00,0x00,0x00, +0x12,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x36,0x00,0x00,0x00,0x35,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x38,0x00,0x00,0x00, +0x36,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x82,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x38,0x00,0x00,0x00, +0x39,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x3b,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x37,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00,0x3f,0x00,0x00,0x00, +0x3d,0x00,0x00,0x00,0x3e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x3f,0x00,0x00,0x00, +0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x42,0x00,0x00,0x00, +0x40,0x00,0x00,0x00,0x3b,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x47,0x00,0x00,0x00,0x40,0x00,0x00,0x00, +0x3b,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00, +0x49,0x00,0x00,0x00,0x3d,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x4a,0x00,0x00,0x00, +0x49,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00, +0x4d,0x00,0x00,0x00,0x4c,0x00,0x00,0x00,0x3e,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, +0x4d,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x50,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x4f,0x00,0x00,0x00, +0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x55,0x00,0x00,0x00, +0x50,0x00,0x00,0x00,0x54,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x59,0x00,0x00,0x00,0x50,0x00,0x00,0x00, +0x58,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x5d,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x4f,0x00,0x00,0x00, +0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x64,0x00,0x00,0x00, +0x5d,0x00,0x00,0x00,0x63,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x68,0x00,0x00,0x00,0x5d,0x00,0x00,0x00, +0x67,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x6e,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x73,0x00,0x00,0x00, +0x4e,0x00,0x00,0x00,0x72,0x00,0x00,0x00,0x89,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x78,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, +0x77,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x7d,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x7c,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x81,0x00,0x00,0x00, +0x12,0x00,0x00,0x00,0x80,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x82,0x00,0x00,0x00,0x81,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x83,0x00,0x00,0x00, +0x47,0x00,0x00,0x00,0x82,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x15,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0x85,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x87,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x89,0x00,0x00,0x00,0x47,0x00,0x00,0x00, +0x39,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x8c,0x00,0x00,0x00,0x89,0x00,0x00,0x00,0x82,0x00,0x00,0x00, +0x0c,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x8d,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x26,0x00,0x00,0x00,0x87,0x00,0x00,0x00, +0x8c,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, +0x91,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x90,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x92,0x00,0x00,0x00, +0x91,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x93,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x92,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x95,0x00,0x00,0x00, +0x42,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x15,0x00,0x00,0x00,0x97,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0x96,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x98,0x00,0x00,0x00,0x97,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x99,0x00,0x00,0x00,0x95,0x00,0x00,0x00, +0x98,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x9a,0x00,0x00,0x00,0x93,0x00,0x00,0x00,0x99,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9c,0x00,0x00,0x00, +0x9a,0x00,0x00,0x00,0x83,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x9d,0x00,0x00,0x00,0x9c,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, +0xa2,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0xa1,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xa3,0x00,0x00,0x00, +0xa2,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xa4,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0xa3,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa7,0x00,0x00,0x00, +0x4a,0x00,0x00,0x00,0xa6,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x15,0x00,0x00,0x00,0xa9,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0xa8,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xaa,0x00,0x00,0x00,0xa9,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xab,0x00,0x00,0x00,0xa7,0x00,0x00,0x00, +0xaa,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xac,0x00,0x00,0x00,0xa4,0x00,0x00,0x00,0xab,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xae,0x00,0x00,0x00, +0xac,0x00,0x00,0x00,0x83,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xaf,0x00,0x00,0x00,0xae,0x00,0x00,0x00, +0x39,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xb1,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xb1,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xe5,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, +0x05,0x00,0x00,0x00,0xd0,0x00,0x00,0x00,0xb2,0x00,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00,0xc2,0x00,0x00,0x00, +0xe5,0x02,0x00,0x00,0xc0,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0xb3,0x00,0x00,0x00,0xb2,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0xc2,0x00,0x00,0x00,0xb2,0x00,0x00,0x00, +0xb3,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xb2,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0xcc,0x00,0x00,0x00,0xcd,0x00,0x00,0x00, +0xc9,0x00,0x00,0x00,0xe5,0x02,0x00,0x00,0x3e,0x00,0x03,0x00, +0xcd,0x00,0x00,0x00,0xcb,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xd0,0x00,0x00,0x00,0xe5,0x02,0x00,0x00, +0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xb1,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xb3,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xd3,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xd3,0x00,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xfe,0x02,0x00,0x00, +0xaf,0x00,0x00,0x00,0xb3,0x00,0x00,0x00,0xae,0x01,0x00,0x00, +0xd6,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xfa,0x02,0x00,0x00,0x9d,0x00,0x00,0x00,0xb3,0x00,0x00,0x00, +0xab,0x01,0x00,0x00,0xd6,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xe6,0x02,0x00,0x00,0x83,0x00,0x00,0x00, +0xb3,0x00,0x00,0x00,0x5c,0x02,0x00,0x00,0xd6,0x00,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00,0xda,0x00,0x00,0x00, +0xe6,0x02,0x00,0x00,0x8d,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0xd5,0x00,0x00,0x00,0xd6,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0xda,0x00,0x00,0x00,0xd4,0x00,0x00,0x00, +0xd5,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xd4,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xdc,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xdc,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xf6,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0xd4,0x00,0x00,0x00, +0x5e,0x01,0x00,0x00,0xdd,0x00,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc1,0x00,0x00,0x00,0xe2,0x00,0x00,0x00,0xf6,0x02,0x00,0x00, +0x37,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xde,0x00,0x00,0x00, +0xdd,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0xe2,0x00,0x00,0x00,0xdd,0x00,0x00,0x00,0xde,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xdd,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xe7,0x00,0x00,0x00,0x73,0x00,0x00,0x00, +0xf6,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xea,0x00,0x00,0x00,0xe7,0x00,0x00,0x00,0x98,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xeb,0x00,0x00,0x00, +0xea,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xec,0x00,0x00,0x00,0xfa,0x02,0x00,0x00, +0xeb,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xee,0x00,0x00,0x00,0xec,0x00,0x00,0x00,0x6e,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xf4,0x00,0x00,0x00, +0xe7,0x00,0x00,0x00,0xf3,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xf6,0x00,0x00,0x00,0xf4,0x00,0x00,0x00, +0x6e,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xfa,0x00,0x00,0x00,0xee,0x00,0x00,0x00,0xf9,0x00,0x00,0x00, +0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xfe,0x00,0x00,0x00, +0xee,0x00,0x00,0x00,0xfd,0x00,0x00,0x00,0x41,0x00,0x07,0x00, +0x09,0x01,0x00,0x00,0x0a,0x01,0x00,0x00,0x07,0x01,0x00,0x00, +0x34,0x00,0x00,0x00,0xfa,0x00,0x00,0x00,0x34,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x00,0x01,0x00,0x00,0x0b,0x01,0x00,0x00, +0x0a,0x01,0x00,0x00,0x73,0x00,0x04,0x00,0xc3,0x00,0x00,0x00, +0x0c,0x01,0x00,0x00,0x0b,0x01,0x00,0x00,0x41,0x00,0x07,0x00, +0x09,0x01,0x00,0x00,0x0f,0x01,0x00,0x00,0x07,0x01,0x00,0x00, +0x34,0x00,0x00,0x00,0xfa,0x00,0x00,0x00,0xcf,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x00,0x01,0x00,0x00,0x10,0x01,0x00,0x00, +0x0f,0x01,0x00,0x00,0x73,0x00,0x04,0x00,0xc3,0x00,0x00,0x00, +0x11,0x01,0x00,0x00,0x10,0x01,0x00,0x00,0x41,0x00,0x07,0x00, +0x14,0x01,0x00,0x00,0x15,0x01,0x00,0x00,0x07,0x01,0x00,0x00, +0x34,0x00,0x00,0x00,0xfa,0x00,0x00,0x00,0x85,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x16,0x01,0x00,0x00, +0x15,0x01,0x00,0x00,0xc2,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x1c,0x01,0x00,0x00,0x16,0x01,0x00,0x00,0xfe,0x00,0x00,0x00, +0xc4,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x1d,0x01,0x00,0x00, +0x1c,0x01,0x00,0x00,0xa8,0x00,0x00,0x00,0xc7,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x1e,0x01,0x00,0x00,0x1d,0x01,0x00,0x00, +0xf9,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x13,0x00,0x00,0x00, +0x1f,0x01,0x00,0x00,0x1e,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x23,0x01,0x00,0x00,0xfe,0x00,0x00,0x00, +0x22,0x01,0x00,0x00,0xc2,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x24,0x01,0x00,0x00,0x16,0x01,0x00,0x00,0x23,0x01,0x00,0x00, +0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x25,0x01,0x00,0x00, +0x24,0x01,0x00,0x00,0xf9,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0x26,0x01,0x00,0x00,0x25,0x01,0x00,0x00, +0x41,0x00,0x08,0x00,0x2b,0x01,0x00,0x00,0x2c,0x01,0x00,0x00, +0x07,0x01,0x00,0x00,0x34,0x00,0x00,0x00,0xfa,0x00,0x00,0x00, +0x96,0x00,0x00,0x00,0xfe,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x01,0x01,0x00,0x00,0x2d,0x01,0x00,0x00,0x2c,0x01,0x00,0x00, +0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x2e,0x01,0x00,0x00, +0x2d,0x01,0x00,0x00,0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x33,0x01,0x00,0x00,0x2e,0x01,0x00,0x00,0xfd,0x00,0x00,0x00, +0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x37,0x01,0x00,0x00, +0x1f,0x01,0x00,0x00,0xc5,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x38,0x01,0x00,0x00,0x33,0x01,0x00,0x00,0x37,0x01,0x00,0x00, +0x70,0x00,0x04,0x00,0xc3,0x00,0x00,0x00,0x39,0x01,0x00,0x00, +0x38,0x01,0x00,0x00,0xc2,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x3b,0x01,0x00,0x00,0x2e,0x01,0x00,0x00,0xa8,0x00,0x00,0x00, +0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x3e,0x01,0x00,0x00, +0x26,0x01,0x00,0x00,0xc5,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x3f,0x01,0x00,0x00,0x3b,0x01,0x00,0x00,0x3e,0x01,0x00,0x00, +0x70,0x00,0x04,0x00,0xc3,0x00,0x00,0x00,0x40,0x01,0x00,0x00, +0x3f,0x01,0x00,0x00,0x50,0x00,0x05,0x00,0x2f,0x01,0x00,0x00, +0x41,0x01,0x00,0x00,0x39,0x01,0x00,0x00,0x40,0x01,0x00,0x00, +0x8e,0x00,0x05,0x00,0x2f,0x01,0x00,0x00,0x43,0x01,0x00,0x00, +0x41,0x01,0x00,0x00,0x0c,0x01,0x00,0x00,0x50,0x00,0x05,0x00, +0x2f,0x01,0x00,0x00,0x45,0x01,0x00,0x00,0x11,0x01,0x00,0x00, +0x11,0x01,0x00,0x00,0x81,0x00,0x05,0x00,0x2f,0x01,0x00,0x00, +0x46,0x01,0x00,0x00,0x43,0x01,0x00,0x00,0x45,0x01,0x00,0x00, +0x51,0x00,0x05,0x00,0xc3,0x00,0x00,0x00,0x4e,0x01,0x00,0x00, +0x46,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x73,0x00,0x04,0x00, +0x00,0x01,0x00,0x00,0x4f,0x01,0x00,0x00,0x4e,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0x50,0x01,0x00,0x00,0x51,0x01,0x00,0x00, +0x4b,0x01,0x00,0x00,0xf6,0x00,0x00,0x00,0x3e,0x00,0x03,0x00, +0x51,0x01,0x00,0x00,0x4f,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x53,0x01,0x00,0x00,0xf6,0x00,0x00,0x00, +0xf9,0x00,0x00,0x00,0x51,0x00,0x05,0x00,0xc3,0x00,0x00,0x00, +0x55,0x01,0x00,0x00,0x46,0x01,0x00,0x00,0x01,0x00,0x00,0x00, +0x73,0x00,0x04,0x00,0x00,0x01,0x00,0x00,0x56,0x01,0x00,0x00, +0x55,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0x50,0x01,0x00,0x00, +0x57,0x01,0x00,0x00,0x4b,0x01,0x00,0x00,0x53,0x01,0x00,0x00, +0x3e,0x00,0x03,0x00,0x57,0x01,0x00,0x00,0x56,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x5e,0x01,0x00,0x00, +0xf6,0x02,0x00,0x00,0x5c,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0xdc,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xde,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x60,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x60,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xf7,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0xde,0x00,0x00,0x00, +0xa7,0x01,0x00,0x00,0x63,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc1,0x00,0x00,0x00,0x66,0x01,0x00,0x00,0xf7,0x02,0x00,0x00, +0xa6,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x62,0x01,0x00,0x00, +0x63,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x66,0x01,0x00,0x00,0x61,0x01,0x00,0x00,0x62,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x61,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x6a,0x01,0x00,0x00,0xa7,0x00,0x00,0x00, +0x7d,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x6c,0x01,0x00,0x00,0x6a,0x01,0x00,0x00,0xf7,0x02,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x6d,0x01,0x00,0x00, +0x12,0x00,0x00,0x00,0xcf,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x6e,0x01,0x00,0x00,0x6d,0x01,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00,0x6f,0x01,0x00,0x00, +0x6c,0x01,0x00,0x00,0x6e,0x01,0x00,0x00,0xf7,0x00,0x03,0x00, +0x71,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x6f,0x01,0x00,0x00,0x70,0x01,0x00,0x00,0x71,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x70,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x74,0x01,0x00,0x00,0xe6,0x02,0x00,0x00, +0x78,0x00,0x00,0x00,0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00, +0x76,0x01,0x00,0x00,0x74,0x01,0x00,0x00,0x8d,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x71,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x71,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0xc1,0x00,0x00,0x00, +0x77,0x01,0x00,0x00,0x6f,0x01,0x00,0x00,0x61,0x01,0x00,0x00, +0x76,0x01,0x00,0x00,0x70,0x01,0x00,0x00,0xf7,0x00,0x03,0x00, +0x79,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x77,0x01,0x00,0x00,0x78,0x01,0x00,0x00,0x99,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x78,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x81,0x01,0x00,0x00,0x7d,0x00,0x00,0x00, +0xf7,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x83,0x01,0x00,0x00,0x81,0x01,0x00,0x00,0x82,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x85,0x01,0x00,0x00, +0x83,0x01,0x00,0x00,0x78,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x90,0x01,0x00,0x00,0x81,0x01,0x00,0x00, +0xaa,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x91,0x01,0x00,0x00,0xfe,0x02,0x00,0x00,0x90,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x93,0x01,0x00,0x00, +0x91,0x01,0x00,0x00,0x78,0x00,0x00,0x00,0x41,0x00,0x06,0x00, +0x94,0x01,0x00,0x00,0x95,0x01,0x00,0x00,0x89,0x01,0x00,0x00, +0x34,0x00,0x00,0x00,0x93,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, +0xc3,0x00,0x00,0x00,0x96,0x01,0x00,0x00,0x95,0x01,0x00,0x00, +0x73,0x00,0x04,0x00,0x00,0x01,0x00,0x00,0x97,0x01,0x00,0x00, +0x96,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0x50,0x01,0x00,0x00, +0x98,0x01,0x00,0x00,0x7e,0x01,0x00,0x00,0x85,0x01,0x00,0x00, +0x3e,0x00,0x03,0x00,0x98,0x01,0x00,0x00,0x97,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0x79,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x99,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x9c,0x01,0x00,0x00,0x7d,0x00,0x00,0x00,0xf7,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9e,0x01,0x00,0x00, +0x9c,0x01,0x00,0x00,0x9d,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xa0,0x01,0x00,0x00,0x9e,0x01,0x00,0x00, +0x78,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x50,0x01,0x00,0x00, +0xa2,0x01,0x00,0x00,0x7e,0x01,0x00,0x00,0xa0,0x01,0x00,0x00, +0x3e,0x00,0x03,0x00,0xa2,0x01,0x00,0x00,0xa1,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0x79,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x79,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x63,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x63,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xa7,0x01,0x00,0x00,0xf7,0x02,0x00,0x00, +0xa5,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x60,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x62,0x01,0x00,0x00,0xe0,0x00,0x04,0x00, +0x0c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0xa8,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xab,0x01,0x00,0x00, +0xfa,0x02,0x00,0x00,0xa9,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xae,0x01,0x00,0x00,0xfe,0x02,0x00,0x00, +0xac,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xb0,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xb0,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0x00,0x03,0x00,0x00,0x3e,0x00,0x00,0x00, +0x62,0x01,0x00,0x00,0x5a,0x02,0x00,0x00,0xb3,0x01,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00,0xb6,0x01,0x00,0x00, +0x00,0x03,0x00,0x00,0x6c,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0xb2,0x01,0x00,0x00,0xb3,0x01,0x00,0x00,0x00,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0xb6,0x01,0x00,0x00,0xb1,0x01,0x00,0x00, +0xb2,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xb1,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0xb8,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xb8,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0x04,0x03,0x00,0x00,0x3e,0x00,0x00,0x00,0xb1,0x01,0x00,0x00, +0xe4,0x01,0x00,0x00,0xbb,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc1,0x00,0x00,0x00,0xbe,0x01,0x00,0x00,0x04,0x03,0x00,0x00, +0x60,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xba,0x01,0x00,0x00, +0xbb,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0xbe,0x01,0x00,0x00,0xb9,0x01,0x00,0x00,0xba,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xb9,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0xc0,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xc0,0x01,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x16,0x03,0x00,0x00, +0x3e,0x00,0x00,0x00,0xb9,0x01,0x00,0x00,0xe2,0x01,0x00,0x00, +0xc1,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00, +0xc6,0x01,0x00,0x00,0x16,0x03,0x00,0x00,0x62,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0xc2,0x01,0x00,0x00,0xc1,0x01,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xc6,0x01,0x00,0x00, +0xc1,0x01,0x00,0x00,0xc2,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xc1,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xcc,0x01,0x00,0x00,0x04,0x03,0x00,0x00,0x62,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xce,0x01,0x00,0x00, +0xcc,0x01,0x00,0x00,0x16,0x03,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xd0,0x01,0x00,0x00,0x55,0x00,0x00,0x00, +0x53,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xd2,0x01,0x00,0x00,0x04,0x03,0x00,0x00,0x61,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xd3,0x01,0x00,0x00, +0xd0,0x01,0x00,0x00,0xd2,0x01,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xd5,0x01,0x00,0x00,0x64,0x00,0x00,0x00, +0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xd6,0x01,0x00,0x00,0xd3,0x01,0x00,0x00,0xd5,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xd8,0x01,0x00,0x00, +0xd6,0x01,0x00,0x00,0x16,0x03,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xda,0x01,0x00,0x00,0xd8,0x01,0x00,0x00, +0xd9,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xdc,0x01,0x00,0x00,0xda,0x01,0x00,0x00,0x00,0x03,0x00,0x00, +0x41,0x00,0x05,0x00,0x50,0x01,0x00,0x00,0xdd,0x01,0x00,0x00, +0x4b,0x01,0x00,0x00,0xdc,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, +0x00,0x01,0x00,0x00,0xde,0x01,0x00,0x00,0xdd,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0xdf,0x01,0x00,0x00,0xe0,0x01,0x00,0x00, +0xca,0x01,0x00,0x00,0xce,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, +0xe0,0x01,0x00,0x00,0xde,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xe2,0x01,0x00,0x00,0x16,0x03,0x00,0x00, +0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xc0,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xc2,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0xbb,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xbb,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xe4,0x01,0x00,0x00, +0x04,0x03,0x00,0x00,0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xb8,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xba,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0xe6,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xe6,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0x05,0x03,0x00,0x00,0x3e,0x00,0x00,0x00,0xba,0x01,0x00,0x00, +0x12,0x02,0x00,0x00,0xe9,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc1,0x00,0x00,0x00,0xec,0x01,0x00,0x00,0x05,0x03,0x00,0x00, +0xbe,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xe8,0x01,0x00,0x00, +0xe9,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0xec,0x01,0x00,0x00,0xe7,0x01,0x00,0x00,0xe8,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xe7,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0xee,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xee,0x01,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x13,0x03,0x00,0x00, +0x3e,0x00,0x00,0x00,0xe7,0x01,0x00,0x00,0x10,0x02,0x00,0x00, +0xef,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00, +0xf4,0x01,0x00,0x00,0x13,0x03,0x00,0x00,0xbb,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0xf0,0x01,0x00,0x00,0xef,0x01,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xf4,0x01,0x00,0x00, +0xef,0x01,0x00,0x00,0xf0,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xef,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xfa,0x01,0x00,0x00,0x05,0x03,0x00,0x00,0xbb,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xfc,0x01,0x00,0x00, +0xfa,0x01,0x00,0x00,0x13,0x03,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xfe,0x01,0x00,0x00,0x59,0x00,0x00,0x00, +0xb8,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x01,0x02,0x00,0x00,0x05,0x03,0x00,0x00,0x00,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x02,0x02,0x00,0x00, +0xfe,0x01,0x00,0x00,0x01,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x04,0x02,0x00,0x00,0x68,0x00,0x00,0x00, +0xbb,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x05,0x02,0x00,0x00,0x02,0x02,0x00,0x00,0x04,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x07,0x02,0x00,0x00, +0x05,0x02,0x00,0x00,0x13,0x03,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x09,0x02,0x00,0x00,0x07,0x02,0x00,0x00, +0x08,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x0b,0x02,0x00,0x00,0x09,0x02,0x00,0x00,0x00,0x03,0x00,0x00, +0x41,0x00,0x05,0x00,0x50,0x01,0x00,0x00,0x0c,0x02,0x00,0x00, +0x7e,0x01,0x00,0x00,0x0b,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, +0x00,0x01,0x00,0x00,0x0d,0x02,0x00,0x00,0x0c,0x02,0x00,0x00, +0x41,0x00,0x05,0x00,0xdf,0x01,0x00,0x00,0x0e,0x02,0x00,0x00, +0xf8,0x01,0x00,0x00,0xfc,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, +0x0e,0x02,0x00,0x00,0x0d,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x10,0x02,0x00,0x00,0x13,0x03,0x00,0x00, +0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xee,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xf0,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0xe9,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xe9,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x12,0x02,0x00,0x00, +0x05,0x03,0x00,0x00,0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xe6,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xe8,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0x14,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x14,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0x06,0x03,0x00,0x00,0x3e,0x00,0x00,0x00,0xe8,0x01,0x00,0x00, +0x58,0x02,0x00,0x00,0x17,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc1,0x00,0x00,0x00,0x1a,0x02,0x00,0x00,0x06,0x03,0x00,0x00, +0xbe,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x16,0x02,0x00,0x00, +0x17,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x1a,0x02,0x00,0x00,0x15,0x02,0x00,0x00,0x16,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x15,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0x1c,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x1c,0x02,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x0a,0x03,0x00,0x00, +0x3e,0x00,0x00,0x00,0x15,0x02,0x00,0x00,0x56,0x02,0x00,0x00, +0x1f,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00, +0x22,0x02,0x00,0x00,0x0a,0x03,0x00,0x00,0x60,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0x1e,0x02,0x00,0x00,0x1f,0x02,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x22,0x02,0x00,0x00, +0x1d,0x02,0x00,0x00,0x1e,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x1d,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x24,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x24,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0x0c,0x03,0x00,0x00,0x3e,0x00,0x00,0x00, +0x1d,0x02,0x00,0x00,0x54,0x02,0x00,0x00,0x27,0x02,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00,0x2a,0x02,0x00,0x00, +0x0c,0x03,0x00,0x00,0xbb,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0x26,0x02,0x00,0x00,0x27,0x02,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x2a,0x02,0x00,0x00,0x25,0x02,0x00,0x00, +0x26,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x25,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0x2c,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x2c,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0x0e,0x03,0x00,0x00,0x3e,0x00,0x00,0x00,0x25,0x02,0x00,0x00, +0x52,0x02,0x00,0x00,0x2d,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc1,0x00,0x00,0x00,0x32,0x02,0x00,0x00,0x0e,0x03,0x00,0x00, +0x62,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x2e,0x02,0x00,0x00, +0x2d,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x32,0x02,0x00,0x00,0x2d,0x02,0x00,0x00,0x2e,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x2d,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x34,0x02,0x00,0x00,0x06,0x03,0x00,0x00, +0xbb,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x36,0x02,0x00,0x00,0x34,0x02,0x00,0x00,0x0c,0x03,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x38,0x02,0x00,0x00, +0x36,0x02,0x00,0x00,0x37,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x3a,0x02,0x00,0x00,0x0a,0x03,0x00,0x00, +0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x3b,0x02,0x00,0x00,0x38,0x02,0x00,0x00,0x3a,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3d,0x02,0x00,0x00, +0x3b,0x02,0x00,0x00,0x0e,0x03,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x41,0x02,0x00,0x00,0x3a,0x02,0x00,0x00, +0x0e,0x03,0x00,0x00,0x41,0x00,0x05,0x00,0xdf,0x01,0x00,0x00, +0x42,0x02,0x00,0x00,0xca,0x01,0x00,0x00,0x41,0x02,0x00,0x00, +0x3d,0x00,0x04,0x00,0x00,0x01,0x00,0x00,0x43,0x02,0x00,0x00, +0x42,0x02,0x00,0x00,0x73,0x00,0x04,0x00,0xc3,0x00,0x00,0x00, +0x44,0x02,0x00,0x00,0x43,0x02,0x00,0x00,0x41,0x00,0x05,0x00, +0xdf,0x01,0x00,0x00,0x49,0x02,0x00,0x00,0xf8,0x01,0x00,0x00, +0x36,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0x00,0x01,0x00,0x00, +0x4a,0x02,0x00,0x00,0x49,0x02,0x00,0x00,0x73,0x00,0x04,0x00, +0xc3,0x00,0x00,0x00,0x4b,0x02,0x00,0x00,0x4a,0x02,0x00,0x00, +0x41,0x00,0x05,0x00,0xcc,0x00,0x00,0x00,0x4d,0x02,0x00,0x00, +0xc9,0x00,0x00,0x00,0x3d,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, +0xc3,0x00,0x00,0x00,0x4e,0x02,0x00,0x00,0x4d,0x02,0x00,0x00, +0x0c,0x00,0x08,0x00,0xc3,0x00,0x00,0x00,0x4f,0x02,0x00,0x00, +0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x44,0x02,0x00,0x00, +0x4b,0x02,0x00,0x00,0x4e,0x02,0x00,0x00,0x3e,0x00,0x03,0x00, +0x4d,0x02,0x00,0x00,0x4f,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x52,0x02,0x00,0x00,0x0e,0x03,0x00,0x00, +0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x2c,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x2e,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0x27,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x27,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x54,0x02,0x00,0x00, +0x0c,0x03,0x00,0x00,0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x24,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x26,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0x1f,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x1f,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x56,0x02,0x00,0x00,0x0a,0x03,0x00,0x00,0xcf,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x1c,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x1e,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x17,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x17,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x58,0x02,0x00,0x00,0x06,0x03,0x00,0x00, +0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x14,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x16,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0xb3,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xb3,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x5a,0x02,0x00,0x00, +0x00,0x03,0x00,0x00,0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xb0,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xb2,0x01,0x00,0x00, +0xe0,0x00,0x04,0x00,0x0c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0xa8,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xd6,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xd6,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x5c,0x02,0x00,0x00,0xe6,0x02,0x00,0x00, +0x6c,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xd3,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xd5,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x61,0x02,0x00,0x00,0x55,0x00,0x00,0x00, +0x53,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x62,0x02,0x00,0x00,0x95,0x00,0x00,0x00,0x61,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x67,0x02,0x00,0x00, +0x59,0x00,0x00,0x00,0xb8,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x68,0x02,0x00,0x00,0xa7,0x00,0x00,0x00, +0x67,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, +0x6d,0x02,0x00,0x00,0x12,0x00,0x00,0x00,0x6c,0x02,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x6e,0x02,0x00,0x00, +0x6d,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x6f,0x02,0x00,0x00,0x0f,0x00,0x00,0x00,0x6e,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x73,0x02,0x00,0x00, +0x47,0x00,0x00,0x00,0x6e,0x02,0x00,0x00,0x41,0x00,0x05,0x00, +0x0d,0x00,0x00,0x00,0x75,0x02,0x00,0x00,0x74,0x02,0x00,0x00, +0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x76,0x02,0x00,0x00,0x75,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x77,0x02,0x00,0x00,0x73,0x02,0x00,0x00, +0x76,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x78,0x02,0x00,0x00,0x6f,0x02,0x00,0x00,0x77,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0x7a,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x7a,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xe7,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0xd5,0x00,0x00,0x00, +0xe0,0x02,0x00,0x00,0x7d,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc1,0x00,0x00,0x00,0x80,0x02,0x00,0x00,0xe7,0x02,0x00,0x00, +0xbe,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x7c,0x02,0x00,0x00, +0x7d,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x80,0x02,0x00,0x00,0x7b,0x02,0x00,0x00,0x7c,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x7b,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0x82,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x82,0x02,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xe8,0x02,0x00,0x00, +0x3e,0x00,0x00,0x00,0x7b,0x02,0x00,0x00,0xde,0x02,0x00,0x00, +0x85,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00, +0x88,0x02,0x00,0x00,0xe8,0x02,0x00,0x00,0x60,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0x84,0x02,0x00,0x00,0x85,0x02,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x88,0x02,0x00,0x00, +0x83,0x02,0x00,0x00,0x84,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x83,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x8c,0x02,0x00,0x00,0xe8,0x02,0x00,0x00,0x61,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8d,0x02,0x00,0x00, +0x62,0x02,0x00,0x00,0x8c,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x8f,0x02,0x00,0x00,0x64,0x00,0x00,0x00, +0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x90,0x02,0x00,0x00,0x8d,0x02,0x00,0x00,0x8f,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x94,0x02,0x00,0x00, +0xe7,0x02,0x00,0x00,0x00,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x95,0x02,0x00,0x00,0x68,0x02,0x00,0x00, +0x94,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x97,0x02,0x00,0x00,0x68,0x00,0x00,0x00,0xbb,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x98,0x02,0x00,0x00, +0x95,0x02,0x00,0x00,0x97,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0x9a,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x9a,0x02,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xea,0x02,0x00,0x00, +0x3e,0x00,0x00,0x00,0x83,0x02,0x00,0x00,0xdc,0x02,0x00,0x00, +0x9d,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00, +0xa0,0x02,0x00,0x00,0xea,0x02,0x00,0x00,0xbb,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0x9c,0x02,0x00,0x00,0x9d,0x02,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xa0,0x02,0x00,0x00, +0x9b,0x02,0x00,0x00,0x9c,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x9b,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0xa2,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0xa2,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xec,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, +0x9b,0x02,0x00,0x00,0xda,0x02,0x00,0x00,0xa5,0x02,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00,0xa8,0x02,0x00,0x00, +0xec,0x02,0x00,0x00,0x62,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0xa4,0x02,0x00,0x00,0xa5,0x02,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0xa8,0x02,0x00,0x00,0xa3,0x02,0x00,0x00, +0xa4,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0xa3,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xab,0x02,0x00,0x00, +0x90,0x02,0x00,0x00,0xec,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc1,0x00,0x00,0x00,0xae,0x02,0x00,0x00,0xab,0x02,0x00,0x00, +0x36,0x00,0x00,0x00,0xf7,0x00,0x03,0x00,0xb0,0x02,0x00,0x00, +0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xae,0x02,0x00,0x00, +0xaf,0x02,0x00,0x00,0xb0,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0xaf,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xb3,0x02,0x00,0x00,0x98,0x02,0x00,0x00,0xea,0x02,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0xb4,0x02,0x00,0x00, +0x12,0x00,0x00,0x00,0xcf,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xb5,0x02,0x00,0x00,0xb4,0x02,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00,0xb6,0x02,0x00,0x00, +0xb3,0x02,0x00,0x00,0xb5,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0xb0,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0xb0,0x02,0x00,0x00, +0xf5,0x00,0x07,0x00,0xc1,0x00,0x00,0x00,0xb7,0x02,0x00,0x00, +0xae,0x02,0x00,0x00,0xa3,0x02,0x00,0x00,0xb6,0x02,0x00,0x00, +0xaf,0x02,0x00,0x00,0xf7,0x00,0x03,0x00,0xb9,0x02,0x00,0x00, +0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xb7,0x02,0x00,0x00, +0xb8,0x02,0x00,0x00,0xb9,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0xb8,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xc1,0x02,0x00,0x00,0x98,0x02,0x00,0x00,0xea,0x02,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0xc3,0x02,0x00,0x00, +0x12,0x00,0x00,0x00,0xc2,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xc4,0x02,0x00,0x00,0xc3,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xc5,0x02,0x00,0x00, +0xc1,0x02,0x00,0x00,0xc4,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xc6,0x02,0x00,0x00,0x78,0x02,0x00,0x00, +0xc5,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xc8,0x02,0x00,0x00,0xc6,0x02,0x00,0x00,0x90,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xca,0x02,0x00,0x00, +0xc8,0x02,0x00,0x00,0xec,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xcc,0x02,0x00,0x00,0xe7,0x02,0x00,0x00, +0xbb,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xce,0x02,0x00,0x00,0xcc,0x02,0x00,0x00,0xea,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xd0,0x02,0x00,0x00, +0xce,0x02,0x00,0x00,0xcf,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xd2,0x02,0x00,0x00,0xe8,0x02,0x00,0x00, +0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xd3,0x02,0x00,0x00,0xd0,0x02,0x00,0x00,0xd2,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xd5,0x02,0x00,0x00, +0xd3,0x02,0x00,0x00,0xec,0x02,0x00,0x00,0x41,0x00,0x05,0x00, +0xcc,0x00,0x00,0x00,0xd6,0x02,0x00,0x00,0xc9,0x00,0x00,0x00, +0xd5,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0xc3,0x00,0x00,0x00, +0xd7,0x02,0x00,0x00,0xd6,0x02,0x00,0x00,0x41,0x00,0x06,0x00, +0x94,0x01,0x00,0x00,0xd8,0x02,0x00,0x00,0xbd,0x02,0x00,0x00, +0x34,0x00,0x00,0x00,0xca,0x02,0x00,0x00,0x3e,0x00,0x03,0x00, +0xd8,0x02,0x00,0x00,0xd7,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0xb9,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0xb9,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0xa5,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0xa5,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xda,0x02,0x00,0x00,0xec,0x02,0x00,0x00,0xcf,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xa2,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0xa4,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x9d,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x9d,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xdc,0x02,0x00,0x00,0xea,0x02,0x00,0x00, +0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x9a,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x9c,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0x85,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x85,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xde,0x02,0x00,0x00, +0xe8,0x02,0x00,0x00,0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x82,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x84,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0x7d,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x7d,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xe0,0x02,0x00,0x00,0xe7,0x02,0x00,0x00,0xcf,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x7a,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x7c,0x02,0x00,0x00,0xfd,0x00,0x01,0x00,0x38,0x00,0x01,0x00, + +}; +const uint64_t matmul_q5_1_f32_len = 10956; + +unsigned char matmul_q5_1_f32_aligned_data[] = { +0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, +0x3a,0x03,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, +0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x09,0x00,0x00,0x00, +0x11,0x00,0x02,0x00,0x51,0x11,0x00,0x00,0x11,0x00,0x02,0x00, +0x60,0x11,0x00,0x00,0x0b,0x00,0x06,0x00,0x01,0x00,0x00,0x00, +0x47,0x4c,0x53,0x4c,0x2e,0x73,0x74,0x64,0x2e,0x34,0x35,0x30, +0x00,0x00,0x00,0x00,0x0e,0x00,0x03,0x00,0x00,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x0f,0x00,0x0f,0x00,0x05,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x6d,0x61,0x69,0x6e,0x00,0x00,0x00,0x00, +0x0b,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x3d,0x00,0x00,0x00, +0x4c,0x00,0x00,0x00,0x08,0x01,0x00,0x00,0x4c,0x01,0x00,0x00, +0x81,0x01,0x00,0x00,0x89,0x01,0x00,0x00,0x97,0x02,0x00,0x00, +0xe0,0x02,0x00,0x00,0x10,0x00,0x06,0x00,0x04,0x00,0x00,0x00, +0x11,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x0b,0x00,0x00,0x00, +0x0b,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x10,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x10,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x10,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x05,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x10,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, +0x07,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x1c,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x08,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x10,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x24,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, +0x0a,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x28,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x2c,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x10,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x30,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, +0x0d,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x34,0x00,0x00,0x00, +0x47,0x00,0x03,0x00,0x10,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x37,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x3d,0x00,0x00,0x00, +0x0b,0x00,0x00,0x00,0x1a,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x4c,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x1b,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x4f,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x09,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x53,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x60,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x62,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x07,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x6c,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0xa7,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0xb9,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x05,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xbc,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x03,0x01,0x00,0x00,0x06,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x04,0x01,0x00,0x00,0x00,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x04,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x04,0x01,0x00,0x00, +0x02,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x04,0x01,0x00,0x00,0x03,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x05,0x01,0x00,0x00,0x06,0x00,0x00,0x00,0x18,0x00,0x00,0x00, +0x48,0x00,0x04,0x00,0x06,0x01,0x00,0x00,0x00,0x00,0x00,0x00, +0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x06,0x01,0x00,0x00, +0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x03,0x00,0x06,0x01,0x00,0x00,0x02,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x08,0x01,0x00,0x00,0x22,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x08,0x01,0x00,0x00, +0x21,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x59,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x5a,0x01,0x00,0x00,0x0b,0x00,0x00,0x00, +0x19,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x86,0x01,0x00,0x00, +0x06,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x48,0x00,0x04,0x00, +0x87,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x05,0x00,0x00,0x00, +0x48,0x00,0x04,0x00,0x87,0x01,0x00,0x00,0x00,0x00,0x00,0x00, +0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x87,0x01,0x00,0x00, +0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x87,0x01,0x00,0x00,0x00,0x00,0x00,0x00, +0x07,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x47,0x00,0x03,0x00, +0x87,0x01,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x89,0x01,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x89,0x01,0x00,0x00,0x21,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x97,0x02,0x00,0x00, +0x0b,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0xdd,0x02,0x00,0x00,0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x48,0x00,0x04,0x00,0xde,0x02,0x00,0x00,0x00,0x00,0x00,0x00, +0x19,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0xde,0x02,0x00,0x00, +0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x03,0x00,0xde,0x02,0x00,0x00,0x02,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0xe0,0x02,0x00,0x00,0x22,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xe0,0x02,0x00,0x00, +0x21,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x13,0x00,0x02,0x00, +0x02,0x00,0x00,0x00,0x21,0x00,0x03,0x00,0x03,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x15,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x17,0x00,0x04,0x00, +0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x03,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x0a,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x09,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00, +0x0b,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x0d,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x1e,0x00,0x10,0x00,0x10,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x11,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x10,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0x11,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0x09,0x00,0x00,0x00,0x15,0x00,0x04,0x00,0x13,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x08,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x15,0x00,0x00,0x00,0x09,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, +0x21,0x00,0x00,0x00,0x0a,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0x27,0x00,0x00,0x00,0x09,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x2d,0x00,0x00,0x00, +0x07,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, +0x34,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x32,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x40,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00, +0x3d,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x3e,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00,0x4c,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x4f,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x32,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x54,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x53,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x58,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x53,0x00,0x00,0x00, +0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x60,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x61,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x53,0x00,0x00,0x00, +0x60,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x62,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x63,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0x61,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x67,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0x61,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x32,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x10,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x72,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x77,0x00,0x00,0x00, +0x08,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x78,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, +0x77,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x7d,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, +0x77,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, +0x81,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x91,0x00,0x00,0x00, +0x0b,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, +0x97,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0xa2,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xa7,0x00,0x00,0x00, +0x40,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, +0xa9,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xb8,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x32,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xb9,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xba,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0xb9,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xbb,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x62,0x00,0x00,0x00, +0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xbc,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xbd,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0xbb,0x00,0x00,0x00, +0xbc,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xbe,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0xbd,0x00,0x00,0x00, +0x60,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xbf,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0xba,0x00,0x00,0x00, +0xbe,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xc0,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0xb8,0x00,0x00,0x00, +0xbf,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xc1,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0xc0,0x00,0x00,0x00, +0xbc,0x00,0x00,0x00,0x14,0x00,0x02,0x00,0xc2,0x00,0x00,0x00, +0x16,0x00,0x03,0x00,0xc4,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xc5,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xc6,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0xc5,0x00,0x00,0x00,0xbf,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xc7,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0xc6,0x00,0x00,0x00,0xbc,0x00,0x00,0x00, +0x1c,0x00,0x04,0x00,0xc8,0x00,0x00,0x00,0xc4,0x00,0x00,0x00, +0xc7,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0xc9,0x00,0x00,0x00, +0x07,0x00,0x00,0x00,0xc8,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0xc4,0x00,0x00,0x00,0xcc,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0xcd,0x00,0x00,0x00,0x07,0x00,0x00,0x00, +0xc4,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, +0xd0,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xf4,0x00,0x00,0x00,0x80,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xfa,0x00,0x00,0x00,0x10,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xfe,0x00,0x00,0x00, +0x0f,0x00,0x00,0x00,0x16,0x00,0x03,0x00,0x01,0x01,0x00,0x00, +0x10,0x00,0x00,0x00,0x15,0x00,0x04,0x00,0x02,0x01,0x00,0x00, +0x08,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x1c,0x00,0x04,0x00, +0x03,0x01,0x00,0x00,0x02,0x01,0x00,0x00,0xfa,0x00,0x00,0x00, +0x1e,0x00,0x06,0x00,0x04,0x01,0x00,0x00,0x01,0x01,0x00,0x00, +0x01,0x01,0x00,0x00,0x06,0x00,0x00,0x00,0x03,0x01,0x00,0x00, +0x1d,0x00,0x03,0x00,0x05,0x01,0x00,0x00,0x04,0x01,0x00,0x00, +0x1e,0x00,0x03,0x00,0x06,0x01,0x00,0x00,0x05,0x01,0x00,0x00, +0x20,0x00,0x04,0x00,0x07,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, +0x06,0x01,0x00,0x00,0x3b,0x00,0x04,0x00,0x07,0x01,0x00,0x00, +0x08,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x0a,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x01,0x01,0x00,0x00, +0x20,0x00,0x04,0x00,0x15,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x23,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x2c,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x02,0x01,0x00,0x00, +0x17,0x00,0x04,0x00,0x30,0x01,0x00,0x00,0xc4,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x48,0x01,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, +0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x49,0x01,0x00,0x00,0x84,0x00,0x00,0x00,0x37,0x00,0x00,0x00, +0x48,0x01,0x00,0x00,0x1c,0x00,0x04,0x00,0x4a,0x01,0x00,0x00, +0x01,0x01,0x00,0x00,0x49,0x01,0x00,0x00,0x20,0x00,0x04,0x00, +0x4b,0x01,0x00,0x00,0x04,0x00,0x00,0x00,0x4a,0x01,0x00,0x00, +0x3b,0x00,0x04,0x00,0x4b,0x01,0x00,0x00,0x4c,0x01,0x00,0x00, +0x04,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x51,0x01,0x00,0x00, +0x04,0x00,0x00,0x00,0x01,0x01,0x00,0x00,0x32,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x59,0x01,0x00,0x00,0x01,0x00,0x00,0x00, +0x33,0x00,0x06,0x00,0x09,0x00,0x00,0x00,0x5a,0x01,0x00,0x00, +0x59,0x01,0x00,0x00,0x39,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x5b,0x01,0x00,0x00, +0x51,0x00,0x00,0x00,0x5a,0x01,0x00,0x00,0x00,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x5c,0x01,0x00,0x00, +0x84,0x00,0x00,0x00,0x5b,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x5d,0x01,0x00,0x00, +0x86,0x00,0x00,0x00,0x5c,0x01,0x00,0x00,0x6c,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x78,0x01,0x00,0x00, +0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x7d,0x01,0x00,0x00, +0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x7e,0x01,0x00,0x00, +0x84,0x00,0x00,0x00,0xa7,0x00,0x00,0x00,0x7d,0x01,0x00,0x00, +0x1c,0x00,0x04,0x00,0x7f,0x01,0x00,0x00,0x01,0x01,0x00,0x00, +0x7e,0x01,0x00,0x00,0x20,0x00,0x04,0x00,0x80,0x01,0x00,0x00, +0x04,0x00,0x00,0x00,0x7f,0x01,0x00,0x00,0x3b,0x00,0x04,0x00, +0x80,0x01,0x00,0x00,0x81,0x01,0x00,0x00,0x04,0x00,0x00,0x00, +0x17,0x00,0x04,0x00,0x84,0x01,0x00,0x00,0xc4,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x18,0x00,0x04,0x00,0x85,0x01,0x00,0x00, +0x84,0x01,0x00,0x00,0x02,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, +0x86,0x01,0x00,0x00,0x85,0x01,0x00,0x00,0x1e,0x00,0x03,0x00, +0x87,0x01,0x00,0x00,0x86,0x01,0x00,0x00,0x20,0x00,0x04,0x00, +0x88,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x87,0x01,0x00,0x00, +0x3b,0x00,0x04,0x00,0x88,0x01,0x00,0x00,0x89,0x01,0x00,0x00, +0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x8b,0x01,0x00,0x00, +0x0c,0x00,0x00,0x00,0xc4,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x9f,0x01,0x00,0x00,0x03,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xa7,0x01,0x00,0x00, +0x04,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xaf,0x01,0x00,0x00,0x05,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xb7,0x01,0x00,0x00,0x06,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xbf,0x01,0x00,0x00, +0x07,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xc6,0x01,0x00,0x00,0x51,0x00,0x00,0x00,0x5a,0x01,0x00,0x00, +0x00,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xc7,0x01,0x00,0x00,0x84,0x00,0x00,0x00,0xc6,0x01,0x00,0x00, +0x77,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xc8,0x01,0x00,0x00,0x86,0x00,0x00,0x00,0xc7,0x01,0x00,0x00, +0x6c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xcb,0x01,0x00,0x00,0x08,0x01,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xcc,0x01,0x00,0x00,0x86,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xcf,0x01,0x00,0x00,0x86,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x77,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xea,0x01,0x00,0x00,0x84,0x00,0x00,0x00, +0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x1c,0x00,0x04,0x00, +0xeb,0x01,0x00,0x00,0x01,0x01,0x00,0x00,0xea,0x01,0x00,0x00, +0x20,0x00,0x04,0x00,0xec,0x01,0x00,0x00,0x07,0x00,0x00,0x00, +0xeb,0x01,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xfc,0x01,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, +0x39,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x02,0x02,0x00,0x00, +0x07,0x00,0x00,0x00,0x01,0x01,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x18,0x02,0x00,0x00,0x84,0x00,0x00,0x00, +0xbf,0x00,0x00,0x00,0xbc,0x00,0x00,0x00,0x1c,0x00,0x04,0x00, +0x19,0x02,0x00,0x00,0x01,0x01,0x00,0x00,0x18,0x02,0x00,0x00, +0x20,0x00,0x04,0x00,0x1a,0x02,0x00,0x00,0x07,0x00,0x00,0x00, +0x19,0x02,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x23,0x02,0x00,0x00,0x86,0x00,0x00,0x00,0xb9,0x00,0x00,0x00, +0xbf,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x2b,0x02,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, +0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x5a,0x02,0x00,0x00,0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00, +0x62,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, +0x8f,0x02,0x00,0x00,0x0d,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0x0a,0x00,0x00,0x00,0x97,0x02,0x00,0x00,0x01,0x00,0x00,0x00, +0x1d,0x00,0x03,0x00,0xdd,0x02,0x00,0x00,0xc4,0x00,0x00,0x00, +0x1e,0x00,0x03,0x00,0xde,0x02,0x00,0x00,0xdd,0x02,0x00,0x00, +0x20,0x00,0x04,0x00,0xdf,0x02,0x00,0x00,0x0c,0x00,0x00,0x00, +0xde,0x02,0x00,0x00,0x3b,0x00,0x04,0x00,0xdf,0x02,0x00,0x00, +0xe0,0x02,0x00,0x00,0x0c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0xe5,0x02,0x00,0x00,0x05,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xf2,0x02,0x00,0x00, +0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00, +0x36,0x00,0x05,0x00,0x02,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0x05,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0xc9,0x00,0x00,0x00, +0xca,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0xec,0x01,0x00,0x00,0xed,0x01,0x00,0x00,0x07,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0x1a,0x02,0x00,0x00,0x1b,0x02,0x00,0x00, +0x07,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00, +0x0e,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, +0x0e,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, +0x16,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x14,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x17,0x00,0x00,0x00, +0x16,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x18,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x17,0x00,0x00,0x00, +0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x1e,0x00,0x00,0x00, +0x0f,0x00,0x00,0x00,0x17,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x15,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0x21,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x24,0x00,0x00,0x00,0x18,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, +0x28,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x27,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x29,0x00,0x00,0x00, +0x28,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x2a,0x00,0x00,0x00,0x1e,0x00,0x00,0x00,0x29,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, +0x12,0x00,0x00,0x00,0x2d,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x2f,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x30,0x00,0x00,0x00, +0x24,0x00,0x00,0x00,0x2f,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x30,0x00,0x00,0x00, +0x2a,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, +0x35,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x34,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x36,0x00,0x00,0x00, +0x35,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x38,0x00,0x00,0x00,0x36,0x00,0x00,0x00,0x37,0x00,0x00,0x00, +0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, +0x38,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x3b,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, +0x37,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00, +0x3f,0x00,0x00,0x00,0x3d,0x00,0x00,0x00,0x3e,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x40,0x00,0x00,0x00, +0x3f,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x42,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x3b,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x47,0x00,0x00,0x00, +0x40,0x00,0x00,0x00,0x3b,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x0d,0x00,0x00,0x00,0x49,0x00,0x00,0x00,0x3d,0x00,0x00,0x00, +0x39,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x4a,0x00,0x00,0x00,0x49,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x0d,0x00,0x00,0x00,0x4d,0x00,0x00,0x00,0x4c,0x00,0x00,0x00, +0x3e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x4e,0x00,0x00,0x00,0x4d,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x50,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, +0x4f,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x55,0x00,0x00,0x00,0x50,0x00,0x00,0x00,0x54,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x59,0x00,0x00,0x00, +0x50,0x00,0x00,0x00,0x58,0x00,0x00,0x00,0x89,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x5d,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, +0x4f,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x64,0x00,0x00,0x00,0x5d,0x00,0x00,0x00,0x63,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x68,0x00,0x00,0x00, +0x5d,0x00,0x00,0x00,0x67,0x00,0x00,0x00,0x89,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x6e,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, +0x6d,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x73,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x72,0x00,0x00,0x00, +0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x79,0x00,0x00,0x00, +0x4e,0x00,0x00,0x00,0x78,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x7e,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, +0x7d,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, +0x82,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x81,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x83,0x00,0x00,0x00, +0x82,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0x47,0x00,0x00,0x00,0x83,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x87,0x00,0x00,0x00, +0x12,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x88,0x00,0x00,0x00,0x87,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8a,0x00,0x00,0x00, +0x47,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x8d,0x00,0x00,0x00,0x8a,0x00,0x00,0x00, +0x83,0x00,0x00,0x00,0x0c,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0x8e,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x26,0x00,0x00,0x00, +0x88,0x00,0x00,0x00,0x8d,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x15,0x00,0x00,0x00,0x92,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0x91,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x93,0x00,0x00,0x00,0x92,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x94,0x00,0x00,0x00,0x32,0x00,0x00,0x00, +0x93,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x96,0x00,0x00,0x00,0x42,0x00,0x00,0x00,0x37,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x98,0x00,0x00,0x00, +0x12,0x00,0x00,0x00,0x97,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x99,0x00,0x00,0x00,0x98,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9a,0x00,0x00,0x00, +0x96,0x00,0x00,0x00,0x99,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x9b,0x00,0x00,0x00,0x94,0x00,0x00,0x00, +0x9a,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x9d,0x00,0x00,0x00,0x9b,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9e,0x00,0x00,0x00, +0x9d,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x15,0x00,0x00,0x00,0xa3,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0xa2,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xa4,0x00,0x00,0x00,0xa3,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xa5,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, +0xa4,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xa8,0x00,0x00,0x00,0x4a,0x00,0x00,0x00,0xa7,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0xaa,0x00,0x00,0x00, +0x12,0x00,0x00,0x00,0xa9,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xab,0x00,0x00,0x00,0xaa,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xac,0x00,0x00,0x00, +0xa8,0x00,0x00,0x00,0xab,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xad,0x00,0x00,0x00,0xa5,0x00,0x00,0x00, +0xac,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xaf,0x00,0x00,0x00,0xad,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xb0,0x00,0x00,0x00, +0xaf,0x00,0x00,0x00,0x77,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xb2,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xb2,0x00,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x08,0x03,0x00,0x00, +0x3e,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0xd1,0x00,0x00,0x00, +0xb3,0x00,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, +0xc3,0x00,0x00,0x00,0x08,0x03,0x00,0x00,0xc1,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0xb4,0x00,0x00,0x00,0xb3,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xc3,0x00,0x00,0x00, +0xb3,0x00,0x00,0x00,0xb4,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xb3,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0xcd,0x00,0x00,0x00, +0xce,0x00,0x00,0x00,0xca,0x00,0x00,0x00,0x08,0x03,0x00,0x00, +0x3e,0x00,0x03,0x00,0xce,0x00,0x00,0x00,0xcc,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xd1,0x00,0x00,0x00, +0x08,0x03,0x00,0x00,0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xb2,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xb4,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xd4,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xd4,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0x21,0x03,0x00,0x00,0xb0,0x00,0x00,0x00,0xb4,0x00,0x00,0x00, +0xd1,0x01,0x00,0x00,0xd7,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0x1d,0x03,0x00,0x00,0x9e,0x00,0x00,0x00, +0xb4,0x00,0x00,0x00,0xce,0x01,0x00,0x00,0xd7,0x00,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x09,0x03,0x00,0x00, +0x84,0x00,0x00,0x00,0xb4,0x00,0x00,0x00,0x7f,0x02,0x00,0x00, +0xd7,0x00,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, +0xdb,0x00,0x00,0x00,0x09,0x03,0x00,0x00,0x8e,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0xd6,0x00,0x00,0x00,0xd7,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xdb,0x00,0x00,0x00, +0xd5,0x00,0x00,0x00,0xd6,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xd5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xdd,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xdd,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0x19,0x03,0x00,0x00,0x3e,0x00,0x00,0x00, +0xd5,0x00,0x00,0x00,0x5f,0x01,0x00,0x00,0xde,0x00,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0xe3,0x00,0x00,0x00, +0x19,0x03,0x00,0x00,0x37,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0xdf,0x00,0x00,0x00,0xde,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0xe3,0x00,0x00,0x00,0xde,0x00,0x00,0x00, +0xdf,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xde,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xe8,0x00,0x00,0x00, +0x73,0x00,0x00,0x00,0x19,0x03,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xeb,0x00,0x00,0x00,0xe8,0x00,0x00,0x00, +0x99,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xec,0x00,0x00,0x00,0xeb,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xed,0x00,0x00,0x00, +0x1d,0x03,0x00,0x00,0xec,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xef,0x00,0x00,0x00,0xed,0x00,0x00,0x00, +0x6e,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xf5,0x00,0x00,0x00,0xe8,0x00,0x00,0x00,0xf4,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xf7,0x00,0x00,0x00, +0xf5,0x00,0x00,0x00,0x6e,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xfb,0x00,0x00,0x00,0xef,0x00,0x00,0x00, +0xfa,0x00,0x00,0x00,0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xff,0x00,0x00,0x00,0xef,0x00,0x00,0x00,0xfe,0x00,0x00,0x00, +0x41,0x00,0x07,0x00,0x0a,0x01,0x00,0x00,0x0b,0x01,0x00,0x00, +0x08,0x01,0x00,0x00,0x34,0x00,0x00,0x00,0xfb,0x00,0x00,0x00, +0x34,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x01,0x01,0x00,0x00, +0x0c,0x01,0x00,0x00,0x0b,0x01,0x00,0x00,0x73,0x00,0x04,0x00, +0xc4,0x00,0x00,0x00,0x0d,0x01,0x00,0x00,0x0c,0x01,0x00,0x00, +0x41,0x00,0x07,0x00,0x0a,0x01,0x00,0x00,0x10,0x01,0x00,0x00, +0x08,0x01,0x00,0x00,0x34,0x00,0x00,0x00,0xfb,0x00,0x00,0x00, +0xd0,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x01,0x01,0x00,0x00, +0x11,0x01,0x00,0x00,0x10,0x01,0x00,0x00,0x73,0x00,0x04,0x00, +0xc4,0x00,0x00,0x00,0x12,0x01,0x00,0x00,0x11,0x01,0x00,0x00, +0x41,0x00,0x07,0x00,0x15,0x01,0x00,0x00,0x16,0x01,0x00,0x00, +0x08,0x01,0x00,0x00,0x34,0x00,0x00,0x00,0xfb,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x17,0x01,0x00,0x00,0x16,0x01,0x00,0x00,0xc2,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x1d,0x01,0x00,0x00,0x17,0x01,0x00,0x00, +0xff,0x00,0x00,0x00,0xc4,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x1e,0x01,0x00,0x00,0x1d,0x01,0x00,0x00,0xa9,0x00,0x00,0x00, +0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x1f,0x01,0x00,0x00, +0x1e,0x01,0x00,0x00,0xfa,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0x20,0x01,0x00,0x00,0x1f,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x24,0x01,0x00,0x00, +0xff,0x00,0x00,0x00,0x23,0x01,0x00,0x00,0xc2,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x25,0x01,0x00,0x00,0x17,0x01,0x00,0x00, +0x24,0x01,0x00,0x00,0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x26,0x01,0x00,0x00,0x25,0x01,0x00,0x00,0xfa,0x00,0x00,0x00, +0x7c,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x27,0x01,0x00,0x00, +0x26,0x01,0x00,0x00,0x41,0x00,0x08,0x00,0x2c,0x01,0x00,0x00, +0x2d,0x01,0x00,0x00,0x08,0x01,0x00,0x00,0x34,0x00,0x00,0x00, +0xfb,0x00,0x00,0x00,0x97,0x00,0x00,0x00,0xff,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x02,0x01,0x00,0x00,0x2e,0x01,0x00,0x00, +0x2d,0x01,0x00,0x00,0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x2f,0x01,0x00,0x00,0x2e,0x01,0x00,0x00,0xc7,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x34,0x01,0x00,0x00,0x2f,0x01,0x00,0x00, +0xfe,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x38,0x01,0x00,0x00,0x20,0x01,0x00,0x00,0xc5,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x39,0x01,0x00,0x00,0x34,0x01,0x00,0x00, +0x38,0x01,0x00,0x00,0x70,0x00,0x04,0x00,0xc4,0x00,0x00,0x00, +0x3a,0x01,0x00,0x00,0x39,0x01,0x00,0x00,0xc2,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x3c,0x01,0x00,0x00,0x2f,0x01,0x00,0x00, +0xa9,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x3f,0x01,0x00,0x00,0x27,0x01,0x00,0x00,0xc5,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x40,0x01,0x00,0x00,0x3c,0x01,0x00,0x00, +0x3f,0x01,0x00,0x00,0x70,0x00,0x04,0x00,0xc4,0x00,0x00,0x00, +0x41,0x01,0x00,0x00,0x40,0x01,0x00,0x00,0x50,0x00,0x05,0x00, +0x30,0x01,0x00,0x00,0x42,0x01,0x00,0x00,0x3a,0x01,0x00,0x00, +0x41,0x01,0x00,0x00,0x8e,0x00,0x05,0x00,0x30,0x01,0x00,0x00, +0x44,0x01,0x00,0x00,0x42,0x01,0x00,0x00,0x0d,0x01,0x00,0x00, +0x50,0x00,0x05,0x00,0x30,0x01,0x00,0x00,0x46,0x01,0x00,0x00, +0x12,0x01,0x00,0x00,0x12,0x01,0x00,0x00,0x81,0x00,0x05,0x00, +0x30,0x01,0x00,0x00,0x47,0x01,0x00,0x00,0x44,0x01,0x00,0x00, +0x46,0x01,0x00,0x00,0x51,0x00,0x05,0x00,0xc4,0x00,0x00,0x00, +0x4f,0x01,0x00,0x00,0x47,0x01,0x00,0x00,0x00,0x00,0x00,0x00, +0x73,0x00,0x04,0x00,0x01,0x01,0x00,0x00,0x50,0x01,0x00,0x00, +0x4f,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0x51,0x01,0x00,0x00, +0x52,0x01,0x00,0x00,0x4c,0x01,0x00,0x00,0xf7,0x00,0x00,0x00, +0x3e,0x00,0x03,0x00,0x52,0x01,0x00,0x00,0x50,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x54,0x01,0x00,0x00, +0xf7,0x00,0x00,0x00,0xfa,0x00,0x00,0x00,0x51,0x00,0x05,0x00, +0xc4,0x00,0x00,0x00,0x56,0x01,0x00,0x00,0x47,0x01,0x00,0x00, +0x01,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x01,0x01,0x00,0x00, +0x57,0x01,0x00,0x00,0x56,0x01,0x00,0x00,0x41,0x00,0x05,0x00, +0x51,0x01,0x00,0x00,0x58,0x01,0x00,0x00,0x4c,0x01,0x00,0x00, +0x54,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x58,0x01,0x00,0x00, +0x57,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x5f,0x01,0x00,0x00,0x19,0x03,0x00,0x00,0x5d,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0xdd,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xdf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x61,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x61,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0x1a,0x03,0x00,0x00,0x3e,0x00,0x00,0x00, +0xdf,0x00,0x00,0x00,0xca,0x01,0x00,0x00,0x62,0x01,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0x67,0x01,0x00,0x00, +0x1a,0x03,0x00,0x00,0xa7,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0x63,0x01,0x00,0x00,0x62,0x01,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x67,0x01,0x00,0x00,0x62,0x01,0x00,0x00, +0x63,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x62,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x6c,0x01,0x00,0x00, +0x7e,0x00,0x00,0x00,0x1a,0x03,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x6f,0x01,0x00,0x00,0x6c,0x01,0x00,0x00, +0xab,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x70,0x01,0x00,0x00,0x6f,0x01,0x00,0x00,0x77,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x71,0x01,0x00,0x00, +0x21,0x03,0x00,0x00,0x70,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x73,0x01,0x00,0x00,0x71,0x01,0x00,0x00, +0x79,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x79,0x01,0x00,0x00,0x6c,0x01,0x00,0x00,0x78,0x01,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x7b,0x01,0x00,0x00, +0x79,0x00,0x00,0x00,0x77,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x7c,0x01,0x00,0x00,0x79,0x01,0x00,0x00, +0x7b,0x01,0x00,0x00,0x41,0x00,0x08,0x00,0x8b,0x01,0x00,0x00, +0x8c,0x01,0x00,0x00,0x89,0x01,0x00,0x00,0x34,0x00,0x00,0x00, +0x73,0x01,0x00,0x00,0x34,0x00,0x00,0x00,0x3e,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00,0x8d,0x01,0x00,0x00, +0x8c,0x01,0x00,0x00,0x73,0x00,0x04,0x00,0x01,0x01,0x00,0x00, +0x8e,0x01,0x00,0x00,0x8d,0x01,0x00,0x00,0x41,0x00,0x05,0x00, +0x51,0x01,0x00,0x00,0x8f,0x01,0x00,0x00,0x81,0x01,0x00,0x00, +0x7c,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x8f,0x01,0x00,0x00, +0x8e,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x91,0x01,0x00,0x00,0x7c,0x01,0x00,0x00,0x39,0x00,0x00,0x00, +0x41,0x00,0x08,0x00,0x8b,0x01,0x00,0x00,0x93,0x01,0x00,0x00, +0x89,0x01,0x00,0x00,0x34,0x00,0x00,0x00,0x73,0x01,0x00,0x00, +0x34,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0xc4,0x00,0x00,0x00,0x94,0x01,0x00,0x00,0x93,0x01,0x00,0x00, +0x73,0x00,0x04,0x00,0x01,0x01,0x00,0x00,0x95,0x01,0x00,0x00, +0x94,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0x51,0x01,0x00,0x00, +0x96,0x01,0x00,0x00,0x81,0x01,0x00,0x00,0x91,0x01,0x00,0x00, +0x3e,0x00,0x03,0x00,0x96,0x01,0x00,0x00,0x95,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x98,0x01,0x00,0x00, +0x7c,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x41,0x00,0x08,0x00, +0x8b,0x01,0x00,0x00,0x9a,0x01,0x00,0x00,0x89,0x01,0x00,0x00, +0x34,0x00,0x00,0x00,0x73,0x01,0x00,0x00,0x34,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00, +0x9b,0x01,0x00,0x00,0x9a,0x01,0x00,0x00,0x73,0x00,0x04,0x00, +0x01,0x01,0x00,0x00,0x9c,0x01,0x00,0x00,0x9b,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0x51,0x01,0x00,0x00,0x9d,0x01,0x00,0x00, +0x81,0x01,0x00,0x00,0x98,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, +0x9d,0x01,0x00,0x00,0x9c,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xa0,0x01,0x00,0x00,0x7c,0x01,0x00,0x00, +0x9f,0x01,0x00,0x00,0x41,0x00,0x08,0x00,0x8b,0x01,0x00,0x00, +0xa2,0x01,0x00,0x00,0x89,0x01,0x00,0x00,0x34,0x00,0x00,0x00, +0x73,0x01,0x00,0x00,0x34,0x00,0x00,0x00,0x9f,0x01,0x00,0x00, +0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00,0xa3,0x01,0x00,0x00, +0xa2,0x01,0x00,0x00,0x73,0x00,0x04,0x00,0x01,0x01,0x00,0x00, +0xa4,0x01,0x00,0x00,0xa3,0x01,0x00,0x00,0x41,0x00,0x05,0x00, +0x51,0x01,0x00,0x00,0xa5,0x01,0x00,0x00,0x81,0x01,0x00,0x00, +0xa0,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0xa5,0x01,0x00,0x00, +0xa4,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xa8,0x01,0x00,0x00,0x7c,0x01,0x00,0x00,0xa7,0x01,0x00,0x00, +0x41,0x00,0x08,0x00,0x8b,0x01,0x00,0x00,0xaa,0x01,0x00,0x00, +0x89,0x01,0x00,0x00,0x34,0x00,0x00,0x00,0x73,0x01,0x00,0x00, +0xd0,0x00,0x00,0x00,0x3e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0xc4,0x00,0x00,0x00,0xab,0x01,0x00,0x00,0xaa,0x01,0x00,0x00, +0x73,0x00,0x04,0x00,0x01,0x01,0x00,0x00,0xac,0x01,0x00,0x00, +0xab,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0x51,0x01,0x00,0x00, +0xad,0x01,0x00,0x00,0x81,0x01,0x00,0x00,0xa8,0x01,0x00,0x00, +0x3e,0x00,0x03,0x00,0xad,0x01,0x00,0x00,0xac,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xb0,0x01,0x00,0x00, +0x7c,0x01,0x00,0x00,0xaf,0x01,0x00,0x00,0x41,0x00,0x08,0x00, +0x8b,0x01,0x00,0x00,0xb2,0x01,0x00,0x00,0x89,0x01,0x00,0x00, +0x34,0x00,0x00,0x00,0x73,0x01,0x00,0x00,0xd0,0x00,0x00,0x00, +0x39,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00, +0xb3,0x01,0x00,0x00,0xb2,0x01,0x00,0x00,0x73,0x00,0x04,0x00, +0x01,0x01,0x00,0x00,0xb4,0x01,0x00,0x00,0xb3,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0x51,0x01,0x00,0x00,0xb5,0x01,0x00,0x00, +0x81,0x01,0x00,0x00,0xb0,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, +0xb5,0x01,0x00,0x00,0xb4,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xb8,0x01,0x00,0x00,0x7c,0x01,0x00,0x00, +0xb7,0x01,0x00,0x00,0x41,0x00,0x08,0x00,0x8b,0x01,0x00,0x00, +0xba,0x01,0x00,0x00,0x89,0x01,0x00,0x00,0x34,0x00,0x00,0x00, +0x73,0x01,0x00,0x00,0xd0,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00,0xbb,0x01,0x00,0x00, +0xba,0x01,0x00,0x00,0x73,0x00,0x04,0x00,0x01,0x01,0x00,0x00, +0xbc,0x01,0x00,0x00,0xbb,0x01,0x00,0x00,0x41,0x00,0x05,0x00, +0x51,0x01,0x00,0x00,0xbd,0x01,0x00,0x00,0x81,0x01,0x00,0x00, +0xb8,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0xbd,0x01,0x00,0x00, +0xbc,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xc0,0x01,0x00,0x00,0x7c,0x01,0x00,0x00,0xbf,0x01,0x00,0x00, +0x41,0x00,0x08,0x00,0x8b,0x01,0x00,0x00,0xc2,0x01,0x00,0x00, +0x89,0x01,0x00,0x00,0x34,0x00,0x00,0x00,0x73,0x01,0x00,0x00, +0xd0,0x00,0x00,0x00,0x9f,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, +0xc4,0x00,0x00,0x00,0xc3,0x01,0x00,0x00,0xc2,0x01,0x00,0x00, +0x73,0x00,0x04,0x00,0x01,0x01,0x00,0x00,0xc4,0x01,0x00,0x00, +0xc3,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0x51,0x01,0x00,0x00, +0xc5,0x01,0x00,0x00,0x81,0x01,0x00,0x00,0xc0,0x01,0x00,0x00, +0x3e,0x00,0x03,0x00,0xc5,0x01,0x00,0x00,0xc4,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xca,0x01,0x00,0x00, +0x1a,0x03,0x00,0x00,0xc8,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0x61,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x63,0x01,0x00,0x00, +0xe0,0x00,0x04,0x00,0x0c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0xcb,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xce,0x01,0x00,0x00,0x1d,0x03,0x00,0x00,0xcc,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xd1,0x01,0x00,0x00, +0x21,0x03,0x00,0x00,0xcf,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0xd3,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xd3,0x01,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x23,0x03,0x00,0x00, +0x3e,0x00,0x00,0x00,0x63,0x01,0x00,0x00,0x7d,0x02,0x00,0x00, +0xd6,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, +0xd9,0x01,0x00,0x00,0x23,0x03,0x00,0x00,0x6c,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0xd5,0x01,0x00,0x00,0xd6,0x01,0x00,0x00, +0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xd9,0x01,0x00,0x00, +0xd4,0x01,0x00,0x00,0xd5,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xd4,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xdb,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xdb,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0x27,0x03,0x00,0x00,0x3e,0x00,0x00,0x00, +0xd4,0x01,0x00,0x00,0x07,0x02,0x00,0x00,0xde,0x01,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0xe1,0x01,0x00,0x00, +0x27,0x03,0x00,0x00,0x60,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0xdd,0x01,0x00,0x00,0xde,0x01,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0xe1,0x01,0x00,0x00,0xdc,0x01,0x00,0x00, +0xdd,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xdc,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0xe3,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xe3,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0x39,0x03,0x00,0x00,0x3e,0x00,0x00,0x00,0xdc,0x01,0x00,0x00, +0x05,0x02,0x00,0x00,0xe4,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc2,0x00,0x00,0x00,0xe9,0x01,0x00,0x00,0x39,0x03,0x00,0x00, +0x62,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xe5,0x01,0x00,0x00, +0xe4,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0xe9,0x01,0x00,0x00,0xe4,0x01,0x00,0x00,0xe5,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xe4,0x01,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xef,0x01,0x00,0x00,0x27,0x03,0x00,0x00, +0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xf1,0x01,0x00,0x00,0xef,0x01,0x00,0x00,0x39,0x03,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xf3,0x01,0x00,0x00, +0x55,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xf5,0x01,0x00,0x00,0x27,0x03,0x00,0x00, +0x61,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xf6,0x01,0x00,0x00,0xf3,0x01,0x00,0x00,0xf5,0x01,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xf8,0x01,0x00,0x00, +0x64,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xf9,0x01,0x00,0x00,0xf6,0x01,0x00,0x00, +0xf8,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xfb,0x01,0x00,0x00,0xf9,0x01,0x00,0x00,0x39,0x03,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xfd,0x01,0x00,0x00, +0xfb,0x01,0x00,0x00,0xfc,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xff,0x01,0x00,0x00,0xfd,0x01,0x00,0x00, +0x23,0x03,0x00,0x00,0x41,0x00,0x05,0x00,0x51,0x01,0x00,0x00, +0x00,0x02,0x00,0x00,0x4c,0x01,0x00,0x00,0xff,0x01,0x00,0x00, +0x3d,0x00,0x04,0x00,0x01,0x01,0x00,0x00,0x01,0x02,0x00,0x00, +0x00,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x02,0x02,0x00,0x00, +0x03,0x02,0x00,0x00,0xed,0x01,0x00,0x00,0xf1,0x01,0x00,0x00, +0x3e,0x00,0x03,0x00,0x03,0x02,0x00,0x00,0x01,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x05,0x02,0x00,0x00, +0x39,0x03,0x00,0x00,0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xe3,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xe5,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0xde,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xde,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x07,0x02,0x00,0x00,0x27,0x03,0x00,0x00,0xd0,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xdb,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xdd,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x09,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x09,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0x28,0x03,0x00,0x00,0x3e,0x00,0x00,0x00, +0xdd,0x01,0x00,0x00,0x35,0x02,0x00,0x00,0x0c,0x02,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0x0f,0x02,0x00,0x00, +0x28,0x03,0x00,0x00,0xbf,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0x0b,0x02,0x00,0x00,0x0c,0x02,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x0f,0x02,0x00,0x00,0x0a,0x02,0x00,0x00, +0x0b,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x0a,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0x11,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x11,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0x36,0x03,0x00,0x00,0x3e,0x00,0x00,0x00,0x0a,0x02,0x00,0x00, +0x33,0x02,0x00,0x00,0x12,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc2,0x00,0x00,0x00,0x17,0x02,0x00,0x00,0x36,0x03,0x00,0x00, +0xbc,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x13,0x02,0x00,0x00, +0x12,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x17,0x02,0x00,0x00,0x12,0x02,0x00,0x00,0x13,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x12,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x1d,0x02,0x00,0x00,0x28,0x03,0x00,0x00, +0xbc,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x1f,0x02,0x00,0x00,0x1d,0x02,0x00,0x00,0x36,0x03,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x21,0x02,0x00,0x00, +0x59,0x00,0x00,0x00,0xb9,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x24,0x02,0x00,0x00,0x28,0x03,0x00,0x00, +0x23,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x25,0x02,0x00,0x00,0x21,0x02,0x00,0x00,0x24,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x27,0x02,0x00,0x00, +0x68,0x00,0x00,0x00,0xbc,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x28,0x02,0x00,0x00,0x25,0x02,0x00,0x00, +0x27,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x2a,0x02,0x00,0x00,0x28,0x02,0x00,0x00,0x36,0x03,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x2c,0x02,0x00,0x00, +0x2a,0x02,0x00,0x00,0x2b,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x2e,0x02,0x00,0x00,0x2c,0x02,0x00,0x00, +0x23,0x03,0x00,0x00,0x41,0x00,0x05,0x00,0x51,0x01,0x00,0x00, +0x2f,0x02,0x00,0x00,0x81,0x01,0x00,0x00,0x2e,0x02,0x00,0x00, +0x3d,0x00,0x04,0x00,0x01,0x01,0x00,0x00,0x30,0x02,0x00,0x00, +0x2f,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x02,0x02,0x00,0x00, +0x31,0x02,0x00,0x00,0x1b,0x02,0x00,0x00,0x1f,0x02,0x00,0x00, +0x3e,0x00,0x03,0x00,0x31,0x02,0x00,0x00,0x30,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x33,0x02,0x00,0x00, +0x36,0x03,0x00,0x00,0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x11,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x13,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0x0c,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x0c,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x35,0x02,0x00,0x00,0x28,0x03,0x00,0x00,0xd0,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x09,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x0b,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x37,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x37,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0x29,0x03,0x00,0x00,0x3e,0x00,0x00,0x00, +0x0b,0x02,0x00,0x00,0x7b,0x02,0x00,0x00,0x3a,0x02,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0x3d,0x02,0x00,0x00, +0x29,0x03,0x00,0x00,0xbf,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0x39,0x02,0x00,0x00,0x3a,0x02,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x3d,0x02,0x00,0x00,0x38,0x02,0x00,0x00, +0x39,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x38,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0x3f,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x3f,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0x2d,0x03,0x00,0x00,0x3e,0x00,0x00,0x00,0x38,0x02,0x00,0x00, +0x79,0x02,0x00,0x00,0x42,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc2,0x00,0x00,0x00,0x45,0x02,0x00,0x00,0x2d,0x03,0x00,0x00, +0x60,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x41,0x02,0x00,0x00, +0x42,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x45,0x02,0x00,0x00,0x40,0x02,0x00,0x00,0x41,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x40,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0x47,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x47,0x02,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x2f,0x03,0x00,0x00, +0x3e,0x00,0x00,0x00,0x40,0x02,0x00,0x00,0x77,0x02,0x00,0x00, +0x4a,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, +0x4d,0x02,0x00,0x00,0x2f,0x03,0x00,0x00,0xbc,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0x49,0x02,0x00,0x00,0x4a,0x02,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x4d,0x02,0x00,0x00, +0x48,0x02,0x00,0x00,0x49,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x48,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x4f,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x4f,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0x31,0x03,0x00,0x00,0x3e,0x00,0x00,0x00, +0x48,0x02,0x00,0x00,0x75,0x02,0x00,0x00,0x50,0x02,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0x55,0x02,0x00,0x00, +0x31,0x03,0x00,0x00,0x62,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0x51,0x02,0x00,0x00,0x50,0x02,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x55,0x02,0x00,0x00,0x50,0x02,0x00,0x00, +0x51,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x50,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x57,0x02,0x00,0x00, +0x29,0x03,0x00,0x00,0xbc,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x59,0x02,0x00,0x00,0x57,0x02,0x00,0x00, +0x2f,0x03,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x5b,0x02,0x00,0x00,0x59,0x02,0x00,0x00,0x5a,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x5d,0x02,0x00,0x00, +0x2d,0x03,0x00,0x00,0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x5e,0x02,0x00,0x00,0x5b,0x02,0x00,0x00, +0x5d,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x60,0x02,0x00,0x00,0x5e,0x02,0x00,0x00,0x31,0x03,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x64,0x02,0x00,0x00, +0x5d,0x02,0x00,0x00,0x31,0x03,0x00,0x00,0x41,0x00,0x05,0x00, +0x02,0x02,0x00,0x00,0x65,0x02,0x00,0x00,0xed,0x01,0x00,0x00, +0x64,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0x01,0x01,0x00,0x00, +0x66,0x02,0x00,0x00,0x65,0x02,0x00,0x00,0x73,0x00,0x04,0x00, +0xc4,0x00,0x00,0x00,0x67,0x02,0x00,0x00,0x66,0x02,0x00,0x00, +0x41,0x00,0x05,0x00,0x02,0x02,0x00,0x00,0x6c,0x02,0x00,0x00, +0x1b,0x02,0x00,0x00,0x59,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, +0x01,0x01,0x00,0x00,0x6d,0x02,0x00,0x00,0x6c,0x02,0x00,0x00, +0x73,0x00,0x04,0x00,0xc4,0x00,0x00,0x00,0x6e,0x02,0x00,0x00, +0x6d,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0xcd,0x00,0x00,0x00, +0x70,0x02,0x00,0x00,0xca,0x00,0x00,0x00,0x60,0x02,0x00,0x00, +0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00,0x71,0x02,0x00,0x00, +0x70,0x02,0x00,0x00,0x0c,0x00,0x08,0x00,0xc4,0x00,0x00,0x00, +0x72,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00, +0x67,0x02,0x00,0x00,0x6e,0x02,0x00,0x00,0x71,0x02,0x00,0x00, +0x3e,0x00,0x03,0x00,0x70,0x02,0x00,0x00,0x72,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x75,0x02,0x00,0x00, +0x31,0x03,0x00,0x00,0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x4f,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x51,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0x4a,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x4a,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x77,0x02,0x00,0x00,0x2f,0x03,0x00,0x00,0xd0,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x47,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x49,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x42,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x42,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x79,0x02,0x00,0x00,0x2d,0x03,0x00,0x00, +0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x3f,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x41,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0x3a,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x3a,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x7b,0x02,0x00,0x00, +0x29,0x03,0x00,0x00,0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x37,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x39,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0xd6,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xd6,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x7d,0x02,0x00,0x00,0x23,0x03,0x00,0x00,0xd0,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xd3,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xd5,0x01,0x00,0x00,0xe0,0x00,0x04,0x00,0x0c,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0xcb,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0xd7,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xd7,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x7f,0x02,0x00,0x00, +0x09,0x03,0x00,0x00,0x6c,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xd4,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xd6,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x84,0x02,0x00,0x00, +0x55,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x85,0x02,0x00,0x00,0x96,0x00,0x00,0x00, +0x84,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x8a,0x02,0x00,0x00,0x59,0x00,0x00,0x00,0xb9,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8b,0x02,0x00,0x00, +0xa8,0x00,0x00,0x00,0x8a,0x02,0x00,0x00,0x41,0x00,0x05,0x00, +0x15,0x00,0x00,0x00,0x90,0x02,0x00,0x00,0x12,0x00,0x00,0x00, +0x8f,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x91,0x02,0x00,0x00,0x90,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x92,0x02,0x00,0x00,0x0f,0x00,0x00,0x00, +0x91,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x96,0x02,0x00,0x00,0x47,0x00,0x00,0x00,0x91,0x02,0x00,0x00, +0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00,0x98,0x02,0x00,0x00, +0x97,0x02,0x00,0x00,0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x99,0x02,0x00,0x00,0x98,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9a,0x02,0x00,0x00, +0x96,0x02,0x00,0x00,0x99,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x9b,0x02,0x00,0x00,0x92,0x02,0x00,0x00, +0x9a,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x9d,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x9d,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0x0a,0x03,0x00,0x00,0x3e,0x00,0x00,0x00, +0xd6,0x00,0x00,0x00,0x03,0x03,0x00,0x00,0xa0,0x02,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0xa3,0x02,0x00,0x00, +0x0a,0x03,0x00,0x00,0xbf,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0x9f,0x02,0x00,0x00,0xa0,0x02,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0xa3,0x02,0x00,0x00,0x9e,0x02,0x00,0x00, +0x9f,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x9e,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0xa5,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0xa5,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0x0b,0x03,0x00,0x00,0x3e,0x00,0x00,0x00,0x9e,0x02,0x00,0x00, +0x01,0x03,0x00,0x00,0xa8,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc2,0x00,0x00,0x00,0xab,0x02,0x00,0x00,0x0b,0x03,0x00,0x00, +0x60,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xa7,0x02,0x00,0x00, +0xa8,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0xab,0x02,0x00,0x00,0xa6,0x02,0x00,0x00,0xa7,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0xa6,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xaf,0x02,0x00,0x00,0x0b,0x03,0x00,0x00, +0x61,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xb0,0x02,0x00,0x00,0x85,0x02,0x00,0x00,0xaf,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xb2,0x02,0x00,0x00, +0x64,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xb3,0x02,0x00,0x00,0xb0,0x02,0x00,0x00, +0xb2,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xb7,0x02,0x00,0x00,0x0a,0x03,0x00,0x00,0x23,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xb8,0x02,0x00,0x00, +0x8b,0x02,0x00,0x00,0xb7,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xba,0x02,0x00,0x00,0x68,0x00,0x00,0x00, +0xbc,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xbb,0x02,0x00,0x00,0xb8,0x02,0x00,0x00,0xba,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0xbd,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0xbd,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0x0d,0x03,0x00,0x00,0x3e,0x00,0x00,0x00,0xa6,0x02,0x00,0x00, +0xff,0x02,0x00,0x00,0xc0,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc2,0x00,0x00,0x00,0xc3,0x02,0x00,0x00,0x0d,0x03,0x00,0x00, +0xbc,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xbf,0x02,0x00,0x00, +0xc0,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0xc3,0x02,0x00,0x00,0xbe,0x02,0x00,0x00,0xbf,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0xbe,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0xc5,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0xc5,0x02,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x0f,0x03,0x00,0x00, +0x3e,0x00,0x00,0x00,0xbe,0x02,0x00,0x00,0xfd,0x02,0x00,0x00, +0xc8,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, +0xcb,0x02,0x00,0x00,0x0f,0x03,0x00,0x00,0x62,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0xc7,0x02,0x00,0x00,0xc8,0x02,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xcb,0x02,0x00,0x00, +0xc6,0x02,0x00,0x00,0xc7,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0xc6,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xce,0x02,0x00,0x00,0xb3,0x02,0x00,0x00,0x0f,0x03,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0xd1,0x02,0x00,0x00, +0xce,0x02,0x00,0x00,0x36,0x00,0x00,0x00,0xf7,0x00,0x03,0x00, +0xd3,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0xd1,0x02,0x00,0x00,0xd2,0x02,0x00,0x00,0xd3,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0xd2,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xd6,0x02,0x00,0x00,0xbb,0x02,0x00,0x00, +0x0d,0x03,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, +0xd7,0x02,0x00,0x00,0x12,0x00,0x00,0x00,0xd0,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xd8,0x02,0x00,0x00, +0xd7,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, +0xd9,0x02,0x00,0x00,0xd6,0x02,0x00,0x00,0xd8,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0xd3,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0xd3,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0xc2,0x00,0x00,0x00, +0xda,0x02,0x00,0x00,0xd1,0x02,0x00,0x00,0xc6,0x02,0x00,0x00, +0xd9,0x02,0x00,0x00,0xd2,0x02,0x00,0x00,0xf7,0x00,0x03,0x00, +0xdc,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0xda,0x02,0x00,0x00,0xdb,0x02,0x00,0x00,0xdc,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0xdb,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xe4,0x02,0x00,0x00,0xbb,0x02,0x00,0x00, +0x0d,0x03,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, +0xe6,0x02,0x00,0x00,0x12,0x00,0x00,0x00,0xe5,0x02,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xe7,0x02,0x00,0x00, +0xe6,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xe8,0x02,0x00,0x00,0xe4,0x02,0x00,0x00,0xe7,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xe9,0x02,0x00,0x00, +0x9b,0x02,0x00,0x00,0xe8,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xeb,0x02,0x00,0x00,0xe9,0x02,0x00,0x00, +0xb3,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xed,0x02,0x00,0x00,0xeb,0x02,0x00,0x00,0x0f,0x03,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xef,0x02,0x00,0x00, +0x0a,0x03,0x00,0x00,0xbc,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xf1,0x02,0x00,0x00,0xef,0x02,0x00,0x00, +0x0d,0x03,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xf3,0x02,0x00,0x00,0xf1,0x02,0x00,0x00,0xf2,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xf5,0x02,0x00,0x00, +0x0b,0x03,0x00,0x00,0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xf6,0x02,0x00,0x00,0xf3,0x02,0x00,0x00, +0xf5,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xf8,0x02,0x00,0x00,0xf6,0x02,0x00,0x00,0x0f,0x03,0x00,0x00, +0x41,0x00,0x05,0x00,0xcd,0x00,0x00,0x00,0xf9,0x02,0x00,0x00, +0xca,0x00,0x00,0x00,0xf8,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, +0xc4,0x00,0x00,0x00,0xfa,0x02,0x00,0x00,0xf9,0x02,0x00,0x00, +0x41,0x00,0x06,0x00,0x8b,0x01,0x00,0x00,0xfb,0x02,0x00,0x00, +0xe0,0x02,0x00,0x00,0x34,0x00,0x00,0x00,0xed,0x02,0x00,0x00, +0x3e,0x00,0x03,0x00,0xfb,0x02,0x00,0x00,0xfa,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0xdc,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0xdc,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0xc8,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0xc8,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xfd,0x02,0x00,0x00,0x0f,0x03,0x00,0x00, +0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xc5,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0xc7,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0xc0,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0xc0,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xff,0x02,0x00,0x00, +0x0d,0x03,0x00,0x00,0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xbd,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0xbf,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0xa8,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0xa8,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x01,0x03,0x00,0x00,0x0b,0x03,0x00,0x00,0xd0,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xa5,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0xa7,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0xa0,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0xa0,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x03,0x03,0x00,0x00,0x0a,0x03,0x00,0x00, +0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x9d,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x9f,0x02,0x00,0x00,0xfd,0x00,0x01,0x00, +0x38,0x00,0x01,0x00, +}; +const uint64_t matmul_q5_1_f32_aligned_len = 11548; + +unsigned char matmul_q5_1_f32_aligned_fp32_data[] = { +0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, +0x10,0x03,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, +0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x51,0x11,0x00,0x00, +0x11,0x00,0x02,0x00,0x60,0x11,0x00,0x00,0x0b,0x00,0x06,0x00, +0x01,0x00,0x00,0x00,0x47,0x4c,0x53,0x4c,0x2e,0x73,0x74,0x64, +0x2e,0x34,0x35,0x30,0x00,0x00,0x00,0x00,0x0e,0x00,0x03,0x00, +0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x0f,0x00,0x0f,0x00, +0x05,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x6d,0x61,0x69,0x6e, +0x00,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0x3d,0x00,0x00,0x00,0x4c,0x00,0x00,0x00,0x08,0x01,0x00,0x00, +0x4c,0x01,0x00,0x00,0x7f,0x01,0x00,0x00,0x86,0x01,0x00,0x00, +0x6d,0x02,0x00,0x00,0xb6,0x02,0x00,0x00,0x10,0x00,0x06,0x00, +0x04,0x00,0x00,0x00,0x11,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x0b,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x1c,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x10,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x08,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x03,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x10,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x10,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, +0x05,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x14,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x10,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x1c,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, +0x08,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x09,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x24,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x10,0x00,0x00,0x00,0x0a,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x28,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, +0x0b,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x2c,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x30,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x10,0x00,0x00,0x00,0x0d,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x34,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x10,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x37,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x3d,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x1a,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x4c,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, +0x1b,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x4f,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x53,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x60,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x62,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x6c,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x03,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0xa7,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xb9,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0xbc,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x08,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x03,0x01,0x00,0x00,0x06,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x04,0x01,0x00,0x00, +0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x04,0x01,0x00,0x00,0x01,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x04,0x01,0x00,0x00,0x02,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x04,0x01,0x00,0x00, +0x03,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x08,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x05,0x01,0x00,0x00,0x06,0x00,0x00,0x00, +0x18,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0x06,0x01,0x00,0x00, +0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x06,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x06,0x01,0x00,0x00, +0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x08,0x01,0x00,0x00, +0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x08,0x01,0x00,0x00,0x21,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x57,0x01,0x00,0x00,0x01,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x58,0x01,0x00,0x00, +0x0b,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x83,0x01,0x00,0x00,0x06,0x00,0x00,0x00,0x10,0x00,0x00,0x00, +0x48,0x00,0x04,0x00,0x84,0x01,0x00,0x00,0x00,0x00,0x00,0x00, +0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x84,0x01,0x00,0x00, +0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x03,0x00,0x84,0x01,0x00,0x00,0x02,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x86,0x01,0x00,0x00,0x22,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x86,0x01,0x00,0x00, +0x21,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x6d,0x02,0x00,0x00,0x0b,0x00,0x00,0x00,0x18,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0xb3,0x02,0x00,0x00,0x06,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0xb4,0x02,0x00,0x00, +0x00,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0xb4,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0xb4,0x02,0x00,0x00, +0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xb6,0x02,0x00,0x00, +0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0xb6,0x02,0x00,0x00,0x21,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x13,0x00,0x02,0x00,0x02,0x00,0x00,0x00,0x21,0x00,0x03,0x00, +0x03,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x15,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x17,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x03,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x0a,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0x0a,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x0d,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x1e,0x00,0x10,0x00, +0x10,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x11,0x00,0x00,0x00,0x09,0x00,0x00,0x00, +0x10,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x11,0x00,0x00,0x00, +0x12,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x15,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x14,0x00,0x00,0x00, +0x08,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x15,0x00,0x00,0x00, +0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x0a,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x27,0x00,0x00,0x00, +0x09,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, +0x2d,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x37,0x00,0x00,0x00, +0x40,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x39,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0x0a,0x00,0x00,0x00,0x3d,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x3e,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00, +0x4c,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x53,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x54,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x37,0x00,0x00,0x00, +0x53,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x58,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x37,0x00,0x00,0x00, +0x53,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x60,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x61,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0x53,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x32,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x63,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0x61,0x00,0x00,0x00,0x62,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x67,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0x61,0x00,0x00,0x00,0x62,0x00,0x00,0x00, +0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, +0x10,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x6d,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x72,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x77,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x78,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x77,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x7d,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x77,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0x81,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, +0x91,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0x97,0x00,0x00,0x00,0x03,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0xa2,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xa7,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0xa9,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xb8,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00, +0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xb9,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xba,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x53,0x00,0x00,0x00, +0xb9,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xbb,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x4f,0x00,0x00,0x00, +0x62,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xbc,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xbd,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0xbb,0x00,0x00,0x00,0xbc,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xbe,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0xbd,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xbf,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0xba,0x00,0x00,0x00,0xbe,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xc0,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0xb8,0x00,0x00,0x00,0xbf,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xc1,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0xc0,0x00,0x00,0x00,0xbc,0x00,0x00,0x00,0x14,0x00,0x02,0x00, +0xc2,0x00,0x00,0x00,0x16,0x00,0x03,0x00,0xc4,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xc5,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00, +0x62,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xc6,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0xc5,0x00,0x00,0x00, +0xbf,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xc7,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0xc6,0x00,0x00,0x00, +0xbc,0x00,0x00,0x00,0x1c,0x00,0x04,0x00,0xc8,0x00,0x00,0x00, +0xc4,0x00,0x00,0x00,0xc7,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0xc9,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0xc8,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0xc4,0x00,0x00,0x00,0xcc,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0xcd,0x00,0x00,0x00, +0x07,0x00,0x00,0x00,0xc4,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0xd0,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xf4,0x00,0x00,0x00, +0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xfa,0x00,0x00,0x00, +0x10,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xfe,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x16,0x00,0x03,0x00, +0x01,0x01,0x00,0x00,0x10,0x00,0x00,0x00,0x15,0x00,0x04,0x00, +0x02,0x01,0x00,0x00,0x08,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x1c,0x00,0x04,0x00,0x03,0x01,0x00,0x00,0x02,0x01,0x00,0x00, +0xfa,0x00,0x00,0x00,0x1e,0x00,0x06,0x00,0x04,0x01,0x00,0x00, +0x01,0x01,0x00,0x00,0x01,0x01,0x00,0x00,0x06,0x00,0x00,0x00, +0x03,0x01,0x00,0x00,0x1d,0x00,0x03,0x00,0x05,0x01,0x00,0x00, +0x04,0x01,0x00,0x00,0x1e,0x00,0x03,0x00,0x06,0x01,0x00,0x00, +0x05,0x01,0x00,0x00,0x20,0x00,0x04,0x00,0x07,0x01,0x00,0x00, +0x0c,0x00,0x00,0x00,0x06,0x01,0x00,0x00,0x3b,0x00,0x04,0x00, +0x07,0x01,0x00,0x00,0x08,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x0a,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, +0x01,0x01,0x00,0x00,0x20,0x00,0x04,0x00,0x15,0x01,0x00,0x00, +0x0c,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x23,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x2c,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, +0x02,0x01,0x00,0x00,0x17,0x00,0x04,0x00,0x30,0x01,0x00,0x00, +0xc4,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x48,0x01,0x00,0x00,0x80,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x49,0x01,0x00,0x00,0x84,0x00,0x00,0x00, +0x37,0x00,0x00,0x00,0x48,0x01,0x00,0x00,0x1c,0x00,0x04,0x00, +0x4a,0x01,0x00,0x00,0xc4,0x00,0x00,0x00,0x49,0x01,0x00,0x00, +0x20,0x00,0x04,0x00,0x4b,0x01,0x00,0x00,0x04,0x00,0x00,0x00, +0x4a,0x01,0x00,0x00,0x3b,0x00,0x04,0x00,0x4b,0x01,0x00,0x00, +0x4c,0x01,0x00,0x00,0x04,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x50,0x01,0x00,0x00,0x04,0x00,0x00,0x00,0xc4,0x00,0x00,0x00, +0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x57,0x01,0x00,0x00, +0x01,0x00,0x00,0x00,0x33,0x00,0x06,0x00,0x09,0x00,0x00,0x00, +0x58,0x01,0x00,0x00,0x57,0x01,0x00,0x00,0x39,0x00,0x00,0x00, +0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x59,0x01,0x00,0x00,0x51,0x00,0x00,0x00,0x58,0x01,0x00,0x00, +0x00,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x5a,0x01,0x00,0x00,0x84,0x00,0x00,0x00,0x59,0x01,0x00,0x00, +0x0c,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x5b,0x01,0x00,0x00,0x86,0x00,0x00,0x00,0x5a,0x01,0x00,0x00, +0x6c,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x76,0x01,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, +0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x7b,0x01,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, +0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x7c,0x01,0x00,0x00,0x84,0x00,0x00,0x00,0xa7,0x00,0x00,0x00, +0x7b,0x01,0x00,0x00,0x1c,0x00,0x04,0x00,0x7d,0x01,0x00,0x00, +0xc4,0x00,0x00,0x00,0x7c,0x01,0x00,0x00,0x20,0x00,0x04,0x00, +0x7e,0x01,0x00,0x00,0x04,0x00,0x00,0x00,0x7d,0x01,0x00,0x00, +0x3b,0x00,0x04,0x00,0x7e,0x01,0x00,0x00,0x7f,0x01,0x00,0x00, +0x04,0x00,0x00,0x00,0x17,0x00,0x04,0x00,0x82,0x01,0x00,0x00, +0xc4,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, +0x83,0x01,0x00,0x00,0x82,0x01,0x00,0x00,0x1e,0x00,0x03,0x00, +0x84,0x01,0x00,0x00,0x83,0x01,0x00,0x00,0x20,0x00,0x04,0x00, +0x85,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x84,0x01,0x00,0x00, +0x3b,0x00,0x04,0x00,0x85,0x01,0x00,0x00,0x86,0x01,0x00,0x00, +0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x88,0x01,0x00,0x00, +0x0c,0x00,0x00,0x00,0xc4,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x99,0x01,0x00,0x00,0x03,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x9f,0x01,0x00,0x00, +0x51,0x00,0x00,0x00,0x58,0x01,0x00,0x00,0x00,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xa0,0x01,0x00,0x00, +0x84,0x00,0x00,0x00,0x9f,0x01,0x00,0x00,0x77,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xa1,0x01,0x00,0x00, +0x86,0x00,0x00,0x00,0xa0,0x01,0x00,0x00,0x6c,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xa4,0x01,0x00,0x00, +0x08,0x01,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xa5,0x01,0x00,0x00,0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xa8,0x01,0x00,0x00,0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, +0x77,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xc3,0x01,0x00,0x00,0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00, +0x62,0x00,0x00,0x00,0x1c,0x00,0x04,0x00,0xc4,0x01,0x00,0x00, +0xc4,0x00,0x00,0x00,0xc3,0x01,0x00,0x00,0x20,0x00,0x04,0x00, +0xc5,0x01,0x00,0x00,0x07,0x00,0x00,0x00,0xc4,0x01,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xd5,0x01,0x00,0x00, +0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xf0,0x01,0x00,0x00, +0x84,0x00,0x00,0x00,0xbf,0x00,0x00,0x00,0xbc,0x00,0x00,0x00, +0x1c,0x00,0x04,0x00,0xf1,0x01,0x00,0x00,0xc4,0x00,0x00,0x00, +0xf0,0x01,0x00,0x00,0x20,0x00,0x04,0x00,0xf2,0x01,0x00,0x00, +0x07,0x00,0x00,0x00,0xf1,0x01,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xfb,0x01,0x00,0x00,0x86,0x00,0x00,0x00, +0xb9,0x00,0x00,0x00,0xbf,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x03,0x02,0x00,0x00,0x80,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x32,0x02,0x00,0x00,0x84,0x00,0x00,0x00, +0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0x65,0x02,0x00,0x00,0x0d,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00,0x6d,0x02,0x00,0x00, +0x01,0x00,0x00,0x00,0x1d,0x00,0x03,0x00,0xb3,0x02,0x00,0x00, +0xc4,0x00,0x00,0x00,0x1e,0x00,0x03,0x00,0xb4,0x02,0x00,0x00, +0xb3,0x02,0x00,0x00,0x20,0x00,0x04,0x00,0xb5,0x02,0x00,0x00, +0x0c,0x00,0x00,0x00,0xb4,0x02,0x00,0x00,0x3b,0x00,0x04,0x00, +0xb5,0x02,0x00,0x00,0xb6,0x02,0x00,0x00,0x0c,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0xbb,0x02,0x00,0x00, +0x05,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xc8,0x02,0x00,0x00,0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00, +0x62,0x00,0x00,0x00,0x36,0x00,0x05,0x00,0x02,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x03,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0x05,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0xc9,0x00,0x00,0x00,0xca,0x00,0x00,0x00,0x07,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0xc5,0x01,0x00,0x00,0xc6,0x01,0x00,0x00, +0x07,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0xf2,0x01,0x00,0x00, +0xf3,0x01,0x00,0x00,0x07,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x0d,0x00,0x00,0x00,0x0e,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x0f,0x00,0x00,0x00,0x0e,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x15,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0x14,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x17,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, +0x17,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x1e,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x17,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x22,0x00,0x00,0x00, +0x12,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x22,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x24,0x00,0x00,0x00, +0x18,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x15,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0x27,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x29,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x2a,0x00,0x00,0x00,0x1e,0x00,0x00,0x00, +0x29,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, +0x2e,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x2d,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x2f,0x00,0x00,0x00, +0x2e,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x30,0x00,0x00,0x00,0x24,0x00,0x00,0x00,0x2f,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x32,0x00,0x00,0x00, +0x30,0x00,0x00,0x00,0x2a,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x15,0x00,0x00,0x00,0x35,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0x34,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x36,0x00,0x00,0x00,0x35,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x38,0x00,0x00,0x00,0x36,0x00,0x00,0x00, +0x37,0x00,0x00,0x00,0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x3a,0x00,0x00,0x00,0x38,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3b,0x00,0x00,0x00, +0x3a,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x0d,0x00,0x00,0x00,0x3f,0x00,0x00,0x00,0x3d,0x00,0x00,0x00, +0x3e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x40,0x00,0x00,0x00,0x3f,0x00,0x00,0x00,0x89,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x42,0x00,0x00,0x00,0x40,0x00,0x00,0x00, +0x3b,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x47,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x3b,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00,0x49,0x00,0x00,0x00, +0x3d,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x4a,0x00,0x00,0x00,0x49,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00,0x4d,0x00,0x00,0x00, +0x4c,0x00,0x00,0x00,0x3e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x4d,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x50,0x00,0x00,0x00, +0x4e,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x89,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x55,0x00,0x00,0x00,0x50,0x00,0x00,0x00, +0x54,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x59,0x00,0x00,0x00,0x50,0x00,0x00,0x00,0x58,0x00,0x00,0x00, +0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x5d,0x00,0x00,0x00, +0x4e,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x89,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x64,0x00,0x00,0x00,0x5d,0x00,0x00,0x00, +0x63,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x68,0x00,0x00,0x00,0x5d,0x00,0x00,0x00,0x67,0x00,0x00,0x00, +0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x6e,0x00,0x00,0x00, +0x4e,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x73,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, +0x72,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x79,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x78,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x7e,0x00,0x00,0x00, +0x4e,0x00,0x00,0x00,0x7d,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x15,0x00,0x00,0x00,0x82,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0x81,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x83,0x00,0x00,0x00,0x82,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x47,0x00,0x00,0x00, +0x83,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, +0x87,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x88,0x00,0x00,0x00, +0x87,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x8a,0x00,0x00,0x00,0x47,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8d,0x00,0x00,0x00, +0x8a,0x00,0x00,0x00,0x83,0x00,0x00,0x00,0x0c,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0x8e,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x26,0x00,0x00,0x00,0x88,0x00,0x00,0x00,0x8d,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x92,0x00,0x00,0x00, +0x12,0x00,0x00,0x00,0x91,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x93,0x00,0x00,0x00,0x92,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x94,0x00,0x00,0x00, +0x32,0x00,0x00,0x00,0x93,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x96,0x00,0x00,0x00,0x42,0x00,0x00,0x00, +0x37,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, +0x98,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x97,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x99,0x00,0x00,0x00, +0x98,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x9a,0x00,0x00,0x00,0x96,0x00,0x00,0x00,0x99,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9b,0x00,0x00,0x00, +0x94,0x00,0x00,0x00,0x9a,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x9d,0x00,0x00,0x00,0x9b,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x9e,0x00,0x00,0x00,0x9d,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0xa3,0x00,0x00,0x00, +0x12,0x00,0x00,0x00,0xa2,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xa4,0x00,0x00,0x00,0xa3,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa5,0x00,0x00,0x00, +0x0f,0x00,0x00,0x00,0xa4,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xa8,0x00,0x00,0x00,0x4a,0x00,0x00,0x00, +0xa7,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, +0xaa,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0xa9,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xab,0x00,0x00,0x00, +0xaa,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xac,0x00,0x00,0x00,0xa8,0x00,0x00,0x00,0xab,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xad,0x00,0x00,0x00, +0xa5,0x00,0x00,0x00,0xac,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xaf,0x00,0x00,0x00,0xad,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xb0,0x00,0x00,0x00,0xaf,0x00,0x00,0x00,0x77,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xb2,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xb2,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xde,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0x05,0x00,0x00,0x00, +0xd1,0x00,0x00,0x00,0xb3,0x00,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc2,0x00,0x00,0x00,0xc3,0x00,0x00,0x00,0xde,0x02,0x00,0x00, +0xc1,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xb4,0x00,0x00,0x00, +0xb3,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0xc3,0x00,0x00,0x00,0xb3,0x00,0x00,0x00,0xb4,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xb3,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0xcd,0x00,0x00,0x00,0xce,0x00,0x00,0x00,0xca,0x00,0x00,0x00, +0xde,0x02,0x00,0x00,0x3e,0x00,0x03,0x00,0xce,0x00,0x00,0x00, +0xcc,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xd1,0x00,0x00,0x00,0xde,0x02,0x00,0x00,0xd0,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xb2,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xb4,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xd4,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xd4,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xf7,0x02,0x00,0x00,0xb0,0x00,0x00,0x00, +0xb4,0x00,0x00,0x00,0xaa,0x01,0x00,0x00,0xd7,0x00,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xf3,0x02,0x00,0x00, +0x9e,0x00,0x00,0x00,0xb4,0x00,0x00,0x00,0xa7,0x01,0x00,0x00, +0xd7,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xdf,0x02,0x00,0x00,0x84,0x00,0x00,0x00,0xb4,0x00,0x00,0x00, +0x55,0x02,0x00,0x00,0xd7,0x00,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc2,0x00,0x00,0x00,0xdb,0x00,0x00,0x00,0xdf,0x02,0x00,0x00, +0x8e,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xd6,0x00,0x00,0x00, +0xd7,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0xdb,0x00,0x00,0x00,0xd5,0x00,0x00,0x00,0xd6,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xd5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xdd,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xdd,0x00,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xef,0x02,0x00,0x00, +0x3e,0x00,0x00,0x00,0xd5,0x00,0x00,0x00,0x5d,0x01,0x00,0x00, +0xde,0x00,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, +0xe3,0x00,0x00,0x00,0xef,0x02,0x00,0x00,0x37,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0xdf,0x00,0x00,0x00,0xde,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xe3,0x00,0x00,0x00, +0xde,0x00,0x00,0x00,0xdf,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xde,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xe8,0x00,0x00,0x00,0x73,0x00,0x00,0x00,0xef,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xeb,0x00,0x00,0x00, +0xe8,0x00,0x00,0x00,0x99,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xec,0x00,0x00,0x00,0xeb,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xed,0x00,0x00,0x00,0xf3,0x02,0x00,0x00,0xec,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xef,0x00,0x00,0x00, +0xed,0x00,0x00,0x00,0x6e,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xf5,0x00,0x00,0x00,0xe8,0x00,0x00,0x00, +0xf4,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xf7,0x00,0x00,0x00,0xf5,0x00,0x00,0x00,0x6e,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xfb,0x00,0x00,0x00, +0xef,0x00,0x00,0x00,0xfa,0x00,0x00,0x00,0xc7,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xff,0x00,0x00,0x00,0xef,0x00,0x00,0x00, +0xfe,0x00,0x00,0x00,0x41,0x00,0x07,0x00,0x0a,0x01,0x00,0x00, +0x0b,0x01,0x00,0x00,0x08,0x01,0x00,0x00,0x34,0x00,0x00,0x00, +0xfb,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x01,0x01,0x00,0x00,0x0c,0x01,0x00,0x00,0x0b,0x01,0x00,0x00, +0x73,0x00,0x04,0x00,0xc4,0x00,0x00,0x00,0x0d,0x01,0x00,0x00, +0x0c,0x01,0x00,0x00,0x41,0x00,0x07,0x00,0x0a,0x01,0x00,0x00, +0x10,0x01,0x00,0x00,0x08,0x01,0x00,0x00,0x34,0x00,0x00,0x00, +0xfb,0x00,0x00,0x00,0xd0,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x01,0x01,0x00,0x00,0x11,0x01,0x00,0x00,0x10,0x01,0x00,0x00, +0x73,0x00,0x04,0x00,0xc4,0x00,0x00,0x00,0x12,0x01,0x00,0x00, +0x11,0x01,0x00,0x00,0x41,0x00,0x07,0x00,0x15,0x01,0x00,0x00, +0x16,0x01,0x00,0x00,0x08,0x01,0x00,0x00,0x34,0x00,0x00,0x00, +0xfb,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x17,0x01,0x00,0x00,0x16,0x01,0x00,0x00, +0xc2,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x1d,0x01,0x00,0x00, +0x17,0x01,0x00,0x00,0xff,0x00,0x00,0x00,0xc4,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x1e,0x01,0x00,0x00,0x1d,0x01,0x00,0x00, +0xa9,0x00,0x00,0x00,0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x1f,0x01,0x00,0x00,0x1e,0x01,0x00,0x00,0xfa,0x00,0x00,0x00, +0x7c,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x20,0x01,0x00,0x00, +0x1f,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x24,0x01,0x00,0x00,0xff,0x00,0x00,0x00,0x23,0x01,0x00,0x00, +0xc2,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x25,0x01,0x00,0x00, +0x17,0x01,0x00,0x00,0x24,0x01,0x00,0x00,0xc7,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x26,0x01,0x00,0x00,0x25,0x01,0x00,0x00, +0xfa,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x13,0x00,0x00,0x00, +0x27,0x01,0x00,0x00,0x26,0x01,0x00,0x00,0x41,0x00,0x08,0x00, +0x2c,0x01,0x00,0x00,0x2d,0x01,0x00,0x00,0x08,0x01,0x00,0x00, +0x34,0x00,0x00,0x00,0xfb,0x00,0x00,0x00,0x97,0x00,0x00,0x00, +0xff,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x02,0x01,0x00,0x00, +0x2e,0x01,0x00,0x00,0x2d,0x01,0x00,0x00,0x71,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x2f,0x01,0x00,0x00,0x2e,0x01,0x00,0x00, +0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x34,0x01,0x00,0x00, +0x2f,0x01,0x00,0x00,0xfe,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x38,0x01,0x00,0x00,0x20,0x01,0x00,0x00, +0xc5,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x39,0x01,0x00,0x00, +0x34,0x01,0x00,0x00,0x38,0x01,0x00,0x00,0x70,0x00,0x04,0x00, +0xc4,0x00,0x00,0x00,0x3a,0x01,0x00,0x00,0x39,0x01,0x00,0x00, +0xc2,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3c,0x01,0x00,0x00, +0x2f,0x01,0x00,0x00,0xa9,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x3f,0x01,0x00,0x00,0x27,0x01,0x00,0x00, +0xc5,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x40,0x01,0x00,0x00, +0x3c,0x01,0x00,0x00,0x3f,0x01,0x00,0x00,0x70,0x00,0x04,0x00, +0xc4,0x00,0x00,0x00,0x41,0x01,0x00,0x00,0x40,0x01,0x00,0x00, +0x50,0x00,0x05,0x00,0x30,0x01,0x00,0x00,0x42,0x01,0x00,0x00, +0x3a,0x01,0x00,0x00,0x41,0x01,0x00,0x00,0x8e,0x00,0x05,0x00, +0x30,0x01,0x00,0x00,0x44,0x01,0x00,0x00,0x42,0x01,0x00,0x00, +0x0d,0x01,0x00,0x00,0x50,0x00,0x05,0x00,0x30,0x01,0x00,0x00, +0x46,0x01,0x00,0x00,0x12,0x01,0x00,0x00,0x12,0x01,0x00,0x00, +0x81,0x00,0x05,0x00,0x30,0x01,0x00,0x00,0x47,0x01,0x00,0x00, +0x44,0x01,0x00,0x00,0x46,0x01,0x00,0x00,0x51,0x00,0x05,0x00, +0xc4,0x00,0x00,0x00,0x4f,0x01,0x00,0x00,0x47,0x01,0x00,0x00, +0x00,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x50,0x01,0x00,0x00, +0x51,0x01,0x00,0x00,0x4c,0x01,0x00,0x00,0xf7,0x00,0x00,0x00, +0x3e,0x00,0x03,0x00,0x51,0x01,0x00,0x00,0x4f,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x53,0x01,0x00,0x00, +0xf7,0x00,0x00,0x00,0xfa,0x00,0x00,0x00,0x51,0x00,0x05,0x00, +0xc4,0x00,0x00,0x00,0x55,0x01,0x00,0x00,0x47,0x01,0x00,0x00, +0x01,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x50,0x01,0x00,0x00, +0x56,0x01,0x00,0x00,0x4c,0x01,0x00,0x00,0x53,0x01,0x00,0x00, +0x3e,0x00,0x03,0x00,0x56,0x01,0x00,0x00,0x55,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x5d,0x01,0x00,0x00, +0xef,0x02,0x00,0x00,0x5b,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0xdd,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xdf,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x5f,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x5f,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xf0,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0xdf,0x00,0x00,0x00, +0xa3,0x01,0x00,0x00,0x60,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc2,0x00,0x00,0x00,0x65,0x01,0x00,0x00,0xf0,0x02,0x00,0x00, +0xa7,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x61,0x01,0x00,0x00, +0x60,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x65,0x01,0x00,0x00,0x60,0x01,0x00,0x00,0x61,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x60,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x6a,0x01,0x00,0x00,0x7e,0x00,0x00,0x00, +0xf0,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x6d,0x01,0x00,0x00,0x6a,0x01,0x00,0x00,0xab,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x6e,0x01,0x00,0x00, +0x6d,0x01,0x00,0x00,0x77,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x6f,0x01,0x00,0x00,0xf7,0x02,0x00,0x00, +0x6e,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x71,0x01,0x00,0x00,0x6f,0x01,0x00,0x00,0x79,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x77,0x01,0x00,0x00, +0x6a,0x01,0x00,0x00,0x76,0x01,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x79,0x01,0x00,0x00,0x79,0x00,0x00,0x00, +0x77,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x7a,0x01,0x00,0x00,0x77,0x01,0x00,0x00,0x79,0x01,0x00,0x00, +0x41,0x00,0x07,0x00,0x88,0x01,0x00,0x00,0x89,0x01,0x00,0x00, +0x86,0x01,0x00,0x00,0x34,0x00,0x00,0x00,0x71,0x01,0x00,0x00, +0x3e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00, +0x8a,0x01,0x00,0x00,0x89,0x01,0x00,0x00,0x41,0x00,0x05,0x00, +0x50,0x01,0x00,0x00,0x8b,0x01,0x00,0x00,0x7f,0x01,0x00,0x00, +0x7a,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x8b,0x01,0x00,0x00, +0x8a,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x8d,0x01,0x00,0x00,0x7a,0x01,0x00,0x00,0x39,0x00,0x00,0x00, +0x41,0x00,0x07,0x00,0x88,0x01,0x00,0x00,0x8f,0x01,0x00,0x00, +0x86,0x01,0x00,0x00,0x34,0x00,0x00,0x00,0x71,0x01,0x00,0x00, +0x39,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00, +0x90,0x01,0x00,0x00,0x8f,0x01,0x00,0x00,0x41,0x00,0x05,0x00, +0x50,0x01,0x00,0x00,0x91,0x01,0x00,0x00,0x7f,0x01,0x00,0x00, +0x8d,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x91,0x01,0x00,0x00, +0x90,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x93,0x01,0x00,0x00,0x7a,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, +0x41,0x00,0x07,0x00,0x88,0x01,0x00,0x00,0x95,0x01,0x00,0x00, +0x86,0x01,0x00,0x00,0x34,0x00,0x00,0x00,0x71,0x01,0x00,0x00, +0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00, +0x96,0x01,0x00,0x00,0x95,0x01,0x00,0x00,0x41,0x00,0x05,0x00, +0x50,0x01,0x00,0x00,0x97,0x01,0x00,0x00,0x7f,0x01,0x00,0x00, +0x93,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x97,0x01,0x00,0x00, +0x96,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x9a,0x01,0x00,0x00,0x7a,0x01,0x00,0x00,0x99,0x01,0x00,0x00, +0x41,0x00,0x07,0x00,0x88,0x01,0x00,0x00,0x9c,0x01,0x00,0x00, +0x86,0x01,0x00,0x00,0x34,0x00,0x00,0x00,0x71,0x01,0x00,0x00, +0x99,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00, +0x9d,0x01,0x00,0x00,0x9c,0x01,0x00,0x00,0x41,0x00,0x05,0x00, +0x50,0x01,0x00,0x00,0x9e,0x01,0x00,0x00,0x7f,0x01,0x00,0x00, +0x9a,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x9e,0x01,0x00,0x00, +0x9d,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xa3,0x01,0x00,0x00,0xf0,0x02,0x00,0x00,0xa1,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0x5f,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x61,0x01,0x00,0x00,0xe0,0x00,0x04,0x00,0x0c,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0xa4,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xa7,0x01,0x00,0x00,0xf3,0x02,0x00,0x00, +0xa5,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xaa,0x01,0x00,0x00,0xf7,0x02,0x00,0x00,0xa8,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0xac,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xac,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xf9,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0x61,0x01,0x00,0x00, +0x53,0x02,0x00,0x00,0xaf,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc2,0x00,0x00,0x00,0xb2,0x01,0x00,0x00,0xf9,0x02,0x00,0x00, +0x6c,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xae,0x01,0x00,0x00, +0xaf,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0xb2,0x01,0x00,0x00,0xad,0x01,0x00,0x00,0xae,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xad,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0xb4,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xb4,0x01,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xfd,0x02,0x00,0x00, +0x3e,0x00,0x00,0x00,0xad,0x01,0x00,0x00,0xdf,0x01,0x00,0x00, +0xb7,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, +0xba,0x01,0x00,0x00,0xfd,0x02,0x00,0x00,0x60,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0xb6,0x01,0x00,0x00,0xb7,0x01,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xba,0x01,0x00,0x00, +0xb5,0x01,0x00,0x00,0xb6,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xb5,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xbc,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xbc,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0x0f,0x03,0x00,0x00,0x3e,0x00,0x00,0x00, +0xb5,0x01,0x00,0x00,0xdd,0x01,0x00,0x00,0xbd,0x01,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0xc2,0x01,0x00,0x00, +0x0f,0x03,0x00,0x00,0x62,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0xbe,0x01,0x00,0x00,0xbd,0x01,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0xc2,0x01,0x00,0x00,0xbd,0x01,0x00,0x00, +0xbe,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xbd,0x01,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xc8,0x01,0x00,0x00, +0xfd,0x02,0x00,0x00,0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xca,0x01,0x00,0x00,0xc8,0x01,0x00,0x00, +0x0f,0x03,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xcc,0x01,0x00,0x00,0x55,0x00,0x00,0x00,0x53,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xce,0x01,0x00,0x00, +0xfd,0x02,0x00,0x00,0x61,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xcf,0x01,0x00,0x00,0xcc,0x01,0x00,0x00, +0xce,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xd1,0x01,0x00,0x00,0x64,0x00,0x00,0x00,0x62,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xd2,0x01,0x00,0x00, +0xcf,0x01,0x00,0x00,0xd1,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xd4,0x01,0x00,0x00,0xd2,0x01,0x00,0x00, +0x0f,0x03,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xd6,0x01,0x00,0x00,0xd4,0x01,0x00,0x00,0xd5,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xd8,0x01,0x00,0x00, +0xd6,0x01,0x00,0x00,0xf9,0x02,0x00,0x00,0x41,0x00,0x05,0x00, +0x50,0x01,0x00,0x00,0xd9,0x01,0x00,0x00,0x4c,0x01,0x00,0x00, +0xd8,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00, +0xda,0x01,0x00,0x00,0xd9,0x01,0x00,0x00,0x41,0x00,0x05,0x00, +0xcd,0x00,0x00,0x00,0xdb,0x01,0x00,0x00,0xc6,0x01,0x00,0x00, +0xca,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0xdb,0x01,0x00,0x00, +0xda,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xdd,0x01,0x00,0x00,0x0f,0x03,0x00,0x00,0xd0,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xbc,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xbe,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xb7,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xb7,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xdf,0x01,0x00,0x00,0xfd,0x02,0x00,0x00, +0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xb4,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xb6,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0xe1,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xe1,0x01,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xfe,0x02,0x00,0x00, +0x3e,0x00,0x00,0x00,0xb6,0x01,0x00,0x00,0x0d,0x02,0x00,0x00, +0xe4,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, +0xe7,0x01,0x00,0x00,0xfe,0x02,0x00,0x00,0xbf,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0xe3,0x01,0x00,0x00,0xe4,0x01,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xe7,0x01,0x00,0x00, +0xe2,0x01,0x00,0x00,0xe3,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xe2,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xe9,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xe9,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0x0c,0x03,0x00,0x00,0x3e,0x00,0x00,0x00, +0xe2,0x01,0x00,0x00,0x0b,0x02,0x00,0x00,0xea,0x01,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0xef,0x01,0x00,0x00, +0x0c,0x03,0x00,0x00,0xbc,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0xeb,0x01,0x00,0x00,0xea,0x01,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0xef,0x01,0x00,0x00,0xea,0x01,0x00,0x00, +0xeb,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xea,0x01,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xf5,0x01,0x00,0x00, +0xfe,0x02,0x00,0x00,0xbc,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xf7,0x01,0x00,0x00,0xf5,0x01,0x00,0x00, +0x0c,0x03,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xf9,0x01,0x00,0x00,0x59,0x00,0x00,0x00,0xb9,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xfc,0x01,0x00,0x00, +0xfe,0x02,0x00,0x00,0xfb,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xfd,0x01,0x00,0x00,0xf9,0x01,0x00,0x00, +0xfc,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xff,0x01,0x00,0x00,0x68,0x00,0x00,0x00,0xbc,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x00,0x02,0x00,0x00, +0xfd,0x01,0x00,0x00,0xff,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x02,0x02,0x00,0x00,0x00,0x02,0x00,0x00, +0x0c,0x03,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x04,0x02,0x00,0x00,0x02,0x02,0x00,0x00,0x03,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x06,0x02,0x00,0x00, +0x04,0x02,0x00,0x00,0xf9,0x02,0x00,0x00,0x41,0x00,0x05,0x00, +0x50,0x01,0x00,0x00,0x07,0x02,0x00,0x00,0x7f,0x01,0x00,0x00, +0x06,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00, +0x08,0x02,0x00,0x00,0x07,0x02,0x00,0x00,0x41,0x00,0x05,0x00, +0xcd,0x00,0x00,0x00,0x09,0x02,0x00,0x00,0xf3,0x01,0x00,0x00, +0xf7,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x09,0x02,0x00,0x00, +0x08,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x0b,0x02,0x00,0x00,0x0c,0x03,0x00,0x00,0xd0,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xe9,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xeb,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xe4,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xe4,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x0d,0x02,0x00,0x00,0xfe,0x02,0x00,0x00, +0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xe1,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xe3,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0x0f,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x0f,0x02,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xff,0x02,0x00,0x00, +0x3e,0x00,0x00,0x00,0xe3,0x01,0x00,0x00,0x51,0x02,0x00,0x00, +0x12,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, +0x15,0x02,0x00,0x00,0xff,0x02,0x00,0x00,0xbf,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0x11,0x02,0x00,0x00,0x12,0x02,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x15,0x02,0x00,0x00, +0x10,0x02,0x00,0x00,0x11,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x10,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x17,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x17,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0x03,0x03,0x00,0x00,0x3e,0x00,0x00,0x00, +0x10,0x02,0x00,0x00,0x4f,0x02,0x00,0x00,0x1a,0x02,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0x1d,0x02,0x00,0x00, +0x03,0x03,0x00,0x00,0x60,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0x19,0x02,0x00,0x00,0x1a,0x02,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x1d,0x02,0x00,0x00,0x18,0x02,0x00,0x00, +0x19,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x18,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0x1f,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x1f,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0x05,0x03,0x00,0x00,0x3e,0x00,0x00,0x00,0x18,0x02,0x00,0x00, +0x4d,0x02,0x00,0x00,0x22,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc2,0x00,0x00,0x00,0x25,0x02,0x00,0x00,0x05,0x03,0x00,0x00, +0xbc,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x21,0x02,0x00,0x00, +0x22,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x25,0x02,0x00,0x00,0x20,0x02,0x00,0x00,0x21,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x20,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0x27,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x27,0x02,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x07,0x03,0x00,0x00, +0x3e,0x00,0x00,0x00,0x20,0x02,0x00,0x00,0x4b,0x02,0x00,0x00, +0x28,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, +0x2d,0x02,0x00,0x00,0x07,0x03,0x00,0x00,0x62,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0x29,0x02,0x00,0x00,0x28,0x02,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x2d,0x02,0x00,0x00, +0x28,0x02,0x00,0x00,0x29,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x28,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x2f,0x02,0x00,0x00,0xff,0x02,0x00,0x00,0xbc,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x31,0x02,0x00,0x00, +0x2f,0x02,0x00,0x00,0x05,0x03,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x33,0x02,0x00,0x00,0x31,0x02,0x00,0x00, +0x32,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x35,0x02,0x00,0x00,0x03,0x03,0x00,0x00,0x62,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x36,0x02,0x00,0x00, +0x33,0x02,0x00,0x00,0x35,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x38,0x02,0x00,0x00,0x36,0x02,0x00,0x00, +0x07,0x03,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x3c,0x02,0x00,0x00,0x35,0x02,0x00,0x00,0x07,0x03,0x00,0x00, +0x41,0x00,0x05,0x00,0xcd,0x00,0x00,0x00,0x3d,0x02,0x00,0x00, +0xc6,0x01,0x00,0x00,0x3c,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, +0xc4,0x00,0x00,0x00,0x3e,0x02,0x00,0x00,0x3d,0x02,0x00,0x00, +0x41,0x00,0x05,0x00,0xcd,0x00,0x00,0x00,0x43,0x02,0x00,0x00, +0xf3,0x01,0x00,0x00,0x31,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, +0xc4,0x00,0x00,0x00,0x44,0x02,0x00,0x00,0x43,0x02,0x00,0x00, +0x41,0x00,0x05,0x00,0xcd,0x00,0x00,0x00,0x46,0x02,0x00,0x00, +0xca,0x00,0x00,0x00,0x38,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, +0xc4,0x00,0x00,0x00,0x47,0x02,0x00,0x00,0x46,0x02,0x00,0x00, +0x0c,0x00,0x08,0x00,0xc4,0x00,0x00,0x00,0x48,0x02,0x00,0x00, +0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x3e,0x02,0x00,0x00, +0x44,0x02,0x00,0x00,0x47,0x02,0x00,0x00,0x3e,0x00,0x03,0x00, +0x46,0x02,0x00,0x00,0x48,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x4b,0x02,0x00,0x00,0x07,0x03,0x00,0x00, +0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x27,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x29,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0x22,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x22,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x4d,0x02,0x00,0x00, +0x05,0x03,0x00,0x00,0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x1f,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x21,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0x1a,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x1a,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x4f,0x02,0x00,0x00,0x03,0x03,0x00,0x00,0xd0,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x17,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x19,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x12,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x12,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x51,0x02,0x00,0x00,0xff,0x02,0x00,0x00, +0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x0f,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x11,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0xaf,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xaf,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x53,0x02,0x00,0x00, +0xf9,0x02,0x00,0x00,0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xac,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xae,0x01,0x00,0x00, +0xe0,0x00,0x04,0x00,0x0c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0xa4,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xd7,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xd7,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x55,0x02,0x00,0x00,0xdf,0x02,0x00,0x00, +0x6c,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xd4,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xd6,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x5a,0x02,0x00,0x00,0x55,0x00,0x00,0x00, +0x53,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x5b,0x02,0x00,0x00,0x96,0x00,0x00,0x00,0x5a,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x60,0x02,0x00,0x00, +0x59,0x00,0x00,0x00,0xb9,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x61,0x02,0x00,0x00,0xa8,0x00,0x00,0x00, +0x60,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, +0x66,0x02,0x00,0x00,0x12,0x00,0x00,0x00,0x65,0x02,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x67,0x02,0x00,0x00, +0x66,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x68,0x02,0x00,0x00,0x0f,0x00,0x00,0x00,0x67,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x6c,0x02,0x00,0x00, +0x47,0x00,0x00,0x00,0x67,0x02,0x00,0x00,0x41,0x00,0x05,0x00, +0x0d,0x00,0x00,0x00,0x6e,0x02,0x00,0x00,0x6d,0x02,0x00,0x00, +0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x6f,0x02,0x00,0x00,0x6e,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x70,0x02,0x00,0x00,0x6c,0x02,0x00,0x00, +0x6f,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x71,0x02,0x00,0x00,0x68,0x02,0x00,0x00,0x70,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0x73,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x73,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xe0,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0xd6,0x00,0x00,0x00, +0xd9,0x02,0x00,0x00,0x76,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc2,0x00,0x00,0x00,0x79,0x02,0x00,0x00,0xe0,0x02,0x00,0x00, +0xbf,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x75,0x02,0x00,0x00, +0x76,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x79,0x02,0x00,0x00,0x74,0x02,0x00,0x00,0x75,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x74,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0x7b,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x7b,0x02,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xe1,0x02,0x00,0x00, +0x3e,0x00,0x00,0x00,0x74,0x02,0x00,0x00,0xd7,0x02,0x00,0x00, +0x7e,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, +0x81,0x02,0x00,0x00,0xe1,0x02,0x00,0x00,0x60,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0x7d,0x02,0x00,0x00,0x7e,0x02,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x81,0x02,0x00,0x00, +0x7c,0x02,0x00,0x00,0x7d,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x7c,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x85,0x02,0x00,0x00,0xe1,0x02,0x00,0x00,0x61,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x86,0x02,0x00,0x00, +0x5b,0x02,0x00,0x00,0x85,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x88,0x02,0x00,0x00,0x64,0x00,0x00,0x00, +0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x89,0x02,0x00,0x00,0x86,0x02,0x00,0x00,0x88,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8d,0x02,0x00,0x00, +0xe0,0x02,0x00,0x00,0xfb,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x8e,0x02,0x00,0x00,0x61,0x02,0x00,0x00, +0x8d,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x90,0x02,0x00,0x00,0x68,0x00,0x00,0x00,0xbc,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x91,0x02,0x00,0x00, +0x8e,0x02,0x00,0x00,0x90,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0x93,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x93,0x02,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xe3,0x02,0x00,0x00, +0x3e,0x00,0x00,0x00,0x7c,0x02,0x00,0x00,0xd5,0x02,0x00,0x00, +0x96,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, +0x99,0x02,0x00,0x00,0xe3,0x02,0x00,0x00,0xbc,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0x95,0x02,0x00,0x00,0x96,0x02,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x99,0x02,0x00,0x00, +0x94,0x02,0x00,0x00,0x95,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x94,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x9b,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x9b,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xe5,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, +0x94,0x02,0x00,0x00,0xd3,0x02,0x00,0x00,0x9e,0x02,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0xa1,0x02,0x00,0x00, +0xe5,0x02,0x00,0x00,0x62,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0x9d,0x02,0x00,0x00,0x9e,0x02,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0xa1,0x02,0x00,0x00,0x9c,0x02,0x00,0x00, +0x9d,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x9c,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa4,0x02,0x00,0x00, +0x89,0x02,0x00,0x00,0xe5,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc2,0x00,0x00,0x00,0xa7,0x02,0x00,0x00,0xa4,0x02,0x00,0x00, +0x36,0x00,0x00,0x00,0xf7,0x00,0x03,0x00,0xa9,0x02,0x00,0x00, +0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xa7,0x02,0x00,0x00, +0xa8,0x02,0x00,0x00,0xa9,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0xa8,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xac,0x02,0x00,0x00,0x91,0x02,0x00,0x00,0xe3,0x02,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0xad,0x02,0x00,0x00, +0x12,0x00,0x00,0x00,0xd0,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xae,0x02,0x00,0x00,0xad,0x02,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0xaf,0x02,0x00,0x00, +0xac,0x02,0x00,0x00,0xae,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0xa9,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0xa9,0x02,0x00,0x00, +0xf5,0x00,0x07,0x00,0xc2,0x00,0x00,0x00,0xb0,0x02,0x00,0x00, +0xa7,0x02,0x00,0x00,0x9c,0x02,0x00,0x00,0xaf,0x02,0x00,0x00, +0xa8,0x02,0x00,0x00,0xf7,0x00,0x03,0x00,0xb2,0x02,0x00,0x00, +0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xb0,0x02,0x00,0x00, +0xb1,0x02,0x00,0x00,0xb2,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0xb1,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xba,0x02,0x00,0x00,0x91,0x02,0x00,0x00,0xe3,0x02,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0xbc,0x02,0x00,0x00, +0x12,0x00,0x00,0x00,0xbb,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xbd,0x02,0x00,0x00,0xbc,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xbe,0x02,0x00,0x00, +0xba,0x02,0x00,0x00,0xbd,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xbf,0x02,0x00,0x00,0x71,0x02,0x00,0x00, +0xbe,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xc1,0x02,0x00,0x00,0xbf,0x02,0x00,0x00,0x89,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xc3,0x02,0x00,0x00, +0xc1,0x02,0x00,0x00,0xe5,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xc5,0x02,0x00,0x00,0xe0,0x02,0x00,0x00, +0xbc,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xc7,0x02,0x00,0x00,0xc5,0x02,0x00,0x00,0xe3,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xc9,0x02,0x00,0x00, +0xc7,0x02,0x00,0x00,0xc8,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xcb,0x02,0x00,0x00,0xe1,0x02,0x00,0x00, +0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xcc,0x02,0x00,0x00,0xc9,0x02,0x00,0x00,0xcb,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xce,0x02,0x00,0x00, +0xcc,0x02,0x00,0x00,0xe5,0x02,0x00,0x00,0x41,0x00,0x05,0x00, +0xcd,0x00,0x00,0x00,0xcf,0x02,0x00,0x00,0xca,0x00,0x00,0x00, +0xce,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00, +0xd0,0x02,0x00,0x00,0xcf,0x02,0x00,0x00,0x41,0x00,0x06,0x00, +0x88,0x01,0x00,0x00,0xd1,0x02,0x00,0x00,0xb6,0x02,0x00,0x00, +0x34,0x00,0x00,0x00,0xc3,0x02,0x00,0x00,0x3e,0x00,0x03,0x00, +0xd1,0x02,0x00,0x00,0xd0,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0xb2,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0xb2,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0x9e,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x9e,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xd3,0x02,0x00,0x00,0xe5,0x02,0x00,0x00,0xd0,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x9b,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x9d,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x96,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x96,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xd5,0x02,0x00,0x00,0xe3,0x02,0x00,0x00, +0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x93,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x95,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0x7e,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x7e,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xd7,0x02,0x00,0x00, +0xe1,0x02,0x00,0x00,0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x7b,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x7d,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0x76,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x76,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xd9,0x02,0x00,0x00,0xe0,0x02,0x00,0x00,0xd0,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x73,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x75,0x02,0x00,0x00,0xfd,0x00,0x01,0x00,0x38,0x00,0x01,0x00, + +}; +const uint64_t matmul_q5_1_f32_aligned_fp32_len = 10800; + +unsigned char matmul_q5_1_f32_fp32_data[] = { +0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, +0x10,0x03,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, +0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x51,0x11,0x00,0x00, +0x11,0x00,0x02,0x00,0x60,0x11,0x00,0x00,0x0b,0x00,0x06,0x00, +0x01,0x00,0x00,0x00,0x47,0x4c,0x53,0x4c,0x2e,0x73,0x74,0x64, +0x2e,0x34,0x35,0x30,0x00,0x00,0x00,0x00,0x0e,0x00,0x03,0x00, +0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x0f,0x00,0x0f,0x00, +0x05,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x6d,0x61,0x69,0x6e, +0x00,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0x3d,0x00,0x00,0x00,0x4c,0x00,0x00,0x00,0x07,0x01,0x00,0x00, +0x4b,0x01,0x00,0x00,0x7c,0x01,0x00,0x00,0x87,0x01,0x00,0x00, +0x6d,0x02,0x00,0x00,0xb6,0x02,0x00,0x00,0x10,0x00,0x06,0x00, +0x04,0x00,0x00,0x00,0x11,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x0b,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x1c,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x10,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x08,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x03,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x10,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x10,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, +0x05,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x14,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x10,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x1c,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, +0x08,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x09,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x24,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x10,0x00,0x00,0x00,0x0a,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x28,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, +0x0b,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x2c,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x30,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x10,0x00,0x00,0x00,0x0d,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x34,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x10,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x37,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x3d,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x1a,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x4c,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, +0x1b,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x4f,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x53,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x60,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x62,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x6c,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x03,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0xa6,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xb8,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0xbb,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x08,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x02,0x01,0x00,0x00,0x06,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x03,0x01,0x00,0x00, +0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x03,0x01,0x00,0x00,0x01,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x03,0x01,0x00,0x00,0x02,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x03,0x01,0x00,0x00, +0x03,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x08,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x04,0x01,0x00,0x00,0x06,0x00,0x00,0x00, +0x18,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0x05,0x01,0x00,0x00, +0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x05,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x05,0x01,0x00,0x00, +0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x07,0x01,0x00,0x00, +0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x07,0x01,0x00,0x00,0x21,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x56,0x01,0x00,0x00,0x01,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x57,0x01,0x00,0x00, +0x0b,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x84,0x01,0x00,0x00,0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x48,0x00,0x04,0x00,0x85,0x01,0x00,0x00,0x00,0x00,0x00,0x00, +0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x85,0x01,0x00,0x00, +0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x03,0x00,0x85,0x01,0x00,0x00,0x02,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x87,0x01,0x00,0x00,0x22,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x87,0x01,0x00,0x00, +0x21,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x6d,0x02,0x00,0x00,0x0b,0x00,0x00,0x00,0x18,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0xb3,0x02,0x00,0x00,0x06,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0xb4,0x02,0x00,0x00, +0x00,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0xb4,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0xb4,0x02,0x00,0x00, +0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xb6,0x02,0x00,0x00, +0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0xb6,0x02,0x00,0x00,0x21,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x13,0x00,0x02,0x00,0x02,0x00,0x00,0x00,0x21,0x00,0x03,0x00, +0x03,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x15,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x17,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x03,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x0a,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0x0a,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x0d,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x1e,0x00,0x10,0x00, +0x10,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x11,0x00,0x00,0x00,0x09,0x00,0x00,0x00, +0x10,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x11,0x00,0x00,0x00, +0x12,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x15,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x14,0x00,0x00,0x00, +0x08,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x15,0x00,0x00,0x00, +0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x0a,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x27,0x00,0x00,0x00, +0x09,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, +0x2d,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x37,0x00,0x00,0x00, +0x40,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x39,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0x0a,0x00,0x00,0x00,0x3d,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x3e,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00, +0x4c,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x53,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x54,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x37,0x00,0x00,0x00, +0x53,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x58,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x37,0x00,0x00,0x00, +0x53,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x60,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x61,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0x53,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x32,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x63,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0x61,0x00,0x00,0x00,0x62,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x67,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0x61,0x00,0x00,0x00,0x62,0x00,0x00,0x00, +0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, +0x10,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x6d,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x72,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x77,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, +0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x7c,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, +0x39,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, +0x80,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0x85,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x90,0x00,0x00,0x00, +0x0b,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, +0x96,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0xa1,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xa6,0x00,0x00,0x00, +0x40,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, +0xa8,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xb7,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x32,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xb8,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xb9,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0xb8,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xba,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x62,0x00,0x00,0x00, +0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xbb,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xbc,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0xba,0x00,0x00,0x00, +0xbb,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xbd,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0xbc,0x00,0x00,0x00, +0x60,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xbe,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0xb9,0x00,0x00,0x00, +0xbd,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xbf,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0xb7,0x00,0x00,0x00, +0xbe,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xc0,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0xbf,0x00,0x00,0x00, +0xbb,0x00,0x00,0x00,0x14,0x00,0x02,0x00,0xc1,0x00,0x00,0x00, +0x16,0x00,0x03,0x00,0xc3,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xc4,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xc5,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0xc4,0x00,0x00,0x00,0xbe,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xc6,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0xc5,0x00,0x00,0x00,0xbb,0x00,0x00,0x00, +0x1c,0x00,0x04,0x00,0xc7,0x00,0x00,0x00,0xc3,0x00,0x00,0x00, +0xc6,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0xc8,0x00,0x00,0x00, +0x07,0x00,0x00,0x00,0xc7,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0xc3,0x00,0x00,0x00,0xcb,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0xcc,0x00,0x00,0x00,0x07,0x00,0x00,0x00, +0xc3,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, +0xcf,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xf3,0x00,0x00,0x00,0x80,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xf9,0x00,0x00,0x00,0x10,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xfd,0x00,0x00,0x00, +0x0f,0x00,0x00,0x00,0x16,0x00,0x03,0x00,0x00,0x01,0x00,0x00, +0x10,0x00,0x00,0x00,0x15,0x00,0x04,0x00,0x01,0x01,0x00,0x00, +0x08,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x1c,0x00,0x04,0x00, +0x02,0x01,0x00,0x00,0x01,0x01,0x00,0x00,0xf9,0x00,0x00,0x00, +0x1e,0x00,0x06,0x00,0x03,0x01,0x00,0x00,0x00,0x01,0x00,0x00, +0x00,0x01,0x00,0x00,0x06,0x00,0x00,0x00,0x02,0x01,0x00,0x00, +0x1d,0x00,0x03,0x00,0x04,0x01,0x00,0x00,0x03,0x01,0x00,0x00, +0x1e,0x00,0x03,0x00,0x05,0x01,0x00,0x00,0x04,0x01,0x00,0x00, +0x20,0x00,0x04,0x00,0x06,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, +0x05,0x01,0x00,0x00,0x3b,0x00,0x04,0x00,0x06,0x01,0x00,0x00, +0x07,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x09,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,0x01,0x00,0x00, +0x20,0x00,0x04,0x00,0x14,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x22,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x2b,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x01,0x01,0x00,0x00, +0x17,0x00,0x04,0x00,0x2f,0x01,0x00,0x00,0xc3,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x47,0x01,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, +0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x48,0x01,0x00,0x00,0x84,0x00,0x00,0x00,0x37,0x00,0x00,0x00, +0x47,0x01,0x00,0x00,0x1c,0x00,0x04,0x00,0x49,0x01,0x00,0x00, +0xc3,0x00,0x00,0x00,0x48,0x01,0x00,0x00,0x20,0x00,0x04,0x00, +0x4a,0x01,0x00,0x00,0x04,0x00,0x00,0x00,0x49,0x01,0x00,0x00, +0x3b,0x00,0x04,0x00,0x4a,0x01,0x00,0x00,0x4b,0x01,0x00,0x00, +0x04,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x4f,0x01,0x00,0x00, +0x04,0x00,0x00,0x00,0xc3,0x00,0x00,0x00,0x32,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x56,0x01,0x00,0x00,0x01,0x00,0x00,0x00, +0x33,0x00,0x06,0x00,0x09,0x00,0x00,0x00,0x57,0x01,0x00,0x00, +0x56,0x01,0x00,0x00,0x39,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x58,0x01,0x00,0x00, +0x51,0x00,0x00,0x00,0x57,0x01,0x00,0x00,0x00,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x59,0x01,0x00,0x00, +0x84,0x00,0x00,0x00,0x58,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x5a,0x01,0x00,0x00, +0x86,0x00,0x00,0x00,0x59,0x01,0x00,0x00,0x6c,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x78,0x01,0x00,0x00, +0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x79,0x01,0x00,0x00, +0x84,0x00,0x00,0x00,0xa6,0x00,0x00,0x00,0x78,0x01,0x00,0x00, +0x1c,0x00,0x04,0x00,0x7a,0x01,0x00,0x00,0xc3,0x00,0x00,0x00, +0x79,0x01,0x00,0x00,0x20,0x00,0x04,0x00,0x7b,0x01,0x00,0x00, +0x04,0x00,0x00,0x00,0x7a,0x01,0x00,0x00,0x3b,0x00,0x04,0x00, +0x7b,0x01,0x00,0x00,0x7c,0x01,0x00,0x00,0x04,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x80,0x01,0x00,0x00, +0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0x1d,0x00,0x03,0x00,0x84,0x01,0x00,0x00,0xc3,0x00,0x00,0x00, +0x1e,0x00,0x03,0x00,0x85,0x01,0x00,0x00,0x84,0x01,0x00,0x00, +0x20,0x00,0x04,0x00,0x86,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, +0x85,0x01,0x00,0x00,0x3b,0x00,0x04,0x00,0x86,0x01,0x00,0x00, +0x87,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x92,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0xc3,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x9a,0x01,0x00,0x00, +0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x9f,0x01,0x00,0x00, +0x51,0x00,0x00,0x00,0x57,0x01,0x00,0x00,0x00,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xa0,0x01,0x00,0x00, +0x84,0x00,0x00,0x00,0x9f,0x01,0x00,0x00,0x39,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xa1,0x01,0x00,0x00, +0x86,0x00,0x00,0x00,0xa0,0x01,0x00,0x00,0x6c,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xa4,0x01,0x00,0x00, +0x08,0x01,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xa5,0x01,0x00,0x00,0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xa8,0x01,0x00,0x00,0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, +0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xc3,0x01,0x00,0x00,0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00, +0x62,0x00,0x00,0x00,0x1c,0x00,0x04,0x00,0xc4,0x01,0x00,0x00, +0xc3,0x00,0x00,0x00,0xc3,0x01,0x00,0x00,0x20,0x00,0x04,0x00, +0xc5,0x01,0x00,0x00,0x07,0x00,0x00,0x00,0xc4,0x01,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xd5,0x01,0x00,0x00, +0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xf0,0x01,0x00,0x00, +0x84,0x00,0x00,0x00,0xbe,0x00,0x00,0x00,0xbb,0x00,0x00,0x00, +0x1c,0x00,0x04,0x00,0xf1,0x01,0x00,0x00,0xc3,0x00,0x00,0x00, +0xf0,0x01,0x00,0x00,0x20,0x00,0x04,0x00,0xf2,0x01,0x00,0x00, +0x07,0x00,0x00,0x00,0xf1,0x01,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xfb,0x01,0x00,0x00,0x86,0x00,0x00,0x00, +0xb8,0x00,0x00,0x00,0xbe,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x03,0x02,0x00,0x00,0x80,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x32,0x02,0x00,0x00,0x84,0x00,0x00,0x00, +0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0x65,0x02,0x00,0x00,0x0d,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00,0x6d,0x02,0x00,0x00, +0x01,0x00,0x00,0x00,0x1d,0x00,0x03,0x00,0xb3,0x02,0x00,0x00, +0xc3,0x00,0x00,0x00,0x1e,0x00,0x03,0x00,0xb4,0x02,0x00,0x00, +0xb3,0x02,0x00,0x00,0x20,0x00,0x04,0x00,0xb5,0x02,0x00,0x00, +0x0c,0x00,0x00,0x00,0xb4,0x02,0x00,0x00,0x3b,0x00,0x04,0x00, +0xb5,0x02,0x00,0x00,0xb6,0x02,0x00,0x00,0x0c,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0xbb,0x02,0x00,0x00, +0x05,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xc8,0x02,0x00,0x00,0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00, +0x62,0x00,0x00,0x00,0x36,0x00,0x05,0x00,0x02,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x03,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0x05,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0xc8,0x00,0x00,0x00,0xc9,0x00,0x00,0x00,0x07,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0xc5,0x01,0x00,0x00,0xc6,0x01,0x00,0x00, +0x07,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0xf2,0x01,0x00,0x00, +0xf3,0x01,0x00,0x00,0x07,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x0d,0x00,0x00,0x00,0x0e,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x0f,0x00,0x00,0x00,0x0e,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x15,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0x14,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x17,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, +0x17,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x1e,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x17,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x22,0x00,0x00,0x00, +0x12,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x22,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x24,0x00,0x00,0x00, +0x18,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x15,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0x27,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x29,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x2a,0x00,0x00,0x00,0x1e,0x00,0x00,0x00, +0x29,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, +0x2e,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x2d,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x2f,0x00,0x00,0x00, +0x2e,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x30,0x00,0x00,0x00,0x24,0x00,0x00,0x00,0x2f,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x32,0x00,0x00,0x00, +0x30,0x00,0x00,0x00,0x2a,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x15,0x00,0x00,0x00,0x35,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0x34,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x36,0x00,0x00,0x00,0x35,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x38,0x00,0x00,0x00,0x36,0x00,0x00,0x00, +0x37,0x00,0x00,0x00,0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x3a,0x00,0x00,0x00,0x38,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3b,0x00,0x00,0x00, +0x3a,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x0d,0x00,0x00,0x00,0x3f,0x00,0x00,0x00,0x3d,0x00,0x00,0x00, +0x3e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x40,0x00,0x00,0x00,0x3f,0x00,0x00,0x00,0x89,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x42,0x00,0x00,0x00,0x40,0x00,0x00,0x00, +0x3b,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x47,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x3b,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00,0x49,0x00,0x00,0x00, +0x3d,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x4a,0x00,0x00,0x00,0x49,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00,0x4d,0x00,0x00,0x00, +0x4c,0x00,0x00,0x00,0x3e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x4d,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x50,0x00,0x00,0x00, +0x4e,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x89,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x55,0x00,0x00,0x00,0x50,0x00,0x00,0x00, +0x54,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x59,0x00,0x00,0x00,0x50,0x00,0x00,0x00,0x58,0x00,0x00,0x00, +0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x5d,0x00,0x00,0x00, +0x4e,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x89,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x64,0x00,0x00,0x00,0x5d,0x00,0x00,0x00, +0x63,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x68,0x00,0x00,0x00,0x5d,0x00,0x00,0x00,0x67,0x00,0x00,0x00, +0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x6e,0x00,0x00,0x00, +0x4e,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x73,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, +0x72,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x78,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x77,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x7d,0x00,0x00,0x00, +0x4e,0x00,0x00,0x00,0x7c,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x15,0x00,0x00,0x00,0x81,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0x80,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x82,0x00,0x00,0x00,0x81,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x83,0x00,0x00,0x00,0x47,0x00,0x00,0x00, +0x82,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x85,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x87,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x89,0x00,0x00,0x00,0x47,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8c,0x00,0x00,0x00, +0x89,0x00,0x00,0x00,0x82,0x00,0x00,0x00,0x0c,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0x8d,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x26,0x00,0x00,0x00,0x87,0x00,0x00,0x00,0x8c,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x91,0x00,0x00,0x00, +0x12,0x00,0x00,0x00,0x90,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x92,0x00,0x00,0x00,0x91,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x93,0x00,0x00,0x00, +0x32,0x00,0x00,0x00,0x92,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x95,0x00,0x00,0x00,0x42,0x00,0x00,0x00, +0x37,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, +0x97,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x96,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x98,0x00,0x00,0x00, +0x97,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x99,0x00,0x00,0x00,0x95,0x00,0x00,0x00,0x98,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9a,0x00,0x00,0x00, +0x93,0x00,0x00,0x00,0x99,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x9c,0x00,0x00,0x00,0x9a,0x00,0x00,0x00, +0x83,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x9d,0x00,0x00,0x00,0x9c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0xa2,0x00,0x00,0x00, +0x12,0x00,0x00,0x00,0xa1,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xa3,0x00,0x00,0x00,0xa2,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa4,0x00,0x00,0x00, +0x0f,0x00,0x00,0x00,0xa3,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xa7,0x00,0x00,0x00,0x4a,0x00,0x00,0x00, +0xa6,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, +0xa9,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0xa8,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xaa,0x00,0x00,0x00, +0xa9,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xab,0x00,0x00,0x00,0xa7,0x00,0x00,0x00,0xaa,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xac,0x00,0x00,0x00, +0xa4,0x00,0x00,0x00,0xab,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xae,0x00,0x00,0x00,0xac,0x00,0x00,0x00, +0x83,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xaf,0x00,0x00,0x00,0xae,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xb1,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xb1,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xde,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0x05,0x00,0x00,0x00, +0xd0,0x00,0x00,0x00,0xb2,0x00,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc1,0x00,0x00,0x00,0xc2,0x00,0x00,0x00,0xde,0x02,0x00,0x00, +0xc0,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xb3,0x00,0x00,0x00, +0xb2,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0xc2,0x00,0x00,0x00,0xb2,0x00,0x00,0x00,0xb3,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xb2,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0xcc,0x00,0x00,0x00,0xcd,0x00,0x00,0x00,0xc9,0x00,0x00,0x00, +0xde,0x02,0x00,0x00,0x3e,0x00,0x03,0x00,0xcd,0x00,0x00,0x00, +0xcb,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xd0,0x00,0x00,0x00,0xde,0x02,0x00,0x00,0xcf,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xb1,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xb3,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xd3,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xd3,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xf7,0x02,0x00,0x00,0xaf,0x00,0x00,0x00, +0xb3,0x00,0x00,0x00,0xaa,0x01,0x00,0x00,0xd6,0x00,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xf3,0x02,0x00,0x00, +0x9d,0x00,0x00,0x00,0xb3,0x00,0x00,0x00,0xa7,0x01,0x00,0x00, +0xd6,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xdf,0x02,0x00,0x00,0x83,0x00,0x00,0x00,0xb3,0x00,0x00,0x00, +0x55,0x02,0x00,0x00,0xd6,0x00,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc1,0x00,0x00,0x00,0xda,0x00,0x00,0x00,0xdf,0x02,0x00,0x00, +0x8d,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xd5,0x00,0x00,0x00, +0xd6,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0xda,0x00,0x00,0x00,0xd4,0x00,0x00,0x00,0xd5,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xd4,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xdc,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xdc,0x00,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xef,0x02,0x00,0x00, +0x3e,0x00,0x00,0x00,0xd4,0x00,0x00,0x00,0x5c,0x01,0x00,0x00, +0xdd,0x00,0x00,0x00,0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00, +0xe2,0x00,0x00,0x00,0xef,0x02,0x00,0x00,0x37,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0xde,0x00,0x00,0x00,0xdd,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xe2,0x00,0x00,0x00, +0xdd,0x00,0x00,0x00,0xde,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xdd,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xe7,0x00,0x00,0x00,0x73,0x00,0x00,0x00,0xef,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xea,0x00,0x00,0x00, +0xe7,0x00,0x00,0x00,0x98,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xeb,0x00,0x00,0x00,0xea,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xec,0x00,0x00,0x00,0xf3,0x02,0x00,0x00,0xeb,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xee,0x00,0x00,0x00, +0xec,0x00,0x00,0x00,0x6e,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xf4,0x00,0x00,0x00,0xe7,0x00,0x00,0x00, +0xf3,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xf6,0x00,0x00,0x00,0xf4,0x00,0x00,0x00,0x6e,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xfa,0x00,0x00,0x00, +0xee,0x00,0x00,0x00,0xf9,0x00,0x00,0x00,0xc7,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xfe,0x00,0x00,0x00,0xee,0x00,0x00,0x00, +0xfd,0x00,0x00,0x00,0x41,0x00,0x07,0x00,0x09,0x01,0x00,0x00, +0x0a,0x01,0x00,0x00,0x07,0x01,0x00,0x00,0x34,0x00,0x00,0x00, +0xfa,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x00,0x01,0x00,0x00,0x0b,0x01,0x00,0x00,0x0a,0x01,0x00,0x00, +0x73,0x00,0x04,0x00,0xc3,0x00,0x00,0x00,0x0c,0x01,0x00,0x00, +0x0b,0x01,0x00,0x00,0x41,0x00,0x07,0x00,0x09,0x01,0x00,0x00, +0x0f,0x01,0x00,0x00,0x07,0x01,0x00,0x00,0x34,0x00,0x00,0x00, +0xfa,0x00,0x00,0x00,0xcf,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x00,0x01,0x00,0x00,0x10,0x01,0x00,0x00,0x0f,0x01,0x00,0x00, +0x73,0x00,0x04,0x00,0xc3,0x00,0x00,0x00,0x11,0x01,0x00,0x00, +0x10,0x01,0x00,0x00,0x41,0x00,0x07,0x00,0x14,0x01,0x00,0x00, +0x15,0x01,0x00,0x00,0x07,0x01,0x00,0x00,0x34,0x00,0x00,0x00, +0xfa,0x00,0x00,0x00,0x85,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x16,0x01,0x00,0x00,0x15,0x01,0x00,0x00, +0xc2,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x1c,0x01,0x00,0x00, +0x16,0x01,0x00,0x00,0xfe,0x00,0x00,0x00,0xc4,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x1d,0x01,0x00,0x00,0x1c,0x01,0x00,0x00, +0xa8,0x00,0x00,0x00,0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x1e,0x01,0x00,0x00,0x1d,0x01,0x00,0x00,0xf9,0x00,0x00,0x00, +0x7c,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x1f,0x01,0x00,0x00, +0x1e,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x23,0x01,0x00,0x00,0xfe,0x00,0x00,0x00,0x22,0x01,0x00,0x00, +0xc2,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x24,0x01,0x00,0x00, +0x16,0x01,0x00,0x00,0x23,0x01,0x00,0x00,0xc7,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x25,0x01,0x00,0x00,0x24,0x01,0x00,0x00, +0xf9,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x13,0x00,0x00,0x00, +0x26,0x01,0x00,0x00,0x25,0x01,0x00,0x00,0x41,0x00,0x08,0x00, +0x2b,0x01,0x00,0x00,0x2c,0x01,0x00,0x00,0x07,0x01,0x00,0x00, +0x34,0x00,0x00,0x00,0xfa,0x00,0x00,0x00,0x96,0x00,0x00,0x00, +0xfe,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x01,0x01,0x00,0x00, +0x2d,0x01,0x00,0x00,0x2c,0x01,0x00,0x00,0x71,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x2e,0x01,0x00,0x00,0x2d,0x01,0x00,0x00, +0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x33,0x01,0x00,0x00, +0x2e,0x01,0x00,0x00,0xfd,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x37,0x01,0x00,0x00,0x1f,0x01,0x00,0x00, +0xc5,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x38,0x01,0x00,0x00, +0x33,0x01,0x00,0x00,0x37,0x01,0x00,0x00,0x70,0x00,0x04,0x00, +0xc3,0x00,0x00,0x00,0x39,0x01,0x00,0x00,0x38,0x01,0x00,0x00, +0xc2,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3b,0x01,0x00,0x00, +0x2e,0x01,0x00,0x00,0xa8,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x3e,0x01,0x00,0x00,0x26,0x01,0x00,0x00, +0xc5,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3f,0x01,0x00,0x00, +0x3b,0x01,0x00,0x00,0x3e,0x01,0x00,0x00,0x70,0x00,0x04,0x00, +0xc3,0x00,0x00,0x00,0x40,0x01,0x00,0x00,0x3f,0x01,0x00,0x00, +0x50,0x00,0x05,0x00,0x2f,0x01,0x00,0x00,0x41,0x01,0x00,0x00, +0x39,0x01,0x00,0x00,0x40,0x01,0x00,0x00,0x8e,0x00,0x05,0x00, +0x2f,0x01,0x00,0x00,0x43,0x01,0x00,0x00,0x41,0x01,0x00,0x00, +0x0c,0x01,0x00,0x00,0x50,0x00,0x05,0x00,0x2f,0x01,0x00,0x00, +0x45,0x01,0x00,0x00,0x11,0x01,0x00,0x00,0x11,0x01,0x00,0x00, +0x81,0x00,0x05,0x00,0x2f,0x01,0x00,0x00,0x46,0x01,0x00,0x00, +0x43,0x01,0x00,0x00,0x45,0x01,0x00,0x00,0x51,0x00,0x05,0x00, +0xc3,0x00,0x00,0x00,0x4e,0x01,0x00,0x00,0x46,0x01,0x00,0x00, +0x00,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x4f,0x01,0x00,0x00, +0x50,0x01,0x00,0x00,0x4b,0x01,0x00,0x00,0xf6,0x00,0x00,0x00, +0x3e,0x00,0x03,0x00,0x50,0x01,0x00,0x00,0x4e,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x52,0x01,0x00,0x00, +0xf6,0x00,0x00,0x00,0xf9,0x00,0x00,0x00,0x51,0x00,0x05,0x00, +0xc3,0x00,0x00,0x00,0x54,0x01,0x00,0x00,0x46,0x01,0x00,0x00, +0x01,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x4f,0x01,0x00,0x00, +0x55,0x01,0x00,0x00,0x4b,0x01,0x00,0x00,0x52,0x01,0x00,0x00, +0x3e,0x00,0x03,0x00,0x55,0x01,0x00,0x00,0x54,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x5c,0x01,0x00,0x00, +0xef,0x02,0x00,0x00,0x5a,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0xdc,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xde,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x5e,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x5e,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xf0,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0xde,0x00,0x00,0x00, +0xa3,0x01,0x00,0x00,0x61,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc1,0x00,0x00,0x00,0x64,0x01,0x00,0x00,0xf0,0x02,0x00,0x00, +0xa6,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x60,0x01,0x00,0x00, +0x61,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x64,0x01,0x00,0x00,0x5f,0x01,0x00,0x00,0x60,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x5f,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x68,0x01,0x00,0x00,0xa7,0x00,0x00,0x00, +0x7d,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x6a,0x01,0x00,0x00,0x68,0x01,0x00,0x00,0xf0,0x02,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x6b,0x01,0x00,0x00, +0x12,0x00,0x00,0x00,0xcf,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x6c,0x01,0x00,0x00,0x6b,0x01,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00,0x6d,0x01,0x00,0x00, +0x6a,0x01,0x00,0x00,0x6c,0x01,0x00,0x00,0xf7,0x00,0x03,0x00, +0x6f,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x6d,0x01,0x00,0x00,0x6e,0x01,0x00,0x00,0x6f,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x6e,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x72,0x01,0x00,0x00,0xdf,0x02,0x00,0x00, +0x78,0x00,0x00,0x00,0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00, +0x74,0x01,0x00,0x00,0x72,0x01,0x00,0x00,0x8d,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x6f,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x6f,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0xc1,0x00,0x00,0x00, +0x75,0x01,0x00,0x00,0x6d,0x01,0x00,0x00,0x5f,0x01,0x00,0x00, +0x74,0x01,0x00,0x00,0x6e,0x01,0x00,0x00,0xf7,0x00,0x03,0x00, +0x77,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x75,0x01,0x00,0x00,0x76,0x01,0x00,0x00,0x96,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x76,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x7f,0x01,0x00,0x00,0x7d,0x00,0x00,0x00, +0xf0,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x81,0x01,0x00,0x00,0x7f,0x01,0x00,0x00,0x80,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x83,0x01,0x00,0x00, +0x81,0x01,0x00,0x00,0x78,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x8e,0x01,0x00,0x00,0x7f,0x01,0x00,0x00, +0xaa,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x8f,0x01,0x00,0x00,0xf7,0x02,0x00,0x00,0x8e,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x91,0x01,0x00,0x00, +0x8f,0x01,0x00,0x00,0x78,0x00,0x00,0x00,0x41,0x00,0x06,0x00, +0x92,0x01,0x00,0x00,0x93,0x01,0x00,0x00,0x87,0x01,0x00,0x00, +0x34,0x00,0x00,0x00,0x91,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, +0xc3,0x00,0x00,0x00,0x94,0x01,0x00,0x00,0x93,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0x4f,0x01,0x00,0x00,0x95,0x01,0x00,0x00, +0x7c,0x01,0x00,0x00,0x83,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, +0x95,0x01,0x00,0x00,0x94,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0x77,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x96,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x99,0x01,0x00,0x00, +0x7d,0x00,0x00,0x00,0xf0,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x9b,0x01,0x00,0x00,0x99,0x01,0x00,0x00, +0x9a,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x9d,0x01,0x00,0x00,0x9b,0x01,0x00,0x00,0x78,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x4f,0x01,0x00,0x00,0x9e,0x01,0x00,0x00, +0x7c,0x01,0x00,0x00,0x9d,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, +0x9e,0x01,0x00,0x00,0xcb,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x77,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x77,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0x61,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x61,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xa3,0x01,0x00,0x00,0xf0,0x02,0x00,0x00,0xa1,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0x5e,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x60,0x01,0x00,0x00,0xe0,0x00,0x04,0x00,0x0c,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0xa4,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xa7,0x01,0x00,0x00,0xf3,0x02,0x00,0x00, +0xa5,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xaa,0x01,0x00,0x00,0xf7,0x02,0x00,0x00,0xa8,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0xac,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xac,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xf9,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0x60,0x01,0x00,0x00, +0x53,0x02,0x00,0x00,0xaf,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc1,0x00,0x00,0x00,0xb2,0x01,0x00,0x00,0xf9,0x02,0x00,0x00, +0x6c,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xae,0x01,0x00,0x00, +0xaf,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0xb2,0x01,0x00,0x00,0xad,0x01,0x00,0x00,0xae,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xad,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0xb4,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xb4,0x01,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xfd,0x02,0x00,0x00, +0x3e,0x00,0x00,0x00,0xad,0x01,0x00,0x00,0xdf,0x01,0x00,0x00, +0xb7,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00, +0xba,0x01,0x00,0x00,0xfd,0x02,0x00,0x00,0x60,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0xb6,0x01,0x00,0x00,0xb7,0x01,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xba,0x01,0x00,0x00, +0xb5,0x01,0x00,0x00,0xb6,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xb5,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xbc,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xbc,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0x0f,0x03,0x00,0x00,0x3e,0x00,0x00,0x00, +0xb5,0x01,0x00,0x00,0xdd,0x01,0x00,0x00,0xbd,0x01,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00,0xc2,0x01,0x00,0x00, +0x0f,0x03,0x00,0x00,0x62,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0xbe,0x01,0x00,0x00,0xbd,0x01,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0xc2,0x01,0x00,0x00,0xbd,0x01,0x00,0x00, +0xbe,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xbd,0x01,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xc8,0x01,0x00,0x00, +0xfd,0x02,0x00,0x00,0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xca,0x01,0x00,0x00,0xc8,0x01,0x00,0x00, +0x0f,0x03,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xcc,0x01,0x00,0x00,0x55,0x00,0x00,0x00,0x53,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xce,0x01,0x00,0x00, +0xfd,0x02,0x00,0x00,0x61,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xcf,0x01,0x00,0x00,0xcc,0x01,0x00,0x00, +0xce,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xd1,0x01,0x00,0x00,0x64,0x00,0x00,0x00,0x62,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xd2,0x01,0x00,0x00, +0xcf,0x01,0x00,0x00,0xd1,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xd4,0x01,0x00,0x00,0xd2,0x01,0x00,0x00, +0x0f,0x03,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xd6,0x01,0x00,0x00,0xd4,0x01,0x00,0x00,0xd5,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xd8,0x01,0x00,0x00, +0xd6,0x01,0x00,0x00,0xf9,0x02,0x00,0x00,0x41,0x00,0x05,0x00, +0x4f,0x01,0x00,0x00,0xd9,0x01,0x00,0x00,0x4b,0x01,0x00,0x00, +0xd8,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0xc3,0x00,0x00,0x00, +0xda,0x01,0x00,0x00,0xd9,0x01,0x00,0x00,0x41,0x00,0x05,0x00, +0xcc,0x00,0x00,0x00,0xdb,0x01,0x00,0x00,0xc6,0x01,0x00,0x00, +0xca,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0xdb,0x01,0x00,0x00, +0xda,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xdd,0x01,0x00,0x00,0x0f,0x03,0x00,0x00,0xcf,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xbc,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xbe,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xb7,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xb7,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xdf,0x01,0x00,0x00,0xfd,0x02,0x00,0x00, +0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xb4,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xb6,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0xe1,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xe1,0x01,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xfe,0x02,0x00,0x00, +0x3e,0x00,0x00,0x00,0xb6,0x01,0x00,0x00,0x0d,0x02,0x00,0x00, +0xe4,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00, +0xe7,0x01,0x00,0x00,0xfe,0x02,0x00,0x00,0xbe,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0xe3,0x01,0x00,0x00,0xe4,0x01,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xe7,0x01,0x00,0x00, +0xe2,0x01,0x00,0x00,0xe3,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xe2,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xe9,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xe9,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0x0c,0x03,0x00,0x00,0x3e,0x00,0x00,0x00, +0xe2,0x01,0x00,0x00,0x0b,0x02,0x00,0x00,0xea,0x01,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00,0xef,0x01,0x00,0x00, +0x0c,0x03,0x00,0x00,0xbb,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0xeb,0x01,0x00,0x00,0xea,0x01,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0xef,0x01,0x00,0x00,0xea,0x01,0x00,0x00, +0xeb,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xea,0x01,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xf5,0x01,0x00,0x00, +0xfe,0x02,0x00,0x00,0xbb,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xf7,0x01,0x00,0x00,0xf5,0x01,0x00,0x00, +0x0c,0x03,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xf9,0x01,0x00,0x00,0x59,0x00,0x00,0x00,0xb8,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xfc,0x01,0x00,0x00, +0xfe,0x02,0x00,0x00,0xfb,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xfd,0x01,0x00,0x00,0xf9,0x01,0x00,0x00, +0xfc,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xff,0x01,0x00,0x00,0x68,0x00,0x00,0x00,0xbb,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x00,0x02,0x00,0x00, +0xfd,0x01,0x00,0x00,0xff,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x02,0x02,0x00,0x00,0x00,0x02,0x00,0x00, +0x0c,0x03,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x04,0x02,0x00,0x00,0x02,0x02,0x00,0x00,0x03,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x06,0x02,0x00,0x00, +0x04,0x02,0x00,0x00,0xf9,0x02,0x00,0x00,0x41,0x00,0x05,0x00, +0x4f,0x01,0x00,0x00,0x07,0x02,0x00,0x00,0x7c,0x01,0x00,0x00, +0x06,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0xc3,0x00,0x00,0x00, +0x08,0x02,0x00,0x00,0x07,0x02,0x00,0x00,0x41,0x00,0x05,0x00, +0xcc,0x00,0x00,0x00,0x09,0x02,0x00,0x00,0xf3,0x01,0x00,0x00, +0xf7,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x09,0x02,0x00,0x00, +0x08,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x0b,0x02,0x00,0x00,0x0c,0x03,0x00,0x00,0xcf,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xe9,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xeb,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xe4,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xe4,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x0d,0x02,0x00,0x00,0xfe,0x02,0x00,0x00, +0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xe1,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xe3,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0x0f,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x0f,0x02,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xff,0x02,0x00,0x00, +0x3e,0x00,0x00,0x00,0xe3,0x01,0x00,0x00,0x51,0x02,0x00,0x00, +0x12,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00, +0x15,0x02,0x00,0x00,0xff,0x02,0x00,0x00,0xbe,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0x11,0x02,0x00,0x00,0x12,0x02,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x15,0x02,0x00,0x00, +0x10,0x02,0x00,0x00,0x11,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x10,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x17,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x17,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0x03,0x03,0x00,0x00,0x3e,0x00,0x00,0x00, +0x10,0x02,0x00,0x00,0x4f,0x02,0x00,0x00,0x1a,0x02,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00,0x1d,0x02,0x00,0x00, +0x03,0x03,0x00,0x00,0x60,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0x19,0x02,0x00,0x00,0x1a,0x02,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x1d,0x02,0x00,0x00,0x18,0x02,0x00,0x00, +0x19,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x18,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0x1f,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x1f,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0x05,0x03,0x00,0x00,0x3e,0x00,0x00,0x00,0x18,0x02,0x00,0x00, +0x4d,0x02,0x00,0x00,0x22,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc1,0x00,0x00,0x00,0x25,0x02,0x00,0x00,0x05,0x03,0x00,0x00, +0xbb,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x21,0x02,0x00,0x00, +0x22,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x25,0x02,0x00,0x00,0x20,0x02,0x00,0x00,0x21,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x20,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0x27,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x27,0x02,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x07,0x03,0x00,0x00, +0x3e,0x00,0x00,0x00,0x20,0x02,0x00,0x00,0x4b,0x02,0x00,0x00, +0x28,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00, +0x2d,0x02,0x00,0x00,0x07,0x03,0x00,0x00,0x62,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0x29,0x02,0x00,0x00,0x28,0x02,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x2d,0x02,0x00,0x00, +0x28,0x02,0x00,0x00,0x29,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x28,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x2f,0x02,0x00,0x00,0xff,0x02,0x00,0x00,0xbb,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x31,0x02,0x00,0x00, +0x2f,0x02,0x00,0x00,0x05,0x03,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x33,0x02,0x00,0x00,0x31,0x02,0x00,0x00, +0x32,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x35,0x02,0x00,0x00,0x03,0x03,0x00,0x00,0x62,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x36,0x02,0x00,0x00, +0x33,0x02,0x00,0x00,0x35,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x38,0x02,0x00,0x00,0x36,0x02,0x00,0x00, +0x07,0x03,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x3c,0x02,0x00,0x00,0x35,0x02,0x00,0x00,0x07,0x03,0x00,0x00, +0x41,0x00,0x05,0x00,0xcc,0x00,0x00,0x00,0x3d,0x02,0x00,0x00, +0xc6,0x01,0x00,0x00,0x3c,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, +0xc3,0x00,0x00,0x00,0x3e,0x02,0x00,0x00,0x3d,0x02,0x00,0x00, +0x41,0x00,0x05,0x00,0xcc,0x00,0x00,0x00,0x43,0x02,0x00,0x00, +0xf3,0x01,0x00,0x00,0x31,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, +0xc3,0x00,0x00,0x00,0x44,0x02,0x00,0x00,0x43,0x02,0x00,0x00, +0x41,0x00,0x05,0x00,0xcc,0x00,0x00,0x00,0x46,0x02,0x00,0x00, +0xc9,0x00,0x00,0x00,0x38,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, +0xc3,0x00,0x00,0x00,0x47,0x02,0x00,0x00,0x46,0x02,0x00,0x00, +0x0c,0x00,0x08,0x00,0xc3,0x00,0x00,0x00,0x48,0x02,0x00,0x00, +0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x3e,0x02,0x00,0x00, +0x44,0x02,0x00,0x00,0x47,0x02,0x00,0x00,0x3e,0x00,0x03,0x00, +0x46,0x02,0x00,0x00,0x48,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x4b,0x02,0x00,0x00,0x07,0x03,0x00,0x00, +0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x27,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x29,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0x22,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x22,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x4d,0x02,0x00,0x00, +0x05,0x03,0x00,0x00,0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x1f,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x21,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0x1a,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x1a,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x4f,0x02,0x00,0x00,0x03,0x03,0x00,0x00,0xcf,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x17,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x19,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x12,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x12,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x51,0x02,0x00,0x00,0xff,0x02,0x00,0x00, +0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x0f,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x11,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0xaf,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xaf,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x53,0x02,0x00,0x00, +0xf9,0x02,0x00,0x00,0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xac,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xae,0x01,0x00,0x00, +0xe0,0x00,0x04,0x00,0x0c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0xa4,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xd6,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xd6,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x55,0x02,0x00,0x00,0xdf,0x02,0x00,0x00, +0x6c,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xd3,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xd5,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x5a,0x02,0x00,0x00,0x55,0x00,0x00,0x00, +0x53,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x5b,0x02,0x00,0x00,0x95,0x00,0x00,0x00,0x5a,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x60,0x02,0x00,0x00, +0x59,0x00,0x00,0x00,0xb8,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x61,0x02,0x00,0x00,0xa7,0x00,0x00,0x00, +0x60,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, +0x66,0x02,0x00,0x00,0x12,0x00,0x00,0x00,0x65,0x02,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x67,0x02,0x00,0x00, +0x66,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x68,0x02,0x00,0x00,0x0f,0x00,0x00,0x00,0x67,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x6c,0x02,0x00,0x00, +0x47,0x00,0x00,0x00,0x67,0x02,0x00,0x00,0x41,0x00,0x05,0x00, +0x0d,0x00,0x00,0x00,0x6e,0x02,0x00,0x00,0x6d,0x02,0x00,0x00, +0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x6f,0x02,0x00,0x00,0x6e,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x70,0x02,0x00,0x00,0x6c,0x02,0x00,0x00, +0x6f,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x71,0x02,0x00,0x00,0x68,0x02,0x00,0x00,0x70,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0x73,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x73,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xe0,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0xd5,0x00,0x00,0x00, +0xd9,0x02,0x00,0x00,0x76,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc1,0x00,0x00,0x00,0x79,0x02,0x00,0x00,0xe0,0x02,0x00,0x00, +0xbe,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x75,0x02,0x00,0x00, +0x76,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x79,0x02,0x00,0x00,0x74,0x02,0x00,0x00,0x75,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x74,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0x7b,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x7b,0x02,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xe1,0x02,0x00,0x00, +0x3e,0x00,0x00,0x00,0x74,0x02,0x00,0x00,0xd7,0x02,0x00,0x00, +0x7e,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00, +0x81,0x02,0x00,0x00,0xe1,0x02,0x00,0x00,0x60,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0x7d,0x02,0x00,0x00,0x7e,0x02,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x81,0x02,0x00,0x00, +0x7c,0x02,0x00,0x00,0x7d,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x7c,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x85,0x02,0x00,0x00,0xe1,0x02,0x00,0x00,0x61,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x86,0x02,0x00,0x00, +0x5b,0x02,0x00,0x00,0x85,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x88,0x02,0x00,0x00,0x64,0x00,0x00,0x00, +0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x89,0x02,0x00,0x00,0x86,0x02,0x00,0x00,0x88,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8d,0x02,0x00,0x00, +0xe0,0x02,0x00,0x00,0xfb,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x8e,0x02,0x00,0x00,0x61,0x02,0x00,0x00, +0x8d,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x90,0x02,0x00,0x00,0x68,0x00,0x00,0x00,0xbb,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x91,0x02,0x00,0x00, +0x8e,0x02,0x00,0x00,0x90,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0x93,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x93,0x02,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xe3,0x02,0x00,0x00, +0x3e,0x00,0x00,0x00,0x7c,0x02,0x00,0x00,0xd5,0x02,0x00,0x00, +0x96,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00, +0x99,0x02,0x00,0x00,0xe3,0x02,0x00,0x00,0xbb,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0x95,0x02,0x00,0x00,0x96,0x02,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x99,0x02,0x00,0x00, +0x94,0x02,0x00,0x00,0x95,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x94,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x9b,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x9b,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xe5,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, +0x94,0x02,0x00,0x00,0xd3,0x02,0x00,0x00,0x9e,0x02,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00,0xa1,0x02,0x00,0x00, +0xe5,0x02,0x00,0x00,0x62,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0x9d,0x02,0x00,0x00,0x9e,0x02,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0xa1,0x02,0x00,0x00,0x9c,0x02,0x00,0x00, +0x9d,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x9c,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa4,0x02,0x00,0x00, +0x89,0x02,0x00,0x00,0xe5,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc1,0x00,0x00,0x00,0xa7,0x02,0x00,0x00,0xa4,0x02,0x00,0x00, +0x36,0x00,0x00,0x00,0xf7,0x00,0x03,0x00,0xa9,0x02,0x00,0x00, +0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xa7,0x02,0x00,0x00, +0xa8,0x02,0x00,0x00,0xa9,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0xa8,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xac,0x02,0x00,0x00,0x91,0x02,0x00,0x00,0xe3,0x02,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0xad,0x02,0x00,0x00, +0x12,0x00,0x00,0x00,0xcf,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xae,0x02,0x00,0x00,0xad,0x02,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00,0xaf,0x02,0x00,0x00, +0xac,0x02,0x00,0x00,0xae,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0xa9,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0xa9,0x02,0x00,0x00, +0xf5,0x00,0x07,0x00,0xc1,0x00,0x00,0x00,0xb0,0x02,0x00,0x00, +0xa7,0x02,0x00,0x00,0x9c,0x02,0x00,0x00,0xaf,0x02,0x00,0x00, +0xa8,0x02,0x00,0x00,0xf7,0x00,0x03,0x00,0xb2,0x02,0x00,0x00, +0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xb0,0x02,0x00,0x00, +0xb1,0x02,0x00,0x00,0xb2,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0xb1,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xba,0x02,0x00,0x00,0x91,0x02,0x00,0x00,0xe3,0x02,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0xbc,0x02,0x00,0x00, +0x12,0x00,0x00,0x00,0xbb,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xbd,0x02,0x00,0x00,0xbc,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xbe,0x02,0x00,0x00, +0xba,0x02,0x00,0x00,0xbd,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xbf,0x02,0x00,0x00,0x71,0x02,0x00,0x00, +0xbe,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xc1,0x02,0x00,0x00,0xbf,0x02,0x00,0x00,0x89,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xc3,0x02,0x00,0x00, +0xc1,0x02,0x00,0x00,0xe5,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xc5,0x02,0x00,0x00,0xe0,0x02,0x00,0x00, +0xbb,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xc7,0x02,0x00,0x00,0xc5,0x02,0x00,0x00,0xe3,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xc9,0x02,0x00,0x00, +0xc7,0x02,0x00,0x00,0xc8,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xcb,0x02,0x00,0x00,0xe1,0x02,0x00,0x00, +0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xcc,0x02,0x00,0x00,0xc9,0x02,0x00,0x00,0xcb,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xce,0x02,0x00,0x00, +0xcc,0x02,0x00,0x00,0xe5,0x02,0x00,0x00,0x41,0x00,0x05,0x00, +0xcc,0x00,0x00,0x00,0xcf,0x02,0x00,0x00,0xc9,0x00,0x00,0x00, +0xce,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0xc3,0x00,0x00,0x00, +0xd0,0x02,0x00,0x00,0xcf,0x02,0x00,0x00,0x41,0x00,0x06,0x00, +0x92,0x01,0x00,0x00,0xd1,0x02,0x00,0x00,0xb6,0x02,0x00,0x00, +0x34,0x00,0x00,0x00,0xc3,0x02,0x00,0x00,0x3e,0x00,0x03,0x00, +0xd1,0x02,0x00,0x00,0xd0,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0xb2,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0xb2,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0x9e,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x9e,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xd3,0x02,0x00,0x00,0xe5,0x02,0x00,0x00,0xcf,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x9b,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x9d,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x96,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x96,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xd5,0x02,0x00,0x00,0xe3,0x02,0x00,0x00, +0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x93,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x95,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0x7e,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x7e,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xd7,0x02,0x00,0x00, +0xe1,0x02,0x00,0x00,0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x7b,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x7d,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0x76,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x76,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xd9,0x02,0x00,0x00,0xe0,0x02,0x00,0x00,0xcf,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x73,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x75,0x02,0x00,0x00,0xfd,0x00,0x01,0x00,0x38,0x00,0x01,0x00, + +}; +const uint64_t matmul_q5_1_f32_fp32_len = 10836; + +unsigned char matmul_q8_0_f32_data[] = { +0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, +0xf2,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, +0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x09,0x00,0x00,0x00, +0x11,0x00,0x02,0x00,0x51,0x11,0x00,0x00,0x11,0x00,0x02,0x00, +0x60,0x11,0x00,0x00,0x0b,0x00,0x06,0x00,0x01,0x00,0x00,0x00, +0x47,0x4c,0x53,0x4c,0x2e,0x73,0x74,0x64,0x2e,0x34,0x35,0x30, +0x00,0x00,0x00,0x00,0x0e,0x00,0x03,0x00,0x00,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x0f,0x00,0x0f,0x00,0x05,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x6d,0x61,0x69,0x6e,0x00,0x00,0x00,0x00, +0x0b,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x3d,0x00,0x00,0x00, +0x4c,0x00,0x00,0x00,0x0a,0x01,0x00,0x00,0x28,0x01,0x00,0x00, +0x5b,0x01,0x00,0x00,0x66,0x01,0x00,0x00,0x51,0x02,0x00,0x00, +0x9a,0x02,0x00,0x00,0x10,0x00,0x06,0x00,0x04,0x00,0x00,0x00, +0x11,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x0b,0x00,0x00,0x00, +0x0b,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x10,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x10,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x10,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x05,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x10,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, +0x07,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x1c,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x08,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x10,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x24,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, +0x0a,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x28,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x2c,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x10,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x30,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, +0x0d,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x34,0x00,0x00,0x00, +0x47,0x00,0x03,0x00,0x10,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x37,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x3d,0x00,0x00,0x00, +0x0b,0x00,0x00,0x00,0x1a,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x4c,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x1b,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x4f,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x09,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x53,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x60,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x62,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x07,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x6c,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0xa6,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0xb8,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x05,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xbb,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x05,0x01,0x00,0x00,0x06,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x06,0x01,0x00,0x00,0x00,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x06,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x07,0x01,0x00,0x00, +0x06,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x48,0x00,0x04,0x00, +0x08,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x08,0x01,0x00,0x00,0x00,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, +0x08,0x01,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x0a,0x01,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x0a,0x01,0x00,0x00,0x21,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x35,0x01,0x00,0x00, +0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x36,0x01,0x00,0x00,0x0b,0x00,0x00,0x00,0x19,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x63,0x01,0x00,0x00,0x06,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0x64,0x01,0x00,0x00, +0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x64,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x64,0x01,0x00,0x00, +0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x66,0x01,0x00,0x00, +0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x66,0x01,0x00,0x00,0x21,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x51,0x02,0x00,0x00,0x0b,0x00,0x00,0x00, +0x18,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x97,0x02,0x00,0x00, +0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00, +0x98,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x19,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x98,0x02,0x00,0x00,0x00,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, +0x98,0x02,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x9a,0x02,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x9a,0x02,0x00,0x00,0x21,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x13,0x00,0x02,0x00,0x02,0x00,0x00,0x00, +0x21,0x00,0x03,0x00,0x03,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x15,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x17,0x00,0x04,0x00,0x09,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x0a,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x09,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x0d,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x1e,0x00,0x10,0x00,0x10,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x11,0x00,0x00,0x00, +0x09,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0x11,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x09,0x00,0x00,0x00, +0x15,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, +0x14,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x15,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x21,0x00,0x00,0x00, +0x0a,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, +0x27,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0x2d,0x00,0x00,0x00,0x07,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x34,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x37,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00,0x3d,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x3e,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0x0a,0x00,0x00,0x00,0x4c,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x4f,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x53,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x54,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0x37,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x58,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0x37,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x32,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x61,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x60,0x00,0x00,0x00, +0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x62,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x63,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x61,0x00,0x00,0x00, +0x62,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x67,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x61,0x00,0x00,0x00, +0x62,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x72,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x77,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x7c,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0x80,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x85,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, +0x90,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0x96,0x00,0x00,0x00,0x03,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0xa1,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xa6,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0xa8,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xb7,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00, +0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xb8,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xb9,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x53,0x00,0x00,0x00, +0xb8,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xba,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x4f,0x00,0x00,0x00, +0x62,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xbb,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xbc,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0xba,0x00,0x00,0x00,0xbb,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xbd,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0xbc,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xbe,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0xb9,0x00,0x00,0x00,0xbd,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xbf,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0xb7,0x00,0x00,0x00,0xbe,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xc0,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0xbf,0x00,0x00,0x00,0xbb,0x00,0x00,0x00,0x14,0x00,0x02,0x00, +0xc1,0x00,0x00,0x00,0x16,0x00,0x03,0x00,0xc3,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xc4,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00, +0x62,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xc5,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0xc4,0x00,0x00,0x00, +0xbe,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xc6,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0xc5,0x00,0x00,0x00, +0xbb,0x00,0x00,0x00,0x1c,0x00,0x04,0x00,0xc7,0x00,0x00,0x00, +0xc3,0x00,0x00,0x00,0xc6,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0xc8,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0xc7,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0xc3,0x00,0x00,0x00,0xcb,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0xcc,0x00,0x00,0x00, +0x07,0x00,0x00,0x00,0xc3,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0xcf,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xf3,0x00,0x00,0x00, +0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xfa,0x00,0x00,0x00, +0x10,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xfe,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x16,0x00,0x03,0x00, +0x02,0x01,0x00,0x00,0x10,0x00,0x00,0x00,0x15,0x00,0x04,0x00, +0x03,0x01,0x00,0x00,0x08,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x04,0x01,0x00,0x00, +0x20,0x00,0x00,0x00,0x1c,0x00,0x04,0x00,0x05,0x01,0x00,0x00, +0x03,0x01,0x00,0x00,0x04,0x01,0x00,0x00,0x1e,0x00,0x04,0x00, +0x06,0x01,0x00,0x00,0x02,0x01,0x00,0x00,0x05,0x01,0x00,0x00, +0x1d,0x00,0x03,0x00,0x07,0x01,0x00,0x00,0x06,0x01,0x00,0x00, +0x1e,0x00,0x03,0x00,0x08,0x01,0x00,0x00,0x07,0x01,0x00,0x00, +0x20,0x00,0x04,0x00,0x09,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, +0x08,0x01,0x00,0x00,0x3b,0x00,0x04,0x00,0x09,0x01,0x00,0x00, +0x0a,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x0c,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x02,0x01,0x00,0x00, +0x17,0x00,0x04,0x00,0x10,0x01,0x00,0x00,0xc3,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x15,0x01,0x00,0x00, +0x0c,0x00,0x00,0x00,0x03,0x01,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x24,0x01,0x00,0x00,0x80,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x25,0x01,0x00,0x00,0x84,0x00,0x00,0x00, +0x37,0x00,0x00,0x00,0x24,0x01,0x00,0x00,0x1c,0x00,0x04,0x00, +0x26,0x01,0x00,0x00,0x02,0x01,0x00,0x00,0x25,0x01,0x00,0x00, +0x20,0x00,0x04,0x00,0x27,0x01,0x00,0x00,0x04,0x00,0x00,0x00, +0x26,0x01,0x00,0x00,0x3b,0x00,0x04,0x00,0x27,0x01,0x00,0x00, +0x28,0x01,0x00,0x00,0x04,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x2d,0x01,0x00,0x00,0x04,0x00,0x00,0x00,0x02,0x01,0x00,0x00, +0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x35,0x01,0x00,0x00, +0x01,0x00,0x00,0x00,0x33,0x00,0x06,0x00,0x09,0x00,0x00,0x00, +0x36,0x01,0x00,0x00,0x35,0x01,0x00,0x00,0x39,0x00,0x00,0x00, +0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x37,0x01,0x00,0x00,0x51,0x00,0x00,0x00,0x36,0x01,0x00,0x00, +0x00,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x38,0x01,0x00,0x00,0x84,0x00,0x00,0x00,0x37,0x01,0x00,0x00, +0x0c,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x39,0x01,0x00,0x00,0x86,0x00,0x00,0x00,0x38,0x01,0x00,0x00, +0x6c,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x57,0x01,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, +0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x58,0x01,0x00,0x00,0x84,0x00,0x00,0x00,0xa6,0x00,0x00,0x00, +0x57,0x01,0x00,0x00,0x1c,0x00,0x04,0x00,0x59,0x01,0x00,0x00, +0x02,0x01,0x00,0x00,0x58,0x01,0x00,0x00,0x20,0x00,0x04,0x00, +0x5a,0x01,0x00,0x00,0x04,0x00,0x00,0x00,0x59,0x01,0x00,0x00, +0x3b,0x00,0x04,0x00,0x5a,0x01,0x00,0x00,0x5b,0x01,0x00,0x00, +0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x5f,0x01,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, +0x39,0x00,0x00,0x00,0x1d,0x00,0x03,0x00,0x63,0x01,0x00,0x00, +0xc3,0x00,0x00,0x00,0x1e,0x00,0x03,0x00,0x64,0x01,0x00,0x00, +0x63,0x01,0x00,0x00,0x20,0x00,0x04,0x00,0x65,0x01,0x00,0x00, +0x0c,0x00,0x00,0x00,0x64,0x01,0x00,0x00,0x3b,0x00,0x04,0x00, +0x65,0x01,0x00,0x00,0x66,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x71,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, +0xc3,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x7a,0x01,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, +0x39,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x02,0x01,0x00,0x00, +0x7e,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x80,0x01,0x00,0x00,0x51,0x00,0x00,0x00, +0x36,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x81,0x01,0x00,0x00,0x84,0x00,0x00,0x00, +0x80,0x01,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x82,0x01,0x00,0x00,0x86,0x00,0x00,0x00, +0x81,0x01,0x00,0x00,0x6c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x85,0x01,0x00,0x00,0x08,0x01,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x86,0x01,0x00,0x00, +0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x89,0x01,0x00,0x00, +0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xa4,0x01,0x00,0x00, +0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00, +0x1c,0x00,0x04,0x00,0xa5,0x01,0x00,0x00,0x02,0x01,0x00,0x00, +0xa4,0x01,0x00,0x00,0x20,0x00,0x04,0x00,0xa6,0x01,0x00,0x00, +0x07,0x00,0x00,0x00,0xa5,0x01,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xb6,0x01,0x00,0x00,0x80,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0xbc,0x01,0x00,0x00,0x07,0x00,0x00,0x00,0x02,0x01,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xd2,0x01,0x00,0x00, +0x84,0x00,0x00,0x00,0xbe,0x00,0x00,0x00,0xbb,0x00,0x00,0x00, +0x1c,0x00,0x04,0x00,0xd3,0x01,0x00,0x00,0x02,0x01,0x00,0x00, +0xd2,0x01,0x00,0x00,0x20,0x00,0x04,0x00,0xd4,0x01,0x00,0x00, +0x07,0x00,0x00,0x00,0xd3,0x01,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xdd,0x01,0x00,0x00,0x86,0x00,0x00,0x00, +0xb8,0x00,0x00,0x00,0xbe,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xe5,0x01,0x00,0x00,0x80,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x14,0x02,0x00,0x00,0x84,0x00,0x00,0x00, +0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0x49,0x02,0x00,0x00,0x0d,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00,0x51,0x02,0x00,0x00, +0x01,0x00,0x00,0x00,0x1d,0x00,0x03,0x00,0x97,0x02,0x00,0x00, +0xc3,0x00,0x00,0x00,0x1e,0x00,0x03,0x00,0x98,0x02,0x00,0x00, +0x97,0x02,0x00,0x00,0x20,0x00,0x04,0x00,0x99,0x02,0x00,0x00, +0x0c,0x00,0x00,0x00,0x98,0x02,0x00,0x00,0x3b,0x00,0x04,0x00, +0x99,0x02,0x00,0x00,0x9a,0x02,0x00,0x00,0x0c,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x9f,0x02,0x00,0x00, +0x05,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xac,0x02,0x00,0x00,0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00, +0x62,0x00,0x00,0x00,0x36,0x00,0x05,0x00,0x02,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x03,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0x05,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0xc8,0x00,0x00,0x00,0xc9,0x00,0x00,0x00,0x07,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0xa6,0x01,0x00,0x00,0xa7,0x01,0x00,0x00, +0x07,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0xd4,0x01,0x00,0x00, +0xd5,0x01,0x00,0x00,0x07,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x0d,0x00,0x00,0x00,0x0e,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x0f,0x00,0x00,0x00,0x0e,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x15,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0x14,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x17,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, +0x17,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x1e,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x17,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x22,0x00,0x00,0x00, +0x12,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x22,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x24,0x00,0x00,0x00, +0x18,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x15,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0x27,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x29,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x2a,0x00,0x00,0x00,0x1e,0x00,0x00,0x00, +0x29,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, +0x2e,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x2d,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x2f,0x00,0x00,0x00, +0x2e,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x30,0x00,0x00,0x00,0x24,0x00,0x00,0x00,0x2f,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x32,0x00,0x00,0x00, +0x30,0x00,0x00,0x00,0x2a,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x15,0x00,0x00,0x00,0x35,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0x34,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x36,0x00,0x00,0x00,0x35,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x38,0x00,0x00,0x00,0x36,0x00,0x00,0x00, +0x37,0x00,0x00,0x00,0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x3a,0x00,0x00,0x00,0x38,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3b,0x00,0x00,0x00, +0x3a,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x0d,0x00,0x00,0x00,0x3f,0x00,0x00,0x00,0x3d,0x00,0x00,0x00, +0x3e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x40,0x00,0x00,0x00,0x3f,0x00,0x00,0x00,0x89,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x42,0x00,0x00,0x00,0x40,0x00,0x00,0x00, +0x3b,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x47,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x3b,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00,0x49,0x00,0x00,0x00, +0x3d,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x4a,0x00,0x00,0x00,0x49,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00,0x4d,0x00,0x00,0x00, +0x4c,0x00,0x00,0x00,0x3e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x4d,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x50,0x00,0x00,0x00, +0x4e,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x89,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x55,0x00,0x00,0x00,0x50,0x00,0x00,0x00, +0x54,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x59,0x00,0x00,0x00,0x50,0x00,0x00,0x00,0x58,0x00,0x00,0x00, +0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x5d,0x00,0x00,0x00, +0x4e,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x89,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x64,0x00,0x00,0x00,0x5d,0x00,0x00,0x00, +0x63,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x68,0x00,0x00,0x00,0x5d,0x00,0x00,0x00,0x67,0x00,0x00,0x00, +0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x6e,0x00,0x00,0x00, +0x4e,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x73,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, +0x72,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x78,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x77,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x7d,0x00,0x00,0x00, +0x4e,0x00,0x00,0x00,0x7c,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x15,0x00,0x00,0x00,0x81,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0x80,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x82,0x00,0x00,0x00,0x81,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x83,0x00,0x00,0x00,0x47,0x00,0x00,0x00, +0x82,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x85,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x87,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x89,0x00,0x00,0x00,0x47,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8c,0x00,0x00,0x00, +0x89,0x00,0x00,0x00,0x82,0x00,0x00,0x00,0x0c,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0x8d,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x26,0x00,0x00,0x00,0x87,0x00,0x00,0x00,0x8c,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x91,0x00,0x00,0x00, +0x12,0x00,0x00,0x00,0x90,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x92,0x00,0x00,0x00,0x91,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x93,0x00,0x00,0x00, +0x32,0x00,0x00,0x00,0x92,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x95,0x00,0x00,0x00,0x42,0x00,0x00,0x00, +0x37,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, +0x97,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x96,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x98,0x00,0x00,0x00, +0x97,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x99,0x00,0x00,0x00,0x95,0x00,0x00,0x00,0x98,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9a,0x00,0x00,0x00, +0x93,0x00,0x00,0x00,0x99,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x9c,0x00,0x00,0x00,0x9a,0x00,0x00,0x00, +0x83,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x9d,0x00,0x00,0x00,0x9c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0xa2,0x00,0x00,0x00, +0x12,0x00,0x00,0x00,0xa1,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xa3,0x00,0x00,0x00,0xa2,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa4,0x00,0x00,0x00, +0x0f,0x00,0x00,0x00,0xa3,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xa7,0x00,0x00,0x00,0x4a,0x00,0x00,0x00, +0xa6,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, +0xa9,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0xa8,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xaa,0x00,0x00,0x00, +0xa9,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xab,0x00,0x00,0x00,0xa7,0x00,0x00,0x00,0xaa,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xac,0x00,0x00,0x00, +0xa4,0x00,0x00,0x00,0xab,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xae,0x00,0x00,0x00,0xac,0x00,0x00,0x00, +0x83,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xaf,0x00,0x00,0x00,0xae,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xb1,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xb1,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xc0,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0x05,0x00,0x00,0x00, +0xd0,0x00,0x00,0x00,0xb2,0x00,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc1,0x00,0x00,0x00,0xc2,0x00,0x00,0x00,0xc0,0x02,0x00,0x00, +0xc0,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xb3,0x00,0x00,0x00, +0xb2,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0xc2,0x00,0x00,0x00,0xb2,0x00,0x00,0x00,0xb3,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xb2,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0xcc,0x00,0x00,0x00,0xcd,0x00,0x00,0x00,0xc9,0x00,0x00,0x00, +0xc0,0x02,0x00,0x00,0x3e,0x00,0x03,0x00,0xcd,0x00,0x00,0x00, +0xcb,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xd0,0x00,0x00,0x00,0xc0,0x02,0x00,0x00,0xcf,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xb1,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xb3,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xd3,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xd3,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xd9,0x02,0x00,0x00,0xaf,0x00,0x00,0x00, +0xb3,0x00,0x00,0x00,0x8b,0x01,0x00,0x00,0xd6,0x00,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xd5,0x02,0x00,0x00, +0x9d,0x00,0x00,0x00,0xb3,0x00,0x00,0x00,0x88,0x01,0x00,0x00, +0xd6,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xc1,0x02,0x00,0x00,0x83,0x00,0x00,0x00,0xb3,0x00,0x00,0x00, +0x39,0x02,0x00,0x00,0xd6,0x00,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc1,0x00,0x00,0x00,0xda,0x00,0x00,0x00,0xc1,0x02,0x00,0x00, +0x8d,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xd5,0x00,0x00,0x00, +0xd6,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0xda,0x00,0x00,0x00,0xd4,0x00,0x00,0x00,0xd5,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xd4,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xdc,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xdc,0x00,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xd1,0x02,0x00,0x00, +0x3e,0x00,0x00,0x00,0xd4,0x00,0x00,0x00,0x3b,0x01,0x00,0x00, +0xdd,0x00,0x00,0x00,0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00, +0xe2,0x00,0x00,0x00,0xd1,0x02,0x00,0x00,0x37,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0xde,0x00,0x00,0x00,0xdd,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xe2,0x00,0x00,0x00, +0xdd,0x00,0x00,0x00,0xde,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xdd,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xe7,0x00,0x00,0x00,0x73,0x00,0x00,0x00,0xd1,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xea,0x00,0x00,0x00, +0xe7,0x00,0x00,0x00,0x98,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xeb,0x00,0x00,0x00,0xea,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xec,0x00,0x00,0x00,0xd5,0x02,0x00,0x00,0xeb,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xee,0x00,0x00,0x00, +0xec,0x00,0x00,0x00,0x6e,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xf4,0x00,0x00,0x00,0xe7,0x00,0x00,0x00, +0xf3,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xf6,0x00,0x00,0x00,0x6e,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xf7,0x00,0x00,0x00, +0xf4,0x00,0x00,0x00,0xf6,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xfb,0x00,0x00,0x00,0xee,0x00,0x00,0x00, +0xfa,0x00,0x00,0x00,0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xff,0x00,0x00,0x00,0xee,0x00,0x00,0x00,0xfe,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x00,0x01,0x00,0x00, +0xff,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x41,0x00,0x07,0x00, +0x0c,0x01,0x00,0x00,0x0d,0x01,0x00,0x00,0x0a,0x01,0x00,0x00, +0x34,0x00,0x00,0x00,0xfb,0x00,0x00,0x00,0x34,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x02,0x01,0x00,0x00,0x0e,0x01,0x00,0x00, +0x0d,0x01,0x00,0x00,0x73,0x00,0x04,0x00,0xc3,0x00,0x00,0x00, +0x0f,0x01,0x00,0x00,0x0e,0x01,0x00,0x00,0x41,0x00,0x08,0x00, +0x15,0x01,0x00,0x00,0x16,0x01,0x00,0x00,0x0a,0x01,0x00,0x00, +0x34,0x00,0x00,0x00,0xfb,0x00,0x00,0x00,0xcf,0x00,0x00,0x00, +0x00,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0x03,0x01,0x00,0x00, +0x17,0x01,0x00,0x00,0x16,0x01,0x00,0x00,0x72,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0x18,0x01,0x00,0x00,0x17,0x01,0x00,0x00, +0x6f,0x00,0x04,0x00,0xc3,0x00,0x00,0x00,0x19,0x01,0x00,0x00, +0x18,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x1c,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x39,0x00,0x00,0x00, +0x41,0x00,0x08,0x00,0x15,0x01,0x00,0x00,0x1d,0x01,0x00,0x00, +0x0a,0x01,0x00,0x00,0x34,0x00,0x00,0x00,0xfb,0x00,0x00,0x00, +0xcf,0x00,0x00,0x00,0x1c,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, +0x03,0x01,0x00,0x00,0x1e,0x01,0x00,0x00,0x1d,0x01,0x00,0x00, +0x72,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x1f,0x01,0x00,0x00, +0x1e,0x01,0x00,0x00,0x6f,0x00,0x04,0x00,0xc3,0x00,0x00,0x00, +0x20,0x01,0x00,0x00,0x1f,0x01,0x00,0x00,0x50,0x00,0x05,0x00, +0x10,0x01,0x00,0x00,0x21,0x01,0x00,0x00,0x19,0x01,0x00,0x00, +0x20,0x01,0x00,0x00,0x8e,0x00,0x05,0x00,0x10,0x01,0x00,0x00, +0x23,0x01,0x00,0x00,0x21,0x01,0x00,0x00,0x0f,0x01,0x00,0x00, +0x51,0x00,0x05,0x00,0xc3,0x00,0x00,0x00,0x2b,0x01,0x00,0x00, +0x23,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x73,0x00,0x04,0x00, +0x02,0x01,0x00,0x00,0x2c,0x01,0x00,0x00,0x2b,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0x2d,0x01,0x00,0x00,0x2e,0x01,0x00,0x00, +0x28,0x01,0x00,0x00,0xf7,0x00,0x00,0x00,0x3e,0x00,0x03,0x00, +0x2e,0x01,0x00,0x00,0x2c,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x30,0x01,0x00,0x00,0xf7,0x00,0x00,0x00, +0x39,0x00,0x00,0x00,0x51,0x00,0x05,0x00,0xc3,0x00,0x00,0x00, +0x32,0x01,0x00,0x00,0x23,0x01,0x00,0x00,0x01,0x00,0x00,0x00, +0x73,0x00,0x04,0x00,0x02,0x01,0x00,0x00,0x33,0x01,0x00,0x00, +0x32,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0x2d,0x01,0x00,0x00, +0x34,0x01,0x00,0x00,0x28,0x01,0x00,0x00,0x30,0x01,0x00,0x00, +0x3e,0x00,0x03,0x00,0x34,0x01,0x00,0x00,0x33,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3b,0x01,0x00,0x00, +0xd1,0x02,0x00,0x00,0x39,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0xdc,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xde,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x3d,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x3d,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xd2,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0xde,0x00,0x00,0x00, +0x84,0x01,0x00,0x00,0x40,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc1,0x00,0x00,0x00,0x43,0x01,0x00,0x00,0xd2,0x02,0x00,0x00, +0xa6,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x3f,0x01,0x00,0x00, +0x40,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x43,0x01,0x00,0x00,0x3e,0x01,0x00,0x00,0x3f,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x3e,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x47,0x01,0x00,0x00,0xa7,0x00,0x00,0x00, +0x7d,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x49,0x01,0x00,0x00,0x47,0x01,0x00,0x00,0xd2,0x02,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x4a,0x01,0x00,0x00, +0x12,0x00,0x00,0x00,0xcf,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x4b,0x01,0x00,0x00,0x4a,0x01,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00,0x4c,0x01,0x00,0x00, +0x49,0x01,0x00,0x00,0x4b,0x01,0x00,0x00,0xf7,0x00,0x03,0x00, +0x4e,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x4c,0x01,0x00,0x00,0x4d,0x01,0x00,0x00,0x4e,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x4d,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x51,0x01,0x00,0x00,0xc1,0x02,0x00,0x00, +0x78,0x00,0x00,0x00,0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00, +0x53,0x01,0x00,0x00,0x51,0x01,0x00,0x00,0x8d,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x4e,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x4e,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0xc1,0x00,0x00,0x00, +0x54,0x01,0x00,0x00,0x4c,0x01,0x00,0x00,0x3e,0x01,0x00,0x00, +0x53,0x01,0x00,0x00,0x4d,0x01,0x00,0x00,0xf7,0x00,0x03,0x00, +0x56,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x54,0x01,0x00,0x00,0x55,0x01,0x00,0x00,0x76,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x55,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x5e,0x01,0x00,0x00,0x7d,0x00,0x00,0x00, +0xd2,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x60,0x01,0x00,0x00,0x5e,0x01,0x00,0x00,0x5f,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x62,0x01,0x00,0x00, +0x60,0x01,0x00,0x00,0x78,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x6d,0x01,0x00,0x00,0x5e,0x01,0x00,0x00, +0xaa,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x6e,0x01,0x00,0x00,0xd9,0x02,0x00,0x00,0x6d,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x70,0x01,0x00,0x00, +0x6e,0x01,0x00,0x00,0x78,0x00,0x00,0x00,0x41,0x00,0x06,0x00, +0x71,0x01,0x00,0x00,0x72,0x01,0x00,0x00,0x66,0x01,0x00,0x00, +0x34,0x00,0x00,0x00,0x70,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, +0xc3,0x00,0x00,0x00,0x73,0x01,0x00,0x00,0x72,0x01,0x00,0x00, +0x73,0x00,0x04,0x00,0x02,0x01,0x00,0x00,0x74,0x01,0x00,0x00, +0x73,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0x2d,0x01,0x00,0x00, +0x75,0x01,0x00,0x00,0x5b,0x01,0x00,0x00,0x62,0x01,0x00,0x00, +0x3e,0x00,0x03,0x00,0x75,0x01,0x00,0x00,0x74,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0x56,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x76,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x79,0x01,0x00,0x00,0x7d,0x00,0x00,0x00,0xd2,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x7b,0x01,0x00,0x00, +0x79,0x01,0x00,0x00,0x7a,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x7d,0x01,0x00,0x00,0x7b,0x01,0x00,0x00, +0x78,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x2d,0x01,0x00,0x00, +0x7f,0x01,0x00,0x00,0x5b,0x01,0x00,0x00,0x7d,0x01,0x00,0x00, +0x3e,0x00,0x03,0x00,0x7f,0x01,0x00,0x00,0x7e,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0x56,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x56,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x40,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x40,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x84,0x01,0x00,0x00,0xd2,0x02,0x00,0x00, +0x82,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x3d,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x3f,0x01,0x00,0x00,0xe0,0x00,0x04,0x00, +0x0c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x85,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x88,0x01,0x00,0x00, +0xd5,0x02,0x00,0x00,0x86,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x8b,0x01,0x00,0x00,0xd9,0x02,0x00,0x00, +0x89,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x8d,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x8d,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xdb,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, +0x3f,0x01,0x00,0x00,0x37,0x02,0x00,0x00,0x90,0x01,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00,0x93,0x01,0x00,0x00, +0xdb,0x02,0x00,0x00,0x6c,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0x8f,0x01,0x00,0x00,0x90,0x01,0x00,0x00,0x00,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x93,0x01,0x00,0x00,0x8e,0x01,0x00,0x00, +0x8f,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x8e,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0x95,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x95,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xdf,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0x8e,0x01,0x00,0x00, +0xc1,0x01,0x00,0x00,0x98,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc1,0x00,0x00,0x00,0x9b,0x01,0x00,0x00,0xdf,0x02,0x00,0x00, +0x60,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x97,0x01,0x00,0x00, +0x98,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x9b,0x01,0x00,0x00,0x96,0x01,0x00,0x00,0x97,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x96,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0x9d,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x9d,0x01,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xf1,0x02,0x00,0x00, +0x3e,0x00,0x00,0x00,0x96,0x01,0x00,0x00,0xbf,0x01,0x00,0x00, +0x9e,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00, +0xa3,0x01,0x00,0x00,0xf1,0x02,0x00,0x00,0x62,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0x9f,0x01,0x00,0x00,0x9e,0x01,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xa3,0x01,0x00,0x00, +0x9e,0x01,0x00,0x00,0x9f,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x9e,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xa9,0x01,0x00,0x00,0xdf,0x02,0x00,0x00,0x62,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xab,0x01,0x00,0x00, +0xa9,0x01,0x00,0x00,0xf1,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xad,0x01,0x00,0x00,0x55,0x00,0x00,0x00, +0x53,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xaf,0x01,0x00,0x00,0xdf,0x02,0x00,0x00,0x61,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xb0,0x01,0x00,0x00, +0xad,0x01,0x00,0x00,0xaf,0x01,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xb2,0x01,0x00,0x00,0x64,0x00,0x00,0x00, +0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xb3,0x01,0x00,0x00,0xb0,0x01,0x00,0x00,0xb2,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xb5,0x01,0x00,0x00, +0xb3,0x01,0x00,0x00,0xf1,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xb7,0x01,0x00,0x00,0xb5,0x01,0x00,0x00, +0xb6,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xb9,0x01,0x00,0x00,0xb7,0x01,0x00,0x00,0xdb,0x02,0x00,0x00, +0x41,0x00,0x05,0x00,0x2d,0x01,0x00,0x00,0xba,0x01,0x00,0x00, +0x28,0x01,0x00,0x00,0xb9,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, +0x02,0x01,0x00,0x00,0xbb,0x01,0x00,0x00,0xba,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0xbc,0x01,0x00,0x00,0xbd,0x01,0x00,0x00, +0xa7,0x01,0x00,0x00,0xab,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, +0xbd,0x01,0x00,0x00,0xbb,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xbf,0x01,0x00,0x00,0xf1,0x02,0x00,0x00, +0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x9d,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x9f,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0x98,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x98,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xc1,0x01,0x00,0x00, +0xdf,0x02,0x00,0x00,0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x95,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x97,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0xc3,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xc3,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xe0,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0x97,0x01,0x00,0x00, +0xef,0x01,0x00,0x00,0xc6,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc1,0x00,0x00,0x00,0xc9,0x01,0x00,0x00,0xe0,0x02,0x00,0x00, +0xbe,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xc5,0x01,0x00,0x00, +0xc6,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0xc9,0x01,0x00,0x00,0xc4,0x01,0x00,0x00,0xc5,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xc4,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0xcb,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xcb,0x01,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xee,0x02,0x00,0x00, +0x3e,0x00,0x00,0x00,0xc4,0x01,0x00,0x00,0xed,0x01,0x00,0x00, +0xcc,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00, +0xd1,0x01,0x00,0x00,0xee,0x02,0x00,0x00,0xbb,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0xcd,0x01,0x00,0x00,0xcc,0x01,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xd1,0x01,0x00,0x00, +0xcc,0x01,0x00,0x00,0xcd,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xcc,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xd7,0x01,0x00,0x00,0xe0,0x02,0x00,0x00,0xbb,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xd9,0x01,0x00,0x00, +0xd7,0x01,0x00,0x00,0xee,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xdb,0x01,0x00,0x00,0x59,0x00,0x00,0x00, +0xb8,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xde,0x01,0x00,0x00,0xe0,0x02,0x00,0x00,0xdd,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xdf,0x01,0x00,0x00, +0xdb,0x01,0x00,0x00,0xde,0x01,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xe1,0x01,0x00,0x00,0x68,0x00,0x00,0x00, +0xbb,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xe2,0x01,0x00,0x00,0xdf,0x01,0x00,0x00,0xe1,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xe4,0x01,0x00,0x00, +0xe2,0x01,0x00,0x00,0xee,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xe6,0x01,0x00,0x00,0xe4,0x01,0x00,0x00, +0xe5,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xe8,0x01,0x00,0x00,0xe6,0x01,0x00,0x00,0xdb,0x02,0x00,0x00, +0x41,0x00,0x05,0x00,0x2d,0x01,0x00,0x00,0xe9,0x01,0x00,0x00, +0x5b,0x01,0x00,0x00,0xe8,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, +0x02,0x01,0x00,0x00,0xea,0x01,0x00,0x00,0xe9,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0xbc,0x01,0x00,0x00,0xeb,0x01,0x00,0x00, +0xd5,0x01,0x00,0x00,0xd9,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, +0xeb,0x01,0x00,0x00,0xea,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xed,0x01,0x00,0x00,0xee,0x02,0x00,0x00, +0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xcb,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xcd,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0xc6,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xc6,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xef,0x01,0x00,0x00, +0xe0,0x02,0x00,0x00,0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xc3,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xc5,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0xf1,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xf1,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xe1,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0xc5,0x01,0x00,0x00, +0x35,0x02,0x00,0x00,0xf4,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc1,0x00,0x00,0x00,0xf7,0x01,0x00,0x00,0xe1,0x02,0x00,0x00, +0xbe,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xf3,0x01,0x00,0x00, +0xf4,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0xf7,0x01,0x00,0x00,0xf2,0x01,0x00,0x00,0xf3,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xf2,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0xf9,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xf9,0x01,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xe5,0x02,0x00,0x00, +0x3e,0x00,0x00,0x00,0xf2,0x01,0x00,0x00,0x33,0x02,0x00,0x00, +0xfc,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00, +0xff,0x01,0x00,0x00,0xe5,0x02,0x00,0x00,0x60,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0xfb,0x01,0x00,0x00,0xfc,0x01,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xff,0x01,0x00,0x00, +0xfa,0x01,0x00,0x00,0xfb,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xfa,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x01,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x01,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xe7,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, +0xfa,0x01,0x00,0x00,0x31,0x02,0x00,0x00,0x04,0x02,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00,0x07,0x02,0x00,0x00, +0xe7,0x02,0x00,0x00,0xbb,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0x03,0x02,0x00,0x00,0x04,0x02,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x07,0x02,0x00,0x00,0x02,0x02,0x00,0x00, +0x03,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x02,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0x09,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x09,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xe9,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0x02,0x02,0x00,0x00, +0x2f,0x02,0x00,0x00,0x0a,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc1,0x00,0x00,0x00,0x0f,0x02,0x00,0x00,0xe9,0x02,0x00,0x00, +0x62,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x0b,0x02,0x00,0x00, +0x0a,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x0f,0x02,0x00,0x00,0x0a,0x02,0x00,0x00,0x0b,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x0a,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x11,0x02,0x00,0x00,0xe1,0x02,0x00,0x00, +0xbb,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x13,0x02,0x00,0x00,0x11,0x02,0x00,0x00,0xe7,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x15,0x02,0x00,0x00, +0x13,0x02,0x00,0x00,0x14,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x17,0x02,0x00,0x00,0xe5,0x02,0x00,0x00, +0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x18,0x02,0x00,0x00,0x15,0x02,0x00,0x00,0x17,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x1a,0x02,0x00,0x00, +0x18,0x02,0x00,0x00,0xe9,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x1e,0x02,0x00,0x00,0x17,0x02,0x00,0x00, +0xe9,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0xbc,0x01,0x00,0x00, +0x1f,0x02,0x00,0x00,0xa7,0x01,0x00,0x00,0x1e,0x02,0x00,0x00, +0x3d,0x00,0x04,0x00,0x02,0x01,0x00,0x00,0x20,0x02,0x00,0x00, +0x1f,0x02,0x00,0x00,0x73,0x00,0x04,0x00,0xc3,0x00,0x00,0x00, +0x21,0x02,0x00,0x00,0x20,0x02,0x00,0x00,0x41,0x00,0x05,0x00, +0xbc,0x01,0x00,0x00,0x26,0x02,0x00,0x00,0xd5,0x01,0x00,0x00, +0x13,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0x02,0x01,0x00,0x00, +0x27,0x02,0x00,0x00,0x26,0x02,0x00,0x00,0x73,0x00,0x04,0x00, +0xc3,0x00,0x00,0x00,0x28,0x02,0x00,0x00,0x27,0x02,0x00,0x00, +0x41,0x00,0x05,0x00,0xcc,0x00,0x00,0x00,0x2a,0x02,0x00,0x00, +0xc9,0x00,0x00,0x00,0x1a,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, +0xc3,0x00,0x00,0x00,0x2b,0x02,0x00,0x00,0x2a,0x02,0x00,0x00, +0x0c,0x00,0x08,0x00,0xc3,0x00,0x00,0x00,0x2c,0x02,0x00,0x00, +0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x21,0x02,0x00,0x00, +0x28,0x02,0x00,0x00,0x2b,0x02,0x00,0x00,0x3e,0x00,0x03,0x00, +0x2a,0x02,0x00,0x00,0x2c,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x2f,0x02,0x00,0x00,0xe9,0x02,0x00,0x00, +0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x09,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x0b,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0x04,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x04,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x31,0x02,0x00,0x00, +0xe7,0x02,0x00,0x00,0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x01,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x03,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0xfc,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xfc,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x33,0x02,0x00,0x00,0xe5,0x02,0x00,0x00,0xcf,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xf9,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xfb,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xf4,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xf4,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x35,0x02,0x00,0x00,0xe1,0x02,0x00,0x00, +0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xf1,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xf3,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0x90,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x90,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x37,0x02,0x00,0x00, +0xdb,0x02,0x00,0x00,0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x8d,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x8f,0x01,0x00,0x00, +0xe0,0x00,0x04,0x00,0x0c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x85,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xd6,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xd6,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x39,0x02,0x00,0x00,0xc1,0x02,0x00,0x00, +0x6c,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xd3,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xd5,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x3e,0x02,0x00,0x00,0x55,0x00,0x00,0x00, +0x53,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x3f,0x02,0x00,0x00,0x95,0x00,0x00,0x00,0x3e,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x44,0x02,0x00,0x00, +0x59,0x00,0x00,0x00,0xb8,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x45,0x02,0x00,0x00,0xa7,0x00,0x00,0x00, +0x44,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, +0x4a,0x02,0x00,0x00,0x12,0x00,0x00,0x00,0x49,0x02,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x4b,0x02,0x00,0x00, +0x4a,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x4c,0x02,0x00,0x00,0x0f,0x00,0x00,0x00,0x4b,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x50,0x02,0x00,0x00, +0x47,0x00,0x00,0x00,0x4b,0x02,0x00,0x00,0x41,0x00,0x05,0x00, +0x0d,0x00,0x00,0x00,0x52,0x02,0x00,0x00,0x51,0x02,0x00,0x00, +0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x53,0x02,0x00,0x00,0x52,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x54,0x02,0x00,0x00,0x50,0x02,0x00,0x00, +0x53,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x55,0x02,0x00,0x00,0x4c,0x02,0x00,0x00,0x54,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0x57,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x57,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xc2,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0xd5,0x00,0x00,0x00, +0xbd,0x02,0x00,0x00,0x5a,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc1,0x00,0x00,0x00,0x5d,0x02,0x00,0x00,0xc2,0x02,0x00,0x00, +0xbe,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x59,0x02,0x00,0x00, +0x5a,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x5d,0x02,0x00,0x00,0x58,0x02,0x00,0x00,0x59,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x58,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0x5f,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x5f,0x02,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xc3,0x02,0x00,0x00, +0x3e,0x00,0x00,0x00,0x58,0x02,0x00,0x00,0xbb,0x02,0x00,0x00, +0x62,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00, +0x65,0x02,0x00,0x00,0xc3,0x02,0x00,0x00,0x60,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0x61,0x02,0x00,0x00,0x62,0x02,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x65,0x02,0x00,0x00, +0x60,0x02,0x00,0x00,0x61,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x60,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x69,0x02,0x00,0x00,0xc3,0x02,0x00,0x00,0x61,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x6a,0x02,0x00,0x00, +0x3f,0x02,0x00,0x00,0x69,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x6c,0x02,0x00,0x00,0x64,0x00,0x00,0x00, +0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x6d,0x02,0x00,0x00,0x6a,0x02,0x00,0x00,0x6c,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x71,0x02,0x00,0x00, +0xc2,0x02,0x00,0x00,0xdd,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x72,0x02,0x00,0x00,0x45,0x02,0x00,0x00, +0x71,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x74,0x02,0x00,0x00,0x68,0x00,0x00,0x00,0xbb,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x75,0x02,0x00,0x00, +0x72,0x02,0x00,0x00,0x74,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0x77,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x77,0x02,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xc5,0x02,0x00,0x00, +0x3e,0x00,0x00,0x00,0x60,0x02,0x00,0x00,0xb9,0x02,0x00,0x00, +0x7a,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00, +0x7d,0x02,0x00,0x00,0xc5,0x02,0x00,0x00,0xbb,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0x79,0x02,0x00,0x00,0x7a,0x02,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x7d,0x02,0x00,0x00, +0x78,0x02,0x00,0x00,0x79,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x78,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x7f,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x7f,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xc7,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, +0x78,0x02,0x00,0x00,0xb7,0x02,0x00,0x00,0x82,0x02,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00,0x85,0x02,0x00,0x00, +0xc7,0x02,0x00,0x00,0x62,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0x81,0x02,0x00,0x00,0x82,0x02,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x85,0x02,0x00,0x00,0x80,0x02,0x00,0x00, +0x81,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x80,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x88,0x02,0x00,0x00, +0x6d,0x02,0x00,0x00,0xc7,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc1,0x00,0x00,0x00,0x8b,0x02,0x00,0x00,0x88,0x02,0x00,0x00, +0x36,0x00,0x00,0x00,0xf7,0x00,0x03,0x00,0x8d,0x02,0x00,0x00, +0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x8b,0x02,0x00,0x00, +0x8c,0x02,0x00,0x00,0x8d,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x8c,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x90,0x02,0x00,0x00,0x75,0x02,0x00,0x00,0xc5,0x02,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x91,0x02,0x00,0x00, +0x12,0x00,0x00,0x00,0xcf,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x92,0x02,0x00,0x00,0x91,0x02,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00,0x93,0x02,0x00,0x00, +0x90,0x02,0x00,0x00,0x92,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0x8d,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x8d,0x02,0x00,0x00, +0xf5,0x00,0x07,0x00,0xc1,0x00,0x00,0x00,0x94,0x02,0x00,0x00, +0x8b,0x02,0x00,0x00,0x80,0x02,0x00,0x00,0x93,0x02,0x00,0x00, +0x8c,0x02,0x00,0x00,0xf7,0x00,0x03,0x00,0x96,0x02,0x00,0x00, +0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x94,0x02,0x00,0x00, +0x95,0x02,0x00,0x00,0x96,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x95,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x9e,0x02,0x00,0x00,0x75,0x02,0x00,0x00,0xc5,0x02,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0xa0,0x02,0x00,0x00, +0x12,0x00,0x00,0x00,0x9f,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xa1,0x02,0x00,0x00,0xa0,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa2,0x02,0x00,0x00, +0x9e,0x02,0x00,0x00,0xa1,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xa3,0x02,0x00,0x00,0x55,0x02,0x00,0x00, +0xa2,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xa5,0x02,0x00,0x00,0xa3,0x02,0x00,0x00,0x6d,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa7,0x02,0x00,0x00, +0xa5,0x02,0x00,0x00,0xc7,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xa9,0x02,0x00,0x00,0xc2,0x02,0x00,0x00, +0xbb,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xab,0x02,0x00,0x00,0xa9,0x02,0x00,0x00,0xc5,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xad,0x02,0x00,0x00, +0xab,0x02,0x00,0x00,0xac,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xaf,0x02,0x00,0x00,0xc3,0x02,0x00,0x00, +0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xb0,0x02,0x00,0x00,0xad,0x02,0x00,0x00,0xaf,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xb2,0x02,0x00,0x00, +0xb0,0x02,0x00,0x00,0xc7,0x02,0x00,0x00,0x41,0x00,0x05,0x00, +0xcc,0x00,0x00,0x00,0xb3,0x02,0x00,0x00,0xc9,0x00,0x00,0x00, +0xb2,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0xc3,0x00,0x00,0x00, +0xb4,0x02,0x00,0x00,0xb3,0x02,0x00,0x00,0x41,0x00,0x06,0x00, +0x71,0x01,0x00,0x00,0xb5,0x02,0x00,0x00,0x9a,0x02,0x00,0x00, +0x34,0x00,0x00,0x00,0xa7,0x02,0x00,0x00,0x3e,0x00,0x03,0x00, +0xb5,0x02,0x00,0x00,0xb4,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0x96,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x96,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0x82,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x82,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xb7,0x02,0x00,0x00,0xc7,0x02,0x00,0x00,0xcf,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x7f,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x81,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x7a,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x7a,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xb9,0x02,0x00,0x00,0xc5,0x02,0x00,0x00, +0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x77,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x79,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0x62,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x62,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xbb,0x02,0x00,0x00, +0xc3,0x02,0x00,0x00,0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x5f,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x61,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0x5a,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x5a,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xbd,0x02,0x00,0x00,0xc2,0x02,0x00,0x00,0xcf,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x57,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x59,0x02,0x00,0x00,0xfd,0x00,0x01,0x00,0x38,0x00,0x01,0x00, + +}; +const uint64_t matmul_q8_0_f32_len = 10608; + +unsigned char matmul_q8_0_f32_aligned_data[] = { +0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, +0x15,0x03,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, +0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x09,0x00,0x00,0x00, +0x11,0x00,0x02,0x00,0x51,0x11,0x00,0x00,0x11,0x00,0x02,0x00, +0x60,0x11,0x00,0x00,0x0b,0x00,0x06,0x00,0x01,0x00,0x00,0x00, +0x47,0x4c,0x53,0x4c,0x2e,0x73,0x74,0x64,0x2e,0x34,0x35,0x30, +0x00,0x00,0x00,0x00,0x0e,0x00,0x03,0x00,0x00,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x0f,0x00,0x0f,0x00,0x05,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x6d,0x61,0x69,0x6e,0x00,0x00,0x00,0x00, +0x0b,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x3d,0x00,0x00,0x00, +0x4c,0x00,0x00,0x00,0x0b,0x01,0x00,0x00,0x29,0x01,0x00,0x00, +0x5e,0x01,0x00,0x00,0x66,0x01,0x00,0x00,0x74,0x02,0x00,0x00, +0xbd,0x02,0x00,0x00,0x10,0x00,0x06,0x00,0x04,0x00,0x00,0x00, +0x11,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x0b,0x00,0x00,0x00, +0x0b,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x10,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x10,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x10,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x05,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x10,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, +0x07,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x1c,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x08,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x10,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x24,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, +0x0a,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x28,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x2c,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x10,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x30,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, +0x0d,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x34,0x00,0x00,0x00, +0x47,0x00,0x03,0x00,0x10,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x37,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x3d,0x00,0x00,0x00, +0x0b,0x00,0x00,0x00,0x1a,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x4c,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x1b,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x4f,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x09,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x53,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x60,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x62,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x07,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x6c,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0xa7,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0xb9,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x05,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xbc,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x06,0x01,0x00,0x00,0x06,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x07,0x01,0x00,0x00,0x00,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x07,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x08,0x01,0x00,0x00, +0x06,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x48,0x00,0x04,0x00, +0x09,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x09,0x01,0x00,0x00,0x00,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, +0x09,0x01,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x0b,0x01,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x0b,0x01,0x00,0x00,0x21,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x36,0x01,0x00,0x00, +0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x37,0x01,0x00,0x00,0x0b,0x00,0x00,0x00,0x19,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x63,0x01,0x00,0x00,0x06,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0x64,0x01,0x00,0x00, +0x00,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0x48,0x00,0x04,0x00, +0x64,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x64,0x01,0x00,0x00,0x00,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x64,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x07,0x00,0x00,0x00, +0x10,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x64,0x01,0x00,0x00, +0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x66,0x01,0x00,0x00, +0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x66,0x01,0x00,0x00,0x21,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x74,0x02,0x00,0x00,0x0b,0x00,0x00,0x00, +0x18,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xba,0x02,0x00,0x00, +0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00, +0xbb,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x19,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0xbb,0x02,0x00,0x00,0x00,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, +0xbb,0x02,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0xbd,0x02,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0xbd,0x02,0x00,0x00,0x21,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x13,0x00,0x02,0x00,0x02,0x00,0x00,0x00, +0x21,0x00,0x03,0x00,0x03,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x15,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x17,0x00,0x04,0x00,0x09,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x0a,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x09,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x0d,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x1e,0x00,0x10,0x00,0x10,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x11,0x00,0x00,0x00, +0x09,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0x11,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x09,0x00,0x00,0x00, +0x15,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, +0x14,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x15,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x21,0x00,0x00,0x00, +0x0a,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, +0x27,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0x2d,0x00,0x00,0x00,0x07,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x34,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x37,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00,0x3d,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x3e,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0x0a,0x00,0x00,0x00,0x4c,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x4f,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x53,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x54,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0x37,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x58,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0x37,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x32,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x61,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x60,0x00,0x00,0x00, +0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x62,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x63,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x61,0x00,0x00,0x00, +0x62,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x67,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x61,0x00,0x00,0x00, +0x62,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x72,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x77,0x00,0x00,0x00,0x08,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x78,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x77,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x7d,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x77,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x81,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0x91,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x97,0x00,0x00,0x00, +0x03,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, +0xa2,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x32,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xa7,0x00,0x00,0x00,0x40,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0xa9,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xb8,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00, +0x62,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xb9,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xba,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0x53,0x00,0x00,0x00,0xb9,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xbb,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0x4f,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x32,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xbc,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xbd,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0xbb,0x00,0x00,0x00,0xbc,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xbe,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0xbd,0x00,0x00,0x00,0x60,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xbf,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0xba,0x00,0x00,0x00,0xbe,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xc0,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0xb8,0x00,0x00,0x00,0xbf,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xc1,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0xc0,0x00,0x00,0x00,0xbc,0x00,0x00,0x00, +0x14,0x00,0x02,0x00,0xc2,0x00,0x00,0x00,0x16,0x00,0x03,0x00, +0xc4,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xc5,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xc6,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0xc5,0x00,0x00,0x00,0xbf,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xc7,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0xc6,0x00,0x00,0x00,0xbc,0x00,0x00,0x00,0x1c,0x00,0x04,0x00, +0xc8,0x00,0x00,0x00,0xc4,0x00,0x00,0x00,0xc7,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0xc9,0x00,0x00,0x00,0x07,0x00,0x00,0x00, +0xc8,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0xc4,0x00,0x00,0x00, +0xcc,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0xcd,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0xc4,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0xd0,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xf4,0x00,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, +0x39,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xfb,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xff,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, +0x16,0x00,0x03,0x00,0x03,0x01,0x00,0x00,0x10,0x00,0x00,0x00, +0x15,0x00,0x04,0x00,0x04,0x01,0x00,0x00,0x08,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x05,0x01,0x00,0x00,0x20,0x00,0x00,0x00,0x1c,0x00,0x04,0x00, +0x06,0x01,0x00,0x00,0x04,0x01,0x00,0x00,0x05,0x01,0x00,0x00, +0x1e,0x00,0x04,0x00,0x07,0x01,0x00,0x00,0x03,0x01,0x00,0x00, +0x06,0x01,0x00,0x00,0x1d,0x00,0x03,0x00,0x08,0x01,0x00,0x00, +0x07,0x01,0x00,0x00,0x1e,0x00,0x03,0x00,0x09,0x01,0x00,0x00, +0x08,0x01,0x00,0x00,0x20,0x00,0x04,0x00,0x0a,0x01,0x00,0x00, +0x0c,0x00,0x00,0x00,0x09,0x01,0x00,0x00,0x3b,0x00,0x04,0x00, +0x0a,0x01,0x00,0x00,0x0b,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x0d,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, +0x03,0x01,0x00,0x00,0x17,0x00,0x04,0x00,0x11,0x01,0x00,0x00, +0xc4,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x16,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x04,0x01,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x25,0x01,0x00,0x00, +0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x26,0x01,0x00,0x00, +0x84,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x25,0x01,0x00,0x00, +0x1c,0x00,0x04,0x00,0x27,0x01,0x00,0x00,0x03,0x01,0x00,0x00, +0x26,0x01,0x00,0x00,0x20,0x00,0x04,0x00,0x28,0x01,0x00,0x00, +0x04,0x00,0x00,0x00,0x27,0x01,0x00,0x00,0x3b,0x00,0x04,0x00, +0x28,0x01,0x00,0x00,0x29,0x01,0x00,0x00,0x04,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x2e,0x01,0x00,0x00,0x04,0x00,0x00,0x00, +0x03,0x01,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x36,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x33,0x00,0x06,0x00, +0x09,0x00,0x00,0x00,0x37,0x01,0x00,0x00,0x36,0x01,0x00,0x00, +0x39,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x38,0x01,0x00,0x00,0x51,0x00,0x00,0x00, +0x37,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x39,0x01,0x00,0x00,0x84,0x00,0x00,0x00, +0x38,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x3a,0x01,0x00,0x00,0x86,0x00,0x00,0x00, +0x39,0x01,0x00,0x00,0x6c,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x55,0x01,0x00,0x00,0x80,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x5a,0x01,0x00,0x00,0x80,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x5b,0x01,0x00,0x00,0x84,0x00,0x00,0x00, +0xa7,0x00,0x00,0x00,0x5a,0x01,0x00,0x00,0x1c,0x00,0x04,0x00, +0x5c,0x01,0x00,0x00,0x03,0x01,0x00,0x00,0x5b,0x01,0x00,0x00, +0x20,0x00,0x04,0x00,0x5d,0x01,0x00,0x00,0x04,0x00,0x00,0x00, +0x5c,0x01,0x00,0x00,0x3b,0x00,0x04,0x00,0x5d,0x01,0x00,0x00, +0x5e,0x01,0x00,0x00,0x04,0x00,0x00,0x00,0x17,0x00,0x04,0x00, +0x61,0x01,0x00,0x00,0xc4,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x18,0x00,0x04,0x00,0x62,0x01,0x00,0x00,0x61,0x01,0x00,0x00, +0x02,0x00,0x00,0x00,0x1d,0x00,0x03,0x00,0x63,0x01,0x00,0x00, +0x62,0x01,0x00,0x00,0x1e,0x00,0x03,0x00,0x64,0x01,0x00,0x00, +0x63,0x01,0x00,0x00,0x20,0x00,0x04,0x00,0x65,0x01,0x00,0x00, +0x0c,0x00,0x00,0x00,0x64,0x01,0x00,0x00,0x3b,0x00,0x04,0x00, +0x65,0x01,0x00,0x00,0x66,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x68,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, +0xc4,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x7c,0x01,0x00,0x00,0x03,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x84,0x01,0x00,0x00,0x04,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x8c,0x01,0x00,0x00, +0x05,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x94,0x01,0x00,0x00,0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x9c,0x01,0x00,0x00,0x07,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xa3,0x01,0x00,0x00, +0x51,0x00,0x00,0x00,0x37,0x01,0x00,0x00,0x00,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xa4,0x01,0x00,0x00, +0x84,0x00,0x00,0x00,0xa3,0x01,0x00,0x00,0x77,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xa5,0x01,0x00,0x00, +0x86,0x00,0x00,0x00,0xa4,0x01,0x00,0x00,0x6c,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xa8,0x01,0x00,0x00, +0x08,0x01,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xa9,0x01,0x00,0x00,0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xac,0x01,0x00,0x00,0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, +0x77,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xc7,0x01,0x00,0x00,0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00, +0x62,0x00,0x00,0x00,0x1c,0x00,0x04,0x00,0xc8,0x01,0x00,0x00, +0x03,0x01,0x00,0x00,0xc7,0x01,0x00,0x00,0x20,0x00,0x04,0x00, +0xc9,0x01,0x00,0x00,0x07,0x00,0x00,0x00,0xc8,0x01,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xd9,0x01,0x00,0x00, +0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0xdf,0x01,0x00,0x00,0x07,0x00,0x00,0x00, +0x03,0x01,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xf5,0x01,0x00,0x00,0x84,0x00,0x00,0x00,0xbf,0x00,0x00,0x00, +0xbc,0x00,0x00,0x00,0x1c,0x00,0x04,0x00,0xf6,0x01,0x00,0x00, +0x03,0x01,0x00,0x00,0xf5,0x01,0x00,0x00,0x20,0x00,0x04,0x00, +0xf7,0x01,0x00,0x00,0x07,0x00,0x00,0x00,0xf6,0x01,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x00,0x02,0x00,0x00, +0x86,0x00,0x00,0x00,0xb9,0x00,0x00,0x00,0xbf,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x08,0x02,0x00,0x00, +0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x37,0x02,0x00,0x00, +0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x6c,0x02,0x00,0x00, +0x0d,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00, +0x74,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, +0xba,0x02,0x00,0x00,0xc4,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, +0xbb,0x02,0x00,0x00,0xba,0x02,0x00,0x00,0x20,0x00,0x04,0x00, +0xbc,0x02,0x00,0x00,0x0c,0x00,0x00,0x00,0xbb,0x02,0x00,0x00, +0x3b,0x00,0x04,0x00,0xbc,0x02,0x00,0x00,0xbd,0x02,0x00,0x00, +0x0c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, +0xc2,0x02,0x00,0x00,0x05,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xcf,0x02,0x00,0x00,0x84,0x00,0x00,0x00, +0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x36,0x00,0x05,0x00, +0x02,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x03,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x05,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0xc9,0x00,0x00,0x00,0xca,0x00,0x00,0x00, +0x07,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0xc9,0x01,0x00,0x00, +0xca,0x01,0x00,0x00,0x07,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0xf7,0x01,0x00,0x00,0xf8,0x01,0x00,0x00,0x07,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00,0x0e,0x00,0x00,0x00, +0x0b,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x0e,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x16,0x00,0x00,0x00, +0x12,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x17,0x00,0x00,0x00,0x16,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x18,0x00,0x00,0x00, +0x0f,0x00,0x00,0x00,0x17,0x00,0x00,0x00,0x89,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x1e,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, +0x17,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, +0x22,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x21,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x22,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x24,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x28,0x00,0x00,0x00, +0x12,0x00,0x00,0x00,0x27,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x29,0x00,0x00,0x00,0x28,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x2a,0x00,0x00,0x00, +0x1e,0x00,0x00,0x00,0x29,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x15,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0x2d,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x2f,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x30,0x00,0x00,0x00,0x24,0x00,0x00,0x00, +0x2f,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x32,0x00,0x00,0x00,0x30,0x00,0x00,0x00,0x2a,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x35,0x00,0x00,0x00, +0x12,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x36,0x00,0x00,0x00,0x35,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x38,0x00,0x00,0x00, +0x36,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x82,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x38,0x00,0x00,0x00, +0x39,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x3b,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x37,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00,0x3f,0x00,0x00,0x00, +0x3d,0x00,0x00,0x00,0x3e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x3f,0x00,0x00,0x00, +0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x42,0x00,0x00,0x00, +0x40,0x00,0x00,0x00,0x3b,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x47,0x00,0x00,0x00,0x40,0x00,0x00,0x00, +0x3b,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00, +0x49,0x00,0x00,0x00,0x3d,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x4a,0x00,0x00,0x00, +0x49,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00, +0x4d,0x00,0x00,0x00,0x4c,0x00,0x00,0x00,0x3e,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, +0x4d,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x50,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x4f,0x00,0x00,0x00, +0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x55,0x00,0x00,0x00, +0x50,0x00,0x00,0x00,0x54,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x59,0x00,0x00,0x00,0x50,0x00,0x00,0x00, +0x58,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x5d,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x4f,0x00,0x00,0x00, +0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x64,0x00,0x00,0x00, +0x5d,0x00,0x00,0x00,0x63,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x68,0x00,0x00,0x00,0x5d,0x00,0x00,0x00, +0x67,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x6e,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x73,0x00,0x00,0x00, +0x4e,0x00,0x00,0x00,0x72,0x00,0x00,0x00,0x89,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x79,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, +0x78,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x7e,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x7d,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x82,0x00,0x00,0x00, +0x12,0x00,0x00,0x00,0x81,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x83,0x00,0x00,0x00,0x82,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0x47,0x00,0x00,0x00,0x83,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x15,0x00,0x00,0x00,0x87,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x88,0x00,0x00,0x00,0x87,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x8a,0x00,0x00,0x00,0x47,0x00,0x00,0x00, +0x39,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x8d,0x00,0x00,0x00,0x8a,0x00,0x00,0x00,0x83,0x00,0x00,0x00, +0x0c,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x8e,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x26,0x00,0x00,0x00,0x88,0x00,0x00,0x00, +0x8d,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, +0x92,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x91,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x93,0x00,0x00,0x00, +0x92,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x94,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x93,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x96,0x00,0x00,0x00, +0x42,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x15,0x00,0x00,0x00,0x98,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0x97,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x99,0x00,0x00,0x00,0x98,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x9a,0x00,0x00,0x00,0x96,0x00,0x00,0x00, +0x99,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x9b,0x00,0x00,0x00,0x94,0x00,0x00,0x00,0x9a,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9d,0x00,0x00,0x00, +0x9b,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x9e,0x00,0x00,0x00,0x9d,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, +0xa3,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0xa2,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xa4,0x00,0x00,0x00, +0xa3,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xa5,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0xa4,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa8,0x00,0x00,0x00, +0x4a,0x00,0x00,0x00,0xa7,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x15,0x00,0x00,0x00,0xaa,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0xa9,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xab,0x00,0x00,0x00,0xaa,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xac,0x00,0x00,0x00,0xa8,0x00,0x00,0x00, +0xab,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xad,0x00,0x00,0x00,0xa5,0x00,0x00,0x00,0xac,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xaf,0x00,0x00,0x00, +0xad,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xb0,0x00,0x00,0x00,0xaf,0x00,0x00,0x00, +0x77,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xb2,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xb2,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xe3,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, +0x05,0x00,0x00,0x00,0xd1,0x00,0x00,0x00,0xb3,0x00,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0xc3,0x00,0x00,0x00, +0xe3,0x02,0x00,0x00,0xc1,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0xb4,0x00,0x00,0x00,0xb3,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0xc3,0x00,0x00,0x00,0xb3,0x00,0x00,0x00, +0xb4,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xb3,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0xcd,0x00,0x00,0x00,0xce,0x00,0x00,0x00, +0xca,0x00,0x00,0x00,0xe3,0x02,0x00,0x00,0x3e,0x00,0x03,0x00, +0xce,0x00,0x00,0x00,0xcc,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xd1,0x00,0x00,0x00,0xe3,0x02,0x00,0x00, +0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xb2,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xb4,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xd4,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xd4,0x00,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xfc,0x02,0x00,0x00, +0xb0,0x00,0x00,0x00,0xb4,0x00,0x00,0x00,0xae,0x01,0x00,0x00, +0xd7,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xf8,0x02,0x00,0x00,0x9e,0x00,0x00,0x00,0xb4,0x00,0x00,0x00, +0xab,0x01,0x00,0x00,0xd7,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xe4,0x02,0x00,0x00,0x84,0x00,0x00,0x00, +0xb4,0x00,0x00,0x00,0x5c,0x02,0x00,0x00,0xd7,0x00,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0xdb,0x00,0x00,0x00, +0xe4,0x02,0x00,0x00,0x8e,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0xd6,0x00,0x00,0x00,0xd7,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0xdb,0x00,0x00,0x00,0xd5,0x00,0x00,0x00, +0xd6,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xd5,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xdd,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xdd,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xf4,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0xd5,0x00,0x00,0x00, +0x3c,0x01,0x00,0x00,0xde,0x00,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc2,0x00,0x00,0x00,0xe3,0x00,0x00,0x00,0xf4,0x02,0x00,0x00, +0x37,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xdf,0x00,0x00,0x00, +0xde,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0xe3,0x00,0x00,0x00,0xde,0x00,0x00,0x00,0xdf,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xde,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xe8,0x00,0x00,0x00,0x73,0x00,0x00,0x00, +0xf4,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xeb,0x00,0x00,0x00,0xe8,0x00,0x00,0x00,0x99,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xec,0x00,0x00,0x00, +0xeb,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xed,0x00,0x00,0x00,0xf8,0x02,0x00,0x00, +0xec,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xef,0x00,0x00,0x00,0xed,0x00,0x00,0x00,0x6e,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xf5,0x00,0x00,0x00, +0xe8,0x00,0x00,0x00,0xf4,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xf7,0x00,0x00,0x00,0x6e,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xf8,0x00,0x00,0x00,0xf5,0x00,0x00,0x00,0xf7,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xfc,0x00,0x00,0x00, +0xef,0x00,0x00,0x00,0xfb,0x00,0x00,0x00,0xc7,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0xef,0x00,0x00,0x00, +0xff,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x01,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, +0x41,0x00,0x07,0x00,0x0d,0x01,0x00,0x00,0x0e,0x01,0x00,0x00, +0x0b,0x01,0x00,0x00,0x34,0x00,0x00,0x00,0xfc,0x00,0x00,0x00, +0x34,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x03,0x01,0x00,0x00, +0x0f,0x01,0x00,0x00,0x0e,0x01,0x00,0x00,0x73,0x00,0x04,0x00, +0xc4,0x00,0x00,0x00,0x10,0x01,0x00,0x00,0x0f,0x01,0x00,0x00, +0x41,0x00,0x08,0x00,0x16,0x01,0x00,0x00,0x17,0x01,0x00,0x00, +0x0b,0x01,0x00,0x00,0x34,0x00,0x00,0x00,0xfc,0x00,0x00,0x00, +0xd0,0x00,0x00,0x00,0x01,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, +0x04,0x01,0x00,0x00,0x18,0x01,0x00,0x00,0x17,0x01,0x00,0x00, +0x72,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x19,0x01,0x00,0x00, +0x18,0x01,0x00,0x00,0x6f,0x00,0x04,0x00,0xc4,0x00,0x00,0x00, +0x1a,0x01,0x00,0x00,0x19,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x1d,0x01,0x00,0x00,0x01,0x01,0x00,0x00, +0x39,0x00,0x00,0x00,0x41,0x00,0x08,0x00,0x16,0x01,0x00,0x00, +0x1e,0x01,0x00,0x00,0x0b,0x01,0x00,0x00,0x34,0x00,0x00,0x00, +0xfc,0x00,0x00,0x00,0xd0,0x00,0x00,0x00,0x1d,0x01,0x00,0x00, +0x3d,0x00,0x04,0x00,0x04,0x01,0x00,0x00,0x1f,0x01,0x00,0x00, +0x1e,0x01,0x00,0x00,0x72,0x00,0x04,0x00,0x13,0x00,0x00,0x00, +0x20,0x01,0x00,0x00,0x1f,0x01,0x00,0x00,0x6f,0x00,0x04,0x00, +0xc4,0x00,0x00,0x00,0x21,0x01,0x00,0x00,0x20,0x01,0x00,0x00, +0x50,0x00,0x05,0x00,0x11,0x01,0x00,0x00,0x22,0x01,0x00,0x00, +0x1a,0x01,0x00,0x00,0x21,0x01,0x00,0x00,0x8e,0x00,0x05,0x00, +0x11,0x01,0x00,0x00,0x24,0x01,0x00,0x00,0x22,0x01,0x00,0x00, +0x10,0x01,0x00,0x00,0x51,0x00,0x05,0x00,0xc4,0x00,0x00,0x00, +0x2c,0x01,0x00,0x00,0x24,0x01,0x00,0x00,0x00,0x00,0x00,0x00, +0x73,0x00,0x04,0x00,0x03,0x01,0x00,0x00,0x2d,0x01,0x00,0x00, +0x2c,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0x2e,0x01,0x00,0x00, +0x2f,0x01,0x00,0x00,0x29,0x01,0x00,0x00,0xf8,0x00,0x00,0x00, +0x3e,0x00,0x03,0x00,0x2f,0x01,0x00,0x00,0x2d,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x31,0x01,0x00,0x00, +0xf8,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x51,0x00,0x05,0x00, +0xc4,0x00,0x00,0x00,0x33,0x01,0x00,0x00,0x24,0x01,0x00,0x00, +0x01,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x03,0x01,0x00,0x00, +0x34,0x01,0x00,0x00,0x33,0x01,0x00,0x00,0x41,0x00,0x05,0x00, +0x2e,0x01,0x00,0x00,0x35,0x01,0x00,0x00,0x29,0x01,0x00,0x00, +0x31,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x35,0x01,0x00,0x00, +0x34,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x3c,0x01,0x00,0x00,0xf4,0x02,0x00,0x00,0x3a,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0xdd,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xdf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x3e,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x3e,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xf5,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, +0xdf,0x00,0x00,0x00,0xa7,0x01,0x00,0x00,0x3f,0x01,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0x44,0x01,0x00,0x00, +0xf5,0x02,0x00,0x00,0xa7,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0x40,0x01,0x00,0x00,0x3f,0x01,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x44,0x01,0x00,0x00,0x3f,0x01,0x00,0x00, +0x40,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x3f,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x49,0x01,0x00,0x00, +0x7e,0x00,0x00,0x00,0xf5,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x4c,0x01,0x00,0x00,0x49,0x01,0x00,0x00, +0xab,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x4d,0x01,0x00,0x00,0x4c,0x01,0x00,0x00,0x77,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x4e,0x01,0x00,0x00, +0xfc,0x02,0x00,0x00,0x4d,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x50,0x01,0x00,0x00,0x4e,0x01,0x00,0x00, +0x79,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x56,0x01,0x00,0x00,0x49,0x01,0x00,0x00,0x55,0x01,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x58,0x01,0x00,0x00, +0x79,0x00,0x00,0x00,0x77,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x59,0x01,0x00,0x00,0x56,0x01,0x00,0x00, +0x58,0x01,0x00,0x00,0x41,0x00,0x08,0x00,0x68,0x01,0x00,0x00, +0x69,0x01,0x00,0x00,0x66,0x01,0x00,0x00,0x34,0x00,0x00,0x00, +0x50,0x01,0x00,0x00,0x34,0x00,0x00,0x00,0x3e,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00,0x6a,0x01,0x00,0x00, +0x69,0x01,0x00,0x00,0x73,0x00,0x04,0x00,0x03,0x01,0x00,0x00, +0x6b,0x01,0x00,0x00,0x6a,0x01,0x00,0x00,0x41,0x00,0x05,0x00, +0x2e,0x01,0x00,0x00,0x6c,0x01,0x00,0x00,0x5e,0x01,0x00,0x00, +0x59,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x6c,0x01,0x00,0x00, +0x6b,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x6e,0x01,0x00,0x00,0x59,0x01,0x00,0x00,0x39,0x00,0x00,0x00, +0x41,0x00,0x08,0x00,0x68,0x01,0x00,0x00,0x70,0x01,0x00,0x00, +0x66,0x01,0x00,0x00,0x34,0x00,0x00,0x00,0x50,0x01,0x00,0x00, +0x34,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0xc4,0x00,0x00,0x00,0x71,0x01,0x00,0x00,0x70,0x01,0x00,0x00, +0x73,0x00,0x04,0x00,0x03,0x01,0x00,0x00,0x72,0x01,0x00,0x00, +0x71,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0x2e,0x01,0x00,0x00, +0x73,0x01,0x00,0x00,0x5e,0x01,0x00,0x00,0x6e,0x01,0x00,0x00, +0x3e,0x00,0x03,0x00,0x73,0x01,0x00,0x00,0x72,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x75,0x01,0x00,0x00, +0x59,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x41,0x00,0x08,0x00, +0x68,0x01,0x00,0x00,0x77,0x01,0x00,0x00,0x66,0x01,0x00,0x00, +0x34,0x00,0x00,0x00,0x50,0x01,0x00,0x00,0x34,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00, +0x78,0x01,0x00,0x00,0x77,0x01,0x00,0x00,0x73,0x00,0x04,0x00, +0x03,0x01,0x00,0x00,0x79,0x01,0x00,0x00,0x78,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0x2e,0x01,0x00,0x00,0x7a,0x01,0x00,0x00, +0x5e,0x01,0x00,0x00,0x75,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, +0x7a,0x01,0x00,0x00,0x79,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x7d,0x01,0x00,0x00,0x59,0x01,0x00,0x00, +0x7c,0x01,0x00,0x00,0x41,0x00,0x08,0x00,0x68,0x01,0x00,0x00, +0x7f,0x01,0x00,0x00,0x66,0x01,0x00,0x00,0x34,0x00,0x00,0x00, +0x50,0x01,0x00,0x00,0x34,0x00,0x00,0x00,0x7c,0x01,0x00,0x00, +0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00,0x80,0x01,0x00,0x00, +0x7f,0x01,0x00,0x00,0x73,0x00,0x04,0x00,0x03,0x01,0x00,0x00, +0x81,0x01,0x00,0x00,0x80,0x01,0x00,0x00,0x41,0x00,0x05,0x00, +0x2e,0x01,0x00,0x00,0x82,0x01,0x00,0x00,0x5e,0x01,0x00,0x00, +0x7d,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x82,0x01,0x00,0x00, +0x81,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x85,0x01,0x00,0x00,0x59,0x01,0x00,0x00,0x84,0x01,0x00,0x00, +0x41,0x00,0x08,0x00,0x68,0x01,0x00,0x00,0x87,0x01,0x00,0x00, +0x66,0x01,0x00,0x00,0x34,0x00,0x00,0x00,0x50,0x01,0x00,0x00, +0xd0,0x00,0x00,0x00,0x3e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0xc4,0x00,0x00,0x00,0x88,0x01,0x00,0x00,0x87,0x01,0x00,0x00, +0x73,0x00,0x04,0x00,0x03,0x01,0x00,0x00,0x89,0x01,0x00,0x00, +0x88,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0x2e,0x01,0x00,0x00, +0x8a,0x01,0x00,0x00,0x5e,0x01,0x00,0x00,0x85,0x01,0x00,0x00, +0x3e,0x00,0x03,0x00,0x8a,0x01,0x00,0x00,0x89,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8d,0x01,0x00,0x00, +0x59,0x01,0x00,0x00,0x8c,0x01,0x00,0x00,0x41,0x00,0x08,0x00, +0x68,0x01,0x00,0x00,0x8f,0x01,0x00,0x00,0x66,0x01,0x00,0x00, +0x34,0x00,0x00,0x00,0x50,0x01,0x00,0x00,0xd0,0x00,0x00,0x00, +0x39,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00, +0x90,0x01,0x00,0x00,0x8f,0x01,0x00,0x00,0x73,0x00,0x04,0x00, +0x03,0x01,0x00,0x00,0x91,0x01,0x00,0x00,0x90,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0x2e,0x01,0x00,0x00,0x92,0x01,0x00,0x00, +0x5e,0x01,0x00,0x00,0x8d,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, +0x92,0x01,0x00,0x00,0x91,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x95,0x01,0x00,0x00,0x59,0x01,0x00,0x00, +0x94,0x01,0x00,0x00,0x41,0x00,0x08,0x00,0x68,0x01,0x00,0x00, +0x97,0x01,0x00,0x00,0x66,0x01,0x00,0x00,0x34,0x00,0x00,0x00, +0x50,0x01,0x00,0x00,0xd0,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00,0x98,0x01,0x00,0x00, +0x97,0x01,0x00,0x00,0x73,0x00,0x04,0x00,0x03,0x01,0x00,0x00, +0x99,0x01,0x00,0x00,0x98,0x01,0x00,0x00,0x41,0x00,0x05,0x00, +0x2e,0x01,0x00,0x00,0x9a,0x01,0x00,0x00,0x5e,0x01,0x00,0x00, +0x95,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x9a,0x01,0x00,0x00, +0x99,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x9d,0x01,0x00,0x00,0x59,0x01,0x00,0x00,0x9c,0x01,0x00,0x00, +0x41,0x00,0x08,0x00,0x68,0x01,0x00,0x00,0x9f,0x01,0x00,0x00, +0x66,0x01,0x00,0x00,0x34,0x00,0x00,0x00,0x50,0x01,0x00,0x00, +0xd0,0x00,0x00,0x00,0x7c,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, +0xc4,0x00,0x00,0x00,0xa0,0x01,0x00,0x00,0x9f,0x01,0x00,0x00, +0x73,0x00,0x04,0x00,0x03,0x01,0x00,0x00,0xa1,0x01,0x00,0x00, +0xa0,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0x2e,0x01,0x00,0x00, +0xa2,0x01,0x00,0x00,0x5e,0x01,0x00,0x00,0x9d,0x01,0x00,0x00, +0x3e,0x00,0x03,0x00,0xa2,0x01,0x00,0x00,0xa1,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa7,0x01,0x00,0x00, +0xf5,0x02,0x00,0x00,0xa5,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0x3e,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x40,0x01,0x00,0x00, +0xe0,0x00,0x04,0x00,0x0c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0xa8,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xab,0x01,0x00,0x00,0xf8,0x02,0x00,0x00,0xa9,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xae,0x01,0x00,0x00, +0xfc,0x02,0x00,0x00,0xac,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0xb0,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xb0,0x01,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xfe,0x02,0x00,0x00, +0x3e,0x00,0x00,0x00,0x40,0x01,0x00,0x00,0x5a,0x02,0x00,0x00, +0xb3,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, +0xb6,0x01,0x00,0x00,0xfe,0x02,0x00,0x00,0x6c,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0xb2,0x01,0x00,0x00,0xb3,0x01,0x00,0x00, +0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xb6,0x01,0x00,0x00, +0xb1,0x01,0x00,0x00,0xb2,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xb1,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xb8,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xb8,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0x02,0x03,0x00,0x00,0x3e,0x00,0x00,0x00, +0xb1,0x01,0x00,0x00,0xe4,0x01,0x00,0x00,0xbb,0x01,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0xbe,0x01,0x00,0x00, +0x02,0x03,0x00,0x00,0x60,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0xba,0x01,0x00,0x00,0xbb,0x01,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0xbe,0x01,0x00,0x00,0xb9,0x01,0x00,0x00, +0xba,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xb9,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0xc0,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xc0,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0x14,0x03,0x00,0x00,0x3e,0x00,0x00,0x00,0xb9,0x01,0x00,0x00, +0xe2,0x01,0x00,0x00,0xc1,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc2,0x00,0x00,0x00,0xc6,0x01,0x00,0x00,0x14,0x03,0x00,0x00, +0x62,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xc2,0x01,0x00,0x00, +0xc1,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0xc6,0x01,0x00,0x00,0xc1,0x01,0x00,0x00,0xc2,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xc1,0x01,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xcc,0x01,0x00,0x00,0x02,0x03,0x00,0x00, +0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xce,0x01,0x00,0x00,0xcc,0x01,0x00,0x00,0x14,0x03,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xd0,0x01,0x00,0x00, +0x55,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xd2,0x01,0x00,0x00,0x02,0x03,0x00,0x00, +0x61,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xd3,0x01,0x00,0x00,0xd0,0x01,0x00,0x00,0xd2,0x01,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xd5,0x01,0x00,0x00, +0x64,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xd6,0x01,0x00,0x00,0xd3,0x01,0x00,0x00, +0xd5,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xd8,0x01,0x00,0x00,0xd6,0x01,0x00,0x00,0x14,0x03,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xda,0x01,0x00,0x00, +0xd8,0x01,0x00,0x00,0xd9,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xdc,0x01,0x00,0x00,0xda,0x01,0x00,0x00, +0xfe,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x2e,0x01,0x00,0x00, +0xdd,0x01,0x00,0x00,0x29,0x01,0x00,0x00,0xdc,0x01,0x00,0x00, +0x3d,0x00,0x04,0x00,0x03,0x01,0x00,0x00,0xde,0x01,0x00,0x00, +0xdd,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0xdf,0x01,0x00,0x00, +0xe0,0x01,0x00,0x00,0xca,0x01,0x00,0x00,0xce,0x01,0x00,0x00, +0x3e,0x00,0x03,0x00,0xe0,0x01,0x00,0x00,0xde,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xe2,0x01,0x00,0x00, +0x14,0x03,0x00,0x00,0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xc0,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xc2,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0xbb,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xbb,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xe4,0x01,0x00,0x00,0x02,0x03,0x00,0x00,0xd0,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xb8,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xba,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xe6,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xe6,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0x03,0x03,0x00,0x00,0x3e,0x00,0x00,0x00, +0xba,0x01,0x00,0x00,0x12,0x02,0x00,0x00,0xe9,0x01,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0xec,0x01,0x00,0x00, +0x03,0x03,0x00,0x00,0xbf,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0xe8,0x01,0x00,0x00,0xe9,0x01,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0xec,0x01,0x00,0x00,0xe7,0x01,0x00,0x00, +0xe8,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xe7,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0xee,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xee,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0x11,0x03,0x00,0x00,0x3e,0x00,0x00,0x00,0xe7,0x01,0x00,0x00, +0x10,0x02,0x00,0x00,0xef,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc2,0x00,0x00,0x00,0xf4,0x01,0x00,0x00,0x11,0x03,0x00,0x00, +0xbc,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xf0,0x01,0x00,0x00, +0xef,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0xf4,0x01,0x00,0x00,0xef,0x01,0x00,0x00,0xf0,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xef,0x01,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xfa,0x01,0x00,0x00,0x03,0x03,0x00,0x00, +0xbc,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xfc,0x01,0x00,0x00,0xfa,0x01,0x00,0x00,0x11,0x03,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xfe,0x01,0x00,0x00, +0x59,0x00,0x00,0x00,0xb9,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x01,0x02,0x00,0x00,0x03,0x03,0x00,0x00, +0x00,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x02,0x02,0x00,0x00,0xfe,0x01,0x00,0x00,0x01,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x04,0x02,0x00,0x00, +0x68,0x00,0x00,0x00,0xbc,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x05,0x02,0x00,0x00,0x02,0x02,0x00,0x00, +0x04,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x07,0x02,0x00,0x00,0x05,0x02,0x00,0x00,0x11,0x03,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x09,0x02,0x00,0x00, +0x07,0x02,0x00,0x00,0x08,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x0b,0x02,0x00,0x00,0x09,0x02,0x00,0x00, +0xfe,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x2e,0x01,0x00,0x00, +0x0c,0x02,0x00,0x00,0x5e,0x01,0x00,0x00,0x0b,0x02,0x00,0x00, +0x3d,0x00,0x04,0x00,0x03,0x01,0x00,0x00,0x0d,0x02,0x00,0x00, +0x0c,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0xdf,0x01,0x00,0x00, +0x0e,0x02,0x00,0x00,0xf8,0x01,0x00,0x00,0xfc,0x01,0x00,0x00, +0x3e,0x00,0x03,0x00,0x0e,0x02,0x00,0x00,0x0d,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x10,0x02,0x00,0x00, +0x11,0x03,0x00,0x00,0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xee,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xf0,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0xe9,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xe9,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x12,0x02,0x00,0x00,0x03,0x03,0x00,0x00,0xd0,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xe6,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xe8,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x14,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x14,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0x04,0x03,0x00,0x00,0x3e,0x00,0x00,0x00, +0xe8,0x01,0x00,0x00,0x58,0x02,0x00,0x00,0x17,0x02,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0x1a,0x02,0x00,0x00, +0x04,0x03,0x00,0x00,0xbf,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0x16,0x02,0x00,0x00,0x17,0x02,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x1a,0x02,0x00,0x00,0x15,0x02,0x00,0x00, +0x16,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x15,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0x1c,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x1c,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0x08,0x03,0x00,0x00,0x3e,0x00,0x00,0x00,0x15,0x02,0x00,0x00, +0x56,0x02,0x00,0x00,0x1f,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc2,0x00,0x00,0x00,0x22,0x02,0x00,0x00,0x08,0x03,0x00,0x00, +0x60,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x1e,0x02,0x00,0x00, +0x1f,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x22,0x02,0x00,0x00,0x1d,0x02,0x00,0x00,0x1e,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x1d,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0x24,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x24,0x02,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x0a,0x03,0x00,0x00, +0x3e,0x00,0x00,0x00,0x1d,0x02,0x00,0x00,0x54,0x02,0x00,0x00, +0x27,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, +0x2a,0x02,0x00,0x00,0x0a,0x03,0x00,0x00,0xbc,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0x26,0x02,0x00,0x00,0x27,0x02,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x2a,0x02,0x00,0x00, +0x25,0x02,0x00,0x00,0x26,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x25,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x2c,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x2c,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0x0c,0x03,0x00,0x00,0x3e,0x00,0x00,0x00, +0x25,0x02,0x00,0x00,0x52,0x02,0x00,0x00,0x2d,0x02,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0x32,0x02,0x00,0x00, +0x0c,0x03,0x00,0x00,0x62,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0x2e,0x02,0x00,0x00,0x2d,0x02,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x32,0x02,0x00,0x00,0x2d,0x02,0x00,0x00, +0x2e,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x2d,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x34,0x02,0x00,0x00, +0x04,0x03,0x00,0x00,0xbc,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x36,0x02,0x00,0x00,0x34,0x02,0x00,0x00, +0x0a,0x03,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x38,0x02,0x00,0x00,0x36,0x02,0x00,0x00,0x37,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3a,0x02,0x00,0x00, +0x08,0x03,0x00,0x00,0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x3b,0x02,0x00,0x00,0x38,0x02,0x00,0x00, +0x3a,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x3d,0x02,0x00,0x00,0x3b,0x02,0x00,0x00,0x0c,0x03,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x41,0x02,0x00,0x00, +0x3a,0x02,0x00,0x00,0x0c,0x03,0x00,0x00,0x41,0x00,0x05,0x00, +0xdf,0x01,0x00,0x00,0x42,0x02,0x00,0x00,0xca,0x01,0x00,0x00, +0x41,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0x03,0x01,0x00,0x00, +0x43,0x02,0x00,0x00,0x42,0x02,0x00,0x00,0x73,0x00,0x04,0x00, +0xc4,0x00,0x00,0x00,0x44,0x02,0x00,0x00,0x43,0x02,0x00,0x00, +0x41,0x00,0x05,0x00,0xdf,0x01,0x00,0x00,0x49,0x02,0x00,0x00, +0xf8,0x01,0x00,0x00,0x36,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, +0x03,0x01,0x00,0x00,0x4a,0x02,0x00,0x00,0x49,0x02,0x00,0x00, +0x73,0x00,0x04,0x00,0xc4,0x00,0x00,0x00,0x4b,0x02,0x00,0x00, +0x4a,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0xcd,0x00,0x00,0x00, +0x4d,0x02,0x00,0x00,0xca,0x00,0x00,0x00,0x3d,0x02,0x00,0x00, +0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00,0x4e,0x02,0x00,0x00, +0x4d,0x02,0x00,0x00,0x0c,0x00,0x08,0x00,0xc4,0x00,0x00,0x00, +0x4f,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00, +0x44,0x02,0x00,0x00,0x4b,0x02,0x00,0x00,0x4e,0x02,0x00,0x00, +0x3e,0x00,0x03,0x00,0x4d,0x02,0x00,0x00,0x4f,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x52,0x02,0x00,0x00, +0x0c,0x03,0x00,0x00,0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x2c,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x2e,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0x27,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x27,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x54,0x02,0x00,0x00,0x0a,0x03,0x00,0x00,0xd0,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x24,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x26,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x1f,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x1f,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x56,0x02,0x00,0x00,0x08,0x03,0x00,0x00, +0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x1c,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x1e,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0x17,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x17,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x58,0x02,0x00,0x00, +0x04,0x03,0x00,0x00,0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x14,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x16,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0xb3,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xb3,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x5a,0x02,0x00,0x00,0xfe,0x02,0x00,0x00,0xd0,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xb0,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xb2,0x01,0x00,0x00,0xe0,0x00,0x04,0x00,0x0c,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0xa8,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0xd7,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xd7,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x5c,0x02,0x00,0x00, +0xe4,0x02,0x00,0x00,0x6c,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xd4,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xd6,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x61,0x02,0x00,0x00, +0x55,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x62,0x02,0x00,0x00,0x96,0x00,0x00,0x00, +0x61,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x67,0x02,0x00,0x00,0x59,0x00,0x00,0x00,0xb9,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x68,0x02,0x00,0x00, +0xa8,0x00,0x00,0x00,0x67,0x02,0x00,0x00,0x41,0x00,0x05,0x00, +0x15,0x00,0x00,0x00,0x6d,0x02,0x00,0x00,0x12,0x00,0x00,0x00, +0x6c,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x6e,0x02,0x00,0x00,0x6d,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x6f,0x02,0x00,0x00,0x0f,0x00,0x00,0x00, +0x6e,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x73,0x02,0x00,0x00,0x47,0x00,0x00,0x00,0x6e,0x02,0x00,0x00, +0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00,0x75,0x02,0x00,0x00, +0x74,0x02,0x00,0x00,0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x76,0x02,0x00,0x00,0x75,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x77,0x02,0x00,0x00, +0x73,0x02,0x00,0x00,0x76,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x78,0x02,0x00,0x00,0x6f,0x02,0x00,0x00, +0x77,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x7a,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x7a,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xe5,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, +0xd6,0x00,0x00,0x00,0xe0,0x02,0x00,0x00,0x7d,0x02,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0x80,0x02,0x00,0x00, +0xe5,0x02,0x00,0x00,0xbf,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0x7c,0x02,0x00,0x00,0x7d,0x02,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x80,0x02,0x00,0x00,0x7b,0x02,0x00,0x00, +0x7c,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x7b,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0x82,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x82,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xe6,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0x7b,0x02,0x00,0x00, +0xde,0x02,0x00,0x00,0x85,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc2,0x00,0x00,0x00,0x88,0x02,0x00,0x00,0xe6,0x02,0x00,0x00, +0x60,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x84,0x02,0x00,0x00, +0x85,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x88,0x02,0x00,0x00,0x83,0x02,0x00,0x00,0x84,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x83,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x8c,0x02,0x00,0x00,0xe6,0x02,0x00,0x00, +0x61,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x8d,0x02,0x00,0x00,0x62,0x02,0x00,0x00,0x8c,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8f,0x02,0x00,0x00, +0x64,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x90,0x02,0x00,0x00,0x8d,0x02,0x00,0x00, +0x8f,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x94,0x02,0x00,0x00,0xe5,0x02,0x00,0x00,0x00,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x95,0x02,0x00,0x00, +0x68,0x02,0x00,0x00,0x94,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x97,0x02,0x00,0x00,0x68,0x00,0x00,0x00, +0xbc,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x98,0x02,0x00,0x00,0x95,0x02,0x00,0x00,0x97,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0x9a,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x9a,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xe8,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0x83,0x02,0x00,0x00, +0xdc,0x02,0x00,0x00,0x9d,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc2,0x00,0x00,0x00,0xa0,0x02,0x00,0x00,0xe8,0x02,0x00,0x00, +0xbc,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x9c,0x02,0x00,0x00, +0x9d,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0xa0,0x02,0x00,0x00,0x9b,0x02,0x00,0x00,0x9c,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x9b,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0xa2,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0xa2,0x02,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xea,0x02,0x00,0x00, +0x3e,0x00,0x00,0x00,0x9b,0x02,0x00,0x00,0xda,0x02,0x00,0x00, +0xa5,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, +0xa8,0x02,0x00,0x00,0xea,0x02,0x00,0x00,0x62,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0xa4,0x02,0x00,0x00,0xa5,0x02,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xa8,0x02,0x00,0x00, +0xa3,0x02,0x00,0x00,0xa4,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0xa3,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xab,0x02,0x00,0x00,0x90,0x02,0x00,0x00,0xea,0x02,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0xae,0x02,0x00,0x00, +0xab,0x02,0x00,0x00,0x36,0x00,0x00,0x00,0xf7,0x00,0x03,0x00, +0xb0,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0xae,0x02,0x00,0x00,0xaf,0x02,0x00,0x00,0xb0,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0xaf,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xb3,0x02,0x00,0x00,0x98,0x02,0x00,0x00, +0xe8,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, +0xb4,0x02,0x00,0x00,0x12,0x00,0x00,0x00,0xd0,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xb5,0x02,0x00,0x00, +0xb4,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, +0xb6,0x02,0x00,0x00,0xb3,0x02,0x00,0x00,0xb5,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0xb0,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0xb0,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0xc2,0x00,0x00,0x00, +0xb7,0x02,0x00,0x00,0xae,0x02,0x00,0x00,0xa3,0x02,0x00,0x00, +0xb6,0x02,0x00,0x00,0xaf,0x02,0x00,0x00,0xf7,0x00,0x03,0x00, +0xb9,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0xb7,0x02,0x00,0x00,0xb8,0x02,0x00,0x00,0xb9,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0xb8,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xc1,0x02,0x00,0x00,0x98,0x02,0x00,0x00, +0xe8,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, +0xc3,0x02,0x00,0x00,0x12,0x00,0x00,0x00,0xc2,0x02,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xc4,0x02,0x00,0x00, +0xc3,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xc5,0x02,0x00,0x00,0xc1,0x02,0x00,0x00,0xc4,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xc6,0x02,0x00,0x00, +0x78,0x02,0x00,0x00,0xc5,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xc8,0x02,0x00,0x00,0xc6,0x02,0x00,0x00, +0x90,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xca,0x02,0x00,0x00,0xc8,0x02,0x00,0x00,0xea,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xcc,0x02,0x00,0x00, +0xe5,0x02,0x00,0x00,0xbc,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xce,0x02,0x00,0x00,0xcc,0x02,0x00,0x00, +0xe8,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xd0,0x02,0x00,0x00,0xce,0x02,0x00,0x00,0xcf,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xd2,0x02,0x00,0x00, +0xe6,0x02,0x00,0x00,0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xd3,0x02,0x00,0x00,0xd0,0x02,0x00,0x00, +0xd2,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xd5,0x02,0x00,0x00,0xd3,0x02,0x00,0x00,0xea,0x02,0x00,0x00, +0x41,0x00,0x05,0x00,0xcd,0x00,0x00,0x00,0xd6,0x02,0x00,0x00, +0xca,0x00,0x00,0x00,0xd5,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, +0xc4,0x00,0x00,0x00,0xd7,0x02,0x00,0x00,0xd6,0x02,0x00,0x00, +0x41,0x00,0x06,0x00,0x68,0x01,0x00,0x00,0xd8,0x02,0x00,0x00, +0xbd,0x02,0x00,0x00,0x34,0x00,0x00,0x00,0xca,0x02,0x00,0x00, +0x3e,0x00,0x03,0x00,0xd8,0x02,0x00,0x00,0xd7,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0xb9,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0xb9,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0xa5,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0xa5,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xda,0x02,0x00,0x00,0xea,0x02,0x00,0x00, +0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xa2,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0xa4,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0x9d,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x9d,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xdc,0x02,0x00,0x00, +0xe8,0x02,0x00,0x00,0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x9a,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x9c,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0x85,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x85,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xde,0x02,0x00,0x00,0xe6,0x02,0x00,0x00,0xd0,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x82,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x84,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x7d,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x7d,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xe0,0x02,0x00,0x00,0xe5,0x02,0x00,0x00, +0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x7a,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x7c,0x02,0x00,0x00,0xfd,0x00,0x01,0x00, +0x38,0x00,0x01,0x00, +}; +const uint64_t matmul_q8_0_f32_aligned_len = 11200; + +unsigned char matmul_q8_0_f32_aligned_fp32_data[] = { +0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, +0xeb,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, +0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x51,0x11,0x00,0x00, +0x11,0x00,0x02,0x00,0x60,0x11,0x00,0x00,0x0b,0x00,0x06,0x00, +0x01,0x00,0x00,0x00,0x47,0x4c,0x53,0x4c,0x2e,0x73,0x74,0x64, +0x2e,0x34,0x35,0x30,0x00,0x00,0x00,0x00,0x0e,0x00,0x03,0x00, +0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x0f,0x00,0x0f,0x00, +0x05,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x6d,0x61,0x69,0x6e, +0x00,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0x3d,0x00,0x00,0x00,0x4c,0x00,0x00,0x00,0x0b,0x01,0x00,0x00, +0x29,0x01,0x00,0x00,0x5c,0x01,0x00,0x00,0x63,0x01,0x00,0x00, +0x4a,0x02,0x00,0x00,0x93,0x02,0x00,0x00,0x10,0x00,0x06,0x00, +0x04,0x00,0x00,0x00,0x11,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x0b,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x1c,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x10,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x08,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x03,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x10,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x10,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, +0x05,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x14,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x10,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x1c,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, +0x08,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x09,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x24,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x10,0x00,0x00,0x00,0x0a,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x28,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, +0x0b,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x2c,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x30,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x10,0x00,0x00,0x00,0x0d,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x34,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x10,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x37,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x3d,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x1a,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x4c,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, +0x1b,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x4f,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x53,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x60,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x62,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x6c,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x03,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0xa7,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xb9,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0xbc,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x08,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x06,0x01,0x00,0x00,0x06,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x07,0x01,0x00,0x00, +0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x07,0x01,0x00,0x00,0x01,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x08,0x01,0x00,0x00,0x06,0x00,0x00,0x00,0x22,0x00,0x00,0x00, +0x48,0x00,0x04,0x00,0x09,0x01,0x00,0x00,0x00,0x00,0x00,0x00, +0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x09,0x01,0x00,0x00, +0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x03,0x00,0x09,0x01,0x00,0x00,0x02,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x0b,0x01,0x00,0x00,0x22,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x0b,0x01,0x00,0x00, +0x21,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x34,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x35,0x01,0x00,0x00,0x0b,0x00,0x00,0x00, +0x19,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x60,0x01,0x00,0x00, +0x06,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x48,0x00,0x04,0x00, +0x61,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x61,0x01,0x00,0x00,0x00,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, +0x61,0x01,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x63,0x01,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x63,0x01,0x00,0x00,0x21,0x00,0x00,0x00, 0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x4a,0x02,0x00,0x00, 0x0b,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x47,0x00,0x04,0x00, 0x90,0x02,0x00,0x00,0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00, @@ -33313,136 +36461,147 @@ unsigned char matmul_f32_aligned_l_fp32_data[] = { 0x06,0x00,0x00,0x00,0x67,0x00,0x00,0x00,0x86,0x00,0x00,0x00, 0x61,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x32,0x00,0x04,0x00, 0x06,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x10,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x72,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x77,0x00,0x00,0x00, 0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x6e,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x6d,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x73,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x6d,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, -0x77,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0x7c,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x87,0x00,0x00,0x00, +0x78,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, +0x77,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x7d,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, +0x77,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, +0x81,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x91,0x00,0x00,0x00, 0x0b,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, -0x8d,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0x98,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x9d,0x00,0x00,0x00, +0x97,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0xa2,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xa7,0x00,0x00,0x00, 0x40,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, -0x9f,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xae,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0xa9,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xb8,0x00,0x00,0x00,0x84,0x00,0x00,0x00, 0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x32,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xaf,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xb0,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0xaf,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xb1,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x62,0x00,0x00,0x00, -0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xb2,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xb3,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0xb1,0x00,0x00,0x00, -0xb2,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xb4,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0xb3,0x00,0x00,0x00, -0x60,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xb5,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0xb0,0x00,0x00,0x00, -0xb4,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xb6,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0xae,0x00,0x00,0x00, -0xb5,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xb7,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0xb6,0x00,0x00,0x00, -0xb2,0x00,0x00,0x00,0x14,0x00,0x02,0x00,0xb8,0x00,0x00,0x00, -0x16,0x00,0x03,0x00,0xba,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0xb9,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xba,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0xb9,0x00,0x00,0x00, 0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xbb,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x62,0x00,0x00,0x00, +0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xbc,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xbd,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0xbb,0x00,0x00,0x00, +0xbc,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xbe,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0xbd,0x00,0x00,0x00, +0x60,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xbf,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0xba,0x00,0x00,0x00, +0xbe,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xc0,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0xb8,0x00,0x00,0x00, +0xbf,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xc1,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0xc0,0x00,0x00,0x00, +0xbc,0x00,0x00,0x00,0x14,0x00,0x02,0x00,0xc2,0x00,0x00,0x00, +0x16,0x00,0x03,0x00,0xc4,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xc5,0x00,0x00,0x00, 0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xbc,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0xbb,0x00,0x00,0x00,0xb5,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xbd,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0xbc,0x00,0x00,0x00,0xb2,0x00,0x00,0x00, -0x1c,0x00,0x04,0x00,0xbe,0x00,0x00,0x00,0xba,0x00,0x00,0x00, -0xbd,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0xbf,0x00,0x00,0x00, -0x07,0x00,0x00,0x00,0xbe,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0xba,0x00,0x00,0x00,0xc2,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0xc3,0x00,0x00,0x00,0x07,0x00,0x00,0x00, -0xba,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, -0xc6,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xe6,0x00,0x00,0x00,0x80,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xe7,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0x37,0x00,0x00,0x00,0xe6,0x00,0x00,0x00,0x1c,0x00,0x04,0x00, -0xe8,0x00,0x00,0x00,0xba,0x00,0x00,0x00,0xe7,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0xe9,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0xe8,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0xe9,0x00,0x00,0x00, -0xea,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xee,0x00,0x00,0x00,0x80,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x17,0x00,0x04,0x00, -0xf4,0x00,0x00,0x00,0xba,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x1d,0x00,0x03,0x00,0xf5,0x00,0x00,0x00,0xf4,0x00,0x00,0x00, -0x1e,0x00,0x03,0x00,0xf6,0x00,0x00,0x00,0xf5,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0xf7,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0xf6,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0xf7,0x00,0x00,0x00, -0xf8,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0xfa,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0xba,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0xfd,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0xba,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x02,0x01,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x0f,0x01,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x1c,0x01,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x21,0x01,0x00,0x00,0x03,0x00,0x00,0x00,0x32,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x27,0x01,0x00,0x00,0x01,0x00,0x00,0x00, -0x33,0x00,0x06,0x00,0x09,0x00,0x00,0x00,0x28,0x01,0x00,0x00, -0x27,0x01,0x00,0x00,0x39,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x29,0x01,0x00,0x00, -0x51,0x00,0x00,0x00,0x28,0x01,0x00,0x00,0x00,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x2a,0x01,0x00,0x00, -0x84,0x00,0x00,0x00,0x29,0x01,0x00,0x00,0x6d,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x2b,0x01,0x00,0x00, -0x86,0x00,0x00,0x00,0x2a,0x01,0x00,0x00,0x6c,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x42,0x01,0x00,0x00, -0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x43,0x01,0x00,0x00, -0x84,0x00,0x00,0x00,0x9d,0x00,0x00,0x00,0x42,0x01,0x00,0x00, -0x1c,0x00,0x04,0x00,0x44,0x01,0x00,0x00,0xba,0x00,0x00,0x00, -0x43,0x01,0x00,0x00,0x20,0x00,0x04,0x00,0x45,0x01,0x00,0x00, -0x04,0x00,0x00,0x00,0x44,0x01,0x00,0x00,0x3b,0x00,0x04,0x00, -0x45,0x01,0x00,0x00,0x46,0x01,0x00,0x00,0x04,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x4a,0x01,0x00,0x00, -0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x1d,0x00,0x03,0x00,0x50,0x01,0x00,0x00,0xf4,0x00,0x00,0x00, -0x1e,0x00,0x03,0x00,0x51,0x01,0x00,0x00,0x50,0x01,0x00,0x00, -0x20,0x00,0x04,0x00,0x52,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, -0x51,0x01,0x00,0x00,0x3b,0x00,0x04,0x00,0x52,0x01,0x00,0x00, -0x53,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x5b,0x01,0x00,0x00,0x80,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x68,0x01,0x00,0x00,0x80,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x75,0x01,0x00,0x00,0x80,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xc6,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0xc5,0x00,0x00,0x00,0xbf,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xc7,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0xc6,0x00,0x00,0x00,0xbc,0x00,0x00,0x00, +0x1c,0x00,0x04,0x00,0xc8,0x00,0x00,0x00,0xc4,0x00,0x00,0x00, +0xc7,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0xc9,0x00,0x00,0x00, +0x07,0x00,0x00,0x00,0xc8,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0xc4,0x00,0x00,0x00,0xcc,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0xcd,0x00,0x00,0x00,0x07,0x00,0x00,0x00, +0xc4,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, +0xd0,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xf4,0x00,0x00,0x00,0x80,0x00,0x00,0x00, 0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xfb,0x00,0x00,0x00,0x10,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xff,0x00,0x00,0x00, +0x0f,0x00,0x00,0x00,0x16,0x00,0x03,0x00,0x03,0x01,0x00,0x00, +0x10,0x00,0x00,0x00,0x15,0x00,0x04,0x00,0x04,0x01,0x00,0x00, +0x08,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x05,0x01,0x00,0x00,0x20,0x00,0x00,0x00, +0x1c,0x00,0x04,0x00,0x06,0x01,0x00,0x00,0x04,0x01,0x00,0x00, +0x05,0x01,0x00,0x00,0x1e,0x00,0x04,0x00,0x07,0x01,0x00,0x00, +0x03,0x01,0x00,0x00,0x06,0x01,0x00,0x00,0x1d,0x00,0x03,0x00, +0x08,0x01,0x00,0x00,0x07,0x01,0x00,0x00,0x1e,0x00,0x03,0x00, +0x09,0x01,0x00,0x00,0x08,0x01,0x00,0x00,0x20,0x00,0x04,0x00, +0x0a,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x09,0x01,0x00,0x00, +0x3b,0x00,0x04,0x00,0x0a,0x01,0x00,0x00,0x0b,0x01,0x00,0x00, +0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x0d,0x01,0x00,0x00, +0x0c,0x00,0x00,0x00,0x03,0x01,0x00,0x00,0x17,0x00,0x04,0x00, +0x11,0x01,0x00,0x00,0xc4,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x16,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, +0x04,0x01,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x25,0x01,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, +0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x26,0x01,0x00,0x00,0x84,0x00,0x00,0x00,0x37,0x00,0x00,0x00, +0x25,0x01,0x00,0x00,0x1c,0x00,0x04,0x00,0x27,0x01,0x00,0x00, +0xc4,0x00,0x00,0x00,0x26,0x01,0x00,0x00,0x20,0x00,0x04,0x00, +0x28,0x01,0x00,0x00,0x04,0x00,0x00,0x00,0x27,0x01,0x00,0x00, +0x3b,0x00,0x04,0x00,0x28,0x01,0x00,0x00,0x29,0x01,0x00,0x00, +0x04,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x2d,0x01,0x00,0x00, +0x04,0x00,0x00,0x00,0xc4,0x00,0x00,0x00,0x32,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x34,0x01,0x00,0x00,0x01,0x00,0x00,0x00, +0x33,0x00,0x06,0x00,0x09,0x00,0x00,0x00,0x35,0x01,0x00,0x00, +0x34,0x01,0x00,0x00,0x39,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x36,0x01,0x00,0x00, +0x51,0x00,0x00,0x00,0x35,0x01,0x00,0x00,0x00,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x37,0x01,0x00,0x00, +0x84,0x00,0x00,0x00,0x36,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x38,0x01,0x00,0x00, +0x86,0x00,0x00,0x00,0x37,0x01,0x00,0x00,0x6c,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x53,0x01,0x00,0x00, +0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x58,0x01,0x00,0x00, +0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x59,0x01,0x00,0x00, +0x84,0x00,0x00,0x00,0xa7,0x00,0x00,0x00,0x58,0x01,0x00,0x00, +0x1c,0x00,0x04,0x00,0x5a,0x01,0x00,0x00,0xc4,0x00,0x00,0x00, +0x59,0x01,0x00,0x00,0x20,0x00,0x04,0x00,0x5b,0x01,0x00,0x00, +0x04,0x00,0x00,0x00,0x5a,0x01,0x00,0x00,0x3b,0x00,0x04,0x00, +0x5b,0x01,0x00,0x00,0x5c,0x01,0x00,0x00,0x04,0x00,0x00,0x00, +0x17,0x00,0x04,0x00,0x5f,0x01,0x00,0x00,0xc4,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x1d,0x00,0x03,0x00,0x60,0x01,0x00,0x00, +0x5f,0x01,0x00,0x00,0x1e,0x00,0x03,0x00,0x61,0x01,0x00,0x00, +0x60,0x01,0x00,0x00,0x20,0x00,0x04,0x00,0x62,0x01,0x00,0x00, +0x0c,0x00,0x00,0x00,0x61,0x01,0x00,0x00,0x3b,0x00,0x04,0x00, +0x62,0x01,0x00,0x00,0x63,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x65,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, +0xc4,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x76,0x01,0x00,0x00,0x03,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x7c,0x01,0x00,0x00,0x51,0x00,0x00,0x00, +0x35,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x7d,0x01,0x00,0x00,0x84,0x00,0x00,0x00, +0x7c,0x01,0x00,0x00,0x77,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x7e,0x01,0x00,0x00,0x86,0x00,0x00,0x00, +0x7d,0x01,0x00,0x00,0x6c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, 0x06,0x00,0x00,0x00,0x81,0x01,0x00,0x00,0x08,0x01,0x00,0x00, 0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x82,0x01,0x00,0x00, -0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, 0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x85,0x01,0x00,0x00, -0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x77,0x00,0x00,0x00, 0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xa0,0x01,0x00,0x00, 0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00, -0x1c,0x00,0x04,0x00,0xa1,0x01,0x00,0x00,0xba,0x00,0x00,0x00, +0x1c,0x00,0x04,0x00,0xa1,0x01,0x00,0x00,0xc4,0x00,0x00,0x00, 0xa0,0x01,0x00,0x00,0x20,0x00,0x04,0x00,0xa2,0x01,0x00,0x00, 0x07,0x00,0x00,0x00,0xa1,0x01,0x00,0x00,0x34,0x00,0x06,0x00, 0x06,0x00,0x00,0x00,0xb2,0x01,0x00,0x00,0x80,0x00,0x00,0x00, 0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, 0x06,0x00,0x00,0x00,0xcd,0x01,0x00,0x00,0x84,0x00,0x00,0x00, -0xb5,0x00,0x00,0x00,0xb2,0x00,0x00,0x00,0x1c,0x00,0x04,0x00, -0xce,0x01,0x00,0x00,0xba,0x00,0x00,0x00,0xcd,0x01,0x00,0x00, +0xbf,0x00,0x00,0x00,0xbc,0x00,0x00,0x00,0x1c,0x00,0x04,0x00, +0xce,0x01,0x00,0x00,0xc4,0x00,0x00,0x00,0xcd,0x01,0x00,0x00, 0x20,0x00,0x04,0x00,0xcf,0x01,0x00,0x00,0x07,0x00,0x00,0x00, 0xce,0x01,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xd8,0x01,0x00,0x00,0x86,0x00,0x00,0x00,0xaf,0x00,0x00,0x00, -0xb5,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xd8,0x01,0x00,0x00,0x86,0x00,0x00,0x00,0xb9,0x00,0x00,0x00, +0xbf,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, 0xe0,0x01,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, 0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, 0x0f,0x02,0x00,0x00,0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00, 0x62,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, 0x42,0x02,0x00,0x00,0x0d,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, 0x0a,0x00,0x00,0x00,0x4a,0x02,0x00,0x00,0x01,0x00,0x00,0x00, -0x1d,0x00,0x03,0x00,0x90,0x02,0x00,0x00,0xba,0x00,0x00,0x00, +0x1d,0x00,0x03,0x00,0x90,0x02,0x00,0x00,0xc4,0x00,0x00,0x00, 0x1e,0x00,0x03,0x00,0x91,0x02,0x00,0x00,0x90,0x02,0x00,0x00, 0x20,0x00,0x04,0x00,0x92,0x02,0x00,0x00,0x0c,0x00,0x00,0x00, 0x91,0x02,0x00,0x00,0x3b,0x00,0x04,0x00,0x92,0x02,0x00,0x00, @@ -33452,8 +36611,8 @@ unsigned char matmul_f32_aligned_l_fp32_data[] = { 0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00, 0x36,0x00,0x05,0x00,0x02,0x00,0x00,0x00,0x04,0x00,0x00,0x00, 0x00,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x05,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0xbf,0x00,0x00,0x00, -0xc0,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0x05,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0xc9,0x00,0x00,0x00, +0xca,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, 0xa2,0x01,0x00,0x00,0xa3,0x01,0x00,0x00,0x07,0x00,0x00,0x00, 0x3b,0x00,0x04,0x00,0xcf,0x01,0x00,0x00,0xd0,0x01,0x00,0x00, 0x07,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00, @@ -33512,1633 +36671,612 @@ unsigned char matmul_f32_aligned_l_fp32_data[] = { 0x64,0x00,0x00,0x00,0x5d,0x00,0x00,0x00,0x63,0x00,0x00,0x00, 0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x68,0x00,0x00,0x00, 0x5d,0x00,0x00,0x00,0x67,0x00,0x00,0x00,0x89,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x6f,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, -0x6e,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x74,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x73,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x78,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x77,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x79,0x00,0x00,0x00,0x78,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x7a,0x00,0x00,0x00, -0x47,0x00,0x00,0x00,0x79,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x7d,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x7c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x7e,0x00,0x00,0x00,0x7d,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x80,0x00,0x00,0x00,0x47,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x83,0x00,0x00,0x00,0x80,0x00,0x00,0x00,0x79,0x00,0x00,0x00, -0x0c,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x26,0x00,0x00,0x00,0x7e,0x00,0x00,0x00, -0x83,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0x88,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x87,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x89,0x00,0x00,0x00, -0x88,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x8a,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x89,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8c,0x00,0x00,0x00, -0x42,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x8e,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x8d,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x8f,0x00,0x00,0x00,0x8e,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x90,0x00,0x00,0x00,0x8c,0x00,0x00,0x00, -0x8f,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x91,0x00,0x00,0x00,0x8a,0x00,0x00,0x00,0x90,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x93,0x00,0x00,0x00, -0x91,0x00,0x00,0x00,0x7a,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x94,0x00,0x00,0x00,0x93,0x00,0x00,0x00, -0x6d,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0x99,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x98,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x9a,0x00,0x00,0x00, -0x99,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x9b,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x9a,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9e,0x00,0x00,0x00, -0x4a,0x00,0x00,0x00,0x9d,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0xa0,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x9f,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xa1,0x00,0x00,0x00,0xa0,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xa2,0x00,0x00,0x00,0x9e,0x00,0x00,0x00, -0xa1,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xa3,0x00,0x00,0x00,0x9b,0x00,0x00,0x00,0xa2,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa5,0x00,0x00,0x00, -0xa3,0x00,0x00,0x00,0x7a,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xa6,0x00,0x00,0x00,0xa5,0x00,0x00,0x00, -0x6d,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xa8,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xa8,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xb7,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, -0x05,0x00,0x00,0x00,0xc7,0x00,0x00,0x00,0xa9,0x00,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00,0xb9,0x00,0x00,0x00, -0xb7,0x02,0x00,0x00,0xb7,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0xaa,0x00,0x00,0x00,0xa9,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xb9,0x00,0x00,0x00,0xa9,0x00,0x00,0x00, -0xaa,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xa9,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0xc3,0x00,0x00,0x00,0xc4,0x00,0x00,0x00, -0xc0,0x00,0x00,0x00,0xb7,0x02,0x00,0x00,0x3e,0x00,0x03,0x00, -0xc4,0x00,0x00,0x00,0xc2,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xc7,0x00,0x00,0x00,0xb7,0x02,0x00,0x00, -0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xa8,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xaa,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xca,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xca,0x00,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xd0,0x02,0x00,0x00, -0xa6,0x00,0x00,0x00,0xaa,0x00,0x00,0x00,0x87,0x01,0x00,0x00, -0xcd,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xcc,0x02,0x00,0x00,0x94,0x00,0x00,0x00,0xaa,0x00,0x00,0x00, -0x84,0x01,0x00,0x00,0xcd,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xb8,0x02,0x00,0x00,0x7a,0x00,0x00,0x00, -0xaa,0x00,0x00,0x00,0x32,0x02,0x00,0x00,0xcd,0x00,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00,0xd1,0x00,0x00,0x00, -0xb8,0x02,0x00,0x00,0x84,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0xcc,0x00,0x00,0x00,0xcd,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xd1,0x00,0x00,0x00,0xcb,0x00,0x00,0x00, -0xcc,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xcb,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xd3,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xd3,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xc8,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0xcb,0x00,0x00,0x00, -0x2d,0x01,0x00,0x00,0xd4,0x00,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb8,0x00,0x00,0x00,0xd9,0x00,0x00,0x00,0xc8,0x02,0x00,0x00, -0x37,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xd5,0x00,0x00,0x00, -0xd4,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0xd9,0x00,0x00,0x00,0xd4,0x00,0x00,0x00,0xd5,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xd4,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xde,0x00,0x00,0x00,0x74,0x00,0x00,0x00, -0xc8,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xe1,0x00,0x00,0x00,0xde,0x00,0x00,0x00,0x8f,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xe2,0x00,0x00,0x00, -0xe1,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xe3,0x00,0x00,0x00,0xcc,0x02,0x00,0x00, -0xe2,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xe5,0x00,0x00,0x00,0xe3,0x00,0x00,0x00,0x6f,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xef,0x00,0x00,0x00, -0xde,0x00,0x00,0x00,0xee,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xf1,0x00,0x00,0x00,0x6f,0x00,0x00,0x00, -0x6d,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xf2,0x00,0x00,0x00,0xef,0x00,0x00,0x00,0xf1,0x00,0x00,0x00, -0x41,0x00,0x07,0x00,0xfa,0x00,0x00,0x00,0xfb,0x00,0x00,0x00, -0xf8,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0xe5,0x00,0x00,0x00, -0x3e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0xba,0x00,0x00,0x00, -0xfc,0x00,0x00,0x00,0xfb,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0xfd,0x00,0x00,0x00,0xfe,0x00,0x00,0x00,0xea,0x00,0x00,0x00, -0xf2,0x00,0x00,0x00,0x3e,0x00,0x03,0x00,0xfe,0x00,0x00,0x00, -0xfc,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x03,0x01,0x00,0x00,0xde,0x00,0x00,0x00,0x02,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x06,0x01,0x00,0x00, -0x03,0x01,0x00,0x00,0xf1,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x07,0x01,0x00,0x00,0x06,0x01,0x00,0x00, -0x39,0x00,0x00,0x00,0x41,0x00,0x07,0x00,0xfa,0x00,0x00,0x00, -0x09,0x01,0x00,0x00,0xf8,0x00,0x00,0x00,0x34,0x00,0x00,0x00, -0xe5,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0xba,0x00,0x00,0x00,0x0a,0x01,0x00,0x00,0x09,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0xfd,0x00,0x00,0x00,0x0b,0x01,0x00,0x00, -0xea,0x00,0x00,0x00,0x07,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0x0b,0x01,0x00,0x00,0x0a,0x01,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x10,0x01,0x00,0x00,0xde,0x00,0x00,0x00, -0x0f,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x13,0x01,0x00,0x00,0x10,0x01,0x00,0x00,0xf1,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x14,0x01,0x00,0x00, -0x13,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x41,0x00,0x07,0x00, -0xfa,0x00,0x00,0x00,0x16,0x01,0x00,0x00,0xf8,0x00,0x00,0x00, -0x34,0x00,0x00,0x00,0xe5,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0xba,0x00,0x00,0x00,0x17,0x01,0x00,0x00, -0x16,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0xfd,0x00,0x00,0x00, -0x18,0x01,0x00,0x00,0xea,0x00,0x00,0x00,0x14,0x01,0x00,0x00, -0x3e,0x00,0x03,0x00,0x18,0x01,0x00,0x00,0x17,0x01,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x1d,0x01,0x00,0x00, -0xde,0x00,0x00,0x00,0x1c,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x20,0x01,0x00,0x00,0x1d,0x01,0x00,0x00, -0xf1,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x22,0x01,0x00,0x00,0x20,0x01,0x00,0x00,0x21,0x01,0x00,0x00, -0x41,0x00,0x07,0x00,0xfa,0x00,0x00,0x00,0x24,0x01,0x00,0x00, -0xf8,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0xe5,0x00,0x00,0x00, -0x21,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0xba,0x00,0x00,0x00, -0x25,0x01,0x00,0x00,0x24,0x01,0x00,0x00,0x41,0x00,0x05,0x00, -0xfd,0x00,0x00,0x00,0x26,0x01,0x00,0x00,0xea,0x00,0x00,0x00, -0x22,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x26,0x01,0x00,0x00, -0x25,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x2d,0x01,0x00,0x00,0xc8,0x02,0x00,0x00,0x2b,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0xd3,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xd5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x2f,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x2f,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xc9,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, -0xd5,0x00,0x00,0x00,0x80,0x01,0x00,0x00,0x30,0x01,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00,0x35,0x01,0x00,0x00, -0xc9,0x02,0x00,0x00,0x9d,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0x31,0x01,0x00,0x00,0x30,0x01,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x35,0x01,0x00,0x00,0x30,0x01,0x00,0x00, -0x31,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x30,0x01,0x00,0x00, +0x06,0x00,0x00,0x00,0x6e,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, +0x6d,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x73,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x72,0x00,0x00,0x00, +0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x79,0x00,0x00,0x00, +0x4e,0x00,0x00,0x00,0x78,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x7e,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, +0x7d,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, +0x82,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x81,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x83,0x00,0x00,0x00, +0x82,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0x47,0x00,0x00,0x00,0x83,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x87,0x00,0x00,0x00, +0x12,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x88,0x00,0x00,0x00,0x87,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8a,0x00,0x00,0x00, +0x47,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x8d,0x00,0x00,0x00,0x8a,0x00,0x00,0x00, +0x83,0x00,0x00,0x00,0x0c,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0x8e,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x26,0x00,0x00,0x00, +0x88,0x00,0x00,0x00,0x8d,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x15,0x00,0x00,0x00,0x92,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0x91,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x93,0x00,0x00,0x00,0x92,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x94,0x00,0x00,0x00,0x32,0x00,0x00,0x00, +0x93,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x96,0x00,0x00,0x00,0x42,0x00,0x00,0x00,0x37,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x98,0x00,0x00,0x00, +0x12,0x00,0x00,0x00,0x97,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x99,0x00,0x00,0x00,0x98,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9a,0x00,0x00,0x00, +0x96,0x00,0x00,0x00,0x99,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x9b,0x00,0x00,0x00,0x94,0x00,0x00,0x00, +0x9a,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x9d,0x00,0x00,0x00,0x9b,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9e,0x00,0x00,0x00, +0x9d,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x15,0x00,0x00,0x00,0xa3,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0xa2,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xa4,0x00,0x00,0x00,0xa3,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xa5,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, +0xa4,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xa8,0x00,0x00,0x00,0x4a,0x00,0x00,0x00,0xa7,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0xaa,0x00,0x00,0x00, +0x12,0x00,0x00,0x00,0xa9,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xab,0x00,0x00,0x00,0xaa,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xac,0x00,0x00,0x00, +0xa8,0x00,0x00,0x00,0xab,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xad,0x00,0x00,0x00,0xa5,0x00,0x00,0x00, +0xac,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xaf,0x00,0x00,0x00,0xad,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xb0,0x00,0x00,0x00, +0xaf,0x00,0x00,0x00,0x77,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xb2,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xb2,0x00,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xb9,0x02,0x00,0x00, +0x3e,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0xd1,0x00,0x00,0x00, +0xb3,0x00,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, +0xc3,0x00,0x00,0x00,0xb9,0x02,0x00,0x00,0xc1,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0xb4,0x00,0x00,0x00,0xb3,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xc3,0x00,0x00,0x00, +0xb3,0x00,0x00,0x00,0xb4,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xb3,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0xcd,0x00,0x00,0x00, +0xce,0x00,0x00,0x00,0xca,0x00,0x00,0x00,0xb9,0x02,0x00,0x00, +0x3e,0x00,0x03,0x00,0xce,0x00,0x00,0x00,0xcc,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xd1,0x00,0x00,0x00, +0xb9,0x02,0x00,0x00,0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xb2,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xb4,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xd4,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xd4,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xd2,0x02,0x00,0x00,0xb0,0x00,0x00,0x00,0xb4,0x00,0x00,0x00, +0x87,0x01,0x00,0x00,0xd7,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xce,0x02,0x00,0x00,0x9e,0x00,0x00,0x00, +0xb4,0x00,0x00,0x00,0x84,0x01,0x00,0x00,0xd7,0x00,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xba,0x02,0x00,0x00, +0x84,0x00,0x00,0x00,0xb4,0x00,0x00,0x00,0x32,0x02,0x00,0x00, +0xd7,0x00,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, +0xdb,0x00,0x00,0x00,0xba,0x02,0x00,0x00,0x8e,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0xd6,0x00,0x00,0x00,0xd7,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xdb,0x00,0x00,0x00, +0xd5,0x00,0x00,0x00,0xd6,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xd5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xdd,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xdd,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xca,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, +0xd5,0x00,0x00,0x00,0x3a,0x01,0x00,0x00,0xde,0x00,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0xe3,0x00,0x00,0x00, +0xca,0x02,0x00,0x00,0x37,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0xdf,0x00,0x00,0x00,0xde,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0xe3,0x00,0x00,0x00,0xde,0x00,0x00,0x00, +0xdf,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xde,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xe8,0x00,0x00,0x00, +0x73,0x00,0x00,0x00,0xca,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xeb,0x00,0x00,0x00,0xe8,0x00,0x00,0x00, +0x99,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xec,0x00,0x00,0x00,0xeb,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xed,0x00,0x00,0x00, +0xce,0x02,0x00,0x00,0xec,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xef,0x00,0x00,0x00,0xed,0x00,0x00,0x00, +0x6e,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xf5,0x00,0x00,0x00,0xe8,0x00,0x00,0x00,0xf4,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xf7,0x00,0x00,0x00, +0x6e,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xf8,0x00,0x00,0x00,0xf5,0x00,0x00,0x00, +0xf7,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xfc,0x00,0x00,0x00,0xef,0x00,0x00,0x00,0xfb,0x00,0x00,0x00, +0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x00,0x01,0x00,0x00, +0xef,0x00,0x00,0x00,0xff,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x01,0x01,0x00,0x00,0x00,0x01,0x00,0x00, +0x0c,0x00,0x00,0x00,0x41,0x00,0x07,0x00,0x0d,0x01,0x00,0x00, +0x0e,0x01,0x00,0x00,0x0b,0x01,0x00,0x00,0x34,0x00,0x00,0x00, +0xfc,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x03,0x01,0x00,0x00,0x0f,0x01,0x00,0x00,0x0e,0x01,0x00,0x00, +0x73,0x00,0x04,0x00,0xc4,0x00,0x00,0x00,0x10,0x01,0x00,0x00, +0x0f,0x01,0x00,0x00,0x41,0x00,0x08,0x00,0x16,0x01,0x00,0x00, +0x17,0x01,0x00,0x00,0x0b,0x01,0x00,0x00,0x34,0x00,0x00,0x00, +0xfc,0x00,0x00,0x00,0xd0,0x00,0x00,0x00,0x01,0x01,0x00,0x00, +0x3d,0x00,0x04,0x00,0x04,0x01,0x00,0x00,0x18,0x01,0x00,0x00, +0x17,0x01,0x00,0x00,0x72,0x00,0x04,0x00,0x13,0x00,0x00,0x00, +0x19,0x01,0x00,0x00,0x18,0x01,0x00,0x00,0x6f,0x00,0x04,0x00, +0xc4,0x00,0x00,0x00,0x1a,0x01,0x00,0x00,0x19,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x1d,0x01,0x00,0x00, +0x01,0x01,0x00,0x00,0x39,0x00,0x00,0x00,0x41,0x00,0x08,0x00, +0x16,0x01,0x00,0x00,0x1e,0x01,0x00,0x00,0x0b,0x01,0x00,0x00, +0x34,0x00,0x00,0x00,0xfc,0x00,0x00,0x00,0xd0,0x00,0x00,0x00, +0x1d,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0x04,0x01,0x00,0x00, +0x1f,0x01,0x00,0x00,0x1e,0x01,0x00,0x00,0x72,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0x20,0x01,0x00,0x00,0x1f,0x01,0x00,0x00, +0x6f,0x00,0x04,0x00,0xc4,0x00,0x00,0x00,0x21,0x01,0x00,0x00, +0x20,0x01,0x00,0x00,0x50,0x00,0x05,0x00,0x11,0x01,0x00,0x00, +0x22,0x01,0x00,0x00,0x1a,0x01,0x00,0x00,0x21,0x01,0x00,0x00, +0x8e,0x00,0x05,0x00,0x11,0x01,0x00,0x00,0x24,0x01,0x00,0x00, +0x22,0x01,0x00,0x00,0x10,0x01,0x00,0x00,0x51,0x00,0x05,0x00, +0xc4,0x00,0x00,0x00,0x2c,0x01,0x00,0x00,0x24,0x01,0x00,0x00, +0x00,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x2d,0x01,0x00,0x00, +0x2e,0x01,0x00,0x00,0x29,0x01,0x00,0x00,0xf8,0x00,0x00,0x00, +0x3e,0x00,0x03,0x00,0x2e,0x01,0x00,0x00,0x2c,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x30,0x01,0x00,0x00, +0xf8,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x51,0x00,0x05,0x00, +0xc4,0x00,0x00,0x00,0x32,0x01,0x00,0x00,0x24,0x01,0x00,0x00, +0x01,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x2d,0x01,0x00,0x00, +0x33,0x01,0x00,0x00,0x29,0x01,0x00,0x00,0x30,0x01,0x00,0x00, +0x3e,0x00,0x03,0x00,0x33,0x01,0x00,0x00,0x32,0x01,0x00,0x00, 0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3a,0x01,0x00,0x00, -0x74,0x00,0x00,0x00,0xc9,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x3d,0x01,0x00,0x00,0x3a,0x01,0x00,0x00, -0xa1,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x3e,0x01,0x00,0x00,0x3d,0x01,0x00,0x00,0x6d,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3f,0x01,0x00,0x00, -0xd0,0x02,0x00,0x00,0x3e,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x41,0x01,0x00,0x00,0x3f,0x01,0x00,0x00, -0x6f,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x4b,0x01,0x00,0x00,0x3a,0x01,0x00,0x00,0x4a,0x01,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x4d,0x01,0x00,0x00, -0x6f,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x4e,0x01,0x00,0x00,0x4b,0x01,0x00,0x00, -0x4d,0x01,0x00,0x00,0x41,0x00,0x07,0x00,0xfa,0x00,0x00,0x00, -0x55,0x01,0x00,0x00,0x53,0x01,0x00,0x00,0x34,0x00,0x00,0x00, -0x41,0x01,0x00,0x00,0x3e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0xba,0x00,0x00,0x00,0x56,0x01,0x00,0x00,0x55,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0xfd,0x00,0x00,0x00,0x57,0x01,0x00,0x00, -0x46,0x01,0x00,0x00,0x4e,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0x57,0x01,0x00,0x00,0x56,0x01,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x5c,0x01,0x00,0x00,0x3a,0x01,0x00,0x00, -0x5b,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x5f,0x01,0x00,0x00,0x5c,0x01,0x00,0x00,0x4d,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x60,0x01,0x00,0x00, -0x5f,0x01,0x00,0x00,0x39,0x00,0x00,0x00,0x41,0x00,0x07,0x00, -0xfa,0x00,0x00,0x00,0x62,0x01,0x00,0x00,0x53,0x01,0x00,0x00, -0x34,0x00,0x00,0x00,0x41,0x01,0x00,0x00,0x39,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0xba,0x00,0x00,0x00,0x63,0x01,0x00,0x00, -0x62,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0xfd,0x00,0x00,0x00, -0x64,0x01,0x00,0x00,0x46,0x01,0x00,0x00,0x60,0x01,0x00,0x00, -0x3e,0x00,0x03,0x00,0x64,0x01,0x00,0x00,0x63,0x01,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x69,0x01,0x00,0x00, -0x3a,0x01,0x00,0x00,0x68,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x6c,0x01,0x00,0x00,0x69,0x01,0x00,0x00, -0x4d,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x6d,0x01,0x00,0x00,0x6c,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, -0x41,0x00,0x07,0x00,0xfa,0x00,0x00,0x00,0x6f,0x01,0x00,0x00, -0x53,0x01,0x00,0x00,0x34,0x00,0x00,0x00,0x41,0x01,0x00,0x00, -0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0xba,0x00,0x00,0x00, -0x70,0x01,0x00,0x00,0x6f,0x01,0x00,0x00,0x41,0x00,0x05,0x00, -0xfd,0x00,0x00,0x00,0x71,0x01,0x00,0x00,0x46,0x01,0x00,0x00, -0x6d,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x71,0x01,0x00,0x00, -0x70,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x76,0x01,0x00,0x00,0x3a,0x01,0x00,0x00,0x75,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x79,0x01,0x00,0x00, -0x76,0x01,0x00,0x00,0x4d,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x7a,0x01,0x00,0x00,0x79,0x01,0x00,0x00, -0x21,0x01,0x00,0x00,0x41,0x00,0x07,0x00,0xfa,0x00,0x00,0x00, -0x7c,0x01,0x00,0x00,0x53,0x01,0x00,0x00,0x34,0x00,0x00,0x00, -0x41,0x01,0x00,0x00,0x21,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, -0xba,0x00,0x00,0x00,0x7d,0x01,0x00,0x00,0x7c,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0xfd,0x00,0x00,0x00,0x7e,0x01,0x00,0x00, -0x46,0x01,0x00,0x00,0x7a,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0x7e,0x01,0x00,0x00,0x7d,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x80,0x01,0x00,0x00,0xc9,0x02,0x00,0x00, -0x2b,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x2f,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x31,0x01,0x00,0x00,0xe0,0x00,0x04,0x00, -0x0c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x81,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x84,0x01,0x00,0x00, -0xcc,0x02,0x00,0x00,0x82,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x87,0x01,0x00,0x00,0xd0,0x02,0x00,0x00, -0x85,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x89,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x89,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xd2,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, -0x31,0x01,0x00,0x00,0x30,0x02,0x00,0x00,0x8c,0x01,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00,0x8f,0x01,0x00,0x00, -0xd2,0x02,0x00,0x00,0x6c,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0x8b,0x01,0x00,0x00,0x8c,0x01,0x00,0x00,0x00,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x8f,0x01,0x00,0x00,0x8a,0x01,0x00,0x00, -0x8b,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x8a,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0x91,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x91,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xd6,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0x8a,0x01,0x00,0x00, -0xbc,0x01,0x00,0x00,0x94,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb8,0x00,0x00,0x00,0x97,0x01,0x00,0x00,0xd6,0x02,0x00,0x00, -0x60,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x93,0x01,0x00,0x00, -0x94,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x97,0x01,0x00,0x00,0x92,0x01,0x00,0x00,0x93,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x92,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0x99,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x99,0x01,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xe8,0x02,0x00,0x00, -0x3e,0x00,0x00,0x00,0x92,0x01,0x00,0x00,0xba,0x01,0x00,0x00, -0x9a,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00, -0x9f,0x01,0x00,0x00,0xe8,0x02,0x00,0x00,0x62,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0x9b,0x01,0x00,0x00,0x9a,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x9f,0x01,0x00,0x00, -0x9a,0x01,0x00,0x00,0x9b,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x9a,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xa5,0x01,0x00,0x00,0xd6,0x02,0x00,0x00,0x62,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa7,0x01,0x00,0x00, -0xa5,0x01,0x00,0x00,0xe8,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xa9,0x01,0x00,0x00,0x55,0x00,0x00,0x00, -0x53,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xab,0x01,0x00,0x00,0xd6,0x02,0x00,0x00,0x61,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xac,0x01,0x00,0x00, -0xa9,0x01,0x00,0x00,0xab,0x01,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xae,0x01,0x00,0x00,0x64,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xaf,0x01,0x00,0x00,0xac,0x01,0x00,0x00,0xae,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xb1,0x01,0x00,0x00, -0xaf,0x01,0x00,0x00,0xe8,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xb3,0x01,0x00,0x00,0xb1,0x01,0x00,0x00, -0xb2,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xb5,0x01,0x00,0x00,0xb3,0x01,0x00,0x00,0xd2,0x02,0x00,0x00, -0x41,0x00,0x05,0x00,0xfd,0x00,0x00,0x00,0xb6,0x01,0x00,0x00, -0xea,0x00,0x00,0x00,0xb5,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, -0xba,0x00,0x00,0x00,0xb7,0x01,0x00,0x00,0xb6,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0xc3,0x00,0x00,0x00,0xb8,0x01,0x00,0x00, -0xa3,0x01,0x00,0x00,0xa7,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0xb8,0x01,0x00,0x00,0xb7,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xba,0x01,0x00,0x00,0xe8,0x02,0x00,0x00, -0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x99,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x9b,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0x94,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x94,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xbc,0x01,0x00,0x00, -0xd6,0x02,0x00,0x00,0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x91,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x93,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0xbe,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xbe,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xd7,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0x93,0x01,0x00,0x00, -0xea,0x01,0x00,0x00,0xc1,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb8,0x00,0x00,0x00,0xc4,0x01,0x00,0x00,0xd7,0x02,0x00,0x00, -0xb5,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xc0,0x01,0x00,0x00, -0xc1,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0xc4,0x01,0x00,0x00,0xbf,0x01,0x00,0x00,0xc0,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xbf,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0xc6,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xc6,0x01,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xe5,0x02,0x00,0x00, -0x3e,0x00,0x00,0x00,0xbf,0x01,0x00,0x00,0xe8,0x01,0x00,0x00, -0xc7,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00, -0xcc,0x01,0x00,0x00,0xe5,0x02,0x00,0x00,0xb2,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0xc8,0x01,0x00,0x00,0xc7,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xcc,0x01,0x00,0x00, -0xc7,0x01,0x00,0x00,0xc8,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xc7,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xd2,0x01,0x00,0x00,0xd7,0x02,0x00,0x00,0xb2,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xd4,0x01,0x00,0x00, -0xd2,0x01,0x00,0x00,0xe5,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xd6,0x01,0x00,0x00,0x59,0x00,0x00,0x00, -0xaf,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xd9,0x01,0x00,0x00,0xd7,0x02,0x00,0x00,0xd8,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xda,0x01,0x00,0x00, -0xd6,0x01,0x00,0x00,0xd9,0x01,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xdc,0x01,0x00,0x00,0x68,0x00,0x00,0x00, -0xb2,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xdd,0x01,0x00,0x00,0xda,0x01,0x00,0x00,0xdc,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xdf,0x01,0x00,0x00, -0xdd,0x01,0x00,0x00,0xe5,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xe1,0x01,0x00,0x00,0xdf,0x01,0x00,0x00, -0xe0,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xe3,0x01,0x00,0x00,0xe1,0x01,0x00,0x00,0xd2,0x02,0x00,0x00, -0x41,0x00,0x05,0x00,0xfd,0x00,0x00,0x00,0xe4,0x01,0x00,0x00, -0x46,0x01,0x00,0x00,0xe3,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, -0xba,0x00,0x00,0x00,0xe5,0x01,0x00,0x00,0xe4,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0xc3,0x00,0x00,0x00,0xe6,0x01,0x00,0x00, -0xd0,0x01,0x00,0x00,0xd4,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0xe6,0x01,0x00,0x00,0xe5,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xe8,0x01,0x00,0x00,0xe5,0x02,0x00,0x00, -0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xc6,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xc8,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0xc1,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xc1,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xea,0x01,0x00,0x00, -0xd7,0x02,0x00,0x00,0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xbe,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xc0,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0xec,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xec,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xd8,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0xc0,0x01,0x00,0x00, -0x2e,0x02,0x00,0x00,0xef,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb8,0x00,0x00,0x00,0xf2,0x01,0x00,0x00,0xd8,0x02,0x00,0x00, -0xb5,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xee,0x01,0x00,0x00, -0xef,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0xf2,0x01,0x00,0x00,0xed,0x01,0x00,0x00,0xee,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xed,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0xf4,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xf4,0x01,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xdc,0x02,0x00,0x00, -0x3e,0x00,0x00,0x00,0xed,0x01,0x00,0x00,0x2c,0x02,0x00,0x00, -0xf7,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00, -0xfa,0x01,0x00,0x00,0xdc,0x02,0x00,0x00,0x60,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0xf6,0x01,0x00,0x00,0xf7,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xfa,0x01,0x00,0x00, -0xf5,0x01,0x00,0x00,0xf6,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xf5,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xfc,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xfc,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xde,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, -0xf5,0x01,0x00,0x00,0x2a,0x02,0x00,0x00,0xff,0x01,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00,0x02,0x02,0x00,0x00, -0xde,0x02,0x00,0x00,0xb2,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0xfe,0x01,0x00,0x00,0xff,0x01,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x02,0x02,0x00,0x00,0xfd,0x01,0x00,0x00, -0xfe,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xfd,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0x04,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x04,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xe0,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0xfd,0x01,0x00,0x00, -0x28,0x02,0x00,0x00,0x05,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb8,0x00,0x00,0x00,0x0a,0x02,0x00,0x00,0xe0,0x02,0x00,0x00, -0x62,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x06,0x02,0x00,0x00, -0x05,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x0a,0x02,0x00,0x00,0x05,0x02,0x00,0x00,0x06,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x05,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x0c,0x02,0x00,0x00,0xd8,0x02,0x00,0x00, -0xb2,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x0e,0x02,0x00,0x00,0x0c,0x02,0x00,0x00,0xde,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x10,0x02,0x00,0x00, -0x0e,0x02,0x00,0x00,0x0f,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x12,0x02,0x00,0x00,0xdc,0x02,0x00,0x00, -0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x13,0x02,0x00,0x00,0x10,0x02,0x00,0x00,0x12,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x15,0x02,0x00,0x00, -0x13,0x02,0x00,0x00,0xe0,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x19,0x02,0x00,0x00,0x12,0x02,0x00,0x00, -0xe0,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0xc3,0x00,0x00,0x00, -0x1a,0x02,0x00,0x00,0xa3,0x01,0x00,0x00,0x19,0x02,0x00,0x00, -0x3d,0x00,0x04,0x00,0xba,0x00,0x00,0x00,0x1b,0x02,0x00,0x00, -0x1a,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0xc3,0x00,0x00,0x00, -0x20,0x02,0x00,0x00,0xd0,0x01,0x00,0x00,0x0e,0x02,0x00,0x00, -0x3d,0x00,0x04,0x00,0xba,0x00,0x00,0x00,0x21,0x02,0x00,0x00, -0x20,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0xc3,0x00,0x00,0x00, -0x23,0x02,0x00,0x00,0xc0,0x00,0x00,0x00,0x15,0x02,0x00,0x00, -0x3d,0x00,0x04,0x00,0xba,0x00,0x00,0x00,0x24,0x02,0x00,0x00, -0x23,0x02,0x00,0x00,0x0c,0x00,0x08,0x00,0xba,0x00,0x00,0x00, -0x25,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00, -0x1b,0x02,0x00,0x00,0x21,0x02,0x00,0x00,0x24,0x02,0x00,0x00, -0x3e,0x00,0x03,0x00,0x23,0x02,0x00,0x00,0x25,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x28,0x02,0x00,0x00, -0xe0,0x02,0x00,0x00,0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x04,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x06,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0xff,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xff,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x2a,0x02,0x00,0x00,0xde,0x02,0x00,0x00,0xc6,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xfc,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xfe,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xf7,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xf7,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x2c,0x02,0x00,0x00,0xdc,0x02,0x00,0x00, -0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xf4,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xf6,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0xef,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xef,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x2e,0x02,0x00,0x00, -0xd8,0x02,0x00,0x00,0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xec,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xee,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0x8c,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x8c,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x30,0x02,0x00,0x00,0xd2,0x02,0x00,0x00,0xc6,0x00,0x00,0x00, +0xca,0x02,0x00,0x00,0x38,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0xdd,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xdf,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x3c,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x3c,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xcb,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0xdf,0x00,0x00,0x00, +0x80,0x01,0x00,0x00,0x3d,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc2,0x00,0x00,0x00,0x42,0x01,0x00,0x00,0xcb,0x02,0x00,0x00, +0xa7,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x3e,0x01,0x00,0x00, +0x3d,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x42,0x01,0x00,0x00,0x3d,0x01,0x00,0x00,0x3e,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x3d,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x47,0x01,0x00,0x00,0x7e,0x00,0x00,0x00, +0xcb,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x4a,0x01,0x00,0x00,0x47,0x01,0x00,0x00,0xab,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x4b,0x01,0x00,0x00, +0x4a,0x01,0x00,0x00,0x77,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x4c,0x01,0x00,0x00,0xd2,0x02,0x00,0x00, +0x4b,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x4e,0x01,0x00,0x00,0x4c,0x01,0x00,0x00,0x79,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x54,0x01,0x00,0x00, +0x47,0x01,0x00,0x00,0x53,0x01,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x56,0x01,0x00,0x00,0x79,0x00,0x00,0x00, +0x77,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x57,0x01,0x00,0x00,0x54,0x01,0x00,0x00,0x56,0x01,0x00,0x00, +0x41,0x00,0x07,0x00,0x65,0x01,0x00,0x00,0x66,0x01,0x00,0x00, +0x63,0x01,0x00,0x00,0x34,0x00,0x00,0x00,0x4e,0x01,0x00,0x00, +0x3e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00, +0x67,0x01,0x00,0x00,0x66,0x01,0x00,0x00,0x41,0x00,0x05,0x00, +0x2d,0x01,0x00,0x00,0x68,0x01,0x00,0x00,0x5c,0x01,0x00,0x00, +0x57,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x68,0x01,0x00,0x00, +0x67,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x6a,0x01,0x00,0x00,0x57,0x01,0x00,0x00,0x39,0x00,0x00,0x00, +0x41,0x00,0x07,0x00,0x65,0x01,0x00,0x00,0x6c,0x01,0x00,0x00, +0x63,0x01,0x00,0x00,0x34,0x00,0x00,0x00,0x4e,0x01,0x00,0x00, +0x39,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00, +0x6d,0x01,0x00,0x00,0x6c,0x01,0x00,0x00,0x41,0x00,0x05,0x00, +0x2d,0x01,0x00,0x00,0x6e,0x01,0x00,0x00,0x5c,0x01,0x00,0x00, +0x6a,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x6e,0x01,0x00,0x00, +0x6d,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x70,0x01,0x00,0x00,0x57,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, +0x41,0x00,0x07,0x00,0x65,0x01,0x00,0x00,0x72,0x01,0x00,0x00, +0x63,0x01,0x00,0x00,0x34,0x00,0x00,0x00,0x4e,0x01,0x00,0x00, +0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00, +0x73,0x01,0x00,0x00,0x72,0x01,0x00,0x00,0x41,0x00,0x05,0x00, +0x2d,0x01,0x00,0x00,0x74,0x01,0x00,0x00,0x5c,0x01,0x00,0x00, +0x70,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x74,0x01,0x00,0x00, +0x73,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x77,0x01,0x00,0x00,0x57,0x01,0x00,0x00,0x76,0x01,0x00,0x00, +0x41,0x00,0x07,0x00,0x65,0x01,0x00,0x00,0x79,0x01,0x00,0x00, +0x63,0x01,0x00,0x00,0x34,0x00,0x00,0x00,0x4e,0x01,0x00,0x00, +0x76,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00, +0x7a,0x01,0x00,0x00,0x79,0x01,0x00,0x00,0x41,0x00,0x05,0x00, +0x2d,0x01,0x00,0x00,0x7b,0x01,0x00,0x00,0x5c,0x01,0x00,0x00, +0x77,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x7b,0x01,0x00,0x00, +0x7a,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x80,0x01,0x00,0x00,0xcb,0x02,0x00,0x00,0x7e,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0x3c,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x3e,0x01,0x00,0x00,0xe0,0x00,0x04,0x00,0x0c,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x81,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x84,0x01,0x00,0x00,0xce,0x02,0x00,0x00, +0x82,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x87,0x01,0x00,0x00,0xd2,0x02,0x00,0x00,0x85,0x01,0x00,0x00, 0xf9,0x00,0x02,0x00,0x89,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x8b,0x01,0x00,0x00,0xe0,0x00,0x04,0x00,0x0c,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x81,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0xcd,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xcd,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x32,0x02,0x00,0x00, -0xb8,0x02,0x00,0x00,0x6c,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xca,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xcc,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x37,0x02,0x00,0x00, -0x55,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x38,0x02,0x00,0x00,0x8c,0x00,0x00,0x00, -0x37,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x3d,0x02,0x00,0x00,0x59,0x00,0x00,0x00,0xaf,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3e,0x02,0x00,0x00, -0x9e,0x00,0x00,0x00,0x3d,0x02,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x43,0x02,0x00,0x00,0x12,0x00,0x00,0x00, -0x42,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x44,0x02,0x00,0x00,0x43,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x45,0x02,0x00,0x00,0x0f,0x00,0x00,0x00, -0x44,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x49,0x02,0x00,0x00,0x47,0x00,0x00,0x00,0x44,0x02,0x00,0x00, -0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00,0x4b,0x02,0x00,0x00, -0x4a,0x02,0x00,0x00,0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x4c,0x02,0x00,0x00,0x4b,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x4d,0x02,0x00,0x00, -0x49,0x02,0x00,0x00,0x4c,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x4e,0x02,0x00,0x00,0x45,0x02,0x00,0x00, -0x4d,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x50,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x50,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xb9,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, -0xcc,0x00,0x00,0x00,0xb6,0x02,0x00,0x00,0x53,0x02,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00,0x56,0x02,0x00,0x00, -0xb9,0x02,0x00,0x00,0xb5,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0x52,0x02,0x00,0x00,0x53,0x02,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x56,0x02,0x00,0x00,0x51,0x02,0x00,0x00, -0x52,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x51,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0x58,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x58,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xba,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0x51,0x02,0x00,0x00, -0xb4,0x02,0x00,0x00,0x5b,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb8,0x00,0x00,0x00,0x5e,0x02,0x00,0x00,0xba,0x02,0x00,0x00, -0x60,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x5a,0x02,0x00,0x00, -0x5b,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x5e,0x02,0x00,0x00,0x59,0x02,0x00,0x00,0x5a,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x59,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x62,0x02,0x00,0x00,0xba,0x02,0x00,0x00, -0x61,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x63,0x02,0x00,0x00,0x38,0x02,0x00,0x00,0x62,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x65,0x02,0x00,0x00, -0x64,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x66,0x02,0x00,0x00,0x63,0x02,0x00,0x00, -0x65,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x6a,0x02,0x00,0x00,0xb9,0x02,0x00,0x00,0xd8,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x6b,0x02,0x00,0x00, -0x3e,0x02,0x00,0x00,0x6a,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x6d,0x02,0x00,0x00,0x68,0x00,0x00,0x00, -0xb2,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x6e,0x02,0x00,0x00,0x6b,0x02,0x00,0x00,0x6d,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0x70,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x70,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xbc,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0x59,0x02,0x00,0x00, -0xb2,0x02,0x00,0x00,0x73,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb8,0x00,0x00,0x00,0x76,0x02,0x00,0x00,0xbc,0x02,0x00,0x00, -0xb2,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x72,0x02,0x00,0x00, -0x73,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x76,0x02,0x00,0x00,0x71,0x02,0x00,0x00,0x72,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x71,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x78,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x78,0x02,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xbe,0x02,0x00,0x00, -0x3e,0x00,0x00,0x00,0x71,0x02,0x00,0x00,0xb0,0x02,0x00,0x00, -0x7b,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00, -0x7e,0x02,0x00,0x00,0xbe,0x02,0x00,0x00,0x62,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0x7a,0x02,0x00,0x00,0x7b,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x7e,0x02,0x00,0x00, -0x79,0x02,0x00,0x00,0x7a,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x79,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x81,0x02,0x00,0x00,0x66,0x02,0x00,0x00,0xbe,0x02,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00,0x84,0x02,0x00,0x00, -0x81,0x02,0x00,0x00,0x36,0x00,0x00,0x00,0xf7,0x00,0x03,0x00, -0x86,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x84,0x02,0x00,0x00,0x85,0x02,0x00,0x00,0x86,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x85,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x89,0x02,0x00,0x00,0x6e,0x02,0x00,0x00, -0xbc,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0x8a,0x02,0x00,0x00,0x12,0x00,0x00,0x00,0xc6,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x8b,0x02,0x00,0x00, -0x8a,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00, -0x8c,0x02,0x00,0x00,0x89,0x02,0x00,0x00,0x8b,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0x86,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x86,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0xb8,0x00,0x00,0x00, -0x8d,0x02,0x00,0x00,0x84,0x02,0x00,0x00,0x79,0x02,0x00,0x00, -0x8c,0x02,0x00,0x00,0x85,0x02,0x00,0x00,0xf7,0x00,0x03,0x00, -0x8f,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x8d,0x02,0x00,0x00,0x8e,0x02,0x00,0x00,0x8f,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x8e,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x97,0x02,0x00,0x00,0x6e,0x02,0x00,0x00, -0xbc,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0x99,0x02,0x00,0x00,0x12,0x00,0x00,0x00,0x98,0x02,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x9a,0x02,0x00,0x00, -0x99,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x9b,0x02,0x00,0x00,0x97,0x02,0x00,0x00,0x9a,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9c,0x02,0x00,0x00, -0x4e,0x02,0x00,0x00,0x9b,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x9e,0x02,0x00,0x00,0x9c,0x02,0x00,0x00, -0x66,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xa0,0x02,0x00,0x00,0x9e,0x02,0x00,0x00,0xbe,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa2,0x02,0x00,0x00, -0xb9,0x02,0x00,0x00,0xb2,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xa4,0x02,0x00,0x00,0xa2,0x02,0x00,0x00, -0xbc,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xa6,0x02,0x00,0x00,0xa4,0x02,0x00,0x00,0xa5,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa8,0x02,0x00,0x00, -0xba,0x02,0x00,0x00,0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xa9,0x02,0x00,0x00,0xa6,0x02,0x00,0x00, -0xa8,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xab,0x02,0x00,0x00,0xa9,0x02,0x00,0x00,0xbe,0x02,0x00,0x00, -0x41,0x00,0x05,0x00,0xc3,0x00,0x00,0x00,0xac,0x02,0x00,0x00, -0xc0,0x00,0x00,0x00,0xab,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, -0xba,0x00,0x00,0x00,0xad,0x02,0x00,0x00,0xac,0x02,0x00,0x00, -0x41,0x00,0x06,0x00,0xfa,0x00,0x00,0x00,0xae,0x02,0x00,0x00, -0x93,0x02,0x00,0x00,0x34,0x00,0x00,0x00,0xa0,0x02,0x00,0x00, -0x3e,0x00,0x03,0x00,0xae,0x02,0x00,0x00,0xad,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0x8f,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x8f,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x7b,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x7b,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xb0,0x02,0x00,0x00,0xbe,0x02,0x00,0x00, -0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x78,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x7a,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x73,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x73,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xb2,0x02,0x00,0x00, -0xbc,0x02,0x00,0x00,0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x70,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x72,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0x5b,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x5b,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xb4,0x02,0x00,0x00,0xba,0x02,0x00,0x00,0xc6,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x58,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x5a,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x53,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x53,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xb6,0x02,0x00,0x00,0xb9,0x02,0x00,0x00, -0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x50,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x52,0x02,0x00,0x00,0xfd,0x00,0x01,0x00, -0x38,0x00,0x01,0x00, -}; -const uint64_t matmul_f32_aligned_l_fp32_len = 10348; - -unsigned char matmul_f32_aligned_m_data[] = { -0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, -0x6a,0x03,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, -0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x09,0x00,0x00,0x00, -0x0b,0x00,0x06,0x00,0x01,0x00,0x00,0x00,0x47,0x4c,0x53,0x4c, -0x2e,0x73,0x74,0x64,0x2e,0x34,0x35,0x30,0x00,0x00,0x00,0x00, -0x0e,0x00,0x03,0x00,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x0f,0x00,0x0f,0x00,0x05,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x6d,0x61,0x69,0x6e,0x00,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x3d,0x00,0x00,0x00,0x4c,0x00,0x00,0x00, -0xeb,0x00,0x00,0x00,0xfa,0x00,0x00,0x00,0x88,0x01,0x00,0x00, -0x95,0x01,0x00,0x00,0xcb,0x02,0x00,0x00,0x14,0x03,0x00,0x00, -0x10,0x00,0x06,0x00,0x04,0x00,0x00,0x00,0x11,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x0b,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x1c,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x10,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x08,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x03,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x10,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x14,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x18,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x07,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x10,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x24,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x0a,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x10,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x2c,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x30,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x0d,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0x47,0x00,0x03,0x00, -0x10,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x37,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x3d,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x1a,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x4c,0x00,0x00,0x00, -0x0b,0x00,0x00,0x00,0x1b,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x4f,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x53,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x60,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x62,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x07,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x6c,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x03,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x9d,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0xaf,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x05,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0xb2,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x08,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xf7,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x48,0x00,0x04,0x00, -0xf8,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x05,0x00,0x00,0x00, -0x48,0x00,0x04,0x00,0xf8,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0xf8,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0xf8,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x07,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x47,0x00,0x03,0x00, -0xf8,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0xfa,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0xfa,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x69,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x6a,0x01,0x00,0x00,0x0b,0x00,0x00,0x00,0x19,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x92,0x01,0x00,0x00,0x06,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0x93,0x01,0x00,0x00, -0x00,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0x48,0x00,0x04,0x00, -0x93,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x93,0x01,0x00,0x00,0x00,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x93,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x07,0x00,0x00,0x00, -0x10,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x93,0x01,0x00,0x00, -0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x95,0x01,0x00,0x00, -0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x95,0x01,0x00,0x00,0x21,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0xcb,0x02,0x00,0x00,0x0b,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x11,0x03,0x00,0x00, -0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00, -0x12,0x03,0x00,0x00,0x00,0x00,0x00,0x00,0x19,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x12,0x03,0x00,0x00,0x00,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, -0x12,0x03,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x14,0x03,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x14,0x03,0x00,0x00,0x21,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x13,0x00,0x02,0x00,0x02,0x00,0x00,0x00, -0x21,0x00,0x03,0x00,0x03,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x15,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x17,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x0a,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x0d,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x1e,0x00,0x10,0x00,0x10,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x11,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x11,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x15,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, -0x14,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x15,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x0a,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, -0x27,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0x2d,0x00,0x00,0x00,0x07,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x34,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x37,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00,0x3d,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x3e,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x0a,0x00,0x00,0x00,0x4c,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x4f,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x53,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x54,0x00,0x00,0x00,0x86,0x00,0x00,0x00, -0x37,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x58,0x00,0x00,0x00,0x86,0x00,0x00,0x00, -0x37,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x32,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x61,0x00,0x00,0x00, -0x86,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x60,0x00,0x00,0x00, -0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x62,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x63,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x61,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x67,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x61,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x08,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x6e,0x00,0x00,0x00, -0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x73,0x00,0x00,0x00, -0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x77,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, -0x7c,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0x87,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x8d,0x00,0x00,0x00, -0x03,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, -0x98,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x32,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x9d,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x9f,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xae,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xaf,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xb0,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0x53,0x00,0x00,0x00,0xaf,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xb1,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0x4f,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x32,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xb2,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xb3,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0xb1,0x00,0x00,0x00,0xb2,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xb4,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0xb3,0x00,0x00,0x00,0x60,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xb5,0x00,0x00,0x00, -0x86,0x00,0x00,0x00,0xb0,0x00,0x00,0x00,0xb4,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xb6,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0xae,0x00,0x00,0x00,0xb5,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xb7,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0xb6,0x00,0x00,0x00,0xb2,0x00,0x00,0x00, -0x14,0x00,0x02,0x00,0xb8,0x00,0x00,0x00,0x16,0x00,0x03,0x00, -0xba,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xbb,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xbc,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0xbb,0x00,0x00,0x00,0xb5,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xbd,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0xbc,0x00,0x00,0x00,0xb2,0x00,0x00,0x00,0x1c,0x00,0x04,0x00, -0xbe,0x00,0x00,0x00,0xba,0x00,0x00,0x00,0xbd,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0xbf,0x00,0x00,0x00,0x07,0x00,0x00,0x00, -0xbe,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0xba,0x00,0x00,0x00, -0xc2,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0xc3,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0xba,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0xc6,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x16,0x00,0x03,0x00,0xe6,0x00,0x00,0x00, -0x10,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xe7,0x00,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xe8,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x37,0x00,0x00,0x00, -0xe7,0x00,0x00,0x00,0x1c,0x00,0x04,0x00,0xe9,0x00,0x00,0x00, -0xe6,0x00,0x00,0x00,0xe8,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0xea,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0xe9,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0xea,0x00,0x00,0x00,0xeb,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xef,0x00,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x17,0x00,0x04,0x00,0xf5,0x00,0x00,0x00, -0xba,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x18,0x00,0x04,0x00, -0xf6,0x00,0x00,0x00,0xf5,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x1d,0x00,0x03,0x00,0xf7,0x00,0x00,0x00,0xf6,0x00,0x00,0x00, -0x1e,0x00,0x03,0x00,0xf8,0x00,0x00,0x00,0xf7,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0xf9,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0xf8,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0xf9,0x00,0x00,0x00, -0xfa,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0xfc,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0xba,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x00,0x01,0x00,0x00,0x04,0x00,0x00,0x00, -0xe6,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x05,0x01,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x13,0x01,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x21,0x01,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x26,0x01,0x00,0x00,0x03,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x30,0x01,0x00,0x00,0x80,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x35,0x01,0x00,0x00,0x04,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x3f,0x01,0x00,0x00, -0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x44,0x01,0x00,0x00, -0x05,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x4e,0x01,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x53,0x01,0x00,0x00,0x06,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x5d,0x01,0x00,0x00,0x80,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x62,0x01,0x00,0x00,0x07,0x00,0x00,0x00, -0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x69,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0x33,0x00,0x06,0x00,0x09,0x00,0x00,0x00, -0x6a,0x01,0x00,0x00,0x69,0x01,0x00,0x00,0x39,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x6b,0x01,0x00,0x00,0x51,0x00,0x00,0x00,0x6a,0x01,0x00,0x00, -0x00,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x6c,0x01,0x00,0x00,0x84,0x00,0x00,0x00,0x6b,0x01,0x00,0x00, -0x6d,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x6d,0x01,0x00,0x00,0x86,0x00,0x00,0x00,0x6c,0x01,0x00,0x00, -0x6c,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x84,0x01,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x85,0x01,0x00,0x00,0x84,0x00,0x00,0x00,0x9d,0x00,0x00,0x00, -0x84,0x01,0x00,0x00,0x1c,0x00,0x04,0x00,0x86,0x01,0x00,0x00, -0xe6,0x00,0x00,0x00,0x85,0x01,0x00,0x00,0x20,0x00,0x04,0x00, -0x87,0x01,0x00,0x00,0x04,0x00,0x00,0x00,0x86,0x01,0x00,0x00, -0x3b,0x00,0x04,0x00,0x87,0x01,0x00,0x00,0x88,0x01,0x00,0x00, -0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x8c,0x01,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x1d,0x00,0x03,0x00,0x92,0x01,0x00,0x00, -0xf6,0x00,0x00,0x00,0x1e,0x00,0x03,0x00,0x93,0x01,0x00,0x00, -0x92,0x01,0x00,0x00,0x20,0x00,0x04,0x00,0x94,0x01,0x00,0x00, -0x0c,0x00,0x00,0x00,0x93,0x01,0x00,0x00,0x3b,0x00,0x04,0x00, -0x94,0x01,0x00,0x00,0x95,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x9e,0x01,0x00,0x00, -0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xac,0x01,0x00,0x00, -0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xba,0x01,0x00,0x00, -0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xc8,0x01,0x00,0x00, -0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xd6,0x01,0x00,0x00, -0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xe4,0x01,0x00,0x00, -0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xf2,0x01,0x00,0x00, -0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xff,0x01,0x00,0x00, -0x08,0x01,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x00,0x02,0x00,0x00,0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x6d,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x03,0x02,0x00,0x00,0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x6d,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x1e,0x02,0x00,0x00,0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x1c,0x00,0x04,0x00,0x1f,0x02,0x00,0x00, -0xe6,0x00,0x00,0x00,0x1e,0x02,0x00,0x00,0x20,0x00,0x04,0x00, -0x20,0x02,0x00,0x00,0x07,0x00,0x00,0x00,0x1f,0x02,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x30,0x02,0x00,0x00, -0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x36,0x02,0x00,0x00,0x07,0x00,0x00,0x00, -0xe6,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x4c,0x02,0x00,0x00,0x84,0x00,0x00,0x00,0xb5,0x00,0x00,0x00, -0xb2,0x00,0x00,0x00,0x1c,0x00,0x04,0x00,0x4d,0x02,0x00,0x00, -0xe6,0x00,0x00,0x00,0x4c,0x02,0x00,0x00,0x20,0x00,0x04,0x00, -0x4e,0x02,0x00,0x00,0x07,0x00,0x00,0x00,0x4d,0x02,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x57,0x02,0x00,0x00, -0x86,0x00,0x00,0x00,0xaf,0x00,0x00,0x00,0xb5,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x5f,0x02,0x00,0x00, -0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x8e,0x02,0x00,0x00, -0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0xc3,0x02,0x00,0x00, -0x0d,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00, -0xcb,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, -0x11,0x03,0x00,0x00,0xba,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, -0x12,0x03,0x00,0x00,0x11,0x03,0x00,0x00,0x20,0x00,0x04,0x00, -0x13,0x03,0x00,0x00,0x0c,0x00,0x00,0x00,0x12,0x03,0x00,0x00, -0x3b,0x00,0x04,0x00,0x13,0x03,0x00,0x00,0x14,0x03,0x00,0x00, -0x0c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, -0x19,0x03,0x00,0x00,0x05,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x26,0x03,0x00,0x00,0x84,0x00,0x00,0x00, -0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x36,0x00,0x05,0x00, -0x02,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x03,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x05,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0xbf,0x00,0x00,0x00,0xc0,0x00,0x00,0x00, -0x07,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x20,0x02,0x00,0x00, -0x21,0x02,0x00,0x00,0x07,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x4e,0x02,0x00,0x00,0x4f,0x02,0x00,0x00,0x07,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00,0x0e,0x00,0x00,0x00, -0x0b,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x0e,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x17,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x18,0x00,0x00,0x00, -0x0f,0x00,0x00,0x00,0x17,0x00,0x00,0x00,0x89,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x1e,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, -0x17,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0x22,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x22,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x24,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x28,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x27,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x29,0x00,0x00,0x00,0x28,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x2a,0x00,0x00,0x00, -0x1e,0x00,0x00,0x00,0x29,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x2d,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x2f,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x30,0x00,0x00,0x00,0x24,0x00,0x00,0x00, -0x2f,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x32,0x00,0x00,0x00,0x30,0x00,0x00,0x00,0x2a,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x35,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x36,0x00,0x00,0x00,0x35,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x38,0x00,0x00,0x00, -0x36,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x82,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x38,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x3b,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x37,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00,0x3f,0x00,0x00,0x00, -0x3d,0x00,0x00,0x00,0x3e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x3f,0x00,0x00,0x00, -0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x42,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0x3b,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x47,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x3b,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00, -0x49,0x00,0x00,0x00,0x3d,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x4a,0x00,0x00,0x00, -0x49,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00, -0x4d,0x00,0x00,0x00,0x4c,0x00,0x00,0x00,0x3e,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, -0x4d,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x50,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x4f,0x00,0x00,0x00, -0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x55,0x00,0x00,0x00, -0x50,0x00,0x00,0x00,0x54,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x59,0x00,0x00,0x00,0x50,0x00,0x00,0x00, -0x58,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x5d,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x4f,0x00,0x00,0x00, -0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x64,0x00,0x00,0x00, -0x5d,0x00,0x00,0x00,0x63,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x68,0x00,0x00,0x00,0x5d,0x00,0x00,0x00, -0x67,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x6f,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x6e,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x74,0x00,0x00,0x00, -0x4e,0x00,0x00,0x00,0x73,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x78,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x77,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x79,0x00,0x00,0x00,0x78,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x7a,0x00,0x00,0x00,0x47,0x00,0x00,0x00, -0x79,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0x7d,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x7c,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x7e,0x00,0x00,0x00, -0x7d,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x80,0x00,0x00,0x00,0x47,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x83,0x00,0x00,0x00, -0x80,0x00,0x00,0x00,0x79,0x00,0x00,0x00,0x0c,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x26,0x00,0x00,0x00,0x7e,0x00,0x00,0x00,0x83,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x88,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x87,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x89,0x00,0x00,0x00,0x88,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8a,0x00,0x00,0x00, -0x32,0x00,0x00,0x00,0x89,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x8c,0x00,0x00,0x00,0x42,0x00,0x00,0x00, -0x37,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0x8e,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x8d,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x8f,0x00,0x00,0x00, -0x8e,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x90,0x00,0x00,0x00,0x8c,0x00,0x00,0x00,0x8f,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x91,0x00,0x00,0x00, -0x8a,0x00,0x00,0x00,0x90,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x93,0x00,0x00,0x00,0x91,0x00,0x00,0x00, -0x7a,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x94,0x00,0x00,0x00,0x93,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x99,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x98,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x9a,0x00,0x00,0x00,0x99,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9b,0x00,0x00,0x00, -0x0f,0x00,0x00,0x00,0x9a,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x9e,0x00,0x00,0x00,0x4a,0x00,0x00,0x00, -0x9d,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0xa0,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x9f,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xa1,0x00,0x00,0x00, -0xa0,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xa2,0x00,0x00,0x00,0x9e,0x00,0x00,0x00,0xa1,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa3,0x00,0x00,0x00, -0x9b,0x00,0x00,0x00,0xa2,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xa5,0x00,0x00,0x00,0xa3,0x00,0x00,0x00, -0x7a,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xa6,0x00,0x00,0x00,0xa5,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xa8,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xa8,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0x38,0x03,0x00,0x00,0x3e,0x00,0x00,0x00,0x05,0x00,0x00,0x00, -0xc7,0x00,0x00,0x00,0xa9,0x00,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb8,0x00,0x00,0x00,0xb9,0x00,0x00,0x00,0x38,0x03,0x00,0x00, -0xb7,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xaa,0x00,0x00,0x00, -0xa9,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0xb9,0x00,0x00,0x00,0xa9,0x00,0x00,0x00,0xaa,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xa9,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0xc3,0x00,0x00,0x00,0xc4,0x00,0x00,0x00,0xc0,0x00,0x00,0x00, -0x38,0x03,0x00,0x00,0x3e,0x00,0x03,0x00,0xc4,0x00,0x00,0x00, -0xc2,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xc7,0x00,0x00,0x00,0x38,0x03,0x00,0x00,0xc6,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xa8,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xaa,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xca,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xca,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0x51,0x03,0x00,0x00,0xa6,0x00,0x00,0x00, -0xaa,0x00,0x00,0x00,0x05,0x02,0x00,0x00,0xcd,0x00,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x4d,0x03,0x00,0x00, -0x94,0x00,0x00,0x00,0xaa,0x00,0x00,0x00,0x02,0x02,0x00,0x00, -0xcd,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0x39,0x03,0x00,0x00,0x7a,0x00,0x00,0x00,0xaa,0x00,0x00,0x00, -0xb3,0x02,0x00,0x00,0xcd,0x00,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb8,0x00,0x00,0x00,0xd1,0x00,0x00,0x00,0x39,0x03,0x00,0x00, -0x84,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xcc,0x00,0x00,0x00, -0xcd,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0xd1,0x00,0x00,0x00,0xcb,0x00,0x00,0x00,0xcc,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xcb,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xd3,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xd3,0x00,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x49,0x03,0x00,0x00, -0x3e,0x00,0x00,0x00,0xcb,0x00,0x00,0x00,0x6f,0x01,0x00,0x00, -0xd4,0x00,0x00,0x00,0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00, -0xd9,0x00,0x00,0x00,0x49,0x03,0x00,0x00,0x37,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0xd5,0x00,0x00,0x00,0xd4,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xd9,0x00,0x00,0x00, -0xd4,0x00,0x00,0x00,0xd5,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xd4,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xde,0x00,0x00,0x00,0x74,0x00,0x00,0x00,0x49,0x03,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xe1,0x00,0x00,0x00, -0xde,0x00,0x00,0x00,0x8f,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xe2,0x00,0x00,0x00,0xe1,0x00,0x00,0x00, -0x6d,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xe3,0x00,0x00,0x00,0x4d,0x03,0x00,0x00,0xe2,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xe5,0x00,0x00,0x00, -0xe3,0x00,0x00,0x00,0x6f,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xf0,0x00,0x00,0x00,0xde,0x00,0x00,0x00, -0xef,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xf2,0x00,0x00,0x00,0x6f,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xf3,0x00,0x00,0x00, -0xf0,0x00,0x00,0x00,0xf2,0x00,0x00,0x00,0x41,0x00,0x08,0x00, -0xfc,0x00,0x00,0x00,0xfd,0x00,0x00,0x00,0xfa,0x00,0x00,0x00, -0x34,0x00,0x00,0x00,0xe5,0x00,0x00,0x00,0x34,0x00,0x00,0x00, -0x3e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0xba,0x00,0x00,0x00, -0xfe,0x00,0x00,0x00,0xfd,0x00,0x00,0x00,0x73,0x00,0x04,0x00, -0xe6,0x00,0x00,0x00,0xff,0x00,0x00,0x00,0xfe,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x00,0x01,0x00,0x00,0x01,0x01,0x00,0x00, -0xeb,0x00,0x00,0x00,0xf3,0x00,0x00,0x00,0x3e,0x00,0x03,0x00, -0x01,0x01,0x00,0x00,0xff,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x06,0x01,0x00,0x00,0xde,0x00,0x00,0x00, -0x05,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x09,0x01,0x00,0x00,0x06,0x01,0x00,0x00,0xf2,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x0a,0x01,0x00,0x00, -0x09,0x01,0x00,0x00,0x39,0x00,0x00,0x00,0x41,0x00,0x08,0x00, -0xfc,0x00,0x00,0x00,0x0c,0x01,0x00,0x00,0xfa,0x00,0x00,0x00, -0x34,0x00,0x00,0x00,0xe5,0x00,0x00,0x00,0x34,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0xba,0x00,0x00,0x00, -0x0d,0x01,0x00,0x00,0x0c,0x01,0x00,0x00,0x73,0x00,0x04,0x00, -0xe6,0x00,0x00,0x00,0x0e,0x01,0x00,0x00,0x0d,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0x00,0x01,0x00,0x00,0x0f,0x01,0x00,0x00, -0xeb,0x00,0x00,0x00,0x0a,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0x0f,0x01,0x00,0x00,0x0e,0x01,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x14,0x01,0x00,0x00,0xde,0x00,0x00,0x00, -0x13,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x17,0x01,0x00,0x00,0x14,0x01,0x00,0x00,0xf2,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x18,0x01,0x00,0x00, -0x17,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x41,0x00,0x08,0x00, -0xfc,0x00,0x00,0x00,0x1a,0x01,0x00,0x00,0xfa,0x00,0x00,0x00, -0x34,0x00,0x00,0x00,0xe5,0x00,0x00,0x00,0x34,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0xba,0x00,0x00,0x00, -0x1b,0x01,0x00,0x00,0x1a,0x01,0x00,0x00,0x73,0x00,0x04,0x00, -0xe6,0x00,0x00,0x00,0x1c,0x01,0x00,0x00,0x1b,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0x00,0x01,0x00,0x00,0x1d,0x01,0x00,0x00, -0xeb,0x00,0x00,0x00,0x18,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0x1d,0x01,0x00,0x00,0x1c,0x01,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x22,0x01,0x00,0x00,0xde,0x00,0x00,0x00, -0x21,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x25,0x01,0x00,0x00,0x22,0x01,0x00,0x00,0xf2,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x27,0x01,0x00,0x00, -0x25,0x01,0x00,0x00,0x26,0x01,0x00,0x00,0x41,0x00,0x08,0x00, -0xfc,0x00,0x00,0x00,0x29,0x01,0x00,0x00,0xfa,0x00,0x00,0x00, -0x34,0x00,0x00,0x00,0xe5,0x00,0x00,0x00,0x34,0x00,0x00,0x00, -0x26,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0xba,0x00,0x00,0x00, -0x2a,0x01,0x00,0x00,0x29,0x01,0x00,0x00,0x73,0x00,0x04,0x00, -0xe6,0x00,0x00,0x00,0x2b,0x01,0x00,0x00,0x2a,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0x00,0x01,0x00,0x00,0x2c,0x01,0x00,0x00, -0xeb,0x00,0x00,0x00,0x27,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0x2c,0x01,0x00,0x00,0x2b,0x01,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x31,0x01,0x00,0x00,0xde,0x00,0x00,0x00, -0x30,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x34,0x01,0x00,0x00,0x31,0x01,0x00,0x00,0xf2,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x36,0x01,0x00,0x00, -0x34,0x01,0x00,0x00,0x35,0x01,0x00,0x00,0x41,0x00,0x08,0x00, -0xfc,0x00,0x00,0x00,0x38,0x01,0x00,0x00,0xfa,0x00,0x00,0x00, -0x34,0x00,0x00,0x00,0xe5,0x00,0x00,0x00,0xc6,0x00,0x00,0x00, -0x3e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0xba,0x00,0x00,0x00, -0x39,0x01,0x00,0x00,0x38,0x01,0x00,0x00,0x73,0x00,0x04,0x00, -0xe6,0x00,0x00,0x00,0x3a,0x01,0x00,0x00,0x39,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0x00,0x01,0x00,0x00,0x3b,0x01,0x00,0x00, -0xeb,0x00,0x00,0x00,0x36,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0x3b,0x01,0x00,0x00,0x3a,0x01,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x40,0x01,0x00,0x00,0xde,0x00,0x00,0x00, -0x3f,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x43,0x01,0x00,0x00,0x40,0x01,0x00,0x00,0xf2,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x45,0x01,0x00,0x00, -0x43,0x01,0x00,0x00,0x44,0x01,0x00,0x00,0x41,0x00,0x08,0x00, -0xfc,0x00,0x00,0x00,0x47,0x01,0x00,0x00,0xfa,0x00,0x00,0x00, -0x34,0x00,0x00,0x00,0xe5,0x00,0x00,0x00,0xc6,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0xba,0x00,0x00,0x00, -0x48,0x01,0x00,0x00,0x47,0x01,0x00,0x00,0x73,0x00,0x04,0x00, -0xe6,0x00,0x00,0x00,0x49,0x01,0x00,0x00,0x48,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0x00,0x01,0x00,0x00,0x4a,0x01,0x00,0x00, -0xeb,0x00,0x00,0x00,0x45,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0x4a,0x01,0x00,0x00,0x49,0x01,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x4f,0x01,0x00,0x00,0xde,0x00,0x00,0x00, -0x4e,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x52,0x01,0x00,0x00,0x4f,0x01,0x00,0x00,0xf2,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x54,0x01,0x00,0x00, -0x52,0x01,0x00,0x00,0x53,0x01,0x00,0x00,0x41,0x00,0x08,0x00, -0xfc,0x00,0x00,0x00,0x56,0x01,0x00,0x00,0xfa,0x00,0x00,0x00, -0x34,0x00,0x00,0x00,0xe5,0x00,0x00,0x00,0xc6,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0xba,0x00,0x00,0x00, -0x57,0x01,0x00,0x00,0x56,0x01,0x00,0x00,0x73,0x00,0x04,0x00, -0xe6,0x00,0x00,0x00,0x58,0x01,0x00,0x00,0x57,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0x00,0x01,0x00,0x00,0x59,0x01,0x00,0x00, -0xeb,0x00,0x00,0x00,0x54,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0x59,0x01,0x00,0x00,0x58,0x01,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x5e,0x01,0x00,0x00,0xde,0x00,0x00,0x00, -0x5d,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x61,0x01,0x00,0x00,0x5e,0x01,0x00,0x00,0xf2,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x63,0x01,0x00,0x00, -0x61,0x01,0x00,0x00,0x62,0x01,0x00,0x00,0x41,0x00,0x08,0x00, -0xfc,0x00,0x00,0x00,0x65,0x01,0x00,0x00,0xfa,0x00,0x00,0x00, -0x34,0x00,0x00,0x00,0xe5,0x00,0x00,0x00,0xc6,0x00,0x00,0x00, -0x26,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0xba,0x00,0x00,0x00, -0x66,0x01,0x00,0x00,0x65,0x01,0x00,0x00,0x73,0x00,0x04,0x00, -0xe6,0x00,0x00,0x00,0x67,0x01,0x00,0x00,0x66,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0x00,0x01,0x00,0x00,0x68,0x01,0x00,0x00, -0xeb,0x00,0x00,0x00,0x63,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0x68,0x01,0x00,0x00,0x67,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x6f,0x01,0x00,0x00,0x49,0x03,0x00,0x00, -0x6d,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xd3,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xd5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x71,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x71,0x01,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x4a,0x03,0x00,0x00, -0x3e,0x00,0x00,0x00,0xd5,0x00,0x00,0x00,0xfe,0x01,0x00,0x00, -0x72,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00, -0x77,0x01,0x00,0x00,0x4a,0x03,0x00,0x00,0x9d,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0x73,0x01,0x00,0x00,0x72,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x77,0x01,0x00,0x00, -0x72,0x01,0x00,0x00,0x73,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x72,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x7c,0x01,0x00,0x00,0x74,0x00,0x00,0x00,0x4a,0x03,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x7f,0x01,0x00,0x00, -0x7c,0x01,0x00,0x00,0xa1,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x80,0x01,0x00,0x00,0x7f,0x01,0x00,0x00, -0x6d,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x81,0x01,0x00,0x00,0x51,0x03,0x00,0x00,0x80,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x83,0x01,0x00,0x00, -0x81,0x01,0x00,0x00,0x6f,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x8d,0x01,0x00,0x00,0x7c,0x01,0x00,0x00, -0x8c,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x8f,0x01,0x00,0x00,0x6f,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x90,0x01,0x00,0x00, -0x8d,0x01,0x00,0x00,0x8f,0x01,0x00,0x00,0x41,0x00,0x08,0x00, -0xfc,0x00,0x00,0x00,0x97,0x01,0x00,0x00,0x95,0x01,0x00,0x00, -0x34,0x00,0x00,0x00,0x83,0x01,0x00,0x00,0x34,0x00,0x00,0x00, -0x3e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0xba,0x00,0x00,0x00, -0x98,0x01,0x00,0x00,0x97,0x01,0x00,0x00,0x73,0x00,0x04,0x00, -0xe6,0x00,0x00,0x00,0x99,0x01,0x00,0x00,0x98,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0x00,0x01,0x00,0x00,0x9a,0x01,0x00,0x00, -0x88,0x01,0x00,0x00,0x90,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0x9a,0x01,0x00,0x00,0x99,0x01,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x9f,0x01,0x00,0x00,0x7c,0x01,0x00,0x00, -0x9e,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xa2,0x01,0x00,0x00,0x9f,0x01,0x00,0x00,0x8f,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa3,0x01,0x00,0x00, -0xa2,0x01,0x00,0x00,0x39,0x00,0x00,0x00,0x41,0x00,0x08,0x00, -0xfc,0x00,0x00,0x00,0xa5,0x01,0x00,0x00,0x95,0x01,0x00,0x00, -0x34,0x00,0x00,0x00,0x83,0x01,0x00,0x00,0x34,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0xba,0x00,0x00,0x00, -0xa6,0x01,0x00,0x00,0xa5,0x01,0x00,0x00,0x73,0x00,0x04,0x00, -0xe6,0x00,0x00,0x00,0xa7,0x01,0x00,0x00,0xa6,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0x00,0x01,0x00,0x00,0xa8,0x01,0x00,0x00, -0x88,0x01,0x00,0x00,0xa3,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0xa8,0x01,0x00,0x00,0xa7,0x01,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xad,0x01,0x00,0x00,0x7c,0x01,0x00,0x00, -0xac,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xb0,0x01,0x00,0x00,0xad,0x01,0x00,0x00,0x8f,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xb1,0x01,0x00,0x00, -0xb0,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x41,0x00,0x08,0x00, -0xfc,0x00,0x00,0x00,0xb3,0x01,0x00,0x00,0x95,0x01,0x00,0x00, -0x34,0x00,0x00,0x00,0x83,0x01,0x00,0x00,0x34,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0xba,0x00,0x00,0x00, -0xb4,0x01,0x00,0x00,0xb3,0x01,0x00,0x00,0x73,0x00,0x04,0x00, -0xe6,0x00,0x00,0x00,0xb5,0x01,0x00,0x00,0xb4,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0x00,0x01,0x00,0x00,0xb6,0x01,0x00,0x00, -0x88,0x01,0x00,0x00,0xb1,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0xb6,0x01,0x00,0x00,0xb5,0x01,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xbb,0x01,0x00,0x00,0x7c,0x01,0x00,0x00, -0xba,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xbe,0x01,0x00,0x00,0xbb,0x01,0x00,0x00,0x8f,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xbf,0x01,0x00,0x00, -0xbe,0x01,0x00,0x00,0x26,0x01,0x00,0x00,0x41,0x00,0x08,0x00, -0xfc,0x00,0x00,0x00,0xc1,0x01,0x00,0x00,0x95,0x01,0x00,0x00, -0x34,0x00,0x00,0x00,0x83,0x01,0x00,0x00,0x34,0x00,0x00,0x00, -0x26,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0xba,0x00,0x00,0x00, -0xc2,0x01,0x00,0x00,0xc1,0x01,0x00,0x00,0x73,0x00,0x04,0x00, -0xe6,0x00,0x00,0x00,0xc3,0x01,0x00,0x00,0xc2,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0x00,0x01,0x00,0x00,0xc4,0x01,0x00,0x00, -0x88,0x01,0x00,0x00,0xbf,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0xc4,0x01,0x00,0x00,0xc3,0x01,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xc9,0x01,0x00,0x00,0x7c,0x01,0x00,0x00, -0xc8,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xcc,0x01,0x00,0x00,0xc9,0x01,0x00,0x00,0x8f,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xcd,0x01,0x00,0x00, -0xcc,0x01,0x00,0x00,0x35,0x01,0x00,0x00,0x41,0x00,0x08,0x00, -0xfc,0x00,0x00,0x00,0xcf,0x01,0x00,0x00,0x95,0x01,0x00,0x00, -0x34,0x00,0x00,0x00,0x83,0x01,0x00,0x00,0xc6,0x00,0x00,0x00, -0x3e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0xba,0x00,0x00,0x00, -0xd0,0x01,0x00,0x00,0xcf,0x01,0x00,0x00,0x73,0x00,0x04,0x00, -0xe6,0x00,0x00,0x00,0xd1,0x01,0x00,0x00,0xd0,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0x00,0x01,0x00,0x00,0xd2,0x01,0x00,0x00, -0x88,0x01,0x00,0x00,0xcd,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0xd2,0x01,0x00,0x00,0xd1,0x01,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xd7,0x01,0x00,0x00,0x7c,0x01,0x00,0x00, -0xd6,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xda,0x01,0x00,0x00,0xd7,0x01,0x00,0x00,0x8f,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xdb,0x01,0x00,0x00, -0xda,0x01,0x00,0x00,0x44,0x01,0x00,0x00,0x41,0x00,0x08,0x00, -0xfc,0x00,0x00,0x00,0xdd,0x01,0x00,0x00,0x95,0x01,0x00,0x00, -0x34,0x00,0x00,0x00,0x83,0x01,0x00,0x00,0xc6,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0xba,0x00,0x00,0x00, -0xde,0x01,0x00,0x00,0xdd,0x01,0x00,0x00,0x73,0x00,0x04,0x00, -0xe6,0x00,0x00,0x00,0xdf,0x01,0x00,0x00,0xde,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0x00,0x01,0x00,0x00,0xe0,0x01,0x00,0x00, -0x88,0x01,0x00,0x00,0xdb,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0xe0,0x01,0x00,0x00,0xdf,0x01,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xe5,0x01,0x00,0x00,0x7c,0x01,0x00,0x00, -0xe4,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xe8,0x01,0x00,0x00,0xe5,0x01,0x00,0x00,0x8f,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xe9,0x01,0x00,0x00, -0xe8,0x01,0x00,0x00,0x53,0x01,0x00,0x00,0x41,0x00,0x08,0x00, -0xfc,0x00,0x00,0x00,0xeb,0x01,0x00,0x00,0x95,0x01,0x00,0x00, -0x34,0x00,0x00,0x00,0x83,0x01,0x00,0x00,0xc6,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0xba,0x00,0x00,0x00, -0xec,0x01,0x00,0x00,0xeb,0x01,0x00,0x00,0x73,0x00,0x04,0x00, -0xe6,0x00,0x00,0x00,0xed,0x01,0x00,0x00,0xec,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0x00,0x01,0x00,0x00,0xee,0x01,0x00,0x00, -0x88,0x01,0x00,0x00,0xe9,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0xee,0x01,0x00,0x00,0xed,0x01,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xf3,0x01,0x00,0x00,0x7c,0x01,0x00,0x00, -0xf2,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xf6,0x01,0x00,0x00,0xf3,0x01,0x00,0x00,0x8f,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xf7,0x01,0x00,0x00, -0xf6,0x01,0x00,0x00,0x62,0x01,0x00,0x00,0x41,0x00,0x08,0x00, -0xfc,0x00,0x00,0x00,0xf9,0x01,0x00,0x00,0x95,0x01,0x00,0x00, -0x34,0x00,0x00,0x00,0x83,0x01,0x00,0x00,0xc6,0x00,0x00,0x00, -0x26,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0xba,0x00,0x00,0x00, -0xfa,0x01,0x00,0x00,0xf9,0x01,0x00,0x00,0x73,0x00,0x04,0x00, -0xe6,0x00,0x00,0x00,0xfb,0x01,0x00,0x00,0xfa,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0x00,0x01,0x00,0x00,0xfc,0x01,0x00,0x00, -0x88,0x01,0x00,0x00,0xf7,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0xfc,0x01,0x00,0x00,0xfb,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xfe,0x01,0x00,0x00,0x4a,0x03,0x00,0x00, -0x6d,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x71,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x73,0x01,0x00,0x00,0xe0,0x00,0x04,0x00, -0x0c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0xff,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x02,0x02,0x00,0x00, -0x4d,0x03,0x00,0x00,0x00,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x05,0x02,0x00,0x00,0x51,0x03,0x00,0x00, -0x03,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x07,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x07,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0x53,0x03,0x00,0x00,0x3e,0x00,0x00,0x00, -0x73,0x01,0x00,0x00,0xb1,0x02,0x00,0x00,0x0a,0x02,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00,0x0d,0x02,0x00,0x00, -0x53,0x03,0x00,0x00,0x6c,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0x09,0x02,0x00,0x00,0x0a,0x02,0x00,0x00,0x00,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x0d,0x02,0x00,0x00,0x08,0x02,0x00,0x00, -0x09,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x08,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0x0f,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x0f,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0x57,0x03,0x00,0x00,0x3e,0x00,0x00,0x00,0x08,0x02,0x00,0x00, -0x3b,0x02,0x00,0x00,0x12,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb8,0x00,0x00,0x00,0x15,0x02,0x00,0x00,0x57,0x03,0x00,0x00, -0x60,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x11,0x02,0x00,0x00, -0x12,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x15,0x02,0x00,0x00,0x10,0x02,0x00,0x00,0x11,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x10,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x17,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x17,0x02,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x69,0x03,0x00,0x00, -0x3e,0x00,0x00,0x00,0x10,0x02,0x00,0x00,0x39,0x02,0x00,0x00, -0x18,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00, -0x1d,0x02,0x00,0x00,0x69,0x03,0x00,0x00,0x62,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0x19,0x02,0x00,0x00,0x18,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x1d,0x02,0x00,0x00, -0x18,0x02,0x00,0x00,0x19,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x18,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x23,0x02,0x00,0x00,0x57,0x03,0x00,0x00,0x62,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x25,0x02,0x00,0x00, -0x23,0x02,0x00,0x00,0x69,0x03,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x27,0x02,0x00,0x00,0x55,0x00,0x00,0x00, -0x53,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x29,0x02,0x00,0x00,0x57,0x03,0x00,0x00,0x61,0x00,0x00,0x00, +0x89,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xd4,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0x3e,0x01,0x00,0x00, +0x30,0x02,0x00,0x00,0x8c,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc2,0x00,0x00,0x00,0x8f,0x01,0x00,0x00,0xd4,0x02,0x00,0x00, +0x6c,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x8b,0x01,0x00,0x00, +0x8c,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x8f,0x01,0x00,0x00,0x8a,0x01,0x00,0x00,0x8b,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x8a,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0x91,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x91,0x01,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xd8,0x02,0x00,0x00, +0x3e,0x00,0x00,0x00,0x8a,0x01,0x00,0x00,0xbc,0x01,0x00,0x00, +0x94,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, +0x97,0x01,0x00,0x00,0xd8,0x02,0x00,0x00,0x60,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0x93,0x01,0x00,0x00,0x94,0x01,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x97,0x01,0x00,0x00, +0x92,0x01,0x00,0x00,0x93,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x92,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x99,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x99,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xea,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, +0x92,0x01,0x00,0x00,0xba,0x01,0x00,0x00,0x9a,0x01,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0x9f,0x01,0x00,0x00, +0xea,0x02,0x00,0x00,0x62,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0x9b,0x01,0x00,0x00,0x9a,0x01,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x9f,0x01,0x00,0x00,0x9a,0x01,0x00,0x00, +0x9b,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x9a,0x01,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa5,0x01,0x00,0x00, +0xd8,0x02,0x00,0x00,0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xa7,0x01,0x00,0x00,0xa5,0x01,0x00,0x00, +0xea,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xa9,0x01,0x00,0x00,0x55,0x00,0x00,0x00,0x53,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xab,0x01,0x00,0x00, +0xd8,0x02,0x00,0x00,0x61,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xac,0x01,0x00,0x00,0xa9,0x01,0x00,0x00, +0xab,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xae,0x01,0x00,0x00,0x64,0x00,0x00,0x00,0x62,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xaf,0x01,0x00,0x00, +0xac,0x01,0x00,0x00,0xae,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xb1,0x01,0x00,0x00,0xaf,0x01,0x00,0x00, +0xea,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xb3,0x01,0x00,0x00,0xb1,0x01,0x00,0x00,0xb2,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xb5,0x01,0x00,0x00, +0xb3,0x01,0x00,0x00,0xd4,0x02,0x00,0x00,0x41,0x00,0x05,0x00, +0x2d,0x01,0x00,0x00,0xb6,0x01,0x00,0x00,0x29,0x01,0x00,0x00, +0xb5,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00, +0xb7,0x01,0x00,0x00,0xb6,0x01,0x00,0x00,0x41,0x00,0x05,0x00, +0xcd,0x00,0x00,0x00,0xb8,0x01,0x00,0x00,0xa3,0x01,0x00,0x00, +0xa7,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0xb8,0x01,0x00,0x00, +0xb7,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xba,0x01,0x00,0x00,0xea,0x02,0x00,0x00,0xd0,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x99,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x9b,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x94,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x94,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xbc,0x01,0x00,0x00,0xd8,0x02,0x00,0x00, +0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x91,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x93,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0xbe,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xbe,0x01,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xd9,0x02,0x00,0x00, +0x3e,0x00,0x00,0x00,0x93,0x01,0x00,0x00,0xea,0x01,0x00,0x00, +0xc1,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, +0xc4,0x01,0x00,0x00,0xd9,0x02,0x00,0x00,0xbf,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0xc0,0x01,0x00,0x00,0xc1,0x01,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xc4,0x01,0x00,0x00, +0xbf,0x01,0x00,0x00,0xc0,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xbf,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xc6,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xc6,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xe7,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, +0xbf,0x01,0x00,0x00,0xe8,0x01,0x00,0x00,0xc7,0x01,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0xcc,0x01,0x00,0x00, +0xe7,0x02,0x00,0x00,0xbc,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0xc8,0x01,0x00,0x00,0xc7,0x01,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0xcc,0x01,0x00,0x00,0xc7,0x01,0x00,0x00, +0xc8,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xc7,0x01,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xd2,0x01,0x00,0x00, +0xd9,0x02,0x00,0x00,0xbc,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xd4,0x01,0x00,0x00,0xd2,0x01,0x00,0x00, +0xe7,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xd6,0x01,0x00,0x00,0x59,0x00,0x00,0x00,0xb9,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xd9,0x01,0x00,0x00, +0xd9,0x02,0x00,0x00,0xd8,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xda,0x01,0x00,0x00,0xd6,0x01,0x00,0x00, +0xd9,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xdc,0x01,0x00,0x00,0x68,0x00,0x00,0x00,0xbc,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xdd,0x01,0x00,0x00, +0xda,0x01,0x00,0x00,0xdc,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xdf,0x01,0x00,0x00,0xdd,0x01,0x00,0x00, +0xe7,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xe1,0x01,0x00,0x00,0xdf,0x01,0x00,0x00,0xe0,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xe3,0x01,0x00,0x00, +0xe1,0x01,0x00,0x00,0xd4,0x02,0x00,0x00,0x41,0x00,0x05,0x00, +0x2d,0x01,0x00,0x00,0xe4,0x01,0x00,0x00,0x5c,0x01,0x00,0x00, +0xe3,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00, +0xe5,0x01,0x00,0x00,0xe4,0x01,0x00,0x00,0x41,0x00,0x05,0x00, +0xcd,0x00,0x00,0x00,0xe6,0x01,0x00,0x00,0xd0,0x01,0x00,0x00, +0xd4,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0xe6,0x01,0x00,0x00, +0xe5,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xe8,0x01,0x00,0x00,0xe7,0x02,0x00,0x00,0xd0,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xc6,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xc8,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xc1,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xc1,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xea,0x01,0x00,0x00,0xd9,0x02,0x00,0x00, +0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xbe,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xc0,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0xec,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xec,0x01,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xda,0x02,0x00,0x00, +0x3e,0x00,0x00,0x00,0xc0,0x01,0x00,0x00,0x2e,0x02,0x00,0x00, +0xef,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, +0xf2,0x01,0x00,0x00,0xda,0x02,0x00,0x00,0xbf,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0xee,0x01,0x00,0x00,0xef,0x01,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xf2,0x01,0x00,0x00, +0xed,0x01,0x00,0x00,0xee,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xed,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xf4,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xf4,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xde,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, +0xed,0x01,0x00,0x00,0x2c,0x02,0x00,0x00,0xf7,0x01,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0xfa,0x01,0x00,0x00, +0xde,0x02,0x00,0x00,0x60,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0xf6,0x01,0x00,0x00,0xf7,0x01,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0xfa,0x01,0x00,0x00,0xf5,0x01,0x00,0x00, +0xf6,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xf5,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0xfc,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xfc,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xe0,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0xf5,0x01,0x00,0x00, +0x2a,0x02,0x00,0x00,0xff,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc2,0x00,0x00,0x00,0x02,0x02,0x00,0x00,0xe0,0x02,0x00,0x00, +0xbc,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xfe,0x01,0x00,0x00, +0xff,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x02,0x02,0x00,0x00,0xfd,0x01,0x00,0x00,0xfe,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xfd,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0x04,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x04,0x02,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xe2,0x02,0x00,0x00, +0x3e,0x00,0x00,0x00,0xfd,0x01,0x00,0x00,0x28,0x02,0x00,0x00, +0x05,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, +0x0a,0x02,0x00,0x00,0xe2,0x02,0x00,0x00,0x62,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0x06,0x02,0x00,0x00,0x05,0x02,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x0a,0x02,0x00,0x00, +0x05,0x02,0x00,0x00,0x06,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x05,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x0c,0x02,0x00,0x00,0xda,0x02,0x00,0x00,0xbc,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x0e,0x02,0x00,0x00, +0x0c,0x02,0x00,0x00,0xe0,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x10,0x02,0x00,0x00,0x0e,0x02,0x00,0x00, +0x0f,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x12,0x02,0x00,0x00,0xde,0x02,0x00,0x00,0x62,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x13,0x02,0x00,0x00, +0x10,0x02,0x00,0x00,0x12,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x15,0x02,0x00,0x00,0x13,0x02,0x00,0x00, +0xe2,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x19,0x02,0x00,0x00,0x12,0x02,0x00,0x00,0xe2,0x02,0x00,0x00, +0x41,0x00,0x05,0x00,0xcd,0x00,0x00,0x00,0x1a,0x02,0x00,0x00, +0xa3,0x01,0x00,0x00,0x19,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, +0xc4,0x00,0x00,0x00,0x1b,0x02,0x00,0x00,0x1a,0x02,0x00,0x00, +0x41,0x00,0x05,0x00,0xcd,0x00,0x00,0x00,0x20,0x02,0x00,0x00, +0xd0,0x01,0x00,0x00,0x0e,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, +0xc4,0x00,0x00,0x00,0x21,0x02,0x00,0x00,0x20,0x02,0x00,0x00, +0x41,0x00,0x05,0x00,0xcd,0x00,0x00,0x00,0x23,0x02,0x00,0x00, +0xca,0x00,0x00,0x00,0x15,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, +0xc4,0x00,0x00,0x00,0x24,0x02,0x00,0x00,0x23,0x02,0x00,0x00, +0x0c,0x00,0x08,0x00,0xc4,0x00,0x00,0x00,0x25,0x02,0x00,0x00, +0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x1b,0x02,0x00,0x00, +0x21,0x02,0x00,0x00,0x24,0x02,0x00,0x00,0x3e,0x00,0x03,0x00, +0x23,0x02,0x00,0x00,0x25,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x28,0x02,0x00,0x00,0xe2,0x02,0x00,0x00, +0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x04,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x06,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0xff,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xff,0x01,0x00,0x00, 0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x2a,0x02,0x00,0x00, -0x27,0x02,0x00,0x00,0x29,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x2c,0x02,0x00,0x00,0x64,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x2d,0x02,0x00,0x00,0x2a,0x02,0x00,0x00,0x2c,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x2f,0x02,0x00,0x00, -0x2d,0x02,0x00,0x00,0x69,0x03,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x31,0x02,0x00,0x00,0x2f,0x02,0x00,0x00, -0x30,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x33,0x02,0x00,0x00,0x31,0x02,0x00,0x00,0x53,0x03,0x00,0x00, -0x41,0x00,0x05,0x00,0x00,0x01,0x00,0x00,0x34,0x02,0x00,0x00, -0xeb,0x00,0x00,0x00,0x33,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, -0xe6,0x00,0x00,0x00,0x35,0x02,0x00,0x00,0x34,0x02,0x00,0x00, -0x41,0x00,0x05,0x00,0x36,0x02,0x00,0x00,0x37,0x02,0x00,0x00, -0x21,0x02,0x00,0x00,0x25,0x02,0x00,0x00,0x3e,0x00,0x03,0x00, -0x37,0x02,0x00,0x00,0x35,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x39,0x02,0x00,0x00,0x69,0x03,0x00,0x00, -0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x17,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x19,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x12,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x12,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3b,0x02,0x00,0x00, -0x57,0x03,0x00,0x00,0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x0f,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x11,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0x3d,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x3d,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0x58,0x03,0x00,0x00,0x3e,0x00,0x00,0x00,0x11,0x02,0x00,0x00, -0x69,0x02,0x00,0x00,0x40,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb8,0x00,0x00,0x00,0x43,0x02,0x00,0x00,0x58,0x03,0x00,0x00, -0xb5,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x3f,0x02,0x00,0x00, -0x40,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x43,0x02,0x00,0x00,0x3e,0x02,0x00,0x00,0x3f,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x3e,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x45,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x45,0x02,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x66,0x03,0x00,0x00, -0x3e,0x00,0x00,0x00,0x3e,0x02,0x00,0x00,0x67,0x02,0x00,0x00, -0x46,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00, -0x4b,0x02,0x00,0x00,0x66,0x03,0x00,0x00,0xb2,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0x47,0x02,0x00,0x00,0x46,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x4b,0x02,0x00,0x00, -0x46,0x02,0x00,0x00,0x47,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x46,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x51,0x02,0x00,0x00,0x58,0x03,0x00,0x00,0xb2,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x53,0x02,0x00,0x00, -0x51,0x02,0x00,0x00,0x66,0x03,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x55,0x02,0x00,0x00,0x59,0x00,0x00,0x00, -0xaf,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x58,0x02,0x00,0x00,0x58,0x03,0x00,0x00,0x57,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x59,0x02,0x00,0x00, -0x55,0x02,0x00,0x00,0x58,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x5b,0x02,0x00,0x00,0x68,0x00,0x00,0x00, -0xb2,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x5c,0x02,0x00,0x00,0x59,0x02,0x00,0x00,0x5b,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x5e,0x02,0x00,0x00, -0x5c,0x02,0x00,0x00,0x66,0x03,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x60,0x02,0x00,0x00,0x5e,0x02,0x00,0x00, -0x5f,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x62,0x02,0x00,0x00,0x60,0x02,0x00,0x00,0x53,0x03,0x00,0x00, -0x41,0x00,0x05,0x00,0x00,0x01,0x00,0x00,0x63,0x02,0x00,0x00, -0x88,0x01,0x00,0x00,0x62,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, -0xe6,0x00,0x00,0x00,0x64,0x02,0x00,0x00,0x63,0x02,0x00,0x00, -0x41,0x00,0x05,0x00,0x36,0x02,0x00,0x00,0x65,0x02,0x00,0x00, -0x4f,0x02,0x00,0x00,0x53,0x02,0x00,0x00,0x3e,0x00,0x03,0x00, -0x65,0x02,0x00,0x00,0x64,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x67,0x02,0x00,0x00,0x66,0x03,0x00,0x00, -0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x45,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x47,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x40,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x40,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x69,0x02,0x00,0x00, -0x58,0x03,0x00,0x00,0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x3d,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x3f,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0x6b,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x6b,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0x59,0x03,0x00,0x00,0x3e,0x00,0x00,0x00,0x3f,0x02,0x00,0x00, -0xaf,0x02,0x00,0x00,0x6e,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb8,0x00,0x00,0x00,0x71,0x02,0x00,0x00,0x59,0x03,0x00,0x00, -0xb5,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x6d,0x02,0x00,0x00, -0x6e,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x71,0x02,0x00,0x00,0x6c,0x02,0x00,0x00,0x6d,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x6c,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x73,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x73,0x02,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x5d,0x03,0x00,0x00, -0x3e,0x00,0x00,0x00,0x6c,0x02,0x00,0x00,0xad,0x02,0x00,0x00, -0x76,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00, -0x79,0x02,0x00,0x00,0x5d,0x03,0x00,0x00,0x60,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0x75,0x02,0x00,0x00,0x76,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x79,0x02,0x00,0x00, -0x74,0x02,0x00,0x00,0x75,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x74,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x7b,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x7b,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0x5f,0x03,0x00,0x00,0x3e,0x00,0x00,0x00, -0x74,0x02,0x00,0x00,0xab,0x02,0x00,0x00,0x7e,0x02,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00,0x81,0x02,0x00,0x00, -0x5f,0x03,0x00,0x00,0xb2,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0x7d,0x02,0x00,0x00,0x7e,0x02,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x81,0x02,0x00,0x00,0x7c,0x02,0x00,0x00, -0x7d,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x7c,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0x83,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x83,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0x61,0x03,0x00,0x00,0x3e,0x00,0x00,0x00,0x7c,0x02,0x00,0x00, -0xa9,0x02,0x00,0x00,0x84,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb8,0x00,0x00,0x00,0x89,0x02,0x00,0x00,0x61,0x03,0x00,0x00, -0x62,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x85,0x02,0x00,0x00, -0x84,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x89,0x02,0x00,0x00,0x84,0x02,0x00,0x00,0x85,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x84,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x8b,0x02,0x00,0x00,0x59,0x03,0x00,0x00, -0xb2,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x8d,0x02,0x00,0x00,0x8b,0x02,0x00,0x00,0x5f,0x03,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8f,0x02,0x00,0x00, -0x8d,0x02,0x00,0x00,0x8e,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x91,0x02,0x00,0x00,0x5d,0x03,0x00,0x00, -0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x92,0x02,0x00,0x00,0x8f,0x02,0x00,0x00,0x91,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x94,0x02,0x00,0x00, -0x92,0x02,0x00,0x00,0x61,0x03,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x98,0x02,0x00,0x00,0x91,0x02,0x00,0x00, -0x61,0x03,0x00,0x00,0x41,0x00,0x05,0x00,0x36,0x02,0x00,0x00, -0x99,0x02,0x00,0x00,0x21,0x02,0x00,0x00,0x98,0x02,0x00,0x00, -0x3d,0x00,0x04,0x00,0xe6,0x00,0x00,0x00,0x9a,0x02,0x00,0x00, -0x99,0x02,0x00,0x00,0x73,0x00,0x04,0x00,0xba,0x00,0x00,0x00, -0x9b,0x02,0x00,0x00,0x9a,0x02,0x00,0x00,0x41,0x00,0x05,0x00, -0x36,0x02,0x00,0x00,0xa0,0x02,0x00,0x00,0x4f,0x02,0x00,0x00, -0x8d,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0xe6,0x00,0x00,0x00, -0xa1,0x02,0x00,0x00,0xa0,0x02,0x00,0x00,0x73,0x00,0x04,0x00, -0xba,0x00,0x00,0x00,0xa2,0x02,0x00,0x00,0xa1,0x02,0x00,0x00, -0x41,0x00,0x05,0x00,0xc3,0x00,0x00,0x00,0xa4,0x02,0x00,0x00, -0xc0,0x00,0x00,0x00,0x94,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, -0xba,0x00,0x00,0x00,0xa5,0x02,0x00,0x00,0xa4,0x02,0x00,0x00, -0x0c,0x00,0x08,0x00,0xba,0x00,0x00,0x00,0xa6,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x9b,0x02,0x00,0x00, -0xa2,0x02,0x00,0x00,0xa5,0x02,0x00,0x00,0x3e,0x00,0x03,0x00, -0xa4,0x02,0x00,0x00,0xa6,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xa9,0x02,0x00,0x00,0x61,0x03,0x00,0x00, -0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x83,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x85,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x7e,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x7e,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xab,0x02,0x00,0x00, -0x5f,0x03,0x00,0x00,0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x7b,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x7d,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0x76,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x76,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xad,0x02,0x00,0x00,0x5d,0x03,0x00,0x00,0xc6,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x73,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x75,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x6e,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x6e,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xaf,0x02,0x00,0x00,0x59,0x03,0x00,0x00, -0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x6b,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x6d,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x0a,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x0a,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xb1,0x02,0x00,0x00, -0x53,0x03,0x00,0x00,0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x07,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x09,0x02,0x00,0x00, +0xe0,0x02,0x00,0x00,0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xfc,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xfe,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0xf7,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xf7,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x2c,0x02,0x00,0x00,0xde,0x02,0x00,0x00,0xd0,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xf4,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xf6,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xef,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xef,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x2e,0x02,0x00,0x00,0xda,0x02,0x00,0x00, +0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xec,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xee,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0x8c,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x8c,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x30,0x02,0x00,0x00, +0xd4,0x02,0x00,0x00,0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x89,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x8b,0x01,0x00,0x00, 0xe0,0x00,0x04,0x00,0x0c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0xff,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xcd,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xcd,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xb3,0x02,0x00,0x00,0x39,0x03,0x00,0x00, -0x6c,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xca,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xcc,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xb8,0x02,0x00,0x00,0x55,0x00,0x00,0x00, +0x81,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xd7,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xd7,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x32,0x02,0x00,0x00,0xba,0x02,0x00,0x00, +0x6c,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xd4,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xd6,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x37,0x02,0x00,0x00,0x55,0x00,0x00,0x00, 0x53,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xb9,0x02,0x00,0x00,0x8c,0x00,0x00,0x00,0xb8,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xbe,0x02,0x00,0x00, -0x59,0x00,0x00,0x00,0xaf,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xbf,0x02,0x00,0x00,0x9e,0x00,0x00,0x00, -0xbe,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0xc4,0x02,0x00,0x00,0x12,0x00,0x00,0x00,0xc3,0x02,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xc5,0x02,0x00,0x00, -0xc4,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xc6,0x02,0x00,0x00,0x0f,0x00,0x00,0x00,0xc5,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xca,0x02,0x00,0x00, -0x47,0x00,0x00,0x00,0xc5,0x02,0x00,0x00,0x41,0x00,0x05,0x00, -0x0d,0x00,0x00,0x00,0xcc,0x02,0x00,0x00,0xcb,0x02,0x00,0x00, +0x38,0x02,0x00,0x00,0x96,0x00,0x00,0x00,0x37,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3d,0x02,0x00,0x00, +0x59,0x00,0x00,0x00,0xb9,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x3e,0x02,0x00,0x00,0xa8,0x00,0x00,0x00, +0x3d,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, +0x43,0x02,0x00,0x00,0x12,0x00,0x00,0x00,0x42,0x02,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x44,0x02,0x00,0x00, +0x43,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x45,0x02,0x00,0x00,0x0f,0x00,0x00,0x00,0x44,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x49,0x02,0x00,0x00, +0x47,0x00,0x00,0x00,0x44,0x02,0x00,0x00,0x41,0x00,0x05,0x00, +0x0d,0x00,0x00,0x00,0x4b,0x02,0x00,0x00,0x4a,0x02,0x00,0x00, 0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xcd,0x02,0x00,0x00,0xcc,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xce,0x02,0x00,0x00,0xca,0x02,0x00,0x00, -0xcd,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xcf,0x02,0x00,0x00,0xc6,0x02,0x00,0x00,0xce,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0xd1,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0xd1,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0x3a,0x03,0x00,0x00,0x3e,0x00,0x00,0x00,0xcc,0x00,0x00,0x00, -0x37,0x03,0x00,0x00,0xd4,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb8,0x00,0x00,0x00,0xd7,0x02,0x00,0x00,0x3a,0x03,0x00,0x00, -0xb5,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xd3,0x02,0x00,0x00, -0xd4,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0xd7,0x02,0x00,0x00,0xd2,0x02,0x00,0x00,0xd3,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0xd2,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0xd9,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0xd9,0x02,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x3b,0x03,0x00,0x00, -0x3e,0x00,0x00,0x00,0xd2,0x02,0x00,0x00,0x35,0x03,0x00,0x00, -0xdc,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00, -0xdf,0x02,0x00,0x00,0x3b,0x03,0x00,0x00,0x60,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0xdb,0x02,0x00,0x00,0xdc,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xdf,0x02,0x00,0x00, -0xda,0x02,0x00,0x00,0xdb,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0xda,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xe3,0x02,0x00,0x00,0x3b,0x03,0x00,0x00,0x61,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xe4,0x02,0x00,0x00, -0xb9,0x02,0x00,0x00,0xe3,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xe6,0x02,0x00,0x00,0x64,0x00,0x00,0x00, +0x4c,0x02,0x00,0x00,0x4b,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x4d,0x02,0x00,0x00,0x49,0x02,0x00,0x00, +0x4c,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x4e,0x02,0x00,0x00,0x45,0x02,0x00,0x00,0x4d,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0x50,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x50,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xbb,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0xd6,0x00,0x00,0x00, +0xb6,0x02,0x00,0x00,0x53,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc2,0x00,0x00,0x00,0x56,0x02,0x00,0x00,0xbb,0x02,0x00,0x00, +0xbf,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x52,0x02,0x00,0x00, +0x53,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x56,0x02,0x00,0x00,0x51,0x02,0x00,0x00,0x52,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x51,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0x58,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x58,0x02,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xbc,0x02,0x00,0x00, +0x3e,0x00,0x00,0x00,0x51,0x02,0x00,0x00,0xb4,0x02,0x00,0x00, +0x5b,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, +0x5e,0x02,0x00,0x00,0xbc,0x02,0x00,0x00,0x60,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0x5a,0x02,0x00,0x00,0x5b,0x02,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x5e,0x02,0x00,0x00, +0x59,0x02,0x00,0x00,0x5a,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x59,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x62,0x02,0x00,0x00,0xbc,0x02,0x00,0x00,0x61,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x63,0x02,0x00,0x00, +0x38,0x02,0x00,0x00,0x62,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x65,0x02,0x00,0x00,0x64,0x00,0x00,0x00, 0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xe7,0x02,0x00,0x00,0xe4,0x02,0x00,0x00,0xe6,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xeb,0x02,0x00,0x00, -0x3a,0x03,0x00,0x00,0x57,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xec,0x02,0x00,0x00,0xbf,0x02,0x00,0x00, -0xeb,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xee,0x02,0x00,0x00,0x68,0x00,0x00,0x00,0xb2,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xef,0x02,0x00,0x00, -0xec,0x02,0x00,0x00,0xee,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0xf1,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0xf1,0x02,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x3d,0x03,0x00,0x00, -0x3e,0x00,0x00,0x00,0xda,0x02,0x00,0x00,0x33,0x03,0x00,0x00, -0xf4,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00, -0xf7,0x02,0x00,0x00,0x3d,0x03,0x00,0x00,0xb2,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0xf3,0x02,0x00,0x00,0xf4,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xf7,0x02,0x00,0x00, -0xf2,0x02,0x00,0x00,0xf3,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0xf2,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0xf9,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0xf9,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0x3f,0x03,0x00,0x00,0x3e,0x00,0x00,0x00, -0xf2,0x02,0x00,0x00,0x31,0x03,0x00,0x00,0xfc,0x02,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00,0xff,0x02,0x00,0x00, -0x3f,0x03,0x00,0x00,0x62,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0xfb,0x02,0x00,0x00,0xfc,0x02,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xff,0x02,0x00,0x00,0xfa,0x02,0x00,0x00, -0xfb,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0xfa,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x02,0x03,0x00,0x00, -0xe7,0x02,0x00,0x00,0x3f,0x03,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb8,0x00,0x00,0x00,0x05,0x03,0x00,0x00,0x02,0x03,0x00,0x00, -0x36,0x00,0x00,0x00,0xf7,0x00,0x03,0x00,0x07,0x03,0x00,0x00, -0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x05,0x03,0x00,0x00, -0x06,0x03,0x00,0x00,0x07,0x03,0x00,0x00,0xf8,0x00,0x02,0x00, -0x06,0x03,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x0a,0x03,0x00,0x00,0xef,0x02,0x00,0x00,0x3d,0x03,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x0b,0x03,0x00,0x00, -0x12,0x00,0x00,0x00,0xc6,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x0c,0x03,0x00,0x00,0x0b,0x03,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00,0x0d,0x03,0x00,0x00, -0x0a,0x03,0x00,0x00,0x0c,0x03,0x00,0x00,0xf9,0x00,0x02,0x00, -0x07,0x03,0x00,0x00,0xf8,0x00,0x02,0x00,0x07,0x03,0x00,0x00, -0xf5,0x00,0x07,0x00,0xb8,0x00,0x00,0x00,0x0e,0x03,0x00,0x00, -0x05,0x03,0x00,0x00,0xfa,0x02,0x00,0x00,0x0d,0x03,0x00,0x00, -0x06,0x03,0x00,0x00,0xf7,0x00,0x03,0x00,0x10,0x03,0x00,0x00, -0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x0e,0x03,0x00,0x00, -0x0f,0x03,0x00,0x00,0x10,0x03,0x00,0x00,0xf8,0x00,0x02,0x00, -0x0f,0x03,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x18,0x03,0x00,0x00,0xef,0x02,0x00,0x00,0x3d,0x03,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x1a,0x03,0x00,0x00, -0x12,0x00,0x00,0x00,0x19,0x03,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x1b,0x03,0x00,0x00,0x1a,0x03,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x1c,0x03,0x00,0x00, -0x18,0x03,0x00,0x00,0x1b,0x03,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x1d,0x03,0x00,0x00,0xcf,0x02,0x00,0x00, -0x1c,0x03,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x1f,0x03,0x00,0x00,0x1d,0x03,0x00,0x00,0xe7,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x21,0x03,0x00,0x00, -0x1f,0x03,0x00,0x00,0x3f,0x03,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x23,0x03,0x00,0x00,0x3a,0x03,0x00,0x00, -0xb2,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x25,0x03,0x00,0x00,0x23,0x03,0x00,0x00,0x3d,0x03,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x27,0x03,0x00,0x00, -0x25,0x03,0x00,0x00,0x26,0x03,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x29,0x03,0x00,0x00,0x3b,0x03,0x00,0x00, +0x66,0x02,0x00,0x00,0x63,0x02,0x00,0x00,0x65,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x6a,0x02,0x00,0x00, +0xbb,0x02,0x00,0x00,0xd8,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x6b,0x02,0x00,0x00,0x3e,0x02,0x00,0x00, +0x6a,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x6d,0x02,0x00,0x00,0x68,0x00,0x00,0x00,0xbc,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x6e,0x02,0x00,0x00, +0x6b,0x02,0x00,0x00,0x6d,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0x70,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x70,0x02,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xbe,0x02,0x00,0x00, +0x3e,0x00,0x00,0x00,0x59,0x02,0x00,0x00,0xb2,0x02,0x00,0x00, +0x73,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, +0x76,0x02,0x00,0x00,0xbe,0x02,0x00,0x00,0xbc,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0x72,0x02,0x00,0x00,0x73,0x02,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x76,0x02,0x00,0x00, +0x71,0x02,0x00,0x00,0x72,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x71,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x78,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x78,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xc0,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, +0x71,0x02,0x00,0x00,0xb0,0x02,0x00,0x00,0x7b,0x02,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0x7e,0x02,0x00,0x00, +0xc0,0x02,0x00,0x00,0x62,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0x7a,0x02,0x00,0x00,0x7b,0x02,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x7e,0x02,0x00,0x00,0x79,0x02,0x00,0x00, +0x7a,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x79,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x81,0x02,0x00,0x00, +0x66,0x02,0x00,0x00,0xc0,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc2,0x00,0x00,0x00,0x84,0x02,0x00,0x00,0x81,0x02,0x00,0x00, +0x36,0x00,0x00,0x00,0xf7,0x00,0x03,0x00,0x86,0x02,0x00,0x00, +0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x84,0x02,0x00,0x00, +0x85,0x02,0x00,0x00,0x86,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x85,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x89,0x02,0x00,0x00,0x6e,0x02,0x00,0x00,0xbe,0x02,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x8a,0x02,0x00,0x00, +0x12,0x00,0x00,0x00,0xd0,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x8b,0x02,0x00,0x00,0x8a,0x02,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0x8c,0x02,0x00,0x00, +0x89,0x02,0x00,0x00,0x8b,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0x86,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x86,0x02,0x00,0x00, +0xf5,0x00,0x07,0x00,0xc2,0x00,0x00,0x00,0x8d,0x02,0x00,0x00, +0x84,0x02,0x00,0x00,0x79,0x02,0x00,0x00,0x8c,0x02,0x00,0x00, +0x85,0x02,0x00,0x00,0xf7,0x00,0x03,0x00,0x8f,0x02,0x00,0x00, +0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x8d,0x02,0x00,0x00, +0x8e,0x02,0x00,0x00,0x8f,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x8e,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x97,0x02,0x00,0x00,0x6e,0x02,0x00,0x00,0xbe,0x02,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x99,0x02,0x00,0x00, +0x12,0x00,0x00,0x00,0x98,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x9a,0x02,0x00,0x00,0x99,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9b,0x02,0x00,0x00, +0x97,0x02,0x00,0x00,0x9a,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x9c,0x02,0x00,0x00,0x4e,0x02,0x00,0x00, +0x9b,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x9e,0x02,0x00,0x00,0x9c,0x02,0x00,0x00,0x66,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa0,0x02,0x00,0x00, +0x9e,0x02,0x00,0x00,0xc0,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xa2,0x02,0x00,0x00,0xbb,0x02,0x00,0x00, +0xbc,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xa4,0x02,0x00,0x00,0xa2,0x02,0x00,0x00,0xbe,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa6,0x02,0x00,0x00, +0xa4,0x02,0x00,0x00,0xa5,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xa8,0x02,0x00,0x00,0xbc,0x02,0x00,0x00, 0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x2a,0x03,0x00,0x00,0x27,0x03,0x00,0x00,0x29,0x03,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x2c,0x03,0x00,0x00, -0x2a,0x03,0x00,0x00,0x3f,0x03,0x00,0x00,0x41,0x00,0x05,0x00, -0xc3,0x00,0x00,0x00,0x2d,0x03,0x00,0x00,0xc0,0x00,0x00,0x00, -0x2c,0x03,0x00,0x00,0x3d,0x00,0x04,0x00,0xba,0x00,0x00,0x00, -0x2e,0x03,0x00,0x00,0x2d,0x03,0x00,0x00,0x41,0x00,0x06,0x00, -0xfc,0x00,0x00,0x00,0x2f,0x03,0x00,0x00,0x14,0x03,0x00,0x00, -0x34,0x00,0x00,0x00,0x21,0x03,0x00,0x00,0x3e,0x00,0x03,0x00, -0x2f,0x03,0x00,0x00,0x2e,0x03,0x00,0x00,0xf9,0x00,0x02,0x00, -0x10,0x03,0x00,0x00,0xf8,0x00,0x02,0x00,0x10,0x03,0x00,0x00, -0xf9,0x00,0x02,0x00,0xfc,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0xfc,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x31,0x03,0x00,0x00,0x3f,0x03,0x00,0x00,0xc6,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xf9,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0xfb,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0xf4,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0xf4,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x33,0x03,0x00,0x00,0x3d,0x03,0x00,0x00, -0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xf1,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0xf3,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0xdc,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0xdc,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x35,0x03,0x00,0x00, -0x3b,0x03,0x00,0x00,0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xd9,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0xdb,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0xd4,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0xd4,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x37,0x03,0x00,0x00,0x3a,0x03,0x00,0x00,0xc6,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xd1,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0xd3,0x02,0x00,0x00,0xfd,0x00,0x01,0x00,0x38,0x00,0x01,0x00, +0xa9,0x02,0x00,0x00,0xa6,0x02,0x00,0x00,0xa8,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xab,0x02,0x00,0x00, +0xa9,0x02,0x00,0x00,0xc0,0x02,0x00,0x00,0x41,0x00,0x05,0x00, +0xcd,0x00,0x00,0x00,0xac,0x02,0x00,0x00,0xca,0x00,0x00,0x00, +0xab,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00, +0xad,0x02,0x00,0x00,0xac,0x02,0x00,0x00,0x41,0x00,0x06,0x00, +0x65,0x01,0x00,0x00,0xae,0x02,0x00,0x00,0x93,0x02,0x00,0x00, +0x34,0x00,0x00,0x00,0xa0,0x02,0x00,0x00,0x3e,0x00,0x03,0x00, +0xae,0x02,0x00,0x00,0xad,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0x8f,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x8f,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0x7b,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x7b,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xb0,0x02,0x00,0x00,0xc0,0x02,0x00,0x00,0xd0,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x78,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x7a,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x73,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x73,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xb2,0x02,0x00,0x00,0xbe,0x02,0x00,0x00, +0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x70,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x72,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0x5b,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x5b,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xb4,0x02,0x00,0x00, +0xbc,0x02,0x00,0x00,0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x58,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x5a,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0x53,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x53,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xb6,0x02,0x00,0x00,0xbb,0x02,0x00,0x00,0xd0,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x50,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x52,0x02,0x00,0x00,0xfd,0x00,0x01,0x00,0x38,0x00,0x01,0x00, }; -const uint64_t matmul_f32_aligned_m_len = 12168; +const uint64_t matmul_q8_0_f32_aligned_fp32_len = 10452; -unsigned char matmul_f32_aligned_m_fp32_data[] = { +unsigned char matmul_q8_0_f32_fp32_data[] = { 0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, -0xe9,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, -0x01,0x00,0x00,0x00,0x0b,0x00,0x06,0x00,0x01,0x00,0x00,0x00, -0x47,0x4c,0x53,0x4c,0x2e,0x73,0x74,0x64,0x2e,0x34,0x35,0x30, -0x00,0x00,0x00,0x00,0x0e,0x00,0x03,0x00,0x00,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x0f,0x00,0x0f,0x00,0x05,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x6d,0x61,0x69,0x6e,0x00,0x00,0x00,0x00, -0x0b,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x3d,0x00,0x00,0x00, -0x4c,0x00,0x00,0x00,0xea,0x00,0x00,0x00,0xf8,0x00,0x00,0x00, -0x46,0x01,0x00,0x00,0x53,0x01,0x00,0x00,0x4a,0x02,0x00,0x00, -0x93,0x02,0x00,0x00,0x10,0x00,0x06,0x00,0x04,0x00,0x00,0x00, -0x11,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x0b,0x00,0x00,0x00, -0x0b,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x10,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x10,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x10,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x05,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x10,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x07,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x1c,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x08,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x10,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x24,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x0a,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x28,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x2c,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x10,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x30,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x0d,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x34,0x00,0x00,0x00, -0x47,0x00,0x03,0x00,0x10,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x37,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x3d,0x00,0x00,0x00, -0x0b,0x00,0x00,0x00,0x1a,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x4c,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x1b,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x4f,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x53,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x60,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x62,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x07,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x6c,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x9d,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0xaf,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x05,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xb2,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0xf5,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x10,0x00,0x00,0x00, -0x48,0x00,0x04,0x00,0xf6,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0xf6,0x00,0x00,0x00, +0xeb,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, +0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x51,0x11,0x00,0x00, +0x11,0x00,0x02,0x00,0x60,0x11,0x00,0x00,0x0b,0x00,0x06,0x00, +0x01,0x00,0x00,0x00,0x47,0x4c,0x53,0x4c,0x2e,0x73,0x74,0x64, +0x2e,0x34,0x35,0x30,0x00,0x00,0x00,0x00,0x0e,0x00,0x03,0x00, +0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x0f,0x00,0x0f,0x00, +0x05,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x6d,0x61,0x69,0x6e, +0x00,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0x3d,0x00,0x00,0x00,0x4c,0x00,0x00,0x00,0x0a,0x01,0x00,0x00, +0x28,0x01,0x00,0x00,0x59,0x01,0x00,0x00,0x64,0x01,0x00,0x00, +0x4a,0x02,0x00,0x00,0x93,0x02,0x00,0x00,0x10,0x00,0x06,0x00, +0x04,0x00,0x00,0x00,0x11,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x0b,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x1c,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x10,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x08,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x03,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x10,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x10,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, +0x05,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x14,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x10,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x1c,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, +0x08,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x09,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x24,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x10,0x00,0x00,0x00,0x0a,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x28,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, +0x0b,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x2c,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x30,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x10,0x00,0x00,0x00,0x0d,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x34,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x10,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x37,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x3d,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x1a,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x4c,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, +0x1b,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x4f,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x53,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x60,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x62,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x6c,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x03,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0xa6,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xb8,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0xbb,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x08,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x05,0x01,0x00,0x00,0x06,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x06,0x01,0x00,0x00, 0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x03,0x00,0xf6,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0xf8,0x00,0x00,0x00,0x22,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xf8,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x06,0x01,0x00,0x00,0x01,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x07,0x01,0x00,0x00,0x06,0x00,0x00,0x00,0x22,0x00,0x00,0x00, +0x48,0x00,0x04,0x00,0x08,0x01,0x00,0x00,0x00,0x00,0x00,0x00, +0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x08,0x01,0x00,0x00, +0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x03,0x00,0x08,0x01,0x00,0x00,0x02,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x0a,0x01,0x00,0x00,0x22,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x0a,0x01,0x00,0x00, 0x21,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x27,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x28,0x01,0x00,0x00,0x0b,0x00,0x00,0x00, -0x19,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x50,0x01,0x00,0x00, -0x06,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x48,0x00,0x04,0x00, -0x51,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x51,0x01,0x00,0x00,0x00,0x00,0x00,0x00, +0x33,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x34,0x01,0x00,0x00,0x0b,0x00,0x00,0x00, +0x19,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x61,0x01,0x00,0x00, +0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00, +0x62,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x62,0x01,0x00,0x00,0x00,0x00,0x00,0x00, 0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, -0x51,0x01,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x53,0x01,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x53,0x01,0x00,0x00,0x21,0x00,0x00,0x00, +0x62,0x01,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x64,0x01,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x64,0x01,0x00,0x00,0x21,0x00,0x00,0x00, 0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x4a,0x02,0x00,0x00, 0x0b,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x47,0x00,0x04,0x00, 0x90,0x02,0x00,0x00,0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00, @@ -35199,2022 +37337,145 @@ unsigned char matmul_f32_aligned_m_fp32_data[] = { 0x06,0x00,0x00,0x00,0x67,0x00,0x00,0x00,0x86,0x00,0x00,0x00, 0x61,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x32,0x00,0x04,0x00, 0x06,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x10,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x6e,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x6d,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x73,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x6d,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, -0x77,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0x7c,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x87,0x00,0x00,0x00, -0x0b,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, -0x8d,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0x98,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x9d,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, -0x9f,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xae,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x32,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xaf,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xb0,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0xaf,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xb1,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x62,0x00,0x00,0x00, -0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xb2,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xb3,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0xb1,0x00,0x00,0x00, -0xb2,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xb4,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0xb3,0x00,0x00,0x00, -0x60,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xb5,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0xb0,0x00,0x00,0x00, -0xb4,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xb6,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0xae,0x00,0x00,0x00, -0xb5,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xb7,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0xb6,0x00,0x00,0x00, -0xb2,0x00,0x00,0x00,0x14,0x00,0x02,0x00,0xb8,0x00,0x00,0x00, -0x16,0x00,0x03,0x00,0xba,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xbb,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xbc,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0xbb,0x00,0x00,0x00,0xb5,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xbd,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0xbc,0x00,0x00,0x00,0xb2,0x00,0x00,0x00, -0x1c,0x00,0x04,0x00,0xbe,0x00,0x00,0x00,0xba,0x00,0x00,0x00, -0xbd,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0xbf,0x00,0x00,0x00, -0x07,0x00,0x00,0x00,0xbe,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0xba,0x00,0x00,0x00,0xc2,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0xc3,0x00,0x00,0x00,0x07,0x00,0x00,0x00, -0xba,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, -0xc6,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xe6,0x00,0x00,0x00,0x80,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xe7,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0x37,0x00,0x00,0x00,0xe6,0x00,0x00,0x00,0x1c,0x00,0x04,0x00, -0xe8,0x00,0x00,0x00,0xba,0x00,0x00,0x00,0xe7,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0xe9,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0xe8,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0xe9,0x00,0x00,0x00, -0xea,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xee,0x00,0x00,0x00,0x80,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x17,0x00,0x04,0x00, -0xf4,0x00,0x00,0x00,0xba,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x1d,0x00,0x03,0x00,0xf5,0x00,0x00,0x00,0xf4,0x00,0x00,0x00, -0x1e,0x00,0x03,0x00,0xf6,0x00,0x00,0x00,0xf5,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0xf7,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0xf6,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0xf7,0x00,0x00,0x00, -0xf8,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0xfa,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0xba,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0xfd,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0xba,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x02,0x01,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x0f,0x01,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x1c,0x01,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x21,0x01,0x00,0x00,0x03,0x00,0x00,0x00,0x32,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x27,0x01,0x00,0x00,0x01,0x00,0x00,0x00, -0x33,0x00,0x06,0x00,0x09,0x00,0x00,0x00,0x28,0x01,0x00,0x00, -0x27,0x01,0x00,0x00,0x39,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x29,0x01,0x00,0x00, -0x51,0x00,0x00,0x00,0x28,0x01,0x00,0x00,0x00,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x2a,0x01,0x00,0x00, -0x84,0x00,0x00,0x00,0x29,0x01,0x00,0x00,0x6d,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x2b,0x01,0x00,0x00, -0x86,0x00,0x00,0x00,0x2a,0x01,0x00,0x00,0x6c,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x42,0x01,0x00,0x00, -0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x43,0x01,0x00,0x00, -0x84,0x00,0x00,0x00,0x9d,0x00,0x00,0x00,0x42,0x01,0x00,0x00, -0x1c,0x00,0x04,0x00,0x44,0x01,0x00,0x00,0xba,0x00,0x00,0x00, -0x43,0x01,0x00,0x00,0x20,0x00,0x04,0x00,0x45,0x01,0x00,0x00, -0x04,0x00,0x00,0x00,0x44,0x01,0x00,0x00,0x3b,0x00,0x04,0x00, -0x45,0x01,0x00,0x00,0x46,0x01,0x00,0x00,0x04,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x4a,0x01,0x00,0x00, -0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x1d,0x00,0x03,0x00,0x50,0x01,0x00,0x00,0xf4,0x00,0x00,0x00, -0x1e,0x00,0x03,0x00,0x51,0x01,0x00,0x00,0x50,0x01,0x00,0x00, -0x20,0x00,0x04,0x00,0x52,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, -0x51,0x01,0x00,0x00,0x3b,0x00,0x04,0x00,0x52,0x01,0x00,0x00, -0x53,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x5b,0x01,0x00,0x00,0x80,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x68,0x01,0x00,0x00,0x80,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x75,0x01,0x00,0x00,0x80,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x81,0x01,0x00,0x00,0x08,0x01,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x82,0x01,0x00,0x00, -0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x85,0x01,0x00,0x00, -0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xa0,0x01,0x00,0x00, -0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00, -0x1c,0x00,0x04,0x00,0xa1,0x01,0x00,0x00,0xba,0x00,0x00,0x00, -0xa0,0x01,0x00,0x00,0x20,0x00,0x04,0x00,0xa2,0x01,0x00,0x00, -0x07,0x00,0x00,0x00,0xa1,0x01,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xb2,0x01,0x00,0x00,0x80,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xcd,0x01,0x00,0x00,0x84,0x00,0x00,0x00, -0xb5,0x00,0x00,0x00,0xb2,0x00,0x00,0x00,0x1c,0x00,0x04,0x00, -0xce,0x01,0x00,0x00,0xba,0x00,0x00,0x00,0xcd,0x01,0x00,0x00, -0x20,0x00,0x04,0x00,0xcf,0x01,0x00,0x00,0x07,0x00,0x00,0x00, -0xce,0x01,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xd8,0x01,0x00,0x00,0x86,0x00,0x00,0x00,0xaf,0x00,0x00,0x00, -0xb5,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xe0,0x01,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x0f,0x02,0x00,0x00,0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, -0x42,0x02,0x00,0x00,0x0d,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x0a,0x00,0x00,0x00,0x4a,0x02,0x00,0x00,0x01,0x00,0x00,0x00, -0x1d,0x00,0x03,0x00,0x90,0x02,0x00,0x00,0xba,0x00,0x00,0x00, -0x1e,0x00,0x03,0x00,0x91,0x02,0x00,0x00,0x90,0x02,0x00,0x00, -0x20,0x00,0x04,0x00,0x92,0x02,0x00,0x00,0x0c,0x00,0x00,0x00, -0x91,0x02,0x00,0x00,0x3b,0x00,0x04,0x00,0x92,0x02,0x00,0x00, -0x93,0x02,0x00,0x00,0x0c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0x98,0x02,0x00,0x00,0x05,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xa5,0x02,0x00,0x00, -0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00, -0x36,0x00,0x05,0x00,0x02,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x05,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0xbf,0x00,0x00,0x00, -0xc0,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0xa2,0x01,0x00,0x00,0xa3,0x01,0x00,0x00,0x07,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0xcf,0x01,0x00,0x00,0xd0,0x01,0x00,0x00, -0x07,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00, -0x0e,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, -0x0e,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x14,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x17,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x17,0x00,0x00,0x00, -0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x1e,0x00,0x00,0x00, -0x0f,0x00,0x00,0x00,0x17,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x21,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x24,0x00,0x00,0x00,0x18,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0x28,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x27,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x29,0x00,0x00,0x00, -0x28,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x2a,0x00,0x00,0x00,0x1e,0x00,0x00,0x00,0x29,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x2d,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x2f,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x30,0x00,0x00,0x00, -0x24,0x00,0x00,0x00,0x2f,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x30,0x00,0x00,0x00, -0x2a,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0x35,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x34,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x36,0x00,0x00,0x00, -0x35,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x38,0x00,0x00,0x00,0x36,0x00,0x00,0x00,0x37,0x00,0x00,0x00, -0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, -0x38,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x3b,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, -0x37,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00, -0x3f,0x00,0x00,0x00,0x3d,0x00,0x00,0x00,0x3e,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x3f,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x42,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x3b,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x47,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0x3b,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x0d,0x00,0x00,0x00,0x49,0x00,0x00,0x00,0x3d,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x4a,0x00,0x00,0x00,0x49,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x0d,0x00,0x00,0x00,0x4d,0x00,0x00,0x00,0x4c,0x00,0x00,0x00, -0x3e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x4e,0x00,0x00,0x00,0x4d,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x50,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, -0x4f,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x55,0x00,0x00,0x00,0x50,0x00,0x00,0x00,0x54,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x59,0x00,0x00,0x00, -0x50,0x00,0x00,0x00,0x58,0x00,0x00,0x00,0x89,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x5d,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, -0x4f,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x64,0x00,0x00,0x00,0x5d,0x00,0x00,0x00,0x63,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x68,0x00,0x00,0x00, -0x5d,0x00,0x00,0x00,0x67,0x00,0x00,0x00,0x89,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x6f,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, -0x6e,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x74,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x73,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x78,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x77,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x79,0x00,0x00,0x00,0x78,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x7a,0x00,0x00,0x00, -0x47,0x00,0x00,0x00,0x79,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x7d,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x7c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x7e,0x00,0x00,0x00,0x7d,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x80,0x00,0x00,0x00,0x47,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x83,0x00,0x00,0x00,0x80,0x00,0x00,0x00,0x79,0x00,0x00,0x00, -0x0c,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x26,0x00,0x00,0x00,0x7e,0x00,0x00,0x00, -0x83,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0x88,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x87,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x89,0x00,0x00,0x00, -0x88,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x8a,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x89,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8c,0x00,0x00,0x00, -0x42,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x8e,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x8d,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x8f,0x00,0x00,0x00,0x8e,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x90,0x00,0x00,0x00,0x8c,0x00,0x00,0x00, -0x8f,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x91,0x00,0x00,0x00,0x8a,0x00,0x00,0x00,0x90,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x93,0x00,0x00,0x00, -0x91,0x00,0x00,0x00,0x7a,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x94,0x00,0x00,0x00,0x93,0x00,0x00,0x00, -0x6d,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0x99,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x98,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x9a,0x00,0x00,0x00, -0x99,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x9b,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x9a,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9e,0x00,0x00,0x00, -0x4a,0x00,0x00,0x00,0x9d,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0xa0,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x9f,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xa1,0x00,0x00,0x00,0xa0,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xa2,0x00,0x00,0x00,0x9e,0x00,0x00,0x00, -0xa1,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xa3,0x00,0x00,0x00,0x9b,0x00,0x00,0x00,0xa2,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa5,0x00,0x00,0x00, -0xa3,0x00,0x00,0x00,0x7a,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xa6,0x00,0x00,0x00,0xa5,0x00,0x00,0x00, -0x6d,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xa8,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xa8,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xb7,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, -0x05,0x00,0x00,0x00,0xc7,0x00,0x00,0x00,0xa9,0x00,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00,0xb9,0x00,0x00,0x00, -0xb7,0x02,0x00,0x00,0xb7,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0xaa,0x00,0x00,0x00,0xa9,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xb9,0x00,0x00,0x00,0xa9,0x00,0x00,0x00, -0xaa,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xa9,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0xc3,0x00,0x00,0x00,0xc4,0x00,0x00,0x00, -0xc0,0x00,0x00,0x00,0xb7,0x02,0x00,0x00,0x3e,0x00,0x03,0x00, -0xc4,0x00,0x00,0x00,0xc2,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xc7,0x00,0x00,0x00,0xb7,0x02,0x00,0x00, -0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xa8,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xaa,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xca,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xca,0x00,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xd0,0x02,0x00,0x00, -0xa6,0x00,0x00,0x00,0xaa,0x00,0x00,0x00,0x87,0x01,0x00,0x00, -0xcd,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xcc,0x02,0x00,0x00,0x94,0x00,0x00,0x00,0xaa,0x00,0x00,0x00, -0x84,0x01,0x00,0x00,0xcd,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xb8,0x02,0x00,0x00,0x7a,0x00,0x00,0x00, -0xaa,0x00,0x00,0x00,0x32,0x02,0x00,0x00,0xcd,0x00,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00,0xd1,0x00,0x00,0x00, -0xb8,0x02,0x00,0x00,0x84,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0xcc,0x00,0x00,0x00,0xcd,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xd1,0x00,0x00,0x00,0xcb,0x00,0x00,0x00, -0xcc,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xcb,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xd3,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xd3,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xc8,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0xcb,0x00,0x00,0x00, -0x2d,0x01,0x00,0x00,0xd4,0x00,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb8,0x00,0x00,0x00,0xd9,0x00,0x00,0x00,0xc8,0x02,0x00,0x00, -0x37,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xd5,0x00,0x00,0x00, -0xd4,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0xd9,0x00,0x00,0x00,0xd4,0x00,0x00,0x00,0xd5,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xd4,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xde,0x00,0x00,0x00,0x74,0x00,0x00,0x00, -0xc8,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xe1,0x00,0x00,0x00,0xde,0x00,0x00,0x00,0x8f,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xe2,0x00,0x00,0x00, -0xe1,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xe3,0x00,0x00,0x00,0xcc,0x02,0x00,0x00, -0xe2,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xe5,0x00,0x00,0x00,0xe3,0x00,0x00,0x00,0x6f,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xef,0x00,0x00,0x00, -0xde,0x00,0x00,0x00,0xee,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xf1,0x00,0x00,0x00,0x6f,0x00,0x00,0x00, -0x6d,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xf2,0x00,0x00,0x00,0xef,0x00,0x00,0x00,0xf1,0x00,0x00,0x00, -0x41,0x00,0x07,0x00,0xfa,0x00,0x00,0x00,0xfb,0x00,0x00,0x00, -0xf8,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0xe5,0x00,0x00,0x00, -0x3e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0xba,0x00,0x00,0x00, -0xfc,0x00,0x00,0x00,0xfb,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0xfd,0x00,0x00,0x00,0xfe,0x00,0x00,0x00,0xea,0x00,0x00,0x00, -0xf2,0x00,0x00,0x00,0x3e,0x00,0x03,0x00,0xfe,0x00,0x00,0x00, -0xfc,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x03,0x01,0x00,0x00,0xde,0x00,0x00,0x00,0x02,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x06,0x01,0x00,0x00, -0x03,0x01,0x00,0x00,0xf1,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x07,0x01,0x00,0x00,0x06,0x01,0x00,0x00, -0x39,0x00,0x00,0x00,0x41,0x00,0x07,0x00,0xfa,0x00,0x00,0x00, -0x09,0x01,0x00,0x00,0xf8,0x00,0x00,0x00,0x34,0x00,0x00,0x00, -0xe5,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0xba,0x00,0x00,0x00,0x0a,0x01,0x00,0x00,0x09,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0xfd,0x00,0x00,0x00,0x0b,0x01,0x00,0x00, -0xea,0x00,0x00,0x00,0x07,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0x0b,0x01,0x00,0x00,0x0a,0x01,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x10,0x01,0x00,0x00,0xde,0x00,0x00,0x00, -0x0f,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x13,0x01,0x00,0x00,0x10,0x01,0x00,0x00,0xf1,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x14,0x01,0x00,0x00, -0x13,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x41,0x00,0x07,0x00, -0xfa,0x00,0x00,0x00,0x16,0x01,0x00,0x00,0xf8,0x00,0x00,0x00, -0x34,0x00,0x00,0x00,0xe5,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0xba,0x00,0x00,0x00,0x17,0x01,0x00,0x00, -0x16,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0xfd,0x00,0x00,0x00, -0x18,0x01,0x00,0x00,0xea,0x00,0x00,0x00,0x14,0x01,0x00,0x00, -0x3e,0x00,0x03,0x00,0x18,0x01,0x00,0x00,0x17,0x01,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x1d,0x01,0x00,0x00, -0xde,0x00,0x00,0x00,0x1c,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x20,0x01,0x00,0x00,0x1d,0x01,0x00,0x00, -0xf1,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x22,0x01,0x00,0x00,0x20,0x01,0x00,0x00,0x21,0x01,0x00,0x00, -0x41,0x00,0x07,0x00,0xfa,0x00,0x00,0x00,0x24,0x01,0x00,0x00, -0xf8,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0xe5,0x00,0x00,0x00, -0x21,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0xba,0x00,0x00,0x00, -0x25,0x01,0x00,0x00,0x24,0x01,0x00,0x00,0x41,0x00,0x05,0x00, -0xfd,0x00,0x00,0x00,0x26,0x01,0x00,0x00,0xea,0x00,0x00,0x00, -0x22,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x26,0x01,0x00,0x00, -0x25,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x2d,0x01,0x00,0x00,0xc8,0x02,0x00,0x00,0x2b,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0xd3,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xd5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x2f,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x2f,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xc9,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, -0xd5,0x00,0x00,0x00,0x80,0x01,0x00,0x00,0x30,0x01,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00,0x35,0x01,0x00,0x00, -0xc9,0x02,0x00,0x00,0x9d,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0x31,0x01,0x00,0x00,0x30,0x01,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x35,0x01,0x00,0x00,0x30,0x01,0x00,0x00, -0x31,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x30,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3a,0x01,0x00,0x00, -0x74,0x00,0x00,0x00,0xc9,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x3d,0x01,0x00,0x00,0x3a,0x01,0x00,0x00, -0xa1,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x3e,0x01,0x00,0x00,0x3d,0x01,0x00,0x00,0x6d,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3f,0x01,0x00,0x00, -0xd0,0x02,0x00,0x00,0x3e,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x41,0x01,0x00,0x00,0x3f,0x01,0x00,0x00, -0x6f,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x4b,0x01,0x00,0x00,0x3a,0x01,0x00,0x00,0x4a,0x01,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x4d,0x01,0x00,0x00, -0x6f,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x4e,0x01,0x00,0x00,0x4b,0x01,0x00,0x00, -0x4d,0x01,0x00,0x00,0x41,0x00,0x07,0x00,0xfa,0x00,0x00,0x00, -0x55,0x01,0x00,0x00,0x53,0x01,0x00,0x00,0x34,0x00,0x00,0x00, -0x41,0x01,0x00,0x00,0x3e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0xba,0x00,0x00,0x00,0x56,0x01,0x00,0x00,0x55,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0xfd,0x00,0x00,0x00,0x57,0x01,0x00,0x00, -0x46,0x01,0x00,0x00,0x4e,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0x57,0x01,0x00,0x00,0x56,0x01,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x5c,0x01,0x00,0x00,0x3a,0x01,0x00,0x00, -0x5b,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x5f,0x01,0x00,0x00,0x5c,0x01,0x00,0x00,0x4d,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x60,0x01,0x00,0x00, -0x5f,0x01,0x00,0x00,0x39,0x00,0x00,0x00,0x41,0x00,0x07,0x00, -0xfa,0x00,0x00,0x00,0x62,0x01,0x00,0x00,0x53,0x01,0x00,0x00, -0x34,0x00,0x00,0x00,0x41,0x01,0x00,0x00,0x39,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0xba,0x00,0x00,0x00,0x63,0x01,0x00,0x00, -0x62,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0xfd,0x00,0x00,0x00, -0x64,0x01,0x00,0x00,0x46,0x01,0x00,0x00,0x60,0x01,0x00,0x00, -0x3e,0x00,0x03,0x00,0x64,0x01,0x00,0x00,0x63,0x01,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x69,0x01,0x00,0x00, -0x3a,0x01,0x00,0x00,0x68,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x6c,0x01,0x00,0x00,0x69,0x01,0x00,0x00, -0x4d,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x6d,0x01,0x00,0x00,0x6c,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, -0x41,0x00,0x07,0x00,0xfa,0x00,0x00,0x00,0x6f,0x01,0x00,0x00, -0x53,0x01,0x00,0x00,0x34,0x00,0x00,0x00,0x41,0x01,0x00,0x00, -0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0xba,0x00,0x00,0x00, -0x70,0x01,0x00,0x00,0x6f,0x01,0x00,0x00,0x41,0x00,0x05,0x00, -0xfd,0x00,0x00,0x00,0x71,0x01,0x00,0x00,0x46,0x01,0x00,0x00, -0x6d,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x71,0x01,0x00,0x00, -0x70,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x76,0x01,0x00,0x00,0x3a,0x01,0x00,0x00,0x75,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x79,0x01,0x00,0x00, -0x76,0x01,0x00,0x00,0x4d,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x7a,0x01,0x00,0x00,0x79,0x01,0x00,0x00, -0x21,0x01,0x00,0x00,0x41,0x00,0x07,0x00,0xfa,0x00,0x00,0x00, -0x7c,0x01,0x00,0x00,0x53,0x01,0x00,0x00,0x34,0x00,0x00,0x00, -0x41,0x01,0x00,0x00,0x21,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, -0xba,0x00,0x00,0x00,0x7d,0x01,0x00,0x00,0x7c,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0xfd,0x00,0x00,0x00,0x7e,0x01,0x00,0x00, -0x46,0x01,0x00,0x00,0x7a,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0x7e,0x01,0x00,0x00,0x7d,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x80,0x01,0x00,0x00,0xc9,0x02,0x00,0x00, -0x2b,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x2f,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x31,0x01,0x00,0x00,0xe0,0x00,0x04,0x00, -0x0c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x81,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x84,0x01,0x00,0x00, -0xcc,0x02,0x00,0x00,0x82,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x87,0x01,0x00,0x00,0xd0,0x02,0x00,0x00, -0x85,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x89,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x89,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xd2,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, -0x31,0x01,0x00,0x00,0x30,0x02,0x00,0x00,0x8c,0x01,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00,0x8f,0x01,0x00,0x00, -0xd2,0x02,0x00,0x00,0x6c,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0x8b,0x01,0x00,0x00,0x8c,0x01,0x00,0x00,0x00,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x8f,0x01,0x00,0x00,0x8a,0x01,0x00,0x00, -0x8b,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x8a,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0x91,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x91,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xd6,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0x8a,0x01,0x00,0x00, -0xbc,0x01,0x00,0x00,0x94,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb8,0x00,0x00,0x00,0x97,0x01,0x00,0x00,0xd6,0x02,0x00,0x00, -0x60,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x93,0x01,0x00,0x00, -0x94,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x97,0x01,0x00,0x00,0x92,0x01,0x00,0x00,0x93,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x92,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0x99,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x99,0x01,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xe8,0x02,0x00,0x00, -0x3e,0x00,0x00,0x00,0x92,0x01,0x00,0x00,0xba,0x01,0x00,0x00, -0x9a,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00, -0x9f,0x01,0x00,0x00,0xe8,0x02,0x00,0x00,0x62,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0x9b,0x01,0x00,0x00,0x9a,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x9f,0x01,0x00,0x00, -0x9a,0x01,0x00,0x00,0x9b,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x9a,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xa5,0x01,0x00,0x00,0xd6,0x02,0x00,0x00,0x62,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa7,0x01,0x00,0x00, -0xa5,0x01,0x00,0x00,0xe8,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xa9,0x01,0x00,0x00,0x55,0x00,0x00,0x00, -0x53,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xab,0x01,0x00,0x00,0xd6,0x02,0x00,0x00,0x61,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xac,0x01,0x00,0x00, -0xa9,0x01,0x00,0x00,0xab,0x01,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xae,0x01,0x00,0x00,0x64,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xaf,0x01,0x00,0x00,0xac,0x01,0x00,0x00,0xae,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xb1,0x01,0x00,0x00, -0xaf,0x01,0x00,0x00,0xe8,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xb3,0x01,0x00,0x00,0xb1,0x01,0x00,0x00, -0xb2,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xb5,0x01,0x00,0x00,0xb3,0x01,0x00,0x00,0xd2,0x02,0x00,0x00, -0x41,0x00,0x05,0x00,0xfd,0x00,0x00,0x00,0xb6,0x01,0x00,0x00, -0xea,0x00,0x00,0x00,0xb5,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, -0xba,0x00,0x00,0x00,0xb7,0x01,0x00,0x00,0xb6,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0xc3,0x00,0x00,0x00,0xb8,0x01,0x00,0x00, -0xa3,0x01,0x00,0x00,0xa7,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0xb8,0x01,0x00,0x00,0xb7,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xba,0x01,0x00,0x00,0xe8,0x02,0x00,0x00, -0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x99,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x9b,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0x94,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x94,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xbc,0x01,0x00,0x00, -0xd6,0x02,0x00,0x00,0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x91,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x93,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0xbe,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xbe,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xd7,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0x93,0x01,0x00,0x00, -0xea,0x01,0x00,0x00,0xc1,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb8,0x00,0x00,0x00,0xc4,0x01,0x00,0x00,0xd7,0x02,0x00,0x00, -0xb5,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xc0,0x01,0x00,0x00, -0xc1,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0xc4,0x01,0x00,0x00,0xbf,0x01,0x00,0x00,0xc0,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xbf,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0xc6,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xc6,0x01,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xe5,0x02,0x00,0x00, -0x3e,0x00,0x00,0x00,0xbf,0x01,0x00,0x00,0xe8,0x01,0x00,0x00, -0xc7,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00, -0xcc,0x01,0x00,0x00,0xe5,0x02,0x00,0x00,0xb2,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0xc8,0x01,0x00,0x00,0xc7,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xcc,0x01,0x00,0x00, -0xc7,0x01,0x00,0x00,0xc8,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xc7,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xd2,0x01,0x00,0x00,0xd7,0x02,0x00,0x00,0xb2,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xd4,0x01,0x00,0x00, -0xd2,0x01,0x00,0x00,0xe5,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xd6,0x01,0x00,0x00,0x59,0x00,0x00,0x00, -0xaf,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xd9,0x01,0x00,0x00,0xd7,0x02,0x00,0x00,0xd8,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xda,0x01,0x00,0x00, -0xd6,0x01,0x00,0x00,0xd9,0x01,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xdc,0x01,0x00,0x00,0x68,0x00,0x00,0x00, -0xb2,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xdd,0x01,0x00,0x00,0xda,0x01,0x00,0x00,0xdc,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xdf,0x01,0x00,0x00, -0xdd,0x01,0x00,0x00,0xe5,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xe1,0x01,0x00,0x00,0xdf,0x01,0x00,0x00, -0xe0,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xe3,0x01,0x00,0x00,0xe1,0x01,0x00,0x00,0xd2,0x02,0x00,0x00, -0x41,0x00,0x05,0x00,0xfd,0x00,0x00,0x00,0xe4,0x01,0x00,0x00, -0x46,0x01,0x00,0x00,0xe3,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, -0xba,0x00,0x00,0x00,0xe5,0x01,0x00,0x00,0xe4,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0xc3,0x00,0x00,0x00,0xe6,0x01,0x00,0x00, -0xd0,0x01,0x00,0x00,0xd4,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0xe6,0x01,0x00,0x00,0xe5,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xe8,0x01,0x00,0x00,0xe5,0x02,0x00,0x00, -0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xc6,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xc8,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0xc1,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xc1,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xea,0x01,0x00,0x00, -0xd7,0x02,0x00,0x00,0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xbe,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xc0,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0xec,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xec,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xd8,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0xc0,0x01,0x00,0x00, -0x2e,0x02,0x00,0x00,0xef,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb8,0x00,0x00,0x00,0xf2,0x01,0x00,0x00,0xd8,0x02,0x00,0x00, -0xb5,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xee,0x01,0x00,0x00, -0xef,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0xf2,0x01,0x00,0x00,0xed,0x01,0x00,0x00,0xee,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xed,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0xf4,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xf4,0x01,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xdc,0x02,0x00,0x00, -0x3e,0x00,0x00,0x00,0xed,0x01,0x00,0x00,0x2c,0x02,0x00,0x00, -0xf7,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00, -0xfa,0x01,0x00,0x00,0xdc,0x02,0x00,0x00,0x60,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0xf6,0x01,0x00,0x00,0xf7,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xfa,0x01,0x00,0x00, -0xf5,0x01,0x00,0x00,0xf6,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xf5,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xfc,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xfc,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xde,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, -0xf5,0x01,0x00,0x00,0x2a,0x02,0x00,0x00,0xff,0x01,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00,0x02,0x02,0x00,0x00, -0xde,0x02,0x00,0x00,0xb2,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0xfe,0x01,0x00,0x00,0xff,0x01,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x02,0x02,0x00,0x00,0xfd,0x01,0x00,0x00, -0xfe,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xfd,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0x04,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x04,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xe0,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0xfd,0x01,0x00,0x00, -0x28,0x02,0x00,0x00,0x05,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb8,0x00,0x00,0x00,0x0a,0x02,0x00,0x00,0xe0,0x02,0x00,0x00, -0x62,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x06,0x02,0x00,0x00, -0x05,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x0a,0x02,0x00,0x00,0x05,0x02,0x00,0x00,0x06,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x05,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x0c,0x02,0x00,0x00,0xd8,0x02,0x00,0x00, -0xb2,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x0e,0x02,0x00,0x00,0x0c,0x02,0x00,0x00,0xde,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x10,0x02,0x00,0x00, -0x0e,0x02,0x00,0x00,0x0f,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x12,0x02,0x00,0x00,0xdc,0x02,0x00,0x00, -0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x13,0x02,0x00,0x00,0x10,0x02,0x00,0x00,0x12,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x15,0x02,0x00,0x00, -0x13,0x02,0x00,0x00,0xe0,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x19,0x02,0x00,0x00,0x12,0x02,0x00,0x00, -0xe0,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0xc3,0x00,0x00,0x00, -0x1a,0x02,0x00,0x00,0xa3,0x01,0x00,0x00,0x19,0x02,0x00,0x00, -0x3d,0x00,0x04,0x00,0xba,0x00,0x00,0x00,0x1b,0x02,0x00,0x00, -0x1a,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0xc3,0x00,0x00,0x00, -0x20,0x02,0x00,0x00,0xd0,0x01,0x00,0x00,0x0e,0x02,0x00,0x00, -0x3d,0x00,0x04,0x00,0xba,0x00,0x00,0x00,0x21,0x02,0x00,0x00, -0x20,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0xc3,0x00,0x00,0x00, -0x23,0x02,0x00,0x00,0xc0,0x00,0x00,0x00,0x15,0x02,0x00,0x00, -0x3d,0x00,0x04,0x00,0xba,0x00,0x00,0x00,0x24,0x02,0x00,0x00, -0x23,0x02,0x00,0x00,0x0c,0x00,0x08,0x00,0xba,0x00,0x00,0x00, -0x25,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00, -0x1b,0x02,0x00,0x00,0x21,0x02,0x00,0x00,0x24,0x02,0x00,0x00, -0x3e,0x00,0x03,0x00,0x23,0x02,0x00,0x00,0x25,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x28,0x02,0x00,0x00, -0xe0,0x02,0x00,0x00,0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x04,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x06,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0xff,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xff,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x2a,0x02,0x00,0x00,0xde,0x02,0x00,0x00,0xc6,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xfc,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xfe,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xf7,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xf7,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x2c,0x02,0x00,0x00,0xdc,0x02,0x00,0x00, -0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xf4,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xf6,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0xef,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xef,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x2e,0x02,0x00,0x00, -0xd8,0x02,0x00,0x00,0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xec,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xee,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0x8c,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x8c,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x30,0x02,0x00,0x00,0xd2,0x02,0x00,0x00,0xc6,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x89,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x8b,0x01,0x00,0x00,0xe0,0x00,0x04,0x00,0x0c,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x81,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0xcd,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xcd,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x32,0x02,0x00,0x00, -0xb8,0x02,0x00,0x00,0x6c,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xca,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xcc,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x37,0x02,0x00,0x00, -0x55,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x38,0x02,0x00,0x00,0x8c,0x00,0x00,0x00, -0x37,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x3d,0x02,0x00,0x00,0x59,0x00,0x00,0x00,0xaf,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3e,0x02,0x00,0x00, -0x9e,0x00,0x00,0x00,0x3d,0x02,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x43,0x02,0x00,0x00,0x12,0x00,0x00,0x00, -0x42,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x44,0x02,0x00,0x00,0x43,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x45,0x02,0x00,0x00,0x0f,0x00,0x00,0x00, -0x44,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x49,0x02,0x00,0x00,0x47,0x00,0x00,0x00,0x44,0x02,0x00,0x00, -0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00,0x4b,0x02,0x00,0x00, -0x4a,0x02,0x00,0x00,0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x4c,0x02,0x00,0x00,0x4b,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x4d,0x02,0x00,0x00, -0x49,0x02,0x00,0x00,0x4c,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x4e,0x02,0x00,0x00,0x45,0x02,0x00,0x00, -0x4d,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x50,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x50,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xb9,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, -0xcc,0x00,0x00,0x00,0xb6,0x02,0x00,0x00,0x53,0x02,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00,0x56,0x02,0x00,0x00, -0xb9,0x02,0x00,0x00,0xb5,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0x52,0x02,0x00,0x00,0x53,0x02,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x56,0x02,0x00,0x00,0x51,0x02,0x00,0x00, -0x52,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x51,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0x58,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x58,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xba,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0x51,0x02,0x00,0x00, -0xb4,0x02,0x00,0x00,0x5b,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb8,0x00,0x00,0x00,0x5e,0x02,0x00,0x00,0xba,0x02,0x00,0x00, -0x60,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x5a,0x02,0x00,0x00, -0x5b,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x5e,0x02,0x00,0x00,0x59,0x02,0x00,0x00,0x5a,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x59,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x62,0x02,0x00,0x00,0xba,0x02,0x00,0x00, -0x61,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x63,0x02,0x00,0x00,0x38,0x02,0x00,0x00,0x62,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x65,0x02,0x00,0x00, -0x64,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x66,0x02,0x00,0x00,0x63,0x02,0x00,0x00, -0x65,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x6a,0x02,0x00,0x00,0xb9,0x02,0x00,0x00,0xd8,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x6b,0x02,0x00,0x00, -0x3e,0x02,0x00,0x00,0x6a,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x6d,0x02,0x00,0x00,0x68,0x00,0x00,0x00, -0xb2,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x6e,0x02,0x00,0x00,0x6b,0x02,0x00,0x00,0x6d,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0x70,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x70,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xbc,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0x59,0x02,0x00,0x00, -0xb2,0x02,0x00,0x00,0x73,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb8,0x00,0x00,0x00,0x76,0x02,0x00,0x00,0xbc,0x02,0x00,0x00, -0xb2,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x72,0x02,0x00,0x00, -0x73,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x76,0x02,0x00,0x00,0x71,0x02,0x00,0x00,0x72,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x71,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x78,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x78,0x02,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xbe,0x02,0x00,0x00, -0x3e,0x00,0x00,0x00,0x71,0x02,0x00,0x00,0xb0,0x02,0x00,0x00, -0x7b,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00, -0x7e,0x02,0x00,0x00,0xbe,0x02,0x00,0x00,0x62,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0x7a,0x02,0x00,0x00,0x7b,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x7e,0x02,0x00,0x00, -0x79,0x02,0x00,0x00,0x7a,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x79,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x81,0x02,0x00,0x00,0x66,0x02,0x00,0x00,0xbe,0x02,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00,0x84,0x02,0x00,0x00, -0x81,0x02,0x00,0x00,0x36,0x00,0x00,0x00,0xf7,0x00,0x03,0x00, -0x86,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x84,0x02,0x00,0x00,0x85,0x02,0x00,0x00,0x86,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x85,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x89,0x02,0x00,0x00,0x6e,0x02,0x00,0x00, -0xbc,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0x8a,0x02,0x00,0x00,0x12,0x00,0x00,0x00,0xc6,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x8b,0x02,0x00,0x00, -0x8a,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00, -0x8c,0x02,0x00,0x00,0x89,0x02,0x00,0x00,0x8b,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0x86,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x86,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0xb8,0x00,0x00,0x00, -0x8d,0x02,0x00,0x00,0x84,0x02,0x00,0x00,0x79,0x02,0x00,0x00, -0x8c,0x02,0x00,0x00,0x85,0x02,0x00,0x00,0xf7,0x00,0x03,0x00, -0x8f,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x8d,0x02,0x00,0x00,0x8e,0x02,0x00,0x00,0x8f,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x8e,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x97,0x02,0x00,0x00,0x6e,0x02,0x00,0x00, -0xbc,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0x99,0x02,0x00,0x00,0x12,0x00,0x00,0x00,0x98,0x02,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x9a,0x02,0x00,0x00, -0x99,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x9b,0x02,0x00,0x00,0x97,0x02,0x00,0x00,0x9a,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9c,0x02,0x00,0x00, -0x4e,0x02,0x00,0x00,0x9b,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x9e,0x02,0x00,0x00,0x9c,0x02,0x00,0x00, -0x66,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xa0,0x02,0x00,0x00,0x9e,0x02,0x00,0x00,0xbe,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa2,0x02,0x00,0x00, -0xb9,0x02,0x00,0x00,0xb2,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xa4,0x02,0x00,0x00,0xa2,0x02,0x00,0x00, -0xbc,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xa6,0x02,0x00,0x00,0xa4,0x02,0x00,0x00,0xa5,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa8,0x02,0x00,0x00, -0xba,0x02,0x00,0x00,0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xa9,0x02,0x00,0x00,0xa6,0x02,0x00,0x00, -0xa8,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xab,0x02,0x00,0x00,0xa9,0x02,0x00,0x00,0xbe,0x02,0x00,0x00, -0x41,0x00,0x05,0x00,0xc3,0x00,0x00,0x00,0xac,0x02,0x00,0x00, -0xc0,0x00,0x00,0x00,0xab,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, -0xba,0x00,0x00,0x00,0xad,0x02,0x00,0x00,0xac,0x02,0x00,0x00, -0x41,0x00,0x06,0x00,0xfa,0x00,0x00,0x00,0xae,0x02,0x00,0x00, -0x93,0x02,0x00,0x00,0x34,0x00,0x00,0x00,0xa0,0x02,0x00,0x00, -0x3e,0x00,0x03,0x00,0xae,0x02,0x00,0x00,0xad,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0x8f,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x8f,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x7b,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x7b,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xb0,0x02,0x00,0x00,0xbe,0x02,0x00,0x00, -0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x78,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x7a,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x73,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x73,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xb2,0x02,0x00,0x00, -0xbc,0x02,0x00,0x00,0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x70,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x72,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0x5b,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x5b,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xb4,0x02,0x00,0x00,0xba,0x02,0x00,0x00,0xc6,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x58,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x5a,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x53,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x53,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xb6,0x02,0x00,0x00,0xb9,0x02,0x00,0x00, -0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x50,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x52,0x02,0x00,0x00,0xfd,0x00,0x01,0x00, -0x38,0x00,0x01,0x00, -}; -const uint64_t matmul_f32_aligned_m_fp32_len = 10348; - -unsigned char matmul_f32_aligned_s_data[] = { -0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, -0x6a,0x03,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, -0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x09,0x00,0x00,0x00, -0x0b,0x00,0x06,0x00,0x01,0x00,0x00,0x00,0x47,0x4c,0x53,0x4c, -0x2e,0x73,0x74,0x64,0x2e,0x34,0x35,0x30,0x00,0x00,0x00,0x00, -0x0e,0x00,0x03,0x00,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x0f,0x00,0x0f,0x00,0x05,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x6d,0x61,0x69,0x6e,0x00,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x3d,0x00,0x00,0x00,0x4c,0x00,0x00,0x00, -0xeb,0x00,0x00,0x00,0xfa,0x00,0x00,0x00,0x88,0x01,0x00,0x00, -0x95,0x01,0x00,0x00,0xcb,0x02,0x00,0x00,0x14,0x03,0x00,0x00, -0x10,0x00,0x06,0x00,0x04,0x00,0x00,0x00,0x11,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x0b,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x1c,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x10,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x08,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x03,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x10,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x14,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x18,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x07,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x10,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x24,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x0a,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x10,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x2c,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x30,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x0d,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0x47,0x00,0x03,0x00, -0x10,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x37,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x3d,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x1a,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x4c,0x00,0x00,0x00, -0x0b,0x00,0x00,0x00,0x1b,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x4f,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x53,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x60,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x62,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x07,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x6c,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x03,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x9d,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0xaf,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x05,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0xb2,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x08,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xf7,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x48,0x00,0x04,0x00, -0xf8,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x05,0x00,0x00,0x00, -0x48,0x00,0x04,0x00,0xf8,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0xf8,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0xf8,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x07,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x47,0x00,0x03,0x00, -0xf8,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0xfa,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0xfa,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x69,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x6a,0x01,0x00,0x00,0x0b,0x00,0x00,0x00,0x19,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x92,0x01,0x00,0x00,0x06,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0x93,0x01,0x00,0x00, -0x00,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0x48,0x00,0x04,0x00, -0x93,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x93,0x01,0x00,0x00,0x00,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x93,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x07,0x00,0x00,0x00, -0x10,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x93,0x01,0x00,0x00, -0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x95,0x01,0x00,0x00, -0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x95,0x01,0x00,0x00,0x21,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0xcb,0x02,0x00,0x00,0x0b,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x11,0x03,0x00,0x00, -0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00, -0x12,0x03,0x00,0x00,0x00,0x00,0x00,0x00,0x19,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x12,0x03,0x00,0x00,0x00,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, -0x12,0x03,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x14,0x03,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x14,0x03,0x00,0x00,0x21,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x13,0x00,0x02,0x00,0x02,0x00,0x00,0x00, -0x21,0x00,0x03,0x00,0x03,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x15,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x17,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x0a,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x0d,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x1e,0x00,0x10,0x00,0x10,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x11,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x11,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x15,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, -0x14,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x15,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x0a,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, -0x27,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0x2d,0x00,0x00,0x00,0x07,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x34,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x37,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00,0x3d,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x3e,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x0a,0x00,0x00,0x00,0x4c,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x4f,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x53,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x54,0x00,0x00,0x00,0x86,0x00,0x00,0x00, -0x37,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x58,0x00,0x00,0x00,0x86,0x00,0x00,0x00, -0x37,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x32,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x61,0x00,0x00,0x00, -0x86,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x60,0x00,0x00,0x00, -0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x62,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x63,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x61,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x67,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x61,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x08,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x6e,0x00,0x00,0x00, -0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x73,0x00,0x00,0x00, -0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x77,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x72,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x77,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x7c,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x80,0x00,0x00,0x00, 0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, -0x7c,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0x87,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x8d,0x00,0x00,0x00, +0x85,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0x90,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x96,0x00,0x00,0x00, 0x03,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, -0x98,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x32,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x9d,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x9f,0x00,0x00,0x00, +0xa1,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x32,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xa6,0x00,0x00,0x00,0x40,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0xa8,0x00,0x00,0x00, 0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xae,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00, +0xb7,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00, 0x62,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xaf,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xb0,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0x53,0x00,0x00,0x00,0xaf,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xb1,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0xb8,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xb9,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0x53,0x00,0x00,0x00,0xb8,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xba,0x00,0x00,0x00,0x84,0x00,0x00,0x00, 0x4f,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x32,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xb2,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xb3,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0xb1,0x00,0x00,0x00,0xb2,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xb4,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0xb3,0x00,0x00,0x00,0x60,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xb5,0x00,0x00,0x00, -0x86,0x00,0x00,0x00,0xb0,0x00,0x00,0x00,0xb4,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xb6,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0xae,0x00,0x00,0x00,0xb5,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xb7,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0xb6,0x00,0x00,0x00,0xb2,0x00,0x00,0x00, -0x14,0x00,0x02,0x00,0xb8,0x00,0x00,0x00,0x16,0x00,0x03,0x00, -0xba,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xbb,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0xbb,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xbc,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0xba,0x00,0x00,0x00,0xbb,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xbd,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0xbc,0x00,0x00,0x00,0x60,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xbe,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0xb9,0x00,0x00,0x00,0xbd,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xbf,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0xb7,0x00,0x00,0x00,0xbe,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xc0,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0xbf,0x00,0x00,0x00,0xbb,0x00,0x00,0x00, +0x14,0x00,0x02,0x00,0xc1,0x00,0x00,0x00,0x16,0x00,0x03,0x00, +0xc3,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xc4,0x00,0x00,0x00,0x84,0x00,0x00,0x00, 0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xbc,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0xbb,0x00,0x00,0x00,0xb5,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xbd,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0xbc,0x00,0x00,0x00,0xb2,0x00,0x00,0x00,0x1c,0x00,0x04,0x00, -0xbe,0x00,0x00,0x00,0xba,0x00,0x00,0x00,0xbd,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0xbf,0x00,0x00,0x00,0x07,0x00,0x00,0x00, -0xbe,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0xba,0x00,0x00,0x00, -0xc2,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0xc3,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0xba,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0xc6,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x16,0x00,0x03,0x00,0xe6,0x00,0x00,0x00, -0x10,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xe7,0x00,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xe8,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x37,0x00,0x00,0x00, -0xe7,0x00,0x00,0x00,0x1c,0x00,0x04,0x00,0xe9,0x00,0x00,0x00, -0xe6,0x00,0x00,0x00,0xe8,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0xea,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0xe9,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0xea,0x00,0x00,0x00,0xeb,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xef,0x00,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x17,0x00,0x04,0x00,0xf5,0x00,0x00,0x00, -0xba,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x18,0x00,0x04,0x00, -0xf6,0x00,0x00,0x00,0xf5,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x1d,0x00,0x03,0x00,0xf7,0x00,0x00,0x00,0xf6,0x00,0x00,0x00, -0x1e,0x00,0x03,0x00,0xf8,0x00,0x00,0x00,0xf7,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0xf9,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0xf8,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0xf9,0x00,0x00,0x00, -0xfa,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0xfc,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0xba,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x00,0x01,0x00,0x00,0x04,0x00,0x00,0x00, -0xe6,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x05,0x01,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x13,0x01,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x21,0x01,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0xc5,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0xc4,0x00,0x00,0x00,0xbe,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xc6,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0xc5,0x00,0x00,0x00,0xbb,0x00,0x00,0x00,0x1c,0x00,0x04,0x00, +0xc7,0x00,0x00,0x00,0xc3,0x00,0x00,0x00,0xc6,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0xc8,0x00,0x00,0x00,0x07,0x00,0x00,0x00, +0xc7,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0xc3,0x00,0x00,0x00, +0xcb,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0xcc,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0xc3,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0xcf,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xf3,0x00,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, 0x39,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x26,0x01,0x00,0x00,0x03,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x30,0x01,0x00,0x00,0x80,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x35,0x01,0x00,0x00,0x04,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x3f,0x01,0x00,0x00, +0xfa,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xfe,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, +0x16,0x00,0x03,0x00,0x02,0x01,0x00,0x00,0x10,0x00,0x00,0x00, +0x15,0x00,0x04,0x00,0x03,0x01,0x00,0x00,0x08,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x04,0x01,0x00,0x00,0x20,0x00,0x00,0x00,0x1c,0x00,0x04,0x00, +0x05,0x01,0x00,0x00,0x03,0x01,0x00,0x00,0x04,0x01,0x00,0x00, +0x1e,0x00,0x04,0x00,0x06,0x01,0x00,0x00,0x02,0x01,0x00,0x00, +0x05,0x01,0x00,0x00,0x1d,0x00,0x03,0x00,0x07,0x01,0x00,0x00, +0x06,0x01,0x00,0x00,0x1e,0x00,0x03,0x00,0x08,0x01,0x00,0x00, +0x07,0x01,0x00,0x00,0x20,0x00,0x04,0x00,0x09,0x01,0x00,0x00, +0x0c,0x00,0x00,0x00,0x08,0x01,0x00,0x00,0x3b,0x00,0x04,0x00, +0x09,0x01,0x00,0x00,0x0a,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x0c,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, +0x02,0x01,0x00,0x00,0x17,0x00,0x04,0x00,0x10,0x01,0x00,0x00, +0xc3,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x15,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x03,0x01,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x24,0x01,0x00,0x00, 0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x44,0x01,0x00,0x00, -0x05,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x4e,0x01,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x53,0x01,0x00,0x00,0x06,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x25,0x01,0x00,0x00, +0x84,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x24,0x01,0x00,0x00, +0x1c,0x00,0x04,0x00,0x26,0x01,0x00,0x00,0xc3,0x00,0x00,0x00, +0x25,0x01,0x00,0x00,0x20,0x00,0x04,0x00,0x27,0x01,0x00,0x00, +0x04,0x00,0x00,0x00,0x26,0x01,0x00,0x00,0x3b,0x00,0x04,0x00, +0x27,0x01,0x00,0x00,0x28,0x01,0x00,0x00,0x04,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x2c,0x01,0x00,0x00,0x04,0x00,0x00,0x00, +0xc3,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x33,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x33,0x00,0x06,0x00, +0x09,0x00,0x00,0x00,0x34,0x01,0x00,0x00,0x33,0x01,0x00,0x00, +0x39,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x35,0x01,0x00,0x00,0x51,0x00,0x00,0x00, +0x34,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x36,0x01,0x00,0x00,0x84,0x00,0x00,0x00, +0x35,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x37,0x01,0x00,0x00,0x86,0x00,0x00,0x00, +0x36,0x01,0x00,0x00,0x6c,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x55,0x01,0x00,0x00,0x80,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x56,0x01,0x00,0x00,0x84,0x00,0x00,0x00, +0xa6,0x00,0x00,0x00,0x55,0x01,0x00,0x00,0x1c,0x00,0x04,0x00, +0x57,0x01,0x00,0x00,0xc3,0x00,0x00,0x00,0x56,0x01,0x00,0x00, +0x20,0x00,0x04,0x00,0x58,0x01,0x00,0x00,0x04,0x00,0x00,0x00, +0x57,0x01,0x00,0x00,0x3b,0x00,0x04,0x00,0x58,0x01,0x00,0x00, +0x59,0x01,0x00,0x00,0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00, 0x06,0x00,0x00,0x00,0x5d,0x01,0x00,0x00,0x80,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x62,0x01,0x00,0x00,0x07,0x00,0x00,0x00, -0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x69,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0x33,0x00,0x06,0x00,0x09,0x00,0x00,0x00, -0x6a,0x01,0x00,0x00,0x69,0x01,0x00,0x00,0x39,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x6b,0x01,0x00,0x00,0x51,0x00,0x00,0x00,0x6a,0x01,0x00,0x00, -0x00,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x6c,0x01,0x00,0x00,0x84,0x00,0x00,0x00,0x6b,0x01,0x00,0x00, -0x6d,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x6d,0x01,0x00,0x00,0x86,0x00,0x00,0x00,0x6c,0x01,0x00,0x00, -0x6c,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x84,0x01,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x85,0x01,0x00,0x00,0x84,0x00,0x00,0x00,0x9d,0x00,0x00,0x00, -0x84,0x01,0x00,0x00,0x1c,0x00,0x04,0x00,0x86,0x01,0x00,0x00, -0xe6,0x00,0x00,0x00,0x85,0x01,0x00,0x00,0x20,0x00,0x04,0x00, -0x87,0x01,0x00,0x00,0x04,0x00,0x00,0x00,0x86,0x01,0x00,0x00, -0x3b,0x00,0x04,0x00,0x87,0x01,0x00,0x00,0x88,0x01,0x00,0x00, -0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x8c,0x01,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x1d,0x00,0x03,0x00,0x92,0x01,0x00,0x00, -0xf6,0x00,0x00,0x00,0x1e,0x00,0x03,0x00,0x93,0x01,0x00,0x00, -0x92,0x01,0x00,0x00,0x20,0x00,0x04,0x00,0x94,0x01,0x00,0x00, -0x0c,0x00,0x00,0x00,0x93,0x01,0x00,0x00,0x3b,0x00,0x04,0x00, -0x94,0x01,0x00,0x00,0x95,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x9e,0x01,0x00,0x00, -0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xac,0x01,0x00,0x00, -0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xba,0x01,0x00,0x00, -0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xc8,0x01,0x00,0x00, -0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xd6,0x01,0x00,0x00, -0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xe4,0x01,0x00,0x00, -0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xf2,0x01,0x00,0x00, -0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xff,0x01,0x00,0x00, -0x08,0x01,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x00,0x02,0x00,0x00,0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x6d,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x03,0x02,0x00,0x00,0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x6d,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x1e,0x02,0x00,0x00,0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x1c,0x00,0x04,0x00,0x1f,0x02,0x00,0x00, -0xe6,0x00,0x00,0x00,0x1e,0x02,0x00,0x00,0x20,0x00,0x04,0x00, -0x20,0x02,0x00,0x00,0x07,0x00,0x00,0x00,0x1f,0x02,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x30,0x02,0x00,0x00, -0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x36,0x02,0x00,0x00,0x07,0x00,0x00,0x00, -0xe6,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x4c,0x02,0x00,0x00,0x84,0x00,0x00,0x00,0xb5,0x00,0x00,0x00, -0xb2,0x00,0x00,0x00,0x1c,0x00,0x04,0x00,0x4d,0x02,0x00,0x00, -0xe6,0x00,0x00,0x00,0x4c,0x02,0x00,0x00,0x20,0x00,0x04,0x00, -0x4e,0x02,0x00,0x00,0x07,0x00,0x00,0x00,0x4d,0x02,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x57,0x02,0x00,0x00, -0x86,0x00,0x00,0x00,0xaf,0x00,0x00,0x00,0xb5,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x5f,0x02,0x00,0x00, -0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x8e,0x02,0x00,0x00, -0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0xc3,0x02,0x00,0x00, -0x0d,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00, -0xcb,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, -0x11,0x03,0x00,0x00,0xba,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, -0x12,0x03,0x00,0x00,0x11,0x03,0x00,0x00,0x20,0x00,0x04,0x00, -0x13,0x03,0x00,0x00,0x0c,0x00,0x00,0x00,0x12,0x03,0x00,0x00, -0x3b,0x00,0x04,0x00,0x13,0x03,0x00,0x00,0x14,0x03,0x00,0x00, -0x0c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, -0x19,0x03,0x00,0x00,0x05,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x26,0x03,0x00,0x00,0x84,0x00,0x00,0x00, -0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x36,0x00,0x05,0x00, -0x02,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x03,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x05,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0xbf,0x00,0x00,0x00,0xc0,0x00,0x00,0x00, -0x07,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x20,0x02,0x00,0x00, -0x21,0x02,0x00,0x00,0x07,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x4e,0x02,0x00,0x00,0x4f,0x02,0x00,0x00,0x07,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00,0x0e,0x00,0x00,0x00, -0x0b,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x0e,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x17,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x18,0x00,0x00,0x00, -0x0f,0x00,0x00,0x00,0x17,0x00,0x00,0x00,0x89,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x1e,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, -0x17,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0x22,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x22,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x24,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x28,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x27,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x29,0x00,0x00,0x00,0x28,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x2a,0x00,0x00,0x00, -0x1e,0x00,0x00,0x00,0x29,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x2d,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x2f,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x30,0x00,0x00,0x00,0x24,0x00,0x00,0x00, -0x2f,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x32,0x00,0x00,0x00,0x30,0x00,0x00,0x00,0x2a,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x35,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x36,0x00,0x00,0x00,0x35,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x38,0x00,0x00,0x00, -0x36,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x82,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x38,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x3b,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x37,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00,0x3f,0x00,0x00,0x00, -0x3d,0x00,0x00,0x00,0x3e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x3f,0x00,0x00,0x00, -0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x42,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0x3b,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x47,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x3b,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00, -0x49,0x00,0x00,0x00,0x3d,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x4a,0x00,0x00,0x00, -0x49,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00, -0x4d,0x00,0x00,0x00,0x4c,0x00,0x00,0x00,0x3e,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, -0x4d,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x50,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x4f,0x00,0x00,0x00, -0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x55,0x00,0x00,0x00, -0x50,0x00,0x00,0x00,0x54,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x59,0x00,0x00,0x00,0x50,0x00,0x00,0x00, -0x58,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x5d,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x4f,0x00,0x00,0x00, -0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x64,0x00,0x00,0x00, -0x5d,0x00,0x00,0x00,0x63,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x68,0x00,0x00,0x00,0x5d,0x00,0x00,0x00, -0x67,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x6f,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x6e,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x74,0x00,0x00,0x00, -0x4e,0x00,0x00,0x00,0x73,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x78,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x77,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x79,0x00,0x00,0x00,0x78,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x7a,0x00,0x00,0x00,0x47,0x00,0x00,0x00, -0x79,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0x7d,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x7c,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x7e,0x00,0x00,0x00, -0x7d,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x80,0x00,0x00,0x00,0x47,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x83,0x00,0x00,0x00, -0x80,0x00,0x00,0x00,0x79,0x00,0x00,0x00,0x0c,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x26,0x00,0x00,0x00,0x7e,0x00,0x00,0x00,0x83,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x88,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x87,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x89,0x00,0x00,0x00,0x88,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8a,0x00,0x00,0x00, -0x32,0x00,0x00,0x00,0x89,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x8c,0x00,0x00,0x00,0x42,0x00,0x00,0x00, -0x37,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0x8e,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x8d,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x8f,0x00,0x00,0x00, -0x8e,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x90,0x00,0x00,0x00,0x8c,0x00,0x00,0x00,0x8f,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x91,0x00,0x00,0x00, -0x8a,0x00,0x00,0x00,0x90,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x93,0x00,0x00,0x00,0x91,0x00,0x00,0x00, -0x7a,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x94,0x00,0x00,0x00,0x93,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x99,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x98,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x9a,0x00,0x00,0x00,0x99,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9b,0x00,0x00,0x00, -0x0f,0x00,0x00,0x00,0x9a,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x9e,0x00,0x00,0x00,0x4a,0x00,0x00,0x00, -0x9d,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0xa0,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x9f,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xa1,0x00,0x00,0x00, -0xa0,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xa2,0x00,0x00,0x00,0x9e,0x00,0x00,0x00,0xa1,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa3,0x00,0x00,0x00, -0x9b,0x00,0x00,0x00,0xa2,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xa5,0x00,0x00,0x00,0xa3,0x00,0x00,0x00, -0x7a,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xa6,0x00,0x00,0x00,0xa5,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xa8,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xa8,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0x38,0x03,0x00,0x00,0x3e,0x00,0x00,0x00,0x05,0x00,0x00,0x00, -0xc7,0x00,0x00,0x00,0xa9,0x00,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb8,0x00,0x00,0x00,0xb9,0x00,0x00,0x00,0x38,0x03,0x00,0x00, -0xb7,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xaa,0x00,0x00,0x00, -0xa9,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0xb9,0x00,0x00,0x00,0xa9,0x00,0x00,0x00,0xaa,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xa9,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0xc3,0x00,0x00,0x00,0xc4,0x00,0x00,0x00,0xc0,0x00,0x00,0x00, -0x38,0x03,0x00,0x00,0x3e,0x00,0x03,0x00,0xc4,0x00,0x00,0x00, -0xc2,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xc7,0x00,0x00,0x00,0x38,0x03,0x00,0x00,0xc6,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xa8,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xaa,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xca,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xca,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0x51,0x03,0x00,0x00,0xa6,0x00,0x00,0x00, -0xaa,0x00,0x00,0x00,0x05,0x02,0x00,0x00,0xcd,0x00,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x4d,0x03,0x00,0x00, -0x94,0x00,0x00,0x00,0xaa,0x00,0x00,0x00,0x02,0x02,0x00,0x00, -0xcd,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0x39,0x03,0x00,0x00,0x7a,0x00,0x00,0x00,0xaa,0x00,0x00,0x00, -0xb3,0x02,0x00,0x00,0xcd,0x00,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb8,0x00,0x00,0x00,0xd1,0x00,0x00,0x00,0x39,0x03,0x00,0x00, -0x84,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xcc,0x00,0x00,0x00, -0xcd,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0xd1,0x00,0x00,0x00,0xcb,0x00,0x00,0x00,0xcc,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xcb,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xd3,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xd3,0x00,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x49,0x03,0x00,0x00, -0x3e,0x00,0x00,0x00,0xcb,0x00,0x00,0x00,0x6f,0x01,0x00,0x00, -0xd4,0x00,0x00,0x00,0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00, -0xd9,0x00,0x00,0x00,0x49,0x03,0x00,0x00,0x37,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0xd5,0x00,0x00,0x00,0xd4,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xd9,0x00,0x00,0x00, -0xd4,0x00,0x00,0x00,0xd5,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xd4,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xde,0x00,0x00,0x00,0x74,0x00,0x00,0x00,0x49,0x03,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xe1,0x00,0x00,0x00, -0xde,0x00,0x00,0x00,0x8f,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xe2,0x00,0x00,0x00,0xe1,0x00,0x00,0x00, -0x6d,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xe3,0x00,0x00,0x00,0x4d,0x03,0x00,0x00,0xe2,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xe5,0x00,0x00,0x00, -0xe3,0x00,0x00,0x00,0x6f,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xf0,0x00,0x00,0x00,0xde,0x00,0x00,0x00, -0xef,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xf2,0x00,0x00,0x00,0x6f,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xf3,0x00,0x00,0x00, -0xf0,0x00,0x00,0x00,0xf2,0x00,0x00,0x00,0x41,0x00,0x08,0x00, -0xfc,0x00,0x00,0x00,0xfd,0x00,0x00,0x00,0xfa,0x00,0x00,0x00, -0x34,0x00,0x00,0x00,0xe5,0x00,0x00,0x00,0x34,0x00,0x00,0x00, -0x3e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0xba,0x00,0x00,0x00, -0xfe,0x00,0x00,0x00,0xfd,0x00,0x00,0x00,0x73,0x00,0x04,0x00, -0xe6,0x00,0x00,0x00,0xff,0x00,0x00,0x00,0xfe,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x00,0x01,0x00,0x00,0x01,0x01,0x00,0x00, -0xeb,0x00,0x00,0x00,0xf3,0x00,0x00,0x00,0x3e,0x00,0x03,0x00, -0x01,0x01,0x00,0x00,0xff,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x06,0x01,0x00,0x00,0xde,0x00,0x00,0x00, -0x05,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x09,0x01,0x00,0x00,0x06,0x01,0x00,0x00,0xf2,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x0a,0x01,0x00,0x00, -0x09,0x01,0x00,0x00,0x39,0x00,0x00,0x00,0x41,0x00,0x08,0x00, -0xfc,0x00,0x00,0x00,0x0c,0x01,0x00,0x00,0xfa,0x00,0x00,0x00, -0x34,0x00,0x00,0x00,0xe5,0x00,0x00,0x00,0x34,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0xba,0x00,0x00,0x00, -0x0d,0x01,0x00,0x00,0x0c,0x01,0x00,0x00,0x73,0x00,0x04,0x00, -0xe6,0x00,0x00,0x00,0x0e,0x01,0x00,0x00,0x0d,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0x00,0x01,0x00,0x00,0x0f,0x01,0x00,0x00, -0xeb,0x00,0x00,0x00,0x0a,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0x0f,0x01,0x00,0x00,0x0e,0x01,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x14,0x01,0x00,0x00,0xde,0x00,0x00,0x00, -0x13,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x17,0x01,0x00,0x00,0x14,0x01,0x00,0x00,0xf2,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x18,0x01,0x00,0x00, -0x17,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x41,0x00,0x08,0x00, -0xfc,0x00,0x00,0x00,0x1a,0x01,0x00,0x00,0xfa,0x00,0x00,0x00, -0x34,0x00,0x00,0x00,0xe5,0x00,0x00,0x00,0x34,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0xba,0x00,0x00,0x00, -0x1b,0x01,0x00,0x00,0x1a,0x01,0x00,0x00,0x73,0x00,0x04,0x00, -0xe6,0x00,0x00,0x00,0x1c,0x01,0x00,0x00,0x1b,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0x00,0x01,0x00,0x00,0x1d,0x01,0x00,0x00, -0xeb,0x00,0x00,0x00,0x18,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0x1d,0x01,0x00,0x00,0x1c,0x01,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x22,0x01,0x00,0x00,0xde,0x00,0x00,0x00, -0x21,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x25,0x01,0x00,0x00,0x22,0x01,0x00,0x00,0xf2,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x27,0x01,0x00,0x00, -0x25,0x01,0x00,0x00,0x26,0x01,0x00,0x00,0x41,0x00,0x08,0x00, -0xfc,0x00,0x00,0x00,0x29,0x01,0x00,0x00,0xfa,0x00,0x00,0x00, -0x34,0x00,0x00,0x00,0xe5,0x00,0x00,0x00,0x34,0x00,0x00,0x00, -0x26,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0xba,0x00,0x00,0x00, -0x2a,0x01,0x00,0x00,0x29,0x01,0x00,0x00,0x73,0x00,0x04,0x00, -0xe6,0x00,0x00,0x00,0x2b,0x01,0x00,0x00,0x2a,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0x00,0x01,0x00,0x00,0x2c,0x01,0x00,0x00, -0xeb,0x00,0x00,0x00,0x27,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0x2c,0x01,0x00,0x00,0x2b,0x01,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x31,0x01,0x00,0x00,0xde,0x00,0x00,0x00, -0x30,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x34,0x01,0x00,0x00,0x31,0x01,0x00,0x00,0xf2,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x36,0x01,0x00,0x00, -0x34,0x01,0x00,0x00,0x35,0x01,0x00,0x00,0x41,0x00,0x08,0x00, -0xfc,0x00,0x00,0x00,0x38,0x01,0x00,0x00,0xfa,0x00,0x00,0x00, -0x34,0x00,0x00,0x00,0xe5,0x00,0x00,0x00,0xc6,0x00,0x00,0x00, -0x3e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0xba,0x00,0x00,0x00, -0x39,0x01,0x00,0x00,0x38,0x01,0x00,0x00,0x73,0x00,0x04,0x00, -0xe6,0x00,0x00,0x00,0x3a,0x01,0x00,0x00,0x39,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0x00,0x01,0x00,0x00,0x3b,0x01,0x00,0x00, -0xeb,0x00,0x00,0x00,0x36,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0x3b,0x01,0x00,0x00,0x3a,0x01,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x40,0x01,0x00,0x00,0xde,0x00,0x00,0x00, -0x3f,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x43,0x01,0x00,0x00,0x40,0x01,0x00,0x00,0xf2,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x45,0x01,0x00,0x00, -0x43,0x01,0x00,0x00,0x44,0x01,0x00,0x00,0x41,0x00,0x08,0x00, -0xfc,0x00,0x00,0x00,0x47,0x01,0x00,0x00,0xfa,0x00,0x00,0x00, -0x34,0x00,0x00,0x00,0xe5,0x00,0x00,0x00,0xc6,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0xba,0x00,0x00,0x00, -0x48,0x01,0x00,0x00,0x47,0x01,0x00,0x00,0x73,0x00,0x04,0x00, -0xe6,0x00,0x00,0x00,0x49,0x01,0x00,0x00,0x48,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0x00,0x01,0x00,0x00,0x4a,0x01,0x00,0x00, -0xeb,0x00,0x00,0x00,0x45,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0x4a,0x01,0x00,0x00,0x49,0x01,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x4f,0x01,0x00,0x00,0xde,0x00,0x00,0x00, -0x4e,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x52,0x01,0x00,0x00,0x4f,0x01,0x00,0x00,0xf2,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x54,0x01,0x00,0x00, -0x52,0x01,0x00,0x00,0x53,0x01,0x00,0x00,0x41,0x00,0x08,0x00, -0xfc,0x00,0x00,0x00,0x56,0x01,0x00,0x00,0xfa,0x00,0x00,0x00, -0x34,0x00,0x00,0x00,0xe5,0x00,0x00,0x00,0xc6,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0xba,0x00,0x00,0x00, -0x57,0x01,0x00,0x00,0x56,0x01,0x00,0x00,0x73,0x00,0x04,0x00, -0xe6,0x00,0x00,0x00,0x58,0x01,0x00,0x00,0x57,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0x00,0x01,0x00,0x00,0x59,0x01,0x00,0x00, -0xeb,0x00,0x00,0x00,0x54,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0x59,0x01,0x00,0x00,0x58,0x01,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x5e,0x01,0x00,0x00,0xde,0x00,0x00,0x00, -0x5d,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x61,0x01,0x00,0x00,0x5e,0x01,0x00,0x00,0xf2,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x63,0x01,0x00,0x00, -0x61,0x01,0x00,0x00,0x62,0x01,0x00,0x00,0x41,0x00,0x08,0x00, -0xfc,0x00,0x00,0x00,0x65,0x01,0x00,0x00,0xfa,0x00,0x00,0x00, -0x34,0x00,0x00,0x00,0xe5,0x00,0x00,0x00,0xc6,0x00,0x00,0x00, -0x26,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0xba,0x00,0x00,0x00, -0x66,0x01,0x00,0x00,0x65,0x01,0x00,0x00,0x73,0x00,0x04,0x00, -0xe6,0x00,0x00,0x00,0x67,0x01,0x00,0x00,0x66,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0x00,0x01,0x00,0x00,0x68,0x01,0x00,0x00, -0xeb,0x00,0x00,0x00,0x63,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0x68,0x01,0x00,0x00,0x67,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x6f,0x01,0x00,0x00,0x49,0x03,0x00,0x00, -0x6d,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xd3,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xd5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x71,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x71,0x01,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x4a,0x03,0x00,0x00, -0x3e,0x00,0x00,0x00,0xd5,0x00,0x00,0x00,0xfe,0x01,0x00,0x00, -0x72,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00, -0x77,0x01,0x00,0x00,0x4a,0x03,0x00,0x00,0x9d,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0x73,0x01,0x00,0x00,0x72,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x77,0x01,0x00,0x00, -0x72,0x01,0x00,0x00,0x73,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x72,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x7c,0x01,0x00,0x00,0x74,0x00,0x00,0x00,0x4a,0x03,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x7f,0x01,0x00,0x00, -0x7c,0x01,0x00,0x00,0xa1,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x80,0x01,0x00,0x00,0x7f,0x01,0x00,0x00, -0x6d,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x81,0x01,0x00,0x00,0x51,0x03,0x00,0x00,0x80,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x83,0x01,0x00,0x00, -0x81,0x01,0x00,0x00,0x6f,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x8d,0x01,0x00,0x00,0x7c,0x01,0x00,0x00, -0x8c,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x8f,0x01,0x00,0x00,0x6f,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x90,0x01,0x00,0x00, -0x8d,0x01,0x00,0x00,0x8f,0x01,0x00,0x00,0x41,0x00,0x08,0x00, -0xfc,0x00,0x00,0x00,0x97,0x01,0x00,0x00,0x95,0x01,0x00,0x00, -0x34,0x00,0x00,0x00,0x83,0x01,0x00,0x00,0x34,0x00,0x00,0x00, -0x3e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0xba,0x00,0x00,0x00, -0x98,0x01,0x00,0x00,0x97,0x01,0x00,0x00,0x73,0x00,0x04,0x00, -0xe6,0x00,0x00,0x00,0x99,0x01,0x00,0x00,0x98,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0x00,0x01,0x00,0x00,0x9a,0x01,0x00,0x00, -0x88,0x01,0x00,0x00,0x90,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0x9a,0x01,0x00,0x00,0x99,0x01,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x9f,0x01,0x00,0x00,0x7c,0x01,0x00,0x00, -0x9e,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xa2,0x01,0x00,0x00,0x9f,0x01,0x00,0x00,0x8f,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa3,0x01,0x00,0x00, -0xa2,0x01,0x00,0x00,0x39,0x00,0x00,0x00,0x41,0x00,0x08,0x00, -0xfc,0x00,0x00,0x00,0xa5,0x01,0x00,0x00,0x95,0x01,0x00,0x00, -0x34,0x00,0x00,0x00,0x83,0x01,0x00,0x00,0x34,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0xba,0x00,0x00,0x00, -0xa6,0x01,0x00,0x00,0xa5,0x01,0x00,0x00,0x73,0x00,0x04,0x00, -0xe6,0x00,0x00,0x00,0xa7,0x01,0x00,0x00,0xa6,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0x00,0x01,0x00,0x00,0xa8,0x01,0x00,0x00, -0x88,0x01,0x00,0x00,0xa3,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0xa8,0x01,0x00,0x00,0xa7,0x01,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xad,0x01,0x00,0x00,0x7c,0x01,0x00,0x00, -0xac,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xb0,0x01,0x00,0x00,0xad,0x01,0x00,0x00,0x8f,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xb1,0x01,0x00,0x00, -0xb0,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x41,0x00,0x08,0x00, -0xfc,0x00,0x00,0x00,0xb3,0x01,0x00,0x00,0x95,0x01,0x00,0x00, -0x34,0x00,0x00,0x00,0x83,0x01,0x00,0x00,0x34,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0xba,0x00,0x00,0x00, -0xb4,0x01,0x00,0x00,0xb3,0x01,0x00,0x00,0x73,0x00,0x04,0x00, -0xe6,0x00,0x00,0x00,0xb5,0x01,0x00,0x00,0xb4,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0x00,0x01,0x00,0x00,0xb6,0x01,0x00,0x00, -0x88,0x01,0x00,0x00,0xb1,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0xb6,0x01,0x00,0x00,0xb5,0x01,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xbb,0x01,0x00,0x00,0x7c,0x01,0x00,0x00, -0xba,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xbe,0x01,0x00,0x00,0xbb,0x01,0x00,0x00,0x8f,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xbf,0x01,0x00,0x00, -0xbe,0x01,0x00,0x00,0x26,0x01,0x00,0x00,0x41,0x00,0x08,0x00, -0xfc,0x00,0x00,0x00,0xc1,0x01,0x00,0x00,0x95,0x01,0x00,0x00, -0x34,0x00,0x00,0x00,0x83,0x01,0x00,0x00,0x34,0x00,0x00,0x00, -0x26,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0xba,0x00,0x00,0x00, -0xc2,0x01,0x00,0x00,0xc1,0x01,0x00,0x00,0x73,0x00,0x04,0x00, -0xe6,0x00,0x00,0x00,0xc3,0x01,0x00,0x00,0xc2,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0x00,0x01,0x00,0x00,0xc4,0x01,0x00,0x00, -0x88,0x01,0x00,0x00,0xbf,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0xc4,0x01,0x00,0x00,0xc3,0x01,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xc9,0x01,0x00,0x00,0x7c,0x01,0x00,0x00, -0xc8,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xcc,0x01,0x00,0x00,0xc9,0x01,0x00,0x00,0x8f,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xcd,0x01,0x00,0x00, -0xcc,0x01,0x00,0x00,0x35,0x01,0x00,0x00,0x41,0x00,0x08,0x00, -0xfc,0x00,0x00,0x00,0xcf,0x01,0x00,0x00,0x95,0x01,0x00,0x00, -0x34,0x00,0x00,0x00,0x83,0x01,0x00,0x00,0xc6,0x00,0x00,0x00, -0x3e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0xba,0x00,0x00,0x00, -0xd0,0x01,0x00,0x00,0xcf,0x01,0x00,0x00,0x73,0x00,0x04,0x00, -0xe6,0x00,0x00,0x00,0xd1,0x01,0x00,0x00,0xd0,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0x00,0x01,0x00,0x00,0xd2,0x01,0x00,0x00, -0x88,0x01,0x00,0x00,0xcd,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0xd2,0x01,0x00,0x00,0xd1,0x01,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xd7,0x01,0x00,0x00,0x7c,0x01,0x00,0x00, -0xd6,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xda,0x01,0x00,0x00,0xd7,0x01,0x00,0x00,0x8f,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xdb,0x01,0x00,0x00, -0xda,0x01,0x00,0x00,0x44,0x01,0x00,0x00,0x41,0x00,0x08,0x00, -0xfc,0x00,0x00,0x00,0xdd,0x01,0x00,0x00,0x95,0x01,0x00,0x00, -0x34,0x00,0x00,0x00,0x83,0x01,0x00,0x00,0xc6,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0xba,0x00,0x00,0x00, -0xde,0x01,0x00,0x00,0xdd,0x01,0x00,0x00,0x73,0x00,0x04,0x00, -0xe6,0x00,0x00,0x00,0xdf,0x01,0x00,0x00,0xde,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0x00,0x01,0x00,0x00,0xe0,0x01,0x00,0x00, -0x88,0x01,0x00,0x00,0xdb,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0xe0,0x01,0x00,0x00,0xdf,0x01,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xe5,0x01,0x00,0x00,0x7c,0x01,0x00,0x00, -0xe4,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xe8,0x01,0x00,0x00,0xe5,0x01,0x00,0x00,0x8f,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xe9,0x01,0x00,0x00, -0xe8,0x01,0x00,0x00,0x53,0x01,0x00,0x00,0x41,0x00,0x08,0x00, -0xfc,0x00,0x00,0x00,0xeb,0x01,0x00,0x00,0x95,0x01,0x00,0x00, -0x34,0x00,0x00,0x00,0x83,0x01,0x00,0x00,0xc6,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0xba,0x00,0x00,0x00, -0xec,0x01,0x00,0x00,0xeb,0x01,0x00,0x00,0x73,0x00,0x04,0x00, -0xe6,0x00,0x00,0x00,0xed,0x01,0x00,0x00,0xec,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0x00,0x01,0x00,0x00,0xee,0x01,0x00,0x00, -0x88,0x01,0x00,0x00,0xe9,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0xee,0x01,0x00,0x00,0xed,0x01,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xf3,0x01,0x00,0x00,0x7c,0x01,0x00,0x00, -0xf2,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xf6,0x01,0x00,0x00,0xf3,0x01,0x00,0x00,0x8f,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xf7,0x01,0x00,0x00, -0xf6,0x01,0x00,0x00,0x62,0x01,0x00,0x00,0x41,0x00,0x08,0x00, -0xfc,0x00,0x00,0x00,0xf9,0x01,0x00,0x00,0x95,0x01,0x00,0x00, -0x34,0x00,0x00,0x00,0x83,0x01,0x00,0x00,0xc6,0x00,0x00,0x00, -0x26,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0xba,0x00,0x00,0x00, -0xfa,0x01,0x00,0x00,0xf9,0x01,0x00,0x00,0x73,0x00,0x04,0x00, -0xe6,0x00,0x00,0x00,0xfb,0x01,0x00,0x00,0xfa,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0x00,0x01,0x00,0x00,0xfc,0x01,0x00,0x00, -0x88,0x01,0x00,0x00,0xf7,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0xfc,0x01,0x00,0x00,0xfb,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xfe,0x01,0x00,0x00,0x4a,0x03,0x00,0x00, -0x6d,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x71,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x73,0x01,0x00,0x00,0xe0,0x00,0x04,0x00, -0x0c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0xff,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x02,0x02,0x00,0x00, -0x4d,0x03,0x00,0x00,0x00,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x05,0x02,0x00,0x00,0x51,0x03,0x00,0x00, -0x03,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x07,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x07,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0x53,0x03,0x00,0x00,0x3e,0x00,0x00,0x00, -0x73,0x01,0x00,0x00,0xb1,0x02,0x00,0x00,0x0a,0x02,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00,0x0d,0x02,0x00,0x00, -0x53,0x03,0x00,0x00,0x6c,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0x09,0x02,0x00,0x00,0x0a,0x02,0x00,0x00,0x00,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x0d,0x02,0x00,0x00,0x08,0x02,0x00,0x00, -0x09,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x08,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0x0f,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x0f,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0x57,0x03,0x00,0x00,0x3e,0x00,0x00,0x00,0x08,0x02,0x00,0x00, -0x3b,0x02,0x00,0x00,0x12,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb8,0x00,0x00,0x00,0x15,0x02,0x00,0x00,0x57,0x03,0x00,0x00, -0x60,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x11,0x02,0x00,0x00, -0x12,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x15,0x02,0x00,0x00,0x10,0x02,0x00,0x00,0x11,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x10,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x17,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x17,0x02,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x69,0x03,0x00,0x00, -0x3e,0x00,0x00,0x00,0x10,0x02,0x00,0x00,0x39,0x02,0x00,0x00, -0x18,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00, -0x1d,0x02,0x00,0x00,0x69,0x03,0x00,0x00,0x62,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0x19,0x02,0x00,0x00,0x18,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x1d,0x02,0x00,0x00, -0x18,0x02,0x00,0x00,0x19,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x18,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x23,0x02,0x00,0x00,0x57,0x03,0x00,0x00,0x62,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x25,0x02,0x00,0x00, -0x23,0x02,0x00,0x00,0x69,0x03,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x27,0x02,0x00,0x00,0x55,0x00,0x00,0x00, -0x53,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x29,0x02,0x00,0x00,0x57,0x03,0x00,0x00,0x61,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x2a,0x02,0x00,0x00, -0x27,0x02,0x00,0x00,0x29,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x2c,0x02,0x00,0x00,0x64,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x2d,0x02,0x00,0x00,0x2a,0x02,0x00,0x00,0x2c,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x2f,0x02,0x00,0x00, -0x2d,0x02,0x00,0x00,0x69,0x03,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x31,0x02,0x00,0x00,0x2f,0x02,0x00,0x00, -0x30,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x33,0x02,0x00,0x00,0x31,0x02,0x00,0x00,0x53,0x03,0x00,0x00, -0x41,0x00,0x05,0x00,0x00,0x01,0x00,0x00,0x34,0x02,0x00,0x00, -0xeb,0x00,0x00,0x00,0x33,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, -0xe6,0x00,0x00,0x00,0x35,0x02,0x00,0x00,0x34,0x02,0x00,0x00, -0x41,0x00,0x05,0x00,0x36,0x02,0x00,0x00,0x37,0x02,0x00,0x00, -0x21,0x02,0x00,0x00,0x25,0x02,0x00,0x00,0x3e,0x00,0x03,0x00, -0x37,0x02,0x00,0x00,0x35,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x39,0x02,0x00,0x00,0x69,0x03,0x00,0x00, -0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x17,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x19,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x12,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x12,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3b,0x02,0x00,0x00, -0x57,0x03,0x00,0x00,0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x0f,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x11,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0x3d,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x3d,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0x58,0x03,0x00,0x00,0x3e,0x00,0x00,0x00,0x11,0x02,0x00,0x00, -0x69,0x02,0x00,0x00,0x40,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb8,0x00,0x00,0x00,0x43,0x02,0x00,0x00,0x58,0x03,0x00,0x00, -0xb5,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x3f,0x02,0x00,0x00, -0x40,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x43,0x02,0x00,0x00,0x3e,0x02,0x00,0x00,0x3f,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x3e,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x45,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x45,0x02,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x66,0x03,0x00,0x00, -0x3e,0x00,0x00,0x00,0x3e,0x02,0x00,0x00,0x67,0x02,0x00,0x00, -0x46,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00, -0x4b,0x02,0x00,0x00,0x66,0x03,0x00,0x00,0xb2,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0x47,0x02,0x00,0x00,0x46,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x4b,0x02,0x00,0x00, -0x46,0x02,0x00,0x00,0x47,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x46,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x51,0x02,0x00,0x00,0x58,0x03,0x00,0x00,0xb2,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x53,0x02,0x00,0x00, -0x51,0x02,0x00,0x00,0x66,0x03,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x55,0x02,0x00,0x00,0x59,0x00,0x00,0x00, -0xaf,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x58,0x02,0x00,0x00,0x58,0x03,0x00,0x00,0x57,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x59,0x02,0x00,0x00, -0x55,0x02,0x00,0x00,0x58,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x5b,0x02,0x00,0x00,0x68,0x00,0x00,0x00, -0xb2,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x5c,0x02,0x00,0x00,0x59,0x02,0x00,0x00,0x5b,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x5e,0x02,0x00,0x00, -0x5c,0x02,0x00,0x00,0x66,0x03,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x60,0x02,0x00,0x00,0x5e,0x02,0x00,0x00, -0x5f,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x62,0x02,0x00,0x00,0x60,0x02,0x00,0x00,0x53,0x03,0x00,0x00, -0x41,0x00,0x05,0x00,0x00,0x01,0x00,0x00,0x63,0x02,0x00,0x00, -0x88,0x01,0x00,0x00,0x62,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, -0xe6,0x00,0x00,0x00,0x64,0x02,0x00,0x00,0x63,0x02,0x00,0x00, -0x41,0x00,0x05,0x00,0x36,0x02,0x00,0x00,0x65,0x02,0x00,0x00, -0x4f,0x02,0x00,0x00,0x53,0x02,0x00,0x00,0x3e,0x00,0x03,0x00, -0x65,0x02,0x00,0x00,0x64,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x67,0x02,0x00,0x00,0x66,0x03,0x00,0x00, -0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x45,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x47,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x40,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x40,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x69,0x02,0x00,0x00, -0x58,0x03,0x00,0x00,0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x3d,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x3f,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0x6b,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x6b,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0x59,0x03,0x00,0x00,0x3e,0x00,0x00,0x00,0x3f,0x02,0x00,0x00, -0xaf,0x02,0x00,0x00,0x6e,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb8,0x00,0x00,0x00,0x71,0x02,0x00,0x00,0x59,0x03,0x00,0x00, -0xb5,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x6d,0x02,0x00,0x00, -0x6e,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x71,0x02,0x00,0x00,0x6c,0x02,0x00,0x00,0x6d,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x6c,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x73,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x73,0x02,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x5d,0x03,0x00,0x00, -0x3e,0x00,0x00,0x00,0x6c,0x02,0x00,0x00,0xad,0x02,0x00,0x00, -0x76,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00, -0x79,0x02,0x00,0x00,0x5d,0x03,0x00,0x00,0x60,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0x75,0x02,0x00,0x00,0x76,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x79,0x02,0x00,0x00, -0x74,0x02,0x00,0x00,0x75,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x74,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x7b,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x7b,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0x5f,0x03,0x00,0x00,0x3e,0x00,0x00,0x00, -0x74,0x02,0x00,0x00,0xab,0x02,0x00,0x00,0x7e,0x02,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00,0x81,0x02,0x00,0x00, -0x5f,0x03,0x00,0x00,0xb2,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0x7d,0x02,0x00,0x00,0x7e,0x02,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x81,0x02,0x00,0x00,0x7c,0x02,0x00,0x00, -0x7d,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x7c,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0x83,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x83,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0x61,0x03,0x00,0x00,0x3e,0x00,0x00,0x00,0x7c,0x02,0x00,0x00, -0xa9,0x02,0x00,0x00,0x84,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb8,0x00,0x00,0x00,0x89,0x02,0x00,0x00,0x61,0x03,0x00,0x00, -0x62,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x85,0x02,0x00,0x00, -0x84,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x89,0x02,0x00,0x00,0x84,0x02,0x00,0x00,0x85,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x84,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x8b,0x02,0x00,0x00,0x59,0x03,0x00,0x00, -0xb2,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x8d,0x02,0x00,0x00,0x8b,0x02,0x00,0x00,0x5f,0x03,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8f,0x02,0x00,0x00, -0x8d,0x02,0x00,0x00,0x8e,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x91,0x02,0x00,0x00,0x5d,0x03,0x00,0x00, -0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x92,0x02,0x00,0x00,0x8f,0x02,0x00,0x00,0x91,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x94,0x02,0x00,0x00, -0x92,0x02,0x00,0x00,0x61,0x03,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x98,0x02,0x00,0x00,0x91,0x02,0x00,0x00, -0x61,0x03,0x00,0x00,0x41,0x00,0x05,0x00,0x36,0x02,0x00,0x00, -0x99,0x02,0x00,0x00,0x21,0x02,0x00,0x00,0x98,0x02,0x00,0x00, -0x3d,0x00,0x04,0x00,0xe6,0x00,0x00,0x00,0x9a,0x02,0x00,0x00, -0x99,0x02,0x00,0x00,0x73,0x00,0x04,0x00,0xba,0x00,0x00,0x00, -0x9b,0x02,0x00,0x00,0x9a,0x02,0x00,0x00,0x41,0x00,0x05,0x00, -0x36,0x02,0x00,0x00,0xa0,0x02,0x00,0x00,0x4f,0x02,0x00,0x00, -0x8d,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0xe6,0x00,0x00,0x00, -0xa1,0x02,0x00,0x00,0xa0,0x02,0x00,0x00,0x73,0x00,0x04,0x00, -0xba,0x00,0x00,0x00,0xa2,0x02,0x00,0x00,0xa1,0x02,0x00,0x00, -0x41,0x00,0x05,0x00,0xc3,0x00,0x00,0x00,0xa4,0x02,0x00,0x00, -0xc0,0x00,0x00,0x00,0x94,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, -0xba,0x00,0x00,0x00,0xa5,0x02,0x00,0x00,0xa4,0x02,0x00,0x00, -0x0c,0x00,0x08,0x00,0xba,0x00,0x00,0x00,0xa6,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x9b,0x02,0x00,0x00, -0xa2,0x02,0x00,0x00,0xa5,0x02,0x00,0x00,0x3e,0x00,0x03,0x00, -0xa4,0x02,0x00,0x00,0xa6,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xa9,0x02,0x00,0x00,0x61,0x03,0x00,0x00, -0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x83,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x85,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x7e,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x7e,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xab,0x02,0x00,0x00, -0x5f,0x03,0x00,0x00,0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x7b,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x7d,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0x76,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x76,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xad,0x02,0x00,0x00,0x5d,0x03,0x00,0x00,0xc6,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x73,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x75,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x6e,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x6e,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xaf,0x02,0x00,0x00,0x59,0x03,0x00,0x00, -0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x6b,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x6d,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x0a,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x0a,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xb1,0x02,0x00,0x00, -0x53,0x03,0x00,0x00,0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x07,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x09,0x02,0x00,0x00, -0xe0,0x00,0x04,0x00,0x0c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0xff,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xcd,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xcd,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xb3,0x02,0x00,0x00,0x39,0x03,0x00,0x00, -0x6c,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xca,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xcc,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xb8,0x02,0x00,0x00,0x55,0x00,0x00,0x00, -0x53,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xb9,0x02,0x00,0x00,0x8c,0x00,0x00,0x00,0xb8,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xbe,0x02,0x00,0x00, -0x59,0x00,0x00,0x00,0xaf,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xbf,0x02,0x00,0x00,0x9e,0x00,0x00,0x00, -0xbe,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0xc4,0x02,0x00,0x00,0x12,0x00,0x00,0x00,0xc3,0x02,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xc5,0x02,0x00,0x00, -0xc4,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xc6,0x02,0x00,0x00,0x0f,0x00,0x00,0x00,0xc5,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xca,0x02,0x00,0x00, -0x47,0x00,0x00,0x00,0xc5,0x02,0x00,0x00,0x41,0x00,0x05,0x00, -0x0d,0x00,0x00,0x00,0xcc,0x02,0x00,0x00,0xcb,0x02,0x00,0x00, -0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xcd,0x02,0x00,0x00,0xcc,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xce,0x02,0x00,0x00,0xca,0x02,0x00,0x00, -0xcd,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xcf,0x02,0x00,0x00,0xc6,0x02,0x00,0x00,0xce,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0xd1,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0xd1,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0x3a,0x03,0x00,0x00,0x3e,0x00,0x00,0x00,0xcc,0x00,0x00,0x00, -0x37,0x03,0x00,0x00,0xd4,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb8,0x00,0x00,0x00,0xd7,0x02,0x00,0x00,0x3a,0x03,0x00,0x00, -0xb5,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xd3,0x02,0x00,0x00, -0xd4,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0xd7,0x02,0x00,0x00,0xd2,0x02,0x00,0x00,0xd3,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0xd2,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0xd9,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0xd9,0x02,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x3b,0x03,0x00,0x00, -0x3e,0x00,0x00,0x00,0xd2,0x02,0x00,0x00,0x35,0x03,0x00,0x00, -0xdc,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00, -0xdf,0x02,0x00,0x00,0x3b,0x03,0x00,0x00,0x60,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0xdb,0x02,0x00,0x00,0xdc,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xdf,0x02,0x00,0x00, -0xda,0x02,0x00,0x00,0xdb,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0xda,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xe3,0x02,0x00,0x00,0x3b,0x03,0x00,0x00,0x61,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xe4,0x02,0x00,0x00, -0xb9,0x02,0x00,0x00,0xe3,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xe6,0x02,0x00,0x00,0x64,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xe7,0x02,0x00,0x00,0xe4,0x02,0x00,0x00,0xe6,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xeb,0x02,0x00,0x00, -0x3a,0x03,0x00,0x00,0x57,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xec,0x02,0x00,0x00,0xbf,0x02,0x00,0x00, -0xeb,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xee,0x02,0x00,0x00,0x68,0x00,0x00,0x00,0xb2,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xef,0x02,0x00,0x00, -0xec,0x02,0x00,0x00,0xee,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0xf1,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0xf1,0x02,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x3d,0x03,0x00,0x00, -0x3e,0x00,0x00,0x00,0xda,0x02,0x00,0x00,0x33,0x03,0x00,0x00, -0xf4,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00, -0xf7,0x02,0x00,0x00,0x3d,0x03,0x00,0x00,0xb2,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0xf3,0x02,0x00,0x00,0xf4,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xf7,0x02,0x00,0x00, -0xf2,0x02,0x00,0x00,0xf3,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0xf2,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0xf9,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0xf9,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0x3f,0x03,0x00,0x00,0x3e,0x00,0x00,0x00, -0xf2,0x02,0x00,0x00,0x31,0x03,0x00,0x00,0xfc,0x02,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00,0xff,0x02,0x00,0x00, -0x3f,0x03,0x00,0x00,0x62,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0xfb,0x02,0x00,0x00,0xfc,0x02,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xff,0x02,0x00,0x00,0xfa,0x02,0x00,0x00, -0xfb,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0xfa,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x02,0x03,0x00,0x00, -0xe7,0x02,0x00,0x00,0x3f,0x03,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb8,0x00,0x00,0x00,0x05,0x03,0x00,0x00,0x02,0x03,0x00,0x00, -0x36,0x00,0x00,0x00,0xf7,0x00,0x03,0x00,0x07,0x03,0x00,0x00, -0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x05,0x03,0x00,0x00, -0x06,0x03,0x00,0x00,0x07,0x03,0x00,0x00,0xf8,0x00,0x02,0x00, -0x06,0x03,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x0a,0x03,0x00,0x00,0xef,0x02,0x00,0x00,0x3d,0x03,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x0b,0x03,0x00,0x00, -0x12,0x00,0x00,0x00,0xc6,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x0c,0x03,0x00,0x00,0x0b,0x03,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00,0x0d,0x03,0x00,0x00, -0x0a,0x03,0x00,0x00,0x0c,0x03,0x00,0x00,0xf9,0x00,0x02,0x00, -0x07,0x03,0x00,0x00,0xf8,0x00,0x02,0x00,0x07,0x03,0x00,0x00, -0xf5,0x00,0x07,0x00,0xb8,0x00,0x00,0x00,0x0e,0x03,0x00,0x00, -0x05,0x03,0x00,0x00,0xfa,0x02,0x00,0x00,0x0d,0x03,0x00,0x00, -0x06,0x03,0x00,0x00,0xf7,0x00,0x03,0x00,0x10,0x03,0x00,0x00, -0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x0e,0x03,0x00,0x00, -0x0f,0x03,0x00,0x00,0x10,0x03,0x00,0x00,0xf8,0x00,0x02,0x00, -0x0f,0x03,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x18,0x03,0x00,0x00,0xef,0x02,0x00,0x00,0x3d,0x03,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x1a,0x03,0x00,0x00, -0x12,0x00,0x00,0x00,0x19,0x03,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x1b,0x03,0x00,0x00,0x1a,0x03,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x1c,0x03,0x00,0x00, -0x18,0x03,0x00,0x00,0x1b,0x03,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x1d,0x03,0x00,0x00,0xcf,0x02,0x00,0x00, -0x1c,0x03,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x1f,0x03,0x00,0x00,0x1d,0x03,0x00,0x00,0xe7,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x21,0x03,0x00,0x00, -0x1f,0x03,0x00,0x00,0x3f,0x03,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x23,0x03,0x00,0x00,0x3a,0x03,0x00,0x00, -0xb2,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x25,0x03,0x00,0x00,0x23,0x03,0x00,0x00,0x3d,0x03,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x27,0x03,0x00,0x00, -0x25,0x03,0x00,0x00,0x26,0x03,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x29,0x03,0x00,0x00,0x3b,0x03,0x00,0x00, -0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x2a,0x03,0x00,0x00,0x27,0x03,0x00,0x00,0x29,0x03,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x2c,0x03,0x00,0x00, -0x2a,0x03,0x00,0x00,0x3f,0x03,0x00,0x00,0x41,0x00,0x05,0x00, -0xc3,0x00,0x00,0x00,0x2d,0x03,0x00,0x00,0xc0,0x00,0x00,0x00, -0x2c,0x03,0x00,0x00,0x3d,0x00,0x04,0x00,0xba,0x00,0x00,0x00, -0x2e,0x03,0x00,0x00,0x2d,0x03,0x00,0x00,0x41,0x00,0x06,0x00, -0xfc,0x00,0x00,0x00,0x2f,0x03,0x00,0x00,0x14,0x03,0x00,0x00, -0x34,0x00,0x00,0x00,0x21,0x03,0x00,0x00,0x3e,0x00,0x03,0x00, -0x2f,0x03,0x00,0x00,0x2e,0x03,0x00,0x00,0xf9,0x00,0x02,0x00, -0x10,0x03,0x00,0x00,0xf8,0x00,0x02,0x00,0x10,0x03,0x00,0x00, -0xf9,0x00,0x02,0x00,0xfc,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0xfc,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x31,0x03,0x00,0x00,0x3f,0x03,0x00,0x00,0xc6,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xf9,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0xfb,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0xf4,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0xf4,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x33,0x03,0x00,0x00,0x3d,0x03,0x00,0x00, -0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xf1,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0xf3,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0xdc,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0xdc,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x35,0x03,0x00,0x00, -0x3b,0x03,0x00,0x00,0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xd9,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0xdb,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0xd4,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0xd4,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x37,0x03,0x00,0x00,0x3a,0x03,0x00,0x00,0xc6,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xd1,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0xd3,0x02,0x00,0x00,0xfd,0x00,0x01,0x00,0x38,0x00,0x01,0x00, - -}; -const uint64_t matmul_f32_aligned_s_len = 12168; - -unsigned char matmul_f32_aligned_s_fp32_data[] = { -0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, -0xe9,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, -0x01,0x00,0x00,0x00,0x0b,0x00,0x06,0x00,0x01,0x00,0x00,0x00, -0x47,0x4c,0x53,0x4c,0x2e,0x73,0x74,0x64,0x2e,0x34,0x35,0x30, -0x00,0x00,0x00,0x00,0x0e,0x00,0x03,0x00,0x00,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x0f,0x00,0x0f,0x00,0x05,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x6d,0x61,0x69,0x6e,0x00,0x00,0x00,0x00, -0x0b,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x3d,0x00,0x00,0x00, -0x4c,0x00,0x00,0x00,0xea,0x00,0x00,0x00,0xf8,0x00,0x00,0x00, -0x46,0x01,0x00,0x00,0x53,0x01,0x00,0x00,0x4a,0x02,0x00,0x00, -0x93,0x02,0x00,0x00,0x10,0x00,0x06,0x00,0x04,0x00,0x00,0x00, -0x11,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x0b,0x00,0x00,0x00, -0x0b,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x10,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x10,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x10,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x05,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x10,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x07,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x1c,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x08,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x10,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x24,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x0a,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x28,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x2c,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x10,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x30,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x0d,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x34,0x00,0x00,0x00, -0x47,0x00,0x03,0x00,0x10,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x37,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x3d,0x00,0x00,0x00, -0x0b,0x00,0x00,0x00,0x1a,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x4c,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x1b,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x4f,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x53,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x60,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x62,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x07,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x6c,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x9d,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0xaf,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x05,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xb2,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0xf5,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x10,0x00,0x00,0x00, -0x48,0x00,0x04,0x00,0xf6,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0xf6,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x03,0x00,0xf6,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0xf8,0x00,0x00,0x00,0x22,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xf8,0x00,0x00,0x00, -0x21,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x27,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x28,0x01,0x00,0x00,0x0b,0x00,0x00,0x00, -0x19,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x50,0x01,0x00,0x00, -0x06,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x48,0x00,0x04,0x00, -0x51,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x51,0x01,0x00,0x00,0x00,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, -0x51,0x01,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x53,0x01,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x53,0x01,0x00,0x00,0x21,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x4a,0x02,0x00,0x00, -0x0b,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x90,0x02,0x00,0x00,0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x48,0x00,0x04,0x00,0x91,0x02,0x00,0x00,0x00,0x00,0x00,0x00, -0x19,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x91,0x02,0x00,0x00, -0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x03,0x00,0x91,0x02,0x00,0x00,0x02,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x93,0x02,0x00,0x00,0x22,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x93,0x02,0x00,0x00, -0x21,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x13,0x00,0x02,0x00, -0x02,0x00,0x00,0x00,0x21,0x00,0x03,0x00,0x03,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x15,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x17,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x03,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x0a,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00, -0x0b,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x0d,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x1e,0x00,0x10,0x00,0x10,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x11,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x10,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x11,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x15,0x00,0x04,0x00,0x13,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x08,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x15,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, -0x21,0x00,0x00,0x00,0x0a,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0x27,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x2d,0x00,0x00,0x00, -0x07,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, -0x34,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x32,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00, -0x3d,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x3e,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00,0x4c,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x4f,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x32,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x54,0x00,0x00,0x00, -0x86,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x53,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x58,0x00,0x00,0x00, -0x86,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x53,0x00,0x00,0x00, -0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x60,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x61,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x53,0x00,0x00,0x00, -0x60,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x63,0x00,0x00,0x00,0x86,0x00,0x00,0x00, -0x61,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x67,0x00,0x00,0x00,0x86,0x00,0x00,0x00, -0x61,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x32,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x10,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x6e,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x6d,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x73,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x6d,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, -0x77,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0x7c,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x87,0x00,0x00,0x00, -0x0b,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, -0x8d,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0x98,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x9d,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, -0x9f,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xae,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x32,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xaf,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xb0,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0xaf,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xb1,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x62,0x00,0x00,0x00, -0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xb2,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xb3,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0xb1,0x00,0x00,0x00, -0xb2,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xb4,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0xb3,0x00,0x00,0x00, -0x60,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xb5,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0xb0,0x00,0x00,0x00, -0xb4,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xb6,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0xae,0x00,0x00,0x00, -0xb5,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xb7,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0xb6,0x00,0x00,0x00, -0xb2,0x00,0x00,0x00,0x14,0x00,0x02,0x00,0xb8,0x00,0x00,0x00, -0x16,0x00,0x03,0x00,0xba,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xbb,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xbc,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0xbb,0x00,0x00,0x00,0xb5,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xbd,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0xbc,0x00,0x00,0x00,0xb2,0x00,0x00,0x00, -0x1c,0x00,0x04,0x00,0xbe,0x00,0x00,0x00,0xba,0x00,0x00,0x00, -0xbd,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0xbf,0x00,0x00,0x00, -0x07,0x00,0x00,0x00,0xbe,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0xba,0x00,0x00,0x00,0xc2,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0xc3,0x00,0x00,0x00,0x07,0x00,0x00,0x00, -0xba,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, -0xc6,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xe6,0x00,0x00,0x00,0x80,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, +0x61,0x01,0x00,0x00,0xc3,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, +0x62,0x01,0x00,0x00,0x61,0x01,0x00,0x00,0x20,0x00,0x04,0x00, +0x63,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x62,0x01,0x00,0x00, +0x3b,0x00,0x04,0x00,0x63,0x01,0x00,0x00,0x64,0x01,0x00,0x00, +0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x6f,0x01,0x00,0x00, +0x0c,0x00,0x00,0x00,0xc3,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x77,0x01,0x00,0x00,0x80,0x00,0x00,0x00, 0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xe7,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0x37,0x00,0x00,0x00,0xe6,0x00,0x00,0x00,0x1c,0x00,0x04,0x00, -0xe8,0x00,0x00,0x00,0xba,0x00,0x00,0x00,0xe7,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0xe9,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0xe8,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0xe9,0x00,0x00,0x00, -0xea,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xee,0x00,0x00,0x00,0x80,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x17,0x00,0x04,0x00, -0xf4,0x00,0x00,0x00,0xba,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x1d,0x00,0x03,0x00,0xf5,0x00,0x00,0x00,0xf4,0x00,0x00,0x00, -0x1e,0x00,0x03,0x00,0xf6,0x00,0x00,0x00,0xf5,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0xf7,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0xf6,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0xf7,0x00,0x00,0x00, -0xf8,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0xfa,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0xba,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0xfd,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0xba,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x02,0x01,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x0f,0x01,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x1c,0x01,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x21,0x01,0x00,0x00,0x03,0x00,0x00,0x00,0x32,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x27,0x01,0x00,0x00,0x01,0x00,0x00,0x00, -0x33,0x00,0x06,0x00,0x09,0x00,0x00,0x00,0x28,0x01,0x00,0x00, -0x27,0x01,0x00,0x00,0x39,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x29,0x01,0x00,0x00, -0x51,0x00,0x00,0x00,0x28,0x01,0x00,0x00,0x00,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x2a,0x01,0x00,0x00, -0x84,0x00,0x00,0x00,0x29,0x01,0x00,0x00,0x6d,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x2b,0x01,0x00,0x00, -0x86,0x00,0x00,0x00,0x2a,0x01,0x00,0x00,0x6c,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x42,0x01,0x00,0x00, -0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x43,0x01,0x00,0x00, -0x84,0x00,0x00,0x00,0x9d,0x00,0x00,0x00,0x42,0x01,0x00,0x00, -0x1c,0x00,0x04,0x00,0x44,0x01,0x00,0x00,0xba,0x00,0x00,0x00, -0x43,0x01,0x00,0x00,0x20,0x00,0x04,0x00,0x45,0x01,0x00,0x00, -0x04,0x00,0x00,0x00,0x44,0x01,0x00,0x00,0x3b,0x00,0x04,0x00, -0x45,0x01,0x00,0x00,0x46,0x01,0x00,0x00,0x04,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x4a,0x01,0x00,0x00, -0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x1d,0x00,0x03,0x00,0x50,0x01,0x00,0x00,0xf4,0x00,0x00,0x00, -0x1e,0x00,0x03,0x00,0x51,0x01,0x00,0x00,0x50,0x01,0x00,0x00, -0x20,0x00,0x04,0x00,0x52,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, -0x51,0x01,0x00,0x00,0x3b,0x00,0x04,0x00,0x52,0x01,0x00,0x00, -0x53,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x5b,0x01,0x00,0x00,0x80,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x68,0x01,0x00,0x00,0x80,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x75,0x01,0x00,0x00,0x80,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x7c,0x01,0x00,0x00,0x51,0x00,0x00,0x00, +0x34,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x7d,0x01,0x00,0x00,0x84,0x00,0x00,0x00, +0x7c,0x01,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x7e,0x01,0x00,0x00,0x86,0x00,0x00,0x00, +0x7d,0x01,0x00,0x00,0x6c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, 0x06,0x00,0x00,0x00,0x81,0x01,0x00,0x00,0x08,0x01,0x00,0x00, 0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x82,0x01,0x00,0x00, -0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, 0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x85,0x01,0x00,0x00, -0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, 0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xa0,0x01,0x00,0x00, 0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00, -0x1c,0x00,0x04,0x00,0xa1,0x01,0x00,0x00,0xba,0x00,0x00,0x00, +0x1c,0x00,0x04,0x00,0xa1,0x01,0x00,0x00,0xc3,0x00,0x00,0x00, 0xa0,0x01,0x00,0x00,0x20,0x00,0x04,0x00,0xa2,0x01,0x00,0x00, 0x07,0x00,0x00,0x00,0xa1,0x01,0x00,0x00,0x34,0x00,0x06,0x00, 0x06,0x00,0x00,0x00,0xb2,0x01,0x00,0x00,0x80,0x00,0x00,0x00, 0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, 0x06,0x00,0x00,0x00,0xcd,0x01,0x00,0x00,0x84,0x00,0x00,0x00, -0xb5,0x00,0x00,0x00,0xb2,0x00,0x00,0x00,0x1c,0x00,0x04,0x00, -0xce,0x01,0x00,0x00,0xba,0x00,0x00,0x00,0xcd,0x01,0x00,0x00, +0xbe,0x00,0x00,0x00,0xbb,0x00,0x00,0x00,0x1c,0x00,0x04,0x00, +0xce,0x01,0x00,0x00,0xc3,0x00,0x00,0x00,0xcd,0x01,0x00,0x00, 0x20,0x00,0x04,0x00,0xcf,0x01,0x00,0x00,0x07,0x00,0x00,0x00, 0xce,0x01,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xd8,0x01,0x00,0x00,0x86,0x00,0x00,0x00,0xaf,0x00,0x00,0x00, -0xb5,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xd8,0x01,0x00,0x00,0x86,0x00,0x00,0x00,0xb8,0x00,0x00,0x00, +0xbe,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, 0xe0,0x01,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, 0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, 0x0f,0x02,0x00,0x00,0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00, 0x62,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, 0x42,0x02,0x00,0x00,0x0d,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, 0x0a,0x00,0x00,0x00,0x4a,0x02,0x00,0x00,0x01,0x00,0x00,0x00, -0x1d,0x00,0x03,0x00,0x90,0x02,0x00,0x00,0xba,0x00,0x00,0x00, +0x1d,0x00,0x03,0x00,0x90,0x02,0x00,0x00,0xc3,0x00,0x00,0x00, 0x1e,0x00,0x03,0x00,0x91,0x02,0x00,0x00,0x90,0x02,0x00,0x00, 0x20,0x00,0x04,0x00,0x92,0x02,0x00,0x00,0x0c,0x00,0x00,0x00, 0x91,0x02,0x00,0x00,0x3b,0x00,0x04,0x00,0x92,0x02,0x00,0x00, @@ -37224,8 +37485,8 @@ unsigned char matmul_f32_aligned_s_fp32_data[] = { 0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00, 0x36,0x00,0x05,0x00,0x02,0x00,0x00,0x00,0x04,0x00,0x00,0x00, 0x00,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x05,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0xbf,0x00,0x00,0x00, -0xc0,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0x05,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0xc8,0x00,0x00,0x00, +0xc9,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, 0xa2,0x01,0x00,0x00,0xa3,0x01,0x00,0x00,0x07,0x00,0x00,0x00, 0x3b,0x00,0x04,0x00,0xcf,0x01,0x00,0x00,0xd0,0x01,0x00,0x00, 0x07,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00, @@ -37284,5951 +37545,1135 @@ unsigned char matmul_f32_aligned_s_fp32_data[] = { 0x64,0x00,0x00,0x00,0x5d,0x00,0x00,0x00,0x63,0x00,0x00,0x00, 0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x68,0x00,0x00,0x00, 0x5d,0x00,0x00,0x00,0x67,0x00,0x00,0x00,0x89,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x6f,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, -0x6e,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x74,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x73,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x78,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x77,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x79,0x00,0x00,0x00,0x78,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x7a,0x00,0x00,0x00, -0x47,0x00,0x00,0x00,0x79,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x7d,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x7c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x7e,0x00,0x00,0x00,0x7d,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x80,0x00,0x00,0x00,0x47,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x83,0x00,0x00,0x00,0x80,0x00,0x00,0x00,0x79,0x00,0x00,0x00, -0x0c,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x26,0x00,0x00,0x00,0x7e,0x00,0x00,0x00, -0x83,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0x88,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x87,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x89,0x00,0x00,0x00, -0x88,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x8a,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x89,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8c,0x00,0x00,0x00, -0x42,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x8e,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x8d,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x8f,0x00,0x00,0x00,0x8e,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x90,0x00,0x00,0x00,0x8c,0x00,0x00,0x00, -0x8f,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x91,0x00,0x00,0x00,0x8a,0x00,0x00,0x00,0x90,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x93,0x00,0x00,0x00, -0x91,0x00,0x00,0x00,0x7a,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x94,0x00,0x00,0x00,0x93,0x00,0x00,0x00, -0x6d,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0x99,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x98,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x9a,0x00,0x00,0x00, -0x99,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x9b,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x9a,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9e,0x00,0x00,0x00, -0x4a,0x00,0x00,0x00,0x9d,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0xa0,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x9f,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xa1,0x00,0x00,0x00,0xa0,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xa2,0x00,0x00,0x00,0x9e,0x00,0x00,0x00, -0xa1,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xa3,0x00,0x00,0x00,0x9b,0x00,0x00,0x00,0xa2,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa5,0x00,0x00,0x00, -0xa3,0x00,0x00,0x00,0x7a,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xa6,0x00,0x00,0x00,0xa5,0x00,0x00,0x00, -0x6d,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xa8,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xa8,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xb7,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, -0x05,0x00,0x00,0x00,0xc7,0x00,0x00,0x00,0xa9,0x00,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00,0xb9,0x00,0x00,0x00, -0xb7,0x02,0x00,0x00,0xb7,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0xaa,0x00,0x00,0x00,0xa9,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xb9,0x00,0x00,0x00,0xa9,0x00,0x00,0x00, -0xaa,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xa9,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0xc3,0x00,0x00,0x00,0xc4,0x00,0x00,0x00, -0xc0,0x00,0x00,0x00,0xb7,0x02,0x00,0x00,0x3e,0x00,0x03,0x00, -0xc4,0x00,0x00,0x00,0xc2,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xc7,0x00,0x00,0x00,0xb7,0x02,0x00,0x00, -0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xa8,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xaa,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xca,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xca,0x00,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xd0,0x02,0x00,0x00, -0xa6,0x00,0x00,0x00,0xaa,0x00,0x00,0x00,0x87,0x01,0x00,0x00, -0xcd,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xcc,0x02,0x00,0x00,0x94,0x00,0x00,0x00,0xaa,0x00,0x00,0x00, -0x84,0x01,0x00,0x00,0xcd,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xb8,0x02,0x00,0x00,0x7a,0x00,0x00,0x00, -0xaa,0x00,0x00,0x00,0x32,0x02,0x00,0x00,0xcd,0x00,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00,0xd1,0x00,0x00,0x00, -0xb8,0x02,0x00,0x00,0x84,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0xcc,0x00,0x00,0x00,0xcd,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xd1,0x00,0x00,0x00,0xcb,0x00,0x00,0x00, -0xcc,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xcb,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x6e,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, +0x6d,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x73,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x72,0x00,0x00,0x00, +0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x78,0x00,0x00,0x00, +0x4e,0x00,0x00,0x00,0x77,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x7d,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, +0x7c,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, +0x81,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x80,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x82,0x00,0x00,0x00, +0x81,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x83,0x00,0x00,0x00,0x47,0x00,0x00,0x00,0x82,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0x12,0x00,0x00,0x00,0x85,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x87,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x89,0x00,0x00,0x00, +0x47,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x8c,0x00,0x00,0x00,0x89,0x00,0x00,0x00, +0x82,0x00,0x00,0x00,0x0c,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0x8d,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x26,0x00,0x00,0x00, +0x87,0x00,0x00,0x00,0x8c,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x15,0x00,0x00,0x00,0x91,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0x90,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x92,0x00,0x00,0x00,0x91,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x93,0x00,0x00,0x00,0x32,0x00,0x00,0x00, +0x92,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x95,0x00,0x00,0x00,0x42,0x00,0x00,0x00,0x37,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x97,0x00,0x00,0x00, +0x12,0x00,0x00,0x00,0x96,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x98,0x00,0x00,0x00,0x97,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x99,0x00,0x00,0x00, +0x95,0x00,0x00,0x00,0x98,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x9a,0x00,0x00,0x00,0x93,0x00,0x00,0x00, +0x99,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x9c,0x00,0x00,0x00,0x9a,0x00,0x00,0x00,0x83,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9d,0x00,0x00,0x00, +0x9c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x15,0x00,0x00,0x00,0xa2,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0xa1,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xa3,0x00,0x00,0x00,0xa2,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xa4,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, +0xa3,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xa7,0x00,0x00,0x00,0x4a,0x00,0x00,0x00,0xa6,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0xa9,0x00,0x00,0x00, +0x12,0x00,0x00,0x00,0xa8,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xaa,0x00,0x00,0x00,0xa9,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xab,0x00,0x00,0x00, +0xa7,0x00,0x00,0x00,0xaa,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xac,0x00,0x00,0x00,0xa4,0x00,0x00,0x00, +0xab,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xae,0x00,0x00,0x00,0xac,0x00,0x00,0x00,0x83,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xaf,0x00,0x00,0x00, +0xae,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xb1,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xb1,0x00,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xb9,0x02,0x00,0x00, +0x3e,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0xd0,0x00,0x00,0x00, +0xb2,0x00,0x00,0x00,0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00, +0xc2,0x00,0x00,0x00,0xb9,0x02,0x00,0x00,0xc0,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0xb3,0x00,0x00,0x00,0xb2,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xc2,0x00,0x00,0x00, +0xb2,0x00,0x00,0x00,0xb3,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xb2,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0xcc,0x00,0x00,0x00, +0xcd,0x00,0x00,0x00,0xc9,0x00,0x00,0x00,0xb9,0x02,0x00,0x00, +0x3e,0x00,0x03,0x00,0xcd,0x00,0x00,0x00,0xcb,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xd0,0x00,0x00,0x00, +0xb9,0x02,0x00,0x00,0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xb1,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xb3,0x00,0x00,0x00, 0xf9,0x00,0x02,0x00,0xd3,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, 0xd3,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xc8,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0xcb,0x00,0x00,0x00, -0x2d,0x01,0x00,0x00,0xd4,0x00,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb8,0x00,0x00,0x00,0xd9,0x00,0x00,0x00,0xc8,0x02,0x00,0x00, -0x37,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xd5,0x00,0x00,0x00, -0xd4,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0xd9,0x00,0x00,0x00,0xd4,0x00,0x00,0x00,0xd5,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xd4,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xde,0x00,0x00,0x00,0x74,0x00,0x00,0x00, -0xc8,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xe1,0x00,0x00,0x00,0xde,0x00,0x00,0x00,0x8f,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xe2,0x00,0x00,0x00, -0xe1,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xe3,0x00,0x00,0x00,0xcc,0x02,0x00,0x00, -0xe2,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xe5,0x00,0x00,0x00,0xe3,0x00,0x00,0x00,0x6f,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xef,0x00,0x00,0x00, -0xde,0x00,0x00,0x00,0xee,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xf1,0x00,0x00,0x00,0x6f,0x00,0x00,0x00, -0x6d,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xf2,0x00,0x00,0x00,0xef,0x00,0x00,0x00,0xf1,0x00,0x00,0x00, -0x41,0x00,0x07,0x00,0xfa,0x00,0x00,0x00,0xfb,0x00,0x00,0x00, -0xf8,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0xe5,0x00,0x00,0x00, -0x3e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0xba,0x00,0x00,0x00, -0xfc,0x00,0x00,0x00,0xfb,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0xfd,0x00,0x00,0x00,0xfe,0x00,0x00,0x00,0xea,0x00,0x00,0x00, -0xf2,0x00,0x00,0x00,0x3e,0x00,0x03,0x00,0xfe,0x00,0x00,0x00, -0xfc,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x03,0x01,0x00,0x00,0xde,0x00,0x00,0x00,0x02,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x06,0x01,0x00,0x00, -0x03,0x01,0x00,0x00,0xf1,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x07,0x01,0x00,0x00,0x06,0x01,0x00,0x00, -0x39,0x00,0x00,0x00,0x41,0x00,0x07,0x00,0xfa,0x00,0x00,0x00, -0x09,0x01,0x00,0x00,0xf8,0x00,0x00,0x00,0x34,0x00,0x00,0x00, -0xe5,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0xba,0x00,0x00,0x00,0x0a,0x01,0x00,0x00,0x09,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0xfd,0x00,0x00,0x00,0x0b,0x01,0x00,0x00, -0xea,0x00,0x00,0x00,0x07,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0x0b,0x01,0x00,0x00,0x0a,0x01,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x10,0x01,0x00,0x00,0xde,0x00,0x00,0x00, -0x0f,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x13,0x01,0x00,0x00,0x10,0x01,0x00,0x00,0xf1,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x14,0x01,0x00,0x00, -0x13,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x41,0x00,0x07,0x00, -0xfa,0x00,0x00,0x00,0x16,0x01,0x00,0x00,0xf8,0x00,0x00,0x00, -0x34,0x00,0x00,0x00,0xe5,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0xba,0x00,0x00,0x00,0x17,0x01,0x00,0x00, -0x16,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0xfd,0x00,0x00,0x00, -0x18,0x01,0x00,0x00,0xea,0x00,0x00,0x00,0x14,0x01,0x00,0x00, -0x3e,0x00,0x03,0x00,0x18,0x01,0x00,0x00,0x17,0x01,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x1d,0x01,0x00,0x00, -0xde,0x00,0x00,0x00,0x1c,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x20,0x01,0x00,0x00,0x1d,0x01,0x00,0x00, -0xf1,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x22,0x01,0x00,0x00,0x20,0x01,0x00,0x00,0x21,0x01,0x00,0x00, -0x41,0x00,0x07,0x00,0xfa,0x00,0x00,0x00,0x24,0x01,0x00,0x00, -0xf8,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0xe5,0x00,0x00,0x00, -0x21,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0xba,0x00,0x00,0x00, -0x25,0x01,0x00,0x00,0x24,0x01,0x00,0x00,0x41,0x00,0x05,0x00, -0xfd,0x00,0x00,0x00,0x26,0x01,0x00,0x00,0xea,0x00,0x00,0x00, -0x22,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x26,0x01,0x00,0x00, -0x25,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x2d,0x01,0x00,0x00,0xc8,0x02,0x00,0x00,0x2b,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0xd3,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xd5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x2f,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x2f,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xc9,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, -0xd5,0x00,0x00,0x00,0x80,0x01,0x00,0x00,0x30,0x01,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00,0x35,0x01,0x00,0x00, -0xc9,0x02,0x00,0x00,0x9d,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0x31,0x01,0x00,0x00,0x30,0x01,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x35,0x01,0x00,0x00,0x30,0x01,0x00,0x00, -0x31,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x30,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3a,0x01,0x00,0x00, -0x74,0x00,0x00,0x00,0xc9,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x3d,0x01,0x00,0x00,0x3a,0x01,0x00,0x00, -0xa1,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x3e,0x01,0x00,0x00,0x3d,0x01,0x00,0x00,0x6d,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3f,0x01,0x00,0x00, -0xd0,0x02,0x00,0x00,0x3e,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x41,0x01,0x00,0x00,0x3f,0x01,0x00,0x00, -0x6f,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x4b,0x01,0x00,0x00,0x3a,0x01,0x00,0x00,0x4a,0x01,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x4d,0x01,0x00,0x00, -0x6f,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x4e,0x01,0x00,0x00,0x4b,0x01,0x00,0x00, -0x4d,0x01,0x00,0x00,0x41,0x00,0x07,0x00,0xfa,0x00,0x00,0x00, -0x55,0x01,0x00,0x00,0x53,0x01,0x00,0x00,0x34,0x00,0x00,0x00, -0x41,0x01,0x00,0x00,0x3e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0xba,0x00,0x00,0x00,0x56,0x01,0x00,0x00,0x55,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0xfd,0x00,0x00,0x00,0x57,0x01,0x00,0x00, -0x46,0x01,0x00,0x00,0x4e,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0x57,0x01,0x00,0x00,0x56,0x01,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x5c,0x01,0x00,0x00,0x3a,0x01,0x00,0x00, -0x5b,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x5f,0x01,0x00,0x00,0x5c,0x01,0x00,0x00,0x4d,0x01,0x00,0x00, +0xd2,0x02,0x00,0x00,0xaf,0x00,0x00,0x00,0xb3,0x00,0x00,0x00, +0x87,0x01,0x00,0x00,0xd6,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xce,0x02,0x00,0x00,0x9d,0x00,0x00,0x00, +0xb3,0x00,0x00,0x00,0x84,0x01,0x00,0x00,0xd6,0x00,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xba,0x02,0x00,0x00, +0x83,0x00,0x00,0x00,0xb3,0x00,0x00,0x00,0x32,0x02,0x00,0x00, +0xd6,0x00,0x00,0x00,0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00, +0xda,0x00,0x00,0x00,0xba,0x02,0x00,0x00,0x8d,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0xd5,0x00,0x00,0x00,0xd6,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xda,0x00,0x00,0x00, +0xd4,0x00,0x00,0x00,0xd5,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xd4,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xdc,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xdc,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xca,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, +0xd4,0x00,0x00,0x00,0x39,0x01,0x00,0x00,0xdd,0x00,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00,0xe2,0x00,0x00,0x00, +0xca,0x02,0x00,0x00,0x37,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0xde,0x00,0x00,0x00,0xdd,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0xe2,0x00,0x00,0x00,0xdd,0x00,0x00,0x00, +0xde,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xdd,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xe7,0x00,0x00,0x00, +0x73,0x00,0x00,0x00,0xca,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xea,0x00,0x00,0x00,0xe7,0x00,0x00,0x00, +0x98,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xeb,0x00,0x00,0x00,0xea,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xec,0x00,0x00,0x00, +0xce,0x02,0x00,0x00,0xeb,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xee,0x00,0x00,0x00,0xec,0x00,0x00,0x00, +0x6e,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xf4,0x00,0x00,0x00,0xe7,0x00,0x00,0x00,0xf3,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xf6,0x00,0x00,0x00, +0x6e,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xf7,0x00,0x00,0x00,0xf4,0x00,0x00,0x00, +0xf6,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xfb,0x00,0x00,0x00,0xee,0x00,0x00,0x00,0xfa,0x00,0x00,0x00, +0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xff,0x00,0x00,0x00, +0xee,0x00,0x00,0x00,0xfe,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0xff,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x41,0x00,0x07,0x00,0x0c,0x01,0x00,0x00, +0x0d,0x01,0x00,0x00,0x0a,0x01,0x00,0x00,0x34,0x00,0x00,0x00, +0xfb,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x02,0x01,0x00,0x00,0x0e,0x01,0x00,0x00,0x0d,0x01,0x00,0x00, +0x73,0x00,0x04,0x00,0xc3,0x00,0x00,0x00,0x0f,0x01,0x00,0x00, +0x0e,0x01,0x00,0x00,0x41,0x00,0x08,0x00,0x15,0x01,0x00,0x00, +0x16,0x01,0x00,0x00,0x0a,0x01,0x00,0x00,0x34,0x00,0x00,0x00, +0xfb,0x00,0x00,0x00,0xcf,0x00,0x00,0x00,0x00,0x01,0x00,0x00, +0x3d,0x00,0x04,0x00,0x03,0x01,0x00,0x00,0x17,0x01,0x00,0x00, +0x16,0x01,0x00,0x00,0x72,0x00,0x04,0x00,0x13,0x00,0x00,0x00, +0x18,0x01,0x00,0x00,0x17,0x01,0x00,0x00,0x6f,0x00,0x04,0x00, +0xc3,0x00,0x00,0x00,0x19,0x01,0x00,0x00,0x18,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x1c,0x01,0x00,0x00, +0x00,0x01,0x00,0x00,0x39,0x00,0x00,0x00,0x41,0x00,0x08,0x00, +0x15,0x01,0x00,0x00,0x1d,0x01,0x00,0x00,0x0a,0x01,0x00,0x00, +0x34,0x00,0x00,0x00,0xfb,0x00,0x00,0x00,0xcf,0x00,0x00,0x00, +0x1c,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0x03,0x01,0x00,0x00, +0x1e,0x01,0x00,0x00,0x1d,0x01,0x00,0x00,0x72,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0x1f,0x01,0x00,0x00,0x1e,0x01,0x00,0x00, +0x6f,0x00,0x04,0x00,0xc3,0x00,0x00,0x00,0x20,0x01,0x00,0x00, +0x1f,0x01,0x00,0x00,0x50,0x00,0x05,0x00,0x10,0x01,0x00,0x00, +0x21,0x01,0x00,0x00,0x19,0x01,0x00,0x00,0x20,0x01,0x00,0x00, +0x8e,0x00,0x05,0x00,0x10,0x01,0x00,0x00,0x23,0x01,0x00,0x00, +0x21,0x01,0x00,0x00,0x0f,0x01,0x00,0x00,0x51,0x00,0x05,0x00, +0xc3,0x00,0x00,0x00,0x2b,0x01,0x00,0x00,0x23,0x01,0x00,0x00, +0x00,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x2c,0x01,0x00,0x00, +0x2d,0x01,0x00,0x00,0x28,0x01,0x00,0x00,0xf7,0x00,0x00,0x00, +0x3e,0x00,0x03,0x00,0x2d,0x01,0x00,0x00,0x2b,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x2f,0x01,0x00,0x00, +0xf7,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x51,0x00,0x05,0x00, +0xc3,0x00,0x00,0x00,0x31,0x01,0x00,0x00,0x23,0x01,0x00,0x00, +0x01,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x2c,0x01,0x00,0x00, +0x32,0x01,0x00,0x00,0x28,0x01,0x00,0x00,0x2f,0x01,0x00,0x00, +0x3e,0x00,0x03,0x00,0x32,0x01,0x00,0x00,0x31,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x39,0x01,0x00,0x00, +0xca,0x02,0x00,0x00,0x37,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0xdc,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xde,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x3b,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x3b,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xcb,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0xde,0x00,0x00,0x00, +0x80,0x01,0x00,0x00,0x3e,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc1,0x00,0x00,0x00,0x41,0x01,0x00,0x00,0xcb,0x02,0x00,0x00, +0xa6,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x3d,0x01,0x00,0x00, +0x3e,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x41,0x01,0x00,0x00,0x3c,0x01,0x00,0x00,0x3d,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x3c,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x45,0x01,0x00,0x00,0xa7,0x00,0x00,0x00, +0x7d,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x47,0x01,0x00,0x00,0x45,0x01,0x00,0x00,0xcb,0x02,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x48,0x01,0x00,0x00, +0x12,0x00,0x00,0x00,0xcf,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x49,0x01,0x00,0x00,0x48,0x01,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00,0x4a,0x01,0x00,0x00, +0x47,0x01,0x00,0x00,0x49,0x01,0x00,0x00,0xf7,0x00,0x03,0x00, +0x4c,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x4a,0x01,0x00,0x00,0x4b,0x01,0x00,0x00,0x4c,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x4b,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x4f,0x01,0x00,0x00,0xba,0x02,0x00,0x00, +0x78,0x00,0x00,0x00,0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00, +0x51,0x01,0x00,0x00,0x4f,0x01,0x00,0x00,0x8d,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x4c,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x4c,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0xc1,0x00,0x00,0x00, +0x52,0x01,0x00,0x00,0x4a,0x01,0x00,0x00,0x3c,0x01,0x00,0x00, +0x51,0x01,0x00,0x00,0x4b,0x01,0x00,0x00,0xf7,0x00,0x03,0x00, +0x54,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x52,0x01,0x00,0x00,0x53,0x01,0x00,0x00,0x73,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x53,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x5c,0x01,0x00,0x00,0x7d,0x00,0x00,0x00, +0xcb,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x5e,0x01,0x00,0x00,0x5c,0x01,0x00,0x00,0x5d,0x01,0x00,0x00, 0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x60,0x01,0x00,0x00, -0x5f,0x01,0x00,0x00,0x39,0x00,0x00,0x00,0x41,0x00,0x07,0x00, -0xfa,0x00,0x00,0x00,0x62,0x01,0x00,0x00,0x53,0x01,0x00,0x00, -0x34,0x00,0x00,0x00,0x41,0x01,0x00,0x00,0x39,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0xba,0x00,0x00,0x00,0x63,0x01,0x00,0x00, -0x62,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0xfd,0x00,0x00,0x00, -0x64,0x01,0x00,0x00,0x46,0x01,0x00,0x00,0x60,0x01,0x00,0x00, -0x3e,0x00,0x03,0x00,0x64,0x01,0x00,0x00,0x63,0x01,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x69,0x01,0x00,0x00, -0x3a,0x01,0x00,0x00,0x68,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x6c,0x01,0x00,0x00,0x69,0x01,0x00,0x00, -0x4d,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x6d,0x01,0x00,0x00,0x6c,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, -0x41,0x00,0x07,0x00,0xfa,0x00,0x00,0x00,0x6f,0x01,0x00,0x00, -0x53,0x01,0x00,0x00,0x34,0x00,0x00,0x00,0x41,0x01,0x00,0x00, -0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0xba,0x00,0x00,0x00, -0x70,0x01,0x00,0x00,0x6f,0x01,0x00,0x00,0x41,0x00,0x05,0x00, -0xfd,0x00,0x00,0x00,0x71,0x01,0x00,0x00,0x46,0x01,0x00,0x00, -0x6d,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x71,0x01,0x00,0x00, -0x70,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x76,0x01,0x00,0x00,0x3a,0x01,0x00,0x00,0x75,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x79,0x01,0x00,0x00, -0x76,0x01,0x00,0x00,0x4d,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x7a,0x01,0x00,0x00,0x79,0x01,0x00,0x00, -0x21,0x01,0x00,0x00,0x41,0x00,0x07,0x00,0xfa,0x00,0x00,0x00, -0x7c,0x01,0x00,0x00,0x53,0x01,0x00,0x00,0x34,0x00,0x00,0x00, -0x41,0x01,0x00,0x00,0x21,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, -0xba,0x00,0x00,0x00,0x7d,0x01,0x00,0x00,0x7c,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0xfd,0x00,0x00,0x00,0x7e,0x01,0x00,0x00, -0x46,0x01,0x00,0x00,0x7a,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0x7e,0x01,0x00,0x00,0x7d,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x80,0x01,0x00,0x00,0xc9,0x02,0x00,0x00, -0x2b,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x2f,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x31,0x01,0x00,0x00,0xe0,0x00,0x04,0x00, -0x0c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x81,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x84,0x01,0x00,0x00, -0xcc,0x02,0x00,0x00,0x82,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x87,0x01,0x00,0x00,0xd0,0x02,0x00,0x00, -0x85,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x89,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x89,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xd2,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, -0x31,0x01,0x00,0x00,0x30,0x02,0x00,0x00,0x8c,0x01,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00,0x8f,0x01,0x00,0x00, -0xd2,0x02,0x00,0x00,0x6c,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0x8b,0x01,0x00,0x00,0x8c,0x01,0x00,0x00,0x00,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x8f,0x01,0x00,0x00,0x8a,0x01,0x00,0x00, -0x8b,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x8a,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0x91,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x91,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xd6,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0x8a,0x01,0x00,0x00, -0xbc,0x01,0x00,0x00,0x94,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb8,0x00,0x00,0x00,0x97,0x01,0x00,0x00,0xd6,0x02,0x00,0x00, -0x60,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x93,0x01,0x00,0x00, -0x94,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x97,0x01,0x00,0x00,0x92,0x01,0x00,0x00,0x93,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x92,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0x99,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x99,0x01,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xe8,0x02,0x00,0x00, -0x3e,0x00,0x00,0x00,0x92,0x01,0x00,0x00,0xba,0x01,0x00,0x00, -0x9a,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00, -0x9f,0x01,0x00,0x00,0xe8,0x02,0x00,0x00,0x62,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0x9b,0x01,0x00,0x00,0x9a,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x9f,0x01,0x00,0x00, -0x9a,0x01,0x00,0x00,0x9b,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x9a,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xa5,0x01,0x00,0x00,0xd6,0x02,0x00,0x00,0x62,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa7,0x01,0x00,0x00, -0xa5,0x01,0x00,0x00,0xe8,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xa9,0x01,0x00,0x00,0x55,0x00,0x00,0x00, -0x53,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xab,0x01,0x00,0x00,0xd6,0x02,0x00,0x00,0x61,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xac,0x01,0x00,0x00, -0xa9,0x01,0x00,0x00,0xab,0x01,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xae,0x01,0x00,0x00,0x64,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xaf,0x01,0x00,0x00,0xac,0x01,0x00,0x00,0xae,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xb1,0x01,0x00,0x00, -0xaf,0x01,0x00,0x00,0xe8,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xb3,0x01,0x00,0x00,0xb1,0x01,0x00,0x00, -0xb2,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xb5,0x01,0x00,0x00,0xb3,0x01,0x00,0x00,0xd2,0x02,0x00,0x00, -0x41,0x00,0x05,0x00,0xfd,0x00,0x00,0x00,0xb6,0x01,0x00,0x00, -0xea,0x00,0x00,0x00,0xb5,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, -0xba,0x00,0x00,0x00,0xb7,0x01,0x00,0x00,0xb6,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0xc3,0x00,0x00,0x00,0xb8,0x01,0x00,0x00, -0xa3,0x01,0x00,0x00,0xa7,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0xb8,0x01,0x00,0x00,0xb7,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xba,0x01,0x00,0x00,0xe8,0x02,0x00,0x00, -0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x99,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x9b,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0x94,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x94,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xbc,0x01,0x00,0x00, -0xd6,0x02,0x00,0x00,0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x91,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x93,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0xbe,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xbe,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xd7,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0x93,0x01,0x00,0x00, -0xea,0x01,0x00,0x00,0xc1,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb8,0x00,0x00,0x00,0xc4,0x01,0x00,0x00,0xd7,0x02,0x00,0x00, -0xb5,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xc0,0x01,0x00,0x00, -0xc1,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0xc4,0x01,0x00,0x00,0xbf,0x01,0x00,0x00,0xc0,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xbf,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0xc6,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xc6,0x01,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xe5,0x02,0x00,0x00, -0x3e,0x00,0x00,0x00,0xbf,0x01,0x00,0x00,0xe8,0x01,0x00,0x00, -0xc7,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00, -0xcc,0x01,0x00,0x00,0xe5,0x02,0x00,0x00,0xb2,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0xc8,0x01,0x00,0x00,0xc7,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xcc,0x01,0x00,0x00, -0xc7,0x01,0x00,0x00,0xc8,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xc7,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xd2,0x01,0x00,0x00,0xd7,0x02,0x00,0x00,0xb2,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xd4,0x01,0x00,0x00, -0xd2,0x01,0x00,0x00,0xe5,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xd6,0x01,0x00,0x00,0x59,0x00,0x00,0x00, -0xaf,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xd9,0x01,0x00,0x00,0xd7,0x02,0x00,0x00,0xd8,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xda,0x01,0x00,0x00, -0xd6,0x01,0x00,0x00,0xd9,0x01,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xdc,0x01,0x00,0x00,0x68,0x00,0x00,0x00, -0xb2,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xdd,0x01,0x00,0x00,0xda,0x01,0x00,0x00,0xdc,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xdf,0x01,0x00,0x00, -0xdd,0x01,0x00,0x00,0xe5,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xe1,0x01,0x00,0x00,0xdf,0x01,0x00,0x00, -0xe0,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xe3,0x01,0x00,0x00,0xe1,0x01,0x00,0x00,0xd2,0x02,0x00,0x00, -0x41,0x00,0x05,0x00,0xfd,0x00,0x00,0x00,0xe4,0x01,0x00,0x00, -0x46,0x01,0x00,0x00,0xe3,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, -0xba,0x00,0x00,0x00,0xe5,0x01,0x00,0x00,0xe4,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0xc3,0x00,0x00,0x00,0xe6,0x01,0x00,0x00, -0xd0,0x01,0x00,0x00,0xd4,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0xe6,0x01,0x00,0x00,0xe5,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xe8,0x01,0x00,0x00,0xe5,0x02,0x00,0x00, -0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xc6,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xc8,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0xc1,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xc1,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xea,0x01,0x00,0x00, -0xd7,0x02,0x00,0x00,0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xbe,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xc0,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0xec,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xec,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xd8,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0xc0,0x01,0x00,0x00, -0x2e,0x02,0x00,0x00,0xef,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb8,0x00,0x00,0x00,0xf2,0x01,0x00,0x00,0xd8,0x02,0x00,0x00, -0xb5,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xee,0x01,0x00,0x00, -0xef,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0xf2,0x01,0x00,0x00,0xed,0x01,0x00,0x00,0xee,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xed,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0xf4,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xf4,0x01,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xdc,0x02,0x00,0x00, -0x3e,0x00,0x00,0x00,0xed,0x01,0x00,0x00,0x2c,0x02,0x00,0x00, -0xf7,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00, -0xfa,0x01,0x00,0x00,0xdc,0x02,0x00,0x00,0x60,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0xf6,0x01,0x00,0x00,0xf7,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xfa,0x01,0x00,0x00, -0xf5,0x01,0x00,0x00,0xf6,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xf5,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xfc,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xfc,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xde,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, -0xf5,0x01,0x00,0x00,0x2a,0x02,0x00,0x00,0xff,0x01,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00,0x02,0x02,0x00,0x00, -0xde,0x02,0x00,0x00,0xb2,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0xfe,0x01,0x00,0x00,0xff,0x01,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x02,0x02,0x00,0x00,0xfd,0x01,0x00,0x00, -0xfe,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xfd,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0x04,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x04,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xe0,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0xfd,0x01,0x00,0x00, -0x28,0x02,0x00,0x00,0x05,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb8,0x00,0x00,0x00,0x0a,0x02,0x00,0x00,0xe0,0x02,0x00,0x00, -0x62,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x06,0x02,0x00,0x00, -0x05,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x0a,0x02,0x00,0x00,0x05,0x02,0x00,0x00,0x06,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x05,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x0c,0x02,0x00,0x00,0xd8,0x02,0x00,0x00, -0xb2,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x0e,0x02,0x00,0x00,0x0c,0x02,0x00,0x00,0xde,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x10,0x02,0x00,0x00, -0x0e,0x02,0x00,0x00,0x0f,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x12,0x02,0x00,0x00,0xdc,0x02,0x00,0x00, -0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x13,0x02,0x00,0x00,0x10,0x02,0x00,0x00,0x12,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x15,0x02,0x00,0x00, -0x13,0x02,0x00,0x00,0xe0,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x19,0x02,0x00,0x00,0x12,0x02,0x00,0x00, -0xe0,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0xc3,0x00,0x00,0x00, -0x1a,0x02,0x00,0x00,0xa3,0x01,0x00,0x00,0x19,0x02,0x00,0x00, -0x3d,0x00,0x04,0x00,0xba,0x00,0x00,0x00,0x1b,0x02,0x00,0x00, -0x1a,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0xc3,0x00,0x00,0x00, -0x20,0x02,0x00,0x00,0xd0,0x01,0x00,0x00,0x0e,0x02,0x00,0x00, -0x3d,0x00,0x04,0x00,0xba,0x00,0x00,0x00,0x21,0x02,0x00,0x00, -0x20,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0xc3,0x00,0x00,0x00, -0x23,0x02,0x00,0x00,0xc0,0x00,0x00,0x00,0x15,0x02,0x00,0x00, -0x3d,0x00,0x04,0x00,0xba,0x00,0x00,0x00,0x24,0x02,0x00,0x00, -0x23,0x02,0x00,0x00,0x0c,0x00,0x08,0x00,0xba,0x00,0x00,0x00, -0x25,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00, -0x1b,0x02,0x00,0x00,0x21,0x02,0x00,0x00,0x24,0x02,0x00,0x00, -0x3e,0x00,0x03,0x00,0x23,0x02,0x00,0x00,0x25,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x28,0x02,0x00,0x00, -0xe0,0x02,0x00,0x00,0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x04,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x06,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0xff,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xff,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x2a,0x02,0x00,0x00,0xde,0x02,0x00,0x00,0xc6,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xfc,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xfe,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xf7,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xf7,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x2c,0x02,0x00,0x00,0xdc,0x02,0x00,0x00, -0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xf4,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xf6,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0xef,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xef,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x2e,0x02,0x00,0x00, -0xd8,0x02,0x00,0x00,0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xec,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xee,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0x8c,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x8c,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x30,0x02,0x00,0x00,0xd2,0x02,0x00,0x00,0xc6,0x00,0x00,0x00, +0x5e,0x01,0x00,0x00,0x78,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x6b,0x01,0x00,0x00,0x5c,0x01,0x00,0x00, +0xaa,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x6c,0x01,0x00,0x00,0xd2,0x02,0x00,0x00,0x6b,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x6e,0x01,0x00,0x00, +0x6c,0x01,0x00,0x00,0x78,0x00,0x00,0x00,0x41,0x00,0x06,0x00, +0x6f,0x01,0x00,0x00,0x70,0x01,0x00,0x00,0x64,0x01,0x00,0x00, +0x34,0x00,0x00,0x00,0x6e,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, +0xc3,0x00,0x00,0x00,0x71,0x01,0x00,0x00,0x70,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0x2c,0x01,0x00,0x00,0x72,0x01,0x00,0x00, +0x59,0x01,0x00,0x00,0x60,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, +0x72,0x01,0x00,0x00,0x71,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0x54,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x73,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x76,0x01,0x00,0x00, +0x7d,0x00,0x00,0x00,0xcb,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x78,0x01,0x00,0x00,0x76,0x01,0x00,0x00, +0x77,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x7a,0x01,0x00,0x00,0x78,0x01,0x00,0x00,0x78,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x2c,0x01,0x00,0x00,0x7b,0x01,0x00,0x00, +0x59,0x01,0x00,0x00,0x7a,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, +0x7b,0x01,0x00,0x00,0xcb,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x54,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x54,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0x3e,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x3e,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x80,0x01,0x00,0x00,0xcb,0x02,0x00,0x00,0x7e,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0x3b,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x3d,0x01,0x00,0x00,0xe0,0x00,0x04,0x00,0x0c,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x81,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x84,0x01,0x00,0x00,0xce,0x02,0x00,0x00, +0x82,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x87,0x01,0x00,0x00,0xd2,0x02,0x00,0x00,0x85,0x01,0x00,0x00, 0xf9,0x00,0x02,0x00,0x89,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x8b,0x01,0x00,0x00,0xe0,0x00,0x04,0x00,0x0c,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x81,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0xcd,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xcd,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x32,0x02,0x00,0x00, -0xb8,0x02,0x00,0x00,0x6c,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xca,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xcc,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x37,0x02,0x00,0x00, -0x55,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x38,0x02,0x00,0x00,0x8c,0x00,0x00,0x00, -0x37,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x3d,0x02,0x00,0x00,0x59,0x00,0x00,0x00,0xaf,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3e,0x02,0x00,0x00, -0x9e,0x00,0x00,0x00,0x3d,0x02,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x43,0x02,0x00,0x00,0x12,0x00,0x00,0x00, -0x42,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x44,0x02,0x00,0x00,0x43,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x45,0x02,0x00,0x00,0x0f,0x00,0x00,0x00, -0x44,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x49,0x02,0x00,0x00,0x47,0x00,0x00,0x00,0x44,0x02,0x00,0x00, -0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00,0x4b,0x02,0x00,0x00, -0x4a,0x02,0x00,0x00,0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x4c,0x02,0x00,0x00,0x4b,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x4d,0x02,0x00,0x00, -0x49,0x02,0x00,0x00,0x4c,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x4e,0x02,0x00,0x00,0x45,0x02,0x00,0x00, -0x4d,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x50,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x50,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xb9,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, -0xcc,0x00,0x00,0x00,0xb6,0x02,0x00,0x00,0x53,0x02,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00,0x56,0x02,0x00,0x00, -0xb9,0x02,0x00,0x00,0xb5,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0x52,0x02,0x00,0x00,0x53,0x02,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x56,0x02,0x00,0x00,0x51,0x02,0x00,0x00, -0x52,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x51,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0x58,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x58,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xba,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0x51,0x02,0x00,0x00, -0xb4,0x02,0x00,0x00,0x5b,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb8,0x00,0x00,0x00,0x5e,0x02,0x00,0x00,0xba,0x02,0x00,0x00, -0x60,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x5a,0x02,0x00,0x00, -0x5b,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x5e,0x02,0x00,0x00,0x59,0x02,0x00,0x00,0x5a,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x59,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x62,0x02,0x00,0x00,0xba,0x02,0x00,0x00, -0x61,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x63,0x02,0x00,0x00,0x38,0x02,0x00,0x00,0x62,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x65,0x02,0x00,0x00, -0x64,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x66,0x02,0x00,0x00,0x63,0x02,0x00,0x00, -0x65,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x6a,0x02,0x00,0x00,0xb9,0x02,0x00,0x00,0xd8,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x6b,0x02,0x00,0x00, -0x3e,0x02,0x00,0x00,0x6a,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x6d,0x02,0x00,0x00,0x68,0x00,0x00,0x00, -0xb2,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x6e,0x02,0x00,0x00,0x6b,0x02,0x00,0x00,0x6d,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0x70,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x70,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xbc,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0x59,0x02,0x00,0x00, -0xb2,0x02,0x00,0x00,0x73,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb8,0x00,0x00,0x00,0x76,0x02,0x00,0x00,0xbc,0x02,0x00,0x00, -0xb2,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x72,0x02,0x00,0x00, -0x73,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x76,0x02,0x00,0x00,0x71,0x02,0x00,0x00,0x72,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x71,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x78,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x78,0x02,0x00,0x00, +0x89,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xd4,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0x3d,0x01,0x00,0x00, +0x30,0x02,0x00,0x00,0x8c,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc1,0x00,0x00,0x00,0x8f,0x01,0x00,0x00,0xd4,0x02,0x00,0x00, +0x6c,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x8b,0x01,0x00,0x00, +0x8c,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x8f,0x01,0x00,0x00,0x8a,0x01,0x00,0x00,0x8b,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x8a,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0x91,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x91,0x01,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xd8,0x02,0x00,0x00, +0x3e,0x00,0x00,0x00,0x8a,0x01,0x00,0x00,0xbc,0x01,0x00,0x00, +0x94,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00, +0x97,0x01,0x00,0x00,0xd8,0x02,0x00,0x00,0x60,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0x93,0x01,0x00,0x00,0x94,0x01,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x97,0x01,0x00,0x00, +0x92,0x01,0x00,0x00,0x93,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x92,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x99,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x99,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xea,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, +0x92,0x01,0x00,0x00,0xba,0x01,0x00,0x00,0x9a,0x01,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00,0x9f,0x01,0x00,0x00, +0xea,0x02,0x00,0x00,0x62,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0x9b,0x01,0x00,0x00,0x9a,0x01,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x9f,0x01,0x00,0x00,0x9a,0x01,0x00,0x00, +0x9b,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x9a,0x01,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa5,0x01,0x00,0x00, +0xd8,0x02,0x00,0x00,0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xa7,0x01,0x00,0x00,0xa5,0x01,0x00,0x00, +0xea,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xa9,0x01,0x00,0x00,0x55,0x00,0x00,0x00,0x53,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xab,0x01,0x00,0x00, +0xd8,0x02,0x00,0x00,0x61,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xac,0x01,0x00,0x00,0xa9,0x01,0x00,0x00, +0xab,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xae,0x01,0x00,0x00,0x64,0x00,0x00,0x00,0x62,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xaf,0x01,0x00,0x00, +0xac,0x01,0x00,0x00,0xae,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xb1,0x01,0x00,0x00,0xaf,0x01,0x00,0x00, +0xea,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xb3,0x01,0x00,0x00,0xb1,0x01,0x00,0x00,0xb2,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xb5,0x01,0x00,0x00, +0xb3,0x01,0x00,0x00,0xd4,0x02,0x00,0x00,0x41,0x00,0x05,0x00, +0x2c,0x01,0x00,0x00,0xb6,0x01,0x00,0x00,0x28,0x01,0x00,0x00, +0xb5,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0xc3,0x00,0x00,0x00, +0xb7,0x01,0x00,0x00,0xb6,0x01,0x00,0x00,0x41,0x00,0x05,0x00, +0xcc,0x00,0x00,0x00,0xb8,0x01,0x00,0x00,0xa3,0x01,0x00,0x00, +0xa7,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0xb8,0x01,0x00,0x00, +0xb7,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xba,0x01,0x00,0x00,0xea,0x02,0x00,0x00,0xcf,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x99,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x9b,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x94,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x94,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xbc,0x01,0x00,0x00,0xd8,0x02,0x00,0x00, +0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x91,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x93,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0xbe,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xbe,0x01,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xd9,0x02,0x00,0x00, +0x3e,0x00,0x00,0x00,0x93,0x01,0x00,0x00,0xea,0x01,0x00,0x00, +0xc1,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00, +0xc4,0x01,0x00,0x00,0xd9,0x02,0x00,0x00,0xbe,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0xc0,0x01,0x00,0x00,0xc1,0x01,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xc4,0x01,0x00,0x00, +0xbf,0x01,0x00,0x00,0xc0,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xbf,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xc6,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xc6,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xe7,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, +0xbf,0x01,0x00,0x00,0xe8,0x01,0x00,0x00,0xc7,0x01,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00,0xcc,0x01,0x00,0x00, +0xe7,0x02,0x00,0x00,0xbb,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0xc8,0x01,0x00,0x00,0xc7,0x01,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0xcc,0x01,0x00,0x00,0xc7,0x01,0x00,0x00, +0xc8,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xc7,0x01,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xd2,0x01,0x00,0x00, +0xd9,0x02,0x00,0x00,0xbb,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xd4,0x01,0x00,0x00,0xd2,0x01,0x00,0x00, +0xe7,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xd6,0x01,0x00,0x00,0x59,0x00,0x00,0x00,0xb8,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xd9,0x01,0x00,0x00, +0xd9,0x02,0x00,0x00,0xd8,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xda,0x01,0x00,0x00,0xd6,0x01,0x00,0x00, +0xd9,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xdc,0x01,0x00,0x00,0x68,0x00,0x00,0x00,0xbb,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xdd,0x01,0x00,0x00, +0xda,0x01,0x00,0x00,0xdc,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xdf,0x01,0x00,0x00,0xdd,0x01,0x00,0x00, +0xe7,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xe1,0x01,0x00,0x00,0xdf,0x01,0x00,0x00,0xe0,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xe3,0x01,0x00,0x00, +0xe1,0x01,0x00,0x00,0xd4,0x02,0x00,0x00,0x41,0x00,0x05,0x00, +0x2c,0x01,0x00,0x00,0xe4,0x01,0x00,0x00,0x59,0x01,0x00,0x00, +0xe3,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0xc3,0x00,0x00,0x00, +0xe5,0x01,0x00,0x00,0xe4,0x01,0x00,0x00,0x41,0x00,0x05,0x00, +0xcc,0x00,0x00,0x00,0xe6,0x01,0x00,0x00,0xd0,0x01,0x00,0x00, +0xd4,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0xe6,0x01,0x00,0x00, +0xe5,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xe8,0x01,0x00,0x00,0xe7,0x02,0x00,0x00,0xcf,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xc6,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xc8,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xc1,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xc1,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xea,0x01,0x00,0x00,0xd9,0x02,0x00,0x00, +0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xbe,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xc0,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0xec,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xec,0x01,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xda,0x02,0x00,0x00, +0x3e,0x00,0x00,0x00,0xc0,0x01,0x00,0x00,0x2e,0x02,0x00,0x00, +0xef,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00, +0xf2,0x01,0x00,0x00,0xda,0x02,0x00,0x00,0xbe,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0xee,0x01,0x00,0x00,0xef,0x01,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xf2,0x01,0x00,0x00, +0xed,0x01,0x00,0x00,0xee,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xed,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xf4,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xf4,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xde,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, +0xed,0x01,0x00,0x00,0x2c,0x02,0x00,0x00,0xf7,0x01,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00,0xfa,0x01,0x00,0x00, +0xde,0x02,0x00,0x00,0x60,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0xf6,0x01,0x00,0x00,0xf7,0x01,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0xfa,0x01,0x00,0x00,0xf5,0x01,0x00,0x00, +0xf6,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xf5,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0xfc,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xfc,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xe0,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0xf5,0x01,0x00,0x00, +0x2a,0x02,0x00,0x00,0xff,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc1,0x00,0x00,0x00,0x02,0x02,0x00,0x00,0xe0,0x02,0x00,0x00, +0xbb,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xfe,0x01,0x00,0x00, +0xff,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x02,0x02,0x00,0x00,0xfd,0x01,0x00,0x00,0xfe,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xfd,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0x04,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x04,0x02,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xe2,0x02,0x00,0x00, +0x3e,0x00,0x00,0x00,0xfd,0x01,0x00,0x00,0x28,0x02,0x00,0x00, +0x05,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00, +0x0a,0x02,0x00,0x00,0xe2,0x02,0x00,0x00,0x62,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0x06,0x02,0x00,0x00,0x05,0x02,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x0a,0x02,0x00,0x00, +0x05,0x02,0x00,0x00,0x06,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x05,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x0c,0x02,0x00,0x00,0xda,0x02,0x00,0x00,0xbb,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x0e,0x02,0x00,0x00, +0x0c,0x02,0x00,0x00,0xe0,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x10,0x02,0x00,0x00,0x0e,0x02,0x00,0x00, +0x0f,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x12,0x02,0x00,0x00,0xde,0x02,0x00,0x00,0x62,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x13,0x02,0x00,0x00, +0x10,0x02,0x00,0x00,0x12,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x15,0x02,0x00,0x00,0x13,0x02,0x00,0x00, +0xe2,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x19,0x02,0x00,0x00,0x12,0x02,0x00,0x00,0xe2,0x02,0x00,0x00, +0x41,0x00,0x05,0x00,0xcc,0x00,0x00,0x00,0x1a,0x02,0x00,0x00, +0xa3,0x01,0x00,0x00,0x19,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, +0xc3,0x00,0x00,0x00,0x1b,0x02,0x00,0x00,0x1a,0x02,0x00,0x00, +0x41,0x00,0x05,0x00,0xcc,0x00,0x00,0x00,0x20,0x02,0x00,0x00, +0xd0,0x01,0x00,0x00,0x0e,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, +0xc3,0x00,0x00,0x00,0x21,0x02,0x00,0x00,0x20,0x02,0x00,0x00, +0x41,0x00,0x05,0x00,0xcc,0x00,0x00,0x00,0x23,0x02,0x00,0x00, +0xc9,0x00,0x00,0x00,0x15,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, +0xc3,0x00,0x00,0x00,0x24,0x02,0x00,0x00,0x23,0x02,0x00,0x00, +0x0c,0x00,0x08,0x00,0xc3,0x00,0x00,0x00,0x25,0x02,0x00,0x00, +0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x1b,0x02,0x00,0x00, +0x21,0x02,0x00,0x00,0x24,0x02,0x00,0x00,0x3e,0x00,0x03,0x00, +0x23,0x02,0x00,0x00,0x25,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x28,0x02,0x00,0x00,0xe2,0x02,0x00,0x00, +0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x04,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x06,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0xff,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xff,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x2a,0x02,0x00,0x00, +0xe0,0x02,0x00,0x00,0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xfc,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xfe,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0xf7,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xf7,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x2c,0x02,0x00,0x00,0xde,0x02,0x00,0x00,0xcf,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xf4,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xf6,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xef,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xef,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x2e,0x02,0x00,0x00,0xda,0x02,0x00,0x00, +0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xec,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xee,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0x8c,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x8c,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x30,0x02,0x00,0x00, +0xd4,0x02,0x00,0x00,0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x89,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x8b,0x01,0x00,0x00, +0xe0,0x00,0x04,0x00,0x0c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x81,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xd6,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xd6,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x32,0x02,0x00,0x00,0xba,0x02,0x00,0x00, +0x6c,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xd3,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xd5,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x37,0x02,0x00,0x00,0x55,0x00,0x00,0x00, +0x53,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x38,0x02,0x00,0x00,0x95,0x00,0x00,0x00,0x37,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3d,0x02,0x00,0x00, +0x59,0x00,0x00,0x00,0xb8,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x3e,0x02,0x00,0x00,0xa7,0x00,0x00,0x00, +0x3d,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, +0x43,0x02,0x00,0x00,0x12,0x00,0x00,0x00,0x42,0x02,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x44,0x02,0x00,0x00, +0x43,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x45,0x02,0x00,0x00,0x0f,0x00,0x00,0x00,0x44,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x49,0x02,0x00,0x00, +0x47,0x00,0x00,0x00,0x44,0x02,0x00,0x00,0x41,0x00,0x05,0x00, +0x0d,0x00,0x00,0x00,0x4b,0x02,0x00,0x00,0x4a,0x02,0x00,0x00, +0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x4c,0x02,0x00,0x00,0x4b,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x4d,0x02,0x00,0x00,0x49,0x02,0x00,0x00, +0x4c,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x4e,0x02,0x00,0x00,0x45,0x02,0x00,0x00,0x4d,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0x50,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x50,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xbb,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0xd5,0x00,0x00,0x00, +0xb6,0x02,0x00,0x00,0x53,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc1,0x00,0x00,0x00,0x56,0x02,0x00,0x00,0xbb,0x02,0x00,0x00, +0xbe,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x52,0x02,0x00,0x00, +0x53,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x56,0x02,0x00,0x00,0x51,0x02,0x00,0x00,0x52,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x51,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0x58,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x58,0x02,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xbc,0x02,0x00,0x00, +0x3e,0x00,0x00,0x00,0x51,0x02,0x00,0x00,0xb4,0x02,0x00,0x00, +0x5b,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00, +0x5e,0x02,0x00,0x00,0xbc,0x02,0x00,0x00,0x60,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0x5a,0x02,0x00,0x00,0x5b,0x02,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x5e,0x02,0x00,0x00, +0x59,0x02,0x00,0x00,0x5a,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x59,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x62,0x02,0x00,0x00,0xbc,0x02,0x00,0x00,0x61,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x63,0x02,0x00,0x00, +0x38,0x02,0x00,0x00,0x62,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x65,0x02,0x00,0x00,0x64,0x00,0x00,0x00, +0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x66,0x02,0x00,0x00,0x63,0x02,0x00,0x00,0x65,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x6a,0x02,0x00,0x00, +0xbb,0x02,0x00,0x00,0xd8,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x6b,0x02,0x00,0x00,0x3e,0x02,0x00,0x00, +0x6a,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x6d,0x02,0x00,0x00,0x68,0x00,0x00,0x00,0xbb,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x6e,0x02,0x00,0x00, +0x6b,0x02,0x00,0x00,0x6d,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0x70,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x70,0x02,0x00,0x00, 0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xbe,0x02,0x00,0x00, -0x3e,0x00,0x00,0x00,0x71,0x02,0x00,0x00,0xb0,0x02,0x00,0x00, -0x7b,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00, -0x7e,0x02,0x00,0x00,0xbe,0x02,0x00,0x00,0x62,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0x7a,0x02,0x00,0x00,0x7b,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x7e,0x02,0x00,0x00, -0x79,0x02,0x00,0x00,0x7a,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x79,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x81,0x02,0x00,0x00,0x66,0x02,0x00,0x00,0xbe,0x02,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00,0x84,0x02,0x00,0x00, -0x81,0x02,0x00,0x00,0x36,0x00,0x00,0x00,0xf7,0x00,0x03,0x00, -0x86,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x84,0x02,0x00,0x00,0x85,0x02,0x00,0x00,0x86,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x85,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x89,0x02,0x00,0x00,0x6e,0x02,0x00,0x00, -0xbc,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0x8a,0x02,0x00,0x00,0x12,0x00,0x00,0x00,0xc6,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x8b,0x02,0x00,0x00, -0x8a,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xb8,0x00,0x00,0x00, -0x8c,0x02,0x00,0x00,0x89,0x02,0x00,0x00,0x8b,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0x86,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x86,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0xb8,0x00,0x00,0x00, -0x8d,0x02,0x00,0x00,0x84,0x02,0x00,0x00,0x79,0x02,0x00,0x00, -0x8c,0x02,0x00,0x00,0x85,0x02,0x00,0x00,0xf7,0x00,0x03,0x00, -0x8f,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x8d,0x02,0x00,0x00,0x8e,0x02,0x00,0x00,0x8f,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x8e,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x97,0x02,0x00,0x00,0x6e,0x02,0x00,0x00, -0xbc,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0x99,0x02,0x00,0x00,0x12,0x00,0x00,0x00,0x98,0x02,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x9a,0x02,0x00,0x00, -0x99,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x9b,0x02,0x00,0x00,0x97,0x02,0x00,0x00,0x9a,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9c,0x02,0x00,0x00, -0x4e,0x02,0x00,0x00,0x9b,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x9e,0x02,0x00,0x00,0x9c,0x02,0x00,0x00, -0x66,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xa0,0x02,0x00,0x00,0x9e,0x02,0x00,0x00,0xbe,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa2,0x02,0x00,0x00, -0xb9,0x02,0x00,0x00,0xb2,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xa4,0x02,0x00,0x00,0xa2,0x02,0x00,0x00, -0xbc,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xa6,0x02,0x00,0x00,0xa4,0x02,0x00,0x00,0xa5,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa8,0x02,0x00,0x00, -0xba,0x02,0x00,0x00,0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xa9,0x02,0x00,0x00,0xa6,0x02,0x00,0x00, -0xa8,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xab,0x02,0x00,0x00,0xa9,0x02,0x00,0x00,0xbe,0x02,0x00,0x00, -0x41,0x00,0x05,0x00,0xc3,0x00,0x00,0x00,0xac,0x02,0x00,0x00, -0xc0,0x00,0x00,0x00,0xab,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, -0xba,0x00,0x00,0x00,0xad,0x02,0x00,0x00,0xac,0x02,0x00,0x00, -0x41,0x00,0x06,0x00,0xfa,0x00,0x00,0x00,0xae,0x02,0x00,0x00, -0x93,0x02,0x00,0x00,0x34,0x00,0x00,0x00,0xa0,0x02,0x00,0x00, -0x3e,0x00,0x03,0x00,0xae,0x02,0x00,0x00,0xad,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0x8f,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x8f,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x7b,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x7b,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xb0,0x02,0x00,0x00,0xbe,0x02,0x00,0x00, -0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x78,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x7a,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x73,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x73,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xb2,0x02,0x00,0x00, -0xbc,0x02,0x00,0x00,0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x70,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x72,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0x5b,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x5b,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xb4,0x02,0x00,0x00,0xba,0x02,0x00,0x00,0xc6,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x58,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x5a,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x53,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x53,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xb6,0x02,0x00,0x00,0xb9,0x02,0x00,0x00, -0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x50,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x52,0x02,0x00,0x00,0xfd,0x00,0x01,0x00, -0x38,0x00,0x01,0x00, -}; -const uint64_t matmul_f32_aligned_s_fp32_len = 10348; - -unsigned char matmul_f32_l_data[] = { -0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, -0xcd,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, -0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x09,0x00,0x00,0x00, -0x0b,0x00,0x06,0x00,0x01,0x00,0x00,0x00,0x47,0x4c,0x53,0x4c, -0x2e,0x73,0x74,0x64,0x2e,0x34,0x35,0x30,0x00,0x00,0x00,0x00, -0x0e,0x00,0x03,0x00,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x0f,0x00,0x0f,0x00,0x05,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x6d,0x61,0x69,0x6e,0x00,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x3d,0x00,0x00,0x00,0x4c,0x00,0x00,0x00, -0xf1,0x00,0x00,0x00,0xfc,0x00,0x00,0x00,0x3d,0x01,0x00,0x00, -0x48,0x01,0x00,0x00,0x2e,0x02,0x00,0x00,0x77,0x02,0x00,0x00, -0x10,0x00,0x06,0x00,0x04,0x00,0x00,0x00,0x11,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x0b,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x1c,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x10,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x08,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x03,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x10,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x14,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x18,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x07,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x10,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x24,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x0a,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x10,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x2c,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x30,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x0d,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0x47,0x00,0x03,0x00, -0x10,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x37,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x3d,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x1a,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x4c,0x00,0x00,0x00, -0x0b,0x00,0x00,0x00,0x1b,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x4f,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x53,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x60,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x62,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x07,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x6c,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x03,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x9c,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0xae,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x05,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0xb1,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x08,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xf9,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00, -0xfa,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0xfa,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, -0xfa,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0xfc,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0xfc,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x17,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x18,0x01,0x00,0x00,0x0b,0x00,0x00,0x00,0x19,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x45,0x01,0x00,0x00,0x06,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0x46,0x01,0x00,0x00, -0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x46,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x46,0x01,0x00,0x00, -0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x48,0x01,0x00,0x00, -0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x48,0x01,0x00,0x00,0x21,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x2e,0x02,0x00,0x00,0x0b,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x74,0x02,0x00,0x00, -0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00, -0x75,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x19,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x75,0x02,0x00,0x00,0x00,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, -0x75,0x02,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x77,0x02,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x77,0x02,0x00,0x00,0x21,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x13,0x00,0x02,0x00,0x02,0x00,0x00,0x00, -0x21,0x00,0x03,0x00,0x03,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x15,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x17,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x0a,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x0d,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x1e,0x00,0x10,0x00,0x10,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x11,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x11,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x15,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, -0x14,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x15,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x0a,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, -0x27,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0x2d,0x00,0x00,0x00,0x07,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x34,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x37,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00,0x3d,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x3e,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x0a,0x00,0x00,0x00,0x4c,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x4f,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x53,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x54,0x00,0x00,0x00,0x86,0x00,0x00,0x00, -0x37,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x58,0x00,0x00,0x00,0x86,0x00,0x00,0x00, -0x37,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x32,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x61,0x00,0x00,0x00, -0x86,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x60,0x00,0x00,0x00, -0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x62,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x63,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x61,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x67,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x61,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x86,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x72,0x00,0x00,0x00,0x86,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0x76,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x7b,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, -0x86,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0x8c,0x00,0x00,0x00,0x03,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x97,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x9c,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0x9e,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xad,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00, -0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xae,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xaf,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x53,0x00,0x00,0x00, -0xae,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xb0,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x4f,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xb1,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xb2,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0xb0,0x00,0x00,0x00,0xb1,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xb3,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0xb2,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xb4,0x00,0x00,0x00,0x86,0x00,0x00,0x00, -0xaf,0x00,0x00,0x00,0xb3,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xb5,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0xad,0x00,0x00,0x00,0xb4,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xb6,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0xb5,0x00,0x00,0x00,0xb1,0x00,0x00,0x00,0x14,0x00,0x02,0x00, -0xb7,0x00,0x00,0x00,0x16,0x00,0x03,0x00,0xb9,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xba,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xbb,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0xba,0x00,0x00,0x00, -0xb4,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xbc,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0xbb,0x00,0x00,0x00, -0xb1,0x00,0x00,0x00,0x1c,0x00,0x04,0x00,0xbd,0x00,0x00,0x00, -0xb9,0x00,0x00,0x00,0xbc,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0xbe,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0xbd,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0xb9,0x00,0x00,0x00,0xc1,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0xc2,0x00,0x00,0x00, -0x07,0x00,0x00,0x00,0xb9,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0xc5,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x16,0x00,0x03,0x00,0xec,0x00,0x00,0x00,0x10,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xed,0x00,0x00,0x00, -0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xee,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0xed,0x00,0x00,0x00, -0x1c,0x00,0x04,0x00,0xef,0x00,0x00,0x00,0xec,0x00,0x00,0x00, -0xee,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0xf0,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0xef,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0xf0,0x00,0x00,0x00,0xf1,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xf5,0x00,0x00,0x00, -0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x1d,0x00,0x03,0x00,0xf9,0x00,0x00,0x00,0xb9,0x00,0x00,0x00, -0x1e,0x00,0x03,0x00,0xfa,0x00,0x00,0x00,0xf9,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0xfb,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0xfa,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0xfb,0x00,0x00,0x00, -0xfc,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x07,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0xb9,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x0b,0x01,0x00,0x00,0x04,0x00,0x00,0x00, -0xec,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x11,0x01,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0xec,0x00,0x00,0x00, -0x15,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x32,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x17,0x01,0x00,0x00,0x01,0x00,0x00,0x00, -0x33,0x00,0x06,0x00,0x09,0x00,0x00,0x00,0x18,0x01,0x00,0x00, -0x17,0x01,0x00,0x00,0x39,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x19,0x01,0x00,0x00, -0x51,0x00,0x00,0x00,0x18,0x01,0x00,0x00,0x00,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x1a,0x01,0x00,0x00, -0x84,0x00,0x00,0x00,0x19,0x01,0x00,0x00,0x39,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x1b,0x01,0x00,0x00, -0x86,0x00,0x00,0x00,0x1a,0x01,0x00,0x00,0x6c,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x39,0x01,0x00,0x00, -0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x3a,0x01,0x00,0x00, -0x84,0x00,0x00,0x00,0x9c,0x00,0x00,0x00,0x39,0x01,0x00,0x00, -0x1c,0x00,0x04,0x00,0x3b,0x01,0x00,0x00,0xec,0x00,0x00,0x00, -0x3a,0x01,0x00,0x00,0x20,0x00,0x04,0x00,0x3c,0x01,0x00,0x00, -0x04,0x00,0x00,0x00,0x3b,0x01,0x00,0x00,0x3b,0x00,0x04,0x00, -0x3c,0x01,0x00,0x00,0x3d,0x01,0x00,0x00,0x04,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x41,0x01,0x00,0x00, -0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x1d,0x00,0x03,0x00,0x45,0x01,0x00,0x00,0xb9,0x00,0x00,0x00, -0x1e,0x00,0x03,0x00,0x46,0x01,0x00,0x00,0x45,0x01,0x00,0x00, -0x20,0x00,0x04,0x00,0x47,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, -0x46,0x01,0x00,0x00,0x3b,0x00,0x04,0x00,0x47,0x01,0x00,0x00, -0x48,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x5b,0x01,0x00,0x00,0x80,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x62,0x01,0x00,0x00,0x08,0x01,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x63,0x01,0x00,0x00, -0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x66,0x01,0x00,0x00, -0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x81,0x01,0x00,0x00, -0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00, -0x1c,0x00,0x04,0x00,0x82,0x01,0x00,0x00,0xec,0x00,0x00,0x00, -0x81,0x01,0x00,0x00,0x20,0x00,0x04,0x00,0x83,0x01,0x00,0x00, -0x07,0x00,0x00,0x00,0x82,0x01,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x93,0x01,0x00,0x00,0x80,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x99,0x01,0x00,0x00,0x07,0x00,0x00,0x00,0xec,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xaf,0x01,0x00,0x00, -0x84,0x00,0x00,0x00,0xb4,0x00,0x00,0x00,0xb1,0x00,0x00,0x00, -0x1c,0x00,0x04,0x00,0xb0,0x01,0x00,0x00,0xec,0x00,0x00,0x00, -0xaf,0x01,0x00,0x00,0x20,0x00,0x04,0x00,0xb1,0x01,0x00,0x00, -0x07,0x00,0x00,0x00,0xb0,0x01,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xba,0x01,0x00,0x00,0x86,0x00,0x00,0x00, -0xae,0x00,0x00,0x00,0xb4,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xc2,0x01,0x00,0x00,0x80,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xf1,0x01,0x00,0x00,0x84,0x00,0x00,0x00, -0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0x26,0x02,0x00,0x00,0x0d,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00,0x2e,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0x1d,0x00,0x03,0x00,0x74,0x02,0x00,0x00, -0xb9,0x00,0x00,0x00,0x1e,0x00,0x03,0x00,0x75,0x02,0x00,0x00, -0x74,0x02,0x00,0x00,0x20,0x00,0x04,0x00,0x76,0x02,0x00,0x00, -0x0c,0x00,0x00,0x00,0x75,0x02,0x00,0x00,0x3b,0x00,0x04,0x00, -0x76,0x02,0x00,0x00,0x77,0x02,0x00,0x00,0x0c,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x7c,0x02,0x00,0x00, -0x05,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x89,0x02,0x00,0x00,0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x36,0x00,0x05,0x00,0x02,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x03,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0x05,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0xbe,0x00,0x00,0x00,0xbf,0x00,0x00,0x00,0x07,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x83,0x01,0x00,0x00,0x84,0x01,0x00,0x00, -0x07,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0xb1,0x01,0x00,0x00, -0xb2,0x01,0x00,0x00,0x07,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x0d,0x00,0x00,0x00,0x0e,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x0f,0x00,0x00,0x00,0x0e,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x14,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x17,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, -0x17,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x1e,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x17,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x22,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x22,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x24,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x27,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x29,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x2a,0x00,0x00,0x00,0x1e,0x00,0x00,0x00, -0x29,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x2d,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x2f,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x30,0x00,0x00,0x00,0x24,0x00,0x00,0x00,0x2f,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x32,0x00,0x00,0x00, -0x30,0x00,0x00,0x00,0x2a,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x35,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x34,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x36,0x00,0x00,0x00,0x35,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x38,0x00,0x00,0x00,0x36,0x00,0x00,0x00, -0x37,0x00,0x00,0x00,0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x3a,0x00,0x00,0x00,0x38,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3b,0x00,0x00,0x00, -0x3a,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x0d,0x00,0x00,0x00,0x3f,0x00,0x00,0x00,0x3d,0x00,0x00,0x00, -0x3e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0x3f,0x00,0x00,0x00,0x89,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x42,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x3b,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x47,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x3b,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00,0x49,0x00,0x00,0x00, -0x3d,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x4a,0x00,0x00,0x00,0x49,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00,0x4d,0x00,0x00,0x00, -0x4c,0x00,0x00,0x00,0x3e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x4d,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x50,0x00,0x00,0x00, -0x4e,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x89,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x55,0x00,0x00,0x00,0x50,0x00,0x00,0x00, -0x54,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x59,0x00,0x00,0x00,0x50,0x00,0x00,0x00,0x58,0x00,0x00,0x00, -0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x5d,0x00,0x00,0x00, -0x4e,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x89,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x64,0x00,0x00,0x00,0x5d,0x00,0x00,0x00, -0x63,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x68,0x00,0x00,0x00,0x5d,0x00,0x00,0x00,0x67,0x00,0x00,0x00, -0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x6e,0x00,0x00,0x00, -0x4e,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x73,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, -0x72,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0x77,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x76,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x78,0x00,0x00,0x00, -0x77,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x79,0x00,0x00,0x00,0x47,0x00,0x00,0x00,0x78,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x7c,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x7b,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x7d,0x00,0x00,0x00,0x7c,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x7f,0x00,0x00,0x00, -0x47,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x82,0x00,0x00,0x00,0x7f,0x00,0x00,0x00, -0x78,0x00,0x00,0x00,0x0c,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0x83,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x26,0x00,0x00,0x00, -0x7d,0x00,0x00,0x00,0x82,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x87,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x86,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x88,0x00,0x00,0x00,0x87,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x89,0x00,0x00,0x00,0x32,0x00,0x00,0x00, -0x88,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x8b,0x00,0x00,0x00,0x42,0x00,0x00,0x00,0x37,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x8d,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x8c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x8e,0x00,0x00,0x00,0x8d,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8f,0x00,0x00,0x00, -0x8b,0x00,0x00,0x00,0x8e,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x90,0x00,0x00,0x00,0x89,0x00,0x00,0x00, -0x8f,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x92,0x00,0x00,0x00,0x90,0x00,0x00,0x00,0x79,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x93,0x00,0x00,0x00, -0x92,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x98,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x97,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x99,0x00,0x00,0x00,0x98,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x9a,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, -0x99,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x9d,0x00,0x00,0x00,0x4a,0x00,0x00,0x00,0x9c,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x9f,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x9e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xa0,0x00,0x00,0x00,0x9f,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa1,0x00,0x00,0x00, -0x9d,0x00,0x00,0x00,0xa0,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xa2,0x00,0x00,0x00,0x9a,0x00,0x00,0x00, -0xa1,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xa4,0x00,0x00,0x00,0xa2,0x00,0x00,0x00,0x79,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa5,0x00,0x00,0x00, -0xa4,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xa7,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xa7,0x00,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x9b,0x02,0x00,0x00, -0x3e,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0xc6,0x00,0x00,0x00, -0xa8,0x00,0x00,0x00,0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00, -0xb8,0x00,0x00,0x00,0x9b,0x02,0x00,0x00,0xb6,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0xa9,0x00,0x00,0x00,0xa8,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xb8,0x00,0x00,0x00, -0xa8,0x00,0x00,0x00,0xa9,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xa8,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, -0xc3,0x00,0x00,0x00,0xbf,0x00,0x00,0x00,0x9b,0x02,0x00,0x00, -0x3e,0x00,0x03,0x00,0xc3,0x00,0x00,0x00,0xc1,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xc6,0x00,0x00,0x00, -0x9b,0x02,0x00,0x00,0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xa7,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xa9,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xc9,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xc9,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xb4,0x02,0x00,0x00,0xa5,0x00,0x00,0x00,0xa9,0x00,0x00,0x00, -0x68,0x01,0x00,0x00,0xcc,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xb0,0x02,0x00,0x00,0x93,0x00,0x00,0x00, -0xa9,0x00,0x00,0x00,0x65,0x01,0x00,0x00,0xcc,0x00,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x9c,0x02,0x00,0x00, -0x79,0x00,0x00,0x00,0xa9,0x00,0x00,0x00,0x16,0x02,0x00,0x00, -0xcc,0x00,0x00,0x00,0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00, -0xd0,0x00,0x00,0x00,0x9c,0x02,0x00,0x00,0x83,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0xcb,0x00,0x00,0x00,0xcc,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xd0,0x00,0x00,0x00, -0xca,0x00,0x00,0x00,0xcb,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xca,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xd2,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xd2,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xac,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, -0xca,0x00,0x00,0x00,0x1d,0x01,0x00,0x00,0xd5,0x00,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00,0xd8,0x00,0x00,0x00, -0xac,0x02,0x00,0x00,0x37,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0xd4,0x00,0x00,0x00,0xd5,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xd8,0x00,0x00,0x00,0xd3,0x00,0x00,0x00, -0xd4,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xd3,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xdc,0x00,0x00,0x00, -0x8b,0x00,0x00,0x00,0x73,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xde,0x00,0x00,0x00,0xdc,0x00,0x00,0x00, -0xac,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00, -0xe1,0x00,0x00,0x00,0xde,0x00,0x00,0x00,0x36,0x00,0x00,0x00, -0xf7,0x00,0x03,0x00,0xe3,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xe1,0x00,0x00,0x00,0xe2,0x00,0x00,0x00, -0xe3,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xe2,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xe6,0x00,0x00,0x00, -0x9c,0x02,0x00,0x00,0x6e,0x00,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb7,0x00,0x00,0x00,0xe8,0x00,0x00,0x00,0xe6,0x00,0x00,0x00, -0x83,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xe3,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xe3,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, -0xb7,0x00,0x00,0x00,0xe9,0x00,0x00,0x00,0xe1,0x00,0x00,0x00, -0xd3,0x00,0x00,0x00,0xe8,0x00,0x00,0x00,0xe2,0x00,0x00,0x00, -0xf7,0x00,0x03,0x00,0xeb,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xe9,0x00,0x00,0x00,0xea,0x00,0x00,0x00, -0x0d,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xea,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xf4,0x00,0x00,0x00, -0x73,0x00,0x00,0x00,0xac,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xf6,0x00,0x00,0x00,0xf4,0x00,0x00,0x00, -0xf5,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xf8,0x00,0x00,0x00,0xf6,0x00,0x00,0x00,0x6e,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x03,0x01,0x00,0x00, -0xf4,0x00,0x00,0x00,0x8e,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x04,0x01,0x00,0x00,0xb0,0x02,0x00,0x00, -0x03,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x06,0x01,0x00,0x00,0x04,0x01,0x00,0x00,0x6e,0x00,0x00,0x00, -0x41,0x00,0x06,0x00,0x07,0x01,0x00,0x00,0x08,0x01,0x00,0x00, -0xfc,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0x06,0x01,0x00,0x00, -0x3d,0x00,0x04,0x00,0xb9,0x00,0x00,0x00,0x09,0x01,0x00,0x00, -0x08,0x01,0x00,0x00,0x73,0x00,0x04,0x00,0xec,0x00,0x00,0x00, -0x0a,0x01,0x00,0x00,0x09,0x01,0x00,0x00,0x41,0x00,0x05,0x00, -0x0b,0x01,0x00,0x00,0x0c,0x01,0x00,0x00,0xf1,0x00,0x00,0x00, -0xf8,0x00,0x00,0x00,0x3e,0x00,0x03,0x00,0x0c,0x01,0x00,0x00, -0x0a,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xeb,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0x0d,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x10,0x01,0x00,0x00,0x73,0x00,0x00,0x00, -0xac,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x12,0x01,0x00,0x00,0x10,0x01,0x00,0x00,0x11,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x14,0x01,0x00,0x00, -0x12,0x01,0x00,0x00,0x6e,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x0b,0x01,0x00,0x00,0x16,0x01,0x00,0x00,0xf1,0x00,0x00,0x00, -0x14,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x16,0x01,0x00,0x00, -0x15,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xeb,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xeb,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xd5,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xd5,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x1d,0x01,0x00,0x00, -0xac,0x02,0x00,0x00,0x1b,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0xd2,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xd4,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x1f,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x1f,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xad,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0xd4,0x00,0x00,0x00, -0x61,0x01,0x00,0x00,0x22,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb7,0x00,0x00,0x00,0x25,0x01,0x00,0x00,0xad,0x02,0x00,0x00, -0x9c,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x21,0x01,0x00,0x00, -0x22,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x25,0x01,0x00,0x00,0x20,0x01,0x00,0x00,0x21,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x20,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x29,0x01,0x00,0x00,0x9d,0x00,0x00,0x00, -0x73,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x2b,0x01,0x00,0x00,0x29,0x01,0x00,0x00,0xad,0x02,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x2c,0x01,0x00,0x00, -0x12,0x00,0x00,0x00,0xc5,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x2d,0x01,0x00,0x00,0x2c,0x01,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00,0x2e,0x01,0x00,0x00, -0x2b,0x01,0x00,0x00,0x2d,0x01,0x00,0x00,0xf7,0x00,0x03,0x00, -0x30,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x2e,0x01,0x00,0x00,0x2f,0x01,0x00,0x00,0x30,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x2f,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x33,0x01,0x00,0x00,0x9c,0x02,0x00,0x00, -0x6e,0x00,0x00,0x00,0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00, -0x35,0x01,0x00,0x00,0x33,0x01,0x00,0x00,0x83,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x30,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x30,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0xb7,0x00,0x00,0x00, -0x36,0x01,0x00,0x00,0x2e,0x01,0x00,0x00,0x20,0x01,0x00,0x00, -0x35,0x01,0x00,0x00,0x2f,0x01,0x00,0x00,0xf7,0x00,0x03,0x00, -0x38,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x36,0x01,0x00,0x00,0x37,0x01,0x00,0x00,0x57,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x37,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x40,0x01,0x00,0x00,0x73,0x00,0x00,0x00, -0xad,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x42,0x01,0x00,0x00,0x40,0x01,0x00,0x00,0x41,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x44,0x01,0x00,0x00, -0x42,0x01,0x00,0x00,0x6e,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x4f,0x01,0x00,0x00,0x40,0x01,0x00,0x00, -0xa0,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x50,0x01,0x00,0x00,0xb4,0x02,0x00,0x00,0x4f,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x52,0x01,0x00,0x00, -0x50,0x01,0x00,0x00,0x6e,0x00,0x00,0x00,0x41,0x00,0x06,0x00, -0x07,0x01,0x00,0x00,0x53,0x01,0x00,0x00,0x48,0x01,0x00,0x00, -0x34,0x00,0x00,0x00,0x52,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, -0xb9,0x00,0x00,0x00,0x54,0x01,0x00,0x00,0x53,0x01,0x00,0x00, -0x73,0x00,0x04,0x00,0xec,0x00,0x00,0x00,0x55,0x01,0x00,0x00, -0x54,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0x0b,0x01,0x00,0x00, -0x56,0x01,0x00,0x00,0x3d,0x01,0x00,0x00,0x44,0x01,0x00,0x00, -0x3e,0x00,0x03,0x00,0x56,0x01,0x00,0x00,0x55,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0x38,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x57,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x5a,0x01,0x00,0x00,0x73,0x00,0x00,0x00,0xad,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x5c,0x01,0x00,0x00, -0x5a,0x01,0x00,0x00,0x5b,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x5e,0x01,0x00,0x00,0x5c,0x01,0x00,0x00, -0x6e,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x0b,0x01,0x00,0x00, -0x5f,0x01,0x00,0x00,0x3d,0x01,0x00,0x00,0x5e,0x01,0x00,0x00, -0x3e,0x00,0x03,0x00,0x5f,0x01,0x00,0x00,0x15,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0x38,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x38,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x22,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x22,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x61,0x01,0x00,0x00,0xad,0x02,0x00,0x00, -0x1b,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x1f,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x21,0x01,0x00,0x00,0xe0,0x00,0x04,0x00, -0x0c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x62,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x65,0x01,0x00,0x00, -0xb0,0x02,0x00,0x00,0x63,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x68,0x01,0x00,0x00,0xb4,0x02,0x00,0x00, -0x66,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x6a,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x6a,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xb6,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, -0x21,0x01,0x00,0x00,0x14,0x02,0x00,0x00,0x6d,0x01,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00,0x70,0x01,0x00,0x00, -0xb6,0x02,0x00,0x00,0x6c,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0x6c,0x01,0x00,0x00,0x6d,0x01,0x00,0x00,0x00,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x70,0x01,0x00,0x00,0x6b,0x01,0x00,0x00, -0x6c,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x6b,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0x72,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x72,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xba,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0x6b,0x01,0x00,0x00, -0x9e,0x01,0x00,0x00,0x75,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb7,0x00,0x00,0x00,0x78,0x01,0x00,0x00,0xba,0x02,0x00,0x00, -0x60,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x74,0x01,0x00,0x00, -0x75,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x78,0x01,0x00,0x00,0x73,0x01,0x00,0x00,0x74,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x73,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0x7a,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x7a,0x01,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xcc,0x02,0x00,0x00, -0x3e,0x00,0x00,0x00,0x73,0x01,0x00,0x00,0x9c,0x01,0x00,0x00, -0x7b,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00, -0x80,0x01,0x00,0x00,0xcc,0x02,0x00,0x00,0x62,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0x7c,0x01,0x00,0x00,0x7b,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x80,0x01,0x00,0x00, -0x7b,0x01,0x00,0x00,0x7c,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x7b,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x86,0x01,0x00,0x00,0xba,0x02,0x00,0x00,0x62,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x88,0x01,0x00,0x00, -0x86,0x01,0x00,0x00,0xcc,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x8a,0x01,0x00,0x00,0x55,0x00,0x00,0x00, -0x53,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x8c,0x01,0x00,0x00,0xba,0x02,0x00,0x00,0x61,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8d,0x01,0x00,0x00, -0x8a,0x01,0x00,0x00,0x8c,0x01,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x8f,0x01,0x00,0x00,0x64,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x90,0x01,0x00,0x00,0x8d,0x01,0x00,0x00,0x8f,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x92,0x01,0x00,0x00, -0x90,0x01,0x00,0x00,0xcc,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x94,0x01,0x00,0x00,0x92,0x01,0x00,0x00, -0x93,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x96,0x01,0x00,0x00,0x94,0x01,0x00,0x00,0xb6,0x02,0x00,0x00, -0x41,0x00,0x05,0x00,0x0b,0x01,0x00,0x00,0x97,0x01,0x00,0x00, -0xf1,0x00,0x00,0x00,0x96,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, -0xec,0x00,0x00,0x00,0x98,0x01,0x00,0x00,0x97,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0x99,0x01,0x00,0x00,0x9a,0x01,0x00,0x00, -0x84,0x01,0x00,0x00,0x88,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0x9a,0x01,0x00,0x00,0x98,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x9c,0x01,0x00,0x00,0xcc,0x02,0x00,0x00, -0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x7a,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x7c,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0x75,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x75,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9e,0x01,0x00,0x00, -0xba,0x02,0x00,0x00,0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x72,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x74,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0xa0,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xa0,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xbb,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0x74,0x01,0x00,0x00, -0xcc,0x01,0x00,0x00,0xa3,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb7,0x00,0x00,0x00,0xa6,0x01,0x00,0x00,0xbb,0x02,0x00,0x00, -0xb4,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xa2,0x01,0x00,0x00, -0xa3,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0xa6,0x01,0x00,0x00,0xa1,0x01,0x00,0x00,0xa2,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xa1,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0xa8,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xa8,0x01,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xc9,0x02,0x00,0x00, -0x3e,0x00,0x00,0x00,0xa1,0x01,0x00,0x00,0xca,0x01,0x00,0x00, -0xa9,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00, -0xae,0x01,0x00,0x00,0xc9,0x02,0x00,0x00,0xb1,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0xaa,0x01,0x00,0x00,0xa9,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xae,0x01,0x00,0x00, -0xa9,0x01,0x00,0x00,0xaa,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xa9,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xb4,0x01,0x00,0x00,0xbb,0x02,0x00,0x00,0xb1,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xb6,0x01,0x00,0x00, -0xb4,0x01,0x00,0x00,0xc9,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xb8,0x01,0x00,0x00,0x59,0x00,0x00,0x00, -0xae,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xbb,0x01,0x00,0x00,0xbb,0x02,0x00,0x00,0xba,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xbc,0x01,0x00,0x00, -0xb8,0x01,0x00,0x00,0xbb,0x01,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xbe,0x01,0x00,0x00,0x68,0x00,0x00,0x00, -0xb1,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xbf,0x01,0x00,0x00,0xbc,0x01,0x00,0x00,0xbe,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xc1,0x01,0x00,0x00, -0xbf,0x01,0x00,0x00,0xc9,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xc3,0x01,0x00,0x00,0xc1,0x01,0x00,0x00, -0xc2,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xc5,0x01,0x00,0x00,0xc3,0x01,0x00,0x00,0xb6,0x02,0x00,0x00, -0x41,0x00,0x05,0x00,0x0b,0x01,0x00,0x00,0xc6,0x01,0x00,0x00, -0x3d,0x01,0x00,0x00,0xc5,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, -0xec,0x00,0x00,0x00,0xc7,0x01,0x00,0x00,0xc6,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0x99,0x01,0x00,0x00,0xc8,0x01,0x00,0x00, -0xb2,0x01,0x00,0x00,0xb6,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0xc8,0x01,0x00,0x00,0xc7,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xca,0x01,0x00,0x00,0xc9,0x02,0x00,0x00, -0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xa8,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xaa,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0xa3,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xa3,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xcc,0x01,0x00,0x00, -0xbb,0x02,0x00,0x00,0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xa0,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xa2,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0xce,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xce,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xbc,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0xa2,0x01,0x00,0x00, -0x12,0x02,0x00,0x00,0xd1,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb7,0x00,0x00,0x00,0xd4,0x01,0x00,0x00,0xbc,0x02,0x00,0x00, -0xb4,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xd0,0x01,0x00,0x00, -0xd1,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0xd4,0x01,0x00,0x00,0xcf,0x01,0x00,0x00,0xd0,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xcf,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0xd6,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xd6,0x01,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xc0,0x02,0x00,0x00, -0x3e,0x00,0x00,0x00,0xcf,0x01,0x00,0x00,0x10,0x02,0x00,0x00, -0xd9,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00, -0xdc,0x01,0x00,0x00,0xc0,0x02,0x00,0x00,0x60,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0xd8,0x01,0x00,0x00,0xd9,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xdc,0x01,0x00,0x00, -0xd7,0x01,0x00,0x00,0xd8,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xd7,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xde,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xde,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xc2,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, -0xd7,0x01,0x00,0x00,0x0e,0x02,0x00,0x00,0xe1,0x01,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00,0xe4,0x01,0x00,0x00, -0xc2,0x02,0x00,0x00,0xb1,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0xe0,0x01,0x00,0x00,0xe1,0x01,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xe4,0x01,0x00,0x00,0xdf,0x01,0x00,0x00, -0xe0,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xdf,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0xe6,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xe6,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xc4,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0xdf,0x01,0x00,0x00, -0x0c,0x02,0x00,0x00,0xe7,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb7,0x00,0x00,0x00,0xec,0x01,0x00,0x00,0xc4,0x02,0x00,0x00, -0x62,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xe8,0x01,0x00,0x00, -0xe7,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0xec,0x01,0x00,0x00,0xe7,0x01,0x00,0x00,0xe8,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xe7,0x01,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xee,0x01,0x00,0x00,0xbc,0x02,0x00,0x00, -0xb1,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xf0,0x01,0x00,0x00,0xee,0x01,0x00,0x00,0xc2,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xf2,0x01,0x00,0x00, -0xf0,0x01,0x00,0x00,0xf1,0x01,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xf4,0x01,0x00,0x00,0xc0,0x02,0x00,0x00, -0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xf5,0x01,0x00,0x00,0xf2,0x01,0x00,0x00,0xf4,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xf7,0x01,0x00,0x00, -0xf5,0x01,0x00,0x00,0xc4,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xfb,0x01,0x00,0x00,0xf4,0x01,0x00,0x00, -0xc4,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x99,0x01,0x00,0x00, -0xfc,0x01,0x00,0x00,0x84,0x01,0x00,0x00,0xfb,0x01,0x00,0x00, -0x3d,0x00,0x04,0x00,0xec,0x00,0x00,0x00,0xfd,0x01,0x00,0x00, -0xfc,0x01,0x00,0x00,0x73,0x00,0x04,0x00,0xb9,0x00,0x00,0x00, -0xfe,0x01,0x00,0x00,0xfd,0x01,0x00,0x00,0x41,0x00,0x05,0x00, -0x99,0x01,0x00,0x00,0x03,0x02,0x00,0x00,0xb2,0x01,0x00,0x00, -0xf0,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0xec,0x00,0x00,0x00, -0x04,0x02,0x00,0x00,0x03,0x02,0x00,0x00,0x73,0x00,0x04,0x00, -0xb9,0x00,0x00,0x00,0x05,0x02,0x00,0x00,0x04,0x02,0x00,0x00, -0x41,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0x07,0x02,0x00,0x00, -0xbf,0x00,0x00,0x00,0xf7,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, -0xb9,0x00,0x00,0x00,0x08,0x02,0x00,0x00,0x07,0x02,0x00,0x00, -0x0c,0x00,0x08,0x00,0xb9,0x00,0x00,0x00,0x09,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0xfe,0x01,0x00,0x00, -0x05,0x02,0x00,0x00,0x08,0x02,0x00,0x00,0x3e,0x00,0x03,0x00, -0x07,0x02,0x00,0x00,0x09,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x0c,0x02,0x00,0x00,0xc4,0x02,0x00,0x00, -0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xe6,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xe8,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0xe1,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xe1,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x0e,0x02,0x00,0x00, -0xc2,0x02,0x00,0x00,0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xde,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xe0,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0xd9,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xd9,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x10,0x02,0x00,0x00,0xc0,0x02,0x00,0x00,0xc5,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xd6,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xd8,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xd1,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xd1,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x12,0x02,0x00,0x00,0xbc,0x02,0x00,0x00, -0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xce,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xd0,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0x6d,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x6d,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x14,0x02,0x00,0x00, -0xb6,0x02,0x00,0x00,0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x6a,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x6c,0x01,0x00,0x00, -0xe0,0x00,0x04,0x00,0x0c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x62,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xcc,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xcc,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x16,0x02,0x00,0x00,0x9c,0x02,0x00,0x00, -0x6c,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xc9,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xcb,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x1b,0x02,0x00,0x00,0x55,0x00,0x00,0x00, -0x53,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x1c,0x02,0x00,0x00,0x8b,0x00,0x00,0x00,0x1b,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x21,0x02,0x00,0x00, -0x59,0x00,0x00,0x00,0xae,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x22,0x02,0x00,0x00,0x9d,0x00,0x00,0x00, -0x21,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0x27,0x02,0x00,0x00,0x12,0x00,0x00,0x00,0x26,0x02,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x28,0x02,0x00,0x00, -0x27,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x29,0x02,0x00,0x00,0x0f,0x00,0x00,0x00,0x28,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x2d,0x02,0x00,0x00, -0x47,0x00,0x00,0x00,0x28,0x02,0x00,0x00,0x41,0x00,0x05,0x00, -0x0d,0x00,0x00,0x00,0x2f,0x02,0x00,0x00,0x2e,0x02,0x00,0x00, -0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x30,0x02,0x00,0x00,0x2f,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x31,0x02,0x00,0x00,0x2d,0x02,0x00,0x00, -0x30,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x32,0x02,0x00,0x00,0x29,0x02,0x00,0x00,0x31,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0x34,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x34,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0x9d,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0xcb,0x00,0x00,0x00, -0x9a,0x02,0x00,0x00,0x37,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb7,0x00,0x00,0x00,0x3a,0x02,0x00,0x00,0x9d,0x02,0x00,0x00, -0xb4,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x36,0x02,0x00,0x00, -0x37,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x3a,0x02,0x00,0x00,0x35,0x02,0x00,0x00,0x36,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x35,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x3c,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x3c,0x02,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x9e,0x02,0x00,0x00, -0x3e,0x00,0x00,0x00,0x35,0x02,0x00,0x00,0x98,0x02,0x00,0x00, -0x3f,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00, -0x42,0x02,0x00,0x00,0x9e,0x02,0x00,0x00,0x60,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0x3e,0x02,0x00,0x00,0x3f,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x42,0x02,0x00,0x00, -0x3d,0x02,0x00,0x00,0x3e,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x3d,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x46,0x02,0x00,0x00,0x9e,0x02,0x00,0x00,0x61,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x47,0x02,0x00,0x00, -0x1c,0x02,0x00,0x00,0x46,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x49,0x02,0x00,0x00,0x64,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x4a,0x02,0x00,0x00,0x47,0x02,0x00,0x00,0x49,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x4e,0x02,0x00,0x00, -0x9d,0x02,0x00,0x00,0xba,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x4f,0x02,0x00,0x00,0x22,0x02,0x00,0x00, -0x4e,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x51,0x02,0x00,0x00,0x68,0x00,0x00,0x00,0xb1,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x52,0x02,0x00,0x00, -0x4f,0x02,0x00,0x00,0x51,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x54,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x54,0x02,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xa0,0x02,0x00,0x00, -0x3e,0x00,0x00,0x00,0x3d,0x02,0x00,0x00,0x96,0x02,0x00,0x00, -0x57,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00, -0x5a,0x02,0x00,0x00,0xa0,0x02,0x00,0x00,0xb1,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0x56,0x02,0x00,0x00,0x57,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x5a,0x02,0x00,0x00, -0x55,0x02,0x00,0x00,0x56,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x55,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x5c,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x5c,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xa2,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, -0x55,0x02,0x00,0x00,0x94,0x02,0x00,0x00,0x5f,0x02,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00,0x62,0x02,0x00,0x00, -0xa2,0x02,0x00,0x00,0x62,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0x5e,0x02,0x00,0x00,0x5f,0x02,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x62,0x02,0x00,0x00,0x5d,0x02,0x00,0x00, -0x5e,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x5d,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x65,0x02,0x00,0x00, -0x4a,0x02,0x00,0x00,0xa2,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb7,0x00,0x00,0x00,0x68,0x02,0x00,0x00,0x65,0x02,0x00,0x00, -0x36,0x00,0x00,0x00,0xf7,0x00,0x03,0x00,0x6a,0x02,0x00,0x00, -0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x68,0x02,0x00,0x00, -0x69,0x02,0x00,0x00,0x6a,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x69,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x6d,0x02,0x00,0x00,0x52,0x02,0x00,0x00,0xa0,0x02,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x6e,0x02,0x00,0x00, -0x12,0x00,0x00,0x00,0xc5,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x6f,0x02,0x00,0x00,0x6e,0x02,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00,0x70,0x02,0x00,0x00, -0x6d,0x02,0x00,0x00,0x6f,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x6a,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x6a,0x02,0x00,0x00, -0xf5,0x00,0x07,0x00,0xb7,0x00,0x00,0x00,0x71,0x02,0x00,0x00, -0x68,0x02,0x00,0x00,0x5d,0x02,0x00,0x00,0x70,0x02,0x00,0x00, -0x69,0x02,0x00,0x00,0xf7,0x00,0x03,0x00,0x73,0x02,0x00,0x00, -0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x71,0x02,0x00,0x00, -0x72,0x02,0x00,0x00,0x73,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x72,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x7b,0x02,0x00,0x00,0x52,0x02,0x00,0x00,0xa0,0x02,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x7d,0x02,0x00,0x00, -0x12,0x00,0x00,0x00,0x7c,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x7e,0x02,0x00,0x00,0x7d,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x7f,0x02,0x00,0x00, -0x7b,0x02,0x00,0x00,0x7e,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x80,0x02,0x00,0x00,0x32,0x02,0x00,0x00, -0x7f,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x82,0x02,0x00,0x00,0x80,0x02,0x00,0x00,0x4a,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x84,0x02,0x00,0x00, -0x82,0x02,0x00,0x00,0xa2,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x86,0x02,0x00,0x00,0x9d,0x02,0x00,0x00, -0xb1,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x88,0x02,0x00,0x00,0x86,0x02,0x00,0x00,0xa0,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8a,0x02,0x00,0x00, -0x88,0x02,0x00,0x00,0x89,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x8c,0x02,0x00,0x00,0x9e,0x02,0x00,0x00, -0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x8d,0x02,0x00,0x00,0x8a,0x02,0x00,0x00,0x8c,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8f,0x02,0x00,0x00, -0x8d,0x02,0x00,0x00,0xa2,0x02,0x00,0x00,0x41,0x00,0x05,0x00, -0xc2,0x00,0x00,0x00,0x90,0x02,0x00,0x00,0xbf,0x00,0x00,0x00, -0x8f,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0xb9,0x00,0x00,0x00, -0x91,0x02,0x00,0x00,0x90,0x02,0x00,0x00,0x41,0x00,0x06,0x00, -0x07,0x01,0x00,0x00,0x92,0x02,0x00,0x00,0x77,0x02,0x00,0x00, -0x34,0x00,0x00,0x00,0x84,0x02,0x00,0x00,0x3e,0x00,0x03,0x00, -0x92,0x02,0x00,0x00,0x91,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x73,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x73,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0x5f,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x5f,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x94,0x02,0x00,0x00,0xa2,0x02,0x00,0x00,0xc5,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x5c,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x5e,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x57,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x57,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x96,0x02,0x00,0x00,0xa0,0x02,0x00,0x00, -0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x54,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x56,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x3f,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x3f,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x98,0x02,0x00,0x00, -0x9e,0x02,0x00,0x00,0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x3c,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x3e,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0x37,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x37,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x9a,0x02,0x00,0x00,0x9d,0x02,0x00,0x00,0xc5,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x34,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x36,0x02,0x00,0x00,0xfd,0x00,0x01,0x00,0x38,0x00,0x01,0x00, - -}; -const uint64_t matmul_f32_l_len = 10164; - -unsigned char matmul_f32_l_fp32_data[] = { -0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, -0xc6,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, -0x01,0x00,0x00,0x00,0x0b,0x00,0x06,0x00,0x01,0x00,0x00,0x00, -0x47,0x4c,0x53,0x4c,0x2e,0x73,0x74,0x64,0x2e,0x34,0x35,0x30, -0x00,0x00,0x00,0x00,0x0e,0x00,0x03,0x00,0x00,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x0f,0x00,0x0f,0x00,0x05,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x6d,0x61,0x69,0x6e,0x00,0x00,0x00,0x00, -0x0b,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x3d,0x00,0x00,0x00, -0x4c,0x00,0x00,0x00,0xf0,0x00,0x00,0x00,0xfb,0x00,0x00,0x00, -0x3a,0x01,0x00,0x00,0x45,0x01,0x00,0x00,0x27,0x02,0x00,0x00, -0x70,0x02,0x00,0x00,0x10,0x00,0x06,0x00,0x04,0x00,0x00,0x00, -0x11,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x0b,0x00,0x00,0x00, -0x0b,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x10,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x10,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x10,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x05,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x10,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x07,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x1c,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x08,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x10,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x24,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x0a,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x28,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x2c,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x10,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x30,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x0d,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x34,0x00,0x00,0x00, -0x47,0x00,0x03,0x00,0x10,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x37,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x3d,0x00,0x00,0x00, -0x0b,0x00,0x00,0x00,0x1a,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x4c,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x1b,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x4f,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x53,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x60,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x62,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x07,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x6c,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x9c,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0xae,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x05,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xb1,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0xf8,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x48,0x00,0x04,0x00,0xf9,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0xf9,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x03,0x00,0xf9,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0xfb,0x00,0x00,0x00,0x22,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xfb,0x00,0x00,0x00, -0x21,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x14,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x15,0x01,0x00,0x00,0x0b,0x00,0x00,0x00, -0x19,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x42,0x01,0x00,0x00, -0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00, -0x43,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x43,0x01,0x00,0x00,0x00,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, -0x43,0x01,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x45,0x01,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x45,0x01,0x00,0x00,0x21,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x27,0x02,0x00,0x00, -0x0b,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x6d,0x02,0x00,0x00,0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x48,0x00,0x04,0x00,0x6e,0x02,0x00,0x00,0x00,0x00,0x00,0x00, -0x19,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x6e,0x02,0x00,0x00, -0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x03,0x00,0x6e,0x02,0x00,0x00,0x02,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x70,0x02,0x00,0x00,0x22,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x70,0x02,0x00,0x00, -0x21,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x13,0x00,0x02,0x00, -0x02,0x00,0x00,0x00,0x21,0x00,0x03,0x00,0x03,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x15,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x17,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x03,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x0a,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00, -0x0b,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x0d,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x1e,0x00,0x10,0x00,0x10,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x11,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x10,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x11,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x15,0x00,0x04,0x00,0x13,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x08,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x15,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, -0x21,0x00,0x00,0x00,0x0a,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0x27,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x2d,0x00,0x00,0x00, -0x07,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, -0x34,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x32,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00, -0x3d,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x3e,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00,0x4c,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x4f,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x32,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x54,0x00,0x00,0x00, -0x86,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x53,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x58,0x00,0x00,0x00, -0x86,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x53,0x00,0x00,0x00, -0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x60,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x61,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x53,0x00,0x00,0x00, -0x60,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x63,0x00,0x00,0x00,0x86,0x00,0x00,0x00, -0x61,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x67,0x00,0x00,0x00,0x86,0x00,0x00,0x00, -0x61,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x32,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x10,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, -0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x72,0x00,0x00,0x00, -0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x76,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, -0x7b,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x8c,0x00,0x00,0x00, -0x03,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, -0x97,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x32,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x9c,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x9e,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xad,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xae,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xaf,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0x53,0x00,0x00,0x00,0xae,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xb0,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0x4f,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x32,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xb1,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xb2,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0xb0,0x00,0x00,0x00,0xb1,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xb3,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0xb2,0x00,0x00,0x00,0x60,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xb4,0x00,0x00,0x00, -0x86,0x00,0x00,0x00,0xaf,0x00,0x00,0x00,0xb3,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xb5,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0xad,0x00,0x00,0x00,0xb4,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xb6,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0xb5,0x00,0x00,0x00,0xb1,0x00,0x00,0x00, -0x14,0x00,0x02,0x00,0xb7,0x00,0x00,0x00,0x16,0x00,0x03,0x00, -0xb9,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xba,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xbb,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0xba,0x00,0x00,0x00,0xb4,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xbc,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0xbb,0x00,0x00,0x00,0xb1,0x00,0x00,0x00,0x1c,0x00,0x04,0x00, -0xbd,0x00,0x00,0x00,0xb9,0x00,0x00,0x00,0xbc,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0xbe,0x00,0x00,0x00,0x07,0x00,0x00,0x00, -0xbd,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0xb9,0x00,0x00,0x00, -0xc1,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0xc2,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0xb9,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0xc5,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xec,0x00,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xed,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x37,0x00,0x00,0x00, -0xec,0x00,0x00,0x00,0x1c,0x00,0x04,0x00,0xee,0x00,0x00,0x00, -0xb9,0x00,0x00,0x00,0xed,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0xef,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0xee,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0xef,0x00,0x00,0x00,0xf0,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xf4,0x00,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x1d,0x00,0x03,0x00,0xf8,0x00,0x00,0x00, -0xb9,0x00,0x00,0x00,0x1e,0x00,0x03,0x00,0xf9,0x00,0x00,0x00, -0xf8,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0xfa,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0xf9,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0xfa,0x00,0x00,0x00,0xfb,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x06,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, -0xb9,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x09,0x01,0x00,0x00, -0x04,0x00,0x00,0x00,0xb9,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x0f,0x01,0x00,0x00,0x80,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x32,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x14,0x01,0x00,0x00,0x01,0x00,0x00,0x00, -0x33,0x00,0x06,0x00,0x09,0x00,0x00,0x00,0x15,0x01,0x00,0x00, -0x14,0x01,0x00,0x00,0x39,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x16,0x01,0x00,0x00, -0x51,0x00,0x00,0x00,0x15,0x01,0x00,0x00,0x00,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x17,0x01,0x00,0x00, -0x84,0x00,0x00,0x00,0x16,0x01,0x00,0x00,0x39,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x18,0x01,0x00,0x00, -0x86,0x00,0x00,0x00,0x17,0x01,0x00,0x00,0x6c,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x36,0x01,0x00,0x00, -0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x37,0x01,0x00,0x00, -0x84,0x00,0x00,0x00,0x9c,0x00,0x00,0x00,0x36,0x01,0x00,0x00, -0x1c,0x00,0x04,0x00,0x38,0x01,0x00,0x00,0xb9,0x00,0x00,0x00, -0x37,0x01,0x00,0x00,0x20,0x00,0x04,0x00,0x39,0x01,0x00,0x00, -0x04,0x00,0x00,0x00,0x38,0x01,0x00,0x00,0x3b,0x00,0x04,0x00, -0x39,0x01,0x00,0x00,0x3a,0x01,0x00,0x00,0x04,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x3e,0x01,0x00,0x00, -0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x1d,0x00,0x03,0x00,0x42,0x01,0x00,0x00,0xb9,0x00,0x00,0x00, -0x1e,0x00,0x03,0x00,0x43,0x01,0x00,0x00,0x42,0x01,0x00,0x00, -0x20,0x00,0x04,0x00,0x44,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, -0x43,0x01,0x00,0x00,0x3b,0x00,0x04,0x00,0x44,0x01,0x00,0x00, -0x45,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x57,0x01,0x00,0x00,0x80,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x5e,0x01,0x00,0x00,0x08,0x01,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x5f,0x01,0x00,0x00, -0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x62,0x01,0x00,0x00, -0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x7d,0x01,0x00,0x00, -0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00, -0x1c,0x00,0x04,0x00,0x7e,0x01,0x00,0x00,0xb9,0x00,0x00,0x00, -0x7d,0x01,0x00,0x00,0x20,0x00,0x04,0x00,0x7f,0x01,0x00,0x00, -0x07,0x00,0x00,0x00,0x7e,0x01,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x8f,0x01,0x00,0x00,0x80,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xaa,0x01,0x00,0x00,0x84,0x00,0x00,0x00, -0xb4,0x00,0x00,0x00,0xb1,0x00,0x00,0x00,0x1c,0x00,0x04,0x00, -0xab,0x01,0x00,0x00,0xb9,0x00,0x00,0x00,0xaa,0x01,0x00,0x00, -0x20,0x00,0x04,0x00,0xac,0x01,0x00,0x00,0x07,0x00,0x00,0x00, -0xab,0x01,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xb5,0x01,0x00,0x00,0x86,0x00,0x00,0x00,0xae,0x00,0x00,0x00, -0xb4,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xbd,0x01,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xec,0x01,0x00,0x00,0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, -0x1f,0x02,0x00,0x00,0x0d,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x0a,0x00,0x00,0x00,0x27,0x02,0x00,0x00,0x01,0x00,0x00,0x00, -0x1d,0x00,0x03,0x00,0x6d,0x02,0x00,0x00,0xb9,0x00,0x00,0x00, -0x1e,0x00,0x03,0x00,0x6e,0x02,0x00,0x00,0x6d,0x02,0x00,0x00, -0x20,0x00,0x04,0x00,0x6f,0x02,0x00,0x00,0x0c,0x00,0x00,0x00, -0x6e,0x02,0x00,0x00,0x3b,0x00,0x04,0x00,0x6f,0x02,0x00,0x00, -0x70,0x02,0x00,0x00,0x0c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0x75,0x02,0x00,0x00,0x05,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x82,0x02,0x00,0x00, -0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00, -0x36,0x00,0x05,0x00,0x02,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x05,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0xbe,0x00,0x00,0x00, -0xbf,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x7f,0x01,0x00,0x00,0x80,0x01,0x00,0x00,0x07,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0xac,0x01,0x00,0x00,0xad,0x01,0x00,0x00, -0x07,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00, -0x0e,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, -0x0e,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x14,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x17,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x17,0x00,0x00,0x00, -0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x1e,0x00,0x00,0x00, -0x0f,0x00,0x00,0x00,0x17,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x21,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x24,0x00,0x00,0x00,0x18,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0x28,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x27,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x29,0x00,0x00,0x00, -0x28,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x2a,0x00,0x00,0x00,0x1e,0x00,0x00,0x00,0x29,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x2d,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x2f,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x30,0x00,0x00,0x00, -0x24,0x00,0x00,0x00,0x2f,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x30,0x00,0x00,0x00, -0x2a,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0x35,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x34,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x36,0x00,0x00,0x00, -0x35,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x38,0x00,0x00,0x00,0x36,0x00,0x00,0x00,0x37,0x00,0x00,0x00, -0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, -0x38,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x3b,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, -0x37,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00, -0x3f,0x00,0x00,0x00,0x3d,0x00,0x00,0x00,0x3e,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x3f,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x42,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x3b,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x47,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0x3b,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x0d,0x00,0x00,0x00,0x49,0x00,0x00,0x00,0x3d,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x4a,0x00,0x00,0x00,0x49,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x0d,0x00,0x00,0x00,0x4d,0x00,0x00,0x00,0x4c,0x00,0x00,0x00, -0x3e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x4e,0x00,0x00,0x00,0x4d,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x50,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, -0x4f,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x55,0x00,0x00,0x00,0x50,0x00,0x00,0x00,0x54,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x59,0x00,0x00,0x00, -0x50,0x00,0x00,0x00,0x58,0x00,0x00,0x00,0x89,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x5d,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, -0x4f,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x64,0x00,0x00,0x00,0x5d,0x00,0x00,0x00,0x63,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x68,0x00,0x00,0x00, -0x5d,0x00,0x00,0x00,0x67,0x00,0x00,0x00,0x89,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x6e,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, -0x6d,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x73,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x72,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x77,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x76,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x78,0x00,0x00,0x00,0x77,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x79,0x00,0x00,0x00, -0x47,0x00,0x00,0x00,0x78,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x7c,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x7b,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x7d,0x00,0x00,0x00,0x7c,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x7f,0x00,0x00,0x00,0x47,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x82,0x00,0x00,0x00,0x7f,0x00,0x00,0x00,0x78,0x00,0x00,0x00, -0x0c,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x83,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x26,0x00,0x00,0x00,0x7d,0x00,0x00,0x00, -0x82,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0x87,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x86,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x88,0x00,0x00,0x00, -0x87,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x89,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x88,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8b,0x00,0x00,0x00, -0x42,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x8d,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x8c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x8e,0x00,0x00,0x00,0x8d,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x8f,0x00,0x00,0x00,0x8b,0x00,0x00,0x00, -0x8e,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x90,0x00,0x00,0x00,0x89,0x00,0x00,0x00,0x8f,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x92,0x00,0x00,0x00, -0x90,0x00,0x00,0x00,0x79,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x93,0x00,0x00,0x00,0x92,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0x98,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x97,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x99,0x00,0x00,0x00, -0x98,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x9a,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x99,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9d,0x00,0x00,0x00, -0x4a,0x00,0x00,0x00,0x9c,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x9f,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x9e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xa0,0x00,0x00,0x00,0x9f,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xa1,0x00,0x00,0x00,0x9d,0x00,0x00,0x00, -0xa0,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xa2,0x00,0x00,0x00,0x9a,0x00,0x00,0x00,0xa1,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa4,0x00,0x00,0x00, -0xa2,0x00,0x00,0x00,0x79,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xa5,0x00,0x00,0x00,0xa4,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xa7,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xa7,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0x94,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, -0x05,0x00,0x00,0x00,0xc6,0x00,0x00,0x00,0xa8,0x00,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00,0xb8,0x00,0x00,0x00, -0x94,0x02,0x00,0x00,0xb6,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0xa9,0x00,0x00,0x00,0xa8,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xb8,0x00,0x00,0x00,0xa8,0x00,0x00,0x00, -0xa9,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xa8,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0xc3,0x00,0x00,0x00, -0xbf,0x00,0x00,0x00,0x94,0x02,0x00,0x00,0x3e,0x00,0x03,0x00, -0xc3,0x00,0x00,0x00,0xc1,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xc6,0x00,0x00,0x00,0x94,0x02,0x00,0x00, -0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xa7,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xa9,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xc9,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xc9,0x00,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xad,0x02,0x00,0x00, -0xa5,0x00,0x00,0x00,0xa9,0x00,0x00,0x00,0x64,0x01,0x00,0x00, -0xcc,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xa9,0x02,0x00,0x00,0x93,0x00,0x00,0x00,0xa9,0x00,0x00,0x00, -0x61,0x01,0x00,0x00,0xcc,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0x95,0x02,0x00,0x00,0x79,0x00,0x00,0x00, -0xa9,0x00,0x00,0x00,0x0f,0x02,0x00,0x00,0xcc,0x00,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00,0xd0,0x00,0x00,0x00, -0x95,0x02,0x00,0x00,0x83,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0xcb,0x00,0x00,0x00,0xcc,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xd0,0x00,0x00,0x00,0xca,0x00,0x00,0x00, -0xcb,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xca,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xd2,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xd2,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xa5,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0xca,0x00,0x00,0x00, -0x1a,0x01,0x00,0x00,0xd5,0x00,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb7,0x00,0x00,0x00,0xd8,0x00,0x00,0x00,0xa5,0x02,0x00,0x00, -0x37,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xd4,0x00,0x00,0x00, -0xd5,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0xd8,0x00,0x00,0x00,0xd3,0x00,0x00,0x00,0xd4,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xd3,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xdc,0x00,0x00,0x00,0x8b,0x00,0x00,0x00, -0x73,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xde,0x00,0x00,0x00,0xdc,0x00,0x00,0x00,0xa5,0x02,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00,0xe1,0x00,0x00,0x00, -0xde,0x00,0x00,0x00,0x36,0x00,0x00,0x00,0xf7,0x00,0x03,0x00, -0xe3,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0xe1,0x00,0x00,0x00,0xe2,0x00,0x00,0x00,0xe3,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xe2,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xe6,0x00,0x00,0x00,0x95,0x02,0x00,0x00, -0x6e,0x00,0x00,0x00,0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00, -0xe8,0x00,0x00,0x00,0xe6,0x00,0x00,0x00,0x83,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xe3,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xe3,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0xb7,0x00,0x00,0x00, -0xe9,0x00,0x00,0x00,0xe1,0x00,0x00,0x00,0xd3,0x00,0x00,0x00, -0xe8,0x00,0x00,0x00,0xe2,0x00,0x00,0x00,0xf7,0x00,0x03,0x00, -0xeb,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0xe9,0x00,0x00,0x00,0xea,0x00,0x00,0x00,0x0b,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xea,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xf3,0x00,0x00,0x00,0x73,0x00,0x00,0x00, -0xa5,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xf5,0x00,0x00,0x00,0xf3,0x00,0x00,0x00,0xf4,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xf7,0x00,0x00,0x00, -0xf5,0x00,0x00,0x00,0x6e,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x02,0x01,0x00,0x00,0xf3,0x00,0x00,0x00, -0x8e,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x03,0x01,0x00,0x00,0xa9,0x02,0x00,0x00,0x02,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x05,0x01,0x00,0x00, -0x03,0x01,0x00,0x00,0x6e,0x00,0x00,0x00,0x41,0x00,0x06,0x00, -0x06,0x01,0x00,0x00,0x07,0x01,0x00,0x00,0xfb,0x00,0x00,0x00, -0x34,0x00,0x00,0x00,0x05,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, -0xb9,0x00,0x00,0x00,0x08,0x01,0x00,0x00,0x07,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0x09,0x01,0x00,0x00,0x0a,0x01,0x00,0x00, -0xf0,0x00,0x00,0x00,0xf7,0x00,0x00,0x00,0x3e,0x00,0x03,0x00, -0x0a,0x01,0x00,0x00,0x08,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0xeb,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x0b,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x0e,0x01,0x00,0x00, -0x73,0x00,0x00,0x00,0xa5,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x10,0x01,0x00,0x00,0x0e,0x01,0x00,0x00, -0x0f,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x12,0x01,0x00,0x00,0x10,0x01,0x00,0x00,0x6e,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x09,0x01,0x00,0x00,0x13,0x01,0x00,0x00, -0xf0,0x00,0x00,0x00,0x12,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0x13,0x01,0x00,0x00,0xc1,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xeb,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xeb,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xd5,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xd5,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x1a,0x01,0x00,0x00,0xa5,0x02,0x00,0x00,0x18,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0xd2,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xd4,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x1c,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x1c,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xa6,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, -0xd4,0x00,0x00,0x00,0x5d,0x01,0x00,0x00,0x1f,0x01,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00,0x22,0x01,0x00,0x00, -0xa6,0x02,0x00,0x00,0x9c,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0x1e,0x01,0x00,0x00,0x1f,0x01,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x22,0x01,0x00,0x00,0x1d,0x01,0x00,0x00, -0x1e,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x1d,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x26,0x01,0x00,0x00, -0x9d,0x00,0x00,0x00,0x73,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x28,0x01,0x00,0x00,0x26,0x01,0x00,0x00, -0xa6,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0x29,0x01,0x00,0x00,0x12,0x00,0x00,0x00,0xc5,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x2a,0x01,0x00,0x00, -0x29,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00, -0x2b,0x01,0x00,0x00,0x28,0x01,0x00,0x00,0x2a,0x01,0x00,0x00, -0xf7,0x00,0x03,0x00,0x2d,0x01,0x00,0x00,0x00,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x2b,0x01,0x00,0x00,0x2c,0x01,0x00,0x00, -0x2d,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x2c,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x30,0x01,0x00,0x00, -0x95,0x02,0x00,0x00,0x6e,0x00,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb7,0x00,0x00,0x00,0x32,0x01,0x00,0x00,0x30,0x01,0x00,0x00, -0x83,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x2d,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x2d,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, -0xb7,0x00,0x00,0x00,0x33,0x01,0x00,0x00,0x2b,0x01,0x00,0x00, -0x1d,0x01,0x00,0x00,0x32,0x01,0x00,0x00,0x2c,0x01,0x00,0x00, -0xf7,0x00,0x03,0x00,0x35,0x01,0x00,0x00,0x00,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x33,0x01,0x00,0x00,0x34,0x01,0x00,0x00, -0x53,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x34,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3d,0x01,0x00,0x00, -0x73,0x00,0x00,0x00,0xa6,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x3f,0x01,0x00,0x00,0x3d,0x01,0x00,0x00, -0x3e,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x41,0x01,0x00,0x00,0x3f,0x01,0x00,0x00,0x6e,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x4c,0x01,0x00,0x00, -0x3d,0x01,0x00,0x00,0xa0,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x4d,0x01,0x00,0x00,0xad,0x02,0x00,0x00, -0x4c,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x4f,0x01,0x00,0x00,0x4d,0x01,0x00,0x00,0x6e,0x00,0x00,0x00, -0x41,0x00,0x06,0x00,0x06,0x01,0x00,0x00,0x50,0x01,0x00,0x00, -0x45,0x01,0x00,0x00,0x34,0x00,0x00,0x00,0x4f,0x01,0x00,0x00, -0x3d,0x00,0x04,0x00,0xb9,0x00,0x00,0x00,0x51,0x01,0x00,0x00, -0x50,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0x09,0x01,0x00,0x00, -0x52,0x01,0x00,0x00,0x3a,0x01,0x00,0x00,0x41,0x01,0x00,0x00, -0x3e,0x00,0x03,0x00,0x52,0x01,0x00,0x00,0x51,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0x35,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x53,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x56,0x01,0x00,0x00,0x73,0x00,0x00,0x00,0xa6,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x58,0x01,0x00,0x00, -0x56,0x01,0x00,0x00,0x57,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x5a,0x01,0x00,0x00,0x58,0x01,0x00,0x00, -0x6e,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x09,0x01,0x00,0x00, -0x5b,0x01,0x00,0x00,0x3a,0x01,0x00,0x00,0x5a,0x01,0x00,0x00, -0x3e,0x00,0x03,0x00,0x5b,0x01,0x00,0x00,0xc1,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x35,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x35,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x1f,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x1f,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x5d,0x01,0x00,0x00,0xa6,0x02,0x00,0x00, -0x18,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x1c,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x1e,0x01,0x00,0x00,0xe0,0x00,0x04,0x00, -0x0c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x5e,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x61,0x01,0x00,0x00, -0xa9,0x02,0x00,0x00,0x5f,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x64,0x01,0x00,0x00,0xad,0x02,0x00,0x00, -0x62,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x66,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x66,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xaf,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, -0x1e,0x01,0x00,0x00,0x0d,0x02,0x00,0x00,0x69,0x01,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00,0x6c,0x01,0x00,0x00, -0xaf,0x02,0x00,0x00,0x6c,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0x68,0x01,0x00,0x00,0x69,0x01,0x00,0x00,0x00,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x6c,0x01,0x00,0x00,0x67,0x01,0x00,0x00, -0x68,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x67,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0x6e,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x6e,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xb3,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0x67,0x01,0x00,0x00, -0x99,0x01,0x00,0x00,0x71,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb7,0x00,0x00,0x00,0x74,0x01,0x00,0x00,0xb3,0x02,0x00,0x00, -0x60,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x70,0x01,0x00,0x00, -0x71,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x74,0x01,0x00,0x00,0x6f,0x01,0x00,0x00,0x70,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x6f,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0x76,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x76,0x01,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xc5,0x02,0x00,0x00, -0x3e,0x00,0x00,0x00,0x6f,0x01,0x00,0x00,0x97,0x01,0x00,0x00, -0x77,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00, -0x7c,0x01,0x00,0x00,0xc5,0x02,0x00,0x00,0x62,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0x78,0x01,0x00,0x00,0x77,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x7c,0x01,0x00,0x00, -0x77,0x01,0x00,0x00,0x78,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x77,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x82,0x01,0x00,0x00,0xb3,0x02,0x00,0x00,0x62,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x84,0x01,0x00,0x00, -0x82,0x01,0x00,0x00,0xc5,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x86,0x01,0x00,0x00,0x55,0x00,0x00,0x00, -0x53,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x88,0x01,0x00,0x00,0xb3,0x02,0x00,0x00,0x61,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x89,0x01,0x00,0x00, -0x86,0x01,0x00,0x00,0x88,0x01,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x8b,0x01,0x00,0x00,0x64,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x8c,0x01,0x00,0x00,0x89,0x01,0x00,0x00,0x8b,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8e,0x01,0x00,0x00, -0x8c,0x01,0x00,0x00,0xc5,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x90,0x01,0x00,0x00,0x8e,0x01,0x00,0x00, -0x8f,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x92,0x01,0x00,0x00,0x90,0x01,0x00,0x00,0xaf,0x02,0x00,0x00, -0x41,0x00,0x05,0x00,0x09,0x01,0x00,0x00,0x93,0x01,0x00,0x00, -0xf0,0x00,0x00,0x00,0x92,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, -0xb9,0x00,0x00,0x00,0x94,0x01,0x00,0x00,0x93,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0x95,0x01,0x00,0x00, -0x80,0x01,0x00,0x00,0x84,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0x95,0x01,0x00,0x00,0x94,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x97,0x01,0x00,0x00,0xc5,0x02,0x00,0x00, -0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x76,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x78,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0x71,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x71,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x99,0x01,0x00,0x00, -0xb3,0x02,0x00,0x00,0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x6e,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x70,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0x9b,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x9b,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xb4,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0x70,0x01,0x00,0x00, -0xc7,0x01,0x00,0x00,0x9e,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb7,0x00,0x00,0x00,0xa1,0x01,0x00,0x00,0xb4,0x02,0x00,0x00, -0xb4,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x9d,0x01,0x00,0x00, -0x9e,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0xa1,0x01,0x00,0x00,0x9c,0x01,0x00,0x00,0x9d,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x9c,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0xa3,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xa3,0x01,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xc2,0x02,0x00,0x00, -0x3e,0x00,0x00,0x00,0x9c,0x01,0x00,0x00,0xc5,0x01,0x00,0x00, -0xa4,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00, -0xa9,0x01,0x00,0x00,0xc2,0x02,0x00,0x00,0xb1,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0xa5,0x01,0x00,0x00,0xa4,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xa9,0x01,0x00,0x00, -0xa4,0x01,0x00,0x00,0xa5,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xa4,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xaf,0x01,0x00,0x00,0xb4,0x02,0x00,0x00,0xb1,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xb1,0x01,0x00,0x00, -0xaf,0x01,0x00,0x00,0xc2,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xb3,0x01,0x00,0x00,0x59,0x00,0x00,0x00, -0xae,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xb6,0x01,0x00,0x00,0xb4,0x02,0x00,0x00,0xb5,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xb7,0x01,0x00,0x00, -0xb3,0x01,0x00,0x00,0xb6,0x01,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xb9,0x01,0x00,0x00,0x68,0x00,0x00,0x00, -0xb1,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xba,0x01,0x00,0x00,0xb7,0x01,0x00,0x00,0xb9,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xbc,0x01,0x00,0x00, -0xba,0x01,0x00,0x00,0xc2,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xbe,0x01,0x00,0x00,0xbc,0x01,0x00,0x00, -0xbd,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xc0,0x01,0x00,0x00,0xbe,0x01,0x00,0x00,0xaf,0x02,0x00,0x00, -0x41,0x00,0x05,0x00,0x09,0x01,0x00,0x00,0xc1,0x01,0x00,0x00, -0x3a,0x01,0x00,0x00,0xc0,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, -0xb9,0x00,0x00,0x00,0xc2,0x01,0x00,0x00,0xc1,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0xc3,0x01,0x00,0x00, -0xad,0x01,0x00,0x00,0xb1,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0xc3,0x01,0x00,0x00,0xc2,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xc5,0x01,0x00,0x00,0xc2,0x02,0x00,0x00, -0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xa3,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xa5,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0x9e,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x9e,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xc7,0x01,0x00,0x00, -0xb4,0x02,0x00,0x00,0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x9b,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x9d,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0xc9,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xc9,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xb5,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0x9d,0x01,0x00,0x00, -0x0b,0x02,0x00,0x00,0xcc,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb7,0x00,0x00,0x00,0xcf,0x01,0x00,0x00,0xb5,0x02,0x00,0x00, -0xb4,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xcb,0x01,0x00,0x00, -0xcc,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0xcf,0x01,0x00,0x00,0xca,0x01,0x00,0x00,0xcb,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xca,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0xd1,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xd1,0x01,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xb9,0x02,0x00,0x00, -0x3e,0x00,0x00,0x00,0xca,0x01,0x00,0x00,0x09,0x02,0x00,0x00, -0xd4,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00, -0xd7,0x01,0x00,0x00,0xb9,0x02,0x00,0x00,0x60,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0xd3,0x01,0x00,0x00,0xd4,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xd7,0x01,0x00,0x00, -0xd2,0x01,0x00,0x00,0xd3,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xd2,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xd9,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xd9,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xbb,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, -0xd2,0x01,0x00,0x00,0x07,0x02,0x00,0x00,0xdc,0x01,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00,0xdf,0x01,0x00,0x00, -0xbb,0x02,0x00,0x00,0xb1,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0xdb,0x01,0x00,0x00,0xdc,0x01,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xdf,0x01,0x00,0x00,0xda,0x01,0x00,0x00, -0xdb,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xda,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0xe1,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xe1,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xbd,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0xda,0x01,0x00,0x00, -0x05,0x02,0x00,0x00,0xe2,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb7,0x00,0x00,0x00,0xe7,0x01,0x00,0x00,0xbd,0x02,0x00,0x00, -0x62,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xe3,0x01,0x00,0x00, -0xe2,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0xe7,0x01,0x00,0x00,0xe2,0x01,0x00,0x00,0xe3,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xe2,0x01,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xe9,0x01,0x00,0x00,0xb5,0x02,0x00,0x00, -0xb1,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xeb,0x01,0x00,0x00,0xe9,0x01,0x00,0x00,0xbb,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xed,0x01,0x00,0x00, -0xeb,0x01,0x00,0x00,0xec,0x01,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xef,0x01,0x00,0x00,0xb9,0x02,0x00,0x00, -0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xf0,0x01,0x00,0x00,0xed,0x01,0x00,0x00,0xef,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xf2,0x01,0x00,0x00, -0xf0,0x01,0x00,0x00,0xbd,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xf6,0x01,0x00,0x00,0xef,0x01,0x00,0x00, -0xbd,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, -0xf7,0x01,0x00,0x00,0x80,0x01,0x00,0x00,0xf6,0x01,0x00,0x00, -0x3d,0x00,0x04,0x00,0xb9,0x00,0x00,0x00,0xf8,0x01,0x00,0x00, -0xf7,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, -0xfd,0x01,0x00,0x00,0xad,0x01,0x00,0x00,0xeb,0x01,0x00,0x00, -0x3d,0x00,0x04,0x00,0xb9,0x00,0x00,0x00,0xfe,0x01,0x00,0x00, -0xfd,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, -0x00,0x02,0x00,0x00,0xbf,0x00,0x00,0x00,0xf2,0x01,0x00,0x00, -0x3d,0x00,0x04,0x00,0xb9,0x00,0x00,0x00,0x01,0x02,0x00,0x00, -0x00,0x02,0x00,0x00,0x0c,0x00,0x08,0x00,0xb9,0x00,0x00,0x00, -0x02,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00, -0xf8,0x01,0x00,0x00,0xfe,0x01,0x00,0x00,0x01,0x02,0x00,0x00, -0x3e,0x00,0x03,0x00,0x00,0x02,0x00,0x00,0x02,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x05,0x02,0x00,0x00, -0xbd,0x02,0x00,0x00,0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xe1,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xe3,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0xdc,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xdc,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x07,0x02,0x00,0x00,0xbb,0x02,0x00,0x00,0xc5,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xd9,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xdb,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xd4,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xd4,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x09,0x02,0x00,0x00,0xb9,0x02,0x00,0x00, -0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xd1,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xd3,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0xcc,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xcc,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x0b,0x02,0x00,0x00, -0xb5,0x02,0x00,0x00,0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xc9,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xcb,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0x69,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x69,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x0d,0x02,0x00,0x00,0xaf,0x02,0x00,0x00,0xc5,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x66,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x68,0x01,0x00,0x00,0xe0,0x00,0x04,0x00,0x0c,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x5e,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0xcc,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xcc,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x0f,0x02,0x00,0x00, -0x95,0x02,0x00,0x00,0x6c,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xc9,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xcb,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x14,0x02,0x00,0x00, -0x55,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x15,0x02,0x00,0x00,0x8b,0x00,0x00,0x00, -0x14,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x1a,0x02,0x00,0x00,0x59,0x00,0x00,0x00,0xae,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x1b,0x02,0x00,0x00, -0x9d,0x00,0x00,0x00,0x1a,0x02,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x20,0x02,0x00,0x00,0x12,0x00,0x00,0x00, -0x1f,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x21,0x02,0x00,0x00,0x20,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x22,0x02,0x00,0x00,0x0f,0x00,0x00,0x00, -0x21,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x26,0x02,0x00,0x00,0x47,0x00,0x00,0x00,0x21,0x02,0x00,0x00, -0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00,0x28,0x02,0x00,0x00, -0x27,0x02,0x00,0x00,0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x29,0x02,0x00,0x00,0x28,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x2a,0x02,0x00,0x00, -0x26,0x02,0x00,0x00,0x29,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x2b,0x02,0x00,0x00,0x22,0x02,0x00,0x00, -0x2a,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x2d,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x2d,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0x96,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, -0xcb,0x00,0x00,0x00,0x93,0x02,0x00,0x00,0x30,0x02,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00,0x33,0x02,0x00,0x00, -0x96,0x02,0x00,0x00,0xb4,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0x2f,0x02,0x00,0x00,0x30,0x02,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x33,0x02,0x00,0x00,0x2e,0x02,0x00,0x00, -0x2f,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x2e,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0x35,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x35,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0x97,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0x2e,0x02,0x00,0x00, -0x91,0x02,0x00,0x00,0x38,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb7,0x00,0x00,0x00,0x3b,0x02,0x00,0x00,0x97,0x02,0x00,0x00, -0x60,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x37,0x02,0x00,0x00, -0x38,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x3b,0x02,0x00,0x00,0x36,0x02,0x00,0x00,0x37,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x36,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x3f,0x02,0x00,0x00,0x97,0x02,0x00,0x00, -0x61,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x40,0x02,0x00,0x00,0x15,0x02,0x00,0x00,0x3f,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x42,0x02,0x00,0x00, -0x64,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x43,0x02,0x00,0x00,0x40,0x02,0x00,0x00, -0x42,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x47,0x02,0x00,0x00,0x96,0x02,0x00,0x00,0xb5,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x48,0x02,0x00,0x00, -0x1b,0x02,0x00,0x00,0x47,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x4a,0x02,0x00,0x00,0x68,0x00,0x00,0x00, -0xb1,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x4b,0x02,0x00,0x00,0x48,0x02,0x00,0x00,0x4a,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0x4d,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x4d,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0x99,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0x36,0x02,0x00,0x00, -0x8f,0x02,0x00,0x00,0x50,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb7,0x00,0x00,0x00,0x53,0x02,0x00,0x00,0x99,0x02,0x00,0x00, -0xb1,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x4f,0x02,0x00,0x00, -0x50,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x53,0x02,0x00,0x00,0x4e,0x02,0x00,0x00,0x4f,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x4e,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x55,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x55,0x02,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x9b,0x02,0x00,0x00, -0x3e,0x00,0x00,0x00,0x4e,0x02,0x00,0x00,0x8d,0x02,0x00,0x00, -0x58,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00, -0x5b,0x02,0x00,0x00,0x9b,0x02,0x00,0x00,0x62,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0x57,0x02,0x00,0x00,0x58,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x5b,0x02,0x00,0x00, -0x56,0x02,0x00,0x00,0x57,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x56,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x5e,0x02,0x00,0x00,0x43,0x02,0x00,0x00,0x9b,0x02,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00,0x61,0x02,0x00,0x00, -0x5e,0x02,0x00,0x00,0x36,0x00,0x00,0x00,0xf7,0x00,0x03,0x00, -0x63,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x61,0x02,0x00,0x00,0x62,0x02,0x00,0x00,0x63,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x62,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x66,0x02,0x00,0x00,0x4b,0x02,0x00,0x00, -0x99,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0x67,0x02,0x00,0x00,0x12,0x00,0x00,0x00,0xc5,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x68,0x02,0x00,0x00, -0x67,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00, -0x69,0x02,0x00,0x00,0x66,0x02,0x00,0x00,0x68,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0x63,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x63,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0xb7,0x00,0x00,0x00, -0x6a,0x02,0x00,0x00,0x61,0x02,0x00,0x00,0x56,0x02,0x00,0x00, -0x69,0x02,0x00,0x00,0x62,0x02,0x00,0x00,0xf7,0x00,0x03,0x00, -0x6c,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x6a,0x02,0x00,0x00,0x6b,0x02,0x00,0x00,0x6c,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x6b,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x74,0x02,0x00,0x00,0x4b,0x02,0x00,0x00, -0x99,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0x76,0x02,0x00,0x00,0x12,0x00,0x00,0x00,0x75,0x02,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x77,0x02,0x00,0x00, -0x76,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x78,0x02,0x00,0x00,0x74,0x02,0x00,0x00,0x77,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x79,0x02,0x00,0x00, -0x2b,0x02,0x00,0x00,0x78,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x7b,0x02,0x00,0x00,0x79,0x02,0x00,0x00, -0x43,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x7d,0x02,0x00,0x00,0x7b,0x02,0x00,0x00,0x9b,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x7f,0x02,0x00,0x00, -0x96,0x02,0x00,0x00,0xb1,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x81,0x02,0x00,0x00,0x7f,0x02,0x00,0x00, -0x99,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x83,0x02,0x00,0x00,0x81,0x02,0x00,0x00,0x82,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x85,0x02,0x00,0x00, -0x97,0x02,0x00,0x00,0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x86,0x02,0x00,0x00,0x83,0x02,0x00,0x00, +0x3e,0x00,0x00,0x00,0x59,0x02,0x00,0x00,0xb2,0x02,0x00,0x00, +0x73,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00, +0x76,0x02,0x00,0x00,0xbe,0x02,0x00,0x00,0xbb,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0x72,0x02,0x00,0x00,0x73,0x02,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x76,0x02,0x00,0x00, +0x71,0x02,0x00,0x00,0x72,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x71,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x78,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x78,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xc0,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, +0x71,0x02,0x00,0x00,0xb0,0x02,0x00,0x00,0x7b,0x02,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00,0x7e,0x02,0x00,0x00, +0xc0,0x02,0x00,0x00,0x62,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0x7a,0x02,0x00,0x00,0x7b,0x02,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x7e,0x02,0x00,0x00,0x79,0x02,0x00,0x00, +0x7a,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x79,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x81,0x02,0x00,0x00, +0x66,0x02,0x00,0x00,0xc0,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc1,0x00,0x00,0x00,0x84,0x02,0x00,0x00,0x81,0x02,0x00,0x00, +0x36,0x00,0x00,0x00,0xf7,0x00,0x03,0x00,0x86,0x02,0x00,0x00, +0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x84,0x02,0x00,0x00, +0x85,0x02,0x00,0x00,0x86,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, 0x85,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x88,0x02,0x00,0x00,0x86,0x02,0x00,0x00,0x9b,0x02,0x00,0x00, -0x41,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0x89,0x02,0x00,0x00, -0xbf,0x00,0x00,0x00,0x88,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, -0xb9,0x00,0x00,0x00,0x8a,0x02,0x00,0x00,0x89,0x02,0x00,0x00, -0x41,0x00,0x06,0x00,0x06,0x01,0x00,0x00,0x8b,0x02,0x00,0x00, -0x70,0x02,0x00,0x00,0x34,0x00,0x00,0x00,0x7d,0x02,0x00,0x00, -0x3e,0x00,0x03,0x00,0x8b,0x02,0x00,0x00,0x8a,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0x6c,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x6c,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x58,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x58,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x8d,0x02,0x00,0x00,0x9b,0x02,0x00,0x00, -0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x55,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x57,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x50,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x50,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8f,0x02,0x00,0x00, -0x99,0x02,0x00,0x00,0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x4d,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x4f,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0x38,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x38,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x91,0x02,0x00,0x00,0x97,0x02,0x00,0x00,0xc5,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x35,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x37,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x30,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x30,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x93,0x02,0x00,0x00,0x96,0x02,0x00,0x00, -0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x2d,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x2f,0x02,0x00,0x00,0xfd,0x00,0x01,0x00, -0x38,0x00,0x01,0x00, -}; -const uint64_t matmul_f32_l_fp32_len = 10048; - -unsigned char matmul_f32_m_data[] = { -0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, -0xcd,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, -0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x09,0x00,0x00,0x00, -0x0b,0x00,0x06,0x00,0x01,0x00,0x00,0x00,0x47,0x4c,0x53,0x4c, -0x2e,0x73,0x74,0x64,0x2e,0x34,0x35,0x30,0x00,0x00,0x00,0x00, -0x0e,0x00,0x03,0x00,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x0f,0x00,0x0f,0x00,0x05,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x6d,0x61,0x69,0x6e,0x00,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x3d,0x00,0x00,0x00,0x4c,0x00,0x00,0x00, -0xf1,0x00,0x00,0x00,0xfc,0x00,0x00,0x00,0x3d,0x01,0x00,0x00, -0x48,0x01,0x00,0x00,0x2e,0x02,0x00,0x00,0x77,0x02,0x00,0x00, -0x10,0x00,0x06,0x00,0x04,0x00,0x00,0x00,0x11,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x0b,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x1c,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x10,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x08,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x03,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x10,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x14,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x18,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x07,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x10,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x24,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x0a,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x10,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x2c,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x30,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x0d,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0x47,0x00,0x03,0x00, -0x10,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x37,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x3d,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x1a,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x4c,0x00,0x00,0x00, -0x0b,0x00,0x00,0x00,0x1b,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x4f,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x53,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x60,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x62,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x07,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x6c,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x03,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x9c,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0xae,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x05,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0xb1,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x08,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xf9,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00, -0xfa,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0xfa,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, -0xfa,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0xfc,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0xfc,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x17,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x18,0x01,0x00,0x00,0x0b,0x00,0x00,0x00,0x19,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x45,0x01,0x00,0x00,0x06,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0x46,0x01,0x00,0x00, -0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x46,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x46,0x01,0x00,0x00, -0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x48,0x01,0x00,0x00, -0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x48,0x01,0x00,0x00,0x21,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x2e,0x02,0x00,0x00,0x0b,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x74,0x02,0x00,0x00, -0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00, -0x75,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x19,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x75,0x02,0x00,0x00,0x00,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, -0x75,0x02,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x77,0x02,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x77,0x02,0x00,0x00,0x21,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x13,0x00,0x02,0x00,0x02,0x00,0x00,0x00, -0x21,0x00,0x03,0x00,0x03,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x15,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x17,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x0a,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x0d,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x1e,0x00,0x10,0x00,0x10,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x11,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x11,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x15,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, -0x14,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x15,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x0a,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, -0x27,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0x2d,0x00,0x00,0x00,0x07,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x34,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x37,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00,0x3d,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x3e,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x0a,0x00,0x00,0x00,0x4c,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x4f,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x53,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x54,0x00,0x00,0x00,0x86,0x00,0x00,0x00, -0x37,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x58,0x00,0x00,0x00,0x86,0x00,0x00,0x00, -0x37,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x32,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x61,0x00,0x00,0x00, -0x86,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x60,0x00,0x00,0x00, -0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x62,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x63,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x61,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x67,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x61,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x86,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x72,0x00,0x00,0x00,0x86,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0x76,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x7b,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, -0x86,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0x8c,0x00,0x00,0x00,0x03,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x97,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x9c,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0x9e,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xad,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00, -0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xae,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xaf,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x53,0x00,0x00,0x00, -0xae,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xb0,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x4f,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xb1,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xb2,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0xb0,0x00,0x00,0x00,0xb1,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xb3,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0xb2,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xb4,0x00,0x00,0x00,0x86,0x00,0x00,0x00, -0xaf,0x00,0x00,0x00,0xb3,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xb5,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0xad,0x00,0x00,0x00,0xb4,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xb6,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0xb5,0x00,0x00,0x00,0xb1,0x00,0x00,0x00,0x14,0x00,0x02,0x00, -0xb7,0x00,0x00,0x00,0x16,0x00,0x03,0x00,0xb9,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xba,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xbb,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0xba,0x00,0x00,0x00, -0xb4,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xbc,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0xbb,0x00,0x00,0x00, -0xb1,0x00,0x00,0x00,0x1c,0x00,0x04,0x00,0xbd,0x00,0x00,0x00, -0xb9,0x00,0x00,0x00,0xbc,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0xbe,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0xbd,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0xb9,0x00,0x00,0x00,0xc1,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0xc2,0x00,0x00,0x00, -0x07,0x00,0x00,0x00,0xb9,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0xc5,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x16,0x00,0x03,0x00,0xec,0x00,0x00,0x00,0x10,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xed,0x00,0x00,0x00, -0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xee,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0xed,0x00,0x00,0x00, -0x1c,0x00,0x04,0x00,0xef,0x00,0x00,0x00,0xec,0x00,0x00,0x00, -0xee,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0xf0,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0xef,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0xf0,0x00,0x00,0x00,0xf1,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xf5,0x00,0x00,0x00, -0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x1d,0x00,0x03,0x00,0xf9,0x00,0x00,0x00,0xb9,0x00,0x00,0x00, -0x1e,0x00,0x03,0x00,0xfa,0x00,0x00,0x00,0xf9,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0xfb,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0xfa,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0xfb,0x00,0x00,0x00, -0xfc,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x07,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0xb9,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x0b,0x01,0x00,0x00,0x04,0x00,0x00,0x00, -0xec,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x11,0x01,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0xec,0x00,0x00,0x00, -0x15,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x32,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x17,0x01,0x00,0x00,0x01,0x00,0x00,0x00, -0x33,0x00,0x06,0x00,0x09,0x00,0x00,0x00,0x18,0x01,0x00,0x00, -0x17,0x01,0x00,0x00,0x39,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x19,0x01,0x00,0x00, -0x51,0x00,0x00,0x00,0x18,0x01,0x00,0x00,0x00,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x1a,0x01,0x00,0x00, -0x84,0x00,0x00,0x00,0x19,0x01,0x00,0x00,0x39,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x1b,0x01,0x00,0x00, -0x86,0x00,0x00,0x00,0x1a,0x01,0x00,0x00,0x6c,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x39,0x01,0x00,0x00, -0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x3a,0x01,0x00,0x00, -0x84,0x00,0x00,0x00,0x9c,0x00,0x00,0x00,0x39,0x01,0x00,0x00, -0x1c,0x00,0x04,0x00,0x3b,0x01,0x00,0x00,0xec,0x00,0x00,0x00, -0x3a,0x01,0x00,0x00,0x20,0x00,0x04,0x00,0x3c,0x01,0x00,0x00, -0x04,0x00,0x00,0x00,0x3b,0x01,0x00,0x00,0x3b,0x00,0x04,0x00, -0x3c,0x01,0x00,0x00,0x3d,0x01,0x00,0x00,0x04,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x41,0x01,0x00,0x00, -0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x1d,0x00,0x03,0x00,0x45,0x01,0x00,0x00,0xb9,0x00,0x00,0x00, -0x1e,0x00,0x03,0x00,0x46,0x01,0x00,0x00,0x45,0x01,0x00,0x00, -0x20,0x00,0x04,0x00,0x47,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, -0x46,0x01,0x00,0x00,0x3b,0x00,0x04,0x00,0x47,0x01,0x00,0x00, -0x48,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x5b,0x01,0x00,0x00,0x80,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x62,0x01,0x00,0x00,0x08,0x01,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x63,0x01,0x00,0x00, -0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x66,0x01,0x00,0x00, -0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x81,0x01,0x00,0x00, -0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00, -0x1c,0x00,0x04,0x00,0x82,0x01,0x00,0x00,0xec,0x00,0x00,0x00, -0x81,0x01,0x00,0x00,0x20,0x00,0x04,0x00,0x83,0x01,0x00,0x00, -0x07,0x00,0x00,0x00,0x82,0x01,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x93,0x01,0x00,0x00,0x80,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x99,0x01,0x00,0x00,0x07,0x00,0x00,0x00,0xec,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xaf,0x01,0x00,0x00, -0x84,0x00,0x00,0x00,0xb4,0x00,0x00,0x00,0xb1,0x00,0x00,0x00, -0x1c,0x00,0x04,0x00,0xb0,0x01,0x00,0x00,0xec,0x00,0x00,0x00, -0xaf,0x01,0x00,0x00,0x20,0x00,0x04,0x00,0xb1,0x01,0x00,0x00, -0x07,0x00,0x00,0x00,0xb0,0x01,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xba,0x01,0x00,0x00,0x86,0x00,0x00,0x00, -0xae,0x00,0x00,0x00,0xb4,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xc2,0x01,0x00,0x00,0x80,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xf1,0x01,0x00,0x00,0x84,0x00,0x00,0x00, -0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0x26,0x02,0x00,0x00,0x0d,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00,0x2e,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0x1d,0x00,0x03,0x00,0x74,0x02,0x00,0x00, -0xb9,0x00,0x00,0x00,0x1e,0x00,0x03,0x00,0x75,0x02,0x00,0x00, -0x74,0x02,0x00,0x00,0x20,0x00,0x04,0x00,0x76,0x02,0x00,0x00, -0x0c,0x00,0x00,0x00,0x75,0x02,0x00,0x00,0x3b,0x00,0x04,0x00, -0x76,0x02,0x00,0x00,0x77,0x02,0x00,0x00,0x0c,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x7c,0x02,0x00,0x00, -0x05,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x89,0x02,0x00,0x00,0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x36,0x00,0x05,0x00,0x02,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x03,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0x05,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0xbe,0x00,0x00,0x00,0xbf,0x00,0x00,0x00,0x07,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x83,0x01,0x00,0x00,0x84,0x01,0x00,0x00, -0x07,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0xb1,0x01,0x00,0x00, -0xb2,0x01,0x00,0x00,0x07,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x0d,0x00,0x00,0x00,0x0e,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x0f,0x00,0x00,0x00,0x0e,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x14,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x17,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, -0x17,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x1e,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x17,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x22,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x22,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x24,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x27,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x29,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x2a,0x00,0x00,0x00,0x1e,0x00,0x00,0x00, -0x29,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x2d,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x2f,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x30,0x00,0x00,0x00,0x24,0x00,0x00,0x00,0x2f,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x32,0x00,0x00,0x00, -0x30,0x00,0x00,0x00,0x2a,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x35,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x34,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x36,0x00,0x00,0x00,0x35,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x38,0x00,0x00,0x00,0x36,0x00,0x00,0x00, -0x37,0x00,0x00,0x00,0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x3a,0x00,0x00,0x00,0x38,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3b,0x00,0x00,0x00, -0x3a,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x0d,0x00,0x00,0x00,0x3f,0x00,0x00,0x00,0x3d,0x00,0x00,0x00, -0x3e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0x3f,0x00,0x00,0x00,0x89,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x42,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x3b,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x47,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x3b,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00,0x49,0x00,0x00,0x00, -0x3d,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x4a,0x00,0x00,0x00,0x49,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00,0x4d,0x00,0x00,0x00, -0x4c,0x00,0x00,0x00,0x3e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x4d,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x50,0x00,0x00,0x00, -0x4e,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x89,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x55,0x00,0x00,0x00,0x50,0x00,0x00,0x00, -0x54,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x59,0x00,0x00,0x00,0x50,0x00,0x00,0x00,0x58,0x00,0x00,0x00, -0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x5d,0x00,0x00,0x00, -0x4e,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x89,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x64,0x00,0x00,0x00,0x5d,0x00,0x00,0x00, -0x63,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x68,0x00,0x00,0x00,0x5d,0x00,0x00,0x00,0x67,0x00,0x00,0x00, -0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x6e,0x00,0x00,0x00, -0x4e,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x73,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, -0x72,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0x77,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x76,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x78,0x00,0x00,0x00, -0x77,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x79,0x00,0x00,0x00,0x47,0x00,0x00,0x00,0x78,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x7c,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x7b,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x7d,0x00,0x00,0x00,0x7c,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x7f,0x00,0x00,0x00, -0x47,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x82,0x00,0x00,0x00,0x7f,0x00,0x00,0x00, -0x78,0x00,0x00,0x00,0x0c,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0x83,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x26,0x00,0x00,0x00, -0x7d,0x00,0x00,0x00,0x82,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x87,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x86,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x88,0x00,0x00,0x00,0x87,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x89,0x00,0x00,0x00,0x32,0x00,0x00,0x00, -0x88,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x8b,0x00,0x00,0x00,0x42,0x00,0x00,0x00,0x37,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x8d,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x8c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x8e,0x00,0x00,0x00,0x8d,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8f,0x00,0x00,0x00, -0x8b,0x00,0x00,0x00,0x8e,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x90,0x00,0x00,0x00,0x89,0x00,0x00,0x00, -0x8f,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x92,0x00,0x00,0x00,0x90,0x00,0x00,0x00,0x79,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x93,0x00,0x00,0x00, -0x92,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x98,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x97,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x99,0x00,0x00,0x00,0x98,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x9a,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, -0x99,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x9d,0x00,0x00,0x00,0x4a,0x00,0x00,0x00,0x9c,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x9f,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x9e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xa0,0x00,0x00,0x00,0x9f,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa1,0x00,0x00,0x00, -0x9d,0x00,0x00,0x00,0xa0,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xa2,0x00,0x00,0x00,0x9a,0x00,0x00,0x00, -0xa1,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xa4,0x00,0x00,0x00,0xa2,0x00,0x00,0x00,0x79,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa5,0x00,0x00,0x00, -0xa4,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xa7,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xa7,0x00,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x9b,0x02,0x00,0x00, -0x3e,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0xc6,0x00,0x00,0x00, -0xa8,0x00,0x00,0x00,0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00, -0xb8,0x00,0x00,0x00,0x9b,0x02,0x00,0x00,0xb6,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0xa9,0x00,0x00,0x00,0xa8,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xb8,0x00,0x00,0x00, -0xa8,0x00,0x00,0x00,0xa9,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xa8,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, -0xc3,0x00,0x00,0x00,0xbf,0x00,0x00,0x00,0x9b,0x02,0x00,0x00, -0x3e,0x00,0x03,0x00,0xc3,0x00,0x00,0x00,0xc1,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xc6,0x00,0x00,0x00, -0x9b,0x02,0x00,0x00,0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xa7,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xa9,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xc9,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xc9,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xb4,0x02,0x00,0x00,0xa5,0x00,0x00,0x00,0xa9,0x00,0x00,0x00, -0x68,0x01,0x00,0x00,0xcc,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xb0,0x02,0x00,0x00,0x93,0x00,0x00,0x00, -0xa9,0x00,0x00,0x00,0x65,0x01,0x00,0x00,0xcc,0x00,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x9c,0x02,0x00,0x00, -0x79,0x00,0x00,0x00,0xa9,0x00,0x00,0x00,0x16,0x02,0x00,0x00, -0xcc,0x00,0x00,0x00,0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00, -0xd0,0x00,0x00,0x00,0x9c,0x02,0x00,0x00,0x83,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0xcb,0x00,0x00,0x00,0xcc,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xd0,0x00,0x00,0x00, -0xca,0x00,0x00,0x00,0xcb,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xca,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xd2,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xd2,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xac,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, -0xca,0x00,0x00,0x00,0x1d,0x01,0x00,0x00,0xd5,0x00,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00,0xd8,0x00,0x00,0x00, -0xac,0x02,0x00,0x00,0x37,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0xd4,0x00,0x00,0x00,0xd5,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xd8,0x00,0x00,0x00,0xd3,0x00,0x00,0x00, -0xd4,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xd3,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xdc,0x00,0x00,0x00, -0x8b,0x00,0x00,0x00,0x73,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xde,0x00,0x00,0x00,0xdc,0x00,0x00,0x00, -0xac,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00, -0xe1,0x00,0x00,0x00,0xde,0x00,0x00,0x00,0x36,0x00,0x00,0x00, -0xf7,0x00,0x03,0x00,0xe3,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xe1,0x00,0x00,0x00,0xe2,0x00,0x00,0x00, -0xe3,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xe2,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xe6,0x00,0x00,0x00, -0x9c,0x02,0x00,0x00,0x6e,0x00,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb7,0x00,0x00,0x00,0xe8,0x00,0x00,0x00,0xe6,0x00,0x00,0x00, -0x83,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xe3,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xe3,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, -0xb7,0x00,0x00,0x00,0xe9,0x00,0x00,0x00,0xe1,0x00,0x00,0x00, -0xd3,0x00,0x00,0x00,0xe8,0x00,0x00,0x00,0xe2,0x00,0x00,0x00, -0xf7,0x00,0x03,0x00,0xeb,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xe9,0x00,0x00,0x00,0xea,0x00,0x00,0x00, -0x0d,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xea,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xf4,0x00,0x00,0x00, -0x73,0x00,0x00,0x00,0xac,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xf6,0x00,0x00,0x00,0xf4,0x00,0x00,0x00, -0xf5,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xf8,0x00,0x00,0x00,0xf6,0x00,0x00,0x00,0x6e,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x03,0x01,0x00,0x00, -0xf4,0x00,0x00,0x00,0x8e,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x04,0x01,0x00,0x00,0xb0,0x02,0x00,0x00, -0x03,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x06,0x01,0x00,0x00,0x04,0x01,0x00,0x00,0x6e,0x00,0x00,0x00, -0x41,0x00,0x06,0x00,0x07,0x01,0x00,0x00,0x08,0x01,0x00,0x00, -0xfc,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0x06,0x01,0x00,0x00, -0x3d,0x00,0x04,0x00,0xb9,0x00,0x00,0x00,0x09,0x01,0x00,0x00, -0x08,0x01,0x00,0x00,0x73,0x00,0x04,0x00,0xec,0x00,0x00,0x00, -0x0a,0x01,0x00,0x00,0x09,0x01,0x00,0x00,0x41,0x00,0x05,0x00, -0x0b,0x01,0x00,0x00,0x0c,0x01,0x00,0x00,0xf1,0x00,0x00,0x00, -0xf8,0x00,0x00,0x00,0x3e,0x00,0x03,0x00,0x0c,0x01,0x00,0x00, -0x0a,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xeb,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0x0d,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x10,0x01,0x00,0x00,0x73,0x00,0x00,0x00, -0xac,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x12,0x01,0x00,0x00,0x10,0x01,0x00,0x00,0x11,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x14,0x01,0x00,0x00, -0x12,0x01,0x00,0x00,0x6e,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x0b,0x01,0x00,0x00,0x16,0x01,0x00,0x00,0xf1,0x00,0x00,0x00, -0x14,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x16,0x01,0x00,0x00, -0x15,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xeb,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xeb,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xd5,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xd5,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x1d,0x01,0x00,0x00, -0xac,0x02,0x00,0x00,0x1b,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0xd2,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xd4,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x1f,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x1f,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xad,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0xd4,0x00,0x00,0x00, -0x61,0x01,0x00,0x00,0x22,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb7,0x00,0x00,0x00,0x25,0x01,0x00,0x00,0xad,0x02,0x00,0x00, -0x9c,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x21,0x01,0x00,0x00, -0x22,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x25,0x01,0x00,0x00,0x20,0x01,0x00,0x00,0x21,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x20,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x29,0x01,0x00,0x00,0x9d,0x00,0x00,0x00, -0x73,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x2b,0x01,0x00,0x00,0x29,0x01,0x00,0x00,0xad,0x02,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x2c,0x01,0x00,0x00, -0x12,0x00,0x00,0x00,0xc5,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x2d,0x01,0x00,0x00,0x2c,0x01,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00,0x2e,0x01,0x00,0x00, -0x2b,0x01,0x00,0x00,0x2d,0x01,0x00,0x00,0xf7,0x00,0x03,0x00, -0x30,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x2e,0x01,0x00,0x00,0x2f,0x01,0x00,0x00,0x30,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x2f,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x33,0x01,0x00,0x00,0x9c,0x02,0x00,0x00, -0x6e,0x00,0x00,0x00,0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00, -0x35,0x01,0x00,0x00,0x33,0x01,0x00,0x00,0x83,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x30,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x30,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0xb7,0x00,0x00,0x00, -0x36,0x01,0x00,0x00,0x2e,0x01,0x00,0x00,0x20,0x01,0x00,0x00, -0x35,0x01,0x00,0x00,0x2f,0x01,0x00,0x00,0xf7,0x00,0x03,0x00, -0x38,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x36,0x01,0x00,0x00,0x37,0x01,0x00,0x00,0x57,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x37,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x40,0x01,0x00,0x00,0x73,0x00,0x00,0x00, -0xad,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x42,0x01,0x00,0x00,0x40,0x01,0x00,0x00,0x41,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x44,0x01,0x00,0x00, -0x42,0x01,0x00,0x00,0x6e,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x4f,0x01,0x00,0x00,0x40,0x01,0x00,0x00, -0xa0,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x50,0x01,0x00,0x00,0xb4,0x02,0x00,0x00,0x4f,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x52,0x01,0x00,0x00, -0x50,0x01,0x00,0x00,0x6e,0x00,0x00,0x00,0x41,0x00,0x06,0x00, -0x07,0x01,0x00,0x00,0x53,0x01,0x00,0x00,0x48,0x01,0x00,0x00, -0x34,0x00,0x00,0x00,0x52,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, -0xb9,0x00,0x00,0x00,0x54,0x01,0x00,0x00,0x53,0x01,0x00,0x00, -0x73,0x00,0x04,0x00,0xec,0x00,0x00,0x00,0x55,0x01,0x00,0x00, -0x54,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0x0b,0x01,0x00,0x00, -0x56,0x01,0x00,0x00,0x3d,0x01,0x00,0x00,0x44,0x01,0x00,0x00, -0x3e,0x00,0x03,0x00,0x56,0x01,0x00,0x00,0x55,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0x38,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x57,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x5a,0x01,0x00,0x00,0x73,0x00,0x00,0x00,0xad,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x5c,0x01,0x00,0x00, -0x5a,0x01,0x00,0x00,0x5b,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x5e,0x01,0x00,0x00,0x5c,0x01,0x00,0x00, -0x6e,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x0b,0x01,0x00,0x00, -0x5f,0x01,0x00,0x00,0x3d,0x01,0x00,0x00,0x5e,0x01,0x00,0x00, -0x3e,0x00,0x03,0x00,0x5f,0x01,0x00,0x00,0x15,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0x38,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x38,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x22,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x22,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x61,0x01,0x00,0x00,0xad,0x02,0x00,0x00, -0x1b,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x1f,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x21,0x01,0x00,0x00,0xe0,0x00,0x04,0x00, -0x0c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x62,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x65,0x01,0x00,0x00, -0xb0,0x02,0x00,0x00,0x63,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x68,0x01,0x00,0x00,0xb4,0x02,0x00,0x00, -0x66,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x6a,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x6a,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xb6,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, -0x21,0x01,0x00,0x00,0x14,0x02,0x00,0x00,0x6d,0x01,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00,0x70,0x01,0x00,0x00, -0xb6,0x02,0x00,0x00,0x6c,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0x6c,0x01,0x00,0x00,0x6d,0x01,0x00,0x00,0x00,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x70,0x01,0x00,0x00,0x6b,0x01,0x00,0x00, -0x6c,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x6b,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0x72,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x72,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xba,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0x6b,0x01,0x00,0x00, -0x9e,0x01,0x00,0x00,0x75,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb7,0x00,0x00,0x00,0x78,0x01,0x00,0x00,0xba,0x02,0x00,0x00, -0x60,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x74,0x01,0x00,0x00, -0x75,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x78,0x01,0x00,0x00,0x73,0x01,0x00,0x00,0x74,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x73,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0x7a,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x7a,0x01,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xcc,0x02,0x00,0x00, -0x3e,0x00,0x00,0x00,0x73,0x01,0x00,0x00,0x9c,0x01,0x00,0x00, -0x7b,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00, -0x80,0x01,0x00,0x00,0xcc,0x02,0x00,0x00,0x62,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0x7c,0x01,0x00,0x00,0x7b,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x80,0x01,0x00,0x00, -0x7b,0x01,0x00,0x00,0x7c,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x7b,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x86,0x01,0x00,0x00,0xba,0x02,0x00,0x00,0x62,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x88,0x01,0x00,0x00, -0x86,0x01,0x00,0x00,0xcc,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x8a,0x01,0x00,0x00,0x55,0x00,0x00,0x00, -0x53,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x8c,0x01,0x00,0x00,0xba,0x02,0x00,0x00,0x61,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8d,0x01,0x00,0x00, -0x8a,0x01,0x00,0x00,0x8c,0x01,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x8f,0x01,0x00,0x00,0x64,0x00,0x00,0x00, +0x89,0x02,0x00,0x00,0x6e,0x02,0x00,0x00,0xbe,0x02,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x8a,0x02,0x00,0x00, +0x12,0x00,0x00,0x00,0xcf,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x8b,0x02,0x00,0x00,0x8a,0x02,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00,0x8c,0x02,0x00,0x00, +0x89,0x02,0x00,0x00,0x8b,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0x86,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x86,0x02,0x00,0x00, +0xf5,0x00,0x07,0x00,0xc1,0x00,0x00,0x00,0x8d,0x02,0x00,0x00, +0x84,0x02,0x00,0x00,0x79,0x02,0x00,0x00,0x8c,0x02,0x00,0x00, +0x85,0x02,0x00,0x00,0xf7,0x00,0x03,0x00,0x8f,0x02,0x00,0x00, +0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x8d,0x02,0x00,0x00, +0x8e,0x02,0x00,0x00,0x8f,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x8e,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x97,0x02,0x00,0x00,0x6e,0x02,0x00,0x00,0xbe,0x02,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x99,0x02,0x00,0x00, +0x12,0x00,0x00,0x00,0x98,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x9a,0x02,0x00,0x00,0x99,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9b,0x02,0x00,0x00, +0x97,0x02,0x00,0x00,0x9a,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x9c,0x02,0x00,0x00,0x4e,0x02,0x00,0x00, +0x9b,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x9e,0x02,0x00,0x00,0x9c,0x02,0x00,0x00,0x66,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa0,0x02,0x00,0x00, +0x9e,0x02,0x00,0x00,0xc0,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xa2,0x02,0x00,0x00,0xbb,0x02,0x00,0x00, +0xbb,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xa4,0x02,0x00,0x00,0xa2,0x02,0x00,0x00,0xbe,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa6,0x02,0x00,0x00, +0xa4,0x02,0x00,0x00,0xa5,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xa8,0x02,0x00,0x00,0xbc,0x02,0x00,0x00, 0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x90,0x01,0x00,0x00,0x8d,0x01,0x00,0x00,0x8f,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x92,0x01,0x00,0x00, -0x90,0x01,0x00,0x00,0xcc,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x94,0x01,0x00,0x00,0x92,0x01,0x00,0x00, -0x93,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x96,0x01,0x00,0x00,0x94,0x01,0x00,0x00,0xb6,0x02,0x00,0x00, -0x41,0x00,0x05,0x00,0x0b,0x01,0x00,0x00,0x97,0x01,0x00,0x00, -0xf1,0x00,0x00,0x00,0x96,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, -0xec,0x00,0x00,0x00,0x98,0x01,0x00,0x00,0x97,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0x99,0x01,0x00,0x00,0x9a,0x01,0x00,0x00, -0x84,0x01,0x00,0x00,0x88,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0x9a,0x01,0x00,0x00,0x98,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x9c,0x01,0x00,0x00,0xcc,0x02,0x00,0x00, -0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x7a,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x7c,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0x75,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x75,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9e,0x01,0x00,0x00, -0xba,0x02,0x00,0x00,0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x72,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x74,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0xa0,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xa0,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xbb,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0x74,0x01,0x00,0x00, -0xcc,0x01,0x00,0x00,0xa3,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb7,0x00,0x00,0x00,0xa6,0x01,0x00,0x00,0xbb,0x02,0x00,0x00, -0xb4,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xa2,0x01,0x00,0x00, -0xa3,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0xa6,0x01,0x00,0x00,0xa1,0x01,0x00,0x00,0xa2,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xa1,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0xa8,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xa8,0x01,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xc9,0x02,0x00,0x00, -0x3e,0x00,0x00,0x00,0xa1,0x01,0x00,0x00,0xca,0x01,0x00,0x00, -0xa9,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00, -0xae,0x01,0x00,0x00,0xc9,0x02,0x00,0x00,0xb1,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0xaa,0x01,0x00,0x00,0xa9,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xae,0x01,0x00,0x00, -0xa9,0x01,0x00,0x00,0xaa,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xa9,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xb4,0x01,0x00,0x00,0xbb,0x02,0x00,0x00,0xb1,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xb6,0x01,0x00,0x00, -0xb4,0x01,0x00,0x00,0xc9,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xb8,0x01,0x00,0x00,0x59,0x00,0x00,0x00, -0xae,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xbb,0x01,0x00,0x00,0xbb,0x02,0x00,0x00,0xba,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xbc,0x01,0x00,0x00, -0xb8,0x01,0x00,0x00,0xbb,0x01,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xbe,0x01,0x00,0x00,0x68,0x00,0x00,0x00, -0xb1,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xbf,0x01,0x00,0x00,0xbc,0x01,0x00,0x00,0xbe,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xc1,0x01,0x00,0x00, -0xbf,0x01,0x00,0x00,0xc9,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xc3,0x01,0x00,0x00,0xc1,0x01,0x00,0x00, -0xc2,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xc5,0x01,0x00,0x00,0xc3,0x01,0x00,0x00,0xb6,0x02,0x00,0x00, -0x41,0x00,0x05,0x00,0x0b,0x01,0x00,0x00,0xc6,0x01,0x00,0x00, -0x3d,0x01,0x00,0x00,0xc5,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, -0xec,0x00,0x00,0x00,0xc7,0x01,0x00,0x00,0xc6,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0x99,0x01,0x00,0x00,0xc8,0x01,0x00,0x00, -0xb2,0x01,0x00,0x00,0xb6,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0xc8,0x01,0x00,0x00,0xc7,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xca,0x01,0x00,0x00,0xc9,0x02,0x00,0x00, -0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xa8,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xaa,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0xa3,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xa3,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xcc,0x01,0x00,0x00, -0xbb,0x02,0x00,0x00,0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xa0,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xa2,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0xce,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xce,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xbc,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0xa2,0x01,0x00,0x00, -0x12,0x02,0x00,0x00,0xd1,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb7,0x00,0x00,0x00,0xd4,0x01,0x00,0x00,0xbc,0x02,0x00,0x00, -0xb4,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xd0,0x01,0x00,0x00, -0xd1,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0xd4,0x01,0x00,0x00,0xcf,0x01,0x00,0x00,0xd0,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xcf,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0xd6,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xd6,0x01,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xc0,0x02,0x00,0x00, -0x3e,0x00,0x00,0x00,0xcf,0x01,0x00,0x00,0x10,0x02,0x00,0x00, -0xd9,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00, -0xdc,0x01,0x00,0x00,0xc0,0x02,0x00,0x00,0x60,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0xd8,0x01,0x00,0x00,0xd9,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xdc,0x01,0x00,0x00, -0xd7,0x01,0x00,0x00,0xd8,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xd7,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xde,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xde,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xc2,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, -0xd7,0x01,0x00,0x00,0x0e,0x02,0x00,0x00,0xe1,0x01,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00,0xe4,0x01,0x00,0x00, -0xc2,0x02,0x00,0x00,0xb1,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0xe0,0x01,0x00,0x00,0xe1,0x01,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xe4,0x01,0x00,0x00,0xdf,0x01,0x00,0x00, -0xe0,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xdf,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0xe6,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xe6,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xc4,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0xdf,0x01,0x00,0x00, -0x0c,0x02,0x00,0x00,0xe7,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb7,0x00,0x00,0x00,0xec,0x01,0x00,0x00,0xc4,0x02,0x00,0x00, -0x62,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xe8,0x01,0x00,0x00, -0xe7,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0xec,0x01,0x00,0x00,0xe7,0x01,0x00,0x00,0xe8,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xe7,0x01,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xee,0x01,0x00,0x00,0xbc,0x02,0x00,0x00, -0xb1,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xf0,0x01,0x00,0x00,0xee,0x01,0x00,0x00,0xc2,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xf2,0x01,0x00,0x00, -0xf0,0x01,0x00,0x00,0xf1,0x01,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xf4,0x01,0x00,0x00,0xc0,0x02,0x00,0x00, -0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xf5,0x01,0x00,0x00,0xf2,0x01,0x00,0x00,0xf4,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xf7,0x01,0x00,0x00, -0xf5,0x01,0x00,0x00,0xc4,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xfb,0x01,0x00,0x00,0xf4,0x01,0x00,0x00, -0xc4,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x99,0x01,0x00,0x00, -0xfc,0x01,0x00,0x00,0x84,0x01,0x00,0x00,0xfb,0x01,0x00,0x00, -0x3d,0x00,0x04,0x00,0xec,0x00,0x00,0x00,0xfd,0x01,0x00,0x00, -0xfc,0x01,0x00,0x00,0x73,0x00,0x04,0x00,0xb9,0x00,0x00,0x00, -0xfe,0x01,0x00,0x00,0xfd,0x01,0x00,0x00,0x41,0x00,0x05,0x00, -0x99,0x01,0x00,0x00,0x03,0x02,0x00,0x00,0xb2,0x01,0x00,0x00, -0xf0,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0xec,0x00,0x00,0x00, -0x04,0x02,0x00,0x00,0x03,0x02,0x00,0x00,0x73,0x00,0x04,0x00, -0xb9,0x00,0x00,0x00,0x05,0x02,0x00,0x00,0x04,0x02,0x00,0x00, -0x41,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0x07,0x02,0x00,0x00, -0xbf,0x00,0x00,0x00,0xf7,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, -0xb9,0x00,0x00,0x00,0x08,0x02,0x00,0x00,0x07,0x02,0x00,0x00, -0x0c,0x00,0x08,0x00,0xb9,0x00,0x00,0x00,0x09,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0xfe,0x01,0x00,0x00, -0x05,0x02,0x00,0x00,0x08,0x02,0x00,0x00,0x3e,0x00,0x03,0x00, -0x07,0x02,0x00,0x00,0x09,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x0c,0x02,0x00,0x00,0xc4,0x02,0x00,0x00, -0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xe6,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xe8,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0xe1,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xe1,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x0e,0x02,0x00,0x00, -0xc2,0x02,0x00,0x00,0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xde,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xe0,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0xd9,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xd9,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x10,0x02,0x00,0x00,0xc0,0x02,0x00,0x00,0xc5,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xd6,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xd8,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xd1,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xd1,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x12,0x02,0x00,0x00,0xbc,0x02,0x00,0x00, -0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xce,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xd0,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0x6d,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x6d,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x14,0x02,0x00,0x00, -0xb6,0x02,0x00,0x00,0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x6a,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x6c,0x01,0x00,0x00, -0xe0,0x00,0x04,0x00,0x0c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x62,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xcc,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xcc,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x16,0x02,0x00,0x00,0x9c,0x02,0x00,0x00, -0x6c,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xc9,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xcb,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x1b,0x02,0x00,0x00,0x55,0x00,0x00,0x00, -0x53,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x1c,0x02,0x00,0x00,0x8b,0x00,0x00,0x00,0x1b,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x21,0x02,0x00,0x00, -0x59,0x00,0x00,0x00,0xae,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x22,0x02,0x00,0x00,0x9d,0x00,0x00,0x00, -0x21,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0x27,0x02,0x00,0x00,0x12,0x00,0x00,0x00,0x26,0x02,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x28,0x02,0x00,0x00, -0x27,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x29,0x02,0x00,0x00,0x0f,0x00,0x00,0x00,0x28,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x2d,0x02,0x00,0x00, -0x47,0x00,0x00,0x00,0x28,0x02,0x00,0x00,0x41,0x00,0x05,0x00, -0x0d,0x00,0x00,0x00,0x2f,0x02,0x00,0x00,0x2e,0x02,0x00,0x00, -0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x30,0x02,0x00,0x00,0x2f,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x31,0x02,0x00,0x00,0x2d,0x02,0x00,0x00, -0x30,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x32,0x02,0x00,0x00,0x29,0x02,0x00,0x00,0x31,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0x34,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x34,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0x9d,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0xcb,0x00,0x00,0x00, -0x9a,0x02,0x00,0x00,0x37,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb7,0x00,0x00,0x00,0x3a,0x02,0x00,0x00,0x9d,0x02,0x00,0x00, -0xb4,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x36,0x02,0x00,0x00, -0x37,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x3a,0x02,0x00,0x00,0x35,0x02,0x00,0x00,0x36,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x35,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x3c,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x3c,0x02,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x9e,0x02,0x00,0x00, -0x3e,0x00,0x00,0x00,0x35,0x02,0x00,0x00,0x98,0x02,0x00,0x00, -0x3f,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00, -0x42,0x02,0x00,0x00,0x9e,0x02,0x00,0x00,0x60,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0x3e,0x02,0x00,0x00,0x3f,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x42,0x02,0x00,0x00, -0x3d,0x02,0x00,0x00,0x3e,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x3d,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x46,0x02,0x00,0x00,0x9e,0x02,0x00,0x00,0x61,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x47,0x02,0x00,0x00, -0x1c,0x02,0x00,0x00,0x46,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x49,0x02,0x00,0x00,0x64,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x4a,0x02,0x00,0x00,0x47,0x02,0x00,0x00,0x49,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x4e,0x02,0x00,0x00, -0x9d,0x02,0x00,0x00,0xba,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x4f,0x02,0x00,0x00,0x22,0x02,0x00,0x00, -0x4e,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x51,0x02,0x00,0x00,0x68,0x00,0x00,0x00,0xb1,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x52,0x02,0x00,0x00, -0x4f,0x02,0x00,0x00,0x51,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x54,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x54,0x02,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xa0,0x02,0x00,0x00, -0x3e,0x00,0x00,0x00,0x3d,0x02,0x00,0x00,0x96,0x02,0x00,0x00, -0x57,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00, -0x5a,0x02,0x00,0x00,0xa0,0x02,0x00,0x00,0xb1,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0x56,0x02,0x00,0x00,0x57,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x5a,0x02,0x00,0x00, -0x55,0x02,0x00,0x00,0x56,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x55,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x5c,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x5c,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xa2,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, -0x55,0x02,0x00,0x00,0x94,0x02,0x00,0x00,0x5f,0x02,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00,0x62,0x02,0x00,0x00, -0xa2,0x02,0x00,0x00,0x62,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0x5e,0x02,0x00,0x00,0x5f,0x02,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x62,0x02,0x00,0x00,0x5d,0x02,0x00,0x00, -0x5e,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x5d,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x65,0x02,0x00,0x00, -0x4a,0x02,0x00,0x00,0xa2,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb7,0x00,0x00,0x00,0x68,0x02,0x00,0x00,0x65,0x02,0x00,0x00, -0x36,0x00,0x00,0x00,0xf7,0x00,0x03,0x00,0x6a,0x02,0x00,0x00, -0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x68,0x02,0x00,0x00, -0x69,0x02,0x00,0x00,0x6a,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x69,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x6d,0x02,0x00,0x00,0x52,0x02,0x00,0x00,0xa0,0x02,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x6e,0x02,0x00,0x00, -0x12,0x00,0x00,0x00,0xc5,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x6f,0x02,0x00,0x00,0x6e,0x02,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00,0x70,0x02,0x00,0x00, -0x6d,0x02,0x00,0x00,0x6f,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x6a,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x6a,0x02,0x00,0x00, -0xf5,0x00,0x07,0x00,0xb7,0x00,0x00,0x00,0x71,0x02,0x00,0x00, -0x68,0x02,0x00,0x00,0x5d,0x02,0x00,0x00,0x70,0x02,0x00,0x00, -0x69,0x02,0x00,0x00,0xf7,0x00,0x03,0x00,0x73,0x02,0x00,0x00, -0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x71,0x02,0x00,0x00, -0x72,0x02,0x00,0x00,0x73,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x72,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x7b,0x02,0x00,0x00,0x52,0x02,0x00,0x00,0xa0,0x02,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x7d,0x02,0x00,0x00, -0x12,0x00,0x00,0x00,0x7c,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x7e,0x02,0x00,0x00,0x7d,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x7f,0x02,0x00,0x00, -0x7b,0x02,0x00,0x00,0x7e,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x80,0x02,0x00,0x00,0x32,0x02,0x00,0x00, -0x7f,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x82,0x02,0x00,0x00,0x80,0x02,0x00,0x00,0x4a,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x84,0x02,0x00,0x00, -0x82,0x02,0x00,0x00,0xa2,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x86,0x02,0x00,0x00,0x9d,0x02,0x00,0x00, -0xb1,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x88,0x02,0x00,0x00,0x86,0x02,0x00,0x00,0xa0,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8a,0x02,0x00,0x00, -0x88,0x02,0x00,0x00,0x89,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x8c,0x02,0x00,0x00,0x9e,0x02,0x00,0x00, -0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x8d,0x02,0x00,0x00,0x8a,0x02,0x00,0x00,0x8c,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8f,0x02,0x00,0x00, -0x8d,0x02,0x00,0x00,0xa2,0x02,0x00,0x00,0x41,0x00,0x05,0x00, -0xc2,0x00,0x00,0x00,0x90,0x02,0x00,0x00,0xbf,0x00,0x00,0x00, -0x8f,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0xb9,0x00,0x00,0x00, -0x91,0x02,0x00,0x00,0x90,0x02,0x00,0x00,0x41,0x00,0x06,0x00, -0x07,0x01,0x00,0x00,0x92,0x02,0x00,0x00,0x77,0x02,0x00,0x00, -0x34,0x00,0x00,0x00,0x84,0x02,0x00,0x00,0x3e,0x00,0x03,0x00, -0x92,0x02,0x00,0x00,0x91,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x73,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x73,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0x5f,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x5f,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x94,0x02,0x00,0x00,0xa2,0x02,0x00,0x00,0xc5,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x5c,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x5e,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x57,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x57,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x96,0x02,0x00,0x00,0xa0,0x02,0x00,0x00, -0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x54,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x56,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x3f,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x3f,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x98,0x02,0x00,0x00, -0x9e,0x02,0x00,0x00,0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x3c,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x3e,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0x37,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x37,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x9a,0x02,0x00,0x00,0x9d,0x02,0x00,0x00,0xc5,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x34,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x36,0x02,0x00,0x00,0xfd,0x00,0x01,0x00,0x38,0x00,0x01,0x00, +0xa9,0x02,0x00,0x00,0xa6,0x02,0x00,0x00,0xa8,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xab,0x02,0x00,0x00, +0xa9,0x02,0x00,0x00,0xc0,0x02,0x00,0x00,0x41,0x00,0x05,0x00, +0xcc,0x00,0x00,0x00,0xac,0x02,0x00,0x00,0xc9,0x00,0x00,0x00, +0xab,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0xc3,0x00,0x00,0x00, +0xad,0x02,0x00,0x00,0xac,0x02,0x00,0x00,0x41,0x00,0x06,0x00, +0x6f,0x01,0x00,0x00,0xae,0x02,0x00,0x00,0x93,0x02,0x00,0x00, +0x34,0x00,0x00,0x00,0xa0,0x02,0x00,0x00,0x3e,0x00,0x03,0x00, +0xae,0x02,0x00,0x00,0xad,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0x8f,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x8f,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0x7b,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x7b,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xb0,0x02,0x00,0x00,0xc0,0x02,0x00,0x00,0xcf,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x78,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x7a,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x73,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x73,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xb2,0x02,0x00,0x00,0xbe,0x02,0x00,0x00, +0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x70,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x72,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0x5b,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x5b,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xb4,0x02,0x00,0x00, +0xbc,0x02,0x00,0x00,0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x58,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x5a,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0x53,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x53,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xb6,0x02,0x00,0x00,0xbb,0x02,0x00,0x00,0xcf,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x50,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x52,0x02,0x00,0x00,0xfd,0x00,0x01,0x00,0x38,0x00,0x01,0x00, }; -const uint64_t matmul_f32_m_len = 10164; - -unsigned char matmul_f32_m_fp32_data[] = { -0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, -0xc6,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, -0x01,0x00,0x00,0x00,0x0b,0x00,0x06,0x00,0x01,0x00,0x00,0x00, -0x47,0x4c,0x53,0x4c,0x2e,0x73,0x74,0x64,0x2e,0x34,0x35,0x30, -0x00,0x00,0x00,0x00,0x0e,0x00,0x03,0x00,0x00,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x0f,0x00,0x0f,0x00,0x05,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x6d,0x61,0x69,0x6e,0x00,0x00,0x00,0x00, -0x0b,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x3d,0x00,0x00,0x00, -0x4c,0x00,0x00,0x00,0xf0,0x00,0x00,0x00,0xfb,0x00,0x00,0x00, -0x3a,0x01,0x00,0x00,0x45,0x01,0x00,0x00,0x27,0x02,0x00,0x00, -0x70,0x02,0x00,0x00,0x10,0x00,0x06,0x00,0x04,0x00,0x00,0x00, -0x11,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x0b,0x00,0x00,0x00, -0x0b,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x10,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x10,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x10,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x05,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x10,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x07,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x1c,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x08,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x10,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x24,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x0a,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x28,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x2c,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x10,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x30,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x0d,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x34,0x00,0x00,0x00, -0x47,0x00,0x03,0x00,0x10,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x37,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x3d,0x00,0x00,0x00, -0x0b,0x00,0x00,0x00,0x1a,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x4c,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x1b,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x4f,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x53,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x60,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x62,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x07,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x6c,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x9c,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0xae,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x05,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xb1,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0xf8,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x48,0x00,0x04,0x00,0xf9,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0xf9,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x03,0x00,0xf9,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0xfb,0x00,0x00,0x00,0x22,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xfb,0x00,0x00,0x00, -0x21,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x14,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x15,0x01,0x00,0x00,0x0b,0x00,0x00,0x00, -0x19,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x42,0x01,0x00,0x00, -0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00, -0x43,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x43,0x01,0x00,0x00,0x00,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, -0x43,0x01,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x45,0x01,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x45,0x01,0x00,0x00,0x21,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x27,0x02,0x00,0x00, -0x0b,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x6d,0x02,0x00,0x00,0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x48,0x00,0x04,0x00,0x6e,0x02,0x00,0x00,0x00,0x00,0x00,0x00, -0x19,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x6e,0x02,0x00,0x00, -0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x03,0x00,0x6e,0x02,0x00,0x00,0x02,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x70,0x02,0x00,0x00,0x22,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x70,0x02,0x00,0x00, -0x21,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x13,0x00,0x02,0x00, -0x02,0x00,0x00,0x00,0x21,0x00,0x03,0x00,0x03,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x15,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x17,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x03,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x0a,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00, -0x0b,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x0d,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x1e,0x00,0x10,0x00,0x10,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x11,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x10,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x11,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x15,0x00,0x04,0x00,0x13,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x08,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x15,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, -0x21,0x00,0x00,0x00,0x0a,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0x27,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x2d,0x00,0x00,0x00, -0x07,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, -0x34,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x32,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00, -0x3d,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x3e,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00,0x4c,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x4f,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x32,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x54,0x00,0x00,0x00, -0x86,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x53,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x58,0x00,0x00,0x00, -0x86,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x53,0x00,0x00,0x00, -0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x60,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x61,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x53,0x00,0x00,0x00, -0x60,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x63,0x00,0x00,0x00,0x86,0x00,0x00,0x00, -0x61,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x67,0x00,0x00,0x00,0x86,0x00,0x00,0x00, -0x61,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x32,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x10,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, -0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x72,0x00,0x00,0x00, -0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x76,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, -0x7b,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x8c,0x00,0x00,0x00, -0x03,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, -0x97,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x32,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x9c,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x9e,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xad,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xae,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xaf,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0x53,0x00,0x00,0x00,0xae,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xb0,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0x4f,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x32,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xb1,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xb2,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0xb0,0x00,0x00,0x00,0xb1,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xb3,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0xb2,0x00,0x00,0x00,0x60,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xb4,0x00,0x00,0x00, -0x86,0x00,0x00,0x00,0xaf,0x00,0x00,0x00,0xb3,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xb5,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0xad,0x00,0x00,0x00,0xb4,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xb6,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0xb5,0x00,0x00,0x00,0xb1,0x00,0x00,0x00, -0x14,0x00,0x02,0x00,0xb7,0x00,0x00,0x00,0x16,0x00,0x03,0x00, -0xb9,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xba,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xbb,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0xba,0x00,0x00,0x00,0xb4,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xbc,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0xbb,0x00,0x00,0x00,0xb1,0x00,0x00,0x00,0x1c,0x00,0x04,0x00, -0xbd,0x00,0x00,0x00,0xb9,0x00,0x00,0x00,0xbc,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0xbe,0x00,0x00,0x00,0x07,0x00,0x00,0x00, -0xbd,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0xb9,0x00,0x00,0x00, -0xc1,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0xc2,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0xb9,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0xc5,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xec,0x00,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xed,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x37,0x00,0x00,0x00, -0xec,0x00,0x00,0x00,0x1c,0x00,0x04,0x00,0xee,0x00,0x00,0x00, -0xb9,0x00,0x00,0x00,0xed,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0xef,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0xee,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0xef,0x00,0x00,0x00,0xf0,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xf4,0x00,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x1d,0x00,0x03,0x00,0xf8,0x00,0x00,0x00, -0xb9,0x00,0x00,0x00,0x1e,0x00,0x03,0x00,0xf9,0x00,0x00,0x00, -0xf8,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0xfa,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0xf9,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0xfa,0x00,0x00,0x00,0xfb,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x06,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, -0xb9,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x09,0x01,0x00,0x00, -0x04,0x00,0x00,0x00,0xb9,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x0f,0x01,0x00,0x00,0x80,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x32,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x14,0x01,0x00,0x00,0x01,0x00,0x00,0x00, -0x33,0x00,0x06,0x00,0x09,0x00,0x00,0x00,0x15,0x01,0x00,0x00, -0x14,0x01,0x00,0x00,0x39,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x16,0x01,0x00,0x00, -0x51,0x00,0x00,0x00,0x15,0x01,0x00,0x00,0x00,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x17,0x01,0x00,0x00, -0x84,0x00,0x00,0x00,0x16,0x01,0x00,0x00,0x39,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x18,0x01,0x00,0x00, -0x86,0x00,0x00,0x00,0x17,0x01,0x00,0x00,0x6c,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x36,0x01,0x00,0x00, -0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x37,0x01,0x00,0x00, -0x84,0x00,0x00,0x00,0x9c,0x00,0x00,0x00,0x36,0x01,0x00,0x00, -0x1c,0x00,0x04,0x00,0x38,0x01,0x00,0x00,0xb9,0x00,0x00,0x00, -0x37,0x01,0x00,0x00,0x20,0x00,0x04,0x00,0x39,0x01,0x00,0x00, -0x04,0x00,0x00,0x00,0x38,0x01,0x00,0x00,0x3b,0x00,0x04,0x00, -0x39,0x01,0x00,0x00,0x3a,0x01,0x00,0x00,0x04,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x3e,0x01,0x00,0x00, -0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x1d,0x00,0x03,0x00,0x42,0x01,0x00,0x00,0xb9,0x00,0x00,0x00, -0x1e,0x00,0x03,0x00,0x43,0x01,0x00,0x00,0x42,0x01,0x00,0x00, -0x20,0x00,0x04,0x00,0x44,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, -0x43,0x01,0x00,0x00,0x3b,0x00,0x04,0x00,0x44,0x01,0x00,0x00, -0x45,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x57,0x01,0x00,0x00,0x80,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x5e,0x01,0x00,0x00,0x08,0x01,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x5f,0x01,0x00,0x00, -0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x62,0x01,0x00,0x00, -0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x7d,0x01,0x00,0x00, -0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00, -0x1c,0x00,0x04,0x00,0x7e,0x01,0x00,0x00,0xb9,0x00,0x00,0x00, -0x7d,0x01,0x00,0x00,0x20,0x00,0x04,0x00,0x7f,0x01,0x00,0x00, -0x07,0x00,0x00,0x00,0x7e,0x01,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x8f,0x01,0x00,0x00,0x80,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xaa,0x01,0x00,0x00,0x84,0x00,0x00,0x00, -0xb4,0x00,0x00,0x00,0xb1,0x00,0x00,0x00,0x1c,0x00,0x04,0x00, -0xab,0x01,0x00,0x00,0xb9,0x00,0x00,0x00,0xaa,0x01,0x00,0x00, -0x20,0x00,0x04,0x00,0xac,0x01,0x00,0x00,0x07,0x00,0x00,0x00, -0xab,0x01,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xb5,0x01,0x00,0x00,0x86,0x00,0x00,0x00,0xae,0x00,0x00,0x00, -0xb4,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xbd,0x01,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xec,0x01,0x00,0x00,0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, -0x1f,0x02,0x00,0x00,0x0d,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x0a,0x00,0x00,0x00,0x27,0x02,0x00,0x00,0x01,0x00,0x00,0x00, -0x1d,0x00,0x03,0x00,0x6d,0x02,0x00,0x00,0xb9,0x00,0x00,0x00, -0x1e,0x00,0x03,0x00,0x6e,0x02,0x00,0x00,0x6d,0x02,0x00,0x00, -0x20,0x00,0x04,0x00,0x6f,0x02,0x00,0x00,0x0c,0x00,0x00,0x00, -0x6e,0x02,0x00,0x00,0x3b,0x00,0x04,0x00,0x6f,0x02,0x00,0x00, -0x70,0x02,0x00,0x00,0x0c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0x75,0x02,0x00,0x00,0x05,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x82,0x02,0x00,0x00, -0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00, -0x36,0x00,0x05,0x00,0x02,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x05,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0xbe,0x00,0x00,0x00, -0xbf,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x7f,0x01,0x00,0x00,0x80,0x01,0x00,0x00,0x07,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0xac,0x01,0x00,0x00,0xad,0x01,0x00,0x00, -0x07,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00, -0x0e,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, -0x0e,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x14,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x17,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x17,0x00,0x00,0x00, -0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x1e,0x00,0x00,0x00, -0x0f,0x00,0x00,0x00,0x17,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x21,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x24,0x00,0x00,0x00,0x18,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0x28,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x27,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x29,0x00,0x00,0x00, -0x28,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x2a,0x00,0x00,0x00,0x1e,0x00,0x00,0x00,0x29,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x2d,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x2f,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x30,0x00,0x00,0x00, -0x24,0x00,0x00,0x00,0x2f,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x30,0x00,0x00,0x00, -0x2a,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0x35,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x34,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x36,0x00,0x00,0x00, -0x35,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x38,0x00,0x00,0x00,0x36,0x00,0x00,0x00,0x37,0x00,0x00,0x00, -0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, -0x38,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x3b,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, -0x37,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00, -0x3f,0x00,0x00,0x00,0x3d,0x00,0x00,0x00,0x3e,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x3f,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x42,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x3b,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x47,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0x3b,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x0d,0x00,0x00,0x00,0x49,0x00,0x00,0x00,0x3d,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x4a,0x00,0x00,0x00,0x49,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x0d,0x00,0x00,0x00,0x4d,0x00,0x00,0x00,0x4c,0x00,0x00,0x00, -0x3e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x4e,0x00,0x00,0x00,0x4d,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x50,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, -0x4f,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x55,0x00,0x00,0x00,0x50,0x00,0x00,0x00,0x54,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x59,0x00,0x00,0x00, -0x50,0x00,0x00,0x00,0x58,0x00,0x00,0x00,0x89,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x5d,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, -0x4f,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x64,0x00,0x00,0x00,0x5d,0x00,0x00,0x00,0x63,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x68,0x00,0x00,0x00, -0x5d,0x00,0x00,0x00,0x67,0x00,0x00,0x00,0x89,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x6e,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, -0x6d,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x73,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x72,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x77,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x76,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x78,0x00,0x00,0x00,0x77,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x79,0x00,0x00,0x00, -0x47,0x00,0x00,0x00,0x78,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x7c,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x7b,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x7d,0x00,0x00,0x00,0x7c,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x7f,0x00,0x00,0x00,0x47,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x82,0x00,0x00,0x00,0x7f,0x00,0x00,0x00,0x78,0x00,0x00,0x00, -0x0c,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x83,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x26,0x00,0x00,0x00,0x7d,0x00,0x00,0x00, -0x82,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0x87,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x86,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x88,0x00,0x00,0x00, -0x87,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x89,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x88,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8b,0x00,0x00,0x00, -0x42,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x8d,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x8c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x8e,0x00,0x00,0x00,0x8d,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x8f,0x00,0x00,0x00,0x8b,0x00,0x00,0x00, -0x8e,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x90,0x00,0x00,0x00,0x89,0x00,0x00,0x00,0x8f,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x92,0x00,0x00,0x00, -0x90,0x00,0x00,0x00,0x79,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x93,0x00,0x00,0x00,0x92,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0x98,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x97,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x99,0x00,0x00,0x00, -0x98,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x9a,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x99,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9d,0x00,0x00,0x00, -0x4a,0x00,0x00,0x00,0x9c,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x9f,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x9e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xa0,0x00,0x00,0x00,0x9f,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xa1,0x00,0x00,0x00,0x9d,0x00,0x00,0x00, -0xa0,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xa2,0x00,0x00,0x00,0x9a,0x00,0x00,0x00,0xa1,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa4,0x00,0x00,0x00, -0xa2,0x00,0x00,0x00,0x79,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xa5,0x00,0x00,0x00,0xa4,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xa7,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xa7,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0x94,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, -0x05,0x00,0x00,0x00,0xc6,0x00,0x00,0x00,0xa8,0x00,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00,0xb8,0x00,0x00,0x00, -0x94,0x02,0x00,0x00,0xb6,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0xa9,0x00,0x00,0x00,0xa8,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xb8,0x00,0x00,0x00,0xa8,0x00,0x00,0x00, -0xa9,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xa8,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0xc3,0x00,0x00,0x00, -0xbf,0x00,0x00,0x00,0x94,0x02,0x00,0x00,0x3e,0x00,0x03,0x00, -0xc3,0x00,0x00,0x00,0xc1,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xc6,0x00,0x00,0x00,0x94,0x02,0x00,0x00, -0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xa7,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xa9,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xc9,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xc9,0x00,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xad,0x02,0x00,0x00, -0xa5,0x00,0x00,0x00,0xa9,0x00,0x00,0x00,0x64,0x01,0x00,0x00, -0xcc,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xa9,0x02,0x00,0x00,0x93,0x00,0x00,0x00,0xa9,0x00,0x00,0x00, -0x61,0x01,0x00,0x00,0xcc,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0x95,0x02,0x00,0x00,0x79,0x00,0x00,0x00, -0xa9,0x00,0x00,0x00,0x0f,0x02,0x00,0x00,0xcc,0x00,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00,0xd0,0x00,0x00,0x00, -0x95,0x02,0x00,0x00,0x83,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0xcb,0x00,0x00,0x00,0xcc,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xd0,0x00,0x00,0x00,0xca,0x00,0x00,0x00, -0xcb,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xca,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xd2,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xd2,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xa5,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0xca,0x00,0x00,0x00, -0x1a,0x01,0x00,0x00,0xd5,0x00,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb7,0x00,0x00,0x00,0xd8,0x00,0x00,0x00,0xa5,0x02,0x00,0x00, -0x37,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xd4,0x00,0x00,0x00, -0xd5,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0xd8,0x00,0x00,0x00,0xd3,0x00,0x00,0x00,0xd4,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xd3,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xdc,0x00,0x00,0x00,0x8b,0x00,0x00,0x00, -0x73,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xde,0x00,0x00,0x00,0xdc,0x00,0x00,0x00,0xa5,0x02,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00,0xe1,0x00,0x00,0x00, -0xde,0x00,0x00,0x00,0x36,0x00,0x00,0x00,0xf7,0x00,0x03,0x00, -0xe3,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0xe1,0x00,0x00,0x00,0xe2,0x00,0x00,0x00,0xe3,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xe2,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xe6,0x00,0x00,0x00,0x95,0x02,0x00,0x00, -0x6e,0x00,0x00,0x00,0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00, -0xe8,0x00,0x00,0x00,0xe6,0x00,0x00,0x00,0x83,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xe3,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xe3,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0xb7,0x00,0x00,0x00, -0xe9,0x00,0x00,0x00,0xe1,0x00,0x00,0x00,0xd3,0x00,0x00,0x00, -0xe8,0x00,0x00,0x00,0xe2,0x00,0x00,0x00,0xf7,0x00,0x03,0x00, -0xeb,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0xe9,0x00,0x00,0x00,0xea,0x00,0x00,0x00,0x0b,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xea,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xf3,0x00,0x00,0x00,0x73,0x00,0x00,0x00, -0xa5,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xf5,0x00,0x00,0x00,0xf3,0x00,0x00,0x00,0xf4,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xf7,0x00,0x00,0x00, -0xf5,0x00,0x00,0x00,0x6e,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x02,0x01,0x00,0x00,0xf3,0x00,0x00,0x00, -0x8e,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x03,0x01,0x00,0x00,0xa9,0x02,0x00,0x00,0x02,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x05,0x01,0x00,0x00, -0x03,0x01,0x00,0x00,0x6e,0x00,0x00,0x00,0x41,0x00,0x06,0x00, -0x06,0x01,0x00,0x00,0x07,0x01,0x00,0x00,0xfb,0x00,0x00,0x00, -0x34,0x00,0x00,0x00,0x05,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, -0xb9,0x00,0x00,0x00,0x08,0x01,0x00,0x00,0x07,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0x09,0x01,0x00,0x00,0x0a,0x01,0x00,0x00, -0xf0,0x00,0x00,0x00,0xf7,0x00,0x00,0x00,0x3e,0x00,0x03,0x00, -0x0a,0x01,0x00,0x00,0x08,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0xeb,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x0b,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x0e,0x01,0x00,0x00, -0x73,0x00,0x00,0x00,0xa5,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x10,0x01,0x00,0x00,0x0e,0x01,0x00,0x00, -0x0f,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x12,0x01,0x00,0x00,0x10,0x01,0x00,0x00,0x6e,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x09,0x01,0x00,0x00,0x13,0x01,0x00,0x00, -0xf0,0x00,0x00,0x00,0x12,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0x13,0x01,0x00,0x00,0xc1,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xeb,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xeb,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xd5,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xd5,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x1a,0x01,0x00,0x00,0xa5,0x02,0x00,0x00,0x18,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0xd2,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xd4,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x1c,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x1c,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xa6,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, -0xd4,0x00,0x00,0x00,0x5d,0x01,0x00,0x00,0x1f,0x01,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00,0x22,0x01,0x00,0x00, -0xa6,0x02,0x00,0x00,0x9c,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0x1e,0x01,0x00,0x00,0x1f,0x01,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x22,0x01,0x00,0x00,0x1d,0x01,0x00,0x00, -0x1e,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x1d,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x26,0x01,0x00,0x00, -0x9d,0x00,0x00,0x00,0x73,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x28,0x01,0x00,0x00,0x26,0x01,0x00,0x00, -0xa6,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0x29,0x01,0x00,0x00,0x12,0x00,0x00,0x00,0xc5,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x2a,0x01,0x00,0x00, -0x29,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00, -0x2b,0x01,0x00,0x00,0x28,0x01,0x00,0x00,0x2a,0x01,0x00,0x00, -0xf7,0x00,0x03,0x00,0x2d,0x01,0x00,0x00,0x00,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x2b,0x01,0x00,0x00,0x2c,0x01,0x00,0x00, -0x2d,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x2c,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x30,0x01,0x00,0x00, -0x95,0x02,0x00,0x00,0x6e,0x00,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb7,0x00,0x00,0x00,0x32,0x01,0x00,0x00,0x30,0x01,0x00,0x00, -0x83,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x2d,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x2d,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, -0xb7,0x00,0x00,0x00,0x33,0x01,0x00,0x00,0x2b,0x01,0x00,0x00, -0x1d,0x01,0x00,0x00,0x32,0x01,0x00,0x00,0x2c,0x01,0x00,0x00, -0xf7,0x00,0x03,0x00,0x35,0x01,0x00,0x00,0x00,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x33,0x01,0x00,0x00,0x34,0x01,0x00,0x00, -0x53,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x34,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3d,0x01,0x00,0x00, -0x73,0x00,0x00,0x00,0xa6,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x3f,0x01,0x00,0x00,0x3d,0x01,0x00,0x00, -0x3e,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x41,0x01,0x00,0x00,0x3f,0x01,0x00,0x00,0x6e,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x4c,0x01,0x00,0x00, -0x3d,0x01,0x00,0x00,0xa0,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x4d,0x01,0x00,0x00,0xad,0x02,0x00,0x00, -0x4c,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x4f,0x01,0x00,0x00,0x4d,0x01,0x00,0x00,0x6e,0x00,0x00,0x00, -0x41,0x00,0x06,0x00,0x06,0x01,0x00,0x00,0x50,0x01,0x00,0x00, -0x45,0x01,0x00,0x00,0x34,0x00,0x00,0x00,0x4f,0x01,0x00,0x00, -0x3d,0x00,0x04,0x00,0xb9,0x00,0x00,0x00,0x51,0x01,0x00,0x00, -0x50,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0x09,0x01,0x00,0x00, -0x52,0x01,0x00,0x00,0x3a,0x01,0x00,0x00,0x41,0x01,0x00,0x00, -0x3e,0x00,0x03,0x00,0x52,0x01,0x00,0x00,0x51,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0x35,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x53,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x56,0x01,0x00,0x00,0x73,0x00,0x00,0x00,0xa6,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x58,0x01,0x00,0x00, -0x56,0x01,0x00,0x00,0x57,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x5a,0x01,0x00,0x00,0x58,0x01,0x00,0x00, -0x6e,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x09,0x01,0x00,0x00, -0x5b,0x01,0x00,0x00,0x3a,0x01,0x00,0x00,0x5a,0x01,0x00,0x00, -0x3e,0x00,0x03,0x00,0x5b,0x01,0x00,0x00,0xc1,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x35,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x35,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x1f,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x1f,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x5d,0x01,0x00,0x00,0xa6,0x02,0x00,0x00, -0x18,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x1c,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x1e,0x01,0x00,0x00,0xe0,0x00,0x04,0x00, -0x0c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x5e,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x61,0x01,0x00,0x00, -0xa9,0x02,0x00,0x00,0x5f,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x64,0x01,0x00,0x00,0xad,0x02,0x00,0x00, -0x62,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x66,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x66,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xaf,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, -0x1e,0x01,0x00,0x00,0x0d,0x02,0x00,0x00,0x69,0x01,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00,0x6c,0x01,0x00,0x00, -0xaf,0x02,0x00,0x00,0x6c,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0x68,0x01,0x00,0x00,0x69,0x01,0x00,0x00,0x00,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x6c,0x01,0x00,0x00,0x67,0x01,0x00,0x00, -0x68,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x67,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0x6e,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x6e,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xb3,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0x67,0x01,0x00,0x00, -0x99,0x01,0x00,0x00,0x71,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb7,0x00,0x00,0x00,0x74,0x01,0x00,0x00,0xb3,0x02,0x00,0x00, -0x60,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x70,0x01,0x00,0x00, -0x71,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x74,0x01,0x00,0x00,0x6f,0x01,0x00,0x00,0x70,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x6f,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0x76,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x76,0x01,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xc5,0x02,0x00,0x00, -0x3e,0x00,0x00,0x00,0x6f,0x01,0x00,0x00,0x97,0x01,0x00,0x00, -0x77,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00, -0x7c,0x01,0x00,0x00,0xc5,0x02,0x00,0x00,0x62,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0x78,0x01,0x00,0x00,0x77,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x7c,0x01,0x00,0x00, -0x77,0x01,0x00,0x00,0x78,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x77,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x82,0x01,0x00,0x00,0xb3,0x02,0x00,0x00,0x62,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x84,0x01,0x00,0x00, -0x82,0x01,0x00,0x00,0xc5,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x86,0x01,0x00,0x00,0x55,0x00,0x00,0x00, -0x53,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x88,0x01,0x00,0x00,0xb3,0x02,0x00,0x00,0x61,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x89,0x01,0x00,0x00, -0x86,0x01,0x00,0x00,0x88,0x01,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x8b,0x01,0x00,0x00,0x64,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x8c,0x01,0x00,0x00,0x89,0x01,0x00,0x00,0x8b,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8e,0x01,0x00,0x00, -0x8c,0x01,0x00,0x00,0xc5,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x90,0x01,0x00,0x00,0x8e,0x01,0x00,0x00, -0x8f,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x92,0x01,0x00,0x00,0x90,0x01,0x00,0x00,0xaf,0x02,0x00,0x00, -0x41,0x00,0x05,0x00,0x09,0x01,0x00,0x00,0x93,0x01,0x00,0x00, -0xf0,0x00,0x00,0x00,0x92,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, -0xb9,0x00,0x00,0x00,0x94,0x01,0x00,0x00,0x93,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0x95,0x01,0x00,0x00, -0x80,0x01,0x00,0x00,0x84,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0x95,0x01,0x00,0x00,0x94,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x97,0x01,0x00,0x00,0xc5,0x02,0x00,0x00, -0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x76,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x78,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0x71,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x71,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x99,0x01,0x00,0x00, -0xb3,0x02,0x00,0x00,0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x6e,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x70,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0x9b,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x9b,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xb4,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0x70,0x01,0x00,0x00, -0xc7,0x01,0x00,0x00,0x9e,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb7,0x00,0x00,0x00,0xa1,0x01,0x00,0x00,0xb4,0x02,0x00,0x00, -0xb4,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x9d,0x01,0x00,0x00, -0x9e,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0xa1,0x01,0x00,0x00,0x9c,0x01,0x00,0x00,0x9d,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x9c,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0xa3,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xa3,0x01,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xc2,0x02,0x00,0x00, -0x3e,0x00,0x00,0x00,0x9c,0x01,0x00,0x00,0xc5,0x01,0x00,0x00, -0xa4,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00, -0xa9,0x01,0x00,0x00,0xc2,0x02,0x00,0x00,0xb1,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0xa5,0x01,0x00,0x00,0xa4,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xa9,0x01,0x00,0x00, -0xa4,0x01,0x00,0x00,0xa5,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xa4,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xaf,0x01,0x00,0x00,0xb4,0x02,0x00,0x00,0xb1,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xb1,0x01,0x00,0x00, -0xaf,0x01,0x00,0x00,0xc2,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xb3,0x01,0x00,0x00,0x59,0x00,0x00,0x00, -0xae,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xb6,0x01,0x00,0x00,0xb4,0x02,0x00,0x00,0xb5,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xb7,0x01,0x00,0x00, -0xb3,0x01,0x00,0x00,0xb6,0x01,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xb9,0x01,0x00,0x00,0x68,0x00,0x00,0x00, -0xb1,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xba,0x01,0x00,0x00,0xb7,0x01,0x00,0x00,0xb9,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xbc,0x01,0x00,0x00, -0xba,0x01,0x00,0x00,0xc2,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xbe,0x01,0x00,0x00,0xbc,0x01,0x00,0x00, -0xbd,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xc0,0x01,0x00,0x00,0xbe,0x01,0x00,0x00,0xaf,0x02,0x00,0x00, -0x41,0x00,0x05,0x00,0x09,0x01,0x00,0x00,0xc1,0x01,0x00,0x00, -0x3a,0x01,0x00,0x00,0xc0,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, -0xb9,0x00,0x00,0x00,0xc2,0x01,0x00,0x00,0xc1,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0xc3,0x01,0x00,0x00, -0xad,0x01,0x00,0x00,0xb1,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0xc3,0x01,0x00,0x00,0xc2,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xc5,0x01,0x00,0x00,0xc2,0x02,0x00,0x00, -0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xa3,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xa5,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0x9e,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x9e,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xc7,0x01,0x00,0x00, -0xb4,0x02,0x00,0x00,0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x9b,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x9d,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0xc9,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xc9,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xb5,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0x9d,0x01,0x00,0x00, -0x0b,0x02,0x00,0x00,0xcc,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb7,0x00,0x00,0x00,0xcf,0x01,0x00,0x00,0xb5,0x02,0x00,0x00, -0xb4,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xcb,0x01,0x00,0x00, -0xcc,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0xcf,0x01,0x00,0x00,0xca,0x01,0x00,0x00,0xcb,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xca,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0xd1,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xd1,0x01,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xb9,0x02,0x00,0x00, -0x3e,0x00,0x00,0x00,0xca,0x01,0x00,0x00,0x09,0x02,0x00,0x00, -0xd4,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00, -0xd7,0x01,0x00,0x00,0xb9,0x02,0x00,0x00,0x60,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0xd3,0x01,0x00,0x00,0xd4,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xd7,0x01,0x00,0x00, -0xd2,0x01,0x00,0x00,0xd3,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xd2,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xd9,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xd9,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xbb,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, -0xd2,0x01,0x00,0x00,0x07,0x02,0x00,0x00,0xdc,0x01,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00,0xdf,0x01,0x00,0x00, -0xbb,0x02,0x00,0x00,0xb1,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0xdb,0x01,0x00,0x00,0xdc,0x01,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xdf,0x01,0x00,0x00,0xda,0x01,0x00,0x00, -0xdb,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xda,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0xe1,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xe1,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xbd,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0xda,0x01,0x00,0x00, -0x05,0x02,0x00,0x00,0xe2,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb7,0x00,0x00,0x00,0xe7,0x01,0x00,0x00,0xbd,0x02,0x00,0x00, -0x62,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xe3,0x01,0x00,0x00, -0xe2,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0xe7,0x01,0x00,0x00,0xe2,0x01,0x00,0x00,0xe3,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xe2,0x01,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xe9,0x01,0x00,0x00,0xb5,0x02,0x00,0x00, -0xb1,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xeb,0x01,0x00,0x00,0xe9,0x01,0x00,0x00,0xbb,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xed,0x01,0x00,0x00, -0xeb,0x01,0x00,0x00,0xec,0x01,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xef,0x01,0x00,0x00,0xb9,0x02,0x00,0x00, -0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xf0,0x01,0x00,0x00,0xed,0x01,0x00,0x00,0xef,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xf2,0x01,0x00,0x00, -0xf0,0x01,0x00,0x00,0xbd,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xf6,0x01,0x00,0x00,0xef,0x01,0x00,0x00, -0xbd,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, -0xf7,0x01,0x00,0x00,0x80,0x01,0x00,0x00,0xf6,0x01,0x00,0x00, -0x3d,0x00,0x04,0x00,0xb9,0x00,0x00,0x00,0xf8,0x01,0x00,0x00, -0xf7,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, -0xfd,0x01,0x00,0x00,0xad,0x01,0x00,0x00,0xeb,0x01,0x00,0x00, -0x3d,0x00,0x04,0x00,0xb9,0x00,0x00,0x00,0xfe,0x01,0x00,0x00, -0xfd,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, -0x00,0x02,0x00,0x00,0xbf,0x00,0x00,0x00,0xf2,0x01,0x00,0x00, -0x3d,0x00,0x04,0x00,0xb9,0x00,0x00,0x00,0x01,0x02,0x00,0x00, -0x00,0x02,0x00,0x00,0x0c,0x00,0x08,0x00,0xb9,0x00,0x00,0x00, -0x02,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00, -0xf8,0x01,0x00,0x00,0xfe,0x01,0x00,0x00,0x01,0x02,0x00,0x00, -0x3e,0x00,0x03,0x00,0x00,0x02,0x00,0x00,0x02,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x05,0x02,0x00,0x00, -0xbd,0x02,0x00,0x00,0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xe1,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xe3,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0xdc,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xdc,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x07,0x02,0x00,0x00,0xbb,0x02,0x00,0x00,0xc5,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xd9,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xdb,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xd4,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xd4,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x09,0x02,0x00,0x00,0xb9,0x02,0x00,0x00, -0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xd1,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xd3,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0xcc,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xcc,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x0b,0x02,0x00,0x00, -0xb5,0x02,0x00,0x00,0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xc9,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xcb,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0x69,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x69,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x0d,0x02,0x00,0x00,0xaf,0x02,0x00,0x00,0xc5,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x66,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x68,0x01,0x00,0x00,0xe0,0x00,0x04,0x00,0x0c,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x5e,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0xcc,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xcc,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x0f,0x02,0x00,0x00, -0x95,0x02,0x00,0x00,0x6c,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xc9,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xcb,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x14,0x02,0x00,0x00, -0x55,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x15,0x02,0x00,0x00,0x8b,0x00,0x00,0x00, -0x14,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x1a,0x02,0x00,0x00,0x59,0x00,0x00,0x00,0xae,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x1b,0x02,0x00,0x00, -0x9d,0x00,0x00,0x00,0x1a,0x02,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x20,0x02,0x00,0x00,0x12,0x00,0x00,0x00, -0x1f,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x21,0x02,0x00,0x00,0x20,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x22,0x02,0x00,0x00,0x0f,0x00,0x00,0x00, -0x21,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x26,0x02,0x00,0x00,0x47,0x00,0x00,0x00,0x21,0x02,0x00,0x00, -0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00,0x28,0x02,0x00,0x00, -0x27,0x02,0x00,0x00,0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x29,0x02,0x00,0x00,0x28,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x2a,0x02,0x00,0x00, -0x26,0x02,0x00,0x00,0x29,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x2b,0x02,0x00,0x00,0x22,0x02,0x00,0x00, -0x2a,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x2d,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x2d,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0x96,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, -0xcb,0x00,0x00,0x00,0x93,0x02,0x00,0x00,0x30,0x02,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00,0x33,0x02,0x00,0x00, -0x96,0x02,0x00,0x00,0xb4,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0x2f,0x02,0x00,0x00,0x30,0x02,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x33,0x02,0x00,0x00,0x2e,0x02,0x00,0x00, -0x2f,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x2e,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0x35,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x35,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0x97,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0x2e,0x02,0x00,0x00, -0x91,0x02,0x00,0x00,0x38,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb7,0x00,0x00,0x00,0x3b,0x02,0x00,0x00,0x97,0x02,0x00,0x00, -0x60,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x37,0x02,0x00,0x00, -0x38,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x3b,0x02,0x00,0x00,0x36,0x02,0x00,0x00,0x37,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x36,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x3f,0x02,0x00,0x00,0x97,0x02,0x00,0x00, -0x61,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x40,0x02,0x00,0x00,0x15,0x02,0x00,0x00,0x3f,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x42,0x02,0x00,0x00, -0x64,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x43,0x02,0x00,0x00,0x40,0x02,0x00,0x00, -0x42,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x47,0x02,0x00,0x00,0x96,0x02,0x00,0x00,0xb5,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x48,0x02,0x00,0x00, -0x1b,0x02,0x00,0x00,0x47,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x4a,0x02,0x00,0x00,0x68,0x00,0x00,0x00, -0xb1,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x4b,0x02,0x00,0x00,0x48,0x02,0x00,0x00,0x4a,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0x4d,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x4d,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0x99,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0x36,0x02,0x00,0x00, -0x8f,0x02,0x00,0x00,0x50,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb7,0x00,0x00,0x00,0x53,0x02,0x00,0x00,0x99,0x02,0x00,0x00, -0xb1,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x4f,0x02,0x00,0x00, -0x50,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x53,0x02,0x00,0x00,0x4e,0x02,0x00,0x00,0x4f,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x4e,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x55,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x55,0x02,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x9b,0x02,0x00,0x00, -0x3e,0x00,0x00,0x00,0x4e,0x02,0x00,0x00,0x8d,0x02,0x00,0x00, -0x58,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00, -0x5b,0x02,0x00,0x00,0x9b,0x02,0x00,0x00,0x62,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0x57,0x02,0x00,0x00,0x58,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x5b,0x02,0x00,0x00, -0x56,0x02,0x00,0x00,0x57,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x56,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x5e,0x02,0x00,0x00,0x43,0x02,0x00,0x00,0x9b,0x02,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00,0x61,0x02,0x00,0x00, -0x5e,0x02,0x00,0x00,0x36,0x00,0x00,0x00,0xf7,0x00,0x03,0x00, -0x63,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x61,0x02,0x00,0x00,0x62,0x02,0x00,0x00,0x63,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x62,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x66,0x02,0x00,0x00,0x4b,0x02,0x00,0x00, -0x99,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0x67,0x02,0x00,0x00,0x12,0x00,0x00,0x00,0xc5,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x68,0x02,0x00,0x00, -0x67,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00, -0x69,0x02,0x00,0x00,0x66,0x02,0x00,0x00,0x68,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0x63,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x63,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0xb7,0x00,0x00,0x00, -0x6a,0x02,0x00,0x00,0x61,0x02,0x00,0x00,0x56,0x02,0x00,0x00, -0x69,0x02,0x00,0x00,0x62,0x02,0x00,0x00,0xf7,0x00,0x03,0x00, -0x6c,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x6a,0x02,0x00,0x00,0x6b,0x02,0x00,0x00,0x6c,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x6b,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x74,0x02,0x00,0x00,0x4b,0x02,0x00,0x00, -0x99,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0x76,0x02,0x00,0x00,0x12,0x00,0x00,0x00,0x75,0x02,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x77,0x02,0x00,0x00, -0x76,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x78,0x02,0x00,0x00,0x74,0x02,0x00,0x00,0x77,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x79,0x02,0x00,0x00, -0x2b,0x02,0x00,0x00,0x78,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x7b,0x02,0x00,0x00,0x79,0x02,0x00,0x00, -0x43,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x7d,0x02,0x00,0x00,0x7b,0x02,0x00,0x00,0x9b,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x7f,0x02,0x00,0x00, -0x96,0x02,0x00,0x00,0xb1,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x81,0x02,0x00,0x00,0x7f,0x02,0x00,0x00, -0x99,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x83,0x02,0x00,0x00,0x81,0x02,0x00,0x00,0x82,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x85,0x02,0x00,0x00, -0x97,0x02,0x00,0x00,0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x86,0x02,0x00,0x00,0x83,0x02,0x00,0x00, -0x85,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x88,0x02,0x00,0x00,0x86,0x02,0x00,0x00,0x9b,0x02,0x00,0x00, -0x41,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0x89,0x02,0x00,0x00, -0xbf,0x00,0x00,0x00,0x88,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, -0xb9,0x00,0x00,0x00,0x8a,0x02,0x00,0x00,0x89,0x02,0x00,0x00, -0x41,0x00,0x06,0x00,0x06,0x01,0x00,0x00,0x8b,0x02,0x00,0x00, -0x70,0x02,0x00,0x00,0x34,0x00,0x00,0x00,0x7d,0x02,0x00,0x00, -0x3e,0x00,0x03,0x00,0x8b,0x02,0x00,0x00,0x8a,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0x6c,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x6c,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x58,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x58,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x8d,0x02,0x00,0x00,0x9b,0x02,0x00,0x00, -0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x55,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x57,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x50,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x50,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8f,0x02,0x00,0x00, -0x99,0x02,0x00,0x00,0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x4d,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x4f,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0x38,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x38,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x91,0x02,0x00,0x00,0x97,0x02,0x00,0x00,0xc5,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x35,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x37,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x30,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x30,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x93,0x02,0x00,0x00,0x96,0x02,0x00,0x00, -0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x2d,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x2f,0x02,0x00,0x00,0xfd,0x00,0x01,0x00, -0x38,0x00,0x01,0x00, -}; -const uint64_t matmul_f32_m_fp32_len = 10048; - -unsigned char matmul_f32_s_data[] = { -0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, -0xcd,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, -0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x09,0x00,0x00,0x00, -0x0b,0x00,0x06,0x00,0x01,0x00,0x00,0x00,0x47,0x4c,0x53,0x4c, -0x2e,0x73,0x74,0x64,0x2e,0x34,0x35,0x30,0x00,0x00,0x00,0x00, -0x0e,0x00,0x03,0x00,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x0f,0x00,0x0f,0x00,0x05,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x6d,0x61,0x69,0x6e,0x00,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x3d,0x00,0x00,0x00,0x4c,0x00,0x00,0x00, -0xf1,0x00,0x00,0x00,0xfc,0x00,0x00,0x00,0x3d,0x01,0x00,0x00, -0x48,0x01,0x00,0x00,0x2e,0x02,0x00,0x00,0x77,0x02,0x00,0x00, -0x10,0x00,0x06,0x00,0x04,0x00,0x00,0x00,0x11,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x0b,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x1c,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x10,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x08,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x03,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x10,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x14,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x18,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x07,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x10,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x24,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x0a,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x10,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x2c,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x30,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x0d,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0x47,0x00,0x03,0x00, -0x10,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x37,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x3d,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x1a,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x4c,0x00,0x00,0x00, -0x0b,0x00,0x00,0x00,0x1b,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x4f,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x53,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x60,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x62,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x07,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x6c,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x03,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x9c,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0xae,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x05,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0xb1,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x08,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xf9,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00, -0xfa,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0xfa,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, -0xfa,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0xfc,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0xfc,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x17,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x18,0x01,0x00,0x00,0x0b,0x00,0x00,0x00,0x19,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x45,0x01,0x00,0x00,0x06,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0x46,0x01,0x00,0x00, -0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x46,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x46,0x01,0x00,0x00, -0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x48,0x01,0x00,0x00, -0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x48,0x01,0x00,0x00,0x21,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x2e,0x02,0x00,0x00,0x0b,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x74,0x02,0x00,0x00, -0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00, -0x75,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x19,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x75,0x02,0x00,0x00,0x00,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, -0x75,0x02,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x77,0x02,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x77,0x02,0x00,0x00,0x21,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x13,0x00,0x02,0x00,0x02,0x00,0x00,0x00, -0x21,0x00,0x03,0x00,0x03,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x15,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x17,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x0a,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x0d,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x1e,0x00,0x10,0x00,0x10,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x11,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x11,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x15,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, -0x14,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x15,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x0a,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, -0x27,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0x2d,0x00,0x00,0x00,0x07,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x34,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x37,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00,0x3d,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x3e,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x0a,0x00,0x00,0x00,0x4c,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x4f,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x53,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x54,0x00,0x00,0x00,0x86,0x00,0x00,0x00, -0x37,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x58,0x00,0x00,0x00,0x86,0x00,0x00,0x00, -0x37,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x32,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x61,0x00,0x00,0x00, -0x86,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x60,0x00,0x00,0x00, -0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x62,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x63,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x61,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x67,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x61,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x86,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x72,0x00,0x00,0x00,0x86,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0x76,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x7b,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, -0x86,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0x8c,0x00,0x00,0x00,0x03,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x97,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x9c,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0x9e,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xad,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00, -0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xae,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xaf,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x53,0x00,0x00,0x00, -0xae,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xb0,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x4f,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xb1,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xb2,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0xb0,0x00,0x00,0x00,0xb1,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xb3,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0xb2,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xb4,0x00,0x00,0x00,0x86,0x00,0x00,0x00, -0xaf,0x00,0x00,0x00,0xb3,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xb5,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0xad,0x00,0x00,0x00,0xb4,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xb6,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0xb5,0x00,0x00,0x00,0xb1,0x00,0x00,0x00,0x14,0x00,0x02,0x00, -0xb7,0x00,0x00,0x00,0x16,0x00,0x03,0x00,0xb9,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xba,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xbb,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0xba,0x00,0x00,0x00, -0xb4,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xbc,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0xbb,0x00,0x00,0x00, -0xb1,0x00,0x00,0x00,0x1c,0x00,0x04,0x00,0xbd,0x00,0x00,0x00, -0xb9,0x00,0x00,0x00,0xbc,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0xbe,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0xbd,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0xb9,0x00,0x00,0x00,0xc1,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0xc2,0x00,0x00,0x00, -0x07,0x00,0x00,0x00,0xb9,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0xc5,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x16,0x00,0x03,0x00,0xec,0x00,0x00,0x00,0x10,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xed,0x00,0x00,0x00, -0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xee,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0xed,0x00,0x00,0x00, -0x1c,0x00,0x04,0x00,0xef,0x00,0x00,0x00,0xec,0x00,0x00,0x00, -0xee,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0xf0,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0xef,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0xf0,0x00,0x00,0x00,0xf1,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xf5,0x00,0x00,0x00, -0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x1d,0x00,0x03,0x00,0xf9,0x00,0x00,0x00,0xb9,0x00,0x00,0x00, -0x1e,0x00,0x03,0x00,0xfa,0x00,0x00,0x00,0xf9,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0xfb,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0xfa,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0xfb,0x00,0x00,0x00, -0xfc,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x07,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0xb9,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x0b,0x01,0x00,0x00,0x04,0x00,0x00,0x00, -0xec,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x11,0x01,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0xec,0x00,0x00,0x00, -0x15,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x32,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x17,0x01,0x00,0x00,0x01,0x00,0x00,0x00, -0x33,0x00,0x06,0x00,0x09,0x00,0x00,0x00,0x18,0x01,0x00,0x00, -0x17,0x01,0x00,0x00,0x39,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x19,0x01,0x00,0x00, -0x51,0x00,0x00,0x00,0x18,0x01,0x00,0x00,0x00,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x1a,0x01,0x00,0x00, -0x84,0x00,0x00,0x00,0x19,0x01,0x00,0x00,0x39,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x1b,0x01,0x00,0x00, -0x86,0x00,0x00,0x00,0x1a,0x01,0x00,0x00,0x6c,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x39,0x01,0x00,0x00, -0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x3a,0x01,0x00,0x00, -0x84,0x00,0x00,0x00,0x9c,0x00,0x00,0x00,0x39,0x01,0x00,0x00, -0x1c,0x00,0x04,0x00,0x3b,0x01,0x00,0x00,0xec,0x00,0x00,0x00, -0x3a,0x01,0x00,0x00,0x20,0x00,0x04,0x00,0x3c,0x01,0x00,0x00, -0x04,0x00,0x00,0x00,0x3b,0x01,0x00,0x00,0x3b,0x00,0x04,0x00, -0x3c,0x01,0x00,0x00,0x3d,0x01,0x00,0x00,0x04,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x41,0x01,0x00,0x00, -0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x1d,0x00,0x03,0x00,0x45,0x01,0x00,0x00,0xb9,0x00,0x00,0x00, -0x1e,0x00,0x03,0x00,0x46,0x01,0x00,0x00,0x45,0x01,0x00,0x00, -0x20,0x00,0x04,0x00,0x47,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, -0x46,0x01,0x00,0x00,0x3b,0x00,0x04,0x00,0x47,0x01,0x00,0x00, -0x48,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x5b,0x01,0x00,0x00,0x80,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x62,0x01,0x00,0x00,0x08,0x01,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x63,0x01,0x00,0x00, -0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x66,0x01,0x00,0x00, -0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x81,0x01,0x00,0x00, -0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00, -0x1c,0x00,0x04,0x00,0x82,0x01,0x00,0x00,0xec,0x00,0x00,0x00, -0x81,0x01,0x00,0x00,0x20,0x00,0x04,0x00,0x83,0x01,0x00,0x00, -0x07,0x00,0x00,0x00,0x82,0x01,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x93,0x01,0x00,0x00,0x80,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x99,0x01,0x00,0x00,0x07,0x00,0x00,0x00,0xec,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xaf,0x01,0x00,0x00, -0x84,0x00,0x00,0x00,0xb4,0x00,0x00,0x00,0xb1,0x00,0x00,0x00, -0x1c,0x00,0x04,0x00,0xb0,0x01,0x00,0x00,0xec,0x00,0x00,0x00, -0xaf,0x01,0x00,0x00,0x20,0x00,0x04,0x00,0xb1,0x01,0x00,0x00, -0x07,0x00,0x00,0x00,0xb0,0x01,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xba,0x01,0x00,0x00,0x86,0x00,0x00,0x00, -0xae,0x00,0x00,0x00,0xb4,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xc2,0x01,0x00,0x00,0x80,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xf1,0x01,0x00,0x00,0x84,0x00,0x00,0x00, -0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0x26,0x02,0x00,0x00,0x0d,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00,0x2e,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0x1d,0x00,0x03,0x00,0x74,0x02,0x00,0x00, -0xb9,0x00,0x00,0x00,0x1e,0x00,0x03,0x00,0x75,0x02,0x00,0x00, -0x74,0x02,0x00,0x00,0x20,0x00,0x04,0x00,0x76,0x02,0x00,0x00, -0x0c,0x00,0x00,0x00,0x75,0x02,0x00,0x00,0x3b,0x00,0x04,0x00, -0x76,0x02,0x00,0x00,0x77,0x02,0x00,0x00,0x0c,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x7c,0x02,0x00,0x00, -0x05,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x89,0x02,0x00,0x00,0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x36,0x00,0x05,0x00,0x02,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x03,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0x05,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0xbe,0x00,0x00,0x00,0xbf,0x00,0x00,0x00,0x07,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x83,0x01,0x00,0x00,0x84,0x01,0x00,0x00, -0x07,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0xb1,0x01,0x00,0x00, -0xb2,0x01,0x00,0x00,0x07,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x0d,0x00,0x00,0x00,0x0e,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x0f,0x00,0x00,0x00,0x0e,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x14,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x17,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, -0x17,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x1e,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x17,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x22,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x22,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x24,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x27,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x29,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x2a,0x00,0x00,0x00,0x1e,0x00,0x00,0x00, -0x29,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x2d,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x2f,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x30,0x00,0x00,0x00,0x24,0x00,0x00,0x00,0x2f,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x32,0x00,0x00,0x00, -0x30,0x00,0x00,0x00,0x2a,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x35,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x34,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x36,0x00,0x00,0x00,0x35,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x38,0x00,0x00,0x00,0x36,0x00,0x00,0x00, -0x37,0x00,0x00,0x00,0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x3a,0x00,0x00,0x00,0x38,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3b,0x00,0x00,0x00, -0x3a,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x0d,0x00,0x00,0x00,0x3f,0x00,0x00,0x00,0x3d,0x00,0x00,0x00, -0x3e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0x3f,0x00,0x00,0x00,0x89,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x42,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x3b,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x47,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x3b,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00,0x49,0x00,0x00,0x00, -0x3d,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x4a,0x00,0x00,0x00,0x49,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00,0x4d,0x00,0x00,0x00, -0x4c,0x00,0x00,0x00,0x3e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x4d,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x50,0x00,0x00,0x00, -0x4e,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x89,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x55,0x00,0x00,0x00,0x50,0x00,0x00,0x00, -0x54,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x59,0x00,0x00,0x00,0x50,0x00,0x00,0x00,0x58,0x00,0x00,0x00, -0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x5d,0x00,0x00,0x00, -0x4e,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x89,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x64,0x00,0x00,0x00,0x5d,0x00,0x00,0x00, -0x63,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x68,0x00,0x00,0x00,0x5d,0x00,0x00,0x00,0x67,0x00,0x00,0x00, -0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x6e,0x00,0x00,0x00, -0x4e,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x73,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, -0x72,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0x77,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x76,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x78,0x00,0x00,0x00, -0x77,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x79,0x00,0x00,0x00,0x47,0x00,0x00,0x00,0x78,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x7c,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x7b,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x7d,0x00,0x00,0x00,0x7c,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x7f,0x00,0x00,0x00, -0x47,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x82,0x00,0x00,0x00,0x7f,0x00,0x00,0x00, -0x78,0x00,0x00,0x00,0x0c,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0x83,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x26,0x00,0x00,0x00, -0x7d,0x00,0x00,0x00,0x82,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x87,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x86,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x88,0x00,0x00,0x00,0x87,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x89,0x00,0x00,0x00,0x32,0x00,0x00,0x00, -0x88,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x8b,0x00,0x00,0x00,0x42,0x00,0x00,0x00,0x37,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x8d,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x8c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x8e,0x00,0x00,0x00,0x8d,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8f,0x00,0x00,0x00, -0x8b,0x00,0x00,0x00,0x8e,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x90,0x00,0x00,0x00,0x89,0x00,0x00,0x00, -0x8f,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x92,0x00,0x00,0x00,0x90,0x00,0x00,0x00,0x79,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x93,0x00,0x00,0x00, -0x92,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x98,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x97,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x99,0x00,0x00,0x00,0x98,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x9a,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, -0x99,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x9d,0x00,0x00,0x00,0x4a,0x00,0x00,0x00,0x9c,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x9f,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x9e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xa0,0x00,0x00,0x00,0x9f,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa1,0x00,0x00,0x00, -0x9d,0x00,0x00,0x00,0xa0,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xa2,0x00,0x00,0x00,0x9a,0x00,0x00,0x00, -0xa1,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xa4,0x00,0x00,0x00,0xa2,0x00,0x00,0x00,0x79,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa5,0x00,0x00,0x00, -0xa4,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xa7,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xa7,0x00,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x9b,0x02,0x00,0x00, -0x3e,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0xc6,0x00,0x00,0x00, -0xa8,0x00,0x00,0x00,0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00, -0xb8,0x00,0x00,0x00,0x9b,0x02,0x00,0x00,0xb6,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0xa9,0x00,0x00,0x00,0xa8,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xb8,0x00,0x00,0x00, -0xa8,0x00,0x00,0x00,0xa9,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xa8,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, -0xc3,0x00,0x00,0x00,0xbf,0x00,0x00,0x00,0x9b,0x02,0x00,0x00, -0x3e,0x00,0x03,0x00,0xc3,0x00,0x00,0x00,0xc1,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xc6,0x00,0x00,0x00, -0x9b,0x02,0x00,0x00,0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xa7,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xa9,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xc9,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xc9,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xb4,0x02,0x00,0x00,0xa5,0x00,0x00,0x00,0xa9,0x00,0x00,0x00, -0x68,0x01,0x00,0x00,0xcc,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xb0,0x02,0x00,0x00,0x93,0x00,0x00,0x00, -0xa9,0x00,0x00,0x00,0x65,0x01,0x00,0x00,0xcc,0x00,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x9c,0x02,0x00,0x00, -0x79,0x00,0x00,0x00,0xa9,0x00,0x00,0x00,0x16,0x02,0x00,0x00, -0xcc,0x00,0x00,0x00,0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00, -0xd0,0x00,0x00,0x00,0x9c,0x02,0x00,0x00,0x83,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0xcb,0x00,0x00,0x00,0xcc,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xd0,0x00,0x00,0x00, -0xca,0x00,0x00,0x00,0xcb,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xca,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xd2,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xd2,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xac,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, -0xca,0x00,0x00,0x00,0x1d,0x01,0x00,0x00,0xd5,0x00,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00,0xd8,0x00,0x00,0x00, -0xac,0x02,0x00,0x00,0x37,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0xd4,0x00,0x00,0x00,0xd5,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xd8,0x00,0x00,0x00,0xd3,0x00,0x00,0x00, -0xd4,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xd3,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xdc,0x00,0x00,0x00, -0x8b,0x00,0x00,0x00,0x73,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xde,0x00,0x00,0x00,0xdc,0x00,0x00,0x00, -0xac,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00, -0xe1,0x00,0x00,0x00,0xde,0x00,0x00,0x00,0x36,0x00,0x00,0x00, -0xf7,0x00,0x03,0x00,0xe3,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xe1,0x00,0x00,0x00,0xe2,0x00,0x00,0x00, -0xe3,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xe2,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xe6,0x00,0x00,0x00, -0x9c,0x02,0x00,0x00,0x6e,0x00,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb7,0x00,0x00,0x00,0xe8,0x00,0x00,0x00,0xe6,0x00,0x00,0x00, -0x83,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xe3,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xe3,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, -0xb7,0x00,0x00,0x00,0xe9,0x00,0x00,0x00,0xe1,0x00,0x00,0x00, -0xd3,0x00,0x00,0x00,0xe8,0x00,0x00,0x00,0xe2,0x00,0x00,0x00, -0xf7,0x00,0x03,0x00,0xeb,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xe9,0x00,0x00,0x00,0xea,0x00,0x00,0x00, -0x0d,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xea,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xf4,0x00,0x00,0x00, -0x73,0x00,0x00,0x00,0xac,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xf6,0x00,0x00,0x00,0xf4,0x00,0x00,0x00, -0xf5,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xf8,0x00,0x00,0x00,0xf6,0x00,0x00,0x00,0x6e,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x03,0x01,0x00,0x00, -0xf4,0x00,0x00,0x00,0x8e,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x04,0x01,0x00,0x00,0xb0,0x02,0x00,0x00, -0x03,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x06,0x01,0x00,0x00,0x04,0x01,0x00,0x00,0x6e,0x00,0x00,0x00, -0x41,0x00,0x06,0x00,0x07,0x01,0x00,0x00,0x08,0x01,0x00,0x00, -0xfc,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0x06,0x01,0x00,0x00, -0x3d,0x00,0x04,0x00,0xb9,0x00,0x00,0x00,0x09,0x01,0x00,0x00, -0x08,0x01,0x00,0x00,0x73,0x00,0x04,0x00,0xec,0x00,0x00,0x00, -0x0a,0x01,0x00,0x00,0x09,0x01,0x00,0x00,0x41,0x00,0x05,0x00, -0x0b,0x01,0x00,0x00,0x0c,0x01,0x00,0x00,0xf1,0x00,0x00,0x00, -0xf8,0x00,0x00,0x00,0x3e,0x00,0x03,0x00,0x0c,0x01,0x00,0x00, -0x0a,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xeb,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0x0d,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x10,0x01,0x00,0x00,0x73,0x00,0x00,0x00, -0xac,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x12,0x01,0x00,0x00,0x10,0x01,0x00,0x00,0x11,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x14,0x01,0x00,0x00, -0x12,0x01,0x00,0x00,0x6e,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x0b,0x01,0x00,0x00,0x16,0x01,0x00,0x00,0xf1,0x00,0x00,0x00, -0x14,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x16,0x01,0x00,0x00, -0x15,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xeb,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xeb,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xd5,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xd5,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x1d,0x01,0x00,0x00, -0xac,0x02,0x00,0x00,0x1b,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0xd2,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xd4,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x1f,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x1f,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xad,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0xd4,0x00,0x00,0x00, -0x61,0x01,0x00,0x00,0x22,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb7,0x00,0x00,0x00,0x25,0x01,0x00,0x00,0xad,0x02,0x00,0x00, -0x9c,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x21,0x01,0x00,0x00, -0x22,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x25,0x01,0x00,0x00,0x20,0x01,0x00,0x00,0x21,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x20,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x29,0x01,0x00,0x00,0x9d,0x00,0x00,0x00, -0x73,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x2b,0x01,0x00,0x00,0x29,0x01,0x00,0x00,0xad,0x02,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x2c,0x01,0x00,0x00, -0x12,0x00,0x00,0x00,0xc5,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x2d,0x01,0x00,0x00,0x2c,0x01,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00,0x2e,0x01,0x00,0x00, -0x2b,0x01,0x00,0x00,0x2d,0x01,0x00,0x00,0xf7,0x00,0x03,0x00, -0x30,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x2e,0x01,0x00,0x00,0x2f,0x01,0x00,0x00,0x30,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x2f,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x33,0x01,0x00,0x00,0x9c,0x02,0x00,0x00, -0x6e,0x00,0x00,0x00,0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00, -0x35,0x01,0x00,0x00,0x33,0x01,0x00,0x00,0x83,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x30,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x30,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0xb7,0x00,0x00,0x00, -0x36,0x01,0x00,0x00,0x2e,0x01,0x00,0x00,0x20,0x01,0x00,0x00, -0x35,0x01,0x00,0x00,0x2f,0x01,0x00,0x00,0xf7,0x00,0x03,0x00, -0x38,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x36,0x01,0x00,0x00,0x37,0x01,0x00,0x00,0x57,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x37,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x40,0x01,0x00,0x00,0x73,0x00,0x00,0x00, -0xad,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x42,0x01,0x00,0x00,0x40,0x01,0x00,0x00,0x41,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x44,0x01,0x00,0x00, -0x42,0x01,0x00,0x00,0x6e,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x4f,0x01,0x00,0x00,0x40,0x01,0x00,0x00, -0xa0,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x50,0x01,0x00,0x00,0xb4,0x02,0x00,0x00,0x4f,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x52,0x01,0x00,0x00, -0x50,0x01,0x00,0x00,0x6e,0x00,0x00,0x00,0x41,0x00,0x06,0x00, -0x07,0x01,0x00,0x00,0x53,0x01,0x00,0x00,0x48,0x01,0x00,0x00, -0x34,0x00,0x00,0x00,0x52,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, -0xb9,0x00,0x00,0x00,0x54,0x01,0x00,0x00,0x53,0x01,0x00,0x00, -0x73,0x00,0x04,0x00,0xec,0x00,0x00,0x00,0x55,0x01,0x00,0x00, -0x54,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0x0b,0x01,0x00,0x00, -0x56,0x01,0x00,0x00,0x3d,0x01,0x00,0x00,0x44,0x01,0x00,0x00, -0x3e,0x00,0x03,0x00,0x56,0x01,0x00,0x00,0x55,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0x38,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x57,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x5a,0x01,0x00,0x00,0x73,0x00,0x00,0x00,0xad,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x5c,0x01,0x00,0x00, -0x5a,0x01,0x00,0x00,0x5b,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x5e,0x01,0x00,0x00,0x5c,0x01,0x00,0x00, -0x6e,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x0b,0x01,0x00,0x00, -0x5f,0x01,0x00,0x00,0x3d,0x01,0x00,0x00,0x5e,0x01,0x00,0x00, -0x3e,0x00,0x03,0x00,0x5f,0x01,0x00,0x00,0x15,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0x38,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x38,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x22,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x22,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x61,0x01,0x00,0x00,0xad,0x02,0x00,0x00, -0x1b,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x1f,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x21,0x01,0x00,0x00,0xe0,0x00,0x04,0x00, -0x0c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x62,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x65,0x01,0x00,0x00, -0xb0,0x02,0x00,0x00,0x63,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x68,0x01,0x00,0x00,0xb4,0x02,0x00,0x00, -0x66,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x6a,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x6a,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xb6,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, -0x21,0x01,0x00,0x00,0x14,0x02,0x00,0x00,0x6d,0x01,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00,0x70,0x01,0x00,0x00, -0xb6,0x02,0x00,0x00,0x6c,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0x6c,0x01,0x00,0x00,0x6d,0x01,0x00,0x00,0x00,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x70,0x01,0x00,0x00,0x6b,0x01,0x00,0x00, -0x6c,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x6b,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0x72,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x72,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xba,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0x6b,0x01,0x00,0x00, -0x9e,0x01,0x00,0x00,0x75,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb7,0x00,0x00,0x00,0x78,0x01,0x00,0x00,0xba,0x02,0x00,0x00, -0x60,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x74,0x01,0x00,0x00, -0x75,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x78,0x01,0x00,0x00,0x73,0x01,0x00,0x00,0x74,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x73,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0x7a,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x7a,0x01,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xcc,0x02,0x00,0x00, -0x3e,0x00,0x00,0x00,0x73,0x01,0x00,0x00,0x9c,0x01,0x00,0x00, -0x7b,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00, -0x80,0x01,0x00,0x00,0xcc,0x02,0x00,0x00,0x62,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0x7c,0x01,0x00,0x00,0x7b,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x80,0x01,0x00,0x00, -0x7b,0x01,0x00,0x00,0x7c,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x7b,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x86,0x01,0x00,0x00,0xba,0x02,0x00,0x00,0x62,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x88,0x01,0x00,0x00, -0x86,0x01,0x00,0x00,0xcc,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x8a,0x01,0x00,0x00,0x55,0x00,0x00,0x00, -0x53,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x8c,0x01,0x00,0x00,0xba,0x02,0x00,0x00,0x61,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8d,0x01,0x00,0x00, -0x8a,0x01,0x00,0x00,0x8c,0x01,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x8f,0x01,0x00,0x00,0x64,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x90,0x01,0x00,0x00,0x8d,0x01,0x00,0x00,0x8f,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x92,0x01,0x00,0x00, -0x90,0x01,0x00,0x00,0xcc,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x94,0x01,0x00,0x00,0x92,0x01,0x00,0x00, -0x93,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x96,0x01,0x00,0x00,0x94,0x01,0x00,0x00,0xb6,0x02,0x00,0x00, -0x41,0x00,0x05,0x00,0x0b,0x01,0x00,0x00,0x97,0x01,0x00,0x00, -0xf1,0x00,0x00,0x00,0x96,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, -0xec,0x00,0x00,0x00,0x98,0x01,0x00,0x00,0x97,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0x99,0x01,0x00,0x00,0x9a,0x01,0x00,0x00, -0x84,0x01,0x00,0x00,0x88,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0x9a,0x01,0x00,0x00,0x98,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x9c,0x01,0x00,0x00,0xcc,0x02,0x00,0x00, -0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x7a,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x7c,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0x75,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x75,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9e,0x01,0x00,0x00, -0xba,0x02,0x00,0x00,0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x72,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x74,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0xa0,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xa0,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xbb,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0x74,0x01,0x00,0x00, -0xcc,0x01,0x00,0x00,0xa3,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb7,0x00,0x00,0x00,0xa6,0x01,0x00,0x00,0xbb,0x02,0x00,0x00, -0xb4,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xa2,0x01,0x00,0x00, -0xa3,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0xa6,0x01,0x00,0x00,0xa1,0x01,0x00,0x00,0xa2,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xa1,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0xa8,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xa8,0x01,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xc9,0x02,0x00,0x00, -0x3e,0x00,0x00,0x00,0xa1,0x01,0x00,0x00,0xca,0x01,0x00,0x00, -0xa9,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00, -0xae,0x01,0x00,0x00,0xc9,0x02,0x00,0x00,0xb1,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0xaa,0x01,0x00,0x00,0xa9,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xae,0x01,0x00,0x00, -0xa9,0x01,0x00,0x00,0xaa,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xa9,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xb4,0x01,0x00,0x00,0xbb,0x02,0x00,0x00,0xb1,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xb6,0x01,0x00,0x00, -0xb4,0x01,0x00,0x00,0xc9,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xb8,0x01,0x00,0x00,0x59,0x00,0x00,0x00, -0xae,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xbb,0x01,0x00,0x00,0xbb,0x02,0x00,0x00,0xba,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xbc,0x01,0x00,0x00, -0xb8,0x01,0x00,0x00,0xbb,0x01,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xbe,0x01,0x00,0x00,0x68,0x00,0x00,0x00, -0xb1,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xbf,0x01,0x00,0x00,0xbc,0x01,0x00,0x00,0xbe,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xc1,0x01,0x00,0x00, -0xbf,0x01,0x00,0x00,0xc9,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xc3,0x01,0x00,0x00,0xc1,0x01,0x00,0x00, -0xc2,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xc5,0x01,0x00,0x00,0xc3,0x01,0x00,0x00,0xb6,0x02,0x00,0x00, -0x41,0x00,0x05,0x00,0x0b,0x01,0x00,0x00,0xc6,0x01,0x00,0x00, -0x3d,0x01,0x00,0x00,0xc5,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, -0xec,0x00,0x00,0x00,0xc7,0x01,0x00,0x00,0xc6,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0x99,0x01,0x00,0x00,0xc8,0x01,0x00,0x00, -0xb2,0x01,0x00,0x00,0xb6,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0xc8,0x01,0x00,0x00,0xc7,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xca,0x01,0x00,0x00,0xc9,0x02,0x00,0x00, -0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xa8,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xaa,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0xa3,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xa3,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xcc,0x01,0x00,0x00, -0xbb,0x02,0x00,0x00,0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xa0,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xa2,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0xce,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xce,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xbc,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0xa2,0x01,0x00,0x00, -0x12,0x02,0x00,0x00,0xd1,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb7,0x00,0x00,0x00,0xd4,0x01,0x00,0x00,0xbc,0x02,0x00,0x00, -0xb4,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xd0,0x01,0x00,0x00, -0xd1,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0xd4,0x01,0x00,0x00,0xcf,0x01,0x00,0x00,0xd0,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xcf,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0xd6,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xd6,0x01,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xc0,0x02,0x00,0x00, -0x3e,0x00,0x00,0x00,0xcf,0x01,0x00,0x00,0x10,0x02,0x00,0x00, -0xd9,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00, -0xdc,0x01,0x00,0x00,0xc0,0x02,0x00,0x00,0x60,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0xd8,0x01,0x00,0x00,0xd9,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xdc,0x01,0x00,0x00, -0xd7,0x01,0x00,0x00,0xd8,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xd7,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xde,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xde,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xc2,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, -0xd7,0x01,0x00,0x00,0x0e,0x02,0x00,0x00,0xe1,0x01,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00,0xe4,0x01,0x00,0x00, -0xc2,0x02,0x00,0x00,0xb1,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0xe0,0x01,0x00,0x00,0xe1,0x01,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xe4,0x01,0x00,0x00,0xdf,0x01,0x00,0x00, -0xe0,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xdf,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0xe6,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xe6,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xc4,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0xdf,0x01,0x00,0x00, -0x0c,0x02,0x00,0x00,0xe7,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb7,0x00,0x00,0x00,0xec,0x01,0x00,0x00,0xc4,0x02,0x00,0x00, -0x62,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xe8,0x01,0x00,0x00, -0xe7,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0xec,0x01,0x00,0x00,0xe7,0x01,0x00,0x00,0xe8,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xe7,0x01,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xee,0x01,0x00,0x00,0xbc,0x02,0x00,0x00, -0xb1,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xf0,0x01,0x00,0x00,0xee,0x01,0x00,0x00,0xc2,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xf2,0x01,0x00,0x00, -0xf0,0x01,0x00,0x00,0xf1,0x01,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xf4,0x01,0x00,0x00,0xc0,0x02,0x00,0x00, -0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xf5,0x01,0x00,0x00,0xf2,0x01,0x00,0x00,0xf4,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xf7,0x01,0x00,0x00, -0xf5,0x01,0x00,0x00,0xc4,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xfb,0x01,0x00,0x00,0xf4,0x01,0x00,0x00, -0xc4,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x99,0x01,0x00,0x00, -0xfc,0x01,0x00,0x00,0x84,0x01,0x00,0x00,0xfb,0x01,0x00,0x00, -0x3d,0x00,0x04,0x00,0xec,0x00,0x00,0x00,0xfd,0x01,0x00,0x00, -0xfc,0x01,0x00,0x00,0x73,0x00,0x04,0x00,0xb9,0x00,0x00,0x00, -0xfe,0x01,0x00,0x00,0xfd,0x01,0x00,0x00,0x41,0x00,0x05,0x00, -0x99,0x01,0x00,0x00,0x03,0x02,0x00,0x00,0xb2,0x01,0x00,0x00, -0xf0,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0xec,0x00,0x00,0x00, -0x04,0x02,0x00,0x00,0x03,0x02,0x00,0x00,0x73,0x00,0x04,0x00, -0xb9,0x00,0x00,0x00,0x05,0x02,0x00,0x00,0x04,0x02,0x00,0x00, -0x41,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0x07,0x02,0x00,0x00, -0xbf,0x00,0x00,0x00,0xf7,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, -0xb9,0x00,0x00,0x00,0x08,0x02,0x00,0x00,0x07,0x02,0x00,0x00, -0x0c,0x00,0x08,0x00,0xb9,0x00,0x00,0x00,0x09,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0xfe,0x01,0x00,0x00, -0x05,0x02,0x00,0x00,0x08,0x02,0x00,0x00,0x3e,0x00,0x03,0x00, -0x07,0x02,0x00,0x00,0x09,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x0c,0x02,0x00,0x00,0xc4,0x02,0x00,0x00, -0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xe6,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xe8,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0xe1,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xe1,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x0e,0x02,0x00,0x00, -0xc2,0x02,0x00,0x00,0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xde,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xe0,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0xd9,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xd9,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x10,0x02,0x00,0x00,0xc0,0x02,0x00,0x00,0xc5,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xd6,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xd8,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xd1,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xd1,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x12,0x02,0x00,0x00,0xbc,0x02,0x00,0x00, -0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xce,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xd0,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0x6d,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x6d,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x14,0x02,0x00,0x00, -0xb6,0x02,0x00,0x00,0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x6a,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x6c,0x01,0x00,0x00, -0xe0,0x00,0x04,0x00,0x0c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x62,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xcc,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xcc,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x16,0x02,0x00,0x00,0x9c,0x02,0x00,0x00, -0x6c,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xc9,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xcb,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x1b,0x02,0x00,0x00,0x55,0x00,0x00,0x00, -0x53,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x1c,0x02,0x00,0x00,0x8b,0x00,0x00,0x00,0x1b,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x21,0x02,0x00,0x00, -0x59,0x00,0x00,0x00,0xae,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x22,0x02,0x00,0x00,0x9d,0x00,0x00,0x00, -0x21,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0x27,0x02,0x00,0x00,0x12,0x00,0x00,0x00,0x26,0x02,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x28,0x02,0x00,0x00, -0x27,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x29,0x02,0x00,0x00,0x0f,0x00,0x00,0x00,0x28,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x2d,0x02,0x00,0x00, -0x47,0x00,0x00,0x00,0x28,0x02,0x00,0x00,0x41,0x00,0x05,0x00, -0x0d,0x00,0x00,0x00,0x2f,0x02,0x00,0x00,0x2e,0x02,0x00,0x00, -0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x30,0x02,0x00,0x00,0x2f,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x31,0x02,0x00,0x00,0x2d,0x02,0x00,0x00, -0x30,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x32,0x02,0x00,0x00,0x29,0x02,0x00,0x00,0x31,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0x34,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x34,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0x9d,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0xcb,0x00,0x00,0x00, -0x9a,0x02,0x00,0x00,0x37,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb7,0x00,0x00,0x00,0x3a,0x02,0x00,0x00,0x9d,0x02,0x00,0x00, -0xb4,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x36,0x02,0x00,0x00, -0x37,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x3a,0x02,0x00,0x00,0x35,0x02,0x00,0x00,0x36,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x35,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x3c,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x3c,0x02,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x9e,0x02,0x00,0x00, -0x3e,0x00,0x00,0x00,0x35,0x02,0x00,0x00,0x98,0x02,0x00,0x00, -0x3f,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00, -0x42,0x02,0x00,0x00,0x9e,0x02,0x00,0x00,0x60,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0x3e,0x02,0x00,0x00,0x3f,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x42,0x02,0x00,0x00, -0x3d,0x02,0x00,0x00,0x3e,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x3d,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x46,0x02,0x00,0x00,0x9e,0x02,0x00,0x00,0x61,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x47,0x02,0x00,0x00, -0x1c,0x02,0x00,0x00,0x46,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x49,0x02,0x00,0x00,0x64,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x4a,0x02,0x00,0x00,0x47,0x02,0x00,0x00,0x49,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x4e,0x02,0x00,0x00, -0x9d,0x02,0x00,0x00,0xba,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x4f,0x02,0x00,0x00,0x22,0x02,0x00,0x00, -0x4e,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x51,0x02,0x00,0x00,0x68,0x00,0x00,0x00,0xb1,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x52,0x02,0x00,0x00, -0x4f,0x02,0x00,0x00,0x51,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x54,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x54,0x02,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xa0,0x02,0x00,0x00, -0x3e,0x00,0x00,0x00,0x3d,0x02,0x00,0x00,0x96,0x02,0x00,0x00, -0x57,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00, -0x5a,0x02,0x00,0x00,0xa0,0x02,0x00,0x00,0xb1,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0x56,0x02,0x00,0x00,0x57,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x5a,0x02,0x00,0x00, -0x55,0x02,0x00,0x00,0x56,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x55,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x5c,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x5c,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xa2,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, -0x55,0x02,0x00,0x00,0x94,0x02,0x00,0x00,0x5f,0x02,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00,0x62,0x02,0x00,0x00, -0xa2,0x02,0x00,0x00,0x62,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0x5e,0x02,0x00,0x00,0x5f,0x02,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x62,0x02,0x00,0x00,0x5d,0x02,0x00,0x00, -0x5e,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x5d,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x65,0x02,0x00,0x00, -0x4a,0x02,0x00,0x00,0xa2,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb7,0x00,0x00,0x00,0x68,0x02,0x00,0x00,0x65,0x02,0x00,0x00, -0x36,0x00,0x00,0x00,0xf7,0x00,0x03,0x00,0x6a,0x02,0x00,0x00, -0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x68,0x02,0x00,0x00, -0x69,0x02,0x00,0x00,0x6a,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x69,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x6d,0x02,0x00,0x00,0x52,0x02,0x00,0x00,0xa0,0x02,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x6e,0x02,0x00,0x00, -0x12,0x00,0x00,0x00,0xc5,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x6f,0x02,0x00,0x00,0x6e,0x02,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00,0x70,0x02,0x00,0x00, -0x6d,0x02,0x00,0x00,0x6f,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x6a,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x6a,0x02,0x00,0x00, -0xf5,0x00,0x07,0x00,0xb7,0x00,0x00,0x00,0x71,0x02,0x00,0x00, -0x68,0x02,0x00,0x00,0x5d,0x02,0x00,0x00,0x70,0x02,0x00,0x00, -0x69,0x02,0x00,0x00,0xf7,0x00,0x03,0x00,0x73,0x02,0x00,0x00, -0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x71,0x02,0x00,0x00, -0x72,0x02,0x00,0x00,0x73,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x72,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x7b,0x02,0x00,0x00,0x52,0x02,0x00,0x00,0xa0,0x02,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x7d,0x02,0x00,0x00, -0x12,0x00,0x00,0x00,0x7c,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x7e,0x02,0x00,0x00,0x7d,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x7f,0x02,0x00,0x00, -0x7b,0x02,0x00,0x00,0x7e,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x80,0x02,0x00,0x00,0x32,0x02,0x00,0x00, -0x7f,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x82,0x02,0x00,0x00,0x80,0x02,0x00,0x00,0x4a,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x84,0x02,0x00,0x00, -0x82,0x02,0x00,0x00,0xa2,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x86,0x02,0x00,0x00,0x9d,0x02,0x00,0x00, -0xb1,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x88,0x02,0x00,0x00,0x86,0x02,0x00,0x00,0xa0,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8a,0x02,0x00,0x00, -0x88,0x02,0x00,0x00,0x89,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x8c,0x02,0x00,0x00,0x9e,0x02,0x00,0x00, -0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x8d,0x02,0x00,0x00,0x8a,0x02,0x00,0x00,0x8c,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8f,0x02,0x00,0x00, -0x8d,0x02,0x00,0x00,0xa2,0x02,0x00,0x00,0x41,0x00,0x05,0x00, -0xc2,0x00,0x00,0x00,0x90,0x02,0x00,0x00,0xbf,0x00,0x00,0x00, -0x8f,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0xb9,0x00,0x00,0x00, -0x91,0x02,0x00,0x00,0x90,0x02,0x00,0x00,0x41,0x00,0x06,0x00, -0x07,0x01,0x00,0x00,0x92,0x02,0x00,0x00,0x77,0x02,0x00,0x00, -0x34,0x00,0x00,0x00,0x84,0x02,0x00,0x00,0x3e,0x00,0x03,0x00, -0x92,0x02,0x00,0x00,0x91,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x73,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x73,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0x5f,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x5f,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x94,0x02,0x00,0x00,0xa2,0x02,0x00,0x00,0xc5,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x5c,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x5e,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x57,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x57,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x96,0x02,0x00,0x00,0xa0,0x02,0x00,0x00, -0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x54,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x56,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x3f,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x3f,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x98,0x02,0x00,0x00, -0x9e,0x02,0x00,0x00,0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x3c,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x3e,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0x37,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x37,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x9a,0x02,0x00,0x00,0x9d,0x02,0x00,0x00,0xc5,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x34,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x36,0x02,0x00,0x00,0xfd,0x00,0x01,0x00,0x38,0x00,0x01,0x00, - -}; -const uint64_t matmul_f32_s_len = 10164; - -unsigned char matmul_f32_s_fp32_data[] = { -0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, -0xc6,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, -0x01,0x00,0x00,0x00,0x0b,0x00,0x06,0x00,0x01,0x00,0x00,0x00, -0x47,0x4c,0x53,0x4c,0x2e,0x73,0x74,0x64,0x2e,0x34,0x35,0x30, -0x00,0x00,0x00,0x00,0x0e,0x00,0x03,0x00,0x00,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x0f,0x00,0x0f,0x00,0x05,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x6d,0x61,0x69,0x6e,0x00,0x00,0x00,0x00, -0x0b,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x3d,0x00,0x00,0x00, -0x4c,0x00,0x00,0x00,0xf0,0x00,0x00,0x00,0xfb,0x00,0x00,0x00, -0x3a,0x01,0x00,0x00,0x45,0x01,0x00,0x00,0x27,0x02,0x00,0x00, -0x70,0x02,0x00,0x00,0x10,0x00,0x06,0x00,0x04,0x00,0x00,0x00, -0x11,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x0b,0x00,0x00,0x00, -0x0b,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x10,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x10,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x10,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x05,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x10,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x07,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x1c,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x08,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x10,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x24,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x0a,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x28,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x2c,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x10,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x30,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x10,0x00,0x00,0x00, -0x0d,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x34,0x00,0x00,0x00, -0x47,0x00,0x03,0x00,0x10,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x37,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x3d,0x00,0x00,0x00, -0x0b,0x00,0x00,0x00,0x1a,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x4c,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x1b,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x4f,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x53,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x60,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x62,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x07,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x6c,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x9c,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0xae,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x05,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xb1,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0xf8,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x48,0x00,0x04,0x00,0xf9,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0xf9,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x03,0x00,0xf9,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0xfb,0x00,0x00,0x00,0x22,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xfb,0x00,0x00,0x00, -0x21,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x14,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x15,0x01,0x00,0x00,0x0b,0x00,0x00,0x00, -0x19,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x42,0x01,0x00,0x00, -0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00, -0x43,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x43,0x01,0x00,0x00,0x00,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, -0x43,0x01,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x45,0x01,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x45,0x01,0x00,0x00,0x21,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x27,0x02,0x00,0x00, -0x0b,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x6d,0x02,0x00,0x00,0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x48,0x00,0x04,0x00,0x6e,0x02,0x00,0x00,0x00,0x00,0x00,0x00, -0x19,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x6e,0x02,0x00,0x00, -0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x03,0x00,0x6e,0x02,0x00,0x00,0x02,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x70,0x02,0x00,0x00,0x22,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x70,0x02,0x00,0x00, -0x21,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x13,0x00,0x02,0x00, -0x02,0x00,0x00,0x00,0x21,0x00,0x03,0x00,0x03,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x15,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x17,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x03,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x0a,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00, -0x0b,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x0d,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x1e,0x00,0x10,0x00,0x10,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x11,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x10,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x11,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x15,0x00,0x04,0x00,0x13,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x08,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x15,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, -0x21,0x00,0x00,0x00,0x0a,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0x27,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x2d,0x00,0x00,0x00, -0x07,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, -0x34,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x32,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00, -0x3d,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x3e,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00,0x4c,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x4f,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x32,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x54,0x00,0x00,0x00, -0x86,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x53,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x58,0x00,0x00,0x00, -0x86,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x53,0x00,0x00,0x00, -0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x60,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x61,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x53,0x00,0x00,0x00, -0x60,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x63,0x00,0x00,0x00,0x86,0x00,0x00,0x00, -0x61,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x67,0x00,0x00,0x00,0x86,0x00,0x00,0x00, -0x61,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x32,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x10,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, -0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x72,0x00,0x00,0x00, -0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x76,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, -0x7b,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x8c,0x00,0x00,0x00, -0x03,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, -0x97,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x32,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x9c,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x9e,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xad,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xae,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xaf,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0x53,0x00,0x00,0x00,0xae,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xb0,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0x4f,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x32,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xb1,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xb2,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0xb0,0x00,0x00,0x00,0xb1,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xb3,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0xb2,0x00,0x00,0x00,0x60,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xb4,0x00,0x00,0x00, -0x86,0x00,0x00,0x00,0xaf,0x00,0x00,0x00,0xb3,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xb5,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0xad,0x00,0x00,0x00,0xb4,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xb6,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0xb5,0x00,0x00,0x00,0xb1,0x00,0x00,0x00, -0x14,0x00,0x02,0x00,0xb7,0x00,0x00,0x00,0x16,0x00,0x03,0x00, -0xb9,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xba,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xbb,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0xba,0x00,0x00,0x00,0xb4,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xbc,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0xbb,0x00,0x00,0x00,0xb1,0x00,0x00,0x00,0x1c,0x00,0x04,0x00, -0xbd,0x00,0x00,0x00,0xb9,0x00,0x00,0x00,0xbc,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0xbe,0x00,0x00,0x00,0x07,0x00,0x00,0x00, -0xbd,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0xb9,0x00,0x00,0x00, -0xc1,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0xc2,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0xb9,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0xc5,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xec,0x00,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xed,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x37,0x00,0x00,0x00, -0xec,0x00,0x00,0x00,0x1c,0x00,0x04,0x00,0xee,0x00,0x00,0x00, -0xb9,0x00,0x00,0x00,0xed,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0xef,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0xee,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0xef,0x00,0x00,0x00,0xf0,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xf4,0x00,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x1d,0x00,0x03,0x00,0xf8,0x00,0x00,0x00, -0xb9,0x00,0x00,0x00,0x1e,0x00,0x03,0x00,0xf9,0x00,0x00,0x00, -0xf8,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0xfa,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0xf9,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0xfa,0x00,0x00,0x00,0xfb,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x06,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, -0xb9,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x09,0x01,0x00,0x00, -0x04,0x00,0x00,0x00,0xb9,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x0f,0x01,0x00,0x00,0x80,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x32,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x14,0x01,0x00,0x00,0x01,0x00,0x00,0x00, -0x33,0x00,0x06,0x00,0x09,0x00,0x00,0x00,0x15,0x01,0x00,0x00, -0x14,0x01,0x00,0x00,0x39,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x16,0x01,0x00,0x00, -0x51,0x00,0x00,0x00,0x15,0x01,0x00,0x00,0x00,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x17,0x01,0x00,0x00, -0x84,0x00,0x00,0x00,0x16,0x01,0x00,0x00,0x39,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x18,0x01,0x00,0x00, -0x86,0x00,0x00,0x00,0x17,0x01,0x00,0x00,0x6c,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x36,0x01,0x00,0x00, -0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x37,0x01,0x00,0x00, -0x84,0x00,0x00,0x00,0x9c,0x00,0x00,0x00,0x36,0x01,0x00,0x00, -0x1c,0x00,0x04,0x00,0x38,0x01,0x00,0x00,0xb9,0x00,0x00,0x00, -0x37,0x01,0x00,0x00,0x20,0x00,0x04,0x00,0x39,0x01,0x00,0x00, -0x04,0x00,0x00,0x00,0x38,0x01,0x00,0x00,0x3b,0x00,0x04,0x00, -0x39,0x01,0x00,0x00,0x3a,0x01,0x00,0x00,0x04,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x3e,0x01,0x00,0x00, -0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x1d,0x00,0x03,0x00,0x42,0x01,0x00,0x00,0xb9,0x00,0x00,0x00, -0x1e,0x00,0x03,0x00,0x43,0x01,0x00,0x00,0x42,0x01,0x00,0x00, -0x20,0x00,0x04,0x00,0x44,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, -0x43,0x01,0x00,0x00,0x3b,0x00,0x04,0x00,0x44,0x01,0x00,0x00, -0x45,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x57,0x01,0x00,0x00,0x80,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x5e,0x01,0x00,0x00,0x08,0x01,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x5f,0x01,0x00,0x00, -0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x62,0x01,0x00,0x00, -0x86,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x7d,0x01,0x00,0x00, -0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00, -0x1c,0x00,0x04,0x00,0x7e,0x01,0x00,0x00,0xb9,0x00,0x00,0x00, -0x7d,0x01,0x00,0x00,0x20,0x00,0x04,0x00,0x7f,0x01,0x00,0x00, -0x07,0x00,0x00,0x00,0x7e,0x01,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x8f,0x01,0x00,0x00,0x80,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xaa,0x01,0x00,0x00,0x84,0x00,0x00,0x00, -0xb4,0x00,0x00,0x00,0xb1,0x00,0x00,0x00,0x1c,0x00,0x04,0x00, -0xab,0x01,0x00,0x00,0xb9,0x00,0x00,0x00,0xaa,0x01,0x00,0x00, -0x20,0x00,0x04,0x00,0xac,0x01,0x00,0x00,0x07,0x00,0x00,0x00, -0xab,0x01,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xb5,0x01,0x00,0x00,0x86,0x00,0x00,0x00,0xae,0x00,0x00,0x00, -0xb4,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xbd,0x01,0x00,0x00,0x80,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xec,0x01,0x00,0x00,0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, -0x1f,0x02,0x00,0x00,0x0d,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x0a,0x00,0x00,0x00,0x27,0x02,0x00,0x00,0x01,0x00,0x00,0x00, -0x1d,0x00,0x03,0x00,0x6d,0x02,0x00,0x00,0xb9,0x00,0x00,0x00, -0x1e,0x00,0x03,0x00,0x6e,0x02,0x00,0x00,0x6d,0x02,0x00,0x00, -0x20,0x00,0x04,0x00,0x6f,0x02,0x00,0x00,0x0c,0x00,0x00,0x00, -0x6e,0x02,0x00,0x00,0x3b,0x00,0x04,0x00,0x6f,0x02,0x00,0x00, -0x70,0x02,0x00,0x00,0x0c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0x75,0x02,0x00,0x00,0x05,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x82,0x02,0x00,0x00, -0x84,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x62,0x00,0x00,0x00, -0x36,0x00,0x05,0x00,0x02,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x05,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0xbe,0x00,0x00,0x00, -0xbf,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x7f,0x01,0x00,0x00,0x80,0x01,0x00,0x00,0x07,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0xac,0x01,0x00,0x00,0xad,0x01,0x00,0x00, -0x07,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00, -0x0e,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, -0x0e,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x14,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x17,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x17,0x00,0x00,0x00, -0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x1e,0x00,0x00,0x00, -0x0f,0x00,0x00,0x00,0x17,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x21,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x24,0x00,0x00,0x00,0x18,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0x28,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x27,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x29,0x00,0x00,0x00, -0x28,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x2a,0x00,0x00,0x00,0x1e,0x00,0x00,0x00,0x29,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x2d,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x2f,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x30,0x00,0x00,0x00, -0x24,0x00,0x00,0x00,0x2f,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x30,0x00,0x00,0x00, -0x2a,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0x35,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x34,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x36,0x00,0x00,0x00, -0x35,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x38,0x00,0x00,0x00,0x36,0x00,0x00,0x00,0x37,0x00,0x00,0x00, -0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, -0x38,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x3b,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, -0x37,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00, -0x3f,0x00,0x00,0x00,0x3d,0x00,0x00,0x00,0x3e,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x3f,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x42,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x3b,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x47,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0x3b,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x0d,0x00,0x00,0x00,0x49,0x00,0x00,0x00,0x3d,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x4a,0x00,0x00,0x00,0x49,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x0d,0x00,0x00,0x00,0x4d,0x00,0x00,0x00,0x4c,0x00,0x00,0x00, -0x3e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x4e,0x00,0x00,0x00,0x4d,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x50,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, -0x4f,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x55,0x00,0x00,0x00,0x50,0x00,0x00,0x00,0x54,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x59,0x00,0x00,0x00, -0x50,0x00,0x00,0x00,0x58,0x00,0x00,0x00,0x89,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x5d,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, -0x4f,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x64,0x00,0x00,0x00,0x5d,0x00,0x00,0x00,0x63,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x68,0x00,0x00,0x00, -0x5d,0x00,0x00,0x00,0x67,0x00,0x00,0x00,0x89,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x6e,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, -0x6d,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x73,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x72,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x77,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x76,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x78,0x00,0x00,0x00,0x77,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x79,0x00,0x00,0x00, -0x47,0x00,0x00,0x00,0x78,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x7c,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x7b,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x7d,0x00,0x00,0x00,0x7c,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x7f,0x00,0x00,0x00,0x47,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x82,0x00,0x00,0x00,0x7f,0x00,0x00,0x00,0x78,0x00,0x00,0x00, -0x0c,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x83,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x26,0x00,0x00,0x00,0x7d,0x00,0x00,0x00, -0x82,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0x87,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x86,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x88,0x00,0x00,0x00, -0x87,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x89,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x88,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8b,0x00,0x00,0x00, -0x42,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x8d,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x8c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x8e,0x00,0x00,0x00,0x8d,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x8f,0x00,0x00,0x00,0x8b,0x00,0x00,0x00, -0x8e,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x90,0x00,0x00,0x00,0x89,0x00,0x00,0x00,0x8f,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x92,0x00,0x00,0x00, -0x90,0x00,0x00,0x00,0x79,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x93,0x00,0x00,0x00,0x92,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0x98,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x97,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x99,0x00,0x00,0x00, -0x98,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x9a,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x99,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9d,0x00,0x00,0x00, -0x4a,0x00,0x00,0x00,0x9c,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x9f,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x9e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xa0,0x00,0x00,0x00,0x9f,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xa1,0x00,0x00,0x00,0x9d,0x00,0x00,0x00, -0xa0,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xa2,0x00,0x00,0x00,0x9a,0x00,0x00,0x00,0xa1,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa4,0x00,0x00,0x00, -0xa2,0x00,0x00,0x00,0x79,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xa5,0x00,0x00,0x00,0xa4,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xa7,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xa7,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0x94,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, -0x05,0x00,0x00,0x00,0xc6,0x00,0x00,0x00,0xa8,0x00,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00,0xb8,0x00,0x00,0x00, -0x94,0x02,0x00,0x00,0xb6,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0xa9,0x00,0x00,0x00,0xa8,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xb8,0x00,0x00,0x00,0xa8,0x00,0x00,0x00, -0xa9,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xa8,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0xc3,0x00,0x00,0x00, -0xbf,0x00,0x00,0x00,0x94,0x02,0x00,0x00,0x3e,0x00,0x03,0x00, -0xc3,0x00,0x00,0x00,0xc1,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xc6,0x00,0x00,0x00,0x94,0x02,0x00,0x00, -0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xa7,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xa9,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xc9,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xc9,0x00,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xad,0x02,0x00,0x00, -0xa5,0x00,0x00,0x00,0xa9,0x00,0x00,0x00,0x64,0x01,0x00,0x00, -0xcc,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xa9,0x02,0x00,0x00,0x93,0x00,0x00,0x00,0xa9,0x00,0x00,0x00, -0x61,0x01,0x00,0x00,0xcc,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0x95,0x02,0x00,0x00,0x79,0x00,0x00,0x00, -0xa9,0x00,0x00,0x00,0x0f,0x02,0x00,0x00,0xcc,0x00,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00,0xd0,0x00,0x00,0x00, -0x95,0x02,0x00,0x00,0x83,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0xcb,0x00,0x00,0x00,0xcc,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xd0,0x00,0x00,0x00,0xca,0x00,0x00,0x00, -0xcb,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xca,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xd2,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xd2,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xa5,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0xca,0x00,0x00,0x00, -0x1a,0x01,0x00,0x00,0xd5,0x00,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb7,0x00,0x00,0x00,0xd8,0x00,0x00,0x00,0xa5,0x02,0x00,0x00, -0x37,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xd4,0x00,0x00,0x00, -0xd5,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0xd8,0x00,0x00,0x00,0xd3,0x00,0x00,0x00,0xd4,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xd3,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xdc,0x00,0x00,0x00,0x8b,0x00,0x00,0x00, -0x73,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xde,0x00,0x00,0x00,0xdc,0x00,0x00,0x00,0xa5,0x02,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00,0xe1,0x00,0x00,0x00, -0xde,0x00,0x00,0x00,0x36,0x00,0x00,0x00,0xf7,0x00,0x03,0x00, -0xe3,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0xe1,0x00,0x00,0x00,0xe2,0x00,0x00,0x00,0xe3,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xe2,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xe6,0x00,0x00,0x00,0x95,0x02,0x00,0x00, -0x6e,0x00,0x00,0x00,0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00, -0xe8,0x00,0x00,0x00,0xe6,0x00,0x00,0x00,0x83,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xe3,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xe3,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0xb7,0x00,0x00,0x00, -0xe9,0x00,0x00,0x00,0xe1,0x00,0x00,0x00,0xd3,0x00,0x00,0x00, -0xe8,0x00,0x00,0x00,0xe2,0x00,0x00,0x00,0xf7,0x00,0x03,0x00, -0xeb,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0xe9,0x00,0x00,0x00,0xea,0x00,0x00,0x00,0x0b,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xea,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xf3,0x00,0x00,0x00,0x73,0x00,0x00,0x00, -0xa5,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xf5,0x00,0x00,0x00,0xf3,0x00,0x00,0x00,0xf4,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xf7,0x00,0x00,0x00, -0xf5,0x00,0x00,0x00,0x6e,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x02,0x01,0x00,0x00,0xf3,0x00,0x00,0x00, -0x8e,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x03,0x01,0x00,0x00,0xa9,0x02,0x00,0x00,0x02,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x05,0x01,0x00,0x00, -0x03,0x01,0x00,0x00,0x6e,0x00,0x00,0x00,0x41,0x00,0x06,0x00, -0x06,0x01,0x00,0x00,0x07,0x01,0x00,0x00,0xfb,0x00,0x00,0x00, -0x34,0x00,0x00,0x00,0x05,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, -0xb9,0x00,0x00,0x00,0x08,0x01,0x00,0x00,0x07,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0x09,0x01,0x00,0x00,0x0a,0x01,0x00,0x00, -0xf0,0x00,0x00,0x00,0xf7,0x00,0x00,0x00,0x3e,0x00,0x03,0x00, -0x0a,0x01,0x00,0x00,0x08,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0xeb,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x0b,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x0e,0x01,0x00,0x00, -0x73,0x00,0x00,0x00,0xa5,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x10,0x01,0x00,0x00,0x0e,0x01,0x00,0x00, -0x0f,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x12,0x01,0x00,0x00,0x10,0x01,0x00,0x00,0x6e,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x09,0x01,0x00,0x00,0x13,0x01,0x00,0x00, -0xf0,0x00,0x00,0x00,0x12,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0x13,0x01,0x00,0x00,0xc1,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xeb,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xeb,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xd5,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xd5,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x1a,0x01,0x00,0x00,0xa5,0x02,0x00,0x00,0x18,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0xd2,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xd4,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x1c,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x1c,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xa6,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, -0xd4,0x00,0x00,0x00,0x5d,0x01,0x00,0x00,0x1f,0x01,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00,0x22,0x01,0x00,0x00, -0xa6,0x02,0x00,0x00,0x9c,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0x1e,0x01,0x00,0x00,0x1f,0x01,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x22,0x01,0x00,0x00,0x1d,0x01,0x00,0x00, -0x1e,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x1d,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x26,0x01,0x00,0x00, -0x9d,0x00,0x00,0x00,0x73,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x28,0x01,0x00,0x00,0x26,0x01,0x00,0x00, -0xa6,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0x29,0x01,0x00,0x00,0x12,0x00,0x00,0x00,0xc5,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x2a,0x01,0x00,0x00, -0x29,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00, -0x2b,0x01,0x00,0x00,0x28,0x01,0x00,0x00,0x2a,0x01,0x00,0x00, -0xf7,0x00,0x03,0x00,0x2d,0x01,0x00,0x00,0x00,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x2b,0x01,0x00,0x00,0x2c,0x01,0x00,0x00, -0x2d,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x2c,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x30,0x01,0x00,0x00, -0x95,0x02,0x00,0x00,0x6e,0x00,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb7,0x00,0x00,0x00,0x32,0x01,0x00,0x00,0x30,0x01,0x00,0x00, -0x83,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x2d,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x2d,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, -0xb7,0x00,0x00,0x00,0x33,0x01,0x00,0x00,0x2b,0x01,0x00,0x00, -0x1d,0x01,0x00,0x00,0x32,0x01,0x00,0x00,0x2c,0x01,0x00,0x00, -0xf7,0x00,0x03,0x00,0x35,0x01,0x00,0x00,0x00,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x33,0x01,0x00,0x00,0x34,0x01,0x00,0x00, -0x53,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x34,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3d,0x01,0x00,0x00, -0x73,0x00,0x00,0x00,0xa6,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x3f,0x01,0x00,0x00,0x3d,0x01,0x00,0x00, -0x3e,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x41,0x01,0x00,0x00,0x3f,0x01,0x00,0x00,0x6e,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x4c,0x01,0x00,0x00, -0x3d,0x01,0x00,0x00,0xa0,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x4d,0x01,0x00,0x00,0xad,0x02,0x00,0x00, -0x4c,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x4f,0x01,0x00,0x00,0x4d,0x01,0x00,0x00,0x6e,0x00,0x00,0x00, -0x41,0x00,0x06,0x00,0x06,0x01,0x00,0x00,0x50,0x01,0x00,0x00, -0x45,0x01,0x00,0x00,0x34,0x00,0x00,0x00,0x4f,0x01,0x00,0x00, -0x3d,0x00,0x04,0x00,0xb9,0x00,0x00,0x00,0x51,0x01,0x00,0x00, -0x50,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0x09,0x01,0x00,0x00, -0x52,0x01,0x00,0x00,0x3a,0x01,0x00,0x00,0x41,0x01,0x00,0x00, -0x3e,0x00,0x03,0x00,0x52,0x01,0x00,0x00,0x51,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0x35,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x53,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x56,0x01,0x00,0x00,0x73,0x00,0x00,0x00,0xa6,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x58,0x01,0x00,0x00, -0x56,0x01,0x00,0x00,0x57,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x5a,0x01,0x00,0x00,0x58,0x01,0x00,0x00, -0x6e,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x09,0x01,0x00,0x00, -0x5b,0x01,0x00,0x00,0x3a,0x01,0x00,0x00,0x5a,0x01,0x00,0x00, -0x3e,0x00,0x03,0x00,0x5b,0x01,0x00,0x00,0xc1,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x35,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x35,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x1f,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x1f,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x5d,0x01,0x00,0x00,0xa6,0x02,0x00,0x00, -0x18,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x1c,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x1e,0x01,0x00,0x00,0xe0,0x00,0x04,0x00, -0x0c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x5e,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x61,0x01,0x00,0x00, -0xa9,0x02,0x00,0x00,0x5f,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x64,0x01,0x00,0x00,0xad,0x02,0x00,0x00, -0x62,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x66,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x66,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xaf,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, -0x1e,0x01,0x00,0x00,0x0d,0x02,0x00,0x00,0x69,0x01,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00,0x6c,0x01,0x00,0x00, -0xaf,0x02,0x00,0x00,0x6c,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0x68,0x01,0x00,0x00,0x69,0x01,0x00,0x00,0x00,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x6c,0x01,0x00,0x00,0x67,0x01,0x00,0x00, -0x68,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x67,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0x6e,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x6e,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xb3,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0x67,0x01,0x00,0x00, -0x99,0x01,0x00,0x00,0x71,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb7,0x00,0x00,0x00,0x74,0x01,0x00,0x00,0xb3,0x02,0x00,0x00, -0x60,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x70,0x01,0x00,0x00, -0x71,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x74,0x01,0x00,0x00,0x6f,0x01,0x00,0x00,0x70,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x6f,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0x76,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x76,0x01,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xc5,0x02,0x00,0x00, -0x3e,0x00,0x00,0x00,0x6f,0x01,0x00,0x00,0x97,0x01,0x00,0x00, -0x77,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00, -0x7c,0x01,0x00,0x00,0xc5,0x02,0x00,0x00,0x62,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0x78,0x01,0x00,0x00,0x77,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x7c,0x01,0x00,0x00, -0x77,0x01,0x00,0x00,0x78,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x77,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x82,0x01,0x00,0x00,0xb3,0x02,0x00,0x00,0x62,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x84,0x01,0x00,0x00, -0x82,0x01,0x00,0x00,0xc5,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x86,0x01,0x00,0x00,0x55,0x00,0x00,0x00, -0x53,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x88,0x01,0x00,0x00,0xb3,0x02,0x00,0x00,0x61,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x89,0x01,0x00,0x00, -0x86,0x01,0x00,0x00,0x88,0x01,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x8b,0x01,0x00,0x00,0x64,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x8c,0x01,0x00,0x00,0x89,0x01,0x00,0x00,0x8b,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8e,0x01,0x00,0x00, -0x8c,0x01,0x00,0x00,0xc5,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x90,0x01,0x00,0x00,0x8e,0x01,0x00,0x00, -0x8f,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x92,0x01,0x00,0x00,0x90,0x01,0x00,0x00,0xaf,0x02,0x00,0x00, -0x41,0x00,0x05,0x00,0x09,0x01,0x00,0x00,0x93,0x01,0x00,0x00, -0xf0,0x00,0x00,0x00,0x92,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, -0xb9,0x00,0x00,0x00,0x94,0x01,0x00,0x00,0x93,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0x95,0x01,0x00,0x00, -0x80,0x01,0x00,0x00,0x84,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0x95,0x01,0x00,0x00,0x94,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x97,0x01,0x00,0x00,0xc5,0x02,0x00,0x00, -0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x76,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x78,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0x71,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x71,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x99,0x01,0x00,0x00, -0xb3,0x02,0x00,0x00,0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x6e,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x70,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0x9b,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x9b,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xb4,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0x70,0x01,0x00,0x00, -0xc7,0x01,0x00,0x00,0x9e,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb7,0x00,0x00,0x00,0xa1,0x01,0x00,0x00,0xb4,0x02,0x00,0x00, -0xb4,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x9d,0x01,0x00,0x00, -0x9e,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0xa1,0x01,0x00,0x00,0x9c,0x01,0x00,0x00,0x9d,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x9c,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0xa3,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xa3,0x01,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xc2,0x02,0x00,0x00, -0x3e,0x00,0x00,0x00,0x9c,0x01,0x00,0x00,0xc5,0x01,0x00,0x00, -0xa4,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00, -0xa9,0x01,0x00,0x00,0xc2,0x02,0x00,0x00,0xb1,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0xa5,0x01,0x00,0x00,0xa4,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xa9,0x01,0x00,0x00, -0xa4,0x01,0x00,0x00,0xa5,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xa4,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xaf,0x01,0x00,0x00,0xb4,0x02,0x00,0x00,0xb1,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xb1,0x01,0x00,0x00, -0xaf,0x01,0x00,0x00,0xc2,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xb3,0x01,0x00,0x00,0x59,0x00,0x00,0x00, -0xae,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xb6,0x01,0x00,0x00,0xb4,0x02,0x00,0x00,0xb5,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xb7,0x01,0x00,0x00, -0xb3,0x01,0x00,0x00,0xb6,0x01,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xb9,0x01,0x00,0x00,0x68,0x00,0x00,0x00, -0xb1,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xba,0x01,0x00,0x00,0xb7,0x01,0x00,0x00,0xb9,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xbc,0x01,0x00,0x00, -0xba,0x01,0x00,0x00,0xc2,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xbe,0x01,0x00,0x00,0xbc,0x01,0x00,0x00, -0xbd,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xc0,0x01,0x00,0x00,0xbe,0x01,0x00,0x00,0xaf,0x02,0x00,0x00, -0x41,0x00,0x05,0x00,0x09,0x01,0x00,0x00,0xc1,0x01,0x00,0x00, -0x3a,0x01,0x00,0x00,0xc0,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, -0xb9,0x00,0x00,0x00,0xc2,0x01,0x00,0x00,0xc1,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0xc3,0x01,0x00,0x00, -0xad,0x01,0x00,0x00,0xb1,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0xc3,0x01,0x00,0x00,0xc2,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xc5,0x01,0x00,0x00,0xc2,0x02,0x00,0x00, -0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xa3,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xa5,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0x9e,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x9e,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xc7,0x01,0x00,0x00, -0xb4,0x02,0x00,0x00,0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x9b,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x9d,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0xc9,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xc9,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xb5,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0x9d,0x01,0x00,0x00, -0x0b,0x02,0x00,0x00,0xcc,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb7,0x00,0x00,0x00,0xcf,0x01,0x00,0x00,0xb5,0x02,0x00,0x00, -0xb4,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xcb,0x01,0x00,0x00, -0xcc,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0xcf,0x01,0x00,0x00,0xca,0x01,0x00,0x00,0xcb,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xca,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0xd1,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xd1,0x01,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xb9,0x02,0x00,0x00, -0x3e,0x00,0x00,0x00,0xca,0x01,0x00,0x00,0x09,0x02,0x00,0x00, -0xd4,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00, -0xd7,0x01,0x00,0x00,0xb9,0x02,0x00,0x00,0x60,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0xd3,0x01,0x00,0x00,0xd4,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xd7,0x01,0x00,0x00, -0xd2,0x01,0x00,0x00,0xd3,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xd2,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xd9,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xd9,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xbb,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, -0xd2,0x01,0x00,0x00,0x07,0x02,0x00,0x00,0xdc,0x01,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00,0xdf,0x01,0x00,0x00, -0xbb,0x02,0x00,0x00,0xb1,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0xdb,0x01,0x00,0x00,0xdc,0x01,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xdf,0x01,0x00,0x00,0xda,0x01,0x00,0x00, -0xdb,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xda,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0xe1,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xe1,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xbd,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0xda,0x01,0x00,0x00, -0x05,0x02,0x00,0x00,0xe2,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb7,0x00,0x00,0x00,0xe7,0x01,0x00,0x00,0xbd,0x02,0x00,0x00, -0x62,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xe3,0x01,0x00,0x00, -0xe2,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0xe7,0x01,0x00,0x00,0xe2,0x01,0x00,0x00,0xe3,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xe2,0x01,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xe9,0x01,0x00,0x00,0xb5,0x02,0x00,0x00, -0xb1,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xeb,0x01,0x00,0x00,0xe9,0x01,0x00,0x00,0xbb,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xed,0x01,0x00,0x00, -0xeb,0x01,0x00,0x00,0xec,0x01,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xef,0x01,0x00,0x00,0xb9,0x02,0x00,0x00, -0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xf0,0x01,0x00,0x00,0xed,0x01,0x00,0x00,0xef,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xf2,0x01,0x00,0x00, -0xf0,0x01,0x00,0x00,0xbd,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xf6,0x01,0x00,0x00,0xef,0x01,0x00,0x00, -0xbd,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, -0xf7,0x01,0x00,0x00,0x80,0x01,0x00,0x00,0xf6,0x01,0x00,0x00, -0x3d,0x00,0x04,0x00,0xb9,0x00,0x00,0x00,0xf8,0x01,0x00,0x00, -0xf7,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, -0xfd,0x01,0x00,0x00,0xad,0x01,0x00,0x00,0xeb,0x01,0x00,0x00, -0x3d,0x00,0x04,0x00,0xb9,0x00,0x00,0x00,0xfe,0x01,0x00,0x00, -0xfd,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, -0x00,0x02,0x00,0x00,0xbf,0x00,0x00,0x00,0xf2,0x01,0x00,0x00, -0x3d,0x00,0x04,0x00,0xb9,0x00,0x00,0x00,0x01,0x02,0x00,0x00, -0x00,0x02,0x00,0x00,0x0c,0x00,0x08,0x00,0xb9,0x00,0x00,0x00, -0x02,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00, -0xf8,0x01,0x00,0x00,0xfe,0x01,0x00,0x00,0x01,0x02,0x00,0x00, -0x3e,0x00,0x03,0x00,0x00,0x02,0x00,0x00,0x02,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x05,0x02,0x00,0x00, -0xbd,0x02,0x00,0x00,0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xe1,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xe3,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0xdc,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xdc,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x07,0x02,0x00,0x00,0xbb,0x02,0x00,0x00,0xc5,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xd9,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xdb,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xd4,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xd4,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x09,0x02,0x00,0x00,0xb9,0x02,0x00,0x00, -0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xd1,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xd3,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0xcc,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xcc,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x0b,0x02,0x00,0x00, -0xb5,0x02,0x00,0x00,0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xc9,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xcb,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0x69,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x69,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x0d,0x02,0x00,0x00,0xaf,0x02,0x00,0x00,0xc5,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x66,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x68,0x01,0x00,0x00,0xe0,0x00,0x04,0x00,0x0c,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x5e,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0xcc,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xcc,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x0f,0x02,0x00,0x00, -0x95,0x02,0x00,0x00,0x6c,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xc9,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xcb,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x14,0x02,0x00,0x00, -0x55,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x15,0x02,0x00,0x00,0x8b,0x00,0x00,0x00, -0x14,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x1a,0x02,0x00,0x00,0x59,0x00,0x00,0x00,0xae,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x1b,0x02,0x00,0x00, -0x9d,0x00,0x00,0x00,0x1a,0x02,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x00,0x00,0x00,0x20,0x02,0x00,0x00,0x12,0x00,0x00,0x00, -0x1f,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x21,0x02,0x00,0x00,0x20,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x22,0x02,0x00,0x00,0x0f,0x00,0x00,0x00, -0x21,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x26,0x02,0x00,0x00,0x47,0x00,0x00,0x00,0x21,0x02,0x00,0x00, -0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00,0x28,0x02,0x00,0x00, -0x27,0x02,0x00,0x00,0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x29,0x02,0x00,0x00,0x28,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x2a,0x02,0x00,0x00, -0x26,0x02,0x00,0x00,0x29,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x2b,0x02,0x00,0x00,0x22,0x02,0x00,0x00, -0x2a,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x2d,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x2d,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0x96,0x02,0x00,0x00,0x3e,0x00,0x00,0x00, -0xcb,0x00,0x00,0x00,0x93,0x02,0x00,0x00,0x30,0x02,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00,0x33,0x02,0x00,0x00, -0x96,0x02,0x00,0x00,0xb4,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0x2f,0x02,0x00,0x00,0x30,0x02,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x33,0x02,0x00,0x00,0x2e,0x02,0x00,0x00, -0x2f,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x2e,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0x35,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x35,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0x97,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0x2e,0x02,0x00,0x00, -0x91,0x02,0x00,0x00,0x38,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb7,0x00,0x00,0x00,0x3b,0x02,0x00,0x00,0x97,0x02,0x00,0x00, -0x60,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x37,0x02,0x00,0x00, -0x38,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x3b,0x02,0x00,0x00,0x36,0x02,0x00,0x00,0x37,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x36,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x3f,0x02,0x00,0x00,0x97,0x02,0x00,0x00, -0x61,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x40,0x02,0x00,0x00,0x15,0x02,0x00,0x00,0x3f,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x42,0x02,0x00,0x00, -0x64,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x43,0x02,0x00,0x00,0x40,0x02,0x00,0x00, -0x42,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x47,0x02,0x00,0x00,0x96,0x02,0x00,0x00,0xb5,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x48,0x02,0x00,0x00, -0x1b,0x02,0x00,0x00,0x47,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x4a,0x02,0x00,0x00,0x68,0x00,0x00,0x00, -0xb1,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x4b,0x02,0x00,0x00,0x48,0x02,0x00,0x00,0x4a,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0x4d,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x4d,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0x99,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0x36,0x02,0x00,0x00, -0x8f,0x02,0x00,0x00,0x50,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, -0xb7,0x00,0x00,0x00,0x53,0x02,0x00,0x00,0x99,0x02,0x00,0x00, -0xb1,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x4f,0x02,0x00,0x00, -0x50,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x53,0x02,0x00,0x00,0x4e,0x02,0x00,0x00,0x4f,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x4e,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x55,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x55,0x02,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x9b,0x02,0x00,0x00, -0x3e,0x00,0x00,0x00,0x4e,0x02,0x00,0x00,0x8d,0x02,0x00,0x00, -0x58,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00, -0x5b,0x02,0x00,0x00,0x9b,0x02,0x00,0x00,0x62,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0x57,0x02,0x00,0x00,0x58,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x5b,0x02,0x00,0x00, -0x56,0x02,0x00,0x00,0x57,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x56,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x5e,0x02,0x00,0x00,0x43,0x02,0x00,0x00,0x9b,0x02,0x00,0x00, -0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00,0x61,0x02,0x00,0x00, -0x5e,0x02,0x00,0x00,0x36,0x00,0x00,0x00,0xf7,0x00,0x03,0x00, -0x63,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x61,0x02,0x00,0x00,0x62,0x02,0x00,0x00,0x63,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x62,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x66,0x02,0x00,0x00,0x4b,0x02,0x00,0x00, -0x99,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0x67,0x02,0x00,0x00,0x12,0x00,0x00,0x00,0xc5,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x68,0x02,0x00,0x00, -0x67,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xb7,0x00,0x00,0x00, -0x69,0x02,0x00,0x00,0x66,0x02,0x00,0x00,0x68,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0x63,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x63,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0xb7,0x00,0x00,0x00, -0x6a,0x02,0x00,0x00,0x61,0x02,0x00,0x00,0x56,0x02,0x00,0x00, -0x69,0x02,0x00,0x00,0x62,0x02,0x00,0x00,0xf7,0x00,0x03,0x00, -0x6c,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x6a,0x02,0x00,0x00,0x6b,0x02,0x00,0x00,0x6c,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x6b,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x74,0x02,0x00,0x00,0x4b,0x02,0x00,0x00, -0x99,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, -0x76,0x02,0x00,0x00,0x12,0x00,0x00,0x00,0x75,0x02,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x77,0x02,0x00,0x00, -0x76,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x78,0x02,0x00,0x00,0x74,0x02,0x00,0x00,0x77,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x79,0x02,0x00,0x00, -0x2b,0x02,0x00,0x00,0x78,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x7b,0x02,0x00,0x00,0x79,0x02,0x00,0x00, -0x43,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x7d,0x02,0x00,0x00,0x7b,0x02,0x00,0x00,0x9b,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x7f,0x02,0x00,0x00, -0x96,0x02,0x00,0x00,0xb1,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x81,0x02,0x00,0x00,0x7f,0x02,0x00,0x00, -0x99,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x83,0x02,0x00,0x00,0x81,0x02,0x00,0x00,0x82,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x85,0x02,0x00,0x00, -0x97,0x02,0x00,0x00,0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x86,0x02,0x00,0x00,0x83,0x02,0x00,0x00, -0x85,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x88,0x02,0x00,0x00,0x86,0x02,0x00,0x00,0x9b,0x02,0x00,0x00, -0x41,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0x89,0x02,0x00,0x00, -0xbf,0x00,0x00,0x00,0x88,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, -0xb9,0x00,0x00,0x00,0x8a,0x02,0x00,0x00,0x89,0x02,0x00,0x00, -0x41,0x00,0x06,0x00,0x06,0x01,0x00,0x00,0x8b,0x02,0x00,0x00, -0x70,0x02,0x00,0x00,0x34,0x00,0x00,0x00,0x7d,0x02,0x00,0x00, -0x3e,0x00,0x03,0x00,0x8b,0x02,0x00,0x00,0x8a,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0x6c,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x6c,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x58,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x58,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x8d,0x02,0x00,0x00,0x9b,0x02,0x00,0x00, -0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x55,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x57,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x50,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x50,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8f,0x02,0x00,0x00, -0x99,0x02,0x00,0x00,0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x4d,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x4f,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0x38,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x38,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x91,0x02,0x00,0x00,0x97,0x02,0x00,0x00,0xc5,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x35,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x37,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x30,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x30,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x93,0x02,0x00,0x00,0x96,0x02,0x00,0x00, -0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x2d,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x2f,0x02,0x00,0x00,0xfd,0x00,0x01,0x00, -0x38,0x00,0x01,0x00, -}; -const uint64_t matmul_f32_s_fp32_len = 10048; +const uint64_t matmul_q8_0_f32_fp32_len = 10488; unsigned char mul_f32_data[] = { 0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, -0x3e,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, +0x67,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, 0x01,0x00,0x00,0x00,0x0b,0x00,0x06,0x00,0x01,0x00,0x00,0x00, 0x47,0x4c,0x53,0x4c,0x2e,0x73,0x74,0x64,0x2e,0x34,0x35,0x30, 0x00,0x00,0x00,0x00,0x0e,0x00,0x03,0x00,0x00,0x00,0x00,0x00, 0x01,0x00,0x00,0x00,0x0f,0x00,0x0a,0x00,0x05,0x00,0x00,0x00, 0x04,0x00,0x00,0x00,0x6d,0x61,0x69,0x6e,0x00,0x00,0x00,0x00, -0x0b,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x22,0x00,0x00,0x00, -0x27,0x00,0x00,0x00,0x2f,0x00,0x00,0x00,0x10,0x00,0x06,0x00, +0x16,0x00,0x00,0x00,0x2e,0x01,0x00,0x00,0x3f,0x01,0x00,0x00, +0x4b,0x01,0x00,0x00,0x56,0x01,0x00,0x00,0x10,0x00,0x06,0x00, 0x04,0x00,0x00,0x00,0x11,0x00,0x00,0x00,0x00,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x0b,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x1c,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x12,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x12,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x12,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x08,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x12,0x00,0x00,0x00,0x03,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x47,0x00,0x03,0x00, -0x12,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x1f,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x48,0x00,0x04,0x00,0x20,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x19,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x20,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x14,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x14,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x14,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x14,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x14,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x10,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x14,0x00,0x00,0x00,0x05,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x14,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x14,0x00,0x00,0x00, +0x07,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x1c,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x14,0x00,0x00,0x00,0x08,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x14,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x24,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x14,0x00,0x00,0x00, +0x0a,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x28,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x14,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x2c,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x14,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x30,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x14,0x00,0x00,0x00, +0x0d,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x34,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x14,0x00,0x00,0x00,0x0e,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x38,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x14,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x3c,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x14,0x00,0x00,0x00, +0x10,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x40,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x14,0x00,0x00,0x00,0x11,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x44,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x14,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x48,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x14,0x00,0x00,0x00, +0x13,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x4c,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x14,0x00,0x00,0x00,0x14,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x50,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x14,0x00,0x00,0x00,0x15,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x54,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x14,0x00,0x00,0x00, +0x16,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x58,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x14,0x00,0x00,0x00,0x17,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x5c,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x14,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x60,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x14,0x00,0x00,0x00, +0x19,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x64,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x14,0x00,0x00,0x00,0x1a,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x68,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x14,0x00,0x00,0x00,0x1b,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x14,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x2e,0x01,0x00,0x00, +0x0b,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x3c,0x01,0x00,0x00,0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x48,0x00,0x04,0x00,0x3d,0x01,0x00,0x00,0x00,0x00,0x00,0x00, +0x19,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x3d,0x01,0x00,0x00, 0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x03,0x00,0x20,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x22,0x00,0x00,0x00,0x22,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x22,0x00,0x00,0x00, +0x47,0x00,0x03,0x00,0x3d,0x01,0x00,0x00,0x02,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x3f,0x01,0x00,0x00,0x22,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x3f,0x01,0x00,0x00, 0x21,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x24,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x48,0x00,0x04,0x00,0x25,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x25,0x00,0x00,0x00, +0x48,0x01,0x00,0x00,0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x48,0x00,0x04,0x00,0x49,0x01,0x00,0x00,0x00,0x00,0x00,0x00, +0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x49,0x01,0x00,0x00, 0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x03,0x00,0x25,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x27,0x00,0x00,0x00,0x22,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x27,0x00,0x00,0x00, +0x47,0x00,0x03,0x00,0x49,0x01,0x00,0x00,0x02,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x4b,0x01,0x00,0x00,0x22,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x4b,0x01,0x00,0x00, 0x21,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x2c,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x48,0x00,0x04,0x00,0x2d,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x2d,0x00,0x00,0x00, +0x53,0x01,0x00,0x00,0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x48,0x00,0x04,0x00,0x54,0x01,0x00,0x00,0x00,0x00,0x00,0x00, +0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x54,0x01,0x00,0x00, 0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x03,0x00,0x2d,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x2f,0x00,0x00,0x00,0x22,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x2f,0x00,0x00,0x00, +0x47,0x00,0x03,0x00,0x54,0x01,0x00,0x00,0x02,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x56,0x01,0x00,0x00,0x22,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x56,0x01,0x00,0x00, 0x21,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x3b,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x19,0x00,0x00,0x00, +0x61,0x01,0x00,0x00,0x0b,0x00,0x00,0x00,0x19,0x00,0x00,0x00, 0x13,0x00,0x02,0x00,0x02,0x00,0x00,0x00,0x21,0x00,0x03,0x00, 0x03,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x15,0x00,0x04,0x00, 0x06,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x17,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x03,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x0a,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x0a,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x0d,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x16,0x00,0x03,0x00, -0x11,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x1e,0x00,0x06,0x00, -0x12,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x11,0x00,0x00,0x00,0x11,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x14,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x15,0x00,0x04,0x00,0x15,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x15,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x17,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x14,0x00,0x02,0x00,0x1a,0x00,0x00,0x00, -0x1d,0x00,0x03,0x00,0x1f,0x00,0x00,0x00,0x11,0x00,0x00,0x00, -0x1e,0x00,0x03,0x00,0x20,0x00,0x00,0x00,0x1f,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x21,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x21,0x00,0x00,0x00, -0x22,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, -0x24,0x00,0x00,0x00,0x11,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, -0x25,0x00,0x00,0x00,0x24,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x26,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x25,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x26,0x00,0x00,0x00,0x27,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x29,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x11,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, -0x2c,0x00,0x00,0x00,0x11,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, -0x2d,0x00,0x00,0x00,0x2c,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x2e,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x2d,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x2e,0x00,0x00,0x00,0x2f,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x15,0x00,0x00,0x00, -0x31,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x00,0x02,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x2c,0x00,0x06,0x00,0x09,0x00,0x00,0x00, -0x3b,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, -0x3a,0x00,0x00,0x00,0x36,0x00,0x05,0x00,0x02,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x03,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0x05,0x00,0x00,0x00,0xf7,0x00,0x03,0x00, -0x3c,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xfb,0x00,0x03,0x00, -0x0c,0x00,0x00,0x00,0x3d,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x3d,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00, -0x0e,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, -0x0e,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x19,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0xae,0x00,0x05,0x00,0x1a,0x00,0x00,0x00, -0x1b,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x19,0x00,0x00,0x00, -0xf7,0x00,0x03,0x00,0x1d,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x1b,0x00,0x00,0x00,0x1c,0x00,0x00,0x00, -0x1d,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x1c,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x3c,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x1d,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x29,0x00,0x00,0x00, -0x2a,0x00,0x00,0x00,0x27,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0x0f,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x11,0x00,0x00,0x00, -0x2b,0x00,0x00,0x00,0x2a,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x17,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x14,0x00,0x00,0x00, -0x31,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x33,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x89,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, -0x33,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x29,0x00,0x00,0x00, -0x35,0x00,0x00,0x00,0x2f,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0x34,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x11,0x00,0x00,0x00, -0x36,0x00,0x00,0x00,0x35,0x00,0x00,0x00,0x85,0x00,0x05,0x00, -0x11,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x2b,0x00,0x00,0x00, -0x36,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x29,0x00,0x00,0x00, -0x38,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0x0f,0x00,0x00,0x00,0x3e,0x00,0x03,0x00,0x38,0x00,0x00,0x00, -0x37,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x3c,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0x3c,0x00,0x00,0x00,0xfd,0x00,0x01,0x00, +0x1e,0x00,0x1e,0x00,0x14,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x15,0x00,0x00,0x00,0x09,0x00,0x00,0x00, +0x14,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x15,0x00,0x00,0x00, +0x16,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x15,0x00,0x04,0x00, +0x17,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x17,0x00,0x00,0x00,0x18,0x00,0x00,0x00, +0x03,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x19,0x00,0x00,0x00, +0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x17,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x17,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x17,0x00,0x00,0x00, +0x57,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x17,0x00,0x00,0x00,0x5c,0x00,0x00,0x00,0x07,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x17,0x00,0x00,0x00,0x62,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x17,0x00,0x00,0x00, +0x68,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x17,0x00,0x00,0x00,0xac,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x17,0x00,0x00,0x00,0xb0,0x00,0x00,0x00, +0x10,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x17,0x00,0x00,0x00, +0xb5,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x17,0x00,0x00,0x00,0xb9,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x17,0x00,0x00,0x00,0xbf,0x00,0x00,0x00, +0x0a,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x17,0x00,0x00,0x00, +0xc3,0x00,0x00,0x00,0x0e,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x17,0x00,0x00,0x00,0xc9,0x00,0x00,0x00,0x09,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x17,0x00,0x00,0x00,0xcd,0x00,0x00,0x00, +0x0d,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x17,0x00,0x00,0x00, +0xd6,0x00,0x00,0x00,0x13,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x17,0x00,0x00,0x00,0xd9,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x17,0x00,0x00,0x00,0xdd,0x00,0x00,0x00, +0x11,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x17,0x00,0x00,0x00, +0x14,0x01,0x00,0x00,0x18,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x17,0x00,0x00,0x00,0x19,0x01,0x00,0x00,0x17,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x17,0x00,0x00,0x00,0x1f,0x01,0x00,0x00, +0x16,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x17,0x00,0x00,0x00, +0x25,0x01,0x00,0x00,0x15,0x00,0x00,0x00,0x17,0x00,0x04,0x00, +0x2c,0x01,0x00,0x00,0x06,0x00,0x00,0x00,0x03,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x2d,0x01,0x00,0x00,0x01,0x00,0x00,0x00, +0x2c,0x01,0x00,0x00,0x3b,0x00,0x04,0x00,0x2d,0x01,0x00,0x00, +0x2e,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x2f,0x01,0x00,0x00,0x00,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x30,0x01,0x00,0x00,0x01,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x17,0x00,0x00,0x00, +0x33,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x14,0x00,0x02,0x00, +0x36,0x01,0x00,0x00,0x16,0x00,0x03,0x00,0x3b,0x01,0x00,0x00, +0x20,0x00,0x00,0x00,0x1d,0x00,0x03,0x00,0x3c,0x01,0x00,0x00, +0x3b,0x01,0x00,0x00,0x1e,0x00,0x03,0x00,0x3d,0x01,0x00,0x00, +0x3c,0x01,0x00,0x00,0x20,0x00,0x04,0x00,0x3e,0x01,0x00,0x00, +0x0c,0x00,0x00,0x00,0x3d,0x01,0x00,0x00,0x3b,0x00,0x04,0x00, +0x3e,0x01,0x00,0x00,0x3f,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x17,0x00,0x00,0x00,0x40,0x01,0x00,0x00, +0x19,0x00,0x00,0x00,0x1d,0x00,0x03,0x00,0x48,0x01,0x00,0x00, +0x3b,0x01,0x00,0x00,0x1e,0x00,0x03,0x00,0x49,0x01,0x00,0x00, +0x48,0x01,0x00,0x00,0x20,0x00,0x04,0x00,0x4a,0x01,0x00,0x00, +0x0c,0x00,0x00,0x00,0x49,0x01,0x00,0x00,0x3b,0x00,0x04,0x00, +0x4a,0x01,0x00,0x00,0x4b,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x50,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, +0x3b,0x01,0x00,0x00,0x1d,0x00,0x03,0x00,0x53,0x01,0x00,0x00, +0x3b,0x01,0x00,0x00,0x1e,0x00,0x03,0x00,0x54,0x01,0x00,0x00, +0x53,0x01,0x00,0x00,0x20,0x00,0x04,0x00,0x55,0x01,0x00,0x00, +0x0c,0x00,0x00,0x00,0x54,0x01,0x00,0x00,0x3b,0x00,0x04,0x00, +0x55,0x01,0x00,0x00,0x56,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x5f,0x01,0x00,0x00, +0x00,0x02,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x60,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x2c,0x00,0x06,0x00, +0x2c,0x01,0x00,0x00,0x61,0x01,0x00,0x00,0x5f,0x01,0x00,0x00, +0x60,0x01,0x00,0x00,0x60,0x01,0x00,0x00,0x36,0x00,0x05,0x00, +0x02,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x03,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x05,0x00,0x00,0x00, +0xf7,0x00,0x03,0x00,0x62,0x01,0x00,0x00,0x00,0x00,0x00,0x00, +0xfb,0x00,0x03,0x00,0x2f,0x01,0x00,0x00,0x63,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x63,0x01,0x00,0x00,0x41,0x00,0x05,0x00, +0x30,0x01,0x00,0x00,0x31,0x01,0x00,0x00,0x2e,0x01,0x00,0x00, +0x2f,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x32,0x01,0x00,0x00,0x31,0x01,0x00,0x00,0x41,0x00,0x05,0x00, +0x19,0x00,0x00,0x00,0x34,0x01,0x00,0x00,0x16,0x00,0x00,0x00, +0x33,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x35,0x01,0x00,0x00,0x34,0x01,0x00,0x00,0xae,0x00,0x05,0x00, +0x36,0x01,0x00,0x00,0x37,0x01,0x00,0x00,0x32,0x01,0x00,0x00, +0x35,0x01,0x00,0x00,0xf7,0x00,0x03,0x00,0x39,0x01,0x00,0x00, +0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x37,0x01,0x00,0x00, +0x38,0x01,0x00,0x00,0x39,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x38,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x62,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x39,0x01,0x00,0x00,0x41,0x00,0x05,0x00, +0x19,0x00,0x00,0x00,0x41,0x01,0x00,0x00,0x16,0x00,0x00,0x00, +0x40,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x42,0x01,0x00,0x00,0x41,0x01,0x00,0x00,0x41,0x00,0x05,0x00, +0x19,0x00,0x00,0x00,0x71,0x01,0x00,0x00,0x16,0x00,0x00,0x00, +0xd6,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x72,0x01,0x00,0x00,0x71,0x01,0x00,0x00,0x41,0x00,0x05,0x00, +0x19,0x00,0x00,0x00,0x73,0x01,0x00,0x00,0x16,0x00,0x00,0x00, +0xd9,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x74,0x01,0x00,0x00,0x73,0x01,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x75,0x01,0x00,0x00,0x72,0x01,0x00,0x00, +0x74,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0x19,0x00,0x00,0x00, +0x76,0x01,0x00,0x00,0x16,0x00,0x00,0x00,0xdd,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x77,0x01,0x00,0x00, +0x76,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x78,0x01,0x00,0x00,0x75,0x01,0x00,0x00,0x77,0x01,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x79,0x01,0x00,0x00, +0x32,0x01,0x00,0x00,0x78,0x01,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x7d,0x01,0x00,0x00,0x79,0x01,0x00,0x00, +0x72,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x80,0x01,0x00,0x00,0x7d,0x01,0x00,0x00,0x74,0x01,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x83,0x01,0x00,0x00, +0x80,0x01,0x00,0x00,0x77,0x01,0x00,0x00,0x82,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x86,0x01,0x00,0x00,0x32,0x01,0x00,0x00, +0x83,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x8b,0x01,0x00,0x00,0x74,0x01,0x00,0x00,0x77,0x01,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8c,0x01,0x00,0x00, +0x86,0x01,0x00,0x00,0x8b,0x01,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x90,0x01,0x00,0x00,0x8c,0x01,0x00,0x00, +0x74,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x93,0x01,0x00,0x00,0x90,0x01,0x00,0x00,0x77,0x01,0x00,0x00, +0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x98,0x01,0x00,0x00, +0x86,0x01,0x00,0x00,0x93,0x01,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x9b,0x01,0x00,0x00,0x98,0x01,0x00,0x00, +0x77,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xa4,0x01,0x00,0x00,0x9b,0x01,0x00,0x00,0x77,0x01,0x00,0x00, +0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa5,0x01,0x00,0x00, +0x98,0x01,0x00,0x00,0xa4,0x01,0x00,0x00,0x41,0x00,0x05,0x00, +0x19,0x00,0x00,0x00,0xa7,0x01,0x00,0x00,0x16,0x00,0x00,0x00, +0x14,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xa8,0x01,0x00,0x00,0xa7,0x01,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xa9,0x01,0x00,0x00,0x79,0x01,0x00,0x00, +0xa8,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0x19,0x00,0x00,0x00, +0xab,0x01,0x00,0x00,0x16,0x00,0x00,0x00,0x19,0x01,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xac,0x01,0x00,0x00, +0xab,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xad,0x01,0x00,0x00,0x8c,0x01,0x00,0x00,0xac,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xae,0x01,0x00,0x00, +0xa9,0x01,0x00,0x00,0xad,0x01,0x00,0x00,0x41,0x00,0x05,0x00, +0x19,0x00,0x00,0x00,0xb0,0x01,0x00,0x00,0x16,0x00,0x00,0x00, +0x1f,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xb1,0x01,0x00,0x00,0xb0,0x01,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xb2,0x01,0x00,0x00,0x9b,0x01,0x00,0x00, +0xb1,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xb3,0x01,0x00,0x00,0xae,0x01,0x00,0x00,0xb2,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0x19,0x00,0x00,0x00,0xb5,0x01,0x00,0x00, +0x16,0x00,0x00,0x00,0x25,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xb6,0x01,0x00,0x00,0xb5,0x01,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xb7,0x01,0x00,0x00, +0xa5,0x01,0x00,0x00,0xb6,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xb8,0x01,0x00,0x00,0xb3,0x01,0x00,0x00, +0xb7,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x47,0x01,0x00,0x00,0x42,0x01,0x00,0x00,0xb8,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0x19,0x00,0x00,0x00,0xc2,0x01,0x00,0x00, +0x16,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xc3,0x01,0x00,0x00,0xc2,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0x19,0x00,0x00,0x00,0xc4,0x01,0x00,0x00, +0x16,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xc5,0x01,0x00,0x00,0xc4,0x01,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xc6,0x01,0x00,0x00, +0xc3,0x01,0x00,0x00,0xc5,0x01,0x00,0x00,0x41,0x00,0x05,0x00, +0x19,0x00,0x00,0x00,0xc7,0x01,0x00,0x00,0x16,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xc8,0x01,0x00,0x00,0xc7,0x01,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xc9,0x01,0x00,0x00,0xc6,0x01,0x00,0x00, +0xc8,0x01,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xca,0x01,0x00,0x00,0x32,0x01,0x00,0x00,0xc9,0x01,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xce,0x01,0x00,0x00, +0xca,0x01,0x00,0x00,0xc3,0x01,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xd1,0x01,0x00,0x00,0xce,0x01,0x00,0x00, +0xc5,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xd4,0x01,0x00,0x00,0xd1,0x01,0x00,0x00,0xc8,0x01,0x00,0x00, +0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xd7,0x01,0x00,0x00, +0x32,0x01,0x00,0x00,0xd4,0x01,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xdc,0x01,0x00,0x00,0xc5,0x01,0x00,0x00, +0xc8,0x01,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xdd,0x01,0x00,0x00,0xd7,0x01,0x00,0x00,0xdc,0x01,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xe1,0x01,0x00,0x00, +0xdd,0x01,0x00,0x00,0xc5,0x01,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xe4,0x01,0x00,0x00,0xe1,0x01,0x00,0x00, +0xc8,0x01,0x00,0x00,0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xe9,0x01,0x00,0x00,0xd7,0x01,0x00,0x00,0xe4,0x01,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xec,0x01,0x00,0x00, +0xe9,0x01,0x00,0x00,0xc8,0x01,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xf5,0x01,0x00,0x00,0xec,0x01,0x00,0x00, +0xc8,0x01,0x00,0x00,0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xf6,0x01,0x00,0x00,0xe9,0x01,0x00,0x00,0xf5,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0x19,0x00,0x00,0x00,0xf8,0x01,0x00,0x00, +0x16,0x00,0x00,0x00,0x57,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xf9,0x01,0x00,0x00,0xf8,0x01,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xfa,0x01,0x00,0x00, +0xca,0x01,0x00,0x00,0xf9,0x01,0x00,0x00,0x41,0x00,0x05,0x00, +0x19,0x00,0x00,0x00,0xfc,0x01,0x00,0x00,0x16,0x00,0x00,0x00, +0x5c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xfd,0x01,0x00,0x00,0xfc,0x01,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xfe,0x01,0x00,0x00,0xdd,0x01,0x00,0x00, +0xfd,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xff,0x01,0x00,0x00,0xfa,0x01,0x00,0x00,0xfe,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0x19,0x00,0x00,0x00,0x01,0x02,0x00,0x00, +0x16,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x02,0x02,0x00,0x00,0x01,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x03,0x02,0x00,0x00, +0xec,0x01,0x00,0x00,0x02,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x04,0x02,0x00,0x00,0xff,0x01,0x00,0x00, +0x03,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x19,0x00,0x00,0x00, +0x06,0x02,0x00,0x00,0x16,0x00,0x00,0x00,0x68,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x07,0x02,0x00,0x00, +0x06,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x08,0x02,0x00,0x00,0xf6,0x01,0x00,0x00,0x07,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x09,0x02,0x00,0x00, +0x04,0x02,0x00,0x00,0x08,0x02,0x00,0x00,0x41,0x00,0x06,0x00, +0x50,0x01,0x00,0x00,0x51,0x01,0x00,0x00,0x4b,0x01,0x00,0x00, +0x33,0x01,0x00,0x00,0x09,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, +0x3b,0x01,0x00,0x00,0x52,0x01,0x00,0x00,0x51,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0x19,0x00,0x00,0x00,0x49,0x02,0x00,0x00, +0x16,0x00,0x00,0x00,0xac,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x4a,0x02,0x00,0x00,0x49,0x02,0x00,0x00, +0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x4b,0x02,0x00,0x00, +0xca,0x01,0x00,0x00,0x4a,0x02,0x00,0x00,0x41,0x00,0x05,0x00, +0x19,0x00,0x00,0x00,0x4c,0x02,0x00,0x00,0x16,0x00,0x00,0x00, +0xb0,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x4d,0x02,0x00,0x00,0x4c,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x4e,0x02,0x00,0x00,0x4b,0x02,0x00,0x00, +0x4d,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x19,0x00,0x00,0x00, +0x50,0x02,0x00,0x00,0x16,0x00,0x00,0x00,0xb5,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x51,0x02,0x00,0x00, +0x50,0x02,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x52,0x02,0x00,0x00,0xdd,0x01,0x00,0x00,0x51,0x02,0x00,0x00, +0x41,0x00,0x05,0x00,0x19,0x00,0x00,0x00,0x53,0x02,0x00,0x00, +0x16,0x00,0x00,0x00,0xb9,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x54,0x02,0x00,0x00,0x53,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x55,0x02,0x00,0x00, +0x52,0x02,0x00,0x00,0x54,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x56,0x02,0x00,0x00,0x4e,0x02,0x00,0x00, +0x55,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x19,0x00,0x00,0x00, +0x58,0x02,0x00,0x00,0x16,0x00,0x00,0x00,0xbf,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x59,0x02,0x00,0x00, +0x58,0x02,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x5a,0x02,0x00,0x00,0xec,0x01,0x00,0x00,0x59,0x02,0x00,0x00, +0x41,0x00,0x05,0x00,0x19,0x00,0x00,0x00,0x5b,0x02,0x00,0x00, +0x16,0x00,0x00,0x00,0xc3,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x5c,0x02,0x00,0x00,0x5b,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x5d,0x02,0x00,0x00, +0x5a,0x02,0x00,0x00,0x5c,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x5e,0x02,0x00,0x00,0x56,0x02,0x00,0x00, +0x5d,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x19,0x00,0x00,0x00, +0x60,0x02,0x00,0x00,0x16,0x00,0x00,0x00,0xc9,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x61,0x02,0x00,0x00, +0x60,0x02,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x62,0x02,0x00,0x00,0xf6,0x01,0x00,0x00,0x61,0x02,0x00,0x00, +0x41,0x00,0x05,0x00,0x19,0x00,0x00,0x00,0x63,0x02,0x00,0x00, +0x16,0x00,0x00,0x00,0xcd,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x64,0x02,0x00,0x00,0x63,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x65,0x02,0x00,0x00, +0x62,0x02,0x00,0x00,0x64,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x66,0x02,0x00,0x00,0x5e,0x02,0x00,0x00, +0x65,0x02,0x00,0x00,0x41,0x00,0x06,0x00,0x50,0x01,0x00,0x00, +0x5b,0x01,0x00,0x00,0x56,0x01,0x00,0x00,0x33,0x01,0x00,0x00, +0x66,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0x3b,0x01,0x00,0x00, +0x5c,0x01,0x00,0x00,0x5b,0x01,0x00,0x00,0x85,0x00,0x05,0x00, +0x3b,0x01,0x00,0x00,0x5d,0x01,0x00,0x00,0x52,0x01,0x00,0x00, +0x5c,0x01,0x00,0x00,0x41,0x00,0x06,0x00,0x50,0x01,0x00,0x00, +0x5e,0x01,0x00,0x00,0x3f,0x01,0x00,0x00,0x33,0x01,0x00,0x00, +0x47,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x5e,0x01,0x00,0x00, +0x5d,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x62,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x62,0x01,0x00,0x00,0xfd,0x00,0x01,0x00, 0x38,0x00,0x01,0x00, }; -const uint64_t mul_f32_len = 1456; +const uint64_t mul_f32_len = 4276; unsigned char mul_mat_vec_f16_f32_data[] = { 0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, -0xb6,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, +0xb4,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, 0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x51,0x11,0x00,0x00, 0x0b,0x00,0x06,0x00,0x01,0x00,0x00,0x00,0x47,0x4c,0x53,0x4c, 0x2e,0x73,0x74,0x64,0x2e,0x34,0x35,0x30,0x00,0x00,0x00,0x00, 0x0e,0x00,0x03,0x00,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00, 0x0f,0x00,0x0c,0x00,0x05,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x6d,0x61,0x69,0x6e,0x00,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x13,0x00,0x00,0x00,0x1b,0x00,0x00,0x00,0x2a,0x00,0x00,0x00, -0x51,0x00,0x00,0x00,0x65,0x00,0x00,0x00,0xaa,0x00,0x00,0x00, +0x6d,0x61,0x69,0x6e,0x00,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, +0x11,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x26,0x00,0x00,0x00, +0x4f,0x00,0x00,0x00,0x63,0x00,0x00,0x00,0xa5,0x00,0x00,0x00, 0x10,0x00,0x06,0x00,0x04,0x00,0x00,0x00,0x11,0x00,0x00,0x00, 0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x0c,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x1a,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x13,0x00,0x00,0x00, -0x0b,0x00,0x00,0x00,0x1b,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x28,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x28,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x28,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x47,0x00,0x03,0x00, -0x28,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x4e,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x48,0x00,0x04,0x00,0x4f,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x4f,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x03,0x00,0x4f,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x51,0x00,0x00,0x00,0x22,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x51,0x00,0x00,0x00, -0x21,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x62,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x48,0x00,0x04,0x00,0x63,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x63,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x03,0x00,0x63,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x65,0x00,0x00,0x00,0x22,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x65,0x00,0x00,0x00, -0x21,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0xa7,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x48,0x00,0x04,0x00,0xa8,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x19,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0xa8,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x03,0x00,0xa8,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0xaa,0x00,0x00,0x00,0x22,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xaa,0x00,0x00,0x00, -0x21,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0xb2,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x19,0x00,0x00,0x00, -0x13,0x00,0x02,0x00,0x02,0x00,0x00,0x00,0x21,0x00,0x03,0x00, -0x03,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x15,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x15,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x17,0x00,0x04,0x00,0x0a,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x0b,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x0a,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x0b,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x0d,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x0e,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x0b,0x00,0x00,0x00,0x13,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x16,0x00,0x03,0x00,0x17,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x1c,0x00,0x04,0x00, -0x19,0x00,0x00,0x00,0x17,0x00,0x00,0x00,0x18,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x1a,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x19,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x1a,0x00,0x00,0x00, -0x1b,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x17,0x00,0x00,0x00,0x1d,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x1e,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x17,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x21,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x1e,0x00,0x05,0x00, -0x28,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x29,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x29,0x00,0x00,0x00,0x2a,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x2b,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x14,0x00,0x02,0x00, -0x30,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x35,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x16,0x00,0x03,0x00, -0x4d,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, -0x4e,0x00,0x00,0x00,0x4d,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, -0x4f,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x50,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x4f,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x50,0x00,0x00,0x00,0x51,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x54,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x4d,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, -0x62,0x00,0x00,0x00,0x17,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, -0x63,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x64,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x63,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x64,0x00,0x00,0x00,0x65,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x6d,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x17,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x88,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x89,0x00,0x00,0x00, -0x08,0x01,0x00,0x00,0x1d,0x00,0x03,0x00,0xa7,0x00,0x00,0x00, -0x17,0x00,0x00,0x00,0x1e,0x00,0x03,0x00,0xa8,0x00,0x00,0x00, -0xa7,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0xa9,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0xa8,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0xa9,0x00,0x00,0x00,0xaa,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x2c,0x00,0x06,0x00,0x0a,0x00,0x00,0x00,0xb2,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x18,0x00,0x00,0x00, -0x36,0x00,0x05,0x00,0x02,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x05,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x0e,0x00,0x00,0x00, -0x0f,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x0d,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x10,0x00,0x00,0x00, -0x0f,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x11,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x0e,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x13,0x00,0x00,0x00, -0x0d,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x15,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x15,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x1e,0x00,0x00,0x00,0x1f,0x00,0x00,0x00, -0x1b,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x3e,0x00,0x03,0x00, -0x1f,0x00,0x00,0x00,0x1d,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x22,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x22,0x00,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xb5,0x00,0x00,0x00, -0x21,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0x87,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x2b,0x00,0x00,0x00, -0x2c,0x00,0x00,0x00,0x2a,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x2d,0x00,0x00,0x00, -0x2c,0x00,0x00,0x00,0x87,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x2f,0x00,0x00,0x00,0x2d,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0xb1,0x00,0x05,0x00,0x30,0x00,0x00,0x00,0x31,0x00,0x00,0x00, -0xb5,0x00,0x00,0x00,0x2f,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0x24,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x31,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x24,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x23,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x37,0x00,0x00,0x00, -0x35,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x38,0x00,0x00,0x00,0xb5,0x00,0x00,0x00, -0x37,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x3d,0x00,0x00,0x00,0x11,0x00,0x00,0x00,0x2d,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3f,0x00,0x00,0x00, -0x3d,0x00,0x00,0x00,0x38,0x00,0x00,0x00,0x87,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x3f,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x8b,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x43,0x00,0x00,0x00,0x38,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x87,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x44,0x00,0x00,0x00, -0x43,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x82,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x49,0x00,0x00,0x00,0x38,0x00,0x00,0x00, -0x43,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x54,0x00,0x00,0x00, -0x55,0x00,0x00,0x00,0x51,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0x56,0x00,0x00,0x00,0x55,0x00,0x00,0x00,0x73,0x00,0x04,0x00, -0x17,0x00,0x00,0x00,0x57,0x00,0x00,0x00,0x56,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x59,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x41,0x00,0x06,0x00, -0x54,0x00,0x00,0x00,0x5a,0x00,0x00,0x00,0x51,0x00,0x00,0x00, -0x21,0x00,0x00,0x00,0x59,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0x5b,0x00,0x00,0x00,0x5a,0x00,0x00,0x00, -0x73,0x00,0x04,0x00,0x17,0x00,0x00,0x00,0x5c,0x00,0x00,0x00, -0x5b,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x2b,0x00,0x00,0x00, -0x66,0x00,0x00,0x00,0x2a,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x67,0x00,0x00,0x00, -0x66,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x69,0x00,0x00,0x00,0x67,0x00,0x00,0x00,0x49,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x6b,0x00,0x00,0x00, -0x69,0x00,0x00,0x00,0x44,0x00,0x00,0x00,0x41,0x00,0x06,0x00, -0x6d,0x00,0x00,0x00,0x6e,0x00,0x00,0x00,0x65,0x00,0x00,0x00, -0x21,0x00,0x00,0x00,0x6b,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x17,0x00,0x00,0x00,0x6f,0x00,0x00,0x00,0x6e,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x17,0x00,0x00,0x00,0x72,0x00,0x00,0x00, -0x1f,0x00,0x00,0x00,0x0c,0x00,0x08,0x00,0x17,0x00,0x00,0x00, -0x73,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00, -0x57,0x00,0x00,0x00,0x6f,0x00,0x00,0x00,0x72,0x00,0x00,0x00, -0x3e,0x00,0x03,0x00,0x1f,0x00,0x00,0x00,0x73,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x7e,0x00,0x00,0x00, -0x6b,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x41,0x00,0x06,0x00, -0x6d,0x00,0x00,0x00,0x7f,0x00,0x00,0x00,0x65,0x00,0x00,0x00, -0x21,0x00,0x00,0x00,0x7e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x17,0x00,0x00,0x00,0x80,0x00,0x00,0x00,0x7f,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x17,0x00,0x00,0x00,0x83,0x00,0x00,0x00, -0x1f,0x00,0x00,0x00,0x0c,0x00,0x08,0x00,0x17,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00, -0x5c,0x00,0x00,0x00,0x80,0x00,0x00,0x00,0x83,0x00,0x00,0x00, -0x3e,0x00,0x03,0x00,0x1f,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x87,0x00,0x00,0x00, -0xb5,0x00,0x00,0x00,0x35,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x22,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x24,0x00,0x00,0x00, -0xe0,0x00,0x04,0x00,0x88,0x00,0x00,0x00,0x88,0x00,0x00,0x00, -0x89,0x00,0x00,0x00,0xaa,0x00,0x05,0x00,0x30,0x00,0x00,0x00, -0xa4,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0xf7,0x00,0x03,0x00,0xa6,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xa4,0x00,0x00,0x00,0xa5,0x00,0x00,0x00, -0xa6,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xa5,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x2b,0x00,0x00,0x00,0xab,0x00,0x00,0x00, -0x2a,0x00,0x00,0x00,0x35,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xac,0x00,0x00,0x00,0xab,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xae,0x00,0x00,0x00, -0xac,0x00,0x00,0x00,0x11,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x1e,0x00,0x00,0x00,0xaf,0x00,0x00,0x00,0x1b,0x00,0x00,0x00, -0x21,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x17,0x00,0x00,0x00, -0xb0,0x00,0x00,0x00,0xaf,0x00,0x00,0x00,0x41,0x00,0x06,0x00, -0x6d,0x00,0x00,0x00,0xb1,0x00,0x00,0x00,0xaa,0x00,0x00,0x00, -0x21,0x00,0x00,0x00,0xae,0x00,0x00,0x00,0x3e,0x00,0x03,0x00, -0xb1,0x00,0x00,0x00,0xb0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xa6,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xa6,0x00,0x00,0x00, -0xfd,0x00,0x01,0x00,0x38,0x00,0x01,0x00, +0x47,0x00,0x04,0x00,0x0b,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, +0x1a,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x11,0x00,0x00,0x00, +0x0b,0x00,0x00,0x00,0x1b,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x15,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x24,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x24,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x24,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x08,0x00,0x00,0x00, +0x47,0x00,0x03,0x00,0x24,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x4c,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x4d,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x4d,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x4f,0x00,0x00,0x00, +0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x4f,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x60,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0x61,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x61,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x61,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x63,0x00,0x00,0x00, +0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x63,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0xa2,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0xa3,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0xa3,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0xa3,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xa5,0x00,0x00,0x00, +0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0xa5,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0xae,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xaf,0x00,0x00,0x00, +0x0b,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x13,0x00,0x02,0x00, +0x02,0x00,0x00,0x00,0x21,0x00,0x03,0x00,0x03,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x15,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x17,0x00,0x04,0x00, +0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x03,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x0a,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x09,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00, +0x0b,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x0d,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00, +0x11,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x16,0x00,0x03,0x00, +0x14,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x32,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x15,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0x1c,0x00,0x04,0x00,0x16,0x00,0x00,0x00,0x14,0x00,0x00,0x00, +0x15,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x17,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0x17,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0x1a,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x1b,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x1e,0x00,0x05,0x00, +0x24,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x25,0x00,0x00,0x00, +0x09,0x00,0x00,0x00,0x24,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0x25,0x00,0x00,0x00,0x26,0x00,0x00,0x00,0x09,0x00,0x00,0x00, +0x15,0x00,0x04,0x00,0x27,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x27,0x00,0x00,0x00, +0x28,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x29,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x14,0x00,0x02,0x00,0x2d,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x3d,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x16,0x00,0x03,0x00,0x4b,0x00,0x00,0x00, +0x10,0x00,0x00,0x00,0x1d,0x00,0x03,0x00,0x4c,0x00,0x00,0x00, +0x4b,0x00,0x00,0x00,0x1e,0x00,0x03,0x00,0x4d,0x00,0x00,0x00, +0x4c,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x4e,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x4d,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0x4e,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x52,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x4b,0x00,0x00,0x00,0x1d,0x00,0x03,0x00,0x60,0x00,0x00,0x00, +0x14,0x00,0x00,0x00,0x1e,0x00,0x03,0x00,0x61,0x00,0x00,0x00, +0x60,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x62,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x61,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0x62,0x00,0x00,0x00,0x63,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x27,0x00,0x00,0x00,0x64,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x6c,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x83,0x00,0x00,0x00,0x08,0x01,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x85,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0x15,0x00,0x00,0x00,0x32,0x00,0x00,0x00, +0x1d,0x00,0x03,0x00,0xa2,0x00,0x00,0x00,0x14,0x00,0x00,0x00, +0x1e,0x00,0x03,0x00,0xa3,0x00,0x00,0x00,0xa2,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0xa4,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0xa3,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0xa4,0x00,0x00,0x00, +0xa5,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x27,0x00,0x00,0x00,0xa6,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xae,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x33,0x00,0x06,0x00,0x09,0x00,0x00,0x00, +0xaf,0x00,0x00,0x00,0xae,0x00,0x00,0x00,0x3d,0x00,0x00,0x00, +0x3d,0x00,0x00,0x00,0x36,0x00,0x05,0x00,0x02,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x03,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0x05,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x0d,0x00,0x00,0x00,0x0e,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x0f,0x00,0x00,0x00,0x0e,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x0d,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x11,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x13,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x1b,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x18,0x00,0x00,0x00, +0x13,0x00,0x00,0x00,0x3e,0x00,0x03,0x00,0x1c,0x00,0x00,0x00, +0x1a,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x1e,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0x1e,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xb2,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x05,0x00,0x00,0x00,0x82,0x00,0x00,0x00,0x1f,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x29,0x00,0x00,0x00,0x2a,0x00,0x00,0x00, +0x26,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x2b,0x00,0x00,0x00,0x2a,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x2c,0x00,0x00,0x00, +0x2b,0x00,0x00,0x00,0x15,0x00,0x00,0x00,0xb0,0x00,0x05,0x00, +0x2d,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0xb2,0x00,0x00,0x00, +0x2c,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x20,0x00,0x00,0x00, +0x1f,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x2e,0x00,0x00,0x00,0x1f,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0x1f,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x31,0x00,0x00,0x00,0xb2,0x00,0x00,0x00, +0x15,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x34,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x13,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x35,0x00,0x00,0x00, +0x31,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, +0x2b,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x3c,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x35,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3e,0x00,0x00,0x00, +0x3c,0x00,0x00,0x00,0x3d,0x00,0x00,0x00,0x89,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x41,0x00,0x00,0x00,0x35,0x00,0x00,0x00, +0x3d,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x42,0x00,0x00,0x00,0x41,0x00,0x00,0x00,0x3d,0x00,0x00,0x00, +0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x47,0x00,0x00,0x00, +0x35,0x00,0x00,0x00,0x41,0x00,0x00,0x00,0x41,0x00,0x06,0x00, +0x52,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x4f,0x00,0x00,0x00, +0x28,0x00,0x00,0x00,0x3e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x4b,0x00,0x00,0x00,0x54,0x00,0x00,0x00,0x53,0x00,0x00,0x00, +0x73,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0x55,0x00,0x00,0x00, +0x54,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x57,0x00,0x00,0x00,0x3e,0x00,0x00,0x00,0x3d,0x00,0x00,0x00, +0x41,0x00,0x06,0x00,0x52,0x00,0x00,0x00,0x58,0x00,0x00,0x00, +0x4f,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0x57,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x4b,0x00,0x00,0x00,0x59,0x00,0x00,0x00, +0x58,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x14,0x00,0x00,0x00, +0x5a,0x00,0x00,0x00,0x59,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x29,0x00,0x00,0x00,0x65,0x00,0x00,0x00,0x26,0x00,0x00,0x00, +0x64,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x66,0x00,0x00,0x00,0x65,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x68,0x00,0x00,0x00,0x66,0x00,0x00,0x00, +0x47,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x6a,0x00,0x00,0x00,0x68,0x00,0x00,0x00,0x42,0x00,0x00,0x00, +0x41,0x00,0x06,0x00,0x6c,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, +0x63,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0x6a,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0x6e,0x00,0x00,0x00, +0x6d,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x78,0x00,0x00,0x00,0x6a,0x00,0x00,0x00,0x3d,0x00,0x00,0x00, +0x41,0x00,0x06,0x00,0x6c,0x00,0x00,0x00,0x79,0x00,0x00,0x00, +0x63,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0x78,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0x7a,0x00,0x00,0x00, +0x79,0x00,0x00,0x00,0x85,0x00,0x05,0x00,0x14,0x00,0x00,0x00, +0x7b,0x00,0x00,0x00,0x5a,0x00,0x00,0x00,0x7a,0x00,0x00,0x00, +0x0c,0x00,0x08,0x00,0x14,0x00,0x00,0x00,0x7c,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x55,0x00,0x00,0x00, +0x6e,0x00,0x00,0x00,0x7b,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x14,0x00,0x00,0x00,0x7e,0x00,0x00,0x00,0x1c,0x00,0x00,0x00, +0x81,0x00,0x05,0x00,0x14,0x00,0x00,0x00,0x7f,0x00,0x00,0x00, +0x7e,0x00,0x00,0x00,0x7c,0x00,0x00,0x00,0x3e,0x00,0x03,0x00, +0x1c,0x00,0x00,0x00,0x7f,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x82,0x00,0x00,0x00,0xb2,0x00,0x00,0x00, +0x32,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x1e,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0x20,0x00,0x00,0x00,0xe0,0x00,0x04,0x00, +0x32,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x83,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x86,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0x86,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xb3,0x00,0x00,0x00,0x85,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0x9d,0x00,0x00,0x00,0x89,0x00,0x00,0x00,0xac,0x00,0x05,0x00, +0x2d,0x00,0x00,0x00,0x8c,0x00,0x00,0x00,0xb3,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x88,0x00,0x00,0x00, +0x89,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x8c,0x00,0x00,0x00,0x87,0x00,0x00,0x00,0x88,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0x87,0x00,0x00,0x00,0xb0,0x00,0x05,0x00, +0x2d,0x00,0x00,0x00,0x8f,0x00,0x00,0x00,0x13,0x00,0x00,0x00, +0xb3,0x00,0x00,0x00,0xf7,0x00,0x03,0x00,0x91,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x8f,0x00,0x00,0x00, +0x90,0x00,0x00,0x00,0x91,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0x90,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x95,0x00,0x00,0x00,0x13,0x00,0x00,0x00,0xb3,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x1b,0x00,0x00,0x00,0x96,0x00,0x00,0x00, +0x18,0x00,0x00,0x00,0x95,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x14,0x00,0x00,0x00,0x97,0x00,0x00,0x00,0x96,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0x99,0x00,0x00,0x00, +0x1c,0x00,0x00,0x00,0x81,0x00,0x05,0x00,0x14,0x00,0x00,0x00, +0x9a,0x00,0x00,0x00,0x99,0x00,0x00,0x00,0x97,0x00,0x00,0x00, +0x3e,0x00,0x03,0x00,0x1c,0x00,0x00,0x00,0x9a,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x91,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0x91,0x00,0x00,0x00,0xe0,0x00,0x04,0x00,0x32,0x00,0x00,0x00, +0x32,0x00,0x00,0x00,0x83,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x89,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x89,0x00,0x00,0x00, +0xc2,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9d,0x00,0x00,0x00, +0xb3,0x00,0x00,0x00,0x64,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x86,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x88,0x00,0x00,0x00, +0xaa,0x00,0x05,0x00,0x2d,0x00,0x00,0x00,0x9f,0x00,0x00,0x00, +0x13,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0xf7,0x00,0x03,0x00, +0xa1,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x9f,0x00,0x00,0x00,0xa0,0x00,0x00,0x00,0xa1,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xa0,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x29,0x00,0x00,0x00,0xa7,0x00,0x00,0x00,0x26,0x00,0x00,0x00, +0xa6,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xa8,0x00,0x00,0x00,0xa7,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xaa,0x00,0x00,0x00,0xa8,0x00,0x00,0x00, +0x0f,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x1b,0x00,0x00,0x00, +0xab,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x28,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0xac,0x00,0x00,0x00, +0xab,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x6c,0x00,0x00,0x00, +0xad,0x00,0x00,0x00,0xa5,0x00,0x00,0x00,0x28,0x00,0x00,0x00, +0xaa,0x00,0x00,0x00,0x3e,0x00,0x03,0x00,0xad,0x00,0x00,0x00, +0xac,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xa1,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xa1,0x00,0x00,0x00,0xfd,0x00,0x01,0x00, +0x38,0x00,0x01,0x00, }; -const uint64_t mul_mat_vec_f16_f32_len = 2372; +const uint64_t mul_mat_vec_f16_f32_len = 2776; unsigned char mul_mat_vec_nc_f16_f32_data[] = { 0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, @@ -43707,7 +39152,7 @@ const uint64_t mul_mat_vec_p021_f16_f32_len = 2768; unsigned char mul_mat_vec_q2_K_f32_data[] = { 0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, -0xc6,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, +0xdc,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, 0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x27,0x00,0x00,0x00, 0x11,0x00,0x02,0x00,0x51,0x11,0x00,0x00,0x11,0x00,0x02,0x00, 0x60,0x11,0x00,0x00,0x0b,0x00,0x06,0x00,0x01,0x00,0x00,0x00, @@ -43715,34 +39160,676 @@ unsigned char mul_mat_vec_q2_K_f32_data[] = { 0x00,0x00,0x00,0x00,0x0e,0x00,0x03,0x00,0x00,0x00,0x00,0x00, 0x01,0x00,0x00,0x00,0x0f,0x00,0x0c,0x00,0x05,0x00,0x00,0x00, 0x04,0x00,0x00,0x00,0x6d,0x61,0x69,0x6e,0x00,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x15,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x4b,0x00,0x00,0x00,0x71,0x00,0x00,0x00,0x8e,0x00,0x00,0x00, -0xb4,0x02,0x00,0x00,0x10,0x00,0x06,0x00,0x04,0x00,0x00,0x00, +0x0b,0x00,0x00,0x00,0x13,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0x47,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x8b,0x00,0x00,0x00, +0xca,0x02,0x00,0x00,0x10,0x00,0x06,0x00,0x04,0x00,0x00,0x00, 0x11,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x0c,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x0b,0x00,0x00,0x00, 0x0b,0x00,0x00,0x00,0x1a,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x13,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x13,0x00,0x00,0x00, +0x11,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x11,0x00,0x00,0x00, 0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x13,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x11,0x00,0x00,0x00,0x02,0x00,0x00,0x00, 0x23,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x47,0x00,0x03,0x00, -0x13,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x21,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x1b,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x68,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x6a,0x00,0x00,0x00, +0x11,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x20,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x1b,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x63,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x65,0x00,0x00,0x00, 0x06,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x6d,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x6d,0x00,0x00,0x00, +0x68,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x68,0x00,0x00,0x00, 0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x10,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x6d,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x68,0x00,0x00,0x00,0x02,0x00,0x00,0x00, 0x23,0x00,0x00,0x00,0x50,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x6e,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x54,0x00,0x00,0x00, -0x48,0x00,0x04,0x00,0x6f,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x6f,0x00,0x00,0x00, +0x69,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x54,0x00,0x00,0x00, +0x48,0x00,0x04,0x00,0x6a,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x6a,0x00,0x00,0x00, 0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x03,0x00,0x6f,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x71,0x00,0x00,0x00,0x22,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x71,0x00,0x00,0x00, +0x47,0x00,0x03,0x00,0x6a,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x6c,0x00,0x00,0x00,0x22,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x6c,0x00,0x00,0x00, +0x21,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x88,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x48,0x00,0x04,0x00,0x89,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x89,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x03,0x00,0x89,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x8b,0x00,0x00,0x00,0x22,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x8b,0x00,0x00,0x00, +0x21,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0xc7,0x02,0x00,0x00,0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x48,0x00,0x04,0x00,0xc8,0x02,0x00,0x00,0x00,0x00,0x00,0x00, +0x19,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0xc8,0x02,0x00,0x00, +0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x03,0x00,0xc8,0x02,0x00,0x00,0x02,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0xca,0x02,0x00,0x00,0x22,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xca,0x02,0x00,0x00, +0x21,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0xd2,0x02,0x00,0x00,0x0b,0x00,0x00,0x00,0x19,0x00,0x00,0x00, +0x13,0x00,0x02,0x00,0x02,0x00,0x00,0x00,0x21,0x00,0x03,0x00, +0x03,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x15,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x17,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x03,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x0a,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0x0a,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x0d,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x1e,0x00,0x05,0x00, +0x11,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x12,0x00,0x00,0x00, +0x09,0x00,0x00,0x00,0x11,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0x12,0x00,0x00,0x00,0x13,0x00,0x00,0x00,0x09,0x00,0x00,0x00, +0x15,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x14,0x00,0x00,0x00, +0x15,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x16,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x19,0x00,0x00,0x00, +0x00,0x01,0x00,0x00,0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x2b,0x00,0x00,0x00, +0x08,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x36,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x3f,0x00,0x00,0x00,0x80,0x00,0x00,0x00, +0x16,0x00,0x03,0x00,0x44,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0x1c,0x00,0x04,0x00,0x45,0x00,0x00,0x00,0x44,0x00,0x00,0x00, +0x36,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x46,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x45,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0x46,0x00,0x00,0x00,0x47,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x48,0x00,0x00,0x00, +0x10,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x44,0x00,0x00,0x00, +0x4d,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x4e,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x44,0x00,0x00,0x00, +0x14,0x00,0x02,0x00,0x59,0x00,0x00,0x00,0x15,0x00,0x04,0x00, +0x62,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x1c,0x00,0x04,0x00,0x63,0x00,0x00,0x00,0x62,0x00,0x00,0x00, +0x48,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x64,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x1c,0x00,0x04,0x00, +0x65,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x64,0x00,0x00,0x00, +0x16,0x00,0x03,0x00,0x66,0x00,0x00,0x00,0x10,0x00,0x00,0x00, +0x17,0x00,0x04,0x00,0x67,0x00,0x00,0x00,0x66,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x1e,0x00,0x05,0x00,0x68,0x00,0x00,0x00, +0x63,0x00,0x00,0x00,0x65,0x00,0x00,0x00,0x67,0x00,0x00,0x00, +0x1d,0x00,0x03,0x00,0x69,0x00,0x00,0x00,0x68,0x00,0x00,0x00, +0x1e,0x00,0x03,0x00,0x6a,0x00,0x00,0x00,0x69,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x6b,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x6a,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x6b,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x14,0x00,0x00,0x00,0x70,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x71,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x66,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x79,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, +0x88,0x00,0x00,0x00,0x44,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, +0x89,0x00,0x00,0x00,0x88,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x8a,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x89,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0x8a,0x00,0x00,0x00,0x8b,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x14,0x00,0x00,0x00, +0x8c,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x95,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x44,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x9d,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x62,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x14,0x00,0x00,0x00, +0xa2,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x14,0x00,0x00,0x00,0xb3,0x00,0x00,0x00,0x03,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x0c,0x01,0x00,0x00, +0x30,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x14,0x01,0x00,0x00,0x03,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x3c,0x01,0x00,0x00,0x04,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0x4f,0x01,0x00,0x00, +0x04,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x5e,0x01,0x00,0x00,0x50,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x66,0x01,0x00,0x00,0x05,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x87,0x01,0x00,0x00, +0x60,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x8f,0x01,0x00,0x00,0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x14,0x00,0x00,0x00,0xa2,0x01,0x00,0x00,0x06,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xb1,0x01,0x00,0x00, +0x70,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xb9,0x01,0x00,0x00,0x07,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xa9,0x02,0x00,0x00,0x08,0x01,0x00,0x00, +0x1d,0x00,0x03,0x00,0xc7,0x02,0x00,0x00,0x44,0x00,0x00,0x00, +0x1e,0x00,0x03,0x00,0xc8,0x02,0x00,0x00,0xc7,0x02,0x00,0x00, +0x20,0x00,0x04,0x00,0xc9,0x02,0x00,0x00,0x0c,0x00,0x00,0x00, +0xc8,0x02,0x00,0x00,0x3b,0x00,0x04,0x00,0xc9,0x02,0x00,0x00, +0xca,0x02,0x00,0x00,0x0c,0x00,0x00,0x00,0x2c,0x00,0x06,0x00, +0x09,0x00,0x00,0x00,0xd2,0x02,0x00,0x00,0x36,0x00,0x00,0x00, +0x79,0x00,0x00,0x00,0x79,0x00,0x00,0x00,0x36,0x00,0x05,0x00, +0x02,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x03,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x05,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00,0x0e,0x00,0x00,0x00, +0x0b,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x0e,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x16,0x00,0x00,0x00,0x17,0x00,0x00,0x00, +0x13,0x00,0x00,0x00,0x15,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x17,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x1a,0x00,0x00,0x00, +0x18,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x1e,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, +0x1a,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00, +0x21,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x22,0x00,0x00,0x00, +0x21,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x24,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x28,0x00,0x00,0x00, +0x22,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x2c,0x00,0x00,0x00,0x24,0x00,0x00,0x00, +0x2b,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x30,0x00,0x00,0x00,0x2b,0x00,0x00,0x00,0x2c,0x00,0x00,0x00, +0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x31,0x00,0x00,0x00, +0x24,0x00,0x00,0x00,0x30,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x31,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x38,0x00,0x00,0x00,0x36,0x00,0x00,0x00,0x2c,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, +0x38,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x41,0x00,0x00,0x00,0x3f,0x00,0x00,0x00, +0x2c,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x43,0x00,0x00,0x00,0x41,0x00,0x00,0x00,0x34,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x4a,0x00,0x00,0x00, +0x48,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x4c,0x00,0x00,0x00,0x4a,0x00,0x00,0x00, +0x24,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x4e,0x00,0x00,0x00, +0x4f,0x00,0x00,0x00,0x47,0x00,0x00,0x00,0x4c,0x00,0x00,0x00, +0x3e,0x00,0x03,0x00,0x4f,0x00,0x00,0x00,0x4d,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x52,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0x52,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xd3,0x02,0x00,0x00,0x28,0x00,0x00,0x00,0x05,0x00,0x00,0x00, +0xa8,0x02,0x00,0x00,0x55,0x00,0x00,0x00,0xb0,0x00,0x05,0x00, +0x59,0x00,0x00,0x00,0x5a,0x00,0x00,0x00,0xd3,0x02,0x00,0x00, +0x1a,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x54,0x00,0x00,0x00, +0x55,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x5a,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x54,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0x53,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x5d,0x00,0x00,0x00,0xd3,0x02,0x00,0x00, +0x19,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x5f,0x00,0x00,0x00,0x5d,0x00,0x00,0x00,0x43,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x6f,0x00,0x00,0x00, +0x1e,0x00,0x00,0x00,0xd3,0x02,0x00,0x00,0x41,0x00,0x08,0x00, +0x71,0x00,0x00,0x00,0x72,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, +0x15,0x00,0x00,0x00,0x6f,0x00,0x00,0x00,0x70,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x66,0x00,0x00,0x00, +0x73,0x00,0x00,0x00,0x72,0x00,0x00,0x00,0x73,0x00,0x04,0x00, +0x44,0x00,0x00,0x00,0x74,0x00,0x00,0x00,0x73,0x00,0x00,0x00, +0x41,0x00,0x08,0x00,0x71,0x00,0x00,0x00,0x7a,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x15,0x00,0x00,0x00,0x6f,0x00,0x00,0x00, +0x70,0x00,0x00,0x00,0x79,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x66,0x00,0x00,0x00,0x7b,0x00,0x00,0x00,0x7a,0x00,0x00,0x00, +0x73,0x00,0x04,0x00,0x44,0x00,0x00,0x00,0x7c,0x00,0x00,0x00, +0x7b,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x81,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0x81,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, +0x44,0x00,0x00,0x00,0xd8,0x02,0x00,0x00,0x4d,0x00,0x00,0x00, +0x53,0x00,0x00,0x00,0x95,0x02,0x00,0x00,0x82,0x00,0x00,0x00, +0xf5,0x00,0x07,0x00,0x44,0x00,0x00,0x00,0xd7,0x02,0x00,0x00, +0x4d,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0xd4,0x01,0x00,0x00, +0x82,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x14,0x00,0x00,0x00, +0xd6,0x02,0x00,0x00,0x15,0x00,0x00,0x00,0x53,0x00,0x00,0x00, +0x97,0x02,0x00,0x00,0x82,0x00,0x00,0x00,0xb1,0x00,0x05,0x00, +0x59,0x00,0x00,0x00,0x87,0x00,0x00,0x00,0xd6,0x02,0x00,0x00, +0x70,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x83,0x00,0x00,0x00, +0x82,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x87,0x00,0x00,0x00,0x82,0x00,0x00,0x00,0x83,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0x82,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x16,0x00,0x00,0x00,0x8d,0x00,0x00,0x00,0x13,0x00,0x00,0x00, +0x8c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x8e,0x00,0x00,0x00,0x8d,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x90,0x00,0x00,0x00,0x8e,0x00,0x00,0x00, +0x5f,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x92,0x00,0x00,0x00,0xd6,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x93,0x00,0x00,0x00,0x90,0x00,0x00,0x00, +0x92,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x95,0x00,0x00,0x00, +0x96,0x00,0x00,0x00,0x8b,0x00,0x00,0x00,0x15,0x00,0x00,0x00, +0x93,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x44,0x00,0x00,0x00, +0x97,0x00,0x00,0x00,0x96,0x00,0x00,0x00,0x41,0x00,0x08,0x00, +0x9d,0x00,0x00,0x00,0x9e,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, +0x15,0x00,0x00,0x00,0x6f,0x00,0x00,0x00,0x15,0x00,0x00,0x00, +0x30,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x62,0x00,0x00,0x00, +0x9f,0x00,0x00,0x00,0x9e,0x00,0x00,0x00,0x71,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xa0,0x00,0x00,0x00,0x9f,0x00,0x00,0x00, +0x7c,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0xa1,0x00,0x00,0x00, +0xa0,0x00,0x00,0x00,0xc7,0x00,0x05,0x00,0x14,0x00,0x00,0x00, +0xa3,0x00,0x00,0x00,0xa1,0x00,0x00,0x00,0xa2,0x00,0x00,0x00, +0x6f,0x00,0x04,0x00,0x44,0x00,0x00,0x00,0xa4,0x00,0x00,0x00, +0xa3,0x00,0x00,0x00,0x85,0x00,0x05,0x00,0x44,0x00,0x00,0x00, +0xa5,0x00,0x00,0x00,0x97,0x00,0x00,0x00,0xa4,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xac,0x00,0x00,0x00, +0x3a,0x00,0x00,0x00,0x92,0x00,0x00,0x00,0x41,0x00,0x08,0x00, +0x9d,0x00,0x00,0x00,0xae,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, +0x15,0x00,0x00,0x00,0x6f,0x00,0x00,0x00,0x8c,0x00,0x00,0x00, +0xac,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x62,0x00,0x00,0x00, +0xaf,0x00,0x00,0x00,0xae,0x00,0x00,0x00,0xc2,0x00,0x05,0x00, +0x62,0x00,0x00,0x00,0xb0,0x00,0x00,0x00,0xaf,0x00,0x00,0x00, +0x15,0x00,0x00,0x00,0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xb1,0x00,0x00,0x00,0xb0,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, +0x14,0x00,0x00,0x00,0xb2,0x00,0x00,0x00,0xb1,0x00,0x00,0x00, +0xc7,0x00,0x05,0x00,0x14,0x00,0x00,0x00,0xb4,0x00,0x00,0x00, +0xb2,0x00,0x00,0x00,0xb3,0x00,0x00,0x00,0x6f,0x00,0x04,0x00, +0x44,0x00,0x00,0x00,0xb5,0x00,0x00,0x00,0xb4,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xbe,0x00,0x00,0x00, +0x93,0x00,0x00,0x00,0x48,0x00,0x00,0x00,0x41,0x00,0x06,0x00, +0x95,0x00,0x00,0x00,0xbf,0x00,0x00,0x00,0x8b,0x00,0x00,0x00, +0x15,0x00,0x00,0x00,0xbe,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x44,0x00,0x00,0x00,0xc0,0x00,0x00,0x00,0xbf,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xc5,0x00,0x00,0x00, +0x30,0x00,0x00,0x00,0x79,0x00,0x00,0x00,0x41,0x00,0x08,0x00, +0x9d,0x00,0x00,0x00,0xc6,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, +0x15,0x00,0x00,0x00,0x6f,0x00,0x00,0x00,0x15,0x00,0x00,0x00, +0xc5,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x62,0x00,0x00,0x00, +0xc7,0x00,0x00,0x00,0xc6,0x00,0x00,0x00,0x71,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xc8,0x00,0x00,0x00,0xc7,0x00,0x00,0x00, +0x7c,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0xc9,0x00,0x00,0x00, +0xc8,0x00,0x00,0x00,0xc7,0x00,0x05,0x00,0x14,0x00,0x00,0x00, +0xca,0x00,0x00,0x00,0xc9,0x00,0x00,0x00,0xa2,0x00,0x00,0x00, +0x6f,0x00,0x04,0x00,0x44,0x00,0x00,0x00,0xcb,0x00,0x00,0x00, +0xca,0x00,0x00,0x00,0x85,0x00,0x05,0x00,0x44,0x00,0x00,0x00, +0xcc,0x00,0x00,0x00,0xc0,0x00,0x00,0x00,0xcb,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xd4,0x00,0x00,0x00, +0xac,0x00,0x00,0x00,0x48,0x00,0x00,0x00,0x41,0x00,0x08,0x00, +0x9d,0x00,0x00,0x00,0xd5,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, +0x15,0x00,0x00,0x00,0x6f,0x00,0x00,0x00,0x8c,0x00,0x00,0x00, +0xd4,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x62,0x00,0x00,0x00, +0xd6,0x00,0x00,0x00,0xd5,0x00,0x00,0x00,0xc2,0x00,0x05,0x00, +0x62,0x00,0x00,0x00,0xd7,0x00,0x00,0x00,0xd6,0x00,0x00,0x00, +0x15,0x00,0x00,0x00,0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xd8,0x00,0x00,0x00,0xd7,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, +0x14,0x00,0x00,0x00,0xd9,0x00,0x00,0x00,0xd8,0x00,0x00,0x00, +0xc7,0x00,0x05,0x00,0x14,0x00,0x00,0x00,0xda,0x00,0x00,0x00, +0xd9,0x00,0x00,0x00,0xb3,0x00,0x00,0x00,0x6f,0x00,0x04,0x00, +0x44,0x00,0x00,0x00,0xdb,0x00,0x00,0x00,0xda,0x00,0x00,0x00, +0x85,0x00,0x05,0x00,0x44,0x00,0x00,0x00,0xdc,0x00,0x00,0x00, +0xcc,0x00,0x00,0x00,0xdb,0x00,0x00,0x00,0x0c,0x00,0x08,0x00, +0x44,0x00,0x00,0x00,0xdd,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x32,0x00,0x00,0x00,0xa5,0x00,0x00,0x00,0xb5,0x00,0x00,0x00, +0xdc,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xe5,0x00,0x00,0x00,0x93,0x00,0x00,0x00,0x36,0x00,0x00,0x00, +0x41,0x00,0x06,0x00,0x95,0x00,0x00,0x00,0xe6,0x00,0x00,0x00, +0x8b,0x00,0x00,0x00,0x15,0x00,0x00,0x00,0xe5,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x44,0x00,0x00,0x00,0xe7,0x00,0x00,0x00, +0xe6,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xec,0x00,0x00,0x00,0x30,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x41,0x00,0x08,0x00,0x9d,0x00,0x00,0x00,0xed,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x15,0x00,0x00,0x00,0x6f,0x00,0x00,0x00, +0x15,0x00,0x00,0x00,0xec,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x62,0x00,0x00,0x00,0xee,0x00,0x00,0x00,0xed,0x00,0x00,0x00, +0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xef,0x00,0x00,0x00, +0xee,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x14,0x00,0x00,0x00, +0xf0,0x00,0x00,0x00,0xef,0x00,0x00,0x00,0xc7,0x00,0x05,0x00, +0x14,0x00,0x00,0x00,0xf1,0x00,0x00,0x00,0xf0,0x00,0x00,0x00, +0xa2,0x00,0x00,0x00,0x6f,0x00,0x04,0x00,0x44,0x00,0x00,0x00, +0xf2,0x00,0x00,0x00,0xf1,0x00,0x00,0x00,0x85,0x00,0x05,0x00, +0x44,0x00,0x00,0x00,0xf3,0x00,0x00,0x00,0xe7,0x00,0x00,0x00, +0xf2,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x62,0x00,0x00,0x00, +0xfd,0x00,0x00,0x00,0xae,0x00,0x00,0x00,0xc2,0x00,0x05,0x00, +0x62,0x00,0x00,0x00,0xfe,0x00,0x00,0x00,0xfd,0x00,0x00,0x00, +0x70,0x00,0x00,0x00,0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xff,0x00,0x00,0x00,0xfe,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, +0x14,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0xff,0x00,0x00,0x00, +0xc7,0x00,0x05,0x00,0x14,0x00,0x00,0x00,0x01,0x01,0x00,0x00, +0x00,0x01,0x00,0x00,0xb3,0x00,0x00,0x00,0x6f,0x00,0x04,0x00, +0x44,0x00,0x00,0x00,0x02,0x01,0x00,0x00,0x01,0x01,0x00,0x00, +0x0c,0x00,0x08,0x00,0x44,0x00,0x00,0x00,0x04,0x01,0x00,0x00, +0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0xf3,0x00,0x00,0x00, +0x02,0x01,0x00,0x00,0xdd,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x0d,0x01,0x00,0x00,0x93,0x00,0x00,0x00, +0x0c,0x01,0x00,0x00,0x41,0x00,0x06,0x00,0x95,0x00,0x00,0x00, +0x0e,0x01,0x00,0x00,0x8b,0x00,0x00,0x00,0x15,0x00,0x00,0x00, +0x0d,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0x44,0x00,0x00,0x00, +0x0f,0x01,0x00,0x00,0x0e,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x15,0x01,0x00,0x00,0x30,0x00,0x00,0x00, +0x14,0x01,0x00,0x00,0x41,0x00,0x08,0x00,0x9d,0x00,0x00,0x00, +0x16,0x01,0x00,0x00,0x6c,0x00,0x00,0x00,0x15,0x00,0x00,0x00, +0x6f,0x00,0x00,0x00,0x15,0x00,0x00,0x00,0x15,0x01,0x00,0x00, +0x3d,0x00,0x04,0x00,0x62,0x00,0x00,0x00,0x17,0x01,0x00,0x00, +0x16,0x01,0x00,0x00,0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x18,0x01,0x00,0x00,0x17,0x01,0x00,0x00,0x7c,0x00,0x04,0x00, +0x14,0x00,0x00,0x00,0x19,0x01,0x00,0x00,0x18,0x01,0x00,0x00, +0xc7,0x00,0x05,0x00,0x14,0x00,0x00,0x00,0x1a,0x01,0x00,0x00, +0x19,0x01,0x00,0x00,0xa2,0x00,0x00,0x00,0x6f,0x00,0x04,0x00, +0x44,0x00,0x00,0x00,0x1b,0x01,0x00,0x00,0x1a,0x01,0x00,0x00, +0x85,0x00,0x05,0x00,0x44,0x00,0x00,0x00,0x1c,0x01,0x00,0x00, +0x0f,0x01,0x00,0x00,0x1b,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, +0x62,0x00,0x00,0x00,0x26,0x01,0x00,0x00,0xd5,0x00,0x00,0x00, +0xc2,0x00,0x05,0x00,0x62,0x00,0x00,0x00,0x27,0x01,0x00,0x00, +0x26,0x01,0x00,0x00,0x70,0x00,0x00,0x00,0x71,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x28,0x01,0x00,0x00,0x27,0x01,0x00,0x00, +0x7c,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0x29,0x01,0x00,0x00, +0x28,0x01,0x00,0x00,0xc7,0x00,0x05,0x00,0x14,0x00,0x00,0x00, +0x2a,0x01,0x00,0x00,0x29,0x01,0x00,0x00,0xb3,0x00,0x00,0x00, +0x6f,0x00,0x04,0x00,0x44,0x00,0x00,0x00,0x2b,0x01,0x00,0x00, +0x2a,0x01,0x00,0x00,0x0c,0x00,0x08,0x00,0x44,0x00,0x00,0x00, +0x2d,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00, +0x1c,0x01,0x00,0x00,0x2b,0x01,0x00,0x00,0x04,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x35,0x01,0x00,0x00, +0x93,0x00,0x00,0x00,0x64,0x00,0x00,0x00,0x41,0x00,0x06,0x00, +0x95,0x00,0x00,0x00,0x36,0x01,0x00,0x00,0x8b,0x00,0x00,0x00, +0x15,0x00,0x00,0x00,0x35,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, +0x44,0x00,0x00,0x00,0x37,0x01,0x00,0x00,0x36,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3d,0x01,0x00,0x00, +0x30,0x00,0x00,0x00,0x3c,0x01,0x00,0x00,0x41,0x00,0x08,0x00, +0x9d,0x00,0x00,0x00,0x3e,0x01,0x00,0x00,0x6c,0x00,0x00,0x00, +0x15,0x00,0x00,0x00,0x6f,0x00,0x00,0x00,0x15,0x00,0x00,0x00, +0x3d,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0x62,0x00,0x00,0x00, +0x3f,0x01,0x00,0x00,0x3e,0x01,0x00,0x00,0x71,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x40,0x01,0x00,0x00,0x3f,0x01,0x00,0x00, +0x7c,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0x41,0x01,0x00,0x00, +0x40,0x01,0x00,0x00,0xc7,0x00,0x05,0x00,0x14,0x00,0x00,0x00, +0x42,0x01,0x00,0x00,0x41,0x01,0x00,0x00,0xa2,0x00,0x00,0x00, +0x6f,0x00,0x04,0x00,0x44,0x00,0x00,0x00,0x43,0x01,0x00,0x00, +0x42,0x01,0x00,0x00,0x85,0x00,0x05,0x00,0x44,0x00,0x00,0x00, +0x44,0x01,0x00,0x00,0x37,0x01,0x00,0x00,0x43,0x01,0x00,0x00, +0x3d,0x00,0x04,0x00,0x62,0x00,0x00,0x00,0x4e,0x01,0x00,0x00, +0xae,0x00,0x00,0x00,0xc2,0x00,0x05,0x00,0x62,0x00,0x00,0x00, +0x50,0x01,0x00,0x00,0x4e,0x01,0x00,0x00,0x4f,0x01,0x00,0x00, +0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x51,0x01,0x00,0x00, +0x50,0x01,0x00,0x00,0x7c,0x00,0x04,0x00,0x14,0x00,0x00,0x00, +0x52,0x01,0x00,0x00,0x51,0x01,0x00,0x00,0xc7,0x00,0x05,0x00, +0x14,0x00,0x00,0x00,0x53,0x01,0x00,0x00,0x52,0x01,0x00,0x00, +0xb3,0x00,0x00,0x00,0x6f,0x00,0x04,0x00,0x44,0x00,0x00,0x00, +0x54,0x01,0x00,0x00,0x53,0x01,0x00,0x00,0x0c,0x00,0x08,0x00, +0x44,0x00,0x00,0x00,0x56,0x01,0x00,0x00,0x01,0x00,0x00,0x00, +0x32,0x00,0x00,0x00,0x44,0x01,0x00,0x00,0x54,0x01,0x00,0x00, +0x2d,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x5f,0x01,0x00,0x00,0x93,0x00,0x00,0x00,0x5e,0x01,0x00,0x00, +0x41,0x00,0x06,0x00,0x95,0x00,0x00,0x00,0x60,0x01,0x00,0x00, +0x8b,0x00,0x00,0x00,0x15,0x00,0x00,0x00,0x5f,0x01,0x00,0x00, +0x3d,0x00,0x04,0x00,0x44,0x00,0x00,0x00,0x61,0x01,0x00,0x00, +0x60,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x67,0x01,0x00,0x00,0x30,0x00,0x00,0x00,0x66,0x01,0x00,0x00, +0x41,0x00,0x08,0x00,0x9d,0x00,0x00,0x00,0x68,0x01,0x00,0x00, +0x6c,0x00,0x00,0x00,0x15,0x00,0x00,0x00,0x6f,0x00,0x00,0x00, +0x15,0x00,0x00,0x00,0x67,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, +0x62,0x00,0x00,0x00,0x69,0x01,0x00,0x00,0x68,0x01,0x00,0x00, +0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x6a,0x01,0x00,0x00, +0x69,0x01,0x00,0x00,0x7c,0x00,0x04,0x00,0x14,0x00,0x00,0x00, +0x6b,0x01,0x00,0x00,0x6a,0x01,0x00,0x00,0xc7,0x00,0x05,0x00, +0x14,0x00,0x00,0x00,0x6c,0x01,0x00,0x00,0x6b,0x01,0x00,0x00, +0xa2,0x00,0x00,0x00,0x6f,0x00,0x04,0x00,0x44,0x00,0x00,0x00, +0x6d,0x01,0x00,0x00,0x6c,0x01,0x00,0x00,0x85,0x00,0x05,0x00, +0x44,0x00,0x00,0x00,0x6e,0x01,0x00,0x00,0x61,0x01,0x00,0x00, +0x6d,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0x62,0x00,0x00,0x00, +0x78,0x01,0x00,0x00,0xd5,0x00,0x00,0x00,0xc2,0x00,0x05,0x00, +0x62,0x00,0x00,0x00,0x79,0x01,0x00,0x00,0x78,0x01,0x00,0x00, +0x4f,0x01,0x00,0x00,0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x7a,0x01,0x00,0x00,0x79,0x01,0x00,0x00,0x7c,0x00,0x04,0x00, +0x14,0x00,0x00,0x00,0x7b,0x01,0x00,0x00,0x7a,0x01,0x00,0x00, +0xc7,0x00,0x05,0x00,0x14,0x00,0x00,0x00,0x7c,0x01,0x00,0x00, +0x7b,0x01,0x00,0x00,0xb3,0x00,0x00,0x00,0x6f,0x00,0x04,0x00, +0x44,0x00,0x00,0x00,0x7d,0x01,0x00,0x00,0x7c,0x01,0x00,0x00, +0x0c,0x00,0x08,0x00,0x44,0x00,0x00,0x00,0x7f,0x01,0x00,0x00, +0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x6e,0x01,0x00,0x00, +0x7d,0x01,0x00,0x00,0x56,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x88,0x01,0x00,0x00,0x93,0x00,0x00,0x00, +0x87,0x01,0x00,0x00,0x41,0x00,0x06,0x00,0x95,0x00,0x00,0x00, +0x89,0x01,0x00,0x00,0x8b,0x00,0x00,0x00,0x15,0x00,0x00,0x00, +0x88,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0x44,0x00,0x00,0x00, +0x8a,0x01,0x00,0x00,0x89,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x90,0x01,0x00,0x00,0x30,0x00,0x00,0x00, +0x8f,0x01,0x00,0x00,0x41,0x00,0x08,0x00,0x9d,0x00,0x00,0x00, +0x91,0x01,0x00,0x00,0x6c,0x00,0x00,0x00,0x15,0x00,0x00,0x00, +0x6f,0x00,0x00,0x00,0x15,0x00,0x00,0x00,0x90,0x01,0x00,0x00, +0x3d,0x00,0x04,0x00,0x62,0x00,0x00,0x00,0x92,0x01,0x00,0x00, +0x91,0x01,0x00,0x00,0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x93,0x01,0x00,0x00,0x92,0x01,0x00,0x00,0x7c,0x00,0x04,0x00, +0x14,0x00,0x00,0x00,0x94,0x01,0x00,0x00,0x93,0x01,0x00,0x00, +0xc7,0x00,0x05,0x00,0x14,0x00,0x00,0x00,0x95,0x01,0x00,0x00, +0x94,0x01,0x00,0x00,0xa2,0x00,0x00,0x00,0x6f,0x00,0x04,0x00, +0x44,0x00,0x00,0x00,0x96,0x01,0x00,0x00,0x95,0x01,0x00,0x00, +0x85,0x00,0x05,0x00,0x44,0x00,0x00,0x00,0x97,0x01,0x00,0x00, +0x8a,0x01,0x00,0x00,0x96,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, +0x62,0x00,0x00,0x00,0xa1,0x01,0x00,0x00,0xae,0x00,0x00,0x00, +0xc2,0x00,0x05,0x00,0x62,0x00,0x00,0x00,0xa3,0x01,0x00,0x00, +0xa1,0x01,0x00,0x00,0xa2,0x01,0x00,0x00,0x71,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xa4,0x01,0x00,0x00,0xa3,0x01,0x00,0x00, +0x7c,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0xa5,0x01,0x00,0x00, +0xa4,0x01,0x00,0x00,0xc7,0x00,0x05,0x00,0x14,0x00,0x00,0x00, +0xa6,0x01,0x00,0x00,0xa5,0x01,0x00,0x00,0xb3,0x00,0x00,0x00, +0x6f,0x00,0x04,0x00,0x44,0x00,0x00,0x00,0xa7,0x01,0x00,0x00, +0xa6,0x01,0x00,0x00,0x0c,0x00,0x08,0x00,0x44,0x00,0x00,0x00, +0xa9,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00, +0x97,0x01,0x00,0x00,0xa7,0x01,0x00,0x00,0x7f,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xb2,0x01,0x00,0x00, +0x93,0x00,0x00,0x00,0xb1,0x01,0x00,0x00,0x41,0x00,0x06,0x00, +0x95,0x00,0x00,0x00,0xb3,0x01,0x00,0x00,0x8b,0x00,0x00,0x00, +0x15,0x00,0x00,0x00,0xb2,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, +0x44,0x00,0x00,0x00,0xb4,0x01,0x00,0x00,0xb3,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xba,0x01,0x00,0x00, +0x30,0x00,0x00,0x00,0xb9,0x01,0x00,0x00,0x41,0x00,0x08,0x00, +0x9d,0x00,0x00,0x00,0xbb,0x01,0x00,0x00,0x6c,0x00,0x00,0x00, +0x15,0x00,0x00,0x00,0x6f,0x00,0x00,0x00,0x15,0x00,0x00,0x00, +0xba,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0x62,0x00,0x00,0x00, +0xbc,0x01,0x00,0x00,0xbb,0x01,0x00,0x00,0x71,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xbd,0x01,0x00,0x00,0xbc,0x01,0x00,0x00, +0x7c,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0xbe,0x01,0x00,0x00, +0xbd,0x01,0x00,0x00,0xc7,0x00,0x05,0x00,0x14,0x00,0x00,0x00, +0xbf,0x01,0x00,0x00,0xbe,0x01,0x00,0x00,0xa2,0x00,0x00,0x00, +0x6f,0x00,0x04,0x00,0x44,0x00,0x00,0x00,0xc0,0x01,0x00,0x00, +0xbf,0x01,0x00,0x00,0x85,0x00,0x05,0x00,0x44,0x00,0x00,0x00, +0xc1,0x01,0x00,0x00,0xb4,0x01,0x00,0x00,0xc0,0x01,0x00,0x00, +0x3d,0x00,0x04,0x00,0x62,0x00,0x00,0x00,0xcb,0x01,0x00,0x00, +0xd5,0x00,0x00,0x00,0xc2,0x00,0x05,0x00,0x62,0x00,0x00,0x00, +0xcc,0x01,0x00,0x00,0xcb,0x01,0x00,0x00,0xa2,0x01,0x00,0x00, +0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xcd,0x01,0x00,0x00, +0xcc,0x01,0x00,0x00,0x7c,0x00,0x04,0x00,0x14,0x00,0x00,0x00, +0xce,0x01,0x00,0x00,0xcd,0x01,0x00,0x00,0xc7,0x00,0x05,0x00, +0x14,0x00,0x00,0x00,0xcf,0x01,0x00,0x00,0xce,0x01,0x00,0x00, +0xb3,0x00,0x00,0x00,0x6f,0x00,0x04,0x00,0x44,0x00,0x00,0x00, +0xd0,0x01,0x00,0x00,0xcf,0x01,0x00,0x00,0x0c,0x00,0x08,0x00, +0x44,0x00,0x00,0x00,0xd2,0x01,0x00,0x00,0x01,0x00,0x00,0x00, +0x32,0x00,0x00,0x00,0xc1,0x01,0x00,0x00,0xd0,0x01,0x00,0x00, +0xa9,0x01,0x00,0x00,0x81,0x00,0x05,0x00,0x44,0x00,0x00,0x00, +0xd4,0x01,0x00,0x00,0xd7,0x02,0x00,0x00,0xd2,0x01,0x00,0x00, +0x3d,0x00,0x04,0x00,0x44,0x00,0x00,0x00,0xde,0x01,0x00,0x00, +0x96,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x62,0x00,0x00,0x00, +0xe5,0x01,0x00,0x00,0x9e,0x00,0x00,0x00,0xc2,0x00,0x05,0x00, +0x62,0x00,0x00,0x00,0xe6,0x01,0x00,0x00,0xe5,0x01,0x00,0x00, +0x4f,0x01,0x00,0x00,0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xe7,0x01,0x00,0x00,0xe6,0x01,0x00,0x00,0x7c,0x00,0x04,0x00, +0x14,0x00,0x00,0x00,0xe8,0x01,0x00,0x00,0xe7,0x01,0x00,0x00, +0xc7,0x00,0x05,0x00,0x14,0x00,0x00,0x00,0xe9,0x01,0x00,0x00, +0xe8,0x01,0x00,0x00,0xa2,0x00,0x00,0x00,0x6f,0x00,0x04,0x00, +0x44,0x00,0x00,0x00,0xea,0x01,0x00,0x00,0xe9,0x01,0x00,0x00, +0x3d,0x00,0x04,0x00,0x44,0x00,0x00,0x00,0xf5,0x01,0x00,0x00, +0xbf,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x62,0x00,0x00,0x00, +0xfc,0x01,0x00,0x00,0xc6,0x00,0x00,0x00,0xc2,0x00,0x05,0x00, +0x62,0x00,0x00,0x00,0xfd,0x01,0x00,0x00,0xfc,0x01,0x00,0x00, +0x4f,0x01,0x00,0x00,0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xfe,0x01,0x00,0x00,0xfd,0x01,0x00,0x00,0x7c,0x00,0x04,0x00, +0x14,0x00,0x00,0x00,0xff,0x01,0x00,0x00,0xfe,0x01,0x00,0x00, +0xc7,0x00,0x05,0x00,0x14,0x00,0x00,0x00,0x00,0x02,0x00,0x00, +0xff,0x01,0x00,0x00,0xa2,0x00,0x00,0x00,0x6f,0x00,0x04,0x00, +0x44,0x00,0x00,0x00,0x01,0x02,0x00,0x00,0x00,0x02,0x00,0x00, +0x85,0x00,0x05,0x00,0x44,0x00,0x00,0x00,0x02,0x02,0x00,0x00, +0xf5,0x01,0x00,0x00,0x01,0x02,0x00,0x00,0x0c,0x00,0x08,0x00, +0x44,0x00,0x00,0x00,0x03,0x02,0x00,0x00,0x01,0x00,0x00,0x00, +0x32,0x00,0x00,0x00,0xde,0x01,0x00,0x00,0xea,0x01,0x00,0x00, +0x02,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0x44,0x00,0x00,0x00, +0x0d,0x02,0x00,0x00,0xe6,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x62,0x00,0x00,0x00,0x14,0x02,0x00,0x00,0xed,0x00,0x00,0x00, +0xc2,0x00,0x05,0x00,0x62,0x00,0x00,0x00,0x15,0x02,0x00,0x00, +0x14,0x02,0x00,0x00,0x4f,0x01,0x00,0x00,0x71,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x16,0x02,0x00,0x00,0x15,0x02,0x00,0x00, +0x7c,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0x17,0x02,0x00,0x00, +0x16,0x02,0x00,0x00,0xc7,0x00,0x05,0x00,0x14,0x00,0x00,0x00, +0x18,0x02,0x00,0x00,0x17,0x02,0x00,0x00,0xa2,0x00,0x00,0x00, +0x6f,0x00,0x04,0x00,0x44,0x00,0x00,0x00,0x19,0x02,0x00,0x00, +0x18,0x02,0x00,0x00,0x0c,0x00,0x08,0x00,0x44,0x00,0x00,0x00, +0x1b,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00, +0x0d,0x02,0x00,0x00,0x19,0x02,0x00,0x00,0x03,0x02,0x00,0x00, +0x3d,0x00,0x04,0x00,0x44,0x00,0x00,0x00,0x25,0x02,0x00,0x00, +0x0e,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0x62,0x00,0x00,0x00, +0x2c,0x02,0x00,0x00,0x16,0x01,0x00,0x00,0xc2,0x00,0x05,0x00, +0x62,0x00,0x00,0x00,0x2d,0x02,0x00,0x00,0x2c,0x02,0x00,0x00, +0x4f,0x01,0x00,0x00,0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x2e,0x02,0x00,0x00,0x2d,0x02,0x00,0x00,0x7c,0x00,0x04,0x00, +0x14,0x00,0x00,0x00,0x2f,0x02,0x00,0x00,0x2e,0x02,0x00,0x00, +0xc7,0x00,0x05,0x00,0x14,0x00,0x00,0x00,0x30,0x02,0x00,0x00, +0x2f,0x02,0x00,0x00,0xa2,0x00,0x00,0x00,0x6f,0x00,0x04,0x00, +0x44,0x00,0x00,0x00,0x31,0x02,0x00,0x00,0x30,0x02,0x00,0x00, +0x0c,0x00,0x08,0x00,0x44,0x00,0x00,0x00,0x33,0x02,0x00,0x00, +0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x25,0x02,0x00,0x00, +0x31,0x02,0x00,0x00,0x1b,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, +0x44,0x00,0x00,0x00,0x3d,0x02,0x00,0x00,0x36,0x01,0x00,0x00, +0x3d,0x00,0x04,0x00,0x62,0x00,0x00,0x00,0x44,0x02,0x00,0x00, +0x3e,0x01,0x00,0x00,0xc2,0x00,0x05,0x00,0x62,0x00,0x00,0x00, +0x45,0x02,0x00,0x00,0x44,0x02,0x00,0x00,0x4f,0x01,0x00,0x00, +0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x46,0x02,0x00,0x00, +0x45,0x02,0x00,0x00,0x7c,0x00,0x04,0x00,0x14,0x00,0x00,0x00, +0x47,0x02,0x00,0x00,0x46,0x02,0x00,0x00,0xc7,0x00,0x05,0x00, +0x14,0x00,0x00,0x00,0x48,0x02,0x00,0x00,0x47,0x02,0x00,0x00, +0xa2,0x00,0x00,0x00,0x6f,0x00,0x04,0x00,0x44,0x00,0x00,0x00, +0x49,0x02,0x00,0x00,0x48,0x02,0x00,0x00,0x0c,0x00,0x08,0x00, +0x44,0x00,0x00,0x00,0x4b,0x02,0x00,0x00,0x01,0x00,0x00,0x00, +0x32,0x00,0x00,0x00,0x3d,0x02,0x00,0x00,0x49,0x02,0x00,0x00, +0x33,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0x44,0x00,0x00,0x00, +0x55,0x02,0x00,0x00,0x60,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, +0x62,0x00,0x00,0x00,0x5c,0x02,0x00,0x00,0x68,0x01,0x00,0x00, +0xc2,0x00,0x05,0x00,0x62,0x00,0x00,0x00,0x5d,0x02,0x00,0x00, +0x5c,0x02,0x00,0x00,0x4f,0x01,0x00,0x00,0x71,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x5e,0x02,0x00,0x00,0x5d,0x02,0x00,0x00, +0x7c,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0x5f,0x02,0x00,0x00, +0x5e,0x02,0x00,0x00,0xc7,0x00,0x05,0x00,0x14,0x00,0x00,0x00, +0x60,0x02,0x00,0x00,0x5f,0x02,0x00,0x00,0xa2,0x00,0x00,0x00, +0x6f,0x00,0x04,0x00,0x44,0x00,0x00,0x00,0x61,0x02,0x00,0x00, +0x60,0x02,0x00,0x00,0x0c,0x00,0x08,0x00,0x44,0x00,0x00,0x00, +0x63,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00, +0x55,0x02,0x00,0x00,0x61,0x02,0x00,0x00,0x4b,0x02,0x00,0x00, +0x3d,0x00,0x04,0x00,0x44,0x00,0x00,0x00,0x6d,0x02,0x00,0x00, +0x89,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0x62,0x00,0x00,0x00, +0x74,0x02,0x00,0x00,0x91,0x01,0x00,0x00,0xc2,0x00,0x05,0x00, +0x62,0x00,0x00,0x00,0x75,0x02,0x00,0x00,0x74,0x02,0x00,0x00, +0x4f,0x01,0x00,0x00,0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x76,0x02,0x00,0x00,0x75,0x02,0x00,0x00,0x7c,0x00,0x04,0x00, +0x14,0x00,0x00,0x00,0x77,0x02,0x00,0x00,0x76,0x02,0x00,0x00, +0xc7,0x00,0x05,0x00,0x14,0x00,0x00,0x00,0x78,0x02,0x00,0x00, +0x77,0x02,0x00,0x00,0xa2,0x00,0x00,0x00,0x6f,0x00,0x04,0x00, +0x44,0x00,0x00,0x00,0x79,0x02,0x00,0x00,0x78,0x02,0x00,0x00, +0x0c,0x00,0x08,0x00,0x44,0x00,0x00,0x00,0x7b,0x02,0x00,0x00, +0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x6d,0x02,0x00,0x00, +0x79,0x02,0x00,0x00,0x63,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, +0x44,0x00,0x00,0x00,0x85,0x02,0x00,0x00,0xb3,0x01,0x00,0x00, +0x3d,0x00,0x04,0x00,0x62,0x00,0x00,0x00,0x8c,0x02,0x00,0x00, +0xbb,0x01,0x00,0x00,0xc2,0x00,0x05,0x00,0x62,0x00,0x00,0x00, +0x8d,0x02,0x00,0x00,0x8c,0x02,0x00,0x00,0x4f,0x01,0x00,0x00, +0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x8e,0x02,0x00,0x00, +0x8d,0x02,0x00,0x00,0x7c,0x00,0x04,0x00,0x14,0x00,0x00,0x00, +0x8f,0x02,0x00,0x00,0x8e,0x02,0x00,0x00,0xc7,0x00,0x05,0x00, +0x14,0x00,0x00,0x00,0x90,0x02,0x00,0x00,0x8f,0x02,0x00,0x00, +0xa2,0x00,0x00,0x00,0x6f,0x00,0x04,0x00,0x44,0x00,0x00,0x00, +0x91,0x02,0x00,0x00,0x90,0x02,0x00,0x00,0x0c,0x00,0x08,0x00, +0x44,0x00,0x00,0x00,0x93,0x02,0x00,0x00,0x01,0x00,0x00,0x00, +0x32,0x00,0x00,0x00,0x85,0x02,0x00,0x00,0x91,0x02,0x00,0x00, +0x7b,0x02,0x00,0x00,0x81,0x00,0x05,0x00,0x44,0x00,0x00,0x00, +0x95,0x02,0x00,0x00,0xd8,0x02,0x00,0x00,0x93,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x14,0x00,0x00,0x00,0x97,0x02,0x00,0x00, +0xd6,0x02,0x00,0x00,0x8c,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x81,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x83,0x00,0x00,0x00, +0x85,0x00,0x05,0x00,0x44,0x00,0x00,0x00,0xa1,0x02,0x00,0x00, +0x7c,0x00,0x00,0x00,0xd8,0x02,0x00,0x00,0x7f,0x00,0x04,0x00, +0x44,0x00,0x00,0x00,0xdb,0x02,0x00,0x00,0xa1,0x02,0x00,0x00, +0x0c,0x00,0x08,0x00,0x44,0x00,0x00,0x00,0xa2,0x02,0x00,0x00, +0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x74,0x00,0x00,0x00, +0xd7,0x02,0x00,0x00,0xdb,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, +0x44,0x00,0x00,0x00,0xa4,0x02,0x00,0x00,0x4f,0x00,0x00,0x00, +0x81,0x00,0x05,0x00,0x44,0x00,0x00,0x00,0xa5,0x02,0x00,0x00, +0xa4,0x02,0x00,0x00,0xa2,0x02,0x00,0x00,0x3e,0x00,0x03,0x00, +0x4f,0x00,0x00,0x00,0xa5,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0x55,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x55,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa8,0x02,0x00,0x00, +0xd3,0x02,0x00,0x00,0x23,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x52,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x54,0x00,0x00,0x00, +0xe0,0x00,0x04,0x00,0x23,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0xa9,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0xab,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0xab,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xd4,0x02,0x00,0x00,0x48,0x00,0x00,0x00, +0x54,0x00,0x00,0x00,0xc2,0x02,0x00,0x00,0xae,0x02,0x00,0x00, +0xac,0x00,0x05,0x00,0x59,0x00,0x00,0x00,0xb1,0x02,0x00,0x00, +0xd4,0x02,0x00,0x00,0x0c,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0xad,0x02,0x00,0x00,0xae,0x02,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0xb1,0x02,0x00,0x00,0xac,0x02,0x00,0x00, +0xad,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0xac,0x02,0x00,0x00, +0xb0,0x00,0x05,0x00,0x59,0x00,0x00,0x00,0xb4,0x02,0x00,0x00, +0x24,0x00,0x00,0x00,0xd4,0x02,0x00,0x00,0xf7,0x00,0x03,0x00, +0xb6,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0xb4,0x02,0x00,0x00,0xb5,0x02,0x00,0x00,0xb6,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0xb5,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xba,0x02,0x00,0x00,0x24,0x00,0x00,0x00, +0xd4,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x4e,0x00,0x00,0x00, +0xbb,0x02,0x00,0x00,0x47,0x00,0x00,0x00,0xba,0x02,0x00,0x00, +0x3d,0x00,0x04,0x00,0x44,0x00,0x00,0x00,0xbc,0x02,0x00,0x00, +0xbb,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x4e,0x00,0x00,0x00, +0xbd,0x02,0x00,0x00,0x47,0x00,0x00,0x00,0x24,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x44,0x00,0x00,0x00,0xbe,0x02,0x00,0x00, +0xbd,0x02,0x00,0x00,0x81,0x00,0x05,0x00,0x44,0x00,0x00,0x00, +0xbf,0x02,0x00,0x00,0xbe,0x02,0x00,0x00,0xbc,0x02,0x00,0x00, +0x3e,0x00,0x03,0x00,0xbd,0x02,0x00,0x00,0xbf,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0xb6,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0xb6,0x02,0x00,0x00,0xe0,0x00,0x04,0x00,0x23,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0xa9,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0xae,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0xae,0x02,0x00,0x00, +0xc2,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xc2,0x02,0x00,0x00, +0xd4,0x02,0x00,0x00,0x8c,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xab,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0xad,0x02,0x00,0x00, +0xaa,0x00,0x05,0x00,0x59,0x00,0x00,0x00,0xc4,0x02,0x00,0x00, +0x24,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0xf7,0x00,0x03,0x00, +0xc6,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0xc4,0x02,0x00,0x00,0xc5,0x02,0x00,0x00,0xc6,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0xc5,0x02,0x00,0x00,0x41,0x00,0x05,0x00, +0x16,0x00,0x00,0x00,0xcb,0x02,0x00,0x00,0x13,0x00,0x00,0x00, +0x70,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xcc,0x02,0x00,0x00,0xcb,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xce,0x02,0x00,0x00,0xcc,0x02,0x00,0x00, +0x0f,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x4e,0x00,0x00,0x00, +0xcf,0x02,0x00,0x00,0x47,0x00,0x00,0x00,0x15,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x44,0x00,0x00,0x00,0xd0,0x02,0x00,0x00, +0xcf,0x02,0x00,0x00,0x41,0x00,0x06,0x00,0x95,0x00,0x00,0x00, +0xd1,0x02,0x00,0x00,0xca,0x02,0x00,0x00,0x15,0x00,0x00,0x00, +0xce,0x02,0x00,0x00,0x3e,0x00,0x03,0x00,0xd1,0x02,0x00,0x00, +0xd0,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0xc6,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0xc6,0x02,0x00,0x00,0xfd,0x00,0x01,0x00, +0x38,0x00,0x01,0x00, +}; +const uint64_t mul_mat_vec_q2_K_f32_len = 7612; + +unsigned char mul_mat_vec_q3_K_f32_data[] = { +0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, +0x1d,0x03,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, +0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x27,0x00,0x00,0x00, +0x11,0x00,0x02,0x00,0x51,0x11,0x00,0x00,0x11,0x00,0x02,0x00, +0x60,0x11,0x00,0x00,0x0b,0x00,0x06,0x00,0x01,0x00,0x00,0x00, +0x47,0x4c,0x53,0x4c,0x2e,0x73,0x74,0x64,0x2e,0x34,0x35,0x30, +0x00,0x00,0x00,0x00,0x0e,0x00,0x03,0x00,0x00,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x0f,0x00,0x0c,0x00,0x05,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x6d,0x61,0x69,0x6e,0x00,0x00,0x00,0x00, +0x0b,0x00,0x00,0x00,0x13,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0x4f,0x00,0x00,0x00,0x77,0x00,0x00,0x00,0x8e,0x00,0x00,0x00, +0x0e,0x03,0x00,0x00,0x10,0x00,0x06,0x00,0x04,0x00,0x00,0x00, +0x11,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x0b,0x00,0x00,0x00, +0x0b,0x00,0x00,0x00,0x1a,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x11,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x11,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x11,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x47,0x00,0x03,0x00, +0x11,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x20,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x1b,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x6d,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x6f,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x71,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x73,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x73,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x73,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x60,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x73,0x00,0x00,0x00,0x03,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x74,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x6e,0x00,0x00,0x00, +0x48,0x00,0x04,0x00,0x75,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x75,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x03,0x00,0x75,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x77,0x00,0x00,0x00,0x22,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x77,0x00,0x00,0x00, 0x21,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, 0x8b,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00, 0x48,0x00,0x04,0x00,0x8c,0x00,0x00,0x00,0x00,0x00,0x00,0x00, @@ -43752,1923 +39839,1279 @@ unsigned char mul_mat_vec_q2_K_f32_data[] = { 0x47,0x00,0x04,0x00,0x8e,0x00,0x00,0x00,0x22,0x00,0x00,0x00, 0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x8e,0x00,0x00,0x00, 0x21,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0xb1,0x02,0x00,0x00,0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x48,0x00,0x04,0x00,0xb2,0x02,0x00,0x00,0x00,0x00,0x00,0x00, -0x19,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0xb2,0x02,0x00,0x00, +0x0b,0x03,0x00,0x00,0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x48,0x00,0x04,0x00,0x0c,0x03,0x00,0x00,0x00,0x00,0x00,0x00, +0x19,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x0c,0x03,0x00,0x00, 0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x03,0x00,0xb2,0x02,0x00,0x00,0x02,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0xb4,0x02,0x00,0x00,0x22,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xb4,0x02,0x00,0x00, +0x47,0x00,0x03,0x00,0x0c,0x03,0x00,0x00,0x02,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x0e,0x03,0x00,0x00,0x22,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x0e,0x03,0x00,0x00, 0x21,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0xbc,0x02,0x00,0x00,0x0b,0x00,0x00,0x00,0x19,0x00,0x00,0x00, +0x17,0x03,0x00,0x00,0x0b,0x00,0x00,0x00,0x19,0x00,0x00,0x00, 0x13,0x00,0x02,0x00,0x02,0x00,0x00,0x00,0x21,0x00,0x03,0x00, 0x03,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x15,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x15,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x17,0x00,0x04,0x00,0x0a,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x0b,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x0a,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x0b,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x0d,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x0e,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x1e,0x00,0x05,0x00,0x13,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x14,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x13,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0x15,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x17,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x1a,0x00,0x00,0x00, -0x00,0x01,0x00,0x00,0x3b,0x00,0x04,0x00,0x0b,0x00,0x00,0x00, -0x21,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x25,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x08,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x42,0x00,0x00,0x00,0x80,0x00,0x00,0x00, -0x16,0x00,0x03,0x00,0x47,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x48,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x1c,0x00,0x04,0x00,0x49,0x00,0x00,0x00, -0x47,0x00,0x00,0x00,0x48,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x4a,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x49,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0x4b,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x17,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x03,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x0a,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0x0a,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x0d,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x1e,0x00,0x05,0x00, +0x11,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x12,0x00,0x00,0x00, +0x09,0x00,0x00,0x00,0x11,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0x12,0x00,0x00,0x00,0x13,0x00,0x00,0x00,0x09,0x00,0x00,0x00, +0x15,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x14,0x00,0x00,0x00, +0x15,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x16,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x19,0x00,0x00,0x00, +0x00,0x01,0x00,0x00,0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x2b,0x00,0x00,0x00, +0x08,0x00,0x00,0x00,0x15,0x00,0x04,0x00,0x32,0x00,0x00,0x00, +0x08,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x14,0x00,0x00,0x00,0x35,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x36,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x15,0x00,0x04,0x00,0x3a,0x00,0x00,0x00, +0x08,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x41,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x47,0x00,0x00,0x00, +0x80,0x00,0x00,0x00,0x16,0x00,0x03,0x00,0x4c,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0x1c,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, +0x4c,0x00,0x00,0x00,0x41,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x4e,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x4d,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0x4e,0x00,0x00,0x00,0x4f,0x00,0x00,0x00, 0x04,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x4c,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x47,0x00,0x00,0x00,0x51,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x52,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x47,0x00,0x00,0x00,0x14,0x00,0x02,0x00,0x5d,0x00,0x00,0x00, -0x15,0x00,0x04,0x00,0x66,0x00,0x00,0x00,0x08,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x67,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x1c,0x00,0x04,0x00, -0x68,0x00,0x00,0x00,0x66,0x00,0x00,0x00,0x67,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x69,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0x1c,0x00,0x04,0x00,0x6a,0x00,0x00,0x00, -0x66,0x00,0x00,0x00,0x69,0x00,0x00,0x00,0x16,0x00,0x03,0x00, -0x6b,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x17,0x00,0x04,0x00, -0x6c,0x00,0x00,0x00,0x6b,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x1e,0x00,0x05,0x00,0x6d,0x00,0x00,0x00,0x68,0x00,0x00,0x00, -0x6a,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, -0x6e,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, -0x6f,0x00,0x00,0x00,0x6e,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x70,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x6f,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x70,0x00,0x00,0x00,0x71,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x75,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x6b,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x7d,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x1d,0x00,0x03,0x00,0x8b,0x00,0x00,0x00,0x47,0x00,0x00,0x00, -0x1e,0x00,0x03,0x00,0x8c,0x00,0x00,0x00,0x8b,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x8d,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x8c,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x8d,0x00,0x00,0x00, -0x8e,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x8f,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x50,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x4c,0x00,0x00,0x00,0x55,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x56,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x4c,0x00,0x00,0x00,0x14,0x00,0x02,0x00,0x64,0x00,0x00,0x00, +0x1c,0x00,0x04,0x00,0x6d,0x00,0x00,0x00,0x32,0x00,0x00,0x00, +0x41,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x6e,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x1c,0x00,0x04,0x00, +0x6f,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x6e,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x70,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x1c,0x00,0x04,0x00,0x71,0x00,0x00,0x00, +0x32,0x00,0x00,0x00,0x70,0x00,0x00,0x00,0x16,0x00,0x03,0x00, +0x72,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x1e,0x00,0x06,0x00, +0x73,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x6f,0x00,0x00,0x00, +0x71,0x00,0x00,0x00,0x72,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, +0x74,0x00,0x00,0x00,0x73,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, +0x75,0x00,0x00,0x00,0x74,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x76,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x75,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0x76,0x00,0x00,0x00,0x77,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x14,0x00,0x00,0x00, +0x7b,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x7c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x72,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0x89,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x1d,0x00,0x03,0x00,0x8b,0x00,0x00,0x00, +0x4c,0x00,0x00,0x00,0x1e,0x00,0x03,0x00,0x8c,0x00,0x00,0x00, +0x8b,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x8d,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x8c,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0x8d,0x00,0x00,0x00,0x8e,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, 0x20,0x00,0x04,0x00,0x97,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x47,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x9f,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x66,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xa4,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xb4,0x00,0x00,0x00, -0x03,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x08,0x01,0x00,0x00,0x30,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x2e,0x01,0x00,0x00,0x40,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x36,0x01,0x00,0x00, -0x04,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x55,0x01,0x00,0x00,0x50,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x5d,0x01,0x00,0x00,0x05,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x7c,0x01,0x00,0x00, -0x60,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x84,0x01,0x00,0x00,0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xa3,0x01,0x00,0x00,0x70,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xab,0x01,0x00,0x00, -0x07,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x92,0x02,0x00,0x00,0x02,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x93,0x02,0x00,0x00,0x08,0x01,0x00,0x00, -0x1d,0x00,0x03,0x00,0xb1,0x02,0x00,0x00,0x47,0x00,0x00,0x00, -0x1e,0x00,0x03,0x00,0xb2,0x02,0x00,0x00,0xb1,0x02,0x00,0x00, -0x20,0x00,0x04,0x00,0xb3,0x02,0x00,0x00,0x0c,0x00,0x00,0x00, -0xb2,0x02,0x00,0x00,0x3b,0x00,0x04,0x00,0xb3,0x02,0x00,0x00, -0xb4,0x02,0x00,0x00,0x0c,0x00,0x00,0x00,0x2c,0x00,0x06,0x00, -0x0a,0x00,0x00,0x00,0xbc,0x02,0x00,0x00,0x48,0x00,0x00,0x00, -0x7d,0x00,0x00,0x00,0x7d,0x00,0x00,0x00,0x36,0x00,0x05,0x00, +0x4c,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x9d,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x14,0x00,0x00,0x00,0xa4,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0xa9,0x00,0x00,0x00, +0x08,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x14,0x00,0x00,0x00, +0xb2,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x14,0x00,0x00,0x00,0xb7,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0xf1,0x00,0x00,0x00, +0x0a,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x70,0x01,0x00,0x00,0x60,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x14,0x00,0x00,0x00,0x77,0x01,0x00,0x00,0x06,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0xc9,0x01,0x00,0x00, +0x09,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x02,0x02,0x00,0x00,0x30,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x14,0x00,0x00,0x00,0x13,0x02,0x00,0x00,0x0b,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x4d,0x02,0x00,0x00, +0x50,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x14,0x00,0x00,0x00, +0x54,0x02,0x00,0x00,0x05,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x98,0x02,0x00,0x00,0x70,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0x9f,0x02,0x00,0x00, +0x07,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xed,0x02,0x00,0x00,0x08,0x01,0x00,0x00,0x1d,0x00,0x03,0x00, +0x0b,0x03,0x00,0x00,0x4c,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, +0x0c,0x03,0x00,0x00,0x0b,0x03,0x00,0x00,0x20,0x00,0x04,0x00, +0x0d,0x03,0x00,0x00,0x0c,0x00,0x00,0x00,0x0c,0x03,0x00,0x00, +0x3b,0x00,0x04,0x00,0x0d,0x03,0x00,0x00,0x0e,0x03,0x00,0x00, +0x0c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x16,0x03,0x00,0x00,0x01,0x00,0x00,0x00,0x2c,0x00,0x06,0x00, +0x09,0x00,0x00,0x00,0x17,0x03,0x00,0x00,0x41,0x00,0x00,0x00, +0x16,0x03,0x00,0x00,0x16,0x03,0x00,0x00,0x36,0x00,0x05,0x00, 0x02,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x00, 0x03,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x05,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x0e,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x0d,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x11,0x00,0x00,0x00, -0x10,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x15,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x19,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x87,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x1b,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x1a,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x1f,0x00,0x00,0x00, -0x11,0x00,0x00,0x00,0x1b,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x0e,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x0d,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x24,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x87,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x26,0x00,0x00,0x00, -0x24,0x00,0x00,0x00,0x25,0x00,0x00,0x00,0x8b,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x2b,0x00,0x00,0x00,0x24,0x00,0x00,0x00, -0x25,0x00,0x00,0x00,0x87,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x2f,0x00,0x00,0x00,0x26,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x33,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x2f,0x00,0x00,0x00,0x82,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0x26,0x00,0x00,0x00, -0x33,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x37,0x00,0x00,0x00,0x25,0x00,0x00,0x00,0x34,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3b,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x2f,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x3d,0x00,0x00,0x00,0x3b,0x00,0x00,0x00, -0x37,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x44,0x00,0x00,0x00,0x42,0x00,0x00,0x00,0x2f,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x46,0x00,0x00,0x00, -0x44,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x4c,0x00,0x00,0x00, -0x2b,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x50,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x26,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x52,0x00,0x00,0x00,0x53,0x00,0x00,0x00, -0x4b,0x00,0x00,0x00,0x50,0x00,0x00,0x00,0x3e,0x00,0x03,0x00, -0x53,0x00,0x00,0x00,0x51,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x56,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x56,0x00,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xbd,0x02,0x00,0x00, -0x2b,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0x91,0x02,0x00,0x00, -0x59,0x00,0x00,0x00,0xb1,0x00,0x05,0x00,0x5d,0x00,0x00,0x00, -0x5e,0x00,0x00,0x00,0xbd,0x02,0x00,0x00,0x1b,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0x58,0x00,0x00,0x00,0x59,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x5e,0x00,0x00,0x00, -0x57,0x00,0x00,0x00,0x58,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x57,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x61,0x00,0x00,0x00,0xbd,0x02,0x00,0x00,0x1a,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x63,0x00,0x00,0x00, -0x61,0x00,0x00,0x00,0x46,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x74,0x00,0x00,0x00,0x1f,0x00,0x00,0x00, -0xbd,0x02,0x00,0x00,0x41,0x00,0x08,0x00,0x75,0x00,0x00,0x00, -0x76,0x00,0x00,0x00,0x71,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0x74,0x00,0x00,0x00,0x25,0x00,0x00,0x00,0x0d,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x6b,0x00,0x00,0x00,0x77,0x00,0x00,0x00, -0x76,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x47,0x00,0x00,0x00, -0x78,0x00,0x00,0x00,0x77,0x00,0x00,0x00,0x41,0x00,0x08,0x00, -0x75,0x00,0x00,0x00,0x7e,0x00,0x00,0x00,0x71,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x74,0x00,0x00,0x00,0x25,0x00,0x00,0x00, -0x7d,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x6b,0x00,0x00,0x00, -0x7f,0x00,0x00,0x00,0x7e,0x00,0x00,0x00,0x73,0x00,0x04,0x00, -0x47,0x00,0x00,0x00,0x80,0x00,0x00,0x00,0x7f,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x84,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x84,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x47,0x00,0x00,0x00, -0xc2,0x02,0x00,0x00,0x51,0x00,0x00,0x00,0x57,0x00,0x00,0x00, -0x7e,0x02,0x00,0x00,0x85,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, -0x47,0x00,0x00,0x00,0xc1,0x02,0x00,0x00,0x51,0x00,0x00,0x00, -0x57,0x00,0x00,0x00,0xc5,0x01,0x00,0x00,0x85,0x00,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xc0,0x02,0x00,0x00, -0x16,0x00,0x00,0x00,0x57,0x00,0x00,0x00,0x80,0x02,0x00,0x00, -0x85,0x00,0x00,0x00,0xb1,0x00,0x05,0x00,0x5d,0x00,0x00,0x00, -0x8a,0x00,0x00,0x00,0xc0,0x02,0x00,0x00,0x25,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0x86,0x00,0x00,0x00,0x85,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x8a,0x00,0x00,0x00, -0x85,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x85,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00, -0x90,0x00,0x00,0x00,0x15,0x00,0x00,0x00,0x8f,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x91,0x00,0x00,0x00, -0x90,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x93,0x00,0x00,0x00,0x91,0x00,0x00,0x00,0x63,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x95,0x00,0x00,0x00, -0x93,0x00,0x00,0x00,0xc0,0x02,0x00,0x00,0x41,0x00,0x06,0x00, -0x97,0x00,0x00,0x00,0x98,0x00,0x00,0x00,0x8e,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x95,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x47,0x00,0x00,0x00,0x99,0x00,0x00,0x00,0x98,0x00,0x00,0x00, -0x41,0x00,0x08,0x00,0x9f,0x00,0x00,0x00,0xa0,0x00,0x00,0x00, -0x71,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x74,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x33,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x66,0x00,0x00,0x00,0xa1,0x00,0x00,0x00,0xa0,0x00,0x00,0x00, -0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0xa2,0x00,0x00,0x00, -0xa1,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xa3,0x00,0x00,0x00,0xa2,0x00,0x00,0x00,0xc7,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xa5,0x00,0x00,0x00,0xa3,0x00,0x00,0x00, -0xa4,0x00,0x00,0x00,0x6f,0x00,0x04,0x00,0x47,0x00,0x00,0x00, -0xa6,0x00,0x00,0x00,0xa5,0x00,0x00,0x00,0x85,0x00,0x05,0x00, -0x47,0x00,0x00,0x00,0xa7,0x00,0x00,0x00,0x99,0x00,0x00,0x00, -0xa6,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xad,0x00,0x00,0x00,0x3d,0x00,0x00,0x00,0xc0,0x02,0x00,0x00, -0x41,0x00,0x08,0x00,0x9f,0x00,0x00,0x00,0xaf,0x00,0x00,0x00, -0x71,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x74,0x00,0x00,0x00, -0x8f,0x00,0x00,0x00,0xad,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x66,0x00,0x00,0x00,0xb0,0x00,0x00,0x00,0xaf,0x00,0x00,0x00, -0xc2,0x00,0x05,0x00,0x66,0x00,0x00,0x00,0xb1,0x00,0x00,0x00, -0xb0,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x71,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0xb2,0x00,0x00,0x00,0xb1,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xb3,0x00,0x00,0x00, -0xb2,0x00,0x00,0x00,0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xb5,0x00,0x00,0x00,0xb3,0x00,0x00,0x00,0xb4,0x00,0x00,0x00, -0x6f,0x00,0x04,0x00,0x47,0x00,0x00,0x00,0xb6,0x00,0x00,0x00, -0xb5,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xbe,0x00,0x00,0x00,0x95,0x00,0x00,0x00,0x4c,0x00,0x00,0x00, -0x41,0x00,0x06,0x00,0x97,0x00,0x00,0x00,0xbf,0x00,0x00,0x00, -0x8e,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0xbe,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x47,0x00,0x00,0x00,0xc0,0x00,0x00,0x00, -0xbf,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xc5,0x00,0x00,0x00,0x33,0x00,0x00,0x00,0x8f,0x00,0x00,0x00, -0x41,0x00,0x08,0x00,0x9f,0x00,0x00,0x00,0xc6,0x00,0x00,0x00, -0x71,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x74,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0xc5,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x66,0x00,0x00,0x00,0xc7,0x00,0x00,0x00,0xc6,0x00,0x00,0x00, -0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0xc8,0x00,0x00,0x00, -0xc7,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xc9,0x00,0x00,0x00,0xc8,0x00,0x00,0x00,0xc7,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xca,0x00,0x00,0x00,0xc9,0x00,0x00,0x00, -0xa4,0x00,0x00,0x00,0x6f,0x00,0x04,0x00,0x47,0x00,0x00,0x00, -0xcb,0x00,0x00,0x00,0xca,0x00,0x00,0x00,0x85,0x00,0x05,0x00, -0x47,0x00,0x00,0x00,0xcc,0x00,0x00,0x00,0xc0,0x00,0x00,0x00, -0xcb,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xd3,0x00,0x00,0x00,0xad,0x00,0x00,0x00,0x4c,0x00,0x00,0x00, -0x41,0x00,0x08,0x00,0x9f,0x00,0x00,0x00,0xd4,0x00,0x00,0x00, -0x71,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x74,0x00,0x00,0x00, -0x8f,0x00,0x00,0x00,0xd3,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x66,0x00,0x00,0x00,0xd5,0x00,0x00,0x00,0xd4,0x00,0x00,0x00, -0xc2,0x00,0x05,0x00,0x66,0x00,0x00,0x00,0xd6,0x00,0x00,0x00, -0xd5,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x71,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0xd7,0x00,0x00,0x00,0xd6,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xd8,0x00,0x00,0x00, -0xd7,0x00,0x00,0x00,0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xd9,0x00,0x00,0x00,0xd8,0x00,0x00,0x00,0xb4,0x00,0x00,0x00, -0x6f,0x00,0x04,0x00,0x47,0x00,0x00,0x00,0xda,0x00,0x00,0x00, -0xd9,0x00,0x00,0x00,0x85,0x00,0x05,0x00,0x47,0x00,0x00,0x00, -0xdb,0x00,0x00,0x00,0xcc,0x00,0x00,0x00,0xda,0x00,0x00,0x00, -0x0c,0x00,0x08,0x00,0x47,0x00,0x00,0x00,0xdc,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0xa7,0x00,0x00,0x00, -0xb6,0x00,0x00,0x00,0xdb,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xe3,0x00,0x00,0x00,0x95,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x97,0x00,0x00,0x00, -0xe4,0x00,0x00,0x00,0x8e,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0xe3,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x47,0x00,0x00,0x00, -0xe5,0x00,0x00,0x00,0xe4,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xea,0x00,0x00,0x00,0x33,0x00,0x00,0x00, -0x25,0x00,0x00,0x00,0x41,0x00,0x08,0x00,0x9f,0x00,0x00,0x00, -0xeb,0x00,0x00,0x00,0x71,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0x74,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0xea,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x66,0x00,0x00,0x00,0xec,0x00,0x00,0x00, -0xeb,0x00,0x00,0x00,0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0xed,0x00,0x00,0x00,0xec,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xee,0x00,0x00,0x00,0xed,0x00,0x00,0x00, -0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xef,0x00,0x00,0x00, -0xee,0x00,0x00,0x00,0xa4,0x00,0x00,0x00,0x6f,0x00,0x04,0x00, -0x47,0x00,0x00,0x00,0xf0,0x00,0x00,0x00,0xef,0x00,0x00,0x00, -0x85,0x00,0x05,0x00,0x47,0x00,0x00,0x00,0xf1,0x00,0x00,0x00, -0xe5,0x00,0x00,0x00,0xf0,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x66,0x00,0x00,0x00,0xfa,0x00,0x00,0x00,0xaf,0x00,0x00,0x00, -0xc2,0x00,0x05,0x00,0x66,0x00,0x00,0x00,0xfb,0x00,0x00,0x00, -0xfa,0x00,0x00,0x00,0x25,0x00,0x00,0x00,0x71,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0xfc,0x00,0x00,0x00,0xfb,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xfd,0x00,0x00,0x00, -0xfc,0x00,0x00,0x00,0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xfe,0x00,0x00,0x00,0xfd,0x00,0x00,0x00,0xb4,0x00,0x00,0x00, -0x6f,0x00,0x04,0x00,0x47,0x00,0x00,0x00,0xff,0x00,0x00,0x00, -0xfe,0x00,0x00,0x00,0x0c,0x00,0x08,0x00,0x47,0x00,0x00,0x00, -0x01,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00, -0xf1,0x00,0x00,0x00,0xff,0x00,0x00,0x00,0xdc,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x09,0x01,0x00,0x00, -0x95,0x00,0x00,0x00,0x08,0x01,0x00,0x00,0x41,0x00,0x06,0x00, -0x97,0x00,0x00,0x00,0x0a,0x01,0x00,0x00,0x8e,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x09,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, -0x47,0x00,0x00,0x00,0x0b,0x01,0x00,0x00,0x0a,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x10,0x01,0x00,0x00, -0x33,0x00,0x00,0x00,0xb4,0x00,0x00,0x00,0x41,0x00,0x08,0x00, -0x9f,0x00,0x00,0x00,0x11,0x01,0x00,0x00,0x71,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x74,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0x10,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0x66,0x00,0x00,0x00, -0x12,0x01,0x00,0x00,0x11,0x01,0x00,0x00,0x71,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x13,0x01,0x00,0x00,0x12,0x01,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x14,0x01,0x00,0x00, -0x13,0x01,0x00,0x00,0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x15,0x01,0x00,0x00,0x14,0x01,0x00,0x00,0xa4,0x00,0x00,0x00, -0x6f,0x00,0x04,0x00,0x47,0x00,0x00,0x00,0x16,0x01,0x00,0x00, -0x15,0x01,0x00,0x00,0x85,0x00,0x05,0x00,0x47,0x00,0x00,0x00, -0x17,0x01,0x00,0x00,0x0b,0x01,0x00,0x00,0x16,0x01,0x00,0x00, -0x3d,0x00,0x04,0x00,0x66,0x00,0x00,0x00,0x20,0x01,0x00,0x00, -0xd4,0x00,0x00,0x00,0xc2,0x00,0x05,0x00,0x66,0x00,0x00,0x00, -0x21,0x01,0x00,0x00,0x20,0x01,0x00,0x00,0x25,0x00,0x00,0x00, -0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x22,0x01,0x00,0x00, -0x21,0x01,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x23,0x01,0x00,0x00,0x22,0x01,0x00,0x00,0xc7,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x24,0x01,0x00,0x00,0x23,0x01,0x00,0x00, -0xb4,0x00,0x00,0x00,0x6f,0x00,0x04,0x00,0x47,0x00,0x00,0x00, -0x25,0x01,0x00,0x00,0x24,0x01,0x00,0x00,0x0c,0x00,0x08,0x00, -0x47,0x00,0x00,0x00,0x27,0x01,0x00,0x00,0x01,0x00,0x00,0x00, -0x32,0x00,0x00,0x00,0x17,0x01,0x00,0x00,0x25,0x01,0x00,0x00, -0x01,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x2f,0x01,0x00,0x00,0x95,0x00,0x00,0x00,0x2e,0x01,0x00,0x00, -0x41,0x00,0x06,0x00,0x97,0x00,0x00,0x00,0x30,0x01,0x00,0x00, -0x8e,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x2f,0x01,0x00,0x00, -0x3d,0x00,0x04,0x00,0x47,0x00,0x00,0x00,0x31,0x01,0x00,0x00, -0x30,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x37,0x01,0x00,0x00,0x33,0x00,0x00,0x00,0x36,0x01,0x00,0x00, -0x41,0x00,0x08,0x00,0x9f,0x00,0x00,0x00,0x38,0x01,0x00,0x00, -0x71,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x74,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x37,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, -0x66,0x00,0x00,0x00,0x39,0x01,0x00,0x00,0x38,0x01,0x00,0x00, -0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x3a,0x01,0x00,0x00, -0x39,0x01,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x3b,0x01,0x00,0x00,0x3a,0x01,0x00,0x00,0xc7,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x3c,0x01,0x00,0x00,0x3b,0x01,0x00,0x00, -0xa4,0x00,0x00,0x00,0x6f,0x00,0x04,0x00,0x47,0x00,0x00,0x00, -0x3d,0x01,0x00,0x00,0x3c,0x01,0x00,0x00,0x85,0x00,0x05,0x00, -0x47,0x00,0x00,0x00,0x3e,0x01,0x00,0x00,0x31,0x01,0x00,0x00, -0x3d,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0x66,0x00,0x00,0x00, -0x47,0x01,0x00,0x00,0xaf,0x00,0x00,0x00,0xc2,0x00,0x05,0x00, -0x66,0x00,0x00,0x00,0x48,0x01,0x00,0x00,0x47,0x01,0x00,0x00, -0x36,0x01,0x00,0x00,0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x49,0x01,0x00,0x00,0x48,0x01,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x4a,0x01,0x00,0x00,0x49,0x01,0x00,0x00, -0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x4b,0x01,0x00,0x00, -0x4a,0x01,0x00,0x00,0xb4,0x00,0x00,0x00,0x6f,0x00,0x04,0x00, -0x47,0x00,0x00,0x00,0x4c,0x01,0x00,0x00,0x4b,0x01,0x00,0x00, -0x0c,0x00,0x08,0x00,0x47,0x00,0x00,0x00,0x4e,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x3e,0x01,0x00,0x00, -0x4c,0x01,0x00,0x00,0x27,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x56,0x01,0x00,0x00,0x95,0x00,0x00,0x00, -0x55,0x01,0x00,0x00,0x41,0x00,0x06,0x00,0x97,0x00,0x00,0x00, -0x57,0x01,0x00,0x00,0x8e,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0x56,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0x47,0x00,0x00,0x00, -0x58,0x01,0x00,0x00,0x57,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x5e,0x01,0x00,0x00,0x33,0x00,0x00,0x00, -0x5d,0x01,0x00,0x00,0x41,0x00,0x08,0x00,0x9f,0x00,0x00,0x00, -0x5f,0x01,0x00,0x00,0x71,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0x74,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x5e,0x01,0x00,0x00, -0x3d,0x00,0x04,0x00,0x66,0x00,0x00,0x00,0x60,0x01,0x00,0x00, -0x5f,0x01,0x00,0x00,0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x61,0x01,0x00,0x00,0x60,0x01,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x62,0x01,0x00,0x00,0x61,0x01,0x00,0x00, -0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x63,0x01,0x00,0x00, -0x62,0x01,0x00,0x00,0xa4,0x00,0x00,0x00,0x6f,0x00,0x04,0x00, -0x47,0x00,0x00,0x00,0x64,0x01,0x00,0x00,0x63,0x01,0x00,0x00, -0x85,0x00,0x05,0x00,0x47,0x00,0x00,0x00,0x65,0x01,0x00,0x00, -0x58,0x01,0x00,0x00,0x64,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, -0x66,0x00,0x00,0x00,0x6e,0x01,0x00,0x00,0xd4,0x00,0x00,0x00, -0xc2,0x00,0x05,0x00,0x66,0x00,0x00,0x00,0x6f,0x01,0x00,0x00, -0x6e,0x01,0x00,0x00,0x36,0x01,0x00,0x00,0x71,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x70,0x01,0x00,0x00,0x6f,0x01,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x71,0x01,0x00,0x00, -0x70,0x01,0x00,0x00,0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x72,0x01,0x00,0x00,0x71,0x01,0x00,0x00,0xb4,0x00,0x00,0x00, -0x6f,0x00,0x04,0x00,0x47,0x00,0x00,0x00,0x73,0x01,0x00,0x00, -0x72,0x01,0x00,0x00,0x0c,0x00,0x08,0x00,0x47,0x00,0x00,0x00, -0x75,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00, -0x65,0x01,0x00,0x00,0x73,0x01,0x00,0x00,0x4e,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x7d,0x01,0x00,0x00, -0x95,0x00,0x00,0x00,0x7c,0x01,0x00,0x00,0x41,0x00,0x06,0x00, -0x97,0x00,0x00,0x00,0x7e,0x01,0x00,0x00,0x8e,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x7d,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, -0x47,0x00,0x00,0x00,0x7f,0x01,0x00,0x00,0x7e,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x85,0x01,0x00,0x00, -0x33,0x00,0x00,0x00,0x84,0x01,0x00,0x00,0x41,0x00,0x08,0x00, -0x9f,0x00,0x00,0x00,0x86,0x01,0x00,0x00,0x71,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x74,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0x85,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0x66,0x00,0x00,0x00, -0x87,0x01,0x00,0x00,0x86,0x01,0x00,0x00,0x71,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x88,0x01,0x00,0x00,0x87,0x01,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x89,0x01,0x00,0x00, -0x88,0x01,0x00,0x00,0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x8a,0x01,0x00,0x00,0x89,0x01,0x00,0x00,0xa4,0x00,0x00,0x00, -0x6f,0x00,0x04,0x00,0x47,0x00,0x00,0x00,0x8b,0x01,0x00,0x00, -0x8a,0x01,0x00,0x00,0x85,0x00,0x05,0x00,0x47,0x00,0x00,0x00, -0x8c,0x01,0x00,0x00,0x7f,0x01,0x00,0x00,0x8b,0x01,0x00,0x00, -0x3d,0x00,0x04,0x00,0x66,0x00,0x00,0x00,0x95,0x01,0x00,0x00, -0xaf,0x00,0x00,0x00,0xc2,0x00,0x05,0x00,0x66,0x00,0x00,0x00, -0x96,0x01,0x00,0x00,0x95,0x01,0x00,0x00,0x84,0x01,0x00,0x00, -0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x97,0x01,0x00,0x00, -0x96,0x01,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x98,0x01,0x00,0x00,0x97,0x01,0x00,0x00,0xc7,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x99,0x01,0x00,0x00,0x98,0x01,0x00,0x00, -0xb4,0x00,0x00,0x00,0x6f,0x00,0x04,0x00,0x47,0x00,0x00,0x00, -0x9a,0x01,0x00,0x00,0x99,0x01,0x00,0x00,0x0c,0x00,0x08,0x00, -0x47,0x00,0x00,0x00,0x9c,0x01,0x00,0x00,0x01,0x00,0x00,0x00, -0x32,0x00,0x00,0x00,0x8c,0x01,0x00,0x00,0x9a,0x01,0x00,0x00, -0x75,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xa4,0x01,0x00,0x00,0x95,0x00,0x00,0x00,0xa3,0x01,0x00,0x00, -0x41,0x00,0x06,0x00,0x97,0x00,0x00,0x00,0xa5,0x01,0x00,0x00, -0x8e,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0xa4,0x01,0x00,0x00, -0x3d,0x00,0x04,0x00,0x47,0x00,0x00,0x00,0xa6,0x01,0x00,0x00, -0xa5,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xac,0x01,0x00,0x00,0x33,0x00,0x00,0x00,0xab,0x01,0x00,0x00, -0x41,0x00,0x08,0x00,0x9f,0x00,0x00,0x00,0xad,0x01,0x00,0x00, -0x71,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x74,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0xac,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, -0x66,0x00,0x00,0x00,0xae,0x01,0x00,0x00,0xad,0x01,0x00,0x00, -0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0xaf,0x01,0x00,0x00, -0xae,0x01,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xb0,0x01,0x00,0x00,0xaf,0x01,0x00,0x00,0xc7,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xb1,0x01,0x00,0x00,0xb0,0x01,0x00,0x00, -0xa4,0x00,0x00,0x00,0x6f,0x00,0x04,0x00,0x47,0x00,0x00,0x00, -0xb2,0x01,0x00,0x00,0xb1,0x01,0x00,0x00,0x85,0x00,0x05,0x00, -0x47,0x00,0x00,0x00,0xb3,0x01,0x00,0x00,0xa6,0x01,0x00,0x00, -0xb2,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0x66,0x00,0x00,0x00, -0xbc,0x01,0x00,0x00,0xd4,0x00,0x00,0x00,0xc2,0x00,0x05,0x00, -0x66,0x00,0x00,0x00,0xbd,0x01,0x00,0x00,0xbc,0x01,0x00,0x00, -0x84,0x01,0x00,0x00,0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0xbe,0x01,0x00,0x00,0xbd,0x01,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xbf,0x01,0x00,0x00,0xbe,0x01,0x00,0x00, -0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xc0,0x01,0x00,0x00, -0xbf,0x01,0x00,0x00,0xb4,0x00,0x00,0x00,0x6f,0x00,0x04,0x00, -0x47,0x00,0x00,0x00,0xc1,0x01,0x00,0x00,0xc0,0x01,0x00,0x00, -0x0c,0x00,0x08,0x00,0x47,0x00,0x00,0x00,0xc3,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0xb3,0x01,0x00,0x00, -0xc1,0x01,0x00,0x00,0x9c,0x01,0x00,0x00,0x81,0x00,0x05,0x00, -0x47,0x00,0x00,0x00,0xc5,0x01,0x00,0x00,0xc1,0x02,0x00,0x00, -0xc3,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0x47,0x00,0x00,0x00, -0xce,0x01,0x00,0x00,0x98,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x66,0x00,0x00,0x00,0xd5,0x01,0x00,0x00,0xa0,0x00,0x00,0x00, -0xc2,0x00,0x05,0x00,0x66,0x00,0x00,0x00,0xd6,0x01,0x00,0x00, -0xd5,0x01,0x00,0x00,0x36,0x01,0x00,0x00,0x71,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0xd7,0x01,0x00,0x00,0xd6,0x01,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xd8,0x01,0x00,0x00, -0xd7,0x01,0x00,0x00,0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xd9,0x01,0x00,0x00,0xd8,0x01,0x00,0x00,0xa4,0x00,0x00,0x00, -0x6f,0x00,0x04,0x00,0x47,0x00,0x00,0x00,0xda,0x01,0x00,0x00, -0xd9,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0x47,0x00,0x00,0x00, -0xe4,0x01,0x00,0x00,0xbf,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x66,0x00,0x00,0x00,0xeb,0x01,0x00,0x00,0xc6,0x00,0x00,0x00, -0xc2,0x00,0x05,0x00,0x66,0x00,0x00,0x00,0xec,0x01,0x00,0x00, -0xeb,0x01,0x00,0x00,0x36,0x01,0x00,0x00,0x71,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0xed,0x01,0x00,0x00,0xec,0x01,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xee,0x01,0x00,0x00, -0xed,0x01,0x00,0x00,0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xef,0x01,0x00,0x00,0xee,0x01,0x00,0x00,0xa4,0x00,0x00,0x00, -0x6f,0x00,0x04,0x00,0x47,0x00,0x00,0x00,0xf0,0x01,0x00,0x00, -0xef,0x01,0x00,0x00,0x85,0x00,0x05,0x00,0x47,0x00,0x00,0x00, -0xf1,0x01,0x00,0x00,0xe4,0x01,0x00,0x00,0xf0,0x01,0x00,0x00, -0x0c,0x00,0x08,0x00,0x47,0x00,0x00,0x00,0xf2,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0xce,0x01,0x00,0x00, -0xda,0x01,0x00,0x00,0xf1,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, -0x47,0x00,0x00,0x00,0xfb,0x01,0x00,0x00,0xe4,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x66,0x00,0x00,0x00,0x02,0x02,0x00,0x00, -0xeb,0x00,0x00,0x00,0xc2,0x00,0x05,0x00,0x66,0x00,0x00,0x00, -0x03,0x02,0x00,0x00,0x02,0x02,0x00,0x00,0x36,0x01,0x00,0x00, -0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x04,0x02,0x00,0x00, -0x03,0x02,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x05,0x02,0x00,0x00,0x04,0x02,0x00,0x00,0xc7,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x06,0x02,0x00,0x00,0x05,0x02,0x00,0x00, -0xa4,0x00,0x00,0x00,0x6f,0x00,0x04,0x00,0x47,0x00,0x00,0x00, -0x07,0x02,0x00,0x00,0x06,0x02,0x00,0x00,0x0c,0x00,0x08,0x00, -0x47,0x00,0x00,0x00,0x09,0x02,0x00,0x00,0x01,0x00,0x00,0x00, -0x32,0x00,0x00,0x00,0xfb,0x01,0x00,0x00,0x07,0x02,0x00,0x00, -0xf2,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0x47,0x00,0x00,0x00, -0x12,0x02,0x00,0x00,0x0a,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, -0x66,0x00,0x00,0x00,0x19,0x02,0x00,0x00,0x11,0x01,0x00,0x00, -0xc2,0x00,0x05,0x00,0x66,0x00,0x00,0x00,0x1a,0x02,0x00,0x00, -0x19,0x02,0x00,0x00,0x36,0x01,0x00,0x00,0x71,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x1b,0x02,0x00,0x00,0x1a,0x02,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x1c,0x02,0x00,0x00, -0x1b,0x02,0x00,0x00,0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x1d,0x02,0x00,0x00,0x1c,0x02,0x00,0x00,0xa4,0x00,0x00,0x00, -0x6f,0x00,0x04,0x00,0x47,0x00,0x00,0x00,0x1e,0x02,0x00,0x00, -0x1d,0x02,0x00,0x00,0x0c,0x00,0x08,0x00,0x47,0x00,0x00,0x00, -0x20,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00, -0x12,0x02,0x00,0x00,0x1e,0x02,0x00,0x00,0x09,0x02,0x00,0x00, -0x3d,0x00,0x04,0x00,0x47,0x00,0x00,0x00,0x29,0x02,0x00,0x00, -0x30,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0x66,0x00,0x00,0x00, -0x30,0x02,0x00,0x00,0x38,0x01,0x00,0x00,0xc2,0x00,0x05,0x00, -0x66,0x00,0x00,0x00,0x31,0x02,0x00,0x00,0x30,0x02,0x00,0x00, -0x36,0x01,0x00,0x00,0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x32,0x02,0x00,0x00,0x31,0x02,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x33,0x02,0x00,0x00,0x32,0x02,0x00,0x00, -0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x34,0x02,0x00,0x00, -0x33,0x02,0x00,0x00,0xa4,0x00,0x00,0x00,0x6f,0x00,0x04,0x00, -0x47,0x00,0x00,0x00,0x35,0x02,0x00,0x00,0x34,0x02,0x00,0x00, -0x0c,0x00,0x08,0x00,0x47,0x00,0x00,0x00,0x37,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x29,0x02,0x00,0x00, -0x35,0x02,0x00,0x00,0x20,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, -0x47,0x00,0x00,0x00,0x40,0x02,0x00,0x00,0x57,0x01,0x00,0x00, -0x3d,0x00,0x04,0x00,0x66,0x00,0x00,0x00,0x47,0x02,0x00,0x00, -0x5f,0x01,0x00,0x00,0xc2,0x00,0x05,0x00,0x66,0x00,0x00,0x00, -0x48,0x02,0x00,0x00,0x47,0x02,0x00,0x00,0x36,0x01,0x00,0x00, -0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x49,0x02,0x00,0x00, -0x48,0x02,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x4a,0x02,0x00,0x00,0x49,0x02,0x00,0x00,0xc7,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x4b,0x02,0x00,0x00,0x4a,0x02,0x00,0x00, -0xa4,0x00,0x00,0x00,0x6f,0x00,0x04,0x00,0x47,0x00,0x00,0x00, -0x4c,0x02,0x00,0x00,0x4b,0x02,0x00,0x00,0x0c,0x00,0x08,0x00, -0x47,0x00,0x00,0x00,0x4e,0x02,0x00,0x00,0x01,0x00,0x00,0x00, -0x32,0x00,0x00,0x00,0x40,0x02,0x00,0x00,0x4c,0x02,0x00,0x00, -0x37,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0x47,0x00,0x00,0x00, -0x57,0x02,0x00,0x00,0x7e,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, -0x66,0x00,0x00,0x00,0x5e,0x02,0x00,0x00,0x86,0x01,0x00,0x00, -0xc2,0x00,0x05,0x00,0x66,0x00,0x00,0x00,0x5f,0x02,0x00,0x00, -0x5e,0x02,0x00,0x00,0x36,0x01,0x00,0x00,0x71,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x60,0x02,0x00,0x00,0x5f,0x02,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x61,0x02,0x00,0x00, -0x60,0x02,0x00,0x00,0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x62,0x02,0x00,0x00,0x61,0x02,0x00,0x00,0xa4,0x00,0x00,0x00, -0x6f,0x00,0x04,0x00,0x47,0x00,0x00,0x00,0x63,0x02,0x00,0x00, -0x62,0x02,0x00,0x00,0x0c,0x00,0x08,0x00,0x47,0x00,0x00,0x00, -0x65,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00, -0x57,0x02,0x00,0x00,0x63,0x02,0x00,0x00,0x4e,0x02,0x00,0x00, -0x3d,0x00,0x04,0x00,0x47,0x00,0x00,0x00,0x6e,0x02,0x00,0x00, -0xa5,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0x66,0x00,0x00,0x00, -0x75,0x02,0x00,0x00,0xad,0x01,0x00,0x00,0xc2,0x00,0x05,0x00, -0x66,0x00,0x00,0x00,0x76,0x02,0x00,0x00,0x75,0x02,0x00,0x00, -0x36,0x01,0x00,0x00,0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x77,0x02,0x00,0x00,0x76,0x02,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x78,0x02,0x00,0x00,0x77,0x02,0x00,0x00, -0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x79,0x02,0x00,0x00, -0x78,0x02,0x00,0x00,0xa4,0x00,0x00,0x00,0x6f,0x00,0x04,0x00, -0x47,0x00,0x00,0x00,0x7a,0x02,0x00,0x00,0x79,0x02,0x00,0x00, -0x0c,0x00,0x08,0x00,0x47,0x00,0x00,0x00,0x7c,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x6e,0x02,0x00,0x00, -0x7a,0x02,0x00,0x00,0x65,0x02,0x00,0x00,0x81,0x00,0x05,0x00, -0x47,0x00,0x00,0x00,0x7e,0x02,0x00,0x00,0xc2,0x02,0x00,0x00, -0x7c,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x80,0x02,0x00,0x00,0xc0,0x02,0x00,0x00,0x8f,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x84,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x86,0x00,0x00,0x00,0x85,0x00,0x05,0x00,0x47,0x00,0x00,0x00, -0x8a,0x02,0x00,0x00,0x80,0x00,0x00,0x00,0xc2,0x02,0x00,0x00, -0x7f,0x00,0x04,0x00,0x47,0x00,0x00,0x00,0xc5,0x02,0x00,0x00, -0x8a,0x02,0x00,0x00,0x0c,0x00,0x08,0x00,0x47,0x00,0x00,0x00, -0x8b,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00, -0x78,0x00,0x00,0x00,0xc1,0x02,0x00,0x00,0xc5,0x02,0x00,0x00, -0x3d,0x00,0x04,0x00,0x47,0x00,0x00,0x00,0x8d,0x02,0x00,0x00, -0x53,0x00,0x00,0x00,0x81,0x00,0x05,0x00,0x47,0x00,0x00,0x00, -0x8e,0x02,0x00,0x00,0x8d,0x02,0x00,0x00,0x8b,0x02,0x00,0x00, -0x3e,0x00,0x03,0x00,0x53,0x00,0x00,0x00,0x8e,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0x59,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x59,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x91,0x02,0x00,0x00,0xbd,0x02,0x00,0x00,0x25,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x56,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x58,0x00,0x00,0x00,0xe0,0x00,0x04,0x00,0x92,0x02,0x00,0x00, -0x92,0x02,0x00,0x00,0x93,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x95,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x95,0x02,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xbe,0x02,0x00,0x00, -0x4c,0x00,0x00,0x00,0x58,0x00,0x00,0x00,0xac,0x02,0x00,0x00, -0x98,0x02,0x00,0x00,0xad,0x00,0x05,0x00,0x5d,0x00,0x00,0x00, -0x9b,0x02,0x00,0x00,0xbe,0x02,0x00,0x00,0x16,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0x97,0x02,0x00,0x00,0x98,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x9b,0x02,0x00,0x00, -0x96,0x02,0x00,0x00,0x97,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x96,0x02,0x00,0x00,0xb1,0x00,0x05,0x00,0x5d,0x00,0x00,0x00, -0x9e,0x02,0x00,0x00,0x26,0x00,0x00,0x00,0xbe,0x02,0x00,0x00, -0xf7,0x00,0x03,0x00,0xa0,0x02,0x00,0x00,0x00,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x9e,0x02,0x00,0x00,0x9f,0x02,0x00,0x00, -0xa0,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x9f,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa4,0x02,0x00,0x00, -0x26,0x00,0x00,0x00,0xbe,0x02,0x00,0x00,0x41,0x00,0x05,0x00, -0x52,0x00,0x00,0x00,0xa5,0x02,0x00,0x00,0x4b,0x00,0x00,0x00, -0xa4,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0x47,0x00,0x00,0x00, -0xa6,0x02,0x00,0x00,0xa5,0x02,0x00,0x00,0x41,0x00,0x05,0x00, -0x52,0x00,0x00,0x00,0xa7,0x02,0x00,0x00,0x4b,0x00,0x00,0x00, -0x26,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x47,0x00,0x00,0x00, -0xa8,0x02,0x00,0x00,0xa7,0x02,0x00,0x00,0x81,0x00,0x05,0x00, -0x47,0x00,0x00,0x00,0xa9,0x02,0x00,0x00,0xa8,0x02,0x00,0x00, -0xa6,0x02,0x00,0x00,0x3e,0x00,0x03,0x00,0xa7,0x02,0x00,0x00, -0xa9,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0xa0,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0xa0,0x02,0x00,0x00,0xe0,0x00,0x04,0x00, -0x92,0x02,0x00,0x00,0x92,0x02,0x00,0x00,0x93,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0x98,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x98,0x02,0x00,0x00,0xc3,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xac,0x02,0x00,0x00,0xbe,0x02,0x00,0x00,0x8f,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x95,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x97,0x02,0x00,0x00,0xaa,0x00,0x05,0x00,0x5d,0x00,0x00,0x00, -0xae,0x02,0x00,0x00,0x26,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0xf7,0x00,0x03,0x00,0xb0,0x02,0x00,0x00,0x00,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xae,0x02,0x00,0x00,0xaf,0x02,0x00,0x00, -0xb0,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0xaf,0x02,0x00,0x00, -0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00,0xb5,0x02,0x00,0x00, -0x15,0x00,0x00,0x00,0x25,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xb6,0x02,0x00,0x00,0xb5,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xb8,0x02,0x00,0x00, -0xb6,0x02,0x00,0x00,0x11,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x52,0x00,0x00,0x00,0xb9,0x02,0x00,0x00,0x4b,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x47,0x00,0x00,0x00, -0xba,0x02,0x00,0x00,0xb9,0x02,0x00,0x00,0x41,0x00,0x06,0x00, -0x97,0x00,0x00,0x00,0xbb,0x02,0x00,0x00,0xb4,0x02,0x00,0x00, -0x16,0x00,0x00,0x00,0xb8,0x02,0x00,0x00,0x3e,0x00,0x03,0x00, -0xbb,0x02,0x00,0x00,0xba,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0xb0,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0xb0,0x02,0x00,0x00, +0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00,0x0e,0x00,0x00,0x00, +0x0b,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x0e,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x16,0x00,0x00,0x00,0x17,0x00,0x00,0x00, +0x13,0x00,0x00,0x00,0x15,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x17,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x1a,0x00,0x00,0x00, +0x18,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x1e,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, +0x1a,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00, +0x21,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x22,0x00,0x00,0x00, +0x21,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x24,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x28,0x00,0x00,0x00, +0x22,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x2c,0x00,0x00,0x00,0x24,0x00,0x00,0x00, +0x2b,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x30,0x00,0x00,0x00,0x2b,0x00,0x00,0x00,0x2c,0x00,0x00,0x00, +0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x31,0x00,0x00,0x00, +0x24,0x00,0x00,0x00,0x30,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x38,0x00,0x00,0x00,0x36,0x00,0x00,0x00, +0x2c,0x00,0x00,0x00,0xc4,0x00,0x05,0x00,0x14,0x00,0x00,0x00, +0x39,0x00,0x00,0x00,0x35,0x00,0x00,0x00,0x38,0x00,0x00,0x00, +0x72,0x00,0x04,0x00,0x3a,0x00,0x00,0x00,0x3b,0x00,0x00,0x00, +0x39,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x32,0x00,0x00,0x00, +0x3c,0x00,0x00,0x00,0x3b,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x3f,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x31,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x43,0x00,0x00,0x00,0x41,0x00,0x00,0x00,0x2c,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x45,0x00,0x00,0x00, +0x43,0x00,0x00,0x00,0x3f,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x49,0x00,0x00,0x00,0x47,0x00,0x00,0x00, +0x2c,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x4b,0x00,0x00,0x00,0x49,0x00,0x00,0x00,0x3f,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x52,0x00,0x00,0x00, +0x50,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x54,0x00,0x00,0x00,0x52,0x00,0x00,0x00, +0x24,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x56,0x00,0x00,0x00, +0x57,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x54,0x00,0x00,0x00, +0x3e,0x00,0x03,0x00,0x57,0x00,0x00,0x00,0x55,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x5d,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0x5d,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0x18,0x03,0x00,0x00,0x28,0x00,0x00,0x00,0x05,0x00,0x00,0x00, +0xec,0x02,0x00,0x00,0x60,0x00,0x00,0x00,0xb0,0x00,0x05,0x00, +0x64,0x00,0x00,0x00,0x65,0x00,0x00,0x00,0x18,0x03,0x00,0x00, +0x1a,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x5f,0x00,0x00,0x00, +0x60,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x65,0x00,0x00,0x00,0x5e,0x00,0x00,0x00,0x5f,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0x5e,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x68,0x00,0x00,0x00,0x18,0x03,0x00,0x00, +0x19,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x6a,0x00,0x00,0x00,0x68,0x00,0x00,0x00,0x4b,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x7a,0x00,0x00,0x00, +0x1e,0x00,0x00,0x00,0x18,0x03,0x00,0x00,0x41,0x00,0x07,0x00, +0x7c,0x00,0x00,0x00,0x7d,0x00,0x00,0x00,0x77,0x00,0x00,0x00, +0x15,0x00,0x00,0x00,0x7a,0x00,0x00,0x00,0x7b,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x72,0x00,0x00,0x00,0x7e,0x00,0x00,0x00, +0x7d,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x4c,0x00,0x00,0x00, +0x7f,0x00,0x00,0x00,0x7e,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x83,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x83,0x00,0x00,0x00, +0xf5,0x00,0x07,0x00,0x4c,0x00,0x00,0x00,0x1c,0x03,0x00,0x00, +0x55,0x00,0x00,0x00,0x5e,0x00,0x00,0x00,0xdd,0x02,0x00,0x00, +0x84,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x14,0x00,0x00,0x00, +0x1b,0x03,0x00,0x00,0x15,0x00,0x00,0x00,0x5e,0x00,0x00,0x00, +0xdf,0x02,0x00,0x00,0x84,0x00,0x00,0x00,0xb1,0x00,0x05,0x00, +0x64,0x00,0x00,0x00,0x8a,0x00,0x00,0x00,0x1b,0x03,0x00,0x00, +0x89,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x85,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x8a,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x85,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0x84,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x16,0x00,0x00,0x00,0x8f,0x00,0x00,0x00,0x13,0x00,0x00,0x00, +0x35,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x90,0x00,0x00,0x00,0x8f,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x92,0x00,0x00,0x00,0x90,0x00,0x00,0x00, +0x6a,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x94,0x00,0x00,0x00,0x1b,0x03,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x95,0x00,0x00,0x00,0x92,0x00,0x00,0x00, +0x94,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x97,0x00,0x00,0x00, +0x98,0x00,0x00,0x00,0x8e,0x00,0x00,0x00,0x15,0x00,0x00,0x00, +0x95,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x4c,0x00,0x00,0x00, +0x99,0x00,0x00,0x00,0x98,0x00,0x00,0x00,0x41,0x00,0x08,0x00, +0x9d,0x00,0x00,0x00,0x9e,0x00,0x00,0x00,0x77,0x00,0x00,0x00, +0x15,0x00,0x00,0x00,0x7a,0x00,0x00,0x00,0x89,0x00,0x00,0x00, +0x15,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x32,0x00,0x00,0x00, +0x9f,0x00,0x00,0x00,0x9e,0x00,0x00,0x00,0xc2,0x00,0x05,0x00, +0x32,0x00,0x00,0x00,0xa1,0x00,0x00,0x00,0x9f,0x00,0x00,0x00, +0x38,0x00,0x00,0x00,0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xa2,0x00,0x00,0x00,0xa1,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, +0x14,0x00,0x00,0x00,0xa3,0x00,0x00,0x00,0xa2,0x00,0x00,0x00, +0xc7,0x00,0x05,0x00,0x14,0x00,0x00,0x00,0xa5,0x00,0x00,0x00, +0xa3,0x00,0x00,0x00,0xa4,0x00,0x00,0x00,0x41,0x00,0x08,0x00, +0x9d,0x00,0x00,0x00,0xaa,0x00,0x00,0x00,0x77,0x00,0x00,0x00, +0x15,0x00,0x00,0x00,0x7a,0x00,0x00,0x00,0x89,0x00,0x00,0x00, +0xa9,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x32,0x00,0x00,0x00, +0xab,0x00,0x00,0x00,0xaa,0x00,0x00,0x00,0xc2,0x00,0x05,0x00, +0x32,0x00,0x00,0x00,0xae,0x00,0x00,0x00,0xab,0x00,0x00,0x00, +0x38,0x00,0x00,0x00,0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xaf,0x00,0x00,0x00,0xae,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, +0x14,0x00,0x00,0x00,0xb0,0x00,0x00,0x00,0xaf,0x00,0x00,0x00, +0xc7,0x00,0x05,0x00,0x14,0x00,0x00,0x00,0xb1,0x00,0x00,0x00, +0xb0,0x00,0x00,0x00,0x7b,0x00,0x00,0x00,0xc4,0x00,0x05,0x00, +0x14,0x00,0x00,0x00,0xb3,0x00,0x00,0x00,0xb1,0x00,0x00,0x00, +0xb2,0x00,0x00,0x00,0xc5,0x00,0x05,0x00,0x14,0x00,0x00,0x00, +0xb4,0x00,0x00,0x00,0xa5,0x00,0x00,0x00,0xb3,0x00,0x00,0x00, +0x72,0x00,0x04,0x00,0x3a,0x00,0x00,0x00,0xb5,0x00,0x00,0x00, +0xb4,0x00,0x00,0x00,0x72,0x00,0x04,0x00,0x14,0x00,0x00,0x00, +0xb6,0x00,0x00,0x00,0xb5,0x00,0x00,0x00,0x82,0x00,0x05,0x00, +0x14,0x00,0x00,0x00,0xb8,0x00,0x00,0x00,0xb6,0x00,0x00,0x00, +0xb7,0x00,0x00,0x00,0x6f,0x00,0x04,0x00,0x4c,0x00,0x00,0x00, +0xb9,0x00,0x00,0x00,0xb8,0x00,0x00,0x00,0x85,0x00,0x05,0x00, +0x4c,0x00,0x00,0x00,0xba,0x00,0x00,0x00,0x99,0x00,0x00,0x00, +0xb9,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xc1,0x00,0x00,0x00,0x45,0x00,0x00,0x00,0x94,0x00,0x00,0x00, +0x41,0x00,0x08,0x00,0x9d,0x00,0x00,0x00,0xc2,0x00,0x00,0x00, +0x77,0x00,0x00,0x00,0x15,0x00,0x00,0x00,0x7a,0x00,0x00,0x00, +0x35,0x00,0x00,0x00,0xc1,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x32,0x00,0x00,0x00,0xc3,0x00,0x00,0x00,0xc2,0x00,0x00,0x00, +0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xc4,0x00,0x00,0x00, +0xc3,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x14,0x00,0x00,0x00, +0xc5,0x00,0x00,0x00,0xc4,0x00,0x00,0x00,0xc7,0x00,0x05,0x00, +0x14,0x00,0x00,0x00,0xc6,0x00,0x00,0x00,0xc5,0x00,0x00,0x00, +0x7b,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xcd,0x00,0x00,0x00,0x3f,0x00,0x00,0x00,0x94,0x00,0x00,0x00, +0x41,0x00,0x08,0x00,0x9d,0x00,0x00,0x00,0xce,0x00,0x00,0x00, +0x77,0x00,0x00,0x00,0x15,0x00,0x00,0x00,0x7a,0x00,0x00,0x00, +0x15,0x00,0x00,0x00,0xcd,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x32,0x00,0x00,0x00,0xcf,0x00,0x00,0x00,0xce,0x00,0x00,0x00, +0xc4,0x00,0x05,0x00,0x32,0x00,0x00,0x00,0xd1,0x00,0x00,0x00, +0x3c,0x00,0x00,0x00,0x15,0x00,0x00,0x00,0xc7,0x00,0x05,0x00, +0x32,0x00,0x00,0x00,0xd2,0x00,0x00,0x00,0xcf,0x00,0x00,0x00, +0xd1,0x00,0x00,0x00,0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xd3,0x00,0x00,0x00,0xd2,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, +0x14,0x00,0x00,0x00,0xd4,0x00,0x00,0x00,0xd3,0x00,0x00,0x00, +0xab,0x00,0x05,0x00,0x64,0x00,0x00,0x00,0xd5,0x00,0x00,0x00, +0xd4,0x00,0x00,0x00,0x15,0x00,0x00,0x00,0xa9,0x00,0x06,0x00, +0x14,0x00,0x00,0x00,0xd6,0x00,0x00,0x00,0xd5,0x00,0x00,0x00, +0x15,0x00,0x00,0x00,0xb2,0x00,0x00,0x00,0x82,0x00,0x05,0x00, +0x14,0x00,0x00,0x00,0xd7,0x00,0x00,0x00,0xc6,0x00,0x00,0x00, +0xd6,0x00,0x00,0x00,0x6f,0x00,0x04,0x00,0x4c,0x00,0x00,0x00, +0xd8,0x00,0x00,0x00,0xd7,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xe1,0x00,0x00,0x00,0x95,0x00,0x00,0x00, +0x41,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x97,0x00,0x00,0x00, +0xe2,0x00,0x00,0x00,0x8e,0x00,0x00,0x00,0x15,0x00,0x00,0x00, +0xe1,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x4c,0x00,0x00,0x00, +0xe3,0x00,0x00,0x00,0xe2,0x00,0x00,0x00,0x41,0x00,0x08,0x00, +0x9d,0x00,0x00,0x00,0xe7,0x00,0x00,0x00,0x77,0x00,0x00,0x00, +0x15,0x00,0x00,0x00,0x7a,0x00,0x00,0x00,0x89,0x00,0x00,0x00, +0x89,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x32,0x00,0x00,0x00, +0xe8,0x00,0x00,0x00,0xe7,0x00,0x00,0x00,0xc2,0x00,0x05,0x00, +0x32,0x00,0x00,0x00,0xea,0x00,0x00,0x00,0xe8,0x00,0x00,0x00, +0x38,0x00,0x00,0x00,0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xeb,0x00,0x00,0x00,0xea,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, +0x14,0x00,0x00,0x00,0xec,0x00,0x00,0x00,0xeb,0x00,0x00,0x00, +0xc7,0x00,0x05,0x00,0x14,0x00,0x00,0x00,0xed,0x00,0x00,0x00, +0xec,0x00,0x00,0x00,0xa4,0x00,0x00,0x00,0x41,0x00,0x08,0x00, +0x9d,0x00,0x00,0x00,0xf2,0x00,0x00,0x00,0x77,0x00,0x00,0x00, +0x15,0x00,0x00,0x00,0x7a,0x00,0x00,0x00,0x89,0x00,0x00,0x00, +0xf1,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x32,0x00,0x00,0x00, +0xf3,0x00,0x00,0x00,0xf2,0x00,0x00,0x00,0xc2,0x00,0x05,0x00, +0x32,0x00,0x00,0x00,0xf6,0x00,0x00,0x00,0xf3,0x00,0x00,0x00, +0x38,0x00,0x00,0x00,0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xf7,0x00,0x00,0x00,0xf6,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, +0x14,0x00,0x00,0x00,0xf8,0x00,0x00,0x00,0xf7,0x00,0x00,0x00, +0xc7,0x00,0x05,0x00,0x14,0x00,0x00,0x00,0xf9,0x00,0x00,0x00, +0xf8,0x00,0x00,0x00,0x7b,0x00,0x00,0x00,0xc4,0x00,0x05,0x00, +0x14,0x00,0x00,0x00,0xfa,0x00,0x00,0x00,0xf9,0x00,0x00,0x00, +0xb2,0x00,0x00,0x00,0xc5,0x00,0x05,0x00,0x14,0x00,0x00,0x00, +0xfb,0x00,0x00,0x00,0xed,0x00,0x00,0x00,0xfa,0x00,0x00,0x00, +0x72,0x00,0x04,0x00,0x3a,0x00,0x00,0x00,0xfc,0x00,0x00,0x00, +0xfb,0x00,0x00,0x00,0x72,0x00,0x04,0x00,0x14,0x00,0x00,0x00, +0xfd,0x00,0x00,0x00,0xfc,0x00,0x00,0x00,0x82,0x00,0x05,0x00, +0x14,0x00,0x00,0x00,0xfe,0x00,0x00,0x00,0xfd,0x00,0x00,0x00, +0xb7,0x00,0x00,0x00,0x6f,0x00,0x04,0x00,0x4c,0x00,0x00,0x00, +0xff,0x00,0x00,0x00,0xfe,0x00,0x00,0x00,0x85,0x00,0x05,0x00, +0x4c,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0xe3,0x00,0x00,0x00, +0xff,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x32,0x00,0x00,0x00, +0x09,0x01,0x00,0x00,0xc2,0x00,0x00,0x00,0xc2,0x00,0x05,0x00, +0x32,0x00,0x00,0x00,0x0a,0x01,0x00,0x00,0x09,0x01,0x00,0x00, +0x89,0x00,0x00,0x00,0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x0b,0x01,0x00,0x00,0x0a,0x01,0x00,0x00,0x7c,0x00,0x04,0x00, +0x14,0x00,0x00,0x00,0x0c,0x01,0x00,0x00,0x0b,0x01,0x00,0x00, +0xc7,0x00,0x05,0x00,0x14,0x00,0x00,0x00,0x0d,0x01,0x00,0x00, +0x0c,0x01,0x00,0x00,0x7b,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x32,0x00,0x00,0x00,0x16,0x01,0x00,0x00,0xce,0x00,0x00,0x00, +0xc4,0x00,0x05,0x00,0x32,0x00,0x00,0x00,0x18,0x01,0x00,0x00, +0x3c,0x00,0x00,0x00,0x35,0x00,0x00,0x00,0xc7,0x00,0x05,0x00, +0x32,0x00,0x00,0x00,0x19,0x01,0x00,0x00,0x16,0x01,0x00,0x00, +0x18,0x01,0x00,0x00,0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x1a,0x01,0x00,0x00,0x19,0x01,0x00,0x00,0x7c,0x00,0x04,0x00, +0x14,0x00,0x00,0x00,0x1b,0x01,0x00,0x00,0x1a,0x01,0x00,0x00, +0xab,0x00,0x05,0x00,0x64,0x00,0x00,0x00,0x1c,0x01,0x00,0x00, +0x1b,0x01,0x00,0x00,0x15,0x00,0x00,0x00,0xa9,0x00,0x06,0x00, +0x14,0x00,0x00,0x00,0x1d,0x01,0x00,0x00,0x1c,0x01,0x00,0x00, +0x15,0x00,0x00,0x00,0xb2,0x00,0x00,0x00,0x82,0x00,0x05,0x00, +0x14,0x00,0x00,0x00,0x1e,0x01,0x00,0x00,0x0d,0x01,0x00,0x00, +0x1d,0x01,0x00,0x00,0x6f,0x00,0x04,0x00,0x4c,0x00,0x00,0x00, +0x1f,0x01,0x00,0x00,0x1e,0x01,0x00,0x00,0x85,0x00,0x05,0x00, +0x4c,0x00,0x00,0x00,0x20,0x01,0x00,0x00,0x00,0x01,0x00,0x00, +0x1f,0x01,0x00,0x00,0x0c,0x00,0x08,0x00,0x4c,0x00,0x00,0x00, +0x21,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00, +0xba,0x00,0x00,0x00,0xd8,0x00,0x00,0x00,0x20,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x29,0x01,0x00,0x00, +0x95,0x00,0x00,0x00,0x6e,0x00,0x00,0x00,0x41,0x00,0x06,0x00, +0x97,0x00,0x00,0x00,0x2a,0x01,0x00,0x00,0x8e,0x00,0x00,0x00, +0x15,0x00,0x00,0x00,0x29,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, +0x4c,0x00,0x00,0x00,0x2b,0x01,0x00,0x00,0x2a,0x01,0x00,0x00, +0x41,0x00,0x08,0x00,0x9d,0x00,0x00,0x00,0x2f,0x01,0x00,0x00, +0x77,0x00,0x00,0x00,0x15,0x00,0x00,0x00,0x7a,0x00,0x00,0x00, +0x89,0x00,0x00,0x00,0xb2,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x32,0x00,0x00,0x00,0x30,0x01,0x00,0x00,0x2f,0x01,0x00,0x00, +0xc2,0x00,0x05,0x00,0x32,0x00,0x00,0x00,0x32,0x01,0x00,0x00, +0x30,0x01,0x00,0x00,0x38,0x00,0x00,0x00,0x71,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x33,0x01,0x00,0x00,0x32,0x01,0x00,0x00, +0x7c,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0x34,0x01,0x00,0x00, +0x33,0x01,0x00,0x00,0xc7,0x00,0x05,0x00,0x14,0x00,0x00,0x00, +0x35,0x01,0x00,0x00,0x34,0x01,0x00,0x00,0xa4,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x32,0x00,0x00,0x00,0x3a,0x01,0x00,0x00, +0xaa,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x3c,0x01,0x00,0x00,0x38,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0xc2,0x00,0x05,0x00,0x32,0x00,0x00,0x00,0x3d,0x01,0x00,0x00, +0x3a,0x01,0x00,0x00,0x3c,0x01,0x00,0x00,0x71,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x3e,0x01,0x00,0x00,0x3d,0x01,0x00,0x00, +0x7c,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0x3f,0x01,0x00,0x00, +0x3e,0x01,0x00,0x00,0xc7,0x00,0x05,0x00,0x14,0x00,0x00,0x00, +0x40,0x01,0x00,0x00,0x3f,0x01,0x00,0x00,0x7b,0x00,0x00,0x00, +0xc4,0x00,0x05,0x00,0x14,0x00,0x00,0x00,0x41,0x01,0x00,0x00, +0x40,0x01,0x00,0x00,0xb2,0x00,0x00,0x00,0xc5,0x00,0x05,0x00, +0x14,0x00,0x00,0x00,0x42,0x01,0x00,0x00,0x35,0x01,0x00,0x00, +0x41,0x01,0x00,0x00,0x72,0x00,0x04,0x00,0x3a,0x00,0x00,0x00, +0x43,0x01,0x00,0x00,0x42,0x01,0x00,0x00,0x72,0x00,0x04,0x00, +0x14,0x00,0x00,0x00,0x44,0x01,0x00,0x00,0x43,0x01,0x00,0x00, +0x82,0x00,0x05,0x00,0x14,0x00,0x00,0x00,0x45,0x01,0x00,0x00, +0x44,0x01,0x00,0x00,0xb7,0x00,0x00,0x00,0x6f,0x00,0x04,0x00, +0x4c,0x00,0x00,0x00,0x46,0x01,0x00,0x00,0x45,0x01,0x00,0x00, +0x85,0x00,0x05,0x00,0x4c,0x00,0x00,0x00,0x47,0x01,0x00,0x00, +0x2b,0x01,0x00,0x00,0x46,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, +0x32,0x00,0x00,0x00,0x50,0x01,0x00,0x00,0xc2,0x00,0x00,0x00, +0xc2,0x00,0x05,0x00,0x32,0x00,0x00,0x00,0x51,0x01,0x00,0x00, +0x50,0x01,0x00,0x00,0xb2,0x00,0x00,0x00,0x71,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x52,0x01,0x00,0x00,0x51,0x01,0x00,0x00, +0x7c,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0x53,0x01,0x00,0x00, +0x52,0x01,0x00,0x00,0xc7,0x00,0x05,0x00,0x14,0x00,0x00,0x00, +0x54,0x01,0x00,0x00,0x53,0x01,0x00,0x00,0x7b,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x32,0x00,0x00,0x00,0x5d,0x01,0x00,0x00, +0xce,0x00,0x00,0x00,0xc4,0x00,0x05,0x00,0x32,0x00,0x00,0x00, +0x5f,0x01,0x00,0x00,0x3c,0x00,0x00,0x00,0x89,0x00,0x00,0x00, +0xc7,0x00,0x05,0x00,0x32,0x00,0x00,0x00,0x60,0x01,0x00,0x00, +0x5d,0x01,0x00,0x00,0x5f,0x01,0x00,0x00,0x71,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x61,0x01,0x00,0x00,0x60,0x01,0x00,0x00, +0x7c,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0x62,0x01,0x00,0x00, +0x61,0x01,0x00,0x00,0xab,0x00,0x05,0x00,0x64,0x00,0x00,0x00, +0x63,0x01,0x00,0x00,0x62,0x01,0x00,0x00,0x15,0x00,0x00,0x00, +0xa9,0x00,0x06,0x00,0x14,0x00,0x00,0x00,0x64,0x01,0x00,0x00, +0x63,0x01,0x00,0x00,0x15,0x00,0x00,0x00,0xb2,0x00,0x00,0x00, +0x82,0x00,0x05,0x00,0x14,0x00,0x00,0x00,0x65,0x01,0x00,0x00, +0x54,0x01,0x00,0x00,0x64,0x01,0x00,0x00,0x6f,0x00,0x04,0x00, +0x4c,0x00,0x00,0x00,0x66,0x01,0x00,0x00,0x65,0x01,0x00,0x00, +0x0c,0x00,0x08,0x00,0x4c,0x00,0x00,0x00,0x68,0x01,0x00,0x00, +0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x47,0x01,0x00,0x00, +0x66,0x01,0x00,0x00,0x21,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x71,0x01,0x00,0x00,0x95,0x00,0x00,0x00, +0x70,0x01,0x00,0x00,0x41,0x00,0x06,0x00,0x97,0x00,0x00,0x00, +0x72,0x01,0x00,0x00,0x8e,0x00,0x00,0x00,0x15,0x00,0x00,0x00, +0x71,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0x4c,0x00,0x00,0x00, +0x73,0x01,0x00,0x00,0x72,0x01,0x00,0x00,0x41,0x00,0x08,0x00, +0x9d,0x00,0x00,0x00,0x78,0x01,0x00,0x00,0x77,0x00,0x00,0x00, +0x15,0x00,0x00,0x00,0x7a,0x00,0x00,0x00,0x89,0x00,0x00,0x00, +0x77,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0x32,0x00,0x00,0x00, +0x79,0x01,0x00,0x00,0x78,0x01,0x00,0x00,0xc2,0x00,0x05,0x00, +0x32,0x00,0x00,0x00,0x7b,0x01,0x00,0x00,0x79,0x01,0x00,0x00, +0x38,0x00,0x00,0x00,0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x7c,0x01,0x00,0x00,0x7b,0x01,0x00,0x00,0x7c,0x00,0x04,0x00, +0x14,0x00,0x00,0x00,0x7d,0x01,0x00,0x00,0x7c,0x01,0x00,0x00, +0xc7,0x00,0x05,0x00,0x14,0x00,0x00,0x00,0x7e,0x01,0x00,0x00, +0x7d,0x01,0x00,0x00,0xa4,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x32,0x00,0x00,0x00,0x83,0x01,0x00,0x00,0xf2,0x00,0x00,0x00, +0xc2,0x00,0x05,0x00,0x32,0x00,0x00,0x00,0x86,0x01,0x00,0x00, +0x83,0x01,0x00,0x00,0x3c,0x01,0x00,0x00,0x71,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x87,0x01,0x00,0x00,0x86,0x01,0x00,0x00, +0x7c,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0x88,0x01,0x00,0x00, +0x87,0x01,0x00,0x00,0xc7,0x00,0x05,0x00,0x14,0x00,0x00,0x00, +0x89,0x01,0x00,0x00,0x88,0x01,0x00,0x00,0x7b,0x00,0x00,0x00, +0xc4,0x00,0x05,0x00,0x14,0x00,0x00,0x00,0x8a,0x01,0x00,0x00, +0x89,0x01,0x00,0x00,0xb2,0x00,0x00,0x00,0xc5,0x00,0x05,0x00, +0x14,0x00,0x00,0x00,0x8b,0x01,0x00,0x00,0x7e,0x01,0x00,0x00, +0x8a,0x01,0x00,0x00,0x72,0x00,0x04,0x00,0x3a,0x00,0x00,0x00, +0x8c,0x01,0x00,0x00,0x8b,0x01,0x00,0x00,0x72,0x00,0x04,0x00, +0x14,0x00,0x00,0x00,0x8d,0x01,0x00,0x00,0x8c,0x01,0x00,0x00, +0x82,0x00,0x05,0x00,0x14,0x00,0x00,0x00,0x8e,0x01,0x00,0x00, +0x8d,0x01,0x00,0x00,0xb7,0x00,0x00,0x00,0x6f,0x00,0x04,0x00, +0x4c,0x00,0x00,0x00,0x8f,0x01,0x00,0x00,0x8e,0x01,0x00,0x00, +0x85,0x00,0x05,0x00,0x4c,0x00,0x00,0x00,0x90,0x01,0x00,0x00, +0x73,0x01,0x00,0x00,0x8f,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, +0x32,0x00,0x00,0x00,0x99,0x01,0x00,0x00,0xc2,0x00,0x00,0x00, +0xc2,0x00,0x05,0x00,0x32,0x00,0x00,0x00,0x9a,0x01,0x00,0x00, +0x99,0x01,0x00,0x00,0x77,0x01,0x00,0x00,0x71,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x9b,0x01,0x00,0x00,0x9a,0x01,0x00,0x00, +0x7c,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0x9c,0x01,0x00,0x00, +0x9b,0x01,0x00,0x00,0xc7,0x00,0x05,0x00,0x14,0x00,0x00,0x00, +0x9d,0x01,0x00,0x00,0x9c,0x01,0x00,0x00,0x7b,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x32,0x00,0x00,0x00,0xa6,0x01,0x00,0x00, +0xce,0x00,0x00,0x00,0xc4,0x00,0x05,0x00,0x32,0x00,0x00,0x00, +0xa8,0x01,0x00,0x00,0x3c,0x00,0x00,0x00,0x7b,0x00,0x00,0x00, +0xc7,0x00,0x05,0x00,0x32,0x00,0x00,0x00,0xa9,0x01,0x00,0x00, +0xa6,0x01,0x00,0x00,0xa8,0x01,0x00,0x00,0x71,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xaa,0x01,0x00,0x00,0xa9,0x01,0x00,0x00, +0x7c,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0xab,0x01,0x00,0x00, +0xaa,0x01,0x00,0x00,0xab,0x00,0x05,0x00,0x64,0x00,0x00,0x00, +0xac,0x01,0x00,0x00,0xab,0x01,0x00,0x00,0x15,0x00,0x00,0x00, +0xa9,0x00,0x06,0x00,0x14,0x00,0x00,0x00,0xad,0x01,0x00,0x00, +0xac,0x01,0x00,0x00,0x15,0x00,0x00,0x00,0xb2,0x00,0x00,0x00, +0x82,0x00,0x05,0x00,0x14,0x00,0x00,0x00,0xae,0x01,0x00,0x00, +0x9d,0x01,0x00,0x00,0xad,0x01,0x00,0x00,0x6f,0x00,0x04,0x00, +0x4c,0x00,0x00,0x00,0xaf,0x01,0x00,0x00,0xae,0x01,0x00,0x00, +0x0c,0x00,0x08,0x00,0x4c,0x00,0x00,0x00,0xb1,0x01,0x00,0x00, +0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x90,0x01,0x00,0x00, +0xaf,0x01,0x00,0x00,0x68,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xb9,0x01,0x00,0x00,0x95,0x00,0x00,0x00, +0x50,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x97,0x00,0x00,0x00, +0xba,0x01,0x00,0x00,0x8e,0x00,0x00,0x00,0x15,0x00,0x00,0x00, +0xb9,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0x4c,0x00,0x00,0x00, +0xbb,0x01,0x00,0x00,0xba,0x01,0x00,0x00,0x41,0x00,0x08,0x00, +0x9d,0x00,0x00,0x00,0xbf,0x01,0x00,0x00,0x77,0x00,0x00,0x00, +0x15,0x00,0x00,0x00,0x7a,0x00,0x00,0x00,0x89,0x00,0x00,0x00, +0x35,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x32,0x00,0x00,0x00, +0xc0,0x01,0x00,0x00,0xbf,0x01,0x00,0x00,0xc2,0x00,0x05,0x00, +0x32,0x00,0x00,0x00,0xc2,0x01,0x00,0x00,0xc0,0x01,0x00,0x00, +0x38,0x00,0x00,0x00,0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xc3,0x01,0x00,0x00,0xc2,0x01,0x00,0x00,0x7c,0x00,0x04,0x00, +0x14,0x00,0x00,0x00,0xc4,0x01,0x00,0x00,0xc3,0x01,0x00,0x00, +0xc7,0x00,0x05,0x00,0x14,0x00,0x00,0x00,0xc5,0x01,0x00,0x00, +0xc4,0x01,0x00,0x00,0xa4,0x00,0x00,0x00,0x41,0x00,0x08,0x00, +0x9d,0x00,0x00,0x00,0xca,0x01,0x00,0x00,0x77,0x00,0x00,0x00, +0x15,0x00,0x00,0x00,0x7a,0x00,0x00,0x00,0x89,0x00,0x00,0x00, +0xc9,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0x32,0x00,0x00,0x00, +0xcb,0x01,0x00,0x00,0xca,0x01,0x00,0x00,0xc2,0x00,0x05,0x00, +0x32,0x00,0x00,0x00,0xce,0x01,0x00,0x00,0xcb,0x01,0x00,0x00, +0x38,0x00,0x00,0x00,0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xcf,0x01,0x00,0x00,0xce,0x01,0x00,0x00,0x7c,0x00,0x04,0x00, +0x14,0x00,0x00,0x00,0xd0,0x01,0x00,0x00,0xcf,0x01,0x00,0x00, +0xc7,0x00,0x05,0x00,0x14,0x00,0x00,0x00,0xd1,0x01,0x00,0x00, +0xd0,0x01,0x00,0x00,0x7b,0x00,0x00,0x00,0xc4,0x00,0x05,0x00, +0x14,0x00,0x00,0x00,0xd2,0x01,0x00,0x00,0xd1,0x01,0x00,0x00, +0xb2,0x00,0x00,0x00,0xc5,0x00,0x05,0x00,0x14,0x00,0x00,0x00, +0xd3,0x01,0x00,0x00,0xc5,0x01,0x00,0x00,0xd2,0x01,0x00,0x00, +0x72,0x00,0x04,0x00,0x3a,0x00,0x00,0x00,0xd4,0x01,0x00,0x00, +0xd3,0x01,0x00,0x00,0x72,0x00,0x04,0x00,0x14,0x00,0x00,0x00, +0xd5,0x01,0x00,0x00,0xd4,0x01,0x00,0x00,0x82,0x00,0x05,0x00, +0x14,0x00,0x00,0x00,0xd6,0x01,0x00,0x00,0xd5,0x01,0x00,0x00, +0xb7,0x00,0x00,0x00,0x6f,0x00,0x04,0x00,0x4c,0x00,0x00,0x00, +0xd7,0x01,0x00,0x00,0xd6,0x01,0x00,0x00,0x85,0x00,0x05,0x00, +0x4c,0x00,0x00,0x00,0xd8,0x01,0x00,0x00,0xbb,0x01,0x00,0x00, +0xd7,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xe0,0x01,0x00,0x00,0xc1,0x00,0x00,0x00,0x50,0x00,0x00,0x00, +0x41,0x00,0x08,0x00,0x9d,0x00,0x00,0x00,0xe1,0x01,0x00,0x00, +0x77,0x00,0x00,0x00,0x15,0x00,0x00,0x00,0x7a,0x00,0x00,0x00, +0x35,0x00,0x00,0x00,0xe0,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, +0x32,0x00,0x00,0x00,0xe2,0x01,0x00,0x00,0xe1,0x01,0x00,0x00, +0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xe3,0x01,0x00,0x00, +0xe2,0x01,0x00,0x00,0x7c,0x00,0x04,0x00,0x14,0x00,0x00,0x00, +0xe4,0x01,0x00,0x00,0xe3,0x01,0x00,0x00,0xc7,0x00,0x05,0x00, +0x14,0x00,0x00,0x00,0xe5,0x01,0x00,0x00,0xe4,0x01,0x00,0x00, +0x7b,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xed,0x01,0x00,0x00,0xcd,0x00,0x00,0x00,0x50,0x00,0x00,0x00, +0x41,0x00,0x08,0x00,0x9d,0x00,0x00,0x00,0xee,0x01,0x00,0x00, +0x77,0x00,0x00,0x00,0x15,0x00,0x00,0x00,0x7a,0x00,0x00,0x00, +0x15,0x00,0x00,0x00,0xed,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, +0x32,0x00,0x00,0x00,0xef,0x01,0x00,0x00,0xee,0x01,0x00,0x00, +0xc7,0x00,0x05,0x00,0x32,0x00,0x00,0x00,0xf2,0x01,0x00,0x00, +0xef,0x01,0x00,0x00,0xd1,0x00,0x00,0x00,0x71,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xf3,0x01,0x00,0x00,0xf2,0x01,0x00,0x00, +0x7c,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0xf4,0x01,0x00,0x00, +0xf3,0x01,0x00,0x00,0xab,0x00,0x05,0x00,0x64,0x00,0x00,0x00, +0xf5,0x01,0x00,0x00,0xf4,0x01,0x00,0x00,0x15,0x00,0x00,0x00, +0xa9,0x00,0x06,0x00,0x14,0x00,0x00,0x00,0xf6,0x01,0x00,0x00, +0xf5,0x01,0x00,0x00,0x15,0x00,0x00,0x00,0xb2,0x00,0x00,0x00, +0x82,0x00,0x05,0x00,0x14,0x00,0x00,0x00,0xf7,0x01,0x00,0x00, +0xe5,0x01,0x00,0x00,0xf6,0x01,0x00,0x00,0x6f,0x00,0x04,0x00, +0x4c,0x00,0x00,0x00,0xf8,0x01,0x00,0x00,0xf7,0x01,0x00,0x00, +0x0c,0x00,0x08,0x00,0x4c,0x00,0x00,0x00,0xfa,0x01,0x00,0x00, +0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0xd8,0x01,0x00,0x00, +0xf8,0x01,0x00,0x00,0xb1,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x03,0x02,0x00,0x00,0x95,0x00,0x00,0x00, +0x02,0x02,0x00,0x00,0x41,0x00,0x06,0x00,0x97,0x00,0x00,0x00, +0x04,0x02,0x00,0x00,0x8e,0x00,0x00,0x00,0x15,0x00,0x00,0x00, +0x03,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0x4c,0x00,0x00,0x00, +0x05,0x02,0x00,0x00,0x04,0x02,0x00,0x00,0x41,0x00,0x08,0x00, +0x9d,0x00,0x00,0x00,0x09,0x02,0x00,0x00,0x77,0x00,0x00,0x00, +0x15,0x00,0x00,0x00,0x7a,0x00,0x00,0x00,0x89,0x00,0x00,0x00, +0x7b,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x32,0x00,0x00,0x00, +0x0a,0x02,0x00,0x00,0x09,0x02,0x00,0x00,0xc2,0x00,0x05,0x00, +0x32,0x00,0x00,0x00,0x0c,0x02,0x00,0x00,0x0a,0x02,0x00,0x00, +0x38,0x00,0x00,0x00,0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x0d,0x02,0x00,0x00,0x0c,0x02,0x00,0x00,0x7c,0x00,0x04,0x00, +0x14,0x00,0x00,0x00,0x0e,0x02,0x00,0x00,0x0d,0x02,0x00,0x00, +0xc7,0x00,0x05,0x00,0x14,0x00,0x00,0x00,0x0f,0x02,0x00,0x00, +0x0e,0x02,0x00,0x00,0xa4,0x00,0x00,0x00,0x41,0x00,0x08,0x00, +0x9d,0x00,0x00,0x00,0x14,0x02,0x00,0x00,0x77,0x00,0x00,0x00, +0x15,0x00,0x00,0x00,0x7a,0x00,0x00,0x00,0x89,0x00,0x00,0x00, +0x13,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0x32,0x00,0x00,0x00, +0x15,0x02,0x00,0x00,0x14,0x02,0x00,0x00,0xc2,0x00,0x05,0x00, +0x32,0x00,0x00,0x00,0x18,0x02,0x00,0x00,0x15,0x02,0x00,0x00, +0x38,0x00,0x00,0x00,0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x19,0x02,0x00,0x00,0x18,0x02,0x00,0x00,0x7c,0x00,0x04,0x00, +0x14,0x00,0x00,0x00,0x1a,0x02,0x00,0x00,0x19,0x02,0x00,0x00, +0xc7,0x00,0x05,0x00,0x14,0x00,0x00,0x00,0x1b,0x02,0x00,0x00, +0x1a,0x02,0x00,0x00,0x7b,0x00,0x00,0x00,0xc4,0x00,0x05,0x00, +0x14,0x00,0x00,0x00,0x1c,0x02,0x00,0x00,0x1b,0x02,0x00,0x00, +0xb2,0x00,0x00,0x00,0xc5,0x00,0x05,0x00,0x14,0x00,0x00,0x00, +0x1d,0x02,0x00,0x00,0x0f,0x02,0x00,0x00,0x1c,0x02,0x00,0x00, +0x72,0x00,0x04,0x00,0x3a,0x00,0x00,0x00,0x1e,0x02,0x00,0x00, +0x1d,0x02,0x00,0x00,0x72,0x00,0x04,0x00,0x14,0x00,0x00,0x00, +0x1f,0x02,0x00,0x00,0x1e,0x02,0x00,0x00,0x82,0x00,0x05,0x00, +0x14,0x00,0x00,0x00,0x20,0x02,0x00,0x00,0x1f,0x02,0x00,0x00, +0xb7,0x00,0x00,0x00,0x6f,0x00,0x04,0x00,0x4c,0x00,0x00,0x00, +0x21,0x02,0x00,0x00,0x20,0x02,0x00,0x00,0x85,0x00,0x05,0x00, +0x4c,0x00,0x00,0x00,0x22,0x02,0x00,0x00,0x05,0x02,0x00,0x00, +0x21,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0x32,0x00,0x00,0x00, +0x2c,0x02,0x00,0x00,0xe1,0x01,0x00,0x00,0xc2,0x00,0x05,0x00, +0x32,0x00,0x00,0x00,0x2d,0x02,0x00,0x00,0x2c,0x02,0x00,0x00, +0x89,0x00,0x00,0x00,0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x2e,0x02,0x00,0x00,0x2d,0x02,0x00,0x00,0x7c,0x00,0x04,0x00, +0x14,0x00,0x00,0x00,0x2f,0x02,0x00,0x00,0x2e,0x02,0x00,0x00, +0xc7,0x00,0x05,0x00,0x14,0x00,0x00,0x00,0x30,0x02,0x00,0x00, +0x2f,0x02,0x00,0x00,0x7b,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x32,0x00,0x00,0x00,0x3a,0x02,0x00,0x00,0xee,0x01,0x00,0x00, +0xc7,0x00,0x05,0x00,0x32,0x00,0x00,0x00,0x3d,0x02,0x00,0x00, +0x3a,0x02,0x00,0x00,0x18,0x01,0x00,0x00,0x71,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x3e,0x02,0x00,0x00,0x3d,0x02,0x00,0x00, +0x7c,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0x3f,0x02,0x00,0x00, +0x3e,0x02,0x00,0x00,0xab,0x00,0x05,0x00,0x64,0x00,0x00,0x00, +0x40,0x02,0x00,0x00,0x3f,0x02,0x00,0x00,0x15,0x00,0x00,0x00, +0xa9,0x00,0x06,0x00,0x14,0x00,0x00,0x00,0x41,0x02,0x00,0x00, +0x40,0x02,0x00,0x00,0x15,0x00,0x00,0x00,0xb2,0x00,0x00,0x00, +0x82,0x00,0x05,0x00,0x14,0x00,0x00,0x00,0x42,0x02,0x00,0x00, +0x30,0x02,0x00,0x00,0x41,0x02,0x00,0x00,0x6f,0x00,0x04,0x00, +0x4c,0x00,0x00,0x00,0x43,0x02,0x00,0x00,0x42,0x02,0x00,0x00, +0x0c,0x00,0x08,0x00,0x4c,0x00,0x00,0x00,0x45,0x02,0x00,0x00, +0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x22,0x02,0x00,0x00, +0x43,0x02,0x00,0x00,0xfa,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x4e,0x02,0x00,0x00,0x95,0x00,0x00,0x00, +0x4d,0x02,0x00,0x00,0x41,0x00,0x06,0x00,0x97,0x00,0x00,0x00, +0x4f,0x02,0x00,0x00,0x8e,0x00,0x00,0x00,0x15,0x00,0x00,0x00, +0x4e,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0x4c,0x00,0x00,0x00, +0x50,0x02,0x00,0x00,0x4f,0x02,0x00,0x00,0x41,0x00,0x08,0x00, +0x9d,0x00,0x00,0x00,0x55,0x02,0x00,0x00,0x77,0x00,0x00,0x00, +0x15,0x00,0x00,0x00,0x7a,0x00,0x00,0x00,0x89,0x00,0x00,0x00, +0x54,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0x32,0x00,0x00,0x00, +0x56,0x02,0x00,0x00,0x55,0x02,0x00,0x00,0xc2,0x00,0x05,0x00, +0x32,0x00,0x00,0x00,0x58,0x02,0x00,0x00,0x56,0x02,0x00,0x00, +0x38,0x00,0x00,0x00,0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x59,0x02,0x00,0x00,0x58,0x02,0x00,0x00,0x7c,0x00,0x04,0x00, +0x14,0x00,0x00,0x00,0x5a,0x02,0x00,0x00,0x59,0x02,0x00,0x00, +0xc7,0x00,0x05,0x00,0x14,0x00,0x00,0x00,0x5b,0x02,0x00,0x00, +0x5a,0x02,0x00,0x00,0xa4,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x32,0x00,0x00,0x00,0x60,0x02,0x00,0x00,0xca,0x01,0x00,0x00, +0xc2,0x00,0x05,0x00,0x32,0x00,0x00,0x00,0x63,0x02,0x00,0x00, +0x60,0x02,0x00,0x00,0x3c,0x01,0x00,0x00,0x71,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x64,0x02,0x00,0x00,0x63,0x02,0x00,0x00, +0x7c,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0x65,0x02,0x00,0x00, +0x64,0x02,0x00,0x00,0xc7,0x00,0x05,0x00,0x14,0x00,0x00,0x00, +0x66,0x02,0x00,0x00,0x65,0x02,0x00,0x00,0x7b,0x00,0x00,0x00, +0xc4,0x00,0x05,0x00,0x14,0x00,0x00,0x00,0x67,0x02,0x00,0x00, +0x66,0x02,0x00,0x00,0xb2,0x00,0x00,0x00,0xc5,0x00,0x05,0x00, +0x14,0x00,0x00,0x00,0x68,0x02,0x00,0x00,0x5b,0x02,0x00,0x00, +0x67,0x02,0x00,0x00,0x72,0x00,0x04,0x00,0x3a,0x00,0x00,0x00, +0x69,0x02,0x00,0x00,0x68,0x02,0x00,0x00,0x72,0x00,0x04,0x00, +0x14,0x00,0x00,0x00,0x6a,0x02,0x00,0x00,0x69,0x02,0x00,0x00, +0x82,0x00,0x05,0x00,0x14,0x00,0x00,0x00,0x6b,0x02,0x00,0x00, +0x6a,0x02,0x00,0x00,0xb7,0x00,0x00,0x00,0x6f,0x00,0x04,0x00, +0x4c,0x00,0x00,0x00,0x6c,0x02,0x00,0x00,0x6b,0x02,0x00,0x00, +0x85,0x00,0x05,0x00,0x4c,0x00,0x00,0x00,0x6d,0x02,0x00,0x00, +0x50,0x02,0x00,0x00,0x6c,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, +0x32,0x00,0x00,0x00,0x77,0x02,0x00,0x00,0xe1,0x01,0x00,0x00, +0xc2,0x00,0x05,0x00,0x32,0x00,0x00,0x00,0x78,0x02,0x00,0x00, +0x77,0x02,0x00,0x00,0xb2,0x00,0x00,0x00,0x71,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x79,0x02,0x00,0x00,0x78,0x02,0x00,0x00, +0x7c,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0x7a,0x02,0x00,0x00, +0x79,0x02,0x00,0x00,0xc7,0x00,0x05,0x00,0x14,0x00,0x00,0x00, +0x7b,0x02,0x00,0x00,0x7a,0x02,0x00,0x00,0x7b,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x32,0x00,0x00,0x00,0x85,0x02,0x00,0x00, +0xee,0x01,0x00,0x00,0xc7,0x00,0x05,0x00,0x32,0x00,0x00,0x00, +0x88,0x02,0x00,0x00,0x85,0x02,0x00,0x00,0x5f,0x01,0x00,0x00, +0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x89,0x02,0x00,0x00, +0x88,0x02,0x00,0x00,0x7c,0x00,0x04,0x00,0x14,0x00,0x00,0x00, +0x8a,0x02,0x00,0x00,0x89,0x02,0x00,0x00,0xab,0x00,0x05,0x00, +0x64,0x00,0x00,0x00,0x8b,0x02,0x00,0x00,0x8a,0x02,0x00,0x00, +0x15,0x00,0x00,0x00,0xa9,0x00,0x06,0x00,0x14,0x00,0x00,0x00, +0x8c,0x02,0x00,0x00,0x8b,0x02,0x00,0x00,0x15,0x00,0x00,0x00, +0xb2,0x00,0x00,0x00,0x82,0x00,0x05,0x00,0x14,0x00,0x00,0x00, +0x8d,0x02,0x00,0x00,0x7b,0x02,0x00,0x00,0x8c,0x02,0x00,0x00, +0x6f,0x00,0x04,0x00,0x4c,0x00,0x00,0x00,0x8e,0x02,0x00,0x00, +0x8d,0x02,0x00,0x00,0x0c,0x00,0x08,0x00,0x4c,0x00,0x00,0x00, +0x90,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00, +0x6d,0x02,0x00,0x00,0x8e,0x02,0x00,0x00,0x45,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x99,0x02,0x00,0x00, +0x95,0x00,0x00,0x00,0x98,0x02,0x00,0x00,0x41,0x00,0x06,0x00, +0x97,0x00,0x00,0x00,0x9a,0x02,0x00,0x00,0x8e,0x00,0x00,0x00, +0x15,0x00,0x00,0x00,0x99,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, +0x4c,0x00,0x00,0x00,0x9b,0x02,0x00,0x00,0x9a,0x02,0x00,0x00, +0x41,0x00,0x08,0x00,0x9d,0x00,0x00,0x00,0xa0,0x02,0x00,0x00, +0x77,0x00,0x00,0x00,0x15,0x00,0x00,0x00,0x7a,0x00,0x00,0x00, +0x89,0x00,0x00,0x00,0x9f,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, +0x32,0x00,0x00,0x00,0xa1,0x02,0x00,0x00,0xa0,0x02,0x00,0x00, +0xc2,0x00,0x05,0x00,0x32,0x00,0x00,0x00,0xa3,0x02,0x00,0x00, +0xa1,0x02,0x00,0x00,0x38,0x00,0x00,0x00,0x71,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xa4,0x02,0x00,0x00,0xa3,0x02,0x00,0x00, +0x7c,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0xa5,0x02,0x00,0x00, +0xa4,0x02,0x00,0x00,0xc7,0x00,0x05,0x00,0x14,0x00,0x00,0x00, +0xa6,0x02,0x00,0x00,0xa5,0x02,0x00,0x00,0xa4,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x32,0x00,0x00,0x00,0xab,0x02,0x00,0x00, +0x14,0x02,0x00,0x00,0xc2,0x00,0x05,0x00,0x32,0x00,0x00,0x00, +0xae,0x02,0x00,0x00,0xab,0x02,0x00,0x00,0x3c,0x01,0x00,0x00, +0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xaf,0x02,0x00,0x00, +0xae,0x02,0x00,0x00,0x7c,0x00,0x04,0x00,0x14,0x00,0x00,0x00, +0xb0,0x02,0x00,0x00,0xaf,0x02,0x00,0x00,0xc7,0x00,0x05,0x00, +0x14,0x00,0x00,0x00,0xb1,0x02,0x00,0x00,0xb0,0x02,0x00,0x00, +0x7b,0x00,0x00,0x00,0xc4,0x00,0x05,0x00,0x14,0x00,0x00,0x00, +0xb2,0x02,0x00,0x00,0xb1,0x02,0x00,0x00,0xb2,0x00,0x00,0x00, +0xc5,0x00,0x05,0x00,0x14,0x00,0x00,0x00,0xb3,0x02,0x00,0x00, +0xa6,0x02,0x00,0x00,0xb2,0x02,0x00,0x00,0x72,0x00,0x04,0x00, +0x3a,0x00,0x00,0x00,0xb4,0x02,0x00,0x00,0xb3,0x02,0x00,0x00, +0x72,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0xb5,0x02,0x00,0x00, +0xb4,0x02,0x00,0x00,0x82,0x00,0x05,0x00,0x14,0x00,0x00,0x00, +0xb6,0x02,0x00,0x00,0xb5,0x02,0x00,0x00,0xb7,0x00,0x00,0x00, +0x6f,0x00,0x04,0x00,0x4c,0x00,0x00,0x00,0xb7,0x02,0x00,0x00, +0xb6,0x02,0x00,0x00,0x85,0x00,0x05,0x00,0x4c,0x00,0x00,0x00, +0xb8,0x02,0x00,0x00,0x9b,0x02,0x00,0x00,0xb7,0x02,0x00,0x00, +0x3d,0x00,0x04,0x00,0x32,0x00,0x00,0x00,0xc2,0x02,0x00,0x00, +0xe1,0x01,0x00,0x00,0xc2,0x00,0x05,0x00,0x32,0x00,0x00,0x00, +0xc3,0x02,0x00,0x00,0xc2,0x02,0x00,0x00,0x77,0x01,0x00,0x00, +0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xc4,0x02,0x00,0x00, +0xc3,0x02,0x00,0x00,0x7c,0x00,0x04,0x00,0x14,0x00,0x00,0x00, +0xc5,0x02,0x00,0x00,0xc4,0x02,0x00,0x00,0xc7,0x00,0x05,0x00, +0x14,0x00,0x00,0x00,0xc6,0x02,0x00,0x00,0xc5,0x02,0x00,0x00, +0x7b,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x32,0x00,0x00,0x00, +0xd0,0x02,0x00,0x00,0xee,0x01,0x00,0x00,0xc7,0x00,0x05,0x00, +0x32,0x00,0x00,0x00,0xd3,0x02,0x00,0x00,0xd0,0x02,0x00,0x00, +0xa8,0x01,0x00,0x00,0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xd4,0x02,0x00,0x00,0xd3,0x02,0x00,0x00,0x7c,0x00,0x04,0x00, +0x14,0x00,0x00,0x00,0xd5,0x02,0x00,0x00,0xd4,0x02,0x00,0x00, +0xab,0x00,0x05,0x00,0x64,0x00,0x00,0x00,0xd6,0x02,0x00,0x00, +0xd5,0x02,0x00,0x00,0x15,0x00,0x00,0x00,0xa9,0x00,0x06,0x00, +0x14,0x00,0x00,0x00,0xd7,0x02,0x00,0x00,0xd6,0x02,0x00,0x00, +0x15,0x00,0x00,0x00,0xb2,0x00,0x00,0x00,0x82,0x00,0x05,0x00, +0x14,0x00,0x00,0x00,0xd8,0x02,0x00,0x00,0xc6,0x02,0x00,0x00, +0xd7,0x02,0x00,0x00,0x6f,0x00,0x04,0x00,0x4c,0x00,0x00,0x00, +0xd9,0x02,0x00,0x00,0xd8,0x02,0x00,0x00,0x0c,0x00,0x08,0x00, +0x4c,0x00,0x00,0x00,0xdb,0x02,0x00,0x00,0x01,0x00,0x00,0x00, +0x32,0x00,0x00,0x00,0xb8,0x02,0x00,0x00,0xd9,0x02,0x00,0x00, +0x90,0x02,0x00,0x00,0x81,0x00,0x05,0x00,0x4c,0x00,0x00,0x00, +0xdd,0x02,0x00,0x00,0x1c,0x03,0x00,0x00,0xdb,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x14,0x00,0x00,0x00,0xdf,0x02,0x00,0x00, +0x1b,0x03,0x00,0x00,0x35,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x83,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x85,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x4c,0x00,0x00,0x00,0xe8,0x02,0x00,0x00, +0x57,0x00,0x00,0x00,0x0c,0x00,0x08,0x00,0x4c,0x00,0x00,0x00, +0xe9,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00, +0x7f,0x00,0x00,0x00,0x1c,0x03,0x00,0x00,0xe8,0x02,0x00,0x00, +0x3e,0x00,0x03,0x00,0x57,0x00,0x00,0x00,0xe9,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0x60,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0x60,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xec,0x02,0x00,0x00,0x18,0x03,0x00,0x00,0x23,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x5d,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0x5f,0x00,0x00,0x00,0xe0,0x00,0x04,0x00,0x23,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0xed,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0xef,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0xef,0x02,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x19,0x03,0x00,0x00, +0x50,0x00,0x00,0x00,0x5f,0x00,0x00,0x00,0x06,0x03,0x00,0x00, +0xf2,0x02,0x00,0x00,0xac,0x00,0x05,0x00,0x64,0x00,0x00,0x00, +0xf5,0x02,0x00,0x00,0x19,0x03,0x00,0x00,0x0c,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0xf1,0x02,0x00,0x00,0xf2,0x02,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xf5,0x02,0x00,0x00, +0xf0,0x02,0x00,0x00,0xf1,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0xf0,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0x64,0x00,0x00,0x00, +0xf8,0x02,0x00,0x00,0x24,0x00,0x00,0x00,0x19,0x03,0x00,0x00, +0xf7,0x00,0x03,0x00,0xfa,0x02,0x00,0x00,0x00,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0xf8,0x02,0x00,0x00,0xf9,0x02,0x00,0x00, +0xfa,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0xf9,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xfe,0x02,0x00,0x00, +0x24,0x00,0x00,0x00,0x19,0x03,0x00,0x00,0x41,0x00,0x05,0x00, +0x56,0x00,0x00,0x00,0xff,0x02,0x00,0x00,0x4f,0x00,0x00,0x00, +0xfe,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0x4c,0x00,0x00,0x00, +0x00,0x03,0x00,0x00,0xff,0x02,0x00,0x00,0x41,0x00,0x05,0x00, +0x56,0x00,0x00,0x00,0x01,0x03,0x00,0x00,0x4f,0x00,0x00,0x00, +0x24,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x4c,0x00,0x00,0x00, +0x02,0x03,0x00,0x00,0x01,0x03,0x00,0x00,0x81,0x00,0x05,0x00, +0x4c,0x00,0x00,0x00,0x03,0x03,0x00,0x00,0x02,0x03,0x00,0x00, +0x00,0x03,0x00,0x00,0x3e,0x00,0x03,0x00,0x01,0x03,0x00,0x00, +0x03,0x03,0x00,0x00,0xf9,0x00,0x02,0x00,0xfa,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0xfa,0x02,0x00,0x00,0xe0,0x00,0x04,0x00, +0x23,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0xed,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0xf2,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0xf2,0x02,0x00,0x00,0xc2,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x06,0x03,0x00,0x00,0x19,0x03,0x00,0x00,0x35,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xef,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0xf1,0x02,0x00,0x00,0xaa,0x00,0x05,0x00,0x64,0x00,0x00,0x00, +0x08,0x03,0x00,0x00,0x24,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0xf7,0x00,0x03,0x00,0x0a,0x03,0x00,0x00,0x00,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x08,0x03,0x00,0x00,0x09,0x03,0x00,0x00, +0x0a,0x03,0x00,0x00,0xf8,0x00,0x02,0x00,0x09,0x03,0x00,0x00, +0x41,0x00,0x05,0x00,0x16,0x00,0x00,0x00,0x0f,0x03,0x00,0x00, +0x13,0x00,0x00,0x00,0x89,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x10,0x03,0x00,0x00,0x0f,0x03,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x12,0x03,0x00,0x00, +0x10,0x03,0x00,0x00,0x0f,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x56,0x00,0x00,0x00,0x13,0x03,0x00,0x00,0x4f,0x00,0x00,0x00, +0x15,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x4c,0x00,0x00,0x00, +0x14,0x03,0x00,0x00,0x13,0x03,0x00,0x00,0x41,0x00,0x06,0x00, +0x97,0x00,0x00,0x00,0x15,0x03,0x00,0x00,0x0e,0x03,0x00,0x00, +0x15,0x00,0x00,0x00,0x12,0x03,0x00,0x00,0x3e,0x00,0x03,0x00, +0x15,0x03,0x00,0x00,0x14,0x03,0x00,0x00,0xf9,0x00,0x02,0x00, +0x0a,0x03,0x00,0x00,0xf8,0x00,0x02,0x00,0x0a,0x03,0x00,0x00, 0xfd,0x00,0x01,0x00,0x38,0x00,0x01,0x00, }; -const uint64_t mul_mat_vec_q2_K_f32_len = 7628; - -unsigned char mul_mat_vec_q3_K_f32_data[] = { -0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, -0x08,0x03,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, -0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x27,0x00,0x00,0x00, -0x11,0x00,0x02,0x00,0x51,0x11,0x00,0x00,0x11,0x00,0x02,0x00, -0x60,0x11,0x00,0x00,0x0b,0x00,0x06,0x00,0x01,0x00,0x00,0x00, -0x47,0x4c,0x53,0x4c,0x2e,0x73,0x74,0x64,0x2e,0x34,0x35,0x30, -0x00,0x00,0x00,0x00,0x0e,0x00,0x03,0x00,0x00,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x0f,0x00,0x0c,0x00,0x05,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x6d,0x61,0x69,0x6e,0x00,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x15,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x53,0x00,0x00,0x00,0x7d,0x00,0x00,0x00,0x92,0x00,0x00,0x00, -0xf9,0x02,0x00,0x00,0x10,0x00,0x06,0x00,0x04,0x00,0x00,0x00, -0x11,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x0c,0x00,0x00,0x00, -0x0b,0x00,0x00,0x00,0x1a,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x13,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x13,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x13,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x47,0x00,0x03,0x00, -0x13,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x21,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x1b,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x73,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x75,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x77,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x79,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x79,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x79,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x60,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x79,0x00,0x00,0x00,0x03,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x7a,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x6e,0x00,0x00,0x00, -0x48,0x00,0x04,0x00,0x7b,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x7b,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x03,0x00,0x7b,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x7d,0x00,0x00,0x00,0x22,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x7d,0x00,0x00,0x00, -0x21,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x8f,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x48,0x00,0x04,0x00,0x90,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x90,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x03,0x00,0x90,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x92,0x00,0x00,0x00,0x22,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x92,0x00,0x00,0x00, -0x21,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0xf6,0x02,0x00,0x00,0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x48,0x00,0x04,0x00,0xf7,0x02,0x00,0x00,0x00,0x00,0x00,0x00, -0x19,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0xf7,0x02,0x00,0x00, -0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x03,0x00,0xf7,0x02,0x00,0x00,0x02,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0xf9,0x02,0x00,0x00,0x22,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xf9,0x02,0x00,0x00, -0x21,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x02,0x03,0x00,0x00,0x0b,0x00,0x00,0x00,0x19,0x00,0x00,0x00, -0x13,0x00,0x02,0x00,0x02,0x00,0x00,0x00,0x21,0x00,0x03,0x00, -0x03,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x15,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x15,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x17,0x00,0x04,0x00,0x0a,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x0b,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x0a,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x0b,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x0d,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x0e,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x1e,0x00,0x05,0x00,0x13,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x14,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x13,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0x15,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x17,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x1a,0x00,0x00,0x00, -0x00,0x01,0x00,0x00,0x3b,0x00,0x04,0x00,0x0b,0x00,0x00,0x00, -0x21,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x25,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x08,0x00,0x00,0x00,0x15,0x00,0x04,0x00,0x35,0x00,0x00,0x00, -0x08,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x38,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x15,0x00,0x04,0x00,0x3d,0x00,0x00,0x00, -0x08,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x44,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x4a,0x00,0x00,0x00, -0x80,0x00,0x00,0x00,0x16,0x00,0x03,0x00,0x4f,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x50,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x1c,0x00,0x04,0x00, -0x51,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x50,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x52,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x51,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x52,0x00,0x00,0x00, -0x53,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x54,0x00,0x00,0x00,0x10,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x4f,0x00,0x00,0x00,0x59,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x5a,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x14,0x00,0x02,0x00, -0x6a,0x00,0x00,0x00,0x1c,0x00,0x04,0x00,0x73,0x00,0x00,0x00, -0x35,0x00,0x00,0x00,0x50,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x74,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x1c,0x00,0x04,0x00,0x75,0x00,0x00,0x00,0x35,0x00,0x00,0x00, -0x74,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x76,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x1c,0x00,0x04,0x00, -0x77,0x00,0x00,0x00,0x35,0x00,0x00,0x00,0x76,0x00,0x00,0x00, -0x16,0x00,0x03,0x00,0x78,0x00,0x00,0x00,0x10,0x00,0x00,0x00, -0x1e,0x00,0x06,0x00,0x79,0x00,0x00,0x00,0x73,0x00,0x00,0x00, -0x75,0x00,0x00,0x00,0x77,0x00,0x00,0x00,0x78,0x00,0x00,0x00, -0x1d,0x00,0x03,0x00,0x7a,0x00,0x00,0x00,0x79,0x00,0x00,0x00, -0x1e,0x00,0x03,0x00,0x7b,0x00,0x00,0x00,0x7a,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x7c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x7b,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x7c,0x00,0x00,0x00, -0x7d,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x81,0x00,0x00,0x00,0x03,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x82,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x78,0x00,0x00,0x00,0x1d,0x00,0x03,0x00,0x8f,0x00,0x00,0x00, -0x4f,0x00,0x00,0x00,0x1e,0x00,0x03,0x00,0x90,0x00,0x00,0x00, -0x8f,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x91,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x90,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x91,0x00,0x00,0x00,0x92,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x9a,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x4f,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0xa0,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x35,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xa7,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xee,0x00,0x00,0x00, -0x0a,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x23,0x01,0x00,0x00,0x40,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x37,0x01,0x00,0x00,0x02,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x69,0x01,0x00,0x00, -0x60,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x70,0x01,0x00,0x00,0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xbf,0x01,0x00,0x00,0x09,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xf5,0x01,0x00,0x00, -0x30,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x06,0x02,0x00,0x00,0x0b,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x3d,0x02,0x00,0x00,0x50,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x44,0x02,0x00,0x00, -0x05,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x85,0x02,0x00,0x00,0x70,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x8c,0x02,0x00,0x00,0x07,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0xd8,0x02,0x00,0x00, -0x08,0x01,0x00,0x00,0x1d,0x00,0x03,0x00,0xf6,0x02,0x00,0x00, -0x4f,0x00,0x00,0x00,0x1e,0x00,0x03,0x00,0xf7,0x02,0x00,0x00, -0xf6,0x02,0x00,0x00,0x20,0x00,0x04,0x00,0xf8,0x02,0x00,0x00, -0x0c,0x00,0x00,0x00,0xf7,0x02,0x00,0x00,0x3b,0x00,0x04,0x00, -0xf8,0x02,0x00,0x00,0xf9,0x02,0x00,0x00,0x0c,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x01,0x03,0x00,0x00, -0x01,0x00,0x00,0x00,0x2c,0x00,0x06,0x00,0x0a,0x00,0x00,0x00, -0x02,0x03,0x00,0x00,0x50,0x00,0x00,0x00,0x01,0x03,0x00,0x00, -0x01,0x03,0x00,0x00,0x36,0x00,0x05,0x00,0x02,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x03,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0x05,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x0e,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x0d,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x10,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x11,0x00,0x00,0x00,0x10,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00,0x18,0x00,0x00,0x00, -0x15,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x18,0x00,0x00,0x00, -0x87,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x1b,0x00,0x00,0x00, -0x19,0x00,0x00,0x00,0x1a,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x1f,0x00,0x00,0x00,0x11,0x00,0x00,0x00, -0x1b,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x0e,0x00,0x00,0x00, -0x22,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x0d,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x22,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x24,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x87,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x26,0x00,0x00,0x00,0x24,0x00,0x00,0x00, -0x25,0x00,0x00,0x00,0x8b,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x2b,0x00,0x00,0x00,0x24,0x00,0x00,0x00,0x25,0x00,0x00,0x00, -0x87,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x2f,0x00,0x00,0x00, -0x26,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x33,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x2f,0x00,0x00,0x00,0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x34,0x00,0x00,0x00,0x26,0x00,0x00,0x00,0x33,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3b,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x2f,0x00,0x00,0x00,0xc4,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x3c,0x00,0x00,0x00,0x38,0x00,0x00,0x00, -0x3b,0x00,0x00,0x00,0x72,0x00,0x04,0x00,0x3d,0x00,0x00,0x00, -0x3e,0x00,0x00,0x00,0x3c,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x35,0x00,0x00,0x00,0x3f,0x00,0x00,0x00,0x3e,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x42,0x00,0x00,0x00, -0x25,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x46,0x00,0x00,0x00,0x44,0x00,0x00,0x00, -0x2f,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x48,0x00,0x00,0x00,0x46,0x00,0x00,0x00,0x42,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x4c,0x00,0x00,0x00, -0x4a,0x00,0x00,0x00,0x2f,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x4c,0x00,0x00,0x00, -0x42,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x56,0x00,0x00,0x00,0x54,0x00,0x00,0x00,0x2b,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x58,0x00,0x00,0x00, -0x56,0x00,0x00,0x00,0x26,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x5a,0x00,0x00,0x00,0x5b,0x00,0x00,0x00,0x53,0x00,0x00,0x00, -0x58,0x00,0x00,0x00,0x3e,0x00,0x03,0x00,0x5b,0x00,0x00,0x00, -0x59,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x60,0x00,0x00,0x00,0x3b,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x63,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x63,0x00,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x03,0x03,0x00,0x00, -0x2b,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0xd7,0x02,0x00,0x00, -0x66,0x00,0x00,0x00,0xb1,0x00,0x05,0x00,0x6a,0x00,0x00,0x00, -0x6b,0x00,0x00,0x00,0x03,0x03,0x00,0x00,0x1b,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0x65,0x00,0x00,0x00,0x66,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x6b,0x00,0x00,0x00, -0x64,0x00,0x00,0x00,0x65,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x64,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x6e,0x00,0x00,0x00,0x03,0x03,0x00,0x00,0x1a,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x70,0x00,0x00,0x00, -0x6e,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x80,0x00,0x00,0x00,0x1f,0x00,0x00,0x00, -0x03,0x03,0x00,0x00,0x41,0x00,0x07,0x00,0x82,0x00,0x00,0x00, -0x83,0x00,0x00,0x00,0x7d,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0x80,0x00,0x00,0x00,0x81,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x78,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x83,0x00,0x00,0x00, -0x73,0x00,0x04,0x00,0x4f,0x00,0x00,0x00,0x85,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x88,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0x88,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, -0x4f,0x00,0x00,0x00,0x07,0x03,0x00,0x00,0x59,0x00,0x00,0x00, -0x64,0x00,0x00,0x00,0xc8,0x02,0x00,0x00,0x89,0x00,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x06,0x03,0x00,0x00, -0x16,0x00,0x00,0x00,0x64,0x00,0x00,0x00,0xca,0x02,0x00,0x00, -0x89,0x00,0x00,0x00,0xb1,0x00,0x05,0x00,0x6a,0x00,0x00,0x00, -0x8e,0x00,0x00,0x00,0x06,0x03,0x00,0x00,0x25,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0x8a,0x00,0x00,0x00,0x89,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x8e,0x00,0x00,0x00, -0x89,0x00,0x00,0x00,0x8a,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x89,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00, -0x93,0x00,0x00,0x00,0x15,0x00,0x00,0x00,0x38,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x94,0x00,0x00,0x00, -0x93,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x96,0x00,0x00,0x00,0x94,0x00,0x00,0x00,0x70,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x98,0x00,0x00,0x00, -0x96,0x00,0x00,0x00,0x06,0x03,0x00,0x00,0x41,0x00,0x06,0x00, -0x9a,0x00,0x00,0x00,0x9b,0x00,0x00,0x00,0x92,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x98,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4f,0x00,0x00,0x00,0x9c,0x00,0x00,0x00,0x9b,0x00,0x00,0x00, -0x41,0x00,0x08,0x00,0xa0,0x00,0x00,0x00,0xa1,0x00,0x00,0x00, -0x7d,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x80,0x00,0x00,0x00, -0x25,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x35,0x00,0x00,0x00,0xa2,0x00,0x00,0x00,0xa1,0x00,0x00,0x00, -0xc2,0x00,0x05,0x00,0x35,0x00,0x00,0x00,0xa4,0x00,0x00,0x00, -0xa2,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x71,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0xa5,0x00,0x00,0x00,0xa4,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xa6,0x00,0x00,0x00, -0xa5,0x00,0x00,0x00,0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xa8,0x00,0x00,0x00,0xa6,0x00,0x00,0x00,0xa7,0x00,0x00,0x00, -0x41,0x00,0x08,0x00,0xa0,0x00,0x00,0x00,0xac,0x00,0x00,0x00, -0x7d,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x80,0x00,0x00,0x00, -0x25,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x35,0x00,0x00,0x00,0xad,0x00,0x00,0x00,0xac,0x00,0x00,0x00, -0xc2,0x00,0x05,0x00,0x35,0x00,0x00,0x00,0xb0,0x00,0x00,0x00, -0xad,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x71,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0xb1,0x00,0x00,0x00,0xb0,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xb2,0x00,0x00,0x00, -0xb1,0x00,0x00,0x00,0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xb3,0x00,0x00,0x00,0xb2,0x00,0x00,0x00,0x81,0x00,0x00,0x00, -0xc4,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xb4,0x00,0x00,0x00, -0xb3,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0xc5,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xb5,0x00,0x00,0x00,0xa8,0x00,0x00,0x00, -0xb4,0x00,0x00,0x00,0x72,0x00,0x04,0x00,0x3d,0x00,0x00,0x00, -0xb6,0x00,0x00,0x00,0xb5,0x00,0x00,0x00,0x72,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xb7,0x00,0x00,0x00,0xb6,0x00,0x00,0x00, -0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xb8,0x00,0x00,0x00, -0xb7,0x00,0x00,0x00,0x44,0x00,0x00,0x00,0x6f,0x00,0x04,0x00, -0x4f,0x00,0x00,0x00,0xb9,0x00,0x00,0x00,0xb8,0x00,0x00,0x00, -0x85,0x00,0x05,0x00,0x4f,0x00,0x00,0x00,0xba,0x00,0x00,0x00, -0x9c,0x00,0x00,0x00,0xb9,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xc0,0x00,0x00,0x00,0x48,0x00,0x00,0x00, -0x06,0x03,0x00,0x00,0x41,0x00,0x08,0x00,0xa0,0x00,0x00,0x00, -0xc1,0x00,0x00,0x00,0x7d,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0x80,0x00,0x00,0x00,0x38,0x00,0x00,0x00,0xc0,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x35,0x00,0x00,0x00,0xc2,0x00,0x00,0x00, -0xc1,0x00,0x00,0x00,0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0xc3,0x00,0x00,0x00,0xc2,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xc4,0x00,0x00,0x00,0xc3,0x00,0x00,0x00, -0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xc5,0x00,0x00,0x00, -0xc4,0x00,0x00,0x00,0x81,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xcb,0x00,0x00,0x00,0x42,0x00,0x00,0x00, -0x06,0x03,0x00,0x00,0x41,0x00,0x08,0x00,0xa0,0x00,0x00,0x00, -0xcc,0x00,0x00,0x00,0x7d,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0x80,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0xcb,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x35,0x00,0x00,0x00,0xcd,0x00,0x00,0x00, -0xcc,0x00,0x00,0x00,0xc4,0x00,0x05,0x00,0x35,0x00,0x00,0x00, -0xcf,0x00,0x00,0x00,0x3f,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0xc7,0x00,0x05,0x00,0x35,0x00,0x00,0x00,0xd0,0x00,0x00,0x00, -0xcd,0x00,0x00,0x00,0xcf,0x00,0x00,0x00,0x71,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0xd1,0x00,0x00,0x00,0xd0,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xd2,0x00,0x00,0x00, -0xd1,0x00,0x00,0x00,0xab,0x00,0x05,0x00,0x6a,0x00,0x00,0x00, -0xd3,0x00,0x00,0x00,0xd2,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0xa9,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xd4,0x00,0x00,0x00, -0xd3,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xd5,0x00,0x00,0x00, -0xc5,0x00,0x00,0x00,0xd4,0x00,0x00,0x00,0x6f,0x00,0x04,0x00, -0x4f,0x00,0x00,0x00,0xd6,0x00,0x00,0x00,0xd5,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xde,0x00,0x00,0x00, -0x98,0x00,0x00,0x00,0x44,0x00,0x00,0x00,0x41,0x00,0x06,0x00, -0x9a,0x00,0x00,0x00,0xdf,0x00,0x00,0x00,0x92,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0xde,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4f,0x00,0x00,0x00,0xe0,0x00,0x00,0x00,0xdf,0x00,0x00,0x00, -0x41,0x00,0x08,0x00,0xa0,0x00,0x00,0x00,0xe4,0x00,0x00,0x00, -0x7d,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x80,0x00,0x00,0x00, -0x25,0x00,0x00,0x00,0x25,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x35,0x00,0x00,0x00,0xe5,0x00,0x00,0x00,0xe4,0x00,0x00,0x00, -0xc2,0x00,0x05,0x00,0x35,0x00,0x00,0x00,0xe7,0x00,0x00,0x00, -0xe5,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x71,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0xe8,0x00,0x00,0x00,0xe7,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xe9,0x00,0x00,0x00, -0xe8,0x00,0x00,0x00,0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xea,0x00,0x00,0x00,0xe9,0x00,0x00,0x00,0xa7,0x00,0x00,0x00, -0x41,0x00,0x08,0x00,0xa0,0x00,0x00,0x00,0xef,0x00,0x00,0x00, -0x7d,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x80,0x00,0x00,0x00, -0x25,0x00,0x00,0x00,0xee,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x35,0x00,0x00,0x00,0xf0,0x00,0x00,0x00,0xef,0x00,0x00,0x00, -0xc2,0x00,0x05,0x00,0x35,0x00,0x00,0x00,0xf3,0x00,0x00,0x00, -0xf0,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x71,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0xf4,0x00,0x00,0x00,0xf3,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xf5,0x00,0x00,0x00, -0xf4,0x00,0x00,0x00,0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xf6,0x00,0x00,0x00,0xf5,0x00,0x00,0x00,0x81,0x00,0x00,0x00, -0xc4,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xf7,0x00,0x00,0x00, -0xf6,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0xc5,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xf8,0x00,0x00,0x00,0xea,0x00,0x00,0x00, -0xf7,0x00,0x00,0x00,0x72,0x00,0x04,0x00,0x3d,0x00,0x00,0x00, -0xf9,0x00,0x00,0x00,0xf8,0x00,0x00,0x00,0x72,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xfa,0x00,0x00,0x00,0xf9,0x00,0x00,0x00, -0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xfb,0x00,0x00,0x00, -0xfa,0x00,0x00,0x00,0x44,0x00,0x00,0x00,0x6f,0x00,0x04,0x00, -0x4f,0x00,0x00,0x00,0xfc,0x00,0x00,0x00,0xfb,0x00,0x00,0x00, -0x85,0x00,0x05,0x00,0x4f,0x00,0x00,0x00,0xfd,0x00,0x00,0x00, -0xe0,0x00,0x00,0x00,0xfc,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x35,0x00,0x00,0x00,0x05,0x01,0x00,0x00,0xc1,0x00,0x00,0x00, -0xc2,0x00,0x05,0x00,0x35,0x00,0x00,0x00,0x06,0x01,0x00,0x00, -0x05,0x01,0x00,0x00,0x25,0x00,0x00,0x00,0x71,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x07,0x01,0x00,0x00,0x06,0x01,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x08,0x01,0x00,0x00, -0x07,0x01,0x00,0x00,0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x09,0x01,0x00,0x00,0x08,0x01,0x00,0x00,0x81,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x35,0x00,0x00,0x00,0x11,0x01,0x00,0x00, -0xcc,0x00,0x00,0x00,0xc4,0x00,0x05,0x00,0x35,0x00,0x00,0x00, -0x13,0x01,0x00,0x00,0x3f,0x00,0x00,0x00,0x38,0x00,0x00,0x00, -0xc7,0x00,0x05,0x00,0x35,0x00,0x00,0x00,0x14,0x01,0x00,0x00, -0x11,0x01,0x00,0x00,0x13,0x01,0x00,0x00,0x71,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x15,0x01,0x00,0x00,0x14,0x01,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x16,0x01,0x00,0x00, -0x15,0x01,0x00,0x00,0xab,0x00,0x05,0x00,0x6a,0x00,0x00,0x00, -0x17,0x01,0x00,0x00,0x16,0x01,0x00,0x00,0x16,0x00,0x00,0x00, -0xa9,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x18,0x01,0x00,0x00, -0x17,0x01,0x00,0x00,0x16,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x19,0x01,0x00,0x00, -0x09,0x01,0x00,0x00,0x18,0x01,0x00,0x00,0x6f,0x00,0x04,0x00, -0x4f,0x00,0x00,0x00,0x1a,0x01,0x00,0x00,0x19,0x01,0x00,0x00, -0x85,0x00,0x05,0x00,0x4f,0x00,0x00,0x00,0x1b,0x01,0x00,0x00, -0xfd,0x00,0x00,0x00,0x1a,0x01,0x00,0x00,0x0c,0x00,0x08,0x00, -0x4f,0x00,0x00,0x00,0x1c,0x01,0x00,0x00,0x01,0x00,0x00,0x00, -0x32,0x00,0x00,0x00,0xba,0x00,0x00,0x00,0xd6,0x00,0x00,0x00, -0x1b,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x24,0x01,0x00,0x00,0x98,0x00,0x00,0x00,0x23,0x01,0x00,0x00, -0x41,0x00,0x06,0x00,0x9a,0x00,0x00,0x00,0x25,0x01,0x00,0x00, -0x92,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x24,0x01,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4f,0x00,0x00,0x00,0x26,0x01,0x00,0x00, -0x25,0x01,0x00,0x00,0x41,0x00,0x08,0x00,0xa0,0x00,0x00,0x00, -0x2a,0x01,0x00,0x00,0x7d,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0x80,0x00,0x00,0x00,0x25,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x35,0x00,0x00,0x00,0x2b,0x01,0x00,0x00, -0x2a,0x01,0x00,0x00,0xc2,0x00,0x05,0x00,0x35,0x00,0x00,0x00, -0x2d,0x01,0x00,0x00,0x2b,0x01,0x00,0x00,0x60,0x00,0x00,0x00, -0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x2e,0x01,0x00,0x00, -0x2d,0x01,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x2f,0x01,0x00,0x00,0x2e,0x01,0x00,0x00,0xc7,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x30,0x01,0x00,0x00,0x2f,0x01,0x00,0x00, -0xa7,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x35,0x00,0x00,0x00, -0x35,0x01,0x00,0x00,0xac,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0x38,0x01,0x00,0x00,0x60,0x00,0x00,0x00, -0x37,0x01,0x00,0x00,0xc2,0x00,0x05,0x00,0x35,0x00,0x00,0x00, -0x39,0x01,0x00,0x00,0x35,0x01,0x00,0x00,0x38,0x01,0x00,0x00, -0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x3a,0x01,0x00,0x00, -0x39,0x01,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x3b,0x01,0x00,0x00,0x3a,0x01,0x00,0x00,0xc7,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x3c,0x01,0x00,0x00,0x3b,0x01,0x00,0x00, -0x81,0x00,0x00,0x00,0xc4,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x3d,0x01,0x00,0x00,0x3c,0x01,0x00,0x00,0x39,0x00,0x00,0x00, -0xc5,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3e,0x01,0x00,0x00, -0x30,0x01,0x00,0x00,0x3d,0x01,0x00,0x00,0x72,0x00,0x04,0x00, -0x3d,0x00,0x00,0x00,0x3f,0x01,0x00,0x00,0x3e,0x01,0x00,0x00, -0x72,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x40,0x01,0x00,0x00, -0x3f,0x01,0x00,0x00,0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x41,0x01,0x00,0x00,0x40,0x01,0x00,0x00,0x44,0x00,0x00,0x00, -0x6f,0x00,0x04,0x00,0x4f,0x00,0x00,0x00,0x42,0x01,0x00,0x00, -0x41,0x01,0x00,0x00,0x85,0x00,0x05,0x00,0x4f,0x00,0x00,0x00, -0x43,0x01,0x00,0x00,0x26,0x01,0x00,0x00,0x42,0x01,0x00,0x00, -0x3d,0x00,0x04,0x00,0x35,0x00,0x00,0x00,0x4b,0x01,0x00,0x00, -0xc1,0x00,0x00,0x00,0xc2,0x00,0x05,0x00,0x35,0x00,0x00,0x00, -0x4c,0x01,0x00,0x00,0x4b,0x01,0x00,0x00,0x39,0x00,0x00,0x00, -0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x4d,0x01,0x00,0x00, -0x4c,0x01,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x4e,0x01,0x00,0x00,0x4d,0x01,0x00,0x00,0xc7,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x4f,0x01,0x00,0x00,0x4e,0x01,0x00,0x00, -0x81,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x35,0x00,0x00,0x00, -0x57,0x01,0x00,0x00,0xcc,0x00,0x00,0x00,0xc4,0x00,0x05,0x00, -0x35,0x00,0x00,0x00,0x59,0x01,0x00,0x00,0x3f,0x00,0x00,0x00, -0x25,0x00,0x00,0x00,0xc7,0x00,0x05,0x00,0x35,0x00,0x00,0x00, -0x5a,0x01,0x00,0x00,0x57,0x01,0x00,0x00,0x59,0x01,0x00,0x00, -0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x5b,0x01,0x00,0x00, -0x5a,0x01,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x5c,0x01,0x00,0x00,0x5b,0x01,0x00,0x00,0xab,0x00,0x05,0x00, -0x6a,0x00,0x00,0x00,0x5d,0x01,0x00,0x00,0x5c,0x01,0x00,0x00, -0x16,0x00,0x00,0x00,0xa9,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x5e,0x01,0x00,0x00,0x5d,0x01,0x00,0x00,0x16,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x5f,0x01,0x00,0x00,0x4f,0x01,0x00,0x00,0x5e,0x01,0x00,0x00, -0x6f,0x00,0x04,0x00,0x4f,0x00,0x00,0x00,0x60,0x01,0x00,0x00, -0x5f,0x01,0x00,0x00,0x0c,0x00,0x08,0x00,0x4f,0x00,0x00,0x00, -0x62,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00, -0x43,0x01,0x00,0x00,0x60,0x01,0x00,0x00,0x1c,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x6a,0x01,0x00,0x00, -0x98,0x00,0x00,0x00,0x69,0x01,0x00,0x00,0x41,0x00,0x06,0x00, -0x9a,0x00,0x00,0x00,0x6b,0x01,0x00,0x00,0x92,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x6a,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4f,0x00,0x00,0x00,0x6c,0x01,0x00,0x00,0x6b,0x01,0x00,0x00, -0x41,0x00,0x08,0x00,0xa0,0x00,0x00,0x00,0x71,0x01,0x00,0x00, -0x7d,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x80,0x00,0x00,0x00, -0x25,0x00,0x00,0x00,0x70,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, -0x35,0x00,0x00,0x00,0x72,0x01,0x00,0x00,0x71,0x01,0x00,0x00, -0xc2,0x00,0x05,0x00,0x35,0x00,0x00,0x00,0x74,0x01,0x00,0x00, -0x72,0x01,0x00,0x00,0x60,0x00,0x00,0x00,0x71,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x75,0x01,0x00,0x00,0x74,0x01,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x76,0x01,0x00,0x00, -0x75,0x01,0x00,0x00,0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x77,0x01,0x00,0x00,0x76,0x01,0x00,0x00,0xa7,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x35,0x00,0x00,0x00,0x7c,0x01,0x00,0x00, -0xef,0x00,0x00,0x00,0xc2,0x00,0x05,0x00,0x35,0x00,0x00,0x00, -0x7f,0x01,0x00,0x00,0x7c,0x01,0x00,0x00,0x38,0x01,0x00,0x00, -0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x80,0x01,0x00,0x00, -0x7f,0x01,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x81,0x01,0x00,0x00,0x80,0x01,0x00,0x00,0xc7,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x82,0x01,0x00,0x00,0x81,0x01,0x00,0x00, -0x81,0x00,0x00,0x00,0xc4,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x83,0x01,0x00,0x00,0x82,0x01,0x00,0x00,0x39,0x00,0x00,0x00, -0xc5,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x84,0x01,0x00,0x00, -0x77,0x01,0x00,0x00,0x83,0x01,0x00,0x00,0x72,0x00,0x04,0x00, -0x3d,0x00,0x00,0x00,0x85,0x01,0x00,0x00,0x84,0x01,0x00,0x00, -0x72,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x86,0x01,0x00,0x00, -0x85,0x01,0x00,0x00,0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x87,0x01,0x00,0x00,0x86,0x01,0x00,0x00,0x44,0x00,0x00,0x00, -0x6f,0x00,0x04,0x00,0x4f,0x00,0x00,0x00,0x88,0x01,0x00,0x00, -0x87,0x01,0x00,0x00,0x85,0x00,0x05,0x00,0x4f,0x00,0x00,0x00, -0x89,0x01,0x00,0x00,0x6c,0x01,0x00,0x00,0x88,0x01,0x00,0x00, -0x3d,0x00,0x04,0x00,0x35,0x00,0x00,0x00,0x91,0x01,0x00,0x00, -0xc1,0x00,0x00,0x00,0xc2,0x00,0x05,0x00,0x35,0x00,0x00,0x00, -0x92,0x01,0x00,0x00,0x91,0x01,0x00,0x00,0x70,0x01,0x00,0x00, -0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x93,0x01,0x00,0x00, -0x92,0x01,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x94,0x01,0x00,0x00,0x93,0x01,0x00,0x00,0xc7,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x95,0x01,0x00,0x00,0x94,0x01,0x00,0x00, -0x81,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x35,0x00,0x00,0x00, -0x9d,0x01,0x00,0x00,0xcc,0x00,0x00,0x00,0xc4,0x00,0x05,0x00, -0x35,0x00,0x00,0x00,0x9f,0x01,0x00,0x00,0x3f,0x00,0x00,0x00, -0x81,0x00,0x00,0x00,0xc7,0x00,0x05,0x00,0x35,0x00,0x00,0x00, -0xa0,0x01,0x00,0x00,0x9d,0x01,0x00,0x00,0x9f,0x01,0x00,0x00, -0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0xa1,0x01,0x00,0x00, -0xa0,0x01,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xa2,0x01,0x00,0x00,0xa1,0x01,0x00,0x00,0xab,0x00,0x05,0x00, -0x6a,0x00,0x00,0x00,0xa3,0x01,0x00,0x00,0xa2,0x01,0x00,0x00, -0x16,0x00,0x00,0x00,0xa9,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xa4,0x01,0x00,0x00,0xa3,0x01,0x00,0x00,0x16,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xa5,0x01,0x00,0x00,0x95,0x01,0x00,0x00,0xa4,0x01,0x00,0x00, -0x6f,0x00,0x04,0x00,0x4f,0x00,0x00,0x00,0xa6,0x01,0x00,0x00, -0xa5,0x01,0x00,0x00,0x0c,0x00,0x08,0x00,0x4f,0x00,0x00,0x00, -0xa8,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00, -0x89,0x01,0x00,0x00,0xa6,0x01,0x00,0x00,0x62,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xaf,0x01,0x00,0x00, -0x98,0x00,0x00,0x00,0x54,0x00,0x00,0x00,0x41,0x00,0x06,0x00, -0x9a,0x00,0x00,0x00,0xb0,0x01,0x00,0x00,0x92,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0xaf,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4f,0x00,0x00,0x00,0xb1,0x01,0x00,0x00,0xb0,0x01,0x00,0x00, -0x41,0x00,0x08,0x00,0xa0,0x00,0x00,0x00,0xb5,0x01,0x00,0x00, -0x7d,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x80,0x00,0x00,0x00, -0x25,0x00,0x00,0x00,0x38,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x35,0x00,0x00,0x00,0xb6,0x01,0x00,0x00,0xb5,0x01,0x00,0x00, -0xc2,0x00,0x05,0x00,0x35,0x00,0x00,0x00,0xb8,0x01,0x00,0x00, -0xb6,0x01,0x00,0x00,0x60,0x00,0x00,0x00,0x71,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0xb9,0x01,0x00,0x00,0xb8,0x01,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xba,0x01,0x00,0x00, -0xb9,0x01,0x00,0x00,0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xbb,0x01,0x00,0x00,0xba,0x01,0x00,0x00,0xa7,0x00,0x00,0x00, -0x41,0x00,0x08,0x00,0xa0,0x00,0x00,0x00,0xc0,0x01,0x00,0x00, -0x7d,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x80,0x00,0x00,0x00, -0x25,0x00,0x00,0x00,0xbf,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, -0x35,0x00,0x00,0x00,0xc1,0x01,0x00,0x00,0xc0,0x01,0x00,0x00, -0xc2,0x00,0x05,0x00,0x35,0x00,0x00,0x00,0xc4,0x01,0x00,0x00, -0xc1,0x01,0x00,0x00,0x60,0x00,0x00,0x00,0x71,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0xc5,0x01,0x00,0x00,0xc4,0x01,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xc6,0x01,0x00,0x00, -0xc5,0x01,0x00,0x00,0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xc7,0x01,0x00,0x00,0xc6,0x01,0x00,0x00,0x81,0x00,0x00,0x00, -0xc4,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xc8,0x01,0x00,0x00, -0xc7,0x01,0x00,0x00,0x39,0x00,0x00,0x00,0xc5,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xc9,0x01,0x00,0x00,0xbb,0x01,0x00,0x00, -0xc8,0x01,0x00,0x00,0x72,0x00,0x04,0x00,0x3d,0x00,0x00,0x00, -0xca,0x01,0x00,0x00,0xc9,0x01,0x00,0x00,0x72,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xcb,0x01,0x00,0x00,0xca,0x01,0x00,0x00, -0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xcc,0x01,0x00,0x00, -0xcb,0x01,0x00,0x00,0x44,0x00,0x00,0x00,0x6f,0x00,0x04,0x00, -0x4f,0x00,0x00,0x00,0xcd,0x01,0x00,0x00,0xcc,0x01,0x00,0x00, -0x85,0x00,0x05,0x00,0x4f,0x00,0x00,0x00,0xce,0x01,0x00,0x00, -0xb1,0x01,0x00,0x00,0xcd,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xd5,0x01,0x00,0x00,0xc0,0x00,0x00,0x00, -0x54,0x00,0x00,0x00,0x41,0x00,0x08,0x00,0xa0,0x00,0x00,0x00, -0xd6,0x01,0x00,0x00,0x7d,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0x80,0x00,0x00,0x00,0x38,0x00,0x00,0x00,0xd5,0x01,0x00,0x00, -0x3d,0x00,0x04,0x00,0x35,0x00,0x00,0x00,0xd7,0x01,0x00,0x00, -0xd6,0x01,0x00,0x00,0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0xd8,0x01,0x00,0x00,0xd7,0x01,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xd9,0x01,0x00,0x00,0xd8,0x01,0x00,0x00, -0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xda,0x01,0x00,0x00, -0xd9,0x01,0x00,0x00,0x81,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xe1,0x01,0x00,0x00,0xcb,0x00,0x00,0x00, -0x54,0x00,0x00,0x00,0x41,0x00,0x08,0x00,0xa0,0x00,0x00,0x00, -0xe2,0x01,0x00,0x00,0x7d,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0x80,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0xe1,0x01,0x00,0x00, -0x3d,0x00,0x04,0x00,0x35,0x00,0x00,0x00,0xe3,0x01,0x00,0x00, -0xe2,0x01,0x00,0x00,0xc7,0x00,0x05,0x00,0x35,0x00,0x00,0x00, -0xe6,0x01,0x00,0x00,0xe3,0x01,0x00,0x00,0xcf,0x00,0x00,0x00, -0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0xe7,0x01,0x00,0x00, -0xe6,0x01,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xe8,0x01,0x00,0x00,0xe7,0x01,0x00,0x00,0xab,0x00,0x05,0x00, -0x6a,0x00,0x00,0x00,0xe9,0x01,0x00,0x00,0xe8,0x01,0x00,0x00, -0x16,0x00,0x00,0x00,0xa9,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xea,0x01,0x00,0x00,0xe9,0x01,0x00,0x00,0x16,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xeb,0x01,0x00,0x00,0xda,0x01,0x00,0x00,0xea,0x01,0x00,0x00, -0x6f,0x00,0x04,0x00,0x4f,0x00,0x00,0x00,0xec,0x01,0x00,0x00, -0xeb,0x01,0x00,0x00,0x0c,0x00,0x08,0x00,0x4f,0x00,0x00,0x00, -0xee,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00, -0xce,0x01,0x00,0x00,0xec,0x01,0x00,0x00,0xa8,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xf6,0x01,0x00,0x00, -0x98,0x00,0x00,0x00,0xf5,0x01,0x00,0x00,0x41,0x00,0x06,0x00, -0x9a,0x00,0x00,0x00,0xf7,0x01,0x00,0x00,0x92,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0xf6,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4f,0x00,0x00,0x00,0xf8,0x01,0x00,0x00,0xf7,0x01,0x00,0x00, -0x41,0x00,0x08,0x00,0xa0,0x00,0x00,0x00,0xfc,0x01,0x00,0x00, -0x7d,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x80,0x00,0x00,0x00, -0x25,0x00,0x00,0x00,0x81,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x35,0x00,0x00,0x00,0xfd,0x01,0x00,0x00,0xfc,0x01,0x00,0x00, -0xc2,0x00,0x05,0x00,0x35,0x00,0x00,0x00,0xff,0x01,0x00,0x00, -0xfd,0x01,0x00,0x00,0x60,0x00,0x00,0x00,0x71,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x00,0x02,0x00,0x00,0xff,0x01,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x01,0x02,0x00,0x00, -0x00,0x02,0x00,0x00,0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x02,0x02,0x00,0x00,0x01,0x02,0x00,0x00,0xa7,0x00,0x00,0x00, -0x41,0x00,0x08,0x00,0xa0,0x00,0x00,0x00,0x07,0x02,0x00,0x00, -0x7d,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x80,0x00,0x00,0x00, -0x25,0x00,0x00,0x00,0x06,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, -0x35,0x00,0x00,0x00,0x08,0x02,0x00,0x00,0x07,0x02,0x00,0x00, -0xc2,0x00,0x05,0x00,0x35,0x00,0x00,0x00,0x0b,0x02,0x00,0x00, -0x08,0x02,0x00,0x00,0x60,0x00,0x00,0x00,0x71,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x0c,0x02,0x00,0x00,0x0b,0x02,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x0d,0x02,0x00,0x00, -0x0c,0x02,0x00,0x00,0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x0e,0x02,0x00,0x00,0x0d,0x02,0x00,0x00,0x81,0x00,0x00,0x00, -0xc4,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x0f,0x02,0x00,0x00, -0x0e,0x02,0x00,0x00,0x39,0x00,0x00,0x00,0xc5,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x10,0x02,0x00,0x00,0x02,0x02,0x00,0x00, -0x0f,0x02,0x00,0x00,0x72,0x00,0x04,0x00,0x3d,0x00,0x00,0x00, -0x11,0x02,0x00,0x00,0x10,0x02,0x00,0x00,0x72,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x12,0x02,0x00,0x00,0x11,0x02,0x00,0x00, -0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x13,0x02,0x00,0x00, -0x12,0x02,0x00,0x00,0x44,0x00,0x00,0x00,0x6f,0x00,0x04,0x00, -0x4f,0x00,0x00,0x00,0x14,0x02,0x00,0x00,0x13,0x02,0x00,0x00, -0x85,0x00,0x05,0x00,0x4f,0x00,0x00,0x00,0x15,0x02,0x00,0x00, -0xf8,0x01,0x00,0x00,0x14,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, -0x35,0x00,0x00,0x00,0x1e,0x02,0x00,0x00,0xd6,0x01,0x00,0x00, -0xc2,0x00,0x05,0x00,0x35,0x00,0x00,0x00,0x1f,0x02,0x00,0x00, -0x1e,0x02,0x00,0x00,0x25,0x00,0x00,0x00,0x71,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x20,0x02,0x00,0x00,0x1f,0x02,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x21,0x02,0x00,0x00, -0x20,0x02,0x00,0x00,0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x22,0x02,0x00,0x00,0x21,0x02,0x00,0x00,0x81,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x35,0x00,0x00,0x00,0x2b,0x02,0x00,0x00, -0xe2,0x01,0x00,0x00,0xc7,0x00,0x05,0x00,0x35,0x00,0x00,0x00, -0x2e,0x02,0x00,0x00,0x2b,0x02,0x00,0x00,0x13,0x01,0x00,0x00, -0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x2f,0x02,0x00,0x00, -0x2e,0x02,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x30,0x02,0x00,0x00,0x2f,0x02,0x00,0x00,0xab,0x00,0x05,0x00, -0x6a,0x00,0x00,0x00,0x31,0x02,0x00,0x00,0x30,0x02,0x00,0x00, -0x16,0x00,0x00,0x00,0xa9,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x32,0x02,0x00,0x00,0x31,0x02,0x00,0x00,0x16,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x33,0x02,0x00,0x00,0x22,0x02,0x00,0x00,0x32,0x02,0x00,0x00, -0x6f,0x00,0x04,0x00,0x4f,0x00,0x00,0x00,0x34,0x02,0x00,0x00, -0x33,0x02,0x00,0x00,0x0c,0x00,0x08,0x00,0x4f,0x00,0x00,0x00, -0x36,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00, -0x15,0x02,0x00,0x00,0x34,0x02,0x00,0x00,0xee,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3e,0x02,0x00,0x00, -0x98,0x00,0x00,0x00,0x3d,0x02,0x00,0x00,0x41,0x00,0x06,0x00, -0x9a,0x00,0x00,0x00,0x3f,0x02,0x00,0x00,0x92,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x3e,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4f,0x00,0x00,0x00,0x40,0x02,0x00,0x00,0x3f,0x02,0x00,0x00, -0x41,0x00,0x08,0x00,0xa0,0x00,0x00,0x00,0x45,0x02,0x00,0x00, -0x7d,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x80,0x00,0x00,0x00, -0x25,0x00,0x00,0x00,0x44,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, -0x35,0x00,0x00,0x00,0x46,0x02,0x00,0x00,0x45,0x02,0x00,0x00, -0xc2,0x00,0x05,0x00,0x35,0x00,0x00,0x00,0x48,0x02,0x00,0x00, -0x46,0x02,0x00,0x00,0x60,0x00,0x00,0x00,0x71,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x49,0x02,0x00,0x00,0x48,0x02,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x4a,0x02,0x00,0x00, -0x49,0x02,0x00,0x00,0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x4b,0x02,0x00,0x00,0x4a,0x02,0x00,0x00,0xa7,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x35,0x00,0x00,0x00,0x50,0x02,0x00,0x00, -0xc0,0x01,0x00,0x00,0xc2,0x00,0x05,0x00,0x35,0x00,0x00,0x00, -0x53,0x02,0x00,0x00,0x50,0x02,0x00,0x00,0x38,0x01,0x00,0x00, -0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x54,0x02,0x00,0x00, -0x53,0x02,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x55,0x02,0x00,0x00,0x54,0x02,0x00,0x00,0xc7,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x56,0x02,0x00,0x00,0x55,0x02,0x00,0x00, -0x81,0x00,0x00,0x00,0xc4,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x57,0x02,0x00,0x00,0x56,0x02,0x00,0x00,0x39,0x00,0x00,0x00, -0xc5,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x58,0x02,0x00,0x00, -0x4b,0x02,0x00,0x00,0x57,0x02,0x00,0x00,0x72,0x00,0x04,0x00, -0x3d,0x00,0x00,0x00,0x59,0x02,0x00,0x00,0x58,0x02,0x00,0x00, -0x72,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x5a,0x02,0x00,0x00, -0x59,0x02,0x00,0x00,0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x5b,0x02,0x00,0x00,0x5a,0x02,0x00,0x00,0x44,0x00,0x00,0x00, -0x6f,0x00,0x04,0x00,0x4f,0x00,0x00,0x00,0x5c,0x02,0x00,0x00, -0x5b,0x02,0x00,0x00,0x85,0x00,0x05,0x00,0x4f,0x00,0x00,0x00, -0x5d,0x02,0x00,0x00,0x40,0x02,0x00,0x00,0x5c,0x02,0x00,0x00, -0x3d,0x00,0x04,0x00,0x35,0x00,0x00,0x00,0x66,0x02,0x00,0x00, -0xd6,0x01,0x00,0x00,0xc2,0x00,0x05,0x00,0x35,0x00,0x00,0x00, -0x67,0x02,0x00,0x00,0x66,0x02,0x00,0x00,0x39,0x00,0x00,0x00, -0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x68,0x02,0x00,0x00, -0x67,0x02,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x69,0x02,0x00,0x00,0x68,0x02,0x00,0x00,0xc7,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x6a,0x02,0x00,0x00,0x69,0x02,0x00,0x00, -0x81,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x35,0x00,0x00,0x00, -0x73,0x02,0x00,0x00,0xe2,0x01,0x00,0x00,0xc7,0x00,0x05,0x00, -0x35,0x00,0x00,0x00,0x76,0x02,0x00,0x00,0x73,0x02,0x00,0x00, -0x59,0x01,0x00,0x00,0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x77,0x02,0x00,0x00,0x76,0x02,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x78,0x02,0x00,0x00,0x77,0x02,0x00,0x00, -0xab,0x00,0x05,0x00,0x6a,0x00,0x00,0x00,0x79,0x02,0x00,0x00, -0x78,0x02,0x00,0x00,0x16,0x00,0x00,0x00,0xa9,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x7a,0x02,0x00,0x00,0x79,0x02,0x00,0x00, -0x16,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x82,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x7b,0x02,0x00,0x00,0x6a,0x02,0x00,0x00, -0x7a,0x02,0x00,0x00,0x6f,0x00,0x04,0x00,0x4f,0x00,0x00,0x00, -0x7c,0x02,0x00,0x00,0x7b,0x02,0x00,0x00,0x0c,0x00,0x08,0x00, -0x4f,0x00,0x00,0x00,0x7e,0x02,0x00,0x00,0x01,0x00,0x00,0x00, -0x32,0x00,0x00,0x00,0x5d,0x02,0x00,0x00,0x7c,0x02,0x00,0x00, -0x36,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x86,0x02,0x00,0x00,0x98,0x00,0x00,0x00,0x85,0x02,0x00,0x00, -0x41,0x00,0x06,0x00,0x9a,0x00,0x00,0x00,0x87,0x02,0x00,0x00, -0x92,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x86,0x02,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4f,0x00,0x00,0x00,0x88,0x02,0x00,0x00, -0x87,0x02,0x00,0x00,0x41,0x00,0x08,0x00,0xa0,0x00,0x00,0x00, -0x8d,0x02,0x00,0x00,0x7d,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0x80,0x00,0x00,0x00,0x25,0x00,0x00,0x00,0x8c,0x02,0x00,0x00, -0x3d,0x00,0x04,0x00,0x35,0x00,0x00,0x00,0x8e,0x02,0x00,0x00, -0x8d,0x02,0x00,0x00,0xc2,0x00,0x05,0x00,0x35,0x00,0x00,0x00, -0x90,0x02,0x00,0x00,0x8e,0x02,0x00,0x00,0x60,0x00,0x00,0x00, -0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x91,0x02,0x00,0x00, -0x90,0x02,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x92,0x02,0x00,0x00,0x91,0x02,0x00,0x00,0xc7,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x93,0x02,0x00,0x00,0x92,0x02,0x00,0x00, -0xa7,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x35,0x00,0x00,0x00, -0x98,0x02,0x00,0x00,0x07,0x02,0x00,0x00,0xc2,0x00,0x05,0x00, -0x35,0x00,0x00,0x00,0x9b,0x02,0x00,0x00,0x98,0x02,0x00,0x00, -0x38,0x01,0x00,0x00,0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x9c,0x02,0x00,0x00,0x9b,0x02,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x9d,0x02,0x00,0x00,0x9c,0x02,0x00,0x00, -0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9e,0x02,0x00,0x00, -0x9d,0x02,0x00,0x00,0x81,0x00,0x00,0x00,0xc4,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x9f,0x02,0x00,0x00,0x9e,0x02,0x00,0x00, -0x39,0x00,0x00,0x00,0xc5,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xa0,0x02,0x00,0x00,0x93,0x02,0x00,0x00,0x9f,0x02,0x00,0x00, -0x72,0x00,0x04,0x00,0x3d,0x00,0x00,0x00,0xa1,0x02,0x00,0x00, -0xa0,0x02,0x00,0x00,0x72,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xa2,0x02,0x00,0x00,0xa1,0x02,0x00,0x00,0x82,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xa3,0x02,0x00,0x00,0xa2,0x02,0x00,0x00, -0x44,0x00,0x00,0x00,0x6f,0x00,0x04,0x00,0x4f,0x00,0x00,0x00, -0xa4,0x02,0x00,0x00,0xa3,0x02,0x00,0x00,0x85,0x00,0x05,0x00, -0x4f,0x00,0x00,0x00,0xa5,0x02,0x00,0x00,0x88,0x02,0x00,0x00, -0xa4,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0x35,0x00,0x00,0x00, -0xae,0x02,0x00,0x00,0xd6,0x01,0x00,0x00,0xc2,0x00,0x05,0x00, -0x35,0x00,0x00,0x00,0xaf,0x02,0x00,0x00,0xae,0x02,0x00,0x00, -0x70,0x01,0x00,0x00,0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0xb0,0x02,0x00,0x00,0xaf,0x02,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xb1,0x02,0x00,0x00,0xb0,0x02,0x00,0x00, -0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xb2,0x02,0x00,0x00, -0xb1,0x02,0x00,0x00,0x81,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x35,0x00,0x00,0x00,0xbb,0x02,0x00,0x00,0xe2,0x01,0x00,0x00, -0xc7,0x00,0x05,0x00,0x35,0x00,0x00,0x00,0xbe,0x02,0x00,0x00, -0xbb,0x02,0x00,0x00,0x9f,0x01,0x00,0x00,0x71,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0xbf,0x02,0x00,0x00,0xbe,0x02,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xc0,0x02,0x00,0x00, -0xbf,0x02,0x00,0x00,0xab,0x00,0x05,0x00,0x6a,0x00,0x00,0x00, -0xc1,0x02,0x00,0x00,0xc0,0x02,0x00,0x00,0x16,0x00,0x00,0x00, -0xa9,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xc2,0x02,0x00,0x00, -0xc1,0x02,0x00,0x00,0x16,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xc3,0x02,0x00,0x00, -0xb2,0x02,0x00,0x00,0xc2,0x02,0x00,0x00,0x6f,0x00,0x04,0x00, -0x4f,0x00,0x00,0x00,0xc4,0x02,0x00,0x00,0xc3,0x02,0x00,0x00, -0x0c,0x00,0x08,0x00,0x4f,0x00,0x00,0x00,0xc6,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0xa5,0x02,0x00,0x00, -0xc4,0x02,0x00,0x00,0x7e,0x02,0x00,0x00,0x81,0x00,0x05,0x00, -0x4f,0x00,0x00,0x00,0xc8,0x02,0x00,0x00,0x07,0x03,0x00,0x00, -0xc6,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xca,0x02,0x00,0x00,0x06,0x03,0x00,0x00,0x38,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x88,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x8a,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x4f,0x00,0x00,0x00, -0xd3,0x02,0x00,0x00,0x5b,0x00,0x00,0x00,0x0c,0x00,0x08,0x00, -0x4f,0x00,0x00,0x00,0xd4,0x02,0x00,0x00,0x01,0x00,0x00,0x00, -0x32,0x00,0x00,0x00,0x85,0x00,0x00,0x00,0x07,0x03,0x00,0x00, -0xd3,0x02,0x00,0x00,0x3e,0x00,0x03,0x00,0x5b,0x00,0x00,0x00, -0xd4,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x66,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0x66,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xd7,0x02,0x00,0x00,0x03,0x03,0x00,0x00, -0x25,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x63,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0x65,0x00,0x00,0x00,0xe0,0x00,0x04,0x00, -0x37,0x01,0x00,0x00,0x37,0x01,0x00,0x00,0xd8,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0xda,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0xda,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0x04,0x03,0x00,0x00,0x54,0x00,0x00,0x00,0x65,0x00,0x00,0x00, -0xf1,0x02,0x00,0x00,0xdd,0x02,0x00,0x00,0xad,0x00,0x05,0x00, -0x6a,0x00,0x00,0x00,0xe0,0x02,0x00,0x00,0x04,0x03,0x00,0x00, -0x16,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xdc,0x02,0x00,0x00, -0xdd,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0xe0,0x02,0x00,0x00,0xdb,0x02,0x00,0x00,0xdc,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0xdb,0x02,0x00,0x00,0xb1,0x00,0x05,0x00, -0x6a,0x00,0x00,0x00,0xe3,0x02,0x00,0x00,0x26,0x00,0x00,0x00, -0x04,0x03,0x00,0x00,0xf7,0x00,0x03,0x00,0xe5,0x02,0x00,0x00, -0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xe3,0x02,0x00,0x00, -0xe4,0x02,0x00,0x00,0xe5,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0xe4,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xe9,0x02,0x00,0x00,0x26,0x00,0x00,0x00,0x04,0x03,0x00,0x00, -0x41,0x00,0x05,0x00,0x5a,0x00,0x00,0x00,0xea,0x02,0x00,0x00, -0x53,0x00,0x00,0x00,0xe9,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4f,0x00,0x00,0x00,0xeb,0x02,0x00,0x00,0xea,0x02,0x00,0x00, -0x41,0x00,0x05,0x00,0x5a,0x00,0x00,0x00,0xec,0x02,0x00,0x00, -0x53,0x00,0x00,0x00,0x26,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4f,0x00,0x00,0x00,0xed,0x02,0x00,0x00,0xec,0x02,0x00,0x00, -0x81,0x00,0x05,0x00,0x4f,0x00,0x00,0x00,0xee,0x02,0x00,0x00, -0xed,0x02,0x00,0x00,0xeb,0x02,0x00,0x00,0x3e,0x00,0x03,0x00, -0xec,0x02,0x00,0x00,0xee,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0xe5,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0xe5,0x02,0x00,0x00, -0xe0,0x00,0x04,0x00,0x37,0x01,0x00,0x00,0x37,0x01,0x00,0x00, -0xd8,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0xdd,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0xdd,0x02,0x00,0x00,0xc3,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xf1,0x02,0x00,0x00,0x04,0x03,0x00,0x00, -0x38,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xda,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0xdc,0x02,0x00,0x00,0xaa,0x00,0x05,0x00, -0x6a,0x00,0x00,0x00,0xf3,0x02,0x00,0x00,0x26,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0xf7,0x00,0x03,0x00,0xf5,0x02,0x00,0x00, -0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xf3,0x02,0x00,0x00, -0xf4,0x02,0x00,0x00,0xf5,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0xf4,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00, -0xfa,0x02,0x00,0x00,0x15,0x00,0x00,0x00,0x25,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xfb,0x02,0x00,0x00, -0xfa,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xfd,0x02,0x00,0x00,0xfb,0x02,0x00,0x00,0x11,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x5a,0x00,0x00,0x00,0xfe,0x02,0x00,0x00, -0x53,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4f,0x00,0x00,0x00,0xff,0x02,0x00,0x00,0xfe,0x02,0x00,0x00, -0x41,0x00,0x06,0x00,0x9a,0x00,0x00,0x00,0x00,0x03,0x00,0x00, -0xf9,0x02,0x00,0x00,0x16,0x00,0x00,0x00,0xfd,0x02,0x00,0x00, -0x3e,0x00,0x03,0x00,0x00,0x03,0x00,0x00,0xff,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0xf5,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0xf5,0x02,0x00,0x00,0xfd,0x00,0x01,0x00,0x38,0x00,0x01,0x00, - -}; -const uint64_t mul_mat_vec_q3_K_f32_len = 9252; +const uint64_t mul_mat_vec_q3_K_f32_len = 9236; unsigned char mul_mat_vec_q4_0_f32_data[] = { 0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, -0xd1,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, +0xcb,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, 0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x51,0x11,0x00,0x00, 0x11,0x00,0x02,0x00,0x60,0x11,0x00,0x00,0x0b,0x00,0x06,0x00, 0x01,0x00,0x00,0x00,0x47,0x4c,0x53,0x4c,0x2e,0x73,0x74,0x64, 0x2e,0x34,0x35,0x30,0x00,0x00,0x00,0x00,0x0e,0x00,0x03,0x00, 0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x0f,0x00,0x0c,0x00, 0x05,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x6d,0x61,0x69,0x6e, -0x00,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x13,0x00,0x00,0x00, -0x1b,0x00,0x00,0x00,0x2a,0x00,0x00,0x00,0x54,0x00,0x00,0x00, -0x7b,0x00,0x00,0x00,0xc2,0x00,0x00,0x00,0x10,0x00,0x06,0x00, -0x04,0x00,0x00,0x00,0x11,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x11,0x00,0x00,0x00, +0x18,0x00,0x00,0x00,0x26,0x00,0x00,0x00,0x52,0x00,0x00,0x00, +0x78,0x00,0x00,0x00,0xba,0x00,0x00,0x00,0x10,0x00,0x06,0x00, +0x04,0x00,0x00,0x00,0x11,0x00,0x00,0x00,0x01,0x00,0x00,0x00, 0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x0c,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x1a,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x1b,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x28,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x28,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x28,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x08,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x28,0x00,0x00,0x00, +0x0b,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x1a,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x11,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, +0x1b,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x15,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x24,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x24,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x24,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x47,0x00,0x03,0x00, +0x24,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x4d,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x4e,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x4e,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00, 0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x4f,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x50,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x50,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x51,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0x52,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x52,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x52,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x54,0x00,0x00,0x00, -0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x54,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x78,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0x79,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x79,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x79,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x7b,0x00,0x00,0x00, -0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x7b,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0xbf,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0xc0,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0xc0,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0xc0,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xc2,0x00,0x00,0x00, -0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0xc2,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0xca,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x19,0x00,0x00,0x00,0x13,0x00,0x02,0x00,0x02,0x00,0x00,0x00, -0x21,0x00,0x03,0x00,0x03,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x15,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x15,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x17,0x00,0x04,0x00, -0x0a,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x03,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x0b,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x0a,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x0b,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x0e,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x0b,0x00,0x00,0x00, -0x13,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x16,0x00,0x03,0x00, -0x17,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x1c,0x00,0x04,0x00,0x19,0x00,0x00,0x00,0x17,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x1a,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x1a,0x00,0x00,0x00,0x1b,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x17,0x00,0x00,0x00,0x1d,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x1e,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x17,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x1e,0x00,0x05,0x00,0x28,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x48,0x00,0x04,0x00, +0x50,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x50,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, +0x50,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x52,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x52,0x00,0x00,0x00,0x21,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x75,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00, +0x76,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x76,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, +0x76,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x78,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x78,0x00,0x00,0x00,0x21,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xb7,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00, +0xb8,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x19,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0xb8,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, +0xb8,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0xba,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0xba,0x00,0x00,0x00,0x21,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xc3,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0xc4,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x19,0x00,0x00,0x00, +0x13,0x00,0x02,0x00,0x02,0x00,0x00,0x00,0x21,0x00,0x03,0x00, +0x03,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x15,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x17,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x03,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x0a,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0x0a,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x0d,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0x0a,0x00,0x00,0x00,0x11,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x16,0x00,0x03,0x00,0x14,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x15,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0x1c,0x00,0x04,0x00,0x16,0x00,0x00,0x00, +0x14,0x00,0x00,0x00,0x15,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x17,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x16,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0x17,0x00,0x00,0x00,0x18,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x14,0x00,0x00,0x00, +0x1a,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x1b,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x14,0x00,0x00,0x00, +0x1e,0x00,0x05,0x00,0x24,0x00,0x00,0x00,0x06,0x00,0x00,0x00, 0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x29,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x28,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x29,0x00,0x00,0x00,0x2a,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x2b,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x14,0x00,0x02,0x00,0x30,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x35,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x16,0x00,0x03,0x00,0x4c,0x00,0x00,0x00,0x10,0x00,0x00,0x00, -0x15,0x00,0x04,0x00,0x4d,0x00,0x00,0x00,0x08,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x4e,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x1c,0x00,0x04,0x00, -0x4f,0x00,0x00,0x00,0x4d,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, -0x1e,0x00,0x04,0x00,0x50,0x00,0x00,0x00,0x4c,0x00,0x00,0x00, -0x4f,0x00,0x00,0x00,0x1d,0x00,0x03,0x00,0x51,0x00,0x00,0x00, -0x50,0x00,0x00,0x00,0x1e,0x00,0x03,0x00,0x52,0x00,0x00,0x00, -0x51,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x53,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x52,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x53,0x00,0x00,0x00,0x54,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x56,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x4c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x5d,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x5f,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x4d,0x00,0x00,0x00, -0x17,0x00,0x04,0x00,0x63,0x00,0x00,0x00,0x17,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x67,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x6b,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x17,0x00,0x00,0x00,0x70,0x00,0x00,0x00, -0x00,0x00,0x00,0x41,0x1d,0x00,0x03,0x00,0x78,0x00,0x00,0x00, -0x17,0x00,0x00,0x00,0x1e,0x00,0x03,0x00,0x79,0x00,0x00,0x00, -0x78,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x7a,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x79,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x7a,0x00,0x00,0x00,0x7b,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x83,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x17,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x8c,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x95,0x00,0x00,0x00,0x10,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0xa0,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0xa1,0x00,0x00,0x00,0x08,0x01,0x00,0x00,0x1d,0x00,0x03,0x00, -0xbf,0x00,0x00,0x00,0x17,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, -0xc0,0x00,0x00,0x00,0xbf,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0xc1,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0xc0,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0xc1,0x00,0x00,0x00,0xc2,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x2c,0x00,0x06,0x00,0x0a,0x00,0x00,0x00, -0xca,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x8c,0x00,0x00,0x00, -0x8c,0x00,0x00,0x00,0x2c,0x00,0x05,0x00,0x63,0x00,0x00,0x00, -0xd0,0x00,0x00,0x00,0x70,0x00,0x00,0x00,0x70,0x00,0x00,0x00, -0x36,0x00,0x05,0x00,0x02,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x05,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x0e,0x00,0x00,0x00, -0x0f,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x0d,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x10,0x00,0x00,0x00, -0x0f,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x11,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x0e,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x13,0x00,0x00,0x00, -0x0d,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x15,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x15,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x1e,0x00,0x00,0x00,0x1f,0x00,0x00,0x00, -0x1b,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x3e,0x00,0x03,0x00, -0x1f,0x00,0x00,0x00,0x1d,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x22,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x22,0x00,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xcd,0x00,0x00,0x00, -0x21,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0x9f,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x2b,0x00,0x00,0x00, -0x2c,0x00,0x00,0x00,0x2a,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x2d,0x00,0x00,0x00, -0x2c,0x00,0x00,0x00,0x87,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x2f,0x00,0x00,0x00,0x2d,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0xb1,0x00,0x05,0x00,0x30,0x00,0x00,0x00,0x31,0x00,0x00,0x00, -0xcd,0x00,0x00,0x00,0x2f,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0x24,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x31,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x24,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x23,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x34,0x00,0x00,0x00, -0xcd,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x35,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x38,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0x37,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3d,0x00,0x00,0x00, -0x11,0x00,0x00,0x00,0x2d,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x3f,0x00,0x00,0x00,0x3d,0x00,0x00,0x00, -0x38,0x00,0x00,0x00,0x87,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0x3f,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x8b,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x43,0x00,0x00,0x00, -0x38,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x87,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x44,0x00,0x00,0x00,0x43,0x00,0x00,0x00, -0x35,0x00,0x00,0x00,0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x49,0x00,0x00,0x00,0x38,0x00,0x00,0x00,0x43,0x00,0x00,0x00, -0x41,0x00,0x07,0x00,0x56,0x00,0x00,0x00,0x57,0x00,0x00,0x00, -0x54,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x21,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x4c,0x00,0x00,0x00, -0x58,0x00,0x00,0x00,0x57,0x00,0x00,0x00,0x73,0x00,0x04,0x00, -0x17,0x00,0x00,0x00,0x59,0x00,0x00,0x00,0x58,0x00,0x00,0x00, -0x41,0x00,0x08,0x00,0x5f,0x00,0x00,0x00,0x60,0x00,0x00,0x00, -0x54,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x5d,0x00,0x00,0x00,0x44,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0x61,0x00,0x00,0x00,0x60,0x00,0x00,0x00, -0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x62,0x00,0x00,0x00, -0x61,0x00,0x00,0x00,0xc7,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0x68,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x67,0x00,0x00,0x00, -0x70,0x00,0x04,0x00,0x17,0x00,0x00,0x00,0x69,0x00,0x00,0x00, -0x68,0x00,0x00,0x00,0xc2,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x6b,0x00,0x00,0x00, -0x70,0x00,0x04,0x00,0x17,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x50,0x00,0x05,0x00,0x63,0x00,0x00,0x00, -0x6e,0x00,0x00,0x00,0x69,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, -0x83,0x00,0x05,0x00,0x63,0x00,0x00,0x00,0x72,0x00,0x00,0x00, -0x6e,0x00,0x00,0x00,0xd0,0x00,0x00,0x00,0x8e,0x00,0x05,0x00, -0x63,0x00,0x00,0x00,0x74,0x00,0x00,0x00,0x72,0x00,0x00,0x00, -0x59,0x00,0x00,0x00,0x51,0x00,0x05,0x00,0x17,0x00,0x00,0x00, -0x77,0x00,0x00,0x00,0x74,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x2b,0x00,0x00,0x00,0x7c,0x00,0x00,0x00, -0x2a,0x00,0x00,0x00,0x5d,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x7d,0x00,0x00,0x00,0x7c,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x7f,0x00,0x00,0x00, -0x7d,0x00,0x00,0x00,0x49,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x81,0x00,0x00,0x00,0x7f,0x00,0x00,0x00, -0x44,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x83,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0x7b,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x81,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x17,0x00,0x00,0x00, -0x85,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x17,0x00,0x00,0x00,0x88,0x00,0x00,0x00,0x1f,0x00,0x00,0x00, -0x0c,0x00,0x08,0x00,0x17,0x00,0x00,0x00,0x89,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x77,0x00,0x00,0x00, -0x85,0x00,0x00,0x00,0x88,0x00,0x00,0x00,0x3e,0x00,0x03,0x00, -0x1f,0x00,0x00,0x00,0x89,0x00,0x00,0x00,0x51,0x00,0x05,0x00, -0x17,0x00,0x00,0x00,0x8e,0x00,0x00,0x00,0x74,0x00,0x00,0x00, +0x25,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x24,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0x25,0x00,0x00,0x00,0x26,0x00,0x00,0x00, +0x09,0x00,0x00,0x00,0x15,0x00,0x04,0x00,0x27,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x27,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x29,0x00,0x00,0x00,0x09,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x14,0x00,0x02,0x00,0x2d,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x32,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x3d,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x16,0x00,0x03,0x00, +0x4a,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x15,0x00,0x04,0x00, +0x4b,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x4c,0x00,0x00,0x00, +0x10,0x00,0x00,0x00,0x1c,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, +0x4b,0x00,0x00,0x00,0x4c,0x00,0x00,0x00,0x1e,0x00,0x04,0x00, +0x4e,0x00,0x00,0x00,0x4a,0x00,0x00,0x00,0x4d,0x00,0x00,0x00, +0x1d,0x00,0x03,0x00,0x4f,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, +0x1e,0x00,0x03,0x00,0x50,0x00,0x00,0x00,0x4f,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x51,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x50,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x51,0x00,0x00,0x00, +0x52,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x54,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x4a,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x27,0x00,0x00,0x00,0x5a,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x5c,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x4b,0x00,0x00,0x00,0x17,0x00,0x04,0x00, +0x60,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x64,0x00,0x00,0x00, +0x0f,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x27,0x00,0x00,0x00, +0x68,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x14,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x00,0x00,0x00,0x41, +0x1d,0x00,0x03,0x00,0x75,0x00,0x00,0x00,0x14,0x00,0x00,0x00, +0x1e,0x00,0x03,0x00,0x76,0x00,0x00,0x00,0x75,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x77,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x76,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x77,0x00,0x00,0x00, +0x78,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x80,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x14,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x98,0x00,0x00,0x00,0x08,0x01,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x9a,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0x15,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, +0xb7,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, +0xb8,0x00,0x00,0x00,0xb7,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0xb9,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0xb8,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0xb9,0x00,0x00,0x00,0xba,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x27,0x00,0x00,0x00, +0xbb,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x32,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xc3,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x33,0x00,0x06,0x00,0x09,0x00,0x00,0x00,0xc4,0x00,0x00,0x00, +0xc3,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0x2c,0x00,0x05,0x00,0x60,0x00,0x00,0x00,0xca,0x00,0x00,0x00, +0x6d,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x36,0x00,0x05,0x00, +0x02,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x03,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x05,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00,0x0e,0x00,0x00,0x00, +0x0b,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x0e,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0x11,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x13,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x1b,0x00,0x00,0x00,0x1c,0x00,0x00,0x00, +0x18,0x00,0x00,0x00,0x13,0x00,0x00,0x00,0x3e,0x00,0x03,0x00, +0x1c,0x00,0x00,0x00,0x1a,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x1e,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x1e,0x00,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xc7,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0x97,0x00,0x00,0x00, +0x1f,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x29,0x00,0x00,0x00, +0x2a,0x00,0x00,0x00,0x26,0x00,0x00,0x00,0x28,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x2b,0x00,0x00,0x00, +0x2a,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x2c,0x00,0x00,0x00,0x2b,0x00,0x00,0x00,0x15,0x00,0x00,0x00, +0xb0,0x00,0x05,0x00,0x2d,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, +0xc7,0x00,0x00,0x00,0x2c,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0x20,0x00,0x00,0x00,0x1f,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x2e,0x00,0x00,0x00,0x1f,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x1f,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x31,0x00,0x00,0x00, +0xc7,0x00,0x00,0x00,0x15,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0x32,0x00,0x00,0x00, +0x13,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x35,0x00,0x00,0x00,0x31,0x00,0x00,0x00,0x34,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, +0x0f,0x00,0x00,0x00,0x2b,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x3c,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, +0x35,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x3e,0x00,0x00,0x00,0x3c,0x00,0x00,0x00,0x3d,0x00,0x00,0x00, +0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x41,0x00,0x00,0x00, +0x35,0x00,0x00,0x00,0x3d,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x42,0x00,0x00,0x00,0x41,0x00,0x00,0x00, +0x32,0x00,0x00,0x00,0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x47,0x00,0x00,0x00,0x35,0x00,0x00,0x00,0x41,0x00,0x00,0x00, +0x41,0x00,0x07,0x00,0x54,0x00,0x00,0x00,0x55,0x00,0x00,0x00, +0x52,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0x3e,0x00,0x00,0x00, +0x28,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, +0x56,0x00,0x00,0x00,0x55,0x00,0x00,0x00,0x73,0x00,0x04,0x00, +0x14,0x00,0x00,0x00,0x57,0x00,0x00,0x00,0x56,0x00,0x00,0x00, +0x41,0x00,0x08,0x00,0x5c,0x00,0x00,0x00,0x5d,0x00,0x00,0x00, +0x52,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0x3e,0x00,0x00,0x00, +0x5a,0x00,0x00,0x00,0x42,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x4b,0x00,0x00,0x00,0x5e,0x00,0x00,0x00,0x5d,0x00,0x00,0x00, +0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x5f,0x00,0x00,0x00, +0x5e,0x00,0x00,0x00,0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x65,0x00,0x00,0x00,0x5f,0x00,0x00,0x00,0x64,0x00,0x00,0x00, +0x70,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0x66,0x00,0x00,0x00, +0x65,0x00,0x00,0x00,0xc2,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x69,0x00,0x00,0x00,0x5f,0x00,0x00,0x00,0x68,0x00,0x00,0x00, +0x70,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0x6a,0x00,0x00,0x00, +0x69,0x00,0x00,0x00,0x50,0x00,0x05,0x00,0x60,0x00,0x00,0x00, +0x6b,0x00,0x00,0x00,0x66,0x00,0x00,0x00,0x6a,0x00,0x00,0x00, +0x83,0x00,0x05,0x00,0x60,0x00,0x00,0x00,0x6f,0x00,0x00,0x00, +0x6b,0x00,0x00,0x00,0xca,0x00,0x00,0x00,0x8e,0x00,0x05,0x00, +0x60,0x00,0x00,0x00,0x71,0x00,0x00,0x00,0x6f,0x00,0x00,0x00, +0x57,0x00,0x00,0x00,0x51,0x00,0x05,0x00,0x14,0x00,0x00,0x00, +0x74,0x00,0x00,0x00,0x71,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x29,0x00,0x00,0x00,0x79,0x00,0x00,0x00, +0x26,0x00,0x00,0x00,0x5a,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x7a,0x00,0x00,0x00,0x79,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x7c,0x00,0x00,0x00, +0x7a,0x00,0x00,0x00,0x47,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x7e,0x00,0x00,0x00,0x7c,0x00,0x00,0x00, +0x42,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x80,0x00,0x00,0x00, +0x81,0x00,0x00,0x00,0x78,0x00,0x00,0x00,0x28,0x00,0x00,0x00, +0x7e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x14,0x00,0x00,0x00, +0x82,0x00,0x00,0x00,0x81,0x00,0x00,0x00,0x51,0x00,0x05,0x00, +0x14,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x71,0x00,0x00,0x00, 0x01,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x96,0x00,0x00,0x00,0x81,0x00,0x00,0x00,0x95,0x00,0x00,0x00, -0x41,0x00,0x06,0x00,0x83,0x00,0x00,0x00,0x97,0x00,0x00,0x00, -0x7b,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x96,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x17,0x00,0x00,0x00,0x98,0x00,0x00,0x00, -0x97,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x17,0x00,0x00,0x00, -0x9b,0x00,0x00,0x00,0x1f,0x00,0x00,0x00,0x0c,0x00,0x08,0x00, -0x17,0x00,0x00,0x00,0x9c,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x32,0x00,0x00,0x00,0x8e,0x00,0x00,0x00,0x98,0x00,0x00,0x00, -0x9b,0x00,0x00,0x00,0x3e,0x00,0x03,0x00,0x1f,0x00,0x00,0x00, -0x9c,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x9f,0x00,0x00,0x00,0xcd,0x00,0x00,0x00,0x35,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x22,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x24,0x00,0x00,0x00,0xe0,0x00,0x04,0x00,0xa0,0x00,0x00,0x00, -0xa0,0x00,0x00,0x00,0xa1,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xa3,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xa3,0x00,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xce,0x00,0x00,0x00, -0x95,0x00,0x00,0x00,0x24,0x00,0x00,0x00,0xba,0x00,0x00,0x00, -0xa6,0x00,0x00,0x00,0xad,0x00,0x05,0x00,0x30,0x00,0x00,0x00, -0xa9,0x00,0x00,0x00,0xce,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0xa5,0x00,0x00,0x00,0xa6,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xa9,0x00,0x00,0x00, -0xa4,0x00,0x00,0x00,0xa5,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xa4,0x00,0x00,0x00,0xb1,0x00,0x05,0x00,0x30,0x00,0x00,0x00, -0xac,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0xce,0x00,0x00,0x00, -0xf7,0x00,0x03,0x00,0xae,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xac,0x00,0x00,0x00,0xad,0x00,0x00,0x00, -0xae,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xad,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xb2,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0xce,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x1e,0x00,0x00,0x00,0xb3,0x00,0x00,0x00,0x1b,0x00,0x00,0x00, -0xb2,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x17,0x00,0x00,0x00, -0xb4,0x00,0x00,0x00,0xb3,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x17,0x00,0x00,0x00,0xb6,0x00,0x00,0x00,0x1f,0x00,0x00,0x00, -0x81,0x00,0x05,0x00,0x17,0x00,0x00,0x00,0xb7,0x00,0x00,0x00, -0xb6,0x00,0x00,0x00,0xb4,0x00,0x00,0x00,0x3e,0x00,0x03,0x00, -0x1f,0x00,0x00,0x00,0xb7,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xae,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xae,0x00,0x00,0x00, -0xe0,0x00,0x04,0x00,0xa0,0x00,0x00,0x00,0xa0,0x00,0x00,0x00, -0xa1,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xa6,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xa6,0x00,0x00,0x00,0xc3,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xba,0x00,0x00,0x00,0xce,0x00,0x00,0x00, -0x5d,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xa3,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xa5,0x00,0x00,0x00,0xaa,0x00,0x05,0x00, -0x30,0x00,0x00,0x00,0xbc,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0x21,0x00,0x00,0x00,0xf7,0x00,0x03,0x00,0xbe,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xbc,0x00,0x00,0x00, -0xbd,0x00,0x00,0x00,0xbe,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xbd,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x2b,0x00,0x00,0x00, -0xc3,0x00,0x00,0x00,0x2a,0x00,0x00,0x00,0x35,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xc4,0x00,0x00,0x00, -0xc3,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xc6,0x00,0x00,0x00,0xc4,0x00,0x00,0x00,0x11,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x1e,0x00,0x00,0x00,0xc7,0x00,0x00,0x00, -0x1b,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x17,0x00,0x00,0x00,0xc8,0x00,0x00,0x00,0xc7,0x00,0x00,0x00, -0x41,0x00,0x06,0x00,0x83,0x00,0x00,0x00,0xc9,0x00,0x00,0x00, -0xc2,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0xc6,0x00,0x00,0x00, -0x3e,0x00,0x03,0x00,0xc9,0x00,0x00,0x00,0xc8,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xbe,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xbe,0x00,0x00,0x00,0xfd,0x00,0x01,0x00,0x38,0x00,0x01,0x00, - +0x8d,0x00,0x00,0x00,0x7e,0x00,0x00,0x00,0x4c,0x00,0x00,0x00, +0x41,0x00,0x06,0x00,0x80,0x00,0x00,0x00,0x8e,0x00,0x00,0x00, +0x78,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0x8d,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0x8f,0x00,0x00,0x00, +0x8e,0x00,0x00,0x00,0x85,0x00,0x05,0x00,0x14,0x00,0x00,0x00, +0x90,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x8f,0x00,0x00,0x00, +0x0c,0x00,0x08,0x00,0x14,0x00,0x00,0x00,0x91,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x74,0x00,0x00,0x00, +0x82,0x00,0x00,0x00,0x90,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x14,0x00,0x00,0x00,0x93,0x00,0x00,0x00,0x1c,0x00,0x00,0x00, +0x81,0x00,0x05,0x00,0x14,0x00,0x00,0x00,0x94,0x00,0x00,0x00, +0x93,0x00,0x00,0x00,0x91,0x00,0x00,0x00,0x3e,0x00,0x03,0x00, +0x1c,0x00,0x00,0x00,0x94,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x97,0x00,0x00,0x00,0xc7,0x00,0x00,0x00, +0x32,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x1e,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0x20,0x00,0x00,0x00,0xe0,0x00,0x04,0x00, +0x32,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x98,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x9b,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0x9b,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xc8,0x00,0x00,0x00,0x9a,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0xb2,0x00,0x00,0x00,0x9e,0x00,0x00,0x00,0xac,0x00,0x05,0x00, +0x2d,0x00,0x00,0x00,0xa1,0x00,0x00,0x00,0xc8,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x9d,0x00,0x00,0x00, +0x9e,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0xa1,0x00,0x00,0x00,0x9c,0x00,0x00,0x00,0x9d,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0x9c,0x00,0x00,0x00,0xb0,0x00,0x05,0x00, +0x2d,0x00,0x00,0x00,0xa4,0x00,0x00,0x00,0x13,0x00,0x00,0x00, +0xc8,0x00,0x00,0x00,0xf7,0x00,0x03,0x00,0xa6,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xa4,0x00,0x00,0x00, +0xa5,0x00,0x00,0x00,0xa6,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xa5,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xaa,0x00,0x00,0x00,0x13,0x00,0x00,0x00,0xc8,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x1b,0x00,0x00,0x00,0xab,0x00,0x00,0x00, +0x18,0x00,0x00,0x00,0xaa,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x14,0x00,0x00,0x00,0xac,0x00,0x00,0x00,0xab,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0xae,0x00,0x00,0x00, +0x1c,0x00,0x00,0x00,0x81,0x00,0x05,0x00,0x14,0x00,0x00,0x00, +0xaf,0x00,0x00,0x00,0xae,0x00,0x00,0x00,0xac,0x00,0x00,0x00, +0x3e,0x00,0x03,0x00,0x1c,0x00,0x00,0x00,0xaf,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xa6,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xa6,0x00,0x00,0x00,0xe0,0x00,0x04,0x00,0x32,0x00,0x00,0x00, +0x32,0x00,0x00,0x00,0x98,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x9e,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x9e,0x00,0x00,0x00, +0xc2,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xb2,0x00,0x00,0x00, +0xc8,0x00,0x00,0x00,0x5a,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x9b,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x9d,0x00,0x00,0x00, +0xaa,0x00,0x05,0x00,0x2d,0x00,0x00,0x00,0xb4,0x00,0x00,0x00, +0x13,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0xf7,0x00,0x03,0x00, +0xb6,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0xb4,0x00,0x00,0x00,0xb5,0x00,0x00,0x00,0xb6,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xb5,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x29,0x00,0x00,0x00,0xbc,0x00,0x00,0x00,0x26,0x00,0x00,0x00, +0xbb,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xbd,0x00,0x00,0x00,0xbc,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xbf,0x00,0x00,0x00,0xbd,0x00,0x00,0x00, +0x0f,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x1b,0x00,0x00,0x00, +0xc0,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x28,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0xc1,0x00,0x00,0x00, +0xc0,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x80,0x00,0x00,0x00, +0xc2,0x00,0x00,0x00,0xba,0x00,0x00,0x00,0x28,0x00,0x00,0x00, +0xbf,0x00,0x00,0x00,0x3e,0x00,0x03,0x00,0xc2,0x00,0x00,0x00, +0xc1,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xb6,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xb6,0x00,0x00,0x00,0xfd,0x00,0x01,0x00, +0x38,0x00,0x01,0x00, }; -const uint64_t mul_mat_vec_q4_0_f32_len = 3180; +const uint64_t mul_mat_vec_q4_0_f32_len = 3184; unsigned char mul_mat_vec_q4_1_f32_data[] = { 0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, -0xd4,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, +0xce,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, 0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x51,0x11,0x00,0x00, 0x11,0x00,0x02,0x00,0x60,0x11,0x00,0x00,0x0b,0x00,0x06,0x00, 0x01,0x00,0x00,0x00,0x47,0x4c,0x53,0x4c,0x2e,0x73,0x74,0x64, 0x2e,0x34,0x35,0x30,0x00,0x00,0x00,0x00,0x0e,0x00,0x03,0x00, 0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x0f,0x00,0x0c,0x00, 0x05,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x6d,0x61,0x69,0x6e, -0x00,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x13,0x00,0x00,0x00, -0x1b,0x00,0x00,0x00,0x2a,0x00,0x00,0x00,0x54,0x00,0x00,0x00, -0x80,0x00,0x00,0x00,0xc7,0x00,0x00,0x00,0x10,0x00,0x06,0x00, -0x04,0x00,0x00,0x00,0x11,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x11,0x00,0x00,0x00, +0x18,0x00,0x00,0x00,0x26,0x00,0x00,0x00,0x52,0x00,0x00,0x00, +0x7e,0x00,0x00,0x00,0xc0,0x00,0x00,0x00,0x10,0x00,0x06,0x00, +0x04,0x00,0x00,0x00,0x11,0x00,0x00,0x00,0x01,0x00,0x00,0x00, 0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x0c,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x1a,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x1b,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x28,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x28,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x28,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x08,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x28,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x4f,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x0b,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x1a,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x11,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, +0x1b,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x15,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x24,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x24,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x24,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x47,0x00,0x03,0x00, +0x24,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x4d,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x4e,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x4e,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x4e,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x4f,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x14,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0x50,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00, 0x50,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x50,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x50,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x51,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x14,0x00,0x00,0x00, -0x48,0x00,0x04,0x00,0x52,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x52,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x03,0x00,0x52,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x54,0x00,0x00,0x00,0x22,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x54,0x00,0x00,0x00, -0x21,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x7d,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x48,0x00,0x04,0x00,0x7e,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x7e,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x03,0x00,0x7e,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x80,0x00,0x00,0x00,0x22,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x80,0x00,0x00,0x00, -0x21,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0xc4,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x48,0x00,0x04,0x00,0xc5,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x19,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0xc5,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x03,0x00,0xc5,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0xc7,0x00,0x00,0x00,0x22,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xc7,0x00,0x00,0x00, -0x21,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0xcf,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x19,0x00,0x00,0x00, -0x13,0x00,0x02,0x00,0x02,0x00,0x00,0x00,0x21,0x00,0x03,0x00, -0x03,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x15,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x15,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x17,0x00,0x04,0x00,0x0a,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x0b,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x0a,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x0b,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x0d,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x0e,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x0b,0x00,0x00,0x00,0x13,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x16,0x00,0x03,0x00,0x17,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x1c,0x00,0x04,0x00, -0x19,0x00,0x00,0x00,0x17,0x00,0x00,0x00,0x18,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x1a,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x19,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x1a,0x00,0x00,0x00, -0x1b,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x17,0x00,0x00,0x00,0x1d,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x1e,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x17,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x21,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x1e,0x00,0x05,0x00, -0x28,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x29,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x29,0x00,0x00,0x00,0x2a,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x2b,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x14,0x00,0x02,0x00, -0x30,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x35,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x16,0x00,0x03,0x00, -0x4c,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x15,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, -0x10,0x00,0x00,0x00,0x1c,0x00,0x04,0x00,0x4f,0x00,0x00,0x00, -0x4d,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x1e,0x00,0x05,0x00, -0x50,0x00,0x00,0x00,0x4c,0x00,0x00,0x00,0x4c,0x00,0x00,0x00, -0x4f,0x00,0x00,0x00,0x1d,0x00,0x03,0x00,0x51,0x00,0x00,0x00, -0x50,0x00,0x00,0x00,0x1e,0x00,0x03,0x00,0x52,0x00,0x00,0x00, -0x51,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x53,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x52,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x53,0x00,0x00,0x00,0x54,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x56,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x4c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x5c,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x64,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x4d,0x00,0x00,0x00, -0x17,0x00,0x04,0x00,0x68,0x00,0x00,0x00,0x17,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x70,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x1d,0x00,0x03,0x00,0x7d,0x00,0x00,0x00,0x17,0x00,0x00,0x00, -0x1e,0x00,0x03,0x00,0x7e,0x00,0x00,0x00,0x7d,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x7f,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x7e,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x7f,0x00,0x00,0x00, -0x80,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x88,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x17,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x91,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x50,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x52,0x00,0x00,0x00, +0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x52,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x7b,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0x7c,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x7c,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x7c,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x7e,0x00,0x00,0x00, +0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x7e,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0xbd,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0xbe,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0xbe,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xc0,0x00,0x00,0x00, +0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0xc0,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0xc8,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xc9,0x00,0x00,0x00, +0x0b,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x13,0x00,0x02,0x00, +0x02,0x00,0x00,0x00,0x21,0x00,0x03,0x00,0x03,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x15,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x17,0x00,0x04,0x00, +0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x03,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x0a,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x09,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00, +0x0b,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x0d,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00, +0x11,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x16,0x00,0x03,0x00, +0x14,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x32,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x15,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0x1c,0x00,0x04,0x00,0x16,0x00,0x00,0x00,0x14,0x00,0x00,0x00, +0x15,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x17,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0x17,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0x1a,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x1b,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x1e,0x00,0x05,0x00, +0x24,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x25,0x00,0x00,0x00, +0x09,0x00,0x00,0x00,0x24,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0x25,0x00,0x00,0x00,0x26,0x00,0x00,0x00,0x09,0x00,0x00,0x00, +0x15,0x00,0x04,0x00,0x27,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x27,0x00,0x00,0x00, +0x28,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x29,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x14,0x00,0x02,0x00,0x2d,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x3d,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0x16,0x00,0x03,0x00,0x4a,0x00,0x00,0x00, +0x10,0x00,0x00,0x00,0x15,0x00,0x04,0x00,0x4b,0x00,0x00,0x00, +0x08,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x4c,0x00,0x00,0x00,0x10,0x00,0x00,0x00, +0x1c,0x00,0x04,0x00,0x4d,0x00,0x00,0x00,0x4b,0x00,0x00,0x00, +0x4c,0x00,0x00,0x00,0x1e,0x00,0x05,0x00,0x4e,0x00,0x00,0x00, +0x4a,0x00,0x00,0x00,0x4a,0x00,0x00,0x00,0x4d,0x00,0x00,0x00, +0x1d,0x00,0x03,0x00,0x4f,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, +0x1e,0x00,0x03,0x00,0x50,0x00,0x00,0x00,0x4f,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x51,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x50,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x51,0x00,0x00,0x00, +0x52,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x54,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x4a,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x27,0x00,0x00,0x00,0x5a,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x27,0x00,0x00,0x00, +0x60,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x62,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x4b,0x00,0x00,0x00, +0x17,0x00,0x04,0x00,0x66,0x00,0x00,0x00,0x14,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x6a,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x27,0x00,0x00,0x00,0x6e,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x1d,0x00,0x03,0x00,0x7b,0x00,0x00,0x00,0x14,0x00,0x00,0x00, +0x1e,0x00,0x03,0x00,0x7c,0x00,0x00,0x00,0x7b,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x7d,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x7c,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x7d,0x00,0x00,0x00, +0x7e,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x86,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x14,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x8a,0x00,0x00,0x00, 0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x9a,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0xa5,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0xa6,0x00,0x00,0x00, -0x08,0x01,0x00,0x00,0x1d,0x00,0x03,0x00,0xc4,0x00,0x00,0x00, -0x17,0x00,0x00,0x00,0x1e,0x00,0x03,0x00,0xc5,0x00,0x00,0x00, -0xc4,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0xc6,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0xc5,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0xc6,0x00,0x00,0x00,0xc7,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x2c,0x00,0x06,0x00,0x0a,0x00,0x00,0x00,0xcf,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x91,0x00,0x00,0x00,0x91,0x00,0x00,0x00, -0x36,0x00,0x05,0x00,0x02,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x05,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x0e,0x00,0x00,0x00, -0x0f,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x0d,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x10,0x00,0x00,0x00, -0x0f,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x11,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x0e,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x13,0x00,0x00,0x00, -0x0d,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x15,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x15,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x1e,0x00,0x00,0x00,0x1f,0x00,0x00,0x00, -0x1b,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x3e,0x00,0x03,0x00, -0x1f,0x00,0x00,0x00,0x1d,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x22,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x22,0x00,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xd2,0x00,0x00,0x00, -0x21,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0xa4,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x2b,0x00,0x00,0x00, -0x2c,0x00,0x00,0x00,0x2a,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x2d,0x00,0x00,0x00, -0x2c,0x00,0x00,0x00,0x87,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x2f,0x00,0x00,0x00,0x2d,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0xb1,0x00,0x05,0x00,0x30,0x00,0x00,0x00,0x31,0x00,0x00,0x00, -0xd2,0x00,0x00,0x00,0x2f,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0x24,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x31,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x24,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x23,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x34,0x00,0x00,0x00, -0xd2,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x35,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x38,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0x37,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3d,0x00,0x00,0x00, -0x11,0x00,0x00,0x00,0x2d,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x3f,0x00,0x00,0x00,0x3d,0x00,0x00,0x00, -0x38,0x00,0x00,0x00,0x87,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0x3f,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x8b,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x43,0x00,0x00,0x00, -0x38,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x87,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x44,0x00,0x00,0x00,0x43,0x00,0x00,0x00, -0x35,0x00,0x00,0x00,0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x49,0x00,0x00,0x00,0x38,0x00,0x00,0x00,0x43,0x00,0x00,0x00, -0x41,0x00,0x07,0x00,0x56,0x00,0x00,0x00,0x57,0x00,0x00,0x00, -0x54,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x21,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x4c,0x00,0x00,0x00, -0x58,0x00,0x00,0x00,0x57,0x00,0x00,0x00,0x73,0x00,0x04,0x00, -0x17,0x00,0x00,0x00,0x59,0x00,0x00,0x00,0x58,0x00,0x00,0x00, -0x41,0x00,0x07,0x00,0x56,0x00,0x00,0x00,0x5d,0x00,0x00,0x00, -0x54,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x5c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x4c,0x00,0x00,0x00, -0x5e,0x00,0x00,0x00,0x5d,0x00,0x00,0x00,0x73,0x00,0x04,0x00, -0x17,0x00,0x00,0x00,0x5f,0x00,0x00,0x00,0x5e,0x00,0x00,0x00, -0x41,0x00,0x08,0x00,0x64,0x00,0x00,0x00,0x65,0x00,0x00,0x00, -0x54,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x35,0x00,0x00,0x00,0x44,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0x66,0x00,0x00,0x00,0x65,0x00,0x00,0x00, -0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x67,0x00,0x00,0x00, -0x66,0x00,0x00,0x00,0xc7,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0x6d,0x00,0x00,0x00,0x67,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x70,0x00,0x04,0x00,0x17,0x00,0x00,0x00,0x6e,0x00,0x00,0x00, -0x6d,0x00,0x00,0x00,0xc2,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0x71,0x00,0x00,0x00,0x67,0x00,0x00,0x00,0x70,0x00,0x00,0x00, -0x70,0x00,0x04,0x00,0x17,0x00,0x00,0x00,0x72,0x00,0x00,0x00, -0x71,0x00,0x00,0x00,0x50,0x00,0x05,0x00,0x68,0x00,0x00,0x00, -0x73,0x00,0x00,0x00,0x6e,0x00,0x00,0x00,0x72,0x00,0x00,0x00, -0x8e,0x00,0x05,0x00,0x68,0x00,0x00,0x00,0x76,0x00,0x00,0x00, -0x73,0x00,0x00,0x00,0x59,0x00,0x00,0x00,0x50,0x00,0x05,0x00, -0x68,0x00,0x00,0x00,0x78,0x00,0x00,0x00,0x5f,0x00,0x00,0x00, -0x5f,0x00,0x00,0x00,0x81,0x00,0x05,0x00,0x68,0x00,0x00,0x00, -0x79,0x00,0x00,0x00,0x76,0x00,0x00,0x00,0x78,0x00,0x00,0x00, -0x51,0x00,0x05,0x00,0x17,0x00,0x00,0x00,0x7c,0x00,0x00,0x00, -0x79,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x2b,0x00,0x00,0x00,0x81,0x00,0x00,0x00,0x2a,0x00,0x00,0x00, -0x5c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x82,0x00,0x00,0x00,0x81,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x82,0x00,0x00,0x00, -0x49,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x86,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x44,0x00,0x00,0x00, -0x41,0x00,0x06,0x00,0x88,0x00,0x00,0x00,0x89,0x00,0x00,0x00, -0x80,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x86,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x17,0x00,0x00,0x00,0x8a,0x00,0x00,0x00, -0x89,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x17,0x00,0x00,0x00, -0x8d,0x00,0x00,0x00,0x1f,0x00,0x00,0x00,0x0c,0x00,0x08,0x00, -0x17,0x00,0x00,0x00,0x8e,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x32,0x00,0x00,0x00,0x7c,0x00,0x00,0x00,0x8a,0x00,0x00,0x00, -0x8d,0x00,0x00,0x00,0x3e,0x00,0x03,0x00,0x1f,0x00,0x00,0x00, -0x8e,0x00,0x00,0x00,0x51,0x00,0x05,0x00,0x17,0x00,0x00,0x00, -0x93,0x00,0x00,0x00,0x79,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9b,0x00,0x00,0x00, -0x86,0x00,0x00,0x00,0x9a,0x00,0x00,0x00,0x41,0x00,0x06,0x00, -0x88,0x00,0x00,0x00,0x9c,0x00,0x00,0x00,0x80,0x00,0x00,0x00, -0x21,0x00,0x00,0x00,0x9b,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x17,0x00,0x00,0x00,0x9d,0x00,0x00,0x00,0x9c,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x17,0x00,0x00,0x00,0xa0,0x00,0x00,0x00, -0x1f,0x00,0x00,0x00,0x0c,0x00,0x08,0x00,0x17,0x00,0x00,0x00, -0xa1,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00, -0x93,0x00,0x00,0x00,0x9d,0x00,0x00,0x00,0xa0,0x00,0x00,0x00, -0x3e,0x00,0x03,0x00,0x1f,0x00,0x00,0x00,0xa1,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa4,0x00,0x00,0x00, -0xd2,0x00,0x00,0x00,0x35,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x22,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x24,0x00,0x00,0x00, -0xe0,0x00,0x04,0x00,0xa5,0x00,0x00,0x00,0xa5,0x00,0x00,0x00, -0xa6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xa8,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xa8,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xd3,0x00,0x00,0x00,0x9a,0x00,0x00,0x00, -0x24,0x00,0x00,0x00,0xbf,0x00,0x00,0x00,0xab,0x00,0x00,0x00, -0xad,0x00,0x05,0x00,0x30,0x00,0x00,0x00,0xae,0x00,0x00,0x00, -0xd3,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0xaa,0x00,0x00,0x00,0xab,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xae,0x00,0x00,0x00,0xa9,0x00,0x00,0x00, -0xaa,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xa9,0x00,0x00,0x00, -0xb1,0x00,0x05,0x00,0x30,0x00,0x00,0x00,0xb1,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0xd3,0x00,0x00,0x00,0xf7,0x00,0x03,0x00, -0xb3,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0xb1,0x00,0x00,0x00,0xb2,0x00,0x00,0x00,0xb3,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xb2,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xb7,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0xd3,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x1e,0x00,0x00,0x00, -0xb8,0x00,0x00,0x00,0x1b,0x00,0x00,0x00,0xb7,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x17,0x00,0x00,0x00,0xb9,0x00,0x00,0x00, -0xb8,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x17,0x00,0x00,0x00, -0xbb,0x00,0x00,0x00,0x1f,0x00,0x00,0x00,0x81,0x00,0x05,0x00, -0x17,0x00,0x00,0x00,0xbc,0x00,0x00,0x00,0xbb,0x00,0x00,0x00, -0xb9,0x00,0x00,0x00,0x3e,0x00,0x03,0x00,0x1f,0x00,0x00,0x00, -0xbc,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xb3,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xb3,0x00,0x00,0x00,0xe0,0x00,0x04,0x00, -0xa5,0x00,0x00,0x00,0xa5,0x00,0x00,0x00,0xa6,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xab,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xab,0x00,0x00,0x00,0xc3,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xbf,0x00,0x00,0x00,0xd3,0x00,0x00,0x00,0x5c,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xa8,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xaa,0x00,0x00,0x00,0xaa,0x00,0x05,0x00,0x30,0x00,0x00,0x00, -0xc1,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0xf7,0x00,0x03,0x00,0xc3,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xc1,0x00,0x00,0x00,0xc2,0x00,0x00,0x00, -0xc3,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xc2,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x2b,0x00,0x00,0x00,0xc8,0x00,0x00,0x00, -0x2a,0x00,0x00,0x00,0x35,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xc9,0x00,0x00,0x00,0xc8,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xcb,0x00,0x00,0x00, -0xc9,0x00,0x00,0x00,0x11,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x1e,0x00,0x00,0x00,0xcc,0x00,0x00,0x00,0x1b,0x00,0x00,0x00, -0x21,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x17,0x00,0x00,0x00, -0xcd,0x00,0x00,0x00,0xcc,0x00,0x00,0x00,0x41,0x00,0x06,0x00, -0x88,0x00,0x00,0x00,0xce,0x00,0x00,0x00,0xc7,0x00,0x00,0x00, -0x21,0x00,0x00,0x00,0xcb,0x00,0x00,0x00,0x3e,0x00,0x03,0x00, -0xce,0x00,0x00,0x00,0xcd,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xc3,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xc3,0x00,0x00,0x00, -0xfd,0x00,0x01,0x00,0x38,0x00,0x01,0x00, +0x9e,0x00,0x00,0x00,0x08,0x01,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xa0,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0x15,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, +0xbd,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, +0xbe,0x00,0x00,0x00,0xbd,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0xbf,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0xbe,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0xbf,0x00,0x00,0x00,0xc0,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xc8,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x33,0x00,0x06,0x00, +0x09,0x00,0x00,0x00,0xc9,0x00,0x00,0x00,0xc8,0x00,0x00,0x00, +0x8a,0x00,0x00,0x00,0x8a,0x00,0x00,0x00,0x36,0x00,0x05,0x00, +0x02,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x03,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x05,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00,0x0e,0x00,0x00,0x00, +0x0b,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x0e,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0x11,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x13,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x1b,0x00,0x00,0x00,0x1c,0x00,0x00,0x00, +0x18,0x00,0x00,0x00,0x13,0x00,0x00,0x00,0x3e,0x00,0x03,0x00, +0x1c,0x00,0x00,0x00,0x1a,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x1e,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x1e,0x00,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xcc,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0x9d,0x00,0x00,0x00, +0x1f,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x29,0x00,0x00,0x00, +0x2a,0x00,0x00,0x00,0x26,0x00,0x00,0x00,0x28,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x2b,0x00,0x00,0x00, +0x2a,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x2c,0x00,0x00,0x00,0x2b,0x00,0x00,0x00,0x15,0x00,0x00,0x00, +0xb0,0x00,0x05,0x00,0x2d,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, +0xcc,0x00,0x00,0x00,0x2c,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0x20,0x00,0x00,0x00,0x1f,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x2e,0x00,0x00,0x00,0x1f,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x1f,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x31,0x00,0x00,0x00, +0xcc,0x00,0x00,0x00,0x15,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0x32,0x00,0x00,0x00, +0x13,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x35,0x00,0x00,0x00,0x31,0x00,0x00,0x00,0x34,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, +0x0f,0x00,0x00,0x00,0x2b,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x3c,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, +0x35,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x3e,0x00,0x00,0x00,0x3c,0x00,0x00,0x00,0x3d,0x00,0x00,0x00, +0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x41,0x00,0x00,0x00, +0x35,0x00,0x00,0x00,0x3d,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x42,0x00,0x00,0x00,0x41,0x00,0x00,0x00, +0x32,0x00,0x00,0x00,0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x47,0x00,0x00,0x00,0x35,0x00,0x00,0x00,0x41,0x00,0x00,0x00, +0x41,0x00,0x07,0x00,0x54,0x00,0x00,0x00,0x55,0x00,0x00,0x00, +0x52,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0x3e,0x00,0x00,0x00, +0x28,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, +0x56,0x00,0x00,0x00,0x55,0x00,0x00,0x00,0x73,0x00,0x04,0x00, +0x14,0x00,0x00,0x00,0x57,0x00,0x00,0x00,0x56,0x00,0x00,0x00, +0x41,0x00,0x07,0x00,0x54,0x00,0x00,0x00,0x5b,0x00,0x00,0x00, +0x52,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0x3e,0x00,0x00,0x00, +0x5a,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, +0x5c,0x00,0x00,0x00,0x5b,0x00,0x00,0x00,0x73,0x00,0x04,0x00, +0x14,0x00,0x00,0x00,0x5d,0x00,0x00,0x00,0x5c,0x00,0x00,0x00, +0x41,0x00,0x08,0x00,0x62,0x00,0x00,0x00,0x63,0x00,0x00,0x00, +0x52,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0x3e,0x00,0x00,0x00, +0x60,0x00,0x00,0x00,0x42,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x4b,0x00,0x00,0x00,0x64,0x00,0x00,0x00,0x63,0x00,0x00,0x00, +0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x65,0x00,0x00,0x00, +0x64,0x00,0x00,0x00,0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x6b,0x00,0x00,0x00,0x65,0x00,0x00,0x00,0x6a,0x00,0x00,0x00, +0x70,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, +0x6b,0x00,0x00,0x00,0xc2,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x6f,0x00,0x00,0x00,0x65,0x00,0x00,0x00,0x6e,0x00,0x00,0x00, +0x70,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0x70,0x00,0x00,0x00, +0x6f,0x00,0x00,0x00,0x50,0x00,0x05,0x00,0x66,0x00,0x00,0x00, +0x71,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x70,0x00,0x00,0x00, +0x8e,0x00,0x05,0x00,0x66,0x00,0x00,0x00,0x74,0x00,0x00,0x00, +0x71,0x00,0x00,0x00,0x57,0x00,0x00,0x00,0x50,0x00,0x05,0x00, +0x66,0x00,0x00,0x00,0x76,0x00,0x00,0x00,0x5d,0x00,0x00,0x00, +0x5d,0x00,0x00,0x00,0x81,0x00,0x05,0x00,0x66,0x00,0x00,0x00, +0x77,0x00,0x00,0x00,0x74,0x00,0x00,0x00,0x76,0x00,0x00,0x00, +0x51,0x00,0x05,0x00,0x14,0x00,0x00,0x00,0x7a,0x00,0x00,0x00, +0x77,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x29,0x00,0x00,0x00,0x7f,0x00,0x00,0x00,0x26,0x00,0x00,0x00, +0x5a,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x80,0x00,0x00,0x00,0x7f,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x82,0x00,0x00,0x00,0x80,0x00,0x00,0x00, +0x47,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0x82,0x00,0x00,0x00,0x42,0x00,0x00,0x00, +0x41,0x00,0x06,0x00,0x86,0x00,0x00,0x00,0x87,0x00,0x00,0x00, +0x7e,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0x88,0x00,0x00,0x00, +0x87,0x00,0x00,0x00,0x51,0x00,0x05,0x00,0x14,0x00,0x00,0x00, +0x8c,0x00,0x00,0x00,0x77,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x93,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0x4c,0x00,0x00,0x00,0x41,0x00,0x06,0x00, +0x86,0x00,0x00,0x00,0x94,0x00,0x00,0x00,0x7e,0x00,0x00,0x00, +0x28,0x00,0x00,0x00,0x93,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x14,0x00,0x00,0x00,0x95,0x00,0x00,0x00,0x94,0x00,0x00,0x00, +0x85,0x00,0x05,0x00,0x14,0x00,0x00,0x00,0x96,0x00,0x00,0x00, +0x8c,0x00,0x00,0x00,0x95,0x00,0x00,0x00,0x0c,0x00,0x08,0x00, +0x14,0x00,0x00,0x00,0x97,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x32,0x00,0x00,0x00,0x7a,0x00,0x00,0x00,0x88,0x00,0x00,0x00, +0x96,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x14,0x00,0x00,0x00, +0x99,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x81,0x00,0x05,0x00, +0x14,0x00,0x00,0x00,0x9a,0x00,0x00,0x00,0x99,0x00,0x00,0x00, +0x97,0x00,0x00,0x00,0x3e,0x00,0x03,0x00,0x1c,0x00,0x00,0x00, +0x9a,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x9d,0x00,0x00,0x00,0xcc,0x00,0x00,0x00,0x32,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x1e,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0x20,0x00,0x00,0x00,0xe0,0x00,0x04,0x00,0x32,0x00,0x00,0x00, +0x32,0x00,0x00,0x00,0x9e,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xa1,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xa1,0x00,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xcd,0x00,0x00,0x00, +0xa0,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0xb8,0x00,0x00,0x00, +0xa4,0x00,0x00,0x00,0xac,0x00,0x05,0x00,0x2d,0x00,0x00,0x00, +0xa7,0x00,0x00,0x00,0xcd,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0xa3,0x00,0x00,0x00,0xa4,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xa7,0x00,0x00,0x00, +0xa2,0x00,0x00,0x00,0xa3,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xa2,0x00,0x00,0x00,0xb0,0x00,0x05,0x00,0x2d,0x00,0x00,0x00, +0xaa,0x00,0x00,0x00,0x13,0x00,0x00,0x00,0xcd,0x00,0x00,0x00, +0xf7,0x00,0x03,0x00,0xac,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0xaa,0x00,0x00,0x00,0xab,0x00,0x00,0x00, +0xac,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xab,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xb0,0x00,0x00,0x00, +0x13,0x00,0x00,0x00,0xcd,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x1b,0x00,0x00,0x00,0xb1,0x00,0x00,0x00,0x18,0x00,0x00,0x00, +0xb0,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x14,0x00,0x00,0x00, +0xb2,0x00,0x00,0x00,0xb1,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x14,0x00,0x00,0x00,0xb4,0x00,0x00,0x00,0x1c,0x00,0x00,0x00, +0x81,0x00,0x05,0x00,0x14,0x00,0x00,0x00,0xb5,0x00,0x00,0x00, +0xb4,0x00,0x00,0x00,0xb2,0x00,0x00,0x00,0x3e,0x00,0x03,0x00, +0x1c,0x00,0x00,0x00,0xb5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xac,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xac,0x00,0x00,0x00, +0xe0,0x00,0x04,0x00,0x32,0x00,0x00,0x00,0x32,0x00,0x00,0x00, +0x9e,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xa4,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xa4,0x00,0x00,0x00,0xc2,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xb8,0x00,0x00,0x00,0xcd,0x00,0x00,0x00, +0x5a,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xa1,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xa3,0x00,0x00,0x00,0xaa,0x00,0x05,0x00, +0x2d,0x00,0x00,0x00,0xba,0x00,0x00,0x00,0x13,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0xf7,0x00,0x03,0x00,0xbc,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xba,0x00,0x00,0x00, +0xbb,0x00,0x00,0x00,0xbc,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xbb,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x29,0x00,0x00,0x00, +0xc1,0x00,0x00,0x00,0x26,0x00,0x00,0x00,0x60,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xc2,0x00,0x00,0x00, +0xc1,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xc4,0x00,0x00,0x00,0xc2,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x1b,0x00,0x00,0x00,0xc5,0x00,0x00,0x00, +0x18,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x14,0x00,0x00,0x00,0xc6,0x00,0x00,0x00,0xc5,0x00,0x00,0x00, +0x41,0x00,0x06,0x00,0x86,0x00,0x00,0x00,0xc7,0x00,0x00,0x00, +0xc0,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0xc4,0x00,0x00,0x00, +0x3e,0x00,0x03,0x00,0xc7,0x00,0x00,0x00,0xc6,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xbc,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xbc,0x00,0x00,0x00,0xfd,0x00,0x01,0x00,0x38,0x00,0x01,0x00, + }; -const uint64_t mul_mat_vec_q4_1_f32_len = 3248; +const uint64_t mul_mat_vec_q4_1_f32_len = 3252; unsigned char mul_mat_vec_q4_K_f32_data[] = { 0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, -0xa2,0x03,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, +0x9e,0x03,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, 0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x27,0x00,0x00,0x00, 0x11,0x00,0x02,0x00,0x51,0x11,0x00,0x00,0x11,0x00,0x02,0x00, 0x60,0x11,0x00,0x00,0x0b,0x00,0x06,0x00,0x01,0x00,0x00,0x00, @@ -45676,3575 +41119,3566 @@ unsigned char mul_mat_vec_q4_K_f32_data[] = { 0x00,0x00,0x00,0x00,0x0e,0x00,0x03,0x00,0x00,0x00,0x00,0x00, 0x01,0x00,0x00,0x00,0x0f,0x00,0x0c,0x00,0x05,0x00,0x00,0x00, 0x04,0x00,0x00,0x00,0x6d,0x61,0x69,0x6e,0x00,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x15,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x51,0x00,0x00,0x00,0x7b,0x00,0x00,0x00,0xef,0x01,0x00,0x00, -0x94,0x03,0x00,0x00,0x10,0x00,0x06,0x00,0x04,0x00,0x00,0x00, +0x0b,0x00,0x00,0x00,0x13,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0x4d,0x00,0x00,0x00,0x76,0x00,0x00,0x00,0xec,0x01,0x00,0x00, +0x90,0x03,0x00,0x00,0x10,0x00,0x06,0x00,0x04,0x00,0x00,0x00, 0x11,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x0c,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x0b,0x00,0x00,0x00, 0x0b,0x00,0x00,0x00,0x1a,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x13,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x13,0x00,0x00,0x00, +0x11,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x11,0x00,0x00,0x00, 0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x13,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x11,0x00,0x00,0x00,0x02,0x00,0x00,0x00, 0x23,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x47,0x00,0x03,0x00, -0x13,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x21,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x1b,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x74,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x76,0x00,0x00,0x00, +0x11,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x20,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x1b,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x70,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x71,0x00,0x00,0x00, 0x06,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x77,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x77,0x00,0x00,0x00, +0x72,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x72,0x00,0x00,0x00, 0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x77,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x72,0x00,0x00,0x00,0x02,0x00,0x00,0x00, 0x23,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x78,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x90,0x00,0x00,0x00, -0x48,0x00,0x04,0x00,0x79,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x79,0x00,0x00,0x00, +0x73,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x90,0x00,0x00,0x00, +0x48,0x00,0x04,0x00,0x74,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x74,0x00,0x00,0x00, 0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x03,0x00,0x79,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x7b,0x00,0x00,0x00,0x22,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x7b,0x00,0x00,0x00, +0x47,0x00,0x03,0x00,0x74,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x76,0x00,0x00,0x00,0x22,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x76,0x00,0x00,0x00, 0x21,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0xec,0x01,0x00,0x00,0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x48,0x00,0x04,0x00,0xed,0x01,0x00,0x00,0x00,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0xed,0x01,0x00,0x00, +0xe9,0x01,0x00,0x00,0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x48,0x00,0x04,0x00,0xea,0x01,0x00,0x00,0x00,0x00,0x00,0x00, +0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0xea,0x01,0x00,0x00, 0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x03,0x00,0xed,0x01,0x00,0x00,0x02,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0xef,0x01,0x00,0x00,0x22,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xef,0x01,0x00,0x00, +0x47,0x00,0x03,0x00,0xea,0x01,0x00,0x00,0x02,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0xec,0x01,0x00,0x00,0x22,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xec,0x01,0x00,0x00, 0x21,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x91,0x03,0x00,0x00,0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x48,0x00,0x04,0x00,0x92,0x03,0x00,0x00,0x00,0x00,0x00,0x00, -0x19,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x92,0x03,0x00,0x00, +0x8d,0x03,0x00,0x00,0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x48,0x00,0x04,0x00,0x8e,0x03,0x00,0x00,0x00,0x00,0x00,0x00, +0x19,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x8e,0x03,0x00,0x00, 0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x03,0x00,0x92,0x03,0x00,0x00,0x02,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x94,0x03,0x00,0x00,0x22,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x94,0x03,0x00,0x00, +0x47,0x00,0x03,0x00,0x8e,0x03,0x00,0x00,0x02,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x90,0x03,0x00,0x00,0x22,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x90,0x03,0x00,0x00, 0x21,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x9c,0x03,0x00,0x00,0x0b,0x00,0x00,0x00,0x19,0x00,0x00,0x00, +0x98,0x03,0x00,0x00,0x0b,0x00,0x00,0x00,0x19,0x00,0x00,0x00, 0x13,0x00,0x02,0x00,0x02,0x00,0x00,0x00,0x21,0x00,0x03,0x00, 0x03,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x15,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x15,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x17,0x00,0x04,0x00,0x0a,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x0b,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x0a,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x0b,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x0d,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x0e,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x1e,0x00,0x05,0x00,0x13,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x14,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x13,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0x15,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x17,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x1a,0x00,0x00,0x00, -0x00,0x01,0x00,0x00,0x3b,0x00,0x04,0x00,0x0b,0x00,0x00,0x00, -0x21,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x25,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x17,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x03,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x0a,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0x0a,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x0d,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x1e,0x00,0x05,0x00, +0x11,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x12,0x00,0x00,0x00, +0x09,0x00,0x00,0x00,0x11,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0x12,0x00,0x00,0x00,0x13,0x00,0x00,0x00,0x09,0x00,0x00,0x00, +0x15,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x14,0x00,0x00,0x00, +0x15,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x16,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x19,0x00,0x00,0x00, +0x00,0x01,0x00,0x00,0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x2b,0x00,0x00,0x00, 0x04,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x42,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x48,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x16,0x00,0x03,0x00,0x4d,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x1c,0x00,0x04,0x00,0x4f,0x00,0x00,0x00, -0x4d,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x50,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x4f,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x50,0x00,0x00,0x00,0x51,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x52,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0x57,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x58,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x4d,0x00,0x00,0x00,0x14,0x00,0x02,0x00,0x63,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x80,0x00,0x00,0x00,0x16,0x00,0x03,0x00,0x70,0x00,0x00,0x00, -0x10,0x00,0x00,0x00,0x17,0x00,0x04,0x00,0x71,0x00,0x00,0x00, -0x70,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x15,0x00,0x04,0x00, -0x72,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x73,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x1c,0x00,0x04,0x00,0x74,0x00,0x00,0x00, -0x72,0x00,0x00,0x00,0x73,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x75,0x00,0x00,0x00,0x80,0x00,0x00,0x00, -0x1c,0x00,0x04,0x00,0x76,0x00,0x00,0x00,0x72,0x00,0x00,0x00, -0x75,0x00,0x00,0x00,0x1e,0x00,0x05,0x00,0x77,0x00,0x00,0x00, -0x71,0x00,0x00,0x00,0x74,0x00,0x00,0x00,0x76,0x00,0x00,0x00, -0x1d,0x00,0x03,0x00,0x78,0x00,0x00,0x00,0x77,0x00,0x00,0x00, -0x1e,0x00,0x03,0x00,0x79,0x00,0x00,0x00,0x78,0x00,0x00,0x00, +0x3f,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x45,0x00,0x00,0x00,0x40,0x00,0x00,0x00, +0x16,0x00,0x03,0x00,0x4a,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0x1c,0x00,0x04,0x00,0x4b,0x00,0x00,0x00,0x4a,0x00,0x00,0x00, +0x3f,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x4c,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x4b,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0x4c,0x00,0x00,0x00,0x4d,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, +0x10,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, +0x53,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x54,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x4a,0x00,0x00,0x00, +0x14,0x00,0x02,0x00,0x5f,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x68,0x00,0x00,0x00,0x80,0x00,0x00,0x00, +0x16,0x00,0x03,0x00,0x6c,0x00,0x00,0x00,0x10,0x00,0x00,0x00, +0x17,0x00,0x04,0x00,0x6d,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x15,0x00,0x04,0x00,0x6e,0x00,0x00,0x00, +0x08,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x6f,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x1c,0x00,0x04,0x00,0x70,0x00,0x00,0x00,0x6e,0x00,0x00,0x00, +0x6f,0x00,0x00,0x00,0x1c,0x00,0x04,0x00,0x71,0x00,0x00,0x00, +0x6e,0x00,0x00,0x00,0x68,0x00,0x00,0x00,0x1e,0x00,0x05,0x00, +0x72,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x70,0x00,0x00,0x00, +0x71,0x00,0x00,0x00,0x1d,0x00,0x03,0x00,0x73,0x00,0x00,0x00, +0x72,0x00,0x00,0x00,0x1e,0x00,0x03,0x00,0x74,0x00,0x00,0x00, +0x73,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x75,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x74,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0x75,0x00,0x00,0x00,0x76,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, 0x20,0x00,0x04,0x00,0x7a,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x79,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x7a,0x00,0x00,0x00, -0x7b,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x7f,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x70,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x87,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x90,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x93,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x72,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x98,0x00,0x00,0x00, -0x3f,0x00,0x00,0x00,0x15,0x00,0x04,0x00,0x9a,0x00,0x00,0x00, -0x08,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xbf,0x00,0x00,0x00,0x05,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xce,0x00,0x00,0x00, -0x08,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xd4,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xdf,0x00,0x00,0x00,0xc0,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xeb,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x64,0x01,0x00,0x00,0x03,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xa2,0x01,0x00,0x00,0x41,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xb0,0x01,0x00,0x00, +0x6c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x82,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x14,0x00,0x00,0x00,0x8b,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x8e,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x6e,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x14,0x00,0x00,0x00, +0x93,0x00,0x00,0x00,0x3f,0x00,0x00,0x00,0x15,0x00,0x04,0x00, +0x95,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xba,0x00,0x00,0x00, +0x05,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xc9,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x14,0x00,0x00,0x00,0xcf,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0xda,0x00,0x00,0x00, +0xc0,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x14,0x00,0x00,0x00, +0xdc,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xe7,0x00,0x00,0x00,0x09,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0x06,0x01,0x00,0x00, +0x04,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x61,0x01,0x00,0x00,0x03,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x9f,0x01,0x00,0x00,0x41,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xad,0x01,0x00,0x00, 0x42,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xbe,0x01,0x00,0x00,0x43,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, -0xec,0x01,0x00,0x00,0x4d,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, -0xed,0x01,0x00,0x00,0xec,0x01,0x00,0x00,0x20,0x00,0x04,0x00, -0xee,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0xed,0x01,0x00,0x00, -0x3b,0x00,0x04,0x00,0xee,0x01,0x00,0x00,0xef,0x01,0x00,0x00, -0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0xf4,0x01,0x00,0x00, -0x0c,0x00,0x00,0x00,0x4d,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x2a,0x02,0x00,0x00,0x21,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x36,0x02,0x00,0x00, +0xbb,0x01,0x00,0x00,0x43,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, +0xe9,0x01,0x00,0x00,0x4a,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, +0xea,0x01,0x00,0x00,0xe9,0x01,0x00,0x00,0x20,0x00,0x04,0x00, +0xeb,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0xea,0x01,0x00,0x00, +0x3b,0x00,0x04,0x00,0xeb,0x01,0x00,0x00,0xec,0x01,0x00,0x00, +0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0xf1,0x01,0x00,0x00, +0x0c,0x00,0x00,0x00,0x4a,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x27,0x02,0x00,0x00,0x21,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x33,0x02,0x00,0x00, 0x22,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x42,0x02,0x00,0x00,0x23,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x72,0x03,0x00,0x00,0x02,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x73,0x03,0x00,0x00, -0x08,0x01,0x00,0x00,0x1d,0x00,0x03,0x00,0x91,0x03,0x00,0x00, -0x4d,0x00,0x00,0x00,0x1e,0x00,0x03,0x00,0x92,0x03,0x00,0x00, -0x91,0x03,0x00,0x00,0x20,0x00,0x04,0x00,0x93,0x03,0x00,0x00, -0x0c,0x00,0x00,0x00,0x92,0x03,0x00,0x00,0x3b,0x00,0x04,0x00, -0x93,0x03,0x00,0x00,0x94,0x03,0x00,0x00,0x0c,0x00,0x00,0x00, -0x2c,0x00,0x06,0x00,0x0a,0x00,0x00,0x00,0x9c,0x03,0x00,0x00, -0x4e,0x00,0x00,0x00,0x87,0x00,0x00,0x00,0x87,0x00,0x00,0x00, -0x36,0x00,0x05,0x00,0x02,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x05,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x0e,0x00,0x00,0x00, -0x0f,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x0d,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x10,0x00,0x00,0x00, -0x0f,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x11,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x17,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x15,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x19,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x87,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x1b,0x00,0x00,0x00,0x19,0x00,0x00,0x00, -0x1a,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x1f,0x00,0x00,0x00,0x11,0x00,0x00,0x00,0x1b,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x0e,0x00,0x00,0x00,0x22,0x00,0x00,0x00, -0x21,0x00,0x00,0x00,0x0d,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x22,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x24,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x87,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x26,0x00,0x00,0x00,0x24,0x00,0x00,0x00,0x25,0x00,0x00,0x00, -0x8b,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x2b,0x00,0x00,0x00, -0x24,0x00,0x00,0x00,0x25,0x00,0x00,0x00,0x87,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x2f,0x00,0x00,0x00,0x26,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x33,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x2f,0x00,0x00,0x00, -0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x34,0x00,0x00,0x00, -0x26,0x00,0x00,0x00,0x33,0x00,0x00,0x00,0x87,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x2f,0x00,0x00,0x00, -0x25,0x00,0x00,0x00,0x8b,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x3a,0x00,0x00,0x00,0x2f,0x00,0x00,0x00,0x25,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3d,0x00,0x00,0x00, -0x25,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x3f,0x00,0x00,0x00,0x3d,0x00,0x00,0x00, -0x3a,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x3f,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x44,0x00,0x00,0x00, -0x42,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x46,0x00,0x00,0x00,0x44,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x4a,0x00,0x00,0x00,0x48,0x00,0x00,0x00,0x37,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x4c,0x00,0x00,0x00, -0x4a,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x54,0x00,0x00,0x00,0x52,0x00,0x00,0x00, -0x2b,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x56,0x00,0x00,0x00,0x54,0x00,0x00,0x00,0x26,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x58,0x00,0x00,0x00,0x59,0x00,0x00,0x00, -0x51,0x00,0x00,0x00,0x56,0x00,0x00,0x00,0x3e,0x00,0x03,0x00, -0x59,0x00,0x00,0x00,0x57,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x5c,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x5c,0x00,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x9d,0x03,0x00,0x00, -0x2b,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0x71,0x03,0x00,0x00, -0x5d,0x00,0x00,0x00,0xb1,0x00,0x05,0x00,0x63,0x00,0x00,0x00, -0x64,0x00,0x00,0x00,0x9d,0x03,0x00,0x00,0x1b,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0x5e,0x00,0x00,0x00,0x5d,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x64,0x00,0x00,0x00, -0x5d,0x00,0x00,0x00,0x5e,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x5d,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x67,0x00,0x00,0x00,0x9d,0x03,0x00,0x00,0x1a,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x69,0x00,0x00,0x00, -0x67,0x00,0x00,0x00,0x4c,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x69,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x7e,0x00,0x00,0x00,0x1f,0x00,0x00,0x00,0x9d,0x03,0x00,0x00, -0x41,0x00,0x08,0x00,0x7f,0x00,0x00,0x00,0x80,0x00,0x00,0x00, -0x7b,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x7e,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x0d,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x70,0x00,0x00,0x00,0x81,0x00,0x00,0x00,0x80,0x00,0x00,0x00, -0x73,0x00,0x04,0x00,0x4d,0x00,0x00,0x00,0x82,0x00,0x00,0x00, -0x81,0x00,0x00,0x00,0x41,0x00,0x08,0x00,0x7f,0x00,0x00,0x00, -0x88,0x00,0x00,0x00,0x7b,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0x7e,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x87,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x70,0x00,0x00,0x00,0x89,0x00,0x00,0x00, -0x88,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0x8a,0x00,0x00,0x00,0x89,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x92,0x00,0x00,0x00,0x37,0x00,0x00,0x00, -0x25,0x00,0x00,0x00,0x41,0x00,0x08,0x00,0x93,0x00,0x00,0x00, -0x94,0x00,0x00,0x00,0x7b,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0x7e,0x00,0x00,0x00,0x90,0x00,0x00,0x00,0x92,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x72,0x00,0x00,0x00,0x95,0x00,0x00,0x00, -0x94,0x00,0x00,0x00,0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x96,0x00,0x00,0x00,0x95,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x97,0x00,0x00,0x00,0x96,0x00,0x00,0x00, -0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x99,0x00,0x00,0x00, -0x97,0x00,0x00,0x00,0x98,0x00,0x00,0x00,0x72,0x00,0x04,0x00, -0x9a,0x00,0x00,0x00,0x9b,0x00,0x00,0x00,0x99,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x72,0x00,0x00,0x00,0x9c,0x00,0x00,0x00, -0x9b,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xa3,0x00,0x00,0x00,0x92,0x00,0x00,0x00,0x90,0x00,0x00,0x00, -0x41,0x00,0x08,0x00,0x93,0x00,0x00,0x00,0xa4,0x00,0x00,0x00, -0x7b,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x7e,0x00,0x00,0x00, -0x90,0x00,0x00,0x00,0xa3,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x72,0x00,0x00,0x00,0xa5,0x00,0x00,0x00,0xa4,0x00,0x00,0x00, -0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0xa6,0x00,0x00,0x00, -0xa5,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xa7,0x00,0x00,0x00,0xa6,0x00,0x00,0x00,0xc7,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xa8,0x00,0x00,0x00,0xa7,0x00,0x00,0x00, -0x98,0x00,0x00,0x00,0x72,0x00,0x04,0x00,0x9a,0x00,0x00,0x00, -0xa9,0x00,0x00,0x00,0xa8,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x72,0x00,0x00,0x00,0xaa,0x00,0x00,0x00,0xa9,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xb1,0x00,0x00,0x00, -0x92,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x41,0x00,0x08,0x00, -0x93,0x00,0x00,0x00,0xb2,0x00,0x00,0x00,0x7b,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x7e,0x00,0x00,0x00,0x90,0x00,0x00,0x00, -0xb1,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x72,0x00,0x00,0x00, -0xb3,0x00,0x00,0x00,0xb2,0x00,0x00,0x00,0x71,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0xb4,0x00,0x00,0x00,0xb3,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xb5,0x00,0x00,0x00, -0xb4,0x00,0x00,0x00,0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xb6,0x00,0x00,0x00,0xb5,0x00,0x00,0x00,0x98,0x00,0x00,0x00, -0x72,0x00,0x04,0x00,0x9a,0x00,0x00,0x00,0xb7,0x00,0x00,0x00, -0xb6,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x72,0x00,0x00,0x00, -0xb8,0x00,0x00,0x00,0xb7,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xc0,0x00,0x00,0x00,0x92,0x00,0x00,0x00, -0xbf,0x00,0x00,0x00,0x41,0x00,0x08,0x00,0x93,0x00,0x00,0x00, -0xc1,0x00,0x00,0x00,0x7b,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0x7e,0x00,0x00,0x00,0x90,0x00,0x00,0x00,0xc0,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x72,0x00,0x00,0x00,0xc2,0x00,0x00,0x00, -0xc1,0x00,0x00,0x00,0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0xc3,0x00,0x00,0x00,0xc2,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xc4,0x00,0x00,0x00,0xc3,0x00,0x00,0x00, -0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xc5,0x00,0x00,0x00, -0xc4,0x00,0x00,0x00,0x98,0x00,0x00,0x00,0x72,0x00,0x04,0x00, -0x9a,0x00,0x00,0x00,0xc6,0x00,0x00,0x00,0xc5,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x72,0x00,0x00,0x00,0xc7,0x00,0x00,0x00, -0xc6,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xcf,0x00,0x00,0x00,0x92,0x00,0x00,0x00,0xce,0x00,0x00,0x00, -0x41,0x00,0x08,0x00,0x93,0x00,0x00,0x00,0xd0,0x00,0x00,0x00, -0x7b,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x7e,0x00,0x00,0x00, -0x90,0x00,0x00,0x00,0xcf,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x72,0x00,0x00,0x00,0xd1,0x00,0x00,0x00,0xd0,0x00,0x00,0x00, -0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0xd2,0x00,0x00,0x00, -0xd1,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xd3,0x00,0x00,0x00,0xd2,0x00,0x00,0x00,0xc7,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xd5,0x00,0x00,0x00,0xd3,0x00,0x00,0x00, -0xd4,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x72,0x00,0x00,0x00, -0xdc,0x00,0x00,0x00,0x94,0x00,0x00,0x00,0x71,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0xdd,0x00,0x00,0x00,0xdc,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xde,0x00,0x00,0x00, -0xdd,0x00,0x00,0x00,0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xe0,0x00,0x00,0x00,0xde,0x00,0x00,0x00,0xdf,0x00,0x00,0x00, -0xc3,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xe1,0x00,0x00,0x00, -0xe0,0x00,0x00,0x00,0x25,0x00,0x00,0x00,0xc5,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xe2,0x00,0x00,0x00,0xd5,0x00,0x00,0x00, -0xe1,0x00,0x00,0x00,0x72,0x00,0x04,0x00,0x9a,0x00,0x00,0x00, -0xe3,0x00,0x00,0x00,0xe2,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x72,0x00,0x00,0x00,0xe4,0x00,0x00,0x00,0xe3,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xec,0x00,0x00,0x00, -0x92,0x00,0x00,0x00,0xeb,0x00,0x00,0x00,0x41,0x00,0x08,0x00, -0x93,0x00,0x00,0x00,0xed,0x00,0x00,0x00,0x7b,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x7e,0x00,0x00,0x00,0x90,0x00,0x00,0x00, -0xec,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x72,0x00,0x00,0x00, -0xee,0x00,0x00,0x00,0xed,0x00,0x00,0x00,0x71,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0xef,0x00,0x00,0x00,0xee,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xf0,0x00,0x00,0x00, -0xef,0x00,0x00,0x00,0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xf1,0x00,0x00,0x00,0xf0,0x00,0x00,0x00,0xd4,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x72,0x00,0x00,0x00,0xf9,0x00,0x00,0x00, -0xa4,0x00,0x00,0x00,0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0xfa,0x00,0x00,0x00,0xf9,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xfb,0x00,0x00,0x00,0xfa,0x00,0x00,0x00, -0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xfc,0x00,0x00,0x00, -0xfb,0x00,0x00,0x00,0xdf,0x00,0x00,0x00,0xc3,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xfd,0x00,0x00,0x00,0xfc,0x00,0x00,0x00, -0x25,0x00,0x00,0x00,0xc5,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xfe,0x00,0x00,0x00,0xf1,0x00,0x00,0x00,0xfd,0x00,0x00,0x00, -0x72,0x00,0x04,0x00,0x9a,0x00,0x00,0x00,0xff,0x00,0x00,0x00, -0xfe,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x72,0x00,0x00,0x00, -0x00,0x01,0x00,0x00,0xff,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x72,0x00,0x00,0x00,0x09,0x01,0x00,0x00,0xd0,0x00,0x00,0x00, -0xc2,0x00,0x05,0x00,0x72,0x00,0x00,0x00,0x0a,0x01,0x00,0x00, -0x09,0x01,0x00,0x00,0x2e,0x00,0x00,0x00,0x71,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x0b,0x01,0x00,0x00,0x0a,0x01,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x0c,0x01,0x00,0x00, -0x0b,0x01,0x00,0x00,0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x0d,0x01,0x00,0x00,0x0c,0x01,0x00,0x00,0xd4,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x72,0x00,0x00,0x00,0x15,0x01,0x00,0x00, -0xb2,0x00,0x00,0x00,0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x16,0x01,0x00,0x00,0x15,0x01,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x17,0x01,0x00,0x00,0x16,0x01,0x00,0x00, -0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x18,0x01,0x00,0x00, -0x17,0x01,0x00,0x00,0xdf,0x00,0x00,0x00,0xc3,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x19,0x01,0x00,0x00,0x18,0x01,0x00,0x00, -0x25,0x00,0x00,0x00,0xc5,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x1a,0x01,0x00,0x00,0x0d,0x01,0x00,0x00,0x19,0x01,0x00,0x00, -0x72,0x00,0x04,0x00,0x9a,0x00,0x00,0x00,0x1b,0x01,0x00,0x00, -0x1a,0x01,0x00,0x00,0x7c,0x00,0x04,0x00,0x72,0x00,0x00,0x00, -0x1c,0x01,0x00,0x00,0x1b,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, -0x72,0x00,0x00,0x00,0x25,0x01,0x00,0x00,0xed,0x00,0x00,0x00, -0xc2,0x00,0x05,0x00,0x72,0x00,0x00,0x00,0x26,0x01,0x00,0x00, -0x25,0x01,0x00,0x00,0x2e,0x00,0x00,0x00,0x71,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x27,0x01,0x00,0x00,0x26,0x01,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x28,0x01,0x00,0x00, -0x27,0x01,0x00,0x00,0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x29,0x01,0x00,0x00,0x28,0x01,0x00,0x00,0xd4,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x72,0x00,0x00,0x00,0x31,0x01,0x00,0x00, -0xc1,0x00,0x00,0x00,0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x32,0x01,0x00,0x00,0x31,0x01,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x33,0x01,0x00,0x00,0x32,0x01,0x00,0x00, -0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x34,0x01,0x00,0x00, -0x33,0x01,0x00,0x00,0xdf,0x00,0x00,0x00,0xc3,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x35,0x01,0x00,0x00,0x34,0x01,0x00,0x00, -0x25,0x00,0x00,0x00,0xc5,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x36,0x01,0x00,0x00,0x29,0x01,0x00,0x00,0x35,0x01,0x00,0x00, -0x72,0x00,0x04,0x00,0x9a,0x00,0x00,0x00,0x37,0x01,0x00,0x00, -0x36,0x01,0x00,0x00,0x7c,0x00,0x04,0x00,0x72,0x00,0x00,0x00, -0x38,0x01,0x00,0x00,0x37,0x01,0x00,0x00,0x41,0x00,0x08,0x00, -0x93,0x00,0x00,0x00,0x3e,0x01,0x00,0x00,0x7b,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x7e,0x00,0x00,0x00,0x25,0x00,0x00,0x00, -0x46,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x72,0x00,0x00,0x00, -0x3f,0x01,0x00,0x00,0x3e,0x01,0x00,0x00,0x71,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x40,0x01,0x00,0x00,0x3f,0x01,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x41,0x01,0x00,0x00, -0x40,0x01,0x00,0x00,0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x42,0x01,0x00,0x00,0x41,0x01,0x00,0x00,0xd4,0x00,0x00,0x00, -0x72,0x00,0x04,0x00,0x9a,0x00,0x00,0x00,0x43,0x01,0x00,0x00, -0x42,0x01,0x00,0x00,0x7c,0x00,0x04,0x00,0x72,0x00,0x00,0x00, -0x44,0x01,0x00,0x00,0x43,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x4a,0x01,0x00,0x00,0x46,0x00,0x00,0x00, -0x90,0x00,0x00,0x00,0x41,0x00,0x08,0x00,0x93,0x00,0x00,0x00, -0x4b,0x01,0x00,0x00,0x7b,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0x7e,0x00,0x00,0x00,0x25,0x00,0x00,0x00,0x4a,0x01,0x00,0x00, -0x3d,0x00,0x04,0x00,0x72,0x00,0x00,0x00,0x4c,0x01,0x00,0x00, -0x4b,0x01,0x00,0x00,0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x4d,0x01,0x00,0x00,0x4c,0x01,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x4e,0x01,0x00,0x00,0x4d,0x01,0x00,0x00, -0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x4f,0x01,0x00,0x00, -0x4e,0x01,0x00,0x00,0xd4,0x00,0x00,0x00,0x72,0x00,0x04,0x00, -0x9a,0x00,0x00,0x00,0x50,0x01,0x00,0x00,0x4f,0x01,0x00,0x00, -0x7c,0x00,0x04,0x00,0x72,0x00,0x00,0x00,0x51,0x01,0x00,0x00, -0x50,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x57,0x01,0x00,0x00,0x46,0x00,0x00,0x00,0x25,0x00,0x00,0x00, -0x41,0x00,0x08,0x00,0x93,0x00,0x00,0x00,0x58,0x01,0x00,0x00, -0x7b,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x7e,0x00,0x00,0x00, -0x25,0x00,0x00,0x00,0x57,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, -0x72,0x00,0x00,0x00,0x59,0x01,0x00,0x00,0x58,0x01,0x00,0x00, -0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x5a,0x01,0x00,0x00, -0x59,0x01,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x5b,0x01,0x00,0x00,0x5a,0x01,0x00,0x00,0xc7,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x5c,0x01,0x00,0x00,0x5b,0x01,0x00,0x00, -0xd4,0x00,0x00,0x00,0x72,0x00,0x04,0x00,0x9a,0x00,0x00,0x00, -0x5d,0x01,0x00,0x00,0x5c,0x01,0x00,0x00,0x7c,0x00,0x04,0x00, -0x72,0x00,0x00,0x00,0x5e,0x01,0x00,0x00,0x5d,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x65,0x01,0x00,0x00, -0x46,0x00,0x00,0x00,0x64,0x01,0x00,0x00,0x41,0x00,0x08,0x00, -0x93,0x00,0x00,0x00,0x66,0x01,0x00,0x00,0x7b,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x7e,0x00,0x00,0x00,0x25,0x00,0x00,0x00, -0x65,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0x72,0x00,0x00,0x00, -0x67,0x01,0x00,0x00,0x66,0x01,0x00,0x00,0x71,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x68,0x01,0x00,0x00,0x67,0x01,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x69,0x01,0x00,0x00, -0x68,0x01,0x00,0x00,0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x6a,0x01,0x00,0x00,0x69,0x01,0x00,0x00,0xd4,0x00,0x00,0x00, -0x72,0x00,0x04,0x00,0x9a,0x00,0x00,0x00,0x6b,0x01,0x00,0x00, -0x6a,0x01,0x00,0x00,0x7c,0x00,0x04,0x00,0x72,0x00,0x00,0x00, -0x6c,0x01,0x00,0x00,0x6b,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, -0x72,0x00,0x00,0x00,0x73,0x01,0x00,0x00,0x3e,0x01,0x00,0x00, -0xc2,0x00,0x05,0x00,0x72,0x00,0x00,0x00,0x74,0x01,0x00,0x00, -0x73,0x01,0x00,0x00,0x2e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x72,0x00,0x00,0x00,0x7c,0x01,0x00,0x00,0x4b,0x01,0x00,0x00, -0xc2,0x00,0x05,0x00,0x72,0x00,0x00,0x00,0x7d,0x01,0x00,0x00, -0x7c,0x01,0x00,0x00,0x2e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x72,0x00,0x00,0x00,0x85,0x01,0x00,0x00,0x58,0x01,0x00,0x00, -0xc2,0x00,0x05,0x00,0x72,0x00,0x00,0x00,0x86,0x01,0x00,0x00, -0x85,0x01,0x00,0x00,0x2e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x72,0x00,0x00,0x00,0x8e,0x01,0x00,0x00,0x66,0x01,0x00,0x00, -0xc2,0x00,0x05,0x00,0x72,0x00,0x00,0x00,0x8f,0x01,0x00,0x00, -0x8e,0x01,0x00,0x00,0x2e,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x95,0x01,0x00,0x00,0x46,0x00,0x00,0x00, -0x48,0x00,0x00,0x00,0x41,0x00,0x08,0x00,0x93,0x00,0x00,0x00, -0x96,0x01,0x00,0x00,0x7b,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0x7e,0x00,0x00,0x00,0x25,0x00,0x00,0x00,0x95,0x01,0x00,0x00, -0x3d,0x00,0x04,0x00,0x72,0x00,0x00,0x00,0x97,0x01,0x00,0x00, -0x96,0x01,0x00,0x00,0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x98,0x01,0x00,0x00,0x97,0x01,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x99,0x01,0x00,0x00,0x98,0x01,0x00,0x00, -0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9a,0x01,0x00,0x00, -0x99,0x01,0x00,0x00,0xd4,0x00,0x00,0x00,0x72,0x00,0x04,0x00, -0x9a,0x00,0x00,0x00,0x9b,0x01,0x00,0x00,0x9a,0x01,0x00,0x00, -0x7c,0x00,0x04,0x00,0x72,0x00,0x00,0x00,0x9c,0x01,0x00,0x00, -0x9b,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xa3,0x01,0x00,0x00,0x46,0x00,0x00,0x00,0xa2,0x01,0x00,0x00, -0x41,0x00,0x08,0x00,0x93,0x00,0x00,0x00,0xa4,0x01,0x00,0x00, -0x7b,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x7e,0x00,0x00,0x00, -0x25,0x00,0x00,0x00,0xa3,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, -0x72,0x00,0x00,0x00,0xa5,0x01,0x00,0x00,0xa4,0x01,0x00,0x00, -0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0xa6,0x01,0x00,0x00, -0xa5,0x01,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xa7,0x01,0x00,0x00,0xa6,0x01,0x00,0x00,0xc7,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xa8,0x01,0x00,0x00,0xa7,0x01,0x00,0x00, -0xd4,0x00,0x00,0x00,0x72,0x00,0x04,0x00,0x9a,0x00,0x00,0x00, -0xa9,0x01,0x00,0x00,0xa8,0x01,0x00,0x00,0x7c,0x00,0x04,0x00, -0x72,0x00,0x00,0x00,0xaa,0x01,0x00,0x00,0xa9,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xb1,0x01,0x00,0x00, -0x46,0x00,0x00,0x00,0xb0,0x01,0x00,0x00,0x41,0x00,0x08,0x00, -0x93,0x00,0x00,0x00,0xb2,0x01,0x00,0x00,0x7b,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x7e,0x00,0x00,0x00,0x25,0x00,0x00,0x00, -0xb1,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0x72,0x00,0x00,0x00, -0xb3,0x01,0x00,0x00,0xb2,0x01,0x00,0x00,0x71,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0xb4,0x01,0x00,0x00,0xb3,0x01,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xb5,0x01,0x00,0x00, -0xb4,0x01,0x00,0x00,0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xb6,0x01,0x00,0x00,0xb5,0x01,0x00,0x00,0xd4,0x00,0x00,0x00, -0x72,0x00,0x04,0x00,0x9a,0x00,0x00,0x00,0xb7,0x01,0x00,0x00, -0xb6,0x01,0x00,0x00,0x7c,0x00,0x04,0x00,0x72,0x00,0x00,0x00, -0xb8,0x01,0x00,0x00,0xb7,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xbf,0x01,0x00,0x00,0x46,0x00,0x00,0x00, -0xbe,0x01,0x00,0x00,0x41,0x00,0x08,0x00,0x93,0x00,0x00,0x00, -0xc0,0x01,0x00,0x00,0x7b,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0x7e,0x00,0x00,0x00,0x25,0x00,0x00,0x00,0xbf,0x01,0x00,0x00, -0x3d,0x00,0x04,0x00,0x72,0x00,0x00,0x00,0xc1,0x01,0x00,0x00, -0xc0,0x01,0x00,0x00,0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0xc2,0x01,0x00,0x00,0xc1,0x01,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xc3,0x01,0x00,0x00,0xc2,0x01,0x00,0x00, -0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xc4,0x01,0x00,0x00, -0xc3,0x01,0x00,0x00,0xd4,0x00,0x00,0x00,0x72,0x00,0x04,0x00, -0x9a,0x00,0x00,0x00,0xc5,0x01,0x00,0x00,0xc4,0x01,0x00,0x00, -0x7c,0x00,0x04,0x00,0x72,0x00,0x00,0x00,0xc6,0x01,0x00,0x00, -0xc5,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0x72,0x00,0x00,0x00, -0xce,0x01,0x00,0x00,0x96,0x01,0x00,0x00,0xc2,0x00,0x05,0x00, -0x72,0x00,0x00,0x00,0xcf,0x01,0x00,0x00,0xce,0x01,0x00,0x00, -0x2e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x72,0x00,0x00,0x00, -0xd7,0x01,0x00,0x00,0xa4,0x01,0x00,0x00,0xc2,0x00,0x05,0x00, -0x72,0x00,0x00,0x00,0xd8,0x01,0x00,0x00,0xd7,0x01,0x00,0x00, -0x2e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x72,0x00,0x00,0x00, -0xe0,0x01,0x00,0x00,0xb2,0x01,0x00,0x00,0xc2,0x00,0x05,0x00, -0x72,0x00,0x00,0x00,0xe1,0x01,0x00,0x00,0xe0,0x01,0x00,0x00, -0x2e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x72,0x00,0x00,0x00, -0xe9,0x01,0x00,0x00,0xc0,0x01,0x00,0x00,0xc2,0x00,0x05,0x00, -0x72,0x00,0x00,0x00,0xea,0x01,0x00,0x00,0xe9,0x01,0x00,0x00, -0x2e,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00, -0xf0,0x01,0x00,0x00,0x15,0x00,0x00,0x00,0x90,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xf1,0x01,0x00,0x00, -0xf0,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xf3,0x01,0x00,0x00,0xf1,0x01,0x00,0x00,0x69,0x00,0x00,0x00, -0x41,0x00,0x06,0x00,0xf4,0x01,0x00,0x00,0xf5,0x01,0x00,0x00, -0xef,0x01,0x00,0x00,0x16,0x00,0x00,0x00,0xf3,0x01,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00,0xf6,0x01,0x00,0x00, -0xf5,0x01,0x00,0x00,0x70,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0xf8,0x01,0x00,0x00,0x44,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xfe,0x01,0x00,0x00,0xf3,0x01,0x00,0x00, -0x90,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0xf4,0x01,0x00,0x00, -0xff,0x01,0x00,0x00,0xef,0x01,0x00,0x00,0x16,0x00,0x00,0x00, -0xfe,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0x00,0x02,0x00,0x00,0xff,0x01,0x00,0x00,0x70,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0x02,0x02,0x00,0x00,0x51,0x01,0x00,0x00, -0x85,0x00,0x05,0x00,0x4d,0x00,0x00,0x00,0x03,0x02,0x00,0x00, -0x00,0x02,0x00,0x00,0x02,0x02,0x00,0x00,0x0c,0x00,0x08,0x00, -0x4d,0x00,0x00,0x00,0x04,0x02,0x00,0x00,0x01,0x00,0x00,0x00, -0x32,0x00,0x00,0x00,0xf6,0x01,0x00,0x00,0xf8,0x01,0x00,0x00, -0x03,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x09,0x02,0x00,0x00,0xf3,0x01,0x00,0x00,0x25,0x00,0x00,0x00, -0x41,0x00,0x06,0x00,0xf4,0x01,0x00,0x00,0x0a,0x02,0x00,0x00, -0xef,0x01,0x00,0x00,0x16,0x00,0x00,0x00,0x09,0x02,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00,0x0b,0x02,0x00,0x00, -0x0a,0x02,0x00,0x00,0x70,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0x0d,0x02,0x00,0x00,0x5e,0x01,0x00,0x00,0x0c,0x00,0x08,0x00, -0x4d,0x00,0x00,0x00,0x0f,0x02,0x00,0x00,0x01,0x00,0x00,0x00, -0x32,0x00,0x00,0x00,0x0b,0x02,0x00,0x00,0x0d,0x02,0x00,0x00, -0x04,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x14,0x02,0x00,0x00,0xf3,0x01,0x00,0x00,0x64,0x01,0x00,0x00, -0x41,0x00,0x06,0x00,0xf4,0x01,0x00,0x00,0x15,0x02,0x00,0x00, -0xef,0x01,0x00,0x00,0x16,0x00,0x00,0x00,0x14,0x02,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00,0x16,0x02,0x00,0x00, -0x15,0x02,0x00,0x00,0x70,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0x18,0x02,0x00,0x00,0x6c,0x01,0x00,0x00,0x0c,0x00,0x08,0x00, -0x4d,0x00,0x00,0x00,0x1a,0x02,0x00,0x00,0x01,0x00,0x00,0x00, -0x32,0x00,0x00,0x00,0x16,0x02,0x00,0x00,0x18,0x02,0x00,0x00, -0x0f,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x20,0x02,0x00,0x00,0xf3,0x01,0x00,0x00,0x42,0x00,0x00,0x00, -0x41,0x00,0x06,0x00,0xf4,0x01,0x00,0x00,0x21,0x02,0x00,0x00, -0xef,0x01,0x00,0x00,0x16,0x00,0x00,0x00,0x20,0x02,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00,0x22,0x02,0x00,0x00, -0x21,0x02,0x00,0x00,0x70,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0x24,0x02,0x00,0x00,0x74,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x2b,0x02,0x00,0x00,0xf3,0x01,0x00,0x00, -0x2a,0x02,0x00,0x00,0x41,0x00,0x06,0x00,0xf4,0x01,0x00,0x00, -0x2c,0x02,0x00,0x00,0xef,0x01,0x00,0x00,0x16,0x00,0x00,0x00, -0x2b,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0x2d,0x02,0x00,0x00,0x2c,0x02,0x00,0x00,0x70,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0x2f,0x02,0x00,0x00,0x7d,0x01,0x00,0x00, -0x85,0x00,0x05,0x00,0x4d,0x00,0x00,0x00,0x30,0x02,0x00,0x00, -0x2d,0x02,0x00,0x00,0x2f,0x02,0x00,0x00,0x0c,0x00,0x08,0x00, -0x4d,0x00,0x00,0x00,0x31,0x02,0x00,0x00,0x01,0x00,0x00,0x00, -0x32,0x00,0x00,0x00,0x22,0x02,0x00,0x00,0x24,0x02,0x00,0x00, -0x30,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x37,0x02,0x00,0x00,0xf3,0x01,0x00,0x00,0x36,0x02,0x00,0x00, -0x41,0x00,0x06,0x00,0xf4,0x01,0x00,0x00,0x38,0x02,0x00,0x00, -0xef,0x01,0x00,0x00,0x16,0x00,0x00,0x00,0x37,0x02,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00,0x39,0x02,0x00,0x00, -0x38,0x02,0x00,0x00,0x70,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0x3b,0x02,0x00,0x00,0x86,0x01,0x00,0x00,0x0c,0x00,0x08,0x00, -0x4d,0x00,0x00,0x00,0x3d,0x02,0x00,0x00,0x01,0x00,0x00,0x00, -0x32,0x00,0x00,0x00,0x39,0x02,0x00,0x00,0x3b,0x02,0x00,0x00, -0x31,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x43,0x02,0x00,0x00,0xf3,0x01,0x00,0x00,0x42,0x02,0x00,0x00, -0x41,0x00,0x06,0x00,0xf4,0x01,0x00,0x00,0x44,0x02,0x00,0x00, -0xef,0x01,0x00,0x00,0x16,0x00,0x00,0x00,0x43,0x02,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00,0x45,0x02,0x00,0x00, -0x44,0x02,0x00,0x00,0x70,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0x47,0x02,0x00,0x00,0x8f,0x01,0x00,0x00,0x0c,0x00,0x08,0x00, -0x4d,0x00,0x00,0x00,0x49,0x02,0x00,0x00,0x01,0x00,0x00,0x00, -0x32,0x00,0x00,0x00,0x45,0x02,0x00,0x00,0x47,0x02,0x00,0x00, -0x3d,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x4e,0x02,0x00,0x00,0xf1,0x01,0x00,0x00,0x6d,0x00,0x00,0x00, -0x41,0x00,0x06,0x00,0xf4,0x01,0x00,0x00,0x4f,0x02,0x00,0x00, -0xef,0x01,0x00,0x00,0x16,0x00,0x00,0x00,0x4e,0x02,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00,0x50,0x02,0x00,0x00, -0x4f,0x02,0x00,0x00,0x70,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0x52,0x02,0x00,0x00,0x9c,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x58,0x02,0x00,0x00,0x4e,0x02,0x00,0x00, -0x90,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0xf4,0x01,0x00,0x00, -0x59,0x02,0x00,0x00,0xef,0x01,0x00,0x00,0x16,0x00,0x00,0x00, -0x58,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0x5a,0x02,0x00,0x00,0x59,0x02,0x00,0x00,0x70,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0x5c,0x02,0x00,0x00,0xaa,0x01,0x00,0x00, -0x85,0x00,0x05,0x00,0x4d,0x00,0x00,0x00,0x5d,0x02,0x00,0x00, -0x5a,0x02,0x00,0x00,0x5c,0x02,0x00,0x00,0x0c,0x00,0x08,0x00, -0x4d,0x00,0x00,0x00,0x5e,0x02,0x00,0x00,0x01,0x00,0x00,0x00, -0x32,0x00,0x00,0x00,0x50,0x02,0x00,0x00,0x52,0x02,0x00,0x00, -0x5d,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x63,0x02,0x00,0x00,0x4e,0x02,0x00,0x00,0x25,0x00,0x00,0x00, -0x41,0x00,0x06,0x00,0xf4,0x01,0x00,0x00,0x64,0x02,0x00,0x00, -0xef,0x01,0x00,0x00,0x16,0x00,0x00,0x00,0x63,0x02,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00,0x65,0x02,0x00,0x00, -0x64,0x02,0x00,0x00,0x70,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0x67,0x02,0x00,0x00,0xb8,0x01,0x00,0x00,0x0c,0x00,0x08,0x00, -0x4d,0x00,0x00,0x00,0x69,0x02,0x00,0x00,0x01,0x00,0x00,0x00, -0x32,0x00,0x00,0x00,0x65,0x02,0x00,0x00,0x67,0x02,0x00,0x00, -0x5e,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x6e,0x02,0x00,0x00,0x4e,0x02,0x00,0x00,0x64,0x01,0x00,0x00, -0x41,0x00,0x06,0x00,0xf4,0x01,0x00,0x00,0x6f,0x02,0x00,0x00, -0xef,0x01,0x00,0x00,0x16,0x00,0x00,0x00,0x6e,0x02,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00,0x70,0x02,0x00,0x00, -0x6f,0x02,0x00,0x00,0x70,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0x72,0x02,0x00,0x00,0xc6,0x01,0x00,0x00,0x0c,0x00,0x08,0x00, -0x4d,0x00,0x00,0x00,0x74,0x02,0x00,0x00,0x01,0x00,0x00,0x00, -0x32,0x00,0x00,0x00,0x70,0x02,0x00,0x00,0x72,0x02,0x00,0x00, -0x69,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x7a,0x02,0x00,0x00,0x4e,0x02,0x00,0x00,0x42,0x00,0x00,0x00, -0x41,0x00,0x06,0x00,0xf4,0x01,0x00,0x00,0x7b,0x02,0x00,0x00, -0xef,0x01,0x00,0x00,0x16,0x00,0x00,0x00,0x7a,0x02,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00,0x7c,0x02,0x00,0x00, -0x7b,0x02,0x00,0x00,0x70,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0x7e,0x02,0x00,0x00,0xcf,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x84,0x02,0x00,0x00,0x4e,0x02,0x00,0x00, -0x2a,0x02,0x00,0x00,0x41,0x00,0x06,0x00,0xf4,0x01,0x00,0x00, -0x85,0x02,0x00,0x00,0xef,0x01,0x00,0x00,0x16,0x00,0x00,0x00, -0x84,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0x86,0x02,0x00,0x00,0x85,0x02,0x00,0x00,0x70,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0x88,0x02,0x00,0x00,0xd8,0x01,0x00,0x00, -0x85,0x00,0x05,0x00,0x4d,0x00,0x00,0x00,0x89,0x02,0x00,0x00, -0x86,0x02,0x00,0x00,0x88,0x02,0x00,0x00,0x0c,0x00,0x08,0x00, -0x4d,0x00,0x00,0x00,0x8a,0x02,0x00,0x00,0x01,0x00,0x00,0x00, -0x32,0x00,0x00,0x00,0x7c,0x02,0x00,0x00,0x7e,0x02,0x00,0x00, -0x89,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x8f,0x02,0x00,0x00,0x4e,0x02,0x00,0x00,0x36,0x02,0x00,0x00, -0x41,0x00,0x06,0x00,0xf4,0x01,0x00,0x00,0x90,0x02,0x00,0x00, -0xef,0x01,0x00,0x00,0x16,0x00,0x00,0x00,0x8f,0x02,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00,0x91,0x02,0x00,0x00, -0x90,0x02,0x00,0x00,0x70,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0x93,0x02,0x00,0x00,0xe1,0x01,0x00,0x00,0x0c,0x00,0x08,0x00, -0x4d,0x00,0x00,0x00,0x95,0x02,0x00,0x00,0x01,0x00,0x00,0x00, -0x32,0x00,0x00,0x00,0x91,0x02,0x00,0x00,0x93,0x02,0x00,0x00, -0x8a,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x9a,0x02,0x00,0x00,0x4e,0x02,0x00,0x00,0x42,0x02,0x00,0x00, -0x41,0x00,0x06,0x00,0xf4,0x01,0x00,0x00,0x9b,0x02,0x00,0x00, -0xef,0x01,0x00,0x00,0x16,0x00,0x00,0x00,0x9a,0x02,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00,0x9c,0x02,0x00,0x00, -0x9b,0x02,0x00,0x00,0x70,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0x9e,0x02,0x00,0x00,0xea,0x01,0x00,0x00,0x0c,0x00,0x08,0x00, -0x4d,0x00,0x00,0x00,0xa0,0x02,0x00,0x00,0x01,0x00,0x00,0x00, -0x32,0x00,0x00,0x00,0x9c,0x02,0x00,0x00,0x9e,0x02,0x00,0x00, -0x95,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0xa7,0x02,0x00,0x00,0xf5,0x01,0x00,0x00,0x70,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0xa9,0x02,0x00,0x00,0xb8,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00,0xb1,0x02,0x00,0x00, -0x21,0x02,0x00,0x00,0x70,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0xb3,0x02,0x00,0x00,0xc7,0x00,0x00,0x00,0x85,0x00,0x05,0x00, -0x4d,0x00,0x00,0x00,0xb4,0x02,0x00,0x00,0xb1,0x02,0x00,0x00, -0xb3,0x02,0x00,0x00,0x0c,0x00,0x08,0x00,0x4d,0x00,0x00,0x00, -0xb5,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00, -0xa7,0x02,0x00,0x00,0xa9,0x02,0x00,0x00,0xb4,0x02,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00,0xbb,0x02,0x00,0x00, -0x4f,0x02,0x00,0x00,0x70,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0xbd,0x02,0x00,0x00,0x1c,0x01,0x00,0x00,0x0c,0x00,0x08,0x00, -0x4d,0x00,0x00,0x00,0xbf,0x02,0x00,0x00,0x01,0x00,0x00,0x00, -0x32,0x00,0x00,0x00,0xbb,0x02,0x00,0x00,0xbd,0x02,0x00,0x00, -0xb5,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0xc6,0x02,0x00,0x00,0x7b,0x02,0x00,0x00,0x70,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0xc8,0x02,0x00,0x00,0x38,0x01,0x00,0x00, -0x0c,0x00,0x08,0x00,0x4d,0x00,0x00,0x00,0xca,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0xc6,0x02,0x00,0x00, -0xc8,0x02,0x00,0x00,0xbf,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0xd1,0x02,0x00,0x00,0xff,0x01,0x00,0x00, -0x0c,0x00,0x08,0x00,0x4d,0x00,0x00,0x00,0xd5,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0xd1,0x02,0x00,0x00, -0xa9,0x02,0x00,0x00,0xca,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0xdc,0x02,0x00,0x00,0x2c,0x02,0x00,0x00, -0x0c,0x00,0x08,0x00,0x4d,0x00,0x00,0x00,0xe0,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0xdc,0x02,0x00,0x00, -0xb3,0x02,0x00,0x00,0xd5,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0xe7,0x02,0x00,0x00,0x59,0x02,0x00,0x00, -0x0c,0x00,0x08,0x00,0x4d,0x00,0x00,0x00,0xeb,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0xe7,0x02,0x00,0x00, -0xbd,0x02,0x00,0x00,0xe0,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0xf2,0x02,0x00,0x00,0x85,0x02,0x00,0x00, -0x0c,0x00,0x08,0x00,0x4d,0x00,0x00,0x00,0xf6,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0xf2,0x02,0x00,0x00, -0xc8,0x02,0x00,0x00,0xeb,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0xfd,0x02,0x00,0x00,0x0a,0x02,0x00,0x00, -0x0c,0x00,0x08,0x00,0x4d,0x00,0x00,0x00,0x01,0x03,0x00,0x00, -0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0xfd,0x02,0x00,0x00, -0xa9,0x02,0x00,0x00,0xf6,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0x08,0x03,0x00,0x00,0x38,0x02,0x00,0x00, -0x0c,0x00,0x08,0x00,0x4d,0x00,0x00,0x00,0x0c,0x03,0x00,0x00, -0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x08,0x03,0x00,0x00, -0xb3,0x02,0x00,0x00,0x01,0x03,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0x13,0x03,0x00,0x00,0x64,0x02,0x00,0x00, -0x0c,0x00,0x08,0x00,0x4d,0x00,0x00,0x00,0x17,0x03,0x00,0x00, -0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x13,0x03,0x00,0x00, -0xbd,0x02,0x00,0x00,0x0c,0x03,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0x1e,0x03,0x00,0x00,0x90,0x02,0x00,0x00, -0x0c,0x00,0x08,0x00,0x4d,0x00,0x00,0x00,0x22,0x03,0x00,0x00, -0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x1e,0x03,0x00,0x00, -0xc8,0x02,0x00,0x00,0x17,0x03,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0x29,0x03,0x00,0x00,0x15,0x02,0x00,0x00, -0x0c,0x00,0x08,0x00,0x4d,0x00,0x00,0x00,0x2d,0x03,0x00,0x00, -0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x29,0x03,0x00,0x00, -0xa9,0x02,0x00,0x00,0x22,0x03,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0x34,0x03,0x00,0x00,0x44,0x02,0x00,0x00, -0x0c,0x00,0x08,0x00,0x4d,0x00,0x00,0x00,0x38,0x03,0x00,0x00, -0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x34,0x03,0x00,0x00, -0xb3,0x02,0x00,0x00,0x2d,0x03,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0x3f,0x03,0x00,0x00,0x6f,0x02,0x00,0x00, -0x0c,0x00,0x08,0x00,0x4d,0x00,0x00,0x00,0x43,0x03,0x00,0x00, -0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x3f,0x03,0x00,0x00, -0xbd,0x02,0x00,0x00,0x38,0x03,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0x4a,0x03,0x00,0x00,0x9b,0x02,0x00,0x00, -0x0c,0x00,0x08,0x00,0x4d,0x00,0x00,0x00,0x4e,0x03,0x00,0x00, -0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x4a,0x03,0x00,0x00, -0xc8,0x02,0x00,0x00,0x43,0x03,0x00,0x00,0x70,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0x56,0x03,0x00,0x00,0x9c,0x00,0x00,0x00, -0x70,0x00,0x04,0x00,0x4d,0x00,0x00,0x00,0x5a,0x03,0x00,0x00, -0xaa,0x00,0x00,0x00,0x85,0x00,0x05,0x00,0x4d,0x00,0x00,0x00, -0x5b,0x03,0x00,0x00,0x49,0x02,0x00,0x00,0x5a,0x03,0x00,0x00, -0x0c,0x00,0x08,0x00,0x4d,0x00,0x00,0x00,0x5c,0x03,0x00,0x00, -0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x1a,0x02,0x00,0x00, -0x56,0x03,0x00,0x00,0x5b,0x03,0x00,0x00,0x70,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0x5f,0x03,0x00,0x00,0xe4,0x00,0x00,0x00, -0x0c,0x00,0x08,0x00,0x4d,0x00,0x00,0x00,0x61,0x03,0x00,0x00, -0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x74,0x02,0x00,0x00, -0x5f,0x03,0x00,0x00,0x5c,0x03,0x00,0x00,0x70,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0x64,0x03,0x00,0x00,0x00,0x01,0x00,0x00, -0x0c,0x00,0x08,0x00,0x4d,0x00,0x00,0x00,0x66,0x03,0x00,0x00, -0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0xa0,0x02,0x00,0x00, -0x64,0x03,0x00,0x00,0x61,0x03,0x00,0x00,0x85,0x00,0x05,0x00, -0x4d,0x00,0x00,0x00,0x6a,0x03,0x00,0x00,0x8a,0x00,0x00,0x00, -0x4e,0x03,0x00,0x00,0x7f,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0xa1,0x03,0x00,0x00,0x6a,0x03,0x00,0x00,0x0c,0x00,0x08,0x00, -0x4d,0x00,0x00,0x00,0x6b,0x03,0x00,0x00,0x01,0x00,0x00,0x00, -0x32,0x00,0x00,0x00,0x82,0x00,0x00,0x00,0x66,0x03,0x00,0x00, -0xa1,0x03,0x00,0x00,0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0x6d,0x03,0x00,0x00,0x59,0x00,0x00,0x00,0x81,0x00,0x05,0x00, -0x4d,0x00,0x00,0x00,0x6e,0x03,0x00,0x00,0x6d,0x03,0x00,0x00, -0x6b,0x03,0x00,0x00,0x3e,0x00,0x03,0x00,0x59,0x00,0x00,0x00, -0x6e,0x03,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x71,0x03,0x00,0x00,0x9d,0x03,0x00,0x00,0x25,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x5c,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x5e,0x00,0x00,0x00,0xe0,0x00,0x04,0x00,0x72,0x03,0x00,0x00, -0x72,0x03,0x00,0x00,0x73,0x03,0x00,0x00,0xf9,0x00,0x02,0x00, -0x75,0x03,0x00,0x00,0xf8,0x00,0x02,0x00,0x75,0x03,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x9e,0x03,0x00,0x00, -0x52,0x00,0x00,0x00,0x5e,0x00,0x00,0x00,0x8c,0x03,0x00,0x00, -0x78,0x03,0x00,0x00,0xad,0x00,0x05,0x00,0x63,0x00,0x00,0x00, -0x7b,0x03,0x00,0x00,0x9e,0x03,0x00,0x00,0x16,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0x77,0x03,0x00,0x00,0x78,0x03,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x7b,0x03,0x00,0x00, -0x76,0x03,0x00,0x00,0x77,0x03,0x00,0x00,0xf8,0x00,0x02,0x00, -0x76,0x03,0x00,0x00,0xb1,0x00,0x05,0x00,0x63,0x00,0x00,0x00, -0x7e,0x03,0x00,0x00,0x26,0x00,0x00,0x00,0x9e,0x03,0x00,0x00, -0xf7,0x00,0x03,0x00,0x80,0x03,0x00,0x00,0x00,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x7e,0x03,0x00,0x00,0x7f,0x03,0x00,0x00, -0x80,0x03,0x00,0x00,0xf8,0x00,0x02,0x00,0x7f,0x03,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x84,0x03,0x00,0x00, -0x26,0x00,0x00,0x00,0x9e,0x03,0x00,0x00,0x41,0x00,0x05,0x00, -0x58,0x00,0x00,0x00,0x85,0x03,0x00,0x00,0x51,0x00,0x00,0x00, -0x84,0x03,0x00,0x00,0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0x86,0x03,0x00,0x00,0x85,0x03,0x00,0x00,0x41,0x00,0x05,0x00, -0x58,0x00,0x00,0x00,0x87,0x03,0x00,0x00,0x51,0x00,0x00,0x00, -0x26,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0x88,0x03,0x00,0x00,0x87,0x03,0x00,0x00,0x81,0x00,0x05,0x00, -0x4d,0x00,0x00,0x00,0x89,0x03,0x00,0x00,0x88,0x03,0x00,0x00, -0x86,0x03,0x00,0x00,0x3e,0x00,0x03,0x00,0x87,0x03,0x00,0x00, -0x89,0x03,0x00,0x00,0xf9,0x00,0x02,0x00,0x80,0x03,0x00,0x00, -0xf8,0x00,0x02,0x00,0x80,0x03,0x00,0x00,0xe0,0x00,0x04,0x00, -0x72,0x03,0x00,0x00,0x72,0x03,0x00,0x00,0x73,0x03,0x00,0x00, -0xf9,0x00,0x02,0x00,0x78,0x03,0x00,0x00,0xf8,0x00,0x02,0x00, -0x78,0x03,0x00,0x00,0xc3,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x8c,0x03,0x00,0x00,0x9e,0x03,0x00,0x00,0x90,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x75,0x03,0x00,0x00,0xf8,0x00,0x02,0x00, -0x77,0x03,0x00,0x00,0xaa,0x00,0x05,0x00,0x63,0x00,0x00,0x00, -0x8e,0x03,0x00,0x00,0x26,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0xf7,0x00,0x03,0x00,0x90,0x03,0x00,0x00,0x00,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x8e,0x03,0x00,0x00,0x8f,0x03,0x00,0x00, -0x90,0x03,0x00,0x00,0xf8,0x00,0x02,0x00,0x8f,0x03,0x00,0x00, -0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00,0x95,0x03,0x00,0x00, -0x15,0x00,0x00,0x00,0x25,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x96,0x03,0x00,0x00,0x95,0x03,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x98,0x03,0x00,0x00, -0x96,0x03,0x00,0x00,0x11,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x58,0x00,0x00,0x00,0x99,0x03,0x00,0x00,0x51,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0x9a,0x03,0x00,0x00,0x99,0x03,0x00,0x00,0x41,0x00,0x06,0x00, -0xf4,0x01,0x00,0x00,0x9b,0x03,0x00,0x00,0x94,0x03,0x00,0x00, -0x16,0x00,0x00,0x00,0x98,0x03,0x00,0x00,0x3e,0x00,0x03,0x00, -0x9b,0x03,0x00,0x00,0x9a,0x03,0x00,0x00,0xf9,0x00,0x02,0x00, -0x90,0x03,0x00,0x00,0xf8,0x00,0x02,0x00,0x90,0x03,0x00,0x00, +0x3f,0x02,0x00,0x00,0x23,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x6f,0x03,0x00,0x00,0x08,0x01,0x00,0x00, +0x1d,0x00,0x03,0x00,0x8d,0x03,0x00,0x00,0x4a,0x00,0x00,0x00, +0x1e,0x00,0x03,0x00,0x8e,0x03,0x00,0x00,0x8d,0x03,0x00,0x00, +0x20,0x00,0x04,0x00,0x8f,0x03,0x00,0x00,0x0c,0x00,0x00,0x00, +0x8e,0x03,0x00,0x00,0x3b,0x00,0x04,0x00,0x8f,0x03,0x00,0x00, +0x90,0x03,0x00,0x00,0x0c,0x00,0x00,0x00,0x2c,0x00,0x06,0x00, +0x09,0x00,0x00,0x00,0x98,0x03,0x00,0x00,0x3f,0x00,0x00,0x00, +0x82,0x00,0x00,0x00,0x82,0x00,0x00,0x00,0x36,0x00,0x05,0x00, +0x02,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x03,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x05,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00,0x0e,0x00,0x00,0x00, +0x0b,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x0e,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x16,0x00,0x00,0x00,0x17,0x00,0x00,0x00, +0x13,0x00,0x00,0x00,0x15,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x17,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x1a,0x00,0x00,0x00, +0x18,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x1e,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, +0x1a,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00, +0x21,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x22,0x00,0x00,0x00, +0x21,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x24,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x28,0x00,0x00,0x00, +0x22,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x2c,0x00,0x00,0x00,0x24,0x00,0x00,0x00, +0x2b,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x30,0x00,0x00,0x00,0x2b,0x00,0x00,0x00,0x2c,0x00,0x00,0x00, +0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x31,0x00,0x00,0x00, +0x24,0x00,0x00,0x00,0x30,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0x2c,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x37,0x00,0x00,0x00,0x2c,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x31,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x3c,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, +0x37,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x3d,0x00,0x00,0x00,0x2b,0x00,0x00,0x00,0x3c,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x41,0x00,0x00,0x00, +0x3f,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x43,0x00,0x00,0x00,0x41,0x00,0x00,0x00, +0x3d,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x47,0x00,0x00,0x00,0x45,0x00,0x00,0x00,0x34,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x49,0x00,0x00,0x00, +0x47,0x00,0x00,0x00,0x3d,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x50,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, +0x28,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x52,0x00,0x00,0x00,0x50,0x00,0x00,0x00,0x24,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x54,0x00,0x00,0x00,0x55,0x00,0x00,0x00, +0x4d,0x00,0x00,0x00,0x52,0x00,0x00,0x00,0x3e,0x00,0x03,0x00, +0x55,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x58,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x58,0x00,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x99,0x03,0x00,0x00, +0x28,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0x6e,0x03,0x00,0x00, +0x59,0x00,0x00,0x00,0xb0,0x00,0x05,0x00,0x5f,0x00,0x00,0x00, +0x60,0x00,0x00,0x00,0x99,0x03,0x00,0x00,0x1a,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0x5a,0x00,0x00,0x00,0x59,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x60,0x00,0x00,0x00, +0x59,0x00,0x00,0x00,0x5a,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0x59,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x63,0x00,0x00,0x00,0x99,0x03,0x00,0x00,0x19,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x65,0x00,0x00,0x00, +0x63,0x00,0x00,0x00,0x49,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x69,0x00,0x00,0x00,0x65,0x00,0x00,0x00, +0x68,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x79,0x00,0x00,0x00,0x1e,0x00,0x00,0x00,0x99,0x03,0x00,0x00, +0x41,0x00,0x08,0x00,0x7a,0x00,0x00,0x00,0x7b,0x00,0x00,0x00, +0x76,0x00,0x00,0x00,0x15,0x00,0x00,0x00,0x79,0x00,0x00,0x00, +0x15,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x6c,0x00,0x00,0x00,0x7c,0x00,0x00,0x00,0x7b,0x00,0x00,0x00, +0x73,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0x7d,0x00,0x00,0x00, +0x7c,0x00,0x00,0x00,0x41,0x00,0x08,0x00,0x7a,0x00,0x00,0x00, +0x83,0x00,0x00,0x00,0x76,0x00,0x00,0x00,0x15,0x00,0x00,0x00, +0x79,0x00,0x00,0x00,0x15,0x00,0x00,0x00,0x82,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x6c,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0x83,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, +0x85,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x8d,0x00,0x00,0x00,0x34,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x41,0x00,0x08,0x00,0x8e,0x00,0x00,0x00, +0x8f,0x00,0x00,0x00,0x76,0x00,0x00,0x00,0x15,0x00,0x00,0x00, +0x79,0x00,0x00,0x00,0x8b,0x00,0x00,0x00,0x8d,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x6e,0x00,0x00,0x00,0x90,0x00,0x00,0x00, +0x8f,0x00,0x00,0x00,0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x91,0x00,0x00,0x00,0x90,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, +0x14,0x00,0x00,0x00,0x92,0x00,0x00,0x00,0x91,0x00,0x00,0x00, +0xc7,0x00,0x05,0x00,0x14,0x00,0x00,0x00,0x94,0x00,0x00,0x00, +0x92,0x00,0x00,0x00,0x93,0x00,0x00,0x00,0x72,0x00,0x04,0x00, +0x95,0x00,0x00,0x00,0x96,0x00,0x00,0x00,0x94,0x00,0x00,0x00, +0x7c,0x00,0x04,0x00,0x6e,0x00,0x00,0x00,0x97,0x00,0x00,0x00, +0x96,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x9e,0x00,0x00,0x00,0x8d,0x00,0x00,0x00,0x82,0x00,0x00,0x00, +0x41,0x00,0x08,0x00,0x8e,0x00,0x00,0x00,0x9f,0x00,0x00,0x00, +0x76,0x00,0x00,0x00,0x15,0x00,0x00,0x00,0x79,0x00,0x00,0x00, +0x8b,0x00,0x00,0x00,0x9e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x6e,0x00,0x00,0x00,0xa0,0x00,0x00,0x00,0x9f,0x00,0x00,0x00, +0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xa1,0x00,0x00,0x00, +0xa0,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x14,0x00,0x00,0x00, +0xa2,0x00,0x00,0x00,0xa1,0x00,0x00,0x00,0xc7,0x00,0x05,0x00, +0x14,0x00,0x00,0x00,0xa3,0x00,0x00,0x00,0xa2,0x00,0x00,0x00, +0x93,0x00,0x00,0x00,0x72,0x00,0x04,0x00,0x95,0x00,0x00,0x00, +0xa4,0x00,0x00,0x00,0xa3,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, +0x6e,0x00,0x00,0x00,0xa5,0x00,0x00,0x00,0xa4,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xac,0x00,0x00,0x00, +0x8d,0x00,0x00,0x00,0x2b,0x00,0x00,0x00,0x41,0x00,0x08,0x00, +0x8e,0x00,0x00,0x00,0xad,0x00,0x00,0x00,0x76,0x00,0x00,0x00, +0x15,0x00,0x00,0x00,0x79,0x00,0x00,0x00,0x8b,0x00,0x00,0x00, +0xac,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x6e,0x00,0x00,0x00, +0xae,0x00,0x00,0x00,0xad,0x00,0x00,0x00,0x71,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xaf,0x00,0x00,0x00,0xae,0x00,0x00,0x00, +0x7c,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0xb0,0x00,0x00,0x00, +0xaf,0x00,0x00,0x00,0xc7,0x00,0x05,0x00,0x14,0x00,0x00,0x00, +0xb1,0x00,0x00,0x00,0xb0,0x00,0x00,0x00,0x93,0x00,0x00,0x00, +0x72,0x00,0x04,0x00,0x95,0x00,0x00,0x00,0xb2,0x00,0x00,0x00, +0xb1,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x6e,0x00,0x00,0x00, +0xb3,0x00,0x00,0x00,0xb2,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xbb,0x00,0x00,0x00,0x8d,0x00,0x00,0x00, +0xba,0x00,0x00,0x00,0x41,0x00,0x08,0x00,0x8e,0x00,0x00,0x00, +0xbc,0x00,0x00,0x00,0x76,0x00,0x00,0x00,0x15,0x00,0x00,0x00, +0x79,0x00,0x00,0x00,0x8b,0x00,0x00,0x00,0xbb,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x6e,0x00,0x00,0x00,0xbd,0x00,0x00,0x00, +0xbc,0x00,0x00,0x00,0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xbe,0x00,0x00,0x00,0xbd,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, +0x14,0x00,0x00,0x00,0xbf,0x00,0x00,0x00,0xbe,0x00,0x00,0x00, +0xc7,0x00,0x05,0x00,0x14,0x00,0x00,0x00,0xc0,0x00,0x00,0x00, +0xbf,0x00,0x00,0x00,0x93,0x00,0x00,0x00,0x72,0x00,0x04,0x00, +0x95,0x00,0x00,0x00,0xc1,0x00,0x00,0x00,0xc0,0x00,0x00,0x00, +0x7c,0x00,0x04,0x00,0x6e,0x00,0x00,0x00,0xc2,0x00,0x00,0x00, +0xc1,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xca,0x00,0x00,0x00,0x8d,0x00,0x00,0x00,0xc9,0x00,0x00,0x00, +0x41,0x00,0x08,0x00,0x8e,0x00,0x00,0x00,0xcb,0x00,0x00,0x00, +0x76,0x00,0x00,0x00,0x15,0x00,0x00,0x00,0x79,0x00,0x00,0x00, +0x8b,0x00,0x00,0x00,0xca,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x6e,0x00,0x00,0x00,0xcc,0x00,0x00,0x00,0xcb,0x00,0x00,0x00, +0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xcd,0x00,0x00,0x00, +0xcc,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x14,0x00,0x00,0x00, +0xce,0x00,0x00,0x00,0xcd,0x00,0x00,0x00,0xc7,0x00,0x05,0x00, +0x14,0x00,0x00,0x00,0xd0,0x00,0x00,0x00,0xce,0x00,0x00,0x00, +0xcf,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x6e,0x00,0x00,0x00, +0xd7,0x00,0x00,0x00,0x8f,0x00,0x00,0x00,0x71,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xd8,0x00,0x00,0x00,0xd7,0x00,0x00,0x00, +0x7c,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0xd9,0x00,0x00,0x00, +0xd8,0x00,0x00,0x00,0xc7,0x00,0x05,0x00,0x14,0x00,0x00,0x00, +0xdb,0x00,0x00,0x00,0xd9,0x00,0x00,0x00,0xda,0x00,0x00,0x00, +0xc3,0x00,0x05,0x00,0x14,0x00,0x00,0x00,0xdd,0x00,0x00,0x00, +0xdb,0x00,0x00,0x00,0xdc,0x00,0x00,0x00,0xc5,0x00,0x05,0x00, +0x14,0x00,0x00,0x00,0xde,0x00,0x00,0x00,0xd0,0x00,0x00,0x00, +0xdd,0x00,0x00,0x00,0x72,0x00,0x04,0x00,0x95,0x00,0x00,0x00, +0xdf,0x00,0x00,0x00,0xde,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, +0x6e,0x00,0x00,0x00,0xe0,0x00,0x00,0x00,0xdf,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xe8,0x00,0x00,0x00, +0x8d,0x00,0x00,0x00,0xe7,0x00,0x00,0x00,0x41,0x00,0x08,0x00, +0x8e,0x00,0x00,0x00,0xe9,0x00,0x00,0x00,0x76,0x00,0x00,0x00, +0x15,0x00,0x00,0x00,0x79,0x00,0x00,0x00,0x8b,0x00,0x00,0x00, +0xe8,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x6e,0x00,0x00,0x00, +0xea,0x00,0x00,0x00,0xe9,0x00,0x00,0x00,0x71,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xeb,0x00,0x00,0x00,0xea,0x00,0x00,0x00, +0x7c,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0xec,0x00,0x00,0x00, +0xeb,0x00,0x00,0x00,0xc7,0x00,0x05,0x00,0x14,0x00,0x00,0x00, +0xed,0x00,0x00,0x00,0xec,0x00,0x00,0x00,0xcf,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x6e,0x00,0x00,0x00,0xf5,0x00,0x00,0x00, +0x9f,0x00,0x00,0x00,0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xf6,0x00,0x00,0x00,0xf5,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, +0x14,0x00,0x00,0x00,0xf7,0x00,0x00,0x00,0xf6,0x00,0x00,0x00, +0xc7,0x00,0x05,0x00,0x14,0x00,0x00,0x00,0xf8,0x00,0x00,0x00, +0xf7,0x00,0x00,0x00,0xda,0x00,0x00,0x00,0xc3,0x00,0x05,0x00, +0x14,0x00,0x00,0x00,0xf9,0x00,0x00,0x00,0xf8,0x00,0x00,0x00, +0xdc,0x00,0x00,0x00,0xc5,0x00,0x05,0x00,0x14,0x00,0x00,0x00, +0xfa,0x00,0x00,0x00,0xed,0x00,0x00,0x00,0xf9,0x00,0x00,0x00, +0x72,0x00,0x04,0x00,0x95,0x00,0x00,0x00,0xfb,0x00,0x00,0x00, +0xfa,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x6e,0x00,0x00,0x00, +0xfc,0x00,0x00,0x00,0xfb,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x6e,0x00,0x00,0x00,0x05,0x01,0x00,0x00,0xcb,0x00,0x00,0x00, +0xc2,0x00,0x05,0x00,0x6e,0x00,0x00,0x00,0x07,0x01,0x00,0x00, +0x05,0x01,0x00,0x00,0x06,0x01,0x00,0x00,0x71,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x08,0x01,0x00,0x00,0x07,0x01,0x00,0x00, +0x7c,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0x09,0x01,0x00,0x00, +0x08,0x01,0x00,0x00,0xc7,0x00,0x05,0x00,0x14,0x00,0x00,0x00, +0x0a,0x01,0x00,0x00,0x09,0x01,0x00,0x00,0xcf,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x6e,0x00,0x00,0x00,0x12,0x01,0x00,0x00, +0xad,0x00,0x00,0x00,0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x13,0x01,0x00,0x00,0x12,0x01,0x00,0x00,0x7c,0x00,0x04,0x00, +0x14,0x00,0x00,0x00,0x14,0x01,0x00,0x00,0x13,0x01,0x00,0x00, +0xc7,0x00,0x05,0x00,0x14,0x00,0x00,0x00,0x15,0x01,0x00,0x00, +0x14,0x01,0x00,0x00,0xda,0x00,0x00,0x00,0xc3,0x00,0x05,0x00, +0x14,0x00,0x00,0x00,0x16,0x01,0x00,0x00,0x15,0x01,0x00,0x00, +0xdc,0x00,0x00,0x00,0xc5,0x00,0x05,0x00,0x14,0x00,0x00,0x00, +0x17,0x01,0x00,0x00,0x0a,0x01,0x00,0x00,0x16,0x01,0x00,0x00, +0x72,0x00,0x04,0x00,0x95,0x00,0x00,0x00,0x18,0x01,0x00,0x00, +0x17,0x01,0x00,0x00,0x7c,0x00,0x04,0x00,0x6e,0x00,0x00,0x00, +0x19,0x01,0x00,0x00,0x18,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, +0x6e,0x00,0x00,0x00,0x22,0x01,0x00,0x00,0xe9,0x00,0x00,0x00, +0xc2,0x00,0x05,0x00,0x6e,0x00,0x00,0x00,0x23,0x01,0x00,0x00, +0x22,0x01,0x00,0x00,0x06,0x01,0x00,0x00,0x71,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x24,0x01,0x00,0x00,0x23,0x01,0x00,0x00, +0x7c,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0x25,0x01,0x00,0x00, +0x24,0x01,0x00,0x00,0xc7,0x00,0x05,0x00,0x14,0x00,0x00,0x00, +0x26,0x01,0x00,0x00,0x25,0x01,0x00,0x00,0xcf,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x6e,0x00,0x00,0x00,0x2e,0x01,0x00,0x00, +0xbc,0x00,0x00,0x00,0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x2f,0x01,0x00,0x00,0x2e,0x01,0x00,0x00,0x7c,0x00,0x04,0x00, +0x14,0x00,0x00,0x00,0x30,0x01,0x00,0x00,0x2f,0x01,0x00,0x00, +0xc7,0x00,0x05,0x00,0x14,0x00,0x00,0x00,0x31,0x01,0x00,0x00, +0x30,0x01,0x00,0x00,0xda,0x00,0x00,0x00,0xc3,0x00,0x05,0x00, +0x14,0x00,0x00,0x00,0x32,0x01,0x00,0x00,0x31,0x01,0x00,0x00, +0xdc,0x00,0x00,0x00,0xc5,0x00,0x05,0x00,0x14,0x00,0x00,0x00, +0x33,0x01,0x00,0x00,0x26,0x01,0x00,0x00,0x32,0x01,0x00,0x00, +0x72,0x00,0x04,0x00,0x95,0x00,0x00,0x00,0x34,0x01,0x00,0x00, +0x33,0x01,0x00,0x00,0x7c,0x00,0x04,0x00,0x6e,0x00,0x00,0x00, +0x35,0x01,0x00,0x00,0x34,0x01,0x00,0x00,0x41,0x00,0x08,0x00, +0x8e,0x00,0x00,0x00,0x3b,0x01,0x00,0x00,0x76,0x00,0x00,0x00, +0x15,0x00,0x00,0x00,0x79,0x00,0x00,0x00,0xdc,0x00,0x00,0x00, +0x43,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x6e,0x00,0x00,0x00, +0x3c,0x01,0x00,0x00,0x3b,0x01,0x00,0x00,0x71,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x3d,0x01,0x00,0x00,0x3c,0x01,0x00,0x00, +0x7c,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0x3e,0x01,0x00,0x00, +0x3d,0x01,0x00,0x00,0xc7,0x00,0x05,0x00,0x14,0x00,0x00,0x00, +0x3f,0x01,0x00,0x00,0x3e,0x01,0x00,0x00,0xcf,0x00,0x00,0x00, +0x72,0x00,0x04,0x00,0x95,0x00,0x00,0x00,0x40,0x01,0x00,0x00, +0x3f,0x01,0x00,0x00,0x7c,0x00,0x04,0x00,0x6e,0x00,0x00,0x00, +0x41,0x01,0x00,0x00,0x40,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x47,0x01,0x00,0x00,0x43,0x00,0x00,0x00, +0x82,0x00,0x00,0x00,0x41,0x00,0x08,0x00,0x8e,0x00,0x00,0x00, +0x48,0x01,0x00,0x00,0x76,0x00,0x00,0x00,0x15,0x00,0x00,0x00, +0x79,0x00,0x00,0x00,0xdc,0x00,0x00,0x00,0x47,0x01,0x00,0x00, +0x3d,0x00,0x04,0x00,0x6e,0x00,0x00,0x00,0x49,0x01,0x00,0x00, +0x48,0x01,0x00,0x00,0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x4a,0x01,0x00,0x00,0x49,0x01,0x00,0x00,0x7c,0x00,0x04,0x00, +0x14,0x00,0x00,0x00,0x4b,0x01,0x00,0x00,0x4a,0x01,0x00,0x00, +0xc7,0x00,0x05,0x00,0x14,0x00,0x00,0x00,0x4c,0x01,0x00,0x00, +0x4b,0x01,0x00,0x00,0xcf,0x00,0x00,0x00,0x72,0x00,0x04,0x00, +0x95,0x00,0x00,0x00,0x4d,0x01,0x00,0x00,0x4c,0x01,0x00,0x00, +0x7c,0x00,0x04,0x00,0x6e,0x00,0x00,0x00,0x4e,0x01,0x00,0x00, +0x4d,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x54,0x01,0x00,0x00,0x43,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x41,0x00,0x08,0x00,0x8e,0x00,0x00,0x00,0x55,0x01,0x00,0x00, +0x76,0x00,0x00,0x00,0x15,0x00,0x00,0x00,0x79,0x00,0x00,0x00, +0xdc,0x00,0x00,0x00,0x54,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, +0x6e,0x00,0x00,0x00,0x56,0x01,0x00,0x00,0x55,0x01,0x00,0x00, +0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x57,0x01,0x00,0x00, +0x56,0x01,0x00,0x00,0x7c,0x00,0x04,0x00,0x14,0x00,0x00,0x00, +0x58,0x01,0x00,0x00,0x57,0x01,0x00,0x00,0xc7,0x00,0x05,0x00, +0x14,0x00,0x00,0x00,0x59,0x01,0x00,0x00,0x58,0x01,0x00,0x00, +0xcf,0x00,0x00,0x00,0x72,0x00,0x04,0x00,0x95,0x00,0x00,0x00, +0x5a,0x01,0x00,0x00,0x59,0x01,0x00,0x00,0x7c,0x00,0x04,0x00, +0x6e,0x00,0x00,0x00,0x5b,0x01,0x00,0x00,0x5a,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x62,0x01,0x00,0x00, +0x43,0x00,0x00,0x00,0x61,0x01,0x00,0x00,0x41,0x00,0x08,0x00, +0x8e,0x00,0x00,0x00,0x63,0x01,0x00,0x00,0x76,0x00,0x00,0x00, +0x15,0x00,0x00,0x00,0x79,0x00,0x00,0x00,0xdc,0x00,0x00,0x00, +0x62,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0x6e,0x00,0x00,0x00, +0x64,0x01,0x00,0x00,0x63,0x01,0x00,0x00,0x71,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x65,0x01,0x00,0x00,0x64,0x01,0x00,0x00, +0x7c,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0x66,0x01,0x00,0x00, +0x65,0x01,0x00,0x00,0xc7,0x00,0x05,0x00,0x14,0x00,0x00,0x00, +0x67,0x01,0x00,0x00,0x66,0x01,0x00,0x00,0xcf,0x00,0x00,0x00, +0x72,0x00,0x04,0x00,0x95,0x00,0x00,0x00,0x68,0x01,0x00,0x00, +0x67,0x01,0x00,0x00,0x7c,0x00,0x04,0x00,0x6e,0x00,0x00,0x00, +0x69,0x01,0x00,0x00,0x68,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, +0x6e,0x00,0x00,0x00,0x70,0x01,0x00,0x00,0x3b,0x01,0x00,0x00, +0xc2,0x00,0x05,0x00,0x6e,0x00,0x00,0x00,0x71,0x01,0x00,0x00, +0x70,0x01,0x00,0x00,0x06,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, +0x6e,0x00,0x00,0x00,0x79,0x01,0x00,0x00,0x48,0x01,0x00,0x00, +0xc2,0x00,0x05,0x00,0x6e,0x00,0x00,0x00,0x7a,0x01,0x00,0x00, +0x79,0x01,0x00,0x00,0x06,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, +0x6e,0x00,0x00,0x00,0x82,0x01,0x00,0x00,0x55,0x01,0x00,0x00, +0xc2,0x00,0x05,0x00,0x6e,0x00,0x00,0x00,0x83,0x01,0x00,0x00, +0x82,0x01,0x00,0x00,0x06,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, +0x6e,0x00,0x00,0x00,0x8b,0x01,0x00,0x00,0x63,0x01,0x00,0x00, +0xc2,0x00,0x05,0x00,0x6e,0x00,0x00,0x00,0x8c,0x01,0x00,0x00, +0x8b,0x01,0x00,0x00,0x06,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x92,0x01,0x00,0x00,0x43,0x00,0x00,0x00, +0x45,0x00,0x00,0x00,0x41,0x00,0x08,0x00,0x8e,0x00,0x00,0x00, +0x93,0x01,0x00,0x00,0x76,0x00,0x00,0x00,0x15,0x00,0x00,0x00, +0x79,0x00,0x00,0x00,0xdc,0x00,0x00,0x00,0x92,0x01,0x00,0x00, +0x3d,0x00,0x04,0x00,0x6e,0x00,0x00,0x00,0x94,0x01,0x00,0x00, +0x93,0x01,0x00,0x00,0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x95,0x01,0x00,0x00,0x94,0x01,0x00,0x00,0x7c,0x00,0x04,0x00, +0x14,0x00,0x00,0x00,0x96,0x01,0x00,0x00,0x95,0x01,0x00,0x00, +0xc7,0x00,0x05,0x00,0x14,0x00,0x00,0x00,0x97,0x01,0x00,0x00, +0x96,0x01,0x00,0x00,0xcf,0x00,0x00,0x00,0x72,0x00,0x04,0x00, +0x95,0x00,0x00,0x00,0x98,0x01,0x00,0x00,0x97,0x01,0x00,0x00, +0x7c,0x00,0x04,0x00,0x6e,0x00,0x00,0x00,0x99,0x01,0x00,0x00, +0x98,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xa0,0x01,0x00,0x00,0x43,0x00,0x00,0x00,0x9f,0x01,0x00,0x00, +0x41,0x00,0x08,0x00,0x8e,0x00,0x00,0x00,0xa1,0x01,0x00,0x00, +0x76,0x00,0x00,0x00,0x15,0x00,0x00,0x00,0x79,0x00,0x00,0x00, +0xdc,0x00,0x00,0x00,0xa0,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, +0x6e,0x00,0x00,0x00,0xa2,0x01,0x00,0x00,0xa1,0x01,0x00,0x00, +0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xa3,0x01,0x00,0x00, +0xa2,0x01,0x00,0x00,0x7c,0x00,0x04,0x00,0x14,0x00,0x00,0x00, +0xa4,0x01,0x00,0x00,0xa3,0x01,0x00,0x00,0xc7,0x00,0x05,0x00, +0x14,0x00,0x00,0x00,0xa5,0x01,0x00,0x00,0xa4,0x01,0x00,0x00, +0xcf,0x00,0x00,0x00,0x72,0x00,0x04,0x00,0x95,0x00,0x00,0x00, +0xa6,0x01,0x00,0x00,0xa5,0x01,0x00,0x00,0x7c,0x00,0x04,0x00, +0x6e,0x00,0x00,0x00,0xa7,0x01,0x00,0x00,0xa6,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xae,0x01,0x00,0x00, +0x43,0x00,0x00,0x00,0xad,0x01,0x00,0x00,0x41,0x00,0x08,0x00, +0x8e,0x00,0x00,0x00,0xaf,0x01,0x00,0x00,0x76,0x00,0x00,0x00, +0x15,0x00,0x00,0x00,0x79,0x00,0x00,0x00,0xdc,0x00,0x00,0x00, +0xae,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0x6e,0x00,0x00,0x00, +0xb0,0x01,0x00,0x00,0xaf,0x01,0x00,0x00,0x71,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xb1,0x01,0x00,0x00,0xb0,0x01,0x00,0x00, +0x7c,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0xb2,0x01,0x00,0x00, +0xb1,0x01,0x00,0x00,0xc7,0x00,0x05,0x00,0x14,0x00,0x00,0x00, +0xb3,0x01,0x00,0x00,0xb2,0x01,0x00,0x00,0xcf,0x00,0x00,0x00, +0x72,0x00,0x04,0x00,0x95,0x00,0x00,0x00,0xb4,0x01,0x00,0x00, +0xb3,0x01,0x00,0x00,0x7c,0x00,0x04,0x00,0x6e,0x00,0x00,0x00, +0xb5,0x01,0x00,0x00,0xb4,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xbc,0x01,0x00,0x00,0x43,0x00,0x00,0x00, +0xbb,0x01,0x00,0x00,0x41,0x00,0x08,0x00,0x8e,0x00,0x00,0x00, +0xbd,0x01,0x00,0x00,0x76,0x00,0x00,0x00,0x15,0x00,0x00,0x00, +0x79,0x00,0x00,0x00,0xdc,0x00,0x00,0x00,0xbc,0x01,0x00,0x00, +0x3d,0x00,0x04,0x00,0x6e,0x00,0x00,0x00,0xbe,0x01,0x00,0x00, +0xbd,0x01,0x00,0x00,0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xbf,0x01,0x00,0x00,0xbe,0x01,0x00,0x00,0x7c,0x00,0x04,0x00, +0x14,0x00,0x00,0x00,0xc0,0x01,0x00,0x00,0xbf,0x01,0x00,0x00, +0xc7,0x00,0x05,0x00,0x14,0x00,0x00,0x00,0xc1,0x01,0x00,0x00, +0xc0,0x01,0x00,0x00,0xcf,0x00,0x00,0x00,0x72,0x00,0x04,0x00, +0x95,0x00,0x00,0x00,0xc2,0x01,0x00,0x00,0xc1,0x01,0x00,0x00, +0x7c,0x00,0x04,0x00,0x6e,0x00,0x00,0x00,0xc3,0x01,0x00,0x00, +0xc2,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0x6e,0x00,0x00,0x00, +0xcb,0x01,0x00,0x00,0x93,0x01,0x00,0x00,0xc2,0x00,0x05,0x00, +0x6e,0x00,0x00,0x00,0xcc,0x01,0x00,0x00,0xcb,0x01,0x00,0x00, +0x06,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0x6e,0x00,0x00,0x00, +0xd4,0x01,0x00,0x00,0xa1,0x01,0x00,0x00,0xc2,0x00,0x05,0x00, +0x6e,0x00,0x00,0x00,0xd5,0x01,0x00,0x00,0xd4,0x01,0x00,0x00, +0x06,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0x6e,0x00,0x00,0x00, +0xdd,0x01,0x00,0x00,0xaf,0x01,0x00,0x00,0xc2,0x00,0x05,0x00, +0x6e,0x00,0x00,0x00,0xde,0x01,0x00,0x00,0xdd,0x01,0x00,0x00, +0x06,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0x6e,0x00,0x00,0x00, +0xe6,0x01,0x00,0x00,0xbd,0x01,0x00,0x00,0xc2,0x00,0x05,0x00, +0x6e,0x00,0x00,0x00,0xe7,0x01,0x00,0x00,0xe6,0x01,0x00,0x00, +0x06,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0x16,0x00,0x00,0x00, +0xed,0x01,0x00,0x00,0x13,0x00,0x00,0x00,0x8b,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xee,0x01,0x00,0x00, +0xed,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xf0,0x01,0x00,0x00,0xee,0x01,0x00,0x00,0x65,0x00,0x00,0x00, +0x41,0x00,0x06,0x00,0xf1,0x01,0x00,0x00,0xf2,0x01,0x00,0x00, +0xec,0x01,0x00,0x00,0x15,0x00,0x00,0x00,0xf0,0x01,0x00,0x00, +0x3d,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0xf3,0x01,0x00,0x00, +0xf2,0x01,0x00,0x00,0x70,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, +0xf5,0x01,0x00,0x00,0x41,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xfb,0x01,0x00,0x00,0xf0,0x01,0x00,0x00, +0x82,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0xf1,0x01,0x00,0x00, +0xfc,0x01,0x00,0x00,0xec,0x01,0x00,0x00,0x15,0x00,0x00,0x00, +0xfb,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, +0xfd,0x01,0x00,0x00,0xfc,0x01,0x00,0x00,0x70,0x00,0x04,0x00, +0x4a,0x00,0x00,0x00,0xff,0x01,0x00,0x00,0x4e,0x01,0x00,0x00, +0x85,0x00,0x05,0x00,0x4a,0x00,0x00,0x00,0x00,0x02,0x00,0x00, +0xfd,0x01,0x00,0x00,0xff,0x01,0x00,0x00,0x0c,0x00,0x08,0x00, +0x4a,0x00,0x00,0x00,0x01,0x02,0x00,0x00,0x01,0x00,0x00,0x00, +0x32,0x00,0x00,0x00,0xf3,0x01,0x00,0x00,0xf5,0x01,0x00,0x00, +0x00,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x06,0x02,0x00,0x00,0xf0,0x01,0x00,0x00,0x23,0x00,0x00,0x00, +0x41,0x00,0x06,0x00,0xf1,0x01,0x00,0x00,0x07,0x02,0x00,0x00, +0xec,0x01,0x00,0x00,0x15,0x00,0x00,0x00,0x06,0x02,0x00,0x00, +0x3d,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0x08,0x02,0x00,0x00, +0x07,0x02,0x00,0x00,0x70,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, +0x0a,0x02,0x00,0x00,0x5b,0x01,0x00,0x00,0x0c,0x00,0x08,0x00, +0x4a,0x00,0x00,0x00,0x0c,0x02,0x00,0x00,0x01,0x00,0x00,0x00, +0x32,0x00,0x00,0x00,0x08,0x02,0x00,0x00,0x0a,0x02,0x00,0x00, +0x01,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x11,0x02,0x00,0x00,0xf0,0x01,0x00,0x00,0x61,0x01,0x00,0x00, +0x41,0x00,0x06,0x00,0xf1,0x01,0x00,0x00,0x12,0x02,0x00,0x00, +0xec,0x01,0x00,0x00,0x15,0x00,0x00,0x00,0x11,0x02,0x00,0x00, +0x3d,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0x13,0x02,0x00,0x00, +0x12,0x02,0x00,0x00,0x70,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, +0x15,0x02,0x00,0x00,0x69,0x01,0x00,0x00,0x0c,0x00,0x08,0x00, +0x4a,0x00,0x00,0x00,0x17,0x02,0x00,0x00,0x01,0x00,0x00,0x00, +0x32,0x00,0x00,0x00,0x13,0x02,0x00,0x00,0x15,0x02,0x00,0x00, +0x0c,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x1d,0x02,0x00,0x00,0xf0,0x01,0x00,0x00,0x3f,0x00,0x00,0x00, +0x41,0x00,0x06,0x00,0xf1,0x01,0x00,0x00,0x1e,0x02,0x00,0x00, +0xec,0x01,0x00,0x00,0x15,0x00,0x00,0x00,0x1d,0x02,0x00,0x00, +0x3d,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0x1f,0x02,0x00,0x00, +0x1e,0x02,0x00,0x00,0x70,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, +0x21,0x02,0x00,0x00,0x71,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x28,0x02,0x00,0x00,0xf0,0x01,0x00,0x00, +0x27,0x02,0x00,0x00,0x41,0x00,0x06,0x00,0xf1,0x01,0x00,0x00, +0x29,0x02,0x00,0x00,0xec,0x01,0x00,0x00,0x15,0x00,0x00,0x00, +0x28,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, +0x2a,0x02,0x00,0x00,0x29,0x02,0x00,0x00,0x70,0x00,0x04,0x00, +0x4a,0x00,0x00,0x00,0x2c,0x02,0x00,0x00,0x7a,0x01,0x00,0x00, +0x85,0x00,0x05,0x00,0x4a,0x00,0x00,0x00,0x2d,0x02,0x00,0x00, +0x2a,0x02,0x00,0x00,0x2c,0x02,0x00,0x00,0x0c,0x00,0x08,0x00, +0x4a,0x00,0x00,0x00,0x2e,0x02,0x00,0x00,0x01,0x00,0x00,0x00, +0x32,0x00,0x00,0x00,0x1f,0x02,0x00,0x00,0x21,0x02,0x00,0x00, +0x2d,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x34,0x02,0x00,0x00,0xf0,0x01,0x00,0x00,0x33,0x02,0x00,0x00, +0x41,0x00,0x06,0x00,0xf1,0x01,0x00,0x00,0x35,0x02,0x00,0x00, +0xec,0x01,0x00,0x00,0x15,0x00,0x00,0x00,0x34,0x02,0x00,0x00, +0x3d,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0x36,0x02,0x00,0x00, +0x35,0x02,0x00,0x00,0x70,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, +0x38,0x02,0x00,0x00,0x83,0x01,0x00,0x00,0x0c,0x00,0x08,0x00, +0x4a,0x00,0x00,0x00,0x3a,0x02,0x00,0x00,0x01,0x00,0x00,0x00, +0x32,0x00,0x00,0x00,0x36,0x02,0x00,0x00,0x38,0x02,0x00,0x00, +0x2e,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x40,0x02,0x00,0x00,0xf0,0x01,0x00,0x00,0x3f,0x02,0x00,0x00, +0x41,0x00,0x06,0x00,0xf1,0x01,0x00,0x00,0x41,0x02,0x00,0x00, +0xec,0x01,0x00,0x00,0x15,0x00,0x00,0x00,0x40,0x02,0x00,0x00, +0x3d,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0x42,0x02,0x00,0x00, +0x41,0x02,0x00,0x00,0x70,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, +0x44,0x02,0x00,0x00,0x8c,0x01,0x00,0x00,0x0c,0x00,0x08,0x00, +0x4a,0x00,0x00,0x00,0x46,0x02,0x00,0x00,0x01,0x00,0x00,0x00, +0x32,0x00,0x00,0x00,0x42,0x02,0x00,0x00,0x44,0x02,0x00,0x00, +0x3a,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x4b,0x02,0x00,0x00,0xee,0x01,0x00,0x00,0x69,0x00,0x00,0x00, +0x41,0x00,0x06,0x00,0xf1,0x01,0x00,0x00,0x4c,0x02,0x00,0x00, +0xec,0x01,0x00,0x00,0x15,0x00,0x00,0x00,0x4b,0x02,0x00,0x00, +0x3d,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0x4d,0x02,0x00,0x00, +0x4c,0x02,0x00,0x00,0x70,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, +0x4f,0x02,0x00,0x00,0x99,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x55,0x02,0x00,0x00,0x4b,0x02,0x00,0x00, +0x82,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0xf1,0x01,0x00,0x00, +0x56,0x02,0x00,0x00,0xec,0x01,0x00,0x00,0x15,0x00,0x00,0x00, +0x55,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, +0x57,0x02,0x00,0x00,0x56,0x02,0x00,0x00,0x70,0x00,0x04,0x00, +0x4a,0x00,0x00,0x00,0x59,0x02,0x00,0x00,0xa7,0x01,0x00,0x00, +0x85,0x00,0x05,0x00,0x4a,0x00,0x00,0x00,0x5a,0x02,0x00,0x00, +0x57,0x02,0x00,0x00,0x59,0x02,0x00,0x00,0x0c,0x00,0x08,0x00, +0x4a,0x00,0x00,0x00,0x5b,0x02,0x00,0x00,0x01,0x00,0x00,0x00, +0x32,0x00,0x00,0x00,0x4d,0x02,0x00,0x00,0x4f,0x02,0x00,0x00, +0x5a,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x60,0x02,0x00,0x00,0x4b,0x02,0x00,0x00,0x23,0x00,0x00,0x00, +0x41,0x00,0x06,0x00,0xf1,0x01,0x00,0x00,0x61,0x02,0x00,0x00, +0xec,0x01,0x00,0x00,0x15,0x00,0x00,0x00,0x60,0x02,0x00,0x00, +0x3d,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0x62,0x02,0x00,0x00, +0x61,0x02,0x00,0x00,0x70,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, +0x64,0x02,0x00,0x00,0xb5,0x01,0x00,0x00,0x0c,0x00,0x08,0x00, +0x4a,0x00,0x00,0x00,0x66,0x02,0x00,0x00,0x01,0x00,0x00,0x00, +0x32,0x00,0x00,0x00,0x62,0x02,0x00,0x00,0x64,0x02,0x00,0x00, +0x5b,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x6b,0x02,0x00,0x00,0x4b,0x02,0x00,0x00,0x61,0x01,0x00,0x00, +0x41,0x00,0x06,0x00,0xf1,0x01,0x00,0x00,0x6c,0x02,0x00,0x00, +0xec,0x01,0x00,0x00,0x15,0x00,0x00,0x00,0x6b,0x02,0x00,0x00, +0x3d,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0x6d,0x02,0x00,0x00, +0x6c,0x02,0x00,0x00,0x70,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, +0x6f,0x02,0x00,0x00,0xc3,0x01,0x00,0x00,0x0c,0x00,0x08,0x00, +0x4a,0x00,0x00,0x00,0x71,0x02,0x00,0x00,0x01,0x00,0x00,0x00, +0x32,0x00,0x00,0x00,0x6d,0x02,0x00,0x00,0x6f,0x02,0x00,0x00, +0x66,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x77,0x02,0x00,0x00,0x4b,0x02,0x00,0x00,0x3f,0x00,0x00,0x00, +0x41,0x00,0x06,0x00,0xf1,0x01,0x00,0x00,0x78,0x02,0x00,0x00, +0xec,0x01,0x00,0x00,0x15,0x00,0x00,0x00,0x77,0x02,0x00,0x00, +0x3d,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0x79,0x02,0x00,0x00, +0x78,0x02,0x00,0x00,0x70,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, +0x7b,0x02,0x00,0x00,0xcc,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x81,0x02,0x00,0x00,0x4b,0x02,0x00,0x00, +0x27,0x02,0x00,0x00,0x41,0x00,0x06,0x00,0xf1,0x01,0x00,0x00, +0x82,0x02,0x00,0x00,0xec,0x01,0x00,0x00,0x15,0x00,0x00,0x00, +0x81,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, +0x83,0x02,0x00,0x00,0x82,0x02,0x00,0x00,0x70,0x00,0x04,0x00, +0x4a,0x00,0x00,0x00,0x85,0x02,0x00,0x00,0xd5,0x01,0x00,0x00, +0x85,0x00,0x05,0x00,0x4a,0x00,0x00,0x00,0x86,0x02,0x00,0x00, +0x83,0x02,0x00,0x00,0x85,0x02,0x00,0x00,0x0c,0x00,0x08,0x00, +0x4a,0x00,0x00,0x00,0x87,0x02,0x00,0x00,0x01,0x00,0x00,0x00, +0x32,0x00,0x00,0x00,0x79,0x02,0x00,0x00,0x7b,0x02,0x00,0x00, +0x86,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x8c,0x02,0x00,0x00,0x4b,0x02,0x00,0x00,0x33,0x02,0x00,0x00, +0x41,0x00,0x06,0x00,0xf1,0x01,0x00,0x00,0x8d,0x02,0x00,0x00, +0xec,0x01,0x00,0x00,0x15,0x00,0x00,0x00,0x8c,0x02,0x00,0x00, +0x3d,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0x8e,0x02,0x00,0x00, +0x8d,0x02,0x00,0x00,0x70,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, +0x90,0x02,0x00,0x00,0xde,0x01,0x00,0x00,0x0c,0x00,0x08,0x00, +0x4a,0x00,0x00,0x00,0x92,0x02,0x00,0x00,0x01,0x00,0x00,0x00, +0x32,0x00,0x00,0x00,0x8e,0x02,0x00,0x00,0x90,0x02,0x00,0x00, +0x87,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x97,0x02,0x00,0x00,0x4b,0x02,0x00,0x00,0x3f,0x02,0x00,0x00, +0x41,0x00,0x06,0x00,0xf1,0x01,0x00,0x00,0x98,0x02,0x00,0x00, +0xec,0x01,0x00,0x00,0x15,0x00,0x00,0x00,0x97,0x02,0x00,0x00, +0x3d,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0x99,0x02,0x00,0x00, +0x98,0x02,0x00,0x00,0x70,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, +0x9b,0x02,0x00,0x00,0xe7,0x01,0x00,0x00,0x0c,0x00,0x08,0x00, +0x4a,0x00,0x00,0x00,0x9d,0x02,0x00,0x00,0x01,0x00,0x00,0x00, +0x32,0x00,0x00,0x00,0x99,0x02,0x00,0x00,0x9b,0x02,0x00,0x00, +0x92,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, +0xa4,0x02,0x00,0x00,0xf2,0x01,0x00,0x00,0x70,0x00,0x04,0x00, +0x4a,0x00,0x00,0x00,0xa6,0x02,0x00,0x00,0xb3,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0xae,0x02,0x00,0x00, +0x1e,0x02,0x00,0x00,0x70,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, +0xb0,0x02,0x00,0x00,0xc2,0x00,0x00,0x00,0x85,0x00,0x05,0x00, +0x4a,0x00,0x00,0x00,0xb1,0x02,0x00,0x00,0xae,0x02,0x00,0x00, +0xb0,0x02,0x00,0x00,0x0c,0x00,0x08,0x00,0x4a,0x00,0x00,0x00, +0xb2,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00, +0xa4,0x02,0x00,0x00,0xa6,0x02,0x00,0x00,0xb1,0x02,0x00,0x00, +0x3d,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0xb8,0x02,0x00,0x00, +0x4c,0x02,0x00,0x00,0x70,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, +0xba,0x02,0x00,0x00,0x19,0x01,0x00,0x00,0x0c,0x00,0x08,0x00, +0x4a,0x00,0x00,0x00,0xbc,0x02,0x00,0x00,0x01,0x00,0x00,0x00, +0x32,0x00,0x00,0x00,0xb8,0x02,0x00,0x00,0xba,0x02,0x00,0x00, +0xb2,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, +0xc3,0x02,0x00,0x00,0x78,0x02,0x00,0x00,0x70,0x00,0x04,0x00, +0x4a,0x00,0x00,0x00,0xc5,0x02,0x00,0x00,0x35,0x01,0x00,0x00, +0x0c,0x00,0x08,0x00,0x4a,0x00,0x00,0x00,0xc7,0x02,0x00,0x00, +0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0xc3,0x02,0x00,0x00, +0xc5,0x02,0x00,0x00,0xbc,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, +0x4a,0x00,0x00,0x00,0xce,0x02,0x00,0x00,0xfc,0x01,0x00,0x00, +0x0c,0x00,0x08,0x00,0x4a,0x00,0x00,0x00,0xd2,0x02,0x00,0x00, +0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0xce,0x02,0x00,0x00, +0xa6,0x02,0x00,0x00,0xc7,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, +0x4a,0x00,0x00,0x00,0xd9,0x02,0x00,0x00,0x29,0x02,0x00,0x00, +0x0c,0x00,0x08,0x00,0x4a,0x00,0x00,0x00,0xdd,0x02,0x00,0x00, +0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0xd9,0x02,0x00,0x00, +0xb0,0x02,0x00,0x00,0xd2,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, +0x4a,0x00,0x00,0x00,0xe4,0x02,0x00,0x00,0x56,0x02,0x00,0x00, +0x0c,0x00,0x08,0x00,0x4a,0x00,0x00,0x00,0xe8,0x02,0x00,0x00, +0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0xe4,0x02,0x00,0x00, +0xba,0x02,0x00,0x00,0xdd,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, +0x4a,0x00,0x00,0x00,0xef,0x02,0x00,0x00,0x82,0x02,0x00,0x00, +0x0c,0x00,0x08,0x00,0x4a,0x00,0x00,0x00,0xf3,0x02,0x00,0x00, +0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0xef,0x02,0x00,0x00, +0xc5,0x02,0x00,0x00,0xe8,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, +0x4a,0x00,0x00,0x00,0xfa,0x02,0x00,0x00,0x07,0x02,0x00,0x00, +0x0c,0x00,0x08,0x00,0x4a,0x00,0x00,0x00,0xfe,0x02,0x00,0x00, +0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0xfa,0x02,0x00,0x00, +0xa6,0x02,0x00,0x00,0xf3,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, +0x4a,0x00,0x00,0x00,0x05,0x03,0x00,0x00,0x35,0x02,0x00,0x00, +0x0c,0x00,0x08,0x00,0x4a,0x00,0x00,0x00,0x09,0x03,0x00,0x00, +0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x05,0x03,0x00,0x00, +0xb0,0x02,0x00,0x00,0xfe,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, +0x4a,0x00,0x00,0x00,0x10,0x03,0x00,0x00,0x61,0x02,0x00,0x00, +0x0c,0x00,0x08,0x00,0x4a,0x00,0x00,0x00,0x14,0x03,0x00,0x00, +0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x10,0x03,0x00,0x00, +0xba,0x02,0x00,0x00,0x09,0x03,0x00,0x00,0x3d,0x00,0x04,0x00, +0x4a,0x00,0x00,0x00,0x1b,0x03,0x00,0x00,0x8d,0x02,0x00,0x00, +0x0c,0x00,0x08,0x00,0x4a,0x00,0x00,0x00,0x1f,0x03,0x00,0x00, +0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x1b,0x03,0x00,0x00, +0xc5,0x02,0x00,0x00,0x14,0x03,0x00,0x00,0x3d,0x00,0x04,0x00, +0x4a,0x00,0x00,0x00,0x26,0x03,0x00,0x00,0x12,0x02,0x00,0x00, +0x0c,0x00,0x08,0x00,0x4a,0x00,0x00,0x00,0x2a,0x03,0x00,0x00, +0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x26,0x03,0x00,0x00, +0xa6,0x02,0x00,0x00,0x1f,0x03,0x00,0x00,0x3d,0x00,0x04,0x00, +0x4a,0x00,0x00,0x00,0x31,0x03,0x00,0x00,0x41,0x02,0x00,0x00, +0x0c,0x00,0x08,0x00,0x4a,0x00,0x00,0x00,0x35,0x03,0x00,0x00, +0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x31,0x03,0x00,0x00, +0xb0,0x02,0x00,0x00,0x2a,0x03,0x00,0x00,0x3d,0x00,0x04,0x00, +0x4a,0x00,0x00,0x00,0x3c,0x03,0x00,0x00,0x6c,0x02,0x00,0x00, +0x0c,0x00,0x08,0x00,0x4a,0x00,0x00,0x00,0x40,0x03,0x00,0x00, +0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x3c,0x03,0x00,0x00, +0xba,0x02,0x00,0x00,0x35,0x03,0x00,0x00,0x3d,0x00,0x04,0x00, +0x4a,0x00,0x00,0x00,0x47,0x03,0x00,0x00,0x98,0x02,0x00,0x00, +0x0c,0x00,0x08,0x00,0x4a,0x00,0x00,0x00,0x4b,0x03,0x00,0x00, +0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x47,0x03,0x00,0x00, +0xc5,0x02,0x00,0x00,0x40,0x03,0x00,0x00,0x70,0x00,0x04,0x00, +0x4a,0x00,0x00,0x00,0x53,0x03,0x00,0x00,0x97,0x00,0x00,0x00, +0x70,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0x57,0x03,0x00,0x00, +0xa5,0x00,0x00,0x00,0x85,0x00,0x05,0x00,0x4a,0x00,0x00,0x00, +0x58,0x03,0x00,0x00,0x46,0x02,0x00,0x00,0x57,0x03,0x00,0x00, +0x0c,0x00,0x08,0x00,0x4a,0x00,0x00,0x00,0x59,0x03,0x00,0x00, +0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x17,0x02,0x00,0x00, +0x53,0x03,0x00,0x00,0x58,0x03,0x00,0x00,0x70,0x00,0x04,0x00, +0x4a,0x00,0x00,0x00,0x5c,0x03,0x00,0x00,0xe0,0x00,0x00,0x00, +0x0c,0x00,0x08,0x00,0x4a,0x00,0x00,0x00,0x5e,0x03,0x00,0x00, +0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x71,0x02,0x00,0x00, +0x5c,0x03,0x00,0x00,0x59,0x03,0x00,0x00,0x70,0x00,0x04,0x00, +0x4a,0x00,0x00,0x00,0x61,0x03,0x00,0x00,0xfc,0x00,0x00,0x00, +0x0c,0x00,0x08,0x00,0x4a,0x00,0x00,0x00,0x63,0x03,0x00,0x00, +0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x9d,0x02,0x00,0x00, +0x61,0x03,0x00,0x00,0x5e,0x03,0x00,0x00,0x85,0x00,0x05,0x00, +0x4a,0x00,0x00,0x00,0x67,0x03,0x00,0x00,0x85,0x00,0x00,0x00, +0x4b,0x03,0x00,0x00,0x7f,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, +0x9d,0x03,0x00,0x00,0x67,0x03,0x00,0x00,0x0c,0x00,0x08,0x00, +0x4a,0x00,0x00,0x00,0x68,0x03,0x00,0x00,0x01,0x00,0x00,0x00, +0x32,0x00,0x00,0x00,0x7d,0x00,0x00,0x00,0x63,0x03,0x00,0x00, +0x9d,0x03,0x00,0x00,0x3d,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, +0x6a,0x03,0x00,0x00,0x55,0x00,0x00,0x00,0x81,0x00,0x05,0x00, +0x4a,0x00,0x00,0x00,0x6b,0x03,0x00,0x00,0x6a,0x03,0x00,0x00, +0x68,0x03,0x00,0x00,0x3e,0x00,0x03,0x00,0x55,0x00,0x00,0x00, +0x6b,0x03,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x6e,0x03,0x00,0x00,0x99,0x03,0x00,0x00,0x23,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x58,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0x5a,0x00,0x00,0x00,0xe0,0x00,0x04,0x00,0x23,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x6f,0x03,0x00,0x00,0xf9,0x00,0x02,0x00, +0x71,0x03,0x00,0x00,0xf8,0x00,0x02,0x00,0x71,0x03,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x9a,0x03,0x00,0x00, +0x4e,0x00,0x00,0x00,0x5a,0x00,0x00,0x00,0x88,0x03,0x00,0x00, +0x74,0x03,0x00,0x00,0xac,0x00,0x05,0x00,0x5f,0x00,0x00,0x00, +0x77,0x03,0x00,0x00,0x9a,0x03,0x00,0x00,0x0c,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0x73,0x03,0x00,0x00,0x74,0x03,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x77,0x03,0x00,0x00, +0x72,0x03,0x00,0x00,0x73,0x03,0x00,0x00,0xf8,0x00,0x02,0x00, +0x72,0x03,0x00,0x00,0xb0,0x00,0x05,0x00,0x5f,0x00,0x00,0x00, +0x7a,0x03,0x00,0x00,0x24,0x00,0x00,0x00,0x9a,0x03,0x00,0x00, +0xf7,0x00,0x03,0x00,0x7c,0x03,0x00,0x00,0x00,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x7a,0x03,0x00,0x00,0x7b,0x03,0x00,0x00, +0x7c,0x03,0x00,0x00,0xf8,0x00,0x02,0x00,0x7b,0x03,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x80,0x03,0x00,0x00, +0x24,0x00,0x00,0x00,0x9a,0x03,0x00,0x00,0x41,0x00,0x05,0x00, +0x54,0x00,0x00,0x00,0x81,0x03,0x00,0x00,0x4d,0x00,0x00,0x00, +0x80,0x03,0x00,0x00,0x3d,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, +0x82,0x03,0x00,0x00,0x81,0x03,0x00,0x00,0x41,0x00,0x05,0x00, +0x54,0x00,0x00,0x00,0x83,0x03,0x00,0x00,0x4d,0x00,0x00,0x00, +0x24,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, +0x84,0x03,0x00,0x00,0x83,0x03,0x00,0x00,0x81,0x00,0x05,0x00, +0x4a,0x00,0x00,0x00,0x85,0x03,0x00,0x00,0x84,0x03,0x00,0x00, +0x82,0x03,0x00,0x00,0x3e,0x00,0x03,0x00,0x83,0x03,0x00,0x00, +0x85,0x03,0x00,0x00,0xf9,0x00,0x02,0x00,0x7c,0x03,0x00,0x00, +0xf8,0x00,0x02,0x00,0x7c,0x03,0x00,0x00,0xe0,0x00,0x04,0x00, +0x23,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x6f,0x03,0x00,0x00, +0xf9,0x00,0x02,0x00,0x74,0x03,0x00,0x00,0xf8,0x00,0x02,0x00, +0x74,0x03,0x00,0x00,0xc2,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x88,0x03,0x00,0x00,0x9a,0x03,0x00,0x00,0x8b,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x71,0x03,0x00,0x00,0xf8,0x00,0x02,0x00, +0x73,0x03,0x00,0x00,0xaa,0x00,0x05,0x00,0x5f,0x00,0x00,0x00, +0x8a,0x03,0x00,0x00,0x24,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0xf7,0x00,0x03,0x00,0x8c,0x03,0x00,0x00,0x00,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x8a,0x03,0x00,0x00,0x8b,0x03,0x00,0x00, +0x8c,0x03,0x00,0x00,0xf8,0x00,0x02,0x00,0x8b,0x03,0x00,0x00, +0x41,0x00,0x05,0x00,0x16,0x00,0x00,0x00,0x91,0x03,0x00,0x00, +0x13,0x00,0x00,0x00,0xdc,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x92,0x03,0x00,0x00,0x91,0x03,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x94,0x03,0x00,0x00, +0x92,0x03,0x00,0x00,0x0f,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x54,0x00,0x00,0x00,0x95,0x03,0x00,0x00,0x4d,0x00,0x00,0x00, +0x15,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, +0x96,0x03,0x00,0x00,0x95,0x03,0x00,0x00,0x41,0x00,0x06,0x00, +0xf1,0x01,0x00,0x00,0x97,0x03,0x00,0x00,0x90,0x03,0x00,0x00, +0x15,0x00,0x00,0x00,0x94,0x03,0x00,0x00,0x3e,0x00,0x03,0x00, +0x97,0x03,0x00,0x00,0x96,0x03,0x00,0x00,0xf9,0x00,0x02,0x00, +0x8c,0x03,0x00,0x00,0xf8,0x00,0x02,0x00,0x8c,0x03,0x00,0x00, 0xfd,0x00,0x01,0x00,0x38,0x00,0x01,0x00, }; -const uint64_t mul_mat_vec_q4_K_f32_len = 9176; +const uint64_t mul_mat_vec_q4_K_f32_len = 9128; unsigned char mul_mat_vec_q5_0_f32_data[] = { 0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, -0xfa,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, +0xf6,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, 0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x51,0x11,0x00,0x00, 0x11,0x00,0x02,0x00,0x60,0x11,0x00,0x00,0x0b,0x00,0x06,0x00, 0x01,0x00,0x00,0x00,0x47,0x4c,0x53,0x4c,0x2e,0x73,0x74,0x64, 0x2e,0x34,0x35,0x30,0x00,0x00,0x00,0x00,0x0e,0x00,0x03,0x00, 0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x0f,0x00,0x0c,0x00, 0x05,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x6d,0x61,0x69,0x6e, -0x00,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x13,0x00,0x00,0x00, -0x1b,0x00,0x00,0x00,0x2a,0x00,0x00,0x00,0x57,0x00,0x00,0x00, -0xa5,0x00,0x00,0x00,0xe9,0x00,0x00,0x00,0x10,0x00,0x06,0x00, -0x04,0x00,0x00,0x00,0x11,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x11,0x00,0x00,0x00, +0x18,0x00,0x00,0x00,0x26,0x00,0x00,0x00,0x54,0x00,0x00,0x00, +0xa3,0x00,0x00,0x00,0xe4,0x00,0x00,0x00,0x10,0x00,0x06,0x00, +0x04,0x00,0x00,0x00,0x11,0x00,0x00,0x00,0x01,0x00,0x00,0x00, 0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x0c,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x1a,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x1b,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x28,0x00,0x00,0x00, +0x0b,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x1a,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x11,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, +0x1b,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x15,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x24,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x24,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x24,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x47,0x00,0x03,0x00, +0x24,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x4c,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x4f,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x50,0x00,0x00,0x00, 0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x28,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x28,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x08,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x28,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x4f,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x52,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x53,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x53,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x53,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x54,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0x55,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x55,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x55,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x57,0x00,0x00,0x00, -0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x57,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0xa2,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0xa3,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0xa3,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0xa3,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xa5,0x00,0x00,0x00, -0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0xa5,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0xe6,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0xe7,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0xe7,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0xe7,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xe9,0x00,0x00,0x00, -0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0xe9,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0xf1,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x19,0x00,0x00,0x00,0x13,0x00,0x02,0x00,0x02,0x00,0x00,0x00, -0x21,0x00,0x03,0x00,0x03,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x15,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x15,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x17,0x00,0x04,0x00, -0x0a,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x03,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x0b,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x0a,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x0b,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x0e,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x0b,0x00,0x00,0x00, -0x13,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x16,0x00,0x03,0x00, -0x17,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x1c,0x00,0x04,0x00,0x19,0x00,0x00,0x00,0x17,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x1a,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x1a,0x00,0x00,0x00,0x1b,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x17,0x00,0x00,0x00,0x1d,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x1e,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x17,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x1e,0x00,0x05,0x00,0x28,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x29,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x28,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x29,0x00,0x00,0x00,0x2a,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x2b,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x14,0x00,0x02,0x00,0x30,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x35,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x16,0x00,0x03,0x00,0x4c,0x00,0x00,0x00,0x10,0x00,0x00,0x00, -0x15,0x00,0x04,0x00,0x4d,0x00,0x00,0x00,0x10,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x4e,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x1c,0x00,0x04,0x00, -0x4f,0x00,0x00,0x00,0x4d,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, -0x15,0x00,0x04,0x00,0x50,0x00,0x00,0x00,0x08,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x51,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x1c,0x00,0x04,0x00, -0x52,0x00,0x00,0x00,0x50,0x00,0x00,0x00,0x51,0x00,0x00,0x00, -0x1e,0x00,0x05,0x00,0x53,0x00,0x00,0x00,0x4c,0x00,0x00,0x00, -0x4f,0x00,0x00,0x00,0x52,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, -0x54,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, -0x55,0x00,0x00,0x00,0x54,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x56,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x55,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x56,0x00,0x00,0x00,0x57,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x59,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x4c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x61,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x4d,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x65,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x72,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x78,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x81,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x50,0x00,0x00,0x00,0x17,0x00,0x04,0x00, -0x85,0x00,0x00,0x00,0x17,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x89,0x00,0x00,0x00, -0x0f,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x92,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x17,0x00,0x00,0x00,0x9a,0x00,0x00,0x00,0x00,0x00,0x80,0x41, -0x1d,0x00,0x03,0x00,0xa2,0x00,0x00,0x00,0x17,0x00,0x00,0x00, -0x1e,0x00,0x03,0x00,0xa3,0x00,0x00,0x00,0xa2,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0xa4,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0xa3,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0xa4,0x00,0x00,0x00, -0xa5,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0xad,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x17,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0xc8,0x00,0x00,0x00, -0x08,0x01,0x00,0x00,0x1d,0x00,0x03,0x00,0xe6,0x00,0x00,0x00, -0x17,0x00,0x00,0x00,0x1e,0x00,0x03,0x00,0xe7,0x00,0x00,0x00, -0xe6,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0xe8,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0xe7,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0xe8,0x00,0x00,0x00,0xe9,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x2c,0x00,0x06,0x00,0x0a,0x00,0x00,0x00,0xf1,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x92,0x00,0x00,0x00,0x92,0x00,0x00,0x00, -0x2c,0x00,0x05,0x00,0x85,0x00,0x00,0x00,0xf9,0x00,0x00,0x00, -0x9a,0x00,0x00,0x00,0x9a,0x00,0x00,0x00,0x36,0x00,0x05,0x00, -0x02,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x03,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x05,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x0e,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x0d,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x11,0x00,0x00,0x00, -0x10,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x0e,0x00,0x00,0x00, -0x14,0x00,0x00,0x00,0x13,0x00,0x00,0x00,0x0d,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x15,0x00,0x00,0x00, -0x14,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x15,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x1e,0x00,0x00,0x00,0x1f,0x00,0x00,0x00,0x1b,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x3e,0x00,0x03,0x00,0x1f,0x00,0x00,0x00, -0x1d,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x22,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0x22,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xf6,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x05,0x00,0x00,0x00,0xc7,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x2b,0x00,0x00,0x00,0x2c,0x00,0x00,0x00, -0x2a,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x2d,0x00,0x00,0x00,0x2c,0x00,0x00,0x00, -0x87,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x2f,0x00,0x00,0x00, -0x2d,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0xb1,0x00,0x05,0x00, -0x30,0x00,0x00,0x00,0x31,0x00,0x00,0x00,0xf6,0x00,0x00,0x00, -0x2f,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x24,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x31,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x24,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0x23,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0xf6,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x37,0x00,0x00,0x00,0x35,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x38,0x00,0x00,0x00, -0x34,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x3d,0x00,0x00,0x00,0x11,0x00,0x00,0x00, -0x2d,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x3f,0x00,0x00,0x00,0x3d,0x00,0x00,0x00,0x38,0x00,0x00,0x00, -0x87,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x3f,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x8b,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x43,0x00,0x00,0x00,0x38,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x87,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x44,0x00,0x00,0x00,0x43,0x00,0x00,0x00,0x35,0x00,0x00,0x00, -0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x49,0x00,0x00,0x00, -0x38,0x00,0x00,0x00,0x43,0x00,0x00,0x00,0x41,0x00,0x07,0x00, -0x59,0x00,0x00,0x00,0x5a,0x00,0x00,0x00,0x57,0x00,0x00,0x00, -0x21,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4c,0x00,0x00,0x00,0x5b,0x00,0x00,0x00, -0x5a,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x17,0x00,0x00,0x00, -0x5c,0x00,0x00,0x00,0x5b,0x00,0x00,0x00,0x41,0x00,0x08,0x00, -0x61,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x57,0x00,0x00,0x00, -0x21,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x60,0x00,0x00,0x00, -0x60,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0x63,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x71,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x64,0x00,0x00,0x00,0x63,0x00,0x00,0x00, -0xc4,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0x66,0x00,0x00,0x00, -0x64,0x00,0x00,0x00,0x65,0x00,0x00,0x00,0x41,0x00,0x08,0x00, -0x61,0x00,0x00,0x00,0x68,0x00,0x00,0x00,0x57,0x00,0x00,0x00, -0x21,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x60,0x00,0x00,0x00, -0x21,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0x69,0x00,0x00,0x00,0x68,0x00,0x00,0x00,0x71,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x6a,0x00,0x00,0x00,0x69,0x00,0x00,0x00, -0xc5,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0x6b,0x00,0x00,0x00, -0x66,0x00,0x00,0x00,0x6a,0x00,0x00,0x00,0xc2,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0x71,0x00,0x00,0x00,0x6b,0x00,0x00,0x00, -0x44,0x00,0x00,0x00,0xc4,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0x73,0x00,0x00,0x00,0x71,0x00,0x00,0x00,0x72,0x00,0x00,0x00, -0xc7,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0x74,0x00,0x00,0x00, -0x73,0x00,0x00,0x00,0x51,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x75,0x00,0x00,0x00,0x74,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x79,0x00,0x00,0x00, -0x44,0x00,0x00,0x00,0x78,0x00,0x00,0x00,0xc2,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0x7a,0x00,0x00,0x00,0x6b,0x00,0x00,0x00, -0x79,0x00,0x00,0x00,0xc7,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0x7b,0x00,0x00,0x00,0x7a,0x00,0x00,0x00,0x51,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x7c,0x00,0x00,0x00, -0x7b,0x00,0x00,0x00,0x41,0x00,0x08,0x00,0x81,0x00,0x00,0x00, -0x82,0x00,0x00,0x00,0x57,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0x35,0x00,0x00,0x00,0x44,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x50,0x00,0x00,0x00,0x83,0x00,0x00,0x00, -0x82,0x00,0x00,0x00,0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0x83,0x00,0x00,0x00,0xc7,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0x8a,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0x89,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x8d,0x00,0x00,0x00,0x75,0x00,0x00,0x00,0xc5,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0x8e,0x00,0x00,0x00,0x8a,0x00,0x00,0x00, -0x8d,0x00,0x00,0x00,0x70,0x00,0x04,0x00,0x17,0x00,0x00,0x00, -0x8f,0x00,0x00,0x00,0x8e,0x00,0x00,0x00,0xc2,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0x91,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0x72,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x95,0x00,0x00,0x00,0x7c,0x00,0x00,0x00,0xc5,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0x96,0x00,0x00,0x00,0x91,0x00,0x00,0x00, -0x95,0x00,0x00,0x00,0x70,0x00,0x04,0x00,0x17,0x00,0x00,0x00, -0x97,0x00,0x00,0x00,0x96,0x00,0x00,0x00,0x50,0x00,0x05,0x00, -0x85,0x00,0x00,0x00,0x98,0x00,0x00,0x00,0x8f,0x00,0x00,0x00, -0x97,0x00,0x00,0x00,0x83,0x00,0x05,0x00,0x85,0x00,0x00,0x00, -0x9c,0x00,0x00,0x00,0x98,0x00,0x00,0x00,0xf9,0x00,0x00,0x00, -0x8e,0x00,0x05,0x00,0x85,0x00,0x00,0x00,0x9e,0x00,0x00,0x00, -0x9c,0x00,0x00,0x00,0x5c,0x00,0x00,0x00,0x51,0x00,0x05,0x00, -0x17,0x00,0x00,0x00,0xa1,0x00,0x00,0x00,0x9e,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x2b,0x00,0x00,0x00, -0xa6,0x00,0x00,0x00,0x2a,0x00,0x00,0x00,0x60,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xa7,0x00,0x00,0x00, -0xa6,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xa9,0x00,0x00,0x00,0xa7,0x00,0x00,0x00,0x49,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xab,0x00,0x00,0x00, -0xa9,0x00,0x00,0x00,0x44,0x00,0x00,0x00,0x41,0x00,0x06,0x00, -0xad,0x00,0x00,0x00,0xae,0x00,0x00,0x00,0xa5,0x00,0x00,0x00, -0x21,0x00,0x00,0x00,0xab,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x17,0x00,0x00,0x00,0xaf,0x00,0x00,0x00,0xae,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x17,0x00,0x00,0x00,0xb2,0x00,0x00,0x00, -0x1f,0x00,0x00,0x00,0x0c,0x00,0x08,0x00,0x17,0x00,0x00,0x00, -0xb3,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00, -0xa1,0x00,0x00,0x00,0xaf,0x00,0x00,0x00,0xb2,0x00,0x00,0x00, -0x3e,0x00,0x03,0x00,0x1f,0x00,0x00,0x00,0xb3,0x00,0x00,0x00, -0x51,0x00,0x05,0x00,0x17,0x00,0x00,0x00,0xb7,0x00,0x00,0x00, -0x9e,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xbe,0x00,0x00,0x00,0xab,0x00,0x00,0x00, -0x65,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0xad,0x00,0x00,0x00, -0xbf,0x00,0x00,0x00,0xa5,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0xbe,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x17,0x00,0x00,0x00, -0xc0,0x00,0x00,0x00,0xbf,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x17,0x00,0x00,0x00,0xc3,0x00,0x00,0x00,0x1f,0x00,0x00,0x00, -0x0c,0x00,0x08,0x00,0x17,0x00,0x00,0x00,0xc4,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0xb7,0x00,0x00,0x00, -0xc0,0x00,0x00,0x00,0xc3,0x00,0x00,0x00,0x3e,0x00,0x03,0x00, -0x1f,0x00,0x00,0x00,0xc4,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xc7,0x00,0x00,0x00,0xf6,0x00,0x00,0x00, -0x35,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x22,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0x24,0x00,0x00,0x00,0xe0,0x00,0x04,0x00, -0x4e,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0xc8,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xca,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xca,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xf7,0x00,0x00,0x00,0x65,0x00,0x00,0x00,0x24,0x00,0x00,0x00, -0xe1,0x00,0x00,0x00,0xcd,0x00,0x00,0x00,0xad,0x00,0x05,0x00, -0x30,0x00,0x00,0x00,0xd0,0x00,0x00,0x00,0xf7,0x00,0x00,0x00, -0x21,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xcc,0x00,0x00,0x00, -0xcd,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0xd0,0x00,0x00,0x00,0xcb,0x00,0x00,0x00,0xcc,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xcb,0x00,0x00,0x00,0xb1,0x00,0x05,0x00, -0x30,0x00,0x00,0x00,0xd3,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0xf7,0x00,0x00,0x00,0xf7,0x00,0x03,0x00,0xd5,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xd3,0x00,0x00,0x00, -0xd4,0x00,0x00,0x00,0xd5,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xd4,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xd9,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0xf7,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x1e,0x00,0x00,0x00,0xda,0x00,0x00,0x00, -0x1b,0x00,0x00,0x00,0xd9,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x17,0x00,0x00,0x00,0xdb,0x00,0x00,0x00,0xda,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x17,0x00,0x00,0x00,0xdd,0x00,0x00,0x00, -0x1f,0x00,0x00,0x00,0x81,0x00,0x05,0x00,0x17,0x00,0x00,0x00, -0xde,0x00,0x00,0x00,0xdd,0x00,0x00,0x00,0xdb,0x00,0x00,0x00, -0x3e,0x00,0x03,0x00,0x1f,0x00,0x00,0x00,0xde,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xd5,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xd5,0x00,0x00,0x00,0xe0,0x00,0x04,0x00,0x4e,0x00,0x00,0x00, -0x4e,0x00,0x00,0x00,0xc8,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xcd,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xcd,0x00,0x00,0x00, -0xc3,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xe1,0x00,0x00,0x00, -0xf7,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xca,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xcc,0x00,0x00,0x00, -0xaa,0x00,0x05,0x00,0x30,0x00,0x00,0x00,0xe3,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0xf7,0x00,0x03,0x00, -0xe5,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0xe3,0x00,0x00,0x00,0xe4,0x00,0x00,0x00,0xe5,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xe4,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x2b,0x00,0x00,0x00,0xea,0x00,0x00,0x00,0x2a,0x00,0x00,0x00, -0x35,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xeb,0x00,0x00,0x00,0xea,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xed,0x00,0x00,0x00,0xeb,0x00,0x00,0x00, -0x11,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x1e,0x00,0x00,0x00, -0xee,0x00,0x00,0x00,0x1b,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x17,0x00,0x00,0x00,0xef,0x00,0x00,0x00, -0xee,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0xad,0x00,0x00,0x00, -0xf0,0x00,0x00,0x00,0xe9,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0xed,0x00,0x00,0x00,0x3e,0x00,0x03,0x00,0xf0,0x00,0x00,0x00, -0xef,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xe5,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xe5,0x00,0x00,0x00,0xfd,0x00,0x01,0x00, -0x38,0x00,0x01,0x00, -}; -const uint64_t mul_mat_vec_q5_0_f32_len = 3676; - -unsigned char mul_mat_vec_q5_1_f32_data[] = { -0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, -0xf5,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, -0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x51,0x11,0x00,0x00, -0x11,0x00,0x02,0x00,0x60,0x11,0x00,0x00,0x0b,0x00,0x06,0x00, -0x01,0x00,0x00,0x00,0x47,0x4c,0x53,0x4c,0x2e,0x73,0x74,0x64, -0x2e,0x34,0x35,0x30,0x00,0x00,0x00,0x00,0x0e,0x00,0x03,0x00, -0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x0f,0x00,0x0c,0x00, -0x05,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x6d,0x61,0x69,0x6e, -0x00,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x13,0x00,0x00,0x00, -0x1b,0x00,0x00,0x00,0x2a,0x00,0x00,0x00,0x54,0x00,0x00,0x00, -0xa0,0x00,0x00,0x00,0xe6,0x00,0x00,0x00,0x10,0x00,0x06,0x00, -0x04,0x00,0x00,0x00,0x11,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x0c,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x1a,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x1b,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x28,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x28,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x28,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x08,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x28,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x4f,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x50,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x50,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x50,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x50,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x08,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x51,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x48,0x00,0x04,0x00, +0x48,0x00,0x05,0x00,0x50,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x50,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x51,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x48,0x00,0x04,0x00, 0x52,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00, 0x48,0x00,0x05,0x00,0x52,0x00,0x00,0x00,0x00,0x00,0x00,0x00, 0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, 0x52,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, 0x54,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, 0x47,0x00,0x04,0x00,0x54,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x9d,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xa0,0x00,0x00,0x00, 0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00, -0x9e,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x9e,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0xa1,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0xa1,0x00,0x00,0x00,0x00,0x00,0x00,0x00, 0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, -0x9e,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0xa0,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0xa0,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xe3,0x00,0x00,0x00, +0xa1,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0xa3,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0xa3,0x00,0x00,0x00,0x21,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xe1,0x00,0x00,0x00, 0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00, -0xe4,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x19,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0xe4,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0xe2,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x19,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0xe2,0x00,0x00,0x00,0x00,0x00,0x00,0x00, 0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, -0xe4,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0xe6,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0xe6,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xee,0x00,0x00,0x00, -0x0b,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x13,0x00,0x02,0x00, -0x02,0x00,0x00,0x00,0x21,0x00,0x03,0x00,0x03,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x15,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x15,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x17,0x00,0x04,0x00,0x0a,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x03,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x0b,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x0a,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x0b,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x0d,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x0e,0x00,0x00,0x00, +0xe2,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0xe4,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0xe4,0x00,0x00,0x00,0x21,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xec,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0xed,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x19,0x00,0x00,0x00, +0x13,0x00,0x02,0x00,0x02,0x00,0x00,0x00,0x21,0x00,0x03,0x00, +0x03,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x15,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x17,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x03,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x0a,0x00,0x00,0x00, 0x01,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x0b,0x00,0x00,0x00,0x13,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x16,0x00,0x03,0x00,0x17,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x18,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x1c,0x00,0x04,0x00,0x19,0x00,0x00,0x00, -0x17,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x1a,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x19,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x1a,0x00,0x00,0x00,0x1b,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x17,0x00,0x00,0x00, -0x1d,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x1e,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x17,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x1e,0x00,0x05,0x00,0x28,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x0a,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x0d,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0x0a,0x00,0x00,0x00,0x11,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x16,0x00,0x03,0x00,0x14,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x15,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0x1c,0x00,0x04,0x00,0x16,0x00,0x00,0x00, +0x14,0x00,0x00,0x00,0x15,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x17,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x16,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0x17,0x00,0x00,0x00,0x18,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x14,0x00,0x00,0x00, +0x1a,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x1b,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x14,0x00,0x00,0x00, +0x1e,0x00,0x05,0x00,0x24,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x25,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x24,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0x25,0x00,0x00,0x00,0x26,0x00,0x00,0x00, +0x09,0x00,0x00,0x00,0x15,0x00,0x04,0x00,0x27,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x27,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0x00,0x00,0x00,0x00, 0x20,0x00,0x04,0x00,0x29,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x28,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x29,0x00,0x00,0x00, -0x2a,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x2b,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x14,0x00,0x02,0x00,0x30,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x35,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x16,0x00,0x03,0x00,0x4c,0x00,0x00,0x00, -0x10,0x00,0x00,0x00,0x15,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x14,0x00,0x02,0x00,0x2d,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x32,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x3d,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x16,0x00,0x03,0x00, +0x4a,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x15,0x00,0x04,0x00, +0x4b,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x1c,0x00,0x04,0x00,0x4c,0x00,0x00,0x00,0x4b,0x00,0x00,0x00, +0x32,0x00,0x00,0x00,0x15,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, 0x08,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x10,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x10,0x00,0x00,0x00, 0x1c,0x00,0x04,0x00,0x4f,0x00,0x00,0x00,0x4d,0x00,0x00,0x00, -0x4e,0x00,0x00,0x00,0x1e,0x00,0x06,0x00,0x50,0x00,0x00,0x00, -0x4c,0x00,0x00,0x00,0x4c,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x4f,0x00,0x00,0x00,0x1d,0x00,0x03,0x00,0x51,0x00,0x00,0x00, -0x50,0x00,0x00,0x00,0x1e,0x00,0x03,0x00,0x52,0x00,0x00,0x00, -0x51,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x53,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x52,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x53,0x00,0x00,0x00,0x54,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x56,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x4c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x5c,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x64,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x69,0x00,0x00,0x00, +0x4e,0x00,0x00,0x00,0x1e,0x00,0x05,0x00,0x50,0x00,0x00,0x00, +0x4a,0x00,0x00,0x00,0x4c,0x00,0x00,0x00,0x4f,0x00,0x00,0x00, +0x1d,0x00,0x03,0x00,0x51,0x00,0x00,0x00,0x50,0x00,0x00,0x00, +0x1e,0x00,0x03,0x00,0x52,0x00,0x00,0x00,0x51,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x53,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x52,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x53,0x00,0x00,0x00, +0x54,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x56,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x4a,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x27,0x00,0x00,0x00,0x5c,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x5d,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x4b,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x27,0x00,0x00,0x00,0x61,0x00,0x00,0x00,0x10,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x27,0x00,0x00,0x00,0x6e,0x00,0x00,0x00, 0x04,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x71,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x7a,0x00,0x00,0x00,0x03,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x7c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x4d,0x00,0x00,0x00,0x17,0x00,0x04,0x00,0x80,0x00,0x00,0x00, -0x17,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x8d,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x1d,0x00,0x03,0x00,0x9d,0x00,0x00,0x00, -0x17,0x00,0x00,0x00,0x1e,0x00,0x03,0x00,0x9e,0x00,0x00,0x00, -0x9d,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x9f,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x9e,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x9f,0x00,0x00,0x00,0xa0,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0xa8,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x17,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xb9,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0xc4,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0xc5,0x00,0x00,0x00, -0x08,0x01,0x00,0x00,0x1d,0x00,0x03,0x00,0xe3,0x00,0x00,0x00, -0x17,0x00,0x00,0x00,0x1e,0x00,0x03,0x00,0xe4,0x00,0x00,0x00, -0xe3,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0xe5,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0xe4,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0xe5,0x00,0x00,0x00,0xe6,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x2c,0x00,0x06,0x00,0x0a,0x00,0x00,0x00,0xee,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x8d,0x00,0x00,0x00,0x8d,0x00,0x00,0x00, -0x36,0x00,0x05,0x00,0x02,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x05,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x0e,0x00,0x00,0x00, -0x0f,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x0d,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x10,0x00,0x00,0x00, -0x0f,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x11,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x0e,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x13,0x00,0x00,0x00, -0x0d,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x15,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x15,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x1e,0x00,0x00,0x00,0x1f,0x00,0x00,0x00, -0x1b,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x3e,0x00,0x03,0x00, -0x1f,0x00,0x00,0x00,0x1d,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x22,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x22,0x00,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xf3,0x00,0x00,0x00, -0x21,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0xc3,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x2b,0x00,0x00,0x00, -0x2c,0x00,0x00,0x00,0x2a,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x2d,0x00,0x00,0x00, -0x2c,0x00,0x00,0x00,0x87,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x2f,0x00,0x00,0x00,0x2d,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0xb1,0x00,0x05,0x00,0x30,0x00,0x00,0x00,0x31,0x00,0x00,0x00, -0xf3,0x00,0x00,0x00,0x2f,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0x24,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x31,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x24,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x23,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x34,0x00,0x00,0x00, -0xf3,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x35,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x38,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0x37,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3d,0x00,0x00,0x00, -0x11,0x00,0x00,0x00,0x2d,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x3f,0x00,0x00,0x00,0x3d,0x00,0x00,0x00, -0x38,0x00,0x00,0x00,0x87,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0x3f,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x8b,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x43,0x00,0x00,0x00, -0x38,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x87,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x44,0x00,0x00,0x00,0x43,0x00,0x00,0x00, -0x35,0x00,0x00,0x00,0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x49,0x00,0x00,0x00,0x38,0x00,0x00,0x00,0x43,0x00,0x00,0x00, +0x74,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x27,0x00,0x00,0x00,0x7c,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x7e,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x4d,0x00,0x00,0x00,0x17,0x00,0x04,0x00,0x82,0x00,0x00,0x00, +0x14,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x90,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x14,0x00,0x00,0x00, +0x98,0x00,0x00,0x00,0x00,0x00,0x80,0x41,0x1d,0x00,0x03,0x00, +0xa0,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, +0xa1,0x00,0x00,0x00,0xa0,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0xa2,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0xa1,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0xa2,0x00,0x00,0x00,0xa3,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0xab,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xc2,0x00,0x00,0x00,0x08,0x01,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xc4,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0x15,0x00,0x00,0x00,0x32,0x00,0x00,0x00, +0x1d,0x00,0x03,0x00,0xe1,0x00,0x00,0x00,0x14,0x00,0x00,0x00, +0x1e,0x00,0x03,0x00,0xe2,0x00,0x00,0x00,0xe1,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0xe3,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0xe2,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0xe3,0x00,0x00,0x00, +0xe4,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x32,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xec,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x33,0x00,0x06,0x00,0x09,0x00,0x00,0x00,0xed,0x00,0x00,0x00, +0xec,0x00,0x00,0x00,0x90,0x00,0x00,0x00,0x90,0x00,0x00,0x00, +0x2c,0x00,0x05,0x00,0x82,0x00,0x00,0x00,0xf5,0x00,0x00,0x00, +0x98,0x00,0x00,0x00,0x98,0x00,0x00,0x00,0x36,0x00,0x05,0x00, +0x02,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x03,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x05,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00,0x0e,0x00,0x00,0x00, +0x0b,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x0e,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0x11,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x13,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x1b,0x00,0x00,0x00,0x1c,0x00,0x00,0x00, +0x18,0x00,0x00,0x00,0x13,0x00,0x00,0x00,0x3e,0x00,0x03,0x00, +0x1c,0x00,0x00,0x00,0x1a,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x1e,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x1e,0x00,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xf2,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0xc1,0x00,0x00,0x00, +0x1f,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x29,0x00,0x00,0x00, +0x2a,0x00,0x00,0x00,0x26,0x00,0x00,0x00,0x28,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x2b,0x00,0x00,0x00, +0x2a,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x2c,0x00,0x00,0x00,0x2b,0x00,0x00,0x00,0x15,0x00,0x00,0x00, +0xb0,0x00,0x05,0x00,0x2d,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, +0xf2,0x00,0x00,0x00,0x2c,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0x20,0x00,0x00,0x00,0x1f,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x2e,0x00,0x00,0x00,0x1f,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x1f,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x31,0x00,0x00,0x00, +0xf2,0x00,0x00,0x00,0x15,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0x32,0x00,0x00,0x00, +0x13,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x35,0x00,0x00,0x00,0x31,0x00,0x00,0x00,0x34,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, +0x0f,0x00,0x00,0x00,0x2b,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x3c,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, +0x35,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x3e,0x00,0x00,0x00,0x3c,0x00,0x00,0x00,0x3d,0x00,0x00,0x00, +0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x41,0x00,0x00,0x00, +0x35,0x00,0x00,0x00,0x3d,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x42,0x00,0x00,0x00,0x41,0x00,0x00,0x00, +0x32,0x00,0x00,0x00,0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x47,0x00,0x00,0x00,0x35,0x00,0x00,0x00,0x41,0x00,0x00,0x00, 0x41,0x00,0x07,0x00,0x56,0x00,0x00,0x00,0x57,0x00,0x00,0x00, -0x54,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x21,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x4c,0x00,0x00,0x00, +0x54,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0x3e,0x00,0x00,0x00, +0x28,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, 0x58,0x00,0x00,0x00,0x57,0x00,0x00,0x00,0x73,0x00,0x04,0x00, -0x17,0x00,0x00,0x00,0x59,0x00,0x00,0x00,0x58,0x00,0x00,0x00, -0x41,0x00,0x07,0x00,0x56,0x00,0x00,0x00,0x5d,0x00,0x00,0x00, -0x54,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x5c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x4c,0x00,0x00,0x00, -0x5e,0x00,0x00,0x00,0x5d,0x00,0x00,0x00,0x73,0x00,0x04,0x00, -0x17,0x00,0x00,0x00,0x5f,0x00,0x00,0x00,0x5e,0x00,0x00,0x00, -0x41,0x00,0x07,0x00,0x64,0x00,0x00,0x00,0x65,0x00,0x00,0x00, -0x54,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x35,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x66,0x00,0x00,0x00,0x65,0x00,0x00,0x00,0xc2,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0x68,0x00,0x00,0x00,0x66,0x00,0x00,0x00, -0x44,0x00,0x00,0x00,0xc4,0x00,0x05,0x00,0x09,0x00,0x00,0x00, -0x6a,0x00,0x00,0x00,0x68,0x00,0x00,0x00,0x69,0x00,0x00,0x00, -0xc7,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0x6b,0x00,0x00,0x00, -0x6a,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x6b,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x6f,0x00,0x00,0x00, -0x65,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x72,0x00,0x00,0x00,0x44,0x00,0x00,0x00,0x71,0x00,0x00,0x00, -0xc2,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0x73,0x00,0x00,0x00, -0x6f,0x00,0x00,0x00,0x72,0x00,0x00,0x00,0xc7,0x00,0x05,0x00, -0x09,0x00,0x00,0x00,0x74,0x00,0x00,0x00,0x73,0x00,0x00,0x00, -0x4e,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x75,0x00,0x00,0x00,0x74,0x00,0x00,0x00,0x41,0x00,0x08,0x00, -0x7c,0x00,0x00,0x00,0x7d,0x00,0x00,0x00,0x54,0x00,0x00,0x00, -0x21,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x7a,0x00,0x00,0x00, -0x44,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0x7e,0x00,0x00,0x00,0x7d,0x00,0x00,0x00,0x71,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x7f,0x00,0x00,0x00,0x7e,0x00,0x00,0x00, -0xc7,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0x85,0x00,0x00,0x00, -0x7f,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x88,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0xc5,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0x89,0x00,0x00,0x00, -0x85,0x00,0x00,0x00,0x88,0x00,0x00,0x00,0x70,0x00,0x04,0x00, -0x17,0x00,0x00,0x00,0x8a,0x00,0x00,0x00,0x89,0x00,0x00,0x00, -0xc2,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0x8c,0x00,0x00,0x00, -0x7f,0x00,0x00,0x00,0x69,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x90,0x00,0x00,0x00,0x75,0x00,0x00,0x00, -0xc5,0x00,0x05,0x00,0x09,0x00,0x00,0x00,0x91,0x00,0x00,0x00, -0x8c,0x00,0x00,0x00,0x90,0x00,0x00,0x00,0x70,0x00,0x04,0x00, -0x17,0x00,0x00,0x00,0x92,0x00,0x00,0x00,0x91,0x00,0x00,0x00, -0x50,0x00,0x05,0x00,0x80,0x00,0x00,0x00,0x93,0x00,0x00,0x00, -0x8a,0x00,0x00,0x00,0x92,0x00,0x00,0x00,0x8e,0x00,0x05,0x00, -0x80,0x00,0x00,0x00,0x96,0x00,0x00,0x00,0x93,0x00,0x00,0x00, -0x59,0x00,0x00,0x00,0x50,0x00,0x05,0x00,0x80,0x00,0x00,0x00, -0x98,0x00,0x00,0x00,0x5f,0x00,0x00,0x00,0x5f,0x00,0x00,0x00, -0x81,0x00,0x05,0x00,0x80,0x00,0x00,0x00,0x99,0x00,0x00,0x00, -0x96,0x00,0x00,0x00,0x98,0x00,0x00,0x00,0x51,0x00,0x05,0x00, -0x17,0x00,0x00,0x00,0x9c,0x00,0x00,0x00,0x99,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x2b,0x00,0x00,0x00, -0xa1,0x00,0x00,0x00,0x2a,0x00,0x00,0x00,0x5c,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xa2,0x00,0x00,0x00, -0xa1,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xa4,0x00,0x00,0x00,0xa2,0x00,0x00,0x00,0x49,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa6,0x00,0x00,0x00, -0xa4,0x00,0x00,0x00,0x44,0x00,0x00,0x00,0x41,0x00,0x06,0x00, -0xa8,0x00,0x00,0x00,0xa9,0x00,0x00,0x00,0xa0,0x00,0x00,0x00, -0x21,0x00,0x00,0x00,0xa6,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x17,0x00,0x00,0x00,0xaa,0x00,0x00,0x00,0xa9,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x17,0x00,0x00,0x00,0xad,0x00,0x00,0x00, -0x1f,0x00,0x00,0x00,0x0c,0x00,0x08,0x00,0x17,0x00,0x00,0x00, -0xae,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00, -0x9c,0x00,0x00,0x00,0xaa,0x00,0x00,0x00,0xad,0x00,0x00,0x00, -0x3e,0x00,0x03,0x00,0x1f,0x00,0x00,0x00,0xae,0x00,0x00,0x00, -0x51,0x00,0x05,0x00,0x17,0x00,0x00,0x00,0xb2,0x00,0x00,0x00, -0x99,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xba,0x00,0x00,0x00,0xa6,0x00,0x00,0x00, -0xb9,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0xa8,0x00,0x00,0x00, -0xbb,0x00,0x00,0x00,0xa0,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0xba,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x17,0x00,0x00,0x00, -0xbc,0x00,0x00,0x00,0xbb,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x17,0x00,0x00,0x00,0xbf,0x00,0x00,0x00,0x1f,0x00,0x00,0x00, -0x0c,0x00,0x08,0x00,0x17,0x00,0x00,0x00,0xc0,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0xb2,0x00,0x00,0x00, -0xbc,0x00,0x00,0x00,0xbf,0x00,0x00,0x00,0x3e,0x00,0x03,0x00, -0x1f,0x00,0x00,0x00,0xc0,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xc3,0x00,0x00,0x00,0xf3,0x00,0x00,0x00, -0x35,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x22,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0x24,0x00,0x00,0x00,0xe0,0x00,0x04,0x00, -0xc4,0x00,0x00,0x00,0xc4,0x00,0x00,0x00,0xc5,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xc7,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xc7,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xf4,0x00,0x00,0x00,0xb9,0x00,0x00,0x00,0x24,0x00,0x00,0x00, -0xde,0x00,0x00,0x00,0xca,0x00,0x00,0x00,0xad,0x00,0x05,0x00, -0x30,0x00,0x00,0x00,0xcd,0x00,0x00,0x00,0xf4,0x00,0x00,0x00, -0x21,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xc9,0x00,0x00,0x00, -0xca,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0xcd,0x00,0x00,0x00,0xc8,0x00,0x00,0x00,0xc9,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xc8,0x00,0x00,0x00,0xb1,0x00,0x05,0x00, -0x30,0x00,0x00,0x00,0xd0,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0xf4,0x00,0x00,0x00,0xf7,0x00,0x03,0x00,0xd2,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xd0,0x00,0x00,0x00, -0xd1,0x00,0x00,0x00,0xd2,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xd1,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xd6,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0xf4,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x1e,0x00,0x00,0x00,0xd7,0x00,0x00,0x00, -0x1b,0x00,0x00,0x00,0xd6,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x17,0x00,0x00,0x00,0xd8,0x00,0x00,0x00,0xd7,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x17,0x00,0x00,0x00,0xda,0x00,0x00,0x00, -0x1f,0x00,0x00,0x00,0x81,0x00,0x05,0x00,0x17,0x00,0x00,0x00, -0xdb,0x00,0x00,0x00,0xda,0x00,0x00,0x00,0xd8,0x00,0x00,0x00, -0x3e,0x00,0x03,0x00,0x1f,0x00,0x00,0x00,0xdb,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xd2,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xd2,0x00,0x00,0x00,0xe0,0x00,0x04,0x00,0xc4,0x00,0x00,0x00, -0xc4,0x00,0x00,0x00,0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xca,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xca,0x00,0x00,0x00, -0xc3,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xde,0x00,0x00,0x00, -0xf4,0x00,0x00,0x00,0x5c,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xc7,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xc9,0x00,0x00,0x00, -0xaa,0x00,0x05,0x00,0x30,0x00,0x00,0x00,0xe0,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0xf7,0x00,0x03,0x00, -0xe2,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0xe0,0x00,0x00,0x00,0xe1,0x00,0x00,0x00,0xe2,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xe1,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x2b,0x00,0x00,0x00,0xe7,0x00,0x00,0x00,0x2a,0x00,0x00,0x00, -0x35,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xe8,0x00,0x00,0x00,0xe7,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xea,0x00,0x00,0x00,0xe8,0x00,0x00,0x00, -0x11,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x1e,0x00,0x00,0x00, -0xeb,0x00,0x00,0x00,0x1b,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x17,0x00,0x00,0x00,0xec,0x00,0x00,0x00, -0xeb,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0xa8,0x00,0x00,0x00, -0xed,0x00,0x00,0x00,0xe6,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0xea,0x00,0x00,0x00,0x3e,0x00,0x03,0x00,0xed,0x00,0x00,0x00, -0xec,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xe2,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xe2,0x00,0x00,0x00,0xfd,0x00,0x01,0x00, -0x38,0x00,0x01,0x00, -}; -const uint64_t mul_mat_vec_q5_1_f32_len = 3604; - -unsigned char mul_mat_vec_q5_K_f32_data[] = { -0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, -0x8d,0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, -0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x27,0x00,0x00,0x00, -0x11,0x00,0x02,0x00,0x51,0x11,0x00,0x00,0x11,0x00,0x02,0x00, -0x60,0x11,0x00,0x00,0x0b,0x00,0x06,0x00,0x01,0x00,0x00,0x00, -0x47,0x4c,0x53,0x4c,0x2e,0x73,0x74,0x64,0x2e,0x34,0x35,0x30, -0x00,0x00,0x00,0x00,0x0e,0x00,0x03,0x00,0x00,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x0f,0x00,0x0c,0x00,0x05,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x6d,0x61,0x69,0x6e,0x00,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x15,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x5e,0x00,0x00,0x00,0x88,0x00,0x00,0x00,0xfa,0x01,0x00,0x00, -0x7f,0x04,0x00,0x00,0x10,0x00,0x06,0x00,0x04,0x00,0x00,0x00, -0x11,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x0c,0x00,0x00,0x00, -0x0b,0x00,0x00,0x00,0x1a,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x13,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x13,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x13,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x47,0x00,0x03,0x00, -0x13,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x21,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x1b,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x80,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x81,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x83,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x84,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x84,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x84,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x10,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x84,0x00,0x00,0x00,0x03,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x30,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x85,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0xb0,0x00,0x00,0x00, -0x48,0x00,0x04,0x00,0x86,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x86,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x03,0x00,0x86,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x88,0x00,0x00,0x00,0x22,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x88,0x00,0x00,0x00, -0x21,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0xf7,0x01,0x00,0x00,0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x48,0x00,0x04,0x00,0xf8,0x01,0x00,0x00,0x00,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0xf8,0x01,0x00,0x00, -0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x03,0x00,0xf8,0x01,0x00,0x00,0x02,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0xfa,0x01,0x00,0x00,0x22,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xfa,0x01,0x00,0x00, -0x21,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x7c,0x04,0x00,0x00,0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x48,0x00,0x04,0x00,0x7d,0x04,0x00,0x00,0x00,0x00,0x00,0x00, -0x19,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x7d,0x04,0x00,0x00, -0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x03,0x00,0x7d,0x04,0x00,0x00,0x02,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x7f,0x04,0x00,0x00,0x22,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x7f,0x04,0x00,0x00, -0x21,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x87,0x04,0x00,0x00,0x0b,0x00,0x00,0x00,0x19,0x00,0x00,0x00, -0x13,0x00,0x02,0x00,0x02,0x00,0x00,0x00,0x21,0x00,0x03,0x00, -0x03,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x15,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x15,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x17,0x00,0x04,0x00,0x0a,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x0b,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x0a,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x0b,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x0d,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x0e,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x1e,0x00,0x05,0x00,0x13,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x14,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x13,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0x15,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x17,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x1a,0x00,0x00,0x00, -0x00,0x01,0x00,0x00,0x3b,0x00,0x04,0x00,0x0b,0x00,0x00,0x00, -0x21,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x25,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x42,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x48,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x15,0x00,0x04,0x00,0x4d,0x00,0x00,0x00,0x08,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x50,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x15,0x00,0x04,0x00, -0x54,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x16,0x00,0x03,0x00,0x5a,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x5b,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x1c,0x00,0x04,0x00,0x5c,0x00,0x00,0x00, -0x5a,0x00,0x00,0x00,0x5b,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x5d,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x5c,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x5d,0x00,0x00,0x00,0x5e,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x5f,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x5a,0x00,0x00,0x00,0x64,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x65,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x5a,0x00,0x00,0x00,0x14,0x00,0x02,0x00,0x70,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x79,0x00,0x00,0x00, -0x80,0x00,0x00,0x00,0x16,0x00,0x03,0x00,0x7d,0x00,0x00,0x00, -0x10,0x00,0x00,0x00,0x17,0x00,0x04,0x00,0x7e,0x00,0x00,0x00, -0x7d,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x7f,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x1c,0x00,0x04,0x00,0x80,0x00,0x00,0x00,0x4d,0x00,0x00,0x00, -0x7f,0x00,0x00,0x00,0x1c,0x00,0x04,0x00,0x81,0x00,0x00,0x00, -0x4d,0x00,0x00,0x00,0x5b,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x82,0x00,0x00,0x00,0x80,0x00,0x00,0x00, -0x1c,0x00,0x04,0x00,0x83,0x00,0x00,0x00,0x4d,0x00,0x00,0x00, -0x82,0x00,0x00,0x00,0x1e,0x00,0x06,0x00,0x84,0x00,0x00,0x00, -0x7e,0x00,0x00,0x00,0x80,0x00,0x00,0x00,0x81,0x00,0x00,0x00, -0x83,0x00,0x00,0x00,0x1d,0x00,0x03,0x00,0x85,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0x1e,0x00,0x03,0x00,0x86,0x00,0x00,0x00, -0x85,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x87,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x87,0x00,0x00,0x00,0x88,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x8c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x7d,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x94,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x9e,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x4d,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xa3,0x00,0x00,0x00, -0x3f,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xc9,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xd8,0x00,0x00,0x00,0x08,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xde,0x00,0x00,0x00, -0x0f,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xe9,0x00,0x00,0x00,0xc0,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xf5,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x47,0x01,0x00,0x00, -0x03,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x6f,0x01,0x00,0x00,0x11,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xad,0x01,0x00,0x00,0x41,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xbb,0x01,0x00,0x00, -0x50,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xc9,0x01,0x00,0x00,0x51,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, -0xf7,0x01,0x00,0x00,0x5a,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, -0xf8,0x01,0x00,0x00,0xf7,0x01,0x00,0x00,0x20,0x00,0x04,0x00, -0xf9,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0xf8,0x01,0x00,0x00, -0x3b,0x00,0x04,0x00,0xf9,0x01,0x00,0x00,0xfa,0x01,0x00,0x00, -0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0xff,0x01,0x00,0x00, -0x0c,0x00,0x00,0x00,0x5a,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x84,0x02,0x00,0x00,0x21,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xa1,0x02,0x00,0x00, -0x30,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xbe,0x02,0x00,0x00,0x31,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x5d,0x04,0x00,0x00,0x02,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x5e,0x04,0x00,0x00, -0x08,0x01,0x00,0x00,0x1d,0x00,0x03,0x00,0x7c,0x04,0x00,0x00, -0x5a,0x00,0x00,0x00,0x1e,0x00,0x03,0x00,0x7d,0x04,0x00,0x00, -0x7c,0x04,0x00,0x00,0x20,0x00,0x04,0x00,0x7e,0x04,0x00,0x00, -0x0c,0x00,0x00,0x00,0x7d,0x04,0x00,0x00,0x3b,0x00,0x04,0x00, -0x7e,0x04,0x00,0x00,0x7f,0x04,0x00,0x00,0x0c,0x00,0x00,0x00, -0x2c,0x00,0x06,0x00,0x0a,0x00,0x00,0x00,0x87,0x04,0x00,0x00, -0x5b,0x00,0x00,0x00,0x94,0x00,0x00,0x00,0x94,0x00,0x00,0x00, -0x36,0x00,0x05,0x00,0x02,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x05,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x0e,0x00,0x00,0x00, -0x0f,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x0d,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x10,0x00,0x00,0x00, -0x0f,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x11,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x17,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x15,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x19,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x87,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x1b,0x00,0x00,0x00,0x19,0x00,0x00,0x00, -0x1a,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x1f,0x00,0x00,0x00,0x11,0x00,0x00,0x00,0x1b,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x0e,0x00,0x00,0x00,0x22,0x00,0x00,0x00, -0x21,0x00,0x00,0x00,0x0d,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x22,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x24,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x87,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x26,0x00,0x00,0x00,0x24,0x00,0x00,0x00,0x25,0x00,0x00,0x00, -0x8b,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x2b,0x00,0x00,0x00, -0x24,0x00,0x00,0x00,0x25,0x00,0x00,0x00,0x87,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x2f,0x00,0x00,0x00,0x26,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x33,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x2f,0x00,0x00,0x00, -0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x34,0x00,0x00,0x00, -0x26,0x00,0x00,0x00,0x33,0x00,0x00,0x00,0x87,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x2f,0x00,0x00,0x00, -0x25,0x00,0x00,0x00,0x8b,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x3a,0x00,0x00,0x00,0x2f,0x00,0x00,0x00,0x25,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3d,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x3f,0x00,0x00,0x00,0x25,0x00,0x00,0x00, -0x3a,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0x3d,0x00,0x00,0x00,0x3f,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x44,0x00,0x00,0x00, -0x42,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x46,0x00,0x00,0x00,0x44,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x4a,0x00,0x00,0x00,0x48,0x00,0x00,0x00,0x37,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x4c,0x00,0x00,0x00, -0x4a,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x52,0x00,0x00,0x00,0x25,0x00,0x00,0x00, -0x37,0x00,0x00,0x00,0xc4,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x53,0x00,0x00,0x00,0x50,0x00,0x00,0x00,0x52,0x00,0x00,0x00, -0x72,0x00,0x04,0x00,0x54,0x00,0x00,0x00,0x55,0x00,0x00,0x00, -0x53,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0x56,0x00,0x00,0x00,0x55,0x00,0x00,0x00,0xc4,0x00,0x05,0x00, -0x4d,0x00,0x00,0x00,0x59,0x00,0x00,0x00,0x56,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x61,0x00,0x00,0x00,0x5f,0x00,0x00,0x00,0x2b,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x63,0x00,0x00,0x00, -0x61,0x00,0x00,0x00,0x26,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x65,0x00,0x00,0x00,0x66,0x00,0x00,0x00,0x5e,0x00,0x00,0x00, -0x63,0x00,0x00,0x00,0x3e,0x00,0x03,0x00,0x66,0x00,0x00,0x00, -0x64,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x69,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0x69,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0x88,0x04,0x00,0x00,0x2b,0x00,0x00,0x00, -0x05,0x00,0x00,0x00,0x5c,0x04,0x00,0x00,0x6a,0x00,0x00,0x00, -0xb1,0x00,0x05,0x00,0x70,0x00,0x00,0x00,0x71,0x00,0x00,0x00, -0x88,0x04,0x00,0x00,0x1b,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0x6b,0x00,0x00,0x00,0x6a,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x71,0x00,0x00,0x00,0x6a,0x00,0x00,0x00, -0x6b,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x6a,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x74,0x00,0x00,0x00, -0x88,0x04,0x00,0x00,0x1a,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x76,0x00,0x00,0x00,0x74,0x00,0x00,0x00, -0x4c,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x7a,0x00,0x00,0x00,0x76,0x00,0x00,0x00,0x79,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8b,0x00,0x00,0x00, -0x1f,0x00,0x00,0x00,0x88,0x04,0x00,0x00,0x41,0x00,0x08,0x00, -0x8c,0x00,0x00,0x00,0x8d,0x00,0x00,0x00,0x88,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x8b,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0x0d,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x7d,0x00,0x00,0x00, -0x8e,0x00,0x00,0x00,0x8d,0x00,0x00,0x00,0x73,0x00,0x04,0x00, -0x5a,0x00,0x00,0x00,0x8f,0x00,0x00,0x00,0x8e,0x00,0x00,0x00, -0x41,0x00,0x08,0x00,0x8c,0x00,0x00,0x00,0x95,0x00,0x00,0x00, -0x88,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x8b,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x94,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x7d,0x00,0x00,0x00,0x96,0x00,0x00,0x00,0x95,0x00,0x00,0x00, -0x73,0x00,0x04,0x00,0x5a,0x00,0x00,0x00,0x97,0x00,0x00,0x00, -0x96,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x9d,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x25,0x00,0x00,0x00, -0x41,0x00,0x08,0x00,0x9e,0x00,0x00,0x00,0x9f,0x00,0x00,0x00, -0x88,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x8b,0x00,0x00,0x00, -0x50,0x00,0x00,0x00,0x9d,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0xa0,0x00,0x00,0x00,0x9f,0x00,0x00,0x00, -0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0xa1,0x00,0x00,0x00, -0xa0,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xa2,0x00,0x00,0x00,0xa1,0x00,0x00,0x00,0xc7,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xa4,0x00,0x00,0x00,0xa2,0x00,0x00,0x00, -0xa3,0x00,0x00,0x00,0x72,0x00,0x04,0x00,0x54,0x00,0x00,0x00, -0xa5,0x00,0x00,0x00,0xa4,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0xa6,0x00,0x00,0x00,0xa5,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xad,0x00,0x00,0x00, -0x9d,0x00,0x00,0x00,0x50,0x00,0x00,0x00,0x41,0x00,0x08,0x00, -0x9e,0x00,0x00,0x00,0xae,0x00,0x00,0x00,0x88,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x8b,0x00,0x00,0x00,0x50,0x00,0x00,0x00, -0xad,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0xaf,0x00,0x00,0x00,0xae,0x00,0x00,0x00,0x71,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0xb0,0x00,0x00,0x00,0xaf,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xb1,0x00,0x00,0x00, -0xb0,0x00,0x00,0x00,0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xb2,0x00,0x00,0x00,0xb1,0x00,0x00,0x00,0xa3,0x00,0x00,0x00, -0x72,0x00,0x04,0x00,0x54,0x00,0x00,0x00,0xb3,0x00,0x00,0x00, -0xb2,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0xb4,0x00,0x00,0x00,0xb3,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xbb,0x00,0x00,0x00,0x9d,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x41,0x00,0x08,0x00,0x9e,0x00,0x00,0x00, -0xbc,0x00,0x00,0x00,0x88,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0x8b,0x00,0x00,0x00,0x50,0x00,0x00,0x00,0xbb,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00,0xbd,0x00,0x00,0x00, -0xbc,0x00,0x00,0x00,0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0xbe,0x00,0x00,0x00,0xbd,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xbf,0x00,0x00,0x00,0xbe,0x00,0x00,0x00, -0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xc0,0x00,0x00,0x00, -0xbf,0x00,0x00,0x00,0xa3,0x00,0x00,0x00,0x72,0x00,0x04,0x00, -0x54,0x00,0x00,0x00,0xc1,0x00,0x00,0x00,0xc0,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x4d,0x00,0x00,0x00,0xc2,0x00,0x00,0x00, -0xc1,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xca,0x00,0x00,0x00,0x9d,0x00,0x00,0x00,0xc9,0x00,0x00,0x00, -0x41,0x00,0x08,0x00,0x9e,0x00,0x00,0x00,0xcb,0x00,0x00,0x00, -0x88,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x8b,0x00,0x00,0x00, -0x50,0x00,0x00,0x00,0xca,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0xcc,0x00,0x00,0x00,0xcb,0x00,0x00,0x00, -0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0xcd,0x00,0x00,0x00, -0xcc,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xce,0x00,0x00,0x00,0xcd,0x00,0x00,0x00,0xc7,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xcf,0x00,0x00,0x00,0xce,0x00,0x00,0x00, -0xa3,0x00,0x00,0x00,0x72,0x00,0x04,0x00,0x54,0x00,0x00,0x00, -0xd0,0x00,0x00,0x00,0xcf,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0xd1,0x00,0x00,0x00,0xd0,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xd9,0x00,0x00,0x00, -0x9d,0x00,0x00,0x00,0xd8,0x00,0x00,0x00,0x41,0x00,0x08,0x00, -0x9e,0x00,0x00,0x00,0xda,0x00,0x00,0x00,0x88,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x8b,0x00,0x00,0x00,0x50,0x00,0x00,0x00, -0xd9,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0xdb,0x00,0x00,0x00,0xda,0x00,0x00,0x00,0x71,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0xdc,0x00,0x00,0x00,0xdb,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xdd,0x00,0x00,0x00, -0xdc,0x00,0x00,0x00,0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xdf,0x00,0x00,0x00,0xdd,0x00,0x00,0x00,0xde,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00,0xe6,0x00,0x00,0x00, -0x9f,0x00,0x00,0x00,0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0xe7,0x00,0x00,0x00,0xe6,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xe8,0x00,0x00,0x00,0xe7,0x00,0x00,0x00, -0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xea,0x00,0x00,0x00, -0xe8,0x00,0x00,0x00,0xe9,0x00,0x00,0x00,0xc3,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xeb,0x00,0x00,0x00,0xea,0x00,0x00,0x00, -0x25,0x00,0x00,0x00,0xc5,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xec,0x00,0x00,0x00,0xdf,0x00,0x00,0x00,0xeb,0x00,0x00,0x00, -0x72,0x00,0x04,0x00,0x54,0x00,0x00,0x00,0xed,0x00,0x00,0x00, -0xec,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0xee,0x00,0x00,0x00,0xed,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xf6,0x00,0x00,0x00,0x9d,0x00,0x00,0x00, -0xf5,0x00,0x00,0x00,0x41,0x00,0x08,0x00,0x9e,0x00,0x00,0x00, -0xf7,0x00,0x00,0x00,0x88,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0x8b,0x00,0x00,0x00,0x50,0x00,0x00,0x00,0xf6,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00,0xf8,0x00,0x00,0x00, -0xf7,0x00,0x00,0x00,0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0xf9,0x00,0x00,0x00,0xf8,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xfa,0x00,0x00,0x00,0xf9,0x00,0x00,0x00, -0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xfb,0x00,0x00,0x00, -0xfa,0x00,0x00,0x00,0xde,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0x03,0x01,0x00,0x00,0xae,0x00,0x00,0x00, -0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x04,0x01,0x00,0x00, -0x03,0x01,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x05,0x01,0x00,0x00,0x04,0x01,0x00,0x00,0xc7,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x06,0x01,0x00,0x00,0x05,0x01,0x00,0x00, -0xe9,0x00,0x00,0x00,0xc3,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x07,0x01,0x00,0x00,0x06,0x01,0x00,0x00,0x25,0x00,0x00,0x00, -0xc5,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x08,0x01,0x00,0x00, -0xfb,0x00,0x00,0x00,0x07,0x01,0x00,0x00,0x72,0x00,0x04,0x00, -0x54,0x00,0x00,0x00,0x09,0x01,0x00,0x00,0x08,0x01,0x00,0x00, -0x7c,0x00,0x04,0x00,0x4d,0x00,0x00,0x00,0x0a,0x01,0x00,0x00, -0x09,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0x13,0x01,0x00,0x00,0xda,0x00,0x00,0x00,0xc2,0x00,0x05,0x00, -0x4d,0x00,0x00,0x00,0x14,0x01,0x00,0x00,0x13,0x01,0x00,0x00, -0x2e,0x00,0x00,0x00,0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x15,0x01,0x00,0x00,0x14,0x01,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x16,0x01,0x00,0x00,0x15,0x01,0x00,0x00, -0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x17,0x01,0x00,0x00, -0x16,0x01,0x00,0x00,0xde,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0x1f,0x01,0x00,0x00,0xbc,0x00,0x00,0x00, -0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x20,0x01,0x00,0x00, -0x1f,0x01,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x21,0x01,0x00,0x00,0x20,0x01,0x00,0x00,0xc7,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x22,0x01,0x00,0x00,0x21,0x01,0x00,0x00, -0xe9,0x00,0x00,0x00,0xc3,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x23,0x01,0x00,0x00,0x22,0x01,0x00,0x00,0x25,0x00,0x00,0x00, -0xc5,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x24,0x01,0x00,0x00, -0x17,0x01,0x00,0x00,0x23,0x01,0x00,0x00,0x72,0x00,0x04,0x00, -0x54,0x00,0x00,0x00,0x25,0x01,0x00,0x00,0x24,0x01,0x00,0x00, -0x7c,0x00,0x04,0x00,0x4d,0x00,0x00,0x00,0x26,0x01,0x00,0x00, -0x25,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0x2f,0x01,0x00,0x00,0xf7,0x00,0x00,0x00,0xc2,0x00,0x05,0x00, -0x4d,0x00,0x00,0x00,0x30,0x01,0x00,0x00,0x2f,0x01,0x00,0x00, -0x2e,0x00,0x00,0x00,0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x31,0x01,0x00,0x00,0x30,0x01,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x32,0x01,0x00,0x00,0x31,0x01,0x00,0x00, -0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x33,0x01,0x00,0x00, -0x32,0x01,0x00,0x00,0xde,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0x3b,0x01,0x00,0x00,0xcb,0x00,0x00,0x00, -0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x3c,0x01,0x00,0x00, -0x3b,0x01,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x3d,0x01,0x00,0x00,0x3c,0x01,0x00,0x00,0xc7,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x3e,0x01,0x00,0x00,0x3d,0x01,0x00,0x00, -0xe9,0x00,0x00,0x00,0xc3,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x3f,0x01,0x00,0x00,0x3e,0x01,0x00,0x00,0x25,0x00,0x00,0x00, -0xc5,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x40,0x01,0x00,0x00, -0x33,0x01,0x00,0x00,0x3f,0x01,0x00,0x00,0x72,0x00,0x04,0x00, -0x54,0x00,0x00,0x00,0x41,0x01,0x00,0x00,0x40,0x01,0x00,0x00, -0x7c,0x00,0x04,0x00,0x4d,0x00,0x00,0x00,0x42,0x01,0x00,0x00, -0x41,0x01,0x00,0x00,0x41,0x00,0x08,0x00,0x9e,0x00,0x00,0x00, -0x49,0x01,0x00,0x00,0x88,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0x8b,0x00,0x00,0x00,0x47,0x01,0x00,0x00,0x46,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00,0x4a,0x01,0x00,0x00, -0x49,0x01,0x00,0x00,0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x4b,0x01,0x00,0x00,0x4a,0x01,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x4c,0x01,0x00,0x00,0x4b,0x01,0x00,0x00, -0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x4d,0x01,0x00,0x00, -0x4c,0x01,0x00,0x00,0xde,0x00,0x00,0x00,0x72,0x00,0x04,0x00, -0x54,0x00,0x00,0x00,0x4e,0x01,0x00,0x00,0x4d,0x01,0x00,0x00, -0x7c,0x00,0x04,0x00,0x4d,0x00,0x00,0x00,0x4f,0x01,0x00,0x00, -0x4e,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x55,0x01,0x00,0x00,0x46,0x00,0x00,0x00,0x50,0x00,0x00,0x00, -0x41,0x00,0x08,0x00,0x9e,0x00,0x00,0x00,0x56,0x01,0x00,0x00, -0x88,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x8b,0x00,0x00,0x00, -0x47,0x01,0x00,0x00,0x55,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0x57,0x01,0x00,0x00,0x56,0x01,0x00,0x00, -0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x58,0x01,0x00,0x00, -0x57,0x01,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x59,0x01,0x00,0x00,0x58,0x01,0x00,0x00,0xc7,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x5a,0x01,0x00,0x00,0x59,0x01,0x00,0x00, -0xde,0x00,0x00,0x00,0x72,0x00,0x04,0x00,0x54,0x00,0x00,0x00, -0x5b,0x01,0x00,0x00,0x5a,0x01,0x00,0x00,0x7c,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0x5c,0x01,0x00,0x00,0x5b,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x62,0x01,0x00,0x00, -0x46,0x00,0x00,0x00,0x5f,0x00,0x00,0x00,0x41,0x00,0x08,0x00, -0x9e,0x00,0x00,0x00,0x63,0x01,0x00,0x00,0x88,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x8b,0x00,0x00,0x00,0x47,0x01,0x00,0x00, -0x62,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0x64,0x01,0x00,0x00,0x63,0x01,0x00,0x00,0x71,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x65,0x01,0x00,0x00,0x64,0x01,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x66,0x01,0x00,0x00, -0x65,0x01,0x00,0x00,0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x67,0x01,0x00,0x00,0x66,0x01,0x00,0x00,0xde,0x00,0x00,0x00, -0x72,0x00,0x04,0x00,0x54,0x00,0x00,0x00,0x68,0x01,0x00,0x00, -0x67,0x01,0x00,0x00,0x7c,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0x69,0x01,0x00,0x00,0x68,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x70,0x01,0x00,0x00,0x46,0x00,0x00,0x00, -0x6f,0x01,0x00,0x00,0x41,0x00,0x08,0x00,0x9e,0x00,0x00,0x00, -0x71,0x01,0x00,0x00,0x88,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0x8b,0x00,0x00,0x00,0x47,0x01,0x00,0x00,0x70,0x01,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00,0x72,0x01,0x00,0x00, -0x71,0x01,0x00,0x00,0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x73,0x01,0x00,0x00,0x72,0x01,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x74,0x01,0x00,0x00,0x73,0x01,0x00,0x00, -0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x75,0x01,0x00,0x00, -0x74,0x01,0x00,0x00,0xde,0x00,0x00,0x00,0x72,0x00,0x04,0x00, -0x54,0x00,0x00,0x00,0x76,0x01,0x00,0x00,0x75,0x01,0x00,0x00, -0x7c,0x00,0x04,0x00,0x4d,0x00,0x00,0x00,0x77,0x01,0x00,0x00, -0x76,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0x7e,0x01,0x00,0x00,0x49,0x01,0x00,0x00,0xc2,0x00,0x05,0x00, -0x4d,0x00,0x00,0x00,0x7f,0x01,0x00,0x00,0x7e,0x01,0x00,0x00, -0x2e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0x87,0x01,0x00,0x00,0x56,0x01,0x00,0x00,0xc2,0x00,0x05,0x00, -0x4d,0x00,0x00,0x00,0x88,0x01,0x00,0x00,0x87,0x01,0x00,0x00, -0x2e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0x90,0x01,0x00,0x00,0x63,0x01,0x00,0x00,0xc2,0x00,0x05,0x00, -0x4d,0x00,0x00,0x00,0x91,0x01,0x00,0x00,0x90,0x01,0x00,0x00, -0x2e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0x99,0x01,0x00,0x00,0x71,0x01,0x00,0x00,0xc2,0x00,0x05,0x00, -0x4d,0x00,0x00,0x00,0x9a,0x01,0x00,0x00,0x99,0x01,0x00,0x00, -0x2e,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xa0,0x01,0x00,0x00,0x46,0x00,0x00,0x00,0x48,0x00,0x00,0x00, -0x41,0x00,0x08,0x00,0x9e,0x00,0x00,0x00,0xa1,0x01,0x00,0x00, -0x88,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x8b,0x00,0x00,0x00, -0x47,0x01,0x00,0x00,0xa0,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0xa2,0x01,0x00,0x00,0xa1,0x01,0x00,0x00, -0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0xa3,0x01,0x00,0x00, -0xa2,0x01,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xa4,0x01,0x00,0x00,0xa3,0x01,0x00,0x00,0xc7,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xa5,0x01,0x00,0x00,0xa4,0x01,0x00,0x00, -0xde,0x00,0x00,0x00,0x72,0x00,0x04,0x00,0x54,0x00,0x00,0x00, -0xa6,0x01,0x00,0x00,0xa5,0x01,0x00,0x00,0x7c,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0xa7,0x01,0x00,0x00,0xa6,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xae,0x01,0x00,0x00, -0x46,0x00,0x00,0x00,0xad,0x01,0x00,0x00,0x41,0x00,0x08,0x00, -0x9e,0x00,0x00,0x00,0xaf,0x01,0x00,0x00,0x88,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x8b,0x00,0x00,0x00,0x47,0x01,0x00,0x00, -0xae,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0xb0,0x01,0x00,0x00,0xaf,0x01,0x00,0x00,0x71,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0xb1,0x01,0x00,0x00,0xb0,0x01,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xb2,0x01,0x00,0x00, -0xb1,0x01,0x00,0x00,0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xb3,0x01,0x00,0x00,0xb2,0x01,0x00,0x00,0xde,0x00,0x00,0x00, -0x72,0x00,0x04,0x00,0x54,0x00,0x00,0x00,0xb4,0x01,0x00,0x00, -0xb3,0x01,0x00,0x00,0x7c,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0xb5,0x01,0x00,0x00,0xb4,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xbc,0x01,0x00,0x00,0x46,0x00,0x00,0x00, -0xbb,0x01,0x00,0x00,0x41,0x00,0x08,0x00,0x9e,0x00,0x00,0x00, -0xbd,0x01,0x00,0x00,0x88,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0x8b,0x00,0x00,0x00,0x47,0x01,0x00,0x00,0xbc,0x01,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00,0xbe,0x01,0x00,0x00, -0xbd,0x01,0x00,0x00,0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0xbf,0x01,0x00,0x00,0xbe,0x01,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xc0,0x01,0x00,0x00,0xbf,0x01,0x00,0x00, -0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xc1,0x01,0x00,0x00, -0xc0,0x01,0x00,0x00,0xde,0x00,0x00,0x00,0x72,0x00,0x04,0x00, -0x54,0x00,0x00,0x00,0xc2,0x01,0x00,0x00,0xc1,0x01,0x00,0x00, -0x7c,0x00,0x04,0x00,0x4d,0x00,0x00,0x00,0xc3,0x01,0x00,0x00, -0xc2,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xca,0x01,0x00,0x00,0x46,0x00,0x00,0x00,0xc9,0x01,0x00,0x00, -0x41,0x00,0x08,0x00,0x9e,0x00,0x00,0x00,0xcb,0x01,0x00,0x00, -0x88,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x8b,0x00,0x00,0x00, -0x47,0x01,0x00,0x00,0xca,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0xcc,0x01,0x00,0x00,0xcb,0x01,0x00,0x00, -0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0xcd,0x01,0x00,0x00, -0xcc,0x01,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xce,0x01,0x00,0x00,0xcd,0x01,0x00,0x00,0xc7,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xcf,0x01,0x00,0x00,0xce,0x01,0x00,0x00, -0xde,0x00,0x00,0x00,0x72,0x00,0x04,0x00,0x54,0x00,0x00,0x00, -0xd0,0x01,0x00,0x00,0xcf,0x01,0x00,0x00,0x7c,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0xd1,0x01,0x00,0x00,0xd0,0x01,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00,0xd9,0x01,0x00,0x00, -0xa1,0x01,0x00,0x00,0xc2,0x00,0x05,0x00,0x4d,0x00,0x00,0x00, -0xda,0x01,0x00,0x00,0xd9,0x01,0x00,0x00,0x2e,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00,0xe2,0x01,0x00,0x00, -0xaf,0x01,0x00,0x00,0xc2,0x00,0x05,0x00,0x4d,0x00,0x00,0x00, -0xe3,0x01,0x00,0x00,0xe2,0x01,0x00,0x00,0x2e,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00,0xeb,0x01,0x00,0x00, -0xbd,0x01,0x00,0x00,0xc2,0x00,0x05,0x00,0x4d,0x00,0x00,0x00, -0xec,0x01,0x00,0x00,0xeb,0x01,0x00,0x00,0x2e,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00,0xf4,0x01,0x00,0x00, -0xcb,0x01,0x00,0x00,0xc2,0x00,0x05,0x00,0x4d,0x00,0x00,0x00, -0xf5,0x01,0x00,0x00,0xf4,0x01,0x00,0x00,0x2e,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00,0xfb,0x01,0x00,0x00, -0x15,0x00,0x00,0x00,0x50,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xfc,0x01,0x00,0x00,0xfb,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xfe,0x01,0x00,0x00, -0xfc,0x01,0x00,0x00,0x76,0x00,0x00,0x00,0x41,0x00,0x06,0x00, -0xff,0x01,0x00,0x00,0x00,0x02,0x00,0x00,0xfa,0x01,0x00,0x00, -0x16,0x00,0x00,0x00,0xfe,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, -0x5a,0x00,0x00,0x00,0x01,0x02,0x00,0x00,0x00,0x02,0x00,0x00, -0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x03,0x02,0x00,0x00, -0x4f,0x01,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x04,0x02,0x00,0x00,0x03,0x02,0x00,0x00,0x41,0x00,0x08,0x00, -0x9e,0x00,0x00,0x00,0x09,0x02,0x00,0x00,0x88,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x8b,0x00,0x00,0x00,0x25,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0x0a,0x02,0x00,0x00,0x09,0x02,0x00,0x00,0xc7,0x00,0x05,0x00, -0x4d,0x00,0x00,0x00,0x0c,0x02,0x00,0x00,0x0a,0x02,0x00,0x00, -0x56,0x00,0x00,0x00,0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x0d,0x02,0x00,0x00,0x0c,0x02,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x0e,0x02,0x00,0x00,0x0d,0x02,0x00,0x00, -0xab,0x00,0x05,0x00,0x70,0x00,0x00,0x00,0x0f,0x02,0x00,0x00, -0x0e,0x02,0x00,0x00,0x16,0x00,0x00,0x00,0xa9,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x10,0x02,0x00,0x00,0x0f,0x02,0x00,0x00, -0x5f,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x11,0x02,0x00,0x00,0x04,0x02,0x00,0x00, -0x10,0x02,0x00,0x00,0x6f,0x00,0x04,0x00,0x5a,0x00,0x00,0x00, -0x12,0x02,0x00,0x00,0x11,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x18,0x02,0x00,0x00,0xfe,0x01,0x00,0x00, -0x50,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0xff,0x01,0x00,0x00, -0x19,0x02,0x00,0x00,0xfa,0x01,0x00,0x00,0x16,0x00,0x00,0x00, -0x18,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0x5a,0x00,0x00,0x00, -0x1a,0x02,0x00,0x00,0x19,0x02,0x00,0x00,0x71,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x1c,0x02,0x00,0x00,0x5c,0x01,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x1d,0x02,0x00,0x00, -0x1c,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x22,0x02,0x00,0x00,0x40,0x00,0x00,0x00,0x50,0x00,0x00,0x00, -0x41,0x00,0x08,0x00,0x9e,0x00,0x00,0x00,0x23,0x02,0x00,0x00, -0x88,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x8b,0x00,0x00,0x00, -0x25,0x00,0x00,0x00,0x22,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0x24,0x02,0x00,0x00,0x23,0x02,0x00,0x00, -0xc7,0x00,0x05,0x00,0x4d,0x00,0x00,0x00,0x26,0x02,0x00,0x00, -0x24,0x02,0x00,0x00,0x56,0x00,0x00,0x00,0x71,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x27,0x02,0x00,0x00,0x26,0x02,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x28,0x02,0x00,0x00, -0x27,0x02,0x00,0x00,0xab,0x00,0x05,0x00,0x70,0x00,0x00,0x00, -0x29,0x02,0x00,0x00,0x28,0x02,0x00,0x00,0x16,0x00,0x00,0x00, -0xa9,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x2a,0x02,0x00,0x00, -0x29,0x02,0x00,0x00,0x5f,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x2b,0x02,0x00,0x00, -0x1d,0x02,0x00,0x00,0x2a,0x02,0x00,0x00,0x6f,0x00,0x04,0x00, -0x5a,0x00,0x00,0x00,0x2c,0x02,0x00,0x00,0x2b,0x02,0x00,0x00, -0x85,0x00,0x05,0x00,0x5a,0x00,0x00,0x00,0x2d,0x02,0x00,0x00, -0x1a,0x02,0x00,0x00,0x2c,0x02,0x00,0x00,0x0c,0x00,0x08,0x00, -0x5a,0x00,0x00,0x00,0x2e,0x02,0x00,0x00,0x01,0x00,0x00,0x00, -0x32,0x00,0x00,0x00,0x01,0x02,0x00,0x00,0x12,0x02,0x00,0x00, -0x2d,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x33,0x02,0x00,0x00,0xfe,0x01,0x00,0x00,0x5f,0x00,0x00,0x00, -0x41,0x00,0x06,0x00,0xff,0x01,0x00,0x00,0x34,0x02,0x00,0x00, -0xfa,0x01,0x00,0x00,0x16,0x00,0x00,0x00,0x33,0x02,0x00,0x00, -0x3d,0x00,0x04,0x00,0x5a,0x00,0x00,0x00,0x35,0x02,0x00,0x00, -0x34,0x02,0x00,0x00,0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x37,0x02,0x00,0x00,0x69,0x01,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x38,0x02,0x00,0x00,0x37,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3d,0x02,0x00,0x00, -0x40,0x00,0x00,0x00,0x5f,0x00,0x00,0x00,0x41,0x00,0x08,0x00, -0x9e,0x00,0x00,0x00,0x3e,0x02,0x00,0x00,0x88,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x8b,0x00,0x00,0x00,0x25,0x00,0x00,0x00, -0x3d,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0x3f,0x02,0x00,0x00,0x3e,0x02,0x00,0x00,0xc7,0x00,0x05,0x00, -0x4d,0x00,0x00,0x00,0x41,0x02,0x00,0x00,0x3f,0x02,0x00,0x00, -0x56,0x00,0x00,0x00,0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x42,0x02,0x00,0x00,0x41,0x02,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x43,0x02,0x00,0x00,0x42,0x02,0x00,0x00, -0xab,0x00,0x05,0x00,0x70,0x00,0x00,0x00,0x44,0x02,0x00,0x00, -0x43,0x02,0x00,0x00,0x16,0x00,0x00,0x00,0xa9,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x45,0x02,0x00,0x00,0x44,0x02,0x00,0x00, -0x5f,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x46,0x02,0x00,0x00,0x38,0x02,0x00,0x00, -0x45,0x02,0x00,0x00,0x6f,0x00,0x04,0x00,0x5a,0x00,0x00,0x00, -0x47,0x02,0x00,0x00,0x46,0x02,0x00,0x00,0x0c,0x00,0x08,0x00, -0x5a,0x00,0x00,0x00,0x49,0x02,0x00,0x00,0x01,0x00,0x00,0x00, -0x32,0x00,0x00,0x00,0x35,0x02,0x00,0x00,0x47,0x02,0x00,0x00, -0x2e,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x4e,0x02,0x00,0x00,0xfe,0x01,0x00,0x00,0x6f,0x01,0x00,0x00, -0x41,0x00,0x06,0x00,0xff,0x01,0x00,0x00,0x4f,0x02,0x00,0x00, -0xfa,0x01,0x00,0x00,0x16,0x00,0x00,0x00,0x4e,0x02,0x00,0x00, -0x3d,0x00,0x04,0x00,0x5a,0x00,0x00,0x00,0x50,0x02,0x00,0x00, -0x4f,0x02,0x00,0x00,0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x52,0x02,0x00,0x00,0x77,0x01,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x53,0x02,0x00,0x00,0x52,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x58,0x02,0x00,0x00, -0x40,0x00,0x00,0x00,0x6f,0x01,0x00,0x00,0x41,0x00,0x08,0x00, -0x9e,0x00,0x00,0x00,0x59,0x02,0x00,0x00,0x88,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x8b,0x00,0x00,0x00,0x25,0x00,0x00,0x00, -0x58,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0x5a,0x02,0x00,0x00,0x59,0x02,0x00,0x00,0xc7,0x00,0x05,0x00, -0x4d,0x00,0x00,0x00,0x5c,0x02,0x00,0x00,0x5a,0x02,0x00,0x00, -0x56,0x00,0x00,0x00,0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x5d,0x02,0x00,0x00,0x5c,0x02,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x5e,0x02,0x00,0x00,0x5d,0x02,0x00,0x00, -0xab,0x00,0x05,0x00,0x70,0x00,0x00,0x00,0x5f,0x02,0x00,0x00, -0x5e,0x02,0x00,0x00,0x16,0x00,0x00,0x00,0xa9,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x60,0x02,0x00,0x00,0x5f,0x02,0x00,0x00, -0x5f,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x61,0x02,0x00,0x00,0x53,0x02,0x00,0x00, -0x60,0x02,0x00,0x00,0x6f,0x00,0x04,0x00,0x5a,0x00,0x00,0x00, -0x62,0x02,0x00,0x00,0x61,0x02,0x00,0x00,0x0c,0x00,0x08,0x00, -0x5a,0x00,0x00,0x00,0x64,0x02,0x00,0x00,0x01,0x00,0x00,0x00, -0x32,0x00,0x00,0x00,0x50,0x02,0x00,0x00,0x62,0x02,0x00,0x00, -0x49,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x6a,0x02,0x00,0x00,0xfe,0x01,0x00,0x00,0x42,0x00,0x00,0x00, -0x41,0x00,0x06,0x00,0xff,0x01,0x00,0x00,0x6b,0x02,0x00,0x00, -0xfa,0x01,0x00,0x00,0x16,0x00,0x00,0x00,0x6a,0x02,0x00,0x00, -0x3d,0x00,0x04,0x00,0x5a,0x00,0x00,0x00,0x6c,0x02,0x00,0x00, -0x6b,0x02,0x00,0x00,0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x6e,0x02,0x00,0x00,0x7f,0x01,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x6f,0x02,0x00,0x00,0x6e,0x02,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00,0x75,0x02,0x00,0x00, -0x09,0x02,0x00,0x00,0xc4,0x00,0x05,0x00,0x4d,0x00,0x00,0x00, -0x77,0x02,0x00,0x00,0x56,0x00,0x00,0x00,0x50,0x00,0x00,0x00, -0xc7,0x00,0x05,0x00,0x4d,0x00,0x00,0x00,0x78,0x02,0x00,0x00, -0x75,0x02,0x00,0x00,0x77,0x02,0x00,0x00,0x71,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x79,0x02,0x00,0x00,0x78,0x02,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x7a,0x02,0x00,0x00, -0x79,0x02,0x00,0x00,0xab,0x00,0x05,0x00,0x70,0x00,0x00,0x00, -0x7b,0x02,0x00,0x00,0x7a,0x02,0x00,0x00,0x16,0x00,0x00,0x00, -0xa9,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x7c,0x02,0x00,0x00, -0x7b,0x02,0x00,0x00,0x5f,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x7d,0x02,0x00,0x00, -0x6f,0x02,0x00,0x00,0x7c,0x02,0x00,0x00,0x6f,0x00,0x04,0x00, -0x5a,0x00,0x00,0x00,0x7e,0x02,0x00,0x00,0x7d,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x85,0x02,0x00,0x00, -0xfe,0x01,0x00,0x00,0x84,0x02,0x00,0x00,0x41,0x00,0x06,0x00, -0xff,0x01,0x00,0x00,0x86,0x02,0x00,0x00,0xfa,0x01,0x00,0x00, -0x16,0x00,0x00,0x00,0x85,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, -0x5a,0x00,0x00,0x00,0x87,0x02,0x00,0x00,0x86,0x02,0x00,0x00, -0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x89,0x02,0x00,0x00, -0x88,0x01,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x8a,0x02,0x00,0x00,0x89,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0x91,0x02,0x00,0x00,0x23,0x02,0x00,0x00, -0xc7,0x00,0x05,0x00,0x4d,0x00,0x00,0x00,0x94,0x02,0x00,0x00, -0x91,0x02,0x00,0x00,0x77,0x02,0x00,0x00,0x71,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x95,0x02,0x00,0x00,0x94,0x02,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x96,0x02,0x00,0x00, -0x95,0x02,0x00,0x00,0xab,0x00,0x05,0x00,0x70,0x00,0x00,0x00, -0x97,0x02,0x00,0x00,0x96,0x02,0x00,0x00,0x16,0x00,0x00,0x00, -0xa9,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x98,0x02,0x00,0x00, -0x97,0x02,0x00,0x00,0x5f,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x99,0x02,0x00,0x00, -0x8a,0x02,0x00,0x00,0x98,0x02,0x00,0x00,0x6f,0x00,0x04,0x00, -0x5a,0x00,0x00,0x00,0x9a,0x02,0x00,0x00,0x99,0x02,0x00,0x00, -0x85,0x00,0x05,0x00,0x5a,0x00,0x00,0x00,0x9b,0x02,0x00,0x00, -0x87,0x02,0x00,0x00,0x9a,0x02,0x00,0x00,0x0c,0x00,0x08,0x00, -0x5a,0x00,0x00,0x00,0x9c,0x02,0x00,0x00,0x01,0x00,0x00,0x00, -0x32,0x00,0x00,0x00,0x6c,0x02,0x00,0x00,0x7e,0x02,0x00,0x00, -0x9b,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xa2,0x02,0x00,0x00,0xfe,0x01,0x00,0x00,0xa1,0x02,0x00,0x00, -0x41,0x00,0x06,0x00,0xff,0x01,0x00,0x00,0xa3,0x02,0x00,0x00, -0xfa,0x01,0x00,0x00,0x16,0x00,0x00,0x00,0xa2,0x02,0x00,0x00, -0x3d,0x00,0x04,0x00,0x5a,0x00,0x00,0x00,0xa4,0x02,0x00,0x00, -0xa3,0x02,0x00,0x00,0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0xa6,0x02,0x00,0x00,0x91,0x01,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xa7,0x02,0x00,0x00,0xa6,0x02,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00,0xae,0x02,0x00,0x00, -0x3e,0x02,0x00,0x00,0xc7,0x00,0x05,0x00,0x4d,0x00,0x00,0x00, -0xb1,0x02,0x00,0x00,0xae,0x02,0x00,0x00,0x77,0x02,0x00,0x00, -0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0xb2,0x02,0x00,0x00, -0xb1,0x02,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xb3,0x02,0x00,0x00,0xb2,0x02,0x00,0x00,0xab,0x00,0x05,0x00, -0x70,0x00,0x00,0x00,0xb4,0x02,0x00,0x00,0xb3,0x02,0x00,0x00, -0x16,0x00,0x00,0x00,0xa9,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xb5,0x02,0x00,0x00,0xb4,0x02,0x00,0x00,0x5f,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xb6,0x02,0x00,0x00,0xa7,0x02,0x00,0x00,0xb5,0x02,0x00,0x00, -0x6f,0x00,0x04,0x00,0x5a,0x00,0x00,0x00,0xb7,0x02,0x00,0x00, -0xb6,0x02,0x00,0x00,0x0c,0x00,0x08,0x00,0x5a,0x00,0x00,0x00, -0xb9,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00, -0xa4,0x02,0x00,0x00,0xb7,0x02,0x00,0x00,0x9c,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xbf,0x02,0x00,0x00, -0xfe,0x01,0x00,0x00,0xbe,0x02,0x00,0x00,0x41,0x00,0x06,0x00, -0xff,0x01,0x00,0x00,0xc0,0x02,0x00,0x00,0xfa,0x01,0x00,0x00, -0x16,0x00,0x00,0x00,0xbf,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, -0x5a,0x00,0x00,0x00,0xc1,0x02,0x00,0x00,0xc0,0x02,0x00,0x00, -0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0xc3,0x02,0x00,0x00, -0x9a,0x01,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xc4,0x02,0x00,0x00,0xc3,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0xcb,0x02,0x00,0x00,0x59,0x02,0x00,0x00, -0xc7,0x00,0x05,0x00,0x4d,0x00,0x00,0x00,0xce,0x02,0x00,0x00, -0xcb,0x02,0x00,0x00,0x77,0x02,0x00,0x00,0x71,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0xcf,0x02,0x00,0x00,0xce,0x02,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xd0,0x02,0x00,0x00, -0xcf,0x02,0x00,0x00,0xab,0x00,0x05,0x00,0x70,0x00,0x00,0x00, -0xd1,0x02,0x00,0x00,0xd0,0x02,0x00,0x00,0x16,0x00,0x00,0x00, -0xa9,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xd2,0x02,0x00,0x00, -0xd1,0x02,0x00,0x00,0x5f,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xd3,0x02,0x00,0x00, -0xc4,0x02,0x00,0x00,0xd2,0x02,0x00,0x00,0x6f,0x00,0x04,0x00, -0x5a,0x00,0x00,0x00,0xd4,0x02,0x00,0x00,0xd3,0x02,0x00,0x00, -0x0c,0x00,0x08,0x00,0x5a,0x00,0x00,0x00,0xd6,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0xc1,0x02,0x00,0x00, -0xd4,0x02,0x00,0x00,0xb9,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xdb,0x02,0x00,0x00,0xfc,0x01,0x00,0x00, -0x7a,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0xff,0x01,0x00,0x00, -0xdc,0x02,0x00,0x00,0xfa,0x01,0x00,0x00,0x16,0x00,0x00,0x00, -0xdb,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0x5a,0x00,0x00,0x00, -0xdd,0x02,0x00,0x00,0xdc,0x02,0x00,0x00,0x71,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0xdf,0x02,0x00,0x00,0xa7,0x01,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xe0,0x02,0x00,0x00, -0xdf,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0xe6,0x02,0x00,0x00,0x09,0x02,0x00,0x00,0xc7,0x00,0x05,0x00, -0x4d,0x00,0x00,0x00,0xe8,0x02,0x00,0x00,0xe6,0x02,0x00,0x00, -0x59,0x00,0x00,0x00,0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0xe9,0x02,0x00,0x00,0xe8,0x02,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xea,0x02,0x00,0x00,0xe9,0x02,0x00,0x00, -0xab,0x00,0x05,0x00,0x70,0x00,0x00,0x00,0xeb,0x02,0x00,0x00, -0xea,0x02,0x00,0x00,0x16,0x00,0x00,0x00,0xa9,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xec,0x02,0x00,0x00,0xeb,0x02,0x00,0x00, -0x5f,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xed,0x02,0x00,0x00,0xe0,0x02,0x00,0x00, -0xec,0x02,0x00,0x00,0x6f,0x00,0x04,0x00,0x5a,0x00,0x00,0x00, -0xee,0x02,0x00,0x00,0xed,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xf4,0x02,0x00,0x00,0xdb,0x02,0x00,0x00, -0x50,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0xff,0x01,0x00,0x00, -0xf5,0x02,0x00,0x00,0xfa,0x01,0x00,0x00,0x16,0x00,0x00,0x00, -0xf4,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0x5a,0x00,0x00,0x00, -0xf6,0x02,0x00,0x00,0xf5,0x02,0x00,0x00,0x71,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0xf8,0x02,0x00,0x00,0xb5,0x01,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xf9,0x02,0x00,0x00, -0xf8,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0x00,0x03,0x00,0x00,0x23,0x02,0x00,0x00,0xc7,0x00,0x05,0x00, -0x4d,0x00,0x00,0x00,0x02,0x03,0x00,0x00,0x00,0x03,0x00,0x00, -0x59,0x00,0x00,0x00,0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x03,0x03,0x00,0x00,0x02,0x03,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x04,0x03,0x00,0x00,0x03,0x03,0x00,0x00, -0xab,0x00,0x05,0x00,0x70,0x00,0x00,0x00,0x05,0x03,0x00,0x00, -0x04,0x03,0x00,0x00,0x16,0x00,0x00,0x00,0xa9,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x06,0x03,0x00,0x00,0x05,0x03,0x00,0x00, -0x5f,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x07,0x03,0x00,0x00,0xf9,0x02,0x00,0x00, -0x06,0x03,0x00,0x00,0x6f,0x00,0x04,0x00,0x5a,0x00,0x00,0x00, -0x08,0x03,0x00,0x00,0x07,0x03,0x00,0x00,0x85,0x00,0x05,0x00, -0x5a,0x00,0x00,0x00,0x09,0x03,0x00,0x00,0xf6,0x02,0x00,0x00, -0x08,0x03,0x00,0x00,0x0c,0x00,0x08,0x00,0x5a,0x00,0x00,0x00, -0x0a,0x03,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00, -0xdd,0x02,0x00,0x00,0xee,0x02,0x00,0x00,0x09,0x03,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x0f,0x03,0x00,0x00, -0xdb,0x02,0x00,0x00,0x5f,0x00,0x00,0x00,0x41,0x00,0x06,0x00, -0xff,0x01,0x00,0x00,0x10,0x03,0x00,0x00,0xfa,0x01,0x00,0x00, -0x16,0x00,0x00,0x00,0x0f,0x03,0x00,0x00,0x3d,0x00,0x04,0x00, -0x5a,0x00,0x00,0x00,0x11,0x03,0x00,0x00,0x10,0x03,0x00,0x00, -0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x13,0x03,0x00,0x00, -0xc3,0x01,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x14,0x03,0x00,0x00,0x13,0x03,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0x1b,0x03,0x00,0x00,0x3e,0x02,0x00,0x00, -0xc7,0x00,0x05,0x00,0x4d,0x00,0x00,0x00,0x1d,0x03,0x00,0x00, -0x1b,0x03,0x00,0x00,0x59,0x00,0x00,0x00,0x71,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x1e,0x03,0x00,0x00,0x1d,0x03,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x1f,0x03,0x00,0x00, -0x1e,0x03,0x00,0x00,0xab,0x00,0x05,0x00,0x70,0x00,0x00,0x00, -0x20,0x03,0x00,0x00,0x1f,0x03,0x00,0x00,0x16,0x00,0x00,0x00, -0xa9,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x21,0x03,0x00,0x00, -0x20,0x03,0x00,0x00,0x5f,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x22,0x03,0x00,0x00, -0x14,0x03,0x00,0x00,0x21,0x03,0x00,0x00,0x6f,0x00,0x04,0x00, -0x5a,0x00,0x00,0x00,0x23,0x03,0x00,0x00,0x22,0x03,0x00,0x00, -0x0c,0x00,0x08,0x00,0x5a,0x00,0x00,0x00,0x25,0x03,0x00,0x00, -0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x11,0x03,0x00,0x00, -0x23,0x03,0x00,0x00,0x0a,0x03,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x2a,0x03,0x00,0x00,0xdb,0x02,0x00,0x00, -0x6f,0x01,0x00,0x00,0x41,0x00,0x06,0x00,0xff,0x01,0x00,0x00, -0x2b,0x03,0x00,0x00,0xfa,0x01,0x00,0x00,0x16,0x00,0x00,0x00, -0x2a,0x03,0x00,0x00,0x3d,0x00,0x04,0x00,0x5a,0x00,0x00,0x00, -0x2c,0x03,0x00,0x00,0x2b,0x03,0x00,0x00,0x71,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x2e,0x03,0x00,0x00,0xd1,0x01,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x2f,0x03,0x00,0x00, -0x2e,0x03,0x00,0x00,0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0x36,0x03,0x00,0x00,0x59,0x02,0x00,0x00,0xc7,0x00,0x05,0x00, -0x4d,0x00,0x00,0x00,0x38,0x03,0x00,0x00,0x36,0x03,0x00,0x00, -0x59,0x00,0x00,0x00,0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x39,0x03,0x00,0x00,0x38,0x03,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x3a,0x03,0x00,0x00,0x39,0x03,0x00,0x00, -0xab,0x00,0x05,0x00,0x70,0x00,0x00,0x00,0x3b,0x03,0x00,0x00, -0x3a,0x03,0x00,0x00,0x16,0x00,0x00,0x00,0xa9,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x3c,0x03,0x00,0x00,0x3b,0x03,0x00,0x00, -0x5f,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x3d,0x03,0x00,0x00,0x2f,0x03,0x00,0x00, -0x3c,0x03,0x00,0x00,0x6f,0x00,0x04,0x00,0x5a,0x00,0x00,0x00, -0x3e,0x03,0x00,0x00,0x3d,0x03,0x00,0x00,0x0c,0x00,0x08,0x00, -0x5a,0x00,0x00,0x00,0x40,0x03,0x00,0x00,0x01,0x00,0x00,0x00, -0x32,0x00,0x00,0x00,0x2c,0x03,0x00,0x00,0x3e,0x03,0x00,0x00, -0x25,0x03,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x46,0x03,0x00,0x00,0xdb,0x02,0x00,0x00,0x42,0x00,0x00,0x00, -0x41,0x00,0x06,0x00,0xff,0x01,0x00,0x00,0x47,0x03,0x00,0x00, -0xfa,0x01,0x00,0x00,0x16,0x00,0x00,0x00,0x46,0x03,0x00,0x00, -0x3d,0x00,0x04,0x00,0x5a,0x00,0x00,0x00,0x48,0x03,0x00,0x00, -0x47,0x03,0x00,0x00,0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x4a,0x03,0x00,0x00,0xda,0x01,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x4b,0x03,0x00,0x00,0x4a,0x03,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00,0x51,0x03,0x00,0x00, -0x09,0x02,0x00,0x00,0xc4,0x00,0x05,0x00,0x4d,0x00,0x00,0x00, -0x53,0x03,0x00,0x00,0x59,0x00,0x00,0x00,0x50,0x00,0x00,0x00, -0xc7,0x00,0x05,0x00,0x4d,0x00,0x00,0x00,0x54,0x03,0x00,0x00, -0x51,0x03,0x00,0x00,0x53,0x03,0x00,0x00,0x71,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x55,0x03,0x00,0x00,0x54,0x03,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x56,0x03,0x00,0x00, -0x55,0x03,0x00,0x00,0xab,0x00,0x05,0x00,0x70,0x00,0x00,0x00, -0x57,0x03,0x00,0x00,0x56,0x03,0x00,0x00,0x16,0x00,0x00,0x00, -0xa9,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x58,0x03,0x00,0x00, -0x57,0x03,0x00,0x00,0x5f,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x59,0x03,0x00,0x00, -0x4b,0x03,0x00,0x00,0x58,0x03,0x00,0x00,0x6f,0x00,0x04,0x00, -0x5a,0x00,0x00,0x00,0x5a,0x03,0x00,0x00,0x59,0x03,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x60,0x03,0x00,0x00, -0xdb,0x02,0x00,0x00,0x84,0x02,0x00,0x00,0x41,0x00,0x06,0x00, -0xff,0x01,0x00,0x00,0x61,0x03,0x00,0x00,0xfa,0x01,0x00,0x00, -0x16,0x00,0x00,0x00,0x60,0x03,0x00,0x00,0x3d,0x00,0x04,0x00, -0x5a,0x00,0x00,0x00,0x62,0x03,0x00,0x00,0x61,0x03,0x00,0x00, -0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x64,0x03,0x00,0x00, -0xe3,0x01,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x65,0x03,0x00,0x00,0x64,0x03,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0x6c,0x03,0x00,0x00,0x23,0x02,0x00,0x00, -0xc7,0x00,0x05,0x00,0x4d,0x00,0x00,0x00,0x6f,0x03,0x00,0x00, -0x6c,0x03,0x00,0x00,0x53,0x03,0x00,0x00,0x71,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x70,0x03,0x00,0x00,0x6f,0x03,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x71,0x03,0x00,0x00, -0x70,0x03,0x00,0x00,0xab,0x00,0x05,0x00,0x70,0x00,0x00,0x00, -0x72,0x03,0x00,0x00,0x71,0x03,0x00,0x00,0x16,0x00,0x00,0x00, -0xa9,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x73,0x03,0x00,0x00, -0x72,0x03,0x00,0x00,0x5f,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x74,0x03,0x00,0x00, -0x65,0x03,0x00,0x00,0x73,0x03,0x00,0x00,0x6f,0x00,0x04,0x00, -0x5a,0x00,0x00,0x00,0x75,0x03,0x00,0x00,0x74,0x03,0x00,0x00, -0x85,0x00,0x05,0x00,0x5a,0x00,0x00,0x00,0x76,0x03,0x00,0x00, -0x62,0x03,0x00,0x00,0x75,0x03,0x00,0x00,0x0c,0x00,0x08,0x00, -0x5a,0x00,0x00,0x00,0x77,0x03,0x00,0x00,0x01,0x00,0x00,0x00, -0x32,0x00,0x00,0x00,0x48,0x03,0x00,0x00,0x5a,0x03,0x00,0x00, -0x76,0x03,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x7c,0x03,0x00,0x00,0xdb,0x02,0x00,0x00,0xa1,0x02,0x00,0x00, -0x41,0x00,0x06,0x00,0xff,0x01,0x00,0x00,0x7d,0x03,0x00,0x00, -0xfa,0x01,0x00,0x00,0x16,0x00,0x00,0x00,0x7c,0x03,0x00,0x00, -0x3d,0x00,0x04,0x00,0x5a,0x00,0x00,0x00,0x7e,0x03,0x00,0x00, -0x7d,0x03,0x00,0x00,0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x80,0x03,0x00,0x00,0xec,0x01,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x81,0x03,0x00,0x00,0x80,0x03,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00,0x88,0x03,0x00,0x00, -0x3e,0x02,0x00,0x00,0xc7,0x00,0x05,0x00,0x4d,0x00,0x00,0x00, -0x8b,0x03,0x00,0x00,0x88,0x03,0x00,0x00,0x53,0x03,0x00,0x00, -0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x8c,0x03,0x00,0x00, -0x8b,0x03,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x8d,0x03,0x00,0x00,0x8c,0x03,0x00,0x00,0xab,0x00,0x05,0x00, -0x70,0x00,0x00,0x00,0x8e,0x03,0x00,0x00,0x8d,0x03,0x00,0x00, -0x16,0x00,0x00,0x00,0xa9,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x8f,0x03,0x00,0x00,0x8e,0x03,0x00,0x00,0x5f,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x90,0x03,0x00,0x00,0x81,0x03,0x00,0x00,0x8f,0x03,0x00,0x00, -0x6f,0x00,0x04,0x00,0x5a,0x00,0x00,0x00,0x91,0x03,0x00,0x00, -0x90,0x03,0x00,0x00,0x0c,0x00,0x08,0x00,0x5a,0x00,0x00,0x00, -0x93,0x03,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00, -0x7e,0x03,0x00,0x00,0x91,0x03,0x00,0x00,0x77,0x03,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x98,0x03,0x00,0x00, -0xdb,0x02,0x00,0x00,0xbe,0x02,0x00,0x00,0x41,0x00,0x06,0x00, -0xff,0x01,0x00,0x00,0x99,0x03,0x00,0x00,0xfa,0x01,0x00,0x00, -0x16,0x00,0x00,0x00,0x98,0x03,0x00,0x00,0x3d,0x00,0x04,0x00, -0x5a,0x00,0x00,0x00,0x9a,0x03,0x00,0x00,0x99,0x03,0x00,0x00, -0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x9c,0x03,0x00,0x00, -0xf5,0x01,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x9d,0x03,0x00,0x00,0x9c,0x03,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0xa4,0x03,0x00,0x00,0x59,0x02,0x00,0x00, -0xc7,0x00,0x05,0x00,0x4d,0x00,0x00,0x00,0xa7,0x03,0x00,0x00, -0xa4,0x03,0x00,0x00,0x53,0x03,0x00,0x00,0x71,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0xa8,0x03,0x00,0x00,0xa7,0x03,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xa9,0x03,0x00,0x00, -0xa8,0x03,0x00,0x00,0xab,0x00,0x05,0x00,0x70,0x00,0x00,0x00, -0xaa,0x03,0x00,0x00,0xa9,0x03,0x00,0x00,0x16,0x00,0x00,0x00, -0xa9,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xab,0x03,0x00,0x00, -0xaa,0x03,0x00,0x00,0x5f,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xac,0x03,0x00,0x00, -0x9d,0x03,0x00,0x00,0xab,0x03,0x00,0x00,0x6f,0x00,0x04,0x00, -0x5a,0x00,0x00,0x00,0xad,0x03,0x00,0x00,0xac,0x03,0x00,0x00, -0x0c,0x00,0x08,0x00,0x5a,0x00,0x00,0x00,0xaf,0x03,0x00,0x00, -0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x9a,0x03,0x00,0x00, -0xad,0x03,0x00,0x00,0x93,0x03,0x00,0x00,0x3d,0x00,0x04,0x00, -0x5a,0x00,0x00,0x00,0xb6,0x03,0x00,0x00,0x00,0x02,0x00,0x00, -0x3d,0x00,0x04,0x00,0x5a,0x00,0x00,0x00,0xbd,0x03,0x00,0x00, -0x19,0x02,0x00,0x00,0x81,0x00,0x05,0x00,0x5a,0x00,0x00,0x00, -0xbe,0x03,0x00,0x00,0xb6,0x03,0x00,0x00,0xbd,0x03,0x00,0x00, -0x3d,0x00,0x04,0x00,0x5a,0x00,0x00,0x00,0xc5,0x03,0x00,0x00, -0x34,0x02,0x00,0x00,0x81,0x00,0x05,0x00,0x5a,0x00,0x00,0x00, -0xc6,0x03,0x00,0x00,0xbe,0x03,0x00,0x00,0xc5,0x03,0x00,0x00, -0x3d,0x00,0x04,0x00,0x5a,0x00,0x00,0x00,0xcd,0x03,0x00,0x00, -0x4f,0x02,0x00,0x00,0x81,0x00,0x05,0x00,0x5a,0x00,0x00,0x00, -0xce,0x03,0x00,0x00,0xc6,0x03,0x00,0x00,0xcd,0x03,0x00,0x00, -0x70,0x00,0x04,0x00,0x5a,0x00,0x00,0x00,0xd0,0x03,0x00,0x00, -0xc2,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x5a,0x00,0x00,0x00, -0xd8,0x03,0x00,0x00,0x6b,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, -0x5a,0x00,0x00,0x00,0xdf,0x03,0x00,0x00,0x86,0x02,0x00,0x00, -0x81,0x00,0x05,0x00,0x5a,0x00,0x00,0x00,0xe0,0x03,0x00,0x00, -0xd8,0x03,0x00,0x00,0xdf,0x03,0x00,0x00,0x3d,0x00,0x04,0x00, -0x5a,0x00,0x00,0x00,0xe7,0x03,0x00,0x00,0xa3,0x02,0x00,0x00, -0x81,0x00,0x05,0x00,0x5a,0x00,0x00,0x00,0xe8,0x03,0x00,0x00, -0xe0,0x03,0x00,0x00,0xe7,0x03,0x00,0x00,0x3d,0x00,0x04,0x00, -0x5a,0x00,0x00,0x00,0xef,0x03,0x00,0x00,0xc0,0x02,0x00,0x00, -0x81,0x00,0x05,0x00,0x5a,0x00,0x00,0x00,0xf0,0x03,0x00,0x00, -0xe8,0x03,0x00,0x00,0xef,0x03,0x00,0x00,0x70,0x00,0x04,0x00, -0x5a,0x00,0x00,0x00,0xf2,0x03,0x00,0x00,0xd1,0x00,0x00,0x00, -0x85,0x00,0x05,0x00,0x5a,0x00,0x00,0x00,0xf3,0x03,0x00,0x00, -0xf0,0x03,0x00,0x00,0xf2,0x03,0x00,0x00,0x0c,0x00,0x08,0x00, -0x5a,0x00,0x00,0x00,0xf4,0x03,0x00,0x00,0x01,0x00,0x00,0x00, -0x32,0x00,0x00,0x00,0xce,0x03,0x00,0x00,0xd0,0x03,0x00,0x00, -0xf3,0x03,0x00,0x00,0x3d,0x00,0x04,0x00,0x5a,0x00,0x00,0x00, -0xfa,0x03,0x00,0x00,0xdc,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, -0x5a,0x00,0x00,0x00,0x01,0x04,0x00,0x00,0xf5,0x02,0x00,0x00, -0x81,0x00,0x05,0x00,0x5a,0x00,0x00,0x00,0x02,0x04,0x00,0x00, -0xfa,0x03,0x00,0x00,0x01,0x04,0x00,0x00,0x3d,0x00,0x04,0x00, -0x5a,0x00,0x00,0x00,0x09,0x04,0x00,0x00,0x10,0x03,0x00,0x00, -0x81,0x00,0x05,0x00,0x5a,0x00,0x00,0x00,0x0a,0x04,0x00,0x00, -0x02,0x04,0x00,0x00,0x09,0x04,0x00,0x00,0x3d,0x00,0x04,0x00, -0x5a,0x00,0x00,0x00,0x11,0x04,0x00,0x00,0x2b,0x03,0x00,0x00, -0x81,0x00,0x05,0x00,0x5a,0x00,0x00,0x00,0x12,0x04,0x00,0x00, -0x0a,0x04,0x00,0x00,0x11,0x04,0x00,0x00,0x70,0x00,0x04,0x00, -0x5a,0x00,0x00,0x00,0x14,0x04,0x00,0x00,0x26,0x01,0x00,0x00, -0x0c,0x00,0x08,0x00,0x5a,0x00,0x00,0x00,0x16,0x04,0x00,0x00, -0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x12,0x04,0x00,0x00, -0x14,0x04,0x00,0x00,0xf4,0x03,0x00,0x00,0x3d,0x00,0x04,0x00, -0x5a,0x00,0x00,0x00,0x1d,0x04,0x00,0x00,0x47,0x03,0x00,0x00, -0x3d,0x00,0x04,0x00,0x5a,0x00,0x00,0x00,0x24,0x04,0x00,0x00, -0x61,0x03,0x00,0x00,0x81,0x00,0x05,0x00,0x5a,0x00,0x00,0x00, -0x25,0x04,0x00,0x00,0x1d,0x04,0x00,0x00,0x24,0x04,0x00,0x00, -0x3d,0x00,0x04,0x00,0x5a,0x00,0x00,0x00,0x2c,0x04,0x00,0x00, -0x7d,0x03,0x00,0x00,0x81,0x00,0x05,0x00,0x5a,0x00,0x00,0x00, -0x2d,0x04,0x00,0x00,0x25,0x04,0x00,0x00,0x2c,0x04,0x00,0x00, -0x3d,0x00,0x04,0x00,0x5a,0x00,0x00,0x00,0x34,0x04,0x00,0x00, -0x99,0x03,0x00,0x00,0x81,0x00,0x05,0x00,0x5a,0x00,0x00,0x00, -0x35,0x04,0x00,0x00,0x2d,0x04,0x00,0x00,0x34,0x04,0x00,0x00, -0x70,0x00,0x04,0x00,0x5a,0x00,0x00,0x00,0x37,0x04,0x00,0x00, -0x42,0x01,0x00,0x00,0x0c,0x00,0x08,0x00,0x5a,0x00,0x00,0x00, -0x39,0x04,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00, -0x35,0x04,0x00,0x00,0x37,0x04,0x00,0x00,0x16,0x04,0x00,0x00, -0x70,0x00,0x04,0x00,0x5a,0x00,0x00,0x00,0x41,0x04,0x00,0x00, -0xa6,0x00,0x00,0x00,0x70,0x00,0x04,0x00,0x5a,0x00,0x00,0x00, -0x45,0x04,0x00,0x00,0xb4,0x00,0x00,0x00,0x85,0x00,0x05,0x00, -0x5a,0x00,0x00,0x00,0x46,0x04,0x00,0x00,0xd6,0x02,0x00,0x00, -0x45,0x04,0x00,0x00,0x0c,0x00,0x08,0x00,0x5a,0x00,0x00,0x00, -0x47,0x04,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00, -0x64,0x02,0x00,0x00,0x41,0x04,0x00,0x00,0x46,0x04,0x00,0x00, -0x70,0x00,0x04,0x00,0x5a,0x00,0x00,0x00,0x4a,0x04,0x00,0x00, -0xee,0x00,0x00,0x00,0x0c,0x00,0x08,0x00,0x5a,0x00,0x00,0x00, -0x4c,0x04,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00, -0x40,0x03,0x00,0x00,0x4a,0x04,0x00,0x00,0x47,0x04,0x00,0x00, -0x70,0x00,0x04,0x00,0x5a,0x00,0x00,0x00,0x4f,0x04,0x00,0x00, -0x0a,0x01,0x00,0x00,0x0c,0x00,0x08,0x00,0x5a,0x00,0x00,0x00, -0x51,0x04,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00, -0xaf,0x03,0x00,0x00,0x4f,0x04,0x00,0x00,0x4c,0x04,0x00,0x00, -0x85,0x00,0x05,0x00,0x5a,0x00,0x00,0x00,0x55,0x04,0x00,0x00, -0x97,0x00,0x00,0x00,0x39,0x04,0x00,0x00,0x7f,0x00,0x04,0x00, -0x5a,0x00,0x00,0x00,0x8c,0x04,0x00,0x00,0x55,0x04,0x00,0x00, -0x0c,0x00,0x08,0x00,0x5a,0x00,0x00,0x00,0x56,0x04,0x00,0x00, -0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x8f,0x00,0x00,0x00, -0x51,0x04,0x00,0x00,0x8c,0x04,0x00,0x00,0x3d,0x00,0x04,0x00, -0x5a,0x00,0x00,0x00,0x58,0x04,0x00,0x00,0x66,0x00,0x00,0x00, -0x81,0x00,0x05,0x00,0x5a,0x00,0x00,0x00,0x59,0x04,0x00,0x00, -0x58,0x04,0x00,0x00,0x56,0x04,0x00,0x00,0x3e,0x00,0x03,0x00, -0x66,0x00,0x00,0x00,0x59,0x04,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x5c,0x04,0x00,0x00,0x88,0x04,0x00,0x00, -0x25,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x69,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0x6b,0x00,0x00,0x00,0xe0,0x00,0x04,0x00, -0x5d,0x04,0x00,0x00,0x5d,0x04,0x00,0x00,0x5e,0x04,0x00,0x00, -0xf9,0x00,0x02,0x00,0x60,0x04,0x00,0x00,0xf8,0x00,0x02,0x00, -0x60,0x04,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0x89,0x04,0x00,0x00,0x5f,0x00,0x00,0x00,0x6b,0x00,0x00,0x00, -0x77,0x04,0x00,0x00,0x63,0x04,0x00,0x00,0xad,0x00,0x05,0x00, -0x70,0x00,0x00,0x00,0x66,0x04,0x00,0x00,0x89,0x04,0x00,0x00, -0x16,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x62,0x04,0x00,0x00, -0x63,0x04,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x66,0x04,0x00,0x00,0x61,0x04,0x00,0x00,0x62,0x04,0x00,0x00, -0xf8,0x00,0x02,0x00,0x61,0x04,0x00,0x00,0xb1,0x00,0x05,0x00, -0x70,0x00,0x00,0x00,0x69,0x04,0x00,0x00,0x26,0x00,0x00,0x00, -0x89,0x04,0x00,0x00,0xf7,0x00,0x03,0x00,0x6b,0x04,0x00,0x00, -0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x69,0x04,0x00,0x00, -0x6a,0x04,0x00,0x00,0x6b,0x04,0x00,0x00,0xf8,0x00,0x02,0x00, -0x6a,0x04,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x6f,0x04,0x00,0x00,0x26,0x00,0x00,0x00,0x89,0x04,0x00,0x00, -0x41,0x00,0x05,0x00,0x65,0x00,0x00,0x00,0x70,0x04,0x00,0x00, -0x5e,0x00,0x00,0x00,0x6f,0x04,0x00,0x00,0x3d,0x00,0x04,0x00, -0x5a,0x00,0x00,0x00,0x71,0x04,0x00,0x00,0x70,0x04,0x00,0x00, -0x41,0x00,0x05,0x00,0x65,0x00,0x00,0x00,0x72,0x04,0x00,0x00, -0x5e,0x00,0x00,0x00,0x26,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x5a,0x00,0x00,0x00,0x73,0x04,0x00,0x00,0x72,0x04,0x00,0x00, -0x81,0x00,0x05,0x00,0x5a,0x00,0x00,0x00,0x74,0x04,0x00,0x00, -0x73,0x04,0x00,0x00,0x71,0x04,0x00,0x00,0x3e,0x00,0x03,0x00, -0x72,0x04,0x00,0x00,0x74,0x04,0x00,0x00,0xf9,0x00,0x02,0x00, -0x6b,0x04,0x00,0x00,0xf8,0x00,0x02,0x00,0x6b,0x04,0x00,0x00, -0xe0,0x00,0x04,0x00,0x5d,0x04,0x00,0x00,0x5d,0x04,0x00,0x00, -0x5e,0x04,0x00,0x00,0xf9,0x00,0x02,0x00,0x63,0x04,0x00,0x00, -0xf8,0x00,0x02,0x00,0x63,0x04,0x00,0x00,0xc3,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x77,0x04,0x00,0x00,0x89,0x04,0x00,0x00, -0x50,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x60,0x04,0x00,0x00, -0xf8,0x00,0x02,0x00,0x62,0x04,0x00,0x00,0xaa,0x00,0x05,0x00, -0x70,0x00,0x00,0x00,0x79,0x04,0x00,0x00,0x26,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0xf7,0x00,0x03,0x00,0x7b,0x04,0x00,0x00, -0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x79,0x04,0x00,0x00, -0x7a,0x04,0x00,0x00,0x7b,0x04,0x00,0x00,0xf8,0x00,0x02,0x00, -0x7a,0x04,0x00,0x00,0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00, -0x80,0x04,0x00,0x00,0x15,0x00,0x00,0x00,0x25,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x81,0x04,0x00,0x00, -0x80,0x04,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x83,0x04,0x00,0x00,0x81,0x04,0x00,0x00,0x11,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x65,0x00,0x00,0x00,0x84,0x04,0x00,0x00, -0x5e,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x5a,0x00,0x00,0x00,0x85,0x04,0x00,0x00,0x84,0x04,0x00,0x00, -0x41,0x00,0x06,0x00,0xff,0x01,0x00,0x00,0x86,0x04,0x00,0x00, -0x7f,0x04,0x00,0x00,0x16,0x00,0x00,0x00,0x83,0x04,0x00,0x00, -0x3e,0x00,0x03,0x00,0x86,0x04,0x00,0x00,0x85,0x04,0x00,0x00, -0xf9,0x00,0x02,0x00,0x7b,0x04,0x00,0x00,0xf8,0x00,0x02,0x00, -0x7b,0x04,0x00,0x00,0xfd,0x00,0x01,0x00,0x38,0x00,0x01,0x00, +0x14,0x00,0x00,0x00,0x59,0x00,0x00,0x00,0x58,0x00,0x00,0x00, +0x41,0x00,0x08,0x00,0x5d,0x00,0x00,0x00,0x5e,0x00,0x00,0x00, +0x54,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0x3e,0x00,0x00,0x00, +0x5c,0x00,0x00,0x00,0x5c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x4b,0x00,0x00,0x00,0x5f,0x00,0x00,0x00,0x5e,0x00,0x00,0x00, +0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x60,0x00,0x00,0x00, +0x5f,0x00,0x00,0x00,0xc4,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x62,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x61,0x00,0x00,0x00, +0x41,0x00,0x08,0x00,0x5d,0x00,0x00,0x00,0x64,0x00,0x00,0x00, +0x54,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0x3e,0x00,0x00,0x00, +0x5c,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x4b,0x00,0x00,0x00,0x65,0x00,0x00,0x00,0x64,0x00,0x00,0x00, +0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x66,0x00,0x00,0x00, +0x65,0x00,0x00,0x00,0xc5,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x67,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x66,0x00,0x00,0x00, +0xc2,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, +0x67,0x00,0x00,0x00,0x42,0x00,0x00,0x00,0xc4,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x6f,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, +0x6e,0x00,0x00,0x00,0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x70,0x00,0x00,0x00,0x6f,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, +0x7c,0x00,0x04,0x00,0x27,0x00,0x00,0x00,0x71,0x00,0x00,0x00, +0x70,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x75,0x00,0x00,0x00,0x42,0x00,0x00,0x00,0x74,0x00,0x00,0x00, +0xc2,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x76,0x00,0x00,0x00, +0x67,0x00,0x00,0x00,0x75,0x00,0x00,0x00,0xc7,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x77,0x00,0x00,0x00,0x76,0x00,0x00,0x00, +0x4e,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x27,0x00,0x00,0x00, +0x78,0x00,0x00,0x00,0x77,0x00,0x00,0x00,0x41,0x00,0x08,0x00, +0x7e,0x00,0x00,0x00,0x7f,0x00,0x00,0x00,0x54,0x00,0x00,0x00, +0x28,0x00,0x00,0x00,0x3e,0x00,0x00,0x00,0x7c,0x00,0x00,0x00, +0x42,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, +0x80,0x00,0x00,0x00,0x7f,0x00,0x00,0x00,0x71,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x81,0x00,0x00,0x00,0x80,0x00,0x00,0x00, +0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x87,0x00,0x00,0x00, +0x81,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x8b,0x00,0x00,0x00,0x71,0x00,0x00,0x00, +0xc5,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8c,0x00,0x00,0x00, +0x87,0x00,0x00,0x00,0x8b,0x00,0x00,0x00,0x70,0x00,0x04,0x00, +0x14,0x00,0x00,0x00,0x8d,0x00,0x00,0x00,0x8c,0x00,0x00,0x00, +0xc2,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8f,0x00,0x00,0x00, +0x81,0x00,0x00,0x00,0x6e,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x93,0x00,0x00,0x00,0x78,0x00,0x00,0x00, +0xc5,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x94,0x00,0x00,0x00, +0x8f,0x00,0x00,0x00,0x93,0x00,0x00,0x00,0x70,0x00,0x04,0x00, +0x14,0x00,0x00,0x00,0x95,0x00,0x00,0x00,0x94,0x00,0x00,0x00, +0x50,0x00,0x05,0x00,0x82,0x00,0x00,0x00,0x96,0x00,0x00,0x00, +0x8d,0x00,0x00,0x00,0x95,0x00,0x00,0x00,0x83,0x00,0x05,0x00, +0x82,0x00,0x00,0x00,0x9a,0x00,0x00,0x00,0x96,0x00,0x00,0x00, +0xf5,0x00,0x00,0x00,0x8e,0x00,0x05,0x00,0x82,0x00,0x00,0x00, +0x9c,0x00,0x00,0x00,0x9a,0x00,0x00,0x00,0x59,0x00,0x00,0x00, +0x51,0x00,0x05,0x00,0x14,0x00,0x00,0x00,0x9f,0x00,0x00,0x00, +0x9c,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x29,0x00,0x00,0x00,0xa4,0x00,0x00,0x00,0x26,0x00,0x00,0x00, +0x5c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xa5,0x00,0x00,0x00,0xa4,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xa7,0x00,0x00,0x00,0xa5,0x00,0x00,0x00, +0x47,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xa9,0x00,0x00,0x00,0xa7,0x00,0x00,0x00,0x42,0x00,0x00,0x00, +0x41,0x00,0x06,0x00,0xab,0x00,0x00,0x00,0xac,0x00,0x00,0x00, +0xa3,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0xa9,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0xad,0x00,0x00,0x00, +0xac,0x00,0x00,0x00,0x51,0x00,0x05,0x00,0x14,0x00,0x00,0x00, +0xb0,0x00,0x00,0x00,0x9c,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xb7,0x00,0x00,0x00, +0xa9,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x41,0x00,0x06,0x00, +0xab,0x00,0x00,0x00,0xb8,0x00,0x00,0x00,0xa3,0x00,0x00,0x00, +0x28,0x00,0x00,0x00,0xb7,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x14,0x00,0x00,0x00,0xb9,0x00,0x00,0x00,0xb8,0x00,0x00,0x00, +0x85,0x00,0x05,0x00,0x14,0x00,0x00,0x00,0xba,0x00,0x00,0x00, +0xb0,0x00,0x00,0x00,0xb9,0x00,0x00,0x00,0x0c,0x00,0x08,0x00, +0x14,0x00,0x00,0x00,0xbb,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x32,0x00,0x00,0x00,0x9f,0x00,0x00,0x00,0xad,0x00,0x00,0x00, +0xba,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x14,0x00,0x00,0x00, +0xbd,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x81,0x00,0x05,0x00, +0x14,0x00,0x00,0x00,0xbe,0x00,0x00,0x00,0xbd,0x00,0x00,0x00, +0xbb,0x00,0x00,0x00,0x3e,0x00,0x03,0x00,0x1c,0x00,0x00,0x00, +0xbe,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xc1,0x00,0x00,0x00,0xf2,0x00,0x00,0x00,0x32,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x1e,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0x20,0x00,0x00,0x00,0xe0,0x00,0x04,0x00,0x32,0x00,0x00,0x00, +0x32,0x00,0x00,0x00,0xc2,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xc5,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xc5,0x00,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xf3,0x00,0x00,0x00, +0xc4,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0xdc,0x00,0x00,0x00, +0xc8,0x00,0x00,0x00,0xac,0x00,0x05,0x00,0x2d,0x00,0x00,0x00, +0xcb,0x00,0x00,0x00,0xf3,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0xc7,0x00,0x00,0x00,0xc8,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xcb,0x00,0x00,0x00, +0xc6,0x00,0x00,0x00,0xc7,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xc6,0x00,0x00,0x00,0xb0,0x00,0x05,0x00,0x2d,0x00,0x00,0x00, +0xce,0x00,0x00,0x00,0x13,0x00,0x00,0x00,0xf3,0x00,0x00,0x00, +0xf7,0x00,0x03,0x00,0xd0,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0xce,0x00,0x00,0x00,0xcf,0x00,0x00,0x00, +0xd0,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xcf,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xd4,0x00,0x00,0x00, +0x13,0x00,0x00,0x00,0xf3,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x1b,0x00,0x00,0x00,0xd5,0x00,0x00,0x00,0x18,0x00,0x00,0x00, +0xd4,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x14,0x00,0x00,0x00, +0xd6,0x00,0x00,0x00,0xd5,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x14,0x00,0x00,0x00,0xd8,0x00,0x00,0x00,0x1c,0x00,0x00,0x00, +0x81,0x00,0x05,0x00,0x14,0x00,0x00,0x00,0xd9,0x00,0x00,0x00, +0xd8,0x00,0x00,0x00,0xd6,0x00,0x00,0x00,0x3e,0x00,0x03,0x00, +0x1c,0x00,0x00,0x00,0xd9,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xd0,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xd0,0x00,0x00,0x00, +0xe0,0x00,0x04,0x00,0x32,0x00,0x00,0x00,0x32,0x00,0x00,0x00, +0xc2,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xc8,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xc8,0x00,0x00,0x00,0xc2,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xdc,0x00,0x00,0x00,0xf3,0x00,0x00,0x00, +0x5c,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xc5,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xc7,0x00,0x00,0x00,0xaa,0x00,0x05,0x00, +0x2d,0x00,0x00,0x00,0xde,0x00,0x00,0x00,0x13,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0xf7,0x00,0x03,0x00,0xe0,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xde,0x00,0x00,0x00, +0xdf,0x00,0x00,0x00,0xe0,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xdf,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x29,0x00,0x00,0x00, +0xe5,0x00,0x00,0x00,0x26,0x00,0x00,0x00,0x7c,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xe6,0x00,0x00,0x00, +0xe5,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xe8,0x00,0x00,0x00,0xe6,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x1b,0x00,0x00,0x00,0xe9,0x00,0x00,0x00, +0x18,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x14,0x00,0x00,0x00,0xea,0x00,0x00,0x00,0xe9,0x00,0x00,0x00, +0x41,0x00,0x06,0x00,0xab,0x00,0x00,0x00,0xeb,0x00,0x00,0x00, +0xe4,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0xe8,0x00,0x00,0x00, +0x3e,0x00,0x03,0x00,0xeb,0x00,0x00,0x00,0xea,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xe0,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xe0,0x00,0x00,0x00,0xfd,0x00,0x01,0x00,0x38,0x00,0x01,0x00, }; -const uint64_t mul_mat_vec_q5_K_f32_len = 12048; +const uint64_t mul_mat_vec_q5_0_f32_len = 3696; -unsigned char mul_mat_vec_q6_K_f32_data[] = { +unsigned char mul_mat_vec_q5_1_f32_data[] = { 0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, -0x97,0x03,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, -0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x27,0x00,0x00,0x00, -0x11,0x00,0x02,0x00,0x51,0x11,0x00,0x00,0x11,0x00,0x02,0x00, -0x60,0x11,0x00,0x00,0x0b,0x00,0x06,0x00,0x01,0x00,0x00,0x00, -0x47,0x4c,0x53,0x4c,0x2e,0x73,0x74,0x64,0x2e,0x34,0x35,0x30, -0x00,0x00,0x00,0x00,0x0e,0x00,0x03,0x00,0x00,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x0f,0x00,0x0c,0x00,0x05,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x6d,0x61,0x69,0x6e,0x00,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x15,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x57,0x00,0x00,0x00,0x7f,0x00,0x00,0x00,0x94,0x00,0x00,0x00, -0x9c,0x01,0x00,0x00,0x10,0x00,0x06,0x00,0x04,0x00,0x00,0x00, -0x11,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x0c,0x00,0x00,0x00, -0x0b,0x00,0x00,0x00,0x1a,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x13,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x13,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x13,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x47,0x00,0x03,0x00, -0x13,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x21,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x1b,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x74,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x76,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x79,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x7b,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x7b,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x80,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x7b,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0xc0,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x7b,0x00,0x00,0x00,0x03,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0xd0,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x7c,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0xd2,0x00,0x00,0x00, -0x48,0x00,0x04,0x00,0x7d,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x7d,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x03,0x00,0x7d,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x7f,0x00,0x00,0x00,0x22,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x7f,0x00,0x00,0x00, -0x21,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x91,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x48,0x00,0x04,0x00,0x92,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x92,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x03,0x00,0x92,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x94,0x00,0x00,0x00,0x22,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x94,0x00,0x00,0x00, -0x21,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x99,0x01,0x00,0x00,0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x48,0x00,0x04,0x00,0x9a,0x01,0x00,0x00,0x00,0x00,0x00,0x00, -0x19,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x9a,0x01,0x00,0x00, -0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x03,0x00,0x9a,0x01,0x00,0x00,0x02,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x9c,0x01,0x00,0x00,0x22,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x9c,0x01,0x00,0x00, -0x21,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0xa5,0x01,0x00,0x00,0x0b,0x00,0x00,0x00,0x19,0x00,0x00,0x00, -0x13,0x00,0x02,0x00,0x02,0x00,0x00,0x00,0x21,0x00,0x03,0x00, -0x03,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x15,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x15,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x17,0x00,0x04,0x00,0x0a,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x0b,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x0a,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x0b,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x0d,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x0e,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x1e,0x00,0x05,0x00,0x13,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x14,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x13,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0x15,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x17,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x1a,0x00,0x00,0x00, -0x00,0x01,0x00,0x00,0x3b,0x00,0x04,0x00,0x0b,0x00,0x00,0x00, -0x21,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x25,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x08,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x36,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x3d,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x43,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x4e,0x00,0x00,0x00,0x80,0x00,0x00,0x00,0x16,0x00,0x03,0x00, -0x53,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x54,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x1c,0x00,0x04,0x00,0x55,0x00,0x00,0x00,0x53,0x00,0x00,0x00, -0x54,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x56,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x55,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x56,0x00,0x00,0x00,0x57,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x58,0x00,0x00,0x00, -0x10,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x53,0x00,0x00,0x00, -0x5d,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x5e,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x53,0x00,0x00,0x00, -0x14,0x00,0x02,0x00,0x69,0x00,0x00,0x00,0x15,0x00,0x04,0x00, -0x72,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x73,0x00,0x00,0x00, -0x80,0x00,0x00,0x00,0x1c,0x00,0x04,0x00,0x74,0x00,0x00,0x00, -0x72,0x00,0x00,0x00,0x73,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x75,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x1c,0x00,0x04,0x00,0x76,0x00,0x00,0x00,0x72,0x00,0x00,0x00, -0x75,0x00,0x00,0x00,0x15,0x00,0x04,0x00,0x77,0x00,0x00,0x00, -0x08,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x78,0x00,0x00,0x00,0x10,0x00,0x00,0x00, -0x1c,0x00,0x04,0x00,0x79,0x00,0x00,0x00,0x77,0x00,0x00,0x00, -0x78,0x00,0x00,0x00,0x16,0x00,0x03,0x00,0x7a,0x00,0x00,0x00, -0x10,0x00,0x00,0x00,0x1e,0x00,0x06,0x00,0x7b,0x00,0x00,0x00, -0x74,0x00,0x00,0x00,0x76,0x00,0x00,0x00,0x79,0x00,0x00,0x00, -0x7a,0x00,0x00,0x00,0x1d,0x00,0x03,0x00,0x7c,0x00,0x00,0x00, -0x7b,0x00,0x00,0x00,0x1e,0x00,0x03,0x00,0x7d,0x00,0x00,0x00, -0x7c,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x7e,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x7d,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x7e,0x00,0x00,0x00,0x7f,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x83,0x00,0x00,0x00, -0x03,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x84,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x7a,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, -0x91,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, -0x92,0x00,0x00,0x00,0x91,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x93,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x92,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x93,0x00,0x00,0x00,0x94,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x95,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x9d,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x53,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0xa5,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x77,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0xb3,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x72,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xb8,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x3b,0x01,0x00,0x00, -0x60,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x43,0x01,0x00,0x00,0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x7a,0x01,0x00,0x00,0x02,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x7b,0x01,0x00,0x00, -0x08,0x01,0x00,0x00,0x1d,0x00,0x03,0x00,0x99,0x01,0x00,0x00, -0x53,0x00,0x00,0x00,0x1e,0x00,0x03,0x00,0x9a,0x01,0x00,0x00, -0x99,0x01,0x00,0x00,0x20,0x00,0x04,0x00,0x9b,0x01,0x00,0x00, -0x0c,0x00,0x00,0x00,0x9a,0x01,0x00,0x00,0x3b,0x00,0x04,0x00, -0x9b,0x01,0x00,0x00,0x9c,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0xa4,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0x2c,0x00,0x06,0x00,0x0a,0x00,0x00,0x00, -0xa5,0x01,0x00,0x00,0x54,0x00,0x00,0x00,0xa4,0x01,0x00,0x00, -0xa4,0x01,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x8e,0x03,0x00,0x00,0x21,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x8f,0x03,0x00,0x00,0x41,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x90,0x03,0x00,0x00, -0x61,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x91,0x03,0x00,0x00,0x22,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x92,0x03,0x00,0x00,0x42,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x93,0x03,0x00,0x00, -0x62,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x94,0x03,0x00,0x00,0x23,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x95,0x03,0x00,0x00,0x43,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x96,0x03,0x00,0x00, -0x63,0x00,0x00,0x00,0x36,0x00,0x05,0x00,0x02,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x03,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0x05,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x0e,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x0d,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x10,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x11,0x00,0x00,0x00,0x10,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00,0x18,0x00,0x00,0x00, -0x15,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x18,0x00,0x00,0x00, -0x87,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x1b,0x00,0x00,0x00, -0x19,0x00,0x00,0x00,0x1a,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x1f,0x00,0x00,0x00,0x11,0x00,0x00,0x00, -0x1b,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x0e,0x00,0x00,0x00, -0x22,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x0d,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x22,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x24,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x87,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x26,0x00,0x00,0x00,0x24,0x00,0x00,0x00, -0x25,0x00,0x00,0x00,0x8b,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x2b,0x00,0x00,0x00,0x24,0x00,0x00,0x00,0x25,0x00,0x00,0x00, -0x87,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x2f,0x00,0x00,0x00, -0x26,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x33,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x2f,0x00,0x00,0x00,0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x34,0x00,0x00,0x00,0x26,0x00,0x00,0x00,0x33,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x38,0x00,0x00,0x00, -0x36,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0x87,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x3b,0x00,0x00,0x00,0x34,0x00,0x00,0x00, -0x36,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x3f,0x00,0x00,0x00,0x3d,0x00,0x00,0x00,0x2f,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x41,0x00,0x00,0x00, -0x3f,0x00,0x00,0x00,0x38,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x45,0x00,0x00,0x00,0x43,0x00,0x00,0x00, -0x2f,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x47,0x00,0x00,0x00,0x45,0x00,0x00,0x00,0x38,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x4c,0x00,0x00,0x00, -0x33,0x00,0x00,0x00,0x3b,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x50,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, -0x2f,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x52,0x00,0x00,0x00,0x50,0x00,0x00,0x00,0x38,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x5a,0x00,0x00,0x00, -0x58,0x00,0x00,0x00,0x2b,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x5c,0x00,0x00,0x00,0x5a,0x00,0x00,0x00, -0x26,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x5e,0x00,0x00,0x00, -0x5f,0x00,0x00,0x00,0x57,0x00,0x00,0x00,0x5c,0x00,0x00,0x00, -0x3e,0x00,0x03,0x00,0x5f,0x00,0x00,0x00,0x5d,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x62,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x62,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xa6,0x01,0x00,0x00,0x2b,0x00,0x00,0x00,0x05,0x00,0x00,0x00, -0x79,0x01,0x00,0x00,0x63,0x00,0x00,0x00,0xb1,0x00,0x05,0x00, -0x69,0x00,0x00,0x00,0x6a,0x00,0x00,0x00,0xa6,0x01,0x00,0x00, -0x1b,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x64,0x00,0x00,0x00, -0x63,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x6a,0x00,0x00,0x00,0x63,0x00,0x00,0x00,0x64,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0x63,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0xa6,0x01,0x00,0x00, -0x1a,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x6f,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x52,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x82,0x00,0x00,0x00, -0x1f,0x00,0x00,0x00,0xa6,0x01,0x00,0x00,0x41,0x00,0x07,0x00, -0x84,0x00,0x00,0x00,0x85,0x00,0x00,0x00,0x7f,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x82,0x00,0x00,0x00,0x83,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x7a,0x00,0x00,0x00,0x86,0x00,0x00,0x00, -0x85,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x53,0x00,0x00,0x00, -0x87,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x17,0x00,0x00,0x00,0x96,0x00,0x00,0x00,0x15,0x00,0x00,0x00, -0x95,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x97,0x00,0x00,0x00,0x96,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x99,0x00,0x00,0x00,0x97,0x00,0x00,0x00, -0x6f,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x9d,0x00,0x00,0x00, -0x9e,0x00,0x00,0x00,0x94,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0x99,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x53,0x00,0x00,0x00, -0x9f,0x00,0x00,0x00,0x9e,0x00,0x00,0x00,0x41,0x00,0x08,0x00, -0xa5,0x00,0x00,0x00,0xa6,0x00,0x00,0x00,0x7f,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x82,0x00,0x00,0x00,0x25,0x00,0x00,0x00, -0x4c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x77,0x00,0x00,0x00, -0xa7,0x00,0x00,0x00,0xa6,0x00,0x00,0x00,0x6f,0x00,0x04,0x00, -0x53,0x00,0x00,0x00,0xa8,0x00,0x00,0x00,0xa7,0x00,0x00,0x00, -0x85,0x00,0x05,0x00,0x53,0x00,0x00,0x00,0xa9,0x00,0x00,0x00, -0x9f,0x00,0x00,0x00,0xa8,0x00,0x00,0x00,0x85,0x00,0x05,0x00, -0x53,0x00,0x00,0x00,0xab,0x00,0x00,0x00,0xa9,0x00,0x00,0x00, -0x87,0x00,0x00,0x00,0x41,0x00,0x08,0x00,0xb3,0x00,0x00,0x00, -0xb4,0x00,0x00,0x00,0x7f,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0x82,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x41,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x72,0x00,0x00,0x00,0xb5,0x00,0x00,0x00, -0xb4,0x00,0x00,0x00,0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0xb6,0x00,0x00,0x00,0xb5,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xb7,0x00,0x00,0x00,0xb6,0x00,0x00,0x00, -0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xb9,0x00,0x00,0x00, -0xb7,0x00,0x00,0x00,0xb8,0x00,0x00,0x00,0x41,0x00,0x08,0x00, -0xb3,0x00,0x00,0x00,0xc0,0x00,0x00,0x00,0x7f,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x82,0x00,0x00,0x00,0x95,0x00,0x00,0x00, -0x47,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x72,0x00,0x00,0x00, -0xc1,0x00,0x00,0x00,0xc0,0x00,0x00,0x00,0xc2,0x00,0x05,0x00, -0x72,0x00,0x00,0x00,0xc2,0x00,0x00,0x00,0xc1,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0xc3,0x00,0x00,0x00,0xc2,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xc4,0x00,0x00,0x00,0xc3,0x00,0x00,0x00, -0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xc5,0x00,0x00,0x00, -0xc4,0x00,0x00,0x00,0x83,0x00,0x00,0x00,0xc4,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xc6,0x00,0x00,0x00,0xc5,0x00,0x00,0x00, -0x36,0x00,0x00,0x00,0xc5,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xc7,0x00,0x00,0x00,0xb9,0x00,0x00,0x00,0xc6,0x00,0x00,0x00, -0x72,0x00,0x04,0x00,0x77,0x00,0x00,0x00,0xc8,0x00,0x00,0x00, -0xc7,0x00,0x00,0x00,0x72,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xc9,0x00,0x00,0x00,0xc8,0x00,0x00,0x00,0x82,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xca,0x00,0x00,0x00,0xc9,0x00,0x00,0x00, -0x43,0x00,0x00,0x00,0x6f,0x00,0x04,0x00,0x53,0x00,0x00,0x00, -0xcb,0x00,0x00,0x00,0xca,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xd3,0x00,0x00,0x00,0x99,0x00,0x00,0x00, -0x43,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x9d,0x00,0x00,0x00, -0xd4,0x00,0x00,0x00,0x94,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0xd3,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x53,0x00,0x00,0x00, -0xd5,0x00,0x00,0x00,0xd4,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xda,0x00,0x00,0x00,0x4c,0x00,0x00,0x00, -0x25,0x00,0x00,0x00,0x41,0x00,0x08,0x00,0xa5,0x00,0x00,0x00, -0xdb,0x00,0x00,0x00,0x7f,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0x82,0x00,0x00,0x00,0x25,0x00,0x00,0x00,0xda,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x77,0x00,0x00,0x00,0xdc,0x00,0x00,0x00, -0xdb,0x00,0x00,0x00,0x6f,0x00,0x04,0x00,0x53,0x00,0x00,0x00, -0xdd,0x00,0x00,0x00,0xdc,0x00,0x00,0x00,0x85,0x00,0x05,0x00, -0x53,0x00,0x00,0x00,0xde,0x00,0x00,0x00,0xd5,0x00,0x00,0x00, -0xdd,0x00,0x00,0x00,0x85,0x00,0x05,0x00,0x53,0x00,0x00,0x00, -0xe0,0x00,0x00,0x00,0xde,0x00,0x00,0x00,0x87,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xe7,0x00,0x00,0x00, -0x41,0x00,0x00,0x00,0x43,0x00,0x00,0x00,0x41,0x00,0x08,0x00, -0xb3,0x00,0x00,0x00,0xe8,0x00,0x00,0x00,0x7f,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x82,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0xe7,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x72,0x00,0x00,0x00, -0xe9,0x00,0x00,0x00,0xe8,0x00,0x00,0x00,0x71,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0xea,0x00,0x00,0x00,0xe9,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xeb,0x00,0x00,0x00, -0xea,0x00,0x00,0x00,0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xec,0x00,0x00,0x00,0xeb,0x00,0x00,0x00,0xb8,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x72,0x00,0x00,0x00,0xf4,0x00,0x00,0x00, -0xc0,0x00,0x00,0x00,0xc2,0x00,0x05,0x00,0x72,0x00,0x00,0x00, -0xf5,0x00,0x00,0x00,0xf4,0x00,0x00,0x00,0x25,0x00,0x00,0x00, -0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0xf6,0x00,0x00,0x00, -0xf5,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xf7,0x00,0x00,0x00,0xf6,0x00,0x00,0x00,0xc7,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xf8,0x00,0x00,0x00,0xf7,0x00,0x00,0x00, -0x83,0x00,0x00,0x00,0xc4,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xf9,0x00,0x00,0x00,0xf8,0x00,0x00,0x00,0x36,0x00,0x00,0x00, -0xc5,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xfa,0x00,0x00,0x00, -0xec,0x00,0x00,0x00,0xf9,0x00,0x00,0x00,0x72,0x00,0x04,0x00, -0x77,0x00,0x00,0x00,0xfb,0x00,0x00,0x00,0xfa,0x00,0x00,0x00, -0x72,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xfc,0x00,0x00,0x00, -0xfb,0x00,0x00,0x00,0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xfd,0x00,0x00,0x00,0xfc,0x00,0x00,0x00,0x43,0x00,0x00,0x00, -0x6f,0x00,0x04,0x00,0x53,0x00,0x00,0x00,0xfe,0x00,0x00,0x00, -0xfd,0x00,0x00,0x00,0x85,0x00,0x05,0x00,0x53,0x00,0x00,0x00, -0xff,0x00,0x00,0x00,0xe0,0x00,0x00,0x00,0xfe,0x00,0x00,0x00, -0x0c,0x00,0x08,0x00,0x53,0x00,0x00,0x00,0x00,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0xab,0x00,0x00,0x00, -0xcb,0x00,0x00,0x00,0xff,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x07,0x01,0x00,0x00,0x99,0x00,0x00,0x00, -0x3d,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x9d,0x00,0x00,0x00, -0x08,0x01,0x00,0x00,0x94,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0x07,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0x53,0x00,0x00,0x00, -0x09,0x01,0x00,0x00,0x08,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x0e,0x01,0x00,0x00,0x4c,0x00,0x00,0x00, -0x36,0x00,0x00,0x00,0x41,0x00,0x08,0x00,0xa5,0x00,0x00,0x00, -0x0f,0x01,0x00,0x00,0x7f,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0x82,0x00,0x00,0x00,0x25,0x00,0x00,0x00,0x0e,0x01,0x00,0x00, -0x3d,0x00,0x04,0x00,0x77,0x00,0x00,0x00,0x10,0x01,0x00,0x00, -0x0f,0x01,0x00,0x00,0x6f,0x00,0x04,0x00,0x53,0x00,0x00,0x00, -0x11,0x01,0x00,0x00,0x10,0x01,0x00,0x00,0x85,0x00,0x05,0x00, -0x53,0x00,0x00,0x00,0x12,0x01,0x00,0x00,0x09,0x01,0x00,0x00, -0x11,0x01,0x00,0x00,0x85,0x00,0x05,0x00,0x53,0x00,0x00,0x00, -0x14,0x01,0x00,0x00,0x12,0x01,0x00,0x00,0x87,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x72,0x00,0x00,0x00,0x1d,0x01,0x00,0x00, -0xb4,0x00,0x00,0x00,0xc2,0x00,0x05,0x00,0x72,0x00,0x00,0x00, -0x1e,0x01,0x00,0x00,0x1d,0x01,0x00,0x00,0x36,0x00,0x00,0x00, -0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x1f,0x01,0x00,0x00, -0x1e,0x01,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x20,0x01,0x00,0x00,0x1f,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, -0x72,0x00,0x00,0x00,0x28,0x01,0x00,0x00,0xc0,0x00,0x00,0x00, -0xc2,0x00,0x05,0x00,0x72,0x00,0x00,0x00,0x29,0x01,0x00,0x00, -0x28,0x01,0x00,0x00,0x36,0x00,0x00,0x00,0x71,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x2a,0x01,0x00,0x00,0x29,0x01,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x2b,0x01,0x00,0x00, -0x2a,0x01,0x00,0x00,0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x2c,0x01,0x00,0x00,0x2b,0x01,0x00,0x00,0x83,0x00,0x00,0x00, -0xc4,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x2d,0x01,0x00,0x00, -0x2c,0x01,0x00,0x00,0x36,0x00,0x00,0x00,0xc5,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x2e,0x01,0x00,0x00,0x20,0x01,0x00,0x00, -0x2d,0x01,0x00,0x00,0x72,0x00,0x04,0x00,0x77,0x00,0x00,0x00, -0x2f,0x01,0x00,0x00,0x2e,0x01,0x00,0x00,0x72,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x30,0x01,0x00,0x00,0x2f,0x01,0x00,0x00, -0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x31,0x01,0x00,0x00, -0x30,0x01,0x00,0x00,0x43,0x00,0x00,0x00,0x6f,0x00,0x04,0x00, -0x53,0x00,0x00,0x00,0x32,0x01,0x00,0x00,0x31,0x01,0x00,0x00, -0x0c,0x00,0x08,0x00,0x53,0x00,0x00,0x00,0x34,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x14,0x01,0x00,0x00, -0x32,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x3c,0x01,0x00,0x00,0x99,0x00,0x00,0x00, -0x3b,0x01,0x00,0x00,0x41,0x00,0x06,0x00,0x9d,0x00,0x00,0x00, -0x3d,0x01,0x00,0x00,0x94,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0x3c,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0x53,0x00,0x00,0x00, -0x3e,0x01,0x00,0x00,0x3d,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x44,0x01,0x00,0x00,0x4c,0x00,0x00,0x00, -0x43,0x01,0x00,0x00,0x41,0x00,0x08,0x00,0xa5,0x00,0x00,0x00, -0x45,0x01,0x00,0x00,0x7f,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0x82,0x00,0x00,0x00,0x25,0x00,0x00,0x00,0x44,0x01,0x00,0x00, -0x3d,0x00,0x04,0x00,0x77,0x00,0x00,0x00,0x46,0x01,0x00,0x00, -0x45,0x01,0x00,0x00,0x6f,0x00,0x04,0x00,0x53,0x00,0x00,0x00, -0x47,0x01,0x00,0x00,0x46,0x01,0x00,0x00,0x85,0x00,0x05,0x00, -0x53,0x00,0x00,0x00,0x48,0x01,0x00,0x00,0x3e,0x01,0x00,0x00, -0x47,0x01,0x00,0x00,0x85,0x00,0x05,0x00,0x53,0x00,0x00,0x00, -0x4a,0x01,0x00,0x00,0x48,0x01,0x00,0x00,0x87,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x72,0x00,0x00,0x00,0x53,0x01,0x00,0x00, -0xe8,0x00,0x00,0x00,0xc2,0x00,0x05,0x00,0x72,0x00,0x00,0x00, -0x54,0x01,0x00,0x00,0x53,0x01,0x00,0x00,0x36,0x00,0x00,0x00, -0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x55,0x01,0x00,0x00, -0x54,0x01,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x56,0x01,0x00,0x00,0x55,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, -0x72,0x00,0x00,0x00,0x5e,0x01,0x00,0x00,0xc0,0x00,0x00,0x00, -0xc2,0x00,0x05,0x00,0x72,0x00,0x00,0x00,0x5f,0x01,0x00,0x00, -0x5e,0x01,0x00,0x00,0x43,0x01,0x00,0x00,0x71,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x60,0x01,0x00,0x00,0x5f,0x01,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x61,0x01,0x00,0x00, -0x60,0x01,0x00,0x00,0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x62,0x01,0x00,0x00,0x61,0x01,0x00,0x00,0x83,0x00,0x00,0x00, -0xc4,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x63,0x01,0x00,0x00, -0x62,0x01,0x00,0x00,0x36,0x00,0x00,0x00,0xc5,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x64,0x01,0x00,0x00,0x56,0x01,0x00,0x00, -0x63,0x01,0x00,0x00,0x72,0x00,0x04,0x00,0x77,0x00,0x00,0x00, -0x65,0x01,0x00,0x00,0x64,0x01,0x00,0x00,0x72,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x66,0x01,0x00,0x00,0x65,0x01,0x00,0x00, -0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x67,0x01,0x00,0x00, -0x66,0x01,0x00,0x00,0x43,0x00,0x00,0x00,0x6f,0x00,0x04,0x00, -0x53,0x00,0x00,0x00,0x68,0x01,0x00,0x00,0x67,0x01,0x00,0x00, -0x0c,0x00,0x08,0x00,0x53,0x00,0x00,0x00,0x6a,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x4a,0x01,0x00,0x00, -0x68,0x01,0x00,0x00,0x34,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xb6,0x01,0x00,0x00,0x99,0x00,0x00,0x00, -0x95,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x9d,0x00,0x00,0x00, -0xb8,0x01,0x00,0x00,0x94,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0xb6,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0x53,0x00,0x00,0x00, -0xb9,0x01,0x00,0x00,0xb8,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, -0x77,0x00,0x00,0x00,0xbd,0x01,0x00,0x00,0xa6,0x00,0x00,0x00, -0x6f,0x00,0x04,0x00,0x53,0x00,0x00,0x00,0xbe,0x01,0x00,0x00, -0xbd,0x01,0x00,0x00,0x85,0x00,0x05,0x00,0x53,0x00,0x00,0x00, -0xbf,0x01,0x00,0x00,0xb9,0x01,0x00,0x00,0xbe,0x01,0x00,0x00, -0x85,0x00,0x05,0x00,0x53,0x00,0x00,0x00,0xc0,0x01,0x00,0x00, -0xbf,0x01,0x00,0x00,0x87,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xc2,0x01,0x00,0x00,0x41,0x00,0x00,0x00, -0x95,0x00,0x00,0x00,0x41,0x00,0x08,0x00,0xb3,0x00,0x00,0x00, -0xc4,0x01,0x00,0x00,0x7f,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0x82,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0xc2,0x01,0x00,0x00, -0x3d,0x00,0x04,0x00,0x72,0x00,0x00,0x00,0xc5,0x01,0x00,0x00, -0xc4,0x01,0x00,0x00,0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0xc6,0x01,0x00,0x00,0xc5,0x01,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xc7,0x01,0x00,0x00,0xc6,0x01,0x00,0x00, -0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xc8,0x01,0x00,0x00, -0xc7,0x01,0x00,0x00,0xb8,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xca,0x01,0x00,0x00,0x47,0x00,0x00,0x00, -0x95,0x00,0x00,0x00,0x41,0x00,0x08,0x00,0xb3,0x00,0x00,0x00, -0xcb,0x01,0x00,0x00,0x7f,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0x82,0x00,0x00,0x00,0x95,0x00,0x00,0x00,0xca,0x01,0x00,0x00, -0x3d,0x00,0x04,0x00,0x72,0x00,0x00,0x00,0xcc,0x01,0x00,0x00, -0xcb,0x01,0x00,0x00,0xc2,0x00,0x05,0x00,0x72,0x00,0x00,0x00, -0xcd,0x01,0x00,0x00,0xcc,0x01,0x00,0x00,0x16,0x00,0x00,0x00, -0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0xce,0x01,0x00,0x00, -0xcd,0x01,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xcf,0x01,0x00,0x00,0xce,0x01,0x00,0x00,0xc7,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xd0,0x01,0x00,0x00,0xcf,0x01,0x00,0x00, -0x83,0x00,0x00,0x00,0xc4,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xd1,0x01,0x00,0x00,0xd0,0x01,0x00,0x00,0x36,0x00,0x00,0x00, -0xc5,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xd2,0x01,0x00,0x00, -0xc8,0x01,0x00,0x00,0xd1,0x01,0x00,0x00,0x72,0x00,0x04,0x00, -0x77,0x00,0x00,0x00,0xd3,0x01,0x00,0x00,0xd2,0x01,0x00,0x00, -0x72,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xd4,0x01,0x00,0x00, -0xd3,0x01,0x00,0x00,0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xd5,0x01,0x00,0x00,0xd4,0x01,0x00,0x00,0x43,0x00,0x00,0x00, -0x6f,0x00,0x04,0x00,0x53,0x00,0x00,0x00,0xd6,0x01,0x00,0x00, -0xd5,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xdc,0x01,0x00,0x00,0x99,0x00,0x00,0x00,0x8e,0x03,0x00,0x00, -0x41,0x00,0x06,0x00,0x9d,0x00,0x00,0x00,0xdd,0x01,0x00,0x00, -0x94,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0xdc,0x01,0x00,0x00, -0x3d,0x00,0x04,0x00,0x53,0x00,0x00,0x00,0xde,0x01,0x00,0x00, -0xdd,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0x77,0x00,0x00,0x00, -0xe2,0x01,0x00,0x00,0xdb,0x00,0x00,0x00,0x6f,0x00,0x04,0x00, -0x53,0x00,0x00,0x00,0xe3,0x01,0x00,0x00,0xe2,0x01,0x00,0x00, -0x85,0x00,0x05,0x00,0x53,0x00,0x00,0x00,0xe4,0x01,0x00,0x00, -0xde,0x01,0x00,0x00,0xe3,0x01,0x00,0x00,0x85,0x00,0x05,0x00, -0x53,0x00,0x00,0x00,0xe5,0x01,0x00,0x00,0xe4,0x01,0x00,0x00, -0x87,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xe8,0x01,0x00,0x00,0x41,0x00,0x00,0x00,0x8e,0x03,0x00,0x00, -0x41,0x00,0x08,0x00,0xb3,0x00,0x00,0x00,0xe9,0x01,0x00,0x00, -0x7f,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x82,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0xe8,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, -0x72,0x00,0x00,0x00,0xea,0x01,0x00,0x00,0xe9,0x01,0x00,0x00, -0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0xeb,0x01,0x00,0x00, -0xea,0x01,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xec,0x01,0x00,0x00,0xeb,0x01,0x00,0x00,0xc7,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xed,0x01,0x00,0x00,0xec,0x01,0x00,0x00, -0xb8,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x72,0x00,0x00,0x00, -0xf1,0x01,0x00,0x00,0xcb,0x01,0x00,0x00,0xc2,0x00,0x05,0x00, -0x72,0x00,0x00,0x00,0xf2,0x01,0x00,0x00,0xf1,0x01,0x00,0x00, -0x25,0x00,0x00,0x00,0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0xf3,0x01,0x00,0x00,0xf2,0x01,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xf4,0x01,0x00,0x00,0xf3,0x01,0x00,0x00, -0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xf5,0x01,0x00,0x00, -0xf4,0x01,0x00,0x00,0x83,0x00,0x00,0x00,0xc4,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xf6,0x01,0x00,0x00,0xf5,0x01,0x00,0x00, -0x36,0x00,0x00,0x00,0xc5,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xf7,0x01,0x00,0x00,0xed,0x01,0x00,0x00,0xf6,0x01,0x00,0x00, -0x72,0x00,0x04,0x00,0x77,0x00,0x00,0x00,0xf8,0x01,0x00,0x00, -0xf7,0x01,0x00,0x00,0x72,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xf9,0x01,0x00,0x00,0xf8,0x01,0x00,0x00,0x82,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xfa,0x01,0x00,0x00,0xf9,0x01,0x00,0x00, -0x43,0x00,0x00,0x00,0x6f,0x00,0x04,0x00,0x53,0x00,0x00,0x00, -0xfb,0x01,0x00,0x00,0xfa,0x01,0x00,0x00,0x85,0x00,0x05,0x00, -0x53,0x00,0x00,0x00,0xfc,0x01,0x00,0x00,0xe5,0x01,0x00,0x00, -0xfb,0x01,0x00,0x00,0x0c,0x00,0x08,0x00,0x53,0x00,0x00,0x00, -0xfd,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00, -0xc0,0x01,0x00,0x00,0xd6,0x01,0x00,0x00,0xfc,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x02,0x02,0x00,0x00, -0x99,0x00,0x00,0x00,0x8f,0x03,0x00,0x00,0x41,0x00,0x06,0x00, -0x9d,0x00,0x00,0x00,0x03,0x02,0x00,0x00,0x94,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x02,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, -0x53,0x00,0x00,0x00,0x04,0x02,0x00,0x00,0x03,0x02,0x00,0x00, -0x3d,0x00,0x04,0x00,0x77,0x00,0x00,0x00,0x08,0x02,0x00,0x00, -0x0f,0x01,0x00,0x00,0x6f,0x00,0x04,0x00,0x53,0x00,0x00,0x00, -0x09,0x02,0x00,0x00,0x08,0x02,0x00,0x00,0x85,0x00,0x05,0x00, -0x53,0x00,0x00,0x00,0x0a,0x02,0x00,0x00,0x04,0x02,0x00,0x00, -0x09,0x02,0x00,0x00,0x85,0x00,0x05,0x00,0x53,0x00,0x00,0x00, -0x0b,0x02,0x00,0x00,0x0a,0x02,0x00,0x00,0x87,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x72,0x00,0x00,0x00,0x10,0x02,0x00,0x00, -0xc4,0x01,0x00,0x00,0xc2,0x00,0x05,0x00,0x72,0x00,0x00,0x00, -0x11,0x02,0x00,0x00,0x10,0x02,0x00,0x00,0x36,0x00,0x00,0x00, -0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x12,0x02,0x00,0x00, -0x11,0x02,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x13,0x02,0x00,0x00,0x12,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, -0x72,0x00,0x00,0x00,0x17,0x02,0x00,0x00,0xcb,0x01,0x00,0x00, -0xc2,0x00,0x05,0x00,0x72,0x00,0x00,0x00,0x18,0x02,0x00,0x00, -0x17,0x02,0x00,0x00,0x36,0x00,0x00,0x00,0x71,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x19,0x02,0x00,0x00,0x18,0x02,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x1a,0x02,0x00,0x00, -0x19,0x02,0x00,0x00,0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x1b,0x02,0x00,0x00,0x1a,0x02,0x00,0x00,0x83,0x00,0x00,0x00, -0xc4,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x1c,0x02,0x00,0x00, -0x1b,0x02,0x00,0x00,0x36,0x00,0x00,0x00,0xc5,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x1d,0x02,0x00,0x00,0x13,0x02,0x00,0x00, -0x1c,0x02,0x00,0x00,0x72,0x00,0x04,0x00,0x77,0x00,0x00,0x00, -0x1e,0x02,0x00,0x00,0x1d,0x02,0x00,0x00,0x72,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x1f,0x02,0x00,0x00,0x1e,0x02,0x00,0x00, -0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x20,0x02,0x00,0x00, -0x1f,0x02,0x00,0x00,0x43,0x00,0x00,0x00,0x6f,0x00,0x04,0x00, -0x53,0x00,0x00,0x00,0x21,0x02,0x00,0x00,0x20,0x02,0x00,0x00, -0x0c,0x00,0x08,0x00,0x53,0x00,0x00,0x00,0x23,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x0b,0x02,0x00,0x00, -0x21,0x02,0x00,0x00,0xfd,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x28,0x02,0x00,0x00,0x99,0x00,0x00,0x00, -0x90,0x03,0x00,0x00,0x41,0x00,0x06,0x00,0x9d,0x00,0x00,0x00, -0x29,0x02,0x00,0x00,0x94,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0x28,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0x53,0x00,0x00,0x00, -0x2a,0x02,0x00,0x00,0x29,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, -0x77,0x00,0x00,0x00,0x2e,0x02,0x00,0x00,0x45,0x01,0x00,0x00, -0x6f,0x00,0x04,0x00,0x53,0x00,0x00,0x00,0x2f,0x02,0x00,0x00, -0x2e,0x02,0x00,0x00,0x85,0x00,0x05,0x00,0x53,0x00,0x00,0x00, -0x30,0x02,0x00,0x00,0x2a,0x02,0x00,0x00,0x2f,0x02,0x00,0x00, -0x85,0x00,0x05,0x00,0x53,0x00,0x00,0x00,0x31,0x02,0x00,0x00, -0x30,0x02,0x00,0x00,0x87,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x72,0x00,0x00,0x00,0x36,0x02,0x00,0x00,0xe9,0x01,0x00,0x00, -0xc2,0x00,0x05,0x00,0x72,0x00,0x00,0x00,0x37,0x02,0x00,0x00, -0x36,0x02,0x00,0x00,0x36,0x00,0x00,0x00,0x71,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x38,0x02,0x00,0x00,0x37,0x02,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x39,0x02,0x00,0x00, -0x38,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0x72,0x00,0x00,0x00, -0x3d,0x02,0x00,0x00,0xcb,0x01,0x00,0x00,0xc2,0x00,0x05,0x00, -0x72,0x00,0x00,0x00,0x3e,0x02,0x00,0x00,0x3d,0x02,0x00,0x00, -0x43,0x01,0x00,0x00,0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x3f,0x02,0x00,0x00,0x3e,0x02,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x40,0x02,0x00,0x00,0x3f,0x02,0x00,0x00, -0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x41,0x02,0x00,0x00, -0x40,0x02,0x00,0x00,0x83,0x00,0x00,0x00,0xc4,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x42,0x02,0x00,0x00,0x41,0x02,0x00,0x00, -0x36,0x00,0x00,0x00,0xc5,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x43,0x02,0x00,0x00,0x39,0x02,0x00,0x00,0x42,0x02,0x00,0x00, -0x72,0x00,0x04,0x00,0x77,0x00,0x00,0x00,0x44,0x02,0x00,0x00, -0x43,0x02,0x00,0x00,0x72,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x45,0x02,0x00,0x00,0x44,0x02,0x00,0x00,0x82,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x46,0x02,0x00,0x00,0x45,0x02,0x00,0x00, -0x43,0x00,0x00,0x00,0x6f,0x00,0x04,0x00,0x53,0x00,0x00,0x00, -0x47,0x02,0x00,0x00,0x46,0x02,0x00,0x00,0x0c,0x00,0x08,0x00, -0x53,0x00,0x00,0x00,0x49,0x02,0x00,0x00,0x01,0x00,0x00,0x00, -0x32,0x00,0x00,0x00,0x31,0x02,0x00,0x00,0x47,0x02,0x00,0x00, -0x23,0x02,0x00,0x00,0x81,0x00,0x05,0x00,0x53,0x00,0x00,0x00, -0x4a,0x02,0x00,0x00,0x6a,0x01,0x00,0x00,0x49,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x56,0x02,0x00,0x00, -0x99,0x00,0x00,0x00,0x25,0x00,0x00,0x00,0x41,0x00,0x06,0x00, -0x9d,0x00,0x00,0x00,0x58,0x02,0x00,0x00,0x94,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x56,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, -0x53,0x00,0x00,0x00,0x59,0x02,0x00,0x00,0x58,0x02,0x00,0x00, -0x3d,0x00,0x04,0x00,0x77,0x00,0x00,0x00,0x5d,0x02,0x00,0x00, -0xa6,0x00,0x00,0x00,0x6f,0x00,0x04,0x00,0x53,0x00,0x00,0x00, -0x5e,0x02,0x00,0x00,0x5d,0x02,0x00,0x00,0x85,0x00,0x05,0x00, -0x53,0x00,0x00,0x00,0x5f,0x02,0x00,0x00,0x59,0x02,0x00,0x00, -0x5e,0x02,0x00,0x00,0x85,0x00,0x05,0x00,0x53,0x00,0x00,0x00, -0x60,0x02,0x00,0x00,0x5f,0x02,0x00,0x00,0x87,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x62,0x02,0x00,0x00, -0x41,0x00,0x00,0x00,0x25,0x00,0x00,0x00,0x41,0x00,0x08,0x00, -0xb3,0x00,0x00,0x00,0x64,0x02,0x00,0x00,0x7f,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x82,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0x62,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0x72,0x00,0x00,0x00, -0x65,0x02,0x00,0x00,0x64,0x02,0x00,0x00,0x71,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x66,0x02,0x00,0x00,0x65,0x02,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x67,0x02,0x00,0x00, -0x66,0x02,0x00,0x00,0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x68,0x02,0x00,0x00,0x67,0x02,0x00,0x00,0xb8,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x6a,0x02,0x00,0x00, -0x47,0x00,0x00,0x00,0x25,0x00,0x00,0x00,0x41,0x00,0x08,0x00, -0xb3,0x00,0x00,0x00,0x6b,0x02,0x00,0x00,0x7f,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x82,0x00,0x00,0x00,0x95,0x00,0x00,0x00, -0x6a,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0x72,0x00,0x00,0x00, -0x6c,0x02,0x00,0x00,0x6b,0x02,0x00,0x00,0xc2,0x00,0x05,0x00, -0x72,0x00,0x00,0x00,0x6d,0x02,0x00,0x00,0x6c,0x02,0x00,0x00, -0x16,0x00,0x00,0x00,0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x6e,0x02,0x00,0x00,0x6d,0x02,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x6f,0x02,0x00,0x00,0x6e,0x02,0x00,0x00, -0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x70,0x02,0x00,0x00, -0x6f,0x02,0x00,0x00,0x83,0x00,0x00,0x00,0xc4,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x71,0x02,0x00,0x00,0x70,0x02,0x00,0x00, -0x36,0x00,0x00,0x00,0xc5,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x72,0x02,0x00,0x00,0x68,0x02,0x00,0x00,0x71,0x02,0x00,0x00, -0x72,0x00,0x04,0x00,0x77,0x00,0x00,0x00,0x73,0x02,0x00,0x00, -0x72,0x02,0x00,0x00,0x72,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x74,0x02,0x00,0x00,0x73,0x02,0x00,0x00,0x82,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x75,0x02,0x00,0x00,0x74,0x02,0x00,0x00, -0x43,0x00,0x00,0x00,0x6f,0x00,0x04,0x00,0x53,0x00,0x00,0x00, -0x76,0x02,0x00,0x00,0x75,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x7c,0x02,0x00,0x00,0x99,0x00,0x00,0x00, -0x91,0x03,0x00,0x00,0x41,0x00,0x06,0x00,0x9d,0x00,0x00,0x00, -0x7d,0x02,0x00,0x00,0x94,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0x7c,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0x53,0x00,0x00,0x00, -0x7e,0x02,0x00,0x00,0x7d,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, -0x77,0x00,0x00,0x00,0x82,0x02,0x00,0x00,0xdb,0x00,0x00,0x00, -0x6f,0x00,0x04,0x00,0x53,0x00,0x00,0x00,0x83,0x02,0x00,0x00, -0x82,0x02,0x00,0x00,0x85,0x00,0x05,0x00,0x53,0x00,0x00,0x00, -0x84,0x02,0x00,0x00,0x7e,0x02,0x00,0x00,0x83,0x02,0x00,0x00, -0x85,0x00,0x05,0x00,0x53,0x00,0x00,0x00,0x85,0x02,0x00,0x00, -0x84,0x02,0x00,0x00,0x87,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x88,0x02,0x00,0x00,0x41,0x00,0x00,0x00, -0x91,0x03,0x00,0x00,0x41,0x00,0x08,0x00,0xb3,0x00,0x00,0x00, -0x89,0x02,0x00,0x00,0x7f,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0x82,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x88,0x02,0x00,0x00, -0x3d,0x00,0x04,0x00,0x72,0x00,0x00,0x00,0x8a,0x02,0x00,0x00, -0x89,0x02,0x00,0x00,0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x8b,0x02,0x00,0x00,0x8a,0x02,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x8c,0x02,0x00,0x00,0x8b,0x02,0x00,0x00, -0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8d,0x02,0x00,0x00, -0x8c,0x02,0x00,0x00,0xb8,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x72,0x00,0x00,0x00,0x91,0x02,0x00,0x00,0x6b,0x02,0x00,0x00, -0xc2,0x00,0x05,0x00,0x72,0x00,0x00,0x00,0x92,0x02,0x00,0x00, -0x91,0x02,0x00,0x00,0x25,0x00,0x00,0x00,0x71,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x93,0x02,0x00,0x00,0x92,0x02,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x94,0x02,0x00,0x00, -0x93,0x02,0x00,0x00,0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x95,0x02,0x00,0x00,0x94,0x02,0x00,0x00,0x83,0x00,0x00,0x00, -0xc4,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x96,0x02,0x00,0x00, -0x95,0x02,0x00,0x00,0x36,0x00,0x00,0x00,0xc5,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x97,0x02,0x00,0x00,0x8d,0x02,0x00,0x00, -0x96,0x02,0x00,0x00,0x72,0x00,0x04,0x00,0x77,0x00,0x00,0x00, -0x98,0x02,0x00,0x00,0x97,0x02,0x00,0x00,0x72,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x99,0x02,0x00,0x00,0x98,0x02,0x00,0x00, -0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9a,0x02,0x00,0x00, -0x99,0x02,0x00,0x00,0x43,0x00,0x00,0x00,0x6f,0x00,0x04,0x00, -0x53,0x00,0x00,0x00,0x9b,0x02,0x00,0x00,0x9a,0x02,0x00,0x00, -0x85,0x00,0x05,0x00,0x53,0x00,0x00,0x00,0x9c,0x02,0x00,0x00, -0x85,0x02,0x00,0x00,0x9b,0x02,0x00,0x00,0x0c,0x00,0x08,0x00, -0x53,0x00,0x00,0x00,0x9d,0x02,0x00,0x00,0x01,0x00,0x00,0x00, -0x32,0x00,0x00,0x00,0x60,0x02,0x00,0x00,0x76,0x02,0x00,0x00, -0x9c,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xa2,0x02,0x00,0x00,0x99,0x00,0x00,0x00,0x92,0x03,0x00,0x00, -0x41,0x00,0x06,0x00,0x9d,0x00,0x00,0x00,0xa3,0x02,0x00,0x00, -0x94,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0xa2,0x02,0x00,0x00, -0x3d,0x00,0x04,0x00,0x53,0x00,0x00,0x00,0xa4,0x02,0x00,0x00, -0xa3,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0x77,0x00,0x00,0x00, -0xa8,0x02,0x00,0x00,0x0f,0x01,0x00,0x00,0x6f,0x00,0x04,0x00, -0x53,0x00,0x00,0x00,0xa9,0x02,0x00,0x00,0xa8,0x02,0x00,0x00, -0x85,0x00,0x05,0x00,0x53,0x00,0x00,0x00,0xaa,0x02,0x00,0x00, -0xa4,0x02,0x00,0x00,0xa9,0x02,0x00,0x00,0x85,0x00,0x05,0x00, -0x53,0x00,0x00,0x00,0xab,0x02,0x00,0x00,0xaa,0x02,0x00,0x00, -0x87,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x72,0x00,0x00,0x00, -0xb0,0x02,0x00,0x00,0x64,0x02,0x00,0x00,0xc2,0x00,0x05,0x00, -0x72,0x00,0x00,0x00,0xb1,0x02,0x00,0x00,0xb0,0x02,0x00,0x00, -0x36,0x00,0x00,0x00,0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0xb2,0x02,0x00,0x00,0xb1,0x02,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xb3,0x02,0x00,0x00,0xb2,0x02,0x00,0x00, -0x3d,0x00,0x04,0x00,0x72,0x00,0x00,0x00,0xb7,0x02,0x00,0x00, -0x6b,0x02,0x00,0x00,0xc2,0x00,0x05,0x00,0x72,0x00,0x00,0x00, -0xb8,0x02,0x00,0x00,0xb7,0x02,0x00,0x00,0x36,0x00,0x00,0x00, -0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0xb9,0x02,0x00,0x00, -0xb8,0x02,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xba,0x02,0x00,0x00,0xb9,0x02,0x00,0x00,0xc7,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xbb,0x02,0x00,0x00,0xba,0x02,0x00,0x00, -0x83,0x00,0x00,0x00,0xc4,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xbc,0x02,0x00,0x00,0xbb,0x02,0x00,0x00,0x36,0x00,0x00,0x00, -0xc5,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xbd,0x02,0x00,0x00, -0xb3,0x02,0x00,0x00,0xbc,0x02,0x00,0x00,0x72,0x00,0x04,0x00, -0x77,0x00,0x00,0x00,0xbe,0x02,0x00,0x00,0xbd,0x02,0x00,0x00, -0x72,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xbf,0x02,0x00,0x00, -0xbe,0x02,0x00,0x00,0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xc0,0x02,0x00,0x00,0xbf,0x02,0x00,0x00,0x43,0x00,0x00,0x00, -0x6f,0x00,0x04,0x00,0x53,0x00,0x00,0x00,0xc1,0x02,0x00,0x00, -0xc0,0x02,0x00,0x00,0x0c,0x00,0x08,0x00,0x53,0x00,0x00,0x00, -0xc3,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00, -0xab,0x02,0x00,0x00,0xc1,0x02,0x00,0x00,0x9d,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xc8,0x02,0x00,0x00, -0x99,0x00,0x00,0x00,0x93,0x03,0x00,0x00,0x41,0x00,0x06,0x00, -0x9d,0x00,0x00,0x00,0xc9,0x02,0x00,0x00,0x94,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0xc8,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, -0x53,0x00,0x00,0x00,0xca,0x02,0x00,0x00,0xc9,0x02,0x00,0x00, -0x3d,0x00,0x04,0x00,0x77,0x00,0x00,0x00,0xce,0x02,0x00,0x00, -0x45,0x01,0x00,0x00,0x6f,0x00,0x04,0x00,0x53,0x00,0x00,0x00, -0xcf,0x02,0x00,0x00,0xce,0x02,0x00,0x00,0x85,0x00,0x05,0x00, -0x53,0x00,0x00,0x00,0xd0,0x02,0x00,0x00,0xca,0x02,0x00,0x00, -0xcf,0x02,0x00,0x00,0x85,0x00,0x05,0x00,0x53,0x00,0x00,0x00, -0xd1,0x02,0x00,0x00,0xd0,0x02,0x00,0x00,0x87,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x72,0x00,0x00,0x00,0xd6,0x02,0x00,0x00, -0x89,0x02,0x00,0x00,0xc2,0x00,0x05,0x00,0x72,0x00,0x00,0x00, -0xd7,0x02,0x00,0x00,0xd6,0x02,0x00,0x00,0x36,0x00,0x00,0x00, -0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0xd8,0x02,0x00,0x00, -0xd7,0x02,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xd9,0x02,0x00,0x00,0xd8,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, -0x72,0x00,0x00,0x00,0xdd,0x02,0x00,0x00,0x6b,0x02,0x00,0x00, -0xc2,0x00,0x05,0x00,0x72,0x00,0x00,0x00,0xde,0x02,0x00,0x00, -0xdd,0x02,0x00,0x00,0x43,0x01,0x00,0x00,0x71,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0xdf,0x02,0x00,0x00,0xde,0x02,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xe0,0x02,0x00,0x00, -0xdf,0x02,0x00,0x00,0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xe1,0x02,0x00,0x00,0xe0,0x02,0x00,0x00,0x83,0x00,0x00,0x00, -0xc4,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xe2,0x02,0x00,0x00, -0xe1,0x02,0x00,0x00,0x36,0x00,0x00,0x00,0xc5,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xe3,0x02,0x00,0x00,0xd9,0x02,0x00,0x00, -0xe2,0x02,0x00,0x00,0x72,0x00,0x04,0x00,0x77,0x00,0x00,0x00, -0xe4,0x02,0x00,0x00,0xe3,0x02,0x00,0x00,0x72,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xe5,0x02,0x00,0x00,0xe4,0x02,0x00,0x00, -0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xe6,0x02,0x00,0x00, -0xe5,0x02,0x00,0x00,0x43,0x00,0x00,0x00,0x6f,0x00,0x04,0x00, -0x53,0x00,0x00,0x00,0xe7,0x02,0x00,0x00,0xe6,0x02,0x00,0x00, -0x0c,0x00,0x08,0x00,0x53,0x00,0x00,0x00,0xe9,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0xd1,0x02,0x00,0x00, -0xe7,0x02,0x00,0x00,0xc3,0x02,0x00,0x00,0x81,0x00,0x05,0x00, -0x53,0x00,0x00,0x00,0xea,0x02,0x00,0x00,0x4a,0x02,0x00,0x00, -0xe9,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xf6,0x02,0x00,0x00,0x99,0x00,0x00,0x00,0x83,0x00,0x00,0x00, -0x41,0x00,0x06,0x00,0x9d,0x00,0x00,0x00,0xf8,0x02,0x00,0x00, -0x94,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0xf6,0x02,0x00,0x00, -0x3d,0x00,0x04,0x00,0x53,0x00,0x00,0x00,0xf9,0x02,0x00,0x00, -0xf8,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0x77,0x00,0x00,0x00, -0xfd,0x02,0x00,0x00,0xa6,0x00,0x00,0x00,0x6f,0x00,0x04,0x00, -0x53,0x00,0x00,0x00,0xfe,0x02,0x00,0x00,0xfd,0x02,0x00,0x00, -0x85,0x00,0x05,0x00,0x53,0x00,0x00,0x00,0xff,0x02,0x00,0x00, -0xf9,0x02,0x00,0x00,0xfe,0x02,0x00,0x00,0x85,0x00,0x05,0x00, -0x53,0x00,0x00,0x00,0x00,0x03,0x00,0x00,0xff,0x02,0x00,0x00, -0x87,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x02,0x03,0x00,0x00,0x41,0x00,0x00,0x00,0x83,0x00,0x00,0x00, -0x41,0x00,0x08,0x00,0xb3,0x00,0x00,0x00,0x04,0x03,0x00,0x00, -0x7f,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x82,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x02,0x03,0x00,0x00,0x3d,0x00,0x04,0x00, -0x72,0x00,0x00,0x00,0x05,0x03,0x00,0x00,0x04,0x03,0x00,0x00, -0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x06,0x03,0x00,0x00, -0x05,0x03,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x07,0x03,0x00,0x00,0x06,0x03,0x00,0x00,0xc7,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x08,0x03,0x00,0x00,0x07,0x03,0x00,0x00, -0xb8,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x0a,0x03,0x00,0x00,0x47,0x00,0x00,0x00,0x83,0x00,0x00,0x00, -0x41,0x00,0x08,0x00,0xb3,0x00,0x00,0x00,0x0b,0x03,0x00,0x00, -0x7f,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x82,0x00,0x00,0x00, -0x95,0x00,0x00,0x00,0x0a,0x03,0x00,0x00,0x3d,0x00,0x04,0x00, -0x72,0x00,0x00,0x00,0x0c,0x03,0x00,0x00,0x0b,0x03,0x00,0x00, -0xc2,0x00,0x05,0x00,0x72,0x00,0x00,0x00,0x0d,0x03,0x00,0x00, -0x0c,0x03,0x00,0x00,0x16,0x00,0x00,0x00,0x71,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x0e,0x03,0x00,0x00,0x0d,0x03,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x0f,0x03,0x00,0x00, -0x0e,0x03,0x00,0x00,0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x10,0x03,0x00,0x00,0x0f,0x03,0x00,0x00,0x83,0x00,0x00,0x00, -0xc4,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x11,0x03,0x00,0x00, -0x10,0x03,0x00,0x00,0x36,0x00,0x00,0x00,0xc5,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x12,0x03,0x00,0x00,0x08,0x03,0x00,0x00, -0x11,0x03,0x00,0x00,0x72,0x00,0x04,0x00,0x77,0x00,0x00,0x00, -0x13,0x03,0x00,0x00,0x12,0x03,0x00,0x00,0x72,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x14,0x03,0x00,0x00,0x13,0x03,0x00,0x00, -0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x15,0x03,0x00,0x00, -0x14,0x03,0x00,0x00,0x43,0x00,0x00,0x00,0x6f,0x00,0x04,0x00, -0x53,0x00,0x00,0x00,0x16,0x03,0x00,0x00,0x15,0x03,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x1c,0x03,0x00,0x00, -0x99,0x00,0x00,0x00,0x94,0x03,0x00,0x00,0x41,0x00,0x06,0x00, -0x9d,0x00,0x00,0x00,0x1d,0x03,0x00,0x00,0x94,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x1c,0x03,0x00,0x00,0x3d,0x00,0x04,0x00, -0x53,0x00,0x00,0x00,0x1e,0x03,0x00,0x00,0x1d,0x03,0x00,0x00, -0x3d,0x00,0x04,0x00,0x77,0x00,0x00,0x00,0x22,0x03,0x00,0x00, -0xdb,0x00,0x00,0x00,0x6f,0x00,0x04,0x00,0x53,0x00,0x00,0x00, -0x23,0x03,0x00,0x00,0x22,0x03,0x00,0x00,0x85,0x00,0x05,0x00, -0x53,0x00,0x00,0x00,0x24,0x03,0x00,0x00,0x1e,0x03,0x00,0x00, -0x23,0x03,0x00,0x00,0x85,0x00,0x05,0x00,0x53,0x00,0x00,0x00, -0x25,0x03,0x00,0x00,0x24,0x03,0x00,0x00,0x87,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x28,0x03,0x00,0x00, -0x41,0x00,0x00,0x00,0x94,0x03,0x00,0x00,0x41,0x00,0x08,0x00, -0xb3,0x00,0x00,0x00,0x29,0x03,0x00,0x00,0x7f,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x82,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0x28,0x03,0x00,0x00,0x3d,0x00,0x04,0x00,0x72,0x00,0x00,0x00, -0x2a,0x03,0x00,0x00,0x29,0x03,0x00,0x00,0x71,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x2b,0x03,0x00,0x00,0x2a,0x03,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x2c,0x03,0x00,0x00, -0x2b,0x03,0x00,0x00,0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x2d,0x03,0x00,0x00,0x2c,0x03,0x00,0x00,0xb8,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x72,0x00,0x00,0x00,0x31,0x03,0x00,0x00, -0x0b,0x03,0x00,0x00,0xc2,0x00,0x05,0x00,0x72,0x00,0x00,0x00, -0x32,0x03,0x00,0x00,0x31,0x03,0x00,0x00,0x25,0x00,0x00,0x00, -0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x33,0x03,0x00,0x00, -0x32,0x03,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x34,0x03,0x00,0x00,0x33,0x03,0x00,0x00,0xc7,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x35,0x03,0x00,0x00,0x34,0x03,0x00,0x00, -0x83,0x00,0x00,0x00,0xc4,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x36,0x03,0x00,0x00,0x35,0x03,0x00,0x00,0x36,0x00,0x00,0x00, -0xc5,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x37,0x03,0x00,0x00, -0x2d,0x03,0x00,0x00,0x36,0x03,0x00,0x00,0x72,0x00,0x04,0x00, -0x77,0x00,0x00,0x00,0x38,0x03,0x00,0x00,0x37,0x03,0x00,0x00, -0x72,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x39,0x03,0x00,0x00, -0x38,0x03,0x00,0x00,0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x3a,0x03,0x00,0x00,0x39,0x03,0x00,0x00,0x43,0x00,0x00,0x00, -0x6f,0x00,0x04,0x00,0x53,0x00,0x00,0x00,0x3b,0x03,0x00,0x00, -0x3a,0x03,0x00,0x00,0x85,0x00,0x05,0x00,0x53,0x00,0x00,0x00, -0x3c,0x03,0x00,0x00,0x25,0x03,0x00,0x00,0x3b,0x03,0x00,0x00, -0x0c,0x00,0x08,0x00,0x53,0x00,0x00,0x00,0x3d,0x03,0x00,0x00, -0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x00,0x03,0x00,0x00, -0x16,0x03,0x00,0x00,0x3c,0x03,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x42,0x03,0x00,0x00,0x99,0x00,0x00,0x00, -0x95,0x03,0x00,0x00,0x41,0x00,0x06,0x00,0x9d,0x00,0x00,0x00, -0x43,0x03,0x00,0x00,0x94,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0x42,0x03,0x00,0x00,0x3d,0x00,0x04,0x00,0x53,0x00,0x00,0x00, -0x44,0x03,0x00,0x00,0x43,0x03,0x00,0x00,0x3d,0x00,0x04,0x00, -0x77,0x00,0x00,0x00,0x48,0x03,0x00,0x00,0x0f,0x01,0x00,0x00, -0x6f,0x00,0x04,0x00,0x53,0x00,0x00,0x00,0x49,0x03,0x00,0x00, -0x48,0x03,0x00,0x00,0x85,0x00,0x05,0x00,0x53,0x00,0x00,0x00, -0x4a,0x03,0x00,0x00,0x44,0x03,0x00,0x00,0x49,0x03,0x00,0x00, -0x85,0x00,0x05,0x00,0x53,0x00,0x00,0x00,0x4b,0x03,0x00,0x00, -0x4a,0x03,0x00,0x00,0x87,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x72,0x00,0x00,0x00,0x50,0x03,0x00,0x00,0x04,0x03,0x00,0x00, -0xc2,0x00,0x05,0x00,0x72,0x00,0x00,0x00,0x51,0x03,0x00,0x00, -0x50,0x03,0x00,0x00,0x36,0x00,0x00,0x00,0x71,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x52,0x03,0x00,0x00,0x51,0x03,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x53,0x03,0x00,0x00, -0x52,0x03,0x00,0x00,0x3d,0x00,0x04,0x00,0x72,0x00,0x00,0x00, -0x57,0x03,0x00,0x00,0x0b,0x03,0x00,0x00,0xc2,0x00,0x05,0x00, -0x72,0x00,0x00,0x00,0x58,0x03,0x00,0x00,0x57,0x03,0x00,0x00, -0x36,0x00,0x00,0x00,0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x59,0x03,0x00,0x00,0x58,0x03,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x5a,0x03,0x00,0x00,0x59,0x03,0x00,0x00, -0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x5b,0x03,0x00,0x00, -0x5a,0x03,0x00,0x00,0x83,0x00,0x00,0x00,0xc4,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x5c,0x03,0x00,0x00,0x5b,0x03,0x00,0x00, -0x36,0x00,0x00,0x00,0xc5,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x5d,0x03,0x00,0x00,0x53,0x03,0x00,0x00,0x5c,0x03,0x00,0x00, -0x72,0x00,0x04,0x00,0x77,0x00,0x00,0x00,0x5e,0x03,0x00,0x00, -0x5d,0x03,0x00,0x00,0x72,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x5f,0x03,0x00,0x00,0x5e,0x03,0x00,0x00,0x82,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x60,0x03,0x00,0x00,0x5f,0x03,0x00,0x00, -0x43,0x00,0x00,0x00,0x6f,0x00,0x04,0x00,0x53,0x00,0x00,0x00, -0x61,0x03,0x00,0x00,0x60,0x03,0x00,0x00,0x0c,0x00,0x08,0x00, -0x53,0x00,0x00,0x00,0x63,0x03,0x00,0x00,0x01,0x00,0x00,0x00, -0x32,0x00,0x00,0x00,0x4b,0x03,0x00,0x00,0x61,0x03,0x00,0x00, -0x3d,0x03,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x68,0x03,0x00,0x00,0x99,0x00,0x00,0x00,0x96,0x03,0x00,0x00, -0x41,0x00,0x06,0x00,0x9d,0x00,0x00,0x00,0x69,0x03,0x00,0x00, -0x94,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x68,0x03,0x00,0x00, -0x3d,0x00,0x04,0x00,0x53,0x00,0x00,0x00,0x6a,0x03,0x00,0x00, -0x69,0x03,0x00,0x00,0x3d,0x00,0x04,0x00,0x77,0x00,0x00,0x00, -0x6e,0x03,0x00,0x00,0x45,0x01,0x00,0x00,0x6f,0x00,0x04,0x00, -0x53,0x00,0x00,0x00,0x6f,0x03,0x00,0x00,0x6e,0x03,0x00,0x00, -0x85,0x00,0x05,0x00,0x53,0x00,0x00,0x00,0x70,0x03,0x00,0x00, -0x6a,0x03,0x00,0x00,0x6f,0x03,0x00,0x00,0x85,0x00,0x05,0x00, -0x53,0x00,0x00,0x00,0x71,0x03,0x00,0x00,0x70,0x03,0x00,0x00, -0x87,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x72,0x00,0x00,0x00, -0x76,0x03,0x00,0x00,0x29,0x03,0x00,0x00,0xc2,0x00,0x05,0x00, -0x72,0x00,0x00,0x00,0x77,0x03,0x00,0x00,0x76,0x03,0x00,0x00, -0x36,0x00,0x00,0x00,0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x78,0x03,0x00,0x00,0x77,0x03,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x79,0x03,0x00,0x00,0x78,0x03,0x00,0x00, -0x3d,0x00,0x04,0x00,0x72,0x00,0x00,0x00,0x7d,0x03,0x00,0x00, -0x0b,0x03,0x00,0x00,0xc2,0x00,0x05,0x00,0x72,0x00,0x00,0x00, -0x7e,0x03,0x00,0x00,0x7d,0x03,0x00,0x00,0x43,0x01,0x00,0x00, -0x71,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x7f,0x03,0x00,0x00, -0x7e,0x03,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x80,0x03,0x00,0x00,0x7f,0x03,0x00,0x00,0xc7,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x81,0x03,0x00,0x00,0x80,0x03,0x00,0x00, -0x83,0x00,0x00,0x00,0xc4,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x82,0x03,0x00,0x00,0x81,0x03,0x00,0x00,0x36,0x00,0x00,0x00, -0xc5,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x83,0x03,0x00,0x00, -0x79,0x03,0x00,0x00,0x82,0x03,0x00,0x00,0x72,0x00,0x04,0x00, -0x77,0x00,0x00,0x00,0x84,0x03,0x00,0x00,0x83,0x03,0x00,0x00, -0x72,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x85,0x03,0x00,0x00, -0x84,0x03,0x00,0x00,0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x86,0x03,0x00,0x00,0x85,0x03,0x00,0x00,0x43,0x00,0x00,0x00, -0x6f,0x00,0x04,0x00,0x53,0x00,0x00,0x00,0x87,0x03,0x00,0x00, -0x86,0x03,0x00,0x00,0x0c,0x00,0x08,0x00,0x53,0x00,0x00,0x00, -0x89,0x03,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00, -0x71,0x03,0x00,0x00,0x87,0x03,0x00,0x00,0x63,0x03,0x00,0x00, -0x81,0x00,0x05,0x00,0x53,0x00,0x00,0x00,0x8a,0x03,0x00,0x00, -0xea,0x02,0x00,0x00,0x89,0x03,0x00,0x00,0x3d,0x00,0x04,0x00, -0x53,0x00,0x00,0x00,0x75,0x01,0x00,0x00,0x5f,0x00,0x00,0x00, -0x81,0x00,0x05,0x00,0x53,0x00,0x00,0x00,0x76,0x01,0x00,0x00, -0x75,0x01,0x00,0x00,0x8a,0x03,0x00,0x00,0x3e,0x00,0x03,0x00, -0x5f,0x00,0x00,0x00,0x76,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x79,0x01,0x00,0x00,0xa6,0x01,0x00,0x00, -0x25,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x62,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0x64,0x00,0x00,0x00,0xe0,0x00,0x04,0x00, -0x7a,0x01,0x00,0x00,0x7a,0x01,0x00,0x00,0x7b,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0x7d,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x7d,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xa7,0x01,0x00,0x00,0x58,0x00,0x00,0x00,0x64,0x00,0x00,0x00, -0x94,0x01,0x00,0x00,0x80,0x01,0x00,0x00,0xad,0x00,0x05,0x00, -0x69,0x00,0x00,0x00,0x83,0x01,0x00,0x00,0xa7,0x01,0x00,0x00, -0x16,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x7f,0x01,0x00,0x00, -0x80,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x83,0x01,0x00,0x00,0x7e,0x01,0x00,0x00,0x7f,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x7e,0x01,0x00,0x00,0xb1,0x00,0x05,0x00, -0x69,0x00,0x00,0x00,0x86,0x01,0x00,0x00,0x26,0x00,0x00,0x00, -0xa7,0x01,0x00,0x00,0xf7,0x00,0x03,0x00,0x88,0x01,0x00,0x00, -0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x86,0x01,0x00,0x00, -0x87,0x01,0x00,0x00,0x88,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x87,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x8c,0x01,0x00,0x00,0x26,0x00,0x00,0x00,0xa7,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0x5e,0x00,0x00,0x00,0x8d,0x01,0x00,0x00, -0x57,0x00,0x00,0x00,0x8c,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, -0x53,0x00,0x00,0x00,0x8e,0x01,0x00,0x00,0x8d,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0x5e,0x00,0x00,0x00,0x8f,0x01,0x00,0x00, -0x57,0x00,0x00,0x00,0x26,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x53,0x00,0x00,0x00,0x90,0x01,0x00,0x00,0x8f,0x01,0x00,0x00, -0x81,0x00,0x05,0x00,0x53,0x00,0x00,0x00,0x91,0x01,0x00,0x00, -0x90,0x01,0x00,0x00,0x8e,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0x8f,0x01,0x00,0x00,0x91,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0x88,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x88,0x01,0x00,0x00, -0xe0,0x00,0x04,0x00,0x7a,0x01,0x00,0x00,0x7a,0x01,0x00,0x00, -0x7b,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x80,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x80,0x01,0x00,0x00,0xc3,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x94,0x01,0x00,0x00,0xa7,0x01,0x00,0x00, -0x95,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x7d,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x7f,0x01,0x00,0x00,0xaa,0x00,0x05,0x00, -0x69,0x00,0x00,0x00,0x96,0x01,0x00,0x00,0x26,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0xf7,0x00,0x03,0x00,0x98,0x01,0x00,0x00, -0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x96,0x01,0x00,0x00, -0x97,0x01,0x00,0x00,0x98,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x97,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00, -0x9d,0x01,0x00,0x00,0x15,0x00,0x00,0x00,0x25,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x9e,0x01,0x00,0x00, -0x9d,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xa0,0x01,0x00,0x00,0x9e,0x01,0x00,0x00,0x11,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x5e,0x00,0x00,0x00,0xa1,0x01,0x00,0x00, -0x57,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x53,0x00,0x00,0x00,0xa2,0x01,0x00,0x00,0xa1,0x01,0x00,0x00, -0x41,0x00,0x06,0x00,0x9d,0x00,0x00,0x00,0xa3,0x01,0x00,0x00, -0x9c,0x01,0x00,0x00,0x16,0x00,0x00,0x00,0xa0,0x01,0x00,0x00, -0x3e,0x00,0x03,0x00,0xa3,0x01,0x00,0x00,0xa2,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0x98,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x98,0x01,0x00,0x00,0xfd,0x00,0x01,0x00,0x38,0x00,0x01,0x00, - -}; -const uint64_t mul_mat_vec_q6_K_f32_len = 10992; - -unsigned char mul_mat_vec_q8_0_f32_data[] = { -0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, -0xc9,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, +0xf0,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, 0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x51,0x11,0x00,0x00, 0x11,0x00,0x02,0x00,0x60,0x11,0x00,0x00,0x0b,0x00,0x06,0x00, 0x01,0x00,0x00,0x00,0x47,0x4c,0x53,0x4c,0x2e,0x73,0x74,0x64, 0x2e,0x34,0x35,0x30,0x00,0x00,0x00,0x00,0x0e,0x00,0x03,0x00, 0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x0f,0x00,0x0c,0x00, 0x05,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x6d,0x61,0x69,0x6e, -0x00,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x13,0x00,0x00,0x00, -0x1b,0x00,0x00,0x00,0x2a,0x00,0x00,0x00,0x54,0x00,0x00,0x00, -0x75,0x00,0x00,0x00,0xbc,0x00,0x00,0x00,0x10,0x00,0x06,0x00, -0x04,0x00,0x00,0x00,0x11,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x11,0x00,0x00,0x00, +0x18,0x00,0x00,0x00,0x26,0x00,0x00,0x00,0x52,0x00,0x00,0x00, +0x9f,0x00,0x00,0x00,0xe0,0x00,0x00,0x00,0x10,0x00,0x06,0x00, +0x04,0x00,0x00,0x00,0x11,0x00,0x00,0x00,0x01,0x00,0x00,0x00, 0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x0c,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x1a,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x1b,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x28,0x00,0x00,0x00, +0x0b,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x1a,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x11,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, +0x1b,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x15,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x24,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x24,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x24,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x47,0x00,0x03,0x00, +0x24,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x4d,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x4e,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x4e,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x4e,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x4e,0x00,0x00,0x00,0x03,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x4f,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x18,0x00,0x00,0x00, +0x48,0x00,0x04,0x00,0x50,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x50,0x00,0x00,0x00, 0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x28,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x28,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x08,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x28,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x4f,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x50,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x50,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x51,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x22,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0x52,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x52,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x52,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x54,0x00,0x00,0x00, -0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x54,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x72,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0x73,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x73,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x73,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x75,0x00,0x00,0x00, -0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x75,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0xb9,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0xba,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0xba,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0xba,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xbc,0x00,0x00,0x00, -0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0xbc,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0xc4,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, +0x47,0x00,0x03,0x00,0x50,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x52,0x00,0x00,0x00,0x22,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x52,0x00,0x00,0x00, +0x21,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x9c,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x48,0x00,0x04,0x00,0x9d,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x9d,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x03,0x00,0x9d,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x9f,0x00,0x00,0x00,0x22,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x9f,0x00,0x00,0x00, +0x21,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0xdd,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x48,0x00,0x04,0x00,0xde,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x19,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0xde,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x03,0x00,0xde,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0xe0,0x00,0x00,0x00,0x22,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xe0,0x00,0x00,0x00, +0x21,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0xe8,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0xe9,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, 0x19,0x00,0x00,0x00,0x13,0x00,0x02,0x00,0x02,0x00,0x00,0x00, 0x21,0x00,0x03,0x00,0x03,0x00,0x00,0x00,0x02,0x00,0x00,0x00, 0x15,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x15,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x17,0x00,0x04,0x00, -0x0a,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x03,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x0b,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x0a,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x0b,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x0e,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x0b,0x00,0x00,0x00, -0x13,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x16,0x00,0x03,0x00, -0x17,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x1c,0x00,0x04,0x00,0x19,0x00,0x00,0x00,0x17,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x1a,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x1a,0x00,0x00,0x00,0x1b,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x17,0x00,0x00,0x00,0x1d,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x1e,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x17,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x1e,0x00,0x05,0x00,0x28,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x29,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x28,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x29,0x00,0x00,0x00,0x2a,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x2b,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x14,0x00,0x02,0x00,0x30,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x35,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x44,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x16,0x00,0x03,0x00,0x4d,0x00,0x00,0x00, -0x10,0x00,0x00,0x00,0x15,0x00,0x04,0x00,0x4e,0x00,0x00,0x00, -0x08,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x1c,0x00,0x04,0x00, -0x4f,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x18,0x00,0x00,0x00, -0x1e,0x00,0x04,0x00,0x50,0x00,0x00,0x00,0x4d,0x00,0x00,0x00, -0x4f,0x00,0x00,0x00,0x1d,0x00,0x03,0x00,0x51,0x00,0x00,0x00, -0x50,0x00,0x00,0x00,0x1e,0x00,0x03,0x00,0x52,0x00,0x00,0x00, -0x51,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x53,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x52,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x53,0x00,0x00,0x00,0x54,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x56,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x4d,0x00,0x00,0x00,0x17,0x00,0x04,0x00,0x5a,0x00,0x00,0x00, -0x17,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x5f,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, -0x1d,0x00,0x03,0x00,0x72,0x00,0x00,0x00,0x17,0x00,0x00,0x00, -0x1e,0x00,0x03,0x00,0x73,0x00,0x00,0x00,0x72,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x74,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x73,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x74,0x00,0x00,0x00, -0x75,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x7d,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x17,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x86,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x99,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x9a,0x00,0x00,0x00,0x08,0x01,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x9c,0x00,0x00,0x00, -0x10,0x00,0x00,0x00,0x1d,0x00,0x03,0x00,0xb9,0x00,0x00,0x00, -0x17,0x00,0x00,0x00,0x1e,0x00,0x03,0x00,0xba,0x00,0x00,0x00, -0xb9,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0xbb,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0xba,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0xbb,0x00,0x00,0x00,0xbc,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x2c,0x00,0x06,0x00,0x0a,0x00,0x00,0x00,0xc4,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x17,0x00,0x04,0x00,0x09,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x0a,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x09,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x0d,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00,0x11,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x16,0x00,0x03,0x00,0x14,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x15,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x1c,0x00,0x04,0x00, +0x16,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x15,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x17,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x16,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x17,0x00,0x00,0x00, +0x18,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x14,0x00,0x00,0x00,0x1a,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x1b,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x14,0x00,0x00,0x00,0x1e,0x00,0x05,0x00,0x24,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x25,0x00,0x00,0x00,0x09,0x00,0x00,0x00, +0x24,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x25,0x00,0x00,0x00, +0x26,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x15,0x00,0x04,0x00, +0x27,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x27,0x00,0x00,0x00,0x28,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x29,0x00,0x00,0x00, +0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x14,0x00,0x02,0x00, +0x2d,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x32,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x3d,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0x16,0x00,0x03,0x00,0x4a,0x00,0x00,0x00,0x10,0x00,0x00,0x00, +0x15,0x00,0x04,0x00,0x4b,0x00,0x00,0x00,0x08,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x4c,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x1c,0x00,0x04,0x00, +0x4d,0x00,0x00,0x00,0x4b,0x00,0x00,0x00,0x4c,0x00,0x00,0x00, +0x1e,0x00,0x06,0x00,0x4e,0x00,0x00,0x00,0x4a,0x00,0x00,0x00, +0x4a,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x4d,0x00,0x00,0x00, +0x1d,0x00,0x03,0x00,0x4f,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, +0x1e,0x00,0x03,0x00,0x50,0x00,0x00,0x00,0x4f,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x51,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x50,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x51,0x00,0x00,0x00, +0x52,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x54,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x4a,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x27,0x00,0x00,0x00,0x5a,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x27,0x00,0x00,0x00, +0x60,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x61,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x27,0x00,0x00,0x00,0x6a,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x70,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x27,0x00,0x00,0x00,0x78,0x00,0x00,0x00,0x03,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x7a,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x4b,0x00,0x00,0x00,0x17,0x00,0x04,0x00,0x7e,0x00,0x00,0x00, +0x14,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x82,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x8c,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x1d,0x00,0x03,0x00,0x9c,0x00,0x00,0x00, +0x14,0x00,0x00,0x00,0x1e,0x00,0x03,0x00,0x9d,0x00,0x00,0x00, +0x9c,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x9e,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x9d,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0x9e,0x00,0x00,0x00,0x9f,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0xa7,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x14,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xbe,0x00,0x00,0x00,0x08,0x01,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xc0,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0x15,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, +0xdd,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, +0xde,0x00,0x00,0x00,0xdd,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0xdf,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0xde,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0xdf,0x00,0x00,0x00,0xe0,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xe8,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x33,0x00,0x06,0x00, +0x09,0x00,0x00,0x00,0xe9,0x00,0x00,0x00,0xe8,0x00,0x00,0x00, +0x8c,0x00,0x00,0x00,0x8c,0x00,0x00,0x00,0x36,0x00,0x05,0x00, +0x02,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x03,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x05,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00,0x0e,0x00,0x00,0x00, +0x0b,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x0e,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0x11,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x13,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x1b,0x00,0x00,0x00,0x1c,0x00,0x00,0x00, +0x18,0x00,0x00,0x00,0x13,0x00,0x00,0x00,0x3e,0x00,0x03,0x00, +0x1c,0x00,0x00,0x00,0x1a,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x1e,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x1e,0x00,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xee,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0xbd,0x00,0x00,0x00, +0x1f,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x29,0x00,0x00,0x00, +0x2a,0x00,0x00,0x00,0x26,0x00,0x00,0x00,0x28,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x2b,0x00,0x00,0x00, +0x2a,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x2c,0x00,0x00,0x00,0x2b,0x00,0x00,0x00,0x15,0x00,0x00,0x00, +0xb0,0x00,0x05,0x00,0x2d,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, +0xee,0x00,0x00,0x00,0x2c,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0x20,0x00,0x00,0x00,0x1f,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x2e,0x00,0x00,0x00,0x1f,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x1f,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x31,0x00,0x00,0x00, +0xee,0x00,0x00,0x00,0x15,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0x32,0x00,0x00,0x00, +0x13,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x35,0x00,0x00,0x00,0x31,0x00,0x00,0x00,0x34,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, +0x0f,0x00,0x00,0x00,0x2b,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x3c,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, +0x35,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x3e,0x00,0x00,0x00,0x3c,0x00,0x00,0x00,0x3d,0x00,0x00,0x00, +0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x41,0x00,0x00,0x00, +0x35,0x00,0x00,0x00,0x3d,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x42,0x00,0x00,0x00,0x41,0x00,0x00,0x00, +0x32,0x00,0x00,0x00,0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x47,0x00,0x00,0x00,0x35,0x00,0x00,0x00,0x41,0x00,0x00,0x00, +0x41,0x00,0x07,0x00,0x54,0x00,0x00,0x00,0x55,0x00,0x00,0x00, +0x52,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0x3e,0x00,0x00,0x00, +0x28,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, +0x56,0x00,0x00,0x00,0x55,0x00,0x00,0x00,0x73,0x00,0x04,0x00, +0x14,0x00,0x00,0x00,0x57,0x00,0x00,0x00,0x56,0x00,0x00,0x00, +0x41,0x00,0x07,0x00,0x54,0x00,0x00,0x00,0x5b,0x00,0x00,0x00, +0x52,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0x3e,0x00,0x00,0x00, +0x5a,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, +0x5c,0x00,0x00,0x00,0x5b,0x00,0x00,0x00,0x73,0x00,0x04,0x00, +0x14,0x00,0x00,0x00,0x5d,0x00,0x00,0x00,0x5c,0x00,0x00,0x00, +0x41,0x00,0x07,0x00,0x61,0x00,0x00,0x00,0x62,0x00,0x00,0x00, +0x52,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0x3e,0x00,0x00,0x00, +0x60,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x63,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0xc2,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x69,0x00,0x00,0x00,0x63,0x00,0x00,0x00, +0x42,0x00,0x00,0x00,0xc4,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x6b,0x00,0x00,0x00,0x69,0x00,0x00,0x00,0x6a,0x00,0x00,0x00, +0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, +0x6b,0x00,0x00,0x00,0x4c,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, +0x27,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x71,0x00,0x00,0x00, +0x42,0x00,0x00,0x00,0x70,0x00,0x00,0x00,0xc2,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x72,0x00,0x00,0x00,0x63,0x00,0x00,0x00, +0x71,0x00,0x00,0x00,0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x73,0x00,0x00,0x00,0x72,0x00,0x00,0x00,0x4c,0x00,0x00,0x00, +0x7c,0x00,0x04,0x00,0x27,0x00,0x00,0x00,0x74,0x00,0x00,0x00, +0x73,0x00,0x00,0x00,0x41,0x00,0x08,0x00,0x7a,0x00,0x00,0x00, +0x7b,0x00,0x00,0x00,0x52,0x00,0x00,0x00,0x28,0x00,0x00,0x00, +0x3e,0x00,0x00,0x00,0x78,0x00,0x00,0x00,0x42,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x4b,0x00,0x00,0x00,0x7c,0x00,0x00,0x00, +0x7b,0x00,0x00,0x00,0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x7d,0x00,0x00,0x00,0x7c,0x00,0x00,0x00,0xc7,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x83,0x00,0x00,0x00,0x7d,0x00,0x00,0x00, +0x82,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x87,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0xc5,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x88,0x00,0x00,0x00,0x83,0x00,0x00,0x00, +0x87,0x00,0x00,0x00,0x70,0x00,0x04,0x00,0x14,0x00,0x00,0x00, +0x89,0x00,0x00,0x00,0x88,0x00,0x00,0x00,0xc2,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x8b,0x00,0x00,0x00,0x7d,0x00,0x00,0x00, +0x6a,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x8f,0x00,0x00,0x00,0x74,0x00,0x00,0x00,0xc5,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x90,0x00,0x00,0x00,0x8b,0x00,0x00,0x00, +0x8f,0x00,0x00,0x00,0x70,0x00,0x04,0x00,0x14,0x00,0x00,0x00, +0x91,0x00,0x00,0x00,0x90,0x00,0x00,0x00,0x50,0x00,0x05,0x00, +0x7e,0x00,0x00,0x00,0x92,0x00,0x00,0x00,0x89,0x00,0x00,0x00, +0x91,0x00,0x00,0x00,0x8e,0x00,0x05,0x00,0x7e,0x00,0x00,0x00, +0x95,0x00,0x00,0x00,0x92,0x00,0x00,0x00,0x57,0x00,0x00,0x00, +0x50,0x00,0x05,0x00,0x7e,0x00,0x00,0x00,0x97,0x00,0x00,0x00, +0x5d,0x00,0x00,0x00,0x5d,0x00,0x00,0x00,0x81,0x00,0x05,0x00, +0x7e,0x00,0x00,0x00,0x98,0x00,0x00,0x00,0x95,0x00,0x00,0x00, +0x97,0x00,0x00,0x00,0x51,0x00,0x05,0x00,0x14,0x00,0x00,0x00, +0x9b,0x00,0x00,0x00,0x98,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x29,0x00,0x00,0x00,0xa0,0x00,0x00,0x00, +0x26,0x00,0x00,0x00,0x5a,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xa1,0x00,0x00,0x00,0xa0,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa3,0x00,0x00,0x00, +0xa1,0x00,0x00,0x00,0x47,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xa5,0x00,0x00,0x00,0xa3,0x00,0x00,0x00, +0x42,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0xa7,0x00,0x00,0x00, +0xa8,0x00,0x00,0x00,0x9f,0x00,0x00,0x00,0x28,0x00,0x00,0x00, +0xa5,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x14,0x00,0x00,0x00, +0xa9,0x00,0x00,0x00,0xa8,0x00,0x00,0x00,0x51,0x00,0x05,0x00, +0x14,0x00,0x00,0x00,0xac,0x00,0x00,0x00,0x98,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xb3,0x00,0x00,0x00,0xa5,0x00,0x00,0x00,0x4c,0x00,0x00,0x00, +0x41,0x00,0x06,0x00,0xa7,0x00,0x00,0x00,0xb4,0x00,0x00,0x00, +0x9f,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0xb3,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0xb5,0x00,0x00,0x00, +0xb4,0x00,0x00,0x00,0x85,0x00,0x05,0x00,0x14,0x00,0x00,0x00, +0xb6,0x00,0x00,0x00,0xac,0x00,0x00,0x00,0xb5,0x00,0x00,0x00, +0x0c,0x00,0x08,0x00,0x14,0x00,0x00,0x00,0xb7,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x9b,0x00,0x00,0x00, +0xa9,0x00,0x00,0x00,0xb6,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x14,0x00,0x00,0x00,0xb9,0x00,0x00,0x00,0x1c,0x00,0x00,0x00, +0x81,0x00,0x05,0x00,0x14,0x00,0x00,0x00,0xba,0x00,0x00,0x00, +0xb9,0x00,0x00,0x00,0xb7,0x00,0x00,0x00,0x3e,0x00,0x03,0x00, +0x1c,0x00,0x00,0x00,0xba,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xbd,0x00,0x00,0x00,0xee,0x00,0x00,0x00, +0x32,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x1e,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0x20,0x00,0x00,0x00,0xe0,0x00,0x04,0x00, +0x32,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0xbe,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xc1,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xc1,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xef,0x00,0x00,0x00,0xc0,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0xd8,0x00,0x00,0x00,0xc4,0x00,0x00,0x00,0xac,0x00,0x05,0x00, +0x2d,0x00,0x00,0x00,0xc7,0x00,0x00,0x00,0xef,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xc3,0x00,0x00,0x00, +0xc4,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0xc7,0x00,0x00,0x00,0xc2,0x00,0x00,0x00,0xc3,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xc2,0x00,0x00,0x00,0xb0,0x00,0x05,0x00, +0x2d,0x00,0x00,0x00,0xca,0x00,0x00,0x00,0x13,0x00,0x00,0x00, +0xef,0x00,0x00,0x00,0xf7,0x00,0x03,0x00,0xcc,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xca,0x00,0x00,0x00, +0xcb,0x00,0x00,0x00,0xcc,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xcb,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xd0,0x00,0x00,0x00,0x13,0x00,0x00,0x00,0xef,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x1b,0x00,0x00,0x00,0xd1,0x00,0x00,0x00, +0x18,0x00,0x00,0x00,0xd0,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x14,0x00,0x00,0x00,0xd2,0x00,0x00,0x00,0xd1,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0xd4,0x00,0x00,0x00, +0x1c,0x00,0x00,0x00,0x81,0x00,0x05,0x00,0x14,0x00,0x00,0x00, +0xd5,0x00,0x00,0x00,0xd4,0x00,0x00,0x00,0xd2,0x00,0x00,0x00, +0x3e,0x00,0x03,0x00,0x1c,0x00,0x00,0x00,0xd5,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xcc,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xcc,0x00,0x00,0x00,0xe0,0x00,0x04,0x00,0x32,0x00,0x00,0x00, +0x32,0x00,0x00,0x00,0xbe,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xc4,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xc4,0x00,0x00,0x00, +0xc2,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xd8,0x00,0x00,0x00, +0xef,0x00,0x00,0x00,0x5a,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xc1,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xc3,0x00,0x00,0x00, +0xaa,0x00,0x05,0x00,0x2d,0x00,0x00,0x00,0xda,0x00,0x00,0x00, +0x13,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0xf7,0x00,0x03,0x00, +0xdc,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0xda,0x00,0x00,0x00,0xdb,0x00,0x00,0x00,0xdc,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xdb,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x29,0x00,0x00,0x00,0xe1,0x00,0x00,0x00,0x26,0x00,0x00,0x00, +0x60,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xe2,0x00,0x00,0x00,0xe1,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xe4,0x00,0x00,0x00,0xe2,0x00,0x00,0x00, +0x0f,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x1b,0x00,0x00,0x00, +0xe5,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x28,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0xe6,0x00,0x00,0x00, +0xe5,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0xa7,0x00,0x00,0x00, +0xe7,0x00,0x00,0x00,0xe0,0x00,0x00,0x00,0x28,0x00,0x00,0x00, +0xe4,0x00,0x00,0x00,0x3e,0x00,0x03,0x00,0xe7,0x00,0x00,0x00, +0xe6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xdc,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xdc,0x00,0x00,0x00,0xfd,0x00,0x01,0x00, +0x38,0x00,0x01,0x00, +}; +const uint64_t mul_mat_vec_q5_1_f32_len = 3592; + +unsigned char mul_mat_vec_q5_K_f32_data[] = { +0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, +0x8a,0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, +0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x27,0x00,0x00,0x00, +0x11,0x00,0x02,0x00,0x51,0x11,0x00,0x00,0x11,0x00,0x02,0x00, +0x60,0x11,0x00,0x00,0x0b,0x00,0x06,0x00,0x01,0x00,0x00,0x00, +0x47,0x4c,0x53,0x4c,0x2e,0x73,0x74,0x64,0x2e,0x34,0x35,0x30, +0x00,0x00,0x00,0x00,0x0e,0x00,0x03,0x00,0x00,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x0f,0x00,0x0c,0x00,0x05,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x6d,0x61,0x69,0x6e,0x00,0x00,0x00,0x00, +0x0b,0x00,0x00,0x00,0x13,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0x5b,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0xf7,0x01,0x00,0x00, +0x7c,0x04,0x00,0x00,0x10,0x00,0x06,0x00,0x04,0x00,0x00,0x00, +0x11,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x0b,0x00,0x00,0x00, +0x0b,0x00,0x00,0x00,0x1a,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x11,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x11,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x11,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x47,0x00,0x03,0x00, +0x11,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x20,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x1b,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x7d,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x7e,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x7f,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x80,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x80,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x80,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x10,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x80,0x00,0x00,0x00,0x03,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x30,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x81,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0xb0,0x00,0x00,0x00, +0x48,0x00,0x04,0x00,0x82,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x82,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x03,0x00,0x82,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x84,0x00,0x00,0x00,0x22,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x84,0x00,0x00,0x00, +0x21,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0xf4,0x01,0x00,0x00,0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x48,0x00,0x04,0x00,0xf5,0x01,0x00,0x00,0x00,0x00,0x00,0x00, +0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0xf5,0x01,0x00,0x00, +0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x03,0x00,0xf5,0x01,0x00,0x00,0x02,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0xf7,0x01,0x00,0x00,0x22,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xf7,0x01,0x00,0x00, +0x21,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x79,0x04,0x00,0x00,0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x48,0x00,0x04,0x00,0x7a,0x04,0x00,0x00,0x00,0x00,0x00,0x00, +0x19,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x7a,0x04,0x00,0x00, +0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x03,0x00,0x7a,0x04,0x00,0x00,0x02,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x7c,0x04,0x00,0x00,0x22,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x7c,0x04,0x00,0x00, +0x21,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x84,0x04,0x00,0x00,0x0b,0x00,0x00,0x00,0x19,0x00,0x00,0x00, +0x13,0x00,0x02,0x00,0x02,0x00,0x00,0x00,0x21,0x00,0x03,0x00, +0x03,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x15,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x17,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x03,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x0a,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0x0a,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x0d,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x1e,0x00,0x05,0x00, +0x11,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x12,0x00,0x00,0x00, +0x09,0x00,0x00,0x00,0x11,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0x12,0x00,0x00,0x00,0x13,0x00,0x00,0x00,0x09,0x00,0x00,0x00, +0x15,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x14,0x00,0x00,0x00, +0x15,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x16,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x19,0x00,0x00,0x00, +0x00,0x01,0x00,0x00,0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x2b,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x3f,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x45,0x00,0x00,0x00,0x40,0x00,0x00,0x00, +0x15,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0x08,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x14,0x00,0x00,0x00, +0x4d,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x15,0x00,0x04,0x00, +0x51,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0x56,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x16,0x00,0x03,0x00,0x58,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0x1c,0x00,0x04,0x00,0x59,0x00,0x00,0x00, +0x58,0x00,0x00,0x00,0x3f,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x5a,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x59,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0x5a,0x00,0x00,0x00,0x5b,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x5c,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x58,0x00,0x00,0x00,0x61,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x62,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x58,0x00,0x00,0x00,0x14,0x00,0x02,0x00,0x6d,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x76,0x00,0x00,0x00, +0x80,0x00,0x00,0x00,0x16,0x00,0x03,0x00,0x7a,0x00,0x00,0x00, +0x10,0x00,0x00,0x00,0x17,0x00,0x04,0x00,0x7b,0x00,0x00,0x00, +0x7a,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x7c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x1c,0x00,0x04,0x00,0x7d,0x00,0x00,0x00,0x4a,0x00,0x00,0x00, +0x7c,0x00,0x00,0x00,0x1c,0x00,0x04,0x00,0x7e,0x00,0x00,0x00, +0x4a,0x00,0x00,0x00,0x3f,0x00,0x00,0x00,0x1c,0x00,0x04,0x00, +0x7f,0x00,0x00,0x00,0x4a,0x00,0x00,0x00,0x76,0x00,0x00,0x00, +0x1e,0x00,0x06,0x00,0x80,0x00,0x00,0x00,0x7b,0x00,0x00,0x00, +0x7d,0x00,0x00,0x00,0x7e,0x00,0x00,0x00,0x7f,0x00,0x00,0x00, +0x1d,0x00,0x03,0x00,0x81,0x00,0x00,0x00,0x80,0x00,0x00,0x00, +0x1e,0x00,0x03,0x00,0x82,0x00,0x00,0x00,0x81,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x83,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x82,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x83,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x88,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x7a,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x90,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x9a,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x4a,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x14,0x00,0x00,0x00,0x9f,0x00,0x00,0x00,0x3f,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xc5,0x00,0x00,0x00, +0x05,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xd4,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x14,0x00,0x00,0x00,0xda,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0xe5,0x00,0x00,0x00, +0xc0,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x14,0x00,0x00,0x00, +0xe7,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xf2,0x00,0x00,0x00,0x09,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0x44,0x01,0x00,0x00, +0x03,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x6c,0x01,0x00,0x00,0x11,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xaa,0x01,0x00,0x00,0x41,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xb8,0x01,0x00,0x00, +0x50,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xc6,0x01,0x00,0x00,0x51,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, +0xf4,0x01,0x00,0x00,0x58,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, +0xf5,0x01,0x00,0x00,0xf4,0x01,0x00,0x00,0x20,0x00,0x04,0x00, +0xf6,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0xf5,0x01,0x00,0x00, +0x3b,0x00,0x04,0x00,0xf6,0x01,0x00,0x00,0xf7,0x01,0x00,0x00, +0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0xfc,0x01,0x00,0x00, +0x0c,0x00,0x00,0x00,0x58,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x14,0x00,0x00,0x00,0x0d,0x02,0x00,0x00,0x10,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x82,0x02,0x00,0x00, +0x21,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x9f,0x02,0x00,0x00,0x30,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xbc,0x02,0x00,0x00,0x31,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x5b,0x04,0x00,0x00, +0x08,0x01,0x00,0x00,0x1d,0x00,0x03,0x00,0x79,0x04,0x00,0x00, +0x58,0x00,0x00,0x00,0x1e,0x00,0x03,0x00,0x7a,0x04,0x00,0x00, +0x79,0x04,0x00,0x00,0x20,0x00,0x04,0x00,0x7b,0x04,0x00,0x00, +0x0c,0x00,0x00,0x00,0x7a,0x04,0x00,0x00,0x3b,0x00,0x04,0x00, +0x7b,0x04,0x00,0x00,0x7c,0x04,0x00,0x00,0x0c,0x00,0x00,0x00, +0x2c,0x00,0x06,0x00,0x09,0x00,0x00,0x00,0x84,0x04,0x00,0x00, +0x3f,0x00,0x00,0x00,0x90,0x00,0x00,0x00,0x90,0x00,0x00,0x00, 0x36,0x00,0x05,0x00,0x02,0x00,0x00,0x00,0x04,0x00,0x00,0x00, 0x00,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x05,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x0e,0x00,0x00,0x00, -0x0f,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x0d,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x10,0x00,0x00,0x00, -0x0f,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x11,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x0e,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x13,0x00,0x00,0x00, -0x0d,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x15,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x15,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x1e,0x00,0x00,0x00,0x1f,0x00,0x00,0x00, -0x1b,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x3e,0x00,0x03,0x00, -0x1f,0x00,0x00,0x00,0x1d,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x22,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x22,0x00,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xc7,0x00,0x00,0x00, -0x21,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0x98,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x2b,0x00,0x00,0x00, -0x2c,0x00,0x00,0x00,0x2a,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x2d,0x00,0x00,0x00, -0x2c,0x00,0x00,0x00,0x87,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x2f,0x00,0x00,0x00,0x2d,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0xb1,0x00,0x05,0x00,0x30,0x00,0x00,0x00,0x31,0x00,0x00,0x00, -0xc7,0x00,0x00,0x00,0x2f,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0x24,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x31,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x24,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x23,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x34,0x00,0x00,0x00, -0xc7,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x35,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x38,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0x37,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3d,0x00,0x00,0x00, -0x11,0x00,0x00,0x00,0x2d,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x3f,0x00,0x00,0x00,0x3d,0x00,0x00,0x00, -0x38,0x00,0x00,0x00,0x87,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0x3f,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x8b,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x43,0x00,0x00,0x00, -0x38,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x87,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x45,0x00,0x00,0x00,0x43,0x00,0x00,0x00, -0x44,0x00,0x00,0x00,0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x4a,0x00,0x00,0x00,0x38,0x00,0x00,0x00,0x43,0x00,0x00,0x00, -0x41,0x00,0x07,0x00,0x56,0x00,0x00,0x00,0x57,0x00,0x00,0x00, -0x54,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x21,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0x58,0x00,0x00,0x00,0x57,0x00,0x00,0x00,0x73,0x00,0x04,0x00, -0x17,0x00,0x00,0x00,0x59,0x00,0x00,0x00,0x58,0x00,0x00,0x00, -0x41,0x00,0x08,0x00,0x5f,0x00,0x00,0x00,0x60,0x00,0x00,0x00, -0x54,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x44,0x00,0x00,0x00,0x45,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x4e,0x00,0x00,0x00,0x61,0x00,0x00,0x00,0x60,0x00,0x00,0x00, -0x72,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x62,0x00,0x00,0x00, -0x61,0x00,0x00,0x00,0x6f,0x00,0x04,0x00,0x17,0x00,0x00,0x00, -0x63,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x66,0x00,0x00,0x00,0x45,0x00,0x00,0x00, -0x44,0x00,0x00,0x00,0x41,0x00,0x08,0x00,0x5f,0x00,0x00,0x00, -0x67,0x00,0x00,0x00,0x54,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0x44,0x00,0x00,0x00,0x66,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x4e,0x00,0x00,0x00,0x68,0x00,0x00,0x00, -0x67,0x00,0x00,0x00,0x72,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x69,0x00,0x00,0x00,0x68,0x00,0x00,0x00,0x6f,0x00,0x04,0x00, -0x17,0x00,0x00,0x00,0x6a,0x00,0x00,0x00,0x69,0x00,0x00,0x00, -0x50,0x00,0x05,0x00,0x5a,0x00,0x00,0x00,0x6b,0x00,0x00,0x00, -0x63,0x00,0x00,0x00,0x6a,0x00,0x00,0x00,0x8e,0x00,0x05,0x00, -0x5a,0x00,0x00,0x00,0x6e,0x00,0x00,0x00,0x6b,0x00,0x00,0x00, -0x59,0x00,0x00,0x00,0x51,0x00,0x05,0x00,0x17,0x00,0x00,0x00, -0x71,0x00,0x00,0x00,0x6e,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x2b,0x00,0x00,0x00,0x76,0x00,0x00,0x00, -0x2a,0x00,0x00,0x00,0x44,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x77,0x00,0x00,0x00,0x76,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x79,0x00,0x00,0x00, -0x77,0x00,0x00,0x00,0x4a,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x7b,0x00,0x00,0x00,0x79,0x00,0x00,0x00, -0x45,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x7d,0x00,0x00,0x00, -0x7e,0x00,0x00,0x00,0x75,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x7b,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x17,0x00,0x00,0x00, -0x7f,0x00,0x00,0x00,0x7e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x17,0x00,0x00,0x00,0x82,0x00,0x00,0x00,0x1f,0x00,0x00,0x00, -0x0c,0x00,0x08,0x00,0x17,0x00,0x00,0x00,0x83,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x71,0x00,0x00,0x00, -0x7f,0x00,0x00,0x00,0x82,0x00,0x00,0x00,0x3e,0x00,0x03,0x00, -0x1f,0x00,0x00,0x00,0x83,0x00,0x00,0x00,0x51,0x00,0x05,0x00, -0x17,0x00,0x00,0x00,0x88,0x00,0x00,0x00,0x6e,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x8f,0x00,0x00,0x00,0x7b,0x00,0x00,0x00,0x44,0x00,0x00,0x00, -0x41,0x00,0x06,0x00,0x7d,0x00,0x00,0x00,0x90,0x00,0x00,0x00, -0x75,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x8f,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x17,0x00,0x00,0x00,0x91,0x00,0x00,0x00, -0x90,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x17,0x00,0x00,0x00, -0x94,0x00,0x00,0x00,0x1f,0x00,0x00,0x00,0x0c,0x00,0x08,0x00, -0x17,0x00,0x00,0x00,0x95,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x32,0x00,0x00,0x00,0x88,0x00,0x00,0x00,0x91,0x00,0x00,0x00, -0x94,0x00,0x00,0x00,0x3e,0x00,0x03,0x00,0x1f,0x00,0x00,0x00, -0x95,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x98,0x00,0x00,0x00,0xc7,0x00,0x00,0x00,0x35,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x22,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x24,0x00,0x00,0x00,0xe0,0x00,0x04,0x00,0x99,0x00,0x00,0x00, -0x99,0x00,0x00,0x00,0x9a,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x9d,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x9d,0x00,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xc8,0x00,0x00,0x00, -0x9c,0x00,0x00,0x00,0x24,0x00,0x00,0x00,0xb4,0x00,0x00,0x00, -0xa0,0x00,0x00,0x00,0xad,0x00,0x05,0x00,0x30,0x00,0x00,0x00, -0xa3,0x00,0x00,0x00,0xc8,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0x9f,0x00,0x00,0x00,0xa0,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xa3,0x00,0x00,0x00, -0x9e,0x00,0x00,0x00,0x9f,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x9e,0x00,0x00,0x00,0xb1,0x00,0x05,0x00,0x30,0x00,0x00,0x00, -0xa6,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0xc8,0x00,0x00,0x00, -0xf7,0x00,0x03,0x00,0xa8,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xa6,0x00,0x00,0x00,0xa7,0x00,0x00,0x00, -0xa8,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xa7,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xac,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0xc8,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x1e,0x00,0x00,0x00,0xad,0x00,0x00,0x00,0x1b,0x00,0x00,0x00, -0xac,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x17,0x00,0x00,0x00, -0xae,0x00,0x00,0x00,0xad,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x17,0x00,0x00,0x00,0xb0,0x00,0x00,0x00,0x1f,0x00,0x00,0x00, -0x81,0x00,0x05,0x00,0x17,0x00,0x00,0x00,0xb1,0x00,0x00,0x00, -0xb0,0x00,0x00,0x00,0xae,0x00,0x00,0x00,0x3e,0x00,0x03,0x00, -0x1f,0x00,0x00,0x00,0xb1,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xa8,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xa8,0x00,0x00,0x00, -0xe0,0x00,0x04,0x00,0x99,0x00,0x00,0x00,0x99,0x00,0x00,0x00, -0x9a,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xa0,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xa0,0x00,0x00,0x00,0xc3,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xb4,0x00,0x00,0x00,0xc8,0x00,0x00,0x00, -0x44,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x9d,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0x9f,0x00,0x00,0x00,0xaa,0x00,0x05,0x00, -0x30,0x00,0x00,0x00,0xb6,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0x21,0x00,0x00,0x00,0xf7,0x00,0x03,0x00,0xb8,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xb6,0x00,0x00,0x00, -0xb7,0x00,0x00,0x00,0xb8,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xb7,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x2b,0x00,0x00,0x00, -0xbd,0x00,0x00,0x00,0x2a,0x00,0x00,0x00,0x35,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xbe,0x00,0x00,0x00, -0xbd,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xc0,0x00,0x00,0x00,0xbe,0x00,0x00,0x00,0x11,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x1e,0x00,0x00,0x00,0xc1,0x00,0x00,0x00, -0x1b,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x17,0x00,0x00,0x00,0xc2,0x00,0x00,0x00,0xc1,0x00,0x00,0x00, -0x41,0x00,0x06,0x00,0x7d,0x00,0x00,0x00,0xc3,0x00,0x00,0x00, -0xbc,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0xc0,0x00,0x00,0x00, -0x3e,0x00,0x03,0x00,0xc3,0x00,0x00,0x00,0xc2,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xb8,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xb8,0x00,0x00,0x00,0xfd,0x00,0x01,0x00,0x38,0x00,0x01,0x00, - +0x05,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00, +0x0e,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, +0x0e,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x16,0x00,0x00,0x00, +0x17,0x00,0x00,0x00,0x13,0x00,0x00,0x00,0x15,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x18,0x00,0x00,0x00, +0x17,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x1a,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x19,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x1e,0x00,0x00,0x00, +0x0f,0x00,0x00,0x00,0x1a,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x0d,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x22,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x24,0x00,0x00,0x00,0x22,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x28,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x2c,0x00,0x00,0x00, +0x24,0x00,0x00,0x00,0x2b,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x30,0x00,0x00,0x00,0x2b,0x00,0x00,0x00, +0x2c,0x00,0x00,0x00,0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x31,0x00,0x00,0x00,0x24,0x00,0x00,0x00,0x30,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x34,0x00,0x00,0x00, +0x2c,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x89,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x2c,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x3a,0x00,0x00,0x00,0x2b,0x00,0x00,0x00,0x31,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3c,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x3d,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, +0x3c,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x41,0x00,0x00,0x00,0x3f,0x00,0x00,0x00,0x34,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x43,0x00,0x00,0x00, +0x41,0x00,0x00,0x00,0x3d,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x47,0x00,0x00,0x00,0x45,0x00,0x00,0x00, +0x34,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x49,0x00,0x00,0x00,0x47,0x00,0x00,0x00,0x3d,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x4f,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0xc4,0x00,0x05,0x00, +0x14,0x00,0x00,0x00,0x50,0x00,0x00,0x00,0x4d,0x00,0x00,0x00, +0x4f,0x00,0x00,0x00,0x72,0x00,0x04,0x00,0x51,0x00,0x00,0x00, +0x52,0x00,0x00,0x00,0x50,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, +0x4a,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x52,0x00,0x00,0x00, +0xc4,0x00,0x05,0x00,0x4a,0x00,0x00,0x00,0x57,0x00,0x00,0x00, +0x53,0x00,0x00,0x00,0x56,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x5e,0x00,0x00,0x00,0x5c,0x00,0x00,0x00, +0x28,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x60,0x00,0x00,0x00,0x5e,0x00,0x00,0x00,0x24,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x62,0x00,0x00,0x00,0x63,0x00,0x00,0x00, +0x5b,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x3e,0x00,0x03,0x00, +0x63,0x00,0x00,0x00,0x61,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x66,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x66,0x00,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x85,0x04,0x00,0x00, +0x28,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0x5a,0x04,0x00,0x00, +0x67,0x00,0x00,0x00,0xb0,0x00,0x05,0x00,0x6d,0x00,0x00,0x00, +0x6e,0x00,0x00,0x00,0x85,0x04,0x00,0x00,0x1a,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0x68,0x00,0x00,0x00,0x67,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x6e,0x00,0x00,0x00, +0x67,0x00,0x00,0x00,0x68,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0x67,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x71,0x00,0x00,0x00,0x85,0x04,0x00,0x00,0x19,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x73,0x00,0x00,0x00, +0x71,0x00,0x00,0x00,0x49,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x77,0x00,0x00,0x00,0x73,0x00,0x00,0x00, +0x76,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x87,0x00,0x00,0x00,0x1e,0x00,0x00,0x00,0x85,0x04,0x00,0x00, +0x41,0x00,0x08,0x00,0x88,0x00,0x00,0x00,0x89,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0x15,0x00,0x00,0x00,0x87,0x00,0x00,0x00, +0x15,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x7a,0x00,0x00,0x00,0x8a,0x00,0x00,0x00,0x89,0x00,0x00,0x00, +0x73,0x00,0x04,0x00,0x58,0x00,0x00,0x00,0x8b,0x00,0x00,0x00, +0x8a,0x00,0x00,0x00,0x41,0x00,0x08,0x00,0x88,0x00,0x00,0x00, +0x91,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x15,0x00,0x00,0x00, +0x87,0x00,0x00,0x00,0x15,0x00,0x00,0x00,0x90,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x7a,0x00,0x00,0x00,0x92,0x00,0x00,0x00, +0x91,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x58,0x00,0x00,0x00, +0x93,0x00,0x00,0x00,0x92,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x99,0x00,0x00,0x00,0x34,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x41,0x00,0x08,0x00,0x9a,0x00,0x00,0x00, +0x9b,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x15,0x00,0x00,0x00, +0x87,0x00,0x00,0x00,0x4d,0x00,0x00,0x00,0x99,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0x9c,0x00,0x00,0x00, +0x9b,0x00,0x00,0x00,0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x9d,0x00,0x00,0x00,0x9c,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, +0x14,0x00,0x00,0x00,0x9e,0x00,0x00,0x00,0x9d,0x00,0x00,0x00, +0xc7,0x00,0x05,0x00,0x14,0x00,0x00,0x00,0xa0,0x00,0x00,0x00, +0x9e,0x00,0x00,0x00,0x9f,0x00,0x00,0x00,0x72,0x00,0x04,0x00, +0x51,0x00,0x00,0x00,0xa1,0x00,0x00,0x00,0xa0,0x00,0x00,0x00, +0x7c,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0xa2,0x00,0x00,0x00, +0xa1,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xa9,0x00,0x00,0x00,0x99,0x00,0x00,0x00,0x90,0x00,0x00,0x00, +0x41,0x00,0x08,0x00,0x9a,0x00,0x00,0x00,0xaa,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0x15,0x00,0x00,0x00,0x87,0x00,0x00,0x00, +0x4d,0x00,0x00,0x00,0xa9,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x4a,0x00,0x00,0x00,0xab,0x00,0x00,0x00,0xaa,0x00,0x00,0x00, +0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xac,0x00,0x00,0x00, +0xab,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x14,0x00,0x00,0x00, +0xad,0x00,0x00,0x00,0xac,0x00,0x00,0x00,0xc7,0x00,0x05,0x00, +0x14,0x00,0x00,0x00,0xae,0x00,0x00,0x00,0xad,0x00,0x00,0x00, +0x9f,0x00,0x00,0x00,0x72,0x00,0x04,0x00,0x51,0x00,0x00,0x00, +0xaf,0x00,0x00,0x00,0xae,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, +0x4a,0x00,0x00,0x00,0xb0,0x00,0x00,0x00,0xaf,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xb7,0x00,0x00,0x00, +0x99,0x00,0x00,0x00,0x2b,0x00,0x00,0x00,0x41,0x00,0x08,0x00, +0x9a,0x00,0x00,0x00,0xb8,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0x15,0x00,0x00,0x00,0x87,0x00,0x00,0x00,0x4d,0x00,0x00,0x00, +0xb7,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, +0xb9,0x00,0x00,0x00,0xb8,0x00,0x00,0x00,0x71,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xba,0x00,0x00,0x00,0xb9,0x00,0x00,0x00, +0x7c,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0xbb,0x00,0x00,0x00, +0xba,0x00,0x00,0x00,0xc7,0x00,0x05,0x00,0x14,0x00,0x00,0x00, +0xbc,0x00,0x00,0x00,0xbb,0x00,0x00,0x00,0x9f,0x00,0x00,0x00, +0x72,0x00,0x04,0x00,0x51,0x00,0x00,0x00,0xbd,0x00,0x00,0x00, +0xbc,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, +0xbe,0x00,0x00,0x00,0xbd,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xc6,0x00,0x00,0x00,0x99,0x00,0x00,0x00, +0xc5,0x00,0x00,0x00,0x41,0x00,0x08,0x00,0x9a,0x00,0x00,0x00, +0xc7,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x15,0x00,0x00,0x00, +0x87,0x00,0x00,0x00,0x4d,0x00,0x00,0x00,0xc6,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0xc8,0x00,0x00,0x00, +0xc7,0x00,0x00,0x00,0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xc9,0x00,0x00,0x00,0xc8,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, +0x14,0x00,0x00,0x00,0xca,0x00,0x00,0x00,0xc9,0x00,0x00,0x00, +0xc7,0x00,0x05,0x00,0x14,0x00,0x00,0x00,0xcb,0x00,0x00,0x00, +0xca,0x00,0x00,0x00,0x9f,0x00,0x00,0x00,0x72,0x00,0x04,0x00, +0x51,0x00,0x00,0x00,0xcc,0x00,0x00,0x00,0xcb,0x00,0x00,0x00, +0x7c,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0xcd,0x00,0x00,0x00, +0xcc,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xd5,0x00,0x00,0x00,0x99,0x00,0x00,0x00,0xd4,0x00,0x00,0x00, +0x41,0x00,0x08,0x00,0x9a,0x00,0x00,0x00,0xd6,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0x15,0x00,0x00,0x00,0x87,0x00,0x00,0x00, +0x4d,0x00,0x00,0x00,0xd5,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x4a,0x00,0x00,0x00,0xd7,0x00,0x00,0x00,0xd6,0x00,0x00,0x00, +0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xd8,0x00,0x00,0x00, +0xd7,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x14,0x00,0x00,0x00, +0xd9,0x00,0x00,0x00,0xd8,0x00,0x00,0x00,0xc7,0x00,0x05,0x00, +0x14,0x00,0x00,0x00,0xdb,0x00,0x00,0x00,0xd9,0x00,0x00,0x00, +0xda,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, +0xe2,0x00,0x00,0x00,0x9b,0x00,0x00,0x00,0x71,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xe3,0x00,0x00,0x00,0xe2,0x00,0x00,0x00, +0x7c,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0xe4,0x00,0x00,0x00, +0xe3,0x00,0x00,0x00,0xc7,0x00,0x05,0x00,0x14,0x00,0x00,0x00, +0xe6,0x00,0x00,0x00,0xe4,0x00,0x00,0x00,0xe5,0x00,0x00,0x00, +0xc3,0x00,0x05,0x00,0x14,0x00,0x00,0x00,0xe8,0x00,0x00,0x00, +0xe6,0x00,0x00,0x00,0xe7,0x00,0x00,0x00,0xc5,0x00,0x05,0x00, +0x14,0x00,0x00,0x00,0xe9,0x00,0x00,0x00,0xdb,0x00,0x00,0x00, +0xe8,0x00,0x00,0x00,0x72,0x00,0x04,0x00,0x51,0x00,0x00,0x00, +0xea,0x00,0x00,0x00,0xe9,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, +0x4a,0x00,0x00,0x00,0xeb,0x00,0x00,0x00,0xea,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xf3,0x00,0x00,0x00, +0x99,0x00,0x00,0x00,0xf2,0x00,0x00,0x00,0x41,0x00,0x08,0x00, +0x9a,0x00,0x00,0x00,0xf4,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0x15,0x00,0x00,0x00,0x87,0x00,0x00,0x00,0x4d,0x00,0x00,0x00, +0xf3,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, +0xf5,0x00,0x00,0x00,0xf4,0x00,0x00,0x00,0x71,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xf6,0x00,0x00,0x00,0xf5,0x00,0x00,0x00, +0x7c,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0xf7,0x00,0x00,0x00, +0xf6,0x00,0x00,0x00,0xc7,0x00,0x05,0x00,0x14,0x00,0x00,0x00, +0xf8,0x00,0x00,0x00,0xf7,0x00,0x00,0x00,0xda,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0x00,0x01,0x00,0x00, +0xaa,0x00,0x00,0x00,0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x01,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x7c,0x00,0x04,0x00, +0x14,0x00,0x00,0x00,0x02,0x01,0x00,0x00,0x01,0x01,0x00,0x00, +0xc7,0x00,0x05,0x00,0x14,0x00,0x00,0x00,0x03,0x01,0x00,0x00, +0x02,0x01,0x00,0x00,0xe5,0x00,0x00,0x00,0xc3,0x00,0x05,0x00, +0x14,0x00,0x00,0x00,0x04,0x01,0x00,0x00,0x03,0x01,0x00,0x00, +0xe7,0x00,0x00,0x00,0xc5,0x00,0x05,0x00,0x14,0x00,0x00,0x00, +0x05,0x01,0x00,0x00,0xf8,0x00,0x00,0x00,0x04,0x01,0x00,0x00, +0x72,0x00,0x04,0x00,0x51,0x00,0x00,0x00,0x06,0x01,0x00,0x00, +0x05,0x01,0x00,0x00,0x7c,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, +0x07,0x01,0x00,0x00,0x06,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, +0x4a,0x00,0x00,0x00,0x10,0x01,0x00,0x00,0xd6,0x00,0x00,0x00, +0xc2,0x00,0x05,0x00,0x4a,0x00,0x00,0x00,0x11,0x01,0x00,0x00, +0x10,0x01,0x00,0x00,0x56,0x00,0x00,0x00,0x71,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x12,0x01,0x00,0x00,0x11,0x01,0x00,0x00, +0x7c,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0x13,0x01,0x00,0x00, +0x12,0x01,0x00,0x00,0xc7,0x00,0x05,0x00,0x14,0x00,0x00,0x00, +0x14,0x01,0x00,0x00,0x13,0x01,0x00,0x00,0xda,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0x1c,0x01,0x00,0x00, +0xb8,0x00,0x00,0x00,0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x1d,0x01,0x00,0x00,0x1c,0x01,0x00,0x00,0x7c,0x00,0x04,0x00, +0x14,0x00,0x00,0x00,0x1e,0x01,0x00,0x00,0x1d,0x01,0x00,0x00, +0xc7,0x00,0x05,0x00,0x14,0x00,0x00,0x00,0x1f,0x01,0x00,0x00, +0x1e,0x01,0x00,0x00,0xe5,0x00,0x00,0x00,0xc3,0x00,0x05,0x00, +0x14,0x00,0x00,0x00,0x20,0x01,0x00,0x00,0x1f,0x01,0x00,0x00, +0xe7,0x00,0x00,0x00,0xc5,0x00,0x05,0x00,0x14,0x00,0x00,0x00, +0x21,0x01,0x00,0x00,0x14,0x01,0x00,0x00,0x20,0x01,0x00,0x00, +0x72,0x00,0x04,0x00,0x51,0x00,0x00,0x00,0x22,0x01,0x00,0x00, +0x21,0x01,0x00,0x00,0x7c,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, +0x23,0x01,0x00,0x00,0x22,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, +0x4a,0x00,0x00,0x00,0x2c,0x01,0x00,0x00,0xf4,0x00,0x00,0x00, +0xc2,0x00,0x05,0x00,0x4a,0x00,0x00,0x00,0x2d,0x01,0x00,0x00, +0x2c,0x01,0x00,0x00,0x56,0x00,0x00,0x00,0x71,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x2e,0x01,0x00,0x00,0x2d,0x01,0x00,0x00, +0x7c,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0x2f,0x01,0x00,0x00, +0x2e,0x01,0x00,0x00,0xc7,0x00,0x05,0x00,0x14,0x00,0x00,0x00, +0x30,0x01,0x00,0x00,0x2f,0x01,0x00,0x00,0xda,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0x38,0x01,0x00,0x00, +0xc7,0x00,0x00,0x00,0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x39,0x01,0x00,0x00,0x38,0x01,0x00,0x00,0x7c,0x00,0x04,0x00, +0x14,0x00,0x00,0x00,0x3a,0x01,0x00,0x00,0x39,0x01,0x00,0x00, +0xc7,0x00,0x05,0x00,0x14,0x00,0x00,0x00,0x3b,0x01,0x00,0x00, +0x3a,0x01,0x00,0x00,0xe5,0x00,0x00,0x00,0xc3,0x00,0x05,0x00, +0x14,0x00,0x00,0x00,0x3c,0x01,0x00,0x00,0x3b,0x01,0x00,0x00, +0xe7,0x00,0x00,0x00,0xc5,0x00,0x05,0x00,0x14,0x00,0x00,0x00, +0x3d,0x01,0x00,0x00,0x30,0x01,0x00,0x00,0x3c,0x01,0x00,0x00, +0x72,0x00,0x04,0x00,0x51,0x00,0x00,0x00,0x3e,0x01,0x00,0x00, +0x3d,0x01,0x00,0x00,0x7c,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, +0x3f,0x01,0x00,0x00,0x3e,0x01,0x00,0x00,0x41,0x00,0x08,0x00, +0x9a,0x00,0x00,0x00,0x46,0x01,0x00,0x00,0x84,0x00,0x00,0x00, +0x15,0x00,0x00,0x00,0x87,0x00,0x00,0x00,0x44,0x01,0x00,0x00, +0x43,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, +0x47,0x01,0x00,0x00,0x46,0x01,0x00,0x00,0x71,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x48,0x01,0x00,0x00,0x47,0x01,0x00,0x00, +0x7c,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0x49,0x01,0x00,0x00, +0x48,0x01,0x00,0x00,0xc7,0x00,0x05,0x00,0x14,0x00,0x00,0x00, +0x4a,0x01,0x00,0x00,0x49,0x01,0x00,0x00,0xda,0x00,0x00,0x00, +0x72,0x00,0x04,0x00,0x51,0x00,0x00,0x00,0x4b,0x01,0x00,0x00, +0x4a,0x01,0x00,0x00,0x7c,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, +0x4c,0x01,0x00,0x00,0x4b,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x52,0x01,0x00,0x00,0x43,0x00,0x00,0x00, +0x90,0x00,0x00,0x00,0x41,0x00,0x08,0x00,0x9a,0x00,0x00,0x00, +0x53,0x01,0x00,0x00,0x84,0x00,0x00,0x00,0x15,0x00,0x00,0x00, +0x87,0x00,0x00,0x00,0x44,0x01,0x00,0x00,0x52,0x01,0x00,0x00, +0x3d,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0x54,0x01,0x00,0x00, +0x53,0x01,0x00,0x00,0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x55,0x01,0x00,0x00,0x54,0x01,0x00,0x00,0x7c,0x00,0x04,0x00, +0x14,0x00,0x00,0x00,0x56,0x01,0x00,0x00,0x55,0x01,0x00,0x00, +0xc7,0x00,0x05,0x00,0x14,0x00,0x00,0x00,0x57,0x01,0x00,0x00, +0x56,0x01,0x00,0x00,0xda,0x00,0x00,0x00,0x72,0x00,0x04,0x00, +0x51,0x00,0x00,0x00,0x58,0x01,0x00,0x00,0x57,0x01,0x00,0x00, +0x7c,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0x59,0x01,0x00,0x00, +0x58,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x5f,0x01,0x00,0x00,0x43,0x00,0x00,0x00,0x5c,0x00,0x00,0x00, +0x41,0x00,0x08,0x00,0x9a,0x00,0x00,0x00,0x60,0x01,0x00,0x00, +0x84,0x00,0x00,0x00,0x15,0x00,0x00,0x00,0x87,0x00,0x00,0x00, +0x44,0x01,0x00,0x00,0x5f,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, +0x4a,0x00,0x00,0x00,0x61,0x01,0x00,0x00,0x60,0x01,0x00,0x00, +0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x62,0x01,0x00,0x00, +0x61,0x01,0x00,0x00,0x7c,0x00,0x04,0x00,0x14,0x00,0x00,0x00, +0x63,0x01,0x00,0x00,0x62,0x01,0x00,0x00,0xc7,0x00,0x05,0x00, +0x14,0x00,0x00,0x00,0x64,0x01,0x00,0x00,0x63,0x01,0x00,0x00, +0xda,0x00,0x00,0x00,0x72,0x00,0x04,0x00,0x51,0x00,0x00,0x00, +0x65,0x01,0x00,0x00,0x64,0x01,0x00,0x00,0x7c,0x00,0x04,0x00, +0x4a,0x00,0x00,0x00,0x66,0x01,0x00,0x00,0x65,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x6d,0x01,0x00,0x00, +0x43,0x00,0x00,0x00,0x6c,0x01,0x00,0x00,0x41,0x00,0x08,0x00, +0x9a,0x00,0x00,0x00,0x6e,0x01,0x00,0x00,0x84,0x00,0x00,0x00, +0x15,0x00,0x00,0x00,0x87,0x00,0x00,0x00,0x44,0x01,0x00,0x00, +0x6d,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, +0x6f,0x01,0x00,0x00,0x6e,0x01,0x00,0x00,0x71,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x70,0x01,0x00,0x00,0x6f,0x01,0x00,0x00, +0x7c,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0x71,0x01,0x00,0x00, +0x70,0x01,0x00,0x00,0xc7,0x00,0x05,0x00,0x14,0x00,0x00,0x00, +0x72,0x01,0x00,0x00,0x71,0x01,0x00,0x00,0xda,0x00,0x00,0x00, +0x72,0x00,0x04,0x00,0x51,0x00,0x00,0x00,0x73,0x01,0x00,0x00, +0x72,0x01,0x00,0x00,0x7c,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, +0x74,0x01,0x00,0x00,0x73,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, +0x4a,0x00,0x00,0x00,0x7b,0x01,0x00,0x00,0x46,0x01,0x00,0x00, +0xc2,0x00,0x05,0x00,0x4a,0x00,0x00,0x00,0x7c,0x01,0x00,0x00, +0x7b,0x01,0x00,0x00,0x56,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x4a,0x00,0x00,0x00,0x84,0x01,0x00,0x00,0x53,0x01,0x00,0x00, +0xc2,0x00,0x05,0x00,0x4a,0x00,0x00,0x00,0x85,0x01,0x00,0x00, +0x84,0x01,0x00,0x00,0x56,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x4a,0x00,0x00,0x00,0x8d,0x01,0x00,0x00,0x60,0x01,0x00,0x00, +0xc2,0x00,0x05,0x00,0x4a,0x00,0x00,0x00,0x8e,0x01,0x00,0x00, +0x8d,0x01,0x00,0x00,0x56,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x4a,0x00,0x00,0x00,0x96,0x01,0x00,0x00,0x6e,0x01,0x00,0x00, +0xc2,0x00,0x05,0x00,0x4a,0x00,0x00,0x00,0x97,0x01,0x00,0x00, +0x96,0x01,0x00,0x00,0x56,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x9d,0x01,0x00,0x00,0x43,0x00,0x00,0x00, +0x45,0x00,0x00,0x00,0x41,0x00,0x08,0x00,0x9a,0x00,0x00,0x00, +0x9e,0x01,0x00,0x00,0x84,0x00,0x00,0x00,0x15,0x00,0x00,0x00, +0x87,0x00,0x00,0x00,0x44,0x01,0x00,0x00,0x9d,0x01,0x00,0x00, +0x3d,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0x9f,0x01,0x00,0x00, +0x9e,0x01,0x00,0x00,0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xa0,0x01,0x00,0x00,0x9f,0x01,0x00,0x00,0x7c,0x00,0x04,0x00, +0x14,0x00,0x00,0x00,0xa1,0x01,0x00,0x00,0xa0,0x01,0x00,0x00, +0xc7,0x00,0x05,0x00,0x14,0x00,0x00,0x00,0xa2,0x01,0x00,0x00, +0xa1,0x01,0x00,0x00,0xda,0x00,0x00,0x00,0x72,0x00,0x04,0x00, +0x51,0x00,0x00,0x00,0xa3,0x01,0x00,0x00,0xa2,0x01,0x00,0x00, +0x7c,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0xa4,0x01,0x00,0x00, +0xa3,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xab,0x01,0x00,0x00,0x43,0x00,0x00,0x00,0xaa,0x01,0x00,0x00, +0x41,0x00,0x08,0x00,0x9a,0x00,0x00,0x00,0xac,0x01,0x00,0x00, +0x84,0x00,0x00,0x00,0x15,0x00,0x00,0x00,0x87,0x00,0x00,0x00, +0x44,0x01,0x00,0x00,0xab,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, +0x4a,0x00,0x00,0x00,0xad,0x01,0x00,0x00,0xac,0x01,0x00,0x00, +0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xae,0x01,0x00,0x00, +0xad,0x01,0x00,0x00,0x7c,0x00,0x04,0x00,0x14,0x00,0x00,0x00, +0xaf,0x01,0x00,0x00,0xae,0x01,0x00,0x00,0xc7,0x00,0x05,0x00, +0x14,0x00,0x00,0x00,0xb0,0x01,0x00,0x00,0xaf,0x01,0x00,0x00, +0xda,0x00,0x00,0x00,0x72,0x00,0x04,0x00,0x51,0x00,0x00,0x00, +0xb1,0x01,0x00,0x00,0xb0,0x01,0x00,0x00,0x7c,0x00,0x04,0x00, +0x4a,0x00,0x00,0x00,0xb2,0x01,0x00,0x00,0xb1,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xb9,0x01,0x00,0x00, +0x43,0x00,0x00,0x00,0xb8,0x01,0x00,0x00,0x41,0x00,0x08,0x00, +0x9a,0x00,0x00,0x00,0xba,0x01,0x00,0x00,0x84,0x00,0x00,0x00, +0x15,0x00,0x00,0x00,0x87,0x00,0x00,0x00,0x44,0x01,0x00,0x00, +0xb9,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, +0xbb,0x01,0x00,0x00,0xba,0x01,0x00,0x00,0x71,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xbc,0x01,0x00,0x00,0xbb,0x01,0x00,0x00, +0x7c,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0xbd,0x01,0x00,0x00, +0xbc,0x01,0x00,0x00,0xc7,0x00,0x05,0x00,0x14,0x00,0x00,0x00, +0xbe,0x01,0x00,0x00,0xbd,0x01,0x00,0x00,0xda,0x00,0x00,0x00, +0x72,0x00,0x04,0x00,0x51,0x00,0x00,0x00,0xbf,0x01,0x00,0x00, +0xbe,0x01,0x00,0x00,0x7c,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, +0xc0,0x01,0x00,0x00,0xbf,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xc7,0x01,0x00,0x00,0x43,0x00,0x00,0x00, +0xc6,0x01,0x00,0x00,0x41,0x00,0x08,0x00,0x9a,0x00,0x00,0x00, +0xc8,0x01,0x00,0x00,0x84,0x00,0x00,0x00,0x15,0x00,0x00,0x00, +0x87,0x00,0x00,0x00,0x44,0x01,0x00,0x00,0xc7,0x01,0x00,0x00, +0x3d,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0xc9,0x01,0x00,0x00, +0xc8,0x01,0x00,0x00,0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xca,0x01,0x00,0x00,0xc9,0x01,0x00,0x00,0x7c,0x00,0x04,0x00, +0x14,0x00,0x00,0x00,0xcb,0x01,0x00,0x00,0xca,0x01,0x00,0x00, +0xc7,0x00,0x05,0x00,0x14,0x00,0x00,0x00,0xcc,0x01,0x00,0x00, +0xcb,0x01,0x00,0x00,0xda,0x00,0x00,0x00,0x72,0x00,0x04,0x00, +0x51,0x00,0x00,0x00,0xcd,0x01,0x00,0x00,0xcc,0x01,0x00,0x00, +0x7c,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0xce,0x01,0x00,0x00, +0xcd,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, +0xd6,0x01,0x00,0x00,0x9e,0x01,0x00,0x00,0xc2,0x00,0x05,0x00, +0x4a,0x00,0x00,0x00,0xd7,0x01,0x00,0x00,0xd6,0x01,0x00,0x00, +0x56,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, +0xdf,0x01,0x00,0x00,0xac,0x01,0x00,0x00,0xc2,0x00,0x05,0x00, +0x4a,0x00,0x00,0x00,0xe0,0x01,0x00,0x00,0xdf,0x01,0x00,0x00, +0x56,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, +0xe8,0x01,0x00,0x00,0xba,0x01,0x00,0x00,0xc2,0x00,0x05,0x00, +0x4a,0x00,0x00,0x00,0xe9,0x01,0x00,0x00,0xe8,0x01,0x00,0x00, +0x56,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, +0xf1,0x01,0x00,0x00,0xc8,0x01,0x00,0x00,0xc2,0x00,0x05,0x00, +0x4a,0x00,0x00,0x00,0xf2,0x01,0x00,0x00,0xf1,0x01,0x00,0x00, +0x56,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x16,0x00,0x00,0x00, +0xf8,0x01,0x00,0x00,0x13,0x00,0x00,0x00,0x4d,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xf9,0x01,0x00,0x00, +0xf8,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xfb,0x01,0x00,0x00,0xf9,0x01,0x00,0x00,0x73,0x00,0x00,0x00, +0x41,0x00,0x06,0x00,0xfc,0x01,0x00,0x00,0xfd,0x01,0x00,0x00, +0xf7,0x01,0x00,0x00,0x15,0x00,0x00,0x00,0xfb,0x01,0x00,0x00, +0x3d,0x00,0x04,0x00,0x58,0x00,0x00,0x00,0xfe,0x01,0x00,0x00, +0xfd,0x01,0x00,0x00,0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x00,0x02,0x00,0x00,0x4c,0x01,0x00,0x00,0x7c,0x00,0x04,0x00, +0x14,0x00,0x00,0x00,0x01,0x02,0x00,0x00,0x00,0x02,0x00,0x00, +0x41,0x00,0x08,0x00,0x9a,0x00,0x00,0x00,0x06,0x02,0x00,0x00, +0x84,0x00,0x00,0x00,0x15,0x00,0x00,0x00,0x87,0x00,0x00,0x00, +0xe7,0x00,0x00,0x00,0x3d,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x4a,0x00,0x00,0x00,0x07,0x02,0x00,0x00,0x06,0x02,0x00,0x00, +0xc7,0x00,0x05,0x00,0x4a,0x00,0x00,0x00,0x09,0x02,0x00,0x00, +0x07,0x02,0x00,0x00,0x53,0x00,0x00,0x00,0x71,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x0a,0x02,0x00,0x00,0x09,0x02,0x00,0x00, +0x7c,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0x0b,0x02,0x00,0x00, +0x0a,0x02,0x00,0x00,0xab,0x00,0x05,0x00,0x6d,0x00,0x00,0x00, +0x0c,0x02,0x00,0x00,0x0b,0x02,0x00,0x00,0x15,0x00,0x00,0x00, +0xa9,0x00,0x06,0x00,0x14,0x00,0x00,0x00,0x0e,0x02,0x00,0x00, +0x0c,0x02,0x00,0x00,0x0d,0x02,0x00,0x00,0x15,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x14,0x00,0x00,0x00,0x0f,0x02,0x00,0x00, +0x01,0x02,0x00,0x00,0x0e,0x02,0x00,0x00,0x6f,0x00,0x04,0x00, +0x58,0x00,0x00,0x00,0x10,0x02,0x00,0x00,0x0f,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x16,0x02,0x00,0x00, +0xfb,0x01,0x00,0x00,0x90,0x00,0x00,0x00,0x41,0x00,0x06,0x00, +0xfc,0x01,0x00,0x00,0x17,0x02,0x00,0x00,0xf7,0x01,0x00,0x00, +0x15,0x00,0x00,0x00,0x16,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, +0x58,0x00,0x00,0x00,0x18,0x02,0x00,0x00,0x17,0x02,0x00,0x00, +0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x1a,0x02,0x00,0x00, +0x59,0x01,0x00,0x00,0x7c,0x00,0x04,0x00,0x14,0x00,0x00,0x00, +0x1b,0x02,0x00,0x00,0x1a,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x20,0x02,0x00,0x00,0x3d,0x00,0x00,0x00, +0x90,0x00,0x00,0x00,0x41,0x00,0x08,0x00,0x9a,0x00,0x00,0x00, +0x21,0x02,0x00,0x00,0x84,0x00,0x00,0x00,0x15,0x00,0x00,0x00, +0x87,0x00,0x00,0x00,0xe7,0x00,0x00,0x00,0x20,0x02,0x00,0x00, +0x3d,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0x22,0x02,0x00,0x00, +0x21,0x02,0x00,0x00,0xc7,0x00,0x05,0x00,0x4a,0x00,0x00,0x00, +0x24,0x02,0x00,0x00,0x22,0x02,0x00,0x00,0x53,0x00,0x00,0x00, +0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x25,0x02,0x00,0x00, +0x24,0x02,0x00,0x00,0x7c,0x00,0x04,0x00,0x14,0x00,0x00,0x00, +0x26,0x02,0x00,0x00,0x25,0x02,0x00,0x00,0xab,0x00,0x05,0x00, +0x6d,0x00,0x00,0x00,0x27,0x02,0x00,0x00,0x26,0x02,0x00,0x00, +0x15,0x00,0x00,0x00,0xa9,0x00,0x06,0x00,0x14,0x00,0x00,0x00, +0x28,0x02,0x00,0x00,0x27,0x02,0x00,0x00,0x0d,0x02,0x00,0x00, +0x15,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x14,0x00,0x00,0x00, +0x29,0x02,0x00,0x00,0x1b,0x02,0x00,0x00,0x28,0x02,0x00,0x00, +0x6f,0x00,0x04,0x00,0x58,0x00,0x00,0x00,0x2a,0x02,0x00,0x00, +0x29,0x02,0x00,0x00,0x85,0x00,0x05,0x00,0x58,0x00,0x00,0x00, +0x2b,0x02,0x00,0x00,0x18,0x02,0x00,0x00,0x2a,0x02,0x00,0x00, +0x0c,0x00,0x08,0x00,0x58,0x00,0x00,0x00,0x2c,0x02,0x00,0x00, +0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0xfe,0x01,0x00,0x00, +0x10,0x02,0x00,0x00,0x2b,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x31,0x02,0x00,0x00,0xfb,0x01,0x00,0x00, +0x5c,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0xfc,0x01,0x00,0x00, +0x32,0x02,0x00,0x00,0xf7,0x01,0x00,0x00,0x15,0x00,0x00,0x00, +0x31,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0x58,0x00,0x00,0x00, +0x33,0x02,0x00,0x00,0x32,0x02,0x00,0x00,0x71,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x35,0x02,0x00,0x00,0x66,0x01,0x00,0x00, +0x7c,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0x36,0x02,0x00,0x00, +0x35,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x3b,0x02,0x00,0x00,0x3d,0x00,0x00,0x00,0x5c,0x00,0x00,0x00, +0x41,0x00,0x08,0x00,0x9a,0x00,0x00,0x00,0x3c,0x02,0x00,0x00, +0x84,0x00,0x00,0x00,0x15,0x00,0x00,0x00,0x87,0x00,0x00,0x00, +0xe7,0x00,0x00,0x00,0x3b,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, +0x4a,0x00,0x00,0x00,0x3d,0x02,0x00,0x00,0x3c,0x02,0x00,0x00, +0xc7,0x00,0x05,0x00,0x4a,0x00,0x00,0x00,0x3f,0x02,0x00,0x00, +0x3d,0x02,0x00,0x00,0x53,0x00,0x00,0x00,0x71,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x40,0x02,0x00,0x00,0x3f,0x02,0x00,0x00, +0x7c,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0x41,0x02,0x00,0x00, +0x40,0x02,0x00,0x00,0xab,0x00,0x05,0x00,0x6d,0x00,0x00,0x00, +0x42,0x02,0x00,0x00,0x41,0x02,0x00,0x00,0x15,0x00,0x00,0x00, +0xa9,0x00,0x06,0x00,0x14,0x00,0x00,0x00,0x43,0x02,0x00,0x00, +0x42,0x02,0x00,0x00,0x0d,0x02,0x00,0x00,0x15,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x14,0x00,0x00,0x00,0x44,0x02,0x00,0x00, +0x36,0x02,0x00,0x00,0x43,0x02,0x00,0x00,0x6f,0x00,0x04,0x00, +0x58,0x00,0x00,0x00,0x45,0x02,0x00,0x00,0x44,0x02,0x00,0x00, +0x0c,0x00,0x08,0x00,0x58,0x00,0x00,0x00,0x47,0x02,0x00,0x00, +0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x33,0x02,0x00,0x00, +0x45,0x02,0x00,0x00,0x2c,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x4c,0x02,0x00,0x00,0xfb,0x01,0x00,0x00, +0x6c,0x01,0x00,0x00,0x41,0x00,0x06,0x00,0xfc,0x01,0x00,0x00, +0x4d,0x02,0x00,0x00,0xf7,0x01,0x00,0x00,0x15,0x00,0x00,0x00, +0x4c,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0x58,0x00,0x00,0x00, +0x4e,0x02,0x00,0x00,0x4d,0x02,0x00,0x00,0x71,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x50,0x02,0x00,0x00,0x74,0x01,0x00,0x00, +0x7c,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0x51,0x02,0x00,0x00, +0x50,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x56,0x02,0x00,0x00,0x3d,0x00,0x00,0x00,0x6c,0x01,0x00,0x00, +0x41,0x00,0x08,0x00,0x9a,0x00,0x00,0x00,0x57,0x02,0x00,0x00, +0x84,0x00,0x00,0x00,0x15,0x00,0x00,0x00,0x87,0x00,0x00,0x00, +0xe7,0x00,0x00,0x00,0x56,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, +0x4a,0x00,0x00,0x00,0x58,0x02,0x00,0x00,0x57,0x02,0x00,0x00, +0xc7,0x00,0x05,0x00,0x4a,0x00,0x00,0x00,0x5a,0x02,0x00,0x00, +0x58,0x02,0x00,0x00,0x53,0x00,0x00,0x00,0x71,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x5b,0x02,0x00,0x00,0x5a,0x02,0x00,0x00, +0x7c,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0x5c,0x02,0x00,0x00, +0x5b,0x02,0x00,0x00,0xab,0x00,0x05,0x00,0x6d,0x00,0x00,0x00, +0x5d,0x02,0x00,0x00,0x5c,0x02,0x00,0x00,0x15,0x00,0x00,0x00, +0xa9,0x00,0x06,0x00,0x14,0x00,0x00,0x00,0x5e,0x02,0x00,0x00, +0x5d,0x02,0x00,0x00,0x0d,0x02,0x00,0x00,0x15,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x14,0x00,0x00,0x00,0x5f,0x02,0x00,0x00, +0x51,0x02,0x00,0x00,0x5e,0x02,0x00,0x00,0x6f,0x00,0x04,0x00, +0x58,0x00,0x00,0x00,0x60,0x02,0x00,0x00,0x5f,0x02,0x00,0x00, +0x0c,0x00,0x08,0x00,0x58,0x00,0x00,0x00,0x62,0x02,0x00,0x00, +0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x4e,0x02,0x00,0x00, +0x60,0x02,0x00,0x00,0x47,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x68,0x02,0x00,0x00,0xfb,0x01,0x00,0x00, +0x3f,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0xfc,0x01,0x00,0x00, +0x69,0x02,0x00,0x00,0xf7,0x01,0x00,0x00,0x15,0x00,0x00,0x00, +0x68,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0x58,0x00,0x00,0x00, +0x6a,0x02,0x00,0x00,0x69,0x02,0x00,0x00,0x71,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x6c,0x02,0x00,0x00,0x7c,0x01,0x00,0x00, +0x7c,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0x6d,0x02,0x00,0x00, +0x6c,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, +0x73,0x02,0x00,0x00,0x06,0x02,0x00,0x00,0xc4,0x00,0x05,0x00, +0x4a,0x00,0x00,0x00,0x75,0x02,0x00,0x00,0x53,0x00,0x00,0x00, +0x4d,0x00,0x00,0x00,0xc7,0x00,0x05,0x00,0x4a,0x00,0x00,0x00, +0x76,0x02,0x00,0x00,0x73,0x02,0x00,0x00,0x75,0x02,0x00,0x00, +0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x77,0x02,0x00,0x00, +0x76,0x02,0x00,0x00,0x7c,0x00,0x04,0x00,0x14,0x00,0x00,0x00, +0x78,0x02,0x00,0x00,0x77,0x02,0x00,0x00,0xab,0x00,0x05,0x00, +0x6d,0x00,0x00,0x00,0x79,0x02,0x00,0x00,0x78,0x02,0x00,0x00, +0x15,0x00,0x00,0x00,0xa9,0x00,0x06,0x00,0x14,0x00,0x00,0x00, +0x7a,0x02,0x00,0x00,0x79,0x02,0x00,0x00,0x0d,0x02,0x00,0x00, +0x15,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x14,0x00,0x00,0x00, +0x7b,0x02,0x00,0x00,0x6d,0x02,0x00,0x00,0x7a,0x02,0x00,0x00, +0x6f,0x00,0x04,0x00,0x58,0x00,0x00,0x00,0x7c,0x02,0x00,0x00, +0x7b,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x83,0x02,0x00,0x00,0xfb,0x01,0x00,0x00,0x82,0x02,0x00,0x00, +0x41,0x00,0x06,0x00,0xfc,0x01,0x00,0x00,0x84,0x02,0x00,0x00, +0xf7,0x01,0x00,0x00,0x15,0x00,0x00,0x00,0x83,0x02,0x00,0x00, +0x3d,0x00,0x04,0x00,0x58,0x00,0x00,0x00,0x85,0x02,0x00,0x00, +0x84,0x02,0x00,0x00,0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x87,0x02,0x00,0x00,0x85,0x01,0x00,0x00,0x7c,0x00,0x04,0x00, +0x14,0x00,0x00,0x00,0x88,0x02,0x00,0x00,0x87,0x02,0x00,0x00, +0x3d,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0x8f,0x02,0x00,0x00, +0x21,0x02,0x00,0x00,0xc7,0x00,0x05,0x00,0x4a,0x00,0x00,0x00, +0x92,0x02,0x00,0x00,0x8f,0x02,0x00,0x00,0x75,0x02,0x00,0x00, +0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x93,0x02,0x00,0x00, +0x92,0x02,0x00,0x00,0x7c,0x00,0x04,0x00,0x14,0x00,0x00,0x00, +0x94,0x02,0x00,0x00,0x93,0x02,0x00,0x00,0xab,0x00,0x05,0x00, +0x6d,0x00,0x00,0x00,0x95,0x02,0x00,0x00,0x94,0x02,0x00,0x00, +0x15,0x00,0x00,0x00,0xa9,0x00,0x06,0x00,0x14,0x00,0x00,0x00, +0x96,0x02,0x00,0x00,0x95,0x02,0x00,0x00,0x0d,0x02,0x00,0x00, +0x15,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x14,0x00,0x00,0x00, +0x97,0x02,0x00,0x00,0x88,0x02,0x00,0x00,0x96,0x02,0x00,0x00, +0x6f,0x00,0x04,0x00,0x58,0x00,0x00,0x00,0x98,0x02,0x00,0x00, +0x97,0x02,0x00,0x00,0x85,0x00,0x05,0x00,0x58,0x00,0x00,0x00, +0x99,0x02,0x00,0x00,0x85,0x02,0x00,0x00,0x98,0x02,0x00,0x00, +0x0c,0x00,0x08,0x00,0x58,0x00,0x00,0x00,0x9a,0x02,0x00,0x00, +0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x6a,0x02,0x00,0x00, +0x7c,0x02,0x00,0x00,0x99,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xa0,0x02,0x00,0x00,0xfb,0x01,0x00,0x00, +0x9f,0x02,0x00,0x00,0x41,0x00,0x06,0x00,0xfc,0x01,0x00,0x00, +0xa1,0x02,0x00,0x00,0xf7,0x01,0x00,0x00,0x15,0x00,0x00,0x00, +0xa0,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0x58,0x00,0x00,0x00, +0xa2,0x02,0x00,0x00,0xa1,0x02,0x00,0x00,0x71,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xa4,0x02,0x00,0x00,0x8e,0x01,0x00,0x00, +0x7c,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0xa5,0x02,0x00,0x00, +0xa4,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, +0xac,0x02,0x00,0x00,0x3c,0x02,0x00,0x00,0xc7,0x00,0x05,0x00, +0x4a,0x00,0x00,0x00,0xaf,0x02,0x00,0x00,0xac,0x02,0x00,0x00, +0x75,0x02,0x00,0x00,0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xb0,0x02,0x00,0x00,0xaf,0x02,0x00,0x00,0x7c,0x00,0x04,0x00, +0x14,0x00,0x00,0x00,0xb1,0x02,0x00,0x00,0xb0,0x02,0x00,0x00, +0xab,0x00,0x05,0x00,0x6d,0x00,0x00,0x00,0xb2,0x02,0x00,0x00, +0xb1,0x02,0x00,0x00,0x15,0x00,0x00,0x00,0xa9,0x00,0x06,0x00, +0x14,0x00,0x00,0x00,0xb3,0x02,0x00,0x00,0xb2,0x02,0x00,0x00, +0x0d,0x02,0x00,0x00,0x15,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x14,0x00,0x00,0x00,0xb4,0x02,0x00,0x00,0xa5,0x02,0x00,0x00, +0xb3,0x02,0x00,0x00,0x6f,0x00,0x04,0x00,0x58,0x00,0x00,0x00, +0xb5,0x02,0x00,0x00,0xb4,0x02,0x00,0x00,0x0c,0x00,0x08,0x00, +0x58,0x00,0x00,0x00,0xb7,0x02,0x00,0x00,0x01,0x00,0x00,0x00, +0x32,0x00,0x00,0x00,0xa2,0x02,0x00,0x00,0xb5,0x02,0x00,0x00, +0x9a,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xbd,0x02,0x00,0x00,0xfb,0x01,0x00,0x00,0xbc,0x02,0x00,0x00, +0x41,0x00,0x06,0x00,0xfc,0x01,0x00,0x00,0xbe,0x02,0x00,0x00, +0xf7,0x01,0x00,0x00,0x15,0x00,0x00,0x00,0xbd,0x02,0x00,0x00, +0x3d,0x00,0x04,0x00,0x58,0x00,0x00,0x00,0xbf,0x02,0x00,0x00, +0xbe,0x02,0x00,0x00,0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xc1,0x02,0x00,0x00,0x97,0x01,0x00,0x00,0x7c,0x00,0x04,0x00, +0x14,0x00,0x00,0x00,0xc2,0x02,0x00,0x00,0xc1,0x02,0x00,0x00, +0x3d,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0xc9,0x02,0x00,0x00, +0x57,0x02,0x00,0x00,0xc7,0x00,0x05,0x00,0x4a,0x00,0x00,0x00, +0xcc,0x02,0x00,0x00,0xc9,0x02,0x00,0x00,0x75,0x02,0x00,0x00, +0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xcd,0x02,0x00,0x00, +0xcc,0x02,0x00,0x00,0x7c,0x00,0x04,0x00,0x14,0x00,0x00,0x00, +0xce,0x02,0x00,0x00,0xcd,0x02,0x00,0x00,0xab,0x00,0x05,0x00, +0x6d,0x00,0x00,0x00,0xcf,0x02,0x00,0x00,0xce,0x02,0x00,0x00, +0x15,0x00,0x00,0x00,0xa9,0x00,0x06,0x00,0x14,0x00,0x00,0x00, +0xd0,0x02,0x00,0x00,0xcf,0x02,0x00,0x00,0x0d,0x02,0x00,0x00, +0x15,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x14,0x00,0x00,0x00, +0xd1,0x02,0x00,0x00,0xc2,0x02,0x00,0x00,0xd0,0x02,0x00,0x00, +0x6f,0x00,0x04,0x00,0x58,0x00,0x00,0x00,0xd2,0x02,0x00,0x00, +0xd1,0x02,0x00,0x00,0x0c,0x00,0x08,0x00,0x58,0x00,0x00,0x00, +0xd4,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00, +0xbf,0x02,0x00,0x00,0xd2,0x02,0x00,0x00,0xb7,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xd9,0x02,0x00,0x00, +0xf9,0x01,0x00,0x00,0x77,0x00,0x00,0x00,0x41,0x00,0x06,0x00, +0xfc,0x01,0x00,0x00,0xda,0x02,0x00,0x00,0xf7,0x01,0x00,0x00, +0x15,0x00,0x00,0x00,0xd9,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, +0x58,0x00,0x00,0x00,0xdb,0x02,0x00,0x00,0xda,0x02,0x00,0x00, +0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xdd,0x02,0x00,0x00, +0xa4,0x01,0x00,0x00,0x7c,0x00,0x04,0x00,0x14,0x00,0x00,0x00, +0xde,0x02,0x00,0x00,0xdd,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, +0x4a,0x00,0x00,0x00,0xe4,0x02,0x00,0x00,0x06,0x02,0x00,0x00, +0xc7,0x00,0x05,0x00,0x4a,0x00,0x00,0x00,0xe6,0x02,0x00,0x00, +0xe4,0x02,0x00,0x00,0x57,0x00,0x00,0x00,0x71,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xe7,0x02,0x00,0x00,0xe6,0x02,0x00,0x00, +0x7c,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0xe8,0x02,0x00,0x00, +0xe7,0x02,0x00,0x00,0xab,0x00,0x05,0x00,0x6d,0x00,0x00,0x00, +0xe9,0x02,0x00,0x00,0xe8,0x02,0x00,0x00,0x15,0x00,0x00,0x00, +0xa9,0x00,0x06,0x00,0x14,0x00,0x00,0x00,0xea,0x02,0x00,0x00, +0xe9,0x02,0x00,0x00,0x0d,0x02,0x00,0x00,0x15,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x14,0x00,0x00,0x00,0xeb,0x02,0x00,0x00, +0xde,0x02,0x00,0x00,0xea,0x02,0x00,0x00,0x6f,0x00,0x04,0x00, +0x58,0x00,0x00,0x00,0xec,0x02,0x00,0x00,0xeb,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xf2,0x02,0x00,0x00, +0xd9,0x02,0x00,0x00,0x90,0x00,0x00,0x00,0x41,0x00,0x06,0x00, +0xfc,0x01,0x00,0x00,0xf3,0x02,0x00,0x00,0xf7,0x01,0x00,0x00, +0x15,0x00,0x00,0x00,0xf2,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, +0x58,0x00,0x00,0x00,0xf4,0x02,0x00,0x00,0xf3,0x02,0x00,0x00, +0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xf6,0x02,0x00,0x00, +0xb2,0x01,0x00,0x00,0x7c,0x00,0x04,0x00,0x14,0x00,0x00,0x00, +0xf7,0x02,0x00,0x00,0xf6,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, +0x4a,0x00,0x00,0x00,0xfe,0x02,0x00,0x00,0x21,0x02,0x00,0x00, +0xc7,0x00,0x05,0x00,0x4a,0x00,0x00,0x00,0x00,0x03,0x00,0x00, +0xfe,0x02,0x00,0x00,0x57,0x00,0x00,0x00,0x71,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x01,0x03,0x00,0x00,0x00,0x03,0x00,0x00, +0x7c,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0x02,0x03,0x00,0x00, +0x01,0x03,0x00,0x00,0xab,0x00,0x05,0x00,0x6d,0x00,0x00,0x00, +0x03,0x03,0x00,0x00,0x02,0x03,0x00,0x00,0x15,0x00,0x00,0x00, +0xa9,0x00,0x06,0x00,0x14,0x00,0x00,0x00,0x04,0x03,0x00,0x00, +0x03,0x03,0x00,0x00,0x0d,0x02,0x00,0x00,0x15,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x14,0x00,0x00,0x00,0x05,0x03,0x00,0x00, +0xf7,0x02,0x00,0x00,0x04,0x03,0x00,0x00,0x6f,0x00,0x04,0x00, +0x58,0x00,0x00,0x00,0x06,0x03,0x00,0x00,0x05,0x03,0x00,0x00, +0x85,0x00,0x05,0x00,0x58,0x00,0x00,0x00,0x07,0x03,0x00,0x00, +0xf4,0x02,0x00,0x00,0x06,0x03,0x00,0x00,0x0c,0x00,0x08,0x00, +0x58,0x00,0x00,0x00,0x08,0x03,0x00,0x00,0x01,0x00,0x00,0x00, +0x32,0x00,0x00,0x00,0xdb,0x02,0x00,0x00,0xec,0x02,0x00,0x00, +0x07,0x03,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x0d,0x03,0x00,0x00,0xd9,0x02,0x00,0x00,0x5c,0x00,0x00,0x00, +0x41,0x00,0x06,0x00,0xfc,0x01,0x00,0x00,0x0e,0x03,0x00,0x00, +0xf7,0x01,0x00,0x00,0x15,0x00,0x00,0x00,0x0d,0x03,0x00,0x00, +0x3d,0x00,0x04,0x00,0x58,0x00,0x00,0x00,0x0f,0x03,0x00,0x00, +0x0e,0x03,0x00,0x00,0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x11,0x03,0x00,0x00,0xc0,0x01,0x00,0x00,0x7c,0x00,0x04,0x00, +0x14,0x00,0x00,0x00,0x12,0x03,0x00,0x00,0x11,0x03,0x00,0x00, +0x3d,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0x19,0x03,0x00,0x00, +0x3c,0x02,0x00,0x00,0xc7,0x00,0x05,0x00,0x4a,0x00,0x00,0x00, +0x1b,0x03,0x00,0x00,0x19,0x03,0x00,0x00,0x57,0x00,0x00,0x00, +0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x1c,0x03,0x00,0x00, +0x1b,0x03,0x00,0x00,0x7c,0x00,0x04,0x00,0x14,0x00,0x00,0x00, +0x1d,0x03,0x00,0x00,0x1c,0x03,0x00,0x00,0xab,0x00,0x05,0x00, +0x6d,0x00,0x00,0x00,0x1e,0x03,0x00,0x00,0x1d,0x03,0x00,0x00, +0x15,0x00,0x00,0x00,0xa9,0x00,0x06,0x00,0x14,0x00,0x00,0x00, +0x1f,0x03,0x00,0x00,0x1e,0x03,0x00,0x00,0x0d,0x02,0x00,0x00, +0x15,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x14,0x00,0x00,0x00, +0x20,0x03,0x00,0x00,0x12,0x03,0x00,0x00,0x1f,0x03,0x00,0x00, +0x6f,0x00,0x04,0x00,0x58,0x00,0x00,0x00,0x21,0x03,0x00,0x00, +0x20,0x03,0x00,0x00,0x0c,0x00,0x08,0x00,0x58,0x00,0x00,0x00, +0x23,0x03,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00, +0x0f,0x03,0x00,0x00,0x21,0x03,0x00,0x00,0x08,0x03,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x28,0x03,0x00,0x00, +0xd9,0x02,0x00,0x00,0x6c,0x01,0x00,0x00,0x41,0x00,0x06,0x00, +0xfc,0x01,0x00,0x00,0x29,0x03,0x00,0x00,0xf7,0x01,0x00,0x00, +0x15,0x00,0x00,0x00,0x28,0x03,0x00,0x00,0x3d,0x00,0x04,0x00, +0x58,0x00,0x00,0x00,0x2a,0x03,0x00,0x00,0x29,0x03,0x00,0x00, +0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x2c,0x03,0x00,0x00, +0xce,0x01,0x00,0x00,0x7c,0x00,0x04,0x00,0x14,0x00,0x00,0x00, +0x2d,0x03,0x00,0x00,0x2c,0x03,0x00,0x00,0x3d,0x00,0x04,0x00, +0x4a,0x00,0x00,0x00,0x34,0x03,0x00,0x00,0x57,0x02,0x00,0x00, +0xc7,0x00,0x05,0x00,0x4a,0x00,0x00,0x00,0x36,0x03,0x00,0x00, +0x34,0x03,0x00,0x00,0x57,0x00,0x00,0x00,0x71,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x37,0x03,0x00,0x00,0x36,0x03,0x00,0x00, +0x7c,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0x38,0x03,0x00,0x00, +0x37,0x03,0x00,0x00,0xab,0x00,0x05,0x00,0x6d,0x00,0x00,0x00, +0x39,0x03,0x00,0x00,0x38,0x03,0x00,0x00,0x15,0x00,0x00,0x00, +0xa9,0x00,0x06,0x00,0x14,0x00,0x00,0x00,0x3a,0x03,0x00,0x00, +0x39,0x03,0x00,0x00,0x0d,0x02,0x00,0x00,0x15,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x14,0x00,0x00,0x00,0x3b,0x03,0x00,0x00, +0x2d,0x03,0x00,0x00,0x3a,0x03,0x00,0x00,0x6f,0x00,0x04,0x00, +0x58,0x00,0x00,0x00,0x3c,0x03,0x00,0x00,0x3b,0x03,0x00,0x00, +0x0c,0x00,0x08,0x00,0x58,0x00,0x00,0x00,0x3e,0x03,0x00,0x00, +0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x2a,0x03,0x00,0x00, +0x3c,0x03,0x00,0x00,0x23,0x03,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x44,0x03,0x00,0x00,0xd9,0x02,0x00,0x00, +0x3f,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0xfc,0x01,0x00,0x00, +0x45,0x03,0x00,0x00,0xf7,0x01,0x00,0x00,0x15,0x00,0x00,0x00, +0x44,0x03,0x00,0x00,0x3d,0x00,0x04,0x00,0x58,0x00,0x00,0x00, +0x46,0x03,0x00,0x00,0x45,0x03,0x00,0x00,0x71,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x48,0x03,0x00,0x00,0xd7,0x01,0x00,0x00, +0x7c,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0x49,0x03,0x00,0x00, +0x48,0x03,0x00,0x00,0x3d,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, +0x4f,0x03,0x00,0x00,0x06,0x02,0x00,0x00,0xc4,0x00,0x05,0x00, +0x4a,0x00,0x00,0x00,0x51,0x03,0x00,0x00,0x57,0x00,0x00,0x00, +0x4d,0x00,0x00,0x00,0xc7,0x00,0x05,0x00,0x4a,0x00,0x00,0x00, +0x52,0x03,0x00,0x00,0x4f,0x03,0x00,0x00,0x51,0x03,0x00,0x00, +0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x53,0x03,0x00,0x00, +0x52,0x03,0x00,0x00,0x7c,0x00,0x04,0x00,0x14,0x00,0x00,0x00, +0x54,0x03,0x00,0x00,0x53,0x03,0x00,0x00,0xab,0x00,0x05,0x00, +0x6d,0x00,0x00,0x00,0x55,0x03,0x00,0x00,0x54,0x03,0x00,0x00, +0x15,0x00,0x00,0x00,0xa9,0x00,0x06,0x00,0x14,0x00,0x00,0x00, +0x56,0x03,0x00,0x00,0x55,0x03,0x00,0x00,0x0d,0x02,0x00,0x00, +0x15,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x14,0x00,0x00,0x00, +0x57,0x03,0x00,0x00,0x49,0x03,0x00,0x00,0x56,0x03,0x00,0x00, +0x6f,0x00,0x04,0x00,0x58,0x00,0x00,0x00,0x58,0x03,0x00,0x00, +0x57,0x03,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x5e,0x03,0x00,0x00,0xd9,0x02,0x00,0x00,0x82,0x02,0x00,0x00, +0x41,0x00,0x06,0x00,0xfc,0x01,0x00,0x00,0x5f,0x03,0x00,0x00, +0xf7,0x01,0x00,0x00,0x15,0x00,0x00,0x00,0x5e,0x03,0x00,0x00, +0x3d,0x00,0x04,0x00,0x58,0x00,0x00,0x00,0x60,0x03,0x00,0x00, +0x5f,0x03,0x00,0x00,0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x62,0x03,0x00,0x00,0xe0,0x01,0x00,0x00,0x7c,0x00,0x04,0x00, +0x14,0x00,0x00,0x00,0x63,0x03,0x00,0x00,0x62,0x03,0x00,0x00, +0x3d,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0x6a,0x03,0x00,0x00, +0x21,0x02,0x00,0x00,0xc7,0x00,0x05,0x00,0x4a,0x00,0x00,0x00, +0x6d,0x03,0x00,0x00,0x6a,0x03,0x00,0x00,0x51,0x03,0x00,0x00, +0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x6e,0x03,0x00,0x00, +0x6d,0x03,0x00,0x00,0x7c,0x00,0x04,0x00,0x14,0x00,0x00,0x00, +0x6f,0x03,0x00,0x00,0x6e,0x03,0x00,0x00,0xab,0x00,0x05,0x00, +0x6d,0x00,0x00,0x00,0x70,0x03,0x00,0x00,0x6f,0x03,0x00,0x00, +0x15,0x00,0x00,0x00,0xa9,0x00,0x06,0x00,0x14,0x00,0x00,0x00, +0x71,0x03,0x00,0x00,0x70,0x03,0x00,0x00,0x0d,0x02,0x00,0x00, +0x15,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x14,0x00,0x00,0x00, +0x72,0x03,0x00,0x00,0x63,0x03,0x00,0x00,0x71,0x03,0x00,0x00, +0x6f,0x00,0x04,0x00,0x58,0x00,0x00,0x00,0x73,0x03,0x00,0x00, +0x72,0x03,0x00,0x00,0x85,0x00,0x05,0x00,0x58,0x00,0x00,0x00, +0x74,0x03,0x00,0x00,0x60,0x03,0x00,0x00,0x73,0x03,0x00,0x00, +0x0c,0x00,0x08,0x00,0x58,0x00,0x00,0x00,0x75,0x03,0x00,0x00, +0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x46,0x03,0x00,0x00, +0x58,0x03,0x00,0x00,0x74,0x03,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x7a,0x03,0x00,0x00,0xd9,0x02,0x00,0x00, +0x9f,0x02,0x00,0x00,0x41,0x00,0x06,0x00,0xfc,0x01,0x00,0x00, +0x7b,0x03,0x00,0x00,0xf7,0x01,0x00,0x00,0x15,0x00,0x00,0x00, +0x7a,0x03,0x00,0x00,0x3d,0x00,0x04,0x00,0x58,0x00,0x00,0x00, +0x7c,0x03,0x00,0x00,0x7b,0x03,0x00,0x00,0x71,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x7e,0x03,0x00,0x00,0xe9,0x01,0x00,0x00, +0x7c,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0x7f,0x03,0x00,0x00, +0x7e,0x03,0x00,0x00,0x3d,0x00,0x04,0x00,0x4a,0x00,0x00,0x00, +0x86,0x03,0x00,0x00,0x3c,0x02,0x00,0x00,0xc7,0x00,0x05,0x00, +0x4a,0x00,0x00,0x00,0x89,0x03,0x00,0x00,0x86,0x03,0x00,0x00, +0x51,0x03,0x00,0x00,0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x8a,0x03,0x00,0x00,0x89,0x03,0x00,0x00,0x7c,0x00,0x04,0x00, +0x14,0x00,0x00,0x00,0x8b,0x03,0x00,0x00,0x8a,0x03,0x00,0x00, +0xab,0x00,0x05,0x00,0x6d,0x00,0x00,0x00,0x8c,0x03,0x00,0x00, +0x8b,0x03,0x00,0x00,0x15,0x00,0x00,0x00,0xa9,0x00,0x06,0x00, +0x14,0x00,0x00,0x00,0x8d,0x03,0x00,0x00,0x8c,0x03,0x00,0x00, +0x0d,0x02,0x00,0x00,0x15,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x14,0x00,0x00,0x00,0x8e,0x03,0x00,0x00,0x7f,0x03,0x00,0x00, +0x8d,0x03,0x00,0x00,0x6f,0x00,0x04,0x00,0x58,0x00,0x00,0x00, +0x8f,0x03,0x00,0x00,0x8e,0x03,0x00,0x00,0x0c,0x00,0x08,0x00, +0x58,0x00,0x00,0x00,0x91,0x03,0x00,0x00,0x01,0x00,0x00,0x00, +0x32,0x00,0x00,0x00,0x7c,0x03,0x00,0x00,0x8f,0x03,0x00,0x00, +0x75,0x03,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x96,0x03,0x00,0x00,0xd9,0x02,0x00,0x00,0xbc,0x02,0x00,0x00, +0x41,0x00,0x06,0x00,0xfc,0x01,0x00,0x00,0x97,0x03,0x00,0x00, +0xf7,0x01,0x00,0x00,0x15,0x00,0x00,0x00,0x96,0x03,0x00,0x00, +0x3d,0x00,0x04,0x00,0x58,0x00,0x00,0x00,0x98,0x03,0x00,0x00, +0x97,0x03,0x00,0x00,0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x9a,0x03,0x00,0x00,0xf2,0x01,0x00,0x00,0x7c,0x00,0x04,0x00, +0x14,0x00,0x00,0x00,0x9b,0x03,0x00,0x00,0x9a,0x03,0x00,0x00, +0x3d,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0xa2,0x03,0x00,0x00, +0x57,0x02,0x00,0x00,0xc7,0x00,0x05,0x00,0x4a,0x00,0x00,0x00, +0xa5,0x03,0x00,0x00,0xa2,0x03,0x00,0x00,0x51,0x03,0x00,0x00, +0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xa6,0x03,0x00,0x00, +0xa5,0x03,0x00,0x00,0x7c,0x00,0x04,0x00,0x14,0x00,0x00,0x00, +0xa7,0x03,0x00,0x00,0xa6,0x03,0x00,0x00,0xab,0x00,0x05,0x00, +0x6d,0x00,0x00,0x00,0xa8,0x03,0x00,0x00,0xa7,0x03,0x00,0x00, +0x15,0x00,0x00,0x00,0xa9,0x00,0x06,0x00,0x14,0x00,0x00,0x00, +0xa9,0x03,0x00,0x00,0xa8,0x03,0x00,0x00,0x0d,0x02,0x00,0x00, +0x15,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x14,0x00,0x00,0x00, +0xaa,0x03,0x00,0x00,0x9b,0x03,0x00,0x00,0xa9,0x03,0x00,0x00, +0x6f,0x00,0x04,0x00,0x58,0x00,0x00,0x00,0xab,0x03,0x00,0x00, +0xaa,0x03,0x00,0x00,0x0c,0x00,0x08,0x00,0x58,0x00,0x00,0x00, +0xad,0x03,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00, +0x98,0x03,0x00,0x00,0xab,0x03,0x00,0x00,0x91,0x03,0x00,0x00, +0x3d,0x00,0x04,0x00,0x58,0x00,0x00,0x00,0xb4,0x03,0x00,0x00, +0xfd,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0x58,0x00,0x00,0x00, +0xbb,0x03,0x00,0x00,0x17,0x02,0x00,0x00,0x81,0x00,0x05,0x00, +0x58,0x00,0x00,0x00,0xbc,0x03,0x00,0x00,0xb4,0x03,0x00,0x00, +0xbb,0x03,0x00,0x00,0x3d,0x00,0x04,0x00,0x58,0x00,0x00,0x00, +0xc3,0x03,0x00,0x00,0x32,0x02,0x00,0x00,0x81,0x00,0x05,0x00, +0x58,0x00,0x00,0x00,0xc4,0x03,0x00,0x00,0xbc,0x03,0x00,0x00, +0xc3,0x03,0x00,0x00,0x3d,0x00,0x04,0x00,0x58,0x00,0x00,0x00, +0xcb,0x03,0x00,0x00,0x4d,0x02,0x00,0x00,0x81,0x00,0x05,0x00, +0x58,0x00,0x00,0x00,0xcc,0x03,0x00,0x00,0xc4,0x03,0x00,0x00, +0xcb,0x03,0x00,0x00,0x70,0x00,0x04,0x00,0x58,0x00,0x00,0x00, +0xce,0x03,0x00,0x00,0xbe,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x58,0x00,0x00,0x00,0xd6,0x03,0x00,0x00,0x69,0x02,0x00,0x00, +0x3d,0x00,0x04,0x00,0x58,0x00,0x00,0x00,0xdd,0x03,0x00,0x00, +0x84,0x02,0x00,0x00,0x81,0x00,0x05,0x00,0x58,0x00,0x00,0x00, +0xde,0x03,0x00,0x00,0xd6,0x03,0x00,0x00,0xdd,0x03,0x00,0x00, +0x3d,0x00,0x04,0x00,0x58,0x00,0x00,0x00,0xe5,0x03,0x00,0x00, +0xa1,0x02,0x00,0x00,0x81,0x00,0x05,0x00,0x58,0x00,0x00,0x00, +0xe6,0x03,0x00,0x00,0xde,0x03,0x00,0x00,0xe5,0x03,0x00,0x00, +0x3d,0x00,0x04,0x00,0x58,0x00,0x00,0x00,0xed,0x03,0x00,0x00, +0xbe,0x02,0x00,0x00,0x81,0x00,0x05,0x00,0x58,0x00,0x00,0x00, +0xee,0x03,0x00,0x00,0xe6,0x03,0x00,0x00,0xed,0x03,0x00,0x00, +0x70,0x00,0x04,0x00,0x58,0x00,0x00,0x00,0xf0,0x03,0x00,0x00, +0xcd,0x00,0x00,0x00,0x85,0x00,0x05,0x00,0x58,0x00,0x00,0x00, +0xf1,0x03,0x00,0x00,0xee,0x03,0x00,0x00,0xf0,0x03,0x00,0x00, +0x0c,0x00,0x08,0x00,0x58,0x00,0x00,0x00,0xf2,0x03,0x00,0x00, +0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0xcc,0x03,0x00,0x00, +0xce,0x03,0x00,0x00,0xf1,0x03,0x00,0x00,0x3d,0x00,0x04,0x00, +0x58,0x00,0x00,0x00,0xf8,0x03,0x00,0x00,0xda,0x02,0x00,0x00, +0x3d,0x00,0x04,0x00,0x58,0x00,0x00,0x00,0xff,0x03,0x00,0x00, +0xf3,0x02,0x00,0x00,0x81,0x00,0x05,0x00,0x58,0x00,0x00,0x00, +0x00,0x04,0x00,0x00,0xf8,0x03,0x00,0x00,0xff,0x03,0x00,0x00, +0x3d,0x00,0x04,0x00,0x58,0x00,0x00,0x00,0x07,0x04,0x00,0x00, +0x0e,0x03,0x00,0x00,0x81,0x00,0x05,0x00,0x58,0x00,0x00,0x00, +0x08,0x04,0x00,0x00,0x00,0x04,0x00,0x00,0x07,0x04,0x00,0x00, +0x3d,0x00,0x04,0x00,0x58,0x00,0x00,0x00,0x0f,0x04,0x00,0x00, +0x29,0x03,0x00,0x00,0x81,0x00,0x05,0x00,0x58,0x00,0x00,0x00, +0x10,0x04,0x00,0x00,0x08,0x04,0x00,0x00,0x0f,0x04,0x00,0x00, +0x70,0x00,0x04,0x00,0x58,0x00,0x00,0x00,0x12,0x04,0x00,0x00, +0x23,0x01,0x00,0x00,0x0c,0x00,0x08,0x00,0x58,0x00,0x00,0x00, +0x14,0x04,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00, +0x10,0x04,0x00,0x00,0x12,0x04,0x00,0x00,0xf2,0x03,0x00,0x00, +0x3d,0x00,0x04,0x00,0x58,0x00,0x00,0x00,0x1b,0x04,0x00,0x00, +0x45,0x03,0x00,0x00,0x3d,0x00,0x04,0x00,0x58,0x00,0x00,0x00, +0x22,0x04,0x00,0x00,0x5f,0x03,0x00,0x00,0x81,0x00,0x05,0x00, +0x58,0x00,0x00,0x00,0x23,0x04,0x00,0x00,0x1b,0x04,0x00,0x00, +0x22,0x04,0x00,0x00,0x3d,0x00,0x04,0x00,0x58,0x00,0x00,0x00, +0x2a,0x04,0x00,0x00,0x7b,0x03,0x00,0x00,0x81,0x00,0x05,0x00, +0x58,0x00,0x00,0x00,0x2b,0x04,0x00,0x00,0x23,0x04,0x00,0x00, +0x2a,0x04,0x00,0x00,0x3d,0x00,0x04,0x00,0x58,0x00,0x00,0x00, +0x32,0x04,0x00,0x00,0x97,0x03,0x00,0x00,0x81,0x00,0x05,0x00, +0x58,0x00,0x00,0x00,0x33,0x04,0x00,0x00,0x2b,0x04,0x00,0x00, +0x32,0x04,0x00,0x00,0x70,0x00,0x04,0x00,0x58,0x00,0x00,0x00, +0x35,0x04,0x00,0x00,0x3f,0x01,0x00,0x00,0x0c,0x00,0x08,0x00, +0x58,0x00,0x00,0x00,0x37,0x04,0x00,0x00,0x01,0x00,0x00,0x00, +0x32,0x00,0x00,0x00,0x33,0x04,0x00,0x00,0x35,0x04,0x00,0x00, +0x14,0x04,0x00,0x00,0x70,0x00,0x04,0x00,0x58,0x00,0x00,0x00, +0x3f,0x04,0x00,0x00,0xa2,0x00,0x00,0x00,0x70,0x00,0x04,0x00, +0x58,0x00,0x00,0x00,0x43,0x04,0x00,0x00,0xb0,0x00,0x00,0x00, +0x85,0x00,0x05,0x00,0x58,0x00,0x00,0x00,0x44,0x04,0x00,0x00, +0xd4,0x02,0x00,0x00,0x43,0x04,0x00,0x00,0x0c,0x00,0x08,0x00, +0x58,0x00,0x00,0x00,0x45,0x04,0x00,0x00,0x01,0x00,0x00,0x00, +0x32,0x00,0x00,0x00,0x62,0x02,0x00,0x00,0x3f,0x04,0x00,0x00, +0x44,0x04,0x00,0x00,0x70,0x00,0x04,0x00,0x58,0x00,0x00,0x00, +0x48,0x04,0x00,0x00,0xeb,0x00,0x00,0x00,0x0c,0x00,0x08,0x00, +0x58,0x00,0x00,0x00,0x4a,0x04,0x00,0x00,0x01,0x00,0x00,0x00, +0x32,0x00,0x00,0x00,0x3e,0x03,0x00,0x00,0x48,0x04,0x00,0x00, +0x45,0x04,0x00,0x00,0x70,0x00,0x04,0x00,0x58,0x00,0x00,0x00, +0x4d,0x04,0x00,0x00,0x07,0x01,0x00,0x00,0x0c,0x00,0x08,0x00, +0x58,0x00,0x00,0x00,0x4f,0x04,0x00,0x00,0x01,0x00,0x00,0x00, +0x32,0x00,0x00,0x00,0xad,0x03,0x00,0x00,0x4d,0x04,0x00,0x00, +0x4a,0x04,0x00,0x00,0x85,0x00,0x05,0x00,0x58,0x00,0x00,0x00, +0x53,0x04,0x00,0x00,0x93,0x00,0x00,0x00,0x37,0x04,0x00,0x00, +0x7f,0x00,0x04,0x00,0x58,0x00,0x00,0x00,0x89,0x04,0x00,0x00, +0x53,0x04,0x00,0x00,0x0c,0x00,0x08,0x00,0x58,0x00,0x00,0x00, +0x54,0x04,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00, +0x8b,0x00,0x00,0x00,0x4f,0x04,0x00,0x00,0x89,0x04,0x00,0x00, +0x3d,0x00,0x04,0x00,0x58,0x00,0x00,0x00,0x56,0x04,0x00,0x00, +0x63,0x00,0x00,0x00,0x81,0x00,0x05,0x00,0x58,0x00,0x00,0x00, +0x57,0x04,0x00,0x00,0x56,0x04,0x00,0x00,0x54,0x04,0x00,0x00, +0x3e,0x00,0x03,0x00,0x63,0x00,0x00,0x00,0x57,0x04,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x5a,0x04,0x00,0x00, +0x85,0x04,0x00,0x00,0x23,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x66,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x68,0x00,0x00,0x00, +0xe0,0x00,0x04,0x00,0x23,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x5b,0x04,0x00,0x00,0xf9,0x00,0x02,0x00,0x5d,0x04,0x00,0x00, +0xf8,0x00,0x02,0x00,0x5d,0x04,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0x86,0x04,0x00,0x00,0x5c,0x00,0x00,0x00, +0x68,0x00,0x00,0x00,0x74,0x04,0x00,0x00,0x60,0x04,0x00,0x00, +0xac,0x00,0x05,0x00,0x6d,0x00,0x00,0x00,0x63,0x04,0x00,0x00, +0x86,0x04,0x00,0x00,0x0c,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0x5f,0x04,0x00,0x00,0x60,0x04,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x63,0x04,0x00,0x00,0x5e,0x04,0x00,0x00, +0x5f,0x04,0x00,0x00,0xf8,0x00,0x02,0x00,0x5e,0x04,0x00,0x00, +0xb0,0x00,0x05,0x00,0x6d,0x00,0x00,0x00,0x66,0x04,0x00,0x00, +0x24,0x00,0x00,0x00,0x86,0x04,0x00,0x00,0xf7,0x00,0x03,0x00, +0x68,0x04,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x66,0x04,0x00,0x00,0x67,0x04,0x00,0x00,0x68,0x04,0x00,0x00, +0xf8,0x00,0x02,0x00,0x67,0x04,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x6c,0x04,0x00,0x00,0x24,0x00,0x00,0x00, +0x86,0x04,0x00,0x00,0x41,0x00,0x05,0x00,0x62,0x00,0x00,0x00, +0x6d,0x04,0x00,0x00,0x5b,0x00,0x00,0x00,0x6c,0x04,0x00,0x00, +0x3d,0x00,0x04,0x00,0x58,0x00,0x00,0x00,0x6e,0x04,0x00,0x00, +0x6d,0x04,0x00,0x00,0x41,0x00,0x05,0x00,0x62,0x00,0x00,0x00, +0x6f,0x04,0x00,0x00,0x5b,0x00,0x00,0x00,0x24,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x58,0x00,0x00,0x00,0x70,0x04,0x00,0x00, +0x6f,0x04,0x00,0x00,0x81,0x00,0x05,0x00,0x58,0x00,0x00,0x00, +0x71,0x04,0x00,0x00,0x70,0x04,0x00,0x00,0x6e,0x04,0x00,0x00, +0x3e,0x00,0x03,0x00,0x6f,0x04,0x00,0x00,0x71,0x04,0x00,0x00, +0xf9,0x00,0x02,0x00,0x68,0x04,0x00,0x00,0xf8,0x00,0x02,0x00, +0x68,0x04,0x00,0x00,0xe0,0x00,0x04,0x00,0x23,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x5b,0x04,0x00,0x00,0xf9,0x00,0x02,0x00, +0x60,0x04,0x00,0x00,0xf8,0x00,0x02,0x00,0x60,0x04,0x00,0x00, +0xc2,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x74,0x04,0x00,0x00, +0x86,0x04,0x00,0x00,0x4d,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x5d,0x04,0x00,0x00,0xf8,0x00,0x02,0x00,0x5f,0x04,0x00,0x00, +0xaa,0x00,0x05,0x00,0x6d,0x00,0x00,0x00,0x76,0x04,0x00,0x00, +0x24,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0xf7,0x00,0x03,0x00, +0x78,0x04,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x76,0x04,0x00,0x00,0x77,0x04,0x00,0x00,0x78,0x04,0x00,0x00, +0xf8,0x00,0x02,0x00,0x77,0x04,0x00,0x00,0x41,0x00,0x05,0x00, +0x16,0x00,0x00,0x00,0x7d,0x04,0x00,0x00,0x13,0x00,0x00,0x00, +0xe7,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x7e,0x04,0x00,0x00,0x7d,0x04,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x80,0x04,0x00,0x00,0x7e,0x04,0x00,0x00, +0x0f,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x62,0x00,0x00,0x00, +0x81,0x04,0x00,0x00,0x5b,0x00,0x00,0x00,0x15,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x58,0x00,0x00,0x00,0x82,0x04,0x00,0x00, +0x81,0x04,0x00,0x00,0x41,0x00,0x06,0x00,0xfc,0x01,0x00,0x00, +0x83,0x04,0x00,0x00,0x7c,0x04,0x00,0x00,0x15,0x00,0x00,0x00, +0x80,0x04,0x00,0x00,0x3e,0x00,0x03,0x00,0x83,0x04,0x00,0x00, +0x82,0x04,0x00,0x00,0xf9,0x00,0x02,0x00,0x78,0x04,0x00,0x00, +0xf8,0x00,0x02,0x00,0x78,0x04,0x00,0x00,0xfd,0x00,0x01,0x00, +0x38,0x00,0x01,0x00, }; -const uint64_t mul_mat_vec_q8_0_f32_len = 3120; +const uint64_t mul_mat_vec_q5_K_f32_len = 12016; + +unsigned char mul_mat_vec_q6_K_f32_data[] = { +0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, +0xc5,0x03,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, +0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x27,0x00,0x00,0x00, +0x11,0x00,0x02,0x00,0x51,0x11,0x00,0x00,0x11,0x00,0x02,0x00, +0x60,0x11,0x00,0x00,0x0b,0x00,0x06,0x00,0x01,0x00,0x00,0x00, +0x47,0x4c,0x53,0x4c,0x2e,0x73,0x74,0x64,0x2e,0x34,0x35,0x30, +0x00,0x00,0x00,0x00,0x0e,0x00,0x03,0x00,0x00,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x0f,0x00,0x0c,0x00,0x05,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x6d,0x61,0x69,0x6e,0x00,0x00,0x00,0x00, +0x0b,0x00,0x00,0x00,0x13,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0x53,0x00,0x00,0x00,0x78,0x00,0x00,0x00,0x8f,0x00,0x00,0x00, +0xa5,0x01,0x00,0x00,0x10,0x00,0x06,0x00,0x04,0x00,0x00,0x00, +0x11,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x0b,0x00,0x00,0x00, +0x0b,0x00,0x00,0x00,0x1a,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x11,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x11,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x11,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x47,0x00,0x03,0x00, +0x11,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x20,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x1b,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x6f,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x70,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x72,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x74,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x74,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x80,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x74,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0xc0,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x74,0x00,0x00,0x00,0x03,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0xd0,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x75,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0xd2,0x00,0x00,0x00, +0x48,0x00,0x04,0x00,0x76,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x76,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x03,0x00,0x76,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x78,0x00,0x00,0x00,0x22,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x78,0x00,0x00,0x00, +0x21,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x8c,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x48,0x00,0x04,0x00,0x8d,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x8d,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x03,0x00,0x8d,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x8f,0x00,0x00,0x00,0x22,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x8f,0x00,0x00,0x00, +0x21,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0xa2,0x01,0x00,0x00,0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x48,0x00,0x04,0x00,0xa3,0x01,0x00,0x00,0x00,0x00,0x00,0x00, +0x19,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0xa3,0x01,0x00,0x00, +0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x03,0x00,0xa3,0x01,0x00,0x00,0x02,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0xa5,0x01,0x00,0x00,0x22,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xa5,0x01,0x00,0x00, +0x21,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0xae,0x01,0x00,0x00,0x0b,0x00,0x00,0x00,0x19,0x00,0x00,0x00, +0x13,0x00,0x02,0x00,0x02,0x00,0x00,0x00,0x21,0x00,0x03,0x00, +0x03,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x15,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x17,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x03,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x0a,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0x0a,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x0d,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x1e,0x00,0x05,0x00, +0x11,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x12,0x00,0x00,0x00, +0x09,0x00,0x00,0x00,0x11,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0x12,0x00,0x00,0x00,0x13,0x00,0x00,0x00,0x09,0x00,0x00,0x00, +0x15,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x14,0x00,0x00,0x00, +0x15,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x16,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x19,0x00,0x00,0x00, +0x00,0x01,0x00,0x00,0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x2b,0x00,0x00,0x00, +0x08,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x33,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x40,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x40,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x4b,0x00,0x00,0x00,0x80,0x00,0x00,0x00,0x16,0x00,0x03,0x00, +0x50,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x1c,0x00,0x04,0x00, +0x51,0x00,0x00,0x00,0x50,0x00,0x00,0x00,0x40,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x52,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x51,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x52,0x00,0x00,0x00, +0x53,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x54,0x00,0x00,0x00,0x10,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x50,0x00,0x00,0x00,0x59,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x5a,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x50,0x00,0x00,0x00,0x14,0x00,0x02,0x00, +0x65,0x00,0x00,0x00,0x15,0x00,0x04,0x00,0x6e,0x00,0x00,0x00, +0x08,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x1c,0x00,0x04,0x00, +0x6f,0x00,0x00,0x00,0x6e,0x00,0x00,0x00,0x4b,0x00,0x00,0x00, +0x1c,0x00,0x04,0x00,0x70,0x00,0x00,0x00,0x6e,0x00,0x00,0x00, +0x3a,0x00,0x00,0x00,0x15,0x00,0x04,0x00,0x71,0x00,0x00,0x00, +0x08,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x1c,0x00,0x04,0x00, +0x72,0x00,0x00,0x00,0x71,0x00,0x00,0x00,0x54,0x00,0x00,0x00, +0x16,0x00,0x03,0x00,0x73,0x00,0x00,0x00,0x10,0x00,0x00,0x00, +0x1e,0x00,0x06,0x00,0x74,0x00,0x00,0x00,0x6f,0x00,0x00,0x00, +0x70,0x00,0x00,0x00,0x72,0x00,0x00,0x00,0x73,0x00,0x00,0x00, +0x1d,0x00,0x03,0x00,0x75,0x00,0x00,0x00,0x74,0x00,0x00,0x00, +0x1e,0x00,0x03,0x00,0x76,0x00,0x00,0x00,0x75,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x77,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x76,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x77,0x00,0x00,0x00, +0x78,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x14,0x00,0x00,0x00,0x7c,0x00,0x00,0x00,0x03,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x7d,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x73,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x14,0x00,0x00,0x00, +0x8a,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, +0x8c,0x00,0x00,0x00,0x50,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, +0x8d,0x00,0x00,0x00,0x8c,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x8e,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x8d,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0x8e,0x00,0x00,0x00,0x8f,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x14,0x00,0x00,0x00, +0x90,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x99,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x50,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0x9f,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0xa2,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x71,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0xb1,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x6e,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0xb6,0x00,0x00,0x00, +0x0f,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x14,0x00,0x00,0x00, +0xc9,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x42,0x01,0x00,0x00,0x60,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x4a,0x01,0x00,0x00, +0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x14,0x00,0x00,0x00, +0x68,0x01,0x00,0x00,0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x84,0x01,0x00,0x00,0x08,0x01,0x00,0x00, +0x1d,0x00,0x03,0x00,0xa2,0x01,0x00,0x00,0x50,0x00,0x00,0x00, +0x1e,0x00,0x03,0x00,0xa3,0x01,0x00,0x00,0xa2,0x01,0x00,0x00, +0x20,0x00,0x04,0x00,0xa4,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, +0xa3,0x01,0x00,0x00,0x3b,0x00,0x04,0x00,0xa4,0x01,0x00,0x00, +0xa5,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xad,0x01,0x00,0x00,0x01,0x00,0x00,0x00, +0x2c,0x00,0x06,0x00,0x09,0x00,0x00,0x00,0xae,0x01,0x00,0x00, +0x40,0x00,0x00,0x00,0xad,0x01,0x00,0x00,0xad,0x01,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xbb,0x03,0x00,0x00, +0x21,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xbc,0x03,0x00,0x00,0x41,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xbd,0x03,0x00,0x00,0x61,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xbe,0x03,0x00,0x00, +0x22,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xbf,0x03,0x00,0x00,0x42,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xc0,0x03,0x00,0x00,0x62,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xc1,0x03,0x00,0x00, +0x03,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xc2,0x03,0x00,0x00,0x23,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xc3,0x03,0x00,0x00,0x43,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xc4,0x03,0x00,0x00, +0x63,0x00,0x00,0x00,0x36,0x00,0x05,0x00,0x02,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x03,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0x05,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x0d,0x00,0x00,0x00,0x0e,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x0f,0x00,0x00,0x00,0x0e,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x16,0x00,0x00,0x00,0x17,0x00,0x00,0x00,0x13,0x00,0x00,0x00, +0x15,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x18,0x00,0x00,0x00,0x17,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x1a,0x00,0x00,0x00,0x18,0x00,0x00,0x00, +0x19,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x1e,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x1a,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00,0x21,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x21,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x24,0x00,0x00,0x00, +0x22,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x89,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0x22,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x2c,0x00,0x00,0x00,0x24,0x00,0x00,0x00,0x2b,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x30,0x00,0x00,0x00, +0x2b,0x00,0x00,0x00,0x2c,0x00,0x00,0x00,0x82,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x31,0x00,0x00,0x00,0x24,0x00,0x00,0x00, +0x30,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x35,0x00,0x00,0x00,0x33,0x00,0x00,0x00,0x31,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x38,0x00,0x00,0x00, +0x31,0x00,0x00,0x00,0x33,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x3c,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, +0x2c,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x3e,0x00,0x00,0x00,0x3c,0x00,0x00,0x00,0x35,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x42,0x00,0x00,0x00, +0x40,0x00,0x00,0x00,0x2c,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x44,0x00,0x00,0x00,0x42,0x00,0x00,0x00, +0x35,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x49,0x00,0x00,0x00,0x30,0x00,0x00,0x00,0x38,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x4d,0x00,0x00,0x00, +0x4b,0x00,0x00,0x00,0x2c,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x4d,0x00,0x00,0x00, +0x35,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x56,0x00,0x00,0x00,0x54,0x00,0x00,0x00,0x28,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x58,0x00,0x00,0x00, +0x56,0x00,0x00,0x00,0x24,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x5a,0x00,0x00,0x00,0x5b,0x00,0x00,0x00,0x53,0x00,0x00,0x00, +0x58,0x00,0x00,0x00,0x3e,0x00,0x03,0x00,0x5b,0x00,0x00,0x00, +0x59,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x5e,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0x5e,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xaf,0x01,0x00,0x00,0x28,0x00,0x00,0x00, +0x05,0x00,0x00,0x00,0x83,0x01,0x00,0x00,0x5f,0x00,0x00,0x00, +0xb0,0x00,0x05,0x00,0x65,0x00,0x00,0x00,0x66,0x00,0x00,0x00, +0xaf,0x01,0x00,0x00,0x1a,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0x60,0x00,0x00,0x00,0x5f,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x66,0x00,0x00,0x00,0x5f,0x00,0x00,0x00, +0x60,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x5f,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x69,0x00,0x00,0x00, +0xaf,0x01,0x00,0x00,0x19,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x6b,0x00,0x00,0x00,0x69,0x00,0x00,0x00, +0x4f,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x7b,0x00,0x00,0x00,0x1e,0x00,0x00,0x00,0xaf,0x01,0x00,0x00, +0x41,0x00,0x07,0x00,0x7d,0x00,0x00,0x00,0x7e,0x00,0x00,0x00, +0x78,0x00,0x00,0x00,0x15,0x00,0x00,0x00,0x7b,0x00,0x00,0x00, +0x7c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x73,0x00,0x00,0x00, +0x7f,0x00,0x00,0x00,0x7e,0x00,0x00,0x00,0x73,0x00,0x04,0x00, +0x50,0x00,0x00,0x00,0x80,0x00,0x00,0x00,0x7f,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x16,0x00,0x00,0x00,0x91,0x00,0x00,0x00, +0x13,0x00,0x00,0x00,0x90,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x92,0x00,0x00,0x00,0x91,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x94,0x00,0x00,0x00, +0x92,0x00,0x00,0x00,0x6b,0x00,0x00,0x00,0x41,0x00,0x06,0x00, +0x99,0x00,0x00,0x00,0x9a,0x00,0x00,0x00,0x8f,0x00,0x00,0x00, +0x15,0x00,0x00,0x00,0x94,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x50,0x00,0x00,0x00,0x9b,0x00,0x00,0x00,0x9a,0x00,0x00,0x00, +0x41,0x00,0x08,0x00,0xa2,0x00,0x00,0x00,0xa3,0x00,0x00,0x00, +0x78,0x00,0x00,0x00,0x15,0x00,0x00,0x00,0x7b,0x00,0x00,0x00, +0x9f,0x00,0x00,0x00,0x49,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x71,0x00,0x00,0x00,0xa4,0x00,0x00,0x00,0xa3,0x00,0x00,0x00, +0x6f,0x00,0x04,0x00,0x50,0x00,0x00,0x00,0xa5,0x00,0x00,0x00, +0xa4,0x00,0x00,0x00,0x85,0x00,0x05,0x00,0x50,0x00,0x00,0x00, +0xa6,0x00,0x00,0x00,0x9b,0x00,0x00,0x00,0xa5,0x00,0x00,0x00, +0x85,0x00,0x05,0x00,0x50,0x00,0x00,0x00,0xa8,0x00,0x00,0x00, +0xa6,0x00,0x00,0x00,0x80,0x00,0x00,0x00,0x41,0x00,0x08,0x00, +0xb1,0x00,0x00,0x00,0xb2,0x00,0x00,0x00,0x78,0x00,0x00,0x00, +0x15,0x00,0x00,0x00,0x7b,0x00,0x00,0x00,0x15,0x00,0x00,0x00, +0x3e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x6e,0x00,0x00,0x00, +0xb3,0x00,0x00,0x00,0xb2,0x00,0x00,0x00,0x71,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xb4,0x00,0x00,0x00,0xb3,0x00,0x00,0x00, +0x7c,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0xb5,0x00,0x00,0x00, +0xb4,0x00,0x00,0x00,0xc7,0x00,0x05,0x00,0x14,0x00,0x00,0x00, +0xb7,0x00,0x00,0x00,0xb5,0x00,0x00,0x00,0xb6,0x00,0x00,0x00, +0x41,0x00,0x08,0x00,0xb1,0x00,0x00,0x00,0xbf,0x00,0x00,0x00, +0x78,0x00,0x00,0x00,0x15,0x00,0x00,0x00,0x7b,0x00,0x00,0x00, +0x90,0x00,0x00,0x00,0x44,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x6e,0x00,0x00,0x00,0xc0,0x00,0x00,0x00,0xbf,0x00,0x00,0x00, +0xc2,0x00,0x05,0x00,0x6e,0x00,0x00,0x00,0xc1,0x00,0x00,0x00, +0xc0,0x00,0x00,0x00,0x15,0x00,0x00,0x00,0x71,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xc2,0x00,0x00,0x00,0xc1,0x00,0x00,0x00, +0x7c,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0xc3,0x00,0x00,0x00, +0xc2,0x00,0x00,0x00,0xc7,0x00,0x05,0x00,0x14,0x00,0x00,0x00, +0xc4,0x00,0x00,0x00,0xc3,0x00,0x00,0x00,0x7c,0x00,0x00,0x00, +0xc4,0x00,0x05,0x00,0x14,0x00,0x00,0x00,0xc5,0x00,0x00,0x00, +0xc4,0x00,0x00,0x00,0x8a,0x00,0x00,0x00,0xc5,0x00,0x05,0x00, +0x14,0x00,0x00,0x00,0xc6,0x00,0x00,0x00,0xb7,0x00,0x00,0x00, +0xc5,0x00,0x00,0x00,0x72,0x00,0x04,0x00,0x71,0x00,0x00,0x00, +0xc7,0x00,0x00,0x00,0xc6,0x00,0x00,0x00,0x72,0x00,0x04,0x00, +0x14,0x00,0x00,0x00,0xc8,0x00,0x00,0x00,0xc7,0x00,0x00,0x00, +0x82,0x00,0x05,0x00,0x14,0x00,0x00,0x00,0xca,0x00,0x00,0x00, +0xc8,0x00,0x00,0x00,0xc9,0x00,0x00,0x00,0x6f,0x00,0x04,0x00, +0x50,0x00,0x00,0x00,0xcb,0x00,0x00,0x00,0xca,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xd4,0x00,0x00,0x00, +0x94,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x41,0x00,0x06,0x00, +0x99,0x00,0x00,0x00,0xd5,0x00,0x00,0x00,0x8f,0x00,0x00,0x00, +0x15,0x00,0x00,0x00,0xd4,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x50,0x00,0x00,0x00,0xd6,0x00,0x00,0x00,0xd5,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xdb,0x00,0x00,0x00, +0x49,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x41,0x00,0x08,0x00, +0xa2,0x00,0x00,0x00,0xdc,0x00,0x00,0x00,0x78,0x00,0x00,0x00, +0x15,0x00,0x00,0x00,0x7b,0x00,0x00,0x00,0x9f,0x00,0x00,0x00, +0xdb,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x71,0x00,0x00,0x00, +0xdd,0x00,0x00,0x00,0xdc,0x00,0x00,0x00,0x6f,0x00,0x04,0x00, +0x50,0x00,0x00,0x00,0xde,0x00,0x00,0x00,0xdd,0x00,0x00,0x00, +0x85,0x00,0x05,0x00,0x50,0x00,0x00,0x00,0xdf,0x00,0x00,0x00, +0xd6,0x00,0x00,0x00,0xde,0x00,0x00,0x00,0x85,0x00,0x05,0x00, +0x50,0x00,0x00,0x00,0xe1,0x00,0x00,0x00,0xdf,0x00,0x00,0x00, +0x80,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xe9,0x00,0x00,0x00,0x3e,0x00,0x00,0x00,0x40,0x00,0x00,0x00, +0x41,0x00,0x08,0x00,0xb1,0x00,0x00,0x00,0xea,0x00,0x00,0x00, +0x78,0x00,0x00,0x00,0x15,0x00,0x00,0x00,0x7b,0x00,0x00,0x00, +0x15,0x00,0x00,0x00,0xe9,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x6e,0x00,0x00,0x00,0xeb,0x00,0x00,0x00,0xea,0x00,0x00,0x00, +0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xec,0x00,0x00,0x00, +0xeb,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x14,0x00,0x00,0x00, +0xed,0x00,0x00,0x00,0xec,0x00,0x00,0x00,0xc7,0x00,0x05,0x00, +0x14,0x00,0x00,0x00,0xee,0x00,0x00,0x00,0xed,0x00,0x00,0x00, +0xb6,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x6e,0x00,0x00,0x00, +0xf7,0x00,0x00,0x00,0xbf,0x00,0x00,0x00,0xc2,0x00,0x05,0x00, +0x6e,0x00,0x00,0x00,0xf8,0x00,0x00,0x00,0xf7,0x00,0x00,0x00, +0x9f,0x00,0x00,0x00,0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xf9,0x00,0x00,0x00,0xf8,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, +0x14,0x00,0x00,0x00,0xfa,0x00,0x00,0x00,0xf9,0x00,0x00,0x00, +0xc7,0x00,0x05,0x00,0x14,0x00,0x00,0x00,0xfb,0x00,0x00,0x00, +0xfa,0x00,0x00,0x00,0x7c,0x00,0x00,0x00,0xc4,0x00,0x05,0x00, +0x14,0x00,0x00,0x00,0xfc,0x00,0x00,0x00,0xfb,0x00,0x00,0x00, +0x8a,0x00,0x00,0x00,0xc5,0x00,0x05,0x00,0x14,0x00,0x00,0x00, +0xfd,0x00,0x00,0x00,0xee,0x00,0x00,0x00,0xfc,0x00,0x00,0x00, +0x72,0x00,0x04,0x00,0x71,0x00,0x00,0x00,0xfe,0x00,0x00,0x00, +0xfd,0x00,0x00,0x00,0x72,0x00,0x04,0x00,0x14,0x00,0x00,0x00, +0xff,0x00,0x00,0x00,0xfe,0x00,0x00,0x00,0x82,0x00,0x05,0x00, +0x14,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0xff,0x00,0x00,0x00, +0xc9,0x00,0x00,0x00,0x6f,0x00,0x04,0x00,0x50,0x00,0x00,0x00, +0x01,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x85,0x00,0x05,0x00, +0x50,0x00,0x00,0x00,0x02,0x01,0x00,0x00,0xe1,0x00,0x00,0x00, +0x01,0x01,0x00,0x00,0x0c,0x00,0x08,0x00,0x50,0x00,0x00,0x00, +0x03,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00, +0xa8,0x00,0x00,0x00,0xcb,0x00,0x00,0x00,0x02,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x0b,0x01,0x00,0x00, +0x94,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x41,0x00,0x06,0x00, +0x99,0x00,0x00,0x00,0x0c,0x01,0x00,0x00,0x8f,0x00,0x00,0x00, +0x15,0x00,0x00,0x00,0x0b,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, +0x50,0x00,0x00,0x00,0x0d,0x01,0x00,0x00,0x0c,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x12,0x01,0x00,0x00, +0x49,0x00,0x00,0x00,0x33,0x00,0x00,0x00,0x41,0x00,0x08,0x00, +0xa2,0x00,0x00,0x00,0x13,0x01,0x00,0x00,0x78,0x00,0x00,0x00, +0x15,0x00,0x00,0x00,0x7b,0x00,0x00,0x00,0x9f,0x00,0x00,0x00, +0x12,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0x71,0x00,0x00,0x00, +0x14,0x01,0x00,0x00,0x13,0x01,0x00,0x00,0x6f,0x00,0x04,0x00, +0x50,0x00,0x00,0x00,0x15,0x01,0x00,0x00,0x14,0x01,0x00,0x00, +0x85,0x00,0x05,0x00,0x50,0x00,0x00,0x00,0x16,0x01,0x00,0x00, +0x0d,0x01,0x00,0x00,0x15,0x01,0x00,0x00,0x85,0x00,0x05,0x00, +0x50,0x00,0x00,0x00,0x18,0x01,0x00,0x00,0x16,0x01,0x00,0x00, +0x80,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x6e,0x00,0x00,0x00, +0x22,0x01,0x00,0x00,0xb2,0x00,0x00,0x00,0xc2,0x00,0x05,0x00, +0x6e,0x00,0x00,0x00,0x23,0x01,0x00,0x00,0x22,0x01,0x00,0x00, +0x8a,0x00,0x00,0x00,0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x24,0x01,0x00,0x00,0x23,0x01,0x00,0x00,0x7c,0x00,0x04,0x00, +0x14,0x00,0x00,0x00,0x25,0x01,0x00,0x00,0x24,0x01,0x00,0x00, +0x3d,0x00,0x04,0x00,0x6e,0x00,0x00,0x00,0x2e,0x01,0x00,0x00, +0xbf,0x00,0x00,0x00,0xc2,0x00,0x05,0x00,0x6e,0x00,0x00,0x00, +0x2f,0x01,0x00,0x00,0x2e,0x01,0x00,0x00,0x8a,0x00,0x00,0x00, +0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x30,0x01,0x00,0x00, +0x2f,0x01,0x00,0x00,0x7c,0x00,0x04,0x00,0x14,0x00,0x00,0x00, +0x31,0x01,0x00,0x00,0x30,0x01,0x00,0x00,0xc7,0x00,0x05,0x00, +0x14,0x00,0x00,0x00,0x32,0x01,0x00,0x00,0x31,0x01,0x00,0x00, +0x7c,0x00,0x00,0x00,0xc4,0x00,0x05,0x00,0x14,0x00,0x00,0x00, +0x33,0x01,0x00,0x00,0x32,0x01,0x00,0x00,0x8a,0x00,0x00,0x00, +0xc5,0x00,0x05,0x00,0x14,0x00,0x00,0x00,0x34,0x01,0x00,0x00, +0x25,0x01,0x00,0x00,0x33,0x01,0x00,0x00,0x72,0x00,0x04,0x00, +0x71,0x00,0x00,0x00,0x35,0x01,0x00,0x00,0x34,0x01,0x00,0x00, +0x72,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0x36,0x01,0x00,0x00, +0x35,0x01,0x00,0x00,0x82,0x00,0x05,0x00,0x14,0x00,0x00,0x00, +0x37,0x01,0x00,0x00,0x36,0x01,0x00,0x00,0xc9,0x00,0x00,0x00, +0x6f,0x00,0x04,0x00,0x50,0x00,0x00,0x00,0x38,0x01,0x00,0x00, +0x37,0x01,0x00,0x00,0x0c,0x00,0x08,0x00,0x50,0x00,0x00,0x00, +0x3a,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00, +0x18,0x01,0x00,0x00,0x38,0x01,0x00,0x00,0x03,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x43,0x01,0x00,0x00, +0x94,0x00,0x00,0x00,0x42,0x01,0x00,0x00,0x41,0x00,0x06,0x00, +0x99,0x00,0x00,0x00,0x44,0x01,0x00,0x00,0x8f,0x00,0x00,0x00, +0x15,0x00,0x00,0x00,0x43,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, +0x50,0x00,0x00,0x00,0x45,0x01,0x00,0x00,0x44,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x4b,0x01,0x00,0x00, +0x49,0x00,0x00,0x00,0x4a,0x01,0x00,0x00,0x41,0x00,0x08,0x00, +0xa2,0x00,0x00,0x00,0x4c,0x01,0x00,0x00,0x78,0x00,0x00,0x00, +0x15,0x00,0x00,0x00,0x7b,0x00,0x00,0x00,0x9f,0x00,0x00,0x00, +0x4b,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0x71,0x00,0x00,0x00, +0x4d,0x01,0x00,0x00,0x4c,0x01,0x00,0x00,0x6f,0x00,0x04,0x00, +0x50,0x00,0x00,0x00,0x4e,0x01,0x00,0x00,0x4d,0x01,0x00,0x00, +0x85,0x00,0x05,0x00,0x50,0x00,0x00,0x00,0x4f,0x01,0x00,0x00, +0x45,0x01,0x00,0x00,0x4e,0x01,0x00,0x00,0x85,0x00,0x05,0x00, +0x50,0x00,0x00,0x00,0x51,0x01,0x00,0x00,0x4f,0x01,0x00,0x00, +0x80,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x6e,0x00,0x00,0x00, +0x5b,0x01,0x00,0x00,0xea,0x00,0x00,0x00,0xc2,0x00,0x05,0x00, +0x6e,0x00,0x00,0x00,0x5c,0x01,0x00,0x00,0x5b,0x01,0x00,0x00, +0x8a,0x00,0x00,0x00,0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x5d,0x01,0x00,0x00,0x5c,0x01,0x00,0x00,0x7c,0x00,0x04,0x00, +0x14,0x00,0x00,0x00,0x5e,0x01,0x00,0x00,0x5d,0x01,0x00,0x00, +0x3d,0x00,0x04,0x00,0x6e,0x00,0x00,0x00,0x67,0x01,0x00,0x00, +0xbf,0x00,0x00,0x00,0xc2,0x00,0x05,0x00,0x6e,0x00,0x00,0x00, +0x69,0x01,0x00,0x00,0x67,0x01,0x00,0x00,0x68,0x01,0x00,0x00, +0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x6a,0x01,0x00,0x00, +0x69,0x01,0x00,0x00,0x7c,0x00,0x04,0x00,0x14,0x00,0x00,0x00, +0x6b,0x01,0x00,0x00,0x6a,0x01,0x00,0x00,0xc7,0x00,0x05,0x00, +0x14,0x00,0x00,0x00,0x6c,0x01,0x00,0x00,0x6b,0x01,0x00,0x00, +0x7c,0x00,0x00,0x00,0xc4,0x00,0x05,0x00,0x14,0x00,0x00,0x00, +0x6d,0x01,0x00,0x00,0x6c,0x01,0x00,0x00,0x8a,0x00,0x00,0x00, +0xc5,0x00,0x05,0x00,0x14,0x00,0x00,0x00,0x6e,0x01,0x00,0x00, +0x5e,0x01,0x00,0x00,0x6d,0x01,0x00,0x00,0x72,0x00,0x04,0x00, +0x71,0x00,0x00,0x00,0x6f,0x01,0x00,0x00,0x6e,0x01,0x00,0x00, +0x72,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0x70,0x01,0x00,0x00, +0x6f,0x01,0x00,0x00,0x82,0x00,0x05,0x00,0x14,0x00,0x00,0x00, +0x71,0x01,0x00,0x00,0x70,0x01,0x00,0x00,0xc9,0x00,0x00,0x00, +0x6f,0x00,0x04,0x00,0x50,0x00,0x00,0x00,0x72,0x01,0x00,0x00, +0x71,0x01,0x00,0x00,0x0c,0x00,0x08,0x00,0x50,0x00,0x00,0x00, +0x74,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00, +0x51,0x01,0x00,0x00,0x72,0x01,0x00,0x00,0x3a,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xc0,0x01,0x00,0x00, +0x94,0x00,0x00,0x00,0xad,0x01,0x00,0x00,0x41,0x00,0x06,0x00, +0x99,0x00,0x00,0x00,0xc2,0x01,0x00,0x00,0x8f,0x00,0x00,0x00, +0x15,0x00,0x00,0x00,0xc0,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, +0x50,0x00,0x00,0x00,0xc3,0x01,0x00,0x00,0xc2,0x01,0x00,0x00, +0x3d,0x00,0x04,0x00,0x71,0x00,0x00,0x00,0xc7,0x01,0x00,0x00, +0xa3,0x00,0x00,0x00,0x6f,0x00,0x04,0x00,0x50,0x00,0x00,0x00, +0xc8,0x01,0x00,0x00,0xc7,0x01,0x00,0x00,0x85,0x00,0x05,0x00, +0x50,0x00,0x00,0x00,0xc9,0x01,0x00,0x00,0xc3,0x01,0x00,0x00, +0xc8,0x01,0x00,0x00,0x85,0x00,0x05,0x00,0x50,0x00,0x00,0x00, +0xca,0x01,0x00,0x00,0xc9,0x01,0x00,0x00,0x80,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xcd,0x01,0x00,0x00, +0x3e,0x00,0x00,0x00,0xad,0x01,0x00,0x00,0x41,0x00,0x08,0x00, +0xb1,0x00,0x00,0x00,0xcf,0x01,0x00,0x00,0x78,0x00,0x00,0x00, +0x15,0x00,0x00,0x00,0x7b,0x00,0x00,0x00,0x15,0x00,0x00,0x00, +0xcd,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0x6e,0x00,0x00,0x00, +0xd0,0x01,0x00,0x00,0xcf,0x01,0x00,0x00,0x71,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xd1,0x01,0x00,0x00,0xd0,0x01,0x00,0x00, +0x7c,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0xd2,0x01,0x00,0x00, +0xd1,0x01,0x00,0x00,0xc7,0x00,0x05,0x00,0x14,0x00,0x00,0x00, +0xd3,0x01,0x00,0x00,0xd2,0x01,0x00,0x00,0xb6,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xd6,0x01,0x00,0x00, +0x44,0x00,0x00,0x00,0xad,0x01,0x00,0x00,0x41,0x00,0x08,0x00, +0xb1,0x00,0x00,0x00,0xd7,0x01,0x00,0x00,0x78,0x00,0x00,0x00, +0x15,0x00,0x00,0x00,0x7b,0x00,0x00,0x00,0x90,0x00,0x00,0x00, +0xd6,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0x6e,0x00,0x00,0x00, +0xd8,0x01,0x00,0x00,0xd7,0x01,0x00,0x00,0xc2,0x00,0x05,0x00, +0x6e,0x00,0x00,0x00,0xd9,0x01,0x00,0x00,0xd8,0x01,0x00,0x00, +0x15,0x00,0x00,0x00,0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xda,0x01,0x00,0x00,0xd9,0x01,0x00,0x00,0x7c,0x00,0x04,0x00, +0x14,0x00,0x00,0x00,0xdb,0x01,0x00,0x00,0xda,0x01,0x00,0x00, +0xc7,0x00,0x05,0x00,0x14,0x00,0x00,0x00,0xdc,0x01,0x00,0x00, +0xdb,0x01,0x00,0x00,0x7c,0x00,0x00,0x00,0xc4,0x00,0x05,0x00, +0x14,0x00,0x00,0x00,0xdd,0x01,0x00,0x00,0xdc,0x01,0x00,0x00, +0x8a,0x00,0x00,0x00,0xc5,0x00,0x05,0x00,0x14,0x00,0x00,0x00, +0xde,0x01,0x00,0x00,0xd3,0x01,0x00,0x00,0xdd,0x01,0x00,0x00, +0x72,0x00,0x04,0x00,0x71,0x00,0x00,0x00,0xdf,0x01,0x00,0x00, +0xde,0x01,0x00,0x00,0x72,0x00,0x04,0x00,0x14,0x00,0x00,0x00, +0xe0,0x01,0x00,0x00,0xdf,0x01,0x00,0x00,0x82,0x00,0x05,0x00, +0x14,0x00,0x00,0x00,0xe1,0x01,0x00,0x00,0xe0,0x01,0x00,0x00, +0xc9,0x00,0x00,0x00,0x6f,0x00,0x04,0x00,0x50,0x00,0x00,0x00, +0xe2,0x01,0x00,0x00,0xe1,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xe9,0x01,0x00,0x00,0x94,0x00,0x00,0x00, +0xbb,0x03,0x00,0x00,0x41,0x00,0x06,0x00,0x99,0x00,0x00,0x00, +0xea,0x01,0x00,0x00,0x8f,0x00,0x00,0x00,0x15,0x00,0x00,0x00, +0xe9,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0x50,0x00,0x00,0x00, +0xeb,0x01,0x00,0x00,0xea,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, +0x71,0x00,0x00,0x00,0xef,0x01,0x00,0x00,0xdc,0x00,0x00,0x00, +0x6f,0x00,0x04,0x00,0x50,0x00,0x00,0x00,0xf0,0x01,0x00,0x00, +0xef,0x01,0x00,0x00,0x85,0x00,0x05,0x00,0x50,0x00,0x00,0x00, +0xf1,0x01,0x00,0x00,0xeb,0x01,0x00,0x00,0xf0,0x01,0x00,0x00, +0x85,0x00,0x05,0x00,0x50,0x00,0x00,0x00,0xf2,0x01,0x00,0x00, +0xf1,0x01,0x00,0x00,0x80,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xf6,0x01,0x00,0x00,0x3e,0x00,0x00,0x00, +0xbb,0x03,0x00,0x00,0x41,0x00,0x08,0x00,0xb1,0x00,0x00,0x00, +0xf7,0x01,0x00,0x00,0x78,0x00,0x00,0x00,0x15,0x00,0x00,0x00, +0x7b,0x00,0x00,0x00,0x15,0x00,0x00,0x00,0xf6,0x01,0x00,0x00, +0x3d,0x00,0x04,0x00,0x6e,0x00,0x00,0x00,0xf8,0x01,0x00,0x00, +0xf7,0x01,0x00,0x00,0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xf9,0x01,0x00,0x00,0xf8,0x01,0x00,0x00,0x7c,0x00,0x04,0x00, +0x14,0x00,0x00,0x00,0xfa,0x01,0x00,0x00,0xf9,0x01,0x00,0x00, +0xc7,0x00,0x05,0x00,0x14,0x00,0x00,0x00,0xfb,0x01,0x00,0x00, +0xfa,0x01,0x00,0x00,0xb6,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x6e,0x00,0x00,0x00,0x00,0x02,0x00,0x00,0xd7,0x01,0x00,0x00, +0xc2,0x00,0x05,0x00,0x6e,0x00,0x00,0x00,0x01,0x02,0x00,0x00, +0x00,0x02,0x00,0x00,0x9f,0x00,0x00,0x00,0x71,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x02,0x02,0x00,0x00,0x01,0x02,0x00,0x00, +0x7c,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0x03,0x02,0x00,0x00, +0x02,0x02,0x00,0x00,0xc7,0x00,0x05,0x00,0x14,0x00,0x00,0x00, +0x04,0x02,0x00,0x00,0x03,0x02,0x00,0x00,0x7c,0x00,0x00,0x00, +0xc4,0x00,0x05,0x00,0x14,0x00,0x00,0x00,0x05,0x02,0x00,0x00, +0x04,0x02,0x00,0x00,0x8a,0x00,0x00,0x00,0xc5,0x00,0x05,0x00, +0x14,0x00,0x00,0x00,0x06,0x02,0x00,0x00,0xfb,0x01,0x00,0x00, +0x05,0x02,0x00,0x00,0x72,0x00,0x04,0x00,0x71,0x00,0x00,0x00, +0x07,0x02,0x00,0x00,0x06,0x02,0x00,0x00,0x72,0x00,0x04,0x00, +0x14,0x00,0x00,0x00,0x08,0x02,0x00,0x00,0x07,0x02,0x00,0x00, +0x82,0x00,0x05,0x00,0x14,0x00,0x00,0x00,0x09,0x02,0x00,0x00, +0x08,0x02,0x00,0x00,0xc9,0x00,0x00,0x00,0x6f,0x00,0x04,0x00, +0x50,0x00,0x00,0x00,0x0a,0x02,0x00,0x00,0x09,0x02,0x00,0x00, +0x85,0x00,0x05,0x00,0x50,0x00,0x00,0x00,0x0b,0x02,0x00,0x00, +0xf2,0x01,0x00,0x00,0x0a,0x02,0x00,0x00,0x0c,0x00,0x08,0x00, +0x50,0x00,0x00,0x00,0x0c,0x02,0x00,0x00,0x01,0x00,0x00,0x00, +0x32,0x00,0x00,0x00,0xca,0x01,0x00,0x00,0xe2,0x01,0x00,0x00, +0x0b,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x12,0x02,0x00,0x00,0x94,0x00,0x00,0x00,0xbc,0x03,0x00,0x00, +0x41,0x00,0x06,0x00,0x99,0x00,0x00,0x00,0x13,0x02,0x00,0x00, +0x8f,0x00,0x00,0x00,0x15,0x00,0x00,0x00,0x12,0x02,0x00,0x00, +0x3d,0x00,0x04,0x00,0x50,0x00,0x00,0x00,0x14,0x02,0x00,0x00, +0x13,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0x71,0x00,0x00,0x00, +0x18,0x02,0x00,0x00,0x13,0x01,0x00,0x00,0x6f,0x00,0x04,0x00, +0x50,0x00,0x00,0x00,0x19,0x02,0x00,0x00,0x18,0x02,0x00,0x00, +0x85,0x00,0x05,0x00,0x50,0x00,0x00,0x00,0x1a,0x02,0x00,0x00, +0x14,0x02,0x00,0x00,0x19,0x02,0x00,0x00,0x85,0x00,0x05,0x00, +0x50,0x00,0x00,0x00,0x1b,0x02,0x00,0x00,0x1a,0x02,0x00,0x00, +0x80,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x6e,0x00,0x00,0x00, +0x21,0x02,0x00,0x00,0xcf,0x01,0x00,0x00,0xc2,0x00,0x05,0x00, +0x6e,0x00,0x00,0x00,0x22,0x02,0x00,0x00,0x21,0x02,0x00,0x00, +0x8a,0x00,0x00,0x00,0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x23,0x02,0x00,0x00,0x22,0x02,0x00,0x00,0x7c,0x00,0x04,0x00, +0x14,0x00,0x00,0x00,0x24,0x02,0x00,0x00,0x23,0x02,0x00,0x00, +0x3d,0x00,0x04,0x00,0x6e,0x00,0x00,0x00,0x29,0x02,0x00,0x00, +0xd7,0x01,0x00,0x00,0xc2,0x00,0x05,0x00,0x6e,0x00,0x00,0x00, +0x2a,0x02,0x00,0x00,0x29,0x02,0x00,0x00,0x8a,0x00,0x00,0x00, +0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x2b,0x02,0x00,0x00, +0x2a,0x02,0x00,0x00,0x7c,0x00,0x04,0x00,0x14,0x00,0x00,0x00, +0x2c,0x02,0x00,0x00,0x2b,0x02,0x00,0x00,0xc7,0x00,0x05,0x00, +0x14,0x00,0x00,0x00,0x2d,0x02,0x00,0x00,0x2c,0x02,0x00,0x00, +0x7c,0x00,0x00,0x00,0xc4,0x00,0x05,0x00,0x14,0x00,0x00,0x00, +0x2e,0x02,0x00,0x00,0x2d,0x02,0x00,0x00,0x8a,0x00,0x00,0x00, +0xc5,0x00,0x05,0x00,0x14,0x00,0x00,0x00,0x2f,0x02,0x00,0x00, +0x24,0x02,0x00,0x00,0x2e,0x02,0x00,0x00,0x72,0x00,0x04,0x00, +0x71,0x00,0x00,0x00,0x30,0x02,0x00,0x00,0x2f,0x02,0x00,0x00, +0x72,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0x31,0x02,0x00,0x00, +0x30,0x02,0x00,0x00,0x82,0x00,0x05,0x00,0x14,0x00,0x00,0x00, +0x32,0x02,0x00,0x00,0x31,0x02,0x00,0x00,0xc9,0x00,0x00,0x00, +0x6f,0x00,0x04,0x00,0x50,0x00,0x00,0x00,0x33,0x02,0x00,0x00, +0x32,0x02,0x00,0x00,0x0c,0x00,0x08,0x00,0x50,0x00,0x00,0x00, +0x35,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00, +0x1b,0x02,0x00,0x00,0x33,0x02,0x00,0x00,0x0c,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3b,0x02,0x00,0x00, +0x94,0x00,0x00,0x00,0xbd,0x03,0x00,0x00,0x41,0x00,0x06,0x00, +0x99,0x00,0x00,0x00,0x3c,0x02,0x00,0x00,0x8f,0x00,0x00,0x00, +0x15,0x00,0x00,0x00,0x3b,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, +0x50,0x00,0x00,0x00,0x3d,0x02,0x00,0x00,0x3c,0x02,0x00,0x00, +0x3d,0x00,0x04,0x00,0x71,0x00,0x00,0x00,0x41,0x02,0x00,0x00, +0x4c,0x01,0x00,0x00,0x6f,0x00,0x04,0x00,0x50,0x00,0x00,0x00, +0x42,0x02,0x00,0x00,0x41,0x02,0x00,0x00,0x85,0x00,0x05,0x00, +0x50,0x00,0x00,0x00,0x43,0x02,0x00,0x00,0x3d,0x02,0x00,0x00, +0x42,0x02,0x00,0x00,0x85,0x00,0x05,0x00,0x50,0x00,0x00,0x00, +0x44,0x02,0x00,0x00,0x43,0x02,0x00,0x00,0x80,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x6e,0x00,0x00,0x00,0x4a,0x02,0x00,0x00, +0xf7,0x01,0x00,0x00,0xc2,0x00,0x05,0x00,0x6e,0x00,0x00,0x00, +0x4b,0x02,0x00,0x00,0x4a,0x02,0x00,0x00,0x8a,0x00,0x00,0x00, +0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x4c,0x02,0x00,0x00, +0x4b,0x02,0x00,0x00,0x7c,0x00,0x04,0x00,0x14,0x00,0x00,0x00, +0x4d,0x02,0x00,0x00,0x4c,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, +0x6e,0x00,0x00,0x00,0x52,0x02,0x00,0x00,0xd7,0x01,0x00,0x00, +0xc2,0x00,0x05,0x00,0x6e,0x00,0x00,0x00,0x53,0x02,0x00,0x00, +0x52,0x02,0x00,0x00,0x68,0x01,0x00,0x00,0x71,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x54,0x02,0x00,0x00,0x53,0x02,0x00,0x00, +0x7c,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0x55,0x02,0x00,0x00, +0x54,0x02,0x00,0x00,0xc7,0x00,0x05,0x00,0x14,0x00,0x00,0x00, +0x56,0x02,0x00,0x00,0x55,0x02,0x00,0x00,0x7c,0x00,0x00,0x00, +0xc4,0x00,0x05,0x00,0x14,0x00,0x00,0x00,0x57,0x02,0x00,0x00, +0x56,0x02,0x00,0x00,0x8a,0x00,0x00,0x00,0xc5,0x00,0x05,0x00, +0x14,0x00,0x00,0x00,0x58,0x02,0x00,0x00,0x4d,0x02,0x00,0x00, +0x57,0x02,0x00,0x00,0x72,0x00,0x04,0x00,0x71,0x00,0x00,0x00, +0x59,0x02,0x00,0x00,0x58,0x02,0x00,0x00,0x72,0x00,0x04,0x00, +0x14,0x00,0x00,0x00,0x5a,0x02,0x00,0x00,0x59,0x02,0x00,0x00, +0x82,0x00,0x05,0x00,0x14,0x00,0x00,0x00,0x5b,0x02,0x00,0x00, +0x5a,0x02,0x00,0x00,0xc9,0x00,0x00,0x00,0x6f,0x00,0x04,0x00, +0x50,0x00,0x00,0x00,0x5c,0x02,0x00,0x00,0x5b,0x02,0x00,0x00, +0x0c,0x00,0x08,0x00,0x50,0x00,0x00,0x00,0x5e,0x02,0x00,0x00, +0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x44,0x02,0x00,0x00, +0x5c,0x02,0x00,0x00,0x35,0x02,0x00,0x00,0x81,0x00,0x05,0x00, +0x50,0x00,0x00,0x00,0x5f,0x02,0x00,0x00,0x74,0x01,0x00,0x00, +0x5e,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x6c,0x02,0x00,0x00,0x94,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x41,0x00,0x06,0x00,0x99,0x00,0x00,0x00,0x6e,0x02,0x00,0x00, +0x8f,0x00,0x00,0x00,0x15,0x00,0x00,0x00,0x6c,0x02,0x00,0x00, +0x3d,0x00,0x04,0x00,0x50,0x00,0x00,0x00,0x6f,0x02,0x00,0x00, +0x6e,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0x71,0x00,0x00,0x00, +0x73,0x02,0x00,0x00,0xa3,0x00,0x00,0x00,0x6f,0x00,0x04,0x00, +0x50,0x00,0x00,0x00,0x74,0x02,0x00,0x00,0x73,0x02,0x00,0x00, +0x85,0x00,0x05,0x00,0x50,0x00,0x00,0x00,0x75,0x02,0x00,0x00, +0x6f,0x02,0x00,0x00,0x74,0x02,0x00,0x00,0x85,0x00,0x05,0x00, +0x50,0x00,0x00,0x00,0x76,0x02,0x00,0x00,0x75,0x02,0x00,0x00, +0x80,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x79,0x02,0x00,0x00,0x3e,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x41,0x00,0x08,0x00,0xb1,0x00,0x00,0x00,0x7b,0x02,0x00,0x00, +0x78,0x00,0x00,0x00,0x15,0x00,0x00,0x00,0x7b,0x00,0x00,0x00, +0x15,0x00,0x00,0x00,0x79,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, +0x6e,0x00,0x00,0x00,0x7c,0x02,0x00,0x00,0x7b,0x02,0x00,0x00, +0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x7d,0x02,0x00,0x00, +0x7c,0x02,0x00,0x00,0x7c,0x00,0x04,0x00,0x14,0x00,0x00,0x00, +0x7e,0x02,0x00,0x00,0x7d,0x02,0x00,0x00,0xc7,0x00,0x05,0x00, +0x14,0x00,0x00,0x00,0x7f,0x02,0x00,0x00,0x7e,0x02,0x00,0x00, +0xb6,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x82,0x02,0x00,0x00,0x44,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x41,0x00,0x08,0x00,0xb1,0x00,0x00,0x00,0x83,0x02,0x00,0x00, +0x78,0x00,0x00,0x00,0x15,0x00,0x00,0x00,0x7b,0x00,0x00,0x00, +0x90,0x00,0x00,0x00,0x82,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, +0x6e,0x00,0x00,0x00,0x84,0x02,0x00,0x00,0x83,0x02,0x00,0x00, +0xc2,0x00,0x05,0x00,0x6e,0x00,0x00,0x00,0x85,0x02,0x00,0x00, +0x84,0x02,0x00,0x00,0x15,0x00,0x00,0x00,0x71,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x86,0x02,0x00,0x00,0x85,0x02,0x00,0x00, +0x7c,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0x87,0x02,0x00,0x00, +0x86,0x02,0x00,0x00,0xc7,0x00,0x05,0x00,0x14,0x00,0x00,0x00, +0x88,0x02,0x00,0x00,0x87,0x02,0x00,0x00,0x7c,0x00,0x00,0x00, +0xc4,0x00,0x05,0x00,0x14,0x00,0x00,0x00,0x89,0x02,0x00,0x00, +0x88,0x02,0x00,0x00,0x8a,0x00,0x00,0x00,0xc5,0x00,0x05,0x00, +0x14,0x00,0x00,0x00,0x8a,0x02,0x00,0x00,0x7f,0x02,0x00,0x00, +0x89,0x02,0x00,0x00,0x72,0x00,0x04,0x00,0x71,0x00,0x00,0x00, +0x8b,0x02,0x00,0x00,0x8a,0x02,0x00,0x00,0x72,0x00,0x04,0x00, +0x14,0x00,0x00,0x00,0x8c,0x02,0x00,0x00,0x8b,0x02,0x00,0x00, +0x82,0x00,0x05,0x00,0x14,0x00,0x00,0x00,0x8d,0x02,0x00,0x00, +0x8c,0x02,0x00,0x00,0xc9,0x00,0x00,0x00,0x6f,0x00,0x04,0x00, +0x50,0x00,0x00,0x00,0x8e,0x02,0x00,0x00,0x8d,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x95,0x02,0x00,0x00, +0x94,0x00,0x00,0x00,0xbe,0x03,0x00,0x00,0x41,0x00,0x06,0x00, +0x99,0x00,0x00,0x00,0x96,0x02,0x00,0x00,0x8f,0x00,0x00,0x00, +0x15,0x00,0x00,0x00,0x95,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, +0x50,0x00,0x00,0x00,0x97,0x02,0x00,0x00,0x96,0x02,0x00,0x00, +0x3d,0x00,0x04,0x00,0x71,0x00,0x00,0x00,0x9b,0x02,0x00,0x00, +0xdc,0x00,0x00,0x00,0x6f,0x00,0x04,0x00,0x50,0x00,0x00,0x00, +0x9c,0x02,0x00,0x00,0x9b,0x02,0x00,0x00,0x85,0x00,0x05,0x00, +0x50,0x00,0x00,0x00,0x9d,0x02,0x00,0x00,0x97,0x02,0x00,0x00, +0x9c,0x02,0x00,0x00,0x85,0x00,0x05,0x00,0x50,0x00,0x00,0x00, +0x9e,0x02,0x00,0x00,0x9d,0x02,0x00,0x00,0x80,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa2,0x02,0x00,0x00, +0x3e,0x00,0x00,0x00,0xbe,0x03,0x00,0x00,0x41,0x00,0x08,0x00, +0xb1,0x00,0x00,0x00,0xa3,0x02,0x00,0x00,0x78,0x00,0x00,0x00, +0x15,0x00,0x00,0x00,0x7b,0x00,0x00,0x00,0x15,0x00,0x00,0x00, +0xa2,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0x6e,0x00,0x00,0x00, +0xa4,0x02,0x00,0x00,0xa3,0x02,0x00,0x00,0x71,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xa5,0x02,0x00,0x00,0xa4,0x02,0x00,0x00, +0x7c,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0xa6,0x02,0x00,0x00, +0xa5,0x02,0x00,0x00,0xc7,0x00,0x05,0x00,0x14,0x00,0x00,0x00, +0xa7,0x02,0x00,0x00,0xa6,0x02,0x00,0x00,0xb6,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x6e,0x00,0x00,0x00,0xac,0x02,0x00,0x00, +0x83,0x02,0x00,0x00,0xc2,0x00,0x05,0x00,0x6e,0x00,0x00,0x00, +0xad,0x02,0x00,0x00,0xac,0x02,0x00,0x00,0x9f,0x00,0x00,0x00, +0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xae,0x02,0x00,0x00, +0xad,0x02,0x00,0x00,0x7c,0x00,0x04,0x00,0x14,0x00,0x00,0x00, +0xaf,0x02,0x00,0x00,0xae,0x02,0x00,0x00,0xc7,0x00,0x05,0x00, +0x14,0x00,0x00,0x00,0xb0,0x02,0x00,0x00,0xaf,0x02,0x00,0x00, +0x7c,0x00,0x00,0x00,0xc4,0x00,0x05,0x00,0x14,0x00,0x00,0x00, +0xb1,0x02,0x00,0x00,0xb0,0x02,0x00,0x00,0x8a,0x00,0x00,0x00, +0xc5,0x00,0x05,0x00,0x14,0x00,0x00,0x00,0xb2,0x02,0x00,0x00, +0xa7,0x02,0x00,0x00,0xb1,0x02,0x00,0x00,0x72,0x00,0x04,0x00, +0x71,0x00,0x00,0x00,0xb3,0x02,0x00,0x00,0xb2,0x02,0x00,0x00, +0x72,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0xb4,0x02,0x00,0x00, +0xb3,0x02,0x00,0x00,0x82,0x00,0x05,0x00,0x14,0x00,0x00,0x00, +0xb5,0x02,0x00,0x00,0xb4,0x02,0x00,0x00,0xc9,0x00,0x00,0x00, +0x6f,0x00,0x04,0x00,0x50,0x00,0x00,0x00,0xb6,0x02,0x00,0x00, +0xb5,0x02,0x00,0x00,0x85,0x00,0x05,0x00,0x50,0x00,0x00,0x00, +0xb7,0x02,0x00,0x00,0x9e,0x02,0x00,0x00,0xb6,0x02,0x00,0x00, +0x0c,0x00,0x08,0x00,0x50,0x00,0x00,0x00,0xb8,0x02,0x00,0x00, +0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x76,0x02,0x00,0x00, +0x8e,0x02,0x00,0x00,0xb7,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xbe,0x02,0x00,0x00,0x94,0x00,0x00,0x00, +0xbf,0x03,0x00,0x00,0x41,0x00,0x06,0x00,0x99,0x00,0x00,0x00, +0xbf,0x02,0x00,0x00,0x8f,0x00,0x00,0x00,0x15,0x00,0x00,0x00, +0xbe,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0x50,0x00,0x00,0x00, +0xc0,0x02,0x00,0x00,0xbf,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, +0x71,0x00,0x00,0x00,0xc4,0x02,0x00,0x00,0x13,0x01,0x00,0x00, +0x6f,0x00,0x04,0x00,0x50,0x00,0x00,0x00,0xc5,0x02,0x00,0x00, +0xc4,0x02,0x00,0x00,0x85,0x00,0x05,0x00,0x50,0x00,0x00,0x00, +0xc6,0x02,0x00,0x00,0xc0,0x02,0x00,0x00,0xc5,0x02,0x00,0x00, +0x85,0x00,0x05,0x00,0x50,0x00,0x00,0x00,0xc7,0x02,0x00,0x00, +0xc6,0x02,0x00,0x00,0x80,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x6e,0x00,0x00,0x00,0xcd,0x02,0x00,0x00,0x7b,0x02,0x00,0x00, +0xc2,0x00,0x05,0x00,0x6e,0x00,0x00,0x00,0xce,0x02,0x00,0x00, +0xcd,0x02,0x00,0x00,0x8a,0x00,0x00,0x00,0x71,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xcf,0x02,0x00,0x00,0xce,0x02,0x00,0x00, +0x7c,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0xd0,0x02,0x00,0x00, +0xcf,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0x6e,0x00,0x00,0x00, +0xd5,0x02,0x00,0x00,0x83,0x02,0x00,0x00,0xc2,0x00,0x05,0x00, +0x6e,0x00,0x00,0x00,0xd6,0x02,0x00,0x00,0xd5,0x02,0x00,0x00, +0x8a,0x00,0x00,0x00,0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xd7,0x02,0x00,0x00,0xd6,0x02,0x00,0x00,0x7c,0x00,0x04,0x00, +0x14,0x00,0x00,0x00,0xd8,0x02,0x00,0x00,0xd7,0x02,0x00,0x00, +0xc7,0x00,0x05,0x00,0x14,0x00,0x00,0x00,0xd9,0x02,0x00,0x00, +0xd8,0x02,0x00,0x00,0x7c,0x00,0x00,0x00,0xc4,0x00,0x05,0x00, +0x14,0x00,0x00,0x00,0xda,0x02,0x00,0x00,0xd9,0x02,0x00,0x00, +0x8a,0x00,0x00,0x00,0xc5,0x00,0x05,0x00,0x14,0x00,0x00,0x00, +0xdb,0x02,0x00,0x00,0xd0,0x02,0x00,0x00,0xda,0x02,0x00,0x00, +0x72,0x00,0x04,0x00,0x71,0x00,0x00,0x00,0xdc,0x02,0x00,0x00, +0xdb,0x02,0x00,0x00,0x72,0x00,0x04,0x00,0x14,0x00,0x00,0x00, +0xdd,0x02,0x00,0x00,0xdc,0x02,0x00,0x00,0x82,0x00,0x05,0x00, +0x14,0x00,0x00,0x00,0xde,0x02,0x00,0x00,0xdd,0x02,0x00,0x00, +0xc9,0x00,0x00,0x00,0x6f,0x00,0x04,0x00,0x50,0x00,0x00,0x00, +0xdf,0x02,0x00,0x00,0xde,0x02,0x00,0x00,0x0c,0x00,0x08,0x00, +0x50,0x00,0x00,0x00,0xe1,0x02,0x00,0x00,0x01,0x00,0x00,0x00, +0x32,0x00,0x00,0x00,0xc7,0x02,0x00,0x00,0xdf,0x02,0x00,0x00, +0xb8,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xe7,0x02,0x00,0x00,0x94,0x00,0x00,0x00,0xc0,0x03,0x00,0x00, +0x41,0x00,0x06,0x00,0x99,0x00,0x00,0x00,0xe8,0x02,0x00,0x00, +0x8f,0x00,0x00,0x00,0x15,0x00,0x00,0x00,0xe7,0x02,0x00,0x00, +0x3d,0x00,0x04,0x00,0x50,0x00,0x00,0x00,0xe9,0x02,0x00,0x00, +0xe8,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0x71,0x00,0x00,0x00, +0xed,0x02,0x00,0x00,0x4c,0x01,0x00,0x00,0x6f,0x00,0x04,0x00, +0x50,0x00,0x00,0x00,0xee,0x02,0x00,0x00,0xed,0x02,0x00,0x00, +0x85,0x00,0x05,0x00,0x50,0x00,0x00,0x00,0xef,0x02,0x00,0x00, +0xe9,0x02,0x00,0x00,0xee,0x02,0x00,0x00,0x85,0x00,0x05,0x00, +0x50,0x00,0x00,0x00,0xf0,0x02,0x00,0x00,0xef,0x02,0x00,0x00, +0x80,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x6e,0x00,0x00,0x00, +0xf6,0x02,0x00,0x00,0xa3,0x02,0x00,0x00,0xc2,0x00,0x05,0x00, +0x6e,0x00,0x00,0x00,0xf7,0x02,0x00,0x00,0xf6,0x02,0x00,0x00, +0x8a,0x00,0x00,0x00,0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xf8,0x02,0x00,0x00,0xf7,0x02,0x00,0x00,0x7c,0x00,0x04,0x00, +0x14,0x00,0x00,0x00,0xf9,0x02,0x00,0x00,0xf8,0x02,0x00,0x00, +0x3d,0x00,0x04,0x00,0x6e,0x00,0x00,0x00,0xfe,0x02,0x00,0x00, +0x83,0x02,0x00,0x00,0xc2,0x00,0x05,0x00,0x6e,0x00,0x00,0x00, +0xff,0x02,0x00,0x00,0xfe,0x02,0x00,0x00,0x68,0x01,0x00,0x00, +0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x00,0x03,0x00,0x00, +0xff,0x02,0x00,0x00,0x7c,0x00,0x04,0x00,0x14,0x00,0x00,0x00, +0x01,0x03,0x00,0x00,0x00,0x03,0x00,0x00,0xc7,0x00,0x05,0x00, +0x14,0x00,0x00,0x00,0x02,0x03,0x00,0x00,0x01,0x03,0x00,0x00, +0x7c,0x00,0x00,0x00,0xc4,0x00,0x05,0x00,0x14,0x00,0x00,0x00, +0x03,0x03,0x00,0x00,0x02,0x03,0x00,0x00,0x8a,0x00,0x00,0x00, +0xc5,0x00,0x05,0x00,0x14,0x00,0x00,0x00,0x04,0x03,0x00,0x00, +0xf9,0x02,0x00,0x00,0x03,0x03,0x00,0x00,0x72,0x00,0x04,0x00, +0x71,0x00,0x00,0x00,0x05,0x03,0x00,0x00,0x04,0x03,0x00,0x00, +0x72,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0x06,0x03,0x00,0x00, +0x05,0x03,0x00,0x00,0x82,0x00,0x05,0x00,0x14,0x00,0x00,0x00, +0x07,0x03,0x00,0x00,0x06,0x03,0x00,0x00,0xc9,0x00,0x00,0x00, +0x6f,0x00,0x04,0x00,0x50,0x00,0x00,0x00,0x08,0x03,0x00,0x00, +0x07,0x03,0x00,0x00,0x0c,0x00,0x08,0x00,0x50,0x00,0x00,0x00, +0x0a,0x03,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00, +0xf0,0x02,0x00,0x00,0x08,0x03,0x00,0x00,0xe1,0x02,0x00,0x00, +0x81,0x00,0x05,0x00,0x50,0x00,0x00,0x00,0x0b,0x03,0x00,0x00, +0x5f,0x02,0x00,0x00,0x0a,0x03,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x18,0x03,0x00,0x00,0x94,0x00,0x00,0x00, +0xc1,0x03,0x00,0x00,0x41,0x00,0x06,0x00,0x99,0x00,0x00,0x00, +0x1a,0x03,0x00,0x00,0x8f,0x00,0x00,0x00,0x15,0x00,0x00,0x00, +0x18,0x03,0x00,0x00,0x3d,0x00,0x04,0x00,0x50,0x00,0x00,0x00, +0x1b,0x03,0x00,0x00,0x1a,0x03,0x00,0x00,0x3d,0x00,0x04,0x00, +0x71,0x00,0x00,0x00,0x1f,0x03,0x00,0x00,0xa3,0x00,0x00,0x00, +0x6f,0x00,0x04,0x00,0x50,0x00,0x00,0x00,0x20,0x03,0x00,0x00, +0x1f,0x03,0x00,0x00,0x85,0x00,0x05,0x00,0x50,0x00,0x00,0x00, +0x21,0x03,0x00,0x00,0x1b,0x03,0x00,0x00,0x20,0x03,0x00,0x00, +0x85,0x00,0x05,0x00,0x50,0x00,0x00,0x00,0x22,0x03,0x00,0x00, +0x21,0x03,0x00,0x00,0x80,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x25,0x03,0x00,0x00,0x3e,0x00,0x00,0x00, +0xc1,0x03,0x00,0x00,0x41,0x00,0x08,0x00,0xb1,0x00,0x00,0x00, +0x27,0x03,0x00,0x00,0x78,0x00,0x00,0x00,0x15,0x00,0x00,0x00, +0x7b,0x00,0x00,0x00,0x15,0x00,0x00,0x00,0x25,0x03,0x00,0x00, +0x3d,0x00,0x04,0x00,0x6e,0x00,0x00,0x00,0x28,0x03,0x00,0x00, +0x27,0x03,0x00,0x00,0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x29,0x03,0x00,0x00,0x28,0x03,0x00,0x00,0x7c,0x00,0x04,0x00, +0x14,0x00,0x00,0x00,0x2a,0x03,0x00,0x00,0x29,0x03,0x00,0x00, +0xc7,0x00,0x05,0x00,0x14,0x00,0x00,0x00,0x2b,0x03,0x00,0x00, +0x2a,0x03,0x00,0x00,0xb6,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x2e,0x03,0x00,0x00,0x44,0x00,0x00,0x00, +0xc1,0x03,0x00,0x00,0x41,0x00,0x08,0x00,0xb1,0x00,0x00,0x00, +0x2f,0x03,0x00,0x00,0x78,0x00,0x00,0x00,0x15,0x00,0x00,0x00, +0x7b,0x00,0x00,0x00,0x90,0x00,0x00,0x00,0x2e,0x03,0x00,0x00, +0x3d,0x00,0x04,0x00,0x6e,0x00,0x00,0x00,0x30,0x03,0x00,0x00, +0x2f,0x03,0x00,0x00,0xc2,0x00,0x05,0x00,0x6e,0x00,0x00,0x00, +0x31,0x03,0x00,0x00,0x30,0x03,0x00,0x00,0x15,0x00,0x00,0x00, +0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x32,0x03,0x00,0x00, +0x31,0x03,0x00,0x00,0x7c,0x00,0x04,0x00,0x14,0x00,0x00,0x00, +0x33,0x03,0x00,0x00,0x32,0x03,0x00,0x00,0xc7,0x00,0x05,0x00, +0x14,0x00,0x00,0x00,0x34,0x03,0x00,0x00,0x33,0x03,0x00,0x00, +0x7c,0x00,0x00,0x00,0xc4,0x00,0x05,0x00,0x14,0x00,0x00,0x00, +0x35,0x03,0x00,0x00,0x34,0x03,0x00,0x00,0x8a,0x00,0x00,0x00, +0xc5,0x00,0x05,0x00,0x14,0x00,0x00,0x00,0x36,0x03,0x00,0x00, +0x2b,0x03,0x00,0x00,0x35,0x03,0x00,0x00,0x72,0x00,0x04,0x00, +0x71,0x00,0x00,0x00,0x37,0x03,0x00,0x00,0x36,0x03,0x00,0x00, +0x72,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0x38,0x03,0x00,0x00, +0x37,0x03,0x00,0x00,0x82,0x00,0x05,0x00,0x14,0x00,0x00,0x00, +0x39,0x03,0x00,0x00,0x38,0x03,0x00,0x00,0xc9,0x00,0x00,0x00, +0x6f,0x00,0x04,0x00,0x50,0x00,0x00,0x00,0x3a,0x03,0x00,0x00, +0x39,0x03,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x41,0x03,0x00,0x00,0x94,0x00,0x00,0x00,0xc2,0x03,0x00,0x00, +0x41,0x00,0x06,0x00,0x99,0x00,0x00,0x00,0x42,0x03,0x00,0x00, +0x8f,0x00,0x00,0x00,0x15,0x00,0x00,0x00,0x41,0x03,0x00,0x00, +0x3d,0x00,0x04,0x00,0x50,0x00,0x00,0x00,0x43,0x03,0x00,0x00, +0x42,0x03,0x00,0x00,0x3d,0x00,0x04,0x00,0x71,0x00,0x00,0x00, +0x47,0x03,0x00,0x00,0xdc,0x00,0x00,0x00,0x6f,0x00,0x04,0x00, +0x50,0x00,0x00,0x00,0x48,0x03,0x00,0x00,0x47,0x03,0x00,0x00, +0x85,0x00,0x05,0x00,0x50,0x00,0x00,0x00,0x49,0x03,0x00,0x00, +0x43,0x03,0x00,0x00,0x48,0x03,0x00,0x00,0x85,0x00,0x05,0x00, +0x50,0x00,0x00,0x00,0x4a,0x03,0x00,0x00,0x49,0x03,0x00,0x00, +0x80,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x4e,0x03,0x00,0x00,0x3e,0x00,0x00,0x00,0xc2,0x03,0x00,0x00, +0x41,0x00,0x08,0x00,0xb1,0x00,0x00,0x00,0x4f,0x03,0x00,0x00, +0x78,0x00,0x00,0x00,0x15,0x00,0x00,0x00,0x7b,0x00,0x00,0x00, +0x15,0x00,0x00,0x00,0x4e,0x03,0x00,0x00,0x3d,0x00,0x04,0x00, +0x6e,0x00,0x00,0x00,0x50,0x03,0x00,0x00,0x4f,0x03,0x00,0x00, +0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x51,0x03,0x00,0x00, +0x50,0x03,0x00,0x00,0x7c,0x00,0x04,0x00,0x14,0x00,0x00,0x00, +0x52,0x03,0x00,0x00,0x51,0x03,0x00,0x00,0xc7,0x00,0x05,0x00, +0x14,0x00,0x00,0x00,0x53,0x03,0x00,0x00,0x52,0x03,0x00,0x00, +0xb6,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x6e,0x00,0x00,0x00, +0x58,0x03,0x00,0x00,0x2f,0x03,0x00,0x00,0xc2,0x00,0x05,0x00, +0x6e,0x00,0x00,0x00,0x59,0x03,0x00,0x00,0x58,0x03,0x00,0x00, +0x9f,0x00,0x00,0x00,0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x5a,0x03,0x00,0x00,0x59,0x03,0x00,0x00,0x7c,0x00,0x04,0x00, +0x14,0x00,0x00,0x00,0x5b,0x03,0x00,0x00,0x5a,0x03,0x00,0x00, +0xc7,0x00,0x05,0x00,0x14,0x00,0x00,0x00,0x5c,0x03,0x00,0x00, +0x5b,0x03,0x00,0x00,0x7c,0x00,0x00,0x00,0xc4,0x00,0x05,0x00, +0x14,0x00,0x00,0x00,0x5d,0x03,0x00,0x00,0x5c,0x03,0x00,0x00, +0x8a,0x00,0x00,0x00,0xc5,0x00,0x05,0x00,0x14,0x00,0x00,0x00, +0x5e,0x03,0x00,0x00,0x53,0x03,0x00,0x00,0x5d,0x03,0x00,0x00, +0x72,0x00,0x04,0x00,0x71,0x00,0x00,0x00,0x5f,0x03,0x00,0x00, +0x5e,0x03,0x00,0x00,0x72,0x00,0x04,0x00,0x14,0x00,0x00,0x00, +0x60,0x03,0x00,0x00,0x5f,0x03,0x00,0x00,0x82,0x00,0x05,0x00, +0x14,0x00,0x00,0x00,0x61,0x03,0x00,0x00,0x60,0x03,0x00,0x00, +0xc9,0x00,0x00,0x00,0x6f,0x00,0x04,0x00,0x50,0x00,0x00,0x00, +0x62,0x03,0x00,0x00,0x61,0x03,0x00,0x00,0x85,0x00,0x05,0x00, +0x50,0x00,0x00,0x00,0x63,0x03,0x00,0x00,0x4a,0x03,0x00,0x00, +0x62,0x03,0x00,0x00,0x0c,0x00,0x08,0x00,0x50,0x00,0x00,0x00, +0x64,0x03,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00, +0x22,0x03,0x00,0x00,0x3a,0x03,0x00,0x00,0x63,0x03,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x6a,0x03,0x00,0x00, +0x94,0x00,0x00,0x00,0xc3,0x03,0x00,0x00,0x41,0x00,0x06,0x00, +0x99,0x00,0x00,0x00,0x6b,0x03,0x00,0x00,0x8f,0x00,0x00,0x00, +0x15,0x00,0x00,0x00,0x6a,0x03,0x00,0x00,0x3d,0x00,0x04,0x00, +0x50,0x00,0x00,0x00,0x6c,0x03,0x00,0x00,0x6b,0x03,0x00,0x00, +0x3d,0x00,0x04,0x00,0x71,0x00,0x00,0x00,0x70,0x03,0x00,0x00, +0x13,0x01,0x00,0x00,0x6f,0x00,0x04,0x00,0x50,0x00,0x00,0x00, +0x71,0x03,0x00,0x00,0x70,0x03,0x00,0x00,0x85,0x00,0x05,0x00, +0x50,0x00,0x00,0x00,0x72,0x03,0x00,0x00,0x6c,0x03,0x00,0x00, +0x71,0x03,0x00,0x00,0x85,0x00,0x05,0x00,0x50,0x00,0x00,0x00, +0x73,0x03,0x00,0x00,0x72,0x03,0x00,0x00,0x80,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x6e,0x00,0x00,0x00,0x79,0x03,0x00,0x00, +0x27,0x03,0x00,0x00,0xc2,0x00,0x05,0x00,0x6e,0x00,0x00,0x00, +0x7a,0x03,0x00,0x00,0x79,0x03,0x00,0x00,0x8a,0x00,0x00,0x00, +0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x7b,0x03,0x00,0x00, +0x7a,0x03,0x00,0x00,0x7c,0x00,0x04,0x00,0x14,0x00,0x00,0x00, +0x7c,0x03,0x00,0x00,0x7b,0x03,0x00,0x00,0x3d,0x00,0x04,0x00, +0x6e,0x00,0x00,0x00,0x81,0x03,0x00,0x00,0x2f,0x03,0x00,0x00, +0xc2,0x00,0x05,0x00,0x6e,0x00,0x00,0x00,0x82,0x03,0x00,0x00, +0x81,0x03,0x00,0x00,0x8a,0x00,0x00,0x00,0x71,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x83,0x03,0x00,0x00,0x82,0x03,0x00,0x00, +0x7c,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0x84,0x03,0x00,0x00, +0x83,0x03,0x00,0x00,0xc7,0x00,0x05,0x00,0x14,0x00,0x00,0x00, +0x85,0x03,0x00,0x00,0x84,0x03,0x00,0x00,0x7c,0x00,0x00,0x00, +0xc4,0x00,0x05,0x00,0x14,0x00,0x00,0x00,0x86,0x03,0x00,0x00, +0x85,0x03,0x00,0x00,0x8a,0x00,0x00,0x00,0xc5,0x00,0x05,0x00, +0x14,0x00,0x00,0x00,0x87,0x03,0x00,0x00,0x7c,0x03,0x00,0x00, +0x86,0x03,0x00,0x00,0x72,0x00,0x04,0x00,0x71,0x00,0x00,0x00, +0x88,0x03,0x00,0x00,0x87,0x03,0x00,0x00,0x72,0x00,0x04,0x00, +0x14,0x00,0x00,0x00,0x89,0x03,0x00,0x00,0x88,0x03,0x00,0x00, +0x82,0x00,0x05,0x00,0x14,0x00,0x00,0x00,0x8a,0x03,0x00,0x00, +0x89,0x03,0x00,0x00,0xc9,0x00,0x00,0x00,0x6f,0x00,0x04,0x00, +0x50,0x00,0x00,0x00,0x8b,0x03,0x00,0x00,0x8a,0x03,0x00,0x00, +0x0c,0x00,0x08,0x00,0x50,0x00,0x00,0x00,0x8d,0x03,0x00,0x00, +0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x73,0x03,0x00,0x00, +0x8b,0x03,0x00,0x00,0x64,0x03,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x93,0x03,0x00,0x00,0x94,0x00,0x00,0x00, +0xc4,0x03,0x00,0x00,0x41,0x00,0x06,0x00,0x99,0x00,0x00,0x00, +0x94,0x03,0x00,0x00,0x8f,0x00,0x00,0x00,0x15,0x00,0x00,0x00, +0x93,0x03,0x00,0x00,0x3d,0x00,0x04,0x00,0x50,0x00,0x00,0x00, +0x95,0x03,0x00,0x00,0x94,0x03,0x00,0x00,0x3d,0x00,0x04,0x00, +0x71,0x00,0x00,0x00,0x99,0x03,0x00,0x00,0x4c,0x01,0x00,0x00, +0x6f,0x00,0x04,0x00,0x50,0x00,0x00,0x00,0x9a,0x03,0x00,0x00, +0x99,0x03,0x00,0x00,0x85,0x00,0x05,0x00,0x50,0x00,0x00,0x00, +0x9b,0x03,0x00,0x00,0x95,0x03,0x00,0x00,0x9a,0x03,0x00,0x00, +0x85,0x00,0x05,0x00,0x50,0x00,0x00,0x00,0x9c,0x03,0x00,0x00, +0x9b,0x03,0x00,0x00,0x80,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x6e,0x00,0x00,0x00,0xa2,0x03,0x00,0x00,0x4f,0x03,0x00,0x00, +0xc2,0x00,0x05,0x00,0x6e,0x00,0x00,0x00,0xa3,0x03,0x00,0x00, +0xa2,0x03,0x00,0x00,0x8a,0x00,0x00,0x00,0x71,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xa4,0x03,0x00,0x00,0xa3,0x03,0x00,0x00, +0x7c,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0xa5,0x03,0x00,0x00, +0xa4,0x03,0x00,0x00,0x3d,0x00,0x04,0x00,0x6e,0x00,0x00,0x00, +0xaa,0x03,0x00,0x00,0x2f,0x03,0x00,0x00,0xc2,0x00,0x05,0x00, +0x6e,0x00,0x00,0x00,0xab,0x03,0x00,0x00,0xaa,0x03,0x00,0x00, +0x68,0x01,0x00,0x00,0x71,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xac,0x03,0x00,0x00,0xab,0x03,0x00,0x00,0x7c,0x00,0x04,0x00, +0x14,0x00,0x00,0x00,0xad,0x03,0x00,0x00,0xac,0x03,0x00,0x00, +0xc7,0x00,0x05,0x00,0x14,0x00,0x00,0x00,0xae,0x03,0x00,0x00, +0xad,0x03,0x00,0x00,0x7c,0x00,0x00,0x00,0xc4,0x00,0x05,0x00, +0x14,0x00,0x00,0x00,0xaf,0x03,0x00,0x00,0xae,0x03,0x00,0x00, +0x8a,0x00,0x00,0x00,0xc5,0x00,0x05,0x00,0x14,0x00,0x00,0x00, +0xb0,0x03,0x00,0x00,0xa5,0x03,0x00,0x00,0xaf,0x03,0x00,0x00, +0x72,0x00,0x04,0x00,0x71,0x00,0x00,0x00,0xb1,0x03,0x00,0x00, +0xb0,0x03,0x00,0x00,0x72,0x00,0x04,0x00,0x14,0x00,0x00,0x00, +0xb2,0x03,0x00,0x00,0xb1,0x03,0x00,0x00,0x82,0x00,0x05,0x00, +0x14,0x00,0x00,0x00,0xb3,0x03,0x00,0x00,0xb2,0x03,0x00,0x00, +0xc9,0x00,0x00,0x00,0x6f,0x00,0x04,0x00,0x50,0x00,0x00,0x00, +0xb4,0x03,0x00,0x00,0xb3,0x03,0x00,0x00,0x0c,0x00,0x08,0x00, +0x50,0x00,0x00,0x00,0xb6,0x03,0x00,0x00,0x01,0x00,0x00,0x00, +0x32,0x00,0x00,0x00,0x9c,0x03,0x00,0x00,0xb4,0x03,0x00,0x00, +0x8d,0x03,0x00,0x00,0x81,0x00,0x05,0x00,0x50,0x00,0x00,0x00, +0xb7,0x03,0x00,0x00,0x0b,0x03,0x00,0x00,0xb6,0x03,0x00,0x00, +0x3d,0x00,0x04,0x00,0x50,0x00,0x00,0x00,0x7f,0x01,0x00,0x00, +0x5b,0x00,0x00,0x00,0x81,0x00,0x05,0x00,0x50,0x00,0x00,0x00, +0x80,0x01,0x00,0x00,0x7f,0x01,0x00,0x00,0xb7,0x03,0x00,0x00, +0x3e,0x00,0x03,0x00,0x5b,0x00,0x00,0x00,0x80,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x83,0x01,0x00,0x00, +0xaf,0x01,0x00,0x00,0x23,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x5e,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x60,0x00,0x00,0x00, +0xe0,0x00,0x04,0x00,0x23,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x84,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x86,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x86,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xb0,0x01,0x00,0x00,0x54,0x00,0x00,0x00, +0x60,0x00,0x00,0x00,0x9d,0x01,0x00,0x00,0x89,0x01,0x00,0x00, +0xac,0x00,0x05,0x00,0x65,0x00,0x00,0x00,0x8c,0x01,0x00,0x00, +0xb0,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0x88,0x01,0x00,0x00,0x89,0x01,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x8c,0x01,0x00,0x00,0x87,0x01,0x00,0x00, +0x88,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x87,0x01,0x00,0x00, +0xb0,0x00,0x05,0x00,0x65,0x00,0x00,0x00,0x8f,0x01,0x00,0x00, +0x24,0x00,0x00,0x00,0xb0,0x01,0x00,0x00,0xf7,0x00,0x03,0x00, +0x91,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x8f,0x01,0x00,0x00,0x90,0x01,0x00,0x00,0x91,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x90,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x95,0x01,0x00,0x00,0x24,0x00,0x00,0x00, +0xb0,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0x5a,0x00,0x00,0x00, +0x96,0x01,0x00,0x00,0x53,0x00,0x00,0x00,0x95,0x01,0x00,0x00, +0x3d,0x00,0x04,0x00,0x50,0x00,0x00,0x00,0x97,0x01,0x00,0x00, +0x96,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0x5a,0x00,0x00,0x00, +0x98,0x01,0x00,0x00,0x53,0x00,0x00,0x00,0x24,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x50,0x00,0x00,0x00,0x99,0x01,0x00,0x00, +0x98,0x01,0x00,0x00,0x81,0x00,0x05,0x00,0x50,0x00,0x00,0x00, +0x9a,0x01,0x00,0x00,0x99,0x01,0x00,0x00,0x97,0x01,0x00,0x00, +0x3e,0x00,0x03,0x00,0x98,0x01,0x00,0x00,0x9a,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0x91,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x91,0x01,0x00,0x00,0xe0,0x00,0x04,0x00,0x23,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x84,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0x89,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x89,0x01,0x00,0x00, +0xc2,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9d,0x01,0x00,0x00, +0xb0,0x01,0x00,0x00,0x90,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x86,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x88,0x01,0x00,0x00, +0xaa,0x00,0x05,0x00,0x65,0x00,0x00,0x00,0x9f,0x01,0x00,0x00, +0x24,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0xf7,0x00,0x03,0x00, +0xa1,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x9f,0x01,0x00,0x00,0xa0,0x01,0x00,0x00,0xa1,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xa0,0x01,0x00,0x00,0x41,0x00,0x05,0x00, +0x16,0x00,0x00,0x00,0xa6,0x01,0x00,0x00,0x13,0x00,0x00,0x00, +0x9f,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xa7,0x01,0x00,0x00,0xa6,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xa9,0x01,0x00,0x00,0xa7,0x01,0x00,0x00, +0x0f,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x5a,0x00,0x00,0x00, +0xaa,0x01,0x00,0x00,0x53,0x00,0x00,0x00,0x15,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x50,0x00,0x00,0x00,0xab,0x01,0x00,0x00, +0xaa,0x01,0x00,0x00,0x41,0x00,0x06,0x00,0x99,0x00,0x00,0x00, +0xac,0x01,0x00,0x00,0xa5,0x01,0x00,0x00,0x15,0x00,0x00,0x00, +0xa9,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0xac,0x01,0x00,0x00, +0xab,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xa1,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xa1,0x01,0x00,0x00,0xfd,0x00,0x01,0x00, +0x38,0x00,0x01,0x00, +}; +const uint64_t mul_mat_vec_q6_K_f32_len = 10960; + +unsigned char mul_mat_vec_q8_0_f32_data[] = { +0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, +0xc4,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, +0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x51,0x11,0x00,0x00, +0x11,0x00,0x02,0x00,0x60,0x11,0x00,0x00,0x0b,0x00,0x06,0x00, +0x01,0x00,0x00,0x00,0x47,0x4c,0x53,0x4c,0x2e,0x73,0x74,0x64, +0x2e,0x34,0x35,0x30,0x00,0x00,0x00,0x00,0x0e,0x00,0x03,0x00, +0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x0f,0x00,0x0c,0x00, +0x05,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x6d,0x61,0x69,0x6e, +0x00,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x11,0x00,0x00,0x00, +0x18,0x00,0x00,0x00,0x26,0x00,0x00,0x00,0x52,0x00,0x00,0x00, +0x74,0x00,0x00,0x00,0xb5,0x00,0x00,0x00,0x10,0x00,0x06,0x00, +0x04,0x00,0x00,0x00,0x11,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x0b,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x1a,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x11,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, +0x1b,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x15,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x24,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x24,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x24,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x47,0x00,0x03,0x00, +0x24,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x4d,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x4e,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x4e,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x4f,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x48,0x00,0x04,0x00, +0x50,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x50,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, +0x50,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x52,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x52,0x00,0x00,0x00,0x21,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x71,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00, +0x72,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x72,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, +0x72,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x74,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x74,0x00,0x00,0x00,0x21,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xb2,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00, +0xb3,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x19,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0xb3,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, +0xb3,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0xb5,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0xb5,0x00,0x00,0x00,0x21,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xbe,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0xbf,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x19,0x00,0x00,0x00, +0x13,0x00,0x02,0x00,0x02,0x00,0x00,0x00,0x21,0x00,0x03,0x00, +0x03,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x15,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x17,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x03,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x0a,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0x0a,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x0d,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0x0a,0x00,0x00,0x00,0x11,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x16,0x00,0x03,0x00,0x14,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x15,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0x1c,0x00,0x04,0x00,0x16,0x00,0x00,0x00, +0x14,0x00,0x00,0x00,0x15,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x17,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x16,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0x17,0x00,0x00,0x00,0x18,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x14,0x00,0x00,0x00, +0x1a,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x1b,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x14,0x00,0x00,0x00, +0x1e,0x00,0x05,0x00,0x24,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x25,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x24,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0x25,0x00,0x00,0x00,0x26,0x00,0x00,0x00, +0x09,0x00,0x00,0x00,0x15,0x00,0x04,0x00,0x27,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x27,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x29,0x00,0x00,0x00,0x09,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x14,0x00,0x02,0x00,0x2d,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x32,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x3d,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x42,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x16,0x00,0x03,0x00,0x4b,0x00,0x00,0x00,0x10,0x00,0x00,0x00, +0x15,0x00,0x04,0x00,0x4c,0x00,0x00,0x00,0x08,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x1c,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, +0x4c,0x00,0x00,0x00,0x3d,0x00,0x00,0x00,0x1e,0x00,0x04,0x00, +0x4e,0x00,0x00,0x00,0x4b,0x00,0x00,0x00,0x4d,0x00,0x00,0x00, +0x1d,0x00,0x03,0x00,0x4f,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, +0x1e,0x00,0x03,0x00,0x50,0x00,0x00,0x00,0x4f,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x51,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x50,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x51,0x00,0x00,0x00, +0x52,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x54,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x4b,0x00,0x00,0x00, +0x17,0x00,0x04,0x00,0x58,0x00,0x00,0x00,0x14,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x27,0x00,0x00,0x00, +0x5c,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x5e,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x4c,0x00,0x00,0x00, +0x1d,0x00,0x03,0x00,0x71,0x00,0x00,0x00,0x14,0x00,0x00,0x00, +0x1e,0x00,0x03,0x00,0x72,0x00,0x00,0x00,0x71,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x73,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x72,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x73,0x00,0x00,0x00, +0x74,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x7c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x14,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x93,0x00,0x00,0x00, +0x08,0x01,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x95,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x15,0x00,0x00,0x00, +0x32,0x00,0x00,0x00,0x1d,0x00,0x03,0x00,0xb2,0x00,0x00,0x00, +0x14,0x00,0x00,0x00,0x1e,0x00,0x03,0x00,0xb3,0x00,0x00,0x00, +0xb2,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0xb4,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0xb3,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0xb4,0x00,0x00,0x00,0xb5,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x27,0x00,0x00,0x00,0xb6,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xbe,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x33,0x00,0x06,0x00, +0x09,0x00,0x00,0x00,0xbf,0x00,0x00,0x00,0xbe,0x00,0x00,0x00, +0x42,0x00,0x00,0x00,0x42,0x00,0x00,0x00,0x36,0x00,0x05,0x00, +0x02,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x03,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x05,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00,0x0e,0x00,0x00,0x00, +0x0b,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x0e,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0x11,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x13,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x1b,0x00,0x00,0x00,0x1c,0x00,0x00,0x00, +0x18,0x00,0x00,0x00,0x13,0x00,0x00,0x00,0x3e,0x00,0x03,0x00, +0x1c,0x00,0x00,0x00,0x1a,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x1e,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x1e,0x00,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xc2,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0x92,0x00,0x00,0x00, +0x1f,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x29,0x00,0x00,0x00, +0x2a,0x00,0x00,0x00,0x26,0x00,0x00,0x00,0x28,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x2b,0x00,0x00,0x00, +0x2a,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x2c,0x00,0x00,0x00,0x2b,0x00,0x00,0x00,0x15,0x00,0x00,0x00, +0xb0,0x00,0x05,0x00,0x2d,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, +0xc2,0x00,0x00,0x00,0x2c,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0x20,0x00,0x00,0x00,0x1f,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x2e,0x00,0x00,0x00,0x1f,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x1f,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x31,0x00,0x00,0x00, +0xc2,0x00,0x00,0x00,0x15,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0x32,0x00,0x00,0x00, +0x13,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x35,0x00,0x00,0x00,0x31,0x00,0x00,0x00,0x34,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, +0x0f,0x00,0x00,0x00,0x2b,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x3c,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, +0x35,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x3e,0x00,0x00,0x00,0x3c,0x00,0x00,0x00,0x3d,0x00,0x00,0x00, +0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x41,0x00,0x00,0x00, +0x35,0x00,0x00,0x00,0x3d,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x43,0x00,0x00,0x00,0x41,0x00,0x00,0x00, +0x42,0x00,0x00,0x00,0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x48,0x00,0x00,0x00,0x35,0x00,0x00,0x00,0x41,0x00,0x00,0x00, +0x41,0x00,0x07,0x00,0x54,0x00,0x00,0x00,0x55,0x00,0x00,0x00, +0x52,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0x3e,0x00,0x00,0x00, +0x28,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x4b,0x00,0x00,0x00, +0x56,0x00,0x00,0x00,0x55,0x00,0x00,0x00,0x73,0x00,0x04,0x00, +0x14,0x00,0x00,0x00,0x57,0x00,0x00,0x00,0x56,0x00,0x00,0x00, +0x41,0x00,0x08,0x00,0x5e,0x00,0x00,0x00,0x5f,0x00,0x00,0x00, +0x52,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0x3e,0x00,0x00,0x00, +0x5c,0x00,0x00,0x00,0x43,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x4c,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x5f,0x00,0x00,0x00, +0x72,0x00,0x04,0x00,0x27,0x00,0x00,0x00,0x61,0x00,0x00,0x00, +0x60,0x00,0x00,0x00,0x6f,0x00,0x04,0x00,0x14,0x00,0x00,0x00, +0x62,0x00,0x00,0x00,0x61,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x65,0x00,0x00,0x00,0x43,0x00,0x00,0x00, +0x42,0x00,0x00,0x00,0x41,0x00,0x08,0x00,0x5e,0x00,0x00,0x00, +0x66,0x00,0x00,0x00,0x52,0x00,0x00,0x00,0x28,0x00,0x00,0x00, +0x3e,0x00,0x00,0x00,0x5c,0x00,0x00,0x00,0x65,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x4c,0x00,0x00,0x00,0x67,0x00,0x00,0x00, +0x66,0x00,0x00,0x00,0x72,0x00,0x04,0x00,0x27,0x00,0x00,0x00, +0x68,0x00,0x00,0x00,0x67,0x00,0x00,0x00,0x6f,0x00,0x04,0x00, +0x14,0x00,0x00,0x00,0x69,0x00,0x00,0x00,0x68,0x00,0x00,0x00, +0x50,0x00,0x05,0x00,0x58,0x00,0x00,0x00,0x6a,0x00,0x00,0x00, +0x62,0x00,0x00,0x00,0x69,0x00,0x00,0x00,0x8e,0x00,0x05,0x00, +0x58,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x6a,0x00,0x00,0x00, +0x57,0x00,0x00,0x00,0x51,0x00,0x05,0x00,0x14,0x00,0x00,0x00, +0x70,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x29,0x00,0x00,0x00,0x75,0x00,0x00,0x00, +0x26,0x00,0x00,0x00,0x5c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x76,0x00,0x00,0x00,0x75,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x78,0x00,0x00,0x00, +0x76,0x00,0x00,0x00,0x48,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x7a,0x00,0x00,0x00,0x78,0x00,0x00,0x00, +0x43,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x7c,0x00,0x00,0x00, +0x7d,0x00,0x00,0x00,0x74,0x00,0x00,0x00,0x28,0x00,0x00,0x00, +0x7a,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x14,0x00,0x00,0x00, +0x7e,0x00,0x00,0x00,0x7d,0x00,0x00,0x00,0x51,0x00,0x05,0x00, +0x14,0x00,0x00,0x00,0x81,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x88,0x00,0x00,0x00,0x7a,0x00,0x00,0x00,0x42,0x00,0x00,0x00, +0x41,0x00,0x06,0x00,0x7c,0x00,0x00,0x00,0x89,0x00,0x00,0x00, +0x74,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0x88,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0x8a,0x00,0x00,0x00, +0x89,0x00,0x00,0x00,0x85,0x00,0x05,0x00,0x14,0x00,0x00,0x00, +0x8b,0x00,0x00,0x00,0x81,0x00,0x00,0x00,0x8a,0x00,0x00,0x00, +0x0c,0x00,0x08,0x00,0x14,0x00,0x00,0x00,0x8c,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x70,0x00,0x00,0x00, +0x7e,0x00,0x00,0x00,0x8b,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x14,0x00,0x00,0x00,0x8e,0x00,0x00,0x00,0x1c,0x00,0x00,0x00, +0x81,0x00,0x05,0x00,0x14,0x00,0x00,0x00,0x8f,0x00,0x00,0x00, +0x8e,0x00,0x00,0x00,0x8c,0x00,0x00,0x00,0x3e,0x00,0x03,0x00, +0x1c,0x00,0x00,0x00,0x8f,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x92,0x00,0x00,0x00,0xc2,0x00,0x00,0x00, +0x32,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x1e,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0x20,0x00,0x00,0x00,0xe0,0x00,0x04,0x00, +0x32,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x93,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x96,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0x96,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xc3,0x00,0x00,0x00,0x95,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0xad,0x00,0x00,0x00,0x99,0x00,0x00,0x00,0xac,0x00,0x05,0x00, +0x2d,0x00,0x00,0x00,0x9c,0x00,0x00,0x00,0xc3,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x98,0x00,0x00,0x00, +0x99,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x9c,0x00,0x00,0x00,0x97,0x00,0x00,0x00,0x98,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0x97,0x00,0x00,0x00,0xb0,0x00,0x05,0x00, +0x2d,0x00,0x00,0x00,0x9f,0x00,0x00,0x00,0x13,0x00,0x00,0x00, +0xc3,0x00,0x00,0x00,0xf7,0x00,0x03,0x00,0xa1,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x9f,0x00,0x00,0x00, +0xa0,0x00,0x00,0x00,0xa1,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xa0,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xa5,0x00,0x00,0x00,0x13,0x00,0x00,0x00,0xc3,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x1b,0x00,0x00,0x00,0xa6,0x00,0x00,0x00, +0x18,0x00,0x00,0x00,0xa5,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x14,0x00,0x00,0x00,0xa7,0x00,0x00,0x00,0xa6,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0xa9,0x00,0x00,0x00, +0x1c,0x00,0x00,0x00,0x81,0x00,0x05,0x00,0x14,0x00,0x00,0x00, +0xaa,0x00,0x00,0x00,0xa9,0x00,0x00,0x00,0xa7,0x00,0x00,0x00, +0x3e,0x00,0x03,0x00,0x1c,0x00,0x00,0x00,0xaa,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xa1,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xa1,0x00,0x00,0x00,0xe0,0x00,0x04,0x00,0x32,0x00,0x00,0x00, +0x32,0x00,0x00,0x00,0x93,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x99,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x99,0x00,0x00,0x00, +0xc2,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xad,0x00,0x00,0x00, +0xc3,0x00,0x00,0x00,0x5c,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x96,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x98,0x00,0x00,0x00, +0xaa,0x00,0x05,0x00,0x2d,0x00,0x00,0x00,0xaf,0x00,0x00,0x00, +0x13,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0xf7,0x00,0x03,0x00, +0xb1,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0xaf,0x00,0x00,0x00,0xb0,0x00,0x00,0x00,0xb1,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xb0,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x29,0x00,0x00,0x00,0xb7,0x00,0x00,0x00,0x26,0x00,0x00,0x00, +0xb6,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xb8,0x00,0x00,0x00,0xb7,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xba,0x00,0x00,0x00,0xb8,0x00,0x00,0x00, +0x0f,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x1b,0x00,0x00,0x00, +0xbb,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x28,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x14,0x00,0x00,0x00,0xbc,0x00,0x00,0x00, +0xbb,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x7c,0x00,0x00,0x00, +0xbd,0x00,0x00,0x00,0xb5,0x00,0x00,0x00,0x28,0x00,0x00,0x00, +0xba,0x00,0x00,0x00,0x3e,0x00,0x03,0x00,0xbd,0x00,0x00,0x00, +0xbc,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xb1,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xb1,0x00,0x00,0x00,0xfd,0x00,0x01,0x00, +0x38,0x00,0x01,0x00, +}; +const uint64_t mul_mat_vec_q8_0_f32_len = 3124; unsigned char norm_f32_data[] = { 0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, @@ -50951,112 +46385,284 @@ const uint64_t rope_neox_f32_len = 3792; unsigned char scale_f32_data[] = { 0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, -0x37,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, +0x9c,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, 0x01,0x00,0x00,0x00,0x0b,0x00,0x06,0x00,0x01,0x00,0x00,0x00, 0x47,0x4c,0x53,0x4c,0x2e,0x73,0x74,0x64,0x2e,0x34,0x35,0x30, 0x00,0x00,0x00,0x00,0x0e,0x00,0x03,0x00,0x00,0x00,0x00,0x00, 0x01,0x00,0x00,0x00,0x0f,0x00,0x09,0x00,0x05,0x00,0x00,0x00, 0x04,0x00,0x00,0x00,0x6d,0x61,0x69,0x6e,0x00,0x00,0x00,0x00, -0x0b,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x22,0x00,0x00,0x00, -0x27,0x00,0x00,0x00,0x10,0x00,0x06,0x00,0x04,0x00,0x00,0x00, +0x14,0x00,0x00,0x00,0xc7,0x00,0x00,0x00,0xd7,0x00,0x00,0x00, +0xe3,0x00,0x00,0x00,0x10,0x00,0x06,0x00,0x04,0x00,0x00,0x00, 0x11,0x00,0x00,0x00,0x00,0x02,0x00,0x00,0x01,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x0b,0x00,0x00,0x00, -0x0b,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x12,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x12,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x12,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x12,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x12,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x1f,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00, -0x20,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x19,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x20,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, -0x20,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x22,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x22,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x24,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00, -0x25,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x25,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, -0x25,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x27,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x27,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x34,0x00,0x00,0x00, -0x0b,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x13,0x00,0x02,0x00, -0x02,0x00,0x00,0x00,0x21,0x00,0x03,0x00,0x03,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x15,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x17,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x03,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x0a,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00, -0x0b,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x0d,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x16,0x00,0x03,0x00,0x11,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x1e,0x00,0x06,0x00,0x12,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x11,0x00,0x00,0x00, -0x11,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x13,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x15,0x00,0x04,0x00,0x15,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x15,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x17,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x14,0x00,0x02,0x00,0x1a,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, -0x1f,0x00,0x00,0x00,0x11,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, -0x20,0x00,0x00,0x00,0x1f,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x21,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x21,0x00,0x00,0x00,0x22,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x1d,0x00,0x03,0x00,0x24,0x00,0x00,0x00, -0x11,0x00,0x00,0x00,0x1e,0x00,0x03,0x00,0x25,0x00,0x00,0x00, -0x24,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x26,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x25,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x26,0x00,0x00,0x00,0x27,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x29,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x12,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x12,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x12,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x08,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x12,0x00,0x00,0x00, +0x03,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x12,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x12,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x14,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x12,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x18,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x12,0x00,0x00,0x00,0x07,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x12,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x12,0x00,0x00,0x00, +0x09,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x24,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x12,0x00,0x00,0x00,0x0a,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x12,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x2c,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x12,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x30,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x12,0x00,0x00,0x00,0x0d,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x12,0x00,0x00,0x00,0x0e,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x38,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x12,0x00,0x00,0x00, +0x0f,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x3c,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x12,0x00,0x00,0x00,0x10,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x12,0x00,0x00,0x00,0x11,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x44,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x12,0x00,0x00,0x00, +0x12,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x48,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x12,0x00,0x00,0x00,0x13,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x4c,0x00,0x00,0x00,0x47,0x00,0x03,0x00, +0x12,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0xc7,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x1c,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0xd4,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0xd5,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0xd5,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0xd5,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xd7,0x00,0x00,0x00, +0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0xd7,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0xe0,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0xe1,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0xe1,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0xe1,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xe3,0x00,0x00,0x00, +0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0xe3,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0xf3,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, +0x19,0x00,0x00,0x00,0x13,0x00,0x02,0x00,0x02,0x00,0x00,0x00, +0x21,0x00,0x03,0x00,0x03,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x15,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x16,0x00,0x03,0x00,0x11,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0x1e,0x00,0x16,0x00,0x12,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x11,0x00,0x00,0x00,0x11,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x14,0x00,0x00,0x00, +0x09,0x00,0x00,0x00,0x15,0x00,0x04,0x00,0x15,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x15,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x03,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x17,0x00,0x00,0x00,0x09,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x15,0x00,0x00,0x00, +0x1a,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x15,0x00,0x00,0x00,0x1e,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x15,0x00,0x00,0x00,0x55,0x00,0x00,0x00, +0x08,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x15,0x00,0x00,0x00, +0x5a,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x15,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x15,0x00,0x00,0x00,0x66,0x00,0x00,0x00, +0x05,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x15,0x00,0x00,0x00, +0x6f,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x15,0x00,0x00,0x00,0x72,0x00,0x00,0x00,0x0a,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x15,0x00,0x00,0x00,0x76,0x00,0x00,0x00, +0x09,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x15,0x00,0x00,0x00, +0xad,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x15,0x00,0x00,0x00,0xb2,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x15,0x00,0x00,0x00,0xb8,0x00,0x00,0x00, +0x0e,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x15,0x00,0x00,0x00, +0xbe,0x00,0x00,0x00,0x0d,0x00,0x00,0x00,0x17,0x00,0x04,0x00, +0xc5,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x03,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0xc6,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0xc5,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0xc6,0x00,0x00,0x00, +0xc7,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xc8,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0xc9,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x15,0x00,0x00,0x00, +0xcc,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x14,0x00,0x02,0x00, +0xcf,0x00,0x00,0x00,0x1d,0x00,0x03,0x00,0xd4,0x00,0x00,0x00, +0x11,0x00,0x00,0x00,0x1e,0x00,0x03,0x00,0xd5,0x00,0x00,0x00, +0xd4,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0xd6,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0xd5,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0xd6,0x00,0x00,0x00,0xd7,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x15,0x00,0x00,0x00,0xd8,0x00,0x00,0x00, +0x11,0x00,0x00,0x00,0x1d,0x00,0x03,0x00,0xe0,0x00,0x00,0x00, +0x11,0x00,0x00,0x00,0x1e,0x00,0x03,0x00,0xe1,0x00,0x00,0x00, +0xe0,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0xe2,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0xe1,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0xe2,0x00,0x00,0x00,0xe3,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0xe8,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, 0x11,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x15,0x00,0x00,0x00, -0x2c,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x2d,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x11,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x32,0x00,0x00,0x00, +0xeb,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0xec,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x11,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xf1,0x00,0x00,0x00, 0x00,0x02,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x33,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x2c,0x00,0x06,0x00, -0x09,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0x32,0x00,0x00,0x00, -0x33,0x00,0x00,0x00,0x33,0x00,0x00,0x00,0x36,0x00,0x05,0x00, +0xf2,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x2c,0x00,0x06,0x00, +0xc5,0x00,0x00,0x00,0xf3,0x00,0x00,0x00,0xf1,0x00,0x00,0x00, +0xf2,0x00,0x00,0x00,0xf2,0x00,0x00,0x00,0x36,0x00,0x05,0x00, 0x02,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x00, 0x03,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x05,0x00,0x00,0x00, -0xf7,0x00,0x03,0x00,0x35,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0xfb,0x00,0x03,0x00,0x0c,0x00,0x00,0x00,0x36,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0x36,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x0d,0x00,0x00,0x00,0x0e,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x0f,0x00,0x00,0x00,0x0e,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x17,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x14,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x19,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0xae,0x00,0x05,0x00, -0x1a,0x00,0x00,0x00,0x1b,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, -0x19,0x00,0x00,0x00,0xf7,0x00,0x03,0x00,0x1d,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x1b,0x00,0x00,0x00, -0x1c,0x00,0x00,0x00,0x1d,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x1c,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x35,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0x1d,0x00,0x00,0x00,0x41,0x00,0x06,0x00, -0x29,0x00,0x00,0x00,0x2a,0x00,0x00,0x00,0x27,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x11,0x00,0x00,0x00,0x2b,0x00,0x00,0x00,0x2a,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x2d,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x14,0x00,0x00,0x00,0x2c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x11,0x00,0x00,0x00,0x2f,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x85,0x00,0x05,0x00,0x11,0x00,0x00,0x00,0x30,0x00,0x00,0x00, -0x2b,0x00,0x00,0x00,0x2f,0x00,0x00,0x00,0x41,0x00,0x06,0x00, -0x29,0x00,0x00,0x00,0x31,0x00,0x00,0x00,0x22,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x3e,0x00,0x03,0x00, -0x31,0x00,0x00,0x00,0x30,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x35,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x35,0x00,0x00,0x00, +0xf7,0x00,0x03,0x00,0xf4,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0xfb,0x00,0x03,0x00,0xc8,0x00,0x00,0x00,0xf5,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xf5,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0xc9,0x00,0x00,0x00,0xca,0x00,0x00,0x00,0xc7,0x00,0x00,0x00, +0xc8,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xcb,0x00,0x00,0x00,0xca,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x17,0x00,0x00,0x00,0xcd,0x00,0x00,0x00,0x14,0x00,0x00,0x00, +0xcc,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xce,0x00,0x00,0x00,0xcd,0x00,0x00,0x00,0xae,0x00,0x05,0x00, +0xcf,0x00,0x00,0x00,0xd0,0x00,0x00,0x00,0xcb,0x00,0x00,0x00, +0xce,0x00,0x00,0x00,0xf7,0x00,0x03,0x00,0xd2,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xd0,0x00,0x00,0x00, +0xd1,0x00,0x00,0x00,0xd2,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xd1,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xf4,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xd2,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x17,0x00,0x00,0x00,0xd9,0x00,0x00,0x00,0x14,0x00,0x00,0x00, +0xd8,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xda,0x00,0x00,0x00,0xd9,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x17,0x00,0x00,0x00,0x03,0x01,0x00,0x00,0x14,0x00,0x00,0x00, +0x6f,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x04,0x01,0x00,0x00,0x03,0x01,0x00,0x00,0x41,0x00,0x05,0x00, +0x17,0x00,0x00,0x00,0x05,0x01,0x00,0x00,0x14,0x00,0x00,0x00, +0x72,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x06,0x01,0x00,0x00,0x05,0x01,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x07,0x01,0x00,0x00,0x04,0x01,0x00,0x00, +0x06,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00, +0x08,0x01,0x00,0x00,0x14,0x00,0x00,0x00,0x76,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x09,0x01,0x00,0x00, +0x08,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x0a,0x01,0x00,0x00,0x07,0x01,0x00,0x00,0x09,0x01,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x0b,0x01,0x00,0x00, +0xcb,0x00,0x00,0x00,0x0a,0x01,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x0f,0x01,0x00,0x00,0x0b,0x01,0x00,0x00, +0x04,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x12,0x01,0x00,0x00,0x0f,0x01,0x00,0x00,0x06,0x01,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x15,0x01,0x00,0x00, +0x12,0x01,0x00,0x00,0x09,0x01,0x00,0x00,0x82,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x18,0x01,0x00,0x00,0xcb,0x00,0x00,0x00, +0x15,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x1d,0x01,0x00,0x00,0x06,0x01,0x00,0x00,0x09,0x01,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x1e,0x01,0x00,0x00, +0x18,0x01,0x00,0x00,0x1d,0x01,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x22,0x01,0x00,0x00,0x1e,0x01,0x00,0x00, +0x06,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x25,0x01,0x00,0x00,0x22,0x01,0x00,0x00,0x09,0x01,0x00,0x00, +0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x2a,0x01,0x00,0x00, +0x18,0x01,0x00,0x00,0x25,0x01,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x2d,0x01,0x00,0x00,0x2a,0x01,0x00,0x00, +0x09,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x36,0x01,0x00,0x00,0x2d,0x01,0x00,0x00,0x09,0x01,0x00,0x00, +0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x37,0x01,0x00,0x00, +0x2a,0x01,0x00,0x00,0x36,0x01,0x00,0x00,0x41,0x00,0x05,0x00, +0x17,0x00,0x00,0x00,0x39,0x01,0x00,0x00,0x14,0x00,0x00,0x00, +0xad,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x3a,0x01,0x00,0x00,0x39,0x01,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x3b,0x01,0x00,0x00,0x0b,0x01,0x00,0x00, +0x3a,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00, +0x3d,0x01,0x00,0x00,0x14,0x00,0x00,0x00,0xb2,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x3e,0x01,0x00,0x00, +0x3d,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x3f,0x01,0x00,0x00,0x1e,0x01,0x00,0x00,0x3e,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x40,0x01,0x00,0x00, +0x3b,0x01,0x00,0x00,0x3f,0x01,0x00,0x00,0x41,0x00,0x05,0x00, +0x17,0x00,0x00,0x00,0x42,0x01,0x00,0x00,0x14,0x00,0x00,0x00, +0xb8,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x43,0x01,0x00,0x00,0x42,0x01,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x44,0x01,0x00,0x00,0x2d,0x01,0x00,0x00, +0x43,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x45,0x01,0x00,0x00,0x40,0x01,0x00,0x00,0x44,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00,0x47,0x01,0x00,0x00, +0x14,0x00,0x00,0x00,0xbe,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x48,0x01,0x00,0x00,0x47,0x01,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x49,0x01,0x00,0x00, +0x37,0x01,0x00,0x00,0x48,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x4a,0x01,0x00,0x00,0x45,0x01,0x00,0x00, +0x49,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xdf,0x00,0x00,0x00,0xda,0x00,0x00,0x00,0x4a,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00,0x54,0x01,0x00,0x00, +0x14,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x55,0x01,0x00,0x00,0x54,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00,0x56,0x01,0x00,0x00, +0x14,0x00,0x00,0x00,0x1a,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x57,0x01,0x00,0x00,0x56,0x01,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x58,0x01,0x00,0x00, +0x55,0x01,0x00,0x00,0x57,0x01,0x00,0x00,0x41,0x00,0x05,0x00, +0x17,0x00,0x00,0x00,0x59,0x01,0x00,0x00,0x14,0x00,0x00,0x00, +0x1e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x5a,0x01,0x00,0x00,0x59,0x01,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x5b,0x01,0x00,0x00,0x58,0x01,0x00,0x00, +0x5a,0x01,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x5c,0x01,0x00,0x00,0xcb,0x00,0x00,0x00,0x5b,0x01,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x60,0x01,0x00,0x00, +0x5c,0x01,0x00,0x00,0x55,0x01,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x63,0x01,0x00,0x00,0x60,0x01,0x00,0x00, +0x57,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x66,0x01,0x00,0x00,0x63,0x01,0x00,0x00,0x5a,0x01,0x00,0x00, +0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x69,0x01,0x00,0x00, +0xcb,0x00,0x00,0x00,0x66,0x01,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x6e,0x01,0x00,0x00,0x57,0x01,0x00,0x00, +0x5a,0x01,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x6f,0x01,0x00,0x00,0x69,0x01,0x00,0x00,0x6e,0x01,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x73,0x01,0x00,0x00, +0x6f,0x01,0x00,0x00,0x57,0x01,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x76,0x01,0x00,0x00,0x73,0x01,0x00,0x00, +0x5a,0x01,0x00,0x00,0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x7b,0x01,0x00,0x00,0x69,0x01,0x00,0x00,0x76,0x01,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x7e,0x01,0x00,0x00, +0x7b,0x01,0x00,0x00,0x5a,0x01,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x87,0x01,0x00,0x00,0x7e,0x01,0x00,0x00, +0x5a,0x01,0x00,0x00,0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x88,0x01,0x00,0x00,0x7b,0x01,0x00,0x00,0x87,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00,0x8a,0x01,0x00,0x00, +0x14,0x00,0x00,0x00,0x55,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x8b,0x01,0x00,0x00,0x8a,0x01,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8c,0x01,0x00,0x00, +0x5c,0x01,0x00,0x00,0x8b,0x01,0x00,0x00,0x41,0x00,0x05,0x00, +0x17,0x00,0x00,0x00,0x8e,0x01,0x00,0x00,0x14,0x00,0x00,0x00, +0x5a,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x8f,0x01,0x00,0x00,0x8e,0x01,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x90,0x01,0x00,0x00,0x6f,0x01,0x00,0x00, +0x8f,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x91,0x01,0x00,0x00,0x8c,0x01,0x00,0x00,0x90,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00,0x93,0x01,0x00,0x00, +0x14,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x94,0x01,0x00,0x00,0x93,0x01,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x95,0x01,0x00,0x00, +0x7e,0x01,0x00,0x00,0x94,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x96,0x01,0x00,0x00,0x91,0x01,0x00,0x00, +0x95,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00, +0x98,0x01,0x00,0x00,0x14,0x00,0x00,0x00,0x66,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x99,0x01,0x00,0x00, +0x98,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x9a,0x01,0x00,0x00,0x88,0x01,0x00,0x00,0x99,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9b,0x01,0x00,0x00, +0x96,0x01,0x00,0x00,0x9a,0x01,0x00,0x00,0x41,0x00,0x06,0x00, +0xe8,0x00,0x00,0x00,0xe9,0x00,0x00,0x00,0xe3,0x00,0x00,0x00, +0xcc,0x00,0x00,0x00,0x9b,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, +0x11,0x00,0x00,0x00,0xea,0x00,0x00,0x00,0xe9,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0xec,0x00,0x00,0x00,0xed,0x00,0x00,0x00, +0x14,0x00,0x00,0x00,0xeb,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x11,0x00,0x00,0x00,0xee,0x00,0x00,0x00,0xed,0x00,0x00,0x00, +0x81,0x00,0x05,0x00,0x11,0x00,0x00,0x00,0xef,0x00,0x00,0x00, +0xea,0x00,0x00,0x00,0xee,0x00,0x00,0x00,0x41,0x00,0x06,0x00, +0xe8,0x00,0x00,0x00,0xf0,0x00,0x00,0x00,0xd7,0x00,0x00,0x00, +0xcc,0x00,0x00,0x00,0xdf,0x00,0x00,0x00,0x3e,0x00,0x03,0x00, +0xf0,0x00,0x00,0x00,0xef,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xf4,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xf4,0x00,0x00,0x00, 0xfd,0x00,0x01,0x00,0x38,0x00,0x01,0x00, }; -const uint64_t scale_f32_len = 1256; +const uint64_t scale_f32_len = 3320; unsigned char silu_f32_data[] = { 0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, @@ -51170,50 +46776,65 @@ const uint64_t silu_f32_len = 1264; unsigned char soft_max_f32_data[] = { 0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, -0x0e,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, +0x5c,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, 0x01,0x00,0x00,0x00,0x0b,0x00,0x06,0x00,0x01,0x00,0x00,0x00, 0x47,0x4c,0x53,0x4c,0x2e,0x73,0x74,0x64,0x2e,0x34,0x35,0x30, 0x00,0x00,0x00,0x00,0x0e,0x00,0x03,0x00,0x00,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x0f,0x00,0x0c,0x00,0x05,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x0f,0x00,0x0d,0x00,0x05,0x00,0x00,0x00, 0x04,0x00,0x00,0x00,0x6d,0x61,0x69,0x6e,0x00,0x00,0x00,0x00, 0x0b,0x00,0x00,0x00,0x11,0x00,0x00,0x00,0x19,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x3d,0x00,0x00,0x00,0x56,0x00,0x00,0x00, -0xc2,0x00,0x00,0x00,0x10,0x00,0x06,0x00,0x04,0x00,0x00,0x00, -0x11,0x00,0x00,0x00,0x00,0x02,0x00,0x00,0x01,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x0b,0x00,0x00,0x00, -0x0b,0x00,0x00,0x00,0x1b,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x11,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x1a,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x17,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x17,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x17,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x08,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x17,0x00,0x00,0x00,0x03,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x47,0x00,0x03,0x00, +0x5c,0x00,0x00,0x00,0x75,0x00,0x00,0x00,0x8c,0x00,0x00,0x00, +0xa3,0x00,0x00,0x00,0x09,0x01,0x00,0x00,0x10,0x00,0x06,0x00, +0x04,0x00,0x00,0x00,0x11,0x00,0x00,0x00,0x00,0x02,0x00,0x00, +0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x0b,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x1b,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x11,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, +0x1a,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x17,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x17,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x17,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x08,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x17,0x00,0x00,0x00, +0x03,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x17,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x17,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x14,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x17,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x18,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x17,0x00,0x00,0x00,0x07,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x47,0x00,0x03,0x00, 0x17,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x3a,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x48,0x00,0x04,0x00,0x3b,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x3b,0x00,0x00,0x00, +0x72,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x48,0x00,0x04,0x00,0x73,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x73,0x00,0x00,0x00, 0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x03,0x00,0x3b,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x3d,0x00,0x00,0x00,0x22,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x3d,0x00,0x00,0x00, +0x47,0x00,0x03,0x00,0x73,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x75,0x00,0x00,0x00,0x22,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x75,0x00,0x00,0x00, 0x21,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x53,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x48,0x00,0x04,0x00,0x54,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x54,0x00,0x00,0x00, +0x89,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x48,0x00,0x04,0x00,0x8a,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x8a,0x00,0x00,0x00, 0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x03,0x00,0x54,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x56,0x00,0x00,0x00,0x22,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x56,0x00,0x00,0x00, +0x47,0x00,0x03,0x00,0x8a,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x8c,0x00,0x00,0x00,0x22,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x8c,0x00,0x00,0x00, 0x21,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0xbf,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0xc0,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0xa0,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x48,0x00,0x04,0x00,0xa1,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0xa1,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x03,0x00,0xa1,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0xa3,0x00,0x00,0x00,0x22,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xa3,0x00,0x00,0x00, +0x21,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x06,0x01,0x00,0x00,0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x07,0x01,0x00,0x00,0x00,0x00,0x00,0x00, 0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, -0xc0,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0xc2,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0xc2,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xff,0x00,0x00,0x00, +0x07,0x01,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x09,0x01,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x09,0x01,0x00,0x00,0x21,0x00,0x00,0x00, +0x03,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x45,0x01,0x00,0x00, 0x0b,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x13,0x00,0x02,0x00, 0x02,0x00,0x00,0x00,0x21,0x00,0x03,0x00,0x03,0x00,0x00,0x00, 0x02,0x00,0x00,0x00,0x15,0x00,0x04,0x00,0x06,0x00,0x00,0x00, @@ -51226,264 +46847,346 @@ unsigned char soft_max_f32_data[] = { 0x20,0x00,0x04,0x00,0x0d,0x00,0x00,0x00,0x01,0x00,0x00,0x00, 0x06,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00, 0x11,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x16,0x00,0x03,0x00, -0x16,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x1e,0x00,0x06,0x00, +0x16,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x1e,0x00,0x0a,0x00, 0x17,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x18,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x17,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x18,0x00,0x00,0x00,0x19,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x15,0x00,0x04,0x00,0x1a,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x1a,0x00,0x00,0x00,0x1b,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x1c,0x00,0x00,0x00,0x09,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x16,0x00,0x00,0x00, +0x16,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x18,0x00,0x00,0x00,0x09,0x00,0x00,0x00, +0x17,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x18,0x00,0x00,0x00, +0x19,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x15,0x00,0x04,0x00, +0x1a,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x1a,0x00,0x00,0x00,0x1b,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x1c,0x00,0x00,0x00, +0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x16,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x1a,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x24,0x00,0x00,0x00, +0x09,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x14,0x00,0x02,0x00, +0x27,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x1a,0x00,0x00,0x00, +0x32,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x1a,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x05,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x1a,0x00,0x00,0x00,0x3d,0x00,0x00,0x00, 0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x00,0x02,0x00,0x00,0x1c,0x00,0x04,0x00, -0x21,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x22,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x21,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x22,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x27,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x1a,0x00,0x00,0x00,0x31,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x14,0x00,0x02,0x00,0x34,0x00,0x00,0x00, -0x1d,0x00,0x03,0x00,0x3a,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0x1e,0x00,0x03,0x00,0x3b,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x3c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x3b,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x3c,0x00,0x00,0x00, -0x3d,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x44,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x1a,0x00,0x00,0x00,0x47,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x48,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, -0x53,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, -0x54,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x55,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x54,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x55,0x00,0x00,0x00,0x56,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x16,0x00,0x00,0x00, -0x60,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x67,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x68,0x00,0x00,0x00, +0x4a,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x4d,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x59,0x00,0x00,0x00, +0x00,0x02,0x00,0x00,0x1c,0x00,0x04,0x00,0x5a,0x00,0x00,0x00, +0x16,0x00,0x00,0x00,0x59,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x5b,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x5a,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0x5b,0x00,0x00,0x00,0x5c,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x60,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x1a,0x00,0x00,0x00,0x6a,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x1d,0x00,0x03,0x00,0x72,0x00,0x00,0x00,0x16,0x00,0x00,0x00, +0x1e,0x00,0x03,0x00,0x73,0x00,0x00,0x00,0x72,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x74,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x73,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x74,0x00,0x00,0x00, +0x75,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x7c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x16,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x1a,0x00,0x00,0x00,0x7f,0x00,0x00,0x00, +0x03,0x00,0x00,0x00,0x1d,0x00,0x03,0x00,0x89,0x00,0x00,0x00, +0x16,0x00,0x00,0x00,0x1e,0x00,0x03,0x00,0x8a,0x00,0x00,0x00, +0x89,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x8b,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x8a,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0x8b,0x00,0x00,0x00,0x8c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x1a,0x00,0x00,0x00,0x98,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x1d,0x00,0x03,0x00,0xa0,0x00,0x00,0x00, +0x16,0x00,0x00,0x00,0x1e,0x00,0x03,0x00,0xa1,0x00,0x00,0x00, +0xa0,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0xa2,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0xa1,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0xa2,0x00,0x00,0x00,0xa3,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xaf,0x00,0x00,0x00, 0x08,0x01,0x00,0x00,0x2b,0x00,0x04,0x00,0x1a,0x00,0x00,0x00, -0x6b,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x1d,0x00,0x03,0x00, -0xbf,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, -0xc0,0x00,0x00,0x00,0xbf,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0xc1,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0xc0,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0xc1,0x00,0x00,0x00,0xc2,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xfe,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x2c,0x00,0x06,0x00, -0x09,0x00,0x00,0x00,0xff,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0xfe,0x00,0x00,0x00,0xfe,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x16,0x00,0x00,0x00,0x0d,0x01,0x00,0x00,0x00,0x00,0x80,0xff, -0x36,0x00,0x05,0x00,0x02,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x05,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00, -0x0e,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, -0x0e,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x11,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x13,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x1c,0x00,0x00,0x00, -0x1d,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x1b,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x1e,0x00,0x00,0x00, -0x1d,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x1f,0x00,0x00,0x00,0x13,0x00,0x00,0x00,0x1e,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x27,0x00,0x00,0x00,0x28,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x3e,0x00,0x03,0x00, -0x28,0x00,0x00,0x00,0x0d,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0x2b,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x2b,0x00,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x00,0x01,0x00,0x00, -0x0f,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0x66,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x1c,0x00,0x00,0x00, -0x32,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x31,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x33,0x00,0x00,0x00, -0x32,0x00,0x00,0x00,0xb0,0x00,0x05,0x00,0x34,0x00,0x00,0x00, -0x35,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x33,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0x2d,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x35,0x00,0x00,0x00, -0x2c,0x00,0x00,0x00,0x2d,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x2c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x16,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x41,0x00,0x00,0x00,0x13,0x00,0x00,0x00, -0x33,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x43,0x00,0x00,0x00,0x41,0x00,0x00,0x00,0x00,0x01,0x00,0x00, -0x41,0x00,0x06,0x00,0x44,0x00,0x00,0x00,0x45,0x00,0x00,0x00, -0x3d,0x00,0x00,0x00,0x31,0x00,0x00,0x00,0x43,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x16,0x00,0x00,0x00,0x46,0x00,0x00,0x00, -0x45,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x48,0x00,0x00,0x00, -0x49,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x47,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x16,0x00,0x00,0x00,0x4a,0x00,0x00,0x00, -0x49,0x00,0x00,0x00,0xac,0x00,0x05,0x00,0x34,0x00,0x00,0x00, -0x4e,0x00,0x00,0x00,0x1e,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0xf7,0x00,0x03,0x00,0x52,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x4e,0x00,0x00,0x00,0x51,0x00,0x00,0x00, -0x5f,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x51,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x5a,0x00,0x00,0x00, -0x1f,0x00,0x00,0x00,0x33,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x5c,0x00,0x00,0x00,0x5a,0x00,0x00,0x00, -0x00,0x01,0x00,0x00,0x41,0x00,0x06,0x00,0x44,0x00,0x00,0x00, -0x5d,0x00,0x00,0x00,0x56,0x00,0x00,0x00,0x31,0x00,0x00,0x00, -0x5c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x16,0x00,0x00,0x00, -0x5e,0x00,0x00,0x00,0x5d,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x52,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x5f,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x52,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x52,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x16,0x00,0x00,0x00, -0x09,0x01,0x00,0x00,0x5e,0x00,0x00,0x00,0x51,0x00,0x00,0x00, -0x60,0x00,0x00,0x00,0x5f,0x00,0x00,0x00,0x0c,0x00,0x08,0x00, -0x16,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x32,0x00,0x00,0x00,0x46,0x00,0x00,0x00,0x4a,0x00,0x00,0x00, -0x09,0x01,0x00,0x00,0x0c,0x00,0x07,0x00,0x16,0x00,0x00,0x00, -0x63,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x28,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x3e,0x00,0x03,0x00, -0x28,0x00,0x00,0x00,0x63,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x2e,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x2e,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x66,0x00,0x00,0x00, -0x00,0x01,0x00,0x00,0x20,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x2b,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x2d,0x00,0x00,0x00, -0xe0,0x00,0x04,0x00,0x67,0x00,0x00,0x00,0x67,0x00,0x00,0x00, -0x68,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x6c,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0x6c,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, -0x1a,0x00,0x00,0x00,0x01,0x01,0x00,0x00,0x6b,0x00,0x00,0x00, -0x2d,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x6f,0x00,0x00,0x00, -0xad,0x00,0x05,0x00,0x34,0x00,0x00,0x00,0x72,0x00,0x00,0x00, -0x01,0x01,0x00,0x00,0x31,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0x6e,0x00,0x00,0x00,0x6f,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x72,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, -0x6e,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x6d,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x75,0x00,0x00,0x00, -0x01,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0x34,0x00,0x00,0x00, -0x76,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x75,0x00,0x00,0x00, -0xf7,0x00,0x03,0x00,0x78,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x76,0x00,0x00,0x00,0x77,0x00,0x00,0x00, -0x78,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x77,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x16,0x00,0x00,0x00,0x7c,0x00,0x00,0x00, -0x28,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x80,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x75,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x27,0x00,0x00,0x00,0x81,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x80,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x16,0x00,0x00,0x00,0x82,0x00,0x00,0x00,0x81,0x00,0x00,0x00, -0x0c,0x00,0x07,0x00,0x16,0x00,0x00,0x00,0x83,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0x7c,0x00,0x00,0x00, -0x82,0x00,0x00,0x00,0x3e,0x00,0x03,0x00,0x28,0x00,0x00,0x00, -0x83,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x78,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0x78,0x00,0x00,0x00,0xe0,0x00,0x04,0x00, -0x67,0x00,0x00,0x00,0x67,0x00,0x00,0x00,0x68,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x6f,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x6f,0x00,0x00,0x00,0xc3,0x00,0x05,0x00,0x1a,0x00,0x00,0x00, -0x86,0x00,0x00,0x00,0x01,0x01,0x00,0x00,0x1b,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x6c,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x6e,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x27,0x00,0x00,0x00, -0x88,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x31,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x16,0x00,0x00,0x00,0x89,0x00,0x00,0x00, -0x88,0x00,0x00,0x00,0xe0,0x00,0x04,0x00,0x67,0x00,0x00,0x00, -0x67,0x00,0x00,0x00,0x68,0x00,0x00,0x00,0x3e,0x00,0x03,0x00, -0x28,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x8e,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x8e,0x00,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x02,0x01,0x00,0x00, -0x0f,0x00,0x00,0x00,0x6e,0x00,0x00,0x00,0xc7,0x00,0x00,0x00, -0x91,0x00,0x00,0x00,0xb0,0x00,0x05,0x00,0x34,0x00,0x00,0x00, -0x96,0x00,0x00,0x00,0x02,0x01,0x00,0x00,0x33,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0x90,0x00,0x00,0x00,0x91,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x96,0x00,0x00,0x00, -0x8f,0x00,0x00,0x00,0x90,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x8f,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x9b,0x00,0x00,0x00,0x13,0x00,0x00,0x00,0x33,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9d,0x00,0x00,0x00, -0x9b,0x00,0x00,0x00,0x02,0x01,0x00,0x00,0x41,0x00,0x06,0x00, -0x44,0x00,0x00,0x00,0xa0,0x00,0x00,0x00,0x3d,0x00,0x00,0x00, -0x31,0x00,0x00,0x00,0x9d,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x16,0x00,0x00,0x00,0xa1,0x00,0x00,0x00,0xa0,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x48,0x00,0x00,0x00,0xa2,0x00,0x00,0x00, -0x19,0x00,0x00,0x00,0x47,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x16,0x00,0x00,0x00,0xa3,0x00,0x00,0x00,0xa2,0x00,0x00,0x00, -0xac,0x00,0x05,0x00,0x34,0x00,0x00,0x00,0xa7,0x00,0x00,0x00, +0xb2,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x1d,0x00,0x03,0x00, +0x06,0x01,0x00,0x00,0x16,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, +0x07,0x01,0x00,0x00,0x06,0x01,0x00,0x00,0x20,0x00,0x04,0x00, +0x08,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x07,0x01,0x00,0x00, +0x3b,0x00,0x04,0x00,0x08,0x01,0x00,0x00,0x09,0x01,0x00,0x00, +0x0c,0x00,0x00,0x00,0x2c,0x00,0x06,0x00,0x09,0x00,0x00,0x00, +0x45,0x01,0x00,0x00,0x59,0x00,0x00,0x00,0x4a,0x00,0x00,0x00, +0x4a,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x16,0x00,0x00,0x00, +0x5b,0x01,0x00,0x00,0x00,0x00,0x80,0xff,0x36,0x00,0x05,0x00, +0x02,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x03,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x05,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00,0x0e,0x00,0x00,0x00, +0x0b,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x0e,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0x11,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x13,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x1c,0x00,0x00,0x00,0x1d,0x00,0x00,0x00, +0x19,0x00,0x00,0x00,0x1b,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x1e,0x00,0x00,0x00,0x1d,0x00,0x00,0x00, +0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x1f,0x00,0x00,0x00, +0x13,0x00,0x00,0x00,0x1e,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x24,0x00,0x00,0x00,0x25,0x00,0x00,0x00,0x19,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x16,0x00,0x00,0x00, +0x26,0x00,0x00,0x00,0x25,0x00,0x00,0x00,0xba,0x00,0x05,0x00, +0x27,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0x26,0x00,0x00,0x00, +0x22,0x00,0x00,0x00,0xf7,0x00,0x03,0x00,0x2a,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x28,0x00,0x00,0x00, +0x29,0x00,0x00,0x00,0x2a,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0x29,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x2f,0x00,0x00,0x00,0x13,0x00,0x00,0x00,0x1e,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x1c,0x00,0x00,0x00,0x33,0x00,0x00,0x00, +0x19,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0x33,0x00,0x00,0x00, +0xb0,0x00,0x05,0x00,0x27,0x00,0x00,0x00,0x35,0x00,0x00,0x00, +0x2f,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0xf7,0x00,0x03,0x00, +0x38,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x35,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x3c,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0x37,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x24,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x19,0x00,0x00,0x00, +0x39,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x16,0x00,0x00,0x00, +0x3b,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x38,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x3c,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x24,0x00,0x00,0x00,0x3e,0x00,0x00,0x00, +0x19,0x00,0x00,0x00,0x3d,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x16,0x00,0x00,0x00,0x3f,0x00,0x00,0x00,0x3e,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x38,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0x38,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x16,0x00,0x00,0x00, +0x46,0x01,0x00,0x00,0x3b,0x00,0x00,0x00,0x37,0x00,0x00,0x00, +0x3f,0x00,0x00,0x00,0x3c,0x00,0x00,0x00,0xf7,0x00,0x03,0x00, +0x48,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x35,0x00,0x00,0x00,0x47,0x00,0x00,0x00,0x4c,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0x47,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x4b,0x00,0x00,0x00,0x2f,0x00,0x00,0x00, +0x4a,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x48,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0x4c,0x00,0x00,0x00,0x82,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x51,0x00,0x00,0x00,0x2f,0x00,0x00,0x00, +0x34,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x52,0x00,0x00,0x00,0x4d,0x00,0x00,0x00,0x51,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x53,0x00,0x00,0x00, +0x52,0x00,0x00,0x00,0x4a,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x48,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x48,0x00,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x47,0x01,0x00,0x00, +0x4b,0x00,0x00,0x00,0x47,0x00,0x00,0x00,0x53,0x00,0x00,0x00, +0x4c,0x00,0x00,0x00,0x70,0x00,0x04,0x00,0x16,0x00,0x00,0x00, +0x57,0x00,0x00,0x00,0x47,0x01,0x00,0x00,0x0c,0x00,0x07,0x00, +0x16,0x00,0x00,0x00,0x58,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x1a,0x00,0x00,0x00,0x46,0x01,0x00,0x00,0x57,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x2a,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0x2a,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x16,0x00,0x00,0x00, +0x54,0x01,0x00,0x00,0x22,0x00,0x00,0x00,0x05,0x00,0x00,0x00, +0x58,0x00,0x00,0x00,0x48,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x60,0x00,0x00,0x00,0x61,0x00,0x00,0x00,0x5c,0x00,0x00,0x00, +0x0f,0x00,0x00,0x00,0x3e,0x00,0x03,0x00,0x61,0x00,0x00,0x00, +0x5b,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x64,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0x64,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0x48,0x01,0x00,0x00,0x0f,0x00,0x00,0x00, +0x2a,0x00,0x00,0x00,0xae,0x00,0x00,0x00,0x67,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x1c,0x00,0x00,0x00,0x6b,0x00,0x00,0x00, +0x19,0x00,0x00,0x00,0x6a,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x6b,0x00,0x00,0x00, +0xb0,0x00,0x05,0x00,0x27,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, +0x48,0x01,0x00,0x00,0x6c,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0x66,0x00,0x00,0x00,0x67,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x6d,0x00,0x00,0x00,0x65,0x00,0x00,0x00, +0x66,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x65,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x16,0x00,0x00,0x00,0x71,0x00,0x00,0x00, +0x61,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x79,0x00,0x00,0x00,0x13,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x7b,0x00,0x00,0x00, +0x79,0x00,0x00,0x00,0x48,0x01,0x00,0x00,0x41,0x00,0x06,0x00, +0x7c,0x00,0x00,0x00,0x7d,0x00,0x00,0x00,0x75,0x00,0x00,0x00, +0x6a,0x00,0x00,0x00,0x7b,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x16,0x00,0x00,0x00,0x7e,0x00,0x00,0x00,0x7d,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x24,0x00,0x00,0x00,0x80,0x00,0x00,0x00, +0x19,0x00,0x00,0x00,0x7f,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x16,0x00,0x00,0x00,0x81,0x00,0x00,0x00,0x80,0x00,0x00,0x00, +0xac,0x00,0x05,0x00,0x27,0x00,0x00,0x00,0x85,0x00,0x00,0x00, 0x1e,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0xf7,0x00,0x03,0x00, -0xaa,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0xa7,0x00,0x00,0x00,0xa9,0x00,0x00,0x00,0xb3,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xa9,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xae,0x00,0x00,0x00,0x1f,0x00,0x00,0x00, -0x33,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xb0,0x00,0x00,0x00,0xae,0x00,0x00,0x00,0x02,0x01,0x00,0x00, -0x41,0x00,0x06,0x00,0x44,0x00,0x00,0x00,0xb1,0x00,0x00,0x00, -0x56,0x00,0x00,0x00,0x31,0x00,0x00,0x00,0xb0,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x16,0x00,0x00,0x00,0xb2,0x00,0x00,0x00, -0xb1,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xaa,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xb3,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xaa,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xaa,0x00,0x00,0x00, -0xf5,0x00,0x07,0x00,0x16,0x00,0x00,0x00,0x06,0x01,0x00,0x00, -0xb2,0x00,0x00,0x00,0xa9,0x00,0x00,0x00,0x60,0x00,0x00,0x00, -0xb3,0x00,0x00,0x00,0x0c,0x00,0x08,0x00,0x16,0x00,0x00,0x00, -0xb5,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00, -0xa1,0x00,0x00,0x00,0xa3,0x00,0x00,0x00,0x06,0x01,0x00,0x00, -0x83,0x00,0x05,0x00,0x16,0x00,0x00,0x00,0xb7,0x00,0x00,0x00, -0xb5,0x00,0x00,0x00,0x89,0x00,0x00,0x00,0x0c,0x00,0x06,0x00, -0x16,0x00,0x00,0x00,0xb8,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x1b,0x00,0x00,0x00,0xb7,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x16,0x00,0x00,0x00,0xbc,0x00,0x00,0x00,0x28,0x00,0x00,0x00, -0x81,0x00,0x05,0x00,0x16,0x00,0x00,0x00,0xbd,0x00,0x00,0x00, -0xbc,0x00,0x00,0x00,0xb8,0x00,0x00,0x00,0x3e,0x00,0x03,0x00, -0x28,0x00,0x00,0x00,0xbd,0x00,0x00,0x00,0x41,0x00,0x06,0x00, -0x44,0x00,0x00,0x00,0xc5,0x00,0x00,0x00,0xc2,0x00,0x00,0x00, -0x31,0x00,0x00,0x00,0x9d,0x00,0x00,0x00,0x3e,0x00,0x03,0x00, -0xc5,0x00,0x00,0x00,0xb8,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x91,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x91,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xc7,0x00,0x00,0x00, -0x02,0x01,0x00,0x00,0x20,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x8e,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x90,0x00,0x00,0x00, -0xe0,0x00,0x04,0x00,0x67,0x00,0x00,0x00,0x67,0x00,0x00,0x00, -0x68,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xc9,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xc9,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, -0x1a,0x00,0x00,0x00,0x03,0x01,0x00,0x00,0x6b,0x00,0x00,0x00, -0x90,0x00,0x00,0x00,0xe2,0x00,0x00,0x00,0xcc,0x00,0x00,0x00, -0xad,0x00,0x05,0x00,0x34,0x00,0x00,0x00,0xcf,0x00,0x00,0x00, -0x03,0x01,0x00,0x00,0x31,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0xcb,0x00,0x00,0x00,0xcc,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xcf,0x00,0x00,0x00,0xca,0x00,0x00,0x00, -0xcb,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xca,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xd2,0x00,0x00,0x00, -0x03,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0x34,0x00,0x00,0x00, -0xd3,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0xd2,0x00,0x00,0x00, -0xf7,0x00,0x03,0x00,0xd5,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xd3,0x00,0x00,0x00,0xd4,0x00,0x00,0x00, -0xd5,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xd4,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xda,0x00,0x00,0x00, -0x0f,0x00,0x00,0x00,0xd2,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x27,0x00,0x00,0x00,0xdb,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0xda,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x16,0x00,0x00,0x00, -0xdc,0x00,0x00,0x00,0xdb,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x16,0x00,0x00,0x00,0xde,0x00,0x00,0x00,0x28,0x00,0x00,0x00, -0x81,0x00,0x05,0x00,0x16,0x00,0x00,0x00,0xdf,0x00,0x00,0x00, -0xde,0x00,0x00,0x00,0xdc,0x00,0x00,0x00,0x3e,0x00,0x03,0x00, -0x28,0x00,0x00,0x00,0xdf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x88,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x85,0x00,0x00,0x00,0x87,0x00,0x00,0x00,0x95,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0x87,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x90,0x00,0x00,0x00,0x1f,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x92,0x00,0x00,0x00,0x90,0x00,0x00,0x00,0x48,0x01,0x00,0x00, +0x41,0x00,0x06,0x00,0x7c,0x00,0x00,0x00,0x93,0x00,0x00,0x00, +0x8c,0x00,0x00,0x00,0x6a,0x00,0x00,0x00,0x92,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x16,0x00,0x00,0x00,0x94,0x00,0x00,0x00, +0x93,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x88,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0x95,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x88,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x88,0x00,0x00,0x00, +0xf5,0x00,0x07,0x00,0x16,0x00,0x00,0x00,0x51,0x01,0x00,0x00, +0x94,0x00,0x00,0x00,0x87,0x00,0x00,0x00,0x22,0x00,0x00,0x00, +0x95,0x00,0x00,0x00,0x0c,0x00,0x08,0x00,0x16,0x00,0x00,0x00, +0x97,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00, +0x7e,0x00,0x00,0x00,0x81,0x00,0x00,0x00,0x51,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0x1c,0x00,0x00,0x00,0x99,0x00,0x00,0x00, +0x19,0x00,0x00,0x00,0x98,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x9a,0x00,0x00,0x00,0x99,0x00,0x00,0x00, +0xac,0x00,0x05,0x00,0x27,0x00,0x00,0x00,0x9b,0x00,0x00,0x00, +0x9a,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0xf7,0x00,0x03,0x00, +0x9e,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x9b,0x00,0x00,0x00,0x9d,0x00,0x00,0x00,0xa8,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0x9d,0x00,0x00,0x00,0x41,0x00,0x06,0x00, +0x7c,0x00,0x00,0x00,0xa5,0x00,0x00,0x00,0xa3,0x00,0x00,0x00, +0x6a,0x00,0x00,0x00,0x48,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, +0x16,0x00,0x00,0x00,0xa6,0x00,0x00,0x00,0xa5,0x00,0x00,0x00, +0x85,0x00,0x05,0x00,0x16,0x00,0x00,0x00,0xa7,0x00,0x00,0x00, +0x54,0x01,0x00,0x00,0xa6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x9e,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xa8,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x9e,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0x9e,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x16,0x00,0x00,0x00, +0x56,0x01,0x00,0x00,0xa7,0x00,0x00,0x00,0x9d,0x00,0x00,0x00, +0x22,0x00,0x00,0x00,0xa8,0x00,0x00,0x00,0x81,0x00,0x05,0x00, +0x16,0x00,0x00,0x00,0xaa,0x00,0x00,0x00,0x97,0x00,0x00,0x00, +0x56,0x01,0x00,0x00,0x0c,0x00,0x07,0x00,0x16,0x00,0x00,0x00, +0xab,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x28,0x00,0x00,0x00, +0x71,0x00,0x00,0x00,0xaa,0x00,0x00,0x00,0x3e,0x00,0x03,0x00, +0x61,0x00,0x00,0x00,0xab,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x67,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x67,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xae,0x00,0x00,0x00, +0x48,0x01,0x00,0x00,0x59,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x64,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x66,0x00,0x00,0x00, +0xe0,0x00,0x04,0x00,0x4d,0x00,0x00,0x00,0x4d,0x00,0x00,0x00, +0xaf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xb3,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xb3,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, +0x1a,0x00,0x00,0x00,0x49,0x01,0x00,0x00,0xb2,0x00,0x00,0x00, +0x66,0x00,0x00,0x00,0xcd,0x00,0x00,0x00,0xb6,0x00,0x00,0x00, +0xad,0x00,0x05,0x00,0x27,0x00,0x00,0x00,0xb9,0x00,0x00,0x00, +0x49,0x01,0x00,0x00,0x6a,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0xb5,0x00,0x00,0x00,0xb6,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0xb9,0x00,0x00,0x00,0xb4,0x00,0x00,0x00, +0xb5,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xb4,0x00,0x00,0x00, +0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xbc,0x00,0x00,0x00, +0x49,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0x27,0x00,0x00,0x00, +0xbd,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0xbc,0x00,0x00,0x00, +0xf7,0x00,0x03,0x00,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0xbd,0x00,0x00,0x00,0xbe,0x00,0x00,0x00, +0xbf,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xbe,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x16,0x00,0x00,0x00,0xc3,0x00,0x00,0x00, +0x61,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xc7,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0xbc,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x60,0x00,0x00,0x00,0xc8,0x00,0x00,0x00, +0x5c,0x00,0x00,0x00,0xc7,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x16,0x00,0x00,0x00,0xc9,0x00,0x00,0x00,0xc8,0x00,0x00,0x00, +0x0c,0x00,0x07,0x00,0x16,0x00,0x00,0x00,0xca,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0xc3,0x00,0x00,0x00, +0xc9,0x00,0x00,0x00,0x3e,0x00,0x03,0x00,0x61,0x00,0x00,0x00, +0xca,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xbf,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xbf,0x00,0x00,0x00,0xe0,0x00,0x04,0x00, +0x4d,0x00,0x00,0x00,0x4d,0x00,0x00,0x00,0xaf,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xb6,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xb6,0x00,0x00,0x00,0xc3,0x00,0x05,0x00,0x1a,0x00,0x00,0x00, +0xcd,0x00,0x00,0x00,0x49,0x01,0x00,0x00,0x1b,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xb3,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xb5,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x60,0x00,0x00,0x00, +0xcf,0x00,0x00,0x00,0x5c,0x00,0x00,0x00,0x6a,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x16,0x00,0x00,0x00,0xd0,0x00,0x00,0x00, +0xcf,0x00,0x00,0x00,0xe0,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, +0x4d,0x00,0x00,0x00,0xaf,0x00,0x00,0x00,0x3e,0x00,0x03,0x00, +0x61,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, 0xd5,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xd5,0x00,0x00,0x00, -0xe0,0x00,0x04,0x00,0x67,0x00,0x00,0x00,0x67,0x00,0x00,0x00, -0x68,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xcc,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xcc,0x00,0x00,0x00,0xc3,0x00,0x05,0x00, -0x1a,0x00,0x00,0x00,0xe2,0x00,0x00,0x00,0x03,0x01,0x00,0x00, -0x1b,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xc9,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xcb,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x16,0x00,0x00,0x00,0xe5,0x00,0x00,0x00,0x88,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xe8,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xe8,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0x04,0x01,0x00,0x00,0x0f,0x00,0x00,0x00,0xcb,0x00,0x00,0x00, -0xfd,0x00,0x00,0x00,0xe9,0x00,0x00,0x00,0xb0,0x00,0x05,0x00, -0x34,0x00,0x00,0x00,0xf0,0x00,0x00,0x00,0x04,0x01,0x00,0x00, -0x33,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xea,0x00,0x00,0x00, -0xe9,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0xf0,0x00,0x00,0x00,0xe9,0x00,0x00,0x00,0xea,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xe9,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xf4,0x00,0x00,0x00,0x13,0x00,0x00,0x00, -0x33,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xf6,0x00,0x00,0x00,0xf4,0x00,0x00,0x00,0x04,0x01,0x00,0x00, -0x41,0x00,0x06,0x00,0x44,0x00,0x00,0x00,0xf8,0x00,0x00,0x00, -0xc2,0x00,0x00,0x00,0x31,0x00,0x00,0x00,0xf6,0x00,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x4a,0x01,0x00,0x00, +0x0f,0x00,0x00,0x00,0xb5,0x00,0x00,0x00,0x0e,0x01,0x00,0x00, +0xd8,0x00,0x00,0x00,0xb0,0x00,0x05,0x00,0x27,0x00,0x00,0x00, +0xdd,0x00,0x00,0x00,0x4a,0x01,0x00,0x00,0x6c,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0xd7,0x00,0x00,0x00,0xd8,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xdd,0x00,0x00,0x00, +0xd6,0x00,0x00,0x00,0xd7,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xd6,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xe2,0x00,0x00,0x00,0x13,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xe4,0x00,0x00,0x00, +0xe2,0x00,0x00,0x00,0x4a,0x01,0x00,0x00,0x41,0x00,0x06,0x00, +0x7c,0x00,0x00,0x00,0xe7,0x00,0x00,0x00,0x75,0x00,0x00,0x00, +0x6a,0x00,0x00,0x00,0xe4,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x16,0x00,0x00,0x00,0xe8,0x00,0x00,0x00,0xe7,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x24,0x00,0x00,0x00,0xe9,0x00,0x00,0x00, +0x19,0x00,0x00,0x00,0x7f,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x16,0x00,0x00,0x00,0xea,0x00,0x00,0x00,0xe9,0x00,0x00,0x00, +0xac,0x00,0x05,0x00,0x27,0x00,0x00,0x00,0xee,0x00,0x00,0x00, +0x1e,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0xf7,0x00,0x03,0x00, +0xf1,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0xee,0x00,0x00,0x00,0xf0,0x00,0x00,0x00,0xfa,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xf0,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xf5,0x00,0x00,0x00,0x1f,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xf7,0x00,0x00,0x00,0xf5,0x00,0x00,0x00,0x4a,0x01,0x00,0x00, +0x41,0x00,0x06,0x00,0x7c,0x00,0x00,0x00,0xf8,0x00,0x00,0x00, +0x8c,0x00,0x00,0x00,0x6a,0x00,0x00,0x00,0xf7,0x00,0x00,0x00, 0x3d,0x00,0x04,0x00,0x16,0x00,0x00,0x00,0xf9,0x00,0x00,0x00, -0xf8,0x00,0x00,0x00,0x88,0x00,0x05,0x00,0x16,0x00,0x00,0x00, -0xfa,0x00,0x00,0x00,0xf9,0x00,0x00,0x00,0xe5,0x00,0x00,0x00, -0x3e,0x00,0x03,0x00,0xf8,0x00,0x00,0x00,0xfa,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xfd,0x00,0x00,0x00, -0x04,0x01,0x00,0x00,0x20,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xe8,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xea,0x00,0x00,0x00, +0xf8,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xf1,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xfa,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xf1,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xf1,0x00,0x00,0x00, +0xf5,0x00,0x07,0x00,0x16,0x00,0x00,0x00,0x4e,0x01,0x00,0x00, +0xf9,0x00,0x00,0x00,0xf0,0x00,0x00,0x00,0x22,0x00,0x00,0x00, +0xfa,0x00,0x00,0x00,0x0c,0x00,0x08,0x00,0x16,0x00,0x00,0x00, +0xfc,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00, +0xe8,0x00,0x00,0x00,0xea,0x00,0x00,0x00,0x4e,0x01,0x00,0x00, +0x83,0x00,0x05,0x00,0x16,0x00,0x00,0x00,0xfe,0x00,0x00,0x00, +0xfc,0x00,0x00,0x00,0xd0,0x00,0x00,0x00,0x0c,0x00,0x06,0x00, +0x16,0x00,0x00,0x00,0xff,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x1b,0x00,0x00,0x00,0xfe,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x16,0x00,0x00,0x00,0x03,0x01,0x00,0x00,0x61,0x00,0x00,0x00, +0x81,0x00,0x05,0x00,0x16,0x00,0x00,0x00,0x04,0x01,0x00,0x00, +0x03,0x01,0x00,0x00,0xff,0x00,0x00,0x00,0x3e,0x00,0x03,0x00, +0x61,0x00,0x00,0x00,0x04,0x01,0x00,0x00,0x41,0x00,0x06,0x00, +0x7c,0x00,0x00,0x00,0x0c,0x01,0x00,0x00,0x09,0x01,0x00,0x00, +0x6a,0x00,0x00,0x00,0xe4,0x00,0x00,0x00,0x3e,0x00,0x03,0x00, +0x0c,0x01,0x00,0x00,0xff,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xd8,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xd8,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x0e,0x01,0x00,0x00, +0x4a,0x01,0x00,0x00,0x59,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xd5,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xd7,0x00,0x00,0x00, +0xe0,0x00,0x04,0x00,0x4d,0x00,0x00,0x00,0x4d,0x00,0x00,0x00, +0xaf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x10,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x10,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, +0x1a,0x00,0x00,0x00,0x4b,0x01,0x00,0x00,0xb2,0x00,0x00,0x00, +0xd7,0x00,0x00,0x00,0x29,0x01,0x00,0x00,0x13,0x01,0x00,0x00, +0xad,0x00,0x05,0x00,0x27,0x00,0x00,0x00,0x16,0x01,0x00,0x00, +0x4b,0x01,0x00,0x00,0x6a,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0x12,0x01,0x00,0x00,0x13,0x01,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x16,0x01,0x00,0x00,0x11,0x01,0x00,0x00, +0x12,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x11,0x01,0x00,0x00, +0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x19,0x01,0x00,0x00, +0x4b,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0x27,0x00,0x00,0x00, +0x1a,0x01,0x00,0x00,0x0f,0x00,0x00,0x00,0x19,0x01,0x00,0x00, +0xf7,0x00,0x03,0x00,0x1c,0x01,0x00,0x00,0x00,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x1a,0x01,0x00,0x00,0x1b,0x01,0x00,0x00, +0x1c,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x1b,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x21,0x01,0x00,0x00, +0x0f,0x00,0x00,0x00,0x19,0x01,0x00,0x00,0x41,0x00,0x05,0x00, +0x60,0x00,0x00,0x00,0x22,0x01,0x00,0x00,0x5c,0x00,0x00,0x00, +0x21,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0x16,0x00,0x00,0x00, +0x23,0x01,0x00,0x00,0x22,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, +0x16,0x00,0x00,0x00,0x25,0x01,0x00,0x00,0x61,0x00,0x00,0x00, +0x81,0x00,0x05,0x00,0x16,0x00,0x00,0x00,0x26,0x01,0x00,0x00, +0x25,0x01,0x00,0x00,0x23,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, +0x61,0x00,0x00,0x00,0x26,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0x1c,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x1c,0x01,0x00,0x00, +0xe0,0x00,0x04,0x00,0x4d,0x00,0x00,0x00,0x4d,0x00,0x00,0x00, +0xaf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x13,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x13,0x01,0x00,0x00,0xc3,0x00,0x05,0x00, +0x1a,0x00,0x00,0x00,0x29,0x01,0x00,0x00,0x4b,0x01,0x00,0x00, +0x1b,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x10,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x12,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, +0x16,0x00,0x00,0x00,0x2c,0x01,0x00,0x00,0xcf,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x2f,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x2f,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0x4c,0x01,0x00,0x00,0x0f,0x00,0x00,0x00,0x12,0x01,0x00,0x00, +0x44,0x01,0x00,0x00,0x30,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, +0x27,0x00,0x00,0x00,0x37,0x01,0x00,0x00,0x4c,0x01,0x00,0x00, +0x6c,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x31,0x01,0x00,0x00, +0x30,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x37,0x01,0x00,0x00,0x30,0x01,0x00,0x00,0x31,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x30,0x01,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x3b,0x01,0x00,0x00,0x13,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x3d,0x01,0x00,0x00,0x3b,0x01,0x00,0x00,0x4c,0x01,0x00,0x00, +0x41,0x00,0x06,0x00,0x7c,0x00,0x00,0x00,0x3f,0x01,0x00,0x00, +0x09,0x01,0x00,0x00,0x6a,0x00,0x00,0x00,0x3d,0x01,0x00,0x00, +0x3d,0x00,0x04,0x00,0x16,0x00,0x00,0x00,0x40,0x01,0x00,0x00, +0x3f,0x01,0x00,0x00,0x88,0x00,0x05,0x00,0x16,0x00,0x00,0x00, +0x41,0x01,0x00,0x00,0x40,0x01,0x00,0x00,0x2c,0x01,0x00,0x00, +0x3e,0x00,0x03,0x00,0x3f,0x01,0x00,0x00,0x41,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x44,0x01,0x00,0x00, +0x4c,0x01,0x00,0x00,0x59,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x2f,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x31,0x01,0x00,0x00, 0xfd,0x00,0x01,0x00,0x38,0x00,0x01,0x00, }; -const uint64_t soft_max_f32_len = 3752; +const uint64_t soft_max_f32_len = 4916; unsigned char split_k_reduce_data[] = { 0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, @@ -51610,105 +47313,277 @@ const uint64_t split_k_reduce_len = 1416; unsigned char sqr_f32_data[] = { 0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, -0x37,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, +0x9c,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, 0x01,0x00,0x00,0x00,0x0b,0x00,0x06,0x00,0x01,0x00,0x00,0x00, 0x47,0x4c,0x53,0x4c,0x2e,0x73,0x74,0x64,0x2e,0x34,0x35,0x30, 0x00,0x00,0x00,0x00,0x0e,0x00,0x03,0x00,0x00,0x00,0x00,0x00, 0x01,0x00,0x00,0x00,0x0f,0x00,0x09,0x00,0x05,0x00,0x00,0x00, 0x04,0x00,0x00,0x00,0x6d,0x61,0x69,0x6e,0x00,0x00,0x00,0x00, -0x0b,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x24,0x00,0x00,0x00, -0x2c,0x00,0x00,0x00,0x10,0x00,0x06,0x00,0x04,0x00,0x00,0x00, +0x14,0x00,0x00,0x00,0xc7,0x00,0x00,0x00,0xd9,0x00,0x00,0x00, +0xe4,0x00,0x00,0x00,0x10,0x00,0x06,0x00,0x04,0x00,0x00,0x00, 0x11,0x00,0x00,0x00,0x00,0x02,0x00,0x00,0x01,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x0b,0x00,0x00,0x00, -0x0b,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x12,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x12,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x12,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x12,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x12,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x21,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00, -0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, -0x22,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x24,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x24,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x29,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00, -0x2a,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x19,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x2a,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, -0x2a,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x2c,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x2c,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x34,0x00,0x00,0x00, -0x0b,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x13,0x00,0x02,0x00, -0x02,0x00,0x00,0x00,0x21,0x00,0x03,0x00,0x03,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x15,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x17,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x03,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x0a,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00, -0x0b,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x0d,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x16,0x00,0x03,0x00,0x11,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x1e,0x00,0x06,0x00,0x12,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x11,0x00,0x00,0x00, -0x11,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x13,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x15,0x00,0x04,0x00,0x15,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x15,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x17,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x14,0x00,0x02,0x00,0x1a,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, -0x21,0x00,0x00,0x00,0x11,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, -0x22,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x23,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x22,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x23,0x00,0x00,0x00,0x24,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x26,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x11,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, -0x29,0x00,0x00,0x00,0x11,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, -0x2a,0x00,0x00,0x00,0x29,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x2b,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x2a,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x2b,0x00,0x00,0x00,0x2c,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x32,0x00,0x00,0x00,0x00,0x02,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x33,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x2c,0x00,0x06,0x00,0x09,0x00,0x00,0x00,0x34,0x00,0x00,0x00, -0x32,0x00,0x00,0x00,0x33,0x00,0x00,0x00,0x33,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x12,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x12,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x12,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x08,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x12,0x00,0x00,0x00, +0x03,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x12,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x12,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x14,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x12,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x18,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x12,0x00,0x00,0x00,0x07,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x12,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x12,0x00,0x00,0x00, +0x09,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x24,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x12,0x00,0x00,0x00,0x0a,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x12,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x2c,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x12,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x30,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x12,0x00,0x00,0x00,0x0d,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x12,0x00,0x00,0x00,0x0e,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x38,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x12,0x00,0x00,0x00, +0x0f,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x3c,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x12,0x00,0x00,0x00,0x10,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x12,0x00,0x00,0x00,0x11,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x44,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x12,0x00,0x00,0x00, +0x12,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x48,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x12,0x00,0x00,0x00,0x13,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x4c,0x00,0x00,0x00,0x47,0x00,0x03,0x00, +0x12,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0xc7,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x1c,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0xd6,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0xd7,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0xd7,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0xd7,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xd9,0x00,0x00,0x00, +0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0xd9,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0xe1,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0xe2,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0xe2,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0xe2,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xe4,0x00,0x00,0x00, +0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0xe4,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0xf3,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, +0x19,0x00,0x00,0x00,0x13,0x00,0x02,0x00,0x02,0x00,0x00,0x00, +0x21,0x00,0x03,0x00,0x03,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x15,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x16,0x00,0x03,0x00,0x11,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0x1e,0x00,0x16,0x00,0x12,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x11,0x00,0x00,0x00,0x11,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x14,0x00,0x00,0x00, +0x09,0x00,0x00,0x00,0x15,0x00,0x04,0x00,0x15,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x15,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x03,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x17,0x00,0x00,0x00,0x09,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x15,0x00,0x00,0x00, +0x1a,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x15,0x00,0x00,0x00,0x1e,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x15,0x00,0x00,0x00,0x55,0x00,0x00,0x00, +0x08,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x15,0x00,0x00,0x00, +0x5a,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x15,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x15,0x00,0x00,0x00,0x66,0x00,0x00,0x00, +0x05,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x15,0x00,0x00,0x00, +0x6f,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x15,0x00,0x00,0x00,0x72,0x00,0x00,0x00,0x0a,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x15,0x00,0x00,0x00,0x76,0x00,0x00,0x00, +0x09,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x15,0x00,0x00,0x00, +0xad,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x15,0x00,0x00,0x00,0xb2,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x15,0x00,0x00,0x00,0xb8,0x00,0x00,0x00, +0x0e,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x15,0x00,0x00,0x00, +0xbe,0x00,0x00,0x00,0x0d,0x00,0x00,0x00,0x17,0x00,0x04,0x00, +0xc5,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x03,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0xc6,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0xc5,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0xc6,0x00,0x00,0x00, +0xc7,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xc8,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0xc9,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x15,0x00,0x00,0x00, +0xcc,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x14,0x00,0x02,0x00, +0xcf,0x00,0x00,0x00,0x1d,0x00,0x03,0x00,0xd6,0x00,0x00,0x00, +0x11,0x00,0x00,0x00,0x1e,0x00,0x03,0x00,0xd7,0x00,0x00,0x00, +0xd6,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0xd8,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0xd7,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0xd8,0x00,0x00,0x00,0xd9,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0xde,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x11,0x00,0x00,0x00,0x1d,0x00,0x03,0x00,0xe1,0x00,0x00,0x00, +0x11,0x00,0x00,0x00,0x1e,0x00,0x03,0x00,0xe2,0x00,0x00,0x00, +0xe1,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0xe3,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0xe2,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0xe3,0x00,0x00,0x00,0xe4,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x15,0x00,0x00,0x00,0xe5,0x00,0x00,0x00, +0x11,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xf1,0x00,0x00,0x00,0x00,0x02,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xf2,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x2c,0x00,0x06,0x00,0xc5,0x00,0x00,0x00,0xf3,0x00,0x00,0x00, +0xf1,0x00,0x00,0x00,0xf2,0x00,0x00,0x00,0xf2,0x00,0x00,0x00, 0x36,0x00,0x05,0x00,0x02,0x00,0x00,0x00,0x04,0x00,0x00,0x00, 0x00,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x05,0x00,0x00,0x00,0xf7,0x00,0x03,0x00,0x35,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0xfb,0x00,0x03,0x00,0x0c,0x00,0x00,0x00, -0x36,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x36,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00,0x0e,0x00,0x00,0x00, -0x0b,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x0e,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00,0x18,0x00,0x00,0x00, +0x05,0x00,0x00,0x00,0xf7,0x00,0x03,0x00,0xf4,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0xfb,0x00,0x03,0x00,0xc8,0x00,0x00,0x00, +0xf5,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xf5,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0xc9,0x00,0x00,0x00,0xca,0x00,0x00,0x00, +0xc7,0x00,0x00,0x00,0xc8,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xcb,0x00,0x00,0x00,0xca,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00,0xcd,0x00,0x00,0x00, +0x14,0x00,0x00,0x00,0xcc,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xce,0x00,0x00,0x00,0xcd,0x00,0x00,0x00, +0xae,0x00,0x05,0x00,0xcf,0x00,0x00,0x00,0xd0,0x00,0x00,0x00, +0xcb,0x00,0x00,0x00,0xce,0x00,0x00,0x00,0xf7,0x00,0x03,0x00, +0xd2,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0xd0,0x00,0x00,0x00,0xd1,0x00,0x00,0x00,0xd2,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xd1,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xf4,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xd2,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00,0x03,0x01,0x00,0x00, 0x14,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x18,0x00,0x00,0x00, -0xae,0x00,0x05,0x00,0x1a,0x00,0x00,0x00,0x1b,0x00,0x00,0x00, -0x0f,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0xf7,0x00,0x03,0x00, -0x1d,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x1b,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x1d,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0x1c,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x35,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x1d,0x00,0x00,0x00, -0x41,0x00,0x06,0x00,0x26,0x00,0x00,0x00,0x27,0x00,0x00,0x00, -0x24,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x11,0x00,0x00,0x00,0x28,0x00,0x00,0x00, -0x27,0x00,0x00,0x00,0x85,0x00,0x05,0x00,0x11,0x00,0x00,0x00, -0x30,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0x28,0x00,0x00,0x00, -0x41,0x00,0x06,0x00,0x26,0x00,0x00,0x00,0x31,0x00,0x00,0x00, -0x2c,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, -0x3e,0x00,0x03,0x00,0x31,0x00,0x00,0x00,0x30,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x35,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x35,0x00,0x00,0x00,0xfd,0x00,0x01,0x00,0x38,0x00,0x01,0x00, +0x06,0x00,0x00,0x00,0x04,0x01,0x00,0x00,0x03,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00,0x05,0x01,0x00,0x00, +0x14,0x00,0x00,0x00,0x1a,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x06,0x01,0x00,0x00,0x05,0x01,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x07,0x01,0x00,0x00, +0x04,0x01,0x00,0x00,0x06,0x01,0x00,0x00,0x41,0x00,0x05,0x00, +0x17,0x00,0x00,0x00,0x08,0x01,0x00,0x00,0x14,0x00,0x00,0x00, +0x1e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x09,0x01,0x00,0x00,0x08,0x01,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x0a,0x01,0x00,0x00,0x07,0x01,0x00,0x00, +0x09,0x01,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x0b,0x01,0x00,0x00,0xcb,0x00,0x00,0x00,0x0a,0x01,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x0f,0x01,0x00,0x00, +0x0b,0x01,0x00,0x00,0x04,0x01,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x12,0x01,0x00,0x00,0x0f,0x01,0x00,0x00, +0x06,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x15,0x01,0x00,0x00,0x12,0x01,0x00,0x00,0x09,0x01,0x00,0x00, +0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x18,0x01,0x00,0x00, +0xcb,0x00,0x00,0x00,0x15,0x01,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x1d,0x01,0x00,0x00,0x06,0x01,0x00,0x00, +0x09,0x01,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x1e,0x01,0x00,0x00,0x18,0x01,0x00,0x00,0x1d,0x01,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x22,0x01,0x00,0x00, +0x1e,0x01,0x00,0x00,0x06,0x01,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x25,0x01,0x00,0x00,0x22,0x01,0x00,0x00, +0x09,0x01,0x00,0x00,0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x2a,0x01,0x00,0x00,0x18,0x01,0x00,0x00,0x25,0x01,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x2d,0x01,0x00,0x00, +0x2a,0x01,0x00,0x00,0x09,0x01,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x36,0x01,0x00,0x00,0x2d,0x01,0x00,0x00, +0x09,0x01,0x00,0x00,0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x37,0x01,0x00,0x00,0x2a,0x01,0x00,0x00,0x36,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00,0x39,0x01,0x00,0x00, +0x14,0x00,0x00,0x00,0x55,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x3a,0x01,0x00,0x00,0x39,0x01,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3b,0x01,0x00,0x00, +0x0b,0x01,0x00,0x00,0x3a,0x01,0x00,0x00,0x41,0x00,0x05,0x00, +0x17,0x00,0x00,0x00,0x3d,0x01,0x00,0x00,0x14,0x00,0x00,0x00, +0x5a,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x3e,0x01,0x00,0x00,0x3d,0x01,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x3f,0x01,0x00,0x00,0x1e,0x01,0x00,0x00, +0x3e,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x40,0x01,0x00,0x00,0x3b,0x01,0x00,0x00,0x3f,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00,0x42,0x01,0x00,0x00, +0x14,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x43,0x01,0x00,0x00,0x42,0x01,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x44,0x01,0x00,0x00, +0x2d,0x01,0x00,0x00,0x43,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x45,0x01,0x00,0x00,0x40,0x01,0x00,0x00, +0x44,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00, +0x47,0x01,0x00,0x00,0x14,0x00,0x00,0x00,0x66,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x48,0x01,0x00,0x00, +0x47,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x49,0x01,0x00,0x00,0x37,0x01,0x00,0x00,0x48,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x4a,0x01,0x00,0x00, +0x45,0x01,0x00,0x00,0x49,0x01,0x00,0x00,0x41,0x00,0x06,0x00, +0xde,0x00,0x00,0x00,0xdf,0x00,0x00,0x00,0xd9,0x00,0x00,0x00, +0xcc,0x00,0x00,0x00,0x4a,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, +0x11,0x00,0x00,0x00,0xe0,0x00,0x00,0x00,0xdf,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00,0xe6,0x00,0x00,0x00, +0x14,0x00,0x00,0x00,0xe5,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xe7,0x00,0x00,0x00,0xe6,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00,0x54,0x01,0x00,0x00, +0x14,0x00,0x00,0x00,0x6f,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x55,0x01,0x00,0x00,0x54,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00,0x56,0x01,0x00,0x00, +0x14,0x00,0x00,0x00,0x72,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x57,0x01,0x00,0x00,0x56,0x01,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x58,0x01,0x00,0x00, +0x55,0x01,0x00,0x00,0x57,0x01,0x00,0x00,0x41,0x00,0x05,0x00, +0x17,0x00,0x00,0x00,0x59,0x01,0x00,0x00,0x14,0x00,0x00,0x00, +0x76,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x5a,0x01,0x00,0x00,0x59,0x01,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x5b,0x01,0x00,0x00,0x58,0x01,0x00,0x00, +0x5a,0x01,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x5c,0x01,0x00,0x00,0xcb,0x00,0x00,0x00,0x5b,0x01,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x60,0x01,0x00,0x00, +0x5c,0x01,0x00,0x00,0x55,0x01,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x63,0x01,0x00,0x00,0x60,0x01,0x00,0x00, +0x57,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x66,0x01,0x00,0x00,0x63,0x01,0x00,0x00,0x5a,0x01,0x00,0x00, +0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x69,0x01,0x00,0x00, +0xcb,0x00,0x00,0x00,0x66,0x01,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x6e,0x01,0x00,0x00,0x57,0x01,0x00,0x00, +0x5a,0x01,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x6f,0x01,0x00,0x00,0x69,0x01,0x00,0x00,0x6e,0x01,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x73,0x01,0x00,0x00, +0x6f,0x01,0x00,0x00,0x57,0x01,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x76,0x01,0x00,0x00,0x73,0x01,0x00,0x00, +0x5a,0x01,0x00,0x00,0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x7b,0x01,0x00,0x00,0x69,0x01,0x00,0x00,0x76,0x01,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x7e,0x01,0x00,0x00, +0x7b,0x01,0x00,0x00,0x5a,0x01,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x87,0x01,0x00,0x00,0x7e,0x01,0x00,0x00, +0x5a,0x01,0x00,0x00,0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x88,0x01,0x00,0x00,0x7b,0x01,0x00,0x00,0x87,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00,0x8a,0x01,0x00,0x00, +0x14,0x00,0x00,0x00,0xad,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x8b,0x01,0x00,0x00,0x8a,0x01,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8c,0x01,0x00,0x00, +0x5c,0x01,0x00,0x00,0x8b,0x01,0x00,0x00,0x41,0x00,0x05,0x00, +0x17,0x00,0x00,0x00,0x8e,0x01,0x00,0x00,0x14,0x00,0x00,0x00, +0xb2,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x8f,0x01,0x00,0x00,0x8e,0x01,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x90,0x01,0x00,0x00,0x6f,0x01,0x00,0x00, +0x8f,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x91,0x01,0x00,0x00,0x8c,0x01,0x00,0x00,0x90,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00,0x93,0x01,0x00,0x00, +0x14,0x00,0x00,0x00,0xb8,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x94,0x01,0x00,0x00,0x93,0x01,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x95,0x01,0x00,0x00, +0x7e,0x01,0x00,0x00,0x94,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x96,0x01,0x00,0x00,0x91,0x01,0x00,0x00, +0x95,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00, +0x98,0x01,0x00,0x00,0x14,0x00,0x00,0x00,0xbe,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x99,0x01,0x00,0x00, +0x98,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x9a,0x01,0x00,0x00,0x88,0x01,0x00,0x00,0x99,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9b,0x01,0x00,0x00, +0x96,0x01,0x00,0x00,0x9a,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xec,0x00,0x00,0x00,0xe7,0x00,0x00,0x00, +0x9b,0x01,0x00,0x00,0x85,0x00,0x05,0x00,0x11,0x00,0x00,0x00, +0xef,0x00,0x00,0x00,0xe0,0x00,0x00,0x00,0xe0,0x00,0x00,0x00, +0x41,0x00,0x06,0x00,0xde,0x00,0x00,0x00,0xf0,0x00,0x00,0x00, +0xe4,0x00,0x00,0x00,0xcc,0x00,0x00,0x00,0xec,0x00,0x00,0x00, +0x3e,0x00,0x03,0x00,0xf0,0x00,0x00,0x00,0xef,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xf4,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xf4,0x00,0x00,0x00,0xfd,0x00,0x01,0x00,0x38,0x00,0x01,0x00, }; -const uint64_t sqr_f32_len = 1188; +const uint64_t sqr_f32_len = 3252; diff --git a/ggml-vulkan.cpp b/ggml-vulkan.cpp index bc316c3f3..5a1b3f477 100644 --- a/ggml-vulkan.cpp +++ b/ggml-vulkan.cpp @@ -69,6 +69,33 @@ struct vk_queue { vk::PipelineStageFlags stage_flags; }; +struct vk_pipeline_struct { + std::string name; + vk::ShaderModule shader_module; + vk::DescriptorSetLayout dsl; + std::vector descriptor_pools; + std::vector descriptor_sets; + uint32_t descriptor_set_idx; + vk::PipelineLayout layout; + vk::Pipeline pipeline; + uint32_t push_constant_size; + uint32_t parameter_count; + std::array wg_denoms; + uint32_t align; +}; + +typedef std::shared_ptr vk_pipeline; +typedef std::weak_ptr vk_pipeline_ref; + +static void ggml_vk_destroy_pipeline(vk::Device& device, vk_pipeline& pipeline); + +struct vk_matmul_pipeline_struct { + vk_pipeline l, m, s; + vk_pipeline a_l, a_m, a_s; +}; + +typedef std::shared_ptr vk_matmul_pipeline; + struct vk_device { vk::PhysicalDevice physical_device; vk::PhysicalDeviceProperties properties; @@ -84,10 +111,61 @@ struct vk_device { uint32_t subgroup_size; bool uma; + bool initialized; + size_t idx; + + vk_matmul_pipeline pipeline_matmul_f32; + vk_matmul_pipeline pipeline_matmul_f16; + vk_matmul_pipeline pipeline_matmul_f16_f32; + vk_pipeline pipeline_matmul_split_k_reduce; + + vk_matmul_pipeline pipeline_dequant_mul_mat_mat[VK_NUM_TYPES]; + + vk_pipeline pipeline_dequant[VK_NUM_TYPES]; + vk_pipeline pipeline_dequant_mul_mat_vec_f32[VK_NUM_TYPES]; + + vk_pipeline pipeline_mul_mat_vec_p021_f16_f32; + vk_pipeline pipeline_mul_mat_vec_nc_f16_f32; + vk_pipeline pipeline_get_rows[VK_NUM_TYPES]; + vk_pipeline pipeline_get_rows_f32[VK_NUM_TYPES]; + vk_pipeline pipeline_mul_f32; + vk_pipeline pipeline_add_f32; + vk_pipeline pipeline_scale_f32; + vk_pipeline pipeline_sqr_f32; + vk_pipeline pipeline_clamp_f32; + vk_pipeline pipeline_cpy_f32_f32, pipeline_cpy_f32_f16, pipeline_cpy_f16_f16; + vk_pipeline pipeline_norm_f32; + vk_pipeline pipeline_rms_norm_f32; + vk_pipeline pipeline_gelu_f32; + vk_pipeline pipeline_silu_f32; + vk_pipeline pipeline_relu_f32; + vk_pipeline pipeline_diag_mask_inf_f32; + vk_pipeline pipeline_soft_max_f32; + vk_pipeline pipeline_rope_f32, pipeline_rope_f16; + vk_pipeline pipeline_rope_neox_f32, pipeline_rope_neox_f16; + vk_pipeline pipeline_argsort_f32; + + std::vector pipelines; + ~vk_device() { #ifdef GGML_VULKAN_DEBUG std::cerr << "destroy device " << name << std::endl; #endif + device.destroyCommandPool(compute_queue.pool); + if (!single_queue) { + device.destroyCommandPool(transfer_queue.pool); + } + + for (auto& pipeline : pipelines) { + if (pipeline.expired()) { + continue; + } + + vk_pipeline pl = pipeline.lock(); + ggml_vk_destroy_pipeline(device, pl); + } + pipelines.clear(); + device.destroy(); } }; @@ -125,21 +203,6 @@ struct vk_subbuffer { uint64_t size; }; -struct vk_pipeline { - std::string name; - vk::ShaderModule shader_module; - vk::DescriptorSetLayout dsl; - std::vector descriptor_pools; - std::vector descriptor_sets; - uint32_t descriptor_set_idx; - vk::PipelineLayout layout; - vk::Pipeline pipeline; - uint32_t push_constant_size; - uint32_t parameter_count; - std::array wg_denoms; - uint32_t align; -}; - struct vk_semaphore { vk::Semaphore s; uint64_t value; @@ -160,11 +223,21 @@ struct vk_op_push_constants { float param2; }; -struct vk_op_cpy_push_constants { +struct vk_op_unary_push_constants { uint32_t ne; - uint32_t ne00; uint32_t ne01; uint32_t nb00; uint32_t nb01; uint32_t nb02; - uint32_t ne10; uint32_t ne11; uint32_t nb10; uint32_t nb11; uint32_t nb12; + uint32_t ne00; uint32_t ne01; uint32_t ne02; uint32_t ne03; uint32_t nb00; uint32_t nb01; uint32_t nb02; uint32_t nb03; + uint32_t ne10; uint32_t ne11; uint32_t ne12; uint32_t ne13; uint32_t nb10; uint32_t nb11; uint32_t nb12; uint32_t nb13; uint32_t d_offset; + float param1; float param2; +}; + +struct vk_op_binary_push_constants { + uint32_t ne; + uint32_t ne00; uint32_t ne01; uint32_t ne02; uint32_t ne03; uint32_t nb00; uint32_t nb01; uint32_t nb02; uint32_t nb03; + uint32_t ne10; uint32_t ne11; uint32_t ne12; uint32_t ne13; uint32_t nb10; uint32_t nb11; uint32_t nb12; uint32_t nb13; + uint32_t ne20; uint32_t ne21; uint32_t ne22; uint32_t ne23; uint32_t nb20; uint32_t nb21; uint32_t nb22; uint32_t nb23; + uint32_t d_offset; + float param1; float param2; }; struct vk_op_diag_mask_push_constants { @@ -196,6 +269,22 @@ struct vk_op_rope_neox_push_constants { float inv_ndims; }; +struct vk_op_soft_max_push_constants { + uint32_t KX; + uint32_t KY; + uint32_t KZ; + float scale; + float max_bias; + float m0; + float m1; + uint32_t n_head_log2; +}; + +struct vk_op_argsort_push_constants { + uint32_t ncols; + bool ascending; +}; + // Allow pre-recording command buffers struct vk_staging_memcpy { vk_staging_memcpy(void * _dst, const void * _src, size_t _n) : dst(_dst), src(_src), n(_n) {} @@ -236,7 +325,6 @@ struct ggml_tensor_extra_gpu { }; struct ggml_vk_garbage_collector { - std::vector pipelines; std::vector tl_semaphores; std::vector semaphores; std::vector events; @@ -247,35 +335,7 @@ struct ggml_vk_garbage_collector { struct ggml_backend_vk_context { std::string name; - std::weak_ptr device; - vk_pipeline pipeline_matmul_f32_l, pipeline_matmul_f32_m, pipeline_matmul_f32_s; - vk_pipeline pipeline_matmul_f32_aligned_l, pipeline_matmul_f32_aligned_m, pipeline_matmul_f32_aligned_s; - vk_pipeline pipeline_matmul_f16_l, pipeline_matmul_f16_m, pipeline_matmul_f16_s; - vk_pipeline pipeline_matmul_f16_aligned_l, pipeline_matmul_f16_aligned_m, pipeline_matmul_f16_aligned_s; - vk_pipeline pipeline_matmul_f16_f32_l, pipeline_matmul_f16_f32_m, pipeline_matmul_f16_f32_s; - vk_pipeline pipeline_matmul_f16_f32_aligned_l, pipeline_matmul_f16_f32_aligned_m, pipeline_matmul_f16_f32_aligned_s; - vk_pipeline pipeline_matmul_split_k_reduce; - vk_pipeline pipeline_dequant[VK_NUM_TYPES]; - vk_pipeline pipeline_dequant_mul_mat_vec_f32[VK_NUM_TYPES]; - vk_pipeline pipeline_mul_mat_vec_p021_f16_f32; - vk_pipeline pipeline_mul_mat_vec_nc_f16_f32; - vk_pipeline pipeline_get_rows[VK_NUM_TYPES]; - vk_pipeline pipeline_get_rows_f32[VK_NUM_TYPES]; - vk_pipeline pipeline_mul_f32; - vk_pipeline pipeline_add_f32; - vk_pipeline pipeline_scale_f32; - vk_pipeline pipeline_sqr_f32; - vk_pipeline pipeline_clamp_f32; - vk_pipeline pipeline_cpy_f32_f32, pipeline_cpy_f32_f16, pipeline_cpy_f16_f16; - vk_pipeline pipeline_norm_f32; - vk_pipeline pipeline_rms_norm_f32; - vk_pipeline pipeline_gelu_f32; - vk_pipeline pipeline_silu_f32; - vk_pipeline pipeline_relu_f32; - vk_pipeline pipeline_diag_mask_inf_f32; - vk_pipeline pipeline_soft_max_f32; - vk_pipeline pipeline_rope_f32, pipeline_rope_f16; - vk_pipeline pipeline_rope_neox_f32, pipeline_rope_neox_f16; + std::shared_ptr device; size_t semaphore_idx, event_idx; ggml_vk_garbage_collector gc; @@ -304,13 +364,31 @@ struct vk_instance { std::vector device_indices; - std::shared_ptr devices[GGML_VK_MAX_DEVICES]; ggml_backend_t backends[GGML_VK_MAX_DEVICES]; ggml_backend_vk_context contexts[GGML_VK_MAX_DEVICES]; ggml_backend_buffer_type buffer_types[GGML_VK_MAX_DEVICES]; bool initialized[GGML_VK_MAX_DEVICES]; }; +static std::shared_ptr ggml_vk_get_device(size_t idx) { +#ifdef GGML_VULKAN_DEBUG + std::cerr << "ggml_vk_get_device(" << idx << ")" << std::endl; +#endif + static std::weak_ptr devices[GGML_VK_MAX_DEVICES]; + + if (devices[idx].expired()) { +#ifdef GGML_VULKAN_DEBUG + std::cerr << "Initializing new vk_device" << std::endl; +#endif + std::shared_ptr device = std::make_shared(); + device->initialized = false; + devices[idx] = device; + return device; + } + + return devices[idx].lock(); +} + #ifdef GGML_VULKAN_CHECK_RESULTS static size_t vk_skip_checks; static size_t vk_output_tensor; @@ -334,14 +412,15 @@ static void ggml_vk_create_pipeline(ggml_backend_vk_context * ctx, vk_pipeline& GGML_ASSERT(parameter_count > 0); GGML_ASSERT(wg_denoms[0] > 0 && wg_denoms[1] > 0 && wg_denoms[2] > 0); // NOLINT - pipeline.name = name; - pipeline.parameter_count = parameter_count; - pipeline.push_constant_size = push_constant_size; - pipeline.wg_denoms = wg_denoms; - pipeline.align = align; + pipeline = std::make_shared(); + pipeline->name = name; + pipeline->parameter_count = parameter_count; + pipeline->push_constant_size = push_constant_size; + pipeline->wg_denoms = wg_denoms; + pipeline->align = align; vk::ShaderModuleCreateInfo shader_module_create_info({}, spv_size, reinterpret_cast(spv_data)); - pipeline.shader_module = ctx->device.lock()->device.createShaderModule(shader_module_create_info); + pipeline->shader_module = ctx->device->device.createShaderModule(shader_module_create_info); std::vector dsl_binding; std::vector dsl_binding_flags; @@ -355,49 +434,49 @@ static void ggml_vk_create_pipeline(ggml_backend_vk_context * ctx, vk_pipeline& vk::PushConstantRange pcr( vk::ShaderStageFlagBits::eCompute, 0, - pipeline.push_constant_size + pipeline->push_constant_size ); vk::DescriptorSetLayoutCreateInfo descriptor_set_layout_create_info( {}, dsl_binding); descriptor_set_layout_create_info.setPNext(&dslbfci); - pipeline.dsl = ctx->device.lock()->device.createDescriptorSetLayout(descriptor_set_layout_create_info); + pipeline->dsl = ctx->device->device.createDescriptorSetLayout(descriptor_set_layout_create_info); // Check if device supports multiple descriptors per pool - if (ctx->device.lock()->descriptor_set_mode == VK_DEVICE_DESCRIPTOR_POOL_MODE_UNKNOWN) { + if (ctx->device->descriptor_set_mode == VK_DEVICE_DESCRIPTOR_POOL_MODE_UNKNOWN) { const uint32_t alloc_count = 2; // Try allocating multiple sets from one pool // This fails on AMD for some reason, so add a fall back to allocating one pool per set - vk::DescriptorPoolSize descriptor_pool_size(vk::DescriptorType::eStorageBuffer, pipeline.parameter_count); + vk::DescriptorPoolSize descriptor_pool_size(vk::DescriptorType::eStorageBuffer, pipeline->parameter_count); vk::DescriptorPoolCreateInfo descriptor_pool_create_info({}, alloc_count, descriptor_pool_size); - vk::DescriptorPool pool = ctx->device.lock()->device.createDescriptorPool(descriptor_pool_create_info); + vk::DescriptorPool pool = ctx->device->device.createDescriptorPool(descriptor_pool_create_info); std::vector layouts(alloc_count); for (uint32_t i = 0; i < alloc_count; i++) { - layouts[i] = pipeline.dsl; + layouts[i] = pipeline->dsl; } try { vk::DescriptorSetAllocateInfo descriptor_set_alloc_info(pool, alloc_count, layouts.data()); - std::vector sets = ctx->device.lock()->device.allocateDescriptorSets(descriptor_set_alloc_info); + std::vector sets = ctx->device->device.allocateDescriptorSets(descriptor_set_alloc_info); } catch(vk::OutOfPoolMemoryError const&) { - ctx->device.lock()->descriptor_set_mode = VK_DEVICE_DESCRIPTOR_POOL_MODE_SINGLE; + ctx->device->descriptor_set_mode = VK_DEVICE_DESCRIPTOR_POOL_MODE_SINGLE; } - ctx->device.lock()->device.destroyDescriptorPool(pool); + ctx->device->device.destroyDescriptorPool(pool); } - if (ctx->device.lock()->descriptor_set_mode == VK_DEVICE_DESCRIPTOR_POOL_MODE_MULTI) { - vk::DescriptorPoolSize descriptor_pool_size(vk::DescriptorType::eStorageBuffer, pipeline.parameter_count); + if (ctx->device->descriptor_set_mode == VK_DEVICE_DESCRIPTOR_POOL_MODE_MULTI) { + vk::DescriptorPoolSize descriptor_pool_size(vk::DescriptorType::eStorageBuffer, pipeline->parameter_count); vk::DescriptorPoolCreateInfo descriptor_pool_create_info({}, 128, descriptor_pool_size); - pipeline.descriptor_pools.push_back(ctx->device.lock()->device.createDescriptorPool(descriptor_pool_create_info)); + pipeline->descriptor_pools.push_back(ctx->device->device.createDescriptorPool(descriptor_pool_create_info)); } - pipeline.descriptor_set_idx = 0; + pipeline->descriptor_set_idx = 0; - vk::PipelineLayoutCreateInfo pipeline_layout_create_info(vk::PipelineLayoutCreateFlags(), pipeline.dsl, pcr); - pipeline.layout = ctx->device.lock()->device.createPipelineLayout(pipeline_layout_create_info); + vk::PipelineLayoutCreateInfo pipeline_layout_create_info(vk::PipelineLayoutCreateFlags(), pipeline->dsl, pcr); + pipeline->layout = ctx->device->device.createPipelineLayout(pipeline_layout_create_info); std::vector specialization_entries(specialization_constants.size()); @@ -417,72 +496,75 @@ static void ggml_vk_create_pipeline(ggml_backend_vk_context * ctx, vk_pipeline& vk::PipelineShaderStageCreateInfo pipeline_shader_create_info( vk::PipelineShaderStageCreateFlags(), vk::ShaderStageFlagBits::eCompute, - pipeline.shader_module, + pipeline->shader_module, entrypoint.c_str(), &specialization_info); vk::ComputePipelineCreateInfo compute_pipeline_create_info( vk::PipelineCreateFlags(), pipeline_shader_create_info, - pipeline.layout); - pipeline.pipeline = ctx->device.lock()->device.createComputePipeline(VK_NULL_HANDLE, compute_pipeline_create_info).value; + pipeline->layout); + pipeline->pipeline = ctx->device->device.createComputePipeline(VK_NULL_HANDLE, compute_pipeline_create_info).value; - ctx->gc.pipelines.push_back(&pipeline); + ctx->device->pipelines.push_back(pipeline); } -static void ggml_vk_destroy_pipeline(ggml_backend_vk_context * ctx, vk_pipeline * pipeline) { +static void ggml_vk_destroy_pipeline(vk::Device& device, vk_pipeline& pipeline) { +#ifdef GGML_VULKAN_DEBUG + std::cerr << "ggml_pipeline_destroy_pipeline(" << pipeline->name << ")" << std::endl; +#endif for (auto& pool : pipeline->descriptor_pools) { - ctx->device.lock()->device.destroyDescriptorPool(pool); + device.destroyDescriptorPool(pool); } pipeline->descriptor_pools.clear(); pipeline->descriptor_sets.clear(); pipeline->descriptor_set_idx = 0; - ctx->device.lock()->device.destroyDescriptorSetLayout(pipeline->dsl); + device.destroyDescriptorSetLayout(pipeline->dsl); - ctx->device.lock()->device.destroyPipelineLayout(pipeline->layout); + device.destroyPipelineLayout(pipeline->layout); - ctx->device.lock()->device.destroyShaderModule(pipeline->shader_module); + device.destroyShaderModule(pipeline->shader_module); - ctx->device.lock()->device.destroyPipeline(pipeline->pipeline); + device.destroyPipeline(pipeline->pipeline); } static void ggml_pipeline_allocate_descriptor_sets(ggml_backend_vk_context * ctx, vk_pipeline& pipeline, uint32_t n) { #ifdef GGML_VULKAN_DEBUG - std::cerr << "ggml_pipeline_allocate_descriptor_sets(" << pipeline.name << ", " << n << ")" << std::endl; + std::cerr << "ggml_pipeline_allocate_descriptor_sets(" << pipeline->name << ", " << n << ")" << std::endl; #endif - if (pipeline.descriptor_sets.size() >= pipeline.descriptor_set_idx + n) { + if (pipeline->descriptor_sets.size() >= pipeline->descriptor_set_idx + n) { // Enough descriptors are available return; } - if (ctx->device.lock()->descriptor_set_mode == VK_DEVICE_DESCRIPTOR_POOL_MODE_MULTI) { - const uint32_t alloc_count = pipeline.descriptor_set_idx + n - pipeline.descriptor_sets.size(); + if (ctx->device->descriptor_set_mode == VK_DEVICE_DESCRIPTOR_POOL_MODE_MULTI) { + const uint32_t alloc_count = pipeline->descriptor_set_idx + n - pipeline->descriptor_sets.size(); std::vector layouts(alloc_count); for (uint32_t i = 0; i < alloc_count; i++) { - layouts[i] = pipeline.dsl; + layouts[i] = pipeline->dsl; } - vk::DescriptorSetAllocateInfo descriptor_set_alloc_info(pipeline.descriptor_pools[0], alloc_count, layouts.data()); - std::vector sets = ctx->device.lock()->device.allocateDescriptorSets(descriptor_set_alloc_info); - pipeline.descriptor_sets.insert(pipeline.descriptor_sets.end(), sets.begin(), sets.end()); + vk::DescriptorSetAllocateInfo descriptor_set_alloc_info(pipeline->descriptor_pools[0], alloc_count, layouts.data()); + std::vector sets = ctx->device->device.allocateDescriptorSets(descriptor_set_alloc_info); + pipeline->descriptor_sets.insert(pipeline->descriptor_sets.end(), sets.begin(), sets.end()); } else { - for (uint32_t i = pipeline.descriptor_sets.size(); i < pipeline.descriptor_set_idx + n; i++) { - vk::DescriptorPoolSize descriptor_pool_size(vk::DescriptorType::eStorageBuffer, pipeline.parameter_count); + for (uint32_t i = pipeline->descriptor_sets.size(); i < pipeline->descriptor_set_idx + n; i++) { + vk::DescriptorPoolSize descriptor_pool_size(vk::DescriptorType::eStorageBuffer, pipeline->parameter_count); vk::DescriptorPoolCreateInfo descriptor_pool_create_info({}, 1, descriptor_pool_size); - pipeline.descriptor_pools.push_back(ctx->device.lock()->device.createDescriptorPool(descriptor_pool_create_info)); + pipeline->descriptor_pools.push_back(ctx->device->device.createDescriptorPool(descriptor_pool_create_info)); - vk::DescriptorSetAllocateInfo descriptor_set_alloc_info(pipeline.descriptor_pools[i], 1, &pipeline.dsl); - std::vector sets = ctx->device.lock()->device.allocateDescriptorSets(descriptor_set_alloc_info); - pipeline.descriptor_sets.push_back(sets[0]); + vk::DescriptorSetAllocateInfo descriptor_set_alloc_info(pipeline->descriptor_pools[i], 1, &pipeline->dsl); + std::vector sets = ctx->device->device.allocateDescriptorSets(descriptor_set_alloc_info); + pipeline->descriptor_sets.push_back(sets[0]); } } } static void ggml_pipeline_cleanup(vk_pipeline& pipeline) { #ifdef GGML_VULKAN_DEBUG - std::cerr << "ggml_pipeline_cleanup(" << pipeline.name << ")" << std::endl; + std::cerr << "ggml_pipeline_cleanup(" << pipeline->name << ")" << std::endl; #endif - pipeline.descriptor_set_idx = 0; + pipeline->descriptor_set_idx = 0; } static vk::CommandBuffer ggml_vk_create_cmd_buffer(ggml_backend_vk_context * ctx, vk_queue& q) { @@ -498,7 +580,7 @@ static vk::CommandBuffer ggml_vk_create_cmd_buffer(ggml_backend_vk_context * ctx q.pool, vk::CommandBufferLevel::ePrimary, 1); - const std::vector cmd_buffers = ctx->device.lock()->device.allocateCommandBuffers(command_buffer_alloc_info); + const std::vector cmd_buffers = ctx->device->device.allocateCommandBuffers(command_buffer_alloc_info); auto buf = cmd_buffers.front(); q.cmd_buffers.push_back(buf); @@ -643,11 +725,11 @@ static void ggml_vk_create_queue(ggml_backend_vk_context * ctx, vk_queue& q, uin q.queue_family_index = queue_family_index; vk::CommandPoolCreateInfo command_pool_create_info_compute(vk::CommandPoolCreateFlags(VK_COMMAND_POOL_CREATE_TRANSIENT_BIT), queue_family_index); - q.pool = ctx->device.lock()->device.createCommandPool(command_pool_create_info_compute); + q.pool = ctx->device->device.createCommandPool(command_pool_create_info_compute); q.cmd_buffer_idx = 0; - q.queue = ctx->device.lock()->device.getQueue(queue_family_index, queue_index); + q.queue = ctx->device->device.getQueue(queue_family_index, queue_index); q.stage_flags = stage_flags; } @@ -671,7 +753,7 @@ static vk_semaphore * ggml_vk_create_binary_semaphore(ggml_backend_vk_context * vk::SemaphoreTypeCreateInfo tci{ vk::SemaphoreType::eBinary, 0 }; vk::SemaphoreCreateInfo ci{}; ci.setPNext(&tci); - vk::Semaphore semaphore = ctx->device.lock()->device.createSemaphore(ci); + vk::Semaphore semaphore = ctx->device->device.createSemaphore(ci); ctx->gc.semaphores.push_back({ semaphore, 0 }); return &ctx->gc.semaphores[ctx->gc.semaphores.size() - 1]; } @@ -684,7 +766,7 @@ static vk_semaphore * ggml_vk_create_timeline_semaphore(ggml_backend_vk_context vk::SemaphoreTypeCreateInfo tci{ vk::SemaphoreType::eTimeline, 0 }; vk::SemaphoreCreateInfo ci{}; ci.setPNext(&tci); - vk::Semaphore semaphore = ctx->device.lock()->device.createSemaphore(ci); + vk::Semaphore semaphore = ctx->device->device.createSemaphore(ci); ctx->gc.tl_semaphores.push_back({ semaphore, 0 }); } return &ctx->gc.tl_semaphores[ctx->semaphore_idx++]; @@ -692,7 +774,7 @@ static vk_semaphore * ggml_vk_create_timeline_semaphore(ggml_backend_vk_context static vk::Event ggml_vk_create_event(ggml_backend_vk_context * ctx) { if (ctx->event_idx >= ctx->gc.events.size()) { - ctx->gc.events.push_back(ctx->device.lock()->device.createEvent({})); + ctx->gc.events.push_back(ctx->device->device.createEvent({})); } return ctx->gc.events[ctx->event_idx++]; } @@ -703,7 +785,7 @@ static void ggml_vk_queue_cleanup(ggml_backend_vk_context * ctx, vk_queue& q) { #endif // Requires command buffers to be done - ctx->device.lock()->device.resetCommandPool(q.pool); + ctx->device->device.resetCommandPool(q.pool); q.cmd_buffer_idx = 0; } @@ -740,11 +822,11 @@ static vk_buffer ggml_vk_create_buffer(ggml_backend_vk_context * ctx, size_t siz nullptr, }; - buf->buffer = ctx->device.lock()->device.createBuffer(buffer_create_info); + buf->buffer = ctx->device->device.createBuffer(buffer_create_info); - vk::MemoryRequirements mem_req = ctx->device.lock()->device.getBufferMemoryRequirements(buf->buffer); + vk::MemoryRequirements mem_req = ctx->device->device.getBufferMemoryRequirements(buf->buffer); - vk::PhysicalDeviceMemoryProperties mem_props = ctx->device.lock()->physical_device.getMemoryProperties(); + vk::PhysicalDeviceMemoryProperties mem_props = ctx->device->physical_device.getMemoryProperties(); uint32_t memory_type_index = UINT32_MAX; @@ -757,30 +839,30 @@ static vk_buffer ggml_vk_create_buffer(ggml_backend_vk_context * ctx, size_t siz } if (memory_type_index == UINT32_MAX) { - ctx->device.lock()->device.destroyBuffer(buf->buffer); + ctx->device->device.destroyBuffer(buf->buffer); buf->size = 0; throw vk::OutOfDeviceMemoryError("No suitable memory type found"); } try { - buf->device_memory = ctx->device.lock()->device.allocateMemory({ mem_req.size, memory_type_index }); + buf->device_memory = ctx->device->device.allocateMemory({ mem_req.size, memory_type_index }); } catch (const vk::SystemError& e) { // Out of Host/Device memory, clean up buffer - ctx->device.lock()->device.destroyBuffer(buf->buffer); + ctx->device->device.destroyBuffer(buf->buffer); buf->size = 0; throw e; } buf->ptr = nullptr; if (buf->memory_property_flags & vk::MemoryPropertyFlagBits::eHostVisible) { - buf->ptr = ctx->device.lock()->device.mapMemory(buf->device_memory, 0, VK_WHOLE_SIZE); + buf->ptr = ctx->device->device.mapMemory(buf->device_memory, 0, VK_WHOLE_SIZE); } - ctx->device.lock()->device.bindBufferMemory(buf->buffer, buf->device_memory, 0); + ctx->device->device.bindBufferMemory(buf->buffer, buf->device_memory, 0); buf->ctx = ctx; - buf->device = ctx->device.lock(); + buf->device = ctx->device; #ifdef GGML_VULKAN_DEBUG std::cerr << "Created buffer " << buf->buffer << std::endl; @@ -802,7 +884,7 @@ static vk_buffer ggml_vk_create_buffer_check(ggml_backend_vk_context * ctx, size static vk_buffer ggml_vk_create_buffer_device(ggml_backend_vk_context * ctx, size_t size) { vk_buffer buf; try { - if (ctx->device.lock()->uma) { + if (ctx->device->uma) { // Fall back to host memory type buf = ggml_vk_create_buffer(ctx, size, vk::MemoryPropertyFlagBits::eDeviceLocal, vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent); } else { @@ -883,10 +965,16 @@ static void ggml_vk_load_shaders(ggml_backend_vk_context * ctx) { std::cerr << "ggml_vk_load_shaders(" << ctx->name << ")" << std::endl; #endif + const std::shared_ptr device = ctx->device; + // mulmat - std::initializer_list warptile_l = { 128, 128, 128, 16, ctx->device.lock()->subgroup_size * 2, 64, 2, 4, 4, ctx->device.lock()->subgroup_size }; - std::initializer_list warptile_m = { 128, 64, 64, 16, ctx->device.lock()->subgroup_size, 32, 2, 4, 2, ctx->device.lock()->subgroup_size }; - std::initializer_list warptile_s = { ctx->device.lock()->subgroup_size, 32, 32, 16, 32, 32, 2, 2, 2, ctx->device.lock()->subgroup_size }; + std::initializer_list warptile_l = { 128, 128, 128, 16, device->subgroup_size * 2, 64, 2, 4, 4, device->subgroup_size }; + std::initializer_list warptile_m = { 128, 64, 64, 16, device->subgroup_size, 32, 2, 4, 2, device->subgroup_size }; + std::initializer_list warptile_s = { device->subgroup_size, 32, 32, 16, 32, 32, 2, 2, 2, device->subgroup_size }; + + std::initializer_list warptile_mmq_l = { 128, 128, 128, 32, device->subgroup_size * 2, 64, 2, 4, 4, device->subgroup_size }; + std::initializer_list warptile_mmq_m = { 128, 64, 64, 32, device->subgroup_size, 32, 2, 4, 2, device->subgroup_size }; + std::initializer_list warptile_mmq_s = { device->subgroup_size, 32, 32, 32, 32, 32, 2, 2, 2, device->subgroup_size }; std::array l_wg_denoms = {128, 128, 1 }; std::array m_wg_denoms = { 64, 64, 1 }; @@ -896,126 +984,206 @@ static void ggml_vk_load_shaders(ggml_backend_vk_context * ctx) { uint32_t m_align = 64; uint32_t s_align = 32; - if (ctx->device.lock()->fp16) { - ggml_vk_create_pipeline(ctx, ctx->pipeline_matmul_f32_l, "matmul_f32_l", matmul_f32_l_len, matmul_f32_l_data, "main", 3, 14 * sizeof(uint32_t), l_wg_denoms, warptile_l, 1); - ggml_vk_create_pipeline(ctx, ctx->pipeline_matmul_f32_m, "matmul_f32_m", matmul_f32_m_len, matmul_f32_m_data, "main", 3, 14 * sizeof(uint32_t), m_wg_denoms, warptile_m, 1); - ggml_vk_create_pipeline(ctx, ctx->pipeline_matmul_f32_s, "matmul_f32_s", matmul_f32_s_len, matmul_f32_s_data, "main", 3, 14 * sizeof(uint32_t), s_wg_denoms, warptile_s, 1); - ggml_vk_create_pipeline(ctx, ctx->pipeline_matmul_f32_aligned_l, "matmul_f32_aligned_l", matmul_f32_aligned_l_len, matmul_f32_aligned_l_data, "main", 3, 14 * sizeof(uint32_t), l_wg_denoms, warptile_l, l_align); - ggml_vk_create_pipeline(ctx, ctx->pipeline_matmul_f32_aligned_m, "matmul_f32_aligned_m", matmul_f32_aligned_m_len, matmul_f32_aligned_m_data, "main", 3, 14 * sizeof(uint32_t), m_wg_denoms, warptile_m, m_align); - ggml_vk_create_pipeline(ctx, ctx->pipeline_matmul_f32_aligned_s, "matmul_f32_aligned_s", matmul_f32_aligned_s_len, matmul_f32_aligned_s_data, "main", 3, 14 * sizeof(uint32_t), s_wg_denoms, warptile_s, s_align); + ctx->device->pipeline_matmul_f32 = std::make_shared(); + ctx->device->pipeline_matmul_f16_f32 = std::make_shared(); + ctx->device->pipeline_matmul_f16 = std::make_shared(); + ctx->device->pipeline_dequant_mul_mat_mat[GGML_TYPE_Q4_0] = std::make_shared(); + ctx->device->pipeline_dequant_mul_mat_mat[GGML_TYPE_Q4_1] = std::make_shared(); + ctx->device->pipeline_dequant_mul_mat_mat[GGML_TYPE_Q5_0] = std::make_shared(); + ctx->device->pipeline_dequant_mul_mat_mat[GGML_TYPE_Q5_1] = std::make_shared(); + ctx->device->pipeline_dequant_mul_mat_mat[GGML_TYPE_Q8_0] = std::make_shared(); - ggml_vk_create_pipeline(ctx, ctx->pipeline_matmul_f16_l, "matmul_f16_l", matmul_f16_l_len, matmul_f16_l_data, "main", 3, 14 * sizeof(uint32_t), l_wg_denoms, warptile_l, 1); - ggml_vk_create_pipeline(ctx, ctx->pipeline_matmul_f16_m, "matmul_f16_m", matmul_f16_m_len, matmul_f16_m_data, "main", 3, 14 * sizeof(uint32_t), m_wg_denoms, warptile_m, 1); - ggml_vk_create_pipeline(ctx, ctx->pipeline_matmul_f16_s, "matmul_f16_s", matmul_f16_s_len, matmul_f16_s_data, "main", 3, 14 * sizeof(uint32_t), s_wg_denoms, warptile_s, 1); - ggml_vk_create_pipeline(ctx, ctx->pipeline_matmul_f16_aligned_l, "matmul_f16_aligned_l", matmul_f16_aligned_l_len, matmul_f16_aligned_l_data, "main", 3, 14 * sizeof(uint32_t), l_wg_denoms, warptile_l, l_align); - ggml_vk_create_pipeline(ctx, ctx->pipeline_matmul_f16_aligned_m, "matmul_f16_aligned_m", matmul_f16_aligned_m_len, matmul_f16_aligned_m_data, "main", 3, 14 * sizeof(uint32_t), m_wg_denoms, warptile_m, m_align); - ggml_vk_create_pipeline(ctx, ctx->pipeline_matmul_f16_aligned_s, "matmul_f16_aligned_s", matmul_f16_aligned_s_len, matmul_f16_aligned_s_data, "main", 3, 14 * sizeof(uint32_t), s_wg_denoms, warptile_s, s_align); + if (device->fp16) { + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_matmul_f32->l, "matmul_f32_l", matmul_f32_len, matmul_f32_data, "main", 3, 14 * sizeof(uint32_t), l_wg_denoms, warptile_l, 1); + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_matmul_f32->m, "matmul_f32_m", matmul_f32_len, matmul_f32_data, "main", 3, 14 * sizeof(uint32_t), m_wg_denoms, warptile_m, 1); + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_matmul_f32->s, "matmul_f32_s", matmul_f32_len, matmul_f32_data, "main", 3, 14 * sizeof(uint32_t), s_wg_denoms, warptile_s, 1); + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_matmul_f32->a_l, "matmul_f32_aligned_l", matmul_f32_aligned_len, matmul_f32_aligned_data, "main", 3, 14 * sizeof(uint32_t), l_wg_denoms, warptile_l, l_align); + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_matmul_f32->a_m, "matmul_f32_aligned_m", matmul_f32_aligned_len, matmul_f32_aligned_data, "main", 3, 14 * sizeof(uint32_t), m_wg_denoms, warptile_m, m_align); + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_matmul_f32->a_s, "matmul_f32_aligned_s", matmul_f32_aligned_len, matmul_f32_aligned_data, "main", 3, 14 * sizeof(uint32_t), s_wg_denoms, warptile_s, s_align); - ggml_vk_create_pipeline(ctx, ctx->pipeline_matmul_f16_f32_l, "matmul_f16_f32_l", matmul_f16_f32_l_len, matmul_f16_f32_l_data, "main", 3, 14 * sizeof(uint32_t), l_wg_denoms, warptile_l, 1); - ggml_vk_create_pipeline(ctx, ctx->pipeline_matmul_f16_f32_m, "matmul_f16_f32_m", matmul_f16_f32_m_len, matmul_f16_f32_m_data, "main", 3, 14 * sizeof(uint32_t), m_wg_denoms, warptile_m, 1); - ggml_vk_create_pipeline(ctx, ctx->pipeline_matmul_f16_f32_s, "matmul_f16_f32_s", matmul_f16_f32_s_len, matmul_f16_f32_s_data, "main", 3, 14 * sizeof(uint32_t), s_wg_denoms, warptile_s, 1); - ggml_vk_create_pipeline(ctx, ctx->pipeline_matmul_f16_f32_aligned_l, "matmul_f16_f32_aligned_l", matmul_f16_f32_aligned_l_len, matmul_f16_f32_aligned_l_data, "main", 3, 14 * sizeof(uint32_t), l_wg_denoms, warptile_l, l_align); - ggml_vk_create_pipeline(ctx, ctx->pipeline_matmul_f16_f32_aligned_m, "matmul_f16_f32_aligned_m", matmul_f16_f32_aligned_m_len, matmul_f16_f32_aligned_m_data, "main", 3, 14 * sizeof(uint32_t), m_wg_denoms, warptile_m, m_align); - ggml_vk_create_pipeline(ctx, ctx->pipeline_matmul_f16_f32_aligned_s, "matmul_f16_f32_aligned_s", matmul_f16_f32_aligned_s_len, matmul_f16_f32_aligned_s_data, "main", 3, 14 * sizeof(uint32_t), s_wg_denoms, warptile_s, s_align); + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_matmul_f16->l, "matmul_f16_l", matmul_f16_len, matmul_f16_data, "main", 3, 14 * sizeof(uint32_t), l_wg_denoms, warptile_l, 1); + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_matmul_f16->m, "matmul_f16_m", matmul_f16_len, matmul_f16_data, "main", 3, 14 * sizeof(uint32_t), m_wg_denoms, warptile_m, 1); + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_matmul_f16->s, "matmul_f16_s", matmul_f16_len, matmul_f16_data, "main", 3, 14 * sizeof(uint32_t), s_wg_denoms, warptile_s, 1); + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_matmul_f16->a_l, "matmul_f16_aligned_l", matmul_f16_aligned_len, matmul_f16_aligned_data, "main", 3, 14 * sizeof(uint32_t), l_wg_denoms, warptile_l, l_align); + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_matmul_f16->a_m, "matmul_f16_aligned_m", matmul_f16_aligned_len, matmul_f16_aligned_data, "main", 3, 14 * sizeof(uint32_t), m_wg_denoms, warptile_m, m_align); + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_matmul_f16->a_s, "matmul_f16_aligned_s", matmul_f16_aligned_len, matmul_f16_aligned_data, "main", 3, 14 * sizeof(uint32_t), s_wg_denoms, warptile_s, s_align); + + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_matmul_f16_f32->l, "matmul_f16_f32_l", matmul_f16_f32_len, matmul_f16_f32_data, "main", 3, 14 * sizeof(uint32_t), l_wg_denoms, warptile_l, 1); + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_matmul_f16_f32->m, "matmul_f16_f32_m", matmul_f16_f32_len, matmul_f16_f32_data, "main", 3, 14 * sizeof(uint32_t), m_wg_denoms, warptile_m, 1); + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_matmul_f16_f32->s, "matmul_f16_f32_s", matmul_f16_f32_len, matmul_f16_f32_data, "main", 3, 14 * sizeof(uint32_t), s_wg_denoms, warptile_s, 1); + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_matmul_f16_f32->a_l, "matmul_f16_f32_aligned_l", matmul_f16_f32_aligned_len, matmul_f16_f32_aligned_data, "main", 3, 14 * sizeof(uint32_t), l_wg_denoms, warptile_l, l_align); + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_matmul_f16_f32->a_m, "matmul_f16_f32_aligned_m", matmul_f16_f32_aligned_len, matmul_f16_f32_aligned_data, "main", 3, 14 * sizeof(uint32_t), m_wg_denoms, warptile_m, m_align); + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_matmul_f16_f32->a_s, "matmul_f16_f32_aligned_s", matmul_f16_f32_aligned_len, matmul_f16_f32_aligned_data, "main", 3, 14 * sizeof(uint32_t), s_wg_denoms, warptile_s, s_align); + + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_dequant_mul_mat_mat[GGML_TYPE_Q4_0]->l, "matmul_q4_0_f32_l", matmul_q4_0_f32_len, matmul_q4_0_f32_data, "main", 3, 14 * sizeof(uint32_t), l_wg_denoms, warptile_mmq_l, l_align); + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_dequant_mul_mat_mat[GGML_TYPE_Q4_0]->m, "matmul_q4_0_f32_m", matmul_q4_0_f32_len, matmul_q4_0_f32_data, "main", 3, 14 * sizeof(uint32_t), m_wg_denoms, warptile_mmq_m, m_align); + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_dequant_mul_mat_mat[GGML_TYPE_Q4_0]->s, "matmul_q4_0_f32_s", matmul_q4_0_f32_len, matmul_q4_0_f32_data, "main", 3, 14 * sizeof(uint32_t), s_wg_denoms, warptile_mmq_s, s_align); + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_dequant_mul_mat_mat[GGML_TYPE_Q4_0]->a_l, "matmul_q4_0_f32_aligned_l", matmul_q4_0_f32_aligned_len, matmul_q4_0_f32_aligned_data, "main", 3, 14 * sizeof(uint32_t), l_wg_denoms, warptile_mmq_l, l_align); + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_dequant_mul_mat_mat[GGML_TYPE_Q4_0]->a_m, "matmul_q4_0_f32_aligned_m", matmul_q4_0_f32_aligned_len, matmul_q4_0_f32_aligned_data, "main", 3, 14 * sizeof(uint32_t), m_wg_denoms, warptile_mmq_m, m_align); + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_dequant_mul_mat_mat[GGML_TYPE_Q4_0]->a_s, "matmul_q4_0_f32_aligned_s", matmul_q4_0_f32_aligned_len, matmul_q4_0_f32_aligned_data, "main", 3, 14 * sizeof(uint32_t), s_wg_denoms, warptile_mmq_s, s_align); + + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_dequant_mul_mat_mat[GGML_TYPE_Q4_1]->l, "matmul_q4_0_f32_l", matmul_q4_1_f32_len, matmul_q4_1_f32_data, "main", 3, 14 * sizeof(uint32_t), l_wg_denoms, warptile_mmq_l, l_align); + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_dequant_mul_mat_mat[GGML_TYPE_Q4_1]->m, "matmul_q4_0_f32_m", matmul_q4_1_f32_len, matmul_q4_1_f32_data, "main", 3, 14 * sizeof(uint32_t), m_wg_denoms, warptile_mmq_m, m_align); + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_dequant_mul_mat_mat[GGML_TYPE_Q4_1]->s, "matmul_q4_0_f32_s", matmul_q4_1_f32_len, matmul_q4_1_f32_data, "main", 3, 14 * sizeof(uint32_t), s_wg_denoms, warptile_mmq_s, s_align); + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_dequant_mul_mat_mat[GGML_TYPE_Q4_1]->a_l, "matmul_q4_0_f32_aligned_l", matmul_q4_1_f32_aligned_len, matmul_q4_1_f32_aligned_data, "main", 3, 14 * sizeof(uint32_t), l_wg_denoms, warptile_mmq_l, l_align); + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_dequant_mul_mat_mat[GGML_TYPE_Q4_1]->a_m, "matmul_q4_0_f32_aligned_m", matmul_q4_1_f32_aligned_len, matmul_q4_1_f32_aligned_data, "main", 3, 14 * sizeof(uint32_t), m_wg_denoms, warptile_mmq_m, m_align); + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_dequant_mul_mat_mat[GGML_TYPE_Q4_1]->a_s, "matmul_q4_0_f32_aligned_s", matmul_q4_1_f32_aligned_len, matmul_q4_1_f32_aligned_data, "main", 3, 14 * sizeof(uint32_t), s_wg_denoms, warptile_mmq_s, s_align); + + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_dequant_mul_mat_mat[GGML_TYPE_Q5_0]->l, "matmul_q5_0_f32_l", matmul_q5_0_f32_len, matmul_q5_0_f32_data, "main", 3, 14 * sizeof(uint32_t), l_wg_denoms, warptile_mmq_l, l_align); + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_dequant_mul_mat_mat[GGML_TYPE_Q5_0]->m, "matmul_q5_0_f32_m", matmul_q5_0_f32_len, matmul_q5_0_f32_data, "main", 3, 14 * sizeof(uint32_t), m_wg_denoms, warptile_mmq_m, m_align); + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_dequant_mul_mat_mat[GGML_TYPE_Q5_0]->s, "matmul_q5_0_f32_s", matmul_q5_0_f32_len, matmul_q5_0_f32_data, "main", 3, 14 * sizeof(uint32_t), s_wg_denoms, warptile_mmq_s, s_align); + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_dequant_mul_mat_mat[GGML_TYPE_Q5_0]->a_l, "matmul_q5_0_f32_aligned_l", matmul_q5_0_f32_aligned_len, matmul_q5_0_f32_aligned_data, "main", 3, 14 * sizeof(uint32_t), l_wg_denoms, warptile_mmq_l, l_align); + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_dequant_mul_mat_mat[GGML_TYPE_Q5_0]->a_m, "matmul_q5_0_f32_aligned_m", matmul_q5_0_f32_aligned_len, matmul_q5_0_f32_aligned_data, "main", 3, 14 * sizeof(uint32_t), m_wg_denoms, warptile_mmq_m, m_align); + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_dequant_mul_mat_mat[GGML_TYPE_Q5_0]->a_s, "matmul_q5_0_f32_aligned_s", matmul_q5_0_f32_aligned_len, matmul_q5_0_f32_aligned_data, "main", 3, 14 * sizeof(uint32_t), s_wg_denoms, warptile_mmq_s, s_align); + + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_dequant_mul_mat_mat[GGML_TYPE_Q5_1]->l, "matmul_q5_1_f32_l", matmul_q5_1_f32_len, matmul_q5_1_f32_data, "main", 3, 14 * sizeof(uint32_t), l_wg_denoms, warptile_mmq_l, l_align); + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_dequant_mul_mat_mat[GGML_TYPE_Q5_1]->m, "matmul_q5_1_f32_m", matmul_q5_1_f32_len, matmul_q5_1_f32_data, "main", 3, 14 * sizeof(uint32_t), m_wg_denoms, warptile_mmq_m, m_align); + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_dequant_mul_mat_mat[GGML_TYPE_Q5_1]->s, "matmul_q5_1_f32_s", matmul_q5_1_f32_len, matmul_q5_1_f32_data, "main", 3, 14 * sizeof(uint32_t), s_wg_denoms, warptile_mmq_s, s_align); + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_dequant_mul_mat_mat[GGML_TYPE_Q5_1]->a_l, "matmul_q5_1_f32_aligned_l", matmul_q5_1_f32_aligned_len, matmul_q5_1_f32_aligned_data, "main", 3, 14 * sizeof(uint32_t), l_wg_denoms, warptile_mmq_l, l_align); + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_dequant_mul_mat_mat[GGML_TYPE_Q5_1]->a_m, "matmul_q5_1_f32_aligned_m", matmul_q5_1_f32_aligned_len, matmul_q5_1_f32_aligned_data, "main", 3, 14 * sizeof(uint32_t), m_wg_denoms, warptile_mmq_m, m_align); + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_dequant_mul_mat_mat[GGML_TYPE_Q5_1]->a_s, "matmul_q5_1_f32_aligned_s", matmul_q5_1_f32_aligned_len, matmul_q5_1_f32_aligned_data, "main", 3, 14 * sizeof(uint32_t), s_wg_denoms, warptile_mmq_s, s_align); + + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_dequant_mul_mat_mat[GGML_TYPE_Q8_0]->l, "matmul_q8_0_f32_l", matmul_q8_0_f32_len, matmul_q8_0_f32_data, "main", 3, 14 * sizeof(uint32_t), l_wg_denoms, warptile_mmq_l, l_align); + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_dequant_mul_mat_mat[GGML_TYPE_Q8_0]->m, "matmul_q8_0_f32_m", matmul_q8_0_f32_len, matmul_q8_0_f32_data, "main", 3, 14 * sizeof(uint32_t), m_wg_denoms, warptile_mmq_m, m_align); + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_dequant_mul_mat_mat[GGML_TYPE_Q8_0]->s, "matmul_q8_0_f32_s", matmul_q8_0_f32_len, matmul_q8_0_f32_data, "main", 3, 14 * sizeof(uint32_t), s_wg_denoms, warptile_mmq_s, s_align); + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_dequant_mul_mat_mat[GGML_TYPE_Q8_0]->a_l, "matmul_q8_0_f32_aligned_l", matmul_q8_0_f32_aligned_len, matmul_q8_0_f32_aligned_data, "main", 3, 14 * sizeof(uint32_t), l_wg_denoms, warptile_mmq_l, l_align); + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_dequant_mul_mat_mat[GGML_TYPE_Q8_0]->a_m, "matmul_q8_0_f32_aligned_m", matmul_q8_0_f32_aligned_len, matmul_q8_0_f32_aligned_data, "main", 3, 14 * sizeof(uint32_t), m_wg_denoms, warptile_mmq_m, m_align); + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_dequant_mul_mat_mat[GGML_TYPE_Q8_0]->a_s, "matmul_q8_0_f32_aligned_s", matmul_q8_0_f32_aligned_len, matmul_q8_0_f32_aligned_data, "main", 3, 14 * sizeof(uint32_t), s_wg_denoms, warptile_mmq_s, s_align); } else { - ggml_vk_create_pipeline(ctx, ctx->pipeline_matmul_f32_l, "matmul_f32_l", matmul_f32_l_fp32_len, matmul_f32_l_fp32_data, "main", 3, 14 * sizeof(uint32_t), l_wg_denoms, warptile_l, 1); - ggml_vk_create_pipeline(ctx, ctx->pipeline_matmul_f32_m, "matmul_f32_m", matmul_f32_m_fp32_len, matmul_f32_m_fp32_data, "main", 3, 14 * sizeof(uint32_t), m_wg_denoms, warptile_m, 1); - ggml_vk_create_pipeline(ctx, ctx->pipeline_matmul_f32_s, "matmul_f32_s", matmul_f32_s_fp32_len, matmul_f32_s_fp32_data, "main", 3, 14 * sizeof(uint32_t), s_wg_denoms, warptile_s, 1); - ggml_vk_create_pipeline(ctx, ctx->pipeline_matmul_f32_aligned_l, "matmul_f32_aligned_l", matmul_f32_aligned_l_fp32_len, matmul_f32_aligned_l_fp32_data, "main", 3, 14 * sizeof(uint32_t), l_wg_denoms, warptile_l, l_align); - ggml_vk_create_pipeline(ctx, ctx->pipeline_matmul_f32_aligned_m, "matmul_f32_aligned_m", matmul_f32_aligned_m_fp32_len, matmul_f32_aligned_m_fp32_data, "main", 3, 14 * sizeof(uint32_t), m_wg_denoms, warptile_m, m_align); - ggml_vk_create_pipeline(ctx, ctx->pipeline_matmul_f32_aligned_s, "matmul_f32_aligned_s", matmul_f32_aligned_s_fp32_len, matmul_f32_aligned_s_fp32_data, "main", 3, 14 * sizeof(uint32_t), s_wg_denoms, warptile_s, s_align); + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_matmul_f32->l, "matmul_f32_l", matmul_f32_fp32_len, matmul_f32_fp32_data, "main", 3, 14 * sizeof(uint32_t), l_wg_denoms, warptile_l, 1); + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_matmul_f32->m, "matmul_f32_m", matmul_f32_fp32_len, matmul_f32_fp32_data, "main", 3, 14 * sizeof(uint32_t), m_wg_denoms, warptile_m, 1); + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_matmul_f32->s, "matmul_f32_s", matmul_f32_fp32_len, matmul_f32_fp32_data, "main", 3, 14 * sizeof(uint32_t), s_wg_denoms, warptile_s, 1); + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_matmul_f32->a_l, "matmul_f32_aligned_l", matmul_f32_aligned_fp32_len, matmul_f32_aligned_fp32_data, "main", 3, 14 * sizeof(uint32_t), l_wg_denoms, warptile_l, l_align); + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_matmul_f32->a_m, "matmul_f32_aligned_m", matmul_f32_aligned_fp32_len, matmul_f32_aligned_fp32_data, "main", 3, 14 * sizeof(uint32_t), m_wg_denoms, warptile_m, m_align); + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_matmul_f32->a_s, "matmul_f32_aligned_s", matmul_f32_aligned_fp32_len, matmul_f32_aligned_fp32_data, "main", 3, 14 * sizeof(uint32_t), s_wg_denoms, warptile_s, s_align); - ggml_vk_create_pipeline(ctx, ctx->pipeline_matmul_f16_l, "matmul_f16_l", matmul_f16_l_fp32_len, matmul_f16_l_fp32_data, "main", 3, 14 * sizeof(uint32_t), l_wg_denoms, warptile_l, 1); - ggml_vk_create_pipeline(ctx, ctx->pipeline_matmul_f16_m, "matmul_f16_m", matmul_f16_m_fp32_len, matmul_f16_m_fp32_data, "main", 3, 14 * sizeof(uint32_t), m_wg_denoms, warptile_m, 1); - ggml_vk_create_pipeline(ctx, ctx->pipeline_matmul_f16_s, "matmul_f16_s", matmul_f16_s_fp32_len, matmul_f16_s_fp32_data, "main", 3, 14 * sizeof(uint32_t), s_wg_denoms, warptile_s, 1); - ggml_vk_create_pipeline(ctx, ctx->pipeline_matmul_f16_aligned_l, "matmul_f16_aligned_l", matmul_f16_aligned_l_fp32_len, matmul_f16_aligned_l_fp32_data, "main", 3, 14 * sizeof(uint32_t), l_wg_denoms, warptile_l, l_align); - ggml_vk_create_pipeline(ctx, ctx->pipeline_matmul_f16_aligned_m, "matmul_f16_aligned_m", matmul_f16_aligned_m_fp32_len, matmul_f16_aligned_m_fp32_data, "main", 3, 14 * sizeof(uint32_t), m_wg_denoms, warptile_m, m_align); - ggml_vk_create_pipeline(ctx, ctx->pipeline_matmul_f16_aligned_s, "matmul_f16_aligned_s", matmul_f16_aligned_s_fp32_len, matmul_f16_aligned_s_fp32_data, "main", 3, 14 * sizeof(uint32_t), s_wg_denoms, warptile_s, s_align); + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_matmul_f16->l, "matmul_f16_l", matmul_f16_fp32_len, matmul_f16_fp32_data, "main", 3, 14 * sizeof(uint32_t), l_wg_denoms, warptile_l, 1); + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_matmul_f16->m, "matmul_f16_m", matmul_f16_fp32_len, matmul_f16_fp32_data, "main", 3, 14 * sizeof(uint32_t), m_wg_denoms, warptile_m, 1); + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_matmul_f16->s, "matmul_f16_s", matmul_f16_fp32_len, matmul_f16_fp32_data, "main", 3, 14 * sizeof(uint32_t), s_wg_denoms, warptile_s, 1); + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_matmul_f16->a_l, "matmul_f16_aligned_l", matmul_f16_aligned_fp32_len, matmul_f16_aligned_fp32_data, "main", 3, 14 * sizeof(uint32_t), l_wg_denoms, warptile_l, l_align); + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_matmul_f16->a_m, "matmul_f16_aligned_m", matmul_f16_aligned_fp32_len, matmul_f16_aligned_fp32_data, "main", 3, 14 * sizeof(uint32_t), m_wg_denoms, warptile_m, m_align); + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_matmul_f16->a_s, "matmul_f16_aligned_s", matmul_f16_aligned_fp32_len, matmul_f16_aligned_fp32_data, "main", 3, 14 * sizeof(uint32_t), s_wg_denoms, warptile_s, s_align); - ggml_vk_create_pipeline(ctx, ctx->pipeline_matmul_f16_f32_l, "matmul_f16_f32_l", matmul_f16_f32_l_fp32_len, matmul_f16_f32_l_fp32_data, "main", 3, 14 * sizeof(uint32_t), l_wg_denoms, warptile_l, 1); - ggml_vk_create_pipeline(ctx, ctx->pipeline_matmul_f16_f32_m, "matmul_f16_f32_m", matmul_f16_f32_m_fp32_len, matmul_f16_f32_m_fp32_data, "main", 3, 14 * sizeof(uint32_t), m_wg_denoms, warptile_m, 1); - ggml_vk_create_pipeline(ctx, ctx->pipeline_matmul_f16_f32_s, "matmul_f16_f32_s", matmul_f16_f32_s_fp32_len, matmul_f16_f32_s_fp32_data, "main", 3, 14 * sizeof(uint32_t), s_wg_denoms, warptile_s, 1); - ggml_vk_create_pipeline(ctx, ctx->pipeline_matmul_f16_f32_aligned_l, "matmul_f16_f32_aligned_l", matmul_f16_f32_aligned_l_fp32_len, matmul_f16_f32_aligned_l_fp32_data, "main", 3, 14 * sizeof(uint32_t), l_wg_denoms, warptile_l, l_align); - ggml_vk_create_pipeline(ctx, ctx->pipeline_matmul_f16_f32_aligned_m, "matmul_f16_f32_aligned_m", matmul_f16_f32_aligned_m_fp32_len, matmul_f16_f32_aligned_m_fp32_data, "main", 3, 14 * sizeof(uint32_t), m_wg_denoms, warptile_m, m_align); - ggml_vk_create_pipeline(ctx, ctx->pipeline_matmul_f16_f32_aligned_s, "matmul_f16_f32_aligned_s", matmul_f16_f32_aligned_s_fp32_len, matmul_f16_f32_aligned_s_fp32_data, "main", 3, 14 * sizeof(uint32_t), s_wg_denoms, warptile_s, s_align); + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_matmul_f16_f32->l, "matmul_f16_f32_l", matmul_f16_f32_fp32_len, matmul_f16_f32_fp32_data, "main", 3, 14 * sizeof(uint32_t), l_wg_denoms, warptile_l, 1); + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_matmul_f16_f32->m, "matmul_f16_f32_m", matmul_f16_f32_fp32_len, matmul_f16_f32_fp32_data, "main", 3, 14 * sizeof(uint32_t), m_wg_denoms, warptile_m, 1); + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_matmul_f16_f32->s, "matmul_f16_f32_s", matmul_f16_f32_fp32_len, matmul_f16_f32_fp32_data, "main", 3, 14 * sizeof(uint32_t), s_wg_denoms, warptile_s, 1); + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_matmul_f16_f32->a_l, "matmul_f16_f32_aligned_l", matmul_f16_f32_aligned_fp32_len, matmul_f16_f32_aligned_fp32_data, "main", 3, 14 * sizeof(uint32_t), l_wg_denoms, warptile_l, l_align); + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_matmul_f16_f32->a_m, "matmul_f16_f32_aligned_m", matmul_f16_f32_aligned_fp32_len, matmul_f16_f32_aligned_fp32_data, "main", 3, 14 * sizeof(uint32_t), m_wg_denoms, warptile_m, m_align); + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_matmul_f16_f32->a_s, "matmul_f16_f32_aligned_s", matmul_f16_f32_aligned_fp32_len, matmul_f16_f32_aligned_fp32_data, "main", 3, 14 * sizeof(uint32_t), s_wg_denoms, warptile_s, s_align); + + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_dequant_mul_mat_mat[GGML_TYPE_Q4_0]->l, "matmul_q4_0_f32_l", matmul_q4_0_f32_fp32_len, matmul_q4_0_f32_fp32_data, "main", 3, 14 * sizeof(uint32_t), l_wg_denoms, warptile_mmq_l, l_align); + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_dequant_mul_mat_mat[GGML_TYPE_Q4_0]->m, "matmul_q4_0_f32_m", matmul_q4_0_f32_fp32_len, matmul_q4_0_f32_fp32_data, "main", 3, 14 * sizeof(uint32_t), m_wg_denoms, warptile_mmq_m, m_align); + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_dequant_mul_mat_mat[GGML_TYPE_Q4_0]->s, "matmul_q4_0_f32_s", matmul_q4_0_f32_fp32_len, matmul_q4_0_f32_fp32_data, "main", 3, 14 * sizeof(uint32_t), s_wg_denoms, warptile_mmq_s, s_align); + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_dequant_mul_mat_mat[GGML_TYPE_Q4_0]->a_l, "matmul_q4_0_f32_aligned_l", matmul_q4_0_f32_aligned_fp32_len, matmul_q4_0_f32_aligned_fp32_data, "main", 3, 14 * sizeof(uint32_t), l_wg_denoms, warptile_mmq_l, l_align); + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_dequant_mul_mat_mat[GGML_TYPE_Q4_0]->a_m, "matmul_q4_0_f32_aligned_m", matmul_q4_0_f32_aligned_fp32_len, matmul_q4_0_f32_aligned_fp32_data, "main", 3, 14 * sizeof(uint32_t), m_wg_denoms, warptile_mmq_m, m_align); + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_dequant_mul_mat_mat[GGML_TYPE_Q4_0]->a_s, "matmul_q4_0_f32_aligned_s", matmul_q4_0_f32_aligned_fp32_len, matmul_q4_0_f32_aligned_fp32_data, "main", 3, 14 * sizeof(uint32_t), s_wg_denoms, warptile_mmq_s, s_align); + + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_dequant_mul_mat_mat[GGML_TYPE_Q4_1]->l, "matmul_q4_1_f32_l", matmul_q4_1_f32_fp32_len, matmul_q4_1_f32_fp32_data, "main", 3, 14 * sizeof(uint32_t), l_wg_denoms, warptile_mmq_l, l_align); + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_dequant_mul_mat_mat[GGML_TYPE_Q4_1]->m, "matmul_q4_1_f32_m", matmul_q4_1_f32_fp32_len, matmul_q4_1_f32_fp32_data, "main", 3, 14 * sizeof(uint32_t), m_wg_denoms, warptile_mmq_m, m_align); + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_dequant_mul_mat_mat[GGML_TYPE_Q4_1]->s, "matmul_q4_1_f32_s", matmul_q4_1_f32_fp32_len, matmul_q4_1_f32_fp32_data, "main", 3, 14 * sizeof(uint32_t), s_wg_denoms, warptile_mmq_s, s_align); + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_dequant_mul_mat_mat[GGML_TYPE_Q4_1]->a_l, "matmul_q4_1_f32_aligned_l", matmul_q4_1_f32_aligned_fp32_len, matmul_q4_1_f32_aligned_fp32_data, "main", 3, 14 * sizeof(uint32_t), l_wg_denoms, warptile_mmq_l, l_align); + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_dequant_mul_mat_mat[GGML_TYPE_Q4_1]->a_m, "matmul_q4_1_f32_aligned_m", matmul_q4_1_f32_aligned_fp32_len, matmul_q4_1_f32_aligned_fp32_data, "main", 3, 14 * sizeof(uint32_t), m_wg_denoms, warptile_mmq_m, m_align); + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_dequant_mul_mat_mat[GGML_TYPE_Q4_1]->a_s, "matmul_q4_1_f32_aligned_s", matmul_q4_1_f32_aligned_fp32_len, matmul_q4_1_f32_aligned_fp32_data, "main", 3, 14 * sizeof(uint32_t), s_wg_denoms, warptile_mmq_s, s_align); + + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_dequant_mul_mat_mat[GGML_TYPE_Q5_0]->l, "matmul_q5_0_f32_l", matmul_q5_0_f32_fp32_len, matmul_q5_0_f32_fp32_data, "main", 3, 14 * sizeof(uint32_t), l_wg_denoms, warptile_mmq_l, l_align); + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_dequant_mul_mat_mat[GGML_TYPE_Q5_0]->m, "matmul_q5_0_f32_m", matmul_q5_0_f32_fp32_len, matmul_q5_0_f32_fp32_data, "main", 3, 14 * sizeof(uint32_t), m_wg_denoms, warptile_mmq_m, m_align); + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_dequant_mul_mat_mat[GGML_TYPE_Q5_0]->s, "matmul_q5_0_f32_s", matmul_q5_0_f32_fp32_len, matmul_q5_0_f32_fp32_data, "main", 3, 14 * sizeof(uint32_t), s_wg_denoms, warptile_mmq_s, s_align); + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_dequant_mul_mat_mat[GGML_TYPE_Q5_0]->a_l, "matmul_q5_0_f32_aligned_l", matmul_q5_0_f32_aligned_fp32_len, matmul_q5_0_f32_aligned_fp32_data, "main", 3, 14 * sizeof(uint32_t), l_wg_denoms, warptile_mmq_l, l_align); + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_dequant_mul_mat_mat[GGML_TYPE_Q5_0]->a_m, "matmul_q5_0_f32_aligned_m", matmul_q5_0_f32_aligned_fp32_len, matmul_q5_0_f32_aligned_fp32_data, "main", 3, 14 * sizeof(uint32_t), m_wg_denoms, warptile_mmq_m, m_align); + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_dequant_mul_mat_mat[GGML_TYPE_Q5_0]->a_s, "matmul_q5_0_f32_aligned_s", matmul_q5_0_f32_aligned_fp32_len, matmul_q5_0_f32_aligned_fp32_data, "main", 3, 14 * sizeof(uint32_t), s_wg_denoms, warptile_mmq_s, s_align); + + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_dequant_mul_mat_mat[GGML_TYPE_Q5_1]->l, "matmul_q5_1_f32_l", matmul_q5_1_f32_fp32_len, matmul_q5_1_f32_fp32_data, "main", 3, 14 * sizeof(uint32_t), l_wg_denoms, warptile_mmq_l, l_align); + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_dequant_mul_mat_mat[GGML_TYPE_Q5_1]->m, "matmul_q5_1_f32_m", matmul_q5_1_f32_fp32_len, matmul_q5_1_f32_fp32_data, "main", 3, 14 * sizeof(uint32_t), m_wg_denoms, warptile_mmq_m, m_align); + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_dequant_mul_mat_mat[GGML_TYPE_Q5_1]->s, "matmul_q5_1_f32_s", matmul_q5_1_f32_fp32_len, matmul_q5_1_f32_fp32_data, "main", 3, 14 * sizeof(uint32_t), s_wg_denoms, warptile_mmq_s, s_align); + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_dequant_mul_mat_mat[GGML_TYPE_Q5_1]->a_l, "matmul_q5_1_f32_aligned_l", matmul_q5_1_f32_aligned_fp32_len, matmul_q5_1_f32_aligned_fp32_data, "main", 3, 14 * sizeof(uint32_t), l_wg_denoms, warptile_mmq_l, l_align); + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_dequant_mul_mat_mat[GGML_TYPE_Q5_1]->a_m, "matmul_q5_1_f32_aligned_m", matmul_q5_1_f32_aligned_fp32_len, matmul_q5_1_f32_aligned_fp32_data, "main", 3, 14 * sizeof(uint32_t), m_wg_denoms, warptile_mmq_m, m_align); + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_dequant_mul_mat_mat[GGML_TYPE_Q5_1]->a_s, "matmul_q5_1_f32_aligned_s", matmul_q5_1_f32_aligned_fp32_len, matmul_q5_1_f32_aligned_fp32_data, "main", 3, 14 * sizeof(uint32_t), s_wg_denoms, warptile_mmq_s, s_align); + + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_dequant_mul_mat_mat[GGML_TYPE_Q8_0]->l, "matmul_q8_0_f32_l", matmul_q8_0_f32_fp32_len, matmul_q8_0_f32_fp32_data, "main", 3, 14 * sizeof(uint32_t), l_wg_denoms, warptile_mmq_l, l_align); + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_dequant_mul_mat_mat[GGML_TYPE_Q8_0]->m, "matmul_q8_0_f32_m", matmul_q8_0_f32_fp32_len, matmul_q8_0_f32_fp32_data, "main", 3, 14 * sizeof(uint32_t), m_wg_denoms, warptile_mmq_m, m_align); + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_dequant_mul_mat_mat[GGML_TYPE_Q8_0]->s, "matmul_q8_0_f32_s", matmul_q8_0_f32_fp32_len, matmul_q8_0_f32_fp32_data, "main", 3, 14 * sizeof(uint32_t), s_wg_denoms, warptile_mmq_s, s_align); + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_dequant_mul_mat_mat[GGML_TYPE_Q8_0]->a_l, "matmul_q8_0_f32_aligned_l", matmul_q8_0_f32_aligned_fp32_len, matmul_q8_0_f32_aligned_fp32_data, "main", 3, 14 * sizeof(uint32_t), l_wg_denoms, warptile_mmq_l, l_align); + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_dequant_mul_mat_mat[GGML_TYPE_Q8_0]->a_m, "matmul_q8_0_f32_aligned_m", matmul_q8_0_f32_aligned_fp32_len, matmul_q8_0_f32_aligned_fp32_data, "main", 3, 14 * sizeof(uint32_t), m_wg_denoms, warptile_mmq_m, m_align); + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_dequant_mul_mat_mat[GGML_TYPE_Q8_0]->a_s, "matmul_q8_0_f32_aligned_s", matmul_q8_0_f32_aligned_fp32_len, matmul_q8_0_f32_aligned_fp32_data, "main", 3, 14 * sizeof(uint32_t), s_wg_denoms, warptile_mmq_s, s_align); } - ggml_vk_create_pipeline(ctx, ctx->pipeline_dequant_mul_mat_vec_f32[GGML_TYPE_F16 ], "mul_mat_vec_f16_f32", mul_mat_vec_f16_f32_len, mul_mat_vec_f16_f32_data, "main", 3, 3 * sizeof(int), {1, 1, 1}, {}, 1); - ggml_vk_create_pipeline(ctx, ctx->pipeline_dequant_mul_mat_vec_f32[GGML_TYPE_Q4_0], "mul_mat_vec_q4_0_f32", mul_mat_vec_q4_0_f32_len, mul_mat_vec_q4_0_f32_data, "main", 3, 3 * sizeof(int), {1, 1, 1}, {}, 1); - ggml_vk_create_pipeline(ctx, ctx->pipeline_dequant_mul_mat_vec_f32[GGML_TYPE_Q4_1], "mul_mat_vec_q4_1_f32", mul_mat_vec_q4_1_f32_len, mul_mat_vec_q4_1_f32_data, "main", 3, 3 * sizeof(int), {1, 1, 1}, {}, 1); - ggml_vk_create_pipeline(ctx, ctx->pipeline_dequant_mul_mat_vec_f32[GGML_TYPE_Q5_0], "mul_mat_vec_q5_0_f32", mul_mat_vec_q5_0_f32_len, mul_mat_vec_q5_0_f32_data, "main", 3, 3 * sizeof(int), {1, 1, 1}, {}, 1); - ggml_vk_create_pipeline(ctx, ctx->pipeline_dequant_mul_mat_vec_f32[GGML_TYPE_Q5_1], "mul_mat_vec_q5_1_f32", mul_mat_vec_q5_1_f32_len, mul_mat_vec_q5_1_f32_data, "main", 3, 3 * sizeof(int), {1, 1, 1}, {}, 1); - ggml_vk_create_pipeline(ctx, ctx->pipeline_dequant_mul_mat_vec_f32[GGML_TYPE_Q8_0], "mul_mat_vec_q8_0_f32", mul_mat_vec_q8_0_f32_len, mul_mat_vec_q8_0_f32_data, "main", 3, 3 * sizeof(int), {1, 1, 1}, {}, 1); - ggml_vk_create_pipeline(ctx, ctx->pipeline_dequant_mul_mat_vec_f32[GGML_TYPE_Q2_K], "mul_mat_vec_q2_K_f32", mul_mat_vec_q2_K_f32_len, mul_mat_vec_q2_K_f32_data, "main", 3, 3 * sizeof(int), {1, 1, 1}, {}, 1); - ggml_vk_create_pipeline(ctx, ctx->pipeline_dequant_mul_mat_vec_f32[GGML_TYPE_Q3_K], "mul_mat_vec_q3_K_f32", mul_mat_vec_q3_K_f32_len, mul_mat_vec_q3_K_f32_data, "main", 3, 3 * sizeof(int), {1, 1, 1}, {}, 1); - ggml_vk_create_pipeline(ctx, ctx->pipeline_dequant_mul_mat_vec_f32[GGML_TYPE_Q4_K], "mul_mat_vec_q4_K_f32", mul_mat_vec_q4_K_f32_len, mul_mat_vec_q4_K_f32_data, "main", 3, 3 * sizeof(int), {1, 1, 1}, {}, 1); - ggml_vk_create_pipeline(ctx, ctx->pipeline_dequant_mul_mat_vec_f32[GGML_TYPE_Q5_K], "mul_mat_vec_q5_K_f32", mul_mat_vec_q5_K_f32_len, mul_mat_vec_q5_K_f32_data, "main", 3, 3 * sizeof(int), {1, 1, 1}, {}, 1); - ggml_vk_create_pipeline(ctx, ctx->pipeline_dequant_mul_mat_vec_f32[GGML_TYPE_Q6_K], "mul_mat_vec_q6_K_f32", mul_mat_vec_q6_K_f32_len, mul_mat_vec_q6_K_f32_data, "main", 3, 3 * sizeof(int), {1, 1, 1}, {}, 1); + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_dequant_mul_mat_vec_f32[GGML_TYPE_F16 ], "mul_mat_vec_f16_f32", mul_mat_vec_f16_f32_len, mul_mat_vec_f16_f32_data, "main", 3, 3 * sizeof(uint32_t), {1, 1, 1}, { device->subgroup_size }, 1); + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_dequant_mul_mat_vec_f32[GGML_TYPE_Q4_0], "mul_mat_vec_q4_0_f32", mul_mat_vec_q4_0_f32_len, mul_mat_vec_q4_0_f32_data, "main", 3, 3 * sizeof(uint32_t), {1, 1, 1}, { device->subgroup_size }, 1); + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_dequant_mul_mat_vec_f32[GGML_TYPE_Q4_1], "mul_mat_vec_q4_1_f32", mul_mat_vec_q4_1_f32_len, mul_mat_vec_q4_1_f32_data, "main", 3, 3 * sizeof(uint32_t), {1, 1, 1}, { device->subgroup_size }, 1); + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_dequant_mul_mat_vec_f32[GGML_TYPE_Q5_0], "mul_mat_vec_q5_0_f32", mul_mat_vec_q5_0_f32_len, mul_mat_vec_q5_0_f32_data, "main", 3, 3 * sizeof(uint32_t), {1, 1, 1}, { device->subgroup_size }, 1); + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_dequant_mul_mat_vec_f32[GGML_TYPE_Q5_1], "mul_mat_vec_q5_1_f32", mul_mat_vec_q5_1_f32_len, mul_mat_vec_q5_1_f32_data, "main", 3, 3 * sizeof(uint32_t), {1, 1, 1}, { device->subgroup_size }, 1); + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_dequant_mul_mat_vec_f32[GGML_TYPE_Q8_0], "mul_mat_vec_q8_0_f32", mul_mat_vec_q8_0_f32_len, mul_mat_vec_q8_0_f32_data, "main", 3, 3 * sizeof(uint32_t), {1, 1, 1}, { device->subgroup_size }, 1); + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_dequant_mul_mat_vec_f32[GGML_TYPE_Q2_K], "mul_mat_vec_q2_K_f32", mul_mat_vec_q2_K_f32_len, mul_mat_vec_q2_K_f32_data, "main", 3, 3 * sizeof(uint32_t), {1, 1, 1}, { device->subgroup_size }, 1); + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_dequant_mul_mat_vec_f32[GGML_TYPE_Q3_K], "mul_mat_vec_q3_K_f32", mul_mat_vec_q3_K_f32_len, mul_mat_vec_q3_K_f32_data, "main", 3, 3 * sizeof(uint32_t), {1, 1, 1}, { device->subgroup_size }, 1); + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_dequant_mul_mat_vec_f32[GGML_TYPE_Q4_K], "mul_mat_vec_q4_K_f32", mul_mat_vec_q4_K_f32_len, mul_mat_vec_q4_K_f32_data, "main", 3, 3 * sizeof(uint32_t), {1, 1, 1}, { device->subgroup_size }, 1); + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_dequant_mul_mat_vec_f32[GGML_TYPE_Q5_K], "mul_mat_vec_q5_K_f32", mul_mat_vec_q5_K_f32_len, mul_mat_vec_q5_K_f32_data, "main", 3, 3 * sizeof(uint32_t), {1, 1, 1}, { device->subgroup_size }, 1); + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_dequant_mul_mat_vec_f32[GGML_TYPE_Q6_K], "mul_mat_vec_q6_K_f32", mul_mat_vec_q6_K_f32_len, mul_mat_vec_q6_K_f32_data, "main", 3, 3 * sizeof(uint32_t), {1, 1, 1}, { device->subgroup_size }, 1); // dequant shaders - ggml_vk_create_pipeline(ctx, ctx->pipeline_dequant[GGML_TYPE_F32 ], "f32_to_f16", f32_to_f16_len, f32_to_f16_data, "main", 2, 4 * sizeof(int), { 64, 1, 1}, {}, 1); - ggml_vk_create_pipeline(ctx, ctx->pipeline_dequant[GGML_TYPE_F16 ], "dequant_f16", dequant_f16_len, dequant_f16_data, "main", 2, 4 * sizeof(int), {256 * 32, 1, 1}, {}, 1); - ggml_vk_create_pipeline(ctx, ctx->pipeline_dequant[GGML_TYPE_Q4_0], "dequant_q4_0", dequant_q4_0_len, dequant_q4_0_data, "main", 2, 4 * sizeof(int), {256 * 32, 1, 1}, {}, 1); - ggml_vk_create_pipeline(ctx, ctx->pipeline_dequant[GGML_TYPE_Q4_1], "dequant_q4_1", dequant_q4_1_len, dequant_q4_1_data, "main", 2, 4 * sizeof(int), {256 * 32, 1, 1}, {}, 1); - ggml_vk_create_pipeline(ctx, ctx->pipeline_dequant[GGML_TYPE_Q5_0], "dequant_q5_0", dequant_q5_0_len, dequant_q5_0_data, "main", 2, 4 * sizeof(int), {256 * 32, 1, 1}, {}, 1); - ggml_vk_create_pipeline(ctx, ctx->pipeline_dequant[GGML_TYPE_Q5_1], "dequant_q5_1", dequant_q5_1_len, dequant_q5_1_data, "main", 2, 4 * sizeof(int), {256 * 32, 1, 1}, {}, 1); - ggml_vk_create_pipeline(ctx, ctx->pipeline_dequant[GGML_TYPE_Q8_0], "dequant_q8_0", dequant_q8_0_len, dequant_q8_0_data, "main", 2, 4 * sizeof(int), {256 * 32, 1, 1}, {}, 1); - ggml_vk_create_pipeline(ctx, ctx->pipeline_dequant[GGML_TYPE_Q2_K], "dequant_q2_K", dequant_q2_K_len, dequant_q2_K_data, "main", 2, 4 * sizeof(int), {256 * 64, 1, 1}, {}, 1); - ggml_vk_create_pipeline(ctx, ctx->pipeline_dequant[GGML_TYPE_Q3_K], "dequant_q3_K", dequant_q3_K_len, dequant_q3_K_data, "main", 2, 4 * sizeof(int), {256 * 64, 1, 1}, {}, 1); - ggml_vk_create_pipeline(ctx, ctx->pipeline_dequant[GGML_TYPE_Q4_K], "dequant_q4_K", dequant_q4_K_len, dequant_q4_K_data, "main", 2, 4 * sizeof(int), {256 * 32, 1, 1}, {}, 1); - ggml_vk_create_pipeline(ctx, ctx->pipeline_dequant[GGML_TYPE_Q5_K], "dequant_q5_K", dequant_q5_K_len, dequant_q5_K_data, "main", 2, 4 * sizeof(int), {256 * 64, 1, 1}, {}, 1); - ggml_vk_create_pipeline(ctx, ctx->pipeline_dequant[GGML_TYPE_Q6_K], "dequant_q6_K", dequant_q6_K_len, dequant_q6_K_data, "main", 2, 4 * sizeof(int), {256 * 64, 1, 1}, {}, 1); + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_dequant[GGML_TYPE_F32 ], "f32_to_f16", dequant_f32_len, dequant_f32_data, "main", 2, 5 * sizeof(uint32_t), {256 * 16, 1, 1}, {}, 1); + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_dequant[GGML_TYPE_Q4_0], "dequant_q4_0", dequant_q4_0_len, dequant_q4_0_data, "main", 2, 5 * sizeof(uint32_t), {256 * 16, 1, 1}, {}, 1); + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_dequant[GGML_TYPE_Q4_1], "dequant_q4_1", dequant_q4_1_len, dequant_q4_1_data, "main", 2, 5 * sizeof(uint32_t), {256 * 16, 1, 1}, {}, 1); + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_dequant[GGML_TYPE_Q5_0], "dequant_q5_0", dequant_q5_0_len, dequant_q5_0_data, "main", 2, 5 * sizeof(uint32_t), {256 * 16, 1, 1}, {}, 1); + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_dequant[GGML_TYPE_Q5_1], "dequant_q5_1", dequant_q5_1_len, dequant_q5_1_data, "main", 2, 5 * sizeof(uint32_t), {256 * 16, 1, 1}, {}, 1); + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_dequant[GGML_TYPE_Q8_0], "dequant_q8_0", dequant_q8_0_len, dequant_q8_0_data, "main", 2, 5 * sizeof(uint32_t), {256 * 16, 1, 1}, {}, 1); + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_dequant[GGML_TYPE_Q2_K], "dequant_q2_K", dequant_q2_K_len, dequant_q2_K_data, "main", 2, 5 * sizeof(uint32_t), {256 * 64, 1, 1}, {}, 1); + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_dequant[GGML_TYPE_Q3_K], "dequant_q3_K", dequant_q3_K_len, dequant_q3_K_data, "main", 2, 5 * sizeof(uint32_t), {256 * 64, 1, 1}, {}, 1); + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_dequant[GGML_TYPE_Q4_K], "dequant_q4_K", dequant_q4_K_len, dequant_q4_K_data, "main", 2, 5 * sizeof(uint32_t), {256 * 32, 1, 1}, {}, 1); + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_dequant[GGML_TYPE_Q5_K], "dequant_q5_K", dequant_q5_K_len, dequant_q5_K_data, "main", 2, 5 * sizeof(uint32_t), {256 * 64, 1, 1}, {}, 1); + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_dequant[GGML_TYPE_Q6_K], "dequant_q6_K", dequant_q6_K_len, dequant_q6_K_data, "main", 2, 5 * sizeof(uint32_t), {256 * 64, 1, 1}, {}, 1); // get_rows - ggml_vk_create_pipeline(ctx, ctx->pipeline_get_rows[GGML_TYPE_F16 ], "get_rows_f16", get_rows_f16_len, get_rows_f16_data, "main", 3, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1); - ggml_vk_create_pipeline(ctx, ctx->pipeline_get_rows[GGML_TYPE_Q4_0], "get_rows_q4_0", get_rows_q4_0_len, get_rows_q4_0_data, "main", 3, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1); - ggml_vk_create_pipeline(ctx, ctx->pipeline_get_rows[GGML_TYPE_Q4_1], "get_rows_q4_1", get_rows_q4_1_len, get_rows_q4_1_data, "main", 3, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1); - ggml_vk_create_pipeline(ctx, ctx->pipeline_get_rows[GGML_TYPE_Q5_0], "get_rows_q5_0", get_rows_q5_0_len, get_rows_q5_0_data, "main", 3, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1); - ggml_vk_create_pipeline(ctx, ctx->pipeline_get_rows[GGML_TYPE_Q5_1], "get_rows_q5_1", get_rows_q5_1_len, get_rows_q5_1_data, "main", 3, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1); - ggml_vk_create_pipeline(ctx, ctx->pipeline_get_rows[GGML_TYPE_Q8_0], "get_rows_q8_0", get_rows_q8_0_len, get_rows_q8_0_data, "main", 3, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1); + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_get_rows[GGML_TYPE_F16 ], "get_rows_f16", get_rows_f16_len, get_rows_f16_data, "main", 3, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1); + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_get_rows[GGML_TYPE_Q4_0], "get_rows_q4_0", get_rows_q4_0_len, get_rows_q4_0_data, "main", 3, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1); + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_get_rows[GGML_TYPE_Q4_1], "get_rows_q4_1", get_rows_q4_1_len, get_rows_q4_1_data, "main", 3, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1); + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_get_rows[GGML_TYPE_Q5_0], "get_rows_q5_0", get_rows_q5_0_len, get_rows_q5_0_data, "main", 3, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1); + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_get_rows[GGML_TYPE_Q5_1], "get_rows_q5_1", get_rows_q5_1_len, get_rows_q5_1_data, "main", 3, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1); + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_get_rows[GGML_TYPE_Q8_0], "get_rows_q8_0", get_rows_q8_0_len, get_rows_q8_0_data, "main", 3, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1); - ggml_vk_create_pipeline(ctx, ctx->pipeline_get_rows_f32[GGML_TYPE_F32 ], "get_rows_f16_f32", get_rows_f16_f32_len, get_rows_f16_f32_data, "main", 3, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1); - ggml_vk_create_pipeline(ctx, ctx->pipeline_get_rows_f32[GGML_TYPE_Q4_0], "get_rows_q4_0_f32", get_rows_q4_0_f32_len, get_rows_q4_0_f32_data, "main", 3, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1); - ggml_vk_create_pipeline(ctx, ctx->pipeline_get_rows_f32[GGML_TYPE_Q4_1], "get_rows_q4_1_f32", get_rows_q4_1_f32_len, get_rows_q4_1_f32_data, "main", 3, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1); - ggml_vk_create_pipeline(ctx, ctx->pipeline_get_rows_f32[GGML_TYPE_Q5_0], "get_rows_q5_0_f32", get_rows_q5_0_f32_len, get_rows_q5_0_f32_data, "main", 3, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1); - ggml_vk_create_pipeline(ctx, ctx->pipeline_get_rows_f32[GGML_TYPE_Q5_1], "get_rows_q5_1_f32", get_rows_q5_1_f32_len, get_rows_q5_1_f32_data, "main", 3, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1); - ggml_vk_create_pipeline(ctx, ctx->pipeline_get_rows_f32[GGML_TYPE_Q8_0], "get_rows_q8_0_f32", get_rows_q8_0_f32_len, get_rows_q8_0_f32_data, "main", 3, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1); + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_get_rows_f32[GGML_TYPE_F32 ], "get_rows_f16_f32", get_rows_f16_f32_len, get_rows_f16_f32_data, "main", 3, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1); + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_get_rows_f32[GGML_TYPE_Q4_0], "get_rows_q4_0_f32", get_rows_q4_0_f32_len, get_rows_q4_0_f32_data, "main", 3, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1); + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_get_rows_f32[GGML_TYPE_Q4_1], "get_rows_q4_1_f32", get_rows_q4_1_f32_len, get_rows_q4_1_f32_data, "main", 3, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1); + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_get_rows_f32[GGML_TYPE_Q5_0], "get_rows_q5_0_f32", get_rows_q5_0_f32_len, get_rows_q5_0_f32_data, "main", 3, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1); + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_get_rows_f32[GGML_TYPE_Q5_1], "get_rows_q5_1_f32", get_rows_q5_1_f32_len, get_rows_q5_1_f32_data, "main", 3, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1); + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_get_rows_f32[GGML_TYPE_Q8_0], "get_rows_q8_0_f32", get_rows_q8_0_f32_len, get_rows_q8_0_f32_data, "main", 3, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1); - ggml_vk_create_pipeline(ctx, ctx->pipeline_matmul_split_k_reduce, "split_k_reduce", split_k_reduce_len, split_k_reduce_data, "main", 2, 2 * sizeof(uint32_t), {256, 1, 1}, {}, 1); + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_matmul_split_k_reduce, "split_k_reduce", split_k_reduce_len, split_k_reduce_data, "main", 2, 2 * sizeof(uint32_t), {256, 1, 1}, {}, 1); - ggml_vk_create_pipeline(ctx, ctx->pipeline_mul_mat_vec_p021_f16_f32, "mul_mat_vec_p021_f16_f32", mul_mat_vec_p021_f16_f32_len, mul_mat_vec_p021_f16_f32_data, "main", 3, 6 * sizeof(uint32_t), {1, 1, 1}, {}, 1); - ggml_vk_create_pipeline(ctx, ctx->pipeline_mul_mat_vec_nc_f16_f32, "mul_mat_vec_nc_f16_f32", mul_mat_vec_nc_f16_f32_len, mul_mat_vec_nc_f16_f32_data, "main", 3, 7 * sizeof(uint32_t), {1, 1, 1}, {}, 1); + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_mul_mat_vec_p021_f16_f32, "mul_mat_vec_p021_f16_f32", mul_mat_vec_p021_f16_f32_len, mul_mat_vec_p021_f16_f32_data, "main", 3, 6 * sizeof(uint32_t), {1, 1, 1}, {}, 1); + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_mul_mat_vec_nc_f16_f32, "mul_mat_vec_nc_f16_f32", mul_mat_vec_nc_f16_f32_len, mul_mat_vec_nc_f16_f32_data, "main", 3, 7 * sizeof(uint32_t), {1, 1, 1}, {}, 1); - ggml_vk_create_pipeline(ctx, ctx->pipeline_norm_f32, "norm_f32", norm_f32_len, norm_f32_data, "main", 2, sizeof(vk_op_push_constants), {1, 1, 1}, {}, 1); - ggml_vk_create_pipeline(ctx, ctx->pipeline_rms_norm_f32, "rms_norm_f32", rms_norm_f32_len, rms_norm_f32_data, "main", 2, sizeof(vk_op_push_constants), {1, 1, 1}, {}, 1); + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_norm_f32, "norm_f32", norm_f32_len, norm_f32_data, "main", 2, sizeof(vk_op_push_constants), {1, 1, 1}, {}, 1); + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_rms_norm_f32, "rms_norm_f32", rms_norm_f32_len, rms_norm_f32_data, "main", 2, sizeof(vk_op_push_constants), {1, 1, 1}, {}, 1); - ggml_vk_create_pipeline(ctx, ctx->pipeline_cpy_f32_f32, "cpy_f32_f32", cpy_f32_f32_len, cpy_f32_f32_data, "main", 2, sizeof(vk_op_cpy_push_constants), {512, 1, 1}, {}, 1); - ggml_vk_create_pipeline(ctx, ctx->pipeline_cpy_f32_f16, "cpy_f32_f16", cpy_f32_f16_len, cpy_f32_f16_data, "main", 2, sizeof(vk_op_cpy_push_constants), {512, 1, 1}, {}, 1); - ggml_vk_create_pipeline(ctx, ctx->pipeline_cpy_f16_f16, "cpy_f16_f16", cpy_f16_f16_len, cpy_f16_f16_data, "main", 2, sizeof(vk_op_cpy_push_constants), {512, 1, 1}, {}, 1); + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_cpy_f32_f32, "cpy_f32_f32", cpy_f32_f32_len, cpy_f32_f32_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1); + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_cpy_f32_f16, "cpy_f32_f16", cpy_f32_f16_len, cpy_f32_f16_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1); + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_cpy_f16_f16, "cpy_f16_f16", cpy_f16_f16_len, cpy_f16_f16_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1); - ggml_vk_create_pipeline(ctx, ctx->pipeline_add_f32, "add_f32", add_f32_len, add_f32_data, "main", 3, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1); + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_add_f32, "add_f32", add_f32_len, add_f32_data, "main", 3, sizeof(vk_op_binary_push_constants), {512, 1, 1}, {}, 1); - ggml_vk_create_pipeline(ctx, ctx->pipeline_mul_f32, "mul_f32", mul_f32_len, mul_f32_data, "main", 3, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1); + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_mul_f32, "mul_f32", mul_f32_len, mul_f32_data, "main", 3, sizeof(vk_op_binary_push_constants), {512, 1, 1}, {}, 1); - ggml_vk_create_pipeline(ctx, ctx->pipeline_scale_f32, "scale_f32", scale_f32_len, scale_f32_data, "main", 2, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1); + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_scale_f32, "scale_f32", scale_f32_len, scale_f32_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1); - ggml_vk_create_pipeline(ctx, ctx->pipeline_sqr_f32, "sqr_f32", sqr_f32_len, sqr_f32_data, "main", 2, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1); + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_sqr_f32, "sqr_f32", sqr_f32_len, sqr_f32_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1); - ggml_vk_create_pipeline(ctx, ctx->pipeline_clamp_f32, "clamp_f32", clamp_f32_len, clamp_f32_data, "main", 2, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1); + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_clamp_f32, "clamp_f32", clamp_f32_len, clamp_f32_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1); - ggml_vk_create_pipeline(ctx, ctx->pipeline_gelu_f32, "gelu_f32", gelu_f32_len, gelu_f32_data, "main", 2, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1); - ggml_vk_create_pipeline(ctx, ctx->pipeline_silu_f32, "silu_f32", silu_f32_len, silu_f32_data, "main", 2, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1); - ggml_vk_create_pipeline(ctx, ctx->pipeline_relu_f32, "relu_f32", relu_f32_len, relu_f32_data, "main", 2, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1); + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_gelu_f32, "gelu_f32", gelu_f32_len, gelu_f32_data, "main", 2, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1); + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_silu_f32, "silu_f32", silu_f32_len, silu_f32_data, "main", 2, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1); + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_relu_f32, "relu_f32", relu_f32_len, relu_f32_data, "main", 2, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1); - ggml_vk_create_pipeline(ctx, ctx->pipeline_diag_mask_inf_f32, "diag_mask_inf_f32", diag_mask_inf_f32_len, diag_mask_inf_f32_data, "main", 2, sizeof(vk_op_diag_mask_push_constants), {512, 1, 1}, {}, 1); + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_diag_mask_inf_f32, "diag_mask_inf_f32", diag_mask_inf_f32_len, diag_mask_inf_f32_data, "main", 2, sizeof(vk_op_diag_mask_push_constants), {512, 1, 1}, {}, 1); - ggml_vk_create_pipeline(ctx, ctx->pipeline_soft_max_f32, "soft_max_f32", soft_max_f32_len, soft_max_f32_data, "main", 3, sizeof(vk_op_push_constants), {1, 1, 1}, {}, 1); + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_soft_max_f32, "soft_max_f32", soft_max_f32_len, soft_max_f32_data, "main", 4, sizeof(vk_op_soft_max_push_constants), {1, 1, 1}, {}, 1); - ggml_vk_create_pipeline(ctx, ctx->pipeline_rope_f32, "rope_f32", rope_f32_len, rope_f32_data, "main", 3, sizeof(vk_op_rope_push_constants), {1, 512, 1}, {}, 1); - ggml_vk_create_pipeline(ctx, ctx->pipeline_rope_f16, "rope_f16", rope_f16_len, rope_f16_data, "main", 3, sizeof(vk_op_rope_push_constants), {1, 512, 1}, {}, 1); + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_rope_f32, "rope_f32", rope_f32_len, rope_f32_data, "main", 3, sizeof(vk_op_rope_push_constants), {1, 512, 1}, {}, 1); + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_rope_f16, "rope_f16", rope_f16_len, rope_f16_data, "main", 3, sizeof(vk_op_rope_push_constants), {1, 512, 1}, {}, 1); - ggml_vk_create_pipeline(ctx, ctx->pipeline_rope_neox_f32, "rope_neox_f32", rope_neox_f32_len, rope_neox_f32_data, "main", 3, sizeof(vk_op_rope_neox_push_constants), {1, 512, 1}, {}, 1); - ggml_vk_create_pipeline(ctx, ctx->pipeline_rope_neox_f16, "rope_neox_f16", rope_neox_f16_len, rope_neox_f16_data, "main", 3, sizeof(vk_op_rope_neox_push_constants), {1, 512, 1}, {}, 1); + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_rope_neox_f32, "rope_neox_f32", rope_neox_f32_len, rope_neox_f32_data, "main", 3, sizeof(vk_op_rope_neox_push_constants), {1, 512, 1}, {}, 1); + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_rope_neox_f16, "rope_neox_f16", rope_neox_f16_len, rope_neox_f16_data, "main", 3, sizeof(vk_op_rope_neox_push_constants), {1, 512, 1}, {}, 1); + + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_argsort_f32, "argsort_f32", argsort_f32_len, argsort_f32_data, "main", 2, sizeof(vk_op_argsort_push_constants), {1024, 1, 1}, {}, 1); } static void ggml_vk_print_gpu_info(size_t idx) { @@ -1057,8 +1225,8 @@ static void ggml_vk_print_gpu_info(size_t idx) { } } - const char* GGML_VULKAN_DISABLE_F16 = getenv("GGML_VULKAN_DISABLE_F16"); - bool force_disable_f16 = GGML_VULKAN_DISABLE_F16 != nullptr; + const char* GGML_VK_DISABLE_F16 = getenv("GGML_VK_DISABLE_F16"); + bool force_disable_f16 = GGML_VK_DISABLE_F16 != nullptr; bool fp16 = !force_disable_f16 && fp16_storage && fp16_compute; @@ -1188,140 +1356,152 @@ static void ggml_vk_init(ggml_backend_vk_context * ctx, size_t idx) { throw std::runtime_error("Device not found"); } - vk_instance.devices[idx] = std::make_shared(); - ctx->device = vk_instance.devices[idx]; - ctx->device.lock()->physical_device = devices[dev_num]; - const std::vector ext_props = ctx->device.lock()->physical_device.enumerateDeviceExtensionProperties(); + ctx->device = ggml_vk_get_device(idx); + if (!ctx->device->initialized) { + ctx->device->physical_device = devices[dev_num]; + const std::vector ext_props = ctx->device->physical_device.enumerateDeviceExtensionProperties(); - bool maintenance4_support = false; + bool maintenance4_support = false; - // Check if maintenance4 is supported - for (const auto& properties : ext_props) { - if (strcmp("VK_KHR_maintenance4", properties.extensionName) == 0) { - maintenance4_support = true; + // Check if maintenance4 is supported + for (const auto& properties : ext_props) { + if (strcmp("VK_KHR_maintenance4", properties.extensionName) == 0) { + maintenance4_support = true; + } } - } - vk::PhysicalDeviceProperties2 props2; - vk::PhysicalDeviceMaintenance3Properties props3; - vk::PhysicalDeviceMaintenance4Properties props4; - vk::PhysicalDeviceSubgroupProperties subgroup_props; - props2.pNext = &props3; - props3.pNext = &subgroup_props; - if (maintenance4_support) { - subgroup_props.pNext = &props4; - } - ctx->device.lock()->physical_device.getProperties2(&props2); - ctx->device.lock()->properties = props2.properties; - - if (maintenance4_support) { - ctx->device.lock()->max_memory_allocation_size = std::min(props3.maxMemoryAllocationSize, props4.maxBufferSize); - } else { - ctx->device.lock()->max_memory_allocation_size = props3.maxMemoryAllocationSize; - } - - ctx->device.lock()->vendor_id = ctx->device.lock()->properties.vendorID; - ctx->device.lock()->subgroup_size = subgroup_props.subgroupSize; - ctx->device.lock()->uma = ctx->device.lock()->properties.deviceType == vk::PhysicalDeviceType::eIntegratedGpu; - - bool fp16_storage = false; - bool fp16_compute = false; - - for (const auto& properties : ext_props) { - if (strcmp("VK_KHR_16bit_storage", properties.extensionName) == 0) { - fp16_storage = true; - } else if (strcmp("VK_KHR_shader_float16_int8", properties.extensionName) == 0) { - fp16_compute = true; + vk::PhysicalDeviceProperties2 props2; + vk::PhysicalDeviceMaintenance3Properties props3; + vk::PhysicalDeviceMaintenance4Properties props4; + vk::PhysicalDeviceSubgroupProperties subgroup_props; + props2.pNext = &props3; + props3.pNext = &subgroup_props; + if (maintenance4_support) { + subgroup_props.pNext = &props4; } - } + ctx->device->physical_device.getProperties2(&props2); + ctx->device->properties = props2.properties; - const char* GGML_VULKAN_DISABLE_F16 = getenv("GGML_VULKAN_DISABLE_F16"); - bool force_disable_f16 = GGML_VULKAN_DISABLE_F16 != nullptr; + const char* GGML_VK_FORCE_MAX_ALLOCATION_SIZE = getenv("GGML_VK_FORCE_MAX_ALLOCATION_SIZE"); - ctx->device.lock()->fp16 = !force_disable_f16 && fp16_storage && fp16_compute; + if (GGML_VK_FORCE_MAX_ALLOCATION_SIZE != nullptr) { + ctx->device->max_memory_allocation_size = std::stoi(GGML_VK_FORCE_MAX_ALLOCATION_SIZE); + } else if (maintenance4_support) { + ctx->device->max_memory_allocation_size = std::min(props3.maxMemoryAllocationSize, props4.maxBufferSize); + } else { + ctx->device->max_memory_allocation_size = props3.maxMemoryAllocationSize; + } - std::vector queue_family_props = ctx->device.lock()->physical_device.getQueueFamilyProperties(); + ctx->device->vendor_id = ctx->device->properties.vendorID; + ctx->device->subgroup_size = subgroup_props.subgroupSize; + ctx->device->uma = ctx->device->properties.deviceType == vk::PhysicalDeviceType::eIntegratedGpu; - // Try to find a non-graphics compute queue and transfer-focused queues - const uint32_t compute_queue_family_index = ggml_vk_find_queue_family_index(queue_family_props, vk::QueueFlagBits::eCompute, vk::QueueFlagBits::eGraphics, -1, 1); - const uint32_t transfer_queue_family_index = ggml_vk_find_queue_family_index(queue_family_props, vk::QueueFlagBits::eTransfer, vk::QueueFlagBits::eCompute | vk::QueueFlagBits::eGraphics, compute_queue_family_index, 1); + bool fp16_storage = false; + bool fp16_compute = false; - const float priorities[] = { 1.0f, 1.0f }; - ctx->device.lock()->single_queue = compute_queue_family_index == transfer_queue_family_index && queue_family_props[compute_queue_family_index].queueCount == 1; + for (const auto& properties : ext_props) { + if (strcmp("VK_KHR_16bit_storage", properties.extensionName) == 0) { + fp16_storage = true; + } else if (strcmp("VK_KHR_shader_float16_int8", properties.extensionName) == 0) { + fp16_compute = true; + } + } - std::vector device_queue_create_infos; - if (compute_queue_family_index != transfer_queue_family_index) { - device_queue_create_infos.push_back({vk::DeviceQueueCreateFlags(), compute_queue_family_index, 1, priorities}); - device_queue_create_infos.push_back({vk::DeviceQueueCreateFlags(), transfer_queue_family_index, 1, priorities + 1}); - } else if(!ctx->device.lock()->single_queue) { - device_queue_create_infos.push_back({vk::DeviceQueueCreateFlags(), compute_queue_family_index, 2, priorities}); - } else { - device_queue_create_infos.push_back({vk::DeviceQueueCreateFlags(), compute_queue_family_index, 1, priorities}); - } - vk::DeviceCreateInfo device_create_info; - std::vector device_extensions; - vk::PhysicalDeviceFeatures device_features = ctx->device.lock()->physical_device.getFeatures(); + const char* GGML_VK_DISABLE_F16 = getenv("GGML_VK_DISABLE_F16"); + const bool force_disable_f16 = GGML_VK_DISABLE_F16 != nullptr; - VkPhysicalDeviceFeatures2 device_features2; - device_features2.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FEATURES_2; - device_features2.pNext = nullptr; - device_features2.features = (VkPhysicalDeviceFeatures)device_features; + ctx->device->fp16 = !force_disable_f16 && fp16_storage && fp16_compute; - VkPhysicalDeviceVulkan11Features vk11_features; - vk11_features.pNext = nullptr; - vk11_features.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_1_FEATURES; - device_features2.pNext = &vk11_features; + std::vector queue_family_props = ctx->device->physical_device.getQueueFamilyProperties(); - VkPhysicalDeviceVulkan12Features vk12_features; - vk12_features.pNext = nullptr; - vk12_features.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_2_FEATURES; - vk11_features.pNext = &vk12_features; + // Try to find a non-graphics compute queue and transfer-focused queues + const uint32_t compute_queue_family_index = ggml_vk_find_queue_family_index(queue_family_props, vk::QueueFlagBits::eCompute, vk::QueueFlagBits::eGraphics, -1, 1); + const uint32_t transfer_queue_family_index = ggml_vk_find_queue_family_index(queue_family_props, vk::QueueFlagBits::eTransfer, vk::QueueFlagBits::eCompute | vk::QueueFlagBits::eGraphics, compute_queue_family_index, 1); - vkGetPhysicalDeviceFeatures2(ctx->device.lock()->physical_device, &device_features2); + const float priorities[] = { 1.0f, 1.0f }; + ctx->device->single_queue = compute_queue_family_index == transfer_queue_family_index && queue_family_props[compute_queue_family_index].queueCount == 1; - ctx->device.lock()->fp16 = ctx->device.lock()->fp16 && vk12_features.shaderFloat16; + std::vector device_queue_create_infos; + if (compute_queue_family_index != transfer_queue_family_index) { + device_queue_create_infos.push_back({vk::DeviceQueueCreateFlags(), compute_queue_family_index, 1, priorities}); + device_queue_create_infos.push_back({vk::DeviceQueueCreateFlags(), transfer_queue_family_index, 1, priorities + 1}); + } else if(!ctx->device->single_queue) { + device_queue_create_infos.push_back({vk::DeviceQueueCreateFlags(), compute_queue_family_index, 2, priorities}); + } else { + device_queue_create_infos.push_back({vk::DeviceQueueCreateFlags(), compute_queue_family_index, 1, priorities}); + } + vk::DeviceCreateInfo device_create_info; + std::vector device_extensions; + vk::PhysicalDeviceFeatures device_features = ctx->device->physical_device.getFeatures(); - if (!vk11_features.storageBuffer16BitAccess) { - std::cerr << "ggml_vulkan: device " << GGML_VK_NAME << idx << " does not support 16-bit storage." << std::endl; - throw std::runtime_error("Unsupported device"); - } + VkPhysicalDeviceFeatures2 device_features2; + device_features2.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FEATURES_2; + device_features2.pNext = nullptr; + device_features2.features = (VkPhysicalDeviceFeatures)device_features; - device_extensions.push_back("VK_KHR_16bit_storage"); + VkPhysicalDeviceVulkan11Features vk11_features; + vk11_features.pNext = nullptr; + vk11_features.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_1_FEATURES; + device_features2.pNext = &vk11_features; + + VkPhysicalDeviceVulkan12Features vk12_features; + vk12_features.pNext = nullptr; + vk12_features.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_2_FEATURES; + vk11_features.pNext = &vk12_features; + + vkGetPhysicalDeviceFeatures2(ctx->device->physical_device, &device_features2); + + ctx->device->fp16 = ctx->device->fp16 && vk12_features.shaderFloat16; + + if (!vk11_features.storageBuffer16BitAccess) { + std::cerr << "ggml_vulkan: device " << GGML_VK_NAME << idx << " does not support 16-bit storage." << std::endl; + throw std::runtime_error("Unsupported device"); + } + + device_extensions.push_back("VK_KHR_16bit_storage"); #ifdef GGML_VULKAN_VALIDATE - device_extensions.push_back("VK_KHR_shader_non_semantic_info"); + device_extensions.push_back("VK_KHR_shader_non_semantic_info"); #endif - if (ctx->device.lock()->fp16) { - device_extensions.push_back("VK_KHR_shader_float16_int8"); - } - ctx->device.lock()->name = ctx->device.lock()->properties.deviceName.data(); + if (ctx->device->fp16) { + device_extensions.push_back("VK_KHR_shader_float16_int8"); + } + ctx->device->name = ctx->device->properties.deviceName.data(); - device_create_info = { - vk::DeviceCreateFlags(), - device_queue_create_infos, - {}, - device_extensions - }; - device_create_info.setPNext(&device_features2); - ctx->device.lock()->device = ctx->device.lock()->physical_device.createDevice(device_create_info); + device_create_info = { + vk::DeviceCreateFlags(), + device_queue_create_infos, + {}, + device_extensions + }; + device_create_info.setPNext(&device_features2); + ctx->device->device = ctx->device->physical_device.createDevice(device_create_info); - ctx->device.lock()->descriptor_set_mode = VK_DEVICE_DESCRIPTOR_POOL_MODE_UNKNOWN; + ctx->device->descriptor_set_mode = VK_DEVICE_DESCRIPTOR_POOL_MODE_UNKNOWN; - // Shaders - ggml_vk_load_shaders(ctx); + // Queues + ggml_vk_create_queue(ctx, ctx->device->compute_queue, compute_queue_family_index, 0, { vk::PipelineStageFlagBits::eComputeShader | vk::PipelineStageFlagBits::eTransfer }); - // Queues - ggml_vk_create_queue(ctx, ctx->device.lock()->compute_queue, compute_queue_family_index, 0, { vk::PipelineStageFlagBits::eComputeShader | vk::PipelineStageFlagBits::eTransfer }); - if (!ctx->device.lock()->single_queue) { - const uint32_t transfer_queue_index = compute_queue_family_index == transfer_queue_family_index ? 1 : 0; - ggml_vk_create_queue(ctx, ctx->device.lock()->transfer_queue, transfer_queue_family_index, transfer_queue_index, { vk::PipelineStageFlagBits::eTransfer }); - } else { - // TODO: Use pointer or reference to avoid copy - ctx->device.lock()->transfer_queue = ctx->device.lock()->compute_queue; + // Shaders + ggml_vk_load_shaders(ctx); + + if (!ctx->device->single_queue) { + const uint32_t transfer_queue_index = compute_queue_family_index == transfer_queue_family_index ? 1 : 0; + ggml_vk_create_queue(ctx, ctx->device->transfer_queue, transfer_queue_family_index, transfer_queue_index, { vk::PipelineStageFlagBits::eTransfer }); + } else { + // TODO: Use pointer or reference to avoid copy + ctx->device->transfer_queue = ctx->device->compute_queue; + } + + ctx->device->idx = dev_num; + ctx->device->initialized = true; + } else if (ctx->device->idx != dev_num) { + std::cerr << "ggml_vulkan: Device " << ctx->device->name << " already initialized with index " << ctx->device->idx << ", but trying to reinitialize with index " << dev_num << std::endl; + throw std::runtime_error("Device already initialized"); } - ctx->fence = ctx->device.lock()->device.createFence({}); + ctx->fence = ctx->device->device.createFence({}); ctx->compute_ctx = nullptr; ctx->transfer_ctx = nullptr; @@ -1339,7 +1519,7 @@ static void ggml_vk_init(ggml_backend_vk_context * ctx, size_t idx) { #endif } -static vk_pipeline* ggml_vk_get_to_fp16(ggml_backend_vk_context * ctx, ggml_type type) { +static vk_pipeline ggml_vk_get_to_fp16(ggml_backend_vk_context * ctx, ggml_type type) { #ifdef GGML_VULKAN_DEBUG std::cerr << "ggml_vk_get_to_fp16()" << std::endl; #endif @@ -1360,10 +1540,36 @@ static vk_pipeline* ggml_vk_get_to_fp16(ggml_backend_vk_context * ctx, ggml_type return nullptr; } - return &ctx->pipeline_dequant[type]; + return ctx->device->pipeline_dequant[type]; } -static vk_pipeline* ggml_vk_get_dequantize_mul_mat_vec(ggml_backend_vk_context * ctx, ggml_type type) { +static vk_matmul_pipeline ggml_vk_get_mul_mat_mat_pipeline(ggml_backend_vk_context * ctx, ggml_type src0_type, ggml_type src1_type) { +#ifdef GGML_VULKAN_DEBUG + std::cerr << "ggml_vk_get_mul_mat_mat_pipeline()" << std::endl; +#endif + if (src0_type == GGML_TYPE_F32 && src1_type == GGML_TYPE_F32) { + return ctx->device->pipeline_matmul_f32; + } + if (src0_type == GGML_TYPE_F16 && src1_type == GGML_TYPE_F32) { + return ctx->device->pipeline_matmul_f16_f32; + } + if (src0_type == GGML_TYPE_F16 && src1_type == GGML_TYPE_F16) { + return ctx->device->pipeline_matmul_f16; + } + + GGML_ASSERT(src1_type == GGML_TYPE_F32); + + switch (src0_type) { + case GGML_TYPE_Q4_0: + break; + default: + return nullptr; + } + + return ctx->device->pipeline_dequant_mul_mat_mat[src0_type]; +} + +static vk_pipeline ggml_vk_get_dequantize_mul_mat_vec(ggml_backend_vk_context * ctx, ggml_type type) { #ifdef GGML_VULKAN_DEBUG std::cerr << "ggml_vk_get_dequantize_mul_mat_vec()" << std::endl; #endif @@ -1384,7 +1590,7 @@ static vk_pipeline* ggml_vk_get_dequantize_mul_mat_vec(ggml_backend_vk_context * return nullptr; } - return &ctx->pipeline_dequant_mul_mat_vec_f32[type]; + return ctx->device->pipeline_dequant_mul_mat_vec_f32[type]; } static vk_buffer ggml_vk_pool_malloc(ggml_backend_vk_context * ctx, size_t size) { @@ -1463,8 +1669,8 @@ static void * ggml_vk_host_malloc(ggml_backend_vk_context * ctx, size_t size) { if(!(buf->memory_property_flags & vk::MemoryPropertyFlagBits::eHostVisible)) { fprintf(stderr, "WARNING: failed to allocate %.2f MB of pinned memory\n", size/1024.0/1024.0); - ctx->device.lock()->device.freeMemory(buf->device_memory); - ctx->device.lock()->device.destroyBuffer(buf->buffer); + ctx->device->device.freeMemory(buf->device_memory); + ctx->device->device.destroyBuffer(buf->buffer); return nullptr; } @@ -1528,30 +1734,30 @@ static vk_submission ggml_vk_begin_submission(ggml_backend_vk_context * ctx, vk_ } static void ggml_vk_dispatch_pipeline(ggml_backend_vk_context * ctx, vk_context * subctx, vk_pipeline& pipeline, std::vector&& buffers, size_t push_constant_size, const void* push_constants, std::array elements) { - const uint32_t wg0 = CEIL_DIV(elements[0], pipeline.wg_denoms[0]); - const uint32_t wg1 = CEIL_DIV(elements[1], pipeline.wg_denoms[1]); - const uint32_t wg2 = CEIL_DIV(elements[2], pipeline.wg_denoms[2]); + const uint32_t wg0 = CEIL_DIV(elements[0], pipeline->wg_denoms[0]); + const uint32_t wg1 = CEIL_DIV(elements[1], pipeline->wg_denoms[1]); + const uint32_t wg2 = CEIL_DIV(elements[2], pipeline->wg_denoms[2]); #ifdef GGML_VULKAN_DEBUG - std::cerr << "ggml_vk_dispatch_pipeline(" << pipeline.name << ", (" << wg0 << "," << wg1 << "," << wg2 << "))" << std::endl; + std::cerr << "ggml_vk_dispatch_pipeline(" << pipeline->name << ", (" << wg0 << "," << wg1 << "," << wg2 << "))" << std::endl; #endif std::vector descriptor_buffer_infos; std::vector write_descriptor_sets; - GGML_ASSERT(pipeline.descriptor_set_idx < pipeline.descriptor_sets.size()); - GGML_ASSERT(buffers.size() == pipeline.parameter_count); - vk::DescriptorSet& descriptor_set = pipeline.descriptor_sets[pipeline.descriptor_set_idx++]; - for (uint32_t i = 0; i < pipeline.parameter_count; i++) { + GGML_ASSERT(pipeline->descriptor_set_idx < pipeline->descriptor_sets.size()); + GGML_ASSERT(buffers.size() == pipeline->parameter_count); + vk::DescriptorSet& descriptor_set = pipeline->descriptor_sets[pipeline->descriptor_set_idx++]; + for (uint32_t i = 0; i < pipeline->parameter_count; i++) { descriptor_buffer_infos.push_back({buffers[i].buffer->buffer, buffers[i].offset, buffers[i].size}); } - for (uint32_t i = 0; i < pipeline.parameter_count; i++) { + for (uint32_t i = 0; i < pipeline->parameter_count; i++) { write_descriptor_sets.push_back({descriptor_set, i, 0, 1, vk::DescriptorType::eStorageBuffer, nullptr, &descriptor_buffer_infos[i]}); } - ctx->device.lock()->device.updateDescriptorSets(write_descriptor_sets, {}); + ctx->device->device.updateDescriptorSets(write_descriptor_sets, {}); - subctx->s->buffer.pushConstants(pipeline.layout, vk::ShaderStageFlagBits::eCompute, 0, push_constant_size, push_constants); - subctx->s->buffer.bindPipeline(vk::PipelineBindPoint::eCompute, pipeline.pipeline); + subctx->s->buffer.pushConstants(pipeline->layout, vk::ShaderStageFlagBits::eCompute, 0, push_constant_size, push_constants); + subctx->s->buffer.bindPipeline(vk::PipelineBindPoint::eCompute, pipeline->pipeline); subctx->s->buffer.bindDescriptorSets(vk::PipelineBindPoint::eCompute, - pipeline.layout, + pipeline->layout, 0, { descriptor_set }, {}); @@ -1810,7 +2016,7 @@ static void ggml_vk_buffer_write_2d(ggml_backend_vk_context * ctx, vk_buffer& ds memcpy((uint8_t *)dst->ptr + offset + i * width, (const uint8_t *) src + i * spitch, width); } } else { - vk_context * subctx = ggml_vk_create_context(ctx, ctx->device.lock()->transfer_queue); + vk_context * subctx = ggml_vk_create_context(ctx, ctx->device->transfer_queue); ggml_vk_ctx_begin(ctx, subctx); ggml_vk_buffer_write_2d_async(ctx, subctx, dst, offset, src, spitch, width, height, true); ggml_vk_ctx_end(subctx); @@ -1820,8 +2026,9 @@ static void ggml_vk_buffer_write_2d(ggml_backend_vk_context * ctx, vk_buffer& ds } ggml_vk_submit(subctx, ctx->fence); - VK_CHECK(ctx->device.lock()->device.waitForFences({ ctx->fence }, true, UINT64_MAX), "vk_buffer_write_2d waitForFences"); - ctx->device.lock()->device.resetFences({ ctx->fence }); + VK_CHECK(ctx->device->device.waitForFences({ ctx->fence }, true, UINT64_MAX), "vk_buffer_write_2d waitForFences"); + ctx->device->device.resetFences({ ctx->fence }); + ggml_vk_queue_cleanup(ctx, ctx->device->transfer_queue); } } @@ -1906,18 +2113,19 @@ static void ggml_vk_buffer_read(ggml_backend_vk_context * ctx, vk_buffer& src, s memcpy(dst, (uint8_t *) src->ptr + offset, size); } else { - vk_context * subctx = ggml_vk_create_context(ctx, ctx->device.lock()->transfer_queue); + vk_context * subctx = ggml_vk_create_context(ctx, ctx->device->transfer_queue); ggml_vk_ctx_begin(ctx, subctx); ggml_vk_buffer_read_async(ctx, subctx, src, offset, dst, size, true); ggml_vk_ctx_end(subctx); ggml_vk_submit(subctx, ctx->fence); - VK_CHECK(ctx->device.lock()->device.waitForFences({ ctx->fence }, true, UINT64_MAX), "vk_buffer_read waitForFences"); - ctx->device.lock()->device.resetFences({ ctx->fence }); + VK_CHECK(ctx->device->device.waitForFences({ ctx->fence }, true, UINT64_MAX), "vk_buffer_read waitForFences"); + ctx->device->device.resetFences({ ctx->fence }); for (auto& cpy : subctx->out_memcpys) { memcpy(cpy.dst, cpy.src, cpy.n); } + ggml_vk_queue_cleanup(ctx, ctx->device->transfer_queue); } } @@ -1941,15 +2149,13 @@ static void ggml_vk_buffer_copy(vk_buffer& dst, size_t dst_offset, vk_buffer& sr // Copy within the device ggml_backend_vk_context * ctx = src->ctx; - VkBufferCopy bc{ src_offset, dst_offset, size }; - - vk_context * subctx = ggml_vk_create_context(ctx, ctx->device.lock()->transfer_queue); + vk_context * subctx = ggml_vk_create_context(ctx, ctx->device->transfer_queue); ggml_vk_ctx_begin(ctx, subctx); ggml_vk_buffer_copy_async(subctx, dst, dst_offset, src, src_offset, size); ggml_vk_ctx_end(subctx); ggml_vk_submit(subctx, ctx->fence); - VK_CHECK(ctx->device.lock()->device.waitForFences({ ctx->fence }, true, UINT64_MAX), "vk_buffer_copy waitForFences"); - ctx->device.lock()->device.resetFences({ ctx->fence }); + VK_CHECK(ctx->device->device.waitForFences({ ctx->fence }, true, UINT64_MAX), "vk_buffer_copy waitForFences"); + ctx->device->device.resetFences({ ctx->fence }); } else { #ifdef GGML_VULKAN_DEBUG std::cerr << "ggml_vk_buffer_copy(MULTI_DEVICE, " << size << ")" << std::endl; @@ -1977,14 +2183,14 @@ static void ggml_vk_buffer_memset(ggml_backend_vk_context * ctx, vk_buffer& dst, // Make sure ctx owns the buffer GGML_ASSERT(dst->ctx == ctx); - vk_context * subctx = ggml_vk_create_context(ctx, ctx->device.lock()->transfer_queue); + vk_context * subctx = ggml_vk_create_context(ctx, ctx->device->transfer_queue); ggml_vk_ctx_begin(ctx, subctx); subctx->s->buffer.fillBuffer(dst->buffer, offset, size, c); ggml_vk_ctx_end(subctx); ggml_vk_submit(subctx, ctx->fence); - VK_CHECK(ctx->device.lock()->device.waitForFences({ ctx->fence }, true, UINT64_MAX), "vk_memset waitForFences"); - ctx->device.lock()->device.resetFences({ ctx->fence }); + VK_CHECK(ctx->device->device.waitForFences({ ctx->fence }, true, UINT64_MAX), "vk_memset waitForFences"); + ctx->device->device.resetFences({ ctx->fence }); } static void ggml_vk_h2d_tensor_2d(ggml_backend_vk_context * ctx, vk_context * subctx, vk_buffer& dst, size_t offset, const ggml_tensor * src, uint64_t i3, uint64_t i2, uint64_t i1) { @@ -2045,176 +2251,63 @@ static void ggml_vk_d2h_tensor_2d(ggml_backend_vk_context * ctx, vk_context * su static uint32_t ggml_vk_guess_split_k(int m, int n, int k) { #ifdef GGML_VULKAN_DEBUG - std::cerr << "ggml_vk_guess_split_k(" << m << ", " << n << ", " << k << ")"; + std::cerr << "ggml_vk_guess_split_k(" << m << ", " << n << ", " << k << ")" << std::endl; #endif if (k > 128 && (m < 128 || n < 128) && m > 2 && n > 2) { -#ifdef GGML_VULKAN_DEBUG - std::cerr << " = 4" << std::endl; -#endif return 4; } -#ifdef GGML_VULKAN_DEBUG - std::cerr << " = 1" << std::endl; -#endif return 1; } -static uint32_t ggml_vk_guess_matmul_pipeline_align(ggml_backend_vk_context * ctx, int m, int n) { +static vk_pipeline ggml_vk_guess_matmul_pipeline_amd(ggml_backend_vk_context * ctx, vk_matmul_pipeline& mmp, int m, int n, bool aligned) { + if (m <= 32 || n <= 32) { + return aligned ? mmp->a_s : mmp->s; + } + return aligned ? mmp->a_m : mmp->m; + + GGML_UNUSED(ctx); +} + +static vk_pipeline ggml_vk_guess_matmul_pipeline_apple(ggml_backend_vk_context * ctx, vk_matmul_pipeline& mmp, bool aligned) { + return aligned ? mmp->a_m : mmp->m; + + GGML_UNUSED(ctx); +} + +static vk_pipeline ggml_vk_guess_matmul_pipeline_intel(ggml_backend_vk_context * ctx, vk_matmul_pipeline& mmp, bool aligned) { + return aligned ? mmp->a_s : mmp->s; + + GGML_UNUSED(ctx); +} + +static vk_pipeline ggml_vk_guess_matmul_pipeline(ggml_backend_vk_context * ctx, vk_matmul_pipeline& mmp, int m, int n, bool aligned) { +#ifdef GGML_VULKAN_DEBUG + std::cerr << "ggml_vk_guess_matmul_pipeline(" << m << ", " << n << ", " << aligned << ")" << std::endl; +#endif + switch (ctx->device->vendor_id) { + case VK_VENDOR_ID_AMD: + return ggml_vk_guess_matmul_pipeline_amd(ctx, mmp, m, n, aligned); + case VK_VENDOR_ID_APPLE: + return ggml_vk_guess_matmul_pipeline_apple(ctx, mmp, aligned); + case VK_VENDOR_ID_INTEL: + return ggml_vk_guess_matmul_pipeline_intel(ctx, mmp, aligned); + } + + if (m <= 32 || n <= 32) { + return aligned ? mmp->a_s : mmp->s; + } + if (m <= 64 || n <= 64) { + return aligned ? mmp->a_m : mmp->m; + } + return aligned ? mmp->a_l : mmp->l; +} + +static uint32_t ggml_vk_guess_matmul_pipeline_align(ggml_backend_vk_context * ctx, vk_matmul_pipeline& mmp, int m, int n) { #ifdef GGML_VULKAN_DEBUG std::cerr << "ggml_vk_guess_matmul_pipeline_align(" << m << ", " << n << ")" << std::endl; #endif - if (m <= 32 || n <= 32) { - return ctx->pipeline_matmul_f32_aligned_s.align; - } - if (ctx->device.lock()->subgroup_size == 64 || m <= 64 || n <= 64) { - return ctx->pipeline_matmul_f32_aligned_m.align; - } - return ctx->pipeline_matmul_f32_aligned_l.align; -} - -static vk_pipeline* ggml_vk_guess_matmul_pipeline_amd(ggml_backend_vk_context * ctx, bool bit16_x, bool bit16_y, int m, int n, bool aligned) { - if (bit16_x && bit16_y) { - if (m <= 32 || n <= 32) { -#ifdef GGML_VULKAN_DEBUG - std::cerr << " S" << std::endl; -#endif - return aligned ? &ctx->pipeline_matmul_f16_aligned_s : &ctx->pipeline_matmul_f16_s; - } -#ifdef GGML_VULKAN_DEBUG - std::cerr << " M" << std::endl; -#endif - return aligned ? &ctx->pipeline_matmul_f16_aligned_m : &ctx->pipeline_matmul_f16_m; - } - if (bit16_x && !bit16_y) { - if (m <= 32 || n <= 32) { -#ifdef GGML_VULKAN_DEBUG - std::cerr << " S" << std::endl; -#endif - return aligned ? &ctx->pipeline_matmul_f16_f32_aligned_s : &ctx->pipeline_matmul_f16_f32_s; - } -#ifdef GGML_VULKAN_DEBUG - std::cerr << " M" << std::endl; -#endif - return aligned ? &ctx->pipeline_matmul_f16_f32_aligned_m : &ctx->pipeline_matmul_f16_f32_m; - } - if (!bit16_x && bit16_y) { - GGML_ASSERT(false); - } - - if (m <= 32 || n <= 32) { -#ifdef GGML_VULKAN_DEBUG - std::cerr << " S" << std::endl; -#endif - return aligned ? &ctx->pipeline_matmul_f32_aligned_s : &ctx->pipeline_matmul_f32_s; - } -#ifdef GGML_VULKAN_DEBUG - std::cerr << " M" << std::endl; -#endif - return aligned ? &ctx->pipeline_matmul_f32_aligned_m : &ctx->pipeline_matmul_f32_m; -} - -static vk_pipeline* ggml_vk_guess_matmul_pipeline_apple(ggml_backend_vk_context * ctx, bool bit16_x, bool bit16_y, bool aligned) { -#ifdef GGML_VULKAN_DEBUG - std::cerr << " M" << std::endl; -#endif - if (bit16_x && bit16_y) { - return aligned ? &ctx->pipeline_matmul_f16_aligned_m : &ctx->pipeline_matmul_f16_m; - } - if (bit16_x && !bit16_y) { - return aligned ? &ctx->pipeline_matmul_f16_f32_aligned_m : &ctx->pipeline_matmul_f16_f32_m; - } - if (!bit16_x && bit16_y) { - GGML_ASSERT(false); - } - return aligned ? &ctx->pipeline_matmul_f32_aligned_m : &ctx->pipeline_matmul_f32_m; -} - -static vk_pipeline* ggml_vk_guess_matmul_pipeline_intel(ggml_backend_vk_context * ctx, bool bit16_x, bool bit16_y, bool aligned) { -#ifdef GGML_VULKAN_DEBUG - std::cerr << " S" << std::endl; -#endif - if (bit16_x && bit16_y) { - return aligned ? &ctx->pipeline_matmul_f16_aligned_s : &ctx->pipeline_matmul_f16_s; - } - if (bit16_x && !bit16_y) { - return aligned ? &ctx->pipeline_matmul_f16_f32_aligned_s : &ctx->pipeline_matmul_f16_f32_s; - } - if (!bit16_x && bit16_y) { - GGML_ASSERT(false); - } - return aligned ? &ctx->pipeline_matmul_f32_aligned_s : &ctx->pipeline_matmul_f32_s; -} - -static vk_pipeline* ggml_vk_guess_matmul_pipeline(ggml_backend_vk_context * ctx, bool bit16_x, bool bit16_y, int m, int n, bool aligned) { -#ifdef GGML_VULKAN_DEBUG - std::cerr << "ggml_vk_guess_matmul_pipeline(" << bit16_x << ", " << bit16_y << ", " << m << ", " << n << ", " << aligned << ")"; -#endif - switch (ctx->device.lock()->vendor_id) { - case VK_VENDOR_ID_AMD: - return ggml_vk_guess_matmul_pipeline_amd(ctx, bit16_x, bit16_y, m, n, aligned); - case VK_VENDOR_ID_APPLE: - return ggml_vk_guess_matmul_pipeline_apple(ctx, bit16_x, bit16_y, aligned); - case VK_VENDOR_ID_INTEL: - return ggml_vk_guess_matmul_pipeline_intel(ctx, bit16_x, bit16_y, aligned); - } - - if (bit16_x && bit16_y) { - if (m <= 32 || n <= 32) { -#ifdef GGML_VULKAN_DEBUG - std::cerr << " S" << std::endl; -#endif - return aligned ? &ctx->pipeline_matmul_f16_aligned_s : &ctx->pipeline_matmul_f16_s; - } - if (m <= 64 || n <= 64) { -#ifdef GGML_VULKAN_DEBUG - std::cerr << " M" << std::endl; -#endif - return aligned ? &ctx->pipeline_matmul_f16_aligned_m : &ctx->pipeline_matmul_f16_m; - } -#ifdef GGML_VULKAN_DEBUG - std::cerr << " L" << std::endl; -#endif - return aligned ? &ctx->pipeline_matmul_f16_aligned_l : &ctx->pipeline_matmul_f16_l; - } - if (bit16_x && !bit16_y) { - if (m <= 32 || n <= 32) { -#ifdef GGML_VULKAN_DEBUG - std::cerr << " S" << std::endl; -#endif - return aligned ? &ctx->pipeline_matmul_f16_f32_aligned_s : &ctx->pipeline_matmul_f16_f32_s; - } - if (m <= 64 || n <= 64) { -#ifdef GGML_VULKAN_DEBUG - std::cerr << " M" << std::endl; -#endif - return aligned ? &ctx->pipeline_matmul_f16_f32_aligned_m : &ctx->pipeline_matmul_f16_f32_m; - } -#ifdef GGML_VULKAN_DEBUG - std::cerr << " L" << std::endl; -#endif - return aligned ? &ctx->pipeline_matmul_f16_f32_aligned_l : &ctx->pipeline_matmul_f16_f32_l; - } - if (!bit16_x && bit16_y) { - GGML_ASSERT(false); - } - - if (m <= 32 || n <= 32) { -#ifdef GGML_VULKAN_DEBUG - std::cerr << " S" << std::endl; -#endif - return aligned ? &ctx->pipeline_matmul_f32_aligned_s : &ctx->pipeline_matmul_f32_s; - } - if (m <= 64 || n <= 64) { -#ifdef GGML_VULKAN_DEBUG - std::cerr << " M" << std::endl; -#endif - return aligned ? &ctx->pipeline_matmul_f32_aligned_m : &ctx->pipeline_matmul_f32_m; - } -#ifdef GGML_VULKAN_DEBUG - std::cerr << " L" << std::endl; -#endif - return aligned ? &ctx->pipeline_matmul_f32_aligned_l : &ctx->pipeline_matmul_f32_l; + return ggml_vk_guess_matmul_pipeline(ctx, mmp, m, n, false)->align; } static void ggml_vk_matmul(ggml_backend_vk_context * ctx, vk_context * subctx, vk_pipeline& pipeline, vk_subbuffer&& a, vk_subbuffer&& b, vk_subbuffer&& d, vk_subbuffer&& split_k_buffer, uint32_t m, uint32_t n, uint32_t k, uint32_t stride_a, uint32_t stride_b, uint32_t stride_d, uint32_t split_k, uint32_t batch, uint32_t ne02, uint32_t ne12, uint32_t broadcast2, uint32_t broadcast3, uint32_t batch_stride_a, uint32_t batch_stride_b, uint32_t batch_stride_d) { @@ -2232,10 +2325,10 @@ static void ggml_vk_matmul(ggml_backend_vk_context * ctx, vk_context * subctx, v const std::array pc1 = { m, n, k, stride_a, stride_b, stride_d, CEIL_DIV(k, split_k), ne02, ne12, broadcast2, broadcast3, batch_stride_a, batch_stride_b, batch_stride_d }; // Make sure enough workgroups get assigned for split k to work - ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, { a, b, split_k_buffer }, pc1.size() * sizeof(uint32_t), pc1.data(), { (CEIL_DIV(m, pipeline.wg_denoms[0]) * pipeline.wg_denoms[0]) * split_k, n, batch }); + ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, { a, b, split_k_buffer }, pc1.size() * sizeof(uint32_t), pc1.data(), { (CEIL_DIV(m, pipeline->wg_denoms[0]) * pipeline->wg_denoms[0]) * split_k, n, batch }); ggml_vk_sync_buffers(subctx); const std::array pc2 = { (uint32_t)(m * n * batch), split_k }; - ggml_vk_dispatch_pipeline(ctx, subctx, ctx->pipeline_matmul_split_k_reduce, { split_k_buffer, d }, pc2.size() * sizeof(uint32_t), pc2.data(), { m * n * batch, 1, 1 }); + ggml_vk_dispatch_pipeline(ctx, subctx, ctx->device->pipeline_matmul_split_k_reduce, { split_k_buffer, d }, pc2.size() * sizeof(uint32_t), pc2.data(), { m * n * batch, 1, 1 }); } static bool ggml_vk_dim01_contiguous(const ggml_tensor * tensor) { @@ -2245,41 +2338,39 @@ static bool ggml_vk_dim01_contiguous(const ggml_tensor * tensor) { tensor->nb[3] == tensor->nb[2]*tensor->ne[2]; } -static vk_pipeline * ggml_vk_get_cpy_pipeline(ggml_backend_vk_context * ctx, ggml_type from, ggml_type to) { +static vk_pipeline ggml_vk_get_cpy_pipeline(ggml_backend_vk_context * ctx, ggml_type from, ggml_type to) { if (from == GGML_TYPE_F32 && to == GGML_TYPE_F32) { - return &ctx->pipeline_cpy_f32_f32; + return ctx->device->pipeline_cpy_f32_f32; } if (from == GGML_TYPE_F32 && to == GGML_TYPE_F16) { - return &ctx->pipeline_cpy_f32_f16; + return ctx->device->pipeline_cpy_f32_f16; } if (from == GGML_TYPE_F16 && to == GGML_TYPE_F16) { - return &ctx->pipeline_cpy_f16_f16; + return ctx->device->pipeline_cpy_f16_f16; } std::cerr << "Missing CPY op for types: " << ggml_type_name(from) << " " << ggml_type_name(to) << std::endl; GGML_ASSERT(false); } -static void ggml_vk_cpy_to_contiguous(ggml_backend_vk_context * ctx, vk_context * subctx, vk_pipeline * pipeline, const ggml_tensor * tensor, vk_subbuffer&& in, vk_subbuffer&& out, ggml_type buffer_type, bool aligned=true) { +static void ggml_vk_cpy_to_contiguous(ggml_backend_vk_context * ctx, vk_context * subctx, vk_pipeline pipeline, const ggml_tensor * tensor, vk_subbuffer&& in, vk_subbuffer&& out) { #ifdef GGML_VULKAN_DEBUG std::cerr << "ggml_vk_cpy_to_contiguous((" << tensor << ", type=" << tensor->type << ", backend=" << tensor->backend << ", ne0=" << tensor->ne[0] << ", ne1=" << tensor->ne[1] << ", ne2=" << tensor->ne[2] << ", ne3=" << tensor->ne[3] << ", nb0=" << tensor->nb[0] << ", nb1=" << tensor->nb[1] << ", nb2=" << tensor->nb[2] << ", nb3=" << tensor->nb[3] << "), "; std::cerr << "buffer in size=" << in.buffer->size << ", buffer out size=" << out.buffer->size << ")" << std::endl; #endif const int tensor_type_size = ggml_type_size(tensor->type); - const int dst_type_size = ggml_type_size(buffer_type); - const uint32_t ne = tensor->ne[0] * tensor->ne[1] * tensor->ne[2]; + const uint32_t ne = ggml_nelements(tensor); - const uint32_t nb2 = aligned ? ggml_vk_align_size(dst_type_size * tensor->ne[0] * tensor->ne[1], ctx->device.lock()->properties.limits.minStorageBufferOffsetAlignment) / dst_type_size : tensor->ne[0] * tensor->ne[1]; - - const vk_op_cpy_push_constants pc = { + const vk_op_unary_push_constants pc = { (uint32_t)ne, - (uint32_t)tensor->ne[0], (uint32_t)tensor->ne[1], (uint32_t)tensor->nb[0] / tensor_type_size, (uint32_t)tensor->nb[1] / tensor_type_size, (uint32_t)tensor->nb[2] / tensor_type_size, - (uint32_t)tensor->ne[0], (uint32_t)tensor->ne[1], 1 , (uint32_t)tensor->ne[0] , nb2, + (uint32_t)tensor->ne[0], (uint32_t)tensor->ne[1], (uint32_t)tensor->ne[2], (uint32_t)tensor->ne[3], (uint32_t)tensor->nb[0] / tensor_type_size, (uint32_t)tensor->nb[1] / tensor_type_size, (uint32_t)tensor->nb[2] / tensor_type_size, (uint32_t)tensor->nb[3] / tensor_type_size, + (uint32_t)tensor->ne[0], (uint32_t)tensor->ne[1], (uint32_t)tensor->ne[2], (uint32_t)tensor->ne[3], 1 , (uint32_t)tensor->ne[0] , (uint32_t)(tensor->ne[0] * tensor->ne[1]) , (uint32_t)(tensor->ne[0] * tensor->ne[1] * tensor->ne[2]), 0, + 0.0f, 0.0f, }; ggml_vk_sync_buffers(subctx); - ggml_vk_dispatch_pipeline(ctx, subctx, *pipeline, { in, out }, sizeof(vk_op_cpy_push_constants), &pc, { ne, 1, 1 }); + ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, { in, out }, sizeof(vk_op_unary_push_constants), &pc, { ne, 1, 1 }); } static void ggml_vk_mul_mat_q_f16(ggml_backend_vk_context * ctx, vk_context * subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { @@ -2319,7 +2410,7 @@ static void ggml_vk_mul_mat_q_f16(ggml_backend_vk_context * ctx, vk_context * su bool src0_uma = false; bool src1_uma = false; - if (ctx->device.lock()->uma) { + if (ctx->device->uma) { ggml_vk_host_get(ctx, src0->data, d_Qx, qx_buf_offset); ggml_vk_host_get(ctx, src1->data, d_Qy, qy_buf_offset); src0_uma = d_Qx != nullptr; @@ -2332,10 +2423,17 @@ static void ggml_vk_mul_mat_q_f16(ggml_backend_vk_context * ctx, vk_context * su const bool x_non_contig = !load_x && !ggml_vk_dim01_contiguous(src0); const bool y_non_contig = !load_y && !ggml_vk_dim01_contiguous(src1); - const bool f16_f32_kernel = src1->type == GGML_TYPE_F32 && !y_non_contig; + const bool y_f32_kernel = src1->type == GGML_TYPE_F32 && !y_non_contig; - const bool qx_needs_dequant = src0->type != GGML_TYPE_F16 || x_non_contig; - const bool qy_needs_dequant = (src1->type != GGML_TYPE_F16 && !f16_f32_kernel) || y_non_contig; + vk_matmul_pipeline mmp = ggml_vk_get_mul_mat_mat_pipeline(ctx, src0->type, y_non_contig ? GGML_TYPE_F16 : src1->type); + + const bool qx_needs_dequant = mmp == nullptr || x_non_contig; + const bool qy_needs_dequant = (src1->type != GGML_TYPE_F16 && !y_f32_kernel) || y_non_contig; + + if (mmp == nullptr) { + // Fall back to dequant + f16 mulmat + mmp = ggml_vk_get_mul_mat_mat_pipeline(ctx, GGML_TYPE_F16, y_f32_kernel ? GGML_TYPE_F32 : GGML_TYPE_F16); + } // Not implemented GGML_ASSERT(y_non_contig || !qy_needs_dequant); // NOLINT @@ -2344,17 +2442,17 @@ static void ggml_vk_mul_mat_q_f16(ggml_backend_vk_context * ctx, vk_context * su const int y_ne = ne11 * ne10; const int d_ne = ne11 * ne01; - const uint32_t kpad = ggml_vk_align_size(ne10, ggml_vk_guess_matmul_pipeline_align(ctx, ne01, ne11)); + const uint32_t kpad = ggml_vk_align_size(ne10, ggml_vk_guess_matmul_pipeline_align(ctx, mmp, ne01, ne11)); const bool aligned = ne10 == kpad; const uint32_t split_k = ggml_vk_guess_split_k(ne01, ne11, ne10); - vk_pipeline * pipeline = ggml_vk_guess_matmul_pipeline(ctx, true, !f16_f32_kernel, ne01, ne11, aligned); + vk_pipeline pipeline = ggml_vk_guess_matmul_pipeline(ctx, mmp, ne01, ne11, aligned); const uint64_t qx_sz = ggml_type_size(src0->type) * x_ne / ggml_blck_size(src0->type); const uint64_t qy_sz = ggml_type_size(src1->type) * y_ne / ggml_blck_size(src1->type); - const uint64_t x_sz = sizeof(ggml_fp16_t) * x_ne; - const uint64_t y_sz = f16_f32_kernel ? sizeof(float) * y_ne : sizeof(ggml_fp16_t) * y_ne; + const uint64_t x_sz = !qx_needs_dequant ? qx_sz : sizeof(ggml_fp16_t) * x_ne; + const uint64_t y_sz = y_f32_kernel ? sizeof(float) * y_ne : sizeof(ggml_fp16_t) * y_ne; const uint64_t d_sz = sizeof(float) * d_ne; vk_buffer d_D = extra->buffer_gpu.lock(); @@ -2385,7 +2483,7 @@ static void ggml_vk_mul_mat_q_f16(ggml_backend_vk_context * ctx, vk_context * su } else { d_X = d_Qx; x_buf_offset = qx_buf_offset; - GGML_ASSERT(qx_sz == x_sz); // NOLINT + GGML_ASSERT(qx_sz == x_sz); } if (qy_needs_dequant) { d_Y = ctx->prealloc_y; @@ -2396,8 +2494,8 @@ static void ggml_vk_mul_mat_q_f16(ggml_backend_vk_context * ctx, vk_context * su GGML_ASSERT(qy_sz == y_sz); } - vk_pipeline * to_fp16_vk_0 = nullptr; - vk_pipeline * to_fp16_vk_1 = nullptr; + vk_pipeline to_fp16_vk_0 = nullptr; + vk_pipeline to_fp16_vk_1 = nullptr; if (x_non_contig) { to_fp16_vk_0 = ggml_vk_get_cpy_pipeline(ctx, src0->type, GGML_TYPE_F16); @@ -2413,19 +2511,19 @@ static void ggml_vk_mul_mat_q_f16(ggml_backend_vk_context * ctx, vk_context * su GGML_ASSERT(!qy_needs_dequant || to_fp16_vk_1 != nullptr); // NOLINT // Allocate descriptor sets - ggml_pipeline_allocate_descriptor_sets(ctx, *pipeline, ne12 * ne13); + ggml_pipeline_allocate_descriptor_sets(ctx, pipeline, 1); if (qx_needs_dequant) { - ggml_pipeline_allocate_descriptor_sets(ctx, *to_fp16_vk_0, x_non_contig ? 1 : ne12 * ne13); + ggml_pipeline_allocate_descriptor_sets(ctx, to_fp16_vk_0, 1); } if (qy_needs_dequant) { - ggml_pipeline_allocate_descriptor_sets(ctx, *to_fp16_vk_1, y_non_contig ? 1 : ne12 * ne13); + ggml_pipeline_allocate_descriptor_sets(ctx, to_fp16_vk_1, 1); } if (split_k > 1) { - ggml_pipeline_allocate_descriptor_sets(ctx, ctx->pipeline_matmul_split_k_reduce, ne12 * ne13); + ggml_pipeline_allocate_descriptor_sets(ctx, ctx->device->pipeline_matmul_split_k_reduce, 1); } if (x_non_contig) { - ggml_vk_cpy_to_contiguous(ctx, subctx, to_fp16_vk_0, src0, { d_Qx, qx_buf_offset, VK_WHOLE_SIZE }, { d_X, 0, VK_WHOLE_SIZE }, dst->type, false); + ggml_vk_cpy_to_contiguous(ctx, subctx, to_fp16_vk_0, src0, { d_Qx, qx_buf_offset, VK_WHOLE_SIZE }, { d_X, 0, VK_WHOLE_SIZE }); } else if (load_x || qx_needs_dequant) { if (load_x) { // copy data to device @@ -2434,13 +2532,13 @@ static void ggml_vk_mul_mat_q_f16(ggml_backend_vk_context * ctx, vk_context * su } if (qx_needs_dequant) { - const std::vector pc = { (int)ne01, (int)ne10, (int)ne10, (int)ne10 }; + const std::vector pc = { (uint32_t)ne01, (uint32_t)ne10, (uint32_t)ne10, (uint32_t)ne10, (uint32_t)(ggml_nelements(src0)) }; ggml_vk_sync_buffers(subctx); - ggml_vk_dispatch_pipeline(ctx, subctx, *to_fp16_vk_0, { { d_Qx, qx_buf_offset, qx_sz * ne02 * ne03 }, { d_X, 0, x_sz * ne02 * ne03 } }, pc.size() * sizeof(int), pc.data(), { (uint32_t)(x_ne * ne02 * ne03), 1, 1}); + ggml_vk_dispatch_pipeline(ctx, subctx, to_fp16_vk_0, { { d_Qx, qx_buf_offset, qx_sz * ne02 * ne03 }, { d_X, 0, x_sz * ne02 * ne03 } }, pc.size() * sizeof(uint32_t), pc.data(), { (uint32_t)(x_ne * ne02 * ne03), 1, 1}); } } if (y_non_contig) { - ggml_vk_cpy_to_contiguous(ctx, subctx, to_fp16_vk_1, src1, { d_Qy, qy_buf_offset, VK_WHOLE_SIZE }, { d_Y, 0, VK_WHOLE_SIZE }, dst->type); + ggml_vk_cpy_to_contiguous(ctx, subctx, to_fp16_vk_1, src1, { d_Qy, qy_buf_offset, VK_WHOLE_SIZE }, { d_Y, 0, VK_WHOLE_SIZE }); } else if (load_y) { ggml_vk_h2d_tensor_2d(ctx, subctx, d_Qy, 0, src1, 0, 0, ggml_nrows(src1)); } @@ -2457,7 +2555,7 @@ static void ggml_vk_mul_mat_q_f16(ggml_backend_vk_context * ctx, vk_context * su } // compute - ggml_vk_matmul(ctx, subctx, *pipeline, { d_X, x_buf_offset, x_sz * ne02 * ne03 }, { d_Y, y_buf_offset, y_sz * ne12 * ne13 }, { d_D, d_buf_offset, d_sz * ne12 * ne13 }, { ctx->prealloc_split_k, 0, d_sz * ne12 * ne13 * split_k }, ne01, ne11, ne10, ne10, ne10, ne01, split_k, ne12*ne13, ne02, ne12, r2, r3, stride_batch_x, stride_batch_y, ne20*ne21); // NOLINT + ggml_vk_matmul(ctx, subctx, pipeline, { d_X, x_buf_offset, x_sz * ne02 * ne03 }, { d_Y, y_buf_offset, y_sz * ne12 * ne13 }, { d_D, d_buf_offset, d_sz * ne12 * ne13 }, { ctx->prealloc_split_k, 0, d_sz * ne12 * ne13 * split_k }, ne01, ne11, ne10, ne10, ne10, ne01, split_k, ne12*ne13, ne02, ne12, r2, r3, stride_batch_x, stride_batch_y, ne20*ne21); // NOLINT if (dst->backend == GGML_BACKEND_TYPE_CPU) { // copy dst to host @@ -2505,7 +2603,7 @@ static void ggml_vk_mul_mat_vec_q_f16(ggml_backend_vk_context * ctx, vk_context bool src0_uma = false; bool src1_uma = false; - if (ctx->device.lock()->uma) { + if (ctx->device->uma) { ggml_vk_host_get(ctx, src0->data, d_Qx, qx_buf_offset); ggml_vk_host_get(ctx, src1->data, d_Qy, qy_buf_offset); src0_uma = d_Qx != nullptr; @@ -2527,9 +2625,9 @@ static void ggml_vk_mul_mat_vec_q_f16(ggml_backend_vk_context * ctx, vk_context const uint64_t y_ne = ne11 * ne10; const uint64_t d_ne = ne11 * ne01; - const uint64_t qx_sz = ggml_vk_align_size(ggml_type_size(src0->type) * x_ne / ggml_blck_size(src0->type), ctx->device.lock()->properties.limits.minStorageBufferOffsetAlignment); + const uint64_t qx_sz = ggml_vk_align_size(ggml_type_size(src0->type) * x_ne / ggml_blck_size(src0->type), ctx->device->properties.limits.minStorageBufferOffsetAlignment); const uint64_t qy_sz = ggml_type_size(src1->type) * y_ne / ggml_blck_size(src1->type); - const uint64_t x_sz = x_non_contig ? ggml_vk_align_size(ggml_type_size(src0->type) * x_ne, ctx->device.lock()->properties.limits.minStorageBufferOffsetAlignment) : qx_sz; + const uint64_t x_sz = x_non_contig ? ggml_vk_align_size(ggml_type_size(src0->type) * x_ne, ctx->device->properties.limits.minStorageBufferOffsetAlignment) : qx_sz; const uint64_t y_sz = f16_f32_kernel ? sizeof(float) * y_ne : sizeof(ggml_fp16_t) * y_ne; const uint64_t d_sz = sizeof(float) * d_ne; @@ -2569,8 +2667,8 @@ static void ggml_vk_mul_mat_vec_q_f16(ggml_backend_vk_context * ctx, vk_context GGML_ASSERT(qy_sz == y_sz); } - vk_pipeline * to_fp16_vk_0 = nullptr; - vk_pipeline* to_fp16_vk_1 = nullptr; + vk_pipeline to_fp16_vk_0 = nullptr; + vk_pipeline to_fp16_vk_1 = nullptr; if (x_non_contig) { to_fp16_vk_0 = ggml_vk_get_cpy_pipeline(ctx, src0->type, src0->type); } @@ -2579,30 +2677,30 @@ static void ggml_vk_mul_mat_vec_q_f16(ggml_backend_vk_context * ctx, vk_context } else { to_fp16_vk_1 = ggml_vk_get_to_fp16(ctx, src1->type); } - vk_pipeline* dmmv = ggml_vk_get_dequantize_mul_mat_vec(ctx, src0->type); + vk_pipeline dmmv = ggml_vk_get_dequantize_mul_mat_vec(ctx, src0->type); GGML_ASSERT(!qx_needs_dequant || to_fp16_vk_0 != nullptr); // NOLINT GGML_ASSERT(!qy_needs_dequant || to_fp16_vk_1 != nullptr); // NOLINT GGML_ASSERT(dmmv != nullptr); // Allocate descriptor sets if (qx_needs_dequant) { - ggml_pipeline_allocate_descriptor_sets(ctx, *to_fp16_vk_0, 1); + ggml_pipeline_allocate_descriptor_sets(ctx, to_fp16_vk_0, 1); } if (qy_needs_dequant) { - ggml_pipeline_allocate_descriptor_sets(ctx, *to_fp16_vk_1, y_non_contig ? 1 : ne12 * ne13); + ggml_pipeline_allocate_descriptor_sets(ctx, to_fp16_vk_1, y_non_contig ? 1 : ne12 * ne13); } - ggml_pipeline_allocate_descriptor_sets(ctx, *dmmv, ne12 * ne13); + ggml_pipeline_allocate_descriptor_sets(ctx, dmmv, ne12 * ne13); if (x_non_contig) { - GGML_ASSERT(x_sz == ggml_vk_align_size(ggml_type_size(src0->type) * x_ne, ctx->device.lock()->properties.limits.minStorageBufferOffsetAlignment)); - ggml_vk_cpy_to_contiguous(ctx, subctx, to_fp16_vk_0, src0, { d_Qx, qx_buf_offset, VK_WHOLE_SIZE }, { d_X, 0, VK_WHOLE_SIZE }, src0->type); + GGML_ASSERT(x_sz == ggml_vk_align_size(ggml_type_size(src0->type) * x_ne, ctx->device->properties.limits.minStorageBufferOffsetAlignment)); + ggml_vk_cpy_to_contiguous(ctx, subctx, to_fp16_vk_0, src0, { d_Qx, qx_buf_offset, VK_WHOLE_SIZE }, { d_X, 0, VK_WHOLE_SIZE }); } else if (load_x) { // copy data to device ggml_vk_h2d_tensor_2d(ctx, subctx, d_Qx, 0, src0, 0, 0, ggml_nrows(src0)); } if (y_non_contig) { GGML_ASSERT(y_sz == ggml_type_size(src1->type) * y_ne); - ggml_vk_cpy_to_contiguous(ctx, subctx, to_fp16_vk_1, src1, { d_Qy, qy_buf_offset, VK_WHOLE_SIZE }, { d_Y, 0, VK_WHOLE_SIZE }, src1->type); + ggml_vk_cpy_to_contiguous(ctx, subctx, to_fp16_vk_1, src1, { d_Qy, qy_buf_offset, VK_WHOLE_SIZE }, { d_Y, 0, VK_WHOLE_SIZE }); } else if (load_y) { ggml_vk_h2d_tensor_2d(ctx, subctx, d_Qy, 0, src1, 0, 0, ggml_nrows(src1)); } @@ -2619,22 +2717,22 @@ static void ggml_vk_mul_mat_vec_q_f16(ggml_backend_vk_context * ctx, vk_context const uint64_t y_offset = y_buf_offset + y_sz * it_idx1; const uint64_t d_offset = d_buf_offset + d_sz * it_idx1; - const uint64_t y_buffer_offset = (y_offset / ctx->device.lock()->properties.limits.minStorageBufferOffsetAlignment) * ctx->device.lock()->properties.limits.minStorageBufferOffsetAlignment; + const uint64_t y_buffer_offset = (y_offset / ctx->device->properties.limits.minStorageBufferOffsetAlignment) * ctx->device->properties.limits.minStorageBufferOffsetAlignment; const uint64_t y_shader_offset = y_offset - y_buffer_offset; - const uint64_t d_buffer_offset = (d_offset / ctx->device.lock()->properties.limits.minStorageBufferOffsetAlignment) * ctx->device.lock()->properties.limits.minStorageBufferOffsetAlignment; + const uint64_t d_buffer_offset = (d_offset / ctx->device->properties.limits.minStorageBufferOffsetAlignment) * ctx->device->properties.limits.minStorageBufferOffsetAlignment; const uint64_t d_shader_offset = d_offset - d_buffer_offset; if (!y_non_contig && qy_needs_dequant) { - const std::vector pc = { (int)ne11, (int)ne10, (int)ne10, (int)ne10 }; + const std::vector pc = { (uint32_t)ne11, (uint32_t)ne10, (uint32_t)ne10, (uint32_t)ne10, (uint32_t)(y_ne / 32) }; ggml_vk_sync_buffers(subctx); - ggml_vk_dispatch_pipeline(ctx, subctx, *to_fp16_vk_1, { { d_Qy, qy_offset, qy_sz }, { d_Y, y_offset, y_sz } }, pc.size() * sizeof(int), pc.data(), { (uint32_t)y_ne, 1, 1}); + ggml_vk_dispatch_pipeline(ctx, subctx, to_fp16_vk_1, { { d_Qy, qy_offset, qy_sz }, { d_Y, y_offset, y_sz } }, pc.size() * sizeof(uint32_t), pc.data(), { (uint32_t)y_ne, 1, 1}); } // compute - const std::array pc = { (int)ne00, (int)(y_shader_offset / ggml_type_size(src1->type)), (int)(d_shader_offset / ggml_type_size(dst->type))}; + const std::array pc = { (uint32_t)ne00, (uint32_t)(y_shader_offset / ggml_type_size(src1->type)), (uint32_t)(d_shader_offset / ggml_type_size(dst->type))}; ggml_vk_sync_buffers(subctx); - ggml_vk_dispatch_pipeline(ctx, subctx, *dmmv, { { d_X, x_offset, x_sz }, { d_Y, y_buffer_offset, y_sz + y_shader_offset }, { d_D, d_buffer_offset, d_sz + d_shader_offset } }, 3 * sizeof(int), &pc, { (uint32_t)ne01, 1, 1}); + ggml_vk_dispatch_pipeline(ctx, subctx, dmmv, { { d_X, x_offset, x_sz }, { d_Y, y_buffer_offset, y_sz + y_shader_offset }, { d_D, d_buffer_offset, d_sz + d_shader_offset } }, 3 * sizeof(int), &pc, { (uint32_t)ne01, 1, 1}); if (dst->backend == GGML_BACKEND_TYPE_CPU) { // copy dst to host @@ -2680,7 +2778,7 @@ static void ggml_vk_mul_mat_vec_p021_f16_f32(ggml_backend_vk_context * ctx, vk_c bool src1_uma = false; - if (ctx->device.lock()->uma) { + if (ctx->device->uma) { ggml_vk_host_get(ctx, src1->data, d_Qy, qy_buf_offset); src1_uma = d_Qy != nullptr; } @@ -2691,7 +2789,7 @@ static void ggml_vk_mul_mat_vec_p021_f16_f32(ggml_backend_vk_context * ctx, vk_c const uint64_t y_ne = ne10 * ne11 * ne12; const uint64_t d_ne = ne01 * ne11 * ne12; - const uint64_t qx_sz = ggml_vk_align_size(ggml_type_size(src0->type) * x_ne / ggml_blck_size(src0->type), ctx->device.lock()->properties.limits.minStorageBufferOffsetAlignment); + const uint64_t qx_sz = ggml_vk_align_size(ggml_type_size(src0->type) * x_ne / ggml_blck_size(src0->type), ctx->device->properties.limits.minStorageBufferOffsetAlignment); const uint64_t qy_sz = ggml_type_size(src1->type) * y_ne / ggml_blck_size(src1->type); const uint64_t d_sz = sizeof(float) * d_ne; @@ -2710,12 +2808,12 @@ static void ggml_vk_mul_mat_vec_p021_f16_f32(ggml_backend_vk_context * ctx, vk_c } // Allocate descriptor sets - ggml_pipeline_allocate_descriptor_sets(ctx, ctx->pipeline_mul_mat_vec_p021_f16_f32, 1); + ggml_pipeline_allocate_descriptor_sets(ctx, ctx->device->pipeline_mul_mat_vec_p021_f16_f32, 1); - const uint64_t qy_buffer_offset = (qy_buf_offset / ctx->device.lock()->properties.limits.minStorageBufferOffsetAlignment) * ctx->device.lock()->properties.limits.minStorageBufferOffsetAlignment; + const uint64_t qy_buffer_offset = (qy_buf_offset / ctx->device->properties.limits.minStorageBufferOffsetAlignment) * ctx->device->properties.limits.minStorageBufferOffsetAlignment; const uint64_t qy_shader_offset = qy_buf_offset - qy_buffer_offset; - const uint64_t d_buffer_offset = (d_buf_offset / ctx->device.lock()->properties.limits.minStorageBufferOffsetAlignment) * ctx->device.lock()->properties.limits.minStorageBufferOffsetAlignment; + const uint64_t d_buffer_offset = (d_buf_offset / ctx->device->properties.limits.minStorageBufferOffsetAlignment) * ctx->device->properties.limits.minStorageBufferOffsetAlignment; const uint64_t d_shader_offset = d_buf_offset - d_buffer_offset; if (load_y) { @@ -2725,7 +2823,7 @@ static void ggml_vk_mul_mat_vec_p021_f16_f32(ggml_backend_vk_context * ctx, vk_c // compute const std::array pc = { (uint32_t)ne00, (uint32_t)ne01, (uint32_t)ne02, (uint32_t)ne12, (uint32_t)(qy_shader_offset / ggml_type_size(src1->type)), (uint32_t)(d_shader_offset / ggml_type_size(dst->type)) }; ggml_vk_sync_buffers(subctx); - ggml_vk_dispatch_pipeline(ctx, subctx, ctx->pipeline_mul_mat_vec_p021_f16_f32, { { d_Qx, qx_buf_offset, qx_sz }, { d_Qy, qy_buffer_offset, qy_sz + qy_shader_offset }, { d_D, d_buffer_offset, d_sz + d_shader_offset } }, 6 * sizeof(uint32_t), &pc, { 1, (uint32_t)ne01, (uint32_t)ne12 }); + ggml_vk_dispatch_pipeline(ctx, subctx, ctx->device->pipeline_mul_mat_vec_p021_f16_f32, { { d_Qx, qx_buf_offset, qx_sz }, { d_Qy, qy_buffer_offset, qy_sz + qy_shader_offset }, { d_D, d_buffer_offset, d_sz + d_shader_offset } }, 6 * sizeof(uint32_t), &pc, { 1, (uint32_t)ne01, (uint32_t)ne12 }); if (dst->backend == GGML_BACKEND_TYPE_CPU) { // copy dst to host @@ -2772,7 +2870,7 @@ static void ggml_vk_mul_mat_vec_nc_f16_f32(ggml_backend_vk_context * ctx, vk_con bool src1_uma = false; - if (ctx->device.lock()->uma) { + if (ctx->device->uma) { ggml_vk_host_get(ctx, src1->data, d_Qy, qy_buf_offset); src1_uma = d_Qy != nullptr; } @@ -2803,12 +2901,12 @@ static void ggml_vk_mul_mat_vec_nc_f16_f32(ggml_backend_vk_context * ctx, vk_con } // Allocate descriptor sets - ggml_pipeline_allocate_descriptor_sets(ctx, ctx->pipeline_mul_mat_vec_nc_f16_f32, 1); + ggml_pipeline_allocate_descriptor_sets(ctx, ctx->device->pipeline_mul_mat_vec_nc_f16_f32, 1); - const uint64_t qy_buffer_offset = (qy_buf_offset / ctx->device.lock()->properties.limits.minStorageBufferOffsetAlignment) * ctx->device.lock()->properties.limits.minStorageBufferOffsetAlignment; + const uint64_t qy_buffer_offset = (qy_buf_offset / ctx->device->properties.limits.minStorageBufferOffsetAlignment) * ctx->device->properties.limits.minStorageBufferOffsetAlignment; const uint64_t qy_shader_offset = qy_buf_offset - qy_buffer_offset; - const uint64_t d_buffer_offset = (d_buf_offset / ctx->device.lock()->properties.limits.minStorageBufferOffsetAlignment) * ctx->device.lock()->properties.limits.minStorageBufferOffsetAlignment; + const uint64_t d_buffer_offset = (d_buf_offset / ctx->device->properties.limits.minStorageBufferOffsetAlignment) * ctx->device->properties.limits.minStorageBufferOffsetAlignment; const uint64_t d_shader_offset = d_buf_offset - d_buffer_offset; if (load_y) { @@ -2818,7 +2916,7 @@ static void ggml_vk_mul_mat_vec_nc_f16_f32(ggml_backend_vk_context * ctx, vk_con // compute const std::array pc = { (uint32_t)ne00, (uint32_t)ne01, row_stride_x, channel_stride_x, (uint32_t)(ne12 / ne02), (uint32_t)(qy_shader_offset / ggml_type_size(src1->type)), (uint32_t)(d_shader_offset / ggml_type_size(dst->type)) }; ggml_vk_sync_buffers(subctx); - ggml_vk_dispatch_pipeline(ctx, subctx, ctx->pipeline_mul_mat_vec_nc_f16_f32, { { d_Qx, qx_buf_offset, qx_sz }, { d_Qy, qy_buffer_offset, qy_sz + qy_shader_offset }, { d_D, d_buffer_offset, d_sz + d_shader_offset } }, 7 * sizeof(uint32_t), &pc, { 1, (uint32_t)ne01, (uint32_t)ne12 }); + ggml_vk_dispatch_pipeline(ctx, subctx, ctx->device->pipeline_mul_mat_vec_nc_f16_f32, { { d_Qx, qx_buf_offset, qx_sz }, { d_Qy, qy_buffer_offset, qy_sz + qy_shader_offset }, { d_D, d_buffer_offset, d_sz + d_shader_offset } }, 7 * sizeof(uint32_t), &pc, { 1, (uint32_t)ne01, (uint32_t)ne12 }); if (dst->backend == GGML_BACKEND_TYPE_CPU) { // copy dst to host @@ -2856,6 +2954,10 @@ static void ggml_vk_mul_mat(ggml_backend_vk_context * ctx, vk_context * subctx, } } +// static void ggml_vk_mul_mat_id(ggml_backend_vk_context * ctx, vk_context * subctx, const struct ggml_tensor * src0, const struct ggml_tensor * src1, struct ggml_tensor * dst) { +// +// } + static void ggml_vk_op_repeat(ggml_backend_vk_context * ctx, vk_context * subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { // guaranteed to be an integer due to the check in ggml_can_repeat const uint64_t ne0 = dst->ne[0]; @@ -2927,40 +3029,40 @@ static void ggml_vk_op_repeat(ggml_backend_vk_context * ctx, vk_context * subctx } -static vk_pipeline* ggml_vk_op_get_pipeline(ggml_backend_vk_context * ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, ggml_op op) { +static vk_pipeline ggml_vk_op_get_pipeline(ggml_backend_vk_context * ctx, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, ggml_tensor * dst, ggml_op op) { switch (op) { case GGML_OP_ADD: if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { - return &ctx->pipeline_add_f32; + return ctx->device->pipeline_add_f32; } return nullptr; case GGML_OP_GET_ROWS: GGML_ASSERT(src1->type == GGML_TYPE_I32); if (dst->type == GGML_TYPE_F16) { - return &ctx->pipeline_get_rows[src0->type]; + return ctx->device->pipeline_get_rows[src0->type]; } if (dst->type == GGML_TYPE_F32) { - return &ctx->pipeline_get_rows_f32[src0->type]; + return ctx->device->pipeline_get_rows_f32[src0->type]; } return nullptr; case GGML_OP_MUL: if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { - return &ctx->pipeline_mul_f32; + return ctx->device->pipeline_mul_f32; } return nullptr; case GGML_OP_SCALE: if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { - return &ctx->pipeline_scale_f32; + return ctx->device->pipeline_scale_f32; } return nullptr; case GGML_OP_SQR: if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { - return &ctx->pipeline_sqr_f32; + return ctx->device->pipeline_sqr_f32; } return nullptr; case GGML_OP_CLAMP: if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { - return &ctx->pipeline_clamp_f32; + return ctx->device->pipeline_clamp_f32; } return nullptr; case GGML_OP_CPY: @@ -2969,29 +3071,29 @@ static vk_pipeline* ggml_vk_op_get_pipeline(ggml_backend_vk_context * ctx, const return ggml_vk_get_cpy_pipeline(ctx, src0->type, dst->type); case GGML_OP_NORM: if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { - return &ctx->pipeline_norm_f32; + return ctx->device->pipeline_norm_f32; } return nullptr; case GGML_OP_RMS_NORM: if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { - return &ctx->pipeline_rms_norm_f32; + return ctx->device->pipeline_rms_norm_f32; } return nullptr; case GGML_OP_UNARY: switch (ggml_get_unary_op(dst)) { case GGML_UNARY_OP_SILU: if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { - return &ctx->pipeline_silu_f32; + return ctx->device->pipeline_silu_f32; } break; case GGML_UNARY_OP_GELU: if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { - return &ctx->pipeline_gelu_f32; + return ctx->device->pipeline_gelu_f32; } break; case GGML_UNARY_OP_RELU: if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { - return &ctx->pipeline_relu_f32; + return ctx->device->pipeline_relu_f32; } break; default: @@ -3000,12 +3102,12 @@ static vk_pipeline* ggml_vk_op_get_pipeline(ggml_backend_vk_context * ctx, const return nullptr; case GGML_OP_DIAG_MASK_INF: if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { - return &ctx->pipeline_diag_mask_inf_f32; + return ctx->device->pipeline_diag_mask_inf_f32; } return nullptr; case GGML_OP_SOFT_MAX: - if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { - return &ctx->pipeline_soft_max_f32; + if (src0->type == GGML_TYPE_F32 && (src1 == nullptr || src1->type == GGML_TYPE_F32) && (src2 == nullptr || src2->type == GGML_TYPE_F32) && dst->type == GGML_TYPE_F32) { + return ctx->device->pipeline_soft_max_f32; } return nullptr; case GGML_OP_ROPE: @@ -3020,21 +3122,26 @@ static vk_pipeline* ggml_vk_op_get_pipeline(ggml_backend_vk_context * ctx, const if (is_neox) { if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { - return &ctx->pipeline_rope_neox_f32; + return ctx->device->pipeline_rope_neox_f32; } if (src0->type == GGML_TYPE_F16 && dst->type == GGML_TYPE_F16) { - return &ctx->pipeline_rope_neox_f16; + return ctx->device->pipeline_rope_neox_f16; } } else { if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { - return &ctx->pipeline_rope_f32; + return ctx->device->pipeline_rope_f32; } if (src0->type == GGML_TYPE_F16 && dst->type == GGML_TYPE_F16) { - return &ctx->pipeline_rope_f16; + return ctx->device->pipeline_rope_f16; } } return nullptr; } + case GGML_OP_ARGSORT: + if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_I32) { + return ctx->device->pipeline_argsort_f32; + } + return nullptr; default: return nullptr; } @@ -3050,17 +3157,19 @@ static ggml_vk_func_t ggml_vk_op_get_func(ggml_op op) { } template -static void ggml_vk_op_f32(ggml_backend_vk_context * ctx, vk_context * subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, ggml_op op, const PC&& pc) { +static void ggml_vk_op_f32(ggml_backend_vk_context * ctx, vk_context * subctx, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, ggml_tensor * dst, ggml_op op, const PC&& pc) { #ifdef GGML_VULKAN_DEBUG std::cerr << "ggml_vk_op_f32((" << src0 << ", name=" << src0->name << ", type=" << src0->type << ", backend=" << src0->backend << ", ne0=" << src0->ne[0] << ", ne1=" << src0->ne[1] << ", ne2=" << src0->ne[2] << ", ne3=" << src0->ne[3] << ", nb0=" << src0->nb[0] << ", nb1=" << src0->nb[1] << ", nb2=" << src0->nb[2] << ", nb3=" << src0->nb[3]; if (src1 != nullptr) { std::cerr << "), (" << src1 << ", name=" << src1->name << ", type=" << src1->type << ", backend=" << src1->backend << ", ne0=" << src1->ne[0] << ", ne1=" << src1->ne[1] << ", ne2=" << src1->ne[2] << ", ne3=" << src1->ne[3] << ", nb0=" << src1->nb[0] << ", nb1=" << src1->nb[1] << ", nb2=" << src1->nb[2] << ", nb3=" << src1->nb[3]; } + if (src2 != nullptr) { + std::cerr << "), (" << src2 << ", name=" << src2->name << ", type=" << src2->type << ", backend=" << src2->backend << ", ne0=" << src2->ne[0] << ", ne1=" << src2->ne[1] << ", ne2=" << src2->ne[2] << ", ne3=" << src2->ne[3] << ", nb0=" << src2->nb[0] << ", nb1=" << src2->nb[1] << ", nb2=" << src2->nb[2] << ", nb3=" << src2->nb[3]; + } std::cerr << "), (" << dst << ", name=" << dst->name << ", type=" << dst->type << ", backend=" << dst->backend << ", ne0=" << dst->ne[0] << ", ne1=" << dst->ne[1] << ", ne2=" << dst->ne[2] << ", ne3=" << dst->ne[3] << ", nb0=" << dst->nb[0] << ", nb1=" << dst->nb[1] << ", nb2=" << dst->nb[2] << ", nb3=" << dst->nb[3] << "), " << ggml_op_name(op) << ")" << std::endl; #endif GGML_ASSERT(!ggml_is_quantized(src0->type) && (src1 == nullptr || !ggml_is_quantized(src1->type))); // NOLINT GGML_ASSERT(op == GGML_OP_CPY || ggml_vk_dim01_contiguous(src0)); // NOLINT - GGML_ASSERT(src1 == nullptr || ggml_vk_dim01_contiguous(src1)); // NOLINT GGML_ASSERT(dst->extra != nullptr); const uint64_t ne00 = src0->ne[0]; const uint64_t ne01 = src0->ne[1]; @@ -3077,7 +3186,10 @@ static void ggml_vk_op_f32(ggml_backend_vk_context * ctx, vk_context * subctx, c const uint64_t nb2 = dst->nb[2]; const uint64_t nb3 = dst->nb[3]; - vk_pipeline * pipeline = ggml_vk_op_get_pipeline(ctx, src0, src1, dst, op); + const bool use_src2 = src2 != nullptr; + const uint64_t ne2 = use_src2 ? src2->ne[0] * src2->ne[1] : 0; + + vk_pipeline pipeline = ggml_vk_op_get_pipeline(ctx, src0, src1, src2, dst, op); ggml_vk_func_t op_func; if (pipeline == nullptr) { @@ -3098,29 +3210,39 @@ static void ggml_vk_op_f32(ggml_backend_vk_context * ctx, vk_context * subctx, c ggml_tensor_extra_gpu * extra = (ggml_tensor_extra_gpu *) dst->extra; ggml_tensor_extra_gpu * extra_src0 = (ggml_tensor_extra_gpu *) src0->extra; ggml_tensor_extra_gpu * extra_src1 = use_src1 ? (ggml_tensor_extra_gpu *) src1->extra : nullptr; + ggml_tensor_extra_gpu * extra_src2 = use_src2 ? (ggml_tensor_extra_gpu *) src2->extra : nullptr; vk_buffer d_X = nullptr; size_t x_buf_offset = 0; vk_buffer d_Y = nullptr; size_t y_buf_offset = 0; + vk_buffer d_Z = nullptr; + size_t z_buf_offset = 0; bool src0_uma = false; bool src1_uma = false; + bool src2_uma = false; - if (ctx->device.lock()->uma) { + if (ctx->device->uma) { ggml_vk_host_get(ctx, src0->data, d_X, x_buf_offset); src0_uma = d_X != nullptr; if (use_src1) { ggml_vk_host_get(ctx, src1->data, d_Y, y_buf_offset); src1_uma = d_Y != nullptr; } + if (use_src2) { + ggml_vk_host_get(ctx, src1->data, d_Z, z_buf_offset); + src2_uma = d_Z != nullptr; + } } const bool transfer_src0 = src0->backend != GGML_BACKEND_TYPE_GPU && !src0_uma; const bool transfer_src1 = use_src1 && src1->backend != GGML_BACKEND_TYPE_GPU && !src1_uma; + const bool transfer_src2 = use_src2 && src2->backend != GGML_BACKEND_TYPE_GPU && !src2_uma; - uint64_t x_sz = ggml_vk_align_size(ggml_type_size(src0->type) * ne0, ctx->device.lock()->properties.limits.minStorageBufferOffsetAlignment); - uint64_t y_sz = use_src1 ? ggml_vk_align_size(ggml_type_size(src1->type) * ne1, ctx->device.lock()->properties.limits.minStorageBufferOffsetAlignment) : 0; + uint64_t x_sz = ggml_vk_align_size(ggml_type_size(src0->type) * ne0, ctx->device->properties.limits.minStorageBufferOffsetAlignment); + uint64_t y_sz = use_src1 ? ggml_vk_align_size(ggml_type_size(src1->type) * ne1, ctx->device->properties.limits.minStorageBufferOffsetAlignment) : 0; + uint64_t z_sz = use_src2 ? ggml_vk_align_size(ggml_type_size(src2->type) * ne2, ctx->device->properties.limits.minStorageBufferOffsetAlignment) : 0; uint64_t d_sz = ggml_type_size(dst->type) * ne0; vk_buffer d_D = extra->buffer_gpu.lock(); @@ -3131,7 +3253,7 @@ static void ggml_vk_op_f32(ggml_backend_vk_context * ctx, vk_context * subctx, c } GGML_ASSERT(d_D != nullptr); - uint64_t d_buf_offset = (extra->offset / ctx->device.lock()->properties.limits.minStorageBufferOffsetAlignment) * ctx->device.lock()->properties.limits.minStorageBufferOffsetAlignment; + uint64_t d_buf_offset = (extra->offset / ctx->device->properties.limits.minStorageBufferOffsetAlignment) * ctx->device->properties.limits.minStorageBufferOffsetAlignment; GGML_ASSERT(d_buf_offset == extra->offset || op == GGML_OP_CPY); // NOLINT if (transfer_src0) { d_X = ctx->prealloc_qx; @@ -3148,6 +3270,13 @@ static void ggml_vk_op_f32(ggml_backend_vk_context * ctx, vk_context * subctx, c GGML_ASSERT(d_Y != nullptr); } + GGML_ASSERT(!transfer_src2); + if (use_src2 && !src2_uma) { + d_Z = extra_src2->buffer_gpu.lock(); + z_buf_offset = extra_src2->offset; + GGML_ASSERT(d_Z != nullptr); + } + if (op == GGML_OP_CPY) { GGML_ASSERT(!transfer_src0); GGML_ASSERT(!transfer_src1); @@ -3175,7 +3304,7 @@ static void ggml_vk_op_f32(ggml_backend_vk_context * ctx, vk_context * subctx, c // Single call if dimension 2 is contiguous if (op == GGML_OP_CPY || (ggml_is_contiguous(src0) && (src1 == nullptr || ggml_is_contiguous(src1)))) { - ggml_pipeline_allocate_descriptor_sets(ctx, *pipeline, 1); + ggml_pipeline_allocate_descriptor_sets(ctx, pipeline, 1); switch (dst->op) { case GGML_OP_NORM: @@ -3204,16 +3333,30 @@ static void ggml_vk_op_f32(ggml_backend_vk_context * ctx, vk_context * subctx, c } } - if (!use_src1 && op == GGML_OP_SOFT_MAX) { - // Empty src1 is possible on soft_max, but the shader needs a buffer + if (op == GGML_OP_SOFT_MAX) { + // Empty src1 and src2 are possible on soft_max, but the shader needs buffers + vk_subbuffer subbuf_y; + if (use_src1) { + subbuf_y = { d_Y, y_buf_offset, y_sz }; + } else { + subbuf_y = { ctx->prealloc_y, 0, ctx->prealloc_y->size }; + } + + vk_subbuffer subbuf_z; + if (use_src2) { + subbuf_z = { d_Z, z_buf_offset, z_sz }; + } else { + subbuf_z = { ctx->prealloc_y, 0, ctx->prealloc_y->size }; + } + ggml_vk_sync_buffers(subctx); - ggml_vk_dispatch_pipeline(ctx, subctx, *pipeline, { { d_X, x_buf_offset, x_sz }, { ctx->prealloc_y, 0, ctx->prealloc_y->size }, { d_D, d_buf_offset, d_sz } }, sizeof(PC), &pc, elements); + ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, { { d_X, x_buf_offset, x_sz }, subbuf_y, subbuf_z, { d_D, d_buf_offset, d_sz } }, sizeof(PC), &pc, elements); } else if (use_src1) { ggml_vk_sync_buffers(subctx); - ggml_vk_dispatch_pipeline(ctx, subctx, *pipeline, { { d_X, x_buf_offset, x_sz }, { d_Y, y_buf_offset, y_sz }, { d_D, d_buf_offset, d_sz } }, sizeof(PC), &pc, elements); + ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, { { d_X, x_buf_offset, x_sz }, { d_Y, y_buf_offset, y_sz }, { d_D, d_buf_offset, d_sz } }, sizeof(PC), &pc, elements); } else { ggml_vk_sync_buffers(subctx); - ggml_vk_dispatch_pipeline(ctx, subctx, *pipeline, { { d_X, x_buf_offset, x_sz }, { d_D, d_buf_offset, d_sz } }, sizeof(PC), &pc, elements); + ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, { { d_X, x_buf_offset, x_sz }, { d_D, d_buf_offset, d_sz } }, sizeof(PC), &pc, elements); } if (dst->backend == GGML_BACKEND_TYPE_CPU && op == GGML_OP_CPY) { ggml_vk_d2h_tensor_2d(ctx, subctx, d_D, 0, dst); @@ -3223,7 +3366,9 @@ static void ggml_vk_op_f32(ggml_backend_vk_context * ctx, vk_context * subctx, c ggml_vk_buffer_read_async(ctx, subctx, d_D, 0, d, d_sz); } } else { - ggml_pipeline_allocate_descriptor_sets(ctx, *pipeline, ne02 * ne03); + GGML_ASSERT(op != GGML_OP_SOFT_MAX); + + ggml_pipeline_allocate_descriptor_sets(ctx, pipeline, ne02 * ne03); switch (dst->op) { case GGML_OP_NORM: @@ -3248,16 +3393,12 @@ static void ggml_vk_op_f32(ggml_backend_vk_context * ctx, vk_context * subctx, c const uint32_t y_offset = y_sz * it_idx1; const uint32_t d_offset = d_sz * it_idx0; - if (!use_src1 && op == GGML_OP_SOFT_MAX) { - // Empty src1 is possible on soft_max, but the shader needs a buffer + if (use_src1) { ggml_vk_sync_buffers(subctx); - ggml_vk_dispatch_pipeline(ctx, subctx, *pipeline, { { d_X, x_buf_offset, x_sz }, { ctx->prealloc_y, 0, ctx->prealloc_y->size }, { d_D, d_buf_offset, d_sz } }, sizeof(PC), &pc, elements); - } else if (use_src1) { - ggml_vk_sync_buffers(subctx); - ggml_vk_dispatch_pipeline(ctx, subctx, *pipeline, { { d_X, x_buf_offset + x_offset, x_sz }, { d_Y, y_buf_offset + y_offset, y_sz }, { d_D, d_buf_offset + d_offset, d_sz } }, sizeof(PC), &pc, elements); + ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, { { d_X, x_buf_offset + x_offset, x_sz }, { d_Y, y_buf_offset + y_offset, y_sz }, { d_D, d_buf_offset + d_offset, d_sz } }, sizeof(PC), &pc, elements); } else { ggml_vk_sync_buffers(subctx); - ggml_vk_dispatch_pipeline(ctx, subctx, *pipeline, { { d_X, x_buf_offset + x_offset, x_sz }, { d_D, d_buf_offset + d_offset, d_sz } }, sizeof(PC), &pc, elements); + ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, { { d_X, x_buf_offset + x_offset, x_sz }, { d_D, d_buf_offset + d_offset, d_sz } }, sizeof(PC), &pc, elements); } if (dst->backend == GGML_BACKEND_TYPE_CPU) { // copy dst to host @@ -3269,69 +3410,141 @@ static void ggml_vk_op_f32(ggml_backend_vk_context * ctx, vk_context * subctx, c } static void ggml_vk_repeat(ggml_backend_vk_context * ctx, vk_context * subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { - ggml_vk_op_f32(ctx, subctx, src0, src1, dst, GGML_OP_REPEAT, { (uint32_t)ggml_nelements(src0), (uint32_t)ggml_nelements(src1), 0.0f, 0.0f }); + ggml_vk_op_f32(ctx, subctx, src0, src1, nullptr, dst, GGML_OP_REPEAT, { (uint32_t)ggml_nelements(src0), (uint32_t)ggml_nelements(src1), 0.0f, 0.0f }); } static void ggml_vk_get_rows(ggml_backend_vk_context * ctx, vk_context * subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { - ggml_vk_op_f32(ctx, subctx, src0, src1, dst, GGML_OP_GET_ROWS, { (uint32_t)ggml_nelements(src0), (uint32_t)ggml_nelements(src1), 0.0f, 0.0f }); + ggml_vk_op_f32(ctx, subctx, src0, src1, nullptr, dst, GGML_OP_GET_ROWS, { (uint32_t)ggml_nelements(src0), (uint32_t)ggml_nelements(src1), 0.0f, 0.0f }); } static void ggml_vk_add(ggml_backend_vk_context * ctx, vk_context * subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { - ggml_vk_op_f32(ctx, subctx, src0, src1, dst, GGML_OP_ADD, { (uint32_t)ggml_nelements(src0), (uint32_t)ggml_nelements(src1), 0.0f, 0.0f }); + const uint32_t src0_type_size = ggml_type_size(src0->type); + const uint32_t src1_type_size = ggml_type_size(src1->type); + const uint32_t dst_type_size = ggml_type_size(dst->type); + + ggml_vk_op_f32(ctx, subctx, src0, src1, nullptr, dst, GGML_OP_ADD, { + (uint32_t)ggml_nelements(src0), + (uint32_t)src0->ne[0], (uint32_t)src0->ne[1], (uint32_t)src0->ne[2],(uint32_t)src0->ne[3], (uint32_t)src0->nb[0] / src0_type_size, (uint32_t)src0->nb[1] / src0_type_size, (uint32_t)src0->nb[2] / src0_type_size, (uint32_t)src0->nb[3] / src0_type_size, + (uint32_t)src1->ne[0], (uint32_t)src1->ne[1], (uint32_t)src1->ne[2],(uint32_t)src1->ne[3], (uint32_t)src1->nb[0] / src1_type_size, (uint32_t)src1->nb[1] / src1_type_size, (uint32_t)src1->nb[2] / src1_type_size, (uint32_t)src1->nb[3] / src1_type_size, + (uint32_t) dst->ne[0], (uint32_t) dst->ne[1], (uint32_t) dst->ne[2],(uint32_t) dst->ne[3], (uint32_t) dst->nb[0] / dst_type_size, (uint32_t) dst->nb[1] / dst_type_size, (uint32_t) dst->nb[2] / dst_type_size, (uint32_t) dst->nb[3] / dst_type_size, + 0, + 0.0f, 0.0f, + }); } static void ggml_vk_mul(ggml_backend_vk_context * ctx, vk_context * subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { - ggml_vk_op_f32(ctx, subctx, src0, src1, dst, GGML_OP_MUL, { (uint32_t)ggml_nelements(src0), (uint32_t)ggml_nelements(src1), 0.0f, 0.0f }); + const uint32_t src0_type_size = ggml_type_size(src0->type); + const uint32_t src1_type_size = ggml_type_size(src1->type); + const uint32_t dst_type_size = ggml_type_size(dst->type); + + ggml_vk_op_f32(ctx, subctx, src0, src1, nullptr, dst, GGML_OP_MUL, { + (uint32_t)ggml_nelements(src0), + (uint32_t)src0->ne[0], (uint32_t)src0->ne[1], (uint32_t)src0->ne[2],(uint32_t)src0->ne[3], (uint32_t)src0->nb[0] / src0_type_size, (uint32_t)src0->nb[1] / src0_type_size, (uint32_t)src0->nb[2] / src0_type_size, (uint32_t)src0->nb[3] / src0_type_size, + (uint32_t)src1->ne[0], (uint32_t)src1->ne[1], (uint32_t)src1->ne[2],(uint32_t)src1->ne[3], (uint32_t)src1->nb[0] / src1_type_size, (uint32_t)src1->nb[1] / src1_type_size, (uint32_t)src1->nb[2] / src1_type_size, (uint32_t)src1->nb[3] / src1_type_size, + (uint32_t) dst->ne[0], (uint32_t) dst->ne[1], (uint32_t) dst->ne[2],(uint32_t) dst->ne[3], (uint32_t) dst->nb[0] / dst_type_size, (uint32_t) dst->nb[1] / dst_type_size, (uint32_t) dst->nb[2] / dst_type_size, (uint32_t) dst->nb[3] / dst_type_size, + 0, + 0.0f, 0.0f, + }); } static void ggml_vk_scale(ggml_backend_vk_context * ctx, vk_context * subctx, const ggml_tensor * src0, ggml_tensor * dst) { float * op_params = (float *)dst->op_params; - ggml_vk_op_f32(ctx, subctx, src0, nullptr, dst, GGML_OP_SCALE, { (uint32_t)ggml_nelements(src0), 0, op_params[0], 0.0f }); + const uint32_t src0_type_size = ggml_type_size(src0->type); + const uint32_t dst_type_size = ggml_type_size(dst->type); + + ggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, dst, GGML_OP_SCALE, { + (uint32_t)ggml_nelements(src0), + (uint32_t)src0->ne[0], (uint32_t)src0->ne[1], (uint32_t)src0->ne[2], (uint32_t)src0->ne[3], (uint32_t)src0->nb[0] / src0_type_size, (uint32_t)src0->nb[1] / src0_type_size, (uint32_t)src0->nb[2] / src0_type_size, (uint32_t)src0->nb[3] / src0_type_size, + (uint32_t) dst->ne[0], (uint32_t) dst->ne[1], (uint32_t) dst->ne[2], (uint32_t) dst->ne[3], (uint32_t) dst->nb[0] / dst_type_size, (uint32_t) dst->nb[1] / dst_type_size, (uint32_t) dst->nb[2] / dst_type_size, (uint32_t) dst->nb[3] / dst_type_size, + 0, + op_params[0], 0.0f + }); } static void ggml_vk_sqr(ggml_backend_vk_context * ctx, vk_context * subctx, const ggml_tensor * src0, ggml_tensor * dst) { - ggml_vk_op_f32(ctx, subctx, src0, nullptr, dst, GGML_OP_SQR, { (uint32_t)ggml_nelements(src0), 0, 0.0f, 0.0f }); + const uint32_t src0_type_size = ggml_type_size(src0->type); + const uint32_t dst_type_size = ggml_type_size(dst->type); + + ggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, dst, GGML_OP_SQR, { + (uint32_t)ggml_nelements(src0), + (uint32_t)src0->ne[0], (uint32_t)src0->ne[1], (uint32_t)src0->ne[2], (uint32_t)src0->ne[3], (uint32_t)src0->nb[0] / src0_type_size, (uint32_t)src0->nb[1] / src0_type_size, (uint32_t)src0->nb[2] / src0_type_size, (uint32_t)src0->nb[3] / src0_type_size, + (uint32_t) dst->ne[0], (uint32_t) dst->ne[1], (uint32_t) dst->ne[2], (uint32_t) dst->ne[3], (uint32_t) dst->nb[0] / dst_type_size, (uint32_t) dst->nb[1] / dst_type_size, (uint32_t) dst->nb[2] / dst_type_size, (uint32_t) dst->nb[3] / dst_type_size, + 0, + 0.0f, 0.0f, + }); } static void ggml_vk_clamp(ggml_backend_vk_context * ctx, vk_context * subctx, const ggml_tensor * src0, ggml_tensor * dst) { float * op_params = (float *)dst->op_params; - ggml_vk_op_f32(ctx, subctx, src0, nullptr, dst, GGML_OP_CLAMP, { (uint32_t)ggml_nelements(src0), 0, op_params[0], op_params[1] }); + const uint32_t src0_type_size = ggml_type_size(src0->type); + const uint32_t dst_type_size = ggml_type_size(dst->type); + + ggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, dst, GGML_OP_CLAMP, { + (uint32_t)ggml_nelements(src0), + (uint32_t)src0->ne[0], (uint32_t)src0->ne[1], (uint32_t)src0->ne[2], (uint32_t)src0->ne[3], (uint32_t)src0->nb[0] / src0_type_size, (uint32_t)src0->nb[1] / src0_type_size, (uint32_t)src0->nb[2] / src0_type_size, (uint32_t)src0->nb[3] / src0_type_size, + (uint32_t) dst->ne[0], (uint32_t) dst->ne[1], (uint32_t) dst->ne[2], (uint32_t) dst->ne[3], (uint32_t) dst->nb[0] / dst_type_size, (uint32_t) dst->nb[1] / dst_type_size, (uint32_t) dst->nb[2] / dst_type_size, (uint32_t) dst->nb[3] / dst_type_size, + 0, + op_params[0], op_params[1], + }); } static void ggml_vk_cpy(ggml_backend_vk_context * ctx, vk_context * subctx, const ggml_tensor * src0, ggml_tensor * dst) { ggml_tensor_extra_gpu * extra = (ggml_tensor_extra_gpu *) dst->extra; - const int src0_type_size = ggml_type_size(src0->type); - const int dst_type_size = ggml_type_size(dst->type); - const uint32_t d_offset = (extra->offset % ctx->device.lock()->properties.limits.minStorageBufferOffsetAlignment) / dst_type_size; - ggml_vk_op_f32(ctx, subctx, src0, nullptr, dst, GGML_OP_CPY, { + const uint32_t src0_type_size = ggml_type_size(src0->type); + const uint32_t dst_type_size = ggml_type_size(dst->type); + const uint32_t d_offset = (extra->offset % ctx->device->properties.limits.minStorageBufferOffsetAlignment) / dst_type_size; + + ggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, dst, GGML_OP_CPY, { (uint32_t)ggml_nelements(src0), - (uint32_t)src0->ne[0], (uint32_t)src0->ne[1], (uint32_t)src0->nb[0] / src0_type_size, (uint32_t)src0->nb[1] / src0_type_size, (uint32_t)src0->nb[2] / src0_type_size, - (uint32_t) dst->ne[0], (uint32_t) dst->ne[1], (uint32_t) dst->nb[0] / dst_type_size, (uint32_t) dst->nb[1] / dst_type_size, (uint32_t) dst->nb[2] / dst_type_size, + (uint32_t)src0->ne[0], (uint32_t)src0->ne[1], (uint32_t)src0->ne[2], (uint32_t)src0->ne[3], (uint32_t)src0->nb[0] / src0_type_size, (uint32_t)src0->nb[1] / src0_type_size, (uint32_t)src0->nb[2] / src0_type_size, (uint32_t)src0->nb[3] / src0_type_size, + (uint32_t) dst->ne[0], (uint32_t) dst->ne[1], (uint32_t) dst->ne[2], (uint32_t) dst->ne[3], (uint32_t) dst->nb[0] / dst_type_size, (uint32_t) dst->nb[1] / dst_type_size, (uint32_t) dst->nb[2] / dst_type_size, (uint32_t) dst->nb[3] / dst_type_size, d_offset, + 0.0f, 0.0f, }); } static void ggml_vk_norm(ggml_backend_vk_context * ctx, vk_context * subctx, const ggml_tensor * src0, ggml_tensor * dst) { - ggml_vk_op_f32(ctx, subctx, src0, nullptr, dst, GGML_OP_NORM, { (uint32_t)src0->ne[0], (uint32_t)src0->ne[1], 0.0f, 0.0f }); + ggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, dst, GGML_OP_NORM, { (uint32_t)src0->ne[0], (uint32_t)src0->ne[1], 0.0f, 0.0f }); } static void ggml_vk_rms_norm(ggml_backend_vk_context * ctx, vk_context * subctx, const ggml_tensor * src0, ggml_tensor * dst) { float * op_params = (float *)dst->op_params; - ggml_vk_op_f32(ctx, subctx, src0, nullptr, dst, GGML_OP_RMS_NORM, { (uint32_t)src0->ne[0], (uint32_t)src0->ne[1], op_params[0], 0.0f }); + ggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, dst, GGML_OP_RMS_NORM, { (uint32_t)src0->ne[0], (uint32_t)src0->ne[1], op_params[0], 0.0f }); } static void ggml_vk_unary(ggml_backend_vk_context * ctx, vk_context * subctx, const ggml_tensor * src0, ggml_tensor * dst) { - ggml_vk_op_f32(ctx, subctx, src0, nullptr, dst, GGML_OP_UNARY, { (uint32_t)ggml_nelements(src0), 0, 0.0f, 0.0f }); + ggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, dst, GGML_OP_UNARY, { (uint32_t)ggml_nelements(src0), 0, 0.0f, 0.0f }); } static void ggml_vk_diag_mask_inf(ggml_backend_vk_context * ctx, vk_context * subctx, const ggml_tensor * src0, ggml_tensor * dst) { int32_t * op_params = (int32_t *)dst->op_params; - ggml_vk_op_f32(ctx, subctx, src0, nullptr, dst, GGML_OP_DIAG_MASK_INF, { (uint32_t)src0->ne[0], (uint32_t)src0->ne[1], op_params[0] }); + ggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, dst, GGML_OP_DIAG_MASK_INF, { (uint32_t)src0->ne[0], (uint32_t)src0->ne[1], op_params[0] }); } -static void ggml_vk_soft_max(ggml_backend_vk_context * ctx, vk_context * subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { +static void ggml_vk_soft_max(ggml_backend_vk_context * ctx, vk_context * subctx, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, ggml_tensor * dst) { float * op_params = (float *)dst->op_params; - ggml_vk_op_f32(ctx, subctx, src0, src1, dst, GGML_OP_SOFT_MAX, { (uint32_t)src0->ne[0], (uint32_t)(src1 != nullptr ? ggml_nrows(src1) : 0), op_params[0], 0.0f }); + + float scale = op_params[0]; + float max_bias = op_params[1]; + + const uint32_t ncols = (uint32_t)src0->ne[0]; + const uint32_t nrows_x = (uint32_t)ggml_nrows(src0); + const uint32_t nrows_y = (uint32_t)src0->ne[1]; + + const uint32_t n_head_kv = nrows_x/nrows_y; + const uint32_t n_head_log2 = 1u << (uint32_t) floorf(log2f((float) n_head_kv)); + + const float m0 = powf(2.0f, -(max_bias ) / n_head_log2); + const float m1 = powf(2.0f, -(max_bias / 2.0f) / n_head_log2); + + ggml_vk_op_f32(ctx, subctx, src0, src1, src2, dst, GGML_OP_SOFT_MAX, { + ncols, + nrows_y, + src2 != nullptr ? (uint32_t)1 : (uint32_t)0, + scale, max_bias, + m0, m1, + n_head_log2, + }); } static void ggml_vk_rope(ggml_backend_vk_context * ctx, vk_context * subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { @@ -3357,12 +3570,17 @@ static void ggml_vk_rope(ggml_backend_vk_context * ctx, vk_context * subctx, con if (is_neox) { const float theta_scale = powf(freq_base, -2.0f/n_dims); const float inv_ndims = -1.0f / n_dims; - ggml_vk_op_f32(ctx, subctx, src0, src1, dst, GGML_OP_ROPE, { (uint32_t)src0->ne[0], (uint32_t)n_dims, freq_scale, (uint32_t)src0->ne[1], freq_base, ext_factor, attn_factor, corr_dims[0], corr_dims[1], 0.0f, 0.0f, theta_scale, inv_ndims }); + ggml_vk_op_f32(ctx, subctx, src0, src1, nullptr, dst, GGML_OP_ROPE, { (uint32_t)src0->ne[0], (uint32_t)n_dims, freq_scale, (uint32_t)src0->ne[1], freq_base, ext_factor, attn_factor, corr_dims[0], corr_dims[1], 0.0f, 0.0f, theta_scale, inv_ndims }); } else { - ggml_vk_op_f32(ctx, subctx, src0, src1, dst, GGML_OP_ROPE, { (uint32_t)src0->ne[0], freq_scale, (uint32_t)src0->ne[1], freq_base, ext_factor, attn_factor, corr_dims[0], corr_dims[1], 0.0f, 0.0f }); + ggml_vk_op_f32(ctx, subctx, src0, src1, nullptr, dst, GGML_OP_ROPE, { (uint32_t)src0->ne[0], freq_scale, (uint32_t)src0->ne[1], freq_base, ext_factor, attn_factor, corr_dims[0], corr_dims[1], 0.0f, 0.0f }); } } +static void ggml_vk_argsort(ggml_backend_vk_context * ctx, vk_context * subctx, const ggml_tensor * src0, ggml_tensor * dst) { + int32_t * op_params = (int32_t *)dst->op_params; + ggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, dst, GGML_OP_ARGSORT, { (uint32_t)src0->ne[0], ((ggml_sort_order) op_params[0]) == GGML_SORT_ORDER_ASC }); +} + static void ggml_vk_nop(ggml_backend_vk_context * ctx, vk_context * subctx, const ggml_tensor * src0, ggml_tensor * dst) { // If backend is CPU, data from src0 has to be copied off the device if (dst->backend == GGML_BACKEND_TYPE_CPU) { @@ -3414,43 +3632,43 @@ static void ggml_vk_test_matmul(ggml_backend_vk_context * ctx, size_t m, size_t const size_t y_ne = k * n * batch; const size_t d_ne = m * n * batch; - vk_pipeline * p; + vk_pipeline p; std::string shname; if (shader_size == 0) { if (std::is_same() && std::is_same()) { - p = &ctx->pipeline_matmul_f32_aligned_s; + p = ctx->device->pipeline_matmul_f32->a_s; shname = "F32_ALIGNED_S"; } else if (std::is_same() && std::is_same()) { - p = &ctx->pipeline_matmul_f16_f32_aligned_s; + p = ctx->device->pipeline_matmul_f16_f32->a_s; shname = "F16_F32_ALIGNED_S"; } else if (std::is_same() && std::is_same()) { - p = &ctx->pipeline_matmul_f16_aligned_s; + p = ctx->device->pipeline_matmul_f16->a_s; shname = "F16_ALIGNED_S"; } else { GGML_ASSERT(false); } } else if (shader_size == 1) { if (std::is_same() && std::is_same()) { - p = &ctx->pipeline_matmul_f32_aligned_m; + p = ctx->device->pipeline_matmul_f32->a_m; shname = "F32_ALIGNED_M"; } else if (std::is_same() && std::is_same()) { - p = &ctx->pipeline_matmul_f16_f32_aligned_m; + p = ctx->device->pipeline_matmul_f16_f32->a_m; shname = "F16_F32_ALIGNED_M"; } else if (std::is_same() && std::is_same()) { - p = &ctx->pipeline_matmul_f16_aligned_m; + p = ctx->device->pipeline_matmul_f16->a_m; shname = "F16_ALIGNED_M"; } else { GGML_ASSERT(false); } } else if (shader_size == 2) { if (std::is_same() && std::is_same()) { - p = &ctx->pipeline_matmul_f32_aligned_l; + p = ctx->device->pipeline_matmul_f32->a_l; shname = "F32_ALIGNED_L"; } else if (std::is_same() && std::is_same()) { - p = &ctx->pipeline_matmul_f16_f32_aligned_l; + p = ctx->device->pipeline_matmul_f16_f32->a_l; shname = "F16_F32_ALIGNED_L"; } else if (std::is_same() && std::is_same()) { - p = &ctx->pipeline_matmul_f16_aligned_l; + p = ctx->device->pipeline_matmul_f16->a_l; shname = "F16_ALIGNED_L"; } else { GGML_ASSERT(false); @@ -3464,43 +3682,43 @@ static void ggml_vk_test_matmul(ggml_backend_vk_context * ctx, size_t m, size_t if (k != kpad) { if (shader_size == 0) { if (std::is_same() && std::is_same()) { - p = &ctx->pipeline_matmul_f32_s; + p = ctx->device->pipeline_matmul_f32->s; shname = "F32_S"; } else if (std::is_same() && std::is_same()) { - p = &ctx->pipeline_matmul_f16_f32_s; + p = ctx->device->pipeline_matmul_f16_f32->s; shname = "F16_F32_S"; } else if (std::is_same() && std::is_same()) { - p = &ctx->pipeline_matmul_f16_s; + p = ctx->device->pipeline_matmul_f16->s; shname = "F16_S"; } } else if (shader_size == 1) { if (std::is_same() && std::is_same()) { - p = &ctx->pipeline_matmul_f32_m; + p = ctx->device->pipeline_matmul_f32->m; shname = "F32_M"; } else if (std::is_same() && std::is_same()) { - p = &ctx->pipeline_matmul_f16_f32_m; + p = ctx->device->pipeline_matmul_f16_f32->m; shname = "F16_F32_M"; } else if (std::is_same() && std::is_same()) { - p = &ctx->pipeline_matmul_f16_m; + p = ctx->device->pipeline_matmul_f16->m; shname = "F16_M"; } } else if (shader_size == 2) { if (std::is_same() && std::is_same()) { - p = &ctx->pipeline_matmul_f32_l; + p = ctx->device->pipeline_matmul_f32->l; shname = "F32_L"; } else if (std::is_same() && std::is_same()) { - p = &ctx->pipeline_matmul_f16_f32_l; + p = ctx->device->pipeline_matmul_f16_f32->l; shname = "F16_F32_L"; } else if (std::is_same() && std::is_same()) { - p = &ctx->pipeline_matmul_f16_l; + p = ctx->device->pipeline_matmul_f16->l; shname = "F16_L"; } } } - ggml_pipeline_allocate_descriptor_sets(ctx, *p, num_it); + ggml_pipeline_allocate_descriptor_sets(ctx, p, num_it); if (split_k > 1) { - ggml_pipeline_allocate_descriptor_sets(ctx, ctx->pipeline_matmul_split_k_reduce, num_it); + ggml_pipeline_allocate_descriptor_sets(ctx, ctx->device->pipeline_matmul_split_k_reduce, num_it); if (ctx->prealloc_split_k == nullptr || ctx->prealloc_split_k->size < sizeof(float) * d_ne * split_k) { // Resize buffer @@ -3530,9 +3748,11 @@ static void ggml_vk_test_matmul(ggml_backend_vk_context * ctx, size_t m, size_t } for (size_t i = 0; i < y_ne; i++) { if (std::is_same()) { - y[i] = (rand() / (float)RAND_MAX) * 2.0f - 1.0f; + // y[i] = (rand() / (float)RAND_MAX) * 2.0f - 1.0f; + y[i] = (i % k == i / k) ? 1.0f : 0.0f; } else if (std::is_same()) { - y[i] = ggml_fp32_to_fp16((rand() / (float)RAND_MAX) * 2.0f - 1.0f); + // y[i] = ggml_fp32_to_fp16((rand() / (float)RAND_MAX) * 2.0f - 1.0f); + y[i] = ggml_fp32_to_fp16((i % k == i / k) ? 1.0f : 0.0f); } else { GGML_ASSERT(false); } @@ -3541,17 +3761,17 @@ static void ggml_vk_test_matmul(ggml_backend_vk_context * ctx, size_t m, size_t ggml_vk_buffer_write(ctx, d_X, 0, x, sizeof(X_TYPE) * k * m * batch); ggml_vk_buffer_write(ctx, d_Y, 0, y, sizeof(Y_TYPE) * k * n * batch); - vk_context * subctx = ggml_vk_create_context(ctx, ctx->device.lock()->compute_queue); + vk_context * subctx = ggml_vk_create_context(ctx, ctx->device->compute_queue); for (size_t i = 0; i < num_it; i++) { ggml_vk_ctx_begin(ctx, subctx); - ggml_vk_matmul(ctx, subctx, *p, ggml_vk_subbuffer(d_X), ggml_vk_subbuffer(d_Y), ggml_vk_subbuffer(d_D), ggml_vk_subbuffer(ctx->prealloc_split_k), m, n, k, k, k, m, split_k, batch, batch, batch, 1, 1, k*m, k*n, m*n); + ggml_vk_matmul(ctx, subctx, p, ggml_vk_subbuffer(d_X), ggml_vk_subbuffer(d_Y), ggml_vk_subbuffer(d_D), ggml_vk_subbuffer(ctx->prealloc_split_k), m, n, k, k, k, m, split_k, batch, batch, batch, 1, 1, k*m, k*n, m*n); ggml_vk_ctx_end(subctx); } auto begin = std::chrono::high_resolution_clock::now(); ggml_vk_submit(subctx, ctx->fence); - VK_CHECK(ctx->device.lock()->device.waitForFences({ ctx->fence }, true, UINT64_MAX), "ggml_vk_test_matmul waitForFences"); - ctx->device.lock()->device.resetFences({ ctx->fence }); + VK_CHECK(ctx->device->device.waitForFences({ ctx->fence }, true, UINT64_MAX), "ggml_vk_test_matmul waitForFences"); + ctx->device->device.resetFences({ ctx->fence }); auto end = std::chrono::high_resolution_clock::now(); double time = std::chrono::duration_cast(end-begin).count() / 1000.0; @@ -3630,6 +3850,8 @@ static void ggml_vk_test_matmul(ggml_backend_vk_context * ctx, size_t m, size_t std::cerr << "m = " << first_err_m << " n = " << first_err_n << " b = " << first_err_b << std::endl; std::cerr << "Actual result: " << std::endl << std::endl; ggml_vk_print_matrix_area(d, GGML_TYPE_F32, m, n, first_err_m, first_err_n, first_err_b); + std::cerr << std::endl; + ggml_vk_print_matrix_area(d, GGML_TYPE_F32, m, n, first_err_m, first_err_n + 15, first_err_b); std::cerr << "Expected result: " << std::endl << std::endl; ggml_vk_print_matrix_area(d_chk, GGML_TYPE_F32, m, n, first_err_m, first_err_n, first_err_b); @@ -3655,15 +3877,15 @@ static void ggml_vk_test_matmul(ggml_backend_vk_context * ctx, size_t m, size_t free(d_chk); - ggml_vk_queue_cleanup(ctx, ctx->device.lock()->transfer_queue); - ggml_vk_queue_cleanup(ctx, ctx->device.lock()->compute_queue); + ggml_vk_queue_cleanup(ctx, ctx->device->transfer_queue); + ggml_vk_queue_cleanup(ctx, ctx->device->compute_queue); ggml_vk_destroy_buffer(d_X); ggml_vk_destroy_buffer(d_Y); ggml_vk_destroy_buffer(d_D); - ggml_pipeline_cleanup(*p); - ggml_pipeline_cleanup(ctx->pipeline_matmul_split_k_reduce); + ggml_pipeline_cleanup(p); + ggml_pipeline_cleanup(ctx->device->pipeline_matmul_split_k_reduce); free(x); free(y); @@ -3736,7 +3958,7 @@ static void ggml_vk_test_h2d_nc(ggml_backend_vk_context * ctx, size_t ne0, size_ data[i] = (rand() / (float)RAND_MAX) * 2.0f - 1.0f; } - vk_context * subctx = ggml_vk_create_context(ctx, ctx->device.lock()->compute_queue); + vk_context * subctx = ggml_vk_create_context(ctx, ctx->device->compute_queue); ggml_vk_ctx_begin(ctx, subctx); vk_buffer buffer = ggml_vk_create_buffer_check(ctx, ggml_nbytes(tensor), vk::MemoryPropertyFlagBits::eDeviceLocal); @@ -3745,8 +3967,8 @@ static void ggml_vk_test_h2d_nc(ggml_backend_vk_context * ctx, size_t ne0, size_ ggml_vk_ctx_end(subctx); ggml_vk_submit(subctx, ctx->fence); - VK_CHECK(ctx->device.lock()->device.waitForFences({ ctx->fence }, true, UINT64_MAX), "ggml_vk_test_h2d_nc waitForFences"); - ctx->device.lock()->device.resetFences({ ctx->fence }); + VK_CHECK(ctx->device->device.waitForFences({ ctx->fence }, true, UINT64_MAX), "ggml_vk_test_h2d_nc waitForFences"); + ctx->device->device.resetFences({ ctx->fence }); ggml_vk_buffer_read(ctx, buffer, 0, result_data, ggml_nbytes(tensor)); @@ -3818,7 +4040,7 @@ static void ggml_vk_test_transfer(ggml_backend_vk_context * ctx, size_t ne, bool x[i] = rand() / (float)RAND_MAX; } - vk_context * subctx = ggml_vk_create_context(ctx, ctx->device.lock()->compute_queue); + vk_context * subctx = ggml_vk_create_context(ctx, ctx->device->compute_queue); ggml_vk_ctx_begin(ctx, subctx); auto begin = std::chrono::high_resolution_clock::now(); @@ -3832,8 +4054,8 @@ static void ggml_vk_test_transfer(ggml_backend_vk_context * ctx, size_t ne, bool ggml_vk_ctx_end(subctx); ggml_vk_submit(subctx, ctx->fence); - VK_CHECK(ctx->device.lock()->device.waitForFences({ ctx->fence }, true, UINT64_MAX), "ggml_vk_test_transfer waitForFences"); - ctx->device.lock()->device.resetFences({ ctx->fence }); + VK_CHECK(ctx->device->device.waitForFences({ ctx->fence }, true, UINT64_MAX), "ggml_vk_test_transfer waitForFences"); + ctx->device->device.resetFences({ ctx->fence }); auto end = std::chrono::high_resolution_clock::now(); @@ -3847,8 +4069,8 @@ static void ggml_vk_test_transfer(ggml_backend_vk_context * ctx, size_t ne, bool ggml_vk_ctx_end(subctx); ggml_vk_submit(subctx, ctx->fence); - VK_CHECK(ctx->device.lock()->device.waitForFences({ ctx->fence }, true, UINT64_MAX), "ggml_vk_test_transfer waitForFences"); - ctx->device.lock()->device.resetFences({ ctx->fence }); + VK_CHECK(ctx->device->device.waitForFences({ ctx->fence }, true, UINT64_MAX), "ggml_vk_test_transfer waitForFences"); + ctx->device->device.resetFences({ ctx->fence }); for (auto& cpy : subctx->out_memcpys) { memcpy(cpy.dst, cpy.src, cpy.n); @@ -3879,6 +4101,48 @@ static void ggml_vk_test_transfer(ggml_backend_vk_context * ctx, size_t ne, bool } } +static void ggml_vk_quantize_data(const float * from, void * to, size_t ne, ggml_type quant) { + std::vector hist_cur(1 << 4, 0); + + switch(quant) { + case GGML_TYPE_F32: + memcpy(to, from, sizeof(float) * ne); + break; + case GGML_TYPE_Q4_0: + ggml_quantize_q4_0(from, to, ne, ne, hist_cur.data()); + break; + case GGML_TYPE_Q4_1: + ggml_quantize_q4_1(from, to, ne, ne, hist_cur.data()); + break; + case GGML_TYPE_Q5_0: + ggml_quantize_q5_0(from, to, ne, ne, hist_cur.data()); + break; + case GGML_TYPE_Q5_1: + ggml_quantize_q5_1(from, to, ne, ne, hist_cur.data()); + break; + case GGML_TYPE_Q8_0: + ggml_quantize_q8_0(from, to, ne, ne, hist_cur.data()); + break; + case GGML_TYPE_Q2_K: + ggml_quantize_q2_K(from, to, ne, ne, hist_cur.data()); + break; + case GGML_TYPE_Q3_K: + ggml_quantize_q3_K(from, to, ne, ne, hist_cur.data()); + break; + case GGML_TYPE_Q4_K: + ggml_quantize_q4_K(from, to, ne, ne, hist_cur.data()); + break; + case GGML_TYPE_Q5_K: + ggml_quantize_q5_K(from, to, ne, ne, hist_cur.data()); + break; + case GGML_TYPE_Q6_K: + ggml_quantize_q6_K(from, to, ne, ne, hist_cur.data()); + break; + default: + GGML_ASSERT(false); + } +} + static void ggml_vk_test_dequant(ggml_backend_vk_context * ctx, size_t ne, ggml_type quant) { #ifdef GGML_VULKAN_DEBUG std::cerr << "ggml_vk_test_dequant(" << ne << ")" << std::endl; @@ -3896,72 +4160,59 @@ static void ggml_vk_test_dequant(ggml_backend_vk_context * ctx, size_t ne, ggml_ x[i] = rand() / (float)RAND_MAX; } - std::vector hist_cur(1 << 4, 0); + vk_pipeline p = ctx->device->pipeline_dequant[quant]; - vk_pipeline& p = ctx->pipeline_dequant[quant]; - - switch(quant) { - case GGML_TYPE_Q4_0: - ggml_quantize_q4_0(x, qx, ne, ne, hist_cur.data()); - break; - case GGML_TYPE_Q4_1: - ggml_quantize_q4_1(x, qx, ne, ne, hist_cur.data()); - break; - case GGML_TYPE_Q5_0: - ggml_quantize_q5_0(x, qx, ne, ne, hist_cur.data()); - break; - case GGML_TYPE_Q5_1: - ggml_quantize_q4_1(x, qx, ne, ne, hist_cur.data()); - break; - case GGML_TYPE_Q8_0: - ggml_quantize_q8_0(x, qx, ne, ne, hist_cur.data()); - break; - case GGML_TYPE_Q2_K: - ggml_quantize_q2_K(x, qx, ne, ne, hist_cur.data()); - break; - case GGML_TYPE_Q3_K: - ggml_quantize_q3_K(x, qx, ne, ne, hist_cur.data()); - break; - case GGML_TYPE_Q4_K: - ggml_quantize_q4_K(x, qx, ne, ne, hist_cur.data()); - break; - case GGML_TYPE_Q5_K: - ggml_quantize_q5_K(x, qx, ne, ne, hist_cur.data()); - break; - case GGML_TYPE_Q6_K: - ggml_quantize_q6_K(x, qx, ne, ne, hist_cur.data()); - break; - default: - GGML_ASSERT(false); - } + ggml_vk_quantize_data(x, qx, ne, quant); ggml_pipeline_allocate_descriptor_sets(ctx, p, 1); ggml_vk_buffer_write(ctx, qx_buf, 0, qx, qx_sz); - vk_context * subctx = ggml_vk_create_context(ctx, ctx->device.lock()->compute_queue); + vk_context * subctx = ggml_vk_create_context(ctx, ctx->device->compute_queue); ggml_vk_ctx_begin(ctx, subctx); - const std::vector pc = { 1, (int)ne, (int)ne, (int)ne }; + const std::vector pc = { 1, (uint32_t)ne, (uint32_t)ne, (uint32_t)ne, (uint32_t)ne }; ggml_vk_dispatch_pipeline(ctx, subctx, p, { { qx_buf, 0, qx_sz }, { x_buf, 0, x_sz_f16 } }, pc.size() * sizeof(int), pc.data(), { (uint32_t)ne, 1, 1}); ggml_vk_ctx_end(subctx); auto begin = std::chrono::high_resolution_clock::now(); ggml_vk_submit(subctx, ctx->fence); - VK_CHECK(ctx->device.lock()->device.waitForFences({ ctx->fence }, true, UINT64_MAX), "ggml_vk_test_dequant waitForFences"); - ctx->device.lock()->device.resetFences({ ctx->fence }); + VK_CHECK(ctx->device->device.waitForFences({ ctx->fence }, true, UINT64_MAX), "ggml_vk_test_dequant waitForFences"); + ctx->device->device.resetFences({ ctx->fence }); auto end = std::chrono::high_resolution_clock::now(); double ms_dequant = std::chrono::duration_cast(end-begin).count() / 1000.0; ggml_vk_buffer_read(ctx, x_buf, 0, x_chk, x_sz_f16); + int first_err = -1; + double avg_err = 0.0; for (size_t i = 0; i < ne; i++) { - avg_err += std::fabs(x[i] - ggml_fp16_to_fp32(x_chk[i])); + double error = std::fabs(x[i] - ggml_fp16_to_fp32(x_chk[i])); + avg_err += error; + + if (first_err < 0 && error > 0.05) { + first_err = i; + } } - std::cerr << "TEST DEQUANT " << ggml_type_name(quant) << " time=" << ms_dequant << "ms avg_err=" << avg_err / ne << std::endl; + avg_err /= ne; + + std::cerr << "TEST DEQUANT " << ggml_type_name(quant) << " time=" << ms_dequant << "ms avg_err=" << avg_err << std::endl; + + if (avg_err > 0.1) { + std::cerr << "first_error = " << first_err << std::endl; + std::cerr << "Actual result: " << std::endl << std::endl; + for (int i = std::max(0, first_err - 5); i < std::min((int)ne, first_err + 5); i++) { + std::cerr << ggml_fp16_to_fp32(x_chk[i]) << ", "; + } + std::cerr << std::endl << "Expected result: " << std::endl << std::endl; + for (int i = std::max(0, first_err - 5); i < std::min((int)ne, first_err + 5); i++) { + std::cerr << x[i] << ", "; + } + std::cerr << std::endl; + } ggml_vk_destroy_buffer(x_buf); ggml_vk_destroy_buffer(qx_buf); @@ -3970,6 +4221,190 @@ static void ggml_vk_test_dequant(ggml_backend_vk_context * ctx, size_t ne, ggml_ free(qx); free(x_chk); } + +static void ggml_vk_test_dequant_matmul(ggml_backend_vk_context * ctx, size_t m, size_t n, size_t k, size_t batch, size_t num_it, size_t split_k, size_t shader_size, ggml_type quant) { +#ifdef GGML_VULKAN_DEBUG + std::cerr << "ggml_vk_test_dequant_matmul(" << m << ", " << n << ", " << k << ", " << batch << ", " << num_it << ", " << split_k << ", " << ggml_type_name(quant) << ")" << std::endl; +#endif + const size_t x_ne = m * k * batch; + const size_t y_ne = k * n * batch; + const size_t d_ne = m * n * batch; + + vk_pipeline p; + std::string shname; + if (shader_size == 0) { + p = ctx->device->pipeline_dequant_mul_mat_mat[quant]->a_s; + shname = std::string(ggml_type_name(quant)) + "_ALIGNED_S"; + } else if (shader_size == 1) { + p = ctx->device->pipeline_dequant_mul_mat_mat[quant]->a_m; + shname = std::string(ggml_type_name(quant)) + "_ALIGNED_M"; + } else if (shader_size == 2) { + p = ctx->device->pipeline_dequant_mul_mat_mat[quant]->a_l; + shname = std::string(ggml_type_name(quant)) + "_ALIGNED_L"; + } else { + GGML_ASSERT(0); + } + + const size_t kpad = ggml_vk_align_size(k, p->align); + + if (k != kpad) { + if (shader_size == 0) { + p = ctx->device->pipeline_dequant_mul_mat_mat[quant]->s; + shname = std::string(ggml_type_name(quant)) + "_S"; + } else if (shader_size == 1) { + p = ctx->device->pipeline_dequant_mul_mat_mat[quant]->m; + shname = std::string(ggml_type_name(quant)) + "_M"; + } else if (shader_size == 2) { + p = ctx->device->pipeline_dequant_mul_mat_mat[quant]->l; + shname = std::string(ggml_type_name(quant)) + "_L"; + } else { + GGML_ASSERT(0); + } + } + + const size_t x_sz = sizeof(float) * x_ne; + const size_t y_sz = sizeof(float) * y_ne; + const size_t qx_sz = x_ne * ggml_type_size(quant)/ggml_blck_size(quant); + const size_t d_sz = sizeof(float) * d_ne; + float * x = (float *) malloc(x_sz); + float * y = (float *) malloc(y_sz); + void * qx = malloc(qx_sz); + vk_buffer qx_buf = ggml_vk_create_buffer_check(ctx, qx_sz, vk::MemoryPropertyFlagBits::eDeviceLocal); + vk_buffer y_buf = ggml_vk_create_buffer_check(ctx, y_sz, vk::MemoryPropertyFlagBits::eDeviceLocal); + vk_buffer d_buf = ggml_vk_create_buffer_check(ctx, d_sz, vk::MemoryPropertyFlagBits::eDeviceLocal); + float * d = (float *) malloc(d_sz); + float * d_chk = (float *) malloc(d_sz); + + for (size_t i = 0; i < x_ne; i++) { + x[i] = (rand() / (float)RAND_MAX) * 2.0f - 1.0f; + } + + ggml_vk_quantize_data(x, qx, x_ne, quant); + + for (size_t i = 0; i < y_ne; i++) { + // y[i] = rand() / (float)RAND_MAX; + y[i] = (i % k == i / k) ? 1.0f : 0.0f; + } + + ggml_pipeline_allocate_descriptor_sets(ctx, p, num_it); + if (split_k > 1) { + ggml_pipeline_allocate_descriptor_sets(ctx, ctx->device->pipeline_matmul_split_k_reduce, num_it); + + if (ctx->prealloc_split_k == nullptr || ctx->prealloc_split_k->size < sizeof(float) * d_ne * split_k) { + // Resize buffer + if (ctx->prealloc_split_k != nullptr) { + ggml_vk_destroy_buffer(ctx->prealloc_split_k); + } + ctx->prealloc_split_k = ggml_vk_create_buffer_check(ctx, sizeof(float) * d_ne * split_k, vk::MemoryPropertyFlagBits::eDeviceLocal); + } + } + + ggml_vk_buffer_write(ctx, qx_buf, 0, qx, qx_sz); + ggml_vk_buffer_write(ctx, y_buf, 0, y, y_sz); + + vk_context * subctx = ggml_vk_create_context(ctx, ctx->device->compute_queue); + for (size_t i = 0; i < num_it; i++) { + ggml_vk_ctx_begin(ctx, subctx); + ggml_vk_matmul(ctx, subctx, p, ggml_vk_subbuffer(qx_buf), ggml_vk_subbuffer(y_buf), ggml_vk_subbuffer(d_buf), ggml_vk_subbuffer(ctx->prealloc_split_k), m, n, k, k, k, m, split_k, batch, batch, batch, 1, 1, k*m, k*n, m*n); + ggml_vk_ctx_end(subctx); + } + + auto begin = std::chrono::high_resolution_clock::now(); + + ggml_vk_submit(subctx, ctx->fence); + VK_CHECK(ctx->device->device.waitForFences({ ctx->fence }, true, UINT64_MAX), "ggml_vk_test_dequant waitForFences"); + ctx->device->device.resetFences({ ctx->fence }); + + auto end = std::chrono::high_resolution_clock::now(); + + double time_ms = std::chrono::duration_cast(end-begin).count() / 1000.0; + ggml_vk_buffer_read(ctx, d_buf, 0, d, d_sz); + + ggml_init_params iparams = { + /*.mem_size =*/ 1024*1024*1024, + /*.mem_buffer =*/ NULL, + /*.no_alloc =*/ true, + }; + + ggml_context * ggml_ctx = ggml_init(iparams); + + ggml_tensor * src0_ggml = ggml_new_tensor_3d(ggml_ctx, quant, k, m, batch); + ggml_tensor * src1_ggml = ggml_new_tensor_3d(ggml_ctx, GGML_TYPE_F32, k, n, batch); + ggml_tensor * tensor_ggml = ggml_mul_mat(ggml_ctx, src0_ggml, src1_ggml); + + src0_ggml->data = qx; + src1_ggml->data = y; + tensor_ggml->data = d_chk; + + ctx->disable = true; + + ggml_cgraph * cgraph = ggml_new_graph(ggml_ctx); + ggml_build_forward_expand(cgraph, tensor_ggml); + + ggml_graph_compute_with_ctx(ggml_ctx, cgraph, 1); + + ctx->disable = false; + + ggml_free(ggml_ctx); + + double avg_err = 0.0; + int first_err_n = -1; + int first_err_m = -1; + int first_err_b = -1; + + for (size_t i = 0; i < m*n*batch; i++) { + double err = std::fabs(d[i] - d_chk[i]); + avg_err += err; + + if ((err > 0.05f || std::isnan(err)) && first_err_n == -1) { + first_err_b = i / (m * n); + first_err_n = (i % (m * n)) / m; + first_err_m = (i % (m * n)) % m; + } + } + + avg_err /= m * n; + + std::cerr << "TEST MMQ " << shname << " m=" << m << " n=" << n << " k=" << k << " batch=" << batch << " split_k=" << split_k << " matmul " << time_ms / num_it << "ms avg_err=" << avg_err << std::endl; + + if (avg_err > 0.1 || std::isnan(avg_err)) { + std::cerr << "m = " << first_err_m << " n = " << first_err_n << " b = " << first_err_b << std::endl; + std::cerr << "Actual result: " << std::endl << std::endl; + ggml_vk_print_matrix_area(d, GGML_TYPE_F32, m, n, first_err_m, first_err_n, first_err_b); + std::cerr << std::endl; + std::cerr << "Expected result: " << std::endl << std::endl; + ggml_vk_print_matrix_area(d_chk, GGML_TYPE_F32, m, n, first_err_m, first_err_n, first_err_b); + + if (split_k > 1) { + float * split_k_buf = (float *) malloc(sizeof(float) * d_ne * split_k); + ggml_vk_buffer_read(ctx, ctx->prealloc_split_k, 0, split_k_buf, sizeof(float) * d_ne * split_k); + + std::cerr << "d_buf0: " << std::endl << std::endl; + ggml_vk_print_matrix_area(split_k_buf, GGML_TYPE_F32, m, n, first_err_m, first_err_n, first_err_b); + + std::cerr << "d_buf1: " << std::endl << std::endl; + ggml_vk_print_matrix_area(split_k_buf + d_ne, GGML_TYPE_F32, m, n, first_err_m, first_err_n, first_err_b); + + std::cerr << "d_buf2: " << std::endl << std::endl; + ggml_vk_print_matrix_area(split_k_buf + 2 * d_ne, GGML_TYPE_F32, m, n, first_err_m, first_err_n, first_err_b); + + std::cerr << "d_buf3: " << std::endl << std::endl; + ggml_vk_print_matrix_area(split_k_buf + 3 * d_ne, GGML_TYPE_F32, m, n, first_err_m, first_err_n, first_err_b); + + free(split_k_buf); + } + } + + ggml_vk_destroy_buffer(qx_buf); + ggml_vk_destroy_buffer(y_buf); + ggml_vk_destroy_buffer(d_buf); + + free(x); + free(qx); + free(y); + free(d); + free(d_chk); +} #endif static ggml_tensor_extra_gpu * ggml_vk_tensor_create_extra(ggml_tensor * tensor) { @@ -3982,18 +4417,8 @@ static ggml_tensor_extra_gpu * ggml_vk_tensor_create_extra(ggml_tensor * tensor) return extra; } -static ggml_tensor * ggml_vk_find_last_use(const ggml_tensor * node, ggml_cgraph * graph) { - GGML_ASSERT(node != nullptr); - - for (int i = graph->n_nodes - 1; i >= 0; i--) { - for (int j = 0; j < GGML_MAX_SRC; j++) { - if (graph->nodes[i]->src[j] == node) { - return graph->nodes[i]; - } - } - } - - return nullptr; +static bool ggml_vk_cpu_assist_op(const ggml_tensor * node) { + return node->op == GGML_OP_MUL_MAT || node->op == GGML_OP_MUL_MAT_ID; } static void ggml_vk_preallocate_buffers_graph(ggml_backend_vk_context * ctx, ggml_tensor * node){ @@ -4004,7 +4429,7 @@ static void ggml_vk_preallocate_buffers_graph(ggml_backend_vk_context * ctx, ggm || (node->src[0] != nullptr && (node->src[0]->backend == GGML_BACKEND_TYPE_GPU || node->src[0]->backend == GGML_BACKEND_TYPE_GPU_SPLIT)) || (node->src[1] != nullptr && (node->src[1]->backend == GGML_BACKEND_TYPE_GPU)); - if (ctx->disable || (!any_on_device && node->op != GGML_OP_MUL_MAT)) { + if (ctx->disable || (!any_on_device && !ggml_vk_cpu_assist_op(node))) { return; } @@ -4035,7 +4460,7 @@ static void ggml_vk_preallocate_buffers_graph(ggml_backend_vk_context * ctx, ggm const bool f16_f32_kernel = use_src1 && src1->type == GGML_TYPE_F32; int split_k; - if (node->op == GGML_OP_MUL_MAT) { + if (node->op == GGML_OP_MUL_MAT || node->op == GGML_OP_MUL_MAT_ID) { split_k = ggml_vk_guess_split_k(ne01, ne11, ne10); } else { split_k = 1; @@ -4044,11 +4469,11 @@ static void ggml_vk_preallocate_buffers_graph(ggml_backend_vk_context * ctx, ggm const uint32_t y_ne = ne10 * ne11; const uint32_t d_ne = ne20 * ne21; - const uint64_t qx_sz = use_src0 ? ggml_vk_align_size(ggml_type_size(src0->type) * x_ne / ggml_blck_size(src0->type), ctx->device.lock()->properties.limits.minStorageBufferOffsetAlignment) * ne02 * ne03 : 0; - const uint64_t qy_sz = use_src1 ? ggml_vk_align_size(ggml_type_size(src1->type) * y_ne / ggml_blck_size(src1->type), ctx->device.lock()->properties.limits.minStorageBufferOffsetAlignment) * ne12 * ne13 : 0; - const uint64_t x_sz = use_src0 ? ggml_vk_align_size(sizeof(ggml_fp16_t) * x_ne, ctx->device.lock()->properties.limits.minStorageBufferOffsetAlignment) * ne02 * ne03 : 0; - const uint64_t y_sz = use_src1 ? ggml_vk_align_size(f16_f32_kernel ? sizeof(float) * y_ne : sizeof(ggml_fp16_t) * y_ne, ctx->device.lock()->properties.limits.minStorageBufferOffsetAlignment) * ne12 * ne13 : 0; - uint64_t d_sz = ggml_vk_align_size(ggml_type_size(node->type) * d_ne, ctx->device.lock()->properties.limits.minStorageBufferOffsetAlignment) * ne22 * ne23; + const uint64_t qx_sz = use_src0 ? ggml_vk_align_size(ggml_type_size(src0->type) * x_ne / ggml_blck_size(src0->type), ctx->device->properties.limits.minStorageBufferOffsetAlignment) * ne02 * ne03 : 0; + const uint64_t qy_sz = use_src1 ? ggml_vk_align_size(ggml_type_size(src1->type) * y_ne / ggml_blck_size(src1->type), ctx->device->properties.limits.minStorageBufferOffsetAlignment) * ne12 * ne13 : 0; + const uint64_t x_sz = use_src0 ? ggml_vk_align_size(sizeof(ggml_fp16_t) * x_ne, ctx->device->properties.limits.minStorageBufferOffsetAlignment) * ne02 * ne03 : 0; + const uint64_t y_sz = use_src1 ? ggml_vk_align_size(f16_f32_kernel ? sizeof(float) * y_ne : sizeof(ggml_fp16_t) * y_ne, ctx->device->properties.limits.minStorageBufferOffsetAlignment) * ne12 * ne13 : 0; + uint64_t d_sz = ggml_vk_align_size(ggml_type_size(node->type) * d_ne, ctx->device->properties.limits.minStorageBufferOffsetAlignment) * ne22 * ne23; const uint64_t split_k_size = split_k > 1 ? d_sz * 4 : 0; if (extra->buffer_gpu.expired()) { @@ -4076,6 +4501,7 @@ static void ggml_vk_preallocate_buffers_graph(ggml_backend_vk_context * ctx, ggm case GGML_OP_DIAG_MASK_INF: case GGML_OP_SOFT_MAX: case GGML_OP_ROPE: + case GGML_OP_ARGSORT: break; case GGML_OP_UNARY: switch (ggml_get_unary_op(node)) { @@ -4088,6 +4514,7 @@ static void ggml_vk_preallocate_buffers_graph(ggml_backend_vk_context * ctx, ggm } break; case GGML_OP_MUL_MAT: + case GGML_OP_MUL_MAT_ID: if (ctx->prealloc_size_qx < qx_sz) { ctx->prealloc_size_qx = qx_sz; } @@ -4121,21 +4548,66 @@ static void ggml_vk_preallocate_buffers(ggml_backend_vk_context * ctx) { #endif #if defined(GGML_VULKAN_RUN_TESTS) ctx->staging = ggml_vk_create_buffer_check(ctx, 100ul * 1024ul * 1024ul, - vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent | vk::MemoryPropertyFlagBits::eHostCached + vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent | vk::MemoryPropertyFlagBits::eHostCached, vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent); ggml_vk_test_transfer(ctx, 8192 * 1000, false); ggml_vk_test_transfer(ctx, 8192 * 1000, true); - ggml_vk_test_dequant(ctx, 2560 * 7680, GGML_TYPE_Q4_0); - ggml_vk_test_dequant(ctx, 2560 * 7680, GGML_TYPE_Q4_1); - ggml_vk_test_dequant(ctx, 2560 * 7680, GGML_TYPE_Q5_0); - ggml_vk_test_dequant(ctx, 2560 * 7680, GGML_TYPE_Q5_1); - ggml_vk_test_dequant(ctx, 2560 * 7680, GGML_TYPE_Q8_0); - ggml_vk_test_dequant(ctx, 2560 * 7680, GGML_TYPE_Q2_K); - ggml_vk_test_dequant(ctx, 2560 * 7680, GGML_TYPE_Q3_K); - ggml_vk_test_dequant(ctx, 2560 * 7680, GGML_TYPE_Q4_K); - ggml_vk_test_dequant(ctx, 2560 * 7680, GGML_TYPE_Q5_K); - ggml_vk_test_dequant(ctx, 2560 * 7680, GGML_TYPE_Q6_K); + ggml_vk_test_dequant(ctx, 7680, GGML_TYPE_F32); + ggml_vk_test_dequant(ctx, 7680, GGML_TYPE_Q4_0); + ggml_vk_test_dequant(ctx, 7680, GGML_TYPE_Q4_1); + ggml_vk_test_dequant(ctx, 7680, GGML_TYPE_Q5_0); + ggml_vk_test_dequant(ctx, 7680, GGML_TYPE_Q5_1); + ggml_vk_test_dequant(ctx, 7680, GGML_TYPE_Q8_0); + ggml_vk_test_dequant(ctx, 7680, GGML_TYPE_Q2_K); + ggml_vk_test_dequant(ctx, 7680, GGML_TYPE_Q3_K); + ggml_vk_test_dequant(ctx, 7680, GGML_TYPE_Q4_K); + ggml_vk_test_dequant(ctx, 7680, GGML_TYPE_Q5_K); + ggml_vk_test_dequant(ctx, 7680, GGML_TYPE_Q6_K); + + ggml_vk_test_matmul(ctx, 128, 512, 512, 2, 100, 1, 0); + ggml_vk_test_matmul(ctx, 128, 512, 512, 2, 100, 1, 1); + ggml_vk_test_matmul(ctx, 128, 512, 512, 2, 100, 1, 2); + ggml_vk_test_matmul(ctx, 128, 512, 512, 2, 100, 4, 0); + ggml_vk_test_matmul(ctx, 128, 512, 512, 2, 100, 4, 1); + ggml_vk_test_matmul(ctx, 128, 512, 512, 2, 100, 4, 2); + + ggml_vk_test_dequant_matmul(ctx, 128, 512, 512, 2, 100, 1, 0, GGML_TYPE_Q4_0); + ggml_vk_test_dequant_matmul(ctx, 128, 512, 512, 2, 100, 1, 1, GGML_TYPE_Q4_0); + ggml_vk_test_dequant_matmul(ctx, 128, 512, 512, 2, 100, 1, 2, GGML_TYPE_Q4_0); + ggml_vk_test_dequant_matmul(ctx, 128, 512, 512, 2, 100, 4, 0, GGML_TYPE_Q4_0); + ggml_vk_test_dequant_matmul(ctx, 128, 512, 512, 2, 100, 4, 1, GGML_TYPE_Q4_0); + ggml_vk_test_dequant_matmul(ctx, 128, 512, 512, 2, 100, 4, 2, GGML_TYPE_Q4_0); + + ggml_vk_test_dequant_matmul(ctx, 128, 512, 512, 2, 100, 1, 0, GGML_TYPE_Q4_1); + ggml_vk_test_dequant_matmul(ctx, 128, 512, 512, 2, 100, 1, 1, GGML_TYPE_Q4_1); + ggml_vk_test_dequant_matmul(ctx, 128, 512, 512, 2, 100, 1, 2, GGML_TYPE_Q4_1); + ggml_vk_test_dequant_matmul(ctx, 128, 512, 512, 2, 100, 4, 0, GGML_TYPE_Q4_1); + ggml_vk_test_dequant_matmul(ctx, 128, 512, 512, 2, 100, 4, 1, GGML_TYPE_Q4_1); + ggml_vk_test_dequant_matmul(ctx, 128, 512, 512, 2, 100, 4, 2, GGML_TYPE_Q4_1); + + ggml_vk_test_dequant_matmul(ctx, 128, 512, 512, 2, 100, 1, 0, GGML_TYPE_Q5_0); + ggml_vk_test_dequant_matmul(ctx, 128, 512, 512, 2, 100, 1, 1, GGML_TYPE_Q5_0); + ggml_vk_test_dequant_matmul(ctx, 128, 512, 512, 2, 100, 1, 2, GGML_TYPE_Q5_0); + ggml_vk_test_dequant_matmul(ctx, 128, 512, 512, 2, 100, 4, 0, GGML_TYPE_Q5_0); + ggml_vk_test_dequant_matmul(ctx, 128, 512, 512, 2, 100, 4, 1, GGML_TYPE_Q5_0); + ggml_vk_test_dequant_matmul(ctx, 128, 512, 512, 2, 100, 4, 2, GGML_TYPE_Q5_0); + + ggml_vk_test_dequant_matmul(ctx, 128, 512, 512, 2, 100, 1, 0, GGML_TYPE_Q5_1); + ggml_vk_test_dequant_matmul(ctx, 128, 512, 512, 2, 100, 1, 1, GGML_TYPE_Q5_1); + ggml_vk_test_dequant_matmul(ctx, 128, 512, 512, 2, 100, 1, 2, GGML_TYPE_Q5_1); + ggml_vk_test_dequant_matmul(ctx, 128, 512, 512, 2, 100, 4, 0, GGML_TYPE_Q5_1); + ggml_vk_test_dequant_matmul(ctx, 128, 512, 512, 2, 100, 4, 1, GGML_TYPE_Q5_1); + ggml_vk_test_dequant_matmul(ctx, 128, 512, 512, 2, 100, 4, 2, GGML_TYPE_Q5_1); + + ggml_vk_test_dequant_matmul(ctx, 128, 512, 512, 2, 100, 1, 0, GGML_TYPE_Q8_0); + ggml_vk_test_dequant_matmul(ctx, 128, 512, 512, 2, 100, 1, 1, GGML_TYPE_Q8_0); + ggml_vk_test_dequant_matmul(ctx, 128, 512, 512, 2, 100, 1, 2, GGML_TYPE_Q8_0); + ggml_vk_test_dequant_matmul(ctx, 128, 512, 512, 2, 100, 4, 0, GGML_TYPE_Q8_0); + ggml_vk_test_dequant_matmul(ctx, 128, 512, 512, 2, 100, 4, 1, GGML_TYPE_Q8_0); + ggml_vk_test_dequant_matmul(ctx, 128, 512, 512, 2, 100, 4, 2, GGML_TYPE_Q8_0); + + std::cerr << std::endl; const std::vector vals { 8, 8, 8, @@ -4225,7 +4697,7 @@ static void ggml_vk_build_graph(ggml_backend_vk_context * ctx, ggml_tensor * nod || (node->src[0] != nullptr && (node->src[0]->backend == GGML_BACKEND_TYPE_GPU || node->src[0]->backend == GGML_BACKEND_TYPE_GPU_SPLIT)) || (node->src[1] != nullptr && node->src[1]->backend == GGML_BACKEND_TYPE_GPU); - if (ctx->disable || (!any_on_device && node->op != GGML_OP_MUL_MAT) || (node->op == GGML_OP_MUL_MAT && !any_on_device && !ggml_vk_can_mul_mat(node->src[0], node->src[1], node))) { + if (ctx->disable || (!any_on_device && !ggml_vk_cpu_assist_op(node)) || (ggml_vk_cpu_assist_op(node) && !any_on_device && !ggml_vk_can_mul_mat(node->src[0], node->src[1], node))) { return; } @@ -4237,6 +4709,7 @@ static void ggml_vk_build_graph(ggml_backend_vk_context * ctx, ggml_tensor * nod const ggml_tensor * src0 = node->src[0]; const ggml_tensor * src1 = node->src[1]; + const ggml_tensor * src2 = node->src[2]; ggml_tensor_extra_gpu * extra = (ggml_tensor_extra_gpu *) node->extra; @@ -4271,7 +4744,9 @@ static void ggml_vk_build_graph(ggml_backend_vk_context * ctx, ggml_tensor * nod case GGML_OP_SOFT_MAX: case GGML_OP_ROPE: case GGML_OP_MUL_MAT: + case GGML_OP_MUL_MAT_ID: case GGML_OP_NONE: + case GGML_OP_ARGSORT: break; default: if (any_on_device) { @@ -4282,7 +4757,7 @@ static void ggml_vk_build_graph(ggml_backend_vk_context * ctx, ggml_tensor * nod } if (ctx->compute_ctx == nullptr) { - ctx->compute_ctx = ggml_vk_create_context(ctx, ctx->device.lock()->compute_queue); + ctx->compute_ctx = ggml_vk_create_context(ctx, ctx->device->compute_queue); ggml_vk_ctx_begin(ctx, ctx->compute_ctx); } @@ -4353,16 +4828,25 @@ static void ggml_vk_build_graph(ggml_backend_vk_context * ctx, ggml_tensor * nod break; case GGML_OP_SOFT_MAX: - ggml_vk_soft_max(ctx, ctx->compute_ctx, src0, src1, node); + ggml_vk_soft_max(ctx, ctx->compute_ctx, src0, src1, src2, node); break; case GGML_OP_ROPE: ggml_vk_rope(ctx, ctx->compute_ctx, src0, src1, node); + break; + case GGML_OP_ARGSORT: + ggml_vk_argsort(ctx, ctx->compute_ctx, src0, node); break; case GGML_OP_MUL_MAT: ggml_vk_mul_mat(ctx, ctx->compute_ctx, src0, src1, node); + break; + case GGML_OP_MUL_MAT_ID: + //ggml_vk_mul_mat_id(ctx, ctx->compute_ctx, src0, src1, node); + std::cerr << "ggml_vulkan: GGML_OP_MUL_MAT_ID not implemented yet." << std::endl; + GGML_ASSERT(false); + break; default: return; @@ -4389,7 +4873,7 @@ static bool ggml_vk_compute_forward(ggml_backend_vk_context * ctx, ggml_compute_ || (tensor->src[0] != nullptr && (tensor->src[0]->backend == GGML_BACKEND_TYPE_GPU || tensor->src[0]->backend == GGML_BACKEND_TYPE_GPU_SPLIT)) || (tensor->src[1] != nullptr && tensor->src[1]->backend == GGML_BACKEND_TYPE_GPU); - if (ctx->disable || (!any_on_device && tensor->op != GGML_OP_MUL_MAT)) { + if (ctx->disable || (!any_on_device && !ggml_vk_cpu_assist_op(tensor))) { return false; } @@ -4415,6 +4899,7 @@ static bool ggml_vk_compute_forward(ggml_backend_vk_context * ctx, ggml_compute_ case GGML_OP_PERMUTE: case GGML_OP_TRANSPOSE: case GGML_OP_NONE: + case GGML_OP_ARGSORT: extra = (ggml_tensor_extra_gpu *) tensor->extra; break; @@ -4430,6 +4915,7 @@ static bool ggml_vk_compute_forward(ggml_backend_vk_context * ctx, ggml_compute_ } break; case GGML_OP_MUL_MAT: + case GGML_OP_MUL_MAT_ID: if (!any_on_device && !ggml_vk_can_mul_mat(tensor->src[0], tensor->src[1], tensor)) { return false; } @@ -4475,8 +4961,8 @@ static bool ggml_vk_compute_forward(ggml_backend_vk_context * ctx, ggml_compute_ } if (tensor == subctx.exit_tensor) { - VK_CHECK(ctx->device.lock()->device.waitForFences({ ctx->fence }, true, UINT64_MAX), "ggml_vk_compute_forward waitForFences"); - ctx->device.lock()->device.resetFences({ ctx->fence }); + VK_CHECK(ctx->device->device.waitForFences({ ctx->fence }, true, UINT64_MAX), "ggml_vk_compute_forward waitForFences"); + ctx->device->device.resetFences({ ctx->fence }); // Do staging buffer copies for (auto& cpy : subctx.out_memcpys) { @@ -4504,20 +4990,25 @@ static void ggml_vk_graph_cleanup(ggml_backend_vk_context * ctx) { } ctx->gc.temp_buffers.clear(); - for (auto * pipeline : ctx->gc.pipelines) { - ggml_pipeline_cleanup(*pipeline); + for (auto& pipeline : ctx->device->pipelines) { + if (pipeline.expired()) { + continue; + } + + vk_pipeline pl = pipeline.lock(); + ggml_pipeline_cleanup(pl); } - ggml_vk_queue_cleanup(ctx, ctx->device.lock()->compute_queue); - ggml_vk_queue_cleanup(ctx, ctx->device.lock()->transfer_queue); + ggml_vk_queue_cleanup(ctx, ctx->device->compute_queue); + ggml_vk_queue_cleanup(ctx, ctx->device->transfer_queue); for (size_t i = 0; i < ctx->gc.semaphores.size(); i++) { - ctx->device.lock()->device.destroySemaphore({ ctx->gc.semaphores[i].s }); + ctx->device->device.destroySemaphore({ ctx->gc.semaphores[i].s }); } ctx->gc.semaphores.clear(); for (size_t i = 0; i < ctx->gc.tl_semaphores.size(); i++) { - ctx->device.lock()->device.destroySemaphore({ ctx->gc.tl_semaphores[i].s }); + ctx->device->device.destroySemaphore({ ctx->gc.tl_semaphores[i].s }); } ctx->gc.tl_semaphores.clear(); ctx->semaphore_idx = 0; @@ -4525,7 +5016,7 @@ static void ggml_vk_graph_cleanup(ggml_backend_vk_context * ctx) { ctx->event_idx = 0; for (auto& event : ctx->gc.events) { - ctx->device.lock()->device.resetEvent(event); + ctx->device->device.resetEvent(event); } ctx->staging_offset = 0; @@ -4562,21 +5053,11 @@ static void ggml_vk_cleanup(ggml_backend_vk_context * ctx) { ctx->staging_size = 0; for (auto& event : ctx->gc.events) { - ctx->device.lock()->device.destroyEvent(event); + ctx->device->device.destroyEvent(event); } ctx->gc.events.clear(); - for (auto* pipeline : ctx->gc.pipelines) { - ggml_vk_destroy_pipeline(ctx, pipeline); - } - ctx->gc.pipelines.clear(); - - ctx->device.lock()->device.destroyFence(ctx->fence); - - ctx->device.lock()->device.destroyCommandPool(ctx->device.lock()->compute_queue.pool); - if (!ctx->device.lock()->single_queue) { - ctx->device.lock()->device.destroyCommandPool(ctx->device.lock()->transfer_queue.pool); - } + ctx->device->device.destroyFence(ctx->fence); } GGML_CALL static int ggml_vk_get_device_count() { @@ -4787,7 +5268,6 @@ GGML_CALL static void ggml_backend_vk_buffer_get_tensor(ggml_backend_buffer_t bu GGML_CALL static bool ggml_backend_vk_buffer_cpy_tensor(ggml_backend_buffer_t buffer, const ggml_tensor * src, ggml_tensor * dst) { if (ggml_backend_buffer_is_vk(src->buffer)) { - ggml_backend_vk_buffer_context * ctx = (ggml_backend_vk_buffer_context *)buffer->context; ggml_tensor_extra_gpu * src_extra = (ggml_tensor_extra_gpu *) src->extra; ggml_tensor_extra_gpu * dst_extra = (ggml_tensor_extra_gpu *) dst->extra; @@ -4799,6 +5279,8 @@ GGML_CALL static bool ggml_backend_vk_buffer_cpy_tensor(ggml_backend_buffer_t bu return true; } return false; + + UNUSED(buffer); } GGML_CALL static void ggml_backend_vk_buffer_clear(ggml_backend_buffer_t buffer, uint8_t value) { @@ -4845,12 +5327,12 @@ GGML_CALL static ggml_backend_buffer_t ggml_backend_vk_buffer_type_alloc_buffer( GGML_CALL static size_t ggml_backend_vk_buffer_type_get_alignment(ggml_backend_buffer_type_t buft) { ggml_backend_vk_buffer_type_context * ctx = (ggml_backend_vk_buffer_type_context *) buft->context; - return ctx->ctx->device.lock()->properties.limits.minStorageBufferOffsetAlignment; + return ctx->ctx->device->properties.limits.minStorageBufferOffsetAlignment; } GGML_CALL static size_t ggml_backend_vk_buffer_type_get_max_size(ggml_backend_buffer_type_t buft) { ggml_backend_vk_buffer_type_context * ctx = (ggml_backend_vk_buffer_type_context *) buft->context; - return ctx->ctx->device.lock()->max_memory_allocation_size; + return ctx->ctx->device->max_memory_allocation_size; } GGML_CALL static size_t ggml_backend_vk_buffer_type_get_alloc_size(ggml_backend_buffer_type_t buft, const ggml_tensor * tensor) { @@ -4936,7 +5418,7 @@ GGML_CALL static ggml_backend_buffer_t ggml_backend_vk_host_buffer_type_alloc_bu } GGML_CALL static size_t ggml_backend_vk_host_buffer_type_get_alignment(ggml_backend_buffer_type_t buft) { - return vk_instance.contexts[0].device.lock()->properties.limits.minMemoryMapAlignment; + return vk_instance.contexts[0].device->properties.limits.minMemoryMapAlignment; UNUSED(buft); } @@ -4981,8 +5463,7 @@ GGML_CALL static void ggml_backend_vk_free(ggml_backend_t backend) { ggml_vk_cleanup(ctx); - // Release device - vk_instance.devices[ctx->idx].reset(); + ctx->device.reset(); ctx->initialized = false; vk_instance.initialized[idx] = false; @@ -5011,7 +5492,7 @@ GGML_CALL static void ggml_backend_vk_set_tensor_async(ggml_backend_t backend, g if (ctx->transfer_ctx == nullptr) { // Initialize new transfer context - ctx->transfer_ctx = ggml_vk_create_context(ctx, ctx->device.lock()->transfer_queue); + ctx->transfer_ctx = ggml_vk_create_context(ctx, ctx->device->transfer_queue); ggml_vk_ctx_begin(ctx, ctx->transfer_ctx); } @@ -5032,7 +5513,7 @@ GGML_CALL static void ggml_backend_vk_get_tensor_async(ggml_backend_t backend, c if (ctx->transfer_ctx == nullptr) { // Initialize new transfer context - ctx->transfer_ctx = ggml_vk_create_context(ctx, ctx->device.lock()->transfer_queue); + ctx->transfer_ctx = ggml_vk_create_context(ctx, ctx->device->transfer_queue); ggml_vk_ctx_begin(ctx, ctx->transfer_ctx); } @@ -5052,7 +5533,7 @@ GGML_CALL static bool ggml_backend_vk_cpy_tensor_async(ggml_backend_t backend, c if (ctx->transfer_ctx == nullptr) { // Initialize new transfer context - ctx->transfer_ctx = ggml_vk_create_context(ctx, ctx->device.lock()->transfer_queue); + ctx->transfer_ctx = ggml_vk_create_context(ctx, ctx->device->transfer_queue); ggml_vk_ctx_begin(ctx, ctx->transfer_ctx); } @@ -5082,8 +5563,8 @@ GGML_CALL static void ggml_backend_vk_synchronize(ggml_backend_t backend) { } ggml_vk_submit(ctx->transfer_ctx, ctx->fence); - VK_CHECK(ctx->device.lock()->device.waitForFences({ ctx->fence }, true, UINT64_MAX), "ggml_backend_vk_synchronize waitForFences"); - ctx->device.lock()->device.resetFences({ ctx->fence }); + VK_CHECK(ctx->device->device.waitForFences({ ctx->fence }, true, UINT64_MAX), "ggml_backend_vk_synchronize waitForFences"); + ctx->device->device.resetFences({ ctx->fence }); for (auto& cpy : ctx->transfer_ctx->out_memcpys) { memcpy(cpy.dst, cpy.src, cpy.n); @@ -5153,6 +5634,7 @@ GGML_CALL static bool ggml_backend_vk_supports_op(ggml_backend_t backend, const } break; case GGML_OP_MUL_MAT: + case GGML_OP_MUL_MAT_ID: { struct ggml_tensor * a; struct ggml_tensor * b; @@ -5226,6 +5708,7 @@ GGML_CALL static bool ggml_backend_vk_supports_op(ggml_backend_t backend, const case GGML_OP_CONT: case GGML_OP_DIAG_MASK_INF: case GGML_OP_SOFT_MAX: + case GGML_OP_ARGSORT: return true; default: return false; @@ -5505,6 +5988,7 @@ static void ggml_vk_check_results_0(ggml_backend_vk_context * ctx, ggml_compute_ ggml_tensor * src0 = tensor->src[0]; ggml_tensor * src1 = tensor->src[1]; + ggml_tensor * src2 = tensor->src[2]; struct ggml_init_params iparams = { /*.mem_size =*/ 1024*1024*1024, @@ -5516,13 +6000,16 @@ static void ggml_vk_check_results_0(ggml_backend_vk_context * ctx, ggml_compute_ struct ggml_tensor * src0_clone = nullptr; struct ggml_tensor * src1_clone = nullptr; + struct ggml_tensor * src2_clone = nullptr; struct ggml_tensor * tensor_clone = nullptr; size_t src0_size; size_t src1_size; + size_t src2_size; void * src0_buffer; void * src1_buffer; + void * src2_buffer; if (src0 != nullptr) { src0_clone = ggml_dup_tensor(ggml_ctx, src0); @@ -5536,12 +6023,12 @@ static void ggml_vk_check_results_0(ggml_backend_vk_context * ctx, ggml_compute_ memcpy(src0_clone->nb, src0->nb, sizeof(size_t) * GGML_MAX_DIMS); } else if (src0->backend == GGML_BACKEND_TYPE_GPU) { ggml_tensor_extra_gpu * extra = (ggml_tensor_extra_gpu *) src0->extra; + vk_buffer buffer_gpu = extra->buffer_gpu.lock(); uint64_t offset = extra->offset; if (!ggml_is_contiguous(src0) && ggml_vk_dim01_contiguous(src0)) { for (int i3 = 0; i3 < src0->ne[3]; i3++) { for (int i2 = 0; i2 < src0->ne[2]; i2++) { const int idx = i3*src0->ne[2] + i2; - vk_buffer buffer_gpu = extra->buffer_gpu.lock(); ggml_vk_buffer_read(ctx, buffer_gpu, offset + idx * src0->nb[2], ((char *)src0_clone->data + idx * src0_clone->nb[2]), src0->ne[1] * src0->nb[1]); } } @@ -5552,7 +6039,6 @@ static void ggml_vk_check_results_0(ggml_backend_vk_context * ctx, ggml_compute_ src0_clone->nb[i] = src0_clone->nb[i - 1]*src0_clone->ne[i - 1]; } } else { - vk_buffer buffer_gpu = extra->buffer_gpu.lock(); if (offset + src0_size >= buffer_gpu->size) { src0_size = buffer_gpu->size - offset; } @@ -5581,12 +6067,12 @@ static void ggml_vk_check_results_0(ggml_backend_vk_context * ctx, ggml_compute_ memcpy(src1_clone->nb, src1->nb, sizeof(size_t) * GGML_MAX_DIMS); } else if (src1->backend == GGML_BACKEND_TYPE_GPU) { ggml_tensor_extra_gpu * extra = (ggml_tensor_extra_gpu *) src1->extra; + vk_buffer buffer_gpu = extra->buffer_gpu.lock(); uint64_t offset = extra->offset; if (!ggml_is_contiguous(src1) && ggml_vk_dim01_contiguous(src1)) { for (int i3 = 0; i3 < src1->ne[3]; i3++) { for (int i2 = 0; i2 < src1->ne[2]; i2++) { const int idx = i3*src1->ne[2] + i2; - vk_buffer buffer_gpu = extra->buffer_gpu.lock(); ggml_vk_buffer_read(ctx, buffer_gpu, offset + idx * src1->nb[2], ((char *)src1_clone->data + idx * src1_clone->nb[2]), src1->ne[1] * src1->nb[1]); } } @@ -5597,7 +6083,6 @@ static void ggml_vk_check_results_0(ggml_backend_vk_context * ctx, ggml_compute_ src1_clone->nb[i] = src1_clone->nb[i - 1]*src1_clone->ne[i - 1]; } } else { - vk_buffer buffer_gpu = extra->buffer_gpu.lock(); if (offset + src1_size >= buffer_gpu->size) { src1_size = buffer_gpu->size - offset; } @@ -5630,6 +6115,66 @@ static void ggml_vk_check_results_0(ggml_backend_vk_context * ctx, ggml_compute_ ggml_vk_check_tensor(std::string(ggml_op_name(tensor->op)) + "->src1", src1_clone); } + if (src2 != nullptr) { + src2_clone = ggml_dup_tensor(ggml_ctx, src2); + + src2_size = ggml_nbytes(src2); + + src2_buffer = malloc(src2_size); + src2_clone->data = src2_buffer; + if (src2->backend == GGML_BACKEND_TYPE_CPU) { + memcpy(src2_clone->data, src2->data, src2_size); + memcpy(src2_clone->nb, src2->nb, sizeof(size_t) * GGML_MAX_DIMS); + } else if (src2->backend == GGML_BACKEND_TYPE_GPU) { + ggml_tensor_extra_gpu * extra = (ggml_tensor_extra_gpu *) src2->extra; + vk_buffer buf = extra->buffer_gpu.lock(); + uint64_t offset = extra->offset; + if (!ggml_is_contiguous(src2) && ggml_vk_dim01_contiguous(src2)) { + for (int i3 = 0; i3 < src2->ne[3]; i3++) { + for (int i2 = 0; i2 < src2->ne[2]; i2++) { + const int idx = i3*src2->ne[2] + i2; + ggml_vk_buffer_read(ctx, buf, offset + idx * src2->nb[2], ((char *)src2_clone->data + idx * src2_clone->nb[2]), src2->ne[1] * src2->nb[1]); + } + } + + src2_clone->nb[0] = src2->nb[0]; + src2_clone->nb[1] = src2->nb[1]; + for (int i = 2; i < GGML_MAX_DIMS; i++) { + src2_clone->nb[i] = src2_clone->nb[i - 1]*src2_clone->ne[i - 1]; + } + } else { + if (offset + src2_size >= buf->size) { + src2_size = buf->size - offset; + } + ggml_vk_buffer_read(ctx, buf, offset, src2_clone->data, src2_size); + memcpy(src2_clone->nb, src2->nb, sizeof(size_t) * GGML_MAX_DIMS); + } + } else { + GGML_ASSERT(false); + } + + if (vk_output_tensor > 0 && vk_output_tensor == check_counter) { + ggml_vk_print_tensor(ctx, src2, "src2"); + std::cerr << "TENSOR CHECK: " << ggml_op_name(src2_clone->op) << " (check " << check_counter << ")" << std::endl; + std::cerr << "src2_clone=" << tensor << " src2_clone->backend: " << src2_clone->backend << " src2_clone->type: " << ggml_type_name(src2_clone->type) << " ne0=" << src2_clone->ne[0] << " nb0=" << src2_clone->nb[0] << " ne1=" << src2_clone->ne[1] << " nb1=" << src2_clone->nb[1] << " ne2=" << src2_clone->ne[2] << " nb2=" << src2_clone->nb[2] << " ne3=" << src2_clone->ne[3] << " nb3=" << src2_clone->nb[3] << std::endl; + if (src2->src[0] != nullptr) { + std::cerr << "src2->src[0]=" << src2->src[0] << " op=" << ggml_op_name(src2->src[0]->op) << " type=" << ggml_type_name(src2->src[0]->type) << " backend=" << src2->src[0]->backend << " ne0=" << src2->src[0]->ne[0] << " nb0=" << src2->src[0]->nb[0] << " ne1=" << src2->src[0]->ne[1] << " nb1=" << src2->src[0]->nb[1] << " ne2=" << src2->src[0]->ne[2] << " nb2=" << src2->src[0]->nb[2] << " ne3=" << src2->src[0]->ne[3] << " nb3=" << src2->src[0]->nb[3] << std::endl; + } + if (src2->src[1] != nullptr) { + std::cerr << "src2->src[1]=" << src2->src[1] << " op=" << ggml_op_name(src2->src[1]->op) << " type=" << ggml_type_name(src2->src[1]->type) << " backend=" << src2->src[1]->backend << " ne0=" << src2->src[1]->ne[0] << " nb0=" << src2->src[1]->nb[0] << " ne1=" << src2->src[1]->ne[1] << " nb1=" << src2->src[1]->nb[1] << " ne2=" << src2->src[1]->ne[2] << " nb2=" << src2->src[1]->nb[2] << " ne3=" << src2->src[1]->ne[3] << " nb3=" << src2->src[1]->nb[3] << std::endl; + } + std::cerr << std::endl << "Result:" << std::endl; + ggml_vk_print_tensor_area(src2_clone, src2_clone->data, 5, 5, 0, 0); + std::cerr << std::endl; + std::cerr << std::endl << "Result:" << std::endl; + ggml_vk_print_tensor_area(src2_clone, src2_clone->data, 5, 5, 1, 0); + std::cerr << std::endl; + std::vector done; + ggml_vk_print_graph_origin(src2_clone, done); + } + + ggml_vk_check_tensor(std::string(ggml_op_name(tensor->op)) + "->src2", src2_clone); + } if (tensor->op == GGML_OP_MUL_MAT) { tensor_clone = ggml_mul_mat(ggml_ctx, src0_clone, src1_clone); @@ -5648,7 +6193,11 @@ static void ggml_vk_check_results_0(ggml_backend_vk_context * ctx, ggml_compute_ } else if (tensor->op == GGML_OP_RMS_NORM) { tensor_clone = ggml_rms_norm(ggml_ctx, src0_clone, *(float *)tensor->op_params); } else if (tensor->op == GGML_OP_SOFT_MAX) { + if (src1 != nullptr) { + tensor_clone = ggml_soft_max_ext(ggml_ctx, src0_clone, src1_clone, src2_clone, ((float *)tensor->op_params)[0], ((float *)tensor->op_params)[1]); + } else { tensor_clone = ggml_soft_max(ggml_ctx, src0_clone); + } } else if (tensor->op == GGML_OP_DIAG_MASK_INF) { tensor_clone = ggml_diag_mask_inf(ggml_ctx, src0_clone, *(float *)tensor->op_params); } else if (tensor->op == GGML_OP_ROPE) { @@ -5728,6 +6277,9 @@ static void ggml_vk_check_results_0(ggml_backend_vk_context * ctx, ggml_compute_ if (src1 != nullptr) { free(src1_buffer); } + if (src2 != nullptr) { + free(src1_buffer); + } ggml_free(ggml_ctx); } diff --git a/ggml-vulkan.h b/ggml-vulkan.h index 9645126b4..e4317c3e0 100644 --- a/ggml-vulkan.h +++ b/ggml-vulkan.h @@ -10,6 +10,7 @@ extern "C" { #define GGML_VK_NAME "Vulkan" #define GGML_VK_MAX_DEVICES 16 +GGML_API void ggml_vk_instance_init(void); GGML_API void ggml_vk_init_cpu_assist(void); GGML_API void ggml_vk_preallocate_buffers_graph_cpu_assist(struct ggml_tensor * node); diff --git a/ggml_vk_generate_shaders.py b/ggml_vk_generate_shaders.py index b2e86e182..4a6f5e323 100644 --- a/ggml_vk_generate_shaders.py +++ b/ggml_vk_generate_shaders.py @@ -64,6 +64,7 @@ struct block_q5_0 #define A_TYPE block_q5_0 """ shader_q5_1_defines = """ +#extension GL_EXT_shader_explicit_arithmetic_types_int16 : require #define QUANT_K 32 #define QUANT_R 2 @@ -187,7 +188,8 @@ v = (v - 16.0f) * d; shader_q5_1_dequant_func = """ #define DEQUANT_FUNC const float d = float(data_a[ib].d); \ const float m = float(data_a[ib].m); \ -const ivec2 qh = ivec2(((data_a[ib].qh >> iqs) << 4) & 0x10, (data_a[ib].qh >> (iqs + 12)) & 0x10); \ +const uint uint_qh = data_a[ib].qh; \ +const ivec2 qh = ivec2(((uint_qh >> iqs) << 4) & 0x10, (uint_qh >> (iqs + 12)) & 0x10); \ const uint vui = uint(data_a[ib].qs[iqs]); \ vec2 v = vec2((vui & 0xF) | qh.x, (vui >> 4) | qh.y); \ v = v*d + m; @@ -206,12 +208,15 @@ mulmat_head = """#version 450 #extension GL_EXT_control_flow_attributes : enable #extension GL_EXT_shader_16bit_storage : require -#ifndef LOAD_VEC -#define LOAD_VEC 1 +#ifndef LOAD_VEC_A +#define LOAD_VEC_A 1 +#endif +#ifndef LOAD_VEC_B +#define LOAD_VEC_B 1 #endif """ -mulmat_body = """ +mulmat_body1 = """ layout(local_size_x_id = 0, local_size_y = 1, local_size_z = 1) in; layout (binding = 0) readonly buffer A {A_TYPE data_a[];}; @@ -240,7 +245,7 @@ layout (push_constant) uniform parameter layout (constant_id = 1) const uint BM = 64; layout (constant_id = 2) const uint BN = 64; -layout (constant_id = 3) const uint BK = 16; +layout (constant_id = 3) const uint BK = 16; // Assumed to be 32 if working with a quant layout (constant_id = 4) const uint WM = 32; layout (constant_id = 5) const uint WN = 32; layout (constant_id = 6) const uint WMITER = 2; @@ -277,16 +282,19 @@ void main() { const uint tiwr = tiw % (WSUBM / TM); const uint tiwc = tiw / (WSUBM / TM); - const uint loadr = gl_LocalInvocationID.x % (BK / LOAD_VEC); - const uint loadc = gl_LocalInvocationID.x / (BK / LOAD_VEC); + const uint loadr_a = gl_LocalInvocationID.x % (BK / LOAD_VEC_A); + const uint loadc_a = gl_LocalInvocationID.x / (BK / LOAD_VEC_A); + const uint loadr_b = gl_LocalInvocationID.x % (BK / LOAD_VEC_B); + const uint loadc_b = gl_LocalInvocationID.x / (BK / LOAD_VEC_B); - const uint loadstride = gl_WorkGroupSize.x * LOAD_VEC / BK; + const uint loadstride_a = gl_WorkGroupSize.x * LOAD_VEC_A / BK; + const uint loadstride_b = gl_WorkGroupSize.x * LOAD_VEC_B / BK; const uint start_k = ik * p.k_split; const uint end_k = min(p.K, (ik + 1) * p.k_split); - uint pos_a = (batch_idx_a * p.batch_stride_a + ir * BM * p.stride_a + start_k) / LOAD_VEC; - uint pos_b = (gl_GlobalInvocationID.z * p.batch_stride_b + ic * BN * p.stride_b + start_k) / LOAD_VEC; + uint pos_a = (batch_idx_a * p.batch_stride_a + ir * BM * p.stride_a + start_k) / LOAD_VEC_A; + uint pos_b = (gl_GlobalInvocationID.z * p.batch_stride_b + ic * BN * p.stride_b + start_k) / LOAD_VEC_B; float sums[WMITER * TM * WNITER * TN]; FLOAT_TYPE cache_a[WMITER * TM]; @@ -297,61 +305,145 @@ void main() { } [[unroll]] for (uint block = start_k; block < end_k; block += BK) { - [[unroll]] for (uint l = 0; l < BM; l += loadstride) { -#if LOAD_VEC == 8 - const uint idx = pos_a + (loadc + l) * p.stride_a / LOAD_VEC + loadr; - buf_a[(loadc + l) * (BK+1) + loadr * LOAD_VEC + 0] = FLOAT_TYPE(data_a[idx][0].x); - buf_a[(loadc + l) * (BK+1) + loadr * LOAD_VEC + 1] = FLOAT_TYPE(data_a[idx][0].y); - buf_a[(loadc + l) * (BK+1) + loadr * LOAD_VEC + 2] = FLOAT_TYPE(data_a[idx][0].z); - buf_a[(loadc + l) * (BK+1) + loadr * LOAD_VEC + 3] = FLOAT_TYPE(data_a[idx][0].w); - buf_a[(loadc + l) * (BK+1) + loadr * LOAD_VEC + 4] = FLOAT_TYPE(data_a[idx][1].x); - buf_a[(loadc + l) * (BK+1) + loadr * LOAD_VEC + 5] = FLOAT_TYPE(data_a[idx][1].y); - buf_a[(loadc + l) * (BK+1) + loadr * LOAD_VEC + 6] = FLOAT_TYPE(data_a[idx][1].z); - buf_a[(loadc + l) * (BK+1) + loadr * LOAD_VEC + 7] = FLOAT_TYPE(data_a[idx][1].w); -#elif LOAD_VEC == 4 - const uint idx = pos_a + (loadc + l) * p.stride_a / LOAD_VEC + loadr; - buf_a[(loadc + l) * (BK+1) + loadr * LOAD_VEC + 0] = FLOAT_TYPE(data_a[idx].x); - buf_a[(loadc + l) * (BK+1) + loadr * LOAD_VEC + 1] = FLOAT_TYPE(data_a[idx].y); - buf_a[(loadc + l) * (BK+1) + loadr * LOAD_VEC + 2] = FLOAT_TYPE(data_a[idx].z); - buf_a[(loadc + l) * (BK+1) + loadr * LOAD_VEC + 3] = FLOAT_TYPE(data_a[idx].w); + [[unroll]] for (uint l = 0; l < BM; l += loadstride_a) {""" + +mulmat_load_scalar = """ +#if LOAD_VEC_A == 8 + const uint idx = pos_a + (loadc_a + l) * p.stride_a / LOAD_VEC_A + loadr_a; + const uint buf_idx = (loadc_a + l) * (BK+1) + loadr_a * LOAD_VEC_A; + buf_a[buf_idx ] = FLOAT_TYPE(data_a[idx][0].x); + buf_a[buf_idx + 1] = FLOAT_TYPE(data_a[idx][0].y); + buf_a[buf_idx + 2] = FLOAT_TYPE(data_a[idx][0].z); + buf_a[buf_idx + 3] = FLOAT_TYPE(data_a[idx][0].w); + buf_a[buf_idx + 4] = FLOAT_TYPE(data_a[idx][1].x); + buf_a[buf_idx + 5] = FLOAT_TYPE(data_a[idx][1].y); + buf_a[buf_idx + 6] = FLOAT_TYPE(data_a[idx][1].z); + buf_a[buf_idx + 7] = FLOAT_TYPE(data_a[idx][1].w); +#elif LOAD_VEC_A == 4 + const uint idx = pos_a + (loadc_a + l) * p.stride_a / LOAD_VEC_A + loadr_a; + const uint buf_idx = (loadc_a + l) * (BK+1) + loadr_a * LOAD_VEC_A; + buf_a[buf_idx ] = FLOAT_TYPE(data_a[idx].x); + buf_a[buf_idx + 1] = FLOAT_TYPE(data_a[idx].y); + buf_a[buf_idx + 2] = FLOAT_TYPE(data_a[idx].z); + buf_a[buf_idx + 3] = FLOAT_TYPE(data_a[idx].w); #else - if (ir * BM + loadc + l < p.M && block + loadr < end_k) { - buf_a[(loadc + l) * (BK+1) + loadr] = FLOAT_TYPE(data_a[pos_a + (loadc + l) * p.stride_a + loadr]); + if (ir * BM + loadc_a + l < p.M && block + loadr_a < end_k) { + buf_a[(loadc_a + l) * (BK+1) + loadr_a] = FLOAT_TYPE(data_a[pos_a + (loadc_a + l) * p.stride_a + loadr_a]); } else { - buf_a[(loadc + l) * (BK+1) + loadr] = FLOAT_TYPE(0.0f); + buf_a[(loadc_a + l) * (BK+1) + loadr_a] = FLOAT_TYPE(0.0f); } #endif +""" + +mulmat_load_q4_0 = """ + const uint idx = pos_a + (loadc_a + l) * p.stride_a / LOAD_VEC_A + loadr_a; + const uint buf_idx = (loadc_a + l) * (BK+1) + loadr_a; + + const uint ib = idx / 16; + const uint iqs = idx & 0xF; + + const float d = float(data_a[ib].d); + const uint vui = uint(data_a[ib].qs[iqs]); + const vec2 v = (vec2(vui & 0xF, vui >> 4) - 8.0f) * d; + + buf_a[buf_idx ] = FLOAT_TYPE(v.x); + buf_a[buf_idx + 16] = FLOAT_TYPE(v.y);""" + +mulmat_load_q4_1 = """ + const uint idx = pos_a + (loadc_a + l) * p.stride_a / LOAD_VEC_A + loadr_a; + const uint buf_idx = (loadc_a + l) * (BK+1) + loadr_a; + + const uint ib = idx / 16; + const uint iqs = idx & 0xF; + + const float d = float(data_a[ib].d); + const float m = float(data_a[ib].m); + const uint vui = uint(data_a[ib].qs[iqs]); + const vec2 v = vec2(vui & 0xF, vui >> 4) * d + m; + + buf_a[buf_idx ] = FLOAT_TYPE(v.x); + buf_a[buf_idx + 16] = FLOAT_TYPE(v.y);""" + +mulmat_load_q5_0 = """ + const uint idx = pos_a + (loadc_a + l) * p.stride_a / LOAD_VEC_A + loadr_a; + const uint buf_idx = (loadc_a + l) * (BK+1) + loadr_a; + + const uint ib = idx / 16; + const uint iqs = idx & 0xF; + + const float d = float(data_a[ib].d); + const uint uint_qh = uint(data_a[ib].qh[1]) << 16 | data_a[ib].qh[0]; + const ivec2 qh = ivec2(((uint_qh >> iqs) << 4) & 0x10, (uint_qh >> (iqs + 12)) & 0x10); + const uint vui = uint(data_a[ib].qs[iqs]); + const vec2 v = (vec2((vui & 0xF) | qh.x, (vui >> 4) | qh.y) - 16.0f) * d; + + buf_a[buf_idx ] = FLOAT_TYPE(v.x); + buf_a[buf_idx + 16] = FLOAT_TYPE(v.y);""" + +mulmat_load_q5_1 = """ + const uint idx = pos_a + (loadc_a + l) * p.stride_a / LOAD_VEC_A + loadr_a; + const uint buf_idx = (loadc_a + l) * (BK+1) + loadr_a; + + const uint ib = idx / 16; + const uint iqs = idx & 0xF; + + const float d = float(data_a[ib].d); + const float m = float(data_a[ib].m); + const uint uint_qh = data_a[ib].qh; + const ivec2 qh = ivec2(((uint_qh >> iqs) << 4) & 0x10, (uint_qh >> (iqs + 12)) & 0x10); + const uint vui = uint(data_a[ib].qs[iqs]); + const vec2 v = vec2((vui & 0xF) | qh.x, (vui >> 4) | qh.y) * d + m; + + buf_a[buf_idx ] = FLOAT_TYPE(v.x); + buf_a[buf_idx + 16] = FLOAT_TYPE(v.y);""" + +mulmat_load_q8_0 = """ + const uint idx = pos_a + (loadc_a + l) * p.stride_a / LOAD_VEC_A + loadr_a; + const uint buf_idx = (loadc_a + l) * (BK+1) + loadr_a * LOAD_VEC_A; + + const uint ib = idx / 16; + const uint iqs = (idx & 0xF) * 2; + + const float d = float(data_a[ib].d); + const vec2 v = vec2(int(data_a[ib].qs[iqs]), int(data_a[ib].qs[iqs + 1])) * d; + + buf_a[buf_idx ] = FLOAT_TYPE(v.x); + buf_a[buf_idx + 1] = FLOAT_TYPE(v.y);""" + +mulmat_body2 = """ } - [[unroll]] for (uint l = 0; l < BN; l += loadstride) { -#if LOAD_VEC == 8 - const uint idx = pos_b + (loadc + l) * p.stride_b / LOAD_VEC + loadr; - buf_b[(loadc + l) * (BK+1) + loadr * LOAD_VEC + 0] = FLOAT_TYPE(data_b[idx][0].x); - buf_b[(loadc + l) * (BK+1) + loadr * LOAD_VEC + 1] = FLOAT_TYPE(data_b[idx][0].y); - buf_b[(loadc + l) * (BK+1) + loadr * LOAD_VEC + 2] = FLOAT_TYPE(data_b[idx][0].z); - buf_b[(loadc + l) * (BK+1) + loadr * LOAD_VEC + 3] = FLOAT_TYPE(data_b[idx][0].w); - buf_b[(loadc + l) * (BK+1) + loadr * LOAD_VEC + 4] = FLOAT_TYPE(data_b[idx][1].x); - buf_b[(loadc + l) * (BK+1) + loadr * LOAD_VEC + 5] = FLOAT_TYPE(data_b[idx][1].y); - buf_b[(loadc + l) * (BK+1) + loadr * LOAD_VEC + 6] = FLOAT_TYPE(data_b[idx][1].z); - buf_b[(loadc + l) * (BK+1) + loadr * LOAD_VEC + 7] = FLOAT_TYPE(data_b[idx][1].w); -#elif LOAD_VEC == 4 - const uint idx = pos_b + (loadc + l) * p.stride_b / LOAD_VEC + loadr; - buf_b[(loadc + l) * (BK+1) + loadr * LOAD_VEC + 0] = FLOAT_TYPE(data_b[idx].x); - buf_b[(loadc + l) * (BK+1) + loadr * LOAD_VEC + 1] = FLOAT_TYPE(data_b[idx].y); - buf_b[(loadc + l) * (BK+1) + loadr * LOAD_VEC + 2] = FLOAT_TYPE(data_b[idx].z); - buf_b[(loadc + l) * (BK+1) + loadr * LOAD_VEC + 3] = FLOAT_TYPE(data_b[idx].w); + [[unroll]] for (uint l = 0; l < BN; l += loadstride_b) { +#if LOAD_VEC_B == 8 + const uint idx = pos_b + (loadc_b + l) * p.stride_b / LOAD_VEC_B + loadr_b; + const uint buf_idx = (loadc_b + l) * (BK+1) + loadr_b * LOAD_VEC_B; + buf_b[buf_idx + 0] = FLOAT_TYPE(data_b[idx][0].x); + buf_b[buf_idx + 1] = FLOAT_TYPE(data_b[idx][0].y); + buf_b[buf_idx + 2] = FLOAT_TYPE(data_b[idx][0].z); + buf_b[buf_idx + 3] = FLOAT_TYPE(data_b[idx][0].w); + buf_b[buf_idx + 4] = FLOAT_TYPE(data_b[idx][1].x); + buf_b[buf_idx + 5] = FLOAT_TYPE(data_b[idx][1].y); + buf_b[buf_idx + 6] = FLOAT_TYPE(data_b[idx][1].z); + buf_b[buf_idx + 7] = FLOAT_TYPE(data_b[idx][1].w); +#elif LOAD_VEC_B == 4 + const uint idx = pos_b + (loadc_b + l) * p.stride_b / LOAD_VEC_B + loadr_b; + const uint buf_idx = (loadc_b + l) * (BK+1) + loadr_b * LOAD_VEC_B; + buf_b[buf_idx + 0] = FLOAT_TYPE(data_b[idx].x); + buf_b[buf_idx + 1] = FLOAT_TYPE(data_b[idx].y); + buf_b[buf_idx + 2] = FLOAT_TYPE(data_b[idx].z); + buf_b[buf_idx + 3] = FLOAT_TYPE(data_b[idx].w); #else - if (ic * BN + loadc + l < p.N && block + loadr < end_k) { - buf_b[(loadc + l) * (BK+1) + loadr] = FLOAT_TYPE(data_b[pos_b + (loadc + l) * p.stride_b + loadr]); + if (ic * BN + loadc_b + l < p.N && block + loadr_b < end_k) { + buf_b[(loadc_b + l) * (BK+1) + loadr_b] = FLOAT_TYPE(data_b[pos_b + (loadc_b + l) * p.stride_b + loadr_b]); } else { - buf_b[(loadc + l) * (BK+1) + loadr] = FLOAT_TYPE(0.0f); + buf_b[(loadc_b + l) * (BK+1) + loadr_b] = FLOAT_TYPE(0.0f); } #endif } barrier(); - pos_a += BK / LOAD_VEC; - pos_b += BK / LOAD_VEC; + pos_a += BK / LOAD_VEC_A; + pos_b += BK / LOAD_VEC_B; for (uint i = 0; i < BK; i++) { // Load from shared into cache @@ -438,45 +530,191 @@ dequant_head = """#version 450 #extension GL_EXT_control_flow_attributes : require #extension GL_EXT_shader_16bit_storage : require -""" - -dequant_body = """ -layout(local_size_x = 256, local_size_y = 1, local_size_z = 1) in; - -layout (binding = 0) readonly buffer A {A_TYPE data_a[];}; -layout (binding = 1) writeonly buffer D {D_TYPE data_b[];}; layout (push_constant) uniform parameter { - int M; - int K; - int stride_a; - int stride_b; + uint M; + uint K; + uint stride_a; + uint stride_b; + uint nel; } p; +""" + +dequant_f32_body = """ +layout(local_size_x = 256, local_size_y = 1, local_size_z = 1) in; + +layout (binding = 0) readonly buffer A {float data_a[];}; +layout (binding = 1) writeonly buffer D {D_TYPE data_b[];}; void main() { - const int i = int(gl_GlobalInvocationID.x); + const uint i = gl_GlobalInvocationID.x * 16; - // Transposed - const int row = i % (p.K / QUANT_K); - const int col = i / (p.K / QUANT_K); - - if (row * QUANT_K >= p.K || col >= p.M) { + if (i >= p.nel) { return; } - const int stride_a = p.stride_a / QUANT_K; + [[unroll]] for (uint l = 0; l < 16; l++) { + data_b[i + l] = D_TYPE(data_a[i + l]); + } +} +""" - const int ib = col * stride_a + row; +dequant_q4_0_body = """ +layout(local_size_x = 256, local_size_y = 1, local_size_z = 1) in; - const int y_offset = QUANT_R == 1 ? 1 : QUANT_K/2; - const int step = QUANT_R == 1 ? 2 : 1; +layout (binding = 0) readonly buffer A {block_q4_0 data_a[];}; +layout (binding = 1) writeonly buffer D {D_TYPE data_b[];}; - [[unroll]] for (int iqs = 0; iqs < QUANT_K/QUANT_R; iqs += step) { - DEQUANT_FUNC +void main() { + const uint i = gl_WorkGroupID.x * 4 + gl_LocalInvocationID.x / 64; - data_b[col * p.stride_b + row*QUANT_K + iqs + 0 ] = D_TYPE(v.x); - data_b[col * p.stride_b + row*QUANT_K + iqs + y_offset] = D_TYPE(v.y); + const uint tid = gl_LocalInvocationID.x % 64; + const uint il = tid/32; + const uint ir = tid%32; + const uint ib = 32*i + ir; + if (ib >= p.nel / 32) { + return; + } + + const uint b_idx = 1024*i + 32*ir + 8*il; + + const float d = float(data_a[ib].d); + const float dm = -8.0f * d; + + const uint q_idx = 8*il; + + [[unroll]] for (uint l = 0; l < 8; ++l) { + data_b[b_idx + l + 0] = D_TYPE(d * (data_a[ib].qs[q_idx + l] & 0xF) + dm); + data_b[b_idx + l + 16] = D_TYPE(d * (data_a[ib].qs[q_idx + l] >> 4) + dm); + } +} +""" + +dequant_q4_1_body = """ +layout(local_size_x = 256, local_size_y = 1, local_size_z = 1) in; + +layout (binding = 0) readonly buffer A {block_q4_1 data_a[];}; +layout (binding = 1) writeonly buffer D {D_TYPE data_b[];}; + +void main() { + const uint i = gl_WorkGroupID.x * 4 + gl_LocalInvocationID.x / 64; + + const uint tid = gl_LocalInvocationID.x % 64; + const uint il = tid/32; + const uint ir = tid%32; + const uint ib = 32*i + ir; + if (ib >= p.nel / 32) { + return; + } + + const uint b_idx = 1024*i + 32*ir + 8*il; + + const float d = float(data_a[ib].d); + const float m = float(data_a[ib].m); + + const uint q_idx = 8*il; + + [[unroll]] for (uint l = 0; l < 8; ++l) { + data_b[b_idx + l + 0] = D_TYPE(d * (data_a[ib].qs[q_idx + l] & 0xF) + m); + data_b[b_idx + l + 16] = D_TYPE(d * (data_a[ib].qs[q_idx + l] >> 4) + m); + } +} +""" + +dequant_q5_0_body = """ +layout(local_size_x = 256, local_size_y = 1, local_size_z = 1) in; + +layout (binding = 0) readonly buffer A {block_q5_0 data_a[];}; +layout (binding = 1) writeonly buffer D {D_TYPE data_b[];}; + +void main() { + const uint i = gl_WorkGroupID.x * 4 + gl_LocalInvocationID.x / 64; + + const uint tid = gl_LocalInvocationID.x % 64; + const uint il = tid/32; + const uint ir = tid%32; + const uint ib = 32*i + ir; + if (ib >= p.nel / 32) { + return; + } + + const uint b_idx = 1024*i + 32*ir + 8*il; + + const float d = float(data_a[ib].d); + const uint qh = uint(data_a[ib].qh[1]) << 16 | data_a[ib].qh[0]; + + const uint q_idx = 8*il; + + [[unroll]] for (uint l = 0; l < 8; ++l) { + const uint iqs = q_idx + l; + const uint vui = uint(data_a[ib].qs[iqs]); + data_b[b_idx + l + 0] = D_TYPE(d * (((vui & 0xF) | (((qh >> iqs) << 4) & 0x10)) - 16.0f)); + data_b[b_idx + l + 16] = D_TYPE(d * (((vui >> 4) | ((qh >> (iqs + 12)) & 0x10)) - 16.0f)); + } +} +""" + +dequant_q5_1_body = """ +layout(local_size_x = 256, local_size_y = 1, local_size_z = 1) in; + +layout (binding = 0) readonly buffer A {block_q5_1 data_a[];}; +layout (binding = 1) writeonly buffer D {D_TYPE data_b[];}; + +void main() { + const uint i = gl_WorkGroupID.x * 4 + gl_LocalInvocationID.x / 64; + + const uint tid = gl_LocalInvocationID.x % 64; + const uint il = tid/32; + const uint ir = tid%32; + const uint ib = 32*i + ir; + if (ib >= p.nel / 32) { + return; + } + + const uint b_idx = 1024*i + 32*ir + 8*il; + + const float d = float(data_a[ib].d); + const float m = float(data_a[ib].m); + const uint qh = data_a[ib].qh; + + const uint q_idx = 8*il; + + [[unroll]] for (uint l = 0; l < 8; ++l) { + const uint iqs = q_idx + l; + const uint vui = uint(data_a[ib].qs[iqs]); + data_b[b_idx + l + 0] = D_TYPE(d * (((vui & 0xF) | (((qh >> iqs) << 4) & 0x10))) + m); + data_b[b_idx + l + 16] = D_TYPE(d * (((vui >> 4) | ((qh >> (iqs + 12)) & 0x10))) + m); + } +} +""" + +dequant_q8_0_body = """ +layout(local_size_x = 256, local_size_y = 1, local_size_z = 1) in; + +layout (binding = 0) readonly buffer A {block_q8_0 data_a[];}; +layout (binding = 1) writeonly buffer D {D_TYPE data_b[];}; + +void main() { + const uint i = gl_WorkGroupID.x * 4 + gl_LocalInvocationID.x / 64; + + const uint tid = gl_LocalInvocationID.x % 64; + const uint il = tid/32; + const uint ir = tid%32; + const uint ib = 32*i + ir; + if (ib >= p.nel / 32) { + return; + } + + const uint b_idx = 1024*i + 32*ir + 16*il; + + const float d = float(data_a[ib].d); + + const uint q_idx = 16*il; + + [[unroll]] for (uint l = 0; l < 16; l += 2) { + data_b[b_idx + l ] = D_TYPE(d * data_a[ib].qs[q_idx + l ]); + data_b[b_idx + l + 1] = D_TYPE(d * data_a[ib].qs[q_idx + l + 1]); } } """ @@ -488,29 +726,21 @@ layout(local_size_x = 64, local_size_y = 1, local_size_z = 1) in; layout (binding = 0) readonly buffer A {A_TYPE data_a[];}; layout (binding = 1) writeonly buffer D {D_TYPE data_b[];}; -layout (push_constant) uniform parameter -{ - int M; - int K; - int stride_a; - int stride_b; -} p; - void main() { - [[unroll]] for (int wgy = 0; wgy < 256; wgy++) { - const int i = int(gl_WorkGroupID.x * 256 + wgy); + [[unroll]] for (uint wgy = 0; wgy < 256; wgy++) { + const uint i = gl_WorkGroupID.x * 256 + wgy; if (i >= p.M * p.K / QUANT_K) { return; } - const int tid = int(gl_LocalInvocationID.x); - const int ip = tid / 32; - const int il = tid - 32 * ip; - const int is = 8 * ip + il / 16; + const uint tid = gl_LocalInvocationID.x; + const uint ip = tid / 32; + const uint il = tid - 32 * ip; + const uint is = 8 * ip + il / 16; - const int y_idx = i * QUANT_K + 128 * ip + il; + const uint y_idx = i * QUANT_K + 128 * ip + il; - const int ql_idx = 32 * ip + il; + const uint ql_idx = 32 * ip + il; const uint8_t qs = data_a[i].qs[32 * ip + il]; FLOAT_TYPE dall = FLOAT_TYPE(data_a[i].d.x); @@ -528,31 +758,23 @@ layout(local_size_x = 64, local_size_y = 1, local_size_z = 1) in; layout (binding = 0) readonly buffer A {A_TYPE data_a[];}; layout (binding = 1) writeonly buffer D {D_TYPE data_b[];}; -layout (push_constant) uniform parameter -{ - int M; - int K; - int stride_a; - int stride_b; -} p; - void main() { - [[unroll]] for (int wgy = 0; wgy < 256; wgy++) { - const int i = int(gl_WorkGroupID.x * 256 + wgy); + [[unroll]] for (uint wgy = 0; wgy < 256; wgy++) { + const uint i = uint(gl_WorkGroupID.x * 256 + wgy); if (i >= p.M * p.K / QUANT_K) { return; } - const int r = int(gl_LocalInvocationID.x) / 4; - const int tid = r / 2; - const int is0 = r % 2; - const int l0 = 16 * is0 + 4 * (int(gl_LocalInvocationID.x) % 4); - const int n = tid / 4; - const int j = tid - 4*n; + const uint r = gl_LocalInvocationID.x / 4; + const uint tid = r / 2; + const uint is0 = r % 2; + const uint l0 = 16 * is0 + 4 * (gl_LocalInvocationID.x % 4); + const uint n = tid / 4; + const uint j = tid - 4*n; const uint8_t m = uint8_t(1 << (4*n + j)); - const int is = 8*n + 2*j + is0; - const int shift = 2*j; + const uint is = 8*n + 2*j + is0; + const uint shift = 2*j; const int8_t us = int8_t(is < 4 ? (data_a[i].scales[is-0] & 0xF) | (((data_a[i].scales[is+8] >> 0) & 3) << 4) : is < 8 ? (data_a[i].scales[is-0] & 0xF) | (((data_a[i].scales[is+4] >> 2) & 3) << 4) : @@ -561,10 +783,10 @@ void main() { const FLOAT_TYPE d_all = FLOAT_TYPE(data_a[i].d); const FLOAT_TYPE dl = d_all * FLOAT_TYPE(us - 32); - const int y_idx = i * QUANT_K + 128 * n + 32 * j; - const int qs_idx = 32*n; + const uint y_idx = i * QUANT_K + 128 * n + 32 * j; + const uint qs_idx = 32*n; - for (int l = l0; l < l0 + 4; ++l) { + for (uint l = l0; l < l0 + 4; ++l) { data_b[y_idx + l] = D_TYPE(dl * FLOAT_TYPE(int8_t((data_a[i].qs[qs_idx + l] >> shift) & 3) - (((data_a[i].hmask[l] & m) != 0) ? 0 : 4))); } } @@ -576,32 +798,24 @@ layout(local_size_x = 32, local_size_y = 1, local_size_z = 1) in; layout (binding = 0) readonly buffer A {A_TYPE data_a[];}; layout (binding = 1) writeonly buffer D {D_TYPE data_b[];}; -layout (push_constant) uniform parameter -{ - int M; - int K; - int stride_a; - int stride_b; -} p; - void main() { - [[unroll]] for (int wgy = 0; wgy < 256; wgy++) { - const int i = int(gl_WorkGroupID.x * 256 + wgy); + [[unroll]] for (uint wgy = 0; wgy < 256; wgy++) { + const uint i = gl_WorkGroupID.x * 256 + wgy; if (i >= p.M * p.K / QUANT_K) { return; } - const int tid = int(gl_LocalInvocationID.x); - const int il = tid / 8; - const int ir = tid % 8; - const int is = 2 * il; - const int n = 4; + const uint tid = gl_LocalInvocationID.x; + const uint il = tid / 8; + const uint ir = tid % 8; + const uint is = 2 * il; + const uint n = 4; const FLOAT_TYPE dall = FLOAT_TYPE(data_a[i].d.x); const FLOAT_TYPE dmin = FLOAT_TYPE(data_a[i].d.y); - const int y_idx = i * QUANT_K + 64 * il + n * ir; - const int qs_idx = 32*il + n * ir; + const uint y_idx = i * QUANT_K + 64 * il + n * ir; + const uint qs_idx = 32*il + n * ir; uint8_t sc; uint8_t m; @@ -625,7 +839,7 @@ void main() { const FLOAT_TYPE d2 = dall * sc; const FLOAT_TYPE m2 = dmin * m; - [[unroll]] for (int l = 0; l < n; ++l) { + [[unroll]] for (uint l = 0; l < n; ++l) { data_b[y_idx + l ] = D_TYPE(d1 * FLOAT_TYPE(data_a[i].qs[qs_idx + l] & 0xF) - m1); data_b[y_idx + l + 32] = D_TYPE(d2 * FLOAT_TYPE(data_a[i].qs[qs_idx + l] >> 4) - m2); } @@ -638,32 +852,24 @@ layout(local_size_x = 64, local_size_y = 1, local_size_z = 1) in; layout (binding = 0) readonly buffer A {A_TYPE data_a[];}; layout (binding = 1) writeonly buffer D {D_TYPE data_b[];}; -layout (push_constant) uniform parameter -{ - int M; - int K; - int stride_a; - int stride_b; -} p; - void main() { - [[unroll]] for (int wgy = 0; wgy < 256; wgy++) { - const int i = int(gl_WorkGroupID.x * 256 + wgy); + [[unroll]] for (uint wgy = 0; wgy < 256; wgy++) { + const uint i = gl_WorkGroupID.x * 256 + wgy; if (i >= p.M * p.K / QUANT_K) { return; } - const int tid = int(gl_LocalInvocationID.x); - const int il = tid / 16; - const int ir = tid % 16; - const int is = 2 * il; + const uint tid = gl_LocalInvocationID.x; + const uint il = tid / 16; + const uint ir = tid % 16; + const uint is = 2 * il; const FLOAT_TYPE dall = FLOAT_TYPE(data_a[i].d.x); const FLOAT_TYPE dmin = FLOAT_TYPE(data_a[i].d.y); - const int y_idx = i * QUANT_K + 64 * il + 2 * ir; - const int qs_idx = 32*il + 2 * ir; - const int qh_idx = 2 * ir; + const uint y_idx = i * QUANT_K + 64 * il + 2 * ir; + const uint qs_idx = 32*il + 2 * ir; + const uint qh_idx = 2 * ir; uint8_t sc; uint8_t m; @@ -702,28 +908,20 @@ layout(local_size_x = 64, local_size_y = 1, local_size_z = 1) in; layout (binding = 0) readonly buffer A {A_TYPE data_a[];}; layout (binding = 1) writeonly buffer D {D_TYPE data_b[];}; -layout (push_constant) uniform parameter -{ - int M; - int K; - int stride_a; - int stride_b; -} p; - void main() { - [[unroll]] for (int wgy = 0; wgy < 256; wgy++) { - const int i = int(gl_WorkGroupID.x * 256 + wgy); + [[unroll]] for (uint wgy = 0; wgy < 256; wgy++) { + const uint i = gl_WorkGroupID.x * 256 + wgy; if (i >= p.M * p.K / QUANT_K) { return; } - const int tid = int(gl_LocalInvocationID.x); - const int ip = tid / 32; - const int il = tid - 32 * ip; - const int is = 8 * ip + il / 16; + const uint tid = gl_LocalInvocationID.x; + const uint ip = tid / 32; + const uint il = tid - 32 * ip; + const uint is = 8 * ip + il / 16; - const int y_idx = i * QUANT_K + 128 * ip + il; + const uint y_idx = i * QUANT_K + 128 * ip + il; - const int ql_idx = 64 * ip + il; + const uint ql_idx = 64 * ip + il; const uint8_t qh = data_a[i].qh[32 * ip + il]; const FLOAT_TYPE d = FLOAT_TYPE(data_a[i].d); @@ -742,49 +940,50 @@ mul_mat_vec_head = """#version 450 #extension GL_EXT_control_flow_attributes : enable #extension GL_EXT_shader_16bit_storage : require #extension GL_EXT_shader_8bit_storage : require + +layout (push_constant) uniform parameter +{ + uint ncols; + uint b_offset; + uint d_offset; +} p; """ mul_mat_vec_body = """ -layout(local_size_x = QUANT_K, local_size_y = 1, local_size_z = 1) in; +layout(local_size_x_id = 0, local_size_y = 1, local_size_z = 1) in; layout (binding = 0) readonly buffer A {A_TYPE data_a[];}; layout (binding = 1) readonly buffer B {B_TYPE data_b[];}; layout (binding = 2) writeonly buffer D {D_TYPE dst[];}; -layout (push_constant) uniform parameter -{ - int ncols; - int b_offset; - int d_offset; -} p; +layout (constant_id = 0) const uint BLOCK_SIZE = 32; -shared FLOAT_TYPE tmp[QUANT_K]; +shared FLOAT_TYPE tmp[BLOCK_SIZE]; void main() { - const int block_size = int(gl_WorkGroupSize.x); - const int row = int(gl_WorkGroupID.x); - const int tid = int(gl_LocalInvocationID.x); + const uint row = gl_WorkGroupID.x; + const uint tid = gl_LocalInvocationID.x; - const int y_offset = QUANT_R == 1 ? 1 : QUANT_K/2; + const uint y_offset = QUANT_R == 1 ? 1 : QUANT_K/2; tmp[tid] = FLOAT_TYPE(0.0f); - [[unroll]] for (int i = 0; i < p.ncols/block_size; i += 2) { - const int col = i*block_size + 2*tid; - const int ib = (row*p.ncols + col)/QUANT_K; // block index - const int iqs = (col%QUANT_K)/QUANT_R; // quant index - const int iybs = col - col%QUANT_K; // y block start index + [[unroll]] for (uint i = 0; i < p.ncols/BLOCK_SIZE; i += 2) { + const uint col = i*BLOCK_SIZE + 2*tid; + const uint ib = (row*p.ncols + col)/QUANT_K; // block index + const uint iqs = (col%QUANT_K)/QUANT_R; // quant index + const uint iybs = col - col%QUANT_K; // y block start index DEQUANT_FUNC // matrix multiplication - tmp[tid] += FLOAT_TYPE(v.x) * FLOAT_TYPE(data_b[p.b_offset + iybs + iqs + 0]); - tmp[tid] += FLOAT_TYPE(v.y) * FLOAT_TYPE(data_b[p.b_offset + iybs + iqs + y_offset]); + tmp[tid] += FLOAT_TYPE(v.x) * FLOAT_TYPE(data_b[p.b_offset + iybs + iqs + 0]) + + FLOAT_TYPE(v.y) * FLOAT_TYPE(data_b[p.b_offset + iybs + iqs + y_offset]); } // sum up partial sums and write back result barrier(); - [[unroll]] for (int s = block_size/2; s > 0; s >>= 1) { + [[unroll]] for (uint s = BLOCK_SIZE/2; s > 0; s >>= 1) { if (tid < s) { tmp[tid] += tmp[tid + s]; } @@ -804,38 +1003,31 @@ layout (binding = 0) readonly buffer A {A_TYPE data_a[];}; layout (binding = 1) readonly buffer B {B_TYPE data_b[];}; layout (binding = 2) writeonly buffer D {D_TYPE dst[];}; -layout (push_constant) uniform parameter -{ - int ncols; - int b_offset; - int d_offset; -} p; - shared FLOAT_TYPE tmp[32]; void main() { - const int row = int(gl_WorkGroupID.x); + const uint row = gl_WorkGroupID.x; - const int num_blocks_per_row = p.ncols / QUANT_K; - const int ib0 = row*num_blocks_per_row; + const uint num_blocks_per_row = p.ncols / QUANT_K; + const uint ib0 = row*num_blocks_per_row; - const int tid = int(gl_LocalInvocationID.x)/K_QUANTS_PER_ITERATION; // 0...31 or 0...16 - const int ix = int(gl_LocalInvocationID.x)%K_QUANTS_PER_ITERATION; // 0 or 0, 1 + const uint tid = gl_LocalInvocationID.x/K_QUANTS_PER_ITERATION; // 0...31 or 0...16 + const uint ix = gl_LocalInvocationID.x%K_QUANTS_PER_ITERATION; // 0 or 0, 1 - const int step = 16/K_QUANTS_PER_ITERATION; // 16 or 8 + const uint step = 16/K_QUANTS_PER_ITERATION; // 16 or 8 - const int v_im = tid/step; // 0 or 1. 0 computes 0..., 1 computes 128... - const int v_in = tid - step*v_im; // 0...15 or 0...7 + const uint v_im = tid/step; // 0 or 1. 0 computes 0..., 1 computes 128... + const uint v_in = tid - step*v_im; // 0...15 or 0...7 - const int l0 = K_QUANTS_PER_ITERATION*v_in; // 0...15 - const int q_offset = 32*v_im + l0; - const int s_offset = 8*v_im; - const int y_offset = 128*v_im + l0; + const uint l0 = K_QUANTS_PER_ITERATION*v_in; // 0...15 + const uint q_offset = 32*v_im + l0; + const uint s_offset = 8*v_im; + const uint y_offset = 128*v_im + l0; tmp[16 * ix + tid] = FLOAT_TYPE(0.0); // partial sum for thread in warp - [[unroll]] for (int i = ix; i < num_blocks_per_row; i += K_QUANTS_PER_ITERATION) { - const int y_idx = i * QUANT_K + y_offset; + [[unroll]] for (uint i = ix; i < num_blocks_per_row; i += K_QUANTS_PER_ITERATION) { + const uint y_idx = i * QUANT_K + y_offset; const FLOAT_TYPE dall = FLOAT_TYPE(data_a[ib0 + i].d.x); const FLOAT_TYPE dmin = FLOAT_TYPE(data_a[ib0 + i].d.y); @@ -865,7 +1057,7 @@ void main() { // sum up partial sums and write back result barrier(); - [[unroll]] for (int s = 16; s > 0; s >>= 1) { + [[unroll]] for (uint s = 16; s > 0; s >>= 1) { if (tid < s) { tmp[tid] += tmp[tid + s]; } @@ -883,41 +1075,34 @@ layout (binding = 0) readonly buffer A {A_TYPE data_a[];}; layout (binding = 1) readonly buffer B {B_TYPE data_b[];}; layout (binding = 2) writeonly buffer D {D_TYPE dst[];}; -layout (push_constant) uniform parameter -{ - int ncols; - int b_offset; - int d_offset; -} p; - shared FLOAT_TYPE tmp[32]; void main() { - const int row = int(gl_WorkGroupID.x); + const uint row = gl_WorkGroupID.x; - const int num_blocks_per_row = p.ncols / QUANT_K; - const int ib0 = row*num_blocks_per_row; + const uint num_blocks_per_row = p.ncols / QUANT_K; + const uint ib0 = row*num_blocks_per_row; - const int tid = int(gl_LocalInvocationID.x)/K_QUANTS_PER_ITERATION; // 0...31 or 0...16 - const int ix = int(gl_LocalInvocationID.x)%K_QUANTS_PER_ITERATION; // 0 or 0, 1 + const uint tid = gl_LocalInvocationID.x/K_QUANTS_PER_ITERATION; // 0...31 or 0...16 + const uint ix = gl_LocalInvocationID.x%K_QUANTS_PER_ITERATION; // 0 or 0, 1 - const int step = 16/K_QUANTS_PER_ITERATION; // 16 or 8 + const uint step = 16/K_QUANTS_PER_ITERATION; // 16 or 8 - const int v_im = tid/step; // 0 or 1. 0 computes 0..., 1 computes 128... - const int v_in = tid - step*v_im; // 0...15 or 0...7 + const uint v_im = tid/step; // 0 or 1. 0 computes 0..., 1 computes 128... + const uint v_in = tid - step*v_im; // 0...15 or 0...7 const uint8_t m = uint8_t(1 << (4 * v_im)); - const int l0 = K_QUANTS_PER_ITERATION*v_in; // 0...15 - const int q_offset = 32*v_im + l0; - const int y_offset = 128*v_im + l0; + const uint l0 = K_QUANTS_PER_ITERATION*v_in; // 0...15 + const uint q_offset = 32*v_im + l0; + const uint y_offset = 128*v_im + l0; tmp[16 * ix + tid] = FLOAT_TYPE(0.0); // partial sum for thread in warp const uint s_shift = 4 * v_im; - [[unroll]] for (int i = ix; i < num_blocks_per_row; i += K_QUANTS_PER_ITERATION) { - const int y_idx = i * QUANT_K + y_offset; + [[unroll]] for (uint i = ix; i < num_blocks_per_row; i += K_QUANTS_PER_ITERATION) { + const uint y_idx = i * QUANT_K + y_offset; const FLOAT_TYPE d = FLOAT_TYPE(data_a[ib0 + i].d); @@ -937,7 +1122,7 @@ void main() { // sum up partial sums and write back result barrier(); - [[unroll]] for (int s = 16; s > 0; s >>= 1) { + [[unroll]] for (uint s = 16; s > 0; s >>= 1) { if (tid < s) { tmp[tid] += tmp[tid + s]; } @@ -955,42 +1140,35 @@ layout (binding = 0) readonly buffer A {A_TYPE data_a[];}; layout (binding = 1) readonly buffer B {B_TYPE data_b[];}; layout (binding = 2) writeonly buffer D {D_TYPE dst[];}; -layout (push_constant) uniform parameter -{ - int ncols; - int b_offset; - int d_offset; -} p; - shared FLOAT_TYPE tmp[32]; void main() { - const int row = int(gl_WorkGroupID.x); + const uint row = gl_WorkGroupID.x; - const int num_blocks_per_row = p.ncols / QUANT_K; - const int ib0 = row*num_blocks_per_row; + const uint num_blocks_per_row = p.ncols / QUANT_K; + const uint ib0 = row*num_blocks_per_row; - const int tid = int(gl_LocalInvocationID.x)/K_QUANTS_PER_ITERATION; // 0...31 or 0...16 - const int ix = int(gl_LocalInvocationID.x)%K_QUANTS_PER_ITERATION; // 0 or 0, 1 + const uint tid = gl_LocalInvocationID.x/K_QUANTS_PER_ITERATION; // 0...31 or 0...16 + const uint ix = gl_LocalInvocationID.x%K_QUANTS_PER_ITERATION; // 0 or 0, 1 - const int step = 8/K_QUANTS_PER_ITERATION; // 8 or 4 + const uint step = 8/K_QUANTS_PER_ITERATION; // 8 or 4 - const int il = tid/step; // 0...3 - const int ir = tid - step*il; // 0...7 or 0...3 - const int n = 2 * K_QUANTS_PER_ITERATION; // 2 or 4 + const uint il = tid/step; // 0...3 + const uint ir = tid - step*il; // 0...7 or 0...3 + const uint n = 2 * K_QUANTS_PER_ITERATION; // 2 or 4 - const int v_im = il / 2; // 0 or 1. 0 computes 0,32 + 128,160, 1 computes 64,96 + 192,224 - const int v_in = il % 2; + const uint v_im = il / 2; // 0 or 1. 0 computes 0,32 + 128,160, 1 computes 64,96 + 192,224 + const uint v_in = il % 2; - const int l0 = n * (2 * ir + v_in); // 0...15 - const int q_offset = 32*v_im + l0; - const int y_offset = 64*v_im + l0; + const uint l0 = n * (2 * ir + v_in); // 0...15 + const uint q_offset = 32*v_im + l0; + const uint y_offset = 64*v_im + l0; tmp[16 * ix + tid] = FLOAT_TYPE(0.0); // partial sum for thread in warp - [[unroll]] for (int i = ix; i < num_blocks_per_row; i += K_QUANTS_PER_ITERATION) { - const int y1_idx = i * QUANT_K + y_offset; - const int y2_idx = y1_idx + 128; + [[unroll]] for (uint i = ix; i < num_blocks_per_row; i += K_QUANTS_PER_ITERATION) { + const uint y1_idx = i * QUANT_K + y_offset; + const uint y2_idx = y1_idx + 128; const FLOAT_TYPE dall = FLOAT_TYPE(data_a[ib0 + i].d.x); const FLOAT_TYPE dmin = FLOAT_TYPE(data_a[ib0 + i].d.y); @@ -1058,7 +1236,7 @@ void main() { // sum up partial sums and write back result barrier(); - [[unroll]] for (int s = 16; s > 0; s >>= 1) { + [[unroll]] for (uint s = 16; s > 0; s >>= 1) { if (tid < s) { tmp[tid] += tmp[tid + s]; } @@ -1076,42 +1254,35 @@ layout (binding = 0) readonly buffer A {A_TYPE data_a[];}; layout (binding = 1) readonly buffer B {B_TYPE data_b[];}; layout (binding = 2) writeonly buffer D {D_TYPE dst[];}; -layout (push_constant) uniform parameter -{ - int ncols; - int b_offset; - int d_offset; -} p; - shared FLOAT_TYPE tmp[32]; void main() { - const int row = int(gl_WorkGroupID.x); + const uint row = gl_WorkGroupID.x; - const int num_blocks_per_row = p.ncols / QUANT_K; - const int ib0 = row*num_blocks_per_row; + const uint num_blocks_per_row = p.ncols / QUANT_K; + const uint ib0 = row*num_blocks_per_row; - const int tid = int(gl_LocalInvocationID.x)/2; // 0...31 or 0...16 - const int ix = int(gl_LocalInvocationID.x)%2; // 0 or 0, 1 + const uint tid = gl_LocalInvocationID.x/2; // 0...31 or 0...16 + const uint ix = gl_LocalInvocationID.x%2; // 0 or 0, 1 - const int il = tid/4; // 0...3 - const int ir = tid - 4*il; // 0...7 or 0...3 + const uint il = tid/4; // 0...3 + const uint ir = tid - 4*il; // 0...7 or 0...3 - const int v_im = il / 2; // 0 or 1. 0 computes 0,32 + 128,160, 1 computes 64,96 + 192,224 - const int v_in = il % 2; + const uint v_im = il / 2; // 0 or 1. 0 computes 0,32 + 128,160, 1 computes 64,96 + 192,224 + const uint v_in = il % 2; - const int l0 = 4*ir + 2*v_in; // 0...15 - const int q_offset = 32*v_im + l0; - const int y_offset = 64*v_im + l0; + const uint l0 = 4*ir + 2*v_in; // 0...15 + const uint q_offset = 32*v_im + l0; + const uint y_offset = 64*v_im + l0; const uint8_t hm1 = uint8_t(1 << (2*v_im)); const uint8_t hm2 = uint8_t(hm1 << 4); tmp[16 * ix + tid] = FLOAT_TYPE(0.0); // partial sum for thread in warp - [[unroll]] for (int i = ix; i < num_blocks_per_row; i += 2) { - const int y1_idx = i * QUANT_K + y_offset; - const int y2_idx = y1_idx + 128; + [[unroll]] for (uint i = ix; i < num_blocks_per_row; i += 2) { + const uint y1_idx = i * QUANT_K + y_offset; + const uint y2_idx = y1_idx + 128; const FLOAT_TYPE dall = FLOAT_TYPE(data_a[ib0 + i].d.x); const FLOAT_TYPE dmin = FLOAT_TYPE(data_a[ib0 + i].d.y); @@ -1175,7 +1346,7 @@ void main() { // sum up partial sums and write back result barrier(); - [[unroll]] for (int s = 16; s > 0; s >>= 1) { + [[unroll]] for (uint s = 16; s > 0; s >>= 1) { if (tid < s) { tmp[tid] += tmp[tid + s]; } @@ -1193,46 +1364,40 @@ layout (binding = 0) readonly buffer A {A_TYPE data_a[];}; layout (binding = 1) readonly buffer B {B_TYPE data_b[];}; layout (binding = 2) writeonly buffer D {D_TYPE dst[];}; -layout (push_constant) uniform parameter -{ - int ncols; - int b_offset; - int d_offset; -} p; - shared FLOAT_TYPE tmp[32]; void main() { - const int row = int(gl_WorkGroupID.x); + const uint block_size = gl_WorkGroupSize.x; + const uint row = gl_WorkGroupID.x; - const int num_blocks_per_row = p.ncols / QUANT_K; - const int ib0 = row*num_blocks_per_row; + const uint num_blocks_per_row = p.ncols / QUANT_K; + const uint ib0 = row*num_blocks_per_row; - const int tid = int(gl_LocalInvocationID.x)/K_QUANTS_PER_ITERATION; // 0...31 or 0...16 - const int ix = int(gl_LocalInvocationID.x)%K_QUANTS_PER_ITERATION; // 0 or 0, 1 + const uint tid = gl_LocalInvocationID.x/K_QUANTS_PER_ITERATION; // 0...31 or 0...16 + const uint ix = gl_LocalInvocationID.x%K_QUANTS_PER_ITERATION; // 0 or 0, 1 - const int step = 16/K_QUANTS_PER_ITERATION; // 16 or 8 + const uint step = 16/K_QUANTS_PER_ITERATION; // 16 or 8 - const int v_im = tid/step; // 0 or 1. 0 computes 0..., 1 computes 128... - const int v_in = tid - step*v_im; // 0...15 or 0...7 + const uint v_im = tid/step; // 0 or 1. 0 computes 0..., 1 computes 128... + const uint v_in = tid - step*v_im; // 0...15 or 0...7 #if K_QUANTS_PER_ITERATION == 1 - const int l0 = K_QUANTS_PER_ITERATION*v_in; // 0...15 - const int is = 0; + const uint l0 = v_in; // 0...15 + const uint is = 0; #else - const int l0 = 4 * v_in; // 0, 4, 8, ..., 28 - const int is = v_in / 4; + const uint l0 = 4 * v_in; // 0, 4, 8, ..., 28 + const uint is = v_in / 4; #endif - const int ql_offset = 64*v_im + l0; - const int qh_offset = 32*v_im + l0; - const int s_offset = 8*v_im + is; - const int y_offset = 128*v_im + l0; + const uint ql_offset = 64*v_im + l0; + const uint qh_offset = 32*v_im + l0; + const uint s_offset = 8*v_im + is; + const uint y_offset = 128*v_im + l0; tmp[16 * ix + tid] = FLOAT_TYPE(0.0); // partial sum for thread in warp - [[unroll]] for (int i = ix; i < num_blocks_per_row; i += K_QUANTS_PER_ITERATION) { - const int y_idx = i * QUANT_K + y_offset; + [[unroll]] for (uint i = ix; i < num_blocks_per_row; i += K_QUANTS_PER_ITERATION) { + const uint y_idx = i * QUANT_K + y_offset; const FLOAT_TYPE d = FLOAT_TYPE(data_a[ib0 + i].d); @@ -1260,7 +1425,7 @@ void main() { // sum up partial sums and write back result barrier(); - [[unroll]] for (int s = 16; s > 0; s >>= 1) { + [[unroll]] for (uint s = 16; s > 0; s >>= 1) { if (tid < s) { tmp[tid] += tmp[tid + s]; } @@ -1421,34 +1586,6 @@ void main() { } """ -# F16 to F32 -f32_to_f16_src = """#version 450 - -#extension GL_EXT_shader_16bit_storage : require - -layout(local_size_x = 64, local_size_y = 1, local_size_z = 1) in; - -layout (binding = 0) readonly buffer A {float data_a[];}; -layout (binding = 1) writeonly buffer D {float16_t data_b[];}; - -layout (push_constant) uniform parameter -{ - int M; - int K; - int stride_a; - int stride_b; -} p; - -void main() { - const int row = int(gl_GlobalInvocationID.x % p.K); - const int col = int(gl_GlobalInvocationID.x / p.K); - - if (row < p.K && col < p.M) { - data_b[col * p.stride_b + row] = float16_t(data_a[col * p.stride_a + row]); - } -} -""" - generic_head = """ #version 450 @@ -1463,107 +1600,17 @@ layout (push_constant) uniform parameter } p; """ -# MUL F32 -mul_body = """layout(local_size_x = 512, local_size_y = 1, local_size_z = 1) in; - -layout (binding = 0) readonly buffer X {A_TYPE data_a[];}; -layout (binding = 1) readonly buffer Y {B_TYPE data_b[];}; -layout (binding = 2) writeonly buffer D {D_TYPE data_d[];}; - -void main() { - const uint idx = gl_GlobalInvocationID.x; - - if (idx >= p.KX) { - return; - } - - data_d[idx] = D_TYPE(FLOAT_TYPE(data_a[idx]) * FLOAT_TYPE(data_b[idx % p.KY])); -} -""" - -# ADD -add_body = """ -layout(local_size_x = 512, local_size_y = 1, local_size_z = 1) in; - -layout (binding = 0) readonly buffer X {A_TYPE data_a[];}; -layout (binding = 1) readonly buffer Y {B_TYPE data_b[];}; -layout (binding = 2) writeonly buffer D {D_TYPE data_d[];}; - -void main() { - const uint idx = gl_GlobalInvocationID.x; - - if (idx >= p.KX) { - return; - } - - data_d[idx] = D_TYPE(FLOAT_TYPE(data_a[idx]) + FLOAT_TYPE(data_b[idx % p.KY])); -} -""" - -# SCALE -scale_body = """layout(local_size_x = 512, local_size_y = 1, local_size_z = 1) in; - -layout (binding = 0) readonly buffer X {A_TYPE data_a[];}; -layout (binding = 1) writeonly buffer D {D_TYPE data_d[];}; - -void main() { - const uint idx = gl_GlobalInvocationID.x; - - if (idx >= p.KX) { - return; - } - - data_d[idx] = D_TYPE(FLOAT_TYPE(data_a[idx]) * FLOAT_TYPE(p.param1)); -} -""" - -# SQR -sqr_body = """layout(local_size_x = 512, local_size_y = 1, local_size_z = 1) in; - -layout (binding = 0) readonly buffer X {A_TYPE data_a[];}; -layout (binding = 1) writeonly buffer D {D_TYPE data_d[];}; - -void main() { - const uint idx = gl_GlobalInvocationID.x; - - if (idx >= p.KX) { - return; - } - - const FLOAT_TYPE val = FLOAT_TYPE(data_a[idx]); - data_d[idx] = D_TYPE(val * val); -} -""" - -# CLAMP -clamp_body = """layout(local_size_x = 512, local_size_y = 1, local_size_z = 1) in; - -layout (binding = 0) readonly buffer X {A_TYPE data_a[];}; -layout (binding = 1) writeonly buffer D {D_TYPE data_d[];}; - -void main() { - const uint idx = gl_GlobalInvocationID.x; - - if (idx >= p.KX) { - return; - } - - const FLOAT_TYPE val = FLOAT_TYPE(data_a[idx]); - data_d[idx] = D_TYPE(val < p.param1 ? p.param1 : (val > p.param2 ? p.param2 : val)); -} -""" - -# CPY -cpy_src = """#version 450 +generic_unary_op_head = """#version 450 #extension GL_EXT_shader_16bit_storage : require layout (push_constant) uniform parameter { uint ne; - uint ne00; uint ne01; uint nb00; uint nb01; uint nb02; - uint ne10; uint ne11; uint nb10; uint nb11; uint nb12; + uint ne00; uint ne01; uint ne02; uint ne03; uint nb00; uint nb01; uint nb02; uint nb03; + uint ne10; uint ne11; uint ne12; uint ne13; uint nb10; uint nb11; uint nb12; uint nb13; uint d_offset; + float param1; float param2; } p; layout(local_size_x = 512, local_size_y = 1, local_size_z = 1) in; @@ -1571,28 +1618,129 @@ layout(local_size_x = 512, local_size_y = 1, local_size_z = 1) in; layout (binding = 0) readonly buffer A {A_TYPE data_a[];}; layout (binding = 1) writeonly buffer D {D_TYPE data_d[];}; +uint src0_idx(uint idx) { + const uint i03 = idx / (p.ne02*p.ne01*p.ne00); + const uint i03_offset = i03 * p.ne02*p.ne01*p.ne00; + const uint i02 = (idx - i03_offset) / (p.ne01*p.ne00); + const uint i02_offset = i02*p.ne01*p.ne00; + const uint i01 = (idx - i03_offset - i02_offset) / p.ne00; + const uint i00 = idx - i03_offset - i02_offset - i01*p.ne00; + return i03*p.nb03 + i02*p.nb02 + i01*p.nb01 + i00*p.nb00; +} + +uint dst_idx(uint idx) { + const uint i13 = idx / (p.ne12*p.ne11*p.ne10); + const uint i13_offset = i13 * p.ne12*p.ne11*p.ne10; + const uint i12 = (idx - i13_offset) / (p.ne11*p.ne10); + const uint i12_offset = i12*p.ne11*p.ne10; + const uint i11 = (idx - i13_offset - i12_offset) / p.ne10; + const uint i10 = idx - i13_offset - i12_offset - i11*p.ne10; + return i13*p.nb13 + i12*p.nb12 + i11*p.nb11 + i10*p.nb10; +} + void main() { if (gl_GlobalInvocationID.x >= p.ne) { return; } - - const uint i02 = gl_GlobalInvocationID.x / (p.ne00*p.ne01); - const uint i01 = (gl_GlobalInvocationID.x - i02*p.ne01*p.ne00) / p.ne00; - const uint i00 = gl_GlobalInvocationID.x - i02*p.ne01*p.ne00 - i01*p.ne00; - const uint a_idx = i00*p.nb00 + i01*p.nb01 + i02*p.nb02; - - const uint i12 = gl_GlobalInvocationID.x / (p.ne10*p.ne11); - const uint i11 = (gl_GlobalInvocationID.x - i12*p.ne11*p.ne10) / p.ne10; - const uint i10 = gl_GlobalInvocationID.x - i12*p.ne11*p.ne10 - i11*p.ne10; - const uint d_idx = i10*p.nb10 + i11*p.nb11 + i12*p.nb12; """ + +generic_binary_op_head = """#version 450 + +#extension GL_EXT_shader_16bit_storage : require + +layout (push_constant) uniform parameter +{ + uint ne; + uint ne00; uint ne01; uint ne02; uint ne03; uint nb00; uint nb01; uint nb02; uint nb03; + uint ne10; uint ne11; uint ne12; uint ne13; uint nb10; uint nb11; uint nb12; uint nb13; + uint ne20; uint ne21; uint ne22; uint ne23; uint nb20; uint nb21; uint nb22; uint nb23; + uint d_offset; + uint param1; uint param2; +} p; + +layout(local_size_x = 512, local_size_y = 1, local_size_z = 1) in; + +layout (binding = 0) readonly buffer A {A_TYPE data_a[];}; +layout (binding = 1) readonly buffer B {A_TYPE data_b[];}; +layout (binding = 2) writeonly buffer D {D_TYPE data_d[];}; + +uint src0_idx(uint idx) { + const uint i03 = idx / (p.ne02*p.ne01*p.ne00); + const uint i03_offset = i03 * p.ne02*p.ne01*p.ne00; + const uint i02 = (idx - i03_offset) / (p.ne01*p.ne00); + const uint i02_offset = i02*p.ne01*p.ne00; + const uint i01 = (idx - i03_offset - i02_offset) / p.ne00; + const uint i00 = idx - i03_offset - i02_offset - i01*p.ne00; + return i03*p.nb03 + i02*p.nb02 + i01*p.nb01 + i00*p.nb00; +} + +uint src1_idx(uint idx) { + const uint i03 = idx / (p.ne02*p.ne01*p.ne00); + const uint i03_offset = i03 * p.ne02*p.ne01*p.ne00; + const uint i02 = (idx - i03_offset) / (p.ne01*p.ne00); + const uint i02_offset = i02*p.ne01*p.ne00; + const uint i01 = (idx - i03_offset - i02_offset) / p.ne00; + const uint i00 = idx - i03_offset - i02_offset - i01*p.ne00; + + return (i03 % p.ne13)*p.nb13 + (i02 % p.ne12)*p.nb12 + (i01 % p.ne11)*p.nb11 + (i00 % p.ne10)*p.nb10; +} + +uint dst_idx(uint idx) { + const uint i23 = idx / (p.ne22*p.ne21*p.ne20); + const uint i23_offset = i23 * p.ne22*p.ne21*p.ne20; + const uint i22 = (idx - i23_offset) / (p.ne21*p.ne20); + const uint i22_offset = i22*p.ne21*p.ne20; + const uint i21 = (idx - i23_offset - i22_offset) / p.ne20; + const uint i20 = idx - i23_offset - i22_offset - i21*p.ne20; + return i23*p.nb23 + i22*p.nb22 + i21*p.nb21 + i20*p.nb20; +} + +void main() { + if (gl_GlobalInvocationID.x >= p.ne) { + return; + } +""" + +# MUL F32 +mul_body = """ + data_d[p.d_offset + dst_idx(gl_GlobalInvocationID.x)] = D_TYPE(FLOAT_TYPE(data_a[src0_idx(gl_GlobalInvocationID.x)]) * FLOAT_TYPE(data_b[src1_idx(gl_GlobalInvocationID.x)])); +} +""" + +# ADD +add_body = """ + data_d[p.d_offset + dst_idx(gl_GlobalInvocationID.x)] = D_TYPE(FLOAT_TYPE(data_a[src0_idx(gl_GlobalInvocationID.x)]) + FLOAT_TYPE(data_b[src1_idx(gl_GlobalInvocationID.x)])); +} +""" + +# SCALE +scale_body = """ + data_d[p.d_offset + dst_idx(gl_GlobalInvocationID.x)] = D_TYPE(FLOAT_TYPE(data_a[src0_idx(gl_GlobalInvocationID.x)]) + FLOAT_TYPE(p.param1)); +} +""" + +# SQR +sqr_body = """ + const FLOAT_TYPE val = FLOAT_TYPE(data_a[src0_idx(gl_GlobalInvocationID.x)]); + data_d[p.d_offset + dst_idx(gl_GlobalInvocationID.x)] = D_TYPE(val * val); +} +""" + +# CLAMP +clamp_body = """ + const FLOAT_TYPE val = FLOAT_TYPE(data_a[src0_idx(gl_GlobalInvocationID.x)]); + data_d[p.d_offset + dst_idx(gl_GlobalInvocationID.x)] = D_TYPE(val < p.param1 ? p.param1 : (val > p.param2 ? p.param2 : val)); +} +""" + +# CPY cpy_end = """ - data_d[p.d_offset + d_idx] = D_TYPE(data_a[a_idx]); + data_d[p.d_offset + dst_idx(gl_GlobalInvocationID.x)] = D_TYPE(data_a[src0_idx(gl_GlobalInvocationID.x)]); } """ # Causes an optimization error otherwise cpy_f16_f16_end = """ - data_d[p.d_offset + d_idx] = data_a[a_idx]; + data_d[p.d_offset + dst_idx(gl_GlobalInvocationID.x)] = data_a[src0_idx(gl_GlobalInvocationID.x)]; } """ @@ -1815,6 +1963,24 @@ void main() { """ # SOFT_MAX +soft_max_head = """ +#version 450 + +#extension GL_EXT_shader_16bit_storage : require + +layout (push_constant) uniform parameter +{ + uint KX; + uint KY; + uint KZ; + float scale; + float max_bias; + float m0; + float m1; + uint n_head_log2; +} p; +""" + soft_max_body = """ #extension GL_EXT_control_flow_attributes : enable #define BLOCK_SIZE 512 @@ -1823,7 +1989,8 @@ layout(local_size_x = BLOCK_SIZE, local_size_y = 1, local_size_z = 1) in; layout (binding = 0) readonly buffer X {A_TYPE data_a[];}; layout (binding = 1) readonly buffer Y {B_TYPE data_b[];}; -layout (binding = 2) buffer D {D_TYPE data_d[];}; +layout (binding = 2) readonly buffer Z {C_TYPE data_c[];}; +layout (binding = 3) buffer D {D_TYPE data_d[];}; shared FLOAT_TYPE vals[BLOCK_SIZE]; @@ -1832,11 +1999,23 @@ void main() { const uint rowx = gl_WorkGroupID.x; const uint rowy = rowx % p.KY; + float slope = 0.0f; + + // ALiBi + if (p.max_bias > 0.0f) { + const uint h = rowx/p.KY; // head index + + const float base = h < p.n_head_log2 ? p.m0 : p.m1; + const uint exp = h < p.n_head_log2 ? h + 1 : 2*(h - p.n_head_log2) + 1; + + slope = pow(base, exp); + } + // Find max vals[tid] = uintBitsToFloat(0xFF800000); [[unroll]] for (uint col = tid; col < p.KX; col += BLOCK_SIZE) { - vals[tid] = max(vals[tid], FLOAT_TYPE(data_a[rowx * p.KX + col]) * p.param1 + (p.KY > 0 ? FLOAT_TYPE(data_b[rowy * p.KX + col]) : FLOAT_TYPE(0.0f))); + vals[tid] = max(vals[tid], FLOAT_TYPE(data_a[rowx * p.KX + col]) * p.scale + (p.KY > 0 ? FLOAT_TYPE(data_b[rowy * p.KX + col]) : FLOAT_TYPE(0.0f)) + (p.KZ > 0 ? slope * data_c[col] : 0.0f)); } barrier(); @@ -1855,7 +2034,7 @@ void main() { [[unroll]] for (uint col = tid; col < p.KX; col += BLOCK_SIZE) { const uint i = rowx * p.KX + col; - const FLOAT_TYPE val = exp(FLOAT_TYPE(data_a[i]) * p.param1 + (p.KY > 0 ? FLOAT_TYPE(data_b[rowy * p.KX + col]) : FLOAT_TYPE(0.0f)) - max_val); + const FLOAT_TYPE val = exp(FLOAT_TYPE(data_a[i]) * p.scale + (p.KY > 0 ? FLOAT_TYPE(data_b[rowy * p.KX + col]) : FLOAT_TYPE(0.0f)) - max_val); vals[tid] += val; data_d[i] = D_TYPE(val); } @@ -2028,6 +2207,65 @@ void main() { } """ +argsort_src = """ +#version 450 + +#extension GL_EXT_shader_16bit_storage : require + +layout(local_size_x = 1024, local_size_y = 1, local_size_z = 1) in; + +layout (binding = 0) readonly buffer A {A_TYPE data_a[];}; +layout (binding = 1) buffer D {int data_d[];}; + +layout (push_constant) uniform parameter { + uint ncols; + bool ascending; +} p; + +void swap(uint idx0, uint idx1) { + int tmp = data_d[idx0]; + data_d[idx0] = data_d[idx1]; + data_d[idx1] = tmp; +} + +void main() { + // bitonic sort + const int col = int(gl_LocalInvocationID.x); + const uint row = gl_WorkGroupID.y; + + if (col >= p.ncols) { + return; + } + + const uint a_idx = row * p.ncols; + const uint d_idx = row * p.ncols; + + // initialize indices + if (col < p.ncols) { + data_d[col] = col; + } + barrier(); + + for (uint k = 2; k <= p.ncols; k *= 2) { + for (uint j = k / 2; j > 0; j /= 2) { + const uint ixj = col ^ j; + if (ixj > col) { + if ((col & k) == 0) { + if (p.ascending ? data_a[a_idx + data_d[d_idx + col]] > data_a[a_idx + data_d[d_idx + ixj]] : data_a[a_idx + data_d[d_idx + col]] < data_a[a_idx + data_d[d_idx + ixj]]) { + swap(d_idx + col, d_idx + ixj); + } + } else { + if (p.ascending ? data_a[a_idx + data_d[d_idx + col]] < data_a[a_idx + data_d[d_idx + ixj]] : data_a[a_idx + data_d[d_idx + col]] > data_a[a_idx + data_d[d_idx + ixj]]) { + swap(d_idx + col, d_idx + ixj); + } + } + } + barrier(); + } + } +} +""" + GLSLC = "glslc" VK_NUM_TYPES = 16 @@ -2129,6 +2367,8 @@ async def main(): tasks = [] + stream = [] + for fp16 in (False, True): # mulmat if fp16: @@ -2142,28 +2382,41 @@ async def main(): vec_type_f16 = "f16vec4" vec_type = "vec4" - stream = [] - stream.extend((mulmat_head, shader_float_type, mulmat_body)) - tasks.append(string_to_spv("matmul_f32_l", "".join(stream), {"A_TYPE": "float", "B_TYPE": "float", "D_TYPE": "float"}, fp16)) - tasks.append(string_to_spv("matmul_f32_m", "".join(stream), {"A_TYPE": "float", "B_TYPE": "float", "D_TYPE": "float"}, fp16)) - tasks.append(string_to_spv("matmul_f32_s", "".join(stream), {"A_TYPE": "float", "B_TYPE": "float", "D_TYPE": "float"}, fp16)) - tasks.append(string_to_spv("matmul_f32_aligned_l", "".join(stream), {"LOAD_VEC": load_vec, "A_TYPE": vec_type, "B_TYPE": vec_type, "D_TYPE": "float"}, fp16)) - tasks.append(string_to_spv("matmul_f32_aligned_m", "".join(stream), {"LOAD_VEC": load_vec, "A_TYPE": vec_type, "B_TYPE": vec_type, "D_TYPE": "float"}, fp16)) - tasks.append(string_to_spv("matmul_f32_aligned_s", "".join(stream), {"LOAD_VEC": load_vec, "A_TYPE": vec_type, "B_TYPE": vec_type, "D_TYPE": "float"}, fp16)) + stream.clear() + stream.extend((mulmat_head, shader_float_type, mulmat_body1, mulmat_load_scalar, mulmat_body2)) + tasks.append(string_to_spv("matmul_f32", "".join(stream), {"A_TYPE": "float", "B_TYPE": "float", "D_TYPE": "float"}, fp16)) + tasks.append(string_to_spv("matmul_f32_aligned", "".join(stream), {"LOAD_VEC_A": load_vec, "LOAD_VEC_B": load_vec, "A_TYPE": vec_type, "B_TYPE": vec_type, "D_TYPE": "float"}, fp16)) - tasks.append(string_to_spv("matmul_f16_l", "".join(stream), {"A_TYPE": "float16_t", "B_TYPE": "float16_t", "D_TYPE": "float"}, fp16)) - tasks.append(string_to_spv("matmul_f16_m", "".join(stream), {"A_TYPE": "float16_t", "B_TYPE": "float16_t", "D_TYPE": "float"}, fp16)) - tasks.append(string_to_spv("matmul_f16_s", "".join(stream), {"A_TYPE": "float16_t", "B_TYPE": "float16_t", "D_TYPE": "float"}, fp16)) - tasks.append(string_to_spv("matmul_f16_aligned_l", "".join(stream), {"LOAD_VEC": load_vec, "A_TYPE": vec_type_f16, "B_TYPE": vec_type_f16, "D_TYPE": "float"}, fp16)) - tasks.append(string_to_spv("matmul_f16_aligned_m", "".join(stream), {"LOAD_VEC": load_vec, "A_TYPE": vec_type_f16, "B_TYPE": vec_type_f16, "D_TYPE": "float"}, fp16)) - tasks.append(string_to_spv("matmul_f16_aligned_s", "".join(stream), {"LOAD_VEC": load_vec, "A_TYPE": vec_type_f16, "B_TYPE": vec_type_f16, "D_TYPE": "float"}, fp16)) + tasks.append(string_to_spv("matmul_f16", "".join(stream), {"A_TYPE": "float16_t", "B_TYPE": "float16_t", "D_TYPE": "float"}, fp16)) + tasks.append(string_to_spv("matmul_f16_aligned", "".join(stream), {"LOAD_VEC_A": load_vec, "LOAD_VEC_B": load_vec, "A_TYPE": vec_type_f16, "B_TYPE": vec_type_f16, "D_TYPE": "float"}, fp16)) - tasks.append(string_to_spv("matmul_f16_f32_l", "".join(stream), {"A_TYPE": "float16_t", "B_TYPE": "float", "D_TYPE": "float"}, fp16)) - tasks.append(string_to_spv("matmul_f16_f32_m", "".join(stream), {"A_TYPE": "float16_t", "B_TYPE": "float", "D_TYPE": "float"}, fp16)) - tasks.append(string_to_spv("matmul_f16_f32_s", "".join(stream), {"A_TYPE": "float16_t", "B_TYPE": "float", "D_TYPE": "float"}, fp16)) - tasks.append(string_to_spv("matmul_f16_f32_aligned_l", "".join(stream), {"LOAD_VEC": load_vec, "A_TYPE": vec_type_f16, "B_TYPE": vec_type, "D_TYPE": "float"}, fp16)) - tasks.append(string_to_spv("matmul_f16_f32_aligned_m", "".join(stream), {"LOAD_VEC": load_vec, "A_TYPE": vec_type_f16, "B_TYPE": vec_type, "D_TYPE": "float"}, fp16)) - tasks.append(string_to_spv("matmul_f16_f32_aligned_s", "".join(stream), {"LOAD_VEC": load_vec, "A_TYPE": vec_type_f16, "B_TYPE": vec_type, "D_TYPE": "float"}, fp16)) + tasks.append(string_to_spv("matmul_f16_f32", "".join(stream), {"A_TYPE": "float16_t", "B_TYPE": "float", "D_TYPE": "float"}, fp16)) + tasks.append(string_to_spv("matmul_f16_f32_aligned", "".join(stream), {"LOAD_VEC_A": load_vec, "LOAD_VEC_B": load_vec, "A_TYPE": vec_type_f16, "B_TYPE": vec_type, "D_TYPE": "float"}, fp16)) + + stream.clear() + stream.extend((mulmat_head, shader_int8_ext, shader_float_type, shader_q4_0_defines, mulmat_body1, mulmat_load_q4_0, mulmat_body2)) + tasks.append(string_to_spv("matmul_q4_0_f32", "".join(stream), {"LOAD_VEC_A": 2, "A_TYPE": "block_q4_0", "B_TYPE": "float", "D_TYPE": "float"}, fp16)) + tasks.append(string_to_spv("matmul_q4_0_f32_aligned", "".join(stream), {"LOAD_VEC_A": 2, "LOAD_VEC_B": load_vec, "A_TYPE": "block_q4_0", "B_TYPE": vec_type, "D_TYPE": "float"}, fp16)) + + stream.clear() + stream.extend((mulmat_head, shader_int8_ext, shader_float_type, shader_q4_1_defines, mulmat_body1, mulmat_load_q4_1, mulmat_body2)) + tasks.append(string_to_spv("matmul_q4_1_f32", "".join(stream), {"LOAD_VEC_A": 2, "A_TYPE": "block_q4_1", "B_TYPE": "float", "D_TYPE": "float"}, fp16)) + tasks.append(string_to_spv("matmul_q4_1_f32_aligned", "".join(stream), {"LOAD_VEC_A": 2, "LOAD_VEC_B": load_vec, "A_TYPE": "block_q4_1", "B_TYPE": vec_type, "D_TYPE": "float"}, fp16)) + + stream.clear() + stream.extend((mulmat_head, shader_int8_ext, shader_float_type, shader_q5_0_defines, mulmat_body1, mulmat_load_q5_0, mulmat_body2)) + tasks.append(string_to_spv("matmul_q5_0_f32", "".join(stream), {"LOAD_VEC_A": 2, "A_TYPE": "block_q5_0", "B_TYPE": "float", "D_TYPE": "float"}, fp16)) + tasks.append(string_to_spv("matmul_q5_0_f32_aligned", "".join(stream), {"LOAD_VEC_A": 2, "LOAD_VEC_B": load_vec, "A_TYPE": "block_q5_0", "B_TYPE": vec_type, "D_TYPE": "float"}, fp16)) + + stream.clear() + stream.extend((mulmat_head, shader_int8_ext, shader_float_type, shader_q5_1_defines, mulmat_body1, mulmat_load_q5_1, mulmat_body2)) + tasks.append(string_to_spv("matmul_q5_1_f32", "".join(stream), {"LOAD_VEC_A": 2, "A_TYPE": "block_q5_1", "B_TYPE": "float", "D_TYPE": "float"}, fp16)) + tasks.append(string_to_spv("matmul_q5_1_f32_aligned", "".join(stream), {"LOAD_VEC_A": 2, "LOAD_VEC_B": load_vec, "A_TYPE": "block_q5_1", "B_TYPE": vec_type, "D_TYPE": "float"}, fp16)) + + stream.clear() + stream.extend((mulmat_head, shader_int8_ext, shader_float_type, shader_q8_0_defines, mulmat_body1, mulmat_load_q8_0, mulmat_body2)) + tasks.append(string_to_spv("matmul_q8_0_f32", "".join(stream), {"LOAD_VEC_A": 2, "A_TYPE": "block_q8_0", "B_TYPE": "float", "D_TYPE": "float"}, fp16)) + tasks.append(string_to_spv("matmul_q8_0_f32_aligned", "".join(stream), {"LOAD_VEC_A": 2, "LOAD_VEC_B": load_vec, "A_TYPE": "block_q8_0", "B_TYPE": vec_type, "D_TYPE": "float"}, fp16)) # Shaders where precision is needed, so no fp16 version @@ -2205,18 +2458,18 @@ async def main(): stream.extend((dequant_head, shader_int8_ext, shader_f32)) - if i == GGML_TYPE_F16: - stream.extend((shader_f16_defines, shader_f16_dequant_func, dequant_body)) + if i == GGML_TYPE_F32: + stream.append(dequant_f32_body) elif i == GGML_TYPE_Q4_0: - stream.extend((shader_q4_0_defines, shader_q4_0_dequant_func, dequant_body)) + stream.extend((shader_q4_0_defines, dequant_q4_0_body)) elif i == GGML_TYPE_Q4_1: - stream.extend((shader_q4_1_defines, shader_q4_1_dequant_func, dequant_body)) + stream.extend((shader_q4_1_defines, dequant_q4_1_body)) elif i == GGML_TYPE_Q5_0: - stream.extend((shader_q5_0_defines, shader_q5_0_dequant_func, dequant_body)) + stream.extend((shader_q5_0_defines, dequant_q5_0_body)) elif i == GGML_TYPE_Q5_1: - stream.extend((shader_q5_1_defines, shader_q5_1_dequant_func, dequant_body)) + stream.extend((shader_q5_1_defines, dequant_q5_1_body)) elif i == GGML_TYPE_Q8_0: - stream.extend((shader_q8_0_defines, shader_q8_0_dequant_func, dequant_body)) + stream.extend((shader_q8_0_defines, dequant_q8_0_body)) elif i == GGML_TYPE_Q2_K: stream.extend((shader_q2_K_defines, dequant_q2_K_body)) elif i == GGML_TYPE_Q3_K: @@ -2232,8 +2485,6 @@ async def main(): tasks.append(string_to_spv(f"dequant_{type_names[i]}", "".join(stream), {"D_TYPE": "float16_t"})) - tasks.append(string_to_spv("f32_to_f16", f32_to_f16_src, {})) - # get_rows for i in range(0, VK_NUM_TYPES): stream.clear() @@ -2264,20 +2515,20 @@ async def main(): tasks.append(string_to_spv("norm_f32", f"{generic_head}\n{shader_f32}\n{norm_body}", {"A_TYPE": "float", "D_TYPE": "float"})) tasks.append(string_to_spv("rms_norm_f32", f"{generic_head}\n{shader_f32}\n{rms_norm_body}", {"A_TYPE": "float", "D_TYPE": "float"})) - tasks.append(string_to_spv("cpy_f32_f32", f"{cpy_src}\n{cpy_end}", {"A_TYPE": "float", "D_TYPE": "float"})) - tasks.append(string_to_spv("cpy_f32_f16", f"{cpy_src}\n{cpy_end}", {"A_TYPE": "float", "D_TYPE": "float16_t"})) - tasks.append(string_to_spv("cpy_f16_f16", f"{cpy_src}\n{cpy_f16_f16_end}", {"A_TYPE": "float16_t", "D_TYPE": "float16_t"})) + tasks.append(string_to_spv("cpy_f32_f32", f"{generic_unary_op_head}\n{cpy_end}", {"A_TYPE": "float", "D_TYPE": "float"})) + tasks.append(string_to_spv("cpy_f32_f16", f"{generic_unary_op_head}\n{cpy_end}", {"A_TYPE": "float", "D_TYPE": "float16_t"})) + tasks.append(string_to_spv("cpy_f16_f16", f"{generic_unary_op_head}\n{cpy_f16_f16_end}", {"A_TYPE": "float16_t", "D_TYPE": "float16_t"})) - tasks.append(string_to_spv("add_f32", f"{generic_head}\n{shader_f32}\n{add_body}", {"A_TYPE": "float", "B_TYPE": "float", "D_TYPE": "float"})) + tasks.append(string_to_spv("add_f32", f"{generic_binary_op_head}\n{add_body}", {"A_TYPE": "float", "B_TYPE": "float", "D_TYPE": "float", "FLOAT_TYPE": "float"})) tasks.append(string_to_spv("split_k_reduce", mulmat_split_k_reduce_src, {})) - tasks.append(string_to_spv("mul_f32", f"{generic_head}\n{shader_f32}\n{mul_body}", {"A_TYPE": "float", "B_TYPE": "float", "D_TYPE": "float"})) + tasks.append(string_to_spv("mul_f32", f"{generic_binary_op_head}\n{mul_body}", {"A_TYPE": "float", "B_TYPE": "float", "D_TYPE": "float", "FLOAT_TYPE": "float"})) - tasks.append(string_to_spv("scale_f32", f"{generic_head}\n{shader_f32}\n{scale_body}", {"A_TYPE": "float", "D_TYPE": "float"})) + tasks.append(string_to_spv("scale_f32", f"{generic_unary_op_head}\n{scale_body}", {"A_TYPE": "float", "D_TYPE": "float", "FLOAT_TYPE": "float"})) - tasks.append(string_to_spv("sqr_f32", f"{generic_head}\n{shader_f32}\n{sqr_body}", {"A_TYPE": "float", "D_TYPE": "float"})) + tasks.append(string_to_spv("sqr_f32", f"{generic_unary_op_head}\n{sqr_body}", {"A_TYPE": "float", "D_TYPE": "float", "FLOAT_TYPE": "float"})) - tasks.append(string_to_spv("clamp_f32", f"{generic_head}\n{shader_f32}\n{clamp_body}", {"A_TYPE": "float", "D_TYPE": "float"})) + tasks.append(string_to_spv("clamp_f32", f"{generic_unary_op_head}\n{clamp_body}", {"A_TYPE": "float", "D_TYPE": "float", "FLOAT_TYPE": "float"})) tasks.append(string_to_spv("gelu_f32", f"{generic_head}\n{shader_f32}\n{gelu_body}", {"A_TYPE": "float", "D_TYPE": "float"})) tasks.append(string_to_spv("silu_f32", f"{generic_head}\n{shader_f32}\n{silu_body}", {"A_TYPE": "float", "D_TYPE": "float"})) @@ -2285,7 +2536,7 @@ async def main(): tasks.append(string_to_spv("diag_mask_inf_f32", f"{diag_mask_inf_head}\n{shader_f32}\n{diag_mask_inf_body}", {"A_TYPE": "float", "D_TYPE": "float"})) - tasks.append(string_to_spv("soft_max_f32", f"{generic_head}\n{shader_f32}\n{soft_max_body}", {"A_TYPE": "float", "B_TYPE": "float", "D_TYPE": "float"})) + tasks.append(string_to_spv("soft_max_f32", f"{soft_max_head}\n{shader_f32}\n{soft_max_body}", {"A_TYPE": "float", "B_TYPE": "float", "C_TYPE": "float", "D_TYPE": "float"})) tasks.append(string_to_spv("rope_f32", rope_src, {"A_TYPE": "float", "D_TYPE": "float"})) tasks.append(string_to_spv("rope_f16", rope_src, {"A_TYPE": "float16_t", "D_TYPE": "float16_t"})) @@ -2293,6 +2544,8 @@ async def main(): tasks.append(string_to_spv("rope_neox_f32", rope_neox_src, {"A_TYPE": "float", "D_TYPE": "float"})) tasks.append(string_to_spv("rope_neox_f16", rope_neox_src, {"A_TYPE": "float16_t", "D_TYPE": "float16_t"})) + tasks.append(string_to_spv("argsort_f32", argsort_src, {"A_TYPE": "float"})) + # Helper to decorate tasks with semaphore acquisition. async def withSemaphore(sem, task): async with sem: diff --git a/llama.cpp b/llama.cpp index 76afcbc13..e9192b4fa 100644 --- a/llama.cpp +++ b/llama.cpp @@ -5014,8 +5014,8 @@ static struct ggml_tensor * llm_build_kqv( ggml_mul_mat_set_prec(kq, GGML_PREC_F32); } -#if defined(GGML_USE_VULKAN) || defined(GGML_USE_KOMPUTE) -#pragma message("TODO: ALiBi support in ggml_soft_max_ext is not implemented for Vulkan, and Kompute") +#if defined(GGML_USE_KOMPUTE) +#pragma message("TODO: ALiBi support in ggml_soft_max_ext is not implemented for Kompute") #pragma message(" Falling back to ggml_alibi(). Will become an error in Mar 2024") #pragma message("ref: https://github.com/ggerganov/llama.cpp/pull/5488") if (hparams.f_max_alibi_bias > 0.0f) { From b1a4e994fde929300d4aeb1deb8320c59cb6edec Mon Sep 17 00:00:00 2001 From: ExtReMLapin <3909752+ExtReMLapin@users.noreply.github.com> Date: Tue, 5 Mar 2024 14:44:29 +0100 Subject: [PATCH 543/812] grammars : don't allow to output unescaped new line in string (#5885) * Don't allow grammar json array to output unescaped new line in string * Don't allow new line in json object string --- grammars/json.gbnf | 2 +- grammars/json_arr.gbnf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/grammars/json.gbnf b/grammars/json.gbnf index a9537cdf9..34e014d55 100644 --- a/grammars/json.gbnf +++ b/grammars/json.gbnf @@ -15,7 +15,7 @@ array ::= string ::= "\"" ( - [^"\\] | + [^"\\\n] | "\\" (["\\/bfnrt] | "u" [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F]) # escapes )* "\"" ws diff --git a/grammars/json_arr.gbnf b/grammars/json_arr.gbnf index ef53e77a0..fd5740eeb 100644 --- a/grammars/json_arr.gbnf +++ b/grammars/json_arr.gbnf @@ -24,7 +24,7 @@ array ::= string ::= "\"" ( - [^"\\] | + [^"\\\n] | "\\" (["\\/bfnrt] | "u" [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F]) # escapes )* "\"" ws From 82cb31eb93fd19b74115e0f0133225d1dfdbfdbc Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Tue, 5 Mar 2024 15:56:24 +0200 Subject: [PATCH 544/812] Revert "grammars : don't allow to output unescaped new line in string (#5885)" This reverts commit b1a4e994fde929300d4aeb1deb8320c59cb6edec. --- grammars/json.gbnf | 2 +- grammars/json_arr.gbnf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/grammars/json.gbnf b/grammars/json.gbnf index 34e014d55..a9537cdf9 100644 --- a/grammars/json.gbnf +++ b/grammars/json.gbnf @@ -15,7 +15,7 @@ array ::= string ::= "\"" ( - [^"\\\n] | + [^"\\] | "\\" (["\\/bfnrt] | "u" [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F]) # escapes )* "\"" ws diff --git a/grammars/json_arr.gbnf b/grammars/json_arr.gbnf index fd5740eeb..ef53e77a0 100644 --- a/grammars/json_arr.gbnf +++ b/grammars/json_arr.gbnf @@ -24,7 +24,7 @@ array ::= string ::= "\"" ( - [^"\\\n] | + [^"\\] | "\\" (["\\/bfnrt] | "u" [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F]) # escapes )* "\"" ws From 3de31677d36aa4f82d4d99898902d7bcf398e666 Mon Sep 17 00:00:00 2001 From: ExtReMLapin <3909752+ExtReMLapin@users.noreply.github.com> Date: Tue, 5 Mar 2024 17:33:08 +0100 Subject: [PATCH 545/812] grammars : blacklists character control set (#5888) * Prevent control characters from being served in json string * Prevent control characters from being served in json string (array) --- grammars/json.gbnf | 2 +- grammars/json_arr.gbnf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/grammars/json.gbnf b/grammars/json.gbnf index a9537cdf9..a8a80752e 100644 --- a/grammars/json.gbnf +++ b/grammars/json.gbnf @@ -15,7 +15,7 @@ array ::= string ::= "\"" ( - [^"\\] | + [^"\\\x7F\x00-\x1F] | "\\" (["\\/bfnrt] | "u" [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F]) # escapes )* "\"" ws diff --git a/grammars/json_arr.gbnf b/grammars/json_arr.gbnf index ef53e77a0..31a3202f8 100644 --- a/grammars/json_arr.gbnf +++ b/grammars/json_arr.gbnf @@ -24,7 +24,7 @@ array ::= string ::= "\"" ( - [^"\\] | + [^"\\\x7F\x00-\x1F] | "\\" (["\\/bfnrt] | "u" [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F]) # escapes )* "\"" ws From bd836944f826f07e19b7edcf994a78728da49c1c Mon Sep 17 00:00:00 2001 From: Jared Van Bortel Date: Tue, 5 Mar 2024 11:56:37 -0500 Subject: [PATCH 546/812] quants : use MM256_SET_M128I consistently to fix gcc 7 build (#5889) --- ggml-quants.c | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/ggml-quants.c b/ggml-quants.c index 2a8881d73..e0c125d43 100644 --- a/ggml-quants.c +++ b/ggml-quants.c @@ -51,6 +51,7 @@ #define UNUSED GGML_UNUSED +// some compilers don't provide _mm256_set_m128i, e.g. gcc 7 #define MM256_SET_M128I(a, b) _mm256_insertf128_si256(_mm256_castsi128_si256(b), (a), 1) #if defined(__AVX__) || defined(__AVX2__) || defined(__AVX512F__) || defined(__SSSE3__) @@ -9563,7 +9564,7 @@ void ggml_vec_dot_iq2_xs_q8_K(int n, float * restrict s, size_t bs, const void * const __m128i odd_bits = _mm_shuffle_epi8(bit_helper, partial_sign_bits_for_counting); const __m128i full_sign_bits = _mm_or_si128(partial_sign_bits, odd_bits); - const __m256i full_signs = _mm256_set_m128i(full_sign_bits, full_sign_bits); + const __m256i full_signs = MM256_SET_M128I(full_sign_bits, full_sign_bits); const __m256i q8_1 = _mm256_loadu_si256((const __m256i *)y[i].qs); const __m256i q8_2 = _mm256_loadu_si256((const __m256i *)(y[i].qs+32)); @@ -9585,8 +9586,8 @@ void ggml_vec_dot_iq2_xs_q8_K(int n, float * restrict s, size_t bs, const void * const __m256i dot1 = _mm256_maddubs_epi16(q2_1, q8s_1); const __m256i dot2 = _mm256_maddubs_epi16(q2_2, q8s_2); - const __m256i sc1 = _mm256_set_m128i(_mm_set1_epi16(2*(x[i].scales[0] >> 4)+1), _mm_set1_epi16(2*(x[i].scales[0] & 0xf)+1)); - const __m256i sc2 = _mm256_set_m128i(_mm_set1_epi16(2*(x[i].scales[1] >> 4)+1), _mm_set1_epi16(2*(x[i].scales[1] & 0xf)+1)); + const __m256i sc1 = MM256_SET_M128I(_mm_set1_epi16(2*(x[i].scales[0] >> 4)+1), _mm_set1_epi16(2*(x[i].scales[0] & 0xf)+1)); + const __m256i sc2 = MM256_SET_M128I(_mm_set1_epi16(2*(x[i].scales[1] >> 4)+1), _mm_set1_epi16(2*(x[i].scales[1] & 0xf)+1)); const __m256i sum = _mm256_add_epi32(_mm256_madd_epi16(sc1, dot1), _mm256_madd_epi16(sc2, dot2)); @@ -9653,8 +9654,8 @@ void ggml_vec_dot_iq2_xs_q8_K(int n, float * restrict s, size_t bs, const void * const __m128i full_signs_l = _mm256_castsi256_si128(full_sign_bits); const __m128i full_signs_h = _mm256_extractf128_si256(full_sign_bits, 1); - const __m256i full_signs_1 = _mm256_set_m128i(full_signs_l, full_signs_l); - const __m256i full_signs_2 = _mm256_set_m128i(full_signs_h, full_signs_h); + const __m256i full_signs_1 = MM256_SET_M128I(full_signs_l, full_signs_l); + const __m256i full_signs_2 = MM256_SET_M128I(full_signs_h, full_signs_h); __m256i signs; signs = _mm256_shuffle_epi8(full_signs_1, block_sign_shuffle_1); @@ -10551,10 +10552,10 @@ void ggml_vec_dot_iq4_nl_q8_0(int n, float * restrict s, size_t bs, const void * const __m128i q4bits_2 = _mm_loadu_si128((const __m128i*)x[1].qs); const __m256i q8b_1 = _mm256_loadu_si256((const __m256i *)y[0].qs); const __m256i q8b_2 = _mm256_loadu_si256((const __m256i *)y[1].qs); - const __m256i q4b_1 = _mm256_set_m128i(_mm_shuffle_epi8(values128, _mm_and_si128(_mm_srli_epi16(q4bits_1, 4), m4b)), - _mm_shuffle_epi8(values128, _mm_and_si128(q4bits_1, m4b))); - const __m256i q4b_2 = _mm256_set_m128i(_mm_shuffle_epi8(values128, _mm_and_si128(_mm_srli_epi16(q4bits_2, 4), m4b)), - _mm_shuffle_epi8(values128, _mm_and_si128(q4bits_2, m4b))); + const __m256i q4b_1 = MM256_SET_M128I(_mm_shuffle_epi8(values128, _mm_and_si128(_mm_srli_epi16(q4bits_1, 4), m4b)), + _mm_shuffle_epi8(values128, _mm_and_si128(q4bits_1, m4b))); + const __m256i q4b_2 = MM256_SET_M128I(_mm_shuffle_epi8(values128, _mm_and_si128(_mm_srli_epi16(q4bits_2, 4), m4b)), + _mm_shuffle_epi8(values128, _mm_and_si128(q4bits_2, m4b))); const __m256i p16_1 = mul_add_epi8(q4b_1, q8b_1); const __m256i p16_2 = mul_add_epi8(q4b_2, q8b_2); const __m256i p_1 = _mm256_madd_epi16(p16_1, mone); @@ -10661,10 +10662,10 @@ void ggml_vec_dot_iq4_xs_q8_K(int n, float * restrict s, size_t bs, const void * const __m128i q4bits_2 = _mm_loadu_si128((const __m128i*)qs); qs += 16; const __m256i q8b_1 = _mm256_loadu_si256((const __m256i *)q8); q8 += 32; const __m256i q8b_2 = _mm256_loadu_si256((const __m256i *)q8); q8 += 32; - const __m256i q4b_1 = _mm256_set_m128i(_mm_shuffle_epi8(values128, _mm_and_si128(_mm_srli_epi16(q4bits_1, 4), m4b)), - _mm_shuffle_epi8(values128, _mm_and_si128(q4bits_1, m4b))); - const __m256i q4b_2 = _mm256_set_m128i(_mm_shuffle_epi8(values128, _mm_and_si128(_mm_srli_epi16(q4bits_2, 4), m4b)), - _mm_shuffle_epi8(values128, _mm_and_si128(q4bits_2, m4b))); + const __m256i q4b_1 = MM256_SET_M128I(_mm_shuffle_epi8(values128, _mm_and_si128(_mm_srli_epi16(q4bits_1, 4), m4b)), + _mm_shuffle_epi8(values128, _mm_and_si128(q4bits_1, m4b))); + const __m256i q4b_2 = MM256_SET_M128I(_mm_shuffle_epi8(values128, _mm_and_si128(_mm_srli_epi16(q4bits_2, 4), m4b)), + _mm_shuffle_epi8(values128, _mm_and_si128(q4bits_2, m4b))); const __m256i p16_1 = mul_add_epi8(q4b_1, q8b_1); const __m256i p16_2 = mul_add_epi8(q4b_2, q8b_2); const int16_t ls1 = ((x[ibl].scales_l[ib/2] & 0xf) | ((sh << 4) & 0x30)) - 32; From 652ca2bded3c818320d92c70d2b67f64bdbff5e5 Mon Sep 17 00:00:00 2001 From: slaren Date: Tue, 5 Mar 2024 22:27:29 +0100 Subject: [PATCH 547/812] compare-llama-bench.py : remove mul_mat_q (#5892) --- scripts/compare-llama-bench.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/compare-llama-bench.py b/scripts/compare-llama-bench.py index 39c3e52e5..54a7771d8 100755 --- a/scripts/compare-llama-bench.py +++ b/scripts/compare-llama-bench.py @@ -18,7 +18,7 @@ except ImportError as e: KEY_PROPERTIES = [ "cpu_info", "gpu_info", "n_gpu_layers", "main_gpu", "cuda", "opencl", "metal", "gpu_blas", "blas", "model_filename", "model_type", "model_size", "model_n_params", "n_batch", "n_threads", - "type_k", "type_v", "no_kv_offload", "mul_mat_q", "tensor_split", "n_prompt", "n_gen" + "type_k", "type_v", "no_kv_offload", "tensor_split", "n_prompt", "n_gen" ] # Properties that are boolean and are converted to Yes/No for the table: From 8ced9f7e3225adb8501e9821ed1bbd92e3a5c7ae Mon Sep 17 00:00:00 2001 From: Neo Zhang Jianyu Date: Wed, 6 Mar 2024 12:08:32 +0800 Subject: [PATCH 548/812] add wait() to make code stable (#5895) --- ci/run.sh | 3 ++- ggml-sycl.cpp | 59 +++++++++++++++++++++++++++++++++++++++------------ 2 files changed, 48 insertions(+), 14 deletions(-) diff --git a/ci/run.sh b/ci/run.sh index 35eb3c7aa..51f4c74cc 100755 --- a/ci/run.sh +++ b/ci/run.sh @@ -45,7 +45,8 @@ fi if [ ! -z ${GG_BUILD_SYCL} ]; then if [ -z ${ONEAPI_ROOT} ]; then - echo "Not detected ONEAPI_ROOT, please install oneAPI base toolkit and enable it by:\n source /opt/intel/oneapi/setvars.sh" + echo "Not detected ONEAPI_ROOT, please install oneAPI base toolkit and enable it by:" + echo "source /opt/intel/oneapi/setvars.sh" exit 1 fi diff --git a/ggml-sycl.cpp b/ggml-sycl.cpp index 477f5cb02..ddd951dd6 100644 --- a/ggml-sycl.cpp +++ b/ggml-sycl.cpp @@ -3769,8 +3769,42 @@ void log_ggml_var_device(const char*name, float *src, size_t total_elements, boo std::ofstream logfile; logfile.open(filename); for(size_t i=0; imemcpy(local_buf, src, total_size).wait(); + } + else { + local_buf = (sycl::half *)src; + } + + std::ofstream logfile; + logfile.open(filename); + for(size_t i=0; iwait(); const to_fp32_sycl_t to_fp32_sycl = ggml_get_to_fp32_sycl(GGML_TYPE_F16); to_fp32_sycl(dst_f16.get(), dst_dd_i, row_diff*src1_ncols, stream); } @@ -14159,6 +14193,7 @@ inline void ggml_sycl_op_mul_mat_sycl( dpct::get_value(&alpha, *g_sycl_handles[id]), src0_ddf_i, ne00, src1_ddf1_i, ne10, dpct::get_value(&beta, *g_sycl_handles[id]), dst_dd_i, ldc))); + g_sycl_handles[id]->wait(); } (void) dst; (void) src1_ddq_i; @@ -15295,8 +15330,8 @@ static void ggml_sycl_mul_mat_batched_sycl(const ggml_tensor *src0, sycl_pool_alloc dst_f16; char * dst_t; - dpct::library_data_t cu_compute_type = dpct::library_data_t::real_half; - dpct::library_data_t cu_data_type = dpct::library_data_t::real_half; + dpct::library_data_t cu_compute_type = dpct::library_data_t::real_float; + dpct::library_data_t cu_data_type = dpct::library_data_t::real_float; // dst strides size_t nbd2 = dst->nb[2]; @@ -15308,15 +15343,13 @@ static void ggml_sycl_mul_mat_batched_sycl(const ggml_tensor *src0, const float alpha_f32 = 1.0f; const float beta_f32 = 0.0f; - const void * alpha = &alpha_f16; - const void * beta = &beta_f16; + const void * alpha = &alpha_f32; + const void * beta = &beta_f32; // TODO: Renable (dst->op_params[0] =! GGML_PREC_DEFAULT) pathway - // once oneMKL open source supports half, half, float, float: datatypes - dst_t = (char *) dst_f16.alloc(ne_dst); + // oneMKL open source supports half, half, float, float: datatypes - nbd2 /= sizeof(float) / sizeof(sycl::half); - nbd3 /= sizeof(float) / sizeof(sycl::half); + dst_t = (char *) dst_ddf; GGML_ASSERT(ne12 % ne02 == 0); GGML_ASSERT(ne13 % ne03 == 0); @@ -15356,6 +15389,7 @@ static void ggml_sycl_mul_mat_batched_sycl(const ggml_tensor *src0, nb11 / nb10, nb12 / nb10, beta, (char *)dst_t, cu_data_type, ne01, nb2 / nb0, ne12 * ne13, cu_compute_type))); + g_sycl_handles[g_main_device]->wait(); } else { const int ne23 = ne12*ne13; @@ -15386,7 +15420,7 @@ static void ggml_sycl_mul_mat_batched_sycl(const ggml_tensor *src0, nb02, nb03, nb12_scaled, nb13_scaled, nbd2, nbd3, r2, r3, item_ct1); }); - }); + }).wait(); } SYCL_CHECK(CHECK_TRY_ERROR(dpct::gemm_batch( *g_sycl_handles[g_main_device], oneapi::mkl::transpose::trans, @@ -15397,11 +15431,10 @@ static void ggml_sycl_mul_mat_batched_sycl(const ggml_tensor *src0, dpct::library_data_t::real_half, nb11 / nb10, beta, (void **)(ptrs_dst.get() + 0 * ne23), cu_data_type, ne01, ne23, cu_compute_type))); + g_sycl_handles[g_main_device]->wait(); } #endif - const to_fp32_sycl_t to_fp32_sycl = ggml_get_to_fp32_sycl(GGML_TYPE_F16); - to_fp32_sycl(dst_f16.get(), dst_ddf, ne_dst, main_stream); } catch (sycl::exception const &exc) { std::cerr << exc.what() << "Exception caught at file:" << __FILE__ From 1e35d619a6fb0b9c5e3dc955345980ff056ddbaf Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Wed, 6 Mar 2024 09:12:25 +0200 Subject: [PATCH 549/812] convert : remove AWQ remnants (#5768) --- convert.py | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/convert.py b/convert.py index 6e3a0319b..c15f8c47e 100755 --- a/convert.py +++ b/convert.py @@ -1377,7 +1377,6 @@ def main(args_in: list[str] | None = None) -> None: # We currently only support Q8_0 output on little endian systems. output_choices.append("q8_0") parser = argparse.ArgumentParser(description="Convert a LLaMA model to a GGML compatible file") - parser.add_argument("--awq-path", type=Path, help="Path to scale awq cache file", default=None) parser.add_argument("--dump", action="store_true", help="don't convert, just show what's in the model") parser.add_argument("--dump-single", action="store_true", help="don't convert, just show what's in a single model file") parser.add_argument("--vocab-only", action="store_true", help="extract only the vocab") @@ -1393,18 +1392,6 @@ def main(args_in: list[str] | None = None) -> None: parser.add_argument("--skip-unknown", action="store_true", help="skip unknown tensor names instead of failing") args = parser.parse_args(args_in) - if args.awq_path: - sys.path.insert(1, str(Path(__file__).parent / 'awq-py')) - from awq.apply_awq import add_scale_weights # type: ignore[import-not-found] - tmp_model_path = args.model / "weighted_model" - if tmp_model_path.is_dir(): - print(f"{tmp_model_path} exists as a weighted model.") - else: - tmp_model_path.mkdir(parents=True, exist_ok=True) - print("Saving new weighted model ...") - add_scale_weights(str(args.model), str(args.awq_path), str(tmp_model_path)) - print(f"Saved weighted model at {tmp_model_path}.") - args.model = tmp_model_path if args.dump_single: model_plus = lazy_load_file(args.model) From e25fb4b18fcedb9bed6be4585cf842e9a669b28b Mon Sep 17 00:00:00 2001 From: bobqianic <129547291+bobqianic@users.noreply.github.com> Date: Wed, 6 Mar 2024 07:35:07 +0000 Subject: [PATCH 550/812] ggml : use `uint8x16_t` return type for `ggml_vqtbl1q_u8` (#5894) * use uint8x16_t * Update ggml-quants.c --- ggml-quants.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ggml-quants.c b/ggml-quants.c index e0c125d43..9dcb76def 100644 --- a/ggml-quants.c +++ b/ggml-quants.c @@ -464,8 +464,8 @@ inline static int8x16_t ggml_vqtbl1q_s8(int8x16_t a, uint8x16_t b) { } // NOTE: not tested -inline static int8x16_t ggml_vqtbl1q_u8(uint8x16_t a, uint8x16_t b) { - int8x16_t res; +inline static uint8x16_t ggml_vqtbl1q_u8(uint8x16_t a, uint8x16_t b) { + uint8x16_t res; res[ 0] = a[b[ 0]]; res[ 1] = a[b[ 1]]; From e04e04f8fad549bb0b3ec1c91f0413aeb08baf29 Mon Sep 17 00:00:00 2001 From: Jared Van Bortel Date: Wed, 6 Mar 2024 15:42:23 -0500 Subject: [PATCH 551/812] ggml : use SYS_get_cpu if SYS_getcpu is not defined (#5906) Fixes #5694 Fixes ggerganov/whisper.cpp#1894 --- ggml.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ggml.c b/ggml.c index 6a10bbcb4..92b17ee6e 100644 --- a/ggml.c +++ b/ggml.c @@ -2154,7 +2154,10 @@ void ggml_numa_init(enum ggml_numa_strategy numa_flag) { getcpu_ret = getcpu(¤t_cpu, &g_state.numa.current_node); #else // old glibc doesn't have a wrapper for this call. Fall back on direct syscall - getcpu_ret = syscall(SYS_getcpu,¤t_cpu,&g_state.numa.current_node); +# if !defined(SYS_getcpu) && defined(SYS_get_cpu) +# define SYS_getcpu SYS_get_cpu // some older glibc versions use this name +# endif + getcpu_ret = syscall(SYS_getcpu, ¤t_cpu, &g_state.numa.current_node); #endif if (g_state.numa.n_nodes < 1 || g_state.numa.total_cpus < 1 || getcpu_ret != 0) { From ceca1aef0738b57951cd12c603c3477e75312dec Mon Sep 17 00:00:00 2001 From: Neo Zhang Jianyu Date: Thu, 7 Mar 2024 16:34:31 +0800 Subject: [PATCH 552/812] [SYCL] fix error when set main gpu to non-zero (#5901) * fix error when set main gpu to non-zero * fix delete condition --- ggml-sycl.cpp | 178 ++++++++++++++++++++++++++++++-------------------- ggml-sycl.h | 1 + llama.cpp | 16 +++-- 3 files changed, 119 insertions(+), 76 deletions(-) diff --git a/ggml-sycl.cpp b/ggml-sycl.cpp index ddd951dd6..221d67b8d 100644 --- a/ggml-sycl.cpp +++ b/ggml-sycl.cpp @@ -3559,12 +3559,31 @@ class sycl_gpu_mgr { int work_group_size = 0; std::string gpus_list = ""; + /* + Use all GPU with same top max compute units + */ sycl_gpu_mgr() { detect_sycl_gpu_list_with_max_cu(); get_allow_gpus(); create_context_with_gpus(); } + /* + Use the assigned GPU as only one + */ + sycl_gpu_mgr(int main_gpu_id) { + sycl::device device = dpct::dev_mgr::instance().get_device(main_gpu_id); + dpct::device_info prop; + dpct::get_device_info(prop, device); + gpus.push_back(main_gpu_id); + devices.push_back(device); + work_group_size = prop.get_max_work_group_size(); + max_compute_units = prop.get_max_compute_units(); + + get_allow_gpus(); + create_context_with_gpus(); + } + void create_context_with_gpus() { sycl::context ctx = sycl::context(devices); assert(gpus.size() > 0); @@ -3580,7 +3599,7 @@ class sycl_gpu_mgr { gpus_list += std::to_string(gpus[i]); gpus_list += ","; } - if (gpus_list.length() > 2) { + if (gpus_list.length() > 1) { gpus_list.pop_back(); } } @@ -3629,8 +3648,8 @@ class sycl_gpu_mgr { if (gpus[i] == id) return i; } - assert(false); - return -1; + printf("miss to get device index by id=%d\n", id); + GGML_ASSERT(false); } int get_next_index(int id) { @@ -3639,8 +3658,7 @@ class sycl_gpu_mgr { if (gpus[i] == id) return i; } - assert(false); - return -1; + GGML_ASSERT(false); } }; @@ -3649,6 +3667,7 @@ static int g_device_count = -1; static int g_all_sycl_device_count = -1; static int g_main_device = -1; static int g_main_device_id = -1; +static bool g_ggml_backend_sycl_buffer_type_initialized = false; static std::array g_default_tensor_split = {}; @@ -13225,7 +13244,7 @@ void ggml_backend_sycl_print_sycl_devices() { } void print_gpu_device_list() { - fprintf(stderr, "detect %d SYCL GPUs: [%s] with Max compute units:%d\n", + fprintf(stderr, "detect %d SYCL GPUs: [%s] with top Max compute units:%d\n", g_sycl_gpu_mgr->get_gpu_count(), g_sycl_gpu_mgr->gpus_list.c_str(), g_sycl_gpu_mgr->max_compute_units); @@ -13264,23 +13283,6 @@ void ggml_init_sycl() try { #else fprintf(stderr, "%s: GGML_SYCL_F16: no\n", __func__); #endif - if (CHECK_TRY_ERROR(g_all_sycl_device_count = - dpct::dev_mgr::instance().device_count()) != 0) { - initialized = true; - g_sycl_loaded = false; - return; - } - GGML_ASSERT(g_all_sycl_device_count <= GGML_SYCL_MAX_DEVICES); - ggml_backend_sycl_print_sycl_devices(); - - if (!g_sycl_gpu_mgr) g_sycl_gpu_mgr = new sycl_gpu_mgr(); - - g_device_count = g_sycl_gpu_mgr->get_gpu_count(); - g_work_group_size = g_sycl_gpu_mgr->work_group_size; - - print_gpu_device_list(); - - int64_t total_vram = 0; /* NOT REMOVE, keep it for next optimize for XMX. #if defined(SYCL_USE_XMX) @@ -13289,52 +13291,71 @@ void ggml_init_sycl() try { fprintf(stderr, "%s: SYCL_USE_XMX: no\n", __func__); #endif */ - for (int id = 0; id < GGML_SYCL_MAX_DEVICES; ++id) { - g_device_caps[id].vmm = 0; - g_device_caps[id].device_id = -1; - g_device_caps[id].cc = 0; - g_tensor_split[id] = 0; - g_default_tensor_split[id] = 0; + + if (CHECK_TRY_ERROR(g_all_sycl_device_count = + dpct::dev_mgr::instance().device_count()) != 0) { + initialized = true; + g_sycl_loaded = false; + return; } - - for (int i = 0; i < g_device_count; ++i) { - int device_id = g_sycl_gpu_mgr->gpus[i]; - g_device_caps[i].vmm = 0; - - dpct::device_info prop; - SYCL_CHECK(CHECK_TRY_ERROR(dpct::get_device_info( - prop, dpct::dev_mgr::instance().get_device(device_id)))); - - g_default_tensor_split[i] = total_vram; - total_vram += prop.get_global_mem_size(); - - g_device_caps[i].cc = - 100 * prop.get_major_version() + 10 * prop.get_minor_version(); - } - - for (int i = 0; i < g_device_count; ++i) { - g_default_tensor_split[i] /= total_vram; - } - - for (int i = 0; i < g_device_count; ++i) { - SYCL_CHECK(ggml_sycl_set_device(i)); - - // create sycl streams - for (int is = 0; is < MAX_STREAMS; ++is) { - SYCL_CHECK(CHECK_TRY_ERROR( - g_syclStreams[i][is] = - dpct::get_current_device().create_queue( - g_sycl_gpu_mgr->get_co_ctx(), dpct::get_current_device()))); - } - - const dpct::queue_ptr stream = g_syclStreams[i][0]; - // create sycl handle - SYCL_CHECK(CHECK_TRY_ERROR(g_sycl_handles[i] = stream)); - } - + GGML_ASSERT(g_all_sycl_device_count <= GGML_SYCL_MAX_DEVICES); + ggml_backend_sycl_print_sycl_devices(); + if (!g_sycl_gpu_mgr) g_sycl_gpu_mgr = new sycl_gpu_mgr(); + print_gpu_device_list(); initialized = true; g_sycl_loaded = true; } + + + + g_device_count = g_sycl_gpu_mgr->get_gpu_count(); + g_work_group_size = g_sycl_gpu_mgr->work_group_size; + + int64_t total_vram = 0; + + + for (int id = 0; id < GGML_SYCL_MAX_DEVICES; ++id) { + g_device_caps[id].vmm = 0; + g_device_caps[id].device_id = -1; + g_device_caps[id].cc = 0; + g_tensor_split[id] = 0; + g_default_tensor_split[id] = 0; + } + + for (int i = 0; i < g_device_count; ++i) { + int device_id = g_sycl_gpu_mgr->gpus[i]; + g_device_caps[i].vmm = 0; + + dpct::device_info prop; + SYCL_CHECK(CHECK_TRY_ERROR(dpct::get_device_info( + prop, dpct::dev_mgr::instance().get_device(device_id)))); + + g_default_tensor_split[i] = total_vram; + total_vram += prop.get_global_mem_size(); + + g_device_caps[i].cc = + 100 * prop.get_major_version() + 10 * prop.get_minor_version(); + } + + for (int i = 0; i < g_device_count; ++i) { + g_default_tensor_split[i] /= total_vram; + } + + for (int i = 0; i < g_device_count; ++i) { + SYCL_CHECK(ggml_sycl_set_device(i)); + + // create sycl streams + for (int is = 0; is < MAX_STREAMS; ++is) { + SYCL_CHECK(CHECK_TRY_ERROR( + g_syclStreams[i][is] = + dpct::get_current_device().create_queue( + g_sycl_gpu_mgr->get_co_ctx(), dpct::get_current_device()))); + } + + const dpct::queue_ptr stream = g_syclStreams[i][0]; + // create sycl handle + SYCL_CHECK(CHECK_TRY_ERROR(g_sycl_handles[i] = stream)); + } } catch (sycl::exception const &exc) { std::cerr << exc.what() << "Exception caught at file:" << __FILE__ @@ -16732,22 +16753,24 @@ static ggml_backend_buffer_type_i ggml_backend_sycl_buffer_type_interface = { /* .is_host = */ nullptr, }; -ggml_backend_buffer_type_t ggml_backend_sycl_buffer_type(int device) { +ggml_backend_buffer_type_t ggml_backend_sycl_buffer_type(int device_index) { + if (device_index>=g_device_count or device_index<0) { + printf("ggml_backend_sycl_buffer_type error: device_index:%d is out of range [0, %d], miss to call ggml_backend_sycl_set_single_device()\n", + device_index, g_device_count-1); + GGML_ASSERT(device_indexgpus[i])}, }; } - ggml_backend_sycl_buffer_type_initialized = true; + g_ggml_backend_sycl_buffer_type_initialized = true; } - - return &ggml_backend_sycl_buffer_types[device]; + return &ggml_backend_sycl_buffer_types[device_index]; } // sycl split buffer type @@ -17496,6 +17519,17 @@ GGML_API GGML_CALL int ggml_backend_sycl_get_device_index(int device_id) { return g_sycl_gpu_mgr->get_index(device_id); } +GGML_API GGML_CALL void ggml_backend_sycl_set_single_device(int main_gpu_id) { + GGML_ASSERT(main_gpu_idbackends.push_back(backend); } else { // LLAMA_SPLIT_LAYER requires a backend for each GPU - int id_list[GGML_SYCL_MAX_DEVICES]; - ggml_sycl_get_gpu_list(id_list, GGML_SYCL_MAX_DEVICES); + for (int i = 0; i < ggml_backend_sycl_get_device_count(); ++i) { - int device_id = id_list[i]; ggml_backend_t backend = ggml_backend_sycl_init(i); if (backend == nullptr) { - LLAMA_LOG_ERROR("%s: failed to initialize SYCL%d (index %d)backend\n", __func__, device_id, i); + int id_list[GGML_SYCL_MAX_DEVICES]; + ggml_sycl_get_gpu_list(id_list, GGML_SYCL_MAX_DEVICES); + LLAMA_LOG_ERROR("%s: failed to initialize SYCL%d (index %d)backend\n", __func__, id_list[i], i); llama_free(ctx); return nullptr; } From 2002bc96bf2cbf5ab981a17d7e994d817c9801f5 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Thu, 7 Mar 2024 11:41:53 +0200 Subject: [PATCH 553/812] server : refactor (#5882) * server : refactoring (wip) * server : remove llava/clip objects from build * server : fix empty prompt handling + all slots idle logic * server : normalize id vars * server : code style * server : simplify model chat template validation * server : code style * server : minor * llama : llama_chat_apply_template support null buf * server : do not process embedding requests when disabled * server : reorganize structs and enums + naming fixes * server : merge oai.hpp in utils.hpp * server : refactor system prompt update at start * server : disable cached prompts with self-extend * server : do not process more than n_batch tokens per iter * server: tests: embeddings use a real embeddings model (#5908) * server, tests : bump batch to fit 1 embedding prompt * server: tests: embeddings fix build type Debug is randomly failing (#5911) * server: tests: embeddings, use different KV Cache size * server: tests: embeddings, fixed prompt do not exceed n_batch, increase embedding timeout, reduce number of concurrent embeddings * server: tests: embeddings, no need to wait for server idle as it can timout * server: refactor: clean up http code (#5912) * server : avoid n_available var ggml-ci * server: refactor: better http codes * server : simplify json parsing + add comment about t_last * server : rename server structs * server : allow to override FQDN in tests ggml-ci * server : add comments --------- Co-authored-by: Pierrick Hymbert --- .github/workflows/server.yml | 3 +- Makefile | 5 +- examples/server-embd.py | 2 +- examples/server/CMakeLists.txt | 4 +- examples/server/README.md | 2 +- examples/server/oai.hpp | 225 - examples/server/server.cpp | 3895 ++++++++--------- .../server/tests/features/embeddings.feature | 94 + .../server/tests/features/parallel.feature | 46 - examples/server/tests/features/server.feature | 28 - examples/server/tests/features/steps/steps.py | 86 +- examples/server/tests/requirements.txt | 1 + examples/server/utils.hpp | 703 ++- llama.cpp | 6 +- 14 files changed, 2327 insertions(+), 2773 deletions(-) delete mode 100644 examples/server/oai.hpp create mode 100644 examples/server/tests/features/embeddings.feature diff --git a/.github/workflows/server.yml b/.github/workflows/server.yml index 04e3fc0c1..f9aeefaa8 100644 --- a/.github/workflows/server.yml +++ b/.github/workflows/server.yml @@ -58,7 +58,8 @@ jobs: cmake \ python3-pip \ wget \ - psmisc + psmisc \ + language-pack-en - name: Build id: cmake_build diff --git a/Makefile b/Makefile index 4f26c0463..efce10bb8 100644 --- a/Makefile +++ b/Makefile @@ -724,10 +724,9 @@ save-load-state: examples/save-load-state/save-load-state.cpp ggml.o llama.o $(C $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) -server: examples/server/server.cpp examples/server/oai.hpp examples/server/utils.hpp examples/server/httplib.h examples/server/json.hpp examples/server/index.html.hpp examples/server/index.js.hpp examples/server/completion.js.hpp examples/llava/clip.cpp examples/llava/clip.h examples/llava/llava.h examples/llava/llava.cpp common/stb_image.h ggml.o llama.o $(COMMON_DEPS) grammar-parser.o $(OBJS) +server: examples/server/server.cpp examples/server/utils.hpp examples/server/httplib.h examples/server/json.hpp examples/server/index.html.hpp examples/server/index.js.hpp examples/server/completion.js.hpp common/stb_image.h ggml.o llama.o $(COMMON_DEPS) grammar-parser.o $(OBJS) $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) - $(CXX) $(CXXFLAGS) -c examples/llava/clip.cpp -o $(call GET_OBJ_FILE, examples/llava/clip.cpp) -Wno-cast-qual - $(CXX) $(CXXFLAGS) -Iexamples/server $(filter-out %.h %.hpp $< examples/llava/clip.cpp,$^) $(call GET_OBJ_FILE, $<) $(call GET_OBJ_FILE, examples/llava/clip.cpp) -o $@ $(LDFLAGS) $(LWINSOCK2) + $(CXX) $(CXXFLAGS) $(filter-out %.h %.hpp $<,$^) -Iexamples/server $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) $(LWINSOCK2) gguf: examples/gguf/gguf.cpp ggml.o $(OBJS) $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) diff --git a/examples/server-embd.py b/examples/server-embd.py index c5c4ea87b..118e04271 100644 --- a/examples/server-embd.py +++ b/examples/server-embd.py @@ -13,7 +13,7 @@ async def main(): model_url = "http://127.0.0.1:6900" responses: list[requests.Response] = await asyncio.gather(*[requests_post_async( url= f"{model_url}/embedding", - json= {"content": str(i)*1024} + json= {"content": str(0)*1024} ) for i in range(n)]) for response in responses: diff --git a/examples/server/CMakeLists.txt b/examples/server/CMakeLists.txt index cc13b2d63..c21eba634 100644 --- a/examples/server/CMakeLists.txt +++ b/examples/server/CMakeLists.txt @@ -1,12 +1,12 @@ set(TARGET server) option(LLAMA_SERVER_VERBOSE "Build verbose logging option for Server" ON) include_directories(${CMAKE_CURRENT_SOURCE_DIR}) -add_executable(${TARGET} server.cpp oai.hpp utils.hpp json.hpp httplib.h) +add_executable(${TARGET} server.cpp utils.hpp json.hpp httplib.h) install(TARGETS ${TARGET} RUNTIME) target_compile_definitions(${TARGET} PRIVATE SERVER_VERBOSE=$ ) -target_link_libraries(${TARGET} PRIVATE common llava ${CMAKE_THREAD_LIBS_INIT}) +target_link_libraries(${TARGET} PRIVATE common ${CMAKE_THREAD_LIBS_INIT}) if (WIN32) TARGET_LINK_LIBRARIES(${TARGET} PRIVATE ws2_32) endif() diff --git a/examples/server/README.md b/examples/server/README.md index 21da7a0a0..591f748f8 100644 --- a/examples/server/README.md +++ b/examples/server/README.md @@ -436,7 +436,7 @@ Notice that each `probs` is an array of length `n_probs`. "next_token": { "has_next_token": true, "n_remain": -1, - "num_tokens_predicted": 0, + "n_decoded": 0, "stopped_eos": false, "stopped_limit": false, "stopped_word": false, diff --git a/examples/server/oai.hpp b/examples/server/oai.hpp deleted file mode 100644 index ff4ad6994..000000000 --- a/examples/server/oai.hpp +++ /dev/null @@ -1,225 +0,0 @@ -#pragma once - -#include -#include -#include -#include -#include -#include - -#include "json.hpp" -#include "utils.hpp" - -#define DEFAULT_OAICOMPAT_MODEL "gpt-3.5-turbo-0613" - -using json = nlohmann::json; - -inline static json oaicompat_completion_params_parse( - const struct llama_model * model, - const json &body, /* openai api json semantics */ - const std::string &chat_template) -{ - json llama_params; - - llama_params["__oaicompat"] = true; - - // Map OpenAI parameters to llama.cpp parameters - // - // For parameters that are defined by the OpenAI documentation (e.g. - // temperature), we explicitly specify OpenAI's intended default; we - // need to do that because sometimes OpenAI disagrees with llama.cpp - // - // https://platform.openai.com/docs/api-reference/chat/create - llama_sampling_params default_sparams; - llama_params["model"] = json_value(body, "model", std::string("unknown")); - llama_params["prompt"] = format_chat(model, chat_template, body["messages"]); - llama_params["cache_prompt"] = json_value(body, "cache_prompt", false); - llama_params["temperature"] = json_value(body, "temperature", 0.0); - llama_params["top_k"] = json_value(body, "top_k", default_sparams.top_k); - llama_params["top_p"] = json_value(body, "top_p", 1.0); - llama_params["n_predict"] = json_value(body, "max_tokens", -1); - llama_params["logit_bias"] = json_value(body, "logit_bias",json::object()); - llama_params["frequency_penalty"] = json_value(body, "frequency_penalty", 0.0); - llama_params["presence_penalty"] = json_value(body, "presence_penalty", 0.0); - llama_params["seed"] = json_value(body, "seed", LLAMA_DEFAULT_SEED); - llama_params["stream"] = json_value(body, "stream", false); - llama_params["mirostat"] = json_value(body, "mirostat", default_sparams.mirostat); - llama_params["mirostat_tau"] = json_value(body, "mirostat_tau", default_sparams.mirostat_tau); - llama_params["mirostat_eta"] = json_value(body, "mirostat_eta", default_sparams.mirostat_eta); - llama_params["penalize_nl"] = json_value(body, "penalize_nl", default_sparams.penalize_nl); - llama_params["typical_p"] = json_value(body, "typical_p", default_sparams.typical_p); - llama_params["repeat_last_n"] = json_value(body, "repeat_last_n", default_sparams.penalty_last_n); - llama_params["ignore_eos"] = json_value(body, "ignore_eos", false); - llama_params["tfs_z"] = json_value(body, "tfs_z", default_sparams.tfs_z); - - if (body.count("grammar") != 0) { - llama_params["grammar"] = json_value(body, "grammar", json::object()); - } - - // Handle 'stop' field - if (body.contains("stop") && body["stop"].is_string()) { - llama_params["stop"] = json::array({body["stop"].get()}); - } else { - llama_params["stop"] = json_value(body, "stop", json::array()); - } - - // Ensure there is ChatML-specific end sequence among stop words - llama_params["stop"].push_back("<|im_end|>"); - - return llama_params; -} - -inline static json format_final_response_oaicompat(const json &request, const task_result &response, bool streaming = false) -{ - json result = response.result_json; - - bool stopped_word = result.count("stopped_word") != 0; - bool stopped_eos = json_value(result, "stopped_eos", false); - int num_tokens_predicted = json_value(result, "tokens_predicted", 0); - int num_prompt_tokens = json_value(result, "tokens_evaluated", 0); - std::string content = json_value(result, "content", std::string("")); - - std::string finish_reason = "length"; - if (stopped_word || stopped_eos) { - finish_reason = "stop"; - } - - json choices = - streaming ? json::array({json{{"finish_reason", finish_reason}, - {"index", 0}, - {"delta", json::object()}}}) - : json::array({json{{"finish_reason", finish_reason}, - {"index", 0}, - {"message", json{{"content", content}, - {"role", "assistant"}}}}}); - - std::time_t t = std::time(0); - - json res = - json{{"choices", choices}, - {"created", t}, - {"model", - json_value(request, "model", std::string(DEFAULT_OAICOMPAT_MODEL))}, - {"object", streaming ? "chat.completion.chunk" : "chat.completion"}, - {"usage", - json{{"completion_tokens", num_tokens_predicted}, - {"prompt_tokens", num_prompt_tokens}, - {"total_tokens", num_tokens_predicted + num_prompt_tokens}}}, - {"id", gen_chatcmplid()}}; - - if (server_verbose) { - res["__verbose"] = result; - } - - if (result.contains("completion_probabilities")) { - res["completion_probabilities"] = json_value(result, "completion_probabilities", json::array()); - } - - return res; -} - -// return value is vector as there is one case where we might need to generate two responses -inline static std::vector format_partial_response_oaicompat(const task_result &response) { - json result = response.result_json; - - if (!result.contains("model") || !result.contains("oaicompat_token_ctr")) { - return std::vector({response.result_json}); - } - - bool first = json_value(result, "oaicompat_token_ctr", 0) == 0; - std::string modelname = json_value(result, "model", std::string(DEFAULT_OAICOMPAT_MODEL)); - - bool stopped_word = json_value(result, "stopped_word", false); - bool stopped_eos = json_value(result, "stopped_eos", false); - bool stopped_limit = json_value(result, "stopped_limit", false); - std::string content = json_value(result, "content", std::string("")); - - std::string finish_reason; - if (stopped_word || stopped_eos) { - finish_reason = "stop"; - } - if (stopped_limit) { - finish_reason = "length"; - } - - std::time_t t = std::time(0); - - json choices; - - if (!finish_reason.empty()) { - choices = json::array({json{{"finish_reason", finish_reason}, - {"index", 0}, - {"delta", json::object()}}}); - } else { - if (first) { - if (content.empty()) { - choices = json::array({json{{"finish_reason", nullptr}, - {"index", 0}, - {"delta", json{{"role", "assistant"}}}}}); - } else { - // We have to send this as two updates to conform to openai behavior - json initial_ret = json{{"choices", json::array({json{ - {"finish_reason", nullptr}, - {"index", 0}, - {"delta", json{ - {"role", "assistant"} - }}}})}, - {"created", t}, - {"id", gen_chatcmplid()}, - {"model", modelname}, - {"object", "chat.completion.chunk"}}; - - json second_ret = json{ - {"choices", json::array({json{{"finish_reason", nullptr}, - {"index", 0}, - {"delta", json{ - {"content", content}}} - }})}, - {"created", t}, - {"id", gen_chatcmplid()}, - {"model", modelname}, - {"object", "chat.completion.chunk"}}; - - return std::vector({initial_ret, second_ret}); - } - } else { - // Some idiosyncrasy in task processing logic makes several trailing calls - // with empty content, we ignore these at the calee site. - if (content.empty()) { - return std::vector({json::object()}); - } - - choices = json::array({json{ - {"finish_reason", nullptr}, - {"index", 0}, - {"delta", - json{ - {"content", content}, - }}, - }}); - } - } - - json ret = json{{"choices", choices}, - {"created", t}, - {"id", gen_chatcmplid()}, - {"model", modelname}, - {"object", "chat.completion.chunk"}}; - - return std::vector({ret}); -} - -inline static json format_embeddings_response_oaicompat(const json &request, const json &embeddings) -{ - json res = - json{ - {"model", json_value(request, "model", std::string(DEFAULT_OAICOMPAT_MODEL))}, - {"object", "list"}, - {"usage", - json{{"prompt_tokens", 0}, - {"total_tokens", 0}}}, - {"data", embeddings} - }; - return res; -} - diff --git a/examples/server/server.cpp b/examples/server/server.cpp index 8fe5e0b19..3bdbde954 100644 --- a/examples/server/server.cpp +++ b/examples/server/server.cpp @@ -1,13 +1,8 @@ +#include "utils.hpp" + #include "common.h" #include "llama.h" #include "grammar-parser.h" -#include "utils.hpp" -#include "oai.hpp" - -#include "../llava/clip.h" -#include "../llava/llava.h" - -#include "stb_image.h" #ifndef NDEBUG // crash the server in debug mode, otherwise send an http 500 error @@ -24,46 +19,76 @@ #include "completion.js.hpp" #include "json-schema-to-grammar.mjs.hpp" -#include -#include +#include #include #include -#include +#include +#include +#include +#include #include using json = nlohmann::json; -struct server_params { - std::string hostname = "127.0.0.1"; - std::vector api_keys; - std::string public_path = "examples/server/public"; - std::string chat_template = ""; - int32_t port = 8080; - int32_t read_timeout = 600; - int32_t write_timeout = 600; - bool slots_endpoint = true; - bool metrics_endpoint = false; - int n_threads_http = -1; -}; - bool server_verbose = false; bool server_log_json = true; enum stop_type { - STOP_FULL, - STOP_PARTIAL, + STOP_TYPE_FULL, + STOP_TYPE_PARTIAL, }; -// TODO: can become bool if we can't find use of more states enum slot_state { - IDLE, - PROCESSING, + SLOT_STATE_IDLE, + SLOT_STATE_PROCESSING, }; enum slot_command { - NONE, - LOAD_PROMPT, - RELEASE, + SLOT_COMMAND_NONE, + SLOT_COMMAND_LOAD_PROMPT, + SLOT_COMMAND_RELEASE, +}; + +enum server_state { + SERVER_STATE_LOADING_MODEL, // Server is starting up, model not fully loaded yet + SERVER_STATE_READY, // Server is ready and model is loaded + SERVER_STATE_ERROR // An error occurred, load_model failed +}; + +enum server_task_type { + SERVER_TASK_TYPE_COMPLETION, + SERVER_TASK_TYPE_CANCEL, + SERVER_TASK_TYPE_NEXT_RESPONSE, + SERVER_TASK_TYPE_METRICS +}; + +struct server_task { + int id = -1; // to be filled by server_queue + int id_multi = -1; + int id_target = -1; + + server_task_type type; + json data; + + bool infill = false; + bool embedding = false; +}; + +struct server_task_result { + int id = -1; + int id_multi = -1; + + json data; + + bool stop; + bool error; +}; + +struct server_task_multi { + int id = -1; + + std::set subtasks_remaining; + std::vector results; }; struct slot_params { @@ -80,26 +105,32 @@ struct slot_params { json input_suffix; }; -struct slot_image { - int32_t id; +struct server_params { + int32_t port = 8080; + int32_t read_timeout = 600; + int32_t write_timeout = 600; + int32_t n_threads_http = -1; - bool request_encode_image = false; - float * image_embedding = nullptr; - int32_t image_tokens = 0; + std::string hostname = "127.0.0.1"; + std::string public_path = "examples/server/public"; + std::string chat_template = ""; + std::string system_prompt = ""; - clip_image_u8 * img_data; + std::vector api_keys; - std::string prefix_prompt; // before of this image + bool slots_endpoint = true; + bool metrics_endpoint = false; }; struct server_slot { int id; - int task_id = -1; + int id_task = -1; + int id_multi = -1; struct slot_params params; - slot_state state = IDLE; - slot_command command = NONE; + slot_state state = SLOT_STATE_IDLE; + slot_command command = SLOT_COMMAND_NONE; // used to determine the slot that has been used the longest int64_t t_last_used = -1; @@ -116,27 +147,31 @@ struct server_slot { int32_t n_prompt_tokens_processed = 0; json prompt; + + // when a task is submitted, we first tokenize the prompt and store it here + std::vector prompt_tokens; + std::string generated_text; - llama_token sampled; std::vector cache_tokens; std::vector generated_token_probs; - bool infill = false; - bool embedding = false; + bool infill = false; + bool embedding = false; bool has_next_token = true; - bool truncated = false; - bool stopped_eos = false; - bool stopped_word = false; - bool stopped_limit = false; + bool truncated = false; + bool stopped_eos = false; + bool stopped_word = false; + bool stopped_limit = false; bool oaicompat = false; - std::string oaicompat_model; + std::string oaicompat_model; std::string stopping_word; // sampling + llama_token sampled; struct llama_sampling_params sparams; - llama_sampling_context *ctx_sampling = nullptr; + llama_sampling_context * ctx_sampling = nullptr; int32_t ga_i = 0; // group-attention state int32_t ga_n = 1; // group-attention factor @@ -144,48 +179,32 @@ struct server_slot { int32_t n_past_se = 0; // self-extend - // multimodal - std::vector images; - // stats size_t n_sent_text = 0; // number of sent text character size_t n_sent_token_probs = 0; int64_t t_start_process_prompt; - int64_t t_start_genereration; + int64_t t_start_generation; double t_prompt_processing; // ms double t_token_generation; // ms - // multitasks - int multitask_id = -1; - void reset() { - n_prompt_tokens = 0; - generated_text = ""; - truncated = false; - stopped_eos = false; - stopped_word = false; - stopped_limit = false; - stopping_word = ""; - n_past = 0; - n_sent_text = 0; - n_sent_token_probs = 0; - infill = false; - ga_i = 0; - n_past_se = 0; + n_prompt_tokens = 0; + generated_text = ""; + truncated = false; + stopped_eos = false; + stopped_word = false; + stopped_limit = false; + stopping_word = ""; + n_past = 0; + n_sent_text = 0; + n_sent_token_probs = 0; + infill = false; + ga_i = 0; + n_past_se = 0; generated_token_probs.clear(); - - for (slot_image & img : images) { - free(img.image_embedding); - if (img.img_data) { - clip_image_u8_free(img.img_data); - } - img.prefix_prompt = ""; - } - - images.clear(); } bool has_budget(gpt_params &global_params) { @@ -205,32 +224,29 @@ struct server_slot { } bool available() const { - return state == IDLE && command == NONE; + return state == SLOT_STATE_IDLE && command == SLOT_COMMAND_NONE; } bool is_processing() const { - return (state == IDLE && command == LOAD_PROMPT) || state == PROCESSING; + return (state == SLOT_STATE_IDLE && command == SLOT_COMMAND_LOAD_PROMPT) || state == SLOT_STATE_PROCESSING; } - void add_token_string(const completion_token_output &token) { - if (command == RELEASE) { + void add_token_string(const completion_token_output & token) { + if (command == SLOT_COMMAND_RELEASE) { return; } - cache_tokens.push_back(token.tok); generated_token_probs.push_back(token); } void release() { - if (state == PROCESSING) - { - t_token_generation = (ggml_time_us() - t_start_genereration) / 1e3; - command = RELEASE; + if (state == SLOT_STATE_PROCESSING) { + t_token_generation = (ggml_time_us() - t_start_generation) / 1e3; + command = SLOT_COMMAND_RELEASE; } } - json get_formated_timings() { - return json - { + json get_formated_timings() const { + return json { {"prompt_n", n_prompt_tokens_processed}, {"prompt_ms", t_prompt_processing}, {"prompt_per_token_ms", t_prompt_processing / n_prompt_tokens_processed}, @@ -243,16 +259,47 @@ struct server_slot { }; } + size_t find_stopping_strings(const std::string & text, const size_t last_token_size, const stop_type type) { + size_t stop_pos = std::string::npos; + + for (const std::string & word : params.antiprompt) { + size_t pos; + + if (type == STOP_TYPE_FULL) { + const size_t tmp = word.size() + last_token_size; + const size_t from_pos = text.size() > tmp ? text.size() - tmp : 0; + + pos = text.find(word, from_pos); + } else { + pos = find_partial_stop_string(word, text); + } + + if (pos != std::string::npos && (stop_pos == std::string::npos || pos < stop_pos)) { + if (type == STOP_TYPE_FULL) { + stopped_word = true; + stopping_word = word; + has_next_token = false; + } + stop_pos = pos; + } + } + + return stop_pos; + } + void print_timings() const { - char buffer[512]; + char buffer[512]; + double t_token = t_prompt_processing / n_prompt_tokens_processed; double n_tokens_second = 1e3 / t_prompt_processing * n_prompt_tokens_processed; - sprintf(buffer, "prompt eval time = %10.2f ms / %5d tokens (%8.2f ms per token, %8.2f tokens per second)", + + snprintf(buffer, 512, "prompt eval time = %10.2f ms / %5d tokens (%8.2f ms per token, %8.2f tokens per second)", t_prompt_processing, n_prompt_tokens_processed, t_token, n_tokens_second); + LOG_INFO(buffer, { - {"slot_id", id}, - {"task_id", task_id}, + {"id_slot", id}, + {"id_task", id_task}, {"t_prompt_processing", t_prompt_processing}, {"n_prompt_tokens_processed", n_prompt_tokens_processed}, {"t_token", t_token}, @@ -261,22 +308,25 @@ struct server_slot { t_token = t_token_generation / n_decoded; n_tokens_second = 1e3 / t_token_generation * n_decoded; - sprintf(buffer, "generation eval time = %10.2f ms / %5d runs (%8.2f ms per token, %8.2f tokens per second)", + + snprintf(buffer, 512, "generation eval time = %10.2f ms / %5d runs (%8.2f ms per token, %8.2f tokens per second)", t_token_generation, n_decoded, t_token, n_tokens_second); + LOG_INFO(buffer, { - {"slot_id", id}, - {"task_id", task_id}, + {"id_slot", id}, + {"id_task", id_task}, {"t_token_generation", t_token_generation}, {"n_decoded", n_decoded}, {"t_token", t_token}, {"n_tokens_second", n_tokens_second}, }); - sprintf(buffer, " total time = %10.2f ms", t_prompt_processing + t_token_generation); + snprintf(buffer, 512, " total time = %10.2f ms", t_prompt_processing + t_token_generation); + LOG_INFO(buffer, { - {"slot_id", id}, - {"task_id", task_id}, + {"id_slot", id}, + {"id_task", id_task}, {"t_prompt_processing", t_prompt_processing}, {"t_token_generation", t_token_generation}, {"t_total", t_prompt_processing + t_token_generation}, @@ -291,9 +341,8 @@ struct server_metrics { uint64_t n_prompt_tokens_processed = 0; uint64_t t_prompt_processing = 0; - uint64_t n_tokens_predicted = 0; - uint64_t t_tokens_generation = 0; - + uint64_t n_tokens_predicted = 0; + uint64_t t_tokens_generation = 0; void on_prompt_eval(const server_slot &slot) { n_prompt_tokens_processed_total += slot.n_prompt_tokens_processed; @@ -315,23 +364,261 @@ struct server_metrics { } }; -struct llama_server_context -{ - llama_model *model = nullptr; - llama_context *ctx = nullptr; +struct server_queue { + int id = 0; + bool running; - clip_ctx *clp_ctx = nullptr; + // queues + std::vector queue_tasks; + std::vector queue_tasks_deferred; + + std::vector queue_multitasks; + + std::mutex mutex_tasks; + std::condition_variable condition_tasks; + + // callback functions + std::function callback_new_task; + std::function callback_finish_multitask; + std::function callback_run_slots; + + // Add a new task to the end of the queue + int post(server_task task) { + std::unique_lock lock(mutex_tasks); + if (task.id == -1) { + task.id = id++; + LOG_VERBOSE("new task id", {{"new_id", task.id}}); + } + queue_tasks.push_back(std::move(task)); + condition_tasks.notify_one(); + return task.id; + } + + // Add a new task, but defer until one slot is available + void defer(server_task task) { + std::unique_lock lock(mutex_tasks); + queue_tasks_deferred.push_back(std::move(task)); + } + + // Get the next id for creating anew task + int get_new_id() { + std::unique_lock lock(mutex_tasks); + int new_id = id++; + LOG_VERBOSE("new task id", {{"new_id", new_id}}); + return new_id; + } + + // Register function to process a new task + void on_new_task(std::function callback) { + callback_new_task = std::move(callback); + } + + // Register function to process a multitask when it is finished + void on_finish_multitask(std::function callback) { + callback_finish_multitask = std::move(callback); + } + + // Register the function to be called when all slots data is ready to be processed + void on_run_slots(std::function callback) { + callback_run_slots = std::move(callback); + } + + // Call when the state of one slot is changed + void notify_slot_changed() { + // move deferred tasks back to main loop + std::unique_lock lock(mutex_tasks); + for (auto & task : queue_tasks_deferred) { + queue_tasks.push_back(std::move(task)); + } + queue_tasks_deferred.clear(); + } + + // end the start_loop routine + void terminate() { + std::unique_lock lock(mutex_tasks); + running = false; + condition_tasks.notify_all(); + } + + /** + * Main loop consists of these steps: + * - Wait until a new task arrives + * - Process the task (i.e. maybe copy data into slot) + * - Check if multitask is finished + * - Run all slots + */ + void start_loop() { + running = true; + + while (true) { + LOG_VERBOSE("new task may arrive", {}); + + while (true) { + std::unique_lock lock(mutex_tasks); + if (queue_tasks.empty()) { + lock.unlock(); + break; + } + server_task task = queue_tasks.front(); + queue_tasks.erase(queue_tasks.begin()); + lock.unlock(); + LOG_VERBOSE("callback_new_task", {{"id_task", task.id}}); + callback_new_task(task); + } + + LOG_VERBOSE("update_multitasks", {}); + + // check if we have any finished multitasks + auto queue_iterator = queue_multitasks.begin(); + while (queue_iterator != queue_multitasks.end()) { + if (queue_iterator->subtasks_remaining.empty()) { + // all subtasks done == multitask is done + server_task_multi current_multitask = *queue_iterator; + callback_finish_multitask(current_multitask); + // remove this multitask + queue_iterator = queue_multitasks.erase(queue_iterator); + } else { + ++queue_iterator; + } + } + + // all tasks in the current loop is processed, slots data is now ready + LOG_VERBOSE("callback_run_slots", {}); + + callback_run_slots(); + + LOG_VERBOSE("wait for new task", {}); + { + std::unique_lock lock(mutex_tasks); + if (queue_tasks.empty()) { + if (!running) { + LOG_VERBOSE("ending start_loop", {}); + return; + } + condition_tasks.wait(lock, [&]{ + return (!queue_tasks.empty() || !running); + }); + } + } + } + } + + // + // functions to manage multitasks + // + + // add a multitask by specifying the id of all subtask (subtask is a server_task) + void add_multitask(int id_multi, std::vector & sub_ids) { + std::lock_guard lock(mutex_tasks); + server_task_multi multi; + multi.id = id_multi; + std::copy(sub_ids.begin(), sub_ids.end(), std::inserter(multi.subtasks_remaining, multi.subtasks_remaining.end())); + queue_multitasks.push_back(multi); + } + + // updatethe remaining subtasks, while appending results to multitask + void update_multitask(int id_multi, int id_sub, server_task_result & result) { + std::lock_guard lock(mutex_tasks); + for (auto & multitask : queue_multitasks) { + if (multitask.id == id_multi) { + multitask.subtasks_remaining.erase(id_sub); + multitask.results.push_back(result); + } + } + } +}; + +struct server_response { + typedef std::function callback_multitask_t; + callback_multitask_t callback_update_multitask; + + // for keeping track of all tasks waiting for the result + std::set waiting_task_ids; + + // the main result queue + std::vector queue_results; + + std::mutex mutex_results; + std::condition_variable condition_results; + + // add the id_task to the list of tasks waiting for response + void add_waiting_task_id(int id_task) { + LOG_VERBOSE("waiting for task id", {{"id_task", id_task}}); + + std::unique_lock lock(mutex_results); + waiting_task_ids.insert(id_task); + } + + // when the request is finished, we can remove task associated with it + void remove_waiting_task_id(int id_task) { + LOG_VERBOSE("remove waiting for task id", {{"id_task", id_task}}); + + std::unique_lock lock(mutex_results); + waiting_task_ids.erase(id_task); + } + + // This function blocks the thread until there is a response for this id_task + server_task_result recv(int id_task) { + while (true) { + std::unique_lock lock(mutex_results); + condition_results.wait(lock, [&]{ + return !queue_results.empty(); + }); + + for (int i = 0; i < (int) queue_results.size(); i++) { + if (queue_results[i].id == id_task) { + assert(queue_results[i].id_multi == -1); + server_task_result res = queue_results[i]; + queue_results.erase(queue_results.begin() + i); + return res; + } + } + } + + // should never reach here + } + + // Register the function to update multitask + void on_multitask_update(callback_multitask_t callback) { + callback_update_multitask = std::move(callback); + } + + // Send a new result to a waiting id_task + void send(server_task_result result) { + LOG_VERBOSE("send new result", {{"id_task", result.id}}); + + std::unique_lock lock(mutex_results); + for (const auto & id_task : waiting_task_ids) { + // LOG_TEE("waiting task id %i \n", id_task); + // for now, tasks that have associated parent multitasks just get erased once multitask picks up the result + if (result.id_multi == id_task) { + LOG_VERBOSE("callback_update_multitask", {{"id_task", id_task}}); + callback_update_multitask(id_task, result.id, result); + continue; + } + + if (result.id == id_task) { + LOG_VERBOSE("queue_results.push_back", {{"id_task", id_task}}); + queue_results.push_back(result); + condition_results.notify_all(); + return; + } + } + } +}; + +struct server_context { + llama_model * model = nullptr; + llama_context * ctx = nullptr; gpt_params params; llama_batch batch; - bool multimodal = false; - bool clean_kv_cache = true; - bool all_slots_are_idle = false; - bool add_bos_token = true; + bool clean_kv_cache = true; + bool add_bos_token = true; - int32_t n_ctx; // total context for all clients / slots + int32_t n_ctx; // total context for all clients / slots // system prompt bool system_need_update = false; @@ -346,60 +633,32 @@ struct llama_server_context std::vector slots; json default_generation_settings_for_props; - llama_server_queue queue_tasks; - llama_server_response queue_results; + server_queue queue_tasks; + server_response queue_results; server_metrics metrics; - ~llama_server_context() - { - if (ctx) - { + ~server_context() { + if (ctx) { llama_free(ctx); ctx = nullptr; } - if (model) - { + + if (model) { llama_free_model(model); model = nullptr; } } - bool load_model(const gpt_params ¶ms_) - { + bool load_model(const gpt_params & params_) { params = params_; - if (!params.mmproj.empty()) { - multimodal = true; - LOG_INFO("Multi Modal Mode Enabled", {}); - clp_ctx = clip_model_load(params.mmproj.c_str(), /*verbosity=*/ 1); - if(clp_ctx == nullptr) { - LOG_ERROR("unable to load clip model", {{"model", params.mmproj}}); - return false; - } - - if (params.n_ctx < 2048) { // request larger context for the image embedding - params.n_ctx = 2048; - } - } std::tie(model, ctx) = llama_init_from_gpt_params(params); - if (model == nullptr) - { + if (model == nullptr) { LOG_ERROR("unable to load model", {{"model", params.model}}); return false; } - if (multimodal) { - const int n_embd_clip = clip_n_mmproj_embd(clp_ctx); - const int n_embd_llm = llama_n_embd(model); - if (n_embd_clip != n_embd_llm) { - LOG_TEE("%s: embedding dim of the multimodal projector (%d) is not equal to that of LLaMA (%d). Make sure that you use the correct mmproj file.\n", __func__, n_embd_clip, n_embd_llm); - llama_free(ctx); - llama_free_model(model); - return false; - } - } - n_ctx = llama_n_ctx(ctx); add_bos_token = llama_should_add_bos_token(model); @@ -407,25 +666,19 @@ struct llama_server_context return true; } - void validate_model_chat_template(server_params & sparams) { + bool validate_model_chat_template() const { llama_chat_message chat[] = {{"user", "test"}}; - std::vector buf(1); - int res = llama_chat_apply_template(model, nullptr, chat, 1, true, buf.data(), buf.size()); - if (res < 0) { - LOG_ERROR("The chat template comes with this model is not yet supported, falling back to chatml. This may cause the model to output suboptimal responses", {}); - sparams.chat_template = "chatml"; - } + + const int res = llama_chat_apply_template(model, nullptr, chat, 1, true, nullptr, 0); + + return res > 0; } void initialize() { - // create slots - all_slots_are_idle = true; - const int32_t n_ctx_slot = n_ctx / params.n_parallel; LOG_INFO("initializing slots", {{"n_slots", params.n_parallel}}); - for (int i = 0; i < params.n_parallel; i++) - { + for (int i = 0; i < params.n_parallel; i++) { server_slot slot; slot.id = i; @@ -433,7 +686,7 @@ struct llama_server_context slot.n_predict = params.n_predict; LOG_INFO("new slot", { - {"slot_id", slot.id}, + {"id_slot", slot.id}, {"n_ctx_slot", slot.n_ctx} }); @@ -447,9 +700,9 @@ struct llama_server_context //GGML_ASSERT(n_ctx >= n_ctx_train * ga_n && "n_ctx must be at least n_ctx_train * ga_n"); // NOLINT LOG_INFO("slot self-extend", { - {"slot_id", slot.id}, - {"ga_n", ga_n}, - {"ga_w", ga_w} + {"id_slot", slot.id}, + {"ga_n", ga_n}, + {"ga_w", ga_w} }); } @@ -468,8 +721,7 @@ struct llama_server_context batch = llama_batch_init(n_ctx, 0, params.n_parallel); } - std::vector tokenize(const json & json_prompt, bool add_bos) const - { + std::vector tokenize(const json & json_prompt, bool add_bos) const { // TODO: currently, we tokenize using special tokens by default // this is not always correct (see https://github.com/ggerganov/llama.cpp/pull/4160#issuecomment-1824826216) // but it's better compared to completely ignoring ChatML and other chat templates @@ -479,38 +731,30 @@ struct llama_server_context // or the first element of the json_prompt array is a string. std::vector prompt_tokens; - if (json_prompt.is_array()) - { + if (json_prompt.is_array()) { bool first = true; - for (const auto& p : json_prompt) - { - if (p.is_string()) - { + for (const auto & p : json_prompt) { + if (p.is_string()) { auto s = p.template get(); + std::vector p; - if (first) - { + if (first) { p = ::llama_tokenize(ctx, s, add_bos, TMP_FORCE_SPECIAL); first = false; - } - else - { + } else { p = ::llama_tokenize(ctx, s, false, TMP_FORCE_SPECIAL); } + prompt_tokens.insert(prompt_tokens.end(), p.begin(), p.end()); - } - else - { - if (first) - { + } else { + if (first) { first = false; } + prompt_tokens.push_back(p.template get()); } } - } - else - { + } else { auto s = json_prompt.template get(); prompt_tokens = ::llama_tokenize(ctx, s, add_bos, TMP_FORCE_SPECIAL); } @@ -518,19 +762,18 @@ struct llama_server_context return prompt_tokens; } - server_slot* get_slot(int id) { + server_slot * get_slot(int id) { int64_t t_last = ggml_time_us(); - server_slot *last_used = nullptr; - for (server_slot & slot : slots) - { - if (slot.id == id && slot.available()) - { + server_slot * last_used = nullptr; + + for (server_slot & slot : slots) { + if (slot.id == id && slot.available()) { return &slot; } - if (slot.available() && slot.t_last_used < t_last) - { + // among all available slots, find the one that has been least recently used + if (slot.available() && slot.t_last_used < t_last) { last_used = &slot; t_last = slot.t_last_used; } @@ -539,295 +782,204 @@ struct llama_server_context return last_used; } - bool launch_slot_with_data(server_slot* &slot, json data) { + bool launch_slot_with_data(server_slot & slot, json data) const { slot_params default_params; llama_sampling_params default_sparams; if (data.count("__oaicompat") != 0) { - slot->oaicompat = true; - slot->oaicompat_model = json_value(data, "model", std::string(DEFAULT_OAICOMPAT_MODEL)); + slot.oaicompat = true; + slot.oaicompat_model = json_value(data, "model", std::string(DEFAULT_OAICOMPAT_MODEL)); } else { - slot->oaicompat = false; - slot->oaicompat_model = ""; + slot.oaicompat = false; + slot.oaicompat_model = ""; } - slot->params.stream = json_value(data, "stream", false); - slot->params.cache_prompt = json_value(data, "cache_prompt", false); - slot->params.n_predict = json_value(data, "n_predict", default_params.n_predict); - slot->sparams.top_k = json_value(data, "top_k", default_sparams.top_k); - slot->sparams.top_p = json_value(data, "top_p", default_sparams.top_p); - slot->sparams.min_p = json_value(data, "min_p", default_sparams.min_p); - slot->sparams.tfs_z = json_value(data, "tfs_z", default_sparams.tfs_z); - slot->sparams.typical_p = json_value(data, "typical_p", default_sparams.typical_p); - slot->sparams.temp = json_value(data, "temperature", default_sparams.temp); - slot->sparams.dynatemp_range = json_value(data, "dynatemp_range", default_sparams.dynatemp_range); - slot->sparams.dynatemp_exponent = json_value(data, "dynatemp_exponent", default_sparams.dynatemp_exponent); - slot->sparams.penalty_last_n = json_value(data, "repeat_last_n", default_sparams.penalty_last_n); - slot->sparams.penalty_repeat = json_value(data, "repeat_penalty", default_sparams.penalty_repeat); - slot->sparams.penalty_freq = json_value(data, "frequency_penalty", default_sparams.penalty_freq); - slot->sparams.penalty_present = json_value(data, "presence_penalty", default_sparams.penalty_present); - slot->sparams.mirostat = json_value(data, "mirostat", default_sparams.mirostat); - slot->sparams.mirostat_tau = json_value(data, "mirostat_tau", default_sparams.mirostat_tau); - slot->sparams.mirostat_eta = json_value(data, "mirostat_eta", default_sparams.mirostat_eta); - slot->sparams.penalize_nl = json_value(data, "penalize_nl", default_sparams.penalize_nl); - slot->params.n_keep = json_value(data, "n_keep", slot->params.n_keep); - slot->params.seed = json_value(data, "seed", default_params.seed); - slot->sparams.grammar = json_value(data, "grammar", default_sparams.grammar); - slot->sparams.n_probs = json_value(data, "n_probs", default_sparams.n_probs); - slot->sparams.min_keep = json_value(data, "min_keep", default_sparams.min_keep); + slot.params.stream = json_value(data, "stream", false); + slot.params.cache_prompt = json_value(data, "cache_prompt", false); + slot.params.n_predict = json_value(data, "n_predict", default_params.n_predict); + slot.sparams.top_k = json_value(data, "top_k", default_sparams.top_k); + slot.sparams.top_p = json_value(data, "top_p", default_sparams.top_p); + slot.sparams.min_p = json_value(data, "min_p", default_sparams.min_p); + slot.sparams.tfs_z = json_value(data, "tfs_z", default_sparams.tfs_z); + slot.sparams.typical_p = json_value(data, "typical_p", default_sparams.typical_p); + slot.sparams.temp = json_value(data, "temperature", default_sparams.temp); + slot.sparams.dynatemp_range = json_value(data, "dynatemp_range", default_sparams.dynatemp_range); + slot.sparams.dynatemp_exponent = json_value(data, "dynatemp_exponent", default_sparams.dynatemp_exponent); + slot.sparams.penalty_last_n = json_value(data, "repeat_last_n", default_sparams.penalty_last_n); + slot.sparams.penalty_repeat = json_value(data, "repeat_penalty", default_sparams.penalty_repeat); + slot.sparams.penalty_freq = json_value(data, "frequency_penalty", default_sparams.penalty_freq); + slot.sparams.penalty_present = json_value(data, "presence_penalty", default_sparams.penalty_present); + slot.sparams.mirostat = json_value(data, "mirostat", default_sparams.mirostat); + slot.sparams.mirostat_tau = json_value(data, "mirostat_tau", default_sparams.mirostat_tau); + slot.sparams.mirostat_eta = json_value(data, "mirostat_eta", default_sparams.mirostat_eta); + slot.sparams.penalize_nl = json_value(data, "penalize_nl", default_sparams.penalize_nl); + slot.params.n_keep = json_value(data, "n_keep", slot.params.n_keep); + slot.params.seed = json_value(data, "seed", default_params.seed); + slot.sparams.grammar = json_value(data, "grammar", default_sparams.grammar); + slot.sparams.n_probs = json_value(data, "n_probs", default_sparams.n_probs); + slot.sparams.min_keep = json_value(data, "min_keep", default_sparams.min_keep); - if (slot->n_predict > 0 && slot->params.n_predict > slot->n_predict) { + if (slot.params.cache_prompt && slot.ga_n != 1) { + LOG_WARNING("cache_prompt is not supported with group-attention", {}); + slot.params.cache_prompt = false; + } + + if (slot.n_predict > 0 && slot.params.n_predict > slot.n_predict) { // Might be better to reject the request with a 400 ? LOG_WARNING("Max tokens to predict exceeds server configuration", { - {"params.n_predict", slot->params.n_predict}, - {"slot.n_predict", slot->n_predict}, + {"params.n_predict", slot.params.n_predict}, + {"slot.n_predict", slot.n_predict}, }); - slot->params.n_predict = slot->n_predict; + slot.params.n_predict = slot.n_predict; } // infill - if (data.count("input_prefix") != 0) - { - slot->params.input_prefix = data["input_prefix"]; - } - else - { - slot->params.input_prefix = ""; - } + slot.params.input_prefix = json_value(data, "input_prefix", default_params.input_prefix); + slot.params.input_suffix = json_value(data, "input_suffix", default_params.input_suffix); + slot.prompt = json_value(data, "prompt", std::string("")); - if (data.count("input_suffix") != 0) + // penalize user-provided tokens { - slot->params.input_suffix = data["input_suffix"]; - } - else - { - slot->params.input_suffix = ""; - } + slot.sparams.penalty_prompt_tokens.clear(); + slot.sparams.use_penalty_prompt_tokens = false; - if (data.count("prompt") != 0) - { - slot->prompt = data["prompt"]; - } - else - { - slot->prompt = ""; - } + const auto & penalty_prompt = data.find("penalty_prompt"); - slot->sparams.penalty_prompt_tokens.clear(); - slot->sparams.use_penalty_prompt_tokens = false; - const auto &penalty_prompt = data.find("penalty_prompt"); - if (penalty_prompt != data.end()) - { - if (penalty_prompt->is_string()) - { - const auto penalty_prompt_string = penalty_prompt->get(); - auto penalty_tokens = llama_tokenize(model, penalty_prompt_string, false); - slot->sparams.penalty_prompt_tokens.swap(penalty_tokens); - if (slot->params.n_predict > 0) - { - slot->sparams.penalty_prompt_tokens.reserve(slot->sparams.penalty_prompt_tokens.size() + slot->params.n_predict); - } - slot->sparams.use_penalty_prompt_tokens = true; - } - else if (penalty_prompt->is_array()) - { - const auto n_tokens = penalty_prompt->size(); - slot->sparams.penalty_prompt_tokens.reserve(n_tokens + std::max(0, slot->params.n_predict)); - const int n_vocab = llama_n_vocab(model); - for (const auto &penalty_token : *penalty_prompt) - { - if (penalty_token.is_number_integer()) - { - const auto tok = penalty_token.get(); - if (tok >= 0 && tok < n_vocab) - { - slot->sparams.penalty_prompt_tokens.push_back(tok); - } + if (penalty_prompt != data.end()) { + if (penalty_prompt->is_string()) { + const auto penalty_prompt_string = penalty_prompt->get(); + slot.sparams.penalty_prompt_tokens = llama_tokenize(model, penalty_prompt_string, false); + + if (slot.params.n_predict > 0) { + slot.sparams.penalty_prompt_tokens.reserve(slot.sparams.penalty_prompt_tokens.size() + slot.params.n_predict); } - } - slot->sparams.use_penalty_prompt_tokens = true; - } - } + slot.sparams.use_penalty_prompt_tokens = true; - slot->sparams.logit_bias.clear(); - - if (json_value(data, "ignore_eos", false)) - { - slot->sparams.logit_bias[llama_token_eos(model)] = -INFINITY; - } - - const auto &logit_bias = data.find("logit_bias"); - if (logit_bias != data.end() && logit_bias->is_array()) - { - const int n_vocab = llama_n_vocab(model); - for (const auto &el : *logit_bias) - { - if (el.is_array() && el.size() == 2) - { - float bias; - if (el[1].is_number()) - { - bias = el[1].get(); - } - else if (el[1].is_boolean() && !el[1].get()) - { - bias = -INFINITY; - } - else - { - continue; - } - - if (el[0].is_number_integer()) - { - llama_token tok = el[0].get(); - if (tok >= 0 && tok < n_vocab) - { - slot->sparams.logit_bias[tok] = bias; - } - } - else if (el[0].is_string()) - { - auto toks = llama_tokenize(model, el[0].get(), false); - for (auto tok : toks) - { - slot->sparams.logit_bias[tok] = bias; - } - } - } - } - } - - slot->params.antiprompt.clear(); - - const auto &stop = data.find("stop"); - if (stop != data.end() && stop->is_array()) - { - for (const auto &word : *stop) - { - if (!word.empty()) - { - slot->params.antiprompt.push_back(word); - } - } - } - - const auto &samplers_sequence = data.find("samplers"); - if (samplers_sequence != data.end() && samplers_sequence->is_array()) - { - std::vector sampler_names; - for (const auto &sampler_name : *samplers_sequence) - { - if (sampler_name.is_string()) - { - sampler_names.emplace_back(sampler_name); - } - } - slot->sparams.samplers_sequence = sampler_types_from_names(sampler_names, false); - } - else - { - slot->sparams.samplers_sequence = default_sparams.samplers_sequence; - } - - if (multimodal) - { - const auto &images_data = data.find("image_data"); - if (images_data != data.end() && images_data->is_array()) - { - for (const auto &img : *images_data) - { - const std::vector image_buffer = base64_decode(img["data"].get()); - - slot_image img_sl; - img_sl.id = img.count("id") != 0 ? img["id"].get() : slot->images.size(); - img_sl.img_data = clip_image_u8_init(); - if (!clip_image_load_from_bytes(image_buffer.data(), image_buffer.size(), img_sl.img_data)) - { - LOG_ERROR("failed to load image", { - {"slot_id", slot->id}, - {"img_sl_id", img_sl.id} - }); - return false; - } - LOG_VERBOSE("image loaded", { - {"slot_id", slot->id}, - {"img_sl_id", img_sl.id} + LOG_VERBOSE("penalty_prompt_tokens", { + {"id_slot", slot.id}, + {"tokens", slot.sparams.penalty_prompt_tokens}, }); - img_sl.request_encode_image = true; - slot->images.push_back(img_sl); } - // process prompt - // example: system prompt [img-102] user [img-103] describe [img-134] -> [{id: 102, prefix: 'system prompt '}, {id: 103, prefix: ' user '}, {id: 134, prefix: ' describe '}]} - if (slot->images.size() > 0 && !slot->prompt.is_array()) - { - std::string prompt = slot->prompt.get(); - size_t pos = 0, begin_prefix = 0; - std::string pattern = "[img-"; - while ((pos = prompt.find(pattern, pos)) != std::string::npos) { - size_t end_prefix = pos; - pos += pattern.length(); - size_t end_pos = prompt.find(']', pos); - if (end_pos != std::string::npos) - { - std::string image_id = prompt.substr(pos, end_pos - pos); - try - { - int img_id = std::stoi(image_id); - bool found = false; - for (slot_image &img : slot->images) - { - if (img.id == img_id) { - found = true; - img.prefix_prompt = prompt.substr(begin_prefix, end_prefix - begin_prefix); - begin_prefix = end_pos + 1; - break; - } - } - if (!found) { - LOG_TEE("ERROR: Image with id: %i, not found.\n", img_id); - slot->images.clear(); - return false; - } - } catch (const std::invalid_argument& e) { - LOG_TEE("Invalid image number id in prompt\n"); - slot->images.clear(); - return false; + else if (penalty_prompt->is_array()) { + const auto n_tokens = penalty_prompt->size(); + slot.sparams.penalty_prompt_tokens.reserve(n_tokens + std::max(0, slot.params.n_predict)); + + const int n_vocab = llama_n_vocab(model); + for (const auto & penalty_token : *penalty_prompt) { + if (penalty_token.is_number_integer()) { + const auto tok = penalty_token.get(); + if (tok >= 0 && tok < n_vocab) { + slot.sparams.penalty_prompt_tokens.push_back(tok); } } } - slot->prompt = ""; - slot->params.input_suffix = prompt.substr(begin_prefix); - slot->params.cache_prompt = false; // multimodal doesn't support cache prompt + slot.sparams.use_penalty_prompt_tokens = true; + + LOG_VERBOSE("penalty_prompt_tokens", { + {"id_slot", slot.id}, + {"tokens", slot.sparams.penalty_prompt_tokens}, + }); } } } - if (slot->ctx_sampling != nullptr) { - llama_sampling_free(slot->ctx_sampling); - } - slot->ctx_sampling = llama_sampling_init(slot->sparams); - llama_set_rng_seed(ctx, slot->params.seed); - slot->command = LOAD_PROMPT; + slot.sparams.logit_bias.clear(); - all_slots_are_idle = false; + if (json_value(data, "ignore_eos", false)) { + slot.sparams.logit_bias[llama_token_eos(model)] = -INFINITY; + } + + const auto & logit_bias = data.find("logit_bias"); + if (logit_bias != data.end() && logit_bias->is_array()) { + const int n_vocab = llama_n_vocab(model); + for (const auto & el : *logit_bias) { + if (el.is_array() && el.size() == 2) { + float bias; + if (el[1].is_number()) { + bias = el[1].get(); + } else if (el[1].is_boolean() && !el[1].get()) { + bias = -INFINITY; + } else { + continue; + } + + if (el[0].is_number_integer()) { + llama_token tok = el[0].get(); + if (tok >= 0 && tok < n_vocab) { + slot.sparams.logit_bias[tok] = bias; + } + } else if (el[0].is_string()) { + auto toks = llama_tokenize(model, el[0].get(), false); + for (auto tok : toks) { + slot.sparams.logit_bias[tok] = bias; + } + } + } + } + } + } + + { + slot.params.antiprompt.clear(); + + const auto & stop = data.find("stop"); + if (stop != data.end() && stop->is_array()) { + for (const auto & word : *stop) { + if (!word.empty()) { + slot.params.antiprompt.push_back(word); + } + } + } + } + + { + const auto & samplers_sequence = data.find("samplers"); + if (samplers_sequence != data.end() && samplers_sequence->is_array()) { + std::vector sampler_names; + for (const auto & sampler_name : *samplers_sequence) { + if (sampler_name.is_string()) { + sampler_names.emplace_back(sampler_name); + } + } + slot.sparams.samplers_sequence = sampler_types_from_names(sampler_names, false); + } else { + slot.sparams.samplers_sequence = default_sparams.samplers_sequence; + } + } + + { + if (slot.ctx_sampling != nullptr) { + llama_sampling_free(slot.ctx_sampling); + } + slot.ctx_sampling = llama_sampling_init(slot.sparams); + llama_set_rng_seed(ctx, slot.params.seed); + } + + slot.command = SLOT_COMMAND_LOAD_PROMPT; + slot.prompt_tokens.clear(); LOG_INFO("slot is processing task", { - {"slot_id", slot->id}, - {"task_id", slot->task_id}, + {"id_slot", slot.id}, + {"id_task", slot.id_task}, }); return true; } void kv_cache_clear() { + LOG_VERBOSE("clearing KV cache", {}); + // clear the entire KV cache llama_kv_cache_clear(ctx); clean_kv_cache = false; } void system_prompt_update() { + LOG_VERBOSE("system prompt update", { + {"system_prompt", system_prompt}, + }); + kv_cache_clear(); system_tokens.clear(); @@ -836,13 +988,11 @@ struct llama_server_context llama_batch_clear(batch); - for (int i = 0; i < (int)system_tokens.size(); ++i) - { + for (int i = 0; i < (int)system_tokens.size(); ++i) { llama_batch_add(batch, system_tokens[i], i, { 0 }, false); } - for (int32_t i = 0; i < (int32_t) batch.n_tokens; i += params.n_batch) - { + for (int32_t i = 0; i < (int32_t) batch.n_tokens; i += params.n_batch) { const int32_t n_tokens = std::min(params.n_batch, (int32_t) (batch.n_tokens - i)); llama_batch batch_view = { n_tokens, @@ -854,78 +1004,42 @@ struct llama_server_context batch.logits + i, 0, 0, 0, // unused }; - if (llama_decode(ctx, batch_view) != 0) - { + + if (llama_decode(ctx, batch_view) != 0) { LOG_TEE("%s: llama_decode() failed\n", __func__); return; } } // assign the system KV cache to all parallel sequences - for (int32_t i = 1; i < params.n_parallel; ++i) - { + for (int32_t i = 1; i < params.n_parallel; ++i) { llama_kv_cache_seq_cp(ctx, 0, i, 0, system_tokens.size()); } } - LOG_TEE("system prompt updated\n"); system_need_update = false; } - void system_prompt_notify() { + void system_prompt_set(const json & sys_props) { + system_prompt = sys_props.value("prompt", ""); + name_user = sys_props.value("anti_prompt", ""); + name_assistant = sys_props.value("assistant_name", ""); + + LOG_VERBOSE("system prompt process", { + {"system_prompt", system_prompt}, + {"name_user", name_user}, + {"name_assistant", name_assistant}, + }); + // release all slots - for (server_slot &slot : slots) - { + for (server_slot & slot : slots) { slot.release(); } system_need_update = true; } - void system_prompt_process(const json &sys_props) { - system_prompt = sys_props.value("prompt", ""); - name_user = sys_props.value("anti_prompt", ""); - name_assistant = sys_props.value("assistant_name", ""); - - - system_prompt_notify(); - } - - static size_t find_stopping_strings(const std::string &text, const size_t last_token_size, - const stop_type type, server_slot &slot) - { - size_t stop_pos = std::string::npos; - - for (const std::string &word : slot.params.antiprompt) - { - size_t pos; - if (type == STOP_FULL) - { - const size_t tmp = word.size() + last_token_size; - const size_t from_pos = text.size() > tmp ? text.size() - tmp : 0; - pos = text.find(word, from_pos); - } - else - { - pos = find_partial_stop_string(word, text); - } - if (pos != std::string::npos && - (stop_pos == std::string::npos || pos < stop_pos)) - { - if (type == STOP_FULL) - { - slot.stopped_word = true; - slot.stopping_word = word; - slot.has_next_token = false; - } - stop_pos = pos; - } - } - - return stop_pos; - } - - bool process_token(completion_token_output &result, server_slot &slot) { + bool process_token(completion_token_output & result, server_slot & slot) { // remember which tokens were sampled - used for repetition penalties during sampling const std::string token_str = llama_token_to_piece(ctx, result.tok); slot.sampled = result.tok; @@ -934,34 +1048,26 @@ struct llama_server_context slot.generated_text += token_str; slot.has_next_token = true; - if (slot.ctx_sampling->params.use_penalty_prompt_tokens && result.tok != -1) - { + if (slot.ctx_sampling->params.use_penalty_prompt_tokens && result.tok != -1) { // we can change penalty_prompt_tokens because it is always created from scratch each request slot.ctx_sampling->params.penalty_prompt_tokens.push_back(result.tok); } // check if there is incomplete UTF-8 character at the end bool incomplete = false; - for (unsigned i = 1; i < 5 && i <= slot.generated_text.size(); ++i) - { + for (unsigned i = 1; i < 5 && i <= slot.generated_text.size(); ++i) { unsigned char c = slot.generated_text[slot.generated_text.size() - i]; - if ((c & 0xC0) == 0x80) - { + if ((c & 0xC0) == 0x80) { // continuation byte: 10xxxxxx continue; } - if ((c & 0xE0) == 0xC0) - { + if ((c & 0xE0) == 0xC0) { // 2-byte character: 110xxxxx ... incomplete = i < 2; - } - else if ((c & 0xF0) == 0xE0) - { + } else if ((c & 0xF0) == 0xE0) { // 3-byte character: 1110xxxx ... incomplete = i < 3; - } - else if ((c & 0xF8) == 0xF0) - { + } else if ((c & 0xF8) == 0xF0) { // 4-byte character: 11110xxx ... incomplete = i < 4; } @@ -969,206 +1075,181 @@ struct llama_server_context break; } - if (!incomplete) - { + if (!incomplete) { size_t pos = std::min(slot.n_sent_text, slot.generated_text.size()); + const std::string str_test = slot.generated_text.substr(pos); bool is_stop_full = false; - size_t stop_pos = find_stopping_strings(str_test, token_str.size(), STOP_FULL, slot); - if (stop_pos != std::string::npos) - { + + size_t stop_pos = slot.find_stopping_strings(str_test, token_str.size(), STOP_TYPE_FULL); + if (stop_pos != std::string::npos) { is_stop_full = true; slot.generated_text.erase( slot.generated_text.begin() + pos + stop_pos, slot.generated_text.end()); pos = std::min(slot.n_sent_text, slot.generated_text.size()); - } - else - { + } else { is_stop_full = false; - stop_pos = find_stopping_strings(str_test, token_str.size(), STOP_PARTIAL, slot); + stop_pos = slot.find_stopping_strings(str_test, token_str.size(), STOP_TYPE_PARTIAL); } // check if there is any token to predict - if (stop_pos == std::string::npos || (!slot.has_next_token && !is_stop_full && stop_pos > 0)) - { + if (stop_pos == std::string::npos || (!slot.has_next_token && !is_stop_full && stop_pos > 0)) { // no send the stop word in the response result.text_to_send = slot.generated_text.substr(pos, std::string::npos); slot.n_sent_text += result.text_to_send.size(); // add the token to slot queue and cache } + slot.add_token_string(result); - if (slot.params.stream) - { + if (slot.params.stream) { send_partial_response(slot, result); } } - if (incomplete) - { + if (incomplete) { slot.has_next_token = true; } // check the limits - if (slot.n_decoded > 0 && slot.has_next_token && !slot.has_budget(params)) - { - slot.stopped_limit = true; + if (slot.n_decoded > 0 && slot.has_next_token && !slot.has_budget(params)) { + slot.stopped_limit = true; slot.has_next_token = false; + + LOG_VERBOSE("stopped by limit", { + {"id_slot", slot.id}, + {"n_decoded", slot.n_decoded}, + {"n_predict", slot.params.n_predict}, + }); } - if (!slot.cache_tokens.empty() && result.tok == llama_token_eos(model)) - { - slot.stopped_eos = true; + if (!slot.cache_tokens.empty() && result.tok == llama_token_eos(model)) { + slot.stopped_eos = true; slot.has_next_token = false; + LOG_VERBOSE("eos token found", {}); } LOG_VERBOSE("next token", { - {"token", result.tok}, - {"token_text", tokens_to_output_formatted_string(ctx, result.tok)}, - {"has_next_token", slot.has_next_token}, - {"n_remain", slot.n_remaining}, - {"num_tokens_predicted", slot.n_decoded}, - {"stopped_eos", slot.stopped_eos}, - {"stopped_word", slot.stopped_word}, - {"stopped_limit", slot.stopped_limit}, - {"stopping_word", slot.stopping_word}, - }); + {"token", result.tok}, + {"token_text", tokens_to_output_formatted_string(ctx, result.tok)}, + {"has_next_token", slot.has_next_token}, + {"n_remain", slot.n_remaining}, + {"n_decoded", slot.n_decoded}, + {"stopped_eos", slot.stopped_eos}, + {"stopped_word", slot.stopped_word}, + {"stopped_limit", slot.stopped_limit}, + {"stopping_word", slot.stopping_word}, + }); return slot.has_next_token; // continue } - bool process_images(server_slot &slot) const - { - for (slot_image &img : slot.images) - { - if (!img.request_encode_image) - { - continue; - } - - if (!llava_image_embed_make_with_clip_img(clp_ctx, params.n_threads, img.img_data, &img.image_embedding, &img.image_tokens)) { - LOG_TEE("Error processing the given image"); - return false; - } - - - img.request_encode_image = false; - } - - return slot.images.size() > 0; - } - - void send_error(task_server& task, const std::string &error) - { - LOG_TEE("task %i - error: %s\n", task.id, error.c_str()); - task_result res; - res.id = task.id; - res.multitask_id = task.multitask_id; - res.stop = false; - res.error = true; - res.result_json = { { "content", error } }; - queue_results.send(res); - } - - json get_formated_generation(server_slot &slot) - { + json get_formated_generation(const server_slot & slot) const { const auto eos_bias = slot.sparams.logit_bias.find(llama_token_eos(model)); - const bool ignore_eos = eos_bias != slot.sparams.logit_bias.end() && - eos_bias->second < 0.0f && std::isinf(eos_bias->second); + const bool ignore_eos = eos_bias != slot.sparams.logit_bias.end() && eos_bias->second < 0.0f && std::isinf(eos_bias->second); + std::vector samplers_sequence; - for (const auto &sampler_type : slot.sparams.samplers_sequence) - { + samplers_sequence.reserve(slot.sparams.samplers_sequence.size()); + for (const auto & sampler_type : slot.sparams.samplers_sequence) { samplers_sequence.emplace_back(sampler_type_to_name_string(sampler_type)); } return json { - {"n_ctx", slot.n_ctx}, - {"n_predict", slot.n_predict}, - {"model", params.model_alias}, - {"seed", slot.params.seed}, - {"temperature", slot.sparams.temp}, - {"dynatemp_range", slot.sparams.dynatemp_range}, - {"dynatemp_exponent", slot.sparams.dynatemp_exponent}, - {"top_k", slot.sparams.top_k}, - {"top_p", slot.sparams.top_p}, - {"min_p", slot.sparams.min_p}, - {"tfs_z", slot.sparams.tfs_z}, - {"typical_p", slot.sparams.typical_p}, - {"repeat_last_n", slot.sparams.penalty_last_n}, - {"repeat_penalty", slot.sparams.penalty_repeat}, - {"presence_penalty", slot.sparams.penalty_present}, - {"frequency_penalty", slot.sparams.penalty_freq}, - {"penalty_prompt_tokens", slot.sparams.penalty_prompt_tokens}, + {"n_ctx", slot.n_ctx}, + {"n_predict", slot.n_predict}, + {"model", params.model_alias}, + {"seed", slot.params.seed}, + {"temperature", slot.sparams.temp}, + {"dynatemp_range", slot.sparams.dynatemp_range}, + {"dynatemp_exponent", slot.sparams.dynatemp_exponent}, + {"top_k", slot.sparams.top_k}, + {"top_p", slot.sparams.top_p}, + {"min_p", slot.sparams.min_p}, + {"tfs_z", slot.sparams.tfs_z}, + {"typical_p", slot.sparams.typical_p}, + {"repeat_last_n", slot.sparams.penalty_last_n}, + {"repeat_penalty", slot.sparams.penalty_repeat}, + {"presence_penalty", slot.sparams.penalty_present}, + {"frequency_penalty", slot.sparams.penalty_freq}, + {"penalty_prompt_tokens", slot.sparams.penalty_prompt_tokens}, {"use_penalty_prompt_tokens", slot.sparams.use_penalty_prompt_tokens}, - {"mirostat", slot.sparams.mirostat}, - {"mirostat_tau", slot.sparams.mirostat_tau}, - {"mirostat_eta", slot.sparams.mirostat_eta}, - {"penalize_nl", slot.sparams.penalize_nl}, - {"stop", slot.params.antiprompt}, - {"n_predict", slot.params.n_predict}, - {"n_keep", params.n_keep}, - {"ignore_eos", ignore_eos}, - {"stream", slot.params.stream}, - {"logit_bias", slot.sparams.logit_bias}, - {"n_probs", slot.sparams.n_probs}, - {"min_keep", slot.sparams.min_keep}, - {"grammar", slot.sparams.grammar}, - {"samplers", samplers_sequence} + {"mirostat", slot.sparams.mirostat}, + {"mirostat_tau", slot.sparams.mirostat_tau}, + {"mirostat_eta", slot.sparams.mirostat_eta}, + {"penalize_nl", slot.sparams.penalize_nl}, + {"stop", slot.params.antiprompt}, + {"n_predict", slot.params.n_predict}, + {"n_keep", params.n_keep}, + {"ignore_eos", ignore_eos}, + {"stream", slot.params.stream}, + {"logit_bias", slot.sparams.logit_bias}, + {"n_probs", slot.sparams.n_probs}, + {"min_keep", slot.sparams.min_keep}, + {"grammar", slot.sparams.grammar}, + {"samplers", samplers_sequence} }; } - void send_partial_response(server_slot &slot, completion_token_output tkn) - { - task_result res; - res.id = slot.task_id; - res.multitask_id = slot.multitask_id; - res.error = false; - res.stop = false; + void send_error(const server_task & task, const std::string & error) { + LOG_TEE("task %i - error: %s\n", task.id, error.c_str()); - res.result_json = json - { + server_task_result res; + res.id = task.id; + res.id_multi = task.id_multi; + res.stop = false; + res.error = true; + res.data = { { "content", error } }; + + queue_results.send(res); + } + + void send_partial_response(server_slot & slot, completion_token_output tkn) { + server_task_result res; + res.id = slot.id_task; + res.id_multi = slot.id_multi; + res.error = false; + res.stop = false; + res.data = json { {"content", tkn.text_to_send}, {"stop", false}, - {"slot_id", slot.id}, - {"multimodal", multimodal} + {"id_slot", slot.id}, + {"multimodal", false} }; - if (slot.sparams.n_probs > 0) - { - std::vector probs_output = {}; + if (slot.sparams.n_probs > 0) { const std::vector to_send_toks = llama_tokenize(ctx, tkn.text_to_send, false); - size_t probs_pos = std::min(slot.n_sent_token_probs, slot.generated_token_probs.size()); - size_t probs_stop_pos = std::min(slot.n_sent_token_probs + to_send_toks.size(), slot.generated_token_probs.size()); - if (probs_pos < probs_stop_pos) - { - probs_output = std::vector(slot.generated_token_probs.begin() + probs_pos, slot.generated_token_probs.begin() + probs_stop_pos); + const size_t probs_pos = std::min(slot.n_sent_token_probs, slot.generated_token_probs.size()); + const size_t probs_stop_pos = std::min(slot.n_sent_token_probs + to_send_toks.size(), slot.generated_token_probs.size()); + + std::vector probs_output; + if (probs_pos < probs_stop_pos) { + probs_output = std::vector( + slot.generated_token_probs.begin() + probs_pos, + slot.generated_token_probs.begin() + probs_stop_pos); } slot.n_sent_token_probs = probs_stop_pos; - res.result_json["completion_probabilities"] = probs_vector_to_json(ctx, probs_output); + + res.data["completion_probabilities"] = probs_vector_to_json(ctx, probs_output); } - if (slot.oaicompat) - { - res.result_json["oaicompat_token_ctr"] = slot.n_decoded; - res.result_json["model"] = slot.oaicompat_model; + if (slot.oaicompat) { + res.data["oaicompat_token_ctr"] = slot.n_decoded; + res.data["model"] = slot.oaicompat_model; } queue_results.send(res); } - void send_final_response(server_slot &slot) - { - task_result res; - res.id = slot.task_id; - res.multitask_id = slot.multitask_id; - res.error = false; - res.stop = true; - - res.result_json = json - { + void send_final_response(const server_slot & slot) { + server_task_result res; + res.id = slot.id_task; + res.id_multi = slot.id_multi; + res.error = false; + res.stop = true; + res.data = json { {"content", !slot.params.stream ? slot.generated_text : ""}, - {"slot_id", slot.id}, + {"id_slot", slot.id}, {"stop", true}, {"model", params.model_alias}, {"tokens_predicted", slot.n_decoded}, @@ -1184,96 +1265,87 @@ struct llama_server_context {"timings", slot.get_formated_timings()} }; - if (slot.sparams.n_probs > 0) - { - std::vector probs = {}; - if (!slot.params.stream && slot.stopped_word) - { + if (slot.sparams.n_probs > 0) { + std::vector probs; + if (!slot.params.stream && slot.stopped_word) { const std::vector stop_word_toks = llama_tokenize(ctx, slot.stopping_word, false); - probs = std::vector(slot.generated_token_probs.begin(), slot.generated_token_probs.end() - stop_word_toks.size()); - } - else - { + probs = std::vector( - slot.generated_token_probs.begin(), - slot.generated_token_probs.end()); + slot.generated_token_probs.begin(), + slot.generated_token_probs.end() - stop_word_toks.size()); + } else { + probs = std::vector( + slot.generated_token_probs.begin(), + slot.generated_token_probs.end()); } - res.result_json["completion_probabilities"] = probs_vector_to_json(ctx, probs); + + res.data["completion_probabilities"] = probs_vector_to_json(ctx, probs); } - if (slot.oaicompat) - { - res.result_json["oaicompat_token_ctr"] = slot.n_decoded; - res.result_json["model"] = slot.oaicompat_model; + if (slot.oaicompat) { + res.data["oaicompat_token_ctr"] = slot.n_decoded; + res.data["model"] = slot.oaicompat_model; } queue_results.send(res); } - void send_embedding(server_slot & slot, const llama_batch & batch) - { - task_result res; - res.id = slot.task_id; - res.multitask_id = slot.multitask_id; - res.error = false; - res.stop = true; + void send_embedding(const server_slot & slot, const llama_batch & batch) { + server_task_result res; + res.id = slot.id_task; + res.id_multi = slot.id_multi; + res.error = false; + res.stop = true; const int n_embd = llama_n_embd(model); - if (!params.embedding) - { - LOG_WARNING("embedding disabled", {{"params.embedding", params.embedding}}); - res.result_json = json - { - {"embedding", std::vector(n_embd, 0.0f)}, + for (int i = 0; i < batch.n_tokens; ++i) { + if (!batch.logits[i] || batch.seq_id[i][0] != slot.id) { + continue; + } + + const float * embd = llama_get_embeddings_seq(ctx, batch.seq_id[i][0]); + if (embd == NULL) { + embd = llama_get_embeddings_ith(ctx, i); + } + + if (embd == NULL) { + LOG_ERROR("failed to get embeddings", { + {"token", batch.token [i]}, + {"seq_id", batch.seq_id[i][0]} + }); + + res.data = json { + {"embedding", std::vector(n_embd, 0.0f)}, + }; + + continue; + } + + res.data = json { + {"embedding", std::vector(embd, embd + n_embd)}, }; } - else - { - for (int i = 0; i < batch.n_tokens; ++i) { - if (!batch.logits[i] || batch.seq_id[i][0] != slot.id) { - continue; - } - const float * embd = llama_get_embeddings_seq(ctx, batch.seq_id[i][0]); - if (embd == NULL) { - embd = llama_get_embeddings_ith(ctx, i); - if (embd == NULL) { - LOG_ERROR("failed to get embeddings for token", {{"token", batch.token[i]}, {"seq_id", batch.seq_id[i][0]}}); - res.result_json = json - { - {"embedding", std::vector(n_embd, 0.0f)}, - }; - continue; - } - } - - res.result_json = json - { - {"embedding", std::vector(embd, embd + n_embd)}, - }; - } - } queue_results.send(res); } - void request_completion(int task_id, json data, bool infill, bool embedding, int multitask_id) - { - task_server task; - task.id = task_id; - task.target_id = 0; - task.data = std::move(data); - task.infill_mode = infill; - task.embedding_mode = embedding; - task.type = TASK_TYPE_COMPLETION; - task.multitask_id = multitask_id; + void request_completion(int id_task, int id_multi, json data, bool infill, bool embedding) { + server_task task; + task.id = id_task; + task.id_multi = id_multi; + task.id_target = 0; + task.data = std::move(data); + task.infill = infill; + task.embedding = embedding; + task.type = SERVER_TASK_TYPE_COMPLETION; // when a completion task's prompt array is not a singleton, we split it into multiple requests // otherwise, it's a single-prompt task, we actually queue it // if there's numbers in the prompt array it will be treated as an array of tokens if (task.data.count("prompt") != 0 && task.data.at("prompt").size() > 1) { bool numbers = false; - for (const auto& e : task.data.at("prompt")) { + for (const auto & e : task.data.at("prompt")) { if (e.is_number()) { numbers = true; break; @@ -1288,106 +1360,23 @@ struct llama_server_context if (numbers) { queue_tasks.post(task); } else { - split_multiprompt_task(task_id, task); + split_multiprompt_task(id_task, task); } } else { - // an empty prompt can make slot become buggy - if (task.data.contains("prompt") && task.data["prompt"].is_string() && task.data["prompt"].get().empty()) { - task.data["prompt"] = " "; // add a space so that we have one token - } queue_tasks.post(task); } } - // for multiple images processing - bool ingest_images(server_slot &slot, int n_batch) - { - int image_idx = 0; + void request_cancel(int id_task) { + server_task task; + task.type = SERVER_TASK_TYPE_CANCEL; + task.id_target = id_task; - while (image_idx < (int) slot.images.size()) - { - slot_image &img = slot.images[image_idx]; - - // process prefix prompt - for (int32_t i = 0; i < (int32_t) batch.n_tokens; i += n_batch) - { - const int32_t n_tokens = std::min(n_batch, (int32_t) (batch.n_tokens - i)); - llama_batch batch_view = { - n_tokens, - batch.token + i, - nullptr, - batch.pos + i, - batch.n_seq_id + i, - batch.seq_id + i, - batch.logits + i, - 0, 0, 0, // unused - }; - if (llama_decode(ctx, batch_view)) - { - LOG_TEE("%s : failed to eval\n", __func__); - return false; - } - } - - // process image with llm - for (int i = 0; i < img.image_tokens; i += n_batch) - { - int n_eval = img.image_tokens - i; - if (n_eval > n_batch) - { - n_eval = n_batch; - } - - const int n_embd = llama_n_embd(model); - llama_batch batch_img = { - n_eval, - nullptr, - (img.image_embedding + i * n_embd), - nullptr, - nullptr, - nullptr, - nullptr, - slot.n_past, - 1, 0 - }; - if (llama_decode(ctx, batch_img)) - { - LOG_TEE("%s : failed to eval image\n", __func__); - return false; - } - slot.n_past += n_eval; - } - image_idx++; - - llama_batch_clear(batch); - - // append prefix of next image - const auto json_prompt = (image_idx >= (int) slot.images.size()) ? - slot.params.input_suffix : // no more images, then process suffix prompt - (json)(slot.images[image_idx].prefix_prompt); - - std::vector append_tokens = tokenize(json_prompt, false); // has next image - for (int i = 0; i < (int) append_tokens.size(); ++i) - { - llama_batch_add(batch, append_tokens[i], system_tokens.size() + slot.n_past, { slot.id }, true); - slot.n_past += 1; - } - } - - return true; - } - - void request_cancel(int task_id) - { - task_server task; - task.type = TASK_TYPE_CANCEL; - task.target_id = task_id; queue_tasks.post(task); } - void split_multiprompt_task(int multitask_id, task_server& multiprompt_task) - { - int prompt_count = multiprompt_task.data.at("prompt").size(); + void split_multiprompt_task(int id_multi, const server_task & multiprompt_task) { + const int prompt_count = multiprompt_task.data.at("prompt").size(); if (prompt_count <= 1) { send_error(multiprompt_task, "error while handling multiple prompts"); return; @@ -1395,127 +1384,121 @@ struct llama_server_context // generate all the ID for subtask std::vector subtask_ids(prompt_count); - for (int i = 0; i < prompt_count; i++) - { + for (int i = 0; i < prompt_count; i++) { subtask_ids[i] = queue_tasks.get_new_id(); } // queue up the multitask so we can track its subtask progression - queue_tasks.add_multitask(multitask_id, subtask_ids); + queue_tasks.add_multitask(id_multi, subtask_ids); // add subtasks - for (int i = 0; i < prompt_count; i++) - { + for (int i = 0; i < prompt_count; i++) { json subtask_data = multiprompt_task.data; subtask_data["prompt"] = subtask_data["prompt"][i]; // subtasks inherit everything else (infill mode, embedding mode, etc.) - request_completion(subtask_ids[i], subtask_data, multiprompt_task.infill_mode, multiprompt_task.embedding_mode, multitask_id); + request_completion(subtask_ids[i], id_multi, subtask_data, multiprompt_task.infill, multiprompt_task.embedding); } } - void process_single_task(task_server& task) - { - switch (task.type) - { - case TASK_TYPE_COMPLETION: { - server_slot *slot = get_slot(json_value(task.data, "slot_id", -1)); - if (slot == nullptr) + void process_single_task(const server_task & task) { + switch (task.type) { + case SERVER_TASK_TYPE_COMPLETION: { - // if no slot is available, we defer this task for processing later - LOG_VERBOSE("no slot is available", {{"task_id", task.id}}); - queue_tasks.defer(task); - break; - } - - if (task.data.contains("system_prompt")) - { - if (!all_slots_are_idle) { - send_error(task, "system prompt can only be updated when all slots are idle"); + server_slot * slot = get_slot(json_value(task.data, "id_slot", -1)); + if (slot == nullptr) { + // if no slot is available, we defer this task for processing later + LOG_VERBOSE("no slot is available", {{"id_task", task.id}}); + queue_tasks.defer(task); break; } - system_prompt_process(task.data["system_prompt"]); - // reset cache_tokens for all slots - for (server_slot &slot : slots) - { - slot.cache_tokens.clear(); - slot.n_past = 0; - slot.n_past_se = 0; + if (task.data.contains("system_prompt")) { + system_prompt_set(task.data["system_prompt"]); + + for (server_slot & slot : slots) { + slot.n_past = 0; + slot.n_past_se = 0; + } } - } - slot->reset(); + slot->reset(); - slot->infill = task.infill_mode; - slot->embedding = task.embedding_mode; - slot->task_id = task.id; - slot->multitask_id = task.multitask_id; + slot->id_task = task.id; + slot->id_multi = task.id_multi; + slot->infill = task.infill; + slot->embedding = task.embedding; - if (!launch_slot_with_data(slot, task.data)) - { - // send error result - send_error(task, "internal_error"); - break; - } - } break; - case TASK_TYPE_CANCEL: { // release slot linked with the task id - for (auto & slot : slots) - { - if (slot.task_id == task.target_id) - { - slot.release(); + if (!launch_slot_with_data(*slot, task.data)) { + // send error result + send_error(task, "internal_error"); break; } - } - } break; - case TASK_TYPE_NEXT_RESPONSE: { - // do nothing - } break; - case TASK_TYPE_METRICS: { - json slots_data = json::array(); - int n_idle_slots = 0; - int n_processing_slots = 0; - - for (server_slot &slot: slots) { - json slot_data = get_formated_generation(slot); - slot_data["id"] = slot.id; - slot_data["task_id"] = slot.task_id; - slot_data["state"] = slot.state; - slot_data["prompt"] = slot.prompt; - slot_data["next_token"] = { - {"has_next_token", slot.has_next_token}, - {"n_remain", slot.n_remaining}, - {"num_tokens_predicted", slot.n_decoded}, - {"stopped_eos", slot.stopped_eos}, - {"stopped_word", slot.stopped_word}, - {"stopped_limit", slot.stopped_limit}, - {"stopping_word", slot.stopping_word}, - }; - if (slot_data["state"] == IDLE) { - n_idle_slots++; - } else { - n_processing_slots++; + } break; + case SERVER_TASK_TYPE_CANCEL: + { + // release slot linked with the task id + for (auto & slot : slots) { + if (slot.id_task == task.id_target) { + slot.release(); + break; + } } - slots_data.push_back(slot_data); - } - LOG_INFO("slot data", { - {"task_id", task.id}, - {"n_idle_slots", n_idle_slots}, - {"n_processing_slots", n_processing_slots} - }); - LOG_VERBOSE("slot data", { - {"task_id", task.id}, - {"n_idle_slots", n_idle_slots}, - {"n_processing_slots", n_processing_slots}, - {"slots", slots_data} - }); - task_result res; - res.id = task.id; - res.multitask_id = task.multitask_id; - res.stop = true; - res.error = false; - res.result_json = { + } break; + case SERVER_TASK_TYPE_NEXT_RESPONSE: + { + // do nothing + } break; + case SERVER_TASK_TYPE_METRICS: + { + json slots_data = json::array(); + + int n_idle_slots = 0; + int n_processing_slots = 0; + + for (server_slot & slot : slots) { + json slot_data = get_formated_generation(slot); + slot_data["id"] = slot.id; + slot_data["id_task"] = slot.id_task; + slot_data["state"] = slot.state; + slot_data["prompt"] = slot.prompt; + slot_data["next_token"] = { + {"has_next_token", slot.has_next_token}, + {"n_remain", slot.n_remaining}, + {"n_decoded", slot.n_decoded}, + {"stopped_eos", slot.stopped_eos}, + {"stopped_word", slot.stopped_word}, + {"stopped_limit", slot.stopped_limit}, + {"stopping_word", slot.stopping_word}, + }; + + if (slot_data["state"] == SLOT_STATE_IDLE) { + n_idle_slots++; + } else { + n_processing_slots++; + } + + slots_data.push_back(slot_data); + } + LOG_INFO("slot data", { + {"id_task", task.id}, + {"n_idle_slots", n_idle_slots}, + {"n_processing_slots", n_processing_slots} + }); + + LOG_VERBOSE("slot data", { + {"id_task", task.id}, + {"n_idle_slots", n_idle_slots}, + {"n_processing_slots", n_processing_slots}, + {"slots", slots_data} + }); + + server_task_result res; + res.id = task.id; + res.id_multi = task.id_multi; + res.stop = true; + res.error = false; + res.data = { { "idle", n_idle_slots }, { "processing", n_processing_slots }, { "deferred", queue_tasks.queue_tasks_deferred.size() }, @@ -1532,71 +1515,104 @@ struct llama_server_context { "kv_cache_used_cells", llama_get_kv_cache_used_cells(ctx)}, { "slots", slots_data }, - }; - metrics.reset_bucket(); - queue_results.send(res); - } break; + }; + + metrics.reset_bucket(); + queue_results.send(res); + } break; } } - void on_finish_multitask(task_multi& multitask) - { + void on_finish_multitask(const server_task_multi & multitask) { // all subtasks done == multitask is done - task_result result; - result.id = multitask.id; - result.stop = true; + server_task_result result; + result.id = multitask.id; + result.stop = true; result.error = false; // collect json results into one json result std::vector result_jsons; - for (auto& subres : multitask.results) - { - result_jsons.push_back(subres.result_json); + for (const auto & subres : multitask.results) { + result_jsons.push_back(subres.data); result.error = result.error && subres.error; } - result.result_json = json{ { "results", result_jsons } }; + result.data = json { + { "results", result_jsons } + }; + queue_results.send(result); } bool update_slots() { - if (system_need_update) - { - LOG_INFO("updating system prompt", {}); + if (system_need_update) { system_prompt_update(); } - llama_batch_clear(batch); + // release slots + for (auto & slot : slots) { + if (slot.command == SLOT_COMMAND_RELEASE) { + slot.state = SLOT_STATE_IDLE; + slot.command = SLOT_COMMAND_NONE; + slot.t_last_used = ggml_time_us(); - if (all_slots_are_idle) - { - if (system_prompt.empty() && clean_kv_cache) - { - LOG_INFO("all slots are idle and system prompt is empty, clear the KV cache", {}); - kv_cache_clear(); + LOG_INFO("slot released", { + {"id_slot", slot.id}, + {"id_task", slot.id_task}, + {"n_ctx", n_ctx}, + {"n_past", slot.n_past}, + {"n_system_tokens", system_tokens.size()}, + {"n_cache_tokens", slot.cache_tokens.size()}, + {"truncated", slot.truncated} + }); + + queue_tasks.notify_slot_changed(); } - return true; } - LOG_VERBOSE("posting NEXT_RESPONSE", {}); - task_server task; - task.type = TASK_TYPE_NEXT_RESPONSE; - task.target_id = -1; - queue_tasks.post(task); - - for (server_slot &slot : slots) + // check if all slots are idle { - if (slot.ga_n == 1) - { - if (slot.is_processing() && system_tokens.size() + slot.cache_tokens.size() >= (size_t) slot.n_ctx) - { + bool all_idle = true; + + for (auto & slot : slots) { + if (slot.state != SLOT_STATE_IDLE || slot.command != SLOT_COMMAND_NONE) { + all_idle = false; + break; + } + } + + if (all_idle) { + LOG_INFO("all slots are idle", {}); + if (system_prompt.empty() && clean_kv_cache) { + kv_cache_clear(); + } + + return true; + } + } + + { + LOG_VERBOSE("posting NEXT_RESPONSE", {}); + + server_task task; + task.type = SERVER_TASK_TYPE_NEXT_RESPONSE; + task.id_target = -1; + + queue_tasks.post(task); + } + + // apply context-shift if needed + // TODO: simplify and improve + for (server_slot & slot : slots) { + if (slot.ga_n == 1) { + if (slot.is_processing() && (int) system_tokens.size() + slot.n_past >= slot.n_ctx - 1) { // Shift context const int n_keep = slot.params.n_keep + add_bos_token; const int n_left = (int) system_tokens.size() + slot.n_past - n_keep; const int n_discard = n_left / 2; LOG_INFO("slot context shift", { - {"slot_id", slot.id}, - {"task_id", slot.task_id}, + {"id_slot", slot.id}, + {"id_task", slot.id_task}, {"n_keep", n_keep}, {"n_left", n_left}, {"n_discard", n_discard}, @@ -1605,15 +1621,17 @@ struct llama_server_context {"n_system_tokens", system_tokens.size()}, {"n_cache_tokens", slot.cache_tokens.size()} }); + llama_kv_cache_seq_rm (ctx, slot.id, n_keep , n_keep + n_discard); llama_kv_cache_seq_add(ctx, slot.id, n_keep + n_discard, system_tokens.size() + slot.n_past, -n_discard); - for (size_t i = n_keep + n_discard; i < slot.cache_tokens.size(); i++) - { - slot.cache_tokens[i - n_discard] = slot.cache_tokens[i]; - } + if (slot.params.cache_prompt) { + for (size_t i = n_keep + n_discard; i < slot.cache_tokens.size(); i++) { + slot.cache_tokens[i - n_discard] = slot.cache_tokens[i]; + } - slot.cache_tokens.resize(slot.cache_tokens.size() - n_discard); + slot.cache_tokens.resize(slot.cache_tokens.size() - n_discard); + } slot.n_past -= n_discard; @@ -1622,33 +1640,12 @@ struct llama_server_context } } - // decode any currently ongoing sequences - LOG_VERBOSE("decoding ongoing sequences", {}); - for (auto & slot : slots) - { - // release the slot - if (slot.command == RELEASE) - { - slot.state = IDLE; - slot.command = NONE; - slot.t_last_used = ggml_time_us(); + // start populating the batch for this iteration + llama_batch_clear(batch); - LOG_INFO("slot released", { - {"slot_id", slot.id}, - {"task_id", slot.task_id}, - {"n_ctx", n_ctx}, - {"n_past", slot.n_past}, - {"n_system_tokens", system_tokens.size()}, - {"n_cache_tokens", slot.cache_tokens.size()}, - {"truncated", slot.truncated} - }); - queue_tasks.notify_slot_changed(); - - continue; - } - - if (slot.state == IDLE) - { + // frist, add sampled tokens from any ongoing sequences + for (auto & slot : slots) { + if (slot.state == SLOT_STATE_IDLE) { continue; } @@ -1659,193 +1656,184 @@ struct llama_server_context // TODO: we always have to take into account the "system_tokens" // this is not great and needs to be improved somehow llama_batch_add(batch, slot.sampled, system_tokens.size() + slot_npast, { slot.id }, true); + slot.n_past += 1; + + if (slot.params.cache_prompt) { + slot.cache_tokens.push_back(slot.sampled); + } + + LOG_VERBOSE("slot decode token", { + {"id_slot", slot.id}, + {"id_task", slot.id_task}, + {"n_ctx", n_ctx}, + {"n_past", slot.n_past}, + {"n_system_tokens", system_tokens.size()}, + {"n_cache_tokens", slot.cache_tokens.size()}, + {"truncated", slot.truncated} + }); } // process in chunks of params.n_batch int32_t n_batch = params.n_batch; - // assign workload to the slots - if (params.cont_batching || batch.n_tokens == 0) - { - for (auto & slot : slots) - { - const bool has_prompt = slot.prompt.is_array() || (slot.prompt.is_string() && !slot.prompt.get().empty()) || !slot.images.empty(); + // next, batch any pending prompts without exceeding n_batch + if (params.cont_batching || batch.n_tokens == 0) { + for (auto & slot : slots) { + const bool has_prompt = slot.prompt.is_array() || (slot.prompt.is_string() && !slot.prompt.get().empty()); // empty prompt passed -> release the slot and send empty response // note: infill mode allows empty prompt - if (slot.state == IDLE && slot.command == LOAD_PROMPT && !has_prompt && !slot.infill) - { + if (slot.state == SLOT_STATE_IDLE && slot.command == SLOT_COMMAND_LOAD_PROMPT && !has_prompt && !slot.infill) { + slot.state = SLOT_STATE_PROCESSING; + slot.command = SLOT_COMMAND_NONE; slot.release(); slot.print_timings(); send_final_response(slot); continue; } - // need process the prompt - if (slot.state == IDLE && slot.command == LOAD_PROMPT) - { - slot.state = PROCESSING; - slot.command = NONE; - std::vector prompt_tokens; - slot.t_start_process_prompt = ggml_time_us(); - slot.t_start_genereration = 0; + // this slot still has a prompt to be processed + if (slot.state == SLOT_STATE_IDLE && slot.command == SLOT_COMMAND_LOAD_PROMPT) { + auto & prompt_tokens = slot.prompt_tokens; - if (slot.infill) - { - bool suff_rm_leading_spc = true; - if (params.input_suffix.find_first_of(' ') == 0 && params.input_suffix.size() > 1) - { - params.input_suffix.erase(0, 1); - suff_rm_leading_spc = false; - } - auto prefix_tokens = tokenize(slot.params.input_prefix, false); - auto suffix_tokens = tokenize(slot.params.input_suffix, false); - - const int space_token = 29871; // TODO: this should not be hardcoded - if (suff_rm_leading_spc && !suffix_tokens.empty() && suffix_tokens[0] == space_token) { - suffix_tokens.erase(suffix_tokens.begin()); - } - - prefix_tokens.insert(prefix_tokens.begin(), llama_token_prefix(model)); - prefix_tokens.insert(prefix_tokens.begin(), llama_token_bos(model)); // always add BOS - prefix_tokens.insert(prefix_tokens.end(), llama_token_suffix(model)); - prefix_tokens.insert(prefix_tokens.end(), suffix_tokens.begin(), suffix_tokens.end()); - prefix_tokens.push_back(llama_token_middle(model)); - prompt_tokens = prefix_tokens; - } - else - { - prompt_tokens = tokenize(slot.prompt, system_prompt.empty() && add_bos_token); // add BOS if there isn't system prompt - } - - slot.n_prompt_tokens = prompt_tokens.size(); - - if (slot.params.n_keep < 0) - { - slot.params.n_keep = slot.n_prompt_tokens; - } - slot.params.n_keep = std::min(slot.n_ctx - 4, slot.params.n_keep); - - // if input prompt is too big, truncate it, if group attention self-extend is disabled - if (slot.ga_n == 1 && slot.n_prompt_tokens >= slot.n_ctx) - { - const int n_left = slot.n_ctx - slot.params.n_keep; - const int n_block_size = n_left / 2; - const int erased_blocks = (slot.n_prompt_tokens - slot.params.n_keep - n_block_size) / n_block_size; - - std::vector new_tokens( - prompt_tokens.begin(), - prompt_tokens.begin() + slot.params.n_keep); - new_tokens.insert( - new_tokens.end(), - prompt_tokens.begin() + slot.params.n_keep + erased_blocks * n_block_size, - prompt_tokens.end()); - - LOG_VERBOSE("input truncated", { - {"n_ctx", slot.n_ctx}, - {"n_keep", slot.params.n_keep}, - {"n_left", n_left}, - {"new_tokens", tokens_to_str(ctx, new_tokens.cbegin(), new_tokens.cend())}, + // we haven't tokenized the prompt yet - do it now: + if (prompt_tokens.empty()) { + LOG_VERBOSE("tokenizing prompt", { + {"id_slot", slot.id}, + {"id_task", slot.id_task} }); - slot.truncated = true; - prompt_tokens = new_tokens; - slot.n_prompt_tokens = prompt_tokens.size(); - GGML_ASSERT(slot.n_prompt_tokens < slot.n_ctx); - } + slot.t_start_process_prompt = ggml_time_us(); + slot.t_start_generation = 0; - if (!slot.params.cache_prompt) - { - llama_sampling_reset(slot.ctx_sampling); - - slot.n_past = 0; - slot.n_past_se = 0; - slot.ga_i = 0; - slot.n_prompt_tokens_processed = slot.n_prompt_tokens; - } - else - { - // push the prompt into the sampling context (do not apply grammar) - for (auto &token : prompt_tokens) - { - llama_sampling_accept(slot.ctx_sampling, ctx, token, false); - } - - slot.n_past = common_part(slot.cache_tokens, prompt_tokens); - - // the last token of the cache is not in the KV cache until the next call to llama_decode - // (it was sampled, pushed into the "cache_tokens", but not yet put in the context) - if (slot.n_past > 0 && slot.n_past == (int32_t) slot.cache_tokens.size()) - { - slot.n_past -= 1; - } - - slot.n_prompt_tokens_processed = slot.n_prompt_tokens - slot.n_past; - - if (slot.ga_n != 1) - { - int ga_i = 0; - int32_t ga_n = slot.ga_n; - int32_t ga_w = slot.ga_w; - int32_t slot_npast = 0; - for (int k = 0; k < slot.n_past; ++k) - { - while (slot_npast >= ga_i + ga_w) { - const int bd = (ga_w/ga_n)*(ga_n - 1); - slot_npast -= bd; - ga_i += ga_w/ga_n; - } - slot_npast++; + if (slot.infill) { + bool suff_rm_leading_spc = true; + if (params.input_suffix.find_first_of(' ') == 0 && params.input_suffix.size() > 1) { + params.input_suffix.erase(0, 1); + suff_rm_leading_spc = false; } - slot.n_past_se = slot_npast; - slot.ga_i = ga_i; + + auto prefix_tokens = tokenize(slot.params.input_prefix, false); + auto suffix_tokens = tokenize(slot.params.input_suffix, false); + + const int space_token = 29871; // TODO: this should not be hardcoded + if (suff_rm_leading_spc && !suffix_tokens.empty() && suffix_tokens[0] == space_token) { + suffix_tokens.erase(suffix_tokens.begin()); + } + + prefix_tokens.insert(prefix_tokens.begin(), llama_token_prefix(model)); + prefix_tokens.insert(prefix_tokens.begin(), llama_token_bos(model)); // always add BOS + prefix_tokens.insert(prefix_tokens.end(), llama_token_suffix(model)); + prefix_tokens.insert(prefix_tokens.end(), suffix_tokens.begin(), suffix_tokens.end()); + prefix_tokens.push_back(llama_token_middle(model)); + prompt_tokens = prefix_tokens; + } else { + prompt_tokens = tokenize(slot.prompt, system_prompt.empty() && add_bos_token); // add BOS if there isn't system prompt } - LOG_INFO("slot progression", { - { "slot_id", slot.id }, - { "task_id", slot.task_id }, - { "n_past", slot.n_past }, - { "n_past_se", slot.n_past_se }, - { "ga_i", slot.ga_i }, - { "n_prompt_tokens_processed", slot.n_prompt_tokens_processed } - }); + slot.n_past = 0; + slot.n_prompt_tokens = prompt_tokens.size(); + + if (slot.embedding) { + // this prompt is too large to process - discard it + if (slot.n_prompt_tokens > n_batch) { + slot.state = SLOT_STATE_PROCESSING; + slot.command = SLOT_COMMAND_NONE; + slot.release(); + slot.print_timings(); + send_final_response(slot); + continue; + } + } else { + if (slot.params.n_keep < 0) { + slot.params.n_keep = slot.n_prompt_tokens; + } + slot.params.n_keep = std::min(slot.n_ctx - 4, slot.params.n_keep); + + // if input prompt is too big, truncate it (if group attention self-extend is disabled) + if (slot.ga_n == 1 && slot.n_prompt_tokens >= slot.n_ctx) { + const int n_left = slot.n_ctx - slot.params.n_keep; + + const int n_block_size = n_left / 2; + const int erased_blocks = (slot.n_prompt_tokens - slot.params.n_keep - n_block_size) / n_block_size; + + std::vector new_tokens( + prompt_tokens.begin(), + prompt_tokens.begin() + slot.params.n_keep); + + new_tokens.insert( + new_tokens.end(), + prompt_tokens.begin() + slot.params.n_keep + erased_blocks * n_block_size, + prompt_tokens.end()); + + prompt_tokens = std::move(new_tokens); + + slot.truncated = true; + slot.n_prompt_tokens = prompt_tokens.size(); + + LOG_VERBOSE("input truncated", { + {"n_ctx", slot.n_ctx}, + {"n_keep", slot.params.n_keep}, + {"n_left", n_left}, + {"prompt_tokens", tokens_to_str(ctx, prompt_tokens.cbegin(), prompt_tokens.cend())}, + }); + + GGML_ASSERT(slot.n_prompt_tokens < slot.n_ctx); + } + + llama_sampling_reset(slot.ctx_sampling); + + if (!slot.params.cache_prompt) { + slot.n_past_se = 0; + slot.ga_i = 0; + } else { + GGML_ASSERT(slot.ga_n == 1); + + // reuse any previously computed tokens that are common with the new prompt + slot.n_past = common_part(slot.cache_tokens, prompt_tokens); + + // remove the non-common part from the cache + slot.cache_tokens.resize(slot.n_past); + + // push the prompt into the sampling context (do not apply grammar) + for (int i = 0; i < slot.n_past; ++i) { + llama_sampling_accept(slot.ctx_sampling, ctx, slot.cache_tokens[i], false); + } + } + } + + if (slot.n_past == slot.n_prompt_tokens && slot.n_past > 0) { + // we have to evaluate at least 1 token to generate logits. + LOG_INFO("we have to evaluate at least 1 token to generate logits", { + { "id_slot", slot.id }, + { "id_task", slot.id_task } + }); + + slot.n_past--; + if (slot.ga_i > 0) { + slot.n_past_se--; + } + } + + slot.n_prompt_tokens_processed = 0; } - slot.cache_tokens = prompt_tokens; - - if (slot.n_past == slot.n_prompt_tokens && slot.n_past > 0) - { - // we have to evaluate at least 1 token to generate logits. - LOG_INFO("we have to evaluate at least 1 token to generate logits", { - { "slot_id", slot.id }, - { "task_id", slot.task_id } - }); - slot.n_past--; - if (slot.ga_i > 0) - { - slot.n_past_se--; + if (slot.embedding) { + // cannot fit the prompt in the current batch - will try next iter + if (batch.n_tokens + slot.n_prompt_tokens > n_batch) { + continue; } } - int p0 = (int) system_tokens.size() + slot.n_past; - LOG_INFO("kv cache rm [p0, end)", { - { "slot_id", slot.id }, - { "task_id", slot.task_id }, - { "p0", p0 } - }); + const int p0 = (int) system_tokens.size() + slot.n_past; llama_kv_cache_seq_rm(ctx, slot.id, p0, -1); - LOG_VERBOSE("prompt ingested", { - {"n_past", slot.n_past}, - {"cached", tokens_to_str(ctx, slot.cache_tokens.cbegin(), slot.cache_tokens.cbegin() + slot.n_past)}, - {"to_eval", tokens_to_str(ctx, slot.cache_tokens.cbegin() + slot.n_past, slot.cache_tokens.cend())}, - }); - - const bool has_images = process_images(slot); - - // process the prefix of first image - std::vector prefix_tokens = has_images ? tokenize(slot.images[0].prefix_prompt, add_bos_token) : prompt_tokens; + LOG_INFO("kv cache rm [p0, end)", { + { "id_slot", slot.id }, + { "id_task", slot.id_task }, + { "p0", p0 } + }); int32_t slot_npast = slot.n_past_se > 0 ? slot.n_past_se : slot.n_past; @@ -1853,61 +1841,82 @@ struct llama_server_context int32_t ga_n = slot.ga_n; int32_t ga_w = slot.ga_w; - for (; slot.n_past < (int) prefix_tokens.size(); ++slot.n_past) - { - if (slot.ga_n != 1) - { + // add prompt tokens for processing in the current batch + // TODO: the self-extend stuff here is a mess - simplify and/or abstract it somehow + for (; slot.n_past < slot.n_prompt_tokens && batch.n_tokens < n_batch; ++slot.n_past) { + if (slot.ga_n != 1) { while (slot_npast >= ga_i + ga_w) { const int bd = (ga_w/ga_n)*(ga_n - 1); slot_npast -= bd; ga_i += ga_w/ga_n; } } - llama_batch_add(batch, prefix_tokens[slot.n_past], system_tokens.size() + slot_npast, { slot.id }, false); + + llama_batch_add(batch, prompt_tokens[slot.n_past], system_tokens.size() + slot_npast, { slot.id }, false); + + if (slot.params.cache_prompt) { + slot.cache_tokens.push_back(prompt_tokens[slot.n_past]); + } + + slot.n_prompt_tokens_processed++; slot_npast++; } - if (has_images && !ingest_images(slot, n_batch)) - { - LOG_ERROR("failed processing images", { - {"slot_id", slot.id}, - {"task_id", slot.task_id}, - }); - // FIXME @phymbert: to be properly tested - // early returning without changing the slot state will block the slot for ever - // no one at the moment is checking the return value - return false; - } + LOG_VERBOSE("prompt processing progress", { + {"id_slot", slot.id}, + {"n_past", slot.n_past}, + {"n_ctx", n_ctx}, + {"n_tokens", batch.n_tokens}, + {"progress", (float) slot.n_prompt_tokens_processed / slot.n_prompt_tokens}, + }); - // extract the logits only for the last token - if (batch.n_tokens > 0) - { + // entire prompt has been processed - start decoding new tokens + if (slot.n_past == slot.n_prompt_tokens) { + slot.state = SLOT_STATE_PROCESSING; + slot.command = SLOT_COMMAND_NONE; + + GGML_ASSERT(batch.n_tokens > 0); + + // extract the logits only for the last token batch.logits[batch.n_tokens - 1] = true; - } - slot.n_decoded = 0; - slot.i_batch = batch.n_tokens - 1; + slot.n_decoded = 0; + slot.i_batch = batch.n_tokens - 1; + + LOG_VERBOSE("prompt done", { + {"id_slot", slot.id}, + {"n_past", slot.n_past}, + {"n_ctx", n_ctx}, + {"n_tokens", batch.n_tokens}, + }); + } + } + + if (batch.n_tokens >= n_batch) { + break; } } } - if (batch.n_tokens == 0) - { - all_slots_are_idle = true; + if (batch.n_tokens == 0) { + LOG_VERBOSE("no tokens to decode", {}); + return true; } - for (int32_t i = 0; i < (int32_t) batch.n_tokens; i += n_batch) - { + LOG_VERBOSE("decoding batch", { + {"n_tokens", batch.n_tokens}, + }); + + // process the created batch of tokens + for (int32_t i = 0; i < (int32_t) batch.n_tokens; i += n_batch) { const int32_t n_tokens = std::min(n_batch, batch.n_tokens - i); - for (auto & slot : slots) - { - if (slot.ga_n != 1) - { + for (auto & slot : slots) { + if (slot.ga_n != 1) { // context extension via Self-Extend - while (slot.n_past_se >= slot.ga_i + slot.ga_w) - { + // TODO: simplify and/or abstract this + while (slot.n_past_se >= slot.ga_i + slot.ga_w) { const int ib = (slot.ga_n * slot.ga_i) / slot.ga_w; const int bd = (slot.ga_w / slot.ga_n) * (slot.ga_n - 1); const int dd = (slot.ga_w / slot.ga_n) - ib * bd - slot.ga_w; @@ -1918,8 +1927,8 @@ struct llama_server_context LOG_TEE("shift: [%6d, %6d] + %6d -> [%6d, %6d]\n", slot.ga_i + ib * bd + slot.ga_w, slot.n_past_se + ib * bd, dd, slot.ga_i + ib * bd + slot.ga_w + dd, slot.n_past_se + ib * bd + dd); llama_kv_cache_seq_add(ctx, slot.id, slot.ga_i, slot.n_past_se, ib * bd); - llama_kv_cache_seq_div(ctx, slot.id, slot.ga_i + ib * bd, slot.ga_i + ib * bd + slot.ga_w,slot.ga_n); - llama_kv_cache_seq_add(ctx, slot.id, slot.ga_i + ib * bd + slot.ga_w,slot.n_past_se + ib * bd, dd); + llama_kv_cache_seq_div(ctx, slot.id, slot.ga_i + ib * bd, slot.ga_i + ib * bd + slot.ga_w, slot.ga_n); + llama_kv_cache_seq_add(ctx, slot.id, slot.ga_i + ib * bd + slot.ga_w, slot.n_past_se + ib * bd, dd); slot.n_past_se -= bd; @@ -1927,12 +1936,12 @@ struct llama_server_context LOG_TEE("\nn_past_old = %d, n_past = %d, ga_i = %d\n\n", slot.n_past_se + bd, slot.n_past_se, slot.ga_i); } + slot.n_past_se += n_tokens; } } - llama_batch batch_view = - { + llama_batch batch_view = { n_tokens, batch.token + i, nullptr, @@ -1945,10 +1954,8 @@ struct llama_server_context const int ret = llama_decode(ctx, batch_view); - if (ret != 0) - { - if (n_batch == 1 || ret < 0) - { + if (ret != 0) { + if (n_batch == 1 || ret < 0) { // if you get here, it means the KV cache is full - try increasing it via the context size LOG_TEE("%s : failed to decode the batch, n_batch = %d, ret = %d\n", __func__, n_batch, ret); return false; @@ -1959,19 +1966,17 @@ struct llama_server_context // retry with half the batch size to try to find a free slot in the KV cache n_batch /= 2; i -= n_batch; + continue; } - for (auto & slot : slots) - { - if (slot.i_batch < (int) i || slot.i_batch >= (int) (i + n_tokens)) - { + for (auto & slot : slots) { + if (slot.state != SLOT_STATE_PROCESSING || slot.i_batch < (int) i || slot.i_batch >= (int) (i + n_tokens)) { continue; } // prompt evaluated for embedding - if (slot.embedding) - { + if (slot.embedding) { send_embedding(slot, batch_view); slot.release(); slot.i_batch = -1; @@ -1984,10 +1989,9 @@ struct llama_server_context llama_sampling_accept(slot.ctx_sampling, ctx, id, true); slot.n_decoded += 1; - if (slot.n_decoded == 1) - { - slot.t_start_genereration = ggml_time_us(); - slot.t_prompt_processing = (slot.t_start_genereration - slot.t_start_process_prompt) / 1e3; + if (slot.n_decoded == 1) { + slot.t_start_generation = ggml_time_us(); + slot.t_prompt_processing = (slot.t_start_generation - slot.t_start_process_prompt) / 1e3; metrics.on_prompt_eval(slot); } @@ -1995,19 +1999,19 @@ struct llama_server_context result.tok = id; const int32_t n_probs = slot.sparams.n_probs; - if (slot.sparams.temp <= 0 && n_probs > 0) - { + if (slot.sparams.temp <= 0 && n_probs > 0) { // for llama_sample_token_greedy we need to sort candidates llama_sample_softmax(ctx, &cur_p); } - for (size_t i = 0; i < std::min(cur_p.size, (size_t)n_probs); ++i) - { - result.probs.push_back({cur_p.data[i].id, cur_p.data[i].p}); + for (size_t i = 0; i < std::min(cur_p.size, (size_t) n_probs); ++i) { + result.probs.push_back({ + cur_p.data[i].id, + cur_p.data[i].p + }); } - if (!process_token(result, slot)) - { + if (!process_token(result, slot)) { slot.release(); slot.print_timings(); send_final_response(slot); @@ -2019,24 +2023,23 @@ struct llama_server_context } LOG_VERBOSE("slots updated", {}); + return true; } - json model_meta() { - return json{ - {"vocab_type", llama_vocab_type(model)}, - {"n_vocab", llama_n_vocab(model)}, - {"n_ctx_train", llama_n_ctx_train(model)}, - {"n_embd", llama_n_embd(model)}, - {"n_params", llama_model_n_params(model)}, - {"size", llama_model_size(model)}, + json model_meta() const { + return json { + {"vocab_type", llama_vocab_type (model)}, + {"n_vocab", llama_n_vocab (model)}, + {"n_ctx_train", llama_n_ctx_train (model)}, + {"n_embd", llama_n_embd (model)}, + {"n_params", llama_model_n_params(model)}, + {"size", llama_model_size (model)}, }; } }; -static void server_print_usage(const char *argv0, const gpt_params ¶ms, - const server_params &sparams) -{ +static void server_print_usage(const char * argv0, const gpt_params & params, const server_params & sparams) { printf("usage: %s [options]\n", argv0); printf("\n"); printf("options:\n"); @@ -2054,17 +2057,14 @@ static void server_print_usage(const char *argv0, const gpt_params ¶ms, printf(" --yarn-attn-factor N YaRN: scale sqrt(t) or attention magnitude (default: 1.0)\n"); printf(" --yarn-beta-slow N YaRN: high correction dim or alpha (default: %.1f)\n", params.yarn_beta_slow); printf(" --yarn-beta-fast N YaRN: low correction dim or beta (default: %.1f)\n", params.yarn_beta_fast); - printf(" --pooling {none,mean,cls}\n"); - printf(" pooling type for embeddings, use model default if unspecified\n"); + printf(" --pooling {none,mean,cls} pooling type for embeddings, use model default if unspecified\n"); printf(" -b N, --batch-size N batch size for prompt processing (default: %d)\n", params.n_batch); printf(" --memory-f32 use f32 instead of f16 for memory key+value (default: disabled)\n"); printf(" not recommended: doubles context memory required and no measurable increase in quality\n"); - if (llama_supports_mlock()) - { + if (llama_supports_mlock()) { printf(" --mlock force system to keep model in RAM rather than swapping or compressing\n"); } - if (llama_supports_mmap()) - { + if (llama_supports_mmap()) { printf(" --no-mmap do not memory-map model (slower load but may reduce pageouts if not using mlock)\n"); } printf(" --numa TYPE attempt optimizations that help on some NUMA systems\n"); @@ -2096,7 +2096,7 @@ static void server_print_usage(const char *argv0, const gpt_params ¶ms, printf(" --api-key API_KEY optional api key to enhance server security. If set, requests must include this key for access.\n"); printf(" --api-key-file FNAME path to file containing api keys delimited by new lines. If set, requests must include one of the keys for access.\n"); printf(" -to N, --timeout N server read/write timeout in seconds (default: %d)\n", sparams.read_timeout); - printf(" --embedding enable embedding vector output (default: %s)\n", params.embedding ? "enabled" : "disabled"); + printf(" --embeddings enable embedding vector output (default: %s)\n", params.embedding ? "enabled" : "disabled"); printf(" -np N, --parallel N number of slots for process requests (default: %d)\n", params.n_parallel); printf(" -cb, --cont-batching enable continuous batching (a.k.a dynamic batching) (default: disabled)\n"); printf(" -spf FNAME, --system-prompt-file FNAME\n"); @@ -2105,7 +2105,6 @@ static void server_print_usage(const char *argv0, const gpt_params ¶ms, printf(" KV cache data type for K (default: f16)\n"); printf(" -ctv TYPE, --cache-type-v TYPE\n"); printf(" KV cache data type for V (default: f16)\n"); - printf(" --mmproj MMPROJ_FILE path to a multimodal projector file for LLaVA.\n"); printf(" --log-format log output format: json or text (default: json)\n"); printf(" --log-disable disables logging to a file.\n"); printf(" --slots-endpoint-disable disables slots monitoring endpoint.\n"); @@ -2123,57 +2122,41 @@ static void server_print_usage(const char *argv0, const gpt_params ¶ms, printf("\n"); } -static void server_params_parse(int argc, char **argv, server_params &sparams, - gpt_params ¶ms, llama_server_context& llama) -{ - gpt_params default_params; +static void server_params_parse(int argc, char ** argv, server_params & sparams, gpt_params & params) { + gpt_params default_params; server_params default_sparams; + std::string arg; bool invalid_param = false; - for (int i = 1; i < argc; i++) - { + for (int i = 1; i < argc; i++) { arg = argv[i]; - if (arg == "--port") - { - if (++i >= argc) - { + if (arg == "--port") { + if (++i >= argc) { invalid_param = true; break; } sparams.port = std::stoi(argv[i]); - } - else if (arg == "--host") - { - if (++i >= argc) - { + } else if (arg == "--host") { + if (++i >= argc) { invalid_param = true; break; } sparams.hostname = argv[i]; - } - else if (arg == "--path") - { - if (++i >= argc) - { + } else if (arg == "--path") { + if (++i >= argc) { invalid_param = true; break; } sparams.public_path = argv[i]; - } - else if (arg == "--api-key") - { - if (++i >= argc) - { + } else if (arg == "--api-key") { + if (++i >= argc) { invalid_param = true; break; } sparams.api_keys.emplace_back(argv[i]); - } - else if (arg == "--api-key-file") - { - if (++i >= argc) - { + } else if (arg == "--api-key-file") { + if (++i >= argc) { invalid_param = true; break; } @@ -2190,53 +2173,36 @@ static void server_params_parse(int argc, char **argv, server_params &sparams, } } key_file.close(); - } - else if (arg == "--timeout" || arg == "-to") - { - if (++i >= argc) - { + } else if (arg == "--timeout" || arg == "-to") { + if (++i >= argc) { invalid_param = true; break; } sparams.read_timeout = std::stoi(argv[i]); sparams.write_timeout = std::stoi(argv[i]); - } - else if (arg == "-m" || arg == "--model") - { - if (++i >= argc) - { + } else if (arg == "-m" || arg == "--model") { + if (++i >= argc) { invalid_param = true; break; } params.model = argv[i]; - } - else if (arg == "-a" || arg == "--alias") - { - if (++i >= argc) - { + } else if (arg == "-a" || arg == "--alias") { + if (++i >= argc) { invalid_param = true; break; } params.model_alias = argv[i]; - } - else if (arg == "-h" || arg == "--help") - { + } else if (arg == "-h" || arg == "--help") { server_print_usage(argv[0], default_params, default_sparams); exit(0); - } - else if (arg == "-c" || arg == "--ctx-size" || arg == "--ctx_size") - { - if (++i >= argc) - { + } else if (arg == "-c" || arg == "--ctx-size" || arg == "--ctx_size") { + if (++i >= argc) { invalid_param = true; break; } params.n_ctx = std::stoi(argv[i]); - } - else if (arg == "--rope-scaling") - { - if (++i >= argc) - { + } else if (arg == "--rope-scaling") { + if (++i >= argc) { invalid_param = true; break; } @@ -2245,59 +2211,44 @@ static void server_params_parse(int argc, char **argv, server_params &sparams, else if (value == "linear") { params.rope_scaling_type = LLAMA_ROPE_SCALING_TYPE_LINEAR; } else if (value == "yarn") { params.rope_scaling_type = LLAMA_ROPE_SCALING_TYPE_YARN; } else { invalid_param = true; break; } - } - else if (arg == "--rope-freq-base") - { - if (++i >= argc) - { + } else if (arg == "--rope-freq-base") { + if (++i >= argc) { invalid_param = true; break; } params.rope_freq_base = std::stof(argv[i]); - } - else if (arg == "--rope-freq-scale") - { - if (++i >= argc) - { + } else if (arg == "--rope-freq-scale") { + if (++i >= argc) { invalid_param = true; break; } params.rope_freq_scale = std::stof(argv[i]); - } - else if (arg == "--yarn-ext-factor") - { + } else if (arg == "--yarn-ext-factor") { if (++i >= argc) { invalid_param = true; break; } params.yarn_ext_factor = std::stof(argv[i]); } - else if (arg == "--yarn-attn-factor") - { + else if (arg == "--yarn-attn-factor") { if (++i >= argc) { invalid_param = true; break; } params.yarn_attn_factor = std::stof(argv[i]); - } - else if (arg == "--yarn-beta-fast") - { + } else if (arg == "--yarn-beta-fast") { if (++i >= argc) { invalid_param = true; break; } params.yarn_beta_fast = std::stof(argv[i]); - } - else if (arg == "--yarn-beta-slow") - { + } else if (arg == "--yarn-beta-slow") { if (++i >= argc) { invalid_param = true; break; } params.yarn_beta_slow = std::stof(argv[i]); - } - else if (arg == "--pooling") - { + } else if (arg == "--pooling") { if (++i >= argc) { invalid_param = true; break; @@ -2307,108 +2258,79 @@ static void server_params_parse(int argc, char **argv, server_params &sparams, else if (value == "mean") { params.pooling_type = LLAMA_POOLING_TYPE_MEAN; } else if (value == "cls") { params.pooling_type = LLAMA_POOLING_TYPE_CLS; } else { invalid_param = true; break; } - } - else if (arg == "--threads" || arg == "-t") - { + } else if (arg == "--threads" || arg == "-t") { if (++i >= argc) { invalid_param = true; break; } params.n_threads = std::stoi(argv[i]); - } - else if (arg == "--grp-attn-n" || arg == "-gan") - { + } else if (arg == "--grp-attn-n" || arg == "-gan") { if (++i >= argc) { invalid_param = true; break; } params.grp_attn_n = std::stoi(argv[i]); - } - else if (arg == "--grp-attn-w" || arg == "-gaw") - { - if (++i >= argc) - { + } else if (arg == "--grp-attn-w" || arg == "-gaw") { + if (++i >= argc) { invalid_param = true; break; } params.grp_attn_w = std::stoi(argv[i]); - } - else if (arg == "--threads-batch" || arg == "-tb") - { - if (++i >= argc) - { + } else if (arg == "--threads-batch" || arg == "-tb") { + if (++i >= argc) { invalid_param = true; break; } params.n_threads_batch = std::stoi(argv[i]); - } - else if (arg == "--threads-http") - { - if (++i >= argc) - { + } else if (arg == "--threads-http") { + if (++i >= argc) { invalid_param = true; break; } sparams.n_threads_http = std::stoi(argv[i]); - } - else if (arg == "-b" || arg == "--batch-size") - { - if (++i >= argc) - { + } else if (arg == "-b" || arg == "--batch-size") { + if (++i >= argc) { invalid_param = true; break; } params.n_batch = std::stoi(argv[i]); - } - else if (arg == "--gpu-layers" || arg == "-ngl" || arg == "--n-gpu-layers") - { - if (++i >= argc) - { + } else if (arg == "--gpu-layers" || arg == "-ngl" || arg == "--n-gpu-layers") { + if (++i >= argc) { invalid_param = true; break; } if (llama_supports_gpu_offload()) { params.n_gpu_layers = std::stoi(argv[i]); } else { - LOG_WARNING("Not compiled with GPU offload support, --n-gpu-layers option will be ignored. " - "See main README.md for information on enabling GPU BLAS support", - {{"n_gpu_layers", params.n_gpu_layers}}); + LOG_WARNING( + "Not compiled with GPU offload support, --n-gpu-layers option will be ignored. " + "See main README.md for information on enabling GPU BLAS support", + {{"n_gpu_layers", params.n_gpu_layers}}); } - } - else if (arg == "--split-mode" || arg == "-sm") - { + } else if (arg == "--split-mode" || arg == "-sm") { if (++i >= argc) { invalid_param = true; break; } std::string arg_next = argv[i]; - if (arg_next == "none") - { + if (arg_next == "none") { params.split_mode = LLAMA_SPLIT_MODE_NONE; - } - else if (arg_next == "layer") - { + } else if (arg_next == "layer") { params.split_mode = LLAMA_SPLIT_MODE_LAYER; - } - else if (arg_next == "row") - { + } else if (arg_next == "row") { params.split_mode = LLAMA_SPLIT_MODE_ROW; - } - else { + } else { invalid_param = true; break; } #ifndef GGML_USE_CUBLAS fprintf(stderr, "warning: llama.cpp was compiled without cuBLAS. Setting the split mode has no effect.\n"); #endif // GGML_USE_CUBLAS - } - else if (arg == "--tensor-split" || arg == "-ts") - { - if (++i >= argc) - { + } else if (arg == "--tensor-split" || arg == "-ts") { + if (++i >= argc) { invalid_param = true; break; } @@ -2421,25 +2343,18 @@ static void server_params_parse(int argc, char **argv, server_params &sparams, std::vector split_arg{it, {}}; GGML_ASSERT(split_arg.size() <= llama_max_devices()); - for (size_t i_device = 0; i_device < llama_max_devices(); ++i_device) - { - if (i_device < split_arg.size()) - { + for (size_t i_device = 0; i_device < llama_max_devices(); ++i_device) { + if (i_device < split_arg.size()) { params.tensor_split[i_device] = std::stof(split_arg[i_device]); - } - else - { + } else { params.tensor_split[i_device] = 0.0f; } } #else LOG_WARNING("llama.cpp was compiled without cuBLAS. It is not possible to set a tensor split.\n", {}); #endif // GGML_USE_CUBLAS - } - else if (arg == "--main-gpu" || arg == "-mg") - { - if (++i >= argc) - { + } else if (arg == "--main-gpu" || arg == "-mg") { + if (++i >= argc) { invalid_param = true; break; } @@ -2448,98 +2363,70 @@ static void server_params_parse(int argc, char **argv, server_params &sparams, #else LOG_WARNING("llama.cpp was compiled without cuBLAS. It is not possible to set a main GPU.", {}); #endif - } - else if (arg == "--lora") - { - if (++i >= argc) - { + } else if (arg == "--lora") { + if (++i >= argc) { invalid_param = true; break; } params.lora_adapter.emplace_back(argv[i], 1.0f); params.use_mmap = false; - } - else if (arg == "--lora-scaled") - { - if (++i >= argc) - { + } else if (arg == "--lora-scaled") { + if (++i >= argc) { invalid_param = true; break; } const char * lora_adapter = argv[i]; - if (++i >= argc) - { + if (++i >= argc) { invalid_param = true; break; } params.lora_adapter.emplace_back(lora_adapter, std::stof(argv[i])); params.use_mmap = false; - } - else if (arg == "--lora-base") - { - if (++i >= argc) - { + } else if (arg == "--lora-base") { + if (++i >= argc) { invalid_param = true; break; } params.lora_base = argv[i]; - } - else if (arg == "-v" || arg == "--verbose") - { + } else if (arg == "-v" || arg == "--verbose") { #if SERVER_VERBOSE != 1 LOG_WARNING("server.cpp is not built with verbose logging.", {}); #else server_verbose = true; #endif - } - else if (arg == "--mlock") - { + } else if (arg == "--mlock") { params.use_mlock = true; - } - else if (arg == "--no-mmap") - { + } else if (arg == "--no-mmap") { params.use_mmap = false; - } - else if (arg == "--numa") { + } else if (arg == "--numa") { if (++i >= argc) { invalid_param = true; break; } else { std::string value(argv[i]); /**/ if (value == "distribute" || value == "" ) { params.numa = GGML_NUMA_STRATEGY_DISTRIBUTE; } - else if (value == "isolate") { params.numa = GGML_NUMA_STRATEGY_ISOLATE; } - else if (value == "numactl") { params.numa = GGML_NUMA_STRATEGY_NUMACTL; } + else if (value == "isolate") { params.numa = GGML_NUMA_STRATEGY_ISOLATE; } + else if (value == "numactl") { params.numa = GGML_NUMA_STRATEGY_NUMACTL; } else { invalid_param = true; break; } } - } - else if (arg == "--embedding") - { + } else if (arg == "--embedding" || arg == "--embeddings") { params.embedding = true; - } - else if (arg == "-cb" || arg == "--cont-batching") - { + } else if (arg == "-cb" || arg == "--cont-batching") { params.cont_batching = true; - } - else if (arg == "-np" || arg == "--parallel") - { - if (++i >= argc) - { + } else if (arg == "-np" || arg == "--parallel") { + if (++i >= argc) { invalid_param = true; break; } params.n_parallel = std::stoi(argv[i]); - } else if (arg == "-n" || arg == "--n-predict") - { - if (++i >= argc) - { + } else if (arg == "-n" || arg == "--n-predict") { + if (++i >= argc) { invalid_param = true; break; } params.n_predict = std::stoi(argv[i]); - } else if (arg == "-spf" || arg == "--system-prompt-file") - { - if (++i >= argc) - { + } else if (arg == "-spf" || arg == "--system-prompt-file") { + if (++i >= argc) { invalid_param = true; break; } @@ -2549,67 +2436,39 @@ static void server_params_parse(int argc, char **argv, server_params &sparams, invalid_param = true; break; } - std::string systm_content; + std::string system_prompt; std::copy( std::istreambuf_iterator(file), std::istreambuf_iterator(), - std::back_inserter(systm_content) + std::back_inserter(system_prompt) ); - llama.system_prompt_process(json::parse(systm_content)); - } - else if (arg == "-ctk" || arg == "--cache-type-k") { + sparams.system_prompt = system_prompt; + } else if (arg == "-ctk" || arg == "--cache-type-k") { params.cache_type_k = argv[++i]; - } - else if (arg == "-ctv" || arg == "--cache-type-v") { + } else if (arg == "-ctv" || arg == "--cache-type-v") { params.cache_type_v = argv[++i]; - } - else if(arg == "--mmproj") - { - if (++i >= argc) - { + } else if (arg == "--log-format") { + if (++i >= argc) { invalid_param = true; break; } - params.mmproj = argv[i]; - } - else if (arg == "--log-format") - { - if (++i >= argc) - { - invalid_param = true; - break; - } - if (std::strcmp(argv[i], "json") == 0) - { + if (std::strcmp(argv[i], "json") == 0) { server_log_json = true; - } - else if (std::strcmp(argv[i], "text") == 0) - { + } else if (std::strcmp(argv[i], "text") == 0) { server_log_json = false; - } - else - { + } else { invalid_param = true; break; } - } - else if (arg == "--log-disable") - { + } else if (arg == "--log-disable") { log_set_target(stdout); LOG_INFO("logging to file is disabled.", {}); - } - else if (arg == "--slots-endpoint-disable") - { + } else if (arg == "--slots-endpoint-disable") { sparams.slots_endpoint = false; - } - else if (arg == "--metrics") - { + } else if (arg == "--metrics") { sparams.metrics_endpoint = true; - } - else if (arg == "--chat-template") - { - if (++i >= argc) - { + } else if (arg == "--chat-template") { + if (++i >= argc) { invalid_param = true; break; } @@ -2620,9 +2479,7 @@ static void server_params_parse(int argc, char **argv, server_params &sparams, break; } sparams.chat_template = argv[i]; - } - else if (arg == "--override-kv") - { + } else if (arg == "--override-kv") { if (++i >= argc) { invalid_param = true; break; @@ -2633,6 +2490,7 @@ static void server_params_parse(int argc, char **argv, server_params &sparams, invalid_param = true; break; } + struct llama_model_kv_override kvo; std::strncpy(kvo.key, argv[i], sep - argv[i]); kvo.key[sep - argv[i]] = 0; @@ -2663,67 +2521,28 @@ static void server_params_parse(int argc, char **argv, server_params &sparams, break; } params.kv_overrides.push_back(kvo); - } - else - { + } else { fprintf(stderr, "error: unknown argument: %s\n", arg.c_str()); server_print_usage(argv[0], default_params, default_sparams); exit(1); } } + if (!params.kv_overrides.empty()) { params.kv_overrides.emplace_back(); params.kv_overrides.back().key[0] = 0; } - if (invalid_param) - { + if (invalid_param) { fprintf(stderr, "error: invalid parameter for argument: %s\n", arg.c_str()); server_print_usage(argv[0], default_params, default_sparams); exit(1); } } -/* llama.cpp completion api semantics */ -static json format_partial_response( - llama_server_context &llama, server_slot *slot, const std::string &content, const std::vector &probs -) { - json res = json - { - {"content", content }, - {"stop", false}, - {"slot_id", slot->id }, - {"multimodal", llama.multimodal } - }; - - if (slot->sparams.n_probs > 0) - { - res["completion_probabilities"] = probs_vector_to_json(llama.ctx, probs); - } - - return res; -} - -static json format_tokenizer_response(const std::vector &tokens) -{ - return json { - {"tokens", tokens} - }; -} - -static json format_detokenized_response(std::string content) -{ - return json { - {"content", content} - }; -} - - -static void log_server_request(const httplib::Request &req, const httplib::Response &res) -{ +static void log_server_request(const httplib::Request & req, const httplib::Response & res) { // skip GH copilot requests when using default port - if (req.path == "/v1/health" || req.path == "/v1/completions") - { + if (req.path == "/v1/health" || req.path == "/v1/completions") { return; } @@ -2742,24 +2561,9 @@ static void log_server_request(const httplib::Request &req, const httplib::Respo }); } -static void append_to_generated_text_from_generated_token_probs(llama_server_context &llama, server_slot *slot) -{ - auto & gtps = slot->generated_token_probs; - auto translator = token_translator{llama.ctx}; - auto add_strlen = [=](size_t sum, const completion_token_output & cto) { return sum + translator(cto).size(); }; - const size_t len = std::accumulate(gtps.begin(), gtps.end(), size_t(0), add_strlen); - if (slot->generated_text.capacity() < slot->generated_text.size() + len) - { - slot->generated_text.reserve(slot->generated_text.size() + len); - } - for (const completion_token_output & cto : gtps) - { - slot->generated_text += translator(cto); - } -} - std::function shutdown_handler; std::atomic_flag is_terminating = ATOMIC_FLAG_INIT; + inline void signal_handler(int signal) { if (is_terminating.test_and_set()) { // in case it hangs, we can force terminate the server by hitting Ctrl+C twice @@ -2767,40 +2571,45 @@ inline void signal_handler(int signal) { fprintf(stderr, "Received second interrupt, terminating immediately.\n"); exit(1); } + shutdown_handler(signal); } -int main(int argc, char **argv) -{ +int main(int argc, char ** argv) { #if SERVER_VERBOSE != 1 log_disable(); #endif // own arguments required by this example - gpt_params params; + gpt_params params; server_params sparams; // struct that contains llama context and inference - llama_server_context llama; + server_context ctx_server; - server_params_parse(argc, argv, sparams, params, llama); + server_params_parse(argc, argv, sparams, params); - if (params.model_alias == "unknown") - { + if (!sparams.system_prompt.empty()) { + ctx_server.system_prompt_set(json::parse(sparams.system_prompt)); + } + + if (params.model_alias == "unknown") { params.model_alias = params.model; } llama_backend_init(); llama_numa_init(params.numa); - LOG_INFO("build info", {{"build", LLAMA_BUILD_NUMBER}, - {"commit", LLAMA_COMMIT}}); + LOG_INFO("build info", { + {"build", LLAMA_BUILD_NUMBER}, + {"commit", LLAMA_COMMIT} + }); LOG_INFO("system info", { - {"n_threads", params.n_threads}, - {"n_threads_batch", params.n_threads_batch}, - {"total_threads", std::thread::hardware_concurrency()}, - {"system_info", llama_print_system_info()}, - }); + {"n_threads", params.n_threads}, + {"n_threads_batch", params.n_threads_batch}, + {"total_threads", std::thread::hardware_concurrency()}, + {"system_info", llama_print_system_info()}, + }); httplib::Server svr; @@ -2809,152 +2618,163 @@ int main(int argc, char **argv) svr.set_default_headers({{"Server", "llama.cpp"}}); // CORS preflight - svr.Options(R"(.*)", [](const httplib::Request &req, httplib::Response &res) { - res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin")); + svr.Options(R"(.*)", [](const httplib::Request & req, httplib::Response & res) { + res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin")); res.set_header("Access-Control-Allow-Credentials", "true"); - res.set_header("Access-Control-Allow-Methods", "POST"); - res.set_header("Access-Control-Allow-Headers", "*"); + res.set_header("Access-Control-Allow-Methods", "POST"); + res.set_header("Access-Control-Allow-Headers", "*"); }); - svr.Get("/health", [&](const httplib::Request& req, httplib::Response& res) { + svr.Get("/health", [&](const httplib::Request & req, httplib::Response & res) { server_state current_state = state.load(); - switch(current_state) { - case SERVER_STATE_READY: { - // request slots data using task queue - task_server task; - task.id = llama.queue_tasks.get_new_id(); - task.type = TASK_TYPE_METRICS; - task.target_id = -1; + switch (current_state) { + case SERVER_STATE_READY: + { + // request slots data using task queue + server_task task; + task.id = ctx_server.queue_tasks.get_new_id(); + task.type = SERVER_TASK_TYPE_METRICS; + task.id_target = -1; - llama.queue_results.add_waiting_task_id(task.id); - llama.queue_tasks.post(task); + ctx_server.queue_results.add_waiting_task_id(task.id); + ctx_server.queue_tasks.post(task); - // get the result - task_result result = llama.queue_results.recv(task.id); - llama.queue_results.remove_waiting_task_id(task.id); + // get the result + server_task_result result = ctx_server.queue_results.recv(task.id); + ctx_server.queue_results.remove_waiting_task_id(task.id); - int n_idle_slots = result.result_json["idle"]; - int n_processing_slots = result.result_json["processing"]; + const int n_idle_slots = result.data["idle"]; + const int n_processing_slots = result.data["processing"]; - json health = { + json health = { {"status", "ok"}, {"slots_idle", n_idle_slots}, - {"slots_processing", n_processing_slots}}; - res.status = 200; // HTTP OK - if (sparams.slots_endpoint && req.has_param("include_slots")) { - health["slots"] = result.result_json["slots"]; - } + {"slots_processing", n_processing_slots} + }; - if (n_idle_slots == 0) { - health["status"] = "no slot available"; - if (req.has_param("fail_on_no_slot")) { - res.status = 503; // HTTP Service Unavailable + res.status = 200; // HTTP OK + if (sparams.slots_endpoint && req.has_param("include_slots")) { + health["slots"] = result.data["slots"]; } + + if (n_idle_slots == 0) { + health["status"] = "no slot available"; + if (req.has_param("fail_on_no_slot")) { + res.status = 503; // HTTP Service Unavailable + } + } + + res.set_content(health.dump(), "application/json"); + break; } - res.set_content(health.dump(), "application/json"); - break; - } case SERVER_STATE_LOADING_MODEL: - res.set_content(R"({"status": "loading model"})", "application/json"); - res.status = 503; // HTTP Service Unavailable - break; + { + res.set_content(R"({"status": "loading model"})", "application/json"); + res.status = 503; // HTTP Service Unavailable + } break; case SERVER_STATE_ERROR: - res.set_content(R"({"status": "error", "error": "Model failed to load"})", "application/json"); - res.status = 500; // HTTP Internal Server Error - break; + { + res.set_content(R"({"status": "error", "error": "Model failed to load"})", "application/json"); + res.status = 500; // HTTP Internal Server Error + } break; } }); if (sparams.slots_endpoint) { - svr.Get("/slots", [&](const httplib::Request&, httplib::Response& res) { + svr.Get("/slots", [&](const httplib::Request &, httplib::Response & res) { // request slots data using task queue - task_server task; - task.id = llama.queue_tasks.get_new_id(); - task.type = TASK_TYPE_METRICS; - task.target_id = -1; + server_task task; + task.id = ctx_server.queue_tasks.get_new_id(); + task.id_multi = -1; + task.id_target = -1; + task.type = SERVER_TASK_TYPE_METRICS; - llama.queue_results.add_waiting_task_id(task.id); - llama.queue_tasks.post(task); + ctx_server.queue_results.add_waiting_task_id(task.id); + ctx_server.queue_tasks.post(task); // get the result - task_result result = llama.queue_results.recv(task.id); - llama.queue_results.remove_waiting_task_id(task.id); + server_task_result result = ctx_server.queue_results.recv(task.id); + ctx_server.queue_results.remove_waiting_task_id(task.id); - res.set_content(result.result_json["slots"].dump(), "application/json"); + res.set_content(result.data["slots"].dump(), "application/json"); res.status = 200; // HTTP OK }); } if (sparams.metrics_endpoint) { - svr.Get("/metrics", [&](const httplib::Request&, httplib::Response& res) { + svr.Get("/metrics", [&](const httplib::Request &, httplib::Response & res) { // request slots data using task queue - task_server task; - task.id = llama.queue_tasks.get_new_id(); - task.type = TASK_TYPE_METRICS; - task.target_id = -1; + server_task task; + task.id = ctx_server.queue_tasks.get_new_id(); + task.id_multi = -1; + task.id_target = -1; + task.type = SERVER_TASK_TYPE_METRICS; - llama.queue_results.add_waiting_task_id(task.id); - llama.queue_tasks.post(task); + ctx_server.queue_results.add_waiting_task_id(task.id); + ctx_server.queue_tasks.post(task); // get the result - task_result result = llama.queue_results.recv(task.id); - llama.queue_results.remove_waiting_task_id(task.id); + server_task_result result = ctx_server.queue_results.recv(task.id); + ctx_server.queue_results.remove_waiting_task_id(task.id); - json data = result.result_json; + json data = result.data; - uint64_t n_prompt_tokens_processed = data["n_prompt_tokens_processed"]; - uint64_t t_prompt_processing = data["t_prompt_processing"]; + const uint64_t n_prompt_tokens_processed = data["n_prompt_tokens_processed"]; + const uint64_t t_prompt_processing = data["t_prompt_processing"]; - uint64_t n_tokens_predicted = data["n_tokens_predicted"]; - uint64_t t_tokens_generation = data["t_tokens_generation"]; + const uint64_t n_tokens_predicted = data["n_tokens_predicted"]; + const uint64_t t_tokens_generation = data["t_tokens_generation"]; - int32_t kv_cache_used_cells = data["kv_cache_used_cells"]; + const int32_t kv_cache_used_cells = data["kv_cache_used_cells"]; // metrics definition: https://prometheus.io/docs/practices/naming/#metric-names json all_metrics_def = json { - {"counter", {{ - {"name", "prompt_tokens_total"}, - {"help", "Number of prompt tokens processed."}, - {"value", data["n_prompt_tokens_processed_total"]} - }, { - {"name", "tokens_predicted_total"}, - {"help", "Number of generation tokens processed."}, - {"value", data["n_tokens_predicted_total"]} - }}}, - {"gauge", {{ - {"name", "prompt_tokens_seconds"}, - {"help", "Average prompt throughput in tokens/s."}, - {"value", n_prompt_tokens_processed ? 1e3 / t_prompt_processing * n_prompt_tokens_processed : 0} - },{ - {"name", "predicted_tokens_seconds"}, - {"help", "Average generation throughput in tokens/s."}, - {"value", n_tokens_predicted ? 1e3 / t_tokens_generation * n_tokens_predicted : 0} - },{ - {"name", "kv_cache_usage_ratio"}, - {"help", "KV-cache usage. 1 means 100 percent usage."}, - {"value", 1. * kv_cache_used_cells / params.n_ctx} - },{ - {"name", "kv_cache_tokens"}, - {"help", "KV-cache tokens."}, - {"value", data["kv_cache_tokens_count"]} - },{ - {"name", "requests_processing"}, - {"help", "Number of request processing."}, - {"value", data["processing"]} - },{ - {"name", "requests_deferred"}, - {"help", "Number of request deferred."}, - {"value", data["deferred"]} - }}} + {"counter", {{ + {"name", "prompt_tokens_total"}, + {"help", "Number of prompt tokens processed."}, + {"value", data["n_prompt_tokens_processed_total"]} + }, { + {"name", "tokens_predicted_total"}, + {"help", "Number of generation tokens processed."}, + {"value", data["n_tokens_predicted_total"]} + }}}, + {"gauge", {{ + {"name", "prompt_tokens_seconds"}, + {"help", "Average prompt throughput in tokens/s."}, + {"value", n_prompt_tokens_processed ? 1e3 / t_prompt_processing * n_prompt_tokens_processed : 0} + },{ + {"name", "predicted_tokens_seconds"}, + {"help", "Average generation throughput in tokens/s."}, + {"value", n_tokens_predicted ? 1e3 / t_tokens_generation * n_tokens_predicted : 0} + },{ + {"name", "kv_cache_usage_ratio"}, + {"help", "KV-cache usage. 1 means 100 percent usage."}, + {"value", 1. * kv_cache_used_cells / params.n_ctx} + },{ + {"name", "kv_cache_tokens"}, + {"help", "KV-cache tokens."}, + {"value", data["kv_cache_tokens_count"]} + },{ + {"name", "requests_processing"}, + {"help", "Number of request processing."}, + {"value", data["processing"]} + },{ + {"name", "requests_deferred"}, + {"help", "Number of request deferred."}, + {"value", data["deferred"]} + }}} }; std::stringstream prometheus; - for (const auto& el : all_metrics_def.items()) { - const auto& type = el.key(); - const auto& metrics_def = el.value(); - for (const auto& metric_def : metrics_def) { - std::string name = metric_def["name"]; - std::string help = metric_def["help"]; + + for (const auto & el : all_metrics_def.items()) { + const auto & type = el.key(); + const auto & metrics_def = el.value(); + + for (const auto & metric_def : metrics_def) { + const std::string name = metric_def["name"]; + const std::string help = metric_def["help"]; + auto value = json_value(metric_def, "value", 0); prometheus << "# HELP llamacpp:" << name << " " << help << "\n" << "# TYPE llamacpp:" << name << " " << type << "\n" @@ -2969,49 +2789,39 @@ int main(int argc, char **argv) svr.set_logger(log_server_request); - svr.set_exception_handler([](const httplib::Request &, httplib::Response &res, std::exception_ptr ep) - { - const char fmt[] = "500 Internal Server Error\n%s"; - char buf[BUFSIZ]; - try - { - std::rethrow_exception(std::move(ep)); - } - catch (std::exception &e) - { - snprintf(buf, sizeof(buf), fmt, e.what()); - } - catch (...) - { - snprintf(buf, sizeof(buf), fmt, "Unknown Exception"); - } - res.set_content(buf, "text/plain; charset=utf-8"); - res.status = 500; - }); + svr.set_exception_handler([](const httplib::Request &, httplib::Response & res, std::exception_ptr ep) { + const char fmt[] = "500 Internal Server Error\n%s"; - svr.set_error_handler([](const httplib::Request &, httplib::Response &res) - { - if (res.status == 401) - { - res.set_content("Unauthorized", "text/plain; charset=utf-8"); - } - if (res.status == 400) - { - res.set_content("Invalid request", "text/plain; charset=utf-8"); - } - else if (res.status == 404) - { - res.set_content("File Not Found", "text/plain; charset=utf-8"); - res.status = 404; - } - }); + char buf[BUFSIZ]; + try { + std::rethrow_exception(std::move(ep)); + } catch (std::exception &e) { + snprintf(buf, sizeof(buf), fmt, e.what()); + } catch (...) { + snprintf(buf, sizeof(buf), fmt, "Unknown Exception"); + } + + res.set_content(buf, "text/plain; charset=utf-8"); + res.status = 500; + }); + + svr.set_error_handler([](const httplib::Request &, httplib::Response & res) { + if (res.status == 401) { + res.set_content("Unauthorized", "text/plain; charset=utf-8"); + } + if (res.status == 400) { + res.set_content("Invalid request", "text/plain; charset=utf-8"); + } + if (res.status == 404) { + res.set_content("File Not Found", "text/plain; charset=utf-8"); + } + }); // set timeouts and change hostname and port svr.set_read_timeout (sparams.read_timeout); svr.set_write_timeout(sparams.write_timeout); - if (!svr.bind_to_port(sparams.hostname, sparams.port)) - { + if (!svr.bind_to_port(sparams.hostname, sparams.port)) { fprintf(stderr, "\ncouldn't bind to server socket: hostname=%s port=%d\n\n", sparams.hostname.c_str(), sparams.port); return 1; } @@ -3020,8 +2830,9 @@ int main(int argc, char **argv) svr.set_base_dir(sparams.public_path); std::unordered_map log_data; + log_data["hostname"] = sparams.hostname; - log_data["port"] = std::to_string(sparams.port); + log_data["port"] = std::to_string(sparams.port); if (sparams.api_keys.size() == 1) { log_data["api_key"] = "api_key: ****" + sparams.api_keys[0].substr(sparams.api_keys[0].length() - 4); @@ -3030,20 +2841,23 @@ int main(int argc, char **argv) } // load the model - if (!llama.load_model(params)) - { + if (!ctx_server.load_model(params)) { state.store(SERVER_STATE_ERROR); return 1; } else { - llama.initialize(); + ctx_server.initialize(); state.store(SERVER_STATE_READY); - LOG_INFO("model loaded", {}); } - const auto model_meta = llama.model_meta(); + + LOG_INFO("model loaded", {}); + + const auto model_meta = ctx_server.model_meta(); if (sparams.chat_template.empty()) { // custom chat template is not supplied - // check if the template comes with the model is supported by us - llama.validate_model_chat_template(sparams); + if (!ctx_server.validate_model_chat_template()) { + LOG_ERROR("The chat template that comes with this model is not yet supported, falling back to chatml. This may cause the model to output suboptimal responses", {}); + sparams.chat_template = "chatml"; + } } // Middleware for API key validation @@ -3055,6 +2869,7 @@ int main(int argc, char **argv) // Check for API key in the header auto auth_header = req.get_header_value("Authorization"); + std::string prefix = "Bearer "; if (auth_header.substr(0, prefix.size()) == prefix) { std::string received_api_key = auth_header.substr(prefix.size()); @@ -3073,179 +2888,173 @@ int main(int argc, char **argv) }; // this is only called if no index.html is found in the public --path - svr.Get("/", [](const httplib::Request &, httplib::Response &res) - { - res.set_content(reinterpret_cast(&index_html), index_html_len, "text/html; charset=utf-8"); - return false; - }); + svr.Get("/", [](const httplib::Request &, httplib::Response & res) { + res.set_content(reinterpret_cast(&index_html), index_html_len, "text/html; charset=utf-8"); + return false; + }); // this is only called if no index.js is found in the public --path - svr.Get("/index.js", [](const httplib::Request &, httplib::Response &res) - { - res.set_content(reinterpret_cast(&index_js), index_js_len, "text/javascript; charset=utf-8"); - return false; - }); + svr.Get("/index.js", [](const httplib::Request &, httplib::Response & res) { + res.set_content(reinterpret_cast(&index_js), index_js_len, "text/javascript; charset=utf-8"); + return false; + }); // this is only called if no index.html is found in the public --path - svr.Get("/completion.js", [](const httplib::Request &, httplib::Response &res) - { - res.set_content(reinterpret_cast(&completion_js), completion_js_len, "application/javascript; charset=utf-8"); - return false; - }); + svr.Get("/completion.js", [](const httplib::Request &, httplib::Response & res) { + res.set_content(reinterpret_cast(&completion_js), completion_js_len, "application/javascript; charset=utf-8"); + return false; + }); // this is only called if no index.html is found in the public --path - svr.Get("/json-schema-to-grammar.mjs", [](const httplib::Request &, httplib::Response &res) - { - res.set_content(reinterpret_cast(&json_schema_to_grammar_mjs), json_schema_to_grammar_mjs_len, "application/javascript; charset=utf-8"); - return false; - }); + svr.Get("/json-schema-to-grammar.mjs", [](const httplib::Request &, httplib::Response & res) { + res.set_content(reinterpret_cast(&json_schema_to_grammar_mjs), json_schema_to_grammar_mjs_len, "application/javascript; charset=utf-8"); + return false; + }); - svr.Get("/props", [&llama](const httplib::Request & req, httplib::Response &res) - { - res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin")); - json data = { - { "user_name", llama.name_user.c_str() }, - { "assistant_name", llama.name_assistant.c_str() }, - { "default_generation_settings", llama.default_generation_settings_for_props }, - { "total_slots", llama.params.n_parallel } - }; - res.set_content(data.dump(), "application/json; charset=utf-8"); - }); + svr.Get("/props", [&ctx_server](const httplib::Request & req, httplib::Response & res) { + res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin")); + json data = { + { "user_name", ctx_server.name_user.c_str() }, + { "assistant_name", ctx_server.name_assistant.c_str() }, + { "default_generation_settings", ctx_server.default_generation_settings_for_props }, + { "total_slots", ctx_server.params.n_parallel } + }; - svr.Post("/completion", [&llama, &validate_api_key](const httplib::Request &req, httplib::Response &res) - { - res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin")); - if (!validate_api_key(req, res)) { - return; - } - json data = json::parse(req.body); - const int task_id = llama.queue_tasks.get_new_id(); - llama.queue_results.add_waiting_task_id(task_id); - llama.request_completion(task_id, data, false, false, -1); - if (!json_value(data, "stream", false)) { - std::string completion_text; - task_result result = llama.queue_results.recv(task_id); - if (!result.error && result.stop) { - res.set_content(result.result_json.dump(-1, ' ', false, json::error_handler_t::replace), "application/json; charset=utf-8"); - } - else - { - res.status = 404; - res.set_content(result.result_json["content"], "text/plain; charset=utf-8"); - } - llama.queue_results.remove_waiting_task_id(task_id); - } else { - const auto chunked_content_provider = [task_id, &llama](size_t, httplib::DataSink & sink) - { - while (true) - { - task_result result = llama.queue_results.recv(task_id); - if (!result.error) { - const std::string str = - "data: " + - result.result_json.dump(-1, ' ', false, json::error_handler_t::replace) + - "\n\n"; - LOG_VERBOSE("data stream", { - { "to_send", str } - }); - if (!sink.write(str.c_str(), str.size())) - { - llama.queue_results.remove_waiting_task_id(task_id); - return false; - } - if (result.stop) { - break; - } - } else { - const std::string str = - "error: " + - result.result_json.dump(-1, ' ', false, json::error_handler_t::replace) + - "\n\n"; - LOG_VERBOSE("data stream", { - { "to_send", str } - }); - if (!sink.write(str.c_str(), str.size())) - { - llama.queue_results.remove_waiting_task_id(task_id); - return false; - } - break; - } - } + res.set_content(data.dump(), "application/json; charset=utf-8"); + }); - llama.queue_results.remove_waiting_task_id(task_id); - sink.done(); - return true; - }; - - auto on_complete = [task_id, &llama] (bool) - { - // cancel - llama.request_cancel(task_id); - llama.queue_results.remove_waiting_task_id(task_id); - }; - - res.set_chunked_content_provider("text/event-stream", chunked_content_provider, on_complete); - } - }); - - svr.Get("/v1/models", [¶ms, &model_meta](const httplib::Request& req, httplib::Response& res) - { - res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin")); - std::time_t t = std::time(0); - - json models = { - {"object", "list"}, - {"data", { - { - {"id", params.model_alias}, - {"object", "model"}, - {"created", t}, - {"owned_by", "llamacpp"}, - {"meta", model_meta} - }, - }} - }; - - res.set_content(models.dump(), "application/json; charset=utf-8"); - }); - - const auto chat_completions = [&llama, &validate_api_key, &sparams](const httplib::Request &req, httplib::Response &res) - { + svr.Post("/completion", [&ctx_server, &validate_api_key](const httplib::Request & req, httplib::Response & res) { res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin")); if (!validate_api_key(req, res)) { return; } - json data = oaicompat_completion_params_parse(llama.model, json::parse(req.body), sparams.chat_template); - const int task_id = llama.queue_tasks.get_new_id(); - llama.queue_results.add_waiting_task_id(task_id); - llama.request_completion(task_id, data, false, false, -1); + json data = json::parse(req.body); + + const int id_task = ctx_server.queue_tasks.get_new_id(); + + ctx_server.queue_results.add_waiting_task_id(id_task); + ctx_server.request_completion(id_task, -1, data, false, false); if (!json_value(data, "stream", false)) { - std::string completion_text; - task_result result = llama.queue_results.recv(task_id); - + server_task_result result = ctx_server.queue_results.recv(id_task); if (!result.error && result.stop) { - json oaicompat_result = format_final_response_oaicompat(data, result); - - res.set_content(oaicompat_result.dump(-1, ' ', false, - json::error_handler_t::replace), - "application/json; charset=utf-8"); + res.set_content(result.data.dump(-1, ' ', false, json::error_handler_t::replace), "application/json; charset=utf-8"); } else { res.status = 500; - res.set_content(result.result_json["content"], "text/plain; charset=utf-8"); + res.set_content(result.data["content"], "text/plain; charset=utf-8"); } - llama.queue_results.remove_waiting_task_id(task_id); - } else { - const auto chunked_content_provider = [task_id, &llama](size_t, httplib::DataSink &sink) { - while (true) { - task_result llama_result = llama.queue_results.recv(task_id); - if (!llama_result.error) { - std::vector result_array = format_partial_response_oaicompat( llama_result); - for (auto it = result_array.begin(); it != result_array.end(); ++it) - { + ctx_server.queue_results.remove_waiting_task_id(id_task); + } else { + const auto chunked_content_provider = [id_task, &ctx_server](size_t, httplib::DataSink & sink) { + while (true) { + server_task_result result = ctx_server.queue_results.recv(id_task); + if (!result.error) { + const std::string str = + "data: " + + result.data.dump(-1, ' ', false, json::error_handler_t::replace) + + "\n\n"; + + LOG_VERBOSE("data stream", { + { "to_send", str } + }); + + if (!sink.write(str.c_str(), str.size())) { + ctx_server.queue_results.remove_waiting_task_id(id_task); + return false; + } + + if (result.stop) { + break; + } + } else { + const std::string str = + "error: " + + result.data.dump(-1, ' ', false, json::error_handler_t::replace) + + "\n\n"; + + LOG_VERBOSE("data stream", { + { "to_send", str } + }); + + if (!sink.write(str.c_str(), str.size())) { + ctx_server.queue_results.remove_waiting_task_id(id_task); + return false; + } + + break; + } + } + + ctx_server.queue_results.remove_waiting_task_id(id_task); + sink.done(); + + return true; + }; + + auto on_complete = [id_task, &ctx_server] (bool) { + // cancel + ctx_server.request_cancel(id_task); + ctx_server.queue_results.remove_waiting_task_id(id_task); + }; + + res.set_chunked_content_provider("text/event-stream", chunked_content_provider, on_complete); + } + }); + + svr.Get("/v1/models", [¶ms, &model_meta](const httplib::Request & req, httplib::Response & res) { + res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin")); + + json models = { + {"object", "list"}, + {"data", { + { + {"id", params.model_alias}, + {"object", "model"}, + {"created", std::time(0)}, + {"owned_by", "llamacpp"}, + {"meta", model_meta} + }, + }} + }; + + res.set_content(models.dump(), "application/json; charset=utf-8"); + }); + + const auto chat_completions = [&ctx_server, &validate_api_key, &sparams](const httplib::Request & req, httplib::Response & res) { + res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin")); + if (!validate_api_key(req, res)) { + return; + } + + json data = oaicompat_completion_params_parse(ctx_server.model, json::parse(req.body), sparams.chat_template); + + const int id_task = ctx_server.queue_tasks.get_new_id(); + + ctx_server.queue_results.add_waiting_task_id(id_task); + ctx_server.request_completion(id_task, -1, data, false, false); + + if (!json_value(data, "stream", false)) { + server_task_result result = ctx_server.queue_results.recv(id_task); + + if (!result.error && result.stop) { + json result_oai = format_final_response_oaicompat(data, result.data); + + res.set_content(result_oai.dump(-1, ' ', false, json::error_handler_t::replace), "application/json; charset=utf-8"); + } else { + res.status = 500; + res.set_content(result.data["content"], "text/plain; charset=utf-8"); + } + ctx_server.queue_results.remove_waiting_task_id(id_task); + } else { + const auto chunked_content_provider = [id_task, &ctx_server](size_t, httplib::DataSink & sink) { + while (true) { + server_task_result result = ctx_server.queue_results.recv(id_task); + if (!result.error) { + std::vector result_array = format_partial_response_oaicompat(result.data); + + for (auto it = result_array.begin(); it != result_array.end(); ++it) { if (!it->empty()) { const std::string str = "data: " + @@ -3253,251 +3062,235 @@ int main(int argc, char **argv) "\n\n"; LOG_VERBOSE("data stream", {{"to_send", str}}); if (!sink.write(str.c_str(), str.size())) { - llama.queue_results.remove_waiting_task_id(task_id); + ctx_server.queue_results.remove_waiting_task_id(id_task); return false; } } } - if (llama_result.stop) { + if (result.stop) { break; } } else { const std::string str = "error: " + - llama_result.result_json.dump(-1, ' ', false, - json::error_handler_t::replace) + + result.data.dump(-1, ' ', false, json::error_handler_t::replace) + "\n\n"; LOG_VERBOSE("data stream", {{"to_send", str}}); if (!sink.write(str.c_str(), str.size())) { - llama.queue_results.remove_waiting_task_id(task_id); + ctx_server.queue_results.remove_waiting_task_id(id_task); return false; } break; } } sink.done(); - llama.queue_results.remove_waiting_task_id(task_id); + ctx_server.queue_results.remove_waiting_task_id(id_task); return true; }; - auto on_complete = [task_id, &llama](bool) { + auto on_complete = [id_task, &ctx_server](bool) { // cancel request - llama.request_cancel(task_id); - llama.queue_results.remove_waiting_task_id(task_id); + ctx_server.request_cancel(id_task); + ctx_server.queue_results.remove_waiting_task_id(id_task); }; res.set_chunked_content_provider("text/event-stream", chunked_content_provider, on_complete); } }; - svr.Post("/chat/completions", chat_completions); + svr.Post("/chat/completions", chat_completions); svr.Post("/v1/chat/completions", chat_completions); - svr.Post("/infill", [&llama, &validate_api_key](const httplib::Request &req, httplib::Response &res) - { - res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin")); - if (!validate_api_key(req, res)) { - return; - } - json data = json::parse(req.body); - const int task_id = llama.queue_tasks.get_new_id(); - llama.queue_results.add_waiting_task_id(task_id); - llama.request_completion(task_id, data, true, false, -1); - if (!json_value(data, "stream", false)) { - std::string completion_text; - task_result result = llama.queue_results.recv(task_id); - if (!result.error && result.stop) - { - res.set_content(result.result_json.dump(-1, ' ', false, json::error_handler_t::replace), "application/json; charset=utf-8"); - } - else - { - res.status = 404; - res.set_content(result.result_json["content"], "text/plain; charset=utf-8"); - } - llama.queue_results.remove_waiting_task_id(task_id); - } else { - const auto chunked_content_provider = [task_id, &llama](size_t, httplib::DataSink & sink) { - while (true) - { - task_result result = llama.queue_results.recv(task_id); - if (!result.error) { - const std::string str = - "data: " + - result.result_json.dump(-1, ' ', false, json::error_handler_t::replace) + - "\n\n"; - LOG_VERBOSE("data stream", { - { "to_send", str } - }); - if (!sink.write(str.c_str(), str.size())) - { - llama.queue_results.remove_waiting_task_id(task_id); - return false; - } - if (result.stop) - { - break; - } - } - else - { - break; - } - } - - llama.queue_results.remove_waiting_task_id(task_id); - sink.done(); - return true; - }; - - auto on_complete = [task_id, &llama] (bool) - { - // cancel - llama.request_cancel(task_id); - }; - - res.set_chunked_content_provider("text/event-stream", chunked_content_provider, on_complete); - } - }); - - svr.Options(R"(/.*)", [](const httplib::Request &, httplib::Response &res) - { return res.set_content("", "application/json; charset=utf-8"); }); - - svr.Post("/tokenize", [&llama](const httplib::Request &req, httplib::Response &res) - { - res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin")); - const json body = json::parse(req.body); - std::vector tokens; - if (body.count("content") != 0) - { - tokens = llama.tokenize(body["content"], false); - } - const json data = format_tokenizer_response(tokens); - return res.set_content(data.dump(), "application/json; charset=utf-8"); - }); - - svr.Post("/detokenize", [&llama](const httplib::Request &req, httplib::Response &res) - { - res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin")); - const json body = json::parse(req.body); - std::string content; - if (body.count("tokens") != 0) - { - const std::vector tokens = body["tokens"]; - content = tokens_to_str(llama.ctx, tokens.cbegin(), tokens.cend()); - } - - const json data = format_detokenized_response(content); - return res.set_content(data.dump(), "application/json; charset=utf-8"); - }); - - svr.Post("/embedding", [&llama](const httplib::Request &req, httplib::Response &res) - { - res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin")); - const json body = json::parse(req.body); - json prompt; - if (body.count("content") != 0) - { - prompt = body["content"]; - } - else - { - prompt = ""; - } - - json image_data; - if (body.count("image_data") != 0) { - image_data = body["image_data"]; - } - else - { - image_data = ""; - } - - // create and queue the task - const int task_id = llama.queue_tasks.get_new_id(); - llama.queue_results.add_waiting_task_id(task_id); - llama.request_completion(task_id, { {"prompt", prompt}, { "n_predict", 0}, {"image_data", image_data} }, false, true, -1); - - // get the result - task_result result = llama.queue_results.recv(task_id); - llama.queue_results.remove_waiting_task_id(task_id); - - // send the result - return res.set_content(result.result_json.dump(), "application/json; charset=utf-8"); - }); - - svr.Post("/v1/embeddings", [&llama](const httplib::Request &req, httplib::Response &res) - { - res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin")); - const json body = json::parse(req.body); - - json prompt; - if (body.count("input") != 0) - { - prompt = body["input"]; - // batch - if(prompt.is_array()) { - json data = json::array(); - int i = 0; - for (const json &elem : prompt) { - const int task_id = llama.queue_tasks.get_new_id(); - llama.queue_results.add_waiting_task_id(task_id); - llama.request_completion(task_id, { {"prompt", elem}, { "n_predict", 0} }, false, true, -1); - - // get the result - task_result result = llama.queue_results.recv(task_id); - llama.queue_results.remove_waiting_task_id(task_id); - - json embedding = json{ - {"embedding", json_value(result.result_json, "embedding", json::array())}, - {"index", i++}, - {"object", "embedding"} - }; - data.push_back(embedding); - } - json result = format_embeddings_response_oaicompat(body, data); - return res.set_content(result.dump(), "application/json; charset=utf-8"); - } - } - else - { - prompt = ""; - } - - // create and queue the task - const int task_id = llama.queue_tasks.get_new_id(); - llama.queue_results.add_waiting_task_id(task_id); - llama.request_completion(task_id, { {"prompt", prompt}, { "n_predict", 0}}, false, true, -1); - - // get the result - task_result result = llama.queue_results.recv(task_id); - llama.queue_results.remove_waiting_task_id(task_id); - - json data = json::array({json{ - {"embedding", json_value(result.result_json, "embedding", json::array())}, - {"index", 0}, - {"object", "embedding"} - }} - ); - - json root = format_embeddings_response_oaicompat(body, data); - - // send the result - return res.set_content(root.dump(), "application/json; charset=utf-8"); - }); - - // GG: if I put the main loop inside a thread, it crashes on the first request when build in Debug!? - // "Bus error: 10" - this is on macOS, it does not crash on Linux - //std::thread t2([&]() - /*{ - bool running = true; - while (running) - { - running = llama.update_slots(); + svr.Post("/infill", [&ctx_server, &validate_api_key](const httplib::Request & req, httplib::Response & res) { + res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin")); + if (!validate_api_key(req, res)) { + return; } - }*/ - //); + + json data = json::parse(req.body); + + const int id_task = ctx_server.queue_tasks.get_new_id(); + + ctx_server.queue_results.add_waiting_task_id(id_task); + ctx_server.request_completion(id_task, -1, data, true, false); + + if (!json_value(data, "stream", false)) { + server_task_result result = ctx_server.queue_results.recv(id_task); + if (!result.error && result.stop) { + res.set_content(result.data.dump(-1, ' ', false, json::error_handler_t::replace), "application/json; charset=utf-8"); + } else { + res.status = 404; + res.set_content(result.data["content"], "text/plain; charset=utf-8"); + } + + ctx_server.queue_results.remove_waiting_task_id(id_task); + } else { + const auto chunked_content_provider = [id_task, &ctx_server](size_t, httplib::DataSink & sink) { + while (true) { + server_task_result result = ctx_server.queue_results.recv(id_task); + if (!result.error) { + const std::string str = + "data: " + + result.data.dump(-1, ' ', false, json::error_handler_t::replace) + + "\n\n"; + + LOG_VERBOSE("data stream", { + { "to_send", str } + }); + + if (!sink.write(str.c_str(), str.size())) { + ctx_server.queue_results.remove_waiting_task_id(id_task); + return false; + } + + if (result.stop) { + break; + } + } else { + break; + } + } + + ctx_server.queue_results.remove_waiting_task_id(id_task); + sink.done(); + + return true; + }; + + auto on_complete = [id_task, &ctx_server] (bool) { + ctx_server.request_cancel(id_task); + }; + + res.set_chunked_content_provider("text/event-stream", chunked_content_provider, on_complete); + } + }); + + svr.Options(R"(/.*)", [](const httplib::Request &, httplib::Response & res) { + return res.set_content("", "application/json; charset=utf-8"); + }); + + svr.Post("/tokenize", [&ctx_server](const httplib::Request & req, httplib::Response & res) { + res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin")); + const json body = json::parse(req.body); + + std::vector tokens; + if (body.count("content") != 0) { + tokens = ctx_server.tokenize(body["content"], false); + } + const json data = format_tokenizer_response(tokens); + return res.set_content(data.dump(), "application/json; charset=utf-8"); + }); + + svr.Post("/detokenize", [&ctx_server](const httplib::Request & req, httplib::Response & res) { + res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin")); + const json body = json::parse(req.body); + + std::string content; + if (body.count("tokens") != 0) { + const std::vector tokens = body["tokens"]; + content = tokens_to_str(ctx_server.ctx, tokens.cbegin(), tokens.cend()); + } + + const json data = format_detokenized_response(content); + return res.set_content(data.dump(), "application/json; charset=utf-8"); + }); + + svr.Post("/embedding", [¶ms, &ctx_server](const httplib::Request & req, httplib::Response & res) { + res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin")); + if (!params.embedding) { + res.status = 501; + res.set_content("This server does not support embeddings. Start it with `--embeddings`", "text/plain; charset=utf-8"); + return; + } + + const json body = json::parse(req.body); + + json prompt; + if (body.count("content") != 0) { + prompt = body["content"]; + } else { + prompt = ""; + } + + // create and queue the task + const int id_task = ctx_server.queue_tasks.get_new_id(); + + ctx_server.queue_results.add_waiting_task_id(id_task); + ctx_server.request_completion(id_task, -1, { {"prompt", prompt}, { "n_predict", 0} }, false, true); + + // get the result + server_task_result result = ctx_server.queue_results.recv(id_task); + ctx_server.queue_results.remove_waiting_task_id(id_task); + + // send the result + return res.set_content(result.data.dump(), "application/json; charset=utf-8"); + }); + + svr.Post("/v1/embeddings", [¶ms, &ctx_server](const httplib::Request & req, httplib::Response & res) { + res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin")); + if (!params.embedding) { + res.status = 501; + res.set_content("This server does not support embeddings. Start it with `--embeddings`", "text/plain; charset=utf-8"); + return; + } + + const json body = json::parse(req.body); + + json prompt; + if (body.count("input") != 0) { + prompt = body["input"]; + if (prompt.is_array()) { + json data = json::array(); + + int i = 0; + for (const json & elem : prompt) { + const int id_task = ctx_server.queue_tasks.get_new_id(); + + ctx_server.queue_results.add_waiting_task_id(id_task); + ctx_server.request_completion(id_task, -1, { {"prompt", elem}, { "n_predict", 0} }, false, true); + + // get the result + server_task_result result = ctx_server.queue_results.recv(id_task); + ctx_server.queue_results.remove_waiting_task_id(id_task); + + json embedding = json{ + {"embedding", json_value(result.data, "embedding", json::array())}, + {"index", i++}, + {"object", "embedding"} + }; + + data.push_back(embedding); + } + + json result = format_embeddings_response_oaicompat(body, data); + + return res.set_content(result.dump(), "application/json; charset=utf-8"); + } + } else { + prompt = ""; + } + + // create and queue the task + const int id_task = ctx_server.queue_tasks.get_new_id(); + + ctx_server.queue_results.add_waiting_task_id(id_task); + ctx_server.request_completion(id_task, -1, { {"prompt", prompt}, { "n_predict", 0}}, false, true); + + // get the result + server_task_result result = ctx_server.queue_results.recv(id_task); + ctx_server.queue_results.remove_waiting_task_id(id_task); + + json data = json::array({json{ + {"embedding", json_value(result.data, "embedding", json::array())}, + {"index", 0}, + {"object", "embedding"} + }} + ); + + json root = format_embeddings_response_oaicompat(body, data); + + return res.set_content(root.dump(), "application/json; charset=utf-8"); + }); if (sparams.n_threads_http < 1) { // +2 threads for monitoring endpoints @@ -3507,34 +3300,33 @@ int main(int argc, char **argv) svr.new_task_queue = [&sparams] { return new httplib::ThreadPool(sparams.n_threads_http); }; LOG_INFO("HTTP server listening", log_data); + // run the HTTP server in a thread - see comment below - std::thread t([&]() - { - if (!svr.listen_after_bind()) - { - state.store(SERVER_STATE_ERROR); - return 1; - } + std::thread t([&]() { + if (!svr.listen_after_bind()) { + state.store(SERVER_STATE_ERROR); + return 1; + } - return 0; - }); + return 0; + }); - llama.queue_tasks.on_new_task(std::bind( - &llama_server_context::process_single_task, &llama, std::placeholders::_1)); - llama.queue_tasks.on_finish_multitask(std::bind( - &llama_server_context::on_finish_multitask, &llama, std::placeholders::_1)); - llama.queue_tasks.on_run_slots(std::bind( - &llama_server_context::update_slots, &llama)); - llama.queue_results.on_multitask_update(std::bind( - &llama_server_queue::update_multitask, - &llama.queue_tasks, + ctx_server.queue_tasks.on_new_task(std::bind( + &server_context::process_single_task, &ctx_server, std::placeholders::_1)); + ctx_server.queue_tasks.on_finish_multitask(std::bind( + &server_context::on_finish_multitask, &ctx_server, std::placeholders::_1)); + ctx_server.queue_tasks.on_run_slots(std::bind( + &server_context::update_slots, &ctx_server)); + ctx_server.queue_results.on_multitask_update(std::bind( + &server_queue::update_multitask, + &ctx_server.queue_tasks, std::placeholders::_1, std::placeholders::_2, std::placeholders::_3 )); shutdown_handler = [&](int) { - llama.queue_tasks.terminate(); + ctx_server.queue_tasks.terminate(); }; #if defined (__unix__) || (defined (__APPLE__) && defined (__MACH__)) @@ -3549,10 +3341,13 @@ int main(int argc, char **argv) }; SetConsoleCtrlHandler(reinterpret_cast(console_ctrl_handler), true); #endif - llama.queue_tasks.start_loop(); + + ctx_server.queue_tasks.start_loop(); + svr.stop(); t.join(); llama_backend_free(); + return 0; } diff --git a/examples/server/tests/features/embeddings.feature b/examples/server/tests/features/embeddings.feature new file mode 100644 index 000000000..b47661e94 --- /dev/null +++ b/examples/server/tests/features/embeddings.feature @@ -0,0 +1,94 @@ +@llama.cpp +@embeddings +Feature: llama.cpp server + + Background: Server startup + Given a server listening on localhost:8080 + And a model file bert-bge-small/ggml-model-f16.gguf from HF repo ggml-org/models + And a model alias bert-bge-small + And 42 as server seed + And 2 slots + And 1024 as batch size + And 2048 KV cache size + And embeddings extraction + Then the server is starting + Then the server is healthy + + Scenario: Embedding + When embeddings are computed for: + """ + What is the capital of Bulgaria ? + """ + Then embeddings are generated + + Scenario: OAI Embeddings compatibility + Given a model bert-bge-small + When an OAI compatible embeddings computation request for: + """ + What is the capital of Spain ? + """ + Then embeddings are generated + + Scenario: OAI Embeddings compatibility with multiple inputs + Given a model bert-bge-small + Given a prompt: + """ + In which country Paris is located ? + """ + And a prompt: + """ + Is Madrid the capital of Spain ? + """ + When an OAI compatible embeddings computation request for multiple inputs + Then embeddings are generated + + Scenario: Multi users embeddings + Given a prompt: + """ + Write a very long story about AI. + """ + And a prompt: + """ + Write another very long music lyrics. + """ + And a prompt: + """ + Write a very long poem. + """ + And a prompt: + """ + Write a very long joke. + """ + Given concurrent embedding requests + Then the server is busy + Then the server is idle + Then all embeddings are generated + + Scenario: Multi users OAI compatibility embeddings + Given a prompt: + """ + In which country Paris is located ? + """ + And a prompt: + """ + Is Madrid the capital of Spain ? + """ + And a prompt: + """ + What is the biggest US city ? + """ + And a prompt: + """ + What is the capital of Bulgaria ? + """ + And a model bert-bge-small + Given concurrent OAI embedding requests + Then the server is busy + Then the server is idle + Then all embeddings are generated + + Scenario: All embeddings should be the same + Given 10 fixed prompts + And a model bert-bge-small + Given concurrent OAI embedding requests + Then all embeddings are the same diff --git a/examples/server/tests/features/parallel.feature b/examples/server/tests/features/parallel.feature index 86cdf7282..066698c8e 100644 --- a/examples/server/tests/features/parallel.feature +++ b/examples/server/tests/features/parallel.feature @@ -9,7 +9,6 @@ Feature: Parallel And 512 as batch size And 64 KV cache size And 2 slots - And embeddings extraction And continuous batching Then the server is starting Then the server is healthy @@ -99,48 +98,3 @@ Feature: Parallel Then the server is busy Then the server is idle Then all prompts are predicted - - Scenario: Multi users embeddings - Given a prompt: - """ - Write a very long story about AI. - """ - And a prompt: - """ - Write another very long music lyrics. - """ - And a prompt: - """ - Write a very long poem. - """ - And a prompt: - """ - Write a very long joke. - """ - Given concurrent embedding requests - Then the server is busy - Then the server is idle - Then all embeddings are generated - - Scenario: Multi users OAI compatibility embeddings - Given a prompt: - """ - In which country Paris is located ? - """ - And a prompt: - """ - Is Madrid the capital of Spain ? - """ - And a prompt: - """ - What is the biggest US city ? - """ - And a prompt: - """ - What is the capital of Bulgaria ? - """ - And a model tinyllama-2 - Given concurrent OAI embedding requests - Then the server is busy - Then the server is idle - Then all embeddings are generated diff --git a/examples/server/tests/features/server.feature b/examples/server/tests/features/server.feature index 7c977bcce..f3b758c79 100644 --- a/examples/server/tests/features/server.feature +++ b/examples/server/tests/features/server.feature @@ -49,34 +49,6 @@ Feature: llama.cpp server | llama-2 | Book | What is the best book | 8 | (Mom\|what)+ | 8 | disabled | | codellama70b | You are a coding assistant. | Write the fibonacci function in c++. | 64 | (thanks\|happy\|bird)+ | 32 | enabled | - Scenario: Embedding - When embeddings are computed for: - """ - What is the capital of Bulgaria ? - """ - Then embeddings are generated - - Scenario: OAI Embeddings compatibility - Given a model tinyllama-2 - When an OAI compatible embeddings computation request for: - """ - What is the capital of Spain ? - """ - Then embeddings are generated - - Scenario: OAI Embeddings compatibility with multiple inputs - Given a model tinyllama-2 - Given a prompt: - """ - In which country Paris is located ? - """ - And a prompt: - """ - Is Madrid the capital of Spain ? - """ - When an OAI compatible embeddings computation request for multiple inputs - Then embeddings are generated - Scenario: Tokenize / Detokenize When tokenizing: """ diff --git a/examples/server/tests/features/steps/steps.py b/examples/server/tests/features/steps/steps.py index 319527802..a0b2ffdfe 100644 --- a/examples/server/tests/features/steps/steps.py +++ b/examples/server/tests/features/steps/steps.py @@ -10,6 +10,7 @@ from contextlib import closing from re import RegexFlag import aiohttp +import numpy as np import openai from behave import step from behave.api.async_step import async_run_until_complete @@ -24,6 +25,9 @@ def step_server_config(context, server_fqdn, server_port): if 'PORT' in os.environ: context.server_port = int(os.environ['PORT']) print(f"$PORT set, overriding server port with to {context.server_port}") + if 'FQDN' in os.environ: + context.server_fqdn = os.environ['FQDN'] + print(f"$FQDN set, overriding server fqdn with to {context.server_fqdn}") context.base_url = f'http://{context.server_fqdn}:{context.server_port}' @@ -34,6 +38,7 @@ def step_server_config(context, server_fqdn, server_port): context.n_ga_w = None context.n_gpu_layer = None context.n_predict = None + context.n_prompts = 0 context.n_server_predict = None context.n_slots = None context.prompt_prefix = None @@ -202,6 +207,7 @@ def step_n_tokens_predicted(context, predicted_n): @step(u'a user prompt {user_prompt}') def step_user_prompt(context, user_prompt): context.prompts.append(user_prompt) + context.n_prompts = len(context.prompts) @step(u'a system prompt {system_prompt}') @@ -290,6 +296,12 @@ def step_prompt_passkey(context): context.prompt_passkey = context.text +@step(u'{n_prompts:d} fixed prompts') +def step_fixed_prompts(context, n_prompts): + context.prompts.extend([str(0)*(context.n_batch if context.n_batch is not None else 512) for i in range(n_prompts)]) + context.n_prompts = n_prompts + + @step(u'a "{passkey}" passkey challenge prompt with the passkey inserted every {i_pos:d} junk') def step_prompt_passkey(context, passkey, i_pos): prompt = "" @@ -301,6 +313,7 @@ def step_prompt_passkey(context, passkey, i_pos): passkey_highlight = "\x1b[33m" + passkey + "\x1b[0m" print(f"Passkey challenge:\n```{prompt.replace(passkey, passkey_highlight)}```\n") context.prompts.append(context.prompt_prefix + prompt + context.prompt_suffix) + context.n_prompts = len(context.prompts) @step(u'an OAI compatible chat completions request with {api_error} api error') @@ -341,11 +354,13 @@ async def step_oai_chat_completions(context, api_error): @step(u'a prompt') def step_a_prompt(context): context.prompts.append(context.text) + context.n_prompts = len(context.prompts) @step(u'a prompt {prompt}') def step_a_prompt_prompt(context, prompt): context.prompts.append(prompt) + context.n_prompts = len(context.prompts) @step(u'concurrent completion requests') @@ -430,25 +445,47 @@ async def all_prompts_are_predicted(context, expected_predicted_n=None): @step(u'embeddings are computed for') @async_run_until_complete async def step_compute_embedding(context): + context.n_prompts = 1 context.embeddings = await request_embedding(context.text, base_url=context.base_url) +@step(u'all embeddings are the same') +@async_run_until_complete +async def step_all_embeddings_are_the_same(context): + n_embedding_requests = await gather_tasks_results(context) + assert n_embedding_requests > 0 + embeddings = [] + for i in range(n_embedding_requests): + embedding = context.tasks_result.pop().pop() + embeddings.append(embedding) + assert_embeddings(embedding) + n = len(embeddings) + for i in range(n-1): + for j in range(i+1, n): + embedding1 = np.array(embeddings[i]) + embedding2 = np.array(embeddings[j]) + if context.debug: + print(f"embedding1: {embedding1[-8:]}\n") + print(f"embedding2: {embedding2[-8:]}\n") + similarity = np.dot(embedding1, embedding2) / (np.linalg.norm(embedding1) * np.linalg.norm(embedding2)) + msg = f"Similarity between {i} and {j}: {similarity:.10f}" + if context.debug: + print(f"{msg}\n") + assert np.isclose(similarity, 1.0, rtol=1e-05, atol=1e-08, equal_nan=False), msg + @step(u'embeddings are generated') def step_assert_embeddings(context): - if len(context.prompts) == 0: - assert_embeddings(context.embeddings) - else: - assert len(context.embeddings) == len(context.prompts), (f"unexpected response:\n" - f"context.prompts={context.prompts}\n" - f"context.embeddings={context.embeddings}") - for embedding in context.embeddings: - context.prompts.pop() - assert_embeddings(embedding) + assert context.n_prompts == len(context.embeddings), (f"unexpected response:\n" + f"context.n_prompts={context.n_prompts}\n" + f"context.embeddings={context.embeddings}") + for embedding in context.embeddings: + assert_embeddings(embedding) @step(u'an OAI compatible embeddings computation request for') @async_run_until_complete async def step_oai_compute_embeddings(context): + context.n_prompts = 1 context.embeddings = await request_oai_embeddings(context.text, base_url=context.base_url, user_api_key=context.user_api_key, @@ -462,6 +499,7 @@ async def step_oai_compute_embeddings_multiple_inputs(context): base_url=context.base_url, user_api_key=context.user_api_key, model=context.model) + context.prompts.clear() @step(u'concurrent embedding requests') @@ -488,9 +526,9 @@ async def step_concurrent_oai_embedding_requests(context): @async_run_until_complete() async def all_embeddings_are_generated(context): n_embedding_requests = await gather_tasks_results(context) - assert n_embedding_requests > 0 + assert n_embedding_requests == context.n_prompts for i in range(n_embedding_requests): - assert_embeddings(context.tasks_result.pop()) + assert_embeddings(context.tasks_result.pop().pop()) @step(u'tokenizing') @@ -588,11 +626,11 @@ def step_supported_models(context, i_model, param, preposition, param_value): async def concurrent_requests(context, f_completion, *args, **kwargs): - n_prompts = len(context.prompts) + context.n_prompts = len(context.prompts) if context.debug: - print(f"starting {n_prompts} concurrent completion requests...") - assert n_prompts > 0 - for prompt_no in range(n_prompts): + print(f"starting {context.n_prompts} concurrent completion requests...") + assert context.n_prompts > 0 + for prompt_no in range(context.n_prompts): shifted_args = [context.prompts.pop(), *args] context.concurrent_tasks.append(asyncio.create_task(f_completion(*shifted_args, **kwargs))) await asyncio.sleep(0.1) @@ -765,7 +803,7 @@ async def request_embedding(content, base_url=None): }) as response: assert response.status == 200 response_json = await response.json() - return response_json['embedding'] + return [response_json['embedding']] async def request_oai_embeddings(input, @@ -775,6 +813,7 @@ async def request_oai_embeddings(input, user_api_key = user_api_key if user_api_key is not None else 'nope' if async_client: origin = 'llama.cpp' + headers=[] if user_api_key is not None: headers = {'Authorization': f'Bearer {user_api_key}', 'Origin': origin} async with aiohttp.ClientSession() as session: @@ -783,14 +822,21 @@ async def request_oai_embeddings(input, "input": input, "model": model, }, - headers=headers) as response: + headers=headers, + timeout=3600) as response: assert response.status == 200, f"received status code not expected: {response.status}" assert response.headers['Access-Control-Allow-Origin'] == origin assert response.headers['Content-Type'] == "application/json; charset=utf-8" response_json = await response.json() assert response_json['model'] == model, f"invalid model received: {response_json['model']}" assert response_json['object'] == 'list' - return response_json['data'] + if isinstance(input, collections.abc.Sequence): + embeddings = [] + for an_oai_embeddings in response_json['data']: + embeddings.append(an_oai_embeddings['embedding']) + else: + embeddings = [response_json['data']['embedding']] + return embeddings else: openai.api_key = user_api_key openai.api_base = f'{base_url}/v1' @@ -804,7 +850,7 @@ async def request_oai_embeddings(input, for an_oai_embeddings in oai_embeddings.data: embeddings.append(an_oai_embeddings.embedding) else: - embeddings = oai_embeddings.data.embedding + embeddings = [oai_embeddings.data.embedding] return embeddings @@ -899,6 +945,8 @@ def assert_embeddings(embeddings): assert len(embeddings) > 0 embeddings_computed = False for emb in embeddings: + if not isinstance(emb, float): + assert False, f"Bad embeddings: {embeddings}" if emb != 0: embeddings_computed = True assert embeddings_computed, f"Embeddings: {embeddings}" diff --git a/examples/server/tests/requirements.txt b/examples/server/tests/requirements.txt index 5d4210164..2e4f42ad2 100644 --- a/examples/server/tests/requirements.txt +++ b/examples/server/tests/requirements.txt @@ -1,5 +1,6 @@ aiohttp~=3.9.3 behave~=1.2.6 huggingface_hub~=0.20.3 +numpy~=1.24.4 openai~=0.25.0 prometheus-client~=0.20.0 diff --git a/examples/server/utils.hpp b/examples/server/utils.hpp index b6e49d8b9..df0a27782 100644 --- a/examples/server/utils.hpp +++ b/examples/server/utils.hpp @@ -1,15 +1,16 @@ #pragma once -#include -#include -#include -#include -#include -#include +#include "llama.h" +#include "common.h" #include "json.hpp" -#include "../llava/clip.h" +#include +#include +#include +#include + +#define DEFAULT_OAICOMPAT_MODEL "gpt-3.5-turbo-0613" using json = nlohmann::json; @@ -37,83 +38,35 @@ extern bool server_log_json; #define LOG_WARNING(MSG, ...) server_log("WARN", __func__, __LINE__, MSG, __VA_ARGS__) #define LOG_INFO( MSG, ...) server_log("INFO", __func__, __LINE__, MSG, __VA_ARGS__) -enum server_state { - SERVER_STATE_LOADING_MODEL, // Server is starting up, model not fully loaded yet - SERVER_STATE_READY, // Server is ready and model is loaded - SERVER_STATE_ERROR // An error occurred, load_model failed -}; - -enum task_type { - TASK_TYPE_COMPLETION, - TASK_TYPE_CANCEL, - TASK_TYPE_NEXT_RESPONSE, - TASK_TYPE_METRICS -}; - -struct task_server { - int id = -1; // to be filled by llama_server_queue - int target_id; - task_type type; - json data; - bool infill_mode = false; - bool embedding_mode = false; - int multitask_id = -1; -}; - -struct task_result { - int id; - int multitask_id = -1; - bool stop; - bool error; - json result_json; -}; - -struct task_multi { - int id; - std::set subtasks_remaining{}; - std::vector results{}; -}; - -// completion token output with probabilities -struct completion_token_output { - struct token_prob - { - llama_token tok; - float prob; - }; - - std::vector probs; - llama_token tok; - std::string text_to_send; -}; - -struct token_translator { - llama_context * ctx; - std::string operator()(llama_token tok) const { return llama_token_to_piece(ctx, tok); } - std::string operator()(const completion_token_output &cto) const { return (*this)(cto.tok); } -}; +template +static T json_value(const json &body, const std::string &key, const T &default_value) { + // Fallback null to default value + return body.contains(key) && !body.at(key).is_null() + ? body.value(key, default_value) + : default_value; +} static inline void server_log(const char *level, const char *function, int line, const char *message, const nlohmann::ordered_json &extra) { std::stringstream ss_tid; ss_tid << std::this_thread::get_id(); json log = nlohmann::ordered_json{ - {"tid", ss_tid.str()}, + {"tid", ss_tid.str()}, {"timestamp", time(nullptr)}, }; if (server_log_json) { - log.merge_patch( - { - {"level", level}, - {"function", function}, - {"line", line}, - {"msg", message}, - }); + log.merge_patch( { + {"level", level}, + {"function", function}, + {"line", line}, + {"msg", message}, + }); + if (!extra.empty()) { log.merge_patch(extra); } - std::cout << log.dump(-1, ' ', false, json::error_handler_t::replace) << "\n" << std::flush; + printf("%s\n", log.dump(-1, ' ', false, json::error_handler_t::replace).c_str()); } else { char buf[1024]; snprintf(buf, 1024, "%4s [%24s] %s", level, function, message); @@ -136,22 +89,13 @@ static inline void server_log(const char *level, const char *function, int line, } // -// server utils +// chat template utils // -template -static T json_value(const json &body, const std::string &key, const T &default_value) { - // Fallback null to default value - return body.contains(key) && !body.at(key).is_null() - ? body.value(key, default_value) - : default_value; -} - // Check if the template supplied via "--chat-template" is supported or not. Returns true if it's valid inline bool verify_custom_template(const std::string & tmpl) { llama_chat_message chat[] = {{"user", "test"}}; - std::vector buf(1); - int res = llama_chat_apply_template(nullptr, tmpl.c_str(), chat, 1, true, buf.data(), buf.size()); + int res = llama_chat_apply_template(nullptr, tmpl.c_str(), chat, 1, true, nullptr, 0); return res >= 0; } @@ -163,7 +107,7 @@ inline std::string format_chat(const struct llama_model * model, const std::stri std::vector chat(messages.size()); for (size_t i = 0; i < messages.size(); ++i) { - auto &curr_msg = messages[i]; + const auto & curr_msg = messages[i]; str[i*2 + 0] = json_value(curr_msg, "role", std::string("")); str[i*2 + 1] = json_value(curr_msg, "content", std::string("")); alloc_size += str[i*2 + 1].length(); @@ -183,261 +127,13 @@ inline std::string format_chat(const struct llama_model * model, const std::stri res = llama_chat_apply_template(model, ptr_tmpl, chat.data(), chat.size(), true, buf.data(), buf.size()); } - std::string formatted_chat(buf.data(), res); + const std::string formatted_chat(buf.data(), res); + LOG_VERBOSE("formatted_chat", {{"text", formatted_chat.c_str()}}); return formatted_chat; } -// -// work queue utils -// - -struct llama_server_queue { - int id = 0; - std::mutex mutex_tasks; - bool running; - // queues - std::vector queue_tasks; - std::vector queue_tasks_deferred; - std::vector queue_multitasks; - std::condition_variable condition_tasks; - // callback functions - std::function callback_new_task; - std::function callback_finish_multitask; - std::function callback_run_slots; - - // Add a new task to the end of the queue - int post(task_server task) { - std::unique_lock lock(mutex_tasks); - if (task.id == -1) { - task.id = id++; - LOG_VERBOSE("new task id", {{"new_id", task.id}}); - } - queue_tasks.push_back(std::move(task)); - condition_tasks.notify_one(); - return task.id; - } - - // Add a new task, but defer until one slot is available - void defer(task_server task) { - std::unique_lock lock(mutex_tasks); - queue_tasks_deferred.push_back(std::move(task)); - } - - // Get the next id for creating anew task - int get_new_id() { - std::unique_lock lock(mutex_tasks); - int new_id = id++; - LOG_VERBOSE("new task id", {{"new_id", new_id}}); - return new_id; - } - - // Register function to process a new task - void on_new_task(std::function callback) { - callback_new_task = callback; - } - - // Register function to process a multitask when it is finished - void on_finish_multitask(std::function callback) { - callback_finish_multitask = callback; - } - - // Register the function to be called when all slots data is ready to be processed - void on_run_slots(std::function callback) { - callback_run_slots = callback; - } - - // Call when the state of one slot is changed - void notify_slot_changed() { - // move deferred tasks back to main loop - std::unique_lock lock(mutex_tasks); - for (auto & task : queue_tasks_deferred) { - queue_tasks.push_back(std::move(task)); - } - queue_tasks_deferred.clear(); - } - - // end the start_loop routine - void terminate() { - { - std::unique_lock lock(mutex_tasks); - running = false; - } - condition_tasks.notify_all(); - } - - /** - * Main loop consists of these steps: - * - Wait until a new task arrives - * - Process the task (i.e. maybe copy data into slot) - * - Check if multitask is finished - * - Run all slots - */ - void start_loop() { - running = true; - while (true) { - LOG_VERBOSE("new task may arrive", {}); - { - while (true) - { - std::unique_lock lock(mutex_tasks); - if (queue_tasks.empty()) { - lock.unlock(); - break; - } - task_server task = queue_tasks.front(); - queue_tasks.erase(queue_tasks.begin()); - lock.unlock(); - LOG_VERBOSE("callback_new_task", {{"task_id", task.id}}); - callback_new_task(task); - } - LOG_VERBOSE("update_multitasks", {}); - // check if we have any finished multitasks - auto queue_iterator = queue_multitasks.begin(); - while (queue_iterator != queue_multitasks.end()) - { - if (queue_iterator->subtasks_remaining.empty()) - { - // all subtasks done == multitask is done - task_multi current_multitask = *queue_iterator; - callback_finish_multitask(current_multitask); - // remove this multitask - queue_iterator = queue_multitasks.erase(queue_iterator); - } - else - { - ++queue_iterator; - } - } - // all tasks in the current loop is processed, slots data is now ready - LOG_VERBOSE("callback_run_slots", {}); - callback_run_slots(); - } - LOG_VERBOSE("wait for new task", {}); - // wait for new task - { - std::unique_lock lock(mutex_tasks); - if (queue_tasks.empty()) { - if (!running) { - LOG_VERBOSE("ending start_loop", {}); - return; - } - condition_tasks.wait(lock, [&]{ - return (!queue_tasks.empty() || !running); - }); - } - } - } - } - - // - // functions to manage multitasks - // - - // add a multitask by specifying the id of all subtask (subtask is a task_server) - void add_multitask(int multitask_id, std::vector& sub_ids) - { - std::lock_guard lock(mutex_tasks); - task_multi multi; - multi.id = multitask_id; - std::copy(sub_ids.begin(), sub_ids.end(), std::inserter(multi.subtasks_remaining, multi.subtasks_remaining.end())); - queue_multitasks.push_back(multi); - } - - // updatethe remaining subtasks, while appending results to multitask - void update_multitask(int multitask_id, int subtask_id, task_result& result) - { - std::lock_guard lock(mutex_tasks); - for (auto& multitask : queue_multitasks) - { - if (multitask.id == multitask_id) - { - multitask.subtasks_remaining.erase(subtask_id); - multitask.results.push_back(result); - } - } - } -}; - -struct llama_server_response { - typedef std::function callback_multitask_t; - callback_multitask_t callback_update_multitask; - // for keeping track of all tasks waiting for the result - std::set waiting_task_ids; - // the main result queue - std::vector queue_results; - std::mutex mutex_results; - std::condition_variable condition_results; - - // add the task_id to the list of tasks waiting for response - void add_waiting_task_id(int task_id) { - LOG_VERBOSE("waiting for task id", {{"task_id", task_id}}); - std::unique_lock lock(mutex_results); - waiting_task_ids.insert(task_id); - } - - // when the request is finished, we can remove task associated with it - void remove_waiting_task_id(int task_id) { - LOG_VERBOSE("remove waiting for task id", {{"task_id", task_id}}); - std::unique_lock lock(mutex_results); - waiting_task_ids.erase(task_id); - } - - // This function blocks the thread until there is a response for this task_id - task_result recv(int task_id) { - while (true) - { - std::unique_lock lock(mutex_results); - condition_results.wait(lock, [&]{ - return !queue_results.empty(); - }); - - for (int i = 0; i < (int) queue_results.size(); i++) - { - if (queue_results[i].id == task_id) - { - assert(queue_results[i].multitask_id == -1); - task_result res = queue_results[i]; - queue_results.erase(queue_results.begin() + i); - return res; - } - } - } - - // should never reach here - } - - // Register the function to update multitask - void on_multitask_update(callback_multitask_t callback) { - callback_update_multitask = callback; - } - - // Send a new result to a waiting task_id - void send(task_result result) { - std::unique_lock lock(mutex_results); - LOG_VERBOSE("send new result", {{"task_id", result.id}}); - for (auto& task_id : waiting_task_ids) { - // LOG_TEE("waiting task id %i \n", task_id); - // for now, tasks that have associated parent multitasks just get erased once multitask picks up the result - if (result.multitask_id == task_id) - { - LOG_VERBOSE("callback_update_multitask", {{"task_id", task_id}}); - callback_update_multitask(task_id, result.id, result); - continue; - } - - if (result.id == task_id) - { - LOG_VERBOSE("queue_results.push_back", {{"task_id", task_id}}); - queue_results.push_back(result); - condition_results.notify_all(); - return; - } - } - } -}; - // // base64 utils (TODO: move to common in the future) // @@ -447,13 +143,11 @@ static const std::string base64_chars = "abcdefghijklmnopqrstuvwxyz" "0123456789+/"; -static inline bool is_base64(uint8_t c) -{ +static inline bool is_base64(uint8_t c) { return (isalnum(c) || (c == '+') || (c == '/')); } -static inline std::vector base64_decode(const std::string & encoded_string) -{ +static inline std::vector base64_decode(const std::string & encoded_string) { int i = 0; int j = 0; int in_ = 0; @@ -465,13 +159,10 @@ static inline std::vector base64_decode(const std::string & encoded_str std::vector ret; - while (in_len-- && (encoded_string[in_] != '=') && is_base64(encoded_string[in_])) - { + while (in_len-- && (encoded_string[in_] != '=') && is_base64(encoded_string[in_])) { char_array_4[i++] = encoded_string[in_]; in_++; - if (i == 4) - { - for (i = 0; i <4; i++) - { + if (i == 4) { + for (i = 0; i < 4; i++) { char_array_4[i] = base64_chars.find(char_array_4[i]); } @@ -479,23 +170,20 @@ static inline std::vector base64_decode(const std::string & encoded_str char_array_3[1] = ((char_array_4[1] & 0xf) << 4) + ((char_array_4[2] & 0x3c) >> 2); char_array_3[2] = ((char_array_4[2] & 0x3) << 6) + char_array_4[3]; - for (i = 0; (i < 3); i++) - { + for (i = 0; (i < 3); i++) { ret.push_back(char_array_3[i]); } + i = 0; } } - if (i) - { - for (j = i; j <4; j++) - { + if (i) { + for (j = i; j < 4; j++) { char_array_4[j] = 0; } - for (j = 0; j <4; j++) - { + for (j = 0; j < 4; j++) { char_array_4[j] = base64_chars.find(char_array_4[j]); } @@ -503,8 +191,7 @@ static inline std::vector base64_decode(const std::string & encoded_str char_array_3[1] = ((char_array_4[1] & 0xf) << 4) + ((char_array_4[2] & 0x3c) >> 2); char_array_3[2] = ((char_array_4[2] & 0x3) << 6) + char_array_4[3]; - for (j = 0; (j < i - 1); j++) - { + for (j = 0; j < i - 1; j++) { ret.push_back(char_array_3[j]); } } @@ -516,8 +203,7 @@ static inline std::vector base64_decode(const std::string & encoded_str // random string / id // -static std::string random_string() -{ +static std::string random_string() { static const std::string str("0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"); std::random_device rd; @@ -532,10 +218,10 @@ static std::string random_string() return result; } -static std::string gen_chatcmplid() -{ +static std::string gen_chatcmplid() { std::stringstream chatcmplid; chatcmplid << "chatcmpl-" << random_string(); + return chatcmplid.str(); } @@ -543,91 +229,316 @@ static std::string gen_chatcmplid() // other common utils // -static size_t common_part(const std::vector &a, const std::vector &b) -{ +static size_t common_part(const std::vector & a, const std::vector & b) { size_t i; - for (i = 0; i < a.size() && i < b.size() && a[i] == b[i]; i++) - { - } + for (i = 0; i < a.size() && i < b.size() && a[i] == b[i]; i++) {} + return i; } -static bool ends_with(const std::string &str, const std::string &suffix) -{ - return str.size() >= suffix.size() && - 0 == str.compare(str.size() - suffix.size(), suffix.size(), suffix); +static bool ends_with(const std::string & str, const std::string & suffix) { + return str.size() >= suffix.size() && 0 == str.compare(str.size() - suffix.size(), suffix.size(), suffix); } -static size_t find_partial_stop_string(const std::string &stop, - const std::string &text) -{ - if (!text.empty() && !stop.empty()) - { +static size_t find_partial_stop_string(const std::string &stop, const std::string &text) { + if (!text.empty() && !stop.empty()) { const char text_last_char = text.back(); - for (int64_t char_index = stop.size() - 1; char_index >= 0; char_index--) - { - if (stop[char_index] == text_last_char) - { + for (int64_t char_index = stop.size() - 1; char_index >= 0; char_index--) { + if (stop[char_index] == text_last_char) { const std::string current_partial = stop.substr(0, char_index + 1); - if (ends_with(text, current_partial)) - { + if (ends_with(text, current_partial)) { return text.size() - char_index - 1; } } } } + return std::string::npos; } // TODO: reuse llama_detokenize template -static std::string tokens_to_str(llama_context *ctx, Iter begin, Iter end) -{ +static std::string tokens_to_str(llama_context * ctx, Iter begin, Iter end) { std::string ret; - for (; begin != end; ++begin) - { + for (; begin != end; ++begin) { ret += llama_token_to_piece(ctx, *begin); } + return ret; } // format incomplete utf-8 multibyte character for output -static std::string tokens_to_output_formatted_string(const llama_context *ctx, const llama_token token) -{ +static std::string tokens_to_output_formatted_string(const llama_context * ctx, const llama_token token) { std::string out = token == -1 ? "" : llama_token_to_piece(ctx, token); + // if the size is 1 and first bit is 1, meaning it's a partial character // (size > 1 meaning it's already a known token) - if (out.size() == 1 && (out[0] & 0x80) == 0x80) - { + if (out.size() == 1 && (out[0] & 0x80) == 0x80) { std::stringstream ss; ss << std::hex << (out[0] & 0xff); std::string res(ss.str()); out = "byte: \\x" + res; } + return out; } +struct completion_token_output { + llama_token tok; + std::string text_to_send; + + struct token_prob { + llama_token tok; + float prob; + }; + + std::vector probs; +}; + // convert a vector of completion_token_output to json -static json probs_vector_to_json(const llama_context *ctx, const std::vector &probs) -{ +static json probs_vector_to_json(const llama_context * ctx, const std::vector & probs) { json out = json::array(); - for (const auto &prob : probs) - { + + for (const auto & prob : probs) { json probs_for_token = json::array(); - for (const auto &p : prob.probs) - { - std::string tok_str = tokens_to_output_formatted_string(ctx, p.tok); - probs_for_token.push_back(json - { + + for (const auto & p : prob.probs) { + const std::string tok_str = tokens_to_output_formatted_string(ctx, p.tok); + probs_for_token.push_back(json { {"tok_str", tok_str}, {"prob", p.prob}, }); } - std::string tok_str = tokens_to_output_formatted_string(ctx, prob.tok); - out.push_back(json{ + + const std::string tok_str = tokens_to_output_formatted_string(ctx, prob.tok); + out.push_back(json { {"content", tok_str}, {"probs", probs_for_token}, }); } + return out; } + +// +// OAI utils +// + +static json oaicompat_completion_params_parse( + const struct llama_model * model, + const json & body, /* openai api json semantics */ + const std::string & chat_template) { + json llama_params; + + llama_params["__oaicompat"] = true; + + // Map OpenAI parameters to llama.cpp parameters + // + // For parameters that are defined by the OpenAI documentation (e.g. + // temperature), we explicitly specify OpenAI's intended default; we + // need to do that because sometimes OpenAI disagrees with llama.cpp + // + // https://platform.openai.com/docs/api-reference/chat/create + llama_sampling_params default_sparams; + llama_params["model"] = json_value(body, "model", std::string("unknown")); + llama_params["prompt"] = format_chat(model, chat_template, body["messages"]); + llama_params["cache_prompt"] = json_value(body, "cache_prompt", false); + llama_params["temperature"] = json_value(body, "temperature", 0.0); + llama_params["top_k"] = json_value(body, "top_k", default_sparams.top_k); + llama_params["top_p"] = json_value(body, "top_p", 1.0); + llama_params["n_predict"] = json_value(body, "max_tokens", -1); + llama_params["logit_bias"] = json_value(body, "logit_bias", json::object()); + llama_params["frequency_penalty"] = json_value(body, "frequency_penalty", 0.0); + llama_params["presence_penalty"] = json_value(body, "presence_penalty", 0.0); + llama_params["seed"] = json_value(body, "seed", LLAMA_DEFAULT_SEED); + llama_params["stream"] = json_value(body, "stream", false); + llama_params["mirostat"] = json_value(body, "mirostat", default_sparams.mirostat); + llama_params["mirostat_tau"] = json_value(body, "mirostat_tau", default_sparams.mirostat_tau); + llama_params["mirostat_eta"] = json_value(body, "mirostat_eta", default_sparams.mirostat_eta); + llama_params["penalize_nl"] = json_value(body, "penalize_nl", default_sparams.penalize_nl); + llama_params["typical_p"] = json_value(body, "typical_p", default_sparams.typical_p); + llama_params["repeat_last_n"] = json_value(body, "repeat_last_n", default_sparams.penalty_last_n); + llama_params["ignore_eos"] = json_value(body, "ignore_eos", false); + llama_params["tfs_z"] = json_value(body, "tfs_z", default_sparams.tfs_z); + + if (body.count("grammar") != 0) { + llama_params["grammar"] = json_value(body, "grammar", json::object()); + } + + // Handle 'stop' field + if (body.contains("stop") && body["stop"].is_string()) { + llama_params["stop"] = json::array({body["stop"].get()}); + } else { + llama_params["stop"] = json_value(body, "stop", json::array()); + } + + // Ensure there is ChatML-specific end sequence among stop words + llama_params["stop"].push_back("<|im_end|>"); + + return llama_params; +} + +static json format_final_response_oaicompat(const json & request, json result, bool streaming = false) { + bool stopped_word = result.count("stopped_word") != 0; + bool stopped_eos = json_value(result, "stopped_eos", false); + int num_tokens_predicted = json_value(result, "tokens_predicted", 0); + int num_prompt_tokens = json_value(result, "tokens_evaluated", 0); + std::string content = json_value(result, "content", std::string("")); + + std::string finish_reason = "length"; + if (stopped_word || stopped_eos) { + finish_reason = "stop"; + } + + json choices = + streaming ? json::array({json{{"finish_reason", finish_reason}, + {"index", 0}, + {"delta", json::object()}}}) + : json::array({json{{"finish_reason", finish_reason}, + {"index", 0}, + {"message", json{{"content", content}, + {"role", "assistant"}}}}}); + + std::time_t t = std::time(0); + + json res = json { + {"choices", choices}, + {"created", t}, + {"model", + json_value(request, "model", std::string(DEFAULT_OAICOMPAT_MODEL))}, + {"object", streaming ? "chat.completion.chunk" : "chat.completion"}, + {"usage", json { + {"completion_tokens", num_tokens_predicted}, + {"prompt_tokens", num_prompt_tokens}, + {"total_tokens", num_tokens_predicted + num_prompt_tokens} + }}, + {"id", gen_chatcmplid()} + }; + + if (server_verbose) { + res["__verbose"] = result; + } + + if (result.contains("completion_probabilities")) { + res["completion_probabilities"] = json_value(result, "completion_probabilities", json::array()); + } + + return res; +} + +// return value is vector as there is one case where we might need to generate two responses +static std::vector format_partial_response_oaicompat(json result) { + if (!result.contains("model") || !result.contains("oaicompat_token_ctr")) { + return std::vector({result}); + } + + bool first = json_value(result, "oaicompat_token_ctr", 0) == 0; + std::string modelname = json_value(result, "model", std::string(DEFAULT_OAICOMPAT_MODEL)); + + bool stopped_word = json_value(result, "stopped_word", false); + bool stopped_eos = json_value(result, "stopped_eos", false); + bool stopped_limit = json_value(result, "stopped_limit", false); + std::string content = json_value(result, "content", std::string("")); + + std::string finish_reason; + if (stopped_word || stopped_eos) { + finish_reason = "stop"; + } + if (stopped_limit) { + finish_reason = "length"; + } + + std::time_t t = std::time(0); + + json choices; + + if (!finish_reason.empty()) { + choices = json::array({json{{"finish_reason", finish_reason}, + {"index", 0}, + {"delta", json::object()}}}); + } else { + if (first) { + if (content.empty()) { + choices = json::array({json{{"finish_reason", nullptr}, + {"index", 0}, + {"delta", json{{"role", "assistant"}}}}}); + } else { + // We have to send this as two updates to conform to openai behavior + json initial_ret = json{{"choices", json::array({json{ + {"finish_reason", nullptr}, + {"index", 0}, + {"delta", json{ + {"role", "assistant"} + }}}})}, + {"created", t}, + {"id", gen_chatcmplid()}, + {"model", modelname}, + {"object", "chat.completion.chunk"}}; + + json second_ret = json{ + {"choices", json::array({json{{"finish_reason", nullptr}, + {"index", 0}, + {"delta", json{ + {"content", content}}} + }})}, + {"created", t}, + {"id", gen_chatcmplid()}, + {"model", modelname}, + {"object", "chat.completion.chunk"}}; + + return std::vector({initial_ret, second_ret}); + } + } else { + // Some idiosyncrasy in task processing logic makes several trailing calls + // with empty content, we ignore these at the calee site. + if (content.empty()) { + return std::vector({json::object()}); + } + + choices = json::array({json{ + {"finish_reason", nullptr}, + {"index", 0}, + {"delta", + json{ + {"content", content}, + }}, + }}); + } + } + + json ret = json { + {"choices", choices}, + {"created", t}, + {"id", gen_chatcmplid()}, + {"model", modelname}, + {"object", "chat.completion.chunk"} + }; + + return std::vector({ret}); +} + +static json format_embeddings_response_oaicompat(const json & request, const json & embeddings) { + json res = json { + {"model", json_value(request, "model", std::string(DEFAULT_OAICOMPAT_MODEL))}, + {"object", "list"}, + {"usage", json { + {"prompt_tokens", 0}, + {"total_tokens", 0} + }}, + {"data", embeddings} + }; + + return res; +} + +static json format_tokenizer_response(const std::vector & tokens) { + return json { + {"tokens", tokens} + }; +} + +static json format_detokenized_response(const std::string & content) { + return json { + {"content", content} + }; +} diff --git a/llama.cpp b/llama.cpp index b27aa2728..478099648 100644 --- a/llama.cpp +++ b/llama.cpp @@ -13541,18 +13541,22 @@ LLAMA_API int32_t llama_chat_apply_template( curr_tmpl = std::string(model_template.data(), model_template.size()); } } + // format the chat to string std::vector chat_vec; chat_vec.resize(n_msg); for (size_t i = 0; i < n_msg; i++) { chat_vec[i] = &chat[i]; } + std::string formatted_chat; int32_t res = llama_chat_apply_template_internal(curr_tmpl, chat_vec, formatted_chat, add_ass); if (res < 0) { return res; } - strncpy(buf, formatted_chat.c_str(), length); + if (buf && length > 0) { + strncpy(buf, formatted_chat.c_str(), length); + } return res; } From 55a2a900ff4a02fc33708ac7858d595d289a3f2a Mon Sep 17 00:00:00 2001 From: Minsoo Cheong <54794500+mscheong01@users.noreply.github.com> Date: Thu, 7 Mar 2024 19:42:39 +0900 Subject: [PATCH 554/812] server : add `/v1/completions` endpoint (#5914) * add-`/v1/completions`-endpoint * add legacy comment to `/completion` endpoint --- examples/server/server.cpp | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/examples/server/server.cpp b/examples/server/server.cpp index 3bdbde954..f255ad764 100644 --- a/examples/server/server.cpp +++ b/examples/server/server.cpp @@ -2923,7 +2923,7 @@ int main(int argc, char ** argv) { res.set_content(data.dump(), "application/json; charset=utf-8"); }); - svr.Post("/completion", [&ctx_server, &validate_api_key](const httplib::Request & req, httplib::Response & res) { + const auto completions = [&ctx_server, &validate_api_key](const httplib::Request & req, httplib::Response & res) { res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin")); if (!validate_api_key(req, res)) { return; @@ -3001,7 +3001,11 @@ int main(int argc, char ** argv) { res.set_chunked_content_provider("text/event-stream", chunked_content_provider, on_complete); } - }); + }; + + svr.Post("/completion", completions); // legacy + svr.Post("/completions", completions); + svr.Post("/v1/completions", completions); svr.Get("/v1/models", [¶ms, &model_meta](const httplib::Request & req, httplib::Response & res) { res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin")); From 89fb735fcfd21781a8194b211cf32824beb3f71f Mon Sep 17 00:00:00 2001 From: Neo Zhang Jianyu Date: Thu, 7 Mar 2024 19:14:49 +0800 Subject: [PATCH 555/812] Revert "[SYCL] fix error when set main gpu to non-zero (#5901)" (#5918) This reverts commit ceca1aef0738b57951cd12c603c3477e75312dec. --- ggml-sycl.cpp | 172 ++++++++++++++++++++------------------------------ ggml-sycl.h | 1 - llama.cpp | 16 ++--- 3 files changed, 73 insertions(+), 116 deletions(-) diff --git a/ggml-sycl.cpp b/ggml-sycl.cpp index 221d67b8d..ddd951dd6 100644 --- a/ggml-sycl.cpp +++ b/ggml-sycl.cpp @@ -3559,31 +3559,12 @@ class sycl_gpu_mgr { int work_group_size = 0; std::string gpus_list = ""; - /* - Use all GPU with same top max compute units - */ sycl_gpu_mgr() { detect_sycl_gpu_list_with_max_cu(); get_allow_gpus(); create_context_with_gpus(); } - /* - Use the assigned GPU as only one - */ - sycl_gpu_mgr(int main_gpu_id) { - sycl::device device = dpct::dev_mgr::instance().get_device(main_gpu_id); - dpct::device_info prop; - dpct::get_device_info(prop, device); - gpus.push_back(main_gpu_id); - devices.push_back(device); - work_group_size = prop.get_max_work_group_size(); - max_compute_units = prop.get_max_compute_units(); - - get_allow_gpus(); - create_context_with_gpus(); - } - void create_context_with_gpus() { sycl::context ctx = sycl::context(devices); assert(gpus.size() > 0); @@ -3599,7 +3580,7 @@ class sycl_gpu_mgr { gpus_list += std::to_string(gpus[i]); gpus_list += ","; } - if (gpus_list.length() > 1) { + if (gpus_list.length() > 2) { gpus_list.pop_back(); } } @@ -3648,8 +3629,8 @@ class sycl_gpu_mgr { if (gpus[i] == id) return i; } - printf("miss to get device index by id=%d\n", id); - GGML_ASSERT(false); + assert(false); + return -1; } int get_next_index(int id) { @@ -3658,7 +3639,8 @@ class sycl_gpu_mgr { if (gpus[i] == id) return i; } - GGML_ASSERT(false); + assert(false); + return -1; } }; @@ -3667,7 +3649,6 @@ static int g_device_count = -1; static int g_all_sycl_device_count = -1; static int g_main_device = -1; static int g_main_device_id = -1; -static bool g_ggml_backend_sycl_buffer_type_initialized = false; static std::array g_default_tensor_split = {}; @@ -13244,7 +13225,7 @@ void ggml_backend_sycl_print_sycl_devices() { } void print_gpu_device_list() { - fprintf(stderr, "detect %d SYCL GPUs: [%s] with top Max compute units:%d\n", + fprintf(stderr, "detect %d SYCL GPUs: [%s] with Max compute units:%d\n", g_sycl_gpu_mgr->get_gpu_count(), g_sycl_gpu_mgr->gpus_list.c_str(), g_sycl_gpu_mgr->max_compute_units); @@ -13283,15 +13264,6 @@ void ggml_init_sycl() try { #else fprintf(stderr, "%s: GGML_SYCL_F16: no\n", __func__); #endif - -/* NOT REMOVE, keep it for next optimize for XMX. -#if defined(SYCL_USE_XMX) - fprintf(stderr, "%s: SYCL_USE_XMX: yes\n", __func__); -#else - fprintf(stderr, "%s: SYCL_USE_XMX: no\n", __func__); -#endif -*/ - if (CHECK_TRY_ERROR(g_all_sycl_device_count = dpct::dev_mgr::instance().device_count()) != 0) { initialized = true; @@ -13300,61 +13272,68 @@ void ggml_init_sycl() try { } GGML_ASSERT(g_all_sycl_device_count <= GGML_SYCL_MAX_DEVICES); ggml_backend_sycl_print_sycl_devices(); + if (!g_sycl_gpu_mgr) g_sycl_gpu_mgr = new sycl_gpu_mgr(); + + g_device_count = g_sycl_gpu_mgr->get_gpu_count(); + g_work_group_size = g_sycl_gpu_mgr->work_group_size; + print_gpu_device_list(); - initialized = true; - g_sycl_loaded = true; - } + int64_t total_vram = 0; - - g_device_count = g_sycl_gpu_mgr->get_gpu_count(); - g_work_group_size = g_sycl_gpu_mgr->work_group_size; - - int64_t total_vram = 0; - - - for (int id = 0; id < GGML_SYCL_MAX_DEVICES; ++id) { - g_device_caps[id].vmm = 0; - g_device_caps[id].device_id = -1; - g_device_caps[id].cc = 0; - g_tensor_split[id] = 0; - g_default_tensor_split[id] = 0; - } - - for (int i = 0; i < g_device_count; ++i) { - int device_id = g_sycl_gpu_mgr->gpus[i]; - g_device_caps[i].vmm = 0; - - dpct::device_info prop; - SYCL_CHECK(CHECK_TRY_ERROR(dpct::get_device_info( - prop, dpct::dev_mgr::instance().get_device(device_id)))); - - g_default_tensor_split[i] = total_vram; - total_vram += prop.get_global_mem_size(); - - g_device_caps[i].cc = - 100 * prop.get_major_version() + 10 * prop.get_minor_version(); - } - - for (int i = 0; i < g_device_count; ++i) { - g_default_tensor_split[i] /= total_vram; - } - - for (int i = 0; i < g_device_count; ++i) { - SYCL_CHECK(ggml_sycl_set_device(i)); - - // create sycl streams - for (int is = 0; is < MAX_STREAMS; ++is) { - SYCL_CHECK(CHECK_TRY_ERROR( - g_syclStreams[i][is] = - dpct::get_current_device().create_queue( - g_sycl_gpu_mgr->get_co_ctx(), dpct::get_current_device()))); +/* NOT REMOVE, keep it for next optimize for XMX. +#if defined(SYCL_USE_XMX) + fprintf(stderr, "%s: SYCL_USE_XMX: yes\n", __func__); +#else + fprintf(stderr, "%s: SYCL_USE_XMX: no\n", __func__); +#endif +*/ + for (int id = 0; id < GGML_SYCL_MAX_DEVICES; ++id) { + g_device_caps[id].vmm = 0; + g_device_caps[id].device_id = -1; + g_device_caps[id].cc = 0; + g_tensor_split[id] = 0; + g_default_tensor_split[id] = 0; } - const dpct::queue_ptr stream = g_syclStreams[i][0]; - // create sycl handle - SYCL_CHECK(CHECK_TRY_ERROR(g_sycl_handles[i] = stream)); + for (int i = 0; i < g_device_count; ++i) { + int device_id = g_sycl_gpu_mgr->gpus[i]; + g_device_caps[i].vmm = 0; + + dpct::device_info prop; + SYCL_CHECK(CHECK_TRY_ERROR(dpct::get_device_info( + prop, dpct::dev_mgr::instance().get_device(device_id)))); + + g_default_tensor_split[i] = total_vram; + total_vram += prop.get_global_mem_size(); + + g_device_caps[i].cc = + 100 * prop.get_major_version() + 10 * prop.get_minor_version(); + } + + for (int i = 0; i < g_device_count; ++i) { + g_default_tensor_split[i] /= total_vram; + } + + for (int i = 0; i < g_device_count; ++i) { + SYCL_CHECK(ggml_sycl_set_device(i)); + + // create sycl streams + for (int is = 0; is < MAX_STREAMS; ++is) { + SYCL_CHECK(CHECK_TRY_ERROR( + g_syclStreams[i][is] = + dpct::get_current_device().create_queue( + g_sycl_gpu_mgr->get_co_ctx(), dpct::get_current_device()))); + } + + const dpct::queue_ptr stream = g_syclStreams[i][0]; + // create sycl handle + SYCL_CHECK(CHECK_TRY_ERROR(g_sycl_handles[i] = stream)); + } + + initialized = true; + g_sycl_loaded = true; } } catch (sycl::exception const &exc) { @@ -16753,24 +16732,22 @@ static ggml_backend_buffer_type_i ggml_backend_sycl_buffer_type_interface = { /* .is_host = */ nullptr, }; -ggml_backend_buffer_type_t ggml_backend_sycl_buffer_type(int device_index) { - if (device_index>=g_device_count or device_index<0) { - printf("ggml_backend_sycl_buffer_type error: device_index:%d is out of range [0, %d], miss to call ggml_backend_sycl_set_single_device()\n", - device_index, g_device_count-1); - GGML_ASSERT(device_indexgpus[i])}, }; } - g_ggml_backend_sycl_buffer_type_initialized = true; + ggml_backend_sycl_buffer_type_initialized = true; } - return &ggml_backend_sycl_buffer_types[device_index]; + + return &ggml_backend_sycl_buffer_types[device]; } // sycl split buffer type @@ -17519,17 +17496,6 @@ GGML_API GGML_CALL int ggml_backend_sycl_get_device_index(int device_id) { return g_sycl_gpu_mgr->get_index(device_id); } -GGML_API GGML_CALL void ggml_backend_sycl_set_single_device(int main_gpu_id) { - GGML_ASSERT(main_gpu_idbackends.push_back(backend); } else { // LLAMA_SPLIT_LAYER requires a backend for each GPU - + int id_list[GGML_SYCL_MAX_DEVICES]; + ggml_sycl_get_gpu_list(id_list, GGML_SYCL_MAX_DEVICES); for (int i = 0; i < ggml_backend_sycl_get_device_count(); ++i) { + int device_id = id_list[i]; ggml_backend_t backend = ggml_backend_sycl_init(i); if (backend == nullptr) { - int id_list[GGML_SYCL_MAX_DEVICES]; - ggml_sycl_get_gpu_list(id_list, GGML_SYCL_MAX_DEVICES); - LLAMA_LOG_ERROR("%s: failed to initialize SYCL%d (index %d)backend\n", __func__, id_list[i], i); + LLAMA_LOG_ERROR("%s: failed to initialize SYCL%d (index %d)backend\n", __func__, device_id, i); llama_free(ctx); return nullptr; } From 6cdabe652695167263c8b447520987b11856f7ca Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Thu, 7 Mar 2024 16:32:38 +0200 Subject: [PATCH 556/812] llama-bench : add embeddings option (#5924) * llama-bench : add embeddings option * llama-bench : do not hard code embd default value --------- Co-authored-by: slaren --- examples/llama-bench/llama-bench.cpp | 30 +++++++++++++++++++++++++--- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/examples/llama-bench/llama-bench.cpp b/examples/llama-bench/llama-bench.cpp index aa79d002a..2ff86ef6f 100644 --- a/examples/llama-bench/llama-bench.cpp +++ b/examples/llama-bench/llama-bench.cpp @@ -173,6 +173,7 @@ struct cmd_params { std::vector no_kv_offload; std::vector> tensor_split; std::vector use_mmap; + std::vector embeddings; int reps; bool verbose; output_formats output_format; @@ -192,6 +193,7 @@ static const cmd_params cmd_params_defaults = { /* no_kv_offload */ {false}, /* tensor_split */ {std::vector(llama_max_devices(), 0.0f)}, /* use_mmap */ {true}, + /* embeddings */ {false}, /* reps */ 5, /* verbose */ false, /* output_format */ MARKDOWN @@ -214,6 +216,7 @@ static void print_usage(int /* argc */, char ** argv) { printf(" -mg, --main-gpu (default: %s)\n", join(cmd_params_defaults.main_gpu, ",").c_str()); printf(" -nkvo, --no-kv-offload <0|1> (default: %s)\n", join(cmd_params_defaults.no_kv_offload, ",").c_str()); printf(" -mmp, --mmap <0|1> (default: %s)\n", join(cmd_params_defaults.use_mmap, ",").c_str()); + printf(" -embd, --embeddings <0|1> (default: %s)\n", join(cmd_params_defaults.embeddings, ",").c_str()); printf(" -ts, --tensor_split (default: 0)\n"); printf(" -r, --repetitions (default: %d)\n", cmd_params_defaults.reps); printf(" -o, --output (default: %s)\n", output_format_str(cmd_params_defaults.output_format)); @@ -382,6 +385,13 @@ static cmd_params parse_cmd_params(int argc, char ** argv) { } auto p = split(argv[i], split_delim); params.use_mmap.insert(params.use_mmap.end(), p.begin(), p.end()); + } else if (arg == "-embd" || arg == "--embeddings") { + if (++i >= argc) { + invalid_param = true; + break; + } + auto p = split(argv[i], split_delim); + params.embeddings.insert(params.embeddings.end(), p.begin(), p.end()); } else if (arg == "-ts" || arg == "--tensor-split") { if (++i >= argc) { invalid_param = true; @@ -453,6 +463,7 @@ static cmd_params parse_cmd_params(int argc, char ** argv) { if (params.no_kv_offload.empty()){ params.no_kv_offload = cmd_params_defaults.no_kv_offload; } if (params.tensor_split.empty()) { params.tensor_split = cmd_params_defaults.tensor_split; } if (params.use_mmap.empty()) { params.use_mmap = cmd_params_defaults.use_mmap; } + if (params.embeddings.empty()) { params.embeddings = cmd_params_defaults.embeddings; } if (params.n_threads.empty()) { params.n_threads = cmd_params_defaults.n_threads; } return params; @@ -472,6 +483,7 @@ struct cmd_params_instance { bool no_kv_offload; std::vector tensor_split; bool use_mmap; + bool embeddings; llama_model_params to_llama_mparams() const { llama_model_params mparams = llama_model_default_params(); @@ -502,6 +514,7 @@ struct cmd_params_instance { cparams.type_k = type_k; cparams.type_v = type_v; cparams.offload_kqv = !no_kv_offload; + cparams.embeddings = embeddings; return cparams; } @@ -517,6 +530,7 @@ static std::vector get_cmd_params_instances(const cmd_param for (const auto & mg : params.main_gpu) for (const auto & ts : params.tensor_split) for (const auto & mmp : params.use_mmap) + for (const auto & embd : params.embeddings) for (const auto & nb : params.n_batch) for (const auto & tk : params.type_k) for (const auto & tv : params.type_v) @@ -540,6 +554,7 @@ static std::vector get_cmd_params_instances(const cmd_param /* .no_kv_offload= */ nkvo, /* .tensor_split = */ ts, /* .use_mmap = */ mmp, + /* .embeddings = */ embd, }; instances.push_back(instance); } @@ -562,6 +577,7 @@ static std::vector get_cmd_params_instances(const cmd_param /* .no_kv_offload= */ nkvo, /* .tensor_split = */ ts, /* .use_mmap = */ mmp, + /* .embeddings = */ embd, }; instances.push_back(instance); } @@ -597,6 +613,7 @@ struct test { bool no_kv_offload; std::vector tensor_split; bool use_mmap; + bool embeddings; int n_prompt; int n_gen; std::string test_time; @@ -619,6 +636,7 @@ struct test { no_kv_offload = inst.no_kv_offload; tensor_split = inst.tensor_split; use_mmap = inst.use_mmap; + embeddings = inst.embeddings; n_prompt = inst.n_prompt; n_gen = inst.n_gen; // RFC 3339 date-time format @@ -690,7 +708,7 @@ struct test { "n_batch", "n_threads", "type_k", "type_v", "n_gpu_layers", "split_mode", "main_gpu", "no_kv_offload", - "tensor_split", "use_mmap", + "tensor_split", "use_mmap", "embeddings", "n_prompt", "n_gen", "test_time", "avg_ns", "stddev_ns", "avg_ts", "stddev_ts" @@ -710,7 +728,7 @@ struct test { } if (field == "cuda" || field == "opencl" || field == "vulkan" || field == "kompute" || field == "metal" || field == "gpu_blas" || field == "blas" || field == "sycl" ||field == "f16_kv" || field == "no_kv_offload" || - field == "use_mmap") { + field == "use_mmap" || field == "embeddings") { return BOOL; } if (field == "avg_ts" || field == "stddev_ts") { @@ -744,7 +762,7 @@ struct test { std::to_string(n_batch), std::to_string(n_threads), ggml_type_name(type_k), ggml_type_name(type_v), std::to_string(n_gpu_layers), split_mode_str(split_mode), std::to_string(main_gpu), std::to_string(no_kv_offload), - tensor_split_str, std::to_string(use_mmap), + tensor_split_str, std::to_string(use_mmap), std::to_string(embeddings), std::to_string(n_prompt), std::to_string(n_gen), test_time, std::to_string(avg_ns()), std::to_string(stdev_ns()), std::to_string(avg_ts()), std::to_string(stdev_ts()) @@ -914,6 +932,9 @@ struct markdown_printer : public printer { if (field == "use_mmap") { return "mmap"; } + if (field == "embeddings") { + return "embd"; + } if (field == "tensor_split") { return "ts"; } @@ -957,6 +978,9 @@ struct markdown_printer : public printer { if (params.use_mmap.size() > 1 || params.use_mmap != cmd_params_defaults.use_mmap) { fields.emplace_back("use_mmap"); } + if (params.embeddings.size() > 1 || params.embeddings != cmd_params_defaults.embeddings) { + fields.emplace_back("embeddings"); + } fields.emplace_back("test"); fields.emplace_back("t/s"); From 581ed5c4fe3a8909aaa8313633ac443f471ba755 Mon Sep 17 00:00:00 2001 From: "UEXTM.com" <84163508+uextm@users.noreply.github.com> Date: Fri, 8 Mar 2024 04:35:04 -0500 Subject: [PATCH 557/812] log : fix MSVC compile errors (#5643) MSVC gives the following error with the existing macros: `Error C2059 : syntax error: ','` This patch adds `##` as a prefix to `__VA_ARGS__` to address this error. --- common/log.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/common/log.h b/common/log.h index e4e1b9f4f..eb111e784 100644 --- a/common/log.h +++ b/common/log.h @@ -297,7 +297,7 @@ inline std::string log_filename_generator_impl(LogTriState multilog, const std:: #ifndef _MSC_VER #define LOG(...) LOG_IMPL(__VA_ARGS__, "") #else - #define LOG(str, ...) LOG_IMPL("%s" str, "", __VA_ARGS__, "") + #define LOG(str, ...) LOG_IMPL("%s" str, "", ##__VA_ARGS__, "") #endif // Main TEE macro. @@ -311,7 +311,7 @@ inline std::string log_filename_generator_impl(LogTriState multilog, const std:: #ifndef _MSC_VER #define LOG_TEE(...) LOG_TEE_IMPL(__VA_ARGS__, "") #else - #define LOG_TEE(str, ...) LOG_TEE_IMPL("%s" str, "", __VA_ARGS__, "") + #define LOG_TEE(str, ...) LOG_TEE_IMPL("%s" str, "", ##__VA_ARGS__, "") #endif // LOG macro variants with auto endline. @@ -319,8 +319,8 @@ inline std::string log_filename_generator_impl(LogTriState multilog, const std:: #define LOGLN(...) LOG_IMPL(__VA_ARGS__, "\n") #define LOG_TEELN(...) LOG_TEE_IMPL(__VA_ARGS__, "\n") #else - #define LOGLN(str, ...) LOG_IMPL("%s" str, "", __VA_ARGS__, "\n") - #define LOG_TEELN(str, ...) LOG_TEE_IMPL("%s" str, "", __VA_ARGS__, "\n") + #define LOGLN(str, ...) LOG_IMPL("%s" str, "", ##__VA_ARGS__, "\n") + #define LOG_TEELN(str, ...) LOG_TEE_IMPL("%s" str, "", ##__VA_ARGS__, "\n") #endif // INTERNAL, DO NOT USE From af37fd8b30e37ccbffdd82e6f48559e2fb7ce7dd Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Fri, 8 Mar 2024 12:40:02 +0200 Subject: [PATCH 558/812] server : fix EOS token detection with disabled cache (#5938) --- examples/server/server.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/server/server.cpp b/examples/server/server.cpp index f255ad764..1434095fc 100644 --- a/examples/server/server.cpp +++ b/examples/server/server.cpp @@ -1123,7 +1123,7 @@ struct server_context { }); } - if (!slot.cache_tokens.empty() && result.tok == llama_token_eos(model)) { + if (result.tok == llama_token_eos(model)) { slot.stopped_eos = true; slot.has_next_token = false; From e457fb3540e0aaec47cfde0abf784c213f9216ee Mon Sep 17 00:00:00 2001 From: Don Mahurin Date: Fri, 8 Mar 2024 02:41:50 -0800 Subject: [PATCH 559/812] llama : assume tied weights if lm_head/output weights is missing (#5824) This is to support model configurations with "tie_word_embeddings" set to true. Co-authored-by: Don Mahurin <2797413+dmahurin@users.noreply.github.com> --- llama.cpp | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/llama.cpp b/llama.cpp index 4225f9555..458382b21 100644 --- a/llama.cpp +++ b/llama.cpp @@ -3888,7 +3888,13 @@ static bool llm_load_tensors( { model.output_norm = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}); if (model.arch != LLM_ARCH_MINICPM){ - model.output = ml.create_tensor(ctx_output_split, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}); + model.output = ml.create_tensor(ctx_output_split, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, false); + // if output is NULL, init from the input tok embed + if (model.output == NULL) { + model.output = ml.create_tensor(ctx_output, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}); + ml.n_created--; // artificial tensor + ml.size_data += ggml_nbytes(model.output); + } } } From 76e868821a94072fbc87cb1fcca291694319eae8 Mon Sep 17 00:00:00 2001 From: Pierrick Hymbert Date: Fri, 8 Mar 2024 12:25:04 +0100 Subject: [PATCH 560/812] server: metrics: add llamacpp:prompt_seconds_total and llamacpp:tokens_predicted_seconds_total, reset bucket only on /metrics. Fix values cast to int. Add Process-Start-Time-Unix header. (#5937) Closes #5850 --- examples/server/server.cpp | 47 ++++++++++++++----- examples/server/tests/features/server.feature | 1 + examples/server/tests/features/steps/steps.py | 11 ++++- 3 files changed, 46 insertions(+), 13 deletions(-) diff --git a/examples/server/server.cpp b/examples/server/server.cpp index 1434095fc..109ff7175 100644 --- a/examples/server/server.cpp +++ b/examples/server/server.cpp @@ -335,8 +335,12 @@ struct server_slot { }; struct server_metrics { + const int64_t t_start = ggml_time_us(); + uint64_t n_prompt_tokens_processed_total = 0; + uint64_t t_prompt_processing_total = 0; uint64_t n_tokens_predicted_total = 0; + uint64_t t_tokens_generation_total = 0; uint64_t n_prompt_tokens_processed = 0; uint64_t t_prompt_processing = 0; @@ -348,12 +352,14 @@ struct server_metrics { n_prompt_tokens_processed_total += slot.n_prompt_tokens_processed; n_prompt_tokens_processed += slot.n_prompt_tokens_processed; t_prompt_processing += slot.t_prompt_processing; + t_prompt_processing_total += slot.t_prompt_processing; } void on_prediction(const server_slot &slot) { - n_tokens_predicted_total += slot.n_decoded; - n_tokens_predicted += slot.n_decoded; - t_tokens_generation += slot.t_token_generation; + n_tokens_predicted_total += slot.n_decoded; + n_tokens_predicted += slot.n_decoded; + t_tokens_generation += slot.t_token_generation; + t_tokens_generation_total += slot.t_token_generation; } void reset_bucket() { @@ -1502,9 +1508,12 @@ struct server_context { { "idle", n_idle_slots }, { "processing", n_processing_slots }, { "deferred", queue_tasks.queue_tasks_deferred.size() }, + { "t_start", metrics.t_start}, { "n_prompt_tokens_processed_total", metrics.n_prompt_tokens_processed_total}, + { "t_tokens_generation_total", metrics.t_tokens_generation_total}, { "n_tokens_predicted_total", metrics.n_tokens_predicted_total}, + { "t_prompt_processing_total", metrics.t_prompt_processing_total}, { "n_prompt_tokens_processed", metrics.n_prompt_tokens_processed}, { "t_prompt_processing", metrics.t_prompt_processing}, @@ -1517,7 +1526,9 @@ struct server_context { { "slots", slots_data }, }; - metrics.reset_bucket(); + if (json_value(task.data, "reset_bucket", false)) { + metrics.reset_bucket(); + } queue_results.send(res); } break; } @@ -2709,6 +2720,7 @@ int main(int argc, char ** argv) { task.id_multi = -1; task.id_target = -1; task.type = SERVER_TASK_TYPE_METRICS; + task.data.push_back({{"reset_bucket", true}}); ctx_server.queue_results.add_waiting_task_id(task.id); ctx_server.queue_tasks.post(task); @@ -2732,20 +2744,28 @@ int main(int argc, char ** argv) { {"counter", {{ {"name", "prompt_tokens_total"}, {"help", "Number of prompt tokens processed."}, - {"value", data["n_prompt_tokens_processed_total"]} + {"value", (uint64_t) data["n_prompt_tokens_processed_total"]} + }, { + {"name", "prompt_seconds_total"}, + {"help", "Prompt process time"}, + {"value", (uint64_t) data["t_prompt_processing_total"] / 1.e3} }, { {"name", "tokens_predicted_total"}, {"help", "Number of generation tokens processed."}, - {"value", data["n_tokens_predicted_total"]} + {"value", (uint64_t) data["n_tokens_predicted_total"]} + }, { + {"name", "tokens_predicted_seconds_total"}, + {"help", "Predict process time"}, + {"value", (uint64_t) data["t_tokens_generation_total"] / 1.e3} }}}, {"gauge", {{ {"name", "prompt_tokens_seconds"}, {"help", "Average prompt throughput in tokens/s."}, - {"value", n_prompt_tokens_processed ? 1e3 / t_prompt_processing * n_prompt_tokens_processed : 0} + {"value", n_prompt_tokens_processed ? 1.e3 / t_prompt_processing * n_prompt_tokens_processed : 0.} },{ {"name", "predicted_tokens_seconds"}, {"help", "Average generation throughput in tokens/s."}, - {"value", n_tokens_predicted ? 1e3 / t_tokens_generation * n_tokens_predicted : 0} + {"value", n_tokens_predicted ? 1.e3 / t_tokens_generation * n_tokens_predicted : 0.} },{ {"name", "kv_cache_usage_ratio"}, {"help", "KV-cache usage. 1 means 100 percent usage."}, @@ -2753,15 +2773,15 @@ int main(int argc, char ** argv) { },{ {"name", "kv_cache_tokens"}, {"help", "KV-cache tokens."}, - {"value", data["kv_cache_tokens_count"]} + {"value", (uint64_t) data["kv_cache_tokens_count"]} },{ {"name", "requests_processing"}, {"help", "Number of request processing."}, - {"value", data["processing"]} + {"value", (uint64_t) data["processing"]} },{ {"name", "requests_deferred"}, {"help", "Number of request deferred."}, - {"value", data["deferred"]} + {"value", (uint64_t) data["deferred"]} }}} }; @@ -2775,13 +2795,16 @@ int main(int argc, char ** argv) { const std::string name = metric_def["name"]; const std::string help = metric_def["help"]; - auto value = json_value(metric_def, "value", 0); + auto value = json_value(metric_def, "value", 0.); prometheus << "# HELP llamacpp:" << name << " " << help << "\n" << "# TYPE llamacpp:" << name << " " << type << "\n" << "llamacpp:" << name << " " << value << "\n"; } } + const int64_t t_start = data["t_start"]; + res.set_header("Process-Start-Time-Unix", std::to_string(t_start)); + res.set_content(prometheus.str(), "text/plain; version=0.0.4"); res.status = 200; // HTTP OK }); diff --git a/examples/server/tests/features/server.feature b/examples/server/tests/features/server.feature index f3b758c79..878ac1363 100644 --- a/examples/server/tests/features/server.feature +++ b/examples/server/tests/features/server.feature @@ -29,6 +29,7 @@ Feature: llama.cpp server And a completion request with no api error Then tokens are predicted matching And prometheus metrics are exposed + And metric llamacpp:tokens_predicted is Examples: Prompts | prompt | n_predict | re_content | n_predicted | diff --git a/examples/server/tests/features/steps/steps.py b/examples/server/tests/features/steps/steps.py index a0b2ffdfe..d7f005836 100644 --- a/examples/server/tests/features/steps/steps.py +++ b/examples/server/tests/features/steps/steps.py @@ -586,14 +586,24 @@ async def step_prometheus_metrics_exported(context): metric_exported = False if context.debug: print(f"/metrics answer:\n{metrics_raw}\n") + context.metrics = {} for metric in parser.text_string_to_metric_families(metrics_raw): match metric.name: case "llamacpp:kv_cache_usage_ratio": assert len(metric.samples) > 0 metric_exported = True + context.metrics[metric.name] = metric + assert int(metrics_response.headers["Process-Start-Time-Unix"]) > 0, "no header process start time" assert metric_exported, "No metrics exported" +@step(u'metric {metric_name} is {metric_value:d}') +def step_assert_metric_value(context, metric_name, metric_value): + if metric_name not in context.metrics: + assert False, f"no metric {metric_name} in {context.metrics.keys()}" + assert context.metrics[metric_name].samples[0].value == metric_value, f"metric: {context.metrics[metric_name]}" + + @step(u'available models') def step_available_models(context): # openai client always expects an api_key @@ -879,7 +889,6 @@ def assert_n_tokens_predicted(completion_response, expected_predicted_n=None, re f' {n_predicted} <> {expected_predicted_n}') - async def gather_tasks_results(context): n_tasks = len(context.concurrent_tasks) if context.debug: From 515f7d0d4fce41c752fc253acf30707c3be2531e Mon Sep 17 00:00:00 2001 From: compilade <113953597+compilade@users.noreply.github.com> Date: Fri, 8 Mar 2024 10:53:37 -0500 Subject: [PATCH 561/812] llama : fix quantization of shared token_embd (#5944) --- llama.cpp | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/llama.cpp b/llama.cpp index 458382b21..4a20b7928 100644 --- a/llama.cpp +++ b/llama.cpp @@ -10973,6 +10973,9 @@ struct quantize_state_internal { bool has_imatrix = false; + // used to figure out if a model shares tok_embd with the output weight + bool has_output = false; + quantize_state_internal(const llama_model & model, const llama_model_quantize_params * params) : model(model) , params(params) @@ -11070,8 +11073,7 @@ static ggml_type get_k_quant_type(quantize_state_internal & qs, ggml_type new_ty // for arches that share the same tensor between the token embeddings and the output, we quantize the token embeddings // with the quantization of the output tensor - if (name == tn(LLM_TENSOR_OUTPUT, "weight") || - (LLM_TENSOR_NAMES.at(arch).find(LLM_TENSOR_OUTPUT) == LLM_TENSOR_NAMES.at(arch).end() && name == "token_embd.weight")) { + if (name == tn(LLM_TENSOR_OUTPUT, "weight") || (!qs.has_output && name == tn(LLM_TENSOR_TOKEN_EMBD, "weight"))) { int nx = tensor->ne[0]; if (arch == LLM_ARCH_FALCON || nx % QK_K != 0) { new_type = GGML_TYPE_Q8_0; @@ -11460,6 +11462,9 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s else if (name.find("ffn_up") != std::string::npos) { ++qs.n_ffn_up; } + else if (name == LLM_TN(model.arch)(LLM_TENSOR_OUTPUT, "weight")) { + qs.has_output = true; + } } if (qs.n_attention_wv != qs.n_ffn_down || (uint32_t)qs.n_attention_wv != model.hparams.n_layer) { LLAMA_LOG_WARN("%s ============ Strange model: n_attention_wv = %d, n_ffn_down = %d, hparams.n_layer = %d\n", From c2101a2e909ac7c08976d414e64e96c90ee5fa9e Mon Sep 17 00:00:00 2001 From: compilade <113953597+compilade@users.noreply.github.com> Date: Fri, 8 Mar 2024 17:31:00 -0500 Subject: [PATCH 562/812] llama : support Mamba Selective State Space Models (#5328) * mamba : begin working on support for Mamba SSM * mamba : begin figuring out how to (ab)use the kv cache for Mamba * mamba : recurrent inference almost works, but incoherent * mamba : recurrent inference WORKS!!! * convert : optionally use d_conv and d_state from config.json for Mamba * mamba : refactor recurrent conv, resulting in 20% perf increase It's still slower than I'd like, but I did not really optimize `ggml_exp` yet. I also refactored `ggml_exp` to work with tensors with more than 2 dimensions. * ggml : parallelize ggml_exp This results in 8% faster token generation for Mamba-130M. * mamba : simplify the conv step with a self-overlapping view Turns out the conv_state can be made smaller by one column. Note that this breaks existing GGUFs of Mamba, because the key_value_length field is tied to the conv_state size. Convolution with a self-overlapping view is cool! And it's much simpler than what I initially thought would be necessary to make the convolution step work with more than 1 token at a time. Next step is to make the SSM step work on batches of tokens too, and thus I need to figure out a way to make a parallel selective scan which will keep the ssm_state small and won't make it bigger by a factor of (n_layer * batch_size). * llama : fix Mamba KV self size wrongly displaying as f16 instead of f32 Relatedly, I also tried to see if other types than f32 worked for the states, but they don't, because of the operators used. It's probably better anyway to keep lots of precision there, since the states are small anyway. * mamba : fix self-overlapping view depth stride * mamba : handle batches of more than 1 token This means running Mamba no longer crashes when using the default settings! And probably also slightly faster prompt processing. Both batched and non-batched processing yield the same output. Previously, the state was not cleared when starting a sequence. Next step is to make the KV cache API work as expected for Mamba models. * ggml: add ggml_ssm_scan to help with parallel selective scan If the selective scan was implemented without a custom operator, there would be waaay too many nodes in the graph. For example, for Mamba-130M, with a batch size of 512 (the default), a naive selective scan could add at least 24*512=12288 nodes, which is more than LLAMA_MAX_NODES (8192), and that's only for the smallest Mamba model. So it's much cleaner with a custom operator. Not sure about the name, though. * ggml : in ggml_ssm_scan, merge multiple rows in the same vec operation This will help with performance on CPU if ggml_vec_mul_f32 and ggml_vec_add_f32 are ever optimized with SIMD. * mamba : very basic quantization support Mostly works, but there is currently no difference between the variants of a k-quant (e.g. Q4_K_S and Q4_K_M are the same). Most of the SSM-specific weights can be kept in f32 without affecting the size that much, since they are relatively small. (the linear projection weights are responsible for most of Mamba's size) Too much quantization seems to make the state degrade quite fast, and the model begins to output gibberish. It seems to affect bigger models to a lesser extent than small models, but I'm not sure by how much. Experimentation will be needed to figure out which weights are more important for the _M (and _L?) variants of k-quants for Mamba. * convert : fix wrong name for layer norm weight of offical Mamba models I was using Q-bert/Mamba-* models before, which have a slighlty different naming scheme for the weights. (they start with "model.layers" instead of "backbone.layers") * mamba : fuse more steps of the SSM scan in the ggml_ssm_scan operator This increases performance on CPU by around 30% for prompt processing, and by around 20% for text generation. However, it also makes the ggml_exp and ggml_soft_plus operators unused. Whether or not they should be kept will be decided later. * convert : for Mamba, also consider the "MambaLMHeadModel" arch name It's the name of the class of the official implementation, though they don't use it (yet) in the "architectures" field of config.json * mamba : fix vocab size problems with official models The perplexity was waaaay to high for models with a non-round vocab size. Not sure why, but it needed to be fixed in the metadata. Note that this breaks existing GGUF-converted Mamba models, but **only if** the vocab size was not already rounded. * ggml : remove ggml_exp and ggml_soft_plus They did not exist anyway outside of this branch, and since ggml_ssm_scan fused operations together, they are unused. It's always possible to bring them back if needed. * mamba : remove some useless comments No code change. * convert : fix flake8 linter errors * mamba : apply suggestions from code review * mamba : remove unecessary branch for row-wise ssm_state and C multiplication It was previously done to avoid permuting when only one token is processed at a time (like when generating text), but permuting is cheap, and dynamically changing the compute graph is not future-proof. * ggml : in ggml_ssm_scan, use more appropriate asserts * ggml : rename the destination pointer in ggml_compute_forward_ssm_scan_f32 * mamba : multiple sequences, but one at a time This is a step towards making this Mamba implementation usable with the server example (the way the system prompt is kept when clearing the client slots will need to be changed before this can work, though). The KV cache size for this kind of model is tied to the maximum number of sequences kept at any single time. For now, this number is obtained from n_parallel (plus one, to have an extra sequence to dedicate to the system prompt), but there might be a better way to do this which won't also make the main example use 2 cells even if only 1 is really used. (for this specific case, --parallel 0 helps) Simultaneous sequence processing will probably require changes to ggml_ssm_scan, and possibly a new operator for the conv step. * mamba : support llama_kv_cache_seq_cp This (mis)uses the logic around K shifts, because tokens in a state can't be shifted anyway, and because inp_K_shift has the right shape and type. Using ggml_get_rows is a nice way to do copies, but copy chains can't work. Fortunately, copy chains don't really seem to be used in the examples. Each KV cell is dedicated to the sequence ID corresponding to its own index. * mamba : use a state mask It's cleaner than the previous heuristic of checking for the pos of the first token in the batch. inp_KQ_mask could not be re-used for this, because it has the wrong shape and because it seems more suited to the next step of simultaneous sequence processing (helping with the problem of remembering which token belongs to which sequence(s)/state(s)). * llama : replace the usage of n_ctx with kv_self.size in many places * mamba : use n_tokens directly instead of n_tok * mamba : in comments, properly refer to KV cells instead of slots * mamba : reduce memory usage of ggml_ssm_scan From 290.37 MiB to 140.68 MiB of CPU compute buffer size with Mamba 3B with a batch size of 512. The result tensor of ggml_ssm_scan was previously a big part of the CPU compute buffer size. To make it smaller, it does not contain the intermediate ssm states anymore. Both y and the last ssm state are combined in the result tensor, because it seems only a single tensor can be returned by an operator with the way the graph is built. * mamba : simultaneous sequence processing A batch can now contain tokens from multiple sequences. This is necessary for at least the parallel example, the server example, and the HellaSwag test in the perplexity example. However, for this to be useful, uses of llama_kv_cache_seq_rm/cp will need to be changed to work on whole sequences. * ggml : add ggml_ssm_conv as a new operator for the conv step of Mamba This operator makes it possible to use and update the correct states for each token of the batch in the same way as ggml_ssm_scan. Other solutions which use existing operators would need loops which would add too many nodes to the graph (at least the ones I thought of). Using this operator further reduces the size of the CPU compute buffer from 140.68 MiB to 103.20 MiB with Mamba 3B with a batch size of 512. And (at least on CPU), it's a bit faster than before. Note that "ggml_ssm_conv" is probably not the most appropriate name, and it could be changed if a better one is found. * llama : add inp_s_seq as a new input tensor The most convenient implementation to select the correct state (for Mamba) for each token is to directly get the correct index from a tensor. This is why inp_s_seq is storing int32_t and not floats. The other, less convenient way to select the correct state would be to have inp_KQ_mask contain 1.0f for each state used by a token and 0.0f otherwise. This complicates quickly fetching the first used state of a token, and is also less efficient because a whole row of the mask would always need to be read for each token. Using indexes makes it easy to stop searching when there are no more sequences for a token, and the first sequence assigned is always very quickly available (it's the first element of each row). * mamba : support llama_kv_cache_seq_cp copy chains * mamba : support shifting and dividing the kv cache pos * mamba : make the server and parallel examples work with whole sequences A seq_id is dedicated to the system prompt in both cases. * llama : make llama_kv_cache_seq_rm return whether it succeeded or not * mamba : dedicate an input tensor for state copy indices This is cleaner and makes it easier to adapt when/if token positions (and by extension, inp_K_shift) are no longer integers. * mamba : adapt perplexity, batched, and batched-bench examples * perplexity : limit the max number of sequences This adapts to what the loaded model can provide. * llama : add llama_n_max_seq to get the upper limit for seq_ids Used by the perplexity example. * batched : pass n_parallel to the model's context params This should have been there already, but it wasn't. * batched-bench : reserve sequences to support Mamba * batched-bench : fix tokens being put in wrong sequences Generation quality isn't what's measured in there anyway, but at least using the correct sequences avoids using non-consecutive token positions. * mamba : stop abusing attention metadata This breaks existing converted-to-GGUF Mamba models, but will allow supporting mixed architectures like MambaFormer without needing to break Mamba models. This will also allow changing the size of Mamba's states without having to reconvert models in the future. (e.g. using something else than d_conv - 1 columns for the conv_states will not require breaking existing converted Mamba models again) * gguf-py : add new KV metadata key-value pairs for Mamba * llama : add new metadata key-value pairs for Mamba * llama : guard against divisions by zero when n_head is 0 * mamba : rename "unlimited" KV cache property to "recurrent" * mamba : more correctly update the "used" field of the KV cache * ggml : in ggml_ssm_scan, use a threshold for soft_plus This is how the official Mamba implementation does it, and it's also what torch.nn.Softplus does. * convert : for Mamba, fallback to internal NeoX tokenizer The resulting models are exactly the same as if the tokenizer.json and tokenizer_config.json of GPT-NeoX were there. * mamba : support state saving and restoring * ggml : implicitly pass src tensors through dst for Mamba-related ops * mamba : clarify some comments * server : fix cache_tokens not getting correctly resized Otherwise, when the "we have to evaluate at least 1 token" special case was triggered, an extra token was kept in cache_tokens even if it was removed from the KV cache. For Mamba, this caused useless prompt reprocessing when the previous request triggered the above case. * convert-hf : support new metadata keys for Mamba For the models available at https://huggingface.co/collections/state-spaces/transformers-compatible-mamba-65e7b40ab87e5297e45ae406 * mamba : rename metadata to be more similar to transformers library This breaks existing converted-to-GGUF models, but the metadata names are more "standard". * mamba : support mamba-*-hf models These models share their token_embd.weight with their output.weight * mamba : add missing spaces This is purely a formatting change. * convert-hf : omit output.weight when identical with token_embd.weight Only for Mamba for now, but it might be relevant for other models eventually. Most Mamba models actually share these two tensors, albeit implicitly. * readme : add Mamba to supported models, and add recent API changes * mamba : move state_seq and state_mask views outside layer loop A few tensors were also missing `struct` in front of `ggml_tensor`. --- README.md | 2 + common/common.cpp | 1 + convert-hf-to-gguf.py | 118 ++++ examples/batched-bench/batched-bench.cpp | 13 +- examples/batched/batched.cpp | 3 +- examples/parallel/parallel.cpp | 20 +- examples/perplexity/perplexity.cpp | 9 +- examples/server/server.cpp | 51 +- ggml.c | 379 +++++++++++- ggml.h | 19 + gguf-py/gguf/constants.py | 41 ++ gguf-py/gguf/gguf_writer.py | 12 + gguf-py/gguf/tensor_mapping.py | 46 +- llama.cpp | 698 +++++++++++++++++++++-- llama.h | 4 +- 15 files changed, 1342 insertions(+), 74 deletions(-) diff --git a/README.md b/README.md index f754022de..d7dba73e6 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,7 @@ Inference of Meta's [LLaMA](https://arxiv.org/abs/2302.13971) model (and others) ### Recent API changes +- [2024 Mar 8] `llama_kv_cache_seq_rm()` returns a `bool` instead of `void`, and new `llama_n_max_seq()` returns the upper limit of acceptable `seq_id` in batches (relevant when dealing with multiple sequences) https://github.com/ggerganov/llama.cpp/pull/5328 - [2024 Mar 4] Embeddings API updated https://github.com/ggerganov/llama.cpp/pull/5796 - [2024 Mar 3] `struct llama_context_params` https://github.com/ggerganov/llama.cpp/pull/5849 @@ -110,6 +111,7 @@ Typically finetunes of the base models below are supported as well. - [x] [InternLM2](https://huggingface.co/models?search=internlm2) - [x] [CodeShell](https://github.com/WisdomShell/codeshell) - [x] [Gemma](https://ai.google.dev/gemma) +- [x] [Mamba](https://github.com/state-spaces/mamba) **Multimodal models:** diff --git a/common/common.cpp b/common/common.cpp index c244db644..d7f650ef4 100644 --- a/common/common.cpp +++ b/common/common.cpp @@ -1288,6 +1288,7 @@ struct llama_context_params llama_context_params_from_gpt_params(const gpt_param cparams.n_ctx = params.n_ctx; cparams.n_batch = params.n_batch; + cparams.n_parallel = params.n_parallel; cparams.n_threads = params.n_threads; cparams.n_threads_batch = params.n_threads_batch == -1 ? params.n_threads : params.n_threads_batch; cparams.seed = params.seed; diff --git a/convert-hf-to-gguf.py b/convert-hf-to-gguf.py index f6369af38..5eee32016 100755 --- a/convert-hf-to-gguf.py +++ b/convert-hf-to-gguf.py @@ -1847,6 +1847,124 @@ class StarCoder2Model(Model): model_arch = gguf.MODEL_ARCH.STARCODER2 +@Model.register("MambaForCausalLM", "MambaLMHeadModel") +class MambaModel(Model): + model_arch = gguf.MODEL_ARCH.MAMBA + + def set_vocab(self): + vocab_size = self.hparams["vocab_size"] + # Round vocab size to next multiple of 8 + pad_vocab = self.hparams.get("pad_vocab_size_multiple", 8) + # pad using ceiling division + # ref: https://stackoverflow.com/a/17511341/22827863 + vocab_size = -(vocab_size // -pad_vocab) * pad_vocab + self.hparams["vocab_size"] = vocab_size + + if (self.dir_model / "tokenizer.json").is_file(): + self._set_vocab_gpt2() + else: + # Use the GPT-NeoX tokenizer when no tokenizer files are present + tokenizer_path = Path(sys.path[0]) / "models" / "ggml-vocab-gpt-neox.gguf" + print(f"Using tokenizer from '{os.path.relpath(tokenizer_path, os.getcwd())}'") + neox_reader = gguf.GGUFReader(tokenizer_path, "r") + + field = neox_reader.get_field(gguf.Keys.Tokenizer.MODEL) + self.gguf_writer.add_tokenizer_model(bytes(field.parts[-1])) + field = neox_reader.get_field(gguf.Keys.Tokenizer.LIST) + self.gguf_writer.add_token_list([bytes(field.parts[i]) for i in field.data][:vocab_size]) + field = neox_reader.get_field(gguf.Keys.Tokenizer.TOKEN_TYPE) + self.gguf_writer.add_token_types([field.parts[i].tolist()[0] for i in field.data][:vocab_size]) + field = neox_reader.get_field(gguf.Keys.Tokenizer.MERGES) + self.gguf_writer.add_token_merges([bytes(field.parts[i]) for i in field.data]) + field = neox_reader.get_field(gguf.Keys.Tokenizer.BOS_ID) + self.gguf_writer.add_bos_token_id(field.parts[-1].tolist()[0]) + field = neox_reader.get_field(gguf.Keys.Tokenizer.EOS_ID) + self.gguf_writer.add_eos_token_id(field.parts[-1].tolist()[0]) + field = neox_reader.get_field(gguf.Keys.Tokenizer.UNK_ID) + self.gguf_writer.add_unk_token_id(field.parts[-1].tolist()[0]) + + def set_gguf_parameters(self): + d_model = self.find_hparam(["hidden_size", "d_model"]) + d_conv = self.find_hparam(["conv_kernel", "d_conv"], optional=True) or 4 + d_inner = self.find_hparam(["intermediate_size", "d_inner"], optional=True) or 2 * d_model + d_state = self.find_hparam(["state_size", "d_state"], optional=True) or 16 + # ceiling division + # ref: https://stackoverflow.com/a/17511341/22827863 + # ref: https://github.com/state-spaces/mamba/blob/ce59daea3a090d011d6476c6e5b97f6d58ddad8b/mamba_ssm/modules/mamba_simple.py#L58 + dt_rank = self.find_hparam(["time_step_rank", "dt_rank"], optional=True) or -(d_model // -16) + rms_norm_eps = self.find_hparam(["layer_norm_epsilon", "rms_norm_eps"], optional=True) or 1e-5 + + # Fail early for models which don't have a block expansion factor of 2 + assert d_inner == 2 * d_model + + self.gguf_writer.add_name(self.dir_model.name) + self.gguf_writer.add_context_length(2**20) # arbitrary value; for those who use the default + self.gguf_writer.add_embedding_length(d_model) + self.gguf_writer.add_feed_forward_length(0) # unused, but seemingly required when loading + self.gguf_writer.add_head_count(0) # unused, but seemingly required when loading + self.gguf_writer.add_block_count(self.hparams["n_layer"]) + self.gguf_writer.add_ssm_conv_kernel(d_conv) + self.gguf_writer.add_ssm_inner_size(d_inner) + self.gguf_writer.add_ssm_state_size(d_state) + self.gguf_writer.add_ssm_time_step_rank(dt_rank) + self.gguf_writer.add_layer_norm_rms_eps(rms_norm_eps) + self.gguf_writer.add_file_type(self.ftype) + + def write_tensors(self): + block_count = self.hparams["n_layer"] + tensor_map = gguf.get_tensor_name_map(self.model_arch, block_count) + + tok_embd = None + tok_embd_name = gguf.TENSOR_NAMES[gguf.MODEL_TENSOR.TOKEN_EMBD] + ".weight" + output_name = gguf.TENSOR_NAMES[gguf.MODEL_TENSOR.OUTPUT] + ".weight" + + for name, data_torch in self.get_tensors(): + old_dtype = data_torch.dtype + + # convert any unsupported data types to float32 + if data_torch.dtype not in (torch.float16, torch.float32): + data_torch = data_torch.to(torch.float32) + + # map tensor names + new_name = tensor_map.get_name(name, try_suffixes=(".weight", ".bias")) + if new_name is None: + print(f"Can not map tensor {name!r}") + sys.exit() + + if name.endswith(".A_log"): + print("A_log --> A ==> " + new_name) + data_torch = -torch.exp(data_torch) + + # assuming token_embd.weight is seen before output.weight + if tok_embd is not None and new_name == output_name: + if torch.equal(tok_embd, data_torch): + print(f"{output_name} is equivalent to {tok_embd_name}, omitting") + continue + if new_name == tok_embd_name: + tok_embd = data_torch + + data = data_torch.squeeze().numpy() + + n_dims = len(data.shape) + data_dtype = data.dtype + + # if f32 desired, convert any float16 to float32 + if self.ftype == 0 and data_dtype == np.float16: + data = data.astype(np.float32) + + # TODO: Why cant we use these float16 as-is? There should be not reason to store float16 as float32 + if self.ftype == 1 and data_dtype == np.float16 and n_dims == 1: + data = data.astype(np.float32) + + # if f16 desired, convert big float32 2-dim weight tensors to float16 + if self.ftype == 1 and data_dtype == np.float32 and new_name.removesuffix(".weight").endswith((".ssm_in", ".ssm_out", "token_embd", "output")) and n_dims == 2: + data = data.astype(np.float16) + + print(f"{new_name}, n_dims = {n_dims}, {old_dtype} --> {data.dtype}") + + self.gguf_writer.add_tensor(new_name, data) + + ###### CONVERSION LOGIC ###### diff --git a/examples/batched-bench/batched-bench.cpp b/examples/batched-bench/batched-bench.cpp index 19aff18ae..dff6c68ec 100644 --- a/examples/batched-bench/batched-bench.cpp +++ b/examples/batched-bench/batched-bench.cpp @@ -105,6 +105,9 @@ int main(int argc, char ** argv) { ctx_params.n_threads = params.n_threads; ctx_params.n_threads_batch = params.n_threads_batch == -1 ? params.n_threads : params.n_threads_batch; + // ensure enough sequences are available + ctx_params.n_parallel = *std::max_element(n_pl.begin(), n_pl.end()); + llama_context * ctx = llama_new_context_with_model(model, ctx_params); if (ctx == NULL) { @@ -174,10 +177,10 @@ int main(int argc, char ** argv) { llama_batch_clear(batch); - const int n_tokens = is_pp_shared ? pp : pl*pp; - - for (int i = 0; i < n_tokens; ++i) { - llama_batch_add(batch, 0, i, { 0 }, false); + for (int i = 0; i < pp; ++i) { + for (int j = 0; j < (is_pp_shared ? 1 : pl); ++j) { + llama_batch_add(batch, 0, i, { j }, false); + } } batch.logits[batch.n_tokens - 1] = true; @@ -192,7 +195,7 @@ int main(int argc, char ** argv) { if (is_pp_shared) { for (int32_t i = 1; i < pl; ++i) { - llama_kv_cache_seq_cp(ctx, 0, i, 0, pp); + llama_kv_cache_seq_cp(ctx, 0, i, -1, -1); } } diff --git a/examples/batched/batched.cpp b/examples/batched/batched.cpp index 9be7eb56b..dde4d5a06 100644 --- a/examples/batched/batched.cpp +++ b/examples/batched/batched.cpp @@ -80,6 +80,7 @@ int main(int argc, char ** argv) { ctx_params.seed = 1234; ctx_params.n_ctx = n_kv_req; ctx_params.n_batch = std::max(n_len, n_parallel); + ctx_params.n_parallel = n_parallel; ctx_params.n_threads = params.n_threads; ctx_params.n_threads_batch = params.n_threads_batch == -1 ? params.n_threads : params.n_threads_batch; @@ -132,7 +133,7 @@ int main(int argc, char ** argv) { // assign the system KV cache to all parallel sequences // this way, the parallel sequences will "reuse" the prompt tokens without having to copy them for (int32_t i = 1; i < n_parallel; ++i) { - llama_kv_cache_seq_cp(ctx, 0, i, 0, batch.n_tokens); + llama_kv_cache_seq_cp(ctx, 0, i, -1, -1); } if (n_parallel > 1) { diff --git a/examples/parallel/parallel.cpp b/examples/parallel/parallel.cpp index 7d11fcd59..a2ef0fb03 100644 --- a/examples/parallel/parallel.cpp +++ b/examples/parallel/parallel.cpp @@ -107,6 +107,9 @@ int main(int argc, char ** argv) { // number of simultaneous "clients" to simulate const int32_t n_clients = params.n_parallel; + // dedicate one sequence to the system prompt + params.n_parallel += 1; + // requests to simulate const int32_t n_seq = params.n_sequences; @@ -196,8 +199,8 @@ int main(int argc, char ** argv) { } // assign the system KV cache to all parallel sequences - for (int32_t i = 1; i < n_clients; ++i) { - llama_kv_cache_seq_cp(ctx, 0, i, 0, n_tokens_system); + for (int32_t i = 1; i <= n_clients; ++i) { + llama_kv_cache_seq_cp(ctx, 0, i, -1, -1); } LOG_TEE("\n"); @@ -221,15 +224,17 @@ int main(int argc, char ** argv) { client.i_batch = batch.n_tokens; - llama_batch_add(batch, client.sampled, n_tokens_system + client.n_prompt + client.n_decoded, { client.id }, true); + llama_batch_add(batch, client.sampled, n_tokens_system + client.n_prompt + client.n_decoded, { client.id + 1 }, true); client.n_decoded += 1; } if (batch.n_tokens == 0) { // all sequences have ended - clear the entire KV cache - for (int i = 0; i < n_clients; ++i) { - llama_kv_cache_seq_rm(ctx, i, n_tokens_system, -1); + for (int i = 1; i <= n_clients; ++i) { + llama_kv_cache_seq_rm(ctx, i, -1, -1); + // but keep the system prompt + llama_kv_cache_seq_cp(ctx, 0, i, -1, -1); } LOG_TEE("%s: clearing the KV cache\n", __func__); @@ -255,7 +260,7 @@ int main(int argc, char ** argv) { tokens_prompt = ::llama_tokenize(ctx, client.prompt, false); for (size_t i = 0; i < tokens_prompt.size(); ++i) { - llama_batch_add(batch, tokens_prompt[i], i + n_tokens_system, { client.id }, false); + llama_batch_add(batch, tokens_prompt[i], i + n_tokens_system, { client.id + 1 }, false); } // extract the logits only for the last token @@ -366,7 +371,8 @@ int main(int argc, char ** argv) { } // delete only the generated part of the sequence, i.e. keep the system prompt in the cache - llama_kv_cache_seq_rm(ctx, client.id, n_tokens_system, -1); + llama_kv_cache_seq_rm(ctx, client.id + 1, -1, -1); + llama_kv_cache_seq_cp(ctx, 0, client.id + 1, -1, -1); const auto t_main_end = ggml_time_us(); diff --git a/examples/perplexity/perplexity.cpp b/examples/perplexity/perplexity.cpp index 9ec989389..52789ee63 100644 --- a/examples/perplexity/perplexity.cpp +++ b/examples/perplexity/perplexity.cpp @@ -809,7 +809,7 @@ static void hellaswag_score(llama_context * ctx, const gpt_params & params) { const int n_batch = params.n_batch; const int max_tasks_per_batch = 32; - const int max_seq = 4*max_tasks_per_batch; + const int max_seq = std::min(4*max_tasks_per_batch, (int) llama_n_max_seq(ctx)); llama_batch batch = llama_batch_init(n_ctx, 0, max_seq); @@ -1086,7 +1086,7 @@ static void winogrande_score(llama_context * ctx, const gpt_params & params) { const int n_batch = params.n_batch; const int max_tasks_per_batch = 128; - const int max_seq = 2*max_tasks_per_batch; + const int max_seq = std::min(2*max_tasks_per_batch, (int) llama_n_max_seq(ctx)); llama_batch batch = llama_batch_init(n_ctx, 0, max_seq); @@ -1438,7 +1438,7 @@ static void multiple_choice_score(llama_context * ctx, const gpt_params & params const int n_batch = params.n_batch; const int max_tasks_per_batch = 32; - const int max_seq = 4*max_tasks_per_batch; + const int max_seq = std::min(4*max_tasks_per_batch, (int) llama_n_max_seq(ctx)); llama_batch batch = llama_batch_init(n_ctx, 0, max_seq); @@ -1815,6 +1815,9 @@ int main(int argc, char ** argv) { llama_model * model; llama_context * ctx; + // ensure there's at least enough seq_ids for HellaSwag + params.n_parallel = std::max(4, params.n_parallel); + // load the model and apply lora adapter, if any std::tie(model, ctx) = llama_init_from_gpt_params(params); if (model == NULL) { diff --git a/examples/server/server.cpp b/examples/server/server.cpp index 109ff7175..59a59d56b 100644 --- a/examples/server/server.cpp +++ b/examples/server/server.cpp @@ -659,7 +659,11 @@ struct server_context { bool load_model(const gpt_params & params_) { params = params_; + // dedicate one sequence to the system prompt + params.n_parallel += 1; + std::tie(model, ctx) = llama_init_from_gpt_params(params); + params.n_parallel -= 1; // but be sneaky about it if (model == nullptr) { LOG_ERROR("unable to load model", {{"model", params.model}}); return false; @@ -1018,8 +1022,8 @@ struct server_context { } // assign the system KV cache to all parallel sequences - for (int32_t i = 1; i < params.n_parallel; ++i) { - llama_kv_cache_seq_cp(ctx, 0, i, 0, system_tokens.size()); + for (int32_t i = 1; i <= params.n_parallel; ++i) { + llama_kv_cache_seq_cp(ctx, 0, i, -1, -1); } } @@ -1306,7 +1310,7 @@ struct server_context { const int n_embd = llama_n_embd(model); for (int i = 0; i < batch.n_tokens; ++i) { - if (!batch.logits[i] || batch.seq_id[i][0] != slot.id) { + if (!batch.logits[i] || batch.seq_id[i][0] != slot.id + 1) { continue; } @@ -1633,8 +1637,8 @@ struct server_context { {"n_cache_tokens", slot.cache_tokens.size()} }); - llama_kv_cache_seq_rm (ctx, slot.id, n_keep , n_keep + n_discard); - llama_kv_cache_seq_add(ctx, slot.id, n_keep + n_discard, system_tokens.size() + slot.n_past, -n_discard); + llama_kv_cache_seq_rm (ctx, slot.id + 1, n_keep , n_keep + n_discard); + llama_kv_cache_seq_add(ctx, slot.id + 1, n_keep + n_discard, system_tokens.size() + slot.n_past, -n_discard); if (slot.params.cache_prompt) { for (size_t i = n_keep + n_discard; i < slot.cache_tokens.size(); i++) { @@ -1666,7 +1670,7 @@ struct server_context { // TODO: we always have to take into account the "system_tokens" // this is not great and needs to be improved somehow - llama_batch_add(batch, slot.sampled, system_tokens.size() + slot_npast, { slot.id }, true); + llama_batch_add(batch, slot.sampled, system_tokens.size() + slot_npast, { slot.id + 1 }, true); slot.n_past += 1; @@ -1804,9 +1808,6 @@ struct server_context { // reuse any previously computed tokens that are common with the new prompt slot.n_past = common_part(slot.cache_tokens, prompt_tokens); - // remove the non-common part from the cache - slot.cache_tokens.resize(slot.n_past); - // push the prompt into the sampling context (do not apply grammar) for (int i = 0; i < slot.n_past; ++i) { llama_sampling_accept(slot.ctx_sampling, ctx, slot.cache_tokens[i], false); @@ -1837,8 +1838,28 @@ struct server_context { } } - const int p0 = (int) system_tokens.size() + slot.n_past; - llama_kv_cache_seq_rm(ctx, slot.id, p0, -1); + // keep only the common part + int p0 = (int) system_tokens.size() + slot.n_past; + if (!llama_kv_cache_seq_rm(ctx, slot.id + 1, p0, -1)) { + // could not partially delete (likely using a non-Transformer model) + llama_kv_cache_seq_rm(ctx, slot.id + 1, -1, -1); + + p0 = (int) system_tokens.size(); + if (p0 != 0) { + // copy over the system prompt when there is one + llama_kv_cache_seq_cp(ctx, 0, slot.id + 1, -1, -1); + } + + // there is no common part left (except for the system prompt) + slot.n_past = 0; + slot.n_past_se = 0; + slot.ga_i = 0; + // TODO: is the system prompt ever in the sampling context? + llama_sampling_reset(slot.ctx_sampling); + } + + // remove the non-common part from the cache + slot.cache_tokens.resize(slot.n_past); LOG_INFO("kv cache rm [p0, end)", { { "id_slot", slot.id }, @@ -1863,7 +1884,7 @@ struct server_context { } } - llama_batch_add(batch, prompt_tokens[slot.n_past], system_tokens.size() + slot_npast, { slot.id }, false); + llama_batch_add(batch, prompt_tokens[slot.n_past], system_tokens.size() + slot_npast, { slot.id + 1 }, false); if (slot.params.cache_prompt) { slot.cache_tokens.push_back(prompt_tokens[slot.n_past]); @@ -1937,9 +1958,9 @@ struct server_context { LOG_TEE("div: [%6d, %6d] / %6d -> [%6d, %6d]\n", slot.ga_i + ib * bd, slot.ga_i + ib * bd + slot.ga_w, slot.ga_n, (slot.ga_i + ib * bd) / slot.ga_n, (slot.ga_i + ib * bd + slot.ga_w) / slot.ga_n); LOG_TEE("shift: [%6d, %6d] + %6d -> [%6d, %6d]\n", slot.ga_i + ib * bd + slot.ga_w, slot.n_past_se + ib * bd, dd, slot.ga_i + ib * bd + slot.ga_w + dd, slot.n_past_se + ib * bd + dd); - llama_kv_cache_seq_add(ctx, slot.id, slot.ga_i, slot.n_past_se, ib * bd); - llama_kv_cache_seq_div(ctx, slot.id, slot.ga_i + ib * bd, slot.ga_i + ib * bd + slot.ga_w, slot.ga_n); - llama_kv_cache_seq_add(ctx, slot.id, slot.ga_i + ib * bd + slot.ga_w, slot.n_past_se + ib * bd, dd); + llama_kv_cache_seq_add(ctx, slot.id + 1, slot.ga_i, slot.n_past_se, ib * bd); + llama_kv_cache_seq_div(ctx, slot.id + 1, slot.ga_i + ib * bd, slot.ga_i + ib * bd + slot.ga_w, slot.ga_n); + llama_kv_cache_seq_add(ctx, slot.id + 1, slot.ga_i + ib * bd + slot.ga_w, slot.n_past_se + ib * bd, dd); slot.n_past_se -= bd; diff --git a/ggml.c b/ggml.c index 92b17ee6e..6eff98ab6 100644 --- a/ggml.c +++ b/ggml.c @@ -1841,6 +1841,8 @@ static const char * GGML_OP_NAME[GGML_OP_COUNT] = { "FLASH_ATTN", "FLASH_FF", "FLASH_ATTN_BACK", + "SSM_CONV", + "SSM_SCAN", "WIN_PART", "WIN_UNPART", "GET_REL_POS", @@ -1863,7 +1865,7 @@ static const char * GGML_OP_NAME[GGML_OP_COUNT] = { "CROSS_ENTROPY_LOSS_BACK", }; -static_assert(GGML_OP_COUNT == 74, "GGML_OP_COUNT != 74"); +static_assert(GGML_OP_COUNT == 76, "GGML_OP_COUNT != 76"); static const char * GGML_OP_SYMBOL[GGML_OP_COUNT] = { "none", @@ -1929,6 +1931,8 @@ static const char * GGML_OP_SYMBOL[GGML_OP_COUNT] = { "flash_attn(x)", "flash_ff(x)", "flash_attn_back(x)", + "ssm_conv(x)", + "ssm_scan(x)", "win_part(x)", "win_unpart(x)", "get_rel_pos(x)", @@ -1951,7 +1955,7 @@ static const char * GGML_OP_SYMBOL[GGML_OP_COUNT] = { "cross_entropy_loss_back(x,y)", }; -static_assert(GGML_OP_COUNT == 74, "GGML_OP_COUNT != 74"); +static_assert(GGML_OP_COUNT == 76, "GGML_OP_COUNT != 76"); static_assert(GGML_OP_POOL_COUNT == 2, "GGML_OP_POOL_COUNT != 2"); @@ -6154,6 +6158,108 @@ struct ggml_tensor * ggml_flash_attn_back( return result; } +// ggml_ssm_conv + +struct ggml_tensor * ggml_ssm_conv( + struct ggml_context * ctx, + struct ggml_tensor * s, + struct ggml_tensor * x, + struct ggml_tensor * c, + struct ggml_tensor * sq) { + GGML_ASSERT(ggml_is_3d(s)); + GGML_ASSERT(ggml_is_matrix(x)); + GGML_ASSERT(ggml_is_matrix(c)); + GGML_ASSERT(ggml_is_matrix(sq)); + GGML_ASSERT(sq->type == GGML_TYPE_I32); + + const int64_t d_conv = c->ne[0]; + const int64_t d_inner = c->ne[1]; + const int64_t n_tokens = x->ne[1]; + const int64_t n_kv = s->ne[2]; + + GGML_ASSERT( s->ne[0] == d_conv - 1); + GGML_ASSERT( s->ne[1] == d_inner); + GGML_ASSERT( x->ne[0] == d_inner); + GGML_ASSERT(sq->ne[0] == n_kv); + GGML_ASSERT(sq->ne[1] == n_tokens); + + bool is_node = false; + + if (s->grad || x->grad || c->grad || sq->grad) { + GGML_ASSERT(false); // TODO: implement + is_node = true; + } + + // 2-in-1 concatenated x and conv_states, {d_inner, n_tokens} with {d_conv, d_inner, n_kv} + struct ggml_tensor * result = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, (d_inner*n_tokens) + (d_conv*d_inner*n_kv)); + + result->op = GGML_OP_SSM_CONV; + result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL; + result->src[0] = s; + result->src[1] = x; + result->src[2] = c; + result->src[3] = sq; + + return result; +} + +// ggml_ssm_scan + +struct ggml_tensor * ggml_ssm_scan( + struct ggml_context * ctx, + struct ggml_tensor * s, + struct ggml_tensor * x, + struct ggml_tensor * dt, + struct ggml_tensor * A, + struct ggml_tensor * B, + struct ggml_tensor * C, + struct ggml_tensor * sq) { + GGML_ASSERT(ggml_is_contiguous(s)); + GGML_ASSERT(ggml_is_contiguous(x)); + GGML_ASSERT(ggml_is_contiguous(dt)); + GGML_ASSERT(ggml_is_contiguous(A)); + GGML_ASSERT(sq->type == GGML_TYPE_I32); + GGML_ASSERT(B->nb[0] == ggml_type_size(B->type)); + GGML_ASSERT(C->nb[0] == ggml_type_size(C->type)); + GGML_ASSERT(ggml_are_same_shape(x, dt)); + + { + const int64_t d_state = s->ne[0]; + const int64_t d_inner = s->ne[1]; + const int64_t n_tokens = x->ne[1]; + + GGML_ASSERT(x->ne[0] == d_inner); + GGML_ASSERT(A->ne[0] == d_state); + GGML_ASSERT(A->ne[1] == d_inner); + GGML_ASSERT(B->ne[0] == d_state); + GGML_ASSERT(B->ne[1] == n_tokens); + GGML_ASSERT(C->ne[0] == d_state); + GGML_ASSERT(C->ne[1] == n_tokens); + } + + bool is_node = false; + + if (s->grad || x->grad || dt->grad || A->grad || B->grad || C->grad || sq->grad) { + GGML_ASSERT(false); // TODO: implement + is_node = true; + } + + // 2-in-1 concatenated y and ssm_states, {d_inner, n_tokens} with {d_state, d_inner, n_kv} + struct ggml_tensor * result = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, ggml_nelements(x) + ggml_nelements(s)); + + result->op = GGML_OP_SSM_SCAN; + result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL; + result->src[0] = s; + result->src[1] = x; + result->src[2] = dt; + result->src[3] = A; + result->src[4] = B; + result->src[5] = C; + result->src[6] = sq; + + return result; +} + // ggml_win_part struct ggml_tensor * ggml_win_part( @@ -14771,6 +14877,257 @@ static void ggml_compute_forward_flash_attn_back( } } +// ggml_compute_forward_ssm_conv + +static void ggml_compute_forward_ssm_conv_f32( + const struct ggml_compute_params * params, + struct ggml_tensor * dst) { + if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) { + return; + } + + const struct ggml_tensor * src0 = dst->src[0]; // conv_state + const struct ggml_tensor * src1 = dst->src[1]; // x + const struct ggml_tensor * src2 = dst->src[2]; // conv1d.weight + const struct ggml_tensor * src3 = dst->src[3]; // state_seq + + const int ith = params->ith; + const int nth = params->nth; + + const int nc = src2->ne[0]; // d_conv + const int nr = src0->ne[1]; // d_inner + const int n_t = src1->ne[1]; // n_tokens + const int n_kv = src0->ne[2]; // max number of sequences in the batch + + GGML_ASSERT((nr*n_t) + (nc*nr*n_kv) == ggml_nelements(dst)); + GGML_ASSERT(src0->nb[0] == sizeof(float)); + GGML_ASSERT(src1->nb[0] == sizeof(float)); + GGML_ASSERT(src2->nb[0] == sizeof(float)); + GGML_ASSERT(src3->nb[0] == sizeof(int32_t)); + GGML_ASSERT(src0->nb[1] == src0->ne[0]*sizeof(float)); + // for use with the destination state offset between sequences + GGML_ASSERT(src2->nb[2] == src2->ne[1]*src2->ne[0]*sizeof(float)); + + // rows per thread + const int dr = (nr + nth - 1)/nth; + + // row range for this thread + const int ir0 = dr*ith; + const int ir1 = MIN(ir0 + dr, nr); + const int ir = ir1 - ir0; + + if (n_kv > 1) { + // multiple sequences means it's hard to know when it's the first time a state is read, + // so copy them all over to the destination, just to be sure. + for (int i3 = 0; i3 < n_kv; ++i3) { + float * s0 = (float *) ((char *) src0->data + ir0*(src0->nb[1]) + i3*(src0->nb[2])); + float * s = (float *) ((char *) dst->data + ir0*(src2->nb[1]) + i3*(src2->nb[2]) + nr*n_t*sizeof(float)); + // can't use memcpy because of d_conv vs d_conv - 1 + for (int i1 = 0; i1 < ir; ++i1) { + for (int i0 = 0; i0 < nc - 1; ++i0) { + // copy s0 to last (d_conv - 1) columns of s + s[1 + i0 + i1*nc] = s0[i0 + i1*(nc - 1)]; + } + } + } + } + + for (int i2 = 0; i2 < n_t; ++i2) { + int32_t * sq = (int32_t *) ((char *) src3->data + i2*(src3->nb[1])); // {n_kv, n_tokens} + float * x = (float *) ((char *) dst->data + ir0*sizeof(float) + i2*(nr*sizeof(float))); // {d_inner, n_tokens} + float * s = (float *) ((char *) dst->data + ir0*(src2->nb[1]) + sq[0]*(src2->nb[2]) + nr*n_t*sizeof(float)); // {d_conv, d_inner, n_kv} + float * s0; // {d_conv - 1, d_inner, n_kv} + float * x0 = (float *) ((char *) src1->data + ir0*(src1->nb[0]) + i2*(src1->nb[1])); // {d_inner, n_tokens} + float * c = (float *) ((char *) src2->data + ir0*(src2->nb[1])); // {d_conv, d_inner} + int ne0s0; + + GGML_ASSERT(0 <= sq[0] && sq[0] < n_kv); + + // avoid needing to copy the state for the first token + if (i2 == 0) { + s0 = (float *) ((char *) src0->data + ir0*(src0->nb[1]) + sq[0]*(src0->nb[2])); // {d_conv - 1, d_inner, n_kv} + ne0s0 = src0->ne[0]; + } else { + // the source is the last (d_conv - 1) columns of the destination + s0 = s + 1; + ne0s0 = nc; + } + + // d_inner + for (int i1 = 0; i1 < ir; ++i1) { + // shift state left + for (int i0 = 0; i0 < nc - 1; ++i0) { + s[i0 + i1*nc] = s0[i0 + i1*ne0s0]; + } + // insert x on the last column + s[(nc - 1) + i1*nc] = x0[i1]; + } + + // handle copies when there are multiple output states + for (int i3 = 1; i3 < n_kv; ++i3) { + int32_t seq = sq[i3]; + if (0 <= seq && seq < n_kv) { + float * s1 = s + (seq - sq[0])*nc*nr; + memcpy(s1, s, nc*ir*sizeof(float)); + } else { + // stop at negative or too big seq_ids + break; + } + } + + // it seems a little faster when this is separate from the state shift + for (int i1 = 0; i1 < ir; ++i1) { + // rowwise dot product + float sumf = 0.0f; + for (int i0 = 0; i0 < nc; ++i0) { + int i = i0 + i1*nc; + sumf += s[i] * c[i]; + } + x[i1] = sumf; + } + } +} + +static void ggml_compute_forward_ssm_conv( + const struct ggml_compute_params * params, + struct ggml_tensor * dst) { + switch (dst->src[0]->type) { + case GGML_TYPE_F32: + { + ggml_compute_forward_ssm_conv_f32(params, dst); + } break; + default: + { + GGML_ASSERT(false); + } break; + } +} + +// ggml_compute_forward_ssm_scan + +static void ggml_compute_forward_ssm_scan_f32( + const struct ggml_compute_params * params, + struct ggml_tensor * dst) { + if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) { + return; + } + + const struct ggml_tensor * src0 = dst->src[0]; // s + const struct ggml_tensor * src1 = dst->src[1]; // x + const struct ggml_tensor * src2 = dst->src[2]; // dt + const struct ggml_tensor * src3 = dst->src[3]; // A + const struct ggml_tensor * src4 = dst->src[4]; // B + const struct ggml_tensor * src5 = dst->src[5]; // C + const struct ggml_tensor * src6 = dst->src[6]; // sq + + const int ith = params->ith; + const int nth = params->nth; + + const int64_t nc = src0->ne[0]; // d_state + const int64_t nr = src0->ne[1]; // d_inner + const int64_t n_t = src1->ne[1]; // number of tokens in the batch + const int64_t n_kv = src0->ne[2]; // max number of sequences in the batch + + GGML_ASSERT(ggml_nelements(src1) + ggml_nelements(src0) == ggml_nelements(dst)); + GGML_ASSERT(src0->nb[0] == sizeof(float)); + GGML_ASSERT(src1->nb[0] == sizeof(float)); + GGML_ASSERT(src2->nb[0] == sizeof(float)); + GGML_ASSERT(src3->nb[0] == sizeof(float)); + GGML_ASSERT(src4->nb[0] == sizeof(float)); + GGML_ASSERT(src5->nb[0] == sizeof(float)); + // required for the dot product between s and C, and when copying the states + GGML_ASSERT(src0->nb[1] == src0->ne[0]*sizeof(float)); + // required for per-sequence offsets for states + GGML_ASSERT(src0->nb[2] == src0->ne[0]*src0->ne[1]*sizeof(float)); + // required to get correct offset for state destination (i.e. src1->nb[2]) + GGML_ASSERT(src1->nb[2] == src1->ne[0]*src1->ne[1]*sizeof(float)); + + // rows per thread + const int dr = (nr + nth - 1)/nth; + + // row range for this thread + const int ir0 = dr*ith; + const int ir1 = MIN(ir0 + dr, nr); + const int ir = ir1 - ir0; + + if (n_kv > 1) { + // it's hard to know if the source states have already been copied + // when there are multiple, so copy them already. + for (int i3 = 0; i3 < n_kv; ++i3) { + float * s0 = (float *) ((char *) src0->data + ir0*(src0->nb[1]) + i3*(src0->nb[2])); + float * s = (float *) ((char *) dst->data + ir0*(src0->nb[1]) + i3*(src0->nb[2]) + src1->nb[2]); + memcpy(s, s0, nc*ir*sizeof(float)); + } + } + + for (int i2 = 0; i2 < n_t; ++i2) { + int32_t * sq = (int32_t *) ((char *) src6->data + i2*(src6->nb[1])); // {n_kv, n_tokens} + float * y = (float *) ((char *) dst->data + ir0*(src1->nb[0]) + i2*(src1->nb[1])); // {d_inner, n_tokens} + float * s = (float *) ((char *) dst->data + ir0*(src0->nb[1]) + sq[0]*(src0->nb[2]) + src1->nb[2]); // {d_state, d_inner, n_kv} + float * s0; + float * x = (float *) ((char *) src1->data + ir0*(src1->nb[0]) + i2*(src1->nb[1])); // {d_inner, n_tokens} + float * dt = (float *) ((char *) src2->data + ir0*(src2->nb[0]) + i2*(src2->nb[1])); // {d_inner, n_tokens} + float * A = (float *) ((char *) src3->data + ir0*(src3->nb[1])); // {d_state, d_inner} + float * B = (float *) ((char *) src4->data + i2*(src4->nb[1])); // {d_state, n_tokens} + float * C = (float *) ((char *) src5->data + i2*(src5->nb[1])); // {d_state, n_tokens} + + GGML_ASSERT(0 <= sq[0] && sq[0] < n_kv); + + // avoid needing to copy the state for the first token + if (i2 == 0) { + s0 = (float *) ((char *) src0->data + ir0*(src0->nb[1]) + sq[0]*(src0->nb[2])); // {d_state, d_inner, n_kv} + } else { + // otherwise the source is the same as the destination + s0 = s; + } + + // d_inner + for (int i1 = 0; i1 < ir; ++i1) { + // ref: https://github.com/state-spaces/mamba/blob/34076d664838588a3c97727b263478ab9f621a07/mamba_ssm/ops/triton/selective_state_update.py#L78 + float dt_soft_plus = dt[i1] <= 20.0f ? log1pf(expf(dt[i1])) : dt[i1]; + float x_dt = x[i1] * dt_soft_plus; + float sumf = 0.0f; + // d_state + for (int i0 = 0; i0 < nc; ++i0) { + int i = i0 + i1*nc; + // state = prev_state * dA + dB * x + float state = (s0[i] * expf(dt_soft_plus * A[i])) + (B[i0] * x_dt); + // y = rowwise_dotprod(state, C) + sumf += state * C[i0]; + s[i] = state; + } + y[i1] = sumf; + } + + // handle copies when there are multiple output states + for (int i3 = 1; i3 < n_kv; ++i3) { + int32_t seq = sq[i3]; + if (0 <= seq && seq < n_kv) { + float * s1 = s + (seq - sq[0])*nc*nr; + memcpy(s1, s, nc*ir*sizeof(float)); + } else { + // stop at negative or too big seq_ids + break; + } + } + } +} + +static void ggml_compute_forward_ssm_scan( + const struct ggml_compute_params * params, + struct ggml_tensor * dst) { + switch (dst->src[0]->type) { + case GGML_TYPE_F32: + { + ggml_compute_forward_ssm_scan_f32(params, dst); + } break; + default: + { + GGML_ASSERT(false); + } break; + } +} + // ggml_compute_forward_win_part static void ggml_compute_forward_win_part_f32( @@ -15830,6 +16187,14 @@ static void ggml_compute_forward(struct ggml_compute_params * params, struct ggm bool masked = t != 0; ggml_compute_forward_flash_attn_back(params, masked, tensor); } break; + case GGML_OP_SSM_CONV: + { + ggml_compute_forward_ssm_conv(params, tensor); + } break; + case GGML_OP_SSM_SCAN: + { + ggml_compute_forward_ssm_scan(params, tensor); + } break; case GGML_OP_WIN_PART: { ggml_compute_forward_win_part(params, tensor); @@ -16884,6 +17249,11 @@ static void ggml_compute_backward(struct ggml_context * ctx, struct ggml_tensor { GGML_ASSERT(false); // not supported } break; + case GGML_OP_SSM_CONV: + case GGML_OP_SSM_SCAN: + { + GGML_ASSERT(false); // TODO: not implemented + } break; case GGML_OP_WIN_PART: case GGML_OP_WIN_UNPART: case GGML_OP_UNARY: @@ -17590,6 +17960,11 @@ static int ggml_get_n_tasks(struct ggml_tensor * node, int n_threads) { { n_tasks = n_threads; } break; + case GGML_OP_SSM_CONV: + case GGML_OP_SSM_SCAN: + { + n_tasks = n_threads; + } break; case GGML_OP_WIN_PART: case GGML_OP_WIN_UNPART: case GGML_OP_GET_REL_POS: diff --git a/ggml.h b/ggml.h index 0ea4f8847..a13b0cec4 100644 --- a/ggml.h +++ b/ggml.h @@ -472,6 +472,8 @@ extern "C" { GGML_OP_FLASH_ATTN, GGML_OP_FLASH_FF, GGML_OP_FLASH_ATTN_BACK, + GGML_OP_SSM_CONV, + GGML_OP_SSM_SCAN, GGML_OP_WIN_PART, GGML_OP_WIN_UNPART, GGML_OP_GET_REL_POS, @@ -1728,6 +1730,23 @@ extern "C" { struct ggml_tensor * c0, struct ggml_tensor * c1); + GGML_API struct ggml_tensor * ggml_ssm_conv( + struct ggml_context * ctx, + struct ggml_tensor * s, + struct ggml_tensor * x, + struct ggml_tensor * c, + struct ggml_tensor * sq); + + GGML_API struct ggml_tensor * ggml_ssm_scan( + struct ggml_context * ctx, + struct ggml_tensor * s, + struct ggml_tensor * x, + struct ggml_tensor * dt, + struct ggml_tensor * A, + struct ggml_tensor * B, + struct ggml_tensor * C, + struct ggml_tensor * sq); + // partition into non-overlapping windows with padding if needed // example: // a: 768 64 64 1 diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py index a62139811..b23badb10 100644 --- a/gguf-py/gguf/constants.py +++ b/gguf-py/gguf/constants.py @@ -61,6 +61,12 @@ class Keys: SCALING_ORIG_CTX_LEN = "{arch}.rope.scaling.original_context_length" SCALING_FINETUNED = "{arch}.rope.scaling.finetuned" + class SSM: + CONV_KERNEL = "{arch}.ssm.conv_kernel" + INNER_SIZE = "{arch}.ssm.inner_size" + STATE_SIZE = "{arch}.ssm.state_size" + TIME_STEP_RANK = "{arch}.ssm.time_step_rank" + class Tokenizer: MODEL = "tokenizer.ggml.model" LIST = "tokenizer.ggml.tokens" @@ -113,6 +119,7 @@ class MODEL_ARCH(IntEnum): MINICPM = auto() GEMMA = auto() STARCODER2 = auto() + MAMBA = auto() class MODEL_TENSOR(IntEnum): @@ -144,6 +151,13 @@ class MODEL_TENSOR(IntEnum): ATTN_Q_NORM = auto() ATTN_K_NORM = auto() LAYER_OUT_NORM = auto() + SSM_IN = auto() + SSM_CONV1D = auto() + SSM_X = auto() + SSM_DT = auto() + SSM_A = auto() + SSM_D = auto() + SSM_OUT = auto() MODEL_ARCH_NAMES: dict[MODEL_ARCH, str] = { @@ -171,6 +185,7 @@ MODEL_ARCH_NAMES: dict[MODEL_ARCH, str] = { MODEL_ARCH.MINICPM: "minicpm", MODEL_ARCH.GEMMA: "gemma", MODEL_ARCH.STARCODER2: "starcoder2", + MODEL_ARCH.MAMBA: "mamba", } TENSOR_NAMES: dict[MODEL_TENSOR, str] = { @@ -202,6 +217,13 @@ TENSOR_NAMES: dict[MODEL_TENSOR, str] = { MODEL_TENSOR.FFN_DOWN_EXP: "blk.{bid}.ffn_down.{xid}", MODEL_TENSOR.FFN_UP_EXP: "blk.{bid}.ffn_up.{xid}", MODEL_TENSOR.LAYER_OUT_NORM: "blk.{bid}.layer_output_norm", + MODEL_TENSOR.SSM_IN: "blk.{bid}.ssm_in", + MODEL_TENSOR.SSM_CONV1D: "blk.{bid}.ssm_conv1d", + MODEL_TENSOR.SSM_X: "blk.{bid}.ssm_x", + MODEL_TENSOR.SSM_DT: "blk.{bid}.ssm_dt", + MODEL_TENSOR.SSM_A: "blk.{bid}.ssm_a", + MODEL_TENSOR.SSM_D: "blk.{bid}.ssm_d", + MODEL_TENSOR.SSM_OUT: "blk.{bid}.ssm_out", } MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = { @@ -543,6 +565,19 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = { MODEL_TENSOR.FFN_DOWN, MODEL_TENSOR.FFN_UP, ], + MODEL_ARCH.MAMBA: [ + MODEL_TENSOR.TOKEN_EMBD, + MODEL_TENSOR.OUTPUT_NORM, + MODEL_TENSOR.OUTPUT, + MODEL_TENSOR.ATTN_NORM, + MODEL_TENSOR.SSM_IN, + MODEL_TENSOR.SSM_CONV1D, + MODEL_TENSOR.SSM_X, + MODEL_TENSOR.SSM_DT, + MODEL_TENSOR.SSM_A, + MODEL_TENSOR.SSM_D, + MODEL_TENSOR.SSM_OUT, + ], # TODO } @@ -734,6 +769,12 @@ KEY_ROPE_SCALING_FACTOR = Keys.Rope.SCALING_FACTOR KEY_ROPE_SCALING_ORIG_CTX_LEN = Keys.Rope.SCALING_ORIG_CTX_LEN KEY_ROPE_SCALING_FINETUNED = Keys.Rope.SCALING_FINETUNED +# SSM +KEY_SSM_CONV_KERNEL = Keys.SSM.CONV_KERNEL +KEY_SSM_INNER_SIZE = Keys.SSM.INNER_SIZE +KEY_SSM_STATE_SIZE = Keys.SSM.STATE_SIZE +KEY_SSM_TIME_STEP_RANK = Keys.SSM.TIME_STEP_RANK + # tokenization KEY_TOKENIZER_MODEL = Keys.Tokenizer.MODEL KEY_TOKENIZER_LIST = Keys.Tokenizer.LIST diff --git a/gguf-py/gguf/gguf_writer.py b/gguf-py/gguf/gguf_writer.py index 801160832..e49c5db68 100644 --- a/gguf-py/gguf/gguf_writer.py +++ b/gguf-py/gguf/gguf_writer.py @@ -382,6 +382,18 @@ class GGUFWriter: def add_rope_scaling_finetuned(self, value: bool) -> None: self.add_bool(Keys.Rope.SCALING_FINETUNED.format(arch=self.arch), value) + def add_ssm_conv_kernel(self, value: int) -> None: + self.add_uint32(Keys.SSM.CONV_KERNEL.format(arch=self.arch), value) + + def add_ssm_inner_size(self, value: int) -> None: + self.add_uint32(Keys.SSM.INNER_SIZE.format(arch=self.arch), value) + + def add_ssm_state_size(self, value: int) -> None: + self.add_uint32(Keys.SSM.STATE_SIZE.format(arch=self.arch), value) + + def add_ssm_time_step_rank(self, value: int) -> None: + self.add_uint32(Keys.SSM.TIME_STEP_RANK.format(arch=self.arch), value) + def add_tokenizer_model(self, model: str) -> None: self.add_string(Keys.Tokenizer.MODEL, model) diff --git a/gguf-py/gguf/tensor_mapping.py b/gguf-py/gguf/tensor_mapping.py index db2ec9704..ed89955d8 100644 --- a/gguf-py/gguf/tensor_mapping.py +++ b/gguf-py/gguf/tensor_mapping.py @@ -20,6 +20,9 @@ class TensorNameMap: "wte", # gpt2 "transformer.embd.wte", # phi2 "model.tok_embeddings", # internlm2 + "model.embedding", # mamba-qbert + "backbone.embedding", # mamba + "backbone.embeddings", # mamba-hf ), # Token type embeddings @@ -44,7 +47,7 @@ class TensorNameMap: # Output MODEL_TENSOR.OUTPUT: ( "embed_out", # gptneox - "lm_head", # gpt2 mpt falcon llama-hf baichuan qwen + "lm_head", # gpt2 mpt falcon llama-hf baichuan qwen mamba "output", # llama-pth bloom internlm2 "word_embeddings_for_head", # persimmon "lm_head.linear", # phi2 @@ -61,6 +64,8 @@ class TensorNameMap: "language_model.encoder.final_layernorm", # persimmon "model.final_layernorm", # persimmon "lm_head.ln", # phi2 + "model.norm_f", # mamba-qbert + "backbone.norm_f", # mamba ), # Rope frequencies @@ -86,6 +91,8 @@ class TensorNameMap: "transformer.h.{bid}.ln", # phi2 "model.layers.layers.{bid}.norm", # plamo "model.layers.{bid}.attention_norm", # internlm2 + "model.layers.{bid}.norm", # mamba-qbert + "backbone.layers.{bid}.norm", # mamba ), # Attention norm 2 @@ -282,7 +289,42 @@ class TensorNameMap: MODEL_TENSOR.LAYER_OUT_NORM: ( "encoder.layer.{bid}.output.LayerNorm", # bert "encoder.layers.{bid}.norm2", # nomic-bert - ) + ), + + MODEL_TENSOR.SSM_IN: ( + "model.layers.{bid}.in_proj", + "backbone.layers.{bid}.mixer.in_proj", + ), + + MODEL_TENSOR.SSM_CONV1D: ( + "model.layers.{bid}.conv1d", + "backbone.layers.{bid}.mixer.conv1d", + ), + + MODEL_TENSOR.SSM_X: ( + "model.layers.{bid}.x_proj", + "backbone.layers.{bid}.mixer.x_proj", + ), + + MODEL_TENSOR.SSM_DT: ( + "model.layers.{bid}.dt_proj", + "backbone.layers.{bid}.mixer.dt_proj", + ), + + MODEL_TENSOR.SSM_A: ( + "model.layers.{bid}.A_log", + "backbone.layers.{bid}.mixer.A_log", + ), + + MODEL_TENSOR.SSM_D: ( + "model.layers.{bid}.D", + "backbone.layers.{bid}.mixer.D", + ), + + MODEL_TENSOR.SSM_OUT: ( + "model.layers.{bid}.out_proj", + "backbone.layers.{bid}.mixer.out_proj", + ), } mapping: dict[str, tuple[MODEL_TENSOR, str]] diff --git a/llama.cpp b/llama.cpp index 4a20b7928..8c147a42b 100644 --- a/llama.cpp +++ b/llama.cpp @@ -213,6 +213,7 @@ enum llm_arch { LLM_ARCH_MINICPM, LLM_ARCH_GEMMA, LLM_ARCH_STARCODER2, + LLM_ARCH_MAMBA, LLM_ARCH_UNKNOWN, }; @@ -241,6 +242,7 @@ static const std::map LLM_ARCH_NAMES = { { LLM_ARCH_MINICPM, "minicpm" }, { LLM_ARCH_GEMMA, "gemma" }, { LLM_ARCH_STARCODER2, "starcoder2" }, + { LLM_ARCH_MAMBA, "mamba" }, { LLM_ARCH_UNKNOWN, "(unknown)" }, }; @@ -284,6 +286,11 @@ enum llm_kv { LLM_KV_ROPE_SCALING_ORIG_CTX_LEN, LLM_KV_ROPE_SCALING_FINETUNED, + LLM_KV_SSM_INNER_SIZE, + LLM_KV_SSM_CONV_KERNEL, + LLM_KV_SSM_STATE_SIZE, + LLM_KV_SSM_TIME_STEP_RANK, + LLM_KV_TOKENIZER_MODEL, LLM_KV_TOKENIZER_LIST, LLM_KV_TOKENIZER_TOKEN_TYPE, @@ -342,6 +349,11 @@ static const std::map LLM_KV_NAMES = { { LLM_KV_ROPE_SCALING_ORIG_CTX_LEN, "%s.rope.scaling.original_context_length" }, { LLM_KV_ROPE_SCALING_FINETUNED, "%s.rope.scaling.finetuned" }, + { LLM_KV_SSM_CONV_KERNEL, "%s.ssm.conv_kernel" }, + { LLM_KV_SSM_INNER_SIZE, "%s.ssm.inner_size" }, + { LLM_KV_SSM_STATE_SIZE, "%s.ssm.state_size" }, + { LLM_KV_SSM_TIME_STEP_RANK, "%s.ssm.time_step_rank" }, + { LLM_KV_TOKENIZER_MODEL, "tokenizer.ggml.model" }, { LLM_KV_TOKENIZER_LIST, "tokenizer.ggml.tokens" }, { LLM_KV_TOKENIZER_TOKEN_TYPE, "tokenizer.ggml.token_type" }, @@ -399,6 +411,13 @@ enum llm_tensor { LLM_TENSOR_ATTN_Q_NORM, LLM_TENSOR_ATTN_K_NORM, LLM_TENSOR_LAYER_OUT_NORM, + LLM_TENSOR_SSM_IN, + LLM_TENSOR_SSM_CONV1D, + LLM_TENSOR_SSM_X, + LLM_TENSOR_SSM_DT, + LLM_TENSOR_SSM_A, + LLM_TENSOR_SSM_D, + LLM_TENSOR_SSM_OUT, }; static const std::map> LLM_TENSOR_NAMES = { @@ -801,6 +820,22 @@ static const std::map> LLM_TENSOR_NA { LLM_TENSOR_FFN_UP, "blk.%d.ffn_up" }, }, }, + { + LLM_ARCH_MAMBA, + { + { LLM_TENSOR_TOKEN_EMBD, "token_embd" }, + { LLM_TENSOR_OUTPUT_NORM, "output_norm" }, + { LLM_TENSOR_OUTPUT, "output" }, + { LLM_TENSOR_ATTN_NORM, "blk.%d.attn_norm" }, + { LLM_TENSOR_SSM_IN, "blk.%d.ssm_in" }, + { LLM_TENSOR_SSM_CONV1D, "blk.%d.ssm_conv1d" }, + { LLM_TENSOR_SSM_X, "blk.%d.ssm_x" }, + { LLM_TENSOR_SSM_DT, "blk.%d.ssm_dt" }, + { LLM_TENSOR_SSM_A, "blk.%d.ssm_a" }, + { LLM_TENSOR_SSM_D, "blk.%d.ssm_d" }, + { LLM_TENSOR_SSM_OUT, "blk.%d.ssm_out" }, + }, + }, { LLM_ARCH_UNKNOWN, { @@ -1613,6 +1648,12 @@ struct llama_hparams { float rope_freq_scale_train; uint32_t n_yarn_orig_ctx; + // for State Space Models + uint32_t ssm_d_conv = 0; + uint32_t ssm_d_inner = 0; + uint32_t ssm_d_state = 0; + uint32_t ssm_dt_rank = 0; + float f_clamp_kqv = 0.0f; float f_max_alibi_bias = 0.0f; @@ -1641,6 +1682,11 @@ struct llama_hparams { if (this->rope_finetuned != other.rope_finetuned) return true; if (this->n_yarn_orig_ctx != other.n_yarn_orig_ctx) return true; + if (this->ssm_d_conv != other.ssm_d_conv) return true; + if (this->ssm_d_inner != other.ssm_d_inner) return true; + if (this->ssm_d_state != other.ssm_d_state) return true; + if (this->ssm_dt_rank != other.ssm_dt_rank) return true; + const float EPSILON = 1e-9f; if (!is_float_close(this->f_norm_eps, other.f_norm_eps, EPSILON)) return true; @@ -1652,6 +1698,9 @@ struct llama_hparams { } uint32_t n_gqa() const { + if (n_head_kv == 0) { + return 0; + } return n_head/n_head_kv; } @@ -1662,6 +1711,18 @@ struct llama_hparams { uint32_t n_embd_v_gqa() const { // dimension of value embeddings across all k-v heads return n_embd_head_v * n_head_kv; } + + uint32_t n_embd_k_s() const { // dimension of the rolling state embeddings + // corresponds to Mamba's conv_states size + // TODO: maybe support other convolution strides than 1 + // NOTE: since the first column of the conv_state is shifted out each time, it's not actually needed + return (ssm_d_conv > 0 ? ssm_d_conv - 1 : 0) * ssm_d_inner; + } + + uint32_t n_embd_v_s() const { // dimension of the recurrent state embeddings + // corresponds to Mamba's ssm_states size + return ssm_d_state * ssm_d_inner; + } }; struct llama_cparams { @@ -1739,11 +1800,27 @@ struct llama_layer { struct ggml_tensor * ffn_down_b; // b2 struct ggml_tensor * ffn_up_b; // b3 struct ggml_tensor * ffn_act; + + // mamba proj + struct ggml_tensor * ssm_in; + struct ggml_tensor * ssm_x; + struct ggml_tensor * ssm_dt; + struct ggml_tensor * ssm_out; + + // mamba + struct ggml_tensor * ssm_conv1d; + struct ggml_tensor * ssm_a; + struct ggml_tensor * ssm_d; + + // mamba bias + struct ggml_tensor * ssm_conv1d_b; + struct ggml_tensor * ssm_dt_b; }; struct llama_kv_cell { llama_pos pos = -1; llama_pos delta = 0; + int32_t src = 0; // used by recurrent state models to copy states std::set seq_id; @@ -1764,6 +1841,9 @@ struct llama_kv_cell { struct llama_kv_cache { bool has_shift = false; bool do_defrag = false; + bool do_copy = false; + // with recurrent state models, a cell can hold the state for more than one past token + bool recurrent = false; // Note: The value of head isn't only used to optimize searching // for a free KV slot. llama_decode_internal also uses it, so it @@ -2003,11 +2083,14 @@ struct llama_context { struct ggml_tensor * inp_tokens; // I32 [n_batch] struct ggml_tensor * inp_embd; // F32 [n_embd, n_batch] struct ggml_tensor * inp_pos; // I32 [n_batch] - struct ggml_tensor * inp_KQ_mask; // F32 [n_ctx, n_batch] - struct ggml_tensor * inp_KQ_pos; // F32 [n_ctx] - struct ggml_tensor * inp_K_shift; // I32 [n_ctx] + struct ggml_tensor * inp_KQ_mask; // F32 [kv_size, n_batch] + struct ggml_tensor * inp_KQ_pos; // F32 [kv_size] + struct ggml_tensor * inp_K_shift; // I32 [kv_size] struct ggml_tensor * inp_mean; // F32 [n_batch, n_batch] struct ggml_tensor * inp_cls; // I32 [n_batch] + struct ggml_tensor * inp_s_copy; // I32 [kv_size] + struct ggml_tensor * inp_s_mask; // F32 [kv_size] + struct ggml_tensor * inp_s_seq; // I32 [kv_size, n_batch] #ifdef GGML_USE_MPI ggml_mpi_context * ctx_mpi = NULL; @@ -2023,25 +2106,42 @@ static bool llama_kv_cache_init( const llama_model & model, ggml_type type_k, ggml_type type_v, - uint32_t n_ctx, + uint32_t kv_size, bool offload) { const struct llama_hparams & hparams = model.hparams; - const uint32_t n_embd_k_gqa = hparams.n_embd_k_gqa(); - const uint32_t n_embd_v_gqa = hparams.n_embd_v_gqa(); + const uint32_t n_embd_k_gqa = hparams.n_embd_k_gqa() + hparams.n_embd_k_s(); + const uint32_t n_embd_v_gqa = hparams.n_embd_v_gqa() + hparams.n_embd_v_s(); const int64_t n_layer = hparams.n_layer; cache.has_shift = false; + // TODO: find a nicer way to add other recurrent model architectures + cache.recurrent = model.arch == LLM_ARCH_MAMBA; + + // TODO: support mixed reccurent Transformer architectues + // NOTE: (!a || b) is a logical implication (a -> b) + GGML_ASSERT(!cache.recurrent || n_embd_k_gqa == hparams.n_embd_k_s()); + GGML_ASSERT(!cache.recurrent || n_embd_v_gqa == hparams.n_embd_v_s()); + GGML_ASSERT( cache.recurrent || n_embd_k_gqa == hparams.n_embd_k_gqa()); + GGML_ASSERT( cache.recurrent || n_embd_v_gqa == hparams.n_embd_v_gqa()); + cache.head = 0; - cache.size = n_ctx; + cache.size = kv_size; cache.used = 0; cache.type_k = type_k; cache.type_v = type_v; cache.cells.clear(); - cache.cells.resize(n_ctx); + cache.cells.resize(kv_size); + + if (cache.recurrent) { + // init state copy sources + for (uint32_t i = 0; i < cache.size; ++i) { + cache.cells[i].src = i; + } + } #ifdef GGML_USE_CLBLAST offload = false; @@ -2080,8 +2180,8 @@ static bool llama_kv_cache_init( for (int i = 0; i < (int) n_layer; i++) { struct ggml_context * ctx = offload ? ctx_map.at(model.buft_layer[i].buft) : cache.ctxs.front(); - ggml_tensor * k = ggml_new_tensor_1d(ctx, type_k, n_embd_k_gqa*n_ctx); - ggml_tensor * v = ggml_new_tensor_1d(ctx, type_v, n_embd_v_gqa*n_ctx); + ggml_tensor * k = ggml_new_tensor_1d(ctx, type_k, n_embd_k_gqa*kv_size); + ggml_tensor * v = ggml_new_tensor_1d(ctx, type_v, n_embd_v_gqa*kv_size); ggml_format_name(k, "cache_k_l%d", i); ggml_format_name(v, "cache_v_l%d", i); cache.k_l.push_back(k); @@ -2115,6 +2215,54 @@ static bool llama_kv_cache_find_slot( const uint32_t n_ctx = cache.size; const uint32_t n_tokens = batch.n_tokens; + if (cache.recurrent) { + // For recurrent state architectures (like Mamba), + // each KV cache cell can store the state for a whole sequence. + + llama_seq_id min = cache.size - 1; + llama_seq_id max = 0; + + for (uint32_t i = 0; i < n_tokens; ++i) { + for (int32_t j = 0; j < batch.n_seq_id[i]; ++j) { + llama_seq_id seq_id = batch.seq_id[i][j]; + // make sure it's a valid seq_id + if ((uint32_t) seq_id < cache.size) { + if (seq_id > max) { + max = seq_id; + } + if (seq_id < min) { + min = seq_id; + } + // Assuming the tokens are in-order + if (batch.pos[i] != cache.cells[seq_id].pos + 1) { + // What should happen when the pos backtracks or skips a value? + // Clearing the state mid-batch would require special-casing which isn't done. + LLAMA_LOG_WARN("%s: non-consecutive token position %d after %d for sequence %d\n", + __func__, batch.pos[i], cache.cells[seq_id].pos, seq_id); + } + if (cache.cells[seq_id].pos < 0 && 0 <= batch.pos[i]) { + cache.used += 1; + } + cache.cells[seq_id].pos = batch.pos[i]; + // NOTE: seq_ids are not inserted here; they are handled when the input tensors are set + } else { + // too big seq_id + // TODO: would it be possible to resize the KV cache size instead? + LLAMA_LOG_ERROR("%s: seq_id=%d >= kv_size=%d Try using a bigger --parallel value\n", __func__, seq_id, cache.size); + return false; + } + } + } + + // allow getting the range of used cells, from head to head + n + cache.head = min; + cache.n = max - min + 1; + + // sanity check + return max >= min; + } + // otherwise, one cell per token. + if (n_tokens > n_ctx) { LLAMA_LOG_ERROR("%s: n_tokens=%d > n_ctx=%d\n", __func__, n_tokens, n_ctx); return false; @@ -2184,7 +2332,7 @@ static void llama_kv_cache_clear(struct llama_kv_cache & cache) { cache.used = 0; } -static void llama_kv_cache_seq_rm( +static bool llama_kv_cache_seq_rm( struct llama_kv_cache & cache, llama_seq_id seq_id, llama_pos p0, @@ -2194,6 +2342,25 @@ static void llama_kv_cache_seq_rm( if (p0 < 0) p0 = 0; if (p1 < 0) p1 = std::numeric_limits::max(); + // models like Mamba can't have a state partially erased + if (cache.recurrent) { + if (seq_id >= (int64_t) cache.size) { + // could be fatal + return false; + } + if (0 <= seq_id) { + // partial intersection is invalid + if ((0 < p0 && p0 <= cache.cells[seq_id].pos) || (0 < p1 && p1 <= cache.cells[seq_id].pos)) { + return false; + } + } else { + // seq_id is negative, then the range should include everything or nothing + if (p0 != p1 && (p0 != 0 || p1 != std::numeric_limits::max())) { + return false; + } + } + } + for (uint32_t i = 0; i < cache.size; ++i) { if (cache.cells[i].pos >= p0 && cache.cells[i].pos < p1) { if (seq_id < 0) { @@ -2215,6 +2382,8 @@ static void llama_kv_cache_seq_rm( // If we freed up a slot, set head to it so searching can start there. if (new_head != cache.size && new_head < cache.head) cache.head = new_head; + + return true; } static void llama_kv_cache_seq_cp( @@ -2226,6 +2395,29 @@ static void llama_kv_cache_seq_cp( if (p0 < 0) p0 = 0; if (p1 < 0) p1 = std::numeric_limits::max(); + if (cache.recurrent) { + if ((uint32_t) seq_id_dst < cache.size && (uint32_t) seq_id_src < cache.size) { + seq_id_src = cache.cells[seq_id_src].src; + GGML_ASSERT((uint32_t) seq_id_src < cache.size); + // intent to "copy from" + // supports copy chains thanks to taking the source of the source + cache.cells[seq_id_dst].src = seq_id_src; + + // preserve the "keep or clear" status of the copied sequence + if (cache.cells[seq_id_src].has_seq_id(seq_id_src)) { + cache.cells[seq_id_dst].seq_id.insert(seq_id_dst); + } else { + cache.cells[seq_id_dst].seq_id.erase(seq_id_dst); + } + + cache.do_copy = true; + + cache.cells[seq_id_dst].pos = cache.cells[seq_id_src].pos; + } + return; + } + // otherwise, this is the KV cache of a Transformer-like model + cache.head = 0; for (uint32_t i = 0; i < cache.size; ++i) { @@ -2265,6 +2457,17 @@ static void llama_kv_cache_seq_add( if (p0 < 0) p0 = 0; if (p1 < 0) p1 = std::numeric_limits::max(); + if (cache.recurrent) { + // for Mamba-like models, only the pos needs to be shifted + if (0 <= seq_id && seq_id < (int64_t) cache.size) { + llama_kv_cell & cell = cache.cells[seq_id]; + if (cell.has_seq_id(seq_id) && p0 <= cell.pos && cell.pos < p1) { + cell.pos += delta; + } + } + return; + } + for (uint32_t i = 0; i < cache.size; ++i) { if (cache.cells[i].has_seq_id(seq_id) && cache.cells[i].pos >= p0 && cache.cells[i].pos < p1) { cache.has_shift = true; @@ -2298,6 +2501,17 @@ static void llama_kv_cache_seq_div( if (p0 < 0) p0 = 0; if (p1 < 0) p1 = std::numeric_limits::max(); + if (cache.recurrent) { + // for Mamba-like models, only the pos needs to be changed + if (0 <= seq_id && seq_id < (int64_t) cache.size) { + llama_kv_cell & cell = cache.cells[seq_id]; + if (cell.has_seq_id(seq_id) && p0 <= cell.pos && cell.pos < p1) { + cell.pos /= d; + } + } + return; + } + for (uint32_t i = 0; i < cache.size; ++i) { if (cache.cells[i].has_seq_id(seq_id) && cache.cells[i].pos >= p0 && cache.cells[i].pos < p1) { cache.has_shift = true; @@ -3117,7 +3331,7 @@ static void llm_load_hparams( // sanity check for n_rot (optional) { - hparams.n_rot = hparams.n_embd / hparams.n_head; + hparams.n_rot = (hparams.n_head == 0) ? 0 : hparams.n_embd / hparams.n_head; ml.get_key(LLM_KV_ROPE_DIMENSION_COUNT, hparams.n_rot, false); @@ -3130,10 +3344,10 @@ static void llm_load_hparams( // gpt-j n_rot = rotary_dim } - hparams.n_embd_head_k = hparams.n_embd / hparams.n_head; + hparams.n_embd_head_k = (hparams.n_head == 0) ? 0 : hparams.n_embd / hparams.n_head; ml.get_key(LLM_KV_ATTENTION_KEY_LENGTH, hparams.n_embd_head_k, false); - hparams.n_embd_head_v = hparams.n_embd / hparams.n_head; + hparams.n_embd_head_v = (hparams.n_head == 0) ? 0 : hparams.n_embd / hparams.n_head; ml.get_key(LLM_KV_ATTENTION_VALUE_LENGTH, hparams.n_embd_head_v, false); // arch-specific KVs @@ -3383,6 +3597,36 @@ static void llm_load_hparams( default: model.type = e_model::MODEL_UNKNOWN; } } break; + case LLM_ARCH_MAMBA: + { + ml.get_key(LLM_KV_SSM_CONV_KERNEL, hparams.ssm_d_conv); + ml.get_key(LLM_KV_SSM_INNER_SIZE, hparams.ssm_d_inner); + ml.get_key(LLM_KV_SSM_STATE_SIZE, hparams.ssm_d_state); + ml.get_key(LLM_KV_SSM_TIME_STEP_RANK, hparams.ssm_dt_rank); + + ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps); + + switch (hparams.n_layer) { + case 24: + switch (hparams.n_embd) { + case 768: model.type = e_model::MODEL_SMALL; break; + default: model.type = e_model::MODEL_UNKNOWN; + } break; + case 48: + switch (hparams.n_embd) { + case 1024: model.type = e_model::MODEL_MEDIUM; break; + case 1536: model.type = e_model::MODEL_LARGE; break; + case 2048: model.type = e_model::MODEL_XL; break; + default: model.type = e_model::MODEL_UNKNOWN; + } break; + case 64: + switch (hparams.n_embd) { + case 2560: model.type = e_model::MODEL_3B; break; + default: model.type = e_model::MODEL_UNKNOWN; + } break; + default: model.type = e_model::MODEL_UNKNOWN; + } + } break; default: (void)0; } @@ -3702,6 +3946,10 @@ static void llm_load_print_meta(llama_model_loader & ml, llama_model & model) { LLAMA_LOG_INFO("%s: freq_scale_train = %g\n", __func__, hparams.rope_freq_scale_train); LLAMA_LOG_INFO("%s: n_yarn_orig_ctx = %u\n", __func__, hparams.n_yarn_orig_ctx); LLAMA_LOG_INFO("%s: rope_finetuned = %s\n", __func__, hparams.rope_finetuned ? "yes" : "unknown"); + LLAMA_LOG_INFO("%s: ssm_d_conv = %u\n", __func__, hparams.ssm_d_conv); + LLAMA_LOG_INFO("%s: ssm_d_inner = %u\n", __func__, hparams.ssm_d_inner); + LLAMA_LOG_INFO("%s: ssm_d_state = %u\n", __func__, hparams.ssm_d_state); + LLAMA_LOG_INFO("%s: ssm_dt_rank = %u\n", __func__, hparams.ssm_dt_rank); LLAMA_LOG_INFO("%s: model type = %s\n", __func__, llama_model_type_name(model.type)); LLAMA_LOG_INFO("%s: model ftype = %s\n", __func__, llama_model_ftype_name(model.ftype).c_str()); if (ml.n_elements >= 1e12) { @@ -4609,6 +4857,57 @@ static bool llm_load_tensors( layer.ffn_up_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_UP , "bias", i), { n_ff}); } } break; + case LLM_ARCH_MAMBA: + { + const int64_t d_conv = hparams.ssm_d_conv; + const int64_t d_inner = hparams.ssm_d_inner; + const int64_t d_state = hparams.ssm_d_state; + const int64_t dt_rank = hparams.ssm_dt_rank; + // only an expansion factor of 2 is supported for now + GGML_ASSERT(2 * n_embd == d_inner); + + model.tok_embd = ml.create_tensor(ctx_input, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}); + + // output + { + model.output_norm = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}); + + model.output = ml.create_tensor(ctx_output_split, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, false); + // if output is NULL, init from the input tok embed, duplicated to allow offloading + if (model.output == NULL) { + model.output = ml.create_tensor(ctx_output_split, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}); + ml.n_created--; // artificial tensor + ml.size_data += ggml_nbytes(model.output); + } + } + + for (int i = 0; i < n_layer; ++i) { + ggml_context * ctx_layer = ctx_for_layer(i); + ggml_context * ctx_split = ctx_for_layer_split(i); + + auto & layer = model.layers[i]; + + // norm + layer.attn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}); + + layer.ssm_in = ml.create_tensor(ctx_split, tn(LLM_TENSOR_SSM_IN, "weight", i), {n_embd, 2*d_inner}); + + layer.ssm_conv1d = ml.create_tensor(ctx_split, tn(LLM_TENSOR_SSM_CONV1D, "weight", i), {d_conv, d_inner}); + layer.ssm_conv1d_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_SSM_CONV1D, "bias", i), {d_inner}); + + layer.ssm_x = ml.create_tensor(ctx_split, tn(LLM_TENSOR_SSM_X, "weight", i), {d_inner, dt_rank + 2*d_state}); + + layer.ssm_dt = ml.create_tensor(ctx_split, tn(LLM_TENSOR_SSM_DT, "weight", i), {dt_rank, d_inner}); + layer.ssm_dt_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_SSM_DT, "bias", i), {d_inner}); + + // no "weight" suffix for these + layer.ssm_a = ml.create_tensor(ctx_split, tn(LLM_TENSOR_SSM_A, i), {d_state, d_inner}); + layer.ssm_d = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_SSM_D, i), {d_inner}); + + // out_proj + layer.ssm_out = ml.create_tensor(ctx_split, tn(LLM_TENSOR_SSM_OUT, "weight", i), {d_inner, n_embd}); + } + } break; default: throw std::runtime_error("unknown architecture"); } @@ -4834,6 +5133,8 @@ static void llm_build_kv_store( const int64_t n_embd_k_gqa = hparams.n_embd_k_gqa(); const int64_t n_embd_v_gqa = hparams.n_embd_v_gqa(); + GGML_ASSERT(kv.size == n_ctx); + // compute the transposed [n_tokens, n_embd] V matrix struct ggml_tensor * v_cur_t = ggml_transpose(ctx, ggml_reshape_2d(ctx, v_cur, n_embd_v_gqa, n_tokens)); //struct ggml_tensor * v_cur_t = ggml_transpose(ctx, v_cur); // TODO: reshape above is likely not needed @@ -5043,6 +5344,8 @@ static struct ggml_tensor * llm_build_kqv( cb(kq, "kq_soft_max_ext", il); } + GGML_ASSERT(kv.size == n_ctx); + // split cached v into n_head heads struct ggml_tensor * v = ggml_view_3d(ctx, kv.v_l[il], @@ -5190,8 +5493,8 @@ struct llm_build_context { norm_eps (hparams.f_norm_eps), norm_rms_eps (hparams.f_norm_rms_eps), n_tokens (batch.n_tokens), - n_kv (worst_case ? n_ctx : kv_self.n), - kv_head (worst_case ? n_ctx - n_tokens : kv_self.head), + n_kv (worst_case ? kv_self.size : kv_self.n), + kv_head (worst_case ? (kv_self.recurrent ? 0 : kv_self.size - n_tokens) : kv_self.head), n_orig_ctx (cparams.n_yarn_orig_ctx), pooling_type (cparams.pooling_type), rope_type (hparams.rope_type), @@ -5220,6 +5523,8 @@ struct llm_build_context { struct ggml_cgraph * build_k_shift() { struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, LLAMA_MAX_NODES, false); + GGML_ASSERT(kv_self.size == n_ctx); + for (int il = 0; il < n_layer; ++il) { struct ggml_tensor * tmp = // we rotate only the first n_rot dimensions @@ -5238,6 +5543,27 @@ struct llm_build_context { return gf; } + struct ggml_cgraph * build_s_copy() { + struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, LLAMA_MAX_NODES, false); + + GGML_ASSERT(kv_self.recurrent); + + for (int il = 0; il < n_layer; ++il) { + struct ggml_tensor * conv_states = ggml_reshape_2d(ctx0, kv_self.k_l[il], hparams.n_embd_k_s(), kv_self.size); + struct ggml_tensor * ssm_states = ggml_reshape_2d(ctx0, kv_self.v_l[il], hparams.n_embd_v_s(), kv_self.size); + + conv_states = ggml_get_rows(ctx0, conv_states, lctx.inp_s_copy); + ssm_states = ggml_get_rows(ctx0, ssm_states, lctx.inp_s_copy); + + // TODO: name the intermediate tensors with cb() + + ggml_build_forward_expand(gf, ggml_cpy(ctx0, conv_states, kv_self.k_l[il])); + ggml_build_forward_expand(gf, ggml_cpy(ctx0, ssm_states, kv_self.v_l[il])); + } + + return gf; + } + struct ggml_cgraph * build_defrag(const std::vector & ids) { struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, LLAMA_MAX_NODES, false); @@ -7835,6 +8161,145 @@ struct llm_build_context { return gf; } + + struct ggml_cgraph * build_mamba() { + struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, LLAMA_MAX_NODES, false); + + const int64_t d_model = n_embd; + const int64_t d_conv = hparams.ssm_d_conv; + const int64_t d_inner = hparams.ssm_d_inner; + GGML_ASSERT(2 * d_model == d_inner); + const int64_t d_state = hparams.ssm_d_state; + const int64_t dt_rank = hparams.ssm_dt_rank; + + struct ggml_tensor * cur; + struct ggml_tensor * inpL; + + // {n_embd, n_tokens} + inpL = llm_build_inp_embd(ctx0, hparams, batch, model.tok_embd, lctx.inp_tokens, lctx.inp_embd, cb); + cb(inpL, "inp_embd", -1); + + struct ggml_tensor * state_mask = ggml_view_2d(ctx0, lctx.inp_s_mask, 1, n_kv, lctx.inp_s_mask->nb[0], 0); + struct ggml_tensor * state_seq = ggml_view_2d(ctx0, lctx.inp_s_seq, n_kv, n_tokens, n_kv*ggml_element_size(lctx.inp_s_seq), 0); + + for (int il = 0; il < n_layer; ++il) { + // (ab)using the KV cache to store the states + struct ggml_tensor * conv_states = ggml_reshape_2d(ctx0, kv_self.k_l[il], hparams.n_embd_k_s(), kv_self.size); + struct ggml_tensor * ssm_states = ggml_reshape_2d(ctx0, kv_self.v_l[il], hparams.n_embd_v_s(), kv_self.size); + + // clear states of sequences which are starting at the beginning of this batch + { + conv_states = ggml_mul(ctx0, + ggml_view_2d(ctx0, conv_states, conv_states->ne[0], n_kv, conv_states->nb[1], kv_head*conv_states->nb[1]), + state_mask); + ssm_states = ggml_mul(ctx0, + ggml_view_2d(ctx0, ssm_states, ssm_states->ne[0], n_kv, ssm_states->nb[1], kv_head*ssm_states->nb[1]), + state_mask); + } + + conv_states = ggml_reshape_3d(ctx0, conv_states, d_conv - 1, d_inner, n_kv); + ssm_states = ggml_reshape_3d(ctx0, ssm_states, d_state, d_inner, n_kv); + + // norm + cur = llm_build_norm(ctx0, inpL, hparams, + model.layers[il].attn_norm, NULL, + LLM_NORM_RMS, cb, il); + cb(cur, "attn_norm", il); + + // {n_embd, 2*d_inner} * {n_embd, n_tokens} => {2*d_inner, n_tokens} + struct ggml_tensor * xz = ggml_mul_mat(ctx0, model.layers[il].ssm_in, cur); + // split the above in two + // => {d_inner, n_tokens} + struct ggml_tensor * x = ggml_view_2d(ctx0, xz, d_inner, xz->ne[1], xz->nb[1], 0); + struct ggml_tensor * z = ggml_view_2d(ctx0, xz, d_inner, xz->ne[1], xz->nb[1], ggml_element_size(xz)*d_inner); + + // conv + { + // Custom operator which is needed only to ease simultaneous sequence processing. + // For a single sequence, the equivalent is to concatenate the columns of conv_states and x, + // then make a self-overlapping view of that over d_conv columns at each stride in the 3rd dimension, + // then element-wise multiply that with the conv1d weigth, + // then sum the elements of each row, + // (the last two steps are a dot product over rows (also doable with mul_mat)) + // then permute away the ne[0] dimension, + // and then you're left with the resulting x tensor. + // The new conv_states is the last (d_conv - 1) columns + // of the last 3rd dimensional "layer" of the self-overlapping view. + // For simultaneous sequences, it's more complicated. + struct ggml_tensor * x_conv = ggml_ssm_conv(ctx0, conv_states, x, model.layers[il].ssm_conv1d, state_seq); + + // store last (d_conv - 1) columns of the conv_state part of x_conv back into the KV cache + ggml_build_forward_expand(gf, + ggml_cpy(ctx0, + ggml_view_2d(ctx0, x_conv, d_conv - 1, d_inner*n_kv, d_conv*ggml_element_size(x_conv), (1+d_inner*n_tokens)*ggml_element_size(x_conv)), + ggml_view_1d(ctx0, kv_self.k_l[il], (d_conv - 1)*(d_inner)*(n_kv), kv_self.head*(d_conv - 1)*(d_inner)*ggml_element_size(x_conv)))); + + // extract x from x_conv + x = ggml_view_2d(ctx0, x_conv, d_inner, n_tokens, d_inner*ggml_element_size(x_conv), 0); + + // bias + x = ggml_add(ctx0, x, model.layers[il].ssm_conv1d_b); + + x = ggml_silu(ctx0, x); + } + + // ssm + { + // {d_inner, dt_rank + 2*d_state} * {d_inner, n_tokens} => {dt_rank + 2*d_state, n_tokens} + struct ggml_tensor * x_db = ggml_mul_mat(ctx0, model.layers[il].ssm_x, x); + // split + struct ggml_tensor * dt = ggml_view_2d(ctx0, x_db, dt_rank, n_tokens, x_db->nb[1], 0); + struct ggml_tensor * B = ggml_view_2d(ctx0, x_db, d_state, n_tokens, x_db->nb[1], ggml_element_size(x_db)*dt_rank); + struct ggml_tensor * C = ggml_view_2d(ctx0, x_db, d_state, n_tokens, x_db->nb[1], ggml_element_size(x_db)*(dt_rank+d_state)); + + // {dt_rank, d_inner} * {dt_rank, n_tokens} => {d_inner, n_tokens} + dt = ggml_mul_mat(ctx0, model.layers[il].ssm_dt, dt); + dt = ggml_add(ctx0, dt, model.layers[il].ssm_dt_b); + + // Custom operator to optimize the parallel associative scan + // as described in the Annex D of the Mamba paper. + // => {d_inner, n_tokens} and {d_state, d_inner, n_kv} combined, + // because only a single tensor can be returned. + struct ggml_tensor * y_ssm_states = ggml_ssm_scan(ctx0, ssm_states, x, dt, model.layers[il].ssm_a, B, C, state_seq); + + // store last states (the second part of y_ssm_states) + ggml_build_forward_expand(gf, + ggml_cpy(ctx0, + ggml_view_1d(ctx0, y_ssm_states, d_state*d_inner*n_kv, d_inner*n_tokens*ggml_element_size(y_ssm_states)), + ggml_view_1d(ctx0, kv_self.v_l[il], d_state*d_inner*n_kv, kv_self.head*d_state*d_inner*ggml_element_size(ssm_states)))); + + struct ggml_tensor * y = ggml_view_2d(ctx0, y_ssm_states, d_inner, n_tokens, d_inner*ggml_element_size(y_ssm_states), 0); + + // {d_inner, n_tokens} * {d_inner} => {d_inner, n_tokens} + y = ggml_add(ctx0, y, ggml_mul(ctx0, x, model.layers[il].ssm_d)); + y = ggml_mul(ctx0, y, ggml_silu(ctx0, z)); + + // {d_inner, n_embd} * {d_inner, n_tokens} => {n_embd, n_tokens} + cur = ggml_mul_mat(ctx0, model.layers[il].ssm_out, y); + } + + // residual + cur = ggml_add(ctx0, cur, inpL); + cb(cur, "l_out", il); + + // input for next layer + inpL = cur; + } + + // final rmsnorm + cur = llm_build_norm(ctx0, inpL, hparams, + model.output_norm, NULL, + LLM_NORM_RMS, cb, -1); + cb(cur, "result_norm", -1); + + // lm_head + cur = ggml_mul_mat(ctx0, model.output, cur); + cb(cur, "result_output", -1); + + ggml_build_forward_expand(gf, cur); + + return gf; + } }; static struct ggml_cgraph * llama_build_graph_defrag(llama_context & lctx, const std::vector & ids) { @@ -7871,6 +8336,23 @@ static struct ggml_cgraph * llama_build_graph_k_shift(llama_context & lctx) { return result; } +static struct ggml_cgraph * llama_build_graph_s_copy(llama_context & lctx) { + llama_batch dummy; + dummy.n_tokens = 0; + + llm_build_cb cb = [&](struct ggml_tensor * , const char * , int ) { }; + + struct llm_build_context llm(lctx, dummy, cb, false); + + llm.init(); + + struct ggml_cgraph * result = llm.build_s_copy(); + + llm.free(); + + return result; +} + static struct ggml_cgraph * llama_build_graph( llama_context & lctx, const llama_batch & batch, @@ -7985,6 +8467,10 @@ static struct ggml_cgraph * llama_build_graph( { result = llm.build_starcoder2(); } break; + case LLM_ARCH_MAMBA: + { + result = llm.build_mamba(); + } break; default: GGML_ASSERT(false); } @@ -7995,19 +8481,29 @@ static struct ggml_cgraph * llama_build_graph( } static void llama_set_k_shift(llama_context & lctx) { - const auto & cparams = lctx.cparams; - - const int64_t n_ctx = cparams.n_ctx; + const int64_t kv_size = lctx.kv_self.size; assert(ggml_backend_buffer_is_host(lctx.inp_K_shift->buffer)); int32_t * data = (int32_t *) lctx.inp_K_shift->data; - for (int i = 0; i < n_ctx; ++i) { + for (int i = 0; i < kv_size; ++i) { data[i] = lctx.kv_self.cells[i].delta; } } +static void llama_set_s_copy(llama_context & lctx) { + const int64_t kv_size = lctx.kv_self.size; + + assert(ggml_backend_buffer_is_host(lctx.inp_s_copy->buffer)); + + int32_t * data = (int32_t *) lctx.inp_s_copy->data; + + for (int i = 0; i < kv_size; ++i) { + data[i] = lctx.kv_self.cells[i].src; + } +} + static void llama_set_inputs(llama_context & lctx, const llama_batch & batch) { // // set input data @@ -8044,6 +8540,9 @@ static void llama_set_inputs(llama_context & lctx, const llama_batch & batch) { float * data = (float *) lctx.inp_KQ_mask->data; + // For causal attention, use only the previous KV cells + // of the correct sequence for each token of the batch. + // It's assumed that if a token in the batch has multiple sequences, they are equivalent. for (int h = 0; h < 1; ++h) { for (int j = 0; j < n_tokens; ++j) { const llama_pos pos = batch.pos[j]; @@ -8149,6 +8648,53 @@ static void llama_set_inputs(llama_context & lctx, const llama_batch & batch) { } } } + + if (kv_self.recurrent) { + const int64_t n_kv = kv_self.n; + + { + GGML_ASSERT(ggml_backend_buffer_is_host(lctx.inp_s_mask->buffer)); + float * data = (float *) lctx.inp_s_mask->data; + + // states which are not affected by the current batch are left untouched + for (int i = 0; i < n_kv; ++i) { + llama_seq_id seq_id = i + lctx.kv_self.head; + llama_kv_cell & kv_cell = lctx.kv_self.cells[seq_id]; + bool has_self_seq = kv_cell.has_seq_id(seq_id); + + data[i] = (float) has_self_seq; + + // ensure current sequences will be kept + if (!has_self_seq && kv_cell.pos >= 0) { + kv_cell.seq_id.insert(seq_id); + } + } + } + // For Mamba (and other recurrent architectures), + // update the correct state(s)/sequence(s) for each token of the batch. + // Like with the KQ_mask, if a token in the batch has multiple sequences, + // they are assumed to be equivalent (not here, but in ggml_ssm_scan and ggml_ssm_conv). + { + const int64_t n_tokens = batch.n_tokens; + + GGML_ASSERT(ggml_backend_buffer_is_host(lctx.inp_s_seq->buffer)); + int32_t * data = (int32_t *) lctx.inp_s_seq->data; + + for (int j = 0; j < n_tokens; ++j) { + const int32_t n_seq = batch.n_seq_id[j]; + GGML_ASSERT(0 < n_seq); // a token should be part of at least 1 sequence + + for (int i = 0; i < n_kv; ++i) { + if (i < n_seq) { + // for this type of model, the head is the minimum seq_id of the batch + data[j*n_kv + i] = batch.seq_id[j][i] - kv_self.head; + } else { + data[j*n_kv + i] = -1; + } + } + } + } + } } static void llama_graph_compute( @@ -8271,11 +8817,13 @@ static int llama_decode_internal( return 1; } - // a heuristic, to avoid attending the full cache if it is not yet utilized - // after enough generations, the benefit from this heuristic disappears - // if we start defragmenting the cache, the benefit from this will be more important - kv_self.n = std::min(cparams.n_ctx, std::max(32u, GGML_PAD(llama_kv_cache_cell_max(kv_self), 32))); - //kv_self.n = llama_kv_cache_cell_max(kv_self); + if (!kv_self.recurrent) { + // a heuristic, to avoid attending the full cache if it is not yet utilized + // after enough generations, the benefit from this heuristic disappears + // if we start defragmenting the cache, the benefit from this will be more important + kv_self.n = std::min(kv_self.size, std::max(32u, GGML_PAD(llama_kv_cache_cell_max(kv_self), 32))); + //kv_self.n = llama_kv_cache_cell_max(kv_self); + } } //printf("kv_self.n = %5d, kv_self.used = %5d, kv_self.head = %5d\n", kv_self.n, kv_self.used, kv_self.head); @@ -8701,6 +9249,26 @@ static void llama_kv_cache_update_internal(struct llama_context & lctx) { } } + if (lctx.kv_self.recurrent && lctx.kv_self.do_copy) { + llama_set_s_copy(lctx); + + { + ggml_cgraph * gf = llama_build_graph_s_copy(lctx); + + llama_graph_compute(lctx, gf, lctx.cparams.n_threads); + } + + { + auto & kv_self = lctx.kv_self; + + kv_self.do_copy = false; + + for (uint32_t i = 0; i < kv_self.size; ++i) { + kv_self.cells[i].src = i; + } + } + } + // defragment the KV cache if needed if (lctx.kv_self.do_defrag) { llama_kv_cache_defrag_internal(lctx); @@ -11535,6 +12103,12 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s quantize &= name != LLM_TN(model.arch)(LLM_TENSOR_POS_EMBD, "weight"); quantize &= name != LLM_TN(model.arch)(LLM_TENSOR_TOKEN_TYPES, "weight"); + // do not quantize Mamba's small yet 2D weights + // NOTE: can't use LLM_TN here because the layer number is not known + quantize &= name.find("ssm_conv1d.weight") == std::string::npos; + quantize &= name.find("ssm_x.weight") == std::string::npos; + quantize &= name.find("ssm_dt.weight") == std::string::npos; + enum ggml_type new_type; void * new_data; size_t new_size; @@ -11985,6 +12559,7 @@ struct llama_context_params llama_context_default_params() { /*.seed =*/ LLAMA_DEFAULT_SEED, /*.n_ctx =*/ 512, /*.n_batch =*/ 512, + /*.n_parallel =*/ 1, /*.n_threads =*/ GGML_DEFAULT_N_THREADS, // TODO: better default /*.n_threads_batch =*/ GGML_DEFAULT_N_THREADS, /*.rope_scaling_type =*/ LLAMA_ROPE_SCALING_TYPE_UNSPECIFIED, @@ -12146,6 +12721,7 @@ struct llama_context * llama_new_context_with_model( auto & cparams = ctx->cparams; cparams.n_batch = params.n_batch; + // TODO: maybe add n_parallel here too cparams.n_threads = params.n_threads; cparams.n_threads_batch = params.n_threads_batch; cparams.yarn_ext_factor = params.yarn_ext_factor; @@ -12203,8 +12779,18 @@ struct llama_context * llama_new_context_with_model( ctx->rng = std::mt19937(params.seed); ctx->logits_all = params.logits_all; - const ggml_type type_k = params.type_k; - const ggml_type type_v = params.type_v; + uint32_t kv_size = cparams.n_ctx; + ggml_type type_k = params.type_k; + ggml_type type_v = params.type_v; + + // Mamba only needs a constant number of KV cache cells per sequence + if (model->arch == LLM_ARCH_MAMBA) { + // Mamba needs at least as many KV cells as there are sequences kept at any time + kv_size = std::max((uint32_t) 1, params.n_parallel); + // it's probably best to keep as much precision as possible for the states + type_k = GGML_TYPE_F32; // required by ggml_ssm_conv for Mamba's conv_states + type_v = GGML_TYPE_F32; // required by ggml_ssm_scan for Mamba's ssm_states + } GGML_ASSERT(hparams.n_embd_head_k % ggml_blck_size(type_k) == 0); GGML_ASSERT(hparams.n_embd_head_v % ggml_blck_size(type_v) == 0); @@ -12304,7 +12890,7 @@ struct llama_context * llama_new_context_with_model( } ctx->backends.push_back(ctx->backend_cpu); - if (!llama_kv_cache_init(ctx->kv_self, ctx->model, type_k, type_v, cparams.n_ctx, cparams.offload_kqv)) { + if (!llama_kv_cache_init(ctx->kv_self, ctx->model, type_k, type_v, kv_size, cparams.offload_kqv)) { LLAMA_LOG_ERROR("%s: llama_kv_cache_init() failed for self-attention cache\n", __func__); llama_free(ctx); return nullptr; @@ -12338,7 +12924,7 @@ struct llama_context * llama_new_context_with_model( // graph inputs { ggml_init_params init_params = { - /* .mem_size */ ggml_tensor_overhead()*8, + /* .mem_size */ ggml_tensor_overhead()*(8 + 3*(ctx->kv_self.recurrent)), /* .mem_buffer */ nullptr, /* .no_alloc */ true, }; @@ -12347,11 +12933,16 @@ struct llama_context * llama_new_context_with_model( ctx->inp_tokens = ggml_new_tensor_1d(ctx->ctx_input, GGML_TYPE_I32, cparams.n_batch); ctx->inp_embd = ggml_new_tensor_2d(ctx->ctx_input, GGML_TYPE_F32, hparams.n_embd, cparams.n_batch); ctx->inp_pos = ggml_new_tensor_1d(ctx->ctx_input, GGML_TYPE_I32, cparams.n_batch); - ctx->inp_KQ_mask = ggml_new_tensor_2d(ctx->ctx_input, GGML_TYPE_F32, cparams.n_ctx, cparams.n_batch); - ctx->inp_KQ_pos = ggml_new_tensor_1d(ctx->ctx_input, GGML_TYPE_F32, cparams.n_ctx); - ctx->inp_K_shift = ggml_new_tensor_1d(ctx->ctx_input, GGML_TYPE_I32, cparams.n_ctx); + ctx->inp_KQ_mask = ggml_new_tensor_2d(ctx->ctx_input, GGML_TYPE_F32, kv_size, cparams.n_batch); + ctx->inp_KQ_pos = ggml_new_tensor_1d(ctx->ctx_input, GGML_TYPE_F32, kv_size); + ctx->inp_K_shift = ggml_new_tensor_1d(ctx->ctx_input, GGML_TYPE_I32, kv_size); ctx->inp_mean = ggml_new_tensor_2d(ctx->ctx_input, GGML_TYPE_F32, cparams.n_batch, cparams.n_batch); ctx->inp_cls = ggml_new_tensor_1d(ctx->ctx_input, GGML_TYPE_I32, cparams.n_batch); + if (ctx->kv_self.recurrent) { + ctx->inp_s_copy = ggml_new_tensor_1d(ctx->ctx_input, GGML_TYPE_I32, kv_size); + ctx->inp_s_mask = ggml_new_tensor_1d(ctx->ctx_input, GGML_TYPE_F32, kv_size); + ctx->inp_s_seq = ggml_new_tensor_2d(ctx->ctx_input, GGML_TYPE_I32, kv_size, cparams.n_batch); + } ggml_set_name(ctx->inp_tokens, "inp_tokens"); ggml_set_name(ctx->inp_embd, "inp_embd"); @@ -12361,6 +12952,11 @@ struct llama_context * llama_new_context_with_model( ggml_set_name(ctx->inp_K_shift, "inp_K_shift"); ggml_set_name(ctx->inp_mean, "inp_mean"); ggml_set_name(ctx->inp_cls, "inp_cls"); + if (ctx->kv_self.recurrent) { + ggml_set_name(ctx->inp_s_copy, "inp_s_copy"); + ggml_set_name(ctx->inp_s_mask, "inp_s_mask"); + ggml_set_name(ctx->inp_s_seq, "inp_s_seq"); + } ctx->buf_input = ggml_backend_alloc_ctx_tensors_from_buft(ctx->ctx_input, llama_default_buffer_type_cpu(true)); LLAMA_LOG_INFO("%s: %10s input buffer size = %8.2f MiB\n", __func__, @@ -12447,6 +13043,10 @@ uint32_t llama_n_batch(const struct llama_context * ctx) { return ctx->cparams.n_batch; } +uint32_t llama_n_max_seq(const struct llama_context * ctx) { + return ctx->kv_self.size; +} + enum llama_vocab_type llama_vocab_type(const struct llama_model * model) { return model->vocab.type; } @@ -12460,6 +13060,7 @@ enum llama_rope_type llama_rope_type(const struct llama_model * model) { case LLM_ARCH_MPT: case LLM_ARCH_REFACT: case LLM_ARCH_BLOOM: + case LLM_ARCH_MAMBA: return LLAMA_ROPE_TYPE_NONE; // use what we call a normal RoPE, operating on pairs of consecutive head values @@ -12713,8 +13314,8 @@ void llama_kv_cache_clear(struct llama_context * ctx) { llama_kv_cache_clear(ctx->kv_self); } -void llama_kv_cache_seq_rm(struct llama_context * ctx, llama_seq_id seq_id, llama_pos p0, llama_pos p1) { - llama_kv_cache_seq_rm(ctx->kv_self, seq_id, p0, p1); +bool llama_kv_cache_seq_rm(struct llama_context * ctx, llama_seq_id seq_id, llama_pos p0, llama_pos p1) { + return llama_kv_cache_seq_rm(ctx->kv_self, seq_id, p0, p1); } void llama_kv_cache_seq_cp(struct llama_context * ctx, llama_seq_id seq_id_src, llama_seq_id seq_id_dst, llama_pos p0, llama_pos p1) { @@ -12891,8 +13492,8 @@ static void llama_copy_state_data_internal(struct llama_context * ctx, llama_dat const auto & hparams = ctx->model.hparams; const uint32_t n_layer = hparams.n_layer; - const uint32_t n_embd_k_gqa = hparams.n_embd_k_gqa(); - const uint32_t n_embd_v_gqa = hparams.n_embd_v_gqa(); + const uint32_t n_embd_k_gqa = hparams.n_embd_k_gqa() + hparams.n_embd_k_s(); + const uint32_t n_embd_v_gqa = hparams.n_embd_v_gqa() + hparams.n_embd_v_s(); const size_t kv_buf_size = kv_self.total_size(); const uint32_t kv_head = llama_kv_cache_cell_max(kv_self); @@ -12913,6 +13514,17 @@ static void llama_copy_state_data_internal(struct llama_context * ctx, llama_dat ggml_backend_tensor_get(kv_self.k_l[il], tmp_buf.data(), 0, tmp_buf.size()); data_ctx->write(tmp_buf.data(), tmp_buf.size()); + if (kv_self.recurrent) { + // v is contiguous for recurrent models + // TODO: use other tensors for state models than k and v + const size_t v_size = ggml_row_size(kv_self.v_l[il]->type, n_embd_v_gqa*kv_head); + + tmp_buf.resize(v_size); + ggml_backend_tensor_get(kv_self.v_l[il], tmp_buf.data(), 0, tmp_buf.size()); + data_ctx->write(tmp_buf.data(), tmp_buf.size()); + continue; + } + // v is not contiguous, copy row by row const size_t v_row_size = ggml_row_size(kv_self.v_l[il]->type, kv_head); const size_t v_row_stride = ggml_row_size(kv_self.v_l[il]->type, kv_size); @@ -13005,8 +13617,8 @@ size_t llama_set_state_data(struct llama_context * ctx, const uint8_t * src) { const auto & hparams = ctx->model.hparams; const uint32_t n_layer = hparams.n_layer; - const uint32_t n_embd_k_gqa = hparams.n_embd_k_gqa(); - const uint32_t n_embd_v_gqa = hparams.n_embd_v_gqa(); + const uint32_t n_embd_k_gqa = hparams.n_embd_k_gqa() + hparams.n_embd_k_s(); + const uint32_t n_embd_v_gqa = hparams.n_embd_v_gqa() + hparams.n_embd_v_s(); size_t kv_buf_size; uint32_t kv_head; @@ -13027,6 +13639,16 @@ size_t llama_set_state_data(struct llama_context * ctx, const uint8_t * src) { ggml_backend_tensor_set(kv_self.k_l[il], inp, 0, k_size); inp += k_size; + if (kv_self.recurrent) { + // v is contiguous for recurrent models + // TODO: use other tensors for state models than k and v + const size_t v_size = ggml_row_size(kv_self.v_l[il]->type, n_embd_v_gqa*kv_head); + + ggml_backend_tensor_set(kv_self.v_l[il], inp, 0, v_size); + inp += v_size; + continue; + } + // v is not contiguous, copy row by row const size_t v_row_size = ggml_row_size(kv_self.v_l[il]->type, kv_head); const size_t v_row_stride = ggml_row_size(kv_self.v_l[il]->type, kv_size); diff --git a/llama.h b/llama.h index 3dc162b07..7a107c7f3 100644 --- a/llama.h +++ b/llama.h @@ -235,6 +235,7 @@ extern "C" { uint32_t seed; // RNG seed, -1 for random uint32_t n_ctx; // text context, 0 = from model uint32_t n_batch; // prompt processing maximum batch size + uint32_t n_parallel; // number of parallel sequences (i.e. distinct states for recurrent models) uint32_t n_threads; // number of threads to use for generation uint32_t n_threads_batch; // number of threads to use for batch processing @@ -376,6 +377,7 @@ extern "C" { LLAMA_API uint32_t llama_n_ctx (const struct llama_context * ctx); LLAMA_API uint32_t llama_n_batch (const struct llama_context * ctx); + LLAMA_API uint32_t llama_n_max_seq (const struct llama_context * ctx); LLAMA_API enum llama_vocab_type llama_vocab_type(const struct llama_model * model); LLAMA_API enum llama_rope_type llama_rope_type (const struct llama_model * model); @@ -502,7 +504,7 @@ extern "C" { // seq_id < 0 : match any sequence // p0 < 0 : [0, p1] // p1 < 0 : [p0, inf) - LLAMA_API void llama_kv_cache_seq_rm( + LLAMA_API bool llama_kv_cache_seq_rm( struct llama_context * ctx, llama_seq_id seq_id, llama_pos p0, From fd72d2d2a5e79d61ccef6af3d15f16e5e5cbc352 Mon Sep 17 00:00:00 2001 From: Pierrick Hymbert Date: Sat, 9 Mar 2024 10:30:04 +0100 Subject: [PATCH 563/812] server: tests: add truncated prompt tests, better kv cache size (#5933) * server: tests: add truncated prompt tests, better size * server, tests : update regex --------- Co-authored-by: Georgi Gerganov --- examples/server/server.cpp | 23 +++++++++-- .../server/tests/features/parallel.feature | 5 ++- examples/server/tests/features/server.feature | 41 ++++++++++++++----- examples/server/tests/features/steps/steps.py | 35 +++++++++++++--- 4 files changed, 81 insertions(+), 23 deletions(-) diff --git a/examples/server/server.cpp b/examples/server/server.cpp index 59a59d56b..6f4449984 100644 --- a/examples/server/server.cpp +++ b/examples/server/server.cpp @@ -1128,6 +1128,7 @@ struct server_context { LOG_VERBOSE("stopped by limit", { {"id_slot", slot.id}, + {"id_task", slot.id_task}, {"n_decoded", slot.n_decoded}, {"n_predict", slot.params.n_predict}, }); @@ -1141,6 +1142,8 @@ struct server_context { } LOG_VERBOSE("next token", { + {"id_slot", slot.id}, + {"id_task", slot.id_task}, {"token", result.tok}, {"token_text", tokens_to_output_formatted_string(ctx, result.tok)}, {"has_next_token", slot.has_next_token}, @@ -1750,6 +1753,15 @@ struct server_context { slot.n_past = 0; slot.n_prompt_tokens = prompt_tokens.size(); + LOG_VERBOSE("prompt tokenized", { + {"id_slot", slot.id}, + {"id_task", slot.id_task}, + {"n_ctx", slot.n_ctx}, + {"n_keep", slot.params.n_keep}, + {"n_prompt_tokens", slot.n_prompt_tokens}, + {"prompt_tokens", tokens_to_str(ctx, prompt_tokens.cbegin(), prompt_tokens.cend())}, + }); + if (slot.embedding) { // this prompt is too large to process - discard it if (slot.n_prompt_tokens > n_batch) { @@ -1788,10 +1800,13 @@ struct server_context { slot.n_prompt_tokens = prompt_tokens.size(); LOG_VERBOSE("input truncated", { - {"n_ctx", slot.n_ctx}, - {"n_keep", slot.params.n_keep}, - {"n_left", n_left}, - {"prompt_tokens", tokens_to_str(ctx, prompt_tokens.cbegin(), prompt_tokens.cend())}, + {"id_slot", slot.id}, + {"id_task", slot.id_task}, + {"n_ctx", slot.n_ctx}, + {"n_keep", slot.params.n_keep}, + {"n_left", n_left}, + {"n_prompt_tokens", slot.n_prompt_tokens}, + {"prompt_tokens", tokens_to_str(ctx, prompt_tokens.cbegin(), prompt_tokens.cend())}, }); GGML_ASSERT(slot.n_prompt_tokens < slot.n_ctx); diff --git a/examples/server/tests/features/parallel.feature b/examples/server/tests/features/parallel.feature index 066698c8e..a66fed626 100644 --- a/examples/server/tests/features/parallel.feature +++ b/examples/server/tests/features/parallel.feature @@ -6,8 +6,8 @@ Feature: Parallel Given a server listening on localhost:8080 And a model file tinyllamas/stories260K.gguf from HF repo ggml-org/models And 42 as server seed - And 512 as batch size - And 64 KV cache size + And 128 as batch size + And 256 KV cache size And 2 slots And continuous batching Then the server is starting @@ -76,6 +76,7 @@ Feature: Parallel | disabled | 128 | | enabled | 64 | + Scenario: Multi users with total number of tokens to predict exceeds the KV Cache size #3969 Given a prompt: """ diff --git a/examples/server/tests/features/server.feature b/examples/server/tests/features/server.feature index 878ac1363..aa132fa34 100644 --- a/examples/server/tests/features/server.feature +++ b/examples/server/tests/features/server.feature @@ -10,11 +10,10 @@ Feature: llama.cpp server # KV Cache corresponds to the total amount of tokens # that can be stored across all independent sequences: #4130 # see --ctx-size and #5568 - And 32 KV cache size - And 512 as batch size - And 1 slots - And embeddings extraction - And 32 server max tokens to predict + And 256 KV cache size + And 32 as batch size + And 2 slots + And 64 server max tokens to predict And prometheus compatible metrics exposed Then the server is starting Then the server is healthy @@ -23,18 +22,35 @@ Feature: llama.cpp server Then the server is ready And all slots are idle + Scenario Outline: Completion Given a prompt And max tokens to predict And a completion request with no api error Then tokens are predicted matching + And the completion is truncated + And prompt tokens are processed And prometheus metrics are exposed And metric llamacpp:tokens_predicted is Examples: Prompts - | prompt | n_predict | re_content | n_predicted | - | I believe the meaning of life is | 8 | (read\|going)+ | 8 | - | Write a joke about AI | 64 | (park\|friends\|scared\|always)+ | 32 | + | prompt | n_predict | re_content | n_prompt | n_predicted | truncated | + | I believe the meaning of life is | 8 | (read\|going)+ | 18 | 8 | not | + | Write a joke about AI from a very long prompt which will not be truncated | 256 | (princesses\|everyone\|kids)+ | 46 | 64 | not | + + Scenario: Completion prompt truncated + Given a prompt: + """ + Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. + Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. + Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. + Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum. + """ + And a completion request with no api error + Then 64 tokens are predicted matching fun|Annaks|popcorns + And the completion is truncated + And 109 prompt tokens are processed + Scenario Outline: OAI Compatibility Given a model @@ -44,11 +60,14 @@ Feature: llama.cpp server And streaming is Given an OAI compatible chat completions request with no api error Then tokens are predicted matching + And prompt tokens are processed + And the completion is truncated Examples: Prompts - | model | system_prompt | user_prompt | max_tokens | re_content | n_predicted | enable_streaming | - | llama-2 | Book | What is the best book | 8 | (Mom\|what)+ | 8 | disabled | - | codellama70b | You are a coding assistant. | Write the fibonacci function in c++. | 64 | (thanks\|happy\|bird)+ | 32 | enabled | + | model | system_prompt | user_prompt | max_tokens | re_content | n_prompt | n_predicted | enable_streaming | truncated | + | llama-2 | Book | What is the best book | 8 | (Here\|what)+ | 77 | 8 | disabled | not | + | codellama70b | You are a coding assistant. | Write the fibonacci function in c++. | 128 | (thanks\|happy\|bird)+ | -1 | 64 | enabled | | + Scenario: Tokenize / Detokenize When tokenizing: diff --git a/examples/server/tests/features/steps/steps.py b/examples/server/tests/features/steps/steps.py index d7f005836..0076f805b 100644 --- a/examples/server/tests/features/steps/steps.py +++ b/examples/server/tests/features/steps/steps.py @@ -196,12 +196,30 @@ async def step_request_completion(context, api_error): @step(u'{predicted_n:d} tokens are predicted matching {re_content}') def step_n_tokens_predicted_with_content(context, predicted_n, re_content): - assert_n_tokens_predicted(context.tasks_result.pop(), predicted_n, re_content) + context.completion = context.tasks_result.pop() + assert_n_tokens_predicted(context.completion, predicted_n, re_content) @step(u'{predicted_n:d} tokens are predicted') def step_n_tokens_predicted(context, predicted_n): - assert_n_tokens_predicted(context.tasks_result.pop(), predicted_n) + context.completion = context.tasks_result.pop() + assert_n_tokens_predicted(context.completion, predicted_n) + + +@step(u'the completion is truncated') +def step_assert_completion_truncated(context): + step_assert_completion_truncated(context, '') + + +@step(u'the completion is {truncated} truncated') +def step_assert_completion_truncated(context, truncated): + truncated = truncated != "not" + assert context.completion['truncated'] == truncated, f'{context.completion}' + + +@step(u'{n_prompt:d} prompt tokens are processed') +def step_impl(context, n_prompt): + assert n_prompt < 0 or n_prompt == context.completion['timings']['prompt_n'], f"n_prompt={context.completion['timings']['prompt_n']}" @step(u'a user prompt {user_prompt}') @@ -722,7 +740,8 @@ async def oai_chat_completions(user_prompt, completion_response = { 'content': '', 'timings': { - 'predicted_n': 0 + 'predicted_n': 0, + 'prompt_n': 0 } } if async_client: @@ -763,7 +782,8 @@ async def oai_chat_completions(user_prompt, completion_response = { 'content': chat_completion_raw['choices'][0]['message'], 'timings': { - 'predicted_n': chat_completion_raw['usage']['completion_tokens'] + 'predicted_n': chat_completion_raw['usage']['completion_tokens'], + 'prompt_n': chat_completion_raw['usage']['prompt_tokens'] } } else: @@ -792,13 +812,16 @@ async def oai_chat_completions(user_prompt, if 'content' in delta: completion_response['content'] += delta['content'] completion_response['timings']['predicted_n'] += 1 + completion_response['truncated'] = chunk.choices[0].finish_reason != 'stop' else: assert len(chat_completion.choices) == 1 completion_response = { 'content': chat_completion.choices[0].message.content, 'timings': { - 'predicted_n': chat_completion.usage.completion_tokens - } + 'predicted_n': chat_completion.usage.completion_tokens, + 'prompt_n': chat_completion.usage.prompt_tokens + }, + 'truncated': chat_completion.choices[0].finish_reason != 'stop' } if debug: print("OAI response formatted to llama.cpp:", completion_response) From e1fa9569ba8ce276bc7801a3cebdcf8b1aa116ea Mon Sep 17 00:00:00 2001 From: Gabe Goodhart Date: Sat, 9 Mar 2024 02:57:09 -0700 Subject: [PATCH 564/812] server : add SSL support (#5926) * add cmake build toggle to enable ssl support in server Signed-off-by: Gabe Goodhart * add flags for ssl key/cert files and use SSLServer if set All SSL setup is hidden behind CPPHTTPLIB_OPENSSL_SUPPORT in the same way that the base httlib hides the SSL support Signed-off-by: Gabe Goodhart * Update readme for SSL support in server Signed-off-by: Gabe Goodhart * Add LLAMA_SERVER_SSL variable setup to top-level Makefile Signed-off-by: Gabe Goodhart --------- Signed-off-by: Gabe Goodhart --- Makefile | 4 ++ examples/server/CMakeLists.txt | 6 ++ examples/server/README.md | 26 ++++++++ examples/server/server.cpp | 108 ++++++++++++++++++++++----------- 4 files changed, 110 insertions(+), 34 deletions(-) diff --git a/Makefile b/Makefile index efce10bb8..aea969222 100644 --- a/Makefile +++ b/Makefile @@ -201,6 +201,10 @@ ifdef LLAMA_SERVER_VERBOSE MK_CPPFLAGS += -DSERVER_VERBOSE=$(LLAMA_SERVER_VERBOSE) endif +ifdef LLAMA_SERVER_SSL + MK_CPPFLAGS += -DCPPHTTPLIB_OPENSSL_SUPPORT + MK_LDFLAGS += -lssl -lcrypto +endif ifdef LLAMA_CODE_COVERAGE MK_CXXFLAGS += -fprofile-arcs -ftest-coverage -dumpbase '' diff --git a/examples/server/CMakeLists.txt b/examples/server/CMakeLists.txt index c21eba634..f94de1e99 100644 --- a/examples/server/CMakeLists.txt +++ b/examples/server/CMakeLists.txt @@ -1,5 +1,6 @@ set(TARGET server) option(LLAMA_SERVER_VERBOSE "Build verbose logging option for Server" ON) +option(LLAMA_SERVER_SSL "Build SSL support for the server" OFF) include_directories(${CMAKE_CURRENT_SOURCE_DIR}) add_executable(${TARGET} server.cpp utils.hpp json.hpp httplib.h) install(TARGETS ${TARGET} RUNTIME) @@ -7,6 +8,11 @@ target_compile_definitions(${TARGET} PRIVATE SERVER_VERBOSE=$ ) target_link_libraries(${TARGET} PRIVATE common ${CMAKE_THREAD_LIBS_INIT}) +if (LLAMA_SERVER_SSL) + find_package(OpenSSL REQUIRED) + target_link_libraries(${TARGET} PRIVATE OpenSSL::SSL OpenSSL::Crypto) + target_compile_definitions(${TARGET} PRIVATE CPPHTTPLIB_OPENSSL_SUPPORT) +endif() if (WIN32) TARGET_LINK_LIBRARIES(${TARGET} PRIVATE ws2_32) endif() diff --git a/examples/server/README.md b/examples/server/README.md index 591f748f8..bf8c450b6 100644 --- a/examples/server/README.md +++ b/examples/server/README.md @@ -59,6 +59,10 @@ see https://github.com/ggerganov/llama.cpp/issues/1437 - `--log-disable`: Output logs to stdout only, default: enabled. - `--log-format FORMAT`: Define the log output to FORMAT: json or text (default: json) +**If compiled with `LLAMA_SERVER_SSL=ON`** +- `--ssl-key-file FNAME`: path to file a PEM-encoded SSL private key +- `--ssl-cert-file FNAME`: path to file a PEM-encoded SSL certificate + ## Build server is build alongside everything else from the root of the project @@ -75,6 +79,28 @@ server is build alongside everything else from the root of the project cmake --build . --config Release ``` +## Build with SSL + +server can also be built with SSL support using OpenSSL 3 + +- Using `make`: + + ```bash + # NOTE: For non-system openssl, use the following: + # CXXFLAGS="-I /path/to/openssl/include" + # LDFLAGS="-L /path/to/openssl/lib" + make LLAMA_SERVER_SSL=true server + ``` + +- Using `CMake`: + + ```bash + mkdir build + cd build + cmake .. -DLLAMA_SERVER_SSL=ON + make server + ``` + ## Quick Start To get started right away, run the following command, making sure to use the correct path for the model you have: diff --git a/examples/server/server.cpp b/examples/server/server.cpp index 6f4449984..c3b87c846 100644 --- a/examples/server/server.cpp +++ b/examples/server/server.cpp @@ -27,6 +27,7 @@ #include #include #include +#include using json = nlohmann::json; @@ -118,6 +119,11 @@ struct server_params { std::vector api_keys; +#ifdef CPPHTTPLIB_OPENSSL_SUPPORT + std::string ssl_key_file = ""; + std::string ssl_cert_file = ""; +#endif + bool slots_endpoint = true; bool metrics_endpoint = false; }; @@ -2142,6 +2148,10 @@ static void server_print_usage(const char * argv0, const gpt_params & params, co printf(" --path PUBLIC_PATH path from which to serve static files (default %s)\n", sparams.public_path.c_str()); printf(" --api-key API_KEY optional api key to enhance server security. If set, requests must include this key for access.\n"); printf(" --api-key-file FNAME path to file containing api keys delimited by new lines. If set, requests must include one of the keys for access.\n"); +#ifdef CPPHTTPLIB_OPENSSL_SUPPORT + printf(" --ssl-key-file FNAME path to file a PEM-encoded SSL private key\n"); + printf(" --ssl-cert-file FNAME path to file a PEM-encoded SSL certificate\n"); +#endif printf(" -to N, --timeout N server read/write timeout in seconds (default: %d)\n", sparams.read_timeout); printf(" --embeddings enable embedding vector output (default: %s)\n", params.embedding ? "enabled" : "disabled"); printf(" -np N, --parallel N number of slots for process requests (default: %d)\n", params.n_parallel); @@ -2220,7 +2230,24 @@ static void server_params_parse(int argc, char ** argv, server_params & sparams, } } key_file.close(); - } else if (arg == "--timeout" || arg == "-to") { + + } +#ifdef CPPHTTPLIB_OPENSSL_SUPPORT + else if (arg == "--ssl-key-file") { + if (++i >= argc) { + invalid_param = true; + break; + } + sparams.ssl_key_file = argv[i]; + } else if (arg == "--ssl-cert-file") { + if (++i >= argc) { + invalid_param = true; + break; + } + sparams.ssl_cert_file = argv[i]; + } +#endif + else if (arg == "--timeout" || arg == "-to") { if (++i >= argc) { invalid_param = true; break; @@ -2658,21 +2685,34 @@ int main(int argc, char ** argv) { {"system_info", llama_print_system_info()}, }); - httplib::Server svr; + std::unique_ptr svr; +#ifdef CPPHTTPLIB_OPENSSL_SUPPORT + if (sparams.ssl_key_file != "" && sparams.ssl_cert_file != "") { + LOG_INFO("Running with SSL", {{"key", sparams.ssl_key_file}, {"cert", sparams.ssl_cert_file}}); + svr.reset( + new httplib::SSLServer(sparams.ssl_cert_file.c_str(), sparams.ssl_key_file.c_str()) + ); + } else { + LOG_INFO("Running without SSL", {}); + svr.reset(new httplib::Server()); + } +#else + svr.reset(new httplib::Server()); +#endif std::atomic state{SERVER_STATE_LOADING_MODEL}; - svr.set_default_headers({{"Server", "llama.cpp"}}); + svr->set_default_headers({{"Server", "llama.cpp"}}); // CORS preflight - svr.Options(R"(.*)", [](const httplib::Request & req, httplib::Response & res) { + svr->Options(R"(.*)", [](const httplib::Request & req, httplib::Response & res) { res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin")); res.set_header("Access-Control-Allow-Credentials", "true"); res.set_header("Access-Control-Allow-Methods", "POST"); res.set_header("Access-Control-Allow-Headers", "*"); }); - svr.Get("/health", [&](const httplib::Request & req, httplib::Response & res) { + svr->Get("/health", [&](const httplib::Request & req, httplib::Response & res) { server_state current_state = state.load(); switch (current_state) { case SERVER_STATE_READY: @@ -2728,7 +2768,7 @@ int main(int argc, char ** argv) { }); if (sparams.slots_endpoint) { - svr.Get("/slots", [&](const httplib::Request &, httplib::Response & res) { + svr->Get("/slots", [&](const httplib::Request &, httplib::Response & res) { // request slots data using task queue server_task task; task.id = ctx_server.queue_tasks.get_new_id(); @@ -2749,7 +2789,7 @@ int main(int argc, char ** argv) { } if (sparams.metrics_endpoint) { - svr.Get("/metrics", [&](const httplib::Request &, httplib::Response & res) { + svr->Get("/metrics", [&](const httplib::Request &, httplib::Response & res) { // request slots data using task queue server_task task; task.id = ctx_server.queue_tasks.get_new_id(); @@ -2846,9 +2886,9 @@ int main(int argc, char ** argv) { }); } - svr.set_logger(log_server_request); + svr->set_logger(log_server_request); - svr.set_exception_handler([](const httplib::Request &, httplib::Response & res, std::exception_ptr ep) { + svr->set_exception_handler([](const httplib::Request &, httplib::Response & res, std::exception_ptr ep) { const char fmt[] = "500 Internal Server Error\n%s"; char buf[BUFSIZ]; @@ -2864,7 +2904,7 @@ int main(int argc, char ** argv) { res.status = 500; }); - svr.set_error_handler([](const httplib::Request &, httplib::Response & res) { + svr->set_error_handler([](const httplib::Request &, httplib::Response & res) { if (res.status == 401) { res.set_content("Unauthorized", "text/plain; charset=utf-8"); } @@ -2877,16 +2917,16 @@ int main(int argc, char ** argv) { }); // set timeouts and change hostname and port - svr.set_read_timeout (sparams.read_timeout); - svr.set_write_timeout(sparams.write_timeout); + svr->set_read_timeout (sparams.read_timeout); + svr->set_write_timeout(sparams.write_timeout); - if (!svr.bind_to_port(sparams.hostname, sparams.port)) { + if (!svr->bind_to_port(sparams.hostname, sparams.port)) { fprintf(stderr, "\ncouldn't bind to server socket: hostname=%s port=%d\n\n", sparams.hostname.c_str(), sparams.port); return 1; } // Set the base directory for serving static files - svr.set_base_dir(sparams.public_path); + svr->set_base_dir(sparams.public_path); std::unordered_map log_data; @@ -2947,30 +2987,30 @@ int main(int argc, char ** argv) { }; // this is only called if no index.html is found in the public --path - svr.Get("/", [](const httplib::Request &, httplib::Response & res) { + svr->Get("/", [](const httplib::Request &, httplib::Response & res) { res.set_content(reinterpret_cast(&index_html), index_html_len, "text/html; charset=utf-8"); return false; }); // this is only called if no index.js is found in the public --path - svr.Get("/index.js", [](const httplib::Request &, httplib::Response & res) { + svr->Get("/index.js", [](const httplib::Request &, httplib::Response & res) { res.set_content(reinterpret_cast(&index_js), index_js_len, "text/javascript; charset=utf-8"); return false; }); // this is only called if no index.html is found in the public --path - svr.Get("/completion.js", [](const httplib::Request &, httplib::Response & res) { + svr->Get("/completion.js", [](const httplib::Request &, httplib::Response & res) { res.set_content(reinterpret_cast(&completion_js), completion_js_len, "application/javascript; charset=utf-8"); return false; }); // this is only called if no index.html is found in the public --path - svr.Get("/json-schema-to-grammar.mjs", [](const httplib::Request &, httplib::Response & res) { + svr->Get("/json-schema-to-grammar.mjs", [](const httplib::Request &, httplib::Response & res) { res.set_content(reinterpret_cast(&json_schema_to_grammar_mjs), json_schema_to_grammar_mjs_len, "application/javascript; charset=utf-8"); return false; }); - svr.Get("/props", [&ctx_server](const httplib::Request & req, httplib::Response & res) { + svr->Get("/props", [&ctx_server](const httplib::Request & req, httplib::Response & res) { res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin")); json data = { { "user_name", ctx_server.name_user.c_str() }, @@ -3062,11 +3102,11 @@ int main(int argc, char ** argv) { } }; - svr.Post("/completion", completions); // legacy - svr.Post("/completions", completions); - svr.Post("/v1/completions", completions); + svr->Post("/completion", completions); // legacy + svr->Post("/completions", completions); + svr->Post("/v1/completions", completions); - svr.Get("/v1/models", [¶ms, &model_meta](const httplib::Request & req, httplib::Response & res) { + svr->Get("/v1/models", [¶ms, &model_meta](const httplib::Request & req, httplib::Response & res) { res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin")); json models = { @@ -3161,10 +3201,10 @@ int main(int argc, char ** argv) { } }; - svr.Post("/chat/completions", chat_completions); - svr.Post("/v1/chat/completions", chat_completions); + svr->Post("/chat/completions", chat_completions); + svr->Post("/v1/chat/completions", chat_completions); - svr.Post("/infill", [&ctx_server, &validate_api_key](const httplib::Request & req, httplib::Response & res) { + svr->Post("/infill", [&ctx_server, &validate_api_key](const httplib::Request & req, httplib::Response & res) { res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin")); if (!validate_api_key(req, res)) { return; @@ -3228,11 +3268,11 @@ int main(int argc, char ** argv) { } }); - svr.Options(R"(/.*)", [](const httplib::Request &, httplib::Response & res) { + svr->Options(R"(/.*)", [](const httplib::Request &, httplib::Response & res) { return res.set_content("", "application/json; charset=utf-8"); }); - svr.Post("/tokenize", [&ctx_server](const httplib::Request & req, httplib::Response & res) { + svr->Post("/tokenize", [&ctx_server](const httplib::Request & req, httplib::Response & res) { res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin")); const json body = json::parse(req.body); @@ -3244,7 +3284,7 @@ int main(int argc, char ** argv) { return res.set_content(data.dump(), "application/json; charset=utf-8"); }); - svr.Post("/detokenize", [&ctx_server](const httplib::Request & req, httplib::Response & res) { + svr->Post("/detokenize", [&ctx_server](const httplib::Request & req, httplib::Response & res) { res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin")); const json body = json::parse(req.body); @@ -3258,7 +3298,7 @@ int main(int argc, char ** argv) { return res.set_content(data.dump(), "application/json; charset=utf-8"); }); - svr.Post("/embedding", [¶ms, &ctx_server](const httplib::Request & req, httplib::Response & res) { + svr->Post("/embedding", [¶ms, &ctx_server](const httplib::Request & req, httplib::Response & res) { res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin")); if (!params.embedding) { res.status = 501; @@ -3289,7 +3329,7 @@ int main(int argc, char ** argv) { return res.set_content(result.data.dump(), "application/json; charset=utf-8"); }); - svr.Post("/v1/embeddings", [¶ms, &ctx_server](const httplib::Request & req, httplib::Response & res) { + svr->Post("/v1/embeddings", [¶ms, &ctx_server](const httplib::Request & req, httplib::Response & res) { res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin")); if (!params.embedding) { res.status = 501; @@ -3360,13 +3400,13 @@ int main(int argc, char ** argv) { sparams.n_threads_http = std::max(params.n_parallel + 2, (int32_t) std::thread::hardware_concurrency() - 1); } log_data["n_threads_http"] = std::to_string(sparams.n_threads_http); - svr.new_task_queue = [&sparams] { return new httplib::ThreadPool(sparams.n_threads_http); }; + svr->new_task_queue = [&sparams] { return new httplib::ThreadPool(sparams.n_threads_http); }; LOG_INFO("HTTP server listening", log_data); // run the HTTP server in a thread - see comment below std::thread t([&]() { - if (!svr.listen_after_bind()) { + if (!svr->listen_after_bind()) { state.store(SERVER_STATE_ERROR); return 1; } @@ -3407,7 +3447,7 @@ int main(int argc, char ** argv) { ctx_server.queue_tasks.start_loop(); - svr.stop(); + svr->stop(); t.join(); llama_backend_free(); From 950ba1ab84db199f0bbdecdb2bb911f35261b321 Mon Sep 17 00:00:00 2001 From: Xuan Son Nguyen Date: Sat, 9 Mar 2024 11:27:53 +0100 Subject: [PATCH 565/812] Server: reorganize some http logic (#5939) * refactor static file handler * use set_pre_routing_handler for validate_api_key * merge embedding handlers * correct http verb for endpoints * fix embedding response * fix test case CORS Options * fix code style --- examples/server/README.md | 4 +- examples/server/server.cpp | 725 +++++++++--------- .../server/tests/features/security.feature | 5 +- examples/server/tests/features/steps/steps.py | 3 +- 4 files changed, 379 insertions(+), 358 deletions(-) diff --git a/examples/server/README.md b/examples/server/README.md index bf8c450b6..3abb1abe3 100644 --- a/examples/server/README.md +++ b/examples/server/README.md @@ -42,7 +42,7 @@ see https://github.com/ggerganov/llama.cpp/issues/1437 - `-to N`, `--timeout N`: Server read/write timeout in seconds. Default `600`. - `--host`: Set the hostname or ip address to listen. Default `127.0.0.1`. - `--port`: Set the port to listen. Default: `8080`. -- `--path`: path from which to serve static files (default examples/server/public) +- `--path`: path from which to serve static files (default: disabled) - `--api-key`: Set an api key for request authorization. By default the server responds to every request. With an api key set, the requests must have the Authorization header set with the api key as Bearer token. May be used multiple times to enable multiple valid keys. - `--api-key-file`: path to file containing api keys delimited by new lines. If set, requests must include one of the keys for access. May be used in conjunction with `--api-key`'s. - `--embedding`: Enable embedding extraction, Default: disabled. @@ -558,7 +558,7 @@ The HTTP server supports OAI-like API ### Extending or building alternative Web Front End -The default location for the static files is `examples/server/public`. You can extend the front end by running the server binary with `--path` set to `./your-directory` and importing `/completion.js` to get access to the llamaComplete() method. +You can extend the front end by running the server binary with `--path` set to `./your-directory` and importing `/completion.js` to get access to the llamaComplete() method. Read the documentation in `/completion.js` to see convenient ways to access llama. diff --git a/examples/server/server.cpp b/examples/server/server.cpp index c3b87c846..6e0f8328c 100644 --- a/examples/server/server.cpp +++ b/examples/server/server.cpp @@ -113,7 +113,7 @@ struct server_params { int32_t n_threads_http = -1; std::string hostname = "127.0.0.1"; - std::string public_path = "examples/server/public"; + std::string public_path = ""; std::string chat_template = ""; std::string system_prompt = ""; @@ -2145,7 +2145,7 @@ static void server_print_usage(const char * argv0, const gpt_params & params, co printf(" --lora-base FNAME optional model to use as a base for the layers modified by the LoRA adapter\n"); printf(" --host ip address to listen (default (default: %s)\n", sparams.hostname.c_str()); printf(" --port PORT port to listen (default (default: %d)\n", sparams.port); - printf(" --path PUBLIC_PATH path from which to serve static files (default %s)\n", sparams.public_path.c_str()); + printf(" --path PUBLIC_PATH path from which to serve static files (default: disabled)\n"); printf(" --api-key API_KEY optional api key to enhance server security. If set, requests must include this key for access.\n"); printf(" --api-key-file FNAME path to file containing api keys delimited by new lines. If set, requests must include one of the keys for access.\n"); #ifdef CPPHTTPLIB_OPENSSL_SUPPORT @@ -2211,7 +2211,7 @@ static void server_params_parse(int argc, char ** argv, server_params & sparams, invalid_param = true; break; } - sparams.api_keys.emplace_back(argv[i]); + sparams.api_keys.push_back(argv[i]); } else if (arg == "--api-key-file") { if (++i >= argc) { invalid_param = true; @@ -2712,7 +2712,143 @@ int main(int argc, char ** argv) { res.set_header("Access-Control-Allow-Headers", "*"); }); - svr->Get("/health", [&](const httplib::Request & req, httplib::Response & res) { + svr->set_logger(log_server_request); + + svr->set_exception_handler([](const httplib::Request &, httplib::Response & res, std::exception_ptr ep) { + const char fmt[] = "500 Internal Server Error\n%s"; + + char buf[BUFSIZ]; + try { + std::rethrow_exception(std::move(ep)); + } catch (std::exception &e) { + snprintf(buf, sizeof(buf), fmt, e.what()); + } catch (...) { + snprintf(buf, sizeof(buf), fmt, "Unknown Exception"); + } + + res.set_content(buf, "text/plain; charset=utf-8"); + res.status = 500; + }); + + svr->set_error_handler([](const httplib::Request &, httplib::Response & res) { + if (res.status == 401) { + res.set_content("Unauthorized", "text/plain; charset=utf-8"); + } + if (res.status == 400) { + res.set_content("Invalid request", "text/plain; charset=utf-8"); + } + if (res.status == 404) { + res.set_content("File Not Found", "text/plain; charset=utf-8"); + } + }); + + // set timeouts and change hostname and port + svr->set_read_timeout (sparams.read_timeout); + svr->set_write_timeout(sparams.write_timeout); + + if (!svr->bind_to_port(sparams.hostname, sparams.port)) { + fprintf(stderr, "\ncouldn't bind to server socket: hostname=%s port=%d\n\n", sparams.hostname.c_str(), sparams.port); + return 1; + } + + std::unordered_map log_data; + + log_data["hostname"] = sparams.hostname; + log_data["port"] = std::to_string(sparams.port); + + if (sparams.api_keys.size() == 1) { + auto key = sparams.api_keys[0]; + log_data["api_key"] = "api_key: ****" + key.substr(std::max((int)(key.length() - 4), 0)); + } else if (sparams.api_keys.size() > 1) { + log_data["api_key"] = "api_key: " + std::to_string(sparams.api_keys.size()) + " keys loaded"; + } + + // load the model + if (!ctx_server.load_model(params)) { + state.store(SERVER_STATE_ERROR); + return 1; + } else { + ctx_server.initialize(); + state.store(SERVER_STATE_READY); + } + + LOG_INFO("model loaded", {}); + + const auto model_meta = ctx_server.model_meta(); + + if (sparams.chat_template.empty()) { // custom chat template is not supplied + if (!ctx_server.validate_model_chat_template()) { + LOG_ERROR("The chat template that comes with this model is not yet supported, falling back to chatml. This may cause the model to output suboptimal responses", {}); + sparams.chat_template = "chatml"; + } + } + + // + // Middlewares + // + + auto middleware_validate_api_key = [&sparams](const httplib::Request & req, httplib::Response & res) { + // TODO: should we apply API key to all endpoints, including "/health" and "/models"? + static const std::set protected_endpoints = { + "/props", + "/completion", + "/completions", + "/v1/completions", + "/chat/completions", + "/v1/chat/completions", + "/infill", + "/tokenize", + "/detokenize", + "/embedding", + "/embeddings", + "/v1/embeddings", + }; + + // If API key is not set, skip validation + if (sparams.api_keys.empty()) { + return true; + } + + // If path is not in protected_endpoints list, skip validation + if (protected_endpoints.find(req.path) == protected_endpoints.end()) { + return true; + } + + // Check for API key in the header + auto auth_header = req.get_header_value("Authorization"); + + std::string prefix = "Bearer "; + if (auth_header.substr(0, prefix.size()) == prefix) { + std::string received_api_key = auth_header.substr(prefix.size()); + if (std::find(sparams.api_keys.begin(), sparams.api_keys.end(), received_api_key) != sparams.api_keys.end()) { + return true; // API key is valid + } + } + + // API key is invalid or not provided + // TODO: make another middleware for CORS related logic + res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin")); + res.set_content("Unauthorized: Invalid API Key", "text/plain; charset=utf-8"); + res.status = 401; // Unauthorized + + LOG_WARNING("Unauthorized: Invalid API Key", {}); + + return false; + }; + + // register server middlewares + svr->set_pre_routing_handler([&middleware_validate_api_key](const httplib::Request & req, httplib::Response & res) { + if (!middleware_validate_api_key(req, res)) { + return httplib::Server::HandlerResponse::Handled; + } + return httplib::Server::HandlerResponse::Unhandled; + }); + + // + // Route handlers (or controllers) + // + + const auto handle_health = [&](const httplib::Request & req, httplib::Response & res) { server_state current_state = state.load(); switch (current_state) { case SERVER_STATE_READY: @@ -2765,252 +2901,136 @@ int main(int argc, char ** argv) { res.status = 500; // HTTP Internal Server Error } break; } - }); - - if (sparams.slots_endpoint) { - svr->Get("/slots", [&](const httplib::Request &, httplib::Response & res) { - // request slots data using task queue - server_task task; - task.id = ctx_server.queue_tasks.get_new_id(); - task.id_multi = -1; - task.id_target = -1; - task.type = SERVER_TASK_TYPE_METRICS; - - ctx_server.queue_results.add_waiting_task_id(task.id); - ctx_server.queue_tasks.post(task); - - // get the result - server_task_result result = ctx_server.queue_results.recv(task.id); - ctx_server.queue_results.remove_waiting_task_id(task.id); - - res.set_content(result.data["slots"].dump(), "application/json"); - res.status = 200; // HTTP OK - }); - } - - if (sparams.metrics_endpoint) { - svr->Get("/metrics", [&](const httplib::Request &, httplib::Response & res) { - // request slots data using task queue - server_task task; - task.id = ctx_server.queue_tasks.get_new_id(); - task.id_multi = -1; - task.id_target = -1; - task.type = SERVER_TASK_TYPE_METRICS; - task.data.push_back({{"reset_bucket", true}}); - - ctx_server.queue_results.add_waiting_task_id(task.id); - ctx_server.queue_tasks.post(task); - - // get the result - server_task_result result = ctx_server.queue_results.recv(task.id); - ctx_server.queue_results.remove_waiting_task_id(task.id); - - json data = result.data; - - const uint64_t n_prompt_tokens_processed = data["n_prompt_tokens_processed"]; - const uint64_t t_prompt_processing = data["t_prompt_processing"]; - - const uint64_t n_tokens_predicted = data["n_tokens_predicted"]; - const uint64_t t_tokens_generation = data["t_tokens_generation"]; - - const int32_t kv_cache_used_cells = data["kv_cache_used_cells"]; - - // metrics definition: https://prometheus.io/docs/practices/naming/#metric-names - json all_metrics_def = json { - {"counter", {{ - {"name", "prompt_tokens_total"}, - {"help", "Number of prompt tokens processed."}, - {"value", (uint64_t) data["n_prompt_tokens_processed_total"]} - }, { - {"name", "prompt_seconds_total"}, - {"help", "Prompt process time"}, - {"value", (uint64_t) data["t_prompt_processing_total"] / 1.e3} - }, { - {"name", "tokens_predicted_total"}, - {"help", "Number of generation tokens processed."}, - {"value", (uint64_t) data["n_tokens_predicted_total"]} - }, { - {"name", "tokens_predicted_seconds_total"}, - {"help", "Predict process time"}, - {"value", (uint64_t) data["t_tokens_generation_total"] / 1.e3} - }}}, - {"gauge", {{ - {"name", "prompt_tokens_seconds"}, - {"help", "Average prompt throughput in tokens/s."}, - {"value", n_prompt_tokens_processed ? 1.e3 / t_prompt_processing * n_prompt_tokens_processed : 0.} - },{ - {"name", "predicted_tokens_seconds"}, - {"help", "Average generation throughput in tokens/s."}, - {"value", n_tokens_predicted ? 1.e3 / t_tokens_generation * n_tokens_predicted : 0.} - },{ - {"name", "kv_cache_usage_ratio"}, - {"help", "KV-cache usage. 1 means 100 percent usage."}, - {"value", 1. * kv_cache_used_cells / params.n_ctx} - },{ - {"name", "kv_cache_tokens"}, - {"help", "KV-cache tokens."}, - {"value", (uint64_t) data["kv_cache_tokens_count"]} - },{ - {"name", "requests_processing"}, - {"help", "Number of request processing."}, - {"value", (uint64_t) data["processing"]} - },{ - {"name", "requests_deferred"}, - {"help", "Number of request deferred."}, - {"value", (uint64_t) data["deferred"]} - }}} - }; - - std::stringstream prometheus; - - for (const auto & el : all_metrics_def.items()) { - const auto & type = el.key(); - const auto & metrics_def = el.value(); - - for (const auto & metric_def : metrics_def) { - const std::string name = metric_def["name"]; - const std::string help = metric_def["help"]; - - auto value = json_value(metric_def, "value", 0.); - prometheus << "# HELP llamacpp:" << name << " " << help << "\n" - << "# TYPE llamacpp:" << name << " " << type << "\n" - << "llamacpp:" << name << " " << value << "\n"; - } - } - - const int64_t t_start = data["t_start"]; - res.set_header("Process-Start-Time-Unix", std::to_string(t_start)); - - res.set_content(prometheus.str(), "text/plain; version=0.0.4"); - res.status = 200; // HTTP OK - }); - } - - svr->set_logger(log_server_request); - - svr->set_exception_handler([](const httplib::Request &, httplib::Response & res, std::exception_ptr ep) { - const char fmt[] = "500 Internal Server Error\n%s"; - - char buf[BUFSIZ]; - try { - std::rethrow_exception(std::move(ep)); - } catch (std::exception &e) { - snprintf(buf, sizeof(buf), fmt, e.what()); - } catch (...) { - snprintf(buf, sizeof(buf), fmt, "Unknown Exception"); - } - - res.set_content(buf, "text/plain; charset=utf-8"); - res.status = 500; - }); - - svr->set_error_handler([](const httplib::Request &, httplib::Response & res) { - if (res.status == 401) { - res.set_content("Unauthorized", "text/plain; charset=utf-8"); - } - if (res.status == 400) { - res.set_content("Invalid request", "text/plain; charset=utf-8"); - } - if (res.status == 404) { - res.set_content("File Not Found", "text/plain; charset=utf-8"); - } - }); - - // set timeouts and change hostname and port - svr->set_read_timeout (sparams.read_timeout); - svr->set_write_timeout(sparams.write_timeout); - - if (!svr->bind_to_port(sparams.hostname, sparams.port)) { - fprintf(stderr, "\ncouldn't bind to server socket: hostname=%s port=%d\n\n", sparams.hostname.c_str(), sparams.port); - return 1; - } - - // Set the base directory for serving static files - svr->set_base_dir(sparams.public_path); - - std::unordered_map log_data; - - log_data["hostname"] = sparams.hostname; - log_data["port"] = std::to_string(sparams.port); - - if (sparams.api_keys.size() == 1) { - log_data["api_key"] = "api_key: ****" + sparams.api_keys[0].substr(sparams.api_keys[0].length() - 4); - } else if (sparams.api_keys.size() > 1) { - log_data["api_key"] = "api_key: " + std::to_string(sparams.api_keys.size()) + " keys loaded"; - } - - // load the model - if (!ctx_server.load_model(params)) { - state.store(SERVER_STATE_ERROR); - return 1; - } else { - ctx_server.initialize(); - state.store(SERVER_STATE_READY); - } - - LOG_INFO("model loaded", {}); - - const auto model_meta = ctx_server.model_meta(); - - if (sparams.chat_template.empty()) { // custom chat template is not supplied - if (!ctx_server.validate_model_chat_template()) { - LOG_ERROR("The chat template that comes with this model is not yet supported, falling back to chatml. This may cause the model to output suboptimal responses", {}); - sparams.chat_template = "chatml"; - } - } - - // Middleware for API key validation - auto validate_api_key = [&sparams](const httplib::Request &req, httplib::Response &res) -> bool { - // If API key is not set, skip validation - if (sparams.api_keys.empty()) { - return true; - } - - // Check for API key in the header - auto auth_header = req.get_header_value("Authorization"); - - std::string prefix = "Bearer "; - if (auth_header.substr(0, prefix.size()) == prefix) { - std::string received_api_key = auth_header.substr(prefix.size()); - if (std::find(sparams.api_keys.begin(), sparams.api_keys.end(), received_api_key) != sparams.api_keys.end()) { - return true; // API key is valid - } - } - - // API key is invalid or not provided - res.set_content("Unauthorized: Invalid API Key", "text/plain; charset=utf-8"); - res.status = 401; // Unauthorized - - LOG_WARNING("Unauthorized: Invalid API Key", {}); - - return false; }; - // this is only called if no index.html is found in the public --path - svr->Get("/", [](const httplib::Request &, httplib::Response & res) { - res.set_content(reinterpret_cast(&index_html), index_html_len, "text/html; charset=utf-8"); - return false; - }); + const auto handle_slots = [&](const httplib::Request &, httplib::Response & res) { + if (!sparams.slots_endpoint) { + res.status = 501; + res.set_content("This server does not support slots endpoint.", "text/plain; charset=utf-8"); + return; + } - // this is only called if no index.js is found in the public --path - svr->Get("/index.js", [](const httplib::Request &, httplib::Response & res) { - res.set_content(reinterpret_cast(&index_js), index_js_len, "text/javascript; charset=utf-8"); - return false; - }); + // request slots data using task queue + server_task task; + task.id = ctx_server.queue_tasks.get_new_id(); + task.id_multi = -1; + task.id_target = -1; + task.type = SERVER_TASK_TYPE_METRICS; - // this is only called if no index.html is found in the public --path - svr->Get("/completion.js", [](const httplib::Request &, httplib::Response & res) { - res.set_content(reinterpret_cast(&completion_js), completion_js_len, "application/javascript; charset=utf-8"); - return false; - }); + ctx_server.queue_results.add_waiting_task_id(task.id); + ctx_server.queue_tasks.post(task); - // this is only called if no index.html is found in the public --path - svr->Get("/json-schema-to-grammar.mjs", [](const httplib::Request &, httplib::Response & res) { - res.set_content(reinterpret_cast(&json_schema_to_grammar_mjs), json_schema_to_grammar_mjs_len, "application/javascript; charset=utf-8"); - return false; - }); + // get the result + server_task_result result = ctx_server.queue_results.recv(task.id); + ctx_server.queue_results.remove_waiting_task_id(task.id); - svr->Get("/props", [&ctx_server](const httplib::Request & req, httplib::Response & res) { + res.set_content(result.data["slots"].dump(), "application/json"); + res.status = 200; // HTTP OK + }; + + const auto handle_metrics = [&](const httplib::Request &, httplib::Response & res) { + if (!sparams.metrics_endpoint) { + res.status = 501; + res.set_content("This server does not support metrics endpoint.", "text/plain; charset=utf-8"); + return; + } + + // request slots data using task queue + server_task task; + task.id = ctx_server.queue_tasks.get_new_id(); + task.id_multi = -1; + task.id_target = -1; + task.type = SERVER_TASK_TYPE_METRICS; + task.data.push_back({{"reset_bucket", true}}); + + ctx_server.queue_results.add_waiting_task_id(task.id); + ctx_server.queue_tasks.post(task); + + // get the result + server_task_result result = ctx_server.queue_results.recv(task.id); + ctx_server.queue_results.remove_waiting_task_id(task.id); + + json data = result.data; + + const uint64_t n_prompt_tokens_processed = data["n_prompt_tokens_processed"]; + const uint64_t t_prompt_processing = data["t_prompt_processing"]; + + const uint64_t n_tokens_predicted = data["n_tokens_predicted"]; + const uint64_t t_tokens_generation = data["t_tokens_generation"]; + + const int32_t kv_cache_used_cells = data["kv_cache_used_cells"]; + + // metrics definition: https://prometheus.io/docs/practices/naming/#metric-names + json all_metrics_def = json { + {"counter", {{ + {"name", "prompt_tokens_total"}, + {"help", "Number of prompt tokens processed."}, + {"value", (uint64_t) data["n_prompt_tokens_processed_total"]} + }, { + {"name", "prompt_seconds_total"}, + {"help", "Prompt process time"}, + {"value", (uint64_t) data["t_prompt_processing_total"] / 1.e3} + }, { + {"name", "tokens_predicted_total"}, + {"help", "Number of generation tokens processed."}, + {"value", (uint64_t) data["n_tokens_predicted_total"]} + }, { + {"name", "tokens_predicted_seconds_total"}, + {"help", "Predict process time"}, + {"value", (uint64_t) data["t_tokens_generation_total"] / 1.e3} + }}}, + {"gauge", {{ + {"name", "prompt_tokens_seconds"}, + {"help", "Average prompt throughput in tokens/s."}, + {"value", n_prompt_tokens_processed ? 1.e3 / t_prompt_processing * n_prompt_tokens_processed : 0.} + },{ + {"name", "predicted_tokens_seconds"}, + {"help", "Average generation throughput in tokens/s."}, + {"value", n_tokens_predicted ? 1.e3 / t_tokens_generation * n_tokens_predicted : 0.} + },{ + {"name", "kv_cache_usage_ratio"}, + {"help", "KV-cache usage. 1 means 100 percent usage."}, + {"value", 1. * kv_cache_used_cells / params.n_ctx} + },{ + {"name", "kv_cache_tokens"}, + {"help", "KV-cache tokens."}, + {"value", (uint64_t) data["kv_cache_tokens_count"]} + },{ + {"name", "requests_processing"}, + {"help", "Number of request processing."}, + {"value", (uint64_t) data["processing"]} + },{ + {"name", "requests_deferred"}, + {"help", "Number of request deferred."}, + {"value", (uint64_t) data["deferred"]} + }}} + }; + + std::stringstream prometheus; + + for (const auto & el : all_metrics_def.items()) { + const auto & type = el.key(); + const auto & metrics_def = el.value(); + + for (const auto & metric_def : metrics_def) { + const std::string name = metric_def["name"]; + const std::string help = metric_def["help"]; + + auto value = json_value(metric_def, "value", 0.); + prometheus << "# HELP llamacpp:" << name << " " << help << "\n" + << "# TYPE llamacpp:" << name << " " << type << "\n" + << "llamacpp:" << name << " " << value << "\n"; + } + } + + const int64_t t_start = data["t_start"]; + res.set_header("Process-Start-Time-Unix", std::to_string(t_start)); + + res.set_content(prometheus.str(), "text/plain; version=0.0.4"); + res.status = 200; // HTTP OK + }; + + const auto handle_props = [&ctx_server](const httplib::Request & req, httplib::Response & res) { res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin")); json data = { { "user_name", ctx_server.name_user.c_str() }, @@ -3020,13 +3040,10 @@ int main(int argc, char ** argv) { }; res.set_content(data.dump(), "application/json; charset=utf-8"); - }); + }; - const auto completions = [&ctx_server, &validate_api_key](const httplib::Request & req, httplib::Response & res) { + const auto handle_completions = [&ctx_server](const httplib::Request & req, httplib::Response & res) { res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin")); - if (!validate_api_key(req, res)) { - return; - } json data = json::parse(req.body); @@ -3102,11 +3119,7 @@ int main(int argc, char ** argv) { } }; - svr->Post("/completion", completions); // legacy - svr->Post("/completions", completions); - svr->Post("/v1/completions", completions); - - svr->Get("/v1/models", [¶ms, &model_meta](const httplib::Request & req, httplib::Response & res) { + const auto handle_models = [¶ms, &model_meta](const httplib::Request & req, httplib::Response & res) { res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin")); json models = { @@ -3123,14 +3136,10 @@ int main(int argc, char ** argv) { }; res.set_content(models.dump(), "application/json; charset=utf-8"); - }); + }; - const auto chat_completions = [&ctx_server, &validate_api_key, &sparams](const httplib::Request & req, httplib::Response & res) { + const auto handle_chat_completions = [&ctx_server, &sparams](const httplib::Request & req, httplib::Response & res) { res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin")); - if (!validate_api_key(req, res)) { - return; - } - json data = oaicompat_completion_params_parse(ctx_server.model, json::parse(req.body), sparams.chat_template); const int id_task = ctx_server.queue_tasks.get_new_id(); @@ -3201,14 +3210,8 @@ int main(int argc, char ** argv) { } }; - svr->Post("/chat/completions", chat_completions); - svr->Post("/v1/chat/completions", chat_completions); - - svr->Post("/infill", [&ctx_server, &validate_api_key](const httplib::Request & req, httplib::Response & res) { + const auto handle_infill = [&ctx_server](const httplib::Request & req, httplib::Response & res) { res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin")); - if (!validate_api_key(req, res)) { - return; - } json data = json::parse(req.body); @@ -3266,13 +3269,9 @@ int main(int argc, char ** argv) { res.set_chunked_content_provider("text/event-stream", chunked_content_provider, on_complete); } - }); + }; - svr->Options(R"(/.*)", [](const httplib::Request &, httplib::Response & res) { - return res.set_content("", "application/json; charset=utf-8"); - }); - - svr->Post("/tokenize", [&ctx_server](const httplib::Request & req, httplib::Response & res) { + const auto handle_tokenize = [&ctx_server](const httplib::Request & req, httplib::Response & res) { res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin")); const json body = json::parse(req.body); @@ -3282,9 +3281,9 @@ int main(int argc, char ** argv) { } const json data = format_tokenizer_response(tokens); return res.set_content(data.dump(), "application/json; charset=utf-8"); - }); + }; - svr->Post("/detokenize", [&ctx_server](const httplib::Request & req, httplib::Response & res) { + const auto handle_detokenize = [&ctx_server](const httplib::Request & req, httplib::Response & res) { res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin")); const json body = json::parse(req.body); @@ -3296,9 +3295,9 @@ int main(int argc, char ** argv) { const json data = format_detokenized_response(content); return res.set_content(data.dump(), "application/json; charset=utf-8"); - }); + }; - svr->Post("/embedding", [¶ms, &ctx_server](const httplib::Request & req, httplib::Response & res) { + const auto handle_embeddings = [¶ms, &ctx_server](const httplib::Request & req, httplib::Response & res) { res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin")); if (!params.embedding) { res.status = 501; @@ -3307,94 +3306,114 @@ int main(int argc, char ** argv) { } const json body = json::parse(req.body); + bool is_openai = false; - json prompt; - if (body.count("content") != 0) { - prompt = body["content"]; - } else { - prompt = ""; - } - - // create and queue the task - const int id_task = ctx_server.queue_tasks.get_new_id(); - - ctx_server.queue_results.add_waiting_task_id(id_task); - ctx_server.request_completion(id_task, -1, { {"prompt", prompt}, { "n_predict", 0} }, false, true); - - // get the result - server_task_result result = ctx_server.queue_results.recv(id_task); - ctx_server.queue_results.remove_waiting_task_id(id_task); - - // send the result - return res.set_content(result.data.dump(), "application/json; charset=utf-8"); - }); - - svr->Post("/v1/embeddings", [¶ms, &ctx_server](const httplib::Request & req, httplib::Response & res) { - res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin")); - if (!params.embedding) { - res.status = 501; - res.set_content("This server does not support embeddings. Start it with `--embeddings`", "text/plain; charset=utf-8"); - return; - } - - const json body = json::parse(req.body); - - json prompt; + // an input prompt can string or a list of tokens (integer) + std::vector prompts; if (body.count("input") != 0) { - prompt = body["input"]; - if (prompt.is_array()) { - json data = json::array(); - - int i = 0; - for (const json & elem : prompt) { - const int id_task = ctx_server.queue_tasks.get_new_id(); - - ctx_server.queue_results.add_waiting_task_id(id_task); - ctx_server.request_completion(id_task, -1, { {"prompt", elem}, { "n_predict", 0} }, false, true); - - // get the result - server_task_result result = ctx_server.queue_results.recv(id_task); - ctx_server.queue_results.remove_waiting_task_id(id_task); - - json embedding = json{ - {"embedding", json_value(result.data, "embedding", json::array())}, - {"index", i++}, - {"object", "embedding"} - }; - - data.push_back(embedding); + is_openai = true; + if (body["input"].is_array()) { + // support multiple prompts + for (const json & elem : body["input"]) { + prompts.push_back(elem); } - - json result = format_embeddings_response_oaicompat(body, data); - - return res.set_content(result.dump(), "application/json; charset=utf-8"); + } else { + // single input prompt + prompts.push_back(body["input"]); } + } else if (body.count("content") != 0) { + // only support single prompt here + std::string content = body["content"]; + prompts.push_back(content); } else { - prompt = ""; + // TODO @ngxson : should return an error here + prompts.push_back(""); } - // create and queue the task - const int id_task = ctx_server.queue_tasks.get_new_id(); + // process all prompts + json responses = json::array(); + for (auto & prompt : prompts) { + // TODO @ngxson : maybe support multitask for this endpoint? + // create and queue the task + const int id_task = ctx_server.queue_tasks.get_new_id(); - ctx_server.queue_results.add_waiting_task_id(id_task); - ctx_server.request_completion(id_task, -1, { {"prompt", prompt}, { "n_predict", 0}}, false, true); + ctx_server.queue_results.add_waiting_task_id(id_task); + ctx_server.request_completion(id_task, -1, { {"prompt", prompt}, { "n_predict", 0}}, false, true); - // get the result - server_task_result result = ctx_server.queue_results.recv(id_task); - ctx_server.queue_results.remove_waiting_task_id(id_task); - - json data = json::array({json{ - {"embedding", json_value(result.data, "embedding", json::array())}, - {"index", 0}, - {"object", "embedding"} - }} - ); - - json root = format_embeddings_response_oaicompat(body, data); + // get the result + server_task_result result = ctx_server.queue_results.recv(id_task); + ctx_server.queue_results.remove_waiting_task_id(id_task); + responses.push_back(result.data); + } + // write JSON response + json root; + if (is_openai) { + json res_oai = json::array(); + int i = 0; + for (auto & elem : responses) { + res_oai.push_back(json{ + {"embedding", json_value(elem, "embedding", json::array())}, + {"index", i++}, + {"object", "embedding"} + }); + } + root = format_embeddings_response_oaicompat(body, res_oai); + } else { + root = responses[0]; + } return res.set_content(root.dump(), "application/json; charset=utf-8"); - }); + }; + // + // Router + // + + // register static assets routes + if (!sparams.public_path.empty()) { + // Set the base directory for serving static files + svr->set_base_dir(sparams.public_path); + } + + // using embedded static files + auto handle_static_file = [](unsigned char * content, size_t len, const char * mime_type) { + return [content, len, mime_type](const httplib::Request &, httplib::Response & res) { + res.set_content(reinterpret_cast(content), len, mime_type); + return false; + }; + }; + + svr->Options(R"(/.*)", [](const httplib::Request &, httplib::Response & res) { + // TODO @ngxson : I have no idea what it is... maybe this is redundant? + return res.set_content("", "application/json; charset=utf-8"); + }); + svr->Get("/", handle_static_file(index_html, index_html_len, "text/html; charset=utf-8")); + svr->Get("/index.js", handle_static_file(index_js, index_js_len, "text/javascript; charset=utf-8")); + svr->Get("/completion.js", handle_static_file(completion_js, completion_js_len, "text/javascript; charset=utf-8")); + svr->Get("/json-schema-to-grammar.mjs", handle_static_file( + json_schema_to_grammar_mjs, json_schema_to_grammar_mjs_len, "text/javascript; charset=utf-8")); + + // register API routes + svr->Get ("/health", handle_health); + svr->Get ("/slots", handle_slots); + svr->Get ("/metrics", handle_metrics); + svr->Get ("/props", handle_props); + svr->Get ("/v1/models", handle_models); + svr->Post("/completion", handle_completions); // legacy + svr->Post("/completions", handle_completions); + svr->Post("/v1/completions", handle_completions); + svr->Post("/chat/completions", handle_chat_completions); + svr->Post("/v1/chat/completions", handle_chat_completions); + svr->Post("/infill", handle_infill); + svr->Post("/embedding", handle_embeddings); // legacy + svr->Post("/embeddings", handle_embeddings); + svr->Post("/v1/embeddings", handle_embeddings); + svr->Post("/tokenize", handle_tokenize); + svr->Post("/detokenize", handle_detokenize); + + // + // Start the server + // if (sparams.n_threads_http < 1) { // +2 threads for monitoring endpoints sparams.n_threads_http = std::max(params.n_parallel + 2, (int32_t) std::thread::hardware_concurrency() - 1); diff --git a/examples/server/tests/features/security.feature b/examples/server/tests/features/security.feature index 42a6709a5..1d6aa40ea 100644 --- a/examples/server/tests/features/security.feature +++ b/examples/server/tests/features/security.feature @@ -39,8 +39,9 @@ Feature: Security Scenario Outline: CORS Options - When an OPTIONS request is sent from - Then CORS header is set to + Given a user api key llama.cpp + When an OPTIONS request is sent from + Then CORS header is set to Examples: Headers | origin | cors_header | cors_header_value | diff --git a/examples/server/tests/features/steps/steps.py b/examples/server/tests/features/steps/steps.py index 0076f805b..142048509 100644 --- a/examples/server/tests/features/steps/steps.py +++ b/examples/server/tests/features/steps/steps.py @@ -582,8 +582,9 @@ async def step_detokenize(context): @async_run_until_complete async def step_options_request(context, origin): async with aiohttp.ClientSession() as session: + headers = {'Authorization': f'Bearer {context.user_api_key}', 'Origin': origin} async with session.options(f'{context.base_url}/v1/chat/completions', - headers={"Origin": origin}) as response: + headers=headers) as response: assert response.status == 200 context.options_response = response From 9674aaf35cb81478eb38c3f3ebde713ec72fbb79 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Sat, 9 Mar 2024 12:34:18 +0200 Subject: [PATCH 566/812] server : simplify logic for empty prompts (#5953) --- examples/server/server.cpp | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/examples/server/server.cpp b/examples/server/server.cpp index 6e0f8328c..aedf0afc6 100644 --- a/examples/server/server.cpp +++ b/examples/server/server.cpp @@ -1704,19 +1704,6 @@ struct server_context { // next, batch any pending prompts without exceeding n_batch if (params.cont_batching || batch.n_tokens == 0) { for (auto & slot : slots) { - const bool has_prompt = slot.prompt.is_array() || (slot.prompt.is_string() && !slot.prompt.get().empty()); - - // empty prompt passed -> release the slot and send empty response - // note: infill mode allows empty prompt - if (slot.state == SLOT_STATE_IDLE && slot.command == SLOT_COMMAND_LOAD_PROMPT && !has_prompt && !slot.infill) { - slot.state = SLOT_STATE_PROCESSING; - slot.command = SLOT_COMMAND_NONE; - slot.release(); - slot.print_timings(); - send_final_response(slot); - continue; - } - // this slot still has a prompt to be processed if (slot.state == SLOT_STATE_IDLE && slot.command == SLOT_COMMAND_LOAD_PROMPT) { auto & prompt_tokens = slot.prompt_tokens; @@ -1768,6 +1755,21 @@ struct server_context { {"prompt_tokens", tokens_to_str(ctx, prompt_tokens.cbegin(), prompt_tokens.cend())}, }); + // empty prompt passed -> release the slot and send empty response + if (prompt_tokens.empty()) { + LOG_INFO("empty prompt - releasing slot", { + {"id_slot", slot.id}, + {"id_task", slot.id_task} + }); + + slot.state = SLOT_STATE_PROCESSING; + slot.command = SLOT_COMMAND_NONE; + slot.release(); + slot.print_timings(); + send_final_response(slot); + continue; + } + if (slot.embedding) { // this prompt is too large to process - discard it if (slot.n_prompt_tokens > n_batch) { From 8a3012a4ad08112bb3dc3f1399afec4e93780c44 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Sat, 9 Mar 2024 12:47:57 +0200 Subject: [PATCH 567/812] ggml : add ggml-common.h to deduplicate shared code (#5940) * ggml : add ggml-common.h to shared code ggml-ci * scripts : update sync scripts * sycl : reuse quantum tables ggml-ci * ggml : minor * ggml : minor * sycl : try to fix build --- CMakeLists.txt | 3 +- Makefile | 4 +- ggml-common.h | 779 ++++++++++++++++++++++++++++++++++++++++ ggml-cuda.cu | 743 +------------------------------------- ggml-metal.metal | 707 +----------------------------------- ggml-quants.c | 708 +----------------------------------- ggml-quants.h | 4 +- ggml-sycl.cpp | 384 +------------------- scripts/sync-ggml-am.sh | 2 + scripts/sync-ggml.sh | 1 + 10 files changed, 799 insertions(+), 2536 deletions(-) create mode 100644 ggml-common.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 48880f720..9309ca6bb 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -199,7 +199,8 @@ if (LLAMA_METAL) # get full path to the file #add_compile_definitions(GGML_METAL_DIR_KERNELS="${CMAKE_CURRENT_SOURCE_DIR}/") - # copy ggml-metal.metal to bin directory + # copy ggml-common.h and ggml-metal.metal to bin directory + configure_file(ggml-common.h ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/ggml-common.h COPYONLY) configure_file(ggml-metal.metal ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/ggml-metal.metal COPYONLY) if (LLAMA_METAL_EMBED_LIBRARY) diff --git a/Makefile b/Makefile index aea969222..d809b8b3b 100644 --- a/Makefile +++ b/Makefile @@ -453,7 +453,7 @@ endif # LLAMA_CUDA_PEER_MAX_BATCH_SIZE ifdef LLAMA_CUDA_CCBIN MK_NVCCFLAGS += -ccbin $(LLAMA_CUDA_CCBIN) endif -ggml-cuda.o: ggml-cuda.cu ggml-cuda.h +ggml-cuda.o: ggml-cuda.cu ggml-cuda.h ggml-common.h ifdef JETSON_EOL_MODULE_DETECT $(NVCC) -I. -Icommon -D_XOPEN_SOURCE=600 -D_GNU_SOURCE -DNDEBUG -DGGML_USE_CUBLAS -I/usr/local/cuda/include -I/opt/cuda/include -I/usr/local/cuda/targets/aarch64-linux/include -std=c++11 -O3 $(NVCCFLAGS) $(CPPFLAGS) -Xcompiler "$(CUDA_CXXFLAGS)" -c $< -o $@ else @@ -630,7 +630,7 @@ ggml-alloc.o: ggml-alloc.c ggml.h ggml-alloc.h ggml-backend.o: ggml-backend.c ggml.h ggml-backend.h $(CC) $(CFLAGS) -c $< -o $@ -ggml-quants.o: ggml-quants.c ggml.h ggml-quants.h +ggml-quants.o: ggml-quants.c ggml.h ggml-quants.h ggml-common.h $(CC) $(CFLAGS) -c $< -o $@ OBJS += ggml-alloc.o ggml-backend.o ggml-quants.o diff --git a/ggml-common.h b/ggml-common.h new file mode 100644 index 000000000..4b6d248b6 --- /dev/null +++ b/ggml-common.h @@ -0,0 +1,779 @@ +#pragma once + +#if defined(GGML_COMMON_IMPL_C) +#include + +#define GGML_TABLE_BEGIN(type, name, size) static const type name[size] = { +#define GGML_TABLE_END() }; + +#define GGML_COMMON_IMPL +#elif defined(GGML_COMMON_IMPL_METAL) +#include + +#define GGML_TABLE_BEGIN(type, name, size) static const constant type name[size] = { +#define GGML_TABLE_END() }; + +#define GGML_COMMON_IMPL +#elif defined(GGML_COMMON_IMPL_CUDA) +#include + +#define GGML_TABLE_BEGIN(type, name, size) static const __device__ __constant__ type name[size] = { +#define GGML_TABLE_END() }; + +#define GGML_COMMON_IMPL +#elif defined(GGML_COMMON_IMPL_SYCL) +#include + +#define GGML_TABLE_BEGIN(type, name, size) static dpct::global_memory name(sycl::range<1>(size), { +#define GGML_TABLE_END() }); + +#define GGML_COMMON_IMPL +#endif + +#if defined(GGML_COMMON_IMPL) + +GGML_TABLE_BEGIN(uint8_t, kmask_iq2xs, 8) + 1, 2, 4, 8, 16, 32, 64, 128 +GGML_TABLE_END() + +GGML_TABLE_BEGIN(uint8_t, ksigns_iq2xs, 128) + 0, 129, 130, 3, 132, 5, 6, 135, 136, 9, 10, 139, 12, 141, 142, 15, + 144, 17, 18, 147, 20, 149, 150, 23, 24, 153, 154, 27, 156, 29, 30, 159, + 160, 33, 34, 163, 36, 165, 166, 39, 40, 169, 170, 43, 172, 45, 46, 175, + 48, 177, 178, 51, 180, 53, 54, 183, 184, 57, 58, 187, 60, 189, 190, 63, + 192, 65, 66, 195, 68, 197, 198, 71, 72, 201, 202, 75, 204, 77, 78, 207, + 80, 209, 210, 83, 212, 85, 86, 215, 216, 89, 90, 219, 92, 221, 222, 95, + 96, 225, 226, 99, 228, 101, 102, 231, 232, 105, 106, 235, 108, 237, 238, 111, + 240, 113, 114, 243, 116, 245, 246, 119, 120, 249, 250, 123, 252, 125, 126, 255, +GGML_TABLE_END() + +//#if __CUDA_ARCH__ >= MIN_CC_DP4A // lowest compute capability for integer intrinsics +GGML_TABLE_BEGIN(uint64_t, ksigns64, 128) + 0x0000000000000000, 0xff000000000000ff, 0xff0000000000ff00, 0x000000000000ffff, + 0xff00000000ff0000, 0x0000000000ff00ff, 0x0000000000ffff00, 0xff00000000ffffff, + 0xff000000ff000000, 0x00000000ff0000ff, 0x00000000ff00ff00, 0xff000000ff00ffff, + 0x00000000ffff0000, 0xff000000ffff00ff, 0xff000000ffffff00, 0x00000000ffffffff, + 0xff0000ff00000000, 0x000000ff000000ff, 0x000000ff0000ff00, 0xff0000ff0000ffff, + 0x000000ff00ff0000, 0xff0000ff00ff00ff, 0xff0000ff00ffff00, 0x000000ff00ffffff, + 0x000000ffff000000, 0xff0000ffff0000ff, 0xff0000ffff00ff00, 0x000000ffff00ffff, + 0xff0000ffffff0000, 0x000000ffffff00ff, 0x000000ffffffff00, 0xff0000ffffffffff, + 0xff00ff0000000000, 0x0000ff00000000ff, 0x0000ff000000ff00, 0xff00ff000000ffff, + 0x0000ff0000ff0000, 0xff00ff0000ff00ff, 0xff00ff0000ffff00, 0x0000ff0000ffffff, + 0x0000ff00ff000000, 0xff00ff00ff0000ff, 0xff00ff00ff00ff00, 0x0000ff00ff00ffff, + 0xff00ff00ffff0000, 0x0000ff00ffff00ff, 0x0000ff00ffffff00, 0xff00ff00ffffffff, + 0x0000ffff00000000, 0xff00ffff000000ff, 0xff00ffff0000ff00, 0x0000ffff0000ffff, + 0xff00ffff00ff0000, 0x0000ffff00ff00ff, 0x0000ffff00ffff00, 0xff00ffff00ffffff, + 0xff00ffffff000000, 0x0000ffffff0000ff, 0x0000ffffff00ff00, 0xff00ffffff00ffff, + 0x0000ffffffff0000, 0xff00ffffffff00ff, 0xff00ffffffffff00, 0x0000ffffffffffff, + 0xffff000000000000, 0x00ff0000000000ff, 0x00ff00000000ff00, 0xffff00000000ffff, + 0x00ff000000ff0000, 0xffff000000ff00ff, 0xffff000000ffff00, 0x00ff000000ffffff, + 0x00ff0000ff000000, 0xffff0000ff0000ff, 0xffff0000ff00ff00, 0x00ff0000ff00ffff, + 0xffff0000ffff0000, 0x00ff0000ffff00ff, 0x00ff0000ffffff00, 0xffff0000ffffffff, + 0x00ff00ff00000000, 0xffff00ff000000ff, 0xffff00ff0000ff00, 0x00ff00ff0000ffff, + 0xffff00ff00ff0000, 0x00ff00ff00ff00ff, 0x00ff00ff00ffff00, 0xffff00ff00ffffff, + 0xffff00ffff000000, 0x00ff00ffff0000ff, 0x00ff00ffff00ff00, 0xffff00ffff00ffff, + 0x00ff00ffffff0000, 0xffff00ffffff00ff, 0xffff00ffffffff00, 0x00ff00ffffffffff, + 0x00ffff0000000000, 0xffffff00000000ff, 0xffffff000000ff00, 0x00ffff000000ffff, + 0xffffff0000ff0000, 0x00ffff0000ff00ff, 0x00ffff0000ffff00, 0xffffff0000ffffff, + 0xffffff00ff000000, 0x00ffff00ff0000ff, 0x00ffff00ff00ff00, 0xffffff00ff00ffff, + 0x00ffff00ffff0000, 0xffffff00ffff00ff, 0xffffff00ffffff00, 0x00ffff00ffffffff, + 0xffffffff00000000, 0x00ffffff000000ff, 0x00ffffff0000ff00, 0xffffffff0000ffff, + 0x00ffffff00ff0000, 0xffffffff00ff00ff, 0xffffffff00ffff00, 0x00ffffff00ffffff, + 0x00ffffffff000000, 0xffffffffff0000ff, 0xffffffffff00ff00, 0x00ffffffff00ffff, + 0xffffffffffff0000, 0x00ffffffffff00ff, 0x00ffffffffffff00, 0xffffffffffffffff, +GGML_TABLE_END() +//#endif + + +GGML_TABLE_BEGIN(uint64_t, iq2xxs_grid, 256) + 0x0808080808080808, 0x080808080808082b, 0x0808080808081919, 0x0808080808082b08, + 0x0808080808082b2b, 0x0808080808190819, 0x0808080808191908, 0x08080808082b0808, + 0x08080808082b082b, 0x08080808082b2b08, 0x08080808082b2b2b, 0x0808080819080819, + 0x0808080819081908, 0x0808080819190808, 0x0808080819192b08, 0x08080808192b0819, + 0x08080808192b1908, 0x080808082b080808, 0x080808082b08082b, 0x080808082b082b2b, + 0x080808082b2b082b, 0x0808081908080819, 0x0808081908081908, 0x0808081908190808, + 0x0808081908191919, 0x0808081919080808, 0x080808192b081908, 0x080808192b192b08, + 0x0808082b08080808, 0x0808082b0808082b, 0x0808082b082b082b, 0x0808082b2b08082b, + 0x0808190808080819, 0x0808190808081908, 0x0808190808190808, 0x08081908082b0819, + 0x08081908082b1908, 0x0808190819080808, 0x080819081908082b, 0x0808190819082b08, + 0x08081908192b0808, 0x080819082b080819, 0x080819082b081908, 0x080819082b190808, + 0x080819082b2b1908, 0x0808191908080808, 0x080819190808082b, 0x0808191908082b08, + 0x08081919082b0808, 0x080819191908192b, 0x08081919192b2b19, 0x080819192b080808, + 0x080819192b190819, 0x0808192b08082b19, 0x0808192b08190808, 0x0808192b19080808, + 0x0808192b2b081908, 0x0808192b2b2b1908, 0x08082b0808080808, 0x08082b0808081919, + 0x08082b0808082b08, 0x08082b0808191908, 0x08082b08082b2b08, 0x08082b0819080819, + 0x08082b0819081908, 0x08082b0819190808, 0x08082b081919082b, 0x08082b082b082b08, + 0x08082b1908081908, 0x08082b1919080808, 0x08082b2b0808082b, 0x08082b2b08191908, + 0x0819080808080819, 0x0819080808081908, 0x0819080808190808, 0x08190808082b0819, + 0x0819080819080808, 0x08190808192b0808, 0x081908082b081908, 0x081908082b190808, + 0x081908082b191919, 0x0819081908080808, 0x0819081908082b08, 0x08190819082b0808, + 0x0819081919190808, 0x0819081919192b2b, 0x081908192b080808, 0x0819082b082b1908, + 0x0819082b19081919, 0x0819190808080808, 0x0819190808082b08, 0x08191908082b0808, + 0x08191908082b1919, 0x0819190819082b19, 0x081919082b080808, 0x0819191908192b08, + 0x08191919192b082b, 0x0819192b08080808, 0x0819192b0819192b, 0x08192b0808080819, + 0x08192b0808081908, 0x08192b0808190808, 0x08192b0819080808, 0x08192b082b080819, + 0x08192b1908080808, 0x08192b1908081919, 0x08192b192b2b0808, 0x08192b2b19190819, + 0x082b080808080808, 0x082b08080808082b, 0x082b080808082b2b, 0x082b080819081908, + 0x082b0808192b0819, 0x082b08082b080808, 0x082b08082b08082b, 0x082b0819082b2b19, + 0x082b081919082b08, 0x082b082b08080808, 0x082b082b0808082b, 0x082b190808080819, + 0x082b190808081908, 0x082b190808190808, 0x082b190819080808, 0x082b19081919192b, + 0x082b191908080808, 0x082b191919080819, 0x082b1919192b1908, 0x082b192b2b190808, + 0x082b2b0808082b08, 0x082b2b08082b0808, 0x082b2b082b191908, 0x082b2b2b19081908, + 0x1908080808080819, 0x1908080808081908, 0x1908080808190808, 0x1908080808192b08, + 0x19080808082b0819, 0x19080808082b1908, 0x1908080819080808, 0x1908080819082b08, + 0x190808081919192b, 0x19080808192b0808, 0x190808082b080819, 0x190808082b081908, + 0x190808082b190808, 0x1908081908080808, 0x19080819082b0808, 0x19080819192b0819, + 0x190808192b080808, 0x190808192b081919, 0x1908082b08080819, 0x1908082b08190808, + 0x1908082b19082b08, 0x1908082b1919192b, 0x1908082b192b2b08, 0x1908190808080808, + 0x1908190808082b08, 0x19081908082b0808, 0x190819082b080808, 0x190819082b192b19, + 0x190819190819082b, 0x19081919082b1908, 0x1908192b08080808, 0x19082b0808080819, + 0x19082b0808081908, 0x19082b0808190808, 0x19082b0819080808, 0x19082b0819081919, + 0x19082b1908080808, 0x19082b1919192b08, 0x19082b19192b0819, 0x19082b192b08082b, + 0x19082b2b19081919, 0x19082b2b2b190808, 0x1919080808080808, 0x1919080808082b08, + 0x1919080808190819, 0x1919080808192b19, 0x19190808082b0808, 0x191908082b080808, + 0x191908082b082b08, 0x1919081908081908, 0x191908191908082b, 0x191908192b2b1908, + 0x1919082b2b190819, 0x191919082b190808, 0x191919082b19082b, 0x1919191908082b2b, + 0x1919192b08080819, 0x1919192b19191908, 0x19192b0808080808, 0x19192b0808190819, + 0x19192b0808192b19, 0x19192b08192b1908, 0x19192b1919080808, 0x19192b2b08082b08, + 0x192b080808081908, 0x192b080808190808, 0x192b080819080808, 0x192b0808192b2b08, + 0x192b081908080808, 0x192b081919191919, 0x192b082b08192b08, 0x192b082b192b0808, + 0x192b190808080808, 0x192b190808081919, 0x192b191908190808, 0x192b19190819082b, + 0x192b19192b081908, 0x192b2b081908082b, 0x2b08080808080808, 0x2b0808080808082b, + 0x2b08080808082b2b, 0x2b08080819080819, 0x2b0808082b08082b, 0x2b08081908081908, + 0x2b08081908192b08, 0x2b08081919080808, 0x2b08082b08190819, 0x2b08190808080819, + 0x2b08190808081908, 0x2b08190808190808, 0x2b08190808191919, 0x2b08190819080808, + 0x2b081908192b0808, 0x2b08191908080808, 0x2b0819191908192b, 0x2b0819192b191908, + 0x2b08192b08082b19, 0x2b08192b19080808, 0x2b08192b192b0808, 0x2b082b080808082b, + 0x2b082b1908081908, 0x2b082b2b08190819, 0x2b19080808081908, 0x2b19080808190808, + 0x2b190808082b1908, 0x2b19080819080808, 0x2b1908082b2b0819, 0x2b1908190819192b, + 0x2b1908192b080808, 0x2b19082b19081919, 0x2b19190808080808, 0x2b191908082b082b, + 0x2b19190819081908, 0x2b19191919190819, 0x2b192b082b080819, 0x2b192b19082b0808, + 0x2b2b08080808082b, 0x2b2b080819190808, 0x2b2b08082b081919, 0x2b2b081908082b19, + 0x2b2b082b08080808, 0x2b2b190808192b08, 0x2b2b2b0819190808, 0x2b2b2b1908081908, +GGML_TABLE_END() + +GGML_TABLE_BEGIN(uint64_t, iq2xs_grid, 512) + 0x0808080808080808, 0x080808080808082b, 0x0808080808081919, 0x0808080808082b08, + 0x0808080808082b2b, 0x0808080808190819, 0x0808080808191908, 0x080808080819192b, + 0x0808080808192b19, 0x08080808082b0808, 0x08080808082b082b, 0x08080808082b1919, + 0x08080808082b2b08, 0x0808080819080819, 0x0808080819081908, 0x080808081908192b, + 0x0808080819082b19, 0x0808080819190808, 0x080808081919082b, 0x0808080819191919, + 0x0808080819192b08, 0x08080808192b0819, 0x08080808192b1908, 0x080808082b080808, + 0x080808082b08082b, 0x080808082b081919, 0x080808082b082b08, 0x080808082b190819, + 0x080808082b191908, 0x080808082b192b19, 0x080808082b2b0808, 0x0808081908080819, + 0x0808081908081908, 0x080808190808192b, 0x0808081908082b19, 0x0808081908190808, + 0x080808190819082b, 0x0808081908191919, 0x0808081908192b08, 0x0808081908192b2b, + 0x08080819082b0819, 0x08080819082b1908, 0x0808081919080808, 0x080808191908082b, + 0x0808081919081919, 0x0808081919082b08, 0x0808081919190819, 0x0808081919191908, + 0x08080819192b0808, 0x08080819192b2b08, 0x080808192b080819, 0x080808192b081908, + 0x080808192b190808, 0x0808082b08080808, 0x0808082b0808082b, 0x0808082b08081919, + 0x0808082b08082b08, 0x0808082b08190819, 0x0808082b08191908, 0x0808082b082b0808, + 0x0808082b19080819, 0x0808082b19081908, 0x0808082b19190808, 0x0808082b19191919, + 0x0808082b2b080808, 0x0808082b2b082b2b, 0x0808190808080819, 0x0808190808081908, + 0x080819080808192b, 0x0808190808082b19, 0x0808190808190808, 0x080819080819082b, + 0x0808190808191919, 0x0808190808192b08, 0x08081908082b0819, 0x08081908082b1908, + 0x0808190819080808, 0x080819081908082b, 0x0808190819081919, 0x0808190819082b08, + 0x0808190819190819, 0x0808190819191908, 0x080819081919192b, 0x08081908192b0808, + 0x080819082b080819, 0x080819082b081908, 0x080819082b190808, 0x0808191908080808, + 0x080819190808082b, 0x0808191908081919, 0x0808191908082b08, 0x0808191908190819, + 0x0808191908191908, 0x08081919082b0808, 0x0808191919080819, 0x0808191919081908, + 0x0808191919190808, 0x08081919192b0819, 0x080819192b080808, 0x0808192b08080819, + 0x0808192b08081908, 0x0808192b08190808, 0x0808192b082b192b, 0x0808192b19080808, + 0x0808192b1908082b, 0x0808192b2b081908, 0x08082b0808080808, 0x08082b080808082b, + 0x08082b0808081919, 0x08082b0808082b08, 0x08082b0808082b2b, 0x08082b0808190819, + 0x08082b0808191908, 0x08082b08082b0808, 0x08082b08082b1919, 0x08082b0819080819, + 0x08082b0819081908, 0x08082b0819190808, 0x08082b0819192b08, 0x08082b082b080808, + 0x08082b082b2b0808, 0x08082b082b2b2b2b, 0x08082b1908080819, 0x08082b1908081908, + 0x08082b1908190808, 0x08082b1919080808, 0x08082b192b080819, 0x08082b192b082b19, + 0x08082b2b08080808, 0x08082b2b082b0808, 0x08082b2b082b2b08, 0x08082b2b2b19192b, + 0x08082b2b2b2b0808, 0x0819080808080819, 0x0819080808081908, 0x081908080808192b, + 0x0819080808082b19, 0x0819080808190808, 0x081908080819082b, 0x0819080808191919, + 0x0819080808192b08, 0x08190808082b0819, 0x08190808082b1908, 0x0819080819080808, + 0x081908081908082b, 0x0819080819081919, 0x0819080819082b08, 0x0819080819190819, + 0x0819080819191908, 0x08190808192b0808, 0x08190808192b2b2b, 0x081908082b080819, + 0x081908082b081908, 0x081908082b190808, 0x0819081908080808, 0x081908190808082b, + 0x0819081908081919, 0x0819081908082b08, 0x0819081908190819, 0x0819081908191908, + 0x08190819082b0808, 0x0819081919080819, 0x0819081919081908, 0x0819081919190808, + 0x081908192b080808, 0x081908192b191908, 0x081908192b19192b, 0x0819082b08080819, + 0x0819082b08081908, 0x0819082b0808192b, 0x0819082b08190808, 0x0819082b19080808, + 0x0819082b192b0808, 0x0819190808080808, 0x081919080808082b, 0x0819190808081919, + 0x0819190808082b08, 0x0819190808190819, 0x0819190808191908, 0x08191908082b0808, + 0x0819190819080819, 0x0819190819081908, 0x0819190819082b19, 0x0819190819190808, + 0x08191908192b1908, 0x081919082b080808, 0x0819191908080819, 0x0819191908081908, + 0x0819191908190808, 0x0819191919080808, 0x0819192b08080808, 0x0819192b08191908, + 0x0819192b19082b19, 0x08192b0808080819, 0x08192b0808081908, 0x08192b0808190808, + 0x08192b080819082b, 0x08192b0819080808, 0x08192b0819191908, 0x08192b082b08192b, + 0x08192b1908080808, 0x08192b1908081919, 0x08192b19192b192b, 0x08192b2b19190819, + 0x08192b2b2b2b2b19, 0x082b080808080808, 0x082b08080808082b, 0x082b080808081919, + 0x082b080808082b08, 0x082b080808082b2b, 0x082b080808190819, 0x082b080808191908, + 0x082b0808082b0808, 0x082b080819080819, 0x082b080819081908, 0x082b080819190808, + 0x082b08082b080808, 0x082b08082b2b0808, 0x082b081908080819, 0x082b081908081908, + 0x082b081908190808, 0x082b081919080808, 0x082b081919082b08, 0x082b0819192b1919, + 0x082b082b08080808, 0x082b082b082b082b, 0x082b082b2b080808, 0x082b082b2b2b2b08, + 0x082b190808080819, 0x082b190808081908, 0x082b190808190808, 0x082b1908082b2b19, + 0x082b190819080808, 0x082b191908080808, 0x082b191919080819, 0x082b19191919082b, + 0x082b19192b192b19, 0x082b192b08080819, 0x082b192b08192b2b, 0x082b192b2b2b192b, + 0x082b2b0808080808, 0x082b2b0808082b08, 0x082b2b0808082b2b, 0x082b2b08082b0808, + 0x082b2b0819191919, 0x082b2b082b082b08, 0x082b2b082b2b082b, 0x082b2b19192b2b08, + 0x082b2b192b190808, 0x082b2b2b08082b08, 0x082b2b2b082b0808, 0x082b2b2b2b08082b, + 0x082b2b2b2b082b08, 0x082b2b2b2b082b2b, 0x1908080808080819, 0x1908080808081908, + 0x190808080808192b, 0x1908080808082b19, 0x1908080808190808, 0x190808080819082b, + 0x1908080808191919, 0x1908080808192b08, 0x19080808082b0819, 0x19080808082b1908, + 0x1908080819080808, 0x190808081908082b, 0x1908080819081919, 0x1908080819082b08, + 0x1908080819082b2b, 0x1908080819190819, 0x1908080819191908, 0x19080808192b0808, + 0x19080808192b1919, 0x190808082b080819, 0x190808082b081908, 0x190808082b190808, + 0x1908081908080808, 0x190808190808082b, 0x1908081908081919, 0x1908081908082b08, + 0x1908081908190819, 0x1908081908191908, 0x19080819082b0808, 0x1908081919080819, + 0x1908081919081908, 0x1908081919190808, 0x190808192b080808, 0x190808192b081919, + 0x190808192b2b082b, 0x1908082b08080819, 0x1908082b08081908, 0x1908082b08190808, + 0x1908082b0819082b, 0x1908082b082b2b19, 0x1908082b19080808, 0x1908190808080808, + 0x190819080808082b, 0x1908190808081919, 0x1908190808082b08, 0x1908190808190819, + 0x1908190808191908, 0x1908190808192b19, 0x19081908082b0808, 0x1908190819080819, + 0x1908190819081908, 0x1908190819190808, 0x190819082b080808, 0x190819082b191908, + 0x1908191908080819, 0x1908191908081908, 0x1908191908190808, 0x19081919082b1908, + 0x1908191919080808, 0x190819192b192b2b, 0x1908192b08080808, 0x1908192b08082b2b, + 0x1908192b19081908, 0x1908192b19190808, 0x19082b0808080819, 0x19082b0808081908, + 0x19082b0808190808, 0x19082b0819080808, 0x19082b0819081919, 0x19082b0819191908, + 0x19082b08192b082b, 0x19082b1908080808, 0x19082b1908190819, 0x19082b1919081908, + 0x19082b1919190808, 0x19082b19192b2b19, 0x19082b2b08081908, 0x1919080808080808, + 0x191908080808082b, 0x1919080808081919, 0x1919080808082b08, 0x1919080808190819, + 0x1919080808191908, 0x19190808082b0808, 0x19190808082b2b08, 0x1919080819080819, + 0x1919080819081908, 0x1919080819190808, 0x191908082b080808, 0x1919081908080819, + 0x1919081908081908, 0x1919081908190808, 0x1919081908191919, 0x1919081919080808, + 0x191908191908082b, 0x1919082b08080808, 0x1919082b19081908, 0x1919082b2b2b2b2b, + 0x1919190808080819, 0x1919190808081908, 0x1919190808190808, 0x19191908082b0819, + 0x1919190819080808, 0x19191908192b0808, 0x191919082b080819, 0x191919082b2b0819, + 0x1919191908080808, 0x1919191908082b08, 0x191919192b080808, 0x191919192b082b08, + 0x1919192b082b0819, 0x1919192b192b2b08, 0x1919192b2b2b0819, 0x19192b0808080808, + 0x19192b0808191908, 0x19192b0819080819, 0x19192b0819190808, 0x19192b082b192b19, + 0x19192b1908192b2b, 0x19192b1919080808, 0x19192b191908082b, 0x19192b2b2b081919, + 0x192b080808080819, 0x192b080808081908, 0x192b080808190808, 0x192b080819080808, + 0x192b080819191908, 0x192b0808192b082b, 0x192b08082b08192b, 0x192b08082b2b2b19, + 0x192b081908080808, 0x192b082b082b1908, 0x192b082b19082b2b, 0x192b082b2b19082b, + 0x192b190808080808, 0x192b19080819192b, 0x192b191908190808, 0x192b191919080808, + 0x192b191919081919, 0x192b19192b2b1908, 0x192b2b0808080819, 0x192b2b08192b2b2b, + 0x192b2b19082b1919, 0x192b2b2b0808192b, 0x192b2b2b19191908, 0x192b2b2b192b082b, + 0x2b08080808080808, 0x2b0808080808082b, 0x2b08080808081919, 0x2b08080808082b08, + 0x2b08080808190819, 0x2b08080808191908, 0x2b080808082b0808, 0x2b080808082b2b2b, + 0x2b08080819080819, 0x2b08080819081908, 0x2b08080819190808, 0x2b0808082b080808, + 0x2b0808082b08082b, 0x2b0808082b2b2b08, 0x2b0808082b2b2b2b, 0x2b08081908080819, + 0x2b08081908081908, 0x2b0808190808192b, 0x2b08081908190808, 0x2b08081919080808, + 0x2b08081919190819, 0x2b08081919192b19, 0x2b08082b08080808, 0x2b08082b082b0808, + 0x2b08082b2b080808, 0x2b08082b2b08082b, 0x2b08082b2b2b0808, 0x2b08082b2b2b2b08, + 0x2b08190808080819, 0x2b08190808081908, 0x2b08190808190808, 0x2b0819080819082b, + 0x2b08190808191919, 0x2b08190819080808, 0x2b081908192b0808, 0x2b0819082b082b19, + 0x2b08191908080808, 0x2b08191919081908, 0x2b0819192b2b1919, 0x2b08192b08192b08, + 0x2b08192b192b2b2b, 0x2b082b0808080808, 0x2b082b0808082b08, 0x2b082b08082b1919, + 0x2b082b0819192b2b, 0x2b082b082b080808, 0x2b082b082b08082b, 0x2b082b082b2b2b08, + 0x2b082b190808192b, 0x2b082b2b082b082b, 0x2b082b2b2b080808, 0x2b082b2b2b082b08, + 0x2b082b2b2b19192b, 0x2b082b2b2b2b2b08, 0x2b19080808080819, 0x2b19080808081908, + 0x2b19080808190808, 0x2b19080819080808, 0x2b1908081919192b, 0x2b1908082b081908, + 0x2b19081908080808, 0x2b190819082b082b, 0x2b190819192b1908, 0x2b19082b1919192b, + 0x2b19082b2b082b19, 0x2b19190808080808, 0x2b19190808081919, 0x2b19190819081908, + 0x2b19190819190808, 0x2b19190819192b08, 0x2b191919082b2b19, 0x2b1919192b190808, + 0x2b1919192b19082b, 0x2b19192b19080819, 0x2b192b0819190819, 0x2b192b082b2b192b, + 0x2b192b1919082b19, 0x2b192b2b08191919, 0x2b192b2b192b0808, 0x2b2b080808080808, + 0x2b2b08080808082b, 0x2b2b080808082b08, 0x2b2b080808082b2b, 0x2b2b0808082b0808, + 0x2b2b0808082b2b2b, 0x2b2b08082b2b0808, 0x2b2b081919190819, 0x2b2b081919192b19, + 0x2b2b08192b2b192b, 0x2b2b082b08080808, 0x2b2b082b0808082b, 0x2b2b082b08082b08, + 0x2b2b082b082b2b2b, 0x2b2b082b2b080808, 0x2b2b082b2b2b0808, 0x2b2b190819080808, + 0x2b2b19082b191919, 0x2b2b192b192b1919, 0x2b2b192b2b192b08, 0x2b2b2b0808082b2b, + 0x2b2b2b08082b0808, 0x2b2b2b08082b082b, 0x2b2b2b08082b2b08, 0x2b2b2b082b2b0808, + 0x2b2b2b082b2b2b08, 0x2b2b2b1908081908, 0x2b2b2b192b081908, 0x2b2b2b192b08192b, + 0x2b2b2b2b082b2b08, 0x2b2b2b2b082b2b2b, 0x2b2b2b2b2b190819, 0x2b2b2b2b2b2b2b2b, +GGML_TABLE_END() + +GGML_TABLE_BEGIN(uint64_t, iq2s_grid, 1024) + 0x0808080808080808, 0x080808080808082b, 0x0808080808081919, 0x0808080808082b08, + 0x0808080808082b2b, 0x0808080808190819, 0x0808080808191908, 0x080808080819192b, + 0x0808080808192b19, 0x08080808082b0808, 0x08080808082b082b, 0x08080808082b1919, + 0x08080808082b2b08, 0x0808080819080819, 0x0808080819081908, 0x080808081908192b, + 0x0808080819082b19, 0x0808080819190808, 0x080808081919082b, 0x0808080819191919, + 0x0808080819192b08, 0x08080808192b0819, 0x08080808192b1908, 0x08080808192b192b, + 0x08080808192b2b19, 0x080808082b080808, 0x080808082b08082b, 0x080808082b081919, + 0x080808082b082b08, 0x080808082b190819, 0x080808082b191908, 0x080808082b2b0808, + 0x080808082b2b1919, 0x080808082b2b2b2b, 0x0808081908080819, 0x0808081908081908, + 0x080808190808192b, 0x0808081908082b19, 0x0808081908190808, 0x080808190819082b, + 0x0808081908191919, 0x0808081908192b08, 0x08080819082b0819, 0x08080819082b1908, + 0x0808081919080808, 0x080808191908082b, 0x0808081919081919, 0x0808081919082b08, + 0x0808081919190819, 0x0808081919191908, 0x080808191919192b, 0x0808081919192b19, + 0x08080819192b0808, 0x08080819192b1919, 0x08080819192b2b08, 0x080808192b080819, + 0x080808192b081908, 0x080808192b190808, 0x080808192b19082b, 0x080808192b191919, + 0x080808192b2b0819, 0x080808192b2b1908, 0x0808082b08080808, 0x0808082b0808082b, + 0x0808082b08081919, 0x0808082b08082b08, 0x0808082b08190819, 0x0808082b08191908, + 0x0808082b082b0808, 0x0808082b082b2b2b, 0x0808082b19080819, 0x0808082b19081908, + 0x0808082b1908192b, 0x0808082b19082b19, 0x0808082b19190808, 0x0808082b19191919, + 0x0808082b2b080808, 0x0808082b2b081919, 0x0808082b2b082b2b, 0x0808082b2b191908, + 0x0808082b2b2b082b, 0x0808190808080819, 0x0808190808081908, 0x080819080808192b, + 0x0808190808082b19, 0x0808190808190808, 0x080819080819082b, 0x0808190808191919, + 0x0808190808192b08, 0x08081908082b0819, 0x08081908082b1908, 0x08081908082b192b, + 0x08081908082b2b19, 0x0808190819080808, 0x080819081908082b, 0x0808190819081919, + 0x0808190819082b08, 0x0808190819082b2b, 0x0808190819190819, 0x0808190819191908, + 0x080819081919192b, 0x0808190819192b19, 0x08081908192b0808, 0x08081908192b082b, + 0x08081908192b1919, 0x080819082b080819, 0x080819082b081908, 0x080819082b08192b, + 0x080819082b082b19, 0x080819082b190808, 0x080819082b191919, 0x080819082b192b08, + 0x080819082b2b0819, 0x080819082b2b1908, 0x0808191908080808, 0x080819190808082b, + 0x0808191908081919, 0x0808191908082b08, 0x0808191908082b2b, 0x0808191908190819, + 0x0808191908191908, 0x080819190819192b, 0x0808191908192b19, 0x08081919082b0808, + 0x08081919082b1919, 0x08081919082b2b08, 0x0808191919080819, 0x0808191919081908, + 0x080819191908192b, 0x0808191919082b19, 0x0808191919190808, 0x080819191919082b, + 0x0808191919191919, 0x0808191919192b08, 0x08081919192b0819, 0x08081919192b1908, + 0x080819192b080808, 0x080819192b08082b, 0x080819192b081919, 0x080819192b082b08, + 0x080819192b190819, 0x080819192b191908, 0x080819192b2b0808, 0x0808192b08080819, + 0x0808192b08081908, 0x0808192b0808192b, 0x0808192b08082b19, 0x0808192b08190808, + 0x0808192b08191919, 0x0808192b19080808, 0x0808192b19081919, 0x0808192b19082b08, + 0x0808192b19190819, 0x0808192b19191908, 0x0808192b192b0808, 0x0808192b2b080819, + 0x0808192b2b081908, 0x0808192b2b190808, 0x08082b0808080808, 0x08082b080808082b, + 0x08082b0808081919, 0x08082b0808082b08, 0x08082b0808190819, 0x08082b0808191908, + 0x08082b080819192b, 0x08082b0808192b19, 0x08082b08082b0808, 0x08082b08082b1919, + 0x08082b08082b2b2b, 0x08082b0819080819, 0x08082b0819081908, 0x08082b081908192b, + 0x08082b0819082b19, 0x08082b0819190808, 0x08082b081919082b, 0x08082b0819191919, + 0x08082b0819192b08, 0x08082b08192b0819, 0x08082b08192b1908, 0x08082b082b080808, + 0x08082b082b081919, 0x08082b082b191908, 0x08082b082b2b2b2b, 0x08082b1908080819, + 0x08082b1908081908, 0x08082b1908190808, 0x08082b190819082b, 0x08082b1908191919, + 0x08082b1908192b08, 0x08082b19082b0819, 0x08082b1919080808, 0x08082b1919081919, + 0x08082b1919082b08, 0x08082b1919190819, 0x08082b1919191908, 0x08082b19192b0808, + 0x08082b192b080819, 0x08082b192b190808, 0x08082b2b08080808, 0x08082b2b08190819, + 0x08082b2b08191908, 0x08082b2b082b082b, 0x08082b2b082b2b08, 0x08082b2b082b2b2b, + 0x08082b2b19190808, 0x08082b2b2b192b19, 0x0819080808080819, 0x0819080808081908, + 0x081908080808192b, 0x0819080808082b19, 0x0819080808190808, 0x081908080819082b, + 0x0819080808191919, 0x0819080808192b08, 0x08190808082b0819, 0x08190808082b1908, + 0x08190808082b192b, 0x0819080819080808, 0x081908081908082b, 0x0819080819081919, + 0x0819080819082b08, 0x0819080819190819, 0x0819080819191908, 0x081908081919192b, + 0x0819080819192b19, 0x08190808192b0808, 0x08190808192b082b, 0x08190808192b1919, + 0x08190808192b2b08, 0x081908082b080819, 0x081908082b081908, 0x081908082b08192b, + 0x081908082b190808, 0x081908082b191919, 0x081908082b192b08, 0x081908082b2b0819, + 0x081908082b2b1908, 0x0819081908080808, 0x081908190808082b, 0x0819081908081919, + 0x0819081908082b08, 0x0819081908082b2b, 0x0819081908190819, 0x0819081908191908, + 0x081908190819192b, 0x0819081908192b19, 0x08190819082b0808, 0x08190819082b082b, + 0x08190819082b1919, 0x08190819082b2b08, 0x0819081919080819, 0x0819081919081908, + 0x081908191908192b, 0x0819081919082b19, 0x0819081919190808, 0x081908191919082b, + 0x0819081919191919, 0x0819081919192b08, 0x08190819192b0819, 0x08190819192b1908, + 0x081908192b080808, 0x081908192b08082b, 0x081908192b081919, 0x081908192b082b08, + 0x081908192b190819, 0x081908192b191908, 0x0819082b08080819, 0x0819082b08081908, + 0x0819082b08082b19, 0x0819082b08190808, 0x0819082b08191919, 0x0819082b082b0819, + 0x0819082b082b1908, 0x0819082b19080808, 0x0819082b19081919, 0x0819082b19190819, + 0x0819082b19191908, 0x0819082b2b080819, 0x0819082b2b081908, 0x0819082b2b190808, + 0x0819190808080808, 0x081919080808082b, 0x0819190808081919, 0x0819190808082b08, + 0x0819190808190819, 0x0819190808191908, 0x081919080819192b, 0x0819190808192b19, + 0x08191908082b0808, 0x08191908082b1919, 0x08191908082b2b08, 0x0819190819080819, + 0x0819190819081908, 0x081919081908192b, 0x0819190819082b19, 0x0819190819190808, + 0x081919081919082b, 0x0819190819191919, 0x0819190819192b08, 0x08191908192b0819, + 0x08191908192b1908, 0x081919082b080808, 0x081919082b08082b, 0x081919082b081919, + 0x081919082b082b08, 0x081919082b190819, 0x081919082b191908, 0x081919082b2b0808, + 0x0819191908080819, 0x0819191908081908, 0x081919190808192b, 0x0819191908082b19, + 0x0819191908190808, 0x081919190819082b, 0x0819191908191919, 0x0819191908192b08, + 0x08191919082b0819, 0x08191919082b1908, 0x0819191919080808, 0x081919191908082b, + 0x0819191919081919, 0x0819191919082b08, 0x0819191919190819, 0x0819191919191908, + 0x08191919192b0808, 0x081919192b080819, 0x081919192b081908, 0x081919192b190808, + 0x0819192b08080808, 0x0819192b08081919, 0x0819192b08082b08, 0x0819192b08190819, + 0x0819192b08191908, 0x0819192b082b0808, 0x0819192b19080819, 0x0819192b19081908, + 0x0819192b19190808, 0x0819192b2b080808, 0x0819192b2b2b2b2b, 0x08192b0808080819, + 0x08192b0808081908, 0x08192b080808192b, 0x08192b0808082b19, 0x08192b0808190808, + 0x08192b0808191919, 0x08192b0808192b08, 0x08192b08082b0819, 0x08192b0819080808, + 0x08192b081908082b, 0x08192b0819081919, 0x08192b0819082b08, 0x08192b0819190819, + 0x08192b0819191908, 0x08192b08192b0808, 0x08192b082b080819, 0x08192b082b081908, + 0x08192b1908080808, 0x08192b190808082b, 0x08192b1908081919, 0x08192b1908082b08, + 0x08192b1908190819, 0x08192b1908191908, 0x08192b19082b0808, 0x08192b1919080819, + 0x08192b1919081908, 0x08192b1919190808, 0x08192b19192b2b19, 0x08192b192b2b082b, + 0x08192b2b08081908, 0x08192b2b08190808, 0x08192b2b19080808, 0x08192b2b1919192b, + 0x082b080808080808, 0x082b08080808082b, 0x082b080808081919, 0x082b080808082b08, + 0x082b080808190819, 0x082b080808191908, 0x082b08080819192b, 0x082b080808192b19, + 0x082b0808082b0808, 0x082b0808082b1919, 0x082b0808082b2b2b, 0x082b080819080819, + 0x082b080819081908, 0x082b080819190808, 0x082b08081919082b, 0x082b080819191919, + 0x082b0808192b1908, 0x082b08082b080808, 0x082b08082b082b2b, 0x082b08082b191908, + 0x082b08082b2b2b2b, 0x082b081908080819, 0x082b081908081908, 0x082b081908190808, + 0x082b08190819082b, 0x082b081908191919, 0x082b0819082b0819, 0x082b081919080808, + 0x082b08191908082b, 0x082b081919081919, 0x082b081919190819, 0x082b081919191908, + 0x082b0819192b0808, 0x082b08192b080819, 0x082b08192b081908, 0x082b08192b190808, + 0x082b082b08080808, 0x082b082b08082b2b, 0x082b082b082b082b, 0x082b082b082b2b08, + 0x082b082b082b2b2b, 0x082b082b19081908, 0x082b082b19190808, 0x082b082b2b082b08, + 0x082b082b2b082b2b, 0x082b082b2b2b2b08, 0x082b190808080819, 0x082b190808081908, + 0x082b19080808192b, 0x082b190808082b19, 0x082b190808190808, 0x082b190808191919, + 0x082b190808192b08, 0x082b1908082b0819, 0x082b1908082b1908, 0x082b190819080808, + 0x082b19081908082b, 0x082b190819081919, 0x082b190819082b08, 0x082b190819190819, + 0x082b190819191908, 0x082b1908192b0808, 0x082b19082b080819, 0x082b19082b081908, + 0x082b19082b190808, 0x082b191908080808, 0x082b191908081919, 0x082b191908082b08, + 0x082b191908190819, 0x082b191908191908, 0x082b1919082b0808, 0x082b191919080819, + 0x082b191919081908, 0x082b191919190808, 0x082b1919192b192b, 0x082b19192b080808, + 0x082b192b08080819, 0x082b192b08081908, 0x082b192b08190808, 0x082b192b19080808, + 0x082b192b19192b19, 0x082b2b0808080808, 0x082b2b0808081919, 0x082b2b0808190819, + 0x082b2b0808191908, 0x082b2b0819080819, 0x082b2b0819081908, 0x082b2b0819190808, + 0x082b2b082b082b2b, 0x082b2b082b2b2b2b, 0x082b2b1908080819, 0x082b2b1908081908, + 0x082b2b1908190808, 0x082b2b192b191919, 0x082b2b2b08082b2b, 0x082b2b2b082b082b, + 0x082b2b2b192b1908, 0x082b2b2b2b082b08, 0x082b2b2b2b082b2b, 0x1908080808080819, + 0x1908080808081908, 0x190808080808192b, 0x1908080808082b19, 0x1908080808190808, + 0x190808080819082b, 0x1908080808191919, 0x1908080808192b08, 0x1908080808192b2b, + 0x19080808082b0819, 0x19080808082b1908, 0x19080808082b192b, 0x1908080819080808, + 0x190808081908082b, 0x1908080819081919, 0x1908080819082b08, 0x1908080819082b2b, + 0x1908080819190819, 0x1908080819191908, 0x190808081919192b, 0x1908080819192b19, + 0x19080808192b0808, 0x19080808192b082b, 0x19080808192b1919, 0x190808082b080819, + 0x190808082b081908, 0x190808082b190808, 0x190808082b191919, 0x190808082b192b08, + 0x190808082b2b0819, 0x190808082b2b1908, 0x1908081908080808, 0x190808190808082b, + 0x1908081908081919, 0x1908081908082b08, 0x1908081908190819, 0x1908081908191908, + 0x190808190819192b, 0x1908081908192b19, 0x19080819082b0808, 0x19080819082b082b, + 0x19080819082b1919, 0x1908081919080819, 0x1908081919081908, 0x190808191908192b, + 0x1908081919082b19, 0x1908081919190808, 0x190808191919082b, 0x1908081919191919, + 0x1908081919192b08, 0x19080819192b0819, 0x19080819192b1908, 0x190808192b080808, + 0x190808192b08082b, 0x190808192b081919, 0x190808192b082b08, 0x190808192b190819, + 0x190808192b191908, 0x190808192b2b0808, 0x1908082b08080819, 0x1908082b08081908, + 0x1908082b08190808, 0x1908082b0819082b, 0x1908082b08191919, 0x1908082b08192b08, + 0x1908082b082b1908, 0x1908082b19080808, 0x1908082b19081919, 0x1908082b19082b08, + 0x1908082b19190819, 0x1908082b19191908, 0x1908082b192b0808, 0x1908082b2b080819, + 0x1908082b2b081908, 0x1908190808080808, 0x190819080808082b, 0x1908190808081919, + 0x1908190808082b08, 0x1908190808082b2b, 0x1908190808190819, 0x1908190808191908, + 0x190819080819192b, 0x1908190808192b19, 0x19081908082b0808, 0x19081908082b082b, + 0x19081908082b1919, 0x19081908082b2b08, 0x1908190819080819, 0x1908190819081908, + 0x190819081908192b, 0x1908190819082b19, 0x1908190819190808, 0x190819081919082b, + 0x1908190819191919, 0x1908190819192b08, 0x19081908192b0819, 0x19081908192b1908, + 0x190819082b080808, 0x190819082b08082b, 0x190819082b081919, 0x190819082b082b08, + 0x190819082b190819, 0x190819082b191908, 0x190819082b2b0808, 0x1908191908080819, + 0x1908191908081908, 0x190819190808192b, 0x1908191908082b19, 0x1908191908190808, + 0x190819190819082b, 0x1908191908191919, 0x1908191908192b08, 0x19081919082b0819, + 0x19081919082b1908, 0x1908191919080808, 0x190819191908082b, 0x1908191919081919, + 0x1908191919082b08, 0x1908191919190819, 0x1908191919191908, 0x19081919192b0808, + 0x19081919192b2b2b, 0x190819192b080819, 0x190819192b081908, 0x190819192b190808, + 0x1908192b08080808, 0x1908192b0808082b, 0x1908192b08081919, 0x1908192b08082b08, + 0x1908192b08190819, 0x1908192b08191908, 0x1908192b082b0808, 0x1908192b19080819, + 0x1908192b19081908, 0x1908192b19190808, 0x1908192b2b080808, 0x1908192b2b2b1919, + 0x19082b0808080819, 0x19082b0808081908, 0x19082b0808082b19, 0x19082b0808190808, + 0x19082b080819082b, 0x19082b0808191919, 0x19082b0808192b08, 0x19082b08082b0819, + 0x19082b08082b1908, 0x19082b0819080808, 0x19082b081908082b, 0x19082b0819081919, + 0x19082b0819082b08, 0x19082b0819190819, 0x19082b0819191908, 0x19082b08192b0808, + 0x19082b082b081908, 0x19082b082b190808, 0x19082b1908080808, 0x19082b190808082b, + 0x19082b1908081919, 0x19082b1908082b08, 0x19082b1908190819, 0x19082b1908191908, + 0x19082b19082b0808, 0x19082b1919080819, 0x19082b1919081908, 0x19082b1919190808, + 0x19082b192b080808, 0x19082b192b19192b, 0x19082b2b08080819, 0x19082b2b08081908, + 0x19082b2b08190808, 0x19082b2b19080808, 0x1919080808080808, 0x191908080808082b, + 0x1919080808081919, 0x1919080808082b08, 0x1919080808190819, 0x1919080808191908, + 0x191908080819192b, 0x1919080808192b19, 0x19190808082b0808, 0x19190808082b082b, + 0x19190808082b1919, 0x19190808082b2b08, 0x1919080819080819, 0x1919080819081908, + 0x191908081908192b, 0x1919080819082b19, 0x1919080819190808, 0x191908081919082b, + 0x1919080819191919, 0x1919080819192b08, 0x19190808192b0819, 0x19190808192b1908, + 0x191908082b080808, 0x191908082b08082b, 0x191908082b081919, 0x191908082b082b08, + 0x191908082b190819, 0x191908082b191908, 0x1919081908080819, 0x1919081908081908, + 0x191908190808192b, 0x1919081908082b19, 0x1919081908190808, 0x191908190819082b, + 0x1919081908191919, 0x1919081908192b08, 0x19190819082b0819, 0x19190819082b1908, + 0x1919081919080808, 0x191908191908082b, 0x1919081919081919, 0x1919081919082b08, + 0x1919081919190819, 0x1919081919191908, 0x19190819192b0808, 0x191908192b080819, + 0x191908192b081908, 0x191908192b190808, 0x1919082b08080808, 0x1919082b08081919, + 0x1919082b08082b08, 0x1919082b08190819, 0x1919082b08191908, 0x1919082b082b0808, + 0x1919082b19080819, 0x1919082b19081908, 0x1919082b19190808, 0x1919082b192b2b19, + 0x1919082b2b080808, 0x1919190808080819, 0x1919190808081908, 0x191919080808192b, + 0x1919190808082b19, 0x1919190808190808, 0x191919080819082b, 0x1919190808191919, + 0x1919190808192b08, 0x19191908082b0819, 0x19191908082b1908, 0x1919190819080808, + 0x191919081908082b, 0x1919190819081919, 0x1919190819082b08, 0x1919190819190819, + 0x1919190819191908, 0x19191908192b0808, 0x191919082b080819, 0x191919082b081908, + 0x191919082b190808, 0x1919191908080808, 0x191919190808082b, 0x1919191908081919, + 0x1919191908082b08, 0x1919191908190819, 0x1919191908191908, 0x19191919082b0808, + 0x1919191919080819, 0x1919191919081908, 0x1919191919190808, 0x191919192b080808, + 0x1919192b08080819, 0x1919192b08081908, 0x1919192b08190808, 0x1919192b082b192b, + 0x1919192b19080808, 0x19192b0808080808, 0x19192b080808082b, 0x19192b0808081919, + 0x19192b0808082b08, 0x19192b0808190819, 0x19192b0808191908, 0x19192b08082b0808, + 0x19192b0819080819, 0x19192b0819081908, 0x19192b0819190808, 0x19192b0819192b2b, + 0x19192b082b080808, 0x19192b1908080819, 0x19192b1908081908, 0x19192b1908190808, + 0x19192b1919080808, 0x19192b2b08080808, 0x19192b2b08192b19, 0x19192b2b2b081919, + 0x19192b2b2b2b2b08, 0x192b080808080819, 0x192b080808081908, 0x192b08080808192b, + 0x192b080808190808, 0x192b08080819082b, 0x192b080808191919, 0x192b080808192b08, + 0x192b0808082b0819, 0x192b0808082b1908, 0x192b080819080808, 0x192b080819081919, + 0x192b080819082b08, 0x192b080819190819, 0x192b080819191908, 0x192b0808192b0808, + 0x192b08082b081908, 0x192b08082b190808, 0x192b081908080808, 0x192b08190808082b, + 0x192b081908081919, 0x192b081908082b08, 0x192b081908190819, 0x192b081908191908, + 0x192b0819082b0808, 0x192b081919080819, 0x192b081919081908, 0x192b081919190808, + 0x192b08192b080808, 0x192b08192b192b19, 0x192b082b08081908, 0x192b082b08190808, + 0x192b082b19080808, 0x192b082b1919192b, 0x192b082b2b2b0819, 0x192b190808080808, + 0x192b190808081919, 0x192b190808082b08, 0x192b190808190819, 0x192b190808191908, + 0x192b1908082b0808, 0x192b190819080819, 0x192b190819081908, 0x192b190819190808, + 0x192b19082b080808, 0x192b191908080819, 0x192b191908081908, 0x192b191908190808, + 0x192b191919080808, 0x192b191919082b2b, 0x192b1919192b2b08, 0x192b19192b19082b, + 0x192b192b08080808, 0x192b192b2b191908, 0x192b2b0808080819, 0x192b2b0808081908, + 0x192b2b0808190808, 0x192b2b08192b1919, 0x192b2b082b192b08, 0x192b2b1908080808, + 0x192b2b19082b2b2b, 0x192b2b2b1908082b, 0x192b2b2b2b2b0819, 0x2b08080808080808, + 0x2b0808080808082b, 0x2b08080808081919, 0x2b08080808082b08, 0x2b08080808190819, + 0x2b08080808191908, 0x2b08080808192b19, 0x2b080808082b0808, 0x2b080808082b1919, + 0x2b08080819080819, 0x2b08080819081908, 0x2b08080819190808, 0x2b0808081919082b, + 0x2b08080819191919, 0x2b08080819192b08, 0x2b080808192b0819, 0x2b0808082b080808, + 0x2b0808082b081919, 0x2b0808082b190819, 0x2b0808082b191908, 0x2b08081908080819, + 0x2b08081908081908, 0x2b08081908082b19, 0x2b08081908190808, 0x2b0808190819082b, + 0x2b08081908191919, 0x2b08081908192b08, 0x2b080819082b0819, 0x2b080819082b1908, + 0x2b08081919080808, 0x2b0808191908082b, 0x2b08081919081919, 0x2b08081919082b08, + 0x2b08081919190819, 0x2b08081919191908, 0x2b0808192b080819, 0x2b0808192b081908, + 0x2b0808192b190808, 0x2b0808192b2b2b19, 0x2b08082b08080808, 0x2b08082b08081919, + 0x2b08082b08082b2b, 0x2b08082b08190819, 0x2b08082b08191908, 0x2b08082b19080819, + 0x2b08082b19081908, 0x2b08082b19190808, 0x2b08190808080819, 0x2b08190808081908, + 0x2b0819080808192b, 0x2b08190808082b19, 0x2b08190808190808, 0x2b0819080819082b, + 0x2b08190808191919, 0x2b08190808192b08, 0x2b081908082b0819, 0x2b08190819080808, + 0x2b0819081908082b, 0x2b08190819081919, 0x2b08190819082b08, 0x2b08190819190819, + 0x2b08190819191908, 0x2b081908192b0808, 0x2b0819082b080819, 0x2b0819082b081908, + 0x2b0819082b190808, 0x2b08191908080808, 0x2b0819190808082b, 0x2b08191908081919, + 0x2b08191908082b08, 0x2b08191908190819, 0x2b08191908191908, 0x2b081919082b0808, + 0x2b08191919080819, 0x2b08191919081908, 0x2b08191919190808, 0x2b0819192b080808, + 0x2b0819192b082b2b, 0x2b08192b08080819, 0x2b08192b08081908, 0x2b08192b08190808, + 0x2b08192b082b2b19, 0x2b08192b19080808, 0x2b082b0808080808, 0x2b082b0808081919, + 0x2b082b0808190819, 0x2b082b0808191908, 0x2b082b0819080819, 0x2b082b0819081908, + 0x2b082b0819190808, 0x2b082b082b2b082b, 0x2b082b1908080819, 0x2b082b1908081908, + 0x2b082b1919080808, 0x2b082b19192b1919, 0x2b082b2b082b082b, 0x2b082b2b19192b08, + 0x2b082b2b19192b2b, 0x2b082b2b2b08082b, 0x2b082b2b2b2b082b, 0x2b19080808080819, + 0x2b19080808081908, 0x2b19080808082b19, 0x2b19080808190808, 0x2b1908080819082b, + 0x2b19080808191919, 0x2b19080808192b08, 0x2b190808082b1908, 0x2b19080819080808, + 0x2b1908081908082b, 0x2b19080819081919, 0x2b19080819082b08, 0x2b19080819190819, + 0x2b19080819191908, 0x2b190808192b0808, 0x2b1908082b080819, 0x2b1908082b081908, + 0x2b1908082b190808, 0x2b19081908080808, 0x2b19081908081919, 0x2b19081908190819, + 0x2b19081908191908, 0x2b19081919080819, 0x2b19081919081908, 0x2b19081919190808, + 0x2b19081919192b2b, 0x2b19082b08080819, 0x2b19082b08081908, 0x2b19082b08190808, + 0x2b19082b19080808, 0x2b19082b2b2b192b, 0x2b19190808080808, 0x2b1919080808082b, + 0x2b19190808081919, 0x2b19190808082b08, 0x2b19190808190819, 0x2b19190808191908, + 0x2b191908082b0808, 0x2b19190819080819, 0x2b19190819081908, 0x2b19190819190808, + 0x2b1919082b080808, 0x2b1919082b19192b, 0x2b19191908080819, 0x2b19191908081908, + 0x2b19191908190808, 0x2b19191919080808, 0x2b1919192b192b08, 0x2b1919192b2b0819, + 0x2b19192b08080808, 0x2b19192b1908192b, 0x2b19192b192b1908, 0x2b192b0808080819, + 0x2b192b0808081908, 0x2b192b0808190808, 0x2b192b08082b192b, 0x2b192b0819080808, + 0x2b192b082b2b2b19, 0x2b192b1908080808, 0x2b192b1919082b19, 0x2b192b191919082b, + 0x2b192b2b2b190808, 0x2b2b080808080808, 0x2b2b080808081919, 0x2b2b080808082b2b, + 0x2b2b080808191908, 0x2b2b0808082b082b, 0x2b2b0808082b2b2b, 0x2b2b080819080819, + 0x2b2b080819081908, 0x2b2b080819190808, 0x2b2b08082b2b082b, 0x2b2b08082b2b2b2b, + 0x2b2b081919080808, 0x2b2b0819192b1919, 0x2b2b082b0808082b, 0x2b2b082b08082b2b, + 0x2b2b082b082b082b, 0x2b2b082b082b2b08, 0x2b2b082b082b2b2b, 0x2b2b082b2b08082b, + 0x2b2b082b2b082b08, 0x2b2b082b2b082b2b, 0x2b2b082b2b2b2b08, 0x2b2b190808080819, + 0x2b2b190808081908, 0x2b2b190808190808, 0x2b2b190819080808, 0x2b2b19082b082b19, + 0x2b2b19082b2b1908, 0x2b2b191908080808, 0x2b2b191908192b19, 0x2b2b192b19190819, + 0x2b2b2b0808082b2b, 0x2b2b2b08082b2b08, 0x2b2b2b082b2b082b, 0x2b2b2b1919191908, + 0x2b2b2b192b08192b, 0x2b2b2b2b08082b08, 0x2b2b2b2b08082b2b, 0x2b2b2b2b082b0808, + 0x2b2b2b2b082b082b, 0x2b2b2b2b082b2b08, 0x2b2b2b2b2b082b08, 0x2b2b2b2b2b2b2b2b, +GGML_TABLE_END() + +GGML_TABLE_BEGIN(uint32_t, iq3xxs_grid, 256) + 0x04040404, 0x04040414, 0x04040424, 0x04040c0c, 0x04040c1c, 0x04040c3e, 0x04041404, 0x04041414, + 0x04041c0c, 0x04042414, 0x04043e1c, 0x04043e2c, 0x040c040c, 0x040c041c, 0x040c0c04, 0x040c0c14, + 0x040c140c, 0x040c142c, 0x040c1c04, 0x040c1c14, 0x040c240c, 0x040c2c24, 0x040c3e04, 0x04140404, + 0x04140414, 0x04140424, 0x04140c0c, 0x04141404, 0x04141414, 0x04141c0c, 0x04141c1c, 0x04141c3e, + 0x04142c0c, 0x04142c3e, 0x04143e2c, 0x041c040c, 0x041c043e, 0x041c0c04, 0x041c0c14, 0x041c142c, + 0x041c3e04, 0x04240c1c, 0x04241c3e, 0x04242424, 0x04242c3e, 0x04243e1c, 0x04243e2c, 0x042c040c, + 0x042c043e, 0x042c1c14, 0x042c2c14, 0x04341c2c, 0x04343424, 0x043e0c04, 0x043e0c24, 0x043e0c34, + 0x043e241c, 0x043e340c, 0x0c04040c, 0x0c04041c, 0x0c040c04, 0x0c040c14, 0x0c04140c, 0x0c04141c, + 0x0c041c04, 0x0c041c14, 0x0c041c24, 0x0c04243e, 0x0c042c04, 0x0c0c0404, 0x0c0c0414, 0x0c0c0c0c, + 0x0c0c1404, 0x0c0c1414, 0x0c14040c, 0x0c14041c, 0x0c140c04, 0x0c140c14, 0x0c14140c, 0x0c141c04, + 0x0c143e14, 0x0c1c0404, 0x0c1c0414, 0x0c1c1404, 0x0c1c1c0c, 0x0c1c2434, 0x0c1c3434, 0x0c24040c, + 0x0c24042c, 0x0c242c04, 0x0c2c1404, 0x0c2c1424, 0x0c2c2434, 0x0c2c3e0c, 0x0c34042c, 0x0c3e1414, + 0x0c3e2404, 0x14040404, 0x14040414, 0x14040c0c, 0x14040c1c, 0x14041404, 0x14041414, 0x14041434, + 0x14041c0c, 0x14042414, 0x140c040c, 0x140c041c, 0x140c042c, 0x140c0c04, 0x140c0c14, 0x140c140c, + 0x140c1c04, 0x140c341c, 0x140c343e, 0x140c3e04, 0x14140404, 0x14140414, 0x14140c0c, 0x14140c3e, + 0x14141404, 0x14141414, 0x14141c3e, 0x14142404, 0x14142c2c, 0x141c040c, 0x141c0c04, 0x141c0c24, + 0x141c3e04, 0x141c3e24, 0x14241c2c, 0x14242c1c, 0x142c041c, 0x142c143e, 0x142c240c, 0x142c3e24, + 0x143e040c, 0x143e041c, 0x143e0c34, 0x143e242c, 0x1c04040c, 0x1c040c04, 0x1c040c14, 0x1c04140c, + 0x1c04141c, 0x1c042c04, 0x1c04342c, 0x1c043e14, 0x1c0c0404, 0x1c0c0414, 0x1c0c1404, 0x1c0c1c0c, + 0x1c0c2424, 0x1c0c2434, 0x1c14040c, 0x1c14041c, 0x1c140c04, 0x1c14142c, 0x1c142c14, 0x1c143e14, + 0x1c1c0c0c, 0x1c1c1c1c, 0x1c241c04, 0x1c24243e, 0x1c243e14, 0x1c2c0404, 0x1c2c0434, 0x1c2c1414, + 0x1c2c2c2c, 0x1c340c24, 0x1c341c34, 0x1c34341c, 0x1c3e1c1c, 0x1c3e3404, 0x24040424, 0x24040c3e, + 0x24041c2c, 0x24041c3e, 0x24042c1c, 0x24042c3e, 0x240c3e24, 0x24141404, 0x24141c3e, 0x24142404, + 0x24143404, 0x24143434, 0x241c043e, 0x241c242c, 0x24240424, 0x24242c0c, 0x24243424, 0x242c142c, + 0x242c241c, 0x242c3e04, 0x243e042c, 0x243e0c04, 0x243e0c14, 0x243e1c04, 0x2c040c14, 0x2c04240c, + 0x2c043e04, 0x2c0c0404, 0x2c0c0434, 0x2c0c1434, 0x2c0c2c2c, 0x2c140c24, 0x2c141c14, 0x2c143e14, + 0x2c1c0414, 0x2c1c2c1c, 0x2c240c04, 0x2c24141c, 0x2c24143e, 0x2c243e14, 0x2c2c0414, 0x2c2c1c0c, + 0x2c342c04, 0x2c3e1424, 0x2c3e2414, 0x34041424, 0x34042424, 0x34042434, 0x34043424, 0x340c140c, + 0x340c340c, 0x34140c3e, 0x34143424, 0x341c1c04, 0x341c1c34, 0x34242424, 0x342c042c, 0x342c2c14, + 0x34341c1c, 0x343e041c, 0x343e140c, 0x3e04041c, 0x3e04042c, 0x3e04043e, 0x3e040c04, 0x3e041c14, + 0x3e042c14, 0x3e0c1434, 0x3e0c2404, 0x3e140c14, 0x3e14242c, 0x3e142c14, 0x3e1c0404, 0x3e1c0c2c, + 0x3e1c1c1c, 0x3e1c3404, 0x3e24140c, 0x3e24240c, 0x3e2c0404, 0x3e2c0414, 0x3e2c1424, 0x3e341c04, +GGML_TABLE_END() + +GGML_TABLE_BEGIN(uint32_t, iq3s_grid, 512) + 0x01010101, 0x01010103, 0x01010105, 0x0101010b, 0x0101010f, 0x01010301, 0x01010303, 0x01010305, + 0x01010309, 0x0101030d, 0x01010501, 0x01010503, 0x0101050b, 0x01010707, 0x01010901, 0x01010905, + 0x0101090b, 0x0101090f, 0x01010b03, 0x01010b07, 0x01010d01, 0x01010d05, 0x01010f03, 0x01010f09, + 0x01010f0f, 0x01030101, 0x01030103, 0x01030105, 0x01030109, 0x01030301, 0x01030303, 0x0103030b, + 0x01030501, 0x01030507, 0x0103050f, 0x01030703, 0x0103070b, 0x01030909, 0x01030d03, 0x01030d0b, + 0x01030f05, 0x01050101, 0x01050103, 0x0105010b, 0x0105010f, 0x01050301, 0x01050307, 0x0105030d, + 0x01050503, 0x0105050b, 0x01050701, 0x01050709, 0x01050905, 0x0105090b, 0x0105090f, 0x01050b03, + 0x01050b07, 0x01050f01, 0x01050f07, 0x01070107, 0x01070303, 0x0107030b, 0x01070501, 0x01070505, + 0x01070703, 0x01070707, 0x0107070d, 0x01070909, 0x01070b01, 0x01070b05, 0x01070d0f, 0x01070f03, + 0x01070f0b, 0x01090101, 0x01090307, 0x0109030f, 0x01090503, 0x01090509, 0x01090705, 0x01090901, + 0x01090907, 0x01090b03, 0x01090f01, 0x010b0105, 0x010b0109, 0x010b0501, 0x010b0505, 0x010b050d, + 0x010b0707, 0x010b0903, 0x010b090b, 0x010b090f, 0x010b0d0d, 0x010b0f07, 0x010d010d, 0x010d0303, + 0x010d0307, 0x010d0703, 0x010d0b05, 0x010d0f03, 0x010f0101, 0x010f0105, 0x010f0109, 0x010f0501, + 0x010f0505, 0x010f050d, 0x010f0707, 0x010f0b01, 0x010f0b09, 0x03010101, 0x03010103, 0x03010105, + 0x03010109, 0x03010301, 0x03010303, 0x03010307, 0x0301030b, 0x0301030f, 0x03010501, 0x03010505, + 0x03010703, 0x03010709, 0x0301070d, 0x03010b09, 0x03010b0d, 0x03010d03, 0x03010f05, 0x03030101, + 0x03030103, 0x03030107, 0x0303010d, 0x03030301, 0x03030309, 0x03030503, 0x03030701, 0x03030707, + 0x03030903, 0x03030b01, 0x03030b05, 0x03030f01, 0x03030f0d, 0x03050101, 0x03050305, 0x0305030b, + 0x0305030f, 0x03050501, 0x03050509, 0x03050705, 0x03050901, 0x03050907, 0x03050b0b, 0x03050d01, + 0x03050f05, 0x03070103, 0x03070109, 0x0307010f, 0x03070301, 0x03070307, 0x03070503, 0x0307050f, + 0x03070701, 0x03070709, 0x03070903, 0x03070d05, 0x03070f01, 0x03090107, 0x0309010b, 0x03090305, + 0x03090309, 0x03090703, 0x03090707, 0x03090905, 0x0309090d, 0x03090b01, 0x03090b09, 0x030b0103, + 0x030b0301, 0x030b0307, 0x030b0503, 0x030b0701, 0x030b0705, 0x030b0b03, 0x030d0501, 0x030d0509, + 0x030d050f, 0x030d0909, 0x030d090d, 0x030f0103, 0x030f0107, 0x030f0301, 0x030f0305, 0x030f0503, + 0x030f070b, 0x030f0903, 0x030f0d05, 0x030f0f01, 0x05010101, 0x05010103, 0x05010107, 0x0501010b, + 0x0501010f, 0x05010301, 0x05010305, 0x05010309, 0x0501030d, 0x05010503, 0x05010507, 0x0501050f, + 0x05010701, 0x05010705, 0x05010903, 0x05010907, 0x0501090b, 0x05010b01, 0x05010b05, 0x05010d0f, + 0x05010f01, 0x05010f07, 0x05010f0b, 0x05030101, 0x05030105, 0x05030301, 0x05030307, 0x0503030f, + 0x05030505, 0x0503050b, 0x05030703, 0x05030709, 0x05030905, 0x05030b03, 0x05050103, 0x05050109, + 0x0505010f, 0x05050503, 0x05050507, 0x05050701, 0x0505070f, 0x05050903, 0x05050b07, 0x05050b0f, + 0x05050f03, 0x05050f09, 0x05070101, 0x05070105, 0x0507010b, 0x05070303, 0x05070505, 0x05070509, + 0x05070703, 0x05070707, 0x05070905, 0x05070b01, 0x05070d0d, 0x05090103, 0x0509010f, 0x05090501, + 0x05090507, 0x05090705, 0x0509070b, 0x05090903, 0x05090f05, 0x05090f0b, 0x050b0109, 0x050b0303, + 0x050b0505, 0x050b070f, 0x050b0901, 0x050b0b07, 0x050b0f01, 0x050d0101, 0x050d0105, 0x050d010f, + 0x050d0503, 0x050d0b0b, 0x050d0d03, 0x050f010b, 0x050f0303, 0x050f050d, 0x050f0701, 0x050f0907, + 0x050f0b01, 0x07010105, 0x07010303, 0x07010307, 0x0701030b, 0x0701030f, 0x07010505, 0x07010703, + 0x07010707, 0x0701070b, 0x07010905, 0x07010909, 0x0701090f, 0x07010b03, 0x07010d07, 0x07010f03, + 0x07030103, 0x07030107, 0x0703010b, 0x07030309, 0x07030503, 0x07030507, 0x07030901, 0x07030d01, + 0x07030f05, 0x07030f0d, 0x07050101, 0x07050305, 0x07050501, 0x07050705, 0x07050709, 0x07050b01, + 0x07070103, 0x07070301, 0x07070309, 0x07070503, 0x07070507, 0x0707050f, 0x07070701, 0x07070903, + 0x07070907, 0x0707090f, 0x07070b0b, 0x07070f07, 0x07090107, 0x07090303, 0x0709030d, 0x07090505, + 0x07090703, 0x07090b05, 0x07090d01, 0x07090d09, 0x070b0103, 0x070b0301, 0x070b0305, 0x070b050b, + 0x070b0705, 0x070b0909, 0x070b0b0d, 0x070b0f07, 0x070d030d, 0x070d0903, 0x070f0103, 0x070f0107, + 0x070f0501, 0x070f0505, 0x070f070b, 0x09010101, 0x09010109, 0x09010305, 0x09010501, 0x09010509, + 0x0901050f, 0x09010705, 0x09010903, 0x09010b01, 0x09010f01, 0x09030105, 0x0903010f, 0x09030303, + 0x09030307, 0x09030505, 0x09030701, 0x0903070b, 0x09030907, 0x09030b03, 0x09030b0b, 0x09050103, + 0x09050107, 0x09050301, 0x0905030b, 0x09050503, 0x09050707, 0x09050901, 0x09050b0f, 0x09050d05, + 0x09050f01, 0x09070109, 0x09070303, 0x09070307, 0x09070501, 0x09070505, 0x09070703, 0x0907070b, + 0x09090101, 0x09090105, 0x09090509, 0x0909070f, 0x09090901, 0x09090f03, 0x090b010b, 0x090b010f, + 0x090b0503, 0x090b0d05, 0x090d0307, 0x090d0709, 0x090d0d01, 0x090f0301, 0x090f030b, 0x090f0701, + 0x090f0907, 0x090f0b03, 0x0b010105, 0x0b010301, 0x0b010309, 0x0b010505, 0x0b010901, 0x0b010909, + 0x0b01090f, 0x0b010b05, 0x0b010d0d, 0x0b010f09, 0x0b030103, 0x0b030107, 0x0b03010b, 0x0b030305, + 0x0b030503, 0x0b030705, 0x0b030f05, 0x0b050101, 0x0b050303, 0x0b050507, 0x0b050701, 0x0b05070d, + 0x0b050b07, 0x0b070105, 0x0b07010f, 0x0b070301, 0x0b07050f, 0x0b070909, 0x0b070b03, 0x0b070d0b, + 0x0b070f07, 0x0b090103, 0x0b090109, 0x0b090501, 0x0b090705, 0x0b09090d, 0x0b0b0305, 0x0b0b050d, + 0x0b0b0b03, 0x0b0b0b07, 0x0b0d0905, 0x0b0f0105, 0x0b0f0109, 0x0b0f0505, 0x0d010303, 0x0d010307, + 0x0d01030b, 0x0d010703, 0x0d010707, 0x0d010d01, 0x0d030101, 0x0d030501, 0x0d03050f, 0x0d030d09, + 0x0d050305, 0x0d050709, 0x0d050905, 0x0d050b0b, 0x0d050d05, 0x0d050f01, 0x0d070101, 0x0d070309, + 0x0d070503, 0x0d070901, 0x0d09050b, 0x0d090907, 0x0d090d05, 0x0d0b0101, 0x0d0b0107, 0x0d0b0709, + 0x0d0b0d01, 0x0d0d010b, 0x0d0d0901, 0x0d0f0303, 0x0d0f0307, 0x0f010101, 0x0f010109, 0x0f01010f, + 0x0f010501, 0x0f010505, 0x0f01070d, 0x0f010901, 0x0f010b09, 0x0f010d05, 0x0f030105, 0x0f030303, + 0x0f030509, 0x0f030907, 0x0f03090b, 0x0f050103, 0x0f050109, 0x0f050301, 0x0f05030d, 0x0f050503, + 0x0f050701, 0x0f050b03, 0x0f070105, 0x0f070705, 0x0f07070b, 0x0f070b07, 0x0f090103, 0x0f09010b, + 0x0f090307, 0x0f090501, 0x0f090b01, 0x0f0b0505, 0x0f0b0905, 0x0f0d0105, 0x0f0d0703, 0x0f0f0101, +GGML_TABLE_END() + +#define NGRID_IQ2XXS 512 +GGML_TABLE_BEGIN(uint64_t, iq1s_grid, NGRID_IQ2XXS) + 0xffffffffffff0101, 0xffffffffff01ff00, 0xffffffffff010100, 0xffffffff00000000, + 0xffffffff01ff00ff, 0xffffffff01ff0001, 0xffffffff0101ffff, 0xffffffff0101ff01, + 0xffffff00ff000000, 0xffffff000000ff00, 0xffffff00000000ff, 0xffffff0000000100, + 0xffffff0000010000, 0xffffff0001000000, 0xffffff01ffff00ff, 0xffffff01ff01ff00, + 0xffffff01ff010100, 0xffffff0100000001, 0xffffff0101ffff00, 0xffffff0101ff0101, + 0xffffff0101010100, 0xffff00ffff00ff01, 0xffff00ffff0000ff, 0xffff00ff00ff0100, + 0xffff00ff0100ff00, 0xffff00ff010001ff, 0xffff0000ff0101ff, 0xffff000000ffff00, + 0xffff000000000000, 0xffff00000001ff01, 0xffff000001000101, 0xffff0000010100ff, + 0xffff0001ffff0100, 0xffff00010000ff00, 0xffff000100010101, 0xffff000101000000, + 0xffff01ffffff0000, 0xffff01ffff01ffff, 0xffff01ffff010100, 0xffff01ff00000000, + 0xffff01ff01ffffff, 0xffff01ff01ff0001, 0xffff01ff0101ffff, 0xffff01ff01010001, + 0xffff0100ffffff01, 0xffff01000000ffff, 0xffff010000000100, 0xffff010001ff01ff, + 0xffff010001000000, 0xffff0101ff000000, 0xffff0101000101ff, 0xffff010101ffff01, + 0xffff01010101ff00, 0xff00ffffff000000, 0xff00ffff00ffff00, 0xff00ffff00000001, + 0xff00ffff000001ff, 0xff00ffff01010000, 0xff00ff00ffff0000, 0xff00ff00ff00ff00, + 0xff00ff00ff0000ff, 0xff00ff00ff000100, 0xff00ff00ff010001, 0xff00ff0000ff0001, + 0xff00ff000000ffff, 0xff00ff0000000000, 0xff00ff000001ff00, 0xff00ff0000010100, + 0xff00ff0001ff0000, 0xff00ff000100ff00, 0xff00ff0001000100, 0xff00ff01ff000000, + 0xff00ff0100ff0000, 0xff00ff01000001ff, 0xff00ff0101010001, 0xff0000ff00000000, + 0xff0000ff0001ff00, 0xff0000ff00010100, 0xff000000ffff0101, 0xff000000ff000000, + 0xff000000ff01ff00, 0xff00000000ff0000, 0xff0000000000ff00, 0xff000000000000ff, + 0xff00000000000000, 0xff00000000000001, 0xff00000000000100, 0xff0000000001ffff, + 0xff00000000010000, 0xff00000001000000, 0xff00000001010100, 0xff000001ff00ff01, + 0xff000001ff0100ff, 0xff00000100000000, 0xff0000010001ff00, 0xff00000101ff0100, + 0xff0000010100ff00, 0xff0001ff00ff00ff, 0xff0001ff00000101, 0xff0001ff000100ff, + 0xff0001ff01000000, 0xff000100ff0001ff, 0xff0001000000ff01, 0xff00010000000000, + 0xff00010000010001, 0xff00010000010100, 0xff00010001ffff00, 0xff00010001ff0101, + 0xff00010001010000, 0xff000101ffffffff, 0xff000101ff000101, 0xff00010101ff00ff, + 0xff00010101000001, 0xff000101010100ff, 0xff01ffffff000101, 0xff01ffffff01ffff, + 0xff01ffffff01ff01, 0xff01ffffff0101ff, 0xff01ffff00000000, 0xff01ffff01ff0001, + 0xff01ffff0101ff01, 0xff01ff00ff000000, 0xff01ff0000ff0100, 0xff01ff000000ff01, + 0xff01ff0000010000, 0xff01ff00010000ff, 0xff01ff01ff01ff00, 0xff01ff0100000101, + 0xff0100ffffff0000, 0xff0100ffff010000, 0xff0100ff01ff00ff, 0xff0100ff01000100, + 0xff0100ff010100ff, 0xff010000ffffff01, 0xff01000000000000, 0xff0100000101ff00, + 0xff010001ffff00ff, 0xff010001ff000100, 0xff01000100ffff00, 0xff01000100010001, + 0xff01000101ff0001, 0xff010001010001ff, 0xff0101ffffffffff, 0xff0101ffff01ffff, + 0xff0101ffff010101, 0xff0101ff0000ff00, 0xff0101ff01010001, 0xff010100ff000000, + 0xff010100ff01ff01, 0xff01010000ff0001, 0xff01010000000100, 0xff01010001000000, + 0xff0101010100ffff, 0x00ffffff0000ff01, 0x00ffffff000000ff, 0x00ffffff00000100, + 0x00ffffff00010000, 0x00ffff00ffff0001, 0x00ffff00ff0000ff, 0x00ffff00ff000100, + 0x00ffff0000000000, 0x00ffff0001000100, 0x00ffff0001010001, 0x00ffff01ff00ff01, + 0x00ffff0100ff0100, 0x00ffff010000ff00, 0x00ffff01000100ff, 0x00ffff0101ff00ff, + 0x00ffff010101ff00, 0x00ff00ffffffffff, 0x00ff00ffffff01ff, 0x00ff00ffff000101, + 0x00ff00ff00000000, 0x00ff00ff000101ff, 0x00ff00ff01010101, 0x00ff0000ff000000, + 0x00ff0000ff01ffff, 0x00ff000000ff0000, 0x00ff00000000ff00, 0x00ff0000000000ff, + 0x00ff000000000000, 0x00ff000000000001, 0x00ff000000000100, 0x00ff000000010000, + 0x00ff000001ffff01, 0x00ff000001000000, 0x00ff0001ff000101, 0x00ff000100ffffff, + 0x00ff000100000000, 0x00ff0001010001ff, 0x00ff01ffff000000, 0x00ff01ff0001ff00, + 0x00ff01ff01ff0100, 0x00ff0100ff01ff01, 0x00ff010000ff00ff, 0x00ff010000ff0101, + 0x00ff010000000000, 0x00ff010000010101, 0x00ff01000100ff00, 0x00ff010001010000, + 0x00ff0101ffffff00, 0x00ff01010000ff01, 0x00ff010100000100, 0x00ff010101ff0000, + 0x0000ffffffff0100, 0x0000ffffff00ff00, 0x0000ffffff0000ff, 0x0000ffffff010000, + 0x0000ffff00000000, 0x0000ffff00010101, 0x0000ffff01ffff01, 0x0000ffff01000100, + 0x0000ff00ff000000, 0x0000ff00ff01ff00, 0x0000ff00ff0101ff, 0x0000ff0000ff0000, + 0x0000ff000000ff00, 0x0000ff00000000ff, 0x0000ff0000000000, 0x0000ff0000000001, + 0x0000ff0000000100, 0x0000ff0000010000, 0x0000ff0001ffffff, 0x0000ff0001ff01ff, + 0x0000ff0001000000, 0x0000ff000101ffff, 0x0000ff01ffff0101, 0x0000ff01ff010000, + 0x0000ff0100000000, 0x0000ff0101000101, 0x000000ffffff0001, 0x000000ffff000000, + 0x000000ff00ff0000, 0x000000ff0000ff00, 0x000000ff000000ff, 0x000000ff00000000, + 0x000000ff00000001, 0x000000ff00000100, 0x000000ff00010000, 0x000000ff01000000, + 0x000000ff0101ff00, 0x00000000ffff0000, 0x00000000ff00ff00, 0x00000000ff0000ff, + 0x00000000ff000000, 0x00000000ff000001, 0x00000000ff000100, 0x00000000ff010000, + 0x0000000000ffff00, 0x0000000000ff00ff, 0x0000000000ff0000, 0x0000000000ff0001, + 0x0000000000ff0100, 0x000000000000ffff, 0x000000000000ff00, 0x000000000000ff01, + 0x00000000000000ff, 0x0000000000000001, 0x00000000000001ff, 0x0000000000000100, + 0x0000000000000101, 0x000000000001ff00, 0x00000000000100ff, 0x0000000000010000, + 0x0000000000010001, 0x0000000000010100, 0x0000000001ff0000, 0x000000000100ff00, + 0x00000000010000ff, 0x0000000001000000, 0x0000000001000001, 0x0000000001000100, + 0x0000000001010000, 0x00000001ffff01ff, 0x00000001ff000000, 0x0000000100ff0000, + 0x000000010000ff00, 0x00000001000000ff, 0x0000000100000000, 0x0000000100000001, + 0x0000000100000100, 0x0000000100010000, 0x0000000101000000, 0x000001ffff00ff00, + 0x000001ffff010001, 0x000001ffff0101ff, 0x000001ff00ffff01, 0x000001ff0000ffff, + 0x000001ff00000000, 0x000001ff010000ff, 0x000001ff01010100, 0x00000100ffff0100, + 0x00000100ff000000, 0x0000010000ff0000, 0x000001000000ff00, 0x00000100000000ff, + 0x0000010000000000, 0x0000010000000001, 0x0000010000000100, 0x0000010000010000, + 0x0000010001000000, 0x000001000101ff01, 0x00000101ffff0001, 0x00000101ff01ffff, + 0x0000010100000000, 0x0000010101010100, 0x0001ffffff000000, 0x0001ffff00ffffff, + 0x0001ffff00000100, 0x0001ffff0001ff00, 0x0001ffff01000000, 0x0001ff00ffffff00, + 0x0001ff00ffff01ff, 0x0001ff00ff010000, 0x0001ff0000000000, 0x0001ff0000010001, + 0x0001ff0001ff0000, 0x0001ff0001010100, 0x0001ff01ff0000ff, 0x0001ff01ff000001, + 0x0001ff0100ffffff, 0x0001ff010001ffff, 0x0001ff01000101ff, 0x0001ff010100ff01, + 0x000100ffff00ffff, 0x000100ffff00ff01, 0x000100ffff000100, 0x000100ff00000000, + 0x000100ff000101ff, 0x000100ff01ff0101, 0x000100ff0100ffff, 0x000100ff01010101, + 0x00010000ff000000, 0x00010000ff010100, 0x0001000000ff0000, 0x000100000000ff00, + 0x00010000000000ff, 0x0001000000000000, 0x0001000000000001, 0x0001000000000100, + 0x0001000000010000, 0x0001000001ffff01, 0x0001000001000000, 0x0001000100ff0101, + 0x0001000100000000, 0x00010001010100ff, 0x000101ffffff01ff, 0x000101ffffff0101, + 0x000101ff00010000, 0x000101ff01ff0000, 0x000101ff0100ff01, 0x00010100ffff0000, + 0x0001010000000000, 0x000101000001ffff, 0x0001010000010101, 0x00010100010001ff, + 0x00010101ff00ff00, 0x00010101ff010001, 0x0001010100ffffff, 0x0001010100ff01ff, + 0x00010101000101ff, 0x0001010101ff0000, 0x000101010100ff01, 0x0001010101000101, + 0x01ffffffffff0101, 0x01ffffffff01ffff, 0x01ffffffff01ff01, 0x01ffffffff0101ff, + 0x01ffffffff010101, 0x01ffffff00000000, 0x01ffffff01ff01ff, 0x01ffffff01000101, + 0x01ffffff0101ff01, 0x01ffffff010100ff, 0x01ffff000000ff00, 0x01ffff0000000001, + 0x01ffff00000001ff, 0x01ffff0000010000, 0x01ffff0001ff0000, 0x01ffff01ffffffff, + 0x01ffff01ffff01ff, 0x01ffff01ff000000, 0x01ffff01ff01ffff, 0x01ffff01ff0101ff, + 0x01ffff010100ffff, 0x01ff00ffffff0000, 0x01ff00ffff010000, 0x01ff00ff00ffff01, + 0x01ff0000ff0000ff, 0x01ff000000000000, 0x01ff00000001ff01, 0x01ff000001ffffff, + 0x01ff000001010100, 0x01ff0001ffffff01, 0x01ff0001ff010001, 0x01ff000101ff0100, + 0x01ff000101000001, 0x01ff0001010100ff, 0x01ff01ffff00ffff, 0x01ff01ff00010001, + 0x01ff01ff01000000, 0x01ff01ff010101ff, 0x01ff0100ff000001, 0x01ff010000ffff00, + 0x01ff010000000100, 0x01ff010001ff01ff, 0x01ff01000101ffff, 0x01ff0101ffff00ff, + 0x01ff0101ffff0101, 0x01ff0101ff0101ff, 0x01ff010100010000, 0x0100ffff00ff00ff, + 0x0100ffff00ff0001, 0x0100ffff00000100, 0x0100ffff0100ff00, 0x0100ff00ffff0000, + 0x0100ff00ff00ffff, 0x0100ff00ff00ff01, 0x0100ff00ff000100, 0x0100ff00ff010000, + 0x0100ff0000000000, 0x0100ff00000100ff, 0x0100ff0001ff0101, 0x0100ff0001010101, + 0x0100ff0100ff00ff, 0x0100ff0100ff0001, 0x0100ff0100000100, 0x0100ff0100010001, + 0x0100ff0101000000, 0x010000ffff00ff00, 0x010000ff0000ffff, 0x010000ff00000000, + 0x010000ff010001ff, 0x010000ff01010001, 0x01000000ffffff00, 0x01000000ffff0101, + 0x01000000ff000000, 0x01000000ff0100ff, 0x01000000ff010101, 0x0100000000ff0000, + 0x010000000000ff00, 0x01000000000000ff, 0x0100000000000000, 0x0100000000000001, + 0x0100000000000100, 0x0100000000010000, 0x0100000001000000, 0x0100000100000000, + 0x01000001000101ff, 0x0100000101ffff01, 0x010001ffff000101, 0x010001ff00ff0100, + 0x010001ff0000ff00, 0x010001ff000100ff, 0x010001ff01ffffff, 0x01000100ffff0000, + 0x01000100ff0001ff, 0x0100010000000000, 0x010001000001ff00, 0x0100010001ff0000, + 0x01000100010000ff, 0x0100010001000101, 0x01000101ff00ff01, 0x0100010100ff0100, + 0x010001010000ffff, 0x0100010101010001, 0x0101ffffffff0101, 0x0101ffffff0001ff, + 0x0101ffffff01ffff, 0x0101ffffff010101, 0x0101ffff00000000, 0x0101ffff0101ffff, + 0x0101ffff010101ff, 0x0101ff00ff000000, 0x0101ff0000ff0100, 0x0101ff000000ff00, + 0x0101ff0000010000, 0x0101ff00010000ff, 0x0101ff0001000001, 0x0101ff01ff010101, + 0x0101ff0100000000, 0x0101ff010101ff00, 0x010100ffffff0000, 0x010100ffff010000, + 0x010100ff00ff01ff, 0x010100ff000000ff, 0x010100ff00000101, 0x010100ff01ffff00, + 0x01010000ffffff01, 0x01010000ff000100, 0x01010000ff01ff01, 0x0101000000000000, + 0x01010000000100ff, 0x010100000101ff01, 0x01010001ffff0000, 0x01010001ff00ffff, + 0x01010001ff010000, 0x0101000101ffffff, 0x0101000101ff01ff, 0x0101000101010101, + 0x010101ffff01ffff, 0x010101ff00000000, 0x010101ff0001ff01, 0x010101ff0101ffff, + 0x010101ff010101ff, 0x01010100ffffffff, 0x01010100ff000001, 0x010101000000ff00, + 0x0101010001010000, 0x0101010100ff0001, 0x010101010001ff01, 0x010101010101ffff, +GGML_TABLE_END() + +#endif // GGML_COMMON_IMPL diff --git a/ggml-cuda.cu b/ggml-cuda.cu index 72bcec8cd..c207ff87a 100644 --- a/ggml-cuda.cu +++ b/ggml-cuda.cu @@ -2,6 +2,9 @@ #include "ggml.h" #include "ggml-backend-impl.h" +#define GGML_COMMON_IMPL_CUDA +#include "ggml-common.h" + #include #include #include @@ -1569,746 +1572,6 @@ static __global__ void dequantize_block_q6_K(const void * __restrict__ vx, dst_t #endif } -static const __device__ uint64_t iq2xxs_grid[256] = { - 0x0808080808080808, 0x080808080808082b, 0x0808080808081919, 0x0808080808082b08, - 0x0808080808082b2b, 0x0808080808190819, 0x0808080808191908, 0x08080808082b0808, - 0x08080808082b082b, 0x08080808082b2b08, 0x08080808082b2b2b, 0x0808080819080819, - 0x0808080819081908, 0x0808080819190808, 0x0808080819192b08, 0x08080808192b0819, - 0x08080808192b1908, 0x080808082b080808, 0x080808082b08082b, 0x080808082b082b2b, - 0x080808082b2b082b, 0x0808081908080819, 0x0808081908081908, 0x0808081908190808, - 0x0808081908191919, 0x0808081919080808, 0x080808192b081908, 0x080808192b192b08, - 0x0808082b08080808, 0x0808082b0808082b, 0x0808082b082b082b, 0x0808082b2b08082b, - 0x0808190808080819, 0x0808190808081908, 0x0808190808190808, 0x08081908082b0819, - 0x08081908082b1908, 0x0808190819080808, 0x080819081908082b, 0x0808190819082b08, - 0x08081908192b0808, 0x080819082b080819, 0x080819082b081908, 0x080819082b190808, - 0x080819082b2b1908, 0x0808191908080808, 0x080819190808082b, 0x0808191908082b08, - 0x08081919082b0808, 0x080819191908192b, 0x08081919192b2b19, 0x080819192b080808, - 0x080819192b190819, 0x0808192b08082b19, 0x0808192b08190808, 0x0808192b19080808, - 0x0808192b2b081908, 0x0808192b2b2b1908, 0x08082b0808080808, 0x08082b0808081919, - 0x08082b0808082b08, 0x08082b0808191908, 0x08082b08082b2b08, 0x08082b0819080819, - 0x08082b0819081908, 0x08082b0819190808, 0x08082b081919082b, 0x08082b082b082b08, - 0x08082b1908081908, 0x08082b1919080808, 0x08082b2b0808082b, 0x08082b2b08191908, - 0x0819080808080819, 0x0819080808081908, 0x0819080808190808, 0x08190808082b0819, - 0x0819080819080808, 0x08190808192b0808, 0x081908082b081908, 0x081908082b190808, - 0x081908082b191919, 0x0819081908080808, 0x0819081908082b08, 0x08190819082b0808, - 0x0819081919190808, 0x0819081919192b2b, 0x081908192b080808, 0x0819082b082b1908, - 0x0819082b19081919, 0x0819190808080808, 0x0819190808082b08, 0x08191908082b0808, - 0x08191908082b1919, 0x0819190819082b19, 0x081919082b080808, 0x0819191908192b08, - 0x08191919192b082b, 0x0819192b08080808, 0x0819192b0819192b, 0x08192b0808080819, - 0x08192b0808081908, 0x08192b0808190808, 0x08192b0819080808, 0x08192b082b080819, - 0x08192b1908080808, 0x08192b1908081919, 0x08192b192b2b0808, 0x08192b2b19190819, - 0x082b080808080808, 0x082b08080808082b, 0x082b080808082b2b, 0x082b080819081908, - 0x082b0808192b0819, 0x082b08082b080808, 0x082b08082b08082b, 0x082b0819082b2b19, - 0x082b081919082b08, 0x082b082b08080808, 0x082b082b0808082b, 0x082b190808080819, - 0x082b190808081908, 0x082b190808190808, 0x082b190819080808, 0x082b19081919192b, - 0x082b191908080808, 0x082b191919080819, 0x082b1919192b1908, 0x082b192b2b190808, - 0x082b2b0808082b08, 0x082b2b08082b0808, 0x082b2b082b191908, 0x082b2b2b19081908, - 0x1908080808080819, 0x1908080808081908, 0x1908080808190808, 0x1908080808192b08, - 0x19080808082b0819, 0x19080808082b1908, 0x1908080819080808, 0x1908080819082b08, - 0x190808081919192b, 0x19080808192b0808, 0x190808082b080819, 0x190808082b081908, - 0x190808082b190808, 0x1908081908080808, 0x19080819082b0808, 0x19080819192b0819, - 0x190808192b080808, 0x190808192b081919, 0x1908082b08080819, 0x1908082b08190808, - 0x1908082b19082b08, 0x1908082b1919192b, 0x1908082b192b2b08, 0x1908190808080808, - 0x1908190808082b08, 0x19081908082b0808, 0x190819082b080808, 0x190819082b192b19, - 0x190819190819082b, 0x19081919082b1908, 0x1908192b08080808, 0x19082b0808080819, - 0x19082b0808081908, 0x19082b0808190808, 0x19082b0819080808, 0x19082b0819081919, - 0x19082b1908080808, 0x19082b1919192b08, 0x19082b19192b0819, 0x19082b192b08082b, - 0x19082b2b19081919, 0x19082b2b2b190808, 0x1919080808080808, 0x1919080808082b08, - 0x1919080808190819, 0x1919080808192b19, 0x19190808082b0808, 0x191908082b080808, - 0x191908082b082b08, 0x1919081908081908, 0x191908191908082b, 0x191908192b2b1908, - 0x1919082b2b190819, 0x191919082b190808, 0x191919082b19082b, 0x1919191908082b2b, - 0x1919192b08080819, 0x1919192b19191908, 0x19192b0808080808, 0x19192b0808190819, - 0x19192b0808192b19, 0x19192b08192b1908, 0x19192b1919080808, 0x19192b2b08082b08, - 0x192b080808081908, 0x192b080808190808, 0x192b080819080808, 0x192b0808192b2b08, - 0x192b081908080808, 0x192b081919191919, 0x192b082b08192b08, 0x192b082b192b0808, - 0x192b190808080808, 0x192b190808081919, 0x192b191908190808, 0x192b19190819082b, - 0x192b19192b081908, 0x192b2b081908082b, 0x2b08080808080808, 0x2b0808080808082b, - 0x2b08080808082b2b, 0x2b08080819080819, 0x2b0808082b08082b, 0x2b08081908081908, - 0x2b08081908192b08, 0x2b08081919080808, 0x2b08082b08190819, 0x2b08190808080819, - 0x2b08190808081908, 0x2b08190808190808, 0x2b08190808191919, 0x2b08190819080808, - 0x2b081908192b0808, 0x2b08191908080808, 0x2b0819191908192b, 0x2b0819192b191908, - 0x2b08192b08082b19, 0x2b08192b19080808, 0x2b08192b192b0808, 0x2b082b080808082b, - 0x2b082b1908081908, 0x2b082b2b08190819, 0x2b19080808081908, 0x2b19080808190808, - 0x2b190808082b1908, 0x2b19080819080808, 0x2b1908082b2b0819, 0x2b1908190819192b, - 0x2b1908192b080808, 0x2b19082b19081919, 0x2b19190808080808, 0x2b191908082b082b, - 0x2b19190819081908, 0x2b19191919190819, 0x2b192b082b080819, 0x2b192b19082b0808, - 0x2b2b08080808082b, 0x2b2b080819190808, 0x2b2b08082b081919, 0x2b2b081908082b19, - 0x2b2b082b08080808, 0x2b2b190808192b08, 0x2b2b2b0819190808, 0x2b2b2b1908081908, -}; - -static const __device__ uint64_t iq2xs_grid[512] = { - 0x0808080808080808, 0x080808080808082b, 0x0808080808081919, 0x0808080808082b08, - 0x0808080808082b2b, 0x0808080808190819, 0x0808080808191908, 0x080808080819192b, - 0x0808080808192b19, 0x08080808082b0808, 0x08080808082b082b, 0x08080808082b1919, - 0x08080808082b2b08, 0x0808080819080819, 0x0808080819081908, 0x080808081908192b, - 0x0808080819082b19, 0x0808080819190808, 0x080808081919082b, 0x0808080819191919, - 0x0808080819192b08, 0x08080808192b0819, 0x08080808192b1908, 0x080808082b080808, - 0x080808082b08082b, 0x080808082b081919, 0x080808082b082b08, 0x080808082b190819, - 0x080808082b191908, 0x080808082b192b19, 0x080808082b2b0808, 0x0808081908080819, - 0x0808081908081908, 0x080808190808192b, 0x0808081908082b19, 0x0808081908190808, - 0x080808190819082b, 0x0808081908191919, 0x0808081908192b08, 0x0808081908192b2b, - 0x08080819082b0819, 0x08080819082b1908, 0x0808081919080808, 0x080808191908082b, - 0x0808081919081919, 0x0808081919082b08, 0x0808081919190819, 0x0808081919191908, - 0x08080819192b0808, 0x08080819192b2b08, 0x080808192b080819, 0x080808192b081908, - 0x080808192b190808, 0x0808082b08080808, 0x0808082b0808082b, 0x0808082b08081919, - 0x0808082b08082b08, 0x0808082b08190819, 0x0808082b08191908, 0x0808082b082b0808, - 0x0808082b19080819, 0x0808082b19081908, 0x0808082b19190808, 0x0808082b19191919, - 0x0808082b2b080808, 0x0808082b2b082b2b, 0x0808190808080819, 0x0808190808081908, - 0x080819080808192b, 0x0808190808082b19, 0x0808190808190808, 0x080819080819082b, - 0x0808190808191919, 0x0808190808192b08, 0x08081908082b0819, 0x08081908082b1908, - 0x0808190819080808, 0x080819081908082b, 0x0808190819081919, 0x0808190819082b08, - 0x0808190819190819, 0x0808190819191908, 0x080819081919192b, 0x08081908192b0808, - 0x080819082b080819, 0x080819082b081908, 0x080819082b190808, 0x0808191908080808, - 0x080819190808082b, 0x0808191908081919, 0x0808191908082b08, 0x0808191908190819, - 0x0808191908191908, 0x08081919082b0808, 0x0808191919080819, 0x0808191919081908, - 0x0808191919190808, 0x08081919192b0819, 0x080819192b080808, 0x0808192b08080819, - 0x0808192b08081908, 0x0808192b08190808, 0x0808192b082b192b, 0x0808192b19080808, - 0x0808192b1908082b, 0x0808192b2b081908, 0x08082b0808080808, 0x08082b080808082b, - 0x08082b0808081919, 0x08082b0808082b08, 0x08082b0808082b2b, 0x08082b0808190819, - 0x08082b0808191908, 0x08082b08082b0808, 0x08082b08082b1919, 0x08082b0819080819, - 0x08082b0819081908, 0x08082b0819190808, 0x08082b0819192b08, 0x08082b082b080808, - 0x08082b082b2b0808, 0x08082b082b2b2b2b, 0x08082b1908080819, 0x08082b1908081908, - 0x08082b1908190808, 0x08082b1919080808, 0x08082b192b080819, 0x08082b192b082b19, - 0x08082b2b08080808, 0x08082b2b082b0808, 0x08082b2b082b2b08, 0x08082b2b2b19192b, - 0x08082b2b2b2b0808, 0x0819080808080819, 0x0819080808081908, 0x081908080808192b, - 0x0819080808082b19, 0x0819080808190808, 0x081908080819082b, 0x0819080808191919, - 0x0819080808192b08, 0x08190808082b0819, 0x08190808082b1908, 0x0819080819080808, - 0x081908081908082b, 0x0819080819081919, 0x0819080819082b08, 0x0819080819190819, - 0x0819080819191908, 0x08190808192b0808, 0x08190808192b2b2b, 0x081908082b080819, - 0x081908082b081908, 0x081908082b190808, 0x0819081908080808, 0x081908190808082b, - 0x0819081908081919, 0x0819081908082b08, 0x0819081908190819, 0x0819081908191908, - 0x08190819082b0808, 0x0819081919080819, 0x0819081919081908, 0x0819081919190808, - 0x081908192b080808, 0x081908192b191908, 0x081908192b19192b, 0x0819082b08080819, - 0x0819082b08081908, 0x0819082b0808192b, 0x0819082b08190808, 0x0819082b19080808, - 0x0819082b192b0808, 0x0819190808080808, 0x081919080808082b, 0x0819190808081919, - 0x0819190808082b08, 0x0819190808190819, 0x0819190808191908, 0x08191908082b0808, - 0x0819190819080819, 0x0819190819081908, 0x0819190819082b19, 0x0819190819190808, - 0x08191908192b1908, 0x081919082b080808, 0x0819191908080819, 0x0819191908081908, - 0x0819191908190808, 0x0819191919080808, 0x0819192b08080808, 0x0819192b08191908, - 0x0819192b19082b19, 0x08192b0808080819, 0x08192b0808081908, 0x08192b0808190808, - 0x08192b080819082b, 0x08192b0819080808, 0x08192b0819191908, 0x08192b082b08192b, - 0x08192b1908080808, 0x08192b1908081919, 0x08192b19192b192b, 0x08192b2b19190819, - 0x08192b2b2b2b2b19, 0x082b080808080808, 0x082b08080808082b, 0x082b080808081919, - 0x082b080808082b08, 0x082b080808082b2b, 0x082b080808190819, 0x082b080808191908, - 0x082b0808082b0808, 0x082b080819080819, 0x082b080819081908, 0x082b080819190808, - 0x082b08082b080808, 0x082b08082b2b0808, 0x082b081908080819, 0x082b081908081908, - 0x082b081908190808, 0x082b081919080808, 0x082b081919082b08, 0x082b0819192b1919, - 0x082b082b08080808, 0x082b082b082b082b, 0x082b082b2b080808, 0x082b082b2b2b2b08, - 0x082b190808080819, 0x082b190808081908, 0x082b190808190808, 0x082b1908082b2b19, - 0x082b190819080808, 0x082b191908080808, 0x082b191919080819, 0x082b19191919082b, - 0x082b19192b192b19, 0x082b192b08080819, 0x082b192b08192b2b, 0x082b192b2b2b192b, - 0x082b2b0808080808, 0x082b2b0808082b08, 0x082b2b0808082b2b, 0x082b2b08082b0808, - 0x082b2b0819191919, 0x082b2b082b082b08, 0x082b2b082b2b082b, 0x082b2b19192b2b08, - 0x082b2b192b190808, 0x082b2b2b08082b08, 0x082b2b2b082b0808, 0x082b2b2b2b08082b, - 0x082b2b2b2b082b08, 0x082b2b2b2b082b2b, 0x1908080808080819, 0x1908080808081908, - 0x190808080808192b, 0x1908080808082b19, 0x1908080808190808, 0x190808080819082b, - 0x1908080808191919, 0x1908080808192b08, 0x19080808082b0819, 0x19080808082b1908, - 0x1908080819080808, 0x190808081908082b, 0x1908080819081919, 0x1908080819082b08, - 0x1908080819082b2b, 0x1908080819190819, 0x1908080819191908, 0x19080808192b0808, - 0x19080808192b1919, 0x190808082b080819, 0x190808082b081908, 0x190808082b190808, - 0x1908081908080808, 0x190808190808082b, 0x1908081908081919, 0x1908081908082b08, - 0x1908081908190819, 0x1908081908191908, 0x19080819082b0808, 0x1908081919080819, - 0x1908081919081908, 0x1908081919190808, 0x190808192b080808, 0x190808192b081919, - 0x190808192b2b082b, 0x1908082b08080819, 0x1908082b08081908, 0x1908082b08190808, - 0x1908082b0819082b, 0x1908082b082b2b19, 0x1908082b19080808, 0x1908190808080808, - 0x190819080808082b, 0x1908190808081919, 0x1908190808082b08, 0x1908190808190819, - 0x1908190808191908, 0x1908190808192b19, 0x19081908082b0808, 0x1908190819080819, - 0x1908190819081908, 0x1908190819190808, 0x190819082b080808, 0x190819082b191908, - 0x1908191908080819, 0x1908191908081908, 0x1908191908190808, 0x19081919082b1908, - 0x1908191919080808, 0x190819192b192b2b, 0x1908192b08080808, 0x1908192b08082b2b, - 0x1908192b19081908, 0x1908192b19190808, 0x19082b0808080819, 0x19082b0808081908, - 0x19082b0808190808, 0x19082b0819080808, 0x19082b0819081919, 0x19082b0819191908, - 0x19082b08192b082b, 0x19082b1908080808, 0x19082b1908190819, 0x19082b1919081908, - 0x19082b1919190808, 0x19082b19192b2b19, 0x19082b2b08081908, 0x1919080808080808, - 0x191908080808082b, 0x1919080808081919, 0x1919080808082b08, 0x1919080808190819, - 0x1919080808191908, 0x19190808082b0808, 0x19190808082b2b08, 0x1919080819080819, - 0x1919080819081908, 0x1919080819190808, 0x191908082b080808, 0x1919081908080819, - 0x1919081908081908, 0x1919081908190808, 0x1919081908191919, 0x1919081919080808, - 0x191908191908082b, 0x1919082b08080808, 0x1919082b19081908, 0x1919082b2b2b2b2b, - 0x1919190808080819, 0x1919190808081908, 0x1919190808190808, 0x19191908082b0819, - 0x1919190819080808, 0x19191908192b0808, 0x191919082b080819, 0x191919082b2b0819, - 0x1919191908080808, 0x1919191908082b08, 0x191919192b080808, 0x191919192b082b08, - 0x1919192b082b0819, 0x1919192b192b2b08, 0x1919192b2b2b0819, 0x19192b0808080808, - 0x19192b0808191908, 0x19192b0819080819, 0x19192b0819190808, 0x19192b082b192b19, - 0x19192b1908192b2b, 0x19192b1919080808, 0x19192b191908082b, 0x19192b2b2b081919, - 0x192b080808080819, 0x192b080808081908, 0x192b080808190808, 0x192b080819080808, - 0x192b080819191908, 0x192b0808192b082b, 0x192b08082b08192b, 0x192b08082b2b2b19, - 0x192b081908080808, 0x192b082b082b1908, 0x192b082b19082b2b, 0x192b082b2b19082b, - 0x192b190808080808, 0x192b19080819192b, 0x192b191908190808, 0x192b191919080808, - 0x192b191919081919, 0x192b19192b2b1908, 0x192b2b0808080819, 0x192b2b08192b2b2b, - 0x192b2b19082b1919, 0x192b2b2b0808192b, 0x192b2b2b19191908, 0x192b2b2b192b082b, - 0x2b08080808080808, 0x2b0808080808082b, 0x2b08080808081919, 0x2b08080808082b08, - 0x2b08080808190819, 0x2b08080808191908, 0x2b080808082b0808, 0x2b080808082b2b2b, - 0x2b08080819080819, 0x2b08080819081908, 0x2b08080819190808, 0x2b0808082b080808, - 0x2b0808082b08082b, 0x2b0808082b2b2b08, 0x2b0808082b2b2b2b, 0x2b08081908080819, - 0x2b08081908081908, 0x2b0808190808192b, 0x2b08081908190808, 0x2b08081919080808, - 0x2b08081919190819, 0x2b08081919192b19, 0x2b08082b08080808, 0x2b08082b082b0808, - 0x2b08082b2b080808, 0x2b08082b2b08082b, 0x2b08082b2b2b0808, 0x2b08082b2b2b2b08, - 0x2b08190808080819, 0x2b08190808081908, 0x2b08190808190808, 0x2b0819080819082b, - 0x2b08190808191919, 0x2b08190819080808, 0x2b081908192b0808, 0x2b0819082b082b19, - 0x2b08191908080808, 0x2b08191919081908, 0x2b0819192b2b1919, 0x2b08192b08192b08, - 0x2b08192b192b2b2b, 0x2b082b0808080808, 0x2b082b0808082b08, 0x2b082b08082b1919, - 0x2b082b0819192b2b, 0x2b082b082b080808, 0x2b082b082b08082b, 0x2b082b082b2b2b08, - 0x2b082b190808192b, 0x2b082b2b082b082b, 0x2b082b2b2b080808, 0x2b082b2b2b082b08, - 0x2b082b2b2b19192b, 0x2b082b2b2b2b2b08, 0x2b19080808080819, 0x2b19080808081908, - 0x2b19080808190808, 0x2b19080819080808, 0x2b1908081919192b, 0x2b1908082b081908, - 0x2b19081908080808, 0x2b190819082b082b, 0x2b190819192b1908, 0x2b19082b1919192b, - 0x2b19082b2b082b19, 0x2b19190808080808, 0x2b19190808081919, 0x2b19190819081908, - 0x2b19190819190808, 0x2b19190819192b08, 0x2b191919082b2b19, 0x2b1919192b190808, - 0x2b1919192b19082b, 0x2b19192b19080819, 0x2b192b0819190819, 0x2b192b082b2b192b, - 0x2b192b1919082b19, 0x2b192b2b08191919, 0x2b192b2b192b0808, 0x2b2b080808080808, - 0x2b2b08080808082b, 0x2b2b080808082b08, 0x2b2b080808082b2b, 0x2b2b0808082b0808, - 0x2b2b0808082b2b2b, 0x2b2b08082b2b0808, 0x2b2b081919190819, 0x2b2b081919192b19, - 0x2b2b08192b2b192b, 0x2b2b082b08080808, 0x2b2b082b0808082b, 0x2b2b082b08082b08, - 0x2b2b082b082b2b2b, 0x2b2b082b2b080808, 0x2b2b082b2b2b0808, 0x2b2b190819080808, - 0x2b2b19082b191919, 0x2b2b192b192b1919, 0x2b2b192b2b192b08, 0x2b2b2b0808082b2b, - 0x2b2b2b08082b0808, 0x2b2b2b08082b082b, 0x2b2b2b08082b2b08, 0x2b2b2b082b2b0808, - 0x2b2b2b082b2b2b08, 0x2b2b2b1908081908, 0x2b2b2b192b081908, 0x2b2b2b192b08192b, - 0x2b2b2b2b082b2b08, 0x2b2b2b2b082b2b2b, 0x2b2b2b2b2b190819, 0x2b2b2b2b2b2b2b2b, -}; - -static const __device__ uint64_t iq2s_grid[1024] = { - 0x0808080808080808, 0x080808080808082b, 0x0808080808081919, 0x0808080808082b08, - 0x0808080808082b2b, 0x0808080808190819, 0x0808080808191908, 0x080808080819192b, - 0x0808080808192b19, 0x08080808082b0808, 0x08080808082b082b, 0x08080808082b1919, - 0x08080808082b2b08, 0x0808080819080819, 0x0808080819081908, 0x080808081908192b, - 0x0808080819082b19, 0x0808080819190808, 0x080808081919082b, 0x0808080819191919, - 0x0808080819192b08, 0x08080808192b0819, 0x08080808192b1908, 0x08080808192b192b, - 0x08080808192b2b19, 0x080808082b080808, 0x080808082b08082b, 0x080808082b081919, - 0x080808082b082b08, 0x080808082b190819, 0x080808082b191908, 0x080808082b2b0808, - 0x080808082b2b1919, 0x080808082b2b2b2b, 0x0808081908080819, 0x0808081908081908, - 0x080808190808192b, 0x0808081908082b19, 0x0808081908190808, 0x080808190819082b, - 0x0808081908191919, 0x0808081908192b08, 0x08080819082b0819, 0x08080819082b1908, - 0x0808081919080808, 0x080808191908082b, 0x0808081919081919, 0x0808081919082b08, - 0x0808081919190819, 0x0808081919191908, 0x080808191919192b, 0x0808081919192b19, - 0x08080819192b0808, 0x08080819192b1919, 0x08080819192b2b08, 0x080808192b080819, - 0x080808192b081908, 0x080808192b190808, 0x080808192b19082b, 0x080808192b191919, - 0x080808192b2b0819, 0x080808192b2b1908, 0x0808082b08080808, 0x0808082b0808082b, - 0x0808082b08081919, 0x0808082b08082b08, 0x0808082b08190819, 0x0808082b08191908, - 0x0808082b082b0808, 0x0808082b082b2b2b, 0x0808082b19080819, 0x0808082b19081908, - 0x0808082b1908192b, 0x0808082b19082b19, 0x0808082b19190808, 0x0808082b19191919, - 0x0808082b2b080808, 0x0808082b2b081919, 0x0808082b2b082b2b, 0x0808082b2b191908, - 0x0808082b2b2b082b, 0x0808190808080819, 0x0808190808081908, 0x080819080808192b, - 0x0808190808082b19, 0x0808190808190808, 0x080819080819082b, 0x0808190808191919, - 0x0808190808192b08, 0x08081908082b0819, 0x08081908082b1908, 0x08081908082b192b, - 0x08081908082b2b19, 0x0808190819080808, 0x080819081908082b, 0x0808190819081919, - 0x0808190819082b08, 0x0808190819082b2b, 0x0808190819190819, 0x0808190819191908, - 0x080819081919192b, 0x0808190819192b19, 0x08081908192b0808, 0x08081908192b082b, - 0x08081908192b1919, 0x080819082b080819, 0x080819082b081908, 0x080819082b08192b, - 0x080819082b082b19, 0x080819082b190808, 0x080819082b191919, 0x080819082b192b08, - 0x080819082b2b0819, 0x080819082b2b1908, 0x0808191908080808, 0x080819190808082b, - 0x0808191908081919, 0x0808191908082b08, 0x0808191908082b2b, 0x0808191908190819, - 0x0808191908191908, 0x080819190819192b, 0x0808191908192b19, 0x08081919082b0808, - 0x08081919082b1919, 0x08081919082b2b08, 0x0808191919080819, 0x0808191919081908, - 0x080819191908192b, 0x0808191919082b19, 0x0808191919190808, 0x080819191919082b, - 0x0808191919191919, 0x0808191919192b08, 0x08081919192b0819, 0x08081919192b1908, - 0x080819192b080808, 0x080819192b08082b, 0x080819192b081919, 0x080819192b082b08, - 0x080819192b190819, 0x080819192b191908, 0x080819192b2b0808, 0x0808192b08080819, - 0x0808192b08081908, 0x0808192b0808192b, 0x0808192b08082b19, 0x0808192b08190808, - 0x0808192b08191919, 0x0808192b19080808, 0x0808192b19081919, 0x0808192b19082b08, - 0x0808192b19190819, 0x0808192b19191908, 0x0808192b192b0808, 0x0808192b2b080819, - 0x0808192b2b081908, 0x0808192b2b190808, 0x08082b0808080808, 0x08082b080808082b, - 0x08082b0808081919, 0x08082b0808082b08, 0x08082b0808190819, 0x08082b0808191908, - 0x08082b080819192b, 0x08082b0808192b19, 0x08082b08082b0808, 0x08082b08082b1919, - 0x08082b08082b2b2b, 0x08082b0819080819, 0x08082b0819081908, 0x08082b081908192b, - 0x08082b0819082b19, 0x08082b0819190808, 0x08082b081919082b, 0x08082b0819191919, - 0x08082b0819192b08, 0x08082b08192b0819, 0x08082b08192b1908, 0x08082b082b080808, - 0x08082b082b081919, 0x08082b082b191908, 0x08082b082b2b2b2b, 0x08082b1908080819, - 0x08082b1908081908, 0x08082b1908190808, 0x08082b190819082b, 0x08082b1908191919, - 0x08082b1908192b08, 0x08082b19082b0819, 0x08082b1919080808, 0x08082b1919081919, - 0x08082b1919082b08, 0x08082b1919190819, 0x08082b1919191908, 0x08082b19192b0808, - 0x08082b192b080819, 0x08082b192b190808, 0x08082b2b08080808, 0x08082b2b08190819, - 0x08082b2b08191908, 0x08082b2b082b082b, 0x08082b2b082b2b08, 0x08082b2b082b2b2b, - 0x08082b2b19190808, 0x08082b2b2b192b19, 0x0819080808080819, 0x0819080808081908, - 0x081908080808192b, 0x0819080808082b19, 0x0819080808190808, 0x081908080819082b, - 0x0819080808191919, 0x0819080808192b08, 0x08190808082b0819, 0x08190808082b1908, - 0x08190808082b192b, 0x0819080819080808, 0x081908081908082b, 0x0819080819081919, - 0x0819080819082b08, 0x0819080819190819, 0x0819080819191908, 0x081908081919192b, - 0x0819080819192b19, 0x08190808192b0808, 0x08190808192b082b, 0x08190808192b1919, - 0x08190808192b2b08, 0x081908082b080819, 0x081908082b081908, 0x081908082b08192b, - 0x081908082b190808, 0x081908082b191919, 0x081908082b192b08, 0x081908082b2b0819, - 0x081908082b2b1908, 0x0819081908080808, 0x081908190808082b, 0x0819081908081919, - 0x0819081908082b08, 0x0819081908082b2b, 0x0819081908190819, 0x0819081908191908, - 0x081908190819192b, 0x0819081908192b19, 0x08190819082b0808, 0x08190819082b082b, - 0x08190819082b1919, 0x08190819082b2b08, 0x0819081919080819, 0x0819081919081908, - 0x081908191908192b, 0x0819081919082b19, 0x0819081919190808, 0x081908191919082b, - 0x0819081919191919, 0x0819081919192b08, 0x08190819192b0819, 0x08190819192b1908, - 0x081908192b080808, 0x081908192b08082b, 0x081908192b081919, 0x081908192b082b08, - 0x081908192b190819, 0x081908192b191908, 0x0819082b08080819, 0x0819082b08081908, - 0x0819082b08082b19, 0x0819082b08190808, 0x0819082b08191919, 0x0819082b082b0819, - 0x0819082b082b1908, 0x0819082b19080808, 0x0819082b19081919, 0x0819082b19190819, - 0x0819082b19191908, 0x0819082b2b080819, 0x0819082b2b081908, 0x0819082b2b190808, - 0x0819190808080808, 0x081919080808082b, 0x0819190808081919, 0x0819190808082b08, - 0x0819190808190819, 0x0819190808191908, 0x081919080819192b, 0x0819190808192b19, - 0x08191908082b0808, 0x08191908082b1919, 0x08191908082b2b08, 0x0819190819080819, - 0x0819190819081908, 0x081919081908192b, 0x0819190819082b19, 0x0819190819190808, - 0x081919081919082b, 0x0819190819191919, 0x0819190819192b08, 0x08191908192b0819, - 0x08191908192b1908, 0x081919082b080808, 0x081919082b08082b, 0x081919082b081919, - 0x081919082b082b08, 0x081919082b190819, 0x081919082b191908, 0x081919082b2b0808, - 0x0819191908080819, 0x0819191908081908, 0x081919190808192b, 0x0819191908082b19, - 0x0819191908190808, 0x081919190819082b, 0x0819191908191919, 0x0819191908192b08, - 0x08191919082b0819, 0x08191919082b1908, 0x0819191919080808, 0x081919191908082b, - 0x0819191919081919, 0x0819191919082b08, 0x0819191919190819, 0x0819191919191908, - 0x08191919192b0808, 0x081919192b080819, 0x081919192b081908, 0x081919192b190808, - 0x0819192b08080808, 0x0819192b08081919, 0x0819192b08082b08, 0x0819192b08190819, - 0x0819192b08191908, 0x0819192b082b0808, 0x0819192b19080819, 0x0819192b19081908, - 0x0819192b19190808, 0x0819192b2b080808, 0x0819192b2b2b2b2b, 0x08192b0808080819, - 0x08192b0808081908, 0x08192b080808192b, 0x08192b0808082b19, 0x08192b0808190808, - 0x08192b0808191919, 0x08192b0808192b08, 0x08192b08082b0819, 0x08192b0819080808, - 0x08192b081908082b, 0x08192b0819081919, 0x08192b0819082b08, 0x08192b0819190819, - 0x08192b0819191908, 0x08192b08192b0808, 0x08192b082b080819, 0x08192b082b081908, - 0x08192b1908080808, 0x08192b190808082b, 0x08192b1908081919, 0x08192b1908082b08, - 0x08192b1908190819, 0x08192b1908191908, 0x08192b19082b0808, 0x08192b1919080819, - 0x08192b1919081908, 0x08192b1919190808, 0x08192b19192b2b19, 0x08192b192b2b082b, - 0x08192b2b08081908, 0x08192b2b08190808, 0x08192b2b19080808, 0x08192b2b1919192b, - 0x082b080808080808, 0x082b08080808082b, 0x082b080808081919, 0x082b080808082b08, - 0x082b080808190819, 0x082b080808191908, 0x082b08080819192b, 0x082b080808192b19, - 0x082b0808082b0808, 0x082b0808082b1919, 0x082b0808082b2b2b, 0x082b080819080819, - 0x082b080819081908, 0x082b080819190808, 0x082b08081919082b, 0x082b080819191919, - 0x082b0808192b1908, 0x082b08082b080808, 0x082b08082b082b2b, 0x082b08082b191908, - 0x082b08082b2b2b2b, 0x082b081908080819, 0x082b081908081908, 0x082b081908190808, - 0x082b08190819082b, 0x082b081908191919, 0x082b0819082b0819, 0x082b081919080808, - 0x082b08191908082b, 0x082b081919081919, 0x082b081919190819, 0x082b081919191908, - 0x082b0819192b0808, 0x082b08192b080819, 0x082b08192b081908, 0x082b08192b190808, - 0x082b082b08080808, 0x082b082b08082b2b, 0x082b082b082b082b, 0x082b082b082b2b08, - 0x082b082b082b2b2b, 0x082b082b19081908, 0x082b082b19190808, 0x082b082b2b082b08, - 0x082b082b2b082b2b, 0x082b082b2b2b2b08, 0x082b190808080819, 0x082b190808081908, - 0x082b19080808192b, 0x082b190808082b19, 0x082b190808190808, 0x082b190808191919, - 0x082b190808192b08, 0x082b1908082b0819, 0x082b1908082b1908, 0x082b190819080808, - 0x082b19081908082b, 0x082b190819081919, 0x082b190819082b08, 0x082b190819190819, - 0x082b190819191908, 0x082b1908192b0808, 0x082b19082b080819, 0x082b19082b081908, - 0x082b19082b190808, 0x082b191908080808, 0x082b191908081919, 0x082b191908082b08, - 0x082b191908190819, 0x082b191908191908, 0x082b1919082b0808, 0x082b191919080819, - 0x082b191919081908, 0x082b191919190808, 0x082b1919192b192b, 0x082b19192b080808, - 0x082b192b08080819, 0x082b192b08081908, 0x082b192b08190808, 0x082b192b19080808, - 0x082b192b19192b19, 0x082b2b0808080808, 0x082b2b0808081919, 0x082b2b0808190819, - 0x082b2b0808191908, 0x082b2b0819080819, 0x082b2b0819081908, 0x082b2b0819190808, - 0x082b2b082b082b2b, 0x082b2b082b2b2b2b, 0x082b2b1908080819, 0x082b2b1908081908, - 0x082b2b1908190808, 0x082b2b192b191919, 0x082b2b2b08082b2b, 0x082b2b2b082b082b, - 0x082b2b2b192b1908, 0x082b2b2b2b082b08, 0x082b2b2b2b082b2b, 0x1908080808080819, - 0x1908080808081908, 0x190808080808192b, 0x1908080808082b19, 0x1908080808190808, - 0x190808080819082b, 0x1908080808191919, 0x1908080808192b08, 0x1908080808192b2b, - 0x19080808082b0819, 0x19080808082b1908, 0x19080808082b192b, 0x1908080819080808, - 0x190808081908082b, 0x1908080819081919, 0x1908080819082b08, 0x1908080819082b2b, - 0x1908080819190819, 0x1908080819191908, 0x190808081919192b, 0x1908080819192b19, - 0x19080808192b0808, 0x19080808192b082b, 0x19080808192b1919, 0x190808082b080819, - 0x190808082b081908, 0x190808082b190808, 0x190808082b191919, 0x190808082b192b08, - 0x190808082b2b0819, 0x190808082b2b1908, 0x1908081908080808, 0x190808190808082b, - 0x1908081908081919, 0x1908081908082b08, 0x1908081908190819, 0x1908081908191908, - 0x190808190819192b, 0x1908081908192b19, 0x19080819082b0808, 0x19080819082b082b, - 0x19080819082b1919, 0x1908081919080819, 0x1908081919081908, 0x190808191908192b, - 0x1908081919082b19, 0x1908081919190808, 0x190808191919082b, 0x1908081919191919, - 0x1908081919192b08, 0x19080819192b0819, 0x19080819192b1908, 0x190808192b080808, - 0x190808192b08082b, 0x190808192b081919, 0x190808192b082b08, 0x190808192b190819, - 0x190808192b191908, 0x190808192b2b0808, 0x1908082b08080819, 0x1908082b08081908, - 0x1908082b08190808, 0x1908082b0819082b, 0x1908082b08191919, 0x1908082b08192b08, - 0x1908082b082b1908, 0x1908082b19080808, 0x1908082b19081919, 0x1908082b19082b08, - 0x1908082b19190819, 0x1908082b19191908, 0x1908082b192b0808, 0x1908082b2b080819, - 0x1908082b2b081908, 0x1908190808080808, 0x190819080808082b, 0x1908190808081919, - 0x1908190808082b08, 0x1908190808082b2b, 0x1908190808190819, 0x1908190808191908, - 0x190819080819192b, 0x1908190808192b19, 0x19081908082b0808, 0x19081908082b082b, - 0x19081908082b1919, 0x19081908082b2b08, 0x1908190819080819, 0x1908190819081908, - 0x190819081908192b, 0x1908190819082b19, 0x1908190819190808, 0x190819081919082b, - 0x1908190819191919, 0x1908190819192b08, 0x19081908192b0819, 0x19081908192b1908, - 0x190819082b080808, 0x190819082b08082b, 0x190819082b081919, 0x190819082b082b08, - 0x190819082b190819, 0x190819082b191908, 0x190819082b2b0808, 0x1908191908080819, - 0x1908191908081908, 0x190819190808192b, 0x1908191908082b19, 0x1908191908190808, - 0x190819190819082b, 0x1908191908191919, 0x1908191908192b08, 0x19081919082b0819, - 0x19081919082b1908, 0x1908191919080808, 0x190819191908082b, 0x1908191919081919, - 0x1908191919082b08, 0x1908191919190819, 0x1908191919191908, 0x19081919192b0808, - 0x19081919192b2b2b, 0x190819192b080819, 0x190819192b081908, 0x190819192b190808, - 0x1908192b08080808, 0x1908192b0808082b, 0x1908192b08081919, 0x1908192b08082b08, - 0x1908192b08190819, 0x1908192b08191908, 0x1908192b082b0808, 0x1908192b19080819, - 0x1908192b19081908, 0x1908192b19190808, 0x1908192b2b080808, 0x1908192b2b2b1919, - 0x19082b0808080819, 0x19082b0808081908, 0x19082b0808082b19, 0x19082b0808190808, - 0x19082b080819082b, 0x19082b0808191919, 0x19082b0808192b08, 0x19082b08082b0819, - 0x19082b08082b1908, 0x19082b0819080808, 0x19082b081908082b, 0x19082b0819081919, - 0x19082b0819082b08, 0x19082b0819190819, 0x19082b0819191908, 0x19082b08192b0808, - 0x19082b082b081908, 0x19082b082b190808, 0x19082b1908080808, 0x19082b190808082b, - 0x19082b1908081919, 0x19082b1908082b08, 0x19082b1908190819, 0x19082b1908191908, - 0x19082b19082b0808, 0x19082b1919080819, 0x19082b1919081908, 0x19082b1919190808, - 0x19082b192b080808, 0x19082b192b19192b, 0x19082b2b08080819, 0x19082b2b08081908, - 0x19082b2b08190808, 0x19082b2b19080808, 0x1919080808080808, 0x191908080808082b, - 0x1919080808081919, 0x1919080808082b08, 0x1919080808190819, 0x1919080808191908, - 0x191908080819192b, 0x1919080808192b19, 0x19190808082b0808, 0x19190808082b082b, - 0x19190808082b1919, 0x19190808082b2b08, 0x1919080819080819, 0x1919080819081908, - 0x191908081908192b, 0x1919080819082b19, 0x1919080819190808, 0x191908081919082b, - 0x1919080819191919, 0x1919080819192b08, 0x19190808192b0819, 0x19190808192b1908, - 0x191908082b080808, 0x191908082b08082b, 0x191908082b081919, 0x191908082b082b08, - 0x191908082b190819, 0x191908082b191908, 0x1919081908080819, 0x1919081908081908, - 0x191908190808192b, 0x1919081908082b19, 0x1919081908190808, 0x191908190819082b, - 0x1919081908191919, 0x1919081908192b08, 0x19190819082b0819, 0x19190819082b1908, - 0x1919081919080808, 0x191908191908082b, 0x1919081919081919, 0x1919081919082b08, - 0x1919081919190819, 0x1919081919191908, 0x19190819192b0808, 0x191908192b080819, - 0x191908192b081908, 0x191908192b190808, 0x1919082b08080808, 0x1919082b08081919, - 0x1919082b08082b08, 0x1919082b08190819, 0x1919082b08191908, 0x1919082b082b0808, - 0x1919082b19080819, 0x1919082b19081908, 0x1919082b19190808, 0x1919082b192b2b19, - 0x1919082b2b080808, 0x1919190808080819, 0x1919190808081908, 0x191919080808192b, - 0x1919190808082b19, 0x1919190808190808, 0x191919080819082b, 0x1919190808191919, - 0x1919190808192b08, 0x19191908082b0819, 0x19191908082b1908, 0x1919190819080808, - 0x191919081908082b, 0x1919190819081919, 0x1919190819082b08, 0x1919190819190819, - 0x1919190819191908, 0x19191908192b0808, 0x191919082b080819, 0x191919082b081908, - 0x191919082b190808, 0x1919191908080808, 0x191919190808082b, 0x1919191908081919, - 0x1919191908082b08, 0x1919191908190819, 0x1919191908191908, 0x19191919082b0808, - 0x1919191919080819, 0x1919191919081908, 0x1919191919190808, 0x191919192b080808, - 0x1919192b08080819, 0x1919192b08081908, 0x1919192b08190808, 0x1919192b082b192b, - 0x1919192b19080808, 0x19192b0808080808, 0x19192b080808082b, 0x19192b0808081919, - 0x19192b0808082b08, 0x19192b0808190819, 0x19192b0808191908, 0x19192b08082b0808, - 0x19192b0819080819, 0x19192b0819081908, 0x19192b0819190808, 0x19192b0819192b2b, - 0x19192b082b080808, 0x19192b1908080819, 0x19192b1908081908, 0x19192b1908190808, - 0x19192b1919080808, 0x19192b2b08080808, 0x19192b2b08192b19, 0x19192b2b2b081919, - 0x19192b2b2b2b2b08, 0x192b080808080819, 0x192b080808081908, 0x192b08080808192b, - 0x192b080808190808, 0x192b08080819082b, 0x192b080808191919, 0x192b080808192b08, - 0x192b0808082b0819, 0x192b0808082b1908, 0x192b080819080808, 0x192b080819081919, - 0x192b080819082b08, 0x192b080819190819, 0x192b080819191908, 0x192b0808192b0808, - 0x192b08082b081908, 0x192b08082b190808, 0x192b081908080808, 0x192b08190808082b, - 0x192b081908081919, 0x192b081908082b08, 0x192b081908190819, 0x192b081908191908, - 0x192b0819082b0808, 0x192b081919080819, 0x192b081919081908, 0x192b081919190808, - 0x192b08192b080808, 0x192b08192b192b19, 0x192b082b08081908, 0x192b082b08190808, - 0x192b082b19080808, 0x192b082b1919192b, 0x192b082b2b2b0819, 0x192b190808080808, - 0x192b190808081919, 0x192b190808082b08, 0x192b190808190819, 0x192b190808191908, - 0x192b1908082b0808, 0x192b190819080819, 0x192b190819081908, 0x192b190819190808, - 0x192b19082b080808, 0x192b191908080819, 0x192b191908081908, 0x192b191908190808, - 0x192b191919080808, 0x192b191919082b2b, 0x192b1919192b2b08, 0x192b19192b19082b, - 0x192b192b08080808, 0x192b192b2b191908, 0x192b2b0808080819, 0x192b2b0808081908, - 0x192b2b0808190808, 0x192b2b08192b1919, 0x192b2b082b192b08, 0x192b2b1908080808, - 0x192b2b19082b2b2b, 0x192b2b2b1908082b, 0x192b2b2b2b2b0819, 0x2b08080808080808, - 0x2b0808080808082b, 0x2b08080808081919, 0x2b08080808082b08, 0x2b08080808190819, - 0x2b08080808191908, 0x2b08080808192b19, 0x2b080808082b0808, 0x2b080808082b1919, - 0x2b08080819080819, 0x2b08080819081908, 0x2b08080819190808, 0x2b0808081919082b, - 0x2b08080819191919, 0x2b08080819192b08, 0x2b080808192b0819, 0x2b0808082b080808, - 0x2b0808082b081919, 0x2b0808082b190819, 0x2b0808082b191908, 0x2b08081908080819, - 0x2b08081908081908, 0x2b08081908082b19, 0x2b08081908190808, 0x2b0808190819082b, - 0x2b08081908191919, 0x2b08081908192b08, 0x2b080819082b0819, 0x2b080819082b1908, - 0x2b08081919080808, 0x2b0808191908082b, 0x2b08081919081919, 0x2b08081919082b08, - 0x2b08081919190819, 0x2b08081919191908, 0x2b0808192b080819, 0x2b0808192b081908, - 0x2b0808192b190808, 0x2b0808192b2b2b19, 0x2b08082b08080808, 0x2b08082b08081919, - 0x2b08082b08082b2b, 0x2b08082b08190819, 0x2b08082b08191908, 0x2b08082b19080819, - 0x2b08082b19081908, 0x2b08082b19190808, 0x2b08190808080819, 0x2b08190808081908, - 0x2b0819080808192b, 0x2b08190808082b19, 0x2b08190808190808, 0x2b0819080819082b, - 0x2b08190808191919, 0x2b08190808192b08, 0x2b081908082b0819, 0x2b08190819080808, - 0x2b0819081908082b, 0x2b08190819081919, 0x2b08190819082b08, 0x2b08190819190819, - 0x2b08190819191908, 0x2b081908192b0808, 0x2b0819082b080819, 0x2b0819082b081908, - 0x2b0819082b190808, 0x2b08191908080808, 0x2b0819190808082b, 0x2b08191908081919, - 0x2b08191908082b08, 0x2b08191908190819, 0x2b08191908191908, 0x2b081919082b0808, - 0x2b08191919080819, 0x2b08191919081908, 0x2b08191919190808, 0x2b0819192b080808, - 0x2b0819192b082b2b, 0x2b08192b08080819, 0x2b08192b08081908, 0x2b08192b08190808, - 0x2b08192b082b2b19, 0x2b08192b19080808, 0x2b082b0808080808, 0x2b082b0808081919, - 0x2b082b0808190819, 0x2b082b0808191908, 0x2b082b0819080819, 0x2b082b0819081908, - 0x2b082b0819190808, 0x2b082b082b2b082b, 0x2b082b1908080819, 0x2b082b1908081908, - 0x2b082b1919080808, 0x2b082b19192b1919, 0x2b082b2b082b082b, 0x2b082b2b19192b08, - 0x2b082b2b19192b2b, 0x2b082b2b2b08082b, 0x2b082b2b2b2b082b, 0x2b19080808080819, - 0x2b19080808081908, 0x2b19080808082b19, 0x2b19080808190808, 0x2b1908080819082b, - 0x2b19080808191919, 0x2b19080808192b08, 0x2b190808082b1908, 0x2b19080819080808, - 0x2b1908081908082b, 0x2b19080819081919, 0x2b19080819082b08, 0x2b19080819190819, - 0x2b19080819191908, 0x2b190808192b0808, 0x2b1908082b080819, 0x2b1908082b081908, - 0x2b1908082b190808, 0x2b19081908080808, 0x2b19081908081919, 0x2b19081908190819, - 0x2b19081908191908, 0x2b19081919080819, 0x2b19081919081908, 0x2b19081919190808, - 0x2b19081919192b2b, 0x2b19082b08080819, 0x2b19082b08081908, 0x2b19082b08190808, - 0x2b19082b19080808, 0x2b19082b2b2b192b, 0x2b19190808080808, 0x2b1919080808082b, - 0x2b19190808081919, 0x2b19190808082b08, 0x2b19190808190819, 0x2b19190808191908, - 0x2b191908082b0808, 0x2b19190819080819, 0x2b19190819081908, 0x2b19190819190808, - 0x2b1919082b080808, 0x2b1919082b19192b, 0x2b19191908080819, 0x2b19191908081908, - 0x2b19191908190808, 0x2b19191919080808, 0x2b1919192b192b08, 0x2b1919192b2b0819, - 0x2b19192b08080808, 0x2b19192b1908192b, 0x2b19192b192b1908, 0x2b192b0808080819, - 0x2b192b0808081908, 0x2b192b0808190808, 0x2b192b08082b192b, 0x2b192b0819080808, - 0x2b192b082b2b2b19, 0x2b192b1908080808, 0x2b192b1919082b19, 0x2b192b191919082b, - 0x2b192b2b2b190808, 0x2b2b080808080808, 0x2b2b080808081919, 0x2b2b080808082b2b, - 0x2b2b080808191908, 0x2b2b0808082b082b, 0x2b2b0808082b2b2b, 0x2b2b080819080819, - 0x2b2b080819081908, 0x2b2b080819190808, 0x2b2b08082b2b082b, 0x2b2b08082b2b2b2b, - 0x2b2b081919080808, 0x2b2b0819192b1919, 0x2b2b082b0808082b, 0x2b2b082b08082b2b, - 0x2b2b082b082b082b, 0x2b2b082b082b2b08, 0x2b2b082b082b2b2b, 0x2b2b082b2b08082b, - 0x2b2b082b2b082b08, 0x2b2b082b2b082b2b, 0x2b2b082b2b2b2b08, 0x2b2b190808080819, - 0x2b2b190808081908, 0x2b2b190808190808, 0x2b2b190819080808, 0x2b2b19082b082b19, - 0x2b2b19082b2b1908, 0x2b2b191908080808, 0x2b2b191908192b19, 0x2b2b192b19190819, - 0x2b2b2b0808082b2b, 0x2b2b2b08082b2b08, 0x2b2b2b082b2b082b, 0x2b2b2b1919191908, - 0x2b2b2b192b08192b, 0x2b2b2b2b08082b08, 0x2b2b2b2b08082b2b, 0x2b2b2b2b082b0808, - 0x2b2b2b2b082b082b, 0x2b2b2b2b082b2b08, 0x2b2b2b2b2b082b08, 0x2b2b2b2b2b2b2b2b, -}; - -static const __device__ uint32_t iq3xxs_grid[256] = { - 0x04040404, 0x04040414, 0x04040424, 0x04040c0c, 0x04040c1c, 0x04040c3e, 0x04041404, 0x04041414, - 0x04041c0c, 0x04042414, 0x04043e1c, 0x04043e2c, 0x040c040c, 0x040c041c, 0x040c0c04, 0x040c0c14, - 0x040c140c, 0x040c142c, 0x040c1c04, 0x040c1c14, 0x040c240c, 0x040c2c24, 0x040c3e04, 0x04140404, - 0x04140414, 0x04140424, 0x04140c0c, 0x04141404, 0x04141414, 0x04141c0c, 0x04141c1c, 0x04141c3e, - 0x04142c0c, 0x04142c3e, 0x04143e2c, 0x041c040c, 0x041c043e, 0x041c0c04, 0x041c0c14, 0x041c142c, - 0x041c3e04, 0x04240c1c, 0x04241c3e, 0x04242424, 0x04242c3e, 0x04243e1c, 0x04243e2c, 0x042c040c, - 0x042c043e, 0x042c1c14, 0x042c2c14, 0x04341c2c, 0x04343424, 0x043e0c04, 0x043e0c24, 0x043e0c34, - 0x043e241c, 0x043e340c, 0x0c04040c, 0x0c04041c, 0x0c040c04, 0x0c040c14, 0x0c04140c, 0x0c04141c, - 0x0c041c04, 0x0c041c14, 0x0c041c24, 0x0c04243e, 0x0c042c04, 0x0c0c0404, 0x0c0c0414, 0x0c0c0c0c, - 0x0c0c1404, 0x0c0c1414, 0x0c14040c, 0x0c14041c, 0x0c140c04, 0x0c140c14, 0x0c14140c, 0x0c141c04, - 0x0c143e14, 0x0c1c0404, 0x0c1c0414, 0x0c1c1404, 0x0c1c1c0c, 0x0c1c2434, 0x0c1c3434, 0x0c24040c, - 0x0c24042c, 0x0c242c04, 0x0c2c1404, 0x0c2c1424, 0x0c2c2434, 0x0c2c3e0c, 0x0c34042c, 0x0c3e1414, - 0x0c3e2404, 0x14040404, 0x14040414, 0x14040c0c, 0x14040c1c, 0x14041404, 0x14041414, 0x14041434, - 0x14041c0c, 0x14042414, 0x140c040c, 0x140c041c, 0x140c042c, 0x140c0c04, 0x140c0c14, 0x140c140c, - 0x140c1c04, 0x140c341c, 0x140c343e, 0x140c3e04, 0x14140404, 0x14140414, 0x14140c0c, 0x14140c3e, - 0x14141404, 0x14141414, 0x14141c3e, 0x14142404, 0x14142c2c, 0x141c040c, 0x141c0c04, 0x141c0c24, - 0x141c3e04, 0x141c3e24, 0x14241c2c, 0x14242c1c, 0x142c041c, 0x142c143e, 0x142c240c, 0x142c3e24, - 0x143e040c, 0x143e041c, 0x143e0c34, 0x143e242c, 0x1c04040c, 0x1c040c04, 0x1c040c14, 0x1c04140c, - 0x1c04141c, 0x1c042c04, 0x1c04342c, 0x1c043e14, 0x1c0c0404, 0x1c0c0414, 0x1c0c1404, 0x1c0c1c0c, - 0x1c0c2424, 0x1c0c2434, 0x1c14040c, 0x1c14041c, 0x1c140c04, 0x1c14142c, 0x1c142c14, 0x1c143e14, - 0x1c1c0c0c, 0x1c1c1c1c, 0x1c241c04, 0x1c24243e, 0x1c243e14, 0x1c2c0404, 0x1c2c0434, 0x1c2c1414, - 0x1c2c2c2c, 0x1c340c24, 0x1c341c34, 0x1c34341c, 0x1c3e1c1c, 0x1c3e3404, 0x24040424, 0x24040c3e, - 0x24041c2c, 0x24041c3e, 0x24042c1c, 0x24042c3e, 0x240c3e24, 0x24141404, 0x24141c3e, 0x24142404, - 0x24143404, 0x24143434, 0x241c043e, 0x241c242c, 0x24240424, 0x24242c0c, 0x24243424, 0x242c142c, - 0x242c241c, 0x242c3e04, 0x243e042c, 0x243e0c04, 0x243e0c14, 0x243e1c04, 0x2c040c14, 0x2c04240c, - 0x2c043e04, 0x2c0c0404, 0x2c0c0434, 0x2c0c1434, 0x2c0c2c2c, 0x2c140c24, 0x2c141c14, 0x2c143e14, - 0x2c1c0414, 0x2c1c2c1c, 0x2c240c04, 0x2c24141c, 0x2c24143e, 0x2c243e14, 0x2c2c0414, 0x2c2c1c0c, - 0x2c342c04, 0x2c3e1424, 0x2c3e2414, 0x34041424, 0x34042424, 0x34042434, 0x34043424, 0x340c140c, - 0x340c340c, 0x34140c3e, 0x34143424, 0x341c1c04, 0x341c1c34, 0x34242424, 0x342c042c, 0x342c2c14, - 0x34341c1c, 0x343e041c, 0x343e140c, 0x3e04041c, 0x3e04042c, 0x3e04043e, 0x3e040c04, 0x3e041c14, - 0x3e042c14, 0x3e0c1434, 0x3e0c2404, 0x3e140c14, 0x3e14242c, 0x3e142c14, 0x3e1c0404, 0x3e1c0c2c, - 0x3e1c1c1c, 0x3e1c3404, 0x3e24140c, 0x3e24240c, 0x3e2c0404, 0x3e2c0414, 0x3e2c1424, 0x3e341c04, -}; - -static const __device__ uint32_t iq3s_grid[512] = { - 0x01010101, 0x01010103, 0x01010105, 0x0101010b, 0x0101010f, 0x01010301, 0x01010303, 0x01010305, - 0x01010309, 0x0101030d, 0x01010501, 0x01010503, 0x0101050b, 0x01010707, 0x01010901, 0x01010905, - 0x0101090b, 0x0101090f, 0x01010b03, 0x01010b07, 0x01010d01, 0x01010d05, 0x01010f03, 0x01010f09, - 0x01010f0f, 0x01030101, 0x01030103, 0x01030105, 0x01030109, 0x01030301, 0x01030303, 0x0103030b, - 0x01030501, 0x01030507, 0x0103050f, 0x01030703, 0x0103070b, 0x01030909, 0x01030d03, 0x01030d0b, - 0x01030f05, 0x01050101, 0x01050103, 0x0105010b, 0x0105010f, 0x01050301, 0x01050307, 0x0105030d, - 0x01050503, 0x0105050b, 0x01050701, 0x01050709, 0x01050905, 0x0105090b, 0x0105090f, 0x01050b03, - 0x01050b07, 0x01050f01, 0x01050f07, 0x01070107, 0x01070303, 0x0107030b, 0x01070501, 0x01070505, - 0x01070703, 0x01070707, 0x0107070d, 0x01070909, 0x01070b01, 0x01070b05, 0x01070d0f, 0x01070f03, - 0x01070f0b, 0x01090101, 0x01090307, 0x0109030f, 0x01090503, 0x01090509, 0x01090705, 0x01090901, - 0x01090907, 0x01090b03, 0x01090f01, 0x010b0105, 0x010b0109, 0x010b0501, 0x010b0505, 0x010b050d, - 0x010b0707, 0x010b0903, 0x010b090b, 0x010b090f, 0x010b0d0d, 0x010b0f07, 0x010d010d, 0x010d0303, - 0x010d0307, 0x010d0703, 0x010d0b05, 0x010d0f03, 0x010f0101, 0x010f0105, 0x010f0109, 0x010f0501, - 0x010f0505, 0x010f050d, 0x010f0707, 0x010f0b01, 0x010f0b09, 0x03010101, 0x03010103, 0x03010105, - 0x03010109, 0x03010301, 0x03010303, 0x03010307, 0x0301030b, 0x0301030f, 0x03010501, 0x03010505, - 0x03010703, 0x03010709, 0x0301070d, 0x03010b09, 0x03010b0d, 0x03010d03, 0x03010f05, 0x03030101, - 0x03030103, 0x03030107, 0x0303010d, 0x03030301, 0x03030309, 0x03030503, 0x03030701, 0x03030707, - 0x03030903, 0x03030b01, 0x03030b05, 0x03030f01, 0x03030f0d, 0x03050101, 0x03050305, 0x0305030b, - 0x0305030f, 0x03050501, 0x03050509, 0x03050705, 0x03050901, 0x03050907, 0x03050b0b, 0x03050d01, - 0x03050f05, 0x03070103, 0x03070109, 0x0307010f, 0x03070301, 0x03070307, 0x03070503, 0x0307050f, - 0x03070701, 0x03070709, 0x03070903, 0x03070d05, 0x03070f01, 0x03090107, 0x0309010b, 0x03090305, - 0x03090309, 0x03090703, 0x03090707, 0x03090905, 0x0309090d, 0x03090b01, 0x03090b09, 0x030b0103, - 0x030b0301, 0x030b0307, 0x030b0503, 0x030b0701, 0x030b0705, 0x030b0b03, 0x030d0501, 0x030d0509, - 0x030d050f, 0x030d0909, 0x030d090d, 0x030f0103, 0x030f0107, 0x030f0301, 0x030f0305, 0x030f0503, - 0x030f070b, 0x030f0903, 0x030f0d05, 0x030f0f01, 0x05010101, 0x05010103, 0x05010107, 0x0501010b, - 0x0501010f, 0x05010301, 0x05010305, 0x05010309, 0x0501030d, 0x05010503, 0x05010507, 0x0501050f, - 0x05010701, 0x05010705, 0x05010903, 0x05010907, 0x0501090b, 0x05010b01, 0x05010b05, 0x05010d0f, - 0x05010f01, 0x05010f07, 0x05010f0b, 0x05030101, 0x05030105, 0x05030301, 0x05030307, 0x0503030f, - 0x05030505, 0x0503050b, 0x05030703, 0x05030709, 0x05030905, 0x05030b03, 0x05050103, 0x05050109, - 0x0505010f, 0x05050503, 0x05050507, 0x05050701, 0x0505070f, 0x05050903, 0x05050b07, 0x05050b0f, - 0x05050f03, 0x05050f09, 0x05070101, 0x05070105, 0x0507010b, 0x05070303, 0x05070505, 0x05070509, - 0x05070703, 0x05070707, 0x05070905, 0x05070b01, 0x05070d0d, 0x05090103, 0x0509010f, 0x05090501, - 0x05090507, 0x05090705, 0x0509070b, 0x05090903, 0x05090f05, 0x05090f0b, 0x050b0109, 0x050b0303, - 0x050b0505, 0x050b070f, 0x050b0901, 0x050b0b07, 0x050b0f01, 0x050d0101, 0x050d0105, 0x050d010f, - 0x050d0503, 0x050d0b0b, 0x050d0d03, 0x050f010b, 0x050f0303, 0x050f050d, 0x050f0701, 0x050f0907, - 0x050f0b01, 0x07010105, 0x07010303, 0x07010307, 0x0701030b, 0x0701030f, 0x07010505, 0x07010703, - 0x07010707, 0x0701070b, 0x07010905, 0x07010909, 0x0701090f, 0x07010b03, 0x07010d07, 0x07010f03, - 0x07030103, 0x07030107, 0x0703010b, 0x07030309, 0x07030503, 0x07030507, 0x07030901, 0x07030d01, - 0x07030f05, 0x07030f0d, 0x07050101, 0x07050305, 0x07050501, 0x07050705, 0x07050709, 0x07050b01, - 0x07070103, 0x07070301, 0x07070309, 0x07070503, 0x07070507, 0x0707050f, 0x07070701, 0x07070903, - 0x07070907, 0x0707090f, 0x07070b0b, 0x07070f07, 0x07090107, 0x07090303, 0x0709030d, 0x07090505, - 0x07090703, 0x07090b05, 0x07090d01, 0x07090d09, 0x070b0103, 0x070b0301, 0x070b0305, 0x070b050b, - 0x070b0705, 0x070b0909, 0x070b0b0d, 0x070b0f07, 0x070d030d, 0x070d0903, 0x070f0103, 0x070f0107, - 0x070f0501, 0x070f0505, 0x070f070b, 0x09010101, 0x09010109, 0x09010305, 0x09010501, 0x09010509, - 0x0901050f, 0x09010705, 0x09010903, 0x09010b01, 0x09010f01, 0x09030105, 0x0903010f, 0x09030303, - 0x09030307, 0x09030505, 0x09030701, 0x0903070b, 0x09030907, 0x09030b03, 0x09030b0b, 0x09050103, - 0x09050107, 0x09050301, 0x0905030b, 0x09050503, 0x09050707, 0x09050901, 0x09050b0f, 0x09050d05, - 0x09050f01, 0x09070109, 0x09070303, 0x09070307, 0x09070501, 0x09070505, 0x09070703, 0x0907070b, - 0x09090101, 0x09090105, 0x09090509, 0x0909070f, 0x09090901, 0x09090f03, 0x090b010b, 0x090b010f, - 0x090b0503, 0x090b0d05, 0x090d0307, 0x090d0709, 0x090d0d01, 0x090f0301, 0x090f030b, 0x090f0701, - 0x090f0907, 0x090f0b03, 0x0b010105, 0x0b010301, 0x0b010309, 0x0b010505, 0x0b010901, 0x0b010909, - 0x0b01090f, 0x0b010b05, 0x0b010d0d, 0x0b010f09, 0x0b030103, 0x0b030107, 0x0b03010b, 0x0b030305, - 0x0b030503, 0x0b030705, 0x0b030f05, 0x0b050101, 0x0b050303, 0x0b050507, 0x0b050701, 0x0b05070d, - 0x0b050b07, 0x0b070105, 0x0b07010f, 0x0b070301, 0x0b07050f, 0x0b070909, 0x0b070b03, 0x0b070d0b, - 0x0b070f07, 0x0b090103, 0x0b090109, 0x0b090501, 0x0b090705, 0x0b09090d, 0x0b0b0305, 0x0b0b050d, - 0x0b0b0b03, 0x0b0b0b07, 0x0b0d0905, 0x0b0f0105, 0x0b0f0109, 0x0b0f0505, 0x0d010303, 0x0d010307, - 0x0d01030b, 0x0d010703, 0x0d010707, 0x0d010d01, 0x0d030101, 0x0d030501, 0x0d03050f, 0x0d030d09, - 0x0d050305, 0x0d050709, 0x0d050905, 0x0d050b0b, 0x0d050d05, 0x0d050f01, 0x0d070101, 0x0d070309, - 0x0d070503, 0x0d070901, 0x0d09050b, 0x0d090907, 0x0d090d05, 0x0d0b0101, 0x0d0b0107, 0x0d0b0709, - 0x0d0b0d01, 0x0d0d010b, 0x0d0d0901, 0x0d0f0303, 0x0d0f0307, 0x0f010101, 0x0f010109, 0x0f01010f, - 0x0f010501, 0x0f010505, 0x0f01070d, 0x0f010901, 0x0f010b09, 0x0f010d05, 0x0f030105, 0x0f030303, - 0x0f030509, 0x0f030907, 0x0f03090b, 0x0f050103, 0x0f050109, 0x0f050301, 0x0f05030d, 0x0f050503, - 0x0f050701, 0x0f050b03, 0x0f070105, 0x0f070705, 0x0f07070b, 0x0f070b07, 0x0f090103, 0x0f09010b, - 0x0f090307, 0x0f090501, 0x0f090b01, 0x0f0b0505, 0x0f0b0905, 0x0f0d0105, 0x0f0d0703, 0x0f0f0101, -}; - -static const __device__ uint64_t iq1s_grid[512] = { - 0xffffffffffff0101, 0xffffffffff01ff00, 0xffffffffff010100, 0xffffffff00000000, - 0xffffffff01ff00ff, 0xffffffff01ff0001, 0xffffffff0101ffff, 0xffffffff0101ff01, - 0xffffff00ff000000, 0xffffff000000ff00, 0xffffff00000000ff, 0xffffff0000000100, - 0xffffff0000010000, 0xffffff0001000000, 0xffffff01ffff00ff, 0xffffff01ff01ff00, - 0xffffff01ff010100, 0xffffff0100000001, 0xffffff0101ffff00, 0xffffff0101ff0101, - 0xffffff0101010100, 0xffff00ffff00ff01, 0xffff00ffff0000ff, 0xffff00ff00ff0100, - 0xffff00ff0100ff00, 0xffff00ff010001ff, 0xffff0000ff0101ff, 0xffff000000ffff00, - 0xffff000000000000, 0xffff00000001ff01, 0xffff000001000101, 0xffff0000010100ff, - 0xffff0001ffff0100, 0xffff00010000ff00, 0xffff000100010101, 0xffff000101000000, - 0xffff01ffffff0000, 0xffff01ffff01ffff, 0xffff01ffff010100, 0xffff01ff00000000, - 0xffff01ff01ffffff, 0xffff01ff01ff0001, 0xffff01ff0101ffff, 0xffff01ff01010001, - 0xffff0100ffffff01, 0xffff01000000ffff, 0xffff010000000100, 0xffff010001ff01ff, - 0xffff010001000000, 0xffff0101ff000000, 0xffff0101000101ff, 0xffff010101ffff01, - 0xffff01010101ff00, 0xff00ffffff000000, 0xff00ffff00ffff00, 0xff00ffff00000001, - 0xff00ffff000001ff, 0xff00ffff01010000, 0xff00ff00ffff0000, 0xff00ff00ff00ff00, - 0xff00ff00ff0000ff, 0xff00ff00ff000100, 0xff00ff00ff010001, 0xff00ff0000ff0001, - 0xff00ff000000ffff, 0xff00ff0000000000, 0xff00ff000001ff00, 0xff00ff0000010100, - 0xff00ff0001ff0000, 0xff00ff000100ff00, 0xff00ff0001000100, 0xff00ff01ff000000, - 0xff00ff0100ff0000, 0xff00ff01000001ff, 0xff00ff0101010001, 0xff0000ff00000000, - 0xff0000ff0001ff00, 0xff0000ff00010100, 0xff000000ffff0101, 0xff000000ff000000, - 0xff000000ff01ff00, 0xff00000000ff0000, 0xff0000000000ff00, 0xff000000000000ff, - 0xff00000000000000, 0xff00000000000001, 0xff00000000000100, 0xff0000000001ffff, - 0xff00000000010000, 0xff00000001000000, 0xff00000001010100, 0xff000001ff00ff01, - 0xff000001ff0100ff, 0xff00000100000000, 0xff0000010001ff00, 0xff00000101ff0100, - 0xff0000010100ff00, 0xff0001ff00ff00ff, 0xff0001ff00000101, 0xff0001ff000100ff, - 0xff0001ff01000000, 0xff000100ff0001ff, 0xff0001000000ff01, 0xff00010000000000, - 0xff00010000010001, 0xff00010000010100, 0xff00010001ffff00, 0xff00010001ff0101, - 0xff00010001010000, 0xff000101ffffffff, 0xff000101ff000101, 0xff00010101ff00ff, - 0xff00010101000001, 0xff000101010100ff, 0xff01ffffff000101, 0xff01ffffff01ffff, - 0xff01ffffff01ff01, 0xff01ffffff0101ff, 0xff01ffff00000000, 0xff01ffff01ff0001, - 0xff01ffff0101ff01, 0xff01ff00ff000000, 0xff01ff0000ff0100, 0xff01ff000000ff01, - 0xff01ff0000010000, 0xff01ff00010000ff, 0xff01ff01ff01ff00, 0xff01ff0100000101, - 0xff0100ffffff0000, 0xff0100ffff010000, 0xff0100ff01ff00ff, 0xff0100ff01000100, - 0xff0100ff010100ff, 0xff010000ffffff01, 0xff01000000000000, 0xff0100000101ff00, - 0xff010001ffff00ff, 0xff010001ff000100, 0xff01000100ffff00, 0xff01000100010001, - 0xff01000101ff0001, 0xff010001010001ff, 0xff0101ffffffffff, 0xff0101ffff01ffff, - 0xff0101ffff010101, 0xff0101ff0000ff00, 0xff0101ff01010001, 0xff010100ff000000, - 0xff010100ff01ff01, 0xff01010000ff0001, 0xff01010000000100, 0xff01010001000000, - 0xff0101010100ffff, 0x00ffffff0000ff01, 0x00ffffff000000ff, 0x00ffffff00000100, - 0x00ffffff00010000, 0x00ffff00ffff0001, 0x00ffff00ff0000ff, 0x00ffff00ff000100, - 0x00ffff0000000000, 0x00ffff0001000100, 0x00ffff0001010001, 0x00ffff01ff00ff01, - 0x00ffff0100ff0100, 0x00ffff010000ff00, 0x00ffff01000100ff, 0x00ffff0101ff00ff, - 0x00ffff010101ff00, 0x00ff00ffffffffff, 0x00ff00ffffff01ff, 0x00ff00ffff000101, - 0x00ff00ff00000000, 0x00ff00ff000101ff, 0x00ff00ff01010101, 0x00ff0000ff000000, - 0x00ff0000ff01ffff, 0x00ff000000ff0000, 0x00ff00000000ff00, 0x00ff0000000000ff, - 0x00ff000000000000, 0x00ff000000000001, 0x00ff000000000100, 0x00ff000000010000, - 0x00ff000001ffff01, 0x00ff000001000000, 0x00ff0001ff000101, 0x00ff000100ffffff, - 0x00ff000100000000, 0x00ff0001010001ff, 0x00ff01ffff000000, 0x00ff01ff0001ff00, - 0x00ff01ff01ff0100, 0x00ff0100ff01ff01, 0x00ff010000ff00ff, 0x00ff010000ff0101, - 0x00ff010000000000, 0x00ff010000010101, 0x00ff01000100ff00, 0x00ff010001010000, - 0x00ff0101ffffff00, 0x00ff01010000ff01, 0x00ff010100000100, 0x00ff010101ff0000, - 0x0000ffffffff0100, 0x0000ffffff00ff00, 0x0000ffffff0000ff, 0x0000ffffff010000, - 0x0000ffff00000000, 0x0000ffff00010101, 0x0000ffff01ffff01, 0x0000ffff01000100, - 0x0000ff00ff000000, 0x0000ff00ff01ff00, 0x0000ff00ff0101ff, 0x0000ff0000ff0000, - 0x0000ff000000ff00, 0x0000ff00000000ff, 0x0000ff0000000000, 0x0000ff0000000001, - 0x0000ff0000000100, 0x0000ff0000010000, 0x0000ff0001ffffff, 0x0000ff0001ff01ff, - 0x0000ff0001000000, 0x0000ff000101ffff, 0x0000ff01ffff0101, 0x0000ff01ff010000, - 0x0000ff0100000000, 0x0000ff0101000101, 0x000000ffffff0001, 0x000000ffff000000, - 0x000000ff00ff0000, 0x000000ff0000ff00, 0x000000ff000000ff, 0x000000ff00000000, - 0x000000ff00000001, 0x000000ff00000100, 0x000000ff00010000, 0x000000ff01000000, - 0x000000ff0101ff00, 0x00000000ffff0000, 0x00000000ff00ff00, 0x00000000ff0000ff, - 0x00000000ff000000, 0x00000000ff000001, 0x00000000ff000100, 0x00000000ff010000, - 0x0000000000ffff00, 0x0000000000ff00ff, 0x0000000000ff0000, 0x0000000000ff0001, - 0x0000000000ff0100, 0x000000000000ffff, 0x000000000000ff00, 0x000000000000ff01, - 0x00000000000000ff, 0x0000000000000001, 0x00000000000001ff, 0x0000000000000100, - 0x0000000000000101, 0x000000000001ff00, 0x00000000000100ff, 0x0000000000010000, - 0x0000000000010001, 0x0000000000010100, 0x0000000001ff0000, 0x000000000100ff00, - 0x00000000010000ff, 0x0000000001000000, 0x0000000001000001, 0x0000000001000100, - 0x0000000001010000, 0x00000001ffff01ff, 0x00000001ff000000, 0x0000000100ff0000, - 0x000000010000ff00, 0x00000001000000ff, 0x0000000100000000, 0x0000000100000001, - 0x0000000100000100, 0x0000000100010000, 0x0000000101000000, 0x000001ffff00ff00, - 0x000001ffff010001, 0x000001ffff0101ff, 0x000001ff00ffff01, 0x000001ff0000ffff, - 0x000001ff00000000, 0x000001ff010000ff, 0x000001ff01010100, 0x00000100ffff0100, - 0x00000100ff000000, 0x0000010000ff0000, 0x000001000000ff00, 0x00000100000000ff, - 0x0000010000000000, 0x0000010000000001, 0x0000010000000100, 0x0000010000010000, - 0x0000010001000000, 0x000001000101ff01, 0x00000101ffff0001, 0x00000101ff01ffff, - 0x0000010100000000, 0x0000010101010100, 0x0001ffffff000000, 0x0001ffff00ffffff, - 0x0001ffff00000100, 0x0001ffff0001ff00, 0x0001ffff01000000, 0x0001ff00ffffff00, - 0x0001ff00ffff01ff, 0x0001ff00ff010000, 0x0001ff0000000000, 0x0001ff0000010001, - 0x0001ff0001ff0000, 0x0001ff0001010100, 0x0001ff01ff0000ff, 0x0001ff01ff000001, - 0x0001ff0100ffffff, 0x0001ff010001ffff, 0x0001ff01000101ff, 0x0001ff010100ff01, - 0x000100ffff00ffff, 0x000100ffff00ff01, 0x000100ffff000100, 0x000100ff00000000, - 0x000100ff000101ff, 0x000100ff01ff0101, 0x000100ff0100ffff, 0x000100ff01010101, - 0x00010000ff000000, 0x00010000ff010100, 0x0001000000ff0000, 0x000100000000ff00, - 0x00010000000000ff, 0x0001000000000000, 0x0001000000000001, 0x0001000000000100, - 0x0001000000010000, 0x0001000001ffff01, 0x0001000001000000, 0x0001000100ff0101, - 0x0001000100000000, 0x00010001010100ff, 0x000101ffffff01ff, 0x000101ffffff0101, - 0x000101ff00010000, 0x000101ff01ff0000, 0x000101ff0100ff01, 0x00010100ffff0000, - 0x0001010000000000, 0x000101000001ffff, 0x0001010000010101, 0x00010100010001ff, - 0x00010101ff00ff00, 0x00010101ff010001, 0x0001010100ffffff, 0x0001010100ff01ff, - 0x00010101000101ff, 0x0001010101ff0000, 0x000101010100ff01, 0x0001010101000101, - 0x01ffffffffff0101, 0x01ffffffff01ffff, 0x01ffffffff01ff01, 0x01ffffffff0101ff, - 0x01ffffffff010101, 0x01ffffff00000000, 0x01ffffff01ff01ff, 0x01ffffff01000101, - 0x01ffffff0101ff01, 0x01ffffff010100ff, 0x01ffff000000ff00, 0x01ffff0000000001, - 0x01ffff00000001ff, 0x01ffff0000010000, 0x01ffff0001ff0000, 0x01ffff01ffffffff, - 0x01ffff01ffff01ff, 0x01ffff01ff000000, 0x01ffff01ff01ffff, 0x01ffff01ff0101ff, - 0x01ffff010100ffff, 0x01ff00ffffff0000, 0x01ff00ffff010000, 0x01ff00ff00ffff01, - 0x01ff0000ff0000ff, 0x01ff000000000000, 0x01ff00000001ff01, 0x01ff000001ffffff, - 0x01ff000001010100, 0x01ff0001ffffff01, 0x01ff0001ff010001, 0x01ff000101ff0100, - 0x01ff000101000001, 0x01ff0001010100ff, 0x01ff01ffff00ffff, 0x01ff01ff00010001, - 0x01ff01ff01000000, 0x01ff01ff010101ff, 0x01ff0100ff000001, 0x01ff010000ffff00, - 0x01ff010000000100, 0x01ff010001ff01ff, 0x01ff01000101ffff, 0x01ff0101ffff00ff, - 0x01ff0101ffff0101, 0x01ff0101ff0101ff, 0x01ff010100010000, 0x0100ffff00ff00ff, - 0x0100ffff00ff0001, 0x0100ffff00000100, 0x0100ffff0100ff00, 0x0100ff00ffff0000, - 0x0100ff00ff00ffff, 0x0100ff00ff00ff01, 0x0100ff00ff000100, 0x0100ff00ff010000, - 0x0100ff0000000000, 0x0100ff00000100ff, 0x0100ff0001ff0101, 0x0100ff0001010101, - 0x0100ff0100ff00ff, 0x0100ff0100ff0001, 0x0100ff0100000100, 0x0100ff0100010001, - 0x0100ff0101000000, 0x010000ffff00ff00, 0x010000ff0000ffff, 0x010000ff00000000, - 0x010000ff010001ff, 0x010000ff01010001, 0x01000000ffffff00, 0x01000000ffff0101, - 0x01000000ff000000, 0x01000000ff0100ff, 0x01000000ff010101, 0x0100000000ff0000, - 0x010000000000ff00, 0x01000000000000ff, 0x0100000000000000, 0x0100000000000001, - 0x0100000000000100, 0x0100000000010000, 0x0100000001000000, 0x0100000100000000, - 0x01000001000101ff, 0x0100000101ffff01, 0x010001ffff000101, 0x010001ff00ff0100, - 0x010001ff0000ff00, 0x010001ff000100ff, 0x010001ff01ffffff, 0x01000100ffff0000, - 0x01000100ff0001ff, 0x0100010000000000, 0x010001000001ff00, 0x0100010001ff0000, - 0x01000100010000ff, 0x0100010001000101, 0x01000101ff00ff01, 0x0100010100ff0100, - 0x010001010000ffff, 0x0100010101010001, 0x0101ffffffff0101, 0x0101ffffff0001ff, - 0x0101ffffff01ffff, 0x0101ffffff010101, 0x0101ffff00000000, 0x0101ffff0101ffff, - 0x0101ffff010101ff, 0x0101ff00ff000000, 0x0101ff0000ff0100, 0x0101ff000000ff00, - 0x0101ff0000010000, 0x0101ff00010000ff, 0x0101ff0001000001, 0x0101ff01ff010101, - 0x0101ff0100000000, 0x0101ff010101ff00, 0x010100ffffff0000, 0x010100ffff010000, - 0x010100ff00ff01ff, 0x010100ff000000ff, 0x010100ff00000101, 0x010100ff01ffff00, - 0x01010000ffffff01, 0x01010000ff000100, 0x01010000ff01ff01, 0x0101000000000000, - 0x01010000000100ff, 0x010100000101ff01, 0x01010001ffff0000, 0x01010001ff00ffff, - 0x01010001ff010000, 0x0101000101ffffff, 0x0101000101ff01ff, 0x0101000101010101, - 0x010101ffff01ffff, 0x010101ff00000000, 0x010101ff0001ff01, 0x010101ff0101ffff, - 0x010101ff010101ff, 0x01010100ffffffff, 0x01010100ff000001, 0x010101000000ff00, - 0x0101010001010000, 0x0101010100ff0001, 0x010101010001ff01, 0x010101010101ffff, -}; - -static const __device__ uint8_t ksigns_iq2xs[128] = { - 0, 129, 130, 3, 132, 5, 6, 135, 136, 9, 10, 139, 12, 141, 142, 15, - 144, 17, 18, 147, 20, 149, 150, 23, 24, 153, 154, 27, 156, 29, 30, 159, - 160, 33, 34, 163, 36, 165, 166, 39, 40, 169, 170, 43, 172, 45, 46, 175, - 48, 177, 178, 51, 180, 53, 54, 183, 184, 57, 58, 187, 60, 189, 190, 63, - 192, 65, 66, 195, 68, 197, 198, 71, 72, 201, 202, 75, 204, 77, 78, 207, - 80, 209, 210, 83, 212, 85, 86, 215, 216, 89, 90, 219, 92, 221, 222, 95, - 96, 225, 226, 99, 228, 101, 102, 231, 232, 105, 106, 235, 108, 237, 238, 111, - 240, 113, 114, 243, 116, 245, 246, 119, 120, 249, 250, 123, 252, 125, 126, 255, -}; - -//#if __CUDA_ARCH__ >= MIN_CC_DP4A // lowest compute capability for integer intrinsics -static const __device__ uint64_t ksigns64[128] = { - 0x0000000000000000, 0xff000000000000ff, 0xff0000000000ff00, 0x000000000000ffff, - 0xff00000000ff0000, 0x0000000000ff00ff, 0x0000000000ffff00, 0xff00000000ffffff, - 0xff000000ff000000, 0x00000000ff0000ff, 0x00000000ff00ff00, 0xff000000ff00ffff, - 0x00000000ffff0000, 0xff000000ffff00ff, 0xff000000ffffff00, 0x00000000ffffffff, - 0xff0000ff00000000, 0x000000ff000000ff, 0x000000ff0000ff00, 0xff0000ff0000ffff, - 0x000000ff00ff0000, 0xff0000ff00ff00ff, 0xff0000ff00ffff00, 0x000000ff00ffffff, - 0x000000ffff000000, 0xff0000ffff0000ff, 0xff0000ffff00ff00, 0x000000ffff00ffff, - 0xff0000ffffff0000, 0x000000ffffff00ff, 0x000000ffffffff00, 0xff0000ffffffffff, - 0xff00ff0000000000, 0x0000ff00000000ff, 0x0000ff000000ff00, 0xff00ff000000ffff, - 0x0000ff0000ff0000, 0xff00ff0000ff00ff, 0xff00ff0000ffff00, 0x0000ff0000ffffff, - 0x0000ff00ff000000, 0xff00ff00ff0000ff, 0xff00ff00ff00ff00, 0x0000ff00ff00ffff, - 0xff00ff00ffff0000, 0x0000ff00ffff00ff, 0x0000ff00ffffff00, 0xff00ff00ffffffff, - 0x0000ffff00000000, 0xff00ffff000000ff, 0xff00ffff0000ff00, 0x0000ffff0000ffff, - 0xff00ffff00ff0000, 0x0000ffff00ff00ff, 0x0000ffff00ffff00, 0xff00ffff00ffffff, - 0xff00ffffff000000, 0x0000ffffff0000ff, 0x0000ffffff00ff00, 0xff00ffffff00ffff, - 0x0000ffffffff0000, 0xff00ffffffff00ff, 0xff00ffffffffff00, 0x0000ffffffffffff, - 0xffff000000000000, 0x00ff0000000000ff, 0x00ff00000000ff00, 0xffff00000000ffff, - 0x00ff000000ff0000, 0xffff000000ff00ff, 0xffff000000ffff00, 0x00ff000000ffffff, - 0x00ff0000ff000000, 0xffff0000ff0000ff, 0xffff0000ff00ff00, 0x00ff0000ff00ffff, - 0xffff0000ffff0000, 0x00ff0000ffff00ff, 0x00ff0000ffffff00, 0xffff0000ffffffff, - 0x00ff00ff00000000, 0xffff00ff000000ff, 0xffff00ff0000ff00, 0x00ff00ff0000ffff, - 0xffff00ff00ff0000, 0x00ff00ff00ff00ff, 0x00ff00ff00ffff00, 0xffff00ff00ffffff, - 0xffff00ffff000000, 0x00ff00ffff0000ff, 0x00ff00ffff00ff00, 0xffff00ffff00ffff, - 0x00ff00ffffff0000, 0xffff00ffffff00ff, 0xffff00ffffffff00, 0x00ff00ffffffffff, - 0x00ffff0000000000, 0xffffff00000000ff, 0xffffff000000ff00, 0x00ffff000000ffff, - 0xffffff0000ff0000, 0x00ffff0000ff00ff, 0x00ffff0000ffff00, 0xffffff0000ffffff, - 0xffffff00ff000000, 0x00ffff00ff0000ff, 0x00ffff00ff00ff00, 0xffffff00ff00ffff, - 0x00ffff00ffff0000, 0xffffff00ffff00ff, 0xffffff00ffffff00, 0x00ffff00ffffffff, - 0xffffffff00000000, 0x00ffffff000000ff, 0x00ffffff0000ff00, 0xffffffff0000ffff, - 0x00ffffff00ff0000, 0xffffffff00ff00ff, 0xffffffff00ffff00, 0x00ffffff00ffffff, - 0x00ffffffff000000, 0xffffffffff0000ff, 0xffffffffff00ff00, 0x00ffffffff00ffff, - 0xffffffffffff0000, 0x00ffffffffff00ff, 0x00ffffffffffff00, 0xffffffffffffffff, -}; -//#endif - -static const __device__ uint8_t kmask_iq2xs[8] = {1, 2, 4, 8, 16, 32, 64, 128}; - inline bool ggml_cuda_supports_mmq(enum ggml_type type) { switch (type) { case GGML_TYPE_Q4_0: diff --git a/ggml-metal.metal b/ggml-metal.metal index a65d12641..6ebbbd195 100644 --- a/ggml-metal.metal +++ b/ggml-metal.metal @@ -1,5 +1,8 @@ #include +#define GGML_COMMON_IMPL_METAL +#include "ggml-common.h" + using namespace metal; #define MAX(x, y) ((x) > (y) ? (x) : (y)) @@ -3638,710 +3641,6 @@ kernel void kernel_mul_mv_q6_K_f32( // ======================= "True" 2-bit -constexpr constant static uint64_t iq2xxs_grid[256] = { - 0x0808080808080808, 0x080808080808082b, 0x0808080808081919, 0x0808080808082b08, - 0x0808080808082b2b, 0x0808080808190819, 0x0808080808191908, 0x08080808082b0808, - 0x08080808082b082b, 0x08080808082b2b08, 0x08080808082b2b2b, 0x0808080819080819, - 0x0808080819081908, 0x0808080819190808, 0x0808080819192b08, 0x08080808192b0819, - 0x08080808192b1908, 0x080808082b080808, 0x080808082b08082b, 0x080808082b082b2b, - 0x080808082b2b082b, 0x0808081908080819, 0x0808081908081908, 0x0808081908190808, - 0x0808081908191919, 0x0808081919080808, 0x080808192b081908, 0x080808192b192b08, - 0x0808082b08080808, 0x0808082b0808082b, 0x0808082b082b082b, 0x0808082b2b08082b, - 0x0808190808080819, 0x0808190808081908, 0x0808190808190808, 0x08081908082b0819, - 0x08081908082b1908, 0x0808190819080808, 0x080819081908082b, 0x0808190819082b08, - 0x08081908192b0808, 0x080819082b080819, 0x080819082b081908, 0x080819082b190808, - 0x080819082b2b1908, 0x0808191908080808, 0x080819190808082b, 0x0808191908082b08, - 0x08081919082b0808, 0x080819191908192b, 0x08081919192b2b19, 0x080819192b080808, - 0x080819192b190819, 0x0808192b08082b19, 0x0808192b08190808, 0x0808192b19080808, - 0x0808192b2b081908, 0x0808192b2b2b1908, 0x08082b0808080808, 0x08082b0808081919, - 0x08082b0808082b08, 0x08082b0808191908, 0x08082b08082b2b08, 0x08082b0819080819, - 0x08082b0819081908, 0x08082b0819190808, 0x08082b081919082b, 0x08082b082b082b08, - 0x08082b1908081908, 0x08082b1919080808, 0x08082b2b0808082b, 0x08082b2b08191908, - 0x0819080808080819, 0x0819080808081908, 0x0819080808190808, 0x08190808082b0819, - 0x0819080819080808, 0x08190808192b0808, 0x081908082b081908, 0x081908082b190808, - 0x081908082b191919, 0x0819081908080808, 0x0819081908082b08, 0x08190819082b0808, - 0x0819081919190808, 0x0819081919192b2b, 0x081908192b080808, 0x0819082b082b1908, - 0x0819082b19081919, 0x0819190808080808, 0x0819190808082b08, 0x08191908082b0808, - 0x08191908082b1919, 0x0819190819082b19, 0x081919082b080808, 0x0819191908192b08, - 0x08191919192b082b, 0x0819192b08080808, 0x0819192b0819192b, 0x08192b0808080819, - 0x08192b0808081908, 0x08192b0808190808, 0x08192b0819080808, 0x08192b082b080819, - 0x08192b1908080808, 0x08192b1908081919, 0x08192b192b2b0808, 0x08192b2b19190819, - 0x082b080808080808, 0x082b08080808082b, 0x082b080808082b2b, 0x082b080819081908, - 0x082b0808192b0819, 0x082b08082b080808, 0x082b08082b08082b, 0x082b0819082b2b19, - 0x082b081919082b08, 0x082b082b08080808, 0x082b082b0808082b, 0x082b190808080819, - 0x082b190808081908, 0x082b190808190808, 0x082b190819080808, 0x082b19081919192b, - 0x082b191908080808, 0x082b191919080819, 0x082b1919192b1908, 0x082b192b2b190808, - 0x082b2b0808082b08, 0x082b2b08082b0808, 0x082b2b082b191908, 0x082b2b2b19081908, - 0x1908080808080819, 0x1908080808081908, 0x1908080808190808, 0x1908080808192b08, - 0x19080808082b0819, 0x19080808082b1908, 0x1908080819080808, 0x1908080819082b08, - 0x190808081919192b, 0x19080808192b0808, 0x190808082b080819, 0x190808082b081908, - 0x190808082b190808, 0x1908081908080808, 0x19080819082b0808, 0x19080819192b0819, - 0x190808192b080808, 0x190808192b081919, 0x1908082b08080819, 0x1908082b08190808, - 0x1908082b19082b08, 0x1908082b1919192b, 0x1908082b192b2b08, 0x1908190808080808, - 0x1908190808082b08, 0x19081908082b0808, 0x190819082b080808, 0x190819082b192b19, - 0x190819190819082b, 0x19081919082b1908, 0x1908192b08080808, 0x19082b0808080819, - 0x19082b0808081908, 0x19082b0808190808, 0x19082b0819080808, 0x19082b0819081919, - 0x19082b1908080808, 0x19082b1919192b08, 0x19082b19192b0819, 0x19082b192b08082b, - 0x19082b2b19081919, 0x19082b2b2b190808, 0x1919080808080808, 0x1919080808082b08, - 0x1919080808190819, 0x1919080808192b19, 0x19190808082b0808, 0x191908082b080808, - 0x191908082b082b08, 0x1919081908081908, 0x191908191908082b, 0x191908192b2b1908, - 0x1919082b2b190819, 0x191919082b190808, 0x191919082b19082b, 0x1919191908082b2b, - 0x1919192b08080819, 0x1919192b19191908, 0x19192b0808080808, 0x19192b0808190819, - 0x19192b0808192b19, 0x19192b08192b1908, 0x19192b1919080808, 0x19192b2b08082b08, - 0x192b080808081908, 0x192b080808190808, 0x192b080819080808, 0x192b0808192b2b08, - 0x192b081908080808, 0x192b081919191919, 0x192b082b08192b08, 0x192b082b192b0808, - 0x192b190808080808, 0x192b190808081919, 0x192b191908190808, 0x192b19190819082b, - 0x192b19192b081908, 0x192b2b081908082b, 0x2b08080808080808, 0x2b0808080808082b, - 0x2b08080808082b2b, 0x2b08080819080819, 0x2b0808082b08082b, 0x2b08081908081908, - 0x2b08081908192b08, 0x2b08081919080808, 0x2b08082b08190819, 0x2b08190808080819, - 0x2b08190808081908, 0x2b08190808190808, 0x2b08190808191919, 0x2b08190819080808, - 0x2b081908192b0808, 0x2b08191908080808, 0x2b0819191908192b, 0x2b0819192b191908, - 0x2b08192b08082b19, 0x2b08192b19080808, 0x2b08192b192b0808, 0x2b082b080808082b, - 0x2b082b1908081908, 0x2b082b2b08190819, 0x2b19080808081908, 0x2b19080808190808, - 0x2b190808082b1908, 0x2b19080819080808, 0x2b1908082b2b0819, 0x2b1908190819192b, - 0x2b1908192b080808, 0x2b19082b19081919, 0x2b19190808080808, 0x2b191908082b082b, - 0x2b19190819081908, 0x2b19191919190819, 0x2b192b082b080819, 0x2b192b19082b0808, - 0x2b2b08080808082b, 0x2b2b080819190808, 0x2b2b08082b081919, 0x2b2b081908082b19, - 0x2b2b082b08080808, 0x2b2b190808192b08, 0x2b2b2b0819190808, 0x2b2b2b1908081908, -}; - -constexpr constant static uint64_t iq2xs_grid[512] = { - 0x0808080808080808, 0x080808080808082b, 0x0808080808081919, 0x0808080808082b08, - 0x0808080808082b2b, 0x0808080808190819, 0x0808080808191908, 0x080808080819192b, - 0x0808080808192b19, 0x08080808082b0808, 0x08080808082b082b, 0x08080808082b1919, - 0x08080808082b2b08, 0x0808080819080819, 0x0808080819081908, 0x080808081908192b, - 0x0808080819082b19, 0x0808080819190808, 0x080808081919082b, 0x0808080819191919, - 0x0808080819192b08, 0x08080808192b0819, 0x08080808192b1908, 0x080808082b080808, - 0x080808082b08082b, 0x080808082b081919, 0x080808082b082b08, 0x080808082b190819, - 0x080808082b191908, 0x080808082b192b19, 0x080808082b2b0808, 0x0808081908080819, - 0x0808081908081908, 0x080808190808192b, 0x0808081908082b19, 0x0808081908190808, - 0x080808190819082b, 0x0808081908191919, 0x0808081908192b08, 0x0808081908192b2b, - 0x08080819082b0819, 0x08080819082b1908, 0x0808081919080808, 0x080808191908082b, - 0x0808081919081919, 0x0808081919082b08, 0x0808081919190819, 0x0808081919191908, - 0x08080819192b0808, 0x08080819192b2b08, 0x080808192b080819, 0x080808192b081908, - 0x080808192b190808, 0x0808082b08080808, 0x0808082b0808082b, 0x0808082b08081919, - 0x0808082b08082b08, 0x0808082b08190819, 0x0808082b08191908, 0x0808082b082b0808, - 0x0808082b19080819, 0x0808082b19081908, 0x0808082b19190808, 0x0808082b19191919, - 0x0808082b2b080808, 0x0808082b2b082b2b, 0x0808190808080819, 0x0808190808081908, - 0x080819080808192b, 0x0808190808082b19, 0x0808190808190808, 0x080819080819082b, - 0x0808190808191919, 0x0808190808192b08, 0x08081908082b0819, 0x08081908082b1908, - 0x0808190819080808, 0x080819081908082b, 0x0808190819081919, 0x0808190819082b08, - 0x0808190819190819, 0x0808190819191908, 0x080819081919192b, 0x08081908192b0808, - 0x080819082b080819, 0x080819082b081908, 0x080819082b190808, 0x0808191908080808, - 0x080819190808082b, 0x0808191908081919, 0x0808191908082b08, 0x0808191908190819, - 0x0808191908191908, 0x08081919082b0808, 0x0808191919080819, 0x0808191919081908, - 0x0808191919190808, 0x08081919192b0819, 0x080819192b080808, 0x0808192b08080819, - 0x0808192b08081908, 0x0808192b08190808, 0x0808192b082b192b, 0x0808192b19080808, - 0x0808192b1908082b, 0x0808192b2b081908, 0x08082b0808080808, 0x08082b080808082b, - 0x08082b0808081919, 0x08082b0808082b08, 0x08082b0808082b2b, 0x08082b0808190819, - 0x08082b0808191908, 0x08082b08082b0808, 0x08082b08082b1919, 0x08082b0819080819, - 0x08082b0819081908, 0x08082b0819190808, 0x08082b0819192b08, 0x08082b082b080808, - 0x08082b082b2b0808, 0x08082b082b2b2b2b, 0x08082b1908080819, 0x08082b1908081908, - 0x08082b1908190808, 0x08082b1919080808, 0x08082b192b080819, 0x08082b192b082b19, - 0x08082b2b08080808, 0x08082b2b082b0808, 0x08082b2b082b2b08, 0x08082b2b2b19192b, - 0x08082b2b2b2b0808, 0x0819080808080819, 0x0819080808081908, 0x081908080808192b, - 0x0819080808082b19, 0x0819080808190808, 0x081908080819082b, 0x0819080808191919, - 0x0819080808192b08, 0x08190808082b0819, 0x08190808082b1908, 0x0819080819080808, - 0x081908081908082b, 0x0819080819081919, 0x0819080819082b08, 0x0819080819190819, - 0x0819080819191908, 0x08190808192b0808, 0x08190808192b2b2b, 0x081908082b080819, - 0x081908082b081908, 0x081908082b190808, 0x0819081908080808, 0x081908190808082b, - 0x0819081908081919, 0x0819081908082b08, 0x0819081908190819, 0x0819081908191908, - 0x08190819082b0808, 0x0819081919080819, 0x0819081919081908, 0x0819081919190808, - 0x081908192b080808, 0x081908192b191908, 0x081908192b19192b, 0x0819082b08080819, - 0x0819082b08081908, 0x0819082b0808192b, 0x0819082b08190808, 0x0819082b19080808, - 0x0819082b192b0808, 0x0819190808080808, 0x081919080808082b, 0x0819190808081919, - 0x0819190808082b08, 0x0819190808190819, 0x0819190808191908, 0x08191908082b0808, - 0x0819190819080819, 0x0819190819081908, 0x0819190819082b19, 0x0819190819190808, - 0x08191908192b1908, 0x081919082b080808, 0x0819191908080819, 0x0819191908081908, - 0x0819191908190808, 0x0819191919080808, 0x0819192b08080808, 0x0819192b08191908, - 0x0819192b19082b19, 0x08192b0808080819, 0x08192b0808081908, 0x08192b0808190808, - 0x08192b080819082b, 0x08192b0819080808, 0x08192b0819191908, 0x08192b082b08192b, - 0x08192b1908080808, 0x08192b1908081919, 0x08192b19192b192b, 0x08192b2b19190819, - 0x08192b2b2b2b2b19, 0x082b080808080808, 0x082b08080808082b, 0x082b080808081919, - 0x082b080808082b08, 0x082b080808082b2b, 0x082b080808190819, 0x082b080808191908, - 0x082b0808082b0808, 0x082b080819080819, 0x082b080819081908, 0x082b080819190808, - 0x082b08082b080808, 0x082b08082b2b0808, 0x082b081908080819, 0x082b081908081908, - 0x082b081908190808, 0x082b081919080808, 0x082b081919082b08, 0x082b0819192b1919, - 0x082b082b08080808, 0x082b082b082b082b, 0x082b082b2b080808, 0x082b082b2b2b2b08, - 0x082b190808080819, 0x082b190808081908, 0x082b190808190808, 0x082b1908082b2b19, - 0x082b190819080808, 0x082b191908080808, 0x082b191919080819, 0x082b19191919082b, - 0x082b19192b192b19, 0x082b192b08080819, 0x082b192b08192b2b, 0x082b192b2b2b192b, - 0x082b2b0808080808, 0x082b2b0808082b08, 0x082b2b0808082b2b, 0x082b2b08082b0808, - 0x082b2b0819191919, 0x082b2b082b082b08, 0x082b2b082b2b082b, 0x082b2b19192b2b08, - 0x082b2b192b190808, 0x082b2b2b08082b08, 0x082b2b2b082b0808, 0x082b2b2b2b08082b, - 0x082b2b2b2b082b08, 0x082b2b2b2b082b2b, 0x1908080808080819, 0x1908080808081908, - 0x190808080808192b, 0x1908080808082b19, 0x1908080808190808, 0x190808080819082b, - 0x1908080808191919, 0x1908080808192b08, 0x19080808082b0819, 0x19080808082b1908, - 0x1908080819080808, 0x190808081908082b, 0x1908080819081919, 0x1908080819082b08, - 0x1908080819082b2b, 0x1908080819190819, 0x1908080819191908, 0x19080808192b0808, - 0x19080808192b1919, 0x190808082b080819, 0x190808082b081908, 0x190808082b190808, - 0x1908081908080808, 0x190808190808082b, 0x1908081908081919, 0x1908081908082b08, - 0x1908081908190819, 0x1908081908191908, 0x19080819082b0808, 0x1908081919080819, - 0x1908081919081908, 0x1908081919190808, 0x190808192b080808, 0x190808192b081919, - 0x190808192b2b082b, 0x1908082b08080819, 0x1908082b08081908, 0x1908082b08190808, - 0x1908082b0819082b, 0x1908082b082b2b19, 0x1908082b19080808, 0x1908190808080808, - 0x190819080808082b, 0x1908190808081919, 0x1908190808082b08, 0x1908190808190819, - 0x1908190808191908, 0x1908190808192b19, 0x19081908082b0808, 0x1908190819080819, - 0x1908190819081908, 0x1908190819190808, 0x190819082b080808, 0x190819082b191908, - 0x1908191908080819, 0x1908191908081908, 0x1908191908190808, 0x19081919082b1908, - 0x1908191919080808, 0x190819192b192b2b, 0x1908192b08080808, 0x1908192b08082b2b, - 0x1908192b19081908, 0x1908192b19190808, 0x19082b0808080819, 0x19082b0808081908, - 0x19082b0808190808, 0x19082b0819080808, 0x19082b0819081919, 0x19082b0819191908, - 0x19082b08192b082b, 0x19082b1908080808, 0x19082b1908190819, 0x19082b1919081908, - 0x19082b1919190808, 0x19082b19192b2b19, 0x19082b2b08081908, 0x1919080808080808, - 0x191908080808082b, 0x1919080808081919, 0x1919080808082b08, 0x1919080808190819, - 0x1919080808191908, 0x19190808082b0808, 0x19190808082b2b08, 0x1919080819080819, - 0x1919080819081908, 0x1919080819190808, 0x191908082b080808, 0x1919081908080819, - 0x1919081908081908, 0x1919081908190808, 0x1919081908191919, 0x1919081919080808, - 0x191908191908082b, 0x1919082b08080808, 0x1919082b19081908, 0x1919082b2b2b2b2b, - 0x1919190808080819, 0x1919190808081908, 0x1919190808190808, 0x19191908082b0819, - 0x1919190819080808, 0x19191908192b0808, 0x191919082b080819, 0x191919082b2b0819, - 0x1919191908080808, 0x1919191908082b08, 0x191919192b080808, 0x191919192b082b08, - 0x1919192b082b0819, 0x1919192b192b2b08, 0x1919192b2b2b0819, 0x19192b0808080808, - 0x19192b0808191908, 0x19192b0819080819, 0x19192b0819190808, 0x19192b082b192b19, - 0x19192b1908192b2b, 0x19192b1919080808, 0x19192b191908082b, 0x19192b2b2b081919, - 0x192b080808080819, 0x192b080808081908, 0x192b080808190808, 0x192b080819080808, - 0x192b080819191908, 0x192b0808192b082b, 0x192b08082b08192b, 0x192b08082b2b2b19, - 0x192b081908080808, 0x192b082b082b1908, 0x192b082b19082b2b, 0x192b082b2b19082b, - 0x192b190808080808, 0x192b19080819192b, 0x192b191908190808, 0x192b191919080808, - 0x192b191919081919, 0x192b19192b2b1908, 0x192b2b0808080819, 0x192b2b08192b2b2b, - 0x192b2b19082b1919, 0x192b2b2b0808192b, 0x192b2b2b19191908, 0x192b2b2b192b082b, - 0x2b08080808080808, 0x2b0808080808082b, 0x2b08080808081919, 0x2b08080808082b08, - 0x2b08080808190819, 0x2b08080808191908, 0x2b080808082b0808, 0x2b080808082b2b2b, - 0x2b08080819080819, 0x2b08080819081908, 0x2b08080819190808, 0x2b0808082b080808, - 0x2b0808082b08082b, 0x2b0808082b2b2b08, 0x2b0808082b2b2b2b, 0x2b08081908080819, - 0x2b08081908081908, 0x2b0808190808192b, 0x2b08081908190808, 0x2b08081919080808, - 0x2b08081919190819, 0x2b08081919192b19, 0x2b08082b08080808, 0x2b08082b082b0808, - 0x2b08082b2b080808, 0x2b08082b2b08082b, 0x2b08082b2b2b0808, 0x2b08082b2b2b2b08, - 0x2b08190808080819, 0x2b08190808081908, 0x2b08190808190808, 0x2b0819080819082b, - 0x2b08190808191919, 0x2b08190819080808, 0x2b081908192b0808, 0x2b0819082b082b19, - 0x2b08191908080808, 0x2b08191919081908, 0x2b0819192b2b1919, 0x2b08192b08192b08, - 0x2b08192b192b2b2b, 0x2b082b0808080808, 0x2b082b0808082b08, 0x2b082b08082b1919, - 0x2b082b0819192b2b, 0x2b082b082b080808, 0x2b082b082b08082b, 0x2b082b082b2b2b08, - 0x2b082b190808192b, 0x2b082b2b082b082b, 0x2b082b2b2b080808, 0x2b082b2b2b082b08, - 0x2b082b2b2b19192b, 0x2b082b2b2b2b2b08, 0x2b19080808080819, 0x2b19080808081908, - 0x2b19080808190808, 0x2b19080819080808, 0x2b1908081919192b, 0x2b1908082b081908, - 0x2b19081908080808, 0x2b190819082b082b, 0x2b190819192b1908, 0x2b19082b1919192b, - 0x2b19082b2b082b19, 0x2b19190808080808, 0x2b19190808081919, 0x2b19190819081908, - 0x2b19190819190808, 0x2b19190819192b08, 0x2b191919082b2b19, 0x2b1919192b190808, - 0x2b1919192b19082b, 0x2b19192b19080819, 0x2b192b0819190819, 0x2b192b082b2b192b, - 0x2b192b1919082b19, 0x2b192b2b08191919, 0x2b192b2b192b0808, 0x2b2b080808080808, - 0x2b2b08080808082b, 0x2b2b080808082b08, 0x2b2b080808082b2b, 0x2b2b0808082b0808, - 0x2b2b0808082b2b2b, 0x2b2b08082b2b0808, 0x2b2b081919190819, 0x2b2b081919192b19, - 0x2b2b08192b2b192b, 0x2b2b082b08080808, 0x2b2b082b0808082b, 0x2b2b082b08082b08, - 0x2b2b082b082b2b2b, 0x2b2b082b2b080808, 0x2b2b082b2b2b0808, 0x2b2b190819080808, - 0x2b2b19082b191919, 0x2b2b192b192b1919, 0x2b2b192b2b192b08, 0x2b2b2b0808082b2b, - 0x2b2b2b08082b0808, 0x2b2b2b08082b082b, 0x2b2b2b08082b2b08, 0x2b2b2b082b2b0808, - 0x2b2b2b082b2b2b08, 0x2b2b2b1908081908, 0x2b2b2b192b081908, 0x2b2b2b192b08192b, - 0x2b2b2b2b082b2b08, 0x2b2b2b2b082b2b2b, 0x2b2b2b2b2b190819, 0x2b2b2b2b2b2b2b2b, -}; - -constexpr constant static uint64_t iq2s_grid[1024] = { - 0x0808080808080808, 0x080808080808082b, 0x0808080808081919, 0x0808080808082b08, - 0x0808080808082b2b, 0x0808080808190819, 0x0808080808191908, 0x080808080819192b, - 0x0808080808192b19, 0x08080808082b0808, 0x08080808082b082b, 0x08080808082b1919, - 0x08080808082b2b08, 0x0808080819080819, 0x0808080819081908, 0x080808081908192b, - 0x0808080819082b19, 0x0808080819190808, 0x080808081919082b, 0x0808080819191919, - 0x0808080819192b08, 0x08080808192b0819, 0x08080808192b1908, 0x08080808192b192b, - 0x08080808192b2b19, 0x080808082b080808, 0x080808082b08082b, 0x080808082b081919, - 0x080808082b082b08, 0x080808082b190819, 0x080808082b191908, 0x080808082b2b0808, - 0x080808082b2b1919, 0x080808082b2b2b2b, 0x0808081908080819, 0x0808081908081908, - 0x080808190808192b, 0x0808081908082b19, 0x0808081908190808, 0x080808190819082b, - 0x0808081908191919, 0x0808081908192b08, 0x08080819082b0819, 0x08080819082b1908, - 0x0808081919080808, 0x080808191908082b, 0x0808081919081919, 0x0808081919082b08, - 0x0808081919190819, 0x0808081919191908, 0x080808191919192b, 0x0808081919192b19, - 0x08080819192b0808, 0x08080819192b1919, 0x08080819192b2b08, 0x080808192b080819, - 0x080808192b081908, 0x080808192b190808, 0x080808192b19082b, 0x080808192b191919, - 0x080808192b2b0819, 0x080808192b2b1908, 0x0808082b08080808, 0x0808082b0808082b, - 0x0808082b08081919, 0x0808082b08082b08, 0x0808082b08190819, 0x0808082b08191908, - 0x0808082b082b0808, 0x0808082b082b2b2b, 0x0808082b19080819, 0x0808082b19081908, - 0x0808082b1908192b, 0x0808082b19082b19, 0x0808082b19190808, 0x0808082b19191919, - 0x0808082b2b080808, 0x0808082b2b081919, 0x0808082b2b082b2b, 0x0808082b2b191908, - 0x0808082b2b2b082b, 0x0808190808080819, 0x0808190808081908, 0x080819080808192b, - 0x0808190808082b19, 0x0808190808190808, 0x080819080819082b, 0x0808190808191919, - 0x0808190808192b08, 0x08081908082b0819, 0x08081908082b1908, 0x08081908082b192b, - 0x08081908082b2b19, 0x0808190819080808, 0x080819081908082b, 0x0808190819081919, - 0x0808190819082b08, 0x0808190819082b2b, 0x0808190819190819, 0x0808190819191908, - 0x080819081919192b, 0x0808190819192b19, 0x08081908192b0808, 0x08081908192b082b, - 0x08081908192b1919, 0x080819082b080819, 0x080819082b081908, 0x080819082b08192b, - 0x080819082b082b19, 0x080819082b190808, 0x080819082b191919, 0x080819082b192b08, - 0x080819082b2b0819, 0x080819082b2b1908, 0x0808191908080808, 0x080819190808082b, - 0x0808191908081919, 0x0808191908082b08, 0x0808191908082b2b, 0x0808191908190819, - 0x0808191908191908, 0x080819190819192b, 0x0808191908192b19, 0x08081919082b0808, - 0x08081919082b1919, 0x08081919082b2b08, 0x0808191919080819, 0x0808191919081908, - 0x080819191908192b, 0x0808191919082b19, 0x0808191919190808, 0x080819191919082b, - 0x0808191919191919, 0x0808191919192b08, 0x08081919192b0819, 0x08081919192b1908, - 0x080819192b080808, 0x080819192b08082b, 0x080819192b081919, 0x080819192b082b08, - 0x080819192b190819, 0x080819192b191908, 0x080819192b2b0808, 0x0808192b08080819, - 0x0808192b08081908, 0x0808192b0808192b, 0x0808192b08082b19, 0x0808192b08190808, - 0x0808192b08191919, 0x0808192b19080808, 0x0808192b19081919, 0x0808192b19082b08, - 0x0808192b19190819, 0x0808192b19191908, 0x0808192b192b0808, 0x0808192b2b080819, - 0x0808192b2b081908, 0x0808192b2b190808, 0x08082b0808080808, 0x08082b080808082b, - 0x08082b0808081919, 0x08082b0808082b08, 0x08082b0808190819, 0x08082b0808191908, - 0x08082b080819192b, 0x08082b0808192b19, 0x08082b08082b0808, 0x08082b08082b1919, - 0x08082b08082b2b2b, 0x08082b0819080819, 0x08082b0819081908, 0x08082b081908192b, - 0x08082b0819082b19, 0x08082b0819190808, 0x08082b081919082b, 0x08082b0819191919, - 0x08082b0819192b08, 0x08082b08192b0819, 0x08082b08192b1908, 0x08082b082b080808, - 0x08082b082b081919, 0x08082b082b191908, 0x08082b082b2b2b2b, 0x08082b1908080819, - 0x08082b1908081908, 0x08082b1908190808, 0x08082b190819082b, 0x08082b1908191919, - 0x08082b1908192b08, 0x08082b19082b0819, 0x08082b1919080808, 0x08082b1919081919, - 0x08082b1919082b08, 0x08082b1919190819, 0x08082b1919191908, 0x08082b19192b0808, - 0x08082b192b080819, 0x08082b192b190808, 0x08082b2b08080808, 0x08082b2b08190819, - 0x08082b2b08191908, 0x08082b2b082b082b, 0x08082b2b082b2b08, 0x08082b2b082b2b2b, - 0x08082b2b19190808, 0x08082b2b2b192b19, 0x0819080808080819, 0x0819080808081908, - 0x081908080808192b, 0x0819080808082b19, 0x0819080808190808, 0x081908080819082b, - 0x0819080808191919, 0x0819080808192b08, 0x08190808082b0819, 0x08190808082b1908, - 0x08190808082b192b, 0x0819080819080808, 0x081908081908082b, 0x0819080819081919, - 0x0819080819082b08, 0x0819080819190819, 0x0819080819191908, 0x081908081919192b, - 0x0819080819192b19, 0x08190808192b0808, 0x08190808192b082b, 0x08190808192b1919, - 0x08190808192b2b08, 0x081908082b080819, 0x081908082b081908, 0x081908082b08192b, - 0x081908082b190808, 0x081908082b191919, 0x081908082b192b08, 0x081908082b2b0819, - 0x081908082b2b1908, 0x0819081908080808, 0x081908190808082b, 0x0819081908081919, - 0x0819081908082b08, 0x0819081908082b2b, 0x0819081908190819, 0x0819081908191908, - 0x081908190819192b, 0x0819081908192b19, 0x08190819082b0808, 0x08190819082b082b, - 0x08190819082b1919, 0x08190819082b2b08, 0x0819081919080819, 0x0819081919081908, - 0x081908191908192b, 0x0819081919082b19, 0x0819081919190808, 0x081908191919082b, - 0x0819081919191919, 0x0819081919192b08, 0x08190819192b0819, 0x08190819192b1908, - 0x081908192b080808, 0x081908192b08082b, 0x081908192b081919, 0x081908192b082b08, - 0x081908192b190819, 0x081908192b191908, 0x0819082b08080819, 0x0819082b08081908, - 0x0819082b08082b19, 0x0819082b08190808, 0x0819082b08191919, 0x0819082b082b0819, - 0x0819082b082b1908, 0x0819082b19080808, 0x0819082b19081919, 0x0819082b19190819, - 0x0819082b19191908, 0x0819082b2b080819, 0x0819082b2b081908, 0x0819082b2b190808, - 0x0819190808080808, 0x081919080808082b, 0x0819190808081919, 0x0819190808082b08, - 0x0819190808190819, 0x0819190808191908, 0x081919080819192b, 0x0819190808192b19, - 0x08191908082b0808, 0x08191908082b1919, 0x08191908082b2b08, 0x0819190819080819, - 0x0819190819081908, 0x081919081908192b, 0x0819190819082b19, 0x0819190819190808, - 0x081919081919082b, 0x0819190819191919, 0x0819190819192b08, 0x08191908192b0819, - 0x08191908192b1908, 0x081919082b080808, 0x081919082b08082b, 0x081919082b081919, - 0x081919082b082b08, 0x081919082b190819, 0x081919082b191908, 0x081919082b2b0808, - 0x0819191908080819, 0x0819191908081908, 0x081919190808192b, 0x0819191908082b19, - 0x0819191908190808, 0x081919190819082b, 0x0819191908191919, 0x0819191908192b08, - 0x08191919082b0819, 0x08191919082b1908, 0x0819191919080808, 0x081919191908082b, - 0x0819191919081919, 0x0819191919082b08, 0x0819191919190819, 0x0819191919191908, - 0x08191919192b0808, 0x081919192b080819, 0x081919192b081908, 0x081919192b190808, - 0x0819192b08080808, 0x0819192b08081919, 0x0819192b08082b08, 0x0819192b08190819, - 0x0819192b08191908, 0x0819192b082b0808, 0x0819192b19080819, 0x0819192b19081908, - 0x0819192b19190808, 0x0819192b2b080808, 0x0819192b2b2b2b2b, 0x08192b0808080819, - 0x08192b0808081908, 0x08192b080808192b, 0x08192b0808082b19, 0x08192b0808190808, - 0x08192b0808191919, 0x08192b0808192b08, 0x08192b08082b0819, 0x08192b0819080808, - 0x08192b081908082b, 0x08192b0819081919, 0x08192b0819082b08, 0x08192b0819190819, - 0x08192b0819191908, 0x08192b08192b0808, 0x08192b082b080819, 0x08192b082b081908, - 0x08192b1908080808, 0x08192b190808082b, 0x08192b1908081919, 0x08192b1908082b08, - 0x08192b1908190819, 0x08192b1908191908, 0x08192b19082b0808, 0x08192b1919080819, - 0x08192b1919081908, 0x08192b1919190808, 0x08192b19192b2b19, 0x08192b192b2b082b, - 0x08192b2b08081908, 0x08192b2b08190808, 0x08192b2b19080808, 0x08192b2b1919192b, - 0x082b080808080808, 0x082b08080808082b, 0x082b080808081919, 0x082b080808082b08, - 0x082b080808190819, 0x082b080808191908, 0x082b08080819192b, 0x082b080808192b19, - 0x082b0808082b0808, 0x082b0808082b1919, 0x082b0808082b2b2b, 0x082b080819080819, - 0x082b080819081908, 0x082b080819190808, 0x082b08081919082b, 0x082b080819191919, - 0x082b0808192b1908, 0x082b08082b080808, 0x082b08082b082b2b, 0x082b08082b191908, - 0x082b08082b2b2b2b, 0x082b081908080819, 0x082b081908081908, 0x082b081908190808, - 0x082b08190819082b, 0x082b081908191919, 0x082b0819082b0819, 0x082b081919080808, - 0x082b08191908082b, 0x082b081919081919, 0x082b081919190819, 0x082b081919191908, - 0x082b0819192b0808, 0x082b08192b080819, 0x082b08192b081908, 0x082b08192b190808, - 0x082b082b08080808, 0x082b082b08082b2b, 0x082b082b082b082b, 0x082b082b082b2b08, - 0x082b082b082b2b2b, 0x082b082b19081908, 0x082b082b19190808, 0x082b082b2b082b08, - 0x082b082b2b082b2b, 0x082b082b2b2b2b08, 0x082b190808080819, 0x082b190808081908, - 0x082b19080808192b, 0x082b190808082b19, 0x082b190808190808, 0x082b190808191919, - 0x082b190808192b08, 0x082b1908082b0819, 0x082b1908082b1908, 0x082b190819080808, - 0x082b19081908082b, 0x082b190819081919, 0x082b190819082b08, 0x082b190819190819, - 0x082b190819191908, 0x082b1908192b0808, 0x082b19082b080819, 0x082b19082b081908, - 0x082b19082b190808, 0x082b191908080808, 0x082b191908081919, 0x082b191908082b08, - 0x082b191908190819, 0x082b191908191908, 0x082b1919082b0808, 0x082b191919080819, - 0x082b191919081908, 0x082b191919190808, 0x082b1919192b192b, 0x082b19192b080808, - 0x082b192b08080819, 0x082b192b08081908, 0x082b192b08190808, 0x082b192b19080808, - 0x082b192b19192b19, 0x082b2b0808080808, 0x082b2b0808081919, 0x082b2b0808190819, - 0x082b2b0808191908, 0x082b2b0819080819, 0x082b2b0819081908, 0x082b2b0819190808, - 0x082b2b082b082b2b, 0x082b2b082b2b2b2b, 0x082b2b1908080819, 0x082b2b1908081908, - 0x082b2b1908190808, 0x082b2b192b191919, 0x082b2b2b08082b2b, 0x082b2b2b082b082b, - 0x082b2b2b192b1908, 0x082b2b2b2b082b08, 0x082b2b2b2b082b2b, 0x1908080808080819, - 0x1908080808081908, 0x190808080808192b, 0x1908080808082b19, 0x1908080808190808, - 0x190808080819082b, 0x1908080808191919, 0x1908080808192b08, 0x1908080808192b2b, - 0x19080808082b0819, 0x19080808082b1908, 0x19080808082b192b, 0x1908080819080808, - 0x190808081908082b, 0x1908080819081919, 0x1908080819082b08, 0x1908080819082b2b, - 0x1908080819190819, 0x1908080819191908, 0x190808081919192b, 0x1908080819192b19, - 0x19080808192b0808, 0x19080808192b082b, 0x19080808192b1919, 0x190808082b080819, - 0x190808082b081908, 0x190808082b190808, 0x190808082b191919, 0x190808082b192b08, - 0x190808082b2b0819, 0x190808082b2b1908, 0x1908081908080808, 0x190808190808082b, - 0x1908081908081919, 0x1908081908082b08, 0x1908081908190819, 0x1908081908191908, - 0x190808190819192b, 0x1908081908192b19, 0x19080819082b0808, 0x19080819082b082b, - 0x19080819082b1919, 0x1908081919080819, 0x1908081919081908, 0x190808191908192b, - 0x1908081919082b19, 0x1908081919190808, 0x190808191919082b, 0x1908081919191919, - 0x1908081919192b08, 0x19080819192b0819, 0x19080819192b1908, 0x190808192b080808, - 0x190808192b08082b, 0x190808192b081919, 0x190808192b082b08, 0x190808192b190819, - 0x190808192b191908, 0x190808192b2b0808, 0x1908082b08080819, 0x1908082b08081908, - 0x1908082b08190808, 0x1908082b0819082b, 0x1908082b08191919, 0x1908082b08192b08, - 0x1908082b082b1908, 0x1908082b19080808, 0x1908082b19081919, 0x1908082b19082b08, - 0x1908082b19190819, 0x1908082b19191908, 0x1908082b192b0808, 0x1908082b2b080819, - 0x1908082b2b081908, 0x1908190808080808, 0x190819080808082b, 0x1908190808081919, - 0x1908190808082b08, 0x1908190808082b2b, 0x1908190808190819, 0x1908190808191908, - 0x190819080819192b, 0x1908190808192b19, 0x19081908082b0808, 0x19081908082b082b, - 0x19081908082b1919, 0x19081908082b2b08, 0x1908190819080819, 0x1908190819081908, - 0x190819081908192b, 0x1908190819082b19, 0x1908190819190808, 0x190819081919082b, - 0x1908190819191919, 0x1908190819192b08, 0x19081908192b0819, 0x19081908192b1908, - 0x190819082b080808, 0x190819082b08082b, 0x190819082b081919, 0x190819082b082b08, - 0x190819082b190819, 0x190819082b191908, 0x190819082b2b0808, 0x1908191908080819, - 0x1908191908081908, 0x190819190808192b, 0x1908191908082b19, 0x1908191908190808, - 0x190819190819082b, 0x1908191908191919, 0x1908191908192b08, 0x19081919082b0819, - 0x19081919082b1908, 0x1908191919080808, 0x190819191908082b, 0x1908191919081919, - 0x1908191919082b08, 0x1908191919190819, 0x1908191919191908, 0x19081919192b0808, - 0x19081919192b2b2b, 0x190819192b080819, 0x190819192b081908, 0x190819192b190808, - 0x1908192b08080808, 0x1908192b0808082b, 0x1908192b08081919, 0x1908192b08082b08, - 0x1908192b08190819, 0x1908192b08191908, 0x1908192b082b0808, 0x1908192b19080819, - 0x1908192b19081908, 0x1908192b19190808, 0x1908192b2b080808, 0x1908192b2b2b1919, - 0x19082b0808080819, 0x19082b0808081908, 0x19082b0808082b19, 0x19082b0808190808, - 0x19082b080819082b, 0x19082b0808191919, 0x19082b0808192b08, 0x19082b08082b0819, - 0x19082b08082b1908, 0x19082b0819080808, 0x19082b081908082b, 0x19082b0819081919, - 0x19082b0819082b08, 0x19082b0819190819, 0x19082b0819191908, 0x19082b08192b0808, - 0x19082b082b081908, 0x19082b082b190808, 0x19082b1908080808, 0x19082b190808082b, - 0x19082b1908081919, 0x19082b1908082b08, 0x19082b1908190819, 0x19082b1908191908, - 0x19082b19082b0808, 0x19082b1919080819, 0x19082b1919081908, 0x19082b1919190808, - 0x19082b192b080808, 0x19082b192b19192b, 0x19082b2b08080819, 0x19082b2b08081908, - 0x19082b2b08190808, 0x19082b2b19080808, 0x1919080808080808, 0x191908080808082b, - 0x1919080808081919, 0x1919080808082b08, 0x1919080808190819, 0x1919080808191908, - 0x191908080819192b, 0x1919080808192b19, 0x19190808082b0808, 0x19190808082b082b, - 0x19190808082b1919, 0x19190808082b2b08, 0x1919080819080819, 0x1919080819081908, - 0x191908081908192b, 0x1919080819082b19, 0x1919080819190808, 0x191908081919082b, - 0x1919080819191919, 0x1919080819192b08, 0x19190808192b0819, 0x19190808192b1908, - 0x191908082b080808, 0x191908082b08082b, 0x191908082b081919, 0x191908082b082b08, - 0x191908082b190819, 0x191908082b191908, 0x1919081908080819, 0x1919081908081908, - 0x191908190808192b, 0x1919081908082b19, 0x1919081908190808, 0x191908190819082b, - 0x1919081908191919, 0x1919081908192b08, 0x19190819082b0819, 0x19190819082b1908, - 0x1919081919080808, 0x191908191908082b, 0x1919081919081919, 0x1919081919082b08, - 0x1919081919190819, 0x1919081919191908, 0x19190819192b0808, 0x191908192b080819, - 0x191908192b081908, 0x191908192b190808, 0x1919082b08080808, 0x1919082b08081919, - 0x1919082b08082b08, 0x1919082b08190819, 0x1919082b08191908, 0x1919082b082b0808, - 0x1919082b19080819, 0x1919082b19081908, 0x1919082b19190808, 0x1919082b192b2b19, - 0x1919082b2b080808, 0x1919190808080819, 0x1919190808081908, 0x191919080808192b, - 0x1919190808082b19, 0x1919190808190808, 0x191919080819082b, 0x1919190808191919, - 0x1919190808192b08, 0x19191908082b0819, 0x19191908082b1908, 0x1919190819080808, - 0x191919081908082b, 0x1919190819081919, 0x1919190819082b08, 0x1919190819190819, - 0x1919190819191908, 0x19191908192b0808, 0x191919082b080819, 0x191919082b081908, - 0x191919082b190808, 0x1919191908080808, 0x191919190808082b, 0x1919191908081919, - 0x1919191908082b08, 0x1919191908190819, 0x1919191908191908, 0x19191919082b0808, - 0x1919191919080819, 0x1919191919081908, 0x1919191919190808, 0x191919192b080808, - 0x1919192b08080819, 0x1919192b08081908, 0x1919192b08190808, 0x1919192b082b192b, - 0x1919192b19080808, 0x19192b0808080808, 0x19192b080808082b, 0x19192b0808081919, - 0x19192b0808082b08, 0x19192b0808190819, 0x19192b0808191908, 0x19192b08082b0808, - 0x19192b0819080819, 0x19192b0819081908, 0x19192b0819190808, 0x19192b0819192b2b, - 0x19192b082b080808, 0x19192b1908080819, 0x19192b1908081908, 0x19192b1908190808, - 0x19192b1919080808, 0x19192b2b08080808, 0x19192b2b08192b19, 0x19192b2b2b081919, - 0x19192b2b2b2b2b08, 0x192b080808080819, 0x192b080808081908, 0x192b08080808192b, - 0x192b080808190808, 0x192b08080819082b, 0x192b080808191919, 0x192b080808192b08, - 0x192b0808082b0819, 0x192b0808082b1908, 0x192b080819080808, 0x192b080819081919, - 0x192b080819082b08, 0x192b080819190819, 0x192b080819191908, 0x192b0808192b0808, - 0x192b08082b081908, 0x192b08082b190808, 0x192b081908080808, 0x192b08190808082b, - 0x192b081908081919, 0x192b081908082b08, 0x192b081908190819, 0x192b081908191908, - 0x192b0819082b0808, 0x192b081919080819, 0x192b081919081908, 0x192b081919190808, - 0x192b08192b080808, 0x192b08192b192b19, 0x192b082b08081908, 0x192b082b08190808, - 0x192b082b19080808, 0x192b082b1919192b, 0x192b082b2b2b0819, 0x192b190808080808, - 0x192b190808081919, 0x192b190808082b08, 0x192b190808190819, 0x192b190808191908, - 0x192b1908082b0808, 0x192b190819080819, 0x192b190819081908, 0x192b190819190808, - 0x192b19082b080808, 0x192b191908080819, 0x192b191908081908, 0x192b191908190808, - 0x192b191919080808, 0x192b191919082b2b, 0x192b1919192b2b08, 0x192b19192b19082b, - 0x192b192b08080808, 0x192b192b2b191908, 0x192b2b0808080819, 0x192b2b0808081908, - 0x192b2b0808190808, 0x192b2b08192b1919, 0x192b2b082b192b08, 0x192b2b1908080808, - 0x192b2b19082b2b2b, 0x192b2b2b1908082b, 0x192b2b2b2b2b0819, 0x2b08080808080808, - 0x2b0808080808082b, 0x2b08080808081919, 0x2b08080808082b08, 0x2b08080808190819, - 0x2b08080808191908, 0x2b08080808192b19, 0x2b080808082b0808, 0x2b080808082b1919, - 0x2b08080819080819, 0x2b08080819081908, 0x2b08080819190808, 0x2b0808081919082b, - 0x2b08080819191919, 0x2b08080819192b08, 0x2b080808192b0819, 0x2b0808082b080808, - 0x2b0808082b081919, 0x2b0808082b190819, 0x2b0808082b191908, 0x2b08081908080819, - 0x2b08081908081908, 0x2b08081908082b19, 0x2b08081908190808, 0x2b0808190819082b, - 0x2b08081908191919, 0x2b08081908192b08, 0x2b080819082b0819, 0x2b080819082b1908, - 0x2b08081919080808, 0x2b0808191908082b, 0x2b08081919081919, 0x2b08081919082b08, - 0x2b08081919190819, 0x2b08081919191908, 0x2b0808192b080819, 0x2b0808192b081908, - 0x2b0808192b190808, 0x2b0808192b2b2b19, 0x2b08082b08080808, 0x2b08082b08081919, - 0x2b08082b08082b2b, 0x2b08082b08190819, 0x2b08082b08191908, 0x2b08082b19080819, - 0x2b08082b19081908, 0x2b08082b19190808, 0x2b08190808080819, 0x2b08190808081908, - 0x2b0819080808192b, 0x2b08190808082b19, 0x2b08190808190808, 0x2b0819080819082b, - 0x2b08190808191919, 0x2b08190808192b08, 0x2b081908082b0819, 0x2b08190819080808, - 0x2b0819081908082b, 0x2b08190819081919, 0x2b08190819082b08, 0x2b08190819190819, - 0x2b08190819191908, 0x2b081908192b0808, 0x2b0819082b080819, 0x2b0819082b081908, - 0x2b0819082b190808, 0x2b08191908080808, 0x2b0819190808082b, 0x2b08191908081919, - 0x2b08191908082b08, 0x2b08191908190819, 0x2b08191908191908, 0x2b081919082b0808, - 0x2b08191919080819, 0x2b08191919081908, 0x2b08191919190808, 0x2b0819192b080808, - 0x2b0819192b082b2b, 0x2b08192b08080819, 0x2b08192b08081908, 0x2b08192b08190808, - 0x2b08192b082b2b19, 0x2b08192b19080808, 0x2b082b0808080808, 0x2b082b0808081919, - 0x2b082b0808190819, 0x2b082b0808191908, 0x2b082b0819080819, 0x2b082b0819081908, - 0x2b082b0819190808, 0x2b082b082b2b082b, 0x2b082b1908080819, 0x2b082b1908081908, - 0x2b082b1919080808, 0x2b082b19192b1919, 0x2b082b2b082b082b, 0x2b082b2b19192b08, - 0x2b082b2b19192b2b, 0x2b082b2b2b08082b, 0x2b082b2b2b2b082b, 0x2b19080808080819, - 0x2b19080808081908, 0x2b19080808082b19, 0x2b19080808190808, 0x2b1908080819082b, - 0x2b19080808191919, 0x2b19080808192b08, 0x2b190808082b1908, 0x2b19080819080808, - 0x2b1908081908082b, 0x2b19080819081919, 0x2b19080819082b08, 0x2b19080819190819, - 0x2b19080819191908, 0x2b190808192b0808, 0x2b1908082b080819, 0x2b1908082b081908, - 0x2b1908082b190808, 0x2b19081908080808, 0x2b19081908081919, 0x2b19081908190819, - 0x2b19081908191908, 0x2b19081919080819, 0x2b19081919081908, 0x2b19081919190808, - 0x2b19081919192b2b, 0x2b19082b08080819, 0x2b19082b08081908, 0x2b19082b08190808, - 0x2b19082b19080808, 0x2b19082b2b2b192b, 0x2b19190808080808, 0x2b1919080808082b, - 0x2b19190808081919, 0x2b19190808082b08, 0x2b19190808190819, 0x2b19190808191908, - 0x2b191908082b0808, 0x2b19190819080819, 0x2b19190819081908, 0x2b19190819190808, - 0x2b1919082b080808, 0x2b1919082b19192b, 0x2b19191908080819, 0x2b19191908081908, - 0x2b19191908190808, 0x2b19191919080808, 0x2b1919192b192b08, 0x2b1919192b2b0819, - 0x2b19192b08080808, 0x2b19192b1908192b, 0x2b19192b192b1908, 0x2b192b0808080819, - 0x2b192b0808081908, 0x2b192b0808190808, 0x2b192b08082b192b, 0x2b192b0819080808, - 0x2b192b082b2b2b19, 0x2b192b1908080808, 0x2b192b1919082b19, 0x2b192b191919082b, - 0x2b192b2b2b190808, 0x2b2b080808080808, 0x2b2b080808081919, 0x2b2b080808082b2b, - 0x2b2b080808191908, 0x2b2b0808082b082b, 0x2b2b0808082b2b2b, 0x2b2b080819080819, - 0x2b2b080819081908, 0x2b2b080819190808, 0x2b2b08082b2b082b, 0x2b2b08082b2b2b2b, - 0x2b2b081919080808, 0x2b2b0819192b1919, 0x2b2b082b0808082b, 0x2b2b082b08082b2b, - 0x2b2b082b082b082b, 0x2b2b082b082b2b08, 0x2b2b082b082b2b2b, 0x2b2b082b2b08082b, - 0x2b2b082b2b082b08, 0x2b2b082b2b082b2b, 0x2b2b082b2b2b2b08, 0x2b2b190808080819, - 0x2b2b190808081908, 0x2b2b190808190808, 0x2b2b190819080808, 0x2b2b19082b082b19, - 0x2b2b19082b2b1908, 0x2b2b191908080808, 0x2b2b191908192b19, 0x2b2b192b19190819, - 0x2b2b2b0808082b2b, 0x2b2b2b08082b2b08, 0x2b2b2b082b2b082b, 0x2b2b2b1919191908, - 0x2b2b2b192b08192b, 0x2b2b2b2b08082b08, 0x2b2b2b2b08082b2b, 0x2b2b2b2b082b0808, - 0x2b2b2b2b082b082b, 0x2b2b2b2b082b2b08, 0x2b2b2b2b2b082b08, 0x2b2b2b2b2b2b2b2b, -}; - -constexpr constant static uint32_t iq3xxs_grid[256] = { - 0x04040404, 0x04040414, 0x04040424, 0x04040c0c, 0x04040c1c, 0x04040c3e, 0x04041404, 0x04041414, - 0x04041c0c, 0x04042414, 0x04043e1c, 0x04043e2c, 0x040c040c, 0x040c041c, 0x040c0c04, 0x040c0c14, - 0x040c140c, 0x040c142c, 0x040c1c04, 0x040c1c14, 0x040c240c, 0x040c2c24, 0x040c3e04, 0x04140404, - 0x04140414, 0x04140424, 0x04140c0c, 0x04141404, 0x04141414, 0x04141c0c, 0x04141c1c, 0x04141c3e, - 0x04142c0c, 0x04142c3e, 0x04143e2c, 0x041c040c, 0x041c043e, 0x041c0c04, 0x041c0c14, 0x041c142c, - 0x041c3e04, 0x04240c1c, 0x04241c3e, 0x04242424, 0x04242c3e, 0x04243e1c, 0x04243e2c, 0x042c040c, - 0x042c043e, 0x042c1c14, 0x042c2c14, 0x04341c2c, 0x04343424, 0x043e0c04, 0x043e0c24, 0x043e0c34, - 0x043e241c, 0x043e340c, 0x0c04040c, 0x0c04041c, 0x0c040c04, 0x0c040c14, 0x0c04140c, 0x0c04141c, - 0x0c041c04, 0x0c041c14, 0x0c041c24, 0x0c04243e, 0x0c042c04, 0x0c0c0404, 0x0c0c0414, 0x0c0c0c0c, - 0x0c0c1404, 0x0c0c1414, 0x0c14040c, 0x0c14041c, 0x0c140c04, 0x0c140c14, 0x0c14140c, 0x0c141c04, - 0x0c143e14, 0x0c1c0404, 0x0c1c0414, 0x0c1c1404, 0x0c1c1c0c, 0x0c1c2434, 0x0c1c3434, 0x0c24040c, - 0x0c24042c, 0x0c242c04, 0x0c2c1404, 0x0c2c1424, 0x0c2c2434, 0x0c2c3e0c, 0x0c34042c, 0x0c3e1414, - 0x0c3e2404, 0x14040404, 0x14040414, 0x14040c0c, 0x14040c1c, 0x14041404, 0x14041414, 0x14041434, - 0x14041c0c, 0x14042414, 0x140c040c, 0x140c041c, 0x140c042c, 0x140c0c04, 0x140c0c14, 0x140c140c, - 0x140c1c04, 0x140c341c, 0x140c343e, 0x140c3e04, 0x14140404, 0x14140414, 0x14140c0c, 0x14140c3e, - 0x14141404, 0x14141414, 0x14141c3e, 0x14142404, 0x14142c2c, 0x141c040c, 0x141c0c04, 0x141c0c24, - 0x141c3e04, 0x141c3e24, 0x14241c2c, 0x14242c1c, 0x142c041c, 0x142c143e, 0x142c240c, 0x142c3e24, - 0x143e040c, 0x143e041c, 0x143e0c34, 0x143e242c, 0x1c04040c, 0x1c040c04, 0x1c040c14, 0x1c04140c, - 0x1c04141c, 0x1c042c04, 0x1c04342c, 0x1c043e14, 0x1c0c0404, 0x1c0c0414, 0x1c0c1404, 0x1c0c1c0c, - 0x1c0c2424, 0x1c0c2434, 0x1c14040c, 0x1c14041c, 0x1c140c04, 0x1c14142c, 0x1c142c14, 0x1c143e14, - 0x1c1c0c0c, 0x1c1c1c1c, 0x1c241c04, 0x1c24243e, 0x1c243e14, 0x1c2c0404, 0x1c2c0434, 0x1c2c1414, - 0x1c2c2c2c, 0x1c340c24, 0x1c341c34, 0x1c34341c, 0x1c3e1c1c, 0x1c3e3404, 0x24040424, 0x24040c3e, - 0x24041c2c, 0x24041c3e, 0x24042c1c, 0x24042c3e, 0x240c3e24, 0x24141404, 0x24141c3e, 0x24142404, - 0x24143404, 0x24143434, 0x241c043e, 0x241c242c, 0x24240424, 0x24242c0c, 0x24243424, 0x242c142c, - 0x242c241c, 0x242c3e04, 0x243e042c, 0x243e0c04, 0x243e0c14, 0x243e1c04, 0x2c040c14, 0x2c04240c, - 0x2c043e04, 0x2c0c0404, 0x2c0c0434, 0x2c0c1434, 0x2c0c2c2c, 0x2c140c24, 0x2c141c14, 0x2c143e14, - 0x2c1c0414, 0x2c1c2c1c, 0x2c240c04, 0x2c24141c, 0x2c24143e, 0x2c243e14, 0x2c2c0414, 0x2c2c1c0c, - 0x2c342c04, 0x2c3e1424, 0x2c3e2414, 0x34041424, 0x34042424, 0x34042434, 0x34043424, 0x340c140c, - 0x340c340c, 0x34140c3e, 0x34143424, 0x341c1c04, 0x341c1c34, 0x34242424, 0x342c042c, 0x342c2c14, - 0x34341c1c, 0x343e041c, 0x343e140c, 0x3e04041c, 0x3e04042c, 0x3e04043e, 0x3e040c04, 0x3e041c14, - 0x3e042c14, 0x3e0c1434, 0x3e0c2404, 0x3e140c14, 0x3e14242c, 0x3e142c14, 0x3e1c0404, 0x3e1c0c2c, - 0x3e1c1c1c, 0x3e1c3404, 0x3e24140c, 0x3e24240c, 0x3e2c0404, 0x3e2c0414, 0x3e2c1424, 0x3e341c04, -}; - -constexpr constant static uint32_t iq3s_grid[512] = { - 0x01010101, 0x01010103, 0x01010105, 0x0101010b, 0x0101010f, 0x01010301, 0x01010303, 0x01010305, - 0x01010309, 0x0101030d, 0x01010501, 0x01010503, 0x0101050b, 0x01010707, 0x01010901, 0x01010905, - 0x0101090b, 0x0101090f, 0x01010b03, 0x01010b07, 0x01010d01, 0x01010d05, 0x01010f03, 0x01010f09, - 0x01010f0f, 0x01030101, 0x01030103, 0x01030105, 0x01030109, 0x01030301, 0x01030303, 0x0103030b, - 0x01030501, 0x01030507, 0x0103050f, 0x01030703, 0x0103070b, 0x01030909, 0x01030d03, 0x01030d0b, - 0x01030f05, 0x01050101, 0x01050103, 0x0105010b, 0x0105010f, 0x01050301, 0x01050307, 0x0105030d, - 0x01050503, 0x0105050b, 0x01050701, 0x01050709, 0x01050905, 0x0105090b, 0x0105090f, 0x01050b03, - 0x01050b07, 0x01050f01, 0x01050f07, 0x01070107, 0x01070303, 0x0107030b, 0x01070501, 0x01070505, - 0x01070703, 0x01070707, 0x0107070d, 0x01070909, 0x01070b01, 0x01070b05, 0x01070d0f, 0x01070f03, - 0x01070f0b, 0x01090101, 0x01090307, 0x0109030f, 0x01090503, 0x01090509, 0x01090705, 0x01090901, - 0x01090907, 0x01090b03, 0x01090f01, 0x010b0105, 0x010b0109, 0x010b0501, 0x010b0505, 0x010b050d, - 0x010b0707, 0x010b0903, 0x010b090b, 0x010b090f, 0x010b0d0d, 0x010b0f07, 0x010d010d, 0x010d0303, - 0x010d0307, 0x010d0703, 0x010d0b05, 0x010d0f03, 0x010f0101, 0x010f0105, 0x010f0109, 0x010f0501, - 0x010f0505, 0x010f050d, 0x010f0707, 0x010f0b01, 0x010f0b09, 0x03010101, 0x03010103, 0x03010105, - 0x03010109, 0x03010301, 0x03010303, 0x03010307, 0x0301030b, 0x0301030f, 0x03010501, 0x03010505, - 0x03010703, 0x03010709, 0x0301070d, 0x03010b09, 0x03010b0d, 0x03010d03, 0x03010f05, 0x03030101, - 0x03030103, 0x03030107, 0x0303010d, 0x03030301, 0x03030309, 0x03030503, 0x03030701, 0x03030707, - 0x03030903, 0x03030b01, 0x03030b05, 0x03030f01, 0x03030f0d, 0x03050101, 0x03050305, 0x0305030b, - 0x0305030f, 0x03050501, 0x03050509, 0x03050705, 0x03050901, 0x03050907, 0x03050b0b, 0x03050d01, - 0x03050f05, 0x03070103, 0x03070109, 0x0307010f, 0x03070301, 0x03070307, 0x03070503, 0x0307050f, - 0x03070701, 0x03070709, 0x03070903, 0x03070d05, 0x03070f01, 0x03090107, 0x0309010b, 0x03090305, - 0x03090309, 0x03090703, 0x03090707, 0x03090905, 0x0309090d, 0x03090b01, 0x03090b09, 0x030b0103, - 0x030b0301, 0x030b0307, 0x030b0503, 0x030b0701, 0x030b0705, 0x030b0b03, 0x030d0501, 0x030d0509, - 0x030d050f, 0x030d0909, 0x030d090d, 0x030f0103, 0x030f0107, 0x030f0301, 0x030f0305, 0x030f0503, - 0x030f070b, 0x030f0903, 0x030f0d05, 0x030f0f01, 0x05010101, 0x05010103, 0x05010107, 0x0501010b, - 0x0501010f, 0x05010301, 0x05010305, 0x05010309, 0x0501030d, 0x05010503, 0x05010507, 0x0501050f, - 0x05010701, 0x05010705, 0x05010903, 0x05010907, 0x0501090b, 0x05010b01, 0x05010b05, 0x05010d0f, - 0x05010f01, 0x05010f07, 0x05010f0b, 0x05030101, 0x05030105, 0x05030301, 0x05030307, 0x0503030f, - 0x05030505, 0x0503050b, 0x05030703, 0x05030709, 0x05030905, 0x05030b03, 0x05050103, 0x05050109, - 0x0505010f, 0x05050503, 0x05050507, 0x05050701, 0x0505070f, 0x05050903, 0x05050b07, 0x05050b0f, - 0x05050f03, 0x05050f09, 0x05070101, 0x05070105, 0x0507010b, 0x05070303, 0x05070505, 0x05070509, - 0x05070703, 0x05070707, 0x05070905, 0x05070b01, 0x05070d0d, 0x05090103, 0x0509010f, 0x05090501, - 0x05090507, 0x05090705, 0x0509070b, 0x05090903, 0x05090f05, 0x05090f0b, 0x050b0109, 0x050b0303, - 0x050b0505, 0x050b070f, 0x050b0901, 0x050b0b07, 0x050b0f01, 0x050d0101, 0x050d0105, 0x050d010f, - 0x050d0503, 0x050d0b0b, 0x050d0d03, 0x050f010b, 0x050f0303, 0x050f050d, 0x050f0701, 0x050f0907, - 0x050f0b01, 0x07010105, 0x07010303, 0x07010307, 0x0701030b, 0x0701030f, 0x07010505, 0x07010703, - 0x07010707, 0x0701070b, 0x07010905, 0x07010909, 0x0701090f, 0x07010b03, 0x07010d07, 0x07010f03, - 0x07030103, 0x07030107, 0x0703010b, 0x07030309, 0x07030503, 0x07030507, 0x07030901, 0x07030d01, - 0x07030f05, 0x07030f0d, 0x07050101, 0x07050305, 0x07050501, 0x07050705, 0x07050709, 0x07050b01, - 0x07070103, 0x07070301, 0x07070309, 0x07070503, 0x07070507, 0x0707050f, 0x07070701, 0x07070903, - 0x07070907, 0x0707090f, 0x07070b0b, 0x07070f07, 0x07090107, 0x07090303, 0x0709030d, 0x07090505, - 0x07090703, 0x07090b05, 0x07090d01, 0x07090d09, 0x070b0103, 0x070b0301, 0x070b0305, 0x070b050b, - 0x070b0705, 0x070b0909, 0x070b0b0d, 0x070b0f07, 0x070d030d, 0x070d0903, 0x070f0103, 0x070f0107, - 0x070f0501, 0x070f0505, 0x070f070b, 0x09010101, 0x09010109, 0x09010305, 0x09010501, 0x09010509, - 0x0901050f, 0x09010705, 0x09010903, 0x09010b01, 0x09010f01, 0x09030105, 0x0903010f, 0x09030303, - 0x09030307, 0x09030505, 0x09030701, 0x0903070b, 0x09030907, 0x09030b03, 0x09030b0b, 0x09050103, - 0x09050107, 0x09050301, 0x0905030b, 0x09050503, 0x09050707, 0x09050901, 0x09050b0f, 0x09050d05, - 0x09050f01, 0x09070109, 0x09070303, 0x09070307, 0x09070501, 0x09070505, 0x09070703, 0x0907070b, - 0x09090101, 0x09090105, 0x09090509, 0x0909070f, 0x09090901, 0x09090f03, 0x090b010b, 0x090b010f, - 0x090b0503, 0x090b0d05, 0x090d0307, 0x090d0709, 0x090d0d01, 0x090f0301, 0x090f030b, 0x090f0701, - 0x090f0907, 0x090f0b03, 0x0b010105, 0x0b010301, 0x0b010309, 0x0b010505, 0x0b010901, 0x0b010909, - 0x0b01090f, 0x0b010b05, 0x0b010d0d, 0x0b010f09, 0x0b030103, 0x0b030107, 0x0b03010b, 0x0b030305, - 0x0b030503, 0x0b030705, 0x0b030f05, 0x0b050101, 0x0b050303, 0x0b050507, 0x0b050701, 0x0b05070d, - 0x0b050b07, 0x0b070105, 0x0b07010f, 0x0b070301, 0x0b07050f, 0x0b070909, 0x0b070b03, 0x0b070d0b, - 0x0b070f07, 0x0b090103, 0x0b090109, 0x0b090501, 0x0b090705, 0x0b09090d, 0x0b0b0305, 0x0b0b050d, - 0x0b0b0b03, 0x0b0b0b07, 0x0b0d0905, 0x0b0f0105, 0x0b0f0109, 0x0b0f0505, 0x0d010303, 0x0d010307, - 0x0d01030b, 0x0d010703, 0x0d010707, 0x0d010d01, 0x0d030101, 0x0d030501, 0x0d03050f, 0x0d030d09, - 0x0d050305, 0x0d050709, 0x0d050905, 0x0d050b0b, 0x0d050d05, 0x0d050f01, 0x0d070101, 0x0d070309, - 0x0d070503, 0x0d070901, 0x0d09050b, 0x0d090907, 0x0d090d05, 0x0d0b0101, 0x0d0b0107, 0x0d0b0709, - 0x0d0b0d01, 0x0d0d010b, 0x0d0d0901, 0x0d0f0303, 0x0d0f0307, 0x0f010101, 0x0f010109, 0x0f01010f, - 0x0f010501, 0x0f010505, 0x0f01070d, 0x0f010901, 0x0f010b09, 0x0f010d05, 0x0f030105, 0x0f030303, - 0x0f030509, 0x0f030907, 0x0f03090b, 0x0f050103, 0x0f050109, 0x0f050301, 0x0f05030d, 0x0f050503, - 0x0f050701, 0x0f050b03, 0x0f070105, 0x0f070705, 0x0f07070b, 0x0f070b07, 0x0f090103, 0x0f09010b, - 0x0f090307, 0x0f090501, 0x0f090b01, 0x0f0b0505, 0x0f0b0905, 0x0f0d0105, 0x0f0d0703, 0x0f0f0101, -}; - -#define NGRID_IQ1S 512 -constexpr constant static uint64_t iq1s_grid[NGRID_IQ1S] = { - 0xffffffffffff0101, 0xffffffffff01ff00, 0xffffffffff010100, 0xffffffff00000000, - 0xffffffff01ff00ff, 0xffffffff01ff0001, 0xffffffff0101ffff, 0xffffffff0101ff01, - 0xffffff00ff000000, 0xffffff000000ff00, 0xffffff00000000ff, 0xffffff0000000100, - 0xffffff0000010000, 0xffffff0001000000, 0xffffff01ffff00ff, 0xffffff01ff01ff00, - 0xffffff01ff010100, 0xffffff0100000001, 0xffffff0101ffff00, 0xffffff0101ff0101, - 0xffffff0101010100, 0xffff00ffff00ff01, 0xffff00ffff0000ff, 0xffff00ff00ff0100, - 0xffff00ff0100ff00, 0xffff00ff010001ff, 0xffff0000ff0101ff, 0xffff000000ffff00, - 0xffff000000000000, 0xffff00000001ff01, 0xffff000001000101, 0xffff0000010100ff, - 0xffff0001ffff0100, 0xffff00010000ff00, 0xffff000100010101, 0xffff000101000000, - 0xffff01ffffff0000, 0xffff01ffff01ffff, 0xffff01ffff010100, 0xffff01ff00000000, - 0xffff01ff01ffffff, 0xffff01ff01ff0001, 0xffff01ff0101ffff, 0xffff01ff01010001, - 0xffff0100ffffff01, 0xffff01000000ffff, 0xffff010000000100, 0xffff010001ff01ff, - 0xffff010001000000, 0xffff0101ff000000, 0xffff0101000101ff, 0xffff010101ffff01, - 0xffff01010101ff00, 0xff00ffffff000000, 0xff00ffff00ffff00, 0xff00ffff00000001, - 0xff00ffff000001ff, 0xff00ffff01010000, 0xff00ff00ffff0000, 0xff00ff00ff00ff00, - 0xff00ff00ff0000ff, 0xff00ff00ff000100, 0xff00ff00ff010001, 0xff00ff0000ff0001, - 0xff00ff000000ffff, 0xff00ff0000000000, 0xff00ff000001ff00, 0xff00ff0000010100, - 0xff00ff0001ff0000, 0xff00ff000100ff00, 0xff00ff0001000100, 0xff00ff01ff000000, - 0xff00ff0100ff0000, 0xff00ff01000001ff, 0xff00ff0101010001, 0xff0000ff00000000, - 0xff0000ff0001ff00, 0xff0000ff00010100, 0xff000000ffff0101, 0xff000000ff000000, - 0xff000000ff01ff00, 0xff00000000ff0000, 0xff0000000000ff00, 0xff000000000000ff, - 0xff00000000000000, 0xff00000000000001, 0xff00000000000100, 0xff0000000001ffff, - 0xff00000000010000, 0xff00000001000000, 0xff00000001010100, 0xff000001ff00ff01, - 0xff000001ff0100ff, 0xff00000100000000, 0xff0000010001ff00, 0xff00000101ff0100, - 0xff0000010100ff00, 0xff0001ff00ff00ff, 0xff0001ff00000101, 0xff0001ff000100ff, - 0xff0001ff01000000, 0xff000100ff0001ff, 0xff0001000000ff01, 0xff00010000000000, - 0xff00010000010001, 0xff00010000010100, 0xff00010001ffff00, 0xff00010001ff0101, - 0xff00010001010000, 0xff000101ffffffff, 0xff000101ff000101, 0xff00010101ff00ff, - 0xff00010101000001, 0xff000101010100ff, 0xff01ffffff000101, 0xff01ffffff01ffff, - 0xff01ffffff01ff01, 0xff01ffffff0101ff, 0xff01ffff00000000, 0xff01ffff01ff0001, - 0xff01ffff0101ff01, 0xff01ff00ff000000, 0xff01ff0000ff0100, 0xff01ff000000ff01, - 0xff01ff0000010000, 0xff01ff00010000ff, 0xff01ff01ff01ff00, 0xff01ff0100000101, - 0xff0100ffffff0000, 0xff0100ffff010000, 0xff0100ff01ff00ff, 0xff0100ff01000100, - 0xff0100ff010100ff, 0xff010000ffffff01, 0xff01000000000000, 0xff0100000101ff00, - 0xff010001ffff00ff, 0xff010001ff000100, 0xff01000100ffff00, 0xff01000100010001, - 0xff01000101ff0001, 0xff010001010001ff, 0xff0101ffffffffff, 0xff0101ffff01ffff, - 0xff0101ffff010101, 0xff0101ff0000ff00, 0xff0101ff01010001, 0xff010100ff000000, - 0xff010100ff01ff01, 0xff01010000ff0001, 0xff01010000000100, 0xff01010001000000, - 0xff0101010100ffff, 0x00ffffff0000ff01, 0x00ffffff000000ff, 0x00ffffff00000100, - 0x00ffffff00010000, 0x00ffff00ffff0001, 0x00ffff00ff0000ff, 0x00ffff00ff000100, - 0x00ffff0000000000, 0x00ffff0001000100, 0x00ffff0001010001, 0x00ffff01ff00ff01, - 0x00ffff0100ff0100, 0x00ffff010000ff00, 0x00ffff01000100ff, 0x00ffff0101ff00ff, - 0x00ffff010101ff00, 0x00ff00ffffffffff, 0x00ff00ffffff01ff, 0x00ff00ffff000101, - 0x00ff00ff00000000, 0x00ff00ff000101ff, 0x00ff00ff01010101, 0x00ff0000ff000000, - 0x00ff0000ff01ffff, 0x00ff000000ff0000, 0x00ff00000000ff00, 0x00ff0000000000ff, - 0x00ff000000000000, 0x00ff000000000001, 0x00ff000000000100, 0x00ff000000010000, - 0x00ff000001ffff01, 0x00ff000001000000, 0x00ff0001ff000101, 0x00ff000100ffffff, - 0x00ff000100000000, 0x00ff0001010001ff, 0x00ff01ffff000000, 0x00ff01ff0001ff00, - 0x00ff01ff01ff0100, 0x00ff0100ff01ff01, 0x00ff010000ff00ff, 0x00ff010000ff0101, - 0x00ff010000000000, 0x00ff010000010101, 0x00ff01000100ff00, 0x00ff010001010000, - 0x00ff0101ffffff00, 0x00ff01010000ff01, 0x00ff010100000100, 0x00ff010101ff0000, - 0x0000ffffffff0100, 0x0000ffffff00ff00, 0x0000ffffff0000ff, 0x0000ffffff010000, - 0x0000ffff00000000, 0x0000ffff00010101, 0x0000ffff01ffff01, 0x0000ffff01000100, - 0x0000ff00ff000000, 0x0000ff00ff01ff00, 0x0000ff00ff0101ff, 0x0000ff0000ff0000, - 0x0000ff000000ff00, 0x0000ff00000000ff, 0x0000ff0000000000, 0x0000ff0000000001, - 0x0000ff0000000100, 0x0000ff0000010000, 0x0000ff0001ffffff, 0x0000ff0001ff01ff, - 0x0000ff0001000000, 0x0000ff000101ffff, 0x0000ff01ffff0101, 0x0000ff01ff010000, - 0x0000ff0100000000, 0x0000ff0101000101, 0x000000ffffff0001, 0x000000ffff000000, - 0x000000ff00ff0000, 0x000000ff0000ff00, 0x000000ff000000ff, 0x000000ff00000000, - 0x000000ff00000001, 0x000000ff00000100, 0x000000ff00010000, 0x000000ff01000000, - 0x000000ff0101ff00, 0x00000000ffff0000, 0x00000000ff00ff00, 0x00000000ff0000ff, - 0x00000000ff000000, 0x00000000ff000001, 0x00000000ff000100, 0x00000000ff010000, - 0x0000000000ffff00, 0x0000000000ff00ff, 0x0000000000ff0000, 0x0000000000ff0001, - 0x0000000000ff0100, 0x000000000000ffff, 0x000000000000ff00, 0x000000000000ff01, - 0x00000000000000ff, 0x0000000000000001, 0x00000000000001ff, 0x0000000000000100, - 0x0000000000000101, 0x000000000001ff00, 0x00000000000100ff, 0x0000000000010000, - 0x0000000000010001, 0x0000000000010100, 0x0000000001ff0000, 0x000000000100ff00, - 0x00000000010000ff, 0x0000000001000000, 0x0000000001000001, 0x0000000001000100, - 0x0000000001010000, 0x00000001ffff01ff, 0x00000001ff000000, 0x0000000100ff0000, - 0x000000010000ff00, 0x00000001000000ff, 0x0000000100000000, 0x0000000100000001, - 0x0000000100000100, 0x0000000100010000, 0x0000000101000000, 0x000001ffff00ff00, - 0x000001ffff010001, 0x000001ffff0101ff, 0x000001ff00ffff01, 0x000001ff0000ffff, - 0x000001ff00000000, 0x000001ff010000ff, 0x000001ff01010100, 0x00000100ffff0100, - 0x00000100ff000000, 0x0000010000ff0000, 0x000001000000ff00, 0x00000100000000ff, - 0x0000010000000000, 0x0000010000000001, 0x0000010000000100, 0x0000010000010000, - 0x0000010001000000, 0x000001000101ff01, 0x00000101ffff0001, 0x00000101ff01ffff, - 0x0000010100000000, 0x0000010101010100, 0x0001ffffff000000, 0x0001ffff00ffffff, - 0x0001ffff00000100, 0x0001ffff0001ff00, 0x0001ffff01000000, 0x0001ff00ffffff00, - 0x0001ff00ffff01ff, 0x0001ff00ff010000, 0x0001ff0000000000, 0x0001ff0000010001, - 0x0001ff0001ff0000, 0x0001ff0001010100, 0x0001ff01ff0000ff, 0x0001ff01ff000001, - 0x0001ff0100ffffff, 0x0001ff010001ffff, 0x0001ff01000101ff, 0x0001ff010100ff01, - 0x000100ffff00ffff, 0x000100ffff00ff01, 0x000100ffff000100, 0x000100ff00000000, - 0x000100ff000101ff, 0x000100ff01ff0101, 0x000100ff0100ffff, 0x000100ff01010101, - 0x00010000ff000000, 0x00010000ff010100, 0x0001000000ff0000, 0x000100000000ff00, - 0x00010000000000ff, 0x0001000000000000, 0x0001000000000001, 0x0001000000000100, - 0x0001000000010000, 0x0001000001ffff01, 0x0001000001000000, 0x0001000100ff0101, - 0x0001000100000000, 0x00010001010100ff, 0x000101ffffff01ff, 0x000101ffffff0101, - 0x000101ff00010000, 0x000101ff01ff0000, 0x000101ff0100ff01, 0x00010100ffff0000, - 0x0001010000000000, 0x000101000001ffff, 0x0001010000010101, 0x00010100010001ff, - 0x00010101ff00ff00, 0x00010101ff010001, 0x0001010100ffffff, 0x0001010100ff01ff, - 0x00010101000101ff, 0x0001010101ff0000, 0x000101010100ff01, 0x0001010101000101, - 0x01ffffffffff0101, 0x01ffffffff01ffff, 0x01ffffffff01ff01, 0x01ffffffff0101ff, - 0x01ffffffff010101, 0x01ffffff00000000, 0x01ffffff01ff01ff, 0x01ffffff01000101, - 0x01ffffff0101ff01, 0x01ffffff010100ff, 0x01ffff000000ff00, 0x01ffff0000000001, - 0x01ffff00000001ff, 0x01ffff0000010000, 0x01ffff0001ff0000, 0x01ffff01ffffffff, - 0x01ffff01ffff01ff, 0x01ffff01ff000000, 0x01ffff01ff01ffff, 0x01ffff01ff0101ff, - 0x01ffff010100ffff, 0x01ff00ffffff0000, 0x01ff00ffff010000, 0x01ff00ff00ffff01, - 0x01ff0000ff0000ff, 0x01ff000000000000, 0x01ff00000001ff01, 0x01ff000001ffffff, - 0x01ff000001010100, 0x01ff0001ffffff01, 0x01ff0001ff010001, 0x01ff000101ff0100, - 0x01ff000101000001, 0x01ff0001010100ff, 0x01ff01ffff00ffff, 0x01ff01ff00010001, - 0x01ff01ff01000000, 0x01ff01ff010101ff, 0x01ff0100ff000001, 0x01ff010000ffff00, - 0x01ff010000000100, 0x01ff010001ff01ff, 0x01ff01000101ffff, 0x01ff0101ffff00ff, - 0x01ff0101ffff0101, 0x01ff0101ff0101ff, 0x01ff010100010000, 0x0100ffff00ff00ff, - 0x0100ffff00ff0001, 0x0100ffff00000100, 0x0100ffff0100ff00, 0x0100ff00ffff0000, - 0x0100ff00ff00ffff, 0x0100ff00ff00ff01, 0x0100ff00ff000100, 0x0100ff00ff010000, - 0x0100ff0000000000, 0x0100ff00000100ff, 0x0100ff0001ff0101, 0x0100ff0001010101, - 0x0100ff0100ff00ff, 0x0100ff0100ff0001, 0x0100ff0100000100, 0x0100ff0100010001, - 0x0100ff0101000000, 0x010000ffff00ff00, 0x010000ff0000ffff, 0x010000ff00000000, - 0x010000ff010001ff, 0x010000ff01010001, 0x01000000ffffff00, 0x01000000ffff0101, - 0x01000000ff000000, 0x01000000ff0100ff, 0x01000000ff010101, 0x0100000000ff0000, - 0x010000000000ff00, 0x01000000000000ff, 0x0100000000000000, 0x0100000000000001, - 0x0100000000000100, 0x0100000000010000, 0x0100000001000000, 0x0100000100000000, - 0x01000001000101ff, 0x0100000101ffff01, 0x010001ffff000101, 0x010001ff00ff0100, - 0x010001ff0000ff00, 0x010001ff000100ff, 0x010001ff01ffffff, 0x01000100ffff0000, - 0x01000100ff0001ff, 0x0100010000000000, 0x010001000001ff00, 0x0100010001ff0000, - 0x01000100010000ff, 0x0100010001000101, 0x01000101ff00ff01, 0x0100010100ff0100, - 0x010001010000ffff, 0x0100010101010001, 0x0101ffffffff0101, 0x0101ffffff0001ff, - 0x0101ffffff01ffff, 0x0101ffffff010101, 0x0101ffff00000000, 0x0101ffff0101ffff, - 0x0101ffff010101ff, 0x0101ff00ff000000, 0x0101ff0000ff0100, 0x0101ff000000ff00, - 0x0101ff0000010000, 0x0101ff00010000ff, 0x0101ff0001000001, 0x0101ff01ff010101, - 0x0101ff0100000000, 0x0101ff010101ff00, 0x010100ffffff0000, 0x010100ffff010000, - 0x010100ff00ff01ff, 0x010100ff000000ff, 0x010100ff00000101, 0x010100ff01ffff00, - 0x01010000ffffff01, 0x01010000ff000100, 0x01010000ff01ff01, 0x0101000000000000, - 0x01010000000100ff, 0x010100000101ff01, 0x01010001ffff0000, 0x01010001ff00ffff, - 0x01010001ff010000, 0x0101000101ffffff, 0x0101000101ff01ff, 0x0101000101010101, - 0x010101ffff01ffff, 0x010101ff00000000, 0x010101ff0001ff01, 0x010101ff0101ffff, - 0x010101ff010101ff, 0x01010100ffffffff, 0x01010100ff000001, 0x010101000000ff00, - 0x0101010001010000, 0x0101010100ff0001, 0x010101010001ff01, 0x010101010101ffff, -}; - -constexpr constant static uint8_t ksigns_iq2xs[128] = { - 0, 129, 130, 3, 132, 5, 6, 135, 136, 9, 10, 139, 12, 141, 142, 15, - 144, 17, 18, 147, 20, 149, 150, 23, 24, 153, 154, 27, 156, 29, 30, 159, - 160, 33, 34, 163, 36, 165, 166, 39, 40, 169, 170, 43, 172, 45, 46, 175, - 48, 177, 178, 51, 180, 53, 54, 183, 184, 57, 58, 187, 60, 189, 190, 63, - 192, 65, 66, 195, 68, 197, 198, 71, 72, 201, 202, 75, 204, 77, 78, 207, - 80, 209, 210, 83, 212, 85, 86, 215, 216, 89, 90, 219, 92, 221, 222, 95, - 96, 225, 226, 99, 228, 101, 102, 231, 232, 105, 106, 235, 108, 237, 238, 111, - 240, 113, 114, 243, 116, 245, 246, 119, 120, 249, 250, 123, 252, 125, 126, 255, -}; - -constexpr constant static uint8_t kmask_iq2xs[8] = {1, 2, 4, 8, 16, 32, 64, 128}; - void kernel_mul_mv_iq2_xxs_f32_impl( device const void * src0, device const float * src1, diff --git a/ggml-quants.c b/ggml-quants.c index 9dcb76def..5bb46def3 100644 --- a/ggml-quants.c +++ b/ggml-quants.c @@ -1,6 +1,9 @@ #include "ggml-quants.h" #include "ggml-impl.h" +#define GGML_COMMON_IMPL_C +#include "ggml-common.h" + #include #include #include @@ -3327,711 +3330,6 @@ size_t quantize_q5_1(const float * src, void * dst, int nrow, int n_per_row, int // ====================== "True" 2-bit (de)-quantization -static const uint64_t iq2xxs_grid[256] = { - 0x0808080808080808, 0x080808080808082b, 0x0808080808081919, 0x0808080808082b08, - 0x0808080808082b2b, 0x0808080808190819, 0x0808080808191908, 0x08080808082b0808, - 0x08080808082b082b, 0x08080808082b2b08, 0x08080808082b2b2b, 0x0808080819080819, - 0x0808080819081908, 0x0808080819190808, 0x0808080819192b08, 0x08080808192b0819, - 0x08080808192b1908, 0x080808082b080808, 0x080808082b08082b, 0x080808082b082b2b, - 0x080808082b2b082b, 0x0808081908080819, 0x0808081908081908, 0x0808081908190808, - 0x0808081908191919, 0x0808081919080808, 0x080808192b081908, 0x080808192b192b08, - 0x0808082b08080808, 0x0808082b0808082b, 0x0808082b082b082b, 0x0808082b2b08082b, - 0x0808190808080819, 0x0808190808081908, 0x0808190808190808, 0x08081908082b0819, - 0x08081908082b1908, 0x0808190819080808, 0x080819081908082b, 0x0808190819082b08, - 0x08081908192b0808, 0x080819082b080819, 0x080819082b081908, 0x080819082b190808, - 0x080819082b2b1908, 0x0808191908080808, 0x080819190808082b, 0x0808191908082b08, - 0x08081919082b0808, 0x080819191908192b, 0x08081919192b2b19, 0x080819192b080808, - 0x080819192b190819, 0x0808192b08082b19, 0x0808192b08190808, 0x0808192b19080808, - 0x0808192b2b081908, 0x0808192b2b2b1908, 0x08082b0808080808, 0x08082b0808081919, - 0x08082b0808082b08, 0x08082b0808191908, 0x08082b08082b2b08, 0x08082b0819080819, - 0x08082b0819081908, 0x08082b0819190808, 0x08082b081919082b, 0x08082b082b082b08, - 0x08082b1908081908, 0x08082b1919080808, 0x08082b2b0808082b, 0x08082b2b08191908, - 0x0819080808080819, 0x0819080808081908, 0x0819080808190808, 0x08190808082b0819, - 0x0819080819080808, 0x08190808192b0808, 0x081908082b081908, 0x081908082b190808, - 0x081908082b191919, 0x0819081908080808, 0x0819081908082b08, 0x08190819082b0808, - 0x0819081919190808, 0x0819081919192b2b, 0x081908192b080808, 0x0819082b082b1908, - 0x0819082b19081919, 0x0819190808080808, 0x0819190808082b08, 0x08191908082b0808, - 0x08191908082b1919, 0x0819190819082b19, 0x081919082b080808, 0x0819191908192b08, - 0x08191919192b082b, 0x0819192b08080808, 0x0819192b0819192b, 0x08192b0808080819, - 0x08192b0808081908, 0x08192b0808190808, 0x08192b0819080808, 0x08192b082b080819, - 0x08192b1908080808, 0x08192b1908081919, 0x08192b192b2b0808, 0x08192b2b19190819, - 0x082b080808080808, 0x082b08080808082b, 0x082b080808082b2b, 0x082b080819081908, - 0x082b0808192b0819, 0x082b08082b080808, 0x082b08082b08082b, 0x082b0819082b2b19, - 0x082b081919082b08, 0x082b082b08080808, 0x082b082b0808082b, 0x082b190808080819, - 0x082b190808081908, 0x082b190808190808, 0x082b190819080808, 0x082b19081919192b, - 0x082b191908080808, 0x082b191919080819, 0x082b1919192b1908, 0x082b192b2b190808, - 0x082b2b0808082b08, 0x082b2b08082b0808, 0x082b2b082b191908, 0x082b2b2b19081908, - 0x1908080808080819, 0x1908080808081908, 0x1908080808190808, 0x1908080808192b08, - 0x19080808082b0819, 0x19080808082b1908, 0x1908080819080808, 0x1908080819082b08, - 0x190808081919192b, 0x19080808192b0808, 0x190808082b080819, 0x190808082b081908, - 0x190808082b190808, 0x1908081908080808, 0x19080819082b0808, 0x19080819192b0819, - 0x190808192b080808, 0x190808192b081919, 0x1908082b08080819, 0x1908082b08190808, - 0x1908082b19082b08, 0x1908082b1919192b, 0x1908082b192b2b08, 0x1908190808080808, - 0x1908190808082b08, 0x19081908082b0808, 0x190819082b080808, 0x190819082b192b19, - 0x190819190819082b, 0x19081919082b1908, 0x1908192b08080808, 0x19082b0808080819, - 0x19082b0808081908, 0x19082b0808190808, 0x19082b0819080808, 0x19082b0819081919, - 0x19082b1908080808, 0x19082b1919192b08, 0x19082b19192b0819, 0x19082b192b08082b, - 0x19082b2b19081919, 0x19082b2b2b190808, 0x1919080808080808, 0x1919080808082b08, - 0x1919080808190819, 0x1919080808192b19, 0x19190808082b0808, 0x191908082b080808, - 0x191908082b082b08, 0x1919081908081908, 0x191908191908082b, 0x191908192b2b1908, - 0x1919082b2b190819, 0x191919082b190808, 0x191919082b19082b, 0x1919191908082b2b, - 0x1919192b08080819, 0x1919192b19191908, 0x19192b0808080808, 0x19192b0808190819, - 0x19192b0808192b19, 0x19192b08192b1908, 0x19192b1919080808, 0x19192b2b08082b08, - 0x192b080808081908, 0x192b080808190808, 0x192b080819080808, 0x192b0808192b2b08, - 0x192b081908080808, 0x192b081919191919, 0x192b082b08192b08, 0x192b082b192b0808, - 0x192b190808080808, 0x192b190808081919, 0x192b191908190808, 0x192b19190819082b, - 0x192b19192b081908, 0x192b2b081908082b, 0x2b08080808080808, 0x2b0808080808082b, - 0x2b08080808082b2b, 0x2b08080819080819, 0x2b0808082b08082b, 0x2b08081908081908, - 0x2b08081908192b08, 0x2b08081919080808, 0x2b08082b08190819, 0x2b08190808080819, - 0x2b08190808081908, 0x2b08190808190808, 0x2b08190808191919, 0x2b08190819080808, - 0x2b081908192b0808, 0x2b08191908080808, 0x2b0819191908192b, 0x2b0819192b191908, - 0x2b08192b08082b19, 0x2b08192b19080808, 0x2b08192b192b0808, 0x2b082b080808082b, - 0x2b082b1908081908, 0x2b082b2b08190819, 0x2b19080808081908, 0x2b19080808190808, - 0x2b190808082b1908, 0x2b19080819080808, 0x2b1908082b2b0819, 0x2b1908190819192b, - 0x2b1908192b080808, 0x2b19082b19081919, 0x2b19190808080808, 0x2b191908082b082b, - 0x2b19190819081908, 0x2b19191919190819, 0x2b192b082b080819, 0x2b192b19082b0808, - 0x2b2b08080808082b, 0x2b2b080819190808, 0x2b2b08082b081919, 0x2b2b081908082b19, - 0x2b2b082b08080808, 0x2b2b190808192b08, 0x2b2b2b0819190808, 0x2b2b2b1908081908, -}; - -static const uint64_t iq2xs_grid[512] = { - 0x0808080808080808, 0x080808080808082b, 0x0808080808081919, 0x0808080808082b08, - 0x0808080808082b2b, 0x0808080808190819, 0x0808080808191908, 0x080808080819192b, - 0x0808080808192b19, 0x08080808082b0808, 0x08080808082b082b, 0x08080808082b1919, - 0x08080808082b2b08, 0x0808080819080819, 0x0808080819081908, 0x080808081908192b, - 0x0808080819082b19, 0x0808080819190808, 0x080808081919082b, 0x0808080819191919, - 0x0808080819192b08, 0x08080808192b0819, 0x08080808192b1908, 0x080808082b080808, - 0x080808082b08082b, 0x080808082b081919, 0x080808082b082b08, 0x080808082b190819, - 0x080808082b191908, 0x080808082b192b19, 0x080808082b2b0808, 0x0808081908080819, - 0x0808081908081908, 0x080808190808192b, 0x0808081908082b19, 0x0808081908190808, - 0x080808190819082b, 0x0808081908191919, 0x0808081908192b08, 0x0808081908192b2b, - 0x08080819082b0819, 0x08080819082b1908, 0x0808081919080808, 0x080808191908082b, - 0x0808081919081919, 0x0808081919082b08, 0x0808081919190819, 0x0808081919191908, - 0x08080819192b0808, 0x08080819192b2b08, 0x080808192b080819, 0x080808192b081908, - 0x080808192b190808, 0x0808082b08080808, 0x0808082b0808082b, 0x0808082b08081919, - 0x0808082b08082b08, 0x0808082b08190819, 0x0808082b08191908, 0x0808082b082b0808, - 0x0808082b19080819, 0x0808082b19081908, 0x0808082b19190808, 0x0808082b19191919, - 0x0808082b2b080808, 0x0808082b2b082b2b, 0x0808190808080819, 0x0808190808081908, - 0x080819080808192b, 0x0808190808082b19, 0x0808190808190808, 0x080819080819082b, - 0x0808190808191919, 0x0808190808192b08, 0x08081908082b0819, 0x08081908082b1908, - 0x0808190819080808, 0x080819081908082b, 0x0808190819081919, 0x0808190819082b08, - 0x0808190819190819, 0x0808190819191908, 0x080819081919192b, 0x08081908192b0808, - 0x080819082b080819, 0x080819082b081908, 0x080819082b190808, 0x0808191908080808, - 0x080819190808082b, 0x0808191908081919, 0x0808191908082b08, 0x0808191908190819, - 0x0808191908191908, 0x08081919082b0808, 0x0808191919080819, 0x0808191919081908, - 0x0808191919190808, 0x08081919192b0819, 0x080819192b080808, 0x0808192b08080819, - 0x0808192b08081908, 0x0808192b08190808, 0x0808192b082b192b, 0x0808192b19080808, - 0x0808192b1908082b, 0x0808192b2b081908, 0x08082b0808080808, 0x08082b080808082b, - 0x08082b0808081919, 0x08082b0808082b08, 0x08082b0808082b2b, 0x08082b0808190819, - 0x08082b0808191908, 0x08082b08082b0808, 0x08082b08082b1919, 0x08082b0819080819, - 0x08082b0819081908, 0x08082b0819190808, 0x08082b0819192b08, 0x08082b082b080808, - 0x08082b082b2b0808, 0x08082b082b2b2b2b, 0x08082b1908080819, 0x08082b1908081908, - 0x08082b1908190808, 0x08082b1919080808, 0x08082b192b080819, 0x08082b192b082b19, - 0x08082b2b08080808, 0x08082b2b082b0808, 0x08082b2b082b2b08, 0x08082b2b2b19192b, - 0x08082b2b2b2b0808, 0x0819080808080819, 0x0819080808081908, 0x081908080808192b, - 0x0819080808082b19, 0x0819080808190808, 0x081908080819082b, 0x0819080808191919, - 0x0819080808192b08, 0x08190808082b0819, 0x08190808082b1908, 0x0819080819080808, - 0x081908081908082b, 0x0819080819081919, 0x0819080819082b08, 0x0819080819190819, - 0x0819080819191908, 0x08190808192b0808, 0x08190808192b2b2b, 0x081908082b080819, - 0x081908082b081908, 0x081908082b190808, 0x0819081908080808, 0x081908190808082b, - 0x0819081908081919, 0x0819081908082b08, 0x0819081908190819, 0x0819081908191908, - 0x08190819082b0808, 0x0819081919080819, 0x0819081919081908, 0x0819081919190808, - 0x081908192b080808, 0x081908192b191908, 0x081908192b19192b, 0x0819082b08080819, - 0x0819082b08081908, 0x0819082b0808192b, 0x0819082b08190808, 0x0819082b19080808, - 0x0819082b192b0808, 0x0819190808080808, 0x081919080808082b, 0x0819190808081919, - 0x0819190808082b08, 0x0819190808190819, 0x0819190808191908, 0x08191908082b0808, - 0x0819190819080819, 0x0819190819081908, 0x0819190819082b19, 0x0819190819190808, - 0x08191908192b1908, 0x081919082b080808, 0x0819191908080819, 0x0819191908081908, - 0x0819191908190808, 0x0819191919080808, 0x0819192b08080808, 0x0819192b08191908, - 0x0819192b19082b19, 0x08192b0808080819, 0x08192b0808081908, 0x08192b0808190808, - 0x08192b080819082b, 0x08192b0819080808, 0x08192b0819191908, 0x08192b082b08192b, - 0x08192b1908080808, 0x08192b1908081919, 0x08192b19192b192b, 0x08192b2b19190819, - 0x08192b2b2b2b2b19, 0x082b080808080808, 0x082b08080808082b, 0x082b080808081919, - 0x082b080808082b08, 0x082b080808082b2b, 0x082b080808190819, 0x082b080808191908, - 0x082b0808082b0808, 0x082b080819080819, 0x082b080819081908, 0x082b080819190808, - 0x082b08082b080808, 0x082b08082b2b0808, 0x082b081908080819, 0x082b081908081908, - 0x082b081908190808, 0x082b081919080808, 0x082b081919082b08, 0x082b0819192b1919, - 0x082b082b08080808, 0x082b082b082b082b, 0x082b082b2b080808, 0x082b082b2b2b2b08, - 0x082b190808080819, 0x082b190808081908, 0x082b190808190808, 0x082b1908082b2b19, - 0x082b190819080808, 0x082b191908080808, 0x082b191919080819, 0x082b19191919082b, - 0x082b19192b192b19, 0x082b192b08080819, 0x082b192b08192b2b, 0x082b192b2b2b192b, - 0x082b2b0808080808, 0x082b2b0808082b08, 0x082b2b0808082b2b, 0x082b2b08082b0808, - 0x082b2b0819191919, 0x082b2b082b082b08, 0x082b2b082b2b082b, 0x082b2b19192b2b08, - 0x082b2b192b190808, 0x082b2b2b08082b08, 0x082b2b2b082b0808, 0x082b2b2b2b08082b, - 0x082b2b2b2b082b08, 0x082b2b2b2b082b2b, 0x1908080808080819, 0x1908080808081908, - 0x190808080808192b, 0x1908080808082b19, 0x1908080808190808, 0x190808080819082b, - 0x1908080808191919, 0x1908080808192b08, 0x19080808082b0819, 0x19080808082b1908, - 0x1908080819080808, 0x190808081908082b, 0x1908080819081919, 0x1908080819082b08, - 0x1908080819082b2b, 0x1908080819190819, 0x1908080819191908, 0x19080808192b0808, - 0x19080808192b1919, 0x190808082b080819, 0x190808082b081908, 0x190808082b190808, - 0x1908081908080808, 0x190808190808082b, 0x1908081908081919, 0x1908081908082b08, - 0x1908081908190819, 0x1908081908191908, 0x19080819082b0808, 0x1908081919080819, - 0x1908081919081908, 0x1908081919190808, 0x190808192b080808, 0x190808192b081919, - 0x190808192b2b082b, 0x1908082b08080819, 0x1908082b08081908, 0x1908082b08190808, - 0x1908082b0819082b, 0x1908082b082b2b19, 0x1908082b19080808, 0x1908190808080808, - 0x190819080808082b, 0x1908190808081919, 0x1908190808082b08, 0x1908190808190819, - 0x1908190808191908, 0x1908190808192b19, 0x19081908082b0808, 0x1908190819080819, - 0x1908190819081908, 0x1908190819190808, 0x190819082b080808, 0x190819082b191908, - 0x1908191908080819, 0x1908191908081908, 0x1908191908190808, 0x19081919082b1908, - 0x1908191919080808, 0x190819192b192b2b, 0x1908192b08080808, 0x1908192b08082b2b, - 0x1908192b19081908, 0x1908192b19190808, 0x19082b0808080819, 0x19082b0808081908, - 0x19082b0808190808, 0x19082b0819080808, 0x19082b0819081919, 0x19082b0819191908, - 0x19082b08192b082b, 0x19082b1908080808, 0x19082b1908190819, 0x19082b1919081908, - 0x19082b1919190808, 0x19082b19192b2b19, 0x19082b2b08081908, 0x1919080808080808, - 0x191908080808082b, 0x1919080808081919, 0x1919080808082b08, 0x1919080808190819, - 0x1919080808191908, 0x19190808082b0808, 0x19190808082b2b08, 0x1919080819080819, - 0x1919080819081908, 0x1919080819190808, 0x191908082b080808, 0x1919081908080819, - 0x1919081908081908, 0x1919081908190808, 0x1919081908191919, 0x1919081919080808, - 0x191908191908082b, 0x1919082b08080808, 0x1919082b19081908, 0x1919082b2b2b2b2b, - 0x1919190808080819, 0x1919190808081908, 0x1919190808190808, 0x19191908082b0819, - 0x1919190819080808, 0x19191908192b0808, 0x191919082b080819, 0x191919082b2b0819, - 0x1919191908080808, 0x1919191908082b08, 0x191919192b080808, 0x191919192b082b08, - 0x1919192b082b0819, 0x1919192b192b2b08, 0x1919192b2b2b0819, 0x19192b0808080808, - 0x19192b0808191908, 0x19192b0819080819, 0x19192b0819190808, 0x19192b082b192b19, - 0x19192b1908192b2b, 0x19192b1919080808, 0x19192b191908082b, 0x19192b2b2b081919, - 0x192b080808080819, 0x192b080808081908, 0x192b080808190808, 0x192b080819080808, - 0x192b080819191908, 0x192b0808192b082b, 0x192b08082b08192b, 0x192b08082b2b2b19, - 0x192b081908080808, 0x192b082b082b1908, 0x192b082b19082b2b, 0x192b082b2b19082b, - 0x192b190808080808, 0x192b19080819192b, 0x192b191908190808, 0x192b191919080808, - 0x192b191919081919, 0x192b19192b2b1908, 0x192b2b0808080819, 0x192b2b08192b2b2b, - 0x192b2b19082b1919, 0x192b2b2b0808192b, 0x192b2b2b19191908, 0x192b2b2b192b082b, - 0x2b08080808080808, 0x2b0808080808082b, 0x2b08080808081919, 0x2b08080808082b08, - 0x2b08080808190819, 0x2b08080808191908, 0x2b080808082b0808, 0x2b080808082b2b2b, - 0x2b08080819080819, 0x2b08080819081908, 0x2b08080819190808, 0x2b0808082b080808, - 0x2b0808082b08082b, 0x2b0808082b2b2b08, 0x2b0808082b2b2b2b, 0x2b08081908080819, - 0x2b08081908081908, 0x2b0808190808192b, 0x2b08081908190808, 0x2b08081919080808, - 0x2b08081919190819, 0x2b08081919192b19, 0x2b08082b08080808, 0x2b08082b082b0808, - 0x2b08082b2b080808, 0x2b08082b2b08082b, 0x2b08082b2b2b0808, 0x2b08082b2b2b2b08, - 0x2b08190808080819, 0x2b08190808081908, 0x2b08190808190808, 0x2b0819080819082b, - 0x2b08190808191919, 0x2b08190819080808, 0x2b081908192b0808, 0x2b0819082b082b19, - 0x2b08191908080808, 0x2b08191919081908, 0x2b0819192b2b1919, 0x2b08192b08192b08, - 0x2b08192b192b2b2b, 0x2b082b0808080808, 0x2b082b0808082b08, 0x2b082b08082b1919, - 0x2b082b0819192b2b, 0x2b082b082b080808, 0x2b082b082b08082b, 0x2b082b082b2b2b08, - 0x2b082b190808192b, 0x2b082b2b082b082b, 0x2b082b2b2b080808, 0x2b082b2b2b082b08, - 0x2b082b2b2b19192b, 0x2b082b2b2b2b2b08, 0x2b19080808080819, 0x2b19080808081908, - 0x2b19080808190808, 0x2b19080819080808, 0x2b1908081919192b, 0x2b1908082b081908, - 0x2b19081908080808, 0x2b190819082b082b, 0x2b190819192b1908, 0x2b19082b1919192b, - 0x2b19082b2b082b19, 0x2b19190808080808, 0x2b19190808081919, 0x2b19190819081908, - 0x2b19190819190808, 0x2b19190819192b08, 0x2b191919082b2b19, 0x2b1919192b190808, - 0x2b1919192b19082b, 0x2b19192b19080819, 0x2b192b0819190819, 0x2b192b082b2b192b, - 0x2b192b1919082b19, 0x2b192b2b08191919, 0x2b192b2b192b0808, 0x2b2b080808080808, - 0x2b2b08080808082b, 0x2b2b080808082b08, 0x2b2b080808082b2b, 0x2b2b0808082b0808, - 0x2b2b0808082b2b2b, 0x2b2b08082b2b0808, 0x2b2b081919190819, 0x2b2b081919192b19, - 0x2b2b08192b2b192b, 0x2b2b082b08080808, 0x2b2b082b0808082b, 0x2b2b082b08082b08, - 0x2b2b082b082b2b2b, 0x2b2b082b2b080808, 0x2b2b082b2b2b0808, 0x2b2b190819080808, - 0x2b2b19082b191919, 0x2b2b192b192b1919, 0x2b2b192b2b192b08, 0x2b2b2b0808082b2b, - 0x2b2b2b08082b0808, 0x2b2b2b08082b082b, 0x2b2b2b08082b2b08, 0x2b2b2b082b2b0808, - 0x2b2b2b082b2b2b08, 0x2b2b2b1908081908, 0x2b2b2b192b081908, 0x2b2b2b192b08192b, - 0x2b2b2b2b082b2b08, 0x2b2b2b2b082b2b2b, 0x2b2b2b2b2b190819, 0x2b2b2b2b2b2b2b2b, -}; - -static const uint64_t iq2s_grid[1024] = { - 0x0808080808080808, 0x080808080808082b, 0x0808080808081919, 0x0808080808082b08, - 0x0808080808082b2b, 0x0808080808190819, 0x0808080808191908, 0x080808080819192b, - 0x0808080808192b19, 0x08080808082b0808, 0x08080808082b082b, 0x08080808082b1919, - 0x08080808082b2b08, 0x0808080819080819, 0x0808080819081908, 0x080808081908192b, - 0x0808080819082b19, 0x0808080819190808, 0x080808081919082b, 0x0808080819191919, - 0x0808080819192b08, 0x08080808192b0819, 0x08080808192b1908, 0x08080808192b192b, - 0x08080808192b2b19, 0x080808082b080808, 0x080808082b08082b, 0x080808082b081919, - 0x080808082b082b08, 0x080808082b190819, 0x080808082b191908, 0x080808082b2b0808, - 0x080808082b2b1919, 0x080808082b2b2b2b, 0x0808081908080819, 0x0808081908081908, - 0x080808190808192b, 0x0808081908082b19, 0x0808081908190808, 0x080808190819082b, - 0x0808081908191919, 0x0808081908192b08, 0x08080819082b0819, 0x08080819082b1908, - 0x0808081919080808, 0x080808191908082b, 0x0808081919081919, 0x0808081919082b08, - 0x0808081919190819, 0x0808081919191908, 0x080808191919192b, 0x0808081919192b19, - 0x08080819192b0808, 0x08080819192b1919, 0x08080819192b2b08, 0x080808192b080819, - 0x080808192b081908, 0x080808192b190808, 0x080808192b19082b, 0x080808192b191919, - 0x080808192b2b0819, 0x080808192b2b1908, 0x0808082b08080808, 0x0808082b0808082b, - 0x0808082b08081919, 0x0808082b08082b08, 0x0808082b08190819, 0x0808082b08191908, - 0x0808082b082b0808, 0x0808082b082b2b2b, 0x0808082b19080819, 0x0808082b19081908, - 0x0808082b1908192b, 0x0808082b19082b19, 0x0808082b19190808, 0x0808082b19191919, - 0x0808082b2b080808, 0x0808082b2b081919, 0x0808082b2b082b2b, 0x0808082b2b191908, - 0x0808082b2b2b082b, 0x0808190808080819, 0x0808190808081908, 0x080819080808192b, - 0x0808190808082b19, 0x0808190808190808, 0x080819080819082b, 0x0808190808191919, - 0x0808190808192b08, 0x08081908082b0819, 0x08081908082b1908, 0x08081908082b192b, - 0x08081908082b2b19, 0x0808190819080808, 0x080819081908082b, 0x0808190819081919, - 0x0808190819082b08, 0x0808190819082b2b, 0x0808190819190819, 0x0808190819191908, - 0x080819081919192b, 0x0808190819192b19, 0x08081908192b0808, 0x08081908192b082b, - 0x08081908192b1919, 0x080819082b080819, 0x080819082b081908, 0x080819082b08192b, - 0x080819082b082b19, 0x080819082b190808, 0x080819082b191919, 0x080819082b192b08, - 0x080819082b2b0819, 0x080819082b2b1908, 0x0808191908080808, 0x080819190808082b, - 0x0808191908081919, 0x0808191908082b08, 0x0808191908082b2b, 0x0808191908190819, - 0x0808191908191908, 0x080819190819192b, 0x0808191908192b19, 0x08081919082b0808, - 0x08081919082b1919, 0x08081919082b2b08, 0x0808191919080819, 0x0808191919081908, - 0x080819191908192b, 0x0808191919082b19, 0x0808191919190808, 0x080819191919082b, - 0x0808191919191919, 0x0808191919192b08, 0x08081919192b0819, 0x08081919192b1908, - 0x080819192b080808, 0x080819192b08082b, 0x080819192b081919, 0x080819192b082b08, - 0x080819192b190819, 0x080819192b191908, 0x080819192b2b0808, 0x0808192b08080819, - 0x0808192b08081908, 0x0808192b0808192b, 0x0808192b08082b19, 0x0808192b08190808, - 0x0808192b08191919, 0x0808192b19080808, 0x0808192b19081919, 0x0808192b19082b08, - 0x0808192b19190819, 0x0808192b19191908, 0x0808192b192b0808, 0x0808192b2b080819, - 0x0808192b2b081908, 0x0808192b2b190808, 0x08082b0808080808, 0x08082b080808082b, - 0x08082b0808081919, 0x08082b0808082b08, 0x08082b0808190819, 0x08082b0808191908, - 0x08082b080819192b, 0x08082b0808192b19, 0x08082b08082b0808, 0x08082b08082b1919, - 0x08082b08082b2b2b, 0x08082b0819080819, 0x08082b0819081908, 0x08082b081908192b, - 0x08082b0819082b19, 0x08082b0819190808, 0x08082b081919082b, 0x08082b0819191919, - 0x08082b0819192b08, 0x08082b08192b0819, 0x08082b08192b1908, 0x08082b082b080808, - 0x08082b082b081919, 0x08082b082b191908, 0x08082b082b2b2b2b, 0x08082b1908080819, - 0x08082b1908081908, 0x08082b1908190808, 0x08082b190819082b, 0x08082b1908191919, - 0x08082b1908192b08, 0x08082b19082b0819, 0x08082b1919080808, 0x08082b1919081919, - 0x08082b1919082b08, 0x08082b1919190819, 0x08082b1919191908, 0x08082b19192b0808, - 0x08082b192b080819, 0x08082b192b190808, 0x08082b2b08080808, 0x08082b2b08190819, - 0x08082b2b08191908, 0x08082b2b082b082b, 0x08082b2b082b2b08, 0x08082b2b082b2b2b, - 0x08082b2b19190808, 0x08082b2b2b192b19, 0x0819080808080819, 0x0819080808081908, - 0x081908080808192b, 0x0819080808082b19, 0x0819080808190808, 0x081908080819082b, - 0x0819080808191919, 0x0819080808192b08, 0x08190808082b0819, 0x08190808082b1908, - 0x08190808082b192b, 0x0819080819080808, 0x081908081908082b, 0x0819080819081919, - 0x0819080819082b08, 0x0819080819190819, 0x0819080819191908, 0x081908081919192b, - 0x0819080819192b19, 0x08190808192b0808, 0x08190808192b082b, 0x08190808192b1919, - 0x08190808192b2b08, 0x081908082b080819, 0x081908082b081908, 0x081908082b08192b, - 0x081908082b190808, 0x081908082b191919, 0x081908082b192b08, 0x081908082b2b0819, - 0x081908082b2b1908, 0x0819081908080808, 0x081908190808082b, 0x0819081908081919, - 0x0819081908082b08, 0x0819081908082b2b, 0x0819081908190819, 0x0819081908191908, - 0x081908190819192b, 0x0819081908192b19, 0x08190819082b0808, 0x08190819082b082b, - 0x08190819082b1919, 0x08190819082b2b08, 0x0819081919080819, 0x0819081919081908, - 0x081908191908192b, 0x0819081919082b19, 0x0819081919190808, 0x081908191919082b, - 0x0819081919191919, 0x0819081919192b08, 0x08190819192b0819, 0x08190819192b1908, - 0x081908192b080808, 0x081908192b08082b, 0x081908192b081919, 0x081908192b082b08, - 0x081908192b190819, 0x081908192b191908, 0x0819082b08080819, 0x0819082b08081908, - 0x0819082b08082b19, 0x0819082b08190808, 0x0819082b08191919, 0x0819082b082b0819, - 0x0819082b082b1908, 0x0819082b19080808, 0x0819082b19081919, 0x0819082b19190819, - 0x0819082b19191908, 0x0819082b2b080819, 0x0819082b2b081908, 0x0819082b2b190808, - 0x0819190808080808, 0x081919080808082b, 0x0819190808081919, 0x0819190808082b08, - 0x0819190808190819, 0x0819190808191908, 0x081919080819192b, 0x0819190808192b19, - 0x08191908082b0808, 0x08191908082b1919, 0x08191908082b2b08, 0x0819190819080819, - 0x0819190819081908, 0x081919081908192b, 0x0819190819082b19, 0x0819190819190808, - 0x081919081919082b, 0x0819190819191919, 0x0819190819192b08, 0x08191908192b0819, - 0x08191908192b1908, 0x081919082b080808, 0x081919082b08082b, 0x081919082b081919, - 0x081919082b082b08, 0x081919082b190819, 0x081919082b191908, 0x081919082b2b0808, - 0x0819191908080819, 0x0819191908081908, 0x081919190808192b, 0x0819191908082b19, - 0x0819191908190808, 0x081919190819082b, 0x0819191908191919, 0x0819191908192b08, - 0x08191919082b0819, 0x08191919082b1908, 0x0819191919080808, 0x081919191908082b, - 0x0819191919081919, 0x0819191919082b08, 0x0819191919190819, 0x0819191919191908, - 0x08191919192b0808, 0x081919192b080819, 0x081919192b081908, 0x081919192b190808, - 0x0819192b08080808, 0x0819192b08081919, 0x0819192b08082b08, 0x0819192b08190819, - 0x0819192b08191908, 0x0819192b082b0808, 0x0819192b19080819, 0x0819192b19081908, - 0x0819192b19190808, 0x0819192b2b080808, 0x0819192b2b2b2b2b, 0x08192b0808080819, - 0x08192b0808081908, 0x08192b080808192b, 0x08192b0808082b19, 0x08192b0808190808, - 0x08192b0808191919, 0x08192b0808192b08, 0x08192b08082b0819, 0x08192b0819080808, - 0x08192b081908082b, 0x08192b0819081919, 0x08192b0819082b08, 0x08192b0819190819, - 0x08192b0819191908, 0x08192b08192b0808, 0x08192b082b080819, 0x08192b082b081908, - 0x08192b1908080808, 0x08192b190808082b, 0x08192b1908081919, 0x08192b1908082b08, - 0x08192b1908190819, 0x08192b1908191908, 0x08192b19082b0808, 0x08192b1919080819, - 0x08192b1919081908, 0x08192b1919190808, 0x08192b19192b2b19, 0x08192b192b2b082b, - 0x08192b2b08081908, 0x08192b2b08190808, 0x08192b2b19080808, 0x08192b2b1919192b, - 0x082b080808080808, 0x082b08080808082b, 0x082b080808081919, 0x082b080808082b08, - 0x082b080808190819, 0x082b080808191908, 0x082b08080819192b, 0x082b080808192b19, - 0x082b0808082b0808, 0x082b0808082b1919, 0x082b0808082b2b2b, 0x082b080819080819, - 0x082b080819081908, 0x082b080819190808, 0x082b08081919082b, 0x082b080819191919, - 0x082b0808192b1908, 0x082b08082b080808, 0x082b08082b082b2b, 0x082b08082b191908, - 0x082b08082b2b2b2b, 0x082b081908080819, 0x082b081908081908, 0x082b081908190808, - 0x082b08190819082b, 0x082b081908191919, 0x082b0819082b0819, 0x082b081919080808, - 0x082b08191908082b, 0x082b081919081919, 0x082b081919190819, 0x082b081919191908, - 0x082b0819192b0808, 0x082b08192b080819, 0x082b08192b081908, 0x082b08192b190808, - 0x082b082b08080808, 0x082b082b08082b2b, 0x082b082b082b082b, 0x082b082b082b2b08, - 0x082b082b082b2b2b, 0x082b082b19081908, 0x082b082b19190808, 0x082b082b2b082b08, - 0x082b082b2b082b2b, 0x082b082b2b2b2b08, 0x082b190808080819, 0x082b190808081908, - 0x082b19080808192b, 0x082b190808082b19, 0x082b190808190808, 0x082b190808191919, - 0x082b190808192b08, 0x082b1908082b0819, 0x082b1908082b1908, 0x082b190819080808, - 0x082b19081908082b, 0x082b190819081919, 0x082b190819082b08, 0x082b190819190819, - 0x082b190819191908, 0x082b1908192b0808, 0x082b19082b080819, 0x082b19082b081908, - 0x082b19082b190808, 0x082b191908080808, 0x082b191908081919, 0x082b191908082b08, - 0x082b191908190819, 0x082b191908191908, 0x082b1919082b0808, 0x082b191919080819, - 0x082b191919081908, 0x082b191919190808, 0x082b1919192b192b, 0x082b19192b080808, - 0x082b192b08080819, 0x082b192b08081908, 0x082b192b08190808, 0x082b192b19080808, - 0x082b192b19192b19, 0x082b2b0808080808, 0x082b2b0808081919, 0x082b2b0808190819, - 0x082b2b0808191908, 0x082b2b0819080819, 0x082b2b0819081908, 0x082b2b0819190808, - 0x082b2b082b082b2b, 0x082b2b082b2b2b2b, 0x082b2b1908080819, 0x082b2b1908081908, - 0x082b2b1908190808, 0x082b2b192b191919, 0x082b2b2b08082b2b, 0x082b2b2b082b082b, - 0x082b2b2b192b1908, 0x082b2b2b2b082b08, 0x082b2b2b2b082b2b, 0x1908080808080819, - 0x1908080808081908, 0x190808080808192b, 0x1908080808082b19, 0x1908080808190808, - 0x190808080819082b, 0x1908080808191919, 0x1908080808192b08, 0x1908080808192b2b, - 0x19080808082b0819, 0x19080808082b1908, 0x19080808082b192b, 0x1908080819080808, - 0x190808081908082b, 0x1908080819081919, 0x1908080819082b08, 0x1908080819082b2b, - 0x1908080819190819, 0x1908080819191908, 0x190808081919192b, 0x1908080819192b19, - 0x19080808192b0808, 0x19080808192b082b, 0x19080808192b1919, 0x190808082b080819, - 0x190808082b081908, 0x190808082b190808, 0x190808082b191919, 0x190808082b192b08, - 0x190808082b2b0819, 0x190808082b2b1908, 0x1908081908080808, 0x190808190808082b, - 0x1908081908081919, 0x1908081908082b08, 0x1908081908190819, 0x1908081908191908, - 0x190808190819192b, 0x1908081908192b19, 0x19080819082b0808, 0x19080819082b082b, - 0x19080819082b1919, 0x1908081919080819, 0x1908081919081908, 0x190808191908192b, - 0x1908081919082b19, 0x1908081919190808, 0x190808191919082b, 0x1908081919191919, - 0x1908081919192b08, 0x19080819192b0819, 0x19080819192b1908, 0x190808192b080808, - 0x190808192b08082b, 0x190808192b081919, 0x190808192b082b08, 0x190808192b190819, - 0x190808192b191908, 0x190808192b2b0808, 0x1908082b08080819, 0x1908082b08081908, - 0x1908082b08190808, 0x1908082b0819082b, 0x1908082b08191919, 0x1908082b08192b08, - 0x1908082b082b1908, 0x1908082b19080808, 0x1908082b19081919, 0x1908082b19082b08, - 0x1908082b19190819, 0x1908082b19191908, 0x1908082b192b0808, 0x1908082b2b080819, - 0x1908082b2b081908, 0x1908190808080808, 0x190819080808082b, 0x1908190808081919, - 0x1908190808082b08, 0x1908190808082b2b, 0x1908190808190819, 0x1908190808191908, - 0x190819080819192b, 0x1908190808192b19, 0x19081908082b0808, 0x19081908082b082b, - 0x19081908082b1919, 0x19081908082b2b08, 0x1908190819080819, 0x1908190819081908, - 0x190819081908192b, 0x1908190819082b19, 0x1908190819190808, 0x190819081919082b, - 0x1908190819191919, 0x1908190819192b08, 0x19081908192b0819, 0x19081908192b1908, - 0x190819082b080808, 0x190819082b08082b, 0x190819082b081919, 0x190819082b082b08, - 0x190819082b190819, 0x190819082b191908, 0x190819082b2b0808, 0x1908191908080819, - 0x1908191908081908, 0x190819190808192b, 0x1908191908082b19, 0x1908191908190808, - 0x190819190819082b, 0x1908191908191919, 0x1908191908192b08, 0x19081919082b0819, - 0x19081919082b1908, 0x1908191919080808, 0x190819191908082b, 0x1908191919081919, - 0x1908191919082b08, 0x1908191919190819, 0x1908191919191908, 0x19081919192b0808, - 0x19081919192b2b2b, 0x190819192b080819, 0x190819192b081908, 0x190819192b190808, - 0x1908192b08080808, 0x1908192b0808082b, 0x1908192b08081919, 0x1908192b08082b08, - 0x1908192b08190819, 0x1908192b08191908, 0x1908192b082b0808, 0x1908192b19080819, - 0x1908192b19081908, 0x1908192b19190808, 0x1908192b2b080808, 0x1908192b2b2b1919, - 0x19082b0808080819, 0x19082b0808081908, 0x19082b0808082b19, 0x19082b0808190808, - 0x19082b080819082b, 0x19082b0808191919, 0x19082b0808192b08, 0x19082b08082b0819, - 0x19082b08082b1908, 0x19082b0819080808, 0x19082b081908082b, 0x19082b0819081919, - 0x19082b0819082b08, 0x19082b0819190819, 0x19082b0819191908, 0x19082b08192b0808, - 0x19082b082b081908, 0x19082b082b190808, 0x19082b1908080808, 0x19082b190808082b, - 0x19082b1908081919, 0x19082b1908082b08, 0x19082b1908190819, 0x19082b1908191908, - 0x19082b19082b0808, 0x19082b1919080819, 0x19082b1919081908, 0x19082b1919190808, - 0x19082b192b080808, 0x19082b192b19192b, 0x19082b2b08080819, 0x19082b2b08081908, - 0x19082b2b08190808, 0x19082b2b19080808, 0x1919080808080808, 0x191908080808082b, - 0x1919080808081919, 0x1919080808082b08, 0x1919080808190819, 0x1919080808191908, - 0x191908080819192b, 0x1919080808192b19, 0x19190808082b0808, 0x19190808082b082b, - 0x19190808082b1919, 0x19190808082b2b08, 0x1919080819080819, 0x1919080819081908, - 0x191908081908192b, 0x1919080819082b19, 0x1919080819190808, 0x191908081919082b, - 0x1919080819191919, 0x1919080819192b08, 0x19190808192b0819, 0x19190808192b1908, - 0x191908082b080808, 0x191908082b08082b, 0x191908082b081919, 0x191908082b082b08, - 0x191908082b190819, 0x191908082b191908, 0x1919081908080819, 0x1919081908081908, - 0x191908190808192b, 0x1919081908082b19, 0x1919081908190808, 0x191908190819082b, - 0x1919081908191919, 0x1919081908192b08, 0x19190819082b0819, 0x19190819082b1908, - 0x1919081919080808, 0x191908191908082b, 0x1919081919081919, 0x1919081919082b08, - 0x1919081919190819, 0x1919081919191908, 0x19190819192b0808, 0x191908192b080819, - 0x191908192b081908, 0x191908192b190808, 0x1919082b08080808, 0x1919082b08081919, - 0x1919082b08082b08, 0x1919082b08190819, 0x1919082b08191908, 0x1919082b082b0808, - 0x1919082b19080819, 0x1919082b19081908, 0x1919082b19190808, 0x1919082b192b2b19, - 0x1919082b2b080808, 0x1919190808080819, 0x1919190808081908, 0x191919080808192b, - 0x1919190808082b19, 0x1919190808190808, 0x191919080819082b, 0x1919190808191919, - 0x1919190808192b08, 0x19191908082b0819, 0x19191908082b1908, 0x1919190819080808, - 0x191919081908082b, 0x1919190819081919, 0x1919190819082b08, 0x1919190819190819, - 0x1919190819191908, 0x19191908192b0808, 0x191919082b080819, 0x191919082b081908, - 0x191919082b190808, 0x1919191908080808, 0x191919190808082b, 0x1919191908081919, - 0x1919191908082b08, 0x1919191908190819, 0x1919191908191908, 0x19191919082b0808, - 0x1919191919080819, 0x1919191919081908, 0x1919191919190808, 0x191919192b080808, - 0x1919192b08080819, 0x1919192b08081908, 0x1919192b08190808, 0x1919192b082b192b, - 0x1919192b19080808, 0x19192b0808080808, 0x19192b080808082b, 0x19192b0808081919, - 0x19192b0808082b08, 0x19192b0808190819, 0x19192b0808191908, 0x19192b08082b0808, - 0x19192b0819080819, 0x19192b0819081908, 0x19192b0819190808, 0x19192b0819192b2b, - 0x19192b082b080808, 0x19192b1908080819, 0x19192b1908081908, 0x19192b1908190808, - 0x19192b1919080808, 0x19192b2b08080808, 0x19192b2b08192b19, 0x19192b2b2b081919, - 0x19192b2b2b2b2b08, 0x192b080808080819, 0x192b080808081908, 0x192b08080808192b, - 0x192b080808190808, 0x192b08080819082b, 0x192b080808191919, 0x192b080808192b08, - 0x192b0808082b0819, 0x192b0808082b1908, 0x192b080819080808, 0x192b080819081919, - 0x192b080819082b08, 0x192b080819190819, 0x192b080819191908, 0x192b0808192b0808, - 0x192b08082b081908, 0x192b08082b190808, 0x192b081908080808, 0x192b08190808082b, - 0x192b081908081919, 0x192b081908082b08, 0x192b081908190819, 0x192b081908191908, - 0x192b0819082b0808, 0x192b081919080819, 0x192b081919081908, 0x192b081919190808, - 0x192b08192b080808, 0x192b08192b192b19, 0x192b082b08081908, 0x192b082b08190808, - 0x192b082b19080808, 0x192b082b1919192b, 0x192b082b2b2b0819, 0x192b190808080808, - 0x192b190808081919, 0x192b190808082b08, 0x192b190808190819, 0x192b190808191908, - 0x192b1908082b0808, 0x192b190819080819, 0x192b190819081908, 0x192b190819190808, - 0x192b19082b080808, 0x192b191908080819, 0x192b191908081908, 0x192b191908190808, - 0x192b191919080808, 0x192b191919082b2b, 0x192b1919192b2b08, 0x192b19192b19082b, - 0x192b192b08080808, 0x192b192b2b191908, 0x192b2b0808080819, 0x192b2b0808081908, - 0x192b2b0808190808, 0x192b2b08192b1919, 0x192b2b082b192b08, 0x192b2b1908080808, - 0x192b2b19082b2b2b, 0x192b2b2b1908082b, 0x192b2b2b2b2b0819, 0x2b08080808080808, - 0x2b0808080808082b, 0x2b08080808081919, 0x2b08080808082b08, 0x2b08080808190819, - 0x2b08080808191908, 0x2b08080808192b19, 0x2b080808082b0808, 0x2b080808082b1919, - 0x2b08080819080819, 0x2b08080819081908, 0x2b08080819190808, 0x2b0808081919082b, - 0x2b08080819191919, 0x2b08080819192b08, 0x2b080808192b0819, 0x2b0808082b080808, - 0x2b0808082b081919, 0x2b0808082b190819, 0x2b0808082b191908, 0x2b08081908080819, - 0x2b08081908081908, 0x2b08081908082b19, 0x2b08081908190808, 0x2b0808190819082b, - 0x2b08081908191919, 0x2b08081908192b08, 0x2b080819082b0819, 0x2b080819082b1908, - 0x2b08081919080808, 0x2b0808191908082b, 0x2b08081919081919, 0x2b08081919082b08, - 0x2b08081919190819, 0x2b08081919191908, 0x2b0808192b080819, 0x2b0808192b081908, - 0x2b0808192b190808, 0x2b0808192b2b2b19, 0x2b08082b08080808, 0x2b08082b08081919, - 0x2b08082b08082b2b, 0x2b08082b08190819, 0x2b08082b08191908, 0x2b08082b19080819, - 0x2b08082b19081908, 0x2b08082b19190808, 0x2b08190808080819, 0x2b08190808081908, - 0x2b0819080808192b, 0x2b08190808082b19, 0x2b08190808190808, 0x2b0819080819082b, - 0x2b08190808191919, 0x2b08190808192b08, 0x2b081908082b0819, 0x2b08190819080808, - 0x2b0819081908082b, 0x2b08190819081919, 0x2b08190819082b08, 0x2b08190819190819, - 0x2b08190819191908, 0x2b081908192b0808, 0x2b0819082b080819, 0x2b0819082b081908, - 0x2b0819082b190808, 0x2b08191908080808, 0x2b0819190808082b, 0x2b08191908081919, - 0x2b08191908082b08, 0x2b08191908190819, 0x2b08191908191908, 0x2b081919082b0808, - 0x2b08191919080819, 0x2b08191919081908, 0x2b08191919190808, 0x2b0819192b080808, - 0x2b0819192b082b2b, 0x2b08192b08080819, 0x2b08192b08081908, 0x2b08192b08190808, - 0x2b08192b082b2b19, 0x2b08192b19080808, 0x2b082b0808080808, 0x2b082b0808081919, - 0x2b082b0808190819, 0x2b082b0808191908, 0x2b082b0819080819, 0x2b082b0819081908, - 0x2b082b0819190808, 0x2b082b082b2b082b, 0x2b082b1908080819, 0x2b082b1908081908, - 0x2b082b1919080808, 0x2b082b19192b1919, 0x2b082b2b082b082b, 0x2b082b2b19192b08, - 0x2b082b2b19192b2b, 0x2b082b2b2b08082b, 0x2b082b2b2b2b082b, 0x2b19080808080819, - 0x2b19080808081908, 0x2b19080808082b19, 0x2b19080808190808, 0x2b1908080819082b, - 0x2b19080808191919, 0x2b19080808192b08, 0x2b190808082b1908, 0x2b19080819080808, - 0x2b1908081908082b, 0x2b19080819081919, 0x2b19080819082b08, 0x2b19080819190819, - 0x2b19080819191908, 0x2b190808192b0808, 0x2b1908082b080819, 0x2b1908082b081908, - 0x2b1908082b190808, 0x2b19081908080808, 0x2b19081908081919, 0x2b19081908190819, - 0x2b19081908191908, 0x2b19081919080819, 0x2b19081919081908, 0x2b19081919190808, - 0x2b19081919192b2b, 0x2b19082b08080819, 0x2b19082b08081908, 0x2b19082b08190808, - 0x2b19082b19080808, 0x2b19082b2b2b192b, 0x2b19190808080808, 0x2b1919080808082b, - 0x2b19190808081919, 0x2b19190808082b08, 0x2b19190808190819, 0x2b19190808191908, - 0x2b191908082b0808, 0x2b19190819080819, 0x2b19190819081908, 0x2b19190819190808, - 0x2b1919082b080808, 0x2b1919082b19192b, 0x2b19191908080819, 0x2b19191908081908, - 0x2b19191908190808, 0x2b19191919080808, 0x2b1919192b192b08, 0x2b1919192b2b0819, - 0x2b19192b08080808, 0x2b19192b1908192b, 0x2b19192b192b1908, 0x2b192b0808080819, - 0x2b192b0808081908, 0x2b192b0808190808, 0x2b192b08082b192b, 0x2b192b0819080808, - 0x2b192b082b2b2b19, 0x2b192b1908080808, 0x2b192b1919082b19, 0x2b192b191919082b, - 0x2b192b2b2b190808, 0x2b2b080808080808, 0x2b2b080808081919, 0x2b2b080808082b2b, - 0x2b2b080808191908, 0x2b2b0808082b082b, 0x2b2b0808082b2b2b, 0x2b2b080819080819, - 0x2b2b080819081908, 0x2b2b080819190808, 0x2b2b08082b2b082b, 0x2b2b08082b2b2b2b, - 0x2b2b081919080808, 0x2b2b0819192b1919, 0x2b2b082b0808082b, 0x2b2b082b08082b2b, - 0x2b2b082b082b082b, 0x2b2b082b082b2b08, 0x2b2b082b082b2b2b, 0x2b2b082b2b08082b, - 0x2b2b082b2b082b08, 0x2b2b082b2b082b2b, 0x2b2b082b2b2b2b08, 0x2b2b190808080819, - 0x2b2b190808081908, 0x2b2b190808190808, 0x2b2b190819080808, 0x2b2b19082b082b19, - 0x2b2b19082b2b1908, 0x2b2b191908080808, 0x2b2b191908192b19, 0x2b2b192b19190819, - 0x2b2b2b0808082b2b, 0x2b2b2b08082b2b08, 0x2b2b2b082b2b082b, 0x2b2b2b1919191908, - 0x2b2b2b192b08192b, 0x2b2b2b2b08082b08, 0x2b2b2b2b08082b2b, 0x2b2b2b2b082b0808, - 0x2b2b2b2b082b082b, 0x2b2b2b2b082b2b08, 0x2b2b2b2b2b082b08, 0x2b2b2b2b2b2b2b2b, -}; - -static const uint32_t iq3xxs_grid[256] = { - 0x04040404, 0x04040414, 0x04040424, 0x04040c0c, 0x04040c1c, 0x04040c3e, 0x04041404, 0x04041414, - 0x04041c0c, 0x04042414, 0x04043e1c, 0x04043e2c, 0x040c040c, 0x040c041c, 0x040c0c04, 0x040c0c14, - 0x040c140c, 0x040c142c, 0x040c1c04, 0x040c1c14, 0x040c240c, 0x040c2c24, 0x040c3e04, 0x04140404, - 0x04140414, 0x04140424, 0x04140c0c, 0x04141404, 0x04141414, 0x04141c0c, 0x04141c1c, 0x04141c3e, - 0x04142c0c, 0x04142c3e, 0x04143e2c, 0x041c040c, 0x041c043e, 0x041c0c04, 0x041c0c14, 0x041c142c, - 0x041c3e04, 0x04240c1c, 0x04241c3e, 0x04242424, 0x04242c3e, 0x04243e1c, 0x04243e2c, 0x042c040c, - 0x042c043e, 0x042c1c14, 0x042c2c14, 0x04341c2c, 0x04343424, 0x043e0c04, 0x043e0c24, 0x043e0c34, - 0x043e241c, 0x043e340c, 0x0c04040c, 0x0c04041c, 0x0c040c04, 0x0c040c14, 0x0c04140c, 0x0c04141c, - 0x0c041c04, 0x0c041c14, 0x0c041c24, 0x0c04243e, 0x0c042c04, 0x0c0c0404, 0x0c0c0414, 0x0c0c0c0c, - 0x0c0c1404, 0x0c0c1414, 0x0c14040c, 0x0c14041c, 0x0c140c04, 0x0c140c14, 0x0c14140c, 0x0c141c04, - 0x0c143e14, 0x0c1c0404, 0x0c1c0414, 0x0c1c1404, 0x0c1c1c0c, 0x0c1c2434, 0x0c1c3434, 0x0c24040c, - 0x0c24042c, 0x0c242c04, 0x0c2c1404, 0x0c2c1424, 0x0c2c2434, 0x0c2c3e0c, 0x0c34042c, 0x0c3e1414, - 0x0c3e2404, 0x14040404, 0x14040414, 0x14040c0c, 0x14040c1c, 0x14041404, 0x14041414, 0x14041434, - 0x14041c0c, 0x14042414, 0x140c040c, 0x140c041c, 0x140c042c, 0x140c0c04, 0x140c0c14, 0x140c140c, - 0x140c1c04, 0x140c341c, 0x140c343e, 0x140c3e04, 0x14140404, 0x14140414, 0x14140c0c, 0x14140c3e, - 0x14141404, 0x14141414, 0x14141c3e, 0x14142404, 0x14142c2c, 0x141c040c, 0x141c0c04, 0x141c0c24, - 0x141c3e04, 0x141c3e24, 0x14241c2c, 0x14242c1c, 0x142c041c, 0x142c143e, 0x142c240c, 0x142c3e24, - 0x143e040c, 0x143e041c, 0x143e0c34, 0x143e242c, 0x1c04040c, 0x1c040c04, 0x1c040c14, 0x1c04140c, - 0x1c04141c, 0x1c042c04, 0x1c04342c, 0x1c043e14, 0x1c0c0404, 0x1c0c0414, 0x1c0c1404, 0x1c0c1c0c, - 0x1c0c2424, 0x1c0c2434, 0x1c14040c, 0x1c14041c, 0x1c140c04, 0x1c14142c, 0x1c142c14, 0x1c143e14, - 0x1c1c0c0c, 0x1c1c1c1c, 0x1c241c04, 0x1c24243e, 0x1c243e14, 0x1c2c0404, 0x1c2c0434, 0x1c2c1414, - 0x1c2c2c2c, 0x1c340c24, 0x1c341c34, 0x1c34341c, 0x1c3e1c1c, 0x1c3e3404, 0x24040424, 0x24040c3e, - 0x24041c2c, 0x24041c3e, 0x24042c1c, 0x24042c3e, 0x240c3e24, 0x24141404, 0x24141c3e, 0x24142404, - 0x24143404, 0x24143434, 0x241c043e, 0x241c242c, 0x24240424, 0x24242c0c, 0x24243424, 0x242c142c, - 0x242c241c, 0x242c3e04, 0x243e042c, 0x243e0c04, 0x243e0c14, 0x243e1c04, 0x2c040c14, 0x2c04240c, - 0x2c043e04, 0x2c0c0404, 0x2c0c0434, 0x2c0c1434, 0x2c0c2c2c, 0x2c140c24, 0x2c141c14, 0x2c143e14, - 0x2c1c0414, 0x2c1c2c1c, 0x2c240c04, 0x2c24141c, 0x2c24143e, 0x2c243e14, 0x2c2c0414, 0x2c2c1c0c, - 0x2c342c04, 0x2c3e1424, 0x2c3e2414, 0x34041424, 0x34042424, 0x34042434, 0x34043424, 0x340c140c, - 0x340c340c, 0x34140c3e, 0x34143424, 0x341c1c04, 0x341c1c34, 0x34242424, 0x342c042c, 0x342c2c14, - 0x34341c1c, 0x343e041c, 0x343e140c, 0x3e04041c, 0x3e04042c, 0x3e04043e, 0x3e040c04, 0x3e041c14, - 0x3e042c14, 0x3e0c1434, 0x3e0c2404, 0x3e140c14, 0x3e14242c, 0x3e142c14, 0x3e1c0404, 0x3e1c0c2c, - 0x3e1c1c1c, 0x3e1c3404, 0x3e24140c, 0x3e24240c, 0x3e2c0404, 0x3e2c0414, 0x3e2c1424, 0x3e341c04, -}; - -static const uint32_t iq3s_grid[512] = { - 0x01010101, 0x01010103, 0x01010105, 0x0101010b, 0x0101010f, 0x01010301, 0x01010303, 0x01010305, - 0x01010309, 0x0101030d, 0x01010501, 0x01010503, 0x0101050b, 0x01010707, 0x01010901, 0x01010905, - 0x0101090b, 0x0101090f, 0x01010b03, 0x01010b07, 0x01010d01, 0x01010d05, 0x01010f03, 0x01010f09, - 0x01010f0f, 0x01030101, 0x01030103, 0x01030105, 0x01030109, 0x01030301, 0x01030303, 0x0103030b, - 0x01030501, 0x01030507, 0x0103050f, 0x01030703, 0x0103070b, 0x01030909, 0x01030d03, 0x01030d0b, - 0x01030f05, 0x01050101, 0x01050103, 0x0105010b, 0x0105010f, 0x01050301, 0x01050307, 0x0105030d, - 0x01050503, 0x0105050b, 0x01050701, 0x01050709, 0x01050905, 0x0105090b, 0x0105090f, 0x01050b03, - 0x01050b07, 0x01050f01, 0x01050f07, 0x01070107, 0x01070303, 0x0107030b, 0x01070501, 0x01070505, - 0x01070703, 0x01070707, 0x0107070d, 0x01070909, 0x01070b01, 0x01070b05, 0x01070d0f, 0x01070f03, - 0x01070f0b, 0x01090101, 0x01090307, 0x0109030f, 0x01090503, 0x01090509, 0x01090705, 0x01090901, - 0x01090907, 0x01090b03, 0x01090f01, 0x010b0105, 0x010b0109, 0x010b0501, 0x010b0505, 0x010b050d, - 0x010b0707, 0x010b0903, 0x010b090b, 0x010b090f, 0x010b0d0d, 0x010b0f07, 0x010d010d, 0x010d0303, - 0x010d0307, 0x010d0703, 0x010d0b05, 0x010d0f03, 0x010f0101, 0x010f0105, 0x010f0109, 0x010f0501, - 0x010f0505, 0x010f050d, 0x010f0707, 0x010f0b01, 0x010f0b09, 0x03010101, 0x03010103, 0x03010105, - 0x03010109, 0x03010301, 0x03010303, 0x03010307, 0x0301030b, 0x0301030f, 0x03010501, 0x03010505, - 0x03010703, 0x03010709, 0x0301070d, 0x03010b09, 0x03010b0d, 0x03010d03, 0x03010f05, 0x03030101, - 0x03030103, 0x03030107, 0x0303010d, 0x03030301, 0x03030309, 0x03030503, 0x03030701, 0x03030707, - 0x03030903, 0x03030b01, 0x03030b05, 0x03030f01, 0x03030f0d, 0x03050101, 0x03050305, 0x0305030b, - 0x0305030f, 0x03050501, 0x03050509, 0x03050705, 0x03050901, 0x03050907, 0x03050b0b, 0x03050d01, - 0x03050f05, 0x03070103, 0x03070109, 0x0307010f, 0x03070301, 0x03070307, 0x03070503, 0x0307050f, - 0x03070701, 0x03070709, 0x03070903, 0x03070d05, 0x03070f01, 0x03090107, 0x0309010b, 0x03090305, - 0x03090309, 0x03090703, 0x03090707, 0x03090905, 0x0309090d, 0x03090b01, 0x03090b09, 0x030b0103, - 0x030b0301, 0x030b0307, 0x030b0503, 0x030b0701, 0x030b0705, 0x030b0b03, 0x030d0501, 0x030d0509, - 0x030d050f, 0x030d0909, 0x030d090d, 0x030f0103, 0x030f0107, 0x030f0301, 0x030f0305, 0x030f0503, - 0x030f070b, 0x030f0903, 0x030f0d05, 0x030f0f01, 0x05010101, 0x05010103, 0x05010107, 0x0501010b, - 0x0501010f, 0x05010301, 0x05010305, 0x05010309, 0x0501030d, 0x05010503, 0x05010507, 0x0501050f, - 0x05010701, 0x05010705, 0x05010903, 0x05010907, 0x0501090b, 0x05010b01, 0x05010b05, 0x05010d0f, - 0x05010f01, 0x05010f07, 0x05010f0b, 0x05030101, 0x05030105, 0x05030301, 0x05030307, 0x0503030f, - 0x05030505, 0x0503050b, 0x05030703, 0x05030709, 0x05030905, 0x05030b03, 0x05050103, 0x05050109, - 0x0505010f, 0x05050503, 0x05050507, 0x05050701, 0x0505070f, 0x05050903, 0x05050b07, 0x05050b0f, - 0x05050f03, 0x05050f09, 0x05070101, 0x05070105, 0x0507010b, 0x05070303, 0x05070505, 0x05070509, - 0x05070703, 0x05070707, 0x05070905, 0x05070b01, 0x05070d0d, 0x05090103, 0x0509010f, 0x05090501, - 0x05090507, 0x05090705, 0x0509070b, 0x05090903, 0x05090f05, 0x05090f0b, 0x050b0109, 0x050b0303, - 0x050b0505, 0x050b070f, 0x050b0901, 0x050b0b07, 0x050b0f01, 0x050d0101, 0x050d0105, 0x050d010f, - 0x050d0503, 0x050d0b0b, 0x050d0d03, 0x050f010b, 0x050f0303, 0x050f050d, 0x050f0701, 0x050f0907, - 0x050f0b01, 0x07010105, 0x07010303, 0x07010307, 0x0701030b, 0x0701030f, 0x07010505, 0x07010703, - 0x07010707, 0x0701070b, 0x07010905, 0x07010909, 0x0701090f, 0x07010b03, 0x07010d07, 0x07010f03, - 0x07030103, 0x07030107, 0x0703010b, 0x07030309, 0x07030503, 0x07030507, 0x07030901, 0x07030d01, - 0x07030f05, 0x07030f0d, 0x07050101, 0x07050305, 0x07050501, 0x07050705, 0x07050709, 0x07050b01, - 0x07070103, 0x07070301, 0x07070309, 0x07070503, 0x07070507, 0x0707050f, 0x07070701, 0x07070903, - 0x07070907, 0x0707090f, 0x07070b0b, 0x07070f07, 0x07090107, 0x07090303, 0x0709030d, 0x07090505, - 0x07090703, 0x07090b05, 0x07090d01, 0x07090d09, 0x070b0103, 0x070b0301, 0x070b0305, 0x070b050b, - 0x070b0705, 0x070b0909, 0x070b0b0d, 0x070b0f07, 0x070d030d, 0x070d0903, 0x070f0103, 0x070f0107, - 0x070f0501, 0x070f0505, 0x070f070b, 0x09010101, 0x09010109, 0x09010305, 0x09010501, 0x09010509, - 0x0901050f, 0x09010705, 0x09010903, 0x09010b01, 0x09010f01, 0x09030105, 0x0903010f, 0x09030303, - 0x09030307, 0x09030505, 0x09030701, 0x0903070b, 0x09030907, 0x09030b03, 0x09030b0b, 0x09050103, - 0x09050107, 0x09050301, 0x0905030b, 0x09050503, 0x09050707, 0x09050901, 0x09050b0f, 0x09050d05, - 0x09050f01, 0x09070109, 0x09070303, 0x09070307, 0x09070501, 0x09070505, 0x09070703, 0x0907070b, - 0x09090101, 0x09090105, 0x09090509, 0x0909070f, 0x09090901, 0x09090f03, 0x090b010b, 0x090b010f, - 0x090b0503, 0x090b0d05, 0x090d0307, 0x090d0709, 0x090d0d01, 0x090f0301, 0x090f030b, 0x090f0701, - 0x090f0907, 0x090f0b03, 0x0b010105, 0x0b010301, 0x0b010309, 0x0b010505, 0x0b010901, 0x0b010909, - 0x0b01090f, 0x0b010b05, 0x0b010d0d, 0x0b010f09, 0x0b030103, 0x0b030107, 0x0b03010b, 0x0b030305, - 0x0b030503, 0x0b030705, 0x0b030f05, 0x0b050101, 0x0b050303, 0x0b050507, 0x0b050701, 0x0b05070d, - 0x0b050b07, 0x0b070105, 0x0b07010f, 0x0b070301, 0x0b07050f, 0x0b070909, 0x0b070b03, 0x0b070d0b, - 0x0b070f07, 0x0b090103, 0x0b090109, 0x0b090501, 0x0b090705, 0x0b09090d, 0x0b0b0305, 0x0b0b050d, - 0x0b0b0b03, 0x0b0b0b07, 0x0b0d0905, 0x0b0f0105, 0x0b0f0109, 0x0b0f0505, 0x0d010303, 0x0d010307, - 0x0d01030b, 0x0d010703, 0x0d010707, 0x0d010d01, 0x0d030101, 0x0d030501, 0x0d03050f, 0x0d030d09, - 0x0d050305, 0x0d050709, 0x0d050905, 0x0d050b0b, 0x0d050d05, 0x0d050f01, 0x0d070101, 0x0d070309, - 0x0d070503, 0x0d070901, 0x0d09050b, 0x0d090907, 0x0d090d05, 0x0d0b0101, 0x0d0b0107, 0x0d0b0709, - 0x0d0b0d01, 0x0d0d010b, 0x0d0d0901, 0x0d0f0303, 0x0d0f0307, 0x0f010101, 0x0f010109, 0x0f01010f, - 0x0f010501, 0x0f010505, 0x0f01070d, 0x0f010901, 0x0f010b09, 0x0f010d05, 0x0f030105, 0x0f030303, - 0x0f030509, 0x0f030907, 0x0f03090b, 0x0f050103, 0x0f050109, 0x0f050301, 0x0f05030d, 0x0f050503, - 0x0f050701, 0x0f050b03, 0x0f070105, 0x0f070705, 0x0f07070b, 0x0f070b07, 0x0f090103, 0x0f09010b, - 0x0f090307, 0x0f090501, 0x0f090b01, 0x0f0b0505, 0x0f0b0905, 0x0f0d0105, 0x0f0d0703, 0x0f0f0101, -}; - -#define NGRID_IQ2XXS 512 -static const uint64_t iq1s_grid[NGRID_IQ2XXS] = { - 0xffffffffffff0101, 0xffffffffff01ff00, 0xffffffffff010100, 0xffffffff00000000, - 0xffffffff01ff00ff, 0xffffffff01ff0001, 0xffffffff0101ffff, 0xffffffff0101ff01, - 0xffffff00ff000000, 0xffffff000000ff00, 0xffffff00000000ff, 0xffffff0000000100, - 0xffffff0000010000, 0xffffff0001000000, 0xffffff01ffff00ff, 0xffffff01ff01ff00, - 0xffffff01ff010100, 0xffffff0100000001, 0xffffff0101ffff00, 0xffffff0101ff0101, - 0xffffff0101010100, 0xffff00ffff00ff01, 0xffff00ffff0000ff, 0xffff00ff00ff0100, - 0xffff00ff0100ff00, 0xffff00ff010001ff, 0xffff0000ff0101ff, 0xffff000000ffff00, - 0xffff000000000000, 0xffff00000001ff01, 0xffff000001000101, 0xffff0000010100ff, - 0xffff0001ffff0100, 0xffff00010000ff00, 0xffff000100010101, 0xffff000101000000, - 0xffff01ffffff0000, 0xffff01ffff01ffff, 0xffff01ffff010100, 0xffff01ff00000000, - 0xffff01ff01ffffff, 0xffff01ff01ff0001, 0xffff01ff0101ffff, 0xffff01ff01010001, - 0xffff0100ffffff01, 0xffff01000000ffff, 0xffff010000000100, 0xffff010001ff01ff, - 0xffff010001000000, 0xffff0101ff000000, 0xffff0101000101ff, 0xffff010101ffff01, - 0xffff01010101ff00, 0xff00ffffff000000, 0xff00ffff00ffff00, 0xff00ffff00000001, - 0xff00ffff000001ff, 0xff00ffff01010000, 0xff00ff00ffff0000, 0xff00ff00ff00ff00, - 0xff00ff00ff0000ff, 0xff00ff00ff000100, 0xff00ff00ff010001, 0xff00ff0000ff0001, - 0xff00ff000000ffff, 0xff00ff0000000000, 0xff00ff000001ff00, 0xff00ff0000010100, - 0xff00ff0001ff0000, 0xff00ff000100ff00, 0xff00ff0001000100, 0xff00ff01ff000000, - 0xff00ff0100ff0000, 0xff00ff01000001ff, 0xff00ff0101010001, 0xff0000ff00000000, - 0xff0000ff0001ff00, 0xff0000ff00010100, 0xff000000ffff0101, 0xff000000ff000000, - 0xff000000ff01ff00, 0xff00000000ff0000, 0xff0000000000ff00, 0xff000000000000ff, - 0xff00000000000000, 0xff00000000000001, 0xff00000000000100, 0xff0000000001ffff, - 0xff00000000010000, 0xff00000001000000, 0xff00000001010100, 0xff000001ff00ff01, - 0xff000001ff0100ff, 0xff00000100000000, 0xff0000010001ff00, 0xff00000101ff0100, - 0xff0000010100ff00, 0xff0001ff00ff00ff, 0xff0001ff00000101, 0xff0001ff000100ff, - 0xff0001ff01000000, 0xff000100ff0001ff, 0xff0001000000ff01, 0xff00010000000000, - 0xff00010000010001, 0xff00010000010100, 0xff00010001ffff00, 0xff00010001ff0101, - 0xff00010001010000, 0xff000101ffffffff, 0xff000101ff000101, 0xff00010101ff00ff, - 0xff00010101000001, 0xff000101010100ff, 0xff01ffffff000101, 0xff01ffffff01ffff, - 0xff01ffffff01ff01, 0xff01ffffff0101ff, 0xff01ffff00000000, 0xff01ffff01ff0001, - 0xff01ffff0101ff01, 0xff01ff00ff000000, 0xff01ff0000ff0100, 0xff01ff000000ff01, - 0xff01ff0000010000, 0xff01ff00010000ff, 0xff01ff01ff01ff00, 0xff01ff0100000101, - 0xff0100ffffff0000, 0xff0100ffff010000, 0xff0100ff01ff00ff, 0xff0100ff01000100, - 0xff0100ff010100ff, 0xff010000ffffff01, 0xff01000000000000, 0xff0100000101ff00, - 0xff010001ffff00ff, 0xff010001ff000100, 0xff01000100ffff00, 0xff01000100010001, - 0xff01000101ff0001, 0xff010001010001ff, 0xff0101ffffffffff, 0xff0101ffff01ffff, - 0xff0101ffff010101, 0xff0101ff0000ff00, 0xff0101ff01010001, 0xff010100ff000000, - 0xff010100ff01ff01, 0xff01010000ff0001, 0xff01010000000100, 0xff01010001000000, - 0xff0101010100ffff, 0x00ffffff0000ff01, 0x00ffffff000000ff, 0x00ffffff00000100, - 0x00ffffff00010000, 0x00ffff00ffff0001, 0x00ffff00ff0000ff, 0x00ffff00ff000100, - 0x00ffff0000000000, 0x00ffff0001000100, 0x00ffff0001010001, 0x00ffff01ff00ff01, - 0x00ffff0100ff0100, 0x00ffff010000ff00, 0x00ffff01000100ff, 0x00ffff0101ff00ff, - 0x00ffff010101ff00, 0x00ff00ffffffffff, 0x00ff00ffffff01ff, 0x00ff00ffff000101, - 0x00ff00ff00000000, 0x00ff00ff000101ff, 0x00ff00ff01010101, 0x00ff0000ff000000, - 0x00ff0000ff01ffff, 0x00ff000000ff0000, 0x00ff00000000ff00, 0x00ff0000000000ff, - 0x00ff000000000000, 0x00ff000000000001, 0x00ff000000000100, 0x00ff000000010000, - 0x00ff000001ffff01, 0x00ff000001000000, 0x00ff0001ff000101, 0x00ff000100ffffff, - 0x00ff000100000000, 0x00ff0001010001ff, 0x00ff01ffff000000, 0x00ff01ff0001ff00, - 0x00ff01ff01ff0100, 0x00ff0100ff01ff01, 0x00ff010000ff00ff, 0x00ff010000ff0101, - 0x00ff010000000000, 0x00ff010000010101, 0x00ff01000100ff00, 0x00ff010001010000, - 0x00ff0101ffffff00, 0x00ff01010000ff01, 0x00ff010100000100, 0x00ff010101ff0000, - 0x0000ffffffff0100, 0x0000ffffff00ff00, 0x0000ffffff0000ff, 0x0000ffffff010000, - 0x0000ffff00000000, 0x0000ffff00010101, 0x0000ffff01ffff01, 0x0000ffff01000100, - 0x0000ff00ff000000, 0x0000ff00ff01ff00, 0x0000ff00ff0101ff, 0x0000ff0000ff0000, - 0x0000ff000000ff00, 0x0000ff00000000ff, 0x0000ff0000000000, 0x0000ff0000000001, - 0x0000ff0000000100, 0x0000ff0000010000, 0x0000ff0001ffffff, 0x0000ff0001ff01ff, - 0x0000ff0001000000, 0x0000ff000101ffff, 0x0000ff01ffff0101, 0x0000ff01ff010000, - 0x0000ff0100000000, 0x0000ff0101000101, 0x000000ffffff0001, 0x000000ffff000000, - 0x000000ff00ff0000, 0x000000ff0000ff00, 0x000000ff000000ff, 0x000000ff00000000, - 0x000000ff00000001, 0x000000ff00000100, 0x000000ff00010000, 0x000000ff01000000, - 0x000000ff0101ff00, 0x00000000ffff0000, 0x00000000ff00ff00, 0x00000000ff0000ff, - 0x00000000ff000000, 0x00000000ff000001, 0x00000000ff000100, 0x00000000ff010000, - 0x0000000000ffff00, 0x0000000000ff00ff, 0x0000000000ff0000, 0x0000000000ff0001, - 0x0000000000ff0100, 0x000000000000ffff, 0x000000000000ff00, 0x000000000000ff01, - 0x00000000000000ff, 0x0000000000000001, 0x00000000000001ff, 0x0000000000000100, - 0x0000000000000101, 0x000000000001ff00, 0x00000000000100ff, 0x0000000000010000, - 0x0000000000010001, 0x0000000000010100, 0x0000000001ff0000, 0x000000000100ff00, - 0x00000000010000ff, 0x0000000001000000, 0x0000000001000001, 0x0000000001000100, - 0x0000000001010000, 0x00000001ffff01ff, 0x00000001ff000000, 0x0000000100ff0000, - 0x000000010000ff00, 0x00000001000000ff, 0x0000000100000000, 0x0000000100000001, - 0x0000000100000100, 0x0000000100010000, 0x0000000101000000, 0x000001ffff00ff00, - 0x000001ffff010001, 0x000001ffff0101ff, 0x000001ff00ffff01, 0x000001ff0000ffff, - 0x000001ff00000000, 0x000001ff010000ff, 0x000001ff01010100, 0x00000100ffff0100, - 0x00000100ff000000, 0x0000010000ff0000, 0x000001000000ff00, 0x00000100000000ff, - 0x0000010000000000, 0x0000010000000001, 0x0000010000000100, 0x0000010000010000, - 0x0000010001000000, 0x000001000101ff01, 0x00000101ffff0001, 0x00000101ff01ffff, - 0x0000010100000000, 0x0000010101010100, 0x0001ffffff000000, 0x0001ffff00ffffff, - 0x0001ffff00000100, 0x0001ffff0001ff00, 0x0001ffff01000000, 0x0001ff00ffffff00, - 0x0001ff00ffff01ff, 0x0001ff00ff010000, 0x0001ff0000000000, 0x0001ff0000010001, - 0x0001ff0001ff0000, 0x0001ff0001010100, 0x0001ff01ff0000ff, 0x0001ff01ff000001, - 0x0001ff0100ffffff, 0x0001ff010001ffff, 0x0001ff01000101ff, 0x0001ff010100ff01, - 0x000100ffff00ffff, 0x000100ffff00ff01, 0x000100ffff000100, 0x000100ff00000000, - 0x000100ff000101ff, 0x000100ff01ff0101, 0x000100ff0100ffff, 0x000100ff01010101, - 0x00010000ff000000, 0x00010000ff010100, 0x0001000000ff0000, 0x000100000000ff00, - 0x00010000000000ff, 0x0001000000000000, 0x0001000000000001, 0x0001000000000100, - 0x0001000000010000, 0x0001000001ffff01, 0x0001000001000000, 0x0001000100ff0101, - 0x0001000100000000, 0x00010001010100ff, 0x000101ffffff01ff, 0x000101ffffff0101, - 0x000101ff00010000, 0x000101ff01ff0000, 0x000101ff0100ff01, 0x00010100ffff0000, - 0x0001010000000000, 0x000101000001ffff, 0x0001010000010101, 0x00010100010001ff, - 0x00010101ff00ff00, 0x00010101ff010001, 0x0001010100ffffff, 0x0001010100ff01ff, - 0x00010101000101ff, 0x0001010101ff0000, 0x000101010100ff01, 0x0001010101000101, - 0x01ffffffffff0101, 0x01ffffffff01ffff, 0x01ffffffff01ff01, 0x01ffffffff0101ff, - 0x01ffffffff010101, 0x01ffffff00000000, 0x01ffffff01ff01ff, 0x01ffffff01000101, - 0x01ffffff0101ff01, 0x01ffffff010100ff, 0x01ffff000000ff00, 0x01ffff0000000001, - 0x01ffff00000001ff, 0x01ffff0000010000, 0x01ffff0001ff0000, 0x01ffff01ffffffff, - 0x01ffff01ffff01ff, 0x01ffff01ff000000, 0x01ffff01ff01ffff, 0x01ffff01ff0101ff, - 0x01ffff010100ffff, 0x01ff00ffffff0000, 0x01ff00ffff010000, 0x01ff00ff00ffff01, - 0x01ff0000ff0000ff, 0x01ff000000000000, 0x01ff00000001ff01, 0x01ff000001ffffff, - 0x01ff000001010100, 0x01ff0001ffffff01, 0x01ff0001ff010001, 0x01ff000101ff0100, - 0x01ff000101000001, 0x01ff0001010100ff, 0x01ff01ffff00ffff, 0x01ff01ff00010001, - 0x01ff01ff01000000, 0x01ff01ff010101ff, 0x01ff0100ff000001, 0x01ff010000ffff00, - 0x01ff010000000100, 0x01ff010001ff01ff, 0x01ff01000101ffff, 0x01ff0101ffff00ff, - 0x01ff0101ffff0101, 0x01ff0101ff0101ff, 0x01ff010100010000, 0x0100ffff00ff00ff, - 0x0100ffff00ff0001, 0x0100ffff00000100, 0x0100ffff0100ff00, 0x0100ff00ffff0000, - 0x0100ff00ff00ffff, 0x0100ff00ff00ff01, 0x0100ff00ff000100, 0x0100ff00ff010000, - 0x0100ff0000000000, 0x0100ff00000100ff, 0x0100ff0001ff0101, 0x0100ff0001010101, - 0x0100ff0100ff00ff, 0x0100ff0100ff0001, 0x0100ff0100000100, 0x0100ff0100010001, - 0x0100ff0101000000, 0x010000ffff00ff00, 0x010000ff0000ffff, 0x010000ff00000000, - 0x010000ff010001ff, 0x010000ff01010001, 0x01000000ffffff00, 0x01000000ffff0101, - 0x01000000ff000000, 0x01000000ff0100ff, 0x01000000ff010101, 0x0100000000ff0000, - 0x010000000000ff00, 0x01000000000000ff, 0x0100000000000000, 0x0100000000000001, - 0x0100000000000100, 0x0100000000010000, 0x0100000001000000, 0x0100000100000000, - 0x01000001000101ff, 0x0100000101ffff01, 0x010001ffff000101, 0x010001ff00ff0100, - 0x010001ff0000ff00, 0x010001ff000100ff, 0x010001ff01ffffff, 0x01000100ffff0000, - 0x01000100ff0001ff, 0x0100010000000000, 0x010001000001ff00, 0x0100010001ff0000, - 0x01000100010000ff, 0x0100010001000101, 0x01000101ff00ff01, 0x0100010100ff0100, - 0x010001010000ffff, 0x0100010101010001, 0x0101ffffffff0101, 0x0101ffffff0001ff, - 0x0101ffffff01ffff, 0x0101ffffff010101, 0x0101ffff00000000, 0x0101ffff0101ffff, - 0x0101ffff010101ff, 0x0101ff00ff000000, 0x0101ff0000ff0100, 0x0101ff000000ff00, - 0x0101ff0000010000, 0x0101ff00010000ff, 0x0101ff0001000001, 0x0101ff01ff010101, - 0x0101ff0100000000, 0x0101ff010101ff00, 0x010100ffffff0000, 0x010100ffff010000, - 0x010100ff00ff01ff, 0x010100ff000000ff, 0x010100ff00000101, 0x010100ff01ffff00, - 0x01010000ffffff01, 0x01010000ff000100, 0x01010000ff01ff01, 0x0101000000000000, - 0x01010000000100ff, 0x010100000101ff01, 0x01010001ffff0000, 0x01010001ff00ffff, - 0x01010001ff010000, 0x0101000101ffffff, 0x0101000101ff01ff, 0x0101000101010101, - 0x010101ffff01ffff, 0x010101ff00000000, 0x010101ff0001ff01, 0x010101ff0101ffff, - 0x010101ff010101ff, 0x01010100ffffffff, 0x01010100ff000001, 0x010101000000ff00, - 0x0101010001010000, 0x0101010100ff0001, 0x010101010001ff01, 0x010101010101ffff, - -}; - -static const uint8_t ksigns_iq2xs[128] = { - 0, 129, 130, 3, 132, 5, 6, 135, 136, 9, 10, 139, 12, 141, 142, 15, - 144, 17, 18, 147, 20, 149, 150, 23, 24, 153, 154, 27, 156, 29, 30, 159, - 160, 33, 34, 163, 36, 165, 166, 39, 40, 169, 170, 43, 172, 45, 46, 175, - 48, 177, 178, 51, 180, 53, 54, 183, 184, 57, 58, 187, 60, 189, 190, 63, - 192, 65, 66, 195, 68, 197, 198, 71, 72, 201, 202, 75, 204, 77, 78, 207, - 80, 209, 210, 83, 212, 85, 86, 215, 216, 89, 90, 219, 92, 221, 222, 95, - 96, 225, 226, 99, 228, 101, 102, 231, 232, 105, 106, 235, 108, 237, 238, 111, - 240, 113, 114, 243, 116, 245, 246, 119, 120, 249, 250, 123, 252, 125, 126, 255, -}; - -static const uint8_t kmask_iq2xs[8] = {1, 2, 4, 8, 16, 32, 64, 128}; - void dequantize_row_iq2_xxs(const block_iq2_xxs * restrict x, float * restrict y, int k) { assert(k % QK_K == 0); const int nb = k / QK_K; diff --git a/ggml-quants.h b/ggml-quants.h index 316e35687..d2d25eb44 100644 --- a/ggml-quants.h +++ b/ggml-quants.h @@ -1,9 +1,9 @@ #pragma once -#include "ggml-impl.h" - // GGML internal header +#include "ggml-impl.h" + #include #include diff --git a/ggml-sycl.cpp b/ggml-sycl.cpp index ddd951dd6..6fafa5282 100644 --- a/ggml-sycl.cpp +++ b/ggml-sycl.cpp @@ -3144,6 +3144,8 @@ namespace dpct } // COPY from DPCT head files +#define GGML_COMMON_IMPL_SYCL +#include "ggml-common.h" static int g_ggml_sycl_debug=0; #define GGML_SYCL_DEBUG(...) do{if(g_ggml_sycl_debug) printf(__VA_ARGS__);}while(0) @@ -4745,388 +4747,6 @@ static void dequantize_block_q6_K(const void * __restrict__ vx, dst_t * __restri #endif } -static dpct::global_memory - iq2xxs_grid(sycl::range<1>(256), - { - 0x0808080808080808, 0x080808080808082b, 0x0808080808081919, - 0x0808080808082b08, 0x0808080808082b2b, 0x0808080808190819, - 0x0808080808191908, 0x08080808082b0808, 0x08080808082b082b, - 0x08080808082b2b08, 0x08080808082b2b2b, 0x0808080819080819, - 0x0808080819081908, 0x0808080819190808, 0x0808080819192b08, - 0x08080808192b0819, 0x08080808192b1908, 0x080808082b080808, - 0x080808082b08082b, 0x080808082b082b2b, 0x080808082b2b082b, - 0x0808081908080819, 0x0808081908081908, 0x0808081908190808, - 0x0808081908191919, 0x0808081919080808, 0x080808192b081908, - 0x080808192b192b08, 0x0808082b08080808, 0x0808082b0808082b, - 0x0808082b082b082b, 0x0808082b2b08082b, 0x0808190808080819, - 0x0808190808081908, 0x0808190808190808, 0x08081908082b0819, - 0x08081908082b1908, 0x0808190819080808, 0x080819081908082b, - 0x0808190819082b08, 0x08081908192b0808, 0x080819082b080819, - 0x080819082b081908, 0x080819082b190808, 0x080819082b2b1908, - 0x0808191908080808, 0x080819190808082b, 0x0808191908082b08, - 0x08081919082b0808, 0x080819191908192b, 0x08081919192b2b19, - 0x080819192b080808, 0x080819192b190819, 0x0808192b08082b19, - 0x0808192b08190808, 0x0808192b19080808, 0x0808192b2b081908, - 0x0808192b2b2b1908, 0x08082b0808080808, 0x08082b0808081919, - 0x08082b0808082b08, 0x08082b0808191908, 0x08082b08082b2b08, - 0x08082b0819080819, 0x08082b0819081908, 0x08082b0819190808, - 0x08082b081919082b, 0x08082b082b082b08, 0x08082b1908081908, - 0x08082b1919080808, 0x08082b2b0808082b, 0x08082b2b08191908, - 0x0819080808080819, 0x0819080808081908, 0x0819080808190808, - 0x08190808082b0819, 0x0819080819080808, 0x08190808192b0808, - 0x081908082b081908, 0x081908082b190808, 0x081908082b191919, - 0x0819081908080808, 0x0819081908082b08, 0x08190819082b0808, - 0x0819081919190808, 0x0819081919192b2b, 0x081908192b080808, - 0x0819082b082b1908, 0x0819082b19081919, 0x0819190808080808, - 0x0819190808082b08, 0x08191908082b0808, 0x08191908082b1919, - 0x0819190819082b19, 0x081919082b080808, 0x0819191908192b08, - 0x08191919192b082b, 0x0819192b08080808, 0x0819192b0819192b, - 0x08192b0808080819, 0x08192b0808081908, 0x08192b0808190808, - 0x08192b0819080808, 0x08192b082b080819, 0x08192b1908080808, - 0x08192b1908081919, 0x08192b192b2b0808, 0x08192b2b19190819, - 0x082b080808080808, 0x082b08080808082b, 0x082b080808082b2b, - 0x082b080819081908, 0x082b0808192b0819, 0x082b08082b080808, - 0x082b08082b08082b, 0x082b0819082b2b19, 0x082b081919082b08, - 0x082b082b08080808, 0x082b082b0808082b, 0x082b190808080819, - 0x082b190808081908, 0x082b190808190808, 0x082b190819080808, - 0x082b19081919192b, 0x082b191908080808, 0x082b191919080819, - 0x082b1919192b1908, 0x082b192b2b190808, 0x082b2b0808082b08, - 0x082b2b08082b0808, 0x082b2b082b191908, 0x082b2b2b19081908, - 0x1908080808080819, 0x1908080808081908, 0x1908080808190808, - 0x1908080808192b08, 0x19080808082b0819, 0x19080808082b1908, - 0x1908080819080808, 0x1908080819082b08, 0x190808081919192b, - 0x19080808192b0808, 0x190808082b080819, 0x190808082b081908, - 0x190808082b190808, 0x1908081908080808, 0x19080819082b0808, - 0x19080819192b0819, 0x190808192b080808, 0x190808192b081919, - 0x1908082b08080819, 0x1908082b08190808, 0x1908082b19082b08, - 0x1908082b1919192b, 0x1908082b192b2b08, 0x1908190808080808, - 0x1908190808082b08, 0x19081908082b0808, 0x190819082b080808, - 0x190819082b192b19, 0x190819190819082b, 0x19081919082b1908, - 0x1908192b08080808, 0x19082b0808080819, 0x19082b0808081908, - 0x19082b0808190808, 0x19082b0819080808, 0x19082b0819081919, - 0x19082b1908080808, 0x19082b1919192b08, 0x19082b19192b0819, - 0x19082b192b08082b, 0x19082b2b19081919, 0x19082b2b2b190808, - 0x1919080808080808, 0x1919080808082b08, 0x1919080808190819, - 0x1919080808192b19, 0x19190808082b0808, 0x191908082b080808, - 0x191908082b082b08, 0x1919081908081908, 0x191908191908082b, - 0x191908192b2b1908, 0x1919082b2b190819, 0x191919082b190808, - 0x191919082b19082b, 0x1919191908082b2b, 0x1919192b08080819, - 0x1919192b19191908, 0x19192b0808080808, 0x19192b0808190819, - 0x19192b0808192b19, 0x19192b08192b1908, 0x19192b1919080808, - 0x19192b2b08082b08, 0x192b080808081908, 0x192b080808190808, - 0x192b080819080808, 0x192b0808192b2b08, 0x192b081908080808, - 0x192b081919191919, 0x192b082b08192b08, 0x192b082b192b0808, - 0x192b190808080808, 0x192b190808081919, 0x192b191908190808, - 0x192b19190819082b, 0x192b19192b081908, 0x192b2b081908082b, - 0x2b08080808080808, 0x2b0808080808082b, 0x2b08080808082b2b, - 0x2b08080819080819, 0x2b0808082b08082b, 0x2b08081908081908, - 0x2b08081908192b08, 0x2b08081919080808, 0x2b08082b08190819, - 0x2b08190808080819, 0x2b08190808081908, 0x2b08190808190808, - 0x2b08190808191919, 0x2b08190819080808, 0x2b081908192b0808, - 0x2b08191908080808, 0x2b0819191908192b, 0x2b0819192b191908, - 0x2b08192b08082b19, 0x2b08192b19080808, 0x2b08192b192b0808, - 0x2b082b080808082b, 0x2b082b1908081908, 0x2b082b2b08190819, - 0x2b19080808081908, 0x2b19080808190808, 0x2b190808082b1908, - 0x2b19080819080808, 0x2b1908082b2b0819, 0x2b1908190819192b, - 0x2b1908192b080808, 0x2b19082b19081919, 0x2b19190808080808, - 0x2b191908082b082b, 0x2b19190819081908, 0x2b19191919190819, - 0x2b192b082b080819, 0x2b192b19082b0808, 0x2b2b08080808082b, - 0x2b2b080819190808, 0x2b2b08082b081919, 0x2b2b081908082b19, - 0x2b2b082b08080808, 0x2b2b190808192b08, 0x2b2b2b0819190808, - 0x2b2b2b1908081908, - }); - -static dpct::global_memory - iq2xs_grid(sycl::range<1>(512), - { - 0x0808080808080808, 0x080808080808082b, 0x0808080808081919, - 0x0808080808082b08, 0x0808080808082b2b, 0x0808080808190819, - 0x0808080808191908, 0x080808080819192b, 0x0808080808192b19, - 0x08080808082b0808, 0x08080808082b082b, 0x08080808082b1919, - 0x08080808082b2b08, 0x0808080819080819, 0x0808080819081908, - 0x080808081908192b, 0x0808080819082b19, 0x0808080819190808, - 0x080808081919082b, 0x0808080819191919, 0x0808080819192b08, - 0x08080808192b0819, 0x08080808192b1908, 0x080808082b080808, - 0x080808082b08082b, 0x080808082b081919, 0x080808082b082b08, - 0x080808082b190819, 0x080808082b191908, 0x080808082b192b19, - 0x080808082b2b0808, 0x0808081908080819, 0x0808081908081908, - 0x080808190808192b, 0x0808081908082b19, 0x0808081908190808, - 0x080808190819082b, 0x0808081908191919, 0x0808081908192b08, - 0x0808081908192b2b, 0x08080819082b0819, 0x08080819082b1908, - 0x0808081919080808, 0x080808191908082b, 0x0808081919081919, - 0x0808081919082b08, 0x0808081919190819, 0x0808081919191908, - 0x08080819192b0808, 0x08080819192b2b08, 0x080808192b080819, - 0x080808192b081908, 0x080808192b190808, 0x0808082b08080808, - 0x0808082b0808082b, 0x0808082b08081919, 0x0808082b08082b08, - 0x0808082b08190819, 0x0808082b08191908, 0x0808082b082b0808, - 0x0808082b19080819, 0x0808082b19081908, 0x0808082b19190808, - 0x0808082b19191919, 0x0808082b2b080808, 0x0808082b2b082b2b, - 0x0808190808080819, 0x0808190808081908, 0x080819080808192b, - 0x0808190808082b19, 0x0808190808190808, 0x080819080819082b, - 0x0808190808191919, 0x0808190808192b08, 0x08081908082b0819, - 0x08081908082b1908, 0x0808190819080808, 0x080819081908082b, - 0x0808190819081919, 0x0808190819082b08, 0x0808190819190819, - 0x0808190819191908, 0x080819081919192b, 0x08081908192b0808, - 0x080819082b080819, 0x080819082b081908, 0x080819082b190808, - 0x0808191908080808, 0x080819190808082b, 0x0808191908081919, - 0x0808191908082b08, 0x0808191908190819, 0x0808191908191908, - 0x08081919082b0808, 0x0808191919080819, 0x0808191919081908, - 0x0808191919190808, 0x08081919192b0819, 0x080819192b080808, - 0x0808192b08080819, 0x0808192b08081908, 0x0808192b08190808, - 0x0808192b082b192b, 0x0808192b19080808, 0x0808192b1908082b, - 0x0808192b2b081908, 0x08082b0808080808, 0x08082b080808082b, - 0x08082b0808081919, 0x08082b0808082b08, 0x08082b0808082b2b, - 0x08082b0808190819, 0x08082b0808191908, 0x08082b08082b0808, - 0x08082b08082b1919, 0x08082b0819080819, 0x08082b0819081908, - 0x08082b0819190808, 0x08082b0819192b08, 0x08082b082b080808, - 0x08082b082b2b0808, 0x08082b082b2b2b2b, 0x08082b1908080819, - 0x08082b1908081908, 0x08082b1908190808, 0x08082b1919080808, - 0x08082b192b080819, 0x08082b192b082b19, 0x08082b2b08080808, - 0x08082b2b082b0808, 0x08082b2b082b2b08, 0x08082b2b2b19192b, - 0x08082b2b2b2b0808, 0x0819080808080819, 0x0819080808081908, - 0x081908080808192b, 0x0819080808082b19, 0x0819080808190808, - 0x081908080819082b, 0x0819080808191919, 0x0819080808192b08, - 0x08190808082b0819, 0x08190808082b1908, 0x0819080819080808, - 0x081908081908082b, 0x0819080819081919, 0x0819080819082b08, - 0x0819080819190819, 0x0819080819191908, 0x08190808192b0808, - 0x08190808192b2b2b, 0x081908082b080819, 0x081908082b081908, - 0x081908082b190808, 0x0819081908080808, 0x081908190808082b, - 0x0819081908081919, 0x0819081908082b08, 0x0819081908190819, - 0x0819081908191908, 0x08190819082b0808, 0x0819081919080819, - 0x0819081919081908, 0x0819081919190808, 0x081908192b080808, - 0x081908192b191908, 0x081908192b19192b, 0x0819082b08080819, - 0x0819082b08081908, 0x0819082b0808192b, 0x0819082b08190808, - 0x0819082b19080808, 0x0819082b192b0808, 0x0819190808080808, - 0x081919080808082b, 0x0819190808081919, 0x0819190808082b08, - 0x0819190808190819, 0x0819190808191908, 0x08191908082b0808, - 0x0819190819080819, 0x0819190819081908, 0x0819190819082b19, - 0x0819190819190808, 0x08191908192b1908, 0x081919082b080808, - 0x0819191908080819, 0x0819191908081908, 0x0819191908190808, - 0x0819191919080808, 0x0819192b08080808, 0x0819192b08191908, - 0x0819192b19082b19, 0x08192b0808080819, 0x08192b0808081908, - 0x08192b0808190808, 0x08192b080819082b, 0x08192b0819080808, - 0x08192b0819191908, 0x08192b082b08192b, 0x08192b1908080808, - 0x08192b1908081919, 0x08192b19192b192b, 0x08192b2b19190819, - 0x08192b2b2b2b2b19, 0x082b080808080808, 0x082b08080808082b, - 0x082b080808081919, 0x082b080808082b08, 0x082b080808082b2b, - 0x082b080808190819, 0x082b080808191908, 0x082b0808082b0808, - 0x082b080819080819, 0x082b080819081908, 0x082b080819190808, - 0x082b08082b080808, 0x082b08082b2b0808, 0x082b081908080819, - 0x082b081908081908, 0x082b081908190808, 0x082b081919080808, - 0x082b081919082b08, 0x082b0819192b1919, 0x082b082b08080808, - 0x082b082b082b082b, 0x082b082b2b080808, 0x082b082b2b2b2b08, - 0x082b190808080819, 0x082b190808081908, 0x082b190808190808, - 0x082b1908082b2b19, 0x082b190819080808, 0x082b191908080808, - 0x082b191919080819, 0x082b19191919082b, 0x082b19192b192b19, - 0x082b192b08080819, 0x082b192b08192b2b, 0x082b192b2b2b192b, - 0x082b2b0808080808, 0x082b2b0808082b08, 0x082b2b0808082b2b, - 0x082b2b08082b0808, 0x082b2b0819191919, 0x082b2b082b082b08, - 0x082b2b082b2b082b, 0x082b2b19192b2b08, 0x082b2b192b190808, - 0x082b2b2b08082b08, 0x082b2b2b082b0808, 0x082b2b2b2b08082b, - 0x082b2b2b2b082b08, 0x082b2b2b2b082b2b, 0x1908080808080819, - 0x1908080808081908, 0x190808080808192b, 0x1908080808082b19, - 0x1908080808190808, 0x190808080819082b, 0x1908080808191919, - 0x1908080808192b08, 0x19080808082b0819, 0x19080808082b1908, - 0x1908080819080808, 0x190808081908082b, 0x1908080819081919, - 0x1908080819082b08, 0x1908080819082b2b, 0x1908080819190819, - 0x1908080819191908, 0x19080808192b0808, 0x19080808192b1919, - 0x190808082b080819, 0x190808082b081908, 0x190808082b190808, - 0x1908081908080808, 0x190808190808082b, 0x1908081908081919, - 0x1908081908082b08, 0x1908081908190819, 0x1908081908191908, - 0x19080819082b0808, 0x1908081919080819, 0x1908081919081908, - 0x1908081919190808, 0x190808192b080808, 0x190808192b081919, - 0x190808192b2b082b, 0x1908082b08080819, 0x1908082b08081908, - 0x1908082b08190808, 0x1908082b0819082b, 0x1908082b082b2b19, - 0x1908082b19080808, 0x1908190808080808, 0x190819080808082b, - 0x1908190808081919, 0x1908190808082b08, 0x1908190808190819, - 0x1908190808191908, 0x1908190808192b19, 0x19081908082b0808, - 0x1908190819080819, 0x1908190819081908, 0x1908190819190808, - 0x190819082b080808, 0x190819082b191908, 0x1908191908080819, - 0x1908191908081908, 0x1908191908190808, 0x19081919082b1908, - 0x1908191919080808, 0x190819192b192b2b, 0x1908192b08080808, - 0x1908192b08082b2b, 0x1908192b19081908, 0x1908192b19190808, - 0x19082b0808080819, 0x19082b0808081908, 0x19082b0808190808, - 0x19082b0819080808, 0x19082b0819081919, 0x19082b0819191908, - 0x19082b08192b082b, 0x19082b1908080808, 0x19082b1908190819, - 0x19082b1919081908, 0x19082b1919190808, 0x19082b19192b2b19, - 0x19082b2b08081908, 0x1919080808080808, 0x191908080808082b, - 0x1919080808081919, 0x1919080808082b08, 0x1919080808190819, - 0x1919080808191908, 0x19190808082b0808, 0x19190808082b2b08, - 0x1919080819080819, 0x1919080819081908, 0x1919080819190808, - 0x191908082b080808, 0x1919081908080819, 0x1919081908081908, - 0x1919081908190808, 0x1919081908191919, 0x1919081919080808, - 0x191908191908082b, 0x1919082b08080808, 0x1919082b19081908, - 0x1919082b2b2b2b2b, 0x1919190808080819, 0x1919190808081908, - 0x1919190808190808, 0x19191908082b0819, 0x1919190819080808, - 0x19191908192b0808, 0x191919082b080819, 0x191919082b2b0819, - 0x1919191908080808, 0x1919191908082b08, 0x191919192b080808, - 0x191919192b082b08, 0x1919192b082b0819, 0x1919192b192b2b08, - 0x1919192b2b2b0819, 0x19192b0808080808, 0x19192b0808191908, - 0x19192b0819080819, 0x19192b0819190808, 0x19192b082b192b19, - 0x19192b1908192b2b, 0x19192b1919080808, 0x19192b191908082b, - 0x19192b2b2b081919, 0x192b080808080819, 0x192b080808081908, - 0x192b080808190808, 0x192b080819080808, 0x192b080819191908, - 0x192b0808192b082b, 0x192b08082b08192b, 0x192b08082b2b2b19, - 0x192b081908080808, 0x192b082b082b1908, 0x192b082b19082b2b, - 0x192b082b2b19082b, 0x192b190808080808, 0x192b19080819192b, - 0x192b191908190808, 0x192b191919080808, 0x192b191919081919, - 0x192b19192b2b1908, 0x192b2b0808080819, 0x192b2b08192b2b2b, - 0x192b2b19082b1919, 0x192b2b2b0808192b, 0x192b2b2b19191908, - 0x192b2b2b192b082b, 0x2b08080808080808, 0x2b0808080808082b, - 0x2b08080808081919, 0x2b08080808082b08, 0x2b08080808190819, - 0x2b08080808191908, 0x2b080808082b0808, 0x2b080808082b2b2b, - 0x2b08080819080819, 0x2b08080819081908, 0x2b08080819190808, - 0x2b0808082b080808, 0x2b0808082b08082b, 0x2b0808082b2b2b08, - 0x2b0808082b2b2b2b, 0x2b08081908080819, 0x2b08081908081908, - 0x2b0808190808192b, 0x2b08081908190808, 0x2b08081919080808, - 0x2b08081919190819, 0x2b08081919192b19, 0x2b08082b08080808, - 0x2b08082b082b0808, 0x2b08082b2b080808, 0x2b08082b2b08082b, - 0x2b08082b2b2b0808, 0x2b08082b2b2b2b08, 0x2b08190808080819, - 0x2b08190808081908, 0x2b08190808190808, 0x2b0819080819082b, - 0x2b08190808191919, 0x2b08190819080808, 0x2b081908192b0808, - 0x2b0819082b082b19, 0x2b08191908080808, 0x2b08191919081908, - 0x2b0819192b2b1919, 0x2b08192b08192b08, 0x2b08192b192b2b2b, - 0x2b082b0808080808, 0x2b082b0808082b08, 0x2b082b08082b1919, - 0x2b082b0819192b2b, 0x2b082b082b080808, 0x2b082b082b08082b, - 0x2b082b082b2b2b08, 0x2b082b190808192b, 0x2b082b2b082b082b, - 0x2b082b2b2b080808, 0x2b082b2b2b082b08, 0x2b082b2b2b19192b, - 0x2b082b2b2b2b2b08, 0x2b19080808080819, 0x2b19080808081908, - 0x2b19080808190808, 0x2b19080819080808, 0x2b1908081919192b, - 0x2b1908082b081908, 0x2b19081908080808, 0x2b190819082b082b, - 0x2b190819192b1908, 0x2b19082b1919192b, 0x2b19082b2b082b19, - 0x2b19190808080808, 0x2b19190808081919, 0x2b19190819081908, - 0x2b19190819190808, 0x2b19190819192b08, 0x2b191919082b2b19, - 0x2b1919192b190808, 0x2b1919192b19082b, 0x2b19192b19080819, - 0x2b192b0819190819, 0x2b192b082b2b192b, 0x2b192b1919082b19, - 0x2b192b2b08191919, 0x2b192b2b192b0808, 0x2b2b080808080808, - 0x2b2b08080808082b, 0x2b2b080808082b08, 0x2b2b080808082b2b, - 0x2b2b0808082b0808, 0x2b2b0808082b2b2b, 0x2b2b08082b2b0808, - 0x2b2b081919190819, 0x2b2b081919192b19, 0x2b2b08192b2b192b, - 0x2b2b082b08080808, 0x2b2b082b0808082b, 0x2b2b082b08082b08, - 0x2b2b082b082b2b2b, 0x2b2b082b2b080808, 0x2b2b082b2b2b0808, - 0x2b2b190819080808, 0x2b2b19082b191919, 0x2b2b192b192b1919, - 0x2b2b192b2b192b08, 0x2b2b2b0808082b2b, 0x2b2b2b08082b0808, - 0x2b2b2b08082b082b, 0x2b2b2b08082b2b08, 0x2b2b2b082b2b0808, - 0x2b2b2b082b2b2b08, 0x2b2b2b1908081908, 0x2b2b2b192b081908, - 0x2b2b2b192b08192b, 0x2b2b2b2b082b2b08, 0x2b2b2b2b082b2b2b, - 0x2b2b2b2b2b190819, 0x2b2b2b2b2b2b2b2b, - }); - -static dpct::global_memory iq3xxs_grid( - sycl::range<1>(256), - { - 0x04040404, 0x04040414, 0x04040424, 0x04040c0c, 0x04040c1c, 0x04040c3e, - 0x04041404, 0x04041414, 0x04041c0c, 0x04042414, 0x04043e1c, 0x04043e2c, - 0x040c040c, 0x040c041c, 0x040c0c04, 0x040c0c14, 0x040c140c, 0x040c142c, - 0x040c1c04, 0x040c1c14, 0x040c240c, 0x040c2c24, 0x040c3e04, 0x04140404, - 0x04140414, 0x04140424, 0x04140c0c, 0x04141404, 0x04141414, 0x04141c0c, - 0x04141c1c, 0x04141c3e, 0x04142c0c, 0x04142c3e, 0x04143e2c, 0x041c040c, - 0x041c043e, 0x041c0c04, 0x041c0c14, 0x041c142c, 0x041c3e04, 0x04240c1c, - 0x04241c3e, 0x04242424, 0x04242c3e, 0x04243e1c, 0x04243e2c, 0x042c040c, - 0x042c043e, 0x042c1c14, 0x042c2c14, 0x04341c2c, 0x04343424, 0x043e0c04, - 0x043e0c24, 0x043e0c34, 0x043e241c, 0x043e340c, 0x0c04040c, 0x0c04041c, - 0x0c040c04, 0x0c040c14, 0x0c04140c, 0x0c04141c, 0x0c041c04, 0x0c041c14, - 0x0c041c24, 0x0c04243e, 0x0c042c04, 0x0c0c0404, 0x0c0c0414, 0x0c0c0c0c, - 0x0c0c1404, 0x0c0c1414, 0x0c14040c, 0x0c14041c, 0x0c140c04, 0x0c140c14, - 0x0c14140c, 0x0c141c04, 0x0c143e14, 0x0c1c0404, 0x0c1c0414, 0x0c1c1404, - 0x0c1c1c0c, 0x0c1c2434, 0x0c1c3434, 0x0c24040c, 0x0c24042c, 0x0c242c04, - 0x0c2c1404, 0x0c2c1424, 0x0c2c2434, 0x0c2c3e0c, 0x0c34042c, 0x0c3e1414, - 0x0c3e2404, 0x14040404, 0x14040414, 0x14040c0c, 0x14040c1c, 0x14041404, - 0x14041414, 0x14041434, 0x14041c0c, 0x14042414, 0x140c040c, 0x140c041c, - 0x140c042c, 0x140c0c04, 0x140c0c14, 0x140c140c, 0x140c1c04, 0x140c341c, - 0x140c343e, 0x140c3e04, 0x14140404, 0x14140414, 0x14140c0c, 0x14140c3e, - 0x14141404, 0x14141414, 0x14141c3e, 0x14142404, 0x14142c2c, 0x141c040c, - 0x141c0c04, 0x141c0c24, 0x141c3e04, 0x141c3e24, 0x14241c2c, 0x14242c1c, - 0x142c041c, 0x142c143e, 0x142c240c, 0x142c3e24, 0x143e040c, 0x143e041c, - 0x143e0c34, 0x143e242c, 0x1c04040c, 0x1c040c04, 0x1c040c14, 0x1c04140c, - 0x1c04141c, 0x1c042c04, 0x1c04342c, 0x1c043e14, 0x1c0c0404, 0x1c0c0414, - 0x1c0c1404, 0x1c0c1c0c, 0x1c0c2424, 0x1c0c2434, 0x1c14040c, 0x1c14041c, - 0x1c140c04, 0x1c14142c, 0x1c142c14, 0x1c143e14, 0x1c1c0c0c, 0x1c1c1c1c, - 0x1c241c04, 0x1c24243e, 0x1c243e14, 0x1c2c0404, 0x1c2c0434, 0x1c2c1414, - 0x1c2c2c2c, 0x1c340c24, 0x1c341c34, 0x1c34341c, 0x1c3e1c1c, 0x1c3e3404, - 0x24040424, 0x24040c3e, 0x24041c2c, 0x24041c3e, 0x24042c1c, 0x24042c3e, - 0x240c3e24, 0x24141404, 0x24141c3e, 0x24142404, 0x24143404, 0x24143434, - 0x241c043e, 0x241c242c, 0x24240424, 0x24242c0c, 0x24243424, 0x242c142c, - 0x242c241c, 0x242c3e04, 0x243e042c, 0x243e0c04, 0x243e0c14, 0x243e1c04, - 0x2c040c14, 0x2c04240c, 0x2c043e04, 0x2c0c0404, 0x2c0c0434, 0x2c0c1434, - 0x2c0c2c2c, 0x2c140c24, 0x2c141c14, 0x2c143e14, 0x2c1c0414, 0x2c1c2c1c, - 0x2c240c04, 0x2c24141c, 0x2c24143e, 0x2c243e14, 0x2c2c0414, 0x2c2c1c0c, - 0x2c342c04, 0x2c3e1424, 0x2c3e2414, 0x34041424, 0x34042424, 0x34042434, - 0x34043424, 0x340c140c, 0x340c340c, 0x34140c3e, 0x34143424, 0x341c1c04, - 0x341c1c34, 0x34242424, 0x342c042c, 0x342c2c14, 0x34341c1c, 0x343e041c, - 0x343e140c, 0x3e04041c, 0x3e04042c, 0x3e04043e, 0x3e040c04, 0x3e041c14, - 0x3e042c14, 0x3e0c1434, 0x3e0c2404, 0x3e140c14, 0x3e14242c, 0x3e142c14, - 0x3e1c0404, 0x3e1c0c2c, 0x3e1c1c1c, 0x3e1c3404, 0x3e24140c, 0x3e24240c, - 0x3e2c0404, 0x3e2c0414, 0x3e2c1424, 0x3e341c04, - }); - -static dpct::global_memory ksigns_iq2xs( - sycl::range<1>(128), - { - 0, 129, 130, 3, 132, 5, 6, 135, 136, 9, 10, 139, 12, - 141, 142, 15, 144, 17, 18, 147, 20, 149, 150, 23, 24, 153, - 154, 27, 156, 29, 30, 159, 160, 33, 34, 163, 36, 165, 166, - 39, 40, 169, 170, 43, 172, 45, 46, 175, 48, 177, 178, 51, - 180, 53, 54, 183, 184, 57, 58, 187, 60, 189, 190, 63, 192, - 65, 66, 195, 68, 197, 198, 71, 72, 201, 202, 75, 204, 77, - 78, 207, 80, 209, 210, 83, 212, 85, 86, 215, 216, 89, 90, - 219, 92, 221, 222, 95, 96, 225, 226, 99, 228, 101, 102, 231, - 232, 105, 106, 235, 108, 237, 238, 111, 240, 113, 114, 243, 116, - 245, 246, 119, 120, 249, 250, 123, 252, 125, 126, 255, - }); - -static dpct::global_memory - ksigns64(sycl::range<1>(128), - { - 0x0000000000000000, 0xff000000000000ff, 0xff0000000000ff00, - 0x000000000000ffff, 0xff00000000ff0000, 0x0000000000ff00ff, - 0x0000000000ffff00, 0xff00000000ffffff, 0xff000000ff000000, - 0x00000000ff0000ff, 0x00000000ff00ff00, 0xff000000ff00ffff, - 0x00000000ffff0000, 0xff000000ffff00ff, 0xff000000ffffff00, - 0x00000000ffffffff, 0xff0000ff00000000, 0x000000ff000000ff, - 0x000000ff0000ff00, 0xff0000ff0000ffff, 0x000000ff00ff0000, - 0xff0000ff00ff00ff, 0xff0000ff00ffff00, 0x000000ff00ffffff, - 0x000000ffff000000, 0xff0000ffff0000ff, 0xff0000ffff00ff00, - 0x000000ffff00ffff, 0xff0000ffffff0000, 0x000000ffffff00ff, - 0x000000ffffffff00, 0xff0000ffffffffff, 0xff00ff0000000000, - 0x0000ff00000000ff, 0x0000ff000000ff00, 0xff00ff000000ffff, - 0x0000ff0000ff0000, 0xff00ff0000ff00ff, 0xff00ff0000ffff00, - 0x0000ff0000ffffff, 0x0000ff00ff000000, 0xff00ff00ff0000ff, - 0xff00ff00ff00ff00, 0x0000ff00ff00ffff, 0xff00ff00ffff0000, - 0x0000ff00ffff00ff, 0x0000ff00ffffff00, 0xff00ff00ffffffff, - 0x0000ffff00000000, 0xff00ffff000000ff, 0xff00ffff0000ff00, - 0x0000ffff0000ffff, 0xff00ffff00ff0000, 0x0000ffff00ff00ff, - 0x0000ffff00ffff00, 0xff00ffff00ffffff, 0xff00ffffff000000, - 0x0000ffffff0000ff, 0x0000ffffff00ff00, 0xff00ffffff00ffff, - 0x0000ffffffff0000, 0xff00ffffffff00ff, 0xff00ffffffffff00, - 0x0000ffffffffffff, 0xffff000000000000, 0x00ff0000000000ff, - 0x00ff00000000ff00, 0xffff00000000ffff, 0x00ff000000ff0000, - 0xffff000000ff00ff, 0xffff000000ffff00, 0x00ff000000ffffff, - 0x00ff0000ff000000, 0xffff0000ff0000ff, 0xffff0000ff00ff00, - 0x00ff0000ff00ffff, 0xffff0000ffff0000, 0x00ff0000ffff00ff, - 0x00ff0000ffffff00, 0xffff0000ffffffff, 0x00ff00ff00000000, - 0xffff00ff000000ff, 0xffff00ff0000ff00, 0x00ff00ff0000ffff, - 0xffff00ff00ff0000, 0x00ff00ff00ff00ff, 0x00ff00ff00ffff00, - 0xffff00ff00ffffff, 0xffff00ffff000000, 0x00ff00ffff0000ff, - 0x00ff00ffff00ff00, 0xffff00ffff00ffff, 0x00ff00ffffff0000, - 0xffff00ffffff00ff, 0xffff00ffffffff00, 0x00ff00ffffffffff, - 0x00ffff0000000000, 0xffffff00000000ff, 0xffffff000000ff00, - 0x00ffff000000ffff, 0xffffff0000ff0000, 0x00ffff0000ff00ff, - 0x00ffff0000ffff00, 0xffffff0000ffffff, 0xffffff00ff000000, - 0x00ffff00ff0000ff, 0x00ffff00ff00ff00, 0xffffff00ff00ffff, - 0x00ffff00ffff0000, 0xffffff00ffff00ff, 0xffffff00ffffff00, - 0x00ffff00ffffffff, 0xffffffff00000000, 0x00ffffff000000ff, - 0x00ffffff0000ff00, 0xffffffff0000ffff, 0x00ffffff00ff0000, - 0xffffffff00ff00ff, 0xffffffff00ffff00, 0x00ffffff00ffffff, - 0x00ffffffff000000, 0xffffffffff0000ff, 0xffffffffff00ff00, - 0x00ffffffff00ffff, 0xffffffffffff0000, 0x00ffffffffff00ff, - 0x00ffffffffffff00, 0xffffffffffffffff, - }); -//#endif - -static dpct::global_memory - kmask_iq2xs(sycl::range<1>(8), {1, 2, 4, 8, 16, 32, 64, 128}); - template static void dequantize_block_iq2_xxs(const void * __restrict__ vx, dst_t * __restrict__ yy, const sycl::nd_item<3> &item_ct1, diff --git a/scripts/sync-ggml-am.sh b/scripts/sync-ggml-am.sh index 2c391e641..83e6359b1 100755 --- a/scripts/sync-ggml-am.sh +++ b/scripts/sync-ggml-am.sh @@ -94,6 +94,7 @@ if [ -f $SRC_LLAMA/ggml-src.patch ]; then # src/ggml-alloc.c -> ggml-alloc.c # src/ggml-backend-impl.h -> ggml-backend-impl.h # src/ggml-backend.c -> ggml-backend.c + # src/ggml-common.h -> ggml-common.h # src/ggml-cuda.cu -> ggml-cuda.cu # src/ggml-cuda.h -> ggml-cuda.h # src/ggml-impl.h -> ggml-impl.h @@ -126,6 +127,7 @@ if [ -f $SRC_LLAMA/ggml-src.patch ]; then -e 's/src\/ggml-alloc\.c/ggml-alloc.c/g' \ -e 's/src\/ggml-backend-impl\.h/ggml-backend-impl.h/g' \ -e 's/src\/ggml-backend\.c/ggml-backend.c/g' \ + -e 's/src\/ggml-common\.h/ggml-common.h/g' \ -e 's/src\/ggml-cuda\.cu/ggml-cuda.cu/g' \ -e 's/src\/ggml-cuda\.h/ggml-cuda.h/g' \ -e 's/src\/ggml-impl\.h/ggml-impl.h/g' \ diff --git a/scripts/sync-ggml.sh b/scripts/sync-ggml.sh index feb34bbc8..f1e6f0e57 100755 --- a/scripts/sync-ggml.sh +++ b/scripts/sync-ggml.sh @@ -4,6 +4,7 @@ cp -rpv ../ggml/src/ggml.c ./ggml.c cp -rpv ../ggml/src/ggml-alloc.c ./ggml-alloc.c cp -rpv ../ggml/src/ggml-backend-impl.h ./ggml-backend-impl.h cp -rpv ../ggml/src/ggml-backend.c ./ggml-backend.c +cp -rpv ../ggml/src/ggml-common.h ./ggml-common.h cp -rpv ../ggml/src/ggml-cuda.cu ./ggml-cuda.cu cp -rpv ../ggml/src/ggml-cuda.h ./ggml-cuda.h cp -rpv ../ggml/src/ggml-impl.h ./ggml-impl.h From 0db32beaf09d90b8959d3d0cc493ed1e45685353 Mon Sep 17 00:00:00 2001 From: Alexey Parfenov Date: Sat, 9 Mar 2024 11:16:53 +0000 Subject: [PATCH 568/812] server : fix passing prompt as tokens (#5955) * server: fix passing prompt as tokens * Update examples/server/server.cpp --------- Co-authored-by: Georgi Gerganov --- examples/server/server.cpp | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/examples/server/server.cpp b/examples/server/server.cpp index aedf0afc6..8cff514f2 100644 --- a/examples/server/server.cpp +++ b/examples/server/server.cpp @@ -852,7 +852,16 @@ struct server_context { // infill slot.params.input_prefix = json_value(data, "input_prefix", default_params.input_prefix); slot.params.input_suffix = json_value(data, "input_suffix", default_params.input_suffix); - slot.prompt = json_value(data, "prompt", std::string("")); + + // get prompt + { + const auto & prompt = data.find("prompt"); + if (prompt == data.end()) { + slot.prompt = ""; + } else { + slot.prompt = *prompt; + } + } // penalize user-provided tokens { From 2c4f566c88322ebf2f9bd11b01b5ebdaa0130b89 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Sat, 9 Mar 2024 14:17:11 +0200 Subject: [PATCH 569/812] tests : gitignore ggml-common.h --- tests/.gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/.gitignore b/tests/.gitignore index 9427cf13d..620a48ee4 100644 --- a/tests/.gitignore +++ b/tests/.gitignore @@ -1,3 +1,4 @@ * !*.* *.o +ggml-common.h From fb215c3832236fec7380c4fb618bd7154cb196ef Mon Sep 17 00:00:00 2001 From: SeungWon Jeong <65549245+redlion0929@users.noreply.github.com> Date: Sat, 9 Mar 2024 21:27:58 +0900 Subject: [PATCH 570/812] server : normalize embeddings (#5956) * output normalize embedding in '/v1/embeddings' * common : reuse llama_embd_normalize * common : better normalize impl --------- Co-authored-by: Georgi Gerganov --- common/common.cpp | 15 +++++++++++++++ common/common.h | 7 +++++++ examples/embedding/embedding.cpp | 14 +------------- examples/server/server.cpp | 8 +++++++- 4 files changed, 30 insertions(+), 14 deletions(-) diff --git a/common/common.cpp b/common/common.cpp index d7f650ef4..16ef4d7f7 100644 --- a/common/common.cpp +++ b/common/common.cpp @@ -1852,3 +1852,18 @@ void dump_kv_cache_view_seqs(const llama_kv_cache_view & view, int row_size) { printf("\n=== Done dumping\n"); } + +void llama_embd_normalize(const float * inp, float * out, int n) { + double sum = 0.0; + for (int i = 0; i < n; i++) { + sum += inp[i] * inp[i]; + } + sum = sqrt(sum); + + const float norm = sum > 0.0 ? 1.0f / sum : 0.0f; + + for (int i = 0; i < n; i++) { + out[i] = inp[i] * norm; + } +} + diff --git a/common/common.h b/common/common.h index 977ce419f..f8d82b871 100644 --- a/common/common.h +++ b/common/common.h @@ -260,3 +260,10 @@ void dump_kv_cache_view(const llama_kv_cache_view & view, int row_size = 80); // Dump the KV cache view showing individual sequences in each cell (long output). void dump_kv_cache_view_seqs(const llama_kv_cache_view & view, int row_size = 40); + +// +// Embedding utils +// + +void llama_embd_normalize(const float * inp, float * out, int n); + diff --git a/examples/embedding/embedding.cpp b/examples/embedding/embedding.cpp index ff5883da6..a553ae1c3 100644 --- a/examples/embedding/embedding.cpp +++ b/examples/embedding/embedding.cpp @@ -23,17 +23,6 @@ static void batch_add_seq(llama_batch & batch, const std::vector & toke } } -static void normalize(const float * vec, float * out, int n) { - float norm = 0; - for (int i = 0; i < n; i++) { - norm += vec[i] * vec[i]; - } - norm = sqrt(norm); - for (int i = 0; i < n; i++) { - out[i] = vec[i] / norm; - } -} - static void batch_decode(llama_context * ctx, llama_batch & batch, float * output, int n_seq, int n_embd) { // clear previous kv_cache values (irrelevant for embeddings) llama_kv_cache_clear(ctx); @@ -44,7 +33,6 @@ static void batch_decode(llama_context * ctx, llama_batch & batch, float * outpu fprintf(stderr, "%s : failed to decode\n", __func__); } - // normalize on copy for (int i = 0; i < batch.n_tokens; i++) { if (!batch.logits[i]) { continue; @@ -61,7 +49,7 @@ static void batch_decode(llama_context * ctx, llama_batch & batch, float * outpu } float * out = output + batch.seq_id[i][0] * n_embd; - normalize(embd, out, n_embd); + llama_embd_normalize(embd, out, n_embd); } } diff --git a/examples/server/server.cpp b/examples/server/server.cpp index 8cff514f2..796f3499c 100644 --- a/examples/server/server.cpp +++ b/examples/server/server.cpp @@ -1327,6 +1327,8 @@ struct server_context { const int n_embd = llama_n_embd(model); + std::vector embd_res(n_embd, 0.0f); + for (int i = 0; i < batch.n_tokens; ++i) { if (!batch.logits[i] || batch.seq_id[i][0] != slot.id + 1) { continue; @@ -1350,8 +1352,10 @@ struct server_context { continue; } + llama_embd_normalize(embd, embd_res.data(), n_embd); + res.data = json { - {"embedding", std::vector(embd, embd + n_embd)}, + {"embedding", embd_res}, }; } @@ -3354,6 +3358,8 @@ int main(int argc, char ** argv) { // get the result server_task_result result = ctx_server.queue_results.recv(id_task); ctx_server.queue_results.remove_waiting_task_id(id_task); + + // append to the responses responses.push_back(result.data); } From 97c09585d65a95864773b4d25d66d0f708baf38d Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Sat, 9 Mar 2024 15:47:47 +0200 Subject: [PATCH 571/812] server : clarify some items in the readme (#5957) * server : clarify some items in the readme * server : fix typo --- examples/server/README.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/examples/server/README.md b/examples/server/README.md index 3abb1abe3..23606b32a 100644 --- a/examples/server/README.md +++ b/examples/server/README.md @@ -195,7 +195,11 @@ node index.js *Options:* - `prompt`: Provide the prompt for this completion as a string or as an array of strings or numbers representing tokens. Internally, the prompt is compared to the previous completion and only the "unseen" suffix is evaluated. If the prompt is a string or an array with the first element given as a string, a `bos` token is inserted in the front like `main` does. + `prompt`: Provide the prompt for this completion as a string or as an array of strings or numbers representing tokens. Internally, if `cache_prompt` is `true`, the prompt is compared to the previous completion and only the "unseen" suffix is evaluated. A `BOS` token is inserted at the start, if all of the following conditions are true: + + - The prompt is a string or an array with the first element given as a string + - The model's `tokenizer.ggml.add_bos_token` metadata is `true` + - The system prompt is empty `temperature`: Adjust the randomness of the generated text (default: 0.8). @@ -308,7 +312,7 @@ Notice that each `probs` is an array of length `n_probs`. `content`: Set the text to tokenize. - Note that the special `BOS` token is not added in front of the text and also a space character is not inserted automatically as it is for `/completion`. + Note that a special `BOS` token is never inserted. - **POST** `/detokenize`: Convert tokens to text. From 5b09797321430f08caf0473143a962916ab2ea89 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Sat, 9 Mar 2024 15:53:59 +0200 Subject: [PATCH 572/812] ggml : remove old quantization functions (#5942) * ggml : remove old quantization functions ggml-ci * ggml : simplify ggml_quantize_chunk ggml-ci * ggml : restrict correctness ggml-ci * ggml : remove hist data from the quantization API ggml-ci * tests : remove hist usage in test-backend-ops ggml-ci * vulkan : remove hist and fix typo --- examples/benchmark/benchmark-matmult.cpp | 6 +- examples/llava/clip.cpp | 55 +--- ggml-quants.c | 143 +++------- ggml-quants.h | 42 +-- ggml-vulkan.cpp | 40 +-- ggml.c | 339 +++-------------------- ggml.h | 23 +- llama.cpp | 48 +--- tests/test-backend-ops.cpp | 3 +- 9 files changed, 131 insertions(+), 568 deletions(-) diff --git a/examples/benchmark/benchmark-matmult.cpp b/examples/benchmark/benchmark-matmult.cpp index e89f3de2f..47cb16c69 100644 --- a/examples/benchmark/benchmark-matmult.cpp +++ b/examples/benchmark/benchmark-matmult.cpp @@ -189,12 +189,10 @@ int main(int argc, char ** argv) { int32_t nelements = sizex*sizey; - std::vector hist_cur(1 << 4, 0); - // Set up a the benchmark matrices // printf("Creating new tensor q11 & Running quantize\n"); struct ggml_tensor * q11 = ggml_new_tensor_2d(ctx, qtype, sizex, sizey); - ggml_quantize_chunk(qtype, (const float *) m11->data, q11->data, 0, nelements/m11->ne[0], m11->ne[0], hist_cur.data(), nullptr); + ggml_quantize_chunk(qtype, (const float *) m11->data, q11->data, 0, nelements/m11->ne[0], m11->ne[0], nullptr); // Set up a the compute graph // printf("Creating new tensor q31\n"); @@ -207,7 +205,7 @@ int main(int argc, char ** argv) { // Set up a second graph computation to make sure we override the CPU cache lines // printf("Creating new tensor q12 & Running quantize\n"); struct ggml_tensor * q12 = ggml_new_tensor_2d(ctx, qtype, sizex, sizey); - ggml_quantize_chunk(qtype, (const float *) m12->data, q12->data, 0, nelements/m12->ne[0], m12->ne[0], hist_cur.data(), nullptr); + ggml_quantize_chunk(qtype, (const float *) m12->data, q12->data, 0, nelements/m12->ne[0], m12->ne[0], nullptr); // printf("Creating new tensor q32\n"); struct ggml_tensor * q32 = ggml_mul_mat(ctx, q12, m2); diff --git a/examples/llava/clip.cpp b/examples/llava/clip.cpp index ef9e4ba7a..6653b815d 100644 --- a/examples/llava/clip.cpp +++ b/examples/llava/clip.cpp @@ -1862,7 +1862,6 @@ bool clip_model_quantize(const char * fname_inp, const char * fname_out, const i std::vector work(512); std::vector conv_buf(512); - std::vector hist_all(1 << 4, 0); size_t total_size_org = 0; size_t total_size_new = 0; @@ -1917,48 +1916,7 @@ bool clip_model_quantize(const char * fname_inp, const char * fname_out, const i } new_data = work.data(); - std::vector hist_cur(1 << 4, 0); - - switch (new_type) { - case GGML_TYPE_Q4_0: { - new_size = ggml_quantize_q4_0(f32_data, new_data, n_elms, cur->ne[0], hist_cur.data()); - } break; - case GGML_TYPE_Q4_1: { - new_size = ggml_quantize_q4_1(f32_data, new_data, n_elms, cur->ne[0], hist_cur.data()); - } break; - case GGML_TYPE_Q5_0: { - new_size = ggml_quantize_q5_0(f32_data, new_data, n_elms, cur->ne[0], hist_cur.data()); - } break; - case GGML_TYPE_Q5_1: { - new_size = ggml_quantize_q5_1(f32_data, new_data, n_elms, cur->ne[0], hist_cur.data()); - } break; - case GGML_TYPE_Q8_0: { - new_size = ggml_quantize_q8_0(f32_data, new_data, n_elms, cur->ne[0], hist_cur.data()); - } break; - case GGML_TYPE_Q2_K: { - new_size = ggml_quantize_q2_K(f32_data, new_data, n_elms, cur->ne[0], hist_cur.data()); - } break; - case GGML_TYPE_Q3_K: { - new_size = ggml_quantize_q3_K(f32_data, new_data, n_elms, cur->ne[0], hist_cur.data()); - } break; - case GGML_TYPE_Q4_K: { - new_size = ggml_quantize_q4_K(f32_data, new_data, n_elms, cur->ne[0], hist_cur.data()); - } break; - case GGML_TYPE_Q5_K: { - new_size = ggml_quantize_q5_K(f32_data, new_data, n_elms, cur->ne[0], hist_cur.data()); - } break; - case GGML_TYPE_Q6_K: { - new_size = ggml_quantize_q6_K(f32_data, new_data, n_elms, cur->ne[0], hist_cur.data()); - } break; - default: { - fprintf(stderr, "%s: unsupported quantization type %d\n", __func__, new_type); - return false; - } - } - - for (size_t j = 0; j < hist_cur.size(); ++j) { - hist_all[j] += hist_cur[j]; - } + new_size = ggml_quantize_chunk(new_type, f32_data, new_data, 0, n_elms/cur->ne[0], cur->ne[0], nullptr); } else { new_type = cur->type; new_data = cur->data; @@ -1993,17 +1951,6 @@ bool clip_model_quantize(const char * fname_inp, const char * fname_out, const i { printf("%s: original size = %8.2f MB\n", __func__, total_size_org / 1024.0 / 1024.0); printf("%s: quantized size = %8.2f MB\n", __func__, total_size_new / 1024.0 / 1024.0); - - int64_t sum_all = 0; - for (size_t i = 0; i < hist_all.size(); ++i) { - sum_all += hist_all[i]; - } - - printf("%s: hist: ", __func__); - for (size_t i = 0; i < hist_all.size(); ++i) { - printf("%5.3f ", hist_all[i] / (float)sum_all); - } - printf("\n"); } return true; diff --git a/ggml-quants.c b/ggml-quants.c index 5bb46def3..4ee4e0604 100644 --- a/ggml-quants.c +++ b/ggml-quants.c @@ -1704,16 +1704,6 @@ void quantize_row_q2_K(const float * restrict x, void * restrict vy, int k) { quantize_row_q2_K_reference(x, vy, k); } -size_t ggml_quantize_q2_K(const float * restrict src, void * restrict dst, int n, int k, int64_t * restrict hist) { - (void)hist; // TODO: collect histograms - - for (int j = 0; j < n; j += k) { - block_q2_K * restrict y = (block_q2_K *)dst + j/QK_K; - quantize_row_q2_K_reference(src + j, y, k); - } - return (n/QK_K*sizeof(block_q2_K)); -} - static float make_qkx3_quants(int n, int nmax, const float * restrict x, const float * restrict weights, uint8_t * restrict L, float * restrict the_min, uint8_t * restrict Laux, float rmin, float rdelta, int nstep, bool use_mad) { @@ -1966,8 +1956,7 @@ static void quantize_row_q2_K_impl(const float * restrict x, block_q2_K * restri } } -size_t quantize_q2_K(const float * src, void * dst, int nrow, int n_per_row, int64_t * hist, const float * quant_weights) { - (void)hist; +size_t quantize_q2_K(const float * restrict src, void * restrict dst, int nrow, int n_per_row, const float * quant_weights) { size_t row_size = ggml_row_size(GGML_TYPE_Q2_K, n_per_row); if (!quant_weights) { quantize_row_q2_K_reference(src, dst, nrow*n_per_row); @@ -2186,16 +2175,6 @@ void quantize_row_q3_K(const float * restrict x, void * restrict vy, int k) { quantize_row_q3_K_reference(x, vy, k); } -size_t ggml_quantize_q3_K(const float * restrict src, void * restrict dst, int n, int k, int64_t * restrict hist) { - (void)hist; // TODO: collect histograms - - for (int j = 0; j < n; j += k) { - block_q3_K * restrict y = (block_q3_K *)dst + j/QK_K; - quantize_row_q3_K_reference(src + j, y, k); - } - return (n/QK_K*sizeof(block_q3_K)); -} - static void quantize_row_q3_K_impl(const float * restrict x, block_q3_K * restrict y, int n_per_row, const float * restrict quant_weights) { #if QK_K != 256 (void)quant_weights; @@ -2285,8 +2264,7 @@ static void quantize_row_q3_K_impl(const float * restrict x, block_q3_K * restri #endif } -size_t quantize_q3_K(const float * src, void * dst, int nrow, int n_per_row, int64_t * hist, const float * quant_weights) { - (void)hist; +size_t quantize_q3_K(const float * restrict src, void * restrict dst, int nrow, int n_per_row, const float * quant_weights) { size_t row_size = ggml_row_size(GGML_TYPE_Q3_K, n_per_row); if (!quant_weights) { quantize_row_q3_K_reference(src, dst, nrow*n_per_row); @@ -2456,17 +2434,6 @@ void quantize_row_q4_K(const float * restrict x, void * restrict vy, int k) { quantize_row_q4_K_reference(x, y, k); } -size_t ggml_quantize_q4_K(const float * restrict src, void * restrict dst, int n, int k, int64_t * restrict hist) { - assert(k % QK_K == 0); - (void)hist; // TODO: collect histograms - - for (int j = 0; j < n; j += k) { - block_q4_K * restrict y = (block_q4_K *)dst + j/QK_K; - quantize_row_q4_K_reference(src + j, y, k); - } - return (n/QK_K*sizeof(block_q4_K)); -} - static void quantize_row_q4_K_impl(const float * restrict x, block_q4_K * restrict y, int n_per_row, const float * quant_weights) { #if QK_K != 256 (void)quant_weights; @@ -2545,8 +2512,7 @@ static void quantize_row_q4_K_impl(const float * restrict x, block_q4_K * restri #endif } -size_t quantize_q4_K(const float * src, void * dst, int nrow, int n_per_row, int64_t * hist, const float * quant_weights) { - (void)hist; +size_t quantize_q4_K(const float * restrict src, void * restrict dst, int nrow, int n_per_row, const float * quant_weights) { size_t row_size = ggml_row_size(GGML_TYPE_Q4_K, n_per_row); if (!quant_weights) { quantize_row_q4_K_reference(src, dst, nrow*n_per_row); @@ -2757,17 +2723,6 @@ void quantize_row_q5_K(const float * restrict x, void * restrict vy, int k) { quantize_row_q5_K_reference(x, y, k); } -size_t ggml_quantize_q5_K(const float * restrict src, void * restrict dst, int n, int k, int64_t * restrict hist) { - assert(k % QK_K == 0); - (void)hist; // TODO: collect histograms - - for (int j = 0; j < n; j += k) { - block_q5_K * restrict y = (block_q5_K *)dst + j/QK_K; - quantize_row_q5_K_reference(src + j, y, k); - } - return (n/QK_K*sizeof(block_q5_K)); -} - static void quantize_row_q5_K_impl(const float * restrict x, block_q5_K * restrict y, int n_per_row, const float * quant_weights) { #if QK_K != 256 (void)quant_weights; @@ -2866,8 +2821,7 @@ static void quantize_row_q5_K_impl(const float * restrict x, block_q5_K * restri #endif } -size_t quantize_q5_K(const float * src, void * dst, int nrow, int n_per_row, int64_t * hist, const float * quant_weights) { - (void)hist; +size_t quantize_q5_K(const float * restrict src, void * restrict dst, int nrow, int n_per_row, const float * quant_weights) { size_t row_size = ggml_row_size(GGML_TYPE_Q5_K, n_per_row); if (!quant_weights) { quantize_row_q5_K_reference(src, dst, nrow*n_per_row); @@ -3020,17 +2974,6 @@ void quantize_row_q6_K(const float * restrict x, void * restrict vy, int k) { quantize_row_q6_K_reference(x, y, k); } -size_t ggml_quantize_q6_K(const float * src, void * dst, int n, int k, int64_t * hist) { - assert(k % QK_K == 0); - (void)hist; // TODO: collect histograms - - for (int j = 0; j < n; j += k) { - block_q6_K * restrict y = (block_q6_K *)dst + j/QK_K; - quantize_row_q6_K_reference(src + j, y, k); - } - return (n/QK_K*sizeof(block_q6_K)); -} - static void quantize_row_q6_K_impl(const float * restrict x, block_q6_K * restrict y, int n_per_row, const float * quant_weights) { #if QK_K != 256 (void)quant_weights; @@ -3120,8 +3063,7 @@ static void quantize_row_q6_K_impl(const float * restrict x, block_q6_K * restri #endif } -size_t quantize_q6_K(const float * src, void * dst, int nrow, int n_per_row, int64_t * hist, const float * quant_weights) { - (void)hist; +size_t quantize_q6_K(const float * restrict src, void * restrict dst, int nrow, int n_per_row, const float * quant_weights) { size_t row_size = ggml_row_size(GGML_TYPE_Q6_K, n_per_row); if (!quant_weights) { quantize_row_q6_K_reference(src, dst, nrow*n_per_row); @@ -3165,9 +3107,10 @@ static void quantize_row_q4_0_impl(const float * restrict x, block_q4_0 * restri } } -size_t quantize_q4_0(const float * src, void * dst, int nrow, int n_per_row, int64_t * hist, const float * quant_weights) { +size_t quantize_q4_0(const float * restrict src, void * restrict dst, int nrow, int n_per_row, const float * quant_weights) { if (!quant_weights) { - return ggml_quantize_q4_0(src, dst, nrow*n_per_row, n_per_row, hist); + quantize_row_q4_0_reference(src, dst, nrow*n_per_row); + return nrow * ggml_row_size(GGML_TYPE_Q4_0, n_per_row); } size_t row_size = ggml_row_size(GGML_TYPE_Q4_0, n_per_row); char * qrow = (char *)dst; @@ -3209,9 +3152,10 @@ static void quantize_row_q4_1_impl(const float * restrict x, block_q4_1 * restri } } -size_t quantize_q4_1(const float * src, void * dst, int nrow, int n_per_row, int64_t * hist, const float * quant_weights) { +size_t quantize_q4_1(const float * restrict src, void * restrict dst, int nrow, int n_per_row, const float * quant_weights) { if (!quant_weights) { - return ggml_quantize_q4_1(src, dst, nrow*n_per_row, n_per_row, hist); + quantize_row_q4_1_reference(src, dst, nrow*n_per_row); + return nrow * ggml_row_size(GGML_TYPE_Q4_1, n_per_row); } size_t row_size = ggml_row_size(GGML_TYPE_Q4_1, n_per_row); char * qrow = (char *)dst; @@ -3262,9 +3206,10 @@ static void quantize_row_q5_0_impl(const float * restrict x, block_q5_0 * restri } } -size_t quantize_q5_0(const float * src, void * dst, int nrow, int n_per_row, int64_t * hist, const float * quant_weights) { +size_t quantize_q5_0(const float * restrict src, void * restrict dst, int nrow, int n_per_row, const float * quant_weights) { if (!quant_weights) { - return ggml_quantize_q5_0(src, dst, nrow*n_per_row, n_per_row, hist); + quantize_row_q5_0_reference(src, dst, nrow*n_per_row); + return nrow * ggml_row_size(GGML_TYPE_Q5_0, n_per_row); } size_t row_size = ggml_row_size(GGML_TYPE_Q5_0, n_per_row); char * qrow = (char *)dst; @@ -3314,9 +3259,10 @@ static void quantize_row_q5_1_impl(const float * restrict x, block_q5_1 * restri } } -size_t quantize_q5_1(const float * src, void * dst, int nrow, int n_per_row, int64_t * hist, const float * quant_weights) { +size_t quantize_q5_1(const float * restrict src, void * restrict dst, int nrow, int n_per_row, const float * quant_weights) { if (!quant_weights) { - return ggml_quantize_q5_1(src, dst, nrow*n_per_row, n_per_row, hist); + quantize_row_q5_1_reference(src, dst, nrow*n_per_row); + return nrow * ggml_row_size(GGML_TYPE_Q5_1, n_per_row); } size_t row_size = ggml_row_size(GGML_TYPE_Q5_1, n_per_row); char * qrow = (char *)dst; @@ -3328,6 +3274,13 @@ size_t quantize_q5_1(const float * src, void * dst, int nrow, int n_per_row, int return nrow * row_size; } +size_t quantize_q8_0(const float * restrict src, void * restrict dst, int nrow, int n_per_row, const float * quant_weights) { + (void)quant_weights; // not used + const size_t row_size = ggml_row_size(GGML_TYPE_Q8_0, n_per_row); + quantize_row_q8_0_reference(src, dst, nrow*n_per_row); + return nrow * row_size; +} + // ====================== "True" 2-bit (de)-quantization void dequantize_row_iq2_xxs(const block_iq2_xxs * restrict x, float * restrict y, int k) { @@ -9373,7 +9326,7 @@ void ggml_vec_dot_iq3_xxs_q8_K(int n, float * restrict s, size_t bs, const void #endif } -void ggml_vec_dot_iq3_s_q8_K (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +void ggml_vec_dot_iq3_s_q8_K (int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) { assert(n % QK_K == 0); assert(nrc == 1); UNUSED(nrc); @@ -9620,7 +9573,7 @@ static inline __m256i mul_add_epi8(const __m256i x, const __m256i y) { } #endif -void ggml_vec_dot_iq1_s_q8_K (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +void ggml_vec_dot_iq1_s_q8_K (int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) { assert(n % QK_K == 0); assert(nrc == 1); UNUSED(nrc); @@ -10220,7 +10173,7 @@ void iq2xs_init_impl(enum ggml_type type) { int * kmap_q2xs; uint16_t * kneighbors_q2xs; - printf("================================================================= %s(grid_size = %d)\n", __func__, grid_size); + //printf("================================================================= %s(grid_size = %d)\n", __func__, grid_size); uint64_t * the_grid = (uint64_t *)malloc(grid_size*sizeof(uint64_t)); for (int k = 0; k < grid_size; ++k) { int8_t * pos = (int8_t *)(the_grid + k); @@ -10275,7 +10228,7 @@ void iq2xs_init_impl(enum ggml_type type) { } num_neighbors += n; } - printf("%s: %d neighbours in total\n", __func__, num_neighbors); + //printf("%s: %d neighbours in total\n", __func__, num_neighbors); kneighbors_q2xs = (uint16_t *)malloc((num_neighbors + num_not_in_map)*sizeof(uint16_t)); iq2_data[gindex].neighbours = kneighbors_q2xs; int counter = 0; @@ -10698,8 +10651,7 @@ static void quantize_row_iq2_xs_impl(const float * restrict x, void * restrict v } } -size_t quantize_iq2_xxs(const float * src, void * dst, int nrow, int n_per_row, int64_t * hist, const float * quant_weights) { - (void)hist; +size_t quantize_iq2_xxs(const float * restrict src, void * restrict dst, int nrow, int n_per_row, const float * quant_weights) { GGML_ASSERT(n_per_row%QK_K == 0); int nblock = n_per_row/QK_K; char * qrow = (char *)dst; @@ -10711,8 +10663,7 @@ size_t quantize_iq2_xxs(const float * src, void * dst, int nrow, int n_per_row, return nrow * nblock * sizeof(block_iq2_xxs); } -size_t quantize_iq2_xs(const float * src, void * dst, int nrow, int n_per_row, int64_t * hist, const float * quant_weights) { - (void)hist; +size_t quantize_iq2_xs(const float * restrict src, void * restrict dst, int nrow, int n_per_row, const float * quant_weights) { GGML_ASSERT(n_per_row%QK_K == 0); int nblock = n_per_row/QK_K; char * qrow = (char *)dst; @@ -10816,7 +10767,7 @@ void iq3xs_init_impl(int grid_size) { int * kmap_q3xs; uint16_t * kneighbors_q3xs; - printf("================================================================= %s(grid_size = %d)\n", __func__, grid_size); + //printf("================================================================= %s(grid_size = %d)\n", __func__, grid_size); uint32_t * the_grid = (uint32_t *)malloc(grid_size*sizeof(uint32_t)); for (int k = 0; k < grid_size; ++k) { int8_t * pos = (int8_t *)(the_grid + k); @@ -10871,7 +10822,7 @@ void iq3xs_init_impl(int grid_size) { } num_neighbors += n; } - printf("%s: %d neighbours in total\n", __func__, num_neighbors); + //printf("%s: %d neighbours in total\n", __func__, num_neighbors); kneighbors_q3xs = (uint16_t *)malloc((num_neighbors + num_not_in_map)*sizeof(uint16_t)); iq3_data[gindex].neighbours = kneighbors_q3xs; int counter = 0; @@ -11154,8 +11105,7 @@ static void quantize_row_iq3_xxs_impl(int grid_size, const float * restrict x, v } } -size_t quantize_iq3_xxs(const float * src, void * dst, int nrow, int n_per_row, int64_t * hist, const float * quant_weights) { - (void)hist; +size_t quantize_iq3_xxs(const float * restrict src, void * restrict dst, int nrow, int n_per_row, const float * quant_weights) { GGML_ASSERT(n_per_row%QK_K == 0); int nblock = n_per_row/QK_K; char * qrow = (char *)dst; @@ -11361,8 +11311,7 @@ static void quantize_row_iq3_s_impl(int block_size, const float * restrict x, vo } #define IQ3S_BLOCK_SIZE 32 -size_t quantize_iq3_s(const float * src, void * dst, int nrow, int n_per_row, int64_t * hist, const float * quant_weights) { - (void)hist; +size_t quantize_iq3_s(const float * restrict src, void * restrict dst, int nrow, int n_per_row, const float * quant_weights) { GGML_ASSERT(n_per_row%QK_K == 0); int nblock = n_per_row/QK_K; float scales[QK_K/IQ3S_BLOCK_SIZE]; @@ -11392,7 +11341,7 @@ void quantize_row_iq3_s(const float * restrict x, void * restrict vy, int k) { void quantize_row_iq3_s_reference(const float * restrict x, block_iq3_s * restrict y, int k) { assert(k % QK_K == 0); - quantize_iq3_s(x, y, 1, k, NULL, NULL); + quantize_iq3_s(x, y, 1, k, NULL); } @@ -11587,8 +11536,7 @@ static void quantize_row_iq1_s_impl(const float * restrict x, void * restrict vy } } -size_t quantize_iq1_s(const float * src, void * dst, int nrow, int n_per_row, int64_t * hist, const float * quant_weights) { - (void)hist; +size_t quantize_iq1_s(const float * restrict src, void * restrict dst, int nrow, int n_per_row, const float * quant_weights) { GGML_ASSERT(n_per_row%QK_K == 0); int nblock = n_per_row/QK_K; char * qrow = (char *)dst; @@ -11613,7 +11561,7 @@ static inline int best_index_int8(int n, const int8_t * val, float x) { return x - val[mu-1] < val[mu] - x ? mu-1 : mu; } -static void quantize_row_iq4_nl_impl(const int super_block_size, const int block_size, const float * GGML_RESTRICT x, +static void quantize_row_iq4_nl_impl(const int super_block_size, const int block_size, const float * restrict x, ggml_fp16_t * dh, uint8_t * q4, uint16_t * scales_h, uint8_t * scales_l, float * scales, float * weight, uint8_t * L, const int8_t * values, @@ -11721,8 +11669,7 @@ static void quantize_row_iq4_nl_impl(const int super_block_size, const int block } } -size_t quantize_iq4_nl(const float * src, void * dst, int nrow, int n_per_row, int64_t * hist, const float * quant_weights) { - (void)hist; +size_t quantize_iq4_nl(const float * restrict src, void * restrict dst, int nrow, int n_per_row, const float * quant_weights) { GGML_ASSERT(n_per_row%QK4_NL == 0); int nblock = n_per_row/QK4_NL; char * qrow = (char *)dst; @@ -11752,14 +11699,13 @@ void quantize_row_iq4_nl(const float * restrict x, void * restrict vy, int k) { void quantize_row_iq4_nl_reference(const float * restrict x, block_iq4_nl * restrict y, int k) { assert(k % QK4_NL == 0); - quantize_iq4_nl(x, y, 1, k, NULL, NULL); + quantize_iq4_nl(x, y, 1, k, NULL); } -size_t quantize_iq4_xs(const float * src, void * dst, int nrow, int n_per_row, int64_t * hist, const float * quant_weights) { +size_t quantize_iq4_xs(const float * restrict src, void * restrict dst, int nrow, int n_per_row, const float * quant_weights) { #if QK_K == 64 - return quantize_iq4_nl(src, dst, nrow, n_per_row, hist, quant_weights); + return quantize_iq4_nl(src, dst, nrow, n_per_row, quant_weights); #else - (void)hist; GGML_ASSERT(n_per_row%QK_K == 0); int nblock = n_per_row/QK_K; char * qrow = (char *)dst; @@ -11788,7 +11734,7 @@ void quantize_row_iq4_xs(const float * restrict x, void * restrict vy, int k) { void quantize_row_iq4_xs_reference(const float * restrict x, block_iq4_xs * restrict y, int k) { assert(k % QK_K == 0); - quantize_iq4_xs(x, y, 1, k, NULL, NULL); + quantize_iq4_xs(x, y, 1, k, NULL); } // =============================== 2.5625 bpw @@ -11961,8 +11907,7 @@ static void quantize_row_iq2_s_impl(const float * restrict x, void * restrict vy } } -size_t quantize_iq2_s(const float * src, void * dst, int nrow, int n_per_row, int64_t * hist, const float * quant_weights) { - (void)hist; +size_t quantize_iq2_s(const float * restrict src, void * restrict dst, int nrow, int n_per_row, const float * quant_weights) { GGML_ASSERT(n_per_row%QK_K == 0); int nblock = n_per_row/QK_K; char * qrow = (char *)dst; @@ -11976,7 +11921,7 @@ size_t quantize_iq2_s(const float * src, void * dst, int nrow, int n_per_row, in void quantize_row_iq2_s_reference(const float * restrict x, block_iq2_s * restrict y, int k) { assert(k % QK_K == 0); - quantize_iq2_s(x, y, 1, k, NULL, NULL); + quantize_iq2_s(x, y, 1, k, NULL); } void quantize_row_iq2_s(const float * restrict x, void * restrict vy, int k) { diff --git a/ggml-quants.h b/ggml-quants.h index d2d25eb44..47dd52856 100644 --- a/ggml-quants.h +++ b/ggml-quants.h @@ -261,6 +261,7 @@ void quantize_row_q4_K_reference(const float * GGML_RESTRICT x, block_q4_K * GGM void quantize_row_q5_K_reference(const float * GGML_RESTRICT x, block_q5_K * GGML_RESTRICT y, int k); void quantize_row_q6_K_reference(const float * GGML_RESTRICT x, block_q6_K * GGML_RESTRICT y, int k); void quantize_row_q8_K_reference(const float * GGML_RESTRICT x, block_q8_K * GGML_RESTRICT y, int k); + void quantize_row_iq3_xxs_reference(const float * GGML_RESTRICT x, block_iq3_xxs * GGML_RESTRICT y, int k); void quantize_row_iq4_nl_reference (const float * GGML_RESTRICT x, block_iq4_nl * GGML_RESTRICT y, int k); void quantize_row_iq4_xs_reference (const float * GGML_RESTRICT x, block_iq4_xs * GGML_RESTRICT y, int k); @@ -280,6 +281,7 @@ void quantize_row_q4_K(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, in void quantize_row_q5_K(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int k); void quantize_row_q6_K(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int k); void quantize_row_q8_K(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int k); + void quantize_row_iq3_xxs(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int k); void quantize_row_iq4_nl (const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int k); void quantize_row_iq4_xs (const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int k); @@ -300,6 +302,7 @@ void dequantize_row_q4_K(const block_q4_K * GGML_RESTRICT x, float * GGML_RESTRI void dequantize_row_q5_K(const block_q5_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int k); void dequantize_row_q6_K(const block_q6_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int k); void dequantize_row_q8_K(const block_q8_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int k); + void dequantize_row_iq2_xxs(const block_iq2_xxs * GGML_RESTRICT x, float * GGML_RESTRICT y, int k); void dequantize_row_iq2_xs (const block_iq2_xs * GGML_RESTRICT x, float * GGML_RESTRICT y, int k); void dequantize_row_iq2_s (const block_iq2_s * GGML_RESTRICT x, float * GGML_RESTRICT y, int k); @@ -321,6 +324,7 @@ void ggml_vec_dot_q3_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voi void ggml_vec_dot_q4_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); void ggml_vec_dot_q5_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); void ggml_vec_dot_q6_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); + void ggml_vec_dot_iq2_xxs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); void ggml_vec_dot_iq2_xs_q8_K (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); void ggml_vec_dot_iq2_s_q8_K (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); @@ -330,26 +334,26 @@ void ggml_vec_dot_iq4_nl_q8_0 (int n, float * GGML_RESTRICT s, size_t bs, const void ggml_vec_dot_iq4_xs_q8_K (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); void ggml_vec_dot_iq3_s_q8_K (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); -// // Quantization utilizing an importance matrix (a.k.a. "Activation aWare Quantization") -// -size_t quantize_iq2_xxs(const float * src, void * dst, int nrows, int n_per_row, int64_t * hist, const float * imatrix); -size_t quantize_iq2_xs (const float * src, void * dst, int nrows, int n_per_row, int64_t * hist, const float * imatrix); -size_t quantize_iq2_s (const float * src, void * dst, int nrows, int n_per_row, int64_t * hist, const float * imatrix); -size_t quantize_iq3_xxs(const float * src, void * dst, int nrows, int n_per_row, int64_t * hist, const float * imatrix); -size_t quantize_iq1_s (const float * src, void * dst, int nrows, int n_per_row, int64_t * hist, const float * imatrix); -size_t quantize_iq4_nl (const float * src, void * dst, int nrows, int n_per_row, int64_t * hist, const float * imatrix); -size_t quantize_iq4_xs (const float * src, void * dst, int nrows, int n_per_row, int64_t * hist, const float * imatrix); -size_t quantize_iq3_s (const float * src, void * dst, int nrows, int n_per_row, int64_t * hist, const float * imatrix); -size_t quantize_q2_K (const float * src, void * dst, int nrows, int n_per_row, int64_t * hist, const float * imatrix); -size_t quantize_q3_K (const float * src, void * dst, int nrows, int n_per_row, int64_t * hist, const float * imatrix); -size_t quantize_q4_K (const float * src, void * dst, int nrows, int n_per_row, int64_t * hist, const float * imatrix); -size_t quantize_q5_K (const float * src, void * dst, int nrows, int n_per_row, int64_t * hist, const float * imatrix); -size_t quantize_q6_K (const float * src, void * dst, int nrows, int n_per_row, int64_t * hist, const float * imatrix); -size_t quantize_q4_0 (const float * src, void * dst, int nrows, int n_per_row, int64_t * hist, const float * imatrix); -size_t quantize_q4_1 (const float * src, void * dst, int nrows, int n_per_row, int64_t * hist, const float * imatrix); -size_t quantize_q5_0 (const float * src, void * dst, int nrows, int n_per_row, int64_t * hist, const float * imatrix); -size_t quantize_q5_1 (const float * src, void * dst, int nrows, int n_per_row, int64_t * hist, const float * imatrix); +size_t quantize_iq2_xxs(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int nrows, int n_per_row, const float * imatrix); +size_t quantize_iq2_xs (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int nrows, int n_per_row, const float * imatrix); +size_t quantize_iq2_s (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int nrows, int n_per_row, const float * imatrix); +size_t quantize_iq3_xxs(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int nrows, int n_per_row, const float * imatrix); +size_t quantize_iq1_s (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int nrows, int n_per_row, const float * imatrix); +size_t quantize_iq4_nl (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int nrows, int n_per_row, const float * imatrix); +size_t quantize_iq4_xs (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int nrows, int n_per_row, const float * imatrix); +size_t quantize_iq3_s (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int nrows, int n_per_row, const float * imatrix); + +size_t quantize_q2_K(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int nrows, int n_per_row, const float * imatrix); +size_t quantize_q3_K(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int nrows, int n_per_row, const float * imatrix); +size_t quantize_q4_K(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int nrows, int n_per_row, const float * imatrix); +size_t quantize_q5_K(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int nrows, int n_per_row, const float * imatrix); +size_t quantize_q6_K(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int nrows, int n_per_row, const float * imatrix); +size_t quantize_q4_0(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int nrows, int n_per_row, const float * imatrix); +size_t quantize_q4_1(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int nrows, int n_per_row, const float * imatrix); +size_t quantize_q5_0(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int nrows, int n_per_row, const float * imatrix); +size_t quantize_q5_1(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int nrows, int n_per_row, const float * imatrix); +size_t quantize_q8_0(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int nrows, int n_per_row, const float * imatrix); void iq2xs_init_impl(enum ggml_type type); void iq2xs_free_impl(enum ggml_type type); diff --git a/ggml-vulkan.cpp b/ggml-vulkan.cpp index 5a1b3f477..d41aa7d22 100644 --- a/ggml-vulkan.cpp +++ b/ggml-vulkan.cpp @@ -4102,45 +4102,7 @@ static void ggml_vk_test_transfer(ggml_backend_vk_context * ctx, size_t ne, bool } static void ggml_vk_quantize_data(const float * from, void * to, size_t ne, ggml_type quant) { - std::vector hist_cur(1 << 4, 0); - - switch(quant) { - case GGML_TYPE_F32: - memcpy(to, from, sizeof(float) * ne); - break; - case GGML_TYPE_Q4_0: - ggml_quantize_q4_0(from, to, ne, ne, hist_cur.data()); - break; - case GGML_TYPE_Q4_1: - ggml_quantize_q4_1(from, to, ne, ne, hist_cur.data()); - break; - case GGML_TYPE_Q5_0: - ggml_quantize_q5_0(from, to, ne, ne, hist_cur.data()); - break; - case GGML_TYPE_Q5_1: - ggml_quantize_q5_1(from, to, ne, ne, hist_cur.data()); - break; - case GGML_TYPE_Q8_0: - ggml_quantize_q8_0(from, to, ne, ne, hist_cur.data()); - break; - case GGML_TYPE_Q2_K: - ggml_quantize_q2_K(from, to, ne, ne, hist_cur.data()); - break; - case GGML_TYPE_Q3_K: - ggml_quantize_q3_K(from, to, ne, ne, hist_cur.data()); - break; - case GGML_TYPE_Q4_K: - ggml_quantize_q4_K(from, to, ne, ne, hist_cur.data()); - break; - case GGML_TYPE_Q5_K: - ggml_quantize_q5_K(from, to, ne, ne, hist_cur.data()); - break; - case GGML_TYPE_Q6_K: - ggml_quantize_q6_K(from, to, ne, ne, hist_cur.data()); - break; - default: - GGML_ASSERT(false); - } + ggml_quantize_chunk(quant, from, to, 0, 1, ne, nullptr); } static void ggml_vk_test_dequant(ggml_backend_vk_context * ctx, size_t ne, ggml_type quant) { diff --git a/ggml.c b/ggml.c index 6eff98ab6..80efa6f2a 100644 --- a/ggml.c +++ b/ggml.c @@ -20159,133 +20159,6 @@ void ggml_quantize_free(void) { ggml_critical_section_end(); } -size_t ggml_quantize_q4_0(const float * src, void * dst, int n, int k, int64_t * hist) { - assert(k % QK4_0 == 0); - const int nb = k / QK4_0; - - for (int b = 0; b < n; b += k) { - block_q4_0 * restrict y = (block_q4_0 *) dst + b/QK4_0; - - quantize_row_q4_0_reference(src + b, y, k); - - for (int i = 0; i < nb; i++) { - for (int j = 0; j < QK4_0; j += 2) { - const uint8_t vi0 = y[i].qs[j/2] & 0x0F; - const uint8_t vi1 = y[i].qs[j/2] >> 4; - - hist[vi0]++; - hist[vi1]++; - } - } - } - - return (n/QK4_0*sizeof(block_q4_0)); -} - -size_t ggml_quantize_q4_1(const float * src, void * dst, int n, int k, int64_t * hist) { - assert(k % QK4_1 == 0); - const int nb = k / QK4_1; - - for (int b = 0; b < n; b += k) { - block_q4_1 * restrict y = (block_q4_1 *) dst + b/QK4_1; - - quantize_row_q4_1_reference(src + b, y, k); - - for (int i = 0; i < nb; i++) { - for (int j = 0; j < QK4_1; j += 2) { - const uint8_t vi0 = y[i].qs[j/2] & 0x0F; - const uint8_t vi1 = y[i].qs[j/2] >> 4; - - hist[vi0]++; - hist[vi1]++; - } - } - } - - return (n/QK4_1*sizeof(block_q4_1)); -} - -size_t ggml_quantize_q5_0(const float * src, void * dst, int n, int k, int64_t * hist) { - assert(k % QK5_0 == 0); - const int nb = k / QK5_0; - - for (int b = 0; b < n; b += k) { - block_q5_0 * restrict y = (block_q5_0 *)dst + b/QK5_0; - - quantize_row_q5_0_reference(src + b, y, k); - - for (int i = 0; i < nb; i++) { - uint32_t qh; - memcpy(&qh, &y[i].qh, sizeof(qh)); - - for (int j = 0; j < QK5_0; j += 2) { - const uint8_t vh0 = ((qh & (1u << (j/2 + 0 ))) >> (j/2 + 0 )) << 4; - const uint8_t vh1 = ((qh & (1u << (j/2 + 16))) >> (j/2 + 12)); - - // cast to 16 bins - const uint8_t vi0 = ((y[i].qs[j/2] & 0x0F) | vh0) / 2; - const uint8_t vi1 = ((y[i].qs[j/2] >> 4) | vh1) / 2; - - hist[vi0]++; - hist[vi1]++; - } - } - } - - return (n/QK5_0*sizeof(block_q5_0)); -} - -size_t ggml_quantize_q5_1(const float * src, void * dst, int n, int k, int64_t * hist) { - assert(k % QK5_1 == 0); - const int nb = k / QK5_1; - - for (int b = 0; b < n; b += k) { - block_q5_1 * restrict y = (block_q5_1 *)dst + b/QK5_1; - - quantize_row_q5_1_reference(src + b, y, k); - - for (int i = 0; i < nb; i++) { - uint32_t qh; - memcpy(&qh, &y[i].qh, sizeof(qh)); - - for (int j = 0; j < QK5_1; j += 2) { - const uint8_t vh0 = ((qh & (1u << (j/2 + 0 ))) >> (j/2 + 0 )) << 4; - const uint8_t vh1 = ((qh & (1u << (j/2 + 16))) >> (j/2 + 12)); - - // cast to 16 bins - const uint8_t vi0 = ((y[i].qs[j/2] & 0x0F) | vh0) / 2; - const uint8_t vi1 = ((y[i].qs[j/2] >> 4) | vh1) / 2; - - hist[vi0]++; - hist[vi1]++; - } - } - } - - return (n/QK5_1*sizeof(block_q5_1)); -} - -size_t ggml_quantize_q8_0(const float * src, void * dst, int n, int k, int64_t * hist) { - assert(k % QK8_0 == 0); - const int nb = k / QK8_0; - - for (int b = 0; b < n; b += k) { - block_q8_0 * restrict y = (block_q8_0 *)dst + b/QK8_0; - - quantize_row_q8_0_reference(src + b, y, k); - - for (int i = 0; i < nb; i++) { - for (int j = 0; j < QK8_0; ++j) { - const int8_t vi = y[i].qs[j]; - - hist[vi/16 + 8]++; - } - } - } - - return (n/QK8_0*sizeof(block_q8_0)); -} - bool ggml_quantize_requires_imatrix(enum ggml_type type) { return type == GGML_TYPE_IQ2_XXS || @@ -20293,177 +20166,52 @@ bool ggml_quantize_requires_imatrix(enum ggml_type type) { type == GGML_TYPE_IQ1_S; } -size_t ggml_quantize_chunk(enum ggml_type type, const float * src, void * dst, int start, - int nrows, int n_per_row, int64_t * hist, const float * imatrix) { +size_t ggml_quantize_chunk( + enum ggml_type type, + const float * src, + void * dst, + int start, + int nrows, + int n_per_row, + const float * imatrix) { + const int n = nrows * n_per_row; + + if (ggml_quantize_requires_imatrix(type)) { + GGML_ASSERT(imatrix != NULL); + } + + GGML_ASSERT(start % type_traits[type].blck_size == 0); + GGML_ASSERT(start % n_per_row == 0); + ggml_quantize_init(type); // this is noop if already initialized + + const size_t start_row = start / n_per_row; + const size_t row_size = ggml_row_size(type, n_per_row); + size_t result = 0; - int n = nrows * n_per_row; + switch (type) { - case GGML_TYPE_Q4_0: - { - GGML_ASSERT(start % QK4_0 == 0); - GGML_ASSERT(start % n_per_row == 0); - size_t start_row = start / n_per_row; - size_t row_size = ggml_row_size(type, n_per_row); - result = quantize_q4_0(src + start, (char *)dst + start_row * row_size, nrows, n_per_row, hist, imatrix); - GGML_ASSERT(result == row_size * nrows); - } break; - case GGML_TYPE_Q4_1: - { - GGML_ASSERT(start % QK4_1 == 0); - GGML_ASSERT(start % n_per_row == 0); - size_t start_row = start / n_per_row; - size_t row_size = ggml_row_size(type, n_per_row); - result = quantize_q4_1(src + start, (char *)dst + start_row * row_size, nrows, n_per_row, hist, imatrix); - GGML_ASSERT(result == row_size * nrows); - } break; - case GGML_TYPE_Q5_0: - { - GGML_ASSERT(start % QK5_0 == 0); - GGML_ASSERT(start % n_per_row == 0); - size_t start_row = start / n_per_row; - size_t row_size = ggml_row_size(type, n_per_row); - result = quantize_q5_0(src + start, (char *)dst + start_row * row_size, nrows, n_per_row, hist, imatrix); - GGML_ASSERT(result == row_size * nrows); - } break; - case GGML_TYPE_Q5_1: - { - GGML_ASSERT(start % QK5_1 == 0); - GGML_ASSERT(start % n_per_row == 0); - size_t start_row = start / n_per_row; - size_t row_size = ggml_row_size(type, n_per_row); - result = quantize_q5_1(src + start, (char *)dst + start_row * row_size, nrows, n_per_row, hist, imatrix); - GGML_ASSERT(result == row_size * nrows); - } break; - case GGML_TYPE_Q8_0: - { - GGML_ASSERT(start % QK8_0 == 0); - block_q8_0 * block = (block_q8_0*)dst + start / QK8_0; - result = ggml_quantize_q8_0(src + start, block, n, n, hist); - } break; - case GGML_TYPE_Q2_K: - { - GGML_ASSERT(start % QK_K == 0); - GGML_ASSERT(start % n_per_row == 0); - size_t start_row = start / n_per_row; - size_t row_size = ggml_row_size(type, n_per_row); - result = quantize_q2_K(src + start, (char *)dst + start_row * row_size, nrows, n_per_row, hist, imatrix); - GGML_ASSERT(result == row_size * nrows); - } break; - case GGML_TYPE_Q3_K: - { - GGML_ASSERT(start % QK_K == 0); - GGML_ASSERT(start % n_per_row == 0); - size_t start_row = start / n_per_row; - size_t row_size = ggml_row_size(type, n_per_row); - result = quantize_q3_K(src + start, (char *)dst + start_row * row_size, nrows, n_per_row, hist, imatrix); - GGML_ASSERT(result == row_size * nrows); - } break; - case GGML_TYPE_Q4_K: - { - GGML_ASSERT(start % QK_K == 0); - GGML_ASSERT(start % n_per_row == 0); - size_t start_row = start / n_per_row; - size_t row_size = ggml_row_size(type, n_per_row); - result = quantize_q4_K(src + start, (char *)dst + start_row * row_size, nrows, n_per_row, hist, imatrix); - GGML_ASSERT(result == row_size * nrows); - } break; - case GGML_TYPE_Q5_K: - { - GGML_ASSERT(start % QK_K == 0); - GGML_ASSERT(start % n_per_row == 0); - size_t start_row = start / n_per_row; - size_t row_size = ggml_row_size(type, n_per_row); - result = quantize_q5_K(src + start, (char *)dst + start_row * row_size, nrows, n_per_row, hist, imatrix); - GGML_ASSERT(result == row_size * nrows); - } break; - case GGML_TYPE_Q6_K: - { - GGML_ASSERT(start % QK_K == 0); - GGML_ASSERT(start % n_per_row == 0); - size_t start_row = start / n_per_row; - size_t row_size = ggml_row_size(type, n_per_row); - result = quantize_q6_K(src + start, (char *)dst + start_row * row_size, nrows, n_per_row, hist, imatrix); - GGML_ASSERT(result == row_size * nrows); - } break; - case GGML_TYPE_IQ2_XXS: - { - GGML_ASSERT(start % QK_K == 0); - GGML_ASSERT(start % n_per_row == 0); - GGML_ASSERT(imatrix); - size_t start_row = start / n_per_row; - size_t row_size = ggml_row_size(type, n_per_row); - result = quantize_iq2_xxs(src + start, (char *)dst + start_row * row_size, nrows, n_per_row, hist, imatrix); - GGML_ASSERT(result == row_size * nrows); - } break; - case GGML_TYPE_IQ2_XS: - { - GGML_ASSERT(start % QK_K == 0); - GGML_ASSERT(start % n_per_row == 0); - GGML_ASSERT(imatrix); - size_t start_row = start / n_per_row; - size_t row_size = ggml_row_size(type, n_per_row); - result = quantize_iq2_xs(src + start, (char *)dst + start_row * row_size, nrows, n_per_row, hist, imatrix); - GGML_ASSERT(result == row_size * nrows); - } break; - case GGML_TYPE_IQ3_XXS: - { - GGML_ASSERT(start % QK_K == 0); - GGML_ASSERT(start % n_per_row == 0); - size_t start_row = start / n_per_row; - size_t row_size = ggml_row_size(type, n_per_row); - result = quantize_iq3_xxs(src + start, (char *)dst + start_row * row_size, nrows, n_per_row, hist, imatrix); - GGML_ASSERT(result == row_size * nrows); - } break; - case GGML_TYPE_IQ3_S: - { - GGML_ASSERT(start % QK_K == 0); - GGML_ASSERT(start % n_per_row == 0); - size_t start_row = start / n_per_row; - size_t row_size = ggml_row_size(type, n_per_row); - result = quantize_iq3_s(src + start, (char *)dst + start_row * row_size, nrows, n_per_row, hist, imatrix); - GGML_ASSERT(result == row_size * nrows); - } break; - case GGML_TYPE_IQ2_S: - { - GGML_ASSERT(start % QK_K == 0); - GGML_ASSERT(start % n_per_row == 0); - size_t start_row = start / n_per_row; - size_t row_size = ggml_row_size(type, n_per_row); - result = quantize_iq2_s(src + start, (char *)dst + start_row * row_size, nrows, n_per_row, hist, imatrix); - GGML_ASSERT(result == row_size * nrows); - } break; - case GGML_TYPE_IQ1_S: - { - GGML_ASSERT(start % QK_K == 0); - GGML_ASSERT(start % n_per_row == 0); - size_t start_row = start / n_per_row; - size_t row_size = ggml_row_size(type, n_per_row); - result = quantize_iq1_s(src + start, (char *)dst + start_row * row_size, nrows, n_per_row, hist, imatrix); - GGML_ASSERT(result == row_size * nrows); - } break; - case GGML_TYPE_IQ4_NL: + case GGML_TYPE_Q4_0: result = quantize_q4_0(src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break; + case GGML_TYPE_Q4_1: result = quantize_q4_1(src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break; + case GGML_TYPE_Q5_0: result = quantize_q5_0(src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break; + case GGML_TYPE_Q5_1: result = quantize_q5_1(src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break; + case GGML_TYPE_Q8_0: result = quantize_q8_0(src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break; + case GGML_TYPE_Q2_K: result = quantize_q2_K(src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break; + case GGML_TYPE_Q3_K: result = quantize_q3_K(src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break; + case GGML_TYPE_Q4_K: result = quantize_q4_K(src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break; + case GGML_TYPE_Q5_K: result = quantize_q5_K(src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break; + case GGML_TYPE_Q6_K: result = quantize_q6_K(src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break; + case GGML_TYPE_IQ2_XXS: result = quantize_iq2_xxs(src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break; + case GGML_TYPE_IQ2_XS: result = quantize_iq2_xs (src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break; + case GGML_TYPE_IQ3_XXS: result = quantize_iq3_xxs(src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break; + case GGML_TYPE_IQ3_S: result = quantize_iq3_s (src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break; + case GGML_TYPE_IQ2_S: result = quantize_iq2_s (src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break; + case GGML_TYPE_IQ1_S: result = quantize_iq1_s (src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break; + case GGML_TYPE_IQ4_NL: result = quantize_iq4_nl (src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break; #if QK_K == 64 - case GGML_TYPE_IQ4_XS: -#endif - { - GGML_ASSERT(start % QK4_NL == 0); - GGML_ASSERT(start % n_per_row == 0); - size_t start_row = start / n_per_row; - size_t row_size = ggml_row_size(type, n_per_row); - result = quantize_iq4_nl(src + start, (char *)dst + start_row * row_size, nrows, n_per_row, hist, imatrix); - GGML_ASSERT(result == row_size * nrows); - } break; -#if QK_K != 64 - case GGML_TYPE_IQ4_XS: - { - GGML_ASSERT(start % QK_K == 0); - GGML_ASSERT(start % n_per_row == 0); - size_t start_row = start / n_per_row; - size_t row_size = ggml_row_size(type, n_per_row); - result = quantize_iq4_xs(src + start, (char *)dst + start_row * row_size, nrows, n_per_row, hist, imatrix); - GGML_ASSERT(result == row_size * nrows); - } break; + case GGML_TYPE_IQ4_XS: result = quantize_iq4_nl (src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break; +#else + case GGML_TYPE_IQ4_XS: result = quantize_iq4_xs (src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break; #endif case GGML_TYPE_F16: { @@ -20480,6 +20228,9 @@ size_t ggml_quantize_chunk(enum ggml_type type, const float * src, void * dst, i default: assert(false); } + + GGML_ASSERT(result == nrows * row_size); + return result; } diff --git a/ggml.h b/ggml.h index a13b0cec4..1171088a9 100644 --- a/ggml.h +++ b/ggml.h @@ -2194,25 +2194,18 @@ extern "C" { GGML_API void ggml_quantize_init(enum ggml_type type); GGML_API void ggml_quantize_free(void); - // TODO: these would probably get removed in favor of the more general ggml_quantize_chunk - GGML_API size_t ggml_quantize_q4_0(const float * src, void * dst, int n, int k, int64_t * hist); - GGML_API size_t ggml_quantize_q4_1(const float * src, void * dst, int n, int k, int64_t * hist); - GGML_API size_t ggml_quantize_q5_0(const float * src, void * dst, int n, int k, int64_t * hist); - GGML_API size_t ggml_quantize_q5_1(const float * src, void * dst, int n, int k, int64_t * hist); - GGML_API size_t ggml_quantize_q8_0(const float * src, void * dst, int n, int k, int64_t * hist); - - GGML_API size_t ggml_quantize_q2_K(const float * src, void * dst, int n, int k, int64_t * hist); - GGML_API size_t ggml_quantize_q3_K(const float * src, void * dst, int n, int k, int64_t * hist); - GGML_API size_t ggml_quantize_q4_K(const float * src, void * dst, int n, int k, int64_t * hist); - GGML_API size_t ggml_quantize_q5_K(const float * src, void * dst, int n, int k, int64_t * hist); - GGML_API size_t ggml_quantize_q6_K(const float * src, void * dst, int n, int k, int64_t * hist); - // some quantization type cannot be used without an importance matrix GGML_API bool ggml_quantize_requires_imatrix(enum ggml_type type); // calls ggml_quantize_init internally (i.e. can allocate memory) - GGML_API size_t ggml_quantize_chunk(enum ggml_type type, const float * src, void * dst, - int start, int nrows, int n_per_row, int64_t * hist, const float * imatrix); + GGML_API size_t ggml_quantize_chunk( + enum ggml_type type, + const float * src, + void * dst, + int start, + int nrows, + int n_per_row, + const float * imatrix); // // gguf diff --git a/llama.cpp b/llama.cpp index 8c147a42b..c58a029f7 100644 --- a/llama.cpp +++ b/llama.cpp @@ -11890,17 +11890,16 @@ static ggml_type get_k_quant_type(quantize_state_internal & qs, ggml_type new_ty return new_type; } -static int32_t llama_tensor_quantize_internal(enum ggml_type new_type, const float * f32_data, void * new_data, const int chunk_size, int nrows, int n_per_row, int64_t * hist_cur, const float * imatrix, std::vector & workers, const int nthread) { +static int32_t llama_tensor_quantize_internal(enum ggml_type new_type, const float * f32_data, void * new_data, const int chunk_size, int nrows, int n_per_row, const float * imatrix, std::vector & workers, const int nthread) { std::mutex mutex; int counter = 0; size_t new_size = 0; if (nthread < 2) { // single-thread - return ggml_quantize_chunk(new_type, f32_data, new_data, 0, nrows, n_per_row, hist_cur, imatrix); + return ggml_quantize_chunk(new_type, f32_data, new_data, 0, nrows, n_per_row, imatrix); } - auto compute = [&mutex, &counter, &hist_cur, &new_size, new_type, f32_data, new_data, chunk_size, + auto compute = [&mutex, &counter, &new_size, new_type, f32_data, new_data, chunk_size, nrows, n_per_row, imatrix]() { - std::array local_hist = {}; const int nrows_per_chunk = chunk_size / n_per_row; size_t local_size = 0; while (true) { @@ -11908,17 +11907,13 @@ static int32_t llama_tensor_quantize_internal(enum ggml_type new_type, const flo int first_row = counter; counter += nrows_per_chunk; if (first_row >= nrows) { if (local_size > 0) { - for (int j=0; j hist_all(1 << 4, 0); std::vector workers; workers.reserve(nthread); @@ -12175,7 +12169,6 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s work.resize(nelements * 4); // upper bound on size } new_data = work.data(); - std::array hist_cur = {}; const int n_per_row = tensor->ne[0]; const int nrows = nelements / n_per_row; @@ -12185,22 +12178,9 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s const int nchunk = (nelements + chunk_size - 1)/chunk_size; const int nthread_use = nthread > 1 ? std::max(1, std::min(nthread, nchunk)) : 1; - new_size = llama_tensor_quantize_internal(new_type, f32_data, new_data, chunk_size, nrows, n_per_row, hist_cur.data(), imatrix, workers, nthread_use); + new_size = llama_tensor_quantize_internal(new_type, f32_data, new_data, chunk_size, nrows, n_per_row, imatrix, workers, nthread_use); - LLAMA_LOG_INFO("size = %8.2f MiB -> %8.2f MiB", ggml_nbytes(tensor)/1024.0/1024.0, new_size/1024.0/1024.0); - int64_t tot_count = 0; - for (size_t i = 0; i < hist_cur.size(); i++) { - hist_all[i] += hist_cur[i]; - tot_count += hist_cur[i]; - } - - if (tot_count > 0) { - LLAMA_LOG_INFO(" | hist: "); - for (size_t i = 0; i < hist_cur.size(); i++) { - LLAMA_LOG_INFO("%5.3f ", hist_cur[i] / float(nelements)); - } - } - LLAMA_LOG_INFO("\n"); + LLAMA_LOG_INFO("size = %8.2f MiB -> %8.2f MiB\n", ggml_nbytes(tensor)/1024.0/1024.0, new_size/1024.0/1024.0); } total_size_org += ggml_nbytes(tensor); total_size_new += new_size; @@ -12229,22 +12209,6 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s LLAMA_LOG_INFO("%s: model size = %8.2f MB\n", __func__, total_size_org/1024.0/1024.0); LLAMA_LOG_INFO("%s: quant size = %8.2f MB\n", __func__, total_size_new/1024.0/1024.0); - // print histogram for all tensors - { - int64_t sum_all = 0; - for (size_t i = 0; i < hist_all.size(); i++) { - sum_all += hist_all[i]; - } - - if (sum_all > 0) { - LLAMA_LOG_INFO("%s: hist: ", __func__); - for (size_t i = 0; i < hist_all.size(); i++) { - LLAMA_LOG_INFO("%5.3f ", hist_all[i] / float(sum_all)); - } - LLAMA_LOG_INFO("\n"); - } - } - if (qs.n_fallback > 0) { LLAMA_LOG_WARN("%s: WARNING: %d of %d tensor(s) incompatible with k-quants and required fallback quantization\n", __func__, qs.n_fallback, qs.n_k_quantized + qs.n_fallback); diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index 8a6999f21..fc5edcc4b 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -53,7 +53,6 @@ static void init_tensor_uniform(ggml_tensor * tensor, float min = -1.0f, float m } else if (ggml_is_quantized(tensor->type) || tensor->type == GGML_TYPE_F16) { GGML_ASSERT(size % ggml_blck_size(tensor->type) == 0); std::vector dataq(ggml_row_size(tensor->type, size)); - int64_t hist[16]; std::vector imatrix(tensor->ne[0], 1.0f); // dummy importance matrix const float * im = imatrix.data(); if (!ggml_quantize_requires_imatrix(tensor->type)) { @@ -63,7 +62,7 @@ static void init_tensor_uniform(ggml_tensor * tensor, float min = -1.0f, float m im = nullptr; } } - ggml_quantize_chunk(tensor->type, data.data(), dataq.data(), 0, size/tensor->ne[0], tensor->ne[0], hist, im); + ggml_quantize_chunk(tensor->type, data.data(), dataq.data(), 0, size/tensor->ne[0], tensor->ne[0], im); ggml_backend_tensor_set(tensor, dataq.data(), 0, dataq.size()); } else if (tensor->type == GGML_TYPE_I8 || tensor->type == GGML_TYPE_I16 || tensor->type == GGML_TYPE_I32) { // This is going to create some weird integers though. From 58308a0ecce7cc261b802f4803c38d420063db21 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Sat, 9 Mar 2024 17:34:15 +0200 Subject: [PATCH 573/812] server : fix metrics init (#5964) --- examples/server/server.cpp | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/examples/server/server.cpp b/examples/server/server.cpp index 796f3499c..2374b7e4a 100644 --- a/examples/server/server.cpp +++ b/examples/server/server.cpp @@ -341,7 +341,7 @@ struct server_slot { }; struct server_metrics { - const int64_t t_start = ggml_time_us(); + int64_t t_start = 0; uint64_t n_prompt_tokens_processed_total = 0; uint64_t t_prompt_processing_total = 0; @@ -354,14 +354,18 @@ struct server_metrics { uint64_t n_tokens_predicted = 0; uint64_t t_tokens_generation = 0; - void on_prompt_eval(const server_slot &slot) { + void init() { + t_start = ggml_time_us(); + } + + void on_prompt_eval(const server_slot & slot) { n_prompt_tokens_processed_total += slot.n_prompt_tokens_processed; n_prompt_tokens_processed += slot.n_prompt_tokens_processed; t_prompt_processing += slot.t_prompt_processing; t_prompt_processing_total += slot.t_prompt_processing; } - void on_prediction(const server_slot &slot) { + void on_prediction(const server_slot & slot) { n_tokens_predicted_total += slot.n_decoded; n_tokens_predicted += slot.n_decoded; t_tokens_generation += slot.t_token_generation; @@ -690,10 +694,11 @@ struct server_context { return res > 0; } - void initialize() { + void init() { const int32_t n_ctx_slot = n_ctx / params.n_parallel; LOG_INFO("initializing slots", {{"n_slots", params.n_parallel}}); + for (int i = 0; i < params.n_parallel; i++) { server_slot slot; @@ -735,6 +740,8 @@ struct server_context { default_generation_settings_for_props["seed"] = -1; batch = llama_batch_init(n_ctx, 0, params.n_parallel); + + metrics.init(); } std::vector tokenize(const json & json_prompt, bool add_bos) const { @@ -2783,7 +2790,7 @@ int main(int argc, char ** argv) { state.store(SERVER_STATE_ERROR); return 1; } else { - ctx_server.initialize(); + ctx_server.init(); state.store(SERVER_STATE_READY); } From 8380ecfb219c2e73cc706fcc83935b7c806cc7c3 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Sat, 9 Mar 2024 17:36:20 +0200 Subject: [PATCH 574/812] ggml : fix unnecessary f32 -> f16 -> f32 casts (mmla) (#5951) --- ggml-quants.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ggml-quants.c b/ggml-quants.c index 4ee4e0604..807c5e391 100644 --- a/ggml-quants.c +++ b/ggml-quants.c @@ -4059,10 +4059,10 @@ void ggml_vec_dot_q4_1_q8_1(int n, float * restrict s, size_t bs, const void * r const int8x16_t y1_h = vld1q_s8(b_y1->qs + 16); // mmla into int32x4_t - float32x4_t scale = {GGML_FP16_TO_FP32(b_x0->d)*GGML_FP16_TO_FP32(b_y0->d), - GGML_FP16_TO_FP32(b_x0->d)*GGML_FP16_TO_FP32(b_y1->d), - GGML_FP16_TO_FP32(b_x1->d)*GGML_FP16_TO_FP32(b_y0->d), - GGML_FP16_TO_FP32(b_x1->d)*GGML_FP16_TO_FP32(b_y1->d)}; + float32x4_t scale = {GGML_FP16_TO_FP32(b_x0->d)*b_y0->d, + GGML_FP16_TO_FP32(b_x0->d)*b_y1->d, + GGML_FP16_TO_FP32(b_x1->d)*b_y0->d, + GGML_FP16_TO_FP32(b_x1->d)*b_y1->d}; int8x16_t l0 = vreinterpretq_s8_s64(vzip1q_s64(vreinterpretq_s64_s8(x0_l), vreinterpretq_s64_s8(x1_l))); int8x16_t l1 = vreinterpretq_s8_s64(vzip2q_s64(vreinterpretq_s64_s8(x0_l), vreinterpretq_s64_s8(x1_l))); From 098dbaab449f5309a54871ba7e5acef72ae696de Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Sat, 9 Mar 2024 18:14:13 +0200 Subject: [PATCH 575/812] readme : update hot topics --- README.md | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index d7dba73e6..e3ec0817a 100644 --- a/README.md +++ b/README.md @@ -8,6 +8,11 @@ Inference of Meta's [LLaMA](https://arxiv.org/abs/2302.13971) model (and others) in pure C/C++ +> [!IMPORTANT] +> **Quantization blind testing: https://github.com/ggerganov/llama.cpp/discussions/5962** +> +> Vote for which quantization type provides better responses, all other parameters being the same. + ### Recent API changes - [2024 Mar 8] `llama_kv_cache_seq_rm()` returns a `bool` instead of `void`, and new `llama_n_max_seq()` returns the upper limit of acceptable `seq_id` in batches (relevant when dealing with multiple sequences) https://github.com/ggerganov/llama.cpp/pull/5328 @@ -16,11 +21,7 @@ Inference of Meta's [LLaMA](https://arxiv.org/abs/2302.13971) model (and others) ### Hot topics -- The `api_like_OAI.py` script has been removed - use `server` instead ([#5766](https://github.com/ggerganov/llama.cpp/issues/5766#issuecomment-1969037761)) -- Support for chat templates: [Wiki (contributions welcome)](https://github.com/ggerganov/llama.cpp/wiki/Templates-supported-by-llama_chat_apply_template) -- Support for Gemma models: https://github.com/ggerganov/llama.cpp/pull/5631 -- Non-linear quantization IQ4_NL: https://github.com/ggerganov/llama.cpp/pull/5590 -- Looking for contributions to improve and maintain the `server` example: https://github.com/ggerganov/llama.cpp/issues/4216 +- Initial Mamba support has been added: https://github.com/ggerganov/llama.cpp/pull/5328 ---- From d894f352bf433157232dc8dc54eacd50014e898e Mon Sep 17 00:00:00 2001 From: slaren Date: Sat, 9 Mar 2024 19:55:54 +0100 Subject: [PATCH 576/812] perplexity : support using multiple sequences to allow larger batch sizes (#5946) * perplexity : support using multiple sequences to allow larger batch sizes ggml-ci * set cparams.n_parallel to the number of sequences * print tested n_ctx, add assert --- examples/perplexity/perplexity.cpp | 137 +++++++++++++++++++---------- llama.cpp | 22 +++-- 2 files changed, 107 insertions(+), 52 deletions(-) diff --git a/examples/perplexity/perplexity.cpp b/examples/perplexity/perplexity.cpp index 52789ee63..293eb52c3 100644 --- a/examples/perplexity/perplexity.cpp +++ b/examples/perplexity/perplexity.cpp @@ -442,7 +442,7 @@ static results_perplexity perplexity_v2(llama_context * ctx, const gpt_params & return {tokens, std::exp(nll / count), logit_history, prob_history}; } -static results_perplexity perplexity(llama_context * ctx, const gpt_params & params) { +static results_perplexity perplexity(llama_context * ctx, const gpt_params & params, const int32_t n_ctx) { if (params.ppl_stride > 0) { return perplexity_v2(ctx, params); } @@ -453,7 +453,6 @@ static results_perplexity perplexity(llama_context * ctx, const gpt_params & par // BOS tokens will be added for each chunk before eval const bool add_bos = llama_should_add_bos_token(llama_get_model(ctx)); - const int n_ctx = llama_n_ctx(ctx); std::ofstream logits_stream; if (!params.logits_file.empty()) { @@ -499,13 +498,19 @@ static results_perplexity perplexity(llama_context * ctx, const gpt_params & par double nll2 = 0.0; const int num_batches = (n_ctx + n_batch - 1) / n_batch; + const int n_seq = std::max(1, n_batch / n_ctx); + + GGML_ASSERT(n_batch < n_ctx || n_batch % n_ctx == 0); + GGML_ASSERT(params.n_ctx == n_seq * n_ctx); + + llama_batch batch = llama_batch_init(std::min(n_batch, n_ctx*n_seq), 0, 1); std::vector logits; if (num_batches > 1) { logits.reserve((size_t)n_ctx * n_vocab); } - fprintf(stderr, "%s: calculating perplexity over %d chunks, batch_size=%d\n", __func__, n_chunk, n_batch); + fprintf(stderr, "%s: calculating perplexity over %d chunks, n_ctx=%d, batch_size=%d, n_seq=%d\n", __func__, n_chunk, n_ctx, n_batch, n_seq); std::vector workers(std::thread::hardware_concurrency() - 1); @@ -518,10 +523,26 @@ static results_perplexity perplexity(llama_context * ctx, const gpt_params & par log_probs.resize(n_ctx * nv); } - for (int i = 0; i < n_chunk; ++i) { + // We get the logits for all the tokens in the context window (params.n_ctx) + // from llama_eval above. Now, based on https://huggingface.co/docs/transformers/perplexity, + // calculate the perplexity over the last half of the window (so the model always has + // some context to predict the token). + // + // We rely on the fact that attention in the forward pass only looks at previous + // tokens here, so the logits returned for each token are an accurate representation + // of what the model would have predicted at that point. + // + // Example, we have a context window of 512, we will compute perplexity for each of the + // last 256 tokens. Then, we split the input up into context window size chunks to + // process the entire prompt. + const int first = n_ctx/2; + + for (int i = 0; i < n_chunk; i += n_seq) { const int start = i * n_ctx; const int end = start + n_ctx; + const int n_seq_batch = std::min(n_seq, n_chunk - i); + const auto t_start = std::chrono::high_resolution_clock::now(); // clear the KV cache @@ -531,22 +552,37 @@ static results_perplexity perplexity(llama_context * ctx, const gpt_params & par const int batch_start = start + j * n_batch; const int batch_size = std::min(end - batch_start, n_batch); - // save original token and restore it after eval - const auto token_org = tokens[batch_start]; + batch.n_tokens = 0; + for (int seq = 0; seq < n_seq_batch; seq++) { + int seq_start = batch_start + seq*n_ctx; - // add BOS token for the first batch of each chunk - if (add_bos && j == 0) { - tokens[batch_start] = llama_token_bos(llama_get_model(ctx)); + // save original token and restore it after eval + const auto token_org = tokens[seq_start]; + + // add BOS token for the first batch of each chunk + if (add_bos && j == 0) { + tokens[seq_start] = llama_token_bos(llama_get_model(ctx)); + } + + for (int k = 0; k < batch_size; ++k) { + const int idx = seq*n_ctx + k; + batch.token[idx] = tokens[seq_start + k]; + batch.pos[idx] = j*n_batch + k; + batch.n_seq_id[idx] = 1; + batch.seq_id[idx][0] = seq; + batch.logits[idx] = batch.pos[idx] >= first ? 1 : 0; + } + batch.n_tokens += batch_size; + + // restore the original token in case it was set to BOS + tokens[seq_start] = token_org; } - if (llama_decode(ctx, llama_batch_get_one(tokens.data() + batch_start, batch_size, j * n_batch, 0))) { + if (llama_decode(ctx, batch)) { fprintf(stderr, "%s : failed to eval\n", __func__); return {tokens, -1, logit_history, prob_history}; } - // restore the original token in case it was set to BOS - tokens[batch_start] = token_org; - if (num_batches > 1) { const auto * batch_logits = llama_get_logits(ctx); logits.insert(logits.end(), batch_logits, batch_logits + batch_size * n_vocab); @@ -558,7 +594,7 @@ static results_perplexity perplexity(llama_context * ctx, const gpt_params & par if (i == 0) { const float t_total = std::chrono::duration(t_end - t_start).count(); fprintf(stderr, "%s: %.2f seconds per pass - ETA ", __func__, t_total); - int total_seconds = (int)(t_total * n_chunk); + int total_seconds = (int)(t_total*n_chunk/n_seq); if (total_seconds >= 60*60) { fprintf(stderr, "%d hours ", total_seconds / (60*60)); total_seconds = total_seconds % (60*60); @@ -566,37 +602,31 @@ static results_perplexity perplexity(llama_context * ctx, const gpt_params & par fprintf(stderr, "%.2f minutes\n", total_seconds / 60.0); } - // We get the logits for all the tokens in the context window (params.n_ctx) - // from llama_eval above. Now, based on https://huggingface.co/docs/transformers/perplexity, - // calculate the perplexity over the last half of the window (so the model always has - // some context to predict the token). - // - // We rely on the fact that attention in the forward pass only looks at previous - // tokens here, so the logits returned for each token are an accurate representation - // of what the model would have predicted at that point. - // - // Example, we have a context window of 512, we will compute perplexity for each of the - // last 256 tokens. Then, we split the input up into context window size chunks to - // process the entire prompt. - const int first = n_ctx/2; - const float * all_logits = num_batches > 1 ? logits.data() : llama_get_logits(ctx); - if (!params.logits_file.empty()) { - process_logits(logits_stream, n_vocab, all_logits + first*n_vocab, tokens.data() + start + first, n_ctx - 1 - first, - workers, log_probs, nll, nll2); - } else { - process_logits(n_vocab, all_logits + first*n_vocab, tokens.data() + start + first, n_ctx - 1 - first, - workers, nll, nll2, logit_history.data() + start + first, prob_history.data() + start + first); - } - count += n_ctx - first - 1; + for (int seq = 0; seq < n_seq_batch; seq++) { + const float * all_logits = num_batches > 1 ? logits.data() : llama_get_logits_ith(ctx, seq*n_ctx); + llama_token * tokens_data = tokens.data() + start + seq*n_ctx + first; + if (!params.logits_file.empty()) { + process_logits(logits_stream, n_vocab, all_logits + first*n_vocab, + tokens_data, n_ctx - 1 - first, + workers, log_probs, nll, nll2); + } else { + process_logits(n_vocab, all_logits + first*n_vocab, + tokens_data, n_ctx - 1 - first, + workers, nll, nll2, + logit_history.data() + start + seq*n_ctx + first, + prob_history.data() + start + seq*n_ctx + first); + } + count += n_ctx - first - 1; - // perplexity is e^(average negative log-likelihood) - if (params.ppl_output_type == 0) { - printf("[%d]%.4lf,", i + 1, std::exp(nll / count)); - } else { - double av = nll/count; - double av2 = nll2/count - av*av; - if (av2 > 0) av2 = sqrt(av2/(count-1)); - printf("%8d %.4lf %4lf %4lf\n", i*n_ctx, std::exp(nll / count), av, av2); + // perplexity is e^(average negative log-likelihood) + if (params.ppl_output_type == 0) { + printf("[%d]%.4lf,", i + seq + 1, std::exp(nll / count)); + } else { + double av = nll/count; + double av2 = nll2/count - av*av; + if (av2 > 0) av2 = sqrt(av2/(count-1)); + printf("%8d %.4lf %4lf %4lf\n", i*n_ctx, std::exp(nll / count), av, av2); + } } fflush(stdout); @@ -615,6 +645,8 @@ static results_perplexity perplexity(llama_context * ctx, const gpt_params & par printf("Unexpected negative standard deviation of log(prob)\n"); } + llama_batch_free(batch); + return {tokens, ppl, logit_history, prob_history}; } @@ -1782,13 +1814,24 @@ static void kl_divergence(llama_context * ctx, const gpt_params & params) { int main(int argc, char ** argv) { gpt_params params; - params.n_batch = 512; if (!gpt_params_parse(argc, argv, params)) { return 1; } params.logits_all = true; - params.n_batch = std::min(params.n_batch, params.n_ctx); + + const int32_t n_ctx = params.n_ctx; + + const bool ppl = !params.hellaswag && !params.winogrande && !params.multiple_choice && !params.kl_divergence; + if (ppl) { + int n_seq = std::max(1, params.n_batch / n_ctx); + int32_t n_kv = n_seq * n_ctx; + params.n_parallel = n_seq; + params.n_ctx = n_kv; + params.n_batch = std::min(params.n_batch, n_kv); + } else { + params.n_batch = std::min(params.n_batch, params.n_ctx); + } if (params.ppl_stride > 0) { fprintf(stderr, "Will perform strided perplexity calculation -> adjusting context size from %d to %d\n", @@ -1847,7 +1890,7 @@ int main(int argc, char ** argv) { } else if (params.kl_divergence) { kl_divergence(ctx, params); } else { - results = perplexity(ctx, params); + results = perplexity(ctx, params, n_ctx); } llama_print_timings(ctx); diff --git a/llama.cpp b/llama.cpp index c58a029f7..b19616e8f 100644 --- a/llama.cpp +++ b/llama.cpp @@ -8925,17 +8925,29 @@ static int llama_decode_internal( if (batch.logits) { logits_out.resize(n_vocab * n_tokens); + int32_t i_first = -1; for (uint32_t i = 0; i < n_tokens; i++) { - if (batch.logits[i] == 0) { - continue; + if (batch.logits[i] && i_first == -1) { + i_first = (int32_t) i; + } + if (batch.logits[i] == 0 || i == n_tokens - 1) { + if (i_first != -1) { + int i_last = batch.logits[i] == 0 ? i : i + 1; + // extract logits for the range [i_first, i_last) + // group the requests to minimize the number of calls to the backend + ggml_backend_tensor_get_async(backend_res, res, + logits_out.data() + (n_vocab*i_first), + (n_vocab*i_first)*sizeof(float), + (i_last - i_first)*n_vocab*sizeof(float)); + i_first = -1; + } } - ggml_backend_tensor_get_async(backend_res, res, logits_out.data() + (n_vocab*i), (n_vocab*i)*sizeof(float), n_vocab*sizeof(float)); #ifndef NDEBUG - logits_valid[i] = true; + logits_valid[i] = batch.logits[i] != 0; #endif } } else if (lctx.logits_all) { - logits_out.resize(n_vocab * n_tokens); + logits_out.resize(n_vocab*n_tokens); ggml_backend_tensor_get_async(backend_res, res, logits_out.data(), 0, n_vocab*n_tokens*sizeof(float)); #ifndef NDEBUG std::fill(logits_valid.begin(), logits_valid.end(), true); From 77d1ac7e00bf049b9f2bba1b5a310a78318c49c4 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Sat, 9 Mar 2024 22:04:00 +0200 Subject: [PATCH 577/812] server : print chat template info --- examples/server/server.cpp | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/examples/server/server.cpp b/examples/server/server.cpp index 2374b7e4a..b14cca61b 100644 --- a/examples/server/server.cpp +++ b/examples/server/server.cpp @@ -2197,7 +2197,8 @@ static void server_print_usage(const char * argv0, const gpt_params & params, co printf(" -gaw N, --grp-attn-w N set the group attention width to extend context size through self-extend(default: 512), used together with group attention factor `--grp-attn-n`\n"); printf(" --chat-template JINJA_TEMPLATE\n"); printf(" set custom jinja chat template (default: template taken from model's metadata)\n"); - printf(" Note: only commonly used templates are accepted, since we don't have jinja parser\n"); + printf(" only commonly used templates are accepted:\n"); + printf(" https://github.com/ggerganov/llama.cpp/wiki/Templates-supported-by-llama_chat_apply_template\n"); printf("\n"); } @@ -2798,13 +2799,30 @@ int main(int argc, char ** argv) { const auto model_meta = ctx_server.model_meta(); - if (sparams.chat_template.empty()) { // custom chat template is not supplied + // if a custom chat template is not supplied, we will use the one that comes with the model (if any) + if (sparams.chat_template.empty()) { if (!ctx_server.validate_model_chat_template()) { LOG_ERROR("The chat template that comes with this model is not yet supported, falling back to chatml. This may cause the model to output suboptimal responses", {}); sparams.chat_template = "chatml"; } } + // print sample chat example to make it clear which template is used + { + json chat; + chat.push_back({{"role", "system"}, {"content", "You are a helpful assistant"}}); + chat.push_back({{"role", "user"}, {"content", "Hello"}}); + chat.push_back({{"role", "assistant"}, {"content", "Hi there"}}); + chat.push_back({{"role", "user"}, {"content", "How are you?"}}); + + const std::string chat_example = format_chat(ctx_server.model, sparams.chat_template, chat); + + LOG_INFO("chat template", { + {"chat_example", chat_example}, + {"built_in", sparams.chat_template.empty()}, + }); + } + // // Middlewares // From 621e86b331f8b0e71f79fd82a4ae1cd54c3e4396 Mon Sep 17 00:00:00 2001 From: Pierrick Hymbert Date: Sat, 9 Mar 2024 23:41:49 +0100 Subject: [PATCH 578/812] server: benchmark: chat/completions scenario and other llm servers comparison (#5941) * server: bench: Init a bench scenario with K6 See #5827 * server: bench: EOL EOF * server: bench: PR feedback and improved k6 script configuration * server: bench: remove llamacpp_completions_tokens_seconds as it include prompt processing time and it's misleading server: bench: add max_tokens from SERVER_BENCH_MAX_TOKENS server: bench: increase truncated rate to 80% before failing * server: bench: fix doc * server: bench: change gauge custom metrics to trend * server: bench: change gauge custom metrics to trend server: bench: add trend custom metrics for total tokens per second average * server: bench: doc add an option to debug http request * server: bench: filter dataset too short and too long sequences * server: bench: allow to filter out conversation in the dataset based on env variable * server: bench: fix assistant message sent instead of user message * server: bench: fix assistant message sent instead of user message * server : add defrag thold parameter * server: bench: select prompts based on the current iteration id not randomly to make the bench more reproducible --------- Co-authored-by: Georgi Gerganov --- examples/server/bench/README.md | 88 +++++++++++++++++++++++ examples/server/bench/script.js | 120 ++++++++++++++++++++++++++++++++ examples/server/server.cpp | 8 +++ 3 files changed, 216 insertions(+) create mode 100644 examples/server/bench/README.md create mode 100644 examples/server/bench/script.js diff --git a/examples/server/bench/README.md b/examples/server/bench/README.md new file mode 100644 index 000000000..a53ad64d7 --- /dev/null +++ b/examples/server/bench/README.md @@ -0,0 +1,88 @@ +### Server benchmark tools + +Benchmark is using [k6](https://k6.io/). + +##### Install k6 + +Follow instruction from: https://k6.io/docs/get-started/installation/ + +Example for ubuntu: +```shell +snap install k6 +``` + +#### Download a dataset + +This dataset was originally proposed in [vLLM benchmarks](https://github.com/vllm-project/vllm/blob/main/benchmarks/README.md). + +```shell +wget https://huggingface.co/datasets/anon8231489123/ShareGPT_Vicuna_unfiltered/resolve/main/ShareGPT_V3_unfiltered_cleaned_split.json +``` + +#### Download a model +Example for PHI-2 + +```shell +../../../scripts/hf.sh --repo ggml-org/models --file phi-2/ggml-model-q4_0.gguf +``` + +#### Start the server +The server must answer OAI Chat completion requests on `http://localhost:8080/v1` or according to the environment variable `SERVER_BENCH_URL`. + +Example: +```shell +server --host localhost --port 8080 \ + --model ggml-model-q4_0.gguf \ + --cont-batching \ + --metrics \ + --parallel 8 \ + --batch-size 512 \ + --ctx-size 4096 \ + --log-format text \ + -ngl 33 +``` + +#### Run the benchmark + +For 500 chat completions request with 8 concurrent users during maximum 10 minutes, run: +```shell +k6 run script.js --duration 10m --iterations 500 --vus 8 +``` + +The benchmark values can be overridden with: +- `SERVER_BENCH_URL` server url prefix for chat completions, default `http://localhost:8080/v1` +- `SERVER_BENCH_N_PROMPTS` total prompts to randomly select in the benchmark, default `480` +- `SERVER_BENCH_MODEL_ALIAS` model alias to pass in the completion request, default `my-model` +- `SERVER_BENCH_MAX_TOKENS` max tokens to predict, default: `512` +- `SERVER_BENCH_DATASET` path to the benchmark dataset file +- `SERVER_BENCH_MAX_PROMPT_TOKENS` maximum prompt tokens to filter out in the dataset: default `1024` +- `SERVER_BENCH_MAX_CONTEXT` maximum context size of the completions request to filter out in the dataset: prompt + predicted tokens, default `2048` + +Note: the local tokenizer is just a string space split, real number of tokens will differ. + +Or with [k6 options](https://k6.io/docs/using-k6/k6-options/reference/): + +```shell +SERVER_BENCH_N_PROMPTS=500 k6 run script.js --duration 10m --iterations 500 --vus 8 +``` + +To [debug http request](https://k6.io/docs/using-k6/http-debugging/) use `--http-debug="full"`. + +#### Metrics + +Following metrics are available computed from the OAI chat completions response `usage`: +- `llamacpp_tokens_second` Trend of `usage.total_tokens / request duration` +- `llamacpp_prompt_tokens` Trend of `usage.prompt_tokens` +- `llamacpp_prompt_tokens_total_counter` Counter of `usage.prompt_tokens` +- `llamacpp_completion_tokens` Trend of `usage.completion_tokens` +- `llamacpp_completion_tokens_total_counter` Counter of `usage.completion_tokens` +- `llamacpp_completions_truncated_rate` Rate of completions truncated, i.e. if `finish_reason === 'length'` +- `llamacpp_completions_stop_rate` Rate of completions stopped by the model, i.e. if `finish_reason === 'stop'` + +The script will fail if too many completions are truncated, see `llamacpp_completions_truncated_rate`. + +K6 metrics might be compared against [server metrics](../README.md), with: + +```shell +curl http://localhost:8080/metrics +``` diff --git a/examples/server/bench/script.js b/examples/server/bench/script.js new file mode 100644 index 000000000..a4f5ac5ab --- /dev/null +++ b/examples/server/bench/script.js @@ -0,0 +1,120 @@ +import http from 'k6/http' +import {check, sleep} from 'k6' +import {SharedArray} from 'k6/data' +import {Counter, Rate, Trend} from 'k6/metrics' +import exec from 'k6/execution'; + +// Server chat completions prefix +const server_url = __ENV.SERVER_BENCH_URL ? __ENV.SERVER_BENCH_URL : 'http://localhost:8080/v1' + +// Number of total prompts in the dataset - default 10m / 10 seconds/request * number of users +const n_prompt = __ENV.SERVER_BENCH_N_PROMPTS ? parseInt(__ENV.SERVER_BENCH_N_PROMPTS) : 600 / 10 * 8 + +// Model name to request +const model = __ENV.SERVER_BENCH_MODEL_ALIAS ? __ENV.SERVER_BENCH_MODEL_ALIAS : 'my-model' + +// Dataset path +const dataset_path = __ENV.SERVER_BENCH_DATASET ? __ENV.SERVER_BENCH_DATASET : './ShareGPT_V3_unfiltered_cleaned_split.json' + +// Max tokens to predict +const max_tokens = __ENV.SERVER_BENCH_MAX_TOKENS ? parseInt(__ENV.SERVER_BENCH_MAX_TOKENS) : 512 + +// Max prompt tokens +const n_prompt_tokens = __ENV.SERVER_BENCH_MAX_PROMPT_TOKENS ? parseInt(__ENV.SERVER_BENCH_MAX_PROMPT_TOKENS) : 1024 + +// Max slot context +const n_ctx_slot = __ENV.SERVER_BENCH_MAX_CONTEXT ? parseInt(__ENV.SERVER_BENCH_MAX_CONTEXT) : 2048 + +export function setup() { + console.info(`Benchmark config: server_url=${server_url} n_prompt=${n_prompt} model=${model} dataset_path=${dataset_path} max_tokens=${max_tokens}`) +} + +const data = new SharedArray('conversations', function () { + const tokenizer = (message) => message.split(/[\s,'".?]/) + + return JSON.parse(open(dataset_path)) + // Filter out the conversations with less than 2 turns. + .filter(data => data["conversations"].length >= 2) + .filter(data => data["conversations"][0]["from"] === "human") + .map(data => { + return { + prompt: data["conversations"][0]["value"], + n_prompt_tokens: tokenizer(data["conversations"][0]["value"]).length, + n_completion_tokens: tokenizer(data["conversations"][1]["value"]).length, + } + }) + // Filter out too short sequences + .filter(conv => conv.n_prompt_tokens >= 4 && conv.n_completion_tokens >= 4) + // Filter out too long sequences. + .filter(conv => conv.n_prompt_tokens <= n_prompt_tokens && conv.n_prompt_tokens + conv.n_completion_tokens <= n_ctx_slot) + // Keep only first n prompts + .slice(0, n_prompt) +}) + +const llamacpp_prompt_tokens = new Trend('llamacpp_prompt_tokens') +const llamacpp_completion_tokens = new Trend('llamacpp_completion_tokens') +const llamacpp_tokens_second = new Trend('llamacpp_tokens_second') + +const llamacpp_prompt_tokens_total_counter = new Counter('llamacpp_prompt_tokens_total_counter') +const llamacpp_completion_tokens_total_counter = new Counter('llamacpp_completion_tokens_total_counter') + +const llamacpp_completions_truncated_rate = new Rate('llamacpp_completions_truncated_rate') +const llamacpp_completions_stop_rate = new Rate('llamacpp_completions_stop_rate') + +export const options = { + thresholds: { + llamacpp_completions_truncated_rate: [ + // more than 80% of truncated input will abort the test + {threshold: 'rate < 0.8', abortOnFail: true, delayAbortEval: '1m'}, + ], + }, + duration: '10m', + vus: 8, +} + +export default function () { + const conversation = data[exec.scenario.iterationInInstance % data.length] + const payload = { + "messages": [ + { + "role": "system", + "content": "You are ChatGPT, an AI assistant.", + }, + { + "role": "user", + "content": conversation.prompt, + } + ], + "model": model, + "stream": false, + "max_tokens": max_tokens + } + + const body = JSON.stringify(payload) + + let res = http.post(`${server_url}/chat/completions`, body, { + headers: {'Content-Type': 'application/json'}, + timeout: '300s' + }) + + check(res, {'success completion': (r) => r.status === 200}) + + if (res.status === 200) { + const completions = res.json() + + llamacpp_prompt_tokens.add(completions.usage.prompt_tokens) + llamacpp_prompt_tokens_total_counter.add(completions.usage.prompt_tokens) + + llamacpp_completion_tokens.add(completions.usage.completion_tokens) + llamacpp_completion_tokens_total_counter.add(completions.usage.completion_tokens) + + llamacpp_completions_truncated_rate.add(completions.choices[0].finish_reason === 'length') + llamacpp_completions_stop_rate.add(completions.choices[0].finish_reason === 'stop') + + llamacpp_tokens_second.add(completions.usage.total_tokens / res.timings.duration * 1.e3) + } else { + console.error(`response: ${res.body} request=${payload}`) + } + + sleep(0.3) +} diff --git a/examples/server/server.cpp b/examples/server/server.cpp index b14cca61b..c7d3ed01b 100644 --- a/examples/server/server.cpp +++ b/examples/server/server.cpp @@ -2133,6 +2133,8 @@ static void server_print_usage(const char * argv0, const gpt_params & params, co printf(" --yarn-beta-slow N YaRN: high correction dim or alpha (default: %.1f)\n", params.yarn_beta_slow); printf(" --yarn-beta-fast N YaRN: low correction dim or beta (default: %.1f)\n", params.yarn_beta_fast); printf(" --pooling {none,mean,cls} pooling type for embeddings, use model default if unspecified\n"); + printf(" -dt N, --defrag-thold N\n"); + printf(" KV cache defragmentation threshold (default: %.1f, < 0 - disabled)\n", params.defrag_thold); printf(" -b N, --batch-size N batch size for prompt processing (default: %d)\n", params.n_batch); printf(" --memory-f32 use f32 instead of f16 for memory key+value (default: disabled)\n"); printf(" not recommended: doubles context memory required and no measurable increase in quality\n"); @@ -2355,6 +2357,12 @@ static void server_params_parse(int argc, char ** argv, server_params & sparams, else if (value == "mean") { params.pooling_type = LLAMA_POOLING_TYPE_MEAN; } else if (value == "cls") { params.pooling_type = LLAMA_POOLING_TYPE_CLS; } else { invalid_param = true; break; } + } else if (arg == "--defrag-thold" || arg == "-dt") { + if (++i >= argc) { + invalid_param = true; + break; + } + params.defrag_thold = std::stof(argv[i]); } else if (arg == "--threads" || arg == "-t") { if (++i >= argc) { From c78541479cf835dd9eb568ccd9a2083198a7203d Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Sun, 10 Mar 2024 16:43:08 +0200 Subject: [PATCH 579/812] nix: update flake.lock (#5969) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/1536926ef5621b09bba54035ae2bb6d806d72ac8' (2024-02-29) → 'github:NixOS/nixpkgs/9df3e30ce24fd28c7b3e2de0d986769db5d6225d' (2024-03-06) Co-authored-by: github-actions[bot] --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index b1b091656..f9865d5e4 100644 --- a/flake.lock +++ b/flake.lock @@ -20,11 +20,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1709237383, - "narHash": "sha256-cy6ArO4k5qTx+l5o+0mL9f5fa86tYUX3ozE1S+Txlds=", + "lastModified": 1709703039, + "narHash": "sha256-6hqgQ8OK6gsMu1VtcGKBxKQInRLHtzulDo9Z5jxHEFY=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "1536926ef5621b09bba54035ae2bb6d806d72ac8", + "rev": "9df3e30ce24fd28c7b3e2de0d986769db5d6225d", "type": "github" }, "original": { From 2960eae847f8dbde23be6d170a61bcf44ebf32de Mon Sep 17 00:00:00 2001 From: Clint Herron Date: Sun, 10 Mar 2024 11:17:43 -0400 Subject: [PATCH 580/812] grammar : verify parsed state (#5950) --- common/grammar-parser.cpp | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/common/grammar-parser.cpp b/common/grammar-parser.cpp index bf89a96f3..2a1301569 100644 --- a/common/grammar-parser.cpp +++ b/common/grammar-parser.cpp @@ -278,6 +278,22 @@ namespace grammar_parser { while (*pos) { pos = parse_rule(state, pos); } + // Validate the state to ensure that all rules are defined + for (const auto & rule : state.rules) { + for (const auto & elem : rule) { + if (elem.type == LLAMA_GRETYPE_RULE_REF) { + // Ensure that the rule at that location exists + if (elem.value >= state.rules.size() || state.rules[elem.value].empty()) { + // Get the name of the rule that is missing + for (const auto & kv : state.symbol_ids) { + if (kv.second == elem.value) { + throw std::runtime_error("Undefined rule identifier '" + kv.first + "'"); + } + } + } + } + } + } return state; } catch (const std::exception & err) { fprintf(stderr, "%s: error parsing grammar: %s\n", __func__, err.what()); From bcebd7dbf62fd7b293d5ed089023e4e733269c71 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?DAN=E2=84=A2?= Date: Sun, 10 Mar 2024 11:56:30 -0400 Subject: [PATCH 581/812] llama : add support for GritLM (#5959) * add gritlm example * gritlm results match * tabs to spaces * comment out debug printing * rebase to new embed * gritlm embeddings are back babeee * add to gitignore * allow to toggle embedding mode * Clean-up GritLM sample code. * Fix types. * Flush stdout and output ending newline if streaming. * mostly style fixes; correct KQ_mask comment * add causal_attn flag to llama_cparams * gritml : minor * llama : minor --------- Co-authored-by: Douglas Hanley Co-authored-by: Georgi Gerganov --- .gitignore | 1 + Makefile | 6 +- examples/CMakeLists.txt | 1 + examples/gritlm/CMakeLists.txt | 5 + examples/gritlm/gritlm.cpp | 229 +++++++++++++++++++++++++++++++++ llama.cpp | 25 +++- llama.h | 4 + 7 files changed, 267 insertions(+), 4 deletions(-) create mode 100644 examples/gritlm/CMakeLists.txt create mode 100644 examples/gritlm/gritlm.cpp diff --git a/.gitignore b/.gitignore index 62b6b8b1a..d28f4d1b8 100644 --- a/.gitignore +++ b/.gitignore @@ -45,6 +45,7 @@ models-mnt /embedding /gguf /gguf-llama-simple +/gritlm /imatrix /infill /libllama.so diff --git a/Makefile b/Makefile index d809b8b3b..6492da8a0 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ BUILD_TARGETS = \ main quantize quantize-stats perplexity imatrix embedding vdot q8dot train-text-from-scratch convert-llama2c-to-ggml \ simple batched batched-bench save-load-state server gguf llama-bench libllava.a llava-cli baby-llama beam-search \ - speculative infill tokenize benchmark-matmult parallel finetune export-lora lookahead lookup passkey tests/test-c.o + speculative infill tokenize benchmark-matmult parallel finetune export-lora lookahead lookup passkey gritlm tests/test-c.o # Binaries only useful for tests TEST_TARGETS = \ @@ -724,6 +724,10 @@ embedding: examples/embedding/embedding.cpp ggml.o llama.o $(C $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) +gritlm: examples/gritlm/gritlm.cpp ggml.o llama.o $(COMMON_DEPS) $(OBJS) + $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) + $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) + save-load-state: examples/save-load-state/save-load-state.cpp ggml.o llama.o $(COMMON_DEPS) $(OBJS) $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index 653abc73a..e762cf8b9 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -20,6 +20,7 @@ else() add_subdirectory(convert-llama2c-to-ggml) add_subdirectory(embedding) add_subdirectory(finetune) + add_subdirectory(gritlm) add_subdirectory(infill) add_subdirectory(llama-bench) add_subdirectory(llava) diff --git a/examples/gritlm/CMakeLists.txt b/examples/gritlm/CMakeLists.txt new file mode 100644 index 000000000..ac4a5ae79 --- /dev/null +++ b/examples/gritlm/CMakeLists.txt @@ -0,0 +1,5 @@ +set(TARGET gritlm) +add_executable(${TARGET} gritlm.cpp) +install(TARGETS ${TARGET} RUNTIME) +target_link_libraries(${TARGET} PRIVATE common llama ${CMAKE_THREAD_LIBS_INIT}) +target_compile_features(${TARGET} PRIVATE cxx_std_11) diff --git a/examples/gritlm/gritlm.cpp b/examples/gritlm/gritlm.cpp new file mode 100644 index 000000000..3d4b085d6 --- /dev/null +++ b/examples/gritlm/gritlm.cpp @@ -0,0 +1,229 @@ +#include "common.h" +#include "llama.h" + +#include +#include + +// #define GRIT_DEBUG + +static float dot_product(const std::vector & v1, const std::vector & v2) { + float dot = 0.0f; + for (uint64_t i = 0; i < v1.size(); ++i) { + dot += v1[i] * v2[i]; + } + return dot; +} + +static float norm(const std::vector & v) { + return std::sqrt(dot_product(v, v)); +} + +static float cosine_similarity(const std::vector & v1, const std::vector & v2) { + return dot_product(v1, v2) / (norm(v1) * norm(v2)); +} + +static std::vector> encode(llama_context * ctx, const std::vector & sentences, const std::string & instruction) { + std::vector> result; + + const llama_model * mdl = llama_get_model(ctx); + + llama_batch batch = llama_batch_init(llama_n_batch(ctx), 0, 1); + + for (uint64_t i = 0; i < sentences.size(); i++) { + llama_batch_clear(batch); + + const std::string input_string = instruction + sentences[i]; + + std::vector inputs = llama_tokenize(mdl, input_string, true, false); + + const int32_t n_toks = inputs.size(); + + // GritLM seems to have EOS = "" + // https://github.com/ContextualAI/gritlm/blob/92025b16534712b31b3c4aaaf069350e222bd5f8/gritlm/gritlm.py#L18 + // inputs.push_back(llama_token_eos(mdl)); + + // we want to ignore instruction tokens for mean pooling + const int32_t n_inst = llama_tokenize(mdl, instruction, true, false).size(); + +#ifdef GRIT_DEBUG + // debug tokens - should be matching as referenced in the GritLM sample + std::for_each(inputs.begin(), inputs.end(), [&ctx](llama_token t) { + std::printf("[%u:%s]", t, llama_token_to_piece(ctx, t).c_str()); + }); + std::printf("\n"); +#endif + + // add input to batch (this increments n_tokens) + for (int32_t j = 0; j < n_toks; j++) { + llama_batch_add(batch, inputs[j], j, { 0 }, j >= n_inst); + } + + // clear previous kv_cache values (irrelevant for embeddings) + llama_kv_cache_clear(ctx); + llama_set_causal_attn(ctx, false); + + // run model + llama_decode(ctx, batch); + + // get embedding dimensions + uint64_t n_embd = llama_n_embd(mdl); + + // allocate embedding output + std::vector emb_unorm(n_embd, 0.0f); + + // sum up all token embeddings + for (int32_t k = n_inst; k < n_toks; k++) { + float * emb = llama_get_embeddings_ith(ctx, k); + for (uint64_t j = 0; j < n_embd; j++) { + emb_unorm[j] += emb[j]; + } + } + + // divide by number of tokens (mean pooling) + { + const uint64_t n_sent = n_toks - n_inst; + + for (uint64_t j = 0; j < n_embd; j++) { + emb_unorm[j] /= n_sent; + } + } + + std::vector emb_norm(emb_unorm.size()); + llama_embd_normalize(emb_unorm.data(), emb_norm.data(), n_embd); + result.push_back(emb_norm); + +#ifdef GRIT_DEBUG + // print out emb_norm + std::printf("embedding %ld: ", i); + for (uint64_t j = 0; j < n_embd; j++) { + std::printf("%.5f ", emb_norm[j]); + } + std::printf("\n\n"); +#endif + } + + llama_batch_free(batch); + + return result; +} + +static std::string generate(llama_context * ctx, const std::string & prompt, bool stream) { + std::string result; + + const llama_model * mdl = llama_get_model(ctx); + llama_token eos_token = llama_token_eos(mdl); + + llama_kv_cache_clear(ctx); + llama_set_causal_attn(ctx, true); + llama_batch bat = llama_batch_init(llama_n_batch(ctx), 0, 1); + + std::vector inputs = llama_tokenize(mdl, prompt, false, true); + int32_t i_current_token = 0; + + while (true) { + llama_batch_clear(bat); + auto n_inputs = (int32_t)inputs.size(); + for (int32_t i = 0; i < n_inputs; i++) { + llama_batch_add(bat, inputs[i], i_current_token++, { 0 }, i == n_inputs - 1); + } + inputs.clear(); + + llama_decode(ctx, bat); + auto logits = llama_get_logits_ith(ctx, bat.n_tokens - 1); + + auto candidates = std::vector(llama_n_vocab(mdl)); + auto n_candidates = (int32_t)candidates.size(); + for (int32_t token = 0; token < n_candidates; token++) { + candidates[token] = llama_token_data{ token, logits[token], 0.0f }; + } + auto candidates_p = llama_token_data_array{ candidates.data(), candidates.size(), false }; + + llama_token token = llama_sample_token_greedy(ctx, &candidates_p); + if (token == eos_token) { + break; + } + + std::string piece = llama_token_to_piece(ctx, token); + if (stream) { + std::printf("%s", piece.c_str()); + std::fflush(stdout); + } + + inputs.push_back(token); + + result += piece; + } + + if (stream) { + std::printf("\n"); + } + + llama_batch_free(bat); + + return result; +} + +static std::string gritlm_instruction(const std::string & instruction) { + return !instruction.empty() ? "<|user|>\n" + instruction + "\n<|embed|>\n" : "<|embed|>\n"; +} + +int main(int argc, char * argv[]) { + gpt_params params; + if (!gpt_params_parse(argc, argv, params)) { + return 1; + } + + llama_model_params mparams = llama_model_params_from_gpt_params(params); + llama_context_params cparams = llama_context_params_from_gpt_params(params); + + llama_backend_init(); + + llama_model * mdl = llama_load_model_from_file(params.model.c_str(), mparams); + + // create new context - set to embedding mode + cparams.embeddings = true; + llama_context * ctx = llama_new_context_with_model(mdl, cparams); + + // ### Embedding/Representation ### + // samples taken from: https://github.com/ContextualAI/gritlm#basic + { + const std::string instruction = "Given a scientific paper title, retrieve the paper's abstract"; + + const std::vector queries = { + "Bitcoin: A Peer-to-Peer Electronic Cash System", + "Generative Representational Instruction Tuning", + }; + + const std::vector documents = { + "A purely peer-to-peer version of electronic cash would allow online payments to be sent directly from one party to another without going through a financial institution. Digital signatures provide part of the solution, but the main benefits are lost if a trusted third party is still required to prevent double-spending. We propose a solution to the double-spending problem using a peer-to-peer network. The network timestamps transactions by hashing them into an ongoing chain of hash-based proof-of-work, forming a record that cannot be changed without redoing the proof-of-work. The longest chain not only serves as proof of the sequence of events witnessed, but proof that it came from the largest pool of CPU power. As long as a majority of CPU power is controlled by nodes that are not cooperating to attack the network, they'll generate the longest chain and outpace attackers. The network itself requires minimal structure. Messages are broadcast on a best effort basis, and nodes can leave and rejoin the network at will, accepting the longest proof-of-work chain as proof of what happened while they were gone.", + "All text-based language problems can be reduced to either generation or embedding. Current models only perform well at one or the other. We introduce generative representational instruction tuning (GRIT) whereby a large language model is trained to handle both generative and embedding tasks by distinguishing between them through instructions. Compared to other open models, our resulting GritLM 7B sets a new state of the art on the Massive Text Embedding Benchmark (MTEB) and outperforms all models up to its size on a range of generative tasks. By scaling up further, GritLM 8X7B outperforms all open generative language models that we tried while still being among the best embedding models. Notably, we find that GRIT matches training on only generative or embedding data, thus we can unify both at no performance loss. Among other benefits, the unification via GRIT speeds up Retrieval-Augmented Generation (RAG) by > 60% for long documents, by no longer requiring separate retrieval and generation models. Models, code, etc. are freely available at https://github.com/ContextualAI/gritlm.", + }; + + // No need to add instruction for retrieval documents + const std::vector> d_rep = encode(ctx, documents, gritlm_instruction("")); + const std::vector> q_rep = encode(ctx, queries, gritlm_instruction(instruction)); + + const float cosine_sim_q0_d0 = cosine_similarity(q_rep[0], d_rep[0]); + const float cosine_sim_q0_d1 = cosine_similarity(q_rep[0], d_rep[1]); + const float cosine_sim_q1_d0 = cosine_similarity(q_rep[1], d_rep[0]); + const float cosine_sim_q1_d1 = cosine_similarity(q_rep[1], d_rep[1]); + + std::printf("Cosine similarity between \"%.50s\" and \"%.50s\" is: %.3f\n", queries[0].c_str(), documents[0].c_str(), cosine_sim_q0_d0); + std::printf("Cosine similarity between \"%.50s\" and \"%.50s\" is: %.3f\n", queries[0].c_str(), documents[1].c_str(), cosine_sim_q0_d1); + std::printf("Cosine similarity between \"%.50s\" and \"%.50s\" is: %.3f\n", queries[1].c_str(), documents[0].c_str(), cosine_sim_q1_d0); + std::printf("Cosine similarity between \"%.50s\" and \"%.50s\" is: %.3f\n", queries[1].c_str(), documents[1].c_str(), cosine_sim_q1_d1); + } + + // ### Generation ### + // GritLM models are not finetuned with system prompts, as you can just include system-like instructions together with your user instruction + { + const std::string prompt = "<|user|>\nPlease write me a poem about my recent hike of Mt. Fuji at midnight in the style of Shakespeare.\n<|assistant|>\n"; + std::string response = generate(ctx, prompt, true); + } + + llama_free(ctx); + llama_free_model(mdl); + llama_backend_free(); + + return 0; +} diff --git a/llama.cpp b/llama.cpp index b19616e8f..249442166 100644 --- a/llama.cpp +++ b/llama.cpp @@ -1744,6 +1744,7 @@ struct llama_cparams { float defrag_thold; bool embeddings; + bool causal_attn; bool offload_kqv; enum llama_pooling_type pooling_type; @@ -3939,6 +3940,7 @@ static void llm_load_print_meta(llama_model_loader & ml, llama_model & model) { LLAMA_LOG_INFO("%s: n_ff = %u\n", __func__, hparams.n_ff); LLAMA_LOG_INFO("%s: n_expert = %u\n", __func__, hparams.n_expert); LLAMA_LOG_INFO("%s: n_expert_used = %u\n", __func__, hparams.n_expert_used); + LLAMA_LOG_INFO("%s: causal attm = %d\n", __func__, hparams.causal_attn); LLAMA_LOG_INFO("%s: pooling type = %d\n", __func__, hparams.pooling_type); LLAMA_LOG_INFO("%s: rope type = %d\n", __func__, hparams.rope_type); LLAMA_LOG_INFO("%s: rope scaling = %s\n", __func__, rope_scaling_type); @@ -8532,7 +8534,13 @@ static void llama_set_inputs(llama_context & lctx, const llama_batch & batch) { ggml_backend_tensor_set(lctx.inp_pos, batch.pos, 0, n_tokens*ggml_element_size(lctx.inp_pos)); } - if (hparams.causal_attn) { + GGML_ASSERT( + (hparams.causal_attn || !cparams.causal_attn) && + "non-causal attention with generative models is not supported" + ); + + // NOTE: hparams.causal_attn indicates the model is capable of generation and uses the kv cache. + if (cparams.causal_attn) { const int64_t n_kv = kv_self.n; const int64_t n_tokens = batch.n_tokens; @@ -8560,8 +8568,9 @@ static void llama_set_inputs(llama_context & lctx, const llama_batch & batch) { } } } else { - // non-causal attention attends only the tokens within the batch (i.e. the KV cache is not used) + // when using kv cache, the mask needs to match the kv cache size const int64_t n_tokens = batch.n_tokens; + const int64_t n_stride = hparams.causal_attn ? kv_self.n : n_tokens; assert(ggml_backend_buffer_is_host(lctx.inp_KQ_mask->buffer)); @@ -8580,7 +8589,11 @@ static void llama_set_inputs(llama_context & lctx, const llama_batch & batch) { } } - data[h*(n_tokens*n_tokens) + j*n_tokens + i] = f; + data[h*(n_tokens*n_tokens) + j*n_stride + i] = f; + } + + for (int i = n_tokens; i < n_stride; ++i) { + data[h*(n_tokens*n_tokens) + j*n_stride + i] = -INFINITY; } } } @@ -12733,6 +12746,8 @@ struct llama_context * llama_new_context_with_model( cparams.yarn_ext_factor = rope_scaling_type == LLAMA_ROPE_SCALING_TYPE_YARN ? 1.0f : 0.0f; } + cparams.causal_attn = hparams.causal_attn; + if (cparams.pooling_type == LLAMA_POOLING_TYPE_UNSPECIFIED) { if (hparams.pooling_type == LLAMA_POOLING_TYPE_UNSPECIFIED) { cparams.pooling_type = LLAMA_POOLING_TYPE_NONE; @@ -13767,6 +13782,10 @@ void llama_set_abort_callback(struct llama_context * ctx, bool (*abort_callback) ctx->abort_callback_data = abort_callback_data; } +void llama_set_causal_attn(struct llama_context * ctx, bool causal_attn) { + ctx->cparams.causal_attn = causal_attn; +} + struct llama_batch llama_batch_get_one( llama_token * tokens, int32_t n_tokens, diff --git a/llama.h b/llama.h index 7a107c7f3..c8e05aadd 100644 --- a/llama.h +++ b/llama.h @@ -643,6 +643,10 @@ extern "C" { // n_threads_batch is the number of threads used for prompt and batch processing (multiple tokens) LLAMA_API void llama_set_n_threads(struct llama_context * ctx, uint32_t n_threads, uint32_t n_threads_batch); + // Set whether to use causal attention or not + // If set to true, the model will only attend to the past tokens + LLAMA_API void llama_set_causal_attn(struct llama_context * ctx, bool causal_attn); + // Set abort callback LLAMA_API void llama_set_abort_callback(struct llama_context * ctx, ggml_abort_callback abort_callback, void * abort_callback_data); From fa8a809a9119411bc9c3f00026550d0343f4d9b7 Mon Sep 17 00:00:00 2001 From: Pierrick Hymbert Date: Sun, 10 Mar 2024 18:17:47 +0100 Subject: [PATCH 582/812] server: ci: windows build and tests (#5968) * server: ci: windows build and tests * server: ci: remove tmp push branch * server: ci: EOF EOL * Use builti Co-authored-by: Jared Van Bortel * server: tests: server graceful shutdown, then kill, then hard kill * server: tests: remove python2 unicode string * server: tests: remove wrong comment on server starting, close_fds is always true * server: tests: server kill, if pid exists * server: tests: remove dependency to killall * server: tests: ci windows: pid exists better handling --------- Co-authored-by: Jared Van Bortel --- .github/workflows/server.yml | 46 ++++- examples/server/tests/features/environment.py | 66 +++++-- examples/server/tests/features/server.feature | 2 +- examples/server/tests/features/steps/steps.py | 176 ++++++++++-------- 4 files changed, 191 insertions(+), 99 deletions(-) diff --git a/.github/workflows/server.yml b/.github/workflows/server.yml index f9aeefaa8..e385e03f3 100644 --- a/.github/workflows/server.yml +++ b/.github/workflows/server.yml @@ -47,6 +47,8 @@ jobs: - name: Clone id: checkout uses: actions/checkout@v3 + with: + fetch-depth: 0 - name: Dependencies id: depends @@ -58,7 +60,6 @@ jobs: cmake \ python3-pip \ wget \ - psmisc \ language-pack-en - name: Build @@ -90,3 +91,46 @@ jobs: run: | cd examples/server/tests PORT=8888 ./tests.sh --stop --no-skipped --no-capture --tags slow + + + server-windows: + runs-on: windows-latest + + steps: + - name: Clone + id: checkout + uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Build + id: cmake_build + run: | + mkdir build + cd build + cmake .. -DLLAMA_BUILD_SERVER=ON -DCMAKE_BUILD_TYPE=Release ; + cmake --build . --config Release -j ${env:NUMBER_OF_PROCESSORS} --target server + + - name: Python setup + id: setup_python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Tests dependencies + id: test_dependencies + run: | + pip install -r examples/server/tests/requirements.txt + + - name: Tests + id: server_integration_tests + run: | + cd examples/server/tests + behave.exe --summary --stop --no-capture --exclude 'issues|wrong_usages|passkey' --tags llama.cpp + + - name: Slow tests + id: server_integration_tests_slow + if: ${{ github.event.schedule != '' || github.event.inputs.slow_tests == 'true' }} + run: | + cd examples/server/tests + behave.exe --stop --no-skipped --no-capture --tags slow diff --git a/examples/server/tests/features/environment.py b/examples/server/tests/features/environment.py index 9fd330db6..8ad987e1b 100644 --- a/examples/server/tests/features/environment.py +++ b/examples/server/tests/features/environment.py @@ -1,9 +1,10 @@ +import errno import os import socket import subprocess import time from contextlib import closing -from signal import SIGKILL +import signal def before_scenario(context, scenario): @@ -29,44 +30,71 @@ def after_scenario(context, scenario): for line in f: print(line) if not is_server_listening(context.server_fqdn, context.server_port): - print("\x1b[33;101mERROR: Server stopped listening\x1b[0m") + print("\x1b[33;101mERROR: Server stopped listening\x1b[0m\n") if not pid_exists(context.server_process.pid): assert False, f"Server not running pid={context.server_process.pid} ..." - print(f"stopping server pid={context.server_process.pid} ...") - context.server_process.kill() + server_graceful_shutdown(context) + # Wait few for socket to free up time.sleep(0.05) attempts = 0 - while is_server_listening(context.server_fqdn, context.server_port): - print(f"stopping server pid={context.server_process.pid} ...") - os.kill(context.server_process.pid, SIGKILL) + while pid_exists(context.server_process.pid) or is_server_listening(context.server_fqdn, context.server_port): + server_kill(context) time.sleep(0.1) attempts += 1 if attempts > 5: - print(f"Server dangling exits, killing all {context.server_path} ...") - process = subprocess.run(['killall', '-9', context.server_path], - stderr=subprocess.PIPE, - universal_newlines=True) - print(process) + server_kill_hard(context) + + +def server_graceful_shutdown(context): + print(f"shutting down server pid={context.server_process.pid} ...\n") + if os.name == 'nt': + os.kill(context.server_process.pid, signal.CTRL_C_EVENT) + else: + os.kill(context.server_process.pid, signal.SIGINT) + + +def server_kill(context): + print(f"killing server pid={context.server_process.pid} ...\n") + context.server_process.kill() + + +def server_kill_hard(context): + pid = context.server_process.pid + path = context.server_path + + print(f"Server dangling exits, hard killing force {pid}={path}...\n") + if os.name == 'nt': + process = subprocess.check_output(['taskkill', '/F', '/pid', str(pid)]).decode() + print(process) + else: + os.kill(-pid, signal.SIGKILL) def is_server_listening(server_fqdn, server_port): with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as sock: result = sock.connect_ex((server_fqdn, server_port)) - return result == 0 + _is_server_listening = result == 0 + if _is_server_listening: + print(f"server is listening on {server_fqdn}:{server_port}...\n") + return _is_server_listening def pid_exists(pid): """Check whether pid exists in the current process table.""" - import errno if pid < 0: return False - try: - os.kill(pid, 0) - except OSError as e: - return e.errno == errno.EPERM + if os.name == 'nt': + output = subprocess.check_output(['TASKLIST', '/FI', f'pid eq {pid}']).decode() + print(output) + return "No tasks are running" not in output else: - return True + try: + os.kill(pid, 0) + except OSError as e: + return e.errno == errno.EPERM + else: + return True diff --git a/examples/server/tests/features/server.feature b/examples/server/tests/features/server.feature index aa132fa34..5014f326d 100644 --- a/examples/server/tests/features/server.feature +++ b/examples/server/tests/features/server.feature @@ -47,7 +47,7 @@ Feature: llama.cpp server Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum. """ And a completion request with no api error - Then 64 tokens are predicted matching fun|Annaks|popcorns + Then 64 tokens are predicted matching fun|Annaks|popcorns|pictry And the completion is truncated And 109 prompt tokens are processed diff --git a/examples/server/tests/features/steps/steps.py b/examples/server/tests/features/steps/steps.py index 142048509..4e81a255a 100644 --- a/examples/server/tests/features/steps/steps.py +++ b/examples/server/tests/features/steps/steps.py @@ -18,7 +18,7 @@ from huggingface_hub import hf_hub_download from prometheus_client import parser -@step(u"a server listening on {server_fqdn}:{server_port}") +@step("a server listening on {server_fqdn}:{server_port}") def step_server_config(context, server_fqdn, server_port): context.server_fqdn = server_fqdn context.server_port = int(server_port) @@ -57,24 +57,24 @@ def step_server_config(context, server_fqdn, server_port): context.prompts = [] -@step(u'a model file {hf_file} from HF repo {hf_repo}') +@step('a model file {hf_file} from HF repo {hf_repo}') def step_download_hf_model(context, hf_file, hf_repo): context.model_file = hf_hub_download(repo_id=hf_repo, filename=hf_file) if context.debug: print(f"model file: {context.model_file}\n") -@step(u'a model alias {model_alias}') +@step('a model alias {model_alias}') def step_model_alias(context, model_alias): context.model_alias = model_alias -@step(u'{seed:d} as server seed') +@step('{seed:d} as server seed') def step_seed(context, seed): context.server_seed = seed -@step(u'{ngl:d} GPU offloaded layers') +@step('{ngl:d} GPU offloaded layers') def step_n_gpu_layer(context, ngl): if 'N_GPU_LAYERS' in os.environ: new_ngl = int(os.environ['N_GPU_LAYERS']) @@ -84,37 +84,37 @@ def step_n_gpu_layer(context, ngl): context.n_gpu_layer = ngl -@step(u'{n_ctx:d} KV cache size') +@step('{n_ctx:d} KV cache size') def step_n_ctx(context, n_ctx): context.n_ctx = n_ctx -@step(u'{n_slots:d} slots') +@step('{n_slots:d} slots') def step_n_slots(context, n_slots): context.n_slots = n_slots -@step(u'{n_predict:d} server max tokens to predict') +@step('{n_predict:d} server max tokens to predict') def step_server_n_predict(context, n_predict): context.n_server_predict = n_predict -@step(u'continuous batching') +@step('continuous batching') def step_server_continuous_batching(context): context.server_continuous_batching = True -@step(u'embeddings extraction') +@step('embeddings extraction') def step_server_embeddings(context): context.server_embeddings = True -@step(u'prometheus compatible metrics exposed') +@step('prometheus compatible metrics exposed') def step_server_metrics(context): context.server_metrics = True -@step(u"the server is starting") +@step("the server is starting") def step_start_server(context): start_server_background(context) attempts = 0 @@ -131,7 +131,7 @@ def step_start_server(context): time.sleep(0.1) -@step(u"the server is {expecting_status}") +@step("the server is {expecting_status}") @async_run_until_complete async def step_wait_for_the_server_to_be_started(context, expecting_status): match expecting_status: @@ -160,7 +160,7 @@ async def step_wait_for_the_server_to_be_started(context, expecting_status): assert False, "unknown status" -@step(u'all slots are {expected_slot_status_string}') +@step('all slots are {expected_slot_status_string}') @async_run_until_complete async def step_all_slots_status(context, expected_slot_status_string): match expected_slot_status_string: @@ -176,7 +176,7 @@ async def step_all_slots_status(context, expected_slot_status_string): await request_slots_status(context, expected_slots) -@step(u'a completion request with {api_error} api error') +@step('a completion request with {api_error} api error') @async_run_until_complete async def step_request_completion(context, api_error): expect_api_error = api_error == 'raised' @@ -194,133 +194,133 @@ async def step_request_completion(context, api_error): assert completion == 401, f"completion must be an 401 status code: {completion}" -@step(u'{predicted_n:d} tokens are predicted matching {re_content}') +@step('{predicted_n:d} tokens are predicted matching {re_content}') def step_n_tokens_predicted_with_content(context, predicted_n, re_content): context.completion = context.tasks_result.pop() assert_n_tokens_predicted(context.completion, predicted_n, re_content) -@step(u'{predicted_n:d} tokens are predicted') +@step('{predicted_n:d} tokens are predicted') def step_n_tokens_predicted(context, predicted_n): context.completion = context.tasks_result.pop() assert_n_tokens_predicted(context.completion, predicted_n) -@step(u'the completion is truncated') +@step('the completion is truncated') def step_assert_completion_truncated(context): step_assert_completion_truncated(context, '') -@step(u'the completion is {truncated} truncated') +@step('the completion is {truncated} truncated') def step_assert_completion_truncated(context, truncated): truncated = truncated != "not" assert context.completion['truncated'] == truncated, f'{context.completion}' -@step(u'{n_prompt:d} prompt tokens are processed') +@step('{n_prompt:d} prompt tokens are processed') def step_impl(context, n_prompt): assert n_prompt < 0 or n_prompt == context.completion['timings']['prompt_n'], f"n_prompt={context.completion['timings']['prompt_n']}" -@step(u'a user prompt {user_prompt}') +@step('a user prompt {user_prompt}') def step_user_prompt(context, user_prompt): context.prompts.append(user_prompt) context.n_prompts = len(context.prompts) -@step(u'a system prompt {system_prompt}') +@step('a system prompt {system_prompt}') def step_system_prompt(context, system_prompt): context.system_prompt = system_prompt -@step(u'a model {model}') +@step('a model {model}') def step_model(context, model): context.model = model -@step(u'{max_tokens:d} max tokens to predict') +@step('{max_tokens:d} max tokens to predict') def step_max_tokens(context, max_tokens): context.n_predict = max_tokens -@step(u'streaming is {enable_streaming}') +@step('streaming is {enable_streaming}') def step_streaming(context, enable_streaming): context.enable_streaming = enable_streaming == 'enabled' -@step(u'a user api key {user_api_key}') +@step('a user api key {user_api_key}') def step_user_api_key(context, user_api_key): context.user_api_key = user_api_key -@step(u'no user api key') +@step('no user api key') def step_no_user_api_key(context): context.user_api_key = None -@step(u'a user api key ') +@step('a user api key ') def step_no_user_api_key_space(context): context.user_api_key = None -@step(u'a server api key {server_api_key}') +@step('a server api key {server_api_key}') def step_server_api_key(context, server_api_key): context.server_api_key = server_api_key -@step(u'{n_junk:d} as number of junk') +@step('{n_junk:d} as number of junk') def step_n_junk(context, n_junk): context.n_junk = n_junk -@step(u'{n_batch:d} as batch size') +@step('{n_batch:d} as batch size') def step_n_batch(context, n_batch): context.n_batch = n_batch -@step(u'{seed:d} as seed') +@step('{seed:d} as seed') def step_seed(context, seed): context.seed = seed -@step(u'a prefix prompt') +@step('a prefix prompt') def step_prompt_prefix(context): - context.prompt_prefix = context.text + context.prompt_prefix = context_text(context) -@step(u'a junk suffix prompt') +@step('a junk suffix prompt') def step_prompt_junk_suffix(context): - context.prompt_junk_suffix = context.text + context.prompt_junk_suffix = context_text(context) -@step(u'a suffix prompt') +@step('a suffix prompt') def step_prompt_suffix(context): - context.prompt_suffix = context.text + context.prompt_suffix = context_text(context) -@step(u'{n_ga:d} group attention factor' - u' to extend context size through self-extend') +@step('{n_ga:d} group attention factor' + ' to extend context size through self-extend') def step_impl(context, n_ga): context.n_ga = n_ga -@step(u'{n_ga_w:d} group attention width to extend context size through self-extend') +@step('{n_ga_w:d} group attention width to extend context size through self-extend') def step_impl(context, n_ga_w): context.n_ga_w = n_ga_w -@step(u'a passkey prompt template') +@step('a passkey prompt template') def step_prompt_passkey(context): - context.prompt_passkey = context.text + context.prompt_passkey = context_text(context) -@step(u'{n_prompts:d} fixed prompts') +@step('{n_prompts:d} fixed prompts') def step_fixed_prompts(context, n_prompts): context.prompts.extend([str(0)*(context.n_batch if context.n_batch is not None else 512) for i in range(n_prompts)]) context.n_prompts = n_prompts -@step(u'a "{passkey}" passkey challenge prompt with the passkey inserted every {i_pos:d} junk') +@step('a "{passkey}" passkey challenge prompt with the passkey inserted every {i_pos:d} junk') def step_prompt_passkey(context, passkey, i_pos): prompt = "" for i in range(context.n_junk): @@ -334,7 +334,7 @@ def step_prompt_passkey(context, passkey, i_pos): context.n_prompts = len(context.prompts) -@step(u'an OAI compatible chat completions request with {api_error} api error') +@step('an OAI compatible chat completions request with {api_error} api error') @async_run_until_complete async def step_oai_chat_completions(context, api_error): if context.debug: @@ -369,19 +369,19 @@ async def step_oai_chat_completions(context, api_error): print(f"Completion response: {completion}") -@step(u'a prompt') +@step('a prompt') def step_a_prompt(context): - context.prompts.append(context.text) + context.prompts.append(context_text(context)) context.n_prompts = len(context.prompts) -@step(u'a prompt {prompt}') +@step('a prompt {prompt}') def step_a_prompt_prompt(context, prompt): context.prompts.append(prompt) context.n_prompts = len(context.prompts) -@step(u'concurrent completion requests') +@step('concurrent completion requests') @async_run_until_complete() async def step_concurrent_completion_requests(context): await concurrent_requests(context, @@ -397,7 +397,7 @@ async def step_concurrent_completion_requests(context): 'user_api_key') else None) -@step(u'concurrent OAI completions requests') +@step('concurrent OAI completions requests') @async_run_until_complete async def step_oai_chat_completions(context): await concurrent_requests(context, oai_chat_completions, @@ -417,7 +417,7 @@ async def step_oai_chat_completions(context): if hasattr(context, 'user_api_key') else None) -@step(u'concurrent OAI completions requests no v1') +@step('concurrent OAI completions requests no v1') @async_run_until_complete async def step_oai_chat_completions(context): await concurrent_requests(context, oai_chat_completions, @@ -440,13 +440,13 @@ async def step_oai_chat_completions(context): if hasattr(context, 'user_api_key') else None) -@step(u'all prompts are predicted') +@step('all prompts are predicted') @async_run_until_complete async def step_all_prompts_are_predicted(context): await all_prompts_are_predicted(context) -@step(u'all prompts are predicted with {n_expected_predicted:d} tokens') +@step('all prompts are predicted with {n_expected_predicted:d} tokens') @async_run_until_complete async def step_all_prompts_are_predicted_with_n_tokens(context, n_expected_predicted): await all_prompts_are_predicted(context, n_expected_predicted) @@ -460,14 +460,14 @@ async def all_prompts_are_predicted(context, expected_predicted_n=None): assert len(context.concurrent_tasks) == 0, f"{len(context.concurrent_tasks)} pending requests" -@step(u'embeddings are computed for') +@step('embeddings are computed for') @async_run_until_complete async def step_compute_embedding(context): context.n_prompts = 1 - context.embeddings = await request_embedding(context.text, base_url=context.base_url) + context.embeddings = await request_embedding(context_text(context), base_url=context.base_url) -@step(u'all embeddings are the same') +@step('all embeddings are the same') @async_run_until_complete async def step_all_embeddings_are_the_same(context): n_embedding_requests = await gather_tasks_results(context) @@ -491,7 +491,8 @@ async def step_all_embeddings_are_the_same(context): print(f"{msg}\n") assert np.isclose(similarity, 1.0, rtol=1e-05, atol=1e-08, equal_nan=False), msg -@step(u'embeddings are generated') + +@step('embeddings are generated') def step_assert_embeddings(context): assert context.n_prompts == len(context.embeddings), (f"unexpected response:\n" f"context.n_prompts={context.n_prompts}\n" @@ -500,17 +501,17 @@ def step_assert_embeddings(context): assert_embeddings(embedding) -@step(u'an OAI compatible embeddings computation request for') +@step('an OAI compatible embeddings computation request for') @async_run_until_complete async def step_oai_compute_embeddings(context): context.n_prompts = 1 - context.embeddings = await request_oai_embeddings(context.text, + context.embeddings = await request_oai_embeddings(context_text(context), base_url=context.base_url, user_api_key=context.user_api_key, model=context.model) -@step(u'an OAI compatible embeddings computation request for multiple inputs') +@step('an OAI compatible embeddings computation request for multiple inputs') @async_run_until_complete async def step_oai_compute_embeddings_multiple_inputs(context): context.embeddings = await request_oai_embeddings(context.prompts, @@ -520,7 +521,7 @@ async def step_oai_compute_embeddings_multiple_inputs(context): context.prompts.clear() -@step(u'concurrent embedding requests') +@step('concurrent embedding requests') @async_run_until_complete() async def step_concurrent_embedding_requests(context): await concurrent_requests(context, @@ -529,7 +530,7 @@ async def step_concurrent_embedding_requests(context): base_url=context.base_url) -@step(u'concurrent OAI embedding requests') +@step('concurrent OAI embedding requests') @async_run_until_complete() async def step_concurrent_oai_embedding_requests(context): await concurrent_requests(context, @@ -540,7 +541,7 @@ async def step_concurrent_oai_embedding_requests(context): model=context.model) -@step(u'all embeddings are generated') +@step('all embeddings are generated') @async_run_until_complete() async def all_embeddings_are_generated(context): n_embedding_requests = await gather_tasks_results(context) @@ -549,10 +550,10 @@ async def all_embeddings_are_generated(context): assert_embeddings(context.tasks_result.pop().pop()) -@step(u'tokenizing') +@step('tokenizing') @async_run_until_complete async def step_tokenize(context): - context.tokenized_text = context.text + context.tokenized_text = context_text(context) async with aiohttp.ClientSession() as session: async with session.post(f'{context.base_url}/tokenize', json={ @@ -563,7 +564,7 @@ async def step_tokenize(context): context.tokens = tokenize_json['tokens'] -@step(u'tokens can be detokenize') +@step('tokens can be detokenize') @async_run_until_complete async def step_detokenize(context): assert len(context.tokens) > 0 @@ -578,7 +579,7 @@ async def step_detokenize(context): assert context.tokenized_text == detokenize_json['content'].strip() -@step(u'an OPTIONS request is sent from {origin}') +@step('an OPTIONS request is sent from {origin}') @async_run_until_complete async def step_options_request(context, origin): async with aiohttp.ClientSession() as session: @@ -589,12 +590,12 @@ async def step_options_request(context, origin): context.options_response = response -@step(u'CORS header {cors_header} is set to {cors_header_value}') +@step('CORS header {cors_header} is set to {cors_header_value}') def step_check_options_header_value(context, cors_header, cors_header_value): assert context.options_response.headers[cors_header] == cors_header_value -@step(u'prometheus metrics are exposed') +@step('prometheus metrics are exposed') @async_run_until_complete async def step_prometheus_metrics_exported(context): async with aiohttp.ClientSession() as session: @@ -616,14 +617,14 @@ async def step_prometheus_metrics_exported(context): assert metric_exported, "No metrics exported" -@step(u'metric {metric_name} is {metric_value:d}') +@step('metric {metric_name} is {metric_value:d}') def step_assert_metric_value(context, metric_name, metric_value): if metric_name not in context.metrics: assert False, f"no metric {metric_name} in {context.metrics.keys()}" assert context.metrics[metric_name].samples[0].value == metric_value, f"metric: {context.metrics[metric_name]}" -@step(u'available models') +@step('available models') def step_available_models(context): # openai client always expects an api_key openai.api_key = context.user_api_key if context.user_api_key is not None else 'nope' @@ -631,14 +632,14 @@ def step_available_models(context): context.models = openai.Model.list().data -@step(u'{n_model:d} models are supported') +@step('{n_model:d} models are supported') def step_supported_models(context, n_model): if context.debug: print("server models available:", context.models) assert len(context.models) == n_model -@step(u'model {i_model:d} is {param} {preposition} {param_value}') +@step('model {i_model:d} is {param} {preposition} {param_value}') def step_supported_models(context, i_model, param, preposition, param_value): assert i_model < len(context.models) model = context.models[i_model] @@ -1007,12 +1008,22 @@ async def completions_seed(context): else context.server_seed if hasattr(context, 'server_seed') else None +def context_text(context): + return context.text.replace('\r', '') + + def start_server_background(context): - context.server_path = '../../../build/bin/server' + if os.name == 'nt': + context.server_path = '../../../build/bin/Release/server.exe' + else: + context.server_path = '../../../build/bin/server' if 'LLAMA_SERVER_BIN_PATH' in os.environ: context.server_path = os.environ['LLAMA_SERVER_BIN_PATH'] + server_listen_addr = context.server_fqdn + if os.name == 'nt': + server_listen_addr = '0.0.0.0' server_args = [ - '--host', context.server_fqdn, + '--host', server_listen_addr, '--port', context.server_port, '--model', context.model_file ] @@ -1045,7 +1056,16 @@ def start_server_background(context): if 'SERVER_LOG_FORMAT_JSON' not in os.environ: server_args.extend(['--log-format', "text"]) print(f"starting server with: {context.server_path} {server_args}\n") + flags = 0 + if 'nt' == os.name: + flags |= subprocess.DETACHED_PROCESS + flags |= subprocess.CREATE_NEW_PROCESS_GROUP + flags |= subprocess.CREATE_NO_WINDOW + + pkwargs = { + 'creationflags': flags, + } context.server_process = subprocess.Popen( [str(arg) for arg in [context.server_path, *server_args]], - close_fds=True) - print(f"server pid={context.server_process.pid}") + **pkwargs) + print(f"server pid={context.server_process.pid}, behave pid={os.getpid()}") From bf47a5eefc669fdba71af096942af999bc1167d4 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Sun, 10 Mar 2024 20:09:24 +0200 Subject: [PATCH 583/812] ggml : remove __constant__ specifier for CUDA tables (#5940) --- ggml-common.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ggml-common.h b/ggml-common.h index 4b6d248b6..84981212c 100644 --- a/ggml-common.h +++ b/ggml-common.h @@ -17,7 +17,7 @@ #elif defined(GGML_COMMON_IMPL_CUDA) #include -#define GGML_TABLE_BEGIN(type, name, size) static const __device__ __constant__ type name[size] = { +#define GGML_TABLE_BEGIN(type, name, size) static const __device__ type name[size] = { #define GGML_TABLE_END() }; #define GGML_COMMON_IMPL From df4dc3e7cb43162862f339525638ba4febcb6158 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Fri, 8 Mar 2024 23:45:07 +0200 Subject: [PATCH 584/812] ggml : try fix 32-bit arm compat (whisper/1938) * ggml : try fix 32-bit arm compat * ggml : fix cont --- ggml-quants.c | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/ggml-quants.c b/ggml-quants.c index 807c5e391..42d8a5d80 100644 --- a/ggml-quants.c +++ b/ggml-quants.c @@ -9009,8 +9009,8 @@ void ggml_vec_dot_iq2_s_q8_K(int n, float * restrict s, size_t bs, const void * static const uint8_t k_mask2[16] = {0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80,}; - const uint8x16x2_t mask1 = vld1q_u8_x2(k_mask1); - const uint8x16_t mask2 = vld1q_u8(k_mask2); + const ggml_uint8x16x2_t mask1 = ggml_vld1q_u8_x2(k_mask1); + const uint8x16_t mask2 = vld1q_u8(k_mask2); const uint8x16_t m1 = vdupq_n_u8(1); const int32x4_t vzero = vdupq_n_s32(0); @@ -9354,11 +9354,12 @@ void ggml_vec_dot_iq3_s_q8_K (int n, float * restrict s, size_t bs, const void * static const int16_t k_shift[8] = {8, 7, 6, 5, 4, 3, 2, 1}; - const uint8x16x2_t mask1 = vld1q_u8_x2(k_mask1); - const uint8x16_t mask2 = vld1q_u8(k_mask2); - const int16x8_t hshift = vld1q_s16(k_shift); - const uint16x8_t m256 = vdupq_n_u16(256); - const uint8x16_t m1 = vdupq_n_u8(1); + const ggml_uint8x16x2_t mask1 = ggml_vld1q_u8_x2(k_mask1); + const uint8x16_t mask2 = vld1q_u8(k_mask2); + + const int16x8_t hshift = vld1q_s16(k_shift); + const uint16x8_t m256 = vdupq_n_u16(256); + const uint8x16_t m1 = vdupq_n_u8(1); uint8x16x2_t vs; ggml_int8x16x4_t q3s; From b838b53ad6de2e53f23ddf8f3ad5e6891cc3dd05 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Sun, 10 Mar 2024 20:10:46 +0200 Subject: [PATCH 585/812] sync : ggml --- scripts/sync-ggml.last | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/sync-ggml.last b/scripts/sync-ggml.last index afe68077d..7c30162e2 100644 --- a/scripts/sync-ggml.last +++ b/scripts/sync-ggml.last @@ -1 +1 @@ -8695910a39102609073d0e099aa7c97d6bcb3bf9 +43a6d4af1971ee2912ff7bc2404011ff327b6a60 From d9f65c97c3dc3aa6fa27470b8c6e69b437ec1a27 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Sun, 10 Mar 2024 20:58:26 +0200 Subject: [PATCH 586/812] readme : update hot topics --- README.md | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index e3ec0817a..98fdc6808 100644 --- a/README.md +++ b/README.md @@ -8,11 +8,6 @@ Inference of Meta's [LLaMA](https://arxiv.org/abs/2302.13971) model (and others) in pure C/C++ -> [!IMPORTANT] -> **Quantization blind testing: https://github.com/ggerganov/llama.cpp/discussions/5962** -> -> Vote for which quantization type provides better responses, all other parameters being the same. - ### Recent API changes - [2024 Mar 8] `llama_kv_cache_seq_rm()` returns a `bool` instead of `void`, and new `llama_n_max_seq()` returns the upper limit of acceptable `seq_id` in batches (relevant when dealing with multiple sequences) https://github.com/ggerganov/llama.cpp/pull/5328 @@ -21,6 +16,8 @@ Inference of Meta's [LLaMA](https://arxiv.org/abs/2302.13971) model (and others) ### Hot topics +- Looking for contributions to add Deepseek support: https://github.com/ggerganov/llama.cpp/issues/5981 +- Quantization blind testing: https://github.com/ggerganov/llama.cpp/discussions/5962 - Initial Mamba support has been added: https://github.com/ggerganov/llama.cpp/pull/5328 ---- From 7ab7b733bb48250b2df26c12b00256ef42c76932 Mon Sep 17 00:00:00 2001 From: Dean Date: Mon, 11 Mar 2024 04:03:17 +0800 Subject: [PATCH 587/812] android : fix utf8 decoding error (#5935) * examples: fix utf8 decoding error some models have a tokenizer that decodes an id into an incomplete utf8 sequence, need to validate and wait for next token one example would be: https://huggingface.co/Qwen/Qwen1.5-1.8B-Chat-GGUF/resolve/main/qwen1_5-1_8b-chat-q4_0.gguf and and an example of the token is 18137 * android : minor --------- Co-authored-by: zhangfuwen Co-authored-by: Georgi Gerganov --- .../app/src/main/cpp/llama-android.cpp | 53 ++++++++++++++++++- .../src/main/java/com/example/llama/Llm.kt | 4 +- 2 files changed, 53 insertions(+), 4 deletions(-) diff --git a/examples/llama.android/app/src/main/cpp/llama-android.cpp b/examples/llama.android/app/src/main/cpp/llama-android.cpp index 2beb1e0d5..ce8ab3b70 100644 --- a/examples/llama.android/app/src/main/cpp/llama-android.cpp +++ b/examples/llama.android/app/src/main/cpp/llama-android.cpp @@ -33,6 +33,45 @@ jclass la_int_var; jmethodID la_int_var_value; jmethodID la_int_var_inc; +std::string cached_token_chars; + +bool is_valid_utf8(const char * string) { + if (!string) { + return true; + } + + const unsigned char * bytes = (const unsigned char *)string; + int num; + + while (*bytes != 0x00) { + if ((*bytes & 0x80) == 0x00) { + // U+0000 to U+007F + num = 1; + } else if ((*bytes & 0xE0) == 0xC0) { + // U+0080 to U+07FF + num = 2; + } else if ((*bytes & 0xF0) == 0xE0) { + // U+0800 to U+FFFF + num = 3; + } else if ((*bytes & 0xF8) == 0xF0) { + // U+10000 to U+10FFFF + num = 4; + } else { + return false; + } + + bytes += 1; + for (int i = 1; i < num; ++i) { + if ((*bytes & 0xC0) != 0x80) { + return false; + } + bytes += 1; + } + } + + return true; +} + static void log_callback(ggml_log_level level, const char * fmt, void * data) { if (level == GGML_LOG_LEVEL_ERROR) __android_log_print(ANDROID_LOG_ERROR, TAG, fmt, data); else if (level == GGML_LOG_LEVEL_INFO) __android_log_print(ANDROID_LOG_INFO, TAG, fmt, data); @@ -295,6 +334,8 @@ Java_com_example_llama_Llm_completion_1init( jint n_len ) { + cached_token_chars.clear(); + const auto text = env->GetStringUTFChars(jtext, 0); const auto context = reinterpret_cast(context_pointer); const auto batch = reinterpret_cast(batch_pointer); @@ -372,8 +413,16 @@ Java_com_example_llama_Llm_completion_1loop( } auto new_token_chars = llama_token_to_piece(context, new_token_id); - LOGi("new_token_chars: `%s`", new_token_chars.c_str()); - auto new_token = env->NewStringUTF(new_token_chars.c_str()); + cached_token_chars += new_token_chars; + + jstring new_token = nullptr; + if (is_valid_utf8(cached_token_chars.c_str())) { + new_token = env->NewStringUTF(cached_token_chars.c_str()); + LOGi("cached: %s, new_token_chars: `%s`, id: %d", cached_token_chars.c_str(), new_token_chars.c_str(), new_token_id); + cached_token_chars.clear(); + } else { + new_token = env->NewStringUTF(""); + } llama_batch_clear(*batch); llama_batch_add(*batch, new_token_id, n_cur, { 0 }, true); diff --git a/examples/llama.android/app/src/main/java/com/example/llama/Llm.kt b/examples/llama.android/app/src/main/java/com/example/llama/Llm.kt index 5f3270372..d86afee37 100644 --- a/examples/llama.android/app/src/main/java/com/example/llama/Llm.kt +++ b/examples/llama.android/app/src/main/java/com/example/llama/Llm.kt @@ -71,7 +71,7 @@ class Llm { batch: Long, nLen: Int, ncur: IntVar - ): String + ): String? private external fun kv_cache_clear(context: Long) @@ -115,7 +115,7 @@ class Llm { val ncur = IntVar(completion_init(state.context, state.batch, message, nlen)) while (ncur.value <= nlen) { val str = completion_loop(state.context, state.batch, nlen, ncur) - if (str.isEmpty()) { + if (str == null) { break } emit(str) From bb6d00bbf98476215596b9df3870b5504ef5a29a Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Sun, 10 Mar 2024 23:12:48 +0200 Subject: [PATCH 588/812] metal : move mm_id indices to shared mem (#5982) --- ggml-metal.m | 6 +++--- ggml-metal.metal | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/ggml-metal.m b/ggml-metal.m index 00df22838..3cf80de7b 100644 --- a/ggml-metal.m +++ b/ggml-metal.m @@ -1642,8 +1642,8 @@ static enum ggml_status ggml_metal_graph_compute( // TODO: make this more general GGML_ASSERT(n_as <= 8); - // max size of the src1ids array in the kernel stack - GGML_ASSERT(ne11 <= 512); + // max size of the src1ids array in the kernel shared buffer + GGML_ASSERT(ne11 <= 4096); const int64_t ne20 = src2 ? src2->ne[0] : 0; const int64_t ne21 = src2 ? src2->ne[1] : 0; @@ -1741,7 +1741,7 @@ static enum ggml_status ggml_metal_graph_compute( [encoder setBuffer:id_src_cur offset:offs_src_cur atIndex:19 + j]; } - [encoder setThreadgroupMemoryLength:8192 atIndex:0]; + [encoder setThreadgroupMemoryLength:GGML_PAD(8192 + 2*ne11, 16) atIndex:0]; [encoder dispatchThreadgroups:MTLSizeMake((ne11 + 31)/32, (ne21 + 63)/64, n_as*ne12*ne13) threadsPerThreadgroup:MTLSizeMake(128, 1, 1)]; } else { diff --git a/ggml-metal.metal b/ggml-metal.metal index 6ebbbd195..50185ae4d 100644 --- a/ggml-metal.metal +++ b/ggml-metal.metal @@ -5386,7 +5386,7 @@ template Date: Mon, 11 Mar 2024 01:13:57 +0000 Subject: [PATCH 589/812] [SYCL] Add support for SYCL Nvidia target (#5738) * Add support for nvidia target in CMake * Update sycl read-me for Nvidia target * Fix errors --- CMakeLists.txt | 14 +++++++++++++- README-sycl.md | 26 ++++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 9309ca6bb..00a26391a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -116,6 +116,7 @@ option(LLAMA_MPI "llama: use MPI" option(LLAMA_QKK_64 "llama: use super-block size of 64 for k-quants" OFF) option(LLAMA_SYCL "llama: use SYCL" OFF) option(LLAMA_SYCL_F16 "llama: use 16 bit floats for sycl calculations" OFF) +set(LLAMA_SYCL_TARGET "INTEL" CACHE STRING "llama: sycl target device") option(LLAMA_CPU_HBM "llama: use memkind for CPU HBM" OFF) option(LLAMA_BUILD_TESTS "llama: build tests" ${LLAMA_STANDALONE}) @@ -534,6 +535,10 @@ if (LLAMA_HIPBLAS) endif() if (LLAMA_SYCL) + if (NOT LLAMA_SYCL_TARGET MATCHES "^(INTEL|NVIDIA)$") + message(FATAL_ERROR "Invalid backend chosen, supported options are INTEL or NVIDIA") + endif() + if ( NOT DEFINED ENV{ONEAPI_ROOT}) message(FATAL_ERROR "Not detect ENV {ONEAPI_ROOT}, please install oneAPI & source it, like: source /opt/intel/oneapi/setvars.sh") endif() @@ -555,6 +560,9 @@ if (LLAMA_SYCL) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-narrowing") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O3") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsycl -L${MKLROOT}/lib") + if (LLAMA_SYCL_TARGET STREQUAL "NVIDIA") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsycl-targets=nvptx64-nvidia-cuda") + endif() set(GGML_HEADERS_SYCL ggml-sycl.h) set(GGML_SOURCES_SYCL ggml-sycl.cpp) @@ -562,7 +570,11 @@ if (LLAMA_SYCL) if (WIN32) set(LLAMA_EXTRA_LIBS ${LLAMA_EXTRA_LIBS} -fsycl sycl7 OpenCL mkl_sycl_blas_dll.lib mkl_intel_ilp64_dll.lib mkl_sequential_dll.lib mkl_core_dll.lib) else() - set(LLAMA_EXTRA_LIBS ${LLAMA_EXTRA_LIBS} -fsycl OpenCL mkl_core pthread m dl mkl_sycl_blas mkl_intel_ilp64 mkl_tbb_thread) + if (LLAMA_SYCL_TARGET STREQUAL "INTEL") + set(LLAMA_EXTRA_LIBS ${LLAMA_EXTRA_LIBS} -fsycl OpenCL mkl_core pthread m dl mkl_sycl_blas mkl_intel_ilp64 mkl_tbb_thread) + elseif (LLAMA_SYCL_TARGET STREQUAL "NVIDIA") + set(LLAMA_EXTRA_LIBS ${LLAMA_EXTRA_LIBS} -fsycl pthread m dl onemkl) + endif() endif() endif() diff --git a/README-sycl.md b/README-sycl.md index 85eb16f2b..9359a9490 100644 --- a/README-sycl.md +++ b/README-sycl.md @@ -73,6 +73,29 @@ For iGPU, please make sure the shared memory from host memory is enough. For lla For dGPU, please make sure the device memory is enough. For llama-2-7b.Q4_0, recommend the device memory is 4GB+. +## Nvidia GPU + +### Verified + +|Intel GPU| Status | Verified Model| +|-|-|-| +|Ampere Series| Support| A100| + +### oneMKL + +The current oneMKL release does not contain the oneMKL cuBlas backend. +As a result for Nvidia GPU's oneMKL must be built from source. + +``` +git clone https://github.com/oneapi-src/oneMKL +cd oneMKL +mkdir build +cd build +cmake -G Ninja .. -DCMAKE_CXX_COMPILER=icpx -DCMAKE_C_COMPILER=icx -DENABLE_MKLGPU_BACKEND=OFF -DENABLE_MKLCPU_BACKEND=OFF -DENABLE_CUBLAS_BACKEND=ON +ninja +// Add paths as necessary +``` + ## Docker Note: @@ -186,6 +209,9 @@ source /opt/intel/oneapi/setvars.sh # Or, for FP32: cmake .. -DLLAMA_SYCL=ON -DCMAKE_C_COMPILER=icx -DCMAKE_CXX_COMPILER=icpx +# For Nvidia GPUs +cmake .. -DLLAMA_SYCL=ON -DLLAMA_SYCL_TARGET=NVIDIA -DCMAKE_C_COMPILER=icx -DCMAKE_CXX_COMPILER=icpx + # Build example/main only #cmake --build . --config Release --target main From ef3ced26a3817d92890b97b83acaeb018ade02d0 Mon Sep 17 00:00:00 2001 From: Abhilash Majumder <30946547+abhilash1910@users.noreply.github.com> Date: Mon, 11 Mar 2024 10:27:56 +0530 Subject: [PATCH 590/812] [SYCL] Add q3_s and q1_s (#5886) * Add q3_s and q1_s * fix compilation * fix build * fix build * fix build * enable ops * rm macro * increase grid space --- ggml-sycl.cpp | 378 +++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 375 insertions(+), 3 deletions(-) diff --git a/ggml-sycl.cpp b/ggml-sycl.cpp index 6fafa5282..6d5684582 100644 --- a/ggml-sycl.cpp +++ b/ggml-sycl.cpp @@ -3494,6 +3494,31 @@ typedef struct dpct_type_block_iq3_xxs { } block_iq3_xxs; static_assert(sizeof(block_iq3_xxs) == sizeof(ggml_fp16_t) + 3*(QK_K/8), "wrong iq3_xxs block size/padding"); +#define QR3_XS 8 +#define QI3_XS (QK_K / (4*QR3_XS)) +#if QK_K == 64 +#define IQ3S_N_SCALE 2 +#else +#define IQ3S_N_SCALE QK_K/64 +#endif +typedef struct { + sycl::half d; + uint8_t qs[QK_K/4]; + uint8_t qh[QK_K/32]; + uint8_t signs[QK_K/8]; + uint8_t scales[IQ3S_N_SCALE]; +} block_iq3_s; +static_assert(sizeof(block_iq3_s) == sizeof(ggml_fp16_t) + 13*(QK_K/32) + IQ3S_N_SCALE, "wrong iq3_s block size/padding"); + +#define QR1_S 8 +#define QI1_S (QK_K / (4*QR1_S)) +typedef struct { + sycl::half d; + uint8_t qs[QK_K/8]; + uint8_t scales[QK_K/16]; +} block_iq1_s; +static_assert(sizeof(block_iq1_s) == sizeof(ggml_fp16_t) + QK_K/8 + QK_K/16, "wrong iq1_s block size/padding"); + #define WARP_SIZE 32 #define MATRIX_ROW_PADDING 512 // last row of quant. matrices is a multiple of this to avoid out-of-bounds memory accesses @@ -4833,6 +4858,62 @@ static void dequantize_block_iq3_xxs(const void * __restrict__ vx, dst_t * __res } +template +static void dequantize_block_iq3_s(const void * __restrict__ vx, dst_t * __restrict__ yy, + const sycl::nd_item<3> &item_ct1, + const uint32_t *iq3s_grid, + const uint8_t *ksigns_iq2xs, + const uint8_t *kmask_iq2xs) { + + const int i = item_ct1.get_group(2); + const block_iq3_s * x = (const block_iq3_s *) vx; + + const int tid = item_ct1.get_local_id(2); +#if QK_K == 256 + const int il = tid/8; // 0...3 + const int ib = tid%8; // 0...7 + dst_t * y = yy + i*QK_K + 32*ib + 8*il; + const uint8_t * qs = x[i].qs + 8*ib; + const uint8_t * grid1 = (const uint8_t *)(iq3s_grid + qs[2*il+0]); + const uint8_t * grid2 = (const uint8_t *)(iq3s_grid + qs[2*il+1]); + const float d = (float)x[i].d * (1 + 2*((x[i].scales[ib/2] >> 4*(ib%2)) & 0xf)); + const uint8_t signs = x[i].signs[4*ib + il]; + for (int j = 0; j < 4; ++j) { + y[j+0] = d * grid1[j] * (signs & kmask_iq2xs[j+0] ? -1.f : 1.f); + y[j+4] = d * grid2[j] * (signs & kmask_iq2xs[j+4] ? -1.f : 1.f); + } +#else + assert(false); +#endif + +} + +template +static void dequantize_block_iq1_s(const void * __restrict__ vx, dst_t * __restrict__ yy, + const sycl::nd_item<3> &item_ct1, + const uint64_t *iq1s_grid, + const uint8_t *ksigns_iq2xs, + const uint8_t *kmask_iq2xs) { + + const int i = item_ct1.get_group(2); + const block_iq1_s * x = (const block_iq1_s *) vx; + + const int tid = item_ct1.get_local_id(2); +#if QK_K == 256 + const int il = tid/8; // 0...3 + const int ib = tid%8; // 0...7 + dst_t * y = yy + i*QK_K + 32*ib + 8*il; + const int i8 = 4*ib+il; + uint8_t h = x[i].scales[i8/2] >> 4*(i8%2); + const int8_t * grid = (const int8_t *)(iq1s_grid + (x[i].qs[i8] | ((h & 8) << 5))); + const float d = (float)x[i].d * (2*(h & 7) + 1); + for (int j = 0; j < 8; ++j) y[j] = d * grid[j]; +#else + assert(false); +#endif + +} + /* DPCT1110:4: The total declared local variable size in device function dequantize_mul_mat_vec_q2_k exceeds 128 bytes and may cause high register @@ -7679,6 +7760,76 @@ vec_dot_iq3_xxs_q8_1(const void *__restrict__ vbq, #endif } +static __dpct_inline__ float +vec_dot_iq3_s_q8_1(const void *__restrict__ vbq, + const block_q8_1 *__restrict__ bq8_1, const int &iqs, + const uint32_t *iq3s_grid, const uint64_t *ksigns64) { +#if DPCT_COMPATIBILITY_TEMP >= \ + MIN_CC_DP4A // lowest compute capability for integer intrinsics +#if QK_K == 256 + const block_iq3_s * bq2 = (const block_iq3_s *) vbq; + + const int ib32 = iqs; + const uint8_t * qs = bq2->qs + 8*ib32; + const int8_t * q8 = bq8_1[ib32].qs; + int sumi = 0; + for (int l = 0; l < 4; ++l) { + const uint32_t * grid1 = iq3s_grid + (qs[2*l+0] | ((bq2->qh[ib32] << (8 - 2*l)) & 256)); + const uint32_t * grid2 = iq3s_grid + (qs[2*l+1] | ((bq2->qh[ib32] << (7 - 2*l)) & 256)); + uint32_t signs0 = dpct::vectorized_binary( + ((bq2->signs[4*ib32+l] & 0xf) * 0x01010101) & 0x08040201, 0x08040201, std::equal_to<>()); + uint32_t signs1 = dpct::vectorized_binary( + ((bq2->signs[4*ib32+l] >> 4) * 0x01010101) & 0x08040201, 0x08040201, std::equal_to<>()); + const int grid_l = dpct::vectorized_binary( + grid1[0] ^ signs0, signs0, std::minus<>()); + const int grid_h = dpct::vectorized_binary( + grid2[0] ^ signs1, signs1, std::minus<>()); + sumi = dpct::dp4a(grid_l, *((int *)q8 + 0), sumi); + sumi = dpct::dp4a(grid_h, *((int *)q8 + 1), sumi); + q8 += 8; + } + const float d = (float)bq2->d * (1 + 2*((bq2->scales[ib32/2] >> 4*(ib32%2)) & 0xf)) * bq8_1[ib32].ds[0]; + return d * sumi; +#else + assert(false); + return 0.f; +#endif +#else + assert(false); + return 0.f; +#endif +} + +static __dpct_inline__ float +vec_dot_iq1_s_q8_1(const void *__restrict__ vbq, + const block_q8_1 *__restrict__ bq8_1, const int &iqs, + const uint64_t *iq1s_grid, const uint64_t *ksigns64) { +#if QK_K == 256 + const block_iq1_s * bq1 = (const block_iq1_s *) vbq; + + const int ib32 = iqs; + int sumi1 = 0, sumi2 = 0, sumi3 = 0, sumi4 = 0; + const uint8_t h1 = bq1->scales[2*ib32+0]; + const uint8_t h2 = bq1->scales[2*ib32+1]; + const int * q8 = (const int *)bq8_1[ib32].qs; + const int * grid1 = (const int *)(iq1s_grid + (bq1->qs[4*ib32+0] | ((h1 & 0x08) << 5))); + const int * grid2 = (const int *)(iq1s_grid + (bq1->qs[4*ib32+1] | ((h1 & 0x80) << 1))); + const int * grid3 = (const int *)(iq1s_grid + (bq1->qs[4*ib32+2] | ((h2 & 0x08) << 5))); + const int * grid4 = (const int *)(iq1s_grid + (bq1->qs[4*ib32+3] | ((h2 & 0x80) << 1))); + for (int j = 0; j < 2; ++j) { + sumi1 = dpct::dp4a(q8[j+0], grid1[j], sumi1); + sumi2 = dpct::dp4a(q8[j+2], grid2[j], sumi2); + sumi3 = dpct::dp4a(q8[j+4], grid3[j], sumi3); + sumi4 = dpct::dp4a(q8[j+6], grid4[j], sumi4); + } + const float d = (float)bq1->d * bq8_1[ib32].ds[0]; + return d * (sumi1 * (2*(h1 & 7) + 1) + sumi2 * (2*((h1 >> 4) & 7) + 1) + + sumi3 * (2*(h2 & 7) + 1) + sumi4 * (2*((h2 >> 4) & 7) + 1)); +#else + assert(false); + return 0.f; +#endif +} template +static void mul_mat_vec_q_iq3_s_q8_1(const void * __restrict__ vx, const void * __restrict__ vy, float * __restrict__ dst, const int ncols, const int nrows, + const sycl::nd_item<3> &item_ct1, + const uint32_t *iq3s_grid_ptr, const uint64_t *ksigns64_ptr ) { + const int row = item_ct1.get_group(2) * item_ct1.get_local_range(1) + + item_ct1.get_local_id(1); + + if (row >= nrows) { + return; + } + + const int blocks_per_row = ncols / qk; + const int blocks_per_warp = vdr * WARP_SIZE / qi; + +// partial sum for each thread + float tmp = 0.0f; + + const block_q_t * x = (const block_q_t *) vx; + const block_q8_1 * y = (const block_q8_1 *) vy; + + for (int i = item_ct1.get_local_id(2) / (qi / vdr); i < blocks_per_row; + i += blocks_per_warp) { + const int ibx = row*blocks_per_row + i; // x block index + + const int iby = i * (qk/QK8_1); // y block index that aligns with ibx + + const int iqs = + vdr * + (item_ct1.get_local_id(2) % + (qi / vdr)); // x block quant index when casting the quants to int + + tmp += vec_dot_iq3_s_q8_1(&x[ibx], &y[iby], iqs, iq3s_grid_ptr, ksigns64_ptr); + } + + // sum up partial sums and write back result +#pragma unroll + for (int mask = 16; mask > 0; mask >>= 1) { + tmp += + dpct::permute_sub_group_by_xor(item_ct1.get_sub_group(), tmp, mask); + } + + if (item_ct1.get_local_id(2) == 0) { + dst[row] = tmp; + } +} + +template +static void mul_mat_vec_q_iq1_s_q8_1(const void * __restrict__ vx, const void * __restrict__ vy, float * __restrict__ dst, const int ncols, const int nrows, + const sycl::nd_item<3> &item_ct1, + const uint64_t *iq1s_grid_ptr, const uint64_t *ksigns64_ptr ) { + const int row = item_ct1.get_group(2) * item_ct1.get_local_range(1) + + item_ct1.get_local_id(1); + + if (row >= nrows) { + return; + } + + const int blocks_per_row = ncols / qk; + const int blocks_per_warp = vdr * WARP_SIZE / qi; + +// partial sum for each thread + float tmp = 0.0f; + + const block_q_t * x = (const block_q_t *) vx; + const block_q8_1 * y = (const block_q8_1 *) vy; + + for (int i = item_ct1.get_local_id(2) / (qi / vdr); i < blocks_per_row; + i += blocks_per_warp) { + const int ibx = row*blocks_per_row + i; // x block index + + const int iby = i * (qk/QK8_1); // y block index that aligns with ibx + + const int iqs = + vdr * + (item_ct1.get_local_id(2) % + (qi / vdr)); // x block quant index when casting the quants to int + + tmp += vec_dot_iq1_s_q8_1(&x[ibx], &y[iby], iqs, iq1s_grid_ptr, ksigns64_ptr); + } + + // sum up partial sums and write back result +#pragma unroll + for (int mask = 16; mask > 0; mask >>= 1) { + tmp += + dpct::permute_sub_group_by_xor(item_ct1.get_sub_group(), tmp, mask); + } + + if (item_ct1.get_local_id(2) == 0) { + dst[row] = tmp; + } +} + template static void dequantize_mul_mat_vec(const void * __restrict__ vx, const dfloat * __restrict__ y, float * __restrict__ dst, const int ncols, const int nrows, const sycl::nd_item<3> &item_ct1) { @@ -10129,6 +10372,64 @@ static void dequantize_row_iq3_xxs_sycl(const void *vx, dst_t *y, const int k, } } +template +static void dequantize_row_iq3_s_sycl(const void *vx, dst_t *y, const int k, + dpct::queue_ptr stream) { + const int nb = k / QK_K; + { + iq3s_grid.init(*stream); + ksigns_iq2xs.init(*stream); + kmask_iq2xs.init(*stream); + + dpct::has_capability_or_fail(stream->get_device(), + {sycl::aspect::fp16}); + + stream->submit([&](sycl::handler &cgh) { + auto iq3s_grid_ptr_ct1 = iq3s_grid.get_ptr(); + auto ksigns_iq2xs_ptr_ct1 = ksigns_iq2xs.get_ptr(); + auto kmask_iq2xs_ptr_ct1 = kmask_iq2xs.get_ptr(); + + cgh.parallel_for(sycl::nd_range<3>(sycl::range<3>(1, 1, nb) * + sycl::range<3>(1, 1, 32), + sycl::range<3>(1, 1, 32)), + [=](sycl::nd_item<3> item_ct1) { + dequantize_block_iq3_s( + vx, y, item_ct1, iq3s_grid_ptr_ct1, + ksigns_iq2xs_ptr_ct1, kmask_iq2xs_ptr_ct1); + }); + }); + } +} + +template +static void dequantize_row_iq1_s_sycl(const void *vx, dst_t *y, const int k, + dpct::queue_ptr stream) { + const int nb = k / QK_K; + { + iq1s_grid.init(*stream); + ksigns_iq2xs.init(*stream); + kmask_iq2xs.init(*stream); + + dpct::has_capability_or_fail(stream->get_device(), + {sycl::aspect::fp16}); + + stream->submit([&](sycl::handler &cgh) { + auto iq1s_grid_ptr_ct1 = iq1s_grid.get_ptr(); + auto ksigns_iq2xs_ptr_ct1 = ksigns_iq2xs.get_ptr(); + auto kmask_iq2xs_ptr_ct1 = kmask_iq2xs.get_ptr(); + + cgh.parallel_for(sycl::nd_range<3>(sycl::range<3>(1, 1, nb) * + sycl::range<3>(1, 1, 32), + sycl::range<3>(1, 1, 32)), + [=](sycl::nd_item<3> item_ct1) { + dequantize_block_iq1_s( + vx, y, item_ct1, iq1s_grid_ptr_ct1, + ksigns_iq2xs_ptr_ct1, kmask_iq2xs_ptr_ct1); + }); + }); + } +} + template static void convert_unary_sycl(const void *__restrict__ vx, dst_t *__restrict__ y, const int k, @@ -10179,6 +10480,10 @@ static to_fp16_sycl_t ggml_get_to_fp16_sycl(ggml_type type) try { return dequantize_row_iq2_xs_sycl; case GGML_TYPE_IQ3_XXS: return dequantize_row_iq3_xxs_sycl; + case GGML_TYPE_IQ3_S: + return dequantize_row_iq3_s_sycl; + case GGML_TYPE_IQ1_S: + return dequantize_row_iq1_s_sycl; case GGML_TYPE_F32: return convert_unary_sycl; default: @@ -10219,6 +10524,10 @@ static to_fp32_sycl_t ggml_get_to_fp32_sycl(ggml_type type) { return dequantize_row_iq2_xs_sycl; case GGML_TYPE_IQ3_XXS: return dequantize_row_iq3_xxs_sycl; + case GGML_TYPE_IQ3_S: + return dequantize_row_iq3_s_sycl; + case GGML_TYPE_IQ1_S: + return dequantize_row_iq1_s_sycl; case GGML_TYPE_F16: return convert_unary_sycl; default: @@ -10808,6 +11117,61 @@ static void mul_mat_vec_iq3_xxs_q8_1_sycl(const void *vx, const void *vy, } } +static void mul_mat_vec_iq3_s_q8_1_sycl(const void *vx, const void *vy, + float *dst, const int ncols, + const int nrows, + dpct::queue_ptr stream) { + GGML_ASSERT(ncols % QK_K == 0); + const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y; + const sycl::range<3> block_nums(1, 1, block_num_y); + const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE); + { + iq3s_grid.init(*stream); + ksigns64.init(*stream); + + stream->submit([&](sycl::handler &cgh) { + auto iq3s_grid_ptr_ct1 = iq3s_grid.get_ptr(); + auto ksigns64_ptr_ct1 = ksigns64.get_ptr(); + + cgh.parallel_for( + sycl::nd_range<3>(block_nums * block_dims, block_dims), + [=](sycl::nd_item<3> item_ct1) + [[intel::reqd_sub_group_size(32)]] { + mul_mat_vec_q_iq3_s_q8_1( + vx, vy, dst, ncols, nrows, item_ct1, + iq3s_grid_ptr_ct1, ksigns64_ptr_ct1); + }); + }); + } +} + +static void mul_mat_vec_iq1_s_q8_1_sycl(const void *vx, const void *vy, + float *dst, const int ncols, + const int nrows, + dpct::queue_ptr stream) { + GGML_ASSERT(ncols % QK_K == 0); + const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y; + const sycl::range<3> block_nums(1, 1, block_num_y); + const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE); + { + iq1s_grid.init(*stream); + ksigns64.init(*stream); + + stream->submit([&](sycl::handler &cgh) { + auto iq1s_grid_ptr_ct1 = iq1s_grid.get_ptr(); + auto ksigns64_ptr_ct1 = ksigns64.get_ptr(); + + cgh.parallel_for( + sycl::nd_range<3>(block_nums * block_dims, block_dims), + [=](sycl::nd_item<3> item_ct1) + [[intel::reqd_sub_group_size(32)]] { + mul_mat_vec_q_iq1_s_q8_1( + vx, vy, dst, ncols, nrows, item_ct1, + iq1s_grid_ptr_ct1, ksigns64_ptr_ct1); + }); + }); + } +} static void ggml_mul_mat_q4_0_q8_1_sycl(const void *vx, const void *vy, float *dst, const int ncols_x, @@ -13556,8 +13920,11 @@ static int64_t get_row_rounding(ggml_type type, const std::array= VER_GEN9 ? 128 : 64; + case GGML_TYPE_IQ3_S: + return max_compute_capability >= VER_GEN9 ? 128 : 64; case GGML_TYPE_Q6_K: return 64; default: @@ -13618,6 +13985,12 @@ inline void ggml_sycl_op_mul_mat_vec_q( case GGML_TYPE_IQ3_XXS: mul_mat_vec_iq3_xxs_q8_1_sycl(src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, stream); break; + case GGML_TYPE_IQ3_S: + mul_mat_vec_iq3_s_q8_1_sycl(src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, stream); + break; + case GGML_TYPE_IQ1_S: + mul_mat_vec_iq1_s_q8_1_sycl(src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, stream); + break; default: GGML_ASSERT(false); break; @@ -16963,9 +17336,8 @@ GGML_CALL static bool ggml_backend_sycl_supports_op(ggml_backend_t backend, cons return false; } ggml_type a_type = a->type; - if (a_type == GGML_TYPE_IQ2_XXS || a_type == GGML_TYPE_IQ2_XS || a_type == GGML_TYPE_IQ3_XXS || - a_type == GGML_TYPE_IQ1_S || a_type == GGML_TYPE_IQ4_NL || a_type == GGML_TYPE_IQ3_S || - a_type == GGML_TYPE_IQ2_S || a_type == GGML_TYPE_IQ4_XS) { + if (a_type == GGML_TYPE_IQ4_NL || a_type == GGML_TYPE_IQ2_S || + a_type == GGML_TYPE_IQ4_XS) { return false; } return true; From be858f620508385ad12d0e5e862010e666ca729c Mon Sep 17 00:00:00 2001 From: Kawrakow <48489457+ikawrakow@users.noreply.github.com> Date: Mon, 11 Mar 2024 07:51:49 +0100 Subject: [PATCH 591/812] Better 1.5 bit quantization (#5971) * Trying blocvks of 16 for IQ1_S - seems slightly better * iq1s_blocks16: Adjust scale fudge factor to 1.125 * iq1s_blocks16: going to blocks of 32 with 2048 lattice points, so same bpw. This is even better than blocks of 16. Should I try blocks of 64? But to keep the same bpw, when I go to 4096 lattice points, I need to remove blocks alltogether and just have superblocks of 256 weights. * iq1s_blocks16: Use 2* as sigma2 in weight adjustment * iq1s_blocks16: scalar and AVX2 dot products * iq1s_blocks16: CUDA dot product * iq1s_blocks16: Metal works, Neon does not Metal works but TG is dog slow (35 t/s). PP is OKish (493 t/s). Not seeing the bug in the Neon implementation for now. * iq1s_blocks16: fixed Neon * iq1s_blocks16: very slightly faster TG on Metal Still pathetic at 37 t/s * iq1s_blocks16: speedup Metal by packing codebook into uint32_t's * Formatting * iq1s_blocks16: uint32_t codebook is also better in CUDA TG-128 is now 204 t/s up from 194 t/s. PP-512 is 5890 t/s, so significantly better than other quants * iq1s_blocks16: slightly faster Neon dot product * iq1s_blocks16: faster AVX2 dot product * iq1s_blocks16: adjust to ggml-common.h --------- Co-authored-by: Iwan Kawrakow --- ggml-common.h | 903 ++++++++++++++++++++++++++++++++++++++++------- ggml-cuda.cu | 62 ++-- ggml-metal.metal | 66 ++-- ggml-quants.c | 510 +++++++++++++++----------- ggml-quants.h | 4 +- 5 files changed, 1152 insertions(+), 393 deletions(-) diff --git a/ggml-common.h b/ggml-common.h index 84981212c..5dd918081 100644 --- a/ggml-common.h +++ b/ggml-common.h @@ -644,136 +644,781 @@ GGML_TABLE_BEGIN(uint32_t, iq3s_grid, 512) 0x0f090307, 0x0f090501, 0x0f090b01, 0x0f0b0505, 0x0f0b0905, 0x0f0d0105, 0x0f0d0703, 0x0f0f0101, GGML_TABLE_END() -#define NGRID_IQ2XXS 512 -GGML_TABLE_BEGIN(uint64_t, iq1s_grid, NGRID_IQ2XXS) - 0xffffffffffff0101, 0xffffffffff01ff00, 0xffffffffff010100, 0xffffffff00000000, - 0xffffffff01ff00ff, 0xffffffff01ff0001, 0xffffffff0101ffff, 0xffffffff0101ff01, - 0xffffff00ff000000, 0xffffff000000ff00, 0xffffff00000000ff, 0xffffff0000000100, - 0xffffff0000010000, 0xffffff0001000000, 0xffffff01ffff00ff, 0xffffff01ff01ff00, - 0xffffff01ff010100, 0xffffff0100000001, 0xffffff0101ffff00, 0xffffff0101ff0101, - 0xffffff0101010100, 0xffff00ffff00ff01, 0xffff00ffff0000ff, 0xffff00ff00ff0100, - 0xffff00ff0100ff00, 0xffff00ff010001ff, 0xffff0000ff0101ff, 0xffff000000ffff00, - 0xffff000000000000, 0xffff00000001ff01, 0xffff000001000101, 0xffff0000010100ff, - 0xffff0001ffff0100, 0xffff00010000ff00, 0xffff000100010101, 0xffff000101000000, - 0xffff01ffffff0000, 0xffff01ffff01ffff, 0xffff01ffff010100, 0xffff01ff00000000, - 0xffff01ff01ffffff, 0xffff01ff01ff0001, 0xffff01ff0101ffff, 0xffff01ff01010001, - 0xffff0100ffffff01, 0xffff01000000ffff, 0xffff010000000100, 0xffff010001ff01ff, - 0xffff010001000000, 0xffff0101ff000000, 0xffff0101000101ff, 0xffff010101ffff01, - 0xffff01010101ff00, 0xff00ffffff000000, 0xff00ffff00ffff00, 0xff00ffff00000001, - 0xff00ffff000001ff, 0xff00ffff01010000, 0xff00ff00ffff0000, 0xff00ff00ff00ff00, - 0xff00ff00ff0000ff, 0xff00ff00ff000100, 0xff00ff00ff010001, 0xff00ff0000ff0001, - 0xff00ff000000ffff, 0xff00ff0000000000, 0xff00ff000001ff00, 0xff00ff0000010100, - 0xff00ff0001ff0000, 0xff00ff000100ff00, 0xff00ff0001000100, 0xff00ff01ff000000, - 0xff00ff0100ff0000, 0xff00ff01000001ff, 0xff00ff0101010001, 0xff0000ff00000000, - 0xff0000ff0001ff00, 0xff0000ff00010100, 0xff000000ffff0101, 0xff000000ff000000, - 0xff000000ff01ff00, 0xff00000000ff0000, 0xff0000000000ff00, 0xff000000000000ff, - 0xff00000000000000, 0xff00000000000001, 0xff00000000000100, 0xff0000000001ffff, - 0xff00000000010000, 0xff00000001000000, 0xff00000001010100, 0xff000001ff00ff01, - 0xff000001ff0100ff, 0xff00000100000000, 0xff0000010001ff00, 0xff00000101ff0100, - 0xff0000010100ff00, 0xff0001ff00ff00ff, 0xff0001ff00000101, 0xff0001ff000100ff, - 0xff0001ff01000000, 0xff000100ff0001ff, 0xff0001000000ff01, 0xff00010000000000, - 0xff00010000010001, 0xff00010000010100, 0xff00010001ffff00, 0xff00010001ff0101, - 0xff00010001010000, 0xff000101ffffffff, 0xff000101ff000101, 0xff00010101ff00ff, - 0xff00010101000001, 0xff000101010100ff, 0xff01ffffff000101, 0xff01ffffff01ffff, - 0xff01ffffff01ff01, 0xff01ffffff0101ff, 0xff01ffff00000000, 0xff01ffff01ff0001, - 0xff01ffff0101ff01, 0xff01ff00ff000000, 0xff01ff0000ff0100, 0xff01ff000000ff01, - 0xff01ff0000010000, 0xff01ff00010000ff, 0xff01ff01ff01ff00, 0xff01ff0100000101, - 0xff0100ffffff0000, 0xff0100ffff010000, 0xff0100ff01ff00ff, 0xff0100ff01000100, - 0xff0100ff010100ff, 0xff010000ffffff01, 0xff01000000000000, 0xff0100000101ff00, - 0xff010001ffff00ff, 0xff010001ff000100, 0xff01000100ffff00, 0xff01000100010001, - 0xff01000101ff0001, 0xff010001010001ff, 0xff0101ffffffffff, 0xff0101ffff01ffff, - 0xff0101ffff010101, 0xff0101ff0000ff00, 0xff0101ff01010001, 0xff010100ff000000, - 0xff010100ff01ff01, 0xff01010000ff0001, 0xff01010000000100, 0xff01010001000000, - 0xff0101010100ffff, 0x00ffffff0000ff01, 0x00ffffff000000ff, 0x00ffffff00000100, - 0x00ffffff00010000, 0x00ffff00ffff0001, 0x00ffff00ff0000ff, 0x00ffff00ff000100, - 0x00ffff0000000000, 0x00ffff0001000100, 0x00ffff0001010001, 0x00ffff01ff00ff01, - 0x00ffff0100ff0100, 0x00ffff010000ff00, 0x00ffff01000100ff, 0x00ffff0101ff00ff, - 0x00ffff010101ff00, 0x00ff00ffffffffff, 0x00ff00ffffff01ff, 0x00ff00ffff000101, - 0x00ff00ff00000000, 0x00ff00ff000101ff, 0x00ff00ff01010101, 0x00ff0000ff000000, - 0x00ff0000ff01ffff, 0x00ff000000ff0000, 0x00ff00000000ff00, 0x00ff0000000000ff, - 0x00ff000000000000, 0x00ff000000000001, 0x00ff000000000100, 0x00ff000000010000, - 0x00ff000001ffff01, 0x00ff000001000000, 0x00ff0001ff000101, 0x00ff000100ffffff, - 0x00ff000100000000, 0x00ff0001010001ff, 0x00ff01ffff000000, 0x00ff01ff0001ff00, - 0x00ff01ff01ff0100, 0x00ff0100ff01ff01, 0x00ff010000ff00ff, 0x00ff010000ff0101, - 0x00ff010000000000, 0x00ff010000010101, 0x00ff01000100ff00, 0x00ff010001010000, - 0x00ff0101ffffff00, 0x00ff01010000ff01, 0x00ff010100000100, 0x00ff010101ff0000, - 0x0000ffffffff0100, 0x0000ffffff00ff00, 0x0000ffffff0000ff, 0x0000ffffff010000, - 0x0000ffff00000000, 0x0000ffff00010101, 0x0000ffff01ffff01, 0x0000ffff01000100, - 0x0000ff00ff000000, 0x0000ff00ff01ff00, 0x0000ff00ff0101ff, 0x0000ff0000ff0000, - 0x0000ff000000ff00, 0x0000ff00000000ff, 0x0000ff0000000000, 0x0000ff0000000001, - 0x0000ff0000000100, 0x0000ff0000010000, 0x0000ff0001ffffff, 0x0000ff0001ff01ff, - 0x0000ff0001000000, 0x0000ff000101ffff, 0x0000ff01ffff0101, 0x0000ff01ff010000, - 0x0000ff0100000000, 0x0000ff0101000101, 0x000000ffffff0001, 0x000000ffff000000, - 0x000000ff00ff0000, 0x000000ff0000ff00, 0x000000ff000000ff, 0x000000ff00000000, - 0x000000ff00000001, 0x000000ff00000100, 0x000000ff00010000, 0x000000ff01000000, - 0x000000ff0101ff00, 0x00000000ffff0000, 0x00000000ff00ff00, 0x00000000ff0000ff, - 0x00000000ff000000, 0x00000000ff000001, 0x00000000ff000100, 0x00000000ff010000, - 0x0000000000ffff00, 0x0000000000ff00ff, 0x0000000000ff0000, 0x0000000000ff0001, +#define NGRID_IQ1S 2048 +#if defined(GGML_COMMON_IMPL_C) +GGML_TABLE_BEGIN(uint64_t, iq1s_grid, NGRID_IQ1S) + 0xffffffffffffffff, 0xffffffffffffff01, 0xffffffffffff0000, 0xffffffffffff01ff, + 0xffffffffffff0101, 0xffffffffff00ff00, 0xffffffffff000000, 0xffffffffff01ffff, + 0xffffffffff01ff01, 0xffffffffff0101ff, 0xffffffffff010101, 0xffffffff00ff0000, + 0xffffffff0000ff00, 0xffffffff000000ff, 0xffffffff00000001, 0xffffffff00010000, + 0xffffffff01ffffff, 0xffffffff01ffff01, 0xffffffff01ff01ff, 0xffffffff01ff0101, + 0xffffffff01000000, 0xffffffff0101ffff, 0xffffffff0101ff01, 0xffffffff010101ff, + 0xffffffff01010101, 0xffffff00ffff00ff, 0xffffff00ffff0000, 0xffffff00ff00ff00, + 0xffffff00ff0000ff, 0xffffff00ff000001, 0xffffff00ff000100, 0xffffff00ff000101, + 0xffffff00ff010000, 0xffffff0000ffff00, 0xffffff0000ff0001, 0xffffff0000ff0100, + 0xffffff000000ff01, 0xffffff0000000000, 0xffffff0000000101, 0xffffff000001ff00, + 0xffffff00000100ff, 0xffffff0000010001, 0xffffff00000101ff, 0xffffff0001ff0000, + 0xffffff000100ff00, 0xffffff00010000ff, 0xffffff0001000001, 0xffffff0001010000, + 0xffffff01ffffffff, 0xffffff01ffffff01, 0xffffff01ffff01ff, 0xffffff01ffff0101, + 0xffffff01ff000000, 0xffffff01ff01ffff, 0xffffff01ff01ff01, 0xffffff01ff0101ff, + 0xffffff01ff010101, 0xffffff0100ff0000, 0xffffff010000ff00, 0xffffff0100000100, + 0xffffff01000100ff, 0xffffff0100010100, 0xffffff0101ffffff, 0xffffff0101ffff01, + 0xffffff0101ff01ff, 0xffffff0101ff0101, 0xffffff010100ff00, 0xffffff0101000000, + 0xffffff0101000100, 0xffffff010101ffff, 0xffffff010101ff01, 0xffffff01010101ff, + 0xffffff0101010101, 0xffff00ffff00ff00, 0xffff00ffff0000ff, 0xffff00ffff000001, + 0xffff00ffff010000, 0xffff00ff00ffff00, 0xffff00ff00ff0100, 0xffff00ff00000000, + 0xffff00ff00000101, 0xffff00ff000100ff, 0xffff00ff00010000, 0xffff00ff0100ff00, + 0xffff00ff01000100, 0xffff00ff01010000, 0xffff0000ffffff00, 0xffff0000ffff00ff, + 0xffff0000ffff0000, 0xffff0000ffff0001, 0xffff0000ff000000, 0xffff0000ff0001ff, + 0xffff0000ff000101, 0xffff0000ff010100, 0xffff000000ffffff, 0xffff000000ff0000, + 0xffff000000ff0101, 0xffff00000000ffff, 0xffff00000000ff00, 0xffff0000000000ff, + 0xffff000000000000, 0xffff000000000001, 0xffff000000000100, 0xffff00000001ffff, + 0xffff00000001ff01, 0xffff000000010000, 0xffff0000000101ff, 0xffff000000010101, + 0xffff000001ffff00, 0xffff00000100ff00, 0xffff000001000000, 0xffff0000010001ff, + 0xffff000001000101, 0xffff00000101ff00, 0xffff0000010100ff, 0xffff000001010000, + 0xffff000001010001, 0xffff000001010100, 0xffff0001ff0000ff, 0xffff0001ff000100, + 0xffff000100ffff00, 0xffff000100ff00ff, 0xffff00010000ffff, 0xffff00010000ff01, + 0xffff000100000000, 0xffff0001000001ff, 0xffff00010001ffff, 0xffff00010001ff00, + 0xffff000100010001, 0xffff000100010100, 0xffff000101ff0000, 0xffff00010100ff00, + 0xffff0001010000ff, 0xffff000101000100, 0xffff01ffffffffff, 0xffff01ffffffff01, + 0xffff01ffffff01ff, 0xffff01ffffff0101, 0xffff01ffff000000, 0xffff01ffff01ffff, + 0xffff01ffff01ff01, 0xffff01ffff0101ff, 0xffff01ffff010101, 0xffff01ff00ff0000, + 0xffff01ff0000ff00, 0xffff01ff00000001, 0xffff01ff00010000, 0xffff01ff01ffffff, + 0xffff01ff01ffff01, 0xffff01ff01ff01ff, 0xffff01ff01ff0101, 0xffff01ff01000000, + 0xffff01ff0101ffff, 0xffff01ff0101ff01, 0xffff01ff010101ff, 0xffff01ff01010101, + 0xffff0100ffff0000, 0xffff0100ff00ff00, 0xffff0100ff0000ff, 0xffff0100ff000100, + 0xffff0100ff0100ff, 0xffff0100ff010000, 0xffff010000ffff00, 0xffff01000000ffff, + 0xffff01000000ff00, 0xffff010000000000, 0xffff01000001ff00, 0xffff0100000100ff, + 0xffff010000010100, 0xffff01000100ff00, 0xffff0100010000ff, 0xffff010001000001, + 0xffff010001000100, 0xffff010001010000, 0xffff0101ffffffff, 0xffff0101ffffff01, + 0xffff0101ffff01ff, 0xffff0101ffff0101, 0xffff0101ff000000, 0xffff0101ff01ffff, + 0xffff0101ff01ff01, 0xffff0101ff0101ff, 0xffff0101ff010101, 0xffff010100ff0000, + 0xffff01010000ff00, 0xffff010100000100, 0xffff01010001ff00, 0xffff010100010000, + 0xffff010101ffffff, 0xffff010101ffff01, 0xffff010101ff0000, 0xffff010101ff01ff, + 0xffff010101ff0101, 0xffff010101000000, 0xffff01010101ffff, 0xffff01010101ff01, + 0xffff0101010101ff, 0xffff010101010101, 0xff00ffffff00ffff, 0xff00ffffff00ff00, + 0xff00ffffff0000ff, 0xff00ffffff000100, 0xff00ffffff0100ff, 0xff00ffffff010000, + 0xff00ffff00ffff00, 0xff00ffff00ff00ff, 0xff00ffff0000ffff, 0xff00ffff00000000, + 0xff00ffff000001ff, 0xff00ffff0001ff00, 0xff00ffff000100ff, 0xff00ffff00010000, + 0xff00ffff00010100, 0xff00ffff0100ff00, 0xff00ffff010000ff, 0xff00ffff01000001, + 0xff00ffff0101ff00, 0xff00ffff01010000, 0xff00ff00ffffff00, 0xff00ff00ffff00ff, + 0xff00ff00ffff0001, 0xff00ff00ffff0100, 0xff00ff00ff00ffff, 0xff00ff00ff00ff01, + 0xff00ff00ff000000, 0xff00ff00ff0001ff, 0xff00ff00ff01ff00, 0xff00ff00ff0100ff, + 0xff00ff00ff010100, 0xff00ff0000ff0000, 0xff00ff0000ff0101, 0xff00ff000000ffff, + 0xff00ff000000ff00, 0xff00ff000000ff01, 0xff00ff00000000ff, 0xff00ff0000000000, + 0xff00ff0000000001, 0xff00ff0000000100, 0xff00ff000001ffff, 0xff00ff0000010000, + 0xff00ff0001ff00ff, 0xff00ff000100ff01, 0xff00ff0001000000, 0xff00ff000101ff00, + 0xff00ff00010100ff, 0xff00ff01ff00ff00, 0xff00ff01ff0000ff, 0xff00ff01ff000001, + 0xff00ff01ff010000, 0xff00ff0100ffffff, 0xff00ff0100ff0001, 0xff00ff0100ff0100, + 0xff00ff010000ff01, 0xff00ff0100000000, 0xff00ff01000001ff, 0xff00ff0100000101, + 0xff00ff01000100ff, 0xff00ff0100010001, 0xff00ff0101ff0000, 0xff00ff010100ff00, + 0xff00ff01010000ff, 0xff00ff0101000001, 0xff00ff0101010000, 0xff0000ffffffff00, + 0xff0000ffffff0001, 0xff0000ffffff0100, 0xff0000ffff0000ff, 0xff0000ffff000000, + 0xff0000ffff0001ff, 0xff0000ffff000100, 0xff0000ffff01ff00, 0xff0000ffff010001, + 0xff0000ff00ffff00, 0xff0000ff00ff0000, 0xff0000ff00ff0001, 0xff0000ff00ff01ff, + 0xff0000ff00ff0101, 0xff0000ff0000ff00, 0xff0000ff000000ff, 0xff0000ff00000000, + 0xff0000ff00000001, 0xff0000ff00000100, 0xff0000ff0001ff01, 0xff0000ff00010000, + 0xff0000ff000101ff, 0xff0000ff01ff00ff, 0xff0000ff01ff0100, 0xff0000ff0100ffff, + 0xff0000ff010000ff, 0xff0000ff01000000, 0xff0000ff010001ff, 0xff0000ff01000100, + 0xff0000ff01000101, 0xff0000ff0101ff00, 0xff0000ff010100ff, 0xff0000ff01010000, + 0xff0000ff01010100, 0xff000000ffffff01, 0xff000000ffff0000, 0xff000000ffff0101, + 0xff000000ff00ff00, 0xff000000ff0000ff, 0xff000000ff000000, 0xff000000ff000001, + 0xff000000ff000100, 0xff000000ff01ffff, 0xff000000ff01ff01, 0xff000000ff010000, + 0xff000000ff0101ff, 0xff000000ff010101, 0xff00000000ffff00, 0xff00000000ff00ff, + 0xff00000000ff0000, 0xff00000000ff0001, 0xff0000000000ff00, 0xff0000000000ff01, + 0xff000000000000ff, 0xff00000000000000, 0xff00000000000001, 0xff00000000000100, + 0xff00000000000101, 0xff0000000001ff00, 0xff000000000100ff, 0xff00000000010000, + 0xff00000000010001, 0xff00000000010100, 0xff00000001ffffff, 0xff00000001ffff01, + 0xff00000001ff00ff, 0xff00000001ff0000, 0xff00000001ff01ff, 0xff00000001ff0101, + 0xff0000000100ffff, 0xff0000000100ff00, 0xff000000010000ff, 0xff00000001000000, + 0xff00000001000001, 0xff00000001000100, 0xff00000001000101, 0xff0000000101ffff, + 0xff0000000101ff01, 0xff00000001010000, 0xff000001ffffff00, 0xff000001ffff00ff, + 0xff000001ffff0000, 0xff000001ffff0001, 0xff000001ff000000, 0xff000001ff000001, + 0xff000001ff0001ff, 0xff000001ff000101, 0xff000001ff01ff00, 0xff000001ff010001, + 0xff00000100ffffff, 0xff00000100ffff01, 0xff00000100ff00ff, 0xff00000100ff0000, + 0xff00000100ff01ff, 0xff00000100ff0101, 0xff0000010000ff00, 0xff00000100000000, + 0xff00000100000001, 0xff000001000001ff, 0xff00000100000100, 0xff0000010001ff00, + 0xff000001000100ff, 0xff00000100010000, 0xff000001000101ff, 0xff00000100010100, + 0xff00000100010101, 0xff00000101ff0001, 0xff00000101ff0101, 0xff0000010100ff01, + 0xff00000101000000, 0xff000001010100ff, 0xff00000101010100, 0xff0001ffff00ff00, + 0xff0001ffff000001, 0xff0001ffff010000, 0xff0001ff00ffff00, 0xff0001ff00ff00ff, + 0xff0001ff00ff0001, 0xff0001ff00ff0100, 0xff0001ff0000ffff, 0xff0001ff00000000, + 0xff0001ff000001ff, 0xff0001ff00000101, 0xff0001ff0001ffff, 0xff0001ff0001ff00, + 0xff0001ff000100ff, 0xff0001ff00010001, 0xff0001ff00010100, 0xff0001ff01ff0000, + 0xff0001ff0100ff00, 0xff0001ff010000ff, 0xff0001ff01010000, 0xff000100ff00ffff, + 0xff000100ff00ff01, 0xff000100ff000000, 0xff000100ff000101, 0xff000100ff01ff00, + 0xff000100ff010000, 0xff00010000ffff01, 0xff00010000ff00ff, 0xff00010000ff0000, + 0xff00010000ff01ff, 0xff0001000000ff00, 0xff000100000000ff, 0xff00010000000000, + 0xff00010000000001, 0xff00010000000100, 0xff00010000000101, 0xff0001000001ffff, + 0xff00010000010000, 0xff00010000010101, 0xff00010001ff0100, 0xff0001000100ff00, + 0xff0001000100ff01, 0xff00010001000000, 0xff000100010001ff, 0xff0001000101ff00, + 0xff00010001010001, 0xff00010001010100, 0xff000101ffff0100, 0xff000101ff000001, + 0xff000101ff0100ff, 0xff000101ff010001, 0xff00010100ff00ff, 0xff00010100ff0001, + 0xff00010100ff0100, 0xff0001010000ffff, 0xff0001010000ff01, 0xff00010100000000, + 0xff000101000001ff, 0xff0001010001ff00, 0xff00010100010001, 0xff00010100010100, + 0xff00010101ff0000, 0xff0001010100ff00, 0xff00010101000001, 0xff00010101000101, + 0xff01ffffffffffff, 0xff01ffffffffff01, 0xff01ffffffff01ff, 0xff01ffffffff0101, + 0xff01ffffff000000, 0xff01ffffff01ffff, 0xff01ffffff01ff01, 0xff01ffffff010000, + 0xff01ffffff0101ff, 0xff01ffffff010101, 0xff01ffff00ff0000, 0xff01ffff0000ff00, + 0xff01ffff00000100, 0xff01ffff0001ff00, 0xff01ffff00010000, 0xff01ffff01ffffff, + 0xff01ffff01ffff01, 0xff01ffff01ff01ff, 0xff01ffff01ff0101, 0xff01ffff01000000, + 0xff01ffff0101ffff, 0xff01ffff0101ff01, 0xff01ffff01010000, 0xff01ffff010101ff, + 0xff01ffff01010101, 0xff01ff00ffff0000, 0xff01ff00ff00ff00, 0xff01ff00ff0000ff, + 0xff01ff00ff000100, 0xff01ff00ff010000, 0xff01ff0000ffff01, 0xff01ff0000ff00ff, + 0xff01ff0000ff0100, 0xff01ff0000000000, 0xff01ff00000001ff, 0xff01ff0000000101, + 0xff01ff000001ff00, 0xff01ff00000100ff, 0xff01ff0000010000, 0xff01ff0000010001, + 0xff01ff0001ff0000, 0xff01ff000100ffff, 0xff01ff0001000001, 0xff01ff0001000100, + 0xff01ff0001010000, 0xff01ff01ffffff00, 0xff01ff01ffff01ff, 0xff01ff01ffff0101, + 0xff01ff01ff00ff00, 0xff01ff01ff000000, 0xff01ff01ff01ffff, 0xff01ff01ff01ff01, + 0xff01ff01ff0101ff, 0xff01ff01ff010101, 0xff01ff0100ff0000, 0xff01ff010000ff00, + 0xff01ff0100000001, 0xff01ff0100000100, 0xff01ff0100010000, 0xff01ff0101ffff00, + 0xff01ff0101ff01ff, 0xff01ff0101ff0101, 0xff01ff010100ff00, 0xff01ff0101000000, + 0xff01ff010101ffff, 0xff01ff010101ff01, 0xff01ff01010101ff, 0xff01ff0101010101, + 0xff0100ffffff0000, 0xff0100ffff0000ff, 0xff0100ffff000001, 0xff0100ffff000100, + 0xff0100ffff010000, 0xff0100ff00ff00ff, 0xff0100ff00ff0000, 0xff0100ff00ff0001, + 0xff0100ff00ff0100, 0xff0100ff0000ff01, 0xff0100ff00000000, 0xff0100ff000001ff, + 0xff0100ff00000101, 0xff0100ff00010001, 0xff0100ff01ff0000, 0xff0100ff0100ff00, + 0xff0100ff010000ff, 0xff0100ff01000100, 0xff0100ff0101ff00, 0xff0100ff01010000, + 0xff010000ffff0100, 0xff010000ff000000, 0xff010000ff01ff00, 0xff010000ff010100, + 0xff01000000ffffff, 0xff01000000ff0000, 0xff01000000ff01ff, 0xff0100000000ff00, + 0xff010000000000ff, 0xff01000000000000, 0xff01000000000100, 0xff0100000001ff01, + 0xff01000000010000, 0xff010000000101ff, 0xff01000001ff0100, 0xff0100000100ffff, + 0xff010000010000ff, 0xff01000001000000, 0xff010000010001ff, 0xff01000001000101, + 0xff0100000101ff00, 0xff010000010100ff, 0xff01000001010001, 0xff01000001010100, + 0xff010001ffff0000, 0xff010001ff00ffff, 0xff010001ff00ff01, 0xff010001ff000100, + 0xff010001ff010000, 0xff01000100ffff00, 0xff01000100ff0100, 0xff01000100000000, + 0xff0100010001ffff, 0xff0100010001ff00, 0xff01000100010100, 0xff01000101ff00ff, + 0xff01000101ff0001, 0xff0100010100ffff, 0xff01000101000101, 0xff0101ffffffffff, + 0xff0101ffffffff01, 0xff0101ffffff01ff, 0xff0101ffffff0101, 0xff0101ffff000000, + 0xff0101ffff01ffff, 0xff0101ffff01ff01, 0xff0101ffff0101ff, 0xff0101ffff010101, + 0xff0101ff00ff0000, 0xff0101ff0000ff00, 0xff0101ff000000ff, 0xff0101ff00010000, + 0xff0101ff01ffffff, 0xff0101ff01ffff01, 0xff0101ff01ff01ff, 0xff0101ff01ff0101, + 0xff0101ff0101ffff, 0xff0101ff0101ff01, 0xff0101ff010101ff, 0xff0101ff01010101, + 0xff010100ffff0100, 0xff010100ff00ff00, 0xff010100ff0000ff, 0xff010100ff000100, + 0xff010100ff010000, 0xff01010000ff0001, 0xff01010000ff0100, 0xff0101000000ff01, + 0xff01010000000000, 0xff0101000001ff00, 0xff010100000100ff, 0xff01010000010001, + 0xff01010000010100, 0xff01010001ff0000, 0xff0101000100ffff, 0xff01010001000001, + 0xff01010001000100, 0xff010100010100ff, 0xff01010001010000, 0xff010101ffffffff, + 0xff010101ffffff01, 0xff010101ffff01ff, 0xff010101ffff0101, 0xff010101ff01ffff, + 0xff010101ff01ff01, 0xff010101ff0101ff, 0xff010101ff010101, 0xff01010100ff0000, + 0xff0101010000ff00, 0xff01010100000001, 0xff01010100000100, 0xff01010100010000, + 0xff01010101ffffff, 0xff01010101ffff01, 0xff01010101ff01ff, 0xff01010101ff0101, + 0xff01010101000000, 0xff0101010101ffff, 0xff0101010101ff01, 0xff010101010101ff, + 0xff01010101010101, 0x00ffffffffff0000, 0x00ffffffff00ff00, 0x00ffffffff000001, + 0x00ffffffff010000, 0x00ffffff00ff0100, 0x00ffffff0000ff01, 0x00ffffff00000000, + 0x00ffffff000001ff, 0x00ffffff00000101, 0x00ffffff0001ff00, 0x00ffffff000100ff, + 0x00ffffff00010001, 0x00ffffff010000ff, 0x00ffffff01000100, 0x00ffffff0101ff00, + 0x00ffffff01010001, 0x00ffff00ffffffff, 0x00ffff00ffffff00, 0x00ffff00ffff00ff, + 0x00ffff00ffff0001, 0x00ffff00ffff0100, 0x00ffff00ff00ff01, 0x00ffff00ff000000, + 0x00ffff00ff000001, 0x00ffff00ff0001ff, 0x00ffff00ff000101, 0x00ffff00ff01ff00, + 0x00ffff00ff010001, 0x00ffff00ff010100, 0x00ffff0000ff0000, 0x00ffff0000ff01ff, + 0x00ffff0000ff0101, 0x00ffff000000ff00, 0x00ffff00000000ff, 0x00ffff0000000000, + 0x00ffff0000000001, 0x00ffff0000000100, 0x00ffff0000000101, 0x00ffff0000010000, + 0x00ffff00000101ff, 0x00ffff0000010101, 0x00ffff0001ffff00, 0x00ffff0001ff00ff, + 0x00ffff0001ff0001, 0x00ffff000100ffff, 0x00ffff000100ff01, 0x00ffff0001000000, + 0x00ffff000101ffff, 0x00ffff000101ff00, 0x00ffff000101ff01, 0x00ffff01ffff0000, + 0x00ffff01ff00ff00, 0x00ffff01ff0000ff, 0x00ffff01ff000001, 0x00ffff01ff010000, + 0x00ffff0100ffff00, 0x00ffff010000ff01, 0x00ffff0100000000, 0x00ffff0100000101, + 0x00ffff01000100ff, 0x00ffff0100010100, 0x00ffff0101ff0100, 0x00ffff01010000ff, + 0x00ffff0101010000, 0x00ff00ffffffff00, 0x00ff00ffff000000, 0x00ff00ffff000100, + 0x00ff00ffff010100, 0x00ff00ff00ff0000, 0x00ff00ff00ff01ff, 0x00ff00ff00ff0101, + 0x00ff00ff0000ff00, 0x00ff00ff000000ff, 0x00ff00ff00000000, 0x00ff00ff00000001, + 0x00ff00ff0001ff00, 0x00ff00ff0001ff01, 0x00ff00ff00010000, 0x00ff00ff000101ff, + 0x00ff00ff00010101, 0x00ff00ff01ffff00, 0x00ff00ff01ff0001, 0x00ff00ff01ff0100, + 0x00ff00ff0100ffff, 0x00ff00ff0100ff01, 0x00ff00ff01000000, 0x00ff00ff0101ffff, + 0x00ff00ff0101ff00, 0x00ff00ff01010100, 0x00ff0000ffffff00, 0x00ff0000ffffff01, + 0x00ff0000ffff0000, 0x00ff0000ffff0101, 0x00ff0000ff00ff00, 0x00ff0000ff0000ff, + 0x00ff0000ff000000, 0x00ff0000ff000001, 0x00ff0000ff000100, 0x00ff0000ff01ffff, + 0x00ff0000ff010000, 0x00ff0000ff010101, 0x00ff000000ffff00, 0x00ff000000ff00ff, + 0x00ff000000ff0000, 0x00ff000000ff0001, 0x00ff000000ff0100, 0x00ff00000000ffff, + 0x00ff00000000ff00, 0x00ff0000000000ff, 0x00ff000000000000, 0x00ff000000000001, + 0x00ff0000000001ff, 0x00ff000000000100, 0x00ff00000001ff00, 0x00ff0000000100ff, + 0x00ff000000010000, 0x00ff000000010001, 0x00ff000000010100, 0x00ff000001ffff01, + 0x00ff000001ff00ff, 0x00ff000001ff0000, 0x00ff000001ff01ff, 0x00ff00000100ff00, + 0x00ff0000010000ff, 0x00ff000001000000, 0x00ff000001000001, 0x00ff000001000100, + 0x00ff000001000101, 0x00ff000001010000, 0x00ff0000010101ff, 0x00ff000001010101, + 0x00ff0001ffffff00, 0x00ff0001ffff0000, 0x00ff0001ffff0100, 0x00ff0001ff0000ff, + 0x00ff0001ff000000, 0x00ff0001ff0001ff, 0x00ff0001ff000101, 0x00ff0001ff01ff00, + 0x00ff0001ff0100ff, 0x00ff0001ff010100, 0x00ff000100ffffff, 0x00ff000100ffff01, + 0x00ff000100ff0000, 0x00ff000100ff01ff, 0x00ff00010000ffff, 0x00ff00010000ff00, + 0x00ff00010000ff01, 0x00ff000100000000, 0x00ff000100000001, 0x00ff000100000100, + 0x00ff00010001ff01, 0x00ff000100010000, 0x00ff0001000101ff, 0x00ff000101ffff00, + 0x00ff000101ff0000, 0x00ff000101ff0101, 0x00ff0001010000ff, 0x00ff000101000000, + 0x00ff00010101ff00, 0x00ff0001010100ff, 0x00ff000101010001, 0x00ff01ffffff0000, + 0x00ff01ffff00ff00, 0x00ff01ffff000000, 0x00ff01ffff000101, 0x00ff01ffff010000, + 0x00ff01ff00ffff01, 0x00ff01ff00ff0100, 0x00ff01ff0000ffff, 0x00ff01ff00000000, + 0x00ff01ff000001ff, 0x00ff01ff0001ff00, 0x00ff01ff000100ff, 0x00ff01ff00010001, + 0x00ff01ff00010100, 0x00ff01ff01ff0000, 0x00ff01ff0100ff00, 0x00ff01ff010000ff, + 0x00ff01ff01000001, 0x00ff01ff01000100, 0x00ff01ff01010000, 0x00ff0100ffffff00, + 0x00ff0100ffff0000, 0x00ff0100ffff0001, 0x00ff0100ffff0101, 0x00ff0100ff00ffff, + 0x00ff0100ff0000ff, 0x00ff0100ff000000, 0x00ff0100ff0001ff, 0x00ff0100ff01ff00, + 0x00ff0100ff0100ff, 0x00ff0100ff010001, 0x00ff010000ffffff, 0x00ff010000ff0000, + 0x00ff010000ff0101, 0x00ff01000000ff00, 0x00ff01000000ff01, 0x00ff0100000000ff, + 0x00ff010000000000, 0x00ff010000000001, 0x00ff010000000100, 0x00ff01000001ffff, + 0x00ff01000001ff01, 0x00ff010000010000, 0x00ff010000010001, 0x00ff010000010101, + 0x00ff010001ff0001, 0x00ff010001ff0100, 0x00ff01000100ff01, 0x00ff010001000000, + 0x00ff010001000001, 0x00ff0100010001ff, 0x00ff01000101ff00, 0x00ff0100010100ff, + 0x00ff010001010001, 0x00ff010001010100, 0x00ff0101ff000001, 0x00ff010100ff00ff, + 0x00ff010100ff0001, 0x00ff010100ff0100, 0x00ff010100000000, 0x00ff0101000001ff, + 0x00ff010100000101, 0x00ff0101000100ff, 0x00ff010100010100, 0x00ff0101010000ff, + 0x00ff010101010000, 0x0000ffffffffff00, 0x0000ffffffff00ff, 0x0000ffffffff0000, + 0x0000ffffffff0001, 0x0000ffffffff0100, 0x0000ffffff00ff01, 0x0000ffffff000000, + 0x0000ffffff000101, 0x0000ffffff01ff00, 0x0000ffffff0100ff, 0x0000ffffff010100, + 0x0000ffff00ffffff, 0x0000ffff00ff0000, 0x0000ffff00ff01ff, 0x0000ffff0000ff00, + 0x0000ffff000000ff, 0x0000ffff00000000, 0x0000ffff00000001, 0x0000ffff00000100, + 0x0000ffff00010000, 0x0000ffff000101ff, 0x0000ffff01ff0001, 0x0000ffff01ff0100, + 0x0000ffff01000000, 0x0000ffff010001ff, 0x0000ffff0101ffff, 0x0000ffff0101ff00, + 0x0000ffff01010001, 0x0000ffff01010100, 0x0000ff00ffff0000, 0x0000ff00ffff01ff, + 0x0000ff00ffff0100, 0x0000ff00ffff0101, 0x0000ff00ff00ff00, 0x0000ff00ff0000ff, + 0x0000ff00ff000000, 0x0000ff00ff000001, 0x0000ff00ff0001ff, 0x0000ff00ff000100, + 0x0000ff00ff01ffff, 0x0000ff00ff010000, 0x0000ff00ff010001, 0x0000ff00ff0101ff, + 0x0000ff00ff010101, 0x0000ff0000ffff00, 0x0000ff0000ff00ff, 0x0000ff0000ff0000, + 0x0000ff0000ff0001, 0x0000ff0000ff0100, 0x0000ff000000ffff, 0x0000ff000000ff00, + 0x0000ff000000ff01, 0x0000ff00000000ff, 0x0000ff0000000000, 0x0000ff0000000001, + 0x0000ff00000001ff, 0x0000ff0000000100, 0x0000ff0000000101, 0x0000ff000001ff00, + 0x0000ff00000100ff, 0x0000ff0000010000, 0x0000ff0000010001, 0x0000ff0000010100, + 0x0000ff0001ffff01, 0x0000ff0001ff0000, 0x0000ff000100ff00, 0x0000ff00010000ff, + 0x0000ff0001000000, 0x0000ff0001000001, 0x0000ff0001000100, 0x0000ff000101ffff, + 0x0000ff0001010000, 0x0000ff0001010101, 0x0000ff01ffffff00, 0x0000ff01ffff0001, + 0x0000ff01ff00ff01, 0x0000ff01ff000000, 0x0000ff01ff000101, 0x0000ff01ff01ff00, + 0x0000ff01ff0100ff, 0x0000ff0100ffff01, 0x0000ff0100ff0000, 0x0000ff0100ff0101, + 0x0000ff010000ff00, 0x0000ff01000000ff, 0x0000ff0100000000, 0x0000ff0100000001, + 0x0000ff0100000100, 0x0000ff010001ff01, 0x0000ff0100010000, 0x0000ff0101ff0000, + 0x0000ff010100ffff, 0x0000ff010100ff01, 0x0000ff0101000000, 0x0000ff0101000100, + 0x0000ff0101000101, 0x0000ff01010100ff, 0x000000ffffff00ff, 0x000000ffffff0000, + 0x000000ffff00ff00, 0x000000ffff0000ff, 0x000000ffff000000, 0x000000ffff000001, + 0x000000ffff0001ff, 0x000000ffff000100, 0x000000ffff01ff00, 0x000000ffff010000, + 0x000000ffff0101ff, 0x000000ffff010101, 0x000000ff00ffff00, 0x000000ff00ff00ff, + 0x000000ff00ff0000, 0x000000ff00ff0001, 0x000000ff00ff0100, 0x000000ff00ff0101, + 0x000000ff0000ffff, 0x000000ff0000ff00, 0x000000ff000000ff, 0x000000ff00000000, + 0x000000ff00000001, 0x000000ff000001ff, 0x000000ff00000100, 0x000000ff00000101, + 0x000000ff0001ff00, 0x000000ff0001ff01, 0x000000ff000100ff, 0x000000ff00010000, + 0x000000ff00010001, 0x000000ff00010100, 0x000000ff01ffffff, 0x000000ff01ff01ff, + 0x000000ff01ff0101, 0x000000ff0100ff00, 0x000000ff010000ff, 0x000000ff01000000, + 0x000000ff01000001, 0x000000ff01000100, 0x000000ff0101ff00, 0x000000ff010100ff, + 0x000000ff01010000, 0x000000ff01010101, 0x00000000ffffff00, 0x00000000ffffff01, + 0x00000000ffff00ff, 0x00000000ffff0000, 0x00000000ffff0001, 0x00000000ffff0100, + 0x00000000ff00ffff, 0x00000000ff00ff00, 0x00000000ff00ff01, 0x00000000ff0000ff, + 0x00000000ff000000, 0x00000000ff000001, 0x00000000ff000100, 0x00000000ff000101, + 0x00000000ff01ff00, 0x00000000ff0100ff, 0x00000000ff010000, 0x00000000ff010001, + 0x00000000ff010100, 0x0000000000ffffff, 0x0000000000ffff00, 0x0000000000ffff01, + 0x0000000000ff00ff, 0x0000000000ff0000, 0x0000000000ff0001, 0x0000000000ff01ff, 0x0000000000ff0100, 0x000000000000ffff, 0x000000000000ff00, 0x000000000000ff01, - 0x00000000000000ff, 0x0000000000000001, 0x00000000000001ff, 0x0000000000000100, - 0x0000000000000101, 0x000000000001ff00, 0x00000000000100ff, 0x0000000000010000, - 0x0000000000010001, 0x0000000000010100, 0x0000000001ff0000, 0x000000000100ff00, - 0x00000000010000ff, 0x0000000001000000, 0x0000000001000001, 0x0000000001000100, - 0x0000000001010000, 0x00000001ffff01ff, 0x00000001ff000000, 0x0000000100ff0000, - 0x000000010000ff00, 0x00000001000000ff, 0x0000000100000000, 0x0000000100000001, - 0x0000000100000100, 0x0000000100010000, 0x0000000101000000, 0x000001ffff00ff00, - 0x000001ffff010001, 0x000001ffff0101ff, 0x000001ff00ffff01, 0x000001ff0000ffff, - 0x000001ff00000000, 0x000001ff010000ff, 0x000001ff01010100, 0x00000100ffff0100, - 0x00000100ff000000, 0x0000010000ff0000, 0x000001000000ff00, 0x00000100000000ff, - 0x0000010000000000, 0x0000010000000001, 0x0000010000000100, 0x0000010000010000, - 0x0000010001000000, 0x000001000101ff01, 0x00000101ffff0001, 0x00000101ff01ffff, - 0x0000010100000000, 0x0000010101010100, 0x0001ffffff000000, 0x0001ffff00ffffff, - 0x0001ffff00000100, 0x0001ffff0001ff00, 0x0001ffff01000000, 0x0001ff00ffffff00, - 0x0001ff00ffff01ff, 0x0001ff00ff010000, 0x0001ff0000000000, 0x0001ff0000010001, - 0x0001ff0001ff0000, 0x0001ff0001010100, 0x0001ff01ff0000ff, 0x0001ff01ff000001, - 0x0001ff0100ffffff, 0x0001ff010001ffff, 0x0001ff01000101ff, 0x0001ff010100ff01, - 0x000100ffff00ffff, 0x000100ffff00ff01, 0x000100ffff000100, 0x000100ff00000000, - 0x000100ff000101ff, 0x000100ff01ff0101, 0x000100ff0100ffff, 0x000100ff01010101, - 0x00010000ff000000, 0x00010000ff010100, 0x0001000000ff0000, 0x000100000000ff00, - 0x00010000000000ff, 0x0001000000000000, 0x0001000000000001, 0x0001000000000100, - 0x0001000000010000, 0x0001000001ffff01, 0x0001000001000000, 0x0001000100ff0101, - 0x0001000100000000, 0x00010001010100ff, 0x000101ffffff01ff, 0x000101ffffff0101, - 0x000101ff00010000, 0x000101ff01ff0000, 0x000101ff0100ff01, 0x00010100ffff0000, - 0x0001010000000000, 0x000101000001ffff, 0x0001010000010101, 0x00010100010001ff, - 0x00010101ff00ff00, 0x00010101ff010001, 0x0001010100ffffff, 0x0001010100ff01ff, - 0x00010101000101ff, 0x0001010101ff0000, 0x000101010100ff01, 0x0001010101000101, - 0x01ffffffffff0101, 0x01ffffffff01ffff, 0x01ffffffff01ff01, 0x01ffffffff0101ff, - 0x01ffffffff010101, 0x01ffffff00000000, 0x01ffffff01ff01ff, 0x01ffffff01000101, - 0x01ffffff0101ff01, 0x01ffffff010100ff, 0x01ffff000000ff00, 0x01ffff0000000001, - 0x01ffff00000001ff, 0x01ffff0000010000, 0x01ffff0001ff0000, 0x01ffff01ffffffff, - 0x01ffff01ffff01ff, 0x01ffff01ff000000, 0x01ffff01ff01ffff, 0x01ffff01ff0101ff, - 0x01ffff010100ffff, 0x01ff00ffffff0000, 0x01ff00ffff010000, 0x01ff00ff00ffff01, - 0x01ff0000ff0000ff, 0x01ff000000000000, 0x01ff00000001ff01, 0x01ff000001ffffff, - 0x01ff000001010100, 0x01ff0001ffffff01, 0x01ff0001ff010001, 0x01ff000101ff0100, - 0x01ff000101000001, 0x01ff0001010100ff, 0x01ff01ffff00ffff, 0x01ff01ff00010001, - 0x01ff01ff01000000, 0x01ff01ff010101ff, 0x01ff0100ff000001, 0x01ff010000ffff00, - 0x01ff010000000100, 0x01ff010001ff01ff, 0x01ff01000101ffff, 0x01ff0101ffff00ff, - 0x01ff0101ffff0101, 0x01ff0101ff0101ff, 0x01ff010100010000, 0x0100ffff00ff00ff, - 0x0100ffff00ff0001, 0x0100ffff00000100, 0x0100ffff0100ff00, 0x0100ff00ffff0000, - 0x0100ff00ff00ffff, 0x0100ff00ff00ff01, 0x0100ff00ff000100, 0x0100ff00ff010000, - 0x0100ff0000000000, 0x0100ff00000100ff, 0x0100ff0001ff0101, 0x0100ff0001010101, - 0x0100ff0100ff00ff, 0x0100ff0100ff0001, 0x0100ff0100000100, 0x0100ff0100010001, - 0x0100ff0101000000, 0x010000ffff00ff00, 0x010000ff0000ffff, 0x010000ff00000000, - 0x010000ff010001ff, 0x010000ff01010001, 0x01000000ffffff00, 0x01000000ffff0101, - 0x01000000ff000000, 0x01000000ff0100ff, 0x01000000ff010101, 0x0100000000ff0000, - 0x010000000000ff00, 0x01000000000000ff, 0x0100000000000000, 0x0100000000000001, - 0x0100000000000100, 0x0100000000010000, 0x0100000001000000, 0x0100000100000000, - 0x01000001000101ff, 0x0100000101ffff01, 0x010001ffff000101, 0x010001ff00ff0100, - 0x010001ff0000ff00, 0x010001ff000100ff, 0x010001ff01ffffff, 0x01000100ffff0000, - 0x01000100ff0001ff, 0x0100010000000000, 0x010001000001ff00, 0x0100010001ff0000, - 0x01000100010000ff, 0x0100010001000101, 0x01000101ff00ff01, 0x0100010100ff0100, - 0x010001010000ffff, 0x0100010101010001, 0x0101ffffffff0101, 0x0101ffffff0001ff, - 0x0101ffffff01ffff, 0x0101ffffff010101, 0x0101ffff00000000, 0x0101ffff0101ffff, - 0x0101ffff010101ff, 0x0101ff00ff000000, 0x0101ff0000ff0100, 0x0101ff000000ff00, - 0x0101ff0000010000, 0x0101ff00010000ff, 0x0101ff0001000001, 0x0101ff01ff010101, - 0x0101ff0100000000, 0x0101ff010101ff00, 0x010100ffffff0000, 0x010100ffff010000, - 0x010100ff00ff01ff, 0x010100ff000000ff, 0x010100ff00000101, 0x010100ff01ffff00, - 0x01010000ffffff01, 0x01010000ff000100, 0x01010000ff01ff01, 0x0101000000000000, - 0x01010000000100ff, 0x010100000101ff01, 0x01010001ffff0000, 0x01010001ff00ffff, - 0x01010001ff010000, 0x0101000101ffffff, 0x0101000101ff01ff, 0x0101000101010101, - 0x010101ffff01ffff, 0x010101ff00000000, 0x010101ff0001ff01, 0x010101ff0101ffff, - 0x010101ff010101ff, 0x01010100ffffffff, 0x01010100ff000001, 0x010101000000ff00, - 0x0101010001010000, 0x0101010100ff0001, 0x010101010001ff01, 0x010101010101ffff, + 0x00000000000000ff, 0x0000000000000000, 0x0000000000000001, 0x00000000000001ff, + 0x0000000000000100, 0x0000000000000101, 0x000000000001ffff, 0x000000000001ff00, + 0x00000000000100ff, 0x0000000000010000, 0x0000000000010001, 0x00000000000101ff, + 0x0000000000010100, 0x0000000000010101, 0x0000000001ffff00, 0x0000000001ff00ff, + 0x0000000001ff0000, 0x0000000001ff0100, 0x0000000001ff0101, 0x000000000100ffff, + 0x000000000100ff00, 0x00000000010000ff, 0x0000000001000000, 0x0000000001000001, + 0x00000000010001ff, 0x0000000001000100, 0x000000000101ff00, 0x00000000010100ff, + 0x0000000001010000, 0x0000000001010001, 0x0000000001010100, 0x00000001ffffffff, + 0x00000001ffffff00, 0x00000001ffffff01, 0x00000001ffff00ff, 0x00000001ffff0001, + 0x00000001ffff01ff, 0x00000001ffff0100, 0x00000001ff00ff00, 0x00000001ff0000ff, + 0x00000001ff000000, 0x00000001ff0001ff, 0x00000001ff000100, 0x00000001ff01ffff, + 0x00000001ff01ff00, 0x00000001ff01ff01, 0x00000001ff0100ff, 0x00000001ff010000, + 0x00000001ff010001, 0x00000001ff0101ff, 0x00000001ff010100, 0x0000000100ffff00, + 0x0000000100ff0000, 0x0000000100ff0001, 0x0000000100ff01ff, 0x0000000100ff0100, + 0x0000000100ff0101, 0x000000010000ffff, 0x000000010000ff00, 0x000000010000ff01, + 0x00000001000000ff, 0x0000000100000000, 0x0000000100000001, 0x00000001000001ff, + 0x0000000100000100, 0x0000000100000101, 0x000000010001ff00, 0x00000001000100ff, + 0x0000000100010000, 0x0000000100010100, 0x0000000101ffff01, 0x0000000101ff0000, + 0x0000000101ff0001, 0x0000000101ff01ff, 0x0000000101ff0100, 0x0000000101ff0101, + 0x000000010100ff00, 0x0000000101000000, 0x0000000101000101, 0x000000010101ff01, + 0x0000000101010000, 0x0000000101010001, 0x00000001010101ff, 0x0000000101010100, + 0x000001ffffff00ff, 0x000001ffffff0000, 0x000001ffffff0001, 0x000001ffffff0100, + 0x000001ffff00ffff, 0x000001ffff000000, 0x000001ffff0001ff, 0x000001ffff01ff00, + 0x000001ffff010101, 0x000001ff00ff0000, 0x000001ff00ff01ff, 0x000001ff00ff0101, + 0x000001ff0000ff00, 0x000001ff000000ff, 0x000001ff00000000, 0x000001ff00000001, + 0x000001ff000001ff, 0x000001ff00000100, 0x000001ff0001ffff, 0x000001ff0001ff01, + 0x000001ff000100ff, 0x000001ff00010000, 0x000001ff01ffff01, 0x000001ff01ff0100, + 0x000001ff0100ffff, 0x000001ff0100ff01, 0x000001ff01000000, 0x000001ff010001ff, + 0x000001ff0101ff00, 0x000001ff01010100, 0x00000100ffffff00, 0x00000100ffffff01, + 0x00000100ffff0000, 0x00000100ffff0101, 0x00000100ff00ff00, 0x00000100ff0000ff, + 0x00000100ff000000, 0x00000100ff000001, 0x00000100ff000100, 0x00000100ff010000, + 0x0000010000ffff00, 0x0000010000ff00ff, 0x0000010000ff0000, 0x0000010000ff0001, + 0x0000010000ff0100, 0x000001000000ffff, 0x000001000000ff00, 0x000001000000ff01, + 0x00000100000000ff, 0x0000010000000000, 0x0000010000000001, 0x00000100000001ff, + 0x0000010000000100, 0x0000010000000101, 0x000001000001ff00, 0x00000100000100ff, + 0x0000010000010000, 0x0000010000010001, 0x0000010000010100, 0x0000010001ffff00, + 0x0000010001ff0000, 0x0000010001ff0100, 0x000001000100ff00, 0x00000100010000ff, + 0x0000010001000000, 0x0000010001000001, 0x00000100010001ff, 0x0000010001000100, + 0x0000010001010000, 0x00000101ffff00ff, 0x00000101ffff01ff, 0x00000101ff000000, + 0x00000101ff000101, 0x00000101ff01ffff, 0x00000101ff010000, 0x00000101ff010001, + 0x00000101ff010100, 0x0000010100ff0000, 0x0000010100ff01ff, 0x0000010100ff0100, + 0x000001010000ff00, 0x0000010100000000, 0x0000010100000001, 0x00000101000001ff, + 0x0000010100000100, 0x000001010001ff01, 0x0000010100010000, 0x00000101000101ff, + 0x0000010100010101, 0x0000010101ffff00, 0x0000010101ff0101, 0x000001010100ff01, + 0x0000010101000000, 0x0000010101000001, 0x00000101010001ff, 0x0000010101000101, + 0x000001010101ff00, 0x0001ffffffff0000, 0x0001ffffff0000ff, 0x0001ffffff000001, + 0x0001ffffff000100, 0x0001ffffff010000, 0x0001ffff00ff00ff, 0x0001ffff0000ffff, + 0x0001ffff00000000, 0x0001ffff00000001, 0x0001ffff000001ff, 0x0001ffff00000101, + 0x0001ffff0001ff00, 0x0001ffff000100ff, 0x0001ffff00010001, 0x0001ffff00010100, + 0x0001ffff01ffff00, 0x0001ffff01000001, 0x0001ffff01010000, 0x0001ff00ffffff00, + 0x0001ff00ffff00ff, 0x0001ff00ffff0001, 0x0001ff00ffff0100, 0x0001ff00ff00ff01, + 0x0001ff00ff000000, 0x0001ff00ff01ff00, 0x0001ff00ff01ff01, 0x0001ff00ff010001, + 0x0001ff00ff010100, 0x0001ff0000ff0000, 0x0001ff0000ff0100, 0x0001ff000000ff00, + 0x0001ff0000000000, 0x0001ff0000000001, 0x0001ff0000000100, 0x0001ff0000010000, + 0x0001ff0000010001, 0x0001ff0000010101, 0x0001ff0001ff00ff, 0x0001ff0001ff0101, + 0x0001ff000100ff01, 0x0001ff0001000000, 0x0001ff000101ff00, 0x0001ff0001010001, + 0x0001ff0001010100, 0x0001ff01ff00ff00, 0x0001ff01ff000001, 0x0001ff01ff000100, + 0x0001ff0100ffffff, 0x0001ff0100ffff00, 0x0001ff0100ff0001, 0x0001ff0100000000, + 0x0001ff0100000001, 0x0001ff01000001ff, 0x0001ff010001ffff, 0x0001ff0101ff0000, + 0x0001ff010100ff00, 0x0001ff0101000001, 0x0001ff0101010000, 0x000100ffff00ff00, + 0x000100ffff00ff01, 0x000100ffff000000, 0x000100ffff000001, 0x000100ffff000101, + 0x000100ffff01ff00, 0x000100ffff010001, 0x000100ffff010100, 0x000100ff00ffffff, + 0x000100ff00ffff01, 0x000100ff00ff0000, 0x000100ff00ff01ff, 0x000100ff00ff0101, + 0x000100ff0000ff00, 0x000100ff000000ff, 0x000100ff00000000, 0x000100ff00000001, + 0x000100ff00000100, 0x000100ff00000101, 0x000100ff0001ffff, 0x000100ff0001ff01, + 0x000100ff00010000, 0x000100ff01ff00ff, 0x000100ff01ff0000, 0x000100ff01ff0100, + 0x000100ff0100ffff, 0x000100ff0100ff01, 0x000100ff010000ff, 0x000100ff01000000, + 0x000100ff01000001, 0x000100ff010001ff, 0x000100ff01000101, 0x000100ff0101ff00, + 0x000100ff010100ff, 0x000100ff01010100, 0x00010000ffff0000, 0x00010000ffff01ff, + 0x00010000ffff0101, 0x00010000ff00ff00, 0x00010000ff000000, 0x00010000ff000001, + 0x00010000ff000100, 0x0001000000ff00ff, 0x0001000000ff0000, 0x0001000000ff0001, + 0x0001000000ff0100, 0x000100000000ffff, 0x000100000000ff00, 0x00010000000000ff, + 0x0001000000000000, 0x0001000000000001, 0x0001000000000100, 0x000100000001ff00, + 0x00010000000100ff, 0x0001000000010000, 0x0001000000010001, 0x0001000000010100, + 0x0001000001ff0001, 0x0001000001ff0100, 0x0001000001ff0101, 0x000100000100ff00, + 0x0001000001000000, 0x0001000001000001, 0x0001000001000100, 0x0001000001000101, + 0x000100000101ff01, 0x0001000001010000, 0x0001000001010001, 0x00010000010101ff, + 0x00010001ffffff01, 0x00010001ffff0100, 0x00010001ff000000, 0x00010001ff01ffff, + 0x00010001ff010001, 0x00010001ff0101ff, 0x00010001ff010100, 0x0001000100ffffff, + 0x0001000100ff0000, 0x0001000100ff01ff, 0x0001000100ff0101, 0x000100010000ff00, + 0x00010001000000ff, 0x0001000100000000, 0x0001000100000001, 0x00010001000001ff, + 0x0001000100000101, 0x000100010001ffff, 0x0001000100010000, 0x00010001000101ff, + 0x0001000101ffffff, 0x0001000101ffff01, 0x0001000101ff0000, 0x0001000101ff0101, + 0x00010001010000ff, 0x0001000101000001, 0x00010001010001ff, 0x0001000101000100, + 0x000100010101ffff, 0x00010001010100ff, 0x0001000101010001, 0x0001000101010101, + 0x000101ffff000001, 0x000101ffff000100, 0x000101ffff010000, 0x000101ff00ffff00, + 0x000101ff0000ff01, 0x000101ff00000000, 0x000101ff00000101, 0x000101ff0001ff00, + 0x000101ff00010100, 0x000101ff01ff0000, 0x000101ff0100ff00, 0x000101ff010001ff, + 0x000101ff01010001, 0x00010100ffffff00, 0x00010100ffff00ff, 0x00010100ff00ffff, + 0x00010100ff000000, 0x00010100ff01ff00, 0x00010100ff0100ff, 0x00010100ff010001, + 0x00010100ff010100, 0x0001010000ffffff, 0x0001010000ffff00, 0x0001010000ff0000, + 0x0001010000ff0001, 0x0001010000ff01ff, 0x000101000000ff00, 0x00010100000000ff, + 0x0001010000000000, 0x0001010000000001, 0x0001010000000100, 0x000101000001ffff, + 0x0001010000010000, 0x0001010000010101, 0x0001010001ffff01, 0x0001010001ff00ff, + 0x0001010001ff0101, 0x0001010001000000, 0x000101000101ff00, 0x00010100010100ff, + 0x0001010001010000, 0x0001010001010100, 0x00010101ff00ff00, 0x00010101ff000001, + 0x00010101ff0001ff, 0x0001010100ffff00, 0x0001010100ff00ff, 0x0001010100ff0100, + 0x000101010000ffff, 0x0001010100000000, 0x00010101000001ff, 0x0001010100000101, + 0x00010101000100ff, 0x0001010100010000, 0x0001010100010100, 0x0001010101ff0001, + 0x00010101010000ff, 0x00010101010001ff, 0x0001010101000101, 0x0001010101010001, + 0x01ffffffffffffff, 0x01ffffffffffff01, 0x01ffffffffff01ff, 0x01ffffffffff0101, + 0x01ffffffff01ffff, 0x01ffffffff01ff01, 0x01ffffffff0101ff, 0x01ffffffff010101, + 0x01ffffff00ff0000, 0x01ffffff0000ffff, 0x01ffffff0000ff00, 0x01ffffff000000ff, + 0x01ffffff00000001, 0x01ffffff00000100, 0x01ffffff00010000, 0x01ffffff01ffffff, + 0x01ffffff01ffff01, 0x01ffffff01ff01ff, 0x01ffffff01ff0101, 0x01ffffff01000000, + 0x01ffffff0101ffff, 0x01ffffff0101ff01, 0x01ffffff010101ff, 0x01ffffff01010101, + 0x01ffff00ffff0000, 0x01ffff00ff00ff00, 0x01ffff00ff0000ff, 0x01ffff00ff000001, + 0x01ffff00ff000100, 0x01ffff00ff010000, 0x01ffff0000ffff00, 0x01ffff0000ff00ff, + 0x01ffff0000ff0100, 0x01ffff000000ffff, 0x01ffff000000ff01, 0x01ffff0000000000, + 0x01ffff0000000001, 0x01ffff00000001ff, 0x01ffff0000000100, 0x01ffff00000100ff, + 0x01ffff0000010001, 0x01ffff0000010100, 0x01ffff0001ff0000, 0x01ffff0001ff0100, + 0x01ffff00010000ff, 0x01ffff0001000001, 0x01ffff0001000100, 0x01ffff0001010000, + 0x01ffff01ffffffff, 0x01ffff01ffffff01, 0x01ffff01ffff01ff, 0x01ffff01ffff0101, + 0x01ffff01ff000000, 0x01ffff01ff01ffff, 0x01ffff01ff01ff01, 0x01ffff01ff0101ff, + 0x01ffff01ff010101, 0x01ffff010000ff00, 0x01ffff01000000ff, 0x01ffff0100000100, + 0x01ffff0100010000, 0x01ffff0101ffffff, 0x01ffff0101ffff01, 0x01ffff0101ff01ff, + 0x01ffff0101ff0101, 0x01ffff0101000000, 0x01ffff010101ffff, 0x01ffff010101ff01, + 0x01ffff01010101ff, 0x01ffff0101010101, 0x01ff00ffff0000ff, 0x01ff00ffff000100, + 0x01ff00ff00ffff00, 0x01ff00ff00ff00ff, 0x01ff00ff0000ff00, 0x01ff00ff00000000, + 0x01ff00ff00000101, 0x01ff00ff0001ff00, 0x01ff00ff000100ff, 0x01ff00ff00010100, + 0x01ff00ff010000ff, 0x01ff00ff01000100, 0x01ff0000ffffff00, 0x01ff0000ffff0100, + 0x01ff0000ff00ff01, 0x01ff0000ff000000, 0x01ff0000ff000101, 0x01ff0000ff010001, + 0x01ff0000ff010100, 0x01ff000000ffffff, 0x01ff000000ffff00, 0x01ff000000ff0000, + 0x01ff000000ff01ff, 0x01ff00000000ff00, 0x01ff0000000000ff, 0x01ff000000000000, + 0x01ff000000000001, 0x01ff000000000100, 0x01ff000000000101, 0x01ff000000010000, + 0x01ff000000010001, 0x01ff0000000101ff, 0x01ff000000010101, 0x01ff000001ffff00, + 0x01ff000001ff00ff, 0x01ff000001ff0001, 0x01ff000001ff0100, 0x01ff00000100ffff, + 0x01ff00000100ff01, 0x01ff000001000000, 0x01ff0000010001ff, 0x01ff000001010001, + 0x01ff0001ff00ff00, 0x01ff0001ff000001, 0x01ff0001ff000100, 0x01ff0001ff010000, + 0x01ff000100ffff00, 0x01ff000100ff00ff, 0x01ff000100ff0100, 0x01ff000100ff0101, + 0x01ff00010000ffff, 0x01ff000100000000, 0x01ff000100000100, 0x01ff000100000101, + 0x01ff00010001ff00, 0x01ff000100010001, 0x01ff000100010101, 0x01ff000101ff0000, + 0x01ff00010100ff00, 0x01ff000101000101, 0x01ff0001010100ff, 0x01ff01ffffffffff, + 0x01ff01ffffffff01, 0x01ff01ffffff01ff, 0x01ff01ffffff0101, 0x01ff01ffff000000, + 0x01ff01ffff01ffff, 0x01ff01ffff01ff01, 0x01ff01ffff0101ff, 0x01ff01ffff010101, + 0x01ff01ff00ffff00, 0x01ff01ff00ff0000, 0x01ff01ff0000ff00, 0x01ff01ff000000ff, + 0x01ff01ff00000100, 0x01ff01ff00010000, 0x01ff01ff00010100, 0x01ff01ff01ffffff, + 0x01ff01ff01ffff01, 0x01ff01ff01ff01ff, 0x01ff01ff01ff0101, 0x01ff01ff01000000, + 0x01ff01ff0101ffff, 0x01ff01ff0101ff01, 0x01ff01ff010101ff, 0x01ff01ff01010101, + 0x01ff0100ffff0000, 0x01ff0100ffff0001, 0x01ff0100ff00ff00, 0x01ff0100ff0000ff, + 0x01ff0100ff000001, 0x01ff0100ff010000, 0x01ff010000ffff00, 0x01ff010000ff00ff, + 0x01ff010000ff0001, 0x01ff010000ff0100, 0x01ff01000000ffff, 0x01ff01000000ff01, + 0x01ff010000000000, 0x01ff010000000101, 0x01ff01000001ff00, 0x01ff0100000100ff, + 0x01ff010001ff0000, 0x01ff010001000001, 0x01ff010001000100, 0x01ff010001010000, + 0x01ff0101ffffffff, 0x01ff0101ffffff01, 0x01ff0101ffff01ff, 0x01ff0101ffff0101, + 0x01ff0101ff000000, 0x01ff0101ff01ffff, 0x01ff0101ff01ff01, 0x01ff0101ff0101ff, + 0x01ff0101ff010101, 0x01ff010100ff0000, 0x01ff01010000ff00, 0x01ff0101000000ff, + 0x01ff010100000001, 0x01ff010101ffffff, 0x01ff010101ffff01, 0x01ff010101ff01ff, + 0x01ff010101ff0101, 0x01ff010101000000, 0x01ff01010101ffff, 0x01ff01010101ff01, + 0x01ff0101010101ff, 0x01ff010101010101, 0x0100ffffffff0000, 0x0100ffffff00ff00, + 0x0100ffffff000001, 0x0100ffffff0001ff, 0x0100ffffff000100, 0x0100ffffff010000, + 0x0100ffff00ffff00, 0x0100ffff00ff0001, 0x0100ffff00ff0100, 0x0100ffff00000000, + 0x0100ffff000001ff, 0x0100ffff00000101, 0x0100ffff00010100, 0x0100ffff00010101, + 0x0100ffff01ff0000, 0x0100ffff0100ff00, 0x0100ffff010000ff, 0x0100ffff01000001, + 0x0100ffff01000100, 0x0100ffff01010000, 0x0100ff00ffffff00, 0x0100ff00ffff00ff, + 0x0100ff00ffff0001, 0x0100ff00ffff0100, 0x0100ff00ff00ffff, 0x0100ff00ff000000, + 0x0100ff00ff0001ff, 0x0100ff00ff000101, 0x0100ff00ff01ff00, 0x0100ff00ff0100ff, + 0x0100ff00ff010001, 0x0100ff00ff010100, 0x0100ff0000ffffff, 0x0100ff0000ff0000, + 0x0100ff000000ffff, 0x0100ff000000ff00, 0x0100ff00000000ff, 0x0100ff0000000000, + 0x0100ff0000000001, 0x0100ff0000000100, 0x0100ff000001ff01, 0x0100ff0000010000, + 0x0100ff0001ff00ff, 0x0100ff0001ff0001, 0x0100ff000100ff01, 0x0100ff0001000000, + 0x0100ff00010001ff, 0x0100ff000101ff00, 0x0100ff00010100ff, 0x0100ff0001010001, + 0x0100ff0001010100, 0x0100ff01ffff0000, 0x0100ff01ff00ff00, 0x0100ff01ff0000ff, + 0x0100ff01ff000100, 0x0100ff01ff010000, 0x0100ff0100ff00ff, 0x0100ff0100ff0001, + 0x0100ff0100ff0100, 0x0100ff010000ffff, 0x0100ff010000ff01, 0x0100ff0100000000, + 0x0100ff01000001ff, 0x0100ff0100010001, 0x0100ff0100010100, 0x0100ff0101ff0000, + 0x0100ff01010000ff, 0x0100ff0101000001, 0x0100ff0101010100, 0x010000ffffffff00, + 0x010000ffffff00ff, 0x010000ffffff0001, 0x010000ffff00ffff, 0x010000ffff000000, + 0x010000ffff0001ff, 0x010000ffff010001, 0x010000ff00ffffff, 0x010000ff00ff0101, + 0x010000ff0000ff00, 0x010000ff000000ff, 0x010000ff00000000, 0x010000ff00000001, + 0x010000ff000001ff, 0x010000ff00000100, 0x010000ff0001ffff, 0x010000ff0001ff00, + 0x010000ff0001ff01, 0x010000ff00010000, 0x010000ff01ff00ff, 0x010000ff01ff0001, + 0x010000ff0100ff01, 0x010000ff010000ff, 0x010000ff01000000, 0x010000ff010001ff, + 0x010000ff0101ff00, 0x010000ff01010100, 0x01000000ffffffff, 0x01000000ffff0000, + 0x01000000ffff01ff, 0x01000000ffff0101, 0x01000000ff00ffff, 0x01000000ff00ff00, + 0x01000000ff0000ff, 0x01000000ff000000, 0x01000000ff000001, 0x01000000ff000100, + 0x01000000ff01ff00, 0x01000000ff010000, 0x01000000ff010100, 0x01000000ff010101, + 0x0100000000ffff00, 0x0100000000ff00ff, 0x0100000000ff0000, 0x0100000000ff0001, + 0x0100000000ff0100, 0x010000000000ffff, 0x010000000000ff00, 0x010000000000ff01, + 0x01000000000000ff, 0x0100000000000000, 0x0100000000000001, 0x01000000000001ff, + 0x0100000000000100, 0x0100000000000101, 0x010000000001ff00, 0x01000000000100ff, + 0x0100000000010000, 0x0100000000010001, 0x0100000000010100, 0x0100000001ffff00, + 0x0100000001ff0000, 0x0100000001ff01ff, 0x010000000100ff00, 0x010000000100ff01, + 0x01000000010000ff, 0x0100000001000000, 0x0100000001000001, 0x0100000001000100, + 0x0100000001000101, 0x010000000101ffff, 0x010000000101ff01, 0x0100000001010000, + 0x01000000010101ff, 0x0100000001010101, 0x01000001ffffff00, 0x01000001ffff00ff, + 0x01000001ff00ffff, 0x01000001ff000000, 0x01000001ff000100, 0x01000001ff01ffff, + 0x01000001ff010001, 0x01000001ff010100, 0x0100000100ff0000, 0x0100000100ff01ff, + 0x0100000100ff0100, 0x010000010000ff00, 0x010000010000ff01, 0x0100000100000000, + 0x0100000100000001, 0x0100000100000100, 0x0100000100010000, 0x01000001000101ff, + 0x0100000101ffff01, 0x0100000101ff00ff, 0x0100000101ff0100, 0x0100000101ff0101, + 0x010000010100ff01, 0x01000001010000ff, 0x0100000101000000, 0x01000001010100ff, + 0x0100000101010001, 0x0100000101010100, 0x010001ffffff0000, 0x010001ffff000001, + 0x010001ffff000100, 0x010001ffff010000, 0x010001ff00ffff00, 0x010001ff00ff0001, + 0x010001ff0000ffff, 0x010001ff0000ff01, 0x010001ff00000000, 0x010001ff00000001, + 0x010001ff00000101, 0x010001ff000100ff, 0x010001ff00010000, 0x010001ff01ff0000, + 0x010001ff0100ff00, 0x010001ff01000001, 0x010001ff01000100, 0x010001ff01010000, + 0x01000100ffff00ff, 0x01000100ffff0001, 0x01000100ffff0100, 0x01000100ff00ffff, + 0x01000100ff00ff01, 0x01000100ff000000, 0x01000100ff0001ff, 0x01000100ff000101, + 0x01000100ff01ffff, 0x01000100ff01ff00, 0x01000100ff0100ff, 0x01000100ff010001, + 0x0100010000ffffff, 0x0100010000ffff01, 0x0100010000ff0000, 0x0100010000ff01ff, + 0x0100010000ff0101, 0x010001000000ff00, 0x01000100000000ff, 0x0100010000000000, + 0x0100010000000001, 0x0100010000000100, 0x010001000001ff01, 0x0100010000010000, + 0x0100010000010001, 0x0100010000010101, 0x0100010001ffff00, 0x0100010001ff00ff, + 0x010001000100ffff, 0x010001000100ff01, 0x0100010001000000, 0x0100010001000101, + 0x010001000101ff00, 0x0100010001010001, 0x01000101ffff0000, 0x01000101ff000000, + 0x01000101ff010000, 0x0100010100ff00ff, 0x0100010100ff0001, 0x0100010100ff0100, + 0x010001010000ffff, 0x0100010100000000, 0x01000101000001ff, 0x010001010001ff00, + 0x0100010101ff0000, 0x010001010100ff00, 0x01000101010000ff, 0x0100010101000000, + 0x0100010101000001, 0x0101ffffffffffff, 0x0101ffffffffff01, 0x0101ffffffff01ff, + 0x0101ffffffff0101, 0x0101ffffff000000, 0x0101ffffff01ffff, 0x0101ffffff01ff01, + 0x0101ffffff0101ff, 0x0101ffffff010101, 0x0101ffff00ff0000, 0x0101ffff0000ff00, + 0x0101ffff000000ff, 0x0101ffff00000001, 0x0101ffff00000100, 0x0101ffff01ffffff, + 0x0101ffff01ffff01, 0x0101ffff01ff01ff, 0x0101ffff01ff0101, 0x0101ffff01000000, + 0x0101ffff0101ffff, 0x0101ffff0101ff01, 0x0101ffff010101ff, 0x0101ffff01010101, + 0x0101ff00ffff0000, 0x0101ff00ffff0100, 0x0101ff00ff00ff00, 0x0101ff00ff0000ff, + 0x0101ff00ff000001, 0x0101ff00ff000100, 0x0101ff00ff000101, 0x0101ff0000ff0001, + 0x0101ff0000ff0100, 0x0101ff000000ff00, 0x0101ff0000000000, 0x0101ff00000001ff, + 0x0101ff0000000101, 0x0101ff000001ff00, 0x0101ff00000100ff, 0x0101ff0001ff0000, + 0x0101ff000100ffff, 0x0101ff000100ff01, 0x0101ff0001000001, 0x0101ff0001000100, + 0x0101ff01ffffff01, 0x0101ff01ffff01ff, 0x0101ff01ffff0101, 0x0101ff01ff00ffff, + 0x0101ff01ff000100, 0x0101ff01ff01ff01, 0x0101ff01ff0101ff, 0x0101ff01ff010101, + 0x0101ff0100ff0000, 0x0101ff010000ff00, 0x0101ff0100000001, 0x0101ff0100000100, + 0x0101ff0100010000, 0x0101ff0101ffffff, 0x0101ff0101ffff01, 0x0101ff0101ff01ff, + 0x0101ff0101ff0101, 0x0101ff0101000000, 0x0101ff010101ffff, 0x0101ff010101ff01, + 0x0101ff01010101ff, 0x0101ff0101010101, 0x010100ffff000100, 0x010100ffff010000, + 0x010100ff00ffff00, 0x010100ff00ff00ff, 0x010100ff0000ffff, 0x010100ff000000ff, + 0x010100ff00000000, 0x010100ff000001ff, 0x010100ff00000101, 0x010100ff0001ff00, + 0x010100ff00010000, 0x010100ff00010001, 0x010100ff000101ff, 0x010100ff00010100, + 0x010100ff01ff0000, 0x01010000ffff0001, 0x01010000ffff0100, 0x01010000ff00ffff, + 0x01010000ff00ff01, 0x01010000ff000000, 0x01010000ff0001ff, 0x01010000ff010001, + 0x01010000ff010100, 0x0101000000ffff01, 0x0101000000ff0000, 0x010100000000ff00, + 0x01010000000000ff, 0x0101000000000000, 0x0101000000000001, 0x0101000000000100, + 0x0101000000010000, 0x0101000000010101, 0x0101000001ffff00, 0x0101000001ff00ff, + 0x0101000001ff0000, 0x0101000001ff0001, 0x0101000001ff0100, 0x010100000100ff01, + 0x0101000001000000, 0x01010000010001ff, 0x01010001ffff0000, 0x01010001ff00ff00, + 0x01010001ff000001, 0x01010001ff000101, 0x01010001ff01ff00, 0x01010001ff010000, + 0x0101000100ff00ff, 0x0101000100ff0001, 0x0101000100ff0101, 0x010100010000ff01, + 0x0101000100000000, 0x0101000100000001, 0x01010001000001ff, 0x010100010001ffff, + 0x010100010001ff01, 0x0101000101ff0001, 0x010100010100ffff, 0x0101000101000000, + 0x0101000101000001, 0x0101000101000100, 0x010100010101ff00, 0x01010001010100ff, + 0x0101000101010001, 0x010101ffffffffff, 0x010101ffffffff01, 0x010101ffffff01ff, + 0x010101ffffff0101, 0x010101ffff01ffff, 0x010101ffff01ff01, 0x010101ffff0101ff, + 0x010101ffff010101, 0x010101ff0000ff00, 0x010101ff000000ff, 0x010101ff00000001, + 0x010101ff00000100, 0x010101ff01ffffff, 0x010101ff01ffff01, 0x010101ff01ff01ff, + 0x010101ff01ff0101, 0x010101ff01000000, 0x010101ff0101ffff, 0x010101ff0101ff01, + 0x010101ff010101ff, 0x010101ff01010101, 0x01010100ffff0000, 0x01010100ff0000ff, + 0x01010100ff000100, 0x01010100ff01ff00, 0x01010100ff010000, 0x0101010000ffff00, + 0x010101000000ffff, 0x0101010000000000, 0x0101010000000101, 0x010101000001ff00, + 0x0101010000010001, 0x0101010000010100, 0x010101000100ffff, 0x0101010001000001, + 0x01010101ffffffff, 0x01010101ffffff01, 0x01010101ffff01ff, 0x01010101ffff0101, + 0x01010101ff01ffff, 0x01010101ff01ff01, 0x01010101ff0101ff, 0x01010101ff010101, + 0x010101010000ff00, 0x01010101000000ff, 0x0101010100000001, 0x0101010101ffffff, + 0x0101010101ffff01, 0x0101010101ff01ff, 0x0101010101ff0101, 0x0101010101000000, + 0x010101010101ffff, 0x010101010101ff01, 0x01010101010101ff, 0x0101010101010101, GGML_TABLE_END() +#else +GGML_TABLE_BEGIN(uint32_t, iq1s_grid_gpu, NGRID_IQ1S) + 0x00000000, 0x00000002, 0x00000101, 0x00000200, 0x00000202, 0x00010001, 0x00010101, 0x00020000, + 0x00020002, 0x00020200, 0x00020202, 0x01000101, 0x01010001, 0x01010100, 0x01010102, 0x01020101, + 0x02000000, 0x02000002, 0x02000200, 0x02000202, 0x02010101, 0x02020000, 0x02020002, 0x02020200, + 0x02020202, 0x00000110, 0x00000111, 0x00010011, 0x00010110, 0x00010112, 0x00010211, 0x00010212, + 0x00020111, 0x01000011, 0x01000112, 0x01000211, 0x01010012, 0x01010111, 0x01010212, 0x01020011, + 0x01020110, 0x01020112, 0x01020210, 0x02000111, 0x02010011, 0x02010110, 0x02010112, 0x02020111, + 0x00000020, 0x00000022, 0x00000220, 0x00000222, 0x00010121, 0x00020020, 0x00020022, 0x00020220, + 0x00020222, 0x01000121, 0x01010021, 0x01010221, 0x01020120, 0x01020221, 0x02000020, 0x02000022, + 0x02000220, 0x02000222, 0x02010021, 0x02010121, 0x02010221, 0x02020020, 0x02020022, 0x02020220, + 0x02020222, 0x00011001, 0x00011100, 0x00011102, 0x00021101, 0x01001001, 0x01001201, 0x01011101, + 0x01011202, 0x01021100, 0x01021101, 0x02011001, 0x02011201, 0x02021101, 0x00001011, 0x00001110, + 0x00001111, 0x00001112, 0x00011111, 0x00011210, 0x00011212, 0x00021211, 0x01001010, 0x01001111, + 0x01001212, 0x01011010, 0x01011011, 0x01011110, 0x01011111, 0x01011112, 0x01011211, 0x01021010, + 0x01021012, 0x01021111, 0x01021210, 0x01021212, 0x02001011, 0x02011011, 0x02011111, 0x02011210, + 0x02011212, 0x02021011, 0x02021110, 0x02021111, 0x02021112, 0x02021211, 0x00011120, 0x00011221, + 0x01001021, 0x01001120, 0x01011020, 0x01011022, 0x01011121, 0x01011220, 0x01021020, 0x01021021, + 0x01021122, 0x01021221, 0x02001121, 0x02011021, 0x02011120, 0x02011221, 0x00002000, 0x00002002, + 0x00002200, 0x00002202, 0x00012101, 0x00022000, 0x00022002, 0x00022200, 0x00022202, 0x01002101, + 0x01012001, 0x01012102, 0x01022101, 0x02002000, 0x02002002, 0x02002200, 0x02002202, 0x02012101, + 0x02022000, 0x02022002, 0x02022200, 0x02022202, 0x00002111, 0x00012011, 0x00012110, 0x00012211, + 0x00022110, 0x00022111, 0x01002011, 0x01012010, 0x01012011, 0x01012111, 0x01022011, 0x01022110, + 0x01022211, 0x02012011, 0x02012110, 0x02012112, 0x02012211, 0x02022111, 0x00002020, 0x00002022, + 0x00002220, 0x00002222, 0x00012121, 0x00022020, 0x00022022, 0x00022220, 0x00022222, 0x01002121, + 0x01012021, 0x01012221, 0x01022021, 0x01022121, 0x02002020, 0x02002022, 0x02002121, 0x02002220, + 0x02002222, 0x02012121, 0x02022020, 0x02022022, 0x02022220, 0x02022222, 0x00110000, 0x00110001, + 0x00110100, 0x00110201, 0x00120100, 0x00120101, 0x01100001, 0x01100100, 0x01110000, 0x01110101, + 0x01110200, 0x01120001, 0x01120100, 0x01120101, 0x01120201, 0x02110001, 0x02110100, 0x02110102, + 0x02120001, 0x02120101, 0x00100011, 0x00100110, 0x00100112, 0x00100211, 0x00110010, 0x00110012, + 0x00110111, 0x00110210, 0x00120011, 0x00120110, 0x00120211, 0x01100111, 0x01100212, 0x01110010, + 0x01110011, 0x01110012, 0x01110110, 0x01110111, 0x01110112, 0x01110211, 0x01120010, 0x01120111, + 0x02100110, 0x02110012, 0x02110111, 0x02120011, 0x02120110, 0x00110021, 0x00110120, 0x00110122, + 0x00120121, 0x01100020, 0x01100122, 0x01100221, 0x01110022, 0x01110121, 0x01110220, 0x01110222, + 0x01120120, 0x01120122, 0x02100121, 0x02110021, 0x02110120, 0x02110122, 0x02120121, 0x00101001, + 0x00101102, 0x00101201, 0x00111100, 0x00111101, 0x00111200, 0x00111201, 0x00121001, 0x00121102, + 0x01101001, 0x01101101, 0x01101102, 0x01101200, 0x01101202, 0x01111001, 0x01111100, 0x01111101, + 0x01111102, 0x01111201, 0x01121002, 0x01121101, 0x01121200, 0x02101100, 0x02101201, 0x02111000, + 0x02111100, 0x02111101, 0x02111200, 0x02111201, 0x02111202, 0x02121001, 0x02121100, 0x02121101, + 0x02121201, 0x00101012, 0x00101111, 0x00101212, 0x00111011, 0x00111110, 0x00111111, 0x00111112, + 0x00111211, 0x00121010, 0x00121012, 0x00121111, 0x00121210, 0x00121212, 0x01101011, 0x01101110, + 0x01101111, 0x01101112, 0x01111011, 0x01111012, 0x01111110, 0x01111111, 0x01111112, 0x01111211, + 0x01111212, 0x01121011, 0x01121110, 0x01121111, 0x01121112, 0x01121211, 0x02101010, 0x02101012, + 0x02101110, 0x02101111, 0x02101210, 0x02101212, 0x02111010, 0x02111011, 0x02111110, 0x02111111, + 0x02111112, 0x02111211, 0x02111212, 0x02121010, 0x02121012, 0x02121111, 0x00101021, 0x00101120, + 0x00101121, 0x00101122, 0x00111121, 0x00111122, 0x00111220, 0x00111222, 0x00121021, 0x00121122, + 0x01101020, 0x01101022, 0x01101120, 0x01101121, 0x01101220, 0x01101222, 0x01111021, 0x01111121, + 0x01111122, 0x01111220, 0x01111221, 0x01121021, 0x01121120, 0x01121121, 0x01121220, 0x01121221, + 0x01121222, 0x02101122, 0x02101222, 0x02111022, 0x02111121, 0x02121120, 0x02121221, 0x00112001, + 0x00112102, 0x00122101, 0x01102001, 0x01102100, 0x01102102, 0x01102201, 0x01112000, 0x01112101, + 0x01112200, 0x01112202, 0x01122000, 0x01122001, 0x01122100, 0x01122102, 0x01122201, 0x02102101, + 0x02112001, 0x02112100, 0x02122101, 0x00112010, 0x00112012, 0x00112111, 0x00112212, 0x00122011, + 0x00122111, 0x01102012, 0x01102110, 0x01102111, 0x01102210, 0x01112011, 0x01112110, 0x01112111, + 0x01112112, 0x01112211, 0x01112212, 0x01122010, 0x01122111, 0x01122212, 0x02102211, 0x02112011, + 0x02112012, 0x02112111, 0x02112210, 0x02122011, 0x02122112, 0x02122211, 0x00102221, 0x00112122, + 0x00122120, 0x00122122, 0x01102120, 0x01102122, 0x01102221, 0x01112020, 0x01112022, 0x01112121, + 0x01112220, 0x01122021, 0x01122122, 0x01122221, 0x02102121, 0x02112021, 0x02112122, 0x02112222, + 0x00200000, 0x00200002, 0x00200200, 0x00200202, 0x00210101, 0x00220000, 0x00220002, 0x00220101, + 0x00220200, 0x00220202, 0x01200101, 0x01210001, 0x01210201, 0x01220001, 0x01220101, 0x02200000, + 0x02200002, 0x02200200, 0x02200202, 0x02210101, 0x02220000, 0x02220002, 0x02220101, 0x02220200, + 0x02220202, 0x00200111, 0x00210011, 0x00210110, 0x00210211, 0x00220111, 0x01200012, 0x01200110, + 0x01200211, 0x01210111, 0x01210210, 0x01210212, 0x01220011, 0x01220110, 0x01220111, 0x01220112, + 0x02200111, 0x02210010, 0x02210112, 0x02210211, 0x02220111, 0x00200021, 0x00200220, 0x00200222, + 0x00210021, 0x00210121, 0x00220020, 0x00220022, 0x00220220, 0x00220222, 0x01200121, 0x01210021, + 0x01210122, 0x01210221, 0x01220121, 0x02200021, 0x02200220, 0x02200222, 0x02210021, 0x02210121, + 0x02220020, 0x02220022, 0x02220220, 0x02220222, 0x00201101, 0x00211100, 0x00211102, 0x00211201, + 0x00221101, 0x01201100, 0x01201101, 0x01201102, 0x01201201, 0x01211002, 0x01211101, 0x01211200, + 0x01211202, 0x01221102, 0x02201101, 0x02211001, 0x02211100, 0x02211201, 0x02221001, 0x02221101, + 0x00201211, 0x00211111, 0x00221011, 0x00221211, 0x01201010, 0x01201111, 0x01201210, 0x01211011, + 0x01211110, 0x01211111, 0x01211211, 0x01221012, 0x01221111, 0x01221210, 0x02201211, 0x02211010, + 0x02211110, 0x02211111, 0x02211210, 0x02211212, 0x02221011, 0x02221110, 0x02221112, 0x02221211, + 0x00201121, 0x00211020, 0x00211022, 0x00211221, 0x00221121, 0x01201021, 0x01201221, 0x01211121, + 0x01221020, 0x01221021, 0x01221221, 0x02201120, 0x02201122, 0x02211020, 0x02211222, 0x00202000, + 0x00202002, 0x00202200, 0x00202202, 0x00212101, 0x00222000, 0x00222002, 0x00222200, 0x00222202, + 0x01202101, 0x01212001, 0x01212100, 0x01222101, 0x02202000, 0x02202002, 0x02202200, 0x02202202, + 0x02222000, 0x02222002, 0x02222200, 0x02222202, 0x00202211, 0x00212011, 0x00212110, 0x00212211, + 0x00222111, 0x01202112, 0x01202211, 0x01212012, 0x01212111, 0x01222011, 0x01222110, 0x01222112, + 0x01222211, 0x02202111, 0x02212010, 0x02212112, 0x02212211, 0x02222110, 0x02222111, 0x00202020, + 0x00202022, 0x00202220, 0x00202222, 0x00222020, 0x00222022, 0x00222220, 0x00222222, 0x01202121, + 0x01212021, 0x01212122, 0x01212221, 0x01222121, 0x02202020, 0x02202022, 0x02202220, 0x02202222, + 0x02212121, 0x02222020, 0x02222022, 0x02222220, 0x02222222, 0x10000101, 0x10010001, 0x10010102, + 0x10020101, 0x11000201, 0x11010002, 0x11010101, 0x11010200, 0x11010202, 0x11020001, 0x11020100, + 0x11020102, 0x12010100, 0x12010201, 0x12020001, 0x12020102, 0x10000010, 0x10000011, 0x10000110, + 0x10000112, 0x10000211, 0x10010012, 0x10010111, 0x10010112, 0x10010210, 0x10010212, 0x10020011, + 0x10020112, 0x10020211, 0x11000111, 0x11000210, 0x11000212, 0x11010011, 0x11010110, 0x11010111, + 0x11010112, 0x11010211, 0x11010212, 0x11020111, 0x11020210, 0x11020212, 0x12000011, 0x12000110, + 0x12000112, 0x12010010, 0x12010012, 0x12010111, 0x12020010, 0x12020011, 0x12020012, 0x10000121, + 0x10010021, 0x10010120, 0x10010122, 0x10020121, 0x11000021, 0x11010022, 0x11010121, 0x11010222, + 0x11020120, 0x11020221, 0x12000221, 0x12010120, 0x12020121, 0x10001001, 0x10011101, 0x10011201, + 0x10021201, 0x11001101, 0x11001200, 0x11001202, 0x11011001, 0x11011100, 0x11011101, 0x11011102, + 0x11021001, 0x11021002, 0x11021101, 0x11021200, 0x11021202, 0x12001001, 0x12001102, 0x12001201, + 0x12011000, 0x12011002, 0x12011101, 0x12021000, 0x12021001, 0x12021201, 0x10001011, 0x10001012, + 0x10001111, 0x10001212, 0x10011011, 0x10011110, 0x10011111, 0x10011112, 0x10011211, 0x10021010, + 0x10021111, 0x10021212, 0x11001011, 0x11001110, 0x11001111, 0x11001112, 0x11001211, 0x11011010, + 0x11011011, 0x11011110, 0x11011111, 0x11011112, 0x11011210, 0x11011211, 0x11021011, 0x11021110, + 0x11021111, 0x11021112, 0x11021211, 0x12001012, 0x12001110, 0x12001111, 0x12001210, 0x12011011, + 0x12011110, 0x12011111, 0x12011112, 0x12011211, 0x12011212, 0x12021111, 0x12021210, 0x12021212, + 0x10001021, 0x10001121, 0x10001221, 0x10011120, 0x10011121, 0x10011220, 0x10011222, 0x10021021, + 0x10021120, 0x10021221, 0x11001020, 0x11001022, 0x11001121, 0x11001220, 0x11011020, 0x11011021, + 0x11011022, 0x11011121, 0x11011122, 0x11011221, 0x11021022, 0x11021121, 0x11021220, 0x12001021, + 0x12001121, 0x12001222, 0x12011120, 0x12011121, 0x12021021, 0x12021120, 0x12021122, 0x10002101, + 0x10012001, 0x10012101, 0x10012202, 0x10022101, 0x11002002, 0x11002201, 0x11012000, 0x11012101, + 0x11012200, 0x11022001, 0x11022100, 0x11022102, 0x11022201, 0x12002101, 0x12012001, 0x12012100, + 0x12012102, 0x12012201, 0x12022101, 0x10002011, 0x10002111, 0x10002112, 0x10002212, 0x10012010, + 0x10012110, 0x10012111, 0x10012210, 0x10022011, 0x10022110, 0x10022112, 0x11002010, 0x11002111, + 0x11002212, 0x11012011, 0x11012012, 0x11012110, 0x11012111, 0x11012112, 0x11012211, 0x11022010, + 0x11022012, 0x11022111, 0x11022112, 0x11022212, 0x12002112, 0x12002211, 0x12012012, 0x12012111, + 0x12012112, 0x12012210, 0x12022011, 0x12022110, 0x12022112, 0x12022211, 0x10012122, 0x11002120, + 0x11002122, 0x11002221, 0x11012121, 0x11012220, 0x11012222, 0x11022120, 0x11022221, 0x12012120, + 0x12022121, 0x10100001, 0x10100100, 0x10100101, 0x10100102, 0x10100201, 0x10110002, 0x10110101, + 0x10110202, 0x10120001, 0x10120100, 0x10120201, 0x11100000, 0x11100101, 0x11100200, 0x11110001, + 0x11110100, 0x11110101, 0x11110102, 0x11110201, 0x11120101, 0x11120200, 0x12100102, 0x12100201, + 0x12110101, 0x12110200, 0x12120000, 0x12120001, 0x12120102, 0x12120201, 0x10100111, 0x10100210, + 0x10100211, 0x10100212, 0x10110011, 0x10110110, 0x10110111, 0x10110112, 0x10110210, 0x10110211, + 0x10120010, 0x10120111, 0x10120112, 0x10120210, 0x10120212, 0x11100011, 0x11100110, 0x11100111, + 0x11100112, 0x11100211, 0x11110010, 0x11110011, 0x11110012, 0x11110110, 0x11110111, 0x11110112, + 0x11110210, 0x11110211, 0x11110212, 0x11120011, 0x11120110, 0x11120111, 0x11120112, 0x11120211, + 0x12100012, 0x12100111, 0x12110011, 0x12110110, 0x12110111, 0x12110112, 0x12110211, 0x12120010, + 0x12120111, 0x12120212, 0x10100021, 0x10100122, 0x10110022, 0x10110121, 0x10110222, 0x10120021, + 0x10120120, 0x11100022, 0x11100121, 0x11100222, 0x11110021, 0x11110120, 0x11110121, 0x11110122, + 0x11110221, 0x11120022, 0x11120121, 0x12100121, 0x12110020, 0x12110022, 0x12110121, 0x12110221, + 0x12110222, 0x12120120, 0x10101100, 0x10101101, 0x10111001, 0x10111100, 0x10111101, 0x10111102, + 0x10111200, 0x10111201, 0x10121001, 0x10121101, 0x10121200, 0x10121202, 0x11101001, 0x11101100, + 0x11101101, 0x11101102, 0x11101201, 0x11101202, 0x11111000, 0x11111001, 0x11111100, 0x11111101, + 0x11111102, 0x11111200, 0x11111201, 0x11111202, 0x11121001, 0x11121002, 0x11121100, 0x11121101, + 0x11121102, 0x11121201, 0x12101000, 0x12101200, 0x12101202, 0x12111001, 0x12111100, 0x12111101, + 0x12111102, 0x12111201, 0x12121001, 0x12121100, 0x12121101, 0x12121202, 0x10101011, 0x10101012, + 0x10101110, 0x10101111, 0x10101112, 0x10101211, 0x10111010, 0x10111011, 0x10111012, 0x10111110, + 0x10111111, 0x10111112, 0x10111211, 0x10111212, 0x10121011, 0x10121110, 0x10121111, 0x10121112, + 0x10121211, 0x11101010, 0x11101011, 0x11101012, 0x11101110, 0x11101111, 0x11101112, 0x11101210, + 0x11101211, 0x11111010, 0x11111011, 0x11111012, 0x11111110, 0x11111111, 0x11111112, 0x11111210, + 0x11111211, 0x11111212, 0x11121010, 0x11121011, 0x11121110, 0x11121111, 0x11121112, 0x11121210, + 0x11121211, 0x11121212, 0x12101011, 0x12101110, 0x12101111, 0x12101211, 0x12101212, 0x12111010, + 0x12111011, 0x12111110, 0x12111111, 0x12111112, 0x12111210, 0x12111211, 0x12121011, 0x12121110, + 0x12121111, 0x12121112, 0x12121211, 0x10101020, 0x10101021, 0x10101022, 0x10101120, 0x10101122, + 0x10101220, 0x10101221, 0x10111021, 0x10111120, 0x10111121, 0x10111220, 0x10111221, 0x10121020, + 0x10121021, 0x10121022, 0x10121120, 0x10121121, 0x10121122, 0x10121220, 0x10121221, 0x11101021, + 0x11101121, 0x11101122, 0x11101220, 0x11101221, 0x11101222, 0x11111020, 0x11111021, 0x11111022, + 0x11111120, 0x11111121, 0x11111122, 0x11111220, 0x11111221, 0x11111222, 0x11121021, 0x11121120, + 0x11121121, 0x11121221, 0x12101022, 0x12101121, 0x12101122, 0x12101220, 0x12101221, 0x12101222, + 0x12111021, 0x12111121, 0x12111222, 0x12121022, 0x12121121, 0x12121122, 0x12121220, 0x12121221, + 0x10102100, 0x10102101, 0x10102102, 0x10102201, 0x10112000, 0x10112101, 0x10112200, 0x10122001, + 0x10122202, 0x11102101, 0x11102200, 0x11102202, 0x11112001, 0x11112100, 0x11112101, 0x11112102, + 0x11112200, 0x11112201, 0x11122000, 0x11122002, 0x11122100, 0x11122101, 0x12102002, 0x12102201, + 0x12112000, 0x12112002, 0x12112101, 0x12112200, 0x12122001, 0x12122201, 0x10102011, 0x10102012, + 0x10102111, 0x10102212, 0x10112011, 0x10112110, 0x10112111, 0x10112112, 0x10112211, 0x10122111, + 0x11102011, 0x11102110, 0x11102111, 0x11102112, 0x11102211, 0x11112010, 0x11112011, 0x11112012, + 0x11112110, 0x11112111, 0x11112112, 0x11112210, 0x11112211, 0x11112212, 0x11122011, 0x11122110, + 0x11122111, 0x11122112, 0x11122211, 0x12102011, 0x12102111, 0x12102211, 0x12112011, 0x12112110, + 0x12112111, 0x12112112, 0x12112210, 0x12112211, 0x12122111, 0x10102120, 0x10102220, 0x10112121, + 0x10112222, 0x10122020, 0x10122121, 0x10122122, 0x10122221, 0x11102121, 0x11102220, 0x11102221, + 0x11112021, 0x11112121, 0x11112122, 0x11112220, 0x11112221, 0x11122022, 0x11122121, 0x11122220, + 0x11122222, 0x12102021, 0x12102222, 0x12112022, 0x12112121, 0x12112122, 0x12112220, 0x12112222, + 0x12122021, 0x10200101, 0x10210100, 0x10210102, 0x10210201, 0x10220101, 0x11200100, 0x11210000, + 0x11210101, 0x11210102, 0x11210200, 0x11210202, 0x11220001, 0x11220100, 0x11220102, 0x11220201, + 0x12200001, 0x12210102, 0x12220101, 0x10200011, 0x10200110, 0x10200112, 0x10200211, 0x10210012, + 0x10210111, 0x10220011, 0x10220012, 0x10220112, 0x10220211, 0x11200111, 0x11200211, 0x11210011, + 0x11210111, 0x11210112, 0x11210211, 0x11220111, 0x11220112, 0x11220212, 0x12200110, 0x12200212, + 0x12210012, 0x12210111, 0x12220011, 0x12220112, 0x12220211, 0x10210021, 0x10210122, 0x10210221, + 0x11200020, 0x11200021, 0x11200122, 0x11210121, 0x11210122, 0x11210220, 0x11220020, 0x12200121, + 0x12210021, 0x12210122, 0x12220121, 0x10211001, 0x10211002, 0x10211101, 0x10211102, 0x10211202, + 0x10221001, 0x10221102, 0x10221201, 0x11201000, 0x11201002, 0x11201101, 0x11201200, 0x11201202, + 0x11211001, 0x11211100, 0x11211101, 0x11211102, 0x11211201, 0x11211202, 0x11221000, 0x11221002, + 0x11221101, 0x12201100, 0x12201101, 0x12201201, 0x12211000, 0x12211002, 0x12211100, 0x12211101, + 0x12211102, 0x12211200, 0x12211202, 0x12221001, 0x12221100, 0x12221201, 0x10201111, 0x10201210, + 0x10201212, 0x10211011, 0x10211111, 0x10211112, 0x10211211, 0x11201110, 0x11201111, 0x11201112, + 0x11201211, 0x11211010, 0x11211011, 0x11211110, 0x11211111, 0x11211112, 0x11211211, 0x11221011, + 0x11221110, 0x11221111, 0x11221112, 0x11221211, 0x12201112, 0x12201211, 0x12201212, 0x12211011, + 0x12211111, 0x12211112, 0x12211211, 0x12211212, 0x12221012, 0x12221111, 0x12221112, 0x12221210, + 0x10201022, 0x10201221, 0x10211121, 0x10221020, 0x10221122, 0x10221220, 0x10221221, 0x11201020, + 0x11201121, 0x11201220, 0x11201222, 0x11211021, 0x11211120, 0x11211121, 0x11211122, 0x11211220, + 0x11211222, 0x11221020, 0x11221121, 0x11221220, 0x12201020, 0x12201022, 0x12201121, 0x12201222, + 0x12211120, 0x12211122, 0x12211220, 0x12211221, 0x12221020, 0x12221120, 0x12221122, 0x12221222, + 0x10212102, 0x10212201, 0x10222101, 0x11202001, 0x11212002, 0x11212101, 0x11212202, 0x11222001, + 0x11222201, 0x12202101, 0x12212001, 0x12212200, 0x12222102, 0x10202011, 0x10202110, 0x10212010, + 0x10212111, 0x10222011, 0x10222110, 0x10222112, 0x10222211, 0x11202010, 0x11202011, 0x11202111, + 0x11202112, 0x11202210, 0x11212011, 0x11212110, 0x11212111, 0x11212112, 0x11212211, 0x11222010, + 0x11222111, 0x11222212, 0x12202012, 0x12202110, 0x12202212, 0x12212111, 0x12222011, 0x12222110, + 0x12222111, 0x12222211, 0x10212021, 0x10212122, 0x10212220, 0x11202021, 0x11202120, 0x11202221, + 0x11212020, 0x11212121, 0x11212220, 0x11212222, 0x11222120, 0x11222121, 0x11222221, 0x12202122, + 0x12212120, 0x12212220, 0x12212222, 0x12222122, 0x20000000, 0x20000002, 0x20000200, 0x20000202, + 0x20020000, 0x20020002, 0x20020200, 0x20020202, 0x21000101, 0x21010000, 0x21010001, 0x21010100, + 0x21010102, 0x21010201, 0x21020101, 0x22000000, 0x22000002, 0x22000200, 0x22000202, 0x22010101, + 0x22020000, 0x22020002, 0x22020200, 0x22020202, 0x20000111, 0x20010011, 0x20010110, 0x20010112, + 0x20010211, 0x20020111, 0x21000011, 0x21000110, 0x21000211, 0x21010010, 0x21010012, 0x21010111, + 0x21010112, 0x21010210, 0x21010211, 0x21020110, 0x21020112, 0x21020211, 0x22000111, 0x22000211, + 0x22010110, 0x22010112, 0x22010211, 0x22020111, 0x20000020, 0x20000022, 0x20000220, 0x20000222, + 0x20010121, 0x20020020, 0x20020022, 0x20020220, 0x20020222, 0x21010021, 0x21010120, 0x21010221, + 0x21020121, 0x22000020, 0x22000022, 0x22000220, 0x22000222, 0x22010121, 0x22020020, 0x22020022, + 0x22020220, 0x22020222, 0x20011100, 0x20011201, 0x21001001, 0x21001100, 0x21011001, 0x21011101, + 0x21011202, 0x21021001, 0x21021100, 0x21021201, 0x22011100, 0x22011201, 0x20001011, 0x20001211, + 0x20011012, 0x20011111, 0x20011212, 0x20021112, 0x20021211, 0x21001010, 0x21001011, 0x21001111, + 0x21001210, 0x21011011, 0x21011110, 0x21011111, 0x21011112, 0x21011211, 0x21011212, 0x21021111, + 0x21021112, 0x21021210, 0x21021212, 0x22001011, 0x22001110, 0x22001112, 0x22001211, 0x22011010, + 0x22011012, 0x22011111, 0x22011210, 0x22021112, 0x20011021, 0x20011122, 0x20011221, 0x20021121, + 0x21001021, 0x21001120, 0x21001221, 0x21001222, 0x21011020, 0x21011121, 0x21011221, 0x21011222, + 0x21021021, 0x21021122, 0x21021222, 0x22001121, 0x22011021, 0x22011222, 0x22021120, 0x20002000, + 0x20002002, 0x20002200, 0x20002202, 0x20012101, 0x20022000, 0x20022002, 0x20022200, 0x20022202, + 0x21002001, 0x21002101, 0x21012001, 0x21012100, 0x21012201, 0x21022101, 0x21022201, 0x22002000, + 0x22002002, 0x22002200, 0x22002202, 0x22012101, 0x22022000, 0x22022002, 0x22022200, 0x22022202, + 0x20002111, 0x20002112, 0x20012011, 0x20012110, 0x20012112, 0x20022111, 0x21002011, 0x21002110, + 0x21002112, 0x21002211, 0x21012010, 0x21012012, 0x21012111, 0x21012212, 0x21022011, 0x21022110, + 0x22002111, 0x22012112, 0x22012211, 0x22022111, 0x20002020, 0x20002022, 0x20002220, 0x20002222, + 0x20012121, 0x20022020, 0x20022022, 0x20022220, 0x20022222, 0x21002121, 0x21012021, 0x21012120, + 0x21012122, 0x22002020, 0x22002022, 0x22002220, 0x22002222, 0x22012121, 0x22022020, 0x22022022, + 0x22022220, 0x22022222, 0x20100101, 0x20110001, 0x20110102, 0x20110200, 0x20110201, 0x20120101, + 0x21100001, 0x21100102, 0x21100201, 0x21110101, 0x21110200, 0x21110202, 0x21120201, 0x21120202, + 0x22100101, 0x22110001, 0x22110100, 0x22110102, 0x22110201, 0x22120101, 0x20100011, 0x20100110, + 0x20100112, 0x20100211, 0x20110010, 0x20110111, 0x20110210, 0x20110212, 0x20120011, 0x20120110, + 0x20120112, 0x20120211, 0x21100010, 0x21100111, 0x21110010, 0x21110011, 0x21110110, 0x21110111, + 0x21110112, 0x21110211, 0x21120012, 0x21120111, 0x22100110, 0x22100112, 0x22110012, 0x22110111, + 0x22110210, 0x22120011, 0x22120110, 0x22120112, 0x22120211, 0x20100121, 0x20110021, 0x20110120, + 0x20110221, 0x20120121, 0x21100120, 0x21100122, 0x21100221, 0x21110020, 0x21110022, 0x21110121, + 0x21110220, 0x21120122, 0x21120221, 0x22100121, 0x22110120, 0x22110122, 0x22120221, 0x20101001, + 0x20101100, 0x20101102, 0x20111000, 0x20111101, 0x20111200, 0x20121102, 0x21101000, 0x21101202, + 0x21111001, 0x21111100, 0x21111101, 0x21111102, 0x21111200, 0x21111201, 0x21121000, 0x21121001, + 0x21121002, 0x21121101, 0x22101100, 0x22101102, 0x22111002, 0x22111100, 0x22111101, 0x22111200, + 0x22121001, 0x22121201, 0x20101010, 0x20101111, 0x20101210, 0x20101212, 0x20111010, 0x20111011, + 0x20111110, 0x20111111, 0x20111112, 0x20111211, 0x20121011, 0x20121111, 0x20121211, 0x20121212, + 0x21101011, 0x21101110, 0x21101111, 0x21101112, 0x21101211, 0x21111010, 0x21111011, 0x21111012, + 0x21111110, 0x21111111, 0x21111112, 0x21111210, 0x21111211, 0x21111212, 0x21121011, 0x21121110, + 0x21121111, 0x21121112, 0x21121211, 0x22101011, 0x22101111, 0x22101210, 0x22111011, 0x22111012, + 0x22111110, 0x22111111, 0x22111112, 0x22111211, 0x22111212, 0x22121010, 0x22121012, 0x22121111, + 0x22121210, 0x22121212, 0x20101021, 0x20101120, 0x20111020, 0x20111121, 0x20111221, 0x20121020, + 0x20121122, 0x20121221, 0x21101121, 0x21101220, 0x21101221, 0x21111021, 0x21111022, 0x21111121, + 0x21111122, 0x21111221, 0x21121121, 0x21121220, 0x22101022, 0x22101120, 0x22101221, 0x22101222, + 0x22111022, 0x22111120, 0x22111121, 0x22121120, 0x22121122, 0x22121221, 0x20102101, 0x20112102, + 0x20112201, 0x20122101, 0x21102001, 0x21102102, 0x21112000, 0x21112002, 0x21112101, 0x21112102, + 0x21112202, 0x21122100, 0x21122101, 0x22102101, 0x22112001, 0x22112102, 0x22112201, 0x22122101, + 0x20102110, 0x20102112, 0x20102211, 0x20112010, 0x20112012, 0x20112111, 0x20112210, 0x20112212, + 0x20122010, 0x20122011, 0x20122110, 0x20122112, 0x21102010, 0x21102012, 0x21102111, 0x21102210, + 0x21102212, 0x21112011, 0x21112110, 0x21112111, 0x21112112, 0x21112211, 0x21122012, 0x21122111, + 0x21122112, 0x21122212, 0x22102011, 0x22102110, 0x22112010, 0x22112012, 0x22112111, 0x22112212, + 0x22122011, 0x22122112, 0x20102121, 0x20112121, 0x20122121, 0x21102120, 0x21102122, 0x21102221, + 0x21112020, 0x21112121, 0x21112220, 0x21122021, 0x22102121, 0x22112021, 0x22112120, 0x22112121, + 0x22112122, 0x20200000, 0x20200002, 0x20200200, 0x20200202, 0x20210101, 0x20220000, 0x20220002, + 0x20220200, 0x20220202, 0x21200101, 0x21210001, 0x21210100, 0x21210102, 0x21210201, 0x22200000, + 0x22200002, 0x22200200, 0x22200202, 0x22210101, 0x22220000, 0x22220002, 0x22220200, 0x22220202, + 0x20200111, 0x20200211, 0x20210011, 0x20210110, 0x20210112, 0x20210211, 0x20210212, 0x21200112, + 0x21200211, 0x21210011, 0x21210111, 0x21210210, 0x21210212, 0x21220011, 0x21220110, 0x22200111, + 0x22210010, 0x22210012, 0x22210112, 0x22210211, 0x20200022, 0x20200220, 0x20200222, 0x20210020, + 0x20210221, 0x20220022, 0x20220220, 0x20220222, 0x21200121, 0x21210021, 0x21210122, 0x21210221, + 0x21220121, 0x22200020, 0x22200022, 0x22200220, 0x22200222, 0x22210121, 0x22220020, 0x22220022, + 0x22220220, 0x22220222, 0x20211201, 0x20221101, 0x21201001, 0x21201100, 0x21211000, 0x21211100, + 0x21211101, 0x21211200, 0x21211202, 0x21221001, 0x21221101, 0x21221102, 0x21221200, 0x21221201, + 0x22201101, 0x20201112, 0x20201211, 0x20211010, 0x20211012, 0x20211111, 0x20211210, 0x20221112, + 0x20221211, 0x21201012, 0x21201111, 0x21211011, 0x21211110, 0x21211111, 0x21211112, 0x21211211, + 0x21221111, 0x21221212, 0x22201011, 0x22201110, 0x22201111, 0x22201112, 0x22201211, 0x22211012, + 0x22211111, 0x22211210, 0x20201121, 0x20211021, 0x20211122, 0x20211222, 0x20221021, 0x20221121, + 0x21201120, 0x21201122, 0x21201222, 0x21211022, 0x21211121, 0x21211122, 0x21211220, 0x21221020, + 0x21221022, 0x22201122, 0x22211020, 0x22211121, 0x22211122, 0x22211221, 0x22221021, 0x22221120, + 0x22221122, 0x20202000, 0x20202002, 0x20202200, 0x20202202, 0x20222000, 0x20222002, 0x20222200, + 0x20222202, 0x21212001, 0x21212100, 0x21212102, 0x21212201, 0x22202000, 0x22202002, 0x22202200, + 0x22202202, 0x22212101, 0x22222000, 0x22222002, 0x22222200, 0x22222202, 0x20202111, 0x20212110, + 0x20212211, 0x20222011, 0x20222111, 0x21202011, 0x21212010, 0x21212111, 0x21212212, 0x21222011, + 0x21222112, 0x21222211, 0x22212010, 0x22212112, 0x20202020, 0x20202022, 0x20202220, 0x20202222, + 0x20222020, 0x20222022, 0x20222220, 0x20222222, 0x21212021, 0x21212120, 0x21212122, 0x22202020, + 0x22202022, 0x22202220, 0x22202222, 0x22212121, 0x22222020, 0x22222022, 0x22222220, 0x22222222, +GGML_TABLE_END() +#endif #endif // GGML_COMMON_IMPL diff --git a/ggml-cuda.cu b/ggml-cuda.cu index c207ff87a..d2945d3c2 100644 --- a/ggml-cuda.cu +++ b/ggml-cuda.cu @@ -565,8 +565,8 @@ static_assert(sizeof(block_iq3_s) == sizeof(ggml_fp16_t) + 13*(QK_K/32) + IQ3S_N #define QI1_S (QK_K / (4*QR1_S)) typedef struct { half d; - uint8_t qs[QK_K/8]; - uint8_t scales[QK_K/16]; + uint8_t qs[QK_K/8]; + uint16_t qh[QK_K/32]; } block_iq1_s; static_assert(sizeof(block_iq1_s) == sizeof(ggml_fp16_t) + QK_K/8 + QK_K/16, "wrong iq1_s block size/padding"); @@ -1722,11 +1722,22 @@ static __global__ void dequantize_block_iq1_s(const void * __restrict__ vx, dst_ const int il = tid/8; // 0...3 const int ib = tid%8; // 0...7 dst_t * y = yy + i*QK_K + 32*ib + 8*il; - const int i8 = 4*ib+il; - uint8_t h = x[i].scales[i8/2] >> 4*(i8%2); - const int8_t * grid = (const int8_t *)(iq1s_grid + (x[i].qs[i8] | ((h & 8) << 5))); - const float d = (float)x[i].d * (2*(h & 7) + 1); - for (int j = 0; j < 8; ++j) y[j] = d * grid[j]; + const float d = (float)x[i].d * (2*((x[i].qh[ib] >> 12) & 0xf) + 1); +#if __CUDA_ARCH__ >= MIN_CC_DP4A // lowest compute capability for integer intrinsics + int grid32[2]; const int8_t * q = (const int8_t *)grid32; + grid32[0] = *((const int *)(iq1s_grid_gpu + (x[i].qs[4*ib+il] | (((x[i].qh[ib] >> 3*il) & 7) << 8)))); + grid32[1] = __vsub4((grid32[0] >> 4) & 0x0f0f0f0f, 0x01010101); + grid32[0] = __vsub4(grid32[0] & 0x0f0f0f0f, 0x01010101); + for (int j = 0; j < 8; ++j) { + y[j] = d * q[j]; + } +#else + const uint8_t * grid = (const uint8_t *)(iq1s_grid_gpu + (x[i].qs[4*ib+il] | (((x[i].qh[ib] >> 3*il) & 7) << 8))); + for (int j = 0; j < 4; ++j) { + y[j+0] = d * ((grid[j] & 0xf) - 1); + y[j+4] = d * ((grid[j] >> 4) - 1); + } +#endif #else assert(false); #endif @@ -4538,44 +4549,33 @@ static __device__ __forceinline__ float vec_dot_iq3_s_q8_1( #endif } - static __device__ __forceinline__ float vec_dot_iq1_s_q8_1( const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) { #if QK_K == 256 const block_iq1_s * bq1 = (const block_iq1_s *) vbq; const int ib32 = iqs; - int sumi1 = 0, sumi2 = 0, sumi3 = 0, sumi4 = 0; - const uint8_t h1 = bq1->scales[2*ib32+0]; - const uint8_t h2 = bq1->scales[2*ib32+1]; + int sumi = 0; #if __CUDA_ARCH__ >= MIN_CC_DP4A // lowest compute capability for integer intrinsics const int * q8 = (const int *)bq8_1[ib32].qs; - const int * grid1 = (const int *)(iq1s_grid + (bq1->qs[4*ib32+0] | ((h1 & 0x08) << 5))); - const int * grid2 = (const int *)(iq1s_grid + (bq1->qs[4*ib32+1] | ((h1 & 0x80) << 1))); - const int * grid3 = (const int *)(iq1s_grid + (bq1->qs[4*ib32+2] | ((h2 & 0x08) << 5))); - const int * grid4 = (const int *)(iq1s_grid + (bq1->qs[4*ib32+3] | ((h2 & 0x80) << 1))); - for (int j = 0; j < 2; ++j) { - sumi1 = __dp4a(q8[j+0], grid1[j], sumi1); - sumi2 = __dp4a(q8[j+2], grid2[j], sumi2); - sumi3 = __dp4a(q8[j+4], grid3[j], sumi3); - sumi4 = __dp4a(q8[j+6], grid4[j], sumi4); + for (int l = 0; l < 4; ++l) { + const int * grid = (const int *)(iq1s_grid_gpu + (bq1->qs[4*ib32+l] | (((bq1->qh[ib32] >> 3*l) & 7) << 8))); + int grid0 = __vsub4(grid[0] & 0x0f0f0f0f, 0x01010101); + int grid1 = __vsub4((grid[0] >> 4) & 0x0f0f0f0f, 0x01010101); + sumi = __dp4a(q8[2*l+1], grid1, __dp4a(q8[2*l+0], grid0, sumi)); } #else const int8_t * q8 = bq8_1[ib32].qs; - const int8_t * grid1 = (const int8_t *)(iq1s_grid + (bq1->qs[4*ib32+0] | ((h1 & 0x08) << 5))); - const int8_t * grid2 = (const int8_t *)(iq1s_grid + (bq1->qs[4*ib32+1] | ((h1 & 0x80) << 1))); - const int8_t * grid3 = (const int8_t *)(iq1s_grid + (bq1->qs[4*ib32+2] | ((h2 & 0x08) << 5))); - const int8_t * grid4 = (const int8_t *)(iq1s_grid + (bq1->qs[4*ib32+3] | ((h2 & 0x80) << 1))); - for (int j = 0; j < 8; ++j) { - sumi1 += q8[j+ 0] * grid1[j]; - sumi2 += q8[j+ 8] * grid2[j]; - sumi3 += q8[j+16] * grid3[j]; - sumi4 += q8[j+24] * grid4[j]; + for (int l = 0; l < 4; ++l) { + const uint8_t * grid = (const uint8_t *)(iq1s_grid_gpu + (bq1->qs[4*ib32+l] | (((bq1->qh[ib32] >> 3*l) & 7) << 8))); + for (int j = 0; j < 4; ++j) { + sumi += q8[j] * ((grid[j] & 0xf) - 1) + q8[j+4] * ((grid[j] >> 4) - 1); + } + q8 += 8; } #endif const float d = (float)bq1->d * __low2float(bq8_1[ib32].ds); - return d * (sumi1 * (2*(h1 & 7) + 1) + sumi2 * (2*((h1 >> 4) & 7) + 1) + - sumi3 * (2*(h2 & 7) + 1) + sumi4 * (2*((h2 >> 4) & 7) + 1)); + return d * sumi * (2*(bq1->qh[ib32] >> 12) + 1); #else assert(false); return 0.f; diff --git a/ggml-metal.metal b/ggml-metal.metal index 50185ae4d..912822ee6 100644 --- a/ggml-metal.metal +++ b/ggml-metal.metal @@ -2595,8 +2595,8 @@ typedef struct { typedef struct { half d; - uint8_t qs[QK_K/8]; - uint8_t scales[QK_K/16]; + uint8_t qs[QK_K/8]; + uint16_t qh[QK_K/32]; } block_iq1_s; // Non-linear quants @@ -4338,48 +4338,53 @@ void kernel_mul_mv_iq1_s_f32_impl( device const block_iq1_s * x = (device const block_iq1_s *) src0 + ib_row + offset0; device const float * y = (device const float *) src1 + r1*ne10 + im*ne00*ne1; - float yl[16]; + float yl[32]; float sumf[N_DST]={0.f}, all_sum; const int nb32 = nb * (QK_K / 32); - const int ix = tiisg/2; - const int il = tiisg%2; + const int ix = tiisg; - device const float * y4 = y + 32 * ix + 16 * il; + device const float * y4 = y + 32 * ix; - for (int ib32 = ix; ib32 < nb32; ib32 += 16) { + for (int ib32 = ix; ib32 < nb32; ib32 += 32) { - for (int i = 0; i < 16; ++i) { + float sumy = 0; + for (int i = 0; i < 32; ++i) { yl[i] = y4[i]; + sumy += yl[i]; } const int ibl = ib32 / (QK_K / 32); const int ib = ib32 % (QK_K / 32); device const block_iq1_s * xr = x + ibl; - device const uint8_t * qs = xr->qs + 4 * ib + 2 * il; - device const uint8_t * sc = xr->scales + 2 * ib + il; - device const half * dh = &xr->d; + device const uint8_t * qs = xr->qs + 4 * ib; + device const uint16_t * qh = xr->qh + ib; + device const half * dh = &xr->d; for (int row = 0; row < N_DST; row++) { - constant int8_t * grid1 = (constant int8_t *)(iq1s_grid + (qs[0] | ((sc[0] & 0x08) << 5))); - constant int8_t * grid2 = (constant int8_t *)(iq1s_grid + (qs[1] | ((sc[0] & 0x80) << 1))); + constant uint8_t * grid1 = (constant uint8_t *)(iq1s_grid_gpu + (qs[0] | ((qh[0] << 8) & 0x700))); + constant uint8_t * grid2 = (constant uint8_t *)(iq1s_grid_gpu + (qs[1] | ((qh[0] << 5) & 0x700))); + constant uint8_t * grid3 = (constant uint8_t *)(iq1s_grid_gpu + (qs[2] | ((qh[0] << 2) & 0x700))); + constant uint8_t * grid4 = (constant uint8_t *)(iq1s_grid_gpu + (qs[3] | ((qh[0] >> 1) & 0x700))); - float2 sum = {0}; - for (int j = 0; j < 8; ++j) { - sum[0] += yl[j+ 0] * grid1[j]; - sum[1] += yl[j+ 8] * grid2[j]; + float sum = 0; + for (int j = 0; j < 4; ++j) { + sum += yl[j+ 0] * (grid1[j] & 0xf) + yl[j+ 4] * (grid1[j] >> 4) + + yl[j+ 8] * (grid2[j] & 0xf) + yl[j+12] * (grid2[j] >> 4) + + yl[j+16] * (grid3[j] & 0xf) + yl[j+20] * (grid3[j] >> 4) + + yl[j+24] * (grid4[j] & 0xf) + yl[j+28] * (grid4[j] >> 4); } - sumf[row] += (float)dh[0] * (sum[0] * (2*(sc[0] & 7) + 1) + sum[1] * (2*((sc[0] >> 4) & 7) + 1)); + sumf[row] += (float)dh[0] * (sum - sumy) * (2*(qh[0] >> 12) + 1); dh += nb*sizeof(block_iq1_s)/2; qs += nb*sizeof(block_iq1_s); - sc += nb*sizeof(block_iq1_s); + qh += nb*sizeof(block_iq1_s)/2; } - y4 += 16 * 32; + y4 += 32 * 32; } for (int row = 0; row < N_DST; ++row) { @@ -5066,16 +5071,19 @@ void dequantize_iq2_s(device const block_iq2_s * xb, short il, thread type4x4 & template void dequantize_iq1_s(device const block_iq1_s * xb, short il, thread type4x4 & reg) { // il is 0...15 for QK_K = 256 => index of block of 32 is il/2 + const int ib32 = il/2; + il = il%2; const float d = xb->d; - device const uint8_t * qs = xb->qs + 2*il; - device const uint8_t * sc = xb->scales + il; - const float dl1 = d * (2*(sc[0] & 7) + 1); - const float dl2 = d * (2*((sc[0] >> 4) & 7) + 1); - constant int8_t * grid1 = (constant int8_t *)(iq1s_grid + (qs[0] | ((sc[0] & 0x08) << 5))); - constant int8_t * grid2 = (constant int8_t *)(iq1s_grid + (qs[1] | ((sc[0] & 0x80) << 1))); - for (int i = 0; i < 8; ++i) { - reg[i/4+0][i%4] = dl1 * grid1[i]; - reg[i/4+2][i%4] = dl2 * grid2[i]; + device const uint8_t * qs = xb->qs + 4*ib32 + 2*il; + device const uint16_t * qh = xb->qh; + const float dl = d * (2*(qh[ib32] >> 12) + 1); + constant uint8_t * grid1 = (constant uint8_t *)(iq1s_grid_gpu + (qs[0] | (((qh[ib32] >> (6*il+0)) & 7) << 8))); + constant uint8_t * grid2 = (constant uint8_t *)(iq1s_grid_gpu + (qs[1] | (((qh[ib32] >> (6*il+3)) & 7) << 8))); + for (int i = 0; i < 4; ++i) { + reg[0][i] = dl * (grid1[i] & 0xf) - dl; + reg[1][i] = dl * (grid1[i] >> 4) - dl; + reg[2][i] = dl * (grid2[i] & 0xf) - dl; + reg[3][i] = dl * (grid2[i] >> 4) - dl; } } diff --git a/ggml-quants.c b/ggml-quants.c index 42d8a5d80..f9a3d9fd2 100644 --- a/ggml-quants.c +++ b/ggml-quants.c @@ -3449,39 +3449,22 @@ void dequantize_row_iq1_s(const block_iq1_s * restrict x, float * restrict y, in assert(k % QK_K == 0); const int nb = k / QK_K; - float db[4]; - uint16_t idx[4]; - //const int8_t * grid[4]; - for (int i = 0; i < nb; i++) { const float d = GGML_FP16_TO_FP32(x[i].d); - const uint8_t * sc = x[i].scales; - const uint8_t * qs = x[i].qs; + const uint8_t * qs = x[i].qs; + const uint16_t * qh = x[i].qh; - for (int i8 = 0; i8 < QK_K/8; i8 += 4) { - idx[0] = qs[0] | ((sc[0] & 0x08) << 5); - idx[1] = qs[1] | ((sc[0] & 0x80) << 1); - idx[2] = qs[2] | ((sc[1] & 0x08) << 5); - idx[3] = qs[3] | ((sc[1] & 0x80) << 1); - //grid[0] = (const int8_t *)(iq1s_grid + (qs[0] | ((sc[0] & 0x08) << 5))); - //grid[1] = (const int8_t *)(iq1s_grid + (qs[1] | ((sc[0] & 0x80) << 1))); - //grid[2] = (const int8_t *)(iq1s_grid + (qs[2] | ((sc[1] & 0x08) << 5))); - //grid[3] = (const int8_t *)(iq1s_grid + (qs[3] | ((sc[1] & 0x80) << 1))); - db[0] = d * (2*(sc[0] & 7) + 1); - db[1] = d * (2*((sc[0] >> 4) & 7) + 1); - db[2] = d * (2*(sc[1] & 7) + 1); - db[3] = d * (2*((sc[1] >> 4) & 7) + 1); + for (int ib = 0; ib < QK_K/32; ++ib) { + const float dl = d * (2*(qh[ib] >> 12) + 1); for (int l = 0; l < 4; ++l) { - const int8_t * grid = (const int8_t *)(iq1s_grid + idx[l]); + const int8_t * grid = (const int8_t *)(iq1s_grid + (qs[l] | (((qh[ib] >> 3*l) & 7) << 8))); for (int j = 0; j < 8; ++j) { - //y[j] = db[l] * grid[l][j]; - y[j] = db[l] * grid[j]; + y[j] = dl * grid[j]; } y += 8; } qs += 4; - sc += 2; } } } @@ -9587,113 +9570,72 @@ void ggml_vec_dot_iq1_s_q8_K (int n, float * restrict s, size_t bs, const void const int nb = n / QK_K; - // TODO: implement for QK_K = 64 -#if defined __ARM_NEON && QK_K == 256 +#if defined __ARM_NEON - const uint8x16_t m8 = vdupq_n_u8(0x08); - const uint8x16_t m7 = vdupq_n_u8(0x07); - const uint8x16_t m1 = vdupq_n_u8(0x01); - const int32x4_t vzero = vdupq_n_s32(0); - - uint16_t gindex[8]; - uint16x8x2_t vindex; - int8x16x4_t q1b; + ggml_int8x16x4_t q1b; ggml_int8x16x4_t q8b; - uint16x8x4_t scales; - int32x4x2_t sumi; - int32x4x2_t dotq; float sumf = 0; for (int i = 0; i < nb; ++i) { - const int8_t * q8 = y[i].qs; - const uint8_t * qs = x[i].qs; - const uint8_t * sc = x[i].scales; + const int8_t * q8 = y[i].qs; + const uint8_t * qs = x[i].qs; + const uint16_t * qh = x[i].qh; - sumi.val[0] = sumi.val[1] = vzero; + int sumi1 = 0, sumi2 = 0; - for (int i128 = 0; i128 < QK_K/128; ++i128) { - const uint8x16_t ql = vld1q_u8(qs); qs += 16; - const uint8x8_t tm1 = vld1_u8 (sc); sc += 8; - const uint8x8_t tm2 = vshr_n_u8(tm1, 4); - const uint8x16_t qh = vcombine_u8(vzip1_u8(tm1, tm2), vzip2_u8(tm1, tm2)); - const uint8x16_t hbit = vandq_u8(qh, m8); - vindex.val[0] = vorrq_u16(vmovl_u8(vget_low_u8 (ql)), vshlq_n_u16(vmovl_u8(vget_low_u8 (hbit)), 5)); - vindex.val[1] = vorrq_u16(vmovl_u8(vget_high_u8(ql)), vshlq_n_u16(vmovl_u8(vget_high_u8(hbit)), 5)); - const uint8x16_t scales8 = vorrq_u8(vshlq_n_u8(vandq_u8(qh, m7), 1), m1); - scales.val[0] = vmovl_u8(vget_low_u8 (scales8)); - scales.val[1] = vmovl_u8(vget_high_u8 (scales8)); + for (int ib = 0; ib < QK_K/32; ib += 2) { - for (int l = 0; l < 2; ++l) { - vst1q_u16(gindex+0, vindex.val[l]); - q1b.val[0] = vcombine_s8(vld1_s8((const void *)(iq1s_grid+gindex[0])), vld1_s8((const void *)(iq1s_grid+gindex[1]))); - q1b.val[1] = vcombine_s8(vld1_s8((const void *)(iq1s_grid+gindex[2])), vld1_s8((const void *)(iq1s_grid+gindex[3]))); - q1b.val[2] = vcombine_s8(vld1_s8((const void *)(iq1s_grid+gindex[4])), vld1_s8((const void *)(iq1s_grid+gindex[5]))); - q1b.val[3] = vcombine_s8(vld1_s8((const void *)(iq1s_grid+gindex[6])), vld1_s8((const void *)(iq1s_grid+gindex[7]))); - q8b = ggml_vld1q_s8_x4(q8); q8 += 64; + q1b.val[0] = vcombine_s8(vld1_s8((const int8_t *)(iq1s_grid + (qs[0] | ((qh[ib+0] << 8) & 0x700)))), + vld1_s8((const int8_t *)(iq1s_grid + (qs[1] | ((qh[ib+0] << 5) & 0x700))))); + q1b.val[1] = vcombine_s8(vld1_s8((const int8_t *)(iq1s_grid + (qs[2] | ((qh[ib+0] << 2) & 0x700)))), + vld1_s8((const int8_t *)(iq1s_grid + (qs[3] | ((qh[ib+0] >> 1) & 0x700))))); + q1b.val[2] = vcombine_s8(vld1_s8((const int8_t *)(iq1s_grid + (qs[4] | ((qh[ib+1] << 8) & 0x700)))), + vld1_s8((const int8_t *)(iq1s_grid + (qs[5] | ((qh[ib+1] << 5) & 0x700))))); + q1b.val[3] = vcombine_s8(vld1_s8((const int8_t *)(iq1s_grid + (qs[6] | ((qh[ib+1] << 2) & 0x700)))), + vld1_s8((const int8_t *)(iq1s_grid + (qs[7] | ((qh[ib+1] >> 1) & 0x700))))); + qs += 8; - dotq.val[0] = vpaddq_s32(ggml_vdotq_s32(vzero, q1b.val[0], q8b.val[0]), ggml_vdotq_s32(vzero, q1b.val[1], q8b.val[1])); - dotq.val[1] = vpaddq_s32(ggml_vdotq_s32(vzero, q1b.val[2], q8b.val[2]), ggml_vdotq_s32(vzero, q1b.val[3], q8b.val[3])); + q8b = ggml_vld1q_s8_x4(q8); q8 += 64; + + const int32x4_t p1 = ggml_vdotq_s32(ggml_vdotq_s32(vdupq_n_s32(0), q1b.val[0], q8b.val[0]), q1b.val[1], q8b.val[1]); + const int32x4_t p2 = ggml_vdotq_s32(ggml_vdotq_s32(vdupq_n_s32(0), q1b.val[2], q8b.val[2]), q1b.val[3], q8b.val[3]); + + sumi1 += vaddvq_s32(p1) * (2*(qh[ib+0] >> 12) + 1); + sumi2 += vaddvq_s32(p2) * (2*(qh[ib+1] >> 12) + 1); - sumi.val[0] = vmlaq_s32(sumi.val[0], dotq.val[0], vreinterpretq_s32_u32(vmovl_u16(vget_low_u16 (scales.val[l])))); - sumi.val[1] = vmlaq_s32(sumi.val[1], dotq.val[1], vreinterpretq_s32_u32(vmovl_u16(vget_high_u16(scales.val[l])))); - } } - sumf += y[i].d * GGML_FP16_TO_FP32(x[i].d) * vaddvq_s32(vaddq_s32(sumi.val[0], sumi.val[1])); + sumf += y[i].d * GGML_FP16_TO_FP32(x[i].d) * (sumi1 + sumi2); } *s = sumf; - // TODO: implement for QK_K = 64 -#elif defined __AVX2__ && QK_K == 256 - - const __m128i m8 = _mm_set1_epi8(0x08); - const __m128i m7 = _mm_set1_epi8(0x07); - const __m128i m1 = _mm_set1_epi8(0x01); - const __m128i shuffle_h = _mm_set_epi8(15, 7, 14, 6, 13, 5, 12, 4, 11, 3, 10, 2, 9, 1, 8, 0); - const __m128i shuffle_s[4] = { - _mm_set_epi32(0x03030303, 0x02020202, 0x01010101, 0x00000000), - _mm_set_epi32(0x07070707, 0x06060606, 0x05050505, 0x04040404), - _mm_set_epi32(0x0b0b0b0b, 0x0a0a0a0a, 0x09090909, 0x08080808), - _mm_set_epi32(0x0f0f0f0f, 0x0e0e0e0e, 0x0d0d0d0d, 0x0c0c0c0c) - }; - - uint64_t aux64; - - typedef union m256i_uint16 { - __m256i reg; - uint16_t s[16]; - } m256i_uint16_t; - - m256i_uint16_t v_gindex; +#elif defined __AVX2__ __m256 accum = _mm256_setzero_ps(); for (int i = 0; i < nb; ++i) { - const int8_t * q8 = y[i].qs; - const uint8_t * qs = x[i].qs; - const uint8_t * sc = x[i].scales; + const int8_t * q8 = y[i].qs; + const uint8_t * qs = x[i].qs; + const uint16_t * qh = x[i].qh; __m256i sumi = _mm256_setzero_si256(); - for (int i128 = 0; i128 < QK_K/128; ++i128) { - const __m128i ql = _mm_loadu_si128((const __m128i*)qs); qs += 16; - memcpy(&aux64, sc, 8); sc += 8; - const __m128i qh = _mm_shuffle_epi8(_mm_set_epi64x(aux64 >> 4, aux64), shuffle_h); - const __m256i hbit = _mm256_cvtepu8_epi16(_mm_and_si128(qh, m8)); - v_gindex.reg = _mm256_or_si256(_mm256_cvtepu8_epi16(ql), _mm256_slli_epi16(hbit, 5)); - const __m128i scales = _mm_or_si128(_mm_slli_epi16(_mm_and_si128(qh, m7), 1), m1); + for (int ib = 0; ib < QK_K/32; ib += 2) { + const __m256i q1b_1 = _mm256_set_epi64x(iq1s_grid[qs[3] | ((qh[ib+0] >> 1) & 0x700)], iq1s_grid[qs[2] | ((qh[ib+0] << 2) & 0x700)], + iq1s_grid[qs[1] | ((qh[ib+0] << 5) & 0x700)], iq1s_grid[qs[0] | ((qh[ib+0] << 8) & 0x700)]); + const __m256i q1b_2 = _mm256_set_epi64x(iq1s_grid[qs[7] | ((qh[ib+1] >> 1) & 0x700)], iq1s_grid[qs[6] | ((qh[ib+1] << 2) & 0x700)], + iq1s_grid[qs[5] | ((qh[ib+1] << 5) & 0x700)], iq1s_grid[qs[4] | ((qh[ib+1] << 8) & 0x700)]); + qs += 8; + const __m256i q8b_1 = _mm256_loadu_si256((const __m256i*)q8); q8 += 32; + const __m256i q8b_2 = _mm256_loadu_si256((const __m256i*)q8); q8 += 32; - for (int i32 = 0; i32 < 4; ++i32) { - const __m256i q8b = _mm256_loadu_si256((const __m256i*)q8); q8 += 32; - const __m256i q1b = _mm256_set_epi64x(iq1s_grid[v_gindex.s[4*i32+3]], iq1s_grid[v_gindex.s[4*i32+2]], - iq1s_grid[v_gindex.s[4*i32+1]], iq1s_grid[v_gindex.s[4*i32+0]]); - const __m256i dot = mul_add_epi8(q1b, q8b); - const __m256i s16 = _mm256_cvtepi8_epi16(_mm_shuffle_epi8(scales, shuffle_s[i32])); - const __m256i p = _mm256_madd_epi16(s16, dot); - sumi = _mm256_add_epi32(sumi, p); - } + const __m256i dot1 = mul_add_epi8(q1b_1, q8b_1); + const __m256i dot2 = mul_add_epi8(q1b_2, q8b_2); + const __m256i p1 = _mm256_madd_epi16(dot1, _mm256_set1_epi16(2*(qh[ib+0] >> 12) + 1)); + const __m256i p2 = _mm256_madd_epi16(dot2, _mm256_set1_epi16(2*(qh[ib+1] >> 12) + 1)); + sumi = _mm256_add_epi32(sumi, _mm256_add_epi32(p1, p2)); } accum = _mm256_fmadd_ps(_mm256_set1_ps(y[i].d * GGML_FP16_TO_FP32(x[i].d)), _mm256_cvtepi32_ps(sumi), accum); @@ -9704,35 +9646,26 @@ void ggml_vec_dot_iq1_s_q8_K (int n, float * restrict s, size_t bs, const void #else - int db[4]; - uint16_t idx[4]; - float sumf = 0; - for (int i = 0; i < nb; ++i) { + for (int i = 0; i < nb; i++) { - const int8_t * q8 = y[i].qs; - const uint8_t * qs = x[i].qs; - const uint8_t * sc = x[i].scales; + const int8_t * q8 = y[i].qs; + const uint8_t * qs = x[i].qs; + const uint16_t * qh = x[i].qh; int sumi = 0; - for (int i32 = 0; i32 < QK_K/32; ++i32) { - idx[0] = qs[0] | ((sc[0] & 0x08) << 5); - idx[1] = qs[1] | ((sc[0] & 0x80) << 1); - idx[2] = qs[2] | ((sc[1] & 0x08) << 5); - idx[3] = qs[3] | ((sc[1] & 0x80) << 1); - db[0] = (2*(sc[0] & 7) + 1); - db[1] = (2*((sc[0] >> 4) & 7) + 1); - db[2] = (2*(sc[1] & 7) + 1); - db[3] = (2*((sc[1] >> 4) & 7) + 1); + for (int ib = 0; ib < QK_K/32; ++ib) { + const int ls = 2*(qh[ib] >> 12) + 1; + int lsum = 0; for (int l = 0; l < 4; ++l) { - const int8_t * grid = (const int8_t *)(iq1s_grid + idx[l]); - int suml = 0; - for (int j = 0; j < 8; ++j) suml += q8[j] * grid[j]; - sumi += db[l] * suml; + const int8_t * grid = (const int8_t *)(iq1s_grid + (qs[l] | (((qh[ib] >> 3*l) & 7) << 8))); + for (int j = 0; j < 8; ++j) { + lsum += q8[j] * grid[j]; + } q8 += 8; } + sumi += ls * lsum; qs += 4; - sc += 2; } sumf += GGML_FP16_TO_FP32(x[i].d) * y[i].d * sumi; @@ -9996,7 +9929,7 @@ static inline int iq2_grid_size(enum ggml_type type) { GGML_ASSERT(type == GGML_TYPE_IQ2_XXS || type == GGML_TYPE_IQ2_XS || type == GGML_TYPE_IQ1_S || type == GGML_TYPE_IQ2_S); return type == GGML_TYPE_IQ2_XXS ? 256 : type == GGML_TYPE_IQ2_XS ? 512 : - type == GGML_TYPE_IQ1_S ? 512 : 1024; + type == GGML_TYPE_IQ1_S ? NGRID_IQ1S : 1024; } static int iq2_compare_func(const void * left, const void * right) { @@ -10063,39 +9996,135 @@ void iq2xs_init_impl(enum ggml_type type) { 40962, 40968, 40970, 40992, 41002, 41120, 41297, 41305, 41382, 41472, 41474, 41480, 41514, 41600, 41632, 42048, 42133, 42597, 42648, 43018, 43040, 43042, 43048, 43168, 43176, 43268, 43396, 43398, 43560, 43562, 43665, 43690, }; - static const uint16_t kgrid_1bit_512[512] = { - 10, 33, 41, 85, 132, 134, 160, 162, 277, 337, 340, 345, 357, 405, 516, 545, - 553, 598, 641, 650, 681, 1042, 1044, 1097, 1169, 1176, 1320, 1345, 1365, 1378, 1434, 1444, - 1545, 1617, 1642, 1685, 2053, 2080, 2089, 2133, 2176, 2182, 2208, 2214, 2306, 2384, 2393, 2440, - 2453, 2581, 2664, 2690, 2721, 4117, 4161, 4182, 4184, 4261, 4357, 4369, 4372, 4377, 4390, 4422, - 4432, 4437, 4449, 4457, 4485, 4497, 4505, 4629, 4677, 4696, 4774, 5205, 5217, 5225, 5386, 5397, - 5409, 5445, 5457, 5460, 5461, 5462, 5465, 5472, 5477, 5525, 5545, 5650, 5668, 5717, 5729, 5769, - 5777, 6212, 6234, 6244, 6293, 6424, 6482, 6485, 6502, 6505, 6529, 6538, 6565, 6656, 6682, 6788, - 6806, 6820, 8218, 8224, 8226, 8232, 8277, 8326, 8354, 8469, 8521, 8530, 8549, 8596, 8737, 8794, - 9221, 9253, 9348, 9369, 9380, 9474, 9557, 9633, 9732, 9753, 9793, 9830, 9862, 9880, 10240, 10272, - 10282, 10321, 10406, 10517, 10530, 10566, 10585, 10645, 10896, 16466, 16468, 16473, 16485, 16646, 16660, 16665, - 16725, 16793, 16806, 16914, 16969, 16977, 16996, 17028, 17057, 17408, 17416, 17434, 17493, 17512, 17578, 17685, - 17696, 17733, 17745, 17748, 17749, 17750, 17753, 17765, 17794, 17813, 17946, 17984, 18005, 18072, 18453, 18529, - 18569, 18722, 18756, 18762, 18773, 18794, 18833, 18853, 18945, 19026, 19033, 19077, 20489, 20497, 20500, 20517, - 20565, 20586, 20610, 20633, 20757, 20769, 20776, 20805, 20817, 20820, 20821, 20822, 20825, 20837, 20864, 20872, - 20885, 20896, 21002, 21029, 21077, 21146, 21510, 21525, 21573, 21585, 21588, 21589, 21590, 21593, 21605, 21653, - 21665, 21765, 21777, 21780, 21781, 21782, 21785, 21797, 21825, 21828, 21829, 21830, 21833, 21840, 21841, 21842, - 21844, 21846, 21848, 21849, 21850, 21857, 21860, 21861, 21862, 21865, 21893, 21905, 21908, 21909, 21910, 21913, - 21925, 22024, 22037, 22085, 22097, 22100, 22101, 22102, 22105, 22117, 22165, 22545, 22566, 22568, 22594, 22608, - 22613, 22676, 22697, 22793, 22805, 22853, 22865, 22868, 22869, 22870, 22873, 22885, 22933, 22946, 23046, 23072, - 23125, 23209, 24597, 24640, 24665, 24673, 24725, 24833, 24840, 24869, 24917, 24934, 24965, 25001, 25108, 25110, - 25152, 25184, 25192, 25234, 25616, 25618, 25625, 25685, 25704, 25738, 25744, 25770, 25877, 25897, 25925, 25937, - 25940, 25941, 25942, 25945, 25957, 25986, 26005, 26186, 26197, 26276, 26632, 26634, 26725, 26757, 26770, 26885, - 26965, 26976, 26986, 27032, 27153, 27174, 27200, 27208, 27240, 27269, 27282, 27290, 32778, 32800, 32802, 32808, - 32810, 32853, 32904, 32922, 32930, 32932, 33105, 33110, 33112, 33125, 33157, 33280, 33288, 33301, 33312, 33320, - 33424, 33797, 33829, 33858, 34068, 34133, 34146, 34176, 34217, 34306, 34342, 34441, 34454, 34468, 34832, 34918, - 34965, 34984, 35094, 35137, 35161, 35208, 35232, 35332, 35338, 35368, 35429, 36932, 36934, 36953, 37009, 37125, - 37136, 37138, 37145, 37157, 37205, 37220, 37258, 37290, 37444, 37446, 37465, 37478, 37525, 37905, 37968, 37973, - 38040, 38054, 38145, 38154, 38165, 38180, 38186, 38213, 38225, 38228, 38229, 38230, 38233, 38245, 38293, 38485, - 38504, 38530, 38938, 38985, 38993, 39012, 39040, 39173, 39192, 39253, 39265, 39301, 39316, 39322, 39442, 39497, - 39504, 39590, 40970, 40984, 40992, 41002, 41045, 41120, 41128, 41237, 41289, 41297, 41317, 41364, 41366, 41514, - 41557, 41633, 41989, 42021, 42056, 42068, 42074, 42113, 42242, 42265, 42274, 42325, 42340, 42402, 42501, 42512, - 42533, 42624, 42632, 42666, 43040, 43093, 43106, 43168, 43176, 43264, 43286, 43345, 43429, 43590, 43618, 43680, + static const uint16_t kgrid_1bit_2048[NGRID_IQ1S] = { + 0, 2, 5, 8, 10, 17, 21, 32, 34, 40, 42, 69, 81, 84, 86, 101, + 128, 130, 136, 138, 149, 160, 162, 168, 170, 260, 261, 273, 276, 278, 281, 282, + 293, 321, 326, 329, 338, 341, 346, 353, 356, 358, 360, 389, 401, 404, 406, 421, + 512, 514, 520, 522, 533, 544, 546, 552, 554, 581, 593, 601, 612, 617, 640, 642, + 648, 650, 657, 661, 665, 672, 674, 680, 682, 1041, 1044, 1046, 1061, 1089, 1097, 1109, + 1114, 1124, 1125, 1169, 1177, 1189, 1281, 1284, 1285, 1286, 1301, 1304, 1306, 1321, 1344, 1349, + 1354, 1360, 1361, 1364, 1365, 1366, 1369, 1376, 1378, 1381, 1384, 1386, 1409, 1425, 1429, 1432, + 1434, 1441, 1444, 1445, 1446, 1449, 1556, 1561, 1601, 1604, 1616, 1618, 1621, 1624, 1632, 1633, + 1638, 1641, 1669, 1681, 1684, 1689, 2048, 2050, 2056, 2058, 2069, 2080, 2082, 2088, 2090, 2117, + 2129, 2134, 2149, 2176, 2178, 2184, 2186, 2197, 2208, 2210, 2216, 2218, 2309, 2321, 2324, 2329, + 2340, 2341, 2369, 2384, 2385, 2389, 2401, 2404, 2409, 2449, 2452, 2454, 2457, 2469, 2560, 2562, + 2568, 2570, 2581, 2592, 2594, 2600, 2602, 2629, 2641, 2649, 2657, 2661, 2688, 2690, 2693, 2696, + 2698, 2709, 2720, 2722, 2728, 2730, 4112, 4113, 4116, 4121, 4132, 4133, 4161, 4164, 4176, 4181, + 4184, 4193, 4196, 4197, 4201, 4241, 4244, 4246, 4257, 4261, 4353, 4356, 4358, 4361, 4368, 4370, + 4373, 4376, 4385, 4388, 4393, 4421, 4426, 4432, 4433, 4434, 4436, 4437, 4438, 4441, 4448, 4453, + 4484, 4498, 4501, 4513, 4516, 4625, 4628, 4630, 4645, 4672, 4678, 4681, 4690, 4693, 4696, 4698, + 4708, 4710, 4741, 4753, 4756, 4758, 4773, 5121, 5126, 5129, 5140, 5141, 5144, 5145, 5153, 5158, + 5185, 5189, 5190, 5192, 5194, 5201, 5204, 5205, 5206, 5209, 5218, 5221, 5224, 5252, 5257, 5264, + 5268, 5269, 5272, 5273, 5274, 5281, 5284, 5285, 5289, 5378, 5381, 5386, 5393, 5396, 5397, 5398, + 5401, 5408, 5410, 5413, 5416, 5418, 5441, 5444, 5445, 5446, 5457, 5458, 5460, 5461, 5462, 5465, + 5466, 5473, 5476, 5477, 5478, 5481, 5504, 5506, 5508, 5509, 5512, 5514, 5520, 5521, 5524, 5525, + 5526, 5529, 5530, 5536, 5538, 5541, 5633, 5636, 5637, 5638, 5653, 5654, 5656, 5658, 5665, 5670, + 5696, 5698, 5700, 5701, 5704, 5706, 5713, 5717, 5718, 5720, 5721, 5729, 5732, 5733, 5736, 5737, + 5738, 5766, 5770, 5778, 5781, 5796, 5801, 6161, 6166, 6181, 6209, 6212, 6214, 6217, 6224, 6229, + 6232, 6234, 6240, 6241, 6244, 6246, 6249, 6277, 6289, 6292, 6309, 6416, 6418, 6421, 6426, 6433, + 6437, 6466, 6468, 6469, 6472, 6481, 6484, 6485, 6486, 6489, 6490, 6496, 6501, 6506, 6537, 6545, + 6546, 6549, 6552, 6561, 6566, 6569, 6665, 6678, 6692, 6694, 6724, 6726, 6729, 6736, 6738, 6741, + 6744, 6753, 6758, 6761, 6789, 6801, 6806, 6810, 8192, 8194, 8200, 8202, 8213, 8224, 8226, 8229, + 8232, 8234, 8261, 8273, 8281, 8289, 8293, 8320, 8322, 8328, 8330, 8341, 8352, 8354, 8357, 8360, + 8362, 8453, 8465, 8468, 8473, 8485, 8514, 8516, 8521, 8533, 8536, 8538, 8545, 8548, 8549, 8550, + 8581, 8592, 8598, 8601, 8613, 8705, 8712, 8714, 8721, 8725, 8736, 8738, 8744, 8746, 8773, 8785, + 8790, 8793, 8805, 8833, 8840, 8842, 8849, 8853, 8864, 8866, 8872, 8874, 9221, 9236, 9238, 9241, + 9253, 9284, 9285, 9286, 9289, 9298, 9301, 9304, 9306, 9318, 9349, 9361, 9364, 9369, 9377, 9381, + 9481, 9493, 9505, 9513, 9536, 9541, 9544, 9553, 9556, 9557, 9561, 9570, 9573, 9576, 9609, 9616, + 9620, 9621, 9624, 9626, 9633, 9636, 9638, 9641, 9733, 9744, 9746, 9753, 9765, 9793, 9801, 9813, + 9824, 9825, 9833, 9860, 9862, 9872, 9882, 10240, 10242, 10248, 10250, 10261, 10272, 10274, 10280, 10282, + 10309, 10321, 10324, 10341, 10368, 10370, 10376, 10378, 10400, 10402, 10408, 10410, 10505, 10513, 10516, 10521, + 10533, 10566, 10569, 10578, 10581, 10593, 10596, 10598, 10601, 10629, 10640, 10646, 10649, 10660, 10661, 10752, + 10754, 10760, 10762, 10784, 10786, 10792, 10794, 10821, 10833, 10838, 10841, 10853, 10880, 10882, 10888, 10890, + 10901, 10912, 10914, 10920, 10922, 16389, 16401, 16406, 16421, 16457, 16466, 16469, 16472, 16474, 16481, 16484, + 16486, 16532, 16537, 16545, 16550, 16640, 16641, 16644, 16646, 16649, 16658, 16661, 16662, 16664, 16666, 16673, + 16678, 16681, 16709, 16712, 16714, 16721, 16724, 16725, 16726, 16729, 16730, 16741, 16744, 16746, 16769, 16772, + 16774, 16784, 16786, 16789, 16800, 16801, 16802, 16901, 16913, 16916, 16918, 16933, 16961, 16978, 16981, 16986, + 16996, 17001, 17033, 17044, 17061, 17409, 17429, 17433, 17449, 17477, 17480, 17482, 17489, 17492, 17493, 17494, + 17505, 17506, 17509, 17512, 17514, 17537, 17542, 17545, 17552, 17554, 17557, 17568, 17569, 17577, 17665, 17666, + 17669, 17674, 17681, 17684, 17685, 17686, 17689, 17696, 17701, 17706, 17729, 17732, 17733, 17734, 17737, 17744, + 17745, 17748, 17749, 17750, 17752, 17753, 17761, 17764, 17765, 17766, 17769, 17794, 17796, 17797, 17800, 17809, + 17812, 17813, 17814, 17817, 17818, 17829, 17832, 17834, 17921, 17925, 17929, 17940, 17941, 17944, 17946, 17953, + 17956, 17961, 17984, 17986, 17989, 17992, 18000, 18001, 18002, 18005, 18006, 18009, 18018, 18021, 18024, 18049, + 18053, 18058, 18068, 18069, 18081, 18084, 18086, 18437, 18449, 18453, 18458, 18469, 18498, 18505, 18512, 18517, + 18520, 18529, 18532, 18534, 18537, 18565, 18577, 18580, 18582, 18585, 18597, 18689, 18693, 18694, 18698, 18704, + 18708, 18709, 18712, 18721, 18724, 18726, 18752, 18757, 18762, 18769, 18770, 18772, 18773, 18774, 18777, 18784, + 18786, 18789, 18790, 18794, 18822, 18825, 18834, 18837, 18838, 18840, 18849, 18852, 18854, 18857, 18966, 19012, + 19014, 19017, 19029, 19032, 19034, 19044, 19049, 19092, 19109, 20481, 20484, 20485, 20486, 20489, 20498, 20501, + 20506, 20513, 20516, 20521, 20544, 20549, 20552, 20561, 20564, 20565, 20566, 20569, 20581, 20584, 20614, 20617, + 20629, 20632, 20640, 20641, 20646, 20649, 20741, 20744, 20745, 20746, 20753, 20756, 20757, 20758, 20760, 20761, + 20768, 20773, 20774, 20776, 20778, 20801, 20804, 20805, 20806, 20809, 20816, 20817, 20818, 20820, 20821, 20822, + 20824, 20825, 20826, 20833, 20836, 20837, 20838, 20841, 20866, 20869, 20881, 20884, 20885, 20886, 20889, 20896, + 20901, 20906, 20993, 20998, 21010, 21013, 21018, 21025, 21028, 21058, 21061, 21066, 21073, 21076, 21077, 21078, + 21081, 21090, 21093, 21125, 21136, 21138, 21141, 21145, 21146, 21156, 21508, 21509, 21521, 21524, 21525, 21526, + 21528, 21529, 21537, 21541, 21544, 21546, 21569, 21572, 21573, 21574, 21577, 21578, 21584, 21585, 21588, 21589, + 21590, 21592, 21593, 21594, 21601, 21602, 21604, 21605, 21606, 21609, 21632, 21640, 21642, 21649, 21652, 21653, + 21654, 21657, 21665, 21668, 21669, 21674, 21761, 21762, 21764, 21765, 21766, 21769, 21776, 21777, 21778, 21780, + 21781, 21782, 21785, 21786, 21793, 21796, 21797, 21798, 21801, 21824, 21825, 21826, 21828, 21829, 21830, 21832, + 21833, 21840, 21841, 21842, 21844, 21845, 21846, 21848, 21849, 21850, 21856, 21857, 21860, 21861, 21862, 21864, + 21865, 21866, 21889, 21892, 21893, 21897, 21898, 21904, 21905, 21908, 21909, 21910, 21912, 21913, 21921, 21924, + 21925, 21926, 21929, 22016, 22017, 22018, 22020, 22022, 22024, 22025, 22033, 22036, 22037, 22040, 22041, 22048, + 22049, 22050, 22052, 22053, 22054, 22056, 22057, 22081, 22085, 22086, 22088, 22089, 22090, 22096, 22097, 22098, + 22100, 22101, 22102, 22104, 22105, 22106, 22113, 22116, 22117, 22121, 22146, 22149, 22150, 22152, 22153, 22154, + 22161, 22165, 22170, 22178, 22181, 22182, 22184, 22185, 22532, 22533, 22534, 22537, 22544, 22549, 22552, 22561, + 22570, 22597, 22600, 22602, 22609, 22612, 22613, 22614, 22616, 22617, 22624, 22626, 22628, 22629, 22658, 22665, + 22672, 22674, 22677, 22680, 22689, 22697, 22785, 22786, 22789, 22794, 22801, 22804, 22805, 22806, 22809, 22821, + 22849, 22852, 22853, 22854, 22857, 22864, 22865, 22866, 22868, 22869, 22870, 22872, 22873, 22874, 22881, 22884, + 22885, 22886, 22889, 22913, 22917, 22921, 22929, 22932, 22933, 22934, 22936, 22937, 22949, 23044, 23048, 23061, + 23066, 23072, 23077, 23078, 23081, 23109, 23112, 23113, 23121, 23125, 23126, 23128, 23129, 23138, 23141, 23144, + 23146, 23169, 23178, 23186, 23189, 23190, 23192, 23194, 23201, 24581, 24596, 24598, 24601, 24613, 24644, 24656, + 24661, 24662, 24664, 24666, 24673, 24676, 24678, 24681, 24705, 24726, 24741, 24833, 24836, 24838, 24841, 24850, + 24853, 24865, 24866, 24870, 24873, 24901, 24905, 24913, 24917, 24918, 24921, 24933, 24934, 24938, 24964, 24970, + 24978, 24981, 24993, 24998, 25001, 25105, 25110, 25113, 25152, 25153, 25158, 25173, 25174, 25176, 25184, 25221, + 25233, 25238, 25253, 25617, 25618, 25621, 25622, 25626, 25633, 25638, 25641, 25664, 25666, 25669, 25672, 25674, + 25681, 25684, 25685, 25686, 25689, 25690, 25696, 25698, 25701, 25732, 25733, 25737, 25744, 25746, 25748, 25749, + 25750, 25752, 25754, 25761, 25764, 25769, 25861, 25864, 25866, 25873, 25877, 25878, 25881, 25924, 25925, 25926, + 25929, 25936, 25937, 25940, 25941, 25942, 25945, 25953, 25956, 25957, 25958, 25961, 25990, 25993, 25994, 26001, + 26005, 26006, 26009, 26010, 26018, 26021, 26022, 26024, 26114, 26121, 26133, 26144, 26150, 26152, 26153, 26176, + 26181, 26184, 26186, 26193, 26196, 26197, 26198, 26200, 26202, 26208, 26213, 26216, 26240, 26242, 26245, 26250, + 26260, 26262, 26264, 26265, 26272, 26276, 26278, 26282, 26646, 26649, 26661, 26689, 26706, 26709, 26714, 26721, + 26729, 26757, 26769, 26776, 26790, 26881, 26884, 26896, 26901, 26913, 26916, 26918, 26921, 26944, 26945, 26949, + 26950, 26952, 26961, 26964, 26965, 26966, 26969, 26976, 26981, 26986, 27010, 27012, 27018, 27029, 27041, 27044, + 27045, 27049, 27153, 27158, 27160, 27201, 27204, 27209, 27216, 27221, 27224, 27226, 27236, 27237, 27241, 27270, + 27284, 27288, 27290, 27302, 32768, 32770, 32776, 32778, 32800, 32802, 32808, 32810, 32837, 32848, 32849, 32852, + 32854, 32857, 32869, 32896, 32898, 32904, 32906, 32917, 32928, 32930, 32936, 32938, 33029, 33041, 33044, 33046, + 33049, 33061, 33089, 33092, 33097, 33104, 33106, 33109, 33110, 33112, 33113, 33124, 33126, 33129, 33157, 33161, + 33172, 33174, 33177, 33189, 33280, 33282, 33288, 33290, 33301, 33312, 33314, 33320, 33322, 33361, 33364, 33369, + 33381, 33408, 33410, 33416, 33418, 33429, 33440, 33442, 33448, 33450, 33812, 33817, 33857, 33860, 33873, 33877, + 33882, 33889, 33892, 33897, 33940, 33945, 34049, 34057, 34066, 34069, 34074, 34086, 34089, 34112, 34113, 34117, + 34120, 34129, 34132, 34133, 34134, 34137, 34138, 34149, 34150, 34152, 34154, 34177, 34180, 34182, 34185, 34192, + 34194, 34197, 34200, 34214, 34321, 34326, 34329, 34341, 34369, 34372, 34377, 34378, 34384, 34389, 34393, 34394, + 34401, 34406, 34410, 34437, 34449, 34458, 34468, 34816, 34818, 34824, 34826, 34837, 34848, 34850, 34856, 34858, + 34881, 34885, 34897, 34900, 34905, 34917, 34921, 34944, 34946, 34952, 34954, 34965, 34976, 34978, 34984, 34986, + 35077, 35078, 35089, 35092, 35094, 35109, 35137, 35140, 35142, 35145, 35152, 35154, 35157, 35162, 35169, 35172, + 35205, 35222, 35225, 35237, 35328, 35330, 35336, 35338, 35349, 35360, 35362, 35368, 35370, 35397, 35409, 35412, + 35414, 35456, 35458, 35464, 35466, 35477, 35488, 35490, 35496, 35498, 36869, 36881, 36886, 36888, 36889, 36901, + 36929, 36934, 36937, 36949, 36952, 36954, 36969, 36970, 36997, 37009, 37012, 37014, 37017, 37029, 37121, 37124, + 37126, 37129, 37136, 37141, 37144, 37146, 37153, 37156, 37158, 37161, 37184, 37189, 37200, 37201, 37204, 37205, + 37206, 37209, 37218, 37221, 37252, 37254, 37266, 37269, 37272, 37281, 37284, 37286, 37289, 37381, 37393, 37396, + 37401, 37413, 37444, 37446, 37449, 37456, 37458, 37461, 37464, 37478, 37481, 37509, 37524, 37526, 37545, 37889, + 37892, 37894, 37904, 37909, 37912, 37926, 37952, 37962, 37969, 37972, 37973, 37974, 37976, 37977, 37984, 37985, + 37986, 37989, 38020, 38022, 38034, 38036, 38037, 38040, 38049, 38057, 38144, 38149, 38152, 38154, 38160, 38161, + 38164, 38165, 38166, 38169, 38177, 38181, 38185, 38186, 38209, 38212, 38213, 38214, 38217, 38224, 38225, 38226, + 38228, 38229, 38230, 38232, 38233, 38234, 38241, 38244, 38245, 38246, 38249, 38273, 38277, 38280, 38289, 38290, + 38292, 38293, 38294, 38297, 38298, 38304, 38306, 38309, 38312, 38314, 38401, 38404, 38416, 38421, 38425, 38432, + 38438, 38441, 38469, 38472, 38473, 38481, 38482, 38485, 38486, 38489, 38501, 38504, 38530, 38532, 38537, 38538, + 38546, 38548, 38549, 38564, 38566, 38569, 38917, 38934, 38937, 38949, 38977, 38982, 38992, 38994, 38997, 38998, + 39002, 39012, 39013, 39045, 39057, 39062, 39065, 39077, 39172, 39174, 39177, 39184, 39186, 39189, 39192, 39194, + 39200, 39201, 39204, 39206, 39232, 39234, 39237, 39240, 39242, 39249, 39252, 39253, 39254, 39257, 39266, 39269, + 39270, 39274, 39297, 39300, 39312, 39314, 39317, 39322, 39329, 39334, 39429, 39445, 39461, 39492, 39494, 39497, + 39504, 39509, 39512, 39521, 39557, 39569, 39572, 39573, 39574, 40960, 40962, 40968, 40970, 40981, 40992, 40994, + 41000, 41002, 41029, 41041, 41044, 41046, 41049, 41088, 41090, 41096, 41098, 41109, 41120, 41122, 41128, 41130, + 41221, 41225, 41233, 41236, 41238, 41241, 41242, 41286, 41289, 41297, 41301, 41304, 41306, 41313, 41316, 41349, + 41360, 41362, 41366, 41369, 41474, 41480, 41482, 41488, 41497, 41506, 41512, 41514, 41541, 41553, 41558, 41561, + 41573, 41600, 41602, 41608, 41610, 41621, 41632, 41634, 41640, 41642, 42009, 42021, 42049, 42052, 42064, 42068, + 42069, 42072, 42074, 42081, 42085, 42086, 42088, 42089, 42117, 42246, 42249, 42256, 42258, 42261, 42264, 42278, + 42281, 42306, 42309, 42321, 42324, 42325, 42326, 42329, 42341, 42346, 42369, 42372, 42373, 42374, 42377, 42386, + 42389, 42392, 42501, 42513, 42518, 42522, 42529, 42533, 42564, 42566, 42570, 42578, 42581, 42582, 42584, 42592, + 42594, 42630, 42640, 42645, 42646, 42649, 42657, 42660, 42662, 43008, 43010, 43016, 43018, 43040, 43042, 43048, + 43050, 43089, 43092, 43094, 43097, 43136, 43138, 43144, 43146, 43157, 43168, 43170, 43176, 43178, 43269, 43284, + 43289, 43297, 43301, 43329, 43344, 43349, 43354, 43361, 43366, 43369, 43408, 43414, 43520, 43522, 43528, 43530, + 43552, 43554, 43560, 43562, 43601, 43604, 43606, 43648, 43650, 43656, 43658, 43669, 43680, 43682, 43688, 43690, }; static const uint16_t kgrid_2bit_1024[1024] = { 0, 2, 5, 8, 10, 17, 20, 22, 25, 32, 34, 37, 40, 65, 68, 70, @@ -10169,7 +10198,7 @@ void iq2xs_init_impl(enum ggml_type type) { const int nwant = type == GGML_TYPE_IQ1_S ? 3 : type == GGML_TYPE_IQ2_S ? 1 : 2; const uint16_t * kgrid = type == GGML_TYPE_IQ2_XXS ? kgrid_2bit_256 : type == GGML_TYPE_IQ2_XS ? kgrid_2bit_512 : - type == GGML_TYPE_IQ1_S ? kgrid_1bit_512 : kgrid_2bit_1024; + type == GGML_TYPE_IQ1_S ? kgrid_1bit_2048 : kgrid_2bit_1024; uint64_t * kgrid_q2xs; int * kmap_q2xs; uint16_t * kneighbors_q2xs; @@ -11408,12 +11437,70 @@ static int iq1_find_best_neighbour(const uint16_t * restrict neighbours, const u return grid_index; } +static int iq1_find_best_neighbour2(const uint16_t * restrict neighbours, const uint64_t * restrict grid, + const float * restrict xval, const float * restrict weight, float scale, int8_t * restrict L, int ngrid) { + int num_neighbors = neighbours[0]; + GGML_ASSERT(num_neighbors > 0); + float best_score = FLT_MAX; + int grid_index = -1; + for (int j = 1; j <= num_neighbors; ++j) { + const int8_t * pg = (const int8_t *)(grid + neighbours[j]); + float d2 = 0; + for (int i = 0; i < 8; ++i) { + float q = (pg[i] - 3)/2; + float w = weight[i]; + float diff = scale*q - xval[i]; + d2 += w*diff*diff; + } + if (d2 < best_score) { + best_score = d2; + grid_index = neighbours[j]; + } + } + if (grid_index < 0) { + for (int i = 0; i < ngrid; ++i) { + const int8_t * grid_i = (const int8_t *)(grid + i); + float d2 = 0; + for (int j = 0; j < 8; ++j) { + float w = weight[j]; + float q = (grid_i[j] - 3)/2; + float diff = scale*q - xval[i]; + d2 += w*diff*diff; + } + if (d2 < best_score) { + best_score = d2; + grid_index = i; + } + } + } + if (grid_index < 0) { + printf("Oops, did not find grid point\n"); + printf("Have %d neighbours\n", num_neighbors); + for (int j = 1; j <= num_neighbors; ++j) { + const int8_t * pg = (const int8_t *)(grid + neighbours[j]); + float sumqx = 0, sumq2 = 0; + for (int i = 0; i < 8; ++i) { + float q = (pg[i] - 3)/2; + float w = weight[i]; + sumqx += w*q*xval[i]; + sumq2 += w*q*q; + } + printf(" neighbour %d: sumqx = %g sumq2 = %g\n", j, (double)sumqx, (double)sumq2); + } + } + GGML_ASSERT(grid_index >= 0); + const int8_t * pg = (const int8_t *)(grid + grid_index); + for (int i = 0; i < 8; ++i) L[i] = (pg[i] - 1)/2; + return grid_index; +} + static int iq1_sort_helper(const void * left, const void * right) { const float * l = left; const float * r = right; return *l < *r ? -1 : *l > *r ? 1 : 0; } +#define IQ1S_BLOCK_SIZE 32 static void quantize_row_iq1_s_impl(const float * restrict x, void * restrict vy, int n, const float * restrict quant_weights) { const int gindex = iq2_data_index(GGML_TYPE_IQ1_S); @@ -11432,37 +11519,37 @@ static void quantize_row_iq1_s_impl(const float * restrict x, void * restrict vy block_iq1_s * y = vy; - float scales[QK_K/8]; - float weight[8]; - int8_t L[8]; - float sumx[9]; - float sumw[9]; - float pairs[16]; + float scales[QK_K/IQ1S_BLOCK_SIZE]; + float weight[IQ1S_BLOCK_SIZE]; + int8_t L[IQ1S_BLOCK_SIZE]; + float sumx[IQ1S_BLOCK_SIZE+1]; + float sumw[IQ1S_BLOCK_SIZE+1]; + float pairs[2*IQ1S_BLOCK_SIZE]; int * idx = (int *)(pairs + 1); - uint8_t hbit[QK_K/8]; + uint16_t index[IQ1S_BLOCK_SIZE/8]; for (int ibl = 0; ibl < nbl; ++ibl) { y[ibl].d = GGML_FP32_TO_FP16(0.f); memset(y[ibl].qs, 0, QK_K/8); - memset(y[ibl].scales, 0, QK_K/16); + memset(y[ibl].qh, 0, QK_K/16); float max_scale = 0; const float * xbl = x + QK_K*ibl; float sumx2 = 0; for (int i = 0; i < QK_K; ++i) sumx2 += xbl[i]*xbl[i]; - float sigma2 = sumx2/QK_K; + float sigma2 = 2*sumx2/QK_K; - for (int ib = 0; ib < QK_K/8; ++ib) { - const float * xb = xbl + 8*ib; - const float * qw = quant_weights + QK_K*ibl + 8*ib; - for (int i = 0; i < 8; ++i) weight[i] = qw[i] * sqrtf(sigma2 + xb[i]*xb[i]); + for (int ib = 0; ib < QK_K/IQ1S_BLOCK_SIZE; ++ib) { + const float * xb = xbl + IQ1S_BLOCK_SIZE*ib; + const float * qw = quant_weights + QK_K*ibl + IQ1S_BLOCK_SIZE*ib; + for (int i = 0; i < IQ1S_BLOCK_SIZE; ++i) weight[i] = qw[i] * sqrtf(sigma2 + xb[i]*xb[i]); float max = fabsf(xb[0]); - for (int i = 1; i < 8; ++i) max = MAX(max, fabsf(xb[i])); + for (int i = 1; i < IQ1S_BLOCK_SIZE; ++i) max = MAX(max, fabsf(xb[i])); if (!max) { scales[ib] = 0; - memset(L, 1, 8); + memset(L, 1, IQ1S_BLOCK_SIZE); continue; } // Here we solve exactly the sum of squared difference (SSD) weighted minimization problem. @@ -11471,14 +11558,14 @@ static void quantize_row_iq1_s_impl(const float * restrict x, void * restrict vy // in ascending order, compute Si = sum[weight[j] xb[j], j = 0...i] and // Wi = sum[weight[j], j = 0...i], and use these to quckly get get the optimum scale // for each possible and score for each split. - for (int j = 0; j < 8; ++j) { + for (int j = 0; j < IQ1S_BLOCK_SIZE; ++j) { pairs[2*j] = xb[j]; idx[2*j] = j; } - qsort(pairs, 8, 2*sizeof(float), iq1_sort_helper); + qsort(pairs, IQ1S_BLOCK_SIZE, 2*sizeof(float), iq1_sort_helper); { sumx[0] = sumw[0] = 0; - for (int j = 0; j < 8; ++j) { + for (int j = 0; j < IQ1S_BLOCK_SIZE; ++j) { int i = idx[2*j]; sumx[j+1] = sumx[j] + weight[i]*xb[i]; sumw[j+1] = sumw[j] + weight[i]; @@ -11486,10 +11573,10 @@ static void quantize_row_iq1_s_impl(const float * restrict x, void * restrict vy } float best_score = 0, scale = max; int besti1 = 0, besti2 = 0; - for (int i1 = 0; i1 <= 8; ++i1) { - for (int i2 = i1; i2 <= 8; ++i2) { - float sumqx = -(sumx[i1] - sumx[0]) + (sumx[8] - sumx[i2]); - float sumq2 = (sumw[i1] - sumw[0]) + (sumw[8] - sumw[i2]); + for (int i1 = 0; i1 <= IQ1S_BLOCK_SIZE; ++i1) { + for (int i2 = i1; i2 <= IQ1S_BLOCK_SIZE; ++i2) { + float sumqx = -(sumx[i1] - sumx[0]) + (sumx[IQ1S_BLOCK_SIZE] - sumx[i2]); + float sumq2 = (sumw[i1] - sumw[0]) + (sumw[IQ1S_BLOCK_SIZE] - sumw[i2]); if (sumq2 > 0 && sumqx*sumqx > best_score*sumq2) { scale = sumqx/sumq2; best_score = scale*sumqx; besti1 = i1; besti2 = i2; @@ -11498,23 +11585,43 @@ static void quantize_row_iq1_s_impl(const float * restrict x, void * restrict vy } for (int j = 0; j < besti1; ++j) L[idx[2*j]] = 0; for (int j = besti1; j < besti2; ++j) L[idx[2*j]] = 1; - for (int j = besti2; j < 8; ++j) L[idx[2*j]] = 2; + for (int j = besti2; j < IQ1S_BLOCK_SIZE; ++j) L[idx[2*j]] = 2; if (scale < 0) { - for (int j = 0; j < 8; ++j) L[j] = 2 - L[j]; + for (int j = 0; j < IQ1S_BLOCK_SIZE; ++j) L[j] = 2 - L[j]; scale = -scale; } - // Now we check if the solution found above corresponds to a grid point and, if not, use a neighbouring - // grid point that minimizes SSD. - uint16_t u = 0; - for (int j = 0; j < 8; ++j) u |= (L[j] << 2*j); - int grid_index = kmap_q2xs[u]; - if (grid_index < 0) { - const uint16_t * neighbours = kneighbors_q2xs - kmap_q2xs[u] - 1; - grid_index = iq1_find_best_neighbour(neighbours, kgrid_q2xs, xb, weight, &scale, L, NGRID_IQ2XXS); - GGML_ASSERT(grid_index >= 0); + bool all_on_grid = true; + for (int k = 0; k < IQ1S_BLOCK_SIZE/8; ++k) { + uint16_t u = 0; + for (int j = 0; j < 8; ++j) u |= (L[8*k+j] << 2*j); + int grid_index = kmap_q2xs[u]; + if (grid_index < 0) { + all_on_grid = false; + const uint16_t * neighbours = kneighbors_q2xs - kmap_q2xs[u] - 1; + grid_index = iq1_find_best_neighbour2(neighbours, kgrid_q2xs, xb + 8*k, weight + 8*k, scale, L + 8*k, NGRID_IQ1S); + GGML_ASSERT(grid_index >= 0); + } + index[k] = grid_index; } - y[ibl].qs[ib] = grid_index & 255; - hbit[ib] = grid_index >> 8; + if (!all_on_grid) { + float sumqx = 0, sumq2 = 0; + for (int k = 0; k < IQ1S_BLOCK_SIZE/8; ++k) { + const int8_t * pg = (const int8_t *)(kgrid_q2xs + index[k]); + for (int j = 0; j < 8; ++j) { + float w = weight[8*k + j]; + float q = (pg[j] - 3)/2; + sumqx += w*q*xb[8*k+j]; + sumq2 += w*q*q; + } + } + if (sumqx > 0 && sumq2 > 0) scale = sumqx/sumq2; + } + uint16_t h = 0; + for (int k = 0; k < IQ1S_BLOCK_SIZE/8; ++k) { + y[ibl].qs[(IQ1S_BLOCK_SIZE/8)*ib + k] = index[k] & 255; + h |= (index[k] >> 8) << 3*k; + } + y[ibl].qh[ib] = h; GGML_ASSERT(scale >= 0); scales[ib] = scale; max_scale = MAX(max_scale, scale); @@ -11525,14 +11632,13 @@ static void quantize_row_iq1_s_impl(const float * restrict x, void * restrict vy continue; } - float d = max_scale/15; - y[ibl].d = GGML_FP32_TO_FP16(d*1.085f); // 1.085f is another fudge factor. Don't ask me why it is needed. + float d = max_scale/31; + y[ibl].d = GGML_FP32_TO_FP16(d*1.125f); // 1.085f is another fudge factor. Don't ask me why it is needed. float id = 1/d; - for (int ib = 0; ib < QK_K/8; ++ib) { + for (int ib = 0; ib < QK_K/IQ1S_BLOCK_SIZE; ++ib) { int l = nearest_int(0.5f*(id*scales[ib]-1)); - l = MAX(0, MIN(7, l)); - if (hbit[ib]) l |= 8; - y[ibl].scales[ib/2] |= (l << 4*(ib%2)); + l = MAX(0, MIN(15, l)); + y[ibl].qh[ib] |= (l << 12); } } } diff --git a/ggml-quants.h b/ggml-quants.h index 47dd52856..74aabf415 100644 --- a/ggml-quants.h +++ b/ggml-quants.h @@ -217,8 +217,8 @@ static_assert(sizeof(block_iq3_s) == sizeof(ggml_fp16_t) + 13*(QK_K/32) + IQ3S_N typedef struct { ggml_fp16_t d; - uint8_t qs[QK_K/8]; - uint8_t scales[QK_K/16]; + uint8_t qs[QK_K/8]; + uint16_t qh[QK_K/32]; } block_iq1_s; static_assert(sizeof(block_iq1_s) == sizeof(ggml_fp16_t) + QK_K/8 + QK_K/16, "wrong iq1_s block size/padding"); From ee35600b9061b1ea0c4ea87fce6844297632b2a8 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Mon, 11 Mar 2024 09:56:47 +0200 Subject: [PATCH 592/812] llama : fix F16/F32 downcast + improve names (#5980) --- llama.cpp | 67 +++++++++++++++++++++++++++++-------------------------- llama.h | 2 +- 2 files changed, 36 insertions(+), 33 deletions(-) diff --git a/llama.cpp b/llama.cpp index 249442166..110e509cc 100644 --- a/llama.cpp +++ b/llama.cpp @@ -11636,7 +11636,7 @@ static void llama_tensor_dequantize_internal( workers.clear(); } -static ggml_type get_k_quant_type(quantize_state_internal & qs, ggml_type new_type, const ggml_tensor * tensor, llama_ftype ftype) { +static ggml_type llama_tensor_get_type(quantize_state_internal & qs, ggml_type new_type, const ggml_tensor * tensor, llama_ftype ftype) { const std::string name = ggml_get_name(tensor); // TODO: avoid hardcoded tensor names - use the TN_* constants @@ -11951,40 +11951,40 @@ static int32_t llama_tensor_quantize_internal(enum ggml_type new_type, const flo } static void llama_model_quantize_internal(const std::string & fname_inp, const std::string & fname_out, const llama_model_quantize_params * params) { - ggml_type quantized_type; + ggml_type default_type; llama_ftype ftype = params->ftype; switch (params->ftype) { - case LLAMA_FTYPE_MOSTLY_Q4_0: quantized_type = GGML_TYPE_Q4_0; break; - case LLAMA_FTYPE_MOSTLY_Q4_1: quantized_type = GGML_TYPE_Q4_1; break; - case LLAMA_FTYPE_MOSTLY_Q5_0: quantized_type = GGML_TYPE_Q5_0; break; - case LLAMA_FTYPE_MOSTLY_Q5_1: quantized_type = GGML_TYPE_Q5_1; break; - case LLAMA_FTYPE_MOSTLY_Q8_0: quantized_type = GGML_TYPE_Q8_0; break; - case LLAMA_FTYPE_MOSTLY_F16: quantized_type = GGML_TYPE_F16; break; - case LLAMA_FTYPE_ALL_F32: quantized_type = GGML_TYPE_F32; break; + case LLAMA_FTYPE_MOSTLY_Q4_0: default_type = GGML_TYPE_Q4_0; break; + case LLAMA_FTYPE_MOSTLY_Q4_1: default_type = GGML_TYPE_Q4_1; break; + case LLAMA_FTYPE_MOSTLY_Q5_0: default_type = GGML_TYPE_Q5_0; break; + case LLAMA_FTYPE_MOSTLY_Q5_1: default_type = GGML_TYPE_Q5_1; break; + case LLAMA_FTYPE_MOSTLY_Q8_0: default_type = GGML_TYPE_Q8_0; break; + case LLAMA_FTYPE_MOSTLY_F16: default_type = GGML_TYPE_F16; break; + case LLAMA_FTYPE_ALL_F32: default_type = GGML_TYPE_F32; break; // K-quants case LLAMA_FTYPE_MOSTLY_Q2_K_S: - case LLAMA_FTYPE_MOSTLY_Q2_K: quantized_type = GGML_TYPE_Q2_K; break; - case LLAMA_FTYPE_MOSTLY_IQ3_XS: quantized_type = GGML_TYPE_IQ3_S; break; + case LLAMA_FTYPE_MOSTLY_Q2_K: default_type = GGML_TYPE_Q2_K; break; + case LLAMA_FTYPE_MOSTLY_IQ3_XS: default_type = GGML_TYPE_IQ3_S; break; case LLAMA_FTYPE_MOSTLY_Q3_K_S: case LLAMA_FTYPE_MOSTLY_Q3_K_M: - case LLAMA_FTYPE_MOSTLY_Q3_K_L: quantized_type = GGML_TYPE_Q3_K; break; + case LLAMA_FTYPE_MOSTLY_Q3_K_L: default_type = GGML_TYPE_Q3_K; break; case LLAMA_FTYPE_MOSTLY_Q4_K_S: - case LLAMA_FTYPE_MOSTLY_Q4_K_M: quantized_type = GGML_TYPE_Q4_K; break; + case LLAMA_FTYPE_MOSTLY_Q4_K_M: default_type = GGML_TYPE_Q4_K; break; case LLAMA_FTYPE_MOSTLY_Q5_K_S: - case LLAMA_FTYPE_MOSTLY_Q5_K_M: quantized_type = GGML_TYPE_Q5_K; break; - case LLAMA_FTYPE_MOSTLY_Q6_K: quantized_type = GGML_TYPE_Q6_K; break; - case LLAMA_FTYPE_MOSTLY_IQ2_XXS: quantized_type = GGML_TYPE_IQ2_XXS; break; - case LLAMA_FTYPE_MOSTLY_IQ2_XS: quantized_type = GGML_TYPE_IQ2_XS; break; - case LLAMA_FTYPE_MOSTLY_IQ2_S: quantized_type = GGML_TYPE_IQ2_XS; break; - case LLAMA_FTYPE_MOSTLY_IQ2_M: quantized_type = GGML_TYPE_IQ2_S; break; - case LLAMA_FTYPE_MOSTLY_IQ3_XXS: quantized_type = GGML_TYPE_IQ3_XXS; break; - case LLAMA_FTYPE_MOSTLY_IQ1_S: quantized_type = GGML_TYPE_IQ1_S; break; - case LLAMA_FTYPE_MOSTLY_IQ4_NL: quantized_type = GGML_TYPE_IQ4_NL; break; - case LLAMA_FTYPE_MOSTLY_IQ4_XS: quantized_type = GGML_TYPE_IQ4_XS; break; - case LLAMA_FTYPE_MOSTLY_IQ3_S: quantized_type = GGML_TYPE_IQ3_S; break; - case LLAMA_FTYPE_MOSTLY_IQ3_M: quantized_type = GGML_TYPE_IQ3_S; break; + case LLAMA_FTYPE_MOSTLY_Q5_K_M: default_type = GGML_TYPE_Q5_K; break; + case LLAMA_FTYPE_MOSTLY_Q6_K: default_type = GGML_TYPE_Q6_K; break; + case LLAMA_FTYPE_MOSTLY_IQ2_XXS: default_type = GGML_TYPE_IQ2_XXS; break; + case LLAMA_FTYPE_MOSTLY_IQ2_XS: default_type = GGML_TYPE_IQ2_XS; break; + case LLAMA_FTYPE_MOSTLY_IQ2_S: default_type = GGML_TYPE_IQ2_XS; break; + case LLAMA_FTYPE_MOSTLY_IQ2_M: default_type = GGML_TYPE_IQ2_S; break; + case LLAMA_FTYPE_MOSTLY_IQ3_XXS: default_type = GGML_TYPE_IQ3_XXS; break; + case LLAMA_FTYPE_MOSTLY_IQ1_S: default_type = GGML_TYPE_IQ1_S; break; + case LLAMA_FTYPE_MOSTLY_IQ4_NL: default_type = GGML_TYPE_IQ4_NL; break; + case LLAMA_FTYPE_MOSTLY_IQ4_XS: default_type = GGML_TYPE_IQ4_XS; break; + case LLAMA_FTYPE_MOSTLY_IQ3_S: default_type = GGML_TYPE_IQ3_S; break; + case LLAMA_FTYPE_MOSTLY_IQ3_M: default_type = GGML_TYPE_IQ3_S; break; default: throw std::runtime_error(format("invalid output file type %d\n", ftype)); } @@ -12125,23 +12125,26 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s // do not quantize Mamba's small yet 2D weights // NOTE: can't use LLM_TN here because the layer number is not known quantize &= name.find("ssm_conv1d.weight") == std::string::npos; - quantize &= name.find("ssm_x.weight") == std::string::npos; - quantize &= name.find("ssm_dt.weight") == std::string::npos; + quantize &= name.find("ssm_x.weight") == std::string::npos; + quantize &= name.find("ssm_dt.weight") == std::string::npos; enum ggml_type new_type; void * new_data; size_t new_size; if (quantize) { - new_type = quantized_type; - if (!params->pure) { - new_type = get_k_quant_type(qs, new_type, tensor, ftype); + new_type = default_type; + + // get more optimal quantization type based on the tensor shape, layer, etc. + if (!params->pure && ggml_is_quantized(default_type)) { + new_type = llama_tensor_get_type(qs, new_type, tensor, ftype); } // If we've decided to quantize to the same type the tensor is already // in then there's nothing to do. quantize = tensor->type != new_type; } + if (!quantize) { new_type = tensor->type; new_data = tensor->data; @@ -12187,7 +12190,7 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s f32_data = (float *) f32_conv_buf.data(); } - LLAMA_LOG_INFO("quantizing to %s .. ", ggml_type_name(new_type)); + LLAMA_LOG_INFO("converting to %s .. ", ggml_type_name(new_type)); fflush(stdout); if (work.size() < nelements * 4) { @@ -12235,7 +12238,7 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s LLAMA_LOG_INFO("%s: quant size = %8.2f MB\n", __func__, total_size_new/1024.0/1024.0); if (qs.n_fallback > 0) { - LLAMA_LOG_WARN("%s: WARNING: %d of %d tensor(s) incompatible with k-quants and required fallback quantization\n", + LLAMA_LOG_WARN("%s: WARNING: %d of %d tensor(s) required fallback quantization\n", __func__, qs.n_fallback, qs.n_k_quantized + qs.n_fallback); } } diff --git a/llama.h b/llama.h index c8e05aadd..ccf65ca4e 100644 --- a/llama.h +++ b/llama.h @@ -278,7 +278,7 @@ extern "C" { bool allow_requantize; // allow quantizing non-f32/f16 tensors bool quantize_output_tensor; // quantize output.weight bool only_copy; // only copy tensors - ftype, allow_requantize and quantize_output_tensor are ignored - bool pure; // disable k-quant mixtures and quantize all tensors to the same type + bool pure; // quantize all tensors to the default type void * imatrix; // pointer to importance matrix data } llama_model_quantize_params; From ecab1c75de68de7c41c254e2ae170d3b07bee6d4 Mon Sep 17 00:00:00 2001 From: Gilad S Date: Mon, 11 Mar 2024 10:00:08 +0200 Subject: [PATCH 593/812] cmake : fix subdir for `LLAMA_METAL_EMBED_LIBRARY` (#5985) --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 00a26391a..60b592bf5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -208,7 +208,7 @@ if (LLAMA_METAL) enable_language(ASM) add_compile_definitions(GGML_METAL_EMBED_LIBRARY) - set(METALLIB_SOURCE "${CMAKE_SOURCE_DIR}/ggml-metal.metal") + set(METALLIB_SOURCE "${CMAKE_CURRENT_SOURCE_DIR}/ggml-metal.metal") file(MAKE_DIRECTORY "${CMAKE_BINARY_DIR}/autogenerated") set(EMBED_METALLIB_ASSEMBLY "${CMAKE_BINARY_DIR}/autogenerated/ggml-embed-metallib.s") From 332bdfd7980718abf664bfa5460f2288a3314984 Mon Sep 17 00:00:00 2001 From: Minsoo Cheong <54794500+mscheong01@users.noreply.github.com> Date: Mon, 11 Mar 2024 17:09:32 +0900 Subject: [PATCH 594/812] server : maintain chat completion id for streaming responses (#5988) * server: maintain chat completion id for streaming responses * Update examples/server/utils.hpp * Update examples/server/utils.hpp --------- Co-authored-by: Georgi Gerganov --- examples/server/server.cpp | 7 ++++--- examples/server/utils.hpp | 12 ++++++------ 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/examples/server/server.cpp b/examples/server/server.cpp index c7d3ed01b..3951507aa 100644 --- a/examples/server/server.cpp +++ b/examples/server/server.cpp @@ -3195,11 +3195,12 @@ int main(int argc, char ** argv) { ctx_server.queue_results.add_waiting_task_id(id_task); ctx_server.request_completion(id_task, -1, data, false, false); + const auto completion_id = gen_chatcmplid(); if (!json_value(data, "stream", false)) { server_task_result result = ctx_server.queue_results.recv(id_task); if (!result.error && result.stop) { - json result_oai = format_final_response_oaicompat(data, result.data); + json result_oai = format_final_response_oaicompat(data, result.data, completion_id); res.set_content(result_oai.dump(-1, ' ', false, json::error_handler_t::replace), "application/json; charset=utf-8"); } else { @@ -3208,11 +3209,11 @@ int main(int argc, char ** argv) { } ctx_server.queue_results.remove_waiting_task_id(id_task); } else { - const auto chunked_content_provider = [id_task, &ctx_server](size_t, httplib::DataSink & sink) { + const auto chunked_content_provider = [id_task, &ctx_server, completion_id](size_t, httplib::DataSink & sink) { while (true) { server_task_result result = ctx_server.queue_results.recv(id_task); if (!result.error) { - std::vector result_array = format_partial_response_oaicompat(result.data); + std::vector result_array = format_partial_response_oaicompat(result.data, completion_id); for (auto it = result_array.begin(); it != result_array.end(); ++it) { if (!it->empty()) { diff --git a/examples/server/utils.hpp b/examples/server/utils.hpp index df0a27782..f27af81e9 100644 --- a/examples/server/utils.hpp +++ b/examples/server/utils.hpp @@ -378,7 +378,7 @@ static json oaicompat_completion_params_parse( return llama_params; } -static json format_final_response_oaicompat(const json & request, json result, bool streaming = false) { +static json format_final_response_oaicompat(const json & request, json result, const std::string & completion_id, bool streaming = false) { bool stopped_word = result.count("stopped_word") != 0; bool stopped_eos = json_value(result, "stopped_eos", false); int num_tokens_predicted = json_value(result, "tokens_predicted", 0); @@ -412,7 +412,7 @@ static json format_final_response_oaicompat(const json & request, json result, b {"prompt_tokens", num_prompt_tokens}, {"total_tokens", num_tokens_predicted + num_prompt_tokens} }}, - {"id", gen_chatcmplid()} + {"id", completion_id} }; if (server_verbose) { @@ -427,7 +427,7 @@ static json format_final_response_oaicompat(const json & request, json result, b } // return value is vector as there is one case where we might need to generate two responses -static std::vector format_partial_response_oaicompat(json result) { +static std::vector format_partial_response_oaicompat(json result, const std::string & completion_id) { if (!result.contains("model") || !result.contains("oaicompat_token_ctr")) { return std::vector({result}); } @@ -471,7 +471,7 @@ static std::vector format_partial_response_oaicompat(json result) { {"role", "assistant"} }}}})}, {"created", t}, - {"id", gen_chatcmplid()}, + {"id", completion_id}, {"model", modelname}, {"object", "chat.completion.chunk"}}; @@ -482,7 +482,7 @@ static std::vector format_partial_response_oaicompat(json result) { {"content", content}}} }})}, {"created", t}, - {"id", gen_chatcmplid()}, + {"id", completion_id}, {"model", modelname}, {"object", "chat.completion.chunk"}}; @@ -509,7 +509,7 @@ static std::vector format_partial_response_oaicompat(json result) { json ret = json { {"choices", choices}, {"created", t}, - {"id", gen_chatcmplid()}, + {"id", completion_id}, {"model", modelname}, {"object", "chat.completion.chunk"} }; From 3202361c5b1ba15e695b31209567ef42c22c5c32 Mon Sep 17 00:00:00 2001 From: Michael Podvitskiy Date: Mon, 11 Mar 2024 10:28:51 +0100 Subject: [PATCH 595/812] ggml, ci : Windows ARM runner and build fixes (#5979) * windows arm ci * fix `error C2078: too many initializers` with ggml_vld1q_u32 macro for MSVC ARM64 * fix `warning C4146: unary minus operator applied to unsigned type, result still unsigned` * fix `error C2065: '__fp16': undeclared identifier` --- .github/workflows/build.yml | 4 +++- ggml-impl.h | 8 ++++++-- ggml-quants.c | 16 ++++++++-------- ggml.c | 4 ++-- llama.cpp | 4 ++-- 5 files changed, 21 insertions(+), 15 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 9144f9266..d39cd6bc3 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -425,6 +425,8 @@ jobs: defines: '-DLLAMA_NATIVE=OFF -DLLAMA_BUILD_SERVER=ON -DLLAMA_KOMPUTE=ON -DKOMPUTE_OPT_DISABLE_VULKAN_VERSION_CHECK=ON -DBUILD_SHARED_LIBS=ON' - build: 'vulkan' defines: '-DLLAMA_NATIVE=OFF -DLLAMA_BUILD_SERVER=ON -DLLAMA_VULKAN=ON -DBUILD_SHARED_LIBS=ON' + - build: 'arm64' + defines: '-A ARM64 -DLLAMA_NATIVE=OFF -DLLAMA_BUILD_SERVER=ON -DBUILD_SHARED_LIBS=ON' steps: - name: Clone @@ -520,7 +522,7 @@ jobs: - name: Test id: cmake_test # not all machines have native AVX-512 - if: ${{ matrix.build != 'clblast' && matrix.build != 'kompute' && matrix.build != 'vulkan' && (matrix.build != 'avx512' || env.HAS_AVX512F == '1') }} + if: ${{ matrix.build != 'arm64' && matrix.build != 'clblast' && matrix.build != 'kompute' && matrix.build != 'vulkan' && (matrix.build != 'avx512' || env.HAS_AVX512F == '1') }} run: | cd build ctest -L main -C Release --verbose --timeout 900 diff --git a/ggml-impl.h b/ggml-impl.h index c5637e4d4..e68b72877 100644 --- a/ggml-impl.h +++ b/ggml-impl.h @@ -53,26 +53,30 @@ extern "C" { // #include +typedef __fp16 ggml_fp16_internal_t; + #define GGML_COMPUTE_FP16_TO_FP32(x) ggml_compute_fp16_to_fp32(x) #define GGML_COMPUTE_FP32_TO_FP16(x) ggml_compute_fp32_to_fp16(x) #define GGML_FP16_TO_FP32(x) ggml_compute_fp16_to_fp32(x) static inline float ggml_compute_fp16_to_fp32(ggml_fp16_t h) { - __fp16 tmp; + ggml_fp16_internal_t tmp; memcpy(&tmp, &h, sizeof(ggml_fp16_t)); return (float)tmp; } static inline ggml_fp16_t ggml_compute_fp32_to_fp16(float f) { ggml_fp16_t res; - __fp16 tmp = f; + ggml_fp16_internal_t tmp = f; memcpy(&res, &tmp, sizeof(ggml_fp16_t)); return res; } #else +typedef uint16_t ggml_fp16_internal_t; + #ifdef __wasm_simd128__ #include #else diff --git a/ggml-quants.c b/ggml-quants.c index f9a3d9fd2..86b0764cb 100644 --- a/ggml-quants.c +++ b/ggml-quants.c @@ -9374,15 +9374,15 @@ void ggml_vec_dot_iq3_s_q8_K (int n, float * restrict s, size_t bs, const void * const uint8x16_t idx_l = vld1q_u8(qs); qs += 16; idx.vec_index = vorrq_u16(vmovl_u8(vget_low_u8 (idx_l)), vandq_u16(vshlq_u16(vdupq_n_u16(qh[ib32+0]), hshift), m256)); - const uint32x4_t aux32x4_0 = {iq3s_grid[idx.index[0]], iq3s_grid[idx.index[1]], - iq3s_grid[idx.index[2]], iq3s_grid[idx.index[3]]}; - const uint32x4_t aux32x4_1 = {iq3s_grid[idx.index[4]], iq3s_grid[idx.index[5]], - iq3s_grid[idx.index[6]], iq3s_grid[idx.index[7]]}; + const uint32x4_t aux32x4_0 = ggml_vld1q_u32(iq3s_grid[idx.index[0]], iq3s_grid[idx.index[1]], + iq3s_grid[idx.index[2]], iq3s_grid[idx.index[3]]); + const uint32x4_t aux32x4_1 = ggml_vld1q_u32(iq3s_grid[idx.index[4]], iq3s_grid[idx.index[5]], + iq3s_grid[idx.index[6]], iq3s_grid[idx.index[7]]); idx.vec_index = vorrq_u16(vmovl_u8(vget_high_u8(idx_l)), vandq_u16(vshlq_u16(vdupq_n_u16(qh[ib32+1]), hshift), m256)); - const uint32x4_t aux32x4_2 = {iq3s_grid[idx.index[0]], iq3s_grid[idx.index[1]], - iq3s_grid[idx.index[2]], iq3s_grid[idx.index[3]]}; - const uint32x4_t aux32x4_3 = {iq3s_grid[idx.index[4]], iq3s_grid[idx.index[5]], - iq3s_grid[idx.index[6]], iq3s_grid[idx.index[7]]}; + const uint32x4_t aux32x4_2 = ggml_vld1q_u32(iq3s_grid[idx.index[0]], iq3s_grid[idx.index[1]], + iq3s_grid[idx.index[2]], iq3s_grid[idx.index[3]]); + const uint32x4_t aux32x4_3 = ggml_vld1q_u32(iq3s_grid[idx.index[4]], iq3s_grid[idx.index[5]], + iq3s_grid[idx.index[6]], iq3s_grid[idx.index[7]]); vs.val[0] = vreinterpretq_u8_u32(vdupq_n_u32(signs[0] | (signs[1] << 16))); diff --git a/ggml.c b/ggml.c index 80efa6f2a..9a7bd1d8c 100644 --- a/ggml.c +++ b/ggml.c @@ -857,7 +857,7 @@ inline static float vaddvq_f32(float32x4_t v) { #define GGML_F16x8 float16x8_t #define GGML_F16x8_ZERO vdupq_n_f16(0.0f) #define GGML_F16x8_SET1(x) vdupq_n_f16(x) - #define GGML_F16x8_LOAD(x) vld1q_f16((const __fp16 *)(x)) + #define GGML_F16x8_LOAD(x) vld1q_f16((const ggml_fp16_internal_t *)(x)) #define GGML_F16x8_STORE vst1q_f16 #define GGML_F16x8_FMA(a, b, c) vfmaq_f16(a, b, c) #define GGML_F16x8_ADD vaddq_f16 @@ -900,7 +900,7 @@ inline static float vaddvq_f32(float32x4_t v) { #define GGML_F32Cx4 float32x4_t #define GGML_F32Cx4_ZERO vdupq_n_f32(0.0f) #define GGML_F32Cx4_SET1(x) vdupq_n_f32(x) - #define GGML_F32Cx4_LOAD(x) vcvt_f32_f16(vld1_f16((const __fp16 *)(x))) + #define GGML_F32Cx4_LOAD(x) vcvt_f32_f16(vld1_f16((const ggml_fp16_internal_t *)(x))) #define GGML_F32Cx4_STORE(x, y) vst1_f16(x, vcvt_f16_f32(y)) #define GGML_F32Cx4_FMA(a, b, c) vfmaq_f32(a, b, c) #define GGML_F32Cx4_ADD vaddq_f32 diff --git a/llama.cpp b/llama.cpp index 110e509cc..f61ea791b 100644 --- a/llama.cpp +++ b/llama.cpp @@ -13980,7 +13980,7 @@ int32_t llama_token_to_piece(const struct llama_model * model, llama_token token } else if (llama_is_user_defined_token(model->vocab, token)) { std::string result = model->vocab.id_to_token[token].text; if (length < (int) result.length()) { - return -result.length(); + return -(int) result.length(); } memcpy(buf, result.c_str(), result.length()); return result.length(); @@ -14015,7 +14015,7 @@ int32_t llama_token_to_piece(const struct llama_model * model, llama_token token } else if (llama_is_user_defined_token(model->vocab, token)) { std::string result = model->vocab.id_to_token[token].text; if (length < (int) result.length()) { - return -result.length(); + return -(int) result.length(); } memcpy(buf, result.c_str(), result.length()); return result.length(); From caa106d4e05a0ab94225c220b81f9e2cd522339b Mon Sep 17 00:00:00 2001 From: Xuan Son Nguyen Date: Mon, 11 Mar 2024 10:56:41 +0100 Subject: [PATCH 596/812] Server: format error to json (#5961) * server: format error to json * server: do not crash on grammar error * fix api key test case * revert limit max n_predict * small fix * correct coding style * update completion.js * launch_slot_with_task * update docs * update_slots * update webui * update readme --- examples/server/README.md | 46 +- examples/server/completion.js.hpp | 469 +++++++++--------- examples/server/public/completion.js | 20 +- examples/server/server.cpp | 159 +++--- examples/server/tests/features/steps/steps.py | 2 +- examples/server/utils.hpp | 51 ++ 6 files changed, 430 insertions(+), 317 deletions(-) diff --git a/examples/server/README.md b/examples/server/README.md index 23606b32a..376739051 100644 --- a/examples/server/README.md +++ b/examples/server/README.md @@ -556,9 +556,51 @@ Run with bash: bash chat.sh ``` -### API like OAI +### OAI-like API -The HTTP server supports OAI-like API +The HTTP server supports OAI-like API: https://github.com/openai/openai-openapi + +### API errors + +Server returns error in the same format as OAI: https://github.com/openai/openai-openapi + +Example of an error: + +```json +{ + "error": { + "code": 401, + "message": "Invalid API Key", + "type": "authentication_error" + } +} +``` + +Apart from error types supported by OAI, we also have custom types that are specific to functionalities of llama.cpp: + +**When /metrics or /slots endpoint is disabled** + +```json +{ + "error": { + "code": 501, + "message": "This server does not support metrics endpoint.", + "type": "not_supported_error" + } +} +``` + +**When the server receives invalid grammar via */completions endpoint** + +```json +{ + "error": { + "code": 400, + "message": "Failed to parse grammar", + "type": "invalid_request_error" + } +} +``` ### Extending or building alternative Web Front End diff --git a/examples/server/completion.js.hpp b/examples/server/completion.js.hpp index f5e696e17..10734091e 100644 --- a/examples/server/completion.js.hpp +++ b/examples/server/completion.js.hpp @@ -231,255 +231,256 @@ unsigned char completion_js[] = { 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x2e, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, - 0x73, 0x75, 0x6c, 0x74, 0x2e, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x20, 0x3d, - 0x20, 0x4a, 0x53, 0x4f, 0x4e, 0x2e, 0x70, 0x61, 0x72, 0x73, 0x65, 0x28, - 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x2e, 0x65, 0x72, 0x72, 0x6f, 0x72, - 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, 0x72, 0x65, 0x73, 0x75, 0x6c, - 0x74, 0x2e, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x2e, 0x63, 0x6f, 0x6e, 0x74, - 0x65, 0x6e, 0x74, 0x2e, 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x73, - 0x28, 0x27, 0x73, 0x6c, 0x6f, 0x74, 0x20, 0x75, 0x6e, 0x61, 0x76, 0x61, - 0x69, 0x6c, 0x61, 0x62, 0x6c, 0x65, 0x27, 0x29, 0x29, 0x20, 0x7b, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x74, 0x72, + 0x79, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, + 0x2e, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x20, 0x3d, 0x20, 0x4a, 0x53, 0x4f, + 0x4e, 0x2e, 0x70, 0x61, 0x72, 0x73, 0x65, 0x28, 0x72, 0x65, 0x73, 0x75, + 0x6c, 0x74, 0x2e, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x29, 0x3b, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x69, 0x66, 0x20, 0x28, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x2e, + 0x65, 0x72, 0x72, 0x6f, 0x72, 0x2e, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, + 0x65, 0x2e, 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x73, 0x28, 0x27, + 0x73, 0x6c, 0x6f, 0x74, 0x20, 0x75, 0x6e, 0x61, 0x76, 0x61, 0x69, 0x6c, + 0x61, 0x62, 0x6c, 0x65, 0x27, 0x29, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x2f, 0x2f, 0x20, 0x54, 0x68, 0x72, 0x6f, 0x77, 0x20, 0x61, 0x6e, 0x20, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x20, 0x74, 0x6f, 0x20, 0x62, 0x65, 0x20, 0x63, 0x61, 0x75, 0x67, 0x68, 0x74, 0x20, 0x62, 0x79, 0x20, 0x75, 0x70, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x20, 0x63, 0x61, 0x6c, 0x6c, 0x65, 0x72, 0x73, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x74, 0x68, 0x72, 0x6f, 0x77, - 0x20, 0x6e, 0x65, 0x77, 0x20, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x28, 0x27, - 0x73, 0x6c, 0x6f, 0x74, 0x20, 0x75, 0x6e, 0x61, 0x76, 0x61, 0x69, 0x6c, - 0x61, 0x62, 0x6c, 0x65, 0x27, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x20, 0x65, 0x6c, - 0x73, 0x65, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x6f, - 0x6c, 0x65, 0x2e, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x28, 0x60, 0x6c, 0x6c, - 0x61, 0x6d, 0x61, 0x2e, 0x63, 0x70, 0x70, 0x20, 0x65, 0x72, 0x72, 0x6f, - 0x72, 0x3a, 0x20, 0x24, 0x7b, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x2e, - 0x65, 0x72, 0x72, 0x6f, 0x72, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, - 0x74, 0x7d, 0x60, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x74, 0x68, 0x72, + 0x6f, 0x77, 0x20, 0x6e, 0x65, 0x77, 0x20, 0x45, 0x72, 0x72, 0x6f, 0x72, + 0x28, 0x27, 0x73, 0x6c, 0x6f, 0x74, 0x20, 0x75, 0x6e, 0x61, 0x76, 0x61, + 0x69, 0x6c, 0x61, 0x62, 0x6c, 0x65, 0x27, 0x29, 0x3b, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x7d, 0x20, 0x65, 0x6c, 0x73, 0x65, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x6f, 0x6c, 0x65, 0x2e, 0x65, 0x72, 0x72, + 0x6f, 0x72, 0x28, 0x60, 0x6c, 0x6c, 0x61, 0x6d, 0x61, 0x2e, 0x63, 0x70, + 0x70, 0x20, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x20, 0x5b, 0x24, 0x7b, 0x72, + 0x65, 0x73, 0x75, 0x6c, 0x74, 0x2e, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x2e, + 0x63, 0x6f, 0x64, 0x65, 0x7d, 0x20, 0x2d, 0x20, 0x24, 0x7b, 0x72, 0x65, + 0x73, 0x75, 0x6c, 0x74, 0x2e, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x2e, 0x74, + 0x79, 0x70, 0x65, 0x7d, 0x5d, 0x3a, 0x20, 0x24, 0x7b, 0x72, 0x65, 0x73, + 0x75, 0x6c, 0x74, 0x2e, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x2e, 0x6d, 0x65, + 0x73, 0x73, 0x61, 0x67, 0x65, 0x7d, 0x60, 0x29, 0x3b, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x7d, 0x20, 0x63, 0x61, 0x74, 0x63, 0x68, 0x28, 0x65, 0x29, + 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x6f, 0x6c, 0x65, + 0x2e, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x28, 0x60, 0x6c, 0x6c, 0x61, 0x6d, + 0x61, 0x2e, 0x63, 0x70, 0x70, 0x20, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x20, + 0x24, 0x7b, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x2e, 0x65, 0x72, 0x72, + 0x6f, 0x72, 0x7d, 0x60, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, 0x72, 0x65, - 0x73, 0x75, 0x6c, 0x74, 0x2e, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x29, 0x20, - 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x2e, 0x65, 0x72, 0x72, - 0x6f, 0x72, 0x20, 0x3d, 0x20, 0x4a, 0x53, 0x4f, 0x4e, 0x2e, 0x70, 0x61, - 0x72, 0x73, 0x65, 0x28, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x2e, 0x65, - 0x72, 0x72, 0x6f, 0x72, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x6f, - 0x6c, 0x65, 0x2e, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x28, 0x60, 0x6c, 0x6c, - 0x61, 0x6d, 0x61, 0x2e, 0x63, 0x70, 0x70, 0x20, 0x65, 0x72, 0x72, 0x6f, - 0x72, 0x3a, 0x20, 0x24, 0x7b, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x2e, - 0x65, 0x72, 0x72, 0x6f, 0x72, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, - 0x74, 0x7d, 0x60, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, - 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x7d, 0x20, 0x63, 0x61, - 0x74, 0x63, 0x68, 0x20, 0x28, 0x65, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, - 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, 0x65, 0x2e, 0x6e, 0x61, 0x6d, 0x65, - 0x20, 0x21, 0x3d, 0x3d, 0x20, 0x27, 0x41, 0x62, 0x6f, 0x72, 0x74, 0x45, - 0x72, 0x72, 0x6f, 0x72, 0x27, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x6f, 0x6c, 0x65, 0x2e, 0x65, - 0x72, 0x72, 0x6f, 0x72, 0x28, 0x22, 0x6c, 0x6c, 0x61, 0x6d, 0x61, 0x20, - 0x65, 0x72, 0x72, 0x6f, 0x72, 0x3a, 0x20, 0x22, 0x2c, 0x20, 0x65, 0x29, - 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, - 0x74, 0x68, 0x72, 0x6f, 0x77, 0x20, 0x65, 0x3b, 0x0a, 0x20, 0x20, 0x7d, - 0x0a, 0x20, 0x20, 0x66, 0x69, 0x6e, 0x61, 0x6c, 0x6c, 0x79, 0x20, 0x7b, - 0x0a, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, - 0x6c, 0x65, 0x72, 0x2e, 0x61, 0x62, 0x6f, 0x72, 0x74, 0x28, 0x29, 0x3b, - 0x0a, 0x20, 0x20, 0x7d, 0x0a, 0x0a, 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, - 0x72, 0x6e, 0x20, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x3b, 0x0a, - 0x7d, 0x0a, 0x0a, 0x2f, 0x2f, 0x20, 0x43, 0x61, 0x6c, 0x6c, 0x20, 0x6c, - 0x6c, 0x61, 0x6d, 0x61, 0x2c, 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, - 0x20, 0x61, 0x6e, 0x20, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x20, 0x74, 0x61, - 0x72, 0x67, 0x65, 0x74, 0x20, 0x74, 0x68, 0x61, 0x74, 0x20, 0x79, 0x6f, - 0x75, 0x20, 0x63, 0x61, 0x6e, 0x20, 0x73, 0x75, 0x62, 0x73, 0x63, 0x72, - 0x69, 0x62, 0x65, 0x20, 0x74, 0x6f, 0x0a, 0x2f, 0x2f, 0x0a, 0x2f, 0x2f, - 0x20, 0x45, 0x78, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x3a, 0x0a, 0x2f, 0x2f, - 0x0a, 0x2f, 0x2f, 0x20, 0x20, 0x20, 0x20, 0x69, 0x6d, 0x70, 0x6f, 0x72, - 0x74, 0x20, 0x7b, 0x20, 0x6c, 0x6c, 0x61, 0x6d, 0x61, 0x45, 0x76, 0x65, - 0x6e, 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x20, 0x7d, 0x20, 0x66, - 0x72, 0x6f, 0x6d, 0x20, 0x27, 0x2f, 0x63, 0x6f, 0x6d, 0x70, 0x6c, 0x65, - 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x6a, 0x73, 0x27, 0x0a, 0x2f, 0x2f, 0x0a, - 0x2f, 0x2f, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, - 0x63, 0x6f, 0x6e, 0x6e, 0x20, 0x3d, 0x20, 0x6c, 0x6c, 0x61, 0x6d, 0x61, - 0x45, 0x76, 0x65, 0x6e, 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x28, - 0x70, 0x72, 0x6f, 0x6d, 0x70, 0x74, 0x29, 0x0a, 0x2f, 0x2f, 0x20, 0x20, - 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x6e, 0x2e, 0x61, 0x64, 0x64, 0x45, 0x76, - 0x65, 0x6e, 0x74, 0x4c, 0x69, 0x73, 0x74, 0x65, 0x6e, 0x65, 0x72, 0x28, - 0x22, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x22, 0x2c, 0x20, 0x28, - 0x63, 0x68, 0x75, 0x6e, 0x6b, 0x29, 0x20, 0x3d, 0x3e, 0x20, 0x7b, 0x0a, - 0x2f, 0x2f, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x64, 0x6f, 0x63, 0x75, - 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x77, 0x72, 0x69, 0x74, 0x65, 0x28, 0x63, - 0x68, 0x75, 0x6e, 0x6b, 0x2e, 0x64, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x2e, - 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x29, 0x0a, 0x2f, 0x2f, 0x20, - 0x20, 0x20, 0x20, 0x7d, 0x29, 0x0a, 0x2f, 0x2f, 0x0a, 0x65, 0x78, 0x70, - 0x6f, 0x72, 0x74, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x6c, 0x6c, - 0x61, 0x6d, 0x61, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x54, 0x61, 0x72, 0x67, - 0x65, 0x74, 0x20, 0x3d, 0x20, 0x28, 0x70, 0x72, 0x6f, 0x6d, 0x70, 0x74, - 0x2c, 0x20, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x20, 0x3d, 0x20, 0x7b, - 0x7d, 0x2c, 0x20, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x20, 0x3d, 0x20, - 0x7b, 0x7d, 0x29, 0x20, 0x3d, 0x3e, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x63, - 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x54, 0x61, - 0x72, 0x67, 0x65, 0x74, 0x20, 0x3d, 0x20, 0x6e, 0x65, 0x77, 0x20, 0x45, - 0x76, 0x65, 0x6e, 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x28, 0x29, - 0x3b, 0x0a, 0x20, 0x20, 0x28, 0x61, 0x73, 0x79, 0x6e, 0x63, 0x20, 0x28, - 0x29, 0x20, 0x3d, 0x3e, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x6c, - 0x65, 0x74, 0x20, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x20, 0x3d, - 0x20, 0x22, 0x22, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x66, 0x6f, 0x72, - 0x20, 0x61, 0x77, 0x61, 0x69, 0x74, 0x20, 0x28, 0x63, 0x6f, 0x6e, 0x73, - 0x74, 0x20, 0x63, 0x68, 0x75, 0x6e, 0x6b, 0x20, 0x6f, 0x66, 0x20, 0x6c, - 0x6c, 0x61, 0x6d, 0x61, 0x28, 0x70, 0x72, 0x6f, 0x6d, 0x70, 0x74, 0x2c, - 0x20, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x2c, 0x20, 0x63, 0x6f, 0x6e, - 0x66, 0x69, 0x67, 0x29, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, 0x63, 0x68, 0x75, 0x6e, 0x6b, 0x2e, - 0x64, 0x61, 0x74, 0x61, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x20, - 0x2b, 0x3d, 0x20, 0x63, 0x68, 0x75, 0x6e, 0x6b, 0x2e, 0x64, 0x61, 0x74, - 0x61, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x3b, 0x0a, 0x20, + 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x7d, 0x20, + 0x63, 0x61, 0x74, 0x63, 0x68, 0x20, 0x28, 0x65, 0x29, 0x20, 0x7b, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, 0x65, 0x2e, 0x6e, 0x61, + 0x6d, 0x65, 0x20, 0x21, 0x3d, 0x3d, 0x20, 0x27, 0x41, 0x62, 0x6f, 0x72, + 0x74, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x27, 0x29, 0x20, 0x7b, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x6f, 0x6c, 0x65, + 0x2e, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x28, 0x22, 0x6c, 0x6c, 0x61, 0x6d, + 0x61, 0x20, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x3a, 0x20, 0x22, 0x2c, 0x20, + 0x65, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x74, 0x68, 0x72, 0x6f, 0x77, 0x20, 0x65, 0x3b, 0x0a, 0x20, + 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x66, 0x69, 0x6e, 0x61, 0x6c, 0x6c, 0x79, + 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x74, 0x72, + 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x2e, 0x61, 0x62, 0x6f, 0x72, 0x74, 0x28, + 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x7d, 0x0a, 0x0a, 0x20, 0x20, 0x72, 0x65, + 0x74, 0x75, 0x72, 0x6e, 0x20, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, + 0x3b, 0x0a, 0x7d, 0x0a, 0x0a, 0x2f, 0x2f, 0x20, 0x43, 0x61, 0x6c, 0x6c, + 0x20, 0x6c, 0x6c, 0x61, 0x6d, 0x61, 0x2c, 0x20, 0x72, 0x65, 0x74, 0x75, + 0x72, 0x6e, 0x20, 0x61, 0x6e, 0x20, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x20, + 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x20, 0x74, 0x68, 0x61, 0x74, 0x20, + 0x79, 0x6f, 0x75, 0x20, 0x63, 0x61, 0x6e, 0x20, 0x73, 0x75, 0x62, 0x73, + 0x63, 0x72, 0x69, 0x62, 0x65, 0x20, 0x74, 0x6f, 0x0a, 0x2f, 0x2f, 0x0a, + 0x2f, 0x2f, 0x20, 0x45, 0x78, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x3a, 0x0a, + 0x2f, 0x2f, 0x0a, 0x2f, 0x2f, 0x20, 0x20, 0x20, 0x20, 0x69, 0x6d, 0x70, + 0x6f, 0x72, 0x74, 0x20, 0x7b, 0x20, 0x6c, 0x6c, 0x61, 0x6d, 0x61, 0x45, + 0x76, 0x65, 0x6e, 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x20, 0x7d, + 0x20, 0x66, 0x72, 0x6f, 0x6d, 0x20, 0x27, 0x2f, 0x63, 0x6f, 0x6d, 0x70, + 0x6c, 0x65, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x6a, 0x73, 0x27, 0x0a, 0x2f, + 0x2f, 0x0a, 0x2f, 0x2f, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, + 0x74, 0x20, 0x63, 0x6f, 0x6e, 0x6e, 0x20, 0x3d, 0x20, 0x6c, 0x6c, 0x61, + 0x6d, 0x61, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, + 0x74, 0x28, 0x70, 0x72, 0x6f, 0x6d, 0x70, 0x74, 0x29, 0x0a, 0x2f, 0x2f, + 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x6e, 0x2e, 0x61, 0x64, 0x64, + 0x45, 0x76, 0x65, 0x6e, 0x74, 0x4c, 0x69, 0x73, 0x74, 0x65, 0x6e, 0x65, + 0x72, 0x28, 0x22, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x22, 0x2c, + 0x20, 0x28, 0x63, 0x68, 0x75, 0x6e, 0x6b, 0x29, 0x20, 0x3d, 0x3e, 0x20, + 0x7b, 0x0a, 0x2f, 0x2f, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x64, 0x6f, + 0x63, 0x75, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x77, 0x72, 0x69, 0x74, 0x65, + 0x28, 0x63, 0x68, 0x75, 0x6e, 0x6b, 0x2e, 0x64, 0x65, 0x74, 0x61, 0x69, + 0x6c, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x29, 0x0a, 0x2f, + 0x2f, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x29, 0x0a, 0x2f, 0x2f, 0x0a, 0x65, + 0x78, 0x70, 0x6f, 0x72, 0x74, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, + 0x6c, 0x6c, 0x61, 0x6d, 0x61, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x54, 0x61, + 0x72, 0x67, 0x65, 0x74, 0x20, 0x3d, 0x20, 0x28, 0x70, 0x72, 0x6f, 0x6d, + 0x70, 0x74, 0x2c, 0x20, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x20, 0x3d, + 0x20, 0x7b, 0x7d, 0x2c, 0x20, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x20, + 0x3d, 0x20, 0x7b, 0x7d, 0x29, 0x20, 0x3d, 0x3e, 0x20, 0x7b, 0x0a, 0x20, + 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x65, 0x76, 0x65, 0x6e, 0x74, + 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x20, 0x3d, 0x20, 0x6e, 0x65, 0x77, + 0x20, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, + 0x28, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x28, 0x61, 0x73, 0x79, 0x6e, 0x63, + 0x20, 0x28, 0x29, 0x20, 0x3d, 0x3e, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x6c, 0x65, 0x74, 0x20, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, + 0x20, 0x3d, 0x20, 0x22, 0x22, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x66, + 0x6f, 0x72, 0x20, 0x61, 0x77, 0x61, 0x69, 0x74, 0x20, 0x28, 0x63, 0x6f, + 0x6e, 0x73, 0x74, 0x20, 0x63, 0x68, 0x75, 0x6e, 0x6b, 0x20, 0x6f, 0x66, + 0x20, 0x6c, 0x6c, 0x61, 0x6d, 0x61, 0x28, 0x70, 0x72, 0x6f, 0x6d, 0x70, + 0x74, 0x2c, 0x20, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x2c, 0x20, 0x63, + 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x29, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, 0x63, 0x68, 0x75, 0x6e, + 0x6b, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, + 0x74, 0x20, 0x2b, 0x3d, 0x20, 0x63, 0x68, 0x75, 0x6e, 0x6b, 0x2e, 0x64, + 0x61, 0x74, 0x61, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x3b, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x65, 0x76, 0x65, + 0x6e, 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x2e, 0x64, 0x69, 0x73, + 0x70, 0x61, 0x74, 0x63, 0x68, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x28, 0x6e, + 0x65, 0x77, 0x20, 0x43, 0x75, 0x73, 0x74, 0x6f, 0x6d, 0x45, 0x76, 0x65, + 0x6e, 0x74, 0x28, 0x22, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x22, + 0x2c, 0x20, 0x7b, 0x20, 0x64, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x3a, 0x20, + 0x63, 0x68, 0x75, 0x6e, 0x6b, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x20, 0x7d, + 0x29, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, 0x63, 0x68, + 0x75, 0x6e, 0x6b, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x67, 0x65, 0x6e, + 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x73, 0x65, 0x74, 0x74, + 0x69, 0x6e, 0x67, 0x73, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x54, 0x61, 0x72, + 0x67, 0x65, 0x74, 0x2e, 0x64, 0x69, 0x73, 0x70, 0x61, 0x74, 0x63, 0x68, + 0x45, 0x76, 0x65, 0x6e, 0x74, 0x28, 0x6e, 0x65, 0x77, 0x20, 0x43, 0x75, + 0x73, 0x74, 0x6f, 0x6d, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x28, 0x22, 0x67, + 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x73, 0x65, + 0x74, 0x74, 0x69, 0x6e, 0x67, 0x73, 0x22, 0x2c, 0x20, 0x7b, 0x20, 0x64, + 0x65, 0x74, 0x61, 0x69, 0x6c, 0x3a, 0x20, 0x63, 0x68, 0x75, 0x6e, 0x6b, + 0x2e, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, + 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x73, 0x65, 0x74, 0x74, 0x69, 0x6e, 0x67, + 0x73, 0x20, 0x7d, 0x29, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, + 0x28, 0x63, 0x68, 0x75, 0x6e, 0x6b, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x2e, + 0x74, 0x69, 0x6d, 0x69, 0x6e, 0x67, 0x73, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x2e, 0x64, 0x69, 0x73, 0x70, 0x61, 0x74, 0x63, 0x68, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x28, 0x6e, 0x65, 0x77, 0x20, 0x43, 0x75, 0x73, 0x74, 0x6f, 0x6d, 0x45, 0x76, 0x65, 0x6e, 0x74, - 0x28, 0x22, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x22, 0x2c, 0x20, + 0x28, 0x22, 0x74, 0x69, 0x6d, 0x69, 0x6e, 0x67, 0x73, 0x22, 0x2c, 0x20, 0x7b, 0x20, 0x64, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x3a, 0x20, 0x63, 0x68, - 0x75, 0x6e, 0x6b, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x20, 0x7d, 0x29, 0x29, - 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, 0x63, 0x68, 0x75, 0x6e, - 0x6b, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x67, 0x65, 0x6e, 0x65, 0x72, - 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x73, 0x65, 0x74, 0x74, 0x69, 0x6e, - 0x67, 0x73, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, - 0x74, 0x2e, 0x64, 0x69, 0x73, 0x70, 0x61, 0x74, 0x63, 0x68, 0x45, 0x76, - 0x65, 0x6e, 0x74, 0x28, 0x6e, 0x65, 0x77, 0x20, 0x43, 0x75, 0x73, 0x74, - 0x6f, 0x6d, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x28, 0x22, 0x67, 0x65, 0x6e, - 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x73, 0x65, 0x74, 0x74, - 0x69, 0x6e, 0x67, 0x73, 0x22, 0x2c, 0x20, 0x7b, 0x20, 0x64, 0x65, 0x74, - 0x61, 0x69, 0x6c, 0x3a, 0x20, 0x63, 0x68, 0x75, 0x6e, 0x6b, 0x2e, 0x64, - 0x61, 0x74, 0x61, 0x2e, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x69, - 0x6f, 0x6e, 0x5f, 0x73, 0x65, 0x74, 0x74, 0x69, 0x6e, 0x67, 0x73, 0x20, - 0x7d, 0x29, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, - 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, 0x63, - 0x68, 0x75, 0x6e, 0x6b, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x74, 0x69, - 0x6d, 0x69, 0x6e, 0x67, 0x73, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x54, 0x61, - 0x72, 0x67, 0x65, 0x74, 0x2e, 0x64, 0x69, 0x73, 0x70, 0x61, 0x74, 0x63, - 0x68, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x28, 0x6e, 0x65, 0x77, 0x20, 0x43, - 0x75, 0x73, 0x74, 0x6f, 0x6d, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x28, 0x22, - 0x74, 0x69, 0x6d, 0x69, 0x6e, 0x67, 0x73, 0x22, 0x2c, 0x20, 0x7b, 0x20, - 0x64, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x3a, 0x20, 0x63, 0x68, 0x75, 0x6e, - 0x6b, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x74, 0x69, 0x6d, 0x69, 0x6e, - 0x67, 0x73, 0x20, 0x7d, 0x29, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, - 0x20, 0x20, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, - 0x74, 0x2e, 0x64, 0x69, 0x73, 0x70, 0x61, 0x74, 0x63, 0x68, 0x45, 0x76, - 0x65, 0x6e, 0x74, 0x28, 0x6e, 0x65, 0x77, 0x20, 0x43, 0x75, 0x73, 0x74, - 0x6f, 0x6d, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x28, 0x22, 0x64, 0x6f, 0x6e, - 0x65, 0x22, 0x2c, 0x20, 0x7b, 0x20, 0x64, 0x65, 0x74, 0x61, 0x69, 0x6c, - 0x3a, 0x20, 0x7b, 0x20, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x20, - 0x7d, 0x20, 0x7d, 0x29, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x7d, 0x29, 0x28, - 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, - 0x65, 0x76, 0x65, 0x6e, 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x3b, - 0x0a, 0x7d, 0x0a, 0x0a, 0x2f, 0x2f, 0x20, 0x43, 0x61, 0x6c, 0x6c, 0x20, - 0x6c, 0x6c, 0x61, 0x6d, 0x61, 0x2c, 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, - 0x6e, 0x20, 0x61, 0x20, 0x70, 0x72, 0x6f, 0x6d, 0x69, 0x73, 0x65, 0x20, - 0x74, 0x68, 0x61, 0x74, 0x20, 0x72, 0x65, 0x73, 0x6f, 0x6c, 0x76, 0x65, - 0x73, 0x20, 0x74, 0x6f, 0x20, 0x74, 0x68, 0x65, 0x20, 0x63, 0x6f, 0x6d, - 0x70, 0x6c, 0x65, 0x74, 0x65, 0x64, 0x20, 0x74, 0x65, 0x78, 0x74, 0x2e, - 0x20, 0x54, 0x68, 0x69, 0x73, 0x20, 0x64, 0x6f, 0x65, 0x73, 0x20, 0x6e, - 0x6f, 0x74, 0x20, 0x73, 0x75, 0x70, 0x70, 0x6f, 0x72, 0x74, 0x20, 0x73, - 0x74, 0x72, 0x65, 0x61, 0x6d, 0x69, 0x6e, 0x67, 0x0a, 0x2f, 0x2f, 0x0a, - 0x2f, 0x2f, 0x20, 0x45, 0x78, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x3a, 0x0a, - 0x2f, 0x2f, 0x0a, 0x2f, 0x2f, 0x20, 0x20, 0x20, 0x20, 0x20, 0x6c, 0x6c, - 0x61, 0x6d, 0x61, 0x50, 0x72, 0x6f, 0x6d, 0x69, 0x73, 0x65, 0x28, 0x70, - 0x72, 0x6f, 0x6d, 0x70, 0x74, 0x29, 0x2e, 0x74, 0x68, 0x65, 0x6e, 0x28, - 0x28, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x29, 0x20, 0x3d, 0x3e, - 0x20, 0x7b, 0x0a, 0x2f, 0x2f, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x64, 0x6f, 0x63, 0x75, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x77, 0x72, 0x69, - 0x74, 0x65, 0x28, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x29, 0x0a, - 0x2f, 0x2f, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x29, 0x0a, 0x2f, 0x2f, - 0x0a, 0x2f, 0x2f, 0x20, 0x20, 0x20, 0x20, 0x20, 0x6f, 0x72, 0x0a, 0x2f, - 0x2f, 0x0a, 0x2f, 0x2f, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, - 0x73, 0x74, 0x20, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x20, 0x3d, - 0x20, 0x61, 0x77, 0x61, 0x69, 0x74, 0x20, 0x6c, 0x6c, 0x61, 0x6d, 0x61, - 0x50, 0x72, 0x6f, 0x6d, 0x69, 0x73, 0x65, 0x28, 0x70, 0x72, 0x6f, 0x6d, - 0x70, 0x74, 0x29, 0x0a, 0x2f, 0x2f, 0x20, 0x20, 0x20, 0x20, 0x20, 0x64, - 0x6f, 0x63, 0x75, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x77, 0x72, 0x69, 0x74, - 0x65, 0x28, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x29, 0x0a, 0x2f, - 0x2f, 0x0a, 0x65, 0x78, 0x70, 0x6f, 0x72, 0x74, 0x20, 0x63, 0x6f, 0x6e, - 0x73, 0x74, 0x20, 0x6c, 0x6c, 0x61, 0x6d, 0x61, 0x50, 0x72, 0x6f, 0x6d, - 0x69, 0x73, 0x65, 0x20, 0x3d, 0x20, 0x28, 0x70, 0x72, 0x6f, 0x6d, 0x70, - 0x74, 0x2c, 0x20, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x20, 0x3d, 0x20, - 0x7b, 0x7d, 0x2c, 0x20, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x20, 0x3d, - 0x20, 0x7b, 0x7d, 0x29, 0x20, 0x3d, 0x3e, 0x20, 0x7b, 0x0a, 0x20, 0x20, - 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x6e, 0x65, 0x77, 0x20, 0x50, - 0x72, 0x6f, 0x6d, 0x69, 0x73, 0x65, 0x28, 0x61, 0x73, 0x79, 0x6e, 0x63, - 0x20, 0x28, 0x72, 0x65, 0x73, 0x6f, 0x6c, 0x76, 0x65, 0x2c, 0x20, 0x72, - 0x65, 0x6a, 0x65, 0x63, 0x74, 0x29, 0x20, 0x3d, 0x3e, 0x20, 0x7b, 0x0a, - 0x20, 0x20, 0x20, 0x20, 0x6c, 0x65, 0x74, 0x20, 0x63, 0x6f, 0x6e, 0x74, - 0x65, 0x6e, 0x74, 0x20, 0x3d, 0x20, 0x22, 0x22, 0x3b, 0x0a, 0x20, 0x20, - 0x20, 0x20, 0x74, 0x72, 0x79, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x75, 0x6e, 0x6b, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x74, 0x69, 0x6d, + 0x69, 0x6e, 0x67, 0x73, 0x20, 0x7d, 0x29, 0x29, 0x3b, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x54, 0x61, 0x72, + 0x67, 0x65, 0x74, 0x2e, 0x64, 0x69, 0x73, 0x70, 0x61, 0x74, 0x63, 0x68, + 0x45, 0x76, 0x65, 0x6e, 0x74, 0x28, 0x6e, 0x65, 0x77, 0x20, 0x43, 0x75, + 0x73, 0x74, 0x6f, 0x6d, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x28, 0x22, 0x64, + 0x6f, 0x6e, 0x65, 0x22, 0x2c, 0x20, 0x7b, 0x20, 0x64, 0x65, 0x74, 0x61, + 0x69, 0x6c, 0x3a, 0x20, 0x7b, 0x20, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, + 0x74, 0x20, 0x7d, 0x20, 0x7d, 0x29, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x7d, + 0x29, 0x28, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, + 0x6e, 0x20, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, + 0x74, 0x3b, 0x0a, 0x7d, 0x0a, 0x0a, 0x2f, 0x2f, 0x20, 0x43, 0x61, 0x6c, + 0x6c, 0x20, 0x6c, 0x6c, 0x61, 0x6d, 0x61, 0x2c, 0x20, 0x72, 0x65, 0x74, + 0x75, 0x72, 0x6e, 0x20, 0x61, 0x20, 0x70, 0x72, 0x6f, 0x6d, 0x69, 0x73, + 0x65, 0x20, 0x74, 0x68, 0x61, 0x74, 0x20, 0x72, 0x65, 0x73, 0x6f, 0x6c, + 0x76, 0x65, 0x73, 0x20, 0x74, 0x6f, 0x20, 0x74, 0x68, 0x65, 0x20, 0x63, + 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65, 0x64, 0x20, 0x74, 0x65, 0x78, + 0x74, 0x2e, 0x20, 0x54, 0x68, 0x69, 0x73, 0x20, 0x64, 0x6f, 0x65, 0x73, + 0x20, 0x6e, 0x6f, 0x74, 0x20, 0x73, 0x75, 0x70, 0x70, 0x6f, 0x72, 0x74, + 0x20, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x69, 0x6e, 0x67, 0x0a, 0x2f, + 0x2f, 0x0a, 0x2f, 0x2f, 0x20, 0x45, 0x78, 0x61, 0x6d, 0x70, 0x6c, 0x65, + 0x3a, 0x0a, 0x2f, 0x2f, 0x0a, 0x2f, 0x2f, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x6c, 0x6c, 0x61, 0x6d, 0x61, 0x50, 0x72, 0x6f, 0x6d, 0x69, 0x73, 0x65, + 0x28, 0x70, 0x72, 0x6f, 0x6d, 0x70, 0x74, 0x29, 0x2e, 0x74, 0x68, 0x65, + 0x6e, 0x28, 0x28, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x29, 0x20, + 0x3d, 0x3e, 0x20, 0x7b, 0x0a, 0x2f, 0x2f, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x64, 0x6f, 0x63, 0x75, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x77, + 0x72, 0x69, 0x74, 0x65, 0x28, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, + 0x29, 0x0a, 0x2f, 0x2f, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x29, 0x0a, + 0x2f, 0x2f, 0x0a, 0x2f, 0x2f, 0x20, 0x20, 0x20, 0x20, 0x20, 0x6f, 0x72, + 0x0a, 0x2f, 0x2f, 0x0a, 0x2f, 0x2f, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, + 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, + 0x20, 0x3d, 0x20, 0x61, 0x77, 0x61, 0x69, 0x74, 0x20, 0x6c, 0x6c, 0x61, + 0x6d, 0x61, 0x50, 0x72, 0x6f, 0x6d, 0x69, 0x73, 0x65, 0x28, 0x70, 0x72, + 0x6f, 0x6d, 0x70, 0x74, 0x29, 0x0a, 0x2f, 0x2f, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x64, 0x6f, 0x63, 0x75, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x77, 0x72, + 0x69, 0x74, 0x65, 0x28, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x29, + 0x0a, 0x2f, 0x2f, 0x0a, 0x65, 0x78, 0x70, 0x6f, 0x72, 0x74, 0x20, 0x63, + 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x6c, 0x6c, 0x61, 0x6d, 0x61, 0x50, 0x72, + 0x6f, 0x6d, 0x69, 0x73, 0x65, 0x20, 0x3d, 0x20, 0x28, 0x70, 0x72, 0x6f, + 0x6d, 0x70, 0x74, 0x2c, 0x20, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x20, + 0x3d, 0x20, 0x7b, 0x7d, 0x2c, 0x20, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, + 0x20, 0x3d, 0x20, 0x7b, 0x7d, 0x29, 0x20, 0x3d, 0x3e, 0x20, 0x7b, 0x0a, + 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x6e, 0x65, 0x77, + 0x20, 0x50, 0x72, 0x6f, 0x6d, 0x69, 0x73, 0x65, 0x28, 0x61, 0x73, 0x79, + 0x6e, 0x63, 0x20, 0x28, 0x72, 0x65, 0x73, 0x6f, 0x6c, 0x76, 0x65, 0x2c, + 0x20, 0x72, 0x65, 0x6a, 0x65, 0x63, 0x74, 0x29, 0x20, 0x3d, 0x3e, 0x20, + 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x6c, 0x65, 0x74, 0x20, 0x63, 0x6f, + 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x20, 0x3d, 0x20, 0x22, 0x22, 0x3b, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x74, 0x72, 0x79, 0x20, 0x7b, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x66, 0x6f, 0x72, 0x20, 0x61, 0x77, 0x61, 0x69, + 0x74, 0x20, 0x28, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x63, 0x68, 0x75, + 0x6e, 0x6b, 0x20, 0x6f, 0x66, 0x20, 0x6c, 0x6c, 0x61, 0x6d, 0x61, 0x28, + 0x70, 0x72, 0x6f, 0x6d, 0x70, 0x74, 0x2c, 0x20, 0x70, 0x61, 0x72, 0x61, + 0x6d, 0x73, 0x2c, 0x20, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x29, 0x29, + 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, + 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x20, 0x2b, 0x3d, 0x20, 0x63, 0x68, + 0x75, 0x6e, 0x6b, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x63, 0x6f, 0x6e, + 0x74, 0x65, 0x6e, 0x74, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, 0x73, 0x6f, + 0x6c, 0x76, 0x65, 0x28, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x29, + 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x20, 0x63, 0x61, 0x74, 0x63, + 0x68, 0x20, 0x28, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x29, 0x20, 0x7b, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, 0x6a, 0x65, 0x63, 0x74, + 0x28, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x7d, 0x29, 0x3b, 0x0a, 0x7d, 0x3b, 0x0a, + 0x0a, 0x2f, 0x2a, 0x2a, 0x0a, 0x20, 0x2a, 0x20, 0x28, 0x64, 0x65, 0x70, + 0x72, 0x65, 0x63, 0x61, 0x74, 0x65, 0x64, 0x29, 0x0a, 0x20, 0x2a, 0x2f, + 0x0a, 0x65, 0x78, 0x70, 0x6f, 0x72, 0x74, 0x20, 0x63, 0x6f, 0x6e, 0x73, + 0x74, 0x20, 0x6c, 0x6c, 0x61, 0x6d, 0x61, 0x43, 0x6f, 0x6d, 0x70, 0x6c, + 0x65, 0x74, 0x65, 0x20, 0x3d, 0x20, 0x61, 0x73, 0x79, 0x6e, 0x63, 0x20, + 0x28, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x2c, 0x20, 0x63, 0x6f, 0x6e, + 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x2c, 0x20, 0x63, 0x61, 0x6c, + 0x6c, 0x62, 0x61, 0x63, 0x6b, 0x29, 0x20, 0x3d, 0x3e, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x66, 0x6f, 0x72, 0x20, 0x61, 0x77, 0x61, 0x69, 0x74, 0x20, 0x28, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x63, 0x68, 0x75, 0x6e, 0x6b, - 0x20, 0x6f, 0x66, 0x20, 0x6c, 0x6c, 0x61, 0x6d, 0x61, 0x28, 0x70, 0x72, - 0x6f, 0x6d, 0x70, 0x74, 0x2c, 0x20, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, - 0x2c, 0x20, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x29, 0x29, 0x20, 0x7b, - 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, - 0x74, 0x65, 0x6e, 0x74, 0x20, 0x2b, 0x3d, 0x20, 0x63, 0x68, 0x75, 0x6e, - 0x6b, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x65, - 0x6e, 0x74, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, 0x73, 0x6f, 0x6c, 0x76, - 0x65, 0x28, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x29, 0x3b, 0x0a, - 0x20, 0x20, 0x20, 0x20, 0x7d, 0x20, 0x63, 0x61, 0x74, 0x63, 0x68, 0x20, - 0x28, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, 0x6a, 0x65, 0x63, 0x74, 0x28, 0x65, - 0x72, 0x72, 0x6f, 0x72, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x7d, - 0x0a, 0x20, 0x20, 0x7d, 0x29, 0x3b, 0x0a, 0x7d, 0x3b, 0x0a, 0x0a, 0x2f, - 0x2a, 0x2a, 0x0a, 0x20, 0x2a, 0x20, 0x28, 0x64, 0x65, 0x70, 0x72, 0x65, - 0x63, 0x61, 0x74, 0x65, 0x64, 0x29, 0x0a, 0x20, 0x2a, 0x2f, 0x0a, 0x65, - 0x78, 0x70, 0x6f, 0x72, 0x74, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, - 0x6c, 0x6c, 0x61, 0x6d, 0x61, 0x43, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, - 0x65, 0x20, 0x3d, 0x20, 0x61, 0x73, 0x79, 0x6e, 0x63, 0x20, 0x28, 0x70, - 0x61, 0x72, 0x61, 0x6d, 0x73, 0x2c, 0x20, 0x63, 0x6f, 0x6e, 0x74, 0x72, - 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x2c, 0x20, 0x63, 0x61, 0x6c, 0x6c, 0x62, - 0x61, 0x63, 0x6b, 0x29, 0x20, 0x3d, 0x3e, 0x20, 0x7b, 0x0a, 0x20, 0x20, - 0x66, 0x6f, 0x72, 0x20, 0x61, 0x77, 0x61, 0x69, 0x74, 0x20, 0x28, 0x63, - 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x63, 0x68, 0x75, 0x6e, 0x6b, 0x20, 0x6f, - 0x66, 0x20, 0x6c, 0x6c, 0x61, 0x6d, 0x61, 0x28, 0x70, 0x61, 0x72, 0x61, - 0x6d, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x6d, 0x70, 0x74, 0x2c, 0x20, 0x70, - 0x61, 0x72, 0x61, 0x6d, 0x73, 0x2c, 0x20, 0x7b, 0x20, 0x63, 0x6f, 0x6e, - 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x20, 0x7d, 0x29, 0x29, 0x20, - 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x63, 0x61, 0x6c, 0x6c, 0x62, 0x61, - 0x63, 0x6b, 0x28, 0x63, 0x68, 0x75, 0x6e, 0x6b, 0x29, 0x3b, 0x0a, 0x20, - 0x20, 0x7d, 0x0a, 0x7d, 0x0a, 0x0a, 0x2f, 0x2f, 0x20, 0x47, 0x65, 0x74, - 0x20, 0x74, 0x68, 0x65, 0x20, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x20, 0x69, - 0x6e, 0x66, 0x6f, 0x20, 0x66, 0x72, 0x6f, 0x6d, 0x20, 0x74, 0x68, 0x65, - 0x20, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x2e, 0x20, 0x54, 0x68, 0x69, - 0x73, 0x20, 0x69, 0x73, 0x20, 0x75, 0x73, 0x65, 0x66, 0x75, 0x6c, 0x20, - 0x66, 0x6f, 0x72, 0x20, 0x67, 0x65, 0x74, 0x74, 0x69, 0x6e, 0x67, 0x20, - 0x74, 0x68, 0x65, 0x20, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x20, - 0x77, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x20, 0x61, 0x6e, 0x64, 0x20, 0x73, - 0x6f, 0x20, 0x6f, 0x6e, 0x2e, 0x0a, 0x65, 0x78, 0x70, 0x6f, 0x72, 0x74, - 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x6c, 0x6c, 0x61, 0x6d, 0x61, - 0x4d, 0x6f, 0x64, 0x65, 0x6c, 0x49, 0x6e, 0x66, 0x6f, 0x20, 0x3d, 0x20, - 0x61, 0x73, 0x79, 0x6e, 0x63, 0x20, 0x28, 0x29, 0x20, 0x3d, 0x3e, 0x20, - 0x7b, 0x0a, 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, 0x21, 0x67, 0x65, 0x6e, - 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x73, 0x65, 0x74, 0x74, - 0x69, 0x6e, 0x67, 0x73, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, - 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x70, 0x72, 0x6f, 0x70, 0x73, 0x20, - 0x3d, 0x20, 0x61, 0x77, 0x61, 0x69, 0x74, 0x20, 0x66, 0x65, 0x74, 0x63, - 0x68, 0x28, 0x22, 0x2f, 0x70, 0x72, 0x6f, 0x70, 0x73, 0x22, 0x29, 0x2e, - 0x74, 0x68, 0x65, 0x6e, 0x28, 0x72, 0x20, 0x3d, 0x3e, 0x20, 0x72, 0x2e, - 0x6a, 0x73, 0x6f, 0x6e, 0x28, 0x29, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, - 0x20, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, - 0x73, 0x65, 0x74, 0x74, 0x69, 0x6e, 0x67, 0x73, 0x20, 0x3d, 0x20, 0x70, - 0x72, 0x6f, 0x70, 0x73, 0x2e, 0x64, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, - 0x5f, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, - 0x73, 0x65, 0x74, 0x74, 0x69, 0x6e, 0x67, 0x73, 0x3b, 0x0a, 0x20, 0x20, - 0x7d, 0x0a, 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x67, + 0x20, 0x6f, 0x66, 0x20, 0x6c, 0x6c, 0x61, 0x6d, 0x61, 0x28, 0x70, 0x61, + 0x72, 0x61, 0x6d, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x6d, 0x70, 0x74, 0x2c, + 0x20, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x2c, 0x20, 0x7b, 0x20, 0x63, + 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x20, 0x7d, 0x29, + 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x63, 0x61, 0x6c, 0x6c, + 0x62, 0x61, 0x63, 0x6b, 0x28, 0x63, 0x68, 0x75, 0x6e, 0x6b, 0x29, 0x3b, + 0x0a, 0x20, 0x20, 0x7d, 0x0a, 0x7d, 0x0a, 0x0a, 0x2f, 0x2f, 0x20, 0x47, + 0x65, 0x74, 0x20, 0x74, 0x68, 0x65, 0x20, 0x6d, 0x6f, 0x64, 0x65, 0x6c, + 0x20, 0x69, 0x6e, 0x66, 0x6f, 0x20, 0x66, 0x72, 0x6f, 0x6d, 0x20, 0x74, + 0x68, 0x65, 0x20, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x2e, 0x20, 0x54, + 0x68, 0x69, 0x73, 0x20, 0x69, 0x73, 0x20, 0x75, 0x73, 0x65, 0x66, 0x75, + 0x6c, 0x20, 0x66, 0x6f, 0x72, 0x20, 0x67, 0x65, 0x74, 0x74, 0x69, 0x6e, + 0x67, 0x20, 0x74, 0x68, 0x65, 0x20, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, + 0x74, 0x20, 0x77, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x20, 0x61, 0x6e, 0x64, + 0x20, 0x73, 0x6f, 0x20, 0x6f, 0x6e, 0x2e, 0x0a, 0x65, 0x78, 0x70, 0x6f, + 0x72, 0x74, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x6c, 0x6c, 0x61, + 0x6d, 0x61, 0x4d, 0x6f, 0x64, 0x65, 0x6c, 0x49, 0x6e, 0x66, 0x6f, 0x20, + 0x3d, 0x20, 0x61, 0x73, 0x79, 0x6e, 0x63, 0x20, 0x28, 0x29, 0x20, 0x3d, + 0x3e, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, 0x21, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x73, 0x65, - 0x74, 0x74, 0x69, 0x6e, 0x67, 0x73, 0x3b, 0x0a, 0x7d, 0x0a + 0x74, 0x74, 0x69, 0x6e, 0x67, 0x73, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x70, 0x72, 0x6f, 0x70, + 0x73, 0x20, 0x3d, 0x20, 0x61, 0x77, 0x61, 0x69, 0x74, 0x20, 0x66, 0x65, + 0x74, 0x63, 0x68, 0x28, 0x22, 0x2f, 0x70, 0x72, 0x6f, 0x70, 0x73, 0x22, + 0x29, 0x2e, 0x74, 0x68, 0x65, 0x6e, 0x28, 0x72, 0x20, 0x3d, 0x3e, 0x20, + 0x72, 0x2e, 0x6a, 0x73, 0x6f, 0x6e, 0x28, 0x29, 0x29, 0x3b, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, + 0x6e, 0x5f, 0x73, 0x65, 0x74, 0x74, 0x69, 0x6e, 0x67, 0x73, 0x20, 0x3d, + 0x20, 0x70, 0x72, 0x6f, 0x70, 0x73, 0x2e, 0x64, 0x65, 0x66, 0x61, 0x75, + 0x6c, 0x74, 0x5f, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, + 0x6e, 0x5f, 0x73, 0x65, 0x74, 0x74, 0x69, 0x6e, 0x67, 0x73, 0x3b, 0x0a, + 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, + 0x20, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, + 0x73, 0x65, 0x74, 0x74, 0x69, 0x6e, 0x67, 0x73, 0x3b, 0x0a, 0x7d, 0x0a }; -unsigned int completion_js_len = 5782; +unsigned int completion_js_len = 5796; diff --git a/examples/server/public/completion.js b/examples/server/public/completion.js index ab38a7b40..835ce6e68 100644 --- a/examples/server/public/completion.js +++ b/examples/server/public/completion.js @@ -96,18 +96,18 @@ export async function* llama(prompt, params = {}, config = {}) { } } if (result.error) { - result.error = JSON.parse(result.error); - if (result.error.content.includes('slot unavailable')) { - // Throw an error to be caught by upstream callers - throw new Error('slot unavailable'); - } else { - console.error(`llama.cpp error: ${result.error.content}`); + try { + result.error = JSON.parse(result.error); + if (result.error.message.includes('slot unavailable')) { + // Throw an error to be caught by upstream callers + throw new Error('slot unavailable'); + } else { + console.error(`llama.cpp error [${result.error.code} - ${result.error.type}]: ${result.error.message}`); + } + } catch(e) { + console.error(`llama.cpp error ${result.error}`) } } - if (result.error) { - result.error = JSON.parse(result.error); - console.error(`llama.cpp error: ${result.error.content}`); - } } } } diff --git a/examples/server/server.cpp b/examples/server/server.cpp index 3951507aa..b63a6f243 100644 --- a/examples/server/server.cpp +++ b/examples/server/server.cpp @@ -396,7 +396,7 @@ struct server_queue { // callback functions std::function callback_new_task; std::function callback_finish_multitask; - std::function callback_run_slots; + std::function callback_update_slots; // Add a new task to the end of the queue int post(server_task task) { @@ -435,8 +435,8 @@ struct server_queue { } // Register the function to be called when all slots data is ready to be processed - void on_run_slots(std::function callback) { - callback_run_slots = std::move(callback); + void on_update_slots(std::function callback) { + callback_update_slots = std::move(callback); } // Call when the state of one slot is changed @@ -461,7 +461,7 @@ struct server_queue { * - Wait until a new task arrives * - Process the task (i.e. maybe copy data into slot) * - Check if multitask is finished - * - Run all slots + * - Update all slots */ void start_loop() { running = true; @@ -499,9 +499,9 @@ struct server_queue { } // all tasks in the current loop is processed, slots data is now ready - LOG_VERBOSE("callback_run_slots", {}); + LOG_VERBOSE("callback_update_slots", {}); - callback_run_slots(); + callback_update_slots(); LOG_VERBOSE("wait for new task", {}); { @@ -805,9 +805,10 @@ struct server_context { return last_used; } - bool launch_slot_with_data(server_slot & slot, json data) const { + bool launch_slot_with_task(server_slot & slot, const server_task & task) { slot_params default_params; llama_sampling_params default_sparams; + auto & data = task.data; if (data.count("__oaicompat") != 0) { slot.oaicompat = true; @@ -864,10 +865,15 @@ struct server_context { { const auto & prompt = data.find("prompt"); if (prompt == data.end()) { - slot.prompt = ""; + send_error(task, "Either \"prompt\" or \"messages\" must be provided", ERROR_TYPE_INVALID_REQUEST); + return false; } else { slot.prompt = *prompt; } + if (slot.prompt.is_array() && slot.prompt.size() == 0) { + send_error(task, "\"prompt\" cannot be an empty array", ERROR_TYPE_INVALID_REQUEST); + return false; + } } // penalize user-provided tokens @@ -926,6 +932,7 @@ struct server_context { if (logit_bias != data.end() && logit_bias->is_array()) { const int n_vocab = llama_n_vocab(model); for (const auto & el : *logit_bias) { + // TODO: we may want to throw errors here, in case "el" is incorrect if (el.is_array() && el.size() == 2) { float bias; if (el[1].is_number()) { @@ -985,6 +992,11 @@ struct server_context { llama_sampling_free(slot.ctx_sampling); } slot.ctx_sampling = llama_sampling_init(slot.sparams); + if (slot.ctx_sampling == nullptr) { + // for now, the only error that may happen here is invalid grammar + send_error(task, "Failed to parse grammar", ERROR_TYPE_INVALID_REQUEST); + return false; + } llama_set_rng_seed(ctx, slot.params.seed); } @@ -1226,15 +1238,23 @@ struct server_context { }; } - void send_error(const server_task & task, const std::string & error) { - LOG_TEE("task %i - error: %s\n", task.id, error.c_str()); + void send_error(const server_task & task, const std::string & error, const enum error_type type = ERROR_TYPE_SERVER) { + send_error(task.id, task.id_multi, error, type); + } + + void send_error(const server_slot & slot, const std::string & error, const enum error_type type = ERROR_TYPE_SERVER) { + send_error(slot.id_task, slot.id_multi, error, type); + } + + void send_error(const int id_task, const int id_multi, const std::string & error, const enum error_type type = ERROR_TYPE_SERVER) { + LOG_TEE("task %i - error: %s\n", id_task, error.c_str()); server_task_result res; - res.id = task.id; - res.id_multi = task.id_multi; + res.id = id_task; + res.id_multi = id_multi; res.stop = false; res.error = true; - res.data = { { "content", error } }; + res.data = format_error_response(error, type); queue_results.send(res); } @@ -1468,9 +1488,8 @@ struct server_context { slot->infill = task.infill; slot->embedding = task.embedding; - if (!launch_slot_with_data(*slot, task.data)) { - // send error result - send_error(task, "internal_error"); + if (!launch_slot_with_task(*slot, task)) { + LOG_ERROR("error while launching slot", task.data); break; } } break; @@ -1587,7 +1606,7 @@ struct server_context { queue_results.send(result); } - bool update_slots() { + void update_slots() { if (system_need_update) { system_prompt_update(); } @@ -1630,7 +1649,7 @@ struct server_context { kv_cache_clear(); } - return true; + return; } } @@ -1975,8 +1994,7 @@ struct server_context { if (batch.n_tokens == 0) { LOG_VERBOSE("no tokens to decode", {}); - - return true; + return; } LOG_VERBOSE("decoding batch", { @@ -2033,7 +2051,13 @@ struct server_context { if (n_batch == 1 || ret < 0) { // if you get here, it means the KV cache is full - try increasing it via the context size LOG_TEE("%s : failed to decode the batch, n_batch = %d, ret = %d\n", __func__, n_batch, ret); - return false; + for (auto & slot : slots) { + slot.state = SLOT_STATE_PROCESSING; + slot.command = SLOT_COMMAND_NONE; + slot.release(); + send_error(slot, "Input prompt is too big compared to KV size. Please try increasing KV size."); + } + break; // break loop of n_batch } LOG_TEE("%s : failed to find free space in the KV cache, retrying with smaller n_batch = %d\n", __func__, n_batch / 2); @@ -2042,12 +2066,12 @@ struct server_context { n_batch /= 2; i -= n_batch; - continue; + continue; // continue loop of n_batch } for (auto & slot : slots) { if (slot.state != SLOT_STATE_PROCESSING || slot.i_batch < (int) i || slot.i_batch >= (int) (i + n_tokens)) { - continue; + continue; // continue loop of slots } // prompt evaluated for embedding @@ -2055,7 +2079,7 @@ struct server_context { send_embedding(slot, batch_view); slot.release(); slot.i_batch = -1; - continue; + continue; // continue loop of slots } completion_token_output result; @@ -2097,9 +2121,7 @@ struct server_context { } } - LOG_VERBOSE("slots updated", {}); - - return true; + LOG_VERBOSE("run slots completed", {}); } json model_meta() const { @@ -2745,32 +2767,32 @@ int main(int argc, char ** argv) { svr->set_logger(log_server_request); - svr->set_exception_handler([](const httplib::Request &, httplib::Response & res, std::exception_ptr ep) { - const char fmt[] = "500 Internal Server Error\n%s"; + auto res_error = [](httplib::Response & res, json error_data) { + json final_response {{"error", error_data}}; + res.set_content(final_response.dump(), "application/json; charset=utf-8"); + res.status = json_value(error_data, "code", 500); + }; - char buf[BUFSIZ]; + svr->set_exception_handler([&res_error](const httplib::Request &, httplib::Response & res, std::exception_ptr ep) { + std::string message; try { std::rethrow_exception(std::move(ep)); - } catch (std::exception &e) { - snprintf(buf, sizeof(buf), fmt, e.what()); + } catch (std::exception & e) { + message = e.what(); } catch (...) { - snprintf(buf, sizeof(buf), fmt, "Unknown Exception"); + message = "Unknown Exception"; } - res.set_content(buf, "text/plain; charset=utf-8"); - res.status = 500; + json formatted_error = format_error_response(message, ERROR_TYPE_SERVER); + LOG_VERBOSE("Got exception", formatted_error); + res_error(res, formatted_error); }); - svr->set_error_handler([](const httplib::Request &, httplib::Response & res) { - if (res.status == 401) { - res.set_content("Unauthorized", "text/plain; charset=utf-8"); - } - if (res.status == 400) { - res.set_content("Invalid request", "text/plain; charset=utf-8"); - } + svr->set_error_handler([&res_error](const httplib::Request &, httplib::Response & res) { if (res.status == 404) { - res.set_content("File Not Found", "text/plain; charset=utf-8"); + res_error(res, format_error_response("File Not Found", ERROR_TYPE_NOT_FOUND)); } + // for other error codes, we skip processing here because it's already done by res_error() }); // set timeouts and change hostname and port @@ -2835,7 +2857,7 @@ int main(int argc, char ** argv) { // Middlewares // - auto middleware_validate_api_key = [&sparams](const httplib::Request & req, httplib::Response & res) { + auto middleware_validate_api_key = [&sparams, &res_error](const httplib::Request & req, httplib::Response & res) { // TODO: should we apply API key to all endpoints, including "/health" and "/models"? static const std::set protected_endpoints = { "/props", @@ -2876,8 +2898,7 @@ int main(int argc, char ** argv) { // API key is invalid or not provided // TODO: make another middleware for CORS related logic res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin")); - res.set_content("Unauthorized: Invalid API Key", "text/plain; charset=utf-8"); - res.status = 401; // Unauthorized + res_error(res, format_error_response("Invalid API Key", ERROR_TYPE_AUTHENTICATION)); LOG_WARNING("Unauthorized: Invalid API Key", {}); @@ -2940,21 +2961,18 @@ int main(int argc, char ** argv) { } case SERVER_STATE_LOADING_MODEL: { - res.set_content(R"({"status": "loading model"})", "application/json"); - res.status = 503; // HTTP Service Unavailable + res_error(res, format_error_response("Loading model", ERROR_TYPE_UNAVAILABLE)); } break; case SERVER_STATE_ERROR: { - res.set_content(R"({"status": "error", "error": "Model failed to load"})", "application/json"); - res.status = 500; // HTTP Internal Server Error + res_error(res, format_error_response("Model failed to load", ERROR_TYPE_SERVER)); } break; } }; const auto handle_slots = [&](const httplib::Request &, httplib::Response & res) { if (!sparams.slots_endpoint) { - res.status = 501; - res.set_content("This server does not support slots endpoint.", "text/plain; charset=utf-8"); + res_error(res, format_error_response("This server does not support slots endpoint.", ERROR_TYPE_NOT_SUPPORTED)); return; } @@ -2978,8 +2996,7 @@ int main(int argc, char ** argv) { const auto handle_metrics = [&](const httplib::Request &, httplib::Response & res) { if (!sparams.metrics_endpoint) { - res.status = 501; - res.set_content("This server does not support metrics endpoint.", "text/plain; charset=utf-8"); + res_error(res, format_error_response("This server does not support metrics endpoint.", ERROR_TYPE_NOT_SUPPORTED)); return; } @@ -3090,7 +3107,7 @@ int main(int argc, char ** argv) { res.set_content(data.dump(), "application/json; charset=utf-8"); }; - const auto handle_completions = [&ctx_server](const httplib::Request & req, httplib::Response & res) { + const auto handle_completions = [&ctx_server, &res_error](const httplib::Request & req, httplib::Response & res) { res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin")); json data = json::parse(req.body); @@ -3105,8 +3122,7 @@ int main(int argc, char ** argv) { if (!result.error && result.stop) { res.set_content(result.data.dump(-1, ' ', false, json::error_handler_t::replace), "application/json; charset=utf-8"); } else { - res.status = 500; - res.set_content(result.data["content"], "text/plain; charset=utf-8"); + res_error(res, result.data); } ctx_server.queue_results.remove_waiting_task_id(id_task); @@ -3186,7 +3202,7 @@ int main(int argc, char ** argv) { res.set_content(models.dump(), "application/json; charset=utf-8"); }; - const auto handle_chat_completions = [&ctx_server, &sparams](const httplib::Request & req, httplib::Response & res) { + const auto handle_chat_completions = [&ctx_server, &sparams, &res_error](const httplib::Request & req, httplib::Response & res) { res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin")); json data = oaicompat_completion_params_parse(ctx_server.model, json::parse(req.body), sparams.chat_template); @@ -3204,8 +3220,7 @@ int main(int argc, char ** argv) { res.set_content(result_oai.dump(-1, ' ', false, json::error_handler_t::replace), "application/json; charset=utf-8"); } else { - res.status = 500; - res.set_content(result.data["content"], "text/plain; charset=utf-8"); + res_error(res, result.data); } ctx_server.queue_results.remove_waiting_task_id(id_task); } else { @@ -3259,7 +3274,7 @@ int main(int argc, char ** argv) { } }; - const auto handle_infill = [&ctx_server](const httplib::Request & req, httplib::Response & res) { + const auto handle_infill = [&ctx_server, &res_error](const httplib::Request & req, httplib::Response & res) { res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin")); json data = json::parse(req.body); @@ -3274,8 +3289,7 @@ int main(int argc, char ** argv) { if (!result.error && result.stop) { res.set_content(result.data.dump(-1, ' ', false, json::error_handler_t::replace), "application/json; charset=utf-8"); } else { - res.status = 404; - res.set_content(result.data["content"], "text/plain; charset=utf-8"); + res_error(res, result.data); } ctx_server.queue_results.remove_waiting_task_id(id_task); @@ -3346,7 +3360,7 @@ int main(int argc, char ** argv) { return res.set_content(data.dump(), "application/json; charset=utf-8"); }; - const auto handle_embeddings = [¶ms, &ctx_server](const httplib::Request & req, httplib::Response & res) { + const auto handle_embeddings = [¶ms, &ctx_server, &res_error](const httplib::Request & req, httplib::Response & res) { res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin")); if (!params.embedding) { res.status = 501; @@ -3375,8 +3389,8 @@ int main(int argc, char ** argv) { std::string content = body["content"]; prompts.push_back(content); } else { - // TODO @ngxson : should return an error here - prompts.push_back(""); + res_error(res, format_error_response("\"input\" or \"content\" must be provided", ERROR_TYPE_INVALID_REQUEST)); + return; } // process all prompts @@ -3392,9 +3406,14 @@ int main(int argc, char ** argv) { // get the result server_task_result result = ctx_server.queue_results.recv(id_task); ctx_server.queue_results.remove_waiting_task_id(id_task); - - // append to the responses - responses.push_back(result.data); + if (!result.error) { + // append to the responses + responses.push_back(result.data); + } else { + // error received, ignore everything else + res_error(res, result.data); + return; + } } // write JSON response @@ -3488,7 +3507,7 @@ int main(int argc, char ** argv) { &server_context::process_single_task, &ctx_server, std::placeholders::_1)); ctx_server.queue_tasks.on_finish_multitask(std::bind( &server_context::on_finish_multitask, &ctx_server, std::placeholders::_1)); - ctx_server.queue_tasks.on_run_slots(std::bind( + ctx_server.queue_tasks.on_update_slots(std::bind( &server_context::update_slots, &ctx_server)); ctx_server.queue_results.on_multitask_update(std::bind( &server_queue::update_multitask, diff --git a/examples/server/tests/features/steps/steps.py b/examples/server/tests/features/steps/steps.py index 4e81a255a..98c2b6174 100644 --- a/examples/server/tests/features/steps/steps.py +++ b/examples/server/tests/features/steps/steps.py @@ -801,7 +801,7 @@ async def oai_chat_completions(user_prompt, stream=enable_streaming, seed=seed ) - except openai.error.APIError as e: + except openai.error.AuthenticationError as e: if expect_api_error is not None and expect_api_error: return 401 else: diff --git a/examples/server/utils.hpp b/examples/server/utils.hpp index f27af81e9..48aeef4eb 100644 --- a/examples/server/utils.hpp +++ b/examples/server/utils.hpp @@ -14,6 +14,17 @@ using json = nlohmann::json; +// https://community.openai.com/t/openai-chat-list-of-error-codes-and-types/357791/11 +enum error_type { + ERROR_TYPE_INVALID_REQUEST, + ERROR_TYPE_AUTHENTICATION, + ERROR_TYPE_SERVER, + ERROR_TYPE_NOT_FOUND, + ERROR_TYPE_PERMISSION, + ERROR_TYPE_UNAVAILABLE, // custom error + ERROR_TYPE_NOT_SUPPORTED, // custom error +}; + extern bool server_verbose; extern bool server_log_json; @@ -542,3 +553,43 @@ static json format_detokenized_response(const std::string & content) { {"content", content} }; } + +static json format_error_response(const std::string & message, const enum error_type type) { + std::string type_str; + int code = 500; + switch (type) { + case ERROR_TYPE_INVALID_REQUEST: + type_str = "invalid_request_error"; + code = 400; + break; + case ERROR_TYPE_AUTHENTICATION: + type_str = "authentication_error"; + code = 401; + break; + case ERROR_TYPE_NOT_FOUND: + type_str = "not_found_error"; + code = 404; + break; + case ERROR_TYPE_SERVER: + type_str = "server_error"; + code = 500; + break; + case ERROR_TYPE_PERMISSION: + type_str = "permission_error"; + code = 403; + break; + case ERROR_TYPE_NOT_SUPPORTED: + type_str = "not_supported_error"; + code = 501; + break; + case ERROR_TYPE_UNAVAILABLE: + type_str = "unavailable_error"; + code = 503; + break; + } + return json { + {"code", code}, + {"message", message}, + {"type", type_str}, + }; +} From 828defefb66fc8a25404f5de845897145bf34061 Mon Sep 17 00:00:00 2001 From: Jakub N Date: Mon, 11 Mar 2024 14:40:42 +0100 Subject: [PATCH 597/812] Update server docker image URLs (#5997) --- examples/server/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/server/README.md b/examples/server/README.md index 376739051..8f8454aff 100644 --- a/examples/server/README.md +++ b/examples/server/README.md @@ -123,10 +123,10 @@ You can consume the endpoints with Postman or NodeJS with axios library. You can ### Docker ```bash -docker run -p 8080:8080 -v /path/to/models:/models ggerganov/llama.cpp:server -m models/7B/ggml-model.gguf -c 512 --host 0.0.0.0 --port 8080 +docker run -p 8080:8080 -v /path/to/models:/models ghcr.io/ggerganov/llama.cpp:server -m models/7B/ggml-model.gguf -c 512 --host 0.0.0.0 --port 8080 # or, with CUDA: -docker run -p 8080:8080 -v /path/to/models:/models --gpus all ggerganov/llama.cpp:server-cuda -m models/7B/ggml-model.gguf -c 512 --host 0.0.0.0 --port 8080 --n-gpu-layers 99 +docker run -p 8080:8080 -v /path/to/models:/models --gpus all ghcr.io/ggerganov/llama.cpp:server-cuda -m models/7B/ggml-model.gguf -c 512 --host 0.0.0.0 --port 8080 --n-gpu-layers 99 ``` ## Testing with CURL From 83796e62bc9f6caae6228168e359890f51e60fee Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Mon, 11 Mar 2024 17:47:47 +0200 Subject: [PATCH 598/812] llama : refactor unicode stuff (#5992) * llama : refactor unicode stuff ggml-ci * unicode : names * make : fix c++ compiler * unicode : names * unicode : straighten tables * zig : fix build * unicode : put nfd normalization behind API ggml-ci * swift : fix build * unicode : add BOM * unicode : add ggml-ci * unicode : pass as cpts as const ref --- CMakeLists.txt | 2 + Makefile | 7 +- Package.swift | 1 + build.zig | 15 +- llama.cpp | 87 +- tests/test-tokenizer-1-bpe.cpp | 4 +- tests/test-tokenizer-1-llama.cpp | 2 +- unicode.cpp | 1672 ++++++++++++++++++++++++++++++ unicode.h | 790 +------------- 9 files changed, 1744 insertions(+), 836 deletions(-) create mode 100644 unicode.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 60b592bf5..7ab13cbd5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1141,6 +1141,8 @@ endif() add_library(llama llama.cpp llama.h + unicode.h + unicode.cpp ) target_include_directories(llama PUBLIC .) diff --git a/Makefile b/Makefile index 6492da8a0..c8fd3f5c5 100644 --- a/Makefile +++ b/Makefile @@ -633,9 +633,12 @@ ggml-backend.o: ggml-backend.c ggml.h ggml-backend.h ggml-quants.o: ggml-quants.c ggml.h ggml-quants.h ggml-common.h $(CC) $(CFLAGS) -c $< -o $@ -OBJS += ggml-alloc.o ggml-backend.o ggml-quants.o +unicode.o: unicode.cpp unicode.h + $(CXX) $(CXXFLAGS) -c $< -o $@ -llama.o: llama.cpp ggml.h ggml-alloc.h ggml-backend.h ggml-cuda.h ggml-metal.h llama.h +OBJS += ggml-alloc.o ggml-backend.o ggml-quants.o unicode.o + +llama.o: llama.cpp unicode.h ggml.h ggml-alloc.h ggml-backend.h ggml-cuda.h ggml-metal.h llama.h $(CXX) $(CXXFLAGS) -c $< -o $@ COMMON_H_DEPS = common/common.h common/sampling.h common/log.h diff --git a/Package.swift b/Package.swift index b24c9204a..4696ae5e4 100644 --- a/Package.swift +++ b/Package.swift @@ -31,6 +31,7 @@ let package = Package( sources: [ "ggml.c", "llama.cpp", + "unicode.cpp", "ggml-alloc.c", "ggml-backend.c", "ggml-quants.c", diff --git a/build.zig b/build.zig index c0af454dc..90609359b 100644 --- a/build.zig +++ b/build.zig @@ -115,6 +115,7 @@ pub fn build(b: *std.build.Builder) !void { const ggml_alloc = make.obj("ggml-alloc", "ggml-alloc.c"); const ggml_backend = make.obj("ggml-backend", "ggml-backend.c"); const ggml_quants = make.obj("ggml-quants", "ggml-quants.c"); + const unicode = make.obj("unicode", "unicode.cpp"); const llama = make.obj("llama", "llama.cpp"); const buildinfo = make.obj("common", "common/build-info.cpp"); const common = make.obj("common", "common/common.cpp"); @@ -125,14 +126,14 @@ pub fn build(b: *std.build.Builder) !void { const clip = make.obj("clip", "examples/llava/clip.cpp"); const llava = make.obj("llava", "examples/llava/llava.cpp"); - _ = make.exe("main", "examples/main/main.cpp", &.{ ggml, ggml_alloc, ggml_backend, ggml_quants, llama, common, buildinfo, sampling, console, grammar_parser }); - _ = make.exe("quantize", "examples/quantize/quantize.cpp", &.{ ggml, ggml_alloc, ggml_backend, ggml_quants, llama, common, buildinfo }); - _ = make.exe("perplexity", "examples/perplexity/perplexity.cpp", &.{ ggml, ggml_alloc, ggml_backend, ggml_quants, llama, common, buildinfo }); - _ = make.exe("embedding", "examples/embedding/embedding.cpp", &.{ ggml, ggml_alloc, ggml_backend, ggml_quants, llama, common, buildinfo }); - _ = make.exe("finetune", "examples/finetune/finetune.cpp", &.{ ggml, ggml_alloc, ggml_backend, ggml_quants, llama, common, buildinfo, train }); - _ = make.exe("train-text-from-scratch", "examples/train-text-from-scratch/train-text-from-scratch.cpp", &.{ ggml, ggml_alloc, ggml_backend, ggml_quants, llama, common, buildinfo, train }); + _ = make.exe("main", "examples/main/main.cpp", &.{ ggml, ggml_alloc, ggml_backend, ggml_quants, llama, unicode, common, buildinfo, sampling, console, grammar_parser }); + _ = make.exe("quantize", "examples/quantize/quantize.cpp", &.{ ggml, ggml_alloc, ggml_backend, ggml_quants, llama, unicode, common, buildinfo }); + _ = make.exe("perplexity", "examples/perplexity/perplexity.cpp", &.{ ggml, ggml_alloc, ggml_backend, ggml_quants, llama, unicode, common, buildinfo }); + _ = make.exe("embedding", "examples/embedding/embedding.cpp", &.{ ggml, ggml_alloc, ggml_backend, ggml_quants, llama, unicode, common, buildinfo }); + _ = make.exe("finetune", "examples/finetune/finetune.cpp", &.{ ggml, ggml_alloc, ggml_backend, ggml_quants, llama, unicode, common, buildinfo, train }); + _ = make.exe("train-text-from-scratch", "examples/train-text-from-scratch/train-text-from-scratch.cpp", &.{ ggml, ggml_alloc, ggml_backend, ggml_quants, llama, unicode, common, buildinfo, train }); - const server = make.exe("server", "examples/server/server.cpp", &.{ ggml, ggml_alloc, ggml_backend, ggml_quants, llama, common, buildinfo, sampling, grammar_parser, clip, llava }); + const server = make.exe("server", "examples/server/server.cpp", &.{ ggml, ggml_alloc, ggml_backend, ggml_quants, llama, unicode, common, buildinfo, sampling, grammar_parser, clip, llava }); if (server.target.isWindows()) { server.linkSystemLibrary("ws2_32"); } diff --git a/llama.cpp b/llama.cpp index f61ea791b..6aa88ce54 100644 --- a/llama.cpp +++ b/llama.cpp @@ -3703,7 +3703,7 @@ static void llm_load_vocab( for (int i = 0; i < n_merges; i++) { const std::string word = gguf_get_arr_str(ctx, merges_keyidx, i); - GGML_ASSERT(codepoints_from_utf8(word).size() > 0); + GGML_ASSERT(unicode_cpts_from_utf8(word).size() > 0); std::string first; std::string second; @@ -3748,7 +3748,7 @@ static void llm_load_vocab( for (uint32_t i = 0; i < n_vocab; i++) { std::string word = gguf_get_arr_str(ctx, token_idx, i); - GGML_ASSERT(codepoints_from_utf8(word).size() > 0); + GGML_ASSERT(unicode_cpts_from_utf8(word).size() > 0); vocab.token_to_id[word] = i; @@ -9340,7 +9340,7 @@ static uint8_t llama_token_to_byte(const llama_vocab& vocab, llama_token id) { } case LLAMA_VOCAB_TYPE_BPE: { GGML_ASSERT(false); - return unicode_to_bytes_bpe(token_data.text); + return unicode_utf8_to_byte(token_data.text); } case LLAMA_VOCAB_TYPE_WPM: { GGML_ASSERT(false); @@ -9365,7 +9365,7 @@ static llama_token llama_byte_to_token(const llama_vocab & vocab, uint8_t ch) { } case LLAMA_VOCAB_TYPE_WPM: case LLAMA_VOCAB_TYPE_BPE: { - return vocab.token_to_id.at(bytes_to_unicode_bpe(ch)); + return vocab.token_to_id.at(unicode_byte_to_utf8(ch)); } default: GGML_ASSERT(false); @@ -9705,9 +9705,9 @@ private: bpe_words.reserve(text.size()); bpe_encoded_words.reserve(text.size()); - auto cps = codepoints_from_utf8(text); - for (size_t i = 0; i < cps.size(); ++i) - text_utf.emplace_back(codepoint_to_utf8(cps[i])); + const auto cpts = unicode_cpts_from_utf8(text); + for (size_t i = 0; i < cpts.size(); ++i) + text_utf.emplace_back(unicode_cpt_to_utf8(cpts[i])); for (int i = 0; i < (int)text_utf.size(); i++) { const std::string & utf_char = text_utf[i]; @@ -9757,40 +9757,40 @@ private: } if (!split_condition && !collecting) { - if (codepoint_type(utf_char) == CODEPOINT_TYPE_LETTER || (!token.size() && utf_char == " " && codepoint_type(utf_char_next) == CODEPOINT_TYPE_LETTER)) { + if (unicode_cpt_type(utf_char) == CODEPOINT_TYPE_LETTER || (!token.size() && utf_char == " " && unicode_cpt_type(utf_char_next) == CODEPOINT_TYPE_LETTER)) { collecting_letter = true; collecting = true; } - else if (codepoint_type(utf_char) == CODEPOINT_TYPE_DIGIT || (!token.size() && utf_char == " " && codepoint_type(utf_char_next) == CODEPOINT_TYPE_DIGIT)) { + else if (unicode_cpt_type(utf_char) == CODEPOINT_TYPE_DIGIT || (!token.size() && utf_char == " " && unicode_cpt_type(utf_char_next) == CODEPOINT_TYPE_DIGIT)) { collecting_numeric = true; collecting = true; } else if ( - ((codepoint_type(utf_char) != CODEPOINT_TYPE_LETTER && codepoint_type(utf_char) != CODEPOINT_TYPE_DIGIT) && (codepoint_type(utf_char) != CODEPOINT_TYPE_WHITESPACE)) || - (!token.size() && utf_char == " " && codepoint_type(utf_char_next) != CODEPOINT_TYPE_LETTER && codepoint_type(utf_char_next) != CODEPOINT_TYPE_DIGIT && codepoint_type(utf_char_next) != CODEPOINT_TYPE_WHITESPACE) + ((unicode_cpt_type(utf_char) != CODEPOINT_TYPE_LETTER && unicode_cpt_type(utf_char) != CODEPOINT_TYPE_DIGIT) && (unicode_cpt_type(utf_char) != CODEPOINT_TYPE_WHITESPACE)) || + (!token.size() && utf_char == " " && unicode_cpt_type(utf_char_next) != CODEPOINT_TYPE_LETTER && unicode_cpt_type(utf_char_next) != CODEPOINT_TYPE_DIGIT && unicode_cpt_type(utf_char_next) != CODEPOINT_TYPE_WHITESPACE) ) { collecting_special = true; collecting = true; } - else if (codepoint_type(utf_char) == CODEPOINT_TYPE_WHITESPACE && codepoint_type(utf_char_next) == CODEPOINT_TYPE_WHITESPACE) { + else if (unicode_cpt_type(utf_char) == CODEPOINT_TYPE_WHITESPACE && unicode_cpt_type(utf_char_next) == CODEPOINT_TYPE_WHITESPACE) { collecting_whitespace_lookahead = true; collecting = true; } - else if (codepoint_type(utf_char) == CODEPOINT_TYPE_WHITESPACE) { + else if (unicode_cpt_type(utf_char) == CODEPOINT_TYPE_WHITESPACE) { split_condition = true; } } else if (!split_condition && collecting) { - if (collecting_letter && codepoint_type(utf_char) != CODEPOINT_TYPE_LETTER) { + if (collecting_letter && unicode_cpt_type(utf_char) != CODEPOINT_TYPE_LETTER) { split_condition = true; } - else if (collecting_numeric && codepoint_type(utf_char) != CODEPOINT_TYPE_DIGIT) { + else if (collecting_numeric && unicode_cpt_type(utf_char) != CODEPOINT_TYPE_DIGIT) { split_condition = true; } - else if (collecting_special && (codepoint_type(utf_char) == CODEPOINT_TYPE_LETTER || codepoint_type(utf_char) == CODEPOINT_TYPE_DIGIT || codepoint_type(utf_char) == CODEPOINT_TYPE_WHITESPACE)) { + else if (collecting_special && (unicode_cpt_type(utf_char) == CODEPOINT_TYPE_LETTER || unicode_cpt_type(utf_char) == CODEPOINT_TYPE_DIGIT || unicode_cpt_type(utf_char) == CODEPOINT_TYPE_WHITESPACE)) { split_condition = true; } - else if (collecting_whitespace_lookahead && (codepoint_type(utf_char_next) == CODEPOINT_TYPE_LETTER || codepoint_type(utf_char_next) == CODEPOINT_TYPE_DIGIT)) { + else if (collecting_whitespace_lookahead && (unicode_cpt_type(utf_char_next) == CODEPOINT_TYPE_LETTER || unicode_cpt_type(utf_char_next) == CODEPOINT_TYPE_DIGIT)) { split_condition = true; } } @@ -9819,7 +9819,7 @@ private: for (std::string & word : bpe_words) { std::string encoded_token = ""; for (char & c : word) { - encoded_token += bytes_to_unicode_bpe(c); + encoded_token += unicode_byte_to_utf8(c); } bpe_encoded_words.emplace_back(encoded_token); } @@ -9893,25 +9893,13 @@ struct llm_tokenizer_wpm { } std::vector preprocess(const std::string & text) { - // normalalization form D - std::vector codepoints = codepoints_from_utf8(text); - std::vector nfd_codepoints; - for (uint32_t code : codepoints) { - auto it = nfd_map.equal_range(code); - if (it.first != it.second) { - for (auto jt = it.first; jt != it.second; jt++) { - nfd_codepoints.push_back(jt->second); - } - } else { - nfd_codepoints.push_back(code); - } - } + std::vector cpts_nfd = unicode_cpts_normalize_nfd(unicode_cpts_from_utf8(text)); // strip accents, strip control, uniformize whitespace, // to lowercase, pad chinese characters, pad punctuation std::string new_str = ""; - for (uint32_t code : nfd_codepoints) { - int type = codepoint_type(code); + for (uint32_t code : cpts_nfd) { + int type = unicode_cpt_type(code); if (type == CODEPOINT_TYPE_ACCENT_MARK || type == CODEPOINT_TYPE_CONTROL) { continue; } @@ -9919,7 +9907,7 @@ struct llm_tokenizer_wpm { if (type == CODEPOINT_TYPE_WHITESPACE) { code = ' '; } - std::string s = codepoint_to_utf8(code); + std::string s = unicode_cpt_to_utf8(code); if (type == CODEPOINT_TYPE_PUNCTUATION || is_ascii_punct(code) || is_chinese_char(code)) { new_str += " "; new_str += s; @@ -9939,8 +9927,7 @@ struct llm_tokenizer_wpm { if (r > l) words.push_back(new_str.substr(l, (r - l))); l = r + 1; r = l; - } - else { + } else { r += 1; } } @@ -9964,17 +9951,17 @@ struct llm_tokenizer_wpm { return code < 256 && ispunct(code); } - bool is_chinese_char(uint32_t codepoint) { - if ((codepoint >= 0x4E00 && codepoint <= 0x9FFF) || - (codepoint >= 0x3400 && codepoint <= 0x4DBF) || - (codepoint >= 0x20000 && codepoint <= 0x2A6DF) || - (codepoint >= 0x2A700 && codepoint <= 0x2B73F) || - (codepoint >= 0x2B740 && codepoint <= 0x2B81F) || - (codepoint >= 0x2B920 && codepoint <= 0x2CEAF) || // this should be 0x2B820 but in hf rust code it is 0x2B920 - (codepoint >= 0xF900 && codepoint <= 0xFAFF) || - (codepoint >= 0x2F800 && codepoint <= 0x2FA1F) || - (codepoint >= 0x3000 && codepoint <= 0x303F) || - (codepoint >= 0xFF00 && codepoint <= 0xFFEF)) { + bool is_chinese_char(uint32_t cpt) { + if ((cpt >= 0x4E00 && cpt <= 0x9FFF) || + (cpt >= 0x3400 && cpt <= 0x4DBF) || + (cpt >= 0x20000 && cpt <= 0x2A6DF) || + (cpt >= 0x2A700 && cpt <= 0x2B73F) || + (cpt >= 0x2B740 && cpt <= 0x2B81F) || + (cpt >= 0x2B920 && cpt <= 0x2CEAF) || // this should be 0x2B820 but in hf rust code it is 0x2B920 + (cpt >= 0xF900 && cpt <= 0xFAFF) || + (cpt >= 0x2F800 && cpt <= 0x2FA1F) || + (cpt >= 0x3000 && cpt <= 0x303F) || + (cpt >= 0xFF00 && cpt <= 0xFFEF)) { return true; // NOLINT } return false; @@ -13953,9 +13940,9 @@ int32_t llama_tokenize( static std::string llama_decode_text(const std::string & text) { std::string decoded_text; - auto unicode_sequences = codepoints_from_utf8(text); - for (auto& unicode_sequence : unicode_sequences) { - decoded_text += unicode_to_bytes_bpe(codepoint_to_utf8(unicode_sequence)); + auto unicode_sequences = unicode_cpts_from_utf8(text); + for (auto & unicode_sequence : unicode_sequences) { + decoded_text += unicode_utf8_to_byte(unicode_cpt_to_utf8(unicode_sequence)); } return decoded_text; diff --git a/tests/test-tokenizer-1-bpe.cpp b/tests/test-tokenizer-1-bpe.cpp index 3596ce55a..a0e2caf94 100644 --- a/tests/test-tokenizer-1-bpe.cpp +++ b/tests/test-tokenizer-1-bpe.cpp @@ -64,7 +64,7 @@ int main(int argc, char **argv) { for (int i = 0; i < n_vocab; ++i) { std::string str = llama_detokenize_bpe(ctx, std::vector(1, i)); try { - auto cps = codepoints_from_utf8(str); + auto cps = unicode_cpts_from_utf8(str); std::vector tokens = llama_tokenize(ctx, str, false); std::string check = llama_detokenize_bpe(ctx, tokens); if (check != str) { @@ -97,7 +97,7 @@ int main(int argc, char **argv) { continue; } - std::string str = codepoint_to_utf8(cp); + std::string str = unicode_cpt_to_utf8(cp); std::vector tokens = llama_tokenize(ctx, str, false); std::string check = llama_detokenize_bpe(ctx, tokens); if (cp != 9601 && str != check) { diff --git a/tests/test-tokenizer-1-llama.cpp b/tests/test-tokenizer-1-llama.cpp index 9333f8686..8caf0b24e 100644 --- a/tests/test-tokenizer-1-llama.cpp +++ b/tests/test-tokenizer-1-llama.cpp @@ -85,7 +85,7 @@ int main(int argc, char **argv) { continue; } - std::string str = codepoint_to_utf8(cp); + std::string str = unicode_cpt_to_utf8(cp); std::vector tokens = llama_tokenize(ctx, str, false); std::string check = llama_detokenize_spm(ctx, tokens); if (cp != 9601 && str != check) { diff --git a/unicode.cpp b/unicode.cpp new file mode 100644 index 000000000..7fce6fb34 --- /dev/null +++ b/unicode.cpp @@ -0,0 +1,1672 @@ +#include "unicode.h" + +#include +#include +#include +#include +#include +#include + +static const std::vector> unicode_ranges_digit = { +{0x00000030, 0x00000039}, {0x000000B2, 0x000000B3}, {0x000000B9, 0x000000B9}, {0x00000660, 0x00000669}, +{0x000006F0, 0x000006F9}, {0x000007C0, 0x000007C9}, {0x00000966, 0x0000096F}, {0x000009E6, 0x000009EF}, +{0x00000A66, 0x00000A6F}, {0x00000AE6, 0x00000AEF}, {0x00000B66, 0x00000B6F}, {0x00000BE6, 0x00000BEF}, +{0x00000C66, 0x00000C6F}, {0x00000CE6, 0x00000CEF}, {0x00000D66, 0x00000D6F}, {0x00000DE6, 0x00000DEF}, +{0x00000E50, 0x00000E59}, {0x00000ED0, 0x00000ED9}, {0x00000F20, 0x00000F29}, {0x00001040, 0x00001049}, +{0x00001090, 0x00001099}, {0x00001369, 0x00001371}, {0x000017E0, 0x000017E9}, {0x00001810, 0x00001819}, +{0x00001946, 0x0000194F}, {0x000019D0, 0x000019DA}, {0x00001A80, 0x00001A89}, {0x00001A90, 0x00001A99}, +{0x00001B50, 0x00001B59}, {0x00001BB0, 0x00001BB9}, {0x00001C40, 0x00001C49}, {0x00001C50, 0x00001C59}, +{0x00002070, 0x00002070}, {0x00002074, 0x00002079}, {0x00002080, 0x00002089}, {0x00002460, 0x00002468}, +{0x00002474, 0x0000247C}, {0x00002488, 0x00002490}, {0x000024EA, 0x000024EA}, {0x000024F5, 0x000024FD}, +{0x000024FF, 0x000024FF}, {0x00002776, 0x0000277E}, {0x00002780, 0x00002788}, {0x0000278A, 0x00002792}, +{0x0000A620, 0x0000A629}, {0x0000A8D0, 0x0000A8D9}, {0x0000A900, 0x0000A909}, {0x0000A9D0, 0x0000A9D9}, +{0x0000A9F0, 0x0000A9F9}, {0x0000AA50, 0x0000AA59}, {0x0000ABF0, 0x0000ABF9}, {0x0000FF10, 0x0000FF19}, +{0x000104A0, 0x000104A9}, {0x00010A40, 0x00010A43}, {0x00010D30, 0x00010D39}, {0x00010E60, 0x00010E68}, +{0x00011052, 0x0001105A}, {0x00011066, 0x0001106F}, {0x000110F0, 0x000110F9}, {0x00011136, 0x0001113F}, +{0x000111D0, 0x000111D9}, {0x000112F0, 0x000112F9}, {0x00011450, 0x00011459}, {0x000114D0, 0x000114D9}, +{0x00011650, 0x00011659}, {0x000116C0, 0x000116C9}, {0x00011730, 0x00011739}, {0x000118E0, 0x000118E9}, +{0x00011950, 0x00011959}, {0x00011C50, 0x00011C59}, {0x00011D50, 0x00011D59}, {0x00011DA0, 0x00011DA9}, +{0x00016A60, 0x00016A69}, {0x00016B50, 0x00016B59}, {0x0001D7CE, 0x0001D7FF}, {0x0001E140, 0x0001E149}, +{0x0001E2F0, 0x0001E2F9}, {0x0001E950, 0x0001E959}, {0x0001F100, 0x0001F10A}, {0x0001FBF0, 0x0001FBF9}, +}; + +static const std::vector> unicode_ranges_letter = { +{0x00000041, 0x0000005A}, {0x00000061, 0x0000007A}, {0x000000AA, 0x000000AA}, {0x000000B5, 0x000000B5}, +{0x000000BA, 0x000000BA}, {0x000000C0, 0x000000D6}, {0x000000D8, 0x000000F6}, {0x000000F8, 0x000002C1}, +{0x000002C6, 0x000002D1}, {0x000002E0, 0x000002E4}, {0x000002EC, 0x000002EC}, {0x000002EE, 0x000002EE}, +{0x00000370, 0x00000374}, {0x00000376, 0x00000377}, {0x0000037A, 0x0000037D}, {0x0000037F, 0x0000037F}, +{0x00000386, 0x00000386}, {0x00000388, 0x0000038A}, {0x0000038C, 0x0000038C}, {0x0000038E, 0x000003A1}, +{0x000003A3, 0x000003F5}, {0x000003F7, 0x00000481}, {0x0000048A, 0x0000052F}, {0x00000531, 0x00000556}, +{0x00000559, 0x00000559}, {0x00000560, 0x00000588}, {0x000005D0, 0x000005EA}, {0x000005EF, 0x000005F2}, +{0x00000620, 0x0000064A}, {0x0000066E, 0x0000066F}, {0x00000671, 0x000006D3}, {0x000006D5, 0x000006D5}, +{0x000006E5, 0x000006E6}, {0x000006EE, 0x000006EF}, {0x000006FA, 0x000006FC}, {0x000006FF, 0x000006FF}, +{0x00000710, 0x00000710}, {0x00000712, 0x0000072F}, {0x0000074D, 0x000007A5}, {0x000007B1, 0x000007B1}, +{0x000007CA, 0x000007EA}, {0x000007F4, 0x000007F5}, {0x000007FA, 0x000007FA}, {0x00000800, 0x00000815}, +{0x0000081A, 0x0000081A}, {0x00000824, 0x00000824}, {0x00000828, 0x00000828}, {0x00000840, 0x00000858}, +{0x00000860, 0x0000086A}, {0x000008A0, 0x000008B4}, {0x000008B6, 0x000008C7}, {0x00000904, 0x00000939}, +{0x0000093D, 0x0000093D}, {0x00000950, 0x00000950}, {0x00000958, 0x00000961}, {0x00000971, 0x00000980}, +{0x00000985, 0x0000098C}, {0x0000098F, 0x00000990}, {0x00000993, 0x000009A8}, {0x000009AA, 0x000009B0}, +{0x000009B2, 0x000009B2}, {0x000009B6, 0x000009B9}, {0x000009BD, 0x000009BD}, {0x000009CE, 0x000009CE}, +{0x000009DC, 0x000009DD}, {0x000009DF, 0x000009E1}, {0x000009F0, 0x000009F1}, {0x000009FC, 0x000009FC}, +{0x00000A05, 0x00000A0A}, {0x00000A0F, 0x00000A10}, {0x00000A13, 0x00000A28}, {0x00000A2A, 0x00000A30}, +{0x00000A32, 0x00000A33}, {0x00000A35, 0x00000A36}, {0x00000A38, 0x00000A39}, {0x00000A59, 0x00000A5C}, +{0x00000A5E, 0x00000A5E}, {0x00000A72, 0x00000A74}, {0x00000A85, 0x00000A8D}, {0x00000A8F, 0x00000A91}, +{0x00000A93, 0x00000AA8}, {0x00000AAA, 0x00000AB0}, {0x00000AB2, 0x00000AB3}, {0x00000AB5, 0x00000AB9}, +{0x00000ABD, 0x00000ABD}, {0x00000AD0, 0x00000AD0}, {0x00000AE0, 0x00000AE1}, {0x00000AF9, 0x00000AF9}, +{0x00000B05, 0x00000B0C}, {0x00000B0F, 0x00000B10}, {0x00000B13, 0x00000B28}, {0x00000B2A, 0x00000B30}, +{0x00000B32, 0x00000B33}, {0x00000B35, 0x00000B39}, {0x00000B3D, 0x00000B3D}, {0x00000B5C, 0x00000B5D}, +{0x00000B5F, 0x00000B61}, {0x00000B71, 0x00000B71}, {0x00000B83, 0x00000B83}, {0x00000B85, 0x00000B8A}, +{0x00000B8E, 0x00000B90}, {0x00000B92, 0x00000B95}, {0x00000B99, 0x00000B9A}, {0x00000B9C, 0x00000B9C}, +{0x00000B9E, 0x00000B9F}, {0x00000BA3, 0x00000BA4}, {0x00000BA8, 0x00000BAA}, {0x00000BAE, 0x00000BB9}, +{0x00000BD0, 0x00000BD0}, {0x00000C05, 0x00000C0C}, {0x00000C0E, 0x00000C10}, {0x00000C12, 0x00000C28}, +{0x00000C2A, 0x00000C39}, {0x00000C3D, 0x00000C3D}, {0x00000C58, 0x00000C5A}, {0x00000C60, 0x00000C61}, +{0x00000C80, 0x00000C80}, {0x00000C85, 0x00000C8C}, {0x00000C8E, 0x00000C90}, {0x00000C92, 0x00000CA8}, +{0x00000CAA, 0x00000CB3}, {0x00000CB5, 0x00000CB9}, {0x00000CBD, 0x00000CBD}, {0x00000CDE, 0x00000CDE}, +{0x00000CE0, 0x00000CE1}, {0x00000CF1, 0x00000CF2}, {0x00000D04, 0x00000D0C}, {0x00000D0E, 0x00000D10}, +{0x00000D12, 0x00000D3A}, {0x00000D3D, 0x00000D3D}, {0x00000D4E, 0x00000D4E}, {0x00000D54, 0x00000D56}, +{0x00000D5F, 0x00000D61}, {0x00000D7A, 0x00000D7F}, {0x00000D85, 0x00000D96}, {0x00000D9A, 0x00000DB1}, +{0x00000DB3, 0x00000DBB}, {0x00000DBD, 0x00000DBD}, {0x00000DC0, 0x00000DC6}, {0x00000E01, 0x00000E30}, +{0x00000E32, 0x00000E33}, {0x00000E40, 0x00000E46}, {0x00000E81, 0x00000E82}, {0x00000E84, 0x00000E84}, +{0x00000E86, 0x00000E8A}, {0x00000E8C, 0x00000EA3}, {0x00000EA5, 0x00000EA5}, {0x00000EA7, 0x00000EB0}, +{0x00000EB2, 0x00000EB3}, {0x00000EBD, 0x00000EBD}, {0x00000EC0, 0x00000EC4}, {0x00000EC6, 0x00000EC6}, +{0x00000EDC, 0x00000EDF}, {0x00000F00, 0x00000F00}, {0x00000F40, 0x00000F47}, {0x00000F49, 0x00000F6C}, +{0x00000F88, 0x00000F8C}, {0x00001000, 0x0000102A}, {0x0000103F, 0x0000103F}, {0x00001050, 0x00001055}, +{0x0000105A, 0x0000105D}, {0x00001061, 0x00001061}, {0x00001065, 0x00001066}, {0x0000106E, 0x00001070}, +{0x00001075, 0x00001081}, {0x0000108E, 0x0000108E}, {0x000010A0, 0x000010C5}, {0x000010C7, 0x000010C7}, +{0x000010CD, 0x000010CD}, {0x000010D0, 0x000010FA}, {0x000010FC, 0x00001248}, {0x0000124A, 0x0000124D}, +{0x00001250, 0x00001256}, {0x00001258, 0x00001258}, {0x0000125A, 0x0000125D}, {0x00001260, 0x00001288}, +{0x0000128A, 0x0000128D}, {0x00001290, 0x000012B0}, {0x000012B2, 0x000012B5}, {0x000012B8, 0x000012BE}, +{0x000012C0, 0x000012C0}, {0x000012C2, 0x000012C5}, {0x000012C8, 0x000012D6}, {0x000012D8, 0x00001310}, +{0x00001312, 0x00001315}, {0x00001318, 0x0000135A}, {0x00001380, 0x0000138F}, {0x000013A0, 0x000013F5}, +{0x000013F8, 0x000013FD}, {0x00001401, 0x0000166C}, {0x0000166F, 0x0000167F}, {0x00001681, 0x0000169A}, +{0x000016A0, 0x000016EA}, {0x000016F1, 0x000016F8}, {0x00001700, 0x0000170C}, {0x0000170E, 0x00001711}, +{0x00001720, 0x00001731}, {0x00001740, 0x00001751}, {0x00001760, 0x0000176C}, {0x0000176E, 0x00001770}, +{0x00001780, 0x000017B3}, {0x000017D7, 0x000017D7}, {0x000017DC, 0x000017DC}, {0x00001820, 0x00001878}, +{0x00001880, 0x00001884}, {0x00001887, 0x000018A8}, {0x000018AA, 0x000018AA}, {0x000018B0, 0x000018F5}, +{0x00001900, 0x0000191E}, {0x00001950, 0x0000196D}, {0x00001970, 0x00001974}, {0x00001980, 0x000019AB}, +{0x000019B0, 0x000019C9}, {0x00001A00, 0x00001A16}, {0x00001A20, 0x00001A54}, {0x00001AA7, 0x00001AA7}, +{0x00001B05, 0x00001B33}, {0x00001B45, 0x00001B4B}, {0x00001B83, 0x00001BA0}, {0x00001BAE, 0x00001BAF}, +{0x00001BBA, 0x00001BE5}, {0x00001C00, 0x00001C23}, {0x00001C4D, 0x00001C4F}, {0x00001C5A, 0x00001C7D}, +{0x00001C80, 0x00001C88}, {0x00001C90, 0x00001CBA}, {0x00001CBD, 0x00001CBF}, {0x00001CE9, 0x00001CEC}, +{0x00001CEE, 0x00001CF3}, {0x00001CF5, 0x00001CF6}, {0x00001CFA, 0x00001CFA}, {0x00001D00, 0x00001DBF}, +{0x00001E00, 0x00001F15}, {0x00001F18, 0x00001F1D}, {0x00001F20, 0x00001F45}, {0x00001F48, 0x00001F4D}, +{0x00001F50, 0x00001F57}, {0x00001F59, 0x00001F59}, {0x00001F5B, 0x00001F5B}, {0x00001F5D, 0x00001F5D}, +{0x00001F5F, 0x00001F7D}, {0x00001F80, 0x00001FB4}, {0x00001FB6, 0x00001FBC}, {0x00001FBE, 0x00001FBE}, +{0x00001FC2, 0x00001FC4}, {0x00001FC6, 0x00001FCC}, {0x00001FD0, 0x00001FD3}, {0x00001FD6, 0x00001FDB}, +{0x00001FE0, 0x00001FEC}, {0x00001FF2, 0x00001FF4}, {0x00001FF6, 0x00001FFC}, {0x00002071, 0x00002071}, +{0x0000207F, 0x0000207F}, {0x00002090, 0x0000209C}, {0x00002102, 0x00002102}, {0x00002107, 0x00002107}, +{0x0000210A, 0x00002113}, {0x00002115, 0x00002115}, {0x00002119, 0x0000211D}, {0x00002124, 0x00002124}, +{0x00002126, 0x00002126}, {0x00002128, 0x00002128}, {0x0000212A, 0x0000212D}, {0x0000212F, 0x00002139}, +{0x0000213C, 0x0000213F}, {0x00002145, 0x00002149}, {0x0000214E, 0x0000214E}, {0x00002183, 0x00002184}, +{0x00002C00, 0x00002C2E}, {0x00002C30, 0x00002C5E}, {0x00002C60, 0x00002CE4}, {0x00002CEB, 0x00002CEE}, +{0x00002CF2, 0x00002CF3}, {0x00002D00, 0x00002D25}, {0x00002D27, 0x00002D27}, {0x00002D2D, 0x00002D2D}, +{0x00002D30, 0x00002D67}, {0x00002D6F, 0x00002D6F}, {0x00002D80, 0x00002D96}, {0x00002DA0, 0x00002DA6}, +{0x00002DA8, 0x00002DAE}, {0x00002DB0, 0x00002DB6}, {0x00002DB8, 0x00002DBE}, {0x00002DC0, 0x00002DC6}, +{0x00002DC8, 0x00002DCE}, {0x00002DD0, 0x00002DD6}, {0x00002DD8, 0x00002DDE}, {0x00002E2F, 0x00002E2F}, +{0x00003005, 0x00003006}, {0x00003031, 0x00003035}, {0x0000303B, 0x0000303C}, {0x00003041, 0x00003096}, +{0x0000309D, 0x0000309F}, {0x000030A1, 0x000030FA}, {0x000030FC, 0x000030FF}, {0x00003105, 0x0000312F}, +{0x00003131, 0x0000318E}, {0x000031A0, 0x000031BF}, {0x000031F0, 0x000031FF}, {0x00003400, 0x00004DBF}, +{0x00004E00, 0x00009FFC}, {0x0000A000, 0x0000A48C}, {0x0000A4D0, 0x0000A4FD}, {0x0000A500, 0x0000A60C}, +{0x0000A610, 0x0000A61F}, {0x0000A62A, 0x0000A62B}, {0x0000A640, 0x0000A66E}, {0x0000A67F, 0x0000A69D}, +{0x0000A6A0, 0x0000A6E5}, {0x0000A717, 0x0000A71F}, {0x0000A722, 0x0000A788}, {0x0000A78B, 0x0000A7BF}, +{0x0000A7C2, 0x0000A7CA}, {0x0000A7F5, 0x0000A801}, {0x0000A803, 0x0000A805}, {0x0000A807, 0x0000A80A}, +{0x0000A80C, 0x0000A822}, {0x0000A840, 0x0000A873}, {0x0000A882, 0x0000A8B3}, {0x0000A8F2, 0x0000A8F7}, +{0x0000A8FB, 0x0000A8FB}, {0x0000A8FD, 0x0000A8FE}, {0x0000A90A, 0x0000A925}, {0x0000A930, 0x0000A946}, +{0x0000A960, 0x0000A97C}, {0x0000A984, 0x0000A9B2}, {0x0000A9CF, 0x0000A9CF}, {0x0000A9E0, 0x0000A9E4}, +{0x0000A9E6, 0x0000A9EF}, {0x0000A9FA, 0x0000A9FE}, {0x0000AA00, 0x0000AA28}, {0x0000AA40, 0x0000AA42}, +{0x0000AA44, 0x0000AA4B}, {0x0000AA60, 0x0000AA76}, {0x0000AA7A, 0x0000AA7A}, {0x0000AA7E, 0x0000AAAF}, +{0x0000AAB1, 0x0000AAB1}, {0x0000AAB5, 0x0000AAB6}, {0x0000AAB9, 0x0000AABD}, {0x0000AAC0, 0x0000AAC0}, +{0x0000AAC2, 0x0000AAC2}, {0x0000AADB, 0x0000AADD}, {0x0000AAE0, 0x0000AAEA}, {0x0000AAF2, 0x0000AAF4}, +{0x0000AB01, 0x0000AB06}, {0x0000AB09, 0x0000AB0E}, {0x0000AB11, 0x0000AB16}, {0x0000AB20, 0x0000AB26}, +{0x0000AB28, 0x0000AB2E}, {0x0000AB30, 0x0000AB5A}, {0x0000AB5C, 0x0000AB69}, {0x0000AB70, 0x0000ABE2}, +{0x0000AC00, 0x0000D7A3}, {0x0000D7B0, 0x0000D7C6}, {0x0000D7CB, 0x0000D7FB}, {0x0000F900, 0x0000FA6D}, +{0x0000FA70, 0x0000FAD9}, {0x0000FB00, 0x0000FB06}, {0x0000FB13, 0x0000FB17}, {0x0000FB1D, 0x0000FB1D}, +{0x0000FB1F, 0x0000FB28}, {0x0000FB2A, 0x0000FB36}, {0x0000FB38, 0x0000FB3C}, {0x0000FB3E, 0x0000FB3E}, +{0x0000FB40, 0x0000FB41}, {0x0000FB43, 0x0000FB44}, {0x0000FB46, 0x0000FBB1}, {0x0000FBD3, 0x0000FD3D}, +{0x0000FD50, 0x0000FD8F}, {0x0000FD92, 0x0000FDC7}, {0x0000FDF0, 0x0000FDFB}, {0x0000FE70, 0x0000FE74}, +{0x0000FE76, 0x0000FEFC}, {0x0000FF21, 0x0000FF3A}, {0x0000FF41, 0x0000FF5A}, {0x0000FF66, 0x0000FFBE}, +{0x0000FFC2, 0x0000FFC7}, {0x0000FFCA, 0x0000FFCF}, {0x0000FFD2, 0x0000FFD7}, {0x0000FFDA, 0x0000FFDC}, +{0x00010000, 0x0001000B}, {0x0001000D, 0x00010026}, {0x00010028, 0x0001003A}, {0x0001003C, 0x0001003D}, +{0x0001003F, 0x0001004D}, {0x00010050, 0x0001005D}, {0x00010080, 0x000100FA}, {0x00010280, 0x0001029C}, +{0x000102A0, 0x000102D0}, {0x00010300, 0x0001031F}, {0x0001032D, 0x00010340}, {0x00010342, 0x00010349}, +{0x00010350, 0x00010375}, {0x00010380, 0x0001039D}, {0x000103A0, 0x000103C3}, {0x000103C8, 0x000103CF}, +{0x00010400, 0x0001049D}, {0x000104B0, 0x000104D3}, {0x000104D8, 0x000104FB}, {0x00010500, 0x00010527}, +{0x00010530, 0x00010563}, {0x00010600, 0x00010736}, {0x00010740, 0x00010755}, {0x00010760, 0x00010767}, +{0x00010800, 0x00010805}, {0x00010808, 0x00010808}, {0x0001080A, 0x00010835}, {0x00010837, 0x00010838}, +{0x0001083C, 0x0001083C}, {0x0001083F, 0x00010855}, {0x00010860, 0x00010876}, {0x00010880, 0x0001089E}, +{0x000108E0, 0x000108F2}, {0x000108F4, 0x000108F5}, {0x00010900, 0x00010915}, {0x00010920, 0x00010939}, +{0x00010980, 0x000109B7}, {0x000109BE, 0x000109BF}, {0x00010A00, 0x00010A00}, {0x00010A10, 0x00010A13}, +{0x00010A15, 0x00010A17}, {0x00010A19, 0x00010A35}, {0x00010A60, 0x00010A7C}, {0x00010A80, 0x00010A9C}, +{0x00010AC0, 0x00010AC7}, {0x00010AC9, 0x00010AE4}, {0x00010B00, 0x00010B35}, {0x00010B40, 0x00010B55}, +{0x00010B60, 0x00010B72}, {0x00010B80, 0x00010B91}, {0x00010C00, 0x00010C48}, {0x00010C80, 0x00010CB2}, +{0x00010CC0, 0x00010CF2}, {0x00010D00, 0x00010D23}, {0x00010E80, 0x00010EA9}, {0x00010EB0, 0x00010EB1}, +{0x00010F00, 0x00010F1C}, {0x00010F27, 0x00010F27}, {0x00010F30, 0x00010F45}, {0x00010FB0, 0x00010FC4}, +{0x00010FE0, 0x00010FF6}, {0x00011003, 0x00011037}, {0x00011083, 0x000110AF}, {0x000110D0, 0x000110E8}, +{0x00011103, 0x00011126}, {0x00011144, 0x00011144}, {0x00011147, 0x00011147}, {0x00011150, 0x00011172}, +{0x00011176, 0x00011176}, {0x00011183, 0x000111B2}, {0x000111C1, 0x000111C4}, {0x000111DA, 0x000111DA}, +{0x000111DC, 0x000111DC}, {0x00011200, 0x00011211}, {0x00011213, 0x0001122B}, {0x00011280, 0x00011286}, +{0x00011288, 0x00011288}, {0x0001128A, 0x0001128D}, {0x0001128F, 0x0001129D}, {0x0001129F, 0x000112A8}, +{0x000112B0, 0x000112DE}, {0x00011305, 0x0001130C}, {0x0001130F, 0x00011310}, {0x00011313, 0x00011328}, +{0x0001132A, 0x00011330}, {0x00011332, 0x00011333}, {0x00011335, 0x00011339}, {0x0001133D, 0x0001133D}, +{0x00011350, 0x00011350}, {0x0001135D, 0x00011361}, {0x00011400, 0x00011434}, {0x00011447, 0x0001144A}, +{0x0001145F, 0x00011461}, {0x00011480, 0x000114AF}, {0x000114C4, 0x000114C5}, {0x000114C7, 0x000114C7}, +{0x00011580, 0x000115AE}, {0x000115D8, 0x000115DB}, {0x00011600, 0x0001162F}, {0x00011644, 0x00011644}, +{0x00011680, 0x000116AA}, {0x000116B8, 0x000116B8}, {0x00011700, 0x0001171A}, {0x00011800, 0x0001182B}, +{0x000118A0, 0x000118DF}, {0x000118FF, 0x00011906}, {0x00011909, 0x00011909}, {0x0001190C, 0x00011913}, +{0x00011915, 0x00011916}, {0x00011918, 0x0001192F}, {0x0001193F, 0x0001193F}, {0x00011941, 0x00011941}, +{0x000119A0, 0x000119A7}, {0x000119AA, 0x000119D0}, {0x000119E1, 0x000119E1}, {0x000119E3, 0x000119E3}, +{0x00011A00, 0x00011A00}, {0x00011A0B, 0x00011A32}, {0x00011A3A, 0x00011A3A}, {0x00011A50, 0x00011A50}, +{0x00011A5C, 0x00011A89}, {0x00011A9D, 0x00011A9D}, {0x00011AC0, 0x00011AF8}, {0x00011C00, 0x00011C08}, +{0x00011C0A, 0x00011C2E}, {0x00011C40, 0x00011C40}, {0x00011C72, 0x00011C8F}, {0x00011D00, 0x00011D06}, +{0x00011D08, 0x00011D09}, {0x00011D0B, 0x00011D30}, {0x00011D46, 0x00011D46}, {0x00011D60, 0x00011D65}, +{0x00011D67, 0x00011D68}, {0x00011D6A, 0x00011D89}, {0x00011D98, 0x00011D98}, {0x00011EE0, 0x00011EF2}, +{0x00011FB0, 0x00011FB0}, {0x00012000, 0x00012399}, {0x00012480, 0x00012543}, {0x00013000, 0x0001342E}, +{0x00014400, 0x00014646}, {0x00016800, 0x00016A38}, {0x00016A40, 0x00016A5E}, {0x00016AD0, 0x00016AED}, +{0x00016B00, 0x00016B2F}, {0x00016B40, 0x00016B43}, {0x00016B63, 0x00016B77}, {0x00016B7D, 0x00016B8F}, +{0x00016E40, 0x00016E7F}, {0x00016F00, 0x00016F4A}, {0x00016F50, 0x00016F50}, {0x00016F93, 0x00016F9F}, +{0x00016FE0, 0x00016FE1}, {0x00016FE3, 0x00016FE3}, {0x00017000, 0x000187F7}, {0x00018800, 0x00018CD5}, +{0x00018D00, 0x00018D08}, {0x0001B000, 0x0001B11E}, {0x0001B150, 0x0001B152}, {0x0001B164, 0x0001B167}, +{0x0001B170, 0x0001B2FB}, {0x0001BC00, 0x0001BC6A}, {0x0001BC70, 0x0001BC7C}, {0x0001BC80, 0x0001BC88}, +{0x0001BC90, 0x0001BC99}, {0x0001D400, 0x0001D454}, {0x0001D456, 0x0001D49C}, {0x0001D49E, 0x0001D49F}, +{0x0001D4A2, 0x0001D4A2}, {0x0001D4A5, 0x0001D4A6}, {0x0001D4A9, 0x0001D4AC}, {0x0001D4AE, 0x0001D4B9}, +{0x0001D4BB, 0x0001D4BB}, {0x0001D4BD, 0x0001D4C3}, {0x0001D4C5, 0x0001D505}, {0x0001D507, 0x0001D50A}, +{0x0001D50D, 0x0001D514}, {0x0001D516, 0x0001D51C}, {0x0001D51E, 0x0001D539}, {0x0001D53B, 0x0001D53E}, +{0x0001D540, 0x0001D544}, {0x0001D546, 0x0001D546}, {0x0001D54A, 0x0001D550}, {0x0001D552, 0x0001D6A5}, +{0x0001D6A8, 0x0001D6C0}, {0x0001D6C2, 0x0001D6DA}, {0x0001D6DC, 0x0001D6FA}, {0x0001D6FC, 0x0001D714}, +{0x0001D716, 0x0001D734}, {0x0001D736, 0x0001D74E}, {0x0001D750, 0x0001D76E}, {0x0001D770, 0x0001D788}, +{0x0001D78A, 0x0001D7A8}, {0x0001D7AA, 0x0001D7C2}, {0x0001D7C4, 0x0001D7CB}, {0x0001E100, 0x0001E12C}, +{0x0001E137, 0x0001E13D}, {0x0001E14E, 0x0001E14E}, {0x0001E2C0, 0x0001E2EB}, {0x0001E800, 0x0001E8C4}, +{0x0001E900, 0x0001E943}, {0x0001E94B, 0x0001E94B}, {0x0001EE00, 0x0001EE03}, {0x0001EE05, 0x0001EE1F}, +{0x0001EE21, 0x0001EE22}, {0x0001EE24, 0x0001EE24}, {0x0001EE27, 0x0001EE27}, {0x0001EE29, 0x0001EE32}, +{0x0001EE34, 0x0001EE37}, {0x0001EE39, 0x0001EE39}, {0x0001EE3B, 0x0001EE3B}, {0x0001EE42, 0x0001EE42}, +{0x0001EE47, 0x0001EE47}, {0x0001EE49, 0x0001EE49}, {0x0001EE4B, 0x0001EE4B}, {0x0001EE4D, 0x0001EE4F}, +{0x0001EE51, 0x0001EE52}, {0x0001EE54, 0x0001EE54}, {0x0001EE57, 0x0001EE57}, {0x0001EE59, 0x0001EE59}, +{0x0001EE5B, 0x0001EE5B}, {0x0001EE5D, 0x0001EE5D}, {0x0001EE5F, 0x0001EE5F}, {0x0001EE61, 0x0001EE62}, +{0x0001EE64, 0x0001EE64}, {0x0001EE67, 0x0001EE6A}, {0x0001EE6C, 0x0001EE72}, {0x0001EE74, 0x0001EE77}, +{0x0001EE79, 0x0001EE7C}, {0x0001EE7E, 0x0001EE7E}, {0x0001EE80, 0x0001EE89}, {0x0001EE8B, 0x0001EE9B}, +{0x0001EEA1, 0x0001EEA3}, {0x0001EEA5, 0x0001EEA9}, {0x0001EEAB, 0x0001EEBB}, {0x00020000, 0x0002A6DD}, +{0x0002A700, 0x0002B734}, {0x0002B740, 0x0002B81D}, {0x0002B820, 0x0002CEA1}, {0x0002CEB0, 0x0002EBE0}, +{0x0002F800, 0x0002FA1D}, {0x00030000, 0x0003134A}, +}; + +static const std::vector> unicode_ranges_whitespace = { +{0x00000009, 0x0000000D}, {0x0000001C, 0x00000020}, {0x00000085, 0x00000085}, {0x000000A0, 0x000000A0}, +{0x00001680, 0x00001680}, {0x00002000, 0x0000200A}, {0x00002028, 0x00002029}, {0x0000202F, 0x0000202F}, +{0x0000205F, 0x0000205F}, {0x00003000, 0x00003000}, +}; + +static const std::vector> unicode_ranges_accent_mark = { +{0x00000300, 0x0000036F}, {0x00000483, 0x00000489}, {0x00000591, 0x000005BD}, {0x000005BF, 0x000005BF}, +{0x000005C1, 0x000005C2}, {0x000005C4, 0x000005C5}, {0x000005C7, 0x000005C7}, {0x00000610, 0x0000061A}, +{0x0000064B, 0x0000065F}, {0x00000670, 0x00000670}, {0x000006D6, 0x000006DC}, {0x000006DF, 0x000006E4}, +{0x000006E7, 0x000006E8}, {0x000006EA, 0x000006ED}, {0x00000711, 0x00000711}, {0x00000730, 0x0000074A}, +{0x000007A6, 0x000007B0}, {0x000007EB, 0x000007F3}, {0x000007FD, 0x000007FD}, {0x00000816, 0x00000819}, +{0x0000081B, 0x00000823}, {0x00000825, 0x00000827}, {0x00000829, 0x0000082D}, {0x00000859, 0x0000085B}, +{0x000008D3, 0x000008E1}, {0x000008E3, 0x00000903}, {0x0000093A, 0x0000093C}, {0x0000093E, 0x0000094F}, +{0x00000951, 0x00000957}, {0x00000962, 0x00000963}, {0x00000981, 0x00000983}, {0x000009BC, 0x000009BC}, +{0x000009BE, 0x000009C4}, {0x000009C7, 0x000009C8}, {0x000009CB, 0x000009CD}, {0x000009D7, 0x000009D7}, +{0x000009E2, 0x000009E3}, {0x000009FE, 0x000009FE}, {0x00000A01, 0x00000A03}, {0x00000A3C, 0x00000A3C}, +{0x00000A3E, 0x00000A42}, {0x00000A47, 0x00000A48}, {0x00000A4B, 0x00000A4D}, {0x00000A51, 0x00000A51}, +{0x00000A70, 0x00000A71}, {0x00000A75, 0x00000A75}, {0x00000A81, 0x00000A83}, {0x00000ABC, 0x00000ABC}, +{0x00000ABE, 0x00000AC5}, {0x00000AC7, 0x00000AC9}, {0x00000ACB, 0x00000ACD}, {0x00000AE2, 0x00000AE3}, +{0x00000AFA, 0x00000AFF}, {0x00000B01, 0x00000B03}, {0x00000B3C, 0x00000B3C}, {0x00000B3E, 0x00000B44}, +{0x00000B47, 0x00000B48}, {0x00000B4B, 0x00000B4D}, {0x00000B55, 0x00000B57}, {0x00000B62, 0x00000B63}, +{0x00000B82, 0x00000B82}, {0x00000BBE, 0x00000BC2}, {0x00000BC6, 0x00000BC8}, {0x00000BCA, 0x00000BCD}, +{0x00000BD7, 0x00000BD7}, {0x00000C00, 0x00000C04}, {0x00000C3E, 0x00000C44}, {0x00000C46, 0x00000C48}, +{0x00000C4A, 0x00000C4D}, {0x00000C55, 0x00000C56}, {0x00000C62, 0x00000C63}, {0x00000C81, 0x00000C83}, +{0x00000CBC, 0x00000CBC}, {0x00000CBE, 0x00000CC4}, {0x00000CC6, 0x00000CC8}, {0x00000CCA, 0x00000CCD}, +{0x00000CD5, 0x00000CD6}, {0x00000CE2, 0x00000CE3}, {0x00000D00, 0x00000D03}, {0x00000D3B, 0x00000D3C}, +{0x00000D3E, 0x00000D44}, {0x00000D46, 0x00000D48}, {0x00000D4A, 0x00000D4D}, {0x00000D57, 0x00000D57}, +{0x00000D62, 0x00000D63}, {0x00000D81, 0x00000D83}, {0x00000DCA, 0x00000DCA}, {0x00000DCF, 0x00000DD4}, +{0x00000DD6, 0x00000DD6}, {0x00000DD8, 0x00000DDF}, {0x00000DF2, 0x00000DF3}, {0x00000E31, 0x00000E31}, +{0x00000E34, 0x00000E3A}, {0x00000E47, 0x00000E4E}, {0x00000EB1, 0x00000EB1}, {0x00000EB4, 0x00000EBC}, +{0x00000EC8, 0x00000ECD}, {0x00000F18, 0x00000F19}, {0x00000F35, 0x00000F35}, {0x00000F37, 0x00000F37}, +{0x00000F39, 0x00000F39}, {0x00000F3E, 0x00000F3F}, {0x00000F71, 0x00000F84}, {0x00000F86, 0x00000F87}, +{0x00000F8D, 0x00000F97}, {0x00000F99, 0x00000FBC}, {0x00000FC6, 0x00000FC6}, {0x0000102B, 0x0000103E}, +{0x00001056, 0x00001059}, {0x0000105E, 0x00001060}, {0x00001062, 0x00001064}, {0x00001067, 0x0000106D}, +{0x00001071, 0x00001074}, {0x00001082, 0x0000108D}, {0x0000108F, 0x0000108F}, {0x0000109A, 0x0000109D}, +{0x0000135D, 0x0000135F}, {0x00001712, 0x00001714}, {0x00001732, 0x00001734}, {0x00001752, 0x00001753}, +{0x00001772, 0x00001773}, {0x000017B4, 0x000017D3}, {0x000017DD, 0x000017DD}, {0x0000180B, 0x0000180D}, +{0x00001885, 0x00001886}, {0x000018A9, 0x000018A9}, {0x00001920, 0x0000192B}, {0x00001930, 0x0000193B}, +{0x00001A17, 0x00001A1B}, {0x00001A55, 0x00001A5E}, {0x00001A60, 0x00001A7C}, {0x00001A7F, 0x00001A7F}, +{0x00001AB0, 0x00001AC0}, {0x00001B00, 0x00001B04}, {0x00001B34, 0x00001B44}, {0x00001B6B, 0x00001B73}, +{0x00001B80, 0x00001B82}, {0x00001BA1, 0x00001BAD}, {0x00001BE6, 0x00001BF3}, {0x00001C24, 0x00001C37}, +{0x00001CD0, 0x00001CD2}, {0x00001CD4, 0x00001CE8}, {0x00001CED, 0x00001CED}, {0x00001CF4, 0x00001CF4}, +{0x00001CF7, 0x00001CF9}, {0x00001DC0, 0x00001DF9}, {0x00001DFB, 0x00001DFF}, {0x000020D0, 0x000020F0}, +{0x00002CEF, 0x00002CF1}, {0x00002D7F, 0x00002D7F}, {0x00002DE0, 0x00002DFF}, {0x0000302A, 0x0000302F}, +{0x00003099, 0x0000309A}, {0x0000A66F, 0x0000A672}, {0x0000A674, 0x0000A67D}, {0x0000A69E, 0x0000A69F}, +{0x0000A6F0, 0x0000A6F1}, {0x0000A802, 0x0000A802}, {0x0000A806, 0x0000A806}, {0x0000A80B, 0x0000A80B}, +{0x0000A823, 0x0000A827}, {0x0000A82C, 0x0000A82C}, {0x0000A880, 0x0000A881}, {0x0000A8B4, 0x0000A8C5}, +{0x0000A8E0, 0x0000A8F1}, {0x0000A8FF, 0x0000A8FF}, {0x0000A926, 0x0000A92D}, {0x0000A947, 0x0000A953}, +{0x0000A980, 0x0000A983}, {0x0000A9B3, 0x0000A9C0}, {0x0000A9E5, 0x0000A9E5}, {0x0000AA29, 0x0000AA36}, +{0x0000AA43, 0x0000AA43}, {0x0000AA4C, 0x0000AA4D}, {0x0000AA7B, 0x0000AA7D}, {0x0000AAB0, 0x0000AAB0}, +{0x0000AAB2, 0x0000AAB4}, {0x0000AAB7, 0x0000AAB8}, {0x0000AABE, 0x0000AABF}, {0x0000AAC1, 0x0000AAC1}, +{0x0000AAEB, 0x0000AAEF}, {0x0000AAF5, 0x0000AAF6}, {0x0000ABE3, 0x0000ABEA}, {0x0000ABEC, 0x0000ABED}, +{0x0000FB1E, 0x0000FB1E}, {0x0000FE00, 0x0000FE0F}, {0x0000FE20, 0x0000FE2F}, {0x000101FD, 0x000101FD}, +{0x000102E0, 0x000102E0}, {0x00010376, 0x0001037A}, {0x00010A01, 0x00010A03}, {0x00010A05, 0x00010A06}, +{0x00010A0C, 0x00010A0F}, {0x00010A38, 0x00010A3A}, {0x00010A3F, 0x00010A3F}, {0x00010AE5, 0x00010AE6}, +{0x00010D24, 0x00010D27}, {0x00010EAB, 0x00010EAC}, {0x00010F46, 0x00010F50}, {0x00011000, 0x00011002}, +{0x00011038, 0x00011046}, {0x0001107F, 0x00011082}, {0x000110B0, 0x000110BA}, {0x00011100, 0x00011102}, +{0x00011127, 0x00011134}, {0x00011145, 0x00011146}, {0x00011173, 0x00011173}, {0x00011180, 0x00011182}, +{0x000111B3, 0x000111C0}, {0x000111C9, 0x000111CC}, {0x000111CE, 0x000111CF}, {0x0001122C, 0x00011237}, +{0x0001123E, 0x0001123E}, {0x000112DF, 0x000112EA}, {0x00011300, 0x00011303}, {0x0001133B, 0x0001133C}, +{0x0001133E, 0x00011344}, {0x00011347, 0x00011348}, {0x0001134B, 0x0001134D}, {0x00011357, 0x00011357}, +{0x00011362, 0x00011363}, {0x00011366, 0x0001136C}, {0x00011370, 0x00011374}, {0x00011435, 0x00011446}, +{0x0001145E, 0x0001145E}, {0x000114B0, 0x000114C3}, {0x000115AF, 0x000115B5}, {0x000115B8, 0x000115C0}, +{0x000115DC, 0x000115DD}, {0x00011630, 0x00011640}, {0x000116AB, 0x000116B7}, {0x0001171D, 0x0001172B}, +{0x0001182C, 0x0001183A}, {0x00011930, 0x00011935}, {0x00011937, 0x00011938}, {0x0001193B, 0x0001193E}, +{0x00011940, 0x00011940}, {0x00011942, 0x00011943}, {0x000119D1, 0x000119D7}, {0x000119DA, 0x000119E0}, +{0x000119E4, 0x000119E4}, {0x00011A01, 0x00011A0A}, {0x00011A33, 0x00011A39}, {0x00011A3B, 0x00011A3E}, +{0x00011A47, 0x00011A47}, {0x00011A51, 0x00011A5B}, {0x00011A8A, 0x00011A99}, {0x00011C2F, 0x00011C36}, +{0x00011C38, 0x00011C3F}, {0x00011C92, 0x00011CA7}, {0x00011CA9, 0x00011CB6}, {0x00011D31, 0x00011D36}, +{0x00011D3A, 0x00011D3A}, {0x00011D3C, 0x00011D3D}, {0x00011D3F, 0x00011D45}, {0x00011D47, 0x00011D47}, +{0x00011D8A, 0x00011D8E}, {0x00011D90, 0x00011D91}, {0x00011D93, 0x00011D97}, {0x00011EF3, 0x00011EF6}, +{0x00016AF0, 0x00016AF4}, {0x00016B30, 0x00016B36}, {0x00016F4F, 0x00016F4F}, {0x00016F51, 0x00016F87}, +{0x00016F8F, 0x00016F92}, {0x00016FE4, 0x00016FE4}, {0x00016FF0, 0x00016FF1}, {0x0001BC9D, 0x0001BC9E}, +{0x0001D165, 0x0001D169}, {0x0001D16D, 0x0001D172}, {0x0001D17B, 0x0001D182}, {0x0001D185, 0x0001D18B}, +{0x0001D1AA, 0x0001D1AD}, {0x0001D242, 0x0001D244}, {0x0001DA00, 0x0001DA36}, {0x0001DA3B, 0x0001DA6C}, +{0x0001DA75, 0x0001DA75}, {0x0001DA84, 0x0001DA84}, {0x0001DA9B, 0x0001DA9F}, {0x0001DAA1, 0x0001DAAF}, +{0x0001E000, 0x0001E006}, {0x0001E008, 0x0001E018}, {0x0001E01B, 0x0001E021}, {0x0001E023, 0x0001E024}, +{0x0001E026, 0x0001E02A}, {0x0001E130, 0x0001E136}, {0x0001E2EC, 0x0001E2EF}, {0x0001E8D0, 0x0001E8D6}, +{0x0001E944, 0x0001E94A}, {0x000E0100, 0x000E01EF}, +}; + +static const std::vector> unicode_ranges_punctuation = { +{0x00000021, 0x00000023}, {0x00000025, 0x0000002A}, {0x0000002C, 0x0000002F}, {0x0000003A, 0x0000003B}, +{0x0000003F, 0x00000040}, {0x0000005B, 0x0000005D}, {0x0000005F, 0x0000005F}, {0x0000007B, 0x0000007B}, +{0x0000007D, 0x0000007D}, {0x000000A1, 0x000000A1}, {0x000000A7, 0x000000A7}, {0x000000AB, 0x000000AB}, +{0x000000B6, 0x000000B7}, {0x000000BB, 0x000000BB}, {0x000000BF, 0x000000BF}, {0x0000037E, 0x0000037E}, +{0x00000387, 0x00000387}, {0x0000055A, 0x0000055F}, {0x00000589, 0x0000058A}, {0x000005BE, 0x000005BE}, +{0x000005C0, 0x000005C0}, {0x000005C3, 0x000005C3}, {0x000005C6, 0x000005C6}, {0x000005F3, 0x000005F4}, +{0x00000609, 0x0000060A}, {0x0000060C, 0x0000060D}, {0x0000061B, 0x0000061B}, {0x0000061E, 0x0000061F}, +{0x0000066A, 0x0000066D}, {0x000006D4, 0x000006D4}, {0x00000700, 0x0000070D}, {0x000007F7, 0x000007F9}, +{0x00000830, 0x0000083E}, {0x0000085E, 0x0000085E}, {0x00000964, 0x00000965}, {0x00000970, 0x00000970}, +{0x000009FD, 0x000009FD}, {0x00000A76, 0x00000A76}, {0x00000AF0, 0x00000AF0}, {0x00000C77, 0x00000C77}, +{0x00000C84, 0x00000C84}, {0x00000DF4, 0x00000DF4}, {0x00000E4F, 0x00000E4F}, {0x00000E5A, 0x00000E5B}, +{0x00000F04, 0x00000F12}, {0x00000F14, 0x00000F14}, {0x00000F3A, 0x00000F3D}, {0x00000F85, 0x00000F85}, +{0x00000FD0, 0x00000FD4}, {0x00000FD9, 0x00000FDA}, {0x0000104A, 0x0000104F}, {0x000010FB, 0x000010FB}, +{0x00001360, 0x00001368}, {0x00001400, 0x00001400}, {0x0000166E, 0x0000166E}, {0x0000169B, 0x0000169C}, +{0x000016EB, 0x000016ED}, {0x00001735, 0x00001736}, {0x000017D4, 0x000017D6}, {0x000017D8, 0x000017DA}, +{0x00001800, 0x0000180A}, {0x00001944, 0x00001945}, {0x00001A1E, 0x00001A1F}, {0x00001AA0, 0x00001AA6}, +{0x00001AA8, 0x00001AAD}, {0x00001B5A, 0x00001B60}, {0x00001BFC, 0x00001BFF}, {0x00001C3B, 0x00001C3F}, +{0x00001C7E, 0x00001C7F}, {0x00001CC0, 0x00001CC7}, {0x00001CD3, 0x00001CD3}, {0x00002010, 0x00002027}, +{0x00002030, 0x00002043}, {0x00002045, 0x00002051}, {0x00002053, 0x0000205E}, {0x0000207D, 0x0000207E}, +{0x0000208D, 0x0000208E}, {0x00002308, 0x0000230B}, {0x00002329, 0x0000232A}, {0x00002768, 0x00002775}, +{0x000027C5, 0x000027C6}, {0x000027E6, 0x000027EF}, {0x00002983, 0x00002998}, {0x000029D8, 0x000029DB}, +{0x000029FC, 0x000029FD}, {0x00002CF9, 0x00002CFC}, {0x00002CFE, 0x00002CFF}, {0x00002D70, 0x00002D70}, +{0x00002E00, 0x00002E2E}, {0x00002E30, 0x00002E4F}, {0x00002E52, 0x00002E52}, {0x00003001, 0x00003003}, +{0x00003008, 0x00003011}, {0x00003014, 0x0000301F}, {0x00003030, 0x00003030}, {0x0000303D, 0x0000303D}, +{0x000030A0, 0x000030A0}, {0x000030FB, 0x000030FB}, {0x0000A4FE, 0x0000A4FF}, {0x0000A60D, 0x0000A60F}, +{0x0000A673, 0x0000A673}, {0x0000A67E, 0x0000A67E}, {0x0000A6F2, 0x0000A6F7}, {0x0000A874, 0x0000A877}, +{0x0000A8CE, 0x0000A8CF}, {0x0000A8F8, 0x0000A8FA}, {0x0000A8FC, 0x0000A8FC}, {0x0000A92E, 0x0000A92F}, +{0x0000A95F, 0x0000A95F}, {0x0000A9C1, 0x0000A9CD}, {0x0000A9DE, 0x0000A9DF}, {0x0000AA5C, 0x0000AA5F}, +{0x0000AADE, 0x0000AADF}, {0x0000AAF0, 0x0000AAF1}, {0x0000ABEB, 0x0000ABEB}, {0x0000FD3E, 0x0000FD3F}, +{0x0000FE10, 0x0000FE19}, {0x0000FE30, 0x0000FE52}, {0x0000FE54, 0x0000FE61}, {0x0000FE63, 0x0000FE63}, +{0x0000FE68, 0x0000FE68}, {0x0000FE6A, 0x0000FE6B}, {0x0000FF01, 0x0000FF03}, {0x0000FF05, 0x0000FF0A}, +{0x0000FF0C, 0x0000FF0F}, {0x0000FF1A, 0x0000FF1B}, {0x0000FF1F, 0x0000FF20}, {0x0000FF3B, 0x0000FF3D}, +{0x0000FF3F, 0x0000FF3F}, {0x0000FF5B, 0x0000FF5B}, {0x0000FF5D, 0x0000FF5D}, {0x0000FF5F, 0x0000FF65}, +{0x00010100, 0x00010102}, {0x0001039F, 0x0001039F}, {0x000103D0, 0x000103D0}, {0x0001056F, 0x0001056F}, +{0x00010857, 0x00010857}, {0x0001091F, 0x0001091F}, {0x0001093F, 0x0001093F}, {0x00010A50, 0x00010A58}, +{0x00010A7F, 0x00010A7F}, {0x00010AF0, 0x00010AF6}, {0x00010B39, 0x00010B3F}, {0x00010B99, 0x00010B9C}, +{0x00010EAD, 0x00010EAD}, {0x00010F55, 0x00010F59}, {0x00011047, 0x0001104D}, {0x000110BB, 0x000110BC}, +{0x000110BE, 0x000110C1}, {0x00011140, 0x00011143}, {0x00011174, 0x00011175}, {0x000111C5, 0x000111C8}, +{0x000111CD, 0x000111CD}, {0x000111DB, 0x000111DB}, {0x000111DD, 0x000111DF}, {0x00011238, 0x0001123D}, +{0x000112A9, 0x000112A9}, {0x0001144B, 0x0001144F}, {0x0001145A, 0x0001145B}, {0x0001145D, 0x0001145D}, +{0x000114C6, 0x000114C6}, {0x000115C1, 0x000115D7}, {0x00011641, 0x00011643}, {0x00011660, 0x0001166C}, +{0x0001173C, 0x0001173E}, {0x0001183B, 0x0001183B}, {0x00011944, 0x00011946}, {0x000119E2, 0x000119E2}, +{0x00011A3F, 0x00011A46}, {0x00011A9A, 0x00011A9C}, {0x00011A9E, 0x00011AA2}, {0x00011C41, 0x00011C45}, +{0x00011C70, 0x00011C71}, {0x00011EF7, 0x00011EF8}, {0x00011FFF, 0x00011FFF}, {0x00012470, 0x00012474}, +{0x00016A6E, 0x00016A6F}, {0x00016AF5, 0x00016AF5}, {0x00016B37, 0x00016B3B}, {0x00016B44, 0x00016B44}, +{0x00016E97, 0x00016E9A}, {0x00016FE2, 0x00016FE2}, {0x0001BC9F, 0x0001BC9F}, {0x0001DA87, 0x0001DA8B}, +{0x0001E95E, 0x0001E95F}, +}; + +static const std::vector> unicode_ranges_symbol = { +{0x00000024, 0x00000024}, {0x0000002B, 0x0000002B}, {0x0000003C, 0x0000003E}, {0x0000005E, 0x0000005E}, +{0x00000060, 0x00000060}, {0x0000007C, 0x0000007C}, {0x0000007E, 0x0000007E}, {0x000000A2, 0x000000A6}, +{0x000000A8, 0x000000A9}, {0x000000AC, 0x000000AC}, {0x000000AE, 0x000000B1}, {0x000000B4, 0x000000B4}, +{0x000000B8, 0x000000B8}, {0x000000D7, 0x000000D7}, {0x000000F7, 0x000000F7}, {0x000002C2, 0x000002C5}, +{0x000002D2, 0x000002DF}, {0x000002E5, 0x000002EB}, {0x000002ED, 0x000002ED}, {0x000002EF, 0x000002FF}, +{0x00000375, 0x00000375}, {0x00000384, 0x00000385}, {0x000003F6, 0x000003F6}, {0x00000482, 0x00000482}, +{0x0000058D, 0x0000058F}, {0x00000606, 0x00000608}, {0x0000060B, 0x0000060B}, {0x0000060E, 0x0000060F}, +{0x000006DE, 0x000006DE}, {0x000006E9, 0x000006E9}, {0x000006FD, 0x000006FE}, {0x000007F6, 0x000007F6}, +{0x000007FE, 0x000007FF}, {0x000009F2, 0x000009F3}, {0x000009FA, 0x000009FB}, {0x00000AF1, 0x00000AF1}, +{0x00000B70, 0x00000B70}, {0x00000BF3, 0x00000BFA}, {0x00000C7F, 0x00000C7F}, {0x00000D4F, 0x00000D4F}, +{0x00000D79, 0x00000D79}, {0x00000E3F, 0x00000E3F}, {0x00000F01, 0x00000F03}, {0x00000F13, 0x00000F13}, +{0x00000F15, 0x00000F17}, {0x00000F1A, 0x00000F1F}, {0x00000F34, 0x00000F34}, {0x00000F36, 0x00000F36}, +{0x00000F38, 0x00000F38}, {0x00000FBE, 0x00000FC5}, {0x00000FC7, 0x00000FCC}, {0x00000FCE, 0x00000FCF}, +{0x00000FD5, 0x00000FD8}, {0x0000109E, 0x0000109F}, {0x00001390, 0x00001399}, {0x0000166D, 0x0000166D}, +{0x000017DB, 0x000017DB}, {0x00001940, 0x00001940}, {0x000019DE, 0x000019FF}, {0x00001B61, 0x00001B6A}, +{0x00001B74, 0x00001B7C}, {0x00001FBD, 0x00001FBD}, {0x00001FBF, 0x00001FC1}, {0x00001FCD, 0x00001FCF}, +{0x00001FDD, 0x00001FDF}, {0x00001FED, 0x00001FEF}, {0x00001FFD, 0x00001FFE}, {0x00002044, 0x00002044}, +{0x00002052, 0x00002052}, {0x0000207A, 0x0000207C}, {0x0000208A, 0x0000208C}, {0x000020A0, 0x000020BF}, +{0x00002100, 0x00002101}, {0x00002103, 0x00002106}, {0x00002108, 0x00002109}, {0x00002114, 0x00002114}, +{0x00002116, 0x00002118}, {0x0000211E, 0x00002123}, {0x00002125, 0x00002125}, {0x00002127, 0x00002127}, +{0x00002129, 0x00002129}, {0x0000212E, 0x0000212E}, {0x0000213A, 0x0000213B}, {0x00002140, 0x00002144}, +{0x0000214A, 0x0000214D}, {0x0000214F, 0x0000214F}, {0x0000218A, 0x0000218B}, {0x00002190, 0x00002307}, +{0x0000230C, 0x00002328}, {0x0000232B, 0x00002426}, {0x00002440, 0x0000244A}, {0x0000249C, 0x000024E9}, +{0x00002500, 0x00002767}, {0x00002794, 0x000027C4}, {0x000027C7, 0x000027E5}, {0x000027F0, 0x00002982}, +{0x00002999, 0x000029D7}, {0x000029DC, 0x000029FB}, {0x000029FE, 0x00002B73}, {0x00002B76, 0x00002B95}, +{0x00002B97, 0x00002BFF}, {0x00002CE5, 0x00002CEA}, {0x00002E50, 0x00002E51}, {0x00002E80, 0x00002E99}, +{0x00002E9B, 0x00002EF3}, {0x00002F00, 0x00002FD5}, {0x00002FF0, 0x00002FFB}, {0x00003004, 0x00003004}, +{0x00003012, 0x00003013}, {0x00003020, 0x00003020}, {0x00003036, 0x00003037}, {0x0000303E, 0x0000303F}, +{0x0000309B, 0x0000309C}, {0x00003190, 0x00003191}, {0x00003196, 0x0000319F}, {0x000031C0, 0x000031E3}, +{0x00003200, 0x0000321E}, {0x0000322A, 0x00003247}, {0x00003250, 0x00003250}, {0x00003260, 0x0000327F}, +{0x0000328A, 0x000032B0}, {0x000032C0, 0x000033FF}, {0x00004DC0, 0x00004DFF}, {0x0000A490, 0x0000A4C6}, +{0x0000A700, 0x0000A716}, {0x0000A720, 0x0000A721}, {0x0000A789, 0x0000A78A}, {0x0000A828, 0x0000A82B}, +{0x0000A836, 0x0000A839}, {0x0000AA77, 0x0000AA79}, {0x0000AB5B, 0x0000AB5B}, {0x0000AB6A, 0x0000AB6B}, +{0x0000FB29, 0x0000FB29}, {0x0000FBB2, 0x0000FBC1}, {0x0000FDFC, 0x0000FDFD}, {0x0000FE62, 0x0000FE62}, +{0x0000FE64, 0x0000FE66}, {0x0000FE69, 0x0000FE69}, {0x0000FF04, 0x0000FF04}, {0x0000FF0B, 0x0000FF0B}, +{0x0000FF1C, 0x0000FF1E}, {0x0000FF3E, 0x0000FF3E}, {0x0000FF40, 0x0000FF40}, {0x0000FF5C, 0x0000FF5C}, +{0x0000FF5E, 0x0000FF5E}, {0x0000FFE0, 0x0000FFE6}, {0x0000FFE8, 0x0000FFEE}, {0x0000FFFC, 0x0000FFFD}, +{0x00010137, 0x0001013F}, {0x00010179, 0x00010189}, {0x0001018C, 0x0001018E}, {0x00010190, 0x0001019C}, +{0x000101A0, 0x000101A0}, {0x000101D0, 0x000101FC}, {0x00010877, 0x00010878}, {0x00010AC8, 0x00010AC8}, +{0x0001173F, 0x0001173F}, {0x00011FD5, 0x00011FF1}, {0x00016B3C, 0x00016B3F}, {0x00016B45, 0x00016B45}, +{0x0001BC9C, 0x0001BC9C}, {0x0001D000, 0x0001D0F5}, {0x0001D100, 0x0001D126}, {0x0001D129, 0x0001D164}, +{0x0001D16A, 0x0001D16C}, {0x0001D183, 0x0001D184}, {0x0001D18C, 0x0001D1A9}, {0x0001D1AE, 0x0001D1E8}, +{0x0001D200, 0x0001D241}, {0x0001D245, 0x0001D245}, {0x0001D300, 0x0001D356}, {0x0001D6C1, 0x0001D6C1}, +{0x0001D6DB, 0x0001D6DB}, {0x0001D6FB, 0x0001D6FB}, {0x0001D715, 0x0001D715}, {0x0001D735, 0x0001D735}, +{0x0001D74F, 0x0001D74F}, {0x0001D76F, 0x0001D76F}, {0x0001D789, 0x0001D789}, {0x0001D7A9, 0x0001D7A9}, +{0x0001D7C3, 0x0001D7C3}, {0x0001D800, 0x0001D9FF}, {0x0001DA37, 0x0001DA3A}, {0x0001DA6D, 0x0001DA74}, +{0x0001DA76, 0x0001DA83}, {0x0001DA85, 0x0001DA86}, {0x0001E14F, 0x0001E14F}, {0x0001E2FF, 0x0001E2FF}, +{0x0001ECAC, 0x0001ECAC}, {0x0001ECB0, 0x0001ECB0}, {0x0001ED2E, 0x0001ED2E}, {0x0001EEF0, 0x0001EEF1}, +{0x0001F000, 0x0001F02B}, {0x0001F030, 0x0001F093}, {0x0001F0A0, 0x0001F0AE}, {0x0001F0B1, 0x0001F0BF}, +{0x0001F0C1, 0x0001F0CF}, {0x0001F0D1, 0x0001F0F5}, {0x0001F10D, 0x0001F1AD}, {0x0001F1E6, 0x0001F202}, +{0x0001F210, 0x0001F23B}, {0x0001F240, 0x0001F248}, {0x0001F250, 0x0001F251}, {0x0001F260, 0x0001F265}, +{0x0001F300, 0x0001F6D7}, {0x0001F6E0, 0x0001F6EC}, {0x0001F6F0, 0x0001F6FC}, {0x0001F700, 0x0001F773}, +{0x0001F780, 0x0001F7D8}, {0x0001F7E0, 0x0001F7EB}, {0x0001F800, 0x0001F80B}, {0x0001F810, 0x0001F847}, +{0x0001F850, 0x0001F859}, {0x0001F860, 0x0001F887}, {0x0001F890, 0x0001F8AD}, {0x0001F8B0, 0x0001F8B1}, +{0x0001F900, 0x0001F978}, {0x0001F97A, 0x0001F9CB}, {0x0001F9CD, 0x0001FA53}, {0x0001FA60, 0x0001FA6D}, +{0x0001FA70, 0x0001FA74}, {0x0001FA78, 0x0001FA7A}, {0x0001FA80, 0x0001FA86}, {0x0001FA90, 0x0001FAA8}, +{0x0001FAB0, 0x0001FAB6}, {0x0001FAC0, 0x0001FAC2}, {0x0001FAD0, 0x0001FAD6}, {0x0001FB00, 0x0001FB92}, +{0x0001FB94, 0x0001FBCA}, +}; + +static const std::vector> unicode_ranges_control = { +{0x00000000, 0x00000008}, {0x0000000E, 0x0000001B}, {0x0000007F, 0x00000084}, {0x00000086, 0x0000009F}, +{0x000000AD, 0x000000AD}, {0x00000378, 0x00000379}, {0x00000380, 0x00000383}, {0x0000038B, 0x0000038B}, +{0x0000038D, 0x0000038D}, {0x000003A2, 0x000003A2}, {0x00000530, 0x00000530}, {0x00000557, 0x00000558}, +{0x0000058B, 0x0000058C}, {0x00000590, 0x00000590}, {0x000005C8, 0x000005CF}, {0x000005EB, 0x000005EE}, +{0x000005F5, 0x00000605}, {0x0000061C, 0x0000061D}, {0x000006DD, 0x000006DD}, {0x0000070E, 0x0000070F}, +{0x0000074B, 0x0000074C}, {0x000007B2, 0x000007BF}, {0x000007FB, 0x000007FC}, {0x0000082E, 0x0000082F}, +{0x0000083F, 0x0000083F}, {0x0000085C, 0x0000085D}, {0x0000085F, 0x0000085F}, {0x0000086B, 0x0000089F}, +{0x000008B5, 0x000008B5}, {0x000008C8, 0x000008D2}, {0x000008E2, 0x000008E2}, {0x00000984, 0x00000984}, +{0x0000098D, 0x0000098E}, {0x00000991, 0x00000992}, {0x000009A9, 0x000009A9}, {0x000009B1, 0x000009B1}, +{0x000009B3, 0x000009B5}, {0x000009BA, 0x000009BB}, {0x000009C5, 0x000009C6}, {0x000009C9, 0x000009CA}, +{0x000009CF, 0x000009D6}, {0x000009D8, 0x000009DB}, {0x000009DE, 0x000009DE}, {0x000009E4, 0x000009E5}, +{0x000009FF, 0x00000A00}, {0x00000A04, 0x00000A04}, {0x00000A0B, 0x00000A0E}, {0x00000A11, 0x00000A12}, +{0x00000A29, 0x00000A29}, {0x00000A31, 0x00000A31}, {0x00000A34, 0x00000A34}, {0x00000A37, 0x00000A37}, +{0x00000A3A, 0x00000A3B}, {0x00000A3D, 0x00000A3D}, {0x00000A43, 0x00000A46}, {0x00000A49, 0x00000A4A}, +{0x00000A4E, 0x00000A50}, {0x00000A52, 0x00000A58}, {0x00000A5D, 0x00000A5D}, {0x00000A5F, 0x00000A65}, +{0x00000A77, 0x00000A80}, {0x00000A84, 0x00000A84}, {0x00000A8E, 0x00000A8E}, {0x00000A92, 0x00000A92}, +{0x00000AA9, 0x00000AA9}, {0x00000AB1, 0x00000AB1}, {0x00000AB4, 0x00000AB4}, {0x00000ABA, 0x00000ABB}, +{0x00000AC6, 0x00000AC6}, {0x00000ACA, 0x00000ACA}, {0x00000ACE, 0x00000ACF}, {0x00000AD1, 0x00000ADF}, +{0x00000AE4, 0x00000AE5}, {0x00000AF2, 0x00000AF8}, {0x00000B00, 0x00000B00}, {0x00000B04, 0x00000B04}, +{0x00000B0D, 0x00000B0E}, {0x00000B11, 0x00000B12}, {0x00000B29, 0x00000B29}, {0x00000B31, 0x00000B31}, +{0x00000B34, 0x00000B34}, {0x00000B3A, 0x00000B3B}, {0x00000B45, 0x00000B46}, {0x00000B49, 0x00000B4A}, +{0x00000B4E, 0x00000B54}, {0x00000B58, 0x00000B5B}, {0x00000B5E, 0x00000B5E}, {0x00000B64, 0x00000B65}, +{0x00000B78, 0x00000B81}, {0x00000B84, 0x00000B84}, {0x00000B8B, 0x00000B8D}, {0x00000B91, 0x00000B91}, +{0x00000B96, 0x00000B98}, {0x00000B9B, 0x00000B9B}, {0x00000B9D, 0x00000B9D}, {0x00000BA0, 0x00000BA2}, +{0x00000BA5, 0x00000BA7}, {0x00000BAB, 0x00000BAD}, {0x00000BBA, 0x00000BBD}, {0x00000BC3, 0x00000BC5}, +{0x00000BC9, 0x00000BC9}, {0x00000BCE, 0x00000BCF}, {0x00000BD1, 0x00000BD6}, {0x00000BD8, 0x00000BE5}, +{0x00000BFB, 0x00000BFF}, {0x00000C0D, 0x00000C0D}, {0x00000C11, 0x00000C11}, {0x00000C29, 0x00000C29}, +{0x00000C3A, 0x00000C3C}, {0x00000C45, 0x00000C45}, {0x00000C49, 0x00000C49}, {0x00000C4E, 0x00000C54}, +{0x00000C57, 0x00000C57}, {0x00000C5B, 0x00000C5F}, {0x00000C64, 0x00000C65}, {0x00000C70, 0x00000C76}, +{0x00000C8D, 0x00000C8D}, {0x00000C91, 0x00000C91}, {0x00000CA9, 0x00000CA9}, {0x00000CB4, 0x00000CB4}, +{0x00000CBA, 0x00000CBB}, {0x00000CC5, 0x00000CC5}, {0x00000CC9, 0x00000CC9}, {0x00000CCE, 0x00000CD4}, +{0x00000CD7, 0x00000CDD}, {0x00000CDF, 0x00000CDF}, {0x00000CE4, 0x00000CE5}, {0x00000CF0, 0x00000CF0}, +{0x00000CF3, 0x00000CFF}, {0x00000D0D, 0x00000D0D}, {0x00000D11, 0x00000D11}, {0x00000D45, 0x00000D45}, +{0x00000D49, 0x00000D49}, {0x00000D50, 0x00000D53}, {0x00000D64, 0x00000D65}, {0x00000D80, 0x00000D80}, +{0x00000D84, 0x00000D84}, {0x00000D97, 0x00000D99}, {0x00000DB2, 0x00000DB2}, {0x00000DBC, 0x00000DBC}, +{0x00000DBE, 0x00000DBF}, {0x00000DC7, 0x00000DC9}, {0x00000DCB, 0x00000DCE}, {0x00000DD5, 0x00000DD5}, +{0x00000DD7, 0x00000DD7}, {0x00000DE0, 0x00000DE5}, {0x00000DF0, 0x00000DF1}, {0x00000DF5, 0x00000E00}, +{0x00000E3B, 0x00000E3E}, {0x00000E5C, 0x00000E80}, {0x00000E83, 0x00000E83}, {0x00000E85, 0x00000E85}, +{0x00000E8B, 0x00000E8B}, {0x00000EA4, 0x00000EA4}, {0x00000EA6, 0x00000EA6}, {0x00000EBE, 0x00000EBF}, +{0x00000EC5, 0x00000EC5}, {0x00000EC7, 0x00000EC7}, {0x00000ECE, 0x00000ECF}, {0x00000EDA, 0x00000EDB}, +{0x00000EE0, 0x00000EFF}, {0x00000F48, 0x00000F48}, {0x00000F6D, 0x00000F70}, {0x00000F98, 0x00000F98}, +{0x00000FBD, 0x00000FBD}, {0x00000FCD, 0x00000FCD}, {0x00000FDB, 0x00000FFF}, {0x000010C6, 0x000010C6}, +{0x000010C8, 0x000010CC}, {0x000010CE, 0x000010CF}, {0x00001249, 0x00001249}, {0x0000124E, 0x0000124F}, +{0x00001257, 0x00001257}, {0x00001259, 0x00001259}, {0x0000125E, 0x0000125F}, {0x00001289, 0x00001289}, +{0x0000128E, 0x0000128F}, {0x000012B1, 0x000012B1}, {0x000012B6, 0x000012B7}, {0x000012BF, 0x000012BF}, +{0x000012C1, 0x000012C1}, {0x000012C6, 0x000012C7}, {0x000012D7, 0x000012D7}, {0x00001311, 0x00001311}, +{0x00001316, 0x00001317}, {0x0000135B, 0x0000135C}, {0x0000137D, 0x0000137F}, {0x0000139A, 0x0000139F}, +{0x000013F6, 0x000013F7}, {0x000013FE, 0x000013FF}, {0x0000169D, 0x0000169F}, {0x000016F9, 0x000016FF}, +{0x0000170D, 0x0000170D}, {0x00001715, 0x0000171F}, {0x00001737, 0x0000173F}, {0x00001754, 0x0000175F}, +{0x0000176D, 0x0000176D}, {0x00001771, 0x00001771}, {0x00001774, 0x0000177F}, {0x000017DE, 0x000017DF}, +{0x000017EA, 0x000017EF}, {0x000017FA, 0x000017FF}, {0x0000180E, 0x0000180F}, {0x0000181A, 0x0000181F}, +{0x00001879, 0x0000187F}, {0x000018AB, 0x000018AF}, {0x000018F6, 0x000018FF}, {0x0000191F, 0x0000191F}, +{0x0000192C, 0x0000192F}, {0x0000193C, 0x0000193F}, {0x00001941, 0x00001943}, {0x0000196E, 0x0000196F}, +{0x00001975, 0x0000197F}, {0x000019AC, 0x000019AF}, {0x000019CA, 0x000019CF}, {0x000019DB, 0x000019DD}, +{0x00001A1C, 0x00001A1D}, {0x00001A5F, 0x00001A5F}, {0x00001A7D, 0x00001A7E}, {0x00001A8A, 0x00001A8F}, +{0x00001A9A, 0x00001A9F}, {0x00001AAE, 0x00001AAF}, {0x00001AC1, 0x00001AFF}, {0x00001B4C, 0x00001B4F}, +{0x00001B7D, 0x00001B7F}, {0x00001BF4, 0x00001BFB}, {0x00001C38, 0x00001C3A}, {0x00001C4A, 0x00001C4C}, +{0x00001C89, 0x00001C8F}, {0x00001CBB, 0x00001CBC}, {0x00001CC8, 0x00001CCF}, {0x00001CFB, 0x00001CFF}, +{0x00001DFA, 0x00001DFA}, {0x00001F16, 0x00001F17}, {0x00001F1E, 0x00001F1F}, {0x00001F46, 0x00001F47}, +{0x00001F4E, 0x00001F4F}, {0x00001F58, 0x00001F58}, {0x00001F5A, 0x00001F5A}, {0x00001F5C, 0x00001F5C}, +{0x00001F5E, 0x00001F5E}, {0x00001F7E, 0x00001F7F}, {0x00001FB5, 0x00001FB5}, {0x00001FC5, 0x00001FC5}, +{0x00001FD4, 0x00001FD5}, {0x00001FDC, 0x00001FDC}, {0x00001FF0, 0x00001FF1}, {0x00001FF5, 0x00001FF5}, +{0x00001FFF, 0x00001FFF}, {0x0000200B, 0x0000200F}, {0x0000202A, 0x0000202E}, {0x00002060, 0x0000206F}, +{0x00002072, 0x00002073}, {0x0000208F, 0x0000208F}, {0x0000209D, 0x0000209F}, {0x000020C0, 0x000020CF}, +{0x000020F1, 0x000020FF}, {0x0000218C, 0x0000218F}, {0x00002427, 0x0000243F}, {0x0000244B, 0x0000245F}, +{0x00002B74, 0x00002B75}, {0x00002B96, 0x00002B96}, {0x00002C2F, 0x00002C2F}, {0x00002C5F, 0x00002C5F}, +{0x00002CF4, 0x00002CF8}, {0x00002D26, 0x00002D26}, {0x00002D28, 0x00002D2C}, {0x00002D2E, 0x00002D2F}, +{0x00002D68, 0x00002D6E}, {0x00002D71, 0x00002D7E}, {0x00002D97, 0x00002D9F}, {0x00002DA7, 0x00002DA7}, +{0x00002DAF, 0x00002DAF}, {0x00002DB7, 0x00002DB7}, {0x00002DBF, 0x00002DBF}, {0x00002DC7, 0x00002DC7}, +{0x00002DCF, 0x00002DCF}, {0x00002DD7, 0x00002DD7}, {0x00002DDF, 0x00002DDF}, {0x00002E53, 0x00002E7F}, +{0x00002E9A, 0x00002E9A}, {0x00002EF4, 0x00002EFF}, {0x00002FD6, 0x00002FEF}, {0x00002FFC, 0x00002FFF}, +{0x00003040, 0x00003040}, {0x00003097, 0x00003098}, {0x00003100, 0x00003104}, {0x00003130, 0x00003130}, +{0x0000318F, 0x0000318F}, {0x000031E4, 0x000031EF}, {0x0000321F, 0x0000321F}, {0x00009FFD, 0x00009FFF}, +{0x0000A48D, 0x0000A48F}, {0x0000A4C7, 0x0000A4CF}, {0x0000A62C, 0x0000A63F}, {0x0000A6F8, 0x0000A6FF}, +{0x0000A7C0, 0x0000A7C1}, {0x0000A7CB, 0x0000A7F4}, {0x0000A82D, 0x0000A82F}, {0x0000A83A, 0x0000A83F}, +{0x0000A878, 0x0000A87F}, {0x0000A8C6, 0x0000A8CD}, {0x0000A8DA, 0x0000A8DF}, {0x0000A954, 0x0000A95E}, +{0x0000A97D, 0x0000A97F}, {0x0000A9CE, 0x0000A9CE}, {0x0000A9DA, 0x0000A9DD}, {0x0000A9FF, 0x0000A9FF}, +{0x0000AA37, 0x0000AA3F}, {0x0000AA4E, 0x0000AA4F}, {0x0000AA5A, 0x0000AA5B}, {0x0000AAC3, 0x0000AADA}, +{0x0000AAF7, 0x0000AB00}, {0x0000AB07, 0x0000AB08}, {0x0000AB0F, 0x0000AB10}, {0x0000AB17, 0x0000AB1F}, +{0x0000AB27, 0x0000AB27}, {0x0000AB2F, 0x0000AB2F}, {0x0000AB6C, 0x0000AB6F}, {0x0000ABEE, 0x0000ABEF}, +{0x0000ABFA, 0x0000ABFF}, {0x0000D7A4, 0x0000D7AF}, {0x0000D7C7, 0x0000D7CA}, {0x0000D7FC, 0x0000F8FF}, +{0x0000FA6E, 0x0000FA6F}, {0x0000FADA, 0x0000FAFF}, {0x0000FB07, 0x0000FB12}, {0x0000FB18, 0x0000FB1C}, +{0x0000FB37, 0x0000FB37}, {0x0000FB3D, 0x0000FB3D}, {0x0000FB3F, 0x0000FB3F}, {0x0000FB42, 0x0000FB42}, +{0x0000FB45, 0x0000FB45}, {0x0000FBC2, 0x0000FBD2}, {0x0000FD40, 0x0000FD4F}, {0x0000FD90, 0x0000FD91}, +{0x0000FDC8, 0x0000FDEF}, {0x0000FDFE, 0x0000FDFF}, {0x0000FE1A, 0x0000FE1F}, {0x0000FE53, 0x0000FE53}, +{0x0000FE67, 0x0000FE67}, {0x0000FE6C, 0x0000FE6F}, {0x0000FE75, 0x0000FE75}, {0x0000FEFD, 0x0000FF00}, +{0x0000FFBF, 0x0000FFC1}, {0x0000FFC8, 0x0000FFC9}, {0x0000FFD0, 0x0000FFD1}, {0x0000FFD8, 0x0000FFD9}, +{0x0000FFDD, 0x0000FFDF}, {0x0000FFE7, 0x0000FFE7}, {0x0000FFEF, 0x0000FFFB}, {0x0000FFFE, 0x0000FFFF}, +{0x0001000C, 0x0001000C}, {0x00010027, 0x00010027}, {0x0001003B, 0x0001003B}, {0x0001003E, 0x0001003E}, +{0x0001004E, 0x0001004F}, {0x0001005E, 0x0001007F}, {0x000100FB, 0x000100FF}, {0x00010103, 0x00010106}, +{0x00010134, 0x00010136}, {0x0001018F, 0x0001018F}, {0x0001019D, 0x0001019F}, {0x000101A1, 0x000101CF}, +{0x000101FE, 0x0001027F}, {0x0001029D, 0x0001029F}, {0x000102D1, 0x000102DF}, {0x000102FC, 0x000102FF}, +{0x00010324, 0x0001032C}, {0x0001034B, 0x0001034F}, {0x0001037B, 0x0001037F}, {0x0001039E, 0x0001039E}, +{0x000103C4, 0x000103C7}, {0x000103D6, 0x000103FF}, {0x0001049E, 0x0001049F}, {0x000104AA, 0x000104AF}, +{0x000104D4, 0x000104D7}, {0x000104FC, 0x000104FF}, {0x00010528, 0x0001052F}, {0x00010564, 0x0001056E}, +{0x00010570, 0x000105FF}, {0x00010737, 0x0001073F}, {0x00010756, 0x0001075F}, {0x00010768, 0x000107FF}, +{0x00010806, 0x00010807}, {0x00010809, 0x00010809}, {0x00010836, 0x00010836}, {0x00010839, 0x0001083B}, +{0x0001083D, 0x0001083E}, {0x00010856, 0x00010856}, {0x0001089F, 0x000108A6}, {0x000108B0, 0x000108DF}, +{0x000108F3, 0x000108F3}, {0x000108F6, 0x000108FA}, {0x0001091C, 0x0001091E}, {0x0001093A, 0x0001093E}, +{0x00010940, 0x0001097F}, {0x000109B8, 0x000109BB}, {0x000109D0, 0x000109D1}, {0x00010A04, 0x00010A04}, +{0x00010A07, 0x00010A0B}, {0x00010A14, 0x00010A14}, {0x00010A18, 0x00010A18}, {0x00010A36, 0x00010A37}, +{0x00010A3B, 0x00010A3E}, {0x00010A49, 0x00010A4F}, {0x00010A59, 0x00010A5F}, {0x00010AA0, 0x00010ABF}, +{0x00010AE7, 0x00010AEA}, {0x00010AF7, 0x00010AFF}, {0x00010B36, 0x00010B38}, {0x00010B56, 0x00010B57}, +{0x00010B73, 0x00010B77}, {0x00010B92, 0x00010B98}, {0x00010B9D, 0x00010BA8}, {0x00010BB0, 0x00010BFF}, +{0x00010C49, 0x00010C7F}, {0x00010CB3, 0x00010CBF}, {0x00010CF3, 0x00010CF9}, {0x00010D28, 0x00010D2F}, +{0x00010D3A, 0x00010E5F}, {0x00010E7F, 0x00010E7F}, {0x00010EAA, 0x00010EAA}, {0x00010EAE, 0x00010EAF}, +{0x00010EB2, 0x00010EFF}, {0x00010F28, 0x00010F2F}, {0x00010F5A, 0x00010FAF}, {0x00010FCC, 0x00010FDF}, +{0x00010FF7, 0x00010FFF}, {0x0001104E, 0x00011051}, {0x00011070, 0x0001107E}, {0x000110BD, 0x000110BD}, +{0x000110C2, 0x000110CF}, {0x000110E9, 0x000110EF}, {0x000110FA, 0x000110FF}, {0x00011135, 0x00011135}, +{0x00011148, 0x0001114F}, {0x00011177, 0x0001117F}, {0x000111E0, 0x000111E0}, {0x000111F5, 0x000111FF}, +{0x00011212, 0x00011212}, {0x0001123F, 0x0001127F}, {0x00011287, 0x00011287}, {0x00011289, 0x00011289}, +{0x0001128E, 0x0001128E}, {0x0001129E, 0x0001129E}, {0x000112AA, 0x000112AF}, {0x000112EB, 0x000112EF}, +{0x000112FA, 0x000112FF}, {0x00011304, 0x00011304}, {0x0001130D, 0x0001130E}, {0x00011311, 0x00011312}, +{0x00011329, 0x00011329}, {0x00011331, 0x00011331}, {0x00011334, 0x00011334}, {0x0001133A, 0x0001133A}, +{0x00011345, 0x00011346}, {0x00011349, 0x0001134A}, {0x0001134E, 0x0001134F}, {0x00011351, 0x00011356}, +{0x00011358, 0x0001135C}, {0x00011364, 0x00011365}, {0x0001136D, 0x0001136F}, {0x00011375, 0x000113FF}, +{0x0001145C, 0x0001145C}, {0x00011462, 0x0001147F}, {0x000114C8, 0x000114CF}, {0x000114DA, 0x0001157F}, +{0x000115B6, 0x000115B7}, {0x000115DE, 0x000115FF}, {0x00011645, 0x0001164F}, {0x0001165A, 0x0001165F}, +{0x0001166D, 0x0001167F}, {0x000116B9, 0x000116BF}, {0x000116CA, 0x000116FF}, {0x0001171B, 0x0001171C}, +{0x0001172C, 0x0001172F}, {0x00011740, 0x000117FF}, {0x0001183C, 0x0001189F}, {0x000118F3, 0x000118FE}, +{0x00011907, 0x00011908}, {0x0001190A, 0x0001190B}, {0x00011914, 0x00011914}, {0x00011917, 0x00011917}, +{0x00011936, 0x00011936}, {0x00011939, 0x0001193A}, {0x00011947, 0x0001194F}, {0x0001195A, 0x0001199F}, +{0x000119A8, 0x000119A9}, {0x000119D8, 0x000119D9}, {0x000119E5, 0x000119FF}, {0x00011A48, 0x00011A4F}, +{0x00011AA3, 0x00011ABF}, {0x00011AF9, 0x00011BFF}, {0x00011C09, 0x00011C09}, {0x00011C37, 0x00011C37}, +{0x00011C46, 0x00011C4F}, {0x00011C6D, 0x00011C6F}, {0x00011C90, 0x00011C91}, {0x00011CA8, 0x00011CA8}, +{0x00011CB7, 0x00011CFF}, {0x00011D07, 0x00011D07}, {0x00011D0A, 0x00011D0A}, {0x00011D37, 0x00011D39}, +{0x00011D3B, 0x00011D3B}, {0x00011D3E, 0x00011D3E}, {0x00011D48, 0x00011D4F}, {0x00011D5A, 0x00011D5F}, +{0x00011D66, 0x00011D66}, {0x00011D69, 0x00011D69}, {0x00011D8F, 0x00011D8F}, {0x00011D92, 0x00011D92}, +{0x00011D99, 0x00011D9F}, {0x00011DAA, 0x00011EDF}, {0x00011EF9, 0x00011FAF}, {0x00011FB1, 0x00011FBF}, +{0x00011FF2, 0x00011FFE}, {0x0001239A, 0x000123FF}, {0x0001246F, 0x0001246F}, {0x00012475, 0x0001247F}, +{0x00012544, 0x00012FFF}, {0x0001342F, 0x000143FF}, {0x00014647, 0x000167FF}, {0x00016A39, 0x00016A3F}, +{0x00016A5F, 0x00016A5F}, {0x00016A6A, 0x00016A6D}, {0x00016A70, 0x00016ACF}, {0x00016AEE, 0x00016AEF}, +{0x00016AF6, 0x00016AFF}, {0x00016B46, 0x00016B4F}, {0x00016B5A, 0x00016B5A}, {0x00016B62, 0x00016B62}, +{0x00016B78, 0x00016B7C}, {0x00016B90, 0x00016E3F}, {0x00016E9B, 0x00016EFF}, {0x00016F4B, 0x00016F4E}, +{0x00016F88, 0x00016F8E}, {0x00016FA0, 0x00016FDF}, {0x00016FE5, 0x00016FEF}, {0x00016FF2, 0x00016FFF}, +{0x000187F8, 0x000187FF}, {0x00018CD6, 0x00018CFF}, {0x00018D09, 0x0001AFFF}, {0x0001B11F, 0x0001B14F}, +{0x0001B153, 0x0001B163}, {0x0001B168, 0x0001B16F}, {0x0001B2FC, 0x0001BBFF}, {0x0001BC6B, 0x0001BC6F}, +{0x0001BC7D, 0x0001BC7F}, {0x0001BC89, 0x0001BC8F}, {0x0001BC9A, 0x0001BC9B}, {0x0001BCA0, 0x0001CFFF}, +{0x0001D0F6, 0x0001D0FF}, {0x0001D127, 0x0001D128}, {0x0001D173, 0x0001D17A}, {0x0001D1E9, 0x0001D1FF}, +{0x0001D246, 0x0001D2DF}, {0x0001D2F4, 0x0001D2FF}, {0x0001D357, 0x0001D35F}, {0x0001D379, 0x0001D3FF}, +{0x0001D455, 0x0001D455}, {0x0001D49D, 0x0001D49D}, {0x0001D4A0, 0x0001D4A1}, {0x0001D4A3, 0x0001D4A4}, +{0x0001D4A7, 0x0001D4A8}, {0x0001D4AD, 0x0001D4AD}, {0x0001D4BA, 0x0001D4BA}, {0x0001D4BC, 0x0001D4BC}, +{0x0001D4C4, 0x0001D4C4}, {0x0001D506, 0x0001D506}, {0x0001D50B, 0x0001D50C}, {0x0001D515, 0x0001D515}, +{0x0001D51D, 0x0001D51D}, {0x0001D53A, 0x0001D53A}, {0x0001D53F, 0x0001D53F}, {0x0001D545, 0x0001D545}, +{0x0001D547, 0x0001D549}, {0x0001D551, 0x0001D551}, {0x0001D6A6, 0x0001D6A7}, {0x0001D7CC, 0x0001D7CD}, +{0x0001DA8C, 0x0001DA9A}, {0x0001DAA0, 0x0001DAA0}, {0x0001DAB0, 0x0001DFFF}, {0x0001E007, 0x0001E007}, +{0x0001E019, 0x0001E01A}, {0x0001E022, 0x0001E022}, {0x0001E025, 0x0001E025}, {0x0001E02B, 0x0001E0FF}, +{0x0001E12D, 0x0001E12F}, {0x0001E13E, 0x0001E13F}, {0x0001E14A, 0x0001E14D}, {0x0001E150, 0x0001E2BF}, +{0x0001E2FA, 0x0001E2FE}, {0x0001E300, 0x0001E7FF}, {0x0001E8C5, 0x0001E8C6}, {0x0001E8D7, 0x0001E8FF}, +{0x0001E94C, 0x0001E94F}, {0x0001E95A, 0x0001E95D}, {0x0001E960, 0x0001EC70}, {0x0001ECB5, 0x0001ED00}, +{0x0001ED3E, 0x0001EDFF}, {0x0001EE04, 0x0001EE04}, {0x0001EE20, 0x0001EE20}, {0x0001EE23, 0x0001EE23}, +{0x0001EE25, 0x0001EE26}, {0x0001EE28, 0x0001EE28}, {0x0001EE33, 0x0001EE33}, {0x0001EE38, 0x0001EE38}, +{0x0001EE3A, 0x0001EE3A}, {0x0001EE3C, 0x0001EE41}, {0x0001EE43, 0x0001EE46}, {0x0001EE48, 0x0001EE48}, +{0x0001EE4A, 0x0001EE4A}, {0x0001EE4C, 0x0001EE4C}, {0x0001EE50, 0x0001EE50}, {0x0001EE53, 0x0001EE53}, +{0x0001EE55, 0x0001EE56}, {0x0001EE58, 0x0001EE58}, {0x0001EE5A, 0x0001EE5A}, {0x0001EE5C, 0x0001EE5C}, +{0x0001EE5E, 0x0001EE5E}, {0x0001EE60, 0x0001EE60}, {0x0001EE63, 0x0001EE63}, {0x0001EE65, 0x0001EE66}, +{0x0001EE6B, 0x0001EE6B}, {0x0001EE73, 0x0001EE73}, {0x0001EE78, 0x0001EE78}, {0x0001EE7D, 0x0001EE7D}, +{0x0001EE7F, 0x0001EE7F}, {0x0001EE8A, 0x0001EE8A}, {0x0001EE9C, 0x0001EEA0}, {0x0001EEA4, 0x0001EEA4}, +{0x0001EEAA, 0x0001EEAA}, {0x0001EEBC, 0x0001EEEF}, {0x0001EEF2, 0x0001EFFF}, {0x0001F02C, 0x0001F02F}, +{0x0001F094, 0x0001F09F}, {0x0001F0AF, 0x0001F0B0}, {0x0001F0C0, 0x0001F0C0}, {0x0001F0D0, 0x0001F0D0}, +{0x0001F0F6, 0x0001F0FF}, {0x0001F1AE, 0x0001F1E5}, {0x0001F203, 0x0001F20F}, {0x0001F23C, 0x0001F23F}, +{0x0001F249, 0x0001F24F}, {0x0001F252, 0x0001F25F}, {0x0001F266, 0x0001F2FF}, {0x0001F6D8, 0x0001F6DF}, +{0x0001F6ED, 0x0001F6EF}, {0x0001F6FD, 0x0001F6FF}, {0x0001F774, 0x0001F77F}, {0x0001F7D9, 0x0001F7DF}, +{0x0001F7EC, 0x0001F7FF}, {0x0001F80C, 0x0001F80F}, {0x0001F848, 0x0001F84F}, {0x0001F85A, 0x0001F85F}, +{0x0001F888, 0x0001F88F}, {0x0001F8AE, 0x0001F8AF}, {0x0001F8B2, 0x0001F8FF}, {0x0001F979, 0x0001F979}, +{0x0001F9CC, 0x0001F9CC}, {0x0001FA54, 0x0001FA5F}, {0x0001FA6E, 0x0001FA6F}, {0x0001FA75, 0x0001FA77}, +{0x0001FA7B, 0x0001FA7F}, {0x0001FA87, 0x0001FA8F}, {0x0001FAA9, 0x0001FAAF}, {0x0001FAB7, 0x0001FABF}, +{0x0001FAC3, 0x0001FACF}, {0x0001FAD7, 0x0001FAFF}, {0x0001FB93, 0x0001FB93}, {0x0001FBCB, 0x0001FBEF}, +{0x0001FBFA, 0x0001FFFF}, {0x0002A6DE, 0x0002A6FF}, {0x0002B735, 0x0002B73F}, {0x0002B81E, 0x0002B81F}, +{0x0002CEA2, 0x0002CEAF}, {0x0002EBE1, 0x0002F7FF}, {0x0002FA1E, 0x0002FFFF}, {0x0003134B, 0x000E00FF}, +{0x000E01F0, 0x0010FFFF}, +}; + +static const std::multimap unicode_map_nfd = { +{0x000000C0, 0x00000041}, {0x000000C0, 0x00000300}, {0x000000C1, 0x00000041}, {0x000000C1, 0x00000301}, +{0x000000C2, 0x00000041}, {0x000000C2, 0x00000302}, {0x000000C3, 0x00000041}, {0x000000C3, 0x00000303}, +{0x000000C4, 0x00000041}, {0x000000C4, 0x00000308}, {0x000000C5, 0x00000041}, {0x000000C5, 0x0000030A}, +{0x000000C7, 0x00000043}, {0x000000C7, 0x00000327}, {0x000000C8, 0x00000045}, {0x000000C8, 0x00000300}, +{0x000000C9, 0x00000045}, {0x000000C9, 0x00000301}, {0x000000CA, 0x00000045}, {0x000000CA, 0x00000302}, +{0x000000CB, 0x00000045}, {0x000000CB, 0x00000308}, {0x000000CC, 0x00000049}, {0x000000CC, 0x00000300}, +{0x000000CD, 0x00000049}, {0x000000CD, 0x00000301}, {0x000000CE, 0x00000049}, {0x000000CE, 0x00000302}, +{0x000000CF, 0x00000049}, {0x000000CF, 0x00000308}, {0x000000D1, 0x0000004E}, {0x000000D1, 0x00000303}, +{0x000000D2, 0x0000004F}, {0x000000D2, 0x00000300}, {0x000000D3, 0x0000004F}, {0x000000D3, 0x00000301}, +{0x000000D4, 0x0000004F}, {0x000000D4, 0x00000302}, {0x000000D5, 0x0000004F}, {0x000000D5, 0x00000303}, +{0x000000D6, 0x0000004F}, {0x000000D6, 0x00000308}, {0x000000D9, 0x00000055}, {0x000000D9, 0x00000300}, +{0x000000DA, 0x00000055}, {0x000000DA, 0x00000301}, {0x000000DB, 0x00000055}, {0x000000DB, 0x00000302}, +{0x000000DC, 0x00000055}, {0x000000DC, 0x00000308}, {0x000000DD, 0x00000059}, {0x000000DD, 0x00000301}, +{0x000000E0, 0x00000061}, {0x000000E0, 0x00000300}, {0x000000E1, 0x00000061}, {0x000000E1, 0x00000301}, +{0x000000E2, 0x00000061}, {0x000000E2, 0x00000302}, {0x000000E3, 0x00000061}, {0x000000E3, 0x00000303}, +{0x000000E4, 0x00000061}, {0x000000E4, 0x00000308}, {0x000000E5, 0x00000061}, {0x000000E5, 0x0000030A}, +{0x000000E7, 0x00000063}, {0x000000E7, 0x00000327}, {0x000000E8, 0x00000065}, {0x000000E8, 0x00000300}, +{0x000000E9, 0x00000065}, {0x000000E9, 0x00000301}, {0x000000EA, 0x00000065}, {0x000000EA, 0x00000302}, +{0x000000EB, 0x00000065}, {0x000000EB, 0x00000308}, {0x000000EC, 0x00000069}, {0x000000EC, 0x00000300}, +{0x000000ED, 0x00000069}, {0x000000ED, 0x00000301}, {0x000000EE, 0x00000069}, {0x000000EE, 0x00000302}, +{0x000000EF, 0x00000069}, {0x000000EF, 0x00000308}, {0x000000F1, 0x0000006E}, {0x000000F1, 0x00000303}, +{0x000000F2, 0x0000006F}, {0x000000F2, 0x00000300}, {0x000000F3, 0x0000006F}, {0x000000F3, 0x00000301}, +{0x000000F4, 0x0000006F}, {0x000000F4, 0x00000302}, {0x000000F5, 0x0000006F}, {0x000000F5, 0x00000303}, +{0x000000F6, 0x0000006F}, {0x000000F6, 0x00000308}, {0x000000F9, 0x00000075}, {0x000000F9, 0x00000300}, +{0x000000FA, 0x00000075}, {0x000000FA, 0x00000301}, {0x000000FB, 0x00000075}, {0x000000FB, 0x00000302}, +{0x000000FC, 0x00000075}, {0x000000FC, 0x00000308}, {0x000000FD, 0x00000079}, {0x000000FD, 0x00000301}, +{0x000000FF, 0x00000079}, {0x000000FF, 0x00000308}, {0x00000100, 0x00000041}, {0x00000100, 0x00000304}, +{0x00000101, 0x00000061}, {0x00000101, 0x00000304}, {0x00000102, 0x00000041}, {0x00000102, 0x00000306}, +{0x00000103, 0x00000061}, {0x00000103, 0x00000306}, {0x00000104, 0x00000041}, {0x00000104, 0x00000328}, +{0x00000105, 0x00000061}, {0x00000105, 0x00000328}, {0x00000106, 0x00000043}, {0x00000106, 0x00000301}, +{0x00000107, 0x00000063}, {0x00000107, 0x00000301}, {0x00000108, 0x00000043}, {0x00000108, 0x00000302}, +{0x00000109, 0x00000063}, {0x00000109, 0x00000302}, {0x0000010A, 0x00000043}, {0x0000010A, 0x00000307}, +{0x0000010B, 0x00000063}, {0x0000010B, 0x00000307}, {0x0000010C, 0x00000043}, {0x0000010C, 0x0000030C}, +{0x0000010D, 0x00000063}, {0x0000010D, 0x0000030C}, {0x0000010E, 0x00000044}, {0x0000010E, 0x0000030C}, +{0x0000010F, 0x00000064}, {0x0000010F, 0x0000030C}, {0x00000112, 0x00000045}, {0x00000112, 0x00000304}, +{0x00000113, 0x00000065}, {0x00000113, 0x00000304}, {0x00000114, 0x00000045}, {0x00000114, 0x00000306}, +{0x00000115, 0x00000065}, {0x00000115, 0x00000306}, {0x00000116, 0x00000045}, {0x00000116, 0x00000307}, +{0x00000117, 0x00000065}, {0x00000117, 0x00000307}, {0x00000118, 0x00000045}, {0x00000118, 0x00000328}, +{0x00000119, 0x00000065}, {0x00000119, 0x00000328}, {0x0000011A, 0x00000045}, {0x0000011A, 0x0000030C}, +{0x0000011B, 0x00000065}, {0x0000011B, 0x0000030C}, {0x0000011C, 0x00000047}, {0x0000011C, 0x00000302}, +{0x0000011D, 0x00000067}, {0x0000011D, 0x00000302}, {0x0000011E, 0x00000047}, {0x0000011E, 0x00000306}, +{0x0000011F, 0x00000067}, {0x0000011F, 0x00000306}, {0x00000120, 0x00000047}, {0x00000120, 0x00000307}, +{0x00000121, 0x00000067}, {0x00000121, 0x00000307}, {0x00000122, 0x00000047}, {0x00000122, 0x00000327}, +{0x00000123, 0x00000067}, {0x00000123, 0x00000327}, {0x00000124, 0x00000048}, {0x00000124, 0x00000302}, +{0x00000125, 0x00000068}, {0x00000125, 0x00000302}, {0x00000128, 0x00000049}, {0x00000128, 0x00000303}, +{0x00000129, 0x00000069}, {0x00000129, 0x00000303}, {0x0000012A, 0x00000049}, {0x0000012A, 0x00000304}, +{0x0000012B, 0x00000069}, {0x0000012B, 0x00000304}, {0x0000012C, 0x00000049}, {0x0000012C, 0x00000306}, +{0x0000012D, 0x00000069}, {0x0000012D, 0x00000306}, {0x0000012E, 0x00000049}, {0x0000012E, 0x00000328}, +{0x0000012F, 0x00000069}, {0x0000012F, 0x00000328}, {0x00000130, 0x00000049}, {0x00000130, 0x00000307}, +{0x00000134, 0x0000004A}, {0x00000134, 0x00000302}, {0x00000135, 0x0000006A}, {0x00000135, 0x00000302}, +{0x00000136, 0x0000004B}, {0x00000136, 0x00000327}, {0x00000137, 0x0000006B}, {0x00000137, 0x00000327}, +{0x00000139, 0x0000004C}, {0x00000139, 0x00000301}, {0x0000013A, 0x0000006C}, {0x0000013A, 0x00000301}, +{0x0000013B, 0x0000004C}, {0x0000013B, 0x00000327}, {0x0000013C, 0x0000006C}, {0x0000013C, 0x00000327}, +{0x0000013D, 0x0000004C}, {0x0000013D, 0x0000030C}, {0x0000013E, 0x0000006C}, {0x0000013E, 0x0000030C}, +{0x00000143, 0x0000004E}, {0x00000143, 0x00000301}, {0x00000144, 0x0000006E}, {0x00000144, 0x00000301}, +{0x00000145, 0x0000004E}, {0x00000145, 0x00000327}, {0x00000146, 0x0000006E}, {0x00000146, 0x00000327}, +{0x00000147, 0x0000004E}, {0x00000147, 0x0000030C}, {0x00000148, 0x0000006E}, {0x00000148, 0x0000030C}, +{0x0000014C, 0x0000004F}, {0x0000014C, 0x00000304}, {0x0000014D, 0x0000006F}, {0x0000014D, 0x00000304}, +{0x0000014E, 0x0000004F}, {0x0000014E, 0x00000306}, {0x0000014F, 0x0000006F}, {0x0000014F, 0x00000306}, +{0x00000150, 0x0000004F}, {0x00000150, 0x0000030B}, {0x00000151, 0x0000006F}, {0x00000151, 0x0000030B}, +{0x00000154, 0x00000052}, {0x00000154, 0x00000301}, {0x00000155, 0x00000072}, {0x00000155, 0x00000301}, +{0x00000156, 0x00000052}, {0x00000156, 0x00000327}, {0x00000157, 0x00000072}, {0x00000157, 0x00000327}, +{0x00000158, 0x00000052}, {0x00000158, 0x0000030C}, {0x00000159, 0x00000072}, {0x00000159, 0x0000030C}, +{0x0000015A, 0x00000053}, {0x0000015A, 0x00000301}, {0x0000015B, 0x00000073}, {0x0000015B, 0x00000301}, +{0x0000015C, 0x00000053}, {0x0000015C, 0x00000302}, {0x0000015D, 0x00000073}, {0x0000015D, 0x00000302}, +{0x0000015E, 0x00000053}, {0x0000015E, 0x00000327}, {0x0000015F, 0x00000073}, {0x0000015F, 0x00000327}, +{0x00000160, 0x00000053}, {0x00000160, 0x0000030C}, {0x00000161, 0x00000073}, {0x00000161, 0x0000030C}, +{0x00000162, 0x00000054}, {0x00000162, 0x00000327}, {0x00000163, 0x00000074}, {0x00000163, 0x00000327}, +{0x00000164, 0x00000054}, {0x00000164, 0x0000030C}, {0x00000165, 0x00000074}, {0x00000165, 0x0000030C}, +{0x00000168, 0x00000055}, {0x00000168, 0x00000303}, {0x00000169, 0x00000075}, {0x00000169, 0x00000303}, +{0x0000016A, 0x00000055}, {0x0000016A, 0x00000304}, {0x0000016B, 0x00000075}, {0x0000016B, 0x00000304}, +{0x0000016C, 0x00000055}, {0x0000016C, 0x00000306}, {0x0000016D, 0x00000075}, {0x0000016D, 0x00000306}, +{0x0000016E, 0x00000055}, {0x0000016E, 0x0000030A}, {0x0000016F, 0x00000075}, {0x0000016F, 0x0000030A}, +{0x00000170, 0x00000055}, {0x00000170, 0x0000030B}, {0x00000171, 0x00000075}, {0x00000171, 0x0000030B}, +{0x00000172, 0x00000055}, {0x00000172, 0x00000328}, {0x00000173, 0x00000075}, {0x00000173, 0x00000328}, +{0x00000174, 0x00000057}, {0x00000174, 0x00000302}, {0x00000175, 0x00000077}, {0x00000175, 0x00000302}, +{0x00000176, 0x00000059}, {0x00000176, 0x00000302}, {0x00000177, 0x00000079}, {0x00000177, 0x00000302}, +{0x00000178, 0x00000059}, {0x00000178, 0x00000308}, {0x00000179, 0x0000005A}, {0x00000179, 0x00000301}, +{0x0000017A, 0x0000007A}, {0x0000017A, 0x00000301}, {0x0000017B, 0x0000005A}, {0x0000017B, 0x00000307}, +{0x0000017C, 0x0000007A}, {0x0000017C, 0x00000307}, {0x0000017D, 0x0000005A}, {0x0000017D, 0x0000030C}, +{0x0000017E, 0x0000007A}, {0x0000017E, 0x0000030C}, {0x000001A0, 0x0000004F}, {0x000001A0, 0x0000031B}, +{0x000001A1, 0x0000006F}, {0x000001A1, 0x0000031B}, {0x000001AF, 0x00000055}, {0x000001AF, 0x0000031B}, +{0x000001B0, 0x00000075}, {0x000001B0, 0x0000031B}, {0x000001CD, 0x00000041}, {0x000001CD, 0x0000030C}, +{0x000001CE, 0x00000061}, {0x000001CE, 0x0000030C}, {0x000001CF, 0x00000049}, {0x000001CF, 0x0000030C}, +{0x000001D0, 0x00000069}, {0x000001D0, 0x0000030C}, {0x000001D1, 0x0000004F}, {0x000001D1, 0x0000030C}, +{0x000001D2, 0x0000006F}, {0x000001D2, 0x0000030C}, {0x000001D3, 0x00000055}, {0x000001D3, 0x0000030C}, +{0x000001D4, 0x00000075}, {0x000001D4, 0x0000030C}, {0x000001D5, 0x00000055}, {0x000001D5, 0x00000308}, +{0x000001D5, 0x00000304}, {0x000001D6, 0x00000075}, {0x000001D6, 0x00000308}, {0x000001D6, 0x00000304}, +{0x000001D7, 0x00000055}, {0x000001D7, 0x00000308}, {0x000001D7, 0x00000301}, {0x000001D8, 0x00000075}, +{0x000001D8, 0x00000308}, {0x000001D8, 0x00000301}, {0x000001D9, 0x00000055}, {0x000001D9, 0x00000308}, +{0x000001D9, 0x0000030C}, {0x000001DA, 0x00000075}, {0x000001DA, 0x00000308}, {0x000001DA, 0x0000030C}, +{0x000001DB, 0x00000055}, {0x000001DB, 0x00000308}, {0x000001DB, 0x00000300}, {0x000001DC, 0x00000075}, +{0x000001DC, 0x00000308}, {0x000001DC, 0x00000300}, {0x000001DE, 0x00000041}, {0x000001DE, 0x00000308}, +{0x000001DE, 0x00000304}, {0x000001DF, 0x00000061}, {0x000001DF, 0x00000308}, {0x000001DF, 0x00000304}, +{0x000001E0, 0x00000041}, {0x000001E0, 0x00000307}, {0x000001E0, 0x00000304}, {0x000001E1, 0x00000061}, +{0x000001E1, 0x00000307}, {0x000001E1, 0x00000304}, {0x000001E2, 0x000000C6}, {0x000001E2, 0x00000304}, +{0x000001E3, 0x000000E6}, {0x000001E3, 0x00000304}, {0x000001E6, 0x00000047}, {0x000001E6, 0x0000030C}, +{0x000001E7, 0x00000067}, {0x000001E7, 0x0000030C}, {0x000001E8, 0x0000004B}, {0x000001E8, 0x0000030C}, +{0x000001E9, 0x0000006B}, {0x000001E9, 0x0000030C}, {0x000001EA, 0x0000004F}, {0x000001EA, 0x00000328}, +{0x000001EB, 0x0000006F}, {0x000001EB, 0x00000328}, {0x000001EC, 0x0000004F}, {0x000001EC, 0x00000328}, +{0x000001EC, 0x00000304}, {0x000001ED, 0x0000006F}, {0x000001ED, 0x00000328}, {0x000001ED, 0x00000304}, +{0x000001EE, 0x000001B7}, {0x000001EE, 0x0000030C}, {0x000001EF, 0x00000292}, {0x000001EF, 0x0000030C}, +{0x000001F0, 0x0000006A}, {0x000001F0, 0x0000030C}, {0x000001F4, 0x00000047}, {0x000001F4, 0x00000301}, +{0x000001F5, 0x00000067}, {0x000001F5, 0x00000301}, {0x000001F8, 0x0000004E}, {0x000001F8, 0x00000300}, +{0x000001F9, 0x0000006E}, {0x000001F9, 0x00000300}, {0x000001FA, 0x00000041}, {0x000001FA, 0x0000030A}, +{0x000001FA, 0x00000301}, {0x000001FB, 0x00000061}, {0x000001FB, 0x0000030A}, {0x000001FB, 0x00000301}, +{0x000001FC, 0x000000C6}, {0x000001FC, 0x00000301}, {0x000001FD, 0x000000E6}, {0x000001FD, 0x00000301}, +{0x000001FE, 0x000000D8}, {0x000001FE, 0x00000301}, {0x000001FF, 0x000000F8}, {0x000001FF, 0x00000301}, +{0x00000200, 0x00000041}, {0x00000200, 0x0000030F}, {0x00000201, 0x00000061}, {0x00000201, 0x0000030F}, +{0x00000202, 0x00000041}, {0x00000202, 0x00000311}, {0x00000203, 0x00000061}, {0x00000203, 0x00000311}, +{0x00000204, 0x00000045}, {0x00000204, 0x0000030F}, {0x00000205, 0x00000065}, {0x00000205, 0x0000030F}, +{0x00000206, 0x00000045}, {0x00000206, 0x00000311}, {0x00000207, 0x00000065}, {0x00000207, 0x00000311}, +{0x00000208, 0x00000049}, {0x00000208, 0x0000030F}, {0x00000209, 0x00000069}, {0x00000209, 0x0000030F}, +{0x0000020A, 0x00000049}, {0x0000020A, 0x00000311}, {0x0000020B, 0x00000069}, {0x0000020B, 0x00000311}, +{0x0000020C, 0x0000004F}, {0x0000020C, 0x0000030F}, {0x0000020D, 0x0000006F}, {0x0000020D, 0x0000030F}, +{0x0000020E, 0x0000004F}, {0x0000020E, 0x00000311}, {0x0000020F, 0x0000006F}, {0x0000020F, 0x00000311}, +{0x00000210, 0x00000052}, {0x00000210, 0x0000030F}, {0x00000211, 0x00000072}, {0x00000211, 0x0000030F}, +{0x00000212, 0x00000052}, {0x00000212, 0x00000311}, {0x00000213, 0x00000072}, {0x00000213, 0x00000311}, +{0x00000214, 0x00000055}, {0x00000214, 0x0000030F}, {0x00000215, 0x00000075}, {0x00000215, 0x0000030F}, +{0x00000216, 0x00000055}, {0x00000216, 0x00000311}, {0x00000217, 0x00000075}, {0x00000217, 0x00000311}, +{0x00000218, 0x00000053}, {0x00000218, 0x00000326}, {0x00000219, 0x00000073}, {0x00000219, 0x00000326}, +{0x0000021A, 0x00000054}, {0x0000021A, 0x00000326}, {0x0000021B, 0x00000074}, {0x0000021B, 0x00000326}, +{0x0000021E, 0x00000048}, {0x0000021E, 0x0000030C}, {0x0000021F, 0x00000068}, {0x0000021F, 0x0000030C}, +{0x00000226, 0x00000041}, {0x00000226, 0x00000307}, {0x00000227, 0x00000061}, {0x00000227, 0x00000307}, +{0x00000228, 0x00000045}, {0x00000228, 0x00000327}, {0x00000229, 0x00000065}, {0x00000229, 0x00000327}, +{0x0000022A, 0x0000004F}, {0x0000022A, 0x00000308}, {0x0000022A, 0x00000304}, {0x0000022B, 0x0000006F}, +{0x0000022B, 0x00000308}, {0x0000022B, 0x00000304}, {0x0000022C, 0x0000004F}, {0x0000022C, 0x00000303}, +{0x0000022C, 0x00000304}, {0x0000022D, 0x0000006F}, {0x0000022D, 0x00000303}, {0x0000022D, 0x00000304}, +{0x0000022E, 0x0000004F}, {0x0000022E, 0x00000307}, {0x0000022F, 0x0000006F}, {0x0000022F, 0x00000307}, +{0x00000230, 0x0000004F}, {0x00000230, 0x00000307}, {0x00000230, 0x00000304}, {0x00000231, 0x0000006F}, +{0x00000231, 0x00000307}, {0x00000231, 0x00000304}, {0x00000232, 0x00000059}, {0x00000232, 0x00000304}, +{0x00000233, 0x00000079}, {0x00000233, 0x00000304}, {0x00000340, 0x00000300}, {0x00000341, 0x00000301}, +{0x00000343, 0x00000313}, {0x00000344, 0x00000308}, {0x00000344, 0x00000301}, {0x00000374, 0x000002B9}, +{0x0000037E, 0x0000003B}, {0x00000385, 0x000000A8}, {0x00000385, 0x00000301}, {0x00000386, 0x00000391}, +{0x00000386, 0x00000301}, {0x00000387, 0x000000B7}, {0x00000388, 0x00000395}, {0x00000388, 0x00000301}, +{0x00000389, 0x00000397}, {0x00000389, 0x00000301}, {0x0000038A, 0x00000399}, {0x0000038A, 0x00000301}, +{0x0000038C, 0x0000039F}, {0x0000038C, 0x00000301}, {0x0000038E, 0x000003A5}, {0x0000038E, 0x00000301}, +{0x0000038F, 0x000003A9}, {0x0000038F, 0x00000301}, {0x00000390, 0x000003B9}, {0x00000390, 0x00000308}, +{0x00000390, 0x00000301}, {0x000003AA, 0x00000399}, {0x000003AA, 0x00000308}, {0x000003AB, 0x000003A5}, +{0x000003AB, 0x00000308}, {0x000003AC, 0x000003B1}, {0x000003AC, 0x00000301}, {0x000003AD, 0x000003B5}, +{0x000003AD, 0x00000301}, {0x000003AE, 0x000003B7}, {0x000003AE, 0x00000301}, {0x000003AF, 0x000003B9}, +{0x000003AF, 0x00000301}, {0x000003B0, 0x000003C5}, {0x000003B0, 0x00000308}, {0x000003B0, 0x00000301}, +{0x000003CA, 0x000003B9}, {0x000003CA, 0x00000308}, {0x000003CB, 0x000003C5}, {0x000003CB, 0x00000308}, +{0x000003CC, 0x000003BF}, {0x000003CC, 0x00000301}, {0x000003CD, 0x000003C5}, {0x000003CD, 0x00000301}, +{0x000003CE, 0x000003C9}, {0x000003CE, 0x00000301}, {0x000003D3, 0x000003D2}, {0x000003D3, 0x00000301}, +{0x000003D4, 0x000003D2}, {0x000003D4, 0x00000308}, {0x00000400, 0x00000415}, {0x00000400, 0x00000300}, +{0x00000401, 0x00000415}, {0x00000401, 0x00000308}, {0x00000403, 0x00000413}, {0x00000403, 0x00000301}, +{0x00000407, 0x00000406}, {0x00000407, 0x00000308}, {0x0000040C, 0x0000041A}, {0x0000040C, 0x00000301}, +{0x0000040D, 0x00000418}, {0x0000040D, 0x00000300}, {0x0000040E, 0x00000423}, {0x0000040E, 0x00000306}, +{0x00000419, 0x00000418}, {0x00000419, 0x00000306}, {0x00000439, 0x00000438}, {0x00000439, 0x00000306}, +{0x00000450, 0x00000435}, {0x00000450, 0x00000300}, {0x00000451, 0x00000435}, {0x00000451, 0x00000308}, +{0x00000453, 0x00000433}, {0x00000453, 0x00000301}, {0x00000457, 0x00000456}, {0x00000457, 0x00000308}, +{0x0000045C, 0x0000043A}, {0x0000045C, 0x00000301}, {0x0000045D, 0x00000438}, {0x0000045D, 0x00000300}, +{0x0000045E, 0x00000443}, {0x0000045E, 0x00000306}, {0x00000476, 0x00000474}, {0x00000476, 0x0000030F}, +{0x00000477, 0x00000475}, {0x00000477, 0x0000030F}, {0x000004C1, 0x00000416}, {0x000004C1, 0x00000306}, +{0x000004C2, 0x00000436}, {0x000004C2, 0x00000306}, {0x000004D0, 0x00000410}, {0x000004D0, 0x00000306}, +{0x000004D1, 0x00000430}, {0x000004D1, 0x00000306}, {0x000004D2, 0x00000410}, {0x000004D2, 0x00000308}, +{0x000004D3, 0x00000430}, {0x000004D3, 0x00000308}, {0x000004D6, 0x00000415}, {0x000004D6, 0x00000306}, +{0x000004D7, 0x00000435}, {0x000004D7, 0x00000306}, {0x000004DA, 0x000004D8}, {0x000004DA, 0x00000308}, +{0x000004DB, 0x000004D9}, {0x000004DB, 0x00000308}, {0x000004DC, 0x00000416}, {0x000004DC, 0x00000308}, +{0x000004DD, 0x00000436}, {0x000004DD, 0x00000308}, {0x000004DE, 0x00000417}, {0x000004DE, 0x00000308}, +{0x000004DF, 0x00000437}, {0x000004DF, 0x00000308}, {0x000004E2, 0x00000418}, {0x000004E2, 0x00000304}, +{0x000004E3, 0x00000438}, {0x000004E3, 0x00000304}, {0x000004E4, 0x00000418}, {0x000004E4, 0x00000308}, +{0x000004E5, 0x00000438}, {0x000004E5, 0x00000308}, {0x000004E6, 0x0000041E}, {0x000004E6, 0x00000308}, +{0x000004E7, 0x0000043E}, {0x000004E7, 0x00000308}, {0x000004EA, 0x000004E8}, {0x000004EA, 0x00000308}, +{0x000004EB, 0x000004E9}, {0x000004EB, 0x00000308}, {0x000004EC, 0x0000042D}, {0x000004EC, 0x00000308}, +{0x000004ED, 0x0000044D}, {0x000004ED, 0x00000308}, {0x000004EE, 0x00000423}, {0x000004EE, 0x00000304}, +{0x000004EF, 0x00000443}, {0x000004EF, 0x00000304}, {0x000004F0, 0x00000423}, {0x000004F0, 0x00000308}, +{0x000004F1, 0x00000443}, {0x000004F1, 0x00000308}, {0x000004F2, 0x00000423}, {0x000004F2, 0x0000030B}, +{0x000004F3, 0x00000443}, {0x000004F3, 0x0000030B}, {0x000004F4, 0x00000427}, {0x000004F4, 0x00000308}, +{0x000004F5, 0x00000447}, {0x000004F5, 0x00000308}, {0x000004F8, 0x0000042B}, {0x000004F8, 0x00000308}, +{0x000004F9, 0x0000044B}, {0x000004F9, 0x00000308}, {0x00000622, 0x00000627}, {0x00000622, 0x00000653}, +{0x00000623, 0x00000627}, {0x00000623, 0x00000654}, {0x00000624, 0x00000648}, {0x00000624, 0x00000654}, +{0x00000625, 0x00000627}, {0x00000625, 0x00000655}, {0x00000626, 0x0000064A}, {0x00000626, 0x00000654}, +{0x000006C0, 0x000006D5}, {0x000006C0, 0x00000654}, {0x000006C2, 0x000006C1}, {0x000006C2, 0x00000654}, +{0x000006D3, 0x000006D2}, {0x000006D3, 0x00000654}, {0x00000929, 0x00000928}, {0x00000929, 0x0000093C}, +{0x00000931, 0x00000930}, {0x00000931, 0x0000093C}, {0x00000934, 0x00000933}, {0x00000934, 0x0000093C}, +{0x00000958, 0x00000915}, {0x00000958, 0x0000093C}, {0x00000959, 0x00000916}, {0x00000959, 0x0000093C}, +{0x0000095A, 0x00000917}, {0x0000095A, 0x0000093C}, {0x0000095B, 0x0000091C}, {0x0000095B, 0x0000093C}, +{0x0000095C, 0x00000921}, {0x0000095C, 0x0000093C}, {0x0000095D, 0x00000922}, {0x0000095D, 0x0000093C}, +{0x0000095E, 0x0000092B}, {0x0000095E, 0x0000093C}, {0x0000095F, 0x0000092F}, {0x0000095F, 0x0000093C}, +{0x000009CB, 0x000009C7}, {0x000009CB, 0x000009BE}, {0x000009CC, 0x000009C7}, {0x000009CC, 0x000009D7}, +{0x000009DC, 0x000009A1}, {0x000009DC, 0x000009BC}, {0x000009DD, 0x000009A2}, {0x000009DD, 0x000009BC}, +{0x000009DF, 0x000009AF}, {0x000009DF, 0x000009BC}, {0x00000A33, 0x00000A32}, {0x00000A33, 0x00000A3C}, +{0x00000A36, 0x00000A38}, {0x00000A36, 0x00000A3C}, {0x00000A59, 0x00000A16}, {0x00000A59, 0x00000A3C}, +{0x00000A5A, 0x00000A17}, {0x00000A5A, 0x00000A3C}, {0x00000A5B, 0x00000A1C}, {0x00000A5B, 0x00000A3C}, +{0x00000A5E, 0x00000A2B}, {0x00000A5E, 0x00000A3C}, {0x00000B48, 0x00000B47}, {0x00000B48, 0x00000B56}, +{0x00000B4B, 0x00000B47}, {0x00000B4B, 0x00000B3E}, {0x00000B4C, 0x00000B47}, {0x00000B4C, 0x00000B57}, +{0x00000B5C, 0x00000B21}, {0x00000B5C, 0x00000B3C}, {0x00000B5D, 0x00000B22}, {0x00000B5D, 0x00000B3C}, +{0x00000B94, 0x00000B92}, {0x00000B94, 0x00000BD7}, {0x00000BCA, 0x00000BC6}, {0x00000BCA, 0x00000BBE}, +{0x00000BCB, 0x00000BC7}, {0x00000BCB, 0x00000BBE}, {0x00000BCC, 0x00000BC6}, {0x00000BCC, 0x00000BD7}, +{0x00000C48, 0x00000C46}, {0x00000C48, 0x00000C56}, {0x00000CC0, 0x00000CBF}, {0x00000CC0, 0x00000CD5}, +{0x00000CC7, 0x00000CC6}, {0x00000CC7, 0x00000CD5}, {0x00000CC8, 0x00000CC6}, {0x00000CC8, 0x00000CD6}, +{0x00000CCA, 0x00000CC6}, {0x00000CCA, 0x00000CC2}, {0x00000CCB, 0x00000CC6}, {0x00000CCB, 0x00000CC2}, +{0x00000CCB, 0x00000CD5}, {0x00000D4A, 0x00000D46}, {0x00000D4A, 0x00000D3E}, {0x00000D4B, 0x00000D47}, +{0x00000D4B, 0x00000D3E}, {0x00000D4C, 0x00000D46}, {0x00000D4C, 0x00000D57}, {0x00000DDA, 0x00000DD9}, +{0x00000DDA, 0x00000DCA}, {0x00000DDC, 0x00000DD9}, {0x00000DDC, 0x00000DCF}, {0x00000DDD, 0x00000DD9}, +{0x00000DDD, 0x00000DCF}, {0x00000DDD, 0x00000DCA}, {0x00000DDE, 0x00000DD9}, {0x00000DDE, 0x00000DDF}, +{0x00000F43, 0x00000F42}, {0x00000F43, 0x00000FB7}, {0x00000F4D, 0x00000F4C}, {0x00000F4D, 0x00000FB7}, +{0x00000F52, 0x00000F51}, {0x00000F52, 0x00000FB7}, {0x00000F57, 0x00000F56}, {0x00000F57, 0x00000FB7}, +{0x00000F5C, 0x00000F5B}, {0x00000F5C, 0x00000FB7}, {0x00000F69, 0x00000F40}, {0x00000F69, 0x00000FB5}, +{0x00000F73, 0x00000F71}, {0x00000F73, 0x00000F72}, {0x00000F75, 0x00000F71}, {0x00000F75, 0x00000F74}, +{0x00000F76, 0x00000FB2}, {0x00000F76, 0x00000F80}, {0x00000F78, 0x00000FB3}, {0x00000F78, 0x00000F80}, +{0x00000F81, 0x00000F71}, {0x00000F81, 0x00000F80}, {0x00000F93, 0x00000F92}, {0x00000F93, 0x00000FB7}, +{0x00000F9D, 0x00000F9C}, {0x00000F9D, 0x00000FB7}, {0x00000FA2, 0x00000FA1}, {0x00000FA2, 0x00000FB7}, +{0x00000FA7, 0x00000FA6}, {0x00000FA7, 0x00000FB7}, {0x00000FAC, 0x00000FAB}, {0x00000FAC, 0x00000FB7}, +{0x00000FB9, 0x00000F90}, {0x00000FB9, 0x00000FB5}, {0x00001026, 0x00001025}, {0x00001026, 0x0000102E}, +{0x00001B06, 0x00001B05}, {0x00001B06, 0x00001B35}, {0x00001B08, 0x00001B07}, {0x00001B08, 0x00001B35}, +{0x00001B0A, 0x00001B09}, {0x00001B0A, 0x00001B35}, {0x00001B0C, 0x00001B0B}, {0x00001B0C, 0x00001B35}, +{0x00001B0E, 0x00001B0D}, {0x00001B0E, 0x00001B35}, {0x00001B12, 0x00001B11}, {0x00001B12, 0x00001B35}, +{0x00001B3B, 0x00001B3A}, {0x00001B3B, 0x00001B35}, {0x00001B3D, 0x00001B3C}, {0x00001B3D, 0x00001B35}, +{0x00001B40, 0x00001B3E}, {0x00001B40, 0x00001B35}, {0x00001B41, 0x00001B3F}, {0x00001B41, 0x00001B35}, +{0x00001B43, 0x00001B42}, {0x00001B43, 0x00001B35}, {0x00001E00, 0x00000041}, {0x00001E00, 0x00000325}, +{0x00001E01, 0x00000061}, {0x00001E01, 0x00000325}, {0x00001E02, 0x00000042}, {0x00001E02, 0x00000307}, +{0x00001E03, 0x00000062}, {0x00001E03, 0x00000307}, {0x00001E04, 0x00000042}, {0x00001E04, 0x00000323}, +{0x00001E05, 0x00000062}, {0x00001E05, 0x00000323}, {0x00001E06, 0x00000042}, {0x00001E06, 0x00000331}, +{0x00001E07, 0x00000062}, {0x00001E07, 0x00000331}, {0x00001E08, 0x00000043}, {0x00001E08, 0x00000327}, +{0x00001E08, 0x00000301}, {0x00001E09, 0x00000063}, {0x00001E09, 0x00000327}, {0x00001E09, 0x00000301}, +{0x00001E0A, 0x00000044}, {0x00001E0A, 0x00000307}, {0x00001E0B, 0x00000064}, {0x00001E0B, 0x00000307}, +{0x00001E0C, 0x00000044}, {0x00001E0C, 0x00000323}, {0x00001E0D, 0x00000064}, {0x00001E0D, 0x00000323}, +{0x00001E0E, 0x00000044}, {0x00001E0E, 0x00000331}, {0x00001E0F, 0x00000064}, {0x00001E0F, 0x00000331}, +{0x00001E10, 0x00000044}, {0x00001E10, 0x00000327}, {0x00001E11, 0x00000064}, {0x00001E11, 0x00000327}, +{0x00001E12, 0x00000044}, {0x00001E12, 0x0000032D}, {0x00001E13, 0x00000064}, {0x00001E13, 0x0000032D}, +{0x00001E14, 0x00000045}, {0x00001E14, 0x00000304}, {0x00001E14, 0x00000300}, {0x00001E15, 0x00000065}, +{0x00001E15, 0x00000304}, {0x00001E15, 0x00000300}, {0x00001E16, 0x00000045}, {0x00001E16, 0x00000304}, +{0x00001E16, 0x00000301}, {0x00001E17, 0x00000065}, {0x00001E17, 0x00000304}, {0x00001E17, 0x00000301}, +{0x00001E18, 0x00000045}, {0x00001E18, 0x0000032D}, {0x00001E19, 0x00000065}, {0x00001E19, 0x0000032D}, +{0x00001E1A, 0x00000045}, {0x00001E1A, 0x00000330}, {0x00001E1B, 0x00000065}, {0x00001E1B, 0x00000330}, +{0x00001E1C, 0x00000045}, {0x00001E1C, 0x00000327}, {0x00001E1C, 0x00000306}, {0x00001E1D, 0x00000065}, +{0x00001E1D, 0x00000327}, {0x00001E1D, 0x00000306}, {0x00001E1E, 0x00000046}, {0x00001E1E, 0x00000307}, +{0x00001E1F, 0x00000066}, {0x00001E1F, 0x00000307}, {0x00001E20, 0x00000047}, {0x00001E20, 0x00000304}, +{0x00001E21, 0x00000067}, {0x00001E21, 0x00000304}, {0x00001E22, 0x00000048}, {0x00001E22, 0x00000307}, +{0x00001E23, 0x00000068}, {0x00001E23, 0x00000307}, {0x00001E24, 0x00000048}, {0x00001E24, 0x00000323}, +{0x00001E25, 0x00000068}, {0x00001E25, 0x00000323}, {0x00001E26, 0x00000048}, {0x00001E26, 0x00000308}, +{0x00001E27, 0x00000068}, {0x00001E27, 0x00000308}, {0x00001E28, 0x00000048}, {0x00001E28, 0x00000327}, +{0x00001E29, 0x00000068}, {0x00001E29, 0x00000327}, {0x00001E2A, 0x00000048}, {0x00001E2A, 0x0000032E}, +{0x00001E2B, 0x00000068}, {0x00001E2B, 0x0000032E}, {0x00001E2C, 0x00000049}, {0x00001E2C, 0x00000330}, +{0x00001E2D, 0x00000069}, {0x00001E2D, 0x00000330}, {0x00001E2E, 0x00000049}, {0x00001E2E, 0x00000308}, +{0x00001E2E, 0x00000301}, {0x00001E2F, 0x00000069}, {0x00001E2F, 0x00000308}, {0x00001E2F, 0x00000301}, +{0x00001E30, 0x0000004B}, {0x00001E30, 0x00000301}, {0x00001E31, 0x0000006B}, {0x00001E31, 0x00000301}, +{0x00001E32, 0x0000004B}, {0x00001E32, 0x00000323}, {0x00001E33, 0x0000006B}, {0x00001E33, 0x00000323}, +{0x00001E34, 0x0000004B}, {0x00001E34, 0x00000331}, {0x00001E35, 0x0000006B}, {0x00001E35, 0x00000331}, +{0x00001E36, 0x0000004C}, {0x00001E36, 0x00000323}, {0x00001E37, 0x0000006C}, {0x00001E37, 0x00000323}, +{0x00001E38, 0x0000004C}, {0x00001E38, 0x00000323}, {0x00001E38, 0x00000304}, {0x00001E39, 0x0000006C}, +{0x00001E39, 0x00000323}, {0x00001E39, 0x00000304}, {0x00001E3A, 0x0000004C}, {0x00001E3A, 0x00000331}, +{0x00001E3B, 0x0000006C}, {0x00001E3B, 0x00000331}, {0x00001E3C, 0x0000004C}, {0x00001E3C, 0x0000032D}, +{0x00001E3D, 0x0000006C}, {0x00001E3D, 0x0000032D}, {0x00001E3E, 0x0000004D}, {0x00001E3E, 0x00000301}, +{0x00001E3F, 0x0000006D}, {0x00001E3F, 0x00000301}, {0x00001E40, 0x0000004D}, {0x00001E40, 0x00000307}, +{0x00001E41, 0x0000006D}, {0x00001E41, 0x00000307}, {0x00001E42, 0x0000004D}, {0x00001E42, 0x00000323}, +{0x00001E43, 0x0000006D}, {0x00001E43, 0x00000323}, {0x00001E44, 0x0000004E}, {0x00001E44, 0x00000307}, +{0x00001E45, 0x0000006E}, {0x00001E45, 0x00000307}, {0x00001E46, 0x0000004E}, {0x00001E46, 0x00000323}, +{0x00001E47, 0x0000006E}, {0x00001E47, 0x00000323}, {0x00001E48, 0x0000004E}, {0x00001E48, 0x00000331}, +{0x00001E49, 0x0000006E}, {0x00001E49, 0x00000331}, {0x00001E4A, 0x0000004E}, {0x00001E4A, 0x0000032D}, +{0x00001E4B, 0x0000006E}, {0x00001E4B, 0x0000032D}, {0x00001E4C, 0x0000004F}, {0x00001E4C, 0x00000303}, +{0x00001E4C, 0x00000301}, {0x00001E4D, 0x0000006F}, {0x00001E4D, 0x00000303}, {0x00001E4D, 0x00000301}, +{0x00001E4E, 0x0000004F}, {0x00001E4E, 0x00000303}, {0x00001E4E, 0x00000308}, {0x00001E4F, 0x0000006F}, +{0x00001E4F, 0x00000303}, {0x00001E4F, 0x00000308}, {0x00001E50, 0x0000004F}, {0x00001E50, 0x00000304}, +{0x00001E50, 0x00000300}, {0x00001E51, 0x0000006F}, {0x00001E51, 0x00000304}, {0x00001E51, 0x00000300}, +{0x00001E52, 0x0000004F}, {0x00001E52, 0x00000304}, {0x00001E52, 0x00000301}, {0x00001E53, 0x0000006F}, +{0x00001E53, 0x00000304}, {0x00001E53, 0x00000301}, {0x00001E54, 0x00000050}, {0x00001E54, 0x00000301}, +{0x00001E55, 0x00000070}, {0x00001E55, 0x00000301}, {0x00001E56, 0x00000050}, {0x00001E56, 0x00000307}, +{0x00001E57, 0x00000070}, {0x00001E57, 0x00000307}, {0x00001E58, 0x00000052}, {0x00001E58, 0x00000307}, +{0x00001E59, 0x00000072}, {0x00001E59, 0x00000307}, {0x00001E5A, 0x00000052}, {0x00001E5A, 0x00000323}, +{0x00001E5B, 0x00000072}, {0x00001E5B, 0x00000323}, {0x00001E5C, 0x00000052}, {0x00001E5C, 0x00000323}, +{0x00001E5C, 0x00000304}, {0x00001E5D, 0x00000072}, {0x00001E5D, 0x00000323}, {0x00001E5D, 0x00000304}, +{0x00001E5E, 0x00000052}, {0x00001E5E, 0x00000331}, {0x00001E5F, 0x00000072}, {0x00001E5F, 0x00000331}, +{0x00001E60, 0x00000053}, {0x00001E60, 0x00000307}, {0x00001E61, 0x00000073}, {0x00001E61, 0x00000307}, +{0x00001E62, 0x00000053}, {0x00001E62, 0x00000323}, {0x00001E63, 0x00000073}, {0x00001E63, 0x00000323}, +{0x00001E64, 0x00000053}, {0x00001E64, 0x00000301}, {0x00001E64, 0x00000307}, {0x00001E65, 0x00000073}, +{0x00001E65, 0x00000301}, {0x00001E65, 0x00000307}, {0x00001E66, 0x00000053}, {0x00001E66, 0x0000030C}, +{0x00001E66, 0x00000307}, {0x00001E67, 0x00000073}, {0x00001E67, 0x0000030C}, {0x00001E67, 0x00000307}, +{0x00001E68, 0x00000053}, {0x00001E68, 0x00000323}, {0x00001E68, 0x00000307}, {0x00001E69, 0x00000073}, +{0x00001E69, 0x00000323}, {0x00001E69, 0x00000307}, {0x00001E6A, 0x00000054}, {0x00001E6A, 0x00000307}, +{0x00001E6B, 0x00000074}, {0x00001E6B, 0x00000307}, {0x00001E6C, 0x00000054}, {0x00001E6C, 0x00000323}, +{0x00001E6D, 0x00000074}, {0x00001E6D, 0x00000323}, {0x00001E6E, 0x00000054}, {0x00001E6E, 0x00000331}, +{0x00001E6F, 0x00000074}, {0x00001E6F, 0x00000331}, {0x00001E70, 0x00000054}, {0x00001E70, 0x0000032D}, +{0x00001E71, 0x00000074}, {0x00001E71, 0x0000032D}, {0x00001E72, 0x00000055}, {0x00001E72, 0x00000324}, +{0x00001E73, 0x00000075}, {0x00001E73, 0x00000324}, {0x00001E74, 0x00000055}, {0x00001E74, 0x00000330}, +{0x00001E75, 0x00000075}, {0x00001E75, 0x00000330}, {0x00001E76, 0x00000055}, {0x00001E76, 0x0000032D}, +{0x00001E77, 0x00000075}, {0x00001E77, 0x0000032D}, {0x00001E78, 0x00000055}, {0x00001E78, 0x00000303}, +{0x00001E78, 0x00000301}, {0x00001E79, 0x00000075}, {0x00001E79, 0x00000303}, {0x00001E79, 0x00000301}, +{0x00001E7A, 0x00000055}, {0x00001E7A, 0x00000304}, {0x00001E7A, 0x00000308}, {0x00001E7B, 0x00000075}, +{0x00001E7B, 0x00000304}, {0x00001E7B, 0x00000308}, {0x00001E7C, 0x00000056}, {0x00001E7C, 0x00000303}, +{0x00001E7D, 0x00000076}, {0x00001E7D, 0x00000303}, {0x00001E7E, 0x00000056}, {0x00001E7E, 0x00000323}, +{0x00001E7F, 0x00000076}, {0x00001E7F, 0x00000323}, {0x00001E80, 0x00000057}, {0x00001E80, 0x00000300}, +{0x00001E81, 0x00000077}, {0x00001E81, 0x00000300}, {0x00001E82, 0x00000057}, {0x00001E82, 0x00000301}, +{0x00001E83, 0x00000077}, {0x00001E83, 0x00000301}, {0x00001E84, 0x00000057}, {0x00001E84, 0x00000308}, +{0x00001E85, 0x00000077}, {0x00001E85, 0x00000308}, {0x00001E86, 0x00000057}, {0x00001E86, 0x00000307}, +{0x00001E87, 0x00000077}, {0x00001E87, 0x00000307}, {0x00001E88, 0x00000057}, {0x00001E88, 0x00000323}, +{0x00001E89, 0x00000077}, {0x00001E89, 0x00000323}, {0x00001E8A, 0x00000058}, {0x00001E8A, 0x00000307}, +{0x00001E8B, 0x00000078}, {0x00001E8B, 0x00000307}, {0x00001E8C, 0x00000058}, {0x00001E8C, 0x00000308}, +{0x00001E8D, 0x00000078}, {0x00001E8D, 0x00000308}, {0x00001E8E, 0x00000059}, {0x00001E8E, 0x00000307}, +{0x00001E8F, 0x00000079}, {0x00001E8F, 0x00000307}, {0x00001E90, 0x0000005A}, {0x00001E90, 0x00000302}, +{0x00001E91, 0x0000007A}, {0x00001E91, 0x00000302}, {0x00001E92, 0x0000005A}, {0x00001E92, 0x00000323}, +{0x00001E93, 0x0000007A}, {0x00001E93, 0x00000323}, {0x00001E94, 0x0000005A}, {0x00001E94, 0x00000331}, +{0x00001E95, 0x0000007A}, {0x00001E95, 0x00000331}, {0x00001E96, 0x00000068}, {0x00001E96, 0x00000331}, +{0x00001E97, 0x00000074}, {0x00001E97, 0x00000308}, {0x00001E98, 0x00000077}, {0x00001E98, 0x0000030A}, +{0x00001E99, 0x00000079}, {0x00001E99, 0x0000030A}, {0x00001E9B, 0x0000017F}, {0x00001E9B, 0x00000307}, +{0x00001EA0, 0x00000041}, {0x00001EA0, 0x00000323}, {0x00001EA1, 0x00000061}, {0x00001EA1, 0x00000323}, +{0x00001EA2, 0x00000041}, {0x00001EA2, 0x00000309}, {0x00001EA3, 0x00000061}, {0x00001EA3, 0x00000309}, +{0x00001EA4, 0x00000041}, {0x00001EA4, 0x00000302}, {0x00001EA4, 0x00000301}, {0x00001EA5, 0x00000061}, +{0x00001EA5, 0x00000302}, {0x00001EA5, 0x00000301}, {0x00001EA6, 0x00000041}, {0x00001EA6, 0x00000302}, +{0x00001EA6, 0x00000300}, {0x00001EA7, 0x00000061}, {0x00001EA7, 0x00000302}, {0x00001EA7, 0x00000300}, +{0x00001EA8, 0x00000041}, {0x00001EA8, 0x00000302}, {0x00001EA8, 0x00000309}, {0x00001EA9, 0x00000061}, +{0x00001EA9, 0x00000302}, {0x00001EA9, 0x00000309}, {0x00001EAA, 0x00000041}, {0x00001EAA, 0x00000302}, +{0x00001EAA, 0x00000303}, {0x00001EAB, 0x00000061}, {0x00001EAB, 0x00000302}, {0x00001EAB, 0x00000303}, +{0x00001EAC, 0x00000041}, {0x00001EAC, 0x00000323}, {0x00001EAC, 0x00000302}, {0x00001EAD, 0x00000061}, +{0x00001EAD, 0x00000323}, {0x00001EAD, 0x00000302}, {0x00001EAE, 0x00000041}, {0x00001EAE, 0x00000306}, +{0x00001EAE, 0x00000301}, {0x00001EAF, 0x00000061}, {0x00001EAF, 0x00000306}, {0x00001EAF, 0x00000301}, +{0x00001EB0, 0x00000041}, {0x00001EB0, 0x00000306}, {0x00001EB0, 0x00000300}, {0x00001EB1, 0x00000061}, +{0x00001EB1, 0x00000306}, {0x00001EB1, 0x00000300}, {0x00001EB2, 0x00000041}, {0x00001EB2, 0x00000306}, +{0x00001EB2, 0x00000309}, {0x00001EB3, 0x00000061}, {0x00001EB3, 0x00000306}, {0x00001EB3, 0x00000309}, +{0x00001EB4, 0x00000041}, {0x00001EB4, 0x00000306}, {0x00001EB4, 0x00000303}, {0x00001EB5, 0x00000061}, +{0x00001EB5, 0x00000306}, {0x00001EB5, 0x00000303}, {0x00001EB6, 0x00000041}, {0x00001EB6, 0x00000323}, +{0x00001EB6, 0x00000306}, {0x00001EB7, 0x00000061}, {0x00001EB7, 0x00000323}, {0x00001EB7, 0x00000306}, +{0x00001EB8, 0x00000045}, {0x00001EB8, 0x00000323}, {0x00001EB9, 0x00000065}, {0x00001EB9, 0x00000323}, +{0x00001EBA, 0x00000045}, {0x00001EBA, 0x00000309}, {0x00001EBB, 0x00000065}, {0x00001EBB, 0x00000309}, +{0x00001EBC, 0x00000045}, {0x00001EBC, 0x00000303}, {0x00001EBD, 0x00000065}, {0x00001EBD, 0x00000303}, +{0x00001EBE, 0x00000045}, {0x00001EBE, 0x00000302}, {0x00001EBE, 0x00000301}, {0x00001EBF, 0x00000065}, +{0x00001EBF, 0x00000302}, {0x00001EBF, 0x00000301}, {0x00001EC0, 0x00000045}, {0x00001EC0, 0x00000302}, +{0x00001EC0, 0x00000300}, {0x00001EC1, 0x00000065}, {0x00001EC1, 0x00000302}, {0x00001EC1, 0x00000300}, +{0x00001EC2, 0x00000045}, {0x00001EC2, 0x00000302}, {0x00001EC2, 0x00000309}, {0x00001EC3, 0x00000065}, +{0x00001EC3, 0x00000302}, {0x00001EC3, 0x00000309}, {0x00001EC4, 0x00000045}, {0x00001EC4, 0x00000302}, +{0x00001EC4, 0x00000303}, {0x00001EC5, 0x00000065}, {0x00001EC5, 0x00000302}, {0x00001EC5, 0x00000303}, +{0x00001EC6, 0x00000045}, {0x00001EC6, 0x00000323}, {0x00001EC6, 0x00000302}, {0x00001EC7, 0x00000065}, +{0x00001EC7, 0x00000323}, {0x00001EC7, 0x00000302}, {0x00001EC8, 0x00000049}, {0x00001EC8, 0x00000309}, +{0x00001EC9, 0x00000069}, {0x00001EC9, 0x00000309}, {0x00001ECA, 0x00000049}, {0x00001ECA, 0x00000323}, +{0x00001ECB, 0x00000069}, {0x00001ECB, 0x00000323}, {0x00001ECC, 0x0000004F}, {0x00001ECC, 0x00000323}, +{0x00001ECD, 0x0000006F}, {0x00001ECD, 0x00000323}, {0x00001ECE, 0x0000004F}, {0x00001ECE, 0x00000309}, +{0x00001ECF, 0x0000006F}, {0x00001ECF, 0x00000309}, {0x00001ED0, 0x0000004F}, {0x00001ED0, 0x00000302}, +{0x00001ED0, 0x00000301}, {0x00001ED1, 0x0000006F}, {0x00001ED1, 0x00000302}, {0x00001ED1, 0x00000301}, +{0x00001ED2, 0x0000004F}, {0x00001ED2, 0x00000302}, {0x00001ED2, 0x00000300}, {0x00001ED3, 0x0000006F}, +{0x00001ED3, 0x00000302}, {0x00001ED3, 0x00000300}, {0x00001ED4, 0x0000004F}, {0x00001ED4, 0x00000302}, +{0x00001ED4, 0x00000309}, {0x00001ED5, 0x0000006F}, {0x00001ED5, 0x00000302}, {0x00001ED5, 0x00000309}, +{0x00001ED6, 0x0000004F}, {0x00001ED6, 0x00000302}, {0x00001ED6, 0x00000303}, {0x00001ED7, 0x0000006F}, +{0x00001ED7, 0x00000302}, {0x00001ED7, 0x00000303}, {0x00001ED8, 0x0000004F}, {0x00001ED8, 0x00000323}, +{0x00001ED8, 0x00000302}, {0x00001ED9, 0x0000006F}, {0x00001ED9, 0x00000323}, {0x00001ED9, 0x00000302}, +{0x00001EDA, 0x0000004F}, {0x00001EDA, 0x0000031B}, {0x00001EDA, 0x00000301}, {0x00001EDB, 0x0000006F}, +{0x00001EDB, 0x0000031B}, {0x00001EDB, 0x00000301}, {0x00001EDC, 0x0000004F}, {0x00001EDC, 0x0000031B}, +{0x00001EDC, 0x00000300}, {0x00001EDD, 0x0000006F}, {0x00001EDD, 0x0000031B}, {0x00001EDD, 0x00000300}, +{0x00001EDE, 0x0000004F}, {0x00001EDE, 0x0000031B}, {0x00001EDE, 0x00000309}, {0x00001EDF, 0x0000006F}, +{0x00001EDF, 0x0000031B}, {0x00001EDF, 0x00000309}, {0x00001EE0, 0x0000004F}, {0x00001EE0, 0x0000031B}, +{0x00001EE0, 0x00000303}, {0x00001EE1, 0x0000006F}, {0x00001EE1, 0x0000031B}, {0x00001EE1, 0x00000303}, +{0x00001EE2, 0x0000004F}, {0x00001EE2, 0x0000031B}, {0x00001EE2, 0x00000323}, {0x00001EE3, 0x0000006F}, +{0x00001EE3, 0x0000031B}, {0x00001EE3, 0x00000323}, {0x00001EE4, 0x00000055}, {0x00001EE4, 0x00000323}, +{0x00001EE5, 0x00000075}, {0x00001EE5, 0x00000323}, {0x00001EE6, 0x00000055}, {0x00001EE6, 0x00000309}, +{0x00001EE7, 0x00000075}, {0x00001EE7, 0x00000309}, {0x00001EE8, 0x00000055}, {0x00001EE8, 0x0000031B}, +{0x00001EE8, 0x00000301}, {0x00001EE9, 0x00000075}, {0x00001EE9, 0x0000031B}, {0x00001EE9, 0x00000301}, +{0x00001EEA, 0x00000055}, {0x00001EEA, 0x0000031B}, {0x00001EEA, 0x00000300}, {0x00001EEB, 0x00000075}, +{0x00001EEB, 0x0000031B}, {0x00001EEB, 0x00000300}, {0x00001EEC, 0x00000055}, {0x00001EEC, 0x0000031B}, +{0x00001EEC, 0x00000309}, {0x00001EED, 0x00000075}, {0x00001EED, 0x0000031B}, {0x00001EED, 0x00000309}, +{0x00001EEE, 0x00000055}, {0x00001EEE, 0x0000031B}, {0x00001EEE, 0x00000303}, {0x00001EEF, 0x00000075}, +{0x00001EEF, 0x0000031B}, {0x00001EEF, 0x00000303}, {0x00001EF0, 0x00000055}, {0x00001EF0, 0x0000031B}, +{0x00001EF0, 0x00000323}, {0x00001EF1, 0x00000075}, {0x00001EF1, 0x0000031B}, {0x00001EF1, 0x00000323}, +{0x00001EF2, 0x00000059}, {0x00001EF2, 0x00000300}, {0x00001EF3, 0x00000079}, {0x00001EF3, 0x00000300}, +{0x00001EF4, 0x00000059}, {0x00001EF4, 0x00000323}, {0x00001EF5, 0x00000079}, {0x00001EF5, 0x00000323}, +{0x00001EF6, 0x00000059}, {0x00001EF6, 0x00000309}, {0x00001EF7, 0x00000079}, {0x00001EF7, 0x00000309}, +{0x00001EF8, 0x00000059}, {0x00001EF8, 0x00000303}, {0x00001EF9, 0x00000079}, {0x00001EF9, 0x00000303}, +{0x00001F00, 0x000003B1}, {0x00001F00, 0x00000313}, {0x00001F01, 0x000003B1}, {0x00001F01, 0x00000314}, +{0x00001F02, 0x000003B1}, {0x00001F02, 0x00000313}, {0x00001F02, 0x00000300}, {0x00001F03, 0x000003B1}, +{0x00001F03, 0x00000314}, {0x00001F03, 0x00000300}, {0x00001F04, 0x000003B1}, {0x00001F04, 0x00000313}, +{0x00001F04, 0x00000301}, {0x00001F05, 0x000003B1}, {0x00001F05, 0x00000314}, {0x00001F05, 0x00000301}, +{0x00001F06, 0x000003B1}, {0x00001F06, 0x00000313}, {0x00001F06, 0x00000342}, {0x00001F07, 0x000003B1}, +{0x00001F07, 0x00000314}, {0x00001F07, 0x00000342}, {0x00001F08, 0x00000391}, {0x00001F08, 0x00000313}, +{0x00001F09, 0x00000391}, {0x00001F09, 0x00000314}, {0x00001F0A, 0x00000391}, {0x00001F0A, 0x00000313}, +{0x00001F0A, 0x00000300}, {0x00001F0B, 0x00000391}, {0x00001F0B, 0x00000314}, {0x00001F0B, 0x00000300}, +{0x00001F0C, 0x00000391}, {0x00001F0C, 0x00000313}, {0x00001F0C, 0x00000301}, {0x00001F0D, 0x00000391}, +{0x00001F0D, 0x00000314}, {0x00001F0D, 0x00000301}, {0x00001F0E, 0x00000391}, {0x00001F0E, 0x00000313}, +{0x00001F0E, 0x00000342}, {0x00001F0F, 0x00000391}, {0x00001F0F, 0x00000314}, {0x00001F0F, 0x00000342}, +{0x00001F10, 0x000003B5}, {0x00001F10, 0x00000313}, {0x00001F11, 0x000003B5}, {0x00001F11, 0x00000314}, +{0x00001F12, 0x000003B5}, {0x00001F12, 0x00000313}, {0x00001F12, 0x00000300}, {0x00001F13, 0x000003B5}, +{0x00001F13, 0x00000314}, {0x00001F13, 0x00000300}, {0x00001F14, 0x000003B5}, {0x00001F14, 0x00000313}, +{0x00001F14, 0x00000301}, {0x00001F15, 0x000003B5}, {0x00001F15, 0x00000314}, {0x00001F15, 0x00000301}, +{0x00001F18, 0x00000395}, {0x00001F18, 0x00000313}, {0x00001F19, 0x00000395}, {0x00001F19, 0x00000314}, +{0x00001F1A, 0x00000395}, {0x00001F1A, 0x00000313}, {0x00001F1A, 0x00000300}, {0x00001F1B, 0x00000395}, +{0x00001F1B, 0x00000314}, {0x00001F1B, 0x00000300}, {0x00001F1C, 0x00000395}, {0x00001F1C, 0x00000313}, +{0x00001F1C, 0x00000301}, {0x00001F1D, 0x00000395}, {0x00001F1D, 0x00000314}, {0x00001F1D, 0x00000301}, +{0x00001F20, 0x000003B7}, {0x00001F20, 0x00000313}, {0x00001F21, 0x000003B7}, {0x00001F21, 0x00000314}, +{0x00001F22, 0x000003B7}, {0x00001F22, 0x00000313}, {0x00001F22, 0x00000300}, {0x00001F23, 0x000003B7}, +{0x00001F23, 0x00000314}, {0x00001F23, 0x00000300}, {0x00001F24, 0x000003B7}, {0x00001F24, 0x00000313}, +{0x00001F24, 0x00000301}, {0x00001F25, 0x000003B7}, {0x00001F25, 0x00000314}, {0x00001F25, 0x00000301}, +{0x00001F26, 0x000003B7}, {0x00001F26, 0x00000313}, {0x00001F26, 0x00000342}, {0x00001F27, 0x000003B7}, +{0x00001F27, 0x00000314}, {0x00001F27, 0x00000342}, {0x00001F28, 0x00000397}, {0x00001F28, 0x00000313}, +{0x00001F29, 0x00000397}, {0x00001F29, 0x00000314}, {0x00001F2A, 0x00000397}, {0x00001F2A, 0x00000313}, +{0x00001F2A, 0x00000300}, {0x00001F2B, 0x00000397}, {0x00001F2B, 0x00000314}, {0x00001F2B, 0x00000300}, +{0x00001F2C, 0x00000397}, {0x00001F2C, 0x00000313}, {0x00001F2C, 0x00000301}, {0x00001F2D, 0x00000397}, +{0x00001F2D, 0x00000314}, {0x00001F2D, 0x00000301}, {0x00001F2E, 0x00000397}, {0x00001F2E, 0x00000313}, +{0x00001F2E, 0x00000342}, {0x00001F2F, 0x00000397}, {0x00001F2F, 0x00000314}, {0x00001F2F, 0x00000342}, +{0x00001F30, 0x000003B9}, {0x00001F30, 0x00000313}, {0x00001F31, 0x000003B9}, {0x00001F31, 0x00000314}, +{0x00001F32, 0x000003B9}, {0x00001F32, 0x00000313}, {0x00001F32, 0x00000300}, {0x00001F33, 0x000003B9}, +{0x00001F33, 0x00000314}, {0x00001F33, 0x00000300}, {0x00001F34, 0x000003B9}, {0x00001F34, 0x00000313}, +{0x00001F34, 0x00000301}, {0x00001F35, 0x000003B9}, {0x00001F35, 0x00000314}, {0x00001F35, 0x00000301}, +{0x00001F36, 0x000003B9}, {0x00001F36, 0x00000313}, {0x00001F36, 0x00000342}, {0x00001F37, 0x000003B9}, +{0x00001F37, 0x00000314}, {0x00001F37, 0x00000342}, {0x00001F38, 0x00000399}, {0x00001F38, 0x00000313}, +{0x00001F39, 0x00000399}, {0x00001F39, 0x00000314}, {0x00001F3A, 0x00000399}, {0x00001F3A, 0x00000313}, +{0x00001F3A, 0x00000300}, {0x00001F3B, 0x00000399}, {0x00001F3B, 0x00000314}, {0x00001F3B, 0x00000300}, +{0x00001F3C, 0x00000399}, {0x00001F3C, 0x00000313}, {0x00001F3C, 0x00000301}, {0x00001F3D, 0x00000399}, +{0x00001F3D, 0x00000314}, {0x00001F3D, 0x00000301}, {0x00001F3E, 0x00000399}, {0x00001F3E, 0x00000313}, +{0x00001F3E, 0x00000342}, {0x00001F3F, 0x00000399}, {0x00001F3F, 0x00000314}, {0x00001F3F, 0x00000342}, +{0x00001F40, 0x000003BF}, {0x00001F40, 0x00000313}, {0x00001F41, 0x000003BF}, {0x00001F41, 0x00000314}, +{0x00001F42, 0x000003BF}, {0x00001F42, 0x00000313}, {0x00001F42, 0x00000300}, {0x00001F43, 0x000003BF}, +{0x00001F43, 0x00000314}, {0x00001F43, 0x00000300}, {0x00001F44, 0x000003BF}, {0x00001F44, 0x00000313}, +{0x00001F44, 0x00000301}, {0x00001F45, 0x000003BF}, {0x00001F45, 0x00000314}, {0x00001F45, 0x00000301}, +{0x00001F48, 0x0000039F}, {0x00001F48, 0x00000313}, {0x00001F49, 0x0000039F}, {0x00001F49, 0x00000314}, +{0x00001F4A, 0x0000039F}, {0x00001F4A, 0x00000313}, {0x00001F4A, 0x00000300}, {0x00001F4B, 0x0000039F}, +{0x00001F4B, 0x00000314}, {0x00001F4B, 0x00000300}, {0x00001F4C, 0x0000039F}, {0x00001F4C, 0x00000313}, +{0x00001F4C, 0x00000301}, {0x00001F4D, 0x0000039F}, {0x00001F4D, 0x00000314}, {0x00001F4D, 0x00000301}, +{0x00001F50, 0x000003C5}, {0x00001F50, 0x00000313}, {0x00001F51, 0x000003C5}, {0x00001F51, 0x00000314}, +{0x00001F52, 0x000003C5}, {0x00001F52, 0x00000313}, {0x00001F52, 0x00000300}, {0x00001F53, 0x000003C5}, +{0x00001F53, 0x00000314}, {0x00001F53, 0x00000300}, {0x00001F54, 0x000003C5}, {0x00001F54, 0x00000313}, +{0x00001F54, 0x00000301}, {0x00001F55, 0x000003C5}, {0x00001F55, 0x00000314}, {0x00001F55, 0x00000301}, +{0x00001F56, 0x000003C5}, {0x00001F56, 0x00000313}, {0x00001F56, 0x00000342}, {0x00001F57, 0x000003C5}, +{0x00001F57, 0x00000314}, {0x00001F57, 0x00000342}, {0x00001F59, 0x000003A5}, {0x00001F59, 0x00000314}, +{0x00001F5B, 0x000003A5}, {0x00001F5B, 0x00000314}, {0x00001F5B, 0x00000300}, {0x00001F5D, 0x000003A5}, +{0x00001F5D, 0x00000314}, {0x00001F5D, 0x00000301}, {0x00001F5F, 0x000003A5}, {0x00001F5F, 0x00000314}, +{0x00001F5F, 0x00000342}, {0x00001F60, 0x000003C9}, {0x00001F60, 0x00000313}, {0x00001F61, 0x000003C9}, +{0x00001F61, 0x00000314}, {0x00001F62, 0x000003C9}, {0x00001F62, 0x00000313}, {0x00001F62, 0x00000300}, +{0x00001F63, 0x000003C9}, {0x00001F63, 0x00000314}, {0x00001F63, 0x00000300}, {0x00001F64, 0x000003C9}, +{0x00001F64, 0x00000313}, {0x00001F64, 0x00000301}, {0x00001F65, 0x000003C9}, {0x00001F65, 0x00000314}, +{0x00001F65, 0x00000301}, {0x00001F66, 0x000003C9}, {0x00001F66, 0x00000313}, {0x00001F66, 0x00000342}, +{0x00001F67, 0x000003C9}, {0x00001F67, 0x00000314}, {0x00001F67, 0x00000342}, {0x00001F68, 0x000003A9}, +{0x00001F68, 0x00000313}, {0x00001F69, 0x000003A9}, {0x00001F69, 0x00000314}, {0x00001F6A, 0x000003A9}, +{0x00001F6A, 0x00000313}, {0x00001F6A, 0x00000300}, {0x00001F6B, 0x000003A9}, {0x00001F6B, 0x00000314}, +{0x00001F6B, 0x00000300}, {0x00001F6C, 0x000003A9}, {0x00001F6C, 0x00000313}, {0x00001F6C, 0x00000301}, +{0x00001F6D, 0x000003A9}, {0x00001F6D, 0x00000314}, {0x00001F6D, 0x00000301}, {0x00001F6E, 0x000003A9}, +{0x00001F6E, 0x00000313}, {0x00001F6E, 0x00000342}, {0x00001F6F, 0x000003A9}, {0x00001F6F, 0x00000314}, +{0x00001F6F, 0x00000342}, {0x00001F70, 0x000003B1}, {0x00001F70, 0x00000300}, {0x00001F71, 0x000003B1}, +{0x00001F71, 0x00000301}, {0x00001F72, 0x000003B5}, {0x00001F72, 0x00000300}, {0x00001F73, 0x000003B5}, +{0x00001F73, 0x00000301}, {0x00001F74, 0x000003B7}, {0x00001F74, 0x00000300}, {0x00001F75, 0x000003B7}, +{0x00001F75, 0x00000301}, {0x00001F76, 0x000003B9}, {0x00001F76, 0x00000300}, {0x00001F77, 0x000003B9}, +{0x00001F77, 0x00000301}, {0x00001F78, 0x000003BF}, {0x00001F78, 0x00000300}, {0x00001F79, 0x000003BF}, +{0x00001F79, 0x00000301}, {0x00001F7A, 0x000003C5}, {0x00001F7A, 0x00000300}, {0x00001F7B, 0x000003C5}, +{0x00001F7B, 0x00000301}, {0x00001F7C, 0x000003C9}, {0x00001F7C, 0x00000300}, {0x00001F7D, 0x000003C9}, +{0x00001F7D, 0x00000301}, {0x00001F80, 0x000003B1}, {0x00001F80, 0x00000313}, {0x00001F80, 0x00000345}, +{0x00001F81, 0x000003B1}, {0x00001F81, 0x00000314}, {0x00001F81, 0x00000345}, {0x00001F82, 0x000003B1}, +{0x00001F82, 0x00000313}, {0x00001F82, 0x00000300}, {0x00001F82, 0x00000345}, {0x00001F83, 0x000003B1}, +{0x00001F83, 0x00000314}, {0x00001F83, 0x00000300}, {0x00001F83, 0x00000345}, {0x00001F84, 0x000003B1}, +{0x00001F84, 0x00000313}, {0x00001F84, 0x00000301}, {0x00001F84, 0x00000345}, {0x00001F85, 0x000003B1}, +{0x00001F85, 0x00000314}, {0x00001F85, 0x00000301}, {0x00001F85, 0x00000345}, {0x00001F86, 0x000003B1}, +{0x00001F86, 0x00000313}, {0x00001F86, 0x00000342}, {0x00001F86, 0x00000345}, {0x00001F87, 0x000003B1}, +{0x00001F87, 0x00000314}, {0x00001F87, 0x00000342}, {0x00001F87, 0x00000345}, {0x00001F88, 0x00000391}, +{0x00001F88, 0x00000313}, {0x00001F88, 0x00000345}, {0x00001F89, 0x00000391}, {0x00001F89, 0x00000314}, +{0x00001F89, 0x00000345}, {0x00001F8A, 0x00000391}, {0x00001F8A, 0x00000313}, {0x00001F8A, 0x00000300}, +{0x00001F8A, 0x00000345}, {0x00001F8B, 0x00000391}, {0x00001F8B, 0x00000314}, {0x00001F8B, 0x00000300}, +{0x00001F8B, 0x00000345}, {0x00001F8C, 0x00000391}, {0x00001F8C, 0x00000313}, {0x00001F8C, 0x00000301}, +{0x00001F8C, 0x00000345}, {0x00001F8D, 0x00000391}, {0x00001F8D, 0x00000314}, {0x00001F8D, 0x00000301}, +{0x00001F8D, 0x00000345}, {0x00001F8E, 0x00000391}, {0x00001F8E, 0x00000313}, {0x00001F8E, 0x00000342}, +{0x00001F8E, 0x00000345}, {0x00001F8F, 0x00000391}, {0x00001F8F, 0x00000314}, {0x00001F8F, 0x00000342}, +{0x00001F8F, 0x00000345}, {0x00001F90, 0x000003B7}, {0x00001F90, 0x00000313}, {0x00001F90, 0x00000345}, +{0x00001F91, 0x000003B7}, {0x00001F91, 0x00000314}, {0x00001F91, 0x00000345}, {0x00001F92, 0x000003B7}, +{0x00001F92, 0x00000313}, {0x00001F92, 0x00000300}, {0x00001F92, 0x00000345}, {0x00001F93, 0x000003B7}, +{0x00001F93, 0x00000314}, {0x00001F93, 0x00000300}, {0x00001F93, 0x00000345}, {0x00001F94, 0x000003B7}, +{0x00001F94, 0x00000313}, {0x00001F94, 0x00000301}, {0x00001F94, 0x00000345}, {0x00001F95, 0x000003B7}, +{0x00001F95, 0x00000314}, {0x00001F95, 0x00000301}, {0x00001F95, 0x00000345}, {0x00001F96, 0x000003B7}, +{0x00001F96, 0x00000313}, {0x00001F96, 0x00000342}, {0x00001F96, 0x00000345}, {0x00001F97, 0x000003B7}, +{0x00001F97, 0x00000314}, {0x00001F97, 0x00000342}, {0x00001F97, 0x00000345}, {0x00001F98, 0x00000397}, +{0x00001F98, 0x00000313}, {0x00001F98, 0x00000345}, {0x00001F99, 0x00000397}, {0x00001F99, 0x00000314}, +{0x00001F99, 0x00000345}, {0x00001F9A, 0x00000397}, {0x00001F9A, 0x00000313}, {0x00001F9A, 0x00000300}, +{0x00001F9A, 0x00000345}, {0x00001F9B, 0x00000397}, {0x00001F9B, 0x00000314}, {0x00001F9B, 0x00000300}, +{0x00001F9B, 0x00000345}, {0x00001F9C, 0x00000397}, {0x00001F9C, 0x00000313}, {0x00001F9C, 0x00000301}, +{0x00001F9C, 0x00000345}, {0x00001F9D, 0x00000397}, {0x00001F9D, 0x00000314}, {0x00001F9D, 0x00000301}, +{0x00001F9D, 0x00000345}, {0x00001F9E, 0x00000397}, {0x00001F9E, 0x00000313}, {0x00001F9E, 0x00000342}, +{0x00001F9E, 0x00000345}, {0x00001F9F, 0x00000397}, {0x00001F9F, 0x00000314}, {0x00001F9F, 0x00000342}, +{0x00001F9F, 0x00000345}, {0x00001FA0, 0x000003C9}, {0x00001FA0, 0x00000313}, {0x00001FA0, 0x00000345}, +{0x00001FA1, 0x000003C9}, {0x00001FA1, 0x00000314}, {0x00001FA1, 0x00000345}, {0x00001FA2, 0x000003C9}, +{0x00001FA2, 0x00000313}, {0x00001FA2, 0x00000300}, {0x00001FA2, 0x00000345}, {0x00001FA3, 0x000003C9}, +{0x00001FA3, 0x00000314}, {0x00001FA3, 0x00000300}, {0x00001FA3, 0x00000345}, {0x00001FA4, 0x000003C9}, +{0x00001FA4, 0x00000313}, {0x00001FA4, 0x00000301}, {0x00001FA4, 0x00000345}, {0x00001FA5, 0x000003C9}, +{0x00001FA5, 0x00000314}, {0x00001FA5, 0x00000301}, {0x00001FA5, 0x00000345}, {0x00001FA6, 0x000003C9}, +{0x00001FA6, 0x00000313}, {0x00001FA6, 0x00000342}, {0x00001FA6, 0x00000345}, {0x00001FA7, 0x000003C9}, +{0x00001FA7, 0x00000314}, {0x00001FA7, 0x00000342}, {0x00001FA7, 0x00000345}, {0x00001FA8, 0x000003A9}, +{0x00001FA8, 0x00000313}, {0x00001FA8, 0x00000345}, {0x00001FA9, 0x000003A9}, {0x00001FA9, 0x00000314}, +{0x00001FA9, 0x00000345}, {0x00001FAA, 0x000003A9}, {0x00001FAA, 0x00000313}, {0x00001FAA, 0x00000300}, +{0x00001FAA, 0x00000345}, {0x00001FAB, 0x000003A9}, {0x00001FAB, 0x00000314}, {0x00001FAB, 0x00000300}, +{0x00001FAB, 0x00000345}, {0x00001FAC, 0x000003A9}, {0x00001FAC, 0x00000313}, {0x00001FAC, 0x00000301}, +{0x00001FAC, 0x00000345}, {0x00001FAD, 0x000003A9}, {0x00001FAD, 0x00000314}, {0x00001FAD, 0x00000301}, +{0x00001FAD, 0x00000345}, {0x00001FAE, 0x000003A9}, {0x00001FAE, 0x00000313}, {0x00001FAE, 0x00000342}, +{0x00001FAE, 0x00000345}, {0x00001FAF, 0x000003A9}, {0x00001FAF, 0x00000314}, {0x00001FAF, 0x00000342}, +{0x00001FAF, 0x00000345}, {0x00001FB0, 0x000003B1}, {0x00001FB0, 0x00000306}, {0x00001FB1, 0x000003B1}, +{0x00001FB1, 0x00000304}, {0x00001FB2, 0x000003B1}, {0x00001FB2, 0x00000300}, {0x00001FB2, 0x00000345}, +{0x00001FB3, 0x000003B1}, {0x00001FB3, 0x00000345}, {0x00001FB4, 0x000003B1}, {0x00001FB4, 0x00000301}, +{0x00001FB4, 0x00000345}, {0x00001FB6, 0x000003B1}, {0x00001FB6, 0x00000342}, {0x00001FB7, 0x000003B1}, +{0x00001FB7, 0x00000342}, {0x00001FB7, 0x00000345}, {0x00001FB8, 0x00000391}, {0x00001FB8, 0x00000306}, +{0x00001FB9, 0x00000391}, {0x00001FB9, 0x00000304}, {0x00001FBA, 0x00000391}, {0x00001FBA, 0x00000300}, +{0x00001FBB, 0x00000391}, {0x00001FBB, 0x00000301}, {0x00001FBC, 0x00000391}, {0x00001FBC, 0x00000345}, +{0x00001FBE, 0x000003B9}, {0x00001FC1, 0x000000A8}, {0x00001FC1, 0x00000342}, {0x00001FC2, 0x000003B7}, +{0x00001FC2, 0x00000300}, {0x00001FC2, 0x00000345}, {0x00001FC3, 0x000003B7}, {0x00001FC3, 0x00000345}, +{0x00001FC4, 0x000003B7}, {0x00001FC4, 0x00000301}, {0x00001FC4, 0x00000345}, {0x00001FC6, 0x000003B7}, +{0x00001FC6, 0x00000342}, {0x00001FC7, 0x000003B7}, {0x00001FC7, 0x00000342}, {0x00001FC7, 0x00000345}, +{0x00001FC8, 0x00000395}, {0x00001FC8, 0x00000300}, {0x00001FC9, 0x00000395}, {0x00001FC9, 0x00000301}, +{0x00001FCA, 0x00000397}, {0x00001FCA, 0x00000300}, {0x00001FCB, 0x00000397}, {0x00001FCB, 0x00000301}, +{0x00001FCC, 0x00000397}, {0x00001FCC, 0x00000345}, {0x00001FCD, 0x00001FBF}, {0x00001FCD, 0x00000300}, +{0x00001FCE, 0x00001FBF}, {0x00001FCE, 0x00000301}, {0x00001FCF, 0x00001FBF}, {0x00001FCF, 0x00000342}, +{0x00001FD0, 0x000003B9}, {0x00001FD0, 0x00000306}, {0x00001FD1, 0x000003B9}, {0x00001FD1, 0x00000304}, +{0x00001FD2, 0x000003B9}, {0x00001FD2, 0x00000308}, {0x00001FD2, 0x00000300}, {0x00001FD3, 0x000003B9}, +{0x00001FD3, 0x00000308}, {0x00001FD3, 0x00000301}, {0x00001FD6, 0x000003B9}, {0x00001FD6, 0x00000342}, +{0x00001FD7, 0x000003B9}, {0x00001FD7, 0x00000308}, {0x00001FD7, 0x00000342}, {0x00001FD8, 0x00000399}, +{0x00001FD8, 0x00000306}, {0x00001FD9, 0x00000399}, {0x00001FD9, 0x00000304}, {0x00001FDA, 0x00000399}, +{0x00001FDA, 0x00000300}, {0x00001FDB, 0x00000399}, {0x00001FDB, 0x00000301}, {0x00001FDD, 0x00001FFE}, +{0x00001FDD, 0x00000300}, {0x00001FDE, 0x00001FFE}, {0x00001FDE, 0x00000301}, {0x00001FDF, 0x00001FFE}, +{0x00001FDF, 0x00000342}, {0x00001FE0, 0x000003C5}, {0x00001FE0, 0x00000306}, {0x00001FE1, 0x000003C5}, +{0x00001FE1, 0x00000304}, {0x00001FE2, 0x000003C5}, {0x00001FE2, 0x00000308}, {0x00001FE2, 0x00000300}, +{0x00001FE3, 0x000003C5}, {0x00001FE3, 0x00000308}, {0x00001FE3, 0x00000301}, {0x00001FE4, 0x000003C1}, +{0x00001FE4, 0x00000313}, {0x00001FE5, 0x000003C1}, {0x00001FE5, 0x00000314}, {0x00001FE6, 0x000003C5}, +{0x00001FE6, 0x00000342}, {0x00001FE7, 0x000003C5}, {0x00001FE7, 0x00000308}, {0x00001FE7, 0x00000342}, +{0x00001FE8, 0x000003A5}, {0x00001FE8, 0x00000306}, {0x00001FE9, 0x000003A5}, {0x00001FE9, 0x00000304}, +{0x00001FEA, 0x000003A5}, {0x00001FEA, 0x00000300}, {0x00001FEB, 0x000003A5}, {0x00001FEB, 0x00000301}, +{0x00001FEC, 0x000003A1}, {0x00001FEC, 0x00000314}, {0x00001FED, 0x000000A8}, {0x00001FED, 0x00000300}, +{0x00001FEE, 0x000000A8}, {0x00001FEE, 0x00000301}, {0x00001FEF, 0x00000060}, {0x00001FF2, 0x000003C9}, +{0x00001FF2, 0x00000300}, {0x00001FF2, 0x00000345}, {0x00001FF3, 0x000003C9}, {0x00001FF3, 0x00000345}, +{0x00001FF4, 0x000003C9}, {0x00001FF4, 0x00000301}, {0x00001FF4, 0x00000345}, {0x00001FF6, 0x000003C9}, +{0x00001FF6, 0x00000342}, {0x00001FF7, 0x000003C9}, {0x00001FF7, 0x00000342}, {0x00001FF7, 0x00000345}, +{0x00001FF8, 0x0000039F}, {0x00001FF8, 0x00000300}, {0x00001FF9, 0x0000039F}, {0x00001FF9, 0x00000301}, +{0x00001FFA, 0x000003A9}, {0x00001FFA, 0x00000300}, {0x00001FFB, 0x000003A9}, {0x00001FFB, 0x00000301}, +{0x00001FFC, 0x000003A9}, {0x00001FFC, 0x00000345}, {0x00001FFD, 0x000000B4}, {0x00002000, 0x00002002}, +{0x00002001, 0x00002003}, {0x00002126, 0x000003A9}, {0x0000212A, 0x0000004B}, {0x0000212B, 0x00000041}, +{0x0000212B, 0x0000030A}, {0x0000219A, 0x00002190}, {0x0000219A, 0x00000338}, {0x0000219B, 0x00002192}, +{0x0000219B, 0x00000338}, {0x000021AE, 0x00002194}, {0x000021AE, 0x00000338}, {0x000021CD, 0x000021D0}, +{0x000021CD, 0x00000338}, {0x000021CE, 0x000021D4}, {0x000021CE, 0x00000338}, {0x000021CF, 0x000021D2}, +{0x000021CF, 0x00000338}, {0x00002204, 0x00002203}, {0x00002204, 0x00000338}, {0x00002209, 0x00002208}, +{0x00002209, 0x00000338}, {0x0000220C, 0x0000220B}, {0x0000220C, 0x00000338}, {0x00002224, 0x00002223}, +{0x00002224, 0x00000338}, {0x00002226, 0x00002225}, {0x00002226, 0x00000338}, {0x00002241, 0x0000223C}, +{0x00002241, 0x00000338}, {0x00002244, 0x00002243}, {0x00002244, 0x00000338}, {0x00002247, 0x00002245}, +{0x00002247, 0x00000338}, {0x00002249, 0x00002248}, {0x00002249, 0x00000338}, {0x00002260, 0x0000003D}, +{0x00002260, 0x00000338}, {0x00002262, 0x00002261}, {0x00002262, 0x00000338}, {0x0000226D, 0x0000224D}, +{0x0000226D, 0x00000338}, {0x0000226E, 0x0000003C}, {0x0000226E, 0x00000338}, {0x0000226F, 0x0000003E}, +{0x0000226F, 0x00000338}, {0x00002270, 0x00002264}, {0x00002270, 0x00000338}, {0x00002271, 0x00002265}, +{0x00002271, 0x00000338}, {0x00002274, 0x00002272}, {0x00002274, 0x00000338}, {0x00002275, 0x00002273}, +{0x00002275, 0x00000338}, {0x00002278, 0x00002276}, {0x00002278, 0x00000338}, {0x00002279, 0x00002277}, +{0x00002279, 0x00000338}, {0x00002280, 0x0000227A}, {0x00002280, 0x00000338}, {0x00002281, 0x0000227B}, +{0x00002281, 0x00000338}, {0x00002284, 0x00002282}, {0x00002284, 0x00000338}, {0x00002285, 0x00002283}, +{0x00002285, 0x00000338}, {0x00002288, 0x00002286}, {0x00002288, 0x00000338}, {0x00002289, 0x00002287}, +{0x00002289, 0x00000338}, {0x000022AC, 0x000022A2}, {0x000022AC, 0x00000338}, {0x000022AD, 0x000022A8}, +{0x000022AD, 0x00000338}, {0x000022AE, 0x000022A9}, {0x000022AE, 0x00000338}, {0x000022AF, 0x000022AB}, +{0x000022AF, 0x00000338}, {0x000022E0, 0x0000227C}, {0x000022E0, 0x00000338}, {0x000022E1, 0x0000227D}, +{0x000022E1, 0x00000338}, {0x000022E2, 0x00002291}, {0x000022E2, 0x00000338}, {0x000022E3, 0x00002292}, +{0x000022E3, 0x00000338}, {0x000022EA, 0x000022B2}, {0x000022EA, 0x00000338}, {0x000022EB, 0x000022B3}, +{0x000022EB, 0x00000338}, {0x000022EC, 0x000022B4}, {0x000022EC, 0x00000338}, {0x000022ED, 0x000022B5}, +{0x000022ED, 0x00000338}, {0x00002329, 0x00003008}, {0x0000232A, 0x00003009}, {0x00002ADC, 0x00002ADD}, +{0x00002ADC, 0x00000338}, {0x0000304C, 0x0000304B}, {0x0000304C, 0x00003099}, {0x0000304E, 0x0000304D}, +{0x0000304E, 0x00003099}, {0x00003050, 0x0000304F}, {0x00003050, 0x00003099}, {0x00003052, 0x00003051}, +{0x00003052, 0x00003099}, {0x00003054, 0x00003053}, {0x00003054, 0x00003099}, {0x00003056, 0x00003055}, +{0x00003056, 0x00003099}, {0x00003058, 0x00003057}, {0x00003058, 0x00003099}, {0x0000305A, 0x00003059}, +{0x0000305A, 0x00003099}, {0x0000305C, 0x0000305B}, {0x0000305C, 0x00003099}, {0x0000305E, 0x0000305D}, +{0x0000305E, 0x00003099}, {0x00003060, 0x0000305F}, {0x00003060, 0x00003099}, {0x00003062, 0x00003061}, +{0x00003062, 0x00003099}, {0x00003065, 0x00003064}, {0x00003065, 0x00003099}, {0x00003067, 0x00003066}, +{0x00003067, 0x00003099}, {0x00003069, 0x00003068}, {0x00003069, 0x00003099}, {0x00003070, 0x0000306F}, +{0x00003070, 0x00003099}, {0x00003071, 0x0000306F}, {0x00003071, 0x0000309A}, {0x00003073, 0x00003072}, +{0x00003073, 0x00003099}, {0x00003074, 0x00003072}, {0x00003074, 0x0000309A}, {0x00003076, 0x00003075}, +{0x00003076, 0x00003099}, {0x00003077, 0x00003075}, {0x00003077, 0x0000309A}, {0x00003079, 0x00003078}, +{0x00003079, 0x00003099}, {0x0000307A, 0x00003078}, {0x0000307A, 0x0000309A}, {0x0000307C, 0x0000307B}, +{0x0000307C, 0x00003099}, {0x0000307D, 0x0000307B}, {0x0000307D, 0x0000309A}, {0x00003094, 0x00003046}, +{0x00003094, 0x00003099}, {0x0000309E, 0x0000309D}, {0x0000309E, 0x00003099}, {0x000030AC, 0x000030AB}, +{0x000030AC, 0x00003099}, {0x000030AE, 0x000030AD}, {0x000030AE, 0x00003099}, {0x000030B0, 0x000030AF}, +{0x000030B0, 0x00003099}, {0x000030B2, 0x000030B1}, {0x000030B2, 0x00003099}, {0x000030B4, 0x000030B3}, +{0x000030B4, 0x00003099}, {0x000030B6, 0x000030B5}, {0x000030B6, 0x00003099}, {0x000030B8, 0x000030B7}, +{0x000030B8, 0x00003099}, {0x000030BA, 0x000030B9}, {0x000030BA, 0x00003099}, {0x000030BC, 0x000030BB}, +{0x000030BC, 0x00003099}, {0x000030BE, 0x000030BD}, {0x000030BE, 0x00003099}, {0x000030C0, 0x000030BF}, +{0x000030C0, 0x00003099}, {0x000030C2, 0x000030C1}, {0x000030C2, 0x00003099}, {0x000030C5, 0x000030C4}, +{0x000030C5, 0x00003099}, {0x000030C7, 0x000030C6}, {0x000030C7, 0x00003099}, {0x000030C9, 0x000030C8}, +{0x000030C9, 0x00003099}, {0x000030D0, 0x000030CF}, {0x000030D0, 0x00003099}, {0x000030D1, 0x000030CF}, +{0x000030D1, 0x0000309A}, {0x000030D3, 0x000030D2}, {0x000030D3, 0x00003099}, {0x000030D4, 0x000030D2}, +{0x000030D4, 0x0000309A}, {0x000030D6, 0x000030D5}, {0x000030D6, 0x00003099}, {0x000030D7, 0x000030D5}, +{0x000030D7, 0x0000309A}, {0x000030D9, 0x000030D8}, {0x000030D9, 0x00003099}, {0x000030DA, 0x000030D8}, +{0x000030DA, 0x0000309A}, {0x000030DC, 0x000030DB}, {0x000030DC, 0x00003099}, {0x000030DD, 0x000030DB}, +{0x000030DD, 0x0000309A}, {0x000030F4, 0x000030A6}, {0x000030F4, 0x00003099}, {0x000030F7, 0x000030EF}, +{0x000030F7, 0x00003099}, {0x000030F8, 0x000030F0}, {0x000030F8, 0x00003099}, {0x000030F9, 0x000030F1}, +{0x000030F9, 0x00003099}, {0x000030FA, 0x000030F2}, {0x000030FA, 0x00003099}, {0x000030FE, 0x000030FD}, +{0x000030FE, 0x00003099}, {0x0000F900, 0x00008C48}, {0x0000F901, 0x000066F4}, {0x0000F902, 0x00008ECA}, +{0x0000F903, 0x00008CC8}, {0x0000F904, 0x00006ED1}, {0x0000F905, 0x00004E32}, {0x0000F906, 0x000053E5}, +{0x0000F907, 0x00009F9C}, {0x0000F908, 0x00009F9C}, {0x0000F909, 0x00005951}, {0x0000F90A, 0x000091D1}, +{0x0000F90B, 0x00005587}, {0x0000F90C, 0x00005948}, {0x0000F90D, 0x000061F6}, {0x0000F90E, 0x00007669}, +{0x0000F90F, 0x00007F85}, {0x0000F910, 0x0000863F}, {0x0000F911, 0x000087BA}, {0x0000F912, 0x000088F8}, +{0x0000F913, 0x0000908F}, {0x0000F914, 0x00006A02}, {0x0000F915, 0x00006D1B}, {0x0000F916, 0x000070D9}, +{0x0000F917, 0x000073DE}, {0x0000F918, 0x0000843D}, {0x0000F919, 0x0000916A}, {0x0000F91A, 0x000099F1}, +{0x0000F91B, 0x00004E82}, {0x0000F91C, 0x00005375}, {0x0000F91D, 0x00006B04}, {0x0000F91E, 0x0000721B}, +{0x0000F91F, 0x0000862D}, {0x0000F920, 0x00009E1E}, {0x0000F921, 0x00005D50}, {0x0000F922, 0x00006FEB}, +{0x0000F923, 0x000085CD}, {0x0000F924, 0x00008964}, {0x0000F925, 0x000062C9}, {0x0000F926, 0x000081D8}, +{0x0000F927, 0x0000881F}, {0x0000F928, 0x00005ECA}, {0x0000F929, 0x00006717}, {0x0000F92A, 0x00006D6A}, +{0x0000F92B, 0x000072FC}, {0x0000F92C, 0x000090CE}, {0x0000F92D, 0x00004F86}, {0x0000F92E, 0x000051B7}, +{0x0000F92F, 0x000052DE}, {0x0000F930, 0x000064C4}, {0x0000F931, 0x00006AD3}, {0x0000F932, 0x00007210}, +{0x0000F933, 0x000076E7}, {0x0000F934, 0x00008001}, {0x0000F935, 0x00008606}, {0x0000F936, 0x0000865C}, +{0x0000F937, 0x00008DEF}, {0x0000F938, 0x00009732}, {0x0000F939, 0x00009B6F}, {0x0000F93A, 0x00009DFA}, +{0x0000F93B, 0x0000788C}, {0x0000F93C, 0x0000797F}, {0x0000F93D, 0x00007DA0}, {0x0000F93E, 0x000083C9}, +{0x0000F93F, 0x00009304}, {0x0000F940, 0x00009E7F}, {0x0000F941, 0x00008AD6}, {0x0000F942, 0x000058DF}, +{0x0000F943, 0x00005F04}, {0x0000F944, 0x00007C60}, {0x0000F945, 0x0000807E}, {0x0000F946, 0x00007262}, +{0x0000F947, 0x000078CA}, {0x0000F948, 0x00008CC2}, {0x0000F949, 0x000096F7}, {0x0000F94A, 0x000058D8}, +{0x0000F94B, 0x00005C62}, {0x0000F94C, 0x00006A13}, {0x0000F94D, 0x00006DDA}, {0x0000F94E, 0x00006F0F}, +{0x0000F94F, 0x00007D2F}, {0x0000F950, 0x00007E37}, {0x0000F951, 0x0000964B}, {0x0000F952, 0x000052D2}, +{0x0000F953, 0x0000808B}, {0x0000F954, 0x000051DC}, {0x0000F955, 0x000051CC}, {0x0000F956, 0x00007A1C}, +{0x0000F957, 0x00007DBE}, {0x0000F958, 0x000083F1}, {0x0000F959, 0x00009675}, {0x0000F95A, 0x00008B80}, +{0x0000F95B, 0x000062CF}, {0x0000F95C, 0x00006A02}, {0x0000F95D, 0x00008AFE}, {0x0000F95E, 0x00004E39}, +{0x0000F95F, 0x00005BE7}, {0x0000F960, 0x00006012}, {0x0000F961, 0x00007387}, {0x0000F962, 0x00007570}, +{0x0000F963, 0x00005317}, {0x0000F964, 0x000078FB}, {0x0000F965, 0x00004FBF}, {0x0000F966, 0x00005FA9}, +{0x0000F967, 0x00004E0D}, {0x0000F968, 0x00006CCC}, {0x0000F969, 0x00006578}, {0x0000F96A, 0x00007D22}, +{0x0000F96B, 0x000053C3}, {0x0000F96C, 0x0000585E}, {0x0000F96D, 0x00007701}, {0x0000F96E, 0x00008449}, +{0x0000F96F, 0x00008AAA}, {0x0000F970, 0x00006BBA}, {0x0000F971, 0x00008FB0}, {0x0000F972, 0x00006C88}, +{0x0000F973, 0x000062FE}, {0x0000F974, 0x000082E5}, {0x0000F975, 0x000063A0}, {0x0000F976, 0x00007565}, +{0x0000F977, 0x00004EAE}, {0x0000F978, 0x00005169}, {0x0000F979, 0x000051C9}, {0x0000F97A, 0x00006881}, +{0x0000F97B, 0x00007CE7}, {0x0000F97C, 0x0000826F}, {0x0000F97D, 0x00008AD2}, {0x0000F97E, 0x000091CF}, +{0x0000F97F, 0x000052F5}, {0x0000F980, 0x00005442}, {0x0000F981, 0x00005973}, {0x0000F982, 0x00005EEC}, +{0x0000F983, 0x000065C5}, {0x0000F984, 0x00006FFE}, {0x0000F985, 0x0000792A}, {0x0000F986, 0x000095AD}, +{0x0000F987, 0x00009A6A}, {0x0000F988, 0x00009E97}, {0x0000F989, 0x00009ECE}, {0x0000F98A, 0x0000529B}, +{0x0000F98B, 0x000066C6}, {0x0000F98C, 0x00006B77}, {0x0000F98D, 0x00008F62}, {0x0000F98E, 0x00005E74}, +{0x0000F98F, 0x00006190}, {0x0000F990, 0x00006200}, {0x0000F991, 0x0000649A}, {0x0000F992, 0x00006F23}, +{0x0000F993, 0x00007149}, {0x0000F994, 0x00007489}, {0x0000F995, 0x000079CA}, {0x0000F996, 0x00007DF4}, +{0x0000F997, 0x0000806F}, {0x0000F998, 0x00008F26}, {0x0000F999, 0x000084EE}, {0x0000F99A, 0x00009023}, +{0x0000F99B, 0x0000934A}, {0x0000F99C, 0x00005217}, {0x0000F99D, 0x000052A3}, {0x0000F99E, 0x000054BD}, +{0x0000F99F, 0x000070C8}, {0x0000F9A0, 0x000088C2}, {0x0000F9A1, 0x00008AAA}, {0x0000F9A2, 0x00005EC9}, +{0x0000F9A3, 0x00005FF5}, {0x0000F9A4, 0x0000637B}, {0x0000F9A5, 0x00006BAE}, {0x0000F9A6, 0x00007C3E}, +{0x0000F9A7, 0x00007375}, {0x0000F9A8, 0x00004EE4}, {0x0000F9A9, 0x000056F9}, {0x0000F9AA, 0x00005BE7}, +{0x0000F9AB, 0x00005DBA}, {0x0000F9AC, 0x0000601C}, {0x0000F9AD, 0x000073B2}, {0x0000F9AE, 0x00007469}, +{0x0000F9AF, 0x00007F9A}, {0x0000F9B0, 0x00008046}, {0x0000F9B1, 0x00009234}, {0x0000F9B2, 0x000096F6}, +{0x0000F9B3, 0x00009748}, {0x0000F9B4, 0x00009818}, {0x0000F9B5, 0x00004F8B}, {0x0000F9B6, 0x000079AE}, +{0x0000F9B7, 0x000091B4}, {0x0000F9B8, 0x000096B8}, {0x0000F9B9, 0x000060E1}, {0x0000F9BA, 0x00004E86}, +{0x0000F9BB, 0x000050DA}, {0x0000F9BC, 0x00005BEE}, {0x0000F9BD, 0x00005C3F}, {0x0000F9BE, 0x00006599}, +{0x0000F9BF, 0x00006A02}, {0x0000F9C0, 0x000071CE}, {0x0000F9C1, 0x00007642}, {0x0000F9C2, 0x000084FC}, +{0x0000F9C3, 0x0000907C}, {0x0000F9C4, 0x00009F8D}, {0x0000F9C5, 0x00006688}, {0x0000F9C6, 0x0000962E}, +{0x0000F9C7, 0x00005289}, {0x0000F9C8, 0x0000677B}, {0x0000F9C9, 0x000067F3}, {0x0000F9CA, 0x00006D41}, +{0x0000F9CB, 0x00006E9C}, {0x0000F9CC, 0x00007409}, {0x0000F9CD, 0x00007559}, {0x0000F9CE, 0x0000786B}, +{0x0000F9CF, 0x00007D10}, {0x0000F9D0, 0x0000985E}, {0x0000F9D1, 0x0000516D}, {0x0000F9D2, 0x0000622E}, +{0x0000F9D3, 0x00009678}, {0x0000F9D4, 0x0000502B}, {0x0000F9D5, 0x00005D19}, {0x0000F9D6, 0x00006DEA}, +{0x0000F9D7, 0x00008F2A}, {0x0000F9D8, 0x00005F8B}, {0x0000F9D9, 0x00006144}, {0x0000F9DA, 0x00006817}, +{0x0000F9DB, 0x00007387}, {0x0000F9DC, 0x00009686}, {0x0000F9DD, 0x00005229}, {0x0000F9DE, 0x0000540F}, +{0x0000F9DF, 0x00005C65}, {0x0000F9E0, 0x00006613}, {0x0000F9E1, 0x0000674E}, {0x0000F9E2, 0x000068A8}, +{0x0000F9E3, 0x00006CE5}, {0x0000F9E4, 0x00007406}, {0x0000F9E5, 0x000075E2}, {0x0000F9E6, 0x00007F79}, +{0x0000F9E7, 0x000088CF}, {0x0000F9E8, 0x000088E1}, {0x0000F9E9, 0x000091CC}, {0x0000F9EA, 0x000096E2}, +{0x0000F9EB, 0x0000533F}, {0x0000F9EC, 0x00006EBA}, {0x0000F9ED, 0x0000541D}, {0x0000F9EE, 0x000071D0}, +{0x0000F9EF, 0x00007498}, {0x0000F9F0, 0x000085FA}, {0x0000F9F1, 0x000096A3}, {0x0000F9F2, 0x00009C57}, +{0x0000F9F3, 0x00009E9F}, {0x0000F9F4, 0x00006797}, {0x0000F9F5, 0x00006DCB}, {0x0000F9F6, 0x000081E8}, +{0x0000F9F7, 0x00007ACB}, {0x0000F9F8, 0x00007B20}, {0x0000F9F9, 0x00007C92}, {0x0000F9FA, 0x000072C0}, +{0x0000F9FB, 0x00007099}, {0x0000F9FC, 0x00008B58}, {0x0000F9FD, 0x00004EC0}, {0x0000F9FE, 0x00008336}, +{0x0000F9FF, 0x0000523A}, {0x0000FA00, 0x00005207}, {0x0000FA01, 0x00005EA6}, {0x0000FA02, 0x000062D3}, +{0x0000FA03, 0x00007CD6}, {0x0000FA04, 0x00005B85}, {0x0000FA05, 0x00006D1E}, {0x0000FA06, 0x000066B4}, +{0x0000FA07, 0x00008F3B}, {0x0000FA08, 0x0000884C}, {0x0000FA09, 0x0000964D}, {0x0000FA0A, 0x0000898B}, +{0x0000FA0B, 0x00005ED3}, {0x0000FA0C, 0x00005140}, {0x0000FA0D, 0x000055C0}, {0x0000FA10, 0x0000585A}, +{0x0000FA12, 0x00006674}, {0x0000FA15, 0x000051DE}, {0x0000FA16, 0x0000732A}, {0x0000FA17, 0x000076CA}, +{0x0000FA18, 0x0000793C}, {0x0000FA19, 0x0000795E}, {0x0000FA1A, 0x00007965}, {0x0000FA1B, 0x0000798F}, +{0x0000FA1C, 0x00009756}, {0x0000FA1D, 0x00007CBE}, {0x0000FA1E, 0x00007FBD}, {0x0000FA20, 0x00008612}, +{0x0000FA22, 0x00008AF8}, {0x0000FA25, 0x00009038}, {0x0000FA26, 0x000090FD}, {0x0000FA2A, 0x000098EF}, +{0x0000FA2B, 0x000098FC}, {0x0000FA2C, 0x00009928}, {0x0000FA2D, 0x00009DB4}, {0x0000FA2E, 0x000090DE}, +{0x0000FA2F, 0x000096B7}, {0x0000FA30, 0x00004FAE}, {0x0000FA31, 0x000050E7}, {0x0000FA32, 0x0000514D}, +{0x0000FA33, 0x000052C9}, {0x0000FA34, 0x000052E4}, {0x0000FA35, 0x00005351}, {0x0000FA36, 0x0000559D}, +{0x0000FA37, 0x00005606}, {0x0000FA38, 0x00005668}, {0x0000FA39, 0x00005840}, {0x0000FA3A, 0x000058A8}, +{0x0000FA3B, 0x00005C64}, {0x0000FA3C, 0x00005C6E}, {0x0000FA3D, 0x00006094}, {0x0000FA3E, 0x00006168}, +{0x0000FA3F, 0x0000618E}, {0x0000FA40, 0x000061F2}, {0x0000FA41, 0x0000654F}, {0x0000FA42, 0x000065E2}, +{0x0000FA43, 0x00006691}, {0x0000FA44, 0x00006885}, {0x0000FA45, 0x00006D77}, {0x0000FA46, 0x00006E1A}, +{0x0000FA47, 0x00006F22}, {0x0000FA48, 0x0000716E}, {0x0000FA49, 0x0000722B}, {0x0000FA4A, 0x00007422}, +{0x0000FA4B, 0x00007891}, {0x0000FA4C, 0x0000793E}, {0x0000FA4D, 0x00007949}, {0x0000FA4E, 0x00007948}, +{0x0000FA4F, 0x00007950}, {0x0000FA50, 0x00007956}, {0x0000FA51, 0x0000795D}, {0x0000FA52, 0x0000798D}, +{0x0000FA53, 0x0000798E}, {0x0000FA54, 0x00007A40}, {0x0000FA55, 0x00007A81}, {0x0000FA56, 0x00007BC0}, +{0x0000FA57, 0x00007DF4}, {0x0000FA58, 0x00007E09}, {0x0000FA59, 0x00007E41}, {0x0000FA5A, 0x00007F72}, +{0x0000FA5B, 0x00008005}, {0x0000FA5C, 0x000081ED}, {0x0000FA5D, 0x00008279}, {0x0000FA5E, 0x00008279}, +{0x0000FA5F, 0x00008457}, {0x0000FA60, 0x00008910}, {0x0000FA61, 0x00008996}, {0x0000FA62, 0x00008B01}, +{0x0000FA63, 0x00008B39}, {0x0000FA64, 0x00008CD3}, {0x0000FA65, 0x00008D08}, {0x0000FA66, 0x00008FB6}, +{0x0000FA67, 0x00009038}, {0x0000FA68, 0x000096E3}, {0x0000FA69, 0x000097FF}, {0x0000FA6A, 0x0000983B}, +{0x0000FA6B, 0x00006075}, {0x0000FA6C, 0x000242EE}, {0x0000FA6D, 0x00008218}, {0x0000FA70, 0x00004E26}, +{0x0000FA71, 0x000051B5}, {0x0000FA72, 0x00005168}, {0x0000FA73, 0x00004F80}, {0x0000FA74, 0x00005145}, +{0x0000FA75, 0x00005180}, {0x0000FA76, 0x000052C7}, {0x0000FA77, 0x000052FA}, {0x0000FA78, 0x0000559D}, +{0x0000FA79, 0x00005555}, {0x0000FA7A, 0x00005599}, {0x0000FA7B, 0x000055E2}, {0x0000FA7C, 0x0000585A}, +{0x0000FA7D, 0x000058B3}, {0x0000FA7E, 0x00005944}, {0x0000FA7F, 0x00005954}, {0x0000FA80, 0x00005A62}, +{0x0000FA81, 0x00005B28}, {0x0000FA82, 0x00005ED2}, {0x0000FA83, 0x00005ED9}, {0x0000FA84, 0x00005F69}, +{0x0000FA85, 0x00005FAD}, {0x0000FA86, 0x000060D8}, {0x0000FA87, 0x0000614E}, {0x0000FA88, 0x00006108}, +{0x0000FA89, 0x0000618E}, {0x0000FA8A, 0x00006160}, {0x0000FA8B, 0x000061F2}, {0x0000FA8C, 0x00006234}, +{0x0000FA8D, 0x000063C4}, {0x0000FA8E, 0x0000641C}, {0x0000FA8F, 0x00006452}, {0x0000FA90, 0x00006556}, +{0x0000FA91, 0x00006674}, {0x0000FA92, 0x00006717}, {0x0000FA93, 0x0000671B}, {0x0000FA94, 0x00006756}, +{0x0000FA95, 0x00006B79}, {0x0000FA96, 0x00006BBA}, {0x0000FA97, 0x00006D41}, {0x0000FA98, 0x00006EDB}, +{0x0000FA99, 0x00006ECB}, {0x0000FA9A, 0x00006F22}, {0x0000FA9B, 0x0000701E}, {0x0000FA9C, 0x0000716E}, +{0x0000FA9D, 0x000077A7}, {0x0000FA9E, 0x00007235}, {0x0000FA9F, 0x000072AF}, {0x0000FAA0, 0x0000732A}, +{0x0000FAA1, 0x00007471}, {0x0000FAA2, 0x00007506}, {0x0000FAA3, 0x0000753B}, {0x0000FAA4, 0x0000761D}, +{0x0000FAA5, 0x0000761F}, {0x0000FAA6, 0x000076CA}, {0x0000FAA7, 0x000076DB}, {0x0000FAA8, 0x000076F4}, +{0x0000FAA9, 0x0000774A}, {0x0000FAAA, 0x00007740}, {0x0000FAAB, 0x000078CC}, {0x0000FAAC, 0x00007AB1}, +{0x0000FAAD, 0x00007BC0}, {0x0000FAAE, 0x00007C7B}, {0x0000FAAF, 0x00007D5B}, {0x0000FAB0, 0x00007DF4}, +{0x0000FAB1, 0x00007F3E}, {0x0000FAB2, 0x00008005}, {0x0000FAB3, 0x00008352}, {0x0000FAB4, 0x000083EF}, +{0x0000FAB5, 0x00008779}, {0x0000FAB6, 0x00008941}, {0x0000FAB7, 0x00008986}, {0x0000FAB8, 0x00008996}, +{0x0000FAB9, 0x00008ABF}, {0x0000FABA, 0x00008AF8}, {0x0000FABB, 0x00008ACB}, {0x0000FABC, 0x00008B01}, +{0x0000FABD, 0x00008AFE}, {0x0000FABE, 0x00008AED}, {0x0000FABF, 0x00008B39}, {0x0000FAC0, 0x00008B8A}, +{0x0000FAC1, 0x00008D08}, {0x0000FAC2, 0x00008F38}, {0x0000FAC3, 0x00009072}, {0x0000FAC4, 0x00009199}, +{0x0000FAC5, 0x00009276}, {0x0000FAC6, 0x0000967C}, {0x0000FAC7, 0x000096E3}, {0x0000FAC8, 0x00009756}, +{0x0000FAC9, 0x000097DB}, {0x0000FACA, 0x000097FF}, {0x0000FACB, 0x0000980B}, {0x0000FACC, 0x0000983B}, +{0x0000FACD, 0x00009B12}, {0x0000FACE, 0x00009F9C}, {0x0000FACF, 0x0002284A}, {0x0000FAD0, 0x00022844}, +{0x0000FAD1, 0x000233D5}, {0x0000FAD2, 0x00003B9D}, {0x0000FAD3, 0x00004018}, {0x0000FAD4, 0x00004039}, +{0x0000FAD5, 0x00025249}, {0x0000FAD6, 0x00025CD0}, {0x0000FAD7, 0x00027ED3}, {0x0000FAD8, 0x00009F43}, +{0x0000FAD9, 0x00009F8E}, {0x0000FB1D, 0x000005D9}, {0x0000FB1D, 0x000005B4}, {0x0000FB1F, 0x000005F2}, +{0x0000FB1F, 0x000005B7}, {0x0000FB2A, 0x000005E9}, {0x0000FB2A, 0x000005C1}, {0x0000FB2B, 0x000005E9}, +{0x0000FB2B, 0x000005C2}, {0x0000FB2C, 0x000005E9}, {0x0000FB2C, 0x000005BC}, {0x0000FB2C, 0x000005C1}, +{0x0000FB2D, 0x000005E9}, {0x0000FB2D, 0x000005BC}, {0x0000FB2D, 0x000005C2}, {0x0000FB2E, 0x000005D0}, +{0x0000FB2E, 0x000005B7}, {0x0000FB2F, 0x000005D0}, {0x0000FB2F, 0x000005B8}, {0x0000FB30, 0x000005D0}, +{0x0000FB30, 0x000005BC}, {0x0000FB31, 0x000005D1}, {0x0000FB31, 0x000005BC}, {0x0000FB32, 0x000005D2}, +{0x0000FB32, 0x000005BC}, {0x0000FB33, 0x000005D3}, {0x0000FB33, 0x000005BC}, {0x0000FB34, 0x000005D4}, +{0x0000FB34, 0x000005BC}, {0x0000FB35, 0x000005D5}, {0x0000FB35, 0x000005BC}, {0x0000FB36, 0x000005D6}, +{0x0000FB36, 0x000005BC}, {0x0000FB38, 0x000005D8}, {0x0000FB38, 0x000005BC}, {0x0000FB39, 0x000005D9}, +{0x0000FB39, 0x000005BC}, {0x0000FB3A, 0x000005DA}, {0x0000FB3A, 0x000005BC}, {0x0000FB3B, 0x000005DB}, +{0x0000FB3B, 0x000005BC}, {0x0000FB3C, 0x000005DC}, {0x0000FB3C, 0x000005BC}, {0x0000FB3E, 0x000005DE}, +{0x0000FB3E, 0x000005BC}, {0x0000FB40, 0x000005E0}, {0x0000FB40, 0x000005BC}, {0x0000FB41, 0x000005E1}, +{0x0000FB41, 0x000005BC}, {0x0000FB43, 0x000005E3}, {0x0000FB43, 0x000005BC}, {0x0000FB44, 0x000005E4}, +{0x0000FB44, 0x000005BC}, {0x0000FB46, 0x000005E6}, {0x0000FB46, 0x000005BC}, {0x0000FB47, 0x000005E7}, +{0x0000FB47, 0x000005BC}, {0x0000FB48, 0x000005E8}, {0x0000FB48, 0x000005BC}, {0x0000FB49, 0x000005E9}, +{0x0000FB49, 0x000005BC}, {0x0000FB4A, 0x000005EA}, {0x0000FB4A, 0x000005BC}, {0x0000FB4B, 0x000005D5}, +{0x0000FB4B, 0x000005B9}, {0x0000FB4C, 0x000005D1}, {0x0000FB4C, 0x000005BF}, {0x0000FB4D, 0x000005DB}, +{0x0000FB4D, 0x000005BF}, {0x0000FB4E, 0x000005E4}, {0x0000FB4E, 0x000005BF}, {0x0001109A, 0x00011099}, +{0x0001109A, 0x000110BA}, {0x0001109C, 0x0001109B}, {0x0001109C, 0x000110BA}, {0x000110AB, 0x000110A5}, +{0x000110AB, 0x000110BA}, {0x0001112E, 0x00011131}, {0x0001112E, 0x00011127}, {0x0001112F, 0x00011132}, +{0x0001112F, 0x00011127}, {0x0001134B, 0x00011347}, {0x0001134B, 0x0001133E}, {0x0001134C, 0x00011347}, +{0x0001134C, 0x00011357}, {0x000114BB, 0x000114B9}, {0x000114BB, 0x000114BA}, {0x000114BC, 0x000114B9}, +{0x000114BC, 0x000114B0}, {0x000114BE, 0x000114B9}, {0x000114BE, 0x000114BD}, {0x000115BA, 0x000115B8}, +{0x000115BA, 0x000115AF}, {0x000115BB, 0x000115B9}, {0x000115BB, 0x000115AF}, {0x0001D15E, 0x0001D157}, +{0x0001D15E, 0x0001D165}, {0x0001D15F, 0x0001D158}, {0x0001D15F, 0x0001D165}, {0x0001D160, 0x0001D158}, +{0x0001D160, 0x0001D165}, {0x0001D160, 0x0001D16E}, {0x0001D161, 0x0001D158}, {0x0001D161, 0x0001D165}, +{0x0001D161, 0x0001D16F}, {0x0001D162, 0x0001D158}, {0x0001D162, 0x0001D165}, {0x0001D162, 0x0001D170}, +{0x0001D163, 0x0001D158}, {0x0001D163, 0x0001D165}, {0x0001D163, 0x0001D171}, {0x0001D164, 0x0001D158}, +{0x0001D164, 0x0001D165}, {0x0001D164, 0x0001D172}, {0x0001D1BB, 0x0001D1B9}, {0x0001D1BB, 0x0001D165}, +{0x0001D1BC, 0x0001D1BA}, {0x0001D1BC, 0x0001D165}, {0x0001D1BD, 0x0001D1B9}, {0x0001D1BD, 0x0001D165}, +{0x0001D1BD, 0x0001D16E}, {0x0001D1BE, 0x0001D1BA}, {0x0001D1BE, 0x0001D165}, {0x0001D1BE, 0x0001D16E}, +{0x0001D1BF, 0x0001D1B9}, {0x0001D1BF, 0x0001D165}, {0x0001D1BF, 0x0001D16F}, {0x0001D1C0, 0x0001D1BA}, +{0x0001D1C0, 0x0001D165}, {0x0001D1C0, 0x0001D16F}, {0x0002F800, 0x00004E3D}, {0x0002F801, 0x00004E38}, +{0x0002F802, 0x00004E41}, {0x0002F803, 0x00020122}, {0x0002F804, 0x00004F60}, {0x0002F805, 0x00004FAE}, +{0x0002F806, 0x00004FBB}, {0x0002F807, 0x00005002}, {0x0002F808, 0x0000507A}, {0x0002F809, 0x00005099}, +{0x0002F80A, 0x000050E7}, {0x0002F80B, 0x000050CF}, {0x0002F80C, 0x0000349E}, {0x0002F80D, 0x0002063A}, +{0x0002F80E, 0x0000514D}, {0x0002F80F, 0x00005154}, {0x0002F810, 0x00005164}, {0x0002F811, 0x00005177}, +{0x0002F812, 0x0002051C}, {0x0002F813, 0x000034B9}, {0x0002F814, 0x00005167}, {0x0002F815, 0x0000518D}, +{0x0002F816, 0x0002054B}, {0x0002F817, 0x00005197}, {0x0002F818, 0x000051A4}, {0x0002F819, 0x00004ECC}, +{0x0002F81A, 0x000051AC}, {0x0002F81B, 0x000051B5}, {0x0002F81C, 0x000291DF}, {0x0002F81D, 0x000051F5}, +{0x0002F81E, 0x00005203}, {0x0002F81F, 0x000034DF}, {0x0002F820, 0x0000523B}, {0x0002F821, 0x00005246}, +{0x0002F822, 0x00005272}, {0x0002F823, 0x00005277}, {0x0002F824, 0x00003515}, {0x0002F825, 0x000052C7}, +{0x0002F826, 0x000052C9}, {0x0002F827, 0x000052E4}, {0x0002F828, 0x000052FA}, {0x0002F829, 0x00005305}, +{0x0002F82A, 0x00005306}, {0x0002F82B, 0x00005317}, {0x0002F82C, 0x00005349}, {0x0002F82D, 0x00005351}, +{0x0002F82E, 0x0000535A}, {0x0002F82F, 0x00005373}, {0x0002F830, 0x0000537D}, {0x0002F831, 0x0000537F}, +{0x0002F832, 0x0000537F}, {0x0002F833, 0x0000537F}, {0x0002F834, 0x00020A2C}, {0x0002F835, 0x00007070}, +{0x0002F836, 0x000053CA}, {0x0002F837, 0x000053DF}, {0x0002F838, 0x00020B63}, {0x0002F839, 0x000053EB}, +{0x0002F83A, 0x000053F1}, {0x0002F83B, 0x00005406}, {0x0002F83C, 0x0000549E}, {0x0002F83D, 0x00005438}, +{0x0002F83E, 0x00005448}, {0x0002F83F, 0x00005468}, {0x0002F840, 0x000054A2}, {0x0002F841, 0x000054F6}, +{0x0002F842, 0x00005510}, {0x0002F843, 0x00005553}, {0x0002F844, 0x00005563}, {0x0002F845, 0x00005584}, +{0x0002F846, 0x00005584}, {0x0002F847, 0x00005599}, {0x0002F848, 0x000055AB}, {0x0002F849, 0x000055B3}, +{0x0002F84A, 0x000055C2}, {0x0002F84B, 0x00005716}, {0x0002F84C, 0x00005606}, {0x0002F84D, 0x00005717}, +{0x0002F84E, 0x00005651}, {0x0002F84F, 0x00005674}, {0x0002F850, 0x00005207}, {0x0002F851, 0x000058EE}, +{0x0002F852, 0x000057CE}, {0x0002F853, 0x000057F4}, {0x0002F854, 0x0000580D}, {0x0002F855, 0x0000578B}, +{0x0002F856, 0x00005832}, {0x0002F857, 0x00005831}, {0x0002F858, 0x000058AC}, {0x0002F859, 0x000214E4}, +{0x0002F85A, 0x000058F2}, {0x0002F85B, 0x000058F7}, {0x0002F85C, 0x00005906}, {0x0002F85D, 0x0000591A}, +{0x0002F85E, 0x00005922}, {0x0002F85F, 0x00005962}, {0x0002F860, 0x000216A8}, {0x0002F861, 0x000216EA}, +{0x0002F862, 0x000059EC}, {0x0002F863, 0x00005A1B}, {0x0002F864, 0x00005A27}, {0x0002F865, 0x000059D8}, +{0x0002F866, 0x00005A66}, {0x0002F867, 0x000036EE}, {0x0002F868, 0x000036FC}, {0x0002F869, 0x00005B08}, +{0x0002F86A, 0x00005B3E}, {0x0002F86B, 0x00005B3E}, {0x0002F86C, 0x000219C8}, {0x0002F86D, 0x00005BC3}, +{0x0002F86E, 0x00005BD8}, {0x0002F86F, 0x00005BE7}, {0x0002F870, 0x00005BF3}, {0x0002F871, 0x00021B18}, +{0x0002F872, 0x00005BFF}, {0x0002F873, 0x00005C06}, {0x0002F874, 0x00005F53}, {0x0002F875, 0x00005C22}, +{0x0002F876, 0x00003781}, {0x0002F877, 0x00005C60}, {0x0002F878, 0x00005C6E}, {0x0002F879, 0x00005CC0}, +{0x0002F87A, 0x00005C8D}, {0x0002F87B, 0x00021DE4}, {0x0002F87C, 0x00005D43}, {0x0002F87D, 0x00021DE6}, +{0x0002F87E, 0x00005D6E}, {0x0002F87F, 0x00005D6B}, {0x0002F880, 0x00005D7C}, {0x0002F881, 0x00005DE1}, +{0x0002F882, 0x00005DE2}, {0x0002F883, 0x0000382F}, {0x0002F884, 0x00005DFD}, {0x0002F885, 0x00005E28}, +{0x0002F886, 0x00005E3D}, {0x0002F887, 0x00005E69}, {0x0002F888, 0x00003862}, {0x0002F889, 0x00022183}, +{0x0002F88A, 0x0000387C}, {0x0002F88B, 0x00005EB0}, {0x0002F88C, 0x00005EB3}, {0x0002F88D, 0x00005EB6}, +{0x0002F88E, 0x00005ECA}, {0x0002F88F, 0x0002A392}, {0x0002F890, 0x00005EFE}, {0x0002F891, 0x00022331}, +{0x0002F892, 0x00022331}, {0x0002F893, 0x00008201}, {0x0002F894, 0x00005F22}, {0x0002F895, 0x00005F22}, +{0x0002F896, 0x000038C7}, {0x0002F897, 0x000232B8}, {0x0002F898, 0x000261DA}, {0x0002F899, 0x00005F62}, +{0x0002F89A, 0x00005F6B}, {0x0002F89B, 0x000038E3}, {0x0002F89C, 0x00005F9A}, {0x0002F89D, 0x00005FCD}, +{0x0002F89E, 0x00005FD7}, {0x0002F89F, 0x00005FF9}, {0x0002F8A0, 0x00006081}, {0x0002F8A1, 0x0000393A}, +{0x0002F8A2, 0x0000391C}, {0x0002F8A3, 0x00006094}, {0x0002F8A4, 0x000226D4}, {0x0002F8A5, 0x000060C7}, +{0x0002F8A6, 0x00006148}, {0x0002F8A7, 0x0000614C}, {0x0002F8A8, 0x0000614E}, {0x0002F8A9, 0x0000614C}, +{0x0002F8AA, 0x0000617A}, {0x0002F8AB, 0x0000618E}, {0x0002F8AC, 0x000061B2}, {0x0002F8AD, 0x000061A4}, +{0x0002F8AE, 0x000061AF}, {0x0002F8AF, 0x000061DE}, {0x0002F8B0, 0x000061F2}, {0x0002F8B1, 0x000061F6}, +{0x0002F8B2, 0x00006210}, {0x0002F8B3, 0x0000621B}, {0x0002F8B4, 0x0000625D}, {0x0002F8B5, 0x000062B1}, +{0x0002F8B6, 0x000062D4}, {0x0002F8B7, 0x00006350}, {0x0002F8B8, 0x00022B0C}, {0x0002F8B9, 0x0000633D}, +{0x0002F8BA, 0x000062FC}, {0x0002F8BB, 0x00006368}, {0x0002F8BC, 0x00006383}, {0x0002F8BD, 0x000063E4}, +{0x0002F8BE, 0x00022BF1}, {0x0002F8BF, 0x00006422}, {0x0002F8C0, 0x000063C5}, {0x0002F8C1, 0x000063A9}, +{0x0002F8C2, 0x00003A2E}, {0x0002F8C3, 0x00006469}, {0x0002F8C4, 0x0000647E}, {0x0002F8C5, 0x0000649D}, +{0x0002F8C6, 0x00006477}, {0x0002F8C7, 0x00003A6C}, {0x0002F8C8, 0x0000654F}, {0x0002F8C9, 0x0000656C}, +{0x0002F8CA, 0x0002300A}, {0x0002F8CB, 0x000065E3}, {0x0002F8CC, 0x000066F8}, {0x0002F8CD, 0x00006649}, +{0x0002F8CE, 0x00003B19}, {0x0002F8CF, 0x00006691}, {0x0002F8D0, 0x00003B08}, {0x0002F8D1, 0x00003AE4}, +{0x0002F8D2, 0x00005192}, {0x0002F8D3, 0x00005195}, {0x0002F8D4, 0x00006700}, {0x0002F8D5, 0x0000669C}, +{0x0002F8D6, 0x000080AD}, {0x0002F8D7, 0x000043D9}, {0x0002F8D8, 0x00006717}, {0x0002F8D9, 0x0000671B}, +{0x0002F8DA, 0x00006721}, {0x0002F8DB, 0x0000675E}, {0x0002F8DC, 0x00006753}, {0x0002F8DD, 0x000233C3}, +{0x0002F8DE, 0x00003B49}, {0x0002F8DF, 0x000067FA}, {0x0002F8E0, 0x00006785}, {0x0002F8E1, 0x00006852}, +{0x0002F8E2, 0x00006885}, {0x0002F8E3, 0x0002346D}, {0x0002F8E4, 0x0000688E}, {0x0002F8E5, 0x0000681F}, +{0x0002F8E6, 0x00006914}, {0x0002F8E7, 0x00003B9D}, {0x0002F8E8, 0x00006942}, {0x0002F8E9, 0x000069A3}, +{0x0002F8EA, 0x000069EA}, {0x0002F8EB, 0x00006AA8}, {0x0002F8EC, 0x000236A3}, {0x0002F8ED, 0x00006ADB}, +{0x0002F8EE, 0x00003C18}, {0x0002F8EF, 0x00006B21}, {0x0002F8F0, 0x000238A7}, {0x0002F8F1, 0x00006B54}, +{0x0002F8F2, 0x00003C4E}, {0x0002F8F3, 0x00006B72}, {0x0002F8F4, 0x00006B9F}, {0x0002F8F5, 0x00006BBA}, +{0x0002F8F6, 0x00006BBB}, {0x0002F8F7, 0x00023A8D}, {0x0002F8F8, 0x00021D0B}, {0x0002F8F9, 0x00023AFA}, +{0x0002F8FA, 0x00006C4E}, {0x0002F8FB, 0x00023CBC}, {0x0002F8FC, 0x00006CBF}, {0x0002F8FD, 0x00006CCD}, +{0x0002F8FE, 0x00006C67}, {0x0002F8FF, 0x00006D16}, {0x0002F900, 0x00006D3E}, {0x0002F901, 0x00006D77}, +{0x0002F902, 0x00006D41}, {0x0002F903, 0x00006D69}, {0x0002F904, 0x00006D78}, {0x0002F905, 0x00006D85}, +{0x0002F906, 0x00023D1E}, {0x0002F907, 0x00006D34}, {0x0002F908, 0x00006E2F}, {0x0002F909, 0x00006E6E}, +{0x0002F90A, 0x00003D33}, {0x0002F90B, 0x00006ECB}, {0x0002F90C, 0x00006EC7}, {0x0002F90D, 0x00023ED1}, +{0x0002F90E, 0x00006DF9}, {0x0002F90F, 0x00006F6E}, {0x0002F910, 0x00023F5E}, {0x0002F911, 0x00023F8E}, +{0x0002F912, 0x00006FC6}, {0x0002F913, 0x00007039}, {0x0002F914, 0x0000701E}, {0x0002F915, 0x0000701B}, +{0x0002F916, 0x00003D96}, {0x0002F917, 0x0000704A}, {0x0002F918, 0x0000707D}, {0x0002F919, 0x00007077}, +{0x0002F91A, 0x000070AD}, {0x0002F91B, 0x00020525}, {0x0002F91C, 0x00007145}, {0x0002F91D, 0x00024263}, +{0x0002F91E, 0x0000719C}, {0x0002F91F, 0x000243AB}, {0x0002F920, 0x00007228}, {0x0002F921, 0x00007235}, +{0x0002F922, 0x00007250}, {0x0002F923, 0x00024608}, {0x0002F924, 0x00007280}, {0x0002F925, 0x00007295}, +{0x0002F926, 0x00024735}, {0x0002F927, 0x00024814}, {0x0002F928, 0x0000737A}, {0x0002F929, 0x0000738B}, +{0x0002F92A, 0x00003EAC}, {0x0002F92B, 0x000073A5}, {0x0002F92C, 0x00003EB8}, {0x0002F92D, 0x00003EB8}, +{0x0002F92E, 0x00007447}, {0x0002F92F, 0x0000745C}, {0x0002F930, 0x00007471}, {0x0002F931, 0x00007485}, +{0x0002F932, 0x000074CA}, {0x0002F933, 0x00003F1B}, {0x0002F934, 0x00007524}, {0x0002F935, 0x00024C36}, +{0x0002F936, 0x0000753E}, {0x0002F937, 0x00024C92}, {0x0002F938, 0x00007570}, {0x0002F939, 0x0002219F}, +{0x0002F93A, 0x00007610}, {0x0002F93B, 0x00024FA1}, {0x0002F93C, 0x00024FB8}, {0x0002F93D, 0x00025044}, +{0x0002F93E, 0x00003FFC}, {0x0002F93F, 0x00004008}, {0x0002F940, 0x000076F4}, {0x0002F941, 0x000250F3}, +{0x0002F942, 0x000250F2}, {0x0002F943, 0x00025119}, {0x0002F944, 0x00025133}, {0x0002F945, 0x0000771E}, +{0x0002F946, 0x0000771F}, {0x0002F947, 0x0000771F}, {0x0002F948, 0x0000774A}, {0x0002F949, 0x00004039}, +{0x0002F94A, 0x0000778B}, {0x0002F94B, 0x00004046}, {0x0002F94C, 0x00004096}, {0x0002F94D, 0x0002541D}, +{0x0002F94E, 0x0000784E}, {0x0002F94F, 0x0000788C}, {0x0002F950, 0x000078CC}, {0x0002F951, 0x000040E3}, +{0x0002F952, 0x00025626}, {0x0002F953, 0x00007956}, {0x0002F954, 0x0002569A}, {0x0002F955, 0x000256C5}, +{0x0002F956, 0x0000798F}, {0x0002F957, 0x000079EB}, {0x0002F958, 0x0000412F}, {0x0002F959, 0x00007A40}, +{0x0002F95A, 0x00007A4A}, {0x0002F95B, 0x00007A4F}, {0x0002F95C, 0x0002597C}, {0x0002F95D, 0x00025AA7}, +{0x0002F95E, 0x00025AA7}, {0x0002F95F, 0x00007AEE}, {0x0002F960, 0x00004202}, {0x0002F961, 0x00025BAB}, +{0x0002F962, 0x00007BC6}, {0x0002F963, 0x00007BC9}, {0x0002F964, 0x00004227}, {0x0002F965, 0x00025C80}, +{0x0002F966, 0x00007CD2}, {0x0002F967, 0x000042A0}, {0x0002F968, 0x00007CE8}, {0x0002F969, 0x00007CE3}, +{0x0002F96A, 0x00007D00}, {0x0002F96B, 0x00025F86}, {0x0002F96C, 0x00007D63}, {0x0002F96D, 0x00004301}, +{0x0002F96E, 0x00007DC7}, {0x0002F96F, 0x00007E02}, {0x0002F970, 0x00007E45}, {0x0002F971, 0x00004334}, +{0x0002F972, 0x00026228}, {0x0002F973, 0x00026247}, {0x0002F974, 0x00004359}, {0x0002F975, 0x000262D9}, +{0x0002F976, 0x00007F7A}, {0x0002F977, 0x0002633E}, {0x0002F978, 0x00007F95}, {0x0002F979, 0x00007FFA}, +{0x0002F97A, 0x00008005}, {0x0002F97B, 0x000264DA}, {0x0002F97C, 0x00026523}, {0x0002F97D, 0x00008060}, +{0x0002F97E, 0x000265A8}, {0x0002F97F, 0x00008070}, {0x0002F980, 0x0002335F}, {0x0002F981, 0x000043D5}, +{0x0002F982, 0x000080B2}, {0x0002F983, 0x00008103}, {0x0002F984, 0x0000440B}, {0x0002F985, 0x0000813E}, +{0x0002F986, 0x00005AB5}, {0x0002F987, 0x000267A7}, {0x0002F988, 0x000267B5}, {0x0002F989, 0x00023393}, +{0x0002F98A, 0x0002339C}, {0x0002F98B, 0x00008201}, {0x0002F98C, 0x00008204}, {0x0002F98D, 0x00008F9E}, +{0x0002F98E, 0x0000446B}, {0x0002F98F, 0x00008291}, {0x0002F990, 0x0000828B}, {0x0002F991, 0x0000829D}, +{0x0002F992, 0x000052B3}, {0x0002F993, 0x000082B1}, {0x0002F994, 0x000082B3}, {0x0002F995, 0x000082BD}, +{0x0002F996, 0x000082E6}, {0x0002F997, 0x00026B3C}, {0x0002F998, 0x000082E5}, {0x0002F999, 0x0000831D}, +{0x0002F99A, 0x00008363}, {0x0002F99B, 0x000083AD}, {0x0002F99C, 0x00008323}, {0x0002F99D, 0x000083BD}, +{0x0002F99E, 0x000083E7}, {0x0002F99F, 0x00008457}, {0x0002F9A0, 0x00008353}, {0x0002F9A1, 0x000083CA}, +{0x0002F9A2, 0x000083CC}, {0x0002F9A3, 0x000083DC}, {0x0002F9A4, 0x00026C36}, {0x0002F9A5, 0x00026D6B}, +{0x0002F9A6, 0x00026CD5}, {0x0002F9A7, 0x0000452B}, {0x0002F9A8, 0x000084F1}, {0x0002F9A9, 0x000084F3}, +{0x0002F9AA, 0x00008516}, {0x0002F9AB, 0x000273CA}, {0x0002F9AC, 0x00008564}, {0x0002F9AD, 0x00026F2C}, +{0x0002F9AE, 0x0000455D}, {0x0002F9AF, 0x00004561}, {0x0002F9B0, 0x00026FB1}, {0x0002F9B1, 0x000270D2}, +{0x0002F9B2, 0x0000456B}, {0x0002F9B3, 0x00008650}, {0x0002F9B4, 0x0000865C}, {0x0002F9B5, 0x00008667}, +{0x0002F9B6, 0x00008669}, {0x0002F9B7, 0x000086A9}, {0x0002F9B8, 0x00008688}, {0x0002F9B9, 0x0000870E}, +{0x0002F9BA, 0x000086E2}, {0x0002F9BB, 0x00008779}, {0x0002F9BC, 0x00008728}, {0x0002F9BD, 0x0000876B}, +{0x0002F9BE, 0x00008786}, {0x0002F9BF, 0x000045D7}, {0x0002F9C0, 0x000087E1}, {0x0002F9C1, 0x00008801}, +{0x0002F9C2, 0x000045F9}, {0x0002F9C3, 0x00008860}, {0x0002F9C4, 0x00008863}, {0x0002F9C5, 0x00027667}, +{0x0002F9C6, 0x000088D7}, {0x0002F9C7, 0x000088DE}, {0x0002F9C8, 0x00004635}, {0x0002F9C9, 0x000088FA}, +{0x0002F9CA, 0x000034BB}, {0x0002F9CB, 0x000278AE}, {0x0002F9CC, 0x00027966}, {0x0002F9CD, 0x000046BE}, +{0x0002F9CE, 0x000046C7}, {0x0002F9CF, 0x00008AA0}, {0x0002F9D0, 0x00008AED}, {0x0002F9D1, 0x00008B8A}, +{0x0002F9D2, 0x00008C55}, {0x0002F9D3, 0x00027CA8}, {0x0002F9D4, 0x00008CAB}, {0x0002F9D5, 0x00008CC1}, +{0x0002F9D6, 0x00008D1B}, {0x0002F9D7, 0x00008D77}, {0x0002F9D8, 0x00027F2F}, {0x0002F9D9, 0x00020804}, +{0x0002F9DA, 0x00008DCB}, {0x0002F9DB, 0x00008DBC}, {0x0002F9DC, 0x00008DF0}, {0x0002F9DD, 0x000208DE}, +{0x0002F9DE, 0x00008ED4}, {0x0002F9DF, 0x00008F38}, {0x0002F9E0, 0x000285D2}, {0x0002F9E1, 0x000285ED}, +{0x0002F9E2, 0x00009094}, {0x0002F9E3, 0x000090F1}, {0x0002F9E4, 0x00009111}, {0x0002F9E5, 0x0002872E}, +{0x0002F9E6, 0x0000911B}, {0x0002F9E7, 0x00009238}, {0x0002F9E8, 0x000092D7}, {0x0002F9E9, 0x000092D8}, +{0x0002F9EA, 0x0000927C}, {0x0002F9EB, 0x000093F9}, {0x0002F9EC, 0x00009415}, {0x0002F9ED, 0x00028BFA}, +{0x0002F9EE, 0x0000958B}, {0x0002F9EF, 0x00004995}, {0x0002F9F0, 0x000095B7}, {0x0002F9F1, 0x00028D77}, +{0x0002F9F2, 0x000049E6}, {0x0002F9F3, 0x000096C3}, {0x0002F9F4, 0x00005DB2}, {0x0002F9F5, 0x00009723}, +{0x0002F9F6, 0x00029145}, {0x0002F9F7, 0x0002921A}, {0x0002F9F8, 0x00004A6E}, {0x0002F9F9, 0x00004A76}, +{0x0002F9FA, 0x000097E0}, {0x0002F9FB, 0x0002940A}, {0x0002F9FC, 0x00004AB2}, {0x0002F9FD, 0x00029496}, +{0x0002F9FE, 0x0000980B}, {0x0002F9FF, 0x0000980B}, {0x0002FA00, 0x00009829}, {0x0002FA01, 0x000295B6}, +{0x0002FA02, 0x000098E2}, {0x0002FA03, 0x00004B33}, {0x0002FA04, 0x00009929}, {0x0002FA05, 0x000099A7}, +{0x0002FA06, 0x000099C2}, {0x0002FA07, 0x000099FE}, {0x0002FA08, 0x00004BCE}, {0x0002FA09, 0x00029B30}, +{0x0002FA0A, 0x00009B12}, {0x0002FA0B, 0x00009C40}, {0x0002FA0C, 0x00009CFD}, {0x0002FA0D, 0x00004CCE}, +{0x0002FA0E, 0x00004CED}, {0x0002FA0F, 0x00009D67}, {0x0002FA10, 0x0002A0CE}, {0x0002FA11, 0x00004CF8}, +{0x0002FA12, 0x0002A105}, {0x0002FA13, 0x0002A20E}, {0x0002FA14, 0x0002A291}, {0x0002FA15, 0x00009EBB}, +{0x0002FA16, 0x00004D56}, {0x0002FA17, 0x00009EF9}, {0x0002FA18, 0x00009EFE}, {0x0002FA19, 0x00009F05}, +{0x0002FA1A, 0x00009F0F}, {0x0002FA1B, 0x00009F16}, {0x0002FA1D, 0x0002A600}, +}; + +static std::string unicode_cpts_to_utf8(const std::vector & cps) { + std::string result; + for (size_t i = 0; i < cps.size(); ++i) { + result.append(unicode_cpt_to_utf8(cps[i])); + } + return result; +} + +static uint32_t unicode_cpt_from_utf8(const std::string & utf8, size_t & offset) { + assert(offset < utf8.size()); + if (!(utf8[offset + 0] & 0x80)) { + auto result = utf8[offset + 0]; + offset += 1; + return result; + } + if (!(utf8[offset + 0] & 0x40)) { + throw std::invalid_argument("invalid character"); + } + if (!(utf8[offset + 0] & 0x20)) { + if (offset + 1 >= utf8.size() || ! ((utf8[offset + 1] & 0xc0) == 0x80)) { + throw std::invalid_argument("invalid character"); + } + auto result = ((utf8[offset + 0] & 0x1f) << 6) | (utf8[offset + 1] & 0x3f); + offset += 2; + return result; + } + if (!(utf8[offset + 0] & 0x10)) { + if (offset + 2 >= utf8.size() || ! ((utf8[offset + 1] & 0xc0) == 0x80) || ! ((utf8[offset + 2] & 0xc0) == 0x80)) { + throw std::invalid_argument("invalid character"); + } + auto result = ((utf8[offset + 0] & 0x0f) << 12) | ((utf8[offset + 1] & 0x3f) << 6) | (utf8[offset + 2] & 0x3f); + offset += 3; + return result; + } + if (!(utf8[offset + 0] & 0x08)) { + if (offset + 3 >= utf8.size() || ! ((utf8[offset + 1] & 0xc0) == 0x80) || ! ((utf8[offset + 2] & 0xc0) == 0x80) || !((utf8[offset + 3] & 0xc0) == 0x80)) { + throw std::invalid_argument("invalid character"); + } + auto result = ((utf8[offset + 0] & 0x07) << 18) | ((utf8[offset + 1] & 0x3f) << 12) | ((utf8[offset + 2] & 0x3f) << 6) | (utf8[offset + 3] & 0x3f); + offset += 4; + return result; + } + throw std::invalid_argument("invalid string"); +} + +static std::vector unicode_cpt_to_utf16(uint32_t cp) { + std::vector result; + if (/* 0x0000 <= cp && */ cp <= 0xffff) { + result.emplace_back(cp); + } + else if (0x10000 <= cp && cp <= 0x10ffff) { + result.emplace_back(0xd800 | ((cp - 0x10000) >> 10)); + result.emplace_back(0xdc00 | ((cp - 0x10000) & 0x03ff)); + } + else { + throw std::invalid_argument("invalid cpt"); + } + return result; +} + +//static std::vector unicode_cpts_to_utf16(const std::vector & cps) { +// std::vector result; +// for (size_t i = 0; i < cps.size(); ++i) { +// auto temp = unicode_cpt_to_utf16(cps[i]); +// result.insert(result.end(), temp.begin(), temp.end()); +// } +// return result; +//} + +static uint32_t cpt_from_utf16(const std::vector & utf16, size_t & offset) { + assert(offset < utf16.size()); + if (((utf16[0] >> 10) << 10) != 0xd800) { + auto result = utf16[offset + 0]; + offset += 1; + return result; + } + + if (offset + 1 >= utf16.size() || !((utf16[1] & 0xdc00) == 0xdc00)) { + throw std::invalid_argument("invalid character"); + } + + auto result = 0x10000 + (((utf16[0] & 0x03ff) << 10) | (utf16[1] & 0x03ff)); + offset += 2; + return result; +} + +//static std::vector unicode_cpts_from_utf16(const std::vector & utf16) { +// std::vector result; +// size_t offset = 0; +// while (offset < utf16.size()) { +// result.push_back(cpt_from_utf16(utf16, offset)); +// } +// return result; +//} + +static std::unordered_map unicode_cpt_type_map() { + std::unordered_map cpt_types; + for (auto p : unicode_ranges_digit) { + for (auto i = p.first; i <= p.second; ++ i) { + cpt_types[i] = CODEPOINT_TYPE_DIGIT; + } + } + for (auto p : unicode_ranges_letter) { + for (auto i = p.first; i <= p.second; ++ i) { + cpt_types[i] = CODEPOINT_TYPE_LETTER; + } + } + for (auto p : unicode_ranges_whitespace) { + for (auto i = p.first; i <= p.second; ++ i) { + cpt_types[i] = CODEPOINT_TYPE_WHITESPACE; + } + } + for (auto p : unicode_ranges_accent_mark) { + for (auto i = p.first; i <= p.second; ++ i) { + cpt_types[i] = CODEPOINT_TYPE_ACCENT_MARK; + } + } + for (auto p : unicode_ranges_punctuation) { + for (auto i = p.first; i <= p.second; ++ i) { + cpt_types[i] = CODEPOINT_TYPE_PUNCTUATION; + } + } + for (auto p : unicode_ranges_symbol) { + for (auto i = p.first; i <= p.second; ++i) { + cpt_types[i] = CODEPOINT_TYPE_SYMBOL; + } + } + for (auto p : unicode_ranges_control) { + for (auto i = p.first; i <= p.second; ++ i) { + cpt_types[i] = CODEPOINT_TYPE_CONTROL; + } + } + return cpt_types; +} + +static std::unordered_map unicode_byte_to_utf8_map() { + std::unordered_map map; + for (int ch = u'!'; ch <= u'~'; ++ch) { + assert(0 <= ch && ch < 256); + map[ch] = unicode_cpt_to_utf8(ch); + } + for (int ch = u'¡'; ch <= u'¬'; ++ch) { + assert(0 <= ch && ch < 256); + map[ch] = unicode_cpt_to_utf8(ch); + } + for (int ch = u'®'; ch <= u'ÿ'; ++ch) { + assert(0 <= ch && ch < 256); + map[ch] = unicode_cpt_to_utf8(ch); + } + auto n = 0; + for (int ch = 0; ch < 256; ++ch) { + if (map.find(ch) == map.end()) { + map[ch] = unicode_cpt_to_utf8(256 + n); + ++n; + } + } + return map; +} + +static std::unordered_map unicode_utf8_to_byte_map() { + std::unordered_map map; + for (int ch = u'!'; ch <= u'~'; ++ch) { + assert(0 <= ch && ch < 256); + map[unicode_cpt_to_utf8(ch)] = ch; + } + for (int ch = u'¡'; ch <= u'¬'; ++ch) { + assert(0 <= ch && ch < 256); + map[unicode_cpt_to_utf8(ch)] = ch; + } + for (int ch = u'®'; ch <= u'ÿ'; ++ch) { + assert(0 <= ch && ch < 256); + map[unicode_cpt_to_utf8(ch)] = ch; + } + auto n = 0; + for (int ch = 0; ch < 256; ++ch) { + if (map.find(unicode_cpt_to_utf8(ch)) == map.end()) { + map[unicode_cpt_to_utf8(256 + n)] = ch; + ++n; + } + } + return map; +} + +// +// interface +// + +std::string unicode_cpt_to_utf8(uint32_t cp) { + std::string result; + if (/* 0x00 <= cp && */ cp <= 0x7f) { + result.push_back(cp); + } + else if (0x80 <= cp && cp <= 0x7ff) { + result.push_back(0xc0 | ((cp >> 6) & 0x1f)); + result.push_back(0x80 | (cp & 0x3f)); + } + else if (0x800 <= cp && cp <= 0xffff) { + result.push_back(0xe0 | ((cp >> 12) & 0x0f)); + result.push_back(0x80 | ((cp >> 6) & 0x3f)); + result.push_back(0x80 | (cp & 0x3f)); + } + else if (0x10000 <= cp && cp <= 0x10ffff) { + result.push_back(0xf0 | ((cp >> 18) & 0x07)); + result.push_back(0x80 | ((cp >> 12) & 0x3f)); + result.push_back(0x80 | ((cp >> 6) & 0x3f)); + result.push_back(0x80 | (cp & 0x3f)); + } + else { + throw std::invalid_argument("invalid codepoint"); + } + return result; +} + +std::vector unicode_cpts_normalize_nfd(const std::vector & cpts) { + std::vector result; + result.reserve(cpts.size()); + for (size_t i = 0; i < cpts.size(); ++i) { + auto it = unicode_map_nfd.find(cpts[i]); + if (it == unicode_map_nfd.end()) { + result.push_back(cpts[i]); + } else { + result.push_back(it->second); + } + } + return result; +} + +std::vector unicode_cpts_from_utf8(const std::string & utf8) { + std::vector result; + size_t offset = 0; + while (offset < utf8.size()) { + result.push_back(unicode_cpt_from_utf8(utf8, offset)); + } + return result; +} + +int unicode_cpt_type(uint32_t cp) { + static std::unordered_map cpt_types = unicode_cpt_type_map(); + const auto it = cpt_types.find(cp); + return it == cpt_types.end() ? CODEPOINT_TYPE_UNIDENTIFIED : it->second; +} + +int unicode_cpt_type(const std::string & utf8) { + if (utf8.length() == 0) { + return CODEPOINT_TYPE_UNIDENTIFIED; + } + size_t offset = 0; + return unicode_cpt_type(unicode_cpt_from_utf8(utf8, offset)); +} + +std::string unicode_byte_to_utf8(uint8_t byte) { + static std::unordered_map map = unicode_byte_to_utf8_map(); + return map.at(byte); +} + +uint8_t unicode_utf8_to_byte(const std::string & utf8) { + static std::unordered_map map = unicode_utf8_to_byte_map(); + return map.at(utf8); +} + diff --git a/unicode.h b/unicode.h index f6be4549b..6d14a5a33 100644 --- a/unicode.h +++ b/unicode.h @@ -1,784 +1,26 @@ -#pragma once +#pragma once -#include -#include -#include +#include #include -#include #include -static const std::vector> digit_ranges = { -{0x30, 0x39}, {0xB2, 0xB3}, {0xB9, 0xB9}, {0x660, 0x669}, {0x6F0, 0x6F9}, {0x7C0, 0x7C9}, {0x966, 0x96F}, {0x9E6, 0x9EF}, {0xA66, 0xA6F}, {0xAE6, 0xAEF}, {0xB66, 0xB6F}, {0xBE6, 0xBEF}, {0xC66, 0xC6F}, -{0xCE6, 0xCEF}, {0xD66, 0xD6F}, {0xDE6, 0xDEF}, {0xE50, 0xE59}, {0xED0, 0xED9}, {0xF20, 0xF29}, {0x1040, 0x1049}, {0x1090, 0x1099}, {0x1369, 0x1371}, {0x17E0, 0x17E9}, {0x1810, 0x1819}, {0x1946, 0x194F}, -{0x19D0, 0x19DA}, {0x1A80, 0x1A89}, {0x1A90, 0x1A99}, {0x1B50, 0x1B59}, {0x1BB0, 0x1BB9}, {0x1C40, 0x1C49}, {0x1C50, 0x1C59}, {0x2070, 0x2070}, {0x2074, 0x2079}, {0x2080, 0x2089}, {0x2460, 0x2468}, -{0x2474, 0x247C}, {0x2488, 0x2490}, {0x24EA, 0x24EA}, {0x24F5, 0x24FD}, {0x24FF, 0x24FF}, {0x2776, 0x277E}, {0x2780, 0x2788}, {0x278A, 0x2792}, {0xA620, 0xA629}, {0xA8D0, 0xA8D9}, {0xA900, 0xA909}, -{0xA9D0, 0xA9D9}, {0xA9F0, 0xA9F9}, {0xAA50, 0xAA59}, {0xABF0, 0xABF9}, {0xFF10, 0xFF19}, {0x104A0, 0x104A9}, {0x10A40, 0x10A43}, {0x10D30, 0x10D39}, {0x10E60, 0x10E68}, {0x11052, 0x1105A}, -{0x11066, 0x1106F}, {0x110F0, 0x110F9}, {0x11136, 0x1113F}, {0x111D0, 0x111D9}, {0x112F0, 0x112F9}, {0x11450, 0x11459}, {0x114D0, 0x114D9}, {0x11650, 0x11659}, {0x116C0, 0x116C9}, {0x11730, 0x11739}, -{0x118E0, 0x118E9}, {0x11950, 0x11959}, {0x11C50, 0x11C59}, {0x11D50, 0x11D59}, {0x11DA0, 0x11DA9}, {0x16A60, 0x16A69}, {0x16B50, 0x16B59}, {0x1D7CE, 0x1D7FF}, {0x1E140, 0x1E149}, {0x1E2F0, 0x1E2F9}, -{0x1E950, 0x1E959}, {0x1F100, 0x1F10A}, {0x1FBF0, 0x1FBF9}, -}; - -static const std::vector> letter_ranges = { -{0x41, 0x5A}, {0x61, 0x7A}, {0xAA, 0xAA}, {0xB5, 0xB5}, {0xBA, 0xBA}, {0xC0, 0xD6}, {0xD8, 0xF6}, {0xF8, 0x2C1}, {0x2C6, 0x2D1}, {0x2E0, 0x2E4}, {0x2EC, 0x2EC}, {0x2EE, 0x2EE}, {0x370, 0x374}, -{0x376, 0x377}, {0x37A, 0x37D}, {0x37F, 0x37F}, {0x386, 0x386}, {0x388, 0x38A}, {0x38C, 0x38C}, {0x38E, 0x3A1}, {0x3A3, 0x3F5}, {0x3F7, 0x481}, {0x48A, 0x52F}, {0x531, 0x556}, {0x559, 0x559}, -{0x560, 0x588}, {0x5D0, 0x5EA}, {0x5EF, 0x5F2}, {0x620, 0x64A}, {0x66E, 0x66F}, {0x671, 0x6D3}, {0x6D5, 0x6D5}, {0x6E5, 0x6E6}, {0x6EE, 0x6EF}, {0x6FA, 0x6FC}, {0x6FF, 0x6FF}, {0x710, 0x710}, -{0x712, 0x72F}, {0x74D, 0x7A5}, {0x7B1, 0x7B1}, {0x7CA, 0x7EA}, {0x7F4, 0x7F5}, {0x7FA, 0x7FA}, {0x800, 0x815}, {0x81A, 0x81A}, {0x824, 0x824}, {0x828, 0x828}, {0x840, 0x858}, {0x860, 0x86A}, -{0x8A0, 0x8B4}, {0x8B6, 0x8C7}, {0x904, 0x939}, {0x93D, 0x93D}, {0x950, 0x950}, {0x958, 0x961}, {0x971, 0x980}, {0x985, 0x98C}, {0x98F, 0x990}, {0x993, 0x9A8}, {0x9AA, 0x9B0}, {0x9B2, 0x9B2}, -{0x9B6, 0x9B9}, {0x9BD, 0x9BD}, {0x9CE, 0x9CE}, {0x9DC, 0x9DD}, {0x9DF, 0x9E1}, {0x9F0, 0x9F1}, {0x9FC, 0x9FC}, {0xA05, 0xA0A}, {0xA0F, 0xA10}, {0xA13, 0xA28}, {0xA2A, 0xA30}, {0xA32, 0xA33}, -{0xA35, 0xA36}, {0xA38, 0xA39}, {0xA59, 0xA5C}, {0xA5E, 0xA5E}, {0xA72, 0xA74}, {0xA85, 0xA8D}, {0xA8F, 0xA91}, {0xA93, 0xAA8}, {0xAAA, 0xAB0}, {0xAB2, 0xAB3}, {0xAB5, 0xAB9}, {0xABD, 0xABD}, -{0xAD0, 0xAD0}, {0xAE0, 0xAE1}, {0xAF9, 0xAF9}, {0xB05, 0xB0C}, {0xB0F, 0xB10}, {0xB13, 0xB28}, {0xB2A, 0xB30}, {0xB32, 0xB33}, {0xB35, 0xB39}, {0xB3D, 0xB3D}, {0xB5C, 0xB5D}, {0xB5F, 0xB61}, -{0xB71, 0xB71}, {0xB83, 0xB83}, {0xB85, 0xB8A}, {0xB8E, 0xB90}, {0xB92, 0xB95}, {0xB99, 0xB9A}, {0xB9C, 0xB9C}, {0xB9E, 0xB9F}, {0xBA3, 0xBA4}, {0xBA8, 0xBAA}, {0xBAE, 0xBB9}, {0xBD0, 0xBD0}, -{0xC05, 0xC0C}, {0xC0E, 0xC10}, {0xC12, 0xC28}, {0xC2A, 0xC39}, {0xC3D, 0xC3D}, {0xC58, 0xC5A}, {0xC60, 0xC61}, {0xC80, 0xC80}, {0xC85, 0xC8C}, {0xC8E, 0xC90}, {0xC92, 0xCA8}, {0xCAA, 0xCB3}, -{0xCB5, 0xCB9}, {0xCBD, 0xCBD}, {0xCDE, 0xCDE}, {0xCE0, 0xCE1}, {0xCF1, 0xCF2}, {0xD04, 0xD0C}, {0xD0E, 0xD10}, {0xD12, 0xD3A}, {0xD3D, 0xD3D}, {0xD4E, 0xD4E}, {0xD54, 0xD56}, {0xD5F, 0xD61}, -{0xD7A, 0xD7F}, {0xD85, 0xD96}, {0xD9A, 0xDB1}, {0xDB3, 0xDBB}, {0xDBD, 0xDBD}, {0xDC0, 0xDC6}, {0xE01, 0xE30}, {0xE32, 0xE33}, {0xE40, 0xE46}, {0xE81, 0xE82}, {0xE84, 0xE84}, {0xE86, 0xE8A}, -{0xE8C, 0xEA3}, {0xEA5, 0xEA5}, {0xEA7, 0xEB0}, {0xEB2, 0xEB3}, {0xEBD, 0xEBD}, {0xEC0, 0xEC4}, {0xEC6, 0xEC6}, {0xEDC, 0xEDF}, {0xF00, 0xF00}, {0xF40, 0xF47}, {0xF49, 0xF6C}, {0xF88, 0xF8C}, -{0x1000, 0x102A}, {0x103F, 0x103F}, {0x1050, 0x1055}, {0x105A, 0x105D}, {0x1061, 0x1061}, {0x1065, 0x1066}, {0x106E, 0x1070}, {0x1075, 0x1081}, {0x108E, 0x108E}, {0x10A0, 0x10C5}, {0x10C7, 0x10C7}, -{0x10CD, 0x10CD}, {0x10D0, 0x10FA}, {0x10FC, 0x1248}, {0x124A, 0x124D}, {0x1250, 0x1256}, {0x1258, 0x1258}, {0x125A, 0x125D}, {0x1260, 0x1288}, {0x128A, 0x128D}, {0x1290, 0x12B0}, {0x12B2, 0x12B5}, -{0x12B8, 0x12BE}, {0x12C0, 0x12C0}, {0x12C2, 0x12C5}, {0x12C8, 0x12D6}, {0x12D8, 0x1310}, {0x1312, 0x1315}, {0x1318, 0x135A}, {0x1380, 0x138F}, {0x13A0, 0x13F5}, {0x13F8, 0x13FD}, {0x1401, 0x166C}, -{0x166F, 0x167F}, {0x1681, 0x169A}, {0x16A0, 0x16EA}, {0x16F1, 0x16F8}, {0x1700, 0x170C}, {0x170E, 0x1711}, {0x1720, 0x1731}, {0x1740, 0x1751}, {0x1760, 0x176C}, {0x176E, 0x1770}, {0x1780, 0x17B3}, -{0x17D7, 0x17D7}, {0x17DC, 0x17DC}, {0x1820, 0x1878}, {0x1880, 0x1884}, {0x1887, 0x18A8}, {0x18AA, 0x18AA}, {0x18B0, 0x18F5}, {0x1900, 0x191E}, {0x1950, 0x196D}, {0x1970, 0x1974}, {0x1980, 0x19AB}, -{0x19B0, 0x19C9}, {0x1A00, 0x1A16}, {0x1A20, 0x1A54}, {0x1AA7, 0x1AA7}, {0x1B05, 0x1B33}, {0x1B45, 0x1B4B}, {0x1B83, 0x1BA0}, {0x1BAE, 0x1BAF}, {0x1BBA, 0x1BE5}, {0x1C00, 0x1C23}, {0x1C4D, 0x1C4F}, -{0x1C5A, 0x1C7D}, {0x1C80, 0x1C88}, {0x1C90, 0x1CBA}, {0x1CBD, 0x1CBF}, {0x1CE9, 0x1CEC}, {0x1CEE, 0x1CF3}, {0x1CF5, 0x1CF6}, {0x1CFA, 0x1CFA}, {0x1D00, 0x1DBF}, {0x1E00, 0x1F15}, {0x1F18, 0x1F1D}, -{0x1F20, 0x1F45}, {0x1F48, 0x1F4D}, {0x1F50, 0x1F57}, {0x1F59, 0x1F59}, {0x1F5B, 0x1F5B}, {0x1F5D, 0x1F5D}, {0x1F5F, 0x1F7D}, {0x1F80, 0x1FB4}, {0x1FB6, 0x1FBC}, {0x1FBE, 0x1FBE}, {0x1FC2, 0x1FC4}, -{0x1FC6, 0x1FCC}, {0x1FD0, 0x1FD3}, {0x1FD6, 0x1FDB}, {0x1FE0, 0x1FEC}, {0x1FF2, 0x1FF4}, {0x1FF6, 0x1FFC}, {0x2071, 0x2071}, {0x207F, 0x207F}, {0x2090, 0x209C}, {0x2102, 0x2102}, {0x2107, 0x2107}, -{0x210A, 0x2113}, {0x2115, 0x2115}, {0x2119, 0x211D}, {0x2124, 0x2124}, {0x2126, 0x2126}, {0x2128, 0x2128}, {0x212A, 0x212D}, {0x212F, 0x2139}, {0x213C, 0x213F}, {0x2145, 0x2149}, {0x214E, 0x214E}, -{0x2183, 0x2184}, {0x2C00, 0x2C2E}, {0x2C30, 0x2C5E}, {0x2C60, 0x2CE4}, {0x2CEB, 0x2CEE}, {0x2CF2, 0x2CF3}, {0x2D00, 0x2D25}, {0x2D27, 0x2D27}, {0x2D2D, 0x2D2D}, {0x2D30, 0x2D67}, {0x2D6F, 0x2D6F}, -{0x2D80, 0x2D96}, {0x2DA0, 0x2DA6}, {0x2DA8, 0x2DAE}, {0x2DB0, 0x2DB6}, {0x2DB8, 0x2DBE}, {0x2DC0, 0x2DC6}, {0x2DC8, 0x2DCE}, {0x2DD0, 0x2DD6}, {0x2DD8, 0x2DDE}, {0x2E2F, 0x2E2F}, {0x3005, 0x3006}, -{0x3031, 0x3035}, {0x303B, 0x303C}, {0x3041, 0x3096}, {0x309D, 0x309F}, {0x30A1, 0x30FA}, {0x30FC, 0x30FF}, {0x3105, 0x312F}, {0x3131, 0x318E}, {0x31A0, 0x31BF}, {0x31F0, 0x31FF}, {0x3400, 0x4DBF}, -{0x4E00, 0x9FFC}, {0xA000, 0xA48C}, {0xA4D0, 0xA4FD}, {0xA500, 0xA60C}, {0xA610, 0xA61F}, {0xA62A, 0xA62B}, {0xA640, 0xA66E}, {0xA67F, 0xA69D}, {0xA6A0, 0xA6E5}, {0xA717, 0xA71F}, {0xA722, 0xA788}, -{0xA78B, 0xA7BF}, {0xA7C2, 0xA7CA}, {0xA7F5, 0xA801}, {0xA803, 0xA805}, {0xA807, 0xA80A}, {0xA80C, 0xA822}, {0xA840, 0xA873}, {0xA882, 0xA8B3}, {0xA8F2, 0xA8F7}, {0xA8FB, 0xA8FB}, {0xA8FD, 0xA8FE}, -{0xA90A, 0xA925}, {0xA930, 0xA946}, {0xA960, 0xA97C}, {0xA984, 0xA9B2}, {0xA9CF, 0xA9CF}, {0xA9E0, 0xA9E4}, {0xA9E6, 0xA9EF}, {0xA9FA, 0xA9FE}, {0xAA00, 0xAA28}, {0xAA40, 0xAA42}, {0xAA44, 0xAA4B}, -{0xAA60, 0xAA76}, {0xAA7A, 0xAA7A}, {0xAA7E, 0xAAAF}, {0xAAB1, 0xAAB1}, {0xAAB5, 0xAAB6}, {0xAAB9, 0xAABD}, {0xAAC0, 0xAAC0}, {0xAAC2, 0xAAC2}, {0xAADB, 0xAADD}, {0xAAE0, 0xAAEA}, {0xAAF2, 0xAAF4}, -{0xAB01, 0xAB06}, {0xAB09, 0xAB0E}, {0xAB11, 0xAB16}, {0xAB20, 0xAB26}, {0xAB28, 0xAB2E}, {0xAB30, 0xAB5A}, {0xAB5C, 0xAB69}, {0xAB70, 0xABE2}, {0xAC00, 0xD7A3}, {0xD7B0, 0xD7C6}, {0xD7CB, 0xD7FB}, -{0xF900, 0xFA6D}, {0xFA70, 0xFAD9}, {0xFB00, 0xFB06}, {0xFB13, 0xFB17}, {0xFB1D, 0xFB1D}, {0xFB1F, 0xFB28}, {0xFB2A, 0xFB36}, {0xFB38, 0xFB3C}, {0xFB3E, 0xFB3E}, {0xFB40, 0xFB41}, {0xFB43, 0xFB44}, -{0xFB46, 0xFBB1}, {0xFBD3, 0xFD3D}, {0xFD50, 0xFD8F}, {0xFD92, 0xFDC7}, {0xFDF0, 0xFDFB}, {0xFE70, 0xFE74}, {0xFE76, 0xFEFC}, {0xFF21, 0xFF3A}, {0xFF41, 0xFF5A}, {0xFF66, 0xFFBE}, {0xFFC2, 0xFFC7}, -{0xFFCA, 0xFFCF}, {0xFFD2, 0xFFD7}, {0xFFDA, 0xFFDC}, {0x10000, 0x1000B}, {0x1000D, 0x10026}, {0x10028, 0x1003A}, {0x1003C, 0x1003D}, {0x1003F, 0x1004D}, {0x10050, 0x1005D}, {0x10080, 0x100FA}, -{0x10280, 0x1029C}, {0x102A0, 0x102D0}, {0x10300, 0x1031F}, {0x1032D, 0x10340}, {0x10342, 0x10349}, {0x10350, 0x10375}, {0x10380, 0x1039D}, {0x103A0, 0x103C3}, {0x103C8, 0x103CF}, {0x10400, 0x1049D}, -{0x104B0, 0x104D3}, {0x104D8, 0x104FB}, {0x10500, 0x10527}, {0x10530, 0x10563}, {0x10600, 0x10736}, {0x10740, 0x10755}, {0x10760, 0x10767}, {0x10800, 0x10805}, {0x10808, 0x10808}, {0x1080A, 0x10835}, -{0x10837, 0x10838}, {0x1083C, 0x1083C}, {0x1083F, 0x10855}, {0x10860, 0x10876}, {0x10880, 0x1089E}, {0x108E0, 0x108F2}, {0x108F4, 0x108F5}, {0x10900, 0x10915}, {0x10920, 0x10939}, {0x10980, 0x109B7}, -{0x109BE, 0x109BF}, {0x10A00, 0x10A00}, {0x10A10, 0x10A13}, {0x10A15, 0x10A17}, {0x10A19, 0x10A35}, {0x10A60, 0x10A7C}, {0x10A80, 0x10A9C}, {0x10AC0, 0x10AC7}, {0x10AC9, 0x10AE4}, {0x10B00, 0x10B35}, -{0x10B40, 0x10B55}, {0x10B60, 0x10B72}, {0x10B80, 0x10B91}, {0x10C00, 0x10C48}, {0x10C80, 0x10CB2}, {0x10CC0, 0x10CF2}, {0x10D00, 0x10D23}, {0x10E80, 0x10EA9}, {0x10EB0, 0x10EB1}, {0x10F00, 0x10F1C}, -{0x10F27, 0x10F27}, {0x10F30, 0x10F45}, {0x10FB0, 0x10FC4}, {0x10FE0, 0x10FF6}, {0x11003, 0x11037}, {0x11083, 0x110AF}, {0x110D0, 0x110E8}, {0x11103, 0x11126}, {0x11144, 0x11144}, {0x11147, 0x11147}, -{0x11150, 0x11172}, {0x11176, 0x11176}, {0x11183, 0x111B2}, {0x111C1, 0x111C4}, {0x111DA, 0x111DA}, {0x111DC, 0x111DC}, {0x11200, 0x11211}, {0x11213, 0x1122B}, {0x11280, 0x11286}, {0x11288, 0x11288}, -{0x1128A, 0x1128D}, {0x1128F, 0x1129D}, {0x1129F, 0x112A8}, {0x112B0, 0x112DE}, {0x11305, 0x1130C}, {0x1130F, 0x11310}, {0x11313, 0x11328}, {0x1132A, 0x11330}, {0x11332, 0x11333}, {0x11335, 0x11339}, -{0x1133D, 0x1133D}, {0x11350, 0x11350}, {0x1135D, 0x11361}, {0x11400, 0x11434}, {0x11447, 0x1144A}, {0x1145F, 0x11461}, {0x11480, 0x114AF}, {0x114C4, 0x114C5}, {0x114C7, 0x114C7}, {0x11580, 0x115AE}, -{0x115D8, 0x115DB}, {0x11600, 0x1162F}, {0x11644, 0x11644}, {0x11680, 0x116AA}, {0x116B8, 0x116B8}, {0x11700, 0x1171A}, {0x11800, 0x1182B}, {0x118A0, 0x118DF}, {0x118FF, 0x11906}, {0x11909, 0x11909}, -{0x1190C, 0x11913}, {0x11915, 0x11916}, {0x11918, 0x1192F}, {0x1193F, 0x1193F}, {0x11941, 0x11941}, {0x119A0, 0x119A7}, {0x119AA, 0x119D0}, {0x119E1, 0x119E1}, {0x119E3, 0x119E3}, {0x11A00, 0x11A00}, -{0x11A0B, 0x11A32}, {0x11A3A, 0x11A3A}, {0x11A50, 0x11A50}, {0x11A5C, 0x11A89}, {0x11A9D, 0x11A9D}, {0x11AC0, 0x11AF8}, {0x11C00, 0x11C08}, {0x11C0A, 0x11C2E}, {0x11C40, 0x11C40}, {0x11C72, 0x11C8F}, -{0x11D00, 0x11D06}, {0x11D08, 0x11D09}, {0x11D0B, 0x11D30}, {0x11D46, 0x11D46}, {0x11D60, 0x11D65}, {0x11D67, 0x11D68}, {0x11D6A, 0x11D89}, {0x11D98, 0x11D98}, {0x11EE0, 0x11EF2}, {0x11FB0, 0x11FB0}, -{0x12000, 0x12399}, {0x12480, 0x12543}, {0x13000, 0x1342E}, {0x14400, 0x14646}, {0x16800, 0x16A38}, {0x16A40, 0x16A5E}, {0x16AD0, 0x16AED}, {0x16B00, 0x16B2F}, {0x16B40, 0x16B43}, {0x16B63, 0x16B77}, -{0x16B7D, 0x16B8F}, {0x16E40, 0x16E7F}, {0x16F00, 0x16F4A}, {0x16F50, 0x16F50}, {0x16F93, 0x16F9F}, {0x16FE0, 0x16FE1}, {0x16FE3, 0x16FE3}, {0x17000, 0x187F7}, {0x18800, 0x18CD5}, {0x18D00, 0x18D08}, -{0x1B000, 0x1B11E}, {0x1B150, 0x1B152}, {0x1B164, 0x1B167}, {0x1B170, 0x1B2FB}, {0x1BC00, 0x1BC6A}, {0x1BC70, 0x1BC7C}, {0x1BC80, 0x1BC88}, {0x1BC90, 0x1BC99}, {0x1D400, 0x1D454}, {0x1D456, 0x1D49C}, -{0x1D49E, 0x1D49F}, {0x1D4A2, 0x1D4A2}, {0x1D4A5, 0x1D4A6}, {0x1D4A9, 0x1D4AC}, {0x1D4AE, 0x1D4B9}, {0x1D4BB, 0x1D4BB}, {0x1D4BD, 0x1D4C3}, {0x1D4C5, 0x1D505}, {0x1D507, 0x1D50A}, {0x1D50D, 0x1D514}, -{0x1D516, 0x1D51C}, {0x1D51E, 0x1D539}, {0x1D53B, 0x1D53E}, {0x1D540, 0x1D544}, {0x1D546, 0x1D546}, {0x1D54A, 0x1D550}, {0x1D552, 0x1D6A5}, {0x1D6A8, 0x1D6C0}, {0x1D6C2, 0x1D6DA}, {0x1D6DC, 0x1D6FA}, -{0x1D6FC, 0x1D714}, {0x1D716, 0x1D734}, {0x1D736, 0x1D74E}, {0x1D750, 0x1D76E}, {0x1D770, 0x1D788}, {0x1D78A, 0x1D7A8}, {0x1D7AA, 0x1D7C2}, {0x1D7C4, 0x1D7CB}, {0x1E100, 0x1E12C}, {0x1E137, 0x1E13D}, -{0x1E14E, 0x1E14E}, {0x1E2C0, 0x1E2EB}, {0x1E800, 0x1E8C4}, {0x1E900, 0x1E943}, {0x1E94B, 0x1E94B}, {0x1EE00, 0x1EE03}, {0x1EE05, 0x1EE1F}, {0x1EE21, 0x1EE22}, {0x1EE24, 0x1EE24}, {0x1EE27, 0x1EE27}, -{0x1EE29, 0x1EE32}, {0x1EE34, 0x1EE37}, {0x1EE39, 0x1EE39}, {0x1EE3B, 0x1EE3B}, {0x1EE42, 0x1EE42}, {0x1EE47, 0x1EE47}, {0x1EE49, 0x1EE49}, {0x1EE4B, 0x1EE4B}, {0x1EE4D, 0x1EE4F}, {0x1EE51, 0x1EE52}, -{0x1EE54, 0x1EE54}, {0x1EE57, 0x1EE57}, {0x1EE59, 0x1EE59}, {0x1EE5B, 0x1EE5B}, {0x1EE5D, 0x1EE5D}, {0x1EE5F, 0x1EE5F}, {0x1EE61, 0x1EE62}, {0x1EE64, 0x1EE64}, {0x1EE67, 0x1EE6A}, {0x1EE6C, 0x1EE72}, -{0x1EE74, 0x1EE77}, {0x1EE79, 0x1EE7C}, {0x1EE7E, 0x1EE7E}, {0x1EE80, 0x1EE89}, {0x1EE8B, 0x1EE9B}, {0x1EEA1, 0x1EEA3}, {0x1EEA5, 0x1EEA9}, {0x1EEAB, 0x1EEBB}, {0x20000, 0x2A6DD}, {0x2A700, 0x2B734}, -{0x2B740, 0x2B81D}, {0x2B820, 0x2CEA1}, {0x2CEB0, 0x2EBE0}, {0x2F800, 0x2FA1D}, {0x30000, 0x3134A}, -}; - -static const std::vector> whitespace_ranges = { -{0x9, 0xD}, {0x1C, 0x20}, {0x85, 0x85}, {0xA0, 0xA0}, {0x1680, 0x1680}, {0x2000, 0x200A}, {0x2028, 0x2029}, {0x202F, 0x202F}, {0x205F, 0x205F}, {0x3000, 0x3000}, -}; - -static const std::vector> accent_mark_ranges = { -{0x300, 0x36F}, {0x483, 0x489}, {0x591, 0x5BD}, {0x5BF, 0x5BF}, {0x5C1, 0x5C2}, {0x5C4, 0x5C5}, {0x5C7, 0x5C7}, {0x610, 0x61A}, {0x64B, 0x65F}, {0x670, 0x670}, {0x6D6, 0x6DC}, {0x6DF, 0x6E4}, -{0x6E7, 0x6E8}, {0x6EA, 0x6ED}, {0x711, 0x711}, {0x730, 0x74A}, {0x7A6, 0x7B0}, {0x7EB, 0x7F3}, {0x7FD, 0x7FD}, {0x816, 0x819}, {0x81B, 0x823}, {0x825, 0x827}, {0x829, 0x82D}, {0x859, 0x85B}, -{0x8D3, 0x8E1}, {0x8E3, 0x903}, {0x93A, 0x93C}, {0x93E, 0x94F}, {0x951, 0x957}, {0x962, 0x963}, {0x981, 0x983}, {0x9BC, 0x9BC}, {0x9BE, 0x9C4}, {0x9C7, 0x9C8}, {0x9CB, 0x9CD}, {0x9D7, 0x9D7}, -{0x9E2, 0x9E3}, {0x9FE, 0x9FE}, {0xA01, 0xA03}, {0xA3C, 0xA3C}, {0xA3E, 0xA42}, {0xA47, 0xA48}, {0xA4B, 0xA4D}, {0xA51, 0xA51}, {0xA70, 0xA71}, {0xA75, 0xA75}, {0xA81, 0xA83}, {0xABC, 0xABC}, -{0xABE, 0xAC5}, {0xAC7, 0xAC9}, {0xACB, 0xACD}, {0xAE2, 0xAE3}, {0xAFA, 0xAFF}, {0xB01, 0xB03}, {0xB3C, 0xB3C}, {0xB3E, 0xB44}, {0xB47, 0xB48}, {0xB4B, 0xB4D}, {0xB55, 0xB57}, {0xB62, 0xB63}, -{0xB82, 0xB82}, {0xBBE, 0xBC2}, {0xBC6, 0xBC8}, {0xBCA, 0xBCD}, {0xBD7, 0xBD7}, {0xC00, 0xC04}, {0xC3E, 0xC44}, {0xC46, 0xC48}, {0xC4A, 0xC4D}, {0xC55, 0xC56}, {0xC62, 0xC63}, {0xC81, 0xC83}, -{0xCBC, 0xCBC}, {0xCBE, 0xCC4}, {0xCC6, 0xCC8}, {0xCCA, 0xCCD}, {0xCD5, 0xCD6}, {0xCE2, 0xCE3}, {0xD00, 0xD03}, {0xD3B, 0xD3C}, {0xD3E, 0xD44}, {0xD46, 0xD48}, {0xD4A, 0xD4D}, {0xD57, 0xD57}, -{0xD62, 0xD63}, {0xD81, 0xD83}, {0xDCA, 0xDCA}, {0xDCF, 0xDD4}, {0xDD6, 0xDD6}, {0xDD8, 0xDDF}, {0xDF2, 0xDF3}, {0xE31, 0xE31}, {0xE34, 0xE3A}, {0xE47, 0xE4E}, {0xEB1, 0xEB1}, {0xEB4, 0xEBC}, -{0xEC8, 0xECD}, {0xF18, 0xF19}, {0xF35, 0xF35}, {0xF37, 0xF37}, {0xF39, 0xF39}, {0xF3E, 0xF3F}, {0xF71, 0xF84}, {0xF86, 0xF87}, {0xF8D, 0xF97}, {0xF99, 0xFBC}, {0xFC6, 0xFC6}, {0x102B, 0x103E}, -{0x1056, 0x1059}, {0x105E, 0x1060}, {0x1062, 0x1064}, {0x1067, 0x106D}, {0x1071, 0x1074}, {0x1082, 0x108D}, {0x108F, 0x108F}, {0x109A, 0x109D}, {0x135D, 0x135F}, {0x1712, 0x1714}, {0x1732, 0x1734}, -{0x1752, 0x1753}, {0x1772, 0x1773}, {0x17B4, 0x17D3}, {0x17DD, 0x17DD}, {0x180B, 0x180D}, {0x1885, 0x1886}, {0x18A9, 0x18A9}, {0x1920, 0x192B}, {0x1930, 0x193B}, {0x1A17, 0x1A1B}, {0x1A55, 0x1A5E}, -{0x1A60, 0x1A7C}, {0x1A7F, 0x1A7F}, {0x1AB0, 0x1AC0}, {0x1B00, 0x1B04}, {0x1B34, 0x1B44}, {0x1B6B, 0x1B73}, {0x1B80, 0x1B82}, {0x1BA1, 0x1BAD}, {0x1BE6, 0x1BF3}, {0x1C24, 0x1C37}, {0x1CD0, 0x1CD2}, -{0x1CD4, 0x1CE8}, {0x1CED, 0x1CED}, {0x1CF4, 0x1CF4}, {0x1CF7, 0x1CF9}, {0x1DC0, 0x1DF9}, {0x1DFB, 0x1DFF}, {0x20D0, 0x20F0}, {0x2CEF, 0x2CF1}, {0x2D7F, 0x2D7F}, {0x2DE0, 0x2DFF}, {0x302A, 0x302F}, -{0x3099, 0x309A}, {0xA66F, 0xA672}, {0xA674, 0xA67D}, {0xA69E, 0xA69F}, {0xA6F0, 0xA6F1}, {0xA802, 0xA802}, {0xA806, 0xA806}, {0xA80B, 0xA80B}, {0xA823, 0xA827}, {0xA82C, 0xA82C}, {0xA880, 0xA881}, -{0xA8B4, 0xA8C5}, {0xA8E0, 0xA8F1}, {0xA8FF, 0xA8FF}, {0xA926, 0xA92D}, {0xA947, 0xA953}, {0xA980, 0xA983}, {0xA9B3, 0xA9C0}, {0xA9E5, 0xA9E5}, {0xAA29, 0xAA36}, {0xAA43, 0xAA43}, {0xAA4C, 0xAA4D}, -{0xAA7B, 0xAA7D}, {0xAAB0, 0xAAB0}, {0xAAB2, 0xAAB4}, {0xAAB7, 0xAAB8}, {0xAABE, 0xAABF}, {0xAAC1, 0xAAC1}, {0xAAEB, 0xAAEF}, {0xAAF5, 0xAAF6}, {0xABE3, 0xABEA}, {0xABEC, 0xABED}, {0xFB1E, 0xFB1E}, -{0xFE00, 0xFE0F}, {0xFE20, 0xFE2F}, {0x101FD, 0x101FD}, {0x102E0, 0x102E0}, {0x10376, 0x1037A}, {0x10A01, 0x10A03}, {0x10A05, 0x10A06}, {0x10A0C, 0x10A0F}, {0x10A38, 0x10A3A}, {0x10A3F, 0x10A3F}, -{0x10AE5, 0x10AE6}, {0x10D24, 0x10D27}, {0x10EAB, 0x10EAC}, {0x10F46, 0x10F50}, {0x11000, 0x11002}, {0x11038, 0x11046}, {0x1107F, 0x11082}, {0x110B0, 0x110BA}, {0x11100, 0x11102}, {0x11127, 0x11134}, -{0x11145, 0x11146}, {0x11173, 0x11173}, {0x11180, 0x11182}, {0x111B3, 0x111C0}, {0x111C9, 0x111CC}, {0x111CE, 0x111CF}, {0x1122C, 0x11237}, {0x1123E, 0x1123E}, {0x112DF, 0x112EA}, {0x11300, 0x11303}, -{0x1133B, 0x1133C}, {0x1133E, 0x11344}, {0x11347, 0x11348}, {0x1134B, 0x1134D}, {0x11357, 0x11357}, {0x11362, 0x11363}, {0x11366, 0x1136C}, {0x11370, 0x11374}, {0x11435, 0x11446}, {0x1145E, 0x1145E}, -{0x114B0, 0x114C3}, {0x115AF, 0x115B5}, {0x115B8, 0x115C0}, {0x115DC, 0x115DD}, {0x11630, 0x11640}, {0x116AB, 0x116B7}, {0x1171D, 0x1172B}, {0x1182C, 0x1183A}, {0x11930, 0x11935}, {0x11937, 0x11938}, -{0x1193B, 0x1193E}, {0x11940, 0x11940}, {0x11942, 0x11943}, {0x119D1, 0x119D7}, {0x119DA, 0x119E0}, {0x119E4, 0x119E4}, {0x11A01, 0x11A0A}, {0x11A33, 0x11A39}, {0x11A3B, 0x11A3E}, {0x11A47, 0x11A47}, -{0x11A51, 0x11A5B}, {0x11A8A, 0x11A99}, {0x11C2F, 0x11C36}, {0x11C38, 0x11C3F}, {0x11C92, 0x11CA7}, {0x11CA9, 0x11CB6}, {0x11D31, 0x11D36}, {0x11D3A, 0x11D3A}, {0x11D3C, 0x11D3D}, {0x11D3F, 0x11D45}, -{0x11D47, 0x11D47}, {0x11D8A, 0x11D8E}, {0x11D90, 0x11D91}, {0x11D93, 0x11D97}, {0x11EF3, 0x11EF6}, {0x16AF0, 0x16AF4}, {0x16B30, 0x16B36}, {0x16F4F, 0x16F4F}, {0x16F51, 0x16F87}, {0x16F8F, 0x16F92}, -{0x16FE4, 0x16FE4}, {0x16FF0, 0x16FF1}, {0x1BC9D, 0x1BC9E}, {0x1D165, 0x1D169}, {0x1D16D, 0x1D172}, {0x1D17B, 0x1D182}, {0x1D185, 0x1D18B}, {0x1D1AA, 0x1D1AD}, {0x1D242, 0x1D244}, {0x1DA00, 0x1DA36}, -{0x1DA3B, 0x1DA6C}, {0x1DA75, 0x1DA75}, {0x1DA84, 0x1DA84}, {0x1DA9B, 0x1DA9F}, {0x1DAA1, 0x1DAAF}, {0x1E000, 0x1E006}, {0x1E008, 0x1E018}, {0x1E01B, 0x1E021}, {0x1E023, 0x1E024}, {0x1E026, 0x1E02A}, -{0x1E130, 0x1E136}, {0x1E2EC, 0x1E2EF}, {0x1E8D0, 0x1E8D6}, {0x1E944, 0x1E94A}, {0xE0100, 0xE01EF}, -}; - -static const std::vector> punctuation_ranges = { -{0x21, 0x23}, {0x25, 0x2A}, {0x2C, 0x2F}, {0x3A, 0x3B}, {0x3F, 0x40}, {0x5B, 0x5D}, {0x5F, 0x5F}, {0x7B, 0x7B}, {0x7D, 0x7D}, {0xA1, 0xA1}, {0xA7, 0xA7}, {0xAB, 0xAB}, {0xB6, 0xB7}, {0xBB, 0xBB}, -{0xBF, 0xBF}, {0x37E, 0x37E}, {0x387, 0x387}, {0x55A, 0x55F}, {0x589, 0x58A}, {0x5BE, 0x5BE}, {0x5C0, 0x5C0}, {0x5C3, 0x5C3}, {0x5C6, 0x5C6}, {0x5F3, 0x5F4}, {0x609, 0x60A}, {0x60C, 0x60D}, -{0x61B, 0x61B}, {0x61E, 0x61F}, {0x66A, 0x66D}, {0x6D4, 0x6D4}, {0x700, 0x70D}, {0x7F7, 0x7F9}, {0x830, 0x83E}, {0x85E, 0x85E}, {0x964, 0x965}, {0x970, 0x970}, {0x9FD, 0x9FD}, {0xA76, 0xA76}, -{0xAF0, 0xAF0}, {0xC77, 0xC77}, {0xC84, 0xC84}, {0xDF4, 0xDF4}, {0xE4F, 0xE4F}, {0xE5A, 0xE5B}, {0xF04, 0xF12}, {0xF14, 0xF14}, {0xF3A, 0xF3D}, {0xF85, 0xF85}, {0xFD0, 0xFD4}, {0xFD9, 0xFDA}, -{0x104A, 0x104F}, {0x10FB, 0x10FB}, {0x1360, 0x1368}, {0x1400, 0x1400}, {0x166E, 0x166E}, {0x169B, 0x169C}, {0x16EB, 0x16ED}, {0x1735, 0x1736}, {0x17D4, 0x17D6}, {0x17D8, 0x17DA}, {0x1800, 0x180A}, -{0x1944, 0x1945}, {0x1A1E, 0x1A1F}, {0x1AA0, 0x1AA6}, {0x1AA8, 0x1AAD}, {0x1B5A, 0x1B60}, {0x1BFC, 0x1BFF}, {0x1C3B, 0x1C3F}, {0x1C7E, 0x1C7F}, {0x1CC0, 0x1CC7}, {0x1CD3, 0x1CD3}, {0x2010, 0x2027}, -{0x2030, 0x2043}, {0x2045, 0x2051}, {0x2053, 0x205E}, {0x207D, 0x207E}, {0x208D, 0x208E}, {0x2308, 0x230B}, {0x2329, 0x232A}, {0x2768, 0x2775}, {0x27C5, 0x27C6}, {0x27E6, 0x27EF}, {0x2983, 0x2998}, -{0x29D8, 0x29DB}, {0x29FC, 0x29FD}, {0x2CF9, 0x2CFC}, {0x2CFE, 0x2CFF}, {0x2D70, 0x2D70}, {0x2E00, 0x2E2E}, {0x2E30, 0x2E4F}, {0x2E52, 0x2E52}, {0x3001, 0x3003}, {0x3008, 0x3011}, {0x3014, 0x301F}, -{0x3030, 0x3030}, {0x303D, 0x303D}, {0x30A0, 0x30A0}, {0x30FB, 0x30FB}, {0xA4FE, 0xA4FF}, {0xA60D, 0xA60F}, {0xA673, 0xA673}, {0xA67E, 0xA67E}, {0xA6F2, 0xA6F7}, {0xA874, 0xA877}, {0xA8CE, 0xA8CF}, -{0xA8F8, 0xA8FA}, {0xA8FC, 0xA8FC}, {0xA92E, 0xA92F}, {0xA95F, 0xA95F}, {0xA9C1, 0xA9CD}, {0xA9DE, 0xA9DF}, {0xAA5C, 0xAA5F}, {0xAADE, 0xAADF}, {0xAAF0, 0xAAF1}, {0xABEB, 0xABEB}, {0xFD3E, 0xFD3F}, -{0xFE10, 0xFE19}, {0xFE30, 0xFE52}, {0xFE54, 0xFE61}, {0xFE63, 0xFE63}, {0xFE68, 0xFE68}, {0xFE6A, 0xFE6B}, {0xFF01, 0xFF03}, {0xFF05, 0xFF0A}, {0xFF0C, 0xFF0F}, {0xFF1A, 0xFF1B}, {0xFF1F, 0xFF20}, -{0xFF3B, 0xFF3D}, {0xFF3F, 0xFF3F}, {0xFF5B, 0xFF5B}, {0xFF5D, 0xFF5D}, {0xFF5F, 0xFF65}, {0x10100, 0x10102}, {0x1039F, 0x1039F}, {0x103D0, 0x103D0}, {0x1056F, 0x1056F}, {0x10857, 0x10857}, -{0x1091F, 0x1091F}, {0x1093F, 0x1093F}, {0x10A50, 0x10A58}, {0x10A7F, 0x10A7F}, {0x10AF0, 0x10AF6}, {0x10B39, 0x10B3F}, {0x10B99, 0x10B9C}, {0x10EAD, 0x10EAD}, {0x10F55, 0x10F59}, {0x11047, 0x1104D}, -{0x110BB, 0x110BC}, {0x110BE, 0x110C1}, {0x11140, 0x11143}, {0x11174, 0x11175}, {0x111C5, 0x111C8}, {0x111CD, 0x111CD}, {0x111DB, 0x111DB}, {0x111DD, 0x111DF}, {0x11238, 0x1123D}, {0x112A9, 0x112A9}, -{0x1144B, 0x1144F}, {0x1145A, 0x1145B}, {0x1145D, 0x1145D}, {0x114C6, 0x114C6}, {0x115C1, 0x115D7}, {0x11641, 0x11643}, {0x11660, 0x1166C}, {0x1173C, 0x1173E}, {0x1183B, 0x1183B}, {0x11944, 0x11946}, -{0x119E2, 0x119E2}, {0x11A3F, 0x11A46}, {0x11A9A, 0x11A9C}, {0x11A9E, 0x11AA2}, {0x11C41, 0x11C45}, {0x11C70, 0x11C71}, {0x11EF7, 0x11EF8}, {0x11FFF, 0x11FFF}, {0x12470, 0x12474}, {0x16A6E, 0x16A6F}, -{0x16AF5, 0x16AF5}, {0x16B37, 0x16B3B}, {0x16B44, 0x16B44}, {0x16E97, 0x16E9A}, {0x16FE2, 0x16FE2}, {0x1BC9F, 0x1BC9F}, {0x1DA87, 0x1DA8B}, {0x1E95E, 0x1E95F}, -}; - -static const std::vector> symbol_ranges = { -{0x24, 0x24}, {0x2B, 0x2B}, {0x3C, 0x3E}, {0x5E, 0x5E}, {0x60, 0x60}, {0x7C, 0x7C}, {0x7E, 0x7E}, {0xA2, 0xA6}, {0xA8, 0xA9}, {0xAC, 0xAC}, {0xAE, 0xB1}, {0xB4, 0xB4}, {0xB8, 0xB8}, {0xD7, 0xD7}, -{0xF7, 0xF7}, {0x2C2, 0x2C5}, {0x2D2, 0x2DF}, {0x2E5, 0x2EB}, {0x2ED, 0x2ED}, {0x2EF, 0x2FF}, {0x375, 0x375}, {0x384, 0x385}, {0x3F6, 0x3F6}, {0x482, 0x482}, {0x58D, 0x58F}, {0x606, 0x608}, -{0x60B, 0x60B}, {0x60E, 0x60F}, {0x6DE, 0x6DE}, {0x6E9, 0x6E9}, {0x6FD, 0x6FE}, {0x7F6, 0x7F6}, {0x7FE, 0x7FF}, {0x9F2, 0x9F3}, {0x9FA, 0x9FB}, {0xAF1, 0xAF1}, {0xB70, 0xB70}, {0xBF3, 0xBFA}, -{0xC7F, 0xC7F}, {0xD4F, 0xD4F}, {0xD79, 0xD79}, {0xE3F, 0xE3F}, {0xF01, 0xF03}, {0xF13, 0xF13}, {0xF15, 0xF17}, {0xF1A, 0xF1F}, {0xF34, 0xF34}, {0xF36, 0xF36}, {0xF38, 0xF38}, {0xFBE, 0xFC5}, -{0xFC7, 0xFCC}, {0xFCE, 0xFCF}, {0xFD5, 0xFD8}, {0x109E, 0x109F}, {0x1390, 0x1399}, {0x166D, 0x166D}, {0x17DB, 0x17DB}, {0x1940, 0x1940}, {0x19DE, 0x19FF}, {0x1B61, 0x1B6A}, {0x1B74, 0x1B7C}, -{0x1FBD, 0x1FBD}, {0x1FBF, 0x1FC1}, {0x1FCD, 0x1FCF}, {0x1FDD, 0x1FDF}, {0x1FED, 0x1FEF}, {0x1FFD, 0x1FFE}, {0x2044, 0x2044}, {0x2052, 0x2052}, {0x207A, 0x207C}, {0x208A, 0x208C}, {0x20A0, 0x20BF}, -{0x2100, 0x2101}, {0x2103, 0x2106}, {0x2108, 0x2109}, {0x2114, 0x2114}, {0x2116, 0x2118}, {0x211E, 0x2123}, {0x2125, 0x2125}, {0x2127, 0x2127}, {0x2129, 0x2129}, {0x212E, 0x212E}, {0x213A, 0x213B}, -{0x2140, 0x2144}, {0x214A, 0x214D}, {0x214F, 0x214F}, {0x218A, 0x218B}, {0x2190, 0x2307}, {0x230C, 0x2328}, {0x232B, 0x2426}, {0x2440, 0x244A}, {0x249C, 0x24E9}, {0x2500, 0x2767}, {0x2794, 0x27C4}, -{0x27C7, 0x27E5}, {0x27F0, 0x2982}, {0x2999, 0x29D7}, {0x29DC, 0x29FB}, {0x29FE, 0x2B73}, {0x2B76, 0x2B95}, {0x2B97, 0x2BFF}, {0x2CE5, 0x2CEA}, {0x2E50, 0x2E51}, {0x2E80, 0x2E99}, {0x2E9B, 0x2EF3}, -{0x2F00, 0x2FD5}, {0x2FF0, 0x2FFB}, {0x3004, 0x3004}, {0x3012, 0x3013}, {0x3020, 0x3020}, {0x3036, 0x3037}, {0x303E, 0x303F}, {0x309B, 0x309C}, {0x3190, 0x3191}, {0x3196, 0x319F}, {0x31C0, 0x31E3}, -{0x3200, 0x321E}, {0x322A, 0x3247}, {0x3250, 0x3250}, {0x3260, 0x327F}, {0x328A, 0x32B0}, {0x32C0, 0x33FF}, {0x4DC0, 0x4DFF}, {0xA490, 0xA4C6}, {0xA700, 0xA716}, {0xA720, 0xA721}, {0xA789, 0xA78A}, -{0xA828, 0xA82B}, {0xA836, 0xA839}, {0xAA77, 0xAA79}, {0xAB5B, 0xAB5B}, {0xAB6A, 0xAB6B}, {0xFB29, 0xFB29}, {0xFBB2, 0xFBC1}, {0xFDFC, 0xFDFD}, {0xFE62, 0xFE62}, {0xFE64, 0xFE66}, {0xFE69, 0xFE69}, -{0xFF04, 0xFF04}, {0xFF0B, 0xFF0B}, {0xFF1C, 0xFF1E}, {0xFF3E, 0xFF3E}, {0xFF40, 0xFF40}, {0xFF5C, 0xFF5C}, {0xFF5E, 0xFF5E}, {0xFFE0, 0xFFE6}, {0xFFE8, 0xFFEE}, {0xFFFC, 0xFFFD}, {0x10137, 0x1013F}, -{0x10179, 0x10189}, {0x1018C, 0x1018E}, {0x10190, 0x1019C}, {0x101A0, 0x101A0}, {0x101D0, 0x101FC}, {0x10877, 0x10878}, {0x10AC8, 0x10AC8}, {0x1173F, 0x1173F}, {0x11FD5, 0x11FF1}, {0x16B3C, 0x16B3F}, -{0x16B45, 0x16B45}, {0x1BC9C, 0x1BC9C}, {0x1D000, 0x1D0F5}, {0x1D100, 0x1D126}, {0x1D129, 0x1D164}, {0x1D16A, 0x1D16C}, {0x1D183, 0x1D184}, {0x1D18C, 0x1D1A9}, {0x1D1AE, 0x1D1E8}, {0x1D200, 0x1D241}, -{0x1D245, 0x1D245}, {0x1D300, 0x1D356}, {0x1D6C1, 0x1D6C1}, {0x1D6DB, 0x1D6DB}, {0x1D6FB, 0x1D6FB}, {0x1D715, 0x1D715}, {0x1D735, 0x1D735}, {0x1D74F, 0x1D74F}, {0x1D76F, 0x1D76F}, {0x1D789, 0x1D789}, -{0x1D7A9, 0x1D7A9}, {0x1D7C3, 0x1D7C3}, {0x1D800, 0x1D9FF}, {0x1DA37, 0x1DA3A}, {0x1DA6D, 0x1DA74}, {0x1DA76, 0x1DA83}, {0x1DA85, 0x1DA86}, {0x1E14F, 0x1E14F}, {0x1E2FF, 0x1E2FF}, {0x1ECAC, 0x1ECAC}, -{0x1ECB0, 0x1ECB0}, {0x1ED2E, 0x1ED2E}, {0x1EEF0, 0x1EEF1}, {0x1F000, 0x1F02B}, {0x1F030, 0x1F093}, {0x1F0A0, 0x1F0AE}, {0x1F0B1, 0x1F0BF}, {0x1F0C1, 0x1F0CF}, {0x1F0D1, 0x1F0F5}, {0x1F10D, 0x1F1AD}, -{0x1F1E6, 0x1F202}, {0x1F210, 0x1F23B}, {0x1F240, 0x1F248}, {0x1F250, 0x1F251}, {0x1F260, 0x1F265}, {0x1F300, 0x1F6D7}, {0x1F6E0, 0x1F6EC}, {0x1F6F0, 0x1F6FC}, {0x1F700, 0x1F773}, {0x1F780, 0x1F7D8}, -{0x1F7E0, 0x1F7EB}, {0x1F800, 0x1F80B}, {0x1F810, 0x1F847}, {0x1F850, 0x1F859}, {0x1F860, 0x1F887}, {0x1F890, 0x1F8AD}, {0x1F8B0, 0x1F8B1}, {0x1F900, 0x1F978}, {0x1F97A, 0x1F9CB}, {0x1F9CD, 0x1FA53}, -{0x1FA60, 0x1FA6D}, {0x1FA70, 0x1FA74}, {0x1FA78, 0x1FA7A}, {0x1FA80, 0x1FA86}, {0x1FA90, 0x1FAA8}, {0x1FAB0, 0x1FAB6}, {0x1FAC0, 0x1FAC2}, {0x1FAD0, 0x1FAD6}, {0x1FB00, 0x1FB92}, {0x1FB94, 0x1FBCA}, -}; - -static const std::vector> control_ranges = { -{0x0, 0x8}, {0xE, 0x1B}, {0x7F, 0x84}, {0x86, 0x9F}, {0xAD, 0xAD}, {0x378, 0x379}, {0x380, 0x383}, {0x38B, 0x38B}, {0x38D, 0x38D}, {0x3A2, 0x3A2}, {0x530, 0x530}, {0x557, 0x558}, {0x58B, 0x58C}, -{0x590, 0x590}, {0x5C8, 0x5CF}, {0x5EB, 0x5EE}, {0x5F5, 0x605}, {0x61C, 0x61D}, {0x6DD, 0x6DD}, {0x70E, 0x70F}, {0x74B, 0x74C}, {0x7B2, 0x7BF}, {0x7FB, 0x7FC}, {0x82E, 0x82F}, {0x83F, 0x83F}, -{0x85C, 0x85D}, {0x85F, 0x85F}, {0x86B, 0x89F}, {0x8B5, 0x8B5}, {0x8C8, 0x8D2}, {0x8E2, 0x8E2}, {0x984, 0x984}, {0x98D, 0x98E}, {0x991, 0x992}, {0x9A9, 0x9A9}, {0x9B1, 0x9B1}, {0x9B3, 0x9B5}, -{0x9BA, 0x9BB}, {0x9C5, 0x9C6}, {0x9C9, 0x9CA}, {0x9CF, 0x9D6}, {0x9D8, 0x9DB}, {0x9DE, 0x9DE}, {0x9E4, 0x9E5}, {0x9FF, 0xA00}, {0xA04, 0xA04}, {0xA0B, 0xA0E}, {0xA11, 0xA12}, {0xA29, 0xA29}, -{0xA31, 0xA31}, {0xA34, 0xA34}, {0xA37, 0xA37}, {0xA3A, 0xA3B}, {0xA3D, 0xA3D}, {0xA43, 0xA46}, {0xA49, 0xA4A}, {0xA4E, 0xA50}, {0xA52, 0xA58}, {0xA5D, 0xA5D}, {0xA5F, 0xA65}, {0xA77, 0xA80}, -{0xA84, 0xA84}, {0xA8E, 0xA8E}, {0xA92, 0xA92}, {0xAA9, 0xAA9}, {0xAB1, 0xAB1}, {0xAB4, 0xAB4}, {0xABA, 0xABB}, {0xAC6, 0xAC6}, {0xACA, 0xACA}, {0xACE, 0xACF}, {0xAD1, 0xADF}, {0xAE4, 0xAE5}, -{0xAF2, 0xAF8}, {0xB00, 0xB00}, {0xB04, 0xB04}, {0xB0D, 0xB0E}, {0xB11, 0xB12}, {0xB29, 0xB29}, {0xB31, 0xB31}, {0xB34, 0xB34}, {0xB3A, 0xB3B}, {0xB45, 0xB46}, {0xB49, 0xB4A}, {0xB4E, 0xB54}, -{0xB58, 0xB5B}, {0xB5E, 0xB5E}, {0xB64, 0xB65}, {0xB78, 0xB81}, {0xB84, 0xB84}, {0xB8B, 0xB8D}, {0xB91, 0xB91}, {0xB96, 0xB98}, {0xB9B, 0xB9B}, {0xB9D, 0xB9D}, {0xBA0, 0xBA2}, {0xBA5, 0xBA7}, -{0xBAB, 0xBAD}, {0xBBA, 0xBBD}, {0xBC3, 0xBC5}, {0xBC9, 0xBC9}, {0xBCE, 0xBCF}, {0xBD1, 0xBD6}, {0xBD8, 0xBE5}, {0xBFB, 0xBFF}, {0xC0D, 0xC0D}, {0xC11, 0xC11}, {0xC29, 0xC29}, {0xC3A, 0xC3C}, -{0xC45, 0xC45}, {0xC49, 0xC49}, {0xC4E, 0xC54}, {0xC57, 0xC57}, {0xC5B, 0xC5F}, {0xC64, 0xC65}, {0xC70, 0xC76}, {0xC8D, 0xC8D}, {0xC91, 0xC91}, {0xCA9, 0xCA9}, {0xCB4, 0xCB4}, {0xCBA, 0xCBB}, -{0xCC5, 0xCC5}, {0xCC9, 0xCC9}, {0xCCE, 0xCD4}, {0xCD7, 0xCDD}, {0xCDF, 0xCDF}, {0xCE4, 0xCE5}, {0xCF0, 0xCF0}, {0xCF3, 0xCFF}, {0xD0D, 0xD0D}, {0xD11, 0xD11}, {0xD45, 0xD45}, {0xD49, 0xD49}, -{0xD50, 0xD53}, {0xD64, 0xD65}, {0xD80, 0xD80}, {0xD84, 0xD84}, {0xD97, 0xD99}, {0xDB2, 0xDB2}, {0xDBC, 0xDBC}, {0xDBE, 0xDBF}, {0xDC7, 0xDC9}, {0xDCB, 0xDCE}, {0xDD5, 0xDD5}, {0xDD7, 0xDD7}, -{0xDE0, 0xDE5}, {0xDF0, 0xDF1}, {0xDF5, 0xE00}, {0xE3B, 0xE3E}, {0xE5C, 0xE80}, {0xE83, 0xE83}, {0xE85, 0xE85}, {0xE8B, 0xE8B}, {0xEA4, 0xEA4}, {0xEA6, 0xEA6}, {0xEBE, 0xEBF}, {0xEC5, 0xEC5}, -{0xEC7, 0xEC7}, {0xECE, 0xECF}, {0xEDA, 0xEDB}, {0xEE0, 0xEFF}, {0xF48, 0xF48}, {0xF6D, 0xF70}, {0xF98, 0xF98}, {0xFBD, 0xFBD}, {0xFCD, 0xFCD}, {0xFDB, 0xFFF}, {0x10C6, 0x10C6}, {0x10C8, 0x10CC}, -{0x10CE, 0x10CF}, {0x1249, 0x1249}, {0x124E, 0x124F}, {0x1257, 0x1257}, {0x1259, 0x1259}, {0x125E, 0x125F}, {0x1289, 0x1289}, {0x128E, 0x128F}, {0x12B1, 0x12B1}, {0x12B6, 0x12B7}, {0x12BF, 0x12BF}, -{0x12C1, 0x12C1}, {0x12C6, 0x12C7}, {0x12D7, 0x12D7}, {0x1311, 0x1311}, {0x1316, 0x1317}, {0x135B, 0x135C}, {0x137D, 0x137F}, {0x139A, 0x139F}, {0x13F6, 0x13F7}, {0x13FE, 0x13FF}, {0x169D, 0x169F}, -{0x16F9, 0x16FF}, {0x170D, 0x170D}, {0x1715, 0x171F}, {0x1737, 0x173F}, {0x1754, 0x175F}, {0x176D, 0x176D}, {0x1771, 0x1771}, {0x1774, 0x177F}, {0x17DE, 0x17DF}, {0x17EA, 0x17EF}, {0x17FA, 0x17FF}, -{0x180E, 0x180F}, {0x181A, 0x181F}, {0x1879, 0x187F}, {0x18AB, 0x18AF}, {0x18F6, 0x18FF}, {0x191F, 0x191F}, {0x192C, 0x192F}, {0x193C, 0x193F}, {0x1941, 0x1943}, {0x196E, 0x196F}, {0x1975, 0x197F}, -{0x19AC, 0x19AF}, {0x19CA, 0x19CF}, {0x19DB, 0x19DD}, {0x1A1C, 0x1A1D}, {0x1A5F, 0x1A5F}, {0x1A7D, 0x1A7E}, {0x1A8A, 0x1A8F}, {0x1A9A, 0x1A9F}, {0x1AAE, 0x1AAF}, {0x1AC1, 0x1AFF}, {0x1B4C, 0x1B4F}, -{0x1B7D, 0x1B7F}, {0x1BF4, 0x1BFB}, {0x1C38, 0x1C3A}, {0x1C4A, 0x1C4C}, {0x1C89, 0x1C8F}, {0x1CBB, 0x1CBC}, {0x1CC8, 0x1CCF}, {0x1CFB, 0x1CFF}, {0x1DFA, 0x1DFA}, {0x1F16, 0x1F17}, {0x1F1E, 0x1F1F}, -{0x1F46, 0x1F47}, {0x1F4E, 0x1F4F}, {0x1F58, 0x1F58}, {0x1F5A, 0x1F5A}, {0x1F5C, 0x1F5C}, {0x1F5E, 0x1F5E}, {0x1F7E, 0x1F7F}, {0x1FB5, 0x1FB5}, {0x1FC5, 0x1FC5}, {0x1FD4, 0x1FD5}, {0x1FDC, 0x1FDC}, -{0x1FF0, 0x1FF1}, {0x1FF5, 0x1FF5}, {0x1FFF, 0x1FFF}, {0x200B, 0x200F}, {0x202A, 0x202E}, {0x2060, 0x206F}, {0x2072, 0x2073}, {0x208F, 0x208F}, {0x209D, 0x209F}, {0x20C0, 0x20CF}, {0x20F1, 0x20FF}, -{0x218C, 0x218F}, {0x2427, 0x243F}, {0x244B, 0x245F}, {0x2B74, 0x2B75}, {0x2B96, 0x2B96}, {0x2C2F, 0x2C2F}, {0x2C5F, 0x2C5F}, {0x2CF4, 0x2CF8}, {0x2D26, 0x2D26}, {0x2D28, 0x2D2C}, {0x2D2E, 0x2D2F}, -{0x2D68, 0x2D6E}, {0x2D71, 0x2D7E}, {0x2D97, 0x2D9F}, {0x2DA7, 0x2DA7}, {0x2DAF, 0x2DAF}, {0x2DB7, 0x2DB7}, {0x2DBF, 0x2DBF}, {0x2DC7, 0x2DC7}, {0x2DCF, 0x2DCF}, {0x2DD7, 0x2DD7}, {0x2DDF, 0x2DDF}, -{0x2E53, 0x2E7F}, {0x2E9A, 0x2E9A}, {0x2EF4, 0x2EFF}, {0x2FD6, 0x2FEF}, {0x2FFC, 0x2FFF}, {0x3040, 0x3040}, {0x3097, 0x3098}, {0x3100, 0x3104}, {0x3130, 0x3130}, {0x318F, 0x318F}, {0x31E4, 0x31EF}, -{0x321F, 0x321F}, {0x9FFD, 0x9FFF}, {0xA48D, 0xA48F}, {0xA4C7, 0xA4CF}, {0xA62C, 0xA63F}, {0xA6F8, 0xA6FF}, {0xA7C0, 0xA7C1}, {0xA7CB, 0xA7F4}, {0xA82D, 0xA82F}, {0xA83A, 0xA83F}, {0xA878, 0xA87F}, -{0xA8C6, 0xA8CD}, {0xA8DA, 0xA8DF}, {0xA954, 0xA95E}, {0xA97D, 0xA97F}, {0xA9CE, 0xA9CE}, {0xA9DA, 0xA9DD}, {0xA9FF, 0xA9FF}, {0xAA37, 0xAA3F}, {0xAA4E, 0xAA4F}, {0xAA5A, 0xAA5B}, {0xAAC3, 0xAADA}, -{0xAAF7, 0xAB00}, {0xAB07, 0xAB08}, {0xAB0F, 0xAB10}, {0xAB17, 0xAB1F}, {0xAB27, 0xAB27}, {0xAB2F, 0xAB2F}, {0xAB6C, 0xAB6F}, {0xABEE, 0xABEF}, {0xABFA, 0xABFF}, {0xD7A4, 0xD7AF}, {0xD7C7, 0xD7CA}, -{0xD7FC, 0xF8FF}, {0xFA6E, 0xFA6F}, {0xFADA, 0xFAFF}, {0xFB07, 0xFB12}, {0xFB18, 0xFB1C}, {0xFB37, 0xFB37}, {0xFB3D, 0xFB3D}, {0xFB3F, 0xFB3F}, {0xFB42, 0xFB42}, {0xFB45, 0xFB45}, {0xFBC2, 0xFBD2}, -{0xFD40, 0xFD4F}, {0xFD90, 0xFD91}, {0xFDC8, 0xFDEF}, {0xFDFE, 0xFDFF}, {0xFE1A, 0xFE1F}, {0xFE53, 0xFE53}, {0xFE67, 0xFE67}, {0xFE6C, 0xFE6F}, {0xFE75, 0xFE75}, {0xFEFD, 0xFF00}, {0xFFBF, 0xFFC1}, -{0xFFC8, 0xFFC9}, {0xFFD0, 0xFFD1}, {0xFFD8, 0xFFD9}, {0xFFDD, 0xFFDF}, {0xFFE7, 0xFFE7}, {0xFFEF, 0xFFFB}, {0xFFFE, 0xFFFF}, {0x1000C, 0x1000C}, {0x10027, 0x10027}, {0x1003B, 0x1003B}, -{0x1003E, 0x1003E}, {0x1004E, 0x1004F}, {0x1005E, 0x1007F}, {0x100FB, 0x100FF}, {0x10103, 0x10106}, {0x10134, 0x10136}, {0x1018F, 0x1018F}, {0x1019D, 0x1019F}, {0x101A1, 0x101CF}, {0x101FE, 0x1027F}, -{0x1029D, 0x1029F}, {0x102D1, 0x102DF}, {0x102FC, 0x102FF}, {0x10324, 0x1032C}, {0x1034B, 0x1034F}, {0x1037B, 0x1037F}, {0x1039E, 0x1039E}, {0x103C4, 0x103C7}, {0x103D6, 0x103FF}, {0x1049E, 0x1049F}, -{0x104AA, 0x104AF}, {0x104D4, 0x104D7}, {0x104FC, 0x104FF}, {0x10528, 0x1052F}, {0x10564, 0x1056E}, {0x10570, 0x105FF}, {0x10737, 0x1073F}, {0x10756, 0x1075F}, {0x10768, 0x107FF}, {0x10806, 0x10807}, -{0x10809, 0x10809}, {0x10836, 0x10836}, {0x10839, 0x1083B}, {0x1083D, 0x1083E}, {0x10856, 0x10856}, {0x1089F, 0x108A6}, {0x108B0, 0x108DF}, {0x108F3, 0x108F3}, {0x108F6, 0x108FA}, {0x1091C, 0x1091E}, -{0x1093A, 0x1093E}, {0x10940, 0x1097F}, {0x109B8, 0x109BB}, {0x109D0, 0x109D1}, {0x10A04, 0x10A04}, {0x10A07, 0x10A0B}, {0x10A14, 0x10A14}, {0x10A18, 0x10A18}, {0x10A36, 0x10A37}, {0x10A3B, 0x10A3E}, -{0x10A49, 0x10A4F}, {0x10A59, 0x10A5F}, {0x10AA0, 0x10ABF}, {0x10AE7, 0x10AEA}, {0x10AF7, 0x10AFF}, {0x10B36, 0x10B38}, {0x10B56, 0x10B57}, {0x10B73, 0x10B77}, {0x10B92, 0x10B98}, {0x10B9D, 0x10BA8}, -{0x10BB0, 0x10BFF}, {0x10C49, 0x10C7F}, {0x10CB3, 0x10CBF}, {0x10CF3, 0x10CF9}, {0x10D28, 0x10D2F}, {0x10D3A, 0x10E5F}, {0x10E7F, 0x10E7F}, {0x10EAA, 0x10EAA}, {0x10EAE, 0x10EAF}, {0x10EB2, 0x10EFF}, -{0x10F28, 0x10F2F}, {0x10F5A, 0x10FAF}, {0x10FCC, 0x10FDF}, {0x10FF7, 0x10FFF}, {0x1104E, 0x11051}, {0x11070, 0x1107E}, {0x110BD, 0x110BD}, {0x110C2, 0x110CF}, {0x110E9, 0x110EF}, {0x110FA, 0x110FF}, -{0x11135, 0x11135}, {0x11148, 0x1114F}, {0x11177, 0x1117F}, {0x111E0, 0x111E0}, {0x111F5, 0x111FF}, {0x11212, 0x11212}, {0x1123F, 0x1127F}, {0x11287, 0x11287}, {0x11289, 0x11289}, {0x1128E, 0x1128E}, -{0x1129E, 0x1129E}, {0x112AA, 0x112AF}, {0x112EB, 0x112EF}, {0x112FA, 0x112FF}, {0x11304, 0x11304}, {0x1130D, 0x1130E}, {0x11311, 0x11312}, {0x11329, 0x11329}, {0x11331, 0x11331}, {0x11334, 0x11334}, -{0x1133A, 0x1133A}, {0x11345, 0x11346}, {0x11349, 0x1134A}, {0x1134E, 0x1134F}, {0x11351, 0x11356}, {0x11358, 0x1135C}, {0x11364, 0x11365}, {0x1136D, 0x1136F}, {0x11375, 0x113FF}, {0x1145C, 0x1145C}, -{0x11462, 0x1147F}, {0x114C8, 0x114CF}, {0x114DA, 0x1157F}, {0x115B6, 0x115B7}, {0x115DE, 0x115FF}, {0x11645, 0x1164F}, {0x1165A, 0x1165F}, {0x1166D, 0x1167F}, {0x116B9, 0x116BF}, {0x116CA, 0x116FF}, -{0x1171B, 0x1171C}, {0x1172C, 0x1172F}, {0x11740, 0x117FF}, {0x1183C, 0x1189F}, {0x118F3, 0x118FE}, {0x11907, 0x11908}, {0x1190A, 0x1190B}, {0x11914, 0x11914}, {0x11917, 0x11917}, {0x11936, 0x11936}, -{0x11939, 0x1193A}, {0x11947, 0x1194F}, {0x1195A, 0x1199F}, {0x119A8, 0x119A9}, {0x119D8, 0x119D9}, {0x119E5, 0x119FF}, {0x11A48, 0x11A4F}, {0x11AA3, 0x11ABF}, {0x11AF9, 0x11BFF}, {0x11C09, 0x11C09}, -{0x11C37, 0x11C37}, {0x11C46, 0x11C4F}, {0x11C6D, 0x11C6F}, {0x11C90, 0x11C91}, {0x11CA8, 0x11CA8}, {0x11CB7, 0x11CFF}, {0x11D07, 0x11D07}, {0x11D0A, 0x11D0A}, {0x11D37, 0x11D39}, {0x11D3B, 0x11D3B}, -{0x11D3E, 0x11D3E}, {0x11D48, 0x11D4F}, {0x11D5A, 0x11D5F}, {0x11D66, 0x11D66}, {0x11D69, 0x11D69}, {0x11D8F, 0x11D8F}, {0x11D92, 0x11D92}, {0x11D99, 0x11D9F}, {0x11DAA, 0x11EDF}, {0x11EF9, 0x11FAF}, -{0x11FB1, 0x11FBF}, {0x11FF2, 0x11FFE}, {0x1239A, 0x123FF}, {0x1246F, 0x1246F}, {0x12475, 0x1247F}, {0x12544, 0x12FFF}, {0x1342F, 0x143FF}, {0x14647, 0x167FF}, {0x16A39, 0x16A3F}, {0x16A5F, 0x16A5F}, -{0x16A6A, 0x16A6D}, {0x16A70, 0x16ACF}, {0x16AEE, 0x16AEF}, {0x16AF6, 0x16AFF}, {0x16B46, 0x16B4F}, {0x16B5A, 0x16B5A}, {0x16B62, 0x16B62}, {0x16B78, 0x16B7C}, {0x16B90, 0x16E3F}, {0x16E9B, 0x16EFF}, -{0x16F4B, 0x16F4E}, {0x16F88, 0x16F8E}, {0x16FA0, 0x16FDF}, {0x16FE5, 0x16FEF}, {0x16FF2, 0x16FFF}, {0x187F8, 0x187FF}, {0x18CD6, 0x18CFF}, {0x18D09, 0x1AFFF}, {0x1B11F, 0x1B14F}, {0x1B153, 0x1B163}, -{0x1B168, 0x1B16F}, {0x1B2FC, 0x1BBFF}, {0x1BC6B, 0x1BC6F}, {0x1BC7D, 0x1BC7F}, {0x1BC89, 0x1BC8F}, {0x1BC9A, 0x1BC9B}, {0x1BCA0, 0x1CFFF}, {0x1D0F6, 0x1D0FF}, {0x1D127, 0x1D128}, {0x1D173, 0x1D17A}, -{0x1D1E9, 0x1D1FF}, {0x1D246, 0x1D2DF}, {0x1D2F4, 0x1D2FF}, {0x1D357, 0x1D35F}, {0x1D379, 0x1D3FF}, {0x1D455, 0x1D455}, {0x1D49D, 0x1D49D}, {0x1D4A0, 0x1D4A1}, {0x1D4A3, 0x1D4A4}, {0x1D4A7, 0x1D4A8}, -{0x1D4AD, 0x1D4AD}, {0x1D4BA, 0x1D4BA}, {0x1D4BC, 0x1D4BC}, {0x1D4C4, 0x1D4C4}, {0x1D506, 0x1D506}, {0x1D50B, 0x1D50C}, {0x1D515, 0x1D515}, {0x1D51D, 0x1D51D}, {0x1D53A, 0x1D53A}, {0x1D53F, 0x1D53F}, -{0x1D545, 0x1D545}, {0x1D547, 0x1D549}, {0x1D551, 0x1D551}, {0x1D6A6, 0x1D6A7}, {0x1D7CC, 0x1D7CD}, {0x1DA8C, 0x1DA9A}, {0x1DAA0, 0x1DAA0}, {0x1DAB0, 0x1DFFF}, {0x1E007, 0x1E007}, {0x1E019, 0x1E01A}, -{0x1E022, 0x1E022}, {0x1E025, 0x1E025}, {0x1E02B, 0x1E0FF}, {0x1E12D, 0x1E12F}, {0x1E13E, 0x1E13F}, {0x1E14A, 0x1E14D}, {0x1E150, 0x1E2BF}, {0x1E2FA, 0x1E2FE}, {0x1E300, 0x1E7FF}, {0x1E8C5, 0x1E8C6}, -{0x1E8D7, 0x1E8FF}, {0x1E94C, 0x1E94F}, {0x1E95A, 0x1E95D}, {0x1E960, 0x1EC70}, {0x1ECB5, 0x1ED00}, {0x1ED3E, 0x1EDFF}, {0x1EE04, 0x1EE04}, {0x1EE20, 0x1EE20}, {0x1EE23, 0x1EE23}, {0x1EE25, 0x1EE26}, -{0x1EE28, 0x1EE28}, {0x1EE33, 0x1EE33}, {0x1EE38, 0x1EE38}, {0x1EE3A, 0x1EE3A}, {0x1EE3C, 0x1EE41}, {0x1EE43, 0x1EE46}, {0x1EE48, 0x1EE48}, {0x1EE4A, 0x1EE4A}, {0x1EE4C, 0x1EE4C}, {0x1EE50, 0x1EE50}, -{0x1EE53, 0x1EE53}, {0x1EE55, 0x1EE56}, {0x1EE58, 0x1EE58}, {0x1EE5A, 0x1EE5A}, {0x1EE5C, 0x1EE5C}, {0x1EE5E, 0x1EE5E}, {0x1EE60, 0x1EE60}, {0x1EE63, 0x1EE63}, {0x1EE65, 0x1EE66}, {0x1EE6B, 0x1EE6B}, -{0x1EE73, 0x1EE73}, {0x1EE78, 0x1EE78}, {0x1EE7D, 0x1EE7D}, {0x1EE7F, 0x1EE7F}, {0x1EE8A, 0x1EE8A}, {0x1EE9C, 0x1EEA0}, {0x1EEA4, 0x1EEA4}, {0x1EEAA, 0x1EEAA}, {0x1EEBC, 0x1EEEF}, {0x1EEF2, 0x1EFFF}, -{0x1F02C, 0x1F02F}, {0x1F094, 0x1F09F}, {0x1F0AF, 0x1F0B0}, {0x1F0C0, 0x1F0C0}, {0x1F0D0, 0x1F0D0}, {0x1F0F6, 0x1F0FF}, {0x1F1AE, 0x1F1E5}, {0x1F203, 0x1F20F}, {0x1F23C, 0x1F23F}, {0x1F249, 0x1F24F}, -{0x1F252, 0x1F25F}, {0x1F266, 0x1F2FF}, {0x1F6D8, 0x1F6DF}, {0x1F6ED, 0x1F6EF}, {0x1F6FD, 0x1F6FF}, {0x1F774, 0x1F77F}, {0x1F7D9, 0x1F7DF}, {0x1F7EC, 0x1F7FF}, {0x1F80C, 0x1F80F}, {0x1F848, 0x1F84F}, -{0x1F85A, 0x1F85F}, {0x1F888, 0x1F88F}, {0x1F8AE, 0x1F8AF}, {0x1F8B2, 0x1F8FF}, {0x1F979, 0x1F979}, {0x1F9CC, 0x1F9CC}, {0x1FA54, 0x1FA5F}, {0x1FA6E, 0x1FA6F}, {0x1FA75, 0x1FA77}, {0x1FA7B, 0x1FA7F}, -{0x1FA87, 0x1FA8F}, {0x1FAA9, 0x1FAAF}, {0x1FAB7, 0x1FABF}, {0x1FAC3, 0x1FACF}, {0x1FAD7, 0x1FAFF}, {0x1FB93, 0x1FB93}, {0x1FBCB, 0x1FBEF}, {0x1FBFA, 0x1FFFF}, {0x2A6DE, 0x2A6FF}, {0x2B735, 0x2B73F}, -{0x2B81E, 0x2B81F}, {0x2CEA2, 0x2CEAF}, {0x2EBE1, 0x2F7FF}, {0x2FA1E, 0x2FFFF}, {0x3134B, 0xE00FF}, {0xE01F0, 0x10FFFF}, -}; - -static const std::multimap nfd_map = { -{0xC0, 0x41}, {0xC0, 0x300}, {0xC1, 0x41}, {0xC1, 0x301}, {0xC2, 0x41}, {0xC2, 0x302}, {0xC3, 0x41}, {0xC3, 0x303}, {0xC4, 0x41}, {0xC4, 0x308}, {0xC5, 0x41}, {0xC5, 0x30A}, {0xC7, 0x43}, -{0xC7, 0x327}, {0xC8, 0x45}, {0xC8, 0x300}, {0xC9, 0x45}, {0xC9, 0x301}, {0xCA, 0x45}, {0xCA, 0x302}, {0xCB, 0x45}, {0xCB, 0x308}, {0xCC, 0x49}, {0xCC, 0x300}, {0xCD, 0x49}, {0xCD, 0x301}, -{0xCE, 0x49}, {0xCE, 0x302}, {0xCF, 0x49}, {0xCF, 0x308}, {0xD1, 0x4E}, {0xD1, 0x303}, {0xD2, 0x4F}, {0xD2, 0x300}, {0xD3, 0x4F}, {0xD3, 0x301}, {0xD4, 0x4F}, {0xD4, 0x302}, {0xD5, 0x4F}, -{0xD5, 0x303}, {0xD6, 0x4F}, {0xD6, 0x308}, {0xD9, 0x55}, {0xD9, 0x300}, {0xDA, 0x55}, {0xDA, 0x301}, {0xDB, 0x55}, {0xDB, 0x302}, {0xDC, 0x55}, {0xDC, 0x308}, {0xDD, 0x59}, {0xDD, 0x301}, -{0xE0, 0x61}, {0xE0, 0x300}, {0xE1, 0x61}, {0xE1, 0x301}, {0xE2, 0x61}, {0xE2, 0x302}, {0xE3, 0x61}, {0xE3, 0x303}, {0xE4, 0x61}, {0xE4, 0x308}, {0xE5, 0x61}, {0xE5, 0x30A}, {0xE7, 0x63}, -{0xE7, 0x327}, {0xE8, 0x65}, {0xE8, 0x300}, {0xE9, 0x65}, {0xE9, 0x301}, {0xEA, 0x65}, {0xEA, 0x302}, {0xEB, 0x65}, {0xEB, 0x308}, {0xEC, 0x69}, {0xEC, 0x300}, {0xED, 0x69}, {0xED, 0x301}, -{0xEE, 0x69}, {0xEE, 0x302}, {0xEF, 0x69}, {0xEF, 0x308}, {0xF1, 0x6E}, {0xF1, 0x303}, {0xF2, 0x6F}, {0xF2, 0x300}, {0xF3, 0x6F}, {0xF3, 0x301}, {0xF4, 0x6F}, {0xF4, 0x302}, {0xF5, 0x6F}, -{0xF5, 0x303}, {0xF6, 0x6F}, {0xF6, 0x308}, {0xF9, 0x75}, {0xF9, 0x300}, {0xFA, 0x75}, {0xFA, 0x301}, {0xFB, 0x75}, {0xFB, 0x302}, {0xFC, 0x75}, {0xFC, 0x308}, {0xFD, 0x79}, {0xFD, 0x301}, -{0xFF, 0x79}, {0xFF, 0x308}, {0x100, 0x41}, {0x100, 0x304}, {0x101, 0x61}, {0x101, 0x304}, {0x102, 0x41}, {0x102, 0x306}, {0x103, 0x61}, {0x103, 0x306}, {0x104, 0x41}, {0x104, 0x328}, {0x105, 0x61}, -{0x105, 0x328}, {0x106, 0x43}, {0x106, 0x301}, {0x107, 0x63}, {0x107, 0x301}, {0x108, 0x43}, {0x108, 0x302}, {0x109, 0x63}, {0x109, 0x302}, {0x10A, 0x43}, {0x10A, 0x307}, {0x10B, 0x63}, -{0x10B, 0x307}, {0x10C, 0x43}, {0x10C, 0x30C}, {0x10D, 0x63}, {0x10D, 0x30C}, {0x10E, 0x44}, {0x10E, 0x30C}, {0x10F, 0x64}, {0x10F, 0x30C}, {0x112, 0x45}, {0x112, 0x304}, {0x113, 0x65}, -{0x113, 0x304}, {0x114, 0x45}, {0x114, 0x306}, {0x115, 0x65}, {0x115, 0x306}, {0x116, 0x45}, {0x116, 0x307}, {0x117, 0x65}, {0x117, 0x307}, {0x118, 0x45}, {0x118, 0x328}, {0x119, 0x65}, -{0x119, 0x328}, {0x11A, 0x45}, {0x11A, 0x30C}, {0x11B, 0x65}, {0x11B, 0x30C}, {0x11C, 0x47}, {0x11C, 0x302}, {0x11D, 0x67}, {0x11D, 0x302}, {0x11E, 0x47}, {0x11E, 0x306}, {0x11F, 0x67}, -{0x11F, 0x306}, {0x120, 0x47}, {0x120, 0x307}, {0x121, 0x67}, {0x121, 0x307}, {0x122, 0x47}, {0x122, 0x327}, {0x123, 0x67}, {0x123, 0x327}, {0x124, 0x48}, {0x124, 0x302}, {0x125, 0x68}, -{0x125, 0x302}, {0x128, 0x49}, {0x128, 0x303}, {0x129, 0x69}, {0x129, 0x303}, {0x12A, 0x49}, {0x12A, 0x304}, {0x12B, 0x69}, {0x12B, 0x304}, {0x12C, 0x49}, {0x12C, 0x306}, {0x12D, 0x69}, -{0x12D, 0x306}, {0x12E, 0x49}, {0x12E, 0x328}, {0x12F, 0x69}, {0x12F, 0x328}, {0x130, 0x49}, {0x130, 0x307}, {0x134, 0x4A}, {0x134, 0x302}, {0x135, 0x6A}, {0x135, 0x302}, {0x136, 0x4B}, -{0x136, 0x327}, {0x137, 0x6B}, {0x137, 0x327}, {0x139, 0x4C}, {0x139, 0x301}, {0x13A, 0x6C}, {0x13A, 0x301}, {0x13B, 0x4C}, {0x13B, 0x327}, {0x13C, 0x6C}, {0x13C, 0x327}, {0x13D, 0x4C}, -{0x13D, 0x30C}, {0x13E, 0x6C}, {0x13E, 0x30C}, {0x143, 0x4E}, {0x143, 0x301}, {0x144, 0x6E}, {0x144, 0x301}, {0x145, 0x4E}, {0x145, 0x327}, {0x146, 0x6E}, {0x146, 0x327}, {0x147, 0x4E}, -{0x147, 0x30C}, {0x148, 0x6E}, {0x148, 0x30C}, {0x14C, 0x4F}, {0x14C, 0x304}, {0x14D, 0x6F}, {0x14D, 0x304}, {0x14E, 0x4F}, {0x14E, 0x306}, {0x14F, 0x6F}, {0x14F, 0x306}, {0x150, 0x4F}, -{0x150, 0x30B}, {0x151, 0x6F}, {0x151, 0x30B}, {0x154, 0x52}, {0x154, 0x301}, {0x155, 0x72}, {0x155, 0x301}, {0x156, 0x52}, {0x156, 0x327}, {0x157, 0x72}, {0x157, 0x327}, {0x158, 0x52}, -{0x158, 0x30C}, {0x159, 0x72}, {0x159, 0x30C}, {0x15A, 0x53}, {0x15A, 0x301}, {0x15B, 0x73}, {0x15B, 0x301}, {0x15C, 0x53}, {0x15C, 0x302}, {0x15D, 0x73}, {0x15D, 0x302}, {0x15E, 0x53}, -{0x15E, 0x327}, {0x15F, 0x73}, {0x15F, 0x327}, {0x160, 0x53}, {0x160, 0x30C}, {0x161, 0x73}, {0x161, 0x30C}, {0x162, 0x54}, {0x162, 0x327}, {0x163, 0x74}, {0x163, 0x327}, {0x164, 0x54}, -{0x164, 0x30C}, {0x165, 0x74}, {0x165, 0x30C}, {0x168, 0x55}, {0x168, 0x303}, {0x169, 0x75}, {0x169, 0x303}, {0x16A, 0x55}, {0x16A, 0x304}, {0x16B, 0x75}, {0x16B, 0x304}, {0x16C, 0x55}, -{0x16C, 0x306}, {0x16D, 0x75}, {0x16D, 0x306}, {0x16E, 0x55}, {0x16E, 0x30A}, {0x16F, 0x75}, {0x16F, 0x30A}, {0x170, 0x55}, {0x170, 0x30B}, {0x171, 0x75}, {0x171, 0x30B}, {0x172, 0x55}, -{0x172, 0x328}, {0x173, 0x75}, {0x173, 0x328}, {0x174, 0x57}, {0x174, 0x302}, {0x175, 0x77}, {0x175, 0x302}, {0x176, 0x59}, {0x176, 0x302}, {0x177, 0x79}, {0x177, 0x302}, {0x178, 0x59}, -{0x178, 0x308}, {0x179, 0x5A}, {0x179, 0x301}, {0x17A, 0x7A}, {0x17A, 0x301}, {0x17B, 0x5A}, {0x17B, 0x307}, {0x17C, 0x7A}, {0x17C, 0x307}, {0x17D, 0x5A}, {0x17D, 0x30C}, {0x17E, 0x7A}, -{0x17E, 0x30C}, {0x1A0, 0x4F}, {0x1A0, 0x31B}, {0x1A1, 0x6F}, {0x1A1, 0x31B}, {0x1AF, 0x55}, {0x1AF, 0x31B}, {0x1B0, 0x75}, {0x1B0, 0x31B}, {0x1CD, 0x41}, {0x1CD, 0x30C}, {0x1CE, 0x61}, -{0x1CE, 0x30C}, {0x1CF, 0x49}, {0x1CF, 0x30C}, {0x1D0, 0x69}, {0x1D0, 0x30C}, {0x1D1, 0x4F}, {0x1D1, 0x30C}, {0x1D2, 0x6F}, {0x1D2, 0x30C}, {0x1D3, 0x55}, {0x1D3, 0x30C}, {0x1D4, 0x75}, -{0x1D4, 0x30C}, {0x1D5, 0x55}, {0x1D5, 0x308}, {0x1D5, 0x304}, {0x1D6, 0x75}, {0x1D6, 0x308}, {0x1D6, 0x304}, {0x1D7, 0x55}, {0x1D7, 0x308}, {0x1D7, 0x301}, {0x1D8, 0x75}, {0x1D8, 0x308}, -{0x1D8, 0x301}, {0x1D9, 0x55}, {0x1D9, 0x308}, {0x1D9, 0x30C}, {0x1DA, 0x75}, {0x1DA, 0x308}, {0x1DA, 0x30C}, {0x1DB, 0x55}, {0x1DB, 0x308}, {0x1DB, 0x300}, {0x1DC, 0x75}, {0x1DC, 0x308}, -{0x1DC, 0x300}, {0x1DE, 0x41}, {0x1DE, 0x308}, {0x1DE, 0x304}, {0x1DF, 0x61}, {0x1DF, 0x308}, {0x1DF, 0x304}, {0x1E0, 0x41}, {0x1E0, 0x307}, {0x1E0, 0x304}, {0x1E1, 0x61}, {0x1E1, 0x307}, -{0x1E1, 0x304}, {0x1E2, 0xC6}, {0x1E2, 0x304}, {0x1E3, 0xE6}, {0x1E3, 0x304}, {0x1E6, 0x47}, {0x1E6, 0x30C}, {0x1E7, 0x67}, {0x1E7, 0x30C}, {0x1E8, 0x4B}, {0x1E8, 0x30C}, {0x1E9, 0x6B}, -{0x1E9, 0x30C}, {0x1EA, 0x4F}, {0x1EA, 0x328}, {0x1EB, 0x6F}, {0x1EB, 0x328}, {0x1EC, 0x4F}, {0x1EC, 0x328}, {0x1EC, 0x304}, {0x1ED, 0x6F}, {0x1ED, 0x328}, {0x1ED, 0x304}, {0x1EE, 0x1B7}, -{0x1EE, 0x30C}, {0x1EF, 0x292}, {0x1EF, 0x30C}, {0x1F0, 0x6A}, {0x1F0, 0x30C}, {0x1F4, 0x47}, {0x1F4, 0x301}, {0x1F5, 0x67}, {0x1F5, 0x301}, {0x1F8, 0x4E}, {0x1F8, 0x300}, {0x1F9, 0x6E}, -{0x1F9, 0x300}, {0x1FA, 0x41}, {0x1FA, 0x30A}, {0x1FA, 0x301}, {0x1FB, 0x61}, {0x1FB, 0x30A}, {0x1FB, 0x301}, {0x1FC, 0xC6}, {0x1FC, 0x301}, {0x1FD, 0xE6}, {0x1FD, 0x301}, {0x1FE, 0xD8}, -{0x1FE, 0x301}, {0x1FF, 0xF8}, {0x1FF, 0x301}, {0x200, 0x41}, {0x200, 0x30F}, {0x201, 0x61}, {0x201, 0x30F}, {0x202, 0x41}, {0x202, 0x311}, {0x203, 0x61}, {0x203, 0x311}, {0x204, 0x45}, -{0x204, 0x30F}, {0x205, 0x65}, {0x205, 0x30F}, {0x206, 0x45}, {0x206, 0x311}, {0x207, 0x65}, {0x207, 0x311}, {0x208, 0x49}, {0x208, 0x30F}, {0x209, 0x69}, {0x209, 0x30F}, {0x20A, 0x49}, -{0x20A, 0x311}, {0x20B, 0x69}, {0x20B, 0x311}, {0x20C, 0x4F}, {0x20C, 0x30F}, {0x20D, 0x6F}, {0x20D, 0x30F}, {0x20E, 0x4F}, {0x20E, 0x311}, {0x20F, 0x6F}, {0x20F, 0x311}, {0x210, 0x52}, -{0x210, 0x30F}, {0x211, 0x72}, {0x211, 0x30F}, {0x212, 0x52}, {0x212, 0x311}, {0x213, 0x72}, {0x213, 0x311}, {0x214, 0x55}, {0x214, 0x30F}, {0x215, 0x75}, {0x215, 0x30F}, {0x216, 0x55}, -{0x216, 0x311}, {0x217, 0x75}, {0x217, 0x311}, {0x218, 0x53}, {0x218, 0x326}, {0x219, 0x73}, {0x219, 0x326}, {0x21A, 0x54}, {0x21A, 0x326}, {0x21B, 0x74}, {0x21B, 0x326}, {0x21E, 0x48}, -{0x21E, 0x30C}, {0x21F, 0x68}, {0x21F, 0x30C}, {0x226, 0x41}, {0x226, 0x307}, {0x227, 0x61}, {0x227, 0x307}, {0x228, 0x45}, {0x228, 0x327}, {0x229, 0x65}, {0x229, 0x327}, {0x22A, 0x4F}, -{0x22A, 0x308}, {0x22A, 0x304}, {0x22B, 0x6F}, {0x22B, 0x308}, {0x22B, 0x304}, {0x22C, 0x4F}, {0x22C, 0x303}, {0x22C, 0x304}, {0x22D, 0x6F}, {0x22D, 0x303}, {0x22D, 0x304}, {0x22E, 0x4F}, -{0x22E, 0x307}, {0x22F, 0x6F}, {0x22F, 0x307}, {0x230, 0x4F}, {0x230, 0x307}, {0x230, 0x304}, {0x231, 0x6F}, {0x231, 0x307}, {0x231, 0x304}, {0x232, 0x59}, {0x232, 0x304}, {0x233, 0x79}, -{0x233, 0x304}, {0x340, 0x300}, {0x341, 0x301}, {0x343, 0x313}, {0x344, 0x308}, {0x344, 0x301}, {0x374, 0x2B9}, {0x37E, 0x3B}, {0x385, 0xA8}, {0x385, 0x301}, {0x386, 0x391}, {0x386, 0x301}, -{0x387, 0xB7}, {0x388, 0x395}, {0x388, 0x301}, {0x389, 0x397}, {0x389, 0x301}, {0x38A, 0x399}, {0x38A, 0x301}, {0x38C, 0x39F}, {0x38C, 0x301}, {0x38E, 0x3A5}, {0x38E, 0x301}, {0x38F, 0x3A9}, -{0x38F, 0x301}, {0x390, 0x3B9}, {0x390, 0x308}, {0x390, 0x301}, {0x3AA, 0x399}, {0x3AA, 0x308}, {0x3AB, 0x3A5}, {0x3AB, 0x308}, {0x3AC, 0x3B1}, {0x3AC, 0x301}, {0x3AD, 0x3B5}, {0x3AD, 0x301}, -{0x3AE, 0x3B7}, {0x3AE, 0x301}, {0x3AF, 0x3B9}, {0x3AF, 0x301}, {0x3B0, 0x3C5}, {0x3B0, 0x308}, {0x3B0, 0x301}, {0x3CA, 0x3B9}, {0x3CA, 0x308}, {0x3CB, 0x3C5}, {0x3CB, 0x308}, {0x3CC, 0x3BF}, -{0x3CC, 0x301}, {0x3CD, 0x3C5}, {0x3CD, 0x301}, {0x3CE, 0x3C9}, {0x3CE, 0x301}, {0x3D3, 0x3D2}, {0x3D3, 0x301}, {0x3D4, 0x3D2}, {0x3D4, 0x308}, {0x400, 0x415}, {0x400, 0x300}, {0x401, 0x415}, -{0x401, 0x308}, {0x403, 0x413}, {0x403, 0x301}, {0x407, 0x406}, {0x407, 0x308}, {0x40C, 0x41A}, {0x40C, 0x301}, {0x40D, 0x418}, {0x40D, 0x300}, {0x40E, 0x423}, {0x40E, 0x306}, {0x419, 0x418}, -{0x419, 0x306}, {0x439, 0x438}, {0x439, 0x306}, {0x450, 0x435}, {0x450, 0x300}, {0x451, 0x435}, {0x451, 0x308}, {0x453, 0x433}, {0x453, 0x301}, {0x457, 0x456}, {0x457, 0x308}, {0x45C, 0x43A}, -{0x45C, 0x301}, {0x45D, 0x438}, {0x45D, 0x300}, {0x45E, 0x443}, {0x45E, 0x306}, {0x476, 0x474}, {0x476, 0x30F}, {0x477, 0x475}, {0x477, 0x30F}, {0x4C1, 0x416}, {0x4C1, 0x306}, {0x4C2, 0x436}, -{0x4C2, 0x306}, {0x4D0, 0x410}, {0x4D0, 0x306}, {0x4D1, 0x430}, {0x4D1, 0x306}, {0x4D2, 0x410}, {0x4D2, 0x308}, {0x4D3, 0x430}, {0x4D3, 0x308}, {0x4D6, 0x415}, {0x4D6, 0x306}, {0x4D7, 0x435}, -{0x4D7, 0x306}, {0x4DA, 0x4D8}, {0x4DA, 0x308}, {0x4DB, 0x4D9}, {0x4DB, 0x308}, {0x4DC, 0x416}, {0x4DC, 0x308}, {0x4DD, 0x436}, {0x4DD, 0x308}, {0x4DE, 0x417}, {0x4DE, 0x308}, {0x4DF, 0x437}, -{0x4DF, 0x308}, {0x4E2, 0x418}, {0x4E2, 0x304}, {0x4E3, 0x438}, {0x4E3, 0x304}, {0x4E4, 0x418}, {0x4E4, 0x308}, {0x4E5, 0x438}, {0x4E5, 0x308}, {0x4E6, 0x41E}, {0x4E6, 0x308}, {0x4E7, 0x43E}, -{0x4E7, 0x308}, {0x4EA, 0x4E8}, {0x4EA, 0x308}, {0x4EB, 0x4E9}, {0x4EB, 0x308}, {0x4EC, 0x42D}, {0x4EC, 0x308}, {0x4ED, 0x44D}, {0x4ED, 0x308}, {0x4EE, 0x423}, {0x4EE, 0x304}, {0x4EF, 0x443}, -{0x4EF, 0x304}, {0x4F0, 0x423}, {0x4F0, 0x308}, {0x4F1, 0x443}, {0x4F1, 0x308}, {0x4F2, 0x423}, {0x4F2, 0x30B}, {0x4F3, 0x443}, {0x4F3, 0x30B}, {0x4F4, 0x427}, {0x4F4, 0x308}, {0x4F5, 0x447}, -{0x4F5, 0x308}, {0x4F8, 0x42B}, {0x4F8, 0x308}, {0x4F9, 0x44B}, {0x4F9, 0x308}, {0x622, 0x627}, {0x622, 0x653}, {0x623, 0x627}, {0x623, 0x654}, {0x624, 0x648}, {0x624, 0x654}, {0x625, 0x627}, -{0x625, 0x655}, {0x626, 0x64A}, {0x626, 0x654}, {0x6C0, 0x6D5}, {0x6C0, 0x654}, {0x6C2, 0x6C1}, {0x6C2, 0x654}, {0x6D3, 0x6D2}, {0x6D3, 0x654}, {0x929, 0x928}, {0x929, 0x93C}, {0x931, 0x930}, -{0x931, 0x93C}, {0x934, 0x933}, {0x934, 0x93C}, {0x958, 0x915}, {0x958, 0x93C}, {0x959, 0x916}, {0x959, 0x93C}, {0x95A, 0x917}, {0x95A, 0x93C}, {0x95B, 0x91C}, {0x95B, 0x93C}, {0x95C, 0x921}, -{0x95C, 0x93C}, {0x95D, 0x922}, {0x95D, 0x93C}, {0x95E, 0x92B}, {0x95E, 0x93C}, {0x95F, 0x92F}, {0x95F, 0x93C}, {0x9CB, 0x9C7}, {0x9CB, 0x9BE}, {0x9CC, 0x9C7}, {0x9CC, 0x9D7}, {0x9DC, 0x9A1}, -{0x9DC, 0x9BC}, {0x9DD, 0x9A2}, {0x9DD, 0x9BC}, {0x9DF, 0x9AF}, {0x9DF, 0x9BC}, {0xA33, 0xA32}, {0xA33, 0xA3C}, {0xA36, 0xA38}, {0xA36, 0xA3C}, {0xA59, 0xA16}, {0xA59, 0xA3C}, {0xA5A, 0xA17}, -{0xA5A, 0xA3C}, {0xA5B, 0xA1C}, {0xA5B, 0xA3C}, {0xA5E, 0xA2B}, {0xA5E, 0xA3C}, {0xB48, 0xB47}, {0xB48, 0xB56}, {0xB4B, 0xB47}, {0xB4B, 0xB3E}, {0xB4C, 0xB47}, {0xB4C, 0xB57}, {0xB5C, 0xB21}, -{0xB5C, 0xB3C}, {0xB5D, 0xB22}, {0xB5D, 0xB3C}, {0xB94, 0xB92}, {0xB94, 0xBD7}, {0xBCA, 0xBC6}, {0xBCA, 0xBBE}, {0xBCB, 0xBC7}, {0xBCB, 0xBBE}, {0xBCC, 0xBC6}, {0xBCC, 0xBD7}, {0xC48, 0xC46}, -{0xC48, 0xC56}, {0xCC0, 0xCBF}, {0xCC0, 0xCD5}, {0xCC7, 0xCC6}, {0xCC7, 0xCD5}, {0xCC8, 0xCC6}, {0xCC8, 0xCD6}, {0xCCA, 0xCC6}, {0xCCA, 0xCC2}, {0xCCB, 0xCC6}, {0xCCB, 0xCC2}, {0xCCB, 0xCD5}, -{0xD4A, 0xD46}, {0xD4A, 0xD3E}, {0xD4B, 0xD47}, {0xD4B, 0xD3E}, {0xD4C, 0xD46}, {0xD4C, 0xD57}, {0xDDA, 0xDD9}, {0xDDA, 0xDCA}, {0xDDC, 0xDD9}, {0xDDC, 0xDCF}, {0xDDD, 0xDD9}, {0xDDD, 0xDCF}, -{0xDDD, 0xDCA}, {0xDDE, 0xDD9}, {0xDDE, 0xDDF}, {0xF43, 0xF42}, {0xF43, 0xFB7}, {0xF4D, 0xF4C}, {0xF4D, 0xFB7}, {0xF52, 0xF51}, {0xF52, 0xFB7}, {0xF57, 0xF56}, {0xF57, 0xFB7}, {0xF5C, 0xF5B}, -{0xF5C, 0xFB7}, {0xF69, 0xF40}, {0xF69, 0xFB5}, {0xF73, 0xF71}, {0xF73, 0xF72}, {0xF75, 0xF71}, {0xF75, 0xF74}, {0xF76, 0xFB2}, {0xF76, 0xF80}, {0xF78, 0xFB3}, {0xF78, 0xF80}, {0xF81, 0xF71}, -{0xF81, 0xF80}, {0xF93, 0xF92}, {0xF93, 0xFB7}, {0xF9D, 0xF9C}, {0xF9D, 0xFB7}, {0xFA2, 0xFA1}, {0xFA2, 0xFB7}, {0xFA7, 0xFA6}, {0xFA7, 0xFB7}, {0xFAC, 0xFAB}, {0xFAC, 0xFB7}, {0xFB9, 0xF90}, -{0xFB9, 0xFB5}, {0x1026, 0x1025}, {0x1026, 0x102E}, {0x1B06, 0x1B05}, {0x1B06, 0x1B35}, {0x1B08, 0x1B07}, {0x1B08, 0x1B35}, {0x1B0A, 0x1B09}, {0x1B0A, 0x1B35}, {0x1B0C, 0x1B0B}, {0x1B0C, 0x1B35}, -{0x1B0E, 0x1B0D}, {0x1B0E, 0x1B35}, {0x1B12, 0x1B11}, {0x1B12, 0x1B35}, {0x1B3B, 0x1B3A}, {0x1B3B, 0x1B35}, {0x1B3D, 0x1B3C}, {0x1B3D, 0x1B35}, {0x1B40, 0x1B3E}, {0x1B40, 0x1B35}, {0x1B41, 0x1B3F}, -{0x1B41, 0x1B35}, {0x1B43, 0x1B42}, {0x1B43, 0x1B35}, {0x1E00, 0x41}, {0x1E00, 0x325}, {0x1E01, 0x61}, {0x1E01, 0x325}, {0x1E02, 0x42}, {0x1E02, 0x307}, {0x1E03, 0x62}, {0x1E03, 0x307}, -{0x1E04, 0x42}, {0x1E04, 0x323}, {0x1E05, 0x62}, {0x1E05, 0x323}, {0x1E06, 0x42}, {0x1E06, 0x331}, {0x1E07, 0x62}, {0x1E07, 0x331}, {0x1E08, 0x43}, {0x1E08, 0x327}, {0x1E08, 0x301}, {0x1E09, 0x63}, -{0x1E09, 0x327}, {0x1E09, 0x301}, {0x1E0A, 0x44}, {0x1E0A, 0x307}, {0x1E0B, 0x64}, {0x1E0B, 0x307}, {0x1E0C, 0x44}, {0x1E0C, 0x323}, {0x1E0D, 0x64}, {0x1E0D, 0x323}, {0x1E0E, 0x44}, {0x1E0E, 0x331}, -{0x1E0F, 0x64}, {0x1E0F, 0x331}, {0x1E10, 0x44}, {0x1E10, 0x327}, {0x1E11, 0x64}, {0x1E11, 0x327}, {0x1E12, 0x44}, {0x1E12, 0x32D}, {0x1E13, 0x64}, {0x1E13, 0x32D}, {0x1E14, 0x45}, {0x1E14, 0x304}, -{0x1E14, 0x300}, {0x1E15, 0x65}, {0x1E15, 0x304}, {0x1E15, 0x300}, {0x1E16, 0x45}, {0x1E16, 0x304}, {0x1E16, 0x301}, {0x1E17, 0x65}, {0x1E17, 0x304}, {0x1E17, 0x301}, {0x1E18, 0x45}, {0x1E18, 0x32D}, -{0x1E19, 0x65}, {0x1E19, 0x32D}, {0x1E1A, 0x45}, {0x1E1A, 0x330}, {0x1E1B, 0x65}, {0x1E1B, 0x330}, {0x1E1C, 0x45}, {0x1E1C, 0x327}, {0x1E1C, 0x306}, {0x1E1D, 0x65}, {0x1E1D, 0x327}, {0x1E1D, 0x306}, -{0x1E1E, 0x46}, {0x1E1E, 0x307}, {0x1E1F, 0x66}, {0x1E1F, 0x307}, {0x1E20, 0x47}, {0x1E20, 0x304}, {0x1E21, 0x67}, {0x1E21, 0x304}, {0x1E22, 0x48}, {0x1E22, 0x307}, {0x1E23, 0x68}, {0x1E23, 0x307}, -{0x1E24, 0x48}, {0x1E24, 0x323}, {0x1E25, 0x68}, {0x1E25, 0x323}, {0x1E26, 0x48}, {0x1E26, 0x308}, {0x1E27, 0x68}, {0x1E27, 0x308}, {0x1E28, 0x48}, {0x1E28, 0x327}, {0x1E29, 0x68}, {0x1E29, 0x327}, -{0x1E2A, 0x48}, {0x1E2A, 0x32E}, {0x1E2B, 0x68}, {0x1E2B, 0x32E}, {0x1E2C, 0x49}, {0x1E2C, 0x330}, {0x1E2D, 0x69}, {0x1E2D, 0x330}, {0x1E2E, 0x49}, {0x1E2E, 0x308}, {0x1E2E, 0x301}, {0x1E2F, 0x69}, -{0x1E2F, 0x308}, {0x1E2F, 0x301}, {0x1E30, 0x4B}, {0x1E30, 0x301}, {0x1E31, 0x6B}, {0x1E31, 0x301}, {0x1E32, 0x4B}, {0x1E32, 0x323}, {0x1E33, 0x6B}, {0x1E33, 0x323}, {0x1E34, 0x4B}, {0x1E34, 0x331}, -{0x1E35, 0x6B}, {0x1E35, 0x331}, {0x1E36, 0x4C}, {0x1E36, 0x323}, {0x1E37, 0x6C}, {0x1E37, 0x323}, {0x1E38, 0x4C}, {0x1E38, 0x323}, {0x1E38, 0x304}, {0x1E39, 0x6C}, {0x1E39, 0x323}, {0x1E39, 0x304}, -{0x1E3A, 0x4C}, {0x1E3A, 0x331}, {0x1E3B, 0x6C}, {0x1E3B, 0x331}, {0x1E3C, 0x4C}, {0x1E3C, 0x32D}, {0x1E3D, 0x6C}, {0x1E3D, 0x32D}, {0x1E3E, 0x4D}, {0x1E3E, 0x301}, {0x1E3F, 0x6D}, {0x1E3F, 0x301}, -{0x1E40, 0x4D}, {0x1E40, 0x307}, {0x1E41, 0x6D}, {0x1E41, 0x307}, {0x1E42, 0x4D}, {0x1E42, 0x323}, {0x1E43, 0x6D}, {0x1E43, 0x323}, {0x1E44, 0x4E}, {0x1E44, 0x307}, {0x1E45, 0x6E}, {0x1E45, 0x307}, -{0x1E46, 0x4E}, {0x1E46, 0x323}, {0x1E47, 0x6E}, {0x1E47, 0x323}, {0x1E48, 0x4E}, {0x1E48, 0x331}, {0x1E49, 0x6E}, {0x1E49, 0x331}, {0x1E4A, 0x4E}, {0x1E4A, 0x32D}, {0x1E4B, 0x6E}, {0x1E4B, 0x32D}, -{0x1E4C, 0x4F}, {0x1E4C, 0x303}, {0x1E4C, 0x301}, {0x1E4D, 0x6F}, {0x1E4D, 0x303}, {0x1E4D, 0x301}, {0x1E4E, 0x4F}, {0x1E4E, 0x303}, {0x1E4E, 0x308}, {0x1E4F, 0x6F}, {0x1E4F, 0x303}, {0x1E4F, 0x308}, -{0x1E50, 0x4F}, {0x1E50, 0x304}, {0x1E50, 0x300}, {0x1E51, 0x6F}, {0x1E51, 0x304}, {0x1E51, 0x300}, {0x1E52, 0x4F}, {0x1E52, 0x304}, {0x1E52, 0x301}, {0x1E53, 0x6F}, {0x1E53, 0x304}, {0x1E53, 0x301}, -{0x1E54, 0x50}, {0x1E54, 0x301}, {0x1E55, 0x70}, {0x1E55, 0x301}, {0x1E56, 0x50}, {0x1E56, 0x307}, {0x1E57, 0x70}, {0x1E57, 0x307}, {0x1E58, 0x52}, {0x1E58, 0x307}, {0x1E59, 0x72}, {0x1E59, 0x307}, -{0x1E5A, 0x52}, {0x1E5A, 0x323}, {0x1E5B, 0x72}, {0x1E5B, 0x323}, {0x1E5C, 0x52}, {0x1E5C, 0x323}, {0x1E5C, 0x304}, {0x1E5D, 0x72}, {0x1E5D, 0x323}, {0x1E5D, 0x304}, {0x1E5E, 0x52}, {0x1E5E, 0x331}, -{0x1E5F, 0x72}, {0x1E5F, 0x331}, {0x1E60, 0x53}, {0x1E60, 0x307}, {0x1E61, 0x73}, {0x1E61, 0x307}, {0x1E62, 0x53}, {0x1E62, 0x323}, {0x1E63, 0x73}, {0x1E63, 0x323}, {0x1E64, 0x53}, {0x1E64, 0x301}, -{0x1E64, 0x307}, {0x1E65, 0x73}, {0x1E65, 0x301}, {0x1E65, 0x307}, {0x1E66, 0x53}, {0x1E66, 0x30C}, {0x1E66, 0x307}, {0x1E67, 0x73}, {0x1E67, 0x30C}, {0x1E67, 0x307}, {0x1E68, 0x53}, {0x1E68, 0x323}, -{0x1E68, 0x307}, {0x1E69, 0x73}, {0x1E69, 0x323}, {0x1E69, 0x307}, {0x1E6A, 0x54}, {0x1E6A, 0x307}, {0x1E6B, 0x74}, {0x1E6B, 0x307}, {0x1E6C, 0x54}, {0x1E6C, 0x323}, {0x1E6D, 0x74}, {0x1E6D, 0x323}, -{0x1E6E, 0x54}, {0x1E6E, 0x331}, {0x1E6F, 0x74}, {0x1E6F, 0x331}, {0x1E70, 0x54}, {0x1E70, 0x32D}, {0x1E71, 0x74}, {0x1E71, 0x32D}, {0x1E72, 0x55}, {0x1E72, 0x324}, {0x1E73, 0x75}, {0x1E73, 0x324}, -{0x1E74, 0x55}, {0x1E74, 0x330}, {0x1E75, 0x75}, {0x1E75, 0x330}, {0x1E76, 0x55}, {0x1E76, 0x32D}, {0x1E77, 0x75}, {0x1E77, 0x32D}, {0x1E78, 0x55}, {0x1E78, 0x303}, {0x1E78, 0x301}, {0x1E79, 0x75}, -{0x1E79, 0x303}, {0x1E79, 0x301}, {0x1E7A, 0x55}, {0x1E7A, 0x304}, {0x1E7A, 0x308}, {0x1E7B, 0x75}, {0x1E7B, 0x304}, {0x1E7B, 0x308}, {0x1E7C, 0x56}, {0x1E7C, 0x303}, {0x1E7D, 0x76}, {0x1E7D, 0x303}, -{0x1E7E, 0x56}, {0x1E7E, 0x323}, {0x1E7F, 0x76}, {0x1E7F, 0x323}, {0x1E80, 0x57}, {0x1E80, 0x300}, {0x1E81, 0x77}, {0x1E81, 0x300}, {0x1E82, 0x57}, {0x1E82, 0x301}, {0x1E83, 0x77}, {0x1E83, 0x301}, -{0x1E84, 0x57}, {0x1E84, 0x308}, {0x1E85, 0x77}, {0x1E85, 0x308}, {0x1E86, 0x57}, {0x1E86, 0x307}, {0x1E87, 0x77}, {0x1E87, 0x307}, {0x1E88, 0x57}, {0x1E88, 0x323}, {0x1E89, 0x77}, {0x1E89, 0x323}, -{0x1E8A, 0x58}, {0x1E8A, 0x307}, {0x1E8B, 0x78}, {0x1E8B, 0x307}, {0x1E8C, 0x58}, {0x1E8C, 0x308}, {0x1E8D, 0x78}, {0x1E8D, 0x308}, {0x1E8E, 0x59}, {0x1E8E, 0x307}, {0x1E8F, 0x79}, {0x1E8F, 0x307}, -{0x1E90, 0x5A}, {0x1E90, 0x302}, {0x1E91, 0x7A}, {0x1E91, 0x302}, {0x1E92, 0x5A}, {0x1E92, 0x323}, {0x1E93, 0x7A}, {0x1E93, 0x323}, {0x1E94, 0x5A}, {0x1E94, 0x331}, {0x1E95, 0x7A}, {0x1E95, 0x331}, -{0x1E96, 0x68}, {0x1E96, 0x331}, {0x1E97, 0x74}, {0x1E97, 0x308}, {0x1E98, 0x77}, {0x1E98, 0x30A}, {0x1E99, 0x79}, {0x1E99, 0x30A}, {0x1E9B, 0x17F}, {0x1E9B, 0x307}, {0x1EA0, 0x41}, {0x1EA0, 0x323}, -{0x1EA1, 0x61}, {0x1EA1, 0x323}, {0x1EA2, 0x41}, {0x1EA2, 0x309}, {0x1EA3, 0x61}, {0x1EA3, 0x309}, {0x1EA4, 0x41}, {0x1EA4, 0x302}, {0x1EA4, 0x301}, {0x1EA5, 0x61}, {0x1EA5, 0x302}, {0x1EA5, 0x301}, -{0x1EA6, 0x41}, {0x1EA6, 0x302}, {0x1EA6, 0x300}, {0x1EA7, 0x61}, {0x1EA7, 0x302}, {0x1EA7, 0x300}, {0x1EA8, 0x41}, {0x1EA8, 0x302}, {0x1EA8, 0x309}, {0x1EA9, 0x61}, {0x1EA9, 0x302}, {0x1EA9, 0x309}, -{0x1EAA, 0x41}, {0x1EAA, 0x302}, {0x1EAA, 0x303}, {0x1EAB, 0x61}, {0x1EAB, 0x302}, {0x1EAB, 0x303}, {0x1EAC, 0x41}, {0x1EAC, 0x323}, {0x1EAC, 0x302}, {0x1EAD, 0x61}, {0x1EAD, 0x323}, {0x1EAD, 0x302}, -{0x1EAE, 0x41}, {0x1EAE, 0x306}, {0x1EAE, 0x301}, {0x1EAF, 0x61}, {0x1EAF, 0x306}, {0x1EAF, 0x301}, {0x1EB0, 0x41}, {0x1EB0, 0x306}, {0x1EB0, 0x300}, {0x1EB1, 0x61}, {0x1EB1, 0x306}, {0x1EB1, 0x300}, -{0x1EB2, 0x41}, {0x1EB2, 0x306}, {0x1EB2, 0x309}, {0x1EB3, 0x61}, {0x1EB3, 0x306}, {0x1EB3, 0x309}, {0x1EB4, 0x41}, {0x1EB4, 0x306}, {0x1EB4, 0x303}, {0x1EB5, 0x61}, {0x1EB5, 0x306}, {0x1EB5, 0x303}, -{0x1EB6, 0x41}, {0x1EB6, 0x323}, {0x1EB6, 0x306}, {0x1EB7, 0x61}, {0x1EB7, 0x323}, {0x1EB7, 0x306}, {0x1EB8, 0x45}, {0x1EB8, 0x323}, {0x1EB9, 0x65}, {0x1EB9, 0x323}, {0x1EBA, 0x45}, {0x1EBA, 0x309}, -{0x1EBB, 0x65}, {0x1EBB, 0x309}, {0x1EBC, 0x45}, {0x1EBC, 0x303}, {0x1EBD, 0x65}, {0x1EBD, 0x303}, {0x1EBE, 0x45}, {0x1EBE, 0x302}, {0x1EBE, 0x301}, {0x1EBF, 0x65}, {0x1EBF, 0x302}, {0x1EBF, 0x301}, -{0x1EC0, 0x45}, {0x1EC0, 0x302}, {0x1EC0, 0x300}, {0x1EC1, 0x65}, {0x1EC1, 0x302}, {0x1EC1, 0x300}, {0x1EC2, 0x45}, {0x1EC2, 0x302}, {0x1EC2, 0x309}, {0x1EC3, 0x65}, {0x1EC3, 0x302}, {0x1EC3, 0x309}, -{0x1EC4, 0x45}, {0x1EC4, 0x302}, {0x1EC4, 0x303}, {0x1EC5, 0x65}, {0x1EC5, 0x302}, {0x1EC5, 0x303}, {0x1EC6, 0x45}, {0x1EC6, 0x323}, {0x1EC6, 0x302}, {0x1EC7, 0x65}, {0x1EC7, 0x323}, {0x1EC7, 0x302}, -{0x1EC8, 0x49}, {0x1EC8, 0x309}, {0x1EC9, 0x69}, {0x1EC9, 0x309}, {0x1ECA, 0x49}, {0x1ECA, 0x323}, {0x1ECB, 0x69}, {0x1ECB, 0x323}, {0x1ECC, 0x4F}, {0x1ECC, 0x323}, {0x1ECD, 0x6F}, {0x1ECD, 0x323}, -{0x1ECE, 0x4F}, {0x1ECE, 0x309}, {0x1ECF, 0x6F}, {0x1ECF, 0x309}, {0x1ED0, 0x4F}, {0x1ED0, 0x302}, {0x1ED0, 0x301}, {0x1ED1, 0x6F}, {0x1ED1, 0x302}, {0x1ED1, 0x301}, {0x1ED2, 0x4F}, {0x1ED2, 0x302}, -{0x1ED2, 0x300}, {0x1ED3, 0x6F}, {0x1ED3, 0x302}, {0x1ED3, 0x300}, {0x1ED4, 0x4F}, {0x1ED4, 0x302}, {0x1ED4, 0x309}, {0x1ED5, 0x6F}, {0x1ED5, 0x302}, {0x1ED5, 0x309}, {0x1ED6, 0x4F}, {0x1ED6, 0x302}, -{0x1ED6, 0x303}, {0x1ED7, 0x6F}, {0x1ED7, 0x302}, {0x1ED7, 0x303}, {0x1ED8, 0x4F}, {0x1ED8, 0x323}, {0x1ED8, 0x302}, {0x1ED9, 0x6F}, {0x1ED9, 0x323}, {0x1ED9, 0x302}, {0x1EDA, 0x4F}, {0x1EDA, 0x31B}, -{0x1EDA, 0x301}, {0x1EDB, 0x6F}, {0x1EDB, 0x31B}, {0x1EDB, 0x301}, {0x1EDC, 0x4F}, {0x1EDC, 0x31B}, {0x1EDC, 0x300}, {0x1EDD, 0x6F}, {0x1EDD, 0x31B}, {0x1EDD, 0x300}, {0x1EDE, 0x4F}, {0x1EDE, 0x31B}, -{0x1EDE, 0x309}, {0x1EDF, 0x6F}, {0x1EDF, 0x31B}, {0x1EDF, 0x309}, {0x1EE0, 0x4F}, {0x1EE0, 0x31B}, {0x1EE0, 0x303}, {0x1EE1, 0x6F}, {0x1EE1, 0x31B}, {0x1EE1, 0x303}, {0x1EE2, 0x4F}, {0x1EE2, 0x31B}, -{0x1EE2, 0x323}, {0x1EE3, 0x6F}, {0x1EE3, 0x31B}, {0x1EE3, 0x323}, {0x1EE4, 0x55}, {0x1EE4, 0x323}, {0x1EE5, 0x75}, {0x1EE5, 0x323}, {0x1EE6, 0x55}, {0x1EE6, 0x309}, {0x1EE7, 0x75}, {0x1EE7, 0x309}, -{0x1EE8, 0x55}, {0x1EE8, 0x31B}, {0x1EE8, 0x301}, {0x1EE9, 0x75}, {0x1EE9, 0x31B}, {0x1EE9, 0x301}, {0x1EEA, 0x55}, {0x1EEA, 0x31B}, {0x1EEA, 0x300}, {0x1EEB, 0x75}, {0x1EEB, 0x31B}, {0x1EEB, 0x300}, -{0x1EEC, 0x55}, {0x1EEC, 0x31B}, {0x1EEC, 0x309}, {0x1EED, 0x75}, {0x1EED, 0x31B}, {0x1EED, 0x309}, {0x1EEE, 0x55}, {0x1EEE, 0x31B}, {0x1EEE, 0x303}, {0x1EEF, 0x75}, {0x1EEF, 0x31B}, {0x1EEF, 0x303}, -{0x1EF0, 0x55}, {0x1EF0, 0x31B}, {0x1EF0, 0x323}, {0x1EF1, 0x75}, {0x1EF1, 0x31B}, {0x1EF1, 0x323}, {0x1EF2, 0x59}, {0x1EF2, 0x300}, {0x1EF3, 0x79}, {0x1EF3, 0x300}, {0x1EF4, 0x59}, {0x1EF4, 0x323}, -{0x1EF5, 0x79}, {0x1EF5, 0x323}, {0x1EF6, 0x59}, {0x1EF6, 0x309}, {0x1EF7, 0x79}, {0x1EF7, 0x309}, {0x1EF8, 0x59}, {0x1EF8, 0x303}, {0x1EF9, 0x79}, {0x1EF9, 0x303}, {0x1F00, 0x3B1}, {0x1F00, 0x313}, -{0x1F01, 0x3B1}, {0x1F01, 0x314}, {0x1F02, 0x3B1}, {0x1F02, 0x313}, {0x1F02, 0x300}, {0x1F03, 0x3B1}, {0x1F03, 0x314}, {0x1F03, 0x300}, {0x1F04, 0x3B1}, {0x1F04, 0x313}, {0x1F04, 0x301}, -{0x1F05, 0x3B1}, {0x1F05, 0x314}, {0x1F05, 0x301}, {0x1F06, 0x3B1}, {0x1F06, 0x313}, {0x1F06, 0x342}, {0x1F07, 0x3B1}, {0x1F07, 0x314}, {0x1F07, 0x342}, {0x1F08, 0x391}, {0x1F08, 0x313}, -{0x1F09, 0x391}, {0x1F09, 0x314}, {0x1F0A, 0x391}, {0x1F0A, 0x313}, {0x1F0A, 0x300}, {0x1F0B, 0x391}, {0x1F0B, 0x314}, {0x1F0B, 0x300}, {0x1F0C, 0x391}, {0x1F0C, 0x313}, {0x1F0C, 0x301}, -{0x1F0D, 0x391}, {0x1F0D, 0x314}, {0x1F0D, 0x301}, {0x1F0E, 0x391}, {0x1F0E, 0x313}, {0x1F0E, 0x342}, {0x1F0F, 0x391}, {0x1F0F, 0x314}, {0x1F0F, 0x342}, {0x1F10, 0x3B5}, {0x1F10, 0x313}, -{0x1F11, 0x3B5}, {0x1F11, 0x314}, {0x1F12, 0x3B5}, {0x1F12, 0x313}, {0x1F12, 0x300}, {0x1F13, 0x3B5}, {0x1F13, 0x314}, {0x1F13, 0x300}, {0x1F14, 0x3B5}, {0x1F14, 0x313}, {0x1F14, 0x301}, -{0x1F15, 0x3B5}, {0x1F15, 0x314}, {0x1F15, 0x301}, {0x1F18, 0x395}, {0x1F18, 0x313}, {0x1F19, 0x395}, {0x1F19, 0x314}, {0x1F1A, 0x395}, {0x1F1A, 0x313}, {0x1F1A, 0x300}, {0x1F1B, 0x395}, -{0x1F1B, 0x314}, {0x1F1B, 0x300}, {0x1F1C, 0x395}, {0x1F1C, 0x313}, {0x1F1C, 0x301}, {0x1F1D, 0x395}, {0x1F1D, 0x314}, {0x1F1D, 0x301}, {0x1F20, 0x3B7}, {0x1F20, 0x313}, {0x1F21, 0x3B7}, -{0x1F21, 0x314}, {0x1F22, 0x3B7}, {0x1F22, 0x313}, {0x1F22, 0x300}, {0x1F23, 0x3B7}, {0x1F23, 0x314}, {0x1F23, 0x300}, {0x1F24, 0x3B7}, {0x1F24, 0x313}, {0x1F24, 0x301}, {0x1F25, 0x3B7}, -{0x1F25, 0x314}, {0x1F25, 0x301}, {0x1F26, 0x3B7}, {0x1F26, 0x313}, {0x1F26, 0x342}, {0x1F27, 0x3B7}, {0x1F27, 0x314}, {0x1F27, 0x342}, {0x1F28, 0x397}, {0x1F28, 0x313}, {0x1F29, 0x397}, -{0x1F29, 0x314}, {0x1F2A, 0x397}, {0x1F2A, 0x313}, {0x1F2A, 0x300}, {0x1F2B, 0x397}, {0x1F2B, 0x314}, {0x1F2B, 0x300}, {0x1F2C, 0x397}, {0x1F2C, 0x313}, {0x1F2C, 0x301}, {0x1F2D, 0x397}, -{0x1F2D, 0x314}, {0x1F2D, 0x301}, {0x1F2E, 0x397}, {0x1F2E, 0x313}, {0x1F2E, 0x342}, {0x1F2F, 0x397}, {0x1F2F, 0x314}, {0x1F2F, 0x342}, {0x1F30, 0x3B9}, {0x1F30, 0x313}, {0x1F31, 0x3B9}, -{0x1F31, 0x314}, {0x1F32, 0x3B9}, {0x1F32, 0x313}, {0x1F32, 0x300}, {0x1F33, 0x3B9}, {0x1F33, 0x314}, {0x1F33, 0x300}, {0x1F34, 0x3B9}, {0x1F34, 0x313}, {0x1F34, 0x301}, {0x1F35, 0x3B9}, -{0x1F35, 0x314}, {0x1F35, 0x301}, {0x1F36, 0x3B9}, {0x1F36, 0x313}, {0x1F36, 0x342}, {0x1F37, 0x3B9}, {0x1F37, 0x314}, {0x1F37, 0x342}, {0x1F38, 0x399}, {0x1F38, 0x313}, {0x1F39, 0x399}, -{0x1F39, 0x314}, {0x1F3A, 0x399}, {0x1F3A, 0x313}, {0x1F3A, 0x300}, {0x1F3B, 0x399}, {0x1F3B, 0x314}, {0x1F3B, 0x300}, {0x1F3C, 0x399}, {0x1F3C, 0x313}, {0x1F3C, 0x301}, {0x1F3D, 0x399}, -{0x1F3D, 0x314}, {0x1F3D, 0x301}, {0x1F3E, 0x399}, {0x1F3E, 0x313}, {0x1F3E, 0x342}, {0x1F3F, 0x399}, {0x1F3F, 0x314}, {0x1F3F, 0x342}, {0x1F40, 0x3BF}, {0x1F40, 0x313}, {0x1F41, 0x3BF}, -{0x1F41, 0x314}, {0x1F42, 0x3BF}, {0x1F42, 0x313}, {0x1F42, 0x300}, {0x1F43, 0x3BF}, {0x1F43, 0x314}, {0x1F43, 0x300}, {0x1F44, 0x3BF}, {0x1F44, 0x313}, {0x1F44, 0x301}, {0x1F45, 0x3BF}, -{0x1F45, 0x314}, {0x1F45, 0x301}, {0x1F48, 0x39F}, {0x1F48, 0x313}, {0x1F49, 0x39F}, {0x1F49, 0x314}, {0x1F4A, 0x39F}, {0x1F4A, 0x313}, {0x1F4A, 0x300}, {0x1F4B, 0x39F}, {0x1F4B, 0x314}, -{0x1F4B, 0x300}, {0x1F4C, 0x39F}, {0x1F4C, 0x313}, {0x1F4C, 0x301}, {0x1F4D, 0x39F}, {0x1F4D, 0x314}, {0x1F4D, 0x301}, {0x1F50, 0x3C5}, {0x1F50, 0x313}, {0x1F51, 0x3C5}, {0x1F51, 0x314}, -{0x1F52, 0x3C5}, {0x1F52, 0x313}, {0x1F52, 0x300}, {0x1F53, 0x3C5}, {0x1F53, 0x314}, {0x1F53, 0x300}, {0x1F54, 0x3C5}, {0x1F54, 0x313}, {0x1F54, 0x301}, {0x1F55, 0x3C5}, {0x1F55, 0x314}, -{0x1F55, 0x301}, {0x1F56, 0x3C5}, {0x1F56, 0x313}, {0x1F56, 0x342}, {0x1F57, 0x3C5}, {0x1F57, 0x314}, {0x1F57, 0x342}, {0x1F59, 0x3A5}, {0x1F59, 0x314}, {0x1F5B, 0x3A5}, {0x1F5B, 0x314}, -{0x1F5B, 0x300}, {0x1F5D, 0x3A5}, {0x1F5D, 0x314}, {0x1F5D, 0x301}, {0x1F5F, 0x3A5}, {0x1F5F, 0x314}, {0x1F5F, 0x342}, {0x1F60, 0x3C9}, {0x1F60, 0x313}, {0x1F61, 0x3C9}, {0x1F61, 0x314}, -{0x1F62, 0x3C9}, {0x1F62, 0x313}, {0x1F62, 0x300}, {0x1F63, 0x3C9}, {0x1F63, 0x314}, {0x1F63, 0x300}, {0x1F64, 0x3C9}, {0x1F64, 0x313}, {0x1F64, 0x301}, {0x1F65, 0x3C9}, {0x1F65, 0x314}, -{0x1F65, 0x301}, {0x1F66, 0x3C9}, {0x1F66, 0x313}, {0x1F66, 0x342}, {0x1F67, 0x3C9}, {0x1F67, 0x314}, {0x1F67, 0x342}, {0x1F68, 0x3A9}, {0x1F68, 0x313}, {0x1F69, 0x3A9}, {0x1F69, 0x314}, -{0x1F6A, 0x3A9}, {0x1F6A, 0x313}, {0x1F6A, 0x300}, {0x1F6B, 0x3A9}, {0x1F6B, 0x314}, {0x1F6B, 0x300}, {0x1F6C, 0x3A9}, {0x1F6C, 0x313}, {0x1F6C, 0x301}, {0x1F6D, 0x3A9}, {0x1F6D, 0x314}, -{0x1F6D, 0x301}, {0x1F6E, 0x3A9}, {0x1F6E, 0x313}, {0x1F6E, 0x342}, {0x1F6F, 0x3A9}, {0x1F6F, 0x314}, {0x1F6F, 0x342}, {0x1F70, 0x3B1}, {0x1F70, 0x300}, {0x1F71, 0x3B1}, {0x1F71, 0x301}, -{0x1F72, 0x3B5}, {0x1F72, 0x300}, {0x1F73, 0x3B5}, {0x1F73, 0x301}, {0x1F74, 0x3B7}, {0x1F74, 0x300}, {0x1F75, 0x3B7}, {0x1F75, 0x301}, {0x1F76, 0x3B9}, {0x1F76, 0x300}, {0x1F77, 0x3B9}, -{0x1F77, 0x301}, {0x1F78, 0x3BF}, {0x1F78, 0x300}, {0x1F79, 0x3BF}, {0x1F79, 0x301}, {0x1F7A, 0x3C5}, {0x1F7A, 0x300}, {0x1F7B, 0x3C5}, {0x1F7B, 0x301}, {0x1F7C, 0x3C9}, {0x1F7C, 0x300}, -{0x1F7D, 0x3C9}, {0x1F7D, 0x301}, {0x1F80, 0x3B1}, {0x1F80, 0x313}, {0x1F80, 0x345}, {0x1F81, 0x3B1}, {0x1F81, 0x314}, {0x1F81, 0x345}, {0x1F82, 0x3B1}, {0x1F82, 0x313}, {0x1F82, 0x300}, -{0x1F82, 0x345}, {0x1F83, 0x3B1}, {0x1F83, 0x314}, {0x1F83, 0x300}, {0x1F83, 0x345}, {0x1F84, 0x3B1}, {0x1F84, 0x313}, {0x1F84, 0x301}, {0x1F84, 0x345}, {0x1F85, 0x3B1}, {0x1F85, 0x314}, -{0x1F85, 0x301}, {0x1F85, 0x345}, {0x1F86, 0x3B1}, {0x1F86, 0x313}, {0x1F86, 0x342}, {0x1F86, 0x345}, {0x1F87, 0x3B1}, {0x1F87, 0x314}, {0x1F87, 0x342}, {0x1F87, 0x345}, {0x1F88, 0x391}, -{0x1F88, 0x313}, {0x1F88, 0x345}, {0x1F89, 0x391}, {0x1F89, 0x314}, {0x1F89, 0x345}, {0x1F8A, 0x391}, {0x1F8A, 0x313}, {0x1F8A, 0x300}, {0x1F8A, 0x345}, {0x1F8B, 0x391}, {0x1F8B, 0x314}, -{0x1F8B, 0x300}, {0x1F8B, 0x345}, {0x1F8C, 0x391}, {0x1F8C, 0x313}, {0x1F8C, 0x301}, {0x1F8C, 0x345}, {0x1F8D, 0x391}, {0x1F8D, 0x314}, {0x1F8D, 0x301}, {0x1F8D, 0x345}, {0x1F8E, 0x391}, -{0x1F8E, 0x313}, {0x1F8E, 0x342}, {0x1F8E, 0x345}, {0x1F8F, 0x391}, {0x1F8F, 0x314}, {0x1F8F, 0x342}, {0x1F8F, 0x345}, {0x1F90, 0x3B7}, {0x1F90, 0x313}, {0x1F90, 0x345}, {0x1F91, 0x3B7}, -{0x1F91, 0x314}, {0x1F91, 0x345}, {0x1F92, 0x3B7}, {0x1F92, 0x313}, {0x1F92, 0x300}, {0x1F92, 0x345}, {0x1F93, 0x3B7}, {0x1F93, 0x314}, {0x1F93, 0x300}, {0x1F93, 0x345}, {0x1F94, 0x3B7}, -{0x1F94, 0x313}, {0x1F94, 0x301}, {0x1F94, 0x345}, {0x1F95, 0x3B7}, {0x1F95, 0x314}, {0x1F95, 0x301}, {0x1F95, 0x345}, {0x1F96, 0x3B7}, {0x1F96, 0x313}, {0x1F96, 0x342}, {0x1F96, 0x345}, -{0x1F97, 0x3B7}, {0x1F97, 0x314}, {0x1F97, 0x342}, {0x1F97, 0x345}, {0x1F98, 0x397}, {0x1F98, 0x313}, {0x1F98, 0x345}, {0x1F99, 0x397}, {0x1F99, 0x314}, {0x1F99, 0x345}, {0x1F9A, 0x397}, -{0x1F9A, 0x313}, {0x1F9A, 0x300}, {0x1F9A, 0x345}, {0x1F9B, 0x397}, {0x1F9B, 0x314}, {0x1F9B, 0x300}, {0x1F9B, 0x345}, {0x1F9C, 0x397}, {0x1F9C, 0x313}, {0x1F9C, 0x301}, {0x1F9C, 0x345}, -{0x1F9D, 0x397}, {0x1F9D, 0x314}, {0x1F9D, 0x301}, {0x1F9D, 0x345}, {0x1F9E, 0x397}, {0x1F9E, 0x313}, {0x1F9E, 0x342}, {0x1F9E, 0x345}, {0x1F9F, 0x397}, {0x1F9F, 0x314}, {0x1F9F, 0x342}, -{0x1F9F, 0x345}, {0x1FA0, 0x3C9}, {0x1FA0, 0x313}, {0x1FA0, 0x345}, {0x1FA1, 0x3C9}, {0x1FA1, 0x314}, {0x1FA1, 0x345}, {0x1FA2, 0x3C9}, {0x1FA2, 0x313}, {0x1FA2, 0x300}, {0x1FA2, 0x345}, -{0x1FA3, 0x3C9}, {0x1FA3, 0x314}, {0x1FA3, 0x300}, {0x1FA3, 0x345}, {0x1FA4, 0x3C9}, {0x1FA4, 0x313}, {0x1FA4, 0x301}, {0x1FA4, 0x345}, {0x1FA5, 0x3C9}, {0x1FA5, 0x314}, {0x1FA5, 0x301}, -{0x1FA5, 0x345}, {0x1FA6, 0x3C9}, {0x1FA6, 0x313}, {0x1FA6, 0x342}, {0x1FA6, 0x345}, {0x1FA7, 0x3C9}, {0x1FA7, 0x314}, {0x1FA7, 0x342}, {0x1FA7, 0x345}, {0x1FA8, 0x3A9}, {0x1FA8, 0x313}, -{0x1FA8, 0x345}, {0x1FA9, 0x3A9}, {0x1FA9, 0x314}, {0x1FA9, 0x345}, {0x1FAA, 0x3A9}, {0x1FAA, 0x313}, {0x1FAA, 0x300}, {0x1FAA, 0x345}, {0x1FAB, 0x3A9}, {0x1FAB, 0x314}, {0x1FAB, 0x300}, -{0x1FAB, 0x345}, {0x1FAC, 0x3A9}, {0x1FAC, 0x313}, {0x1FAC, 0x301}, {0x1FAC, 0x345}, {0x1FAD, 0x3A9}, {0x1FAD, 0x314}, {0x1FAD, 0x301}, {0x1FAD, 0x345}, {0x1FAE, 0x3A9}, {0x1FAE, 0x313}, -{0x1FAE, 0x342}, {0x1FAE, 0x345}, {0x1FAF, 0x3A9}, {0x1FAF, 0x314}, {0x1FAF, 0x342}, {0x1FAF, 0x345}, {0x1FB0, 0x3B1}, {0x1FB0, 0x306}, {0x1FB1, 0x3B1}, {0x1FB1, 0x304}, {0x1FB2, 0x3B1}, -{0x1FB2, 0x300}, {0x1FB2, 0x345}, {0x1FB3, 0x3B1}, {0x1FB3, 0x345}, {0x1FB4, 0x3B1}, {0x1FB4, 0x301}, {0x1FB4, 0x345}, {0x1FB6, 0x3B1}, {0x1FB6, 0x342}, {0x1FB7, 0x3B1}, {0x1FB7, 0x342}, -{0x1FB7, 0x345}, {0x1FB8, 0x391}, {0x1FB8, 0x306}, {0x1FB9, 0x391}, {0x1FB9, 0x304}, {0x1FBA, 0x391}, {0x1FBA, 0x300}, {0x1FBB, 0x391}, {0x1FBB, 0x301}, {0x1FBC, 0x391}, {0x1FBC, 0x345}, -{0x1FBE, 0x3B9}, {0x1FC1, 0xA8}, {0x1FC1, 0x342}, {0x1FC2, 0x3B7}, {0x1FC2, 0x300}, {0x1FC2, 0x345}, {0x1FC3, 0x3B7}, {0x1FC3, 0x345}, {0x1FC4, 0x3B7}, {0x1FC4, 0x301}, {0x1FC4, 0x345}, -{0x1FC6, 0x3B7}, {0x1FC6, 0x342}, {0x1FC7, 0x3B7}, {0x1FC7, 0x342}, {0x1FC7, 0x345}, {0x1FC8, 0x395}, {0x1FC8, 0x300}, {0x1FC9, 0x395}, {0x1FC9, 0x301}, {0x1FCA, 0x397}, {0x1FCA, 0x300}, -{0x1FCB, 0x397}, {0x1FCB, 0x301}, {0x1FCC, 0x397}, {0x1FCC, 0x345}, {0x1FCD, 0x1FBF}, {0x1FCD, 0x300}, {0x1FCE, 0x1FBF}, {0x1FCE, 0x301}, {0x1FCF, 0x1FBF}, {0x1FCF, 0x342}, {0x1FD0, 0x3B9}, -{0x1FD0, 0x306}, {0x1FD1, 0x3B9}, {0x1FD1, 0x304}, {0x1FD2, 0x3B9}, {0x1FD2, 0x308}, {0x1FD2, 0x300}, {0x1FD3, 0x3B9}, {0x1FD3, 0x308}, {0x1FD3, 0x301}, {0x1FD6, 0x3B9}, {0x1FD6, 0x342}, -{0x1FD7, 0x3B9}, {0x1FD7, 0x308}, {0x1FD7, 0x342}, {0x1FD8, 0x399}, {0x1FD8, 0x306}, {0x1FD9, 0x399}, {0x1FD9, 0x304}, {0x1FDA, 0x399}, {0x1FDA, 0x300}, {0x1FDB, 0x399}, {0x1FDB, 0x301}, -{0x1FDD, 0x1FFE}, {0x1FDD, 0x300}, {0x1FDE, 0x1FFE}, {0x1FDE, 0x301}, {0x1FDF, 0x1FFE}, {0x1FDF, 0x342}, {0x1FE0, 0x3C5}, {0x1FE0, 0x306}, {0x1FE1, 0x3C5}, {0x1FE1, 0x304}, {0x1FE2, 0x3C5}, -{0x1FE2, 0x308}, {0x1FE2, 0x300}, {0x1FE3, 0x3C5}, {0x1FE3, 0x308}, {0x1FE3, 0x301}, {0x1FE4, 0x3C1}, {0x1FE4, 0x313}, {0x1FE5, 0x3C1}, {0x1FE5, 0x314}, {0x1FE6, 0x3C5}, {0x1FE6, 0x342}, -{0x1FE7, 0x3C5}, {0x1FE7, 0x308}, {0x1FE7, 0x342}, {0x1FE8, 0x3A5}, {0x1FE8, 0x306}, {0x1FE9, 0x3A5}, {0x1FE9, 0x304}, {0x1FEA, 0x3A5}, {0x1FEA, 0x300}, {0x1FEB, 0x3A5}, {0x1FEB, 0x301}, -{0x1FEC, 0x3A1}, {0x1FEC, 0x314}, {0x1FED, 0xA8}, {0x1FED, 0x300}, {0x1FEE, 0xA8}, {0x1FEE, 0x301}, {0x1FEF, 0x60}, {0x1FF2, 0x3C9}, {0x1FF2, 0x300}, {0x1FF2, 0x345}, {0x1FF3, 0x3C9}, {0x1FF3, 0x345}, -{0x1FF4, 0x3C9}, {0x1FF4, 0x301}, {0x1FF4, 0x345}, {0x1FF6, 0x3C9}, {0x1FF6, 0x342}, {0x1FF7, 0x3C9}, {0x1FF7, 0x342}, {0x1FF7, 0x345}, {0x1FF8, 0x39F}, {0x1FF8, 0x300}, {0x1FF9, 0x39F}, -{0x1FF9, 0x301}, {0x1FFA, 0x3A9}, {0x1FFA, 0x300}, {0x1FFB, 0x3A9}, {0x1FFB, 0x301}, {0x1FFC, 0x3A9}, {0x1FFC, 0x345}, {0x1FFD, 0xB4}, {0x2000, 0x2002}, {0x2001, 0x2003}, {0x2126, 0x3A9}, -{0x212A, 0x4B}, {0x212B, 0x41}, {0x212B, 0x30A}, {0x219A, 0x2190}, {0x219A, 0x338}, {0x219B, 0x2192}, {0x219B, 0x338}, {0x21AE, 0x2194}, {0x21AE, 0x338}, {0x21CD, 0x21D0}, {0x21CD, 0x338}, -{0x21CE, 0x21D4}, {0x21CE, 0x338}, {0x21CF, 0x21D2}, {0x21CF, 0x338}, {0x2204, 0x2203}, {0x2204, 0x338}, {0x2209, 0x2208}, {0x2209, 0x338}, {0x220C, 0x220B}, {0x220C, 0x338}, {0x2224, 0x2223}, -{0x2224, 0x338}, {0x2226, 0x2225}, {0x2226, 0x338}, {0x2241, 0x223C}, {0x2241, 0x338}, {0x2244, 0x2243}, {0x2244, 0x338}, {0x2247, 0x2245}, {0x2247, 0x338}, {0x2249, 0x2248}, {0x2249, 0x338}, -{0x2260, 0x3D}, {0x2260, 0x338}, {0x2262, 0x2261}, {0x2262, 0x338}, {0x226D, 0x224D}, {0x226D, 0x338}, {0x226E, 0x3C}, {0x226E, 0x338}, {0x226F, 0x3E}, {0x226F, 0x338}, {0x2270, 0x2264}, -{0x2270, 0x338}, {0x2271, 0x2265}, {0x2271, 0x338}, {0x2274, 0x2272}, {0x2274, 0x338}, {0x2275, 0x2273}, {0x2275, 0x338}, {0x2278, 0x2276}, {0x2278, 0x338}, {0x2279, 0x2277}, {0x2279, 0x338}, -{0x2280, 0x227A}, {0x2280, 0x338}, {0x2281, 0x227B}, {0x2281, 0x338}, {0x2284, 0x2282}, {0x2284, 0x338}, {0x2285, 0x2283}, {0x2285, 0x338}, {0x2288, 0x2286}, {0x2288, 0x338}, {0x2289, 0x2287}, -{0x2289, 0x338}, {0x22AC, 0x22A2}, {0x22AC, 0x338}, {0x22AD, 0x22A8}, {0x22AD, 0x338}, {0x22AE, 0x22A9}, {0x22AE, 0x338}, {0x22AF, 0x22AB}, {0x22AF, 0x338}, {0x22E0, 0x227C}, {0x22E0, 0x338}, -{0x22E1, 0x227D}, {0x22E1, 0x338}, {0x22E2, 0x2291}, {0x22E2, 0x338}, {0x22E3, 0x2292}, {0x22E3, 0x338}, {0x22EA, 0x22B2}, {0x22EA, 0x338}, {0x22EB, 0x22B3}, {0x22EB, 0x338}, {0x22EC, 0x22B4}, -{0x22EC, 0x338}, {0x22ED, 0x22B5}, {0x22ED, 0x338}, {0x2329, 0x3008}, {0x232A, 0x3009}, {0x2ADC, 0x2ADD}, {0x2ADC, 0x338}, {0x304C, 0x304B}, {0x304C, 0x3099}, {0x304E, 0x304D}, {0x304E, 0x3099}, -{0x3050, 0x304F}, {0x3050, 0x3099}, {0x3052, 0x3051}, {0x3052, 0x3099}, {0x3054, 0x3053}, {0x3054, 0x3099}, {0x3056, 0x3055}, {0x3056, 0x3099}, {0x3058, 0x3057}, {0x3058, 0x3099}, {0x305A, 0x3059}, -{0x305A, 0x3099}, {0x305C, 0x305B}, {0x305C, 0x3099}, {0x305E, 0x305D}, {0x305E, 0x3099}, {0x3060, 0x305F}, {0x3060, 0x3099}, {0x3062, 0x3061}, {0x3062, 0x3099}, {0x3065, 0x3064}, {0x3065, 0x3099}, -{0x3067, 0x3066}, {0x3067, 0x3099}, {0x3069, 0x3068}, {0x3069, 0x3099}, {0x3070, 0x306F}, {0x3070, 0x3099}, {0x3071, 0x306F}, {0x3071, 0x309A}, {0x3073, 0x3072}, {0x3073, 0x3099}, {0x3074, 0x3072}, -{0x3074, 0x309A}, {0x3076, 0x3075}, {0x3076, 0x3099}, {0x3077, 0x3075}, {0x3077, 0x309A}, {0x3079, 0x3078}, {0x3079, 0x3099}, {0x307A, 0x3078}, {0x307A, 0x309A}, {0x307C, 0x307B}, {0x307C, 0x3099}, -{0x307D, 0x307B}, {0x307D, 0x309A}, {0x3094, 0x3046}, {0x3094, 0x3099}, {0x309E, 0x309D}, {0x309E, 0x3099}, {0x30AC, 0x30AB}, {0x30AC, 0x3099}, {0x30AE, 0x30AD}, {0x30AE, 0x3099}, {0x30B0, 0x30AF}, -{0x30B0, 0x3099}, {0x30B2, 0x30B1}, {0x30B2, 0x3099}, {0x30B4, 0x30B3}, {0x30B4, 0x3099}, {0x30B6, 0x30B5}, {0x30B6, 0x3099}, {0x30B8, 0x30B7}, {0x30B8, 0x3099}, {0x30BA, 0x30B9}, {0x30BA, 0x3099}, -{0x30BC, 0x30BB}, {0x30BC, 0x3099}, {0x30BE, 0x30BD}, {0x30BE, 0x3099}, {0x30C0, 0x30BF}, {0x30C0, 0x3099}, {0x30C2, 0x30C1}, {0x30C2, 0x3099}, {0x30C5, 0x30C4}, {0x30C5, 0x3099}, {0x30C7, 0x30C6}, -{0x30C7, 0x3099}, {0x30C9, 0x30C8}, {0x30C9, 0x3099}, {0x30D0, 0x30CF}, {0x30D0, 0x3099}, {0x30D1, 0x30CF}, {0x30D1, 0x309A}, {0x30D3, 0x30D2}, {0x30D3, 0x3099}, {0x30D4, 0x30D2}, {0x30D4, 0x309A}, -{0x30D6, 0x30D5}, {0x30D6, 0x3099}, {0x30D7, 0x30D5}, {0x30D7, 0x309A}, {0x30D9, 0x30D8}, {0x30D9, 0x3099}, {0x30DA, 0x30D8}, {0x30DA, 0x309A}, {0x30DC, 0x30DB}, {0x30DC, 0x3099}, {0x30DD, 0x30DB}, -{0x30DD, 0x309A}, {0x30F4, 0x30A6}, {0x30F4, 0x3099}, {0x30F7, 0x30EF}, {0x30F7, 0x3099}, {0x30F8, 0x30F0}, {0x30F8, 0x3099}, {0x30F9, 0x30F1}, {0x30F9, 0x3099}, {0x30FA, 0x30F2}, {0x30FA, 0x3099}, -{0x30FE, 0x30FD}, {0x30FE, 0x3099}, {0xF900, 0x8C48}, {0xF901, 0x66F4}, {0xF902, 0x8ECA}, {0xF903, 0x8CC8}, {0xF904, 0x6ED1}, {0xF905, 0x4E32}, {0xF906, 0x53E5}, {0xF907, 0x9F9C}, {0xF908, 0x9F9C}, -{0xF909, 0x5951}, {0xF90A, 0x91D1}, {0xF90B, 0x5587}, {0xF90C, 0x5948}, {0xF90D, 0x61F6}, {0xF90E, 0x7669}, {0xF90F, 0x7F85}, {0xF910, 0x863F}, {0xF911, 0x87BA}, {0xF912, 0x88F8}, {0xF913, 0x908F}, -{0xF914, 0x6A02}, {0xF915, 0x6D1B}, {0xF916, 0x70D9}, {0xF917, 0x73DE}, {0xF918, 0x843D}, {0xF919, 0x916A}, {0xF91A, 0x99F1}, {0xF91B, 0x4E82}, {0xF91C, 0x5375}, {0xF91D, 0x6B04}, {0xF91E, 0x721B}, -{0xF91F, 0x862D}, {0xF920, 0x9E1E}, {0xF921, 0x5D50}, {0xF922, 0x6FEB}, {0xF923, 0x85CD}, {0xF924, 0x8964}, {0xF925, 0x62C9}, {0xF926, 0x81D8}, {0xF927, 0x881F}, {0xF928, 0x5ECA}, {0xF929, 0x6717}, -{0xF92A, 0x6D6A}, {0xF92B, 0x72FC}, {0xF92C, 0x90CE}, {0xF92D, 0x4F86}, {0xF92E, 0x51B7}, {0xF92F, 0x52DE}, {0xF930, 0x64C4}, {0xF931, 0x6AD3}, {0xF932, 0x7210}, {0xF933, 0x76E7}, {0xF934, 0x8001}, -{0xF935, 0x8606}, {0xF936, 0x865C}, {0xF937, 0x8DEF}, {0xF938, 0x9732}, {0xF939, 0x9B6F}, {0xF93A, 0x9DFA}, {0xF93B, 0x788C}, {0xF93C, 0x797F}, {0xF93D, 0x7DA0}, {0xF93E, 0x83C9}, {0xF93F, 0x9304}, -{0xF940, 0x9E7F}, {0xF941, 0x8AD6}, {0xF942, 0x58DF}, {0xF943, 0x5F04}, {0xF944, 0x7C60}, {0xF945, 0x807E}, {0xF946, 0x7262}, {0xF947, 0x78CA}, {0xF948, 0x8CC2}, {0xF949, 0x96F7}, {0xF94A, 0x58D8}, -{0xF94B, 0x5C62}, {0xF94C, 0x6A13}, {0xF94D, 0x6DDA}, {0xF94E, 0x6F0F}, {0xF94F, 0x7D2F}, {0xF950, 0x7E37}, {0xF951, 0x964B}, {0xF952, 0x52D2}, {0xF953, 0x808B}, {0xF954, 0x51DC}, {0xF955, 0x51CC}, -{0xF956, 0x7A1C}, {0xF957, 0x7DBE}, {0xF958, 0x83F1}, {0xF959, 0x9675}, {0xF95A, 0x8B80}, {0xF95B, 0x62CF}, {0xF95C, 0x6A02}, {0xF95D, 0x8AFE}, {0xF95E, 0x4E39}, {0xF95F, 0x5BE7}, {0xF960, 0x6012}, -{0xF961, 0x7387}, {0xF962, 0x7570}, {0xF963, 0x5317}, {0xF964, 0x78FB}, {0xF965, 0x4FBF}, {0xF966, 0x5FA9}, {0xF967, 0x4E0D}, {0xF968, 0x6CCC}, {0xF969, 0x6578}, {0xF96A, 0x7D22}, {0xF96B, 0x53C3}, -{0xF96C, 0x585E}, {0xF96D, 0x7701}, {0xF96E, 0x8449}, {0xF96F, 0x8AAA}, {0xF970, 0x6BBA}, {0xF971, 0x8FB0}, {0xF972, 0x6C88}, {0xF973, 0x62FE}, {0xF974, 0x82E5}, {0xF975, 0x63A0}, {0xF976, 0x7565}, -{0xF977, 0x4EAE}, {0xF978, 0x5169}, {0xF979, 0x51C9}, {0xF97A, 0x6881}, {0xF97B, 0x7CE7}, {0xF97C, 0x826F}, {0xF97D, 0x8AD2}, {0xF97E, 0x91CF}, {0xF97F, 0x52F5}, {0xF980, 0x5442}, {0xF981, 0x5973}, -{0xF982, 0x5EEC}, {0xF983, 0x65C5}, {0xF984, 0x6FFE}, {0xF985, 0x792A}, {0xF986, 0x95AD}, {0xF987, 0x9A6A}, {0xF988, 0x9E97}, {0xF989, 0x9ECE}, {0xF98A, 0x529B}, {0xF98B, 0x66C6}, {0xF98C, 0x6B77}, -{0xF98D, 0x8F62}, {0xF98E, 0x5E74}, {0xF98F, 0x6190}, {0xF990, 0x6200}, {0xF991, 0x649A}, {0xF992, 0x6F23}, {0xF993, 0x7149}, {0xF994, 0x7489}, {0xF995, 0x79CA}, {0xF996, 0x7DF4}, {0xF997, 0x806F}, -{0xF998, 0x8F26}, {0xF999, 0x84EE}, {0xF99A, 0x9023}, {0xF99B, 0x934A}, {0xF99C, 0x5217}, {0xF99D, 0x52A3}, {0xF99E, 0x54BD}, {0xF99F, 0x70C8}, {0xF9A0, 0x88C2}, {0xF9A1, 0x8AAA}, {0xF9A2, 0x5EC9}, -{0xF9A3, 0x5FF5}, {0xF9A4, 0x637B}, {0xF9A5, 0x6BAE}, {0xF9A6, 0x7C3E}, {0xF9A7, 0x7375}, {0xF9A8, 0x4EE4}, {0xF9A9, 0x56F9}, {0xF9AA, 0x5BE7}, {0xF9AB, 0x5DBA}, {0xF9AC, 0x601C}, {0xF9AD, 0x73B2}, -{0xF9AE, 0x7469}, {0xF9AF, 0x7F9A}, {0xF9B0, 0x8046}, {0xF9B1, 0x9234}, {0xF9B2, 0x96F6}, {0xF9B3, 0x9748}, {0xF9B4, 0x9818}, {0xF9B5, 0x4F8B}, {0xF9B6, 0x79AE}, {0xF9B7, 0x91B4}, {0xF9B8, 0x96B8}, -{0xF9B9, 0x60E1}, {0xF9BA, 0x4E86}, {0xF9BB, 0x50DA}, {0xF9BC, 0x5BEE}, {0xF9BD, 0x5C3F}, {0xF9BE, 0x6599}, {0xF9BF, 0x6A02}, {0xF9C0, 0x71CE}, {0xF9C1, 0x7642}, {0xF9C2, 0x84FC}, {0xF9C3, 0x907C}, -{0xF9C4, 0x9F8D}, {0xF9C5, 0x6688}, {0xF9C6, 0x962E}, {0xF9C7, 0x5289}, {0xF9C8, 0x677B}, {0xF9C9, 0x67F3}, {0xF9CA, 0x6D41}, {0xF9CB, 0x6E9C}, {0xF9CC, 0x7409}, {0xF9CD, 0x7559}, {0xF9CE, 0x786B}, -{0xF9CF, 0x7D10}, {0xF9D0, 0x985E}, {0xF9D1, 0x516D}, {0xF9D2, 0x622E}, {0xF9D3, 0x9678}, {0xF9D4, 0x502B}, {0xF9D5, 0x5D19}, {0xF9D6, 0x6DEA}, {0xF9D7, 0x8F2A}, {0xF9D8, 0x5F8B}, {0xF9D9, 0x6144}, -{0xF9DA, 0x6817}, {0xF9DB, 0x7387}, {0xF9DC, 0x9686}, {0xF9DD, 0x5229}, {0xF9DE, 0x540F}, {0xF9DF, 0x5C65}, {0xF9E0, 0x6613}, {0xF9E1, 0x674E}, {0xF9E2, 0x68A8}, {0xF9E3, 0x6CE5}, {0xF9E4, 0x7406}, -{0xF9E5, 0x75E2}, {0xF9E6, 0x7F79}, {0xF9E7, 0x88CF}, {0xF9E8, 0x88E1}, {0xF9E9, 0x91CC}, {0xF9EA, 0x96E2}, {0xF9EB, 0x533F}, {0xF9EC, 0x6EBA}, {0xF9ED, 0x541D}, {0xF9EE, 0x71D0}, {0xF9EF, 0x7498}, -{0xF9F0, 0x85FA}, {0xF9F1, 0x96A3}, {0xF9F2, 0x9C57}, {0xF9F3, 0x9E9F}, {0xF9F4, 0x6797}, {0xF9F5, 0x6DCB}, {0xF9F6, 0x81E8}, {0xF9F7, 0x7ACB}, {0xF9F8, 0x7B20}, {0xF9F9, 0x7C92}, {0xF9FA, 0x72C0}, -{0xF9FB, 0x7099}, {0xF9FC, 0x8B58}, {0xF9FD, 0x4EC0}, {0xF9FE, 0x8336}, {0xF9FF, 0x523A}, {0xFA00, 0x5207}, {0xFA01, 0x5EA6}, {0xFA02, 0x62D3}, {0xFA03, 0x7CD6}, {0xFA04, 0x5B85}, {0xFA05, 0x6D1E}, -{0xFA06, 0x66B4}, {0xFA07, 0x8F3B}, {0xFA08, 0x884C}, {0xFA09, 0x964D}, {0xFA0A, 0x898B}, {0xFA0B, 0x5ED3}, {0xFA0C, 0x5140}, {0xFA0D, 0x55C0}, {0xFA10, 0x585A}, {0xFA12, 0x6674}, {0xFA15, 0x51DE}, -{0xFA16, 0x732A}, {0xFA17, 0x76CA}, {0xFA18, 0x793C}, {0xFA19, 0x795E}, {0xFA1A, 0x7965}, {0xFA1B, 0x798F}, {0xFA1C, 0x9756}, {0xFA1D, 0x7CBE}, {0xFA1E, 0x7FBD}, {0xFA20, 0x8612}, {0xFA22, 0x8AF8}, -{0xFA25, 0x9038}, {0xFA26, 0x90FD}, {0xFA2A, 0x98EF}, {0xFA2B, 0x98FC}, {0xFA2C, 0x9928}, {0xFA2D, 0x9DB4}, {0xFA2E, 0x90DE}, {0xFA2F, 0x96B7}, {0xFA30, 0x4FAE}, {0xFA31, 0x50E7}, {0xFA32, 0x514D}, -{0xFA33, 0x52C9}, {0xFA34, 0x52E4}, {0xFA35, 0x5351}, {0xFA36, 0x559D}, {0xFA37, 0x5606}, {0xFA38, 0x5668}, {0xFA39, 0x5840}, {0xFA3A, 0x58A8}, {0xFA3B, 0x5C64}, {0xFA3C, 0x5C6E}, {0xFA3D, 0x6094}, -{0xFA3E, 0x6168}, {0xFA3F, 0x618E}, {0xFA40, 0x61F2}, {0xFA41, 0x654F}, {0xFA42, 0x65E2}, {0xFA43, 0x6691}, {0xFA44, 0x6885}, {0xFA45, 0x6D77}, {0xFA46, 0x6E1A}, {0xFA47, 0x6F22}, {0xFA48, 0x716E}, -{0xFA49, 0x722B}, {0xFA4A, 0x7422}, {0xFA4B, 0x7891}, {0xFA4C, 0x793E}, {0xFA4D, 0x7949}, {0xFA4E, 0x7948}, {0xFA4F, 0x7950}, {0xFA50, 0x7956}, {0xFA51, 0x795D}, {0xFA52, 0x798D}, {0xFA53, 0x798E}, -{0xFA54, 0x7A40}, {0xFA55, 0x7A81}, {0xFA56, 0x7BC0}, {0xFA57, 0x7DF4}, {0xFA58, 0x7E09}, {0xFA59, 0x7E41}, {0xFA5A, 0x7F72}, {0xFA5B, 0x8005}, {0xFA5C, 0x81ED}, {0xFA5D, 0x8279}, {0xFA5E, 0x8279}, -{0xFA5F, 0x8457}, {0xFA60, 0x8910}, {0xFA61, 0x8996}, {0xFA62, 0x8B01}, {0xFA63, 0x8B39}, {0xFA64, 0x8CD3}, {0xFA65, 0x8D08}, {0xFA66, 0x8FB6}, {0xFA67, 0x9038}, {0xFA68, 0x96E3}, {0xFA69, 0x97FF}, -{0xFA6A, 0x983B}, {0xFA6B, 0x6075}, {0xFA6C, 0x242EE}, {0xFA6D, 0x8218}, {0xFA70, 0x4E26}, {0xFA71, 0x51B5}, {0xFA72, 0x5168}, {0xFA73, 0x4F80}, {0xFA74, 0x5145}, {0xFA75, 0x5180}, {0xFA76, 0x52C7}, -{0xFA77, 0x52FA}, {0xFA78, 0x559D}, {0xFA79, 0x5555}, {0xFA7A, 0x5599}, {0xFA7B, 0x55E2}, {0xFA7C, 0x585A}, {0xFA7D, 0x58B3}, {0xFA7E, 0x5944}, {0xFA7F, 0x5954}, {0xFA80, 0x5A62}, {0xFA81, 0x5B28}, -{0xFA82, 0x5ED2}, {0xFA83, 0x5ED9}, {0xFA84, 0x5F69}, {0xFA85, 0x5FAD}, {0xFA86, 0x60D8}, {0xFA87, 0x614E}, {0xFA88, 0x6108}, {0xFA89, 0x618E}, {0xFA8A, 0x6160}, {0xFA8B, 0x61F2}, {0xFA8C, 0x6234}, -{0xFA8D, 0x63C4}, {0xFA8E, 0x641C}, {0xFA8F, 0x6452}, {0xFA90, 0x6556}, {0xFA91, 0x6674}, {0xFA92, 0x6717}, {0xFA93, 0x671B}, {0xFA94, 0x6756}, {0xFA95, 0x6B79}, {0xFA96, 0x6BBA}, {0xFA97, 0x6D41}, -{0xFA98, 0x6EDB}, {0xFA99, 0x6ECB}, {0xFA9A, 0x6F22}, {0xFA9B, 0x701E}, {0xFA9C, 0x716E}, {0xFA9D, 0x77A7}, {0xFA9E, 0x7235}, {0xFA9F, 0x72AF}, {0xFAA0, 0x732A}, {0xFAA1, 0x7471}, {0xFAA2, 0x7506}, -{0xFAA3, 0x753B}, {0xFAA4, 0x761D}, {0xFAA5, 0x761F}, {0xFAA6, 0x76CA}, {0xFAA7, 0x76DB}, {0xFAA8, 0x76F4}, {0xFAA9, 0x774A}, {0xFAAA, 0x7740}, {0xFAAB, 0x78CC}, {0xFAAC, 0x7AB1}, {0xFAAD, 0x7BC0}, -{0xFAAE, 0x7C7B}, {0xFAAF, 0x7D5B}, {0xFAB0, 0x7DF4}, {0xFAB1, 0x7F3E}, {0xFAB2, 0x8005}, {0xFAB3, 0x8352}, {0xFAB4, 0x83EF}, {0xFAB5, 0x8779}, {0xFAB6, 0x8941}, {0xFAB7, 0x8986}, {0xFAB8, 0x8996}, -{0xFAB9, 0x8ABF}, {0xFABA, 0x8AF8}, {0xFABB, 0x8ACB}, {0xFABC, 0x8B01}, {0xFABD, 0x8AFE}, {0xFABE, 0x8AED}, {0xFABF, 0x8B39}, {0xFAC0, 0x8B8A}, {0xFAC1, 0x8D08}, {0xFAC2, 0x8F38}, {0xFAC3, 0x9072}, -{0xFAC4, 0x9199}, {0xFAC5, 0x9276}, {0xFAC6, 0x967C}, {0xFAC7, 0x96E3}, {0xFAC8, 0x9756}, {0xFAC9, 0x97DB}, {0xFACA, 0x97FF}, {0xFACB, 0x980B}, {0xFACC, 0x983B}, {0xFACD, 0x9B12}, {0xFACE, 0x9F9C}, -{0xFACF, 0x2284A}, {0xFAD0, 0x22844}, {0xFAD1, 0x233D5}, {0xFAD2, 0x3B9D}, {0xFAD3, 0x4018}, {0xFAD4, 0x4039}, {0xFAD5, 0x25249}, {0xFAD6, 0x25CD0}, {0xFAD7, 0x27ED3}, {0xFAD8, 0x9F43}, -{0xFAD9, 0x9F8E}, {0xFB1D, 0x5D9}, {0xFB1D, 0x5B4}, {0xFB1F, 0x5F2}, {0xFB1F, 0x5B7}, {0xFB2A, 0x5E9}, {0xFB2A, 0x5C1}, {0xFB2B, 0x5E9}, {0xFB2B, 0x5C2}, {0xFB2C, 0x5E9}, {0xFB2C, 0x5BC}, -{0xFB2C, 0x5C1}, {0xFB2D, 0x5E9}, {0xFB2D, 0x5BC}, {0xFB2D, 0x5C2}, {0xFB2E, 0x5D0}, {0xFB2E, 0x5B7}, {0xFB2F, 0x5D0}, {0xFB2F, 0x5B8}, {0xFB30, 0x5D0}, {0xFB30, 0x5BC}, {0xFB31, 0x5D1}, -{0xFB31, 0x5BC}, {0xFB32, 0x5D2}, {0xFB32, 0x5BC}, {0xFB33, 0x5D3}, {0xFB33, 0x5BC}, {0xFB34, 0x5D4}, {0xFB34, 0x5BC}, {0xFB35, 0x5D5}, {0xFB35, 0x5BC}, {0xFB36, 0x5D6}, {0xFB36, 0x5BC}, -{0xFB38, 0x5D8}, {0xFB38, 0x5BC}, {0xFB39, 0x5D9}, {0xFB39, 0x5BC}, {0xFB3A, 0x5DA}, {0xFB3A, 0x5BC}, {0xFB3B, 0x5DB}, {0xFB3B, 0x5BC}, {0xFB3C, 0x5DC}, {0xFB3C, 0x5BC}, {0xFB3E, 0x5DE}, -{0xFB3E, 0x5BC}, {0xFB40, 0x5E0}, {0xFB40, 0x5BC}, {0xFB41, 0x5E1}, {0xFB41, 0x5BC}, {0xFB43, 0x5E3}, {0xFB43, 0x5BC}, {0xFB44, 0x5E4}, {0xFB44, 0x5BC}, {0xFB46, 0x5E6}, {0xFB46, 0x5BC}, -{0xFB47, 0x5E7}, {0xFB47, 0x5BC}, {0xFB48, 0x5E8}, {0xFB48, 0x5BC}, {0xFB49, 0x5E9}, {0xFB49, 0x5BC}, {0xFB4A, 0x5EA}, {0xFB4A, 0x5BC}, {0xFB4B, 0x5D5}, {0xFB4B, 0x5B9}, {0xFB4C, 0x5D1}, -{0xFB4C, 0x5BF}, {0xFB4D, 0x5DB}, {0xFB4D, 0x5BF}, {0xFB4E, 0x5E4}, {0xFB4E, 0x5BF}, {0x1109A, 0x11099}, {0x1109A, 0x110BA}, {0x1109C, 0x1109B}, {0x1109C, 0x110BA}, {0x110AB, 0x110A5}, -{0x110AB, 0x110BA}, {0x1112E, 0x11131}, {0x1112E, 0x11127}, {0x1112F, 0x11132}, {0x1112F, 0x11127}, {0x1134B, 0x11347}, {0x1134B, 0x1133E}, {0x1134C, 0x11347}, {0x1134C, 0x11357}, {0x114BB, 0x114B9}, -{0x114BB, 0x114BA}, {0x114BC, 0x114B9}, {0x114BC, 0x114B0}, {0x114BE, 0x114B9}, {0x114BE, 0x114BD}, {0x115BA, 0x115B8}, {0x115BA, 0x115AF}, {0x115BB, 0x115B9}, {0x115BB, 0x115AF}, {0x1D15E, 0x1D157}, -{0x1D15E, 0x1D165}, {0x1D15F, 0x1D158}, {0x1D15F, 0x1D165}, {0x1D160, 0x1D158}, {0x1D160, 0x1D165}, {0x1D160, 0x1D16E}, {0x1D161, 0x1D158}, {0x1D161, 0x1D165}, {0x1D161, 0x1D16F}, {0x1D162, 0x1D158}, -{0x1D162, 0x1D165}, {0x1D162, 0x1D170}, {0x1D163, 0x1D158}, {0x1D163, 0x1D165}, {0x1D163, 0x1D171}, {0x1D164, 0x1D158}, {0x1D164, 0x1D165}, {0x1D164, 0x1D172}, {0x1D1BB, 0x1D1B9}, {0x1D1BB, 0x1D165}, -{0x1D1BC, 0x1D1BA}, {0x1D1BC, 0x1D165}, {0x1D1BD, 0x1D1B9}, {0x1D1BD, 0x1D165}, {0x1D1BD, 0x1D16E}, {0x1D1BE, 0x1D1BA}, {0x1D1BE, 0x1D165}, {0x1D1BE, 0x1D16E}, {0x1D1BF, 0x1D1B9}, {0x1D1BF, 0x1D165}, -{0x1D1BF, 0x1D16F}, {0x1D1C0, 0x1D1BA}, {0x1D1C0, 0x1D165}, {0x1D1C0, 0x1D16F}, {0x2F800, 0x4E3D}, {0x2F801, 0x4E38}, {0x2F802, 0x4E41}, {0x2F803, 0x20122}, {0x2F804, 0x4F60}, {0x2F805, 0x4FAE}, -{0x2F806, 0x4FBB}, {0x2F807, 0x5002}, {0x2F808, 0x507A}, {0x2F809, 0x5099}, {0x2F80A, 0x50E7}, {0x2F80B, 0x50CF}, {0x2F80C, 0x349E}, {0x2F80D, 0x2063A}, {0x2F80E, 0x514D}, {0x2F80F, 0x5154}, -{0x2F810, 0x5164}, {0x2F811, 0x5177}, {0x2F812, 0x2051C}, {0x2F813, 0x34B9}, {0x2F814, 0x5167}, {0x2F815, 0x518D}, {0x2F816, 0x2054B}, {0x2F817, 0x5197}, {0x2F818, 0x51A4}, {0x2F819, 0x4ECC}, -{0x2F81A, 0x51AC}, {0x2F81B, 0x51B5}, {0x2F81C, 0x291DF}, {0x2F81D, 0x51F5}, {0x2F81E, 0x5203}, {0x2F81F, 0x34DF}, {0x2F820, 0x523B}, {0x2F821, 0x5246}, {0x2F822, 0x5272}, {0x2F823, 0x5277}, -{0x2F824, 0x3515}, {0x2F825, 0x52C7}, {0x2F826, 0x52C9}, {0x2F827, 0x52E4}, {0x2F828, 0x52FA}, {0x2F829, 0x5305}, {0x2F82A, 0x5306}, {0x2F82B, 0x5317}, {0x2F82C, 0x5349}, {0x2F82D, 0x5351}, -{0x2F82E, 0x535A}, {0x2F82F, 0x5373}, {0x2F830, 0x537D}, {0x2F831, 0x537F}, {0x2F832, 0x537F}, {0x2F833, 0x537F}, {0x2F834, 0x20A2C}, {0x2F835, 0x7070}, {0x2F836, 0x53CA}, {0x2F837, 0x53DF}, -{0x2F838, 0x20B63}, {0x2F839, 0x53EB}, {0x2F83A, 0x53F1}, {0x2F83B, 0x5406}, {0x2F83C, 0x549E}, {0x2F83D, 0x5438}, {0x2F83E, 0x5448}, {0x2F83F, 0x5468}, {0x2F840, 0x54A2}, {0x2F841, 0x54F6}, -{0x2F842, 0x5510}, {0x2F843, 0x5553}, {0x2F844, 0x5563}, {0x2F845, 0x5584}, {0x2F846, 0x5584}, {0x2F847, 0x5599}, {0x2F848, 0x55AB}, {0x2F849, 0x55B3}, {0x2F84A, 0x55C2}, {0x2F84B, 0x5716}, -{0x2F84C, 0x5606}, {0x2F84D, 0x5717}, {0x2F84E, 0x5651}, {0x2F84F, 0x5674}, {0x2F850, 0x5207}, {0x2F851, 0x58EE}, {0x2F852, 0x57CE}, {0x2F853, 0x57F4}, {0x2F854, 0x580D}, {0x2F855, 0x578B}, -{0x2F856, 0x5832}, {0x2F857, 0x5831}, {0x2F858, 0x58AC}, {0x2F859, 0x214E4}, {0x2F85A, 0x58F2}, {0x2F85B, 0x58F7}, {0x2F85C, 0x5906}, {0x2F85D, 0x591A}, {0x2F85E, 0x5922}, {0x2F85F, 0x5962}, -{0x2F860, 0x216A8}, {0x2F861, 0x216EA}, {0x2F862, 0x59EC}, {0x2F863, 0x5A1B}, {0x2F864, 0x5A27}, {0x2F865, 0x59D8}, {0x2F866, 0x5A66}, {0x2F867, 0x36EE}, {0x2F868, 0x36FC}, {0x2F869, 0x5B08}, -{0x2F86A, 0x5B3E}, {0x2F86B, 0x5B3E}, {0x2F86C, 0x219C8}, {0x2F86D, 0x5BC3}, {0x2F86E, 0x5BD8}, {0x2F86F, 0x5BE7}, {0x2F870, 0x5BF3}, {0x2F871, 0x21B18}, {0x2F872, 0x5BFF}, {0x2F873, 0x5C06}, -{0x2F874, 0x5F53}, {0x2F875, 0x5C22}, {0x2F876, 0x3781}, {0x2F877, 0x5C60}, {0x2F878, 0x5C6E}, {0x2F879, 0x5CC0}, {0x2F87A, 0x5C8D}, {0x2F87B, 0x21DE4}, {0x2F87C, 0x5D43}, {0x2F87D, 0x21DE6}, -{0x2F87E, 0x5D6E}, {0x2F87F, 0x5D6B}, {0x2F880, 0x5D7C}, {0x2F881, 0x5DE1}, {0x2F882, 0x5DE2}, {0x2F883, 0x382F}, {0x2F884, 0x5DFD}, {0x2F885, 0x5E28}, {0x2F886, 0x5E3D}, {0x2F887, 0x5E69}, -{0x2F888, 0x3862}, {0x2F889, 0x22183}, {0x2F88A, 0x387C}, {0x2F88B, 0x5EB0}, {0x2F88C, 0x5EB3}, {0x2F88D, 0x5EB6}, {0x2F88E, 0x5ECA}, {0x2F88F, 0x2A392}, {0x2F890, 0x5EFE}, {0x2F891, 0x22331}, -{0x2F892, 0x22331}, {0x2F893, 0x8201}, {0x2F894, 0x5F22}, {0x2F895, 0x5F22}, {0x2F896, 0x38C7}, {0x2F897, 0x232B8}, {0x2F898, 0x261DA}, {0x2F899, 0x5F62}, {0x2F89A, 0x5F6B}, {0x2F89B, 0x38E3}, -{0x2F89C, 0x5F9A}, {0x2F89D, 0x5FCD}, {0x2F89E, 0x5FD7}, {0x2F89F, 0x5FF9}, {0x2F8A0, 0x6081}, {0x2F8A1, 0x393A}, {0x2F8A2, 0x391C}, {0x2F8A3, 0x6094}, {0x2F8A4, 0x226D4}, {0x2F8A5, 0x60C7}, -{0x2F8A6, 0x6148}, {0x2F8A7, 0x614C}, {0x2F8A8, 0x614E}, {0x2F8A9, 0x614C}, {0x2F8AA, 0x617A}, {0x2F8AB, 0x618E}, {0x2F8AC, 0x61B2}, {0x2F8AD, 0x61A4}, {0x2F8AE, 0x61AF}, {0x2F8AF, 0x61DE}, -{0x2F8B0, 0x61F2}, {0x2F8B1, 0x61F6}, {0x2F8B2, 0x6210}, {0x2F8B3, 0x621B}, {0x2F8B4, 0x625D}, {0x2F8B5, 0x62B1}, {0x2F8B6, 0x62D4}, {0x2F8B7, 0x6350}, {0x2F8B8, 0x22B0C}, {0x2F8B9, 0x633D}, -{0x2F8BA, 0x62FC}, {0x2F8BB, 0x6368}, {0x2F8BC, 0x6383}, {0x2F8BD, 0x63E4}, {0x2F8BE, 0x22BF1}, {0x2F8BF, 0x6422}, {0x2F8C0, 0x63C5}, {0x2F8C1, 0x63A9}, {0x2F8C2, 0x3A2E}, {0x2F8C3, 0x6469}, -{0x2F8C4, 0x647E}, {0x2F8C5, 0x649D}, {0x2F8C6, 0x6477}, {0x2F8C7, 0x3A6C}, {0x2F8C8, 0x654F}, {0x2F8C9, 0x656C}, {0x2F8CA, 0x2300A}, {0x2F8CB, 0x65E3}, {0x2F8CC, 0x66F8}, {0x2F8CD, 0x6649}, -{0x2F8CE, 0x3B19}, {0x2F8CF, 0x6691}, {0x2F8D0, 0x3B08}, {0x2F8D1, 0x3AE4}, {0x2F8D2, 0x5192}, {0x2F8D3, 0x5195}, {0x2F8D4, 0x6700}, {0x2F8D5, 0x669C}, {0x2F8D6, 0x80AD}, {0x2F8D7, 0x43D9}, -{0x2F8D8, 0x6717}, {0x2F8D9, 0x671B}, {0x2F8DA, 0x6721}, {0x2F8DB, 0x675E}, {0x2F8DC, 0x6753}, {0x2F8DD, 0x233C3}, {0x2F8DE, 0x3B49}, {0x2F8DF, 0x67FA}, {0x2F8E0, 0x6785}, {0x2F8E1, 0x6852}, -{0x2F8E2, 0x6885}, {0x2F8E3, 0x2346D}, {0x2F8E4, 0x688E}, {0x2F8E5, 0x681F}, {0x2F8E6, 0x6914}, {0x2F8E7, 0x3B9D}, {0x2F8E8, 0x6942}, {0x2F8E9, 0x69A3}, {0x2F8EA, 0x69EA}, {0x2F8EB, 0x6AA8}, -{0x2F8EC, 0x236A3}, {0x2F8ED, 0x6ADB}, {0x2F8EE, 0x3C18}, {0x2F8EF, 0x6B21}, {0x2F8F0, 0x238A7}, {0x2F8F1, 0x6B54}, {0x2F8F2, 0x3C4E}, {0x2F8F3, 0x6B72}, {0x2F8F4, 0x6B9F}, {0x2F8F5, 0x6BBA}, -{0x2F8F6, 0x6BBB}, {0x2F8F7, 0x23A8D}, {0x2F8F8, 0x21D0B}, {0x2F8F9, 0x23AFA}, {0x2F8FA, 0x6C4E}, {0x2F8FB, 0x23CBC}, {0x2F8FC, 0x6CBF}, {0x2F8FD, 0x6CCD}, {0x2F8FE, 0x6C67}, {0x2F8FF, 0x6D16}, -{0x2F900, 0x6D3E}, {0x2F901, 0x6D77}, {0x2F902, 0x6D41}, {0x2F903, 0x6D69}, {0x2F904, 0x6D78}, {0x2F905, 0x6D85}, {0x2F906, 0x23D1E}, {0x2F907, 0x6D34}, {0x2F908, 0x6E2F}, {0x2F909, 0x6E6E}, -{0x2F90A, 0x3D33}, {0x2F90B, 0x6ECB}, {0x2F90C, 0x6EC7}, {0x2F90D, 0x23ED1}, {0x2F90E, 0x6DF9}, {0x2F90F, 0x6F6E}, {0x2F910, 0x23F5E}, {0x2F911, 0x23F8E}, {0x2F912, 0x6FC6}, {0x2F913, 0x7039}, -{0x2F914, 0x701E}, {0x2F915, 0x701B}, {0x2F916, 0x3D96}, {0x2F917, 0x704A}, {0x2F918, 0x707D}, {0x2F919, 0x7077}, {0x2F91A, 0x70AD}, {0x2F91B, 0x20525}, {0x2F91C, 0x7145}, {0x2F91D, 0x24263}, -{0x2F91E, 0x719C}, {0x2F91F, 0x243AB}, {0x2F920, 0x7228}, {0x2F921, 0x7235}, {0x2F922, 0x7250}, {0x2F923, 0x24608}, {0x2F924, 0x7280}, {0x2F925, 0x7295}, {0x2F926, 0x24735}, {0x2F927, 0x24814}, -{0x2F928, 0x737A}, {0x2F929, 0x738B}, {0x2F92A, 0x3EAC}, {0x2F92B, 0x73A5}, {0x2F92C, 0x3EB8}, {0x2F92D, 0x3EB8}, {0x2F92E, 0x7447}, {0x2F92F, 0x745C}, {0x2F930, 0x7471}, {0x2F931, 0x7485}, -{0x2F932, 0x74CA}, {0x2F933, 0x3F1B}, {0x2F934, 0x7524}, {0x2F935, 0x24C36}, {0x2F936, 0x753E}, {0x2F937, 0x24C92}, {0x2F938, 0x7570}, {0x2F939, 0x2219F}, {0x2F93A, 0x7610}, {0x2F93B, 0x24FA1}, -{0x2F93C, 0x24FB8}, {0x2F93D, 0x25044}, {0x2F93E, 0x3FFC}, {0x2F93F, 0x4008}, {0x2F940, 0x76F4}, {0x2F941, 0x250F3}, {0x2F942, 0x250F2}, {0x2F943, 0x25119}, {0x2F944, 0x25133}, {0x2F945, 0x771E}, -{0x2F946, 0x771F}, {0x2F947, 0x771F}, {0x2F948, 0x774A}, {0x2F949, 0x4039}, {0x2F94A, 0x778B}, {0x2F94B, 0x4046}, {0x2F94C, 0x4096}, {0x2F94D, 0x2541D}, {0x2F94E, 0x784E}, {0x2F94F, 0x788C}, -{0x2F950, 0x78CC}, {0x2F951, 0x40E3}, {0x2F952, 0x25626}, {0x2F953, 0x7956}, {0x2F954, 0x2569A}, {0x2F955, 0x256C5}, {0x2F956, 0x798F}, {0x2F957, 0x79EB}, {0x2F958, 0x412F}, {0x2F959, 0x7A40}, -{0x2F95A, 0x7A4A}, {0x2F95B, 0x7A4F}, {0x2F95C, 0x2597C}, {0x2F95D, 0x25AA7}, {0x2F95E, 0x25AA7}, {0x2F95F, 0x7AEE}, {0x2F960, 0x4202}, {0x2F961, 0x25BAB}, {0x2F962, 0x7BC6}, {0x2F963, 0x7BC9}, -{0x2F964, 0x4227}, {0x2F965, 0x25C80}, {0x2F966, 0x7CD2}, {0x2F967, 0x42A0}, {0x2F968, 0x7CE8}, {0x2F969, 0x7CE3}, {0x2F96A, 0x7D00}, {0x2F96B, 0x25F86}, {0x2F96C, 0x7D63}, {0x2F96D, 0x4301}, -{0x2F96E, 0x7DC7}, {0x2F96F, 0x7E02}, {0x2F970, 0x7E45}, {0x2F971, 0x4334}, {0x2F972, 0x26228}, {0x2F973, 0x26247}, {0x2F974, 0x4359}, {0x2F975, 0x262D9}, {0x2F976, 0x7F7A}, {0x2F977, 0x2633E}, -{0x2F978, 0x7F95}, {0x2F979, 0x7FFA}, {0x2F97A, 0x8005}, {0x2F97B, 0x264DA}, {0x2F97C, 0x26523}, {0x2F97D, 0x8060}, {0x2F97E, 0x265A8}, {0x2F97F, 0x8070}, {0x2F980, 0x2335F}, {0x2F981, 0x43D5}, -{0x2F982, 0x80B2}, {0x2F983, 0x8103}, {0x2F984, 0x440B}, {0x2F985, 0x813E}, {0x2F986, 0x5AB5}, {0x2F987, 0x267A7}, {0x2F988, 0x267B5}, {0x2F989, 0x23393}, {0x2F98A, 0x2339C}, {0x2F98B, 0x8201}, -{0x2F98C, 0x8204}, {0x2F98D, 0x8F9E}, {0x2F98E, 0x446B}, {0x2F98F, 0x8291}, {0x2F990, 0x828B}, {0x2F991, 0x829D}, {0x2F992, 0x52B3}, {0x2F993, 0x82B1}, {0x2F994, 0x82B3}, {0x2F995, 0x82BD}, -{0x2F996, 0x82E6}, {0x2F997, 0x26B3C}, {0x2F998, 0x82E5}, {0x2F999, 0x831D}, {0x2F99A, 0x8363}, {0x2F99B, 0x83AD}, {0x2F99C, 0x8323}, {0x2F99D, 0x83BD}, {0x2F99E, 0x83E7}, {0x2F99F, 0x8457}, -{0x2F9A0, 0x8353}, {0x2F9A1, 0x83CA}, {0x2F9A2, 0x83CC}, {0x2F9A3, 0x83DC}, {0x2F9A4, 0x26C36}, {0x2F9A5, 0x26D6B}, {0x2F9A6, 0x26CD5}, {0x2F9A7, 0x452B}, {0x2F9A8, 0x84F1}, {0x2F9A9, 0x84F3}, -{0x2F9AA, 0x8516}, {0x2F9AB, 0x273CA}, {0x2F9AC, 0x8564}, {0x2F9AD, 0x26F2C}, {0x2F9AE, 0x455D}, {0x2F9AF, 0x4561}, {0x2F9B0, 0x26FB1}, {0x2F9B1, 0x270D2}, {0x2F9B2, 0x456B}, {0x2F9B3, 0x8650}, -{0x2F9B4, 0x865C}, {0x2F9B5, 0x8667}, {0x2F9B6, 0x8669}, {0x2F9B7, 0x86A9}, {0x2F9B8, 0x8688}, {0x2F9B9, 0x870E}, {0x2F9BA, 0x86E2}, {0x2F9BB, 0x8779}, {0x2F9BC, 0x8728}, {0x2F9BD, 0x876B}, -{0x2F9BE, 0x8786}, {0x2F9BF, 0x45D7}, {0x2F9C0, 0x87E1}, {0x2F9C1, 0x8801}, {0x2F9C2, 0x45F9}, {0x2F9C3, 0x8860}, {0x2F9C4, 0x8863}, {0x2F9C5, 0x27667}, {0x2F9C6, 0x88D7}, {0x2F9C7, 0x88DE}, -{0x2F9C8, 0x4635}, {0x2F9C9, 0x88FA}, {0x2F9CA, 0x34BB}, {0x2F9CB, 0x278AE}, {0x2F9CC, 0x27966}, {0x2F9CD, 0x46BE}, {0x2F9CE, 0x46C7}, {0x2F9CF, 0x8AA0}, {0x2F9D0, 0x8AED}, {0x2F9D1, 0x8B8A}, -{0x2F9D2, 0x8C55}, {0x2F9D3, 0x27CA8}, {0x2F9D4, 0x8CAB}, {0x2F9D5, 0x8CC1}, {0x2F9D6, 0x8D1B}, {0x2F9D7, 0x8D77}, {0x2F9D8, 0x27F2F}, {0x2F9D9, 0x20804}, {0x2F9DA, 0x8DCB}, {0x2F9DB, 0x8DBC}, -{0x2F9DC, 0x8DF0}, {0x2F9DD, 0x208DE}, {0x2F9DE, 0x8ED4}, {0x2F9DF, 0x8F38}, {0x2F9E0, 0x285D2}, {0x2F9E1, 0x285ED}, {0x2F9E2, 0x9094}, {0x2F9E3, 0x90F1}, {0x2F9E4, 0x9111}, {0x2F9E5, 0x2872E}, -{0x2F9E6, 0x911B}, {0x2F9E7, 0x9238}, {0x2F9E8, 0x92D7}, {0x2F9E9, 0x92D8}, {0x2F9EA, 0x927C}, {0x2F9EB, 0x93F9}, {0x2F9EC, 0x9415}, {0x2F9ED, 0x28BFA}, {0x2F9EE, 0x958B}, {0x2F9EF, 0x4995}, -{0x2F9F0, 0x95B7}, {0x2F9F1, 0x28D77}, {0x2F9F2, 0x49E6}, {0x2F9F3, 0x96C3}, {0x2F9F4, 0x5DB2}, {0x2F9F5, 0x9723}, {0x2F9F6, 0x29145}, {0x2F9F7, 0x2921A}, {0x2F9F8, 0x4A6E}, {0x2F9F9, 0x4A76}, -{0x2F9FA, 0x97E0}, {0x2F9FB, 0x2940A}, {0x2F9FC, 0x4AB2}, {0x2F9FD, 0x29496}, {0x2F9FE, 0x980B}, {0x2F9FF, 0x980B}, {0x2FA00, 0x9829}, {0x2FA01, 0x295B6}, {0x2FA02, 0x98E2}, {0x2FA03, 0x4B33}, -{0x2FA04, 0x9929}, {0x2FA05, 0x99A7}, {0x2FA06, 0x99C2}, {0x2FA07, 0x99FE}, {0x2FA08, 0x4BCE}, {0x2FA09, 0x29B30}, {0x2FA0A, 0x9B12}, {0x2FA0B, 0x9C40}, {0x2FA0C, 0x9CFD}, {0x2FA0D, 0x4CCE}, -{0x2FA0E, 0x4CED}, {0x2FA0F, 0x9D67}, {0x2FA10, 0x2A0CE}, {0x2FA11, 0x4CF8}, {0x2FA12, 0x2A105}, {0x2FA13, 0x2A20E}, {0x2FA14, 0x2A291}, {0x2FA15, 0x9EBB}, {0x2FA16, 0x4D56}, {0x2FA17, 0x9EF9}, -{0x2FA18, 0x9EFE}, {0x2FA19, 0x9F05}, {0x2FA1A, 0x9F0F}, {0x2FA1B, 0x9F16}, {0x2FA1D, 0x2A600}, -}; - -static std::string codepoint_to_utf8(uint32_t cp) { - std::string result; - if (/* 0x00 <= cp && */ cp <= 0x7f) { - result.push_back(cp); - } - else if (0x80 <= cp && cp <= 0x7ff) { - result.push_back(0xc0 | ((cp >> 6) & 0x1f)); - result.push_back(0x80 | (cp & 0x3f)); - } - else if (0x800 <= cp && cp <= 0xffff) { - result.push_back(0xe0 | ((cp >> 12) & 0x0f)); - result.push_back(0x80 | ((cp >> 6) & 0x3f)); - result.push_back(0x80 | (cp & 0x3f)); - } - else if (0x10000 <= cp && cp <= 0x10ffff) { - result.push_back(0xf0 | ((cp >> 18) & 0x07)); - result.push_back(0x80 | ((cp >> 12) & 0x3f)); - result.push_back(0x80 | ((cp >> 6) & 0x3f)); - result.push_back(0x80 | (cp & 0x3f)); - } - else { - throw std::invalid_argument("invalid codepoint"); - } - return result; -} - -static std::string codepoints_to_utf8(const std::vector & cps) { - std::string result; - for (size_t i = 0; i < cps.size(); ++i) { - result.append(codepoint_to_utf8(cps[i])); - } - return result; -} - -static uint32_t codepoint_from_utf8(const std::string & utf8, size_t & offset) { - assert(offset < utf8.size()); - if (!(utf8[offset + 0] & 0x80)) { - auto result = utf8[offset + 0]; - offset += 1; - return result; - } - if (!(utf8[offset + 0] & 0x40)) { - throw std::invalid_argument("invalid character"); - } - if (!(utf8[offset + 0] & 0x20)) { - if (offset + 1 >= utf8.size() || ! ((utf8[offset + 1] & 0xc0) == 0x80)) { - throw std::invalid_argument("invalid character"); - } - auto result = ((utf8[offset + 0] & 0x1f) << 6) | (utf8[offset + 1] & 0x3f); - offset += 2; - return result; - } - if (!(utf8[offset + 0] & 0x10)) { - if (offset + 2 >= utf8.size() || ! ((utf8[offset + 1] & 0xc0) == 0x80) || ! ((utf8[offset + 2] & 0xc0) == 0x80)) { - throw std::invalid_argument("invalid character"); - } - auto result = ((utf8[offset + 0] & 0x0f) << 12) | ((utf8[offset + 1] & 0x3f) << 6) | (utf8[offset + 2] & 0x3f); - offset += 3; - return result; - } - if (!(utf8[offset + 0] & 0x08)) { - if (offset + 3 >= utf8.size() || ! ((utf8[offset + 1] & 0xc0) == 0x80) || ! ((utf8[offset + 2] & 0xc0) == 0x80) || !((utf8[offset + 3] & 0xc0) == 0x80)) { - throw std::invalid_argument("invalid character"); - } - auto result = ((utf8[offset + 0] & 0x07) << 18) | ((utf8[offset + 1] & 0x3f) << 12) | ((utf8[offset + 2] & 0x3f) << 6) | (utf8[offset + 3] & 0x3f); - offset += 4; - return result; - } - throw std::invalid_argument("invalid string"); -} - -static std::vector codepoints_from_utf8(const std::string & utf8) { - std::vector result; - size_t offset = 0; - while (offset < utf8.size()) { - result.push_back(codepoint_from_utf8(utf8, offset)); - } - return result; -} - -static std::vector codepoint_to_utf16(uint32_t cp) { - std::vector result; - if (/* 0x0000 <= cp && */ cp <= 0xffff) { - result.emplace_back(cp); - } - else if (0x10000 <= cp && cp <= 0x10ffff) { - result.emplace_back(0xd800 | ((cp - 0x10000) >> 10)); - result.emplace_back(0xdc00 | ((cp - 0x10000) & 0x03ff)); - } - else { - throw std::invalid_argument("invalid codepoint"); - } - return result; -} - -static std::vector codepoints_to_utf16(const std::vector & cps) { - std::vector result; - for (size_t i = 0; i < cps.size(); ++i) { - auto temp = codepoint_to_utf16(cps[i]); - result.insert(result.end(), temp.begin(), temp.end()); - } - return result; -} - -static uint32_t codepoint_from_utf16(const std::vector & utf16, size_t & offset) { - assert(offset < utf16.size()); - if (((utf16[0] >> 10) << 10) != 0xd800) { - auto result = utf16[offset + 0]; - offset += 1; - return result; - } - - if (offset + 1 >= utf16.size() || !((utf16[1] & 0xdc00) == 0xdc00)) { - throw std::invalid_argument("invalid character"); - } - - auto result = 0x10000 + (((utf16[0] & 0x03ff) << 10) | (utf16[1] & 0x03ff)); - offset += 2; - return result; -} - -static std::vector codepoints_from_utf16(const std::vector & utf16) { - std::vector result; - size_t offset = 0; - while (offset < utf16.size()) { - result.push_back(codepoint_from_utf16(utf16, offset)); - } - return result; -} - #define CODEPOINT_TYPE_UNIDENTIFIED 0 -#define CODEPOINT_TYPE_DIGIT 1 -#define CODEPOINT_TYPE_LETTER 2 -#define CODEPOINT_TYPE_WHITESPACE 3 -#define CODEPOINT_TYPE_ACCENT_MARK 4 -#define CODEPOINT_TYPE_PUNCTUATION 5 -#define CODEPOINT_TYPE_SYMBOL 6 -#define CODEPOINT_TYPE_CONTROL 7 +#define CODEPOINT_TYPE_DIGIT 1 +#define CODEPOINT_TYPE_LETTER 2 +#define CODEPOINT_TYPE_WHITESPACE 3 +#define CODEPOINT_TYPE_ACCENT_MARK 4 +#define CODEPOINT_TYPE_PUNCTUATION 5 +#define CODEPOINT_TYPE_SYMBOL 6 +#define CODEPOINT_TYPE_CONTROL 7 -static std::unordered_map codepoint_type_map() { - std::unordered_map codepoint_types; - for (auto p : digit_ranges) { - for (auto i = p.first; i <= p.second; ++ i) { - codepoint_types[i] = CODEPOINT_TYPE_DIGIT; - } - } - for (auto p : letter_ranges) { - for (auto i = p.first; i <= p.second; ++ i) { - codepoint_types[i] = CODEPOINT_TYPE_LETTER; - } - } - for (auto p : whitespace_ranges) { - for (auto i = p.first; i <= p.second; ++ i) { - codepoint_types[i] = CODEPOINT_TYPE_WHITESPACE; - } - } - for (auto p : accent_mark_ranges) { - for (auto i = p.first; i <= p.second; ++ i) { - codepoint_types[i] = CODEPOINT_TYPE_ACCENT_MARK; - } - } - for (auto p : punctuation_ranges) { - for (auto i = p.first; i <= p.second; ++ i) { - codepoint_types[i] = CODEPOINT_TYPE_PUNCTUATION; - } - } - for (auto p : symbol_ranges) { - for (auto i = p.first; i <= p.second; ++i) { - codepoint_types[i] = CODEPOINT_TYPE_SYMBOL; - } - } - for (auto p : control_ranges) { - for (auto i = p.first; i <= p.second; ++ i) { - codepoint_types[i] = CODEPOINT_TYPE_CONTROL; - } - } - return codepoint_types; -} +std::string unicode_cpt_to_utf8(uint32_t cp); +std::vector unicode_cpts_from_utf8(const std::string & utf8); -static int codepoint_type(uint32_t cp) { - static std::unordered_map codepoint_types = codepoint_type_map(); - const auto it = codepoint_types.find(cp); - return it == codepoint_types.end() ? CODEPOINT_TYPE_UNIDENTIFIED : it->second; -} +std::vector unicode_cpts_normalize_nfd(const std::vector & cpts); -static int codepoint_type(const std::string & utf8) { - if (utf8.length() == 0) { - return CODEPOINT_TYPE_UNIDENTIFIED; - } - size_t offset = 0; - return codepoint_type(codepoint_from_utf8(utf8, offset)); -} +int unicode_cpt_type(uint32_t cp); +int unicode_cpt_type(const std::string & utf8); -static std::unordered_map bytes_to_unicode_map_bpe() { - std::unordered_map map; - for (int ch = u'!'; ch <= u'~'; ++ch) { - assert(0 <= ch && ch < 256); - map[ch] = codepoint_to_utf8(ch); - } - for (int ch = u'¡'; ch <= u'¬'; ++ch) { - assert(0 <= ch && ch < 256); - map[ch] = codepoint_to_utf8(ch); - } - for (int ch = u'®'; ch <= u'ÿ'; ++ch) { - assert(0 <= ch && ch < 256); - map[ch] = codepoint_to_utf8(ch); - } - auto n = 0; - for (int ch = 0; ch < 256; ++ch) { - if (map.find(ch) == map.end()) { - map[ch] = codepoint_to_utf8(256 + n); - ++n; - } - } - return map; -} - -static std::string bytes_to_unicode_bpe(uint8_t byte) { - static std::unordered_map map = bytes_to_unicode_map_bpe(); - return map.at(byte); -} - -static std::unordered_map unicode_to_bytes_map_bpe() { - std::unordered_map map; - for (int ch = u'!'; ch <= u'~'; ++ch) { - assert(0 <= ch && ch < 256); - map[codepoint_to_utf8(ch)] = ch; - } - for (int ch = u'¡'; ch <= u'¬'; ++ch) { - assert(0 <= ch && ch < 256); - map[codepoint_to_utf8(ch)] = ch; - } - for (int ch = u'®'; ch <= u'ÿ'; ++ch) { - assert(0 <= ch && ch < 256); - map[codepoint_to_utf8(ch)] = ch; - } - auto n = 0; - for (int ch = 0; ch < 256; ++ch) { - if (map.find(codepoint_to_utf8(ch)) == map.end()) { - map[codepoint_to_utf8(256 + n)] = ch; - ++n; - } - } - return map; -} - -static uint8_t unicode_to_bytes_bpe(const std::string & utf8) { - static std::unordered_map map = unicode_to_bytes_map_bpe(); - return map.at(utf8); -} +std::string unicode_byte_to_utf8(uint8_t byte); +uint8_t unicode_utf8_to_byte(const std::string & utf8); From 05b06210c954491cf0f12034b0a62bd4d69ce78b Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Mon, 11 Mar 2024 17:49:47 +0200 Subject: [PATCH 599/812] llama : more consistent names of count variables (#5994) * llama : more consistent names of count variables ggml-ci * llama : n_parallel -> n_seq_max * common : fix param name * examples : fix param name --- README.md | 2 +- common/common.cpp | 18 +++++++++--------- examples/batched-bench/batched-bench.cpp | 2 +- examples/batched/batched.cpp | 2 +- examples/main/main.cpp | 1 + examples/perplexity/perplexity.cpp | 6 +++--- llama.cpp | 24 ++++++++++++------------ llama.h | 14 +++++++------- 8 files changed, 35 insertions(+), 34 deletions(-) diff --git a/README.md b/README.md index 98fdc6808..54bf84bec 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ Inference of Meta's [LLaMA](https://arxiv.org/abs/2302.13971) model (and others) ### Recent API changes -- [2024 Mar 8] `llama_kv_cache_seq_rm()` returns a `bool` instead of `void`, and new `llama_n_max_seq()` returns the upper limit of acceptable `seq_id` in batches (relevant when dealing with multiple sequences) https://github.com/ggerganov/llama.cpp/pull/5328 +- [2024 Mar 8] `llama_kv_cache_seq_rm()` returns a `bool` instead of `void`, and new `llama_n_seq_max()` returns the upper limit of acceptable `seq_id` in batches (relevant when dealing with multiple sequences) https://github.com/ggerganov/llama.cpp/pull/5328 - [2024 Mar 4] Embeddings API updated https://github.com/ggerganov/llama.cpp/pull/5796 - [2024 Mar 3] `struct llama_context_params` https://github.com/ggerganov/llama.cpp/pull/5849 diff --git a/common/common.cpp b/common/common.cpp index 16ef4d7f7..2f38ac632 100644 --- a/common/common.cpp +++ b/common/common.cpp @@ -1288,7 +1288,7 @@ struct llama_context_params llama_context_params_from_gpt_params(const gpt_param cparams.n_ctx = params.n_ctx; cparams.n_batch = params.n_batch; - cparams.n_parallel = params.n_parallel; + cparams.n_seq_max = params.n_parallel; cparams.n_threads = params.n_threads; cparams.n_threads_batch = params.n_threads_batch == -1 ? params.n_threads : params.n_threads_batch; cparams.seed = params.seed; @@ -1786,17 +1786,17 @@ void dump_kv_cache_view(const llama_kv_cache_view & view, int row_size) { static const char slot_chars[] = ".123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz+"; printf("=== Dumping KV cache. total cells %d, max sequences per cell %d, populated cells %d, total tokens in cache %d, largest empty slot=%d @ %d", - view.n_cells, view.n_max_seq, view.used_cells, view.token_count, view.max_contiguous, view.max_contiguous_idx); + view.n_cells, view.n_seq_max, view.used_cells, view.token_count, view.max_contiguous, view.max_contiguous_idx); llama_kv_cache_view_cell * c_curr = view.cells; llama_seq_id * cs_curr = view.cells_sequences; - for (int i = 0; i < view.n_cells; i++, c_curr++, cs_curr += view.n_max_seq) { + for (int i = 0; i < view.n_cells; i++, c_curr++, cs_curr += view.n_seq_max) { if (i % row_size == 0) { printf("\n%5d: ", i); } int seq_count = 0; - for (int j = 0; j < view.n_max_seq; j++) { + for (int j = 0; j < view.n_seq_max; j++) { if (cs_curr[j] >= 0) { seq_count++; } } putchar(slot_chars[std::min(sizeof(slot_chars) - 2, size_t(seq_count))]); @@ -1809,14 +1809,14 @@ void dump_kv_cache_view_seqs(const llama_kv_cache_view & view, int row_size) { static const char slot_chars[] = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"; printf("=== Dumping KV cache. total cells %d, max sequences per cell %d, populated cells %d, total tokens in cache %d, largest empty slot=%d @ %d\n", - view.n_cells, view.n_max_seq, view.used_cells, view.token_count, view.max_contiguous, view.max_contiguous_idx); + view.n_cells, view.n_seq_max, view.used_cells, view.token_count, view.max_contiguous, view.max_contiguous_idx); std::unordered_map seqs; llama_kv_cache_view_cell * c_curr = view.cells; llama_seq_id * cs_curr = view.cells_sequences; - for (int i = 0; i < view.n_cells; i++, c_curr++, cs_curr += view.n_max_seq) { - for (int j = 0; j < view.n_max_seq; j++) { + for (int i = 0; i < view.n_cells; i++, c_curr++, cs_curr += view.n_seq_max) { + for (int j = 0; j < view.n_seq_max; j++) { if (cs_curr[j] < 0) { continue; } if (seqs.find(cs_curr[j]) == seqs.end()) { if (seqs.size() + 1 >= sizeof(slot_chars)) { break; } @@ -1835,11 +1835,11 @@ void dump_kv_cache_view_seqs(const llama_kv_cache_view & view, int row_size) { c_curr = view.cells; cs_curr = view.cells_sequences; - for (int i = 0; i < view.n_cells; i++, c_curr++, cs_curr += view.n_max_seq) { + for (int i = 0; i < view.n_cells; i++, c_curr++, cs_curr += view.n_seq_max) { if (i % row_size == 0) { printf("\n%5d: ", i); } - for (int j = 0; j < view.n_max_seq; j++) { + for (int j = 0; j < view.n_seq_max; j++) { if (cs_curr[j] >= 0) { const auto & it = seqs.find(cs_curr[j]); putchar(it != seqs.end() ? int(slot_chars[it->second]) : '+'); diff --git a/examples/batched-bench/batched-bench.cpp b/examples/batched-bench/batched-bench.cpp index dff6c68ec..22bc93bca 100644 --- a/examples/batched-bench/batched-bench.cpp +++ b/examples/batched-bench/batched-bench.cpp @@ -106,7 +106,7 @@ int main(int argc, char ** argv) { ctx_params.n_threads_batch = params.n_threads_batch == -1 ? params.n_threads : params.n_threads_batch; // ensure enough sequences are available - ctx_params.n_parallel = *std::max_element(n_pl.begin(), n_pl.end()); + ctx_params.n_seq_max = *std::max_element(n_pl.begin(), n_pl.end()); llama_context * ctx = llama_new_context_with_model(model, ctx_params); diff --git a/examples/batched/batched.cpp b/examples/batched/batched.cpp index dde4d5a06..ee1f8f1bf 100644 --- a/examples/batched/batched.cpp +++ b/examples/batched/batched.cpp @@ -80,7 +80,7 @@ int main(int argc, char ** argv) { ctx_params.seed = 1234; ctx_params.n_ctx = n_kv_req; ctx_params.n_batch = std::max(n_len, n_parallel); - ctx_params.n_parallel = n_parallel; + ctx_params.n_seq_max = n_parallel; ctx_params.n_threads = params.n_threads; ctx_params.n_threads_batch = params.n_threads_batch == -1 ? params.n_threads : params.n_threads_batch; diff --git a/examples/main/main.cpp b/examples/main/main.cpp index 47059e582..e2d07a631 100644 --- a/examples/main/main.cpp +++ b/examples/main/main.cpp @@ -878,6 +878,7 @@ int main(int argc, char ** argv) { const auto line_pfx = ::llama_tokenize(ctx, params.input_prefix, false, true); const auto line_inp = ::llama_tokenize(ctx, buffer, false, false); const auto line_sfx = ::llama_tokenize(ctx, params.input_suffix, false, true); + LOG("input tokens: %s\n", LOG_TOKENS_TOSTR_PRETTY(ctx, line_inp).c_str()); embd_inp.insert(embd_inp.end(), line_pfx.begin(), line_pfx.end()); diff --git a/examples/perplexity/perplexity.cpp b/examples/perplexity/perplexity.cpp index 293eb52c3..fdfc8f5dc 100644 --- a/examples/perplexity/perplexity.cpp +++ b/examples/perplexity/perplexity.cpp @@ -841,7 +841,7 @@ static void hellaswag_score(llama_context * ctx, const gpt_params & params) { const int n_batch = params.n_batch; const int max_tasks_per_batch = 32; - const int max_seq = std::min(4*max_tasks_per_batch, (int) llama_n_max_seq(ctx)); + const int max_seq = std::min(4*max_tasks_per_batch, (int) llama_n_seq_max(ctx)); llama_batch batch = llama_batch_init(n_ctx, 0, max_seq); @@ -1118,7 +1118,7 @@ static void winogrande_score(llama_context * ctx, const gpt_params & params) { const int n_batch = params.n_batch; const int max_tasks_per_batch = 128; - const int max_seq = std::min(2*max_tasks_per_batch, (int) llama_n_max_seq(ctx)); + const int max_seq = std::min(2*max_tasks_per_batch, (int) llama_n_seq_max(ctx)); llama_batch batch = llama_batch_init(n_ctx, 0, max_seq); @@ -1470,7 +1470,7 @@ static void multiple_choice_score(llama_context * ctx, const gpt_params & params const int n_batch = params.n_batch; const int max_tasks_per_batch = 32; - const int max_seq = std::min(4*max_tasks_per_batch, (int) llama_n_max_seq(ctx)); + const int max_seq = std::min(4*max_tasks_per_batch, (int) llama_n_seq_max(ctx)); llama_batch batch = llama_batch_init(n_ctx, 0, max_seq); diff --git a/llama.cpp b/llama.cpp index 6aa88ce54..98ec147ae 100644 --- a/llama.cpp +++ b/llama.cpp @@ -12538,7 +12538,7 @@ struct llama_context_params llama_context_default_params() { /*.seed =*/ LLAMA_DEFAULT_SEED, /*.n_ctx =*/ 512, /*.n_batch =*/ 512, - /*.n_parallel =*/ 1, + /*.n_seq_max =*/ 1, /*.n_threads =*/ GGML_DEFAULT_N_THREADS, // TODO: better default /*.n_threads_batch =*/ GGML_DEFAULT_N_THREADS, /*.rope_scaling_type =*/ LLAMA_ROPE_SCALING_TYPE_UNSPECIFIED, @@ -12700,7 +12700,7 @@ struct llama_context * llama_new_context_with_model( auto & cparams = ctx->cparams; cparams.n_batch = params.n_batch; - // TODO: maybe add n_parallel here too + // TODO: maybe add n_seq_max here too cparams.n_threads = params.n_threads; cparams.n_threads_batch = params.n_threads_batch; cparams.yarn_ext_factor = params.yarn_ext_factor; @@ -12767,7 +12767,7 @@ struct llama_context * llama_new_context_with_model( // Mamba only needs a constant number of KV cache cells per sequence if (model->arch == LLM_ARCH_MAMBA) { // Mamba needs at least as many KV cells as there are sequences kept at any time - kv_size = std::max((uint32_t) 1, params.n_parallel); + kv_size = std::max((uint32_t) 1, params.n_seq_max); // it's probably best to keep as much precision as possible for the states type_k = GGML_TYPE_F32; // required by ggml_ssm_conv for Mamba's conv_states type_v = GGML_TYPE_F32; // required by ggml_ssm_scan for Mamba's ssm_states @@ -13024,7 +13024,7 @@ uint32_t llama_n_batch(const struct llama_context * ctx) { return ctx->cparams.n_batch; } -uint32_t llama_n_max_seq(const struct llama_context * ctx) { +uint32_t llama_n_seq_max(const struct llama_context * ctx) { return ctx->kv_self.size; } @@ -13188,10 +13188,10 @@ int32_t llama_model_apply_lora_from_file(const struct llama_model * model, const } } -struct llama_kv_cache_view llama_kv_cache_view_init(const struct llama_context * ctx, int32_t n_max_seq) { +struct llama_kv_cache_view llama_kv_cache_view_init(const struct llama_context * ctx, int32_t n_seq_max) { struct llama_kv_cache_view result = { /*.n_cells = */ 0, - /*.n_max_seq = */ n_max_seq, + /*.n_seq_max = */ n_seq_max, /*.token_count = */ 0, /*.used_cells = */ llama_get_kv_cache_used_cells(ctx), /*.max_contiguous = */ 0, @@ -13219,7 +13219,7 @@ void llama_kv_cache_view_update(const struct llama_context * ctx, struct llama_k void * p = realloc(view->cells, sizeof(struct llama_kv_cache_view_cell) * view->n_cells); GGML_ASSERT(p != nullptr && "Failed to alloc kv_cache_view cells"); view->cells = (struct llama_kv_cache_view_cell *)p; - p = realloc(view->cells_sequences, sizeof(llama_seq_id) * view->n_max_seq * view->n_cells); + p = realloc(view->cells_sequences, sizeof(llama_seq_id) * view->n_seq_max * view->n_cells); GGML_ASSERT(p != nullptr && "Failed to alloc kv_cache_view cells sequences"); view->cells_sequences = (llama_seq_id *)p; } @@ -13233,7 +13233,7 @@ void llama_kv_cache_view_update(const struct llama_context * ctx, struct llama_k uint32_t max_contig = 0; int32_t max_contig_idx = -1; - for (int32_t i = 0; i < int32_t(ctx->kv_self.size); i++, c_curr++, cs_curr += view->n_max_seq) { + for (int32_t i = 0; i < int32_t(ctx->kv_self.size); i++, c_curr++, cs_curr += view->n_seq_max) { const size_t curr_size = kv_cells[i].seq_id.size(); token_count += curr_size; c_curr->pos = kv_cells[i].pos + kv_cells[i].delta; @@ -13250,7 +13250,7 @@ void llama_kv_cache_view_update(const struct llama_context * ctx, struct llama_k int seq_idx = 0; for (const llama_seq_id it : kv_cells[i].seq_id) { - if (seq_idx >= view->n_max_seq) { + if (seq_idx >= view->n_seq_max) { break; } cs_curr[seq_idx] = it; @@ -13259,7 +13259,7 @@ void llama_kv_cache_view_update(const struct llama_context * ctx, struct llama_k if (seq_idx != 0) { used_cells++; } - for (; seq_idx < view->n_max_seq; seq_idx++) { + for (; seq_idx < view->n_seq_max; seq_idx++) { cs_curr[seq_idx] = -1; } } @@ -13921,12 +13921,12 @@ int32_t llama_tokenize( const char * text, int32_t text_len, llama_token * tokens, - int32_t n_max_tokens, + int32_t n_tokens_max, bool add_bos, bool special) { auto res = llama_tokenize_internal(model->vocab, std::string(text, text_len), add_bos, special); - if (n_max_tokens < (int) res.size()) { + if (n_tokens_max < (int) res.size()) { // LLAMA_LOG_ERROR("%s: too many tokens\n", __func__); return -((int) res.size()); } diff --git a/llama.h b/llama.h index ccf65ca4e..446899da6 100644 --- a/llama.h +++ b/llama.h @@ -235,7 +235,7 @@ extern "C" { uint32_t seed; // RNG seed, -1 for random uint32_t n_ctx; // text context, 0 = from model uint32_t n_batch; // prompt processing maximum batch size - uint32_t n_parallel; // number of parallel sequences (i.e. distinct states for recurrent models) + uint32_t n_seq_max; // max number of sequences (i.e. distinct states for recurrent models) uint32_t n_threads; // number of threads to use for generation uint32_t n_threads_batch; // number of threads to use for batch processing @@ -377,7 +377,7 @@ extern "C" { LLAMA_API uint32_t llama_n_ctx (const struct llama_context * ctx); LLAMA_API uint32_t llama_n_batch (const struct llama_context * ctx); - LLAMA_API uint32_t llama_n_max_seq (const struct llama_context * ctx); + LLAMA_API uint32_t llama_n_seq_max (const struct llama_context * ctx); LLAMA_API enum llama_vocab_type llama_vocab_type(const struct llama_model * model); LLAMA_API enum llama_rope_type llama_rope_type (const struct llama_model * model); @@ -456,7 +456,7 @@ extern "C" { // Maximum number of sequences that can exist in a cell. It's not an error // if there are more sequences in a cell than this value, however they will // not be visible in the view cells_sequences. - int32_t n_max_seq; + int32_t n_seq_max; // Number of tokens in the cache. For example, if there are two populated // cells, the first with 1 sequence id in it and the second with 2 sequence @@ -476,12 +476,12 @@ extern "C" { // Information for an individual cell. struct llama_kv_cache_view_cell * cells; - // The sequences for each cell. There will be n_max_seq items per cell. + // The sequences for each cell. There will be n_seq_max items per cell. llama_seq_id * cells_sequences; }; // Create an empty KV cache view. (use only for debugging purposes) - LLAMA_API struct llama_kv_cache_view llama_kv_cache_view_init(const struct llama_context * ctx, int32_t n_max_seq); + LLAMA_API struct llama_kv_cache_view llama_kv_cache_view_init(const struct llama_context * ctx, int32_t n_seq_max); // Free a KV cache view. (use only for debugging purposes) LLAMA_API void llama_kv_cache_view_free(struct llama_kv_cache_view * view); @@ -708,7 +708,7 @@ extern "C" { /// @details Convert the provided text into tokens. /// @param tokens The tokens pointer must be large enough to hold the resulting tokens. - /// @return Returns the number of tokens on success, no more than n_max_tokens + /// @return Returns the number of tokens on success, no more than n_tokens_max /// @return Returns a negative number on failure - the number of tokens that would have been returned /// @param special Allow tokenizing special and/or control tokens which otherwise are not exposed and treated as plaintext. /// Does not insert a leading space. @@ -717,7 +717,7 @@ extern "C" { const char * text, int32_t text_len, llama_token * tokens, - int32_t n_max_tokens, + int32_t n_tokens_max, bool add_bos, bool special); From 44ca159faf4fbe1a7ace13a962845ba7cdfd95ec Mon Sep 17 00:00:00 2001 From: Kawrakow <48489457+ikawrakow@users.noreply.github.com> Date: Mon, 11 Mar 2024 16:53:15 +0100 Subject: [PATCH 600/812] 1.5 bit: we can do even better (#5999) * iq1_s: we can do even better Spent one of the 4 scale bits on a signs of a 0.125 shift. I.e., quants are now -1 + delta, delta, 1 + delta, where delta is +/- 0.125. CUDA works, same performance as before. PPL(LLaMA-v2-7B) is now 11.85! * iq1_s: make scalar and AVX2 work with the new version * iq1_s: make Neon work with new version. ~10% drop in performance, so will need some more work. * iq1_s: make Metal work with new version * iq1_s: very slightly faster dequantize on Metal * iq1_s: fix dequantize on the CPU --------- Co-authored-by: Iwan Kawrakow --- ggml-common.h | 1 + ggml-cuda.cu | 36 ++++++++++----------- ggml-metal.metal | 18 ++++++----- ggml-quants.c | 83 ++++++++++++++++++++++++++++++++---------------- 4 files changed, 83 insertions(+), 55 deletions(-) diff --git a/ggml-common.h b/ggml-common.h index 5dd918081..2402b773c 100644 --- a/ggml-common.h +++ b/ggml-common.h @@ -645,6 +645,7 @@ GGML_TABLE_BEGIN(uint32_t, iq3s_grid, 512) GGML_TABLE_END() #define NGRID_IQ1S 2048 +#define IQ1S_DELTA 0.125f #if defined(GGML_COMMON_IMPL_C) GGML_TABLE_BEGIN(uint64_t, iq1s_grid, NGRID_IQ1S) 0xffffffffffffffff, 0xffffffffffffff01, 0xffffffffffff0000, 0xffffffffffff01ff, diff --git a/ggml-cuda.cu b/ggml-cuda.cu index d2945d3c2..01b1f15ee 100644 --- a/ggml-cuda.cu +++ b/ggml-cuda.cu @@ -1722,22 +1722,15 @@ static __global__ void dequantize_block_iq1_s(const void * __restrict__ vx, dst_ const int il = tid/8; // 0...3 const int ib = tid%8; // 0...7 dst_t * y = yy + i*QK_K + 32*ib + 8*il; - const float d = (float)x[i].d * (2*((x[i].qh[ib] >> 12) & 0xf) + 1); -#if __CUDA_ARCH__ >= MIN_CC_DP4A // lowest compute capability for integer intrinsics - int grid32[2]; const int8_t * q = (const int8_t *)grid32; - grid32[0] = *((const int *)(iq1s_grid_gpu + (x[i].qs[4*ib+il] | (((x[i].qh[ib] >> 3*il) & 7) << 8)))); - grid32[1] = __vsub4((grid32[0] >> 4) & 0x0f0f0f0f, 0x01010101); - grid32[0] = __vsub4(grid32[0] & 0x0f0f0f0f, 0x01010101); + const float delta = x[i].qh[ib] & 0x8000 ? -1 - IQ1S_DELTA : -1 + IQ1S_DELTA; + const float d = (float)x[i].d * (2*((x[i].qh[ib] >> 12) & 7) + 1); + uint32_t grid32[2]; const int8_t * q = (const int8_t *)grid32; + grid32[0] = iq1s_grid_gpu[x[i].qs[4*ib+il] | (((x[i].qh[ib] >> 3*il) & 7) << 8)]; + grid32[1] = (grid32[0] >> 4) & 0x0f0f0f0f; + grid32[0] &= 0x0f0f0f0f; for (int j = 0; j < 8; ++j) { - y[j] = d * q[j]; + y[j] = d * (q[j] + delta); } -#else - const uint8_t * grid = (const uint8_t *)(iq1s_grid_gpu + (x[i].qs[4*ib+il] | (((x[i].qh[ib] >> 3*il) & 7) << 8))); - for (int j = 0; j < 4; ++j) { - y[j+0] = d * ((grid[j] & 0xf) - 1); - y[j+4] = d * ((grid[j] >> 4) - 1); - } -#endif #else assert(false); #endif @@ -4560,22 +4553,25 @@ static __device__ __forceinline__ float vec_dot_iq1_s_q8_1( const int * q8 = (const int *)bq8_1[ib32].qs; for (int l = 0; l < 4; ++l) { const int * grid = (const int *)(iq1s_grid_gpu + (bq1->qs[4*ib32+l] | (((bq1->qh[ib32] >> 3*l) & 7) << 8))); - int grid0 = __vsub4(grid[0] & 0x0f0f0f0f, 0x01010101); - int grid1 = __vsub4((grid[0] >> 4) & 0x0f0f0f0f, 0x01010101); + int grid0 = grid[0] & 0x0f0f0f0f; + int grid1 = (grid[0] >> 4) & 0x0f0f0f0f; sumi = __dp4a(q8[2*l+1], grid1, __dp4a(q8[2*l+0], grid0, sumi)); } #else - const int8_t * q8 = bq8_1[ib32].qs; + const int8_t * q8 = bq8_1[ib32].qs; for (int l = 0; l < 4; ++l) { const uint8_t * grid = (const uint8_t *)(iq1s_grid_gpu + (bq1->qs[4*ib32+l] | (((bq1->qh[ib32] >> 3*l) & 7) << 8))); for (int j = 0; j < 4; ++j) { - sumi += q8[j] * ((grid[j] & 0xf) - 1) + q8[j+4] * ((grid[j] >> 4) - 1); + sumi += q8[j] * (grid[j] & 0xf) + q8[j+4] * (grid[j] >> 4); } q8 += 8; } #endif - const float d = (float)bq1->d * __low2float(bq8_1[ib32].ds); - return d * sumi * (2*(bq1->qh[ib32] >> 12) + 1); + const float delta = bq1->qh[ib32] & 0x8000 ? -1-IQ1S_DELTA : -1+IQ1S_DELTA; + const float d1q = (float)bq1->d * (2*((bq1->qh[ib32] >> 12) & 7) + 1); + const float d = d1q * __low2float (bq8_1[ib32].ds); + const float m = d1q * __high2float(bq8_1[ib32].ds); + return d * sumi + m * delta; #else assert(false); return 0.f; diff --git a/ggml-metal.metal b/ggml-metal.metal index 912822ee6..21298a8c2 100644 --- a/ggml-metal.metal +++ b/ggml-metal.metal @@ -4377,7 +4377,7 @@ void kernel_mul_mv_iq1_s_f32_impl( + yl[j+16] * (grid3[j] & 0xf) + yl[j+20] * (grid3[j] >> 4) + yl[j+24] * (grid4[j] & 0xf) + yl[j+28] * (grid4[j] >> 4); } - sumf[row] += (float)dh[0] * (sum - sumy) * (2*(qh[0] >> 12) + 1); + sumf[row] += (float)dh[0] * (sum + sumy * (qh[0] & 0x8000 ? -1 - IQ1S_DELTA : -1 + IQ1S_DELTA)) * (2*((qh[0] >> 12) & 7) + 1); dh += nb*sizeof(block_iq1_s)/2; qs += nb*sizeof(block_iq1_s); @@ -5076,14 +5076,16 @@ void dequantize_iq1_s(device const block_iq1_s * xb, short il, thread type4x4 & const float d = xb->d; device const uint8_t * qs = xb->qs + 4*ib32 + 2*il; device const uint16_t * qh = xb->qh; - const float dl = d * (2*(qh[ib32] >> 12) + 1); - constant uint8_t * grid1 = (constant uint8_t *)(iq1s_grid_gpu + (qs[0] | (((qh[ib32] >> (6*il+0)) & 7) << 8))); - constant uint8_t * grid2 = (constant uint8_t *)(iq1s_grid_gpu + (qs[1] | (((qh[ib32] >> (6*il+3)) & 7) << 8))); + const float dl = d * (2*((qh[ib32] >> 12) & 7) + 1); + const float ml = dl * (qh[ib32] & 0x8000 ? -1 - IQ1S_DELTA : -1 + IQ1S_DELTA); + const uint16_t h = qh[ib32] >> 6*il; + constant uint8_t * grid1 = (constant uint8_t *)(iq1s_grid_gpu + (qs[0] | ((h << 8) & 0x700))); + constant uint8_t * grid2 = (constant uint8_t *)(iq1s_grid_gpu + (qs[1] | ((h << 5) & 0x700))); for (int i = 0; i < 4; ++i) { - reg[0][i] = dl * (grid1[i] & 0xf) - dl; - reg[1][i] = dl * (grid1[i] >> 4) - dl; - reg[2][i] = dl * (grid2[i] & 0xf) - dl; - reg[3][i] = dl * (grid2[i] >> 4) - dl; + reg[0][i] = dl * (grid1[i] & 0xf) + ml; + reg[1][i] = dl * (grid1[i] >> 4) + ml; + reg[2][i] = dl * (grid2[i] & 0xf) + ml; + reg[3][i] = dl * (grid2[i] >> 4) + ml; } } diff --git a/ggml-quants.c b/ggml-quants.c index 86b0764cb..06665eb2d 100644 --- a/ggml-quants.c +++ b/ggml-quants.c @@ -3456,11 +3456,12 @@ void dequantize_row_iq1_s(const block_iq1_s * restrict x, float * restrict y, in const uint16_t * qh = x[i].qh; for (int ib = 0; ib < QK_K/32; ++ib) { - const float dl = d * (2*(qh[ib] >> 12) + 1); + const float dl = d * (2*((qh[ib] >> 12) & 7) + 1); + const float delta = qh[ib] & 0x8000 ? -IQ1S_DELTA : IQ1S_DELTA; for (int l = 0; l < 4; ++l) { const int8_t * grid = (const int8_t *)(iq1s_grid + (qs[l] | (((qh[ib] >> 3*l) & 7) << 8))); for (int j = 0; j < 8; ++j) { - y[j] = dl * grid[j]; + y[j] = dl * (grid[j] + delta); } y += 8; } @@ -9582,7 +9583,7 @@ void ggml_vec_dot_iq1_s_q8_K (int n, float * restrict s, size_t bs, const void const uint8_t * qs = x[i].qs; const uint16_t * qh = x[i].qh; - int sumi1 = 0, sumi2 = 0; + int sumi1 = 0, sumi2 = 0, sumi3 = 0; for (int ib = 0; ib < QK_K/32; ib += 2) { @@ -9601,12 +9602,16 @@ void ggml_vec_dot_iq1_s_q8_K (int n, float * restrict s, size_t bs, const void const int32x4_t p1 = ggml_vdotq_s32(ggml_vdotq_s32(vdupq_n_s32(0), q1b.val[0], q8b.val[0]), q1b.val[1], q8b.val[1]); const int32x4_t p2 = ggml_vdotq_s32(ggml_vdotq_s32(vdupq_n_s32(0), q1b.val[2], q8b.val[2]), q1b.val[3], q8b.val[3]); - sumi1 += vaddvq_s32(p1) * (2*(qh[ib+0] >> 12) + 1); - sumi2 += vaddvq_s32(p2) * (2*(qh[ib+1] >> 12) + 1); + const int ls1 = 2*((qh[ib+0] >> 12) & 7) + 1; + const int ls2 = 2*((qh[ib+1] >> 12) & 7) + 1; + sumi1 += vaddvq_s32(p1) * ls1; + sumi2 += vaddvq_s32(p2) * ls2; + sumi3 += (y[i].bsums[2*ib+0] + y[i].bsums[2*ib+1]) * ls1 * (qh[ib+0] & 0x8000 ? -1 : 1) + + (y[i].bsums[2*ib+2] + y[i].bsums[2*ib+3]) * ls2 * (qh[ib+1] & 0x8000 ? -1 : 1); } - sumf += y[i].d * GGML_FP16_TO_FP32(x[i].d) * (sumi1 + sumi2); + sumf += y[i].d * GGML_FP16_TO_FP32(x[i].d) * (sumi1 + sumi2 + IQ1S_DELTA * sumi3); } *s = sumf; @@ -9614,6 +9619,7 @@ void ggml_vec_dot_iq1_s_q8_K (int n, float * restrict s, size_t bs, const void #elif defined __AVX2__ __m256 accum = _mm256_setzero_ps(); + float accum1 = 0; for (int i = 0; i < nb; ++i) { const int8_t * q8 = y[i].qs; @@ -9621,6 +9627,7 @@ void ggml_vec_dot_iq1_s_q8_K (int n, float * restrict s, size_t bs, const void const uint16_t * qh = x[i].qh; __m256i sumi = _mm256_setzero_si256(); + int sumi1 = 0; for (int ib = 0; ib < QK_K/32; ib += 2) { const __m256i q1b_1 = _mm256_set_epi64x(iq1s_grid[qs[3] | ((qh[ib+0] >> 1) & 0x700)], iq1s_grid[qs[2] | ((qh[ib+0] << 2) & 0x700)], iq1s_grid[qs[1] | ((qh[ib+0] << 5) & 0x700)], iq1s_grid[qs[0] | ((qh[ib+0] << 8) & 0x700)]); @@ -9632,17 +9639,23 @@ void ggml_vec_dot_iq1_s_q8_K (int n, float * restrict s, size_t bs, const void const __m256i dot1 = mul_add_epi8(q1b_1, q8b_1); const __m256i dot2 = mul_add_epi8(q1b_2, q8b_2); - const __m256i p1 = _mm256_madd_epi16(dot1, _mm256_set1_epi16(2*(qh[ib+0] >> 12) + 1)); - const __m256i p2 = _mm256_madd_epi16(dot2, _mm256_set1_epi16(2*(qh[ib+1] >> 12) + 1)); + const int16_t ls1 = 2*((qh[ib+0] >> 12) & 7) + 1; + const int16_t ls2 = 2*((qh[ib+1] >> 12) & 7) + 1; + const __m256i p1 = _mm256_madd_epi16(dot1, _mm256_set1_epi16(ls1)); + const __m256i p2 = _mm256_madd_epi16(dot2, _mm256_set1_epi16(ls2)); sumi = _mm256_add_epi32(sumi, _mm256_add_epi32(p1, p2)); + sumi1 += (y[i].bsums[2*ib+0] + y[i].bsums[2*ib+1]) * (qh[ib+0] & 0x8000 ? -1 : 1) * ls1 + + (y[i].bsums[2*ib+2] + y[i].bsums[2*ib+3]) * (qh[ib+1] & 0x8000 ? -1 : 1) * ls2; } - accum = _mm256_fmadd_ps(_mm256_set1_ps(y[i].d * GGML_FP16_TO_FP32(x[i].d)), _mm256_cvtepi32_ps(sumi), accum); + const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d); + accum = _mm256_fmadd_ps(_mm256_set1_ps(d), _mm256_cvtepi32_ps(sumi), accum); + accum1 += d * sumi1; } - *s = hsum_float_8(accum); + *s = hsum_float_8(accum) + IQ1S_DELTA * accum1; #else @@ -9653,9 +9666,10 @@ void ggml_vec_dot_iq1_s_q8_K (int n, float * restrict s, size_t bs, const void const uint8_t * qs = x[i].qs; const uint16_t * qh = x[i].qh; - int sumi = 0; + int sumi = 0, sumi1 = 0; for (int ib = 0; ib < QK_K/32; ++ib) { - const int ls = 2*(qh[ib] >> 12) + 1; + const int ls = 2*((qh[ib] >> 12) & 7) + 1; + const int delta = qh[ib] & 0x8000 ? -1 : 1; int lsum = 0; for (int l = 0; l < 4; ++l) { const int8_t * grid = (const int8_t *)(iq1s_grid + (qs[l] | (((qh[ib] >> 3*l) & 7) << 8))); @@ -9664,11 +9678,12 @@ void ggml_vec_dot_iq1_s_q8_K (int n, float * restrict s, size_t bs, const void } q8 += 8; } - sumi += ls * lsum; + sumi += ls * lsum; + sumi1 += ls * delta * (y[i].bsums[2*ib+0] + y[i].bsums[2*ib+1]); qs += 4; } - sumf += GGML_FP16_TO_FP32(x[i].d) * y[i].d * sumi; + sumf += GGML_FP16_TO_FP32(x[i].d) * y[i].d * (sumi + IQ1S_DELTA * sumi1); } *s = sumf; @@ -11438,7 +11453,7 @@ static int iq1_find_best_neighbour(const uint16_t * restrict neighbours, const u } static int iq1_find_best_neighbour2(const uint16_t * restrict neighbours, const uint64_t * restrict grid, - const float * restrict xval, const float * restrict weight, float scale, int8_t * restrict L, int ngrid) { + const float * restrict xval, const float * restrict weight, float scale, const float * restrict xg, int8_t * restrict L, int ngrid) { int num_neighbors = neighbours[0]; GGML_ASSERT(num_neighbors > 0); float best_score = FLT_MAX; @@ -11447,7 +11462,7 @@ static int iq1_find_best_neighbour2(const uint16_t * restrict neighbours, const const int8_t * pg = (const int8_t *)(grid + neighbours[j]); float d2 = 0; for (int i = 0; i < 8; ++i) { - float q = (pg[i] - 3)/2; + float q = xg[(pg[i] - 1)/2]; float w = weight[i]; float diff = scale*q - xval[i]; d2 += w*diff*diff; @@ -11463,7 +11478,7 @@ static int iq1_find_best_neighbour2(const uint16_t * restrict neighbours, const float d2 = 0; for (int j = 0; j < 8; ++j) { float w = weight[j]; - float q = (grid_i[j] - 3)/2; + float q = xg[(grid_i[j] - 1)/2]; float diff = scale*q - xval[i]; d2 += w*diff*diff; } @@ -11480,7 +11495,7 @@ static int iq1_find_best_neighbour2(const uint16_t * restrict neighbours, const const int8_t * pg = (const int8_t *)(grid + neighbours[j]); float sumqx = 0, sumq2 = 0; for (int i = 0; i < 8; ++i) { - float q = (pg[i] - 3)/2; + float q = xg[(pg[i] - 1)/2]; float w = weight[i]; sumqx += w*q*xval[i]; sumq2 += w*q*q; @@ -11519,6 +11534,9 @@ static void quantize_row_iq1_s_impl(const float * restrict x, void * restrict vy block_iq1_s * y = vy; + const float x_p[3] = {-1 + IQ1S_DELTA, IQ1S_DELTA, 1 + IQ1S_DELTA}; + const float x_m[3] = {-1 - IQ1S_DELTA, -IQ1S_DELTA, 1 - IQ1S_DELTA}; + float scales[QK_K/IQ1S_BLOCK_SIZE]; float weight[IQ1S_BLOCK_SIZE]; int8_t L[IQ1S_BLOCK_SIZE]; @@ -11527,6 +11545,7 @@ static void quantize_row_iq1_s_impl(const float * restrict x, void * restrict vy float pairs[2*IQ1S_BLOCK_SIZE]; int * idx = (int *)(pairs + 1); uint16_t index[IQ1S_BLOCK_SIZE/8]; + int8_t shifts[QK_K/IQ1S_BLOCK_SIZE]; for (int ibl = 0; ibl < nbl; ++ibl) { @@ -11572,25 +11591,33 @@ static void quantize_row_iq1_s_impl(const float * restrict x, void * restrict vy } } float best_score = 0, scale = max; - int besti1 = 0, besti2 = 0; + int besti1 = -1, besti2 = -1, best_shift = 0; for (int i1 = 0; i1 <= IQ1S_BLOCK_SIZE; ++i1) { for (int i2 = i1; i2 <= IQ1S_BLOCK_SIZE; ++i2) { - float sumqx = -(sumx[i1] - sumx[0]) + (sumx[IQ1S_BLOCK_SIZE] - sumx[i2]); - float sumq2 = (sumw[i1] - sumw[0]) + (sumw[IQ1S_BLOCK_SIZE] - sumw[i2]); + float sumqx = (sumx[i1] - sumx[0])*x_p[0] + (sumx[i2] - sumx[i1])*x_p[1] + (sumx[IQ1S_BLOCK_SIZE] - sumx[i2])*x_p[2]; + float sumq2 = (sumw[i1] - sumw[0])*x_p[0]*x_p[0] + (sumw[i2] - sumw[i1])*x_p[1]*x_p[1] + (sumw[IQ1S_BLOCK_SIZE] - sumw[i2])*x_p[2]*x_p[2]; if (sumq2 > 0 && sumqx*sumqx > best_score*sumq2) { scale = sumqx/sumq2; best_score = scale*sumqx; - besti1 = i1; besti2 = i2; + besti1 = i1; besti2 = i2; best_shift = 1; + } + sumqx = (sumx[i1] - sumx[0])*x_m[0] + (sumx[i2] - sumx[i1])*x_m[1] + (sumx[IQ1S_BLOCK_SIZE] - sumx[i2])*x_m[2]; + sumq2 = (sumw[i1] - sumw[0])*x_m[0]*x_m[0] + (sumw[i2] - sumw[i1])*x_m[1]*x_m[1] + (sumw[IQ1S_BLOCK_SIZE] - sumw[i2])*x_m[2]*x_m[2]; + if (sumq2 > 0 && sumqx*sumqx > best_score*sumq2) { + scale = sumqx/sumq2; best_score = scale*sumqx; + besti1 = i1; besti2 = i2; best_shift = -1; } } } + GGML_ASSERT(besti1 >= 0 && besti2 >= 0 && best_shift != 0); for (int j = 0; j < besti1; ++j) L[idx[2*j]] = 0; for (int j = besti1; j < besti2; ++j) L[idx[2*j]] = 1; for (int j = besti2; j < IQ1S_BLOCK_SIZE; ++j) L[idx[2*j]] = 2; if (scale < 0) { for (int j = 0; j < IQ1S_BLOCK_SIZE; ++j) L[j] = 2 - L[j]; - scale = -scale; + scale = -scale; best_shift = -best_shift; } bool all_on_grid = true; + const float * xx = best_shift == 1 ? x_p : x_m; for (int k = 0; k < IQ1S_BLOCK_SIZE/8; ++k) { uint16_t u = 0; for (int j = 0; j < 8; ++j) u |= (L[8*k+j] << 2*j); @@ -11598,7 +11625,7 @@ static void quantize_row_iq1_s_impl(const float * restrict x, void * restrict vy if (grid_index < 0) { all_on_grid = false; const uint16_t * neighbours = kneighbors_q2xs - kmap_q2xs[u] - 1; - grid_index = iq1_find_best_neighbour2(neighbours, kgrid_q2xs, xb + 8*k, weight + 8*k, scale, L + 8*k, NGRID_IQ1S); + grid_index = iq1_find_best_neighbour2(neighbours, kgrid_q2xs, xb + 8*k, weight + 8*k, scale, xx, L + 8*k, NGRID_IQ1S); GGML_ASSERT(grid_index >= 0); } index[k] = grid_index; @@ -11609,7 +11636,7 @@ static void quantize_row_iq1_s_impl(const float * restrict x, void * restrict vy const int8_t * pg = (const int8_t *)(kgrid_q2xs + index[k]); for (int j = 0; j < 8; ++j) { float w = weight[8*k + j]; - float q = (pg[j] - 3)/2; + float q = xx[(pg[j] - 1)/2]; sumqx += w*q*xb[8*k+j]; sumq2 += w*q*q; } @@ -11624,6 +11651,7 @@ static void quantize_row_iq1_s_impl(const float * restrict x, void * restrict vy y[ibl].qh[ib] = h; GGML_ASSERT(scale >= 0); scales[ib] = scale; + shifts[ib] = best_shift; max_scale = MAX(max_scale, scale); } @@ -11632,12 +11660,13 @@ static void quantize_row_iq1_s_impl(const float * restrict x, void * restrict vy continue; } - float d = max_scale/31; + float d = max_scale/15; y[ibl].d = GGML_FP32_TO_FP16(d*1.125f); // 1.085f is another fudge factor. Don't ask me why it is needed. float id = 1/d; for (int ib = 0; ib < QK_K/IQ1S_BLOCK_SIZE; ++ib) { int l = nearest_int(0.5f*(id*scales[ib]-1)); - l = MAX(0, MIN(15, l)); + l = MAX(0, MIN(7, l)); + if (shifts[ib] == -1) l |= 8; y[ibl].qh[ib] |= (l << 12); } } From 5cdb371731caa2c41fcca42d4d2d43f94f6883b4 Mon Sep 17 00:00:00 2001 From: gliptic Date: Mon, 11 Mar 2024 20:59:03 +0100 Subject: [PATCH 601/812] grammar : fix unnecessarily retained pointer to rules (#6003) --- llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llama.cpp b/llama.cpp index 98ec147ae..ad7b7b7d4 100644 --- a/llama.cpp +++ b/llama.cpp @@ -10538,7 +10538,7 @@ struct llama_grammar * llama_grammar_init( // loop over alternates of start rule to build initial stacks std::vector> stacks; - pos = rules[start_rule_index]; + pos = vec_rules[start_rule_index].data(); do { std::vector stack; if (!llama_grammar_is_end_of_sequence(pos)) { From 48358b2e5b3983c41ba7e61a493e84d3901dc7b9 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Tue, 12 Mar 2024 11:15:05 +0200 Subject: [PATCH 602/812] sycl : update IQ1_S kernels (WIP - not working!) (#5995) * sycl : try to fix after IQ1_S changes * sycl : iq1s_grid -> iq1s_grid_gpu * sycl : fix grid type --- ggml-sycl.cpp | 64 ++++++++++++++++++++++++++------------------------- 1 file changed, 33 insertions(+), 31 deletions(-) diff --git a/ggml-sycl.cpp b/ggml-sycl.cpp index 6d5684582..24187e7d5 100644 --- a/ggml-sycl.cpp +++ b/ggml-sycl.cpp @@ -3514,8 +3514,8 @@ static_assert(sizeof(block_iq3_s) == sizeof(ggml_fp16_t) + 13*(QK_K/32) + IQ3S_N #define QI1_S (QK_K / (4*QR1_S)) typedef struct { sycl::half d; - uint8_t qs[QK_K/8]; - uint8_t scales[QK_K/16]; + uint8_t qs[QK_K/8]; + uint16_t qh[QK_K/32]; } block_iq1_s; static_assert(sizeof(block_iq1_s) == sizeof(ggml_fp16_t) + QK_K/8 + QK_K/16, "wrong iq1_s block size/padding"); @@ -4891,10 +4891,9 @@ static void dequantize_block_iq3_s(const void * __restrict__ vx, dst_t * __restr template static void dequantize_block_iq1_s(const void * __restrict__ vx, dst_t * __restrict__ yy, const sycl::nd_item<3> &item_ct1, - const uint64_t *iq1s_grid, + const uint32_t *iq1s_grid, const uint8_t *ksigns_iq2xs, const uint8_t *kmask_iq2xs) { - const int i = item_ct1.get_group(2); const block_iq1_s * x = (const block_iq1_s *) vx; @@ -4903,11 +4902,15 @@ static void dequantize_block_iq1_s(const void * __restrict__ vx, dst_t * __restr const int il = tid/8; // 0...3 const int ib = tid%8; // 0...7 dst_t * y = yy + i*QK_K + 32*ib + 8*il; - const int i8 = 4*ib+il; - uint8_t h = x[i].scales[i8/2] >> 4*(i8%2); - const int8_t * grid = (const int8_t *)(iq1s_grid + (x[i].qs[i8] | ((h & 8) << 5))); - const float d = (float)x[i].d * (2*(h & 7) + 1); - for (int j = 0; j < 8; ++j) y[j] = d * grid[j]; + const uint8_t * qs = x[i].qs + 8*ib; + const uint8_t * grid1 = (const uint8_t *)(iq1s_grid + qs[2*il+0]); + const uint8_t * grid2 = (const uint8_t *)(iq1s_grid + qs[2*il+1]); + const float d = (float)x[i].d * (2*((x[i].qh[ib] >> 12) & 0xf) + 1); + const uint8_t signs = ksigns_iq2xs[(x[i].qh[ib] >> 3*il) & 7]; + for (int j = 0; j < 4; ++j) { + y[j+0] = d * grid1[j] * (signs & kmask_iq2xs[j+0] ? -1.f : 1.f); + y[j+4] = d * grid2[j] * (signs & kmask_iq2xs[j+4] ? -1.f : 1.f); + } #else assert(false); #endif @@ -7803,28 +7806,27 @@ vec_dot_iq3_s_q8_1(const void *__restrict__ vbq, static __dpct_inline__ float vec_dot_iq1_s_q8_1(const void *__restrict__ vbq, const block_q8_1 *__restrict__ bq8_1, const int &iqs, - const uint64_t *iq1s_grid, const uint64_t *ksigns64) { + const uint32_t *iq1s_grid, const uint64_t *ksigns64) { #if QK_K == 256 const block_iq1_s * bq1 = (const block_iq1_s *) vbq; const int ib32 = iqs; - int sumi1 = 0, sumi2 = 0, sumi3 = 0, sumi4 = 0; - const uint8_t h1 = bq1->scales[2*ib32+0]; - const uint8_t h2 = bq1->scales[2*ib32+1]; - const int * q8 = (const int *)bq8_1[ib32].qs; - const int * grid1 = (const int *)(iq1s_grid + (bq1->qs[4*ib32+0] | ((h1 & 0x08) << 5))); - const int * grid2 = (const int *)(iq1s_grid + (bq1->qs[4*ib32+1] | ((h1 & 0x80) << 1))); - const int * grid3 = (const int *)(iq1s_grid + (bq1->qs[4*ib32+2] | ((h2 & 0x08) << 5))); - const int * grid4 = (const int *)(iq1s_grid + (bq1->qs[4*ib32+3] | ((h2 & 0x80) << 1))); - for (int j = 0; j < 2; ++j) { - sumi1 = dpct::dp4a(q8[j+0], grid1[j], sumi1); - sumi2 = dpct::dp4a(q8[j+2], grid2[j], sumi2); - sumi3 = dpct::dp4a(q8[j+4], grid3[j], sumi3); - sumi4 = dpct::dp4a(q8[j+6], grid4[j], sumi4); + const uint8_t * qs = bq1->qs + 4*ib32; + const int8_t * q8 = bq8_1[ib32].qs; + int sumi = 0; + for (int l = 0; l < 4; ++l) { + const uint32_t * grid = (const uint32_t *)(iq1s_grid + qs[l]); + const uint32_t * signs = (const uint32_t *)(ksigns64 + (qs[l] >> 8)); + const int grid_l = dpct::vectorized_binary( + grid[0] ^ signs[0], signs[0], std::minus<>()); + const int grid_h = dpct::vectorized_binary( + grid[1] ^ signs[1], signs[1], std::minus<>()); + sumi = dpct::dp4a(grid_l, *((int *)q8 + 0), sumi); + sumi = dpct::dp4a(grid_h, *((int *)q8 + 1), sumi); + q8 += 8; } - const float d = (float)bq1->d * bq8_1[ib32].ds[0]; - return d * (sumi1 * (2*(h1 & 7) + 1) + sumi2 * (2*((h1 >> 4) & 7) + 1) + - sumi3 * (2*(h2 & 7) + 1) + sumi4 * (2*((h2 >> 4) & 7) + 1)); + const float d = (float)bq1->d * bq8_1[ib32].ds[0] * 0.25f; + return d * sumi; #else assert(false); return 0.f; @@ -8644,7 +8646,7 @@ static void mul_mat_vec_q_iq3_s_q8_1(const void * __restrict__ vx, const void * template static void mul_mat_vec_q_iq1_s_q8_1(const void * __restrict__ vx, const void * __restrict__ vy, float * __restrict__ dst, const int ncols, const int nrows, const sycl::nd_item<3> &item_ct1, - const uint64_t *iq1s_grid_ptr, const uint64_t *ksigns64_ptr ) { + const uint32_t *iq1s_grid_ptr, const uint64_t *ksigns64_ptr ) { const int row = item_ct1.get_group(2) * item_ct1.get_local_range(1) + item_ct1.get_local_id(1); @@ -10406,7 +10408,7 @@ static void dequantize_row_iq1_s_sycl(const void *vx, dst_t *y, const int k, dpct::queue_ptr stream) { const int nb = k / QK_K; { - iq1s_grid.init(*stream); + iq1s_grid_gpu.init(*stream); ksigns_iq2xs.init(*stream); kmask_iq2xs.init(*stream); @@ -10414,7 +10416,7 @@ static void dequantize_row_iq1_s_sycl(const void *vx, dst_t *y, const int k, {sycl::aspect::fp16}); stream->submit([&](sycl::handler &cgh) { - auto iq1s_grid_ptr_ct1 = iq1s_grid.get_ptr(); + auto iq1s_grid_ptr_ct1 = iq1s_grid_gpu.get_ptr(); auto ksigns_iq2xs_ptr_ct1 = ksigns_iq2xs.get_ptr(); auto kmask_iq2xs_ptr_ct1 = kmask_iq2xs.get_ptr(); @@ -11154,11 +11156,11 @@ static void mul_mat_vec_iq1_s_q8_1_sycl(const void *vx, const void *vy, const sycl::range<3> block_nums(1, 1, block_num_y); const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE); { - iq1s_grid.init(*stream); + iq1s_grid_gpu.init(*stream); ksigns64.init(*stream); stream->submit([&](sycl::handler &cgh) { - auto iq1s_grid_ptr_ct1 = iq1s_grid.get_ptr(); + auto iq1s_grid_ptr_ct1 = iq1s_grid_gpu.get_ptr(); auto ksigns64_ptr_ct1 = ksigns64.get_ptr(); cgh.parallel_for( From 184215e783dfad76aded2c68244c327a5c507df5 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Tue, 12 Mar 2024 13:49:55 +0200 Subject: [PATCH 603/812] ggml : fix UB in IQ2_S and IQ3_S (#6012) --- ggml-quants.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/ggml-quants.c b/ggml-quants.c index 06665eb2d..3ad558a1b 100644 --- a/ggml-quants.c +++ b/ggml-quants.c @@ -9025,7 +9025,7 @@ void ggml_vec_dot_iq2_s_q8_K(int n, float * restrict s, size_t bs, const void * vld1_s8((const int8_t *)(iq2s_grid + (qs[7] | ((qh[ib32+1] << 2) & 0x300))))); qs += 8; - vs.val[0] = vreinterpretq_u8_u32(vdupq_n_u32(signs[0] | (signs[1] << 16))); + vs.val[0] = vreinterpretq_u8_u32(vdupq_n_u32(signs[0] | ((uint32_t) signs[1] << 16))); vs.val[1] = vandq_u8(ggml_vqtbl1q_u8(vs.val[0], mask1.val[1]), mask2); vs.val[0] = vandq_u8(ggml_vqtbl1q_u8(vs.val[0], mask1.val[0]), mask2); vs.val[0] = vceqq_u8(vs.val[0], mask2); @@ -9034,7 +9034,7 @@ void ggml_vec_dot_iq2_s_q8_K(int n, float * restrict s, size_t bs, const void * q2s.val[0] = vmulq_s8(vreinterpretq_s8_u8(vorrq_u8(vs.val[0], m1)), q2s.val[0]); q2s.val[1] = vmulq_s8(vreinterpretq_s8_u8(vorrq_u8(vs.val[1], m1)), q2s.val[1]); - vs.val[0] = vreinterpretq_u8_u32(vdupq_n_u32(signs[2] | (signs[3] << 16))); + vs.val[0] = vreinterpretq_u8_u32(vdupq_n_u32(signs[2] | ((uint32_t) signs[3] << 16))); vs.val[1] = vandq_u8(ggml_vqtbl1q_u8(vs.val[0], mask1.val[1]), mask2); vs.val[0] = vandq_u8(ggml_vqtbl1q_u8(vs.val[0], mask1.val[0]), mask2); vs.val[0] = vceqq_u8(vs.val[0], mask2); @@ -9105,12 +9105,12 @@ void ggml_vec_dot_iq2_s_q8_K(int n, float * restrict s, size_t bs, const void * iq2s_grid[qs[4] | ((qh[ib32+1] << 8) & 0x300)]); qs += 8; - __m256i aux256 = _mm256_set1_epi32(signs[0] | (signs[1] << 16)); + __m256i aux256 = _mm256_set1_epi32(signs[0] | ((uint32_t) signs[1] << 16)); aux256 = _mm256_and_si256(_mm256_shuffle_epi8(aux256,mask1), mask2); const __m256i s2_1 = _mm256_cmpeq_epi8(aux256, mask2); const __m256i q8s_1 = _mm256_sub_epi8(_mm256_xor_si256(s2_1, q8_1), s2_1); - aux256 = _mm256_set1_epi32(signs[2] | (signs[3] << 16)); + aux256 = _mm256_set1_epi32(signs[2] | ((uint32_t) signs[3] << 16)); aux256 = _mm256_and_si256(_mm256_shuffle_epi8(aux256,mask1), mask2); const __m256i s2_2 = _mm256_cmpeq_epi8(aux256, mask2); const __m256i q8s_2 = _mm256_sub_epi8(_mm256_xor_si256(s2_2, q8_2), s2_2); @@ -9386,7 +9386,7 @@ void ggml_vec_dot_iq3_s_q8_K (int n, float * restrict s, size_t bs, const void * iq3s_grid[idx.index[6]], iq3s_grid[idx.index[7]]); - vs.val[0] = vreinterpretq_u8_u32(vdupq_n_u32(signs[0] | (signs[1] << 16))); + vs.val[0] = vreinterpretq_u8_u32(vdupq_n_u32(signs[0] | ((uint32_t) signs[1] << 16))); vs.val[1] = vandq_u8(ggml_vqtbl1q_u8(vs.val[0], mask1.val[1]), mask2); vs.val[0] = vandq_u8(ggml_vqtbl1q_u8(vs.val[0], mask1.val[0]), mask2); vs.val[0] = vorrq_u8(vceqq_u8(vs.val[0], mask2), m1); @@ -9395,7 +9395,7 @@ void ggml_vec_dot_iq3_s_q8_K (int n, float * restrict s, size_t bs, const void * q3s.val[0] = vmulq_s8(vreinterpretq_s8_u8(vs.val[0]), vreinterpretq_s8_u32(aux32x4_0)); q3s.val[1] = vmulq_s8(vreinterpretq_s8_u8(vs.val[1]), vreinterpretq_s8_u32(aux32x4_1)); - vs.val[0] = vreinterpretq_u8_u32(vdupq_n_u32(signs[2] | (signs[3] << 16))); + vs.val[0] = vreinterpretq_u8_u32(vdupq_n_u32(signs[2] | ((uint32_t) signs[3] << 16))); vs.val[1] = vandq_u8(ggml_vqtbl1q_u8(vs.val[0], mask1.val[1]), mask2); vs.val[0] = vandq_u8(ggml_vqtbl1q_u8(vs.val[0], mask1.val[0]), mask2); vs.val[0] = vorrq_u8(vceqq_u8(vs.val[0], mask2), m1); From 8030da7afea2d89f997aeadbd14183d399a017b9 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Tue, 12 Mar 2024 14:27:20 +0200 Subject: [PATCH 604/812] ggml : reuse quantum structs across backends (#5943) * ggml : reuse quant blocks across backends ggml-ci * ggml : define helper constants only for CUDA and SYCL ggml-ci * ggml : define helper quantum constants for SYCL ggml-ci --- ggml-common.h | 409 ++++++++++++++++++++++++++++++++++++++++++++++- ggml-cuda.cu | 244 ++-------------------------- ggml-metal.m | 2 +- ggml-metal.metal | 180 +-------------------- ggml-quants.c | 78 ++++----- ggml-quants.h | 246 +--------------------------- ggml-sycl.cpp | 192 +--------------------- 7 files changed, 470 insertions(+), 881 deletions(-) diff --git a/ggml-common.h b/ggml-common.h index 2402b773c..0257c928c 100644 --- a/ggml-common.h +++ b/ggml-common.h @@ -1,4 +1,408 @@ -#pragma once +#ifndef GGML_COMMON_DECL + +#if defined(GGML_COMMON_DECL_C) +#include + +typedef uint16_t ggml_half; +typedef uint32_t ggml_half2; + +#define GGML_COMMON_AGGR + +#define GGML_COMMON_DECL +#elif defined(GGML_COMMON_DECL_METAL) +#include + +typedef half ggml_half; +typedef half2 ggml_half2; + +#define GGML_COMMON_AGGR + +#define GGML_COMMON_DECL +#elif defined(GGML_COMMON_DECL_CUDA) +#include +#include + +typedef half ggml_half; +typedef half2 ggml_half2; + +#define GGML_COMMON_AGGR data + +#define GGML_COMMON_DECL +#elif defined(GGML_COMMON_DECL_HIP) +#include +#include + +typedef half ggml_half; +typedef half2 ggml_half2; + +#define GGML_COMMON_AGGR data + +#define GGML_COMMON_DECL +#elif defined(GGML_COMMON_DECL_SYCL) +#include +#include + +typedef sycl::half ggml_half; +typedef sycl::half2 ggml_half2; + +#define GGML_COMMON_AGGR data + +#define GGML_COMMON_DECL +#endif + +#if defined(GGML_COMMON_DECL) + +#ifndef __cplusplus +#ifndef static_assert +#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201100L) +#define static_assert(cond, msg) _Static_assert(cond, msg) +#else +#define static_assert(cond, msg) struct global_scope_noop_trick +#endif +#endif +#endif // __cplusplus + +// QK = number of values after dequantization +// QK_K = super-block size + +#ifdef GGML_QKK_64 +#define QK_K 64 +#define K_SCALE_SIZE 4 +#else +#define QK_K 256 +#define K_SCALE_SIZE 12 +#endif // GGML_QKK_64 + +#if defined(GGML_COMMON_DECL_CUDA) || defined(GGML_COMMON_DECL_HIP) || defined(GGML_COMMON_DECL_SYCL) +// QR = QK / number of values before dequantization +// QI = number of 32 bit integers before dequantization + +#define QI4_0 (QK4_0 / (4 * QR4_0)) +#define QR4_0 2 + +#define QI4_1 (QK4_1 / (4 * QR4_1)) +#define QR4_1 2 + +#define QI5_0 (QK5_0 / (4 * QR5_0)) +#define QR5_0 2 + +#define QI5_1 (QK5_1 / (4 * QR5_1)) +#define QR5_1 2 + +#define QI8_0 (QK8_0 / (4 * QR8_0)) +#define QR8_0 1 + +#define QI8_1 (QK8_1 / (4 * QR8_1)) +#define QR8_1 1 + +#define QI2_K (QK_K / (4*QR2_K)) +#define QR2_K 4 + +#define QI3_K (QK_K / (4*QR3_K)) +#define QR3_K 4 + +#define QI4_K (QK_K / (4*QR4_K)) +#define QR4_K 2 + +#define QI5_K (QK_K / (4*QR5_K)) +#define QR5_K 2 + +#define QI6_K (QK_K / (4*QR6_K)) +#define QR6_K 2 + +#define QI2_XXS (QK_K / (4*QR2_XXS)) +#define QR2_XXS 8 + +#define QI2_XS (QK_K / (4*QR2_XS)) +#define QR2_XS 8 + +#define QI2_S (QK_K / (4*QR2_S)) +#define QR2_S 8 + +#define QI3_XXS (QK_K / (4*QR3_XXS)) +#define QR3_XXS 8 + +#define QI3_XS (QK_K / (4*QR3_XS)) +#define QR3_XS 8 + +#define QI1_S (QK_K / (4*QR1_S)) +#define QR1_S 8 + +#define QI4_NL (QK4_NL / (4*QR4_NL)) +#define QR4_NL 2 + +#if QK_K == 64 +#define QI4_XS QI4_NL +#define QR4_XS QR4_NL +#else +#define QI4_XS (QK_K / (4*QR4_XS)) +#define QR4_XS 8 +#endif + +#endif // GGML_COMMON_DECL_CUDA || GGML_COMMON_DECL_HIP + +#define QK4_0 32 +typedef struct { + ggml_half d; // delta + uint8_t qs[QK4_0 / 2]; // nibbles / quants +} block_q4_0; +static_assert(sizeof(block_q4_0) == sizeof(ggml_half) + QK4_0 / 2, "wrong q4_0 block size/padding"); + +#define QK4_1 32 +typedef struct { + union { + struct { + ggml_half d; // delta + ggml_half m; // min + } GGML_COMMON_AGGR; + ggml_half2 dm; + }; + uint8_t qs[QK4_1 / 2]; // nibbles / quants +} block_q4_1; +static_assert(sizeof(block_q4_1) == 2 * sizeof(ggml_half) + QK4_1 / 2, "wrong q4_1 block size/padding"); + +#define QK5_0 32 +typedef struct { + ggml_half d; // delta + uint8_t qh[4]; // 5-th bit of quants + uint8_t qs[QK5_0 / 2]; // nibbles / quants +} block_q5_0; +static_assert(sizeof(block_q5_0) == sizeof(ggml_half) + sizeof(uint32_t) + QK5_0 / 2, "wrong q5_0 block size/padding"); + +#define QK5_1 32 +typedef struct { + union { + struct { + ggml_half d; // delta + ggml_half m; // min + } GGML_COMMON_AGGR; + ggml_half2 dm; + }; + uint8_t qh[4]; // 5-th bit of quants + uint8_t qs[QK5_1 / 2]; // nibbles / quants +} block_q5_1; +static_assert(sizeof(block_q5_1) == 2 * sizeof(ggml_half) + sizeof(uint32_t) + QK5_1 / 2, "wrong q5_1 block size/padding"); + +#define QK8_0 32 +typedef struct { + ggml_half d; // delta + int8_t qs[QK8_0]; // quants +} block_q8_0; +static_assert(sizeof(block_q8_0) == sizeof(ggml_half) + QK8_0, "wrong q8_0 block size/padding"); + +#define QK8_1 32 +typedef struct { + union { + struct { + ggml_half d; // delta + ggml_half s; // d * sum(qs[i]) + } GGML_COMMON_AGGR; + ggml_half2 ds; + }; + int8_t qs[QK8_1]; // quants +} block_q8_1; +static_assert(sizeof(block_q8_1) == 2*sizeof(ggml_half) + QK8_1, "wrong q8_1 block size/padding"); + +// +// Super-block quantization structures +// + +// 2-bit quantization +// weight is represented as x = a * q + b +// 16 blocks of 16 elements each +// Effectively 2.625 bits per weight +typedef struct { + uint8_t scales[QK_K/16]; // scales and mins, quantized with 4 bits + uint8_t qs[QK_K/4]; // quants + union { + struct { + ggml_half d; // super-block scale for quantized scales + ggml_half dmin; // super-block scale for quantized mins + } GGML_COMMON_AGGR; + ggml_half2 dm; + }; +} block_q2_K; +static_assert(sizeof(block_q2_K) == 2*sizeof(ggml_half) + QK_K/16 + QK_K/4, "wrong q2_K block size/padding"); + +// 3-bit quantization +// weight is represented as x = a * q +// 16 blocks of 16 elements each +// Effectively 3.4375 bits per weight +#ifdef GGML_QKK_64 +typedef struct { + uint8_t hmask[QK_K/8]; // quants - high bit + uint8_t qs[QK_K/4]; // quants - low 2 bits + uint8_t scales[2]; + ggml_half d; // super-block scale +} block_q3_K; +static_assert(sizeof(block_q3_K) == sizeof(ggml_half) + QK_K / 4 + QK_K / 8 + 2, "wrong q3_K block size/padding"); +#else +typedef struct { + uint8_t hmask[QK_K/8]; // quants - high bit + uint8_t qs[QK_K/4]; // quants - low 2 bits + uint8_t scales[12]; // scales, quantized with 6 bits + ggml_half d; // super-block scale +} block_q3_K; +static_assert(sizeof(block_q3_K) == sizeof(ggml_half) + QK_K / 4 + QK_K / 8 + 12, "wrong q3_K block size/padding"); +#endif + +// 4-bit quantization +// 8 blocks of 32 elements each +// weight is represented as x = a * q + b +// Effectively 4.5 bits per weight +#ifdef GGML_QKK_64 +typedef struct { + ggml_half d[2]; // super-block scales/mins + uint8_t scales[2]; // 4-bit block scales/mins + uint8_t qs[QK_K/2]; // 4--bit quants +} block_q4_K; +static_assert(sizeof(block_q4_K) == 2*sizeof(ggml_half) + QK_K/2 + 2, "wrong q4_K block size/padding"); +#else +typedef struct { + union { + struct { + ggml_half d; // super-block scale for quantized scales + ggml_half dmin; // super-block scale for quantized mins + } GGML_COMMON_AGGR; + ggml_half2 dm; + }; + uint8_t scales[K_SCALE_SIZE]; // scales and mins, quantized with 6 bits + uint8_t qs[QK_K/2]; // 4--bit quants +} block_q4_K; +static_assert(sizeof(block_q4_K) == 2*sizeof(ggml_half) + K_SCALE_SIZE + QK_K/2, "wrong q4_K block size/padding"); +#endif + +// 5-bit quantization +// 8 blocks of 32 elements each +// weight is represented as x = a * q + b +// Effectively 5.5 bits per weight +#ifdef GGML_QKK_64 +typedef struct { + ggml_half d; // super-block scale + int8_t scales[QK_K/16]; // 8-bit block scales + uint8_t qh[QK_K/8]; // quants, high bit + uint8_t qs[QK_K/2]; // quants, low 4 bits +} block_q5_K; +static_assert(sizeof(block_q5_K) == sizeof(ggml_half) + QK_K/2 + QK_K/8 + QK_K/16, "wrong q5_K block size/padding"); +#else +typedef struct { + union { + struct { + ggml_half d; // super-block scale for quantized scales + ggml_half dmin; // super-block scale for quantized mins + } GGML_COMMON_AGGR; + ggml_half2 dm; + }; + uint8_t scales[K_SCALE_SIZE]; // scales and mins, quantized with 6 bits + uint8_t qh[QK_K/8]; // quants, high bit + uint8_t qs[QK_K/2]; // quants, low 4 bits +} block_q5_K; +static_assert(sizeof(block_q5_K) == 2*sizeof(ggml_half) + K_SCALE_SIZE + QK_K/2 + QK_K/8, "wrong q5_K block size/padding"); +#endif + +// 6-bit quantization +// weight is represented as x = a * q +// 16 blocks of 16 elements each +// Effectively 6.5625 bits per weight +typedef struct { + uint8_t ql[QK_K/2]; // quants, lower 4 bits + uint8_t qh[QK_K/4]; // quants, upper 2 bits + int8_t scales[QK_K/16]; // scales, quantized with 8 bits + ggml_half d; // super-block scale +} block_q6_K; +static_assert(sizeof(block_q6_K) == sizeof(ggml_half) + QK_K / 16 + 3*QK_K/4, "wrong q6_K block size/padding"); + +// This is only used for intermediate quantization and dot products +typedef struct { + float d; // delta + int8_t qs[QK_K]; // quants + int16_t bsums[QK_K/16]; // sum of quants in groups of 16 +} block_q8_K; +static_assert(sizeof(block_q8_K) == sizeof(float) + QK_K + QK_K/16*sizeof(int16_t), "wrong q8_K block size/padding"); + +// (Almost) "true" 2-bit quantization. +// Due to the need to use blocks as per ggml design, it ends up using +// 2.0625 bpw because of the 16-bit scale for each block of 256. +typedef struct { + ggml_half d; + uint16_t qs[QK_K/8]; +} block_iq2_xxs; +static_assert(sizeof(block_iq2_xxs) == sizeof(ggml_half) + QK_K/8*sizeof(uint16_t), "wrong iq2_xxs block size/padding"); + +// 2.3125 bpw quants +typedef struct { + ggml_half d; + uint16_t qs[QK_K/8]; + uint8_t scales[QK_K/32]; +} block_iq2_xs; +static_assert(sizeof(block_iq2_xs) == sizeof(ggml_half) + QK_K/8*sizeof(uint16_t) + QK_K/32, "wrong iq2_xs block size/padding"); + +// 2.5625 bpw quants +typedef struct { + ggml_half d; + uint8_t qs[QK_K/4]; + uint8_t qh[QK_K/32]; + uint8_t scales[QK_K/32]; +} block_iq2_s; +static_assert(sizeof(block_iq2_s) == sizeof(ggml_half) + QK_K/4 + QK_K/16, "wrong iq2_s block size/padding"); + +// (Almost) "true" 3-bit quantization. +// Due to the need to use blocks as per ggml design, it ends up using +// 3.0625 bpw because of the 16-bit scale for each block of 256. +typedef struct { + ggml_half d; + uint8_t qs[3*QK_K/8]; +} block_iq3_xxs; +static_assert(sizeof(block_iq3_xxs) == sizeof(ggml_half) + 3*(QK_K/8), "wrong iq3_xxs block size/padding"); + +// 3.4375 bpw +#if QK_K == 64 +#define IQ3S_N_SCALE 2 +#else +#define IQ3S_N_SCALE QK_K/64 +#endif +typedef struct { + ggml_half d; + uint8_t qs[QK_K/4]; + uint8_t qh[QK_K/32]; + uint8_t signs[QK_K/8]; + uint8_t scales[IQ3S_N_SCALE]; +} block_iq3_s; +static_assert(sizeof(block_iq3_s) == sizeof(ggml_half) + 13*(QK_K/32) + IQ3S_N_SCALE, "wrong iq3_s block size/padding"); + +typedef struct { + ggml_half d; + uint8_t qs[QK_K/8]; + uint16_t qh[QK_K/32]; +} block_iq1_s; +static_assert(sizeof(block_iq1_s) == sizeof(ggml_half) + QK_K/8 + QK_K/16, "wrong iq1_s block size/padding"); + +// Non-linear quants +#define QK4_NL 32 +typedef struct { + ggml_half d; + uint8_t qs[QK4_NL/2]; +} block_iq4_nl; +static_assert(sizeof(block_iq4_nl) == sizeof(ggml_half) + QK4_NL/2, "wrong iq4_nl block size/padding"); + +#if QK_K == 64 +#define block_iq4_xs block_iq4_nl +#else +typedef struct { + ggml_half d; + uint16_t scales_h; + uint8_t scales_l[QK_K/64]; + uint8_t qs[QK_K/2]; +} block_iq4_xs; +static_assert(sizeof(block_iq4_xs) == sizeof(ggml_half) + sizeof(uint16_t) + QK_K/64 + QK_K/2, "wrong iq4_xs block size/padding"); +#endif + +#endif // GGML_COMMON_DECL +#endif // GGML_COMMON_DECL + +//////////////////////////////////////////////////////////////////////////////// + +#ifndef GGML_COMMON_IMPL #if defined(GGML_COMMON_IMPL_C) #include @@ -14,7 +418,7 @@ #define GGML_TABLE_END() }; #define GGML_COMMON_IMPL -#elif defined(GGML_COMMON_IMPL_CUDA) +#elif defined(GGML_COMMON_IMPL_CUDA) || defined(GGML_COMMON_IMPL_HIP) #include #define GGML_TABLE_BEGIN(type, name, size) static const __device__ type name[size] = { @@ -1423,3 +1827,4 @@ GGML_TABLE_END() #endif #endif // GGML_COMMON_IMPL +#endif // GGML_COMMON_IMPL diff --git a/ggml-cuda.cu b/ggml-cuda.cu index 01b1f15ee..b8834ed05 100644 --- a/ggml-cuda.cu +++ b/ggml-cuda.cu @@ -2,7 +2,13 @@ #include "ggml.h" #include "ggml-backend-impl.h" +#if defined(GGML_USE_HIPBLAS) +#define GGML_COMMON_DECL_HIP +#define GGML_COMMON_IMPL_HIP +#else +#define GGML_COMMON_DECL_CUDA #define GGML_COMMON_IMPL_CUDA +#endif #include "ggml-common.h" #include @@ -359,66 +365,6 @@ typedef void (*ggml_cuda_op_flatten_t)( const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, const float * src0_dd, const float * src1_dd, float * dst_dd, cudaStream_t main_stream); -// QK = number of values after dequantization -// QR = QK / number of values before dequantization -// QI = number of 32 bit integers before dequantization - -#define QK4_0 32 -#define QR4_0 2 -#define QI4_0 (QK4_0 / (4 * QR4_0)) -typedef struct { - half d; // delta - uint8_t qs[QK4_0 / 2]; // nibbles / quants -} block_q4_0; -static_assert(sizeof(block_q4_0) == sizeof(ggml_fp16_t) + QK4_0 / 2, "wrong q4_0 block size/padding"); - -#define QK4_1 32 -#define QR4_1 2 -#define QI4_1 (QK4_1 / (4 * QR4_1)) -typedef struct { - half2 dm; // dm.x = delta, dm.y = min - uint8_t qs[QK4_1 / 2]; // nibbles / quants -} block_q4_1; -static_assert(sizeof(block_q4_1) == sizeof(ggml_fp16_t) * 2 + QK4_1 / 2, "wrong q4_1 block size/padding"); - -#define QK5_0 32 -#define QR5_0 2 -#define QI5_0 (QK5_0 / (4 * QR5_0)) -typedef struct { - half d; // delta - uint8_t qh[4]; // 5-th bit of quants - uint8_t qs[QK5_0 / 2]; // nibbles / quants -} block_q5_0; -static_assert(sizeof(block_q5_0) == sizeof(ggml_fp16_t) + sizeof(uint32_t) + QK5_0 / 2, "wrong q5_0 block size/padding"); - -#define QK5_1 32 -#define QR5_1 2 -#define QI5_1 (QK5_1 / (4 * QR5_1)) -typedef struct { - half2 dm; // dm.x = delta, dm.y = min - uint8_t qh[4]; // 5-th bit of quants - uint8_t qs[QK5_1 / 2]; // nibbles / quants -} block_q5_1; -static_assert(sizeof(block_q5_1) == 2 * sizeof(ggml_fp16_t) + sizeof(uint32_t) + QK5_1 / 2, "wrong q5_1 block size/padding"); - -#define QK8_0 32 -#define QR8_0 1 -#define QI8_0 (QK8_0 / (4 * QR8_0)) -typedef struct { - half d; // delta - int8_t qs[QK8_0]; // quants -} block_q8_0; -static_assert(sizeof(block_q8_0) == sizeof(ggml_fp16_t) + QK8_0, "wrong q8_0 block size/padding"); - -#define QK8_1 32 -#define QR8_1 1 -#define QI8_1 (QK8_1 / (4 * QR8_1)) -typedef struct { - half2 ds; // ds.x = delta, ds.y = sum - int8_t qs[QK8_0]; // quants -} block_q8_1; -static_assert(sizeof(block_q8_1) == 2*sizeof(ggml_fp16_t) + QK8_0, "wrong q8_1 block size/padding"); - typedef float (*vec_dot_q_cuda_t)(const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs); typedef void (*allocate_tiles_cuda_t)(int ** x_ql, half2 ** x_dm, int ** x_qh, int ** x_sc); typedef void (*load_tiles_cuda_t)( @@ -428,174 +374,6 @@ typedef float (*vec_dot_q_mul_mat_cuda_t)( const int * __restrict__ x_ql, const half2 * __restrict__ x_dm, const int * __restrict__ x_qh, const int * __restrict__ x_sc, const int * __restrict__ y_qs, const half2 * __restrict__ y_ms, const int & i, const int & j, const int & k); -//================================= k-quants - -#ifdef GGML_QKK_64 -#define QK_K 64 -#define K_SCALE_SIZE 4 -#else -#define QK_K 256 -#define K_SCALE_SIZE 12 -#endif - -#define QR2_K 4 -#define QI2_K (QK_K / (4*QR2_K)) -typedef struct { - uint8_t scales[QK_K/16]; // scales and mins, quantized with 4 bits - uint8_t qs[QK_K/4]; // quants - half2 dm; // super-block scale for quantized scales/mins -} block_q2_K; -static_assert(sizeof(block_q2_K) == 2*sizeof(ggml_fp16_t) + QK_K/16 + QK_K/4, "wrong q2_K block size/padding"); - -#define QR3_K 4 -#define QI3_K (QK_K / (4*QR3_K)) -typedef struct { - uint8_t hmask[QK_K/8]; // quants - high bit - uint8_t qs[QK_K/4]; // quants - low 2 bits -#ifdef GGML_QKK_64 - uint8_t scales[2]; // scales, quantized with 8 bits -#else - uint8_t scales[K_SCALE_SIZE]; // scales, quantized with 6 bits -#endif - half d; // super-block scale -} block_q3_K; -//static_assert(sizeof(block_q3_K) == sizeof(ggml_fp16_t) + QK_K / 4 + QK_K / 8 + K_SCALE_SIZE, "wrong q3_K block size/padding"); - -#define QR4_K 2 -#define QI4_K (QK_K / (4*QR4_K)) -#ifdef GGML_QKK_64 -typedef struct { - half dm[2]; // super-block scales/mins - uint8_t scales[2]; // 4-bit block scales/mins - uint8_t qs[QK_K/2]; // 4--bit quants -} block_q4_K; -static_assert(sizeof(block_q4_K) == sizeof(half2) + QK_K/2 + 2, "wrong q4_K block size/padding"); -#else -typedef struct { - half2 dm; // super-block scale for quantized scales/mins - uint8_t scales[3*QK_K/64]; // scales, quantized with 6 bits - uint8_t qs[QK_K/2]; // 4--bit quants -} block_q4_K; -static_assert(sizeof(block_q4_K) == 2*sizeof(ggml_fp16_t) + 3*QK_K/64 + QK_K/2, "wrong q4_K block size/padding"); -#endif - -#define QR5_K 2 -#define QI5_K (QK_K / (4*QR5_K)) -#ifdef GGML_QKK_64 -typedef struct { - half d; // super-block scale - int8_t scales[QK_K/16]; // block scales - uint8_t qh[QK_K/8]; // quants, high bit - uint8_t qs[QK_K/2]; // quants, low 4 bits -} block_q5_K; -static_assert(sizeof(block_q5_K) == sizeof(ggml_fp16_t) + QK_K/2 + QK_K/8 + QK_K/16, "wrong q5_K block size/padding"); -#else -typedef struct { - half2 dm; // super-block scale for quantized scales/mins - uint8_t scales[K_SCALE_SIZE]; // scales and mins, quantized with 6 bits - uint8_t qh[QK_K/8]; // quants, high bit - uint8_t qs[QK_K/2]; // quants, low 4 bits -} block_q5_K; -static_assert(sizeof(block_q5_K) == 2*sizeof(ggml_fp16_t) + K_SCALE_SIZE + QK_K/2 + QK_K/8, "wrong q5_K block size/padding"); -#endif - -#define QR6_K 2 -#define QI6_K (QK_K / (4*QR6_K)) -typedef struct { - uint8_t ql[QK_K/2]; // quants, lower 4 bits - uint8_t qh[QK_K/4]; // quants, upper 2 bits - int8_t scales[QK_K/16]; // scales - half d; // delta -} block_q6_K; -static_assert(sizeof(block_q6_K) == sizeof(ggml_fp16_t) + 13*QK_K/16, "wrong q6_K block size/padding"); - -#define QR2_XXS 8 -#define QI2_XXS (QK_K / (4*QR2_XXS)) -typedef struct { - half d; - uint16_t qs[QK_K/8]; -} block_iq2_xxs; -static_assert(sizeof(block_iq2_xxs) == sizeof(ggml_fp16_t) + QK_K/8*sizeof(uint16_t), "wrong iq2_xxs block size/padding"); - -#define QR2_XS 8 -#define QI2_XS (QK_K / (4*QR2_XS)) -typedef struct { - half d; - uint16_t qs[QK_K/8]; - uint8_t scales[QK_K/32]; -} block_iq2_xs; -static_assert(sizeof(block_iq2_xs) == sizeof(ggml_fp16_t) + QK_K/8*sizeof(uint16_t) + QK_K/32, "wrong iq2_xs block size/padding"); - -// 2.5625 bpw quants -#define QR2_S 8 -#define QI2_S (QK_K / (4*QR2_S)) -typedef struct { - half d; - uint8_t qs[QK_K/4]; - uint8_t qh[QK_K/32]; - uint8_t scales[QK_K/32]; -} block_iq2_s; -static_assert(sizeof(block_iq2_s) == sizeof(ggml_fp16_t) + QK_K/4 + QK_K/16, "wrong iq2_s block size/padding"); - -#define QR3_XXS 8 -#define QI3_XXS (QK_K / (4*QR3_XXS)) -typedef struct { - half d; - uint8_t qs[3*(QK_K/8)]; -} block_iq3_xxs; -static_assert(sizeof(block_iq3_xxs) == sizeof(ggml_fp16_t) + 3*(QK_K/8), "wrong iq3_xxs block size/padding"); - -#define QR3_XS 8 -#define QI3_XS (QK_K / (4*QR3_XS)) -#if QK_K == 64 -#define IQ3S_N_SCALE 2 -#else -#define IQ3S_N_SCALE QK_K/64 -#endif -typedef struct { - half d; - uint8_t qs[QK_K/4]; - uint8_t qh[QK_K/32]; - uint8_t signs[QK_K/8]; - uint8_t scales[IQ3S_N_SCALE]; -} block_iq3_s; -static_assert(sizeof(block_iq3_s) == sizeof(ggml_fp16_t) + 13*(QK_K/32) + IQ3S_N_SCALE, "wrong iq3_s block size/padding"); - -#define QR1_S 8 -#define QI1_S (QK_K / (4*QR1_S)) -typedef struct { - half d; - uint8_t qs[QK_K/8]; - uint16_t qh[QK_K/32]; -} block_iq1_s; -static_assert(sizeof(block_iq1_s) == sizeof(ggml_fp16_t) + QK_K/8 + QK_K/16, "wrong iq1_s block size/padding"); - -#define QK4_NL 32 -#define QR4_NL 2 -#define QI4_NL (QK4_NL / (4*QR4_NL)) -typedef struct { - half d; - uint8_t qs[QK4_NL/2]; -} block_iq4_nl; -static_assert(sizeof(block_iq4_nl) == sizeof(ggml_fp16_t) + QK4_NL/2, "wrong iq4_nl block size/padding"); - -#if QK_K == 64 -#define block_iq4_xs block_iq4_nl -#define QR4_XS QR4_NL -#define QI4_XS QI4_NL -#else -// QR4_XS = 8 is very slightly faster than QR4_XS = 4 -#define QR4_XS 8 -#define QI4_XS (QK_K / (4*QR4_XS)) -typedef struct { - half d; - uint16_t scales_h; - uint8_t scales_l[QK_K/64]; - uint8_t qs[QK_K/2]; -} block_iq4_xs; -static_assert(sizeof(block_iq4_xs) == sizeof(ggml_fp16_t) + sizeof(uint16_t) + QK_K/64 + QK_K/2, "wrong iq4_xs block size/padding"); -#endif - #define WARP_SIZE 32 #define MATRIX_ROW_PADDING 512 // last row of quant. matrices is a multiple of this to avoid out-of-bounds memory accesses @@ -3570,7 +3348,7 @@ static __device__ __forceinline__ float vec_dot_q2_K_q8_1( #pragma unroll for (int i = 0; i < QR2_K; ++ i) { u[i] = get_int_from_int8_aligned(bq8_1[bq8_offset + i].qs, iqs % QI8_1); - d8[i] = __low2half(bq8_1[bq8_offset + i].ds); + d8[i] = __low2float(bq8_1[bq8_offset + i].ds); } return vec_dot_q2_K_q8_1_impl_mmvq(v, u, scales, bq2_K->dm, d8); @@ -3692,7 +3470,7 @@ static __device__ __forceinline__ float vec_dot_q3_K_q8_1( #pragma unroll for (int i = 0; i < QR3_K; ++i) { u[i] = get_int_from_int8_aligned(bq8_1[bq8_offset + i].qs, iqs % QI8_1); - d8[i] = __low2half(bq8_1[bq8_offset + i].ds); + d8[i] = __low2float(bq8_1[bq8_offset + i].ds); } return vec_dot_q3_K_q8_1_impl_mmvq(vl, vh, u, bq3_K->scales, scale_offset, d, d8); @@ -3861,7 +3639,7 @@ static __device__ __forceinline__ float vec_dot_q4_K_q8_1( for (int i = 0; i < QR4_K; ++i) { const block_q8_1 * bq8i = bq8_1 + bq8_offset + i; - d8[i] = __low2half(bq8i->ds); + d8[i] = __low2float(bq8i->ds); const int * q8 = (const int *)bq8i->qs + ((iqs/2)%4); u[2*i+0] = q8[0]; @@ -4226,7 +4004,7 @@ static __device__ __forceinline__ float vec_dot_q6_K_q8_1( #pragma unroll for (int i = 0; i < QR6_K; ++i) { u[i] = get_int_from_int8_aligned(bq8_1[bq8_offset + 2*i].qs, iqs % QI8_1); - d8[i] = __low2half(bq8_1[bq8_offset + 2*i].ds); + d8[i] = __low2float(bq8_1[bq8_offset + 2*i].ds); } return vec_dot_q6_K_q8_1_impl_mmvq(vl, vh, u, scales, bq6_K->d, d8); @@ -4763,7 +4541,7 @@ static __device__ __forceinline__ void mul_mat_q( *dsi_dst = *dsi_src; } else { float * dfi_dst = (float *) dsi_dst; - *dfi_dst = __low2half(*dsi_src); + *dfi_dst = __low2float(*dsi_src); } } diff --git a/ggml-metal.m b/ggml-metal.m index 3cf80de7b..1825d3320 100644 --- a/ggml-metal.m +++ b/ggml-metal.m @@ -336,7 +336,7 @@ static struct ggml_metal_context * ggml_metal_init(int n_cb) { NSMutableDictionary * prep = [NSMutableDictionary dictionary]; #ifdef GGML_QKK_64 - prep[@"QK_K"] = @(64); + prep[@"GGML_QKK_64"] = @(1); #endif MTLCompileOptions* options = [MTLCompileOptions new]; diff --git a/ggml-metal.metal b/ggml-metal.metal index 21298a8c2..ebf2f5b47 100644 --- a/ggml-metal.metal +++ b/ggml-metal.metal @@ -1,3 +1,7 @@ +#define GGML_COMMON_DECL_METAL +#define GGML_COMMON_IMPL_METAL +#include "ggml-common.h" + #include #define GGML_COMMON_IMPL_METAL @@ -9,41 +13,6 @@ using namespace metal; #define MIN(x, y) ((x) < (y) ? (x) : (y)) #define SWAP(x, y) { auto tmp = (x); (x) = (y); (y) = tmp; } -#define QK4_0 32 -#define QR4_0 2 -typedef struct { - half d; // delta - uint8_t qs[QK4_0 / 2]; // nibbles / quants -} block_q4_0; - -#define QK4_1 32 -typedef struct { - half d; // delta - half m; // min - uint8_t qs[QK4_1 / 2]; // nibbles / quants -} block_q4_1; - -#define QK5_0 32 -typedef struct { - half d; // delta - uint8_t qh[4]; // 5-th bit of quants - uint8_t qs[QK5_0 / 2]; // nibbles / quants -} block_q5_0; - -#define QK5_1 32 -typedef struct { - half d; // delta - half m; // min - uint8_t qh[4]; // 5-th bit of quants - uint8_t qs[QK5_1 / 2]; // nibbles / quants -} block_q5_1; - -#define QK8_0 32 -typedef struct { - half d; // delta - int8_t qs[QK8_0]; // quants -} block_q8_0; - #define N_SIMDWIDTH 32 // assuming SIMD group size is 32 enum ggml_sort_order { @@ -2478,147 +2447,6 @@ kernel void kernel_concat( } } -//============================================ k-quants ====================================================== - -#ifndef QK_K -#define QK_K 256 -#else -static_assert(QK_K == 256 || QK_K == 64, "QK_K must be 256 or 64"); -#endif - -#if QK_K == 256 -#define K_SCALE_SIZE 12 -#else -#define K_SCALE_SIZE 4 -#endif - -typedef struct { - uint8_t scales[QK_K/16]; // scales and mins, quantized with 4 bits - uint8_t qs[QK_K/4]; // quants - half d; // super-block scale for quantized scales - half dmin; // super-block scale for quantized mins -} block_q2_K; -// 84 bytes / block - -typedef struct { - uint8_t hmask[QK_K/8]; // quants - high bit - uint8_t qs[QK_K/4]; // quants - low 2 bits -#if QK_K == 64 - uint8_t scales[2]; -#else - uint8_t scales[K_SCALE_SIZE]; // scales, quantized with 6 bits -#endif - half d; // super-block scale -} block_q3_K; - -#if QK_K == 64 -typedef struct { - half d[2]; // super-block scales/mins - uint8_t scales[2]; - uint8_t qs[QK_K/2]; // 4-bit quants -} block_q4_K; -#else -typedef struct { - half d; // super-block scale for quantized scales - half dmin; // super-block scale for quantized mins - uint8_t scales[K_SCALE_SIZE]; // scales and mins, quantized with 6 bits - uint8_t qs[QK_K/2]; // 4--bit quants -} block_q4_K; -#endif - -#if QK_K == 64 -typedef struct { - half d; // super-block scales/mins - int8_t scales[QK_K/16]; // 8-bit block scales - uint8_t qh[QK_K/8]; // quants, high bit - uint8_t qs[QK_K/2]; // quants, low 4 bits -} block_q5_K; -#else -typedef struct { - half d; // super-block scale for quantized scales - half dmin; // super-block scale for quantized mins - uint8_t scales[3*QK_K/64]; // scales and mins, quantized with 6 bits - uint8_t qh[QK_K/8]; // quants, high bit - uint8_t qs[QK_K/2]; // quants, low 4 bits -} block_q5_K; -// 176 bytes / block -#endif - -typedef struct { - uint8_t ql[QK_K/2]; // quants, lower 4 bits - uint8_t qh[QK_K/4]; // quants, upper 2 bits - int8_t scales[QK_K/16]; // scales, quantized with 8 bits - half d; // super-block scale -} block_q6_K; -// 210 bytes / block - -typedef struct { - half d; - uint16_t qs[QK_K/8]; -} block_iq2_xxs; -// 66 bytes / block for QK_K = 256, so 2.0625 bpw - -typedef struct { - half d; - uint16_t qs[QK_K/8]; - uint8_t scales[QK_K/32]; -} block_iq2_xs; -// 74 bytes / block for QK_K = 256, so 2.3125 bpw - -// 2.5625 bpw quants -typedef struct { - half d; - uint8_t qs[QK_K/4]; - uint8_t qh[QK_K/32]; - uint8_t scales[QK_K/32]; -} block_iq2_s; - -typedef struct { - half d; - uint8_t qs[3*QK_K/8]; -} block_iq3_xxs; -// 98 bytes / block for QK_K = 256, so 3.0625 bpw - -// 3.4375 bpw -#if QK_K == 64 -#define IQ3S_N_SCALE 2 -#else -#define IQ3S_N_SCALE QK_K/64 -#endif -typedef struct { - half d; - uint8_t qs[QK_K/4]; - uint8_t qh[QK_K/32]; - uint8_t signs[QK_K/8]; - uint8_t scales[IQ3S_N_SCALE]; -} block_iq3_s; - -typedef struct { - half d; - uint8_t qs[QK_K/8]; - uint16_t qh[QK_K/32]; -} block_iq1_s; - -// Non-linear quants -#define QK4_NL 32 -typedef struct { - half d; - uint8_t qs[QK4_NL/2]; -} block_iq4_nl; - -#if QK_K == 64 -#define block_iq4_xs block_iq4_nl -#else -typedef struct { - half d; - uint16_t scales_h; - uint8_t scales_l[QK_K/64]; - uint8_t qs[QK_K/2]; -} block_iq4_xs; -#endif - -//====================================== dot products ========================= - void kernel_mul_mv_q2_K_f32_impl( device const void * src0, device const float * src1, diff --git a/ggml-quants.c b/ggml-quants.c index 3ad558a1b..109dd6660 100644 --- a/ggml-quants.c +++ b/ggml-quants.c @@ -1,3 +1,6 @@ +#define GGML_COMMON_IMPL_C +#include "ggml-common.h" + #include "ggml-quants.h" #include "ggml-impl.h" @@ -951,7 +954,7 @@ void quantize_row_q8_1_reference(const float * restrict x, block_q8_1 * restrict const float d = amax / ((1 << 7) - 1); const float id = d ? 1.0f/d : 0.0f; - y[i].d = d; + y[i].d = GGML_FP32_TO_FP16(d); int sum = 0; @@ -966,7 +969,7 @@ void quantize_row_q8_1_reference(const float * restrict x, block_q8_1 * restrict sum += y[i].qs[QK8_1/2 + j]; } - y[i].s = sum*d; + y[i].s = GGML_FP32_TO_FP16(sum*d); } } @@ -994,7 +997,7 @@ void quantize_row_q8_1(const float * restrict x, void * restrict vy, int k) { const float d = amax / ((1 << 7) - 1); const float id = d ? 1.0f/d : 0.0f; - y[i].d = d; + y[i].d = GGML_FP32_TO_FP16(d); int32x4_t accv = vdupq_n_s32(0); @@ -1010,7 +1013,7 @@ void quantize_row_q8_1(const float * restrict x, void * restrict vy, int k) { accv = vaddq_s32(accv, vi); } - y[i].s = d * vaddvq_s32(accv); + y[i].s = GGML_FP32_TO_FP16(d * vaddvq_s32(accv)); } #elif defined(__wasm_simd128__) for (int i = 0; i < nb; i++) { @@ -1033,7 +1036,7 @@ void quantize_row_q8_1(const float * restrict x, void * restrict vy, int k) { const float d = amax / ((1 << 7) - 1); const float id = d ? 1.0f/d : 0.0f; - y[i].d = d; + y[i].d = GGML_FP32_TO_FP16(d); v128_t accv = wasm_i32x4_splat(0); @@ -1049,10 +1052,11 @@ void quantize_row_q8_1(const float * restrict x, void * restrict vy, int k) { accv = wasm_i32x4_add(accv, vi); } - y[i].s = d * (wasm_i32x4_extract_lane(accv, 0) + - wasm_i32x4_extract_lane(accv, 1) + - wasm_i32x4_extract_lane(accv, 2) + - wasm_i32x4_extract_lane(accv, 3)); + y[i].s = GGML_FP32_TO_FP16( + d * (wasm_i32x4_extract_lane(accv, 0) + + wasm_i32x4_extract_lane(accv, 1) + + wasm_i32x4_extract_lane(accv, 2) + + wasm_i32x4_extract_lane(accv, 3))); } #elif defined(__AVX2__) || defined(__AVX__) for (int i = 0; i < nb; i++) { @@ -1077,7 +1081,7 @@ void quantize_row_q8_1(const float * restrict x, void * restrict vy, int k) { // Quantize these floats const float d = maxScalar / 127.f; - y[i].d = d; + y[i].d = GGML_FP32_TO_FP16(d); const float id = ( maxScalar != 0.0f ) ? 127.f / maxScalar : 0.0f; const __m256 mul = _mm256_set1_ps( id ); @@ -1101,7 +1105,7 @@ void quantize_row_q8_1(const float * restrict x, void * restrict vy, int k) { #if defined(__AVX2__) // Compute the sum of the quants and set y[i].s - y[i].s = d * hsum_i32_8(_mm256_add_epi32(_mm256_add_epi32(i0, i1), _mm256_add_epi32(i2, i3))); + y[i].s = GGML_FP32_TO_FP16(d * hsum_i32_8(_mm256_add_epi32(_mm256_add_epi32(i0, i1), _mm256_add_epi32(i2, i3)))); // Convert int32 to int16 i0 = _mm256_packs_epi32( i0, i1 ); // 0, 1, 2, 3, 8, 9, 10, 11, 4, 5, 6, 7, 12, 13, 14, 15 @@ -1131,7 +1135,7 @@ void quantize_row_q8_1(const float * restrict x, void * restrict vy, int k) { // Compute the sum of the quants and set y[i].s const __m128i s0 = _mm_add_epi32(_mm_add_epi32(ni0, ni1), _mm_add_epi32(ni2, ni3)); const __m128i s1 = _mm_add_epi32(_mm_add_epi32(ni4, ni5), _mm_add_epi32(ni6, ni7)); - y[i].s = d * hsum_i32_4(_mm_add_epi32(s0, s1)); + y[i].s = GGML_FP32_TO_FP16(d * hsum_i32_4(_mm_add_epi32(s0, s1))); // Convert int32 to int16 ni0 = _mm_packs_epi32( ni0, ni1 ); @@ -1162,7 +1166,7 @@ void quantize_row_q8_1(const float * restrict x, void * restrict vy, int k) { const float d = amax / ((1 << 7) - 1); const float id = d ? 1.0f/d : 0.0f; - y[i].d = d; + y[i].d = GGML_FP32_TO_FP16(d); vfloat32m4_t x0 = __riscv_vfmul_vf_f32m4(v_x, id, vl); @@ -1179,7 +1183,7 @@ void quantize_row_q8_1(const float * restrict x, void * restrict vy, int k) { // set y[i].s int sum = __riscv_vmv_x_s_i16m1_i16(vwrs); - y[i].s = sum*d; + y[i].s = GGML_FP32_TO_FP16(sum*d); } #else GGML_UNUSED(nb); @@ -4019,10 +4023,10 @@ void ggml_vec_dot_q4_1_q8_1(int n, float * restrict s, size_t bs, const void * r const block_q8_1 * restrict b_y0 = &vy0[i]; const block_q8_1 * restrict b_y1 = &vy1[i]; - float32x4_t summs_t = {GGML_FP16_TO_FP32(b_x0->m) * b_y0->s, - GGML_FP16_TO_FP32(b_x1->m) * b_y0->s, - GGML_FP16_TO_FP32(b_x0->m) * b_y1->s, - GGML_FP16_TO_FP32(b_x1->m) * b_y1->s}; + float32x4_t summs_t = {GGML_FP16_TO_FP32(b_x0->m) * GGML_FP16_TO_FP32(b_y0->s), + GGML_FP16_TO_FP32(b_x1->m) * GGML_FP16_TO_FP32(b_y0->s), + GGML_FP16_TO_FP32(b_x0->m) * GGML_FP16_TO_FP32(b_y1->s), + GGML_FP16_TO_FP32(b_x1->m) * GGML_FP16_TO_FP32(b_y1->s)}; summs0 += summs_t; const uint8x16_t m4b = vdupq_n_u8(0x0F); @@ -4087,7 +4091,7 @@ void ggml_vec_dot_q4_1_q8_1(int n, float * restrict s, size_t bs, const void * r const block_q8_1 * restrict y0 = &y[i + 0]; const block_q8_1 * restrict y1 = &y[i + 1]; - summs += GGML_FP16_TO_FP32(x0->m) * y0->s + GGML_FP16_TO_FP32(x1->m) * y1->s; + summs += GGML_FP16_TO_FP32(x0->m) * GGML_FP16_TO_FP32(y0->s) + GGML_FP16_TO_FP32(x1->m) * GGML_FP16_TO_FP32(y1->s); const uint8x16_t m4b = vdupq_n_u8(0x0F); @@ -4110,8 +4114,8 @@ void ggml_vec_dot_q4_1_q8_1(int n, float * restrict s, size_t bs, const void * r const int32x4_t p_0 = ggml_vdotq_s32(ggml_vdotq_s32(vdupq_n_s32(0), v0_0l, v1_0l), v0_0h, v1_0h); const int32x4_t p_1 = ggml_vdotq_s32(ggml_vdotq_s32(vdupq_n_s32(0), v0_1l, v1_1l), v0_1h, v1_1h); - sumv0 = vmlaq_n_f32(sumv0, vcvtq_f32_s32(p_0), GGML_FP16_TO_FP32(x0->d)*y0->d); - sumv1 = vmlaq_n_f32(sumv1, vcvtq_f32_s32(p_1), GGML_FP16_TO_FP32(x1->d)*y1->d); + sumv0 = vmlaq_n_f32(sumv0, vcvtq_f32_s32(p_0), GGML_FP16_TO_FP32(x0->d)*GGML_FP16_TO_FP32(y0->d)); + sumv1 = vmlaq_n_f32(sumv1, vcvtq_f32_s32(p_1), GGML_FP16_TO_FP32(x1->d)*GGML_FP16_TO_FP32(y1->d)); } *s = vaddvq_f32(sumv0) + vaddvq_f32(sumv1) + summs; @@ -4124,9 +4128,9 @@ void ggml_vec_dot_q4_1_q8_1(int n, float * restrict s, size_t bs, const void * r // Main loop for (int i = 0; i < nb; ++i) { const float d0 = GGML_FP16_TO_FP32(x[i].d); - const float d1 = y[i].d; + const float d1 = GGML_FP16_TO_FP32(y[i].d); - summs += GGML_FP16_TO_FP32(x[i].m) * y[i].s; + summs += GGML_FP16_TO_FP32(x[i].m) * GGML_FP16_TO_FP32(y[i].s); const __m256 d0v = _mm256_set1_ps( d0 ); const __m256 d1v = _mm256_set1_ps( d1 ); @@ -4178,7 +4182,7 @@ void ggml_vec_dot_q4_1_q8_1(int n, float * restrict s, size_t bs, const void * r int sumi = __riscv_vmv_x_s_i32m1_i32(vs2); - sumf += (GGML_FP16_TO_FP32(x[i].d)*y[i].d)*sumi + GGML_FP16_TO_FP32(x[i].m)*y[i].s; + sumf += (GGML_FP16_TO_FP32(x[i].d)*GGML_FP16_TO_FP32(y[i].d))*sumi + GGML_FP16_TO_FP32(x[i].m)*GGML_FP16_TO_FP32(y[i].s); } *s = sumf; @@ -4196,7 +4200,7 @@ void ggml_vec_dot_q4_1_q8_1(int n, float * restrict s, size_t bs, const void * r sumi += (v0 * y[i].qs[j]) + (v1 * y[i].qs[j + qk/2]); } - sumf += (GGML_FP16_TO_FP32(x[i].d)*y[i].d)*sumi + GGML_FP16_TO_FP32(x[i].m)*y[i].s; + sumf += (GGML_FP16_TO_FP32(x[i].d)*GGML_FP16_TO_FP32(y[i].d))*sumi + GGML_FP16_TO_FP32(x[i].m)*GGML_FP16_TO_FP32(y[i].s); } *s = sumf; @@ -4532,8 +4536,8 @@ void ggml_vec_dot_q5_1_q8_1(int n, float * restrict s, size_t bs, const void * r const uint8x16_t m4b = vdupq_n_u8(0x0F); - summs0 += GGML_FP16_TO_FP32(x0->m) * y0->s; - summs1 += GGML_FP16_TO_FP32(x1->m) * y1->s; + summs0 += GGML_FP16_TO_FP32(x0->m) * GGML_FP16_TO_FP32(y0->s); + summs1 += GGML_FP16_TO_FP32(x1->m) * GGML_FP16_TO_FP32(y1->s); // extract the 5th bit via lookup table ((b) << 4) memcpy(&qh0, x0->qh, sizeof(qh0)); @@ -4577,10 +4581,10 @@ void ggml_vec_dot_q5_1_q8_1(int n, float * restrict s, size_t bs, const void * r sumv0 = vmlaq_n_f32(sumv0, vcvtq_f32_s32(vaddq_s32( ggml_vdotq_s32(vdupq_n_s32(0), v0_0lf, v1_0l), - ggml_vdotq_s32(vdupq_n_s32(0), v0_0hf, v1_0h))), GGML_FP16_TO_FP32(x0->d)*y0->d); + ggml_vdotq_s32(vdupq_n_s32(0), v0_0hf, v1_0h))), GGML_FP16_TO_FP32(x0->d)*GGML_FP16_TO_FP32(y0->d)); sumv1 = vmlaq_n_f32(sumv1, vcvtq_f32_s32(vaddq_s32( ggml_vdotq_s32(vdupq_n_s32(0), v0_1lf, v1_1l), - ggml_vdotq_s32(vdupq_n_s32(0), v0_1hf, v1_1h))), GGML_FP16_TO_FP32(x1->d)*y1->d); + ggml_vdotq_s32(vdupq_n_s32(0), v0_1hf, v1_1h))), GGML_FP16_TO_FP32(x1->d)*GGML_FP16_TO_FP32(y1->d)); } *s = vaddvq_f32(sumv0) + vaddvq_f32(sumv1) + summs0 + summs1; @@ -4597,7 +4601,7 @@ void ggml_vec_dot_q5_1_q8_1(int n, float * restrict s, size_t bs, const void * r const block_q5_1 * restrict x0 = &x[i]; const block_q8_1 * restrict y0 = &y[i]; - summs += GGML_FP16_TO_FP32(x0->m) * y0->s; + summs += GGML_FP16_TO_FP32(x0->m) * GGML_FP16_TO_FP32(y0->s); const v128_t m4b = wasm_i8x16_splat(0x0F); @@ -4644,7 +4648,7 @@ void ggml_vec_dot_q5_1_q8_1(int n, float * restrict s, size_t bs, const void * r wasm_i32x4_dot_i16x8(v0lfh, v1lh)), wasm_i32x4_add(wasm_i32x4_dot_i16x8(v0hfl, v1hl), wasm_i32x4_dot_i16x8(v0hfh, v1hh)))), - wasm_f32x4_splat(GGML_FP16_TO_FP32(x0->d) * y0->d))); + wasm_f32x4_splat(GGML_FP16_TO_FP32(x0->d) * GGML_FP16_TO_FP32(y0->d)))); } *s = wasm_f32x4_extract_lane(sumv, 0) + wasm_f32x4_extract_lane(sumv, 1) + @@ -4659,14 +4663,14 @@ void ggml_vec_dot_q5_1_q8_1(int n, float * restrict s, size_t bs, const void * r for (int i = 0; i < nb; i++) { const __m256 dx = _mm256_set1_ps(GGML_FP16_TO_FP32(x[i].d)); - summs += GGML_FP16_TO_FP32(x[i].m) * y[i].s; + summs += GGML_FP16_TO_FP32(x[i].m) * GGML_FP16_TO_FP32(y[i].s); __m256i qx = bytes_from_nibbles_32(x[i].qs); __m256i bxhi = bytes_from_bits_32(x[i].qh); bxhi = _mm256_and_si256(bxhi, _mm256_set1_epi8(0x10)); qx = _mm256_or_si256(qx, bxhi); - const __m256 dy = _mm256_set1_ps(y[i].d); + const __m256 dy = _mm256_set1_ps(GGML_FP16_TO_FP32(y[i].d)); const __m256i qy = _mm256_loadu_si256((const __m256i *)y[i].qs); const __m256 q = mul_sum_us8_pairs_float(qx, qy); @@ -4686,7 +4690,7 @@ void ggml_vec_dot_q5_1_q8_1(int n, float * restrict s, size_t bs, const void * r for (int i = 0; i < nb; i++) { const __m256 dx = _mm256_set1_ps(GGML_FP16_TO_FP32(x[i].d)); - summs += GGML_FP16_TO_FP32(x[i].m) * y[i].s; + summs += GGML_FP16_TO_FP32(x[i].m) * GGML_FP16_TO_FP32(y[i].s); __m256i bx_0 = bytes_from_nibbles_32(x[i].qs); const __m256i bxhi = bytes_from_bits_32(x[i].qh); @@ -4700,7 +4704,7 @@ void ggml_vec_dot_q5_1_q8_1(int n, float * restrict s, size_t bs, const void * r bxh = _mm_or_si128(bxh, bxhih); bx_0 = MM256_SET_M128I(bxh, bxl); - const __m256 dy = _mm256_set1_ps(y[i].d); + const __m256 dy = _mm256_set1_ps(GGML_FP16_TO_FP32(y[i].d)); const __m256i by_0 = _mm256_loadu_si256((const __m256i *)y[i].qs); const __m256 q = mul_sum_us8_pairs_float(bx_0, by_0); @@ -4767,7 +4771,7 @@ void ggml_vec_dot_q5_1_q8_1(int n, float * restrict s, size_t bs, const void * r int sumi = __riscv_vmv_x_s_i32m1_i32(vs2); - sumf += (GGML_FP16_TO_FP32(x[i].d)*y[i].d)*sumi + GGML_FP16_TO_FP32(x[i].m)*y[i].s; + sumf += (GGML_FP16_TO_FP32(x[i].d)*GGML_FP16_TO_FP32(y[i].d))*sumi + GGML_FP16_TO_FP32(x[i].m)*GGML_FP16_TO_FP32(y[i].s); } *s = sumf; @@ -4791,7 +4795,7 @@ void ggml_vec_dot_q5_1_q8_1(int n, float * restrict s, size_t bs, const void * r sumi += (x0 * y[i].qs[j]) + (x1 * y[i].qs[j + qk/2]); } - sumf += (GGML_FP16_TO_FP32(x[i].d)*y[i].d)*sumi + GGML_FP16_TO_FP32(x[i].m)*y[i].s; + sumf += (GGML_FP16_TO_FP32(x[i].d)*GGML_FP16_TO_FP32(y[i].d))*sumi + GGML_FP16_TO_FP32(x[i].m)*GGML_FP16_TO_FP32(y[i].s); } *s = sumf; diff --git a/ggml-quants.h b/ggml-quants.h index 74aabf415..aa7e54a16 100644 --- a/ggml-quants.h +++ b/ggml-quants.h @@ -1,248 +1,12 @@ #pragma once +#define GGML_COMMON_DECL_C +#include "ggml-common.h" + +#include "ggml.h" + // GGML internal header -#include "ggml-impl.h" - -#include -#include - -#define QK4_0 32 -typedef struct { - ggml_fp16_t d; // delta - uint8_t qs[QK4_0 / 2]; // nibbles / quants -} block_q4_0; -static_assert(sizeof(block_q4_0) == sizeof(ggml_fp16_t) + QK4_0 / 2, "wrong q4_0 block size/padding"); - -#define QK4_1 32 -typedef struct { - ggml_fp16_t d; // delta - ggml_fp16_t m; // min - uint8_t qs[QK4_1 / 2]; // nibbles / quants -} block_q4_1; -static_assert(sizeof(block_q4_1) == 2 * sizeof(ggml_fp16_t) + QK4_1 / 2, "wrong q4_1 block size/padding"); - -#define QK5_0 32 -typedef struct { - ggml_fp16_t d; // delta - uint8_t qh[4]; // 5-th bit of quants - uint8_t qs[QK5_0 / 2]; // nibbles / quants -} block_q5_0; -static_assert(sizeof(block_q5_0) == sizeof(ggml_fp16_t) + sizeof(uint32_t) + QK5_0 / 2, "wrong q5_0 block size/padding"); - -#define QK5_1 32 -typedef struct { - ggml_fp16_t d; // delta - ggml_fp16_t m; // min - uint8_t qh[4]; // 5-th bit of quants - uint8_t qs[QK5_1 / 2]; // nibbles / quants -} block_q5_1; -static_assert(sizeof(block_q5_1) == 2 * sizeof(ggml_fp16_t) + sizeof(uint32_t) + QK5_1 / 2, "wrong q5_1 block size/padding"); - -#define QK8_0 32 -typedef struct { - ggml_fp16_t d; // delta - int8_t qs[QK8_0]; // quants -} block_q8_0; -static_assert(sizeof(block_q8_0) == sizeof(ggml_fp16_t) + QK8_0, "wrong q8_0 block size/padding"); - -#define QK8_1 32 -typedef struct { - float d; // delta - float s; // d * sum(qs[i]) - int8_t qs[QK8_1]; // quants -} block_q8_1; -static_assert(sizeof(block_q8_1) == 2*sizeof(float) + QK8_1, "wrong q8_1 block size/padding"); - -// -// Super-block quantization structures -// - -// Super-block size -#ifdef GGML_QKK_64 -#define QK_K 64 -#define K_SCALE_SIZE 4 -#else -#define QK_K 256 -#define K_SCALE_SIZE 12 -#endif - -// 2-bit quantization -// weight is represented as x = a * q + b -// 16 blocks of 16 elements each -// Effectively 2.625 bits per weight -typedef struct { - uint8_t scales[QK_K/16]; // scales and mins, quantized with 4 bits - uint8_t qs[QK_K/4]; // quants - ggml_fp16_t d; // super-block scale for quantized scales - ggml_fp16_t dmin; // super-block scale for quantized mins -} block_q2_K; -static_assert(sizeof(block_q2_K) == 2*sizeof(ggml_fp16_t) + QK_K/16 + QK_K/4, "wrong q2_K block size/padding"); - -// 3-bit quantization -// weight is represented as x = a * q -// 16 blocks of 16 elements each -// Effectively 3.4375 bits per weight -#ifdef GGML_QKK_64 -typedef struct { - uint8_t hmask[QK_K/8]; // quants - high bit - uint8_t qs[QK_K/4]; // quants - low 2 bits - uint8_t scales[2]; - ggml_fp16_t d; // super-block scale -} block_q3_K; -static_assert(sizeof(block_q3_K) == sizeof(ggml_fp16_t) + QK_K / 4 + QK_K / 8 + 2, "wrong q3_K block size/padding"); -#else -typedef struct { - uint8_t hmask[QK_K/8]; // quants - high bit - uint8_t qs[QK_K/4]; // quants - low 2 bits - uint8_t scales[12]; // scales, quantized with 6 bits - ggml_fp16_t d; // super-block scale -} block_q3_K; -static_assert(sizeof(block_q3_K) == sizeof(ggml_fp16_t) + QK_K / 4 + QK_K / 8 + 12, "wrong q3_K block size/padding"); -#endif - -// 4-bit quantization -// 8 blocks of 32 elements each -// weight is represented as x = a * q + b -// Effectively 4.5 bits per weight -#ifdef GGML_QKK_64 -typedef struct { - ggml_fp16_t d[2]; // super-block scales/mins - uint8_t scales[2]; // 4-bit block scales/mins - uint8_t qs[QK_K/2]; // 4--bit quants -} block_q4_K; -static_assert(sizeof(block_q4_K) == 2*sizeof(ggml_fp16_t) + QK_K/2 + 2, "wrong q4_K block size/padding"); -#else -typedef struct { - ggml_fp16_t d; // super-block scale for quantized scales - ggml_fp16_t dmin; // super-block scale for quantized mins - uint8_t scales[K_SCALE_SIZE]; // scales and mins, quantized with 6 bits - uint8_t qs[QK_K/2]; // 4--bit quants -} block_q4_K; -static_assert(sizeof(block_q4_K) == 2*sizeof(ggml_fp16_t) + K_SCALE_SIZE + QK_K/2, "wrong q4_K block size/padding"); -#endif - -// 5-bit quantization -// 8 blocks of 32 elements each -// weight is represented as x = a * q + b -// Effectively 5.5 bits per weight -#ifdef GGML_QKK_64 -typedef struct { - ggml_fp16_t d; // super-block scale - int8_t scales[QK_K/16]; // 8-bit block scales - uint8_t qh[QK_K/8]; // quants, high bit - uint8_t qs[QK_K/2]; // quants, low 4 bits -} block_q5_K; -static_assert(sizeof(block_q5_K) == sizeof(ggml_fp16_t) + QK_K/2 + QK_K/8 + QK_K/16, "wrong q5_K block size/padding"); -#else -typedef struct { - ggml_fp16_t d; // super-block scale for quantized scales - ggml_fp16_t dmin; // super-block scale for quantized mins - uint8_t scales[K_SCALE_SIZE]; // scales and mins, quantized with 6 bits - uint8_t qh[QK_K/8]; // quants, high bit - uint8_t qs[QK_K/2]; // quants, low 4 bits -} block_q5_K; -static_assert(sizeof(block_q5_K) == 2*sizeof(ggml_fp16_t) + K_SCALE_SIZE + QK_K/2 + QK_K/8, "wrong q5_K block size/padding"); -#endif - -// 6-bit quantization -// weight is represented as x = a * q -// 16 blocks of 16 elements each -// Effectively 6.5625 bits per weight -typedef struct { - uint8_t ql[QK_K/2]; // quants, lower 4 bits - uint8_t qh[QK_K/4]; // quants, upper 2 bits - int8_t scales[QK_K/16]; // scales, quantized with 8 bits - ggml_fp16_t d; // super-block scale -} block_q6_K; -static_assert(sizeof(block_q6_K) == sizeof(ggml_fp16_t) + QK_K / 16 + 3*QK_K/4, "wrong q6_K block size/padding"); - -// This is only used for intermediate quantization and dot products -typedef struct { - float d; // delta - int8_t qs[QK_K]; // quants - int16_t bsums[QK_K/16]; // sum of quants in groups of 16 -} block_q8_K; -static_assert(sizeof(block_q8_K) == sizeof(float) + QK_K + QK_K/16*sizeof(int16_t), "wrong q8_K block size/padding"); - -// (Almost) "true" 2-bit quantization. -// Due to the need to use blocks as per ggml design, it ends up using -// 2.0625 bpw because of the 16-bit scale for each block of 256. -typedef struct { - ggml_fp16_t d; - uint16_t qs[QK_K/8]; -} block_iq2_xxs; -static_assert(sizeof(block_iq2_xxs) == sizeof(ggml_fp16_t) + QK_K/8*sizeof(uint16_t), "wrong iq2_xxs block size/padding"); - -// 2.3125 bpw quants -typedef struct { - ggml_fp16_t d; - uint16_t qs[QK_K/8]; - uint8_t scales[QK_K/32]; -} block_iq2_xs; -static_assert(sizeof(block_iq2_xs) == sizeof(ggml_fp16_t) + QK_K/8*sizeof(uint16_t) + QK_K/32, "wrong iq2_xs block size/padding"); - -// 2.5625 bpw quants -typedef struct { - ggml_fp16_t d; - uint8_t qs[QK_K/4]; - uint8_t qh[QK_K/32]; - uint8_t scales[QK_K/32]; -} block_iq2_s; -static_assert(sizeof(block_iq2_s) == sizeof(ggml_fp16_t) + QK_K/4 + QK_K/16, "wrong iq2_s block size/padding"); - -// (Almost) "true" 3-bit quantization. -// Due to the need to use blocks as per ggml design, it ends up using -// 3.0625 bpw because of the 16-bit scale for each block of 256. -typedef struct { - ggml_fp16_t d; - uint8_t qs[3*QK_K/8]; -} block_iq3_xxs; -static_assert(sizeof(block_iq3_xxs) == sizeof(ggml_fp16_t) + 3*(QK_K/8), "wrong iq3_xxs block size/padding"); - -// 3.4375 bpw -#if QK_K == 64 -#define IQ3S_N_SCALE 2 -#else -#define IQ3S_N_SCALE QK_K/64 -#endif -typedef struct { - ggml_fp16_t d; - uint8_t qs[QK_K/4]; - uint8_t qh[QK_K/32]; - uint8_t signs[QK_K/8]; - uint8_t scales[IQ3S_N_SCALE]; -} block_iq3_s; -static_assert(sizeof(block_iq3_s) == sizeof(ggml_fp16_t) + 13*(QK_K/32) + IQ3S_N_SCALE, "wrong iq3_s block size/padding"); - -typedef struct { - ggml_fp16_t d; - uint8_t qs[QK_K/8]; - uint16_t qh[QK_K/32]; -} block_iq1_s; -static_assert(sizeof(block_iq1_s) == sizeof(ggml_fp16_t) + QK_K/8 + QK_K/16, "wrong iq1_s block size/padding"); - -// Non-linear quants -#define QK4_NL 32 -typedef struct { - ggml_fp16_t d; - uint8_t qs[QK4_NL/2]; -} block_iq4_nl; -static_assert(sizeof(block_iq4_nl) == sizeof(ggml_fp16_t) + QK4_NL/2, "wrong iq4_nl block size/padding"); - -#if QK_K == 64 -#define block_iq4_xs block_iq4_nl -//typedef struct block_iq4_nl block_iq4_xs; -#else -typedef struct { - ggml_fp16_t d; - uint16_t scales_h; - uint8_t scales_l[QK_K/64]; - uint8_t qs[QK_K/2]; -} block_iq4_xs; -static_assert(sizeof(block_iq4_xs) == sizeof(ggml_fp16_t) + sizeof(uint16_t) + QK_K/64 + QK_K/2, "wrong iq4_xs block size/padding"); -#endif - #ifdef __cplusplus extern "C" { #endif diff --git a/ggml-sycl.cpp b/ggml-sycl.cpp index 24187e7d5..cfb09934d 100644 --- a/ggml-sycl.cpp +++ b/ggml-sycl.cpp @@ -3144,6 +3144,7 @@ namespace dpct } // COPY from DPCT head files +#define GGML_COMMON_DECL_SYCL #define GGML_COMMON_IMPL_SYCL #include "ggml-common.h" @@ -3312,66 +3313,6 @@ typedef void (*ggml_sycl_op_flatten_t)(const ggml_tensor *src0, const float *src1_dd, float *dst_dd, const dpct::queue_ptr &main_stream); -// QK = number of values after dequantization -// QR = QK / number of values before dequantization -// QI = number of 32 bit integers before dequantization - -#define QK4_0 32 -#define QR4_0 2 -#define QI4_0 (QK4_0 / (4 * QR4_0)) -typedef struct dpct_type_block_q4_0 { - sycl::half d; // delta - uint8_t qs[QK4_0 / 2]; // nibbles / quants -} block_q4_0; -static_assert(sizeof(block_q4_0) == sizeof(ggml_fp16_t) + QK4_0 / 2, "wrong q4_0 block size/padding"); - -#define QK4_1 32 -#define QR4_1 2 -#define QI4_1 (QK4_1 / (4 * QR4_1)) -typedef struct dpct_type_block_q4_1 { - sycl::half2 dm; // dm.x = delta, dm.y = min - uint8_t qs[QK4_1 / 2]; // nibbles / quants -} block_q4_1; -static_assert(sizeof(block_q4_1) == sizeof(ggml_fp16_t) * 2 + QK4_1 / 2, "wrong q4_1 block size/padding"); - -#define QK5_0 32 -#define QR5_0 2 -#define QI5_0 (QK5_0 / (4 * QR5_0)) -typedef struct dpct_type_block_q5_0 { - sycl::half d; // delta - uint8_t qh[4]; // 5-th bit of quants - uint8_t qs[QK5_0 / 2]; // nibbles / quants -} block_q5_0; -static_assert(sizeof(block_q5_0) == sizeof(ggml_fp16_t) + sizeof(uint32_t) + QK5_0 / 2, "wrong q5_0 block size/padding"); - -#define QK5_1 32 -#define QR5_1 2 -#define QI5_1 (QK5_1 / (4 * QR5_1)) -typedef struct dpct_type_block_q5_1 { - sycl::half2 dm; // dm.x = delta, dm.y = min - uint8_t qh[4]; // 5-th bit of quants - uint8_t qs[QK5_1 / 2]; // nibbles / quants -} block_q5_1; -static_assert(sizeof(block_q5_1) == 2 * sizeof(ggml_fp16_t) + sizeof(uint32_t) + QK5_1 / 2, "wrong q5_1 block size/padding"); - -#define QK8_0 32 -#define QR8_0 1 -#define QI8_0 (QK8_0 / (4 * QR8_0)) -typedef struct dpct_type_block_q8_0 { - sycl::half d; // delta - int8_t qs[QK8_0]; // quants -} block_q8_0; -static_assert(sizeof(block_q8_0) == sizeof(ggml_fp16_t) + QK8_0, "wrong q8_0 block size/padding"); - -#define QK8_1 32 -#define QR8_1 1 -#define QI8_1 (QK8_1 / (4 * QR8_1)) -typedef struct dpct_type_block_q8_1 { - sycl::half2 ds; // ds.x = delta, ds.y = sum - int8_t qs[QK8_0]; // quants -} block_q8_1; -static_assert(sizeof(block_q8_1) == 2*sizeof(ggml_fp16_t) + QK8_0, "wrong q8_1 block size/padding"); - typedef float (*vec_dot_q_sycl_t)(const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs); typedef void (*allocate_tiles_sycl_t)(int **x_ql, sycl::half2 **x_dm, int **x_qh, int **x_sc); @@ -3388,137 +3329,6 @@ typedef float (*vec_dot_q_mul_mat_sycl_t)( const int *__restrict__ y_qs, const sycl::half2 *__restrict__ y_ms, const int &i, const int &j, const int &k); -//================================= k-quants - -#ifdef GGML_QKK_64 -#define QK_K 64 -#define K_SCALE_SIZE 4 -#else -#define QK_K 256 -#define K_SCALE_SIZE 12 -#endif - -#define QR2_K 4 -#define QI2_K (QK_K / (4*QR2_K)) -typedef struct dpct_type_block_q2_K { - uint8_t scales[QK_K/16]; // scales and mins, quantized with 4 bits - uint8_t qs[QK_K/4]; // quants - sycl::half2 dm; // super-block scale for quantized scales/mins -} block_q2_K; -static_assert(sizeof(block_q2_K) == 2*sizeof(ggml_fp16_t) + QK_K/16 + QK_K/4, "wrong q2_K block size/padding"); - -#define QR3_K 4 -#define QI3_K (QK_K / (4*QR3_K)) -typedef struct dpct_type_block_q3_K { - uint8_t hmask[QK_K/8]; // quants - high bit - uint8_t qs[QK_K/4]; // quants - low 2 bits -#ifdef GGML_QKK_64 - uint8_t scales[2]; // scales, quantized with 8 bits -#else - uint8_t scales[K_SCALE_SIZE]; // scales, quantized with 6 bits -#endif - sycl::half d; // super-block scale -} block_q3_K; -//static_assert(sizeof(block_q3_K) == sizeof(ggml_fp16_t) + QK_K / 4 + QK_K / 8 + K_SCALE_SIZE, "wrong q3_K block size/padding"); - -#define QR4_K 2 -#define QI4_K (QK_K / (4*QR4_K)) -#ifdef GGML_QKK_64 -typedef struct { - sycl::half dm[2]; // super-block scales/mins - uint8_t scales[2]; // 4-bit block scales/mins - uint8_t qs[QK_K/2]; // 4--bit quants -} block_q4_K; -static_assert(sizeof(block_q4_K) == sizeof(sycl::half2) + QK_K/2 + 2, "wrong q4_K block size/padding"); -#else -typedef struct dpct_type_block_q4_K { - sycl::half2 dm; // super-block scale for quantized scales/mins - uint8_t scales[3*QK_K/64]; // scales, quantized with 6 bits - uint8_t qs[QK_K/2]; // 4--bit quants -} block_q4_K; -static_assert(sizeof(block_q4_K) == 2*sizeof(ggml_fp16_t) + 3*QK_K/64 + QK_K/2, "wrong q4_K block size/padding"); -#endif - -#define QR5_K 2 -#define QI5_K (QK_K / (4*QR5_K)) -#ifdef GGML_QKK_64 -typedef struct { - sycl::half d; // super-block scale - int8_t scales[QK_K/16]; // block scales - uint8_t qh[QK_K/8]; // quants, high bit - uint8_t qs[QK_K/2]; // quants, low 4 bits -} block_q5_K; -static_assert(sizeof(block_q5_K) == sizeof(ggml_fp16_t) + QK_K/2 + QK_K/8 + QK_K/16, "wrong q5_K block size/padding"); -#else -typedef struct dpct_type_block_q5_K { - sycl::half2 dm; // super-block scale for quantized scales/mins - uint8_t scales[K_SCALE_SIZE]; // scales and mins, quantized with 6 bits - uint8_t qh[QK_K/8]; // quants, high bit - uint8_t qs[QK_K/2]; // quants, low 4 bits -} block_q5_K; -static_assert(sizeof(block_q5_K) == 2*sizeof(ggml_fp16_t) + K_SCALE_SIZE + QK_K/2 + QK_K/8, "wrong q5_K block size/padding"); -#endif - -#define QR6_K 2 -#define QI6_K (QK_K / (4*QR6_K)) -typedef struct dpct_type_block_q6_K { - uint8_t ql[QK_K/2]; // quants, lower 4 bits - uint8_t qh[QK_K/4]; // quants, upper 2 bits - int8_t scales[QK_K/16]; // scales - sycl::half d; // delta -} block_q6_K; -static_assert(sizeof(block_q6_K) == sizeof(ggml_fp16_t) + 13*QK_K/16, "wrong q6_K block size/padding"); - -#define QR2_XXS 8 -#define QI2_XXS (QK_K / (4*QR2_XXS)) -typedef struct dpct_type_block_iq2_xxs { - sycl::half d; - uint16_t qs[QK_K/8]; -} block_iq2_xxs; -static_assert(sizeof(block_iq2_xxs) == sizeof(ggml_fp16_t) + QK_K/8*sizeof(uint16_t), "wrong iq2_xxs block size/padding"); - -#define QR2_XS 8 -#define QI2_XS (QK_K / (4*QR2_XS)) -typedef struct dpct_type_block_iq2_xs { - sycl::half d; - uint16_t qs[QK_K/8]; - uint8_t scales[QK_K/32]; -} block_iq2_xs; -static_assert(sizeof(block_iq2_xs) == sizeof(ggml_fp16_t) + QK_K/8*sizeof(uint16_t) + QK_K/32, "wrong iq2_xs block size/padding"); - -#define QR3_XXS 8 -#define QI3_XXS (QK_K / (4*QR3_XXS)) -typedef struct dpct_type_block_iq3_xxs { - sycl::half d; - uint8_t qs[3*(QK_K/8)]; -} block_iq3_xxs; -static_assert(sizeof(block_iq3_xxs) == sizeof(ggml_fp16_t) + 3*(QK_K/8), "wrong iq3_xxs block size/padding"); - -#define QR3_XS 8 -#define QI3_XS (QK_K / (4*QR3_XS)) -#if QK_K == 64 -#define IQ3S_N_SCALE 2 -#else -#define IQ3S_N_SCALE QK_K/64 -#endif -typedef struct { - sycl::half d; - uint8_t qs[QK_K/4]; - uint8_t qh[QK_K/32]; - uint8_t signs[QK_K/8]; - uint8_t scales[IQ3S_N_SCALE]; -} block_iq3_s; -static_assert(sizeof(block_iq3_s) == sizeof(ggml_fp16_t) + 13*(QK_K/32) + IQ3S_N_SCALE, "wrong iq3_s block size/padding"); - -#define QR1_S 8 -#define QI1_S (QK_K / (4*QR1_S)) -typedef struct { - sycl::half d; - uint8_t qs[QK_K/8]; - uint16_t qh[QK_K/32]; -} block_iq1_s; -static_assert(sizeof(block_iq1_s) == sizeof(ggml_fp16_t) + QK_K/8 + QK_K/16, "wrong iq1_s block size/padding"); - #define WARP_SIZE 32 #define MATRIX_ROW_PADDING 512 // last row of quant. matrices is a multiple of this to avoid out-of-bounds memory accesses From 306d34be7ad19e768975409fc80791a274ea0230 Mon Sep 17 00:00:00 2001 From: slaren Date: Tue, 12 Mar 2024 16:55:19 +0100 Subject: [PATCH 605/812] ci : remove tidy-review (#6021) --- .github/workflows/tidy-post.yml | 20 -------------------- .github/workflows/tidy-review.yml | 23 ----------------------- 2 files changed, 43 deletions(-) delete mode 100644 .github/workflows/tidy-post.yml delete mode 100644 .github/workflows/tidy-review.yml diff --git a/.github/workflows/tidy-post.yml b/.github/workflows/tidy-post.yml deleted file mode 100644 index 03652760c..000000000 --- a/.github/workflows/tidy-post.yml +++ /dev/null @@ -1,20 +0,0 @@ -name: clang-tidy review post comments - -on: - workflow_dispatch: - workflows: ["clang-tidy-review"] - types: - - completed - -jobs: - build: - runs-on: ubuntu-latest - - steps: - - uses: ZedThree/clang-tidy-review/post@v0.13.0 - # lgtm_comment_body, max_comments, and annotations need to be set on the posting workflow in a split setup - with: - # adjust options as necessary - lgtm_comment_body: '' - annotations: false - max_comments: 25 diff --git a/.github/workflows/tidy-review.yml b/.github/workflows/tidy-review.yml deleted file mode 100644 index a4bc8d976..000000000 --- a/.github/workflows/tidy-review.yml +++ /dev/null @@ -1,23 +0,0 @@ -name: clang-tidy-review - -on: - pull_request: - branches: - - master - -jobs: - clang-tidy-review: - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v3 - - - uses: ZedThree/clang-tidy-review@v0.13.0 - id: review - with: - lgtm_comment_body: '' - build_dir: build - cmake_command: cmake . -B build -DCMAKE_EXPORT_COMPILE_COMMANDS=on - split_workflow: true - - - uses: ZedThree/clang-tidy-review/upload@v0.13.0 From 99b71c068f624521ad977e08e41589e2971fa1c7 Mon Sep 17 00:00:00 2001 From: Xuan Son Nguyen Date: Wed, 13 Mar 2024 11:39:11 +0100 Subject: [PATCH 606/812] Server: Use multi-task for embeddings endpoint (#6001) * use multitask for embd endpoint * specify types * remove redundant {"n_predict", 0} --- examples/server/server.cpp | 76 ++++++++++++++------------------------ examples/server/utils.hpp | 12 +++++- 2 files changed, 38 insertions(+), 50 deletions(-) diff --git a/examples/server/server.cpp b/examples/server/server.cpp index b63a6f243..3172d96dd 100644 --- a/examples/server/server.cpp +++ b/examples/server/server.cpp @@ -2763,6 +2763,7 @@ int main(int argc, char ** argv) { res.set_header("Access-Control-Allow-Credentials", "true"); res.set_header("Access-Control-Allow-Methods", "POST"); res.set_header("Access-Control-Allow-Headers", "*"); + return res.set_content("", "application/json; charset=utf-8"); }); svr->set_logger(log_server_request); @@ -3371,44 +3372,37 @@ int main(int argc, char ** argv) { const json body = json::parse(req.body); bool is_openai = false; - // an input prompt can string or a list of tokens (integer) - std::vector prompts; + // an input prompt can be a string or a list of tokens (integer) + json prompt; if (body.count("input") != 0) { is_openai = true; - if (body["input"].is_array()) { - // support multiple prompts - for (const json & elem : body["input"]) { - prompts.push_back(elem); - } - } else { - // single input prompt - prompts.push_back(body["input"]); - } + prompt = body["input"]; } else if (body.count("content") != 0) { - // only support single prompt here - std::string content = body["content"]; - prompts.push_back(content); + // with "content", we only support single prompt + prompt = std::vector{body["content"]}; } else { res_error(res, format_error_response("\"input\" or \"content\" must be provided", ERROR_TYPE_INVALID_REQUEST)); return; } - // process all prompts - json responses = json::array(); - for (auto & prompt : prompts) { - // TODO @ngxson : maybe support multitask for this endpoint? - // create and queue the task + // create and queue the task + json responses; + { const int id_task = ctx_server.queue_tasks.get_new_id(); - ctx_server.queue_results.add_waiting_task_id(id_task); - ctx_server.request_completion(id_task, -1, { {"prompt", prompt}, { "n_predict", 0}}, false, true); + ctx_server.request_completion(id_task, -1, {{"prompt", prompt}}, false, true); // get the result server_task_result result = ctx_server.queue_results.recv(id_task); ctx_server.queue_results.remove_waiting_task_id(id_task); if (!result.error) { - // append to the responses - responses.push_back(result.data); + if (result.data.count("results")) { + // result for multi-task + responses = result.data["results"]; + } else { + // result for single task + responses = std::vector{result.data}; + } } else { // error received, ignore everything else res_error(res, result.data); @@ -3417,24 +3411,19 @@ int main(int argc, char ** argv) { } // write JSON response - json root; - if (is_openai) { - json res_oai = json::array(); - int i = 0; - for (auto & elem : responses) { - res_oai.push_back(json{ - {"embedding", json_value(elem, "embedding", json::array())}, - {"index", i++}, - {"object", "embedding"} - }); - } - root = format_embeddings_response_oaicompat(body, res_oai); - } else { - root = responses[0]; - } + json root = is_openai + ? format_embeddings_response_oaicompat(body, responses) + : responses[0]; return res.set_content(root.dump(), "application/json; charset=utf-8"); }; + auto handle_static_file = [](unsigned char * content, size_t len, const char * mime_type) { + return [content, len, mime_type](const httplib::Request &, httplib::Response & res) { + res.set_content(reinterpret_cast(content), len, mime_type); + return false; + }; + }; + // // Router // @@ -3446,17 +3435,6 @@ int main(int argc, char ** argv) { } // using embedded static files - auto handle_static_file = [](unsigned char * content, size_t len, const char * mime_type) { - return [content, len, mime_type](const httplib::Request &, httplib::Response & res) { - res.set_content(reinterpret_cast(content), len, mime_type); - return false; - }; - }; - - svr->Options(R"(/.*)", [](const httplib::Request &, httplib::Response & res) { - // TODO @ngxson : I have no idea what it is... maybe this is redundant? - return res.set_content("", "application/json; charset=utf-8"); - }); svr->Get("/", handle_static_file(index_html, index_html_len, "text/html; charset=utf-8")); svr->Get("/index.js", handle_static_file(index_js, index_js_len, "text/javascript; charset=utf-8")); svr->Get("/completion.js", handle_static_file(completion_js, completion_js_len, "text/javascript; charset=utf-8")); diff --git a/examples/server/utils.hpp b/examples/server/utils.hpp index 48aeef4eb..2ddb2cd21 100644 --- a/examples/server/utils.hpp +++ b/examples/server/utils.hpp @@ -529,6 +529,16 @@ static std::vector format_partial_response_oaicompat(json result, const st } static json format_embeddings_response_oaicompat(const json & request, const json & embeddings) { + json data = json::array(); + int i = 0; + for (auto & elem : embeddings) { + data.push_back(json{ + {"embedding", json_value(elem, "embedding", json::array())}, + {"index", i++}, + {"object", "embedding"} + }); + } + json res = json { {"model", json_value(request, "model", std::string(DEFAULT_OAICOMPAT_MODEL))}, {"object", "list"}, @@ -536,7 +546,7 @@ static json format_embeddings_response_oaicompat(const json & request, const jso {"prompt_tokens", 0}, {"total_tokens", 0} }}, - {"data", embeddings} + {"data", data} }; return res; From b3d978600f07f22e94f2e797f18a8b5f6df23c89 Mon Sep 17 00:00:00 2001 From: AidanBeltonS <87009434+AidanBeltonS@users.noreply.github.com> Date: Wed, 13 Mar 2024 13:17:54 +0000 Subject: [PATCH 607/812] Update get version (#6025) --- ggml-sycl.cpp | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/ggml-sycl.cpp b/ggml-sycl.cpp index cfb09934d..c2ab13034 100644 --- a/ggml-sycl.cpp +++ b/ggml-sycl.cpp @@ -202,24 +202,29 @@ namespace dpct // Version string has the following format: // a. OpenCL // b. + // c. e.g gfx1030 std::string ver; ver = dev.get_info(); std::string::size_type i = 0; - while (i < ver.size()) - { - if (isdigit(ver[i])) - break; - i++; + while (i < ver.size()) { + if (isdigit(ver[i])) + break; + i++; } major = std::stoi(&(ver[i])); - while (i < ver.size()) - { - if (ver[i] == '.') - break; - i++; + while (i < ver.size()) { + if (ver[i] == '.') + break; + i++; + } + if (i < ver.size()) { + // a. and b. + i++; + minor = std::stoi(&(ver[i])); + } else { + // c. + minor = 0; } - i++; - minor = std::stoi(&(ver[i])); } template From d8fd0ccf6ac8b07791ffd1575eed436930854ae3 Mon Sep 17 00:00:00 2001 From: slaren Date: Wed, 13 Mar 2024 14:58:30 +0100 Subject: [PATCH 608/812] test-backend-ops : skip CPU backend by default (#6028) --- tests/test-backend-ops.cpp | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index fc5edcc4b..c2916c3e4 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -2222,8 +2222,8 @@ static void usage(char ** argv) { int main(int argc, char ** argv) { test_mode mode = MODE_TEST; - const char * op_name = NULL; - const char * backend = NULL; + const char * op_name_filter = NULL; + const char * backend_filter = NULL; for (int i = 1; i < argc; i++) { if (strcmp(argv[i], "test") == 0) { @@ -2232,14 +2232,14 @@ int main(int argc, char ** argv) { mode = MODE_PERF; } else if (strcmp(argv[i], "-o") == 0) { if (i + 1 < argc) { - op_name = argv[++i]; + op_name_filter = argv[++i]; } else { usage(argv); return 1; } } else if (strcmp(argv[i], "-b") == 0) { if (i + 1 < argc) { - backend = argv[++i]; + backend_filter = argv[++i]; } else { usage(argv); return 1; @@ -2258,7 +2258,7 @@ int main(int argc, char ** argv) { for (size_t i = 0; i < ggml_backend_reg_get_count(); i++) { printf("Backend %zu/%zu (%s)\n", i + 1, ggml_backend_reg_get_count(), ggml_backend_reg_get_name(i)); - if (backend != NULL && strcmp(backend, ggml_backend_reg_get_name(i)) != 0) { + if (backend_filter != NULL && strcmp(backend_filter, ggml_backend_reg_get_name(i)) != 0) { printf(" Skipping\n"); n_ok++; continue; @@ -2266,9 +2266,17 @@ int main(int argc, char ** argv) { ggml_backend_t backend = ggml_backend_reg_init_backend(i, NULL); GGML_ASSERT(backend != NULL); + + if (backend_filter == NULL && ggml_backend_is_cpu(backend)) { + printf(" Skipping CPU backend\n"); + ggml_backend_free(backend); + n_ok++; + continue; + } + printf(" Backend name: %s\n", ggml_backend_name(backend)); - bool ok = test_backend(backend, mode, op_name); + bool ok = test_backend(backend, mode, op_name_filter); printf(" Backend %s: ", ggml_backend_name(backend)); if (ok) { From f30ea47a87ed4446ad55adb265755dc9102956a2 Mon Sep 17 00:00:00 2001 From: slaren Date: Wed, 13 Mar 2024 18:54:21 +0100 Subject: [PATCH 609/812] llama : add pipeline parallelism support (#6017) * llama : add pipeline parallelism support for batch processing with multiple CUDA GPUs ggml-ci * server : add -ub, --ubatch-size parameter * fix server embedding test * llama : fix Mamba inference for pipeline parallelism Tested to work correctly with both `main` and `parallel` examples. * llama : limit max batch size to n_batch * add LLAMA_SCHED_MAX_COPIES to configure the number of input copies for pipeline parallelism default increase to 4 (from 2) changing this value may improve performance for some systems, but increases memory usage * fix hip build * fix sycl build (disable cpy_tensor_async) * fix hip build * llama : limit n_batch and n_ubatch to n_ctx during context creation * llama : fix norm backend * batched-bench : sync after decode * swiftui : sync after decode * ggml : allow ggml_get_rows to use multiple threads if they are available * check n_ubatch >= n_tokens with non-casual attention * llama : do not limit n_batch to n_ctx with non-casual attn * server : construct batch with size of llama_n_batch * ggml_backend_cpu_graph_compute : fix return value when alloc fails * llama : better n_batch and n_ubatch comment * fix merge * small fix * reduce default n_batch to 2048 --------- Co-authored-by: Francis Couture-Harpin Co-authored-by: Georgi Gerganov --- CMakeLists.txt | 3 + Makefile | 4 + common/common.cpp | 14 +- common/common.h | 3 +- examples/batched-bench/batched-bench.cpp | 2 + examples/embedding/embedding.cpp | 2 +- examples/llama-bench/llama-bench.cpp | 53 +- .../llama.cpp.swift/LibLlama.swift | 2 + examples/perplexity/perplexity.cpp | 3 +- examples/server/server.cpp | 32 +- .../server/tests/features/embeddings.feature | 1 + examples/server/tests/features/steps/steps.py | 8 + ggml-alloc.c | 109 +- ggml-alloc.h | 18 +- ggml-backend-impl.h | 17 +- ggml-backend.c | 517 +++++-- ggml-backend.h | 58 +- ggml-cuda.cu | 175 ++- ggml-kompute.cpp | 5 + ggml-metal.m | 5 + ggml-sycl.cpp | 7 +- ggml-vulkan.cpp | 5 + ggml.c | 113 +- llama.cpp | 1189 +++++++++-------- llama.h | 9 +- 25 files changed, 1467 insertions(+), 887 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 7ab13cbd5..a8abf4088 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -118,6 +118,7 @@ option(LLAMA_SYCL "llama: use SYCL" option(LLAMA_SYCL_F16 "llama: use 16 bit floats for sycl calculations" OFF) set(LLAMA_SYCL_TARGET "INTEL" CACHE STRING "llama: sycl target device") option(LLAMA_CPU_HBM "llama: use memkind for CPU HBM" OFF) +set(LLAMA_SCHED_MAX_COPIES "4" CACHE STRING "llama: max input copies for pipeline parallelism") option(LLAMA_BUILD_TESTS "llama: build tests" ${LLAMA_STANDALONE}) option(LLAMA_BUILD_EXAMPLES "llama: build examples" ${LLAMA_STANDALONE}) @@ -147,6 +148,8 @@ set(THREADS_PREFER_PTHREAD_FLAG ON) find_package(Threads REQUIRED) include(CheckCXXCompilerFlag) +add_compile_definitions(GGML_SCHED_MAX_COPIES=${LLAMA_SCHED_MAX_COPIES}) + # enable libstdc++ assertions for debug builds if (CMAKE_SYSTEM_NAME MATCHES "Linux") add_compile_definitions($<$:_GLIBCXX_ASSERTIONS>) diff --git a/Makefile b/Makefile index c8fd3f5c5..db9968efb 100644 --- a/Makefile +++ b/Makefile @@ -167,6 +167,10 @@ ifeq ($(UNAME_S),OpenBSD) MK_CPPFLAGS += -D_BSD_SOURCE endif +ifdef LLAMA_SCHED_MAX_COPIES + MK_CPPFLAGS += -DGGML_SCHED_MAX_COPIES=$(LLAMA_SCHED_MAX_COPIES) +endif + ifdef LLAMA_DEBUG MK_CFLAGS += -O0 -g MK_CXXFLAGS += -O0 -g diff --git a/common/common.cpp b/common/common.cpp index 2f38ac632..73b1b61ba 100644 --- a/common/common.cpp +++ b/common/common.cpp @@ -483,6 +483,12 @@ bool gpt_params_parse_ex(int argc, char ** argv, gpt_params & params) { break; } params.n_batch = std::stoi(argv[i]); + } else if (arg == "-ub" || arg == "--ubatch-size") { + if (++i >= argc) { + invalid_param = true; + break; + } + params.n_ubatch = std::stoi(argv[i]); } else if (arg == "--keep") { if (++i >= argc) { invalid_param = true; @@ -977,7 +983,9 @@ void gpt_print_usage(int /*argc*/, char ** argv, const gpt_params & params) { printf(" binary file containing multiple choice tasks.\n"); printf(" -n N, --n-predict N number of tokens to predict (default: %d, -1 = infinity, -2 = until context filled)\n", params.n_predict); printf(" -c N, --ctx-size N size of the prompt context (default: %d, 0 = loaded from model)\n", params.n_ctx); - printf(" -b N, --batch-size N batch size for prompt processing (default: %d)\n", params.n_batch); + printf(" -b N, --batch-size N logical maximum batch size (default: %d)\n", params.n_batch); + printf(" -ub N, --ubatch-size N\n"); + printf(" physical maximum batch size (default: %d)\n", params.n_ubatch); printf(" --samplers samplers that will be used for generation in the order, separated by \';\'\n"); printf(" (default: %s)\n", sampler_type_names.c_str()); printf(" --sampling-seq simplified sequence for samplers that will be used (default: %s)\n", sampler_type_chars.c_str()); @@ -1287,8 +1295,9 @@ struct llama_context_params llama_context_params_from_gpt_params(const gpt_param auto cparams = llama_context_default_params(); cparams.n_ctx = params.n_ctx; - cparams.n_batch = params.n_batch; cparams.n_seq_max = params.n_parallel; + cparams.n_batch = params.n_batch; + cparams.n_ubatch = params.n_ubatch; cparams.n_threads = params.n_threads; cparams.n_threads_batch = params.n_threads_batch == -1 ? params.n_threads : params.n_threads_batch; cparams.seed = params.seed; @@ -1379,6 +1388,7 @@ std::tuple llama_init_from_gpt_par std::vector tmp = { llama_token_bos(model), llama_token_eos(model), }; llama_decode(lctx, llama_batch_get_one(tmp.data(), std::min(tmp.size(), (size_t) params.n_batch), 0, 0)); llama_kv_cache_clear(lctx); + llama_synchronize(lctx); llama_reset_timings(lctx); } diff --git a/common/common.h b/common/common.h index f8d82b871..0f178b9eb 100644 --- a/common/common.h +++ b/common/common.h @@ -51,7 +51,8 @@ struct gpt_params { int32_t n_threads_batch_draft = -1; int32_t n_predict = -1; // new tokens to predict int32_t n_ctx = 512; // context size - int32_t n_batch = 512; // batch size for prompt processing (must be >=32 to use BLAS) + int32_t n_batch = 2048; // logical batch size for prompt processing (must be >=32 to use BLAS) + int32_t n_ubatch = 512; // physical batch size for prompt processing (must be >=32 to use BLAS) int32_t n_keep = 0; // number of tokens to keep from initial prompt int32_t n_draft = 5; // number of tokens to draft during speculative decoding int32_t n_chunks = -1; // max number of chunks to process (-1 = unlimited) diff --git a/examples/batched-bench/batched-bench.cpp b/examples/batched-bench/batched-bench.cpp index 22bc93bca..19674dfd3 100644 --- a/examples/batched-bench/batched-bench.cpp +++ b/examples/batched-bench/batched-bench.cpp @@ -138,6 +138,8 @@ int main(int argc, char ** argv) { LOG_TEE("failed to decode the batch, n_batch = %d, ret = %d\n", n_batch, ret); return false; } + + llama_synchronize(ctx); } return true; diff --git a/examples/embedding/embedding.cpp b/examples/embedding/embedding.cpp index a553ae1c3..49302a199 100644 --- a/examples/embedding/embedding.cpp +++ b/examples/embedding/embedding.cpp @@ -107,7 +107,7 @@ int main(int argc, char ** argv) { // max batch size const uint64_t n_batch = params.n_batch; - GGML_ASSERT(params.n_batch == params.n_ctx); + GGML_ASSERT(params.n_batch >= params.n_ctx); // tokenize the prompts and trim std::vector> inputs; diff --git a/examples/llama-bench/llama-bench.cpp b/examples/llama-bench/llama-bench.cpp index 2ff86ef6f..bf94e7e7a 100644 --- a/examples/llama-bench/llama-bench.cpp +++ b/examples/llama-bench/llama-bench.cpp @@ -164,6 +164,7 @@ struct cmd_params { std::vector n_prompt; std::vector n_gen; std::vector n_batch; + std::vector n_ubatch; std::vector type_k; std::vector type_v; std::vector n_threads; @@ -183,7 +184,8 @@ static const cmd_params cmd_params_defaults = { /* model */ {"models/7B/ggml-model-q4_0.gguf"}, /* n_prompt */ {512}, /* n_gen */ {128}, - /* n_batch */ {512}, + /* n_batch */ {2048}, + /* n_ubatch */ {512}, /* type_k */ {GGML_TYPE_F16}, /* type_v */ {GGML_TYPE_F16}, /* n_threads */ {get_num_physical_cores()}, @@ -208,6 +210,7 @@ static void print_usage(int /* argc */, char ** argv) { printf(" -p, --n-prompt (default: %s)\n", join(cmd_params_defaults.n_prompt, ",").c_str()); printf(" -n, --n-gen (default: %s)\n", join(cmd_params_defaults.n_gen, ",").c_str()); printf(" -b, --batch-size (default: %s)\n", join(cmd_params_defaults.n_batch, ",").c_str()); + printf(" -ub N, --ubatch-size (default: %s)\n", join(cmd_params_defaults.n_ubatch, ",").c_str()); printf(" -ctk , --cache-type-k (default: %s)\n", join(transform_to_str(cmd_params_defaults.type_k, ggml_type_name), ",").c_str()); printf(" -ctv , --cache-type-v (default: %s)\n", join(transform_to_str(cmd_params_defaults.type_v, ggml_type_name), ",").c_str()); printf(" -t, --threads (default: %s)\n", join(cmd_params_defaults.n_threads, ",").c_str()); @@ -217,7 +220,7 @@ static void print_usage(int /* argc */, char ** argv) { printf(" -nkvo, --no-kv-offload <0|1> (default: %s)\n", join(cmd_params_defaults.no_kv_offload, ",").c_str()); printf(" -mmp, --mmap <0|1> (default: %s)\n", join(cmd_params_defaults.use_mmap, ",").c_str()); printf(" -embd, --embeddings <0|1> (default: %s)\n", join(cmd_params_defaults.embeddings, ",").c_str()); - printf(" -ts, --tensor_split (default: 0)\n"); + printf(" -ts, --tensor-split (default: 0)\n"); printf(" -r, --repetitions (default: %d)\n", cmd_params_defaults.reps); printf(" -o, --output (default: %s)\n", output_format_str(cmd_params_defaults.output_format)); printf(" -v, --verbose (default: %s)\n", cmd_params_defaults.verbose ? "1" : "0"); @@ -297,6 +300,13 @@ static cmd_params parse_cmd_params(int argc, char ** argv) { } auto p = split(argv[i], split_delim); params.n_batch.insert(params.n_batch.end(), p.begin(), p.end()); + } else if (arg == "-ub" || arg == "--ubatch-size") { + if (++i >= argc) { + invalid_param = true; + break; + } + auto p = split(argv[i], split_delim); + params.n_ubatch.insert(params.n_ubatch.end(), p.begin(), p.end()); } else if (arg == "-ctk" || arg == "--cache-type-k") { if (++i >= argc) { invalid_param = true; @@ -455,6 +465,7 @@ static cmd_params parse_cmd_params(int argc, char ** argv) { if (params.n_prompt.empty()) { params.n_prompt = cmd_params_defaults.n_prompt; } if (params.n_gen.empty()) { params.n_gen = cmd_params_defaults.n_gen; } if (params.n_batch.empty()) { params.n_batch = cmd_params_defaults.n_batch; } + if (params.n_ubatch.empty()) { params.n_ubatch = cmd_params_defaults.n_ubatch; } if (params.type_k.empty()) { params.type_k = cmd_params_defaults.type_k; } if (params.type_v.empty()) { params.type_v = cmd_params_defaults.type_v; } if (params.n_gpu_layers.empty()) { params.n_gpu_layers = cmd_params_defaults.n_gpu_layers; } @@ -474,6 +485,7 @@ struct cmd_params_instance { int n_prompt; int n_gen; int n_batch; + int n_ubatch; ggml_type type_k; ggml_type type_v; int n_threads; @@ -511,6 +523,7 @@ struct cmd_params_instance { cparams.n_ctx = n_prompt + n_gen; cparams.n_batch = n_batch; + cparams.n_ubatch = n_ubatch; cparams.type_k = type_k; cparams.type_v = type_v; cparams.offload_kqv = !no_kv_offload; @@ -532,6 +545,7 @@ static std::vector get_cmd_params_instances(const cmd_param for (const auto & mmp : params.use_mmap) for (const auto & embd : params.embeddings) for (const auto & nb : params.n_batch) + for (const auto & nub : params.n_ubatch) for (const auto & tk : params.type_k) for (const auto & tv : params.type_v) for (const auto & nkvo : params.no_kv_offload) @@ -545,6 +559,7 @@ static std::vector get_cmd_params_instances(const cmd_param /* .n_prompt = */ n_prompt, /* .n_gen = */ 0, /* .n_batch = */ nb, + /* .n_ubatch = */ nub, /* .type_k = */ tk, /* .type_v = */ tv, /* .n_threads = */ nt, @@ -568,6 +583,7 @@ static std::vector get_cmd_params_instances(const cmd_param /* .n_prompt = */ 0, /* .n_gen = */ n_gen, /* .n_batch = */ nb, + /* .n_ubatch = */ nub, /* .type_k = */ tk, /* .type_v = */ tv, /* .n_threads = */ nt, @@ -604,6 +620,7 @@ struct test { uint64_t model_size; uint64_t model_n_params; int n_batch; + int n_ubatch; int n_threads; ggml_type type_k; ggml_type type_v; @@ -627,6 +644,7 @@ struct test { model_size = llama_model_size(lmodel); model_n_params = llama_model_n_params(lmodel); n_batch = inst.n_batch; + n_ubatch = inst.n_ubatch; n_threads = inst.n_threads; type_k = inst.type_k; type_v = inst.type_v; @@ -705,7 +723,8 @@ struct test { "cuda", "opencl", "vulkan", "kompute", "metal", "sycl", "gpu_blas", "blas", "cpu_info", "gpu_info", "model_filename", "model_type", "model_size", "model_n_params", - "n_batch", "n_threads", "type_k", "type_v", + "n_batch", "n_ubatch", + "n_threads", "type_k", "type_v", "n_gpu_layers", "split_mode", "main_gpu", "no_kv_offload", "tensor_split", "use_mmap", "embeddings", @@ -719,7 +738,8 @@ struct test { enum field_type {STRING, BOOL, INT, FLOAT}; static field_type get_field_type(const std::string & field) { - if (field == "build_number" || field == "n_batch" || field == "n_threads" || + if (field == "build_number" || field == "n_batch" || field == "n_ubatch" || + field == "n_threads" || field == "model_size" || field == "model_n_params" || field == "n_gpu_layers" || field == "main_gpu" || field == "n_prompt" || field == "n_gen" || @@ -759,7 +779,8 @@ struct test { std::to_string(metal), std::to_string(sycl), std::to_string(gpu_blas), std::to_string(blas), cpu_info, gpu_info, model_filename, model_type, std::to_string(model_size), std::to_string(model_n_params), - std::to_string(n_batch), std::to_string(n_threads), ggml_type_name(type_k), ggml_type_name(type_v), + std::to_string(n_batch), std::to_string(n_ubatch), + std::to_string(n_threads), ggml_type_name(type_k), ggml_type_name(type_v), std::to_string(n_gpu_layers), split_mode_str(split_mode), std::to_string(main_gpu), std::to_string(no_kv_offload), tensor_split_str, std::to_string(use_mmap), std::to_string(embeddings), @@ -957,6 +978,9 @@ struct markdown_printer : public printer { if (params.n_batch.size() > 1 || params.n_batch != cmd_params_defaults.n_batch) { fields.emplace_back("n_batch"); } + if (params.n_ubatch.size() > 1 || params.n_ubatch != cmd_params_defaults.n_ubatch) { + fields.emplace_back("n_ubatch"); + } if (params.type_k.size() > 1 || params.type_k != cmd_params_defaults.type_k) { fields.emplace_back("type_k"); } @@ -1096,25 +1120,32 @@ struct sql_printer : public printer { }; static void test_prompt(llama_context * ctx, int n_prompt, int n_past, int n_batch, int n_threads) { + llama_set_n_threads(ctx, n_threads, n_threads); + + //std::vector tokens(n_prompt, llama_token_bos(llama_get_model(ctx))); + //llama_decode(ctx, llama_batch_get_one(tokens.data(), n_prompt, n_past, 0)); + //GGML_UNUSED(n_batch); + std::vector tokens(n_batch, llama_token_bos(llama_get_model(ctx))); int n_processed = 0; - llama_set_n_threads(ctx, n_threads, n_threads); - while (n_processed < n_prompt) { int n_tokens = std::min(n_prompt - n_processed, n_batch); llama_decode(ctx, llama_batch_get_one(tokens.data(), n_tokens, n_past + n_processed, 0)); n_processed += n_tokens; } + + llama_synchronize(ctx); } static void test_gen(llama_context * ctx, int n_gen, int n_past, int n_threads) { - llama_token token = llama_token_bos(llama_get_model(ctx)); - llama_set_n_threads(ctx, n_threads, n_threads); + llama_token token = llama_token_bos(llama_get_model(ctx)); + for (int i = 0; i < n_gen; i++) { llama_decode(ctx, llama_batch_get_one(&token, 1, n_past + i, 0)); + llama_synchronize(ctx); } } @@ -1203,7 +1234,8 @@ int main(int argc, char ** argv) { // warmup run if (t.n_prompt > 0) { - test_prompt(ctx, std::min(2, t.n_batch), 0, t.n_batch, t.n_threads); + //test_prompt(ctx, std::min(t.n_batch, std::min(t.n_prompt, 32)), 0, t.n_batch, t.n_threads); + test_prompt(ctx, t.n_prompt, 0, t.n_batch, t.n_threads); } if (t.n_gen > 0) { test_gen(ctx, 1, 0, t.n_threads); @@ -1219,6 +1251,7 @@ int main(int argc, char ** argv) { if (t.n_gen > 0) { test_gen(ctx, t.n_gen, t.n_prompt, t.n_threads); } + uint64_t t_ns = get_time_ns() - t_start; t.samples_ns.push_back(t_ns); } diff --git a/examples/llama.swiftui/llama.cpp.swift/LibLlama.swift b/examples/llama.swiftui/llama.cpp.swift/LibLlama.swift index 58fcf40c6..c249291ae 100644 --- a/examples/llama.swiftui/llama.cpp.swift/LibLlama.swift +++ b/examples/llama.swiftui/llama.cpp.swift/LibLlama.swift @@ -221,6 +221,7 @@ actor LlamaContext { if llama_decode(context, batch) != 0 { print("llama_decode() failed during prompt") } + llama_synchronize(context) let t_pp_end = ggml_time_us() @@ -240,6 +241,7 @@ actor LlamaContext { if llama_decode(context, batch) != 0 { print("llama_decode() failed during text generation") } + llama_synchronize(context) } let t_tg_end = ggml_time_us() diff --git a/examples/perplexity/perplexity.cpp b/examples/perplexity/perplexity.cpp index fdfc8f5dc..d766aef6a 100644 --- a/examples/perplexity/perplexity.cpp +++ b/examples/perplexity/perplexity.cpp @@ -589,9 +589,10 @@ static results_perplexity perplexity(llama_context * ctx, const gpt_params & par } } - const auto t_end = std::chrono::high_resolution_clock::now(); if (i == 0) { + llama_synchronize(ctx); + const auto t_end = std::chrono::high_resolution_clock::now(); const float t_total = std::chrono::duration(t_end - t_start).count(); fprintf(stderr, "%s: %.2f seconds per pass - ETA ", __func__, t_total); int total_seconds = (int)(t_total*n_chunk/n_seq); diff --git a/examples/server/server.cpp b/examples/server/server.cpp index 3172d96dd..895d608fd 100644 --- a/examples/server/server.cpp +++ b/examples/server/server.cpp @@ -147,7 +147,7 @@ struct server_slot { int32_t n_decoded = 0; int32_t n_remaining = -1; int32_t i_batch = -1; - int32_t n_predict = -1; + int32_t n_predict = -1; // TODO: disambiguate from params.n_predict int32_t n_prompt_tokens = 0; int32_t n_prompt_tokens_processed = 0; @@ -739,7 +739,13 @@ struct server_context { default_generation_settings_for_props = get_formated_generation(slots.front()); default_generation_settings_for_props["seed"] = -1; - batch = llama_batch_init(n_ctx, 0, params.n_parallel); + // the update_slots() logic will always submit a maximum of n_batch tokens + // note that n_batch can be > n_ctx (e.g. for non-causal attention models such as BERT where the KV cache is not used) + { + const int32_t n_batch = llama_n_batch(ctx); + + batch = llama_batch_init(n_batch, 0, params.n_parallel); + } metrics.init(); } @@ -1036,8 +1042,10 @@ struct server_context { llama_batch_add(batch, system_tokens[i], i, { 0 }, false); } - for (int32_t i = 0; i < (int32_t) batch.n_tokens; i += params.n_batch) { - const int32_t n_tokens = std::min(params.n_batch, (int32_t) (batch.n_tokens - i)); + const int32_t n_batch = llama_n_batch(ctx); + + for (int32_t i = 0; i < batch.n_tokens; i += n_batch) { + const int32_t n_tokens = std::min(params.n_batch, batch.n_tokens - i); llama_batch batch_view = { n_tokens, batch.token + i, @@ -1226,7 +1234,7 @@ struct server_context { {"mirostat_eta", slot.sparams.mirostat_eta}, {"penalize_nl", slot.sparams.penalize_nl}, {"stop", slot.params.antiprompt}, - {"n_predict", slot.params.n_predict}, + {"n_predict", slot.params.n_predict}, // TODO: fix duplicate key n_predict {"n_keep", params.n_keep}, {"ignore_eos", ignore_eos}, {"stream", slot.params.stream}, @@ -1738,7 +1746,8 @@ struct server_context { } // process in chunks of params.n_batch - int32_t n_batch = params.n_batch; + int32_t n_batch = llama_n_batch(ctx); + int32_t n_ubatch = llama_n_ubatch(ctx); // next, batch any pending prompts without exceeding n_batch if (params.cont_batching || batch.n_tokens == 0) { @@ -1811,7 +1820,7 @@ struct server_context { if (slot.embedding) { // this prompt is too large to process - discard it - if (slot.n_prompt_tokens > n_batch) { + if (slot.n_prompt_tokens > n_ubatch) { slot.state = SLOT_STATE_PROCESSING; slot.command = SLOT_COMMAND_NONE; slot.release(); @@ -2157,7 +2166,8 @@ static void server_print_usage(const char * argv0, const gpt_params & params, co printf(" --pooling {none,mean,cls} pooling type for embeddings, use model default if unspecified\n"); printf(" -dt N, --defrag-thold N\n"); printf(" KV cache defragmentation threshold (default: %.1f, < 0 - disabled)\n", params.defrag_thold); - printf(" -b N, --batch-size N batch size for prompt processing (default: %d)\n", params.n_batch); + printf(" -b N, --batch-size N logical maximum batch size (default: %d)\n", params.n_batch); + printf(" -ub N, --ubatch-size N physical maximum batch size (default: %d)\n", params.n_ubatch); printf(" --memory-f32 use f32 instead of f16 for memory key+value (default: disabled)\n"); printf(" not recommended: doubles context memory required and no measurable increase in quality\n"); if (llama_supports_mlock()) { @@ -2424,6 +2434,12 @@ static void server_params_parse(int argc, char ** argv, server_params & sparams, break; } params.n_batch = std::stoi(argv[i]); + } else if (arg == "-ub" || arg == "--ubatch-size") { + if (++i >= argc) { + invalid_param = true; + break; + } + params.n_ubatch = std::stoi(argv[i]); } else if (arg == "--gpu-layers" || arg == "-ngl" || arg == "--n-gpu-layers") { if (++i >= argc) { invalid_param = true; diff --git a/examples/server/tests/features/embeddings.feature b/examples/server/tests/features/embeddings.feature index b47661e94..57359b267 100644 --- a/examples/server/tests/features/embeddings.feature +++ b/examples/server/tests/features/embeddings.feature @@ -9,6 +9,7 @@ Feature: llama.cpp server And 42 as server seed And 2 slots And 1024 as batch size + And 1024 as ubatch size And 2048 KV cache size And embeddings extraction Then the server is starting diff --git a/examples/server/tests/features/steps/steps.py b/examples/server/tests/features/steps/steps.py index 98c2b6174..cfa9f96ec 100644 --- a/examples/server/tests/features/steps/steps.py +++ b/examples/server/tests/features/steps/steps.py @@ -33,6 +33,7 @@ def step_server_config(context, server_fqdn, server_port): context.model_alias = None context.n_batch = None + context.n_ubatch = None context.n_ctx = None context.n_ga = None context.n_ga_w = None @@ -278,6 +279,11 @@ def step_n_batch(context, n_batch): context.n_batch = n_batch +@step('{n_ubatch:d} as ubatch size') +def step_n_ubatch(context, n_ubatch): + context.n_ubatch = n_ubatch + + @step('{seed:d} as seed') def step_seed(context, seed): context.seed = seed @@ -1029,6 +1035,8 @@ def start_server_background(context): ] if context.n_batch: server_args.extend(['--batch-size', context.n_batch]) + if context.n_ubatch: + server_args.extend(['--ubatch-size', context.n_ubatch]) if context.n_gpu_layer: server_args.extend(['--n-gpu-layers', context.n_gpu_layer]) if context.server_continuous_batching: diff --git a/ggml-alloc.c b/ggml-alloc.c index e675306c8..8ac1d3e51 100644 --- a/ggml-alloc.c +++ b/ggml-alloc.c @@ -61,7 +61,6 @@ static bool ggml_op_can_inplace(enum ggml_op op) { } } -// TODO: GGML_PAD ? static size_t aligned_offset(const void * buffer, size_t offset, size_t alignment) { assert(alignment && !(alignment & (alignment - 1))); // power of 2 size_t align = (alignment - (((uintptr_t)buffer + offset) % alignment)) % alignment; @@ -69,25 +68,14 @@ static size_t aligned_offset(const void * buffer, size_t offset, size_t alignmen } // tallocr -struct ggml_tallocr { - ggml_backend_buffer_t buffer; - void * base; - size_t alignment; - size_t offset; -}; - -ggml_tallocr_t ggml_tallocr_new(ggml_backend_buffer_t buffer) { - ggml_tallocr_t talloc = malloc(sizeof(struct ggml_tallocr)); - if (talloc == NULL) { - return NULL; - } +struct ggml_tallocr ggml_tallocr_new(ggml_backend_buffer_t buffer) { void * base = ggml_backend_buffer_get_base(buffer); size_t align = ggml_backend_buffer_get_alignment(buffer); assert(align && !(align & (align - 1))); // power of 2 - *talloc = (struct ggml_tallocr) { + struct ggml_tallocr talloc = (struct ggml_tallocr) { /*.buffer = */ buffer, /*.base = */ base, /*.alignment = */ align, @@ -96,11 +84,7 @@ ggml_tallocr_t ggml_tallocr_new(ggml_backend_buffer_t buffer) { return talloc; } -void ggml_tallocr_free(ggml_tallocr_t talloc) { - free(talloc); -} - -void ggml_tallocr_alloc(ggml_tallocr_t talloc, struct ggml_tensor * tensor) { +void ggml_tallocr_alloc(struct ggml_tallocr * talloc, struct ggml_tensor * tensor) { size_t size = ggml_backend_buffer_get_alloc_size(talloc->buffer, tensor); size = GGML_PAD(size, talloc->alignment); @@ -354,12 +338,16 @@ struct hash_node { bool allocated; }; -// struct tensor_alloc { size_t offset; size_t size_max; // 0 = pre-allocated, unused, or view }; +struct leaf_alloc { + int buffer_id; + struct tensor_alloc leaf; +}; + struct node_alloc { int buffer_id; struct tensor_alloc dst; @@ -378,7 +366,7 @@ struct ggml_gallocr { struct node_alloc * node_allocs; // [n_nodes] int n_nodes; - struct tensor_alloc * leaf_allocs; // [n_leafs] + struct leaf_alloc * leaf_allocs; // [n_leafs] int n_leafs; }; @@ -543,13 +531,20 @@ static int get_node_buffer_id(const int * node_buffer_ids, int i) { return node_buffer_ids ? node_buffer_ids[i] : 0; } -static void ggml_gallocr_alloc_graph_impl(ggml_gallocr_t galloc, struct ggml_cgraph * graph, const int * node_buffer_ids) { +static void ggml_gallocr_alloc_graph_impl(ggml_gallocr_t galloc, struct ggml_cgraph * graph, const int * node_buffer_ids, const int * leaf_buffer_ids) { // clear hash tables memset(galloc->hash_set.keys, 0, galloc->hash_set.size * sizeof(struct ggml_tensor *)); memset(galloc->hash_values, 0, galloc->hash_set.size * sizeof(struct hash_node)); + // allocate leafs + // these may be tensors that the application is not using in the graph, but may still want to allocate for other purposes + for (int i = 0; i < graph->n_leafs; i++) { + struct ggml_tensor * leaf = graph->leafs[i]; + ggml_gallocr_allocate_node(galloc, leaf, get_node_buffer_id(leaf_buffer_ids, i)); + } + // count number of children and views - // allocate all graph inputs and leafs first to avoid overwriting them + // allocate other graph inputs and leafs first to avoid overwriting them for (int i = 0; i < graph->n_nodes; i++) { struct ggml_tensor * node = graph->nodes[i]; @@ -577,19 +572,6 @@ static void ggml_gallocr_alloc_graph_impl(ggml_gallocr_t galloc, struct ggml_cgr } } - // allocate the remaining leafs that are unused on the graph - // these are effectively static tensors that the application is not using in the graph, but may still want to allocate for other purposes - for (int i = 0; i < graph->n_leafs; i++) { - struct ggml_tensor * leaf = graph->leafs[i]; - struct hash_node * hn = ggml_gallocr_hash_get(galloc, leaf); - - if (hn->n_children == 0) { - assert(!hn->allocated); - // since buffer ids are only given for nodes, these leafs are always allocated in the first buffer - ggml_gallocr_allocate_node(galloc, leaf, 0); - } - } - // allocate tensors for (int i = 0; i < graph->n_nodes; i++) { struct ggml_tensor * node = graph->nodes[i]; @@ -652,7 +634,7 @@ static void ggml_gallocr_alloc_graph_impl(ggml_gallocr_t galloc, struct ggml_cgr } } -bool ggml_gallocr_reserve_n(ggml_gallocr_t galloc, struct ggml_cgraph * graph, const int * node_buffer_ids) { +bool ggml_gallocr_reserve_n(ggml_gallocr_t galloc, struct ggml_cgraph * graph, const int * node_buffer_ids, const int * leaf_buffer_ids) { size_t hash_size = graph->visited_hash_table.size; // initialize hash table @@ -676,7 +658,7 @@ bool ggml_gallocr_reserve_n(ggml_gallocr_t galloc, struct ggml_cgraph * graph, c } // allocate in hash table - ggml_gallocr_alloc_graph_impl(galloc, graph, node_buffer_ids); + ggml_gallocr_alloc_graph_impl(galloc, graph, node_buffer_ids, leaf_buffer_ids); // set the node_allocs from the hash table if (galloc->n_nodes < graph->n_nodes) { @@ -711,15 +693,16 @@ bool ggml_gallocr_reserve_n(ggml_gallocr_t galloc, struct ggml_cgraph * graph, c } if (galloc->n_leafs < graph->n_leafs) { free(galloc->leaf_allocs); - galloc->leaf_allocs = calloc(sizeof(struct tensor_alloc), graph->n_leafs); + galloc->leaf_allocs = calloc(sizeof(galloc->leaf_allocs[0]), graph->n_leafs); GGML_ASSERT(galloc->leaf_allocs != NULL); } galloc->n_leafs = graph->n_leafs; for (int i = 0; i < graph->n_leafs; i++) { struct ggml_tensor * leaf = graph->leafs[i]; struct hash_node * hn = ggml_gallocr_hash_get(galloc, leaf); - galloc->leaf_allocs[i].offset = hn->offset; - galloc->leaf_allocs[i].size_max = ggml_backend_buft_get_alloc_size(galloc->bufts[hn->buffer_id], leaf); + galloc->leaf_allocs[i].buffer_id = hn->buffer_id; + galloc->leaf_allocs[i].leaf.offset = hn->offset; + galloc->leaf_allocs[i].leaf.size_max = ggml_backend_buft_get_alloc_size(galloc->bufts[hn->buffer_id], leaf); } // reallocate buffers if needed @@ -727,7 +710,8 @@ bool ggml_gallocr_reserve_n(ggml_gallocr_t galloc, struct ggml_cgraph * graph, c size_t cur_size = galloc->buffers[i] ? ggml_backend_buffer_get_size(galloc->buffers[i]) : 0; size_t new_size = ggml_dyn_tallocr_max_size(galloc->buf_tallocs[i]); - if (new_size > cur_size) { + // even if there are no tensors allocated in this buffer, we still need to allocate it to initialize views + if (new_size > cur_size || galloc->buffers[i] == NULL) { #ifndef NDEBUG fprintf(stderr, "%s: reallocating %s buffer from size %.02f MiB to %.02f MiB\n", __func__, ggml_backend_buft_name(galloc->bufts[i]), cur_size / 1024.0 / 1024.0, new_size / 1024.0 / 1024.0); #endif @@ -744,30 +728,30 @@ bool ggml_gallocr_reserve_n(ggml_gallocr_t galloc, struct ggml_cgraph * graph, c } bool ggml_gallocr_reserve(ggml_gallocr_t galloc, struct ggml_cgraph *graph) { - return ggml_gallocr_reserve_n(galloc, graph, NULL); + return ggml_gallocr_reserve_n(galloc, graph, NULL, NULL); } -static void ggml_gallocr_init_tensor(ggml_gallocr_t galloc, struct ggml_tensor * node, int buffer_id, struct tensor_alloc * tensor_alloc) { - assert(node->data || node->view_src || ggml_backend_buffer_get_alloc_size(galloc->buffers[buffer_id], node) <= tensor_alloc->size_max); +static void ggml_gallocr_init_tensor(ggml_gallocr_t galloc, struct ggml_tensor * tensor, int buffer_id, struct tensor_alloc * tensor_alloc) { + assert(tensor->data || tensor->view_src || ggml_backend_buffer_get_alloc_size(galloc->buffers[buffer_id], tensor) <= tensor_alloc->size_max); - if (node->view_src != NULL) { - if (node->buffer == NULL) { + if (tensor->view_src != NULL) { + if (tensor->buffer == NULL) { assert(tensor_alloc->offset == SIZE_MAX); - if (node->view_src->buffer == NULL) { + if (tensor->view_src->buffer == NULL) { // this tensor was allocated without ggml-backend return; } - ggml_backend_view_init(galloc->buffers[buffer_id], node); + ggml_backend_view_init(galloc->buffers[buffer_id], tensor); } } else { - if (node->data == NULL) { + if (tensor->data == NULL) { assert(tensor_alloc->offset != SIZE_MAX); - assert(ggml_backend_buffer_get_alloc_size(galloc->buffers[buffer_id], node) <= tensor_alloc->size_max); + assert(ggml_backend_buffer_get_alloc_size(galloc->buffers[buffer_id], tensor) <= tensor_alloc->size_max); void * base = ggml_backend_buffer_get_base(galloc->buffers[buffer_id]); void * addr = (char *)base + tensor_alloc->offset; - ggml_backend_tensor_alloc(galloc->buffers[buffer_id], node, addr); + ggml_backend_tensor_alloc(galloc->buffers[buffer_id], tensor, addr); } else { - if (node->buffer == NULL) { + if (tensor->buffer == NULL) { // this tensor was allocated without ggml-backend return; } @@ -843,13 +827,18 @@ bool ggml_gallocr_alloc_graph(ggml_gallocr_t galloc, struct ggml_cgraph * graph) // reset buffers for (int i = 0; i < galloc->n_buffers; i++) { - // zero size buffers are not allocated if (galloc->buffers[i] != NULL) { ggml_backend_buffer_reset(galloc->buffers[i]); } } // allocate the graph tensors from the previous assignments + // leafs + for (int i = 0; i < graph->n_leafs; i++) { + struct ggml_tensor * leaf = graph->leafs[i]; + struct leaf_alloc * leaf_alloc = &galloc->leaf_allocs[i]; + ggml_gallocr_init_tensor(galloc, leaf, leaf_alloc->buffer_id, &leaf_alloc->leaf); + } // nodes for (int i = 0; i < graph->n_nodes; i++) { struct ggml_tensor * node = graph->nodes[i]; @@ -863,12 +852,6 @@ bool ggml_gallocr_alloc_graph(ggml_gallocr_t galloc, struct ggml_cgraph * graph) } ggml_gallocr_init_tensor(galloc, node, node_alloc->buffer_id, &node_alloc->dst); } - // leafs - for (int i = 0; i < graph->n_leafs; i++) { - struct ggml_tensor * leaf = graph->leafs[i]; - struct tensor_alloc * leaf_alloc = &galloc->leaf_allocs[i]; - ggml_gallocr_init_tensor(galloc, leaf, 0, leaf_alloc); - } return true; } @@ -900,12 +883,12 @@ static bool alloc_tensor_range(struct ggml_context * ctx, return false; } - struct ggml_tallocr * tallocr = ggml_tallocr_new(buffer); + struct ggml_tallocr tallocr = ggml_tallocr_new(buffer); for (struct ggml_tensor * t = first; t != last; t = ggml_get_next_tensor(ctx, t)) { if (t->data == NULL) { if (t->view_src == NULL) { - ggml_tallocr_alloc(tallocr, t); + ggml_tallocr_alloc(&tallocr, t); } else if (t->buffer == NULL) { ggml_backend_view_init(buffer, t); } @@ -917,8 +900,6 @@ static bool alloc_tensor_range(struct ggml_context * ctx, } } - ggml_tallocr_free(tallocr); - *buffers = realloc(*buffers, sizeof(ggml_backend_buffer_t) * (*n_buffers + 1)); (*buffers)[(*n_buffers)++] = buffer; diff --git a/ggml-alloc.h b/ggml-alloc.h index 1d9085d15..434c13b34 100644 --- a/ggml-alloc.h +++ b/ggml-alloc.h @@ -11,11 +11,15 @@ typedef struct ggml_backend_buffer * ggml_backend_buffer_t; typedef struct ggml_backend * ggml_backend_t; // Tensor allocator -typedef struct ggml_tallocr * ggml_tallocr_t; +struct ggml_tallocr { + ggml_backend_buffer_t buffer; + void * base; + size_t alignment; + size_t offset; +}; -GGML_API ggml_tallocr_t ggml_tallocr_new(ggml_backend_buffer_t buffer); -GGML_API void ggml_tallocr_free(ggml_tallocr_t talloc); -GGML_API void ggml_tallocr_alloc(ggml_tallocr_t talloc, struct ggml_tensor * tensor); +GGML_API struct ggml_tallocr ggml_tallocr_new(ggml_backend_buffer_t buffer); +GGML_API void ggml_tallocr_alloc(struct ggml_tallocr * talloc, struct ggml_tensor * tensor); // Graph allocator /* @@ -50,7 +54,11 @@ GGML_API void ggml_gallocr_free(ggml_gallocr_t galloc); // not strictly required for single buffer usage: ggml_gallocr_alloc_graph will reallocate the buffers automatically if needed // returns false if the buffer allocation failed GGML_API bool ggml_gallocr_reserve(ggml_gallocr_t galloc, struct ggml_cgraph * graph); -GGML_API bool ggml_gallocr_reserve_n(ggml_gallocr_t galloc, struct ggml_cgraph * graph, const int * node_buffer_ids); +GGML_API bool ggml_gallocr_reserve_n( + ggml_gallocr_t galloc, + struct ggml_cgraph * graph, + const int * node_buffer_ids, + const int * leaf_buffer_ids); // automatic reallocation if the topology changes when using a single buffer // returns false if using multiple buffers and a re-allocation is needed (call ggml_gallocr_reserve_n first to set the node buffers) diff --git a/ggml-backend-impl.h b/ggml-backend-impl.h index 2e9ba58a9..e475e20e5 100644 --- a/ggml-backend-impl.h +++ b/ggml-backend-impl.h @@ -86,12 +86,12 @@ extern "C" { // (optional) asynchronous tensor data access void (*GGML_CALL set_tensor_async)(ggml_backend_t backend, struct ggml_tensor * tensor, const void * data, size_t offset, size_t size); void (*GGML_CALL get_tensor_async)(ggml_backend_t backend, const struct ggml_tensor * tensor, void * data, size_t offset, size_t size); - bool (*GGML_CALL cpy_tensor_async)(ggml_backend_t backend, const struct ggml_tensor * src, struct ggml_tensor * dst); + bool (*GGML_CALL cpy_tensor_async)(ggml_backend_t backend_src, ggml_backend_t backend_dst, const struct ggml_tensor * src, struct ggml_tensor * dst); // (optional) complete all pending operations void (*GGML_CALL synchronize)(ggml_backend_t backend); - // create a plan for ggml_cgraph and free it + // compute graph with a plan (not used currently) ggml_backend_graph_plan_t (*GGML_CALL graph_plan_create) (ggml_backend_t backend, const struct ggml_cgraph * cgraph); void (*GGML_CALL graph_plan_free) (ggml_backend_t backend, ggml_backend_graph_plan_t plan); @@ -102,16 +102,27 @@ extern "C" { // check if the backend supports an operation bool (*GGML_CALL supports_op)(ggml_backend_t backend, const struct ggml_tensor * op); + + // (optional) event synchronization + ggml_backend_event_t (*GGML_CALL event_new) (ggml_backend_t backend); + void (*GGML_CALL event_free) (ggml_backend_event_t event); + void (*GGML_CALL event_record) (ggml_backend_event_t event); + void (*GGML_CALL event_wait) (ggml_backend_t backend, ggml_backend_event_t event); + void (*GGML_CALL event_synchronize) (ggml_backend_event_t event); }; struct ggml_backend { ggml_guid_t guid; struct ggml_backend_i iface; - ggml_backend_context_t context; }; + struct ggml_backend_event { + ggml_backend_t backend; + void * context; + }; + // // Backend registry // diff --git a/ggml-backend.c b/ggml-backend.c index d60d98414..31f8d5a6d 100644 --- a/ggml-backend.c +++ b/ggml-backend.c @@ -221,29 +221,29 @@ void ggml_backend_tensor_get_async(ggml_backend_t backend, const struct ggml_ten GGML_CALL void ggml_backend_tensor_set(struct ggml_tensor * tensor, const void * data, size_t offset, size_t size) { ggml_backend_buffer_t buf = tensor->view_src ? tensor->view_src->buffer : tensor->buffer; - GGML_ASSERT(tensor->data != NULL && "tensor not allocated"); GGML_ASSERT(buf != NULL && "tensor buffer not set"); + GGML_ASSERT(tensor->data != NULL && "tensor not allocated"); GGML_ASSERT(offset + size <= ggml_nbytes(tensor) && "tensor write out of bounds"); if (!size) { return; } - tensor->buffer->iface.set_tensor(buf, tensor, data, offset, size); + buf->iface.set_tensor(buf, tensor, data, offset, size); } GGML_CALL void ggml_backend_tensor_get(const struct ggml_tensor * tensor, void * data, size_t offset, size_t size) { ggml_backend_buffer_t buf = tensor->view_src ? tensor->view_src->buffer : tensor->buffer; + GGML_ASSERT(buf != NULL && "tensor buffer not set"); GGML_ASSERT(tensor->data != NULL && "tensor not allocated"); - GGML_ASSERT(tensor->buffer != NULL && "tensor buffer not set"); GGML_ASSERT(offset + size <= ggml_nbytes(tensor) && "tensor read out of bounds"); if (!size) { return; } - tensor->buffer->iface.get_tensor(buf, tensor, data, offset, size); + buf->iface.get_tensor(buf, tensor, data, offset, size); } void ggml_backend_synchronize(ggml_backend_t backend) { @@ -255,18 +255,30 @@ void ggml_backend_synchronize(ggml_backend_t backend) { } ggml_backend_graph_plan_t ggml_backend_graph_plan_create(ggml_backend_t backend, struct ggml_cgraph * cgraph) { + GGML_ASSERT(backend->iface.graph_plan_create != NULL); + return backend->iface.graph_plan_create(backend, cgraph); } void ggml_backend_graph_plan_free(ggml_backend_t backend, ggml_backend_graph_plan_t plan) { + GGML_ASSERT(backend->iface.graph_plan_free != NULL); + backend->iface.graph_plan_free(backend, plan); } enum ggml_status ggml_backend_graph_plan_compute(ggml_backend_t backend, ggml_backend_graph_plan_t plan) { + GGML_ASSERT(backend->iface.graph_plan_compute != NULL); + return backend->iface.graph_plan_compute(backend, plan); } enum ggml_status ggml_backend_graph_compute(ggml_backend_t backend, struct ggml_cgraph * cgraph) { + enum ggml_status err = ggml_backend_graph_compute_async(backend, cgraph); + ggml_backend_synchronize(backend); + return err; +} + +bool ggml_backend_graph_compute_async(ggml_backend_t backend, struct ggml_cgraph * cgraph) { return backend->iface.graph_compute(backend, cgraph); } @@ -314,34 +326,68 @@ void ggml_backend_tensor_copy(struct ggml_tensor * src, struct ggml_tensor * dst } } -void ggml_backend_tensor_copy_async(ggml_backend_t backend, struct ggml_tensor * src, struct ggml_tensor * dst) { +void ggml_backend_tensor_copy_async(ggml_backend_t backend_src, ggml_backend_t backend_dst, struct ggml_tensor * src, struct ggml_tensor * dst) { GGML_ASSERT(ggml_are_same_layout(src, dst) && "cannot copy tensors with different layouts"); if (src == dst) { return; } - if (ggml_backend_buft_supports_backend(src->buffer->buft, backend) && ggml_backend_buft_supports_backend(dst->buffer->buft, backend)) { - if (backend->iface.cpy_tensor_async != NULL) { - if (backend->iface.cpy_tensor_async(backend, src, dst)) { - return; - } + if (backend_dst->iface.cpy_tensor_async != NULL) { + if (backend_dst->iface.cpy_tensor_async(backend_src, backend_dst, src, dst)) { + return; } } - size_t nbytes = ggml_nbytes(src); + // an async copy would normally happen after all the queued operations on both backends are completed + // sync src, set_async dst if (ggml_backend_buffer_is_host(src->buffer)) { - ggml_backend_tensor_set_async(backend, dst, src->data, 0, nbytes); - } - else { + ggml_backend_synchronize(backend_src); + ggml_backend_tensor_set_async(backend_dst, dst, src->data, 0, ggml_nbytes(src)); + } else { + ggml_backend_synchronize(backend_src); ggml_backend_tensor_copy(src, dst); + ggml_backend_synchronize(backend_dst); } } +// events + +ggml_backend_event_t ggml_backend_event_new(ggml_backend_t backend) { + if (backend->iface.event_new == NULL) { + return NULL; + } + return backend->iface.event_new(backend); +} + +void ggml_backend_event_free(ggml_backend_event_t event) { + if (event == NULL) { + return; + } + event->backend->iface.event_free(event); +} + +void ggml_backend_event_record(ggml_backend_event_t event) { + GGML_ASSERT(event->backend->iface.event_record != NULL); + + event->backend->iface.event_record(event); +} + +void ggml_backend_event_synchronize(ggml_backend_event_t event) { + GGML_ASSERT(event->backend->iface.event_synchronize != NULL); + + event->backend->iface.event_synchronize(event); +} + +void ggml_backend_event_wait(ggml_backend_t backend, ggml_backend_event_t event) { + GGML_ASSERT(backend->iface.event_wait != NULL); + + backend->iface.event_wait(backend, event); +} // backend registry -#define GGML_MAX_BACKENDS_REG 16 +#define GGML_REG_MAX_BACKENDS 16 struct ggml_backend_reg { char name[128]; @@ -350,7 +396,7 @@ struct ggml_backend_reg { void * user_data; }; -static struct ggml_backend_reg ggml_backend_registry[GGML_MAX_BACKENDS_REG]; +static struct ggml_backend_reg ggml_backend_registry[GGML_REG_MAX_BACKENDS]; static size_t ggml_backend_registry_count = 0; GGML_CALL static ggml_backend_t ggml_backend_reg_cpu_init(const char * params, void * user_data); @@ -395,7 +441,7 @@ GGML_CALL static void ggml_backend_registry_init(void) { } GGML_CALL void ggml_backend_register(const char * name, ggml_backend_init_fn init_fn, ggml_backend_buffer_type_t default_buffer_type, void * user_data) { - GGML_ASSERT(ggml_backend_registry_count < GGML_MAX_BACKENDS_REG); + GGML_ASSERT(ggml_backend_registry_count < GGML_REG_MAX_BACKENDS); size_t id = ggml_backend_registry_count; @@ -746,8 +792,12 @@ GGML_CALL static enum ggml_status ggml_backend_cpu_graph_compute(ggml_backend_t struct ggml_cplan cplan = ggml_graph_plan(cgraph, cpu_ctx->n_threads); if (cpu_ctx->work_size < cplan.work_size) { - // TODO: may be faster to free and use malloc to avoid the copy - cpu_ctx->work_data = realloc(cpu_ctx->work_data, cplan.work_size); + free(cpu_ctx->work_data); + cpu_ctx->work_data = malloc(cplan.work_size); + if (cpu_ctx->work_data == NULL) { + cpu_ctx->work_size = 0; + return GGML_STATUS_ALLOC_FAILED; + } cpu_ctx->work_size = cplan.work_size; } cplan.work_data = cpu_ctx->work_data; @@ -784,6 +834,11 @@ static struct ggml_backend_i cpu_backend_i = { /* .graph_plan_compute = */ ggml_backend_cpu_graph_plan_compute, /* .graph_compute = */ ggml_backend_cpu_graph_compute, /* .supports_op = */ ggml_backend_cpu_supports_op, + /* .event_new = */ NULL, + /* .event_free = */ NULL, + /* .event_record = */ NULL, + /* .event_wait = */ NULL, + /* .event_synchronize = */ NULL, }; static ggml_guid_t ggml_backend_cpu_guid(void) { @@ -939,15 +994,27 @@ static bool ggml_is_view_op(enum ggml_op op) { // scheduler -#define GGML_MAX_BACKENDS 16 -#define GGML_MAX_SPLITS 256 -#define GGML_MAX_SPLIT_INPUTS 16 +#ifndef GGML_SCHED_MAX_BACKENDS +#define GGML_SCHED_MAX_BACKENDS 16 +#endif + +#ifndef GGML_SCHED_MAX_SPLITS +#define GGML_SCHED_MAX_SPLITS 256 +#endif + +#ifndef GGML_SCHED_MAX_SPLIT_INPUTS +#define GGML_SCHED_MAX_SPLIT_INPUTS 16 +#endif + +#ifndef GGML_SCHED_MAX_COPIES +#define GGML_SCHED_MAX_COPIES 4 +#endif struct ggml_backend_sched_split { int backend_id; int i_start; int i_end; - struct ggml_tensor * inputs[GGML_MAX_SPLIT_INPUTS]; + struct ggml_tensor * inputs[GGML_SCHED_MAX_SPLIT_INPUTS]; int n_inputs; // graph view of this split struct ggml_cgraph graph; @@ -955,45 +1022,53 @@ struct ggml_backend_sched_split { struct ggml_backend_sched { bool is_reset; // true if the scheduler has been reset since the last graph split + bool is_alloc; int n_backends; - ggml_backend_t backends[GGML_MAX_BACKENDS]; - ggml_backend_buffer_type_t bufts[GGML_MAX_BACKENDS]; + ggml_backend_t backends[GGML_SCHED_MAX_BACKENDS]; + ggml_backend_buffer_type_t bufts[GGML_SCHED_MAX_BACKENDS]; ggml_gallocr_t galloc; // hash keys of the nodes in the graph struct ggml_hash_set hash_set; // hash values int * tensor_backend_id; - struct ggml_tensor * (* tensor_copies)[GGML_MAX_BACKENDS]; + struct ggml_tensor * (* tensor_copies)[GGML_SCHED_MAX_BACKENDS][GGML_SCHED_MAX_COPIES]; - int * node_backend_ids; // [n_nodes] - int n_nodes; + int * node_backend_ids; // [graph_size] + int * leaf_backend_ids; // [graph_size] // copy of the graph with modified inputs struct ggml_cgraph * graph; - struct ggml_backend_sched_split splits[GGML_MAX_SPLITS]; + // graph splits + struct ggml_backend_sched_split splits[GGML_SCHED_MAX_SPLITS]; int n_splits; + // pipeline parallelism support + int n_copies; + int cur_copy; + ggml_backend_event_t events[GGML_SCHED_MAX_BACKENDS][GGML_SCHED_MAX_COPIES]; + struct ggml_tensor * graph_inputs[GGML_SCHED_MAX_SPLIT_INPUTS]; + int n_graph_inputs; + struct ggml_context * ctx; ggml_backend_sched_eval_callback callback_eval; void * callback_eval_user_data; // align context_buffer to GGML_MEM_ALIGN - #ifdef _MSC_VER +#ifdef _MSC_VER __declspec(align(GGML_MEM_ALIGN)) - #else +#else __attribute__((aligned(GGML_MEM_ALIGN))) - #endif - char context_buffer[GGML_MAX_SPLITS*GGML_MAX_SPLIT_INPUTS*2*sizeof(struct ggml_tensor) + sizeof(struct ggml_cgraph)]; +#endif + char context_buffer[GGML_SCHED_MAX_SPLITS*GGML_SCHED_MAX_SPLIT_INPUTS*2*sizeof(struct ggml_tensor) + sizeof(struct ggml_cgraph)]; }; -#define hash_id(node) ggml_hash_find_or_insert(sched->hash_set, node) -#define tensor_backend_id(node) sched->tensor_backend_id[hash_id(node)] -#define tensor_backend(node) (tensor_backend_id(node) == -1 ? NULL : sched->backends[tensor_backend_id(node)]) +#define hash_id(tensor) ggml_hash_find_or_insert(sched->hash_set, tensor) +#define tensor_backend_id(tensor) sched->tensor_backend_id[hash_id(tensor)] // returns the priority of the backend, lower id is higher priority static int ggml_backend_sched_backend_id(ggml_backend_sched_t sched, ggml_backend_t backend) { @@ -1005,7 +1080,8 @@ static int ggml_backend_sched_backend_id(ggml_backend_sched_t sched, ggml_backen return -1; } -static int ggml_backend_sched_backend_from_buffer(ggml_backend_sched_t sched, ggml_backend_buffer_t buffer) { +static int ggml_backend_sched_backend_from_buffer(ggml_backend_sched_t sched, const struct ggml_tensor * tensor) { + ggml_backend_buffer_t buffer = tensor->buffer; if (buffer == NULL) { return -1; } @@ -1016,12 +1092,16 @@ static int ggml_backend_sched_backend_from_buffer(ggml_backend_sched_t sched, gg return i; } } - GGML_ASSERT(false && "tensor buffer type not supported by any backend"); - return -1; // silence warning + + fprintf(stderr, "%s: error: no backend supports buffer type %s used in tensor %s\n", + __func__, ggml_backend_buffer_name(buffer), tensor->name); + GGML_ASSERT(false); + + return -1; } #if 0 -static char causes[GGML_DEFAULT_GRAPH_SIZE*16 + GGML_MAX_SPLITS*GGML_MAX_SPLIT_INPUTS][128]; // debug only +static char causes[GGML_DEFAULT_GRAPH_SIZE*16 + GGML_SCHED_MAX_SPLITS*GGML_SCHED_MAX_SPLIT_INPUTS][128]; // debug only #define SET_CAUSE(node, ...) sprintf(causes[hash_id(node)], __VA_ARGS__) #define GET_CAUSE(node) causes[hash_id(node)] #else @@ -1035,19 +1115,28 @@ static int ggml_backend_sched_backend_id_from_cur(ggml_backend_sched_t sched, st // assign pre-allocated nodes to their backend // dst - int cur_backend = ggml_backend_sched_backend_from_buffer(sched, tensor->buffer); + int cur_backend = ggml_backend_sched_backend_from_buffer(sched, tensor); if (cur_backend != -1) { - SET_CAUSE(node, "1.dst"); + SET_CAUSE(tensor, "1.dst"); return cur_backend; } + // view_src if (tensor->view_src != NULL) { - cur_backend = ggml_backend_sched_backend_from_buffer(sched, tensor->view_src->buffer); + cur_backend = ggml_backend_sched_backend_from_buffer(sched, tensor->view_src); if (cur_backend != -1) { - SET_CAUSE(node, "1.vsrc"); + SET_CAUSE(tensor, "1.vsrc"); return cur_backend; } } + + // input + if (tensor->flags & GGML_TENSOR_FLAG_INPUT) { + cur_backend = sched->n_backends - 1; // last backend (assumed CPU) + SET_CAUSE(tensor, "1.inp"); + return cur_backend; + } + // assign nodes that use weights to the backend of the weights for (int i = 0; i < GGML_MAX_SRC; i++) { const struct ggml_tensor * src = tensor->src[i]; @@ -1055,9 +1144,9 @@ static int ggml_backend_sched_backend_id_from_cur(ggml_backend_sched_t sched, st continue; } if (src->buffer != NULL && src->buffer->usage == GGML_BACKEND_BUFFER_USAGE_WEIGHTS) { - int src_backend = ggml_backend_sched_backend_from_buffer(sched, src->buffer); + int src_backend = ggml_backend_sched_backend_from_buffer(sched, src); // operations with weights are always run on the same backend as the weights - SET_CAUSE(node, "1.wgt%d", i); + SET_CAUSE(tensor, "1.wgt%d", i); return src_backend; } } @@ -1093,7 +1182,7 @@ static void ggml_backend_sched_print_assignments(ggml_backend_sched_t sched, str if (ggml_is_view_op(node->op)) { continue; } - ggml_backend_t tensor_backend = tensor_backend(node); + ggml_backend_t tensor_backend = ggml_backend_sched_get_tensor_backend(sched, node); fprintf(stderr, "node #%3d (%10.10s): %20.20s (%5.5s) [%5.5s %8.8s]:", i, ggml_op_name(node->op), node->name, fmt_size(ggml_nbytes(node)), tensor_backend ? ggml_backend_name(tensor_backend) : "NULL", GET_CAUSE(node)); for (int j = 0; j < GGML_MAX_SRC; j++) { @@ -1101,7 +1190,7 @@ static void ggml_backend_sched_print_assignments(ggml_backend_sched_t sched, str if (src == NULL) { continue; } - ggml_backend_t src_backend = tensor_backend(src); + ggml_backend_t src_backend = ggml_backend_sched_get_tensor_backend(sched, src); fprintf(stderr, " %20.20s (%5.5s) [%5.5s %8.8s]", src->name, fmt_size(ggml_nbytes(src)), src_backend ? ggml_backend_name(src_backend) : "NULL", GET_CAUSE(src)); } @@ -1118,6 +1207,7 @@ static void ggml_backend_sched_print_assignments(ggml_backend_sched_t sched, str static void ggml_backend_sched_split_graph(ggml_backend_sched_t sched, struct ggml_cgraph * graph) { // reset splits sched->n_splits = 0; + sched->n_graph_inputs = 0; sched->is_reset = false; struct ggml_init_params params = { @@ -1163,7 +1253,7 @@ static void ggml_backend_sched_split_graph(ggml_backend_sched_t sched, struct gg } } #ifdef DEBUG_PASS1 - fprintf(stderr, "PASS 1 ASSIGNMENTS\n"); sched_print_assignments(sched, graph); + fprintf(stderr, "PASS 1 ASSIGNMENTS\n"); ggml_backend_sched_print_assignments(sched, graph); #endif // pass 2: expand current backend assignments @@ -1171,28 +1261,6 @@ static void ggml_backend_sched_split_graph(ggml_backend_sched_t sched, struct gg // expand gpu backends (i.e. non last prio) up and down, ignoring cpu (the lowest priority backend) // thus, cpu will never be used unless weights are on cpu, or there are no gpu ops between cpu ops - // pass 2.1 expand gpu up - { - int cur_backend_id = -1; - for (int i = graph->n_nodes - 1; i >= 0; i--) { - struct ggml_tensor * node = graph->nodes[i]; - if (ggml_is_view_op(node->op)) { - continue; - } - int tensor_backend_id = tensor_backend_id(node); - if (tensor_backend_id != -1) { - if (tensor_backend_id == sched->n_backends - 1) { - // skip cpu (lowest prio backend) - cur_backend_id = -1; - } else { - cur_backend_id = tensor_backend_id; - } - } else { - tensor_backend_id(node) = cur_backend_id; - SET_CAUSE(node, "2.1"); - } - } - } // pass 2.2 expand gpu down { @@ -1217,7 +1285,7 @@ static void ggml_backend_sched_split_graph(ggml_backend_sched_t sched, struct gg } } - // pass 2.3 expand rest up + // pass 2.1 expand gpu up { int cur_backend_id = -1; for (int i = graph->n_nodes - 1; i >= 0; i--) { @@ -1227,14 +1295,20 @@ static void ggml_backend_sched_split_graph(ggml_backend_sched_t sched, struct gg } int tensor_backend_id = tensor_backend_id(node); if (tensor_backend_id != -1) { - cur_backend_id = tensor_backend_id; + if (tensor_backend_id == sched->n_backends - 1) { + // skip cpu (lowest prio backend) + cur_backend_id = -1; + } else { + cur_backend_id = tensor_backend_id; + } } else { tensor_backend_id(node) = cur_backend_id; - SET_CAUSE(node, "2.3"); + SET_CAUSE(node, "2.1"); } } } + // pass 2.4 expand rest down { int cur_backend_id = -1; @@ -1252,8 +1326,26 @@ static void ggml_backend_sched_split_graph(ggml_backend_sched_t sched, struct gg } } } + // pass 2.3 expand rest up + { + int cur_backend_id = -1; + for (int i = graph->n_nodes - 1; i >= 0; i--) { + struct ggml_tensor * node = graph->nodes[i]; + if (ggml_is_view_op(node->op)) { + continue; + } + int tensor_backend_id = tensor_backend_id(node); + if (tensor_backend_id != -1) { + cur_backend_id = tensor_backend_id; + } else { + tensor_backend_id(node) = cur_backend_id; + SET_CAUSE(node, "2.3"); + } + } + } + #ifdef DEBUG_PASS2 - fprintf(stderr, "PASS 2 ASSIGNMENTS\n"); sched_print_assignments(sched, graph); + fprintf(stderr, "PASS 2 ASSIGNMENTS\n"); ggml_backend_sched_print_assignments(sched, graph); #endif // pass 3: assign backends to remaining src from dst and view_src @@ -1283,7 +1375,7 @@ static void ggml_backend_sched_split_graph(ggml_backend_sched_t sched, struct gg } } #ifdef DEBUG_PASS3 - fprintf(stderr, "PASS 3 ASSIGNMENTS\n"); sched_print_assignments(sched, graph); + fprintf(stderr, "PASS 3 ASSIGNMENTS\n"); ggml_backend_sched_print_assignments(sched, graph); #endif // pass 4: split graph, find tensors that need to be copied @@ -1315,7 +1407,7 @@ static void ggml_backend_sched_split_graph(ggml_backend_sched_t sched, struct gg if (tensor_backend_id != cur_backend_id) { sched->splits[cur_split].i_end = i; cur_split++; - GGML_ASSERT(cur_split < GGML_MAX_SPLITS); + GGML_ASSERT(cur_split < GGML_SCHED_MAX_SPLITS); sched->splits[cur_split].backend_id = tensor_backend_id; sched->splits[cur_split].i_start = i; sched->splits[cur_split].n_inputs = 0; @@ -1328,25 +1420,57 @@ static void ggml_backend_sched_split_graph(ggml_backend_sched_t sched, struct gg if (src == NULL) { continue; } + int src_backend_id = tensor_backend_id(src); assert(src_backend_id != -1); // all inputs should be assigned by now + + if (src->flags & GGML_TENSOR_FLAG_INPUT) { + size_t id = hash_id(src); + if (sched->tensor_copies[id][src_backend_id][0] == NULL) { + ggml_backend_t backend = sched->backends[src_backend_id]; + for (int c = 0; c < sched->n_copies; c++) { + struct ggml_tensor * tensor_copy; + if (c == sched->cur_copy) { + tensor_copy = src; // use the original tensor as the current copy + } else { + tensor_copy = ggml_dup_tensor_layout(sched->ctx, src); + ggml_format_name(tensor_copy, "%s#%s#%d", ggml_backend_name(backend), src->name, c); + } + if (sched->n_copies > 1) { + ggml_set_input(tensor_copy); + ggml_set_output(tensor_copy); // prevent ggml-alloc from overwriting the tensor + } + sched->tensor_copies[id][src_backend_id][c] = tensor_copy; + tensor_backend_id(tensor_copy) = src_backend_id; + SET_CAUSE(tensor_copy, "4.cpy"); + } + int n_graph_inputs = sched->n_graph_inputs++; + GGML_ASSERT(n_graph_inputs < GGML_SCHED_MAX_SPLIT_INPUTS); + sched->graph_inputs[n_graph_inputs] = src; + } + } + if (src_backend_id != tensor_backend_id) { // create a copy of the input in the split's backend size_t id = hash_id(src); - if (sched->tensor_copies[id][cur_backend_id] == NULL) { + if (sched->tensor_copies[id][cur_backend_id][0] == NULL) { ggml_backend_t backend = sched->backends[cur_backend_id]; - struct ggml_tensor * tensor_copy = ggml_dup_tensor_layout(sched->ctx, src); - ggml_format_name(tensor_copy, "%s#%s", ggml_backend_name(backend), src->name); - - sched->tensor_copies[id][cur_backend_id] = tensor_copy; - tensor_backend_id(tensor_copy) = cur_backend_id; - SET_CAUSE(tensor_copy, "4.cpy"); - + for (int c = 0; c < sched->n_copies; c++) { + struct ggml_tensor * tensor_copy = ggml_dup_tensor_layout(sched->ctx, src); + ggml_format_name(tensor_copy, "%s#%s#%d", ggml_backend_name(backend), src->name, c); + if (sched->n_copies > 1) { + ggml_set_input(tensor_copy); + ggml_set_output(tensor_copy); // prevent ggml-alloc from overwriting the tensor + } + sched->tensor_copies[id][cur_backend_id][c] = tensor_copy; + tensor_backend_id(tensor_copy) = cur_backend_id; + SET_CAUSE(tensor_copy, "4.cpy"); + } int n_inputs = sched->splits[cur_split].n_inputs++; - GGML_ASSERT(n_inputs < GGML_MAX_SPLIT_INPUTS); + GGML_ASSERT(n_inputs < GGML_SCHED_MAX_SPLIT_INPUTS); sched->splits[cur_split].inputs[n_inputs] = src; } - node->src[j] = sched->tensor_copies[id][cur_backend_id]; + node->src[j] = sched->tensor_copies[id][cur_backend_id][sched->cur_copy]; } } } @@ -1354,37 +1478,39 @@ static void ggml_backend_sched_split_graph(ggml_backend_sched_t sched, struct gg sched->n_splits = cur_split + 1; } #ifdef DEBUG_PASS4 - fprintf(stderr, "PASS 4 ASSIGNMENTS\n"); sched_print_assignments(sched, graph); + fprintf(stderr, "PASS 4 ASSIGNMENTS\n"); ggml_backend_sched_print_assignments(sched, graph); #endif #ifndef NDEBUG // sanity check: all sources should have the same backend as the node for (int i = 0; i < graph->n_nodes; i++) { struct ggml_tensor * node = graph->nodes[i]; - ggml_backend_t tensor_backend = tensor_backend(node); + ggml_backend_t tensor_backend = ggml_backend_sched_get_tensor_backend(sched, node); if (tensor_backend == NULL) { fprintf(stderr, "!!!!!!! %s has no backend\n", node->name); } - if (node->view_src != NULL && tensor_backend != tensor_backend(node->view_src)) { + if (node->view_src != NULL && tensor_backend != ggml_backend_sched_get_tensor_backend(sched, node->view_src)) { fprintf(stderr, "!!!!!!! %s has backend %s, view_src %s has backend %s\n", node->name, tensor_backend ? ggml_backend_name(tensor_backend) : "NULL", - node->view_src->name, tensor_backend(node->view_src) ? ggml_backend_name(tensor_backend(node->view_src)) : "NULL"); + node->view_src->name, ggml_backend_sched_get_tensor_backend(sched, node->view_src) ? + ggml_backend_name(ggml_backend_sched_get_tensor_backend(sched, node->view_src)) : "NULL"); } for (int j = 0; j < GGML_MAX_SRC; j++) { struct ggml_tensor * src = node->src[j]; if (src == NULL) { continue; } - ggml_backend_t src_backend = tensor_backend(src); + ggml_backend_t src_backend = ggml_backend_sched_get_tensor_backend(sched, src); if (src_backend != tensor_backend /* && src_backend != NULL */) { fprintf(stderr, "!!!! %s has backend %s, src %d (%s) has backend %s\n", node->name, tensor_backend ? ggml_backend_name(tensor_backend) : "NULL", j, src->name, src_backend ? ggml_backend_name(src_backend) : "NULL"); } - if (src->view_src != NULL && src_backend != tensor_backend(src->view_src)) { + if (src->view_src != NULL && src_backend != ggml_backend_sched_get_tensor_backend(sched, src->view_src)) { fprintf(stderr, "!!!!!!! [src] %s has backend %s, view_src %s has backend %s\n", src->name, src_backend ? ggml_backend_name(src_backend) : "NULL", - src->view_src->name, tensor_backend(src->view_src) ? ggml_backend_name(tensor_backend(src->view_src)) : "NULL"); + src->view_src->name, ggml_backend_sched_get_tensor_backend(sched, src->view_src) ? + ggml_backend_name(ggml_backend_sched_get_tensor_backend(sched, src->view_src)) : "NULL"); } } } @@ -1392,18 +1518,20 @@ static void ggml_backend_sched_split_graph(ggml_backend_sched_t sched, struct gg #endif // create copies of the graph for each split - // FIXME: avoid this copy, pass split inputs to ggml_gallocr_alloc_graph_n in some other way - struct ggml_cgraph * graph_copy = ggml_new_graph_custom(sched->ctx, graph->n_nodes + sched->n_splits*GGML_MAX_SPLIT_INPUTS, false); + // TODO: avoid this copy + struct ggml_cgraph * graph_copy = ggml_new_graph_custom(sched->ctx, graph->n_nodes + sched->n_splits*GGML_SCHED_MAX_SPLIT_INPUTS, false); for (int i = 0; i < sched->n_splits; i++) { struct ggml_backend_sched_split * split = &sched->splits[i]; split->graph = ggml_graph_view(graph, split->i_start, split->i_end); + // add inputs to the graph copy so that they are allocated by ggml-alloc at the start of the split for (int j = 0; j < split->n_inputs; j++) { struct ggml_tensor * input = split->inputs[j]; - struct ggml_tensor * input_cpy = sched->tensor_copies[hash_id(input)][split->backend_id]; + struct ggml_tensor * input_cpy = sched->tensor_copies[hash_id(input)][split->backend_id][sched->cur_copy]; // add a dependency to the input source so that it is not freed before the copy is done struct ggml_tensor * input_dep = ggml_view_tensor(sched->ctx, input); + input_dep->src[0] = input; sched->node_backend_ids[graph_copy->n_nodes] = tensor_backend_id(input); graph_copy->nodes[graph_copy->n_nodes++] = input_dep; @@ -1417,18 +1545,56 @@ static void ggml_backend_sched_split_graph(ggml_backend_sched_t sched, struct gg graph_copy->nodes[graph_copy->n_nodes++] = graph->nodes[j]; } } + + if (sched->n_copies > 1) { + // add input copies as leafs so that they are allocated first + for (int i = 0; i < sched->n_graph_inputs; i++) { + struct ggml_tensor * input = sched->graph_inputs[i]; + size_t id = hash_id(input); + int backend_id = tensor_backend_id(input); + for (int c = 0; c < sched->n_copies; c++) { + struct ggml_tensor * input_cpy = sched->tensor_copies[id][backend_id][c]; + sched->leaf_backend_ids[graph_copy->n_leafs] = backend_id; + graph_copy->leafs[graph_copy->n_leafs++] = input_cpy; + } + } + + for (int i = 0; i < sched->n_splits; i++) { + struct ggml_backend_sched_split * split = &sched->splits[i]; + int backend_id = split->backend_id; + for (int j = 0; j < split->n_inputs; j++) { + struct ggml_tensor * input = split->inputs[j]; + size_t id = hash_id(input); + for (int c = 0; c < sched->n_copies; c++) { + struct ggml_tensor * input_cpy = sched->tensor_copies[id][backend_id][c]; + sched->leaf_backend_ids[graph_copy->n_leafs] = backend_id; + graph_copy->leafs[graph_copy->n_leafs++] = input_cpy; + } + } + } + } + + // add leafs from the original graph + for (int i = 0; i < graph->n_leafs; i++) { + struct ggml_tensor * leaf = graph->leafs[i]; + sched->leaf_backend_ids[graph_copy->n_leafs] = tensor_backend_id(leaf); + graph_copy->leafs[graph_copy->n_leafs++] = leaf; + } + sched->graph = graph_copy; } static bool ggml_backend_sched_alloc_splits(ggml_backend_sched_t sched) { - // ggml_gallocr_reserve_n(sched->galloc, sched->graph, sched->node_backend_ids); + // allocate graph if (!ggml_gallocr_alloc_graph(sched->galloc, sched->graph)) { + // the re-allocation may cause the split inputs to be moved to a different address + ggml_backend_sched_synchronize(sched); #ifndef NDEBUG - fprintf(stderr, "ggml_backend_sched: failed to allocate graph, reserving\n"); + fprintf(stderr, "%s: failed to allocate graph, reserving\n", __func__); #endif - ggml_gallocr_reserve_n(sched->galloc, sched->graph, sched->node_backend_ids); + ggml_gallocr_reserve_n(sched->galloc, sched->graph, sched->node_backend_ids, sched->leaf_backend_ids); if (!ggml_gallocr_alloc_graph(sched->galloc, sched->graph)) { - fprintf(stderr, "ggml_backend_sched: failed to allocate graph\n"); + fprintf(stderr, "%s: failed to allocate graph\n", __func__); return false; } } @@ -1437,9 +1603,6 @@ static bool ggml_backend_sched_alloc_splits(ggml_backend_sched_t sched) { } static enum ggml_status ggml_backend_sched_compute_splits(ggml_backend_sched_t sched) { - uint64_t copy_us[GGML_MAX_BACKENDS] = {0}; - uint64_t compute_us[GGML_MAX_BACKENDS] = {0}; - struct ggml_backend_sched_split * splits = sched->splits; for (int i = 0; i < sched->n_splits; i++) { @@ -1448,34 +1611,36 @@ static enum ggml_status ggml_backend_sched_compute_splits(ggml_backend_sched_t s ggml_backend_t split_backend = sched->backends[split_backend_id]; // copy the input tensors to the split backend - uint64_t copy_start_us = ggml_time_us(); for (int j = 0; j < split->n_inputs; j++) { + ggml_backend_t input_backend = ggml_backend_sched_get_tensor_backend(sched, split->inputs[j]); struct ggml_tensor * input = split->inputs[j]; - struct ggml_tensor * input_cpy = sched->tensor_copies[hash_id(input)][split_backend_id]; + struct ggml_tensor * input_cpy = sched->tensor_copies[hash_id(input)][split_backend_id][sched->cur_copy]; - GGML_ASSERT(input->buffer != NULL); - GGML_ASSERT(input_cpy->buffer != NULL); + if (input->flags & GGML_TENSOR_FLAG_INPUT) { + // inputs from the user must be copied immediately to prevent the user overwriting the data before the copy is done + if (sched->events[split_backend_id][sched->cur_copy] != NULL) { + ggml_backend_event_synchronize(sched->events[split_backend_id][sched->cur_copy]); + } else { + ggml_backend_synchronize(split_backend); + } + ggml_backend_tensor_copy(input, input_cpy); + } else { + if (sched->events[split_backend_id][sched->cur_copy] != NULL) { + ggml_backend_event_wait(split_backend, sched->events[split_backend_id][sched->cur_copy]); + } else { + ggml_backend_synchronize(split_backend); + ggml_backend_synchronize(input_backend); + } - ggml_backend_tensor_copy_async(split_backend, input, input_cpy); + ggml_backend_tensor_copy_async(input_backend, split_backend, input, input_cpy); + } } - //ggml_backend_synchronize(split_backend); // necessary to measure copy time - int64_t copy_end_us = ggml_time_us(); - copy_us[split_backend_id] += copy_end_us - copy_start_us; -#if 0 - char split_filename[GGML_MAX_NAME]; - snprintf(split_filename, GGML_MAX_NAME, "split_%i_%s.dot", i, ggml_backend_name(split_backend)); - ggml_graph_dump_dot(split->graph, NULL, split_filename); -#endif - - - uint64_t compute_start_us = ggml_time_us(); if (!sched->callback_eval) { - enum ggml_status ec = ggml_backend_graph_compute(split_backend, &split->graph); + enum ggml_status ec = ggml_backend_graph_compute_async(split_backend, &split->graph); if (ec != GGML_STATUS_SUCCESS) { return ec; } - //ggml_backend_synchronize(split_backend); // necessary to measure compute time } else { // similar to ggml_backend_compare_graph_backend for (int j0 = 0; j0 < split->graph.n_nodes; j0++) { @@ -1494,11 +1659,14 @@ static enum ggml_status ggml_backend_sched_compute_splits(ggml_backend_sched_t s struct ggml_cgraph gv = ggml_graph_view(&split->graph, j0, j1 + 1); - enum ggml_status ec = ggml_backend_graph_compute(split_backend, &gv); + enum ggml_status ec = ggml_backend_graph_compute_async(split_backend, &gv); if (ec != GGML_STATUS_SUCCESS) { return ec; } + // TODO: pass backend to the callback, then the user can decide if they want to synchronize + ggml_backend_synchronize(split_backend); + if (need && !sched->callback_eval(t, false, sched->callback_eval_user_data)) { break; } @@ -1506,39 +1674,54 @@ static enum ggml_status ggml_backend_sched_compute_splits(ggml_backend_sched_t s j0 = j1; } } - uint64_t compute_end_us = ggml_time_us(); - compute_us[split_backend_id] += compute_end_us - compute_start_us; - } -#if 0 - // per-backend timings - fprintf(stderr, "sched_compute_splits times (%d splits):\n", sched->n_splits); - for (int i = 0; i < sched->n_backends; i++) { - if (copy_us[i] > 0 || compute_us[i] > 0) { - fprintf(stderr, "\t%5.5s: %lu us copy, %lu us compute\n", ggml_backend_name(sched->backends[i]), copy_us[i], compute_us[i]); + // record the event of this copy + if (split->n_inputs > 0) { + if (sched->events[split_backend_id][sched->cur_copy] != NULL) { + ggml_backend_event_record(sched->events[split_backend_id][sched->cur_copy]); + } } } -#endif + + sched->cur_copy = (sched->cur_copy + 1) % sched->n_copies; return GGML_STATUS_SUCCESS; } -ggml_backend_sched_t ggml_backend_sched_new(ggml_backend_t * backends, ggml_backend_buffer_type_t * bufts, int n_backends, size_t graph_size) { +ggml_backend_sched_t ggml_backend_sched_new( + ggml_backend_t * backends, + ggml_backend_buffer_type_t * bufts, + int n_backends, + size_t graph_size, + bool parallel) { GGML_ASSERT(n_backends > 0); - GGML_ASSERT(n_backends <= GGML_MAX_BACKENDS); + GGML_ASSERT(n_backends <= GGML_SCHED_MAX_BACKENDS); + GGML_ASSERT(ggml_backend_is_cpu(backends[n_backends - 1])); // last backend must be CPU struct ggml_backend_sched * sched = calloc(sizeof(struct ggml_backend_sched), 1); // initialize hash table - sched->hash_set = ggml_hash_set_new(graph_size + GGML_MAX_SPLITS*GGML_MAX_SPLIT_INPUTS); + sched->hash_set = ggml_hash_set_new(graph_size + GGML_SCHED_MAX_SPLITS*GGML_SCHED_MAX_SPLIT_INPUTS); sched->tensor_backend_id = calloc(sizeof(sched->tensor_backend_id[0]), sched->hash_set.size); sched->tensor_copies = calloc(sizeof(sched->tensor_copies[0]), sched->hash_set.size); sched->node_backend_ids = calloc(sizeof(sched->node_backend_ids[0]), graph_size); + sched->leaf_backend_ids = calloc(sizeof(sched->leaf_backend_ids[0]), graph_size); sched->n_backends = n_backends; - for (int i = 0; i < n_backends; i++) { - sched->backends[i] = backends[i]; - sched->bufts[i] = bufts ? bufts[i] : ggml_backend_get_default_buffer_type(backends[i]); + + sched->n_copies = parallel ? GGML_SCHED_MAX_COPIES : 1; + + GGML_ASSERT(sched->n_copies <= GGML_SCHED_MAX_COPIES); + + for (int b = 0; b < n_backends; b++) { + sched->backends[b] = backends[b]; + sched->bufts[b] = bufts ? bufts[b] : ggml_backend_get_default_buffer_type(backends[b]); + GGML_ASSERT(ggml_backend_buft_supports_backend(sched->bufts[b], backends[b])); + if (sched->n_copies > 1) { + for (int c = 0; c < sched->n_copies; c++) { + sched->events[b][c] = ggml_backend_event_new(backends[b]); + } + } } sched->galloc = ggml_gallocr_new_n(sched->bufts, n_backends); @@ -1552,12 +1735,18 @@ void ggml_backend_sched_free(ggml_backend_sched_t sched) { if (sched == NULL) { return; } + for (int b = 0; b < sched->n_backends; b++) { + for (int c = 0; c < sched->n_copies; c++) { + ggml_backend_event_free(sched->events[b][c]); + } + } ggml_gallocr_free(sched->galloc); ggml_free(sched->ctx); free(sched->hash_set.keys); free(sched->tensor_backend_id); free(sched->tensor_copies); free(sched->node_backend_ids); + free(sched->leaf_backend_ids); free(sched); } @@ -1569,34 +1758,63 @@ void ggml_backend_sched_reset(ggml_backend_sched_t sched) { memset(sched->tensor_copies, 0, sizeof(sched->tensor_copies[0]) * hash_size); sched->is_reset = true; + sched->is_alloc = false; } bool ggml_backend_sched_reserve(ggml_backend_sched_t sched, struct ggml_cgraph * measure_graph) { ggml_backend_sched_split_graph(sched, measure_graph); - if (!ggml_gallocr_reserve_n(sched->galloc, sched->graph, sched->node_backend_ids)) { + // TODO: extract this to a separate function + if (!ggml_gallocr_reserve_n(sched->galloc, sched->graph, sched->node_backend_ids, sched->leaf_backend_ids)) { return false; } ggml_backend_sched_reset(sched); + ggml_backend_sched_synchronize(sched); + + return true; +} + +bool ggml_backend_sched_alloc_graph(ggml_backend_sched_t sched, struct ggml_cgraph * graph) { + GGML_ASSERT((int)sched->hash_set.size >= graph->n_nodes + GGML_SCHED_MAX_SPLITS*GGML_SCHED_MAX_SPLIT_INPUTS); + + ggml_backend_sched_split_graph(sched, graph); + + if (!ggml_backend_sched_alloc_splits(sched)) { + return false; + } + + sched->is_alloc = true; + return true; } enum ggml_status ggml_backend_sched_graph_compute(ggml_backend_sched_t sched, struct ggml_cgraph * graph) { - GGML_ASSERT((int)sched->hash_set.size >= graph->n_nodes + GGML_MAX_SPLITS*GGML_MAX_SPLIT_INPUTS); + enum ggml_status err = ggml_backend_sched_graph_compute_async(sched, graph); + ggml_backend_sched_synchronize(sched); + return err; +} - if (!sched->is_reset) { +enum ggml_status ggml_backend_sched_graph_compute_async(ggml_backend_sched_t sched, struct ggml_cgraph * graph) { + if (!sched->is_reset && !sched->is_alloc) { ggml_backend_sched_reset(sched); } - ggml_backend_sched_split_graph(sched, graph); - if (!ggml_backend_sched_alloc_splits(sched)) { - return GGML_STATUS_ALLOC_FAILED; + if (!sched->is_alloc) { + if (!ggml_backend_sched_alloc_graph(sched, graph)) { + return GGML_STATUS_ALLOC_FAILED; + } } return ggml_backend_sched_compute_splits(sched); } +void ggml_backend_sched_synchronize(ggml_backend_sched_t sched) { + for (int i = 0; i < sched->n_backends; i++) { + ggml_backend_synchronize(sched->backends[i]); + } +} + void ggml_backend_sched_set_eval_callback(ggml_backend_sched_t sched, ggml_backend_sched_eval_callback callback, void * user_data) { sched->callback_eval = callback; sched->callback_eval_user_data = user_data; @@ -1606,19 +1824,24 @@ int ggml_backend_sched_get_n_splits(ggml_backend_sched_t sched) { return sched->n_splits; } +int ggml_backend_sched_get_n_copies(ggml_backend_sched_t sched) { + return sched->n_copies; +} + size_t ggml_backend_sched_get_buffer_size(ggml_backend_sched_t sched, ggml_backend_t backend) { int backend_index = ggml_backend_sched_backend_id(sched, backend); GGML_ASSERT(backend_index >= 0 && backend_index < sched->n_backends); + return ggml_gallocr_get_buffer_size(sched->galloc, backend_index); } -void ggml_backend_sched_set_node_backend(ggml_backend_sched_t sched, struct ggml_tensor * node, ggml_backend_t backend) { +void ggml_backend_sched_set_tensor_backend(ggml_backend_sched_t sched, struct ggml_tensor * node, ggml_backend_t backend) { int backend_index = ggml_backend_sched_backend_id(sched, backend); GGML_ASSERT(backend_index >= 0 && backend_index < sched->n_backends); tensor_backend_id(node) = backend_index; } -ggml_backend_t ggml_backend_sched_get_node_backend(ggml_backend_sched_t sched, struct ggml_tensor * node) { +ggml_backend_t ggml_backend_sched_get_tensor_backend(ggml_backend_sched_t sched, struct ggml_tensor * node) { int backend_index = tensor_backend_id(node); if (backend_index == -1) { return NULL; diff --git a/ggml-backend.h b/ggml-backend.h index 8bed22578..099d9c258 100644 --- a/ggml-backend.h +++ b/ggml-backend.h @@ -9,6 +9,7 @@ extern "C" { typedef struct ggml_backend_buffer_type * ggml_backend_buffer_type_t; typedef struct ggml_backend_buffer * ggml_backend_buffer_t; + typedef struct ggml_backend_event * ggml_backend_event_t; typedef struct ggml_backend * ggml_backend_t; typedef void * ggml_backend_graph_plan_t; @@ -72,11 +73,24 @@ extern "C" { GGML_API enum ggml_status ggml_backend_graph_plan_compute(ggml_backend_t backend, ggml_backend_graph_plan_t plan); GGML_API enum ggml_status ggml_backend_graph_compute (ggml_backend_t backend, struct ggml_cgraph * cgraph); + GGML_API bool ggml_backend_graph_compute_async(ggml_backend_t backend, struct ggml_cgraph * cgraph); GGML_API bool ggml_backend_supports_op(ggml_backend_t backend, const struct ggml_tensor * op); // tensor copy between different backends GGML_API void ggml_backend_tensor_copy(struct ggml_tensor * src, struct ggml_tensor * dst); - GGML_API void ggml_backend_tensor_copy_async(ggml_backend_t backend, struct ggml_tensor * src, struct ggml_tensor * dst); // automatic fallback to sync copy + + // asynchronous copy + // the copy is performed after all the currently queued operations in backend_src + // backend_dst will wait for the copy to complete before performing other operations + // automatic fallback to sync copy if async is not supported + GGML_API void ggml_backend_tensor_copy_async(ggml_backend_t backend_src, ggml_backend_t backend_dst, struct ggml_tensor * src, struct ggml_tensor * dst); + + // events + GGML_API ggml_backend_event_t ggml_backend_event_new (ggml_backend_t backend); + GGML_API void ggml_backend_event_free (ggml_backend_event_t event); + GGML_API void ggml_backend_event_record (ggml_backend_event_t event); + GGML_API void ggml_backend_event_synchronize(ggml_backend_event_t event); + GGML_API void ggml_backend_event_wait (ggml_backend_t backend, ggml_backend_event_t event); // wait async on event // // CPU backend @@ -123,27 +137,31 @@ extern "C" { /* Example usage: - sched = ggml_backend_sched_new({backend_gpu, backend_gpu2, backend_cpu}, num_backends); - // sched is initialized with measure allocators and cannot be used until allocated with a measure graph + // operations that use tensors allocated in a buffer with USAGE_WEIGHTS will be asigned + // preferrably to run on the same backend as the buffer + ggml_backend_buffer_set_usage(buf_weights, GGML_BACKEND_BUFFER_USAGE_WEIGHTS); - // initialize buffers from a measure graph - measure_graph = build_graph(sched); // use the allocr to allocate inputs as needed + sched = ggml_backend_sched_new({backend_gpu, backend_gpu2, backend_cpu}, NULL, num_backends, GGML_DEFAULT_GRAPH_SIZE, false); - // in build_graph: - build_graph(...) { - // manually assign nodes to a backend (optional, should not be needed in most cases) - struct ggml_tensor * node = ggml_mul_mat(ctx, ...); - ggml_backend_sched_set_node_backend(sched, node, backend_gpu); - } + // initialize buffers from a max size graph (optional) + reserve_graph = build_graph(sched, max_batch_size); - // allocate backend buffers from measure graph - ggml_backend_sched_init_measure(sched, measure_graph); + // manually assign nodes to a backend (optional, should not be needed in most cases) + struct ggml_tensor * node = ggml_mul_mat(ctx, ...); + ggml_backend_sched_set_tensor_backend(sched, node, backend_gpu); - // the scheduler is now ready to compute graphs + ggml_backend_sched_reserve(sched, reserve_graph); // compute graph = build_graph(sched); ggml_backend_sched_graph_compute(sched, graph); + + // if there are graph inputs: + ggml_backend_sched_reset(sched); + ggml_backend_sched_alloc_graph(sched, graph); + ggml_backend_tensor_set(input_tensor, ...); + ggml_backend_sched_graph_compute(sched, graph); + } */ struct ggml_backend_sched; @@ -158,20 +176,26 @@ extern "C" { typedef bool (*ggml_backend_sched_eval_callback)(struct ggml_tensor * t, bool ask, void * user_data); // Initialize a backend scheduler - GGML_API ggml_backend_sched_t ggml_backend_sched_new(ggml_backend_t * backends, ggml_backend_buffer_type_t * bufts, int n_backends, size_t graph_size); + GGML_API ggml_backend_sched_t ggml_backend_sched_new(ggml_backend_t * backends, ggml_backend_buffer_type_t * bufts, int n_backends, size_t graph_size, bool parallel); GGML_API void ggml_backend_sched_free(ggml_backend_sched_t sched); + // Initialize backend buffers from a measure graph GGML_API bool ggml_backend_sched_reserve(ggml_backend_sched_t sched, struct ggml_cgraph * measure_graph); + // Get the number of splits of the last graph GGML_API int ggml_backend_sched_get_n_splits(ggml_backend_sched_t sched); + GGML_API int ggml_backend_sched_get_n_copies(ggml_backend_sched_t sched); GGML_API size_t ggml_backend_sched_get_buffer_size(ggml_backend_sched_t sched, ggml_backend_t backend); - GGML_API void ggml_backend_sched_set_node_backend(ggml_backend_sched_t sched, struct ggml_tensor * node, ggml_backend_t backend); - GGML_API ggml_backend_t ggml_backend_sched_get_node_backend(ggml_backend_sched_t sched, struct ggml_tensor * node); + GGML_API void ggml_backend_sched_set_tensor_backend(ggml_backend_sched_t sched, struct ggml_tensor * node, ggml_backend_t backend); + GGML_API ggml_backend_t ggml_backend_sched_get_tensor_backend(ggml_backend_sched_t sched, struct ggml_tensor * node); // Allocate and compute graph on the backend scheduler + GGML_API bool ggml_backend_sched_alloc_graph(ggml_backend_sched_t sched, struct ggml_cgraph * graph); GGML_API enum ggml_status ggml_backend_sched_graph_compute(ggml_backend_sched_t sched, struct ggml_cgraph * graph); + GGML_API enum ggml_status ggml_backend_sched_graph_compute_async(ggml_backend_sched_t sched, struct ggml_cgraph * graph); + GGML_API void ggml_backend_sched_synchronize(ggml_backend_sched_t sched); // Reset all assignments and allocators - must be called before changing the node backends GGML_API void ggml_backend_sched_reset(ggml_backend_sched_t sched); diff --git a/ggml-cuda.cu b/ggml-cuda.cu index b8834ed05..d1b5e52ba 100644 --- a/ggml-cuda.cu +++ b/ggml-cuda.cu @@ -72,6 +72,7 @@ #define cudaEventCreateWithFlags hipEventCreateWithFlags #define cudaEventDisableTiming hipEventDisableTiming #define cudaEventRecord hipEventRecord +#define cudaEventSynchronize hipEventSynchronize #define cudaEvent_t hipEvent_t #define cudaEventDestroy hipEventDestroy #define cudaFree hipFree @@ -81,6 +82,7 @@ #define cudaGetDeviceProperties hipGetDeviceProperties #define cudaGetErrorString hipGetErrorString #define cudaGetLastError hipGetLastError +#define cudaLaunchHostFunc hipLaunchHostFunc #ifdef GGML_HIP_UMA #define cudaMalloc hipMallocManaged #define cudaMallocHost(ptr, size) hipHostMalloc(ptr, size) @@ -104,6 +106,7 @@ #define cudaStreamCreateWithFlags hipStreamCreateWithFlags #define cudaStreamFireAndForget hipStreamFireAndForget #define cudaStreamNonBlocking hipStreamNonBlocking +#define cudaStreamPerThread hipStreamPerThread #define cudaStreamSynchronize hipStreamSynchronize #define cudaStreamWaitEvent(stream, event, flags) hipStreamWaitEvent(stream, event, flags) #define cudaStream_t hipStream_t @@ -10641,8 +10644,20 @@ GGML_CALL void ggml_cuda_get_device_description(int device, char * description, #define UNUSED GGML_UNUSED struct ggml_backend_cuda_context { + explicit ggml_backend_cuda_context(int device) : + device(device), + name(GGML_CUDA_NAME + std::to_string(device)) { + } + + ~ggml_backend_cuda_context() { + if (copy_event != nullptr) { + CUDA_CHECK(cudaEventDestroy(copy_event)); + } + } + int device; std::string name; + cudaEvent_t copy_event = nullptr; }; // cuda buffer @@ -10732,9 +10747,8 @@ GGML_CALL static void ggml_backend_cuda_buffer_set_tensor(ggml_backend_buffer_t ggml_backend_cuda_buffer_context * ctx = (ggml_backend_cuda_buffer_context *)buffer->context; ggml_cuda_set_device(ctx->device); - CUDA_CHECK(cudaDeviceSynchronize()); - CUDA_CHECK(cudaMemcpy((char *)tensor->data + offset, data, size, cudaMemcpyHostToDevice)); - CUDA_CHECK(cudaDeviceSynchronize()); + CUDA_CHECK(cudaMemcpyAsync((char *)tensor->data + offset, data, size, cudaMemcpyHostToDevice, cudaStreamPerThread)); + CUDA_CHECK(cudaStreamSynchronize(cudaStreamPerThread)); } GGML_CALL static void ggml_backend_cuda_buffer_get_tensor(ggml_backend_buffer_t buffer, const ggml_tensor * tensor, void * data, size_t offset, size_t size) { @@ -10743,26 +10757,25 @@ GGML_CALL static void ggml_backend_cuda_buffer_get_tensor(ggml_backend_buffer_t ggml_backend_cuda_buffer_context * ctx = (ggml_backend_cuda_buffer_context *)buffer->context; ggml_cuda_set_device(ctx->device); - CUDA_CHECK(cudaDeviceSynchronize()); - CUDA_CHECK(cudaMemcpy(data, (const char *)tensor->data + offset, size, cudaMemcpyDeviceToHost)); - CUDA_CHECK(cudaDeviceSynchronize()); + CUDA_CHECK(cudaMemcpyAsync(data, (const char *)tensor->data + offset, size, cudaMemcpyDeviceToHost, cudaStreamPerThread)); + CUDA_CHECK(cudaStreamSynchronize(cudaStreamPerThread)); } GGML_CALL static bool ggml_backend_cuda_buffer_cpy_tensor(ggml_backend_buffer_t buffer, const ggml_tensor * src, ggml_tensor * dst) { if (ggml_backend_buffer_is_cuda(src->buffer)) { ggml_backend_cuda_buffer_context * src_ctx = (ggml_backend_cuda_buffer_context *)src->buffer->context; - ggml_backend_cuda_buffer_context * dst_ctx = (ggml_backend_cuda_buffer_context *)buffer->context; - - ggml_cuda_set_device(src_ctx->device); - CUDA_CHECK(cudaDeviceSynchronize()); - ggml_cuda_set_device(dst_ctx->device); - CUDA_CHECK(cudaDeviceSynchronize()); - CUDA_CHECK(cudaMemcpy((char *)dst->data, (const char *)src->data, ggml_nbytes(src), cudaMemcpyDeviceToDevice)); - CUDA_CHECK(cudaDeviceSynchronize()); - + ggml_backend_cuda_buffer_context * dst_ctx = (ggml_backend_cuda_buffer_context *)dst->buffer->context; + if (src_ctx->device == dst_ctx->device) { + CUDA_CHECK(cudaMemcpyAsync(dst->data, src->data, ggml_nbytes(src), cudaMemcpyDeviceToDevice, cudaStreamPerThread)); + } else { + CUDA_CHECK(cudaMemcpyPeerAsync(dst->data, dst_ctx->device, src->data, src_ctx->device, ggml_nbytes(src), cudaStreamPerThread)); + } + CUDA_CHECK(cudaStreamSynchronize(cudaStreamPerThread)); return true; } return false; + + UNUSED(buffer); } GGML_CALL static void ggml_backend_cuda_buffer_clear(ggml_backend_buffer_t buffer, uint8_t value) { @@ -11007,7 +11020,11 @@ GGML_CALL static void ggml_backend_cuda_split_buffer_set_tensor(ggml_backend_buf } const char * buf_host = (const char *)data + offset_split; - CUDA_CHECK(cudaMemcpy(extra->data_device[id], buf_host, original_size, cudaMemcpyHostToDevice)); + CUDA_CHECK(cudaMemcpyAsync(extra->data_device[id], buf_host, original_size, cudaMemcpyHostToDevice, cudaStreamPerThread)); + } + + for (int id = 0; id < g_device_count; ++id) { + CUDA_CHECK(cudaStreamSynchronize(cudaStreamPerThread)); } } @@ -11041,7 +11058,11 @@ GGML_CALL static void ggml_backend_cuda_split_buffer_get_tensor(ggml_backend_buf } char * buf_host = (char *)data + offset_split; - CUDA_CHECK(cudaMemcpy(buf_host, extra->data_device[id], original_size, cudaMemcpyDeviceToHost)); + CUDA_CHECK(cudaMemcpyAsync(buf_host, extra->data_device[id], original_size, cudaMemcpyDeviceToHost, cudaStreamPerThread)); + } + + for (int id = 0; id < g_device_count; ++id) { + CUDA_CHECK(cudaStreamSynchronize(cudaStreamPerThread)); } } @@ -11220,6 +11241,10 @@ GGML_CALL ggml_backend_buffer_type_t ggml_backend_cuda_host_buffer_type() { return &ggml_backend_cuda_buffer_type_host; } +//static bool ggml_backend_buffer_is_cuda_host(ggml_backend_buffer_t buffer) { +// return buffer->buft->iface.get_name == ggml_backend_cuda_host_buffer_type_name; +//} + // backend GGML_CALL static const char * ggml_backend_cuda_name(ggml_backend_t backend) { @@ -11243,8 +11268,9 @@ GGML_CALL static ggml_backend_buffer_type_t ggml_backend_cuda_get_default_buffer GGML_CALL static void ggml_backend_cuda_set_tensor_async(ggml_backend_t backend, ggml_tensor * tensor, const void * data, size_t offset, size_t size) { ggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *)backend->context; + ggml_backend_buffer_t buf = tensor->view_src ? tensor->view_src->buffer : tensor->buffer; - GGML_ASSERT(tensor->buffer->buft == ggml_backend_cuda_buffer_type(cuda_ctx->device) && "unsupported buffer type"); + GGML_ASSERT(buf->buft == ggml_backend_cuda_buffer_type(cuda_ctx->device) && "unsupported buffer type"); GGML_ASSERT(tensor->backend == GGML_BACKEND_TYPE_GPU); CUDA_CHECK(cudaMemcpyAsync((char *)tensor->data + offset, data, size, cudaMemcpyHostToDevice, g_cudaStreams[cuda_ctx->device][0])); @@ -11252,22 +11278,61 @@ GGML_CALL static void ggml_backend_cuda_set_tensor_async(ggml_backend_t backend, GGML_CALL static void ggml_backend_cuda_get_tensor_async(ggml_backend_t backend, const ggml_tensor * tensor, void * data, size_t offset, size_t size) { ggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *)backend->context; + ggml_backend_buffer_t buf = tensor->view_src ? tensor->view_src->buffer : tensor->buffer; - GGML_ASSERT(tensor->buffer->buft == ggml_backend_cuda_buffer_type(cuda_ctx->device) && "unsupported buffer type"); + GGML_ASSERT(buf->buft == ggml_backend_cuda_buffer_type(cuda_ctx->device) && "unsupported buffer type"); GGML_ASSERT(tensor->backend == GGML_BACKEND_TYPE_GPU); CUDA_CHECK(cudaMemcpyAsync(data, (const char *)tensor->data + offset, size, cudaMemcpyDeviceToHost, g_cudaStreams[cuda_ctx->device][0])); } -GGML_CALL static bool ggml_backend_cuda_cpy_tensor_async(ggml_backend_t backend, const ggml_tensor * src, ggml_tensor * dst) { - ggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *)backend->context; +GGML_CALL static bool ggml_backend_cuda_cpy_tensor_async(ggml_backend_t backend_src, ggml_backend_t backend_dst, const ggml_tensor * src, ggml_tensor * dst) { + GGML_ASSERT(ggml_backend_is_cuda(backend_src) || ggml_backend_is_cuda(backend_dst)); - if (dst->buffer->buft == ggml_backend_cuda_buffer_type(cuda_ctx->device) && ggml_backend_buffer_is_cuda(src->buffer)) { - CUDA_CHECK(cudaMemcpyAsync(dst->data, src->data, ggml_nbytes(dst), cudaMemcpyDeviceToDevice, g_cudaStreams[cuda_ctx->device][0])); - return true; + ggml_backend_buffer_t buf_src = src->view_src ? src->view_src->buffer : src->buffer; + ggml_backend_buffer_t buf_dst = dst->view_src ? dst->view_src->buffer : dst->buffer; + + if (!ggml_backend_buffer_is_cuda(src->buffer)) { + return false; } - return false; + if (!ggml_backend_buffer_is_cuda(dst->buffer)) { + return false; + } + + // device -> device + ggml_backend_cuda_context * cuda_ctx_src = (ggml_backend_cuda_context *)backend_src->context; + ggml_backend_cuda_context * cuda_ctx_dst = (ggml_backend_cuda_context *)backend_dst->context; + + if (backend_src != backend_dst) { + ggml_backend_cuda_buffer_context * buf_ctx_src = (ggml_backend_cuda_buffer_context *)buf_src->context; + ggml_backend_cuda_buffer_context * buf_ctx_dst = (ggml_backend_cuda_buffer_context *)buf_dst->context; + + GGML_ASSERT(cuda_ctx_src->device == buf_ctx_src->device); + GGML_ASSERT(cuda_ctx_dst->device == buf_ctx_dst->device); + + if (!cuda_ctx_src->copy_event) { + ggml_cuda_set_device(cuda_ctx_src->device); + CUDA_CHECK(cudaEventCreateWithFlags(&cuda_ctx_src->copy_event, cudaEventDisableTiming)); + } + + // copy on src stream + if (cuda_ctx_src->device == cuda_ctx_dst->device) { + CUDA_CHECK(cudaMemcpyAsync(dst->data, src->data, ggml_nbytes(dst), cudaMemcpyDeviceToDevice, g_cudaStreams[cuda_ctx_dst->device][0])); + } else { + CUDA_CHECK(cudaMemcpyPeerAsync(dst->data, cuda_ctx_dst->device, src->data, cuda_ctx_src->device, ggml_nbytes(dst), g_cudaStreams[cuda_ctx_src->device][0])); + } + + // record event on src stream + CUDA_CHECK(cudaEventRecord(cuda_ctx_src->copy_event, g_cudaStreams[cuda_ctx_src->device][0])); + + // wait on dst stream for the copy to complete + CUDA_CHECK(cudaStreamWaitEvent(g_cudaStreams[cuda_ctx_dst->device][0], cuda_ctx_src->copy_event, 0)); + } else { + // src and dst are on the same backend + CUDA_CHECK(cudaMemcpyAsync(dst->data, src->data, ggml_nbytes(dst), cudaMemcpyDeviceToDevice, g_cudaStreams[cuda_ctx_dst->device][0])); + } + return true; } GGML_CALL static void ggml_backend_cuda_synchronize(ggml_backend_t backend) { @@ -11444,6 +11509,52 @@ GGML_CALL static bool ggml_backend_cuda_supports_op(ggml_backend_t backend, cons UNUSED(backend); } +static ggml_backend_event_t ggml_backend_cuda_event_new(ggml_backend_t backend) { + ggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *)backend->context; + + ggml_cuda_set_device(cuda_ctx->device); + + cudaEvent_t event; + CUDA_CHECK(cudaEventCreateWithFlags(&event, cudaEventDisableTiming)); + + return new ggml_backend_event { + /* .backend = */ backend, + /* .context = */ event, + }; +} + +static void ggml_backend_cuda_event_free(ggml_backend_event_t event) { + CUDA_CHECK(cudaEventDestroy((cudaEvent_t)event->context)); + + delete event; +} + +static void ggml_backend_cuda_event_record(ggml_backend_event_t event) { + ggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *)event->backend->context; + + CUDA_CHECK(cudaEventRecord((cudaEvent_t)event->context, g_cudaStreams[cuda_ctx->device][0])); +} + +static void ggml_backend_cuda_event_wait(ggml_backend_t backend, ggml_backend_event_t event) { + ggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *)backend->context; + + if (ggml_backend_is_cuda(event->backend)) { + CUDA_CHECK(cudaStreamWaitEvent(g_cudaStreams[cuda_ctx->device][0], (cudaEvent_t)event->context, 0)); + } else { + // untested + auto wait_fn = [](void * user_data) { + ggml_backend_event_t event = (ggml_backend_event_t)user_data; + ggml_backend_event_synchronize(event); + }; + + CUDA_CHECK(cudaLaunchHostFunc(g_cudaStreams[cuda_ctx->device][0], wait_fn, event)); + } +} + +static void ggml_backend_cuda_event_synchronize(ggml_backend_event_t event) { + CUDA_CHECK(cudaEventSynchronize((cudaEvent_t)event->context)); +} + static ggml_backend_i ggml_backend_cuda_interface = { /* .get_name = */ ggml_backend_cuda_name, /* .free = */ ggml_backend_cuda_free, @@ -11457,6 +11568,11 @@ static ggml_backend_i ggml_backend_cuda_interface = { /* .graph_plan_compute = */ NULL, /* .graph_compute = */ ggml_backend_cuda_graph_compute, /* .supports_op = */ ggml_backend_cuda_supports_op, + /* .event_new = */ ggml_backend_cuda_event_new, + /* .event_free = */ ggml_backend_cuda_event_free, + /* .event_record = */ ggml_backend_cuda_event_record, + /* .event_wait = */ ggml_backend_cuda_event_wait, + /* .event_synchronize = */ ggml_backend_cuda_event_synchronize, }; static ggml_guid_t ggml_backend_cuda_guid() { @@ -11475,10 +11591,11 @@ GGML_CALL ggml_backend_t ggml_backend_cuda_init(int device) { // not strictly necessary, but it may reduce the overhead of the first graph_compute ggml_cuda_set_main_device(device); - ggml_backend_cuda_context * ctx = new ggml_backend_cuda_context { - /* .device = */ device, - /* .name = */ GGML_CUDA_NAME + std::to_string(device), - }; + ggml_backend_cuda_context * ctx = new ggml_backend_cuda_context(device); + if (ctx == nullptr) { + fprintf(stderr, "%s: error: failed to allocate context\n", __func__); + return nullptr; + } ggml_backend_t cuda_backend = new ggml_backend { /* .guid = */ ggml_backend_cuda_guid(), diff --git a/ggml-kompute.cpp b/ggml-kompute.cpp index 83a7822fd..4caf2c9e7 100644 --- a/ggml-kompute.cpp +++ b/ggml-kompute.cpp @@ -1951,6 +1951,11 @@ static struct ggml_backend_i kompute_backend_i = { /* .graph_plan_compute = */ NULL, /* .graph_compute = */ ggml_backend_kompute_graph_compute, /* .supports_op = */ ggml_backend_kompute_supports_op, + /* .event_new = */ NULL, + /* .event_free = */ NULL, + /* .event_record = */ NULL, + /* .event_wait = */ NULL, + /* .event_synchronize = */ NULL, }; static ggml_guid_t ggml_backend_kompute_guid() { diff --git a/ggml-metal.m b/ggml-metal.m index 1825d3320..3a5476c52 100644 --- a/ggml-metal.m +++ b/ggml-metal.m @@ -2820,6 +2820,11 @@ static struct ggml_backend_i ggml_backend_metal_i = { /* .graph_plan_compute = */ NULL, /* .graph_compute = */ ggml_backend_metal_graph_compute, /* .supports_op = */ ggml_backend_metal_supports_op, + /* .event_new = */ NULL, + /* .event_free = */ NULL, + /* .event_record = */ NULL, + /* .event_wait = */ NULL, + /* .event_synchronize = */ NULL, }; void ggml_backend_metal_log_set_callback(ggml_log_callback log_callback, void * user_data) { diff --git a/ggml-sycl.cpp b/ggml-sycl.cpp index c2ab13034..9f6506383 100644 --- a/ggml-sycl.cpp +++ b/ggml-sycl.cpp @@ -17249,13 +17249,18 @@ static ggml_backend_i ggml_backend_sycl_interface = { /* .get_default_buffer_type = */ ggml_backend_sycl_get_default_buffer_type, /* .set_tensor_async = */ ggml_backend_sycl_set_tensor_async, /* .get_tensor_async = */ ggml_backend_sycl_get_tensor_async, - /* .cpy_tensor_async = */ ggml_backend_sycl_cpy_tensor_async, + /* .cpy_tensor_async = */ NULL, //ggml_backend_sycl_cpy_tensor_async, // TODO: update for the new interface /* .synchronize = */ ggml_backend_sycl_synchronize, /* .graph_plan_create = */ NULL, /* .graph_plan_free = */ NULL, /* .graph_plan_compute = */ NULL, /* .graph_compute = */ ggml_backend_sycl_graph_compute, /* .supports_op = */ ggml_backend_sycl_supports_op, + /* .event_new = */ NULL, + /* .event_free = */ NULL, + /* .event_record = */ NULL, + /* .event_wait = */ NULL, + /* .event_synchronize = */ NULL, }; static ggml_guid_t ggml_backend_sycl_guid() { diff --git a/ggml-vulkan.cpp b/ggml-vulkan.cpp index d41aa7d22..7cce616ba 100644 --- a/ggml-vulkan.cpp +++ b/ggml-vulkan.cpp @@ -5693,6 +5693,11 @@ static ggml_backend_i ggml_backend_vk_interface = { /* .graph_plan_compute = */ NULL, /* .graph_compute = */ ggml_backend_vk_graph_compute, /* .supports_op = */ ggml_backend_vk_supports_op, + /* .event_new = */ NULL, + /* .event_free = */ NULL, + /* .event_record = */ NULL, + /* .event_wait = */ NULL, + /* .event_synchronize = */ NULL, }; static ggml_guid_t ggml_backend_vk_guid() { diff --git a/ggml.c b/ggml.c index 9a7bd1d8c..fbc66f65b 100644 --- a/ggml.c +++ b/ggml.c @@ -11560,8 +11560,6 @@ static void ggml_compute_forward_get_rows_q( const struct ggml_tensor * src0 = dst->src[0]; const struct ggml_tensor * src1 = dst->src[1]; - assert(params->ith == 0); - if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) { return; } @@ -11569,7 +11567,7 @@ static void ggml_compute_forward_get_rows_q( GGML_TENSOR_BINARY_OP_LOCALS const int64_t nc = ne00; - const int64_t nr = ggml_nelements(src1); GGML_UNUSED(nr); + const int64_t nr = ggml_nelements(src1); const enum ggml_type type = src0->type; ggml_to_float_t const dequantize_row_q = type_traits[type].to_float; @@ -11579,17 +11577,25 @@ static void ggml_compute_forward_get_rows_q( assert(nb00 == ggml_type_size(type)); assert(ggml_nrows(dst) == nr); - // TODO: multi-thread - for (int64_t i12 = 0; i12 < ne12; ++i12) { - for (int64_t i11 = 0; i11 < ne11; ++i11) { - for (int64_t i10 = 0; i10 < ne10; ++i10) { - const int64_t i01 = *(int32_t *) ((char *) src1->data + i10*nb10 + i11*nb11 + i12*nb12); + const int ith = params->ith; + const int nth = params->nth; - dequantize_row_q( - (const void *) ((char *) src0->data + i01*nb01 + i11*nb02 + i12*nb03), - (float *) ((char *) dst->data + i10*nb1 + i11*nb2 + i12*nb3), nc); - } - } + // rows per thread + const int dr = (nr + nth - 1)/nth; + + // row range for this thread + const int ir0 = dr*ith; + const int ir1 = MIN(ir0 + dr, nr); + + for (int64_t i = ir0; i < ir1; ++i) { + const int64_t i12 = i/(ne11*ne10); + const int64_t i11 = (i - i12*ne11*ne10)/ne10; + const int64_t i10 = (i - i12*ne11*ne10 - i11*ne10); + const int64_t i01 = *(int32_t *) ((char *) src1->data + i10*nb10 + i11*nb11 + i12*nb12); + + dequantize_row_q( + (const void *) ((char *) src0->data + i01*nb01 + i11*nb02 + i12*nb03), + (float *) ((char *) dst->data + i10*nb1 + i11*nb2 + i12*nb3), nc); } } @@ -11600,8 +11606,6 @@ static void ggml_compute_forward_get_rows_f16( const struct ggml_tensor * src0 = dst->src[0]; const struct ggml_tensor * src1 = dst->src[1]; - assert(params->ith == 0); - if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) { return; } @@ -11609,24 +11613,32 @@ static void ggml_compute_forward_get_rows_f16( GGML_TENSOR_BINARY_OP_LOCALS const int64_t nc = ne00; - const int64_t nr = ggml_nelements(src1); GGML_UNUSED(nr); + const int64_t nr = ggml_nelements(src1); assert(ne0 == nc); assert(ne02 == ne11); assert(nb00 == sizeof(ggml_fp16_t)); assert(ggml_nrows(dst) == nr); - // TODO: multi-thread - for (int64_t i12 = 0; i12 < ne12; ++i12) { - for (int64_t i11 = 0; i11 < ne11; ++i11) { - for (int64_t i10 = 0; i10 < ne10; ++i10) { - const int64_t i01 = *(int32_t *) ((char *) src1->data + i10*nb10 + i11*nb11 + i12*nb12); + const int ith = params->ith; + const int nth = params->nth; - ggml_fp16_to_fp32_row( - (const void *) ((char *) src0->data + i01*nb01 + i11*nb02 + i12*nb03), - (float *) ((char *) dst->data + i10*nb1 + i11*nb2 + i12*nb3), nc); - } - } + // rows per thread + const int dr = (nr + nth - 1)/nth; + + // row range for this thread + const int ir0 = dr*ith; + const int ir1 = MIN(ir0 + dr, nr); + + for (int64_t i = ir0; i < ir1; ++i) { + const int64_t i12 = i/(ne11*ne10); + const int64_t i11 = (i - i12*ne11*ne10)/ne10; + const int64_t i10 = (i - i12*ne11*ne10 - i11*ne10); + const int64_t i01 = *(int32_t *) ((char *) src1->data + i10*nb10 + i11*nb11 + i12*nb12); + + ggml_fp16_to_fp32_row( + (const void *) ((char *) src0->data + i01*nb01 + i11*nb02 + i12*nb03), + (float *) ((char *) dst->data + i10*nb1 + i11*nb2 + i12*nb3), nc); } } @@ -11637,8 +11649,6 @@ static void ggml_compute_forward_get_rows_f32( const struct ggml_tensor * src0 = dst->src[0]; const struct ggml_tensor * src1 = dst->src[1]; - assert(params->ith == 0); - if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) { return; } @@ -11646,24 +11656,32 @@ static void ggml_compute_forward_get_rows_f32( GGML_TENSOR_BINARY_OP_LOCALS const int64_t nc = ne00; - const int64_t nr = ggml_nelements(src1); GGML_UNUSED(nr); + const int64_t nr = ggml_nelements(src1); assert(ne0 == nc); assert(ne02 == ne11); assert(nb00 == sizeof(float)); assert(ggml_nrows(dst) == nr); - // TODO: multi-thread - for (int64_t i12 = 0; i12 < ne12; ++i12) { - for (int64_t i11 = 0; i11 < ne11; ++i11) { - for (int64_t i10 = 0; i10 < ne10; ++i10) { - const int64_t i01 = *(int32_t *) ((char *) src1->data + i10*nb10 + i11*nb11 + i12*nb12); + const int ith = params->ith; + const int nth = params->nth; - ggml_vec_cpy_f32(nc, - (float *) ((char *) dst->data + i10*nb1 + i11*nb2 + i12*nb3), - (float *) ((char *) src0->data + i01*nb01 + i11*nb02 + i12*nb03)); - } - } + // rows per thread + const int dr = (nr + nth - 1)/nth; + + // row range for this thread + const int ir0 = dr*ith; + const int ir1 = MIN(ir0 + dr, nr); + + for (int64_t i = ir0; i < ir1; ++i) { + const int64_t i12 = i/(ne11*ne10); + const int64_t i11 = (i - i12*ne11*ne10)/ne10; + const int64_t i10 = (i - i12*ne11*ne10 - i11*ne10); + const int64_t i01 = *(int32_t *) ((char *) src1->data + i10*nb10 + i11*nb11 + i12*nb12); + + ggml_vec_cpy_f32(nc, + (float *) ((char *) dst->data + i10*nb1 + i11*nb2 + i12*nb3), + (float *) ((char *) src0->data + i01*nb01 + i11*nb02 + i12*nb03)); } } @@ -17796,7 +17814,7 @@ static void ggml_graph_compute_perf_stats_node(struct ggml_tensor * node, const node->perf_time_us += time_us_cur; } -static int ggml_get_n_tasks(struct ggml_tensor * node, int n_threads) { +static int ggml_get_n_tasks(struct ggml_tensor * node, int n_threads, int n_cur_threads) { int n_tasks = 0; switch (node->op) { @@ -17877,6 +17895,12 @@ static int ggml_get_n_tasks(struct ggml_tensor * node, int n_threads) { { n_tasks = n_threads; } break; + case GGML_OP_GET_ROWS: + { + // FIXME: the cost of launching additional threads decreases performance with GPU offloading + //n_tasks = MIN(n_threads, ggml_nelements(node->src[1])); + n_tasks = MIN(n_cur_threads, ggml_nelements(node->src[1])); + } break; case GGML_OP_SCALE: case GGML_OP_SET: case GGML_OP_CONT: @@ -17884,7 +17908,6 @@ static int ggml_get_n_tasks(struct ggml_tensor * node, int n_threads) { case GGML_OP_VIEW: case GGML_OP_PERMUTE: case GGML_OP_TRANSPOSE: - case GGML_OP_GET_ROWS: case GGML_OP_GET_ROWS_BACK: case GGML_OP_DIAG: { @@ -18102,7 +18125,7 @@ static thread_ret_t ggml_graph_compute_thread(void * data) { /* FINALIZE */ struct ggml_tensor * node = cgraph->nodes[node_n]; if (GGML_OP_HAS_FINALIZE[node->op]) { - params.nth = ggml_get_n_tasks(node, n_threads); + params.nth = ggml_get_n_tasks(node, n_threads, state->shared->n_threads); ggml_compute_forward(¶ms, node); } ggml_graph_compute_perf_stats_node(node, state->shared); @@ -18112,7 +18135,7 @@ static thread_ret_t ggml_graph_compute_thread(void * data) { while (++node_n < cgraph->n_nodes) { GGML_PRINT_DEBUG_5("%s: %d/%d\n", __func__, node_n, cgraph->n_nodes); struct ggml_tensor * node = cgraph->nodes[node_n]; - const int n_tasks = ggml_get_n_tasks(node, n_threads); + const int n_tasks = ggml_get_n_tasks(node, n_threads, state->shared->n_threads); state->shared->perf_node_start_cycles = ggml_perf_cycles(); state->shared->perf_node_start_time_us = ggml_perf_time_us(); @@ -18160,7 +18183,7 @@ static thread_ret_t ggml_graph_compute_thread(void * data) { /* INIT & COMPUTE */ struct ggml_tensor * node = cgraph->nodes[node_n]; - const int n_tasks = ggml_get_n_tasks(node, n_threads); + const int n_tasks = ggml_get_n_tasks(node, n_threads, state->shared->n_threads); struct ggml_compute_params params = { /*.type =*/ GGML_TASK_TYPE_INIT, @@ -18225,7 +18248,7 @@ struct ggml_cplan ggml_graph_plan(const struct ggml_cgraph * cgraph, int n_threa for (int i = 0; i < cgraph->n_nodes; i++) { struct ggml_tensor * node = cgraph->nodes[i]; - const int n_tasks = ggml_get_n_tasks(node, n_threads); + const int n_tasks = ggml_get_n_tasks(node, n_threads, 1); max_tasks = MAX(max_tasks, n_tasks); diff --git a/llama.cpp b/llama.cpp index ad7b7b7d4..38e7036a7 100644 --- a/llama.cpp +++ b/llama.cpp @@ -978,21 +978,6 @@ static std::string gguf_kv_to_str(const struct gguf_context * ctx_gguf, int i) { } } -// -// ggml helpers -// - -static void ggml_graph_compute_helper(std::vector & buf, ggml_cgraph * graph, int n_threads) { - struct ggml_cplan plan = ggml_graph_plan(graph, n_threads); - - if (plan.work_size > 0) { - buf.resize(plan.work_size); - plan.work_data = buf.data(); - } - - ggml_graph_compute(graph, &plan); -} - // // llama helpers // @@ -1728,6 +1713,7 @@ struct llama_hparams { struct llama_cparams { uint32_t n_ctx; // context size used during inference uint32_t n_batch; + uint32_t n_ubatch; uint32_t n_threads; // number of threads to use for generation uint32_t n_threads_batch; // number of threads to use for batch processing @@ -2024,8 +2010,7 @@ struct llama_context { ggml_vk_free_cpu_assist(); #endif - ggml_backend_buffer_free(buf_input); - ggml_free(ctx_input); + ggml_backend_buffer_free(buf_output); } llama_cparams cparams; @@ -2051,12 +2036,20 @@ struct llama_context { int64_t t_p_eval_us = 0; int64_t t_eval_us = 0; + int64_t t_compute_start_us = 0; + int64_t n_queued_tokens = 0; + int32_t n_sample = 0; // number of tokens sampled int32_t n_p_eval = 0; // number of tokens in eval calls for the prompt (with batch size > 1) int32_t n_eval = 0; // number of eval calls - // logits output (2-dimensional array: [n_tokens][n_vocab]) - std::vector logits; + // host buffer for the model output (logits and embeddings) + ggml_backend_buffer_t buf_output = nullptr; + + // decode output (2-dimensional array: [n_tokens][n_vocab]) + size_t logits_size = 0; + float * logits = nullptr; + #ifndef NDEBUG // guard against access to unset logits std::vector logits_valid; @@ -2065,7 +2058,8 @@ struct llama_context { // embeddings output (2-dimensional array: [n_tokens][n_embd]) // populated only when pooling_type == LLAMA_POOLING_TYPE_NONE - std::vector embd; + size_t embd_size = 0; + float * embd = nullptr; // sequence embeddings output (map of [n_embd] vectors) // populated only when pooling_type != LLAMA_POOLING_TYPE_NONE @@ -2079,8 +2073,6 @@ struct llama_context { void * abort_callback_data = nullptr; // input tensors - ggml_backend_buffer_t buf_input = nullptr; - ggml_context * ctx_input = nullptr; struct ggml_tensor * inp_tokens; // I32 [n_batch] struct ggml_tensor * inp_embd; // F32 [n_embd, n_batch] struct ggml_tensor * inp_pos; // I32 [n_batch] @@ -2090,7 +2082,7 @@ struct llama_context { struct ggml_tensor * inp_mean; // F32 [n_batch, n_batch] struct ggml_tensor * inp_cls; // I32 [n_batch] struct ggml_tensor * inp_s_copy; // I32 [kv_size] - struct ggml_tensor * inp_s_mask; // F32 [kv_size] + struct ggml_tensor * inp_s_mask; // F32 [1, kv_size] struct ggml_tensor * inp_s_seq; // I32 [kv_size, n_batch] #ifdef GGML_USE_MPI @@ -4005,6 +3997,7 @@ static bool llm_load_tensors( // there is very little benefit to offloading the input layer, so always keep it on the CPU model.buft_input = llama_default_buffer_type_cpu(true); + //model.buft_input = llama_default_buffer_type_offload(main_gpu); model.buft_layer.resize(n_layer); @@ -5094,29 +5087,32 @@ enum llm_norm_type { static struct ggml_tensor * llm_build_inp_embd( struct ggml_context * ctx, + struct llama_context & lctx, const llama_hparams & hparams, const llama_batch & batch, struct ggml_tensor * tok_embd, - struct ggml_tensor * inp_tokens, - struct ggml_tensor * inp_embd, const llm_build_cb & cb) { const int64_t n_embd = hparams.n_embd; struct ggml_tensor * inpL; if (batch.token) { - struct ggml_tensor * inp_tokens_v = ggml_view_1d(ctx, inp_tokens, batch.n_tokens, 0); - cb(inp_tokens, "inp_tokens", -1); + lctx.inp_tokens = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, batch.n_tokens); + cb(lctx.inp_tokens, "inp_tokens", -1); + ggml_set_input(lctx.inp_tokens); - inpL = ggml_get_rows(ctx, tok_embd, inp_tokens_v); + inpL = ggml_get_rows(ctx, tok_embd, lctx.inp_tokens); } else { #ifdef GGML_USE_MPI GGML_ASSERT(false && "not implemented"); #endif - - inpL = ggml_view_2d(ctx, inp_embd, n_embd, batch.n_tokens, inp_embd->nb[1], 0); + lctx.inp_embd = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, n_embd, batch.n_tokens); + inpL = lctx.inp_embd; + ggml_set_input(lctx.inp_embd); } + cb(inpL, "inp_embd", -1); + return inpL; } @@ -5420,7 +5416,7 @@ static struct ggml_tensor * llm_build_kv( struct llm_build_context { const llama_model & model; - const llama_context & lctx; + llama_context & lctx; const llama_hparams & hparams; const llama_cparams & cparams; const llama_batch & batch; @@ -5513,6 +5509,18 @@ struct llm_build_context { }; ctx0 = ggml_init(params); + + lctx.inp_tokens = nullptr; + lctx.inp_embd = nullptr; + lctx.inp_pos = nullptr; + lctx.inp_KQ_mask = nullptr; + lctx.inp_KQ_pos = nullptr; + lctx.inp_K_shift = nullptr; + lctx.inp_mean = nullptr; + lctx.inp_cls = nullptr; + lctx.inp_s_copy = nullptr; + lctx.inp_s_mask = nullptr; + lctx.inp_s_seq = nullptr; } void free() { @@ -5527,6 +5535,10 @@ struct llm_build_context { GGML_ASSERT(kv_self.size == n_ctx); + lctx.inp_K_shift = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_ctx); + cb(lctx.inp_K_shift, "K_shift", -1); + ggml_set_input(lctx.inp_K_shift); + for (int il = 0; il < n_layer; ++il) { struct ggml_tensor * tmp = // we rotate only the first n_rot dimensions @@ -5550,12 +5562,14 @@ struct llm_build_context { GGML_ASSERT(kv_self.recurrent); + struct ggml_tensor * state_copy = build_inp_s_copy(); + for (int il = 0; il < n_layer; ++il) { struct ggml_tensor * conv_states = ggml_reshape_2d(ctx0, kv_self.k_l[il], hparams.n_embd_k_s(), kv_self.size); struct ggml_tensor * ssm_states = ggml_reshape_2d(ctx0, kv_self.v_l[il], hparams.n_embd_v_s(), kv_self.size); - conv_states = ggml_get_rows(ctx0, conv_states, lctx.inp_s_copy); - ssm_states = ggml_get_rows(ctx0, ssm_states, lctx.inp_s_copy); + conv_states = ggml_get_rows(ctx0, conv_states, state_copy); + ssm_states = ggml_get_rows(ctx0, ssm_states, state_copy); // TODO: name the intermediate tensors with cb() @@ -5615,6 +5629,66 @@ struct llm_build_context { return gf; } + struct ggml_tensor * build_inp_pos() { + lctx.inp_pos = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_tokens); + cb(lctx.inp_pos, "inp_pos", -1); + ggml_set_input(lctx.inp_pos); + return lctx.inp_pos; + } + + struct ggml_tensor * build_inp_KQ_mask(bool causal = true) { + if (causal) { + lctx.inp_KQ_mask = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_kv, n_tokens); + } else { + lctx.inp_KQ_mask = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_tokens, n_tokens); + } + cb(lctx.inp_KQ_mask, "KQ_mask", -1); + ggml_set_input(lctx.inp_KQ_mask); + return lctx.inp_KQ_mask; + } + + struct ggml_tensor * build_inp_KQ_pos() { + lctx.inp_KQ_pos = ggml_new_tensor_1d(ctx0, GGML_TYPE_F32, n_kv); + cb(lctx.inp_KQ_pos, "KQ_pos", -1); + ggml_set_input(lctx.inp_KQ_pos); + return lctx.inp_KQ_pos; + } + + struct ggml_tensor * build_inp_mean() { + lctx.inp_mean = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_tokens, n_tokens); + cb(lctx.inp_mean, "inp_mean", -1); + ggml_set_input(lctx.inp_mean); + return lctx.inp_mean; + } + + struct ggml_tensor * build_inp_cls() { + lctx.inp_cls = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_tokens); + cb(lctx.inp_cls, "inp_cls", -1); + ggml_set_input(lctx.inp_cls); + return lctx.inp_cls; + } + + struct ggml_tensor * build_inp_s_copy() { + lctx.inp_s_copy = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, kv_self.size); + cb(lctx.inp_s_copy, "inp_s_copy", -1); + ggml_set_input(lctx.inp_s_copy); + return lctx.inp_s_copy; + } + + struct ggml_tensor * build_inp_s_mask() { + lctx.inp_s_mask = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, 1, n_kv); + cb(lctx.inp_s_mask, "inp_s_mask", -1); + ggml_set_input(lctx.inp_s_mask); + return lctx.inp_s_mask; + } + + struct ggml_tensor * build_inp_s_seq() { + lctx.inp_s_seq = ggml_new_tensor_2d(ctx0, GGML_TYPE_I32, n_kv, n_tokens); + cb(lctx.inp_s_seq, "inp_s_seq", -1); + ggml_set_input(lctx.inp_s_seq); + return lctx.inp_s_seq; + } + struct ggml_cgraph * build_llama() { struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, LLAMA_MAX_NODES, false); @@ -5625,16 +5699,13 @@ struct llm_build_context { struct ggml_tensor * cur; struct ggml_tensor * inpL; - inpL = llm_build_inp_embd(ctx0, hparams, batch, model.tok_embd, lctx.inp_tokens, lctx.inp_embd, cb); - cb(inpL, "inp_embd", -1); + inpL = llm_build_inp_embd(ctx0, lctx, hparams, batch, model.tok_embd, cb); // inp_pos - contains the positions - struct ggml_tensor * inp_pos = ggml_view_1d(ctx0, lctx.inp_pos, n_tokens, 0); - cb(inp_pos, "inp_pos", -1); + struct ggml_tensor * inp_pos = build_inp_pos(); // KQ_mask (mask for 1 head, it will be broadcasted to all heads) - struct ggml_tensor * KQ_mask = ggml_view_2d(ctx0, lctx.inp_KQ_mask, n_kv, n_tokens, n_kv*ggml_type_size(lctx.inp_KQ_mask->type), 0); - cb(KQ_mask, "KQ_mask", -1); + struct ggml_tensor * KQ_mask = build_inp_KQ_mask(); for (int il = 0; il < n_layer; ++il) { struct ggml_tensor * inpSA = inpL; @@ -5686,7 +5757,6 @@ struct llm_build_context { cur = llm_build_kv(ctx0, model, hparams, kv_self, gf, model.layers[il].wo, model.layers[il].bo, Kcur, Vcur, Qcur, KQ_mask, nullptr, n_ctx, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il); - cb(cur, "kqv_out", il); } struct ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA); @@ -5804,20 +5874,16 @@ struct llm_build_context { struct ggml_tensor * cur; struct ggml_tensor * inpL; - inpL = llm_build_inp_embd(ctx0, hparams, batch, model.tok_embd, lctx.inp_tokens, lctx.inp_embd, cb); - cb(inpL, "inp_embd", -1); + inpL = llm_build_inp_embd(ctx0, lctx, hparams, batch, model.tok_embd, cb); // inp_pos - contains the positions - struct ggml_tensor * inp_pos = ggml_view_1d(ctx0, lctx.inp_pos, n_tokens, 0); - cb(inp_pos, "inp_pos", -1); + struct ggml_tensor * inp_pos = build_inp_pos(); // KQ_mask (mask for 1 head, it will be broadcasted to all heads) - struct ggml_tensor * KQ_mask = ggml_view_2d(ctx0, lctx.inp_KQ_mask, n_kv, n_tokens, n_kv*ggml_type_size(lctx.inp_KQ_mask->type), 0); - cb(KQ_mask, "KQ_mask", -1); + struct ggml_tensor * KQ_mask = build_inp_KQ_mask(); // positions of the tokens in the KV cache - struct ggml_tensor * KQ_pos = ggml_view_1d(ctx0, lctx.inp_KQ_pos, n_kv, 0); - cb(KQ_pos, "KQ_pos", -1); + struct ggml_tensor * KQ_pos = build_inp_KQ_pos(); for (int il = 0; il < n_layer; ++il) { struct ggml_tensor * inpSA = inpL; @@ -5865,7 +5931,6 @@ struct llm_build_context { cur = llm_build_kv(ctx0, model, hparams, kv_self, gf, model.layers[il].wo, NULL, Kcur, Vcur, Qcur, KQ_mask, KQ_pos, n_ctx, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il); - cb(cur, "kqv_out", il); } struct ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA); @@ -5921,16 +5986,13 @@ struct llm_build_context { struct ggml_tensor * cur; struct ggml_tensor * inpL; - inpL = llm_build_inp_embd(ctx0, hparams, batch, model.tok_embd, lctx.inp_tokens, lctx.inp_embd, cb); - cb(inpL, "inp_embd", -1); + inpL = llm_build_inp_embd(ctx0, lctx, hparams, batch, model.tok_embd, cb); // inp_pos - contains the positions - struct ggml_tensor * inp_pos = ggml_view_1d(ctx0, lctx.inp_pos, n_tokens, 0); - cb(inp_pos, "inp_pos", -1); + struct ggml_tensor * inp_pos = build_inp_pos(); // KQ_mask (mask for 1 head, it will be broadcasted to all heads) - struct ggml_tensor * KQ_mask = ggml_view_2d(ctx0, lctx.inp_KQ_mask, n_kv, n_tokens, n_kv*ggml_type_size(lctx.inp_KQ_mask->type), 0); - cb(KQ_mask, "KQ_mask", -1); + struct ggml_tensor * KQ_mask = build_inp_KQ_mask(); for (int il = 0; il < n_layer; ++il) { struct ggml_tensor * attn_norm; @@ -5984,7 +6046,6 @@ struct llm_build_context { cur = llm_build_kv(ctx0, model, hparams, kv_self, gf, model.layers[il].wo, NULL, Kcur, Vcur, Qcur, KQ_mask, nullptr, n_ctx, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il); - cb(cur, "kqv_out", il); } struct ggml_tensor * ffn_inp = cur; @@ -6035,21 +6096,17 @@ struct llm_build_context { GGML_ASSERT(n_embd_head == hparams.n_embd_head_k); struct ggml_tensor * cur; - struct ggml_tensor * pos; struct ggml_tensor * inpL; - inpL = llm_build_inp_embd(ctx0, hparams, batch, model.tok_embd, lctx.inp_tokens, lctx.inp_embd, cb); - cb(inpL, "inp_embd", -1); + inpL = llm_build_inp_embd(ctx0, lctx, hparams, batch, model.tok_embd, cb); // inp_pos - contains the positions - struct ggml_tensor * inp_pos = ggml_view_1d(ctx0, lctx.inp_pos, n_tokens, 0); - cb(inp_pos, "inp_pos", -1); + struct ggml_tensor * inp_pos = build_inp_pos(); // KQ_mask (mask for 1 head, it will be broadcasted to all heads) - struct ggml_tensor * KQ_mask = ggml_view_2d(ctx0, lctx.inp_KQ_mask, n_kv, n_tokens, n_kv*ggml_type_size(lctx.inp_KQ_mask->type), 0); - cb(KQ_mask, "KQ_mask", -1); + struct ggml_tensor * KQ_mask = build_inp_KQ_mask(); - pos = ggml_get_rows(ctx0, model.pos_embd, inp_pos); + struct ggml_tensor * pos = ggml_get_rows(ctx0, model.pos_embd, inp_pos); cb(pos, "pos_embd", -1); inpL = ggml_add(ctx0, inpL, pos); @@ -6083,7 +6140,6 @@ struct llm_build_context { cur = llm_build_kv(ctx0, model, hparams, kv_self, gf, model.layers[il].wo, model.layers[il].bo, Kcur, Vcur, Qcur, KQ_mask, nullptr, n_ctx, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il); - cb(cur, "kqv_out", il); } // add the input @@ -6135,16 +6191,13 @@ struct llm_build_context { struct ggml_tensor * cur; struct ggml_tensor * inpL; - inpL = llm_build_inp_embd(ctx0, hparams, batch, model.tok_embd, lctx.inp_tokens, lctx.inp_embd, cb); - cb(inpL, "inp_embd", -1); + inpL = llm_build_inp_embd(ctx0, lctx, hparams, batch, model.tok_embd, cb); // inp_pos - contains the positions - struct ggml_tensor * inp_pos = ggml_view_1d(ctx0, lctx.inp_pos, n_tokens, 0); - cb(inp_pos, "inp_pos", -1); + struct ggml_tensor * inp_pos = build_inp_pos(); // KQ_mask (mask for 1 head, it will be broadcasted to all heads) - struct ggml_tensor * KQ_mask = ggml_view_2d(ctx0, lctx.inp_KQ_mask, n_kv, n_tokens, n_kv*ggml_type_size(lctx.inp_KQ_mask->type), 0); - cb(KQ_mask, "KQ_mask", -1); + struct ggml_tensor * KQ_mask = build_inp_KQ_mask(); for (int il = 0; il < n_layer; ++il) { struct ggml_tensor * residual = inpL; @@ -6284,7 +6337,6 @@ struct llm_build_context { cur = llm_build_kv(ctx0, model, hparams, kv_self, gf, model.layers[il].wo, model.layers[il].bo, Kcur, Vcur, Q, KQ_mask, nullptr, n_ctx, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il); - cb(cur, "kqv_out", il); } struct ggml_tensor * ffn_inp = ggml_add(ctx0, residual, cur); @@ -6338,16 +6390,13 @@ struct llm_build_context { struct ggml_tensor * cur; struct ggml_tensor * inpL; - inpL = llm_build_inp_embd(ctx0, hparams, batch, model.tok_embd, lctx.inp_tokens, lctx.inp_embd, cb); - cb(inpL, "inp_embd", -1); + inpL = llm_build_inp_embd(ctx0, lctx, hparams, batch, model.tok_embd, cb); // KQ_mask (mask for 1 head, it will be broadcasted to all heads) - struct ggml_tensor * KQ_mask = ggml_view_2d(ctx0, lctx.inp_KQ_mask, n_kv, n_tokens, n_kv*ggml_type_size(lctx.inp_KQ_mask->type), 0); - cb(KQ_mask, "KQ_mask", -1); + struct ggml_tensor * KQ_mask = build_inp_KQ_mask(); // positions of the tokens in the KV cache - struct ggml_tensor * KQ_pos = ggml_view_1d(ctx0, lctx.inp_KQ_pos, n_kv, 0); - cb(KQ_pos, "KQ_pos", -1); + struct ggml_tensor * KQ_pos = build_inp_KQ_pos(); for (int il = 0; il < n_layer; ++il) { struct ggml_tensor * inpSA = inpL; @@ -6377,7 +6426,6 @@ struct llm_build_context { cur = llm_build_kv(ctx0, model, hparams, kv_self, gf, model.layers[il].wo, NULL, Kcur, Vcur, Qcur, KQ_mask, KQ_pos, n_ctx, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il); - cb(cur, "kqv_out", il); } struct ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA); @@ -6433,15 +6481,12 @@ struct llm_build_context { struct ggml_tensor * cur; struct ggml_tensor * inpL; - // get input vectors with right size - const size_t stride1 = n_tokens * ggml_type_size(lctx.inp_tokens->type); - - struct ggml_tensor * inp_pos = ggml_view_1d(ctx0, lctx.inp_pos, n_tokens, 0); - struct ggml_tensor * inp_mean = ggml_view_2d(ctx0, lctx.inp_mean, n_tokens, n_tokens, stride1, 0); - struct ggml_tensor * inp_cls = ggml_view_1d(ctx0, lctx.inp_cls, n_tokens, 0); + struct ggml_tensor * inp_pos = build_inp_pos(); + struct ggml_tensor * inp_mean = build_inp_mean(); + struct ggml_tensor * inp_cls = build_inp_cls(); // construct input embeddings (token, type, position) - inpL = llm_build_inp_embd(ctx0, hparams, batch, model.tok_embd, lctx.inp_tokens, lctx.inp_embd, cb); + inpL = llm_build_inp_embd(ctx0, lctx, hparams, batch, model.tok_embd, cb); // token types are hardcoded to zero ("Sentence A") struct ggml_tensor * type_row0 = ggml_view_1d(ctx0, model.type_embd, n_embd, 0); @@ -6456,8 +6501,7 @@ struct llm_build_context { cb(inpL, "inp_norm", -1); // KQ_mask (mask for 1 head, it will be broadcasted to all heads) - struct ggml_tensor * KQ_mask = ggml_cont(ctx0, ggml_view_2d(ctx0, lctx.inp_KQ_mask, n_tokens, n_tokens, n_tokens*ggml_type_size(lctx.inp_KQ_mask->type), 0)); - cb(KQ_mask, "KQ_mask", -1); // [n_tokens, n_tokens] + struct ggml_tensor * KQ_mask = build_inp_KQ_mask(false); // iterate layers for (int il = 0; il < n_layer; ++il) { @@ -6619,16 +6663,13 @@ struct llm_build_context { struct ggml_tensor * cur; struct ggml_tensor * inpL; - inpL = llm_build_inp_embd(ctx0, hparams, batch, model.tok_embd, lctx.inp_tokens, lctx.inp_embd, cb); - cb(inpL, "inp_embd", -1); + inpL = llm_build_inp_embd(ctx0, lctx, hparams, batch, model.tok_embd, cb); // KQ_mask (mask for 1 head, it will be broadcasted to all heads) - struct ggml_tensor * KQ_mask = ggml_view_2d(ctx0, lctx.inp_KQ_mask, n_kv, n_tokens, n_kv*ggml_type_size(lctx.inp_KQ_mask->type), 0); - cb(KQ_mask, "KQ_mask", -1); + struct ggml_tensor * KQ_mask = build_inp_KQ_mask(); // positions of the tokens in the KV cache - struct ggml_tensor * KQ_pos = ggml_view_1d(ctx0, lctx.inp_KQ_pos, n_kv, 0); - cb(KQ_pos, "KQ_pos", -1); + struct ggml_tensor * KQ_pos = build_inp_KQ_pos(); inpL = llm_build_norm(ctx0, inpL, hparams, model.tok_norm, @@ -6664,7 +6705,6 @@ struct llm_build_context { cur = llm_build_kv(ctx0, model, hparams, kv_self, gf, model.layers[il].wo, model.layers[il].bo, Kcur, Vcur, Qcur, KQ_mask, KQ_pos, n_ctx, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il); - cb(cur, "kqv_out", il); } // Add the input @@ -6716,16 +6756,13 @@ struct llm_build_context { struct ggml_tensor * cur; struct ggml_tensor * inpL; - inpL = llm_build_inp_embd(ctx0, hparams, batch, model.tok_embd, lctx.inp_tokens, lctx.inp_embd, cb); - cb(inpL, "inp_embd", -1); + inpL = llm_build_inp_embd(ctx0, lctx, hparams, batch, model.tok_embd, cb); // KQ_mask (mask for 1 head, it will be broadcasted to all heads) - struct ggml_tensor * KQ_mask = ggml_view_2d(ctx0, lctx.inp_KQ_mask, n_kv, n_tokens, n_kv*ggml_type_size(lctx.inp_KQ_mask->type), 0); - cb(KQ_mask, "KQ_mask", -1); + struct ggml_tensor * KQ_mask = build_inp_KQ_mask(); // positions of the tokens in the KV cache - struct ggml_tensor * KQ_pos = ggml_view_1d(ctx0, lctx.inp_KQ_pos, n_kv, 0); - cb(KQ_pos, "KQ_pos", -1); + struct ggml_tensor * KQ_pos = build_inp_KQ_pos(); for (int il = 0; il < n_layer; ++il) { struct ggml_tensor * attn_norm; @@ -6766,7 +6803,6 @@ struct llm_build_context { cur = llm_build_kv(ctx0, model, hparams, kv_self, gf, model.layers[il].wo, model.layers[il].bo, Kcur, Vcur, Qcur, KQ_mask, KQ_pos, n_ctx, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il); - cb(cur, "kqv_out", il); } // Add the input @@ -6821,16 +6857,13 @@ struct llm_build_context { struct ggml_tensor * cur; struct ggml_tensor * inpL; - inpL = llm_build_inp_embd(ctx0, hparams, batch, model.tok_embd, lctx.inp_tokens, lctx.inp_embd, cb); - cb(inpL, "inp_embd", -1); + inpL = llm_build_inp_embd(ctx0, lctx, hparams, batch, model.tok_embd, cb); // inp_pos - contains the positions - struct ggml_tensor * inp_pos = ggml_view_1d(ctx0, lctx.inp_pos, n_tokens, 0); - cb(inp_pos, "inp_pos", -1); + struct ggml_tensor * inp_pos = build_inp_pos(); // KQ_mask (mask for 1 head, it will be broadcasted to all heads) - struct ggml_tensor * KQ_mask = ggml_view_2d(ctx0, lctx.inp_KQ_mask, n_kv, n_tokens, n_kv*ggml_type_size(lctx.inp_KQ_mask->type), 0); - cb(KQ_mask, "KQ_mask", -1); + struct ggml_tensor * KQ_mask = build_inp_KQ_mask(); for (int il = 0; il < n_layer; ++il) { struct ggml_tensor * inpSA = inpL; @@ -6883,7 +6916,6 @@ struct llm_build_context { cur = llm_build_kv(ctx0, model, hparams, kv_self, gf, model.layers[il].wo, NULL, Kcur, Vcur, Qcur, KQ_mask, nullptr, n_ctx, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il); - cb(cur, "kqv_out", il); } struct ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA); @@ -6939,16 +6971,13 @@ struct llm_build_context { struct ggml_tensor * cur; struct ggml_tensor * inpL; - inpL = llm_build_inp_embd(ctx0, hparams, batch, model.tok_embd, lctx.inp_tokens, lctx.inp_embd, cb); - cb(inpL, "inp_embd", -1); + inpL = llm_build_inp_embd(ctx0, lctx, hparams, batch, model.tok_embd, cb); // inp_pos - contains the positions - struct ggml_tensor * inp_pos = ggml_view_1d(ctx0, lctx.inp_pos, n_tokens, 0); - cb(inp_pos, "inp_pos", -1); + struct ggml_tensor * inp_pos = build_inp_pos(); // KQ_mask (mask for 1 head, it will be broadcasted to all heads) - struct ggml_tensor * KQ_mask = ggml_view_2d(ctx0, lctx.inp_KQ_mask, n_kv, n_tokens, n_kv*ggml_type_size(lctx.inp_KQ_mask->type), 0); - cb(KQ_mask, "KQ_mask", -1); + struct ggml_tensor * KQ_mask = build_inp_KQ_mask(); for (int il = 0; il < n_layer; ++il) { struct ggml_tensor * inpSA = inpL; @@ -6993,7 +7022,6 @@ struct llm_build_context { cur = llm_build_kv(ctx0, model, hparams, kv_self, gf, model.layers[il].wo, NULL, Kcur, Vcur, Qcur, KQ_mask, nullptr, n_ctx, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il); - cb(cur, "kqv_out", il); } struct ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA); @@ -7048,16 +7076,13 @@ struct llm_build_context { struct ggml_tensor * cur; struct ggml_tensor * inpL; - inpL = llm_build_inp_embd(ctx0, hparams, batch, model.tok_embd, lctx.inp_tokens, lctx.inp_embd, cb); - cb(inpL, "inp_embd", -1); + inpL = llm_build_inp_embd(ctx0, lctx, hparams, batch, model.tok_embd, cb); // inp_pos - contains the positions - struct ggml_tensor * inp_pos = ggml_view_1d(ctx0, lctx.inp_pos, n_tokens, 0); - cb(inp_pos, "inp_pos", -1); + struct ggml_tensor * inp_pos = build_inp_pos(); // KQ_mask (mask for 1 head, it will be broadcasted to all heads) - struct ggml_tensor * KQ_mask = ggml_view_2d(ctx0, lctx.inp_KQ_mask, n_kv, n_tokens, n_kv*ggml_type_size(lctx.inp_KQ_mask->type), 0); - cb(KQ_mask, "KQ_mask", -1); + struct ggml_tensor * KQ_mask = build_inp_KQ_mask(); for (int il = 0; il < n_layer; ++il) { struct ggml_tensor * inpSA = inpL; @@ -7109,7 +7134,6 @@ struct llm_build_context { cur = llm_build_kv(ctx0, model, hparams, kv_self, gf, model.layers[il].wo, model.layers[il].bo, Kcur, Vcur, Qcur, KQ_mask, nullptr, n_ctx, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il); - cb(cur, "kqv_out", il); } struct ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA); @@ -7164,16 +7188,13 @@ struct llm_build_context { struct ggml_tensor * ffn_output; struct ggml_tensor * inpL; - inpL = llm_build_inp_embd(ctx0, hparams, batch, model.tok_embd, lctx.inp_tokens, lctx.inp_embd, cb); - cb(inpL, "inp_embd", -1); + inpL = llm_build_inp_embd(ctx0, lctx, hparams, batch, model.tok_embd, cb); // inp_pos - contains the positions - struct ggml_tensor * inp_pos = ggml_view_1d(ctx0, lctx.inp_pos, n_tokens, 0); - cb(inp_pos, "inp_pos", -1); + struct ggml_tensor * inp_pos = build_inp_pos(); // KQ_mask (mask for 1 head, it will be broadcasted to all heads) - struct ggml_tensor * KQ_mask = ggml_view_2d(ctx0, lctx.inp_KQ_mask, n_kv, n_tokens, n_kv*ggml_type_size(lctx.inp_KQ_mask->type), 0); - cb(KQ_mask, "KQ_mask", -1); + struct ggml_tensor * KQ_mask = build_inp_KQ_mask(); for (int il = 0; il < n_layer; ++il) { attn_norm_output = llm_build_norm(ctx0, inpL, hparams, @@ -7231,7 +7252,6 @@ struct llm_build_context { cur = llm_build_kv(ctx0, model, hparams, kv_self, gf, model.layers[il].wo, model.layers[il].bo, Kcur, Vcur, Qcur, KQ_mask, nullptr, n_ctx, n_tokens, kv_head, n_kv, 1.0f, cb, il); - cb(cur, "kqv_out", il); } // FF @@ -7281,16 +7301,13 @@ struct llm_build_context { struct ggml_tensor * cur; struct ggml_tensor * inpL; - inpL = llm_build_inp_embd(ctx0, hparams, batch, model.tok_embd, lctx.inp_tokens, lctx.inp_embd, cb); - cb(inpL, "inp_embd", -1); + inpL = llm_build_inp_embd(ctx0, lctx, hparams, batch, model.tok_embd, cb); // inp_pos - contains the positions - struct ggml_tensor * inp_pos = ggml_view_1d(ctx0, lctx.inp_pos, n_tokens, 0); - cb(inp_pos, "inp_pos", -1); + struct ggml_tensor * inp_pos = build_inp_pos(); // KQ_mask (mask for 1 head, it will be broadcasted to all heads) - struct ggml_tensor * KQ_mask = ggml_view_2d(ctx0, lctx.inp_KQ_mask, n_kv, n_tokens, n_kv*ggml_type_size(lctx.inp_KQ_mask->type), 0); - cb(KQ_mask, "KQ_mask", -1); + struct ggml_tensor * KQ_mask = build_inp_KQ_mask(); for (int il = 0; il < n_layer; ++il) { @@ -7329,7 +7346,6 @@ struct llm_build_context { cur = llm_build_kv(ctx0, model, hparams, kv_self, gf, model.layers[il].wo, NULL, Kcur, Vcur, Qcur, KQ_mask, nullptr, n_ctx, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il); - cb(cur, "kqv_out", il); } struct ggml_tensor * sa_out = cur; @@ -7383,16 +7399,13 @@ struct llm_build_context { struct ggml_tensor * pos; struct ggml_tensor * inpL; - inpL = llm_build_inp_embd(ctx0, hparams, batch, model.tok_embd, lctx.inp_tokens, lctx.inp_embd, cb); - cb(inpL, "inp_embd", -1); + inpL = llm_build_inp_embd(ctx0, lctx, hparams, batch, model.tok_embd, cb); // inp_pos - contains the positions - struct ggml_tensor * inp_pos = ggml_view_1d(ctx0, lctx.inp_pos, n_tokens, 0); - cb(inp_pos, "inp_pos", -1); + struct ggml_tensor * inp_pos = build_inp_pos(); // KQ_mask (mask for 1 head, it will be broadcasted to all heads) - struct ggml_tensor * KQ_mask = ggml_view_2d(ctx0, lctx.inp_KQ_mask, n_kv, n_tokens, n_kv*ggml_type_size(lctx.inp_KQ_mask->type), 0); - cb(KQ_mask, "KQ_mask", -1); + struct ggml_tensor * KQ_mask = build_inp_KQ_mask(); pos = ggml_get_rows(ctx0, model.pos_embd, inp_pos); cb(pos, "pos_embd", -1); @@ -7428,7 +7441,6 @@ struct llm_build_context { cur = llm_build_kv(ctx0, model, hparams, kv_self, gf, model.layers[il].wo, model.layers[il].bo, Kcur, Vcur, Qcur, KQ_mask, nullptr, n_ctx, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il); - cb(cur, "kqv_out", il); } // add the input @@ -7481,16 +7493,13 @@ struct llm_build_context { struct ggml_tensor * cur; struct ggml_tensor * inpL; - inpL = llm_build_inp_embd(ctx0, hparams, batch, model.tok_embd, lctx.inp_tokens, lctx.inp_embd, cb); - cb(inpL, "inp_embd", -1); + inpL = llm_build_inp_embd(ctx0, lctx, hparams, batch, model.tok_embd, cb); // inp_pos - contains the positions - struct ggml_tensor * inp_pos = ggml_view_1d(ctx0, lctx.inp_pos, n_tokens, 0); - cb(inp_pos, "inp_pos", -1); + struct ggml_tensor * inp_pos = build_inp_pos(); // KQ_mask (mask for 1 head, it will be broadcasted to all heads) - struct ggml_tensor * KQ_mask = ggml_view_2d(ctx0, lctx.inp_KQ_mask, n_kv, n_tokens, n_kv*ggml_type_size(lctx.inp_KQ_mask->type), 0); - cb(KQ_mask, "KQ_mask", -1); + struct ggml_tensor * KQ_mask = build_inp_KQ_mask(); for (int il = 0; il < n_layer; ++il) { cur = llm_build_norm(ctx0, inpL, hparams, @@ -7532,7 +7541,6 @@ struct llm_build_context { cur = llm_build_kv(ctx0, model, hparams, kv_self, gf, model.layers[il].wo, model.layers[il].bo, Kcur, Vcur, Qcur, KQ_mask, nullptr, n_ctx, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il); - cb(cur, "kqv_out", il); } // add the input @@ -7584,16 +7592,13 @@ struct llm_build_context { struct ggml_tensor * cur; struct ggml_tensor * inpL; - inpL = llm_build_inp_embd(ctx0, hparams, batch, model.tok_embd, lctx.inp_tokens, lctx.inp_embd, cb); - cb(inpL, "inp_embd", -1); + inpL = llm_build_inp_embd(ctx0, lctx, hparams, batch, model.tok_embd, cb); // inp_pos - contains the positions - struct ggml_tensor * inp_pos = ggml_view_1d(ctx0, lctx.inp_pos, n_tokens, 0); - cb(inp_pos, "inp_pos", -1); + struct ggml_tensor * inp_pos = build_inp_pos(); // KQ_mask (mask for 1 head, it will be broadcasted to all heads) - struct ggml_tensor * KQ_mask = ggml_view_2d(ctx0, lctx.inp_KQ_mask, n_kv, n_tokens, n_kv*ggml_type_size(lctx.inp_KQ_mask->type), 0); - cb(KQ_mask, "KQ_mask", -1); + struct ggml_tensor * KQ_mask = build_inp_KQ_mask(); for (int il = 0; il < n_layer; ++il) { struct ggml_tensor * inpSA = inpL; @@ -7645,7 +7650,6 @@ struct llm_build_context { cur = llm_build_kv(ctx0, model, hparams, kv_self, gf, model.layers[il].wo, NULL, Kcur, Vcur, Qcur, KQ_mask, nullptr, n_ctx, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il); - cb(cur, "kqv_out", il); } struct ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA); @@ -7698,16 +7702,13 @@ struct llm_build_context { struct ggml_tensor * cur; struct ggml_tensor * inpL; - inpL = llm_build_inp_embd(ctx0, hparams, batch, model.tok_embd, lctx.inp_tokens, lctx.inp_embd, cb); - cb(inpL, "inp_embd", -1); + inpL = llm_build_inp_embd(ctx0, lctx, hparams, batch, model.tok_embd, cb); // inp_pos - contains the positions - struct ggml_tensor * inp_pos = ggml_view_1d(ctx0, lctx.inp_pos, n_tokens, 0); - cb(inp_pos, "inp_pos", -1); + struct ggml_tensor * inp_pos = build_inp_pos(); // KQ_mask (mask for 1 head, it will be broadcasted to all heads) - struct ggml_tensor * KQ_mask = ggml_view_2d(ctx0, lctx.inp_KQ_mask, n_kv, n_tokens, n_kv*ggml_type_size(lctx.inp_KQ_mask->type), 0); - cb(KQ_mask, "KQ_mask", -1); + struct ggml_tensor * KQ_mask = build_inp_KQ_mask(); for (int il = 0; il < n_layer; ++il) { struct ggml_tensor * inpSA = inpL; @@ -7759,7 +7760,6 @@ struct llm_build_context { cur = llm_build_kv(ctx0, model, hparams, kv_self, gf, model.layers[il].wo, model.layers[il].bo, Kcur, Vcur, Qcur, KQ_mask, nullptr, n_ctx, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il); - cb(cur, "kqv_out", il); } struct ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA); @@ -7821,20 +7821,17 @@ struct llm_build_context { struct ggml_tensor * cur; struct ggml_tensor * inpL; - inpL = llm_build_inp_embd(ctx0, hparams, batch, model.tok_embd, lctx.inp_tokens, lctx.inp_embd, cb); - cb(inpL, "inp_embd", -1); + inpL = llm_build_inp_embd(ctx0, lctx, hparams, batch, model.tok_embd, cb); // scale the input embeddings inpL = ggml_scale(ctx0, inpL, scale_embd); cb(inpL, "inp_scaled", -1); // inp_pos - contains the positions - struct ggml_tensor * inp_pos = ggml_view_1d(ctx0, lctx.inp_pos, n_tokens, 0); - cb(inp_pos, "inp_pos", -1); + struct ggml_tensor * inp_pos = build_inp_pos(); // KQ_mask (mask for 1 head, it will be broadcasted to all heads) - struct ggml_tensor * KQ_mask = ggml_view_2d(ctx0, lctx.inp_KQ_mask, n_kv, n_tokens, n_kv*ggml_type_size(lctx.inp_KQ_mask->type), 0); - cb(KQ_mask, "KQ_mask", -1); + struct ggml_tensor * KQ_mask = build_inp_KQ_mask(); for (int il = 0; il < n_layer; ++il) { struct ggml_tensor * inpSA = inpL; @@ -7886,7 +7883,6 @@ struct llm_build_context { cur = llm_build_kv(ctx0, model, hparams, kv_self, gf, model.layers[il].wo, model.layers[il].bo, Kcur, Vcur, Qcur, KQ_mask, nullptr, n_ctx, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il); - cb(cur, "kqv_out", il); } // scale_res - scale the hidden states for residual connection @@ -7953,22 +7949,18 @@ struct llm_build_context { struct ggml_tensor * cur; struct ggml_tensor * inpL; - inpL = llm_build_inp_embd(ctx0, hparams, batch, model.tok_embd, lctx.inp_tokens, lctx.inp_embd, cb); - cb(inpL, "inp_embd", -1); + inpL = llm_build_inp_embd(ctx0, lctx, hparams, batch, model.tok_embd, cb); inpL = ggml_scale(ctx0, inpL, sqrtf(n_embd)); cb(inpL, "inp_scaled", -1); // inp_pos - contains the positions - struct ggml_tensor * inp_pos = ggml_view_1d(ctx0, lctx.inp_pos, n_tokens, 0); - cb(inp_pos, "inp_pos", -1); + struct ggml_tensor * inp_pos = build_inp_pos(); // KQ_mask (mask for 1 head, it will be broadcasted to all heads) - struct ggml_tensor * KQ_mask = ggml_view_2d(ctx0, lctx.inp_KQ_mask, n_kv, n_tokens, n_kv*ggml_type_size(lctx.inp_KQ_mask->type), 0); - cb(KQ_mask, "KQ_mask", -1); + struct ggml_tensor * KQ_mask = build_inp_KQ_mask(); for (int il = 0; il < n_layer; ++il) { - // norm cur = llm_build_norm(ctx0, inpL, hparams, model.layers[il].attn_norm, NULL, @@ -8005,7 +7997,6 @@ struct llm_build_context { cur = llm_build_kv(ctx0, model, hparams, kv_self, gf, model.layers[il].wo, NULL, Kcur, Vcur, Qcur, KQ_mask, nullptr, n_ctx, n_tokens, kv_head, n_kv, 1.0f, cb, il); - cb(cur, "kqv_out", il); } struct ggml_tensor * sa_out = ggml_add(ctx0, cur, inpL); @@ -8060,16 +8051,13 @@ struct llm_build_context { struct ggml_tensor * cur; struct ggml_tensor * inpL; - inpL = llm_build_inp_embd(ctx0, hparams, batch, model.tok_embd, lctx.inp_tokens, lctx.inp_embd, cb); - cb(inpL, "inp_embd", -1); + inpL = llm_build_inp_embd(ctx0, lctx, hparams, batch, model.tok_embd, cb); // inp_pos - contains the positions - struct ggml_tensor * inp_pos = ggml_view_1d(ctx0, lctx.inp_pos, n_tokens, 0); - cb(inp_pos, "inp_pos", -1); + struct ggml_tensor * inp_pos = build_inp_pos(); // KQ_mask (mask for 1 head, it will be broadcasted to all heads) - struct ggml_tensor * KQ_mask = ggml_view_2d(ctx0, lctx.inp_KQ_mask, n_kv, n_tokens, n_kv*ggml_type_size(lctx.inp_KQ_mask->type), 0); - cb(KQ_mask, "KQ_mask", -1); + struct ggml_tensor * KQ_mask = build_inp_KQ_mask(); for (int il = 0; il < n_layer; ++il) { struct ggml_tensor * inpSA = inpL; @@ -8178,11 +8166,10 @@ struct llm_build_context { struct ggml_tensor * inpL; // {n_embd, n_tokens} - inpL = llm_build_inp_embd(ctx0, hparams, batch, model.tok_embd, lctx.inp_tokens, lctx.inp_embd, cb); - cb(inpL, "inp_embd", -1); + inpL = llm_build_inp_embd(ctx0, lctx, hparams, batch, model.tok_embd, cb); - struct ggml_tensor * state_mask = ggml_view_2d(ctx0, lctx.inp_s_mask, 1, n_kv, lctx.inp_s_mask->nb[0], 0); - struct ggml_tensor * state_seq = ggml_view_2d(ctx0, lctx.inp_s_seq, n_kv, n_tokens, n_kv*ggml_element_size(lctx.inp_s_seq), 0); + struct ggml_tensor * state_mask = build_inp_s_mask(); + struct ggml_tensor * state_seq = build_inp_s_seq(); for (int il = 0; il < n_layer; ++il) { // (ab)using the KV cache to store the states @@ -8234,7 +8221,7 @@ struct llm_build_context { ggml_build_forward_expand(gf, ggml_cpy(ctx0, ggml_view_2d(ctx0, x_conv, d_conv - 1, d_inner*n_kv, d_conv*ggml_element_size(x_conv), (1+d_inner*n_tokens)*ggml_element_size(x_conv)), - ggml_view_1d(ctx0, kv_self.k_l[il], (d_conv - 1)*(d_inner)*(n_kv), kv_self.head*(d_conv - 1)*(d_inner)*ggml_element_size(x_conv)))); + ggml_view_1d(ctx0, kv_self.k_l[il], (d_conv - 1)*(d_inner)*(n_kv), kv_head*(d_conv - 1)*(d_inner)*ggml_element_size(x_conv)))); // extract x from x_conv x = ggml_view_2d(ctx0, x_conv, d_inner, n_tokens, d_inner*ggml_element_size(x_conv), 0); @@ -8268,7 +8255,7 @@ struct llm_build_context { ggml_build_forward_expand(gf, ggml_cpy(ctx0, ggml_view_1d(ctx0, y_ssm_states, d_state*d_inner*n_kv, d_inner*n_tokens*ggml_element_size(y_ssm_states)), - ggml_view_1d(ctx0, kv_self.v_l[il], d_state*d_inner*n_kv, kv_self.head*d_state*d_inner*ggml_element_size(ssm_states)))); + ggml_view_1d(ctx0, kv_self.v_l[il], d_state*d_inner*n_kv, kv_head*d_state*d_inner*ggml_element_size(ssm_states)))); struct ggml_tensor * y = ggml_view_2d(ctx0, y_ssm_states, d_inner, n_tokens, d_inner*ggml_element_size(y_ssm_states), 0); @@ -8372,7 +8359,18 @@ static struct ggml_cgraph * llama_build_graph( if (!lctx.cparams.offload_kqv) { if (strcmp(name, "kqv_merged_cont") == 0) { // all nodes between the KV store and the attention output are run on the CPU - ggml_backend_sched_set_node_backend(lctx.sched, cur, lctx.backend_cpu); + ggml_backend_sched_set_tensor_backend(lctx.sched, cur, lctx.backend_cpu); + } + } + + // norm may be automatically assigned to the backend of the previous layer, increasing data transfer between backends + // to fix this, we assign the norm layer manually to the backend of its layer + if (il != -1 && strcmp(name, "norm") == 0) { + for (auto * backend : lctx.backends) { + if (ggml_backend_buft_supports_backend(lctx.model.buft_layer[il].buft, backend)) { + ggml_backend_sched_set_tensor_backend(lctx.sched, cur, backend); + break; + } } } }; @@ -8528,7 +8526,7 @@ static void llama_set_inputs(llama_context & lctx, const llama_batch & batch) { ggml_backend_tensor_set(lctx.inp_embd, batch.embd, 0, n_tokens*n_embd*ggml_element_size(lctx.inp_embd)); } - if (batch.pos) { + if (batch.pos && lctx.inp_pos) { const int64_t n_tokens = batch.n_tokens; ggml_backend_tensor_set(lctx.inp_pos, batch.pos, 0, n_tokens*ggml_element_size(lctx.inp_pos)); @@ -8539,61 +8537,63 @@ static void llama_set_inputs(llama_context & lctx, const llama_batch & batch) { "non-causal attention with generative models is not supported" ); - // NOTE: hparams.causal_attn indicates the model is capable of generation and uses the kv cache. - if (cparams.causal_attn) { - const int64_t n_kv = kv_self.n; - const int64_t n_tokens = batch.n_tokens; + if (lctx.inp_KQ_mask) { + // NOTE: hparams.causal_attn indicates the model is capable of generation and uses the kv cache. + if (cparams.causal_attn) { + const int64_t n_kv = kv_self.n; + const int64_t n_tokens = batch.n_tokens; - assert(ggml_backend_buffer_is_host(lctx.inp_KQ_mask->buffer)); + GGML_ASSERT(ggml_backend_buffer_is_host(lctx.inp_KQ_mask->buffer)); - float * data = (float *) lctx.inp_KQ_mask->data; + float * data = (float *) lctx.inp_KQ_mask->data; - // For causal attention, use only the previous KV cells - // of the correct sequence for each token of the batch. - // It's assumed that if a token in the batch has multiple sequences, they are equivalent. - for (int h = 0; h < 1; ++h) { - for (int j = 0; j < n_tokens; ++j) { - const llama_pos pos = batch.pos[j]; - const llama_seq_id seq_id = batch.seq_id[j][0]; + // For causal attention, use only the previous KV cells + // of the correct sequence for each token of the batch. + // It's assumed that if a token in the batch has multiple sequences, they are equivalent. + for (int h = 0; h < 1; ++h) { + for (int j = 0; j < n_tokens; ++j) { + const llama_pos pos = batch.pos[j]; + const llama_seq_id seq_id = batch.seq_id[j][0]; - for (int i = 0; i < n_kv; ++i) { - float f; - if (!lctx.kv_self.cells[i].has_seq_id(seq_id) || lctx.kv_self.cells[i].pos > pos) { - f = -INFINITY; - } else { - f = 0.0f; + for (int i = 0; i < n_kv; ++i) { + float f; + if (!lctx.kv_self.cells[i].has_seq_id(seq_id) || lctx.kv_self.cells[i].pos > pos) { + f = -INFINITY; + } else { + f = 0.0f; + } + data[h*(n_kv*n_tokens) + j*n_kv + i] = f; } - data[h*(n_kv*n_tokens) + j*n_kv + i] = f; } } - } - } else { - // when using kv cache, the mask needs to match the kv cache size - const int64_t n_tokens = batch.n_tokens; - const int64_t n_stride = hparams.causal_attn ? kv_self.n : n_tokens; + } else { + // when using kv cache, the mask needs to match the kv cache size + const int64_t n_tokens = batch.n_tokens; + const int64_t n_stride = hparams.causal_attn ? kv_self.n : n_tokens; - assert(ggml_backend_buffer_is_host(lctx.inp_KQ_mask->buffer)); + GGML_ASSERT(ggml_backend_buffer_is_host(lctx.inp_KQ_mask->buffer)); - float * data = (float *) lctx.inp_KQ_mask->data; + float * data = (float *) lctx.inp_KQ_mask->data; - for (int h = 0; h < 1; ++h) { - for (int j = 0; j < n_tokens; ++j) { - const llama_seq_id seq_id = batch.seq_id[j][0]; + for (int h = 0; h < 1; ++h) { + for (int j = 0; j < n_tokens; ++j) { + const llama_seq_id seq_id = batch.seq_id[j][0]; - for (int i = 0; i < n_tokens; ++i) { - float f = -INFINITY; - for (int s = 0; s < batch.n_seq_id[i]; ++s) { - if (batch.seq_id[i][s] == seq_id) { - f = 0.0f; - break; + for (int i = 0; i < n_tokens; ++i) { + float f = -INFINITY; + for (int s = 0; s < batch.n_seq_id[i]; ++s) { + if (batch.seq_id[i][s] == seq_id) { + f = 0.0f; + break; + } } + + data[h*(n_tokens*n_tokens) + j*n_stride + i] = f; } - data[h*(n_tokens*n_tokens) + j*n_stride + i] = f; - } - - for (int i = n_tokens; i < n_stride; ++i) { - data[h*(n_tokens*n_tokens) + j*n_stride + i] = -INFINITY; + for (int i = n_tokens; i < n_stride; ++i) { + data[h*(n_tokens*n_tokens) + j*n_stride + i] = -INFINITY; + } } } } @@ -8602,7 +8602,8 @@ static void llama_set_inputs(llama_context & lctx, const llama_batch & batch) { if (hparams.need_kq_pos) { const int64_t n_kv = kv_self.n; - assert(ggml_backend_buffer_is_host(lctx.inp_KQ_pos->buffer)); + GGML_ASSERT(lctx.inp_KQ_pos); + GGML_ASSERT(ggml_backend_buffer_is_host(lctx.inp_KQ_pos->buffer)); float * data = (float *) lctx.inp_KQ_pos->data; @@ -8614,6 +8615,7 @@ static void llama_set_inputs(llama_context & lctx, const llama_batch & batch) { if (cparams.pooling_type == LLAMA_POOLING_TYPE_MEAN) { const int64_t n_tokens = batch.n_tokens; + GGML_ASSERT(lctx.inp_mean); GGML_ASSERT(ggml_backend_buffer_is_host(lctx.inp_mean->buffer)); float * data = (float *) lctx.inp_mean->data; @@ -8645,6 +8647,7 @@ static void llama_set_inputs(llama_context & lctx, const llama_batch & batch) { if (cparams.pooling_type == LLAMA_POOLING_TYPE_CLS) { const int64_t n_tokens = batch.n_tokens; + GGML_ASSERT(lctx.inp_cls); GGML_ASSERT(ggml_backend_buffer_is_host(lctx.inp_cls->buffer)); uint32_t * data = (uint32_t *) lctx.inp_cls->data; @@ -8665,7 +8668,7 @@ static void llama_set_inputs(llama_context & lctx, const llama_batch & batch) { if (kv_self.recurrent) { const int64_t n_kv = kv_self.n; - { + if (lctx.inp_s_mask) { GGML_ASSERT(ggml_backend_buffer_is_host(lctx.inp_s_mask->buffer)); float * data = (float *) lctx.inp_s_mask->data; @@ -8687,7 +8690,7 @@ static void llama_set_inputs(llama_context & lctx, const llama_batch & batch) { // update the correct state(s)/sequence(s) for each token of the batch. // Like with the KQ_mask, if a token in the batch has multiple sequences, // they are assumed to be equivalent (not here, but in ggml_ssm_scan and ggml_ssm_conv). - { + if (lctx.inp_s_seq) { const int64_t n_tokens = batch.n_tokens; GGML_ASSERT(ggml_backend_buffer_is_host(lctx.inp_s_seq->buffer)); @@ -8730,7 +8733,7 @@ static void llama_graph_compute( ggml_backend_cpu_set_abort_callback(lctx.backend_cpu, lctx.abort_callback, lctx.abort_callback_data); } - ggml_backend_sched_graph_compute(lctx.sched, gf); + ggml_backend_sched_graph_compute_async(lctx.sched, gf); // fprintf(stderr, "splits: %d\n", ggml_backend_sched_get_n_splits(lctx.sched)); @@ -8750,10 +8753,11 @@ static void llama_graph_compute( // static int llama_decode_internal( llama_context & lctx, - llama_batch batch) { - const uint32_t n_tokens = batch.n_tokens; + llama_batch batch_all) { // TODO: rename back to batch - if (n_tokens == 0) { + const uint32_t n_tokens_all = batch_all.n_tokens; + + if (n_tokens_all == 0) { LLAMA_LOG_ERROR("%s: n_tokens == 0", __func__); return -1; } @@ -8762,14 +8766,16 @@ static int llama_decode_internal( const auto & hparams = model.hparams; const auto & cparams = lctx.cparams; - const auto n_batch = cparams.n_batch; + GGML_ASSERT((!batch_all.token && batch_all.embd) || (batch_all.token && !batch_all.embd)); // NOLINT - GGML_ASSERT(n_tokens <= n_batch); - GGML_ASSERT((!batch.token && batch.embd) || (batch.token && !batch.embd)); // NOLINT + GGML_ASSERT(n_tokens_all <= cparams.n_batch); - int n_threads = n_tokens == 1 ? cparams.n_threads : cparams.n_threads_batch; + GGML_ASSERT((cparams.causal_attn || cparams.n_ubatch >= n_tokens_all) && "non-causal attention requires n_ubatch >= n_tokens"); - const int64_t t_start_us = ggml_time_us(); + if (lctx.t_compute_start_us == 0) { + lctx.t_compute_start_us = ggml_time_us(); + } + lctx.n_queued_tokens += n_tokens_all; #ifdef GGML_USE_MPI // TODO: needs fix after #3228 @@ -8777,128 +8783,261 @@ static int llama_decode_internal( //ggml_mpi_eval_init(lctx.ctx_mpi, &n_tokens, &n_past, &n_threads); #endif - GGML_ASSERT(n_threads > 0); - auto & kv_self = lctx.kv_self; const int64_t n_embd = hparams.n_embd; const int64_t n_vocab = hparams.n_vocab; - // helpers for smoother batch API transition - // after deprecating the llama_eval calls, these will be removed - std::vector pos; + auto * logits_out = lctx.logits; + +#ifndef NDEBUG + auto & logits_valid = lctx.logits_valid; + logits_valid.clear(); + logits_valid.resize(n_tokens_all); + + memset(logits_out, 0, lctx.logits_size*sizeof(float)); +#endif + + const auto n_ubatch = cparams.n_ubatch; + + std::vector pos; std::vector n_seq_id; std::vector seq_id_arr; std::vector> seq_id; - if (batch.pos == nullptr) { - pos.resize(n_tokens); - for (uint32_t i = 0; i < n_tokens; i++) { - pos[i] = batch.all_pos_0 + i*batch.all_pos_1; - } + for (uint32_t cur_token = 0; cur_token < n_tokens_all; cur_token += n_ubatch) { + const uint32_t n_tokens = std::min(n_ubatch, n_tokens_all - cur_token); + llama_batch u_batch = { + /* .n_tokens = */ (int32_t) n_tokens, + /* .token = */ batch_all.token ? batch_all.token + cur_token : nullptr, + /* .embd = */ batch_all.embd ? batch_all.embd + cur_token*n_embd : nullptr, + /* .pos = */ batch_all.pos ? batch_all.pos + cur_token : nullptr, + /* .n_seq_id = */ batch_all.n_seq_id ? batch_all.n_seq_id + cur_token : nullptr, + /* .seq_id = */ batch_all.seq_id ? batch_all.seq_id + cur_token : nullptr, + /* .logits = */ batch_all.logits ? batch_all.logits + cur_token : nullptr, + /* .all_pos_0 = */ batch_all.all_pos_0 + (llama_pos) cur_token*batch_all.all_pos_1, + /* .all_pos_1 = */ batch_all.all_pos_1, + /* .all_seq_id = */ batch_all.all_seq_id, + }; - batch.pos = pos.data(); - } + int n_threads = n_tokens == 1 ? cparams.n_threads : cparams.n_threads_batch; + GGML_ASSERT(n_threads > 0); - if (batch.seq_id == nullptr) { - n_seq_id.resize(n_tokens); - seq_id.resize(n_tokens); - seq_id_arr.resize(n_tokens); - for (uint32_t i = 0; i < n_tokens; i++) { - n_seq_id[i] = 1; - seq_id[i].resize(1); - seq_id[i][0] = batch.all_seq_id; - seq_id_arr[i] = seq_id[i].data(); - } - - batch.n_seq_id = n_seq_id.data(); - batch.seq_id = seq_id_arr.data(); - } - - // non-causal masks do not use the KV cache - if (hparams.causal_attn) { - llama_kv_cache_update(&lctx); - - // if we have enough unused cells before the current head -> - // better to start searching from the beginning of the cache, hoping to fill it - if (kv_self.head > kv_self.used + 2*n_tokens) { - kv_self.head = 0; - } - - if (!llama_kv_cache_find_slot(kv_self, batch)) { - return 1; - } - - if (!kv_self.recurrent) { - // a heuristic, to avoid attending the full cache if it is not yet utilized - // after enough generations, the benefit from this heuristic disappears - // if we start defragmenting the cache, the benefit from this will be more important - kv_self.n = std::min(kv_self.size, std::max(32u, GGML_PAD(llama_kv_cache_cell_max(kv_self), 32))); - //kv_self.n = llama_kv_cache_cell_max(kv_self); - } - } - - //printf("kv_self.n = %5d, kv_self.used = %5d, kv_self.head = %5d\n", kv_self.n, kv_self.used, kv_self.head); - - ggml_backend_sched_reset(lctx.sched); - ggml_backend_sched_set_eval_callback(lctx.sched, lctx.cparams.cb_eval, lctx.cparams.cb_eval_user_data); - - ggml_cgraph * gf = llama_build_graph(lctx, batch, false); - - // the output is always the last tensor in the graph - struct ggml_tensor * res = gf->nodes[gf->n_nodes - 1]; - struct ggml_tensor * embd = gf->nodes[gf->n_nodes - 2]; - - if (!hparams.causal_attn) { - res = nullptr; // do not extract logits for embedding models such as BERT - - // token or sequence embeddings - embd = gf->nodes[gf->n_nodes - 1]; - - GGML_ASSERT(strcmp(embd->name, "result_embd") == 0 || strcmp(embd->name, "result_embd_pooled") == 0); - } else { - if (strcmp(res->name, "result_output") == 0) { - // the token embeddings could be the second to last tensor, or the third to last tensor - if (strcmp(embd->name, "result_norm") != 0) { - embd = gf->nodes[gf->n_nodes - 3]; - GGML_ASSERT(strcmp(embd->name, "result_norm") == 0); + // helpers for smoother batch API transition + // after deprecating the llama_eval calls, these will be removed + if (u_batch.pos == nullptr) { + pos.resize(n_tokens); + for (uint32_t i = 0; i < n_tokens; i++) { + pos[i] = u_batch.all_pos_0 + i*u_batch.all_pos_1; } + + u_batch.pos = pos.data(); + } + + if (u_batch.seq_id == nullptr) { + n_seq_id.resize(n_tokens); + seq_id.resize(n_tokens); + seq_id_arr.resize(n_tokens); + for (uint32_t i = 0; i < n_tokens; i++) { + n_seq_id[i] = 1; + seq_id[i].resize(1); + seq_id[i][0] = u_batch.all_seq_id; + seq_id_arr[i] = seq_id[i].data(); + } + + u_batch.n_seq_id = n_seq_id.data(); + u_batch.seq_id = seq_id_arr.data(); + } + + // non-causal masks do not use the KV cache + if (hparams.causal_attn) { + llama_kv_cache_update(&lctx); + + // if we have enough unused cells before the current head -> + // better to start searching from the beginning of the cache, hoping to fill it + if (kv_self.head > kv_self.used + 2*n_tokens) { + kv_self.head = 0; + } + + if (!llama_kv_cache_find_slot(kv_self, u_batch)) { + return 1; + } + + if (!kv_self.recurrent) { + // a heuristic, to avoid attending the full cache if it is not yet utilized + // after enough generations, the benefit from this heuristic disappears + // if we start defragmenting the cache, the benefit from this will be more important + kv_self.n = std::min(kv_self.size, std::max(32u, GGML_PAD(llama_kv_cache_cell_max(kv_self), 32))); + //kv_self.n = llama_kv_cache_cell_max(kv_self); + } + } + + //printf("kv_self.n = %5d, kv_self.used = %5d, kv_self.head = %5d\n", kv_self.n, kv_self.used, kv_self.head); + + ggml_backend_sched_reset(lctx.sched); + ggml_backend_sched_set_eval_callback(lctx.sched, lctx.cparams.cb_eval, lctx.cparams.cb_eval_user_data); + + ggml_cgraph * gf = llama_build_graph(lctx, u_batch, false); + + // the output is always the last tensor in the graph + struct ggml_tensor * res = gf->nodes[gf->n_nodes - 1]; + struct ggml_tensor * embd = gf->nodes[gf->n_nodes - 2]; + + if (!hparams.causal_attn) { + res = nullptr; // do not extract logits for embedding models such as BERT + + // token or sequence embeddings + embd = gf->nodes[gf->n_nodes - 1]; + + GGML_ASSERT(strcmp(embd->name, "result_embd") == 0 || strcmp(embd->name, "result_embd_pooled") == 0); } else { - GGML_ASSERT(false && "missing result_output tensor"); + if (strcmp(res->name, "result_output") == 0) { + // the token embeddings could be the second to last tensor, or the third to last tensor + if (strcmp(embd->name, "result_norm") != 0) { + embd = gf->nodes[gf->n_nodes - 3]; + GGML_ASSERT(strcmp(embd->name, "result_norm") == 0); + } + } else { + GGML_ASSERT(false && "missing result_output tensor"); + } + } + // LLAMA_LOG_INFO("graph build time: %.3f ms (%d nodes, %d leafs)\n", (ggml_time_us() - t_start_us)/1000.0, gf->n_nodes, gf->n_leafs); + + // for big prompts, if BLAS is enabled, it is better to use only one thread + // otherwise, the threads are spin-lock waiting for the BLAS calls and are degrading the performance + // TODO: this is mostly important for Apple Silicon where CBLAS is still performing very well + // we still need some threads to process all non-mul_mat ops, but not too much to avoid interfering + // with the BLAS calls. need a better solution + // MoE Special Case: This logic applies when hparams.n_expert == 0, i.e. the model is NOT an MoE model. When an MoE is + // being processed then Accelerate/BLAS will not be involved, so capping would limit performance. + if (n_tokens >= 32 && hparams.n_expert == 0 && ggml_cpu_has_blas() && !ggml_cpu_has_gpublas()) { + n_threads = std::min(4, n_threads); + } + + ggml_backend_sched_alloc_graph(lctx.sched, gf); + + llama_set_inputs(lctx, u_batch); + + llama_graph_compute(lctx, gf, n_threads); + + // update the kv ring buffer + { + kv_self.head += n_tokens; + + // Ensure kv cache head points to a valid index. + if (kv_self.head >= kv_self.size) { + kv_self.head = 0; + } + } + +#ifdef GGML_PERF + // print timing information per ggml operation (for debugging purposes) + // requires GGML_PERF to be defined + ggml_graph_print(gf); +#endif + + // plot the computation graph in dot format (for debugging purposes) + //if (n_past%100 == 0) { + // ggml_graph_dump_dot(gf, NULL, "llama.dot"); + //} + + // extract logits + // TODO: do not compute and extract logits if only embeddings are needed + // update the graphs to skip "result_output" if logits are not needed + if (res) { + ggml_backend_t backend_res = ggml_backend_sched_get_tensor_backend(lctx.sched, res); + GGML_ASSERT(backend_res != nullptr); + if (u_batch.logits) { + int32_t i_first = -1; + for (uint32_t i = 0; i < n_tokens; i++) { + if (u_batch.logits[i] && i_first == -1) { + i_first = (int32_t) i; + } + if (u_batch.logits[i] == 0 || i == n_tokens - 1) { + if (i_first != -1) { + int i_last = u_batch.logits[i] == 0 ? i : i + 1; + // extract logits for the range [i_first, i_last) + // group the requests to minimize the number of calls to the backend + ggml_backend_tensor_get_async(backend_res, res, + logits_out + n_vocab*(cur_token + i_first), + i_first*n_vocab*sizeof(float), + (i_last - i_first)*n_vocab*sizeof(float)); + i_first = -1; + } + } +#ifndef NDEBUG + logits_valid[cur_token + i] = u_batch.logits[i] != 0;; +#endif + } + } else if (lctx.logits_all) { + ggml_backend_tensor_get_async(backend_res, res, logits_out + n_vocab*cur_token, 0, n_vocab*n_tokens*sizeof(float)); +#ifndef NDEBUG + std::fill(logits_valid.begin() + cur_token, logits_valid.begin() + cur_token + n_tokens, true); +#endif + } else { + if (cur_token + n_tokens >= n_tokens_all) { + ggml_backend_tensor_get_async(backend_res, res, logits_out, n_vocab*(n_tokens - 1)*sizeof(float), n_vocab*sizeof(float)); +#ifndef NDEBUG + logits_valid[0] = true; +#endif + } + } + } + + // extract embeddings + if (cparams.embeddings && embd) { + ggml_backend_t backend_embd = ggml_backend_sched_get_tensor_backend(lctx.sched, embd); + GGML_ASSERT(backend_embd != nullptr); + + switch (cparams.pooling_type) { + case LLAMA_POOLING_TYPE_NONE: + { + // extract token embeddings + auto & embd_out = lctx.embd; + + if (u_batch.logits) { + //embd_out.resize(n_embd * n_tokens); + for (uint32_t i = 0; i < n_tokens; i++) { + if (u_batch.logits[i] == 0) { + continue; + } + ggml_backend_tensor_get_async(backend_embd, embd, embd_out + n_embd*(i + cur_token), (n_embd*i)*sizeof(float), n_embd*sizeof(float)); + } + } + } break; + case LLAMA_POOLING_TYPE_CLS: + case LLAMA_POOLING_TYPE_MEAN: + { + GGML_ASSERT(strcmp(embd->name, "result_embd_pooled") == 0); + + // extract sequence embeddings + auto & embd_seq_out = lctx.embd_seq; + embd_seq_out.clear(); + + for (uint32_t i = 0; i < n_tokens; i++) { + const llama_seq_id seq_id = u_batch.seq_id[i][0]; + if (embd_seq_out.find(seq_id) != embd_seq_out.end()) { + continue; + } + embd_seq_out[seq_id].resize(n_embd); + ggml_backend_tensor_get_async(backend_embd, embd, embd_seq_out[seq_id].data(), (n_embd*seq_id)*sizeof(float), n_embd*sizeof(float)); + } + } break; + case LLAMA_POOLING_TYPE_UNSPECIFIED: + { + GGML_ASSERT(false && "unknown pooling type"); + } break; + } } } - // LLAMA_LOG_INFO("graph build time: %.3f ms (%d nodes, %d leafs)\n", (ggml_time_us() - t_start_us)/1000.0, gf->n_nodes, gf->n_leafs); - - // for big prompts, if BLAS is enabled, it is better to use only one thread - // otherwise, the threads are spin-lock waiting for the BLAS calls and are degrading the performance - // TODO: this is mostly important for Apple Silicon where CBLAS is still performing very well - // we still need some threads to process all non-mul_mat ops, but not too much to avoid interfering - // with the BLAS calls. need a better solution - // MoE Special Case: This logic applies when hparams.n_expert == 0, i.e. the model is NOT an MoE model. When an MoE is - // being processed then Accelerate/BLAS will not be involved, so capping would limit performance. - if (n_tokens >= 32 && hparams.n_expert == 0 && ggml_cpu_has_blas() && !ggml_cpu_has_gpublas()) { - n_threads = std::min(4, n_threads); - } - - llama_set_inputs(lctx, batch); - - llama_graph_compute(lctx, gf, n_threads); - - // update the kv ring buffer - { - kv_self.head += n_tokens; - - // Ensure kv cache head points to a valid index. - if (kv_self.head >= kv_self.size) { - kv_self.head = 0; - } - } + // wait for the computation to finish (automatically done when obtaining the model output) + //llama_synchronize(&lctx); // decide if we need to defrag the kv cache if (cparams.defrag_thold >= 0.0f) { - const float fragmentation = kv_self.n >= 128 ? 1.0f - float(kv_self.used + n_tokens)/float(kv_self.n) : 0.0f; + const float fragmentation = kv_self.n >= 128 ? 1.0f - float(kv_self.used + n_tokens_all)/float(kv_self.n) : 0.0f; // queue defragmentation for next llama_kv_cache_update if (fragmentation > cparams.defrag_thold) { @@ -8908,141 +9047,10 @@ static int llama_decode_internal( } } -#ifdef GGML_PERF - // print timing information per ggml operation (for debugging purposes) - // requires GGML_PERF to be defined - ggml_graph_print(gf); -#endif - - // plot the computation graph in dot format (for debugging purposes) - //if (n_past%100 == 0) { - // ggml_graph_dump_dot(gf, NULL, "llama.dot"); - //} - - // extract logits - // TODO: do not compute and extract logits if only embeddings are needed - // need to update the graphs to skip "result_output" - if (res) { - auto & logits_out = lctx.logits; - -#ifndef NDEBUG - auto & logits_valid = lctx.logits_valid; - logits_valid.clear(); - logits_valid.resize(n_tokens); - - logits_out.clear(); -#endif - - ggml_backend_t backend_res = ggml_backend_sched_get_node_backend(lctx.sched, res); - GGML_ASSERT(backend_res != nullptr); - - if (batch.logits) { - logits_out.resize(n_vocab * n_tokens); - int32_t i_first = -1; - for (uint32_t i = 0; i < n_tokens; i++) { - if (batch.logits[i] && i_first == -1) { - i_first = (int32_t) i; - } - if (batch.logits[i] == 0 || i == n_tokens - 1) { - if (i_first != -1) { - int i_last = batch.logits[i] == 0 ? i : i + 1; - // extract logits for the range [i_first, i_last) - // group the requests to minimize the number of calls to the backend - ggml_backend_tensor_get_async(backend_res, res, - logits_out.data() + (n_vocab*i_first), - (n_vocab*i_first)*sizeof(float), - (i_last - i_first)*n_vocab*sizeof(float)); - i_first = -1; - } - } -#ifndef NDEBUG - logits_valid[i] = batch.logits[i] != 0; -#endif - } - } else if (lctx.logits_all) { - logits_out.resize(n_vocab*n_tokens); - ggml_backend_tensor_get_async(backend_res, res, logits_out.data(), 0, n_vocab*n_tokens*sizeof(float)); -#ifndef NDEBUG - std::fill(logits_valid.begin(), logits_valid.end(), true); -#endif - } else { - logits_out.resize(n_vocab); - ggml_backend_tensor_get_async(backend_res, res, logits_out.data(), (n_vocab*(n_tokens - 1))*sizeof(float), n_vocab*sizeof(float)); -#ifndef NDEBUG - logits_valid[0] = true; -#endif - } - ggml_backend_synchronize(backend_res); - } - - // extract embeddings - if (cparams.embeddings && embd) { - ggml_backend_t backend_embd = ggml_backend_sched_get_node_backend(lctx.sched, embd); - GGML_ASSERT(backend_embd != nullptr); - - switch (cparams.pooling_type) { - case LLAMA_POOLING_TYPE_NONE: - { - // extract token embeddings - auto & embd_out = lctx.embd; - - if (batch.logits) { - embd_out.resize(n_embd * n_tokens); - for (uint32_t i = 0; i < n_tokens; i++) { - if (batch.logits[i] == 0) { - continue; - } - - ggml_backend_tensor_get_async(backend_embd, embd, embd_out.data() + (n_embd*i), (n_embd*i)*sizeof(float), n_embd*sizeof(float)); - } - } - } break; - case LLAMA_POOLING_TYPE_CLS: - case LLAMA_POOLING_TYPE_MEAN: - { - GGML_ASSERT(strcmp(embd->name, "result_embd_pooled") == 0); - - // extract sequence embeddings - auto & embd_seq_out = lctx.embd_seq; - embd_seq_out.clear(); - - for (uint32_t i = 0; i < n_tokens; i++) { - const llama_seq_id seq_id = batch.seq_id[i][0]; - if (embd_seq_out.find(seq_id) != embd_seq_out.end()) { - continue; - } - embd_seq_out[seq_id].resize(n_embd); - ggml_backend_tensor_get_async(backend_embd, embd, embd_seq_out[seq_id].data(), (n_embd*seq_id)*sizeof(float), n_embd*sizeof(float)); - } - } break; - case LLAMA_POOLING_TYPE_UNSPECIFIED: - { - GGML_ASSERT(false && "unknown pooling type"); - } break; - } - ggml_backend_synchronize(backend_embd); - } - - // measure the performance only for the single-token evals - if (n_tokens == 1) { - lctx.t_eval_us += ggml_time_us() - t_start_us; - lctx.n_eval++; - } - else if (n_tokens > 1) { - lctx.t_p_eval_us += ggml_time_us() - t_start_us; - lctx.n_p_eval += n_tokens; - } - - // get a more accurate load time, upon first eval - // TODO: fix this - if (!lctx.has_evaluated_once) { - lctx.t_load_us = ggml_time_us() - lctx.t_start_us; - lctx.has_evaluated_once = true; - } - return 0; } + // find holes from the beginning of the KV cache and fill them by moving data from the end of the cache static void llama_kv_cache_defrag_internal(struct llama_context & lctx) { auto & kv_self = lctx.kv_self; @@ -9242,6 +9250,8 @@ static void llama_kv_cache_defrag_internal(struct llama_context & lctx) { #else // ggml_graph defrag + ggml_backend_sched_reset(lctx.sched); + ggml_cgraph * gf = llama_build_graph_defrag(lctx, ids); llama_graph_compute(lctx, gf, lctx.cparams.n_threads); @@ -9253,14 +9263,22 @@ static void llama_kv_cache_defrag_internal(struct llama_context & lctx) { } static void llama_kv_cache_update_internal(struct llama_context & lctx) { + bool need_reserve = false; + // apply K-shift if needed if (lctx.model.hparams.rope_type != LLAMA_ROPE_TYPE_NONE && lctx.kv_self.has_shift) { - llama_set_k_shift(lctx); - { + ggml_backend_sched_reset(lctx.sched); + ggml_cgraph * gf = llama_build_graph_k_shift(lctx); + ggml_backend_sched_alloc_graph(lctx.sched, gf); + + llama_set_k_shift(lctx); + llama_graph_compute(lctx, gf, lctx.cparams.n_threads); + + need_reserve = true; } { @@ -9275,12 +9293,18 @@ static void llama_kv_cache_update_internal(struct llama_context & lctx) { } if (lctx.kv_self.recurrent && lctx.kv_self.do_copy) { - llama_set_s_copy(lctx); - { + ggml_backend_sched_reset(lctx.sched); + ggml_cgraph * gf = llama_build_graph_s_copy(lctx); + ggml_backend_sched_alloc_graph(lctx.sched, gf); + + llama_set_s_copy(lctx); + llama_graph_compute(lctx, gf, lctx.cparams.n_threads); + + need_reserve = true; } { @@ -9298,8 +9322,26 @@ static void llama_kv_cache_update_internal(struct llama_context & lctx) { if (lctx.kv_self.do_defrag) { llama_kv_cache_defrag_internal(lctx); + need_reserve = true; + lctx.kv_self.do_defrag = false; } + + // reserve a worst case graph again + if (need_reserve) { + // TODO: extract to a function + // build worst-case graph + int n_tokens = (int)std::min(lctx.cparams.n_ctx, lctx.cparams.n_ubatch); + int n_past = lctx.cparams.n_ctx - n_tokens; + llama_token token = llama_token_bos(&lctx.model); // not actually used by llama_build_graph, but required to choose between token and embedding inputs graph + ggml_cgraph * gf = llama_build_graph(lctx, llama_batch_get_one(&token, n_tokens, n_past, 0), true); + + // initialize scheduler with the worst-case graph + ggml_backend_sched_reset(lctx.sched); + if (!ggml_backend_sched_reserve(lctx.sched, gf)) { + LLAMA_LOG_ERROR("%s: failed to allocate compute buffers\n", __func__); + } + } } // @@ -12537,7 +12579,8 @@ struct llama_context_params llama_context_default_params() { struct llama_context_params result = { /*.seed =*/ LLAMA_DEFAULT_SEED, /*.n_ctx =*/ 512, - /*.n_batch =*/ 512, + /*.n_batch =*/ 2048, + /*.n_ubatch =*/ 512, /*.n_seq_max =*/ 1, /*.n_threads =*/ GGML_DEFAULT_N_THREADS, // TODO: better default /*.n_threads_batch =*/ GGML_DEFAULT_N_THREADS, @@ -12691,6 +12734,17 @@ struct llama_context * llama_new_context_with_model( struct llama_context_params params) { if (!model) { + LLAMA_LOG_ERROR("%s: model cannot be NULL\n", __func__); + return nullptr; + } + + if (params.n_batch == 0 && params.n_ubatch == 0) { + LLAMA_LOG_ERROR("%s: n_batch and n_ubatch cannot both be zero\n", __func__); + return nullptr; + } + + if (params.n_ctx == 0 && model->hparams.n_ctx_train == 0) { + LLAMA_LOG_ERROR("%s: n_ctx and model->hparams.n_ctx_train cannot both be zero\n", __func__); return nullptr; } @@ -12699,7 +12753,6 @@ struct llama_context * llama_new_context_with_model( const auto & hparams = model->hparams; auto & cparams = ctx->cparams; - cparams.n_batch = params.n_batch; // TODO: maybe add n_seq_max here too cparams.n_threads = params.n_threads; cparams.n_threads_batch = params.n_threads_batch; @@ -12716,6 +12769,11 @@ struct llama_context * llama_new_context_with_model( cparams.rope_freq_base = params.rope_freq_base == 0.0f ? hparams.rope_freq_base_train : params.rope_freq_base; cparams.rope_freq_scale = params.rope_freq_scale == 0.0f ? hparams.rope_freq_scale_train : params.rope_freq_scale; + // with causal attention, the batch size is limited by the context size + cparams.n_batch = hparams.causal_attn ? std::min(cparams.n_ctx, params.n_batch) : params.n_batch; + cparams.n_ubatch = std::min(cparams.n_batch, params.n_ubatch == 0 ? params.n_batch : params.n_ubatch); + + cparams.n_yarn_orig_ctx = params.yarn_orig_ctx != 0 ? params.yarn_orig_ctx : hparams.n_yarn_orig_ctx != 0 ? hparams.n_yarn_orig_ctx : hparams.n_ctx_train; @@ -12751,6 +12809,8 @@ struct llama_context * llama_new_context_with_model( } LLAMA_LOG_INFO("%s: n_ctx = %u\n", __func__, cparams.n_ctx); + LLAMA_LOG_INFO("%s: n_batch = %u\n", __func__, cparams.n_batch); + LLAMA_LOG_INFO("%s: n_ubatch = %u\n", __func__, cparams.n_ubatch); LLAMA_LOG_INFO("%s: freq_base = %.1f\n", __func__, cparams.rope_freq_base); LLAMA_LOG_INFO("%s: freq_scale = %g\n", __func__, cparams.rope_freq_scale); @@ -12895,54 +12955,31 @@ struct llama_context * llama_new_context_with_model( ggml_type_name(type_v), (float)memory_size_v / (1024.0f * 1024.0f)); } - // resized during inference, reserve maximum - ctx->logits.reserve(hparams.n_vocab*cparams.n_batch); - - if (params.embeddings) { - ctx->embd.reserve(hparams.n_embd*cparams.n_batch); - } - - // graph inputs + // graph outputs buffer { - ggml_init_params init_params = { - /* .mem_size */ ggml_tensor_overhead()*(8 + 3*(ctx->kv_self.recurrent)), - /* .mem_buffer */ nullptr, - /* .no_alloc */ true, - }; - ctx->ctx_input = ggml_init(init_params); + // resized during inference, reserve maximum + ctx->logits_size = hparams.n_vocab*cparams.n_batch; + ctx->embd_size = params.embeddings ? hparams.n_embd*cparams.n_batch : 0; - ctx->inp_tokens = ggml_new_tensor_1d(ctx->ctx_input, GGML_TYPE_I32, cparams.n_batch); - ctx->inp_embd = ggml_new_tensor_2d(ctx->ctx_input, GGML_TYPE_F32, hparams.n_embd, cparams.n_batch); - ctx->inp_pos = ggml_new_tensor_1d(ctx->ctx_input, GGML_TYPE_I32, cparams.n_batch); - ctx->inp_KQ_mask = ggml_new_tensor_2d(ctx->ctx_input, GGML_TYPE_F32, kv_size, cparams.n_batch); - ctx->inp_KQ_pos = ggml_new_tensor_1d(ctx->ctx_input, GGML_TYPE_F32, kv_size); - ctx->inp_K_shift = ggml_new_tensor_1d(ctx->ctx_input, GGML_TYPE_I32, kv_size); - ctx->inp_mean = ggml_new_tensor_2d(ctx->ctx_input, GGML_TYPE_F32, cparams.n_batch, cparams.n_batch); - ctx->inp_cls = ggml_new_tensor_1d(ctx->ctx_input, GGML_TYPE_I32, cparams.n_batch); - if (ctx->kv_self.recurrent) { - ctx->inp_s_copy = ggml_new_tensor_1d(ctx->ctx_input, GGML_TYPE_I32, kv_size); - ctx->inp_s_mask = ggml_new_tensor_1d(ctx->ctx_input, GGML_TYPE_F32, kv_size); - ctx->inp_s_seq = ggml_new_tensor_2d(ctx->ctx_input, GGML_TYPE_I32, kv_size, cparams.n_batch); + const size_t buf_output_size = (ctx->logits_size + ctx->embd_size)*sizeof(float); + + ctx->buf_output = ggml_backend_buft_alloc_buffer(llama_default_buffer_type_cpu(true), buf_output_size); + if (ctx->buf_output == nullptr) { + LLAMA_LOG_ERROR("%s: failed to allocate logits buffer\n", __func__); + llama_free(ctx); + return nullptr; + } + ggml_backend_buffer_clear(ctx->buf_output, 0); + + + ctx->logits = (float *) ggml_backend_buffer_get_base(ctx->buf_output); + if (params.embeddings) { + ctx->embd = ctx->logits + ctx->logits_size; } - ggml_set_name(ctx->inp_tokens, "inp_tokens"); - ggml_set_name(ctx->inp_embd, "inp_embd"); - ggml_set_name(ctx->inp_pos, "inp_pos"); - ggml_set_name(ctx->inp_KQ_mask, "inp_KQ_mask"); - ggml_set_name(ctx->inp_KQ_pos, "inp_KQ_pos"); - ggml_set_name(ctx->inp_K_shift, "inp_K_shift"); - ggml_set_name(ctx->inp_mean, "inp_mean"); - ggml_set_name(ctx->inp_cls, "inp_cls"); - if (ctx->kv_self.recurrent) { - ggml_set_name(ctx->inp_s_copy, "inp_s_copy"); - ggml_set_name(ctx->inp_s_mask, "inp_s_mask"); - ggml_set_name(ctx->inp_s_seq, "inp_s_seq"); - } - - ctx->buf_input = ggml_backend_alloc_ctx_tensors_from_buft(ctx->ctx_input, llama_default_buffer_type_cpu(true)); - LLAMA_LOG_INFO("%s: %10s input buffer size = %8.2f MiB\n", __func__, - ggml_backend_buffer_name(ctx->buf_input), - ggml_backend_buffer_get_size(ctx->buf_input) / 1024.0 / 1024.0); + LLAMA_LOG_INFO("%s: %10s output buffer size = %8.2f MiB\n", __func__, + ggml_backend_buffer_name(ctx->buf_output), + ggml_backend_buffer_get_size(ctx->buf_output) / 1024.0 / 1024.0); } // scheduler and compute buffers @@ -12961,10 +12998,21 @@ struct llama_context * llama_new_context_with_model( // buffer used to store the computation graph and the tensor meta data ctx->buf_compute_meta.resize(ggml_tensor_overhead()*LLAMA_MAX_NODES + ggml_graph_overhead_custom(LLAMA_MAX_NODES, false)); - ctx->sched = ggml_backend_sched_new(ctx->backends.data(), backend_buft.data(), ctx->backends.size(), LLAMA_MAX_NODES); + // enabling pipeline parallelism in the scheduler increases memory usage, so it is only done when necessary + bool pipeline_parallel = llama_get_device_count() > 1 && model->n_gpu_layers > (int)model->hparams.n_layer && model->split_mode == LLAMA_SPLIT_MODE_LAYER; +#ifndef GGML_USE_CUBLAS + // pipeline parallelism requires support for async compute and events + // currently this is only implemented in the CUDA backend + pipeline_parallel = false; +#endif + ctx->sched = ggml_backend_sched_new(ctx->backends.data(), backend_buft.data(), ctx->backends.size(), LLAMA_MAX_NODES, pipeline_parallel); + + if (pipeline_parallel) { + LLAMA_LOG_INFO("%s: pipeline parallelism enabled (n_copies=%d)\n", __func__, ggml_backend_sched_get_n_copies(ctx->sched)); + } // build worst-case graph - int n_tokens = (int)std::min(cparams.n_ctx, cparams.n_batch); + int n_tokens = (int)std::min(cparams.n_ctx, cparams.n_ubatch); int n_past = cparams.n_ctx - n_tokens; llama_token token = llama_token_bos(&ctx->model); // not actually used by llama_build_graph, but required to choose between token and embedding inputs graph ggml_cgraph * gf = llama_build_graph(*ctx, llama_batch_get_one(&token, n_tokens, n_past, 0), true); @@ -12987,7 +13035,7 @@ struct llama_context * llama_new_context_with_model( // note: the number of splits during measure is higher than during inference due to the kv shift int n_splits = ggml_backend_sched_get_n_splits(ctx->sched); - LLAMA_LOG_INFO("%s: graph splits (measure): %d\n", __func__, n_splits); + LLAMA_LOG_INFO("%s: graph splits: %d\n", __func__, n_splits); } } @@ -13024,6 +13072,10 @@ uint32_t llama_n_batch(const struct llama_context * ctx) { return ctx->cparams.n_batch; } +uint32_t llama_n_ubatch(const struct llama_context * ctx) { + return ctx->cparams.n_ubatch; +} + uint32_t llama_n_seq_max(const struct llama_context * ctx) { return ctx->kv_self.size; } @@ -13347,9 +13399,9 @@ size_t llama_get_state_size(const struct llama_context * ctx) { const size_t s_rng = LLAMA_MAX_RNG_STATE; const size_t s_logits_size = sizeof(size_t); // assume worst case for logits although only currently set ones are serialized - const size_t s_logits = ctx->logits.capacity() * sizeof(float); + const size_t s_logits = ctx->logits_size * sizeof(float); const size_t s_embedding_size = sizeof(size_t); - const size_t s_embedding = ctx->embd.capacity() * sizeof(float); + const size_t s_embedding = ctx->embd_size * sizeof(float); const size_t s_kv_buf_size = sizeof(size_t); const size_t s_kv_head = sizeof(uint32_t); const size_t s_kv_size = sizeof(uint32_t); @@ -13447,23 +13499,23 @@ static void llama_copy_state_data_internal(struct llama_context * ctx, llama_dat // copy logits { - const size_t logits_size = ctx->logits.size(); + const size_t logits_size = ctx->logits_size; data_ctx->write(&logits_size, sizeof(logits_size)); if (logits_size) { - data_ctx->write(ctx->logits.data(), logits_size * sizeof(float)); + data_ctx->write(ctx->logits, logits_size * sizeof(float)); } } // copy embeddings { - const size_t embeddings_size = ctx->embd.size(); + const size_t embeddings_size = ctx->embd_size; data_ctx->write(&embeddings_size, sizeof(embeddings_size)); if (embeddings_size) { - data_ctx->write(ctx->embd.data(), embeddings_size * sizeof(float)); + data_ctx->write(ctx->embd, embeddings_size * sizeof(float)); } } @@ -13566,12 +13618,10 @@ size_t llama_set_state_data(struct llama_context * ctx, const uint8_t * src) { memcpy(&logits_size, inp, sizeof(logits_size)); inp += sizeof(logits_size); - GGML_ASSERT(ctx->logits.capacity() >= logits_size); + GGML_ASSERT(ctx->logits_size >= logits_size); if (logits_size) { - ctx->logits.resize(logits_size); - - memcpy(ctx->logits.data(), inp, logits_size * sizeof(float)); + memcpy(ctx->logits, inp, logits_size * sizeof(float)); inp += logits_size * sizeof(float); } } @@ -13582,12 +13632,10 @@ size_t llama_set_state_data(struct llama_context * ctx, const uint8_t * src) { memcpy(&embeddings_size, inp, sizeof(embeddings_size)); inp += sizeof(embeddings_size); - GGML_ASSERT(ctx->embd.capacity() == embeddings_size); + GGML_ASSERT(ctx->embd_size == embeddings_size); if (embeddings_size) { - ctx->embd.resize(embeddings_size); - - memcpy(ctx->embd.data(), inp, embeddings_size * sizeof(float)); + memcpy(ctx->embd, inp, embeddings_size * sizeof(float)); inp += embeddings_size * sizeof(float); } } @@ -13842,24 +13890,61 @@ int32_t llama_decode( return ret; } +void llama_synchronize(struct llama_context * ctx) { + ggml_backend_sched_synchronize(ctx->sched); + + // FIXME: if multiple single tokens are evaluated without a synchronization, + // the stats will be added to the prompt evaluation stats + // this should only happen when using batch size 1 to evaluate a batch + + // add the evaluation to the stats + if (ctx->n_queued_tokens == 1) { + ctx->t_eval_us += ggml_time_us() - ctx->t_compute_start_us; + ctx->n_eval++; + } else if (ctx->n_queued_tokens > 1) { + ctx->t_p_eval_us += ggml_time_us() - ctx->t_compute_start_us; + ctx->n_p_eval += ctx->n_queued_tokens; + } + + // get a more accurate load time, upon first eval + if (ctx->n_queued_tokens > 0 && !ctx->has_evaluated_once) { + ctx->t_load_us = ggml_time_us() - ctx->t_start_us; + ctx->has_evaluated_once = true; + } + + ctx->n_queued_tokens = 0; + ctx->t_compute_start_us = 0; +} + float * llama_get_logits(struct llama_context * ctx) { - return ctx->logits.data(); + llama_synchronize(ctx); + + return ctx->logits; } float * llama_get_logits_ith(struct llama_context * ctx, int32_t i) { assert(ctx->logits_valid.at(i)); - return ctx->logits.data() + i*ctx->model.hparams.n_vocab; + + llama_synchronize(ctx); + + return ctx->logits + i*ctx->model.hparams.n_vocab; } float * llama_get_embeddings(struct llama_context * ctx) { - return ctx->embd.data(); + llama_synchronize(ctx); + + return ctx->embd; } float * llama_get_embeddings_ith(struct llama_context * ctx, int32_t i) { - return ctx->embd.data() + i*ctx->model.hparams.n_embd; + llama_synchronize(ctx); + + return ctx->embd + i*ctx->model.hparams.n_embd; } float * llama_get_embeddings_seq(struct llama_context * ctx, llama_seq_id seq_id) { + llama_synchronize(ctx); + auto it = ctx->embd_seq.find(seq_id); if (it == ctx->embd_seq.end()) { return nullptr; diff --git a/llama.h b/llama.h index 446899da6..2d16cc9b9 100644 --- a/llama.h +++ b/llama.h @@ -234,7 +234,8 @@ extern "C" { struct llama_context_params { uint32_t seed; // RNG seed, -1 for random uint32_t n_ctx; // text context, 0 = from model - uint32_t n_batch; // prompt processing maximum batch size + uint32_t n_batch; // logical maximum batch size that can be submitted to llama_decode + uint32_t n_ubatch; // physical maximum batch size uint32_t n_seq_max; // max number of sequences (i.e. distinct states for recurrent models) uint32_t n_threads; // number of threads to use for generation uint32_t n_threads_batch; // number of threads to use for batch processing @@ -377,6 +378,7 @@ extern "C" { LLAMA_API uint32_t llama_n_ctx (const struct llama_context * ctx); LLAMA_API uint32_t llama_n_batch (const struct llama_context * ctx); + LLAMA_API uint32_t llama_n_ubatch (const struct llama_context * ctx); LLAMA_API uint32_t llama_n_seq_max (const struct llama_context * ctx); LLAMA_API enum llama_vocab_type llama_vocab_type(const struct llama_model * model); @@ -650,6 +652,11 @@ extern "C" { // Set abort callback LLAMA_API void llama_set_abort_callback(struct llama_context * ctx, ggml_abort_callback abort_callback, void * abort_callback_data); + // Wait until all computations are finished + // This is automatically done when using one of the functions below to obtain the computation results + // and is not necessary to call it explicitly in most cases + LLAMA_API void llama_synchronize(struct llama_context * ctx); + // Token logits obtained from the last call to llama_decode() // The logits for the last token are stored in the last row // Logits for which llama_batch.logits[i] == 0 are undefined From 463628372d5fe3a0c1e5864aa5fc57deb7387039 Mon Sep 17 00:00:00 2001 From: Clint Herron Date: Wed, 13 Mar 2024 14:10:40 -0400 Subject: [PATCH 610/812] grammar : handle missing "root" node (#6004) --- common/sampling.cpp | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/common/sampling.cpp b/common/sampling.cpp index 823031feb..5a5450982 100644 --- a/common/sampling.cpp +++ b/common/sampling.cpp @@ -17,6 +17,13 @@ struct llama_sampling_context * llama_sampling_init(const struct llama_sampling_ return nullptr; } + // Ensure that there is a "root" node. + if (result->parsed_grammar.symbol_ids.find("root") == result->parsed_grammar.symbol_ids.end()) { + fprintf(stderr, "%s: grammar does not contain a 'root' symbol\n", __func__); + delete result; + return nullptr; + } + std::vector grammar_rules(result->parsed_grammar.c_rules()); result->grammar = llama_grammar_init( From 76a936c8939c249a7c3e8e66dfefbab13eae194f Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Wed, 13 Mar 2024 20:33:56 +0200 Subject: [PATCH 611/812] readme : update API changes and hot topics --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 54bf84bec..80037782f 100644 --- a/README.md +++ b/README.md @@ -10,12 +10,14 @@ Inference of Meta's [LLaMA](https://arxiv.org/abs/2302.13971) model (and others) ### Recent API changes +- [2024 Mar 13] Add `llama_synchronize()` + `llama_context_params.n_ubatch` https://github.com/ggerganov/llama.cpp/pull/6017 - [2024 Mar 8] `llama_kv_cache_seq_rm()` returns a `bool` instead of `void`, and new `llama_n_seq_max()` returns the upper limit of acceptable `seq_id` in batches (relevant when dealing with multiple sequences) https://github.com/ggerganov/llama.cpp/pull/5328 - [2024 Mar 4] Embeddings API updated https://github.com/ggerganov/llama.cpp/pull/5796 - [2024 Mar 3] `struct llama_context_params` https://github.com/ggerganov/llama.cpp/pull/5849 ### Hot topics +- Multi-GPU pipeline parallelizm support https://github.com/ggerganov/llama.cpp/pull/6017 - Looking for contributions to add Deepseek support: https://github.com/ggerganov/llama.cpp/issues/5981 - Quantization blind testing: https://github.com/ggerganov/llama.cpp/discussions/5962 - Initial Mamba support has been added: https://github.com/ggerganov/llama.cpp/pull/5328 From 19885d205e768579ab090d1e99281cae58c21b54 Mon Sep 17 00:00:00 2001 From: Linwei Wang Date: Thu, 14 Mar 2024 02:34:40 +0800 Subject: [PATCH 612/812] readme : update details about running llama in Termux on Android (#6039) --- README.md | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 80037782f..61bedc3f8 100644 --- a/README.md +++ b/README.md @@ -904,6 +904,9 @@ First, install the essential packages for termux: pkg install clang wget git cmake ``` Second, obtain the [Android NDK](https://developer.android.com/ndk) and then build with CMake: + +You can execute the following commands on your computer to avoid downloading the NDK to your mobile. Of course, you can also do this in Termux. + ``` $ mkdir build-android $ cd build-android @@ -912,7 +915,28 @@ $ cmake -DCMAKE_TOOLCHAIN_FILE=$NDK/build/cmake/android.toolchain.cmake -DANDROI $ make ``` Install [termux](https://termux.dev/) on your device and run `termux-setup-storage` to get access to your SD card. -Finally, copy the `llama` binary and the model files to your device storage. Here is a demo of an interactive session running on Pixel 5 phone: +Finally, copy these built `llama` binaries and the model file to your device storage. Because the file permissions in the Android sdcard cannot be changed, you can copy the executable files to the `/data/data/com.termux/files/home/bin` path, and then execute the following commands in Termux to add executable permission: + +(Assumed that you have pushed the built executable files to the /sdcard/llama.cpp/bin path using `adb push`) +``` +$cp -r /sdcard/llama.cpp/bin /data/data/com.termux/files/home/ +$cd /data/data/com.termux/files/home/bin +$chmod +x ./* +``` + +Download model [llama-2-7b-chat.Q4_K_M.gguf](https://huggingface.co/TheBloke/Llama-2-7B-Chat-GGUF/blob/main/llama-2-7b-chat.Q4_K_M.gguf), and push it to `/sdcard/llama.cpp/`, then move it to `/data/data/com.termux/files/home/model/` + +``` +$mv /sdcard/llama.cpp/llama-2-7b-chat.Q4_K_M.gguf /data/data/com.termux/files/home/model/ +``` + +Now, you can start chatting: +``` +$cd /data/data/com.termux/files/home/bin +$./main -m ../model/llama-2-7b-chat.Q4_K_M.gguf -n 128 -cml +``` + +Here is a demo of an interactive session running on Pixel 5 phone: https://user-images.githubusercontent.com/271616/225014776-1d567049-ad71-4ef2-b050-55b0b3b9274c.mp4 From 0fd6c1f015f6cccf3b527f7dbd8386a434728126 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Thu, 14 Mar 2024 10:12:29 +0200 Subject: [PATCH 613/812] embedding : print cosine similarity (#899) --- common/common.cpp | 13 +++++++++++++ common/common.h | 1 + examples/embedding/embedding.cpp | 21 ++++++++++++++++----- examples/gritlm/gritlm.cpp | 26 ++++++-------------------- 4 files changed, 36 insertions(+), 25 deletions(-) diff --git a/common/common.cpp b/common/common.cpp index 73b1b61ba..58fbd05aa 100644 --- a/common/common.cpp +++ b/common/common.cpp @@ -1877,3 +1877,16 @@ void llama_embd_normalize(const float * inp, float * out, int n) { } } +float llama_embd_similarity_cos(const float * embd1, const float * embd2, int n){ + double sum = 0.0; + double sum1 = 0.0; + double sum2 = 0.0; + + for (int i = 0; i < n; i++) { + sum += embd1[i] * embd2[i]; + sum1 += embd1[i] * embd1[i]; + sum2 += embd2[i] * embd2[i]; + } + + return sum / (sqrt(sum1) * sqrt(sum2)); +} diff --git a/common/common.h b/common/common.h index 0f178b9eb..d250eef8b 100644 --- a/common/common.h +++ b/common/common.h @@ -268,3 +268,4 @@ void dump_kv_cache_view_seqs(const llama_kv_cache_view & view, int row_size = 40 void llama_embd_normalize(const float * inp, float * out, int n); +float llama_embd_similarity_cos(const float * embd1, const float * embd2, int n); diff --git a/examples/embedding/embedding.cpp b/examples/embedding/embedding.cpp index 49302a199..f390c4061 100644 --- a/examples/embedding/embedding.cpp +++ b/examples/embedding/embedding.cpp @@ -168,14 +168,25 @@ int main(int argc, char ** argv) { batch_decode(ctx, batch, out, s, n_embd); // print first 3 embeddings + fprintf(stdout, "\n"); for (int j = 0; j < std::min(3, n_prompts); j++) { - fprintf(stderr, "embedding %d: ", j); - for (int i = 0; i < n_embd; i++) { - fprintf(stderr, "%f ", emb[j * n_embd + i]); + fprintf(stdout, "embedding %d: ", j); + for (int i = 0; i < std::min(16, n_embd); i++) { + fprintf(stdout, "%f ", emb[j * n_embd + i]); } - fprintf(stderr, "\n\n"); + fprintf(stdout, "\n"); + } + + // print cosine similarity matrix + fprintf(stdout, "\n"); + printf("cosine similarity matrix:\n\n"); + for (int i = 0; i < n_prompts; i++) { + for (int j = 0; j < n_prompts; j++) { + float sim = llama_embd_similarity_cos(emb + i * n_embd, emb + j * n_embd, n_embd); + fprintf(stdout, "%6.2f ", sim); + } + fprintf(stdout, "\n"); } - fprintf(stderr, "\n"); // clean up llama_print_timings(ctx); diff --git a/examples/gritlm/gritlm.cpp b/examples/gritlm/gritlm.cpp index 3d4b085d6..52fd719b3 100644 --- a/examples/gritlm/gritlm.cpp +++ b/examples/gritlm/gritlm.cpp @@ -6,22 +6,6 @@ // #define GRIT_DEBUG -static float dot_product(const std::vector & v1, const std::vector & v2) { - float dot = 0.0f; - for (uint64_t i = 0; i < v1.size(); ++i) { - dot += v1[i] * v2[i]; - } - return dot; -} - -static float norm(const std::vector & v) { - return std::sqrt(dot_product(v, v)); -} - -static float cosine_similarity(const std::vector & v1, const std::vector & v2) { - return dot_product(v1, v2) / (norm(v1) * norm(v2)); -} - static std::vector> encode(llama_context * ctx, const std::vector & sentences, const std::string & instruction) { std::vector> result; @@ -203,10 +187,12 @@ int main(int argc, char * argv[]) { const std::vector> d_rep = encode(ctx, documents, gritlm_instruction("")); const std::vector> q_rep = encode(ctx, queries, gritlm_instruction(instruction)); - const float cosine_sim_q0_d0 = cosine_similarity(q_rep[0], d_rep[0]); - const float cosine_sim_q0_d1 = cosine_similarity(q_rep[0], d_rep[1]); - const float cosine_sim_q1_d0 = cosine_similarity(q_rep[1], d_rep[0]); - const float cosine_sim_q1_d1 = cosine_similarity(q_rep[1], d_rep[1]); + const int n_embd = llama_n_embd(mdl); + + const float cosine_sim_q0_d0 = llama_embd_similarity_cos(q_rep[0].data(), d_rep[0].data(), n_embd); + const float cosine_sim_q0_d1 = llama_embd_similarity_cos(q_rep[0].data(), d_rep[1].data(), n_embd); + const float cosine_sim_q1_d0 = llama_embd_similarity_cos(q_rep[1].data(), d_rep[0].data(), n_embd); + const float cosine_sim_q1_d1 = llama_embd_similarity_cos(q_rep[1].data(), d_rep[1].data(), n_embd); std::printf("Cosine similarity between \"%.50s\" and \"%.50s\" is: %.3f\n", queries[0].c_str(), documents[0].c_str(), cosine_sim_q0_d0); std::printf("Cosine similarity between \"%.50s\" and \"%.50s\" is: %.3f\n", queries[0].c_str(), documents[1].c_str(), cosine_sim_q0_d1); From 381da2d9f0940d7009e3e918bed36338c8ff2fbb Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Thu, 14 Mar 2024 11:55:23 +0200 Subject: [PATCH 614/812] metal : build metallib + fix embed path (#6015) * metal : build metallib + fix embed path ggml-ci * metal : fix embed build + update library load logic ggml-ci * metal : fix embeded library build ggml-ci * ci : fix iOS builds to use embedded library --- .github/workflows/build.yml | 2 ++ .gitignore | 2 ++ CMakeLists.txt | 70 ++++++++++++++++++++----------------- Makefile | 15 ++++---- ggml-metal.m | 49 +++++++++++++++++--------- ggml-metal.metal | 3 -- 6 files changed, 83 insertions(+), 58 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d39cd6bc3..0da01d5ba 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -333,6 +333,7 @@ jobs: mkdir build cd build cmake -G Xcode .. \ + -DLLAMA_METAL_EMBED_LIBRARY=ON \ -DLLAMA_BUILD_EXAMPLES=OFF \ -DLLAMA_BUILD_TESTS=OFF \ -DLLAMA_BUILD_SERVER=OFF \ @@ -361,6 +362,7 @@ jobs: mkdir build cd build cmake -G Xcode .. \ + -DLLAMA_METAL_EMBED_LIBRARY=ON \ -DLLAMA_BUILD_EXAMPLES=OFF \ -DLLAMA_BUILD_TESTS=OFF \ -DLLAMA_BUILD_SERVER=OFF \ diff --git a/.gitignore b/.gitignore index d28f4d1b8..1ad8d929b 100644 --- a/.gitignore +++ b/.gitignore @@ -25,6 +25,8 @@ .vscode/ .idea/ +ggml-metal-embed.metal + lcov-report/ gcovr-report/ diff --git a/CMakeLists.txt b/CMakeLists.txt index a8abf4088..3ac2804a6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -200,9 +200,6 @@ if (LLAMA_METAL) add_compile_definitions(GGML_METAL_NDEBUG) endif() - # get full path to the file - #add_compile_definitions(GGML_METAL_DIR_KERNELS="${CMAKE_CURRENT_SOURCE_DIR}/") - # copy ggml-common.h and ggml-metal.metal to bin directory configure_file(ggml-common.h ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/ggml-common.h COPYONLY) configure_file(ggml-metal.metal ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/ggml-metal.metal COPYONLY) @@ -211,53 +208,62 @@ if (LLAMA_METAL) enable_language(ASM) add_compile_definitions(GGML_METAL_EMBED_LIBRARY) + set(METALLIB_COMMON "${CMAKE_CURRENT_SOURCE_DIR}/ggml-common.h") set(METALLIB_SOURCE "${CMAKE_CURRENT_SOURCE_DIR}/ggml-metal.metal") + file(MAKE_DIRECTORY "${CMAKE_BINARY_DIR}/autogenerated") - set(EMBED_METALLIB_ASSEMBLY "${CMAKE_BINARY_DIR}/autogenerated/ggml-embed-metallib.s") + + # merge ggml-common.h and ggml-metal.metal into a single file + set(METALLIB_EMBED_ASM "${CMAKE_BINARY_DIR}/autogenerated/ggml-metal-embed.s") + set(METALLIB_SOURCE_EMBED "${CMAKE_BINARY_DIR}/autogenerated/ggml-metal-embed.metal") add_custom_command( - OUTPUT ${EMBED_METALLIB_ASSEMBLY} - COMMAND echo ".section __DATA,__ggml_metallib" > ${EMBED_METALLIB_ASSEMBLY} - COMMAND echo ".globl _ggml_metallib_start" >> ${EMBED_METALLIB_ASSEMBLY} - COMMAND echo "_ggml_metallib_start:" >> ${EMBED_METALLIB_ASSEMBLY} - COMMAND echo ".incbin \\\"${METALLIB_SOURCE}\\\"" >> ${EMBED_METALLIB_ASSEMBLY} - COMMAND echo ".globl _ggml_metallib_end" >> ${EMBED_METALLIB_ASSEMBLY} - COMMAND echo "_ggml_metallib_end:" >> ${EMBED_METALLIB_ASSEMBLY} - DEPENDS ${METALLIB_SOURCE} + OUTPUT ${METALLIB_EMBED_ASM} + COMMAND echo "Embedding Metal library" + COMMAND sed -e '/\#include \"ggml-common.h\"/r ${METALLIB_COMMON}' -e '/\#include \"ggml-common.h\"/d' < ${METALLIB_SOURCE} > ${METALLIB_SOURCE_EMBED} + COMMAND echo ".section __DATA,__ggml_metallib" > ${METALLIB_EMBED_ASM} + COMMAND echo ".globl _ggml_metallib_start" >> ${METALLIB_EMBED_ASM} + COMMAND echo "_ggml_metallib_start:" >> ${METALLIB_EMBED_ASM} + COMMAND echo ".incbin \\\"${METALLIB_SOURCE_EMBED}\\\"" >> ${METALLIB_EMBED_ASM} + COMMAND echo ".globl _ggml_metallib_end" >> ${METALLIB_EMBED_ASM} + COMMAND echo "_ggml_metallib_end:" >> ${METALLIB_EMBED_ASM} + DEPENDS ggml-metal.metal ggml-common.h COMMENT "Generate assembly for embedded Metal library" ) - set(GGML_SOURCES_METAL ${GGML_SOURCES_METAL} ${EMBED_METALLIB_ASSEMBLY}) - endif() - - if (LLAMA_METAL_SHADER_DEBUG) - # custom command to do the following: - # xcrun -sdk macosx metal -fno-fast-math -c ggml-metal.metal -o ggml-metal.air - # xcrun -sdk macosx metallib ggml-metal.air -o default.metallib - # - # note: this is the only way I found to disable fast-math in Metal. it's ugly, but at least it works - # disabling fast math is needed in order to pass tests/test-backend-ops - # note: adding -fno-inline fixes the tests when using MTL_SHADER_VALIDATION=1 - # note: unfortunately, we have to call it default.metallib instead of ggml.metallib - # ref: https://github.com/ggerganov/whisper.cpp/issues/1720 - set(XC_FLAGS -fno-fast-math -fno-inline -g) - if (LLAMA_QKK_64) - set(XC_FLAGS ${XC_FLAGS} -DQK_K=64) + set(GGML_SOURCES_METAL ${GGML_SOURCES_METAL} ${METALLIB_EMBED_ASM}) + else() + if (LLAMA_METAL_SHADER_DEBUG) + # custom command to do the following: + # xcrun -sdk macosx metal -fno-fast-math -c ggml-metal.metal -o ggml-metal.air + # xcrun -sdk macosx metallib ggml-metal.air -o default.metallib + # + # note: this is the only way I found to disable fast-math in Metal. it's ugly, but at least it works + # disabling fast math is needed in order to pass tests/test-backend-ops + # note: adding -fno-inline fixes the tests when using MTL_SHADER_VALIDATION=1 + # note: unfortunately, we have to call it default.metallib instead of ggml.metallib + # ref: https://github.com/ggerganov/whisper.cpp/issues/1720 + set(XC_FLAGS -fno-fast-math -fno-inline -g) + else() + set(XC_FLAGS -O3) endif() add_custom_command( OUTPUT ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/default.metallib COMMAND xcrun -sdk macosx metal ${XC_FLAGS} -c ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/ggml-metal.metal -o ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/ggml-metal.air COMMAND xcrun -sdk macosx metallib ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/ggml-metal.air -o ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/default.metallib - DEPENDS ggml-metal.metal + COMMAND rm -f ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/ggml-metal.air + COMMAND rm -f ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/ggml-common.h + COMMAND rm -f ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/ggml-metal.metal + DEPENDS ggml-metal.metal ggml-common.h COMMENT "Compiling Metal kernels" - ) + ) add_custom_target( ggml-metal ALL DEPENDS ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/default.metallib - ) - endif() + ) + endif() # LLAMA_METAL_EMBED_LIBRARY set(LLAMA_EXTRA_LIBS ${LLAMA_EXTRA_LIBS} ${FOUNDATION_LIBRARY} diff --git a/Makefile b/Makefile index db9968efb..cb597b209 100644 --- a/Makefile +++ b/Makefile @@ -557,15 +557,16 @@ ggml-metal.o: ggml-metal.m ggml-metal.h $(CC) $(CFLAGS) -c $< -o $@ ifdef LLAMA_METAL_EMBED_LIBRARY -ggml-metal-embed.o: ggml-metal.metal +ggml-metal-embed.o: ggml-metal.metal ggml-common.h @echo "Embedding Metal library" + @sed -e '/#include "ggml-common.h"/r ggml-common.h' -e '/#include "ggml-common.h"/d' < ggml-metal.metal > ggml-metal-embed.metal $(eval TEMP_ASSEMBLY=$(shell mktemp)) - @echo ".section __DATA, __ggml_metallib" > $(TEMP_ASSEMBLY) - @echo ".globl _ggml_metallib_start" >> $(TEMP_ASSEMBLY) - @echo "_ggml_metallib_start:" >> $(TEMP_ASSEMBLY) - @echo ".incbin \"$<\"" >> $(TEMP_ASSEMBLY) - @echo ".globl _ggml_metallib_end" >> $(TEMP_ASSEMBLY) - @echo "_ggml_metallib_end:" >> $(TEMP_ASSEMBLY) + @echo ".section __DATA, __ggml_metallib" > $(TEMP_ASSEMBLY) + @echo ".globl _ggml_metallib_start" >> $(TEMP_ASSEMBLY) + @echo "_ggml_metallib_start:" >> $(TEMP_ASSEMBLY) + @echo ".incbin \"ggml-metal-embed.metal\"" >> $(TEMP_ASSEMBLY) + @echo ".globl _ggml_metallib_end" >> $(TEMP_ASSEMBLY) + @echo "_ggml_metallib_end:" >> $(TEMP_ASSEMBLY) @$(AS) $(TEMP_ASSEMBLY) -o $@ @rm -f ${TEMP_ASSEMBLY} endif diff --git a/ggml-metal.m b/ggml-metal.m index 3a5476c52..c3451a79b 100644 --- a/ggml-metal.m +++ b/ggml-metal.m @@ -280,6 +280,11 @@ static struct ggml_metal_context * ggml_metal_init(int n_cb) { id metal_library; // load library + // + // - first check if the library is embedded + // - then check if the library is in the bundle + // - if not found, load the source and compile it + // - if that fails, return NULL { NSBundle * bundle = nil; #ifdef SWIFT_PACKAGE @@ -287,12 +292,21 @@ static struct ggml_metal_context * ggml_metal_init(int n_cb) { #else bundle = [NSBundle bundleForClass:[GGMLMetalClass class]]; #endif + NSError * error = nil; - NSString * libPath = [bundle pathForResource:@"default" ofType:@"metallib"]; - if (libPath != nil) { + +#if GGML_METAL_EMBED_LIBRARY + const bool try_metallib = false; +#else + const bool try_metallib = true; +#endif + + NSString * path_lib = [bundle pathForResource:@"default" ofType:@"metallib"]; + if (try_metallib && path_lib != nil) { // pre-compiled library found - NSURL * libURL = [NSURL fileURLWithPath:libPath]; - GGML_METAL_LOG_INFO("%s: loading '%s'\n", __func__, [libPath UTF8String]); + NSURL * libURL = [NSURL fileURLWithPath:path_lib]; + GGML_METAL_LOG_INFO("%s: loading '%s'\n", __func__, [path_lib UTF8String]); + metal_library = [ctx->device newLibraryWithURL:libURL error:&error]; if (error) { GGML_METAL_LOG_ERROR("%s: error: %s\n", __func__, [[error description] UTF8String]); @@ -305,31 +319,34 @@ static struct ggml_metal_context * ggml_metal_init(int n_cb) { extern const char ggml_metallib_start[]; extern const char ggml_metallib_end[]; - NSString * src = [[NSString alloc] initWithBytes:ggml_metallib_start length:(ggml_metallib_end-ggml_metallib_start) encoding:NSUTF8StringEncoding]; + NSString * src = [[NSString alloc] initWithBytes:ggml_metallib_start length:(ggml_metallib_end-ggml_metallib_start) encoding:NSUTF8StringEncoding]; #else GGML_METAL_LOG_INFO("%s: default.metallib not found, loading from source\n", __func__); - NSString * sourcePath; - NSString * ggmlMetalPathResources = [[NSProcessInfo processInfo].environment objectForKey:@"GGML_METAL_PATH_RESOURCES"]; + NSString * path_source; + NSString * path_resource = [[NSProcessInfo processInfo].environment objectForKey:@"GGML_METAL_PATH_RESOURCES"]; - GGML_METAL_LOG_INFO("%s: GGML_METAL_PATH_RESOURCES = %s\n", __func__, ggmlMetalPathResources ? [ggmlMetalPathResources UTF8String] : "nil"); + GGML_METAL_LOG_INFO("%s: GGML_METAL_PATH_RESOURCES = %s\n", __func__, path_resource ? [path_resource UTF8String] : "nil"); - if (ggmlMetalPathResources) { - sourcePath = [ggmlMetalPathResources stringByAppendingPathComponent:@"ggml-metal.metal"]; + if (path_resource) { + path_source = [path_resource stringByAppendingPathComponent:@"ggml-metal.metal"]; } else { - sourcePath = [bundle pathForResource:@"ggml-metal" ofType:@"metal"]; + path_source = [bundle pathForResource:@"ggml-metal" ofType:@"metal"]; } - if (sourcePath == nil) { + + if (path_source == nil) { GGML_METAL_LOG_WARN("%s: error: could not use bundle path to find ggml-metal.metal, falling back to trying cwd\n", __func__); - sourcePath = @"ggml-metal.metal"; + path_source = @"ggml-metal.metal"; } - GGML_METAL_LOG_INFO("%s: loading '%s'\n", __func__, [sourcePath UTF8String]); - NSString * src = [NSString stringWithContentsOfFile:sourcePath encoding:NSUTF8StringEncoding error:&error]; + + GGML_METAL_LOG_INFO("%s: loading '%s'\n", __func__, [path_source UTF8String]); + + NSString * src = [NSString stringWithContentsOfFile:path_source encoding:NSUTF8StringEncoding error:&error]; if (error) { GGML_METAL_LOG_ERROR("%s: error: %s\n", __func__, [[error description] UTF8String]); return NULL; } -#endif +#endif // GGML_METAL_EMBED_LIBRARY @autoreleasepool { // dictionary of preprocessor macros diff --git a/ggml-metal.metal b/ggml-metal.metal index ebf2f5b47..63de56325 100644 --- a/ggml-metal.metal +++ b/ggml-metal.metal @@ -4,9 +4,6 @@ #include -#define GGML_COMMON_IMPL_METAL -#include "ggml-common.h" - using namespace metal; #define MAX(x, y) ((x) > (y) ? (x) : (y)) From 68265ebfc6a1bed022973ea0c3145be1450b7e70 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Thu, 14 Mar 2024 12:37:20 +0200 Subject: [PATCH 615/812] embedding : print all resulting embeddings (#899) --- examples/embedding/embedding.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/embedding/embedding.cpp b/examples/embedding/embedding.cpp index f390c4061..895469a31 100644 --- a/examples/embedding/embedding.cpp +++ b/examples/embedding/embedding.cpp @@ -167,9 +167,9 @@ int main(int argc, char ** argv) { float * out = emb + p * n_embd; batch_decode(ctx, batch, out, s, n_embd); - // print first 3 embeddings + // print the first part of the embeddings fprintf(stdout, "\n"); - for (int j = 0; j < std::min(3, n_prompts); j++) { + for (int j = 0; j < n_prompts; j++) { fprintf(stdout, "embedding %d: ", j); for (int i = 0; i < std::min(16, n_embd); i++) { fprintf(stdout, "%f ", emb[j * n_embd + i]); From 3fe8d7a17f84bd721cd4d8db35365da44b69f68b Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Thu, 14 Mar 2024 12:38:37 +0200 Subject: [PATCH 616/812] ggml : designate enum vals for integer types (#6050) --- ggml.h | 64 +++++++++++++++++++++++++++++----------------------------- 1 file changed, 32 insertions(+), 32 deletions(-) diff --git a/ggml.h b/ggml.h index 1171088a9..ab26c8f59 100644 --- a/ggml.h +++ b/ggml.h @@ -337,24 +337,24 @@ extern "C" { struct ggml_object; struct ggml_context; + // NOTE: always add types at the end of the enum to keep backward compatibility enum ggml_type { - GGML_TYPE_F32 = 0, - GGML_TYPE_F16 = 1, - GGML_TYPE_Q4_0 = 2, - GGML_TYPE_Q4_1 = 3, + GGML_TYPE_F32 = 0, + GGML_TYPE_F16 = 1, + GGML_TYPE_Q4_0 = 2, + GGML_TYPE_Q4_1 = 3, // GGML_TYPE_Q4_2 = 4, support has been removed - // GGML_TYPE_Q4_3 (5) support has been removed - GGML_TYPE_Q5_0 = 6, - GGML_TYPE_Q5_1 = 7, - GGML_TYPE_Q8_0 = 8, - GGML_TYPE_Q8_1 = 9, - // k-quantizations - GGML_TYPE_Q2_K = 10, - GGML_TYPE_Q3_K = 11, - GGML_TYPE_Q4_K = 12, - GGML_TYPE_Q5_K = 13, - GGML_TYPE_Q6_K = 14, - GGML_TYPE_Q8_K = 15, + // GGML_TYPE_Q4_3 = 5, support has been removed + GGML_TYPE_Q5_0 = 6, + GGML_TYPE_Q5_1 = 7, + GGML_TYPE_Q8_0 = 8, + GGML_TYPE_Q8_1 = 9, + GGML_TYPE_Q2_K = 10, + GGML_TYPE_Q3_K = 11, + GGML_TYPE_Q4_K = 12, + GGML_TYPE_Q5_K = 13, + GGML_TYPE_Q6_K = 14, + GGML_TYPE_Q8_K = 15, GGML_TYPE_IQ2_XXS = 16, GGML_TYPE_IQ2_XS = 17, GGML_TYPE_IQ3_XXS = 18, @@ -363,9 +363,9 @@ extern "C" { GGML_TYPE_IQ3_S = 21, GGML_TYPE_IQ2_S = 22, GGML_TYPE_IQ4_XS = 23, - GGML_TYPE_I8, - GGML_TYPE_I16, - GGML_TYPE_I32, + GGML_TYPE_I8 = 24, + GGML_TYPE_I16 = 25, + GGML_TYPE_I32 = 26, GGML_TYPE_COUNT, }; @@ -383,20 +383,20 @@ extern "C" { // model file types enum ggml_ftype { - GGML_FTYPE_UNKNOWN = -1, - GGML_FTYPE_ALL_F32 = 0, - GGML_FTYPE_MOSTLY_F16 = 1, // except 1d tensors - GGML_FTYPE_MOSTLY_Q4_0 = 2, // except 1d tensors - GGML_FTYPE_MOSTLY_Q4_1 = 3, // except 1d tensors + GGML_FTYPE_UNKNOWN = -1, + GGML_FTYPE_ALL_F32 = 0, + GGML_FTYPE_MOSTLY_F16 = 1, // except 1d tensors + GGML_FTYPE_MOSTLY_Q4_0 = 2, // except 1d tensors + GGML_FTYPE_MOSTLY_Q4_1 = 3, // except 1d tensors GGML_FTYPE_MOSTLY_Q4_1_SOME_F16 = 4, // tok_embeddings.weight and output.weight are F16 - GGML_FTYPE_MOSTLY_Q8_0 = 7, // except 1d tensors - GGML_FTYPE_MOSTLY_Q5_0 = 8, // except 1d tensors - GGML_FTYPE_MOSTLY_Q5_1 = 9, // except 1d tensors - GGML_FTYPE_MOSTLY_Q2_K = 10, // except 1d tensors - GGML_FTYPE_MOSTLY_Q3_K = 11, // except 1d tensors - GGML_FTYPE_MOSTLY_Q4_K = 12, // except 1d tensors - GGML_FTYPE_MOSTLY_Q5_K = 13, // except 1d tensors - GGML_FTYPE_MOSTLY_Q6_K = 14, // except 1d tensors + GGML_FTYPE_MOSTLY_Q8_0 = 7, // except 1d tensors + GGML_FTYPE_MOSTLY_Q5_0 = 8, // except 1d tensors + GGML_FTYPE_MOSTLY_Q5_1 = 9, // except 1d tensors + GGML_FTYPE_MOSTLY_Q2_K = 10, // except 1d tensors + GGML_FTYPE_MOSTLY_Q3_K = 11, // except 1d tensors + GGML_FTYPE_MOSTLY_Q4_K = 12, // except 1d tensors + GGML_FTYPE_MOSTLY_Q5_K = 13, // except 1d tensors + GGML_FTYPE_MOSTLY_Q6_K = 14, // except 1d tensors GGML_FTYPE_MOSTLY_IQ2_XXS = 15, // except 1d tensors GGML_FTYPE_MOSTLY_IQ2_XS = 16, // except 1d tensors GGML_FTYPE_MOSTLY_IQ3_XXS = 17, // except 1d tensors From 3ca23481dd309bd51cc31c73a4cc34f922cc372f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20=C4=8Cert=C3=ADk?= Date: Thu, 14 Mar 2024 04:40:14 -0600 Subject: [PATCH 617/812] gguf-py : add support for I8, I16 and I32 (#6045) * Refactor dtype handling to be extensible This code is equivalent as before, but now it is prepared to easily add more NumPy dtypes. * Add support for I8, I16 and I32 These types are allowed in the GGUF specification. * Add support for I8, I16 and I32 to gguf_writer * Add support for I8, I16, I32 to gguf_reader --- gguf-py/gguf/constants.py | 6 ++++++ gguf-py/gguf/gguf_reader.py | 9 +++++++++ gguf-py/gguf/gguf_writer.py | 16 ++++++++++++---- 3 files changed, 27 insertions(+), 4 deletions(-) diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py index b23badb10..99f71f0a1 100644 --- a/gguf-py/gguf/constants.py +++ b/gguf-py/gguf/constants.py @@ -661,6 +661,9 @@ class GGMLQuantizationType(IntEnum): IQ3_S = 21 IQ2_S = 22 IQ4_XS = 23 + I8 = 24 + I16 = 25 + I32 = 26 class GGUFEndian(IntEnum): @@ -727,6 +730,9 @@ GGML_QUANT_SIZES = { GGMLQuantizationType.IQ3_S: (256, 2 + QK_K // 4 + QK_K // 8 + QK_K // 32 + 4), GGMLQuantizationType.IQ2_S: (256, 2 + QK_K // 4 + QK_K // 16), GGMLQuantizationType.IQ4_XS: (256, 2 + 2 + QK_K // 2 + QK_K // 64), + GGMLQuantizationType.I8: (1, 1), + GGMLQuantizationType.I16: (1, 2), + GGMLQuantizationType.I32: (1, 4), } diff --git a/gguf-py/gguf/gguf_reader.py b/gguf-py/gguf/gguf_reader.py index 5b6d4ba6b..1c10f5753 100644 --- a/gguf-py/gguf/gguf_reader.py +++ b/gguf-py/gguf/gguf_reader.py @@ -248,6 +248,15 @@ class GGUFReader: elif ggml_type == GGMLQuantizationType.F16: item_count = n_elems item_type = np.float16 + elif ggml_type == GGMLQuantizationType.I8: + item_count = n_elems + item_type = np.int8 + elif ggml_type == GGMLQuantizationType.I16: + item_count = n_elems + item_type = np.int16 + elif ggml_type == GGMLQuantizationType.I32: + item_count = n_elems + item_type = np.int32 else: item_count = n_bytes item_type = np.uint8 diff --git a/gguf-py/gguf/gguf_writer.py b/gguf-py/gguf/gguf_writer.py index e49c5db68..9c1eeac31 100644 --- a/gguf-py/gguf/gguf_writer.py +++ b/gguf-py/gguf/gguf_writer.py @@ -196,9 +196,6 @@ class GGUFWriter: if self.state is not WriterState.EMPTY: raise ValueError(f'Expected output file to be empty, got {self.state}') - if raw_dtype is None and tensor_dtype not in (np.float32, np.float16): - raise ValueError("Only F32 and F16 tensors are supported for now") - encoded_name = name.encode("utf8") self.ti_data += self._pack("Q", len(encoded_name)) self.ti_data += encoded_name @@ -207,7 +204,18 @@ class GGUFWriter: for i in range(n_dims): self.ti_data += self._pack("Q", tensor_shape[n_dims - 1 - i]) if raw_dtype is None: - dtype = GGMLQuantizationType.F32 if tensor_dtype == np.float32 else GGMLQuantizationType.F16 + if tensor_shape == np.float32: + dtype = GGMLQuantizationType.F32 + elif tensor_dtype == np.float16: + dtype = GGMLQuantizationType.F16 + elif tensor_dtype == np.int8: + dtype = GGMLQuantizationType.I8 + elif tensor_dtype == np.int16: + dtype = GGMLQuantizationType.I16 + elif tensor_dtype == np.int32: + dtype = GGMLQuantizationType.I32 + else: + raise ValueError("Only F32, F16, I8, I16, I32 tensors are supported for now") else: dtype = raw_dtype self.ti_data += self._pack("I", dtype) From 2c4fb69246834503db7b78bcbedcef506bbc60c4 Mon Sep 17 00:00:00 2001 From: Michael Podvitskiy Date: Thu, 14 Mar 2024 11:56:48 +0100 Subject: [PATCH 618/812] llama : optimize defrag moves + fix fragmentation calculation (#6037) * attempt to reduce the impact of a worst-case scenario * fragmentation calculation fix * Update llama.cpp --------- Co-authored-by: Georgi Gerganov --- llama.cpp | 30 +++++++++++++++++++----------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/llama.cpp b/llama.cpp index 38e7036a7..ff467c575 100644 --- a/llama.cpp +++ b/llama.cpp @@ -9036,8 +9036,8 @@ static int llama_decode_internal( //llama_synchronize(&lctx); // decide if we need to defrag the kv cache - if (cparams.defrag_thold >= 0.0f) { - const float fragmentation = kv_self.n >= 128 ? 1.0f - float(kv_self.used + n_tokens_all)/float(kv_self.n) : 0.0f; + if (cparams.causal_attn && cparams.defrag_thold >= 0.0f) { + const float fragmentation = kv_self.n >= 128 ? 1.0f - float(kv_self.used)/float(kv_self.n) : 0.0f; // queue defragmentation for next llama_kv_cache_update if (fragmentation > cparams.defrag_thold) { @@ -9069,6 +9069,11 @@ static void llama_kv_cache_defrag_internal(struct llama_context & lctx) { // number of cells moved uint32_t n_moves = 0; + // each move requires 6*n_layer tensors (see build_defrag) + // - source view, destination view, copy operation + // - x2 for keys and values + const uint32_t max_moves = LLAMA_MAX_NODES/(6*n_layer); + // determine which KV cells to move where // // cell i moves to ids[i] @@ -9095,15 +9100,6 @@ static void llama_kv_cache_defrag_internal(struct llama_context & lctx) { nh++; } - // each move requires 6*n_layer tensors (see build_defrag) - // - source view, destination view, copy operation - // - x2 for keys and values - // - if (6*(n_moves + nh)*n_layer >= LLAMA_MAX_NODES) { - // the graph is too big, we cannot move more cells - break; - } - uint32_t nf = 0; uint32_t is = n_kv - 1; @@ -9133,11 +9129,19 @@ static void llama_kv_cache_defrag_internal(struct llama_context & lctx) { // are we moving a continuous block of memory? bool cont = false; + // should we stop searching for the next move? + bool stop = false; + // go back and move the nf cells to the hole for (; i1 < n_kv; ++i1) { auto & cell1 = kv_self.cells[i1]; if (cell1.is_empty() || ids[i1] != n_kv) { + if (n_moves == max_moves) { + stop = true; + break; + } + cont = false; continue; } @@ -9164,6 +9168,10 @@ static void llama_kv_cache_defrag_internal(struct llama_context & lctx) { } } + if (stop || n_moves == max_moves) { + break; + } + //LLAMA_LOG_INFO("(tmp log) KV defrag: move [%u, %u) to [%u, %u)\n", is, i1 + 1, i0, i0 + nh); i0 += nh - 1; From a44bc969e4cd62ca9f4332e17fe3c51f2093e7c6 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Thu, 14 Mar 2024 13:13:06 +0200 Subject: [PATCH 619/812] llama : fix typo --- llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llama.cpp b/llama.cpp index ff467c575..eb48b1e90 100644 --- a/llama.cpp +++ b/llama.cpp @@ -3932,7 +3932,7 @@ static void llm_load_print_meta(llama_model_loader & ml, llama_model & model) { LLAMA_LOG_INFO("%s: n_ff = %u\n", __func__, hparams.n_ff); LLAMA_LOG_INFO("%s: n_expert = %u\n", __func__, hparams.n_expert); LLAMA_LOG_INFO("%s: n_expert_used = %u\n", __func__, hparams.n_expert_used); - LLAMA_LOG_INFO("%s: causal attm = %d\n", __func__, hparams.causal_attn); + LLAMA_LOG_INFO("%s: causal attn = %d\n", __func__, hparams.causal_attn); LLAMA_LOG_INFO("%s: pooling type = %d\n", __func__, hparams.pooling_type); LLAMA_LOG_INFO("%s: rope type = %d\n", __func__, hparams.rope_type); LLAMA_LOG_INFO("%s: rope scaling = %s\n", __func__, rope_scaling_type); From 43241adf22e8231ffaf3827d2c9310cc0ffd5ac5 Mon Sep 17 00:00:00 2001 From: Pierrick Hymbert Date: Thu, 14 Mar 2024 12:15:39 +0100 Subject: [PATCH 620/812] server: disable debug release type sanitizer, simplify trigger (#6047) - increase time out for server - do not fail fast --- .github/workflows/server.yml | 17 ++++++++--------- examples/server/tests/features/steps/steps.py | 9 ++++++++- 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/.github/workflows/server.yml b/.github/workflows/server.yml index e385e03f3..5e38b3547 100644 --- a/.github/workflows/server.yml +++ b/.github/workflows/server.yml @@ -25,17 +25,14 @@ jobs: strategy: matrix: sanitizer: [ADDRESS, THREAD, UNDEFINED] - build_type: [Debug, Release] + build_type: [Debug] include: - build_type: Release sanitizer: "" - exclude: - - build_type: Release - sanitizer: ADDRESS - - build_type: Release + - build_type: Debug sanitizer: THREAD - - build_type: Release - sanitizer: UNDEFINED + disabled_on_pr: true + fail-fast: false # While -DLLAMA_SANITIZE_THREAD=ON is broken container: image: ubuntu:latest @@ -81,13 +78,14 @@ jobs: - name: Tests id: server_integration_tests + if: ${{ !matrix.disabled_on_pr || !github.event.pull_request }} run: | cd examples/server/tests PORT=8888 ./tests.sh - name: Slow tests id: server_integration_tests_slow - if: ${{ github.event.schedule != '' && matrix.build_type == 'Release' || github.event.inputs.slow_tests == 'true' }} + if: ${{ (github.event.schedule || github.event.inputs.slow_tests == 'true') && matrix.build_type == 'Release' }} run: | cd examples/server/tests PORT=8888 ./tests.sh --stop --no-skipped --no-capture --tags slow @@ -124,13 +122,14 @@ jobs: - name: Tests id: server_integration_tests + if: ${{ !matrix.disabled_on_pr || !github.event.pull_request }} run: | cd examples/server/tests behave.exe --summary --stop --no-capture --exclude 'issues|wrong_usages|passkey' --tags llama.cpp - name: Slow tests id: server_integration_tests_slow - if: ${{ github.event.schedule != '' || github.event.inputs.slow_tests == 'true' }} + if: ${{ (github.event.schedule || github.event.inputs.slow_tests == 'true') && matrix.build_type == 'Release' }} run: | cd examples/server/tests behave.exe --stop --no-skipped --no-capture --tags slow diff --git a/examples/server/tests/features/steps/steps.py b/examples/server/tests/features/steps/steps.py index cfa9f96ec..a59a52d21 100644 --- a/examples/server/tests/features/steps/steps.py +++ b/examples/server/tests/features/steps/steps.py @@ -119,6 +119,10 @@ def step_server_metrics(context): def step_start_server(context): start_server_background(context) attempts = 0 + max_attempts = 20 + if 'GITHUB_ACTIONS' in os.environ: + max_attempts *= 2 + while True: with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as sock: result = sock.connect_ex((context.server_fqdn, context.server_port)) @@ -126,7 +130,7 @@ def step_start_server(context): print("\x1b[33;46mserver started!\x1b[0m") return attempts += 1 - if attempts > 20: + if attempts > max_attempts: assert False, "server not started" print(f"waiting for server to start, connect error code = {result}...") time.sleep(0.1) @@ -943,6 +947,9 @@ async def wait_for_health_status(context, print(f"Starting checking for health for expected_health_status={expected_health_status}\n") interval = 0.5 counter = 0 + if 'GITHUB_ACTIONS' in os.environ: + timeout *= 2 + async with aiohttp.ClientSession() as session: while True: async with await session.get(f'{base_url}/health', params=params) as health_response: From 15a333260ab637a040ed0864c206a2ceaf806bb8 Mon Sep 17 00:00:00 2001 From: Jian Liao Date: Thu, 14 Mar 2024 04:18:23 -0700 Subject: [PATCH 621/812] readme : improve readme for Llava-1.6 example (#6044) Co-authored-by: Jian Liao --- examples/llava/README.md | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/examples/llava/README.md b/examples/llava/README.md index 35e6d9e5d..67cb0f22b 100644 --- a/examples/llava/README.md +++ b/examples/llava/README.md @@ -63,12 +63,20 @@ Now both the LLaMA part and the image encoder is in the `llava-v1.5-7b` director ```console git clone https://huggingface.co/liuhaotian/llava-v1.6-vicuna-7b ``` -2) Use `llava-surgery-v2.py` which also supports llava-1.5 variants pytorch as well as safetensor models: + +2) Install the required Python packages: + +```sh +pip install -r examples/llava/requirements.txt +``` + +3) Use `llava-surgery-v2.py` which also supports llava-1.5 variants pytorch as well as safetensor models: ```console python examples/llava/llava-surgery-v2.py -C -m ../llava-v1.6-vicuna-7b/ ``` - you will find a llava.projector and a llava.clip file in your model directory -3) Copy the llava.clip file into a subdirectory (like vit), rename it to pytorch_model.bin and add a fitting vit configuration to the directory: + +4) Copy the llava.clip file into a subdirectory (like vit), rename it to pytorch_model.bin and add a fitting vit configuration to the directory: ```console mkdir vit cp ../llava-v1.6-vicuna-7b/llava.clip vit/pytorch_model.bin @@ -76,18 +84,18 @@ cp ../llava-v1.6-vicuna-7b/llava.projector vit/ curl -s -q https://huggingface.co/cmp-nct/llava-1.6-gguf/raw/main/config_vit.json -o vit/config.json ``` -4) Create the visual gguf model: +5) Create the visual gguf model: ```console python ./examples/llava/convert-image-encoder-to-gguf.py -m vit --llava-projector vit/llava.projector --output-dir vit --clip-model-is-vision ``` - This is similar to llava-1.5, the difference is that we tell the encoder that we are working with the pure vision model part of CLIP -5) Then convert the model to gguf format: +6) Then convert the model to gguf format: ```console python ./convert.py ../llava-v1.6-vicuna-7b/ --skip-unknown ``` -6) And finally we can run the llava-cli using the 1.6 model version: +7) And finally we can run the llava-cli using the 1.6 model version: ```console ./llava-cli -m ../llava-v1.6-vicuna-7b/ggml-model-f16.gguf --mmproj vit/mmproj-model-f16.gguf --image some-image.jpg -c 4096 ``` From 77178eedc83d49f31bf757d8e12315d76460be78 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Thu, 14 Mar 2024 13:32:14 +0200 Subject: [PATCH 622/812] gguf-py : fix dtype check (#6045) --- gguf-py/gguf/gguf_writer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gguf-py/gguf/gguf_writer.py b/gguf-py/gguf/gguf_writer.py index 9c1eeac31..4d389be95 100644 --- a/gguf-py/gguf/gguf_writer.py +++ b/gguf-py/gguf/gguf_writer.py @@ -204,7 +204,7 @@ class GGUFWriter: for i in range(n_dims): self.ti_data += self._pack("Q", tensor_shape[n_dims - 1 - i]) if raw_dtype is None: - if tensor_shape == np.float32: + if tensor_dtype == np.float32: dtype = GGMLQuantizationType.F32 elif tensor_dtype == np.float16: dtype = GGMLQuantizationType.F16 From 044ec4b2a567f649459ccd20af2f387c784faa51 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Thu, 14 Mar 2024 15:14:14 +0200 Subject: [PATCH 623/812] embedding : add EOS token if not present (#899) --- examples/embedding/embedding.cpp | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/examples/embedding/embedding.cpp b/examples/embedding/embedding.cpp index 895469a31..cbf9aa2b5 100644 --- a/examples/embedding/embedding.cpp +++ b/examples/embedding/embedding.cpp @@ -112,13 +112,20 @@ int main(int argc, char ** argv) { // tokenize the prompts and trim std::vector> inputs; for (const auto & prompt : prompts) { - auto inp = ::llama_tokenize(ctx, prompt, true); + auto inp = ::llama_tokenize(ctx, prompt, true, false); if (inp.size() > n_batch) { inp.resize(n_batch); } inputs.push_back(inp); } + // add eos if not present + for (auto & inp : inputs) { + if (inp.empty() || inp.back() != llama_token_eos(model)) { + inp.push_back(llama_token_eos(model)); + } + } + // tokenization stats if (params.verbose_prompt) { for (int i = 0; i < (int) inputs.size(); i++) { @@ -172,7 +179,7 @@ int main(int argc, char ** argv) { for (int j = 0; j < n_prompts; j++) { fprintf(stdout, "embedding %d: ", j); for (int i = 0; i < std::min(16, n_embd); i++) { - fprintf(stdout, "%f ", emb[j * n_embd + i]); + fprintf(stdout, "%9.6f ", emb[j * n_embd + i]); } fprintf(stdout, "\n"); } From 69ff61397d2b7b550dcdda4a35b35128892408b0 Mon Sep 17 00:00:00 2001 From: Michael Podvitskiy Date: Thu, 14 Mar 2024 17:21:56 +0100 Subject: [PATCH 624/812] llama : support models without vocabulary (#5798) * additional methods to read model and ctx parameters * vocab size as a part of a model metadata * models without vocabulary, convert.py part * models without vocabulary, llama.cpp part * PR clean up * converter scrypt fixes * llama_vocab_type update (renamed the new key) * pr review fixes * revert function renaming * one more NoVocab assert --- convert.py | 126 +++++++++++++++++++++--------------- gguf-py/gguf/constants.py | 2 + gguf-py/gguf/gguf_writer.py | 3 + llama.cpp | 92 +++++++++++++++++--------- llama.h | 7 +- 5 files changed, 142 insertions(+), 88 deletions(-) diff --git a/convert.py b/convert.py index c15f8c47e..161430f3e 100755 --- a/convert.py +++ b/convert.py @@ -332,6 +332,9 @@ class Params: # class BpeVocab: + tokenizer_model = "gpt2" + name = "bpe" + def __init__(self, fname_tokenizer: Path, fname_added_tokens: Path | None) -> None: self.bpe_tokenizer = json.loads(open(str(fname_tokenizer), encoding="utf-8").read()) if isinstance(self.bpe_tokenizer.get('model'), dict): @@ -390,6 +393,9 @@ class BpeVocab: class SentencePieceVocab: + tokenizer_model = "llama" + name = "spm" + def __init__(self, fname_tokenizer: Path, fname_added_tokens: Path | None) -> None: self.sentencepiece_tokenizer = SentencePieceProcessor(str(fname_tokenizer)) added_tokens: dict[str, int] @@ -453,6 +459,9 @@ class SentencePieceVocab: class HfVocab: + tokenizer_model = "llama" + name = "hfft" + def __init__(self, fname_tokenizer: Path, fname_added_tokens: Path | None = None) -> None: try: from transformers import AutoTokenizer @@ -553,7 +562,15 @@ class HfVocab: return f"" -Vocab: TypeAlias = "BpeVocab | SentencePieceVocab | HfVocab" +class NoVocab: + tokenizer_model = "no_vocab" + name = "no_vocab" + + def __repr__(self) -> str: + return "" + + +Vocab: TypeAlias = "BpeVocab | SentencePieceVocab | HfVocab | NoVocab" # @@ -935,8 +952,10 @@ def check_vocab_size(params: Params, vocab: Vocab, pad_vocab: bool = False) -> N # Handle special case where the model's vocab size is not set if params.n_vocab == -1: raise ValueError( - f"The model's vocab size is set to -1 in params.json. Please update it manually. Maybe {vocab.vocab_size}?" + f"The model's vocab size is set to -1 in params.json. Please update it manually.{f' Maybe {vocab.vocab_size}?' if hasattr(vocab, 'vocab_size') else ''}" ) + if isinstance(vocab, NoVocab): + return # model has no vocab # Check for a vocab size mismatch if params.n_vocab == vocab.vocab_size: @@ -977,6 +996,7 @@ class OutputFile: name = str(params.path_model.parent).split('/')[-1] self.gguf.add_name (name) + self.gguf.add_vocab_size (params.n_vocab) self.gguf.add_context_length (params.n_ctx) self.gguf.add_embedding_length (params.n_embd) self.gguf.add_block_count (params.n_layer) @@ -1013,21 +1033,9 @@ class OutputFile: if params.ftype is not None: self.gguf.add_file_type(params.ftype) - def handle_tokenizer_model(self, vocab: Vocab) -> str: - # Map the vocab types to the supported tokenizer models - tokenizer_model = { - SentencePieceVocab: "llama", - HfVocab: "llama", - BpeVocab: "gpt2", - }.get(type(vocab)) - - # Block if vocab type is not predefined - if tokenizer_model is None: - raise ValueError("Unknown vocab type: Not supported") - - return tokenizer_model - def extract_vocabulary_from_model(self, vocab: Vocab) -> tuple[list[bytes], list[float], list[gguf.TokenType]]: + assert not isinstance(vocab, NoVocab) + tokens = [] scores = [] toktypes = [] @@ -1043,11 +1051,8 @@ class OutputFile: return tokens, scores, toktypes def add_meta_vocab(self, vocab: Vocab) -> None: - # Handle the tokenizer model - tokenizer_model = self.handle_tokenizer_model(vocab) - # Ensure that tokenizer_model is added to the GGUF model - self.gguf.add_tokenizer_model(tokenizer_model) + self.gguf.add_tokenizer_model(vocab.tokenizer_model) # Extract model vocabulary for model conversion tokens, scores, toktypes = self.extract_vocabulary_from_model(vocab) @@ -1074,6 +1079,26 @@ class OutputFile: def write_tensor_info(self) -> None: self.gguf.write_ti_data_to_file() + def write_tensor_data(self, ftype: GGMLFileType, model: LazyModel, concurrency: int) -> None: + ndarrays_inner = bounded_parallel_map(OutputFile.do_item, model.items(), concurrency=concurrency) + if ftype == GGMLFileType.MostlyQ8_0: + ndarrays = bounded_parallel_map( + OutputFile.maybe_do_quantize, ndarrays_inner, concurrency=concurrency, max_workers=concurrency, + use_processpool_executor=True, + ) + else: + ndarrays = map(OutputFile.maybe_do_quantize, ndarrays_inner) + + start = time.time() + for i, ((name, lazy_tensor), ndarray) in enumerate(zip(model.items(), ndarrays)): + elapsed = time.time() - start + size = ' x '.join(f"{dim:6d}" for dim in lazy_tensor.shape) + padi = len(str(len(model))) + print( + f"[{i + 1:{padi}d}/{len(model)}] Writing tensor {name:38s} | size {size:16} | type {lazy_tensor.data_type.name:4} | T+{int(elapsed):4}" + ) + self.gguf.write_tensor_data(ndarray) + def close(self) -> None: self.gguf.close() @@ -1082,7 +1107,7 @@ class OutputFile: fname_out: Path, params: Params, vocab: Vocab, svocab: gguf.SpecialVocab, endianess: gguf.GGUFEndian = gguf.GGUFEndian.LITTLE, pad_vocab: bool = False, ) -> None: - check_vocab_size(params, vocab, pad_vocab = pad_vocab) + check_vocab_size(params, vocab, pad_vocab=pad_vocab) of = OutputFile(fname_out, endianess=endianess) @@ -1120,8 +1145,11 @@ class OutputFile: # meta data of.add_meta_arch(params) - of.add_meta_vocab(vocab) - of.add_meta_special_vocab(svocab) + if isinstance(vocab, NoVocab): + of.gguf.add_tokenizer_model(vocab.tokenizer_model) + else: + of.add_meta_vocab(vocab) + of.add_meta_special_vocab(svocab) # tensor info for name, lazy_tensor in model.items(): @@ -1131,24 +1159,7 @@ class OutputFile: of.write_tensor_info() # tensor data - ndarrays_inner = bounded_parallel_map(OutputFile.do_item, model.items(), concurrency = concurrency) - if ftype == GGMLFileType.MostlyQ8_0: - ndarrays = bounded_parallel_map( - OutputFile.maybe_do_quantize, ndarrays_inner, concurrency=concurrency, max_workers=concurrency, - use_processpool_executor=True, - ) - else: - ndarrays = map(OutputFile.maybe_do_quantize, ndarrays_inner) - - start = time.time() - for i, ((name, lazy_tensor), ndarray) in enumerate(zip(model.items(), ndarrays)): - elapsed = time.time() - start - size = ' x '.join(f"{dim:6d}" for dim in lazy_tensor.shape) - padi = len(str(len(model))) - print( - f"[{i+1:{padi}d}/{len(model)}] Writing tensor {name:38s} | size {size:16} | type {lazy_tensor.data_type.name:4} | T+{int(elapsed):4}" - ) - of.gguf.write_tensor_data(ndarray) + of.write_tensor_data(ftype, model, concurrency) of.close() @@ -1309,8 +1320,8 @@ class VocabFactory: return vtype, path raise FileNotFoundError(f"Could not find any of {[self._FILES[vt] for vt in vocab_types]}") - def _create_special_vocab(self, vocab: Vocab, vocabtype: str, model_parent_path: Path) -> gguf.SpecialVocab: - load_merges = vocabtype == "bpe" + def _create_special_vocab(self, vocab: Vocab, model_parent_path: Path) -> gguf.SpecialVocab: + load_merges = vocab.name == "bpe" n_vocab = vocab.vocab_size if hasattr(vocab, "vocab_size") else None return gguf.SpecialVocab( model_parent_path, @@ -1319,30 +1330,34 @@ class VocabFactory: n_vocab=n_vocab, ) - def load_vocab(self, vocab_types: list[str], model_parent_path: Path) -> tuple[Vocab, gguf.SpecialVocab]: + def _create_vocab_by_path(self, vocab_types: list[str]) -> Vocab: vocab_type, path = self._select_file(vocab_types) print(f"Loading vocab file {path!r}, type {vocab_type!r}") added_tokens_path = path.parent / "added_tokens.json" - vocab: Vocab if vocab_type == "bpe": - vocab = BpeVocab( + return BpeVocab( path, added_tokens_path if added_tokens_path.exists() else None ) - elif vocab_type == "spm": - vocab = SentencePieceVocab( + if vocab_type == "spm": + return SentencePieceVocab( path, added_tokens_path if added_tokens_path.exists() else None ) - elif vocab_type == "hfft": - vocab = HfVocab( + if vocab_type == "hfft": + return HfVocab( path.parent, added_tokens_path if added_tokens_path.exists() else None ) + raise ValueError(vocab_type) + + def load_vocab(self, vocab_types: list[str], model_parent_path: Path) -> tuple[Vocab, gguf.SpecialVocab]: + vocab: Vocab + if len(vocab_types) == 1 and "no_vocab" in vocab_types: + vocab = NoVocab() else: - raise ValueError(vocab_type) + vocab = self._create_vocab_by_path(vocab_types) # FIXME: Respect --vocab-dir? special_vocab = self._create_special_vocab( vocab, - vocab_type, model_parent_path, ) return vocab, special_vocab @@ -1380,6 +1395,7 @@ def main(args_in: list[str] | None = None) -> None: parser.add_argument("--dump", action="store_true", help="don't convert, just show what's in the model") parser.add_argument("--dump-single", action="store_true", help="don't convert, just show what's in a single model file") parser.add_argument("--vocab-only", action="store_true", help="extract only the vocab") + parser.add_argument("--no-vocab", action="store_true", help="store model without the vocab") parser.add_argument("--outtype", choices=output_choices, help="output format - note: q8_0 may be very slow (default: f16 or f32 based on input)") parser.add_argument("--vocab-dir", type=Path, help="directory containing tokenizer.model, if separate from model file") parser.add_argument("--vocab-type", help="vocab types to try in order, choose from 'spm', 'bpe', 'hfft' (default: spm,hfft)", default="spm,hfft") @@ -1392,6 +1408,10 @@ def main(args_in: list[str] | None = None) -> None: parser.add_argument("--skip-unknown", action="store_true", help="skip unknown tensor names instead of failing") args = parser.parse_args(args_in) + if args.no_vocab: + if args.vocab_only: + raise ValueError("no need to specify --vocab-only if using --no-vocab") + args.vocab_type = "no_vocab" if args.dump_single: model_plus = lazy_load_file(args.model) @@ -1442,7 +1462,7 @@ def main(args_in: list[str] | None = None) -> None: print(f"Wrote {outfile}") return - if model_plus.vocab is not None and args.vocab_dir is None: + if model_plus.vocab is not None and args.vocab_dir is None and not args.no_vocab: vocab = model_plus.vocab print(f"Vocab info: {vocab}") diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py index 99f71f0a1..2d7cf16c1 100644 --- a/gguf-py/gguf/constants.py +++ b/gguf-py/gguf/constants.py @@ -32,6 +32,7 @@ class Keys: FILE_TYPE = "general.file_type" class LLM: + VOCAB_SIZE = "{arch}.vocab_size" CONTEXT_LENGTH = "{arch}.context_length" EMBEDDING_LENGTH = "{arch}.embedding_length" BLOCK_COUNT = "{arch}.block_count" @@ -752,6 +753,7 @@ KEY_GENERAL_SOURCE_HF_REPO = Keys.General.SOURCE_HF_REPO KEY_GENERAL_FILE_TYPE = Keys.General.FILE_TYPE # LLM +KEY_VOCAB_SIZE = Keys.LLM.VOCAB_SIZE KEY_CONTEXT_LENGTH = Keys.LLM.CONTEXT_LENGTH KEY_EMBEDDING_LENGTH = Keys.LLM.EMBEDDING_LENGTH KEY_BLOCK_COUNT = Keys.LLM.BLOCK_COUNT diff --git a/gguf-py/gguf/gguf_writer.py b/gguf-py/gguf/gguf_writer.py index 4d389be95..81b2eb884 100644 --- a/gguf-py/gguf/gguf_writer.py +++ b/gguf-py/gguf/gguf_writer.py @@ -321,6 +321,9 @@ class GGUFWriter: self.data_alignment = alignment self.add_uint32(Keys.General.ALIGNMENT, alignment) + def add_vocab_size(self, size: int) -> None: + self.add_uint32(Keys.LLM.VOCAB_SIZE.format(arch=self.arch), size) + def add_context_length(self, length: int) -> None: self.add_uint32(Keys.LLM.CONTEXT_LENGTH.format(arch=self.arch), length) diff --git a/llama.cpp b/llama.cpp index eb48b1e90..10fd53469 100644 --- a/llama.cpp +++ b/llama.cpp @@ -258,6 +258,7 @@ enum llm_kv { LLM_KV_GENERAL_SOURCE_URL, LLM_KV_GENERAL_SOURCE_HF_REPO, + LLM_KV_VOCAB_SIZE, LLM_KV_CONTEXT_LENGTH, LLM_KV_EMBEDDING_LENGTH, LLM_KV_BLOCK_COUNT, @@ -321,6 +322,7 @@ static const std::map LLM_KV_NAMES = { { LLM_KV_GENERAL_SOURCE_URL, "general.source.url" }, { LLM_KV_GENERAL_SOURCE_HF_REPO, "general.source.huggingface.repository" }, + { LLM_KV_VOCAB_SIZE, "%s.vocab_size" }, { LLM_KV_CONTEXT_LENGTH, "%s.context_length" }, { LLM_KV_EMBEDDING_LENGTH, "%s.embedding_length" }, { LLM_KV_BLOCK_COUNT, "%s.block_count" }, @@ -3242,10 +3244,11 @@ static const char * llama_model_type_name(e_model type) { static const char * llama_model_vocab_type_name(enum llama_vocab_type type){ switch (type) { - case LLAMA_VOCAB_TYPE_SPM: return "SPM"; - case LLAMA_VOCAB_TYPE_BPE: return "BPE"; - case LLAMA_VOCAB_TYPE_WPM: return "WPM"; - default: return "unknown"; + case LLAMA_VOCAB_TYPE_NONE: return "no vocab"; + case LLAMA_VOCAB_TYPE_SPM: return "SPM"; + case LLAMA_VOCAB_TYPE_BPE: return "BPE"; + case LLAMA_VOCAB_TYPE_WPM: return "WPM"; + default: return "unknown"; } } @@ -3277,14 +3280,14 @@ static void llm_load_hparams( ml.get_key(LLM_KV_GENERAL_NAME, model.name, false); // get hparams kv - ml.get_arr_n(LLM_KV_TOKENIZER_LIST, hparams.n_vocab); - ml.get_key (LLM_KV_CONTEXT_LENGTH, hparams.n_ctx_train); - ml.get_key (LLM_KV_EMBEDDING_LENGTH, hparams.n_embd); - ml.get_key (LLM_KV_FEED_FORWARD_LENGTH, hparams.n_ff); - ml.get_key (LLM_KV_ATTENTION_HEAD_COUNT, hparams.n_head); - ml.get_key (LLM_KV_BLOCK_COUNT, hparams.n_layer); - ml.get_key (LLM_KV_EXPERT_COUNT, hparams.n_expert, false); - ml.get_key (LLM_KV_EXPERT_USED_COUNT, hparams.n_expert_used, false); + ml.get_key(LLM_KV_VOCAB_SIZE, hparams.n_vocab, false) || ml.get_arr_n(LLM_KV_TOKENIZER_LIST, hparams.n_vocab); + ml.get_key(LLM_KV_CONTEXT_LENGTH, hparams.n_ctx_train); + ml.get_key(LLM_KV_EMBEDDING_LENGTH, hparams.n_embd); + ml.get_key(LLM_KV_FEED_FORWARD_LENGTH, hparams.n_ff); + ml.get_key(LLM_KV_ATTENTION_HEAD_COUNT, hparams.n_head); + ml.get_key(LLM_KV_BLOCK_COUNT, hparams.n_layer); + ml.get_key(LLM_KV_EXPERT_COUNT, hparams.n_expert, false); + ml.get_key(LLM_KV_EXPERT_USED_COUNT, hparams.n_expert_used, false); GGML_ASSERT(hparams.n_expert <= LLAMA_MAX_EXPERTS); GGML_ASSERT(hparams.n_expert_used <= hparams.n_expert); @@ -3645,30 +3648,25 @@ static void llm_load_vocab( const auto kv = LLM_KV(model.arch); - const int token_idx = gguf_find_key(ctx, kv(LLM_KV_TOKENIZER_LIST).c_str()); - if (token_idx == -1) { - throw std::runtime_error("cannot find tokenizer vocab in model file\n"); - } - - const float * scores = nullptr; - const int score_idx = gguf_find_key(ctx, kv(LLM_KV_TOKENIZER_SCORES).c_str()); - if (score_idx != -1) { - scores = (const float * ) gguf_get_arr_data(ctx, score_idx); - } - - const int * toktypes = nullptr; - const int toktype_idx = gguf_find_key(ctx, kv(LLM_KV_TOKENIZER_TOKEN_TYPE).c_str()); - if (toktype_idx != -1) { - toktypes = (const int * ) gguf_get_arr_data(ctx, toktype_idx); - } - // determine vocab type { std::string tokenizer_name; ml.get_key(LLM_KV_TOKENIZER_MODEL, tokenizer_name); - if (tokenizer_name == "llama") { + if (tokenizer_name == "no_vocab") { + vocab.type = LLAMA_VOCAB_TYPE_NONE; + + // default special tokens + vocab.special_bos_id = -1; + vocab.special_eos_id = -1; + vocab.special_unk_id = -1; + vocab.special_sep_id = -1; + vocab.special_pad_id = -1; + vocab.linefeed_id = -1; + + return; + } else if (tokenizer_name == "llama") { vocab.type = LLAMA_VOCAB_TYPE_SPM; // default special tokens @@ -3734,6 +3732,23 @@ static void llm_load_vocab( } } + const int token_idx = gguf_find_key(ctx, kv(LLM_KV_TOKENIZER_LIST).c_str()); + if (token_idx == -1) { + throw std::runtime_error("cannot find tokenizer vocab in model file\n"); + } + + const float * scores = nullptr; + const int score_idx = gguf_find_key(ctx, kv(LLM_KV_TOKENIZER_SCORES).c_str()); + if (score_idx != -1) { + scores = (const float * ) gguf_get_arr_data(ctx, score_idx); + } + + const int * toktypes = nullptr; + const int toktype_idx = gguf_find_key(ctx, kv(LLM_KV_TOKENIZER_TOKEN_TYPE).c_str()); + if (toktype_idx != -1) { + toktypes = (const int * ) gguf_get_arr_data(ctx, toktype_idx); + } + const uint32_t n_vocab = gguf_get_arr_n(ctx, token_idx); vocab.id_to_token.resize(n_vocab); @@ -5023,7 +5038,8 @@ static int llama_model_load(const std::string & fname, llama_model & model, llam llm_load_print_meta(ml, model); - if (model.hparams.n_vocab != model.vocab.id_to_token.size()) { + if (model.vocab.type != LLAMA_VOCAB_TYPE_NONE && + model.hparams.n_vocab != model.vocab.id_to_token.size()) { throw std::runtime_error("vocab size mismatch"); } @@ -9361,26 +9377,32 @@ static enum llama_vocab_type llama_vocab_get_type(const llama_vocab & vocab) { } static bool llama_is_normal_token(const llama_vocab & vocab, llama_token id) { + GGML_ASSERT(vocab.type != LLAMA_VOCAB_TYPE_NONE); return vocab.id_to_token[id].type == LLAMA_TOKEN_TYPE_NORMAL; } static bool llama_is_unknown_token(const llama_vocab & vocab, llama_token id) { + GGML_ASSERT(vocab.type != LLAMA_VOCAB_TYPE_NONE); return vocab.id_to_token[id].type == LLAMA_TOKEN_TYPE_UNKNOWN; } static bool llama_is_control_token(const llama_vocab & vocab, llama_token id) { + GGML_ASSERT(vocab.type != LLAMA_VOCAB_TYPE_NONE); return vocab.id_to_token[id].type == LLAMA_TOKEN_TYPE_CONTROL; } static bool llama_is_byte_token(const llama_vocab & vocab, llama_token id) { + GGML_ASSERT(vocab.type != LLAMA_VOCAB_TYPE_NONE); return vocab.id_to_token[id].type == LLAMA_TOKEN_TYPE_BYTE; } static bool llama_is_user_defined_token(const llama_vocab& vocab, llama_token id) { + GGML_ASSERT(vocab.type != LLAMA_VOCAB_TYPE_NONE); return vocab.id_to_token[id].type == LLAMA_TOKEN_TYPE_USER_DEFINED; } static uint8_t llama_token_to_byte(const llama_vocab& vocab, llama_token id) { + GGML_ASSERT(llama_vocab_get_type(vocab) != LLAMA_VOCAB_TYPE_NONE); GGML_ASSERT(llama_is_byte_token(vocab, id)); const auto& token_data = vocab.id_to_token.at(id); switch (llama_vocab_get_type(vocab)) { @@ -9401,6 +9423,7 @@ static uint8_t llama_token_to_byte(const llama_vocab& vocab, llama_token id) { } static llama_token llama_byte_to_token(const llama_vocab & vocab, uint8_t ch) { + GGML_ASSERT(llama_vocab_get_type(vocab) != LLAMA_VOCAB_TYPE_NONE); static const char * hex = "0123456789ABCDEF"; switch (llama_vocab_get_type(vocab)) { case LLAMA_VOCAB_TYPE_SPM: { @@ -10232,6 +10255,8 @@ static std::vector llama_tokenize_internal(const llama_vocab & } } } break; + case LLAMA_VOCAB_TYPE_NONE: + GGML_ASSERT(false); } return output; @@ -13138,7 +13163,7 @@ enum llama_rope_type llama_rope_type(const struct llama_model * model) { } int32_t llama_n_vocab(const struct llama_model * model) { - return model->vocab.id_to_token.size(); + return model->hparams.n_vocab; } int32_t llama_n_ctx_train(const struct llama_model * model) { @@ -13962,14 +13987,17 @@ float * llama_get_embeddings_seq(struct llama_context * ctx, llama_seq_id seq_id } const char * llama_token_get_text(const struct llama_model * model, llama_token token) { + GGML_ASSERT(model->vocab.type != LLAMA_VOCAB_TYPE_NONE); return model->vocab.id_to_token[token].text.c_str(); } float llama_token_get_score(const struct llama_model * model, llama_token token) { + GGML_ASSERT(model->vocab.type != LLAMA_VOCAB_TYPE_NONE); return model->vocab.id_to_token[token].score; } llama_token_type llama_token_get_type(const struct llama_model * model, llama_token token) { + GGML_ASSERT(model->vocab.type != LLAMA_VOCAB_TYPE_NONE); return model->vocab.id_to_token[token].type; } diff --git a/llama.h b/llama.h index 2d16cc9b9..90aa5372e 100644 --- a/llama.h +++ b/llama.h @@ -59,9 +59,10 @@ extern "C" { typedef int32_t llama_seq_id; enum llama_vocab_type { - LLAMA_VOCAB_TYPE_SPM = 0, // SentencePiece - LLAMA_VOCAB_TYPE_BPE = 1, // Byte Pair Encoding - LLAMA_VOCAB_TYPE_WPM = 2, // WordPiece + LLAMA_VOCAB_TYPE_NONE = 0, // For models without vocab + LLAMA_VOCAB_TYPE_SPM = 1, // SentencePiece + LLAMA_VOCAB_TYPE_BPE = 2, // Byte Pair Encoding + LLAMA_VOCAB_TYPE_WPM = 3, // WordPiece }; // note: these values should be synchronized with ggml_rope From 727107707a73b3dc8a497cf9fc9405722c16dd2b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20=C4=8Cert=C3=ADk?= Date: Thu, 14 Mar 2024 11:57:31 -0600 Subject: [PATCH 625/812] gguf-py : bump version to 0.8.0 (#6060) --- gguf-py/pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gguf-py/pyproject.toml b/gguf-py/pyproject.toml index 9789c2c87..96396e04e 100644 --- a/gguf-py/pyproject.toml +++ b/gguf-py/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "gguf" -version = "0.7.0" +version = "0.8.0" description = "Read and write ML models in GGUF for GGML" authors = ["GGML "] packages = [ From 6e0438da3cc95b89cdbf55f45fa4e324d9076792 Mon Sep 17 00:00:00 2001 From: Steve Grubb Date: Thu, 14 Mar 2024 14:29:32 -0400 Subject: [PATCH 626/812] gguf : fix resource leaks (#6061) There several places where a gguf context is allocated. A call to gguf_free is missing in some error paths. Also on linux, llama-bench was missing a fclose. --- examples/gguf/gguf.cpp | 1 + examples/llama-bench/llama-bench.cpp | 1 + examples/llava/clip.cpp | 4 ++++ examples/train-text-from-scratch/train-text-from-scratch.cpp | 1 + 4 files changed, 7 insertions(+) diff --git a/examples/gguf/gguf.cpp b/examples/gguf/gguf.cpp index e67be4fb2..5444503a5 100644 --- a/examples/gguf/gguf.cpp +++ b/examples/gguf/gguf.cpp @@ -211,6 +211,7 @@ static bool gguf_ex_read_1(const std::string & fname) { for (int j = 0; j < ggml_nelements(cur); ++j) { if (data[j] != 100 + i) { fprintf(stderr, "%s: tensor[%d]: data[%d] = %f\n", __func__, i, j, data[j]); + gguf_free(ctx); return false; } } diff --git a/examples/llama-bench/llama-bench.cpp b/examples/llama-bench/llama-bench.cpp index bf94e7e7a..d6e5e0497 100644 --- a/examples/llama-bench/llama-bench.cpp +++ b/examples/llama-bench/llama-bench.cpp @@ -103,6 +103,7 @@ static std::string get_cpu_info() { } } } + fclose(f); } #endif // TODO: other platforms diff --git a/examples/llava/clip.cpp b/examples/llava/clip.cpp index 6653b815d..2035554ea 100644 --- a/examples/llava/clip.cpp +++ b/examples/llava/clip.cpp @@ -995,6 +995,7 @@ struct clip_ctx * clip_model_load(const char * fname, const int verbosity = 1) { if (!new_clip->ctx_data) { fprintf(stderr, "%s: ggml_init() failed\n", __func__); clip_free(new_clip); + gguf_free(ctx); return nullptr; } @@ -1002,6 +1003,7 @@ struct clip_ctx * clip_model_load(const char * fname, const int verbosity = 1) { if (!fin) { printf("cannot open model file for loading tensors\n"); clip_free(new_clip); + gguf_free(ctx); return nullptr; } @@ -1023,6 +1025,7 @@ struct clip_ctx * clip_model_load(const char * fname, const int verbosity = 1) { if (!fin) { printf("%s: failed to seek for tensor %s\n", __func__, name); clip_free(new_clip); + gguf_free(ctx); return nullptr; } int num_bytes = ggml_nbytes(cur); @@ -1908,6 +1911,7 @@ bool clip_model_quantize(const char * fname_inp, const char * fname_out, const i break; default: printf("Please use an input file in f32 or f16\n"); + gguf_free(ctx_out); return false; } diff --git a/examples/train-text-from-scratch/train-text-from-scratch.cpp b/examples/train-text-from-scratch/train-text-from-scratch.cpp index 7eafe8515..7d06e401b 100644 --- a/examples/train-text-from-scratch/train-text-from-scratch.cpp +++ b/examples/train-text-from-scratch/train-text-from-scratch.cpp @@ -711,6 +711,7 @@ static bool load_checkpoint_file(const char * filename, struct my_llama_model * load_checkpoint_gguf(fctx, f_ggml_ctx, model, train); + gguf_free(fctx); return true; } From 4755afd1cbd40d93c017e5b98c39796f52345314 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Thu, 14 Mar 2024 22:58:41 +0200 Subject: [PATCH 627/812] llama : fix integer overflow during quantization (#6063) --- llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llama.cpp b/llama.cpp index 10fd53469..2c3841974 100644 --- a/llama.cpp +++ b/llama.cpp @@ -11977,7 +11977,7 @@ static ggml_type llama_tensor_get_type(quantize_state_internal & qs, ggml_type n return new_type; } -static int32_t llama_tensor_quantize_internal(enum ggml_type new_type, const float * f32_data, void * new_data, const int chunk_size, int nrows, int n_per_row, const float * imatrix, std::vector & workers, const int nthread) { +static size_t llama_tensor_quantize_internal(enum ggml_type new_type, const float * f32_data, void * new_data, const int chunk_size, int nrows, int n_per_row, const float * imatrix, std::vector & workers, const int nthread) { std::mutex mutex; int counter = 0; size_t new_size = 0; From b0bc9f4a9da7c19f4779106ea83b23feca747566 Mon Sep 17 00:00:00 2001 From: slaren Date: Fri, 15 Mar 2024 09:22:24 +0100 Subject: [PATCH 628/812] llama-bench : use random tokens to improve accuracy with mixtral (#6069) --- examples/llama-bench/llama-bench.cpp | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/examples/llama-bench/llama-bench.cpp b/examples/llama-bench/llama-bench.cpp index d6e5e0497..32eea7869 100644 --- a/examples/llama-bench/llama-bench.cpp +++ b/examples/llama-bench/llama-bench.cpp @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -1123,15 +1124,19 @@ struct sql_printer : public printer { static void test_prompt(llama_context * ctx, int n_prompt, int n_past, int n_batch, int n_threads) { llama_set_n_threads(ctx, n_threads, n_threads); - //std::vector tokens(n_prompt, llama_token_bos(llama_get_model(ctx))); - //llama_decode(ctx, llama_batch_get_one(tokens.data(), n_prompt, n_past, 0)); - //GGML_UNUSED(n_batch); + const llama_model * model = llama_get_model(ctx); + const int32_t n_vocab = llama_n_vocab(model); + + std::vector tokens(n_batch); - std::vector tokens(n_batch, llama_token_bos(llama_get_model(ctx))); int n_processed = 0; while (n_processed < n_prompt) { int n_tokens = std::min(n_prompt - n_processed, n_batch); + tokens[0] = n_processed == 0 && llama_add_bos_token(model) ? llama_token_bos(model) : std::rand() % n_vocab; + for (int i = 1; i < n_tokens; i++) { + tokens[i] = std::rand() % n_vocab; + } llama_decode(ctx, llama_batch_get_one(tokens.data(), n_tokens, n_past + n_processed, 0)); n_processed += n_tokens; } @@ -1142,11 +1147,15 @@ static void test_prompt(llama_context * ctx, int n_prompt, int n_past, int n_bat static void test_gen(llama_context * ctx, int n_gen, int n_past, int n_threads) { llama_set_n_threads(ctx, n_threads, n_threads); - llama_token token = llama_token_bos(llama_get_model(ctx)); + const llama_model * model = llama_get_model(ctx); + const int32_t n_vocab = llama_n_vocab(model); + + llama_token token = llama_add_bos_token(model) ? llama_token_bos(model) : std::rand() % n_vocab; for (int i = 0; i < n_gen; i++) { llama_decode(ctx, llama_batch_get_one(&token, 1, n_past + i, 0)); llama_synchronize(ctx); + token = std::rand() % n_vocab; } } From aab606a11fc0a9740a7f297521c3eef851dfb351 Mon Sep 17 00:00:00 2001 From: Xuan Son Nguyen Date: Fri, 15 Mar 2024 09:44:57 +0100 Subject: [PATCH 629/812] llama : add Orion chat template (#6066) --- llama.cpp | 20 ++++++++++++++++++++ tests/test-chat-template.cpp | 4 ++++ 2 files changed, 24 insertions(+) diff --git a/llama.cpp b/llama.cpp index 2c3841974..b8a8d2723 100644 --- a/llama.cpp +++ b/llama.cpp @@ -14242,6 +14242,26 @@ static int32_t llama_chat_apply_template_internal( if (add_ass) { ss << "model\n"; } + } else if (tmpl == "orion" || tmpl.find("'\\n\\nAssistant: ' + eos_token") != std::string::npos) { + // OrionStarAI/Orion-14B-Chat + std::string system_prompt = ""; + for (auto message : chat) { + std::string role(message->role); + if (role == "system") { + // there is no system message support, we will merge it with user prompt + system_prompt = message->content; + continue; + } else if (role == "user") { + ss << "Human: "; + if (!system_prompt.empty()) { + ss << system_prompt << "\n\n"; + system_prompt = ""; + } + ss << message->content << "\n\nAssistant: "; + } else { + ss << message->content << ""; + } + } } else { // template not supported return -1; diff --git a/tests/test-chat-template.cpp b/tests/test-chat-template.cpp index fa2eb577b..6e9e4bd1e 100644 --- a/tests/test-chat-template.cpp +++ b/tests/test-chat-template.cpp @@ -31,6 +31,8 @@ int main(void) { "{% for message in messages %}{{bos_token + message['role'] + '\\n' + message['content'] + eos_token + '\\n'}}{% endfor %}{% if add_generation_prompt %}{{ bos_token + 'assistant\\n' }}{% endif %}", // google/gemma-7b-it "{% if messages[0]['role'] == 'system' %}{{ raise_exception('System role not supported') }}{% endif %}{% for message in messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if (message['role'] == 'assistant') %}{% set role = 'model' %}{% else %}{% set role = message['role'] %}{% endif %}{{ '' + role + '\\n' + message['content'] | trim + '\\n' }}{% endfor %}{% if add_generation_prompt %}{{'model\\n'}}{% endif %}", + // OrionStarAI/Orion-14B-Chat + "{% for message in messages %}{% if loop.first %}{{ bos_token }}{% endif %}{% if message['role'] == 'user' %}{{ 'Human: ' + message['content'] + '\\n\\nAssistant: ' + eos_token }}{% elif message['role'] == 'assistant' %}{{ message['content'] + eos_token }}{% endif %}{% endfor %}", }; std::vector expected_output = { // teknium/OpenHermes-2.5-Mistral-7B @@ -45,6 +47,8 @@ int main(void) { "system\nYou are a helpful assistant\nuser\nHello\nassistant\nHi there\nuser\nWho are you\nassistant\n I am an assistant \nuser\nAnother question\nassistant\n", // google/gemma-7b-it "user\nYou are a helpful assistant\n\nHello\nmodel\nHi there\nuser\nWho are you\nmodel\nI am an assistant\nuser\nAnother question\nmodel\n", + // OrionStarAI/Orion-14B-Chat + "Human: You are a helpful assistant\n\nHello\n\nAssistant: Hi thereHuman: Who are you\n\nAssistant: I am an assistant Human: Another question\n\nAssistant: ", }; std::vector formatted_chat(1024); int32_t res; From 7ce2c77f88e1ca66ec48417e56f91746bac018c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20=C4=8Cert=C3=ADk?= Date: Fri, 15 Mar 2024 02:46:51 -0600 Subject: [PATCH 630/812] gguf : add support for I64 and F64 arrays (#6062) * gguf : add support for I64 and F64 arrays GGML currently does not support I64 or F64 arrays and they are not often used in machine learning, however if in the future the need arises, it would be nice to add them now, so that the types are next to the other types I8, I16, I32 in the enums, and it also reserves their type number. Furthermore, with this addition the GGUF format becomes very usable for most computational applications of NumPy (being compatible with the most common NumPy dtypes: i8, i16, i32, i64, f32, f64), providing a faster, and more versatile alternative to the `npz` format, and a simpler alternative to the `hdf5` format. The change in this PR seems small, not significantly increasing the maintenance burden. I tested this from Python using GGUFWriter/Reader and `gguf-dump`, as well as from C, everything seems to work. * Fix compiler warnings --- ggml.c | 17 +++++++++++++++++ ggml.h | 2 ++ gguf-py/gguf/constants.py | 4 ++++ gguf-py/gguf/gguf_reader.py | 14 ++++++++++---- gguf-py/gguf/gguf_writer.py | 12 ++++++++---- 5 files changed, 41 insertions(+), 8 deletions(-) diff --git a/ggml.c b/ggml.c index fbc66f65b..c94006e51 100644 --- a/ggml.c +++ b/ggml.c @@ -470,6 +470,19 @@ static const ggml_type_traits_t type_traits[GGML_TYPE_COUNT] = { .type_size = sizeof(int32_t), .is_quantized = false, }, + [GGML_TYPE_I64] = { + .type_name = "i64", + .blck_size = 1, + .type_size = sizeof(int64_t), + .is_quantized = false, + }, + [GGML_TYPE_F64] = { + .type_name = "f64", + .blck_size = 1, + .type_size = sizeof(double), + .is_quantized = false, + .nrows = 1, + }, [GGML_TYPE_F32] = { .type_name = "f32", .blck_size = 1, @@ -12418,6 +12431,8 @@ static void ggml_compute_forward_alibi( case GGML_TYPE_I8: case GGML_TYPE_I16: case GGML_TYPE_I32: + case GGML_TYPE_I64: + case GGML_TYPE_F64: case GGML_TYPE_COUNT: { GGML_ASSERT(false); @@ -12504,6 +12519,8 @@ static void ggml_compute_forward_clamp( case GGML_TYPE_I8: case GGML_TYPE_I16: case GGML_TYPE_I32: + case GGML_TYPE_I64: + case GGML_TYPE_F64: case GGML_TYPE_COUNT: { GGML_ASSERT(false); diff --git a/ggml.h b/ggml.h index ab26c8f59..c937d4a53 100644 --- a/ggml.h +++ b/ggml.h @@ -366,6 +366,8 @@ extern "C" { GGML_TYPE_I8 = 24, GGML_TYPE_I16 = 25, GGML_TYPE_I32 = 26, + GGML_TYPE_I64 = 27, + GGML_TYPE_F64 = 28, GGML_TYPE_COUNT, }; diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py index 2d7cf16c1..458a641dc 100644 --- a/gguf-py/gguf/constants.py +++ b/gguf-py/gguf/constants.py @@ -665,6 +665,8 @@ class GGMLQuantizationType(IntEnum): I8 = 24 I16 = 25 I32 = 26 + I64 = 27 + F64 = 28 class GGUFEndian(IntEnum): @@ -734,6 +736,8 @@ GGML_QUANT_SIZES = { GGMLQuantizationType.I8: (1, 1), GGMLQuantizationType.I16: (1, 2), GGMLQuantizationType.I32: (1, 4), + GGMLQuantizationType.I64: (1, 8), + GGMLQuantizationType.F64: (1, 8), } diff --git a/gguf-py/gguf/gguf_reader.py b/gguf-py/gguf/gguf_reader.py index 1c10f5753..33afac552 100644 --- a/gguf-py/gguf/gguf_reader.py +++ b/gguf-py/gguf/gguf_reader.py @@ -242,12 +242,15 @@ class GGUFReader: n_bytes = n_elems * type_size // block_size data_offs = int(start_offs + offset_tensor[0]) item_type: npt.DTypeLike - if ggml_type == GGMLQuantizationType.F32: - item_count = n_elems - item_type = np.float32 - elif ggml_type == GGMLQuantizationType.F16: + if ggml_type == GGMLQuantizationType.F16: item_count = n_elems item_type = np.float16 + elif ggml_type == GGMLQuantizationType.F32: + item_count = n_elems + item_type = np.float32 + elif ggml_type == GGMLQuantizationType.F64: + item_count = n_elems + item_type = np.float64 elif ggml_type == GGMLQuantizationType.I8: item_count = n_elems item_type = np.int8 @@ -257,6 +260,9 @@ class GGUFReader: elif ggml_type == GGMLQuantizationType.I32: item_count = n_elems item_type = np.int32 + elif ggml_type == GGMLQuantizationType.I64: + item_count = n_elems + item_type = np.int64 else: item_count = n_bytes item_type = np.uint8 diff --git a/gguf-py/gguf/gguf_writer.py b/gguf-py/gguf/gguf_writer.py index 81b2eb884..1967b633c 100644 --- a/gguf-py/gguf/gguf_writer.py +++ b/gguf-py/gguf/gguf_writer.py @@ -204,18 +204,22 @@ class GGUFWriter: for i in range(n_dims): self.ti_data += self._pack("Q", tensor_shape[n_dims - 1 - i]) if raw_dtype is None: - if tensor_dtype == np.float32: - dtype = GGMLQuantizationType.F32 - elif tensor_dtype == np.float16: + if tensor_dtype == np.float16: dtype = GGMLQuantizationType.F16 + elif tensor_dtype == np.float32: + dtype = GGMLQuantizationType.F32 + elif tensor_dtype == np.float64: + dtype = GGMLQuantizationType.F64 elif tensor_dtype == np.int8: dtype = GGMLQuantizationType.I8 elif tensor_dtype == np.int16: dtype = GGMLQuantizationType.I16 elif tensor_dtype == np.int32: dtype = GGMLQuantizationType.I32 + elif tensor_dtype == np.int64: + dtype = GGMLQuantizationType.I64 else: - raise ValueError("Only F32, F16, I8, I16, I32 tensors are supported for now") + raise ValueError("Only F16, F32, F64, I8, I16, I32, I64 tensors are supported for now") else: dtype = raw_dtype self.ti_data += self._pack("I", dtype) From 753e36f650fa2a5869f89188d9ee745dc74cf14b Mon Sep 17 00:00:00 2001 From: AidanBeltonS <87009434+AidanBeltonS@users.noreply.github.com> Date: Fri, 15 Mar 2024 09:26:20 +0000 Subject: [PATCH 631/812] [SYCL] Fix non-intel device selection (#6042) * Fix non-intel device selection * Update ggml-sycl.cpp Co-authored-by: Neo Zhang Jianyu * Update ggml-sycl.cpp Co-authored-by: Neo Zhang Jianyu --------- Co-authored-by: Abhilash Majumder <30946547+abhilash1910@users.noreply.github.com> Co-authored-by: Neo Zhang Jianyu --- ggml-sycl.cpp | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/ggml-sycl.cpp b/ggml-sycl.cpp index 9f6506383..a1ca6aba5 100644 --- a/ggml-sycl.cpp +++ b/ggml-sycl.cpp @@ -3451,7 +3451,7 @@ class sycl_gpu_mgr { dpct::device_info prop; dpct::get_device_info(prop, device); if (max_compute_units == prop.get_max_compute_units() && - prop.get_major_version() == 1) { + is_ext_oneapi_device(device)) { gpus.push_back(id); devices.push_back(device); work_group_size = prop.get_max_work_group_size(); @@ -3484,6 +3484,15 @@ class sycl_gpu_mgr { assert(false); return -1; } + + bool is_ext_oneapi_device(const sycl::device &dev) { + sycl::backend dev_backend = dev.get_backend(); + if (dev_backend == sycl::backend::ext_oneapi_level_zero || + dev_backend == sycl::backend::ext_oneapi_cuda || + dev_backend == sycl::backend::ext_oneapi_hip) + return true; + return false; + } }; static sycl_gpu_mgr *g_sycl_gpu_mgr = NULL; From 131b0584096ee9df4d07cb28759dfea6efe6475f Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Fri, 15 Mar 2024 11:36:50 +0200 Subject: [PATCH 632/812] make : ggml-metal.o depends on ggml.h --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index cb597b209..c0f125036 100644 --- a/Makefile +++ b/Makefile @@ -553,7 +553,7 @@ endif endif # LLAMA_METAL ifdef LLAMA_METAL -ggml-metal.o: ggml-metal.m ggml-metal.h +ggml-metal.o: ggml-metal.m ggml-metal.h ggml.h $(CC) $(CFLAGS) -c $< -o $@ ifdef LLAMA_METAL_EMBED_LIBRARY From 46acb3676718b983157058aecf729a2064fc7d34 Mon Sep 17 00:00:00 2001 From: Neo Zhang Jianyu Date: Fri, 15 Mar 2024 18:53:53 +0800 Subject: [PATCH 633/812] fix set main gpu error (#6073) --- examples/sycl/build.sh | 5 +- examples/sycl/run-llama2.sh | 16 +- ggml-sycl.cpp | 332 ++++++++++++++++++++++++++---------- ggml-sycl.h | 5 + llama.cpp | 23 ++- 5 files changed, 282 insertions(+), 99 deletions(-) diff --git a/examples/sycl/build.sh b/examples/sycl/build.sh index 26ad2f7da..f20391d7a 100755 --- a/examples/sycl/build.sh +++ b/examples/sycl/build.sh @@ -13,8 +13,11 @@ source /opt/intel/oneapi/setvars.sh #for FP32 cmake .. -DLLAMA_SYCL=ON -DCMAKE_C_COMPILER=icx -DCMAKE_CXX_COMPILER=icpx -#build example/main only +#build example/main #cmake --build . --config Release --target main +#build example/llama-bench +#cmake --build . --config Release --target llama-bench + #build all binary cmake --build . --config Release -v diff --git a/examples/sycl/run-llama2.sh b/examples/sycl/run-llama2.sh index 52f7c01a4..c979a52f6 100755 --- a/examples/sycl/run-llama2.sh +++ b/examples/sycl/run-llama2.sh @@ -9,18 +9,28 @@ source /opt/intel/oneapi/setvars.sh if [ $# -gt 0 ]; then GGML_SYCL_DEVICE=$1 + GGML_SYCL_SINGLE_GPU=1 else GGML_SYCL_DEVICE=0 fi -echo "use $GGML_SYCL_DEVICE as main GPU" + #export GGML_SYCL_DEBUG=1 #ZES_ENABLE_SYSMAN=1, Support to get free memory of GPU by sycl::aspect::ext_intel_free_memory. Recommended to use when --split-mode = layer. -#use all GPUs with same max compute units -ZES_ENABLE_SYSMAN=1 ./build/bin/main -m models/llama-2-7b.Q4_0.gguf -p "${INPUT2}" -n 400 -e -ngl 33 -s 0 +if [ $GGML_SYCL_SINGLE_GPU -eq 1 ]; then + echo "use $GGML_SYCL_DEVICE as main GPU" + #use signle GPU only + ZES_ENABLE_SYSMAN=1 ./build/bin/main -m models/llama-2-7b.Q4_0.gguf -p "${INPUT2}" -n 400 -e -ngl 33 -s 0 -mg $GGML_SYCL_DEVICE -sm none +else + #use multiple GPUs with same max compute units + ZES_ENABLE_SYSMAN=1 ./build/bin/main -m models/llama-2-7b.Q4_0.gguf -p "${INPUT2}" -n 400 -e -ngl 33 -s 0 +fi #use main GPU only #ZES_ENABLE_SYSMAN=1 ./build/bin/main -m models/llama-2-7b.Q4_0.gguf -p "${INPUT2}" -n 400 -e -ngl 33 -s 0 -mg $GGML_SYCL_DEVICE -sm none +#use multiple GPUs with same max compute units +#ZES_ENABLE_SYSMAN=1 ./build/bin/main -m models/llama-2-7b.Q4_0.gguf -p "${INPUT2}" -n 400 -e -ngl 33 -s 0 + diff --git a/ggml-sycl.cpp b/ggml-sycl.cpp index a1ca6aba5..6dc5eb20c 100644 --- a/ggml-sycl.cpp +++ b/ggml-sycl.cpp @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include @@ -24,10 +25,9 @@ #include #include #include - #include #include - +#include #include #include @@ -82,6 +82,30 @@ Following definition copied from DPCT head files, which are used by ggml-sycl.cp #define __dpct_noinline__ __attribute__((noinline)) #endif + +std::string get_device_type_name(const sycl::device &Device) { + auto DeviceType = Device.get_info(); + switch (DeviceType) { + case sycl::info::device_type::cpu: + return "cpu"; + case sycl::info::device_type::gpu: + return "gpu"; + case sycl::info::device_type::host: + return "host"; + case sycl::info::device_type::accelerator: + return "acc"; + default: + return "unknown"; + } +} + +std::string get_device_backend_and_type(const sycl::device &device) { + std::stringstream device_type; + sycl::backend backend = device.get_backend(); + device_type << backend << ":" << get_device_type_name(device); + return device_type.str(); +} + namespace dpct { typedef sycl::queue *queue_ptr; @@ -942,17 +966,65 @@ namespace dpct private: mutable std::recursive_mutex m_mutex; + static bool compare_dev(sycl::device &device1, sycl::device &device2) + { + dpct::device_info prop1; + dpct::get_device_info(prop1, device1); + dpct::device_info prop2; + dpct::get_device_info(prop2, device2); + return prop1.get_max_compute_units() > prop2.get_max_compute_units(); + } + static int convert_backend_index(std::string & backend) { + if (backend == "ext_oneapi_level_zero:gpu") return 0; + if (backend == "opencl:gpu") return 1; + if (backend == "opencl:cpu") return 2; + if (backend == "opencl:acc") return 3; + printf("convert_backend_index: can't handle backend=%s\n", backend.c_str()); + GGML_ASSERT(false); + } + static bool compare_backend(std::string &backend1, std::string &backend2) { + return convert_backend_index(backend1) < convert_backend_index(backend2); + } dev_mgr() { sycl::device default_device = sycl::device(sycl::default_selector_v); _devs.push_back(std::make_shared(default_device)); - std::vector sycl_all_devs = - sycl::device::get_devices(sycl::info::device_type::all); + std::vector sycl_all_devs; // Collect other devices except for the default device. if (default_device.is_cpu()) _cpu_device = 0; + + auto Platforms = sycl::platform::get_platforms(); + // Keep track of the number of devices per backend + std::map DeviceNums; + std::map> backend_devices; + + while (!Platforms.empty()) { + auto Platform = Platforms.back(); + Platforms.pop_back(); + auto devices = Platform.get_devices(); + std::string backend_type = get_device_backend_and_type(devices[0]); + for (const auto &device : devices) { + backend_devices[backend_type].push_back(device); + } + } + + std::vector keys; + for(auto it = backend_devices.begin(); it != backend_devices.end(); ++it) { + keys.push_back(it->first); + } + std::sort(keys.begin(), keys.end(), compare_backend); + + for (auto &key : keys) { + std::vector devs = backend_devices[key]; + std::sort(devs.begin(), devs.end(), compare_dev); + for (const auto &dev : devs) { + sycl_all_devs.push_back(dev); + } + } + for (auto &dev : sycl_all_devs) { if (dev == default_device) @@ -3202,6 +3274,11 @@ static int g_work_group_size = 0; #define GGML_SYCL_MMV_Y 1 #endif +enum ggml_sycl_backend_gpu_mode { + SYCL_UNSET_GPU_MODE = -1, + SYCL_SINGLE_GPU_MODE = 0, + SYCL_MUL_GPU_MODE +}; static_assert(sizeof(sycl::half) == sizeof(ggml_fp16_t), "wrong fp16 size"); @@ -3401,12 +3478,31 @@ class sycl_gpu_mgr { int work_group_size = 0; std::string gpus_list = ""; + /* + Use all GPUs with same top max compute units + */ sycl_gpu_mgr() { detect_sycl_gpu_list_with_max_cu(); get_allow_gpus(); create_context_with_gpus(); } + /* + Only use the assigned GPU + */ + sycl_gpu_mgr(int main_gpu_id) { + sycl::device device = dpct::dev_mgr::instance().get_device(main_gpu_id); + dpct::device_info prop; + dpct::get_device_info(prop, device); + gpus.push_back(main_gpu_id); + devices.push_back(device); + work_group_size = prop.get_max_work_group_size(); + max_compute_units = prop.get_max_compute_units(); + + get_allow_gpus(); + create_context_with_gpus(); + } + void create_context_with_gpus() { sycl::context ctx = sycl::context(devices); assert(gpus.size() > 0); @@ -3422,7 +3518,7 @@ class sycl_gpu_mgr { gpus_list += std::to_string(gpus[i]); gpus_list += ","; } - if (gpus_list.length() > 2) { + if (gpus_list.length() > 1) { gpus_list.pop_back(); } } @@ -3471,8 +3567,8 @@ class sycl_gpu_mgr { if (gpus[i] == id) return i; } - assert(false); - return -1; + printf("miss to get device index by id=%d\n", id); + GGML_ASSERT(false); } int get_next_index(int id) { @@ -3481,8 +3577,7 @@ class sycl_gpu_mgr { if (gpus[i] == id) return i; } - assert(false); - return -1; + GGML_ASSERT(false); } bool is_ext_oneapi_device(const sycl::device &dev) { @@ -3500,11 +3595,14 @@ static int g_device_count = -1; static int g_all_sycl_device_count = -1; static int g_main_device = -1; static int g_main_device_id = -1; +static bool g_ggml_backend_sycl_buffer_type_initialized = false; static std::array g_default_tensor_split = {}; static float g_tensor_split[GGML_SYCL_MAX_DEVICES] = {0}; +static ggml_sycl_backend_gpu_mode g_ggml_sycl_backend_gpu_mode = SYCL_UNSET_GPU_MODE; + struct sycl_device_capabilities { int cc; // compute capability bool vmm; // virtual memory support @@ -13008,17 +13106,20 @@ bool ggml_sycl_loaded(void) { return g_sycl_loaded; } -void print_device_detail(int id) { +void print_device_detail(int id, sycl::device &device, std::string device_type) { + dpct::device_info prop; SYCL_CHECK(CHECK_TRY_ERROR( - dpct::get_device_info(prop, dpct::dev_mgr::instance().get_device(id)))); - sycl::device cur_device = dpct::dev_mgr::instance().get_device(id); + dpct::get_device_info(prop, device))); + std::string version; version += std::to_string(prop.get_major_version()); version += "."; version += std::to_string(prop.get_minor_version()); - fprintf(stderr, "|%2d|%45s|%18s|%17d|%14d|%13d|%15lu|\n", id, + device_type = std::regex_replace(device_type, std::regex("ext_oneapi_"), ""); + + fprintf(stderr, "|%2d|%18s|%45s|%10s|%11d|%8d|%7d|%15lu|\n", id, device_type.c_str(), prop.get_name(), version.c_str(), prop.get_max_compute_units(), prop.get_max_work_group_size(), prop.get_max_sub_group_size(), prop.get_global_mem_size()); @@ -13026,19 +13127,35 @@ void print_device_detail(int id) { void ggml_backend_sycl_print_sycl_devices() { int device_count = dpct::dev_mgr::instance().device_count(); + std::map DeviceNums; fprintf(stderr, "found %d SYCL devices:\n", device_count); - fprintf(stderr, "|ID| Name |compute capability|Max compute units|Max work group|Max sub group|Global mem size|\n"); - fprintf(stderr, "|--|---------------------------------------------|------------------|-----------------|--------------|-------------|---------------|\n"); + fprintf(stderr, "| | | |Compute |Max compute|Max work|Max sub| |\n"); + fprintf(stderr, "|ID| Device Type| Name|capability|units |group |group |Global mem size|\n"); + fprintf(stderr, "|--|------------------|---------------------------------------------|----------|-----------|--------|-------|---------------|\n"); for (int id = 0; id < device_count; ++id) { - print_device_detail(id); + sycl::device device = dpct::dev_mgr::instance().get_device(id); + sycl::backend backend = device.get_backend(); + std::string backend_type = get_device_backend_and_type(device); + int type_id=DeviceNums[backend_type]++; + std::stringstream device_type; + device_type << "[" << backend_type << ":" << std::to_string(type_id) << "]"; + print_device_detail(id, device, device_type.str()); } } void print_gpu_device_list() { - fprintf(stderr, "detect %d SYCL GPUs: [%s] with Max compute units:%d\n", - g_sycl_gpu_mgr->get_gpu_count(), - g_sycl_gpu_mgr->gpus_list.c_str(), - g_sycl_gpu_mgr->max_compute_units); + GGML_ASSERT(g_sycl_gpu_mgr); + + char* hint=NULL; + if (g_ggml_sycl_backend_gpu_mode == SYCL_SINGLE_GPU_MODE) { + hint = "use %d SYCL GPUs: [%s] with Max compute units:%d\n"; + } else { + hint = "detect %d SYCL GPUs: [%s] with top Max compute units:%d\n"; + } + fprintf(stderr, hint, + g_sycl_gpu_mgr->get_gpu_count(), + g_sycl_gpu_mgr->gpus_list.c_str(), + g_sycl_gpu_mgr->max_compute_units); } int get_sycl_env(const char *env_name, int default_val) { @@ -13074,23 +13191,6 @@ void ggml_init_sycl() try { #else fprintf(stderr, "%s: GGML_SYCL_F16: no\n", __func__); #endif - if (CHECK_TRY_ERROR(g_all_sycl_device_count = - dpct::dev_mgr::instance().device_count()) != 0) { - initialized = true; - g_sycl_loaded = false; - return; - } - GGML_ASSERT(g_all_sycl_device_count <= GGML_SYCL_MAX_DEVICES); - ggml_backend_sycl_print_sycl_devices(); - - if (!g_sycl_gpu_mgr) g_sycl_gpu_mgr = new sycl_gpu_mgr(); - - g_device_count = g_sycl_gpu_mgr->get_gpu_count(); - g_work_group_size = g_sycl_gpu_mgr->work_group_size; - - print_gpu_device_list(); - - int64_t total_vram = 0; /* NOT REMOVE, keep it for next optimize for XMX. #if defined(SYCL_USE_XMX) @@ -13099,49 +13199,15 @@ void ggml_init_sycl() try { fprintf(stderr, "%s: SYCL_USE_XMX: no\n", __func__); #endif */ - for (int id = 0; id < GGML_SYCL_MAX_DEVICES; ++id) { - g_device_caps[id].vmm = 0; - g_device_caps[id].device_id = -1; - g_device_caps[id].cc = 0; - g_tensor_split[id] = 0; - g_default_tensor_split[id] = 0; + + if (CHECK_TRY_ERROR(g_all_sycl_device_count = + dpct::dev_mgr::instance().device_count()) != 0) { + initialized = true; + g_sycl_loaded = false; + return; } - - for (int i = 0; i < g_device_count; ++i) { - int device_id = g_sycl_gpu_mgr->gpus[i]; - g_device_caps[i].vmm = 0; - - dpct::device_info prop; - SYCL_CHECK(CHECK_TRY_ERROR(dpct::get_device_info( - prop, dpct::dev_mgr::instance().get_device(device_id)))); - - g_default_tensor_split[i] = total_vram; - total_vram += prop.get_global_mem_size(); - - g_device_caps[i].cc = - 100 * prop.get_major_version() + 10 * prop.get_minor_version(); - } - - for (int i = 0; i < g_device_count; ++i) { - g_default_tensor_split[i] /= total_vram; - } - - for (int i = 0; i < g_device_count; ++i) { - SYCL_CHECK(ggml_sycl_set_device(i)); - - // create sycl streams - for (int is = 0; is < MAX_STREAMS; ++is) { - SYCL_CHECK(CHECK_TRY_ERROR( - g_syclStreams[i][is] = - dpct::get_current_device().create_queue( - g_sycl_gpu_mgr->get_co_ctx(), dpct::get_current_device()))); - } - - const dpct::queue_ptr stream = g_syclStreams[i][0]; - // create sycl handle - SYCL_CHECK(CHECK_TRY_ERROR(g_sycl_handles[i] = stream)); - } - + GGML_ASSERT(g_all_sycl_device_count <= GGML_SYCL_MAX_DEVICES); + ggml_backend_sycl_print_sycl_devices(); initialized = true; g_sycl_loaded = true; } @@ -13152,6 +13218,63 @@ catch (sycl::exception const &exc) { std::exit(1); } +void ggml_init_by_gpus(int device_count) try { + g_device_count = device_count; + g_work_group_size = g_sycl_gpu_mgr->work_group_size; + + int64_t total_vram = 0; + + print_gpu_device_list(); + + for (int id = 0; id < GGML_SYCL_MAX_DEVICES; ++id) { + g_device_caps[id].vmm = 0; + g_device_caps[id].device_id = -1; + g_device_caps[id].cc = 0; + g_tensor_split[id] = 0; + g_default_tensor_split[id] = 0; + } + + for (int i = 0; i < g_device_count; ++i) { + int device_id = g_sycl_gpu_mgr->gpus[i]; + g_device_caps[i].vmm = 0; + + dpct::device_info prop; + SYCL_CHECK(CHECK_TRY_ERROR(dpct::get_device_info( + prop, dpct::dev_mgr::instance().get_device(device_id)))); + + g_default_tensor_split[i] = total_vram; + total_vram += prop.get_global_mem_size(); + + g_device_caps[i].cc = + 100 * prop.get_major_version() + 10 * prop.get_minor_version(); + } + + for (int i = 0; i < g_device_count; ++i) { + g_default_tensor_split[i] /= total_vram; + } + + for (int i = 0; i < g_device_count; ++i) { + SYCL_CHECK(ggml_sycl_set_device(i)); + + // create sycl streams + for (int is = 0; is < MAX_STREAMS; ++is) { + SYCL_CHECK(CHECK_TRY_ERROR( + g_syclStreams[i][is] = + dpct::get_current_device().create_queue( + g_sycl_gpu_mgr->get_co_ctx(), dpct::get_current_device()))); + } + + const dpct::queue_ptr stream = g_syclStreams[i][0]; + // create sycl handle + SYCL_CHECK(CHECK_TRY_ERROR(g_sycl_handles[i] = stream)); + } +} +catch (sycl::exception const &exc) { + std::cerr << exc.what() << "Exception caught at file:" << __FILE__ + << ", line:" << __LINE__ << std::endl; + std::exit(1); +} + void *ggml_sycl_host_malloc(size_t size) try { if (getenv("GGML_SYCL_NO_PINNED") != nullptr) { return nullptr; @@ -16551,22 +16674,24 @@ static ggml_backend_buffer_type_i ggml_backend_sycl_buffer_type_interface = { /* .is_host = */ nullptr, }; -ggml_backend_buffer_type_t ggml_backend_sycl_buffer_type(int device) { +ggml_backend_buffer_type_t ggml_backend_sycl_buffer_type(int device_index) { + if (device_index>=g_device_count or device_index<0) { + printf("ggml_backend_sycl_buffer_type error: device_index:%d is out of range [0, %d], miss to call ggml_backend_sycl_set_single_device()\n", + device_index, g_device_count-1); + GGML_ASSERT(device_indexgpus[i])}, }; } - ggml_backend_sycl_buffer_type_initialized = true; + g_ggml_backend_sycl_buffer_type_initialized = true; } - - return &ggml_backend_sycl_buffer_types[device]; + return &ggml_backend_sycl_buffer_types[device_index]; } // sycl split buffer type @@ -17319,11 +17444,42 @@ GGML_API GGML_CALL int ggml_backend_sycl_get_device_index(int device_id) { return g_sycl_gpu_mgr->get_index(device_id); } +GGML_API GGML_CALL int ggml_backend_sycl_get_device_id(int device_index) { + return g_sycl_gpu_mgr->gpus[device_index]; +} + +GGML_API GGML_CALL void ggml_backend_sycl_set_single_device_mode(int main_gpu_id) { + GGML_ASSERT(main_gpu_idget_gpu_count()); + g_ggml_backend_sycl_buffer_type_initialized = false; +} + +GGML_API GGML_CALL void ggml_backend_sycl_set_mul_device_mode() { + if (g_ggml_sycl_backend_gpu_mode == SYCL_MUL_GPU_MODE) { + return; + } + + fprintf(stderr, "ggml_backend_sycl_set_mul_device_mode: true\n"); + + if (g_sycl_gpu_mgr) { + delete g_sycl_gpu_mgr; + } + g_sycl_gpu_mgr = new sycl_gpu_mgr(); + g_ggml_sycl_backend_gpu_mode = SYCL_MUL_GPU_MODE; + ggml_init_by_gpus(g_sycl_gpu_mgr->get_gpu_count()); + g_ggml_backend_sycl_buffer_type_initialized = false; +} + extern "C" int ggml_backend_sycl_reg_devices(); int ggml_backend_sycl_reg_devices() { - if (!g_sycl_gpu_mgr) g_sycl_gpu_mgr = new sycl_gpu_mgr(); - g_device_count = g_sycl_gpu_mgr->get_gpu_count(); + ggml_backend_sycl_set_mul_device_mode(); assert(g_device_count>0); for (int i = 0; i < g_device_count; i++) { int id = g_sycl_gpu_mgr->gpus[i]; diff --git a/ggml-sycl.h b/ggml-sycl.h index bf5b11b36..c549a64a1 100644 --- a/ggml-sycl.h +++ b/ggml-sycl.h @@ -29,6 +29,11 @@ GGML_API GGML_CALL ggml_backend_buffer_type_t ggml_backend_sycl_split_buffer_typ GGML_API GGML_CALL void ggml_backend_sycl_get_device_memory(int device, size_t *free, size_t *total); GGML_API GGML_CALL int ggml_backend_sycl_get_device_index(int device_id); +// TODO: these are temporary +// ref: https://github.com/ggerganov/llama.cpp/pull/6022#issuecomment-1992615670 +GGML_API GGML_CALL int ggml_backend_sycl_get_device_id(int device_index); +GGML_API GGML_CALL void ggml_backend_sycl_set_single_device_mode(int main_gpu_id); +GGML_API GGML_CALL void ggml_backend_sycl_set_mul_device_mode(); #ifdef __cplusplus } #endif diff --git a/llama.cpp b/llama.cpp index b8a8d2723..8e185d4bf 100644 --- a/llama.cpp +++ b/llama.cpp @@ -5064,6 +5064,16 @@ static int llama_model_load(const std::string & fname, llama_model & model, llam } #endif +#ifdef GGML_USE_SYCL + if (params.split_mode == LLAMA_SPLIT_MODE_NONE) { + ggml_backend_sycl_set_single_device_mode(params.main_gpu); + //SYCL use device index (0, 1, 2) directly, uer input device id, then convert to device index. + params.main_gpu = ggml_backend_sycl_get_device_index(params.main_gpu); + } else { + ggml_backend_sycl_set_mul_device_mode(); + } +#endif + if (!llm_load_tensors( ml, model, params.n_gpu_layers, params.split_mode, params.main_gpu, params.tensor_split, params.use_mlock, params.progress_callback, params.progress_callback_user_data @@ -12921,23 +12931,22 @@ struct llama_context * llama_new_context_with_model( if (model->n_gpu_layers > 0) { // with split_mode LLAMA_SPLIT_MODE_NONE or LLAMA_SPLIT_MODE_ROW, only the main GPU backend is used if (model->split_mode == LLAMA_SPLIT_MODE_NONE || model->split_mode == LLAMA_SPLIT_MODE_ROW) { - int main_gpu_index = ggml_backend_sycl_get_device_index(model->main_gpu); - ggml_backend_t backend = ggml_backend_sycl_init(main_gpu_index); + ggml_backend_t backend = ggml_backend_sycl_init(model->main_gpu); if (backend == nullptr) { - LLAMA_LOG_ERROR("%s: failed to initialize SYCL%d (index %d)backend\n", __func__, model->main_gpu, main_gpu_index); + int main_gpu_id = ggml_backend_sycl_get_device_id(model->main_gpu); + LLAMA_LOG_ERROR("%s: failed to initialize SYCL%d (index %d) backend\n", __func__, main_gpu_id, model->main_gpu); llama_free(ctx); return nullptr; } ctx->backends.push_back(backend); } else { // LLAMA_SPLIT_LAYER requires a backend for each GPU - int id_list[GGML_SYCL_MAX_DEVICES]; - ggml_sycl_get_gpu_list(id_list, GGML_SYCL_MAX_DEVICES); for (int i = 0; i < ggml_backend_sycl_get_device_count(); ++i) { - int device_id = id_list[i]; ggml_backend_t backend = ggml_backend_sycl_init(i); if (backend == nullptr) { - LLAMA_LOG_ERROR("%s: failed to initialize SYCL%d (index %d)backend\n", __func__, device_id, i); + int id_list[GGML_SYCL_MAX_DEVICES]; + ggml_sycl_get_gpu_list(id_list, GGML_SYCL_MAX_DEVICES); + LLAMA_LOG_ERROR("%s: failed to initialize SYCL%d (index %d) backend\n", __func__, id_list[i], i); llama_free(ctx); return nullptr; } From 3020327f6cd6d2ce50528dd65f4b199d2ea8b1ae Mon Sep 17 00:00:00 2001 From: slaren Date: Fri, 15 Mar 2024 13:24:03 +0100 Subject: [PATCH 634/812] cuda : disable unused cudaLaunchHostFunc code (#6078) --- ggml-cuda.cu | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ggml-cuda.cu b/ggml-cuda.cu index d1b5e52ba..db595409a 100644 --- a/ggml-cuda.cu +++ b/ggml-cuda.cu @@ -11541,6 +11541,7 @@ static void ggml_backend_cuda_event_wait(ggml_backend_t backend, ggml_backend_ev if (ggml_backend_is_cuda(event->backend)) { CUDA_CHECK(cudaStreamWaitEvent(g_cudaStreams[cuda_ctx->device][0], (cudaEvent_t)event->context, 0)); } else { +#if 0 // untested auto wait_fn = [](void * user_data) { ggml_backend_event_t event = (ggml_backend_event_t)user_data; @@ -11548,6 +11549,8 @@ static void ggml_backend_cuda_event_wait(ggml_backend_t backend, ggml_backend_ev }; CUDA_CHECK(cudaLaunchHostFunc(g_cudaStreams[cuda_ctx->device][0], wait_fn, event)); +#endif + GGML_ASSERT(false); } } From 4e9a7f7f7fb6acbddd1462909c8d696e38edbfcc Mon Sep 17 00:00:00 2001 From: Ting Lou Date: Fri, 15 Mar 2024 22:31:05 +0800 Subject: [PATCH 635/812] llava : change API to pure C style for Rust FFI bindgen (#6079) Co-authored-by: Lou Ting --- examples/llava/clip.cpp | 36 ++++++++++++++++++------------------ examples/llava/clip.h | 6 +++--- examples/llava/llava.cpp | 2 +- examples/llava/llava.h | 4 ++-- 4 files changed, 24 insertions(+), 24 deletions(-) diff --git a/examples/llava/clip.cpp b/examples/llava/clip.cpp index 2035554ea..a0ed82d7e 100644 --- a/examples/llava/clip.cpp +++ b/examples/llava/clip.cpp @@ -1235,16 +1235,16 @@ struct clip_image_f32 * clip_image_f32_init() { void clip_image_u8_free(struct clip_image_u8 * img) { delete img; } void clip_image_f32_free(struct clip_image_f32 * img) { delete img; } -void clip_image_u8_batch_free(struct clip_image_u8_batch & batch) { - if (batch.size > 0) { - delete[] batch.data; - batch.size = 0; +void clip_image_u8_batch_free(struct clip_image_u8_batch * batch) { + if (batch->size > 0) { + delete[] batch->data; + batch->size = 0; } } -void clip_image_f32_batch_free(struct clip_image_f32_batch & batch) { - if (batch.size > 0) { - delete[] batch.data; - batch.size = 0; +void clip_image_f32_batch_free(struct clip_image_f32_batch * batch) { + if (batch->size > 0) { + delete[] batch->data; + batch->size = 0; } } @@ -1497,7 +1497,7 @@ static std::vector divide_to_patches_u8(const clip_image_u8 & im // returns the normalized float tensor for llava-1.5, for spatial_unpad with anyres processing for llava-1.6 it returns the normalized image patch tensors as a vector // res_imgs memory is being allocated here, previous allocations will be freed if found -bool clip_image_preprocess(struct clip_ctx * ctx, const clip_image_u8 * img, clip_image_f32_batch & res_imgs) { +bool clip_image_preprocess(struct clip_ctx * ctx, const clip_image_u8 * img, clip_image_f32_batch * res_imgs) { bool pad_to_square = true; if (!ctx->has_vision_encoder) { printf("This gguf file seems to have no vision encoder\n"); @@ -1509,11 +1509,11 @@ bool clip_image_preprocess(struct clip_ctx * ctx, const clip_image_u8 * img, cli pad_to_square = false; } // free the previous res_imgs if any set - if (res_imgs.size > 0) { + if (res_imgs->size > 0) { clip_image_f32_batch_free(res_imgs); } - res_imgs.data = nullptr; - res_imgs.size = 0; + res_imgs->data = nullptr; + res_imgs->size = 0; // the logic below is to pad the shorter side to the longer side with a background color: rgb(122, 116, 104) // see https://github.com/haotian-liu/LLaVA/blob/e854a2bf85118c504f6f16bf5c3c7c92f8fa8c6b/llava/conversation.py#L113-L156 @@ -1568,11 +1568,11 @@ bool clip_image_preprocess(struct clip_ctx * ctx, const clip_image_u8 * img, cli bicubic_resize(*img, *image_original_resize, params.image_size, params.image_size); // in python this is "shortest_edge", but all CLIP are square patches.insert(patches.begin(), image_original_resize); // clip_image_f32_batch_init(patches.size()); - res_imgs.size = patches.size(); - res_imgs.data = new clip_image_f32[res_imgs.size]; + res_imgs->size = patches.size(); + res_imgs->data = new clip_image_f32[res_imgs->size]; int num=0; for (auto& patch : patches) { - normalize_image_u8_to_f32(patch, &res_imgs.data[num], ctx->image_mean, ctx->image_std); + normalize_image_u8_to_f32(patch, &res_imgs->data[num], ctx->image_mean, ctx->image_std); num++; } @@ -1660,9 +1660,9 @@ bool clip_image_preprocess(struct clip_ctx * ctx, const clip_image_u8 * img, cli // } // res_imgs.push_back(res); - res_imgs.size = 1; - res_imgs.data = new clip_image_f32[res_imgs.size]; - res_imgs.data[0] = *res; + res_imgs->size = 1; + res_imgs->data = new clip_image_f32[res_imgs->size]; + res_imgs->data[0] = *res; clip_image_f32_free(res); return true; diff --git a/examples/llava/clip.h b/examples/llava/clip.h index e5bd54924..45bdad689 100644 --- a/examples/llava/clip.h +++ b/examples/llava/clip.h @@ -60,8 +60,8 @@ CLIP_API struct clip_image_f32 * clip_image_f32_init(); CLIP_API void clip_image_u8_free (struct clip_image_u8 * img); CLIP_API void clip_image_f32_free(struct clip_image_f32 * img); -CLIP_API void clip_image_u8_batch_free (struct clip_image_u8_batch & batch); -CLIP_API void clip_image_f32_batch_free(struct clip_image_f32_batch & batch); +CLIP_API void clip_image_u8_batch_free (struct clip_image_u8_batch * batch); +CLIP_API void clip_image_f32_batch_free(struct clip_image_f32_batch * batch); CLIP_API bool clip_image_load_from_file(const char * fname, struct clip_image_u8 * img); @@ -69,7 +69,7 @@ CLIP_API bool clip_image_load_from_file(const char * fname, struct clip_image_u8 CLIP_API bool clip_image_load_from_bytes(const unsigned char * bytes, size_t bytes_length, struct clip_image_u8 * img); /** preprocess img and store the result in res_imgs, pad_to_square may be overriden to false depending on model configuration */ -CLIP_API bool clip_image_preprocess(struct clip_ctx * ctx, const clip_image_u8 * img, clip_image_f32_batch & res_imgs ); +CLIP_API bool clip_image_preprocess(struct clip_ctx * ctx, const struct clip_image_u8 * img, struct clip_image_f32_batch * res_imgs ); CLIP_API struct ggml_tensor * clip_get_newline_tensor(const struct clip_ctx * ctx); diff --git a/examples/llava/llava.cpp b/examples/llava/llava.cpp index 980128166..29764757a 100644 --- a/examples/llava/llava.cpp +++ b/examples/llava/llava.cpp @@ -223,7 +223,7 @@ static bool encode_image_with_clip(clip_ctx * ctx_clip, int n_threads, const cli clip_image_f32_batch img_res_v; img_res_v.size = 0; img_res_v.data = nullptr; - if (!clip_image_preprocess(ctx_clip, img, img_res_v)) { + if (!clip_image_preprocess(ctx_clip, img, &img_res_v)) { fprintf(stderr, "%s: unable to preprocess image\n", __func__); delete[] img_res_v.data; return false; diff --git a/examples/llava/llava.h b/examples/llava/llava.h index 2d40f3f1d..19212f6e9 100644 --- a/examples/llava/llava.h +++ b/examples/llava/llava.h @@ -29,9 +29,9 @@ struct llava_image_embed { }; /** sanity check for clip <-> llava embed size match */ -LLAVA_API bool llava_validate_embed_size(const llama_context * ctx_llama, const clip_ctx * ctx_clip); +LLAVA_API bool llava_validate_embed_size(const struct llama_context * ctx_llama, const struct clip_ctx * ctx_clip); -LLAVA_API bool llava_image_embed_make_with_clip_img(clip_ctx * ctx_clip, int n_threads, const clip_image_u8 * img, float ** image_embd_out, int * n_img_pos_out); +LLAVA_API bool llava_image_embed_make_with_clip_img(struct clip_ctx * ctx_clip, int n_threads, const struct clip_image_u8 * img, float ** image_embd_out, int * n_img_pos_out); /** build an image embed from image file bytes */ LLAVA_API struct llava_image_embed * llava_image_embed_make_with_bytes(struct clip_ctx * ctx_clip, int n_threads, const unsigned char * image_bytes, int image_bytes_length); From 12247f4c69a173b9482f68aaa174ec37fc909ccf Mon Sep 17 00:00:00 2001 From: Andrew Canis Date: Fri, 15 Mar 2024 16:41:22 -0400 Subject: [PATCH 636/812] llama : add Command-R support (#6033) Information about the Command-R 35B model (128k context) can be found at: https://huggingface.co/CohereForAI/c4ai-command-r-v01 Based on the llama2 model with a few changes: 1) New hyper parameter to scale output logits (logit_scale) 2) Uses LayerNorm instead of RMSNorm 3) Transfomer layers have a single shared LayerNorm that feeds into both the self-attention and FFN layers in parallel. There is no post-attention LayerNorm. 4) No support for Rotary Position Embeddings (RoPE) scaling 5) No biases used Find GGUF files here: https://huggingface.co/andrewcanis/c4ai-command-r-v01-GGUF To convert model to GGUF format yourself: 1) Download Command-R Hugging Face safetensors: git lfs install git clone https://huggingface.co/CohereForAI/c4ai-command-r-v01 2) Run: python3 convert-hf-to-gguf.py --outtype f16 ./c4ai-command-r-v01 --- README.md | 1 + convert-hf-to-gguf.py | 17 ++++ gguf-py/gguf/constants.py | 15 +++ gguf-py/gguf/gguf_writer.py | 3 + llama.cpp | 183 ++++++++++++++++++++++++++++++++++++ 5 files changed, 219 insertions(+) diff --git a/README.md b/README.md index 61bedc3f8..5cbdf7e47 100644 --- a/README.md +++ b/README.md @@ -112,6 +112,7 @@ Typically finetunes of the base models below are supported as well. - [x] [CodeShell](https://github.com/WisdomShell/codeshell) - [x] [Gemma](https://ai.google.dev/gemma) - [x] [Mamba](https://github.com/state-spaces/mamba) +- [x] [Command-R](https://huggingface.co/CohereForAI/c4ai-command-r-v01) **Multimodal models:** diff --git a/convert-hf-to-gguf.py b/convert-hf-to-gguf.py index 5eee32016..cf1f98d66 100755 --- a/convert-hf-to-gguf.py +++ b/convert-hf-to-gguf.py @@ -1965,6 +1965,23 @@ class MambaModel(Model): self.gguf_writer.add_tensor(new_name, data) +@Model.register("CohereForCausalLM") +class CommandR2Model(Model): + model_arch = gguf.MODEL_ARCH.COMMAND_R + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + # max_position_embeddings = 8192 in config.json but model was actually + # trained on 128k context length + self.hparams["max_position_embeddings"] = self.hparams["model_max_length"] + + def set_gguf_parameters(self): + super().set_gguf_parameters() + self.gguf_writer.add_logit_scale(self.hparams["logit_scale"]) + self.gguf_writer.add_rope_scaling_type(gguf.RopeScalingType.NONE) + + ###### CONVERSION LOGIC ###### diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py index 458a641dc..4a4facb06 100644 --- a/gguf-py/gguf/constants.py +++ b/gguf-py/gguf/constants.py @@ -42,6 +42,7 @@ class Keys: EXPERT_COUNT = "{arch}.expert_count" EXPERT_USED_COUNT = "{arch}.expert_used_count" POOLING_TYPE = "{arch}.pooling_type" + LOGIT_SCALE = "{arch}.logit_scale" class Attention: HEAD_COUNT = "{arch}.attention.head_count" @@ -121,6 +122,7 @@ class MODEL_ARCH(IntEnum): GEMMA = auto() STARCODER2 = auto() MAMBA = auto() + COMMAND_R = auto() class MODEL_TENSOR(IntEnum): @@ -187,6 +189,7 @@ MODEL_ARCH_NAMES: dict[MODEL_ARCH, str] = { MODEL_ARCH.GEMMA: "gemma", MODEL_ARCH.STARCODER2: "starcoder2", MODEL_ARCH.MAMBA: "mamba", + MODEL_ARCH.COMMAND_R: "command-r", } TENSOR_NAMES: dict[MODEL_TENSOR, str] = { @@ -579,6 +582,18 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = { MODEL_TENSOR.SSM_D, MODEL_TENSOR.SSM_OUT, ], + MODEL_ARCH.COMMAND_R: [ + MODEL_TENSOR.TOKEN_EMBD, + MODEL_TENSOR.OUTPUT_NORM, + MODEL_TENSOR.ATTN_NORM, + MODEL_TENSOR.ATTN_Q, + MODEL_TENSOR.ATTN_K, + MODEL_TENSOR.ATTN_V, + MODEL_TENSOR.ATTN_OUT, + MODEL_TENSOR.FFN_GATE, + MODEL_TENSOR.FFN_DOWN, + MODEL_TENSOR.FFN_UP, + ], # TODO } diff --git a/gguf-py/gguf/gguf_writer.py b/gguf-py/gguf/gguf_writer.py index 1967b633c..2ae6c814b 100644 --- a/gguf-py/gguf/gguf_writer.py +++ b/gguf-py/gguf/gguf_writer.py @@ -361,6 +361,9 @@ class GGUFWriter: def add_clamp_kqv(self, value: float) -> None: self.add_float32(Keys.Attention.CLAMP_KQV.format(arch=self.arch), value) + def add_logit_scale(self, value: float) -> None: + self.add_float32(Keys.LLM.LOGIT_SCALE.format(arch=self.arch), value) + def add_expert_count(self, count: int) -> None: self.add_uint32(Keys.LLM.EXPERT_COUNT.format(arch=self.arch), count) diff --git a/llama.cpp b/llama.cpp index 8e185d4bf..fc5dd5cb4 100644 --- a/llama.cpp +++ b/llama.cpp @@ -214,6 +214,7 @@ enum llm_arch { LLM_ARCH_GEMMA, LLM_ARCH_STARCODER2, LLM_ARCH_MAMBA, + LLM_ARCH_COMMAND_R, LLM_ARCH_UNKNOWN, }; @@ -243,6 +244,7 @@ static const std::map LLM_ARCH_NAMES = { { LLM_ARCH_GEMMA, "gemma" }, { LLM_ARCH_STARCODER2, "starcoder2" }, { LLM_ARCH_MAMBA, "mamba" }, + { LLM_ARCH_COMMAND_R, "command-r" }, { LLM_ARCH_UNKNOWN, "(unknown)" }, }; @@ -268,6 +270,7 @@ enum llm_kv { LLM_KV_EXPERT_COUNT, LLM_KV_EXPERT_USED_COUNT, LLM_KV_POOLING_TYPE, + LLM_KV_LOGIT_SCALE, LLM_KV_ATTENTION_HEAD_COUNT, LLM_KV_ATTENTION_HEAD_COUNT_KV, @@ -332,6 +335,7 @@ static const std::map LLM_KV_NAMES = { { LLM_KV_EXPERT_COUNT, "%s.expert_count" }, { LLM_KV_EXPERT_USED_COUNT, "%s.expert_used_count" }, { LLM_KV_POOLING_TYPE , "%s.pooling_type" }, + { LLM_KV_LOGIT_SCALE, "%s.logit_scale" }, { LLM_KV_ATTENTION_HEAD_COUNT, "%s.attention.head_count" }, { LLM_KV_ATTENTION_HEAD_COUNT_KV, "%s.attention.head_count_kv" }, @@ -838,6 +842,21 @@ static const std::map> LLM_TENSOR_NA { LLM_TENSOR_SSM_OUT, "blk.%d.ssm_out" }, }, }, + { + LLM_ARCH_COMMAND_R, + { + { LLM_TENSOR_TOKEN_EMBD, "token_embd" }, + { LLM_TENSOR_OUTPUT_NORM, "output_norm" }, + { LLM_TENSOR_ATTN_NORM, "blk.%d.attn_norm" }, + { LLM_TENSOR_ATTN_Q, "blk.%d.attn_q" }, + { LLM_TENSOR_ATTN_K, "blk.%d.attn_k" }, + { LLM_TENSOR_ATTN_V, "blk.%d.attn_v" }, + { LLM_TENSOR_ATTN_OUT, "blk.%d.attn_output" }, + { LLM_TENSOR_FFN_GATE, "blk.%d.ffn_gate" }, + { LLM_TENSOR_FFN_DOWN, "blk.%d.ffn_down" }, + { LLM_TENSOR_FFN_UP, "blk.%d.ffn_up" }, + }, + }, { LLM_ARCH_UNKNOWN, { @@ -1597,6 +1616,7 @@ enum e_model { MODEL_20B, MODEL_30B, MODEL_34B, + MODEL_35B, MODEL_40B, MODEL_65B, MODEL_70B, @@ -1643,6 +1663,7 @@ struct llama_hparams { float f_clamp_kqv = 0.0f; float f_max_alibi_bias = 0.0f; + float f_logit_scale = 0.0f; bool causal_attn = true; bool need_kq_pos = false; @@ -3231,6 +3252,7 @@ static const char * llama_model_type_name(e_model type) { case MODEL_20B: return "20B"; case MODEL_30B: return "30B"; case MODEL_34B: return "34B"; + case MODEL_35B: return "35B"; case MODEL_40B: return "40B"; case MODEL_65B: return "65B"; case MODEL_70B: return "70B"; @@ -3623,6 +3645,15 @@ static void llm_load_hparams( default: model.type = e_model::MODEL_UNKNOWN; } } break; + case LLM_ARCH_COMMAND_R: + { + ml.get_key(LLM_KV_LOGIT_SCALE, hparams.f_logit_scale); + ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps); + switch (hparams.n_layer) { + case 40: model.type = e_model::MODEL_35B; break; + default: model.type = e_model::MODEL_UNKNOWN; + } + } break; default: (void)0; } @@ -3944,6 +3975,7 @@ static void llm_load_print_meta(llama_model_loader & ml, llama_model & model) { LLAMA_LOG_INFO("%s: f_norm_rms_eps = %.1e\n", __func__, hparams.f_norm_rms_eps); LLAMA_LOG_INFO("%s: f_clamp_kqv = %.1e\n", __func__, hparams.f_clamp_kqv); LLAMA_LOG_INFO("%s: f_max_alibi_bias = %.1e\n", __func__, hparams.f_max_alibi_bias); + LLAMA_LOG_INFO("%s: f_logit_scale = %.1e\n", __func__, hparams.f_logit_scale); LLAMA_LOG_INFO("%s: n_ff = %u\n", __func__, hparams.n_ff); LLAMA_LOG_INFO("%s: n_expert = %u\n", __func__, hparams.n_expert); LLAMA_LOG_INFO("%s: n_expert_used = %u\n", __func__, hparams.n_expert_used); @@ -4918,6 +4950,37 @@ static bool llm_load_tensors( layer.ssm_out = ml.create_tensor(ctx_split, tn(LLM_TENSOR_SSM_OUT, "weight", i), {d_inner, n_embd}); } } break; + case LLM_ARCH_COMMAND_R: + { + model.tok_embd = ml.create_tensor(ctx_input, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}); + + // output + { + model.output_norm = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}); + // init output from the input tok embed + model.output = ml.create_tensor(ctx_output, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}); + ml.n_created--; // artificial tensor + ml.size_data += ggml_nbytes(model.output); + } + + for (int i = 0; i < n_layer; ++i) { + ggml_context * ctx_layer = ctx_for_layer(i); + ggml_context * ctx_split = ctx_for_layer_split(i); + + auto & layer = model.layers[i]; + + layer.attn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}); + + layer.wq = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_embd}); + layer.wk = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_K, "weight", i), {n_embd, n_embd_gqa}); + layer.wv = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_V, "weight", i), {n_embd, n_embd_gqa}); + layer.wo = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd}); + + layer.ffn_gate = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff}); + layer.ffn_down = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_DOWN, "weight", i), { n_ff, n_embd}); + layer.ffn_up = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff}); + } + } break; default: throw std::runtime_error("unknown architecture"); } @@ -8315,6 +8378,121 @@ struct llm_build_context { return gf; } + + struct ggml_cgraph * build_command_r() { + + struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, LLAMA_MAX_NODES, false); + + const int64_t n_embd_head = hparams.n_embd_head_v; + GGML_ASSERT(n_embd_head == hparams.n_embd_head_k); + const float f_logit_scale = hparams.f_logit_scale; + + struct ggml_tensor * cur; + struct ggml_tensor * inpL; + + inpL = llm_build_inp_embd(ctx0, lctx, hparams, batch, model.tok_embd, cb); + + // inp_pos - contains the positions + struct ggml_tensor * inp_pos = build_inp_pos(); + + // KQ_mask (mask for 1 head, it will be broadcasted to all heads) + struct ggml_tensor * KQ_mask = build_inp_KQ_mask(); + + for (int il = 0; il < n_layer; ++il) { + + // norm + cur = llm_build_norm(ctx0, inpL, hparams, + model.layers[il].attn_norm, NULL, + LLM_NORM, cb, il); + cb(cur, "attn_norm", il); + struct ggml_tensor * ffn_inp = cur; + + // self-attention + { + // compute Q and K and RoPE them + struct ggml_tensor * Qcur = ggml_mul_mat(ctx0, model.layers[il].wq, cur); + cb(Qcur, "Qcur", il); + if (model.layers[il].bq) { + Qcur = ggml_add(ctx0, Qcur, model.layers[il].bq); + cb(Qcur, "Qcur", il); + } + + struct ggml_tensor * Kcur = ggml_mul_mat(ctx0, model.layers[il].wk, cur); + cb(Kcur, "Kcur", il); + if (model.layers[il].bk) { + Kcur = ggml_add(ctx0, Kcur, model.layers[il].bk); + cb(Kcur, "Kcur", il); + } + + struct ggml_tensor * Vcur = ggml_mul_mat(ctx0, model.layers[il].wv, cur); + cb(Vcur, "Vcur", il); + if (model.layers[il].bv) { + Vcur = ggml_add(ctx0, Vcur, model.layers[il].bv); + cb(Vcur, "Vcur", il); + } + + Qcur = ggml_rope_custom( + ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, + n_rot, rope_type, 0, n_orig_ctx, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow + ); + cb(Qcur, "Qcur", il); + + Kcur = ggml_rope_custom( + ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, + n_rot, rope_type, 0, n_orig_ctx, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow + ); + cb(Kcur, "Kcur", il); + + cur = llm_build_kv(ctx0, model, hparams, kv_self, gf, + model.layers[il].wo, model.layers[il].bo, + Kcur, Vcur, Qcur, KQ_mask, nullptr, n_ctx, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il); + } + + struct ggml_tensor * attn_out = cur; + + // feed-forward network + { + cur = llm_build_ffn(ctx0, ffn_inp, + model.layers[il].ffn_up, NULL, + model.layers[il].ffn_gate, NULL, + model.layers[il].ffn_down, NULL, + NULL, + LLM_FFN_SILU, LLM_FFN_PAR, cb, il); + cb(cur, "ffn_out", il); + } + + // add together residual + FFN + self-attention + cur = ggml_add(ctx0, cur, inpL); + cur = ggml_add(ctx0, cur, attn_out); + cb(cur, "l_out", il); + + // input for next layer + inpL = cur; + } + + cur = inpL; + + cur = llm_build_norm(ctx0, cur, hparams, + model.output_norm, NULL, + LLM_NORM, cb, -1); + cb(cur, "result_norm", -1); + + // lm_head + cur = ggml_mul_mat(ctx0, model.output, cur); + + if (f_logit_scale) { + cur = ggml_scale(ctx0, cur, f_logit_scale); + } + + cb(cur, "result_output", -1); + + ggml_build_forward_expand(gf, cur); + + return gf; + + } }; static struct ggml_cgraph * llama_build_graph_defrag(llama_context & lctx, const std::vector & ids) { @@ -8497,6 +8675,10 @@ static struct ggml_cgraph * llama_build_graph( { result = llm.build_mamba(); } break; + case LLM_ARCH_COMMAND_R: + { + result = llm.build_command_r(); + } break; default: GGML_ASSERT(false); } @@ -13147,6 +13329,7 @@ enum llama_rope_type llama_rope_type(const struct llama_model * model) { case LLM_ARCH_ORION: case LLM_ARCH_INTERNLM2: case LLM_ARCH_MINICPM: + case LLM_ARCH_COMMAND_R: return LLAMA_ROPE_TYPE_NORM; // the pairs of head values are offset by n_rot/2 From 877b4d0c628cc70dddb5df72ed8fc14d126ca7e8 Mon Sep 17 00:00:00 2001 From: Theia Vogel Date: Fri, 15 Mar 2024 13:43:02 -0700 Subject: [PATCH 637/812] llama : add support for control vectors (#5970) * control vector api and implementation * control-vectors : minor code style updates * disable control vector when data == nullptr use -1 for disabled range (also on init) in case we ever support controlling layer 0 (embeddings) --------- Co-authored-by: Georgi Gerganov --- common/common.cpp | 215 ++++++++++++++++++++++++++++++++++++++++++++++ common/common.h | 31 ++++++- llama.cpp | 128 +++++++++++++++++++++++++++ llama.h | 23 ++++- 4 files changed, 392 insertions(+), 5 deletions(-) diff --git a/common/common.cpp b/common/common.cpp index 58fbd05aa..4912237e0 100644 --- a/common/common.cpp +++ b/common/common.cpp @@ -568,6 +568,34 @@ bool gpt_params_parse_ex(int argc, char ** argv, gpt_params & params) { break; } params.lora_base = argv[i]; + } else if (arg == "--control-vector") { + if (++i >= argc) { + invalid_param = true; + break; + } + params.control_vectors.push_back({ 1.0f, argv[i], }); + } else if (arg == "--control-vector-scaled") { + if (++i >= argc) { + invalid_param = true; + break; + } + const char * fname = argv[i]; + if (++i >= argc) { + invalid_param = true; + break; + } + params.control_vectors.push_back({ std::stof(argv[i]), fname, }); + } else if (arg == "--control-vector-layer-range") { + if (++i >= argc) { + invalid_param = true; + break; + } + params.control_vector_layer_start = std::stoi(argv[i]); + if (++i >= argc) { + invalid_param = true; + break; + } + params.control_vector_layer_end = std::stoi(argv[i]); } else if (arg == "--mmproj") { if (++i >= argc) { invalid_param = true; @@ -1095,6 +1123,12 @@ void gpt_print_usage(int /*argc*/, char ** argv, const gpt_params & params) { printf(" --lora FNAME apply LoRA adapter (implies --no-mmap)\n"); printf(" --lora-scaled FNAME S apply LoRA adapter with user defined scaling S (implies --no-mmap)\n"); printf(" --lora-base FNAME optional model to use as a base for the layers modified by the LoRA adapter\n"); + printf(" --control-vector FNAME\n"); + printf(" add a control vector\n"); + printf(" --control-vector-scaled FNAME S\n"); + printf(" add a control vector with user defined scaling S\n"); + printf(" --control-vector-layer-range START END\n"); + printf(" layer range to apply the control vector(s) to, start and end inclusive\n"); printf(" -m FNAME, --model FNAME\n"); printf(" model path (default: %s)\n", params.model.c_str()); printf(" -md FNAME, --model-draft FNAME\n"); @@ -1360,6 +1394,30 @@ std::tuple llama_init_from_gpt_par return std::make_tuple(nullptr, nullptr); } + if (!params.control_vectors.empty()) { + if (params.control_vector_layer_start <= 0) params.control_vector_layer_start = 1; + if (params.control_vector_layer_end <= 0) params.control_vector_layer_end = llama_n_layer(model); + + const auto cvec = llama_control_vector_load(params.control_vectors); + if (cvec.n_embd == -1) { + llama_free(lctx); + llama_free_model(model); + return std::make_tuple(nullptr, nullptr); + } + + int err = llama_control_vector_apply(lctx, + cvec.data.data(), + cvec.data.size(), + cvec.n_embd, + params.control_vector_layer_start, + params.control_vector_layer_end); + if (err) { + llama_free(lctx); + llama_free_model(model); + return std::make_tuple(nullptr, nullptr); + } + } + for (unsigned int i = 0; i < params.lora_adapter.size(); ++i) { const std::string& lora_adapter = std::get<0>(params.lora_adapter[i]); float lora_scale = std::get<1>(params.lora_adapter[i]); @@ -1890,3 +1948,160 @@ float llama_embd_similarity_cos(const float * embd1, const float * embd2, int n) return sum / (sqrt(sum1) * sqrt(sum2)); } + +// +// Control vector utils +// + +static llama_control_vector_data llama_control_vector_load_one(const llama_control_vector_load_info & load_info) { + int32_t n_tensors; + + size_t n_bytes = 0; + + uint32_t max_direction_layer = 0; + + llama_control_vector_data result = { -1, {} }; + + // calculate size of ctx needed for tensors, ensure tensors are f32, and find max layer + { + struct ggml_init_params meta_params = { + /* .mem_size = */ ggml_tensor_overhead() * 128 + ggml_graph_overhead(), + /* .mem_buffer = */ nullptr, + /* .no_alloc = */ true, + }; + ggml_context * meta_ctx = ggml_init(meta_params); + struct gguf_init_params meta_gguf_params = { + /* .no_alloc = */ true, + /* .ctx = */ &meta_ctx, + }; + struct gguf_context * meta_ctx_gguf = gguf_init_from_file(load_info.fname.c_str(), meta_gguf_params); + if (!meta_ctx_gguf) { + fprintf(stderr, "%s: failed to load control vector from %s\n", __func__, load_info.fname.c_str()); + ggml_free(meta_ctx); + return result; + } + + n_tensors = gguf_get_n_tensors(meta_ctx_gguf); + for (int i = 0; i < n_tensors; i++) { + std::string name = gguf_get_tensor_name(meta_ctx_gguf, i); + + // split on '.' + size_t dotpos = name.find('.'); + if (dotpos != std::string::npos && name.substr(0, dotpos) == "direction") { + try { + uint32_t layer = std::stoi(name.substr(dotpos + 1)); + if (layer == 0) { + fprintf(stderr, "%s: direction tensor invalid in %s\n", __func__, load_info.fname.c_str()); + ggml_free(meta_ctx); + gguf_free(meta_ctx_gguf); + return result; + } + if (layer > max_direction_layer) { + max_direction_layer = layer; + } + } catch (...) { + fprintf(stderr, "%s: direction tensor invalid in %s\n", __func__, load_info.fname.c_str()); + ggml_free(meta_ctx); + gguf_free(meta_ctx_gguf); + return result; + } + } + + struct ggml_tensor * tensor_meta = ggml_get_tensor(meta_ctx, name.c_str()); + if (tensor_meta->type != GGML_TYPE_F32 || ggml_n_dims(tensor_meta) != 1) { + fprintf(stderr, "%s: direction tensor invalid in %s\n", __func__, load_info.fname.c_str()); + ggml_free(meta_ctx); + gguf_free(meta_ctx_gguf); + return result; + } + if (result.n_embd == -1) { + result.n_embd = ggml_nelements(tensor_meta); + } else if (ggml_nelements(tensor_meta) != result.n_embd) { + fprintf(stderr, "%s: direction tensor sizes mismatched in %s\n", __func__, load_info.fname.c_str()); + ggml_free(meta_ctx); + gguf_free(meta_ctx_gguf); + return result; + } + n_bytes += ggml_nbytes(tensor_meta); + } + ggml_free(meta_ctx); + gguf_free(meta_ctx_gguf); + } + + if (n_tensors == 0) { + fprintf(stderr, "%s: no direction tensors found in %s\n", __func__, load_info.fname.c_str()); + return result; + } + + // load and scale tensors into final control vector context + struct ggml_init_params ggml_params = { + /* .mem_size = */ ggml_tensor_overhead() * n_tensors + n_bytes, + /* .mem_buffer = */ nullptr, + /* .no_alloc = */ false, + }; + struct ggml_context * ctx = ggml_init(ggml_params); + + struct gguf_init_params params = { + /*.no_alloc = */ false, + /*.ctx = */ &ctx, + }; + struct gguf_context * ctx_gguf = gguf_init_from_file(load_info.fname.c_str(), params); + if (!ctx_gguf) { + fprintf(stderr, "%s: failed to load control vector from %s\n", __func__, load_info.fname.c_str()); + ggml_free(ctx); + return result; + } + + // do not store data for layer 0 (it's not used) + result.data.resize(result.n_embd * max_direction_layer); + + for (uint32_t il = 1; il <= max_direction_layer; il++) { + const std::string name = "direction." + std::to_string(il); + const ggml_tensor * tensor = ggml_get_tensor(ctx, name.c_str()); + + float * dst = result.data.data() + result.n_embd * (il - 1); + + if (tensor) { + const float * src = (const float *) tensor->data; + for (int j = 0; j < result.n_embd; j++) { + dst[j] = src[j] * load_info.strength; + } + } else { + for (int j = 0; j < result.n_embd; j++) { + dst[j] = 0.0f; + } + } + } + + return result; +} + +llama_control_vector_data llama_control_vector_load(const std::vector & load_infos) { + llama_control_vector_data result = { -1, {} }; + + for (const auto & info : load_infos) { + auto cur = llama_control_vector_load_one(info); + + if (cur.n_embd == -1) { + return result; + } + if (result.n_embd != -1 && (result.n_embd != cur.n_embd || result.data.size() != cur.data.size())) { + fprintf(stderr, "%s: control vector in %s does not match previous vector dimensions\n", __func__, info.fname.c_str()); + return result; + } + + if (result.n_embd == -1) { + result = std::move(cur); + } else { + for (size_t i = 0; i < cur.data.size(); i++) { + result.data[i] += cur.data[i]; + } + } + } + + if (result.n_embd == -1) { + fprintf(stderr, "%s: no vectors passed\n", __func__); + } + + return result; +} diff --git a/common/common.h b/common/common.h index d250eef8b..687f3425e 100644 --- a/common/common.h +++ b/common/common.h @@ -37,10 +37,13 @@ extern char const *LLAMA_COMMIT; extern char const *LLAMA_COMPILER; extern char const *LLAMA_BUILD_TARGET; +struct llama_control_vector_load_info; + +int32_t get_num_physical_cores(); + // // CLI argument parsing // -int32_t get_num_physical_cores(); struct gpt_params { uint32_t seed = LLAMA_DEFAULT_SEED; // RNG seed @@ -103,6 +106,11 @@ struct gpt_params { std::vector> lora_adapter; // lora adapter path with user defined scale std::string lora_base = ""; // base model path for the lora adapter + std::vector control_vectors; // control vector with user defined scale + + int32_t control_vector_layer_start = -1; // layer range for control vector + int32_t control_vector_layer_end = -1; // layer range for control vector + int ppl_stride = 0; // stride for perplexity calculations. If left at 0, the pre-existing approach will be used. int ppl_output_type = 0; // = 0 -> ppl output is as usual, = 1 -> ppl output is num_tokens, ppl, one per line // (which is more convenient to use for plotting) @@ -269,3 +277,24 @@ void dump_kv_cache_view_seqs(const llama_kv_cache_view & view, int row_size = 40 void llama_embd_normalize(const float * inp, float * out, int n); float llama_embd_similarity_cos(const float * embd1, const float * embd2, int n); + +// +// Control vector utils +// + +struct llama_control_vector_data { + int n_embd; + + // stores data for layers [1, n_layer] where n_layer = data.size() / n_embd + std::vector data; +}; + +struct llama_control_vector_load_info { + float strength; + + std::string fname; +}; + +// Load control vectors, scale each by strength, and add them together. +// On error, returns {-1, empty} +llama_control_vector_data llama_control_vector_load(const std::vector & load_infos); diff --git a/llama.cpp b/llama.cpp index fc5dd5cb4..52bd718ba 100644 --- a/llama.cpp +++ b/llama.cpp @@ -1894,6 +1894,31 @@ struct llama_kv_cache { } }; +struct llama_control_vector { + std::vector tensors; // per layer + std::vector ctxs; + std::vector bufs; + + int32_t layer_start = -1; + int32_t layer_end = -1; + + ggml_tensor * tensor_for(int il) const { + if (il < 0 || il < layer_start || il > layer_end || (size_t) il >= tensors.size()) { + return nullptr; + } + return tensors[il]; + } + + ~llama_control_vector() { + for (struct ggml_context * ctx : ctxs) { + ggml_free(ctx); + } + for (ggml_backend_buffer_t buf : bufs) { + ggml_backend_buffer_free(buf); + } + } +}; + struct llama_vocab { using id = int32_t; using token = std::string; @@ -2108,6 +2133,9 @@ struct llama_context { struct ggml_tensor * inp_s_mask; // F32 [1, kv_size] struct ggml_tensor * inp_s_seq; // I32 [kv_size, n_batch] + // control vectors + struct llama_control_vector cvec; + #ifdef GGML_USE_MPI ggml_mpi_context * ctx_mpi = NULL; #endif @@ -5931,6 +5959,12 @@ struct llm_build_context { } cur = ggml_add(ctx0, cur, ffn_inp); + cb(cur, "ffn_out", il); + + ggml_tensor * layer_dir = lctx.cvec.tensor_for(il); + if (layer_dir != nullptr) { + cur = ggml_add(ctx0, cur, layer_dir); + } cb(cur, "l_out", il); // input for next layer @@ -13366,6 +13400,10 @@ int32_t llama_n_embd(const struct llama_model * model) { return model->hparams.n_embd; } +int32_t llama_n_layer(const struct llama_model * model) { + return model->hparams.n_layer; +} + float llama_rope_freq_scale_train(const struct llama_model * model) { return model->hparams.rope_freq_scale_train; } @@ -13465,6 +13503,96 @@ int32_t llama_model_apply_lora_from_file(const struct llama_model * model, const } } +static bool llama_control_vector_init(struct llama_control_vector & cvec, const llama_model & model) { + GGML_ASSERT(cvec.tensors.empty()); + GGML_ASSERT(cvec.ctxs.empty()); + GGML_ASSERT(cvec.bufs.empty()); + + // count layer buffer types + std::map buft_layer_count; + for (int64_t i = 0; i < model.hparams.n_layer; i++) { + buft_layer_count[model.buft_layer[i].buft]++; + } + + // allocate contexts + std::map ctx_map; + for (auto & it : buft_layer_count) { + int n_layers = it.second; + struct ggml_init_params params = { + /*.mem_size =*/ n_layers * ggml_tensor_overhead(), + /*.mem_buffer =*/ NULL, + /*.no_alloc =*/ true, + }; + ggml_context * ctx = ggml_init(params); + if (!ctx) { + LLAMA_LOG_ERROR("%s: failed to allocate context for control vector\n", __func__); + return 1; + } + ctx_map[it.first] = ctx; + } + + // make tensors + cvec.tensors.push_back(nullptr); // there's never a tensor for layer 0 + for (size_t il = 1; il < model.hparams.n_layer; il++) { + struct ggml_context * ctx = ctx_map.at(model.buft_layer[il].buft); + ggml_tensor * tensor = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, model.hparams.n_embd); + cvec.tensors.push_back(tensor); + } + + // allocate tensors / buffers and zero + for (auto it : ctx_map) { + ggml_backend_buffer_type_t buft = it.first; + ggml_context * ctx = it.second; + ggml_backend_buffer_t buf = ggml_backend_alloc_ctx_tensors_from_buft(ctx, buft); + if (!buf) { + LLAMA_LOG_ERROR("%s: failed to allocate buffer for control vector\n", __func__); + return false; + } + ggml_backend_buffer_clear(buf, 0); + cvec.ctxs.push_back(ctx); + cvec.bufs.push_back(buf); + } + + return true; +} + +int32_t llama_control_vector_apply(struct llama_context * lctx, const float * data, size_t len, int32_t n_embd, int32_t il_start, int32_t il_end) { + const llama_model & model = lctx->model; + llama_control_vector & cvec = lctx->cvec; + + if (data == nullptr) { + // disable the current control vector (but leave allocated for later) + cvec.layer_start = -1; + cvec.layer_end = -1; + return 0; + } + + if (n_embd != (int) model.hparams.n_embd) { + LLAMA_LOG_ERROR("%s: control vector n_embd does not match model\n", __func__); + return 1; + } + + if (cvec.tensors.empty()) { + if (!llama_control_vector_init(cvec, model)) { + return 1; + } + } + + cvec.layer_start = il_start; + cvec.layer_end = il_end; + + for (size_t il = 1; il < model.hparams.n_layer; il++) { + assert(cvec.tensors[il] != nullptr); + + const size_t off = n_embd * (il - 1); // buffer doesn't have data for layer 0, since it's never present + if (off + n_embd <= len) { + ggml_backend_tensor_set(cvec.tensors[il], data + off, 0, n_embd * ggml_element_size(cvec.tensors[il])); + } + } + + return 0; +} + struct llama_kv_cache_view llama_kv_cache_view_init(const struct llama_context * ctx, int32_t n_seq_max) { struct llama_kv_cache_view result = { /*.n_cells = */ 0, diff --git a/llama.h b/llama.h index 90aa5372e..40dcf54e3 100644 --- a/llama.h +++ b/llama.h @@ -388,6 +388,7 @@ extern "C" { LLAMA_API int32_t llama_n_vocab (const struct llama_model * model); LLAMA_API int32_t llama_n_ctx_train(const struct llama_model * model); LLAMA_API int32_t llama_n_embd (const struct llama_model * model); + LLAMA_API int32_t llama_n_layer (const struct llama_model * model); // Get the model's RoPE frequency scaling factor LLAMA_API float llama_rope_freq_scale_train(const struct llama_model * model); @@ -435,10 +436,24 @@ extern "C" { // Returns 0 on success LLAMA_API int32_t llama_model_apply_lora_from_file( const struct llama_model * model, - const char * path_lora, - float scale, - const char * path_base_model, - int32_t n_threads); + const char * path_lora, + float scale, + const char * path_base_model, + int32_t n_threads); + + // Apply a loaded control vector to a llama_context, or if data is NULL, clear + // the currently loaded vector. + // n_embd should be the size of a single layer's control, and data should point + // to an n_embd x n_layers buffer starting from layer 1. + // il_start and il_end are the layer range the vector should apply to (both inclusive) + // See llama_control_vector_load in common to load a control vector. + LLAMA_API int32_t llama_control_vector_apply( + struct llama_context * lctx, + const float * data, + size_t len, + int32_t n_embd, + int32_t il_start, + int32_t il_end); // // KV cache From d84c48505f60bcd358b82a751d40418c4d235643 Mon Sep 17 00:00:00 2001 From: slaren Date: Fri, 15 Mar 2024 22:14:16 +0100 Subject: [PATCH 638/812] llama : fix Baichuan2 13B (#6092) --- llama.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/llama.cpp b/llama.cpp index 52bd718ba..e4db288dd 100644 --- a/llama.cpp +++ b/llama.cpp @@ -6000,7 +6000,7 @@ struct llm_build_context { inpL = llm_build_inp_embd(ctx0, lctx, hparams, batch, model.tok_embd, cb); // inp_pos - contains the positions - struct ggml_tensor * inp_pos = build_inp_pos(); + struct ggml_tensor * inp_pos = model.type == MODEL_7B ? build_inp_pos() : nullptr; // KQ_mask (mask for 1 head, it will be broadcasted to all heads) struct ggml_tensor * KQ_mask = build_inp_KQ_mask(); @@ -6050,7 +6050,6 @@ struct llm_build_context { cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); - cur = llm_build_kv(ctx0, model, hparams, kv_self, gf, model.layers[il].wo, NULL, Kcur, Vcur, Qcur, KQ_mask, KQ_pos, n_ctx, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il); From a56d09a4407f29c21e149b44fd5308f83aa1cb09 Mon Sep 17 00:00:00 2001 From: Pierrick Hymbert Date: Sat, 16 Mar 2024 13:20:53 +0100 Subject: [PATCH 639/812] ci : close inactive issue with workflow (#6053) * issues: ci - close inactive issue with workflow * ci: close issue, change workflow schedule time --- .github/workflows/close-issue.yml | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 .github/workflows/close-issue.yml diff --git a/.github/workflows/close-issue.yml b/.github/workflows/close-issue.yml new file mode 100644 index 000000000..bc08a72d0 --- /dev/null +++ b/.github/workflows/close-issue.yml @@ -0,0 +1,22 @@ +name: Close inactive issues +on: + schedule: + - cron: "42 0 * * *" + +jobs: + close-issues: + runs-on: ubuntu-latest + permissions: + issues: write + pull-requests: write + steps: + - uses: actions/stale@v5 + with: + days-before-issue-stale: 30 + days-before-issue-close: 14 + stale-issue-label: "stale" + stale-issue-message: "This issue is stale because it has been open for 30 days with no activity." + close-issue-message: "This issue was closed because it has been inactive for 14 days since being marked as stale." + days-before-pr-stale: -1 + days-before-pr-close: -1 + repo-token: ${{ secrets.GITHUB_TOKEN }} From 15961ec04dbd59d21d8984d42e4c0f7e7e7d320a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?DAN=E2=84=A2?= Date: Sat, 16 Mar 2024 11:39:15 -0400 Subject: [PATCH 640/812] common : refactor nested if causing error C1061 on MSVC (#6101) * Refactor nested if causing error C1061 on MSVC. * Revert back and remove else's. * Add flag to track found arguments. --- common/common.cpp | 475 ++++++++++++++++++++++++++++++++++------------ 1 file changed, 356 insertions(+), 119 deletions(-) diff --git a/common/common.cpp b/common/common.cpp index 4912237e0..1b0ba8493 100644 --- a/common/common.cpp +++ b/common/common.cpp @@ -151,13 +151,17 @@ bool gpt_params_parse_ex(int argc, char ** argv, gpt_params & params) { std::replace(arg.begin(), arg.end(), '_', '-'); } + bool arg_found = false; if (arg == "-s" || arg == "--seed") { + arg_found = true; if (++i >= argc) { invalid_param = true; break; } params.seed = std::stoul(argv[i]); - } else if (arg == "-t" || arg == "--threads") { + } + if (arg == "-t" || arg == "--threads") { + arg_found = true; if (++i >= argc) { invalid_param = true; break; @@ -166,7 +170,9 @@ bool gpt_params_parse_ex(int argc, char ** argv, gpt_params & params) { if (params.n_threads <= 0) { params.n_threads = std::thread::hardware_concurrency(); } - } else if (arg == "-tb" || arg == "--threads-batch") { + } + if (arg == "-tb" || arg == "--threads-batch") { + arg_found = true; if (++i >= argc) { invalid_param = true; break; @@ -175,7 +181,9 @@ bool gpt_params_parse_ex(int argc, char ** argv, gpt_params & params) { if (params.n_threads_batch <= 0) { params.n_threads_batch = std::thread::hardware_concurrency(); } - } else if (arg == "-td" || arg == "--threads-draft") { + } + if (arg == "-td" || arg == "--threads-draft") { + arg_found = true; if (++i >= argc) { invalid_param = true; break; @@ -184,7 +192,9 @@ bool gpt_params_parse_ex(int argc, char ** argv, gpt_params & params) { if (params.n_threads_draft <= 0) { params.n_threads_draft = std::thread::hardware_concurrency(); } - } else if (arg == "-tbd" || arg == "--threads-batch-draft") { + } + if (arg == "-tbd" || arg == "--threads-batch-draft") { + arg_found = true; if (++i >= argc) { invalid_param = true; break; @@ -193,25 +203,37 @@ bool gpt_params_parse_ex(int argc, char ** argv, gpt_params & params) { if (params.n_threads_batch_draft <= 0) { params.n_threads_batch_draft = std::thread::hardware_concurrency(); } - } else if (arg == "-p" || arg == "--prompt") { + } + if (arg == "-p" || arg == "--prompt") { + arg_found = true; if (++i >= argc) { invalid_param = true; break; } params.prompt = argv[i]; - } else if (arg == "-e" || arg == "--escape") { + } + if (arg == "-e" || arg == "--escape") { + arg_found = true; params.escape = true; - } else if (arg == "--prompt-cache") { + } + if (arg == "--prompt-cache") { + arg_found = true; if (++i >= argc) { invalid_param = true; break; } params.path_prompt_cache = argv[i]; - } else if (arg == "--prompt-cache-all") { + } + if (arg == "--prompt-cache-all") { + arg_found = true; params.prompt_cache_all = true; - } else if (arg == "--prompt-cache-ro") { + } + if (arg == "--prompt-cache-ro") { + arg_found = true; params.prompt_cache_ro = true; - } else if (arg == "-bf" || arg == "--binary-file") { + } + if (arg == "-bf" || arg == "--binary-file") { + arg_found = true; if (++i >= argc) { invalid_param = true; break; @@ -228,7 +250,9 @@ bool gpt_params_parse_ex(int argc, char ** argv, gpt_params & params) { ss << file.rdbuf(); params.prompt = ss.str(); fprintf(stderr, "Read %zu bytes from binary file %s\n", params.prompt.size(), argv[i]); - } else if (arg == "-f" || arg == "--file") { + } + if (arg == "-f" || arg == "--file") { + arg_found = true; if (++i >= argc) { invalid_param = true; break; @@ -245,51 +269,67 @@ bool gpt_params_parse_ex(int argc, char ** argv, gpt_params & params) { if (!params.prompt.empty() && params.prompt.back() == '\n') { params.prompt.pop_back(); } - } else if (arg == "-n" || arg == "--n-predict") { + } + if (arg == "-n" || arg == "--n-predict") { + arg_found = true; if (++i >= argc) { invalid_param = true; break; } params.n_predict = std::stoi(argv[i]); - } else if (arg == "--top-k") { + } + if (arg == "--top-k") { + arg_found = true; if (++i >= argc) { invalid_param = true; break; } sparams.top_k = std::stoi(argv[i]); - } else if (arg == "-c" || arg == "--ctx-size") { + } + if (arg == "-c" || arg == "--ctx-size") { + arg_found = true; if (++i >= argc) { invalid_param = true; break; } params.n_ctx = std::stoi(argv[i]); - } else if (arg == "--grp-attn-n" || arg == "-gan") { + } + if (arg == "--grp-attn-n" || arg == "-gan") { + arg_found = true; if (++i >= argc) { invalid_param = true; break; } params.grp_attn_n = std::stoi(argv[i]); - } else if (arg == "--grp-attn-w" || arg == "-gaw") { + } + if (arg == "--grp-attn-w" || arg == "-gaw") { + arg_found = true; if (++i >= argc) { invalid_param = true; break; } params.grp_attn_w = std::stoi(argv[i]); - } else if (arg == "--rope-freq-base") { + } + if (arg == "--rope-freq-base") { + arg_found = true; if (++i >= argc) { invalid_param = true; break; } params.rope_freq_base = std::stof(argv[i]); - } else if (arg == "--rope-freq-scale") { + } + if (arg == "--rope-freq-scale") { + arg_found = true; if (++i >= argc) { invalid_param = true; break; } params.rope_freq_scale = std::stof(argv[i]); - } else if (arg == "--rope-scaling") { + } + if (arg == "--rope-scaling") { + arg_found = true; if (++i >= argc) { invalid_param = true; break; @@ -299,43 +339,57 @@ bool gpt_params_parse_ex(int argc, char ** argv, gpt_params & params) { else if (value == "linear") { params.rope_scaling_type = LLAMA_ROPE_SCALING_TYPE_LINEAR; } else if (value == "yarn") { params.rope_scaling_type = LLAMA_ROPE_SCALING_TYPE_YARN; } else { invalid_param = true; break; } - } else if (arg == "--rope-scale") { + } + if (arg == "--rope-scale") { + arg_found = true; if (++i >= argc) { invalid_param = true; break; } params.rope_freq_scale = 1.0f/std::stof(argv[i]); - } else if (arg == "--yarn-orig-ctx") { + } + if (arg == "--yarn-orig-ctx") { + arg_found = true; if (++i >= argc) { invalid_param = true; break; } params.yarn_orig_ctx = std::stoi(argv[i]); - } else if (arg == "--yarn-ext-factor") { + } + if (arg == "--yarn-ext-factor") { + arg_found = true; if (++i >= argc) { invalid_param = true; break; } params.yarn_ext_factor = std::stof(argv[i]); - } else if (arg == "--yarn-attn-factor") { + } + if (arg == "--yarn-attn-factor") { + arg_found = true; if (++i >= argc) { invalid_param = true; break; } params.yarn_attn_factor = std::stof(argv[i]); - } else if (arg == "--yarn-beta-fast") { + } + if (arg == "--yarn-beta-fast") { + arg_found = true; if (++i >= argc) { invalid_param = true; break; } params.yarn_beta_fast = std::stof(argv[i]); - } else if (arg == "--yarn-beta-slow") { + } + if (arg == "--yarn-beta-slow") { + arg_found = true; if (++i >= argc) { invalid_param = true; break; } params.yarn_beta_slow = std::stof(argv[i]); - } else if (arg == "--pooling") { + } + if (arg == "--pooling") { + arg_found = true; if (++i >= argc) { invalid_param = true; break; @@ -345,118 +399,156 @@ bool gpt_params_parse_ex(int argc, char ** argv, gpt_params & params) { else if (value == "mean") { params.pooling_type = LLAMA_POOLING_TYPE_MEAN; } else if (value == "cls") { params.pooling_type = LLAMA_POOLING_TYPE_CLS; } else { invalid_param = true; break; } - } else if (arg == "--defrag-thold" || arg == "-dt") { + } + if (arg == "--defrag-thold" || arg == "-dt") { + arg_found = true; if (++i >= argc) { invalid_param = true; break; } params.defrag_thold = std::stof(argv[i]); - } else if (arg == "--samplers") { + } + if (arg == "--samplers") { + arg_found = true; if (++i >= argc) { invalid_param = true; break; } const auto sampler_names = string_split(argv[i], ';'); sparams.samplers_sequence = sampler_types_from_names(sampler_names, true); - } else if (arg == "--sampling-seq") { + } + if (arg == "--sampling-seq") { + arg_found = true; if (++i >= argc) { invalid_param = true; break; } sparams.samplers_sequence = sampler_types_from_chars(argv[i]); - } else if (arg == "--top-p") { + } + if (arg == "--top-p") { + arg_found = true; if (++i >= argc) { invalid_param = true; break; } sparams.top_p = std::stof(argv[i]); - } else if (arg == "--min-p") { + } + if (arg == "--min-p") { + arg_found = true; if (++i >= argc) { invalid_param = true; break; } sparams.min_p = std::stof(argv[i]); - } else if (arg == "--temp") { + } + if (arg == "--temp") { + arg_found = true; if (++i >= argc) { invalid_param = true; break; } sparams.temp = std::stof(argv[i]); sparams.temp = std::max(sparams.temp, 0.0f); - } else if (arg == "--tfs") { + } + if (arg == "--tfs") { + arg_found = true; if (++i >= argc) { invalid_param = true; break; } sparams.tfs_z = std::stof(argv[i]); - } else if (arg == "--typical") { + } + if (arg == "--typical") { + arg_found = true; if (++i >= argc) { invalid_param = true; break; } sparams.typical_p = std::stof(argv[i]); - } else if (arg == "--repeat-last-n") { + } + if (arg == "--repeat-last-n") { + arg_found = true; if (++i >= argc) { invalid_param = true; break; } sparams.penalty_last_n = std::stoi(argv[i]); sparams.n_prev = std::max(sparams.n_prev, sparams.penalty_last_n); - } else if (arg == "--repeat-penalty") { + } + if (arg == "--repeat-penalty") { + arg_found = true; if (++i >= argc) { invalid_param = true; break; } sparams.penalty_repeat = std::stof(argv[i]); - } else if (arg == "--frequency-penalty") { + } + if (arg == "--frequency-penalty") { + arg_found = true; if (++i >= argc) { invalid_param = true; break; } sparams.penalty_freq = std::stof(argv[i]); - } else if (arg == "--presence-penalty") { + } + if (arg == "--presence-penalty") { + arg_found = true; if (++i >= argc) { invalid_param = true; break; } sparams.penalty_present = std::stof(argv[i]); - } else if (arg == "--dynatemp-range") { + } + if (arg == "--dynatemp-range") { + arg_found = true; if (++i >= argc) { invalid_param = true; break; } sparams.dynatemp_range = std::stof(argv[i]); - } else if (arg == "--dynatemp-exp") { + } + if (arg == "--dynatemp-exp") { + arg_found = true; if (++i >= argc) { invalid_param = true; break; } sparams.dynatemp_exponent = std::stof(argv[i]); - } else if (arg == "--mirostat") { + } + if (arg == "--mirostat") { + arg_found = true; if (++i >= argc) { invalid_param = true; break; } sparams.mirostat = std::stoi(argv[i]); - } else if (arg == "--mirostat-lr") { + } + if (arg == "--mirostat-lr") { + arg_found = true; if (++i >= argc) { invalid_param = true; break; } sparams.mirostat_eta = std::stof(argv[i]); - } else if (arg == "--mirostat-ent") { + } + if (arg == "--mirostat-ent") { + arg_found = true; if (++i >= argc) { invalid_param = true; break; } sparams.mirostat_tau = std::stof(argv[i]); - } else if (arg == "--cfg-negative-prompt") { + } + if (arg == "--cfg-negative-prompt") { + arg_found = true; if (++i >= argc) { invalid_param = true; break; } sparams.cfg_negative_prompt = argv[i]; - } else if (arg == "--cfg-negative-prompt-file") { + } + if (arg == "--cfg-negative-prompt-file") { + arg_found = true; if (++i >= argc) { invalid_param = true; break; @@ -471,86 +563,114 @@ bool gpt_params_parse_ex(int argc, char ** argv, gpt_params & params) { if (!sparams.cfg_negative_prompt.empty() && sparams.cfg_negative_prompt.back() == '\n') { sparams.cfg_negative_prompt.pop_back(); } - } else if (arg == "--cfg-scale") { + } + if (arg == "--cfg-scale") { + arg_found = true; if (++i >= argc) { invalid_param = true; break; } sparams.cfg_scale = std::stof(argv[i]); - } else if (arg == "-b" || arg == "--batch-size") { + } + if (arg == "-b" || arg == "--batch-size") { + arg_found = true; if (++i >= argc) { invalid_param = true; break; } params.n_batch = std::stoi(argv[i]); - } else if (arg == "-ub" || arg == "--ubatch-size") { + } + if (arg == "-ub" || arg == "--ubatch-size") { + arg_found = true; if (++i >= argc) { invalid_param = true; break; } params.n_ubatch = std::stoi(argv[i]); - } else if (arg == "--keep") { + } + if (arg == "--keep") { + arg_found = true; if (++i >= argc) { invalid_param = true; break; } params.n_keep = std::stoi(argv[i]); - } else if (arg == "--draft") { + } + if (arg == "--draft") { + arg_found = true; if (++i >= argc) { invalid_param = true; break; } params.n_draft = std::stoi(argv[i]); - } else if (arg == "--chunks") { + } + if (arg == "--chunks") { + arg_found = true; if (++i >= argc) { invalid_param = true; break; } params.n_chunks = std::stoi(argv[i]); - } else if (arg == "-np" || arg == "--parallel") { + } + if (arg == "-np" || arg == "--parallel") { + arg_found = true; if (++i >= argc) { invalid_param = true; break; } params.n_parallel = std::stoi(argv[i]); - } else if (arg == "-ns" || arg == "--sequences") { + } + if (arg == "-ns" || arg == "--sequences") { + arg_found = true; if (++i >= argc) { invalid_param = true; break; } params.n_sequences = std::stoi(argv[i]); - } else if (arg == "--p-split" || arg == "-ps") { + } + if (arg == "--p-split" || arg == "-ps") { + arg_found = true; if (++i >= argc) { invalid_param = true; break; } params.p_split = std::stof(argv[i]); - } else if (arg == "-m" || arg == "--model") { + } + if (arg == "-m" || arg == "--model") { + arg_found = true; if (++i >= argc) { invalid_param = true; break; } params.model = argv[i]; - } else if (arg == "-md" || arg == "--model-draft") { + } + if (arg == "-md" || arg == "--model-draft") { + arg_found = true; if (++i >= argc) { invalid_param = true; break; } params.model_draft = argv[i]; - } else if (arg == "-a" || arg == "--alias") { + } + if (arg == "-a" || arg == "--alias") { + arg_found = true; if (++i >= argc) { invalid_param = true; break; } params.model_alias = argv[i]; - } else if (arg == "--lora") { + } + if (arg == "--lora") { + arg_found = true; if (++i >= argc) { invalid_param = true; break; } params.lora_adapter.emplace_back(argv[i], 1.0f); params.use_mmap = false; - } else if (arg == "--lora-scaled") { + } + if (arg == "--lora-scaled") { + arg_found = true; if (++i >= argc) { invalid_param = true; break; @@ -562,19 +682,25 @@ bool gpt_params_parse_ex(int argc, char ** argv, gpt_params & params) { } params.lora_adapter.emplace_back(lora_adapter, std::stof(argv[i])); params.use_mmap = false; - } else if (arg == "--lora-base") { + } + if (arg == "--lora-base") { + arg_found = true; if (++i >= argc) { invalid_param = true; break; } params.lora_base = argv[i]; - } else if (arg == "--control-vector") { + } + if (arg == "--control-vector") { + arg_found = true; if (++i >= argc) { invalid_param = true; break; } params.control_vectors.push_back({ 1.0f, argv[i], }); - } else if (arg == "--control-vector-scaled") { + } + if (arg == "--control-vector-scaled") { + arg_found = true; if (++i >= argc) { invalid_param = true; break; @@ -585,7 +711,9 @@ bool gpt_params_parse_ex(int argc, char ** argv, gpt_params & params) { break; } params.control_vectors.push_back({ std::stof(argv[i]), fname, }); - } else if (arg == "--control-vector-layer-range") { + } + if (arg == "--control-vector-layer-range") { + arg_found = true; if (++i >= argc) { invalid_param = true; break; @@ -596,49 +724,85 @@ bool gpt_params_parse_ex(int argc, char ** argv, gpt_params & params) { break; } params.control_vector_layer_end = std::stoi(argv[i]); - } else if (arg == "--mmproj") { + } + if (arg == "--mmproj") { + arg_found = true; if (++i >= argc) { invalid_param = true; break; } params.mmproj = argv[i]; - } else if (arg == "--image") { + } + if (arg == "--image") { + arg_found = true; if (++i >= argc) { invalid_param = true; break; } params.image = argv[i]; - } else if (arg == "-i" || arg == "--interactive") { + } + if (arg == "-i" || arg == "--interactive") { + arg_found = true; params.interactive = true; - } else if (arg == "--embedding") { + } + if (arg == "--embedding") { + arg_found = true; params.embedding = true; - } else if (arg == "--interactive-first") { + } + if (arg == "--interactive-first") { + arg_found = true; params.interactive_first = true; - } else if (arg == "-ins" || arg == "--instruct") { + } + if (arg == "-ins" || arg == "--instruct") { + arg_found = true; params.instruct = true; - } else if (arg == "-cml" || arg == "--chatml") { + } + if (arg == "-cml" || arg == "--chatml") { + arg_found = true; params.chatml = true; - } else if (arg == "--infill") { + } + if (arg == "--infill") { + arg_found = true; params.infill = true; - } else if (arg == "-dkvc" || arg == "--dump-kv-cache") { + } + if (arg == "-dkvc" || arg == "--dump-kv-cache") { + arg_found = true; params.dump_kv_cache = true; - } else if (arg == "-nkvo" || arg == "--no-kv-offload") { + } + if (arg == "-nkvo" || arg == "--no-kv-offload") { + arg_found = true; params.no_kv_offload = true; - } else if (arg == "-ctk" || arg == "--cache-type-k") { + } + if (arg == "-ctk" || arg == "--cache-type-k") { + arg_found = true; params.cache_type_k = argv[++i]; - } else if (arg == "-ctv" || arg == "--cache-type-v") { + } + if (arg == "-ctv" || arg == "--cache-type-v") { + arg_found = true; params.cache_type_v = argv[++i]; - } else if (arg == "--multiline-input") { + } + if (arg == "--multiline-input") { + arg_found = true; params.multiline_input = true; - } else if (arg == "--simple-io") { + } + if (arg == "--simple-io") { + arg_found = true; params.simple_io = true; - } else if (arg == "-cb" || arg == "--cont-batching") { + } + if (arg == "-cb" || arg == "--cont-batching") { + arg_found = true; params.cont_batching = true; - } else if (arg == "--color") { + } + if (arg == "--color") { + arg_found = true; params.use_color = true; - } else if (arg == "--mlock") { + } + if (arg == "--mlock") { + arg_found = true; params.use_mlock = true; - } else if (arg == "--gpu-layers" || arg == "-ngl" || arg == "--n-gpu-layers") { + } + if (arg == "--gpu-layers" || arg == "-ngl" || arg == "--n-gpu-layers") { + arg_found = true; if (++i >= argc) { invalid_param = true; break; @@ -648,7 +812,9 @@ bool gpt_params_parse_ex(int argc, char ** argv, gpt_params & params) { fprintf(stderr, "warning: not compiled with GPU offload support, --n-gpu-layers option will be ignored\n"); fprintf(stderr, "warning: see main README.md for information on enabling GPU BLAS support\n"); } - } else if (arg == "--gpu-layers-draft" || arg == "-ngld" || arg == "--n-gpu-layers-draft") { + } + if (arg == "--gpu-layers-draft" || arg == "-ngld" || arg == "--n-gpu-layers-draft") { + arg_found = true; if (++i >= argc) { invalid_param = true; break; @@ -658,7 +824,9 @@ bool gpt_params_parse_ex(int argc, char ** argv, gpt_params & params) { fprintf(stderr, "warning: not compiled with GPU offload support, --n-gpu-layers-draft option will be ignored\n"); fprintf(stderr, "warning: see main README.md for information on enabling GPU BLAS support\n"); } - } else if (arg == "--main-gpu" || arg == "-mg") { + } + if (arg == "--main-gpu" || arg == "-mg") { + arg_found = true; if (++i >= argc) { invalid_param = true; break; @@ -667,7 +835,9 @@ bool gpt_params_parse_ex(int argc, char ** argv, gpt_params & params) { #ifndef GGML_USE_CUBLAS_SYCL fprintf(stderr, "warning: llama.cpp was compiled without cuBLAS/SYCL. Setting the main GPU has no effect.\n"); #endif // GGML_USE_CUBLAS_SYCL - } else if (arg == "--split-mode" || arg == "-sm") { + } + if (arg == "--split-mode" || arg == "-sm") { + arg_found = true; if (++i >= argc) { invalid_param = true; break; @@ -691,7 +861,9 @@ bool gpt_params_parse_ex(int argc, char ** argv, gpt_params & params) { fprintf(stderr, "warning: llama.cpp was compiled without cuBLAS/SYCL. Setting the split mode has no effect.\n"); #endif // GGML_USE_CUBLAS_SYCL - } else if (arg == "--tensor-split" || arg == "-ts") { + } + if (arg == "--tensor-split" || arg == "-ts") { + arg_found = true; if (++i >= argc) { invalid_param = true; break; @@ -716,9 +888,13 @@ bool gpt_params_parse_ex(int argc, char ** argv, gpt_params & params) { #ifndef GGML_USE_CUBLAS_SYCL_VULKAN fprintf(stderr, "warning: llama.cpp was compiled without cuBLAS/SYCL/Vulkan. Setting a tensor split has no effect.\n"); #endif // GGML_USE_CUBLAS_SYCL - } else if (arg == "--no-mmap") { + } + if (arg == "--no-mmap") { + arg_found = true; params.use_mmap = false; - } else if (arg == "--numa") { + } + if (arg == "--numa") { + arg_found = true; if (++i >= argc) { invalid_param = true; break; @@ -728,17 +904,25 @@ bool gpt_params_parse_ex(int argc, char ** argv, gpt_params & params) { else if (value == "isolate") { params.numa = GGML_NUMA_STRATEGY_ISOLATE; } else if (value == "numactl") { params.numa = GGML_NUMA_STRATEGY_NUMACTL; } else { invalid_param = true; break; } - } else if (arg == "--verbose-prompt") { + } + if (arg == "--verbose-prompt") { + arg_found = true; params.verbose_prompt = true; - } else if (arg == "--no-display-prompt") { + } + if (arg == "--no-display-prompt") { + arg_found = true; params.display_prompt = false; - } else if (arg == "-r" || arg == "--reverse-prompt") { + } + if (arg == "-r" || arg == "--reverse-prompt") { + arg_found = true; if (++i >= argc) { invalid_param = true; break; } params.antiprompt.emplace_back(argv[i]); - } else if (arg == "-ld" || arg == "--logdir") { + } + if (arg == "-ld" || arg == "--logdir") { + arg_found = true; if (++i >= argc) { invalid_param = true; break; @@ -748,63 +932,93 @@ bool gpt_params_parse_ex(int argc, char ** argv, gpt_params & params) { if (params.logdir.back() != DIRECTORY_SEPARATOR) { params.logdir += DIRECTORY_SEPARATOR; } - } else if (arg == "--save-all-logits" || arg == "--kl-divergence-base") { + } + if (arg == "--save-all-logits" || arg == "--kl-divergence-base") { + arg_found = true; if (++i >= argc) { invalid_param = true; break; } params.logits_file = argv[i]; - } else if (arg == "--perplexity" || arg == "--all-logits") { + } + if (arg == "--perplexity" || arg == "--all-logits") { + arg_found = true; params.logits_all = true; - } else if (arg == "--ppl-stride") { + } + if (arg == "--ppl-stride") { + arg_found = true; if (++i >= argc) { invalid_param = true; break; } params.ppl_stride = std::stoi(argv[i]); - } else if (arg == "-ptc" || arg == "--print-token-count") { + } + if (arg == "-ptc" || arg == "--print-token-count") { + arg_found = true; if (++i >= argc) { invalid_param = true; break; } params.n_print = std::stoi(argv[i]); - } else if (arg == "--ppl-output-type") { + } + if (arg == "--ppl-output-type") { + arg_found = true; if (++i >= argc) { invalid_param = true; break; } params.ppl_output_type = std::stoi(argv[i]); - } else if (arg == "--hellaswag") { + } + if (arg == "--hellaswag") { + arg_found = true; params.hellaswag = true; - } else if (arg == "--hellaswag-tasks") { + } + if (arg == "--hellaswag-tasks") { + arg_found = true; if (++i >= argc) { invalid_param = true; break; } params.hellaswag_tasks = std::stoi(argv[i]); - } else if (arg == "--winogrande") { + } + if (arg == "--winogrande") { + arg_found = true; params.winogrande = true; - } else if (arg == "--winogrande-tasks") { + } + if (arg == "--winogrande-tasks") { + arg_found = true; if (++i >= argc) { invalid_param = true; break; } params.winogrande_tasks = std::stoi(argv[i]); - } else if (arg == "--multiple-choice") { + } + if (arg == "--multiple-choice") { + arg_found = true; params.multiple_choice = true; - } else if (arg == "--multiple-choice-tasks") { + } + if (arg == "--multiple-choice-tasks") { + arg_found = true; if (++i >= argc) { invalid_param = true; break; } params.multiple_choice_tasks = std::stoi(argv[i]); - } else if (arg == "--kl-divergence") { + } + if (arg == "--kl-divergence") { + arg_found = true; params.kl_divergence = true; - } else if (arg == "--ignore-eos") { + } + if (arg == "--ignore-eos") { + arg_found = true; params.ignore_eos = true; - } else if (arg == "--no-penalize-nl") { + } + if (arg == "--no-penalize-nl") { + arg_found = true; sparams.penalize_nl = false; - } else if (arg == "-l" || arg == "--logit-bias") { + } + if (arg == "-l" || arg == "--logit-bias") { + arg_found = true; if (++i >= argc) { invalid_param = true; break; @@ -823,36 +1037,51 @@ bool gpt_params_parse_ex(int argc, char ** argv, gpt_params & params) { invalid_param = true; break; } - } else if (arg == "-h" || arg == "--help") { + } + if (arg == "-h" || arg == "--help") { + arg_found = true; return false; - - } else if (arg == "--version") { + } + if (arg == "--version") { + arg_found = true; fprintf(stderr, "version: %d (%s)\n", LLAMA_BUILD_NUMBER, LLAMA_COMMIT); fprintf(stderr, "built with %s for %s\n", LLAMA_COMPILER, LLAMA_BUILD_TARGET); exit(0); - } else if (arg == "--random-prompt") { + } + if (arg == "--random-prompt") { + arg_found = true; params.random_prompt = true; - } else if (arg == "--in-prefix-bos") { + } + if (arg == "--in-prefix-bos") { + arg_found = true; params.input_prefix_bos = true; - } else if (arg == "--in-prefix") { + } + if (arg == "--in-prefix") { + arg_found = true; if (++i >= argc) { invalid_param = true; break; } params.input_prefix = argv[i]; - } else if (arg == "--in-suffix") { + } + if (arg == "--in-suffix") { + arg_found = true; if (++i >= argc) { invalid_param = true; break; } params.input_suffix = argv[i]; - } else if (arg == "--grammar") { + } + if (arg == "--grammar") { + arg_found = true; if (++i >= argc) { invalid_param = true; break; } sparams.grammar = argv[i]; - } else if (arg == "--grammar-file") { + } + if (arg == "--grammar-file") { + arg_found = true; if (++i >= argc) { invalid_param = true; break; @@ -868,7 +1097,9 @@ bool gpt_params_parse_ex(int argc, char ** argv, gpt_params & params) { std::istreambuf_iterator(), std::back_inserter(sparams.grammar) ); - } else if (arg == "--override-kv") { + } + if (arg == "--override-kv") { + arg_found = true; if (++i >= argc) { invalid_param = true; break; @@ -911,10 +1142,14 @@ bool gpt_params_parse_ex(int argc, char ** argv, gpt_params & params) { params.kv_overrides.push_back(kvo); #ifndef LOG_DISABLE_LOGS // Parse args for logging parameters - } else if ( log_param_single_parse( argv[i] ) ) { + } + if ( log_param_single_parse( argv[i] ) ) { + arg_found = true; // Do nothing, log_param_single_parse automatically does it's thing // and returns if a match was found and parsed. - } else if ( log_param_pair_parse( /*check_but_dont_parse*/ true, argv[i] ) ) { + } + if ( log_param_pair_parse( /*check_but_dont_parse*/ true, argv[i] ) ) { + arg_found = true; // We have a matching known parameter requiring an argument, // now we need to check if there is anything after this argv // and flag invalid_param or parse it. @@ -928,7 +1163,9 @@ bool gpt_params_parse_ex(int argc, char ** argv, gpt_params & params) { } // End of Parse args for logging parameters #endif // LOG_DISABLE_LOGS - } else { + } + + if (!arg_found) { throw std::invalid_argument("error: unknown argument: " + arg); } } From dfbfdd60f90207404039c6578d709231496831d9 Mon Sep 17 00:00:00 2001 From: Xuan Son Nguyen Date: Sat, 16 Mar 2024 16:42:08 +0100 Subject: [PATCH 641/812] readme : add wllama as a wasm binding (#6100) --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 5cbdf7e47..c2f3342f0 100644 --- a/README.md +++ b/README.md @@ -134,6 +134,7 @@ Typically finetunes of the base models below are supported as well. - Node.js: [withcatai/node-llama-cpp](https://github.com/withcatai/node-llama-cpp) - JS/TS (llama.cpp server client): [lgrammel/modelfusion](https://modelfusion.dev/integration/model-provider/llamacpp) - JavaScript/Wasm (works in browser): [tangledgroup/llama-cpp-wasm](https://github.com/tangledgroup/llama-cpp-wasm) +- Typescript/Wasm (nicer API, available on npm): [ngxson/wllama](https://github.com/ngxson/wllama) - Ruby: [yoshoku/llama_cpp.rb](https://github.com/yoshoku/llama_cpp.rb) - Rust (nicer API): [mdrokz/rust-llama.cpp](https://github.com/mdrokz/rust-llama.cpp) - Rust (more direct bindings): [utilityai/llama-cpp-rs](https://github.com/utilityai/llama-cpp-rs) From b5f4ae09c3244ae1644b67c03ed9f4227ab25ad2 Mon Sep 17 00:00:00 2001 From: Daniel Bevenius Date: Sat, 16 Mar 2024 16:46:29 +0100 Subject: [PATCH 642/812] gritlm : add initial README.md (#6086) * gritlm: add initial README.md to examples/gritlm This commit adds a suggestion for an initial README.md for the gritlm example. Signed-off-by: Daniel Bevenius * squash! gritlm: add initial README.md to examples/gritlm Use the `scripts/hf.sh` script to download the model file. Signed-off-by: Daniel Bevenius * squash! gritlm: add initial README.md to examples/gritlm Fix editorconfig-checker error in examples/gritlm/README.md. Signed-off-by: Daniel Bevenius --------- Signed-off-by: Daniel Bevenius --- examples/gritlm/README.md | 62 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 examples/gritlm/README.md diff --git a/examples/gritlm/README.md b/examples/gritlm/README.md new file mode 100644 index 000000000..64cc19204 --- /dev/null +++ b/examples/gritlm/README.md @@ -0,0 +1,62 @@ +## Generative Representational Instruction Tuning (GRIT) Example +[gritlm] a model which can generate embeddings as well as "normal" text +generation depending on the instructions in the prompt. + +* Paper: https://arxiv.org/pdf/2402.09906.pdf + +### Retrieval-Augmented Generation (RAG) use case +One use case for `gritlm` is to use it with RAG. If we recall how RAG works is +that we take documents that we want to use as context, to ground the large +language model (LLM), and we create token embeddings for them. We then store +these token embeddings in a vector database. + +When we perform a query, prompt the LLM, we will first create token embeddings +for the query and then search the vector database to retrieve the most +similar vectors, and return those documents so they can be passed to the LLM as +context. Then the query and the context will be passed to the LLM which will +have to _again_ create token embeddings for the query. But because gritlm is used +the first query can be cached and the second query tokenization generation does +not have to be performed at all. + +### Running the example +Download a Grit model: +```console +$ scripts/hf.sh --repo cohesionet/GritLM-7B_gguf --file gritlm-7b_q4_1.gguf +``` + +Run the example using the downloaded model: +```console +$ ./gritlm -m gritlm-7b_q4_1.gguf + +Cosine similarity between "Bitcoin: A Peer-to-Peer Electronic Cash System" and "A purely peer-to-peer version of electronic cash w" is: 0.605 +Cosine similarity between "Bitcoin: A Peer-to-Peer Electronic Cash System" and "All text-based language problems can be reduced to" is: 0.103 +Cosine similarity between "Generative Representational Instruction Tuning" and "A purely peer-to-peer version of electronic cash w" is: 0.112 +Cosine similarity between "Generative Representational Instruction Tuning" and "All text-based language problems can be reduced to" is: 0.547 + +Oh, brave adventurer, who dared to climb +The lofty peak of Mt. Fuji in the night, +When shadows lurk and ghosts do roam, +And darkness reigns, a fearsome sight. + +Thou didst set out, with heart aglow, +To conquer this mountain, so high, +And reach the summit, where the stars do glow, +And the moon shines bright, up in the sky. + +Through the mist and fog, thou didst press on, +With steadfast courage, and a steadfast will, +Through the darkness, thou didst not be gone, +But didst climb on, with a steadfast skill. + +At last, thou didst reach the summit's crest, +And gazed upon the world below, +And saw the beauty of the night's best, +And felt the peace, that only nature knows. + +Oh, brave adventurer, who dared to climb +The lofty peak of Mt. Fuji in the night, +Thou art a hero, in the eyes of all, +For thou didst conquer this mountain, so bright. +``` + +[gritlm]: https://github.com/ContextualAI/gritlm From c47cf414efafb8f60596edc7edb5a2d68065e992 Mon Sep 17 00:00:00 2001 From: AmirAli Mirian <37371367+amiralimi@users.noreply.github.com> Date: Sat, 16 Mar 2024 11:52:02 -0400 Subject: [PATCH 643/812] ggml : add AVX512F SIMD (#6088) --- ggml.c | 95 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 95 insertions(+) diff --git a/ggml.c b/ggml.c index c94006e51..fa23cb3c4 100644 --- a/ggml.c +++ b/ggml.c @@ -931,6 +931,101 @@ inline static float vaddvq_f32(float32x4_t v) { #define GGML_F16_VEC_REDUCE GGML_F32Cx4_REDUCE #endif +#elif defined(__AVX512F__) + +#define GGML_SIMD + +// F32 AVX512 + +#define GGML_F32_STEP 64 +#define GGML_F32_EPR 16 + +#define GGML_F32x16 __m512 +#define GGML_F32x16_ZERO _mm512_setzero_ps() +#define GGML_F32x16_SET1(x) _mm512_set1_ps(x) +#define GGML_F32x16_LOAD _mm512_loadu_ps +#define GGML_F32x16_STORE _mm512_storeu_ps +// _mm512_fmadd_ps is defined in AVX512F so no guard is required +#define GGML_F32x16_FMA(a, b, c) _mm512_fmadd_ps(b, c, a) +#define GGML_F32x16_ADD _mm512_add_ps +#define GGML_F32x16_MUL _mm512_mul_ps +#define GGML_F32x16_REDUCE(res, x) \ +do { \ + int offset = GGML_F32_ARR >> 1; \ + for (int i = 0; i < offset; ++i) { \ + x[i] = _mm512_add_ps(x[i], x[offset+i]); \ + } \ + offset >>= 1; \ + for (int i = 0; i < offset; ++i) { \ + x[i] = _mm512_add_ps(x[i], x[offset+i]); \ + } \ + offset >>= 1; \ + for (int i = 0; i < offset; ++i) { \ + x[i] = _mm512_add_ps(x[i], x[offset+i]); \ + } \ + res = _mm512_reduce_add_ps(x[0]); \ +} while (0) + +// TODO: is this optimal ? + +#define GGML_F32_VEC GGML_F32x16 +#define GGML_F32_VEC_ZERO GGML_F32x16_ZERO +#define GGML_F32_VEC_SET1 GGML_F32x16_SET1 +#define GGML_F32_VEC_LOAD GGML_F32x16_LOAD +#define GGML_F32_VEC_STORE GGML_F32x16_STORE +#define GGML_F32_VEC_FMA GGML_F32x16_FMA +#define GGML_F32_VEC_ADD GGML_F32x16_ADD +#define GGML_F32_VEC_MUL GGML_F32x16_MUL +#define GGML_F32_VEC_REDUCE GGML_F32x16_REDUCE + +// F16 AVX512 + +// F16 AVX + +#define GGML_F16_STEP 64 +#define GGML_F16_EPR 16 + +// AVX512 has FP16 extension (AVX512_FP16) but I don't have it on my machine so I use FP32 instead + +#define GGML_F32Cx16 __m512 +#define GGML_F32Cx16_ZERO _mm512_setzero_ps() +#define GGML_F32Cx16_SET1(x) _mm512_set1_ps(x) + +// unlike _mm256_cvt intrinsics that require F16C, _mm512_cvt is defined in AVX512F +// so F16C guard isn't required +#define GGML_F32Cx16_LOAD(x) _mm512_cvtph_ps(_mm256_loadu_si256((__m256i *)(x))) +#define GGML_F32Cx16_STORE(x, y) _mm256_storeu_si256((__m256i *)(x), _mm512_cvtps_ph(y, 0)) + +#define GGML_F32Cx16_FMA(a, b, c) _mm512_fmadd_ps(b, c, a) +#define GGML_F32Cx16_ADD _mm512_add_ps +#define GGML_F32Cx16_MUL _mm512_mul_ps +#define GGML_F32Cx16_REDUCE(res, x) \ +do { \ + int offset = GGML_F32_ARR >> 1; \ + for (int i = 0; i < offset; ++i) { \ + x[i] = _mm512_add_ps(x[i], x[offset+i]); \ + } \ + offset >>= 1; \ + for (int i = 0; i < offset; ++i) { \ + x[i] = _mm512_add_ps(x[i], x[offset+i]); \ + } \ + offset >>= 1; \ + for (int i = 0; i < offset; ++i) { \ + x[i] = _mm512_add_ps(x[i], x[offset+i]); \ + } \ + res = _mm512_reduce_add_ps(x[0]); \ +} while (0) + +#define GGML_F16_VEC GGML_F32Cx16 +#define GGML_F16_VEC_ZERO GGML_F32Cx16_ZERO +#define GGML_F16_VEC_SET1 GGML_F32Cx16_SET1 +#define GGML_F16_VEC_LOAD(p, i) GGML_F32Cx16_LOAD(p) +#define GGML_F16_VEC_STORE(p, r, i) GGML_F32Cx16_STORE(p, r[i]) +#define GGML_F16_VEC_FMA GGML_F32Cx16_FMA +#define GGML_F16_VEC_ADD GGML_F32Cx16_ADD +#define GGML_F16_VEC_MUL GGML_F32Cx16_MUL +#define GGML_F16_VEC_REDUCE GGML_F32Cx16_REDUCE + #elif defined(__AVX__) #define GGML_SIMD From dc0f6125487dcfbff913360f9d877bc0ccf6aa57 Mon Sep 17 00:00:00 2001 From: GainLee Date: Mon, 18 Mar 2024 01:12:22 +0800 Subject: [PATCH 644/812] ggml:fix finding transfer queue family index error (#6094) Co-authored-by: GainLee --- ggml-vulkan.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/ggml-vulkan.cpp b/ggml-vulkan.cpp index 7cce616ba..698b31496 100644 --- a/ggml-vulkan.cpp +++ b/ggml-vulkan.cpp @@ -710,6 +710,12 @@ static uint32_t ggml_vk_find_queue_family_index(std::vector= 0) { + return compute_index; + } + std::cerr << "ggml_vulkan: No suitable queue family index found." << std::endl; for(auto &q_family : queue_family_props) { From cd776c37c945bf58efc8fe44b370456680cb1b59 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Sun, 17 Mar 2024 19:51:57 +0200 Subject: [PATCH 645/812] ci : close all stale issues at once (#6115) --- .github/workflows/close-issue.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/close-issue.yml b/.github/workflows/close-issue.yml index bc08a72d0..2682f308c 100644 --- a/.github/workflows/close-issue.yml +++ b/.github/workflows/close-issue.yml @@ -19,4 +19,5 @@ jobs: close-issue-message: "This issue was closed because it has been inactive for 14 days since being marked as stale." days-before-pr-stale: -1 days-before-pr-close: -1 + operations-per-run: 1000 repo-token: ${{ secrets.GITHUB_TOKEN }} From d01b3c4c32357567f3531d4e6ceffc5d23e87583 Mon Sep 17 00:00:00 2001 From: Pierrick Hymbert Date: Sun, 17 Mar 2024 19:12:37 +0100 Subject: [PATCH 646/812] common: llama_load_model_from_url using --model-url (#6098) * common: llama_load_model_from_url with libcurl dependency Co-authored-by: Georgi Gerganov --- .github/workflows/build.yml | 22 ++ .github/workflows/server.yml | 20 +- CMakeLists.txt | 1 + Makefile | 5 + common/CMakeLists.txt | 13 +- common/common.cpp | 238 +++++++++++++++++- common/common.h | 4 + examples/main/README.md | 1 + examples/server/README.md | 1 + examples/server/server.cpp | 8 + examples/server/tests/README.md | 2 +- .../server/tests/features/embeddings.feature | 3 +- examples/server/tests/features/environment.py | 93 +++---- examples/server/tests/features/server.feature | 3 +- examples/server/tests/features/steps/steps.py | 37 ++- examples/server/tests/requirements.txt | 1 + 16 files changed, 397 insertions(+), 55 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 0da01d5ba..945df42f8 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -48,6 +48,28 @@ jobs: CC=gcc-8 make tests -j $(nproc) make test -j $(nproc) + ubuntu-focal-make-curl: + runs-on: ubuntu-20.04 + + steps: + - name: Clone + id: checkout + uses: actions/checkout@v3 + + - name: Dependencies + id: depends + run: | + sudo apt-get update + sudo apt-get install build-essential gcc-8 libcurl4-openssl-dev + + - name: Build + id: make_build + env: + LLAMA_FATAL_WARNINGS: 1 + LLAMA_CURL: 1 + run: | + CC=gcc-8 make -j $(nproc) + ubuntu-latest-cmake: runs-on: ubuntu-latest diff --git a/.github/workflows/server.yml b/.github/workflows/server.yml index 5e38b3547..4ea09115a 100644 --- a/.github/workflows/server.yml +++ b/.github/workflows/server.yml @@ -57,7 +57,8 @@ jobs: cmake \ python3-pip \ wget \ - language-pack-en + language-pack-en \ + libcurl4-openssl-dev - name: Build id: cmake_build @@ -67,6 +68,7 @@ jobs: cmake .. \ -DLLAMA_NATIVE=OFF \ -DLLAMA_BUILD_SERVER=ON \ + -DLLAMA_CURL=ON \ -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} \ -DLLAMA_SANITIZE_${{ matrix.sanitizer }}=ON ; cmake --build . --config ${{ matrix.build_type }} -j $(nproc) --target server @@ -101,12 +103,21 @@ jobs: with: fetch-depth: 0 + - name: libCURL + id: get_libcurl + env: + CURL_VERSION: 8.6.0_6 + run: | + curl.exe -o $env:RUNNER_TEMP/curl.zip -L "https://curl.se/windows/dl-${env:CURL_VERSION}/curl-${env:CURL_VERSION}-win64-mingw.zip" + mkdir $env:RUNNER_TEMP/libcurl + tar.exe -xvf $env:RUNNER_TEMP/curl.zip --strip-components=1 -C $env:RUNNER_TEMP/libcurl + - name: Build id: cmake_build run: | mkdir build cd build - cmake .. -DLLAMA_BUILD_SERVER=ON -DCMAKE_BUILD_TYPE=Release ; + cmake .. -DLLAMA_CURL=ON -DCURL_LIBRARY="$env:RUNNER_TEMP/libcurl/lib/libcurl.dll.a" -DCURL_INCLUDE_DIR="$env:RUNNER_TEMP/libcurl/include" cmake --build . --config Release -j ${env:NUMBER_OF_PROCESSORS} --target server - name: Python setup @@ -120,6 +131,11 @@ jobs: run: | pip install -r examples/server/tests/requirements.txt + - name: Copy Libcurl + id: prepare_libcurl + run: | + cp $env:RUNNER_TEMP/libcurl/bin/libcurl-x64.dll ./build/bin/Release/libcurl-x64.dll + - name: Tests id: server_integration_tests if: ${{ !matrix.disabled_on_pr || !github.event.pull_request }} diff --git a/CMakeLists.txt b/CMakeLists.txt index 3ac2804a6..fc4cff28f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -99,6 +99,7 @@ option(LLAMA_CUDA_F16 "llama: use 16 bit floats for some set(LLAMA_CUDA_KQUANTS_ITER "2" CACHE STRING "llama: iters./thread per block for Q2_K/Q6_K") set(LLAMA_CUDA_PEER_MAX_BATCH_SIZE "128" CACHE STRING "llama: max. batch size for using peer access") +option(LLAMA_CURL "llama: use libcurl to download model from an URL" OFF) option(LLAMA_HIPBLAS "llama: use hipBLAS" OFF) option(LLAMA_HIP_UMA "llama: use HIP unified memory architecture" OFF) option(LLAMA_CLBLAST "llama: use CLBlast" OFF) diff --git a/Makefile b/Makefile index c0f125036..838daf5c0 100644 --- a/Makefile +++ b/Makefile @@ -595,6 +595,11 @@ include scripts/get-flags.mk CUDA_CXXFLAGS := $(BASE_CXXFLAGS) $(GF_CXXFLAGS) -Wno-pedantic endif +ifdef LLAMA_CURL +override CXXFLAGS := $(CXXFLAGS) -DLLAMA_USE_CURL +override LDFLAGS := $(LDFLAGS) -lcurl +endif + # # Print build information # diff --git a/common/CMakeLists.txt b/common/CMakeLists.txt index 350bbdf7f..af2629a46 100644 --- a/common/CMakeLists.txt +++ b/common/CMakeLists.txt @@ -68,6 +68,17 @@ if (BUILD_SHARED_LIBS) set_target_properties(${TARGET} PROPERTIES POSITION_INDEPENDENT_CODE ON) endif() +set(LLAMA_COMMON_EXTRA_LIBS build_info) + +# Use curl to download model url +if (LLAMA_CURL) + find_package(CURL REQUIRED) + add_definitions(-DLLAMA_USE_CURL) + include_directories(${CURL_INCLUDE_DIRS}) + find_library(CURL_LIBRARY curl REQUIRED) + set(LLAMA_COMMON_EXTRA_LIBS ${LLAMA_COMMON_EXTRA_LIBS} ${CURL_LIBRARY}) +endif () + target_include_directories(${TARGET} PUBLIC .) target_compile_features(${TARGET} PUBLIC cxx_std_11) -target_link_libraries(${TARGET} PRIVATE build_info PUBLIC llama) +target_link_libraries(${TARGET} PRIVATE ${LLAMA_COMMON_EXTRA_LIBS} PUBLIC llama) diff --git a/common/common.cpp b/common/common.cpp index 1b0ba8493..2f5d965d6 100644 --- a/common/common.cpp +++ b/common/common.cpp @@ -37,6 +37,9 @@ #include #include #endif +#if defined(LLAMA_USE_CURL) +#include +#endif #if defined(_MSC_VER) #pragma warning(disable: 4244 4267) // possible loss of data @@ -50,6 +53,18 @@ #define GGML_USE_CUBLAS_SYCL_VULKAN #endif +#if defined(LLAMA_USE_CURL) +#ifdef __linux__ +#include +#elif defined(_WIN32) +#define PATH_MAX MAX_PATH +#else +#include +#endif +#define LLAMA_CURL_MAX_PATH_LENGTH PATH_MAX +#define LLAMA_CURL_MAX_HEADER_LENGTH 256 +#endif // LLAMA_USE_CURL + int32_t get_num_physical_cores() { #ifdef __linux__ // enumerate the set of thread siblings, num entries is num cores @@ -644,6 +659,13 @@ bool gpt_params_parse_ex(int argc, char ** argv, gpt_params & params) { } params.model = argv[i]; } + if (arg == "-mu" || arg == "--model-url") { + if (++i >= argc) { + invalid_param = true; + break; + } + params.model_url = argv[i]; + } if (arg == "-md" || arg == "--model-draft") { arg_found = true; if (++i >= argc) { @@ -1368,6 +1390,8 @@ void gpt_print_usage(int /*argc*/, char ** argv, const gpt_params & params) { printf(" layer range to apply the control vector(s) to, start and end inclusive\n"); printf(" -m FNAME, --model FNAME\n"); printf(" model path (default: %s)\n", params.model.c_str()); + printf(" -mu MODEL_URL, --model-url MODEL_URL\n"); + printf(" model download url (default: %s)\n", params.model_url.c_str()); printf(" -md FNAME, --model-draft FNAME\n"); printf(" draft model for speculative decoding\n"); printf(" -ld LOGDIR, --logdir LOGDIR\n"); @@ -1613,10 +1637,222 @@ void llama_batch_add( batch.n_tokens++; } +#ifdef LLAMA_USE_CURL + +struct llama_model * llama_load_model_from_url(const char * model_url, const char * path_model, + struct llama_model_params params) { + // Basic validation of the model_url + if (!model_url || strlen(model_url) == 0) { + fprintf(stderr, "%s: invalid model_url\n", __func__); + return NULL; + } + + // Initialize libcurl globally + auto curl = curl_easy_init(); + + if (!curl) { + fprintf(stderr, "%s: error initializing libcurl\n", __func__); + return NULL; + } + + // Set the URL, allow to follow http redirection + curl_easy_setopt(curl, CURLOPT_URL, model_url); + curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L); +#if defined(_WIN32) + // CURLSSLOPT_NATIVE_CA tells libcurl to use standard certificate store of + // operating system. Currently implemented under MS-Windows. + curl_easy_setopt(curl, CURLOPT_SSL_OPTIONS, CURLSSLOPT_NATIVE_CA); +#endif + + // Check if the file already exists locally + struct stat model_file_info; + auto file_exists = (stat(path_model, &model_file_info) == 0); + + // If the file exists, check for ${path_model}.etag or ${path_model}.lastModified files + char etag[LLAMA_CURL_MAX_HEADER_LENGTH] = {0}; + char etag_path[LLAMA_CURL_MAX_PATH_LENGTH] = {0}; + snprintf(etag_path, sizeof(etag_path), "%s.etag", path_model); + + char last_modified[LLAMA_CURL_MAX_HEADER_LENGTH] = {0}; + char last_modified_path[LLAMA_CURL_MAX_PATH_LENGTH] = {0}; + snprintf(last_modified_path, sizeof(last_modified_path), "%s.lastModified", path_model); + + if (file_exists) { + auto * f_etag = fopen(etag_path, "r"); + if (f_etag) { + if (!fgets(etag, sizeof(etag), f_etag)) { + fprintf(stderr, "%s: unable to read file %s\n", __func__, etag_path); + } else { + fprintf(stderr, "%s: previous model file found %s: %s\n", __func__, etag_path, etag); + } + fclose(f_etag); + } + + auto * f_last_modified = fopen(last_modified_path, "r"); + if (f_last_modified) { + if (!fgets(last_modified, sizeof(last_modified), f_last_modified)) { + fprintf(stderr, "%s: unable to read file %s\n", __func__, last_modified_path); + } else { + fprintf(stderr, "%s: previous model file found %s: %s\n", __func__, last_modified_path, + last_modified); + } + fclose(f_last_modified); + } + } + + // Send a HEAD request to retrieve the etag and last-modified headers + struct llama_load_model_from_url_headers { + char etag[LLAMA_CURL_MAX_HEADER_LENGTH] = {0}; + char last_modified[LLAMA_CURL_MAX_HEADER_LENGTH] = {0}; + }; + llama_load_model_from_url_headers headers; + { + typedef size_t(*CURLOPT_HEADERFUNCTION_PTR)(char *, size_t, size_t, void *); + auto header_callback = [](char * buffer, size_t /*size*/, size_t n_items, void * userdata) -> size_t { + llama_load_model_from_url_headers *headers = (llama_load_model_from_url_headers *) userdata; + + const char * etag_prefix = "etag: "; + if (strncmp(buffer, etag_prefix, strlen(etag_prefix)) == 0) { + strncpy(headers->etag, buffer + strlen(etag_prefix), n_items - strlen(etag_prefix) - 2); // Remove CRLF + } + + const char * last_modified_prefix = "last-modified: "; + if (strncmp(buffer, last_modified_prefix, strlen(last_modified_prefix)) == 0) { + strncpy(headers->last_modified, buffer + strlen(last_modified_prefix), + n_items - strlen(last_modified_prefix) - 2); // Remove CRLF + } + return n_items; + }; + + curl_easy_setopt(curl, CURLOPT_NOBODY, 1L); // will trigger the HEAD verb + curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 1L); // hide head request progress + curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, static_cast(header_callback)); + curl_easy_setopt(curl, CURLOPT_HEADERDATA, &headers); + + CURLcode res = curl_easy_perform(curl); + if (res != CURLE_OK) { + curl_easy_cleanup(curl); + fprintf(stderr, "%s: curl_easy_perform() failed: %s\n", __func__, curl_easy_strerror(res)); + return NULL; + } + + long http_code = 0; + curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &http_code); + if (http_code != 200) { + // HEAD not supported, we don't know if the file has changed + // force trigger downloading + file_exists = false; + fprintf(stderr, "%s: HEAD invalid http status code received: %ld\n", __func__, http_code); + } + } + + // If the ETag or the Last-Modified headers are different: trigger a new download + if (!file_exists || strcmp(etag, headers.etag) != 0 || strcmp(last_modified, headers.last_modified) != 0) { + char path_model_temporary[LLAMA_CURL_MAX_PATH_LENGTH] = {0}; + snprintf(path_model_temporary, sizeof(path_model_temporary), "%s.downloadInProgress", path_model); + if (file_exists) { + fprintf(stderr, "%s: deleting previous downloaded model file: %s\n", __func__, path_model); + if (remove(path_model) != 0) { + curl_easy_cleanup(curl); + fprintf(stderr, "%s: unable to delete file: %s\n", __func__, path_model); + return NULL; + } + } + + // Set the output file + auto * outfile = fopen(path_model_temporary, "wb"); + if (!outfile) { + curl_easy_cleanup(curl); + fprintf(stderr, "%s: error opening local file for writing: %s\n", __func__, path_model); + return NULL; + } + + typedef size_t(*CURLOPT_WRITEFUNCTION_PTR)(void * data, size_t size, size_t nmemb, void * fd); + auto write_callback = [](void * data, size_t size, size_t nmemb, void * fd) -> size_t { + return fwrite(data, size, nmemb, (FILE *)fd); + }; + curl_easy_setopt(curl, CURLOPT_NOBODY, 0L); + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, static_cast(write_callback)); + curl_easy_setopt(curl, CURLOPT_WRITEDATA, outfile); + + // display download progress + curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 0L); + + // start the download + fprintf(stderr, "%s: downloading model from %s to %s (server_etag:%s, server_last_modified:%s)...\n", __func__, + model_url, path_model, headers.etag, headers.last_modified); + auto res = curl_easy_perform(curl); + if (res != CURLE_OK) { + fclose(outfile); + curl_easy_cleanup(curl); + fprintf(stderr, "%s: curl_easy_perform() failed: %s\n", __func__, curl_easy_strerror(res)); + return NULL; + } + + long http_code = 0; + curl_easy_getinfo (curl, CURLINFO_RESPONSE_CODE, &http_code); + if (http_code < 200 || http_code >= 400) { + fclose(outfile); + curl_easy_cleanup(curl); + fprintf(stderr, "%s: invalid http status code received: %ld\n", __func__, http_code); + return NULL; + } + + // Clean up + fclose(outfile); + + // Write the new ETag to the .etag file + if (strlen(headers.etag) > 0) { + auto * etag_file = fopen(etag_path, "w"); + if (etag_file) { + fputs(headers.etag, etag_file); + fclose(etag_file); + fprintf(stderr, "%s: model etag saved %s: %s\n", __func__, etag_path, headers.etag); + } + } + + // Write the new lastModified to the .etag file + if (strlen(headers.last_modified) > 0) { + auto * last_modified_file = fopen(last_modified_path, "w"); + if (last_modified_file) { + fputs(headers.last_modified, last_modified_file); + fclose(last_modified_file); + fprintf(stderr, "%s: model last modified saved %s: %s\n", __func__, last_modified_path, + headers.last_modified); + } + } + + if (rename(path_model_temporary, path_model) != 0) { + curl_easy_cleanup(curl); + fprintf(stderr, "%s: unable to rename file: %s to %s\n", __func__, path_model_temporary, path_model); + return NULL; + } + } + + curl_easy_cleanup(curl); + + return llama_load_model_from_file(path_model, params); +} + +#else + +struct llama_model * llama_load_model_from_url(const char * /*model_url*/, const char * /*path_model*/, + struct llama_model_params /*params*/) { + fprintf(stderr, "%s: llama.cpp built without libcurl, downloading from an url not supported.\n", __func__); + return nullptr; +} + +#endif // LLAMA_USE_CURL + std::tuple llama_init_from_gpt_params(gpt_params & params) { auto mparams = llama_model_params_from_gpt_params(params); - llama_model * model = llama_load_model_from_file(params.model.c_str(), mparams); + llama_model * model = nullptr; + if (!params.model_url.empty()) { + model = llama_load_model_from_url(params.model_url.c_str(), params.model.c_str(), mparams); + } else { + model = llama_load_model_from_file(params.model.c_str(), mparams); + } if (model == NULL) { fprintf(stderr, "%s: error: failed to load model '%s'\n", __func__, params.model.c_str()); return std::make_tuple(nullptr, nullptr); diff --git a/common/common.h b/common/common.h index 687f3425e..8dd8a3edc 100644 --- a/common/common.h +++ b/common/common.h @@ -89,6 +89,7 @@ struct gpt_params { struct llama_sampling_params sparams; std::string model = "models/7B/ggml-model-f16.gguf"; // model path + std::string model_url = ""; // model url to download std::string model_draft = ""; // draft model for speculative decoding std::string model_alias = "unknown"; // model alias std::string prompt = ""; @@ -191,6 +192,9 @@ std::tuple llama_init_from_gpt_par struct llama_model_params llama_model_params_from_gpt_params (const gpt_params & params); struct llama_context_params llama_context_params_from_gpt_params(const gpt_params & params); +struct llama_model * llama_load_model_from_url(const char * model_url, const char * path_model, + struct llama_model_params params); + // Batch utils void llama_batch_clear(struct llama_batch & batch); diff --git a/examples/main/README.md b/examples/main/README.md index 7f84e4262..6a8d1e1c5 100644 --- a/examples/main/README.md +++ b/examples/main/README.md @@ -67,6 +67,7 @@ main.exe -m models\7B\ggml-model.bin --ignore-eos -n -1 --random-prompt In this section, we cover the most commonly used options for running the `main` program with the LLaMA models: - `-m FNAME, --model FNAME`: Specify the path to the LLaMA model file (e.g., `models/7B/ggml-model.bin`). +- `-mu MODEL_URL --model-url MODEL_URL`: Specify a remote http url to download the file (e.g https://huggingface.co/ggml-org/models/resolve/main/phi-2/ggml-model-q4_0.gguf). - `-i, --interactive`: Run the program in interactive mode, allowing you to provide input directly and receive real-time responses. - `-ins, --instruct`: Run the program in instruction mode, which is particularly useful when working with Alpaca models. - `-n N, --n-predict N`: Set the number of tokens to predict when generating text. Adjusting this value can influence the length of the generated text. diff --git a/examples/server/README.md b/examples/server/README.md index 8f8454aff..755e1d538 100644 --- a/examples/server/README.md +++ b/examples/server/README.md @@ -20,6 +20,7 @@ The project is under active development, and we are [looking for feedback and co - `-tb N, --threads-batch N`: Set the number of threads to use during batch and prompt processing. If not specified, the number of threads will be set to the number of threads used for generation. - `--threads-http N`: number of threads in the http server pool to process requests (default: `max(std::thread::hardware_concurrency() - 1, --parallel N + 2)`) - `-m FNAME`, `--model FNAME`: Specify the path to the LLaMA model file (e.g., `models/7B/ggml-model.gguf`). +- `-mu MODEL_URL --model-url MODEL_URL`: Specify a remote http url to download the file (e.g https://huggingface.co/ggml-org/models/resolve/main/phi-2/ggml-model-q4_0.gguf). - `-a ALIAS`, `--alias ALIAS`: Set an alias for the model. The alias will be returned in API responses. - `-c N`, `--ctx-size N`: Set the size of the prompt context. The default is 512, but LLaMA models were built with a context of 2048, which will provide better results for longer input/inference. The size may differ in other models, for example, baichuan models were build with a context of 4096. - `-ngl N`, `--n-gpu-layers N`: When compiled with appropriate support (currently CLBlast or cuBLAS), this option allows offloading some layers to the GPU for computation. Generally results in increased performance. diff --git a/examples/server/server.cpp b/examples/server/server.cpp index 895d608fd..d2a8e541d 100644 --- a/examples/server/server.cpp +++ b/examples/server/server.cpp @@ -2195,6 +2195,8 @@ static void server_print_usage(const char * argv0, const gpt_params & params, co } printf(" -m FNAME, --model FNAME\n"); printf(" model path (default: %s)\n", params.model.c_str()); + printf(" -mu MODEL_URL, --model-url MODEL_URL\n"); + printf(" model download url (default: %s)\n", params.model_url.c_str()); printf(" -a ALIAS, --alias ALIAS\n"); printf(" set an alias for the model, will be added as `model` field in completion response\n"); printf(" --lora FNAME apply LoRA adapter (implies --no-mmap)\n"); @@ -2317,6 +2319,12 @@ static void server_params_parse(int argc, char ** argv, server_params & sparams, break; } params.model = argv[i]; + } else if (arg == "-mu" || arg == "--model-url") { + if (++i >= argc) { + invalid_param = true; + break; + } + params.model_url = argv[i]; } else if (arg == "-a" || arg == "--alias") { if (++i >= argc) { invalid_param = true; diff --git a/examples/server/tests/README.md b/examples/server/tests/README.md index 95a0353b6..feb2b1d6c 100644 --- a/examples/server/tests/README.md +++ b/examples/server/tests/README.md @@ -57,7 +57,7 @@ Feature or Scenario must be annotated with `@llama.cpp` to be included in the de To run a scenario annotated with `@bug`, start: ```shell -DEBUG=ON ./tests.sh --no-skipped --tags bug +DEBUG=ON ./tests.sh --no-skipped --tags bug --stop ``` After changing logic in `steps.py`, ensure that `@bug` and `@wrong_usage` scenario are updated. diff --git a/examples/server/tests/features/embeddings.feature b/examples/server/tests/features/embeddings.feature index 57359b267..dcf1434f9 100644 --- a/examples/server/tests/features/embeddings.feature +++ b/examples/server/tests/features/embeddings.feature @@ -4,7 +4,8 @@ Feature: llama.cpp server Background: Server startup Given a server listening on localhost:8080 - And a model file bert-bge-small/ggml-model-f16.gguf from HF repo ggml-org/models + And a model url https://huggingface.co/ggml-org/models/resolve/main/bert-bge-small/ggml-model-f16.gguf + And a model file ggml-model-f16.gguf And a model alias bert-bge-small And 42 as server seed And 2 slots diff --git a/examples/server/tests/features/environment.py b/examples/server/tests/features/environment.py index 8ad987e1b..82104e920 100644 --- a/examples/server/tests/features/environment.py +++ b/examples/server/tests/features/environment.py @@ -1,10 +1,12 @@ -import errno import os -import socket -import subprocess -import time -from contextlib import closing import signal +import socket +import sys +import time +import traceback +from contextlib import closing + +import psutil def before_scenario(context, scenario): @@ -20,33 +22,40 @@ def before_scenario(context, scenario): def after_scenario(context, scenario): - if context.server_process is None: - return - if scenario.status == "failed": - if 'GITHUB_ACTIONS' in os.environ: - print(f"\x1b[33;101mSCENARIO FAILED: {scenario.name} server logs:\x1b[0m\n\n") - if os.path.isfile('llama.log'): - with closing(open('llama.log', 'r')) as f: - for line in f: - print(line) - if not is_server_listening(context.server_fqdn, context.server_port): - print("\x1b[33;101mERROR: Server stopped listening\x1b[0m\n") + try: + if 'server_process' not in context or context.server_process is None: + return + if scenario.status == "failed": + if 'GITHUB_ACTIONS' in os.environ: + print(f"\x1b[33;101mSCENARIO FAILED: {scenario.name} server logs:\x1b[0m\n\n") + if os.path.isfile('llama.log'): + with closing(open('llama.log', 'r')) as f: + for line in f: + print(line) + if not is_server_listening(context.server_fqdn, context.server_port): + print("\x1b[33;101mERROR: Server stopped listening\x1b[0m\n") - if not pid_exists(context.server_process.pid): - assert False, f"Server not running pid={context.server_process.pid} ..." + if not pid_exists(context.server_process.pid): + assert False, f"Server not running pid={context.server_process.pid} ..." - server_graceful_shutdown(context) + server_graceful_shutdown(context) - # Wait few for socket to free up - time.sleep(0.05) + # Wait few for socket to free up + time.sleep(0.05) - attempts = 0 - while pid_exists(context.server_process.pid) or is_server_listening(context.server_fqdn, context.server_port): - server_kill(context) - time.sleep(0.1) - attempts += 1 - if attempts > 5: - server_kill_hard(context) + attempts = 0 + while pid_exists(context.server_process.pid) or is_server_listening(context.server_fqdn, context.server_port): + server_kill(context) + time.sleep(0.1) + attempts += 1 + if attempts > 5: + server_kill_hard(context) + except: + exc = sys.exception() + print("error in after scenario: \n") + print(exc) + print("*** print_tb: \n") + traceback.print_tb(exc.__traceback__, file=sys.stdout) def server_graceful_shutdown(context): @@ -67,11 +76,11 @@ def server_kill_hard(context): path = context.server_path print(f"Server dangling exits, hard killing force {pid}={path}...\n") - if os.name == 'nt': - process = subprocess.check_output(['taskkill', '/F', '/pid', str(pid)]).decode() - print(process) - else: - os.kill(-pid, signal.SIGKILL) + try: + psutil.Process(pid).kill() + except psutil.NoSuchProcess: + return False + return True def is_server_listening(server_fqdn, server_port): @@ -84,17 +93,9 @@ def is_server_listening(server_fqdn, server_port): def pid_exists(pid): - """Check whether pid exists in the current process table.""" - if pid < 0: + try: + psutil.Process(pid) + except psutil.NoSuchProcess: return False - if os.name == 'nt': - output = subprocess.check_output(['TASKLIST', '/FI', f'pid eq {pid}']).decode() - print(output) - return "No tasks are running" not in output - else: - try: - os.kill(pid, 0) - except OSError as e: - return e.errno == errno.EPERM - else: - return True + return True + diff --git a/examples/server/tests/features/server.feature b/examples/server/tests/features/server.feature index 5014f326d..7448986e7 100644 --- a/examples/server/tests/features/server.feature +++ b/examples/server/tests/features/server.feature @@ -4,7 +4,8 @@ Feature: llama.cpp server Background: Server startup Given a server listening on localhost:8080 - And a model file tinyllamas/stories260K.gguf from HF repo ggml-org/models + And a model url https://huggingface.co/ggml-org/models/resolve/main/tinyllamas/stories260K.gguf + And a model file stories260K.gguf And a model alias tinyllama-2 And 42 as server seed # KV Cache corresponds to the total amount of tokens diff --git a/examples/server/tests/features/steps/steps.py b/examples/server/tests/features/steps/steps.py index a59a52d21..9e348d5fc 100644 --- a/examples/server/tests/features/steps/steps.py +++ b/examples/server/tests/features/steps/steps.py @@ -5,6 +5,8 @@ import os import re import socket import subprocess +import sys +import threading import time from contextlib import closing from re import RegexFlag @@ -32,6 +34,8 @@ def step_server_config(context, server_fqdn, server_port): context.base_url = f'http://{context.server_fqdn}:{context.server_port}' context.model_alias = None + context.model_file = None + context.model_url = None context.n_batch = None context.n_ubatch = None context.n_ctx = None @@ -65,6 +69,16 @@ def step_download_hf_model(context, hf_file, hf_repo): print(f"model file: {context.model_file}\n") +@step('a model file {model_file}') +def step_model_file(context, model_file): + context.model_file = model_file + + +@step('a model url {model_url}') +def step_model_url(context, model_url): + context.model_url = model_url + + @step('a model alias {model_alias}') def step_model_alias(context, model_alias): context.model_alias = model_alias @@ -141,7 +155,8 @@ def step_start_server(context): async def step_wait_for_the_server_to_be_started(context, expecting_status): match expecting_status: case 'healthy': - await wait_for_health_status(context, context.base_url, 200, 'ok') + await wait_for_health_status(context, context.base_url, 200, 'ok', + timeout=30) case 'ready' | 'idle': await wait_for_health_status(context, context.base_url, 200, 'ok', @@ -1038,8 +1053,11 @@ def start_server_background(context): server_args = [ '--host', server_listen_addr, '--port', context.server_port, - '--model', context.model_file ] + if context.model_file: + server_args.extend(['--model', context.model_file]) + if context.model_url: + server_args.extend(['--model-url', context.model_url]) if context.n_batch: server_args.extend(['--batch-size', context.n_batch]) if context.n_ubatch: @@ -1079,8 +1097,23 @@ def start_server_background(context): pkwargs = { 'creationflags': flags, + 'stdout': subprocess.PIPE, + 'stderr': subprocess.PIPE } context.server_process = subprocess.Popen( [str(arg) for arg in [context.server_path, *server_args]], **pkwargs) + + def log_stdout(process): + for line in iter(process.stdout.readline, b''): + print(line.decode('utf-8'), end='') + thread_stdout = threading.Thread(target=log_stdout, args=(context.server_process,)) + thread_stdout.start() + + def log_stderr(process): + for line in iter(process.stderr.readline, b''): + print(line.decode('utf-8'), end='', file=sys.stderr) + thread_stderr = threading.Thread(target=log_stderr, args=(context.server_process,)) + thread_stderr.start() + print(f"server pid={context.server_process.pid}, behave pid={os.getpid()}") diff --git a/examples/server/tests/requirements.txt b/examples/server/tests/requirements.txt index 2e4f42ad2..c2c960102 100644 --- a/examples/server/tests/requirements.txt +++ b/examples/server/tests/requirements.txt @@ -3,4 +3,5 @@ behave~=1.2.6 huggingface_hub~=0.20.3 numpy~=1.24.4 openai~=0.25.0 +psutil~=5.9.8 prometheus-client~=0.20.0 From 3a6efdd03c46c5ba08e43880d34260c02dd9999b Mon Sep 17 00:00:00 2001 From: Romain D <90720+Artefact2@users.noreply.github.com> Date: Mon, 18 Mar 2024 09:04:41 +0100 Subject: [PATCH 647/812] convert : use f32 outtype for bf16 tensors (#6106) The old behaviour is to use f16, but bf16 to f16 is not a lossless conversion. Change the outtype to f32 to default to a lossless conversion. --- convert.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/convert.py b/convert.py index 161430f3e..817cb6612 100755 --- a/convert.py +++ b/convert.py @@ -1167,9 +1167,9 @@ class OutputFile: def pick_output_type(model: LazyModel, output_type_str: str | None) -> GGMLFileType: wq_type = model[gguf.TENSOR_NAMES[gguf.MODEL_TENSOR.ATTN_Q].format(bid=0) + ".weight"].data_type - if output_type_str == "f32" or (output_type_str is None and wq_type == DT_F32): + if output_type_str == "f32" or (output_type_str is None and wq_type in (DT_F32, DT_BF16)): return GGMLFileType.AllF32 - if output_type_str == "f16" or (output_type_str is None and wq_type in (DT_F16, DT_BF16)): + if output_type_str == "f16" or (output_type_str is None and wq_type == DT_F16): return GGMLFileType.MostlyF16 if output_type_str == "q8_0": return GGMLFileType.MostlyQ8_0 From 9b03719ad712e2dc36c5c0c20f352bf3e4bda332 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9rence?= <13496987+Royalphax@users.noreply.github.com> Date: Mon, 18 Mar 2024 09:17:00 +0100 Subject: [PATCH 648/812] convert : add support for CamembertModel architecture (#6119) Adding support for CamembertModel architecture used by : https://huggingface.co/dangvantuan/sentence-camembert-large --- convert-hf-to-gguf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/convert-hf-to-gguf.py b/convert-hf-to-gguf.py index cf1f98d66..1e49d56c1 100755 --- a/convert-hf-to-gguf.py +++ b/convert-hf-to-gguf.py @@ -1634,7 +1634,7 @@ in chat mode so that the conversation can end normally.") self.post_write_tensors(tensor_map, name, data_torch) -@Model.register("BertModel") +@Model.register("BertModel", "CamembertModel") class BertModel(Model): model_arch = gguf.MODEL_ARCH.BERT From 496bc79bc2b79bfd6124b8687a8dbd6a646e9b06 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?DAN=E2=84=A2?= Date: Mon, 18 Mar 2024 04:27:44 -0400 Subject: [PATCH 649/812] common : tidy-up argument parsing (#6105) * Tidy-up argument parsing. * Missing ref. * common : minor * common : add static classifier --------- Co-authored-by: Georgi Gerganov --- common/common.cpp | 2057 +++++++++++++++++++++++---------------------- 1 file changed, 1035 insertions(+), 1022 deletions(-) diff --git a/common/common.cpp b/common/common.cpp index 2f5d965d6..919182862 100644 --- a/common/common.cpp +++ b/common/common.cpp @@ -154,6 +154,1040 @@ bool gpt_params_parse(int argc, char ** argv, gpt_params & params) { return result; } +static bool gpt_params_find_arg(int argc, char ** argv, gpt_params & params, int & i, bool & invalid_param) { + std::string arg = argv[i]; + llama_sampling_params& sparams = params.sparams; + + if (arg == "-s" || arg == "--seed") { + if (++i >= argc) { + invalid_param = true; + return true; + } + params.seed = std::stoul(argv[i]); + return true; + } + if (arg == "-t" || arg == "--threads") { + if (++i >= argc) { + invalid_param = true; + return true; + } + params.n_threads = std::stoi(argv[i]); + if (params.n_threads <= 0) { + params.n_threads = std::thread::hardware_concurrency(); + } + return true; + } + if (arg == "-tb" || arg == "--threads-batch") { + if (++i >= argc) { + invalid_param = true; + return true; + } + params.n_threads_batch = std::stoi(argv[i]); + if (params.n_threads_batch <= 0) { + params.n_threads_batch = std::thread::hardware_concurrency(); + } + return true; + } + if (arg == "-td" || arg == "--threads-draft") { + if (++i >= argc) { + invalid_param = true; + return true; + } + params.n_threads_draft = std::stoi(argv[i]); + if (params.n_threads_draft <= 0) { + params.n_threads_draft = std::thread::hardware_concurrency(); + } + return true; + } + if (arg == "-tbd" || arg == "--threads-batch-draft") { + if (++i >= argc) { + invalid_param = true; + return true; + } + params.n_threads_batch_draft = std::stoi(argv[i]); + if (params.n_threads_batch_draft <= 0) { + params.n_threads_batch_draft = std::thread::hardware_concurrency(); + } + return true; + } + if (arg == "-p" || arg == "--prompt") { + if (++i >= argc) { + invalid_param = true; + return true; + } + params.prompt = argv[i]; + return true; + } + if (arg == "-e" || arg == "--escape") { + params.escape = true; + return true; + } + if (arg == "--prompt-cache") { + if (++i >= argc) { + invalid_param = true; + return true; + } + params.path_prompt_cache = argv[i]; + return true; + } + if (arg == "--prompt-cache-all") { + params.prompt_cache_all = true; + return true; + } + if (arg == "--prompt-cache-ro") { + params.prompt_cache_ro = true; + return true; + } + if (arg == "-bf" || arg == "--binary-file") { + if (++i >= argc) { + invalid_param = true; + return true; + } + std::ifstream file(argv[i], std::ios::binary); + if (!file) { + fprintf(stderr, "error: failed to open file '%s'\n", argv[i]); + invalid_param = true; + return true; + } + // store the external file name in params + params.prompt_file = argv[i]; + std::ostringstream ss; + ss << file.rdbuf(); + params.prompt = ss.str(); + fprintf(stderr, "Read %zu bytes from binary file %s\n", params.prompt.size(), argv[i]); + return true; + } + if (arg == "-f" || arg == "--file") { + if (++i >= argc) { + invalid_param = true; + return true; + } + std::ifstream file(argv[i]); + if (!file) { + fprintf(stderr, "error: failed to open file '%s'\n", argv[i]); + invalid_param = true; + return true; + } + // store the external file name in params + params.prompt_file = argv[i]; + std::copy(std::istreambuf_iterator(file), std::istreambuf_iterator(), back_inserter(params.prompt)); + if (!params.prompt.empty() && params.prompt.back() == '\n') { + params.prompt.pop_back(); + } + return true; + } + if (arg == "-n" || arg == "--n-predict") { + if (++i >= argc) { + invalid_param = true; + return true; + } + params.n_predict = std::stoi(argv[i]); + return true; + } + if (arg == "--top-k") { + if (++i >= argc) { + invalid_param = true; + return true; + } + sparams.top_k = std::stoi(argv[i]); + return true; + } + if (arg == "-c" || arg == "--ctx-size") { + if (++i >= argc) { + invalid_param = true; + return true; + } + params.n_ctx = std::stoi(argv[i]); + return true; + } + if (arg == "--grp-attn-n" || arg == "-gan") { + if (++i >= argc) { + invalid_param = true; + return true; + } + params.grp_attn_n = std::stoi(argv[i]); + return true; + } + if (arg == "--grp-attn-w" || arg == "-gaw") { + if (++i >= argc) { + invalid_param = true; + return true; + } + params.grp_attn_w = std::stoi(argv[i]); + return true; + } + if (arg == "--rope-freq-base") { + if (++i >= argc) { + invalid_param = true; + return true; + } + params.rope_freq_base = std::stof(argv[i]); + return true; + } + if (arg == "--rope-freq-scale") { + if (++i >= argc) { + invalid_param = true; + return true; + } + params.rope_freq_scale = std::stof(argv[i]); + return true; + } + if (arg == "--rope-scaling") { + if (++i >= argc) { + invalid_param = true; + return true; + } + std::string value(argv[i]); + /**/ if (value == "none") { params.rope_scaling_type = LLAMA_ROPE_SCALING_TYPE_NONE; } + else if (value == "linear") { params.rope_scaling_type = LLAMA_ROPE_SCALING_TYPE_LINEAR; } + else if (value == "yarn") { params.rope_scaling_type = LLAMA_ROPE_SCALING_TYPE_YARN; } + else { invalid_param = true; } + return true; + } + if (arg == "--rope-scale") { + if (++i >= argc) { + invalid_param = true; + return true; + } + params.rope_freq_scale = 1.0f / std::stof(argv[i]); + return true; + } + if (arg == "--yarn-orig-ctx") { + if (++i >= argc) { + invalid_param = true; + return true; + } + params.yarn_orig_ctx = std::stoi(argv[i]); + return true; + } + if (arg == "--yarn-ext-factor") { + if (++i >= argc) { + invalid_param = true; + return true; + } + params.yarn_ext_factor = std::stof(argv[i]); + return true; + } + if (arg == "--yarn-attn-factor") { + if (++i >= argc) { + invalid_param = true; + return true; + } + params.yarn_attn_factor = std::stof(argv[i]); + return true; + } + if (arg == "--yarn-beta-fast") { + if (++i >= argc) { + invalid_param = true; + return true; + } + params.yarn_beta_fast = std::stof(argv[i]); + return true; + } + if (arg == "--yarn-beta-slow") { + if (++i >= argc) { + invalid_param = true; + return true; + } + params.yarn_beta_slow = std::stof(argv[i]); + return true; + } + if (arg == "--pooling") { + if (++i >= argc) { + invalid_param = true; + return true; + } + std::string value(argv[i]); + /**/ if (value == "none") { params.pooling_type = LLAMA_POOLING_TYPE_NONE; } + else if (value == "mean") { params.pooling_type = LLAMA_POOLING_TYPE_MEAN; } + else if (value == "cls") { params.pooling_type = LLAMA_POOLING_TYPE_CLS; } + else { invalid_param = true; } + return true; + } + if (arg == "--defrag-thold" || arg == "-dt") { + if (++i >= argc) { + invalid_param = true; + return true; + } + params.defrag_thold = std::stof(argv[i]); + return true; + } + if (arg == "--samplers") { + if (++i >= argc) { + invalid_param = true; + return true; + } + const auto sampler_names = string_split(argv[i], ';'); + sparams.samplers_sequence = sampler_types_from_names(sampler_names, true); + return true; + } + if (arg == "--sampling-seq") { + if (++i >= argc) { + invalid_param = true; + return true; + } + sparams.samplers_sequence = sampler_types_from_chars(argv[i]); + return true; + } + if (arg == "--top-p") { + if (++i >= argc) { + invalid_param = true; + return true; + } + sparams.top_p = std::stof(argv[i]); + return true; + } + if (arg == "--min-p") { + if (++i >= argc) { + invalid_param = true; + return true; + } + sparams.min_p = std::stof(argv[i]); + return true; + } + if (arg == "--temp") { + if (++i >= argc) { + invalid_param = true; + return true; + } + sparams.temp = std::stof(argv[i]); + sparams.temp = std::max(sparams.temp, 0.0f); + return true; + } + if (arg == "--tfs") { + if (++i >= argc) { + invalid_param = true; + return true; + } + sparams.tfs_z = std::stof(argv[i]); + return true; + } + if (arg == "--typical") { + if (++i >= argc) { + invalid_param = true; + return true; + } + sparams.typical_p = std::stof(argv[i]); + return true; + } + if (arg == "--repeat-last-n") { + if (++i >= argc) { + invalid_param = true; + return true; + } + sparams.penalty_last_n = std::stoi(argv[i]); + sparams.n_prev = std::max(sparams.n_prev, sparams.penalty_last_n); + return true; + } + if (arg == "--repeat-penalty") { + if (++i >= argc) { + invalid_param = true; + return true; + } + sparams.penalty_repeat = std::stof(argv[i]); + return true; + } + if (arg == "--frequency-penalty") { + if (++i >= argc) { + invalid_param = true; + return true; + } + sparams.penalty_freq = std::stof(argv[i]); + return true; + } + if (arg == "--presence-penalty") { + if (++i >= argc) { + invalid_param = true; + return true; + } + sparams.penalty_present = std::stof(argv[i]); + return true; + } + if (arg == "--dynatemp-range") { + if (++i >= argc) { + invalid_param = true; + return true; + } + sparams.dynatemp_range = std::stof(argv[i]); + return true; + } + if (arg == "--dynatemp-exp") { + if (++i >= argc) { + invalid_param = true; + return true; + } + sparams.dynatemp_exponent = std::stof(argv[i]); + return true; + } + if (arg == "--mirostat") { + if (++i >= argc) { + invalid_param = true; + return true; + } + sparams.mirostat = std::stoi(argv[i]); + return true; + } + if (arg == "--mirostat-lr") { + if (++i >= argc) { + invalid_param = true; + return true; + } + sparams.mirostat_eta = std::stof(argv[i]); + return true; + } + if (arg == "--mirostat-ent") { + if (++i >= argc) { + invalid_param = true; + return true; + } + sparams.mirostat_tau = std::stof(argv[i]); + return true; + } + if (arg == "--cfg-negative-prompt") { + if (++i >= argc) { + invalid_param = true; + return true; + } + sparams.cfg_negative_prompt = argv[i]; + return true; + } + if (arg == "--cfg-negative-prompt-file") { + if (++i >= argc) { + invalid_param = true; + return true; + } + std::ifstream file(argv[i]); + if (!file) { + fprintf(stderr, "error: failed to open file '%s'\n", argv[i]); + invalid_param = true; + return true; + } + std::copy(std::istreambuf_iterator(file), std::istreambuf_iterator(), back_inserter(sparams.cfg_negative_prompt)); + if (!sparams.cfg_negative_prompt.empty() && sparams.cfg_negative_prompt.back() == '\n') { + sparams.cfg_negative_prompt.pop_back(); + } + return true; + } + if (arg == "--cfg-scale") { + if (++i >= argc) { + invalid_param = true; + return true; + } + sparams.cfg_scale = std::stof(argv[i]); + return true; + } + if (arg == "-b" || arg == "--batch-size") { + if (++i >= argc) { + invalid_param = true; + return true; + } + params.n_batch = std::stoi(argv[i]); + return true; + } + if (arg == "-ub" || arg == "--ubatch-size") { + if (++i >= argc) { + invalid_param = true; + return true; + } + params.n_ubatch = std::stoi(argv[i]); + return true; + } + if (arg == "--keep") { + if (++i >= argc) { + invalid_param = true; + return true; + } + params.n_keep = std::stoi(argv[i]); + return true; + } + if (arg == "--draft") { + if (++i >= argc) { + invalid_param = true; + return true; + } + params.n_draft = std::stoi(argv[i]); + return true; + } + if (arg == "--chunks") { + if (++i >= argc) { + invalid_param = true; + return true; + } + params.n_chunks = std::stoi(argv[i]); + return true; + } + if (arg == "-np" || arg == "--parallel") { + if (++i >= argc) { + invalid_param = true; + return true; + } + params.n_parallel = std::stoi(argv[i]); + return true; + } + if (arg == "-ns" || arg == "--sequences") { + if (++i >= argc) { + invalid_param = true; + return true; + } + params.n_sequences = std::stoi(argv[i]); + return true; + } + if (arg == "--p-split" || arg == "-ps") { + if (++i >= argc) { + invalid_param = true; + return true; + } + params.p_split = std::stof(argv[i]); + return true; + } + if (arg == "-m" || arg == "--model") { + if (++i >= argc) { + invalid_param = true; + return true; + } + params.model = argv[i]; + return true; + } + if (arg == "-mu" || arg == "--model-url") { + if (++i >= argc) { + invalid_param = true; + return true; + } + params.model_url = argv[i]; + return true; + } + if (arg == "-md" || arg == "--model-draft") { + if (++i >= argc) { + invalid_param = true; + return true; + } + params.model_draft = argv[i]; + return true; + } + if (arg == "-a" || arg == "--alias") { + if (++i >= argc) { + invalid_param = true; + return true; + } + params.model_alias = argv[i]; + return true; + } + if (arg == "--lora") { + if (++i >= argc) { + invalid_param = true; + return true; + } + params.lora_adapter.emplace_back(argv[i], 1.0f); + params.use_mmap = false; + return true; + } + if (arg == "--lora-scaled") { + if (++i >= argc) { + invalid_param = true; + return true; + } + const char* lora_adapter = argv[i]; + if (++i >= argc) { + invalid_param = true; + return true; + } + params.lora_adapter.emplace_back(lora_adapter, std::stof(argv[i])); + params.use_mmap = false; + return true; + } + if (arg == "--lora-base") { + if (++i >= argc) { + invalid_param = true; + return true; + } + params.lora_base = argv[i]; + return true; + } + if (arg == "--control-vector") { + if (++i >= argc) { + invalid_param = true; + return true; + } + params.control_vectors.push_back({ 1.0f, argv[i], }); + return true; + } + if (arg == "--control-vector-scaled") { + if (++i >= argc) { + invalid_param = true; + return true; + } + const char* fname = argv[i]; + if (++i >= argc) { + invalid_param = true; + return true; + } + params.control_vectors.push_back({ std::stof(argv[i]), fname, }); + return true; + } + if (arg == "--control-vector-layer-range") { + if (++i >= argc) { + invalid_param = true; + return true; + } + params.control_vector_layer_start = std::stoi(argv[i]); + if (++i >= argc) { + invalid_param = true; + return true; + } + params.control_vector_layer_end = std::stoi(argv[i]); + return true; + } + if (arg == "--mmproj") { + if (++i >= argc) { + invalid_param = true; + return true; + } + params.mmproj = argv[i]; + return true; + } + if (arg == "--image") { + if (++i >= argc) { + invalid_param = true; + return true; + } + params.image = argv[i]; + return true; + } + if (arg == "-i" || arg == "--interactive") { + params.interactive = true; + return true; + } + if (arg == "--embedding") { + params.embedding = true; + return true; + } + if (arg == "--interactive-first") { + params.interactive_first = true; + return true; + } + if (arg == "-ins" || arg == "--instruct") { + params.instruct = true; + return true; + } + if (arg == "-cml" || arg == "--chatml") { + params.chatml = true; + return true; + } + if (arg == "--infill") { + params.infill = true; + return true; + } + if (arg == "-dkvc" || arg == "--dump-kv-cache") { + params.dump_kv_cache = true; + return true; + } + if (arg == "-nkvo" || arg == "--no-kv-offload") { + params.no_kv_offload = true; + return true; + } + if (arg == "-ctk" || arg == "--cache-type-k") { + params.cache_type_k = argv[++i]; + return true; + } + if (arg == "-ctv" || arg == "--cache-type-v") { + params.cache_type_v = argv[++i]; + return true; + } + if (arg == "--multiline-input") { + params.multiline_input = true; + return true; + } + if (arg == "--simple-io") { + params.simple_io = true; + return true; + } + if (arg == "-cb" || arg == "--cont-batching") { + params.cont_batching = true; + return true; + } + if (arg == "--color") { + params.use_color = true; + return true; + } + if (arg == "--mlock") { + params.use_mlock = true; + return true; + } + if (arg == "--gpu-layers" || arg == "-ngl" || arg == "--n-gpu-layers") { + if (++i >= argc) { + invalid_param = true; + return true; + } + params.n_gpu_layers = std::stoi(argv[i]); + if (!llama_supports_gpu_offload()) { + fprintf(stderr, "warning: not compiled with GPU offload support, --n-gpu-layers option will be ignored\n"); + fprintf(stderr, "warning: see main README.md for information on enabling GPU BLAS support\n"); + } + return true; + } + if (arg == "--gpu-layers-draft" || arg == "-ngld" || arg == "--n-gpu-layers-draft") { + if (++i >= argc) { + invalid_param = true; + return true; + } + params.n_gpu_layers_draft = std::stoi(argv[i]); + if (!llama_supports_gpu_offload()) { + fprintf(stderr, "warning: not compiled with GPU offload support, --n-gpu-layers-draft option will be ignored\n"); + fprintf(stderr, "warning: see main README.md for information on enabling GPU BLAS support\n"); + } + return true; + } + if (arg == "--main-gpu" || arg == "-mg") { + if (++i >= argc) { + invalid_param = true; + return true; + } + params.main_gpu = std::stoi(argv[i]); +#ifndef GGML_USE_CUBLAS_SYCL + fprintf(stderr, "warning: llama.cpp was compiled without cuBLAS/SYCL. Setting the main GPU has no effect.\n"); +#endif // GGML_USE_CUBLAS_SYCL + return true; + } + if (arg == "--split-mode" || arg == "-sm") { + if (++i >= argc) { + invalid_param = true; + return true; + } + std::string arg_next = argv[i]; + if (arg_next == "none") { + params.split_mode = LLAMA_SPLIT_MODE_NONE; + } + else if (arg_next == "layer") { + params.split_mode = LLAMA_SPLIT_MODE_LAYER; + } + else if (arg_next == "row") { +#ifdef GGML_USE_SYCL + fprintf(stderr, "warning: The split mode value:[row] is not supported by llama.cpp with SYCL. It's developing.\nExit!\n"); + exit(1); +#endif // GGML_USE_SYCL + params.split_mode = LLAMA_SPLIT_MODE_ROW; + } + else { + invalid_param = true; + return true; + } +#ifndef GGML_USE_CUBLAS_SYCL + fprintf(stderr, "warning: llama.cpp was compiled without cuBLAS/SYCL. Setting the split mode has no effect.\n"); +#endif // GGML_USE_CUBLAS_SYCL + return true; + } + if (arg == "--tensor-split" || arg == "-ts") { + if (++i >= argc) { + invalid_param = true; + return true; + } + std::string arg_next = argv[i]; + + // split string by , and / + const std::regex regex{ R"([,/]+)" }; + std::sregex_token_iterator it{ arg_next.begin(), arg_next.end(), regex, -1 }; + std::vector split_arg{ it, {} }; + if (split_arg.size() >= llama_max_devices()) { + invalid_param = true; + return true; + } + for (size_t i = 0; i < llama_max_devices(); ++i) { + if (i < split_arg.size()) { + params.tensor_split[i] = std::stof(split_arg[i]); + } + else { + params.tensor_split[i] = 0.0f; + } + } +#ifndef GGML_USE_CUBLAS_SYCL_VULKAN + fprintf(stderr, "warning: llama.cpp was compiled without cuBLAS/SYCL/Vulkan. Setting a tensor split has no effect.\n"); +#endif // GGML_USE_CUBLAS_SYCL + return true; + } + if (arg == "--no-mmap") { + params.use_mmap = false; + return true; + } + if (arg == "--numa") { + if (++i >= argc) { + invalid_param = true; + return true; + } + std::string value(argv[i]); + /**/ if (value == "distribute" || value == "") { params.numa = GGML_NUMA_STRATEGY_DISTRIBUTE; } + else if (value == "isolate") { params.numa = GGML_NUMA_STRATEGY_ISOLATE; } + else if (value == "numactl") { params.numa = GGML_NUMA_STRATEGY_NUMACTL; } + else { invalid_param = true; } + return true; + } + if (arg == "--verbose-prompt") { + params.verbose_prompt = true; + return true; + } + if (arg == "--no-display-prompt") { + params.display_prompt = false; + return true; + } + if (arg == "-r" || arg == "--reverse-prompt") { + if (++i >= argc) { + invalid_param = true; + return true; + } + params.antiprompt.emplace_back(argv[i]); + return true; + } + if (arg == "-ld" || arg == "--logdir") { + if (++i >= argc) { + invalid_param = true; + return true; + } + params.logdir = argv[i]; + + if (params.logdir.back() != DIRECTORY_SEPARATOR) { + params.logdir += DIRECTORY_SEPARATOR; + } + return true; + } + if (arg == "--save-all-logits" || arg == "--kl-divergence-base") { + if (++i >= argc) { + invalid_param = true; + return true; + } + params.logits_file = argv[i]; + return true; + } + if (arg == "--perplexity" || arg == "--all-logits") { + params.logits_all = true; + return true; + } + if (arg == "--ppl-stride") { + if (++i >= argc) { + invalid_param = true; + return true; + } + params.ppl_stride = std::stoi(argv[i]); + return true; + } + if (arg == "-ptc" || arg == "--print-token-count") { + if (++i >= argc) { + invalid_param = true; + return true; + } + params.n_print = std::stoi(argv[i]); + return true; + } + if (arg == "--ppl-output-type") { + if (++i >= argc) { + invalid_param = true; + return true; + } + params.ppl_output_type = std::stoi(argv[i]); + return true; + } + if (arg == "--hellaswag") { + params.hellaswag = true; + return true; + } + if (arg == "--hellaswag-tasks") { + if (++i >= argc) { + invalid_param = true; + return true; + } + params.hellaswag_tasks = std::stoi(argv[i]); + return true; + } + if (arg == "--winogrande") { + params.winogrande = true; + return true; + } + if (arg == "--winogrande-tasks") { + if (++i >= argc) { + invalid_param = true; + return true; + } + params.winogrande_tasks = std::stoi(argv[i]); + return true; + } + if (arg == "--multiple-choice") { + params.multiple_choice = true; + return true; + } + if (arg == "--multiple-choice-tasks") { + if (++i >= argc) { + invalid_param = true; + return true; + } + params.multiple_choice_tasks = std::stoi(argv[i]); + return true; + } + if (arg == "--kl-divergence") { + params.kl_divergence = true; + return true; + } + if (arg == "--ignore-eos") { + params.ignore_eos = true; + return true; + } + if (arg == "--no-penalize-nl") { + sparams.penalize_nl = false; + return true; + } + if (arg == "-l" || arg == "--logit-bias") { + if (++i >= argc) { + invalid_param = true; + return true; + } + std::stringstream ss(argv[i]); + llama_token key; + char sign; + std::string value_str; + try { + if (ss >> key && ss >> sign && std::getline(ss, value_str) && (sign == '+' || sign == '-')) { + sparams.logit_bias[key] = std::stof(value_str) * ((sign == '-') ? -1.0f : 1.0f); + } + else { + throw std::exception(); + } + } + catch (const std::exception&) { + invalid_param = true; + return true; + } + return true; + } + if (arg == "-h" || arg == "--help") { + return false; + } + if (arg == "--version") { + fprintf(stderr, "version: %d (%s)\n", LLAMA_BUILD_NUMBER, LLAMA_COMMIT); + fprintf(stderr, "built with %s for %s\n", LLAMA_COMPILER, LLAMA_BUILD_TARGET); + exit(0); + } + if (arg == "--random-prompt") { + params.random_prompt = true; + return true; + } + if (arg == "--in-prefix-bos") { + params.input_prefix_bos = true; + return true; + } + if (arg == "--in-prefix") { + if (++i >= argc) { + invalid_param = true; + return true; + } + params.input_prefix = argv[i]; + return true; + } + if (arg == "--in-suffix") { + if (++i >= argc) { + invalid_param = true; + return true; + } + params.input_suffix = argv[i]; + return true; + } + if (arg == "--grammar") { + if (++i >= argc) { + invalid_param = true; + return true; + } + sparams.grammar = argv[i]; + return true; + } + if (arg == "--grammar-file") { + if (++i >= argc) { + invalid_param = true; + return true; + } + std::ifstream file(argv[i]); + if (!file) { + fprintf(stderr, "error: failed to open file '%s'\n", argv[i]); + invalid_param = true; + return true; + } + std::copy( + std::istreambuf_iterator(file), + std::istreambuf_iterator(), + std::back_inserter(sparams.grammar) + ); + return true; + } + if (arg == "--override-kv") { + if (++i >= argc) { + invalid_param = true; + return true; + } + char* sep = strchr(argv[i], '='); + if (sep == nullptr || sep - argv[i] >= 128) { + fprintf(stderr, "error: Malformed KV override: %s\n", argv[i]); + invalid_param = true; + return true; + } + struct llama_model_kv_override kvo; + std::strncpy(kvo.key, argv[i], sep - argv[i]); + kvo.key[sep - argv[i]] = 0; + sep++; + if (strncmp(sep, "int:", 4) == 0) { + sep += 4; + kvo.tag = LLAMA_KV_OVERRIDE_TYPE_INT; + kvo.int_value = std::atol(sep); + } + else if (strncmp(sep, "float:", 6) == 0) { + sep += 6; + kvo.tag = LLAMA_KV_OVERRIDE_TYPE_FLOAT; + kvo.float_value = std::atof(sep); + } + else if (strncmp(sep, "bool:", 5) == 0) { + sep += 5; + kvo.tag = LLAMA_KV_OVERRIDE_TYPE_BOOL; + if (std::strcmp(sep, "true") == 0) { + kvo.bool_value = true; + } + else if (std::strcmp(sep, "false") == 0) { + kvo.bool_value = false; + } + else { + fprintf(stderr, "error: Invalid boolean value for KV override: %s\n", argv[i]); + invalid_param = true; + return true; + } + } + else { + fprintf(stderr, "error: Invalid type for KV override: %s\n", argv[i]); + invalid_param = true; + return true; + } + params.kv_overrides.push_back(kvo); + return true; + } +#ifndef LOG_DISABLE_LOGS + // Parse args for logging parameters + if (log_param_single_parse(argv[i])) { + // Do nothing, log_param_single_parse automatically does it's thing + // and returns if a match was found and parsed. + return true; + } + if (log_param_pair_parse( /*check_but_dont_parse*/ true, argv[i])) { + // We have a matching known parameter requiring an argument, + // now we need to check if there is anything after this argv + // and flag invalid_param or parse it. + if (++i >= argc) { + invalid_param = true; + return true; + } + if (!log_param_pair_parse( /*check_but_dont_parse*/ false, argv[i - 1], argv[i])) { + invalid_param = true; + return true; + } + return true; + } + // End of Parse args for logging parameters +#endif // LOG_DISABLE_LOGS + + return false; +} + bool gpt_params_parse_ex(int argc, char ** argv, gpt_params & params) { bool invalid_param = false; std::string arg; @@ -166,1028 +1200,7 @@ bool gpt_params_parse_ex(int argc, char ** argv, gpt_params & params) { std::replace(arg.begin(), arg.end(), '_', '-'); } - bool arg_found = false; - if (arg == "-s" || arg == "--seed") { - arg_found = true; - if (++i >= argc) { - invalid_param = true; - break; - } - params.seed = std::stoul(argv[i]); - } - if (arg == "-t" || arg == "--threads") { - arg_found = true; - if (++i >= argc) { - invalid_param = true; - break; - } - params.n_threads = std::stoi(argv[i]); - if (params.n_threads <= 0) { - params.n_threads = std::thread::hardware_concurrency(); - } - } - if (arg == "-tb" || arg == "--threads-batch") { - arg_found = true; - if (++i >= argc) { - invalid_param = true; - break; - } - params.n_threads_batch = std::stoi(argv[i]); - if (params.n_threads_batch <= 0) { - params.n_threads_batch = std::thread::hardware_concurrency(); - } - } - if (arg == "-td" || arg == "--threads-draft") { - arg_found = true; - if (++i >= argc) { - invalid_param = true; - break; - } - params.n_threads_draft = std::stoi(argv[i]); - if (params.n_threads_draft <= 0) { - params.n_threads_draft = std::thread::hardware_concurrency(); - } - } - if (arg == "-tbd" || arg == "--threads-batch-draft") { - arg_found = true; - if (++i >= argc) { - invalid_param = true; - break; - } - params.n_threads_batch_draft = std::stoi(argv[i]); - if (params.n_threads_batch_draft <= 0) { - params.n_threads_batch_draft = std::thread::hardware_concurrency(); - } - } - if (arg == "-p" || arg == "--prompt") { - arg_found = true; - if (++i >= argc) { - invalid_param = true; - break; - } - params.prompt = argv[i]; - } - if (arg == "-e" || arg == "--escape") { - arg_found = true; - params.escape = true; - } - if (arg == "--prompt-cache") { - arg_found = true; - if (++i >= argc) { - invalid_param = true; - break; - } - params.path_prompt_cache = argv[i]; - } - if (arg == "--prompt-cache-all") { - arg_found = true; - params.prompt_cache_all = true; - } - if (arg == "--prompt-cache-ro") { - arg_found = true; - params.prompt_cache_ro = true; - } - if (arg == "-bf" || arg == "--binary-file") { - arg_found = true; - if (++i >= argc) { - invalid_param = true; - break; - } - std::ifstream file(argv[i], std::ios::binary); - if (!file) { - fprintf(stderr, "error: failed to open file '%s'\n", argv[i]); - invalid_param = true; - break; - } - // store the external file name in params - params.prompt_file = argv[i]; - std::ostringstream ss; - ss << file.rdbuf(); - params.prompt = ss.str(); - fprintf(stderr, "Read %zu bytes from binary file %s\n", params.prompt.size(), argv[i]); - } - if (arg == "-f" || arg == "--file") { - arg_found = true; - if (++i >= argc) { - invalid_param = true; - break; - } - std::ifstream file(argv[i]); - if (!file) { - fprintf(stderr, "error: failed to open file '%s'\n", argv[i]); - invalid_param = true; - break; - } - // store the external file name in params - params.prompt_file = argv[i]; - std::copy(std::istreambuf_iterator(file), std::istreambuf_iterator(), back_inserter(params.prompt)); - if (!params.prompt.empty() && params.prompt.back() == '\n') { - params.prompt.pop_back(); - } - } - if (arg == "-n" || arg == "--n-predict") { - arg_found = true; - if (++i >= argc) { - invalid_param = true; - break; - } - params.n_predict = std::stoi(argv[i]); - } - if (arg == "--top-k") { - arg_found = true; - if (++i >= argc) { - invalid_param = true; - break; - } - sparams.top_k = std::stoi(argv[i]); - } - if (arg == "-c" || arg == "--ctx-size") { - arg_found = true; - if (++i >= argc) { - invalid_param = true; - break; - } - params.n_ctx = std::stoi(argv[i]); - } - if (arg == "--grp-attn-n" || arg == "-gan") { - arg_found = true; - if (++i >= argc) { - invalid_param = true; - break; - } - - params.grp_attn_n = std::stoi(argv[i]); - } - if (arg == "--grp-attn-w" || arg == "-gaw") { - arg_found = true; - if (++i >= argc) { - invalid_param = true; - break; - } - - params.grp_attn_w = std::stoi(argv[i]); - } - if (arg == "--rope-freq-base") { - arg_found = true; - if (++i >= argc) { - invalid_param = true; - break; - } - params.rope_freq_base = std::stof(argv[i]); - } - if (arg == "--rope-freq-scale") { - arg_found = true; - if (++i >= argc) { - invalid_param = true; - break; - } - params.rope_freq_scale = std::stof(argv[i]); - } - if (arg == "--rope-scaling") { - arg_found = true; - if (++i >= argc) { - invalid_param = true; - break; - } - std::string value(argv[i]); - /**/ if (value == "none") { params.rope_scaling_type = LLAMA_ROPE_SCALING_TYPE_NONE; } - else if (value == "linear") { params.rope_scaling_type = LLAMA_ROPE_SCALING_TYPE_LINEAR; } - else if (value == "yarn") { params.rope_scaling_type = LLAMA_ROPE_SCALING_TYPE_YARN; } - else { invalid_param = true; break; } - } - if (arg == "--rope-scale") { - arg_found = true; - if (++i >= argc) { - invalid_param = true; - break; - } - params.rope_freq_scale = 1.0f/std::stof(argv[i]); - } - if (arg == "--yarn-orig-ctx") { - arg_found = true; - if (++i >= argc) { - invalid_param = true; - break; - } - params.yarn_orig_ctx = std::stoi(argv[i]); - } - if (arg == "--yarn-ext-factor") { - arg_found = true; - if (++i >= argc) { - invalid_param = true; - break; - } - params.yarn_ext_factor = std::stof(argv[i]); - } - if (arg == "--yarn-attn-factor") { - arg_found = true; - if (++i >= argc) { - invalid_param = true; - break; - } - params.yarn_attn_factor = std::stof(argv[i]); - } - if (arg == "--yarn-beta-fast") { - arg_found = true; - if (++i >= argc) { - invalid_param = true; - break; - } - params.yarn_beta_fast = std::stof(argv[i]); - } - if (arg == "--yarn-beta-slow") { - arg_found = true; - if (++i >= argc) { - invalid_param = true; - break; - } - params.yarn_beta_slow = std::stof(argv[i]); - } - if (arg == "--pooling") { - arg_found = true; - if (++i >= argc) { - invalid_param = true; - break; - } - std::string value(argv[i]); - /**/ if (value == "none") { params.pooling_type = LLAMA_POOLING_TYPE_NONE; } - else if (value == "mean") { params.pooling_type = LLAMA_POOLING_TYPE_MEAN; } - else if (value == "cls") { params.pooling_type = LLAMA_POOLING_TYPE_CLS; } - else { invalid_param = true; break; } - } - if (arg == "--defrag-thold" || arg == "-dt") { - arg_found = true; - if (++i >= argc) { - invalid_param = true; - break; - } - params.defrag_thold = std::stof(argv[i]); - } - if (arg == "--samplers") { - arg_found = true; - if (++i >= argc) { - invalid_param = true; - break; - } - const auto sampler_names = string_split(argv[i], ';'); - sparams.samplers_sequence = sampler_types_from_names(sampler_names, true); - } - if (arg == "--sampling-seq") { - arg_found = true; - if (++i >= argc) { - invalid_param = true; - break; - } - sparams.samplers_sequence = sampler_types_from_chars(argv[i]); - } - if (arg == "--top-p") { - arg_found = true; - if (++i >= argc) { - invalid_param = true; - break; - } - sparams.top_p = std::stof(argv[i]); - } - if (arg == "--min-p") { - arg_found = true; - if (++i >= argc) { - invalid_param = true; - break; - } - sparams.min_p = std::stof(argv[i]); - } - if (arg == "--temp") { - arg_found = true; - if (++i >= argc) { - invalid_param = true; - break; - } - sparams.temp = std::stof(argv[i]); - sparams.temp = std::max(sparams.temp, 0.0f); - } - if (arg == "--tfs") { - arg_found = true; - if (++i >= argc) { - invalid_param = true; - break; - } - sparams.tfs_z = std::stof(argv[i]); - } - if (arg == "--typical") { - arg_found = true; - if (++i >= argc) { - invalid_param = true; - break; - } - sparams.typical_p = std::stof(argv[i]); - } - if (arg == "--repeat-last-n") { - arg_found = true; - if (++i >= argc) { - invalid_param = true; - break; - } - sparams.penalty_last_n = std::stoi(argv[i]); - sparams.n_prev = std::max(sparams.n_prev, sparams.penalty_last_n); - } - if (arg == "--repeat-penalty") { - arg_found = true; - if (++i >= argc) { - invalid_param = true; - break; - } - sparams.penalty_repeat = std::stof(argv[i]); - } - if (arg == "--frequency-penalty") { - arg_found = true; - if (++i >= argc) { - invalid_param = true; - break; - } - sparams.penalty_freq = std::stof(argv[i]); - } - if (arg == "--presence-penalty") { - arg_found = true; - if (++i >= argc) { - invalid_param = true; - break; - } - sparams.penalty_present = std::stof(argv[i]); - } - if (arg == "--dynatemp-range") { - arg_found = true; - if (++i >= argc) { - invalid_param = true; - break; - } - sparams.dynatemp_range = std::stof(argv[i]); - } - if (arg == "--dynatemp-exp") { - arg_found = true; - if (++i >= argc) { - invalid_param = true; - break; - } - sparams.dynatemp_exponent = std::stof(argv[i]); - } - if (arg == "--mirostat") { - arg_found = true; - if (++i >= argc) { - invalid_param = true; - break; - } - sparams.mirostat = std::stoi(argv[i]); - } - if (arg == "--mirostat-lr") { - arg_found = true; - if (++i >= argc) { - invalid_param = true; - break; - } - sparams.mirostat_eta = std::stof(argv[i]); - } - if (arg == "--mirostat-ent") { - arg_found = true; - if (++i >= argc) { - invalid_param = true; - break; - } - sparams.mirostat_tau = std::stof(argv[i]); - } - if (arg == "--cfg-negative-prompt") { - arg_found = true; - if (++i >= argc) { - invalid_param = true; - break; - } - sparams.cfg_negative_prompt = argv[i]; - } - if (arg == "--cfg-negative-prompt-file") { - arg_found = true; - if (++i >= argc) { - invalid_param = true; - break; - } - std::ifstream file(argv[i]); - if (!file) { - fprintf(stderr, "error: failed to open file '%s'\n", argv[i]); - invalid_param = true; - break; - } - std::copy(std::istreambuf_iterator(file), std::istreambuf_iterator(), back_inserter(sparams.cfg_negative_prompt)); - if (!sparams.cfg_negative_prompt.empty() && sparams.cfg_negative_prompt.back() == '\n') { - sparams.cfg_negative_prompt.pop_back(); - } - } - if (arg == "--cfg-scale") { - arg_found = true; - if (++i >= argc) { - invalid_param = true; - break; - } - sparams.cfg_scale = std::stof(argv[i]); - } - if (arg == "-b" || arg == "--batch-size") { - arg_found = true; - if (++i >= argc) { - invalid_param = true; - break; - } - params.n_batch = std::stoi(argv[i]); - } - if (arg == "-ub" || arg == "--ubatch-size") { - arg_found = true; - if (++i >= argc) { - invalid_param = true; - break; - } - params.n_ubatch = std::stoi(argv[i]); - } - if (arg == "--keep") { - arg_found = true; - if (++i >= argc) { - invalid_param = true; - break; - } - params.n_keep = std::stoi(argv[i]); - } - if (arg == "--draft") { - arg_found = true; - if (++i >= argc) { - invalid_param = true; - break; - } - params.n_draft = std::stoi(argv[i]); - } - if (arg == "--chunks") { - arg_found = true; - if (++i >= argc) { - invalid_param = true; - break; - } - params.n_chunks = std::stoi(argv[i]); - } - if (arg == "-np" || arg == "--parallel") { - arg_found = true; - if (++i >= argc) { - invalid_param = true; - break; - } - params.n_parallel = std::stoi(argv[i]); - } - if (arg == "-ns" || arg == "--sequences") { - arg_found = true; - if (++i >= argc) { - invalid_param = true; - break; - } - params.n_sequences = std::stoi(argv[i]); - } - if (arg == "--p-split" || arg == "-ps") { - arg_found = true; - if (++i >= argc) { - invalid_param = true; - break; - } - params.p_split = std::stof(argv[i]); - } - if (arg == "-m" || arg == "--model") { - arg_found = true; - if (++i >= argc) { - invalid_param = true; - break; - } - params.model = argv[i]; - } - if (arg == "-mu" || arg == "--model-url") { - if (++i >= argc) { - invalid_param = true; - break; - } - params.model_url = argv[i]; - } - if (arg == "-md" || arg == "--model-draft") { - arg_found = true; - if (++i >= argc) { - invalid_param = true; - break; - } - params.model_draft = argv[i]; - } - if (arg == "-a" || arg == "--alias") { - arg_found = true; - if (++i >= argc) { - invalid_param = true; - break; - } - params.model_alias = argv[i]; - } - if (arg == "--lora") { - arg_found = true; - if (++i >= argc) { - invalid_param = true; - break; - } - params.lora_adapter.emplace_back(argv[i], 1.0f); - params.use_mmap = false; - } - if (arg == "--lora-scaled") { - arg_found = true; - if (++i >= argc) { - invalid_param = true; - break; - } - const char * lora_adapter = argv[i]; - if (++i >= argc) { - invalid_param = true; - break; - } - params.lora_adapter.emplace_back(lora_adapter, std::stof(argv[i])); - params.use_mmap = false; - } - if (arg == "--lora-base") { - arg_found = true; - if (++i >= argc) { - invalid_param = true; - break; - } - params.lora_base = argv[i]; - } - if (arg == "--control-vector") { - arg_found = true; - if (++i >= argc) { - invalid_param = true; - break; - } - params.control_vectors.push_back({ 1.0f, argv[i], }); - } - if (arg == "--control-vector-scaled") { - arg_found = true; - if (++i >= argc) { - invalid_param = true; - break; - } - const char * fname = argv[i]; - if (++i >= argc) { - invalid_param = true; - break; - } - params.control_vectors.push_back({ std::stof(argv[i]), fname, }); - } - if (arg == "--control-vector-layer-range") { - arg_found = true; - if (++i >= argc) { - invalid_param = true; - break; - } - params.control_vector_layer_start = std::stoi(argv[i]); - if (++i >= argc) { - invalid_param = true; - break; - } - params.control_vector_layer_end = std::stoi(argv[i]); - } - if (arg == "--mmproj") { - arg_found = true; - if (++i >= argc) { - invalid_param = true; - break; - } - params.mmproj = argv[i]; - } - if (arg == "--image") { - arg_found = true; - if (++i >= argc) { - invalid_param = true; - break; - } - params.image = argv[i]; - } - if (arg == "-i" || arg == "--interactive") { - arg_found = true; - params.interactive = true; - } - if (arg == "--embedding") { - arg_found = true; - params.embedding = true; - } - if (arg == "--interactive-first") { - arg_found = true; - params.interactive_first = true; - } - if (arg == "-ins" || arg == "--instruct") { - arg_found = true; - params.instruct = true; - } - if (arg == "-cml" || arg == "--chatml") { - arg_found = true; - params.chatml = true; - } - if (arg == "--infill") { - arg_found = true; - params.infill = true; - } - if (arg == "-dkvc" || arg == "--dump-kv-cache") { - arg_found = true; - params.dump_kv_cache = true; - } - if (arg == "-nkvo" || arg == "--no-kv-offload") { - arg_found = true; - params.no_kv_offload = true; - } - if (arg == "-ctk" || arg == "--cache-type-k") { - arg_found = true; - params.cache_type_k = argv[++i]; - } - if (arg == "-ctv" || arg == "--cache-type-v") { - arg_found = true; - params.cache_type_v = argv[++i]; - } - if (arg == "--multiline-input") { - arg_found = true; - params.multiline_input = true; - } - if (arg == "--simple-io") { - arg_found = true; - params.simple_io = true; - } - if (arg == "-cb" || arg == "--cont-batching") { - arg_found = true; - params.cont_batching = true; - } - if (arg == "--color") { - arg_found = true; - params.use_color = true; - } - if (arg == "--mlock") { - arg_found = true; - params.use_mlock = true; - } - if (arg == "--gpu-layers" || arg == "-ngl" || arg == "--n-gpu-layers") { - arg_found = true; - if (++i >= argc) { - invalid_param = true; - break; - } - params.n_gpu_layers = std::stoi(argv[i]); - if (!llama_supports_gpu_offload()) { - fprintf(stderr, "warning: not compiled with GPU offload support, --n-gpu-layers option will be ignored\n"); - fprintf(stderr, "warning: see main README.md for information on enabling GPU BLAS support\n"); - } - } - if (arg == "--gpu-layers-draft" || arg == "-ngld" || arg == "--n-gpu-layers-draft") { - arg_found = true; - if (++i >= argc) { - invalid_param = true; - break; - } - params.n_gpu_layers_draft = std::stoi(argv[i]); - if (!llama_supports_gpu_offload()) { - fprintf(stderr, "warning: not compiled with GPU offload support, --n-gpu-layers-draft option will be ignored\n"); - fprintf(stderr, "warning: see main README.md for information on enabling GPU BLAS support\n"); - } - } - if (arg == "--main-gpu" || arg == "-mg") { - arg_found = true; - if (++i >= argc) { - invalid_param = true; - break; - } - params.main_gpu = std::stoi(argv[i]); -#ifndef GGML_USE_CUBLAS_SYCL - fprintf(stderr, "warning: llama.cpp was compiled without cuBLAS/SYCL. Setting the main GPU has no effect.\n"); -#endif // GGML_USE_CUBLAS_SYCL - } - if (arg == "--split-mode" || arg == "-sm") { - arg_found = true; - if (++i >= argc) { - invalid_param = true; - break; - } - std::string arg_next = argv[i]; - if (arg_next == "none") { - params.split_mode = LLAMA_SPLIT_MODE_NONE; - } else if (arg_next == "layer") { - params.split_mode = LLAMA_SPLIT_MODE_LAYER; - } else if (arg_next == "row") { -#ifdef GGML_USE_SYCL - fprintf(stderr, "warning: The split mode value:[row] is not supported by llama.cpp with SYCL. It's developing.\nExit!\n"); - exit(1); -#endif // GGML_USE_SYCL - params.split_mode = LLAMA_SPLIT_MODE_ROW; - } else { - invalid_param = true; - break; - } -#ifndef GGML_USE_CUBLAS_SYCL - fprintf(stderr, "warning: llama.cpp was compiled without cuBLAS/SYCL. Setting the split mode has no effect.\n"); -#endif // GGML_USE_CUBLAS_SYCL - - } - if (arg == "--tensor-split" || arg == "-ts") { - arg_found = true; - if (++i >= argc) { - invalid_param = true; - break; - } - std::string arg_next = argv[i]; - - // split string by , and / - const std::regex regex{R"([,/]+)"}; - std::sregex_token_iterator it{arg_next.begin(), arg_next.end(), regex, -1}; - std::vector split_arg{it, {}}; - if (split_arg.size() >= llama_max_devices()) { - invalid_param = true; - break; - } - for (size_t i = 0; i < llama_max_devices(); ++i) { - if (i < split_arg.size()) { - params.tensor_split[i] = std::stof(split_arg[i]); - } else { - params.tensor_split[i] = 0.0f; - } - } -#ifndef GGML_USE_CUBLAS_SYCL_VULKAN - fprintf(stderr, "warning: llama.cpp was compiled without cuBLAS/SYCL/Vulkan. Setting a tensor split has no effect.\n"); -#endif // GGML_USE_CUBLAS_SYCL - } - if (arg == "--no-mmap") { - arg_found = true; - params.use_mmap = false; - } - if (arg == "--numa") { - arg_found = true; - if (++i >= argc) { - invalid_param = true; - break; - } - std::string value(argv[i]); - /**/ if (value == "distribute" || value == "") { params.numa = GGML_NUMA_STRATEGY_DISTRIBUTE; } - else if (value == "isolate") { params.numa = GGML_NUMA_STRATEGY_ISOLATE; } - else if (value == "numactl") { params.numa = GGML_NUMA_STRATEGY_NUMACTL; } - else { invalid_param = true; break; } - } - if (arg == "--verbose-prompt") { - arg_found = true; - params.verbose_prompt = true; - } - if (arg == "--no-display-prompt") { - arg_found = true; - params.display_prompt = false; - } - if (arg == "-r" || arg == "--reverse-prompt") { - arg_found = true; - if (++i >= argc) { - invalid_param = true; - break; - } - params.antiprompt.emplace_back(argv[i]); - } - if (arg == "-ld" || arg == "--logdir") { - arg_found = true; - if (++i >= argc) { - invalid_param = true; - break; - } - params.logdir = argv[i]; - - if (params.logdir.back() != DIRECTORY_SEPARATOR) { - params.logdir += DIRECTORY_SEPARATOR; - } - } - if (arg == "--save-all-logits" || arg == "--kl-divergence-base") { - arg_found = true; - if (++i >= argc) { - invalid_param = true; - break; - } - params.logits_file = argv[i]; - } - if (arg == "--perplexity" || arg == "--all-logits") { - arg_found = true; - params.logits_all = true; - } - if (arg == "--ppl-stride") { - arg_found = true; - if (++i >= argc) { - invalid_param = true; - break; - } - params.ppl_stride = std::stoi(argv[i]); - } - if (arg == "-ptc" || arg == "--print-token-count") { - arg_found = true; - if (++i >= argc) { - invalid_param = true; - break; - } - params.n_print = std::stoi(argv[i]); - } - if (arg == "--ppl-output-type") { - arg_found = true; - if (++i >= argc) { - invalid_param = true; - break; - } - params.ppl_output_type = std::stoi(argv[i]); - } - if (arg == "--hellaswag") { - arg_found = true; - params.hellaswag = true; - } - if (arg == "--hellaswag-tasks") { - arg_found = true; - if (++i >= argc) { - invalid_param = true; - break; - } - params.hellaswag_tasks = std::stoi(argv[i]); - } - if (arg == "--winogrande") { - arg_found = true; - params.winogrande = true; - } - if (arg == "--winogrande-tasks") { - arg_found = true; - if (++i >= argc) { - invalid_param = true; - break; - } - params.winogrande_tasks = std::stoi(argv[i]); - } - if (arg == "--multiple-choice") { - arg_found = true; - params.multiple_choice = true; - } - if (arg == "--multiple-choice-tasks") { - arg_found = true; - if (++i >= argc) { - invalid_param = true; - break; - } - params.multiple_choice_tasks = std::stoi(argv[i]); - } - if (arg == "--kl-divergence") { - arg_found = true; - params.kl_divergence = true; - } - if (arg == "--ignore-eos") { - arg_found = true; - params.ignore_eos = true; - } - if (arg == "--no-penalize-nl") { - arg_found = true; - sparams.penalize_nl = false; - } - if (arg == "-l" || arg == "--logit-bias") { - arg_found = true; - if (++i >= argc) { - invalid_param = true; - break; - } - std::stringstream ss(argv[i]); - llama_token key; - char sign; - std::string value_str; - try { - if (ss >> key && ss >> sign && std::getline(ss, value_str) && (sign == '+' || sign == '-')) { - sparams.logit_bias[key] = std::stof(value_str) * ((sign == '-') ? -1.0f : 1.0f); - } else { - throw std::exception(); - } - } catch (const std::exception&) { - invalid_param = true; - break; - } - } - if (arg == "-h" || arg == "--help") { - arg_found = true; - return false; - } - if (arg == "--version") { - arg_found = true; - fprintf(stderr, "version: %d (%s)\n", LLAMA_BUILD_NUMBER, LLAMA_COMMIT); - fprintf(stderr, "built with %s for %s\n", LLAMA_COMPILER, LLAMA_BUILD_TARGET); - exit(0); - } - if (arg == "--random-prompt") { - arg_found = true; - params.random_prompt = true; - } - if (arg == "--in-prefix-bos") { - arg_found = true; - params.input_prefix_bos = true; - } - if (arg == "--in-prefix") { - arg_found = true; - if (++i >= argc) { - invalid_param = true; - break; - } - params.input_prefix = argv[i]; - } - if (arg == "--in-suffix") { - arg_found = true; - if (++i >= argc) { - invalid_param = true; - break; - } - params.input_suffix = argv[i]; - } - if (arg == "--grammar") { - arg_found = true; - if (++i >= argc) { - invalid_param = true; - break; - } - sparams.grammar = argv[i]; - } - if (arg == "--grammar-file") { - arg_found = true; - if (++i >= argc) { - invalid_param = true; - break; - } - std::ifstream file(argv[i]); - if (!file) { - fprintf(stderr, "error: failed to open file '%s'\n", argv[i]); - invalid_param = true; - break; - } - std::copy( - std::istreambuf_iterator(file), - std::istreambuf_iterator(), - std::back_inserter(sparams.grammar) - ); - } - if (arg == "--override-kv") { - arg_found = true; - if (++i >= argc) { - invalid_param = true; - break; - } - char * sep = strchr(argv[i], '='); - if (sep == nullptr || sep - argv[i] >= 128) { - fprintf(stderr, "error: Malformed KV override: %s\n", argv[i]); - invalid_param = true; - break; - } - struct llama_model_kv_override kvo; - std::strncpy(kvo.key, argv[i], sep - argv[i]); - kvo.key[sep - argv[i]] = 0; - sep++; - if (strncmp(sep, "int:", 4) == 0) { - sep += 4; - kvo.tag = LLAMA_KV_OVERRIDE_TYPE_INT; - kvo.int_value = std::atol(sep); - } else if (strncmp(sep, "float:", 6) == 0) { - sep += 6; - kvo.tag = LLAMA_KV_OVERRIDE_TYPE_FLOAT; - kvo.float_value = std::atof(sep); - } else if (strncmp(sep, "bool:", 5) == 0) { - sep += 5; - kvo.tag = LLAMA_KV_OVERRIDE_TYPE_BOOL; - if (std::strcmp(sep, "true") == 0) { - kvo.bool_value = true; - } else if (std::strcmp(sep, "false") == 0) { - kvo.bool_value = false; - } else { - fprintf(stderr, "error: Invalid boolean value for KV override: %s\n", argv[i]); - invalid_param = true; - break; - } - } else { - fprintf(stderr, "error: Invalid type for KV override: %s\n", argv[i]); - invalid_param = true; - break; - } - params.kv_overrides.push_back(kvo); -#ifndef LOG_DISABLE_LOGS - // Parse args for logging parameters - } - if ( log_param_single_parse( argv[i] ) ) { - arg_found = true; - // Do nothing, log_param_single_parse automatically does it's thing - // and returns if a match was found and parsed. - } - if ( log_param_pair_parse( /*check_but_dont_parse*/ true, argv[i] ) ) { - arg_found = true; - // We have a matching known parameter requiring an argument, - // now we need to check if there is anything after this argv - // and flag invalid_param or parse it. - if (++i >= argc) { - invalid_param = true; - break; - } - if( !log_param_pair_parse( /*check_but_dont_parse*/ false, argv[i-1], argv[i]) ) { - invalid_param = true; - break; - } - // End of Parse args for logging parameters -#endif // LOG_DISABLE_LOGS - } - - if (!arg_found) { + if (!gpt_params_find_arg(argc, argv, params, i, invalid_param)) { throw std::invalid_argument("error: unknown argument: " + arg); } } From 2bf8d0f7c4cc1235755ad06961ca761e458c5e55 Mon Sep 17 00:00:00 2001 From: slaren Date: Mon, 18 Mar 2024 11:03:04 +0100 Subject: [PATCH 650/812] backend : offload large batches to GPU (#6083) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * backend : offload large batches to GPU * fix hip * code cleanup * fix CUDA split buffers * Update ggml-backend-impl.h Co-authored-by: Johannes Gäßler * cuda : fix memset without set_device * imatrix : remove sched affix from weight names * sched : add a new split if the current one has too many inputs reduce max inputs per split more cleanup * update backends ggml-ci --------- Co-authored-by: Johannes Gäßler --- examples/imatrix/imatrix.cpp | 32 ++- examples/llama-bench/llama-bench.cpp | 4 +- ggml-alloc.c | 10 +- ggml-backend-impl.h | 5 + ggml-backend.c | 278 ++++++++++++++----------- ggml-backend.h | 8 +- ggml-cuda.cu | 297 +++++++++------------------ ggml-cuda.h | 21 +- ggml-kompute.cpp | 1 + ggml-metal.m | 1 + ggml-sycl.cpp | 1 + ggml-vulkan.cpp | 1 + ggml.c | 19 +- llama.cpp | 67 +++--- 14 files changed, 349 insertions(+), 396 deletions(-) diff --git a/examples/imatrix/imatrix.cpp b/examples/imatrix/imatrix.cpp index f21bc48f3..ea79b9062 100644 --- a/examples/imatrix/imatrix.cpp +++ b/examples/imatrix/imatrix.cpp @@ -56,13 +56,31 @@ bool IMatrixCollector::collect_imatrix(struct ggml_tensor * t, bool ask, void * const struct ggml_tensor * src0 = t->src[0]; const struct ggml_tensor * src1 = t->src[1]; + std::string wname; + { + // remove any prefix and suffixes from the name + // CUDA0#blk.0.attn_k.weight#0 => blk.0.attn_k.weight + const char * p = strchr(src0->name, '#'); + if (p != NULL) { + p = p + 1; + const char * q = strchr(p, '#'); + if (q != NULL) { + wname = std::string(p, q - p); + } else { + wname = p; + } + } else { + wname = src0->name; + } + } + // when ask is true, the scheduler wants to know if we are interested in data from this tensor // if we return true, a follow-up call will be made with ask=false in which we can do the actual collection if (ask) { if (t->op == GGML_OP_MUL_MAT_ID) return true; // collect all indirect matrix multiplications if (t->op != GGML_OP_MUL_MAT) return false; if (src1->ne[1] < 16 || src1->type != GGML_TYPE_F32) return false; - if (!(strncmp(src0->name, "blk.", 4) == 0 || (m_params.collect_output_weight && strcmp(src0->name, "output.weight") == 0))) return false; + if (!(wname.substr(0, 4) == "blk." || (m_params.collect_output_weight && wname == "output.weight"))) return false; return true; } @@ -94,12 +112,12 @@ bool IMatrixCollector::collect_imatrix(struct ggml_tensor * t, bool ask, void * // this is necessary to guarantee equal number of "ncall" for each tensor for (int ex = 0; ex < n_as; ++ex) { src0 = t->src[2 + ex]; - auto& e = m_stats[src0->name]; + auto& e = m_stats[wname]; if (e.values.empty()) { e.values.resize(src1->ne[0], 0); } else if (e.values.size() != (size_t)src1->ne[0]) { - fprintf(stderr, "Oops: inconsistent size for %s (%d vs %d)\n", src0->name, (int)e.values.size(), (int)src1->ne[0]); + fprintf(stderr, "Oops: inconsistent size for %s (%d vs %d)\n", wname.c_str(), (int)e.values.size(), (int)src1->ne[0]); exit(1); //GGML_ASSERT(false); } // NOTE: since we select top-k experts, the number of calls for the expert tensors will be k times larger @@ -107,7 +125,7 @@ bool IMatrixCollector::collect_imatrix(struct ggml_tensor * t, bool ask, void * //if (idx == t->src[0]->ne[0] - 1) ++e.ncall; ++e.ncall; if (m_params.verbosity > 1) { - printf("%s[%d]: %32s, %s, %5d x %5d, %d\n", __func__, m_last_call, src0->name, ggml_op_name(t->op), (int)src1->ne[0], (int)src1->ne[1], (int)src1->type); + printf("%s[%d]: %32s, %s, %5d x %5d, %d\n", __func__, m_last_call, wname.c_str(), ggml_op_name(t->op), (int)src1->ne[0], (int)src1->ne[1], (int)src1->type); } for (int row = 0; row < (int)src1->ne[1]; ++row) { const int excur = m_ids[row*n_as + idx]; @@ -129,17 +147,17 @@ bool IMatrixCollector::collect_imatrix(struct ggml_tensor * t, bool ask, void * } } } else { - auto& e = m_stats[src0->name]; + auto& e = m_stats[wname]; if (e.values.empty()) { e.values.resize(src1->ne[0], 0); } else if (e.values.size() != (size_t)src1->ne[0]) { - fprintf(stderr, "Oops: inconsistent size for %s (%d vs %d)\n", src0->name, (int)e.values.size(), (int)src1->ne[0]); + fprintf(stderr, "Oops: inconsistent size for %s (%d vs %d)\n", wname.c_str(), (int)e.values.size(), (int)src1->ne[0]); exit(1); //GGML_ASSERT(false); } ++e.ncall; if (m_params.verbosity > 1) { - printf("%s[%d]: %32s, %s, %5d x %5d, %d\n", __func__, m_last_call, src0->name, ggml_op_name(t->op), (int)src1->ne[0], (int)src1->ne[1], (int)src1->type); + printf("%s[%d]: %32s, %s, %5d x %5d, %d\n", __func__, m_last_call, wname.c_str(), ggml_op_name(t->op), (int)src1->ne[0], (int)src1->ne[1], (int)src1->type); } for (int row = 0; row < (int)src1->ne[1]; ++row) { const float * x = data + row * src1->ne[0]; diff --git a/examples/llama-bench/llama-bench.cpp b/examples/llama-bench/llama-bench.cpp index 32eea7869..4cb230804 100644 --- a/examples/llama-bench/llama-bench.cpp +++ b/examples/llama-bench/llama-bench.cpp @@ -114,10 +114,10 @@ static std::string get_cpu_info() { static std::string get_gpu_info() { std::string id; #ifdef GGML_USE_CUBLAS - int count = ggml_cuda_get_device_count(); + int count = ggml_backend_cuda_get_device_count(); for (int i = 0; i < count; i++) { char buf[128]; - ggml_cuda_get_device_description(i, buf, sizeof(buf)); + ggml_backend_cuda_get_device_description(i, buf, sizeof(buf)); id += buf; if (i < count - 1) { id += "/"; diff --git a/ggml-alloc.c b/ggml-alloc.c index 8ac1d3e51..643b2e55f 100644 --- a/ggml-alloc.c +++ b/ggml-alloc.c @@ -548,7 +548,11 @@ static void ggml_gallocr_alloc_graph_impl(ggml_gallocr_t galloc, struct ggml_cgr for (int i = 0; i < graph->n_nodes; i++) { struct ggml_tensor * node = graph->nodes[i]; - if (ggml_is_view(node)) { + // TODO: better way to add external dependencies + // GGML_OP_NONE does not appear normally in the graph nodes, but is used by ggml-backend to add dependencies to + // control when some tensors are allocated and freed. in this case, the dependencies are in `src`, but the node + // itself is never used and should not be considered a dependency + if (ggml_is_view(node) && node->op != GGML_OP_NONE) { struct ggml_tensor * view_src = node->view_src; ggml_gallocr_hash_get(galloc, view_src)->n_views += 1; } @@ -565,8 +569,8 @@ static void ggml_gallocr_alloc_graph_impl(ggml_gallocr_t galloc, struct ggml_cgr ggml_gallocr_hash_get(galloc, src)->n_children += 1; - // allocate explicit inputs and leafs - if (src->flags & GGML_TENSOR_FLAG_INPUT || src->op == GGML_OP_NONE) { + // allocate explicit inputs + if (src->flags & GGML_TENSOR_FLAG_INPUT) { ggml_gallocr_allocate_node(galloc, src, get_node_buffer_id(node_buffer_ids, i)); } } diff --git a/ggml-backend-impl.h b/ggml-backend-impl.h index e475e20e5..f121e1de4 100644 --- a/ggml-backend-impl.h +++ b/ggml-backend-impl.h @@ -103,6 +103,11 @@ extern "C" { // check if the backend supports an operation bool (*GGML_CALL supports_op)(ggml_backend_t backend, const struct ggml_tensor * op); + // check if the backend wants to run an operation, even if the weights are allocated in a CPU buffer + // these should be expensive operations with large batch sizes that may benefit from running on this backend + // even if the weight has to be copied from the CPU temporarily + bool (*GGML_CALL offload_op)(ggml_backend_t backend, const struct ggml_tensor * op); + // (optional) event synchronization ggml_backend_event_t (*GGML_CALL event_new) (ggml_backend_t backend); void (*GGML_CALL event_free) (ggml_backend_event_t event); diff --git a/ggml-backend.c b/ggml-backend.c index 31f8d5a6d..9f0084df7 100644 --- a/ggml-backend.c +++ b/ggml-backend.c @@ -278,7 +278,7 @@ enum ggml_status ggml_backend_graph_compute(ggml_backend_t backend, struct ggml_ return err; } -bool ggml_backend_graph_compute_async(ggml_backend_t backend, struct ggml_cgraph * cgraph) { +enum ggml_status ggml_backend_graph_compute_async(ggml_backend_t backend, struct ggml_cgraph * cgraph) { return backend->iface.graph_compute(backend, cgraph); } @@ -286,6 +286,13 @@ bool ggml_backend_supports_op(ggml_backend_t backend, const struct ggml_tensor * return backend->iface.supports_op(backend, op); } +bool ggml_backend_offload_op(ggml_backend_t backend, const struct ggml_tensor * op) { + if (backend->iface.offload_op != NULL) { + return backend->iface.offload_op(backend, op); + } + return false; +} + // backend copy static bool ggml_are_same_layout(const struct ggml_tensor * a, const struct ggml_tensor * b) { @@ -761,6 +768,10 @@ GGML_CALL static ggml_backend_graph_plan_t ggml_backend_cpu_graph_plan_create(gg if (cpu_plan->cplan.work_size > 0) { cpu_plan->cplan.work_data = malloc(cpu_plan->cplan.work_size); + if (cpu_plan->cplan.work_data == NULL) { + free(cpu_plan); + return NULL; + } } cpu_plan->cplan.abort_callback = cpu_ctx->abort_callback; @@ -834,6 +845,7 @@ static struct ggml_backend_i cpu_backend_i = { /* .graph_plan_compute = */ ggml_backend_cpu_graph_plan_compute, /* .graph_compute = */ ggml_backend_cpu_graph_compute, /* .supports_op = */ ggml_backend_cpu_supports_op, + /* .offload_op = */ NULL, /* .event_new = */ NULL, /* .event_free = */ NULL, /* .event_record = */ NULL, @@ -999,11 +1011,11 @@ static bool ggml_is_view_op(enum ggml_op op) { #endif #ifndef GGML_SCHED_MAX_SPLITS -#define GGML_SCHED_MAX_SPLITS 256 +#define GGML_SCHED_MAX_SPLITS 2048 #endif #ifndef GGML_SCHED_MAX_SPLIT_INPUTS -#define GGML_SCHED_MAX_SPLIT_INPUTS 16 +#define GGML_SCHED_MAX_SPLIT_INPUTS 4 #endif #ifndef GGML_SCHED_MAX_COPIES @@ -1043,8 +1055,9 @@ struct ggml_backend_sched { struct ggml_cgraph * graph; // graph splits - struct ggml_backend_sched_split splits[GGML_SCHED_MAX_SPLITS]; + struct ggml_backend_sched_split * splits; int n_splits; + int splits_capacity; // pipeline parallelism support int n_copies; @@ -1114,40 +1127,48 @@ static int ggml_backend_sched_backend_id_from_cur(ggml_backend_sched_t sched, st // TODO: use supports_op to check if the backend supports the op // assign pre-allocated nodes to their backend - // dst - int cur_backend = ggml_backend_sched_backend_from_buffer(sched, tensor); - if (cur_backend != -1) { + int cur_backend_id = ggml_backend_sched_backend_from_buffer(sched, tensor); + if (cur_backend_id != -1) { SET_CAUSE(tensor, "1.dst"); - return cur_backend; + return cur_backend_id; } // view_src if (tensor->view_src != NULL) { - cur_backend = ggml_backend_sched_backend_from_buffer(sched, tensor->view_src); - if (cur_backend != -1) { + cur_backend_id = ggml_backend_sched_backend_from_buffer(sched, tensor->view_src); + if (cur_backend_id != -1) { SET_CAUSE(tensor, "1.vsrc"); - return cur_backend; + return cur_backend_id; } } - // input + // graph input if (tensor->flags & GGML_TENSOR_FLAG_INPUT) { - cur_backend = sched->n_backends - 1; // last backend (assumed CPU) + cur_backend_id = sched->n_backends - 1; // last backend (assumed CPU) SET_CAUSE(tensor, "1.inp"); - return cur_backend; + return cur_backend_id; } // assign nodes that use weights to the backend of the weights + // operations with weights are preferably run on the same backend as the weights for (int i = 0; i < GGML_MAX_SRC; i++) { const struct ggml_tensor * src = tensor->src[i]; if (src == NULL) { continue; } if (src->buffer != NULL && src->buffer->usage == GGML_BACKEND_BUFFER_USAGE_WEIGHTS) { - int src_backend = ggml_backend_sched_backend_from_buffer(sched, src); - // operations with weights are always run on the same backend as the weights + int src_backend_id = ggml_backend_sched_backend_from_buffer(sched, src); + // check if a backend with higher prio wants to offload the op + if (src_backend_id == sched->n_backends - 1) { + for (int b = 0; b < src_backend_id; b++) { + if (ggml_backend_offload_op(sched->backends[b], tensor)) { + SET_CAUSE(tensor, "1.off"); + return b; + } + } + } SET_CAUSE(tensor, "1.wgt%d", i); - return src_backend; + return src_backend_id; } } @@ -1227,28 +1248,31 @@ static void ggml_backend_sched_split_graph(ggml_backend_sched_t sched, struct gg // pass 1: assign backends to ops with pre-allocated inputs for (int i = 0; i < graph->n_leafs; i++) { struct ggml_tensor * leaf = graph->leafs[i]; - if (tensor_backend_id(leaf) != -1) { + int * leaf_backend_id = &tensor_backend_id(leaf); + if (*leaf_backend_id != -1) { // do not overwrite user assignments continue; } - tensor_backend_id(leaf) = ggml_backend_sched_backend_id_from_cur(sched, leaf); + *leaf_backend_id = ggml_backend_sched_backend_id_from_cur(sched, leaf); } for (int i = 0; i < graph->n_nodes; i++) { struct ggml_tensor * node = graph->nodes[i]; - if (tensor_backend_id(node) != -1) { + int * node_backend_id = &tensor_backend_id(node); + if (*node_backend_id != -1) { // do not overwrite user assignments continue; } - tensor_backend_id(node) = ggml_backend_sched_backend_id_from_cur(sched, node); + *node_backend_id = ggml_backend_sched_backend_id_from_cur(sched, node); // src for (int j = 0; j < GGML_MAX_SRC; j++) { struct ggml_tensor * src = node->src[j]; if (src == NULL) { continue; } - if (tensor_backend_id(src) == -1) { - tensor_backend_id(src) = ggml_backend_sched_backend_id_from_cur(sched, src); + int * src_backend_id = &tensor_backend_id(src); + if (*src_backend_id == -1) { + *src_backend_id = ggml_backend_sched_backend_id_from_cur(sched, src); } } } @@ -1270,21 +1294,20 @@ static void ggml_backend_sched_split_graph(ggml_backend_sched_t sched, struct gg if (ggml_is_view_op(node->op)) { continue; } - int tensor_backend_id = tensor_backend_id(node); - if (tensor_backend_id != -1) { - if (tensor_backend_id == sched->n_backends - 1) { + int * node_backend_id = &tensor_backend_id(node); + if (*node_backend_id != -1) { + if (*node_backend_id == sched->n_backends - 1) { // skip cpu (lowest prio backend) cur_backend_id = -1; } else { - cur_backend_id = tensor_backend_id; + cur_backend_id = *node_backend_id; } } else { - tensor_backend_id(node) = cur_backend_id; + *node_backend_id = cur_backend_id; SET_CAUSE(node, "2.2"); } } } - // pass 2.1 expand gpu up { int cur_backend_id = -1; @@ -1293,22 +1316,20 @@ static void ggml_backend_sched_split_graph(ggml_backend_sched_t sched, struct gg if (ggml_is_view_op(node->op)) { continue; } - int tensor_backend_id = tensor_backend_id(node); - if (tensor_backend_id != -1) { - if (tensor_backend_id == sched->n_backends - 1) { + int * node_backend_id = &tensor_backend_id(node); + if (*node_backend_id != -1) { + if (*node_backend_id == sched->n_backends - 1) { // skip cpu (lowest prio backend) cur_backend_id = -1; } else { - cur_backend_id = tensor_backend_id; + cur_backend_id = *node_backend_id; } } else { - tensor_backend_id(node) = cur_backend_id; + *node_backend_id = cur_backend_id; SET_CAUSE(node, "2.1"); } } } - - // pass 2.4 expand rest down { int cur_backend_id = -1; @@ -1317,16 +1338,16 @@ static void ggml_backend_sched_split_graph(ggml_backend_sched_t sched, struct gg if (ggml_is_view_op(node->op)) { continue; } - int tensor_backend_id = tensor_backend_id(node); - if (tensor_backend_id != -1) { - cur_backend_id = tensor_backend_id; + int * node_backend_id = &tensor_backend_id(node); + if (*node_backend_id != -1) { + cur_backend_id = *node_backend_id; } else { - tensor_backend_id(node) = cur_backend_id; + *node_backend_id = cur_backend_id; SET_CAUSE(node, "2.4"); } } } - // pass 2.3 expand rest up + // pass 2.3 expand rest up { int cur_backend_id = -1; for (int i = graph->n_nodes - 1; i >= 0; i--) { @@ -1334,11 +1355,11 @@ static void ggml_backend_sched_split_graph(ggml_backend_sched_t sched, struct gg if (ggml_is_view_op(node->op)) { continue; } - int tensor_backend_id = tensor_backend_id(node); - if (tensor_backend_id != -1) { - cur_backend_id = tensor_backend_id; + int * node_backend_id = &tensor_backend_id(node); + if (*node_backend_id != -1) { + cur_backend_id = *node_backend_id; } else { - tensor_backend_id(node) = cur_backend_id; + *node_backend_id = cur_backend_id; SET_CAUSE(node, "2.3"); } } @@ -1351,9 +1372,9 @@ static void ggml_backend_sched_split_graph(ggml_backend_sched_t sched, struct gg // pass 3: assign backends to remaining src from dst and view_src for (int i = 0; i < graph->n_nodes; i++) { struct ggml_tensor * node = graph->nodes[i]; - int cur_backend_id = tensor_backend_id(node); - if (node->view_src != NULL && cur_backend_id == -1) { - cur_backend_id = tensor_backend_id(node) = tensor_backend_id(node->view_src); + int * cur_backend_id = &tensor_backend_id(node); + if (node->view_src != NULL && *cur_backend_id == -1) { + *cur_backend_id = tensor_backend_id(node->view_src); SET_CAUSE(node, "3.vsrc"); } for (int j = 0; j < GGML_MAX_SRC; j++) { @@ -1361,14 +1382,14 @@ static void ggml_backend_sched_split_graph(ggml_backend_sched_t sched, struct gg if (src == NULL) { continue; } - int src_backend_id = tensor_backend_id(src); - if (src_backend_id == -1) { + int * src_backend_id = &tensor_backend_id(src); + if (*src_backend_id == -1) { if (src->view_src != NULL) { // views are always on the same backend as the source - tensor_backend_id(src) = tensor_backend_id(src->view_src); + *src_backend_id = tensor_backend_id(src->view_src); SET_CAUSE(src, "3.vsrc"); } else { - tensor_backend_id(src) = cur_backend_id; + *src_backend_id = *cur_backend_id; SET_CAUSE(src, "3.cur"); } } @@ -1380,19 +1401,20 @@ static void ggml_backend_sched_split_graph(ggml_backend_sched_t sched, struct gg // pass 4: split graph, find tensors that need to be copied { - int cur_split = 0; + int i_split = 0; + struct ggml_backend_sched_split * split = &sched->splits[0]; // find the backend of the first split, skipping view ops for (int i = 0; i < graph->n_nodes; i++) { struct ggml_tensor * node = graph->nodes[i]; if (!ggml_is_view_op(node->op)) { - sched->splits[0].backend_id = tensor_backend_id(node); + split->backend_id = tensor_backend_id(node); break; } } - sched->splits[0].i_start = 0; - sched->splits[0].n_inputs = 0; - memset(sched->splits[0].inputs, 0, sizeof(sched->splits[0].inputs)); //HACK - int cur_backend_id = sched->splits[0].backend_id; + split->i_start = 0; + split->n_inputs = 0; + memset(split->inputs, 0, sizeof(split->inputs)); //HACK + int cur_backend_id = split->backend_id; for (int i = 0; i < graph->n_nodes; i++) { struct ggml_tensor * node = graph->nodes[i]; @@ -1400,18 +1422,54 @@ static void ggml_backend_sched_split_graph(ggml_backend_sched_t sched, struct gg continue; } - int tensor_backend_id = tensor_backend_id(node); + const int node_backend_id = tensor_backend_id(node); - GGML_ASSERT(tensor_backend_id != -1); // all nodes should be assigned by now + GGML_ASSERT(node_backend_id != -1); // all nodes should be assigned by now - if (tensor_backend_id != cur_backend_id) { - sched->splits[cur_split].i_end = i; - cur_split++; - GGML_ASSERT(cur_split < GGML_SCHED_MAX_SPLITS); - sched->splits[cur_split].backend_id = tensor_backend_id; - sched->splits[cur_split].i_start = i; - sched->splits[cur_split].n_inputs = 0; - cur_backend_id = tensor_backend_id; + // check if we should start a new split based on the sources of the current node + bool need_new_split = false; + if (node_backend_id == cur_backend_id && split->n_inputs > 0) { + for (int j = 0; j < GGML_MAX_SRC; j++) { + struct ggml_tensor * src = node->src[j]; + if (src == NULL) { + continue; + } + // check if a weight is on a different backend + // by starting a new split, the memory of the previously offloaded weights can be reused + if (src->buffer != NULL && src->buffer->usage == GGML_BACKEND_BUFFER_USAGE_WEIGHTS) { + int src_backend_id = tensor_backend_id(src); + if (src_backend_id != -1 && src_backend_id != cur_backend_id) { + need_new_split = true; + break; + } + } + // check if the split has too many inputs + if (split->n_inputs == GGML_SCHED_MAX_SPLIT_INPUTS) { + const size_t id = hash_id(src); + int src_backend_id = sched->tensor_backend_id[id]; + if (src_backend_id != cur_backend_id && sched->tensor_copies[hash_id(src)][cur_backend_id][0] == NULL) { + //printf("starting new split because of too many inputs: node %s, input %s\n", node->name, src->name); + need_new_split = true; + break; + } + } + } + } + + if (node_backend_id != cur_backend_id || need_new_split) { + split->i_end = i; + i_split++; + if (i_split >= sched->splits_capacity) { + sched->splits_capacity *= 2; + sched->splits = realloc(sched->splits, sched->splits_capacity * sizeof(struct ggml_backend_sched_split)); + GGML_ASSERT(sched->splits != NULL); + } + GGML_ASSERT(i_split < GGML_SCHED_MAX_SPLITS); + split = &sched->splits[i_split]; + split->backend_id = node_backend_id; + split->i_start = i; + split->n_inputs = 0; + cur_backend_id = node_backend_id; } // find inputs that are not on the same backend @@ -1421,10 +1479,10 @@ static void ggml_backend_sched_split_graph(ggml_backend_sched_t sched, struct gg continue; } - int src_backend_id = tensor_backend_id(src); + const int src_backend_id = tensor_backend_id(src); assert(src_backend_id != -1); // all inputs should be assigned by now - if (src->flags & GGML_TENSOR_FLAG_INPUT) { + if (src->flags & GGML_TENSOR_FLAG_INPUT && sched->n_copies > 1) { size_t id = hash_id(src); if (sched->tensor_copies[id][src_backend_id][0] == NULL) { ggml_backend_t backend = sched->backends[src_backend_id]; @@ -1441,7 +1499,6 @@ static void ggml_backend_sched_split_graph(ggml_backend_sched_t sched, struct gg ggml_set_output(tensor_copy); // prevent ggml-alloc from overwriting the tensor } sched->tensor_copies[id][src_backend_id][c] = tensor_copy; - tensor_backend_id(tensor_copy) = src_backend_id; SET_CAUSE(tensor_copy, "4.cpy"); } int n_graph_inputs = sched->n_graph_inputs++; @@ -1450,9 +1507,9 @@ static void ggml_backend_sched_split_graph(ggml_backend_sched_t sched, struct gg } } - if (src_backend_id != tensor_backend_id) { + if (src_backend_id != node_backend_id) { // create a copy of the input in the split's backend - size_t id = hash_id(src); + const size_t id = hash_id(src); if (sched->tensor_copies[id][cur_backend_id][0] == NULL) { ggml_backend_t backend = sched->backends[cur_backend_id]; for (int c = 0; c < sched->n_copies; c++) { @@ -1463,76 +1520,42 @@ static void ggml_backend_sched_split_graph(ggml_backend_sched_t sched, struct gg ggml_set_output(tensor_copy); // prevent ggml-alloc from overwriting the tensor } sched->tensor_copies[id][cur_backend_id][c] = tensor_copy; - tensor_backend_id(tensor_copy) = cur_backend_id; SET_CAUSE(tensor_copy, "4.cpy"); } - int n_inputs = sched->splits[cur_split].n_inputs++; + int n_inputs = split->n_inputs++; GGML_ASSERT(n_inputs < GGML_SCHED_MAX_SPLIT_INPUTS); - sched->splits[cur_split].inputs[n_inputs] = src; + split->inputs[n_inputs] = src; } node->src[j] = sched->tensor_copies[id][cur_backend_id][sched->cur_copy]; } } } - sched->splits[cur_split].i_end = graph->n_nodes; - sched->n_splits = cur_split + 1; + split->i_end = graph->n_nodes; + sched->n_splits = i_split + 1; } #ifdef DEBUG_PASS4 fprintf(stderr, "PASS 4 ASSIGNMENTS\n"); ggml_backend_sched_print_assignments(sched, graph); #endif -#ifndef NDEBUG - // sanity check: all sources should have the same backend as the node - for (int i = 0; i < graph->n_nodes; i++) { - struct ggml_tensor * node = graph->nodes[i]; - ggml_backend_t tensor_backend = ggml_backend_sched_get_tensor_backend(sched, node); - if (tensor_backend == NULL) { - fprintf(stderr, "!!!!!!! %s has no backend\n", node->name); - } - if (node->view_src != NULL && tensor_backend != ggml_backend_sched_get_tensor_backend(sched, node->view_src)) { - fprintf(stderr, "!!!!!!! %s has backend %s, view_src %s has backend %s\n", - node->name, tensor_backend ? ggml_backend_name(tensor_backend) : "NULL", - node->view_src->name, ggml_backend_sched_get_tensor_backend(sched, node->view_src) ? - ggml_backend_name(ggml_backend_sched_get_tensor_backend(sched, node->view_src)) : "NULL"); - } - for (int j = 0; j < GGML_MAX_SRC; j++) { - struct ggml_tensor * src = node->src[j]; - if (src == NULL) { - continue; - } - ggml_backend_t src_backend = ggml_backend_sched_get_tensor_backend(sched, src); - if (src_backend != tensor_backend /* && src_backend != NULL */) { - fprintf(stderr, "!!!! %s has backend %s, src %d (%s) has backend %s\n", - node->name, tensor_backend ? ggml_backend_name(tensor_backend) : "NULL", - j, src->name, src_backend ? ggml_backend_name(src_backend) : "NULL"); - } - if (src->view_src != NULL && src_backend != ggml_backend_sched_get_tensor_backend(sched, src->view_src)) { - fprintf(stderr, "!!!!!!! [src] %s has backend %s, view_src %s has backend %s\n", - src->name, src_backend ? ggml_backend_name(src_backend) : "NULL", - src->view_src->name, ggml_backend_sched_get_tensor_backend(sched, src->view_src) ? - ggml_backend_name(ggml_backend_sched_get_tensor_backend(sched, src->view_src)) : "NULL"); - } - } - } - fflush(stderr); -#endif - // create copies of the graph for each split // TODO: avoid this copy - struct ggml_cgraph * graph_copy = ggml_new_graph_custom(sched->ctx, graph->n_nodes + sched->n_splits*GGML_SCHED_MAX_SPLIT_INPUTS, false); + struct ggml_cgraph * graph_copy = ggml_new_graph_custom(sched->ctx, graph->n_nodes + sched->n_splits*GGML_SCHED_MAX_SPLIT_INPUTS*2, false); for (int i = 0; i < sched->n_splits; i++) { struct ggml_backend_sched_split * split = &sched->splits[i]; split->graph = ggml_graph_view(graph, split->i_start, split->i_end); // add inputs to the graph copy so that they are allocated by ggml-alloc at the start of the split for (int j = 0; j < split->n_inputs; j++) { + assert(graph_copy->size > (graph_copy->n_nodes + 1)); + struct ggml_tensor * input = split->inputs[j]; - struct ggml_tensor * input_cpy = sched->tensor_copies[hash_id(input)][split->backend_id][sched->cur_copy]; + const size_t input_id = hash_id(input); + struct ggml_tensor * input_cpy = sched->tensor_copies[input_id][split->backend_id][sched->cur_copy]; // add a dependency to the input source so that it is not freed before the copy is done struct ggml_tensor * input_dep = ggml_view_tensor(sched->ctx, input); input_dep->src[0] = input; - sched->node_backend_ids[graph_copy->n_nodes] = tensor_backend_id(input); + sched->node_backend_ids[graph_copy->n_nodes] = sched->tensor_backend_id[input_id]; graph_copy->nodes[graph_copy->n_nodes++] = input_dep; // add a dependency to the input copy so that it is allocated at the start of the split @@ -1541,6 +1564,7 @@ static void ggml_backend_sched_split_graph(ggml_backend_sched_t sched, struct gg } for (int j = split->i_start; j < split->i_end; j++) { + assert(graph_copy->size > graph_copy->n_nodes); sched->node_backend_ids[graph_copy->n_nodes] = tensor_backend_id(graph->nodes[j]); graph_copy->nodes[graph_copy->n_nodes++] = graph->nodes[j]; } @@ -1625,13 +1649,12 @@ static enum ggml_status ggml_backend_sched_compute_splits(ggml_backend_sched_t s } ggml_backend_tensor_copy(input, input_cpy); } else { + // wait for the split backend to finish using the input before overwriting it if (sched->events[split_backend_id][sched->cur_copy] != NULL) { ggml_backend_event_wait(split_backend, sched->events[split_backend_id][sched->cur_copy]); } else { ggml_backend_synchronize(split_backend); - ggml_backend_synchronize(input_backend); } - ggml_backend_tensor_copy_async(input_backend, split_backend, input, input_cpy); } } @@ -1701,17 +1724,21 @@ ggml_backend_sched_t ggml_backend_sched_new( struct ggml_backend_sched * sched = calloc(sizeof(struct ggml_backend_sched), 1); // initialize hash table - sched->hash_set = ggml_hash_set_new(graph_size + GGML_SCHED_MAX_SPLITS*GGML_SCHED_MAX_SPLIT_INPUTS); + sched->hash_set = ggml_hash_set_new(graph_size); sched->tensor_backend_id = calloc(sizeof(sched->tensor_backend_id[0]), sched->hash_set.size); sched->tensor_copies = calloc(sizeof(sched->tensor_copies[0]), sched->hash_set.size); - sched->node_backend_ids = calloc(sizeof(sched->node_backend_ids[0]), graph_size); - sched->leaf_backend_ids = calloc(sizeof(sched->leaf_backend_ids[0]), graph_size); + + const size_t nodes_size = graph_size + GGML_SCHED_MAX_SPLITS*GGML_SCHED_MAX_SPLIT_INPUTS*2; + sched->node_backend_ids = calloc(sizeof(sched->node_backend_ids[0]), nodes_size); + sched->leaf_backend_ids = calloc(sizeof(sched->leaf_backend_ids[0]), nodes_size); sched->n_backends = n_backends; sched->n_copies = parallel ? GGML_SCHED_MAX_COPIES : 1; - GGML_ASSERT(sched->n_copies <= GGML_SCHED_MAX_COPIES); + const int initial_splits_capacity = 16; + sched->splits = calloc(sizeof(sched->splits[0]), initial_splits_capacity); + sched->splits_capacity = initial_splits_capacity; for (int b = 0; b < n_backends; b++) { sched->backends[b] = backends[b]; @@ -1742,6 +1769,7 @@ void ggml_backend_sched_free(ggml_backend_sched_t sched) { } ggml_gallocr_free(sched->galloc); ggml_free(sched->ctx); + free(sched->splits); free(sched->hash_set.keys); free(sched->tensor_backend_id); free(sched->tensor_copies); @@ -1762,6 +1790,8 @@ void ggml_backend_sched_reset(ggml_backend_sched_t sched) { } bool ggml_backend_sched_reserve(ggml_backend_sched_t sched, struct ggml_cgraph * measure_graph) { + GGML_ASSERT((int)sched->hash_set.size >= measure_graph->n_nodes); + ggml_backend_sched_split_graph(sched, measure_graph); // TODO: extract this to a separate function @@ -1776,7 +1806,7 @@ bool ggml_backend_sched_reserve(ggml_backend_sched_t sched, struct ggml_cgraph * } bool ggml_backend_sched_alloc_graph(ggml_backend_sched_t sched, struct ggml_cgraph * graph) { - GGML_ASSERT((int)sched->hash_set.size >= graph->n_nodes + GGML_SCHED_MAX_SPLITS*GGML_SCHED_MAX_SPLIT_INPUTS); + GGML_ASSERT((int)sched->hash_set.size >= graph->n_nodes); ggml_backend_sched_split_graph(sched, graph); diff --git a/ggml-backend.h b/ggml-backend.h index 099d9c258..422457ab6 100644 --- a/ggml-backend.h +++ b/ggml-backend.h @@ -70,11 +70,11 @@ extern "C" { GGML_API ggml_backend_graph_plan_t ggml_backend_graph_plan_create(ggml_backend_t backend, struct ggml_cgraph * cgraph); GGML_API void ggml_backend_graph_plan_free (ggml_backend_t backend, ggml_backend_graph_plan_t plan); - GGML_API enum ggml_status ggml_backend_graph_plan_compute(ggml_backend_t backend, ggml_backend_graph_plan_t plan); - GGML_API enum ggml_status ggml_backend_graph_compute (ggml_backend_t backend, struct ggml_cgraph * cgraph); - - GGML_API bool ggml_backend_graph_compute_async(ggml_backend_t backend, struct ggml_cgraph * cgraph); + GGML_API enum ggml_status ggml_backend_graph_plan_compute (ggml_backend_t backend, ggml_backend_graph_plan_t plan); + GGML_API enum ggml_status ggml_backend_graph_compute (ggml_backend_t backend, struct ggml_cgraph * cgraph); + GGML_API enum ggml_status ggml_backend_graph_compute_async(ggml_backend_t backend, struct ggml_cgraph * cgraph); GGML_API bool ggml_backend_supports_op(ggml_backend_t backend, const struct ggml_tensor * op); + GGML_API bool ggml_backend_offload_op(ggml_backend_t backend, const struct ggml_tensor * op); // tensor copy between different backends GGML_API void ggml_backend_tensor_copy(struct ggml_tensor * src, struct ggml_tensor * dst); diff --git a/ggml-cuda.cu b/ggml-cuda.cu index db595409a..139025588 100644 --- a/ggml-cuda.cu +++ b/ggml-cuda.cu @@ -82,6 +82,10 @@ #define cudaGetDeviceProperties hipGetDeviceProperties #define cudaGetErrorString hipGetErrorString #define cudaGetLastError hipGetLastError +#define cudaHostRegister hipHostRegister +#define cudaHostRegisterPortable hipHostRegisterPortable +#define cudaHostRegisterReadOnly hipHostRegisterReadOnly +#define cudaHostUnregister hipHostUnregister #define cudaLaunchHostFunc hipLaunchHostFunc #ifdef GGML_HIP_UMA #define cudaMalloc hipMallocManaged @@ -7787,11 +7791,7 @@ struct cuda_pool_alloc { static bool g_cublas_loaded = false; -GGML_CALL bool ggml_cublas_loaded(void) { - return g_cublas_loaded; -} - -GGML_CALL void ggml_init_cublas() { +static void ggml_init_cublas() { static bool initialized = false; if (!initialized) { @@ -7880,7 +7880,7 @@ GGML_CALL void ggml_init_cublas() { } } -GGML_CALL void * ggml_cuda_host_malloc(size_t size) { +static void * ggml_cuda_host_malloc(size_t size) { if (getenv("GGML_CUDA_NO_PINNED") != nullptr) { return nullptr; } @@ -7890,7 +7890,7 @@ GGML_CALL void * ggml_cuda_host_malloc(size_t size) { if (err != cudaSuccess) { // clear the error cudaGetLastError(); - fprintf(stderr, "WARNING: failed to allocate %.2f MB of pinned memory: %s\n", + fprintf(stderr, "%s: warning: failed to allocate %.2f MiB of pinned memory: %s\n", __func__, size/1024.0/1024.0, cudaGetErrorString(err)); return nullptr; } @@ -7898,7 +7898,7 @@ GGML_CALL void * ggml_cuda_host_malloc(size_t size) { return ptr; } -GGML_CALL void ggml_cuda_host_free(void * ptr) { +static void ggml_cuda_host_free(void * ptr) { CUDA_CHECK(cudaFreeHost(ptr)); } @@ -9036,21 +9036,13 @@ static void ggml_cuda_op_soft_max( // positions tensor float * src2_dd = nullptr; - cuda_pool_alloc src2_f; ggml_tensor * src2 = dst->src[2]; const bool use_src2 = src2 != nullptr; if (use_src2) { - const bool src2_on_device = src2->backend == GGML_BACKEND_TYPE_GPU; - - if (src2_on_device) { - ggml_tensor_extra_gpu * src2_extra = (ggml_tensor_extra_gpu *) src2->extra; - src2_dd = (float *) src2_extra->data_device[g_main_device]; - } else { - src2_dd = src2_f.alloc(ggml_nelements(src2)); - CUDA_CHECK(ggml_cuda_cpy_tensor_2d(src2_dd, src2, 0, 0, 0, 1, main_stream)); - } + ggml_tensor_extra_gpu * src2_extra = (ggml_tensor_extra_gpu *) src2->extra; + src2_dd = (float *) src2_extra->data_device[g_main_device]; } soft_max_f32_cuda(src0_dd, src1 ? src1_dd : nullptr, src2_dd, dst_dd, ne00, nrows_x, nrows_y, scale, max_bias, main_stream); @@ -9107,55 +9099,24 @@ static void ggml_cuda_op_flatten(const ggml_tensor * src0, const ggml_tensor * s ggml_tensor_extra_gpu * src1_extra = use_src1 ? (ggml_tensor_extra_gpu *) src1->extra : nullptr; ggml_tensor_extra_gpu * dst_extra = (ggml_tensor_extra_gpu *) dst->extra; - const bool src0_on_device = src0->backend == GGML_BACKEND_TYPE_GPU || src0->backend == GGML_BACKEND_TYPE_GPU_SPLIT; - const bool src1_on_device = use_src1 && src1->backend == GGML_BACKEND_TYPE_GPU; - const bool dst_on_device = dst->backend == GGML_BACKEND_TYPE_GPU; - // dd = data device float * src0_ddf = nullptr; float * src1_ddf = nullptr; float * dst_ddf = nullptr; - cuda_pool_alloc src0_f; - cuda_pool_alloc src1_f; - cuda_pool_alloc dst_f; - ggml_cuda_set_device(g_main_device); cudaStream_t main_stream = g_cudaStreams[g_main_device][0]; - if (src0_on_device) { - src0_ddf = (float *) src0_extra->data_device[g_main_device]; - } else { - src0_ddf = src0_f.alloc(ggml_nelements(src0)); - CUDA_CHECK(ggml_cuda_cpy_tensor_2d(src0_ddf, src0, 0, 0, 0, nrows0, main_stream)); - } + src0_ddf = (float *) src0_extra->data_device[g_main_device]; if (use_src1) { - if (src1_on_device) { - src1_ddf = (float *) src1_extra->data_device[g_main_device]; - } else { - src1_ddf = src1_f.alloc(ggml_nelements(src1)); - CUDA_CHECK(ggml_cuda_cpy_tensor_2d(src1_ddf, src1, 0, 0, 0, nrows1, main_stream)); - } - } - if (dst_on_device) { - dst_ddf = (float *) dst_extra->data_device[g_main_device]; - } else { - dst_ddf = dst_f.alloc(ggml_nelements(dst)); + src1_ddf = (float *) src1_extra->data_device[g_main_device]; } + dst_ddf = (float *) dst_extra->data_device[g_main_device]; // do the computation op(src0, src1, dst, src0_ddf, src1_ddf, dst_ddf, main_stream); CUDA_CHECK(cudaGetLastError()); - - // copy dst to host if necessary - if (!dst_on_device) { - CUDA_CHECK(cudaMemcpyAsync(dst->data, dst_ddf, ggml_nbytes(dst), cudaMemcpyDeviceToHost, main_stream)); - } - - if (dst->backend == GGML_BACKEND_TYPE_CPU) { - CUDA_CHECK(cudaDeviceSynchronize()); - } } static void ggml_cuda_set_peer_access(const int n_tokens) { @@ -9251,7 +9212,6 @@ static void ggml_cuda_op_mul_mat( ggml_tensor_extra_gpu * src1_extra = (ggml_tensor_extra_gpu *) src1->extra; ggml_tensor_extra_gpu * dst_extra = (ggml_tensor_extra_gpu *) dst->extra; - const bool src0_on_device = src0->backend == GGML_BACKEND_TYPE_GPU || src0->backend == GGML_BACKEND_TYPE_GPU_SPLIT; const bool src0_is_contiguous = ggml_is_contiguous(src0); const bool src1_is_contiguous = ggml_is_contiguous(src1); @@ -9322,13 +9282,13 @@ static void ggml_cuda_op_mul_mat( used_devices++; - const bool src1_on_device = src1->backend == GGML_BACKEND_TYPE_GPU && id == g_main_device; - const bool dst_on_device = dst->backend == GGML_BACKEND_TYPE_GPU && id == g_main_device; + const bool src1_on_device = id == g_main_device; // TODO: check from buffer + const bool dst_on_device = id == g_main_device; ggml_cuda_set_device(id); cudaStream_t stream = g_cudaStreams[id][0]; - if (src0_on_device && src0_is_contiguous) { + if (src0_is_contiguous) { dev[id].src0_dd = (char *) src0_extra->data_device[id]; } else { dev[id].src0_dd = dev[id].src0_dd_alloc.alloc(ggml_nbytes(src0)); @@ -9374,8 +9334,8 @@ static void ggml_cuda_op_mul_mat( continue; } - const bool src1_on_device = src1->backend == GGML_BACKEND_TYPE_GPU && id == g_main_device; - const bool dst_on_device = dst->backend == GGML_BACKEND_TYPE_GPU && id == g_main_device; + const bool src1_on_device = id == g_main_device; // TODO: check from buffer + const bool dst_on_device = id == g_main_device; const int64_t row_diff = dev[id].row_high - dev[id].row_low; ggml_cuda_set_device(id); @@ -9400,12 +9360,12 @@ static void ggml_cuda_op_mul_mat( // the main device memory buffer can be on VRAM scratch, with space for all partial results // in that case an offset on dst_ddf_i is needed - if (dst->backend == GGML_BACKEND_TYPE_GPU && id == g_main_device) { + if (id == g_main_device) { dst_dd_i += dev[id].row_low; // offset is 0 if no tensor split } // copy src0, src1 to device if necessary - if (src1->backend == GGML_BACKEND_TYPE_GPU && src1_is_contiguous) { + if (src1_is_contiguous) { if (id != g_main_device) { if (convert_src1_to_q8_1) { char * src1_ddq_i_source = dev[g_main_device].src1_ddq + src1_ddq_i_offset; @@ -9418,19 +9378,19 @@ static void ggml_cuda_op_mul_mat( src1_ncols*ne10*sizeof(float), stream)); } } - } else if (src1->backend == GGML_BACKEND_TYPE_CPU || (src1_on_device && !src1_is_contiguous)) { + } else if (src1_on_device && !src1_is_contiguous) { CUDA_CHECK(ggml_cuda_cpy_tensor_2d( src1_ddf_i, src1, i03, i02, src1_col_0, src1_col_0+src1_ncols, stream)); } else { GGML_ASSERT(false); } - if (convert_src1_to_q8_1 && (src1->backend == GGML_BACKEND_TYPE_CPU || !src1_is_contiguous)) { + if (convert_src1_to_q8_1 && !src1_is_contiguous) { quantize_row_q8_1_cuda(src1_ddf_i, src1_ddq_i, ne10, src1_ncols, src1_padded_col_size, stream); CUDA_CHECK(cudaGetLastError()); } - if (src1_col_0 == 0 && (!src0_on_device || !src0_is_contiguous) && i02 % i02_divisor == 0) { + if (src1_col_0 == 0 && !src0_is_contiguous && i02 % i02_divisor == 0) { CUDA_CHECK(ggml_cuda_cpy_tensor_2d(src0_dd_i, src0, i03, i02/i02_divisor, dev[id].row_low, dev[id].row_high, stream)); } @@ -9441,17 +9401,7 @@ static void ggml_cuda_op_mul_mat( // copy dst to host or other device if necessary if (!dst_on_device) { - void * dst_off_device; - cudaMemcpyKind kind; - if (dst->backend == GGML_BACKEND_TYPE_CPU) { - dst_off_device = dst->data; - kind = cudaMemcpyDeviceToHost; - } else if (dst->backend == GGML_BACKEND_TYPE_GPU) { - dst_off_device = dst_extra->data_device[g_main_device]; - kind = cudaMemcpyDeviceToDevice; - } else { - GGML_ASSERT(false); - } + void * dst_off_device = dst_extra->data_device[g_main_device]; if (split) { // src0 = weight matrix is saved as a transposed matrix for better memory layout. // dst is NOT transposed. @@ -9462,28 +9412,26 @@ static void ggml_cuda_op_mul_mat( GGML_ASSERT(dst->nb[1] == ne0*sizeof(float)); dhf_dst_i += src1_col_0*ne0 + dev[id].row_low; #if !defined(GGML_USE_HIPBLAS) - if (kind == cudaMemcpyDeviceToDevice) { - // cudaMemcpy2DAsync may fail with copies between vmm pools of different devices - cudaMemcpy3DPeerParms p = {}; - p.dstDevice = g_main_device; - p.dstPtr = make_cudaPitchedPtr(dhf_dst_i, ne0*sizeof(float), row_diff, src1_ncols); - p.srcDevice = id; - p.srcPtr = make_cudaPitchedPtr(dst_dd_i, row_diff*sizeof(float), row_diff, src1_ncols); - p.extent = make_cudaExtent(row_diff*sizeof(float), src1_ncols, 1); - CUDA_CHECK(cudaMemcpy3DPeerAsync(&p, stream)); - } else + // cudaMemcpy2DAsync may fail with copies between vmm pools of different devices + cudaMemcpy3DPeerParms p = {}; + p.dstDevice = g_main_device; + p.dstPtr = make_cudaPitchedPtr(dhf_dst_i, ne0*sizeof(float), row_diff, src1_ncols); + p.srcDevice = id; + p.srcPtr = make_cudaPitchedPtr(dst_dd_i, row_diff*sizeof(float), row_diff, src1_ncols); + p.extent = make_cudaExtent(row_diff*sizeof(float), src1_ncols, 1); + CUDA_CHECK(cudaMemcpy3DPeerAsync(&p, stream)); +#else + // HIP does not support cudaMemcpy3DPeerAsync or vmm pools + CUDA_CHECK(cudaMemcpy2DAsync(dhf_dst_i, ne0*sizeof(float), + dst_dd_i, row_diff*sizeof(float), + row_diff*sizeof(float), src1_ncols, + cudaMemcpyDeviceToDevice, stream)); #endif - { - CUDA_CHECK(cudaMemcpy2DAsync(dhf_dst_i, ne0*sizeof(float), - dst_dd_i, row_diff*sizeof(float), - row_diff*sizeof(float), src1_ncols, - kind, stream)); - } } else { float * dhf_dst_i = (float *) ((char *) dst_off_device + i02*nb2 + i03*nb3); GGML_ASSERT(dst->nb[1] == ne0*sizeof(float)); dhf_dst_i += src1_col_0*ne0; - CUDA_CHECK(cudaMemcpyAsync(dhf_dst_i, dst_dd_i, src1_ncols*ne0*sizeof(float), kind, stream)); + CUDA_CHECK(cudaMemcpyAsync(dhf_dst_i, dst_dd_i, src1_ncols*ne0*sizeof(float), cudaMemcpyDeviceToDevice, stream)); } } @@ -9510,11 +9458,6 @@ static void ggml_cuda_op_mul_mat( } } } - - if (dst->backend == GGML_BACKEND_TYPE_CPU) { - ggml_cuda_set_device(g_main_device); - CUDA_CHECK(cudaDeviceSynchronize()); - } } static void ggml_cuda_repeat(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { @@ -9599,36 +9542,19 @@ static void ggml_cuda_pad(const ggml_tensor * src0, const ggml_tensor * src1, gg static void ggml_cuda_arange(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { ggml_tensor_extra_gpu * dst_extra = (ggml_tensor_extra_gpu *) dst->extra; - const bool dst_on_device = dst->backend == GGML_BACKEND_TYPE_GPU; - // dd = data device float * src0_ddf = nullptr; float * src1_ddf = nullptr; float * dst_ddf = nullptr; - cuda_pool_alloc dst_f; - ggml_cuda_set_device(g_main_device); cudaStream_t main_stream = g_cudaStreams[g_main_device][0]; - if (dst_on_device) { - dst_ddf = (float *) dst_extra->data_device[g_main_device]; - } else { - dst_ddf = dst_f.alloc(ggml_nelements(dst)); - } + dst_ddf = (float *) dst_extra->data_device[g_main_device]; // do the computation ggml_cuda_op_arange(src0, src1, dst, src0_ddf, src1_ddf, dst_ddf, main_stream); CUDA_CHECK(cudaGetLastError()); - - // copy dst to host if necessary - if (!dst_on_device) { - CUDA_CHECK(cudaMemcpyAsync(dst->data, dst_ddf, ggml_nbytes(dst), cudaMemcpyDeviceToHost, main_stream)); - } - - if (dst->backend == GGML_BACKEND_TYPE_CPU) { - CUDA_CHECK(cudaDeviceSynchronize()); - } } static void ggml_cuda_timestep_embedding(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { @@ -9639,21 +9565,6 @@ static void ggml_cuda_rms_norm(const ggml_tensor * src0, const ggml_tensor * src ggml_cuda_op_flatten(src0, src1, dst, ggml_cuda_op_rms_norm); } -GGML_CALL bool ggml_cuda_can_mul_mat(const struct ggml_tensor * src0, const struct ggml_tensor * src1, struct ggml_tensor * dst) { - if (!g_cublas_loaded) return false; - - const int64_t ne10 = src1->ne[0]; - - const int64_t ne0 = dst->ne[0]; - const int64_t ne1 = dst->ne[1]; - - // TODO: find the optimal values for these - return (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16 || ggml_is_quantized(src0->type)) && - src1->type == GGML_TYPE_F32 && - dst->type == GGML_TYPE_F32 && - (ne0 >= 32 && ne1 >= 32 && ne10 >= 32); -} - static void ggml_cuda_mul_mat_vec_p021(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst){ GGML_ASSERT(ggml_is_permuted(src0) && ggml_is_permuted(src1)); GGML_ASSERT(src0->backend != GGML_BACKEND_TYPE_GPU_SPLIT); @@ -9891,11 +9802,6 @@ static void ggml_cuda_mul_mat_batched_cublas(const ggml_tensor * src0, const ggm } static void ggml_cuda_mul_mat(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { - const bool all_on_device = - (src0->backend == GGML_BACKEND_TYPE_GPU || src0->backend == GGML_BACKEND_TYPE_GPU_SPLIT) && - (src1->backend == GGML_BACKEND_TYPE_GPU) && - ( dst->backend == GGML_BACKEND_TYPE_GPU); - const bool split = src0->backend == GGML_BACKEND_TYPE_GPU_SPLIT; int64_t min_compute_capability = INT_MAX; @@ -9972,13 +9878,13 @@ static void ggml_cuda_mul_mat(const ggml_tensor * src0, const ggml_tensor * src1 //printf("src0 is contiguous %d, transposed %d, type = %s, name = %s\n", ggml_is_contiguous(src0), ggml_is_transposed(src0), ggml_type_name(src0->type), src0->name); //printf("src1 is contiguous %d, transposed %d, type = %s, name = %s\n", ggml_is_contiguous(src1), ggml_is_transposed(src1), ggml_type_name(src1->type), src1->name); - if (!split && all_on_device && !fp16_performance_good && src0->type == GGML_TYPE_F16 && ggml_is_permuted(src0) && ggml_is_permuted(src1) && src1->ne[1] == 1) { + if (!split && !fp16_performance_good && src0->type == GGML_TYPE_F16 && ggml_is_permuted(src0) && ggml_is_permuted(src1) && src1->ne[1] == 1) { // KQ single-batch ggml_cuda_mul_mat_vec_p021(src0, src1, dst); - } else if (!split && all_on_device && !fp16_performance_good && src0->type == GGML_TYPE_F16 && !ggml_is_contiguous(src0) && !ggml_is_transposed(src1) && src1->ne[1] == 1) { + } else if (!split && !fp16_performance_good && src0->type == GGML_TYPE_F16 && !ggml_is_contiguous(src0) && !ggml_is_transposed(src1) && src1->ne[1] == 1) { // KQV single-batch ggml_cuda_mul_mat_vec_nc(src0, src1, dst); - } else if (!split && all_on_device && fp16_performance_good && src0->type == GGML_TYPE_F16 && !ggml_is_transposed(src0) && !ggml_is_transposed(src1) && src1->ne[2]*src1->ne[3] > 1) { + } else if (!split && fp16_performance_good && src0->type == GGML_TYPE_F16 && !ggml_is_transposed(src0) && !ggml_is_transposed(src1) && src1->ne[2]*src1->ne[3] > 1) { // KQ + KQV multi-batch ggml_cuda_mul_mat_batched_cublas(src0, src1, dst); } else if (use_dequantize_mul_mat_vec) { @@ -10178,6 +10084,7 @@ static void ggml_cuda_mul_mat_id(const ggml_tensor * src0, const ggml_tensor * s ggml_cuda_mul_mat_id_cublas(dst); // TODO: mmq/mmv support #endif + cudaStream_t stream = g_cudaStreams[g_main_device][0]; const size_t nb11 = src1->nb[1]; const size_t nb1 = dst->nb[1]; @@ -10187,16 +10094,9 @@ static void ggml_cuda_mul_mat_id(const ggml_tensor * src0, const ggml_tensor * s const int32_t n_as = ((int32_t *) dst->op_params)[1]; std::vector ids_host(ggml_nbytes(ids)); - - cudaStream_t stream = g_cudaStreams[g_main_device][0]; - - if (ids->backend == GGML_BACKEND_TYPE_GPU) { - const char * ids_dev = (const char *)((const ggml_tensor_extra_gpu *)ids->extra)->data_device[g_main_device]; - CUDA_CHECK(cudaMemcpyAsync(ids_host.data(), ids_dev, ggml_nbytes(ids), cudaMemcpyDeviceToHost, stream)); - CUDA_CHECK(cudaStreamSynchronize(stream)); - } else { - memcpy(ids_host.data(), ids->data, ggml_nbytes(ids)); - } + const char * ids_dev = (const char *)((const ggml_tensor_extra_gpu *)ids->extra)->data_device[g_main_device]; + CUDA_CHECK(cudaMemcpyAsync(ids_host.data(), ids_dev, ggml_nbytes(ids), cudaMemcpyDeviceToHost, stream)); + CUDA_CHECK(cudaStreamSynchronize(stream)); const ggml_tensor_extra_gpu * src1_extra = (const ggml_tensor_extra_gpu *) src1->extra; const ggml_tensor_extra_gpu * dst_extra = (const ggml_tensor_extra_gpu *) dst->extra; @@ -10213,20 +10113,11 @@ static void ggml_cuda_mul_mat_id(const ggml_tensor * src0, const ggml_tensor * s src1_row.extra = &src1_row_extra; dst_row.extra = &dst_row_extra; - char * src1_original = src1->backend == GGML_BACKEND_TYPE_CPU ? - (char *) src1->data : (char *) src1_extra->data_device[g_main_device]; - char * dst_original = dst->backend == GGML_BACKEND_TYPE_CPU ? - (char *) dst->data : (char *) dst_extra->data_device[g_main_device]; + char * src1_original = (char *) src1_extra->data_device[g_main_device]; + char * dst_original = (char *) dst_extra->data_device[g_main_device]; if (src1->ne[1] == 1) { - GGML_ASSERT(src1->backend == GGML_BACKEND_TYPE_GPU); - GGML_ASSERT(dst->backend == GGML_BACKEND_TYPE_GPU); - for (int64_t i01 = 0; i01 < ids->ne[1]; i01++) { - //int32_t row_id; - //CUDA_CHECK(cudaMemcpyAsync(&row_id, ids_dev + i01*ids->nb[1] + id*ids->nb[0], sizeof(int32_t), cudaMemcpyDeviceToHost, g_cudaStreams[g_main_device][0])); - //CUDA_CHECK(cudaStreamSynchronize(g_cudaStreams[g_main_device][0])); - const int32_t row_id = *(const int32_t *) (ids_host.data() + i01*ids->nb[1] + id*ids->nb[0]); GGML_ASSERT(row_id >= 0 && row_id < n_as); @@ -10248,11 +10139,6 @@ static void ggml_cuda_mul_mat_id(const ggml_tensor * src0, const ggml_tensor * s src1_row_extra.data_device[g_main_device] = src1_contiguous.get(); dst_row_extra.data_device[g_main_device] = dst_contiguous.get(); - const cudaMemcpyKind src1_kind = src1->backend == GGML_BACKEND_TYPE_CPU ? - cudaMemcpyHostToDevice : cudaMemcpyDeviceToDevice; - const cudaMemcpyKind dst_kind = dst->backend == GGML_BACKEND_TYPE_CPU ? - cudaMemcpyDeviceToHost : cudaMemcpyDeviceToDevice; - for (int32_t row_id = 0; row_id < n_as; ++row_id) { const struct ggml_tensor * src0_row = dst->src[row_id + 2]; @@ -10267,7 +10153,7 @@ static void ggml_cuda_mul_mat_id(const ggml_tensor * src0, const ggml_tensor * s GGML_ASSERT(row_id >= 0 && row_id < n_as); CUDA_CHECK(cudaMemcpyAsync(src1_contiguous.get() + num_src1_rows*nb11, src1_original + i01*nb11, - nb11, src1_kind, stream)); + nb11, cudaMemcpyDeviceToDevice, stream)); num_src1_rows++; } @@ -10299,15 +10185,11 @@ static void ggml_cuda_mul_mat_id(const ggml_tensor * src0, const ggml_tensor * s GGML_ASSERT(row_id >= 0 && row_id < n_as); CUDA_CHECK(cudaMemcpyAsync(dst_original + i01*nb1, dst_contiguous.get() + num_src1_rows*nb1, - nb1, dst_kind, stream)); + nb1, cudaMemcpyDeviceToDevice, stream)); num_src1_rows++; } } } - - if (dst->backend == GGML_BACKEND_TYPE_CPU) { - CUDA_CHECK(cudaStreamSynchronize(stream)); - } } static void ggml_cuda_scale(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { @@ -10435,7 +10317,7 @@ static size_t ggml_nbytes_split(const struct ggml_tensor * tensor, int nrows_spl return nrows_split*ggml_row_size(tensor->type, tensor->ne[0]); } -GGML_CALL static void ggml_cuda_set_main_device(const int main_device) { +static void ggml_cuda_set_main_device(const int main_device) { if (main_device >= g_device_count) { fprintf(stderr, "warning: cannot set main_device=%d because there are only %d devices. Using device %d instead.\n", main_device, g_device_count, g_main_device); @@ -10450,18 +10332,9 @@ GGML_CALL static void ggml_cuda_set_main_device(const int main_device) { } } -GGML_CALL bool ggml_cuda_compute_forward(struct ggml_compute_params * params, struct ggml_tensor * tensor) { +static bool ggml_cuda_compute_forward(struct ggml_tensor * tensor) { if (!g_cublas_loaded) return false; - ggml_cuda_func_t func; - const bool any_on_device = tensor->backend == GGML_BACKEND_TYPE_GPU - || (tensor->src[0] != nullptr && (tensor->src[0]->backend == GGML_BACKEND_TYPE_GPU || tensor->src[0]->backend == GGML_BACKEND_TYPE_GPU_SPLIT)) - || (tensor->src[1] != nullptr && tensor->src[1]->backend == GGML_BACKEND_TYPE_GPU); - - if (!any_on_device && tensor->op != GGML_OP_MUL_MAT && tensor->op != GGML_OP_MUL_MAT_ID) { - return false; - } - if (tensor->op == GGML_OP_MUL_MAT) { if (tensor->src[0]->ne[3] != tensor->src[1]->ne[3]) { #ifndef NDEBUG @@ -10471,6 +10344,8 @@ GGML_CALL bool ggml_cuda_compute_forward(struct ggml_compute_params * params, st } } + ggml_cuda_func_t func; + switch (tensor->op) { case GGML_OP_REPEAT: func = ggml_cuda_repeat; @@ -10548,15 +10423,9 @@ GGML_CALL bool ggml_cuda_compute_forward(struct ggml_compute_params * params, st func = ggml_cuda_rms_norm; break; case GGML_OP_MUL_MAT: - if (!any_on_device && !ggml_cuda_can_mul_mat(tensor->src[0], tensor->src[1], tensor)) { - return false; - } func = ggml_cuda_mul_mat; break; case GGML_OP_MUL_MAT_ID: - if (!any_on_device && !ggml_cuda_can_mul_mat(tensor->src[2], tensor->src[1], tensor)) { - return false; - } func = ggml_cuda_mul_mat_id; break; case GGML_OP_SCALE: @@ -10613,17 +10482,11 @@ GGML_CALL bool ggml_cuda_compute_forward(struct ggml_compute_params * params, st ggml_cuda_set_peer_access(tensor->src[1]->ne[1]); } - if (params->ith != 0) { - return true; - } - if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) { - return true; - } func(tensor->src[0], tensor->src[1], tensor); return true; } -GGML_CALL int ggml_cuda_get_device_count() { +static int ggml_cuda_get_device_count() { int device_count; if (cudaGetDeviceCount(&device_count) != cudaSuccess) { return 0; @@ -10631,7 +10494,7 @@ GGML_CALL int ggml_cuda_get_device_count() { return device_count; } -GGML_CALL void ggml_cuda_get_device_description(int device, char * description, size_t description_size) { +static void ggml_cuda_get_device_description(int device, char * description, size_t description_size) { cudaDeviceProp prop; CUDA_CHECK(cudaGetDeviceProperties(&prop, device)); snprintf(description, description_size, "%s", prop.name); @@ -10736,6 +10599,7 @@ GGML_CALL static void ggml_backend_cuda_buffer_init_tensor(ggml_backend_buffer_t size_t padded_size = ggml_backend_buft_get_alloc_size(buffer->buft, tensor); if (padded_size > original_size && tensor->view_src == nullptr) { + ggml_cuda_set_device(ctx->device); CUDA_CHECK(cudaMemset((char *)tensor->data + original_size, 0, padded_size - original_size)); } } @@ -10873,6 +10737,8 @@ static ggml_backend_buffer_type_i ggml_backend_cuda_buffer_type_interface = { }; GGML_CALL ggml_backend_buffer_type_t ggml_backend_cuda_buffer_type(int device) { + ggml_init_cublas(); + // FIXME: this is not thread safe if (device >= ggml_backend_cuda_get_device_count()) { return nullptr; @@ -11157,6 +11023,8 @@ static ggml_backend_buffer_type_i ggml_backend_cuda_split_buffer_type_interface }; GGML_CALL ggml_backend_buffer_type_t ggml_backend_cuda_split_buffer_type(const float * tensor_split) { + ggml_init_cublas(); + // FIXME: this is not thread safe static std::map, struct ggml_backend_buffer_type> buft_map; @@ -11348,9 +11216,6 @@ GGML_CALL static enum ggml_status ggml_backend_cuda_graph_compute(ggml_backend_t ggml_cuda_set_main_device(cuda_ctx->device); - ggml_compute_params params = {}; - params.type = GGML_TASK_TYPE_COMPUTE; - params.ith = 0; for (int i = 0; i < cgraph->n_nodes; i++) { ggml_tensor * node = cgraph->nodes[i]; @@ -11372,7 +11237,7 @@ GGML_CALL static enum ggml_status ggml_backend_cuda_graph_compute(ggml_backend_t } #endif - bool ok = ggml_cuda_compute_forward(¶ms, node); + bool ok = ggml_cuda_compute_forward(node); if (!ok) { fprintf(stderr, "%s: error: op not supported %s (%s)\n", __func__, node->name, ggml_op_name(node->op)); } @@ -11509,6 +11374,14 @@ GGML_CALL static bool ggml_backend_cuda_supports_op(ggml_backend_t backend, cons UNUSED(backend); } +GGML_CALL static bool ggml_backend_cuda_offload_op(ggml_backend_t backend, const ggml_tensor * op) { + const int min_batch_size = 32; + + return op->ne[1] >= min_batch_size && op->op != GGML_OP_GET_ROWS; + + UNUSED(backend); +} + static ggml_backend_event_t ggml_backend_cuda_event_new(ggml_backend_t backend) { ggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *)backend->context; @@ -11571,6 +11444,7 @@ static ggml_backend_i ggml_backend_cuda_interface = { /* .graph_plan_compute = */ NULL, /* .graph_compute = */ ggml_backend_cuda_graph_compute, /* .supports_op = */ ggml_backend_cuda_supports_op, + /* .offload_op = */ ggml_backend_cuda_offload_op, /* .event_new = */ ggml_backend_cuda_event_new, /* .event_free = */ ggml_backend_cuda_event_free, /* .event_record = */ ggml_backend_cuda_event_record, @@ -11584,7 +11458,7 @@ static ggml_guid_t ggml_backend_cuda_guid() { } GGML_CALL ggml_backend_t ggml_backend_cuda_init(int device) { - ggml_init_cublas(); // TODO: remove from ggml.c + ggml_init_cublas(); if (device < 0 || device >= ggml_cuda_get_device_count()) { fprintf(stderr, "%s: error: invalid device %d\n", __func__, device); @@ -11627,6 +11501,31 @@ GGML_CALL void ggml_backend_cuda_get_device_memory(int device, size_t * free, si CUDA_CHECK(cudaMemGetInfo(free, total)); } +GGML_CALL bool ggml_backend_cuda_register_host_buffer(void * buffer, size_t size) { + if (getenv("GGML_CUDA_NO_PINNED") != nullptr) { + return false; + } + + cudaError_t err = cudaHostRegister(buffer, size, cudaHostRegisterPortable | cudaHostRegisterReadOnly); + if (err != cudaSuccess) { + // clear the error + cudaGetLastError(); + + fprintf(stderr, "%s: warning: failed to register %.2f MiB of pinned memory: %s\n", __func__, + size/1024.0/1024.0, cudaGetErrorString(err)); + return false; + } + return true; +} + +GGML_CALL void ggml_backend_cuda_unregister_host_buffer(void * buffer) { + cudaError_t err = cudaHostUnregister(buffer); + if (err != cudaSuccess) { + // clear the error + cudaGetLastError(); + } +} + // backend registry GGML_CALL static ggml_backend_t ggml_backend_reg_cuda_init(const char * params, void * user_data) { ggml_backend_t cuda_backend = ggml_backend_cuda_init((int) (intptr_t) user_data); diff --git a/ggml-cuda.h b/ggml-cuda.h index b1ebd61d7..5eb4af40f 100644 --- a/ggml-cuda.h +++ b/ggml-cuda.h @@ -17,29 +17,17 @@ extern "C" { #define GGML_CUDA_MAX_DEVICES 16 -// Always success. To check if CUDA is actually loaded, use `ggml_cublas_loaded`. -GGML_API GGML_CALL void ggml_init_cublas(void); - -// Returns `true` if there are available CUDA devices and cublas loads successfully; otherwise, it returns `false`. -GGML_API GGML_CALL bool ggml_cublas_loaded(void); - -GGML_API GGML_CALL void * ggml_cuda_host_malloc(size_t size); -GGML_API GGML_CALL void ggml_cuda_host_free(void * ptr); - -GGML_API GGML_CALL bool ggml_cuda_can_mul_mat(const struct ggml_tensor * src0, const struct ggml_tensor * src1, struct ggml_tensor * dst); -GGML_API GGML_CALL bool ggml_cuda_compute_forward(struct ggml_compute_params * params, struct ggml_tensor * tensor); - -GGML_API GGML_CALL int ggml_cuda_get_device_count(void); -GGML_API GGML_CALL void ggml_cuda_get_device_description(int device, char * description, size_t description_size); - // backend API GGML_API GGML_CALL ggml_backend_t ggml_backend_cuda_init(int device); GGML_API GGML_CALL bool ggml_backend_is_cuda(ggml_backend_t backend); +// device buffer GGML_API GGML_CALL ggml_backend_buffer_type_t ggml_backend_cuda_buffer_type(int device); + // split tensor buffer that splits matrices by rows across multiple devices GGML_API GGML_CALL ggml_backend_buffer_type_t ggml_backend_cuda_split_buffer_type(const float * tensor_split); + // pinned host buffer for use with the CPU backend for faster copies between CPU and GPU GGML_API GGML_CALL ggml_backend_buffer_type_t ggml_backend_cuda_host_buffer_type(void); @@ -47,6 +35,9 @@ GGML_API GGML_CALL int ggml_backend_cuda_get_device_count(void); GGML_API GGML_CALL void ggml_backend_cuda_get_device_description(int device, char * description, size_t description_size); GGML_API GGML_CALL void ggml_backend_cuda_get_device_memory(int device, size_t * free, size_t * total); +GGML_API GGML_CALL bool ggml_backend_cuda_register_host_buffer(void * buffer, size_t size); +GGML_API GGML_CALL void ggml_backend_cuda_unregister_host_buffer(void * buffer); + #ifdef __cplusplus } #endif diff --git a/ggml-kompute.cpp b/ggml-kompute.cpp index 4caf2c9e7..81dd50678 100644 --- a/ggml-kompute.cpp +++ b/ggml-kompute.cpp @@ -1951,6 +1951,7 @@ static struct ggml_backend_i kompute_backend_i = { /* .graph_plan_compute = */ NULL, /* .graph_compute = */ ggml_backend_kompute_graph_compute, /* .supports_op = */ ggml_backend_kompute_supports_op, + /* .offload_op = */ NULL, /* .event_new = */ NULL, /* .event_free = */ NULL, /* .event_record = */ NULL, diff --git a/ggml-metal.m b/ggml-metal.m index c3451a79b..109e5fe6b 100644 --- a/ggml-metal.m +++ b/ggml-metal.m @@ -2837,6 +2837,7 @@ static struct ggml_backend_i ggml_backend_metal_i = { /* .graph_plan_compute = */ NULL, /* .graph_compute = */ ggml_backend_metal_graph_compute, /* .supports_op = */ ggml_backend_metal_supports_op, + /* .offload_op = */ NULL, /* .event_new = */ NULL, /* .event_free = */ NULL, /* .event_record = */ NULL, diff --git a/ggml-sycl.cpp b/ggml-sycl.cpp index 6dc5eb20c..d51f23b41 100644 --- a/ggml-sycl.cpp +++ b/ggml-sycl.cpp @@ -17390,6 +17390,7 @@ static ggml_backend_i ggml_backend_sycl_interface = { /* .graph_plan_compute = */ NULL, /* .graph_compute = */ ggml_backend_sycl_graph_compute, /* .supports_op = */ ggml_backend_sycl_supports_op, + /* .offload_op = */ NULL, /* .event_new = */ NULL, /* .event_free = */ NULL, /* .event_record = */ NULL, diff --git a/ggml-vulkan.cpp b/ggml-vulkan.cpp index 698b31496..cbceaa19f 100644 --- a/ggml-vulkan.cpp +++ b/ggml-vulkan.cpp @@ -5699,6 +5699,7 @@ static ggml_backend_i ggml_backend_vk_interface = { /* .graph_plan_compute = */ NULL, /* .graph_compute = */ ggml_backend_vk_graph_compute, /* .supports_op = */ ggml_backend_vk_supports_op, + /* .offload_op = */ NULL, /* .event_new = */ NULL, /* .event_free = */ NULL, /* .event_record = */ NULL, diff --git a/ggml.c b/ggml.c index fa23cb3c4..1d5854960 100644 --- a/ggml.c +++ b/ggml.c @@ -282,8 +282,6 @@ inline static void * ggml_calloc(size_t num, size_t size) { #else #include #endif -#elif defined(GGML_USE_CUBLAS) -#include "ggml-cuda.h" #elif defined(GGML_USE_CLBLAST) #include "ggml-opencl.h" #elif defined(GGML_USE_VULKAN) @@ -2640,9 +2638,7 @@ struct ggml_context * ggml_init(struct ggml_init_params params) { GGML_PRINT_DEBUG("%s: g_state initialized in %f ms\n", __func__, (t_end - t_start)/1000.0f); } -#if defined(GGML_USE_CUBLAS) - ggml_init_cublas(); -#elif defined(GGML_USE_CLBLAST) +#if defined(GGML_USE_CLBLAST) ggml_cl_init(); #elif defined(GGML_USE_VULKAN) ggml_vk_init_cpu_assist(); @@ -11105,7 +11101,6 @@ static void ggml_compute_forward_out_prod_f32( // nb01 >= nb00 - src0 is not transposed // compute by src0 rows - // TODO: #if defined(GGML_USE_CUBLAS) ggml_cuda_out_prod // TODO: #if defined(GGML_USE_CLBLAST) #if defined(GGML_USE_ACCELERATE) || defined(GGML_USE_OPENBLAS) @@ -11305,7 +11300,6 @@ static void ggml_compute_forward_out_prod_q_f32( // nb01 >= nb00 - src0 is not transposed // compute by src0 rows - // TODO: #if defined(GGML_USE_CUBLAS) ggml_cuda_out_prod // TODO: #if defined(GGML_USE_ACCELERATE) || defined(GGML_USE_OPENBLAS) || defined(GGML_USE_CLBLAST) if (params->type == GGML_TASK_TYPE_INIT) { @@ -16051,14 +16045,7 @@ static void ggml_compute_forward(struct ggml_compute_params * params, struct ggm return; } -#ifdef GGML_USE_CUBLAS - bool skip_cpu = ggml_cuda_compute_forward(params, tensor); - if (skip_cpu) { - return; - } - GGML_ASSERT(tensor->src[0] == NULL || tensor->src[0]->backend == GGML_BACKEND_TYPE_CPU); - GGML_ASSERT(tensor->src[1] == NULL || tensor->src[1]->backend == GGML_BACKEND_TYPE_CPU); -#elif defined(GGML_USE_VULKAN) +#if defined(GGML_USE_VULKAN) const bool skip_cpu = ggml_vk_compute_forward_cpu_assist(params, tensor); #ifdef GGML_VULKAN_CHECK_RESULTS if (skip_cpu) { @@ -16070,7 +16057,7 @@ static void ggml_compute_forward(struct ggml_compute_params * params, struct ggm } GGML_ASSERT(tensor->src[0] == NULL || tensor->src[0]->backend == GGML_BACKEND_TYPE_CPU); GGML_ASSERT(tensor->src[1] == NULL || tensor->src[1]->backend == GGML_BACKEND_TYPE_CPU); -#endif // GGML_USE_CUBLAS +#endif // GGML_USE_VULKAN #ifdef GGML_USE_SYCL bool skip_cpu = ggml_sycl_compute_forward(params, tensor); diff --git a/llama.cpp b/llama.cpp index e4db288dd..b8bef6daf 100644 --- a/llama.cpp +++ b/llama.cpp @@ -2040,6 +2040,11 @@ struct llama_model { ggml_free(ctx); } for (ggml_backend_buffer_t buf : bufs) { +#ifdef GGML_USE_CUBLAS + if (ggml_backend_buffer_get_type(buf) == ggml_backend_cpu_buffer_type()) { + ggml_backend_cuda_unregister_host_buffer(ggml_backend_buffer_get_base(buf)); + } +#endif ggml_backend_buffer_free(buf); } } @@ -5033,6 +5038,13 @@ static bool llm_load_tensors( size_t first, last; ml.get_mapping_range(&first, &last, ctx); buf = ggml_backend_cpu_buffer_from_ptr((char *) ml.mapping->addr + first, last - first); +#ifdef GGML_USE_CUBLAS + if (n_layer >= n_gpu_layers) { + ggml_backend_cuda_register_host_buffer( + ggml_backend_buffer_get_base(buf), + ggml_backend_buffer_get_size(buf)); + } +#endif } #ifdef GGML_USE_METAL else if (ml.use_mmap && buft == ggml_backend_metal_buffer_type()) { @@ -8231,7 +8243,6 @@ struct llm_build_context { cur = llm_build_kv(ctx0, model, hparams, kv_self, gf, model.layers[il].wo, model.layers[il].bo, Kcur, Vcur, Qcur, KQ_mask, nullptr, n_ctx, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il); - cb(cur, "kqv_out", il); } struct ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA); @@ -8601,12 +8612,15 @@ static struct ggml_cgraph * llama_build_graph( } // norm may be automatically assigned to the backend of the previous layer, increasing data transfer between backends - // to fix this, we assign the norm layer manually to the backend of its layer - if (il != -1 && strcmp(name, "norm") == 0) { - for (auto * backend : lctx.backends) { - if (ggml_backend_buft_supports_backend(lctx.model.buft_layer[il].buft, backend)) { - ggml_backend_sched_set_tensor_backend(lctx.sched, cur, backend); - break; + // FIXME: fix in ggml_backend_sched + const bool full_offload = lctx.model.n_gpu_layers > (int)lctx.model.hparams.n_layer; + if (batch.n_tokens < 32 || full_offload) { + if (il != -1 && strcmp(name, "norm") == 0) { + for (auto * backend : lctx.backends) { + if (ggml_backend_buft_supports_backend(lctx.model.buft_layer[il].buft, backend)) { + ggml_backend_sched_set_tensor_backend(lctx.sched, cur, backend); + break; + } } } } @@ -13107,27 +13121,25 @@ struct llama_context * llama_new_context_with_model( ctx->backends.push_back(ctx->backend_metal); } #elif defined(GGML_USE_CUBLAS) - if (model->n_gpu_layers > 0) { + if (model->split_mode == LLAMA_SPLIT_MODE_NONE || model->split_mode == LLAMA_SPLIT_MODE_ROW) { // with split_mode LLAMA_SPLIT_MODE_NONE or LLAMA_SPLIT_MODE_ROW, only the main GPU backend is used - if (model->split_mode == LLAMA_SPLIT_MODE_NONE || model->split_mode == LLAMA_SPLIT_MODE_ROW) { - ggml_backend_t backend = ggml_backend_cuda_init(model->main_gpu); + ggml_backend_t backend = ggml_backend_cuda_init(model->main_gpu); + if (backend == nullptr) { + LLAMA_LOG_ERROR("%s: failed to initialize CUDA%d backend\n", __func__, model->main_gpu); + llama_free(ctx); + return nullptr; + } + ctx->backends.push_back(backend); + } else { + // LLAMA_SPLIT_MODE_LAYER requires a backend for each GPU + for (int device = 0; device < ggml_backend_cuda_get_device_count(); ++device) { + ggml_backend_t backend = ggml_backend_cuda_init(device); if (backend == nullptr) { - LLAMA_LOG_ERROR("%s: failed to initialize CUDA%d backend\n", __func__, model->main_gpu); + LLAMA_LOG_ERROR("%s: failed to initialize CUDA%d backend\n", __func__, device); llama_free(ctx); return nullptr; } ctx->backends.push_back(backend); - } else { - // LLAMA_SPLIT_MODE_LAYER requires a backend for each GPU - for (int device = 0; device < ggml_backend_cuda_get_device_count(); ++device) { - ggml_backend_t backend = ggml_backend_cuda_init(device); - if (backend == nullptr) { - LLAMA_LOG_ERROR("%s: failed to initialize CUDA%d backend\n", __func__, device); - llama_free(ctx); - return nullptr; - } - ctx->backends.push_back(backend); - } } } #elif defined(GGML_USE_VULKAN) @@ -13285,14 +13297,17 @@ struct llama_context * llama_new_context_with_model( ggml_backend_t backend = ctx->backends[i]; ggml_backend_buffer_type_t buft = backend_buft[i]; size_t size = ggml_backend_sched_get_buffer_size(ctx->sched, backend); - LLAMA_LOG_INFO("%s: %10s compute buffer size = %8.2f MiB\n", __func__, - ggml_backend_buft_name(buft), - size / 1024.0 / 1024.0); + if (size > 1) { + LLAMA_LOG_INFO("%s: %10s compute buffer size = %8.2f MiB\n", __func__, + ggml_backend_buft_name(buft), + size / 1024.0 / 1024.0); + } } // note: the number of splits during measure is higher than during inference due to the kv shift int n_splits = ggml_backend_sched_get_n_splits(ctx->sched); - LLAMA_LOG_INFO("%s: graph splits: %d\n", __func__, n_splits); + LLAMA_LOG_INFO("%s: graph nodes = %d\n", __func__, gf->n_nodes); + LLAMA_LOG_INFO("%s: graph splits = %d\n", __func__, n_splits); } } From 4f6d1337ca5a409dc74aca8c479b7c34408a69c0 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Mon, 18 Mar 2024 13:45:27 +0200 Subject: [PATCH 651/812] ci : temporary disable sanitizer builds (#6128) --- .github/workflows/build.yml | 68 ++++++++++++++++++------------------ .github/workflows/server.yml | 6 ++-- 2 files changed, 37 insertions(+), 37 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 945df42f8..992c34a03 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -98,40 +98,40 @@ jobs: cd build ctest -L main --verbose --timeout 900 - ubuntu-latest-cmake-sanitizer: - runs-on: ubuntu-latest - - continue-on-error: true - - strategy: - matrix: - sanitizer: [ADDRESS, THREAD, UNDEFINED] - build_type: [Debug, Release] - - steps: - - name: Clone - id: checkout - uses: actions/checkout@v3 - - - name: Dependencies - id: depends - run: | - sudo apt-get update - sudo apt-get install build-essential - - - name: Build - id: cmake_build - run: | - mkdir build - cd build - cmake .. -DLLAMA_FATAL_WARNINGS=ON -DLLAMA_SANITIZE_${{ matrix.sanitizer }}=ON -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} - cmake --build . --config ${{ matrix.build_type }} -j $(nproc) - - - name: Test - id: cmake_test - run: | - cd build - ctest -L main --verbose --timeout 900 +# ubuntu-latest-cmake-sanitizer: +# runs-on: ubuntu-latest +# +# continue-on-error: true +# +# strategy: +# matrix: +# sanitizer: [ADDRESS, THREAD, UNDEFINED] +# build_type: [Debug, Release] +# +# steps: +# - name: Clone +# id: checkout +# uses: actions/checkout@v3 +# +# - name: Dependencies +# id: depends +# run: | +# sudo apt-get update +# sudo apt-get install build-essential +# +# - name: Build +# id: cmake_build +# run: | +# mkdir build +# cd build +# cmake .. -DLLAMA_FATAL_WARNINGS=ON -DLLAMA_SANITIZE_${{ matrix.sanitizer }}=ON -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} +# cmake --build . --config ${{ matrix.build_type }} -j $(nproc) +# +# - name: Test +# id: cmake_test +# run: | +# cd build +# ctest -L main --verbose --timeout 900 ubuntu-latest-cmake-mpi: runs-on: ubuntu-latest diff --git a/.github/workflows/server.yml b/.github/workflows/server.yml index 4ea09115a..65ca7d9ca 100644 --- a/.github/workflows/server.yml +++ b/.github/workflows/server.yml @@ -24,13 +24,13 @@ jobs: strategy: matrix: - sanitizer: [ADDRESS, THREAD, UNDEFINED] + # TODO: temporary disabled due to linux kernel issues + #sanitizer: [ADDRESS, THREAD, UNDEFINED] + sanitizer: [UNDEFINED] build_type: [Debug] include: - build_type: Release sanitizer: "" - - build_type: Debug - sanitizer: THREAD disabled_on_pr: true fail-fast: false # While -DLLAMA_SANITIZE_THREAD=ON is broken From ac9ee6a4ad740bc1ee484ede43e9f92b5af244c1 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Mon, 18 Mar 2024 13:45:38 +0200 Subject: [PATCH 652/812] ci : disable stale issue messages (#6126) --- .github/workflows/close-issue.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/close-issue.yml b/.github/workflows/close-issue.yml index 2682f308c..eaffd074d 100644 --- a/.github/workflows/close-issue.yml +++ b/.github/workflows/close-issue.yml @@ -15,7 +15,6 @@ jobs: days-before-issue-stale: 30 days-before-issue-close: 14 stale-issue-label: "stale" - stale-issue-message: "This issue is stale because it has been open for 30 days with no activity." close-issue-message: "This issue was closed because it has been inactive for 14 days since being marked as stale." days-before-pr-stale: -1 days-before-pr-close: -1 From 5e1b7f94a03e0b3b8e4578625bbdadc7bbd2b93c Mon Sep 17 00:00:00 2001 From: slaren Date: Mon, 18 Mar 2024 16:33:44 +0100 Subject: [PATCH 653/812] backend : set max split inputs to GGML_MAX_SRC (#6137) --- ggml-backend.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ggml-backend.c b/ggml-backend.c index 9f0084df7..6026570ae 100644 --- a/ggml-backend.c +++ b/ggml-backend.c @@ -1015,7 +1015,7 @@ static bool ggml_is_view_op(enum ggml_op op) { #endif #ifndef GGML_SCHED_MAX_SPLIT_INPUTS -#define GGML_SCHED_MAX_SPLIT_INPUTS 4 +#define GGML_SCHED_MAX_SPLIT_INPUTS GGML_MAX_SRC #endif #ifndef GGML_SCHED_MAX_COPIES From 104f5e0fc156d48476258295457cafeec2a2af10 Mon Sep 17 00:00:00 2001 From: Felix Date: Mon, 18 Mar 2024 16:40:22 +0100 Subject: [PATCH 654/812] clip : fix memory leak (#6138) --- examples/llava/clip.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/examples/llava/clip.cpp b/examples/llava/clip.cpp index a0ed82d7e..690bca2eb 100644 --- a/examples/llava/clip.cpp +++ b/examples/llava/clip.cpp @@ -497,7 +497,6 @@ struct clip_ctx { // memory buffers to evaluate the model ggml_backend_buffer_t params_buffer = NULL; - ggml_backend_buffer_t compute_buffer = NULL; ggml_backend_t backend = NULL; ggml_gallocr_t compute_alloc = NULL; @@ -1676,6 +1675,9 @@ void clip_free(clip_ctx * ctx) { ggml_free(ctx->ctx_data); gguf_free(ctx->ctx_gguf); + ggml_backend_buffer_free(ctx->params_buffer); + ggml_backend_free(ctx->backend); + ggml_gallocr_free(ctx->compute_alloc); delete ctx; } From d199ca79f279e84ebe27caafe0aa59c461d88969 Mon Sep 17 00:00:00 2001 From: Jared Van Bortel Date: Mon, 18 Mar 2024 12:49:02 -0400 Subject: [PATCH 655/812] mpt : implement backwards compatiblity with duped output tensor (#6139) --- llama.cpp | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/llama.cpp b/llama.cpp index b8bef6daf..1a9fe0c4d 100644 --- a/llama.cpp +++ b/llama.cpp @@ -540,6 +540,7 @@ static const std::map> LLM_TENSOR_NA { { LLM_TENSOR_TOKEN_EMBD, "token_embd" }, { LLM_TENSOR_OUTPUT_NORM, "output_norm" }, + { LLM_TENSOR_OUTPUT, "output"}, { LLM_TENSOR_ATTN_NORM, "blk.%d.attn_norm" }, { LLM_TENSOR_FFN_NORM, "blk.%d.ffn_norm" }, { LLM_TENSOR_ATTN_QKV, "blk.%d.attn_qkv" }, @@ -4300,9 +4301,9 @@ static bool llm_load_tensors( { model.output_norm = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}); model.output_norm_b = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "bias"), {n_embd}); - if (gguf_find_tensor(ml.ctx_gguf, tn(LLM_TENSOR_OUTPUT, "weight").c_str()) >= 0) { - model.output = ml.create_tensor(ctx_output_split, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}); - } else { + + model.output = ml.create_tensor(ctx_output_split, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, false); + if (!model.output) { model.output = ml.create_tensor(ctx_output_split, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}); // needs to be on GPU ml.n_created--; // artificial tensor ml.size_data += ggml_nbytes(model.output); @@ -4507,10 +4508,12 @@ static bool llm_load_tensors( model.output_norm = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}); model.output_norm_b = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "bias"), {n_embd}, false); - // same as tok_embd, duplicated to allow offloading - model.output = ml.create_tensor(ctx_output_split, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}); - ml.n_created--; // artificial tensor - ml.size_data += ggml_nbytes(model.output); + model.output = ml.create_tensor(ctx_output_split, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, false); + if (!model.output) { + model.output = ml.create_tensor(ctx_output_split, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}); // needs to be on GPU + ml.n_created--; // artificial tensor + ml.size_data += ggml_nbytes(model.output); + } } for (int i = 0; i < n_layer; ++i) { From 2d15886bb092c3b780c676b5cc57ff3337af9c83 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Sun, 17 Mar 2024 06:37:44 +0000 Subject: [PATCH 656/812] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/9df3e30ce24fd28c7b3e2de0d986769db5d6225d' (2024-03-06) → 'github:NixOS/nixpkgs/d691274a972b3165335d261cc4671335f5c67de9' (2024-03-14) --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index f9865d5e4..80de76dbf 100644 --- a/flake.lock +++ b/flake.lock @@ -20,11 +20,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1709703039, - "narHash": "sha256-6hqgQ8OK6gsMu1VtcGKBxKQInRLHtzulDo9Z5jxHEFY=", + "lastModified": 1710451336, + "narHash": "sha256-pP86Pcfu3BrAvRO7R64x7hs+GaQrjFes+mEPowCfkxY=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "9df3e30ce24fd28c7b3e2de0d986769db5d6225d", + "rev": "d691274a972b3165335d261cc4671335f5c67de9", "type": "github" }, "original": { From 4c28b8252907561165827125d2d1a4bad6926ac6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?DAN=E2=84=A2?= Date: Tue, 19 Mar 2024 01:59:36 -0400 Subject: [PATCH 657/812] common : print usage on '-h' and '--help' (#6145) --- common/common.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/common/common.cpp b/common/common.cpp index 919182862..5f10718ec 100644 --- a/common/common.cpp +++ b/common/common.cpp @@ -1056,7 +1056,8 @@ static bool gpt_params_find_arg(int argc, char ** argv, gpt_params & params, int return true; } if (arg == "-h" || arg == "--help") { - return false; + gpt_print_usage(argc, argv, gpt_params()); + exit(0); } if (arg == "--version") { fprintf(stderr, "version: %d (%s)\n", LLAMA_BUILD_NUMBER, LLAMA_COMMIT); From 970a48060ab9a6cc67aa063870323781c2a7bd7d Mon Sep 17 00:00:00 2001 From: slaren Date: Tue, 19 Mar 2024 09:06:54 +0100 Subject: [PATCH 658/812] ci : exempt some labels from being tagged as stale (#6140) --- .github/workflows/close-issue.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/close-issue.yml b/.github/workflows/close-issue.yml index eaffd074d..a151c6780 100644 --- a/.github/workflows/close-issue.yml +++ b/.github/workflows/close-issue.yml @@ -12,6 +12,7 @@ jobs: steps: - uses: actions/stale@v5 with: + exempt-issue-labels: "refactor,help wanted,good first issue,research" days-before-issue-stale: 30 days-before-issue-close: 14 stale-issue-label: "stale" From b80cf3b2d1dee0ad325f7a794fecc66befce7336 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Tue, 19 Mar 2024 10:21:54 +0200 Subject: [PATCH 659/812] common : disable repeat penalties by default (#6127) --- common/sampling.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/common/sampling.h b/common/sampling.h index 48b2459d1..79a998be8 100644 --- a/common/sampling.h +++ b/common/sampling.h @@ -32,13 +32,13 @@ typedef struct llama_sampling_params { float dynatemp_range = 0.00f; // 0.0 = disabled float dynatemp_exponent = 1.00f; // controls how entropy maps to temperature in dynamic temperature sampler int32_t penalty_last_n = 64; // last n tokens to penalize (0 = disable penalty, -1 = context size) - float penalty_repeat = 1.10f; // 1.0 = disabled + float penalty_repeat = 1.00f; // 1.0 = disabled float penalty_freq = 0.00f; // 0.0 = disabled float penalty_present = 0.00f; // 0.0 = disabled int32_t mirostat = 0; // 0 = disabled, 1 = mirostat, 2 = mirostat 2.0 float mirostat_tau = 5.00f; // target entropy float mirostat_eta = 0.10f; // learning rate - bool penalize_nl = true; // consider newlines as a repeatable token + bool penalize_nl = false; // consider newlines as a repeatable token std::vector samplers_sequence = { llama_sampler_type::TOP_K, From d0d5de42e5a65865b5fddb6f5c785083539b74c3 Mon Sep 17 00:00:00 2001 From: Pierrick Hymbert Date: Tue, 19 Mar 2024 12:05:44 +0100 Subject: [PATCH 660/812] gguf-split: split and merge gguf per batch of tensors (#6135) * gguf-split: split and merge gguf files per tensor * gguf-split: build with make toolchain * gguf-split: rename `--split-tensors-size` to `--split-max-tensors`. Set general.split_count KV to all split * split : minor style + fix compile warnings * gguf-split: remove --upload not implemented --------- Co-authored-by: Georgi Gerganov --- Makefile | 4 + examples/CMakeLists.txt | 1 + examples/gguf-split/CMakeLists.txt | 5 + examples/gguf-split/README.md | 9 + examples/gguf-split/gguf-split.cpp | 491 +++++++++++++++++++++++++++++ 5 files changed, 510 insertions(+) create mode 100644 examples/gguf-split/CMakeLists.txt create mode 100644 examples/gguf-split/README.md create mode 100644 examples/gguf-split/gguf-split.cpp diff --git a/Makefile b/Makefile index 838daf5c0..1daad45ed 100644 --- a/Makefile +++ b/Makefile @@ -753,6 +753,10 @@ gguf: examples/gguf/gguf.cpp ggml.o $(OBJS) $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) +gguf-split: examples/gguf-split/gguf-split.cpp ggml.o llama.o $(COMMON_DEPS) $(OBJS) + $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) + $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) + train-text-from-scratch: examples/train-text-from-scratch/train-text-from-scratch.cpp ggml.o llama.o $(COMMON_DEPS) train.o $(OBJS) $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index e762cf8b9..b59cc65bf 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -21,6 +21,7 @@ else() add_subdirectory(embedding) add_subdirectory(finetune) add_subdirectory(gritlm) + add_subdirectory(gguf-split) add_subdirectory(infill) add_subdirectory(llama-bench) add_subdirectory(llava) diff --git a/examples/gguf-split/CMakeLists.txt b/examples/gguf-split/CMakeLists.txt new file mode 100644 index 000000000..828e62435 --- /dev/null +++ b/examples/gguf-split/CMakeLists.txt @@ -0,0 +1,5 @@ +set(TARGET gguf-split) +add_executable(${TARGET} gguf-split.cpp) +install(TARGETS ${TARGET} RUNTIME) +target_link_libraries(${TARGET} PRIVATE common llama ${CMAKE_THREAD_LIBS_INIT}) +target_compile_features(${TARGET} PRIVATE cxx_std_11) diff --git a/examples/gguf-split/README.md b/examples/gguf-split/README.md new file mode 100644 index 000000000..ddb1f7649 --- /dev/null +++ b/examples/gguf-split/README.md @@ -0,0 +1,9 @@ +## GGUF split Example + +CLI to split / merge GGUF files. + +**Command line options:** + +- `--split`: split GGUF to multiple GGUF, default operation. +- `--split-max-tensors`: maximum tensors in each split: default(128) +- `--merge`: merge multiple GGUF to a single GGUF. diff --git a/examples/gguf-split/gguf-split.cpp b/examples/gguf-split/gguf-split.cpp new file mode 100644 index 000000000..5d7040ab5 --- /dev/null +++ b/examples/gguf-split/gguf-split.cpp @@ -0,0 +1,491 @@ +#include "llama.h" +#include "ggml.h" +#include "common.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +enum split_operation : uint8_t { + SPLIT_OP_SPLIT, + SPLIT_OP_MERGE, +}; + +static const char * const LLM_KV_GENERAL_SPLIT_I_SPLIT = "general.split"; +static const char * const LLM_KV_GENERAL_SPLIT_N_SPLIT = "general.split_count"; + +static const int SPLIT_FILENAME_MAX = 256; + +static const char * const SPLIT_FILENAME_FORMAT = "%s-%05d-of-%05d.gguf"; + +struct split_params { + split_operation operation = SPLIT_OP_SPLIT; + int n_split_tensors = 128; + std::string input; + std::string output; +}; + +static void split_print_usage(const char * executable) { + const split_params default_params; + printf("\n"); + printf("usage: %s [options] GGUF_IN GGUF_OUT\n", executable); + printf("\n"); + printf("Apply a GGUF operation on IN to OUT."); + printf("\n"); + printf("options:\n"); + printf(" -h, --help show this help message and exit\n"); + printf(" --version show version and build info\n"); + printf(" --split split GGUF to multiple GGUF (default)\n"); + printf(" --split-max-tensors max tensors in each split: default(%d)\n", default_params.n_split_tensors); + printf(" --merge merge multiple GGUF to a single GGUF\n"); + printf("\n"); +} + +static bool split_params_parse_ex(int argc, const char ** argv, split_params & params) { + std::string arg; + const std::string arg_prefix = "--"; + bool invalid_param = false; + + int arg_idx = 1; + for (; arg_idx < argc && strncmp(argv[arg_idx], "--", 2) == 0; arg_idx++) { + arg = argv[arg_idx]; + if (arg.compare(0, arg_prefix.size(), arg_prefix) == 0) { + std::replace(arg.begin(), arg.end(), '_', '-'); + } + + bool arg_found = false; + if (arg == "-h" || arg == "--help") { + split_print_usage(argv[0]); + exit(0); + } + if (arg == "--version") { + fprintf(stderr, "version: %d (%s)\n", LLAMA_BUILD_NUMBER, LLAMA_COMMIT); + fprintf(stderr, "built with %s for %s\n", LLAMA_COMPILER, LLAMA_BUILD_TARGET); + exit(0); + } + + if (arg == "--merge") { + arg_found = true; + params.operation = SPLIT_OP_MERGE; + } + if (arg == "--split") { + arg_found = true; + params.operation = SPLIT_OP_SPLIT; + } + if (arg == "--split-max-tensors") { + if (++arg_idx >= argc) { + invalid_param = true; + break; + } + arg_found = true; + params.n_split_tensors = atoi(argv[arg_idx]); + } + + if (!arg_found) { + throw std::invalid_argument("error: unknown argument: " + arg); + } + } + + if (invalid_param) { + throw std::invalid_argument("error: invalid parameter for argument: " + arg); + } + + if (argc - arg_idx < 2) { + printf("%s: bad arguments\n", argv[0]); + split_print_usage(argv[0]); + return false; + } + + params.input = argv[arg_idx++]; + params.output = argv[arg_idx++]; + + return true; +} + +static bool split_params_parse(int argc, const char ** argv, split_params & params) { + bool result = true; + try { + if (!split_params_parse_ex(argc, argv, params)) { + split_print_usage(argv[0]); + exit(1); + } + } + catch (const std::invalid_argument & ex) { + fprintf(stderr, "%s\n", ex.what()); + split_print_usage(argv[0]); + exit(1); + } + return result; +} + +static void zeros(std::ofstream & file, size_t n) { + char zero = 0; + for (size_t i = 0; i < n; ++i) { + file.write(&zero, 1); + } +} + +static std::string split_file_name(const std::string & path, int i_split, int n_split) { + char f_split[SPLIT_FILENAME_MAX] = {0}; + snprintf(f_split, sizeof(f_split), SPLIT_FILENAME_FORMAT, path.c_str(), i_split + 1, n_split); + return std::string(f_split); +} + +struct split_strategy { + const split_params params; + std::ifstream & f_input; + struct gguf_context * ctx_gguf; + struct ggml_context * ctx_meta = NULL; + const int n_tensors; + + const int n_split; + int i_split = 0; + + int i_tensor = 0; + + std::vector read_data; + + struct gguf_context * ctx_out; + std::ofstream fout; + + split_strategy(const split_params & params, + std::ifstream & f_input, + struct gguf_context * ctx_gguf, + struct ggml_context * ctx_meta) : + params(params), + f_input(f_input), + ctx_gguf(ctx_gguf), + ctx_meta(ctx_meta), + n_tensors(gguf_get_n_tensors(ctx_gguf)), + n_split(std::ceil(1. * n_tensors / params.n_split_tensors)) { + } + + bool should_split() const { + return i_tensor < n_tensors && i_tensor % params.n_split_tensors == 0; + } + + void split_start() { + ctx_out = gguf_init_empty(); + + // Save all metadata in first split only + if (i_split == 0) { + gguf_set_kv(ctx_out, ctx_gguf); + } + gguf_set_val_u8(ctx_out, LLM_KV_GENERAL_SPLIT_I_SPLIT, i_split); + gguf_set_val_u8(ctx_out, LLM_KV_GENERAL_SPLIT_N_SPLIT, n_split); + + // populate the original tensors, so we get an initial metadata + for (int i = i_split * params.n_split_tensors; i < n_tensors && i < (i_split + 1) * params.n_split_tensors; ++i) { + struct ggml_tensor * meta = ggml_get_tensor(ctx_meta, gguf_get_tensor_name(ctx_gguf, i)); + gguf_add_tensor(ctx_out, meta); + } + + auto split_name = split_file_name(params.output, i_split, n_split); + + fprintf(stderr, "%s: %s ...", __func__, split_name.c_str()); + fout = std::ofstream(split_name, std::ios::binary); + fout.exceptions(std::ofstream::failbit); // fail fast on write errors + + auto meta_size = gguf_get_meta_size(ctx_out); + + // placeholder for the meta data + ::zeros(fout, meta_size); + + i_split++; + } + + void next_tensor() { + const char * t_name = gguf_get_tensor_name(ctx_gguf, i_tensor); + struct ggml_tensor * t = ggml_get_tensor(ctx_meta, t_name); + auto n_bytes = ggml_nbytes(t); + + if (read_data.size() < n_bytes) { + read_data.resize(n_bytes); + } + + auto offset = gguf_get_data_offset(ctx_gguf) + gguf_get_tensor_offset(ctx_gguf, i_tensor); + f_input.seekg(offset); + f_input.read((char *)read_data.data(), n_bytes); + + t->data = read_data.data(); + + // write tensor data + padding + fout.write((const char *)t->data, n_bytes); + zeros(fout, GGML_PAD(n_bytes, GGUF_DEFAULT_ALIGNMENT) - n_bytes); + + i_tensor++; + } + + void split_end() { + // go back to beginning of file and write the updated metadata + fout.seekp(0); + std::vector data(gguf_get_meta_size(ctx_out)); + gguf_get_meta_data(ctx_out, data.data()); + fout.write((const char *)data.data(), data.size()); + + fout.close(); + gguf_free(ctx_out); + + fprintf(stderr, "\033[3Ddone\n"); + } +}; + +static void gguf_split(const split_params & split_params) { + struct ggml_context * ctx_meta = NULL; + + struct gguf_init_params params = { + /*.no_alloc = */ true, + /*.ctx = */ &ctx_meta, + }; + + std::ifstream f_input(split_params.input.c_str(), std::ios::binary); + if (!f_input.is_open()) { + fprintf(stderr, "%s: failed to open input GGUF from %s\n", __func__, split_params.input.c_str()); + exit(1); + } + + auto * ctx_gguf = gguf_init_from_file(split_params.input.c_str(), params); + if (!ctx_gguf) { + fprintf(stderr, "%s: failed to load input GGUF from %s\n", __func__, split_params.input.c_str()); + exit(1); + } + + split_strategy strategy(split_params, f_input, ctx_gguf, ctx_meta); + fprintf(stderr, "%s: %s -> %s (%d tensors per file)\n", + __func__, split_params.input.c_str(), + split_file_name(split_params.output, strategy.i_split, strategy.n_split).c_str(), + split_params.n_split_tensors); + + strategy.split_start(); + + while (strategy.i_tensor < strategy.n_tensors) { + strategy.next_tensor(); + if (strategy.should_split()) { + strategy.split_end(); + strategy.split_start(); + } + } + strategy.split_end(); + + gguf_free(ctx_gguf); + f_input.close(); + + fprintf(stderr, "%s: %d gguf split written with a total of %d tensors.\n", + __func__, strategy.n_split, strategy.n_tensors); +} + +static void gguf_merge(const split_params & split_params) { + fprintf(stderr, "%s: %s -> %s\n", + __func__, split_params.input.c_str(), + split_params.output.c_str()); + int n_split = 1; + int total_tensors = 0; + + auto * ctx_out = gguf_init_empty(); + std::ofstream fout(split_params.output.c_str(), std::ios::binary); + fout.exceptions(std::ofstream::failbit); // fail fast on write errors + + std::vector read_data; + std::vector ctx_metas; + std::vector ctx_ggufs; + + std::string split_prefix; + + // First pass to find KV and tensors metadata + for (int i_split = 0; i_split < n_split; i_split++) { + struct ggml_context * ctx_meta = NULL; + + struct gguf_init_params params = { + /*.no_alloc = */ true, + /*.ctx = */ &ctx_meta, + }; + + auto split_name = split_params.input; + if (i_split > 0) { + split_name = split_file_name(split_prefix, i_split, n_split); + } + fprintf(stderr, "%s: reading metadata %s ...", __func__, split_name.c_str()); + + auto * ctx_gguf = gguf_init_from_file(split_name.c_str(), params); + if (!ctx_gguf) { + fprintf(stderr, "\n%s: failed to load input GGUF from %s\n", __func__, split_params.input.c_str()); + exit(1); + } + ctx_ggufs.push_back(ctx_gguf); + ctx_metas.push_back(ctx_meta); + + if (i_split == 0) { + auto key_n_split = gguf_find_key(ctx_gguf, LLM_KV_GENERAL_SPLIT_N_SPLIT); + if (key_n_split < 0) { + fprintf(stderr, + "\n%s: input file does not contain %s metadata\n", + __func__, + LLM_KV_GENERAL_SPLIT_N_SPLIT); + gguf_free(ctx_gguf); + gguf_free(ctx_out); + fout.close(); + exit(1); + } + + n_split = gguf_get_val_u8(ctx_gguf, key_n_split); + if (n_split < 1) { + fprintf(stderr, + "\n%s: input file does not contain a valid split count %d\n", + __func__, + n_split); + gguf_free(ctx_gguf); + gguf_free(ctx_out); + fout.close(); + exit(1); + } + + // Do not trigger merge if we try to merge again the output + gguf_set_val_u8(ctx_out, LLM_KV_GENERAL_SPLIT_N_SPLIT, 0); + + // Set metadata from the first split + gguf_set_kv(ctx_out, ctx_gguf); + } + + // Verify the file naming + { + int i_split_file = 0; + int n_split_file = 0; + const char * i_split_format = "-00000-of-00000.gguf"; + + if (split_name.size() < strlen(i_split_format)) { + fprintf(stderr, "\n%s: unexpected input file name: %s\n", __func__, split_params.input.c_str()); + for (auto * _ctx_gguf : ctx_ggufs) { + gguf_free(_ctx_gguf); + } + gguf_free(ctx_out); + fout.close(); + exit(1); + } + + split_prefix = split_name.substr(0, split_name.size() - strlen(i_split_format)); + + const char * split_name_c_str = split_name.c_str(); + int n_part = sscanf(&split_name_c_str[0] + split_prefix.size(), "-%d-of-%d", &i_split_file, &n_split_file); + + if (n_part != 2 || i_split_file - 1 != i_split || n_split_file != n_split) { + fprintf(stderr, "\n%s: unexpected input file name: %s" + " i_split=%d i_split_file=%d" + " n_split=%d n_split_file=%d\n", __func__, + split_params.input.c_str(), + i_split, i_split_file, + n_split, n_split_file); + for (auto * _ctx_gguf : ctx_ggufs) { + gguf_free(_ctx_gguf); + } + gguf_free(ctx_out); + fout.close(); + exit(1); + } + } + + auto n_tensors = gguf_get_n_tensors(ctx_gguf); + for (int i_tensor = 0; i_tensor < n_tensors; i_tensor++) { + const char * t_name = gguf_get_tensor_name(ctx_gguf, i_tensor); + struct ggml_tensor * t = ggml_get_tensor(ctx_meta, t_name); + gguf_add_tensor(ctx_out, t); + } + total_tensors += n_tensors; + + fprintf(stderr, "\033[3Ddone\n"); + } + + // placeholder for the meta data + { + auto meta_size = gguf_get_meta_size(ctx_out); + ::zeros(fout, meta_size); + } + + // Write tensors data + for (int i_split = 0; i_split < n_split; i_split++) { + auto split_name = split_file_name(split_prefix, i_split, n_split); + std::ifstream f_input(split_name.c_str(), std::ios::binary); + if (!f_input.is_open()) { + fprintf(stderr, "%s: failed to open input GGUF from %s\n", __func__, split_name.c_str()); + for (auto * _ctx_gguf : ctx_ggufs) { + gguf_free(_ctx_gguf); + } + gguf_free(ctx_out); + fout.close(); + exit(1); + } + fprintf(stderr, "%s: writing tensors %s ...", __func__, split_name.c_str()); + + auto * ctx_gguf = ctx_ggufs[i_split]; + auto * ctx_meta = ctx_metas[i_split]; + + auto n_tensors = gguf_get_n_tensors(ctx_gguf); + for (int i_tensor = 0; i_tensor < n_tensors; i_tensor++) { + const char * t_name = gguf_get_tensor_name(ctx_gguf, i_tensor); + struct ggml_tensor * t = ggml_get_tensor(ctx_meta, t_name); + + auto n_bytes = ggml_nbytes(t); + + if (read_data.size() < n_bytes) { + read_data.resize(n_bytes); + } + + auto offset = gguf_get_data_offset(ctx_gguf) + gguf_get_tensor_offset(ctx_gguf, i_tensor); + f_input.seekg(offset); + f_input.read((char *)read_data.data(), n_bytes); + + // write tensor data + padding + fout.write((const char *)read_data.data(), n_bytes); + zeros(fout, GGML_PAD(n_bytes, GGUF_DEFAULT_ALIGNMENT) - n_bytes); + } + + gguf_free(ctx_gguf); + ggml_free(ctx_meta); + f_input.close(); + fprintf(stderr, "\033[3Ddone\n"); + } + + { + // go back to beginning of file and write the updated metadata + fout.seekp(0); + std::vector data(gguf_get_meta_size(ctx_out)); + gguf_get_meta_data(ctx_out, data.data()); + fout.write((const char *)data.data(), data.size()); + + fout.close(); + gguf_free(ctx_out); + } + + fprintf(stderr, "%s: %s merged from %d split with %d tensors.\n", + __func__, split_params.output.c_str(), n_split, total_tensors); +} + +int main(int argc, const char ** argv) { + if (argc < 3) { + split_print_usage(argv[0]); + } + + split_params params; + split_params_parse(argc, argv, params); + + switch (params.operation) { + case SPLIT_OP_SPLIT: gguf_split(params); + break; + case SPLIT_OP_MERGE: gguf_merge(params); + break; + default:split_print_usage(argv[0]); + exit(1); + } + + return 0; +} From d8b009a9456bf5284376149f3deb09300a37701a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?DAN=E2=84=A2?= Date: Tue, 19 Mar 2024 12:16:09 -0400 Subject: [PATCH 661/812] Remove undeed header file. (#6158) --- examples/gguf-split/gguf-split.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/examples/gguf-split/gguf-split.cpp b/examples/gguf-split/gguf-split.cpp index 5d7040ab5..8e12e6493 100644 --- a/examples/gguf-split/gguf-split.cpp +++ b/examples/gguf-split/gguf-split.cpp @@ -13,9 +13,7 @@ #include #include -#include #include -#include enum split_operation : uint8_t { SPLIT_OP_SPLIT, From d26e8b669dbf1f5f5a0afe4d2d885e86cf566302 Mon Sep 17 00:00:00 2001 From: Abhilash Majumder <30946547+abhilash1910@users.noreply.github.com> Date: Wed, 20 Mar 2024 08:28:49 +0530 Subject: [PATCH 662/812] increase igpu cluster limit (#6159) --- ggml-sycl.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ggml-sycl.h b/ggml-sycl.h index c549a64a1..1c9d52115 100644 --- a/ggml-sycl.h +++ b/ggml-sycl.h @@ -13,7 +13,7 @@ extern "C" { #endif -#define GGML_SYCL_MAX_DEVICES 16 +#define GGML_SYCL_MAX_DEVICES 48 #define GGML_SYCL_NAME "SYCL" GGML_API void ggml_init_sycl(void); From 6c0b287748327741b113d7d6018b68c63039b1c5 Mon Sep 17 00:00:00 2001 From: Neo Zhang Jianyu Date: Wed, 20 Mar 2024 11:21:41 +0800 Subject: [PATCH 663/812] update readme sycl for new update (#6151) * update readme sycl for new update * Update README-sycl.md Co-authored-by: Abhilash Majumder <30946547+abhilash1910@users.noreply.github.com> * Update README-sycl.md Co-authored-by: Abhilash Majumder <30946547+abhilash1910@users.noreply.github.com> * Update README-sycl.md Co-authored-by: Abhilash Majumder <30946547+abhilash1910@users.noreply.github.com> * Update README-sycl.md Co-authored-by: Abhilash Majumder <30946547+abhilash1910@users.noreply.github.com> * Update README-sycl.md Co-authored-by: AidanBeltonS <87009434+AidanBeltonS@users.noreply.github.com> * Update README-sycl.md Co-authored-by: AidanBeltonS <87009434+AidanBeltonS@users.noreply.github.com> * update by review comments * update w64devkit link * update for verify device id part * Update README-sycl.md Co-authored-by: Meng, Hengyu --------- Co-authored-by: Abhilash Majumder <30946547+abhilash1910@users.noreply.github.com> Co-authored-by: AidanBeltonS <87009434+AidanBeltonS@users.noreply.github.com> Co-authored-by: Meng, Hengyu --- README-sycl.md | 129 ++++++++++++++++++++++--------- examples/sycl/win-run-llama2.bat | 2 - 2 files changed, 93 insertions(+), 38 deletions(-) diff --git a/README-sycl.md b/README-sycl.md index 9359a9490..32adfda47 100644 --- a/README-sycl.md +++ b/README-sycl.md @@ -29,6 +29,7 @@ For Intel CPU, recommend to use llama.cpp for X86 (Intel MKL building). ## News - 2024.3 + - New base line is ready: [tag b2437](https://github.com/ggerganov/llama.cpp/tree/b2437). - Support multiple cards: **--split-mode**: [none|layer]; not support [row], it's on developing. - Support to assign main GPU by **--main-gpu**, replace $GGML_SYCL_DEVICE. - Support detecting all GPUs with level-zero and same top **Max compute units**. @@ -81,7 +82,7 @@ For dGPU, please make sure the device memory is enough. For llama-2-7b.Q4_0, rec |-|-|-| |Ampere Series| Support| A100| -### oneMKL +### oneMKL for CUDA The current oneMKL release does not contain the oneMKL cuBlas backend. As a result for Nvidia GPU's oneMKL must be built from source. @@ -254,16 +255,16 @@ Run without parameter: Check the ID in startup log, like: ``` -found 4 SYCL devices: - Device 0: Intel(R) Arc(TM) A770 Graphics, compute capability 1.3, - max compute_units 512, max work group size 1024, max sub group size 32, global mem size 16225243136 - Device 1: Intel(R) FPGA Emulation Device, compute capability 1.2, - max compute_units 24, max work group size 67108864, max sub group size 64, global mem size 67065057280 - Device 2: 13th Gen Intel(R) Core(TM) i7-13700K, compute capability 3.0, - max compute_units 24, max work group size 8192, max sub group size 64, global mem size 67065057280 - Device 3: Intel(R) Arc(TM) A770 Graphics, compute capability 3.0, - max compute_units 512, max work group size 1024, max sub group size 32, global mem size 16225243136 - +found 6 SYCL devices: +| | | |Compute |Max compute|Max work|Max sub| | +|ID| Device Type| Name|capability|units |group |group |Global mem size| +|--|------------------|---------------------------------------------|----------|-----------|--------|-------|---------------| +| 0|[level_zero:gpu:0]| Intel(R) Arc(TM) A770 Graphics| 1.3| 512| 1024| 32| 16225243136| +| 1|[level_zero:gpu:1]| Intel(R) UHD Graphics 770| 1.3| 32| 512| 32| 53651849216| +| 2| [opencl:gpu:0]| Intel(R) Arc(TM) A770 Graphics| 3.0| 512| 1024| 32| 16225243136| +| 3| [opencl:gpu:1]| Intel(R) UHD Graphics 770| 3.0| 32| 512| 32| 53651849216| +| 4| [opencl:cpu:0]| 13th Gen Intel(R) Core(TM) i7-13700K| 3.0| 24| 8192| 64| 67064815616| +| 5| [opencl:acc:0]| Intel(R) FPGA Emulation Device| 1.2| 24|67108864| 64| 67064815616| ``` |Attribute|Note| @@ -271,12 +272,35 @@ found 4 SYCL devices: |compute capability 1.3|Level-zero running time, recommended | |compute capability 3.0|OpenCL running time, slower than level-zero in most cases| -4. Set device ID and execute llama.cpp +4. Device selection and execution of llama.cpp -Set device ID = 0 by **GGML_SYCL_DEVICE=0** +There are two device selection modes: + +- Single device: Use one device assigned by user. +- Multiple devices: Automatically choose the devices with the same biggest Max compute units. + +|Device selection|Parameter| +|-|-| +|Single device|--split-mode none --main-gpu DEVICE_ID | +|Multiple devices|--split-mode layer (default)| + +Examples: + +- Use device 0: ```sh -GGML_SYCL_DEVICE=0 ./build/bin/main -m models/llama-2-7b.Q4_0.gguf -p "Building a website can be done in 10 simple steps:" -n 400 -e -ngl 33 +ZES_ENABLE_SYSMAN=1 ./build/bin/main -m models/llama-2-7b.Q4_0.gguf -p "Building a website can be done in 10 simple steps:" -n 400 -e -ngl 33 -sm none -mg 0 +``` +or run by script: + +```sh +./examples/sycl/run_llama2.sh 0 +``` + +- Use multiple devices: + +```sh +ZES_ENABLE_SYSMAN=1 ./build/bin/main -m models/llama-2-7b.Q4_0.gguf -p "Building a website can be done in 10 simple steps:" -n 400 -e -ngl 33 -sm layer ``` or run by script: @@ -289,12 +313,18 @@ Note: - By default, mmap is used to read model file. In some cases, it leads to the hang issue. Recommend to use parameter **--no-mmap** to disable mmap() to skip this issue. -5. Check the device ID in output +5. Verify the device ID in output + +Verify to see if the selected GPU is shown in the output, like: -Like: ``` -Using device **0** (Intel(R) Arc(TM) A770 Graphics) as main device +detect 1 SYCL GPUs: [0] with top Max compute units:512 ``` +Or +``` +use 1 SYCL GPUs: [0] with Max compute units:512 +``` + ## Windows @@ -355,7 +385,7 @@ a. Download & install cmake for Windows: https://cmake.org/download/ b. Download & install mingw-w64 make for Windows provided by w64devkit -- Download the latest fortran version of [w64devkit](https://github.com/skeeto/w64devkit/releases). +- Download the 1.19.0 version of [w64devkit](https://github.com/skeeto/w64devkit/releases/download/v1.19.0/w64devkit-1.19.0.zip). - Extract `w64devkit` on your pc. @@ -430,15 +460,16 @@ build\bin\main.exe Check the ID in startup log, like: ``` -found 4 SYCL devices: - Device 0: Intel(R) Arc(TM) A770 Graphics, compute capability 1.3, - max compute_units 512, max work group size 1024, max sub group size 32, global mem size 16225243136 - Device 1: Intel(R) FPGA Emulation Device, compute capability 1.2, - max compute_units 24, max work group size 67108864, max sub group size 64, global mem size 67065057280 - Device 2: 13th Gen Intel(R) Core(TM) i7-13700K, compute capability 3.0, - max compute_units 24, max work group size 8192, max sub group size 64, global mem size 67065057280 - Device 3: Intel(R) Arc(TM) A770 Graphics, compute capability 3.0, - max compute_units 512, max work group size 1024, max sub group size 32, global mem size 16225243136 +found 6 SYCL devices: +| | | |Compute |Max compute|Max work|Max sub| | +|ID| Device Type| Name|capability|units |group |group |Global mem size| +|--|------------------|---------------------------------------------|----------|-----------|--------|-------|---------------| +| 0|[level_zero:gpu:0]| Intel(R) Arc(TM) A770 Graphics| 1.3| 512| 1024| 32| 16225243136| +| 1|[level_zero:gpu:1]| Intel(R) UHD Graphics 770| 1.3| 32| 512| 32| 53651849216| +| 2| [opencl:gpu:0]| Intel(R) Arc(TM) A770 Graphics| 3.0| 512| 1024| 32| 16225243136| +| 3| [opencl:gpu:1]| Intel(R) UHD Graphics 770| 3.0| 32| 512| 32| 53651849216| +| 4| [opencl:cpu:0]| 13th Gen Intel(R) Core(TM) i7-13700K| 3.0| 24| 8192| 64| 67064815616| +| 5| [opencl:acc:0]| Intel(R) FPGA Emulation Device| 1.2| 24|67108864| 64| 67064815616| ``` @@ -447,13 +478,31 @@ found 4 SYCL devices: |compute capability 1.3|Level-zero running time, recommended | |compute capability 3.0|OpenCL running time, slower than level-zero in most cases| -4. Set device ID and execute llama.cpp -Set device ID = 0 by **set GGML_SYCL_DEVICE=0** +4. Device selection and execution of llama.cpp + +There are two device selection modes: + +- Single device: Use one device assigned by user. +- Multiple devices: Automatically choose the devices with the same biggest Max compute units. + +|Device selection|Parameter| +|-|-| +|Single device|--split-mode none --main-gpu DEVICE_ID | +|Multiple devices|--split-mode layer (default)| + +Examples: + +- Use device 0: ``` -set GGML_SYCL_DEVICE=0 -build\bin\main.exe -m models\llama-2-7b.Q4_0.gguf -p "Building a website can be done in 10 simple steps:\nStep 1:" -n 400 -e -ngl 33 -s 0 +build\bin\main.exe -m models\llama-2-7b.Q4_0.gguf -p "Building a website can be done in 10 simple steps:\nStep 1:" -n 400 -e -ngl 33 -s 0 -sm none -mg 0 +``` + +- Use multiple devices: + +``` +build\bin\main.exe -m models\llama-2-7b.Q4_0.gguf -p "Building a website can be done in 10 simple steps:\nStep 1:" -n 400 -e -ngl 33 -s 0 -sm layer ``` or run by script: @@ -466,11 +515,17 @@ Note: - By default, mmap is used to read model file. In some cases, it leads to the hang issue. Recommend to use parameter **--no-mmap** to disable mmap() to skip this issue. -5. Check the device ID in output -Like: +5. Verify the device ID in output + +Verify to see if the selected GPU is shown in the output, like: + ``` -Using device **0** (Intel(R) Arc(TM) A770 Graphics) as main device +detect 1 SYCL GPUs: [0] with top Max compute units:512 +``` +Or +``` +use 1 SYCL GPUs: [0] with Max compute units:512 ``` ## Environment Variable @@ -489,7 +544,6 @@ Using device **0** (Intel(R) Arc(TM) A770 Graphics) as main device |Name|Value|Function| |-|-|-| -|GGML_SYCL_DEVICE|0 (default) or 1|Set the device id used. Check the device ids by default running output| |GGML_SYCL_DEBUG|0 (default) or 1|Enable log function by macro: GGML_SYCL_DEBUG| |ZES_ENABLE_SYSMAN| 0 (default) or 1|Support to get free memory of GPU by sycl::aspect::ext_intel_free_memory.
    Recommended to use when --split-mode = layer| @@ -507,6 +561,9 @@ Using device **0** (Intel(R) Arc(TM) A770 Graphics) as main device ## Q&A +Note: please add prefix **[SYCL]** in issue title, so that we will check it as soon as possible. + + - Error: `error while loading shared libraries: libsycl.so.7: cannot open shared object file: No such file or directory`. Miss to enable oneAPI running environment. @@ -538,4 +595,4 @@ Using device **0** (Intel(R) Arc(TM) A770 Graphics) as main device ## Todo -- Support multiple cards. +- Support row layer split for multiple card runs. diff --git a/examples/sycl/win-run-llama2.bat b/examples/sycl/win-run-llama2.bat index cf621c675..1d4d7d2cd 100644 --- a/examples/sycl/win-run-llama2.bat +++ b/examples/sycl/win-run-llama2.bat @@ -6,8 +6,6 @@ set INPUT2="Building a website can be done in 10 simple steps:\nStep 1:" @call "C:\Program Files (x86)\Intel\oneAPI\setvars.bat" intel64 --force -set GGML_SYCL_DEVICE=0 -rem set GGML_SYCL_DEBUG=1 .\build\bin\main.exe -m models\llama-2-7b.Q4_0.gguf -p %INPUT2% -n 400 -e -ngl 33 -s 0 From bd60d82d0cc8b6852ec535495a5042dbdf05de24 Mon Sep 17 00:00:00 2001 From: Jared Van Bortel Date: Wed, 20 Mar 2024 01:33:49 -0400 Subject: [PATCH 664/812] server tests : more pythonic process management; fix bare `except:` (#6146) * server tests : remove seemingly redundant newlines in print() * server tests : use built-in subprocess features, not os.kill and psutil * server tests : do not catch e.g. SystemExit; use print_exc * server tests: handle TimeoutExpired exception * server tests: fix connect on dual-stack systems * server: tests: add new tokens regex on windows generated following new repeat penalties default changed in (#6127) * server: tests: remove the hack on windows since now we get the good socket family * server: tests: add new tokens regex following new repeat penalties default changed in (#6127) * server: tests: add new tokens regex following new repeat penalties default changed in (#6127) --------- Co-authored-by: Pierrick HYMBERT --- examples/server/tests/features/environment.py | 74 ++++++------------- examples/server/tests/features/server.feature | 14 ++-- examples/server/tests/features/steps/steps.py | 33 +++++---- examples/server/tests/requirements.txt | 1 - 4 files changed, 46 insertions(+), 76 deletions(-) diff --git a/examples/server/tests/features/environment.py b/examples/server/tests/features/environment.py index 82104e920..e7845dc2f 100644 --- a/examples/server/tests/features/environment.py +++ b/examples/server/tests/features/environment.py @@ -5,15 +5,14 @@ import sys import time import traceback from contextlib import closing - -import psutil +from subprocess import TimeoutExpired def before_scenario(context, scenario): context.debug = 'DEBUG' in os.environ and os.environ['DEBUG'] == 'ON' if context.debug: - print("DEBUG=ON\n") - print(f"\x1b[33;42mStarting new scenario: {scenario.name}!\x1b[0m\n") + print("DEBUG=ON") + print(f"\x1b[33;42mStarting new scenario: {scenario.name}!\x1b[0m") port = 8080 if 'PORT' in os.environ: port = int(os.environ['PORT']) @@ -27,60 +26,40 @@ def after_scenario(context, scenario): return if scenario.status == "failed": if 'GITHUB_ACTIONS' in os.environ: - print(f"\x1b[33;101mSCENARIO FAILED: {scenario.name} server logs:\x1b[0m\n\n") + print(f"\x1b[33;101mSCENARIO FAILED: {scenario.name} server logs:\x1b[0m\n") if os.path.isfile('llama.log'): with closing(open('llama.log', 'r')) as f: for line in f: print(line) if not is_server_listening(context.server_fqdn, context.server_port): - print("\x1b[33;101mERROR: Server stopped listening\x1b[0m\n") + print("\x1b[33;101mERROR: Server stopped listening\x1b[0m") - if not pid_exists(context.server_process.pid): + if context.server_process.poll() is not None: assert False, f"Server not running pid={context.server_process.pid} ..." - server_graceful_shutdown(context) + server_graceful_shutdown(context) # SIGINT - # Wait few for socket to free up - time.sleep(0.05) + try: + context.server_process.wait(0.5) + except TimeoutExpired: + print(f"server still alive after 500ms, force-killing pid={context.server_process.pid} ...") + context.server_process.kill() # SIGKILL + context.server_process.wait() - attempts = 0 - while pid_exists(context.server_process.pid) or is_server_listening(context.server_fqdn, context.server_port): - server_kill(context) + while is_server_listening(context.server_fqdn, context.server_port): time.sleep(0.1) - attempts += 1 - if attempts > 5: - server_kill_hard(context) - except: - exc = sys.exception() - print("error in after scenario: \n") - print(exc) - print("*** print_tb: \n") - traceback.print_tb(exc.__traceback__, file=sys.stdout) + except Exception: + print("ignoring error in after_scenario:") + traceback.print_exc(file=sys.stdout) def server_graceful_shutdown(context): - print(f"shutting down server pid={context.server_process.pid} ...\n") + print(f"shutting down server pid={context.server_process.pid} ...") if os.name == 'nt': - os.kill(context.server_process.pid, signal.CTRL_C_EVENT) + interrupt = signal.CTRL_C_EVENT else: - os.kill(context.server_process.pid, signal.SIGINT) - - -def server_kill(context): - print(f"killing server pid={context.server_process.pid} ...\n") - context.server_process.kill() - - -def server_kill_hard(context): - pid = context.server_process.pid - path = context.server_path - - print(f"Server dangling exits, hard killing force {pid}={path}...\n") - try: - psutil.Process(pid).kill() - except psutil.NoSuchProcess: - return False - return True + interrupt = signal.SIGINT + context.server_process.send_signal(interrupt) def is_server_listening(server_fqdn, server_port): @@ -88,14 +67,5 @@ def is_server_listening(server_fqdn, server_port): result = sock.connect_ex((server_fqdn, server_port)) _is_server_listening = result == 0 if _is_server_listening: - print(f"server is listening on {server_fqdn}:{server_port}...\n") + print(f"server is listening on {server_fqdn}:{server_port}...") return _is_server_listening - - -def pid_exists(pid): - try: - psutil.Process(pid) - except psutil.NoSuchProcess: - return False - return True - diff --git a/examples/server/tests/features/server.feature b/examples/server/tests/features/server.feature index 7448986e7..45a988db6 100644 --- a/examples/server/tests/features/server.feature +++ b/examples/server/tests/features/server.feature @@ -35,9 +35,9 @@ Feature: llama.cpp server And metric llamacpp:tokens_predicted is Examples: Prompts - | prompt | n_predict | re_content | n_prompt | n_predicted | truncated | - | I believe the meaning of life is | 8 | (read\|going)+ | 18 | 8 | not | - | Write a joke about AI from a very long prompt which will not be truncated | 256 | (princesses\|everyone\|kids)+ | 46 | 64 | not | + | prompt | n_predict | re_content | n_prompt | n_predicted | truncated | + | I believe the meaning of life is | 8 | (read\|going)+ | 18 | 8 | not | + | Write a joke about AI from a very long prompt which will not be truncated | 256 | (princesses\|everyone\|kids\|Anna\|forest)+ | 46 | 64 | not | Scenario: Completion prompt truncated Given a prompt: @@ -48,7 +48,7 @@ Feature: llama.cpp server Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum. """ And a completion request with no api error - Then 64 tokens are predicted matching fun|Annaks|popcorns|pictry + Then 64 tokens are predicted matching fun|Annaks|popcorns|pictry|bowl And the completion is truncated And 109 prompt tokens are processed @@ -65,9 +65,9 @@ Feature: llama.cpp server And the completion is truncated Examples: Prompts - | model | system_prompt | user_prompt | max_tokens | re_content | n_prompt | n_predicted | enable_streaming | truncated | - | llama-2 | Book | What is the best book | 8 | (Here\|what)+ | 77 | 8 | disabled | not | - | codellama70b | You are a coding assistant. | Write the fibonacci function in c++. | 128 | (thanks\|happy\|bird)+ | -1 | 64 | enabled | | + | model | system_prompt | user_prompt | max_tokens | re_content | n_prompt | n_predicted | enable_streaming | truncated | + | llama-2 | Book | What is the best book | 8 | (Here\|what)+ | 77 | 8 | disabled | not | + | codellama70b | You are a coding assistant. | Write the fibonacci function in c++. | 128 | (thanks\|happy\|bird\|Annabyear)+ | -1 | 64 | enabled | | Scenario: Tokenize / Detokenize diff --git a/examples/server/tests/features/steps/steps.py b/examples/server/tests/features/steps/steps.py index 9e348d5fc..40c97001a 100644 --- a/examples/server/tests/features/steps/steps.py +++ b/examples/server/tests/features/steps/steps.py @@ -66,7 +66,7 @@ def step_server_config(context, server_fqdn, server_port): def step_download_hf_model(context, hf_file, hf_repo): context.model_file = hf_hub_download(repo_id=hf_repo, filename=hf_file) if context.debug: - print(f"model file: {context.model_file}\n") + print(f"model file: {context.model_file}") @step('a model file {model_file}') @@ -137,9 +137,12 @@ def step_start_server(context): if 'GITHUB_ACTIONS' in os.environ: max_attempts *= 2 + addrs = socket.getaddrinfo(context.server_fqdn, context.server_port, type=socket.SOCK_STREAM) + family, typ, proto, _, sockaddr = addrs[0] + while True: - with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as sock: - result = sock.connect_ex((context.server_fqdn, context.server_port)) + with closing(socket.socket(family, typ, proto)) as sock: + result = sock.connect_ex(sockaddr) if result == 0: print("\x1b[33;46mserver started!\x1b[0m") return @@ -209,7 +212,7 @@ async def step_request_completion(context, api_error): user_api_key=context.user_api_key) context.tasks_result.append(completion) if context.debug: - print(f"Completion response: {completion}\n") + print(f"Completion response: {completion}") if expect_api_error: assert completion == 401, f"completion must be an 401 status code: {completion}" @@ -354,7 +357,7 @@ def step_prompt_passkey(context, passkey, i_pos): prompt += context.prompt_junk_suffix if context.debug: passkey_highlight = "\x1b[33m" + passkey + "\x1b[0m" - print(f"Passkey challenge:\n```{prompt.replace(passkey, passkey_highlight)}```\n") + print(f"Passkey challenge:\n```{prompt.replace(passkey, passkey_highlight)}```") context.prompts.append(context.prompt_prefix + prompt + context.prompt_suffix) context.n_prompts = len(context.prompts) @@ -363,7 +366,7 @@ def step_prompt_passkey(context, passkey, i_pos): @async_run_until_complete async def step_oai_chat_completions(context, api_error): if context.debug: - print(f"Submitting OAI compatible completions request...\n") + print(f"Submitting OAI compatible completions request...") expect_api_error = api_error == 'raised' completion = await oai_chat_completions(context.prompts.pop(), context.system_prompt, @@ -508,12 +511,12 @@ async def step_all_embeddings_are_the_same(context): embedding1 = np.array(embeddings[i]) embedding2 = np.array(embeddings[j]) if context.debug: - print(f"embedding1: {embedding1[-8:]}\n") - print(f"embedding2: {embedding2[-8:]}\n") + print(f"embedding1: {embedding1[-8:]}") + print(f"embedding2: {embedding2[-8:]}") similarity = np.dot(embedding1, embedding2) / (np.linalg.norm(embedding1) * np.linalg.norm(embedding2)) msg = f"Similarity between {i} and {j}: {similarity:.10f}" if context.debug: - print(f"{msg}\n") + print(f"{msg}") assert np.isclose(similarity, 1.0, rtol=1e-05, atol=1e-08, equal_nan=False), msg @@ -630,7 +633,7 @@ async def step_prometheus_metrics_exported(context): metrics_raw = await metrics_response.text() metric_exported = False if context.debug: - print(f"/metrics answer:\n{metrics_raw}\n") + print(f"/metrics answer:\n{metrics_raw}") context.metrics = {} for metric in parser.text_string_to_metric_families(metrics_raw): match metric.name: @@ -932,7 +935,7 @@ def assert_n_tokens_predicted(completion_response, expected_predicted_n=None, re last_match = end highlighted += content[last_match:] if 'DEBUG' in os.environ and os.environ['DEBUG'] == 'ON': - print(f"Checking completion response: {highlighted}\n") + print(f"Checking completion response: {highlighted}") assert last_match > 0, f'/{re_content}/ must match ```{highlighted}```' if expected_predicted_n and expected_predicted_n > 0: assert n_predicted == expected_predicted_n, (f'invalid number of tokens predicted:' @@ -942,7 +945,7 @@ def assert_n_tokens_predicted(completion_response, expected_predicted_n=None, re async def gather_tasks_results(context): n_tasks = len(context.concurrent_tasks) if context.debug: - print(f"Waiting for all {n_tasks} tasks results...\n") + print(f"Waiting for all {n_tasks} tasks results...") for task_no in range(n_tasks): context.tasks_result.append(await context.concurrent_tasks.pop()) n_completions = len(context.tasks_result) @@ -959,7 +962,7 @@ async def wait_for_health_status(context, slots_processing=None, expected_slots=None): if context.debug: - print(f"Starting checking for health for expected_health_status={expected_health_status}\n") + print(f"Starting checking for health for expected_health_status={expected_health_status}") interval = 0.5 counter = 0 if 'GITHUB_ACTIONS' in os.environ: @@ -1048,8 +1051,6 @@ def start_server_background(context): if 'LLAMA_SERVER_BIN_PATH' in os.environ: context.server_path = os.environ['LLAMA_SERVER_BIN_PATH'] server_listen_addr = context.server_fqdn - if os.name == 'nt': - server_listen_addr = '0.0.0.0' server_args = [ '--host', server_listen_addr, '--port', context.server_port, @@ -1088,7 +1089,7 @@ def start_server_background(context): server_args.append('--verbose') if 'SERVER_LOG_FORMAT_JSON' not in os.environ: server_args.extend(['--log-format', "text"]) - print(f"starting server with: {context.server_path} {server_args}\n") + print(f"starting server with: {context.server_path} {server_args}") flags = 0 if 'nt' == os.name: flags |= subprocess.DETACHED_PROCESS diff --git a/examples/server/tests/requirements.txt b/examples/server/tests/requirements.txt index c2c960102..2e4f42ad2 100644 --- a/examples/server/tests/requirements.txt +++ b/examples/server/tests/requirements.txt @@ -3,5 +3,4 @@ behave~=1.2.6 huggingface_hub~=0.20.3 numpy~=1.24.4 openai~=0.25.0 -psutil~=5.9.8 prometheus-client~=0.20.0 From 47cc7a7bf9793f81a6dfe7c2096c499403f64dd6 Mon Sep 17 00:00:00 2001 From: Karthick Date: Wed, 20 Mar 2024 16:32:34 +0530 Subject: [PATCH 665/812] Server: Handle n_keep parameter in the request (#6174) --- examples/server/utils.hpp | 1 + 1 file changed, 1 insertion(+) diff --git a/examples/server/utils.hpp b/examples/server/utils.hpp index 2ddb2cd21..cc5fce691 100644 --- a/examples/server/utils.hpp +++ b/examples/server/utils.hpp @@ -371,6 +371,7 @@ static json oaicompat_completion_params_parse( llama_params["repeat_last_n"] = json_value(body, "repeat_last_n", default_sparams.penalty_last_n); llama_params["ignore_eos"] = json_value(body, "ignore_eos", false); llama_params["tfs_z"] = json_value(body, "tfs_z", default_sparams.tfs_z); + llama_params["n_keep"] = json_value(body, "n_keep", 0); if (body.count("grammar") != 0) { llama_params["grammar"] = json_value(body, "grammar", json::object()); From f8c4e745e1e728204ab26dbadf52853545e6789c Mon Sep 17 00:00:00 2001 From: Ziang Wu <97337387+ZiangWu-77@users.noreply.github.com> Date: Wed, 20 Mar 2024 19:20:37 +0800 Subject: [PATCH 666/812] llava : add a MobileVLM_V2-1.7B backup (#6152) * Add MobileVLM_V2 backup * Update MobileVLM-README.md * Update examples/llava/MobileVLM-README.md Co-authored-by: Georgi Gerganov * Update examples/llava/convert-image-encoder-to-gguf.py Co-authored-by: Georgi Gerganov * clip : fix whitespace --------- Co-authored-by: Georgi Gerganov --- examples/llava/MobileVLM-README.md | 14 ++++++- examples/llava/clip.cpp | 42 ++++++++++++++++++- .../llava/convert-image-encoder-to-gguf.py | 9 ++-- 3 files changed, 59 insertions(+), 6 deletions(-) diff --git a/examples/llava/MobileVLM-README.md b/examples/llava/MobileVLM-README.md index 9eba791da..c1f361d17 100644 --- a/examples/llava/MobileVLM-README.md +++ b/examples/llava/MobileVLM-README.md @@ -1,11 +1,13 @@ # MobileVLM -Currently this implementation supports [MobileVLM-v1.7](https://huggingface.co/mtgv/MobileVLM-1.7B) variants. +Currently this implementation supports [MobileVLM-1.7B](https://huggingface.co/mtgv/MobileVLM-1.7B) / [MobileVLM_V2-1.7B](https://huggingface.co/mtgv/MobileVLM_V2-1.7B) variants. for more information, please go to [Meituan-AutoML/MobileVLM](https://github.com/Meituan-AutoML/MobileVLM) The implementation is based on llava, and is compatible with llava and mobileVLM. The usage is basically same as llava. +Notice: The overall process of model inference for both **MobilVLM** and **MobilVLM_V2** models is the same, but the process of model conversion is a little different. Therefore, using MobiVLM as an example, the different conversion step will be shown. + ## Usage Build with cmake or run `make llava-cli` to build it. @@ -34,7 +36,7 @@ git clone https://huggingface.co/openai/clip-vit-large-patch14-336 python ./examples/llava/llava-surgery.py -m path/to/MobileVLM-1.7B ``` -3. Use `convert-image-encoder-to-gguf.py` with `--projector-type ldp` to convert the LLaVA image encoder to GGUF: +3. Use `convert-image-encoder-to-gguf.py` with `--projector-type ldp` (for **V2** the arg is `--projector-type ldpv2`) to convert the LLaVA image encoder to GGUF: ```sh python ./examples/llava/convert-image-encoder-to-gguf \ @@ -44,6 +46,14 @@ python ./examples/llava/convert-image-encoder-to-gguf \ --projector-type ldp ``` +```sh +python ./examples/llava/convert-image-encoder-to-gguf \ + -m path/to/clip-vit-large-patch14-336 \ + --llava-projector path/to/MobileVLM-1.7B_V2/llava.projector \ + --output-dir path/to/MobileVLM-1.7B_V2 \ + --projector-type ldpv2 +``` + 4. Use `convert.py` to convert the LLaMA part of LLaVA to GGUF: ```sh diff --git a/examples/llava/clip.cpp b/examples/llava/clip.cpp index 690bca2eb..6d7b4950f 100644 --- a/examples/llava/clip.cpp +++ b/examples/llava/clip.cpp @@ -119,6 +119,7 @@ static std::string format(const char * fmt, ...) { #define TN_LLAVA_PROJ "mm.%d.%s" #define TN_MVLM_PROJ_MLP "mm.model.mlp.%d.%s" #define TN_MVLM_PROJ_BLOCK "mm.model.mb_block.%d.block.%d.%s" +#define TN_MVLM_PROJ_PEG "mm.model.peg.%d.%s" #define TN_IMAGE_NEWLINE "model.image_newline" @@ -126,12 +127,14 @@ enum projector_type { PROJECTOR_TYPE_MLP, PROJECTOR_TYPE_MLP_NORM, PROJECTOR_TYPE_LDP, + PROJECTOR_TYPE_LDPV2, PROJECTOR_TYPE_UNKNOWN, }; static std::map PROJECTOR_TYPE_NAMES = { { PROJECTOR_TYPE_MLP, "mlp" }, { PROJECTOR_TYPE_LDP, "ldp" }, + { PROJECTOR_TYPE_LDPV2, "ldpv2"}, }; @@ -807,6 +810,29 @@ static ggml_cgraph * clip_image_build_graph(clip_ctx * ctx, const clip_image_f32 } embeddings = block_1; } + else if (ctx->proj_type == PROJECTOR_TYPE_LDPV2) + { + int n_patch = 24; + struct ggml_tensor * mlp_0 = ggml_mul_mat(ctx0, model.mm_model_mlp_0_w, embeddings); + mlp_0 = ggml_add(ctx0, mlp_0, model.mm_model_mlp_0_b); + mlp_0 = ggml_gelu(ctx0, mlp_0); + struct ggml_tensor * mlp_2 = ggml_mul_mat(ctx0, model.mm_model_mlp_2_w, mlp_0); + mlp_2 = ggml_add(ctx0, mlp_2, model.mm_model_mlp_2_b); + // mlp_2 ne = [2048, 576, 1, 1] + // // AVG Pool Layer 2*2, strides = 2 + mlp_2 = ggml_cont(ctx0, ggml_permute(ctx0, mlp_2, 1, 0, 2, 3)); + // mlp_2 ne = [576, 2048, 1, 1] + mlp_2 = ggml_reshape_4d(ctx0, mlp_2, n_patch, n_patch, mlp_2->ne[1], mlp_2->ne[2]); + // mlp_2 ne [24, 24, 2048, 1] + mlp_2 = ggml_pool_2d(ctx0, mlp_2, GGML_OP_POOL_AVG, 2, 2, 2, 2, 0, 0); + // weight ne = [3, 3, 2048, 1] + struct ggml_tensor * peg_0 = ggml_conv_depthwise_2d(ctx0, model.mm_model_peg_0_w, mlp_2, 1, 1, 1, 1, 1, 1); + peg_0 = ggml_add(ctx0, peg_0, mlp_2); + peg_0 = ggml_cont(ctx0, ggml_permute(ctx0, peg_0, 1, 2, 0, 3)); + peg_0 = ggml_add(ctx0, peg_0, model.mm_model_peg_0_b); + peg_0 = ggml_reshape_3d(ctx0, peg_0, peg_0->ne[0], peg_0->ne[1] * peg_0->ne[2], peg_0->ne[3]); + embeddings = peg_0; + } else { GGML_ASSERT(false); } @@ -1177,7 +1203,18 @@ struct clip_ctx * clip_model_load(const char * fname, const int verbosity = 1) { vision_model.mm_model_block_2_block_2_0_w = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_BLOCK, 2, 2, "0.weight")); vision_model.mm_model_block_2_block_2_1_w = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_BLOCK, 2, 2, "1.weight")); vision_model.mm_model_block_2_block_2_1_b = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_BLOCK, 2, 2, "1.bias")); - } else { + } + else if (new_clip->proj_type == PROJECTOR_TYPE_LDPV2) + { + // MobilVLM_V2 projection + vision_model.mm_model_mlp_0_w = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_MLP, 0, "weight")); + vision_model.mm_model_mlp_0_b = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_MLP, 0, "bias")); + vision_model.mm_model_mlp_2_w = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_MLP, 2, "weight")); + vision_model.mm_model_mlp_2_b = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_MLP, 2, "bias")); + vision_model.mm_model_peg_0_w = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_PEG, 0, "weight")); + vision_model.mm_model_peg_0_b = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_PEG, 0, "bias")); + } + else { std::string proj_type = PROJECTOR_TYPE_NAMES[new_clip->proj_type]; throw std::runtime_error(format("%s: don't support projector with: %s currently\n", __func__, proj_type.c_str())); } @@ -1966,6 +2003,9 @@ int clip_n_mmproj_embd(const struct clip_ctx * ctx) { if (ctx->proj_type == PROJECTOR_TYPE_LDP) { return ctx->vision_model.mm_model_block_1_block_2_1_b->ne[0]; } + if (ctx->proj_type == PROJECTOR_TYPE_LDPV2) { + return ctx->vision_model.mm_model_peg_0_b->ne[0]; + } if (ctx->proj_type == PROJECTOR_TYPE_MLP) { return ctx->vision_model.mm_2_b->ne[0]; } diff --git a/examples/llava/convert-image-encoder-to-gguf.py b/examples/llava/convert-image-encoder-to-gguf.py index c69f89ac2..b00bf7c6d 100644 --- a/examples/llava/convert-image-encoder-to-gguf.py +++ b/examples/llava/convert-image-encoder-to-gguf.py @@ -1,6 +1,7 @@ import argparse import os import json +import re import torch import numpy as np @@ -38,9 +39,11 @@ def should_skip_tensor(name: str, has_text: bool, has_vision: bool, has_llava: b def get_tensor_name(name: str) -> str: if "projection" in name: return name - if "mm_projector" in name: - return name.replace("model.mm_projector", "mm") + name = name.replace("model.mm_projector", "mm") + name = re.sub(r'mm\.mlp\.mlp', 'mm.model.mlp', name, count=1) + name = re.sub(r'mm\.peg\.peg', 'mm.model.peg', name, count=1) + return name return name.replace("text_model", "t").replace("vision_model", "v").replace("encoder.layers", "blk").replace("embeddings.", "").replace("_proj", "").replace("self_attn.", "attn_").replace("layer_norm", "ln").replace("layernorm", "ln").replace("mlp.fc1", "ffn_down").replace("mlp.fc2", "ffn_up").replace("embedding", "embd").replace("final", "post").replace("layrnorm", "ln") @@ -83,7 +86,7 @@ ap.add_argument("--clip-model-is-vision", action="store_true", required=False, ap.add_argument("--clip-model-is-openclip", action="store_true", required=False, help="The clip model is from openclip (for ViT-SO400M type))") ap.add_argument("--llava-projector", help="Path to llava.projector file. If specified, save an image encoder for LLaVA models.") -ap.add_argument("--projector-type", help="Type of projector. Possible values: mlp, ldp", choices=["mlp", "ldp"], default="mlp") +ap.add_argument("--projector-type", help="Type of projector. Possible values: mlp, ldp, ldpv2", choices=["mlp", "ldp", "ldpv2"], default="mlp") ap.add_argument("-o", "--output-dir", help="Directory to save GGUF files. Default is the original model directory", default=None) # Example --image_mean 0.48145466 0.4578275 0.40821073 --image_std 0.26862954 0.26130258 0.27577711 # Example --image_mean 0.5 0.5 0.5 --image_std 0.5 0.5 0.5 From d795988d9e09f75412efe2134512914c42780ad5 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Wed, 20 Mar 2024 13:29:49 +0200 Subject: [PATCH 667/812] Revert "llava : add a MobileVLM_V2-1.7B backup (#6152)" This reverts commit f8c4e745e1e728204ab26dbadf52853545e6789c. --- examples/llava/MobileVLM-README.md | 14 +------ examples/llava/clip.cpp | 42 +------------------ .../llava/convert-image-encoder-to-gguf.py | 9 ++-- 3 files changed, 6 insertions(+), 59 deletions(-) diff --git a/examples/llava/MobileVLM-README.md b/examples/llava/MobileVLM-README.md index c1f361d17..9eba791da 100644 --- a/examples/llava/MobileVLM-README.md +++ b/examples/llava/MobileVLM-README.md @@ -1,13 +1,11 @@ # MobileVLM -Currently this implementation supports [MobileVLM-1.7B](https://huggingface.co/mtgv/MobileVLM-1.7B) / [MobileVLM_V2-1.7B](https://huggingface.co/mtgv/MobileVLM_V2-1.7B) variants. +Currently this implementation supports [MobileVLM-v1.7](https://huggingface.co/mtgv/MobileVLM-1.7B) variants. for more information, please go to [Meituan-AutoML/MobileVLM](https://github.com/Meituan-AutoML/MobileVLM) The implementation is based on llava, and is compatible with llava and mobileVLM. The usage is basically same as llava. -Notice: The overall process of model inference for both **MobilVLM** and **MobilVLM_V2** models is the same, but the process of model conversion is a little different. Therefore, using MobiVLM as an example, the different conversion step will be shown. - ## Usage Build with cmake or run `make llava-cli` to build it. @@ -36,7 +34,7 @@ git clone https://huggingface.co/openai/clip-vit-large-patch14-336 python ./examples/llava/llava-surgery.py -m path/to/MobileVLM-1.7B ``` -3. Use `convert-image-encoder-to-gguf.py` with `--projector-type ldp` (for **V2** the arg is `--projector-type ldpv2`) to convert the LLaVA image encoder to GGUF: +3. Use `convert-image-encoder-to-gguf.py` with `--projector-type ldp` to convert the LLaVA image encoder to GGUF: ```sh python ./examples/llava/convert-image-encoder-to-gguf \ @@ -46,14 +44,6 @@ python ./examples/llava/convert-image-encoder-to-gguf \ --projector-type ldp ``` -```sh -python ./examples/llava/convert-image-encoder-to-gguf \ - -m path/to/clip-vit-large-patch14-336 \ - --llava-projector path/to/MobileVLM-1.7B_V2/llava.projector \ - --output-dir path/to/MobileVLM-1.7B_V2 \ - --projector-type ldpv2 -``` - 4. Use `convert.py` to convert the LLaMA part of LLaVA to GGUF: ```sh diff --git a/examples/llava/clip.cpp b/examples/llava/clip.cpp index 6d7b4950f..690bca2eb 100644 --- a/examples/llava/clip.cpp +++ b/examples/llava/clip.cpp @@ -119,7 +119,6 @@ static std::string format(const char * fmt, ...) { #define TN_LLAVA_PROJ "mm.%d.%s" #define TN_MVLM_PROJ_MLP "mm.model.mlp.%d.%s" #define TN_MVLM_PROJ_BLOCK "mm.model.mb_block.%d.block.%d.%s" -#define TN_MVLM_PROJ_PEG "mm.model.peg.%d.%s" #define TN_IMAGE_NEWLINE "model.image_newline" @@ -127,14 +126,12 @@ enum projector_type { PROJECTOR_TYPE_MLP, PROJECTOR_TYPE_MLP_NORM, PROJECTOR_TYPE_LDP, - PROJECTOR_TYPE_LDPV2, PROJECTOR_TYPE_UNKNOWN, }; static std::map PROJECTOR_TYPE_NAMES = { { PROJECTOR_TYPE_MLP, "mlp" }, { PROJECTOR_TYPE_LDP, "ldp" }, - { PROJECTOR_TYPE_LDPV2, "ldpv2"}, }; @@ -810,29 +807,6 @@ static ggml_cgraph * clip_image_build_graph(clip_ctx * ctx, const clip_image_f32 } embeddings = block_1; } - else if (ctx->proj_type == PROJECTOR_TYPE_LDPV2) - { - int n_patch = 24; - struct ggml_tensor * mlp_0 = ggml_mul_mat(ctx0, model.mm_model_mlp_0_w, embeddings); - mlp_0 = ggml_add(ctx0, mlp_0, model.mm_model_mlp_0_b); - mlp_0 = ggml_gelu(ctx0, mlp_0); - struct ggml_tensor * mlp_2 = ggml_mul_mat(ctx0, model.mm_model_mlp_2_w, mlp_0); - mlp_2 = ggml_add(ctx0, mlp_2, model.mm_model_mlp_2_b); - // mlp_2 ne = [2048, 576, 1, 1] - // // AVG Pool Layer 2*2, strides = 2 - mlp_2 = ggml_cont(ctx0, ggml_permute(ctx0, mlp_2, 1, 0, 2, 3)); - // mlp_2 ne = [576, 2048, 1, 1] - mlp_2 = ggml_reshape_4d(ctx0, mlp_2, n_patch, n_patch, mlp_2->ne[1], mlp_2->ne[2]); - // mlp_2 ne [24, 24, 2048, 1] - mlp_2 = ggml_pool_2d(ctx0, mlp_2, GGML_OP_POOL_AVG, 2, 2, 2, 2, 0, 0); - // weight ne = [3, 3, 2048, 1] - struct ggml_tensor * peg_0 = ggml_conv_depthwise_2d(ctx0, model.mm_model_peg_0_w, mlp_2, 1, 1, 1, 1, 1, 1); - peg_0 = ggml_add(ctx0, peg_0, mlp_2); - peg_0 = ggml_cont(ctx0, ggml_permute(ctx0, peg_0, 1, 2, 0, 3)); - peg_0 = ggml_add(ctx0, peg_0, model.mm_model_peg_0_b); - peg_0 = ggml_reshape_3d(ctx0, peg_0, peg_0->ne[0], peg_0->ne[1] * peg_0->ne[2], peg_0->ne[3]); - embeddings = peg_0; - } else { GGML_ASSERT(false); } @@ -1203,18 +1177,7 @@ struct clip_ctx * clip_model_load(const char * fname, const int verbosity = 1) { vision_model.mm_model_block_2_block_2_0_w = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_BLOCK, 2, 2, "0.weight")); vision_model.mm_model_block_2_block_2_1_w = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_BLOCK, 2, 2, "1.weight")); vision_model.mm_model_block_2_block_2_1_b = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_BLOCK, 2, 2, "1.bias")); - } - else if (new_clip->proj_type == PROJECTOR_TYPE_LDPV2) - { - // MobilVLM_V2 projection - vision_model.mm_model_mlp_0_w = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_MLP, 0, "weight")); - vision_model.mm_model_mlp_0_b = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_MLP, 0, "bias")); - vision_model.mm_model_mlp_2_w = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_MLP, 2, "weight")); - vision_model.mm_model_mlp_2_b = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_MLP, 2, "bias")); - vision_model.mm_model_peg_0_w = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_PEG, 0, "weight")); - vision_model.mm_model_peg_0_b = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_PEG, 0, "bias")); - } - else { + } else { std::string proj_type = PROJECTOR_TYPE_NAMES[new_clip->proj_type]; throw std::runtime_error(format("%s: don't support projector with: %s currently\n", __func__, proj_type.c_str())); } @@ -2003,9 +1966,6 @@ int clip_n_mmproj_embd(const struct clip_ctx * ctx) { if (ctx->proj_type == PROJECTOR_TYPE_LDP) { return ctx->vision_model.mm_model_block_1_block_2_1_b->ne[0]; } - if (ctx->proj_type == PROJECTOR_TYPE_LDPV2) { - return ctx->vision_model.mm_model_peg_0_b->ne[0]; - } if (ctx->proj_type == PROJECTOR_TYPE_MLP) { return ctx->vision_model.mm_2_b->ne[0]; } diff --git a/examples/llava/convert-image-encoder-to-gguf.py b/examples/llava/convert-image-encoder-to-gguf.py index b00bf7c6d..c69f89ac2 100644 --- a/examples/llava/convert-image-encoder-to-gguf.py +++ b/examples/llava/convert-image-encoder-to-gguf.py @@ -1,7 +1,6 @@ import argparse import os import json -import re import torch import numpy as np @@ -39,11 +38,9 @@ def should_skip_tensor(name: str, has_text: bool, has_vision: bool, has_llava: b def get_tensor_name(name: str) -> str: if "projection" in name: return name + if "mm_projector" in name: - name = name.replace("model.mm_projector", "mm") - name = re.sub(r'mm\.mlp\.mlp', 'mm.model.mlp', name, count=1) - name = re.sub(r'mm\.peg\.peg', 'mm.model.peg', name, count=1) - return name + return name.replace("model.mm_projector", "mm") return name.replace("text_model", "t").replace("vision_model", "v").replace("encoder.layers", "blk").replace("embeddings.", "").replace("_proj", "").replace("self_attn.", "attn_").replace("layer_norm", "ln").replace("layernorm", "ln").replace("mlp.fc1", "ffn_down").replace("mlp.fc2", "ffn_up").replace("embedding", "embd").replace("final", "post").replace("layrnorm", "ln") @@ -86,7 +83,7 @@ ap.add_argument("--clip-model-is-vision", action="store_true", required=False, ap.add_argument("--clip-model-is-openclip", action="store_true", required=False, help="The clip model is from openclip (for ViT-SO400M type))") ap.add_argument("--llava-projector", help="Path to llava.projector file. If specified, save an image encoder for LLaVA models.") -ap.add_argument("--projector-type", help="Type of projector. Possible values: mlp, ldp, ldpv2", choices=["mlp", "ldp", "ldpv2"], default="mlp") +ap.add_argument("--projector-type", help="Type of projector. Possible values: mlp, ldp", choices=["mlp", "ldp"], default="mlp") ap.add_argument("-o", "--output-dir", help="Directory to save GGUF files. Default is the original model directory", default=None) # Example --image_mean 0.48145466 0.4578275 0.40821073 --image_std 0.26862954 0.26130258 0.27577711 # Example --image_mean 0.5 0.5 0.5 --image_std 0.5 0.5 0.5 From bc0baab2ea8f806961dd3fb9f534cfeb59a2e1fc Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Wed, 20 Mar 2024 14:14:32 +0200 Subject: [PATCH 668/812] server : allow to override -ngl in tests (#6170) --- examples/server/tests/features/steps/steps.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/examples/server/tests/features/steps/steps.py b/examples/server/tests/features/steps/steps.py index 40c97001a..42cdd8985 100644 --- a/examples/server/tests/features/steps/steps.py +++ b/examples/server/tests/features/steps/steps.py @@ -24,12 +24,16 @@ from prometheus_client import parser def step_server_config(context, server_fqdn, server_port): context.server_fqdn = server_fqdn context.server_port = int(server_port) + context.n_gpu_layer = None if 'PORT' in os.environ: context.server_port = int(os.environ['PORT']) print(f"$PORT set, overriding server port with to {context.server_port}") if 'FQDN' in os.environ: context.server_fqdn = os.environ['FQDN'] print(f"$FQDN set, overriding server fqdn with to {context.server_fqdn}") + if 'N_GPU_LAYERS' in os.environ: + context.n_gpu_layer = int(os.environ['N_GPU_LAYERS']) + print(f"$N_GPU_LAYERS set, overriding n_gpu_layer with to {context.n_gpu_layer}") context.base_url = f'http://{context.server_fqdn}:{context.server_port}' @@ -41,7 +45,6 @@ def step_server_config(context, server_fqdn, server_port): context.n_ctx = None context.n_ga = None context.n_ga_w = None - context.n_gpu_layer = None context.n_predict = None context.n_prompts = 0 context.n_server_predict = None From 6b7e76d28cdf4361740054140708c71828453522 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Wed, 20 Mar 2024 14:17:34 +0200 Subject: [PATCH 669/812] gitignore : ignore curl-related files --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index 1ad8d929b..fbeb9223c 100644 --- a/.gitignore +++ b/.gitignore @@ -12,6 +12,8 @@ *.dot *.bat *.metallib +*.etag +*.lastModified .DS_Store .build/ .cache/ From 91f8ad167dcd24b54615b468d9dd764ebe1d37ad Mon Sep 17 00:00:00 2001 From: Xuan Son Nguyen Date: Wed, 20 Mar 2024 13:30:36 +0100 Subject: [PATCH 670/812] Server: version bump for httplib and json (#6169) * server: version bump for httplib and json * fix build * bring back content_length --- examples/server/httplib.h | 1807 +++++++++++++++++++++++++------------ examples/server/json.hpp | 1664 +++++++++++++++++++--------------- 2 files changed, 2156 insertions(+), 1315 deletions(-) diff --git a/examples/server/httplib.h b/examples/server/httplib.h index 28746000c..f360bd93e 100644 --- a/examples/server/httplib.h +++ b/examples/server/httplib.h @@ -1,14 +1,14 @@ // // httplib.h // -// Copyright (c) 2023 Yuji Hirose. All rights reserved. +// Copyright (c) 2024 Yuji Hirose. All rights reserved. // MIT License // #ifndef CPPHTTPLIB_HTTPLIB_H #define CPPHTTPLIB_HTTPLIB_H -#define CPPHTTPLIB_VERSION "0.12.2" +#define CPPHTTPLIB_VERSION "0.15.3" /* * Configuration @@ -82,6 +82,10 @@ #define CPPHTTPLIB_FORM_URL_ENCODED_PAYLOAD_MAX_LENGTH 8192 #endif +#ifndef CPPHTTPLIB_RANGE_MAX_COUNT +#define CPPHTTPLIB_RANGE_MAX_COUNT 1024 +#endif + #ifndef CPPHTTPLIB_TCP_NODELAY #define CPPHTTPLIB_TCP_NODELAY false #endif @@ -160,10 +164,6 @@ using ssize_t = long; #define WSA_FLAG_NO_HANDLE_INHERIT 0x80 #endif -#ifndef strcasecmp -#define strcasecmp _stricmp -#endif // strcasecmp - using socket_t = SOCKET; #ifdef CPPHTTPLIB_USE_POLL #define poll(fds, nfds, timeout) WSAPoll(fds, nfds, timeout) @@ -172,9 +172,15 @@ using socket_t = SOCKET; #else // not _WIN32 #include -#ifndef _AIX +#if !defined(_AIX) && !defined(__MVS__) #include #endif +#ifdef __MVS__ +#include +#ifndef NI_MAXHOST +#define NI_MAXHOST 1025 +#endif +#endif #include #include #include @@ -187,6 +193,7 @@ using socket_t = SOCKET; #endif #include #include +#include #include #include #include @@ -207,6 +214,7 @@ using socket_t = int; #include #include #include +#include #include #include #include @@ -223,6 +231,9 @@ using socket_t = int; #include #include #include +#include +#include +#include #ifdef CPPHTTPLIB_OPENSSL_SUPPORT #ifdef _WIN32 @@ -237,7 +248,6 @@ using socket_t = int; #ifdef _MSC_VER #pragma comment(lib, "crypt32.lib") -#pragma comment(lib, "cryptui.lib") #endif #elif defined(CPPHTTPLIB_USE_CERTS_FROM_MACOSX_KEYCHAIN) && defined(__APPLE__) #include @@ -259,10 +269,8 @@ using socket_t = int; #include #include -#if OPENSSL_VERSION_NUMBER < 0x1010100fL -#error Sorry, OpenSSL versions prior to 1.1.1 are not supported -#elif OPENSSL_VERSION_NUMBER < 0x30000000L -#define SSL_get1_peer_certificate SSL_get_peer_certificate +#if OPENSSL_VERSION_NUMBER < 0x30000000L +#error Sorry, OpenSSL versions prior to 3.0.0 are not supported #endif #endif @@ -321,7 +329,7 @@ struct scope_exit { explicit scope_exit(std::function &&f) : exit_function(std::move(f)), execute_on_destruction{true} {} - scope_exit(scope_exit &&rhs) + scope_exit(scope_exit &&rhs) noexcept : exit_function(std::move(rhs.exit_function)), execute_on_destruction{rhs.execute_on_destruction} { rhs.release(); @@ -344,6 +352,81 @@ private: } // namespace detail +enum StatusCode { + // Information responses + Continue_100 = 100, + SwitchingProtocol_101 = 101, + Processing_102 = 102, + EarlyHints_103 = 103, + + // Successful responses + OK_200 = 200, + Created_201 = 201, + Accepted_202 = 202, + NonAuthoritativeInformation_203 = 203, + NoContent_204 = 204, + ResetContent_205 = 205, + PartialContent_206 = 206, + MultiStatus_207 = 207, + AlreadyReported_208 = 208, + IMUsed_226 = 226, + + // Redirection messages + MultipleChoices_300 = 300, + MovedPermanently_301 = 301, + Found_302 = 302, + SeeOther_303 = 303, + NotModified_304 = 304, + UseProxy_305 = 305, + unused_306 = 306, + TemporaryRedirect_307 = 307, + PermanentRedirect_308 = 308, + + // Client error responses + BadRequest_400 = 400, + Unauthorized_401 = 401, + PaymentRequired_402 = 402, + Forbidden_403 = 403, + NotFound_404 = 404, + MethodNotAllowed_405 = 405, + NotAcceptable_406 = 406, + ProxyAuthenticationRequired_407 = 407, + RequestTimeout_408 = 408, + Conflict_409 = 409, + Gone_410 = 410, + LengthRequired_411 = 411, + PreconditionFailed_412 = 412, + PayloadTooLarge_413 = 413, + UriTooLong_414 = 414, + UnsupportedMediaType_415 = 415, + RangeNotSatisfiable_416 = 416, + ExpectationFailed_417 = 417, + ImATeapot_418 = 418, + MisdirectedRequest_421 = 421, + UnprocessableContent_422 = 422, + Locked_423 = 423, + FailedDependency_424 = 424, + TooEarly_425 = 425, + UpgradeRequired_426 = 426, + PreconditionRequired_428 = 428, + TooManyRequests_429 = 429, + RequestHeaderFieldsTooLarge_431 = 431, + UnavailableForLegalReasons_451 = 451, + + // Server error responses + InternalServerError_500 = 500, + NotImplemented_501 = 501, + BadGateway_502 = 502, + ServiceUnavailable_503 = 503, + GatewayTimeout_504 = 504, + HttpVersionNotSupported_505 = 505, + VariantAlsoNegotiates_506 = 506, + InsufficientStorage_507 = 507, + LoopDetected_508 = 508, + NotExtended_510 = 510, + NetworkAuthenticationRequired_511 = 511, +}; + using Headers = std::multimap; using Params = std::multimap; @@ -373,17 +456,18 @@ public: DataSink &operator=(DataSink &&) = delete; std::function write; + std::function is_writable; std::function done; std::function done_with_trailer; std::ostream os; private: - class data_sink_streambuf : public std::streambuf { + class data_sink_streambuf final : public std::streambuf { public: explicit data_sink_streambuf(DataSink &sink) : sink_(sink) {} protected: - std::streamsize xsputn(const char *s, std::streamsize n) { + std::streamsize xsputn(const char *s, std::streamsize n) override { sink_.write(s, static_cast(n)); return n; } @@ -465,6 +549,7 @@ struct Request { MultipartFormDataMap files; Ranges ranges; Match matches; + std::unordered_map path_params; // for client ResponseHandler response_handler; @@ -476,8 +561,7 @@ struct Request { bool has_header(const std::string &key) const; std::string get_header_value(const std::string &key, size_t id = 0) const; - template - T get_header_value(const std::string &key, size_t id = 0) const; + uint64_t get_header_value_u64(const std::string &key, size_t id = 0) const; size_t get_header_value_count(const std::string &key) const; void set_header(const std::string &key, const std::string &val); @@ -509,14 +593,14 @@ struct Response { bool has_header(const std::string &key) const; std::string get_header_value(const std::string &key, size_t id = 0) const; - template - T get_header_value(const std::string &key, size_t id = 0) const; + uint64_t get_header_value_u64(const std::string &key, size_t id = 0) const; size_t get_header_value_count(const std::string &key) const; void set_header(const std::string &key, const std::string &val); - void set_redirect(const std::string &url, int status = 302); + void set_redirect(const std::string &url, int status = StatusCode::Found_302); void set_content(const char *s, size_t n, const std::string &content_type); void set_content(const std::string &s, const std::string &content_type); + void set_content(std::string &&s, const std::string &content_type); void set_content_provider( size_t length, const std::string &content_type, ContentProvider provider, @@ -573,15 +657,16 @@ public: TaskQueue() = default; virtual ~TaskQueue() = default; - virtual void enqueue(std::function fn) = 0; + virtual bool enqueue(std::function fn) = 0; virtual void shutdown() = 0; virtual void on_idle() {} }; -class ThreadPool : public TaskQueue { +class ThreadPool final : public TaskQueue { public: - explicit ThreadPool(size_t n) : shutdown_(false) { + explicit ThreadPool(size_t n, size_t mqr = 0) + : shutdown_(false), max_queued_requests_(mqr) { while (n) { threads_.emplace_back(worker(*this)); n--; @@ -591,13 +676,17 @@ public: ThreadPool(const ThreadPool &) = delete; ~ThreadPool() override = default; - void enqueue(std::function fn) override { + bool enqueue(std::function fn) override { { std::unique_lock lock(mutex_); + if (max_queued_requests_ > 0 && jobs_.size() >= max_queued_requests_) { + return false; + } jobs_.push_back(std::move(fn)); } cond_.notify_one(); + return true; } void shutdown() override { @@ -647,6 +736,7 @@ private: std::list> jobs_; bool shutdown_; + size_t max_queued_requests_ = 0; std::condition_variable cond_; std::mutex mutex_; @@ -658,6 +748,82 @@ using SocketOptions = std::function; void default_socket_options(socket_t sock); +const char *status_message(int status); + +std::string get_bearer_token_auth(const Request &req); + +namespace detail { + +class MatcherBase { +public: + virtual ~MatcherBase() = default; + + // Match request path and populate its matches and + virtual bool match(Request &request) const = 0; +}; + +/** + * Captures parameters in request path and stores them in Request::path_params + * + * Capture name is a substring of a pattern from : to /. + * The rest of the pattern is matched agains the request path directly + * Parameters are captured starting from the next character after + * the end of the last matched static pattern fragment until the next /. + * + * Example pattern: + * "/path/fragments/:capture/more/fragments/:second_capture" + * Static fragments: + * "/path/fragments/", "more/fragments/" + * + * Given the following request path: + * "/path/fragments/:1/more/fragments/:2" + * the resulting capture will be + * {{"capture", "1"}, {"second_capture", "2"}} + */ +class PathParamsMatcher final : public MatcherBase { +public: + PathParamsMatcher(const std::string &pattern); + + bool match(Request &request) const override; + +private: + static constexpr char marker = ':'; + // Treat segment separators as the end of path parameter capture + // Does not need to handle query parameters as they are parsed before path + // matching + static constexpr char separator = '/'; + + // Contains static path fragments to match against, excluding the '/' after + // path params + // Fragments are separated by path params + std::vector static_fragments_; + // Stores the names of the path parameters to be used as keys in the + // Request::path_params map + std::vector param_names_; +}; + +/** + * Performs std::regex_match on request path + * and stores the result in Request::matches + * + * Note that regex match is performed directly on the whole request. + * This means that wildcard patterns may match multiple path segments with /: + * "/begin/(.*)/end" will match both "/begin/middle/end" and "/begin/1/2/end". + */ +class RegexMatcher final : public MatcherBase { +public: + RegexMatcher(const std::string &pattern) : regex_(pattern) {} + + bool match(Request &request) const override; + +private: + std::regex regex_; +}; + +ssize_t write_headers(Stream &strm, const Headers &headers); + +} // namespace detail + class Server { public: using Handler = std::function; @@ -702,6 +868,7 @@ public: bool remove_mount_point(const std::string &mount_point); Server &set_file_extension_and_mimetype_mapping(const std::string &ext, const std::string &mime); + Server &set_default_file_mimetype(const std::string &mime); Server &set_file_request_handler(Handler handler); Server &set_error_handler(HandlerWithResponse handler); @@ -718,6 +885,8 @@ public: Server &set_socket_options(SocketOptions socket_options); Server &set_default_headers(Headers headers); + Server & + set_header_writer(std::function const &writer); Server &set_keep_alive_max_count(size_t count); Server &set_keep_alive_timeout(time_t sec); @@ -765,9 +934,14 @@ protected: size_t payload_max_length_ = CPPHTTPLIB_PAYLOAD_MAX_LENGTH; private: - using Handlers = std::vector>; + using Handlers = + std::vector, Handler>>; using HandlersForContentReader = - std::vector>; + std::vector, + HandlerWithContentReader>>; + + static std::unique_ptr + make_matcher(const std::string &pattern); socket_t create_server_socket(const std::string &host, int port, int socket_flags, @@ -778,16 +952,16 @@ private: bool routing(Request &req, Response &res, Stream &strm); bool handle_file_request(const Request &req, Response &res, bool head = false); - bool dispatch_request(Request &req, Response &res, const Handlers &handlers); - bool - dispatch_request_for_content_reader(Request &req, Response &res, - ContentReader content_reader, - const HandlersForContentReader &handlers); + bool dispatch_request(Request &req, Response &res, + const Handlers &handlers) const; + bool dispatch_request_for_content_reader( + Request &req, Response &res, ContentReader content_reader, + const HandlersForContentReader &handlers) const; - bool parse_request_line(const char *s, Request &req); + bool parse_request_line(const char *s, Request &req) const; void apply_ranges(const Request &req, Response &res, - std::string &content_type, std::string &boundary); - bool write_response(Stream &strm, bool close_connection, const Request &req, + std::string &content_type, std::string &boundary) const; + bool write_response(Stream &strm, bool close_connection, Request &req, Response &res); bool write_response_with_content(Stream &strm, bool close_connection, const Request &req, Response &res); @@ -806,21 +980,23 @@ private: bool read_content_core(Stream &strm, Request &req, Response &res, ContentReceiver receiver, MultipartContentHeader multipart_header, - ContentReceiver multipart_receiver); + ContentReceiver multipart_receiver) const; virtual bool process_and_close_socket(socket_t sock); + std::atomic is_running_{false}; + std::atomic done_{false}; + struct MountPointEntry { std::string mount_point; std::string base_dir; Headers headers; }; std::vector base_dirs_; - - std::atomic is_running_{false}; - std::atomic done_{false}; std::map file_extension_and_mimetype_map_; + std::string default_file_mimetype_ = "application/octet-stream"; Handler file_request_handler_; + Handlers get_handlers_; Handlers post_handlers_; HandlersForContentReader post_handlers_for_content_reader_; @@ -831,18 +1007,22 @@ private: Handlers delete_handlers_; HandlersForContentReader delete_handlers_for_content_reader_; Handlers options_handlers_; + HandlerWithResponse error_handler_; ExceptionHandler exception_handler_; HandlerWithResponse pre_routing_handler_; Handler post_routing_handler_; - Logger logger_; Expect100ContinueHandler expect_100_continue_handler_; + Logger logger_; + int address_family_ = AF_UNSPEC; bool tcp_nodelay_ = CPPHTTPLIB_TCP_NODELAY; SocketOptions socket_options_ = default_socket_options; Headers default_headers_; + std::function header_writer_ = + detail::write_headers; }; enum class Error { @@ -860,17 +1040,19 @@ enum class Error { UnsupportedMultipartBoundaryChars, Compression, ConnectionTimeout, + ProxyConnection, // For internal use only SSLPeerCouldBeClosed_, }; -std::string to_string(const Error error); +std::string to_string(Error error); std::ostream &operator<<(std::ostream &os, const Error &obj); class Result { public: + Result() = default; Result(std::unique_ptr &&res, Error err, Headers &&request_headers = Headers{}) : res_(std::move(res)), err_(err), @@ -893,13 +1075,13 @@ public: bool has_request_header(const std::string &key) const; std::string get_request_header_value(const std::string &key, size_t id = 0) const; - template - T get_request_header_value(const std::string &key, size_t id = 0) const; + uint64_t get_request_header_value_u64(const std::string &key, + size_t id = 0) const; size_t get_request_header_value_count(const std::string &key) const; private: std::unique_ptr res_; - Error err_; + Error err_ = Error::Unknown; Headers request_headers_; }; @@ -1059,16 +1241,21 @@ public: bool send(Request &req, Response &res, Error &error); Result send(const Request &req); - size_t is_socket_open() const; - - socket_t socket() const; - void stop(); + std::string host() const; + int port() const; + + size_t is_socket_open() const; + socket_t socket() const; + void set_hostname_addr_map(std::map addr_map); void set_default_headers(Headers headers); + void + set_header_writer(std::function const &writer); + void set_address_family(int family); void set_tcp_nodelay(bool on); void set_socket_options(SocketOptions socket_options); @@ -1117,6 +1304,7 @@ public: void set_ca_cert_path(const std::string &ca_cert_file_path, const std::string &ca_cert_dir_path = std::string()); void set_ca_cert_store(X509_STORE *ca_cert_store); + X509_STORE *create_ca_cert_store(const char *ca_cert, std::size_t size) const; #endif #ifdef CPPHTTPLIB_OPENSSL_SUPPORT @@ -1145,14 +1333,14 @@ protected: // Also, shutdown_ssl and close_socket should also NOT be called concurrently // with a DIFFERENT thread sending requests using that socket. virtual void shutdown_ssl(Socket &socket, bool shutdown_gracefully); - void shutdown_socket(Socket &socket); + void shutdown_socket(Socket &socket) const; void close_socket(Socket &socket); bool process_request(Stream &strm, Request &req, Response &res, bool close_connection, Error &error); bool write_content_with_provider(Stream &strm, const Request &req, - Error &error); + Error &error) const; void copy_settings(const ClientImpl &rhs); @@ -1177,6 +1365,10 @@ protected: // Default headers Headers default_headers_; + // Header writer + std::function header_writer_ = + detail::write_headers; + // Settings std::string client_cert_path_; std::string client_key_path_; @@ -1239,7 +1431,8 @@ private: Result send_(Request &&req); socket_t create_client_socket(Error &error) const; - bool read_response_line(Stream &strm, const Request &req, Response &res); + bool read_response_line(Stream &strm, const Request &req, + Response &res) const; bool write_request(Stream &strm, Request &req, bool close_connection, Error &error); bool redirect(Request &req, Response &res, Error &error); @@ -1258,7 +1451,7 @@ private: const std::string &content_type); ContentProviderWithoutLength get_multipart_content_provider( const std::string &boundary, const MultipartFormDataItems &items, - const MultipartFormDataProviderItems &provider_items); + const MultipartFormDataProviderItems &provider_items) const; std::string adjust_host_string(const std::string &host) const; @@ -1431,16 +1624,21 @@ public: bool send(Request &req, Response &res, Error &error); Result send(const Request &req); - size_t is_socket_open() const; - - socket_t socket() const; - void stop(); + std::string host() const; + int port() const; + + size_t is_socket_open() const; + socket_t socket() const; + void set_hostname_addr_map(std::map addr_map); void set_default_headers(Headers headers); + void + set_header_writer(std::function const &writer); + void set_address_family(int family); void set_tcp_nodelay(bool on); void set_socket_options(SocketOptions socket_options); @@ -1497,6 +1695,7 @@ public: const std::string &ca_cert_dir_path = std::string()); void set_ca_cert_store(X509_STORE *ca_cert_store); + void load_ca_cert_store(const char *ca_cert, std::size_t size); long get_openssl_verify_result() const; @@ -1538,7 +1737,7 @@ private: std::mutex ctx_mutex_; }; -class SSLClient : public ClientImpl { +class SSLClient final : public ClientImpl { public: explicit SSLClient(const std::string &host); @@ -1546,16 +1745,19 @@ public: explicit SSLClient(const std::string &host, int port, const std::string &client_cert_path, - const std::string &client_key_path); + const std::string &client_key_path, + const std::string &private_key_password = std::string()); explicit SSLClient(const std::string &host, int port, X509 *client_cert, - EVP_PKEY *client_key); + EVP_PKEY *client_key, + const std::string &private_key_password = std::string()); ~SSLClient() override; bool is_valid() const override; void set_ca_cert_store(X509_STORE *ca_cert_store); + void load_ca_cert_store(const char *ca_cert, std::size_t size); long get_openssl_verify_result() const; @@ -1564,7 +1766,7 @@ public: private: bool create_and_connect_socket(Socket &socket, Error &error) override; void shutdown_ssl(Socket &socket, bool shutdown_gracefully) override; - void shutdown_ssl_impl(Socket &socket, bool shutdown_socket); + void shutdown_ssl_impl(Socket &socket, bool shutdown_gracefully); bool process_socket(const Socket &socket, std::function callback) override; @@ -1608,15 +1810,9 @@ inline void duration_to_sec_and_usec(const T &duration, U callback) { callback(static_cast(sec), static_cast(usec)); } -template -inline T get_header_value(const Headers & /*headers*/, - const std::string & /*key*/, size_t /*id*/ = 0, - uint64_t /*def*/ = 0) {} - -template <> -inline uint64_t get_header_value(const Headers &headers, - const std::string &key, size_t id, - uint64_t def) { +inline uint64_t get_header_value_u64(const Headers &headers, + const std::string &key, size_t id, + uint64_t def) { auto rng = headers.equal_range(key); auto it = rng.first; std::advance(it, static_cast(id)); @@ -1628,14 +1824,14 @@ inline uint64_t get_header_value(const Headers &headers, } // namespace detail -template -inline T Request::get_header_value(const std::string &key, size_t id) const { - return detail::get_header_value(headers, key, id, 0); +inline uint64_t Request::get_header_value_u64(const std::string &key, + size_t id) const { + return detail::get_header_value_u64(headers, key, id, 0); } -template -inline T Response::get_header_value(const std::string &key, size_t id) const { - return detail::get_header_value(headers, key, id, 0); +inline uint64_t Response::get_header_value_u64(const std::string &key, + size_t id) const { + return detail::get_header_value_u64(headers, key, id, 0); } template @@ -1665,21 +1861,106 @@ inline ssize_t Stream::write_format(const char *fmt, const Args &...args) { inline void default_socket_options(socket_t sock) { int yes = 1; #ifdef _WIN32 - setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, reinterpret_cast(&yes), - sizeof(yes)); + setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, + reinterpret_cast(&yes), sizeof(yes)); setsockopt(sock, SOL_SOCKET, SO_EXCLUSIVEADDRUSE, - reinterpret_cast(&yes), sizeof(yes)); + reinterpret_cast(&yes), sizeof(yes)); #else #ifdef SO_REUSEPORT - setsockopt(sock, SOL_SOCKET, SO_REUSEPORT, reinterpret_cast(&yes), - sizeof(yes)); + setsockopt(sock, SOL_SOCKET, SO_REUSEPORT, + reinterpret_cast(&yes), sizeof(yes)); #else - setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, reinterpret_cast(&yes), - sizeof(yes)); + setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, + reinterpret_cast(&yes), sizeof(yes)); #endif #endif } +inline const char *status_message(int status) { + switch (status) { + case StatusCode::Continue_100: return "Continue"; + case StatusCode::SwitchingProtocol_101: return "Switching Protocol"; + case StatusCode::Processing_102: return "Processing"; + case StatusCode::EarlyHints_103: return "Early Hints"; + case StatusCode::OK_200: return "OK"; + case StatusCode::Created_201: return "Created"; + case StatusCode::Accepted_202: return "Accepted"; + case StatusCode::NonAuthoritativeInformation_203: + return "Non-Authoritative Information"; + case StatusCode::NoContent_204: return "No Content"; + case StatusCode::ResetContent_205: return "Reset Content"; + case StatusCode::PartialContent_206: return "Partial Content"; + case StatusCode::MultiStatus_207: return "Multi-Status"; + case StatusCode::AlreadyReported_208: return "Already Reported"; + case StatusCode::IMUsed_226: return "IM Used"; + case StatusCode::MultipleChoices_300: return "Multiple Choices"; + case StatusCode::MovedPermanently_301: return "Moved Permanently"; + case StatusCode::Found_302: return "Found"; + case StatusCode::SeeOther_303: return "See Other"; + case StatusCode::NotModified_304: return "Not Modified"; + case StatusCode::UseProxy_305: return "Use Proxy"; + case StatusCode::unused_306: return "unused"; + case StatusCode::TemporaryRedirect_307: return "Temporary Redirect"; + case StatusCode::PermanentRedirect_308: return "Permanent Redirect"; + case StatusCode::BadRequest_400: return "Bad Request"; + case StatusCode::Unauthorized_401: return "Unauthorized"; + case StatusCode::PaymentRequired_402: return "Payment Required"; + case StatusCode::Forbidden_403: return "Forbidden"; + case StatusCode::NotFound_404: return "Not Found"; + case StatusCode::MethodNotAllowed_405: return "Method Not Allowed"; + case StatusCode::NotAcceptable_406: return "Not Acceptable"; + case StatusCode::ProxyAuthenticationRequired_407: + return "Proxy Authentication Required"; + case StatusCode::RequestTimeout_408: return "Request Timeout"; + case StatusCode::Conflict_409: return "Conflict"; + case StatusCode::Gone_410: return "Gone"; + case StatusCode::LengthRequired_411: return "Length Required"; + case StatusCode::PreconditionFailed_412: return "Precondition Failed"; + case StatusCode::PayloadTooLarge_413: return "Payload Too Large"; + case StatusCode::UriTooLong_414: return "URI Too Long"; + case StatusCode::UnsupportedMediaType_415: return "Unsupported Media Type"; + case StatusCode::RangeNotSatisfiable_416: return "Range Not Satisfiable"; + case StatusCode::ExpectationFailed_417: return "Expectation Failed"; + case StatusCode::ImATeapot_418: return "I'm a teapot"; + case StatusCode::MisdirectedRequest_421: return "Misdirected Request"; + case StatusCode::UnprocessableContent_422: return "Unprocessable Content"; + case StatusCode::Locked_423: return "Locked"; + case StatusCode::FailedDependency_424: return "Failed Dependency"; + case StatusCode::TooEarly_425: return "Too Early"; + case StatusCode::UpgradeRequired_426: return "Upgrade Required"; + case StatusCode::PreconditionRequired_428: return "Precondition Required"; + case StatusCode::TooManyRequests_429: return "Too Many Requests"; + case StatusCode::RequestHeaderFieldsTooLarge_431: + return "Request Header Fields Too Large"; + case StatusCode::UnavailableForLegalReasons_451: + return "Unavailable For Legal Reasons"; + case StatusCode::NotImplemented_501: return "Not Implemented"; + case StatusCode::BadGateway_502: return "Bad Gateway"; + case StatusCode::ServiceUnavailable_503: return "Service Unavailable"; + case StatusCode::GatewayTimeout_504: return "Gateway Timeout"; + case StatusCode::HttpVersionNotSupported_505: + return "HTTP Version Not Supported"; + case StatusCode::VariantAlsoNegotiates_506: return "Variant Also Negotiates"; + case StatusCode::InsufficientStorage_507: return "Insufficient Storage"; + case StatusCode::LoopDetected_508: return "Loop Detected"; + case StatusCode::NotExtended_510: return "Not Extended"; + case StatusCode::NetworkAuthenticationRequired_511: + return "Network Authentication Required"; + + default: + case StatusCode::InternalServerError_500: return "Internal Server Error"; + } +} + +inline std::string get_bearer_token_auth(const Request &req) { + if (req.has_header("Authorization")) { + static std::string BearerHeaderPrefix = "Bearer "; + return req.get_header_value("Authorization") + .substr(BearerHeaderPrefix.length()); + } + return ""; +} + template inline Server & Server::set_read_timeout(const std::chrono::duration &duration) { @@ -1720,6 +2001,7 @@ inline std::string to_string(const Error error) { return "Unsupported HTTP multipart boundary characters"; case Error::Compression: return "Compression failed"; case Error::ConnectionTimeout: return "Connection timed out"; + case Error::ProxyConnection: return "Proxy connection failed"; case Error::Unknown: return "Unknown"; default: break; } @@ -1733,10 +2015,9 @@ inline std::ostream &operator<<(std::ostream &os, const Error &obj) { return os; } -template -inline T Result::get_request_header_value(const std::string &key, - size_t id) const { - return detail::get_header_value(request_headers_, key, id, 0); +inline uint64_t Result::get_request_header_value_u64(const std::string &key, + size_t id) const { + return detail::get_header_value_u64(request_headers_, key, id, 0); } template @@ -1790,7 +2071,7 @@ void hosted_at(const std::string &hostname, std::vector &addrs); std::string append_query_params(const std::string &path, const Params ¶ms); -std::pair make_range_header(Ranges ranges); +std::pair make_range_header(const Ranges &ranges); std::pair make_basic_authentication_header(const std::string &username, @@ -1810,6 +2091,9 @@ std::string trim_copy(const std::string &s); void split(const char *b, const char *e, char d, std::function fn); +void split(const char *b, const char *e, char d, size_t m, + std::function fn); + bool process_client_socket(socket_t sock, time_t read_timeout_sec, time_t read_timeout_usec, time_t write_timeout_sec, time_t write_timeout_usec, @@ -1844,7 +2128,7 @@ enum class EncodingType { None = 0, Gzip, Brotli }; EncodingType encoding_type(const Request &req, const Response &res); -class BufferStream : public Stream { +class BufferStream final : public Stream { public: BufferStream() = default; ~BufferStream() override = default; @@ -1884,19 +2168,19 @@ public: Callback callback) = 0; }; -class nocompressor : public compressor { +class nocompressor final : public compressor { public: - virtual ~nocompressor() = default; + ~nocompressor() override = default; bool compress(const char *data, size_t data_length, bool /*last*/, Callback callback) override; }; #ifdef CPPHTTPLIB_ZLIB_SUPPORT -class gzip_compressor : public compressor { +class gzip_compressor final : public compressor { public: gzip_compressor(); - ~gzip_compressor(); + ~gzip_compressor() override; bool compress(const char *data, size_t data_length, bool last, Callback callback) override; @@ -1906,10 +2190,10 @@ private: z_stream strm_; }; -class gzip_decompressor : public decompressor { +class gzip_decompressor final : public decompressor { public: gzip_decompressor(); - ~gzip_decompressor(); + ~gzip_decompressor() override; bool is_valid() const override; @@ -1923,7 +2207,7 @@ private: #endif #ifdef CPPHTTPLIB_BROTLI_SUPPORT -class brotli_compressor : public compressor { +class brotli_compressor final : public compressor { public: brotli_compressor(); ~brotli_compressor(); @@ -1935,7 +2219,7 @@ private: BrotliEncoderState *state_ = nullptr; }; -class brotli_decompressor : public decompressor { +class brotli_decompressor final : public decompressor { public: brotli_decompressor(); ~brotli_decompressor(); @@ -1972,6 +2256,29 @@ private: std::string glowable_buffer_; }; +class mmap { +public: + mmap(const char *path); + ~mmap(); + + bool open(const char *path); + void close(); + + bool is_open() const; + size_t size() const; + const char *data() const; + +private: +#if defined(_WIN32) + HANDLE hFile_; + HANDLE hMapping_; +#else + int fd_; +#endif + size_t size_; + void *addr_; +}; + } // namespace detail // ---------------------------------------------------------------------------- @@ -2003,7 +2310,7 @@ inline bool from_hex_to_i(const std::string &s, size_t i, size_t cnt, val = 0; for (; cnt; i++, cnt--) { if (!s[i]) { return false; } - int v = 0; + auto v = 0; if (is_hex(s[i], v)) { val = val * 16 + v; } else { @@ -2014,7 +2321,7 @@ inline bool from_hex_to_i(const std::string &s, size_t i, size_t cnt, } inline std::string from_i_to_hex(size_t n) { - const char *charset = "0123456789abcdef"; + static const auto charset = "0123456789abcdef"; std::string ret; do { ret = charset[n & 15] + ret; @@ -2025,7 +2332,7 @@ inline std::string from_i_to_hex(size_t n) { inline size_t to_utf8(int code, char *buff) { if (code < 0x0080) { - buff[0] = (code & 0x7F); + buff[0] = static_cast(code & 0x7F); return 1; } else if (code < 0x0800) { buff[0] = static_cast(0xC0 | ((code >> 6) & 0x1F)); @@ -2064,8 +2371,8 @@ inline std::string base64_encode(const std::string &in) { std::string out; out.reserve(in.size()); - int val = 0; - int valb = -6; + auto val = 0; + auto valb = -6; for (auto c : in) { val = (val << 8) + static_cast(c); @@ -2112,6 +2419,11 @@ inline bool is_valid_path(const std::string &path) { // Read component auto beg = i; while (i < path.size() && path[i] != '/') { + if (path[i] == '\0') { + return false; + } else if (path[i] == '\\') { + return false; + } i++; } @@ -2196,7 +2508,7 @@ inline std::string decode_url(const std::string &s, for (size_t i = 0; i < s.size(); i++) { if (s[i] == '%' && i + 1 < s.size()) { if (s[i + 1] == 'u') { - int val = 0; + auto val = 0; if (from_hex_to_i(s, i + 2, 4, val)) { // 4 digits Unicode codes char buff[4]; @@ -2207,7 +2519,7 @@ inline std::string decode_url(const std::string &s, result += s[i]; } } else { - int val = 0; + auto val = 0; if (from_hex_to_i(s, i + 1, 2, val)) { // 2 digits hex codes result += static_cast(val); @@ -2260,16 +2572,30 @@ inline std::string trim_copy(const std::string &s) { return s.substr(r.first, r.second - r.first); } +inline std::string trim_double_quotes_copy(const std::string &s) { + if (s.length() >= 2 && s.front() == '"' && s.back() == '"') { + return s.substr(1, s.size() - 2); + } + return s; +} + inline void split(const char *b, const char *e, char d, std::function fn) { + return split(b, e, d, (std::numeric_limits::max)(), std::move(fn)); +} + +inline void split(const char *b, const char *e, char d, size_t m, + std::function fn) { size_t i = 0; size_t beg = 0; + size_t count = 1; while (e ? (b + i < e) : (b[i] != '\0')) { - if (b[i] == d) { + if (b[i] == d && count < m) { auto r = trim(b, e, beg, i); if (r.first < r.second) { fn(&b[r.first], &b[r.second]); } beg = i + 1; + count++; } i++; } @@ -2345,6 +2671,105 @@ inline void stream_line_reader::append(char c) { } } +inline mmap::mmap(const char *path) +#if defined(_WIN32) + : hFile_(NULL), hMapping_(NULL) +#else + : fd_(-1) +#endif + , + size_(0), addr_(nullptr) { + open(path); +} + +inline mmap::~mmap() { close(); } + +inline bool mmap::open(const char *path) { + close(); + +#if defined(_WIN32) + std::wstring wpath; + for (size_t i = 0; i < strlen(path); i++) { + wpath += path[i]; + } + + hFile_ = ::CreateFile2(wpath.c_str(), GENERIC_READ, FILE_SHARE_READ, + OPEN_EXISTING, NULL); + + if (hFile_ == INVALID_HANDLE_VALUE) { return false; } + + LARGE_INTEGER size{}; + if (!::GetFileSizeEx(hFile_, &size)) { return false; } + size_ = static_cast(size.QuadPart); + + hMapping_ = + ::CreateFileMappingFromApp(hFile_, NULL, PAGE_READONLY, size_, NULL); + + if (hMapping_ == NULL) { + close(); + return false; + } + + addr_ = ::MapViewOfFileFromApp(hMapping_, FILE_MAP_READ, 0, 0); +#else + fd_ = ::open(path, O_RDONLY); + if (fd_ == -1) { return false; } + + struct stat sb; + if (fstat(fd_, &sb) == -1) { + close(); + return false; + } + size_ = static_cast(sb.st_size); + + addr_ = ::mmap(NULL, size_, PROT_READ, MAP_PRIVATE, fd_, 0); +#endif + + if (addr_ == nullptr) { + close(); + return false; + } + + return true; +} + +inline bool mmap::is_open() const { return addr_ != nullptr; } + +inline size_t mmap::size() const { return size_; } + +inline const char *mmap::data() const { + return static_cast(addr_); +} + +inline void mmap::close() { +#if defined(_WIN32) + if (addr_) { + ::UnmapViewOfFile(addr_); + addr_ = nullptr; + } + + if (hMapping_) { + ::CloseHandle(hMapping_); + hMapping_ = NULL; + } + + if (hFile_ != INVALID_HANDLE_VALUE) { + ::CloseHandle(hFile_); + hFile_ = INVALID_HANDLE_VALUE; + } +#else + if (addr_ != nullptr) { + munmap(addr_, size_); + addr_ = nullptr; + } + + if (fd_ != -1) { + ::close(fd_); + fd_ = -1; + } +#endif + size_ = 0; +} inline int close_socket(socket_t sock) { #ifdef _WIN32 return closesocket(sock); @@ -2354,7 +2779,7 @@ inline int close_socket(socket_t sock) { } template inline ssize_t handle_EINTR(T fn) { - ssize_t res = false; + ssize_t res = 0; while (true) { res = fn(); if (res < 0 && errno == EINTR) { continue; } @@ -2399,7 +2824,7 @@ inline ssize_t select_read(socket_t sock, time_t sec, time_t usec) { return handle_EINTR([&]() { return poll(&pfd_read, 1, timeout); }); #else #ifndef _WIN32 - if (sock >= FD_SETSIZE) { return 1; } + if (sock >= FD_SETSIZE) { return -1; } #endif fd_set fds; @@ -2427,7 +2852,7 @@ inline ssize_t select_write(socket_t sock, time_t sec, time_t usec) { return handle_EINTR([&]() { return poll(&pfd_read, 1, timeout); }); #else #ifndef _WIN32 - if (sock >= FD_SETSIZE) { return 1; } + if (sock >= FD_SETSIZE) { return -1; } #endif fd_set fds; @@ -2458,7 +2883,7 @@ inline Error wait_until_socket_is_ready(socket_t sock, time_t sec, if (poll_res == 0) { return Error::ConnectionTimeout; } if (poll_res > 0 && pfd_read.revents & (POLLIN | POLLOUT)) { - int error = 0; + auto error = 0; socklen_t len = sizeof(error); auto res = getsockopt(sock, SOL_SOCKET, SO_ERROR, reinterpret_cast(&error), &len); @@ -2490,7 +2915,7 @@ inline Error wait_until_socket_is_ready(socket_t sock, time_t sec, if (ret == 0) { return Error::ConnectionTimeout; } if (ret > 0 && (FD_ISSET(sock, &fdsr) || FD_ISSET(sock, &fdsw))) { - int error = 0; + auto error = 0; socklen_t len = sizeof(error); auto res = getsockopt(sock, SOL_SOCKET, SO_ERROR, reinterpret_cast(&error), &len); @@ -2512,7 +2937,7 @@ inline bool is_socket_alive(socket_t sock) { return detail::read_socket(sock, &buf[0], sizeof(buf), MSG_PEEK) > 0; } -class SocketStream : public Stream { +class SocketStream final : public Stream { public: SocketStream(socket_t sock, time_t read_timeout_sec, time_t read_timeout_usec, time_t write_timeout_sec, time_t write_timeout_usec); @@ -2537,11 +2962,11 @@ private: size_t read_buff_off_ = 0; size_t read_buff_content_size_ = 0; - static const size_t read_buff_size_ = 1024 * 4; + static const size_t read_buff_size_ = 1024l * 4; }; #ifdef CPPHTTPLIB_OPENSSL_SUPPORT -class SSLSocketStream : public Stream { +class SSLSocketStream final : public Stream { public: SSLSocketStream(socket_t sock, SSL *ssl, time_t read_timeout_sec, time_t read_timeout_usec, time_t write_timeout_sec, @@ -2666,7 +3091,7 @@ socket_t create_socket(const std::string &host, const std::string &ip, int port, #ifndef _WIN32 if (hints.ai_family == AF_UNIX) { const auto addrlen = host.length(); - if (addrlen > sizeof(sockaddr_un::sun_path)) return INVALID_SOCKET; + if (addrlen > sizeof(sockaddr_un::sun_path)) { return INVALID_SOCKET; } auto sock = socket(hints.ai_family, hints.ai_socktype, hints.ai_protocol); if (sock != INVALID_SOCKET) { @@ -2735,17 +3160,27 @@ socket_t create_socket(const std::string &host, const std::string &ip, int port, #endif if (tcp_nodelay) { - int yes = 1; - setsockopt(sock, IPPROTO_TCP, TCP_NODELAY, reinterpret_cast(&yes), - sizeof(yes)); + auto yes = 1; +#ifdef _WIN32 + setsockopt(sock, IPPROTO_TCP, TCP_NODELAY, + reinterpret_cast(&yes), sizeof(yes)); +#else + setsockopt(sock, IPPROTO_TCP, TCP_NODELAY, + reinterpret_cast(&yes), sizeof(yes)); +#endif } if (socket_options) { socket_options(sock); } if (rp->ai_family == AF_INET6) { - int no = 0; - setsockopt(sock, IPPROTO_IPV6, IPV6_V6ONLY, reinterpret_cast(&no), - sizeof(no)); + auto no = 0; +#ifdef _WIN32 + setsockopt(sock, IPPROTO_IPV6, IPV6_V6ONLY, + reinterpret_cast(&no), sizeof(no)); +#else + setsockopt(sock, IPPROTO_IPV6, IPV6_V6ONLY, + reinterpret_cast(&no), sizeof(no)); +#endif } // bind or connect @@ -2804,7 +3239,7 @@ inline bool bind_ip_address(socket_t sock, const std::string &host) { return ret; } -#if !defined _WIN32 && !defined ANDROID && !defined _AIX +#if !defined _WIN32 && !defined ANDROID && !defined _AIX && !defined __MVS__ #define USE_IF2IP #endif @@ -2860,7 +3295,7 @@ inline socket_t create_client_socket( #ifdef USE_IF2IP auto ip_from_if = if2ip(address_family, intf); if (ip_from_if.empty()) { ip_from_if = intf; } - if (!bind_ip_address(sock2, ip_from_if.c_str())) { + if (!bind_ip_address(sock2, ip_from_if)) { error = Error::BindIPAddress; return false; } @@ -2888,13 +3323,14 @@ inline socket_t create_client_socket( #ifdef _WIN32 auto timeout = static_cast(read_timeout_sec * 1000 + read_timeout_usec / 1000); - setsockopt(sock2, SOL_SOCKET, SO_RCVTIMEO, (char *)&timeout, - sizeof(timeout)); + setsockopt(sock2, SOL_SOCKET, SO_RCVTIMEO, + reinterpret_cast(&timeout), sizeof(timeout)); #else timeval tv; tv.tv_sec = static_cast(read_timeout_sec); tv.tv_usec = static_cast(read_timeout_usec); - setsockopt(sock2, SOL_SOCKET, SO_RCVTIMEO, (char *)&tv, sizeof(tv)); + setsockopt(sock2, SOL_SOCKET, SO_RCVTIMEO, + reinterpret_cast(&tv), sizeof(tv)); #endif } { @@ -2902,13 +3338,14 @@ inline socket_t create_client_socket( #ifdef _WIN32 auto timeout = static_cast(write_timeout_sec * 1000 + write_timeout_usec / 1000); - setsockopt(sock2, SOL_SOCKET, SO_SNDTIMEO, (char *)&timeout, - sizeof(timeout)); + setsockopt(sock2, SOL_SOCKET, SO_SNDTIMEO, + reinterpret_cast(&timeout), sizeof(timeout)); #else timeval tv; tv.tv_sec = static_cast(write_timeout_sec); tv.tv_usec = static_cast(write_timeout_usec); - setsockopt(sock2, SOL_SOCKET, SO_SNDTIMEO, (char *)&tv, sizeof(tv)); + setsockopt(sock2, SOL_SOCKET, SO_SNDTIMEO, + reinterpret_cast(&tv), sizeof(tv)); #endif } @@ -3008,18 +3445,20 @@ inline constexpr unsigned int operator"" _t(const char *s, size_t l) { } // namespace udl -inline const char * +inline std::string find_content_type(const std::string &path, - const std::map &user_data) { + const std::map &user_data, + const std::string &default_content_type) { auto ext = file_extension(path); auto it = user_data.find(ext); - if (it != user_data.end()) { return it->second.c_str(); } + if (it != user_data.end()) { return it->second; } using udl::operator""_t; switch (str2tag(ext)) { - default: return nullptr; + default: return default_content_type; + case "css"_t: return "text/css"; case "csv"_t: return "text/csv"; case "htm"_t: @@ -3072,76 +3511,6 @@ find_content_type(const std::string &path, } } -inline const char *status_message(int status) { - switch (status) { - case 100: return "Continue"; - case 101: return "Switching Protocol"; - case 102: return "Processing"; - case 103: return "Early Hints"; - case 200: return "OK"; - case 201: return "Created"; - case 202: return "Accepted"; - case 203: return "Non-Authoritative Information"; - case 204: return "No Content"; - case 205: return "Reset Content"; - case 206: return "Partial Content"; - case 207: return "Multi-Status"; - case 208: return "Already Reported"; - case 226: return "IM Used"; - case 300: return "Multiple Choice"; - case 301: return "Moved Permanently"; - case 302: return "Found"; - case 303: return "See Other"; - case 304: return "Not Modified"; - case 305: return "Use Proxy"; - case 306: return "unused"; - case 307: return "Temporary Redirect"; - case 308: return "Permanent Redirect"; - case 400: return "Bad Request"; - case 401: return "Unauthorized"; - case 402: return "Payment Required"; - case 403: return "Forbidden"; - case 404: return "Not Found"; - case 405: return "Method Not Allowed"; - case 406: return "Not Acceptable"; - case 407: return "Proxy Authentication Required"; - case 408: return "Request Timeout"; - case 409: return "Conflict"; - case 410: return "Gone"; - case 411: return "Length Required"; - case 412: return "Precondition Failed"; - case 413: return "Payload Too Large"; - case 414: return "URI Too Long"; - case 415: return "Unsupported Media Type"; - case 416: return "Range Not Satisfiable"; - case 417: return "Expectation Failed"; - case 418: return "I'm a teapot"; - case 421: return "Misdirected Request"; - case 422: return "Unprocessable Entity"; - case 423: return "Locked"; - case 424: return "Failed Dependency"; - case 425: return "Too Early"; - case 426: return "Upgrade Required"; - case 428: return "Precondition Required"; - case 429: return "Too Many Requests"; - case 431: return "Request Header Fields Too Large"; - case 451: return "Unavailable For Legal Reasons"; - case 501: return "Not Implemented"; - case 502: return "Bad Gateway"; - case 503: return "Service Unavailable"; - case 504: return "Gateway Timeout"; - case 505: return "HTTP Version Not Supported"; - case 506: return "Variant Also Negotiates"; - case 507: return "Insufficient Storage"; - case 508: return "Loop Detected"; - case 510: return "Not Extended"; - case 511: return "Network Authentication Required"; - - default: - case 500: return "Internal Server Error"; - } -} - inline bool can_compress_content_type(const std::string &content_type) { using udl::operator""_t; @@ -3218,7 +3587,7 @@ inline bool gzip_compressor::compress(const char *data, size_t data_length, data += strm_.avail_in; auto flush = (last && data_length == 0) ? Z_FINISH : Z_NO_FLUSH; - int ret = Z_OK; + auto ret = Z_OK; std::array buff{}; do { @@ -3262,7 +3631,7 @@ inline bool gzip_decompressor::decompress(const char *data, size_t data_length, Callback callback) { assert(is_valid_); - int ret = Z_OK; + auto ret = Z_OK; do { constexpr size_t max_avail_in = @@ -3276,16 +3645,12 @@ inline bool gzip_decompressor::decompress(const char *data, size_t data_length, data += strm_.avail_in; std::array buff{}; - while (strm_.avail_in > 0) { + while (strm_.avail_in > 0 && ret == Z_OK) { strm_.avail_out = static_cast(buff.size()); strm_.next_out = reinterpret_cast(buff.data()); - auto prev_avail_in = strm_.avail_in; - ret = inflate(&strm_, Z_NO_FLUSH); - if (prev_avail_in - strm_.avail_in == 0) { return false; } - assert(ret != Z_STREAM_ERROR); switch (ret) { case Z_NEED_DICT: @@ -3298,7 +3663,7 @@ inline bool gzip_decompressor::decompress(const char *data, size_t data_length, } } - if (ret != Z_OK && ret != Z_STREAM_END) return false; + if (ret != Z_OK && ret != Z_STREAM_END) { return false; } } while (data_length > 0); @@ -3367,7 +3732,7 @@ inline bool brotli_decompressor::decompress(const char *data, return 0; } - const uint8_t *next_in = (const uint8_t *)data; + auto next_in = reinterpret_cast(data); size_t avail_in = data_length; size_t total_out; @@ -3437,6 +3802,9 @@ inline bool parse_header(const char *beg, const char *end, T fn) { } if (p < end) { + auto key_len = key_end - beg; + if (!key_len) { return false; } + auto key = std::string(beg, key_end); auto val = compare_case_ignore(key, "Location") ? std::string(p, end) @@ -3526,11 +3894,7 @@ inline bool read_content_without_length(Stream &strm, uint64_t r = 0; for (;;) { auto n = strm.read(buf, CPPHTTPLIB_RECV_BUFSIZ); - if (n < 0) { - return false; - } else if (n == 0) { - return true; - } + if (n <= 0) { return true; } if (!out(buf, static_cast(n), r, 0)) { return false; } r += static_cast(n); @@ -3566,7 +3930,7 @@ inline bool read_content_chunked(Stream &strm, T &x, if (!line_reader.getline()) { return false; } - if (strcmp(line_reader.ptr(), "\r\n")) { return false; } + if (strcmp(line_reader.ptr(), "\r\n") != 0) { return false; } if (!line_reader.getline()) { return false; } } @@ -3576,7 +3940,7 @@ inline bool read_content_chunked(Stream &strm, T &x, // Trailer if (!line_reader.getline()) { return false; } - while (strcmp(line_reader.ptr(), "\r\n")) { + while (strcmp(line_reader.ptr(), "\r\n") != 0) { if (line_reader.size() > CPPHTTPLIB_HEADER_MAX_LENGTH) { return false; } // Exclude line terminator @@ -3595,8 +3959,8 @@ inline bool read_content_chunked(Stream &strm, T &x, } inline bool is_chunked_transfer_encoding(const Headers &headers) { - return !strcasecmp(get_header_value(headers, "Transfer-Encoding", 0, ""), - "chunked"); + return compare_case_ignore( + get_header_value(headers, "Transfer-Encoding", 0, ""), "chunked"); } template @@ -3611,14 +3975,14 @@ bool prepare_content_receiver(T &x, int &status, #ifdef CPPHTTPLIB_ZLIB_SUPPORT decompressor = detail::make_unique(); #else - status = 415; + status = StatusCode::UnsupportedMediaType_415; return false; #endif } else if (encoding.find("br") != std::string::npos) { #ifdef CPPHTTPLIB_BROTLI_SUPPORT decompressor = detail::make_unique(); #else - status = 415; + status = StatusCode::UnsupportedMediaType_415; return false; #endif } @@ -3634,7 +3998,7 @@ bool prepare_content_receiver(T &x, int &status, }; return callback(std::move(out)); } else { - status = 500; + status = StatusCode::InternalServerError_500; return false; } } @@ -3662,7 +4026,7 @@ bool read_content(Stream &strm, T &x, size_t payload_max_length, int &status, } else if (!has_header(x.headers, "Content-Length")) { ret = read_content_without_length(strm, out); } else { - auto len = get_header_value(x.headers, "Content-Length"); + auto len = get_header_value_u64(x.headers, "Content-Length", 0, 0); if (len > payload_max_length) { exceed_payload_max_length = true; skip_content_with_length(strm, len); @@ -3672,7 +4036,10 @@ bool read_content(Stream &strm, T &x, size_t payload_max_length, int &status, } } - if (!ret) { status = exceed_payload_max_length ? 413 : 400; } + if (!ret) { + status = exceed_payload_max_length ? StatusCode::PayloadTooLarge_413 + : StatusCode::BadRequest_400; + } return ret; }); } // namespace detail @@ -3720,6 +4087,8 @@ inline bool write_content(Stream &strm, const ContentProvider &content_provider, return ok; }; + data_sink.is_writable = [&]() -> bool { return strm.is_writable(); }; + while (offset < end_offset && !is_shutting_down()) { if (!strm.is_writable()) { error = Error::Write; @@ -3764,6 +4133,8 @@ write_content_without_length(Stream &strm, return ok; }; + data_sink.is_writable = [&]() -> bool { return strm.is_writable(); }; + data_sink.done = [&](void) { data_available = false; }; while (data_available && !is_shutting_down()) { @@ -3814,6 +4185,8 @@ write_content_chunked(Stream &strm, const ContentProvider &content_provider, return ok; }; + data_sink.is_writable = [&]() -> bool { return strm.is_writable(); }; + auto done_with_trailer = [&](const Headers *trailer) { if (!ok) { return; } @@ -3898,7 +4271,8 @@ inline bool redirect(T &cli, Request &req, Response &res, new_req.path = path; new_req.redirect_count_ -= 1; - if (res.status == 303 && (req.method != "GET" && req.method != "HEAD")) { + if (res.status == StatusCode::SeeOther_303 && + (req.method != "GET" && req.method != "HEAD")) { new_req.method = "GET"; new_req.body.clear(); new_req.headers.clear(); @@ -3910,7 +4284,8 @@ inline bool redirect(T &cli, Request &req, Response &res, if (ret) { req = new_req; res = new_res; - res.location = location; + + if (res.location.empty()) { res.location = location; } } return ret; } @@ -3957,14 +4332,34 @@ inline bool parse_multipart_boundary(const std::string &content_type, if (pos == std::string::npos) { return false; } auto end = content_type.find(';', pos); auto beg = pos + strlen(boundary_keyword); - boundary = content_type.substr(beg, end - beg); - if (boundary.length() >= 2 && boundary.front() == '"' && - boundary.back() == '"') { - boundary = boundary.substr(1, boundary.size() - 2); - } + boundary = trim_double_quotes_copy(content_type.substr(beg, end - beg)); return !boundary.empty(); } +inline void parse_disposition_params(const std::string &s, Params ¶ms) { + std::set cache; + split(s.data(), s.data() + s.size(), ';', [&](const char *b, const char *e) { + std::string kv(b, e); + if (cache.find(kv) != cache.end()) { return; } + cache.insert(kv); + + std::string key; + std::string val; + split(b, e, '=', [&](const char *b2, const char *e2) { + if (key.empty()) { + key.assign(b2, e2); + } else { + val.assign(b2, e2); + } + }); + + if (!key.empty()) { + params.emplace(trim_double_quotes_copy((key)), + trim_double_quotes_copy((val))); + } + }); +} + #ifdef CPPHTTPLIB_NO_EXCEPTIONS inline bool parse_range_header(const std::string &s, Ranges &ranges) { #else @@ -3975,9 +4370,9 @@ inline bool parse_range_header(const std::string &s, Ranges &ranges) try { if (std::regex_match(s, m, re_first_range)) { auto pos = static_cast(m.position(1)); auto len = static_cast(m.length(1)); - bool all_valid_ranges = true; + auto all_valid_ranges = true; split(&s[pos], &s[pos + len], ',', [&](const char *b, const char *e) { - if (!all_valid_ranges) return; + if (!all_valid_ranges) { return; } static auto re_another_range = std::regex(R"(\s*(\d*)-(\d*))"); std::cmatch cm; if (std::regex_match(b, e, cm, re_another_range)) { @@ -4022,11 +4417,6 @@ public: bool parse(const char *buf, size_t n, const ContentReceiver &content_callback, const MultipartContentHeader &header_callback) { - // TODO: support 'filename*' - static const std::regex re_content_disposition( - R"~(^Content-Disposition:\s*form-data;\s*name="(.*?)"(?:;\s*filename="(.*?)")?(?:;\s*filename\*=\S+)?\s*$)~", - std::regex_constants::icase); - buf_append(buf, n); while (buf_size() > 0) { @@ -4059,18 +4449,54 @@ public: break; } - static const std::string header_name = "content-type:"; const auto header = buf_head(pos); - if (start_with_case_ignore(header, header_name)) { - file_.content_type = trim_copy(header.substr(header_name.size())); + + if (!parse_header(header.data(), header.data() + header.size(), + [&](std::string &&, std::string &&) {})) { + is_valid_ = false; + return false; + } + + static const std::string header_content_type = "Content-Type:"; + + if (start_with_case_ignore(header, header_content_type)) { + file_.content_type = + trim_copy(header.substr(header_content_type.size())); } else { + static const std::regex re_content_disposition( + R"~(^Content-Disposition:\s*form-data;\s*(.*)$)~", + std::regex_constants::icase); + std::smatch m; if (std::regex_match(header, m, re_content_disposition)) { - file_.name = m[1]; - file_.filename = m[2]; - } else { - is_valid_ = false; - return false; + Params params; + parse_disposition_params(m[1], params); + + auto it = params.find("name"); + if (it != params.end()) { + file_.name = it->second; + } else { + is_valid_ = false; + return false; + } + + it = params.find("filename"); + if (it != params.end()) { file_.filename = it->second; } + + it = params.find("filename*"); + if (it != params.end()) { + // Only allow UTF-8 enconnding... + static const std::regex re_rfc5987_encoding( + R"~(^UTF-8''(.+?)$)~", std::regex_constants::icase); + + std::smatch m2; + if (std::regex_match(it->second, m2, re_rfc5987_encoding)) { + file_.filename = decode_url(m2[1], false); // override... + } else { + is_valid_ = false; + return false; + } + } } } buf_erase(pos + crlf_.size()); @@ -4108,9 +4534,9 @@ public: buf_erase(crlf_.size()); state_ = 1; } else { - if (dash_crlf_.size() > buf_size()) { return true; } - if (buf_start_with(dash_crlf_)) { - buf_erase(dash_crlf_.size()); + if (dash_.size() > buf_size()) { return true; } + if (buf_start_with(dash_)) { + buf_erase(dash_.size()); is_valid_ = true; buf_erase(buf_size()); // Remove epilogue } else { @@ -4143,7 +4569,6 @@ private: const std::string dash_ = "--"; const std::string crlf_ = "\r\n"; - const std::string dash_crlf_ = "--\r\n"; std::string boundary_; std::string dash_boundary_crlf_; std::string crlf_dash_boundary_; @@ -4230,28 +4655,32 @@ inline std::string to_lower(const char *beg, const char *end) { return out; } -inline std::string make_multipart_data_boundary() { +inline std::string random_string(size_t length) { static const char data[] = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"; // std::random_device might actually be deterministic on some // platforms, but due to lack of support in the c++ standard library, // doing better requires either some ugly hacks or breaking portability. - std::random_device seed_gen; + static std::random_device seed_gen; // Request 128 bits of entropy for initialization - std::seed_seq seed_sequence{seed_gen(), seed_gen(), seed_gen(), seed_gen()}; - std::mt19937 engine(seed_sequence); + static std::seed_seq seed_sequence{seed_gen(), seed_gen(), seed_gen(), + seed_gen()}; - std::string result = "--cpp-httplib-multipart-data-"; + static std::mt19937 engine(seed_sequence); - for (auto i = 0; i < 16; i++) { + std::string result; + for (size_t i = 0; i < length; i++) { result += data[engine() % (sizeof(data) - 1)]; } - return result; } +inline std::string make_multipart_data_boundary() { + return "--cpp-httplib-multipart-data-" + detail::random_string(16); +} + inline bool is_multipart_boundary_chars_valid(const std::string &boundary) { auto valid = true; for (size_t i = 0; i < boundary.size(); i++) { @@ -4304,48 +4733,97 @@ serialize_multipart_formdata(const MultipartFormDataItems &items, body += item.content + serialize_multipart_formdata_item_end(); } - if (finish) body += serialize_multipart_formdata_finish(boundary); + if (finish) { body += serialize_multipart_formdata_finish(boundary); } return body; } +inline bool range_error(Request &req, Response &res) { + if (!req.ranges.empty() && 200 <= res.status && res.status < 300) { + ssize_t contant_len = static_cast( + res.content_length_ ? res.content_length_ : res.body.size()); + + ssize_t prev_first_pos = -1; + ssize_t prev_last_pos = -1; + size_t overwrapping_count = 0; + + // NOTE: The following Range check is based on '14.2. Range' in RFC 9110 + // 'HTTP Semantics' to avoid potential denial-of-service attacks. + // https://www.rfc-editor.org/rfc/rfc9110#section-14.2 + + // Too many ranges + if (req.ranges.size() > CPPHTTPLIB_RANGE_MAX_COUNT) { return true; } + + for (auto &r : req.ranges) { + auto &first_pos = r.first; + auto &last_pos = r.second; + + if (first_pos == -1 && last_pos == -1) { + first_pos = 0; + last_pos = contant_len; + } + + if (first_pos == -1) { + first_pos = contant_len - last_pos; + last_pos = contant_len - 1; + } + + if (last_pos == -1) { last_pos = contant_len - 1; } + + // Range must be within content length + if (!(0 <= first_pos && first_pos <= last_pos && + last_pos <= contant_len - 1)) { + return true; + } + + // Ranges must be in ascending order + if (first_pos <= prev_first_pos) { return true; } + + // Request must not have more than two overlapping ranges + if (first_pos <= prev_last_pos) { + overwrapping_count++; + if (overwrapping_count > 2) { return true; } + } + + prev_first_pos = (std::max)(prev_first_pos, first_pos); + prev_last_pos = (std::max)(prev_last_pos, last_pos); + } + } + + return false; +} + inline std::pair -get_range_offset_and_length(const Request &req, size_t content_length, - size_t index) { - auto r = req.ranges[index]; +get_range_offset_and_length(Range r, size_t content_length) { + (void)(content_length); // patch to get rid of "unused parameter" on release build + assert(r.first != -1 && r.second != -1); + assert(0 <= r.first && r.first < static_cast(content_length)); + assert(r.first <= r.second && + r.second < static_cast(content_length)); - if (r.first == -1 && r.second == -1) { - return std::make_pair(0, content_length); - } - - auto slen = static_cast(content_length); - - if (r.first == -1) { - r.first = (std::max)(static_cast(0), slen - r.second); - r.second = slen - 1; - } - - if (r.second == -1) { r.second = slen - 1; } return std::make_pair(r.first, static_cast(r.second - r.first) + 1); } -inline std::string make_content_range_header_field(size_t offset, size_t length, - size_t content_length) { +inline std::string make_content_range_header_field( + const std::pair &offset_and_length, size_t content_length) { + auto st = offset_and_length.first; + auto ed = st + offset_and_length.second - 1; + std::string field = "bytes "; - field += std::to_string(offset); + field += std::to_string(st); field += "-"; - field += std::to_string(offset + length - 1); + field += std::to_string(ed); field += "/"; field += std::to_string(content_length); return field; } template -bool process_multipart_ranges_data(const Request &req, Response &res, +bool process_multipart_ranges_data(const Request &req, const std::string &boundary, const std::string &content_type, - SToken stoken, CToken ctoken, - Content content) { + size_t content_length, SToken stoken, + CToken ctoken, Content content) { for (size_t i = 0; i < req.ranges.size(); i++) { ctoken("--"); stoken(boundary); @@ -4356,50 +4834,51 @@ bool process_multipart_ranges_data(const Request &req, Response &res, ctoken("\r\n"); } - auto offsets = get_range_offset_and_length(req, res.body.size(), i); - auto offset = offsets.first; - auto length = offsets.second; + auto offset_and_length = + get_range_offset_and_length(req.ranges[i], content_length); ctoken("Content-Range: "); - stoken(make_content_range_header_field(offset, length, res.body.size())); + stoken(make_content_range_header_field(offset_and_length, content_length)); ctoken("\r\n"); ctoken("\r\n"); - if (!content(offset, length)) { return false; } + + if (!content(offset_and_length.first, offset_and_length.second)) { + return false; + } ctoken("\r\n"); } ctoken("--"); stoken(boundary); - ctoken("--\r\n"); + ctoken("--"); return true; } -inline bool make_multipart_ranges_data(const Request &req, Response &res, +inline void make_multipart_ranges_data(const Request &req, Response &res, const std::string &boundary, const std::string &content_type, + size_t content_length, std::string &data) { - return process_multipart_ranges_data( - req, res, boundary, content_type, + process_multipart_ranges_data( + req, boundary, content_type, content_length, [&](const std::string &token) { data += token; }, [&](const std::string &token) { data += token; }, [&](size_t offset, size_t length) { - if (offset < res.body.size()) { - data += res.body.substr(offset, length); - return true; - } - return false; + assert(offset + length <= content_length); + data += res.body.substr(offset, length); + return true; }); } -inline size_t -get_multipart_ranges_data_length(const Request &req, Response &res, - const std::string &boundary, - const std::string &content_type) { +inline size_t get_multipart_ranges_data_length(const Request &req, + const std::string &boundary, + const std::string &content_type, + size_t content_length) { size_t data_length = 0; process_multipart_ranges_data( - req, res, boundary, content_type, + req, boundary, content_type, content_length, [&](const std::string &token) { data_length += token.size(); }, [&](const std::string &token) { data_length += token.size(); }, [&](size_t /*offset*/, size_t length) { @@ -4411,13 +4890,13 @@ get_multipart_ranges_data_length(const Request &req, Response &res, } template -inline bool write_multipart_ranges_data(Stream &strm, const Request &req, - Response &res, - const std::string &boundary, - const std::string &content_type, - const T &is_shutting_down) { +inline bool +write_multipart_ranges_data(Stream &strm, const Request &req, Response &res, + const std::string &boundary, + const std::string &content_type, + size_t content_length, const T &is_shutting_down) { return process_multipart_ranges_data( - req, res, boundary, content_type, + req, boundary, content_type, content_length, [&](const std::string &token) { strm.write(token); }, [&](const std::string &token) { strm.write(token); }, [&](size_t offset, size_t length) { @@ -4426,18 +4905,6 @@ inline bool write_multipart_ranges_data(Stream &strm, const Request &req, }); } -inline std::pair -get_range_offset_and_length(const Request &req, const Response &res, - size_t index) { - auto r = req.ranges[index]; - - if (r.second == -1) { - r.second = static_cast(res.content_length_) - 1; - } - - return std::make_pair(r.first, r.second - r.first + 1); -} - inline bool expect_content(const Request &req) { if (req.method == "POST" || req.method == "PUT" || req.method == "PATCH" || req.method == "PRI" || req.method == "DELETE") { @@ -4471,7 +4938,7 @@ inline std::string message_digest(const std::string &s, const EVP_MD *algo) { std::stringstream ss; for (auto i = 0u; i < hash_length; ++i) { ss << std::hex << std::setw(2) << std::setfill('0') - << (unsigned int)hash[i]; + << static_cast(hash[i]); } return ss.str(); @@ -4564,7 +5031,7 @@ inline bool retrieve_root_certs_from_keychain(CFObjectPtr &certs) { inline bool add_certs_to_x509_store(CFArrayRef certs, X509_STORE *store) { auto result = false; - for (int i = 0; i < CFArrayGetCount(certs); ++i) { + for (auto i = 0; i < CFArrayGetCount(certs); ++i) { const auto cert = reinterpret_cast( CFArrayGetValueAtIndex(certs, i)); @@ -4707,7 +5174,7 @@ inline bool parse_www_authenticate(const Response &res, s = s.substr(pos + 1); auto beg = std::sregex_iterator(s.begin(), s.end(), re); for (auto i = beg; i != std::sregex_iterator(); ++i) { - auto m = *i; + const auto &m = *i; auto key = s.substr(static_cast(m.position(1)), static_cast(m.length(1))); auto val = m.length(2) > 0 @@ -4724,20 +5191,6 @@ inline bool parse_www_authenticate(const Response &res, return false; } -// https://stackoverflow.com/questions/440133/how-do-i-create-a-random-alpha-numeric-string-in-c/440240#answer-440240 -inline std::string random_string(size_t length) { - auto randchar = []() -> char { - const char charset[] = "0123456789" - "ABCDEFGHIJKLMNOPQRSTUVWXYZ" - "abcdefghijklmnopqrstuvwxyz"; - const size_t max_index = (sizeof(charset) - 1); - return charset[static_cast(std::rand()) % max_index]; - }; - std::string str(length, 0); - std::generate_n(str.begin(), length, randchar); - return str; -} - class ContentProviderAdapter { public: explicit ContentProviderAdapter( @@ -4782,7 +5235,7 @@ inline void hosted_at(const std::string &hostname, const auto &addr = *reinterpret_cast(rp->ai_addr); std::string ip; - int dummy = -1; + auto dummy = -1; if (detail::get_ip_and_port(addr, sizeof(struct sockaddr_storage), ip, dummy)) { addrs.push_back(ip); @@ -4802,10 +5255,11 @@ inline std::string append_query_params(const std::string &path, } // Header utilities -inline std::pair make_range_header(Ranges ranges) { +inline std::pair +make_range_header(const Ranges &ranges) { std::string field = "bytes="; auto i = 0; - for (auto r : ranges) { + for (const auto &r : ranges) { if (i != 0) { field += ", "; } if (r.first != -1) { field += std::to_string(r.first); } field += '-'; @@ -4924,7 +5378,7 @@ inline void Response::set_redirect(const std::string &url, int stat) { if (300 <= stat && stat < 400) { this->status = stat; } else { - this->status = 302; + this->status = StatusCode::Found_302; } } } @@ -4943,13 +5397,22 @@ inline void Response::set_content(const std::string &s, set_content(s.data(), s.size(), content_type); } +inline void Response::set_content(std::string &&s, + const std::string &content_type) { + body = std::move(s); + + auto rng = headers.equal_range("Content-Type"); + headers.erase(rng.first, rng.second); + set_header("Content-Type", content_type); +} + inline void Response::set_content_provider( size_t in_length, const std::string &content_type, ContentProvider provider, ContentProviderResourceReleaser resource_releaser) { set_header("Content-Type", content_type); content_length_ = in_length; if (in_length > 0) { content_provider_ = std::move(provider); } - content_provider_resource_releaser_ = resource_releaser; + content_provider_resource_releaser_ = std::move(resource_releaser); is_chunked_content_provider_ = false; } @@ -4959,7 +5422,7 @@ inline void Response::set_content_provider( set_header("Content-Type", content_type); content_length_ = 0; content_provider_ = detail::ContentProviderAdapter(std::move(provider)); - content_provider_resource_releaser_ = resource_releaser; + content_provider_resource_releaser_ = std::move(resource_releaser); is_chunked_content_provider_ = false; } @@ -4969,7 +5432,7 @@ inline void Response::set_chunked_content_provider( set_header("Content-Type", content_type); content_length_ = 0; content_provider_ = detail::ContentProviderAdapter(std::move(provider)); - content_provider_resource_releaser_ = resource_releaser; + content_provider_resource_releaser_ = std::move(resource_releaser); is_chunked_content_provider_ = true; } @@ -5010,7 +5473,7 @@ inline SocketStream::SocketStream(socket_t sock, time_t read_timeout_sec, write_timeout_sec_(write_timeout_sec), write_timeout_usec_(write_timeout_usec), read_buff_(read_buff_size_, 0) {} -inline SocketStream::~SocketStream() {} +inline SocketStream::~SocketStream() = default; inline bool SocketStream::is_readable() const { return select_read(sock_, read_timeout_sec_, read_timeout_usec_) > 0; @@ -5120,6 +5583,99 @@ inline socket_t BufferStream::socket() const { return 0; } inline const std::string &BufferStream::get_buffer() const { return buffer; } +inline PathParamsMatcher::PathParamsMatcher(const std::string &pattern) { + // One past the last ending position of a path param substring + std::size_t last_param_end = 0; + +#ifndef CPPHTTPLIB_NO_EXCEPTIONS + // Needed to ensure that parameter names are unique during matcher + // construction + // If exceptions are disabled, only last duplicate path + // parameter will be set + std::unordered_set param_name_set; +#endif + + while (true) { + const auto marker_pos = pattern.find(marker, last_param_end); + if (marker_pos == std::string::npos) { break; } + + static_fragments_.push_back( + pattern.substr(last_param_end, marker_pos - last_param_end)); + + const auto param_name_start = marker_pos + 1; + + auto sep_pos = pattern.find(separator, param_name_start); + if (sep_pos == std::string::npos) { sep_pos = pattern.length(); } + + auto param_name = + pattern.substr(param_name_start, sep_pos - param_name_start); + +#ifndef CPPHTTPLIB_NO_EXCEPTIONS + if (param_name_set.find(param_name) != param_name_set.cend()) { + std::string msg = "Encountered path parameter '" + param_name + + "' multiple times in route pattern '" + pattern + "'."; + throw std::invalid_argument(msg); + } +#endif + + param_names_.push_back(std::move(param_name)); + + last_param_end = sep_pos + 1; + } + + if (last_param_end < pattern.length()) { + static_fragments_.push_back(pattern.substr(last_param_end)); + } +} + +inline bool PathParamsMatcher::match(Request &request) const { + request.matches = std::smatch(); + request.path_params.clear(); + request.path_params.reserve(param_names_.size()); + + // One past the position at which the path matched the pattern last time + std::size_t starting_pos = 0; + for (size_t i = 0; i < static_fragments_.size(); ++i) { + const auto &fragment = static_fragments_[i]; + + if (starting_pos + fragment.length() > request.path.length()) { + return false; + } + + // Avoid unnecessary allocation by using strncmp instead of substr + + // comparison + if (std::strncmp(request.path.c_str() + starting_pos, fragment.c_str(), + fragment.length()) != 0) { + return false; + } + + starting_pos += fragment.length(); + + // Should only happen when we have a static fragment after a param + // Example: '/users/:id/subscriptions' + // The 'subscriptions' fragment here does not have a corresponding param + if (i >= param_names_.size()) { continue; } + + auto sep_pos = request.path.find(separator, starting_pos); + if (sep_pos == std::string::npos) { sep_pos = request.path.length(); } + + const auto ¶m_name = param_names_[i]; + + request.path_params.emplace( + param_name, request.path.substr(starting_pos, sep_pos - starting_pos)); + + // Mark everythin up to '/' as matched + starting_pos = sep_pos + 1; + } + // Returns false if the path is longer than the pattern + return starting_pos >= request.path.length(); +} + +inline bool RegexMatcher::match(Request &request) const { + request.path_params.clear(); + return std::regex_match(request.path, request.matches, regex_); +} + } // namespace detail // HTTP server implementation @@ -5131,69 +5687,72 @@ inline Server::Server() #endif } -inline Server::~Server() {} +inline Server::~Server() = default; + +inline std::unique_ptr +Server::make_matcher(const std::string &pattern) { + if (pattern.find("/:") != std::string::npos) { + return detail::make_unique(pattern); + } else { + return detail::make_unique(pattern); + } +} inline Server &Server::Get(const std::string &pattern, Handler handler) { - get_handlers_.push_back( - std::make_pair(std::regex(pattern), std::move(handler))); + get_handlers_.emplace_back(make_matcher(pattern), std::move(handler)); return *this; } inline Server &Server::Post(const std::string &pattern, Handler handler) { - post_handlers_.push_back( - std::make_pair(std::regex(pattern), std::move(handler))); + post_handlers_.emplace_back(make_matcher(pattern), std::move(handler)); return *this; } inline Server &Server::Post(const std::string &pattern, HandlerWithContentReader handler) { - post_handlers_for_content_reader_.push_back( - std::make_pair(std::regex(pattern), std::move(handler))); + post_handlers_for_content_reader_.emplace_back(make_matcher(pattern), + std::move(handler)); return *this; } inline Server &Server::Put(const std::string &pattern, Handler handler) { - put_handlers_.push_back( - std::make_pair(std::regex(pattern), std::move(handler))); + put_handlers_.emplace_back(make_matcher(pattern), std::move(handler)); return *this; } inline Server &Server::Put(const std::string &pattern, HandlerWithContentReader handler) { - put_handlers_for_content_reader_.push_back( - std::make_pair(std::regex(pattern), std::move(handler))); + put_handlers_for_content_reader_.emplace_back(make_matcher(pattern), + std::move(handler)); return *this; } inline Server &Server::Patch(const std::string &pattern, Handler handler) { - patch_handlers_.push_back( - std::make_pair(std::regex(pattern), std::move(handler))); + patch_handlers_.emplace_back(make_matcher(pattern), std::move(handler)); return *this; } inline Server &Server::Patch(const std::string &pattern, HandlerWithContentReader handler) { - patch_handlers_for_content_reader_.push_back( - std::make_pair(std::regex(pattern), std::move(handler))); + patch_handlers_for_content_reader_.emplace_back(make_matcher(pattern), + std::move(handler)); return *this; } inline Server &Server::Delete(const std::string &pattern, Handler handler) { - delete_handlers_.push_back( - std::make_pair(std::regex(pattern), std::move(handler))); + delete_handlers_.emplace_back(make_matcher(pattern), std::move(handler)); return *this; } inline Server &Server::Delete(const std::string &pattern, HandlerWithContentReader handler) { - delete_handlers_for_content_reader_.push_back( - std::make_pair(std::regex(pattern), std::move(handler))); + delete_handlers_for_content_reader_.emplace_back(make_matcher(pattern), + std::move(handler)); return *this; } inline Server &Server::Options(const std::string &pattern, Handler handler) { - options_handlers_.push_back( - std::make_pair(std::regex(pattern), std::move(handler))); + options_handlers_.emplace_back(make_matcher(pattern), std::move(handler)); return *this; } @@ -5231,6 +5790,11 @@ Server::set_file_extension_and_mimetype_mapping(const std::string &ext, return *this; } +inline Server &Server::set_default_file_mimetype(const std::string &mime) { + default_file_mimetype_ = mime; + return *this; +} + inline Server &Server::set_file_request_handler(Handler handler) { file_request_handler_ = std::move(handler); return *this; @@ -5272,7 +5836,6 @@ inline Server &Server::set_logger(Logger logger) { inline Server & Server::set_expect_100_continue_handler(Expect100ContinueHandler handler) { expect_100_continue_handler_ = std::move(handler); - return *this; } @@ -5296,6 +5859,12 @@ inline Server &Server::set_default_headers(Headers headers) { return *this; } +inline Server &Server::set_header_writer( + std::function const &writer) { + header_writer_ = writer; + return *this; +} + inline Server &Server::set_keep_alive_max_count(size_t count) { keep_alive_max_count_ = count; return *this; @@ -5331,8 +5900,7 @@ inline Server &Server::set_payload_max_length(size_t length) { inline bool Server::bind_to_port(const std::string &host, int port, int socket_flags) { - if (bind_internal(host, port, socket_flags) < 0) return false; - return true; + return bind_internal(host, port, socket_flags) >= 0; } inline int Server::bind_to_any_port(const std::string &host, int socket_flags) { return bind_internal(host, 0, socket_flags); @@ -5366,7 +5934,7 @@ inline void Server::stop() { } } -inline bool Server::parse_request_line(const char *s, Request &req) { +inline bool Server::parse_request_line(const char *s, Request &req) const { auto len = strlen(s); if (len < 2 || s[len - 2] != '\r' || s[len - 1] != '\n') { return false; } len -= 2; @@ -5407,7 +5975,7 @@ inline bool Server::parse_request_line(const char *s, Request &req) { size_t count = 0; detail::split(req.target.data(), req.target.data() + req.target.size(), '?', - [&](const char *b, const char *e) { + 2, [&](const char *b, const char *e) { switch (count) { case 0: req.path = detail::decode_url(std::string(b, e), false); @@ -5430,7 +5998,10 @@ inline bool Server::parse_request_line(const char *s, Request &req) { } inline bool Server::write_response(Stream &strm, bool close_connection, - const Request &req, Response &res) { + Request &req, Response &res) { + // NOTE: `req.ranges` should be empty, otherwise it will be applied + // incorrectly to the error content. + req.ranges.clear(); return write_response_core(strm, close_connection, req, res, false); } @@ -5486,11 +6057,11 @@ inline bool Server::write_response_core(Stream &strm, bool close_connection, detail::BufferStream bstrm; if (!bstrm.write_format("HTTP/1.1 %d %s\r\n", res.status, - detail::status_message(res.status))) { + status_message(res.status))) { return false; } - if (!detail::write_headers(bstrm, res.headers)) { return false; } + if (!header_writer_(bstrm, res.headers)) { return false; } // Flush buffer auto &data = bstrm.get_buffer(); @@ -5508,7 +6079,6 @@ inline bool Server::write_response_core(Stream &strm, bool close_connection, if (write_content_with_provider(strm, req, res, boundary, content_type)) { res.content_provider_success_ = true; } else { - res.content_provider_success_ = false; ret = false; } } @@ -5533,15 +6103,16 @@ Server::write_content_with_provider(Stream &strm, const Request &req, return detail::write_content(strm, res.content_provider_, 0, res.content_length_, is_shutting_down); } else if (req.ranges.size() == 1) { - auto offsets = - detail::get_range_offset_and_length(req, res.content_length_, 0); - auto offset = offsets.first; - auto length = offsets.second; - return detail::write_content(strm, res.content_provider_, offset, length, - is_shutting_down); + auto offset_and_length = detail::get_range_offset_and_length( + req.ranges[0], res.content_length_); + + return detail::write_content(strm, res.content_provider_, + offset_and_length.first, + offset_and_length.second, is_shutting_down); } else { return detail::write_multipart_ranges_data( - strm, req, res, boundary, content_type, is_shutting_down); + strm, req, res, boundary, content_type, res.content_length_, + is_shutting_down); } } else { if (res.is_chunked_content_provider_) { @@ -5598,7 +6169,7 @@ inline bool Server::read_content(Stream &strm, Request &req, Response &res) { const auto &content_type = req.get_header_value("Content-Type"); if (!content_type.find("application/x-www-form-urlencoded")) { if (req.body.size() > CPPHTTPLIB_FORM_URL_ENCODED_PAYLOAD_MAX_LENGTH) { - res.status = 413; // NOTE: should be 414? + res.status = StatusCode::PayloadTooLarge_413; // NOTE: should be 414? return false; } detail::parse_query_text(req.body, req.params); @@ -5617,10 +6188,11 @@ inline bool Server::read_content_with_content_receiver( std::move(multipart_receiver)); } -inline bool Server::read_content_core(Stream &strm, Request &req, Response &res, - ContentReceiver receiver, - MultipartContentHeader multipart_header, - ContentReceiver multipart_receiver) { +inline bool +Server::read_content_core(Stream &strm, Request &req, Response &res, + ContentReceiver receiver, + MultipartContentHeader multipart_header, + ContentReceiver multipart_receiver) const { detail::MultipartFormDataParser multipart_form_data_parser; ContentReceiverWithProgress out; @@ -5628,7 +6200,7 @@ inline bool Server::read_content_core(Stream &strm, Request &req, Response &res, const auto &content_type = req.get_header_value("Content-Type"); std::string boundary; if (!detail::parse_multipart_boundary(content_type, boundary)) { - res.status = 400; + res.status = StatusCode::BadRequest_400; return false; } @@ -5664,7 +6236,7 @@ inline bool Server::read_content_core(Stream &strm, Request &req, Response &res, if (req.is_multipart_form_data()) { if (!multipart_form_data_parser.is_valid()) { - res.status = 400; + res.status = StatusCode::BadRequest_400; return false; } } @@ -5683,17 +6255,26 @@ inline bool Server::handle_file_request(const Request &req, Response &res, if (path.back() == '/') { path += "index.html"; } if (detail::is_file(path)) { - detail::read_file(path, res.body); - auto type = - detail::find_content_type(path, file_extension_and_mimetype_map_); - if (type) { res.set_header("Content-Type", type); } for (const auto &kv : entry.headers) { - res.set_header(kv.first.c_str(), kv.second); + res.set_header(kv.first, kv.second); } - res.status = req.has_header("Range") ? 206 : 200; + + auto mm = std::make_shared(path.c_str()); + if (!mm->is_open()) { return false; } + + res.set_content_provider( + mm->size(), + detail::find_content_type(path, file_extension_and_mimetype_map_, + default_file_mimetype_), + [mm](size_t offset, size_t length, DataSink &sink) -> bool { + sink.write(mm->data() + offset, length); + return true; + }); + if (!head && file_request_handler_) { file_request_handler_(req, res); } + return true; } } @@ -5789,13 +6370,14 @@ inline bool Server::listen_internal() { #ifdef _WIN32 auto timeout = static_cast(read_timeout_sec_ * 1000 + read_timeout_usec_ / 1000); - setsockopt(sock, SOL_SOCKET, SO_RCVTIMEO, (char *)&timeout, - sizeof(timeout)); + setsockopt(sock, SOL_SOCKET, SO_RCVTIMEO, + reinterpret_cast(&timeout), sizeof(timeout)); #else timeval tv; tv.tv_sec = static_cast(read_timeout_sec_); tv.tv_usec = static_cast(read_timeout_usec_); - setsockopt(sock, SOL_SOCKET, SO_RCVTIMEO, (char *)&tv, sizeof(tv)); + setsockopt(sock, SOL_SOCKET, SO_RCVTIMEO, + reinterpret_cast(&tv), sizeof(tv)); #endif } { @@ -5803,17 +6385,22 @@ inline bool Server::listen_internal() { #ifdef _WIN32 auto timeout = static_cast(write_timeout_sec_ * 1000 + write_timeout_usec_ / 1000); - setsockopt(sock, SOL_SOCKET, SO_SNDTIMEO, (char *)&timeout, - sizeof(timeout)); + setsockopt(sock, SOL_SOCKET, SO_SNDTIMEO, + reinterpret_cast(&timeout), sizeof(timeout)); #else timeval tv; tv.tv_sec = static_cast(write_timeout_sec_); tv.tv_usec = static_cast(write_timeout_usec_); - setsockopt(sock, SOL_SOCKET, SO_SNDTIMEO, (char *)&tv, sizeof(tv)); + setsockopt(sock, SOL_SOCKET, SO_SNDTIMEO, + reinterpret_cast(&tv), sizeof(tv)); #endif } - task_queue->enqueue([this, sock]() { process_and_close_socket(sock); }); + if (!task_queue->enqueue( + [this, sock]() { process_and_close_socket(sock); })) { + detail::shutdown_socket(sock); + detail::close_socket(sock); + } } task_queue->shutdown(); @@ -5829,7 +6416,7 @@ inline bool Server::routing(Request &req, Response &res, Stream &strm) { } // File handler - bool is_head_request = req.method == "HEAD"; + auto is_head_request = req.method == "HEAD"; if ((req.method == "GET" || is_head_request) && handle_file_request(req, res, is_head_request)) { return true; @@ -5895,17 +6482,17 @@ inline bool Server::routing(Request &req, Response &res, Stream &strm) { return dispatch_request(req, res, patch_handlers_); } - res.status = 400; + res.status = StatusCode::BadRequest_400; return false; } inline bool Server::dispatch_request(Request &req, Response &res, - const Handlers &handlers) { + const Handlers &handlers) const { for (const auto &x : handlers) { - const auto &pattern = x.first; + const auto &matcher = x.first; const auto &handler = x.second; - if (std::regex_match(req.path, req.matches, pattern)) { + if (matcher->match(req)) { handler(req, res); return true; } @@ -5915,18 +6502,18 @@ inline bool Server::dispatch_request(Request &req, Response &res, inline void Server::apply_ranges(const Request &req, Response &res, std::string &content_type, - std::string &boundary) { + std::string &boundary) const { if (req.ranges.size() > 1) { - boundary = detail::make_multipart_data_boundary(); - auto it = res.headers.find("Content-Type"); if (it != res.headers.end()) { content_type = it->second; res.headers.erase(it); } - res.headers.emplace("Content-Type", - "multipart/byteranges; boundary=" + boundary); + boundary = detail::make_multipart_data_boundary(); + + res.set_header("Content-Type", + "multipart/byteranges; boundary=" + boundary); } auto type = detail::encoding_type(req, res); @@ -5937,16 +6524,17 @@ inline void Server::apply_ranges(const Request &req, Response &res, if (req.ranges.empty()) { length = res.content_length_; } else if (req.ranges.size() == 1) { - auto offsets = - detail::get_range_offset_and_length(req, res.content_length_, 0); - auto offset = offsets.first; - length = offsets.second; + auto offset_and_length = detail::get_range_offset_and_length( + req.ranges[0], res.content_length_); + + length = offset_and_length.second; + auto content_range = detail::make_content_range_header_field( - offset, length, res.content_length_); + offset_and_length, res.content_length_); res.set_header("Content-Range", content_range); } else { - length = detail::get_multipart_ranges_data_length(req, res, boundary, - content_type); + length = detail::get_multipart_ranges_data_length( + req, boundary, content_type, res.content_length_); } res.set_header("Content-Length", std::to_string(length)); } else { @@ -5965,28 +6553,22 @@ inline void Server::apply_ranges(const Request &req, Response &res, if (req.ranges.empty()) { ; } else if (req.ranges.size() == 1) { - auto offsets = - detail::get_range_offset_and_length(req, res.body.size(), 0); - auto offset = offsets.first; - auto length = offsets.second; + auto offset_and_length = + detail::get_range_offset_and_length(req.ranges[0], res.body.size()); + auto offset = offset_and_length.first; + auto length = offset_and_length.second; + auto content_range = detail::make_content_range_header_field( - offset, length, res.body.size()); + offset_and_length, res.body.size()); res.set_header("Content-Range", content_range); - if (offset < res.body.size()) { - res.body = res.body.substr(offset, length); - } else { - res.body.clear(); - res.status = 416; - } + + assert(offset + length <= res.body.size()); + res.body = res.body.substr(offset, length); } else { std::string data; - if (detail::make_multipart_ranges_data(req, res, boundary, content_type, - data)) { - res.body.swap(data); - } else { - res.body.clear(); - res.status = 416; - } + detail::make_multipart_ranges_data(req, res, boundary, content_type, + res.body.size(), data); + res.body.swap(data); } if (type != detail::EncodingType::None) { @@ -6025,12 +6607,12 @@ inline void Server::apply_ranges(const Request &req, Response &res, inline bool Server::dispatch_request_for_content_reader( Request &req, Response &res, ContentReader content_reader, - const HandlersForContentReader &handlers) { + const HandlersForContentReader &handlers) const { for (const auto &x : handlers) { - const auto &pattern = x.first; + const auto &matcher = x.first; const auto &handler = x.second; - if (std::regex_match(req.path, req.matches, pattern)) { + if (matcher->match(req)) { handler(req, res, content_reader); return true; } @@ -6050,15 +6632,10 @@ Server::process_request(Stream &strm, bool close_connection, if (!line_reader.getline()) { return false; } Request req; + Response res; - res.version = "HTTP/1.1"; - - for (const auto &header : default_headers_) { - if (res.headers.find(header.first) == res.headers.end()) { - res.headers.insert(header); - } - } + res.headers = default_headers_; #ifdef _WIN32 // TODO: Increase FD_SETSIZE statically (libzmq), dynamically (MySQL). @@ -6068,7 +6645,7 @@ Server::process_request(Stream &strm, bool close_connection, if (strm.socket() >= FD_SETSIZE) { Headers dummy; detail::read_headers(strm, dummy); - res.status = 500; + res.status = StatusCode::InternalServerError_500; return write_response(strm, close_connection, req, res); } #endif @@ -6078,14 +6655,14 @@ Server::process_request(Stream &strm, bool close_connection, if (line_reader.size() > CPPHTTPLIB_REQUEST_URI_MAX_LENGTH) { Headers dummy; detail::read_headers(strm, dummy); - res.status = 414; + res.status = StatusCode::UriTooLong_414; return write_response(strm, close_connection, req, res); } // Request line and headers if (!parse_request_line(line_reader.ptr(), req) || !detail::read_headers(strm, req.headers)) { - res.status = 400; + res.status = StatusCode::BadRequest_400; return write_response(strm, close_connection, req, res); } @@ -6109,7 +6686,7 @@ Server::process_request(Stream &strm, bool close_connection, if (req.has_header("Range")) { const auto &range_header_value = req.get_header_value("Range"); if (!detail::parse_range_header(range_header_value, req.ranges)) { - res.status = 416; + res.status = StatusCode::RangeNotSatisfiable_416; return write_response(strm, close_connection, req, res); } } @@ -6117,22 +6694,22 @@ Server::process_request(Stream &strm, bool close_connection, if (setup_request) { setup_request(req); } if (req.get_header_value("Expect") == "100-continue") { - auto status = 100; + int status = StatusCode::Continue_100; if (expect_100_continue_handler_) { status = expect_100_continue_handler_(req, res); } switch (status) { - case 100: - case 417: + case StatusCode::Continue_100: + case StatusCode::ExpectationFailed_417: strm.write_format("HTTP/1.1 %d %s\r\n\r\n", status, - detail::status_message(status)); + status_message(status)); break; default: return write_response(strm, close_connection, req, res); } } - // Rounting - bool routed = false; + // Routing + auto routed = false; #ifdef CPPHTTPLIB_NO_EXCEPTIONS routed = routing(req, res, strm); #else @@ -6144,7 +6721,7 @@ Server::process_request(Stream &strm, bool close_connection, exception_handler_(req, res, ep); routed = true; } else { - res.status = 500; + res.status = StatusCode::InternalServerError_500; std::string val; auto s = e.what(); for (size_t i = 0; s[i]; i++) { @@ -6162,17 +6739,29 @@ Server::process_request(Stream &strm, bool close_connection, exception_handler_(req, res, ep); routed = true; } else { - res.status = 500; + res.status = StatusCode::InternalServerError_500; res.set_header("EXCEPTION_WHAT", "UNKNOWN"); } } #endif - if (routed) { - if (res.status == -1) { res.status = req.ranges.empty() ? 200 : 206; } + if (res.status == -1) { + res.status = req.ranges.empty() ? StatusCode::OK_200 + : StatusCode::PartialContent_206; + } + + if (detail::range_error(req, res)) { + res.body.clear(); + res.content_length_ = 0; + res.content_provider_ = nullptr; + res.status = StatusCode::RangeNotSatisfiable_416; + return write_response(strm, close_connection, req, res); + } + return write_response_with_content(strm, close_connection, req, res); } else { - if (res.status == -1) { res.status = 404; } + if (res.status == -1) { res.status = StatusCode::NotFound_404; } + return write_response(strm, close_connection, req, res); } } @@ -6272,7 +6861,7 @@ inline socket_t ClientImpl::create_client_socket(Error &error) const { // Check is custom IP specified for host_ std::string ip; auto it = addr_map_.find(host_); - if (it != addr_map_.end()) ip = it->second; + if (it != addr_map_.end()) { ip = it->second; } return detail::create_client_socket( host_, ip, port_, address_family_, tcp_nodelay_, socket_options_, @@ -6297,7 +6886,7 @@ inline void ClientImpl::shutdown_ssl(Socket & /*socket*/, socket_requests_are_from_thread_ == std::this_thread::get_id()); } -inline void ClientImpl::shutdown_socket(Socket &socket) { +inline void ClientImpl::shutdown_socket(Socket &socket) const { if (socket.sock == INVALID_SOCKET) { return; } detail::shutdown_socket(socket.sock); } @@ -6322,7 +6911,7 @@ inline void ClientImpl::close_socket(Socket &socket) { } inline bool ClientImpl::read_response_line(Stream &strm, const Request &req, - Response &res) { + Response &res) const { std::array buf{}; detail::stream_line_reader line_reader(strm, buf.data(), buf.size()); @@ -6330,9 +6919,9 @@ inline bool ClientImpl::read_response_line(Stream &strm, const Request &req, if (!line_reader.getline()) { return false; } #ifdef CPPHTTPLIB_ALLOW_LF_AS_LINE_TERMINATOR - const static std::regex re("(HTTP/1\\.[01]) (\\d{3})(?: (.*?))?\r\n"); -#else const static std::regex re("(HTTP/1\\.[01]) (\\d{3})(?: (.*?))?\r?\n"); +#else + const static std::regex re("(HTTP/1\\.[01]) (\\d{3})(?: (.*?))?\r\n"); #endif std::cmatch m; @@ -6344,7 +6933,7 @@ inline bool ClientImpl::read_response_line(Stream &strm, const Request &req, res.reason = std::string(m[3]); // Ignore '100 Continue' - while (res.status == 100) { + while (res.status == StatusCode::Continue_100) { if (!line_reader.getline()) { return false; } // CRLF if (!line_reader.getline()) { return false; } // next response line @@ -6492,15 +7081,31 @@ inline bool ClientImpl::handle_request(Stream &strm, Request &req, if (!ret) { return false; } + if (res.get_header_value("Connection") == "close" || + (res.version == "HTTP/1.0" && res.reason != "Connection established")) { + // TODO this requires a not-entirely-obvious chain of calls to be correct + // for this to be safe. + + // This is safe to call because handle_request is only called by send_ + // which locks the request mutex during the process. It would be a bug + // to call it from a different thread since it's a thread-safety issue + // to do these things to the socket if another thread is using the socket. + std::lock_guard guard(socket_mutex_); + shutdown_ssl(socket_, true); + shutdown_socket(socket_); + close_socket(socket_); + } + if (300 < res.status && res.status < 400 && follow_location_) { req = req_save; ret = redirect(req, res, error); } #ifdef CPPHTTPLIB_OPENSSL_SUPPORT - if ((res.status == 401 || res.status == 407) && + if ((res.status == StatusCode::Unauthorized_401 || + res.status == StatusCode::ProxyAuthenticationRequired_407) && req.authorization_count_ < 5) { - auto is_proxy = res.status == 407; + auto is_proxy = res.status == StatusCode::ProxyAuthenticationRequired_407; const auto &username = is_proxy ? proxy_digest_auth_username_ : digest_auth_username_; const auto &password = @@ -6571,7 +7176,7 @@ inline bool ClientImpl::redirect(Request &req, Response &res, Error &error) { } else { if (next_scheme == "https") { #ifdef CPPHTTPLIB_OPENSSL_SUPPORT - SSLClient cli(next_host.c_str(), next_port); + SSLClient cli(next_host, next_port); cli.copy_settings(*this); if (ca_cert_store_) { cli.set_ca_cert_store(ca_cert_store_); } return detail::redirect(cli, req, res, path, location, error); @@ -6579,7 +7184,7 @@ inline bool ClientImpl::redirect(Request &req, Response &res, Error &error) { return false; #endif } else { - ClientImpl cli(next_host.c_str(), next_port); + ClientImpl cli(next_host, next_port); cli.copy_settings(*this); return detail::redirect(cli, req, res, path, location, error); } @@ -6588,7 +7193,7 @@ inline bool ClientImpl::redirect(Request &req, Response &res, Error &error) { inline bool ClientImpl::write_content_with_provider(Stream &strm, const Request &req, - Error &error) { + Error &error) const { auto is_shutting_down = []() { return false; }; if (req.is_chunked_content_provider_) { @@ -6616,32 +7221,32 @@ inline bool ClientImpl::write_request(Stream &strm, Request &req, // Prepare additional headers if (close_connection) { if (!req.has_header("Connection")) { - req.headers.emplace("Connection", "close"); + req.set_header("Connection", "close"); } } if (!req.has_header("Host")) { if (is_ssl()) { if (port_ == 443) { - req.headers.emplace("Host", host_); + req.set_header("Host", host_); } else { - req.headers.emplace("Host", host_and_port_); + req.set_header("Host", host_and_port_); } } else { if (port_ == 80) { - req.headers.emplace("Host", host_); + req.set_header("Host", host_); } else { - req.headers.emplace("Host", host_and_port_); + req.set_header("Host", host_and_port_); } } } - if (!req.has_header("Accept")) { req.headers.emplace("Accept", "*/*"); } + if (!req.has_header("Accept")) { req.set_header("Accept", "*/*"); } #ifndef CPPHTTPLIB_NO_DEFAULT_USER_AGENT if (!req.has_header("User-Agent")) { auto agent = std::string("cpp-httplib/") + CPPHTTPLIB_VERSION; - req.headers.emplace("User-Agent", agent); + req.set_header("User-Agent", agent); } #endif @@ -6650,23 +7255,23 @@ inline bool ClientImpl::write_request(Stream &strm, Request &req, if (!req.is_chunked_content_provider_) { if (!req.has_header("Content-Length")) { auto length = std::to_string(req.content_length_); - req.headers.emplace("Content-Length", length); + req.set_header("Content-Length", length); } } } else { if (req.method == "POST" || req.method == "PUT" || req.method == "PATCH") { - req.headers.emplace("Content-Length", "0"); + req.set_header("Content-Length", "0"); } } } else { if (!req.has_header("Content-Type")) { - req.headers.emplace("Content-Type", "text/plain"); + req.set_header("Content-Type", "text/plain"); } if (!req.has_header("Content-Length")) { auto length = std::to_string(req.body.size()); - req.headers.emplace("Content-Length", length); + req.set_header("Content-Length", length); } } @@ -6706,7 +7311,7 @@ inline bool ClientImpl::write_request(Stream &strm, Request &req, const auto &path = url_encode_ ? detail::encode_url(req.path) : req.path; bstrm.write_format("%s %s HTTP/1.1\r\n", req.method.c_str(), path.c_str()); - detail::write_headers(bstrm, req.headers); + header_writer_(bstrm, req.headers); // Flush buffer auto &data = bstrm.get_buffer(); @@ -6734,12 +7339,10 @@ inline std::unique_ptr ClientImpl::send_with_content_provider( ContentProvider content_provider, ContentProviderWithoutLength content_provider_without_length, const std::string &content_type, Error &error) { - if (!content_type.empty()) { - req.headers.emplace("Content-Type", content_type); - } + if (!content_type.empty()) { req.set_header("Content-Type", content_type); } #ifdef CPPHTTPLIB_ZLIB_SUPPORT - if (compress_) { req.headers.emplace("Content-Encoding", "gzip"); } + if (compress_) { req.set_header("Content-Encoding", "gzip"); } #endif #ifdef CPPHTTPLIB_ZLIB_SUPPORT @@ -6800,10 +7403,9 @@ inline std::unique_ptr ClientImpl::send_with_content_provider( req.content_provider_ = detail::ContentProviderAdapter( std::move(content_provider_without_length)); req.is_chunked_content_provider_ = true; - req.headers.emplace("Transfer-Encoding", "chunked"); + req.set_header("Transfer-Encoding", "chunked"); } else { req.body.assign(body, content_length); - ; } } @@ -6864,7 +7466,8 @@ inline bool ClientImpl::process_request(Stream &strm, Request &req, } // Body - if ((res.status != 204) && req.method != "HEAD" && req.method != "CONNECT") { + if ((res.status != StatusCode::NoContent_204) && req.method != "HEAD" && + req.method != "CONNECT") { auto redirect = 300 < res.status && res.status < 400 && follow_location_; if (req.response_handler && !redirect) { @@ -6909,24 +7512,6 @@ inline bool ClientImpl::process_request(Stream &strm, Request &req, } } - if (res.get_header_value("Connection") == "close" || - (res.version == "HTTP/1.0" && res.reason != "Connection established")) { - // TODO this requires a not-entirely-obvious chain of calls to be correct - // for this to be safe. Maybe a code refactor (such as moving this out to - // the send function and getting rid of the recursiveness of the mutex) - // could make this more obvious. - - // This is safe to call because process_request is only called by - // handle_request which is only called by send, which locks the request - // mutex during the process. It would be a bug to call it from a different - // thread since it's a thread-safety issue to do these things to the socket - // if another thread is using the socket. - std::lock_guard guard(socket_mutex_); - shutdown_ssl(socket_, true); - shutdown_socket(socket_); - close_socket(socket_); - } - // Log if (logger_) { logger_(req, res); } @@ -6935,13 +7520,14 @@ inline bool ClientImpl::process_request(Stream &strm, Request &req, inline ContentProviderWithoutLength ClientImpl::get_multipart_content_provider( const std::string &boundary, const MultipartFormDataItems &items, - const MultipartFormDataProviderItems &provider_items) { - size_t cur_item = 0, cur_start = 0; + const MultipartFormDataProviderItems &provider_items) const { + size_t cur_item = 0; + size_t cur_start = 0; // cur_item and cur_start are copied to within the std::function and maintain // state between successive calls return [&, cur_item, cur_start](size_t offset, DataSink &sink) mutable -> bool { - if (!offset && items.size()) { + if (!offset && !items.empty()) { sink.os << detail::serialize_multipart_formdata(items, boundary, false); return true; } else if (cur_item < provider_items.size()) { @@ -6954,12 +7540,13 @@ inline ContentProviderWithoutLength ClientImpl::get_multipart_content_provider( } DataSink cur_sink; - bool has_data = true; + auto has_data = true; cur_sink.write = sink.write; cur_sink.done = [&]() { has_data = false; }; - if (!provider_items[cur_item].provider(offset - cur_start, cur_sink)) + if (!provider_items[cur_item].provider(offset - cur_start, cur_sink)) { return false; + } if (!has_data) { sink.os << detail::serialize_multipart_formdata_item_end(); @@ -7078,14 +7665,15 @@ inline Result ClientImpl::Get(const std::string &path, const Params ¶ms, if (params.empty()) { return Get(path, headers); } std::string path_with_query = append_query_params(path, params); - return Get(path_with_query.c_str(), headers, progress); + return Get(path_with_query, headers, std::move(progress)); } inline Result ClientImpl::Get(const std::string &path, const Params ¶ms, const Headers &headers, ContentReceiver content_receiver, Progress progress) { - return Get(path, params, headers, nullptr, content_receiver, progress); + return Get(path, params, headers, nullptr, std::move(content_receiver), + std::move(progress)); } inline Result ClientImpl::Get(const std::string &path, const Params ¶ms, @@ -7094,12 +7682,13 @@ inline Result ClientImpl::Get(const std::string &path, const Params ¶ms, ContentReceiver content_receiver, Progress progress) { if (params.empty()) { - return Get(path, headers, response_handler, content_receiver, progress); + return Get(path, headers, std::move(response_handler), + std::move(content_receiver), std::move(progress)); } std::string path_with_query = append_query_params(path, params); - return Get(path_with_query.c_str(), headers, response_handler, - content_receiver, progress); + return Get(path_with_query, headers, std::move(response_handler), + std::move(content_receiver), std::move(progress)); } inline Result ClientImpl::Head(const std::string &path) { @@ -7201,7 +7790,7 @@ inline Result ClientImpl::Post(const std::string &path, const Headers &headers, const auto &content_type = detail::serialize_multipart_formdata_get_content_type(boundary); const auto &body = detail::serialize_multipart_formdata(items, boundary); - return Post(path, headers, body, content_type.c_str()); + return Post(path, headers, body, content_type); } inline Result ClientImpl::Post(const std::string &path, const Headers &headers, @@ -7214,7 +7803,7 @@ inline Result ClientImpl::Post(const std::string &path, const Headers &headers, const auto &content_type = detail::serialize_multipart_formdata_get_content_type(boundary); const auto &body = detail::serialize_multipart_formdata(items, boundary); - return Post(path, headers, body, content_type.c_str()); + return Post(path, headers, body, content_type); } inline Result @@ -7423,9 +8012,7 @@ inline Result ClientImpl::Delete(const std::string &path, req.headers = headers; req.path = path; - if (!content_type.empty()) { - req.headers.emplace("Content-Type", content_type); - } + if (!content_type.empty()) { req.set_header("Content-Type", content_type); } req.body.assign(body, content_length); return send_(std::move(req)); @@ -7458,13 +8045,6 @@ inline Result ClientImpl::Options(const std::string &path, return send_(std::move(req)); } -inline size_t ClientImpl::is_socket_open() const { - std::lock_guard guard(socket_mutex_); - return socket_.is_open(); -} - -inline socket_t ClientImpl::socket() const { return socket_.sock; } - inline void ClientImpl::stop() { std::lock_guard guard(socket_mutex_); @@ -7488,6 +8068,17 @@ inline void ClientImpl::stop() { close_socket(socket_); } +inline std::string ClientImpl::host() const { return host_; } + +inline int ClientImpl::port() const { return port_; } + +inline size_t ClientImpl::is_socket_open() const { + std::lock_guard guard(socket_mutex_); + return socket_.is_open(); +} + +inline socket_t ClientImpl::socket() const { return socket_.sock; } + inline void ClientImpl::set_connection_timeout(time_t sec, time_t usec) { connection_timeout_sec_ = sec; connection_timeout_usec_ = usec; @@ -7536,6 +8127,11 @@ inline void ClientImpl::set_default_headers(Headers headers) { default_headers_ = std::move(headers); } +inline void ClientImpl::set_header_writer( + std::function const &writer) { + header_writer_ = writer; +} + inline void ClientImpl::set_address_family(int family) { address_family_ = family; } @@ -7575,9 +8171,7 @@ inline void ClientImpl::set_proxy_digest_auth(const std::string &username, proxy_digest_auth_username_ = username; proxy_digest_auth_password_ = password; } -#endif -#ifdef CPPHTTPLIB_OPENSSL_SUPPORT inline void ClientImpl::set_ca_cert_path(const std::string &ca_cert_file_path, const std::string &ca_cert_dir_path) { ca_cert_file_path_ = ca_cert_file_path; @@ -7589,9 +8183,34 @@ inline void ClientImpl::set_ca_cert_store(X509_STORE *ca_cert_store) { ca_cert_store_ = ca_cert_store; } } -#endif -#ifdef CPPHTTPLIB_OPENSSL_SUPPORT +inline X509_STORE *ClientImpl::create_ca_cert_store(const char *ca_cert, + std::size_t size) const { + auto mem = BIO_new_mem_buf(ca_cert, static_cast(size)); + if (!mem) { return nullptr; } + + auto inf = PEM_X509_INFO_read_bio(mem, nullptr, nullptr, nullptr); + if (!inf) { + BIO_free_all(mem); + return nullptr; + } + + auto cts = X509_STORE_new(); + if (cts) { + for (auto i = 0; i < static_cast(sk_X509_INFO_num(inf)); i++) { + auto itmp = sk_X509_INFO_value(inf, i); + if (!itmp) { continue; } + + if (itmp->x509) { X509_STORE_add_cert(cts, itmp->x509); } + if (itmp->crl) { X509_STORE_add_crl(cts, itmp->crl); } + } + } + + sk_X509_INFO_pop_free(inf, X509_INFO_free); + BIO_free_all(mem); + return cts; +} + inline void ClientImpl::enable_server_certificate_verification(bool enabled) { server_certificate_verification_ = enabled; } @@ -7655,7 +8274,7 @@ bool ssl_connect_or_accept_nonblocking(socket_t sock, SSL *ssl, U ssl_connect_or_accept, time_t timeout_sec, time_t timeout_usec) { - int res = 0; + auto res = 0; while ((res = ssl_connect_or_accept(ssl)) != 1) { auto err = SSL_get_error(ssl, res); switch (err) { @@ -7718,7 +8337,7 @@ inline SSLSocketStream::SSLSocketStream(socket_t sock, SSL *ssl, SSL_clear_mode(ssl, SSL_MODE_AUTO_RETRY); } -inline SSLSocketStream::~SSLSocketStream() {} +inline SSLSocketStream::~SSLSocketStream() = default; inline bool SSLSocketStream::is_readable() const { return detail::select_read(sock_, read_timeout_sec_, read_timeout_usec_) > 0; @@ -7736,7 +8355,7 @@ inline ssize_t SSLSocketStream::read(char *ptr, size_t size) { auto ret = SSL_read(ssl_, ptr, static_cast(size)); if (ret < 0) { auto err = SSL_get_error(ssl_, ret); - int n = 1000; + auto n = 1000; #ifdef _WIN32 while (--n >= 0 && (err == SSL_ERROR_WANT_READ || (err == SSL_ERROR_SYSCALL && @@ -7769,7 +8388,7 @@ inline ssize_t SSLSocketStream::write(const char *ptr, size_t size) { auto ret = SSL_write(ssl_, ptr, static_cast(handle_size)); if (ret < 0) { auto err = SSL_get_error(ssl_, ret); - int n = 1000; + auto n = 1000; #ifdef _WIN32 while (--n >= 0 && (err == SSL_ERROR_WANT_WRITE || (err == SSL_ERROR_SYSCALL && @@ -7822,10 +8441,10 @@ inline SSLServer::SSLServer(const char *cert_path, const char *private_key_path, SSL_CTX_set_min_proto_version(ctx_, TLS1_1_VERSION); - // add default password callback before opening encrypted private key if (private_key_password != nullptr && (private_key_password[0] != '\0')) { - SSL_CTX_set_default_passwd_cb_userdata(ctx_, - (char *)private_key_password); + SSL_CTX_set_default_passwd_cb_userdata( + ctx_, + reinterpret_cast(const_cast(private_key_password))); } if (SSL_CTX_use_certificate_chain_file(ctx_, cert_path) != 1 || @@ -7927,16 +8546,23 @@ inline SSLClient::SSLClient(const std::string &host, int port) inline SSLClient::SSLClient(const std::string &host, int port, const std::string &client_cert_path, - const std::string &client_key_path) + const std::string &client_key_path, + const std::string &private_key_password) : ClientImpl(host, port, client_cert_path, client_key_path) { ctx_ = SSL_CTX_new(TLS_client_method()); detail::split(&host_[0], &host_[host_.size()], '.', [&](const char *b, const char *e) { - host_components_.emplace_back(std::string(b, e)); + host_components_.emplace_back(b, e); }); if (!client_cert_path.empty() && !client_key_path.empty()) { + if (!private_key_password.empty()) { + SSL_CTX_set_default_passwd_cb_userdata( + ctx_, reinterpret_cast( + const_cast(private_key_password.c_str()))); + } + if (SSL_CTX_use_certificate_file(ctx_, client_cert_path.c_str(), SSL_FILETYPE_PEM) != 1 || SSL_CTX_use_PrivateKey_file(ctx_, client_key_path.c_str(), @@ -7948,16 +8574,23 @@ inline SSLClient::SSLClient(const std::string &host, int port, } inline SSLClient::SSLClient(const std::string &host, int port, - X509 *client_cert, EVP_PKEY *client_key) + X509 *client_cert, EVP_PKEY *client_key, + const std::string &private_key_password) : ClientImpl(host, port) { ctx_ = SSL_CTX_new(TLS_client_method()); detail::split(&host_[0], &host_[host_.size()], '.', [&](const char *b, const char *e) { - host_components_.emplace_back(std::string(b, e)); + host_components_.emplace_back(b, e); }); if (client_cert != nullptr && client_key != nullptr) { + if (!private_key_password.empty()) { + SSL_CTX_set_default_passwd_cb_userdata( + ctx_, reinterpret_cast( + const_cast(private_key_password.c_str()))); + } + if (SSL_CTX_use_certificate(ctx_, client_cert) != 1 || SSL_CTX_use_PrivateKey(ctx_, client_key) != 1) { SSL_CTX_free(ctx_); @@ -7989,6 +8622,11 @@ inline void SSLClient::set_ca_cert_store(X509_STORE *ca_cert_store) { } } +inline void SSLClient::load_ca_cert_store(const char *ca_cert, + std::size_t size) { + set_ca_cert_store(ClientImpl::create_ca_cert_store(ca_cert, size)); +} + inline long SSLClient::get_openssl_verify_result() const { return verify_result_; } @@ -8003,14 +8641,14 @@ inline bool SSLClient::create_and_connect_socket(Socket &socket, Error &error) { inline bool SSLClient::connect_with_proxy(Socket &socket, Response &res, bool &success, Error &error) { success = true; - Response res2; + Response proxy_res; if (!detail::process_client_socket( socket.sock, read_timeout_sec_, read_timeout_usec_, write_timeout_sec_, write_timeout_usec_, [&](Stream &strm) { Request req2; req2.method = "CONNECT"; req2.path = host_and_port_; - return process_request(strm, req2, res2, false, error); + return process_request(strm, req2, proxy_res, false, error); })) { // Thread-safe to close everything because we are assuming there are no // requests in flight @@ -8021,12 +8659,12 @@ inline bool SSLClient::connect_with_proxy(Socket &socket, Response &res, return false; } - if (res2.status == 407) { + if (proxy_res.status == StatusCode::ProxyAuthenticationRequired_407) { if (!proxy_digest_auth_username_.empty() && !proxy_digest_auth_password_.empty()) { std::map auth; - if (detail::parse_www_authenticate(res2, auth, true)) { - Response res3; + if (detail::parse_www_authenticate(proxy_res, auth, true)) { + proxy_res = Response(); if (!detail::process_client_socket( socket.sock, read_timeout_sec_, read_timeout_usec_, write_timeout_sec_, write_timeout_usec_, [&](Stream &strm) { @@ -8037,7 +8675,7 @@ inline bool SSLClient::connect_with_proxy(Socket &socket, Response &res, req3, auth, 1, detail::random_string(10), proxy_digest_auth_username_, proxy_digest_auth_password_, true)); - return process_request(strm, req3, res3, false, error); + return process_request(strm, req3, proxy_res, false, error); })) { // Thread-safe to close everything because we are assuming there are // no requests in flight @@ -8048,17 +8686,28 @@ inline bool SSLClient::connect_with_proxy(Socket &socket, Response &res, return false; } } - } else { - res = res2; - return false; } } + // If status code is not 200, proxy request is failed. + // Set error to ProxyConnection and return proxy response + // as the response of the request + if (proxy_res.status != StatusCode::OK_200) { + error = Error::ProxyConnection; + res = std::move(proxy_res); + // Thread-safe to close everything because we are assuming there are + // no requests in flight + shutdown_ssl(socket, true); + shutdown_socket(socket); + close_socket(socket); + return false; + } + return true; } inline bool SSLClient::load_certs() { - bool ret = true; + auto ret = true; std::call_once(initialize_cert_, [&]() { std::lock_guard guard(ctx_mutex_); @@ -8134,7 +8783,11 @@ inline bool SSLClient::initialize_ssl(Socket &socket, Error &error) { return true; }, [&](SSL *ssl2) { - SSL_set_tlsext_host_name(ssl2, host_.c_str()); + // NOTE: Direct call instead of using the OpenSSL macro to suppress + // -Wold-style-cast warning + // SSL_set_tlsext_host_name(ssl2, host_.c_str()); + SSL_ctrl(ssl2, SSL_CTRL_SET_TLSEXT_HOSTNAME, TLSEXT_NAMETYPE_host_name, + static_cast(const_cast(host_.c_str()))); return true; }); @@ -8208,8 +8861,8 @@ SSLClient::verify_host_with_subject_alt_name(X509 *server_cert) const { auto type = GEN_DNS; - struct in6_addr addr6; - struct in_addr addr; + struct in6_addr addr6 {}; + struct in_addr addr {}; size_t addr_len = 0; #ifndef __MINGW32__ @@ -8234,8 +8887,9 @@ SSLClient::verify_host_with_subject_alt_name(X509 *server_cert) const { for (decltype(count) i = 0; i < count && !dsn_matched; i++) { auto val = sk_GENERAL_NAME_value(alt_names, i); if (val->type == type) { - auto name = (const char *)ASN1_STRING_get0_data(val->d.ia5); - auto name_len = (size_t)ASN1_STRING_length(val->d.ia5); + auto name = + reinterpret_cast(ASN1_STRING_get0_data(val->d.ia5)); + auto name_len = static_cast(ASN1_STRING_length(val->d.ia5)); switch (type) { case GEN_DNS: dsn_matched = check_host_name(name, name_len); break; @@ -8253,7 +8907,8 @@ SSLClient::verify_host_with_subject_alt_name(X509 *server_cert) const { if (dsn_matched || ip_matched) { ret = true; } } - GENERAL_NAMES_free((STACK_OF(GENERAL_NAME) *)alt_names); + GENERAL_NAMES_free(const_cast( + reinterpret_cast(alt_names))); return ret; } @@ -8282,7 +8937,7 @@ inline bool SSLClient::check_host_name(const char *pattern, std::vector pattern_components; detail::split(&pattern[0], &pattern[pattern_len], '.', [&](const char *b, const char *e) { - pattern_components.emplace_back(std::string(b, e)); + pattern_components.emplace_back(b, e); }); if (host_components_.size() != pattern_components.size()) { return false; } @@ -8361,7 +9016,7 @@ inline Client::Client(const std::string &host, int port, : cli_(detail::make_unique(host, port, client_cert_path, client_key_path)) {} -inline Client::~Client() {} +inline Client::~Client() = default; inline bool Client::is_valid() const { return cli_ != nullptr && cli_->is_valid(); @@ -8421,19 +9076,20 @@ inline Result Client::Get(const std::string &path, const Headers &headers, } inline Result Client::Get(const std::string &path, const Params ¶ms, const Headers &headers, Progress progress) { - return cli_->Get(path, params, headers, progress); + return cli_->Get(path, params, headers, std::move(progress)); } inline Result Client::Get(const std::string &path, const Params ¶ms, const Headers &headers, ContentReceiver content_receiver, Progress progress) { - return cli_->Get(path, params, headers, content_receiver, progress); + return cli_->Get(path, params, headers, std::move(content_receiver), + std::move(progress)); } inline Result Client::Get(const std::string &path, const Params ¶ms, const Headers &headers, ResponseHandler response_handler, ContentReceiver content_receiver, Progress progress) { - return cli_->Get(path, params, headers, response_handler, content_receiver, - progress); + return cli_->Get(path, params, headers, std::move(response_handler), + std::move(content_receiver), std::move(progress)); } inline Result Client::Head(const std::string &path) { return cli_->Head(path); } @@ -8665,12 +9321,16 @@ inline bool Client::send(Request &req, Response &res, Error &error) { inline Result Client::send(const Request &req) { return cli_->send(req); } +inline void Client::stop() { cli_->stop(); } + +inline std::string Client::host() const { return cli_->host(); } + +inline int Client::port() const { return cli_->port(); } + inline size_t Client::is_socket_open() const { return cli_->is_socket_open(); } inline socket_t Client::socket() const { return cli_->socket(); } -inline void Client::stop() { cli_->stop(); } - inline void Client::set_hostname_addr_map(std::map addr_map) { cli_->set_hostname_addr_map(std::move(addr_map)); @@ -8680,6 +9340,11 @@ inline void Client::set_default_headers(Headers headers) { cli_->set_default_headers(std::move(headers)); } +inline void Client::set_header_writer( + std::function const &writer) { + cli_->set_header_writer(writer); +} + inline void Client::set_address_family(int family) { cli_->set_address_family(family); } @@ -8754,7 +9419,9 @@ inline void Client::enable_server_certificate_verification(bool enabled) { } #endif -inline void Client::set_logger(Logger logger) { cli_->set_logger(logger); } +inline void Client::set_logger(Logger logger) { + cli_->set_logger(std::move(logger)); +} #ifdef CPPHTTPLIB_OPENSSL_SUPPORT inline void Client::set_ca_cert_path(const std::string &ca_cert_file_path, @@ -8770,6 +9437,10 @@ inline void Client::set_ca_cert_store(X509_STORE *ca_cert_store) { } } +inline void Client::load_ca_cert_store(const char *ca_cert, std::size_t size) { + set_ca_cert_store(cli_->create_ca_cert_store(ca_cert, size)); +} + inline long Client::get_openssl_verify_result() const { if (is_ssl_) { return static_cast(*cli_).get_openssl_verify_result(); diff --git a/examples/server/json.hpp b/examples/server/json.hpp index ea945f346..a858728c4 100644 --- a/examples/server/json.hpp +++ b/examples/server/json.hpp @@ -1,9 +1,9 @@ // __ _____ _____ _____ // __| | __| | | | JSON for Modern C++ -// | | |__ | | | | | | version 3.11.2 +// | | |__ | | | | | | version 3.11.3 // |_____|_____|_____|_|___| https://github.com/nlohmann/json // -// SPDX-FileCopyrightText: 2013-2022 Niels Lohmann +// SPDX-FileCopyrightText: 2013-2023 Niels Lohmann // SPDX-License-Identifier: MIT /****************************************************************************\ @@ -27,7 +27,6 @@ #endif // JSON_NO_IO #include // random_access_iterator_tag #include // unique_ptr -#include // accumulate #include // string, stoi, to_string #include // declval, forward, move, pair, swap #include // vector @@ -35,10 +34,10 @@ // #include // __ _____ _____ _____ // __| | __| | | | JSON for Modern C++ -// | | |__ | | | | | | version 3.11.2 +// | | |__ | | | | | | version 3.11.3 // |_____|_____|_____|_|___| https://github.com/nlohmann/json // -// SPDX-FileCopyrightText: 2013-2022 Niels Lohmann +// SPDX-FileCopyrightText: 2013-2023 Niels Lohmann // SPDX-License-Identifier: MIT @@ -48,10 +47,10 @@ // #include // __ _____ _____ _____ // __| | __| | | | JSON for Modern C++ -// | | |__ | | | | | | version 3.11.2 +// | | |__ | | | | | | version 3.11.3 // |_____|_____|_____|_|___| https://github.com/nlohmann/json // -// SPDX-FileCopyrightText: 2013-2022 Niels Lohmann +// SPDX-FileCopyrightText: 2013-2023 Niels Lohmann // SPDX-License-Identifier: MIT @@ -60,7 +59,7 @@ #ifndef JSON_SKIP_LIBRARY_VERSION_CHECK #if defined(NLOHMANN_JSON_VERSION_MAJOR) && defined(NLOHMANN_JSON_VERSION_MINOR) && defined(NLOHMANN_JSON_VERSION_PATCH) - #if NLOHMANN_JSON_VERSION_MAJOR != 3 || NLOHMANN_JSON_VERSION_MINOR != 11 || NLOHMANN_JSON_VERSION_PATCH != 2 + #if NLOHMANN_JSON_VERSION_MAJOR != 3 || NLOHMANN_JSON_VERSION_MINOR != 11 || NLOHMANN_JSON_VERSION_PATCH != 3 #warning "Already included a different version of the library!" #endif #endif @@ -68,7 +67,7 @@ #define NLOHMANN_JSON_VERSION_MAJOR 3 // NOLINT(modernize-macro-to-enum) #define NLOHMANN_JSON_VERSION_MINOR 11 // NOLINT(modernize-macro-to-enum) -#define NLOHMANN_JSON_VERSION_PATCH 2 // NOLINT(modernize-macro-to-enum) +#define NLOHMANN_JSON_VERSION_PATCH 3 // NOLINT(modernize-macro-to-enum) #ifndef JSON_DIAGNOSTICS #define JSON_DIAGNOSTICS 0 @@ -150,10 +149,10 @@ // #include // __ _____ _____ _____ // __| | __| | | | JSON for Modern C++ -// | | |__ | | | | | | version 3.11.2 +// | | |__ | | | | | | version 3.11.3 // |_____|_____|_____|_|___| https://github.com/nlohmann/json // -// SPDX-FileCopyrightText: 2013-2022 Niels Lohmann +// SPDX-FileCopyrightText: 2013-2023 Niels Lohmann // SPDX-License-Identifier: MIT @@ -173,16 +172,19 @@ // #include // __ _____ _____ _____ // __| | __| | | | JSON for Modern C++ -// | | |__ | | | | | | version 3.11.2 +// | | |__ | | | | | | version 3.11.3 // |_____|_____|_____|_|___| https://github.com/nlohmann/json // -// SPDX-FileCopyrightText: 2013-2022 Niels Lohmann +// SPDX-FileCopyrightText: 2013-2023 Niels Lohmann // SPDX-License-Identifier: MIT #include // nullptr_t #include // exception +#if JSON_DIAGNOSTICS + #include // accumulate +#endif #include // runtime_error #include // to_string #include // vector @@ -190,10 +192,10 @@ // #include // __ _____ _____ _____ // __| | __| | | | JSON for Modern C++ -// | | |__ | | | | | | version 3.11.2 +// | | |__ | | | | | | version 3.11.3 // |_____|_____|_____|_|___| https://github.com/nlohmann/json // -// SPDX-FileCopyrightText: 2013-2022 Niels Lohmann +// SPDX-FileCopyrightText: 2013-2023 Niels Lohmann // SPDX-License-Identifier: MIT @@ -206,10 +208,10 @@ // #include // __ _____ _____ _____ // __| | __| | | | JSON for Modern C++ -// | | |__ | | | | | | version 3.11.2 +// | | |__ | | | | | | version 3.11.3 // |_____|_____|_____|_|___| https://github.com/nlohmann/json // -// SPDX-FileCopyrightText: 2013-2022 Niels Lohmann +// SPDX-FileCopyrightText: 2013-2023 Niels Lohmann // SPDX-License-Identifier: MIT @@ -218,10 +220,10 @@ // #include // __ _____ _____ _____ // __| | __| | | | JSON for Modern C++ -// | | |__ | | | | | | version 3.11.2 +// | | |__ | | | | | | version 3.11.3 // |_____|_____|_____|_|___| https://github.com/nlohmann/json // -// SPDX-FileCopyrightText: 2013-2022 Niels Lohmann +// SPDX-FileCopyrightText: 2013-2023 Niels Lohmann // SPDX-License-Identifier: MIT @@ -231,10 +233,10 @@ // #include // __ _____ _____ _____ // __| | __| | | | JSON for Modern C++ -// | | |__ | | | | | | version 3.11.2 +// | | |__ | | | | | | version 3.11.3 // |_____|_____|_____|_|___| https://github.com/nlohmann/json // -// SPDX-FileCopyrightText: 2013-2022 Niels Lohmann +// SPDX-FileCopyrightText: 2013-2023 Niels Lohmann // SPDX-License-Identifier: MIT @@ -318,10 +320,10 @@ NLOHMANN_JSON_NAMESPACE_END // __ _____ _____ _____ // __| | __| | | | JSON for Modern C++ -// | | |__ | | | | | | version 3.11.2 +// | | |__ | | | | | | version 3.11.3 // |_____|_____|_____|_|___| https://github.com/nlohmann/json // -// SPDX-FileCopyrightText: 2013-2022 Niels Lohmann +// SPDX-FileCopyrightText: 2013-2023 Niels Lohmann // SPDX-FileCopyrightText: 2016-2021 Evan Nemerson // SPDX-License-Identifier: MIT @@ -2483,6 +2485,14 @@ JSON_HEDLEY_DIAGNOSTIC_POP #endif #endif +#ifndef JSON_HAS_STATIC_RTTI + #if !defined(_HAS_STATIC_RTTI) || _HAS_STATIC_RTTI != 0 + #define JSON_HAS_STATIC_RTTI 1 + #else + #define JSON_HAS_STATIC_RTTI 0 + #endif +#endif + #ifdef JSON_HAS_CPP_17 #define JSON_INLINE_VARIABLE inline #else @@ -2590,12 +2600,13 @@ JSON_HEDLEY_DIAGNOSTIC_POP class NumberUnsignedType, class NumberFloatType, \ template class AllocatorType, \ template class JSONSerializer, \ - class BinaryType> + class BinaryType, \ + class CustomBaseClass> #define NLOHMANN_BASIC_JSON_TPL \ basic_json + AllocatorType, JSONSerializer, BinaryType, CustomBaseClass> // Macros to simplify conversion from/to types @@ -2745,7 +2756,10 @@ JSON_HEDLEY_DIAGNOSTIC_POP #define NLOHMANN_DEFINE_TYPE_INTRUSIVE_WITH_DEFAULT(Type, ...) \ friend void to_json(nlohmann::json& nlohmann_json_j, const Type& nlohmann_json_t) { NLOHMANN_JSON_EXPAND(NLOHMANN_JSON_PASTE(NLOHMANN_JSON_TO, __VA_ARGS__)) } \ - friend void from_json(const nlohmann::json& nlohmann_json_j, Type& nlohmann_json_t) { Type nlohmann_json_default_obj; NLOHMANN_JSON_EXPAND(NLOHMANN_JSON_PASTE(NLOHMANN_JSON_FROM_WITH_DEFAULT, __VA_ARGS__)) } + friend void from_json(const nlohmann::json& nlohmann_json_j, Type& nlohmann_json_t) { const Type nlohmann_json_default_obj{}; NLOHMANN_JSON_EXPAND(NLOHMANN_JSON_PASTE(NLOHMANN_JSON_FROM_WITH_DEFAULT, __VA_ARGS__)) } + +#define NLOHMANN_DEFINE_TYPE_INTRUSIVE_ONLY_SERIALIZE(Type, ...) \ + friend void to_json(nlohmann::json& nlohmann_json_j, const Type& nlohmann_json_t) { NLOHMANN_JSON_EXPAND(NLOHMANN_JSON_PASTE(NLOHMANN_JSON_TO, __VA_ARGS__)) } /*! @brief macro @@ -2756,10 +2770,12 @@ JSON_HEDLEY_DIAGNOSTIC_POP inline void to_json(nlohmann::json& nlohmann_json_j, const Type& nlohmann_json_t) { NLOHMANN_JSON_EXPAND(NLOHMANN_JSON_PASTE(NLOHMANN_JSON_TO, __VA_ARGS__)) } \ inline void from_json(const nlohmann::json& nlohmann_json_j, Type& nlohmann_json_t) { NLOHMANN_JSON_EXPAND(NLOHMANN_JSON_PASTE(NLOHMANN_JSON_FROM, __VA_ARGS__)) } +#define NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE_ONLY_SERIALIZE(Type, ...) \ + inline void to_json(nlohmann::json& nlohmann_json_j, const Type& nlohmann_json_t) { NLOHMANN_JSON_EXPAND(NLOHMANN_JSON_PASTE(NLOHMANN_JSON_TO, __VA_ARGS__)) } + #define NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE_WITH_DEFAULT(Type, ...) \ inline void to_json(nlohmann::json& nlohmann_json_j, const Type& nlohmann_json_t) { NLOHMANN_JSON_EXPAND(NLOHMANN_JSON_PASTE(NLOHMANN_JSON_TO, __VA_ARGS__)) } \ - inline void from_json(const nlohmann::json& nlohmann_json_j, Type& nlohmann_json_t) { Type nlohmann_json_default_obj; NLOHMANN_JSON_EXPAND(NLOHMANN_JSON_PASTE(NLOHMANN_JSON_FROM_WITH_DEFAULT, __VA_ARGS__)) } - + inline void from_json(const nlohmann::json& nlohmann_json_j, Type& nlohmann_json_t) { const Type nlohmann_json_default_obj{}; NLOHMANN_JSON_EXPAND(NLOHMANN_JSON_PASTE(NLOHMANN_JSON_FROM_WITH_DEFAULT, __VA_ARGS__)) } // inspired from https://stackoverflow.com/a/26745591 // allows to call any std function as if (e.g. with begin): @@ -2923,10 +2939,10 @@ NLOHMANN_JSON_NAMESPACE_END // #include // __ _____ _____ _____ // __| | __| | | | JSON for Modern C++ -// | | |__ | | | | | | version 3.11.2 +// | | |__ | | | | | | version 3.11.3 // |_____|_____|_____|_|___| https://github.com/nlohmann/json // -// SPDX-FileCopyrightText: 2013-2022 Niels Lohmann +// SPDX-FileCopyrightText: 2013-2023 Niels Lohmann // SPDX-License-Identifier: MIT @@ -2998,10 +3014,10 @@ NLOHMANN_JSON_NAMESPACE_END // #include // __ _____ _____ _____ // __| | __| | | | JSON for Modern C++ -// | | |__ | | | | | | version 3.11.2 +// | | |__ | | | | | | version 3.11.3 // |_____|_____|_____|_|___| https://github.com/nlohmann/json // -// SPDX-FileCopyrightText: 2013-2022 Niels Lohmann +// SPDX-FileCopyrightText: 2013-2023 Niels Lohmann // SPDX-License-Identifier: MIT @@ -3040,10 +3056,10 @@ NLOHMANN_JSON_NAMESPACE_END // #include // __ _____ _____ _____ // __| | __| | | | JSON for Modern C++ -// | | |__ | | | | | | version 3.11.2 +// | | |__ | | | | | | version 3.11.3 // |_____|_____|_____|_|___| https://github.com/nlohmann/json // -// SPDX-FileCopyrightText: 2013-2022 Niels Lohmann +// SPDX-FileCopyrightText: 2013-2023 Niels Lohmann // SPDX-FileCopyrightText: 2018 The Abseil Authors // SPDX-License-Identifier: MIT @@ -3214,10 +3230,10 @@ NLOHMANN_JSON_NAMESPACE_END // #include // __ _____ _____ _____ // __| | __| | | | JSON for Modern C++ -// | | |__ | | | | | | version 3.11.2 +// | | |__ | | | | | | version 3.11.3 // |_____|_____|_____|_|___| https://github.com/nlohmann/json // -// SPDX-FileCopyrightText: 2013-2022 Niels Lohmann +// SPDX-FileCopyrightText: 2013-2023 Niels Lohmann // SPDX-License-Identifier: MIT @@ -3226,14 +3242,15 @@ NLOHMANN_JSON_NAMESPACE_END #include // false_type, is_constructible, is_integral, is_same, true_type #include // declval #include // tuple +#include // char_traits // #include // __ _____ _____ _____ // __| | __| | | | JSON for Modern C++ -// | | |__ | | | | | | version 3.11.2 +// | | |__ | | | | | | version 3.11.3 // |_____|_____|_____|_|___| https://github.com/nlohmann/json // -// SPDX-FileCopyrightText: 2013-2022 Niels Lohmann +// SPDX-FileCopyrightText: 2013-2023 Niels Lohmann // SPDX-License-Identifier: MIT @@ -3298,10 +3315,10 @@ NLOHMANN_JSON_NAMESPACE_END // #include // __ _____ _____ _____ // __| | __| | | | JSON for Modern C++ -// | | |__ | | | | | | version 3.11.2 +// | | |__ | | | | | | version 3.11.3 // |_____|_____|_____|_|___| https://github.com/nlohmann/json // -// SPDX-FileCopyrightText: 2013-2022 Niels Lohmann +// SPDX-FileCopyrightText: 2013-2023 Niels Lohmann // SPDX-License-Identifier: MIT @@ -3318,10 +3335,10 @@ NLOHMANN_JSON_NAMESPACE_END // #include // __ _____ _____ _____ // __| | __| | | | JSON for Modern C++ -// | | |__ | | | | | | version 3.11.2 +// | | |__ | | | | | | version 3.11.3 // |_____|_____|_____|_|___| https://github.com/nlohmann/json // -// SPDX-FileCopyrightText: 2013-2022 Niels Lohmann +// SPDX-FileCopyrightText: 2013-2023 Niels Lohmann // SPDX-License-Identifier: MIT @@ -3342,10 +3359,10 @@ NLOHMANN_JSON_NAMESPACE_END // #include // __ _____ _____ _____ // __| | __| | | | JSON for Modern C++ -// | | |__ | | | | | | version 3.11.2 +// | | |__ | | | | | | version 3.11.3 // |_____|_____|_____|_|___| https://github.com/nlohmann/json // -// SPDX-FileCopyrightText: 2013-2022 Niels Lohmann +// SPDX-FileCopyrightText: 2013-2023 Niels Lohmann // SPDX-License-Identifier: MIT #ifndef INCLUDE_NLOHMANN_JSON_FWD_HPP_ @@ -3389,7 +3406,8 @@ NLOHMANN_JSON_NAMESPACE_END template class AllocatorType = std::allocator, template class JSONSerializer = adl_serializer, - class BinaryType = std::vector> + class BinaryType = std::vector, // cppcheck-suppress syntaxError + class CustomBaseClass = void> class basic_json; /// @brief JSON Pointer defines a string syntax for identifying a specific value within a JSON document @@ -3577,6 +3595,63 @@ struct actual_object_comparator template using actual_object_comparator_t = typename actual_object_comparator::type; +///////////////// +// char_traits // +///////////////// + +// Primary template of char_traits calls std char_traits +template +struct char_traits : std::char_traits +{}; + +// Explicitly define char traits for unsigned char since it is not standard +template<> +struct char_traits : std::char_traits +{ + using char_type = unsigned char; + using int_type = uint64_t; + + // Redefine to_int_type function + static int_type to_int_type(char_type c) noexcept + { + return static_cast(c); + } + + static char_type to_char_type(int_type i) noexcept + { + return static_cast(i); + } + + static constexpr int_type eof() noexcept + { + return static_cast(EOF); + } +}; + +// Explicitly define char traits for signed char since it is not standard +template<> +struct char_traits : std::char_traits +{ + using char_type = signed char; + using int_type = uint64_t; + + // Redefine to_int_type function + static int_type to_int_type(char_type c) noexcept + { + return static_cast(c); + } + + static char_type to_char_type(int_type i) noexcept + { + return static_cast(i); + } + + static constexpr int_type eof() noexcept + { + return static_cast(EOF); + } +}; + /////////////////// // is_ functions // /////////////////// @@ -3613,7 +3688,6 @@ template struct is_default_constructible> : conjunction...> {}; - template struct is_constructible : std::is_constructible {}; @@ -3629,7 +3703,6 @@ struct is_constructible> : is_default_constructible struct is_constructible> : is_default_constructible> {}; - template struct is_iterator_traits : std::false_type {}; @@ -4039,7 +4112,6 @@ struct value_in_range_of_impl2 } }; - template struct value_in_range_of_impl2 { @@ -4138,10 +4210,10 @@ NLOHMANN_JSON_NAMESPACE_END // #include // __ _____ _____ _____ // __| | __| | | | JSON for Modern C++ -// | | |__ | | | | | | version 3.11.2 +// | | |__ | | | | | | version 3.11.3 // |_____|_____|_____|_|___| https://github.com/nlohmann/json // -// SPDX-FileCopyrightText: 2013-2022 Niels Lohmann +// SPDX-FileCopyrightText: 2013-2023 Niels Lohmann // SPDX-License-Identifier: MIT @@ -4165,28 +4237,28 @@ inline std::size_t concat_length() } template -inline std::size_t concat_length(const char* cstr, Args&& ... rest); +inline std::size_t concat_length(const char* cstr, const Args& ... rest); template -inline std::size_t concat_length(const StringType& str, Args&& ... rest); +inline std::size_t concat_length(const StringType& str, const Args& ... rest); template -inline std::size_t concat_length(const char /*c*/, Args&& ... rest) +inline std::size_t concat_length(const char /*c*/, const Args& ... rest) { - return 1 + concat_length(std::forward(rest)...); + return 1 + concat_length(rest...); } template -inline std::size_t concat_length(const char* cstr, Args&& ... rest) +inline std::size_t concat_length(const char* cstr, const Args& ... rest) { // cppcheck-suppress ignoredReturnValue - return ::strlen(cstr) + concat_length(std::forward(rest)...); + return ::strlen(cstr) + concat_length(rest...); } template -inline std::size_t concat_length(const StringType& str, Args&& ... rest) +inline std::size_t concat_length(const StringType& str, const Args& ... rest) { - return str.size() + concat_length(std::forward(rest)...); + return str.size() + concat_length(rest...); } template @@ -4277,7 +4349,7 @@ template inline OutStringType concat(Args && ... args) { OutStringType str; - str.reserve(concat_length(std::forward(args)...)); + str.reserve(concat_length(args...)); concat_into(str, std::forward(args)...); return str; } @@ -4286,7 +4358,6 @@ inline OutStringType concat(Args && ... args) NLOHMANN_JSON_NAMESPACE_END - NLOHMANN_JSON_NAMESPACE_BEGIN namespace detail { @@ -4334,9 +4405,9 @@ class exception : public std::exception { case value_t::array: { - for (std::size_t i = 0; i < current->m_parent->m_value.array->size(); ++i) + for (std::size_t i = 0; i < current->m_parent->m_data.m_value.array->size(); ++i) { - if (¤t->m_parent->m_value.array->operator[](i) == current) + if (¤t->m_parent->m_data.m_value.array->operator[](i) == current) { tokens.emplace_back(std::to_string(i)); break; @@ -4347,7 +4418,7 @@ class exception : public std::exception case value_t::object: { - for (const auto& element : *current->m_parent->m_value.object) + for (const auto& element : *current->m_parent->m_data.m_value.object) { if (&element.second == current) { @@ -4410,17 +4481,17 @@ class parse_error : public exception template::value, int> = 0> static parse_error create(int id_, const position_t& pos, const std::string& what_arg, BasicJsonContext context) { - std::string w = concat(exception::name("parse_error", id_), "parse error", - position_string(pos), ": ", exception::diagnostics(context), what_arg); + const std::string w = concat(exception::name("parse_error", id_), "parse error", + position_string(pos), ": ", exception::diagnostics(context), what_arg); return {id_, pos.chars_read_total, w.c_str()}; } template::value, int> = 0> static parse_error create(int id_, std::size_t byte_, const std::string& what_arg, BasicJsonContext context) { - std::string w = concat(exception::name("parse_error", id_), "parse error", - (byte_ != 0 ? (concat(" at byte ", std::to_string(byte_))) : ""), - ": ", exception::diagnostics(context), what_arg); + const std::string w = concat(exception::name("parse_error", id_), "parse error", + (byte_ != 0 ? (concat(" at byte ", std::to_string(byte_))) : ""), + ": ", exception::diagnostics(context), what_arg); return {id_, byte_, w.c_str()}; } @@ -4454,7 +4525,7 @@ class invalid_iterator : public exception template::value, int> = 0> static invalid_iterator create(int id_, const std::string& what_arg, BasicJsonContext context) { - std::string w = concat(exception::name("invalid_iterator", id_), exception::diagnostics(context), what_arg); + const std::string w = concat(exception::name("invalid_iterator", id_), exception::diagnostics(context), what_arg); return {id_, w.c_str()}; } @@ -4472,7 +4543,7 @@ class type_error : public exception template::value, int> = 0> static type_error create(int id_, const std::string& what_arg, BasicJsonContext context) { - std::string w = concat(exception::name("type_error", id_), exception::diagnostics(context), what_arg); + const std::string w = concat(exception::name("type_error", id_), exception::diagnostics(context), what_arg); return {id_, w.c_str()}; } @@ -4489,7 +4560,7 @@ class out_of_range : public exception template::value, int> = 0> static out_of_range create(int id_, const std::string& what_arg, BasicJsonContext context) { - std::string w = concat(exception::name("out_of_range", id_), exception::diagnostics(context), what_arg); + const std::string w = concat(exception::name("out_of_range", id_), exception::diagnostics(context), what_arg); return {id_, w.c_str()}; } @@ -4506,7 +4577,7 @@ class other_error : public exception template::value, int> = 0> static other_error create(int id_, const std::string& what_arg, BasicJsonContext context) { - std::string w = concat(exception::name("other_error", id_), exception::diagnostics(context), what_arg); + const std::string w = concat(exception::name("other_error", id_), exception::diagnostics(context), what_arg); return {id_, w.c_str()}; } @@ -4525,10 +4596,10 @@ NLOHMANN_JSON_NAMESPACE_END // #include // __ _____ _____ _____ // __| | __| | | | JSON for Modern C++ -// | | |__ | | | | | | version 3.11.2 +// | | |__ | | | | | | version 3.11.3 // |_____|_____|_____|_|___| https://github.com/nlohmann/json // -// SPDX-FileCopyrightText: 2013-2022 Niels Lohmann +// SPDX-FileCopyrightText: 2013-2023 Niels Lohmann // SPDX-License-Identifier: MIT @@ -4549,10 +4620,10 @@ NLOHMANN_JSON_NAMESPACE_END // #include // __ _____ _____ _____ // __| | __| | | | JSON for Modern C++ -// | | |__ | | | | | | version 3.11.2 +// | | |__ | | | | | | version 3.11.3 // |_____|_____|_____|_|___| https://github.com/nlohmann/json // -// SPDX-FileCopyrightText: 2013-2022 Niels Lohmann +// SPDX-FileCopyrightText: 2013-2023 Niels Lohmann // SPDX-License-Identifier: MIT @@ -5055,10 +5126,10 @@ NLOHMANN_JSON_NAMESPACE_END // #include // __ _____ _____ _____ // __| | __| | | | JSON for Modern C++ -// | | |__ | | | | | | version 3.11.2 +// | | |__ | | | | | | version 3.11.3 // |_____|_____|_____|_|___| https://github.com/nlohmann/json // -// SPDX-FileCopyrightText: 2013-2022 Niels Lohmann +// SPDX-FileCopyrightText: 2013-2023 Niels Lohmann // SPDX-License-Identifier: MIT @@ -5075,10 +5146,10 @@ NLOHMANN_JSON_NAMESPACE_END // #include // __ _____ _____ _____ // __| | __| | | | JSON for Modern C++ -// | | |__ | | | | | | version 3.11.2 +// | | |__ | | | | | | version 3.11.3 // |_____|_____|_____|_|___| https://github.com/nlohmann/json // -// SPDX-FileCopyrightText: 2013-2022 Niels Lohmann +// SPDX-FileCopyrightText: 2013-2023 Niels Lohmann // SPDX-License-Identifier: MIT @@ -5147,10 +5218,10 @@ template class iteration_proxy_value // older GCCs are a bit fussy and require explicit noexcept specifiers on defaulted functions iteration_proxy_value(iteration_proxy_value&&) noexcept(std::is_nothrow_move_constructible::value - && std::is_nothrow_move_constructible::value) = default; + && std::is_nothrow_move_constructible::value) = default; // NOLINT(hicpp-noexcept-move,performance-noexcept-move-constructor,cppcoreguidelines-noexcept-move-operations) iteration_proxy_value& operator=(iteration_proxy_value&&) noexcept(std::is_nothrow_move_assignable::value - && std::is_nothrow_move_assignable::value) = default; + && std::is_nothrow_move_assignable::value) = default; // NOLINT(hicpp-noexcept-move,performance-noexcept-move-constructor,cppcoreguidelines-noexcept-move-operations) ~iteration_proxy_value() = default; /// dereference operator (needed for range-based for) @@ -5297,11 +5368,11 @@ namespace std #pragma clang diagnostic ignored "-Wmismatched-tags" #endif template -class tuple_size<::nlohmann::detail::iteration_proxy_value> +class tuple_size<::nlohmann::detail::iteration_proxy_value> // NOLINT(cert-dcl58-cpp) : public std::integral_constant {}; template -class tuple_element> +class tuple_element> // NOLINT(cert-dcl58-cpp) { public: using type = decltype( @@ -5340,7 +5411,7 @@ namespace detail /* * Note all external_constructor<>::construct functions need to call - * j.m_value.destroy(j.m_type) to avoid a memory leak in case j contains an + * j.m_data.m_value.destroy(j.m_data.m_type) to avoid a memory leak in case j contains an * allocated value (e.g., a string). See bug issue * https://github.com/nlohmann/json/issues/2865 for more information. */ @@ -5353,9 +5424,9 @@ struct external_constructor template static void construct(BasicJsonType& j, typename BasicJsonType::boolean_t b) noexcept { - j.m_value.destroy(j.m_type); - j.m_type = value_t::boolean; - j.m_value = b; + j.m_data.m_value.destroy(j.m_data.m_type); + j.m_data.m_type = value_t::boolean; + j.m_data.m_value = b; j.assert_invariant(); } }; @@ -5366,18 +5437,18 @@ struct external_constructor template static void construct(BasicJsonType& j, const typename BasicJsonType::string_t& s) { - j.m_value.destroy(j.m_type); - j.m_type = value_t::string; - j.m_value = s; + j.m_data.m_value.destroy(j.m_data.m_type); + j.m_data.m_type = value_t::string; + j.m_data.m_value = s; j.assert_invariant(); } template static void construct(BasicJsonType& j, typename BasicJsonType::string_t&& s) { - j.m_value.destroy(j.m_type); - j.m_type = value_t::string; - j.m_value = std::move(s); + j.m_data.m_value.destroy(j.m_data.m_type); + j.m_data.m_type = value_t::string; + j.m_data.m_value = std::move(s); j.assert_invariant(); } @@ -5386,9 +5457,9 @@ struct external_constructor int > = 0 > static void construct(BasicJsonType& j, const CompatibleStringType& str) { - j.m_value.destroy(j.m_type); - j.m_type = value_t::string; - j.m_value.string = j.template create(str); + j.m_data.m_value.destroy(j.m_data.m_type); + j.m_data.m_type = value_t::string; + j.m_data.m_value.string = j.template create(str); j.assert_invariant(); } }; @@ -5399,18 +5470,18 @@ struct external_constructor template static void construct(BasicJsonType& j, const typename BasicJsonType::binary_t& b) { - j.m_value.destroy(j.m_type); - j.m_type = value_t::binary; - j.m_value = typename BasicJsonType::binary_t(b); + j.m_data.m_value.destroy(j.m_data.m_type); + j.m_data.m_type = value_t::binary; + j.m_data.m_value = typename BasicJsonType::binary_t(b); j.assert_invariant(); } template static void construct(BasicJsonType& j, typename BasicJsonType::binary_t&& b) { - j.m_value.destroy(j.m_type); - j.m_type = value_t::binary; - j.m_value = typename BasicJsonType::binary_t(std::move(b)); + j.m_data.m_value.destroy(j.m_data.m_type); + j.m_data.m_type = value_t::binary; + j.m_data.m_value = typename BasicJsonType::binary_t(std::move(b)); j.assert_invariant(); } }; @@ -5421,9 +5492,9 @@ struct external_constructor template static void construct(BasicJsonType& j, typename BasicJsonType::number_float_t val) noexcept { - j.m_value.destroy(j.m_type); - j.m_type = value_t::number_float; - j.m_value = val; + j.m_data.m_value.destroy(j.m_data.m_type); + j.m_data.m_type = value_t::number_float; + j.m_data.m_value = val; j.assert_invariant(); } }; @@ -5434,9 +5505,9 @@ struct external_constructor template static void construct(BasicJsonType& j, typename BasicJsonType::number_unsigned_t val) noexcept { - j.m_value.destroy(j.m_type); - j.m_type = value_t::number_unsigned; - j.m_value = val; + j.m_data.m_value.destroy(j.m_data.m_type); + j.m_data.m_type = value_t::number_unsigned; + j.m_data.m_value = val; j.assert_invariant(); } }; @@ -5447,9 +5518,9 @@ struct external_constructor template static void construct(BasicJsonType& j, typename BasicJsonType::number_integer_t val) noexcept { - j.m_value.destroy(j.m_type); - j.m_type = value_t::number_integer; - j.m_value = val; + j.m_data.m_value.destroy(j.m_data.m_type); + j.m_data.m_type = value_t::number_integer; + j.m_data.m_value = val; j.assert_invariant(); } }; @@ -5460,9 +5531,9 @@ struct external_constructor template static void construct(BasicJsonType& j, const typename BasicJsonType::array_t& arr) { - j.m_value.destroy(j.m_type); - j.m_type = value_t::array; - j.m_value = arr; + j.m_data.m_value.destroy(j.m_data.m_type); + j.m_data.m_type = value_t::array; + j.m_data.m_value = arr; j.set_parents(); j.assert_invariant(); } @@ -5470,9 +5541,9 @@ struct external_constructor template static void construct(BasicJsonType& j, typename BasicJsonType::array_t&& arr) { - j.m_value.destroy(j.m_type); - j.m_type = value_t::array; - j.m_value = std::move(arr); + j.m_data.m_value.destroy(j.m_data.m_type); + j.m_data.m_type = value_t::array; + j.m_data.m_value = std::move(arr); j.set_parents(); j.assert_invariant(); } @@ -5485,9 +5556,9 @@ struct external_constructor using std::begin; using std::end; - j.m_value.destroy(j.m_type); - j.m_type = value_t::array; - j.m_value.array = j.template create(begin(arr), end(arr)); + j.m_data.m_value.destroy(j.m_data.m_type); + j.m_data.m_type = value_t::array; + j.m_data.m_value.array = j.template create(begin(arr), end(arr)); j.set_parents(); j.assert_invariant(); } @@ -5495,14 +5566,14 @@ struct external_constructor template static void construct(BasicJsonType& j, const std::vector& arr) { - j.m_value.destroy(j.m_type); - j.m_type = value_t::array; - j.m_value = value_t::array; - j.m_value.array->reserve(arr.size()); + j.m_data.m_value.destroy(j.m_data.m_type); + j.m_data.m_type = value_t::array; + j.m_data.m_value = value_t::array; + j.m_data.m_value.array->reserve(arr.size()); for (const bool x : arr) { - j.m_value.array->push_back(x); - j.set_parent(j.m_value.array->back()); + j.m_data.m_value.array->push_back(x); + j.set_parent(j.m_data.m_value.array->back()); } j.assert_invariant(); } @@ -5511,13 +5582,13 @@ struct external_constructor enable_if_t::value, int> = 0> static void construct(BasicJsonType& j, const std::valarray& arr) { - j.m_value.destroy(j.m_type); - j.m_type = value_t::array; - j.m_value = value_t::array; - j.m_value.array->resize(arr.size()); + j.m_data.m_value.destroy(j.m_data.m_type); + j.m_data.m_type = value_t::array; + j.m_data.m_value = value_t::array; + j.m_data.m_value.array->resize(arr.size()); if (arr.size() > 0) { - std::copy(std::begin(arr), std::end(arr), j.m_value.array->begin()); + std::copy(std::begin(arr), std::end(arr), j.m_data.m_value.array->begin()); } j.set_parents(); j.assert_invariant(); @@ -5530,9 +5601,9 @@ struct external_constructor template static void construct(BasicJsonType& j, const typename BasicJsonType::object_t& obj) { - j.m_value.destroy(j.m_type); - j.m_type = value_t::object; - j.m_value = obj; + j.m_data.m_value.destroy(j.m_data.m_type); + j.m_data.m_type = value_t::object; + j.m_data.m_value = obj; j.set_parents(); j.assert_invariant(); } @@ -5540,9 +5611,9 @@ struct external_constructor template static void construct(BasicJsonType& j, typename BasicJsonType::object_t&& obj) { - j.m_value.destroy(j.m_type); - j.m_type = value_t::object; - j.m_value = std::move(obj); + j.m_data.m_value.destroy(j.m_data.m_type); + j.m_data.m_type = value_t::object; + j.m_data.m_value = std::move(obj); j.set_parents(); j.assert_invariant(); } @@ -5554,9 +5625,9 @@ struct external_constructor using std::begin; using std::end; - j.m_value.destroy(j.m_type); - j.m_type = value_t::object; - j.m_value.object = j.template create(begin(obj), end(obj)); + j.m_data.m_value.destroy(j.m_data.m_type); + j.m_data.m_type = value_t::object; + j.m_data.m_value.object = j.template create(begin(obj), end(obj)); j.set_parents(); j.assert_invariant(); } @@ -5626,7 +5697,8 @@ template::type; - external_constructor::construct(j, static_cast(e)); + static constexpr value_t integral_value_t = std::is_unsigned::value ? value_t::number_unsigned : value_t::number_integer; + external_constructor::construct(j, static_cast(e)); } #endif // JSON_DISABLE_ENUM_SERIALIZATION @@ -5796,10 +5868,10 @@ NLOHMANN_JSON_NAMESPACE_END // #include // __ _____ _____ _____ // __| | __| | | | JSON for Modern C++ -// | | |__ | | | | | | version 3.11.2 +// | | |__ | | | | | | version 3.11.3 // |_____|_____|_____|_|___| https://github.com/nlohmann/json // -// SPDX-FileCopyrightText: 2013-2022 Niels Lohmann +// SPDX-FileCopyrightText: 2013-2023 Niels Lohmann // SPDX-License-Identifier: MIT @@ -5908,10 +5980,10 @@ NLOHMANN_JSON_NAMESPACE_END // #include // __ _____ _____ _____ // __| | __| | | | JSON for Modern C++ -// | | |__ | | | | | | version 3.11.2 +// | | |__ | | | | | | version 3.11.3 // |_____|_____|_____|_|___| https://github.com/nlohmann/json // -// SPDX-FileCopyrightText: 2013-2022 Niels Lohmann +// SPDX-FileCopyrightText: 2013-2023 Niels Lohmann // SPDX-License-Identifier: MIT @@ -6041,10 +6113,10 @@ NLOHMANN_JSON_NAMESPACE_END // #include // __ _____ _____ _____ // __| | __| | | | JSON for Modern C++ -// | | |__ | | | | | | version 3.11.2 +// | | |__ | | | | | | version 3.11.3 // |_____|_____|_____|_|___| https://github.com/nlohmann/json // -// SPDX-FileCopyrightText: 2013-2022 Niels Lohmann +// SPDX-FileCopyrightText: 2013-2023 Niels Lohmann // SPDX-License-Identifier: MIT @@ -6067,10 +6139,10 @@ NLOHMANN_JSON_NAMESPACE_END // #include // __ _____ _____ _____ // __| | __| | | | JSON for Modern C++ -// | | |__ | | | | | | version 3.11.2 +// | | |__ | | | | | | version 3.11.3 // |_____|_____|_____|_|___| https://github.com/nlohmann/json // -// SPDX-FileCopyrightText: 2013-2022 Niels Lohmann +// SPDX-FileCopyrightText: 2013-2023 Niels Lohmann // SPDX-License-Identifier: MIT @@ -6094,6 +6166,8 @@ NLOHMANN_JSON_NAMESPACE_END // #include +// #include + NLOHMANN_JSON_NAMESPACE_BEGIN namespace detail @@ -6140,7 +6214,6 @@ class file_input_adapter std::FILE* m_file; }; - /*! Input adapter for a (caching) istream. Ignores a UFT Byte Order Mark at beginning of input. Does not support changing the underlying std::streambuf @@ -6214,16 +6287,16 @@ class iterator_input_adapter : current(std::move(first)), end(std::move(last)) {} - typename std::char_traits::int_type get_character() + typename char_traits::int_type get_character() { if (JSON_HEDLEY_LIKELY(current != end)) { - auto result = std::char_traits::to_int_type(*current); + auto result = char_traits::to_int_type(*current); std::advance(current, 1); return result; } - return std::char_traits::eof(); + return char_traits::eof(); } private: @@ -6239,7 +6312,6 @@ class iterator_input_adapter } }; - template struct wide_string_input_helper; @@ -6363,7 +6435,7 @@ struct wide_string_input_helper } }; -// Wraps another input apdater to convert wide character types into individual bytes. +// Wraps another input adapter to convert wide character types into individual bytes. template class wide_string_input_adapter { @@ -6408,7 +6480,6 @@ class wide_string_input_adapter std::size_t utf8_bytes_filled = 0; }; - template struct iterator_input_adapter_factory { @@ -6565,10 +6636,10 @@ NLOHMANN_JSON_NAMESPACE_END // #include // __ _____ _____ _____ // __| | __| | | | JSON for Modern C++ -// | | |__ | | | | | | version 3.11.2 +// | | |__ | | | | | | version 3.11.3 // |_____|_____|_____|_|___| https://github.com/nlohmann/json // -// SPDX-FileCopyrightText: 2013-2022 Niels Lohmann +// SPDX-FileCopyrightText: 2013-2023 Niels Lohmann // SPDX-License-Identifier: MIT @@ -6710,7 +6781,6 @@ struct json_sax virtual ~json_sax() = default; }; - namespace detail { /*! @@ -6812,7 +6882,7 @@ class json_sax_dom_parser JSON_ASSERT(ref_stack.back()->is_object()); // add null at given key and store the reference for later - object_element = &(ref_stack.back()->m_value.object->operator[](val)); + object_element = &(ref_stack.back()->m_data.m_value.object->operator[](val)); return true; } @@ -6887,8 +6957,8 @@ class json_sax_dom_parser if (ref_stack.back()->is_array()) { - ref_stack.back()->m_value.array->emplace_back(std::forward(v)); - return &(ref_stack.back()->m_value.array->back()); + ref_stack.back()->m_data.m_value.array->emplace_back(std::forward(v)); + return &(ref_stack.back()->m_data.m_value.array->back()); } JSON_ASSERT(ref_stack.back()->is_object()); @@ -7007,7 +7077,7 @@ class json_sax_dom_callback_parser // add discarded value at given key and store the reference for later if (keep && ref_stack.back()) { - object_element = &(ref_stack.back()->m_value.object->operator[](val) = discarded); + object_element = &(ref_stack.back()->m_data.m_value.object->operator[](val) = discarded); } return true; @@ -7092,7 +7162,7 @@ class json_sax_dom_callback_parser // remove discarded value if (!keep && !ref_stack.empty() && ref_stack.back()->is_array()) { - ref_stack.back()->m_value.array->pop_back(); + ref_stack.back()->m_data.m_value.array->pop_back(); } return true; @@ -7159,7 +7229,7 @@ class json_sax_dom_callback_parser if (ref_stack.empty()) { root = std::move(value); - return {true, &root}; + return {true, & root}; } // skip this value if we already decided to skip the parent @@ -7175,8 +7245,8 @@ class json_sax_dom_callback_parser // array if (ref_stack.back()->is_array()) { - ref_stack.back()->m_value.array->emplace_back(std::move(value)); - return {true, &(ref_stack.back()->m_value.array->back())}; + ref_stack.back()->m_data.m_value.array->emplace_back(std::move(value)); + return {true, & (ref_stack.back()->m_data.m_value.array->back())}; } // object @@ -7201,9 +7271,9 @@ class json_sax_dom_callback_parser /// stack to model hierarchy of values std::vector ref_stack {}; /// stack to manage which values to keep - std::vector keep_stack {}; + std::vector keep_stack {}; // NOLINT(readability-redundant-member-init) /// stack to manage which object keys to keep - std::vector key_keep_stack {}; + std::vector key_keep_stack {}; // NOLINT(readability-redundant-member-init) /// helper to hold the reference for the next object element BasicJsonType* object_element = nullptr; /// whether a syntax error occurred @@ -7298,10 +7368,10 @@ NLOHMANN_JSON_NAMESPACE_END // #include // __ _____ _____ _____ // __| | __| | | | JSON for Modern C++ -// | | |__ | | | | | | version 3.11.2 +// | | |__ | | | | | | version 3.11.3 // |_____|_____|_____|_|___| https://github.com/nlohmann/json // -// SPDX-FileCopyrightText: 2013-2022 Niels Lohmann +// SPDX-FileCopyrightText: 2013-2023 Niels Lohmann // SPDX-License-Identifier: MIT @@ -7322,6 +7392,8 @@ NLOHMANN_JSON_NAMESPACE_END // #include +// #include + NLOHMANN_JSON_NAMESPACE_BEGIN namespace detail @@ -7416,7 +7488,7 @@ class lexer : public lexer_base using number_float_t = typename BasicJsonType::number_float_t; using string_t = typename BasicJsonType::string_t; using char_type = typename InputAdapterType::char_type; - using char_int_type = typename std::char_traits::int_type; + using char_int_type = typename char_traits::int_type; public: using token_type = typename lexer_base::token_type; @@ -7523,7 +7595,7 @@ class lexer : public lexer_base for (auto range = ranges.begin(); range != ranges.end(); ++range) { get(); - if (JSON_HEDLEY_LIKELY(*range <= current && current <= *(++range))) + if (JSON_HEDLEY_LIKELY(*range <= current && current <= *(++range))) // NOLINT(bugprone-inc-dec-in-conditions) { add(current); } @@ -7566,7 +7638,7 @@ class lexer : public lexer_base switch (get()) { // end of file while parsing string - case std::char_traits::eof(): + case char_traits::eof(): { error_message = "invalid string: missing closing quote"; return token_type::parse_error; @@ -8155,7 +8227,7 @@ class lexer : public lexer_base { case '\n': case '\r': - case std::char_traits::eof(): + case char_traits::eof(): case '\0': return true; @@ -8172,7 +8244,7 @@ class lexer : public lexer_base { switch (get()) { - case std::char_traits::eof(): + case char_traits::eof(): case '\0': { error_message = "invalid comment; missing closing '*/'"; @@ -8601,10 +8673,10 @@ scan_number_done: token_type scan_literal(const char_type* literal_text, const std::size_t length, token_type return_type) { - JSON_ASSERT(std::char_traits::to_char_type(current) == literal_text[0]); + JSON_ASSERT(char_traits::to_char_type(current) == literal_text[0]); for (std::size_t i = 1; i < length; ++i) { - if (JSON_HEDLEY_UNLIKELY(std::char_traits::to_char_type(get()) != literal_text[i])) + if (JSON_HEDLEY_UNLIKELY(char_traits::to_char_type(get()) != literal_text[i])) { error_message = "invalid literal"; return token_type::parse_error; @@ -8622,7 +8694,7 @@ scan_number_done: { token_buffer.clear(); token_string.clear(); - token_string.push_back(std::char_traits::to_char_type(current)); + token_string.push_back(char_traits::to_char_type(current)); } /* @@ -8630,7 +8702,7 @@ scan_number_done: This function provides the interface to the used input adapter. It does not throw in case the input reached EOF, but returns a - `std::char_traits::eof()` in that case. Stores the scanned characters + `char_traits::eof()` in that case. Stores the scanned characters for use in error messages. @return character read from the input @@ -8650,9 +8722,9 @@ scan_number_done: current = ia.get_character(); } - if (JSON_HEDLEY_LIKELY(current != std::char_traits::eof())) + if (JSON_HEDLEY_LIKELY(current != char_traits::eof())) { - token_string.push_back(std::char_traits::to_char_type(current)); + token_string.push_back(char_traits::to_char_type(current)); } if (current == '\n') @@ -8691,7 +8763,7 @@ scan_number_done: --position.chars_read_current_line; } - if (JSON_HEDLEY_LIKELY(current != std::char_traits::eof())) + if (JSON_HEDLEY_LIKELY(current != char_traits::eof())) { JSON_ASSERT(!token_string.empty()); token_string.pop_back(); @@ -8885,7 +8957,7 @@ scan_number_done: // end of input (the null byte is needed when parsing from // string literals) case '\0': - case std::char_traits::eof(): + case char_traits::eof(): return token_type::end_of_input; // error @@ -8903,7 +8975,7 @@ scan_number_done: const bool ignore_comments = false; /// the current character - char_int_type current = std::char_traits::eof(); + char_int_type current = char_traits::eof(); /// whether the next get() call should just return current bool next_unget = false; @@ -8937,10 +9009,10 @@ NLOHMANN_JSON_NAMESPACE_END // #include // __ _____ _____ _____ // __| | __| | | | JSON for Modern C++ -// | | |__ | | | | | | version 3.11.2 +// | | |__ | | | | | | version 3.11.3 // |_____|_____|_____|_|___| https://github.com/nlohmann/json // -// SPDX-FileCopyrightText: 2013-2022 Niels Lohmann +// SPDX-FileCopyrightText: 2013-2023 Niels Lohmann // SPDX-License-Identifier: MIT @@ -9129,7 +9201,6 @@ static inline bool little_endianness(int num = 1) noexcept return *reinterpret_cast(&num) == 1; } - /////////////////// // binary reader // /////////////////// @@ -9147,7 +9218,7 @@ class binary_reader using binary_t = typename BasicJsonType::binary_t; using json_sax_t = SAX; using char_type = typename InputAdapterType::char_type; - using char_int_type = typename std::char_traits::int_type; + using char_int_type = typename char_traits::int_type; public: /*! @@ -9220,7 +9291,7 @@ class binary_reader get(); } - if (JSON_HEDLEY_UNLIKELY(current != std::char_traits::eof())) + if (JSON_HEDLEY_UNLIKELY(current != char_traits::eof())) { return sax->parse_error(chars_read, get_token_string(), parse_error::create(110, chars_read, exception_message(input_format, concat("expected end of input; last byte: 0x", get_token_string()), "value"), nullptr)); @@ -9303,7 +9374,7 @@ class binary_reader exception_message(input_format_t::bson, concat("string length must be at least 1, is ", std::to_string(len)), "string"), nullptr)); } - return get_string(input_format_t::bson, len - static_cast(1), result) && get() != std::char_traits::eof(); + return get_string(input_format_t::bson, len - static_cast(1), result) && get() != char_traits::eof(); } /*! @@ -9404,7 +9475,7 @@ class binary_reader { std::array cr{{}}; static_cast((std::snprintf)(cr.data(), cr.size(), "%.2hhX", static_cast(element_type))); // NOLINT(cppcoreguidelines-pro-type-vararg,hicpp-vararg) - std::string cr_str{cr.data()}; + const std::string cr_str{cr.data()}; return sax->parse_error(element_type_parse_position, cr_str, parse_error::create(114, element_type_parse_position, concat("Unsupported BSON record type 0x", cr_str), nullptr)); } @@ -9497,7 +9568,7 @@ class binary_reader switch (get_char ? get() : current) { // EOF - case std::char_traits::eof(): + case char_traits::eof(): return unexpect_eof(input_format_t::cbor, "value"); // Integer 0x00..0x17 (0..23) @@ -10272,7 +10343,7 @@ class binary_reader switch (get()) { // EOF - case std::char_traits::eof(): + case char_traits::eof(): return unexpect_eof(input_format_t::msgpack, "value"); // positive fixint @@ -11339,7 +11410,7 @@ class binary_reader exception_message(input_format, concat("expected '#' after type information; last byte: 0x", last_token), "size"), nullptr)); } - bool is_error = get_ubjson_size_value(result.first, is_ndarray); + const bool is_error = get_ubjson_size_value(result.first, is_ndarray); if (input_format == input_format_t::bjdata && is_ndarray) { if (inside_ndarray) @@ -11354,7 +11425,7 @@ class binary_reader if (current == '#') { - bool is_error = get_ubjson_size_value(result.first, is_ndarray); + const bool is_error = get_ubjson_size_value(result.first, is_ndarray); if (input_format == input_format_t::bjdata && is_ndarray) { return sax->parse_error(chars_read, get_token_string(), parse_error::create(112, chars_read, @@ -11374,7 +11445,7 @@ class binary_reader { switch (prefix) { - case std::char_traits::eof(): // EOF + case char_traits::eof(): // EOF return unexpect_eof(input_format, "value"); case 'T': // true @@ -11819,7 +11890,7 @@ class binary_reader This function provides the interface to the used input adapter. It does not throw in case the input reached EOF, but returns a -'ve valued - `std::char_traits::eof()` in that case. + `char_traits::eof()` in that case. @return character read from the input */ @@ -11961,7 +12032,7 @@ class binary_reader JSON_HEDLEY_NON_NULL(3) bool unexpect_eof(const input_format_t format, const char* context) const { - if (JSON_HEDLEY_UNLIKELY(current == std::char_traits::eof())) + if (JSON_HEDLEY_UNLIKELY(current == char_traits::eof())) { return sax->parse_error(chars_read, "", parse_error::create(110, chars_read, exception_message(format, "unexpected end of input", context), nullptr)); @@ -12028,7 +12099,7 @@ class binary_reader InputAdapterType ia; /// the current character - char_int_type current = std::char_traits::eof(); + char_int_type current = char_traits::eof(); /// the number of characters read std::size_t chars_read = 0; @@ -12090,10 +12161,10 @@ NLOHMANN_JSON_NAMESPACE_END // #include // __ _____ _____ _____ // __| | __| | | | JSON for Modern C++ -// | | |__ | | | | | | version 3.11.2 +// | | |__ | | | | | | version 3.11.3 // |_____|_____|_____|_|___| https://github.com/nlohmann/json // -// SPDX-FileCopyrightText: 2013-2022 Niels Lohmann +// SPDX-FileCopyrightText: 2013-2023 Niels Lohmann // SPDX-License-Identifier: MIT @@ -12439,13 +12510,25 @@ class parser m_lexer.get_token_string(), parse_error::create(101, m_lexer.get_position(), exception_message(token_type::uninitialized, "value"), nullptr)); } + case token_type::end_of_input: + { + if (JSON_HEDLEY_UNLIKELY(m_lexer.get_position().chars_read_total == 1)) + { + return sax->parse_error(m_lexer.get_position(), + m_lexer.get_token_string(), + parse_error::create(101, m_lexer.get_position(), + "attempting to parse an empty input; check that your input string or stream contains the expected JSON", nullptr)); + } + return sax->parse_error(m_lexer.get_position(), + m_lexer.get_token_string(), + parse_error::create(101, m_lexer.get_position(), exception_message(token_type::literal_or_value, "value"), nullptr)); + } case token_type::uninitialized: case token_type::end_array: case token_type::end_object: case token_type::name_separator: case token_type::value_separator: - case token_type::end_of_input: case token_type::literal_or_value: default: // the last token was unexpected { @@ -12607,10 +12690,10 @@ NLOHMANN_JSON_NAMESPACE_END // #include // __ _____ _____ _____ // __| | __| | | | JSON for Modern C++ -// | | |__ | | | | | | version 3.11.2 +// | | |__ | | | | | | version 3.11.3 // |_____|_____|_____|_|___| https://github.com/nlohmann/json // -// SPDX-FileCopyrightText: 2013-2022 Niels Lohmann +// SPDX-FileCopyrightText: 2013-2023 Niels Lohmann // SPDX-License-Identifier: MIT @@ -12620,10 +12703,10 @@ NLOHMANN_JSON_NAMESPACE_END // #include // __ _____ _____ _____ // __| | __| | | | JSON for Modern C++ -// | | |__ | | | | | | version 3.11.2 +// | | |__ | | | | | | version 3.11.3 // |_____|_____|_____|_|___| https://github.com/nlohmann/json // -// SPDX-FileCopyrightText: 2013-2022 Niels Lohmann +// SPDX-FileCopyrightText: 2013-2023 Niels Lohmann // SPDX-License-Identifier: MIT @@ -12779,10 +12862,10 @@ NLOHMANN_JSON_NAMESPACE_END // #include // __ _____ _____ _____ // __| | __| | | | JSON for Modern C++ -// | | |__ | | | | | | version 3.11.2 +// | | |__ | | | | | | version 3.11.3 // |_____|_____|_____|_|___| https://github.com/nlohmann/json // -// SPDX-FileCopyrightText: 2013-2022 Niels Lohmann +// SPDX-FileCopyrightText: 2013-2023 Niels Lohmann // SPDX-License-Identifier: MIT @@ -12887,7 +12970,7 @@ class iter_impl // NOLINT(cppcoreguidelines-special-member-functions,hicpp-speci { JSON_ASSERT(m_object != nullptr); - switch (m_object->m_type) + switch (m_object->m_data.m_type) { case value_t::object: { @@ -12984,17 +13067,17 @@ class iter_impl // NOLINT(cppcoreguidelines-special-member-functions,hicpp-speci { JSON_ASSERT(m_object != nullptr); - switch (m_object->m_type) + switch (m_object->m_data.m_type) { case value_t::object: { - m_it.object_iterator = m_object->m_value.object->begin(); + m_it.object_iterator = m_object->m_data.m_value.object->begin(); break; } case value_t::array: { - m_it.array_iterator = m_object->m_value.array->begin(); + m_it.array_iterator = m_object->m_data.m_value.array->begin(); break; } @@ -13028,17 +13111,17 @@ class iter_impl // NOLINT(cppcoreguidelines-special-member-functions,hicpp-speci { JSON_ASSERT(m_object != nullptr); - switch (m_object->m_type) + switch (m_object->m_data.m_type) { case value_t::object: { - m_it.object_iterator = m_object->m_value.object->end(); + m_it.object_iterator = m_object->m_data.m_value.object->end(); break; } case value_t::array: { - m_it.array_iterator = m_object->m_value.array->end(); + m_it.array_iterator = m_object->m_data.m_value.array->end(); break; } @@ -13067,17 +13150,17 @@ class iter_impl // NOLINT(cppcoreguidelines-special-member-functions,hicpp-speci { JSON_ASSERT(m_object != nullptr); - switch (m_object->m_type) + switch (m_object->m_data.m_type) { case value_t::object: { - JSON_ASSERT(m_it.object_iterator != m_object->m_value.object->end()); + JSON_ASSERT(m_it.object_iterator != m_object->m_data.m_value.object->end()); return m_it.object_iterator->second; } case value_t::array: { - JSON_ASSERT(m_it.array_iterator != m_object->m_value.array->end()); + JSON_ASSERT(m_it.array_iterator != m_object->m_data.m_value.array->end()); return *m_it.array_iterator; } @@ -13111,17 +13194,17 @@ class iter_impl // NOLINT(cppcoreguidelines-special-member-functions,hicpp-speci { JSON_ASSERT(m_object != nullptr); - switch (m_object->m_type) + switch (m_object->m_data.m_type) { case value_t::object: { - JSON_ASSERT(m_it.object_iterator != m_object->m_value.object->end()); + JSON_ASSERT(m_it.object_iterator != m_object->m_data.m_value.object->end()); return &(m_it.object_iterator->second); } case value_t::array: { - JSON_ASSERT(m_it.array_iterator != m_object->m_value.array->end()); + JSON_ASSERT(m_it.array_iterator != m_object->m_data.m_value.array->end()); return &*m_it.array_iterator; } @@ -13164,7 +13247,7 @@ class iter_impl // NOLINT(cppcoreguidelines-special-member-functions,hicpp-speci { JSON_ASSERT(m_object != nullptr); - switch (m_object->m_type) + switch (m_object->m_data.m_type) { case value_t::object: { @@ -13215,7 +13298,7 @@ class iter_impl // NOLINT(cppcoreguidelines-special-member-functions,hicpp-speci { JSON_ASSERT(m_object != nullptr); - switch (m_object->m_type) + switch (m_object->m_data.m_type) { case value_t::object: { @@ -13262,7 +13345,7 @@ class iter_impl // NOLINT(cppcoreguidelines-special-member-functions,hicpp-speci JSON_ASSERT(m_object != nullptr); - switch (m_object->m_type) + switch (m_object->m_data.m_type) { case value_t::object: return (m_it.object_iterator == other.m_it.object_iterator); @@ -13307,7 +13390,7 @@ class iter_impl // NOLINT(cppcoreguidelines-special-member-functions,hicpp-speci JSON_ASSERT(m_object != nullptr); - switch (m_object->m_type) + switch (m_object->m_data.m_type) { case value_t::object: JSON_THROW(invalid_iterator::create(213, "cannot compare order of object iterators", m_object)); @@ -13363,7 +13446,7 @@ class iter_impl // NOLINT(cppcoreguidelines-special-member-functions,hicpp-speci { JSON_ASSERT(m_object != nullptr); - switch (m_object->m_type) + switch (m_object->m_data.m_type) { case value_t::object: JSON_THROW(invalid_iterator::create(209, "cannot use offsets with object iterators", m_object)); @@ -13442,7 +13525,7 @@ class iter_impl // NOLINT(cppcoreguidelines-special-member-functions,hicpp-speci { JSON_ASSERT(m_object != nullptr); - switch (m_object->m_type) + switch (m_object->m_data.m_type) { case value_t::object: JSON_THROW(invalid_iterator::create(209, "cannot use offsets with object iterators", m_object)); @@ -13471,7 +13554,7 @@ class iter_impl // NOLINT(cppcoreguidelines-special-member-functions,hicpp-speci { JSON_ASSERT(m_object != nullptr); - switch (m_object->m_type) + switch (m_object->m_data.m_type) { case value_t::object: JSON_THROW(invalid_iterator::create(208, "cannot use operator[] for object iterators", m_object)); @@ -13541,10 +13624,10 @@ NLOHMANN_JSON_NAMESPACE_END // #include // __ _____ _____ _____ // __| | __| | | | JSON for Modern C++ -// | | |__ | | | | | | version 3.11.2 +// | | |__ | | | | | | version 3.11.3 // |_____|_____|_____|_|___| https://github.com/nlohmann/json // -// SPDX-FileCopyrightText: 2013-2022 Niels Lohmann +// SPDX-FileCopyrightText: 2013-2023 Niels Lohmann // SPDX-License-Identifier: MIT @@ -13673,13 +13756,55 @@ NLOHMANN_JSON_NAMESPACE_END // #include +// #include +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ +// | | |__ | | | | | | version 3.11.3 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013-2023 Niels Lohmann +// SPDX-License-Identifier: MIT + + + +#include // conditional, is_same + +// #include + + +NLOHMANN_JSON_NAMESPACE_BEGIN +namespace detail +{ + +/*! +@brief Default base class of the @ref basic_json class. + +So that the correct implementations of the copy / move ctors / assign operators +of @ref basic_json do not require complex case distinctions +(no base class / custom base class used as customization point), +@ref basic_json always has a base class. +By default, this class is used because it is empty and thus has no effect +on the behavior of @ref basic_json. +*/ +struct json_default_base {}; + +template +using json_base_class = typename std::conditional < + std::is_same::value, + json_default_base, + T + >::type; + +} // namespace detail +NLOHMANN_JSON_NAMESPACE_END + // #include // __ _____ _____ _____ // __| | __| | | | JSON for Modern C++ -// | | |__ | | | | | | version 3.11.2 +// | | |__ | | | | | | version 3.11.3 // |_____|_____|_____|_|___| https://github.com/nlohmann/json // -// SPDX-FileCopyrightText: 2013-2022 Niels Lohmann +// SPDX-FileCopyrightText: 2013-2023 Niels Lohmann // SPDX-License-Identifier: MIT @@ -13911,7 +14036,7 @@ class json_pointer const char* p = s.c_str(); char* p_end = nullptr; errno = 0; // strtoull doesn't reset errno - unsigned long long res = std::strtoull(p, &p_end, 10); // NOLINT(runtime/int) + const unsigned long long res = std::strtoull(p, &p_end, 10); // NOLINT(runtime/int) if (p == p_end // invalid input or empty string || errno == ERANGE // out of range || JSON_HEDLEY_UNLIKELY(static_cast(p_end - p) != s.size())) // incomplete read @@ -14067,7 +14192,7 @@ class json_pointer if (reference_token == "-") { // explicitly treat "-" as index beyond the end - ptr = &ptr->operator[](ptr->m_value.array->size()); + ptr = &ptr->operator[](ptr->m_data.m_value.array->size()); } else { @@ -14119,7 +14244,7 @@ class json_pointer { // "-" always fails the range check JSON_THROW(detail::out_of_range::create(402, detail::concat( - "array index '-' (", std::to_string(ptr->m_value.array->size()), + "array index '-' (", std::to_string(ptr->m_data.m_value.array->size()), ") is out of range"), ptr)); } @@ -14176,7 +14301,7 @@ class json_pointer if (JSON_HEDLEY_UNLIKELY(reference_token == "-")) { // "-" cannot be used for const access - JSON_THROW(detail::out_of_range::create(402, detail::concat("array index '-' (", std::to_string(ptr->m_value.array->size()), ") is out of range"), ptr)); + JSON_THROW(detail::out_of_range::create(402, detail::concat("array index '-' (", std::to_string(ptr->m_data.m_value.array->size()), ") is out of range"), ptr)); } // use unchecked array access @@ -14226,7 +14351,7 @@ class json_pointer { // "-" always fails the range check JSON_THROW(detail::out_of_range::create(402, detail::concat( - "array index '-' (", std::to_string(ptr->m_value.array->size()), + "array index '-' (", std::to_string(ptr->m_data.m_value.array->size()), ") is out of range"), ptr)); } @@ -14421,7 +14546,7 @@ class json_pointer { case detail::value_t::array: { - if (value.m_value.array->empty()) + if (value.m_data.m_value.array->empty()) { // flatten empty array as null result[reference_string] = nullptr; @@ -14429,10 +14554,10 @@ class json_pointer else { // iterate array and use index as reference string - for (std::size_t i = 0; i < value.m_value.array->size(); ++i) + for (std::size_t i = 0; i < value.m_data.m_value.array->size(); ++i) { flatten(detail::concat(reference_string, '/', std::to_string(i)), - value.m_value.array->operator[](i), result); + value.m_data.m_value.array->operator[](i), result); } } break; @@ -14440,7 +14565,7 @@ class json_pointer case detail::value_t::object: { - if (value.m_value.object->empty()) + if (value.m_data.m_value.object->empty()) { // flatten empty object as null result[reference_string] = nullptr; @@ -14448,7 +14573,7 @@ class json_pointer else { // iterate object and use keys as reference string - for (const auto& element : *value.m_value.object) + for (const auto& element : *value.m_data.m_value.object) { flatten(detail::concat(reference_string, '/', detail::escape(element.first)), element.second, result); } @@ -14495,7 +14620,7 @@ class json_pointer BasicJsonType result; // iterate the JSON object values - for (const auto& element : *value.m_value.object) + for (const auto& element : *value.m_data.m_value.object) { if (JSON_HEDLEY_UNLIKELY(!element.second.is_primitive())) { @@ -14671,10 +14796,10 @@ NLOHMANN_JSON_NAMESPACE_END // #include // __ _____ _____ _____ // __| | __| | | | JSON for Modern C++ -// | | |__ | | | | | | version 3.11.2 +// | | |__ | | | | | | version 3.11.3 // |_____|_____|_____|_|___| https://github.com/nlohmann/json // -// SPDX-FileCopyrightText: 2013-2022 Niels Lohmann +// SPDX-FileCopyrightText: 2013-2023 Niels Lohmann // SPDX-License-Identifier: MIT @@ -14763,10 +14888,10 @@ NLOHMANN_JSON_NAMESPACE_END // #include // __ _____ _____ _____ // __| | __| | | | JSON for Modern C++ -// | | |__ | | | | | | version 3.11.2 +// | | |__ | | | | | | version 3.11.3 // |_____|_____|_____|_|___| https://github.com/nlohmann/json // -// SPDX-FileCopyrightText: 2013-2022 Niels Lohmann +// SPDX-FileCopyrightText: 2013-2023 Niels Lohmann // SPDX-License-Identifier: MIT @@ -14789,10 +14914,10 @@ NLOHMANN_JSON_NAMESPACE_END // #include // __ _____ _____ _____ // __| | __| | | | JSON for Modern C++ -// | | |__ | | | | | | version 3.11.2 +// | | |__ | | | | | | version 3.11.3 // |_____|_____|_____|_|___| https://github.com/nlohmann/json // -// SPDX-FileCopyrightText: 2013-2022 Niels Lohmann +// SPDX-FileCopyrightText: 2013-2023 Niels Lohmann // SPDX-License-Identifier: MIT @@ -14978,7 +15103,7 @@ class binary_writer { case value_t::object: { - write_bson_object(*j.m_value.object); + write_bson_object(*j.m_data.m_value.object); break; } @@ -15013,7 +15138,7 @@ class binary_writer case value_t::boolean: { - oa->write_character(j.m_value.boolean + oa->write_character(j.m_data.m_value.boolean ? to_char_type(0xF5) : to_char_type(0xF4)); break; @@ -15021,42 +15146,42 @@ class binary_writer case value_t::number_integer: { - if (j.m_value.number_integer >= 0) + if (j.m_data.m_value.number_integer >= 0) { // CBOR does not differentiate between positive signed // integers and unsigned integers. Therefore, we used the // code from the value_t::number_unsigned case here. - if (j.m_value.number_integer <= 0x17) + if (j.m_data.m_value.number_integer <= 0x17) { - write_number(static_cast(j.m_value.number_integer)); + write_number(static_cast(j.m_data.m_value.number_integer)); } - else if (j.m_value.number_integer <= (std::numeric_limits::max)()) + else if (j.m_data.m_value.number_integer <= (std::numeric_limits::max)()) { oa->write_character(to_char_type(0x18)); - write_number(static_cast(j.m_value.number_integer)); + write_number(static_cast(j.m_data.m_value.number_integer)); } - else if (j.m_value.number_integer <= (std::numeric_limits::max)()) + else if (j.m_data.m_value.number_integer <= (std::numeric_limits::max)()) { oa->write_character(to_char_type(0x19)); - write_number(static_cast(j.m_value.number_integer)); + write_number(static_cast(j.m_data.m_value.number_integer)); } - else if (j.m_value.number_integer <= (std::numeric_limits::max)()) + else if (j.m_data.m_value.number_integer <= (std::numeric_limits::max)()) { oa->write_character(to_char_type(0x1A)); - write_number(static_cast(j.m_value.number_integer)); + write_number(static_cast(j.m_data.m_value.number_integer)); } else { oa->write_character(to_char_type(0x1B)); - write_number(static_cast(j.m_value.number_integer)); + write_number(static_cast(j.m_data.m_value.number_integer)); } } else { // The conversions below encode the sign in the first // byte, and the value is converted to a positive number. - const auto positive_number = -1 - j.m_value.number_integer; - if (j.m_value.number_integer >= -24) + const auto positive_number = -1 - j.m_data.m_value.number_integer; + if (j.m_data.m_value.number_integer >= -24) { write_number(static_cast(0x20 + positive_number)); } @@ -15086,52 +15211,52 @@ class binary_writer case value_t::number_unsigned: { - if (j.m_value.number_unsigned <= 0x17) + if (j.m_data.m_value.number_unsigned <= 0x17) { - write_number(static_cast(j.m_value.number_unsigned)); + write_number(static_cast(j.m_data.m_value.number_unsigned)); } - else if (j.m_value.number_unsigned <= (std::numeric_limits::max)()) + else if (j.m_data.m_value.number_unsigned <= (std::numeric_limits::max)()) { oa->write_character(to_char_type(0x18)); - write_number(static_cast(j.m_value.number_unsigned)); + write_number(static_cast(j.m_data.m_value.number_unsigned)); } - else if (j.m_value.number_unsigned <= (std::numeric_limits::max)()) + else if (j.m_data.m_value.number_unsigned <= (std::numeric_limits::max)()) { oa->write_character(to_char_type(0x19)); - write_number(static_cast(j.m_value.number_unsigned)); + write_number(static_cast(j.m_data.m_value.number_unsigned)); } - else if (j.m_value.number_unsigned <= (std::numeric_limits::max)()) + else if (j.m_data.m_value.number_unsigned <= (std::numeric_limits::max)()) { oa->write_character(to_char_type(0x1A)); - write_number(static_cast(j.m_value.number_unsigned)); + write_number(static_cast(j.m_data.m_value.number_unsigned)); } else { oa->write_character(to_char_type(0x1B)); - write_number(static_cast(j.m_value.number_unsigned)); + write_number(static_cast(j.m_data.m_value.number_unsigned)); } break; } case value_t::number_float: { - if (std::isnan(j.m_value.number_float)) + if (std::isnan(j.m_data.m_value.number_float)) { // NaN is 0xf97e00 in CBOR oa->write_character(to_char_type(0xF9)); oa->write_character(to_char_type(0x7E)); oa->write_character(to_char_type(0x00)); } - else if (std::isinf(j.m_value.number_float)) + else if (std::isinf(j.m_data.m_value.number_float)) { // Infinity is 0xf97c00, -Infinity is 0xf9fc00 oa->write_character(to_char_type(0xf9)); - oa->write_character(j.m_value.number_float > 0 ? to_char_type(0x7C) : to_char_type(0xFC)); + oa->write_character(j.m_data.m_value.number_float > 0 ? to_char_type(0x7C) : to_char_type(0xFC)); oa->write_character(to_char_type(0x00)); } else { - write_compact_float(j.m_value.number_float, detail::input_format_t::cbor); + write_compact_float(j.m_data.m_value.number_float, detail::input_format_t::cbor); } break; } @@ -15139,7 +15264,7 @@ class binary_writer case value_t::string: { // step 1: write control byte and the string length - const auto N = j.m_value.string->size(); + const auto N = j.m_data.m_value.string->size(); if (N <= 0x17) { write_number(static_cast(0x60 + N)); @@ -15169,15 +15294,15 @@ class binary_writer // step 2: write the string oa->write_characters( - reinterpret_cast(j.m_value.string->c_str()), - j.m_value.string->size()); + reinterpret_cast(j.m_data.m_value.string->c_str()), + j.m_data.m_value.string->size()); break; } case value_t::array: { // step 1: write control byte and the array size - const auto N = j.m_value.array->size(); + const auto N = j.m_data.m_value.array->size(); if (N <= 0x17) { write_number(static_cast(0x80 + N)); @@ -15206,7 +15331,7 @@ class binary_writer // LCOV_EXCL_STOP // step 2: write each element - for (const auto& el : *j.m_value.array) + for (const auto& el : *j.m_data.m_value.array) { write_cbor(el); } @@ -15215,32 +15340,32 @@ class binary_writer case value_t::binary: { - if (j.m_value.binary->has_subtype()) + if (j.m_data.m_value.binary->has_subtype()) { - if (j.m_value.binary->subtype() <= (std::numeric_limits::max)()) + if (j.m_data.m_value.binary->subtype() <= (std::numeric_limits::max)()) { write_number(static_cast(0xd8)); - write_number(static_cast(j.m_value.binary->subtype())); + write_number(static_cast(j.m_data.m_value.binary->subtype())); } - else if (j.m_value.binary->subtype() <= (std::numeric_limits::max)()) + else if (j.m_data.m_value.binary->subtype() <= (std::numeric_limits::max)()) { write_number(static_cast(0xd9)); - write_number(static_cast(j.m_value.binary->subtype())); + write_number(static_cast(j.m_data.m_value.binary->subtype())); } - else if (j.m_value.binary->subtype() <= (std::numeric_limits::max)()) + else if (j.m_data.m_value.binary->subtype() <= (std::numeric_limits::max)()) { write_number(static_cast(0xda)); - write_number(static_cast(j.m_value.binary->subtype())); + write_number(static_cast(j.m_data.m_value.binary->subtype())); } - else if (j.m_value.binary->subtype() <= (std::numeric_limits::max)()) + else if (j.m_data.m_value.binary->subtype() <= (std::numeric_limits::max)()) { write_number(static_cast(0xdb)); - write_number(static_cast(j.m_value.binary->subtype())); + write_number(static_cast(j.m_data.m_value.binary->subtype())); } } // step 1: write control byte and the binary array size - const auto N = j.m_value.binary->size(); + const auto N = j.m_data.m_value.binary->size(); if (N <= 0x17) { write_number(static_cast(0x40 + N)); @@ -15270,7 +15395,7 @@ class binary_writer // step 2: write each element oa->write_characters( - reinterpret_cast(j.m_value.binary->data()), + reinterpret_cast(j.m_data.m_value.binary->data()), N); break; @@ -15279,7 +15404,7 @@ class binary_writer case value_t::object: { // step 1: write control byte and the object size - const auto N = j.m_value.object->size(); + const auto N = j.m_data.m_value.object->size(); if (N <= 0x17) { write_number(static_cast(0xA0 + N)); @@ -15308,7 +15433,7 @@ class binary_writer // LCOV_EXCL_STOP // step 2: write each element - for (const auto& el : *j.m_value.object) + for (const auto& el : *j.m_data.m_value.object) { write_cbor(el.first); write_cbor(el.second); @@ -15337,7 +15462,7 @@ class binary_writer case value_t::boolean: // true and false { - oa->write_character(j.m_value.boolean + oa->write_character(j.m_data.m_value.boolean ? to_char_type(0xC3) : to_char_type(0xC2)); break; @@ -15345,75 +15470,75 @@ class binary_writer case value_t::number_integer: { - if (j.m_value.number_integer >= 0) + if (j.m_data.m_value.number_integer >= 0) { // MessagePack does not differentiate between positive // signed integers and unsigned integers. Therefore, we used // the code from the value_t::number_unsigned case here. - if (j.m_value.number_unsigned < 128) + if (j.m_data.m_value.number_unsigned < 128) { // positive fixnum - write_number(static_cast(j.m_value.number_integer)); + write_number(static_cast(j.m_data.m_value.number_integer)); } - else if (j.m_value.number_unsigned <= (std::numeric_limits::max)()) + else if (j.m_data.m_value.number_unsigned <= (std::numeric_limits::max)()) { // uint 8 oa->write_character(to_char_type(0xCC)); - write_number(static_cast(j.m_value.number_integer)); + write_number(static_cast(j.m_data.m_value.number_integer)); } - else if (j.m_value.number_unsigned <= (std::numeric_limits::max)()) + else if (j.m_data.m_value.number_unsigned <= (std::numeric_limits::max)()) { // uint 16 oa->write_character(to_char_type(0xCD)); - write_number(static_cast(j.m_value.number_integer)); + write_number(static_cast(j.m_data.m_value.number_integer)); } - else if (j.m_value.number_unsigned <= (std::numeric_limits::max)()) + else if (j.m_data.m_value.number_unsigned <= (std::numeric_limits::max)()) { // uint 32 oa->write_character(to_char_type(0xCE)); - write_number(static_cast(j.m_value.number_integer)); + write_number(static_cast(j.m_data.m_value.number_integer)); } - else if (j.m_value.number_unsigned <= (std::numeric_limits::max)()) + else if (j.m_data.m_value.number_unsigned <= (std::numeric_limits::max)()) { // uint 64 oa->write_character(to_char_type(0xCF)); - write_number(static_cast(j.m_value.number_integer)); + write_number(static_cast(j.m_data.m_value.number_integer)); } } else { - if (j.m_value.number_integer >= -32) + if (j.m_data.m_value.number_integer >= -32) { // negative fixnum - write_number(static_cast(j.m_value.number_integer)); + write_number(static_cast(j.m_data.m_value.number_integer)); } - else if (j.m_value.number_integer >= (std::numeric_limits::min)() && - j.m_value.number_integer <= (std::numeric_limits::max)()) + else if (j.m_data.m_value.number_integer >= (std::numeric_limits::min)() && + j.m_data.m_value.number_integer <= (std::numeric_limits::max)()) { // int 8 oa->write_character(to_char_type(0xD0)); - write_number(static_cast(j.m_value.number_integer)); + write_number(static_cast(j.m_data.m_value.number_integer)); } - else if (j.m_value.number_integer >= (std::numeric_limits::min)() && - j.m_value.number_integer <= (std::numeric_limits::max)()) + else if (j.m_data.m_value.number_integer >= (std::numeric_limits::min)() && + j.m_data.m_value.number_integer <= (std::numeric_limits::max)()) { // int 16 oa->write_character(to_char_type(0xD1)); - write_number(static_cast(j.m_value.number_integer)); + write_number(static_cast(j.m_data.m_value.number_integer)); } - else if (j.m_value.number_integer >= (std::numeric_limits::min)() && - j.m_value.number_integer <= (std::numeric_limits::max)()) + else if (j.m_data.m_value.number_integer >= (std::numeric_limits::min)() && + j.m_data.m_value.number_integer <= (std::numeric_limits::max)()) { // int 32 oa->write_character(to_char_type(0xD2)); - write_number(static_cast(j.m_value.number_integer)); + write_number(static_cast(j.m_data.m_value.number_integer)); } - else if (j.m_value.number_integer >= (std::numeric_limits::min)() && - j.m_value.number_integer <= (std::numeric_limits::max)()) + else if (j.m_data.m_value.number_integer >= (std::numeric_limits::min)() && + j.m_data.m_value.number_integer <= (std::numeric_limits::max)()) { // int 64 oa->write_character(to_char_type(0xD3)); - write_number(static_cast(j.m_value.number_integer)); + write_number(static_cast(j.m_data.m_value.number_integer)); } } break; @@ -15421,48 +15546,48 @@ class binary_writer case value_t::number_unsigned: { - if (j.m_value.number_unsigned < 128) + if (j.m_data.m_value.number_unsigned < 128) { // positive fixnum - write_number(static_cast(j.m_value.number_integer)); + write_number(static_cast(j.m_data.m_value.number_integer)); } - else if (j.m_value.number_unsigned <= (std::numeric_limits::max)()) + else if (j.m_data.m_value.number_unsigned <= (std::numeric_limits::max)()) { // uint 8 oa->write_character(to_char_type(0xCC)); - write_number(static_cast(j.m_value.number_integer)); + write_number(static_cast(j.m_data.m_value.number_integer)); } - else if (j.m_value.number_unsigned <= (std::numeric_limits::max)()) + else if (j.m_data.m_value.number_unsigned <= (std::numeric_limits::max)()) { // uint 16 oa->write_character(to_char_type(0xCD)); - write_number(static_cast(j.m_value.number_integer)); + write_number(static_cast(j.m_data.m_value.number_integer)); } - else if (j.m_value.number_unsigned <= (std::numeric_limits::max)()) + else if (j.m_data.m_value.number_unsigned <= (std::numeric_limits::max)()) { // uint 32 oa->write_character(to_char_type(0xCE)); - write_number(static_cast(j.m_value.number_integer)); + write_number(static_cast(j.m_data.m_value.number_integer)); } - else if (j.m_value.number_unsigned <= (std::numeric_limits::max)()) + else if (j.m_data.m_value.number_unsigned <= (std::numeric_limits::max)()) { // uint 64 oa->write_character(to_char_type(0xCF)); - write_number(static_cast(j.m_value.number_integer)); + write_number(static_cast(j.m_data.m_value.number_integer)); } break; } case value_t::number_float: { - write_compact_float(j.m_value.number_float, detail::input_format_t::msgpack); + write_compact_float(j.m_data.m_value.number_float, detail::input_format_t::msgpack); break; } case value_t::string: { // step 1: write control byte and the string length - const auto N = j.m_value.string->size(); + const auto N = j.m_data.m_value.string->size(); if (N <= 31) { // fixstr @@ -15489,15 +15614,15 @@ class binary_writer // step 2: write the string oa->write_characters( - reinterpret_cast(j.m_value.string->c_str()), - j.m_value.string->size()); + reinterpret_cast(j.m_data.m_value.string->c_str()), + j.m_data.m_value.string->size()); break; } case value_t::array: { // step 1: write control byte and the array size - const auto N = j.m_value.array->size(); + const auto N = j.m_data.m_value.array->size(); if (N <= 15) { // fixarray @@ -15517,7 +15642,7 @@ class binary_writer } // step 2: write each element - for (const auto& el : *j.m_value.array) + for (const auto& el : *j.m_data.m_value.array) { write_msgpack(el); } @@ -15528,10 +15653,10 @@ class binary_writer { // step 0: determine if the binary type has a set subtype to // determine whether or not to use the ext or fixext types - const bool use_ext = j.m_value.binary->has_subtype(); + const bool use_ext = j.m_data.m_value.binary->has_subtype(); // step 1: write control byte and the byte string length - const auto N = j.m_value.binary->size(); + const auto N = j.m_data.m_value.binary->size(); if (N <= (std::numeric_limits::max)()) { std::uint8_t output_type{}; @@ -15576,18 +15701,18 @@ class binary_writer } else if (N <= (std::numeric_limits::max)()) { - std::uint8_t output_type = use_ext - ? 0xC8 // ext 16 - : 0xC5; // bin 16 + const std::uint8_t output_type = use_ext + ? 0xC8 // ext 16 + : 0xC5; // bin 16 oa->write_character(to_char_type(output_type)); write_number(static_cast(N)); } else if (N <= (std::numeric_limits::max)()) { - std::uint8_t output_type = use_ext - ? 0xC9 // ext 32 - : 0xC6; // bin 32 + const std::uint8_t output_type = use_ext + ? 0xC9 // ext 32 + : 0xC6; // bin 32 oa->write_character(to_char_type(output_type)); write_number(static_cast(N)); @@ -15596,12 +15721,12 @@ class binary_writer // step 1.5: if this is an ext type, write the subtype if (use_ext) { - write_number(static_cast(j.m_value.binary->subtype())); + write_number(static_cast(j.m_data.m_value.binary->subtype())); } // step 2: write the byte string oa->write_characters( - reinterpret_cast(j.m_value.binary->data()), + reinterpret_cast(j.m_data.m_value.binary->data()), N); break; @@ -15610,7 +15735,7 @@ class binary_writer case value_t::object: { // step 1: write control byte and the object size - const auto N = j.m_value.object->size(); + const auto N = j.m_data.m_value.object->size(); if (N <= 15) { // fixmap @@ -15630,7 +15755,7 @@ class binary_writer } // step 2: write each element - for (const auto& el : *j.m_value.object) + for (const auto& el : *j.m_data.m_value.object) { write_msgpack(el.first); write_msgpack(el.second); @@ -15670,7 +15795,7 @@ class binary_writer { if (add_prefix) { - oa->write_character(j.m_value.boolean + oa->write_character(j.m_data.m_value.boolean ? to_char_type('T') : to_char_type('F')); } @@ -15679,19 +15804,19 @@ class binary_writer case value_t::number_integer: { - write_number_with_ubjson_prefix(j.m_value.number_integer, add_prefix, use_bjdata); + write_number_with_ubjson_prefix(j.m_data.m_value.number_integer, add_prefix, use_bjdata); break; } case value_t::number_unsigned: { - write_number_with_ubjson_prefix(j.m_value.number_unsigned, add_prefix, use_bjdata); + write_number_with_ubjson_prefix(j.m_data.m_value.number_unsigned, add_prefix, use_bjdata); break; } case value_t::number_float: { - write_number_with_ubjson_prefix(j.m_value.number_float, add_prefix, use_bjdata); + write_number_with_ubjson_prefix(j.m_data.m_value.number_float, add_prefix, use_bjdata); break; } @@ -15701,10 +15826,10 @@ class binary_writer { oa->write_character(to_char_type('S')); } - write_number_with_ubjson_prefix(j.m_value.string->size(), true, use_bjdata); + write_number_with_ubjson_prefix(j.m_data.m_value.string->size(), true, use_bjdata); oa->write_characters( - reinterpret_cast(j.m_value.string->c_str()), - j.m_value.string->size()); + reinterpret_cast(j.m_data.m_value.string->c_str()), + j.m_data.m_value.string->size()); break; } @@ -15716,7 +15841,7 @@ class binary_writer } bool prefix_required = true; - if (use_type && !j.m_value.array->empty()) + if (use_type && !j.m_data.m_value.array->empty()) { JSON_ASSERT(use_count); const CharType first_prefix = ubjson_prefix(j.front(), use_bjdata); @@ -15739,10 +15864,10 @@ class binary_writer if (use_count) { oa->write_character(to_char_type('#')); - write_number_with_ubjson_prefix(j.m_value.array->size(), true, use_bjdata); + write_number_with_ubjson_prefix(j.m_data.m_value.array->size(), true, use_bjdata); } - for (const auto& el : *j.m_value.array) + for (const auto& el : *j.m_data.m_value.array) { write_ubjson(el, use_count, use_type, prefix_required, use_bjdata); } @@ -15762,7 +15887,7 @@ class binary_writer oa->write_character(to_char_type('[')); } - if (use_type && !j.m_value.binary->empty()) + if (use_type && !j.m_data.m_value.binary->empty()) { JSON_ASSERT(use_count); oa->write_character(to_char_type('$')); @@ -15772,21 +15897,21 @@ class binary_writer if (use_count) { oa->write_character(to_char_type('#')); - write_number_with_ubjson_prefix(j.m_value.binary->size(), true, use_bjdata); + write_number_with_ubjson_prefix(j.m_data.m_value.binary->size(), true, use_bjdata); } if (use_type) { oa->write_characters( - reinterpret_cast(j.m_value.binary->data()), - j.m_value.binary->size()); + reinterpret_cast(j.m_data.m_value.binary->data()), + j.m_data.m_value.binary->size()); } else { - for (size_t i = 0; i < j.m_value.binary->size(); ++i) + for (size_t i = 0; i < j.m_data.m_value.binary->size(); ++i) { oa->write_character(to_char_type('U')); - oa->write_character(j.m_value.binary->data()[i]); + oa->write_character(j.m_data.m_value.binary->data()[i]); } } @@ -15800,9 +15925,9 @@ class binary_writer case value_t::object: { - if (use_bjdata && j.m_value.object->size() == 3 && j.m_value.object->find("_ArrayType_") != j.m_value.object->end() && j.m_value.object->find("_ArraySize_") != j.m_value.object->end() && j.m_value.object->find("_ArrayData_") != j.m_value.object->end()) + if (use_bjdata && j.m_data.m_value.object->size() == 3 && j.m_data.m_value.object->find("_ArrayType_") != j.m_data.m_value.object->end() && j.m_data.m_value.object->find("_ArraySize_") != j.m_data.m_value.object->end() && j.m_data.m_value.object->find("_ArrayData_") != j.m_data.m_value.object->end()) { - if (!write_bjdata_ndarray(*j.m_value.object, use_count, use_type)) // decode bjdata ndarray in the JData format (https://github.com/NeuroJSON/jdata) + if (!write_bjdata_ndarray(*j.m_data.m_value.object, use_count, use_type)) // decode bjdata ndarray in the JData format (https://github.com/NeuroJSON/jdata) { break; } @@ -15814,7 +15939,7 @@ class binary_writer } bool prefix_required = true; - if (use_type && !j.m_value.object->empty()) + if (use_type && !j.m_data.m_value.object->empty()) { JSON_ASSERT(use_count); const CharType first_prefix = ubjson_prefix(j.front(), use_bjdata); @@ -15837,10 +15962,10 @@ class binary_writer if (use_count) { oa->write_character(to_char_type('#')); - write_number_with_ubjson_prefix(j.m_value.object->size(), true, use_bjdata); + write_number_with_ubjson_prefix(j.m_data.m_value.object->size(), true, use_bjdata); } - for (const auto& el : *j.m_value.object) + for (const auto& el : *j.m_data.m_value.object) { write_number_with_ubjson_prefix(el.first.size(), true, use_bjdata); oa->write_characters( @@ -15990,19 +16115,19 @@ class binary_writer void write_bson_unsigned(const string_t& name, const BasicJsonType& j) { - if (j.m_value.number_unsigned <= static_cast((std::numeric_limits::max)())) + if (j.m_data.m_value.number_unsigned <= static_cast((std::numeric_limits::max)())) { write_bson_entry_header(name, 0x10 /* int32 */); - write_number(static_cast(j.m_value.number_unsigned), true); + write_number(static_cast(j.m_data.m_value.number_unsigned), true); } - else if (j.m_value.number_unsigned <= static_cast((std::numeric_limits::max)())) + else if (j.m_data.m_value.number_unsigned <= static_cast((std::numeric_limits::max)())) { write_bson_entry_header(name, 0x12 /* int64 */); - write_number(static_cast(j.m_value.number_unsigned), true); + write_number(static_cast(j.m_data.m_value.number_unsigned), true); } else { - JSON_THROW(out_of_range::create(407, concat("integer number ", std::to_string(j.m_value.number_unsigned), " cannot be represented by BSON as it does not fit int64"), &j)); + JSON_THROW(out_of_range::create(407, concat("integer number ", std::to_string(j.m_data.m_value.number_unsigned), " cannot be represented by BSON as it does not fit int64"), &j)); } } @@ -16083,13 +16208,13 @@ class binary_writer switch (j.type()) { case value_t::object: - return header_size + calc_bson_object_size(*j.m_value.object); + return header_size + calc_bson_object_size(*j.m_data.m_value.object); case value_t::array: - return header_size + calc_bson_array_size(*j.m_value.array); + return header_size + calc_bson_array_size(*j.m_data.m_value.array); case value_t::binary: - return header_size + calc_bson_binary_size(*j.m_value.binary); + return header_size + calc_bson_binary_size(*j.m_data.m_value.binary); case value_t::boolean: return header_size + 1ul; @@ -16098,13 +16223,13 @@ class binary_writer return header_size + 8ul; case value_t::number_integer: - return header_size + calc_bson_integer_size(j.m_value.number_integer); + return header_size + calc_bson_integer_size(j.m_data.m_value.number_integer); case value_t::number_unsigned: - return header_size + calc_bson_unsigned_size(j.m_value.number_unsigned); + return header_size + calc_bson_unsigned_size(j.m_data.m_value.number_unsigned); case value_t::string: - return header_size + calc_bson_string_size(*j.m_value.string); + return header_size + calc_bson_string_size(*j.m_data.m_value.string); case value_t::null: return header_size + 0ul; @@ -16130,28 +16255,28 @@ class binary_writer switch (j.type()) { case value_t::object: - return write_bson_object_entry(name, *j.m_value.object); + return write_bson_object_entry(name, *j.m_data.m_value.object); case value_t::array: - return write_bson_array(name, *j.m_value.array); + return write_bson_array(name, *j.m_data.m_value.array); case value_t::binary: - return write_bson_binary(name, *j.m_value.binary); + return write_bson_binary(name, *j.m_data.m_value.binary); case value_t::boolean: - return write_bson_boolean(name, j.m_value.boolean); + return write_bson_boolean(name, j.m_data.m_value.boolean); case value_t::number_float: - return write_bson_double(name, j.m_value.number_float); + return write_bson_double(name, j.m_data.m_value.number_float); case value_t::number_integer: - return write_bson_integer(name, j.m_value.number_integer); + return write_bson_integer(name, j.m_data.m_value.number_integer); case value_t::number_unsigned: return write_bson_unsigned(name, j); case value_t::string: - return write_bson_string(name, *j.m_value.string); + return write_bson_string(name, *j.m_data.m_value.string); case value_t::null: return write_bson_null(name); @@ -16173,8 +16298,8 @@ class binary_writer */ static std::size_t calc_bson_object_size(const typename BasicJsonType::object_t& value) { - std::size_t document_size = std::accumulate(value.begin(), value.end(), static_cast(0), - [](size_t result, const typename BasicJsonType::object_t::value_type & el) + const std::size_t document_size = std::accumulate(value.begin(), value.end(), static_cast(0), + [](size_t result, const typename BasicJsonType::object_t::value_type & el) { return result += calc_bson_element_size(el.first, el.second); }); @@ -16424,35 +16549,35 @@ class binary_writer return 'Z'; case value_t::boolean: - return j.m_value.boolean ? 'T' : 'F'; + return j.m_data.m_value.boolean ? 'T' : 'F'; case value_t::number_integer: { - if ((std::numeric_limits::min)() <= j.m_value.number_integer && j.m_value.number_integer <= (std::numeric_limits::max)()) + if ((std::numeric_limits::min)() <= j.m_data.m_value.number_integer && j.m_data.m_value.number_integer <= (std::numeric_limits::max)()) { return 'i'; } - if ((std::numeric_limits::min)() <= j.m_value.number_integer && j.m_value.number_integer <= (std::numeric_limits::max)()) + if ((std::numeric_limits::min)() <= j.m_data.m_value.number_integer && j.m_data.m_value.number_integer <= (std::numeric_limits::max)()) { return 'U'; } - if ((std::numeric_limits::min)() <= j.m_value.number_integer && j.m_value.number_integer <= (std::numeric_limits::max)()) + if ((std::numeric_limits::min)() <= j.m_data.m_value.number_integer && j.m_data.m_value.number_integer <= (std::numeric_limits::max)()) { return 'I'; } - if (use_bjdata && ((std::numeric_limits::min)() <= j.m_value.number_integer && j.m_value.number_integer <= (std::numeric_limits::max)())) + if (use_bjdata && ((std::numeric_limits::min)() <= j.m_data.m_value.number_integer && j.m_data.m_value.number_integer <= (std::numeric_limits::max)())) { return 'u'; } - if ((std::numeric_limits::min)() <= j.m_value.number_integer && j.m_value.number_integer <= (std::numeric_limits::max)()) + if ((std::numeric_limits::min)() <= j.m_data.m_value.number_integer && j.m_data.m_value.number_integer <= (std::numeric_limits::max)()) { return 'l'; } - if (use_bjdata && ((std::numeric_limits::min)() <= j.m_value.number_integer && j.m_value.number_integer <= (std::numeric_limits::max)())) + if (use_bjdata && ((std::numeric_limits::min)() <= j.m_data.m_value.number_integer && j.m_data.m_value.number_integer <= (std::numeric_limits::max)())) { return 'm'; } - if ((std::numeric_limits::min)() <= j.m_value.number_integer && j.m_value.number_integer <= (std::numeric_limits::max)()) + if ((std::numeric_limits::min)() <= j.m_data.m_value.number_integer && j.m_data.m_value.number_integer <= (std::numeric_limits::max)()) { return 'L'; } @@ -16462,35 +16587,35 @@ class binary_writer case value_t::number_unsigned: { - if (j.m_value.number_unsigned <= static_cast((std::numeric_limits::max)())) + if (j.m_data.m_value.number_unsigned <= static_cast((std::numeric_limits::max)())) { return 'i'; } - if (j.m_value.number_unsigned <= static_cast((std::numeric_limits::max)())) + if (j.m_data.m_value.number_unsigned <= static_cast((std::numeric_limits::max)())) { return 'U'; } - if (j.m_value.number_unsigned <= static_cast((std::numeric_limits::max)())) + if (j.m_data.m_value.number_unsigned <= static_cast((std::numeric_limits::max)())) { return 'I'; } - if (use_bjdata && j.m_value.number_unsigned <= static_cast((std::numeric_limits::max)())) + if (use_bjdata && j.m_data.m_value.number_unsigned <= static_cast((std::numeric_limits::max)())) { return 'u'; } - if (j.m_value.number_unsigned <= static_cast((std::numeric_limits::max)())) + if (j.m_data.m_value.number_unsigned <= static_cast((std::numeric_limits::max)())) { return 'l'; } - if (use_bjdata && j.m_value.number_unsigned <= static_cast((std::numeric_limits::max)())) + if (use_bjdata && j.m_data.m_value.number_unsigned <= static_cast((std::numeric_limits::max)())) { return 'm'; } - if (j.m_value.number_unsigned <= static_cast((std::numeric_limits::max)())) + if (j.m_data.m_value.number_unsigned <= static_cast((std::numeric_limits::max)())) { return 'L'; } - if (use_bjdata && j.m_value.number_unsigned <= (std::numeric_limits::max)()) + if (use_bjdata && j.m_data.m_value.number_unsigned <= (std::numeric_limits::max)()) { return 'M'; } @@ -16499,7 +16624,7 @@ class binary_writer } case value_t::number_float: - return get_ubjson_float_prefix(j.m_value.number_float); + return get_ubjson_float_prefix(j.m_data.m_value.number_float); case value_t::string: return 'S'; @@ -16548,7 +16673,7 @@ class binary_writer std::size_t len = (value.at(key).empty() ? 0 : 1); for (const auto& el : value.at(key)) { - len *= static_cast(el.m_value.number_unsigned); + len *= static_cast(el.m_data.m_value.number_unsigned); } key = "_ArrayData_"; @@ -16570,70 +16695,70 @@ class binary_writer { for (const auto& el : value.at(key)) { - write_number(static_cast(el.m_value.number_unsigned), true); + write_number(static_cast(el.m_data.m_value.number_unsigned), true); } } else if (dtype == 'i') { for (const auto& el : value.at(key)) { - write_number(static_cast(el.m_value.number_integer), true); + write_number(static_cast(el.m_data.m_value.number_integer), true); } } else if (dtype == 'u') { for (const auto& el : value.at(key)) { - write_number(static_cast(el.m_value.number_unsigned), true); + write_number(static_cast(el.m_data.m_value.number_unsigned), true); } } else if (dtype == 'I') { for (const auto& el : value.at(key)) { - write_number(static_cast(el.m_value.number_integer), true); + write_number(static_cast(el.m_data.m_value.number_integer), true); } } else if (dtype == 'm') { for (const auto& el : value.at(key)) { - write_number(static_cast(el.m_value.number_unsigned), true); + write_number(static_cast(el.m_data.m_value.number_unsigned), true); } } else if (dtype == 'l') { for (const auto& el : value.at(key)) { - write_number(static_cast(el.m_value.number_integer), true); + write_number(static_cast(el.m_data.m_value.number_integer), true); } } else if (dtype == 'M') { for (const auto& el : value.at(key)) { - write_number(static_cast(el.m_value.number_unsigned), true); + write_number(static_cast(el.m_data.m_value.number_unsigned), true); } } else if (dtype == 'L') { for (const auto& el : value.at(key)) { - write_number(static_cast(el.m_value.number_integer), true); + write_number(static_cast(el.m_data.m_value.number_integer), true); } } else if (dtype == 'd') { for (const auto& el : value.at(key)) { - write_number(static_cast(el.m_value.number_float), true); + write_number(static_cast(el.m_data.m_value.number_float), true); } } else if (dtype == 'D') { for (const auto& el : value.at(key)) { - write_number(static_cast(el.m_value.number_float), true); + write_number(static_cast(el.m_data.m_value.number_float), true); } } return false; @@ -16757,11 +16882,11 @@ NLOHMANN_JSON_NAMESPACE_END // #include // __ _____ _____ _____ // __| | __| | | | JSON for Modern C++ -// | | |__ | | | | | | version 3.11.2 +// | | |__ | | | | | | version 3.11.3 // |_____|_____|_____|_|___| https://github.com/nlohmann/json // // SPDX-FileCopyrightText: 2008-2009 Björn Hoehrmann -// SPDX-FileCopyrightText: 2013-2022 Niels Lohmann +// SPDX-FileCopyrightText: 2013-2023 Niels Lohmann // SPDX-License-Identifier: MIT @@ -16782,11 +16907,11 @@ NLOHMANN_JSON_NAMESPACE_END // #include // __ _____ _____ _____ // __| | __| | | | JSON for Modern C++ -// | | |__ | | | | | | version 3.11.2 +// | | |__ | | | | | | version 3.11.3 // |_____|_____|_____|_|___| https://github.com/nlohmann/json // // SPDX-FileCopyrightText: 2009 Florian Loitsch -// SPDX-FileCopyrightText: 2013-2022 Niels Lohmann +// SPDX-FileCopyrightText: 2013-2023 Niels Lohmann // SPDX-License-Identifier: MIT @@ -17692,7 +17817,7 @@ void grisu2(char* buf, int& len, int& decimal_exponent, FloatType value) // NB: If the neighbors are computed for single-precision numbers, there is a single float // (7.0385307e-26f) which can't be recovered using strtod. The resulting double precision // value is off by 1 ulp. -#if 0 +#if 0 // NOLINT(readability-avoid-unconditional-preprocessor-if) const boundaries w = compute_boundaries(static_cast(value)); #else const boundaries w = compute_boundaries(value); @@ -17994,11 +18119,11 @@ class serializer const unsigned int indent_step, const unsigned int current_indent = 0) { - switch (val.m_type) + switch (val.m_data.m_type) { case value_t::object: { - if (val.m_value.object->empty()) + if (val.m_data.m_value.object->empty()) { o->write_characters("{}", 2); return; @@ -18016,8 +18141,8 @@ class serializer } // first n-1 elements - auto i = val.m_value.object->cbegin(); - for (std::size_t cnt = 0; cnt < val.m_value.object->size() - 1; ++cnt, ++i) + auto i = val.m_data.m_value.object->cbegin(); + for (std::size_t cnt = 0; cnt < val.m_data.m_value.object->size() - 1; ++cnt, ++i) { o->write_characters(indent_string.c_str(), new_indent); o->write_character('\"'); @@ -18028,8 +18153,8 @@ class serializer } // last element - JSON_ASSERT(i != val.m_value.object->cend()); - JSON_ASSERT(std::next(i) == val.m_value.object->cend()); + JSON_ASSERT(i != val.m_data.m_value.object->cend()); + JSON_ASSERT(std::next(i) == val.m_data.m_value.object->cend()); o->write_characters(indent_string.c_str(), new_indent); o->write_character('\"'); dump_escaped(i->first, ensure_ascii); @@ -18045,8 +18170,8 @@ class serializer o->write_character('{'); // first n-1 elements - auto i = val.m_value.object->cbegin(); - for (std::size_t cnt = 0; cnt < val.m_value.object->size() - 1; ++cnt, ++i) + auto i = val.m_data.m_value.object->cbegin(); + for (std::size_t cnt = 0; cnt < val.m_data.m_value.object->size() - 1; ++cnt, ++i) { o->write_character('\"'); dump_escaped(i->first, ensure_ascii); @@ -18056,8 +18181,8 @@ class serializer } // last element - JSON_ASSERT(i != val.m_value.object->cend()); - JSON_ASSERT(std::next(i) == val.m_value.object->cend()); + JSON_ASSERT(i != val.m_data.m_value.object->cend()); + JSON_ASSERT(std::next(i) == val.m_data.m_value.object->cend()); o->write_character('\"'); dump_escaped(i->first, ensure_ascii); o->write_characters("\":", 2); @@ -18071,7 +18196,7 @@ class serializer case value_t::array: { - if (val.m_value.array->empty()) + if (val.m_data.m_value.array->empty()) { o->write_characters("[]", 2); return; @@ -18089,8 +18214,8 @@ class serializer } // first n-1 elements - for (auto i = val.m_value.array->cbegin(); - i != val.m_value.array->cend() - 1; ++i) + for (auto i = val.m_data.m_value.array->cbegin(); + i != val.m_data.m_value.array->cend() - 1; ++i) { o->write_characters(indent_string.c_str(), new_indent); dump(*i, true, ensure_ascii, indent_step, new_indent); @@ -18098,9 +18223,9 @@ class serializer } // last element - JSON_ASSERT(!val.m_value.array->empty()); + JSON_ASSERT(!val.m_data.m_value.array->empty()); o->write_characters(indent_string.c_str(), new_indent); - dump(val.m_value.array->back(), true, ensure_ascii, indent_step, new_indent); + dump(val.m_data.m_value.array->back(), true, ensure_ascii, indent_step, new_indent); o->write_character('\n'); o->write_characters(indent_string.c_str(), current_indent); @@ -18111,16 +18236,16 @@ class serializer o->write_character('['); // first n-1 elements - for (auto i = val.m_value.array->cbegin(); - i != val.m_value.array->cend() - 1; ++i) + for (auto i = val.m_data.m_value.array->cbegin(); + i != val.m_data.m_value.array->cend() - 1; ++i) { dump(*i, false, ensure_ascii, indent_step, current_indent); o->write_character(','); } // last element - JSON_ASSERT(!val.m_value.array->empty()); - dump(val.m_value.array->back(), false, ensure_ascii, indent_step, current_indent); + JSON_ASSERT(!val.m_data.m_value.array->empty()); + dump(val.m_data.m_value.array->back(), false, ensure_ascii, indent_step, current_indent); o->write_character(']'); } @@ -18131,7 +18256,7 @@ class serializer case value_t::string: { o->write_character('\"'); - dump_escaped(*val.m_value.string, ensure_ascii); + dump_escaped(*val.m_data.m_value.string, ensure_ascii); o->write_character('\"'); return; } @@ -18153,24 +18278,24 @@ class serializer o->write_characters("\"bytes\": [", 10); - if (!val.m_value.binary->empty()) + if (!val.m_data.m_value.binary->empty()) { - for (auto i = val.m_value.binary->cbegin(); - i != val.m_value.binary->cend() - 1; ++i) + for (auto i = val.m_data.m_value.binary->cbegin(); + i != val.m_data.m_value.binary->cend() - 1; ++i) { dump_integer(*i); o->write_characters(", ", 2); } - dump_integer(val.m_value.binary->back()); + dump_integer(val.m_data.m_value.binary->back()); } o->write_characters("],\n", 3); o->write_characters(indent_string.c_str(), new_indent); o->write_characters("\"subtype\": ", 11); - if (val.m_value.binary->has_subtype()) + if (val.m_data.m_value.binary->has_subtype()) { - dump_integer(val.m_value.binary->subtype()); + dump_integer(val.m_data.m_value.binary->subtype()); } else { @@ -18184,21 +18309,21 @@ class serializer { o->write_characters("{\"bytes\":[", 10); - if (!val.m_value.binary->empty()) + if (!val.m_data.m_value.binary->empty()) { - for (auto i = val.m_value.binary->cbegin(); - i != val.m_value.binary->cend() - 1; ++i) + for (auto i = val.m_data.m_value.binary->cbegin(); + i != val.m_data.m_value.binary->cend() - 1; ++i) { dump_integer(*i); o->write_character(','); } - dump_integer(val.m_value.binary->back()); + dump_integer(val.m_data.m_value.binary->back()); } o->write_characters("],\"subtype\":", 12); - if (val.m_value.binary->has_subtype()) + if (val.m_data.m_value.binary->has_subtype()) { - dump_integer(val.m_value.binary->subtype()); + dump_integer(val.m_data.m_value.binary->subtype()); o->write_character('}'); } else @@ -18211,7 +18336,7 @@ class serializer case value_t::boolean: { - if (val.m_value.boolean) + if (val.m_data.m_value.boolean) { o->write_characters("true", 4); } @@ -18224,19 +18349,19 @@ class serializer case value_t::number_integer: { - dump_integer(val.m_value.number_integer); + dump_integer(val.m_data.m_value.number_integer); return; } case value_t::number_unsigned: { - dump_integer(val.m_value.number_unsigned); + dump_integer(val.m_data.m_value.number_unsigned); return; } case value_t::number_float: { - dump_float(val.m_value.number_float); + dump_float(val.m_data.m_value.number_float); return; } @@ -18810,8 +18935,8 @@ class serializer ? (byte & 0x3fu) | (codep << 6u) : (0xFFu >> type) & (byte); - std::size_t index = 256u + static_cast(state) * 16u + static_cast(type); - JSON_ASSERT(index < 400); + const std::size_t index = 256u + static_cast(state) * 16u + static_cast(type); + JSON_ASSERT(index < utf8d.size()); state = utf8d[index]; return state; } @@ -18878,10 +19003,10 @@ NLOHMANN_JSON_NAMESPACE_END // #include // __ _____ _____ _____ // __| | __| | | | JSON for Modern C++ -// | | |__ | | | | | | version 3.11.2 +// | | |__ | | | | | | version 3.11.3 // |_____|_____|_____|_|___| https://github.com/nlohmann/json // -// SPDX-FileCopyrightText: 2013-2022 Niels Lohmann +// SPDX-FileCopyrightText: 2013-2023 Niels Lohmann // SPDX-License-Identifier: MIT @@ -18998,7 +19123,7 @@ template , template::value, int> = 0> - T & at(KeyType && key) + T & at(KeyType && key) // NOLINT(cppcoreguidelines-missing-std-forward) { for (auto it = this->begin(); it != this->end(); ++it) { @@ -19026,7 +19151,7 @@ template , template::value, int> = 0> - const T & at(KeyType && key) const + const T & at(KeyType && key) const // NOLINT(cppcoreguidelines-missing-std-forward) { for (auto it = this->begin(); it != this->end(); ++it) { @@ -19060,7 +19185,7 @@ template , template::value, int> = 0> - size_type erase(KeyType && key) + size_type erase(KeyType && key) // NOLINT(cppcoreguidelines-missing-std-forward) { for (auto it = this->begin(); it != this->end(); ++it) { @@ -19151,7 +19276,7 @@ template , template::value, int> = 0> - size_type count(KeyType && key) const + size_type count(KeyType && key) const // NOLINT(cppcoreguidelines-missing-std-forward) { for (auto it = this->begin(); it != this->end(); ++it) { @@ -19177,7 +19302,7 @@ template , template::value, int> = 0> - iterator find(KeyType && key) + iterator find(KeyType && key) // NOLINT(cppcoreguidelines-missing-std-forward) { for (auto it = this->begin(); it != this->end(); ++it) { @@ -19240,7 +19365,9 @@ NLOHMANN_JSON_NAMESPACE_END #if defined(JSON_HAS_CPP_17) - #include + #if JSON_HAS_STATIC_RTTI + #include + #endif #include #endif @@ -19271,6 +19398,7 @@ The invariants are checked by member function assert_invariant(). */ NLOHMANN_BASIC_JSON_TPL_DECLARATION class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-special-member-functions) + : public ::nlohmann::detail::json_base_class { private: template friend struct detail::external_constructor; @@ -19297,6 +19425,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec /// workaround type for MSVC using basic_json_t = NLOHMANN_BASIC_JSON_TPL; + using json_base_class_t = ::nlohmann::detail::json_base_class; JSON_PRIVATE_UNLESS_TESTED: // convenience aliases for types residing in namespace detail; @@ -19368,7 +19497,6 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec /// @} - ///////////////////// // container types // ///////////////////// @@ -19410,7 +19538,6 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec /// @} - /// @brief returns the allocator associated with the container /// @sa https://json.nlohmann.me/api/basic_json/get_allocator/ static allocator_type get_allocator() @@ -19425,7 +19552,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec { basic_json result; - result["copyright"] = "(C) 2013-2022 Niels Lohmann"; + result["copyright"] = "(C) 2013-2023 Niels Lohmann"; result["name"] = "JSON for Modern C++"; result["url"] = "https://github.com/nlohmann/json"; result["version"]["string"] = @@ -19473,7 +19600,6 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec result["compiler"] = {{"family", "unknown"}, {"version", "unknown"}}; #endif - #if defined(_MSVC_LANG) result["compiler"]["c++"] = std::to_string(_MSVC_LANG); #elif defined(__cplusplus) @@ -19484,7 +19610,6 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec return result; } - /////////////////////////// // JSON value data types // /////////////////////////// @@ -19692,7 +19817,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec object = nullptr; // silence warning, see #821 if (JSON_HEDLEY_UNLIKELY(t == value_t::null)) { - JSON_THROW(other_error::create(500, "961c151d2e87f2686a955a9be24d316f1362bf21 3.11.2", nullptr)); // LCOV_EXCL_LINE + JSON_THROW(other_error::create(500, "961c151d2e87f2686a955a9be24d316f1362bf21 3.11.3", nullptr)); // LCOV_EXCL_LINE } break; } @@ -19731,6 +19856,16 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec void destroy(value_t t) { + if ( + (t == value_t::object && object == nullptr) || + (t == value_t::array && array == nullptr) || + (t == value_t::string && string == nullptr) || + (t == value_t::binary && binary == nullptr) + ) + { + //not initialized (e.g. due to exception in the ctor) + return; + } if (t == value_t::array || t == value_t::object) { // flatten the current json_value to a heap-allocated stack @@ -19761,18 +19896,18 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec // its children to the stack to be processed later if (current_item.is_array()) { - std::move(current_item.m_value.array->begin(), current_item.m_value.array->end(), std::back_inserter(stack)); + std::move(current_item.m_data.m_value.array->begin(), current_item.m_data.m_value.array->end(), std::back_inserter(stack)); - current_item.m_value.array->clear(); + current_item.m_data.m_value.array->clear(); } else if (current_item.is_object()) { - for (auto&& it : *current_item.m_value.object) + for (auto&& it : *current_item.m_data.m_value.object) { stack.push_back(std::move(it.second)); } - current_item.m_value.object->clear(); + current_item.m_data.m_value.object->clear(); } // it's now safe that current_item get destructed @@ -19849,10 +19984,10 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec */ void assert_invariant(bool check_parents = true) const noexcept { - JSON_ASSERT(m_type != value_t::object || m_value.object != nullptr); - JSON_ASSERT(m_type != value_t::array || m_value.array != nullptr); - JSON_ASSERT(m_type != value_t::string || m_value.string != nullptr); - JSON_ASSERT(m_type != value_t::binary || m_value.binary != nullptr); + JSON_ASSERT(m_data.m_type != value_t::object || m_data.m_value.object != nullptr); + JSON_ASSERT(m_data.m_type != value_t::array || m_data.m_value.array != nullptr); + JSON_ASSERT(m_data.m_type != value_t::string || m_data.m_value.string != nullptr); + JSON_ASSERT(m_data.m_type != value_t::binary || m_data.m_value.binary != nullptr); #if JSON_DIAGNOSTICS JSON_TRY @@ -19871,11 +20006,11 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec void set_parents() { #if JSON_DIAGNOSTICS - switch (m_type) + switch (m_data.m_type) { case value_t::array: { - for (auto& element : *m_value.array) + for (auto& element : *m_data.m_value.array) { element.m_parent = this; } @@ -19884,7 +20019,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec case value_t::object: { - for (auto& element : *m_value.object) + for (auto& element : *m_data.m_value.object) { element.second.m_parent = this; } @@ -19925,7 +20060,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec { // see https://github.com/nlohmann/json/issues/2838 JSON_ASSERT(type() == value_t::array); - if (JSON_HEDLEY_UNLIKELY(m_value.array->capacity() != old_capacity)) + if (JSON_HEDLEY_UNLIKELY(m_data.m_value.array->capacity() != old_capacity)) { // capacity has changed: update all parents set_parents(); @@ -19981,7 +20116,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec /// @brief create an empty value with a given type /// @sa https://json.nlohmann.me/api/basic_json/basic_json/ basic_json(const value_t v) - : m_type(v), m_value(v) + : m_data(v) { assert_invariant(); } @@ -20055,12 +20190,12 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec *this = nullptr; break; case value_t::discarded: - m_type = value_t::discarded; + m_data.m_type = value_t::discarded; break; default: // LCOV_EXCL_LINE JSON_ASSERT(false); // NOLINT(cert-dcl03-c,hicpp-static-assert,misc-static-assert) LCOV_EXCL_LINE } - JSON_ASSERT(m_type == val.type()); + JSON_ASSERT(m_data.m_type == val.type()); set_parents(); assert_invariant(); } @@ -20076,7 +20211,10 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec bool is_an_object = std::all_of(init.begin(), init.end(), [](const detail::json_ref& element_ref) { - return element_ref->is_array() && element_ref->size() == 2 && (*element_ref)[0].is_string(); + // The cast is to ensure op[size_type] is called, bearing in mind size_type may not be int; + // (many string types can be constructed from 0 via its null-pointer guise, so we get a + // broken call to op[key_type], the wrong semantics and a 4804 warning on Windows) + return element_ref->is_array() && element_ref->size() == 2 && (*element_ref)[static_cast(0)].is_string(); }); // adjust type if type deduction is not wanted @@ -20098,22 +20236,22 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec if (is_an_object) { // the initializer list is a list of pairs -> create object - m_type = value_t::object; - m_value = value_t::object; + m_data.m_type = value_t::object; + m_data.m_value = value_t::object; for (auto& element_ref : init) { auto element = element_ref.moved_or_copied(); - m_value.object->emplace( - std::move(*((*element.m_value.array)[0].m_value.string)), - std::move((*element.m_value.array)[1])); + m_data.m_value.object->emplace( + std::move(*((*element.m_data.m_value.array)[0].m_data.m_value.string)), + std::move((*element.m_data.m_value.array)[1])); } } else { // the initializer list describes an array -> create array - m_type = value_t::array; - m_value.array = create(init.begin(), init.end()); + m_data.m_type = value_t::array; + m_data.m_value.array = create(init.begin(), init.end()); } set_parents(); @@ -20126,8 +20264,8 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec static basic_json binary(const typename binary_t::container_type& init) { auto res = basic_json(); - res.m_type = value_t::binary; - res.m_value = init; + res.m_data.m_type = value_t::binary; + res.m_data.m_value = init; return res; } @@ -20137,8 +20275,8 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec static basic_json binary(const typename binary_t::container_type& init, typename binary_t::subtype_type subtype) { auto res = basic_json(); - res.m_type = value_t::binary; - res.m_value = binary_t(init, subtype); + res.m_data.m_type = value_t::binary; + res.m_data.m_value = binary_t(init, subtype); return res; } @@ -20148,8 +20286,8 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec static basic_json binary(typename binary_t::container_type&& init) { auto res = basic_json(); - res.m_type = value_t::binary; - res.m_value = std::move(init); + res.m_data.m_type = value_t::binary; + res.m_data.m_value = std::move(init); return res; } @@ -20159,8 +20297,8 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec static basic_json binary(typename binary_t::container_type&& init, typename binary_t::subtype_type subtype) { auto res = basic_json(); - res.m_type = value_t::binary; - res.m_value = binary_t(std::move(init), subtype); + res.m_data.m_type = value_t::binary; + res.m_data.m_value = binary_t(std::move(init), subtype); return res; } @@ -20182,10 +20320,9 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec /// @brief construct an array with count copies of given value /// @sa https://json.nlohmann.me/api/basic_json/basic_json/ - basic_json(size_type cnt, const basic_json& val) - : m_type(value_t::array) + basic_json(size_type cnt, const basic_json& val): + m_data{cnt, val} { - m_value.array = create(cnt, val); set_parents(); assert_invariant(); } @@ -20207,10 +20344,10 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec } // copy type from first iterator - m_type = first.m_object->m_type; + m_data.m_type = first.m_object->m_data.m_type; // check if iterator range is complete for primitive values - switch (m_type) + switch (m_data.m_type) { case value_t::boolean: case value_t::number_float: @@ -20235,55 +20372,55 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec break; } - switch (m_type) + switch (m_data.m_type) { case value_t::number_integer: { - m_value.number_integer = first.m_object->m_value.number_integer; + m_data.m_value.number_integer = first.m_object->m_data.m_value.number_integer; break; } case value_t::number_unsigned: { - m_value.number_unsigned = first.m_object->m_value.number_unsigned; + m_data.m_value.number_unsigned = first.m_object->m_data.m_value.number_unsigned; break; } case value_t::number_float: { - m_value.number_float = first.m_object->m_value.number_float; + m_data.m_value.number_float = first.m_object->m_data.m_value.number_float; break; } case value_t::boolean: { - m_value.boolean = first.m_object->m_value.boolean; + m_data.m_value.boolean = first.m_object->m_data.m_value.boolean; break; } case value_t::string: { - m_value = *first.m_object->m_value.string; + m_data.m_value = *first.m_object->m_data.m_value.string; break; } case value_t::object: { - m_value.object = create(first.m_it.object_iterator, - last.m_it.object_iterator); + m_data.m_value.object = create(first.m_it.object_iterator, + last.m_it.object_iterator); break; } case value_t::array: { - m_value.array = create(first.m_it.array_iterator, - last.m_it.array_iterator); + m_data.m_value.array = create(first.m_it.array_iterator, + last.m_it.array_iterator); break; } case value_t::binary: { - m_value = *first.m_object->m_value.binary; + m_data.m_value = *first.m_object->m_data.m_value.binary; break; } @@ -20297,7 +20434,6 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec assert_invariant(); } - /////////////////////////////////////// // other constructors and destructor // /////////////////////////////////////// @@ -20310,58 +20446,59 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec /// @brief copy constructor /// @sa https://json.nlohmann.me/api/basic_json/basic_json/ basic_json(const basic_json& other) - : m_type(other.m_type) + : json_base_class_t(other) { + m_data.m_type = other.m_data.m_type; // check of passed value is valid other.assert_invariant(); - switch (m_type) + switch (m_data.m_type) { case value_t::object: { - m_value = *other.m_value.object; + m_data.m_value = *other.m_data.m_value.object; break; } case value_t::array: { - m_value = *other.m_value.array; + m_data.m_value = *other.m_data.m_value.array; break; } case value_t::string: { - m_value = *other.m_value.string; + m_data.m_value = *other.m_data.m_value.string; break; } case value_t::boolean: { - m_value = other.m_value.boolean; + m_data.m_value = other.m_data.m_value.boolean; break; } case value_t::number_integer: { - m_value = other.m_value.number_integer; + m_data.m_value = other.m_data.m_value.number_integer; break; } case value_t::number_unsigned: { - m_value = other.m_value.number_unsigned; + m_data.m_value = other.m_data.m_value.number_unsigned; break; } case value_t::number_float: { - m_value = other.m_value.number_float; + m_data.m_value = other.m_data.m_value.number_float; break; } case value_t::binary: { - m_value = *other.m_value.binary; + m_data.m_value = *other.m_data.m_value.binary; break; } @@ -20378,15 +20515,15 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec /// @brief move constructor /// @sa https://json.nlohmann.me/api/basic_json/basic_json/ basic_json(basic_json&& other) noexcept - : m_type(std::move(other.m_type)), - m_value(std::move(other.m_value)) + : json_base_class_t(std::forward(other)), + m_data(std::move(other.m_data)) { // check that passed value is valid other.assert_invariant(false); // invalidate payload - other.m_type = value_t::null; - other.m_value = {}; + other.m_data.m_type = value_t::null; + other.m_data.m_value = {}; set_parents(); assert_invariant(); @@ -20398,15 +20535,17 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec std::is_nothrow_move_constructible::value&& std::is_nothrow_move_assignable::value&& std::is_nothrow_move_constructible::value&& - std::is_nothrow_move_assignable::value + std::is_nothrow_move_assignable::value&& + std::is_nothrow_move_assignable::value ) { // check that passed value is valid other.assert_invariant(); using std::swap; - swap(m_type, other.m_type); - swap(m_value, other.m_value); + swap(m_data.m_type, other.m_data.m_type); + swap(m_data.m_value, other.m_data.m_value); + json_base_class_t::operator=(std::move(other)); set_parents(); assert_invariant(); @@ -20418,7 +20557,6 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec ~basic_json() noexcept { assert_invariant(false); - m_value.destroy(m_type); } /// @} @@ -20458,7 +20596,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec /// @sa https://json.nlohmann.me/api/basic_json/type/ constexpr value_t type() const noexcept { - return m_type; + return m_data.m_type; } /// @brief return whether type is primitive @@ -20479,14 +20617,14 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec /// @sa https://json.nlohmann.me/api/basic_json/is_null/ constexpr bool is_null() const noexcept { - return m_type == value_t::null; + return m_data.m_type == value_t::null; } /// @brief return whether value is a boolean /// @sa https://json.nlohmann.me/api/basic_json/is_boolean/ constexpr bool is_boolean() const noexcept { - return m_type == value_t::boolean; + return m_data.m_type == value_t::boolean; } /// @brief return whether value is a number @@ -20500,63 +20638,63 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec /// @sa https://json.nlohmann.me/api/basic_json/is_number_integer/ constexpr bool is_number_integer() const noexcept { - return m_type == value_t::number_integer || m_type == value_t::number_unsigned; + return m_data.m_type == value_t::number_integer || m_data.m_type == value_t::number_unsigned; } /// @brief return whether value is an unsigned integer number /// @sa https://json.nlohmann.me/api/basic_json/is_number_unsigned/ constexpr bool is_number_unsigned() const noexcept { - return m_type == value_t::number_unsigned; + return m_data.m_type == value_t::number_unsigned; } /// @brief return whether value is a floating-point number /// @sa https://json.nlohmann.me/api/basic_json/is_number_float/ constexpr bool is_number_float() const noexcept { - return m_type == value_t::number_float; + return m_data.m_type == value_t::number_float; } /// @brief return whether value is an object /// @sa https://json.nlohmann.me/api/basic_json/is_object/ constexpr bool is_object() const noexcept { - return m_type == value_t::object; + return m_data.m_type == value_t::object; } /// @brief return whether value is an array /// @sa https://json.nlohmann.me/api/basic_json/is_array/ constexpr bool is_array() const noexcept { - return m_type == value_t::array; + return m_data.m_type == value_t::array; } /// @brief return whether value is a string /// @sa https://json.nlohmann.me/api/basic_json/is_string/ constexpr bool is_string() const noexcept { - return m_type == value_t::string; + return m_data.m_type == value_t::string; } /// @brief return whether value is a binary array /// @sa https://json.nlohmann.me/api/basic_json/is_binary/ constexpr bool is_binary() const noexcept { - return m_type == value_t::binary; + return m_data.m_type == value_t::binary; } /// @brief return whether value is discarded /// @sa https://json.nlohmann.me/api/basic_json/is_discarded/ constexpr bool is_discarded() const noexcept { - return m_type == value_t::discarded; + return m_data.m_type == value_t::discarded; } /// @brief return the type of the JSON value (implicit) /// @sa https://json.nlohmann.me/api/basic_json/operator_value_t/ constexpr operator value_t() const noexcept { - return m_type; + return m_data.m_type; } /// @} @@ -20571,7 +20709,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec { if (JSON_HEDLEY_LIKELY(is_boolean())) { - return m_value.boolean; + return m_data.m_value.boolean; } JSON_THROW(type_error::create(302, detail::concat("type must be boolean, but is ", type_name()), this)); @@ -20580,97 +20718,97 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec /// get a pointer to the value (object) object_t* get_impl_ptr(object_t* /*unused*/) noexcept { - return is_object() ? m_value.object : nullptr; + return is_object() ? m_data.m_value.object : nullptr; } /// get a pointer to the value (object) constexpr const object_t* get_impl_ptr(const object_t* /*unused*/) const noexcept { - return is_object() ? m_value.object : nullptr; + return is_object() ? m_data.m_value.object : nullptr; } /// get a pointer to the value (array) array_t* get_impl_ptr(array_t* /*unused*/) noexcept { - return is_array() ? m_value.array : nullptr; + return is_array() ? m_data.m_value.array : nullptr; } /// get a pointer to the value (array) constexpr const array_t* get_impl_ptr(const array_t* /*unused*/) const noexcept { - return is_array() ? m_value.array : nullptr; + return is_array() ? m_data.m_value.array : nullptr; } /// get a pointer to the value (string) string_t* get_impl_ptr(string_t* /*unused*/) noexcept { - return is_string() ? m_value.string : nullptr; + return is_string() ? m_data.m_value.string : nullptr; } /// get a pointer to the value (string) constexpr const string_t* get_impl_ptr(const string_t* /*unused*/) const noexcept { - return is_string() ? m_value.string : nullptr; + return is_string() ? m_data.m_value.string : nullptr; } /// get a pointer to the value (boolean) boolean_t* get_impl_ptr(boolean_t* /*unused*/) noexcept { - return is_boolean() ? &m_value.boolean : nullptr; + return is_boolean() ? &m_data.m_value.boolean : nullptr; } /// get a pointer to the value (boolean) constexpr const boolean_t* get_impl_ptr(const boolean_t* /*unused*/) const noexcept { - return is_boolean() ? &m_value.boolean : nullptr; + return is_boolean() ? &m_data.m_value.boolean : nullptr; } /// get a pointer to the value (integer number) number_integer_t* get_impl_ptr(number_integer_t* /*unused*/) noexcept { - return is_number_integer() ? &m_value.number_integer : nullptr; + return is_number_integer() ? &m_data.m_value.number_integer : nullptr; } /// get a pointer to the value (integer number) constexpr const number_integer_t* get_impl_ptr(const number_integer_t* /*unused*/) const noexcept { - return is_number_integer() ? &m_value.number_integer : nullptr; + return is_number_integer() ? &m_data.m_value.number_integer : nullptr; } /// get a pointer to the value (unsigned number) number_unsigned_t* get_impl_ptr(number_unsigned_t* /*unused*/) noexcept { - return is_number_unsigned() ? &m_value.number_unsigned : nullptr; + return is_number_unsigned() ? &m_data.m_value.number_unsigned : nullptr; } /// get a pointer to the value (unsigned number) constexpr const number_unsigned_t* get_impl_ptr(const number_unsigned_t* /*unused*/) const noexcept { - return is_number_unsigned() ? &m_value.number_unsigned : nullptr; + return is_number_unsigned() ? &m_data.m_value.number_unsigned : nullptr; } /// get a pointer to the value (floating-point number) number_float_t* get_impl_ptr(number_float_t* /*unused*/) noexcept { - return is_number_float() ? &m_value.number_float : nullptr; + return is_number_float() ? &m_data.m_value.number_float : nullptr; } /// get a pointer to the value (floating-point number) constexpr const number_float_t* get_impl_ptr(const number_float_t* /*unused*/) const noexcept { - return is_number_float() ? &m_value.number_float : nullptr; + return is_number_float() ? &m_data.m_value.number_float : nullptr; } /// get a pointer to the value (binary) binary_t* get_impl_ptr(binary_t* /*unused*/) noexcept { - return is_binary() ? m_value.binary : nullptr; + return is_binary() ? m_data.m_value.binary : nullptr; } /// get a pointer to the value (binary) constexpr const binary_t* get_impl_ptr(const binary_t* /*unused*/) const noexcept { - return is_binary() ? m_value.binary : nullptr; + return is_binary() ? m_data.m_value.binary : nullptr; } /*! @@ -21053,7 +21191,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec #if defined(JSON_HAS_CPP_17) && (defined(__GNUC__) || (defined(_MSC_VER) && _MSC_VER >= 1910 && _MSC_VER <= 1914)) detail::negation>, #endif -#if defined(JSON_HAS_CPP_17) +#if defined(JSON_HAS_CPP_17) && JSON_HAS_STATIC_RTTI detail::negation>, #endif detail::is_detected_lazy @@ -21090,7 +21228,6 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec /// @} - //////////////////// // element access // //////////////////// @@ -21108,7 +21245,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec { JSON_TRY { - return set_parent(m_value.array->at(idx)); + return set_parent(m_data.m_value.array->at(idx)); } JSON_CATCH (std::out_of_range&) { @@ -21131,7 +21268,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec { JSON_TRY { - return m_value.array->at(idx); + return m_data.m_value.array->at(idx); } JSON_CATCH (std::out_of_range&) { @@ -21155,8 +21292,8 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec JSON_THROW(type_error::create(304, detail::concat("cannot use at() with ", type_name()), this)); } - auto it = m_value.object->find(key); - if (it == m_value.object->end()) + auto it = m_data.m_value.object->find(key); + if (it == m_data.m_value.object->end()) { JSON_THROW(out_of_range::create(403, detail::concat("key '", key, "' not found"), this)); } @@ -21175,8 +21312,8 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec JSON_THROW(type_error::create(304, detail::concat("cannot use at() with ", type_name()), this)); } - auto it = m_value.object->find(std::forward(key)); - if (it == m_value.object->end()) + auto it = m_data.m_value.object->find(std::forward(key)); + if (it == m_data.m_value.object->end()) { JSON_THROW(out_of_range::create(403, detail::concat("key '", string_t(std::forward(key)), "' not found"), this)); } @@ -21193,8 +21330,8 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec JSON_THROW(type_error::create(304, detail::concat("cannot use at() with ", type_name()), this)); } - auto it = m_value.object->find(key); - if (it == m_value.object->end()) + auto it = m_data.m_value.object->find(key); + if (it == m_data.m_value.object->end()) { JSON_THROW(out_of_range::create(403, detail::concat("key '", key, "' not found"), this)); } @@ -21213,8 +21350,8 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec JSON_THROW(type_error::create(304, detail::concat("cannot use at() with ", type_name()), this)); } - auto it = m_value.object->find(std::forward(key)); - if (it == m_value.object->end()) + auto it = m_data.m_value.object->find(std::forward(key)); + if (it == m_data.m_value.object->end()) { JSON_THROW(out_of_range::create(403, detail::concat("key '", string_t(std::forward(key)), "' not found"), this)); } @@ -21228,8 +21365,8 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec // implicitly convert null value to an empty array if (is_null()) { - m_type = value_t::array; - m_value.array = create(); + m_data.m_type = value_t::array; + m_data.m_value.array = create(); assert_invariant(); } @@ -21237,17 +21374,17 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec if (JSON_HEDLEY_LIKELY(is_array())) { // fill up array with null values if given idx is outside range - if (idx >= m_value.array->size()) + if (idx >= m_data.m_value.array->size()) { #if JSON_DIAGNOSTICS // remember array size & capacity before resizing - const auto old_size = m_value.array->size(); - const auto old_capacity = m_value.array->capacity(); + const auto old_size = m_data.m_value.array->size(); + const auto old_capacity = m_data.m_value.array->capacity(); #endif - m_value.array->resize(idx + 1); + m_data.m_value.array->resize(idx + 1); #if JSON_DIAGNOSTICS - if (JSON_HEDLEY_UNLIKELY(m_value.array->capacity() != old_capacity)) + if (JSON_HEDLEY_UNLIKELY(m_data.m_value.array->capacity() != old_capacity)) { // capacity has changed: update all parents set_parents(); @@ -21261,7 +21398,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec assert_invariant(); } - return m_value.array->operator[](idx); + return m_data.m_value.array->operator[](idx); } JSON_THROW(type_error::create(305, detail::concat("cannot use operator[] with a numeric argument with ", type_name()), this)); @@ -21274,7 +21411,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec // const operator[] only works for arrays if (JSON_HEDLEY_LIKELY(is_array())) { - return m_value.array->operator[](idx); + return m_data.m_value.array->operator[](idx); } JSON_THROW(type_error::create(305, detail::concat("cannot use operator[] with a numeric argument with ", type_name()), this)); @@ -21287,15 +21424,15 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec // implicitly convert null value to an empty object if (is_null()) { - m_type = value_t::object; - m_value.object = create(); + m_data.m_type = value_t::object; + m_data.m_value.object = create(); assert_invariant(); } // operator[] only works for objects if (JSON_HEDLEY_LIKELY(is_object())) { - auto result = m_value.object->emplace(std::move(key), nullptr); + auto result = m_data.m_value.object->emplace(std::move(key), nullptr); return set_parent(result.first->second); } @@ -21309,8 +21446,8 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec // const operator[] only works for objects if (JSON_HEDLEY_LIKELY(is_object())) { - auto it = m_value.object->find(key); - JSON_ASSERT(it != m_value.object->end()); + auto it = m_data.m_value.object->find(key); + JSON_ASSERT(it != m_data.m_value.object->end()); return it->second; } @@ -21340,15 +21477,15 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec // implicitly convert null value to an empty object if (is_null()) { - m_type = value_t::object; - m_value.object = create(); + m_data.m_type = value_t::object; + m_data.m_value.object = create(); assert_invariant(); } // operator[] only works for objects if (JSON_HEDLEY_LIKELY(is_object())) { - auto result = m_value.object->emplace(std::forward(key), nullptr); + auto result = m_data.m_value.object->emplace(std::forward(key), nullptr); return set_parent(result.first->second); } @@ -21364,8 +21501,8 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec // const operator[] only works for objects if (JSON_HEDLEY_LIKELY(is_object())) { - auto it = m_value.object->find(std::forward(key)); - JSON_ASSERT(it != m_value.object->end()); + auto it = m_data.m_value.object->find(std::forward(key)); + JSON_ASSERT(it != m_data.m_value.object->end()); return it->second; } @@ -21602,7 +21739,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec IteratorType result = end(); - switch (m_type) + switch (m_data.m_type) { case value_t::boolean: case value_t::number_float: @@ -21619,32 +21756,32 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec if (is_string()) { AllocatorType alloc; - std::allocator_traits::destroy(alloc, m_value.string); - std::allocator_traits::deallocate(alloc, m_value.string, 1); - m_value.string = nullptr; + std::allocator_traits::destroy(alloc, m_data.m_value.string); + std::allocator_traits::deallocate(alloc, m_data.m_value.string, 1); + m_data.m_value.string = nullptr; } else if (is_binary()) { AllocatorType alloc; - std::allocator_traits::destroy(alloc, m_value.binary); - std::allocator_traits::deallocate(alloc, m_value.binary, 1); - m_value.binary = nullptr; + std::allocator_traits::destroy(alloc, m_data.m_value.binary); + std::allocator_traits::deallocate(alloc, m_data.m_value.binary, 1); + m_data.m_value.binary = nullptr; } - m_type = value_t::null; + m_data.m_type = value_t::null; assert_invariant(); break; } case value_t::object: { - result.m_it.object_iterator = m_value.object->erase(pos.m_it.object_iterator); + result.m_it.object_iterator = m_data.m_value.object->erase(pos.m_it.object_iterator); break; } case value_t::array: { - result.m_it.array_iterator = m_value.array->erase(pos.m_it.array_iterator); + result.m_it.array_iterator = m_data.m_value.array->erase(pos.m_it.array_iterator); break; } @@ -21672,7 +21809,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec IteratorType result = end(); - switch (m_type) + switch (m_data.m_type) { case value_t::boolean: case value_t::number_float: @@ -21690,33 +21827,33 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec if (is_string()) { AllocatorType alloc; - std::allocator_traits::destroy(alloc, m_value.string); - std::allocator_traits::deallocate(alloc, m_value.string, 1); - m_value.string = nullptr; + std::allocator_traits::destroy(alloc, m_data.m_value.string); + std::allocator_traits::deallocate(alloc, m_data.m_value.string, 1); + m_data.m_value.string = nullptr; } else if (is_binary()) { AllocatorType alloc; - std::allocator_traits::destroy(alloc, m_value.binary); - std::allocator_traits::deallocate(alloc, m_value.binary, 1); - m_value.binary = nullptr; + std::allocator_traits::destroy(alloc, m_data.m_value.binary); + std::allocator_traits::deallocate(alloc, m_data.m_value.binary, 1); + m_data.m_value.binary = nullptr; } - m_type = value_t::null; + m_data.m_type = value_t::null; assert_invariant(); break; } case value_t::object: { - result.m_it.object_iterator = m_value.object->erase(first.m_it.object_iterator, + result.m_it.object_iterator = m_data.m_value.object->erase(first.m_it.object_iterator, last.m_it.object_iterator); break; } case value_t::array: { - result.m_it.array_iterator = m_value.array->erase(first.m_it.array_iterator, + result.m_it.array_iterator = m_data.m_value.array->erase(first.m_it.array_iterator, last.m_it.array_iterator); break; } @@ -21741,7 +21878,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec JSON_THROW(type_error::create(307, detail::concat("cannot use erase() with ", type_name()), this)); } - return m_value.object->erase(std::forward(key)); + return m_data.m_value.object->erase(std::forward(key)); } template < typename KeyType, detail::enable_if_t < @@ -21754,10 +21891,10 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec JSON_THROW(type_error::create(307, detail::concat("cannot use erase() with ", type_name()), this)); } - const auto it = m_value.object->find(std::forward(key)); - if (it != m_value.object->end()) + const auto it = m_data.m_value.object->find(std::forward(key)); + if (it != m_data.m_value.object->end()) { - m_value.object->erase(it); + m_data.m_value.object->erase(it); return 1; } return 0; @@ -21795,7 +21932,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec JSON_THROW(out_of_range::create(401, detail::concat("array index ", std::to_string(idx), " is out of range"), this)); } - m_value.array->erase(m_value.array->begin() + static_cast(idx)); + m_data.m_value.array->erase(m_data.m_value.array->begin() + static_cast(idx)); } else { @@ -21805,7 +21942,6 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec /// @} - //////////// // lookup // //////////// @@ -21821,7 +21957,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec if (is_object()) { - result.m_it.object_iterator = m_value.object->find(key); + result.m_it.object_iterator = m_data.m_value.object->find(key); } return result; @@ -21835,7 +21971,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec if (is_object()) { - result.m_it.object_iterator = m_value.object->find(key); + result.m_it.object_iterator = m_data.m_value.object->find(key); } return result; @@ -21851,7 +21987,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec if (is_object()) { - result.m_it.object_iterator = m_value.object->find(std::forward(key)); + result.m_it.object_iterator = m_data.m_value.object->find(std::forward(key)); } return result; @@ -21867,7 +22003,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec if (is_object()) { - result.m_it.object_iterator = m_value.object->find(std::forward(key)); + result.m_it.object_iterator = m_data.m_value.object->find(std::forward(key)); } return result; @@ -21878,7 +22014,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec size_type count(const typename object_t::key_type& key) const { // return 0 for all nonobject types - return is_object() ? m_value.object->count(key) : 0; + return is_object() ? m_data.m_value.object->count(key) : 0; } /// @brief returns the number of occurrences of a key in a JSON object @@ -21888,14 +22024,14 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec size_type count(KeyType && key) const { // return 0 for all nonobject types - return is_object() ? m_value.object->count(std::forward(key)) : 0; + return is_object() ? m_data.m_value.object->count(std::forward(key)) : 0; } /// @brief check the existence of an element in a JSON object /// @sa https://json.nlohmann.me/api/basic_json/contains/ bool contains(const typename object_t::key_type& key) const { - return is_object() && m_value.object->find(key) != m_value.object->end(); + return is_object() && m_data.m_value.object->find(key) != m_data.m_value.object->end(); } /// @brief check the existence of an element in a JSON object @@ -21904,7 +22040,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec detail::is_usable_as_basic_json_key_type::value, int> = 0> bool contains(KeyType && key) const { - return is_object() && m_value.object->find(std::forward(key)) != m_value.object->end(); + return is_object() && m_data.m_value.object->find(std::forward(key)) != m_data.m_value.object->end(); } /// @brief check the existence of an element in a JSON object given a JSON pointer @@ -21923,7 +22059,6 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec /// @} - /////////////// // iterators // /////////////// @@ -22062,7 +22197,6 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec /// @} - ////////////// // capacity // ////////////// @@ -22074,7 +22208,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec /// @sa https://json.nlohmann.me/api/basic_json/empty/ bool empty() const noexcept { - switch (m_type) + switch (m_data.m_type) { case value_t::null: { @@ -22085,13 +22219,13 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec case value_t::array: { // delegate call to array_t::empty() - return m_value.array->empty(); + return m_data.m_value.array->empty(); } case value_t::object: { // delegate call to object_t::empty() - return m_value.object->empty(); + return m_data.m_value.object->empty(); } case value_t::string: @@ -22113,7 +22247,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec /// @sa https://json.nlohmann.me/api/basic_json/size/ size_type size() const noexcept { - switch (m_type) + switch (m_data.m_type) { case value_t::null: { @@ -22124,13 +22258,13 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec case value_t::array: { // delegate call to array_t::size() - return m_value.array->size(); + return m_data.m_value.array->size(); } case value_t::object: { // delegate call to object_t::size() - return m_value.object->size(); + return m_data.m_value.object->size(); } case value_t::string: @@ -22152,18 +22286,18 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec /// @sa https://json.nlohmann.me/api/basic_json/max_size/ size_type max_size() const noexcept { - switch (m_type) + switch (m_data.m_type) { case value_t::array: { // delegate call to array_t::max_size() - return m_value.array->max_size(); + return m_data.m_value.array->max_size(); } case value_t::object: { // delegate call to object_t::max_size() - return m_value.object->max_size(); + return m_data.m_value.object->max_size(); } case value_t::null: @@ -22184,7 +22318,6 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec /// @} - /////////////// // modifiers // /////////////// @@ -22196,53 +22329,53 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec /// @sa https://json.nlohmann.me/api/basic_json/clear/ void clear() noexcept { - switch (m_type) + switch (m_data.m_type) { case value_t::number_integer: { - m_value.number_integer = 0; + m_data.m_value.number_integer = 0; break; } case value_t::number_unsigned: { - m_value.number_unsigned = 0; + m_data.m_value.number_unsigned = 0; break; } case value_t::number_float: { - m_value.number_float = 0.0; + m_data.m_value.number_float = 0.0; break; } case value_t::boolean: { - m_value.boolean = false; + m_data.m_value.boolean = false; break; } case value_t::string: { - m_value.string->clear(); + m_data.m_value.string->clear(); break; } case value_t::binary: { - m_value.binary->clear(); + m_data.m_value.binary->clear(); break; } case value_t::array: { - m_value.array->clear(); + m_data.m_value.array->clear(); break; } case value_t::object: { - m_value.object->clear(); + m_data.m_value.object->clear(); break; } @@ -22266,15 +22399,15 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec // transform null object into an array if (is_null()) { - m_type = value_t::array; - m_value = value_t::array; + m_data.m_type = value_t::array; + m_data.m_value = value_t::array; assert_invariant(); } // add element to array (move semantics) - const auto old_capacity = m_value.array->capacity(); - m_value.array->push_back(std::move(val)); - set_parent(m_value.array->back(), old_capacity); + const auto old_capacity = m_data.m_value.array->capacity(); + m_data.m_value.array->push_back(std::move(val)); + set_parent(m_data.m_value.array->back(), old_capacity); // if val is moved from, basic_json move constructor marks it null, so we do not call the destructor } @@ -22299,15 +22432,15 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec // transform null object into an array if (is_null()) { - m_type = value_t::array; - m_value = value_t::array; + m_data.m_type = value_t::array; + m_data.m_value = value_t::array; assert_invariant(); } // add element to array - const auto old_capacity = m_value.array->capacity(); - m_value.array->push_back(val); - set_parent(m_value.array->back(), old_capacity); + const auto old_capacity = m_data.m_value.array->capacity(); + m_data.m_value.array->push_back(val); + set_parent(m_data.m_value.array->back(), old_capacity); } /// @brief add an object to an array @@ -22331,13 +22464,13 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec // transform null object into an object if (is_null()) { - m_type = value_t::object; - m_value = value_t::object; + m_data.m_type = value_t::object; + m_data.m_value = value_t::object; assert_invariant(); } // add element to object - auto res = m_value.object->insert(val); + auto res = m_data.m_value.object->insert(val); set_parent(res.first->second); } @@ -22387,15 +22520,15 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec // transform null object into an array if (is_null()) { - m_type = value_t::array; - m_value = value_t::array; + m_data.m_type = value_t::array; + m_data.m_value = value_t::array; assert_invariant(); } // add element to array (perfect forwarding) - const auto old_capacity = m_value.array->capacity(); - m_value.array->emplace_back(std::forward(args)...); - return set_parent(m_value.array->back(), old_capacity); + const auto old_capacity = m_data.m_value.array->capacity(); + m_data.m_value.array->emplace_back(std::forward(args)...); + return set_parent(m_data.m_value.array->back(), old_capacity); } /// @brief add an object to an object if key does not exist @@ -22412,13 +22545,13 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec // transform null object into an object if (is_null()) { - m_type = value_t::object; - m_value = value_t::object; + m_data.m_type = value_t::object; + m_data.m_value = value_t::object; assert_invariant(); } // add element to array (perfect forwarding) - auto res = m_value.object->emplace(std::forward(args)...); + auto res = m_data.m_value.object->emplace(std::forward(args)...); set_parent(res.first->second); // create result iterator and set iterator to the result of emplace @@ -22436,14 +22569,14 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec iterator insert_iterator(const_iterator pos, Args&& ... args) { iterator result(this); - JSON_ASSERT(m_value.array != nullptr); + JSON_ASSERT(m_data.m_value.array != nullptr); - auto insert_pos = std::distance(m_value.array->begin(), pos.m_it.array_iterator); - m_value.array->insert(pos.m_it.array_iterator, std::forward(args)...); - result.m_it.array_iterator = m_value.array->begin() + insert_pos; + auto insert_pos = std::distance(m_data.m_value.array->begin(), pos.m_it.array_iterator); + m_data.m_value.array->insert(pos.m_it.array_iterator, std::forward(args)...); + result.m_it.array_iterator = m_data.m_value.array->begin() + insert_pos; // This could have been written as: - // result.m_it.array_iterator = m_value.array->insert(pos.m_it.array_iterator, cnt, val); + // result.m_it.array_iterator = m_data.m_value.array->insert(pos.m_it.array_iterator, cnt, val); // but the return value of insert is missing in GCC 4.8, so it is written this way instead. set_parents(); @@ -22570,7 +22703,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec JSON_THROW(invalid_iterator::create(202, "iterators first and last must point to objects", this)); } - m_value.object->insert(first.m_it.object_iterator, last.m_it.object_iterator); + m_data.m_value.object->insert(first.m_it.object_iterator, last.m_it.object_iterator); } /// @brief updates a JSON object from another object, overwriting existing keys @@ -22587,8 +22720,8 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec // implicitly convert null value to an empty object if (is_null()) { - m_type = value_t::object; - m_value.object = create(); + m_data.m_type = value_t::object; + m_data.m_value.object = create(); assert_invariant(); } @@ -22613,16 +22746,16 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec { if (merge_objects && it.value().is_object()) { - auto it2 = m_value.object->find(it.key()); - if (it2 != m_value.object->end()) + auto it2 = m_data.m_value.object->find(it.key()); + if (it2 != m_data.m_value.object->end()) { it2->second.update(it.value(), true); continue; } } - m_value.object->operator[](it.key()) = it.value(); + m_data.m_value.object->operator[](it.key()) = it.value(); #if JSON_DIAGNOSTICS - m_value.object->operator[](it.key()).m_parent = this; + m_data.m_value.object->operator[](it.key()).m_parent = this; #endif } } @@ -22632,12 +22765,12 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec void swap(reference other) noexcept ( std::is_nothrow_move_constructible::value&& std::is_nothrow_move_assignable::value&& - std::is_nothrow_move_constructible::value&& + std::is_nothrow_move_constructible::value&& // NOLINT(cppcoreguidelines-noexcept-swap,performance-noexcept-swap) std::is_nothrow_move_assignable::value ) { - std::swap(m_type, other.m_type); - std::swap(m_value, other.m_value); + std::swap(m_data.m_type, other.m_data.m_type); + std::swap(m_data.m_value, other.m_data.m_value); set_parents(); other.set_parents(); @@ -22649,7 +22782,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec friend void swap(reference left, reference right) noexcept ( std::is_nothrow_move_constructible::value&& std::is_nothrow_move_assignable::value&& - std::is_nothrow_move_constructible::value&& + std::is_nothrow_move_constructible::value&& // NOLINT(cppcoreguidelines-noexcept-swap,performance-noexcept-swap) std::is_nothrow_move_assignable::value ) { @@ -22658,13 +22791,13 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec /// @brief exchanges the values /// @sa https://json.nlohmann.me/api/basic_json/swap/ - void swap(array_t& other) // NOLINT(bugprone-exception-escape) + void swap(array_t& other) // NOLINT(bugprone-exception-escape,cppcoreguidelines-noexcept-swap,performance-noexcept-swap) { // swap only works for arrays if (JSON_HEDLEY_LIKELY(is_array())) { using std::swap; - swap(*(m_value.array), other); + swap(*(m_data.m_value.array), other); } else { @@ -22674,13 +22807,13 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec /// @brief exchanges the values /// @sa https://json.nlohmann.me/api/basic_json/swap/ - void swap(object_t& other) // NOLINT(bugprone-exception-escape) + void swap(object_t& other) // NOLINT(bugprone-exception-escape,cppcoreguidelines-noexcept-swap,performance-noexcept-swap) { // swap only works for objects if (JSON_HEDLEY_LIKELY(is_object())) { using std::swap; - swap(*(m_value.object), other); + swap(*(m_data.m_value.object), other); } else { @@ -22690,13 +22823,13 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec /// @brief exchanges the values /// @sa https://json.nlohmann.me/api/basic_json/swap/ - void swap(string_t& other) // NOLINT(bugprone-exception-escape) + void swap(string_t& other) // NOLINT(bugprone-exception-escape,cppcoreguidelines-noexcept-swap,performance-noexcept-swap) { // swap only works for strings if (JSON_HEDLEY_LIKELY(is_string())) { using std::swap; - swap(*(m_value.string), other); + swap(*(m_data.m_value.string), other); } else { @@ -22706,13 +22839,13 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec /// @brief exchanges the values /// @sa https://json.nlohmann.me/api/basic_json/swap/ - void swap(binary_t& other) // NOLINT(bugprone-exception-escape) + void swap(binary_t& other) // NOLINT(bugprone-exception-escape,cppcoreguidelines-noexcept-swap,performance-noexcept-swap) { // swap only works for strings if (JSON_HEDLEY_LIKELY(is_binary())) { using std::swap; - swap(*(m_value.binary), other); + swap(*(m_data.m_value.binary), other); } else { @@ -22728,7 +22861,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec if (JSON_HEDLEY_LIKELY(is_binary())) { using std::swap; - swap(*(m_value.binary), other); + swap(*(m_data.m_value.binary), other); } else { @@ -22756,31 +22889,31 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec switch (lhs_type) \ { \ case value_t::array: \ - return (*lhs.m_value.array) op (*rhs.m_value.array); \ + return (*lhs.m_data.m_value.array) op (*rhs.m_data.m_value.array); \ \ case value_t::object: \ - return (*lhs.m_value.object) op (*rhs.m_value.object); \ + return (*lhs.m_data.m_value.object) op (*rhs.m_data.m_value.object); \ \ case value_t::null: \ return (null_result); \ \ case value_t::string: \ - return (*lhs.m_value.string) op (*rhs.m_value.string); \ + return (*lhs.m_data.m_value.string) op (*rhs.m_data.m_value.string); \ \ case value_t::boolean: \ - return (lhs.m_value.boolean) op (rhs.m_value.boolean); \ + return (lhs.m_data.m_value.boolean) op (rhs.m_data.m_value.boolean); \ \ case value_t::number_integer: \ - return (lhs.m_value.number_integer) op (rhs.m_value.number_integer); \ + return (lhs.m_data.m_value.number_integer) op (rhs.m_data.m_value.number_integer); \ \ case value_t::number_unsigned: \ - return (lhs.m_value.number_unsigned) op (rhs.m_value.number_unsigned); \ + return (lhs.m_data.m_value.number_unsigned) op (rhs.m_data.m_value.number_unsigned); \ \ case value_t::number_float: \ - return (lhs.m_value.number_float) op (rhs.m_value.number_float); \ + return (lhs.m_data.m_value.number_float) op (rhs.m_data.m_value.number_float); \ \ case value_t::binary: \ - return (*lhs.m_value.binary) op (*rhs.m_value.binary); \ + return (*lhs.m_data.m_value.binary) op (*rhs.m_data.m_value.binary); \ \ case value_t::discarded: \ default: \ @@ -22789,27 +22922,27 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec } \ else if (lhs_type == value_t::number_integer && rhs_type == value_t::number_float) \ { \ - return static_cast(lhs.m_value.number_integer) op rhs.m_value.number_float; \ + return static_cast(lhs.m_data.m_value.number_integer) op rhs.m_data.m_value.number_float; \ } \ else if (lhs_type == value_t::number_float && rhs_type == value_t::number_integer) \ { \ - return lhs.m_value.number_float op static_cast(rhs.m_value.number_integer); \ + return lhs.m_data.m_value.number_float op static_cast(rhs.m_data.m_value.number_integer); \ } \ else if (lhs_type == value_t::number_unsigned && rhs_type == value_t::number_float) \ { \ - return static_cast(lhs.m_value.number_unsigned) op rhs.m_value.number_float; \ + return static_cast(lhs.m_data.m_value.number_unsigned) op rhs.m_data.m_value.number_float; \ } \ else if (lhs_type == value_t::number_float && rhs_type == value_t::number_unsigned) \ { \ - return lhs.m_value.number_float op static_cast(rhs.m_value.number_unsigned); \ + return lhs.m_data.m_value.number_float op static_cast(rhs.m_data.m_value.number_unsigned); \ } \ else if (lhs_type == value_t::number_unsigned && rhs_type == value_t::number_integer) \ { \ - return static_cast(lhs.m_value.number_unsigned) op rhs.m_value.number_integer; \ + return static_cast(lhs.m_data.m_value.number_unsigned) op rhs.m_data.m_value.number_integer; \ } \ else if (lhs_type == value_t::number_integer && rhs_type == value_t::number_unsigned) \ { \ - return lhs.m_value.number_integer op static_cast(rhs.m_value.number_unsigned); \ + return lhs.m_data.m_value.number_integer op static_cast(rhs.m_data.m_value.number_unsigned); \ } \ else if(compares_unordered(lhs, rhs))\ {\ @@ -22826,8 +22959,8 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec // an operation is computed as an odd number of inverses of others static bool compares_unordered(const_reference lhs, const_reference rhs, bool inverse = false) noexcept { - if ((lhs.is_number_float() && std::isnan(lhs.m_value.number_float) && rhs.is_number()) - || (rhs.is_number_float() && std::isnan(rhs.m_value.number_float) && lhs.is_number())) + if ((lhs.is_number_float() && std::isnan(lhs.m_data.m_value.number_float) && rhs.is_number()) + || (rhs.is_number_float() && std::isnan(rhs.m_data.m_value.number_float) && lhs.is_number())) { return true; } @@ -23171,7 +23304,6 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec #endif // JSON_NO_IO /// @} - ///////////////////// // deserialization // ///////////////////// @@ -23328,7 +23460,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec JSON_HEDLEY_RETURNS_NON_NULL const char* type_name() const noexcept { - switch (m_type) + switch (m_data.m_type) { case value_t::null: return "null"; @@ -23352,17 +23484,43 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec } } - JSON_PRIVATE_UNLESS_TESTED: ////////////////////// // member variables // ////////////////////// - /// the type of the current element - value_t m_type = value_t::null; + struct data + { + /// the type of the current element + value_t m_type = value_t::null; - /// the value of the current element - json_value m_value = {}; + /// the value of the current element + json_value m_value = {}; + + data(const value_t v) + : m_type(v), m_value(v) + { + } + + data(size_type cnt, const basic_json& val) + : m_type(value_t::array) + { + m_value.array = create(cnt, val); + } + + data() noexcept = default; + data(data&&) noexcept = default; + data(const data&) noexcept = delete; + data& operator=(data&&) noexcept = delete; + data& operator=(const data&) noexcept = delete; + + ~data() noexcept + { + m_value.destroy(m_type); + } + }; + + data m_data = {}; #if JSON_DIAGNOSTICS /// a pointer to a parent value (for debugging purposes) @@ -23543,7 +23701,6 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec return from_cbor(ptr, ptr + len, strict, allow_exceptions, tag_handler); } - JSON_HEDLEY_WARN_UNUSED_RESULT JSON_HEDLEY_DEPRECATED_FOR(3.8.0, from_cbor(ptr, ptr + len)) static basic_json from_cbor(detail::span_input_adapter&& i, @@ -23667,7 +23824,6 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec return res ? result : basic_json(value_t::discarded); } - /// @brief create a JSON value from an input in BJData format /// @sa https://json.nlohmann.me/api/basic_json/from_bjdata/ template @@ -23890,7 +24046,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec } // make sure the top element of the pointer exists - json_pointer top_pointer = ptr.top(); + json_pointer const top_pointer = ptr.top(); if (top_pointer != ptr) { result.at(top_pointer); @@ -23902,7 +24058,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec // parent must exist when performing patch add per RFC6902 specs basic_json& parent = result.at(ptr); - switch (parent.m_type) + switch (parent.m_data.m_type) { case value_t::null: case value_t::object: @@ -23948,7 +24104,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec }; // wrapper for "remove" operation; remove value at ptr - const auto operation_remove = [this, &result](json_pointer & ptr) + const auto operation_remove = [this, & result](json_pointer & ptr) { // get reference to parent of JSON pointer ptr const auto last_path = ptr.back(); @@ -23991,13 +24147,13 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec bool string_type) -> basic_json & { // find value - auto it = val.m_value.object->find(member); + auto it = val.m_data.m_value.object->find(member); // context-sensitive error message - const auto error_msg = (op == "op") ? "operation" : detail::concat("operation '", op, '\''); + const auto error_msg = (op == "op") ? "operation" : detail::concat("operation '", op, '\''); // NOLINT(bugprone-unused-local-non-trivial-variable) // check if desired value is present - if (JSON_HEDLEY_UNLIKELY(it == val.m_value.object->end())) + if (JSON_HEDLEY_UNLIKELY(it == val.m_data.m_value.object->end())) { // NOLINTNEXTLINE(performance-inefficient-string-concatenation) JSON_THROW(parse_error::create(105, 0, detail::concat(error_msg, " must have member '", member, "'"), &val)); @@ -24052,7 +24208,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec json_pointer from_ptr(from_path); // the "from" location must exist - use at() - basic_json v = result.at(from_ptr); + basic_json const v = result.at(from_ptr); // The move operation is functionally identical to a // "remove" operation on the "from" location, followed @@ -24069,7 +24225,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec const json_pointer from_ptr(from_path); // the "from" location must exist - use at() - basic_json v = result.at(from_ptr); + basic_json const v = result.at(from_ptr); // The copy is functionally identical to an "add" // operation at the target location using the value @@ -24311,7 +24467,11 @@ inline namespace json_literals /// @brief user-defined string literal for JSON values /// @sa https://json.nlohmann.me/api/basic_json/operator_literal_json/ JSON_HEDLEY_NON_NULL(1) -inline nlohmann::json operator "" _json(const char* s, std::size_t n) +#if !defined(JSON_HEDLEY_GCC_VERSION) || JSON_HEDLEY_GCC_VERSION_CHECK(4,9,0) + inline nlohmann::json operator ""_json(const char* s, std::size_t n) +#else + inline nlohmann::json operator "" _json(const char* s, std::size_t n) +#endif { return nlohmann::json::parse(s, s + n); } @@ -24319,7 +24479,11 @@ inline nlohmann::json operator "" _json(const char* s, std::size_t n) /// @brief user-defined string literal for JSON pointer /// @sa https://json.nlohmann.me/api/basic_json/operator_literal_json_pointer/ JSON_HEDLEY_NON_NULL(1) -inline nlohmann::json::json_pointer operator "" _json_pointer(const char* s, std::size_t n) +#if !defined(JSON_HEDLEY_GCC_VERSION) || JSON_HEDLEY_GCC_VERSION_CHECK(4,9,0) + inline nlohmann::json::json_pointer operator ""_json_pointer(const char* s, std::size_t n) +#else + inline nlohmann::json::json_pointer operator "" _json_pointer(const char* s, std::size_t n) +#endif { return nlohmann::json::json_pointer(std::string(s, n)); } @@ -24338,7 +24502,7 @@ namespace std // NOLINT(cert-dcl58-cpp) /// @brief hash value for JSON objects /// @sa https://json.nlohmann.me/api/basic_json/std_hash/ NLOHMANN_BASIC_JSON_TPL_DECLARATION -struct hash +struct hash // NOLINT(cert-dcl58-cpp) { std::size_t operator()(const nlohmann::NLOHMANN_BASIC_JSON_TPL& j) const { @@ -24371,8 +24535,8 @@ struct less< ::nlohmann::detail::value_t> // do not remove the space after '<', /// @brief exchanges the values of two JSON objects /// @sa https://json.nlohmann.me/api/basic_json/std_swap/ NLOHMANN_BASIC_JSON_TPL_DECLARATION -inline void swap(nlohmann::NLOHMANN_BASIC_JSON_TPL& j1, nlohmann::NLOHMANN_BASIC_JSON_TPL& j2) noexcept( // NOLINT(readability-inconsistent-declaration-parameter-name) - is_nothrow_move_constructible::value&& // NOLINT(misc-redundant-expression) +inline void swap(nlohmann::NLOHMANN_BASIC_JSON_TPL& j1, nlohmann::NLOHMANN_BASIC_JSON_TPL& j2) noexcept( // NOLINT(readability-inconsistent-declaration-parameter-name, cert-dcl58-cpp) + is_nothrow_move_constructible::value&& // NOLINT(misc-redundant-expression,cppcoreguidelines-noexcept-swap,performance-noexcept-swap) is_nothrow_move_assignable::value) { j1.swap(j2); @@ -24383,17 +24547,22 @@ inline void swap(nlohmann::NLOHMANN_BASIC_JSON_TPL& j1, nlohmann::NLOHMANN_BASIC } // namespace std #if JSON_USE_GLOBAL_UDLS - using nlohmann::literals::json_literals::operator "" _json; // NOLINT(misc-unused-using-decls,google-global-names-in-headers) - using nlohmann::literals::json_literals::operator "" _json_pointer; //NOLINT(misc-unused-using-decls,google-global-names-in-headers) + #if !defined(JSON_HEDLEY_GCC_VERSION) || JSON_HEDLEY_GCC_VERSION_CHECK(4,9,0) + using nlohmann::literals::json_literals::operator ""_json; // NOLINT(misc-unused-using-decls,google-global-names-in-headers) + using nlohmann::literals::json_literals::operator ""_json_pointer; //NOLINT(misc-unused-using-decls,google-global-names-in-headers) + #else + using nlohmann::literals::json_literals::operator "" _json; // NOLINT(misc-unused-using-decls,google-global-names-in-headers) + using nlohmann::literals::json_literals::operator "" _json_pointer; //NOLINT(misc-unused-using-decls,google-global-names-in-headers) + #endif #endif // #include // __ _____ _____ _____ // __| | __| | | | JSON for Modern C++ -// | | |__ | | | | | | version 3.11.2 +// | | |__ | | | | | | version 3.11.3 // |_____|_____|_____|_|___| https://github.com/nlohmann/json // -// SPDX-FileCopyrightText: 2013-2022 Niels Lohmann +// SPDX-FileCopyrightText: 2013-2023 Niels Lohmann // SPDX-License-Identifier: MIT @@ -24428,16 +24597,17 @@ inline void swap(nlohmann::NLOHMANN_BASIC_JSON_TPL& j1, nlohmann::NLOHMANN_BASIC #undef JSON_HAS_EXPERIMENTAL_FILESYSTEM #undef JSON_HAS_THREE_WAY_COMPARISON #undef JSON_HAS_RANGES + #undef JSON_HAS_STATIC_RTTI #undef JSON_USE_LEGACY_DISCARDED_VALUE_COMPARISON #endif // #include // __ _____ _____ _____ // __| | __| | | | JSON for Modern C++ -// | | |__ | | | | | | version 3.11.2 +// | | |__ | | | | | | version 3.11.3 // |_____|_____|_____|_|___| https://github.com/nlohmann/json // -// SPDX-FileCopyrightText: 2013-2022 Niels Lohmann +// SPDX-FileCopyrightText: 2013-2023 Niels Lohmann // SPDX-License-Identifier: MIT From ccf58aa3ec4d20b10162ba40898dc038ad4c3fad Mon Sep 17 00:00:00 2001 From: slaren Date: Wed, 20 Mar 2024 14:42:59 +0100 Subject: [PATCH 671/812] cuda : refactor to remove global resources (#6170) * cuda : refactor to remove global resources --- ggml-cuda.cu | 3536 +++++++++++++++++++++++++------------------------- 1 file changed, 1760 insertions(+), 1776 deletions(-) diff --git a/ggml-cuda.cu b/ggml-cuda.cu index 139025588..e70b79c30 100644 --- a/ggml-cuda.cu +++ b/ggml-cuda.cu @@ -12,6 +12,7 @@ #include "ggml-common.h" #include +#include #include #include #include @@ -19,12 +20,13 @@ #include #include #include +#include +#include +#include #include #include #include #include -#include -#include // stringize macro for converting __CUDA_ARCH_LIST__ (list of integers) to string #define STRINGIZE_IMPL(...) #__VA_ARGS__ @@ -52,6 +54,7 @@ #define __shfl_xor_sync(mask, var, laneMask, width) __shfl_xor(var, laneMask, width) #define cublasComputeType_t hipblasDatatype_t //deprecated, new hipblasComputeType_t not in 5.6 #define cublasCreate hipblasCreate +#define cublasDestroy hipblasDestroy #define cublasGemmEx hipblasGemmEx #define cublasGemmBatchedEx hipblasGemmBatchedEx #define cublasGemmStridedBatchedEx hipblasGemmStridedBatchedEx @@ -108,6 +111,7 @@ #define cudaOccupancyMaxPotentialBlockSize hipOccupancyMaxPotentialBlockSize #define cudaSetDevice hipSetDevice #define cudaStreamCreateWithFlags hipStreamCreateWithFlags +#define cudaStreamDestroy hipStreamDestroy #define cudaStreamFireAndForget hipStreamFireAndForget #define cudaStreamNonBlocking hipStreamNonBlocking #define cudaStreamPerThread hipStreamPerThread @@ -151,8 +155,6 @@ #define CC_RDNA2 (CC_OFFSET_AMD + 1030) #define CC_RDNA3 (CC_OFFSET_AMD + 1100) -#define GGML_CUDA_MAX_NODES 8192 - // define this if you want to always fallback to MMQ kernels and not use cuBLAS for matrix multiplication // on modern hardware, using cuBLAS is recommended as it utilizes F16 tensor cores which are very performant // for large computational tasks. the drawback is that this requires some extra amount of VRAM: @@ -170,6 +172,1214 @@ #define MMVQ_MAX_BATCH_SIZE 8 // max batch size to use MMVQ kernels #define MMQ_MAX_BATCH_SIZE 32 // max batch size to use MMQ kernels when tensor cores are available +#define MATRIX_ROW_PADDING 512 // last row of quant. matrices is a multiple of this to avoid out-of-bounds memory accesses + + +#if defined(_MSC_VER) +#pragma warning(disable: 4244 4267) // possible loss of data +#endif + +static_assert(sizeof(half) == sizeof(ggml_fp16_t), "wrong fp16 size"); + +[[noreturn]] +static void ggml_cuda_error(const char * stmt, const char * func, const char * file, const int line, const char * msg) { + int id = -1; // in case cudaGetDevice fails + cudaGetDevice(&id); + + fprintf(stderr, "CUDA error: %s\n", msg); + fprintf(stderr, " current device: %d, in function %s at %s:%d\n", id, func, file, line); + fprintf(stderr, " %s\n", stmt); + // abort with GGML_ASSERT to get a stack trace + GGML_ASSERT(!"CUDA error"); +} + +#define CUDA_CHECK_GEN(err, success, error_fn) \ + do { \ + auto err_ = (err); \ + if (err_ != (success)) { \ + ggml_cuda_error(#err, __func__, __FILE__, __LINE__, error_fn(err_)); \ + } \ + } while (0) + +#define CUDA_CHECK(err) CUDA_CHECK_GEN(err, cudaSuccess, cudaGetErrorString) + +#if CUDART_VERSION >= 12000 + static const char * cublas_get_error_str(const cublasStatus_t err) { + return cublasGetStatusString(err); + } +#else + static const char * cublas_get_error_str(const cublasStatus_t err) { + switch (err) { + case CUBLAS_STATUS_SUCCESS: return "CUBLAS_STATUS_SUCCESS"; + case CUBLAS_STATUS_NOT_INITIALIZED: return "CUBLAS_STATUS_NOT_INITIALIZED"; + case CUBLAS_STATUS_ALLOC_FAILED: return "CUBLAS_STATUS_ALLOC_FAILED"; + case CUBLAS_STATUS_INVALID_VALUE: return "CUBLAS_STATUS_INVALID_VALUE"; + case CUBLAS_STATUS_ARCH_MISMATCH: return "CUBLAS_STATUS_ARCH_MISMATCH"; + case CUBLAS_STATUS_MAPPING_ERROR: return "CUBLAS_STATUS_MAPPING_ERROR"; + case CUBLAS_STATUS_EXECUTION_FAILED: return "CUBLAS_STATUS_EXECUTION_FAILED"; + case CUBLAS_STATUS_INTERNAL_ERROR: return "CUBLAS_STATUS_INTERNAL_ERROR"; + case CUBLAS_STATUS_NOT_SUPPORTED: return "CUBLAS_STATUS_NOT_SUPPORTED"; + default: return "unknown error"; + } + } +#endif // CUDART_VERSION >= 12000 + +#define CUBLAS_CHECK(err) CUDA_CHECK_GEN(err, CUBLAS_STATUS_SUCCESS, cublas_get_error_str) + +#if !defined(GGML_USE_HIPBLAS) +static const char * cu_get_error_str(CUresult err) { + const char * err_str; + cuGetErrorString(err, &err_str); + return err_str; +} +#define CU_CHECK(err) CUDA_CHECK_GEN(err, CUDA_SUCCESS, cu_get_error_str) +#endif + +#if CUDART_VERSION >= 11100 +#define GGML_CUDA_ASSUME(x) __builtin_assume(x) +#else +#define GGML_CUDA_ASSUME(x) +#endif // CUDART_VERSION >= 11100 + + +#define GGML_CUDA_MAX_STREAMS 8 + +struct ggml_tensor_extra_gpu { + void * data_device[GGML_CUDA_MAX_DEVICES]; // 1 pointer for each device for split tensors + cudaEvent_t events[GGML_CUDA_MAX_DEVICES][GGML_CUDA_MAX_STREAMS]; // events for synchronizing multiple GPUs +}; + +// this is faster on Windows +// probably because the Windows CUDA libraries forget to make this check before invoking the drivers +static void ggml_cuda_set_device(const int device) { + int current_device; + CUDA_CHECK(cudaGetDevice(¤t_device)); + + if (device == current_device) { + return; + } + + CUDA_CHECK(cudaSetDevice(device)); +} + +static int ggml_cuda_get_device() { + int id; + CUDA_CHECK(cudaGetDevice(&id)); + return id; +} + +struct ggml_cuda_device_info { + int device_count; + + struct cuda_device_info { + int cc; // compute capability + size_t smpb; // max. shared memory per block + bool vmm; // virtual memory support + size_t vmm_granularity; // granularity of virtual memory + size_t total_vram; + }; + + cuda_device_info devices[GGML_CUDA_MAX_DEVICES] = {}; + + std::array default_tensor_split = {}; +}; + +static ggml_cuda_device_info ggml_cuda_init() { +#ifdef __HIP_PLATFORM_AMD__ + // Workaround for a rocBLAS bug when using multiple graphics cards: + // https://github.com/ROCmSoftwarePlatform/rocBLAS/issues/1346 + rocblas_initialize(); + CUDA_CHECK(cudaDeviceSynchronize()); +#endif + + ggml_cuda_device_info info = {}; + + if (cudaGetDeviceCount(&info.device_count) != cudaSuccess) { + fprintf(stderr, "%s: no " GGML_CUDA_NAME " devices found, " GGML_CUDA_NAME " will be disabled\n", __func__); + return info; + } + + GGML_ASSERT(info.device_count <= GGML_CUDA_MAX_DEVICES); + + int64_t total_vram = 0; +#if defined(GGML_CUDA_FORCE_MMQ) + fprintf(stderr, "%s: GGML_CUDA_FORCE_MMQ: yes\n", __func__); +#else + fprintf(stderr, "%s: GGML_CUDA_FORCE_MMQ: no\n", __func__); +#endif +#if defined(CUDA_USE_TENSOR_CORES) + fprintf(stderr, "%s: CUDA_USE_TENSOR_CORES: yes\n", __func__); +#else + fprintf(stderr, "%s: CUDA_USE_TENSOR_CORES: no\n", __func__); +#endif + fprintf(stderr, "%s: found %d " GGML_CUDA_NAME " devices:\n", __func__, info.device_count); + for (int id = 0; id < info.device_count; ++id) { + int device_vmm = 0; + +#if !defined(GGML_USE_HIPBLAS) + CUdevice device; + CU_CHECK(cuDeviceGet(&device, id)); + CU_CHECK(cuDeviceGetAttribute(&device_vmm, CU_DEVICE_ATTRIBUTE_VIRTUAL_MEMORY_MANAGEMENT_SUPPORTED, device)); + + if (device_vmm) { + CUmemAllocationProp alloc_prop = {}; + alloc_prop.type = CU_MEM_ALLOCATION_TYPE_PINNED; + alloc_prop.location.type = CU_MEM_LOCATION_TYPE_DEVICE; + alloc_prop.location.id = id; + CU_CHECK(cuMemGetAllocationGranularity(&info.devices[id].vmm_granularity, &alloc_prop, CU_MEM_ALLOC_GRANULARITY_RECOMMENDED)); + } +#endif // !defined(GGML_USE_HIPBLAS) + info.devices[id].vmm = !!device_vmm; + + cudaDeviceProp prop; + CUDA_CHECK(cudaGetDeviceProperties(&prop, id)); + fprintf(stderr, " Device %d: %s, compute capability %d.%d, VMM: %s\n", id, prop.name, prop.major, prop.minor, device_vmm ? "yes" : "no"); + + info.default_tensor_split[id] = total_vram; + total_vram += prop.totalGlobalMem; + +#if defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__) + info.devices[id].cc = 100*prop.major + 10*prop.minor + CC_OFFSET_AMD; +#else + info.devices[id].cc = 100*prop.major + 10*prop.minor; +#endif // defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__) + info.devices[id].smpb = prop.sharedMemPerBlock; + } + + for (int id = 0; id < info.device_count; ++id) { + info.default_tensor_split[id] /= total_vram; + } + + // configure logging to stdout + // CUBLAS_CHECK(cublasLoggerConfigure(1, 1, 0, nullptr)); + + return info; +} + +static const ggml_cuda_device_info & get_cuda_global_info() { + static ggml_cuda_device_info info = ggml_cuda_init(); + return info; +} + +// #define DEBUG_CUDA_MALLOC + +// buffer pool for cuda (legacy) +struct ggml_cuda_pool { + virtual ~ggml_cuda_pool() = default; + + virtual void * alloc(size_t size, size_t * actual_size) = 0; + virtual void free(void * ptr, size_t size) = 0; + + ggml_cuda_pool() = default; + ggml_cuda_pool(const ggml_cuda_pool &) = delete; + ggml_cuda_pool(ggml_cuda_pool &&) = delete; + ggml_cuda_pool& operator=(const ggml_cuda_pool &) = delete; + ggml_cuda_pool& operator=(ggml_cuda_pool &&) = delete; +}; + +struct ggml_cuda_pool_leg : public ggml_cuda_pool { + static const int MAX_BUFFERS = 256; + + int device; + struct ggml_cuda_buffer { + void * ptr = nullptr; + size_t size = 0; + }; + + ggml_cuda_buffer buffer_pool[MAX_BUFFERS] = {}; + size_t pool_size = 0; + + explicit ggml_cuda_pool_leg(int device) : + device(device) { + } + + ~ggml_cuda_pool_leg() { + ggml_cuda_set_device(device); + for (int i = 0; i < MAX_BUFFERS; ++i) { + ggml_cuda_buffer & b = buffer_pool[i]; + if (b.ptr != nullptr) { + CUDA_CHECK(cudaFree(b.ptr)); + pool_size -= b.size; + } + } + GGML_ASSERT(pool_size == 0); + } + + void * alloc(size_t size, size_t * actual_size) override { +#ifdef DEBUG_CUDA_MALLOC + int nnz = 0; + size_t max_size = 0; +#endif + size_t best_diff = 1ull << 36; + int ibest = -1; + for (int i = 0; i < MAX_BUFFERS; ++i) { + ggml_cuda_buffer& b = buffer_pool[i]; + if (b.ptr != nullptr) { +#ifdef DEBUG_CUDA_MALLOC + ++nnz; + if (b.size > max_size) max_size = b.size; +#endif + if (b.size >= size) { + size_t diff = b.size - size; + if (diff < best_diff) { + best_diff = diff; + ibest = i; + if (!best_diff) { + void * ptr = b.ptr; + *actual_size = b.size; + b.ptr = nullptr; + b.size = 0; + return ptr; + } + } + } + } + } + if (ibest >= 0) { + ggml_cuda_buffer& b = buffer_pool[ibest]; + void * ptr = b.ptr; + *actual_size = b.size; + b.ptr = nullptr; + b.size = 0; + return ptr; + } + void * ptr; + size_t look_ahead_size = (size_t) (1.05 * size); + look_ahead_size = 256 * ((look_ahead_size + 255)/256); + ggml_cuda_set_device(device); + CUDA_CHECK(cudaMalloc((void **) &ptr, look_ahead_size)); + *actual_size = look_ahead_size; + pool_size += look_ahead_size; +#ifdef DEBUG_CUDA_MALLOC + fprintf(stderr, "%s[%d]: %d buffers, max_size = %u MB, pool_size = %u MB, requested %u MB\n", __func__, device, nnz, + (uint32_t)(max_size/1024/1024), (uint32_t)(pool_size/1024/1024), (uint32_t)(size/1024/1024)); +#endif + return ptr; + } + + void free(void * ptr, size_t size) override { + for (int i = 0; i < MAX_BUFFERS; ++i) { + ggml_cuda_buffer& b = buffer_pool[i]; + if (b.ptr == nullptr) { + b.ptr = ptr; + b.size = size; + return; + } + } + fprintf(stderr, "WARNING: cuda buffer pool full, increase MAX_CUDA_BUFFERS\n"); + ggml_cuda_set_device(device); + CUDA_CHECK(cudaFree(ptr)); + pool_size -= size; + } +}; + +// pool with virtual memory +#if !defined(GGML_USE_HIPBLAS) +struct ggml_cuda_pool_vmm : public ggml_cuda_pool { + static const size_t CUDA_POOL_VMM_MAX_SIZE = 1ull << 35; // 32 GB + + int device; + CUdeviceptr pool_addr = 0; + size_t pool_used = 0; + size_t pool_size = 0; + size_t granularity; + + explicit ggml_cuda_pool_vmm(int device) : + device(device), + granularity(get_cuda_global_info().devices[device].vmm_granularity) { + } + + ~ggml_cuda_pool_vmm() { + if (pool_addr != 0) { + CU_CHECK(cuMemUnmap(pool_addr, pool_size)); + CU_CHECK(cuMemAddressFree(pool_addr, CUDA_POOL_VMM_MAX_SIZE)); + } + } + + void * alloc(size_t size, size_t * actual_size) override { + // round up the allocation size to the alignment to ensure that all allocations are aligned for all data types + const size_t alignment = 128; + size = alignment * ((size + alignment - 1) / alignment); + + size_t avail = pool_size - pool_used; + + if (size > avail) { + // round up to the next multiple of the granularity + size_t reserve_size = size - avail; + reserve_size = granularity * ((reserve_size + granularity - 1) / granularity); + + GGML_ASSERT(pool_size + reserve_size <= CUDA_POOL_VMM_MAX_SIZE); + + // allocate more physical memory + CUmemAllocationProp prop = {}; + prop.type = CU_MEM_ALLOCATION_TYPE_PINNED; + prop.location.type = CU_MEM_LOCATION_TYPE_DEVICE; + prop.location.id = device; + CUmemGenericAllocationHandle handle; + CU_CHECK(cuMemCreate(&handle, reserve_size, &prop, 0)); + + // reserve virtual address space (if not already reserved) + if (pool_addr == 0) { + CU_CHECK(cuMemAddressReserve(&pool_addr, CUDA_POOL_VMM_MAX_SIZE, 0, 0, 0)); + } + + // map at the end of the pool + CU_CHECK(cuMemMap(pool_addr + pool_size, reserve_size, 0, handle, 0)); + + // the memory allocation handle is no longer needed after mapping + CU_CHECK(cuMemRelease(handle)); + + // set access + CUmemAccessDesc access = {}; + access.location.type = CU_MEM_LOCATION_TYPE_DEVICE; + access.location.id = device; + access.flags = CU_MEM_ACCESS_FLAGS_PROT_READWRITE; + CU_CHECK(cuMemSetAccess(pool_addr + pool_size, reserve_size, &access, 1)); + + // add to the pool + pool_size += reserve_size; + + //printf("cuda pool[%d]: size increased to %llu MB (reserved %llu MB)\n", + // id, (unsigned long long) (pool_size[id]/1024/1024), + // (unsigned long long) (reserve_size/1024/1024)); + } + + GGML_ASSERT(pool_addr != 0); + + void * ptr = (void *) (pool_addr + pool_used); + *actual_size = size; + pool_used += size; + +#ifdef DEBUG_CUDA_MALLOC + printf("cuda pool[%d]: allocated %llu bytes at %llx\n", device, (unsigned long long) size, ptr); +#endif + + return ptr; + } + + void free(void * ptr, size_t size) override { +#ifdef DEBUG_CUDA_MALLOC + printf("cuda pool[%d]: freed %llu bytes at %llx\n", device, (unsigned long long) size, ptr); +#endif + + pool_used -= size; + + // all deallocations must be in reverse order of the allocations + GGML_ASSERT(ptr == (void *) (pool_addr + pool_used)); + } +}; +#endif // !defined(GGML_USE_HIPBLAS) + +template +struct ggml_cuda_pool_alloc { + ggml_cuda_pool * pool = nullptr; + T * ptr = nullptr; + size_t actual_size = 0; + + ggml_cuda_pool_alloc() = default; + + explicit ggml_cuda_pool_alloc(ggml_cuda_pool & pool) : pool(&pool) { + } + + ggml_cuda_pool_alloc(ggml_cuda_pool & pool, size_t size) : pool(&pool) { + alloc(size); + } + + ~ggml_cuda_pool_alloc() { + if (ptr != nullptr) { + pool->free(ptr, actual_size); + } + } + + // size is in number of elements + T * alloc(size_t size) { + GGML_ASSERT(pool != nullptr); + GGML_ASSERT(ptr == nullptr); + ptr = (T *) pool->alloc(size * sizeof(T), &this->actual_size); + return ptr; + } + + T * alloc(ggml_cuda_pool & pool, size_t size) { + this->pool = &pool; + return alloc(size); + } + + T * get() { + return ptr; + } + + ggml_cuda_pool_alloc(const ggml_cuda_pool_alloc &) = delete; + ggml_cuda_pool_alloc(ggml_cuda_pool_alloc &&) = delete; + ggml_cuda_pool_alloc& operator=(const ggml_cuda_pool_alloc &) = delete; + ggml_cuda_pool_alloc& operator=(ggml_cuda_pool_alloc &&) = delete; +}; + + +// backend interface + +struct ggml_backend_cuda_context { + int device; + std::string name; + cudaEvent_t copy_event = nullptr; + + cudaStream_t streams[GGML_CUDA_MAX_DEVICES][GGML_CUDA_MAX_STREAMS] = { { nullptr } }; + cublasHandle_t cublas_handles[GGML_CUDA_MAX_DEVICES] = {nullptr}; + + explicit ggml_backend_cuda_context(int device) : + device(device), + name(GGML_CUDA_NAME + std::to_string(device)) { + } + + ~ggml_backend_cuda_context() { + if (copy_event != nullptr) { + CUDA_CHECK(cudaEventDestroy(copy_event)); + } + for (int i = 0; i < GGML_CUDA_MAX_DEVICES; ++i) { + for (int j = 0; j < GGML_CUDA_MAX_STREAMS; ++j) { + if (streams[i][j] != nullptr) { + CUDA_CHECK(cudaStreamDestroy(streams[i][j])); + } + } + if (cublas_handles[i] != nullptr) { + CUBLAS_CHECK(cublasDestroy(cublas_handles[i])); + } + } + } + + cudaStream_t stream(int device, int stream) { + if (streams[device][stream] == nullptr) { + ggml_cuda_set_device(device); + CUDA_CHECK(cudaStreamCreateWithFlags(&streams[device][stream], cudaStreamNonBlocking)); + } + return streams[device][stream]; + } + + cudaStream_t stream() { + return stream(device, 0); + } + + cublasHandle_t cublas_handle(int device) { + if (cublas_handles[device] == nullptr) { + ggml_cuda_set_device(device); + CUBLAS_CHECK(cublasCreate(&cublas_handles[device])); + CUBLAS_CHECK(cublasSetMathMode(cublas_handles[device], CUBLAS_TF32_TENSOR_OP_MATH)); + } + return cublas_handles[device]; + } + + cublasHandle_t cublas_handle() { + return cublas_handle(device); + } + + // pool + std::unique_ptr pools[GGML_CUDA_MAX_DEVICES]; + + static std::unique_ptr new_pool_for_device(int device) { +#if !defined(GGML_USE_HIPBLAS) + if (get_cuda_global_info().devices[device].vmm) { + return std::unique_ptr(new ggml_cuda_pool_vmm(device)); + } +#endif + return std::unique_ptr(new ggml_cuda_pool_leg(device)); + } + + ggml_cuda_pool & pool(int device) { + if (pools[device] == nullptr) { + pools[device] = new_pool_for_device(device); + } + return *pools[device]; + } + + ggml_cuda_pool & pool() { + return pool(device); + } +}; + +// cuda buffer + +struct ggml_backend_cuda_buffer_context { + int device; + void * dev_ptr = nullptr; + std::string name; + + ggml_backend_cuda_buffer_context(int device, void * dev_ptr) : + device(device), dev_ptr(dev_ptr), + name(GGML_CUDA_NAME + std::to_string(device)) { + } + + ~ggml_backend_cuda_buffer_context() { + CUDA_CHECK(cudaFree(dev_ptr)); + } +}; + +GGML_CALL static const char * ggml_backend_cuda_buffer_get_name(ggml_backend_buffer_t buffer) { + ggml_backend_cuda_buffer_context * ctx = (ggml_backend_cuda_buffer_context *)buffer->context; + return ctx->name.c_str(); +} + +GGML_CALL static bool ggml_backend_buffer_is_cuda(ggml_backend_buffer_t buffer) { + return buffer->iface.get_name == ggml_backend_cuda_buffer_get_name; +} + +GGML_CALL static void ggml_backend_cuda_buffer_free_buffer(ggml_backend_buffer_t buffer) { + ggml_backend_cuda_buffer_context * ctx = (ggml_backend_cuda_buffer_context *)buffer->context; + delete ctx; +} + +GGML_CALL static void * ggml_backend_cuda_buffer_get_base(ggml_backend_buffer_t buffer) { + ggml_backend_cuda_buffer_context * ctx = (ggml_backend_cuda_buffer_context *)buffer->context; + return ctx->dev_ptr; +} + +GGML_CALL static void ggml_backend_cuda_buffer_init_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor) { + ggml_backend_cuda_buffer_context * ctx = (ggml_backend_cuda_buffer_context *)buffer->context; + + if (tensor->view_src != NULL && tensor->view_offs == 0) { + assert(tensor->view_src->buffer->buft == buffer->buft); + tensor->backend = tensor->view_src->backend; + tensor->extra = tensor->view_src->extra; + return; + } + + if (ggml_is_quantized(tensor->type)) { + // initialize padding to 0 to avoid possible NaN values + size_t original_size = ggml_nbytes(tensor); + size_t padded_size = ggml_backend_buft_get_alloc_size(buffer->buft, tensor); + + if (padded_size > original_size && tensor->view_src == nullptr) { + ggml_cuda_set_device(ctx->device); + CUDA_CHECK(cudaMemset((char *)tensor->data + original_size, 0, padded_size - original_size)); + } + } +} + +GGML_CALL static void ggml_backend_cuda_buffer_set_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor, const void * data, size_t offset, size_t size) { + ggml_backend_cuda_buffer_context * ctx = (ggml_backend_cuda_buffer_context *)buffer->context; + + ggml_cuda_set_device(ctx->device); + CUDA_CHECK(cudaMemcpyAsync((char *)tensor->data + offset, data, size, cudaMemcpyHostToDevice, cudaStreamPerThread)); + CUDA_CHECK(cudaStreamSynchronize(cudaStreamPerThread)); +} + +GGML_CALL static void ggml_backend_cuda_buffer_get_tensor(ggml_backend_buffer_t buffer, const ggml_tensor * tensor, void * data, size_t offset, size_t size) { + ggml_backend_cuda_buffer_context * ctx = (ggml_backend_cuda_buffer_context *)buffer->context; + + ggml_cuda_set_device(ctx->device); + CUDA_CHECK(cudaMemcpyAsync(data, (const char *)tensor->data + offset, size, cudaMemcpyDeviceToHost, cudaStreamPerThread)); + CUDA_CHECK(cudaStreamSynchronize(cudaStreamPerThread)); +} + +GGML_CALL static bool ggml_backend_cuda_buffer_cpy_tensor(ggml_backend_buffer_t buffer, const ggml_tensor * src, ggml_tensor * dst) { + if (ggml_backend_buffer_is_cuda(src->buffer)) { + ggml_backend_cuda_buffer_context * src_ctx = (ggml_backend_cuda_buffer_context *)src->buffer->context; + ggml_backend_cuda_buffer_context * dst_ctx = (ggml_backend_cuda_buffer_context *)dst->buffer->context; + if (src_ctx->device == dst_ctx->device) { + CUDA_CHECK(cudaMemcpyAsync(dst->data, src->data, ggml_nbytes(src), cudaMemcpyDeviceToDevice, cudaStreamPerThread)); + } else { + CUDA_CHECK(cudaMemcpyPeerAsync(dst->data, dst_ctx->device, src->data, src_ctx->device, ggml_nbytes(src), cudaStreamPerThread)); + } + CUDA_CHECK(cudaStreamSynchronize(cudaStreamPerThread)); + return true; + } + return false; + + GGML_UNUSED(buffer); +} + +GGML_CALL static void ggml_backend_cuda_buffer_clear(ggml_backend_buffer_t buffer, uint8_t value) { + ggml_backend_cuda_buffer_context * ctx = (ggml_backend_cuda_buffer_context *)buffer->context; + + ggml_cuda_set_device(ctx->device); + CUDA_CHECK(cudaDeviceSynchronize()); + CUDA_CHECK(cudaMemset(ctx->dev_ptr, value, buffer->size)); + CUDA_CHECK(cudaDeviceSynchronize()); +} + +static ggml_backend_buffer_i ggml_backend_cuda_buffer_interface = { + /* .get_name = */ ggml_backend_cuda_buffer_get_name, + /* .free_buffer = */ ggml_backend_cuda_buffer_free_buffer, + /* .get_base = */ ggml_backend_cuda_buffer_get_base, + /* .init_tensor = */ ggml_backend_cuda_buffer_init_tensor, + /* .set_tensor = */ ggml_backend_cuda_buffer_set_tensor, + /* .get_tensor = */ ggml_backend_cuda_buffer_get_tensor, + /* .cpy_tensor = */ ggml_backend_cuda_buffer_cpy_tensor, + /* .clear = */ ggml_backend_cuda_buffer_clear, + /* .reset = */ NULL, +}; + +// cuda buffer type +struct ggml_backend_cuda_buffer_type_context { + int device; + std::string name; +}; + +GGML_CALL static const char * ggml_backend_cuda_buffer_type_name(ggml_backend_buffer_type_t buft) { + ggml_backend_cuda_buffer_type_context * ctx = (ggml_backend_cuda_buffer_type_context *)buft->context; + + return ctx->name.c_str(); +} + +GGML_CALL static ggml_backend_buffer_t ggml_backend_cuda_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size) { + ggml_backend_cuda_buffer_type_context * buft_ctx = (ggml_backend_cuda_buffer_type_context *)buft->context; + + ggml_cuda_set_device(buft_ctx->device); + + size = std::max(size, (size_t)1); // cudaMalloc returns null for size 0 + + void * dev_ptr; + cudaError_t err = cudaMalloc(&dev_ptr, size); + if (err != cudaSuccess) { + fprintf(stderr, "%s: allocating %.2f MiB on device %d: cudaMalloc failed: %s\n", __func__, size/1024.0/1024.0, buft_ctx->device, cudaGetErrorString(err)); + return nullptr; + } + + ggml_backend_cuda_buffer_context * ctx = new ggml_backend_cuda_buffer_context(buft_ctx->device, dev_ptr); + + return ggml_backend_buffer_init(buft, ggml_backend_cuda_buffer_interface, ctx, size); +} + +GGML_CALL static size_t ggml_backend_cuda_buffer_type_get_alignment(ggml_backend_buffer_type_t buft) { + return 128; + + GGML_UNUSED(buft); +} + +GGML_CALL static size_t ggml_backend_cuda_buffer_type_get_alloc_size(ggml_backend_buffer_type_t buft, const ggml_tensor * tensor) { + size_t size = ggml_nbytes(tensor); + int64_t ne0 = tensor->ne[0]; + + if (ggml_is_quantized(tensor->type)) { + if (ne0 % MATRIX_ROW_PADDING != 0) { + size += ggml_row_size(tensor->type, MATRIX_ROW_PADDING - ne0 % MATRIX_ROW_PADDING); + } + } + + return size; + + GGML_UNUSED(buft); +} + +GGML_CALL static bool ggml_backend_cuda_buffer_type_supports_backend(ggml_backend_buffer_type_t buft, ggml_backend_t backend) { + if (!ggml_backend_is_cuda(backend)) { + return false; + } + + ggml_backend_cuda_buffer_type_context * buft_ctx = (ggml_backend_cuda_buffer_type_context *)buft->context; + ggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *)backend->context; + + return buft_ctx->device == cuda_ctx->device; +} + +static ggml_backend_buffer_type_i ggml_backend_cuda_buffer_type_interface = { + /* .get_name = */ ggml_backend_cuda_buffer_type_name, + /* .alloc_buffer = */ ggml_backend_cuda_buffer_type_alloc_buffer, + /* .get_alignment = */ ggml_backend_cuda_buffer_type_get_alignment, + /* .get_max_size = */ NULL, // defaults to SIZE_MAX + /* .get_alloc_size = */ ggml_backend_cuda_buffer_type_get_alloc_size, + /* .supports_backend = */ ggml_backend_cuda_buffer_type_supports_backend, + /* .is_host = */ NULL, +}; + +GGML_CALL ggml_backend_buffer_type_t ggml_backend_cuda_buffer_type(int device) { + static std::mutex mutex; + std::lock_guard lock(mutex); + + if (device >= ggml_backend_cuda_get_device_count()) { + return nullptr; + } + + static ggml_backend_buffer_type ggml_backend_cuda_buffer_types[GGML_CUDA_MAX_DEVICES]; + + static bool ggml_backend_cuda_buffer_type_initialized = false; + + if (!ggml_backend_cuda_buffer_type_initialized) { + for (int i = 0; i < GGML_CUDA_MAX_DEVICES; i++) { + ggml_backend_cuda_buffer_types[i] = { + /* .iface = */ ggml_backend_cuda_buffer_type_interface, + /* .context = */ new ggml_backend_cuda_buffer_type_context{i, GGML_CUDA_NAME + std::to_string(i)}, + }; + } + ggml_backend_cuda_buffer_type_initialized = true; + } + + return &ggml_backend_cuda_buffer_types[device]; +} + +// cuda split buffer + +static int64_t get_row_rounding(ggml_type type, const std::array & tensor_split) { + int64_t min_compute_capability = INT_MAX; + int64_t max_compute_capability = INT_MIN; + for (int id = 0; id < ggml_backend_cuda_get_device_count(); ++id) { + if (tensor_split[id] < (id + 1 < ggml_backend_cuda_get_device_count() ? tensor_split[id + 1] : 1.0f)) { + if (min_compute_capability > get_cuda_global_info().devices[id].cc) { + min_compute_capability = get_cuda_global_info().devices[id].cc; + } + if (max_compute_capability < get_cuda_global_info().devices[id].cc) { + max_compute_capability = get_cuda_global_info().devices[id].cc; + } + } + } + +#if defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__) + switch(type) { + case GGML_TYPE_Q4_0: + case GGML_TYPE_Q4_1: + case GGML_TYPE_Q5_0: + case GGML_TYPE_Q5_1: + case GGML_TYPE_Q8_0: + return max_compute_capability >= CC_RDNA2 ? 128 : 64; + case GGML_TYPE_F16: + case GGML_TYPE_F32: + return 1; + case GGML_TYPE_Q2_K: + return max_compute_capability >= CC_RDNA2 ? 128 : 32; + case GGML_TYPE_Q3_K: + return min_compute_capability < CC_RDNA2 ? 128 : 64; + case GGML_TYPE_Q4_K: + case GGML_TYPE_Q5_K: + case GGML_TYPE_Q6_K: + case GGML_TYPE_IQ2_XXS: + case GGML_TYPE_IQ2_XS: + case GGML_TYPE_IQ2_S: + case GGML_TYPE_IQ3_XXS: + case GGML_TYPE_IQ1_S: + case GGML_TYPE_IQ4_NL: + case GGML_TYPE_IQ4_XS: + case GGML_TYPE_IQ3_S: + return max_compute_capability >= CC_RDNA2 ? 128 : 64; + default: + GGML_ASSERT(false); + } +#else + switch(type) { + case GGML_TYPE_Q4_0: + case GGML_TYPE_Q4_1: + return max_compute_capability >= CC_VOLTA ? 128 : 64; + case GGML_TYPE_Q5_0: + case GGML_TYPE_Q5_1: + case GGML_TYPE_Q8_0: + return 64; + case GGML_TYPE_F16: + case GGML_TYPE_F32: + return 1; + case GGML_TYPE_Q2_K: + case GGML_TYPE_Q3_K: + case GGML_TYPE_Q4_K: + case GGML_TYPE_Q5_K: + case GGML_TYPE_IQ2_XXS: + case GGML_TYPE_IQ2_XS: + case GGML_TYPE_IQ2_S: + case GGML_TYPE_IQ3_XXS: + case GGML_TYPE_IQ1_S: + case GGML_TYPE_IQ4_NL: + case GGML_TYPE_IQ4_XS: + case GGML_TYPE_IQ3_S: + return max_compute_capability >= CC_VOLTA ? 128 : 64; + case GGML_TYPE_Q6_K: + return 64; + default: + GGML_ASSERT(false); + } +#endif // defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__) +} + +static void get_row_split(int64_t * row_low, int64_t * row_high, const ggml_tensor * tensor, const std::array & tensor_split, int id) { + const int64_t nrows = ggml_nrows(tensor); + const int64_t rounding = get_row_rounding(tensor->type, tensor_split); + + *row_low = id == 0 ? 0 : nrows*tensor_split[id]; + *row_low -= *row_low % rounding; + + if (id == ggml_backend_cuda_get_device_count() - 1) { + *row_high = nrows; + } else { + *row_high = nrows*tensor_split[id + 1]; + *row_high -= *row_high % rounding; + } +} + +static size_t ggml_nbytes_split(const struct ggml_tensor * tensor, int nrows_split) { + static_assert(GGML_MAX_DIMS == 4, "GGML_MAX_DIMS is not 4 - update this function"); + + return nrows_split*ggml_row_size(tensor->type, tensor->ne[0]); +} + +struct ggml_backend_cuda_split_buffer_type_context { + std::array tensor_split; +}; + +struct ggml_backend_cuda_split_buffer_context { + ~ggml_backend_cuda_split_buffer_context() { + for (ggml_tensor_extra_gpu * extra : tensor_extras) { + for (int id = 0; id < GGML_CUDA_MAX_DEVICES; ++id) { + for (int64_t is = 0; is < GGML_CUDA_MAX_STREAMS; ++is) { + if (extra->events[id][is] != nullptr) { + CUDA_CHECK(cudaEventDestroy(extra->events[id][is])); + } + } + if (extra->data_device[id] != nullptr) { + CUDA_CHECK(cudaFree(extra->data_device[id])); + } + } + delete extra; + } + } + + std::vector tensor_extras; +}; + +GGML_CALL static const char * ggml_backend_cuda_split_buffer_get_name(ggml_backend_buffer_t buffer) { + return GGML_CUDA_NAME "_Split"; + + GGML_UNUSED(buffer); +} + +static bool ggml_backend_buffer_is_cuda_split(ggml_backend_buffer_t buffer) { + return buffer->iface.get_name == ggml_backend_cuda_split_buffer_get_name; + GGML_UNUSED(ggml_backend_buffer_is_cuda_split); // only used in debug builds currently, avoid unused function warning in release builds +} + +GGML_CALL static void ggml_backend_cuda_split_buffer_free_buffer(ggml_backend_buffer_t buffer) { + ggml_backend_cuda_split_buffer_context * ctx = (ggml_backend_cuda_split_buffer_context *)buffer->context; + delete ctx; +} + +GGML_CALL static void * ggml_backend_cuda_split_buffer_get_base(ggml_backend_buffer_t buffer) { + // the pointers are stored in the tensor extras, this is just a dummy address and never dereferenced + return (void *)0x1000; + + GGML_UNUSED(buffer); +} + +GGML_CALL static void ggml_backend_cuda_split_buffer_init_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor) { + GGML_ASSERT(tensor->view_src == nullptr); // views of split tensors are not supported + + ggml_backend_cuda_split_buffer_context * ctx = (ggml_backend_cuda_split_buffer_context *)buffer->context; + ggml_backend_cuda_split_buffer_type_context * buft_ctx = (ggml_backend_cuda_split_buffer_type_context *)buffer->buft->context; + + const int64_t ne0 = tensor->ne[0]; + + ggml_tensor_extra_gpu * extra = new ggml_tensor_extra_gpu{}; + ctx->tensor_extras.push_back(extra); + + for (int id = 0; id < ggml_backend_cuda_get_device_count(); ++id) { + int64_t row_low, row_high; + get_row_split(&row_low, &row_high, tensor, buft_ctx->tensor_split, id); + + int64_t nrows_split = row_high - row_low; + if (nrows_split == 0) { + continue; + } + + size_t size = ggml_nbytes_split(tensor, nrows_split); + const size_t original_size = size; + + // pad last row to a multiple of 512 elements to avoid out-of-bounds memory accesses + if (ne0 % MATRIX_ROW_PADDING != 0) { + size += ggml_row_size(tensor->type, MATRIX_ROW_PADDING - ne0 % MATRIX_ROW_PADDING); + } + + // FIXME: do not crash if cudaMalloc fails + // currently, init_tensor cannot fail, it needs to be fixed in ggml-backend first + ggml_cuda_set_device(id); + char * buf; + CUDA_CHECK(cudaMalloc(&buf, size)); + + // set padding to 0 to avoid possible NaN values + if (size > original_size) { + CUDA_CHECK(cudaMemset(buf + original_size, 0, size - original_size)); + } + + extra->data_device[id] = buf; + + for (int64_t is = 0; is < GGML_CUDA_MAX_STREAMS; ++is) { + CUDA_CHECK(cudaEventCreateWithFlags(&extra->events[id][is], cudaEventDisableTiming)); + } + } + tensor->extra = extra; +} + +GGML_CALL static void ggml_backend_cuda_split_buffer_set_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor, const void * data, size_t offset, size_t size) { + // split tensors must always be set in their entirety at once + GGML_ASSERT(offset == 0); + GGML_ASSERT(size == ggml_nbytes(tensor)); + + ggml_backend_cuda_split_buffer_type_context * buft_ctx = (ggml_backend_cuda_split_buffer_type_context *)buffer->buft->context; + + const int64_t ne0 = tensor->ne[0]; + const size_t nb1 = tensor->nb[1]; + ggml_tensor_extra_gpu * extra = (ggml_tensor_extra_gpu *)tensor->extra; + + for (int id = 0; id < ggml_backend_cuda_get_device_count(); ++id) { + int64_t row_low, row_high; + get_row_split(&row_low, &row_high, tensor, buft_ctx->tensor_split, id); + + int64_t nrows_split = row_high - row_low; + if (nrows_split == 0) { + continue; + } + + const size_t offset_split = row_low*nb1; + size_t size = ggml_nbytes_split(tensor, nrows_split); + const size_t original_size = size; + + // pad last row to a multiple of 512 elements to avoid out-of-bounds memory accesses + if (ne0 % MATRIX_ROW_PADDING != 0) { + size += ggml_row_size(tensor->type, MATRIX_ROW_PADDING - ne0 % MATRIX_ROW_PADDING); + } + + const char * buf_host = (const char *)data + offset_split; + CUDA_CHECK(cudaMemcpyAsync(extra->data_device[id], buf_host, original_size, cudaMemcpyHostToDevice, cudaStreamPerThread)); + } + + for (int id = 0; id < ggml_backend_cuda_get_device_count(); ++id) { + CUDA_CHECK(cudaStreamSynchronize(cudaStreamPerThread)); + } +} + +GGML_CALL static void ggml_backend_cuda_split_buffer_get_tensor(ggml_backend_buffer_t buffer, const ggml_tensor * tensor, void * data, size_t offset, size_t size) { + // split tensors must always be set in their entirety at once + GGML_ASSERT(offset == 0); + GGML_ASSERT(size == ggml_nbytes(tensor)); + + ggml_backend_cuda_split_buffer_type_context * buft_ctx = (ggml_backend_cuda_split_buffer_type_context *)buffer->buft->context; + + const int64_t ne0 = tensor->ne[0]; + const size_t nb1 = tensor->nb[1]; + ggml_tensor_extra_gpu * extra = (ggml_tensor_extra_gpu *)tensor->extra; + + for (int id = 0; id < ggml_backend_cuda_get_device_count(); ++id) { + int64_t row_low, row_high; + get_row_split(&row_low, &row_high, tensor, buft_ctx->tensor_split, id); + + int64_t nrows_split = row_high - row_low; + if (nrows_split == 0) { + continue; + } + + const size_t offset_split = row_low*nb1; + size_t size = ggml_nbytes_split(tensor, nrows_split); + const size_t original_size = size; + + // pad last row to a multiple of 512 elements to avoid out-of-bounds memory accesses + if (ne0 % MATRIX_ROW_PADDING != 0) { + size += ggml_row_size(tensor->type, MATRIX_ROW_PADDING - ne0 % MATRIX_ROW_PADDING); + } + + char * buf_host = (char *)data + offset_split; + CUDA_CHECK(cudaMemcpyAsync(buf_host, extra->data_device[id], original_size, cudaMemcpyDeviceToHost, cudaStreamPerThread)); + } + + for (int id = 0; id < ggml_backend_cuda_get_device_count(); ++id) { + CUDA_CHECK(cudaStreamSynchronize(cudaStreamPerThread)); + } +} + +GGML_CALL static void ggml_backend_cuda_split_buffer_clear(ggml_backend_buffer_t buffer, uint8_t value) { + GGML_UNUSED(buffer); + GGML_UNUSED(value); +} + +static struct ggml_backend_buffer_i ggml_backend_cuda_split_buffer_interface = { + /* .get_name = */ ggml_backend_cuda_split_buffer_get_name, + /* .free_buffer = */ ggml_backend_cuda_split_buffer_free_buffer, + /* .get_base = */ ggml_backend_cuda_split_buffer_get_base, + /* .init_tensor = */ ggml_backend_cuda_split_buffer_init_tensor, + /* .set_tensor = */ ggml_backend_cuda_split_buffer_set_tensor, + /* .get_tensor = */ ggml_backend_cuda_split_buffer_get_tensor, + /* .cpy_tensor = */ NULL, + /* .clear = */ ggml_backend_cuda_split_buffer_clear, + /* .reset = */ NULL, +}; + +// cuda split buffer type + +GGML_CALL static const char * ggml_backend_cuda_split_buffer_type_name(ggml_backend_buffer_type_t buft) { + return GGML_CUDA_NAME "_Split"; + + GGML_UNUSED(buft); +} + +GGML_CALL static ggml_backend_buffer_t ggml_backend_cuda_split_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size) { + // since we don't know the exact split after rounding, we cannot allocate the device buffers at this point + // instead, we allocate them for each tensor separately in init_tensor + // however, the size still represents the maximum cumulative size of all the device buffers after the tensors are allocated, + // as returned by get_alloc_size. this limit is enforced during tensor allocation by ggml-alloc, so it must be correct. + ggml_backend_cuda_split_buffer_context * ctx = new ggml_backend_cuda_split_buffer_context(); + + return ggml_backend_buffer_init(buft, ggml_backend_cuda_split_buffer_interface, ctx, size); +} + +GGML_CALL static size_t ggml_backend_cuda_split_buffer_type_get_alignment(ggml_backend_buffer_type_t buft) { + return 128; + + GGML_UNUSED(buft); +} + +GGML_CALL static size_t ggml_backend_cuda_split_buffer_type_get_alloc_size(ggml_backend_buffer_type_t buft, const ggml_tensor * tensor) { + ggml_backend_cuda_split_buffer_type_context * ctx = (ggml_backend_cuda_split_buffer_type_context *)buft->context; + + size_t total_size = 0; + + const int64_t ne0 = tensor->ne[0]; + + for (int id = 0; id < ggml_backend_cuda_get_device_count(); ++id) { + int64_t row_low, row_high; + get_row_split(&row_low, &row_high, tensor, ctx->tensor_split, id); + + int64_t nrows_split = row_high - row_low; + if (nrows_split == 0) { + continue; + } + + total_size += ggml_nbytes_split(tensor, nrows_split); + + // pad last row to a multiple of 512 elements to avoid out-of-bounds memory accesses + if (ne0 % MATRIX_ROW_PADDING != 0) { + total_size += ggml_row_size(tensor->type, MATRIX_ROW_PADDING - ne0 % MATRIX_ROW_PADDING); + } + } + + return total_size; +} + +GGML_CALL static bool ggml_backend_cuda_split_buffer_type_supports_backend(ggml_backend_buffer_type_t buft, ggml_backend_t backend) { + return ggml_backend_is_cuda(backend); + + GGML_UNUSED(buft); +} + +GGML_CALL static bool ggml_backend_cuda_split_buffer_type_is_host(ggml_backend_buffer_type_t buft) { + return false; + + GGML_UNUSED(buft); +} + +static ggml_backend_buffer_type_i ggml_backend_cuda_split_buffer_type_interface = { + /* .get_name = */ ggml_backend_cuda_split_buffer_type_name, + /* .alloc_buffer = */ ggml_backend_cuda_split_buffer_type_alloc_buffer, + /* .get_alignment = */ ggml_backend_cuda_split_buffer_type_get_alignment, + /* .get_max_size = */ NULL, // defaults to SIZE_MAX + /* .get_alloc_size = */ ggml_backend_cuda_split_buffer_type_get_alloc_size, + /* .supports_backend = */ ggml_backend_cuda_split_buffer_type_supports_backend, + /* .is_host = */ ggml_backend_cuda_split_buffer_type_is_host, +}; + +GGML_CALL ggml_backend_buffer_type_t ggml_backend_cuda_split_buffer_type(const float * tensor_split) { + static std::mutex mutex; + std::lock_guard lock(mutex); + + static std::map, struct ggml_backend_buffer_type> buft_map; + + std::array tensor_split_arr = {}; + + bool all_zero = tensor_split == nullptr || std::all_of(tensor_split, tensor_split + GGML_CUDA_MAX_DEVICES, [](float x) { return x == 0.0f; }); + if (all_zero) { + tensor_split_arr = get_cuda_global_info().default_tensor_split; + } else { + float split_sum = 0.0f; + for (int i = 0; i < ggml_backend_cuda_get_device_count(); ++i) { + tensor_split_arr[i] = split_sum; + split_sum += tensor_split[i]; + } + for (int i = 0; i < ggml_backend_cuda_get_device_count(); ++i) { + tensor_split_arr[i] /= split_sum; + } + } + + auto it = buft_map.find(tensor_split_arr); + if (it != buft_map.end()) { + return &it->second; + } + + struct ggml_backend_buffer_type buft { + /* .iface = */ ggml_backend_cuda_split_buffer_type_interface, + /* .context = */ new ggml_backend_cuda_split_buffer_type_context{tensor_split_arr}, + }; + + auto result = buft_map.emplace(tensor_split_arr, buft); + return &result.first->second; +} + +// host buffer type + +GGML_CALL static const char * ggml_backend_cuda_host_buffer_type_name(ggml_backend_buffer_type_t buft) { + return GGML_CUDA_NAME "_Host"; + + GGML_UNUSED(buft); +} + +GGML_CALL static const char * ggml_backend_cuda_host_buffer_name(ggml_backend_buffer_t buffer) { + return GGML_CUDA_NAME "_Host"; + + GGML_UNUSED(buffer); +} + +GGML_CALL static void ggml_backend_cuda_host_buffer_free_buffer(ggml_backend_buffer_t buffer) { + CUDA_CHECK(cudaFreeHost(buffer->context)); +} + +static void * ggml_cuda_host_malloc(size_t size) { + if (getenv("GGML_CUDA_NO_PINNED") != nullptr) { + return nullptr; + } + + void * ptr = nullptr; + cudaError_t err = cudaMallocHost((void **) &ptr, size); + if (err != cudaSuccess) { + // clear the error + cudaGetLastError(); + fprintf(stderr, "%s: warning: failed to allocate %.2f MiB of pinned memory: %s\n", __func__, + size/1024.0/1024.0, cudaGetErrorString(err)); + return nullptr; + } + + return ptr; +} + +GGML_CALL static ggml_backend_buffer_t ggml_backend_cuda_host_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size) { + void * ptr = ggml_cuda_host_malloc(size); + + if (ptr == nullptr) { + // fallback to cpu buffer + return ggml_backend_buft_alloc_buffer(ggml_backend_cpu_buffer_type(), size); + } + + ggml_backend_buffer_t buffer = ggml_backend_cpu_buffer_from_ptr(ptr, size); + buffer->buft = buft; + buffer->iface.get_name = ggml_backend_cuda_host_buffer_name; + buffer->iface.free_buffer = ggml_backend_cuda_host_buffer_free_buffer; + + return buffer; +} + +GGML_CALL ggml_backend_buffer_type_t ggml_backend_cuda_host_buffer_type() { + static struct ggml_backend_buffer_type ggml_backend_cuda_buffer_type_host = { + /* .iface = */ { + /* .get_name = */ ggml_backend_cuda_host_buffer_type_name, + /* .alloc_buffer = */ ggml_backend_cuda_host_buffer_type_alloc_buffer, + /* .get_alignment = */ ggml_backend_cpu_buffer_type()->iface.get_alignment, + /* .get_max_size = */ NULL, // defaults to SIZE_MAX + /* .get_alloc_size = */ ggml_backend_cpu_buffer_type()->iface.get_alloc_size, + /* .supports_backend = */ ggml_backend_cpu_buffer_type()->iface.supports_backend, + /* .is_host = */ ggml_backend_cpu_buffer_type()->iface.is_host, + }, + /* .context = */ nullptr, + }; + + return &ggml_backend_cuda_buffer_type_host; +} + +//static bool ggml_backend_buffer_is_cuda_host(ggml_backend_buffer_t buffer) { +// return buffer->buft->iface.get_name == ggml_backend_cuda_host_buffer_type_name; +//} + + + +/// kernels + + #if defined(GGML_USE_HIPBLAS) #define __CUDA_ARCH__ 1300 @@ -253,71 +1463,6 @@ static __device__ __forceinline__ int __dp4a(const int a, const int b, int c) { } #endif // defined(GGML_USE_HIPBLAS) -#if defined(_MSC_VER) -#pragma warning(disable: 4244 4267) // possible loss of data -#endif - -static_assert(sizeof(half) == sizeof(ggml_fp16_t), "wrong fp16 size"); - -[[noreturn]] -static void ggml_cuda_error(const char * stmt, const char * func, const char * file, const int line, const char * msg) { - int id = -1; // in case cudaGetDevice fails - cudaGetDevice(&id); - - fprintf(stderr, "CUDA error: %s\n", msg); - fprintf(stderr, " current device: %d, in function %s at %s:%d\n", id, func, file, line); - fprintf(stderr, " %s\n", stmt); - // abort with GGML_ASSERT to get a stack trace - GGML_ASSERT(!"CUDA error"); -} - -#define CUDA_CHECK_GEN(err, success, error_fn) \ - do { \ - auto err_ = (err); \ - if (err_ != (success)) { \ - ggml_cuda_error(#err, __func__, __FILE__, __LINE__, error_fn(err_)); \ - } \ - } while (0) - -#define CUDA_CHECK(err) CUDA_CHECK_GEN(err, cudaSuccess, cudaGetErrorString) - -#if CUDART_VERSION >= 12000 - static const char * cublas_get_error_str(const cublasStatus_t err) { - return cublasGetStatusString(err); - } -#else - static const char * cublas_get_error_str(const cublasStatus_t err) { - switch (err) { - case CUBLAS_STATUS_SUCCESS: return "CUBLAS_STATUS_SUCCESS"; - case CUBLAS_STATUS_NOT_INITIALIZED: return "CUBLAS_STATUS_NOT_INITIALIZED"; - case CUBLAS_STATUS_ALLOC_FAILED: return "CUBLAS_STATUS_ALLOC_FAILED"; - case CUBLAS_STATUS_INVALID_VALUE: return "CUBLAS_STATUS_INVALID_VALUE"; - case CUBLAS_STATUS_ARCH_MISMATCH: return "CUBLAS_STATUS_ARCH_MISMATCH"; - case CUBLAS_STATUS_MAPPING_ERROR: return "CUBLAS_STATUS_MAPPING_ERROR"; - case CUBLAS_STATUS_EXECUTION_FAILED: return "CUBLAS_STATUS_EXECUTION_FAILED"; - case CUBLAS_STATUS_INTERNAL_ERROR: return "CUBLAS_STATUS_INTERNAL_ERROR"; - case CUBLAS_STATUS_NOT_SUPPORTED: return "CUBLAS_STATUS_NOT_SUPPORTED"; - default: return "unknown error"; - } - } -#endif // CUDART_VERSION >= 12000 - -#define CUBLAS_CHECK(err) CUDA_CHECK_GEN(err, CUBLAS_STATUS_SUCCESS, cublas_get_error_str) - -#if !defined(GGML_USE_HIPBLAS) -static const char * cu_get_error_str(CUresult err) { - const char * err_str; - cuGetErrorString(err, &err_str); - return err_str; -} -#define CU_CHECK(err) CUDA_CHECK_GEN(err, CUDA_SUCCESS, cu_get_error_str) -#endif - -#if CUDART_VERSION >= 11100 -#define GGML_CUDA_ASSUME(x) __builtin_assume(x) -#else -#define GGML_CUDA_ASSUME(x) -#endif // CUDART_VERSION >= 11100 #ifdef GGML_CUDA_F16 typedef half dfloat; // dequantize float @@ -363,12 +1508,17 @@ typedef to_t_cuda_t to_fp16_cuda_t; typedef void (*dequantize_kernel_t)(const void * vx, const int ib, const int iqs, dfloat2 & v); typedef void (*dot_kernel_k_t)(const void * __restrict__ vx, const int ib, const int iqs, const float * __restrict__ y, float & v); typedef void (*cpy_kernel_t)(const char * cx, char * cdst); -typedef void (*ggml_cuda_func_t)(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst); + +typedef void (*ggml_cuda_func_t)(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst); + typedef void (*ggml_cuda_op_mul_mat_t)( + ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, const char * src0_dd_i, const float * src1_ddf_i, const char * src1_ddq_i, float * dst_dd_i, const int64_t row_low, const int64_t row_high, const int64_t src1_ncols, const int64_t src1_padded_row_size, cudaStream_t stream); + typedef void (*ggml_cuda_op_flatten_t)( + ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, const float * src0_dd, const float * src1_dd, float * dst_dd, cudaStream_t main_stream); @@ -382,7 +1532,6 @@ typedef float (*vec_dot_q_mul_mat_cuda_t)( const int * __restrict__ y_qs, const half2 * __restrict__ y_ms, const int & i, const int & j, const int & k); #define WARP_SIZE 32 -#define MATRIX_ROW_PADDING 512 // last row of quant. matrices is a multiple of this to avoid out-of-bounds memory accesses #define CUDA_GELU_BLOCK_SIZE 256 #define CUDA_SILU_BLOCK_SIZE 256 @@ -432,42 +1581,6 @@ static_assert(K_QUANTS_PER_ITERATION == 1 || K_QUANTS_PER_ITERATION == 2, "K_QUA #define MUL_MAT_SRC1_COL_STRIDE 128 -#define MAX_STREAMS 8 -static cudaStream_t g_cudaStreams[GGML_CUDA_MAX_DEVICES][MAX_STREAMS] = { { nullptr } }; - -struct ggml_tensor_extra_gpu { - void * data_device[GGML_CUDA_MAX_DEVICES]; // 1 pointer for each device for split tensors - cudaEvent_t events[GGML_CUDA_MAX_DEVICES][MAX_STREAMS]; // events for synchronizing multiple GPUs -}; - -// this is faster on Windows -// probably because the Windows CUDA libraries forget to make this check before invoking the drivers -static void ggml_cuda_set_device(const int device) { - int current_device; - CUDA_CHECK(cudaGetDevice(¤t_device)); - - if (device == current_device) { - return; - } - - CUDA_CHECK(cudaSetDevice(device)); -} - -static int g_device_count = -1; -static int g_main_device = 0; -static std::array g_default_tensor_split = {}; - -struct cuda_device_capabilities { - int cc; // compute capability - size_t smpb; // max. shared memory per block - bool vmm; // virtual memory support - size_t vmm_granularity; // granularity of virtual memory -}; - -static cuda_device_capabilities g_device_caps[GGML_CUDA_MAX_DEVICES] = { {0, 0, false, 0} }; - -static cublasHandle_t g_cublas_handles[GGML_CUDA_MAX_DEVICES] = {nullptr}; - [[noreturn]] static __device__ void no_device_code( const char * file_name, const int line, const char * function_name, const int arch, const char * arch_list) { @@ -475,20 +1588,21 @@ static __device__ void no_device_code( #if defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__) printf("%s:%d: ERROR: HIP kernel %s has no device code compatible with HIP arch %d.\n", file_name, line, function_name, arch); - (void) arch_list; + GGML_UNUSED(arch_list); #else printf("%s:%d: ERROR: CUDA kernel %s has no device code compatible with CUDA arch %d. ggml-cuda.cu was compiled for: %s\n", file_name, line, function_name, arch, arch_list); #endif // defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__) __trap(); - (void) no_device_code; // suppress unused function warning + GGML_UNUSED(no_device_code); // suppress unused function warning } #ifdef __CUDA_ARCH__ #define NO_DEVICE_CODE no_device_code(__FILE__, __LINE__, __FUNCTION__, __CUDA_ARCH__, STRINGIZE(__CUDA_ARCH_LIST__)) #else -#define NO_DEVICE_CODE GGML_ASSERT(false && "NO_DEVICE_CODE not valid in host code.") +//#define NO_DEVICE_CODE GGML_ASSERT(false && "NO_DEVICE_CODE not valid in host code.") +#define NO_DEVICE_CODE #endif // __CUDA_ARCH__ static __device__ __forceinline__ float warp_reduce_sum(float x) { @@ -517,7 +1631,7 @@ static __device__ __forceinline__ half2 warp_reduce_sum(half2 a) { } return a; #else - (void) a; + GGML_UNUSED(a); NO_DEVICE_CODE; #endif // !(defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)) && __CUDA_ARCH__ >= CC_PASCAL } @@ -539,7 +1653,7 @@ static __device__ __forceinline__ float warp_reduce_max(float x) { // } // return x; //#else -// (void) x; +// GGML_UNUSED(x); // NO_DEVICE_CODE; //#endif // !(defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)) && __CUDA_ARCH__ >= CC_PASCAL && CUDART_VERSION >= CUDART_HMAX //} @@ -2293,7 +3407,9 @@ static __global__ void dequantize_block_q8_0_f16(const void * __restrict__ vx, h y2[iy/2 + threadIdx.x] = __hmul2(make_half2(qs.x, qs.y), __half2half2(d)); } #else - (void) vx; (void) y; (void) k; + GGML_UNUSED(vx); + GGML_UNUSED(y); + GGML_UNUSED(k); NO_DEVICE_CODE; #endif // __CUDA_ARCH__ >= CC_PASCAL } @@ -2853,7 +3969,8 @@ static __device__ __forceinline__ float vec_dot_q4_0_q8_1( } template static __device__ __forceinline__ void allocate_tiles_q4_0(int ** x_ql, half2 ** x_dm, int ** x_qh, int ** x_sc) { - (void)x_qh; (void)x_sc; + GGML_UNUSED(x_qh); + GGML_UNUSED(x_sc); __shared__ int tile_x_qs[mmq_y * (WARP_SIZE) + mmq_y]; __shared__ float tile_x_d[mmq_y * (WARP_SIZE/QI4_0) + mmq_y/QI4_0]; @@ -2865,7 +3982,7 @@ template static __device__ __forceinline__ void allocate_tiles_q4_0( template static __device__ __forceinline__ void load_tiles_q4_0( const void * __restrict__ vx, int * __restrict__ x_ql, half2 * __restrict__ x_dm, int * __restrict__ x_qh, int * __restrict__ x_sc, const int & i_offset, const int & i_max, const int & k, const int & blocks_per_row) { - (void)x_qh; (void)x_sc; + GGML_UNUSED(x_qh); GGML_UNUSED(x_sc); GGML_CUDA_ASSUME(i_offset >= 0); GGML_CUDA_ASSUME(i_offset < nwarps); GGML_CUDA_ASSUME(k >= 0); @@ -2912,7 +4029,7 @@ template static __device__ __forceinlin static __device__ __forceinline__ float vec_dot_q4_0_q8_1_mul_mat( const int * __restrict__ x_ql, const half2 * __restrict__ x_dm, const int * __restrict__ x_qh, const int * __restrict__ x_sc, const int * __restrict__ y_qs, const half2 * __restrict__ y_ds, const int & i, const int & j, const int & k) { - (void)x_qh; (void)x_sc; + GGML_UNUSED(x_qh); GGML_UNUSED(x_sc); const int kyqs = k % (QI8_1/2) + QI8_1 * (k / (QI8_1/2)); const float * x_dmf = (const float *) x_dm; @@ -2949,7 +4066,7 @@ static __device__ __forceinline__ float vec_dot_q4_1_q8_1( } template static __device__ __forceinline__ void allocate_tiles_q4_1(int ** x_ql, half2 ** x_dm, int ** x_qh, int ** x_sc) { - (void)x_qh; (void)x_sc; + GGML_UNUSED(x_qh); GGML_UNUSED(x_sc); __shared__ int tile_x_qs[mmq_y * (WARP_SIZE) + + mmq_y]; __shared__ half2 tile_x_dm[mmq_y * (WARP_SIZE/QI4_1) + mmq_y/QI4_1]; @@ -2961,7 +4078,7 @@ template static __device__ __forceinline__ void allocate_tiles_q4_1( template static __device__ __forceinline__ void load_tiles_q4_1( const void * __restrict__ vx, int * __restrict__ x_ql, half2 * __restrict__ x_dm, int * __restrict__ x_qh, int * __restrict__ x_sc, const int & i_offset, const int & i_max, const int & k, const int & blocks_per_row) { - (void)x_qh; (void)x_sc; + GGML_UNUSED(x_qh); GGML_UNUSED(x_sc); GGML_CUDA_ASSUME(i_offset >= 0); GGML_CUDA_ASSUME(i_offset < nwarps); @@ -3006,7 +4123,7 @@ template static __device__ __forceinlin static __device__ __forceinline__ float vec_dot_q4_1_q8_1_mul_mat( const int * __restrict__ x_ql, const half2 * __restrict__ x_dm, const int * __restrict__ x_qh, const int * __restrict__ x_sc, const int * __restrict__ y_qs, const half2 * __restrict__ y_ds, const int & i, const int & j, const int & k) { - (void)x_qh; (void)x_sc; + GGML_UNUSED(x_qh); GGML_UNUSED(x_sc); const int kyqs = k % (QI8_1/2) + QI8_1 * (k / (QI8_1/2)); @@ -3044,7 +4161,7 @@ static __device__ __forceinline__ float vec_dot_q5_0_q8_1( } template static __device__ __forceinline__ void allocate_tiles_q5_0(int ** x_ql, half2 ** x_dm, int ** x_qh, int ** x_sc) { - (void)x_qh; (void)x_sc; + GGML_UNUSED(x_qh); GGML_UNUSED(x_sc); __shared__ int tile_x_ql[mmq_y * (2*WARP_SIZE) + mmq_y]; __shared__ float tile_x_d[mmq_y * (WARP_SIZE/QI5_0) + mmq_y/QI5_0]; @@ -3056,7 +4173,7 @@ template static __device__ __forceinline__ void allocate_tiles_q5_0( template static __device__ __forceinline__ void load_tiles_q5_0( const void * __restrict__ vx, int * __restrict__ x_ql, half2 * __restrict__ x_dm, int * __restrict__ x_qh, int * __restrict__ x_sc, const int & i_offset, const int & i_max, const int & k, const int & blocks_per_row) { - (void)x_qh; (void)x_sc; + GGML_UNUSED(x_qh); GGML_UNUSED(x_sc); GGML_CUDA_ASSUME(i_offset >= 0); GGML_CUDA_ASSUME(i_offset < nwarps); @@ -3121,7 +4238,7 @@ template static __device__ __forceinlin static __device__ __forceinline__ float vec_dot_q5_0_q8_1_mul_mat( const int * __restrict__ x_ql, const half2 * __restrict__ x_dm, const int * __restrict__ x_qh, const int * __restrict__ x_sc, const int * __restrict__ y_qs, const half2 * __restrict__ y_ds, const int & i, const int & j, const int & k) { - (void)x_qh; (void)x_sc; + GGML_UNUSED(x_qh); GGML_UNUSED(x_sc); const int kyqs = k % (QI8_1/2) + QI8_1 * (k / (QI8_1/2)); const int index_bx = i * (WARP_SIZE/QI5_0) + i/QI5_0 + k/QI5_0; @@ -3161,7 +4278,7 @@ static __device__ __forceinline__ float vec_dot_q5_1_q8_1( } template static __device__ __forceinline__ void allocate_tiles_q5_1(int ** x_ql, half2 ** x_dm, int ** x_qh, int ** x_sc) { - (void)x_qh; (void)x_sc; + GGML_UNUSED(x_qh); GGML_UNUSED(x_sc); __shared__ int tile_x_ql[mmq_y * (2*WARP_SIZE) + mmq_y]; __shared__ half2 tile_x_dm[mmq_y * (WARP_SIZE/QI5_1) + mmq_y/QI5_1]; @@ -3173,7 +4290,7 @@ template static __device__ __forceinline__ void allocate_tiles_q5_1( template static __device__ __forceinline__ void load_tiles_q5_1( const void * __restrict__ vx, int * __restrict__ x_ql, half2 * __restrict__ x_dm, int * __restrict__ x_qh, int * __restrict__ x_sc, const int & i_offset, const int & i_max, const int & k, const int & blocks_per_row) { - (void)x_qh; (void)x_sc; + GGML_UNUSED(x_qh); GGML_UNUSED(x_sc); GGML_CUDA_ASSUME(i_offset >= 0); GGML_CUDA_ASSUME(i_offset < nwarps); @@ -3235,7 +4352,7 @@ template static __device__ __forceinlin static __device__ __forceinline__ float vec_dot_q5_1_q8_1_mul_mat( const int * __restrict__ x_ql, const half2 * __restrict__ x_dm, const int * __restrict__ x_qh, const int * __restrict__ x_sc, const int * __restrict__ y_qs, const half2 * __restrict__ y_ds, const int & i, const int & j, const int & k) { - (void)x_qh; (void)x_sc; + GGML_UNUSED(x_qh); GGML_UNUSED(x_sc); const int kyqs = k % (QI8_1/2) + QI8_1 * (k / (QI8_1/2)); const int index_bx = i * (WARP_SIZE/QI5_1) + + i/QI5_1 + k/QI5_1; @@ -3270,7 +4387,7 @@ static __device__ __forceinline__ float vec_dot_q8_0_q8_1( } template static __device__ __forceinline__ void allocate_tiles_q8_0(int ** x_ql, half2 ** x_dm, int ** x_qh, int ** x_sc) { - (void)x_qh; (void)x_sc; + GGML_UNUSED(x_qh); GGML_UNUSED(x_sc); __shared__ int tile_x_qs[mmq_y * (WARP_SIZE) + mmq_y]; __shared__ float tile_x_d[mmq_y * (WARP_SIZE/QI8_0) + mmq_y/QI8_0]; @@ -3282,7 +4399,7 @@ template static __device__ __forceinline__ void allocate_tiles_q8_0( template static __device__ __forceinline__ void load_tiles_q8_0( const void * __restrict__ vx, int * __restrict__ x_ql, half2 * __restrict__ x_dm, int * __restrict__ x_qh, int * __restrict__ x_sc, const int & i_offset, const int & i_max, const int & k, const int & blocks_per_row) { - (void)x_qh; (void)x_sc; + GGML_UNUSED(x_qh); GGML_UNUSED(x_sc); GGML_CUDA_ASSUME(i_offset >= 0); GGML_CUDA_ASSUME(i_offset < nwarps); @@ -3328,7 +4445,7 @@ template static __device__ __forceinlin static __device__ __forceinline__ float vec_dot_q8_0_q8_1_mul_mat( const int * __restrict__ x_ql, const half2 * __restrict__ x_dm, const int * __restrict__ x_qh, const int * __restrict__ x_sc, const int * __restrict__ y_qs, const half2 * __restrict__ y_ds, const int & i, const int & j, const int & k) { - (void)x_qh; (void)x_sc; + GGML_UNUSED(x_qh); GGML_UNUSED(x_sc); const float * x_dmf = (const float *) x_dm; const float * y_df = (const float *) y_ds; @@ -3362,7 +4479,7 @@ static __device__ __forceinline__ float vec_dot_q2_K_q8_1( } template static __device__ __forceinline__ void allocate_tiles_q2_K(int ** x_ql, half2 ** x_dm, int ** x_qh, int ** x_sc) { - (void)x_qh; + GGML_UNUSED(x_qh); __shared__ int tile_x_ql[mmq_y * (WARP_SIZE) + mmq_y]; __shared__ half2 tile_x_dm[mmq_y * (WARP_SIZE/QI2_K) + mmq_y/QI2_K]; @@ -3376,7 +4493,7 @@ template static __device__ __forceinline__ void allocate_tiles_q2_K( template static __device__ __forceinline__ void load_tiles_q2_K( const void * __restrict__ vx, int * __restrict__ x_ql, half2 * __restrict__ x_dm, int * __restrict__ x_qh, int * __restrict__ x_sc, const int & i_offset, const int & i_max, const int & k, const int & blocks_per_row) { - (void)x_qh; + GGML_UNUSED(x_qh); GGML_CUDA_ASSUME(i_offset >= 0); GGML_CUDA_ASSUME(i_offset < nwarps); @@ -3434,7 +4551,7 @@ template static __device__ __forceinlin static __device__ __forceinline__ float vec_dot_q2_K_q8_1_mul_mat( const int * __restrict__ x_ql, const half2 * __restrict__ x_dm, const int * __restrict__ x_qh, const int * __restrict__ x_sc, const int * __restrict__ y_qs, const half2 * __restrict__ y_ds, const int & i, const int & j, const int & k) { - (void)x_qh; + GGML_UNUSED(x_qh); const int kbx = k / QI2_K; const int ky = (k % QI2_K) * QR2_K; @@ -3703,7 +4820,7 @@ static __device__ __forceinline__ float vec_dot_q4_K_q8_1( } template static __device__ __forceinline__ void allocate_tiles_q4_K(int ** x_ql, half2 ** x_dm, int ** x_qh, int ** x_sc) { - (void)x_qh; + GGML_UNUSED(x_qh); __shared__ int tile_x_ql[mmq_y * (WARP_SIZE) + mmq_y]; __shared__ half2 tile_x_dm[mmq_y * (WARP_SIZE/QI4_K) + mmq_y/QI4_K]; @@ -3717,7 +4834,7 @@ template static __device__ __forceinline__ void allocate_tiles_q4_K( template static __device__ __forceinline__ void load_tiles_q4_K( const void * __restrict__ vx, int * __restrict__ x_ql, half2 * __restrict__ x_dm, int * __restrict__ x_qh, int * __restrict__ x_sc, const int & i_offset, const int & i_max, const int & k, const int & blocks_per_row) { - (void)x_qh; + GGML_UNUSED(x_qh); GGML_CUDA_ASSUME(i_offset >= 0); GGML_CUDA_ASSUME(i_offset < nwarps); @@ -3787,7 +4904,7 @@ template static __device__ __forceinlin static __device__ __forceinline__ float vec_dot_q4_K_q8_1_mul_mat( const int * __restrict__ x_ql, const half2 * __restrict__ x_dm, const int * __restrict__ x_qh, const int * __restrict__ x_sc, const int * __restrict__ y_qs, const half2 * __restrict__ y_ds, const int & i, const int & j, const int & k) { - (void)x_qh; + GGML_UNUSED(x_qh); const uint8_t * sc = ((const uint8_t *) &x_sc[i * (WARP_SIZE/8) + i/8 + k/16]) + 2*((k % 16) / 8); @@ -3886,7 +5003,7 @@ static __device__ __forceinline__ float vec_dot_q5_K_q8_1( } template static __device__ __forceinline__ void allocate_tiles_q5_K(int ** x_ql, half2 ** x_dm, int ** x_qh, int ** x_sc) { - (void)x_qh; + GGML_UNUSED(x_qh); __shared__ int tile_x_ql[mmq_y * (2*WARP_SIZE) + mmq_y]; __shared__ half2 tile_x_dm[mmq_y * (WARP_SIZE/QI5_K) + mmq_y/QI5_K]; @@ -3900,7 +5017,7 @@ template static __device__ __forceinline__ void allocate_tiles_q5_K( template static __device__ __forceinline__ void load_tiles_q5_K( const void * __restrict__ vx, int * __restrict__ x_ql, half2 * __restrict__ x_dm, int * __restrict__ x_qh, int * __restrict__ x_sc, const int & i_offset, const int & i_max, const int & k, const int & blocks_per_row) { - (void)x_qh; + GGML_UNUSED(x_qh); GGML_CUDA_ASSUME(i_offset >= 0); GGML_CUDA_ASSUME(i_offset < nwarps); @@ -3981,7 +5098,7 @@ template static __device__ __forceinlin static __device__ __forceinline__ float vec_dot_q5_K_q8_1_mul_mat( const int * __restrict__ x_ql, const half2 * __restrict__ x_dm, const int * __restrict__ x_qh, const int * __restrict__ x_sc, const int * __restrict__ y_qs, const half2 * __restrict__ y_ds, const int & i, const int & j, const int & k) { - (void)x_qh; + GGML_UNUSED(x_qh); const uint8_t * sc = ((const uint8_t *) &x_sc[i * (WARP_SIZE/8) + i/8 + k/16]) + 2 * ((k % 16) / 8); @@ -4018,7 +5135,7 @@ static __device__ __forceinline__ float vec_dot_q6_K_q8_1( } template static __device__ __forceinline__ void allocate_tiles_q6_K(int ** x_ql, half2 ** x_dm, int ** x_qh, int ** x_sc) { - (void)x_qh; + GGML_UNUSED(x_qh); __shared__ int tile_x_ql[mmq_y * (2*WARP_SIZE) + mmq_y]; __shared__ half2 tile_x_dm[mmq_y * (WARP_SIZE/QI6_K) + mmq_y/QI6_K]; @@ -4032,7 +5149,7 @@ template static __device__ __forceinline__ void allocate_tiles_q6_K( template static __device__ __forceinline__ void load_tiles_q6_K( const void * __restrict__ vx, int * __restrict__ x_ql, half2 * __restrict__ x_dm, int * __restrict__ x_qh, int * __restrict__ x_sc, const int & i_offset, const int & i_max, const int & k, const int & blocks_per_row) { - (void)x_qh; + GGML_UNUSED(x_qh); GGML_CUDA_ASSUME(i_offset >= 0); GGML_CUDA_ASSUME(i_offset < nwarps); @@ -4104,7 +5221,7 @@ template static __device__ __forceinlin static __device__ __forceinline__ float vec_dot_q6_K_q8_1_mul_mat( const int * __restrict__ x_ql, const half2 * __restrict__ x_dm, const int * __restrict__ x_qh, const int * __restrict__ x_sc, const int * __restrict__ y_qs, const half2 * __restrict__ y_ds, const int & i, const int & j, const int & k) { - (void)x_qh; + GGML_UNUSED(x_qh); const float * x_dmf = (const float *) x_dm; const float * y_df = (const float *) y_ds; @@ -4199,12 +5316,12 @@ static __device__ __forceinline__ float vec_dot_iq2_xs_q8_1( const float d = (float)bq2->d * __low2float(bq8_1[ib32].ds) * 0.25f; return d * ((0.5f + ls1) * sumi1 + (0.5f + ls2) * sumi2); #else - (void) ksigns64; + GGML_UNUSED(ksigns64); assert(false); return 0.f; #endif #else - (void) ksigns64; + GGML_UNUSED(ksigns64); assert(false); return 0.f; #endif @@ -4247,12 +5364,12 @@ static __device__ __forceinline__ float vec_dot_iq2_s_q8_1( const float d = (float)bq2->d * __low2float(bq8_1[ib32].ds) * 0.25f; return d * ((0.5f + ls1) * sumi1 + (0.5f + ls2) * sumi2); #else - (void) ksigns64; + GGML_UNUSED(ksigns64); assert(false); return 0.f; #endif #else - (void) ksigns64; + GGML_UNUSED(ksigns64); assert(false); return 0.f; #endif @@ -4654,7 +5771,7 @@ template static __global__ void load_tiles_q4_0, VDR_Q4_0_Q8_1_MMQ, vec_dot_q4_0_q8_1_mul_mat> (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst); #else - (void) vec_dot_q4_0_q8_1_mul_mat; + GGML_UNUSED(vec_dot_q4_0_q8_1_mul_mat); NO_DEVICE_CODE; #endif // __CUDA_ARCH__ >= CC_VOLTA } @@ -4723,7 +5840,7 @@ template static __global__ void load_tiles_q4_1, VDR_Q4_1_Q8_1_MMQ, vec_dot_q4_1_q8_1_mul_mat> (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst); #else - (void) vec_dot_q4_1_q8_1_mul_mat; + GGML_UNUSED(vec_dot_q4_1_q8_1_mul_mat); NO_DEVICE_CODE; #endif // __CUDA_ARCH__ >= CC_VOLTA } @@ -4790,7 +5907,7 @@ template static __global__ void load_tiles_q5_0, VDR_Q5_0_Q8_1_MMQ, vec_dot_q5_0_q8_1_mul_mat> (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst); #else - (void) vec_dot_q5_0_q8_1_mul_mat; + GGML_UNUSED(vec_dot_q5_0_q8_1_mul_mat); NO_DEVICE_CODE; #endif // __CUDA_ARCH__ >= CC_VOLTA } @@ -4857,7 +5974,7 @@ mul_mat_q5_1( load_tiles_q5_1, VDR_Q5_1_Q8_1_MMQ, vec_dot_q5_1_q8_1_mul_mat> (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst); #else - (void) vec_dot_q5_1_q8_1_mul_mat; + GGML_UNUSED(vec_dot_q5_1_q8_1_mul_mat); NO_DEVICE_CODE; #endif // __CUDA_ARCH__ >= CC_VOLTA } @@ -4924,7 +6041,7 @@ template static __global__ void load_tiles_q8_0, VDR_Q8_0_Q8_1_MMQ, vec_dot_q8_0_q8_1_mul_mat> (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst); #else - (void) vec_dot_q8_0_q8_1_mul_mat; + GGML_UNUSED(vec_dot_q8_0_q8_1_mul_mat); NO_DEVICE_CODE; #endif // __CUDA_ARCH__ >= CC_VOLTA } @@ -4991,7 +6108,7 @@ mul_mat_q2_K( load_tiles_q2_K, VDR_Q2_K_Q8_1_MMQ, vec_dot_q2_K_q8_1_mul_mat> (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst); #else - (void) vec_dot_q2_K_q8_1_mul_mat; + GGML_UNUSED(vec_dot_q2_K_q8_1_mul_mat); NO_DEVICE_CODE; #endif // __CUDA_ARCH__ >= CC_VOLTA } @@ -5060,7 +6177,7 @@ template static __global__ void load_tiles_q3_K, VDR_Q3_K_Q8_1_MMQ, vec_dot_q3_K_q8_1_mul_mat> (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst); #else - (void) vec_dot_q3_K_q8_1_mul_mat; + GGML_UNUSED(vec_dot_q3_K_q8_1_mul_mat); NO_DEVICE_CODE; #endif // __CUDA_ARCH__ >= CC_VOLTA } @@ -5129,7 +6246,7 @@ template static __global__ void load_tiles_q4_K, VDR_Q4_K_Q8_1_MMQ, vec_dot_q4_K_q8_1_mul_mat> (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst); #else - (void) vec_dot_q4_K_q8_1_mul_mat; + GGML_UNUSED(vec_dot_q4_K_q8_1_mul_mat); NO_DEVICE_CODE; #endif // __CUDA_ARCH__ >= CC_VOLTA } @@ -5196,7 +6313,7 @@ mul_mat_q5_K( load_tiles_q5_K, VDR_Q5_K_Q8_1_MMQ, vec_dot_q5_K_q8_1_mul_mat> (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst); #else - (void) vec_dot_q5_K_q8_1_mul_mat; + GGML_UNUSED(vec_dot_q5_K_q8_1_mul_mat); NO_DEVICE_CODE; #endif // __CUDA_ARCH__ >= CC_VOLTA } @@ -5265,7 +6382,7 @@ template static __global__ void load_tiles_q6_K, VDR_Q6_K_Q8_1_MMQ, vec_dot_q6_K_q8_1_mul_mat> (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst); #else - (void) vec_dot_q6_K_q8_1_mul_mat; + GGML_UNUSED(vec_dot_q6_K_q8_1_mul_mat); NO_DEVICE_CODE; #endif // __CUDA_ARCH__ >= CC_VOLTA } @@ -6082,46 +7199,48 @@ static __global__ void pool2d_nchw_kernel( const int kh, const int kw, const int sh, const int sw, const int ph, const int pw, const int parallel_elements, const Ti* src, To* dst, const enum ggml_op_pool op) { - int idx = threadIdx.x + blockIdx.x * blockDim.x; - if (idx >= parallel_elements) { - return; - } + int idx = threadIdx.x + blockIdx.x * blockDim.x; + if (idx >= parallel_elements) { + return; + } - const int I_HW = ih * iw; - const int O_HW = oh * ow; - const int nc = idx / O_HW; - const int cur_oh = idx % O_HW / ow; - const int cur_ow = idx % O_HW % ow; - const Ti* i_ptr = src + nc * I_HW; - To* o_ptr = dst + nc * O_HW; - const int start_h = cur_oh * sh - ph; - const int bh = max(0, start_h); - const int eh = min(ih, start_h + kh); - const int start_w = cur_ow * sw - pw; - const int bw = max(0, start_w); - const int ew = min(iw, start_w + kw); - const To scale = 1. / (kh * kw); - To res = 0; + const int I_HW = ih * iw; + const int O_HW = oh * ow; + const int nc = idx / O_HW; + const int cur_oh = idx % O_HW / ow; + const int cur_ow = idx % O_HW % ow; + const Ti* i_ptr = src + nc * I_HW; + To* o_ptr = dst + nc * O_HW; + const int start_h = cur_oh * sh - ph; + const int bh = max(0, start_h); + const int eh = min(ih, start_h + kh); + const int start_w = cur_ow * sw - pw; + const int bw = max(0, start_w); + const int ew = min(iw, start_w + kw); + const To scale = 1. / (kh * kw); + To res = 0; - switch (op) { - case GGML_OP_POOL_AVG: res = 0; break; - case GGML_OP_POOL_MAX: res = -FLT_MAX; break; - } + switch (op) { + case GGML_OP_POOL_AVG: res = 0; break; + case GGML_OP_POOL_MAX: res = -FLT_MAX; break; + default: assert(false); + } - for (int i = bh; i < eh; i += 1) { - for (int j = bw; j < ew; j += 1) { - #if __CUDA_ARCH__ >= 350 - Ti cur = __ldg(i_ptr + i * iw + j); - #else - Ti cur = i_ptr[i * iw + j]; - #endif - switch (op) { - case GGML_OP_POOL_AVG: res += cur * scale; break; - case GGML_OP_POOL_MAX: res = max(res, (To)cur); break; - } + for (int i = bh; i < eh; i += 1) { + for (int j = bw; j < ew; j += 1) { +#if __CUDA_ARCH__ >= 350 + Ti cur = __ldg(i_ptr + i * iw + j); +#else + Ti cur = i_ptr[i * iw + j]; +#endif + switch (op) { + case GGML_OP_POOL_AVG: res += cur * scale; break; + case GGML_OP_POOL_MAX: res = max(res, (To)cur); break; + default: assert(false); } } - o_ptr[cur_oh * ow + cur_ow] = res; + } + o_ptr[cur_oh * ow + cur_ow] = res; } template @@ -6155,7 +7274,7 @@ static void get_rows_cuda(const ggml_tensor * src0, const ggml_tensor * src1, gg /* nb00,*/ nb01, nb02, nb03, s10, s11, s12/*, s13*/); - (void) dst; + GGML_UNUSED(dst); } template @@ -6187,7 +7306,7 @@ static void get_rows_cuda_float(const ggml_tensor * src0, const ggml_tensor * sr /* nb00,*/ nb01, nb02, nb03, s10, s11, s12/*, s13*/); - (void) dst; + GGML_UNUSED(dst); } template @@ -6580,7 +7699,7 @@ static to_fp16_cuda_t ggml_get_to_fp16_cuda(ggml_type type) { return dequantize_block_cuda; case GGML_TYPE_Q8_0: CUDA_CHECK(cudaGetDevice(&id)); - if (g_device_caps[id].cc >= CC_PASCAL) { + if (get_cuda_global_info().devices[id].cc >= CC_PASCAL) { return dequantize_block_q8_0_f16_cuda; } return dequantize_block_cuda; @@ -6773,7 +7892,7 @@ static void mul_mat_vec_q_cuda( int64_t nwarps = 1; int64_t rows_per_cuda_block = 1; - if (g_device_caps[id].cc < CC_RDNA2) { // NVIDIA and AMD older than RDNA2 + if (get_cuda_global_info().devices[id].cc < CC_RDNA2) { // NVIDIA and AMD older than RDNA2 switch(ncols_y) { case 1: nwarps = 4; @@ -6846,7 +7965,7 @@ static void ggml_mul_mat_q4_0_q8_1_cuda( int id; CUDA_CHECK(cudaGetDevice(&id)); - const int compute_capability = g_device_caps[id].cc; + const int compute_capability = get_cuda_global_info().devices[id].cc; int mmq_x, mmq_y, nwarps; if (compute_capability >= CC_RDNA2) { @@ -6891,7 +8010,7 @@ static void ggml_mul_mat_q4_1_q8_1_cuda( int id; CUDA_CHECK(cudaGetDevice(&id)); - const int compute_capability = g_device_caps[id].cc; + const int compute_capability = get_cuda_global_info().devices[id].cc; int mmq_x, mmq_y, nwarps; if (compute_capability >= CC_RDNA2) { @@ -6936,7 +8055,7 @@ static void ggml_mul_mat_q5_0_q8_1_cuda( int id; CUDA_CHECK(cudaGetDevice(&id)); - const int compute_capability = g_device_caps[id].cc; + const int compute_capability = get_cuda_global_info().devices[id].cc; int mmq_x, mmq_y, nwarps; if (compute_capability >= CC_RDNA2) { @@ -6981,7 +8100,7 @@ static void ggml_mul_mat_q5_1_q8_1_cuda( int id; CUDA_CHECK(cudaGetDevice(&id)); - const int compute_capability = g_device_caps[id].cc; + const int compute_capability = get_cuda_global_info().devices[id].cc; int mmq_x, mmq_y, nwarps; if (compute_capability >= CC_RDNA2) { @@ -7026,7 +8145,7 @@ static void ggml_mul_mat_q8_0_q8_1_cuda( int id; CUDA_CHECK(cudaGetDevice(&id)); - const int compute_capability = g_device_caps[id].cc; + const int compute_capability = get_cuda_global_info().devices[id].cc; int mmq_x, mmq_y, nwarps; if (compute_capability >= CC_RDNA2) { @@ -7071,7 +8190,7 @@ static void ggml_mul_mat_q2_K_q8_1_cuda( int id; CUDA_CHECK(cudaGetDevice(&id)); - const int compute_capability = g_device_caps[id].cc; + const int compute_capability = get_cuda_global_info().devices[id].cc; int mmq_x, mmq_y, nwarps; if (compute_capability >= CC_RDNA2) { @@ -7118,7 +8237,7 @@ static void ggml_mul_mat_q3_K_q8_1_cuda( int id; CUDA_CHECK(cudaGetDevice(&id)); - const int compute_capability = g_device_caps[id].cc; + const int compute_capability = get_cuda_global_info().devices[id].cc; int mmq_x, mmq_y, nwarps; if (compute_capability >= CC_RDNA2) { @@ -7164,7 +8283,7 @@ static void ggml_mul_mat_q4_K_q8_1_cuda( int id; CUDA_CHECK(cudaGetDevice(&id)); - const int compute_capability = g_device_caps[id].cc; + const int compute_capability = get_cuda_global_info().devices[id].cc; int mmq_x, mmq_y, nwarps; if (compute_capability >= CC_RDNA2) { @@ -7209,7 +8328,7 @@ static void ggml_mul_mat_q5_K_q8_1_cuda( int id; CUDA_CHECK(cudaGetDevice(&id)); - const int compute_capability = g_device_caps[id].cc; + const int compute_capability = get_cuda_global_info().devices[id].cc; int mmq_x, mmq_y, nwarps; if (compute_capability >= CC_RDNA2) { @@ -7254,7 +8373,7 @@ static void ggml_mul_mat_q6_K_q8_1_cuda( int id; CUDA_CHECK(cudaGetDevice(&id)); - const int compute_capability = g_device_caps[id].cc; + const int compute_capability = get_cuda_global_info().devices[id].cc; int mmq_x, mmq_y, nwarps; if (compute_capability >= CC_RDNA2) { @@ -7506,7 +8625,7 @@ static void soft_max_f32_cuda(const float * x, const float * mask, const float * const float m0 = powf(2.0f, -(max_bias ) / n_head_log2); const float m1 = powf(2.0f, -(max_bias / 2.0f) / n_head_log2); - if (shmem < g_device_caps[g_main_device].smpb) { + if (shmem < get_cuda_global_info().devices[ggml_cuda_get_device()].smpb) { switch (ncols_x) { case 32: soft_max_f32<<>>(x, mask, pos, dst, ncols_x, nrows_y, scale, max_bias, m0, m1, n_head_log2); @@ -7543,7 +8662,7 @@ static void soft_max_f32_cuda(const float * x, const float * mask, const float * } template -static void im2col_cuda(const float* x, T* dst, +static void im2col_cuda(const float * x, T* dst, int64_t IW, int64_t IH, int64_t OW, int64_t OH, int64_t KW, int64_t KH, int64_t IC, int64_t batch, int64_t batch_offset, int64_t offset_delta, int s0,int s1,int p0,int p1,int d0,int d1, cudaStream_t stream) { @@ -7553,373 +8672,11 @@ static void im2col_cuda(const float* x, T* dst, im2col_kernel<<>>(x, dst, batch_offset, offset_delta, IC, IW, IH, OH, OW, KW, KH, parallel_elements, (IC * KH * KW), s0, s1, p0, p1, d0, d1); } -// buffer pool for cuda -#define MAX_CUDA_BUFFERS 256 - -struct scoped_spin_lock { - std::atomic_flag& lock; - scoped_spin_lock(std::atomic_flag& lock) : lock(lock) { - while (lock.test_and_set(std::memory_order_acquire)) { - ; // spin - } - } - ~scoped_spin_lock() { - lock.clear(std::memory_order_release); - } - scoped_spin_lock(const scoped_spin_lock&) = delete; - scoped_spin_lock& operator=(const scoped_spin_lock&) = delete; -}; - -static std::atomic_flag g_cuda_pool_lock = ATOMIC_FLAG_INIT; - -// #define DEBUG_CUDA_MALLOC -struct ggml_cuda_buffer { - void * ptr = nullptr; - size_t size = 0; -}; - -static ggml_cuda_buffer g_cuda_buffer_pool[GGML_CUDA_MAX_DEVICES][MAX_CUDA_BUFFERS]; -static size_t g_cuda_pool_size[GGML_CUDA_MAX_DEVICES] = {0}; - -static void * ggml_cuda_pool_malloc_leg(int device, size_t size, size_t * actual_size) { - scoped_spin_lock lock(g_cuda_pool_lock); -#ifdef DEBUG_CUDA_MALLOC - int nnz = 0; - size_t max_size = 0; -#endif - size_t best_diff = 1ull << 36; - int ibest = -1; - for (int i = 0; i < MAX_CUDA_BUFFERS; ++i) { - ggml_cuda_buffer& b = g_cuda_buffer_pool[device][i]; - if (b.ptr != nullptr) { -#ifdef DEBUG_CUDA_MALLOC - ++nnz; - if (b.size > max_size) max_size = b.size; -#endif - if (b.size >= size) { - size_t diff = b.size - size; - if (diff < best_diff) { - best_diff = diff; - ibest = i; - if (!best_diff) { - void * ptr = b.ptr; - *actual_size = b.size; - b.ptr = nullptr; - b.size = 0; - return ptr; - } - } - } - } - } - if (ibest >= 0) { - ggml_cuda_buffer& b = g_cuda_buffer_pool[device][ibest]; - void * ptr = b.ptr; - *actual_size = b.size; - b.ptr = nullptr; - b.size = 0; - return ptr; - } - void * ptr; - size_t look_ahead_size = (size_t) (1.05 * size); - look_ahead_size = 256 * ((look_ahead_size + 255)/256); - ggml_cuda_set_device(device); - CUDA_CHECK(cudaMalloc((void **) &ptr, look_ahead_size)); - *actual_size = look_ahead_size; - g_cuda_pool_size[device] += look_ahead_size; -#ifdef DEBUG_CUDA_MALLOC - fprintf(stderr, "%s[%d]: %d buffers, max_size = %u MB, pool_size = %u MB, requested %u MB\n", __func__, device, nnz, - (uint32_t)(max_size/1024/1024), (uint32_t)(g_cuda_pool_size[device]/1024/1024), (uint32_t)(size/1024/1024)); -#endif - return ptr; -} - -static void ggml_cuda_pool_free_leg(int device, void * ptr, size_t size) { - scoped_spin_lock lock(g_cuda_pool_lock); - - for (int i = 0; i < MAX_CUDA_BUFFERS; ++i) { - ggml_cuda_buffer& b = g_cuda_buffer_pool[device][i]; - if (b.ptr == nullptr) { - b.ptr = ptr; - b.size = size; - return; - } - } - fprintf(stderr, "WARNING: cuda buffer pool full, increase MAX_CUDA_BUFFERS\n"); - ggml_cuda_set_device(device); - CUDA_CHECK(cudaFree(ptr)); - g_cuda_pool_size[device] -= size; -} - -#if !defined(GGML_USE_HIPBLAS) -// pool with virtual memory -static CUdeviceptr g_cuda_pool_addr[GGML_CUDA_MAX_DEVICES] = {0}; -static size_t g_cuda_pool_used[GGML_CUDA_MAX_DEVICES] = {0}; -static const size_t CUDA_POOL_VMM_MAX_SIZE = 1ull << 35; // 32 GB - -static void * ggml_cuda_pool_malloc_vmm(int device, size_t size, size_t * actual_size) { - scoped_spin_lock lock(g_cuda_pool_lock); - - // round up the allocation size to the alignment to ensure that all allocations are aligned for all data types - const size_t alignment = 128; - size = alignment * ((size + alignment - 1) / alignment); - - size_t avail = g_cuda_pool_size[device] - g_cuda_pool_used[device]; - - if (size > avail) { - // round up to the next multiple of the granularity - size_t reserve_size = size - avail; - const size_t granularity = g_device_caps[device].vmm_granularity; - reserve_size = granularity * ((reserve_size + granularity - 1) / granularity); - - GGML_ASSERT(g_cuda_pool_size[device] + reserve_size <= CUDA_POOL_VMM_MAX_SIZE); - - // allocate more physical memory - CUmemAllocationProp prop = {}; - prop.type = CU_MEM_ALLOCATION_TYPE_PINNED; - prop.location.type = CU_MEM_LOCATION_TYPE_DEVICE; - prop.location.id = device; - CUmemGenericAllocationHandle handle; - CU_CHECK(cuMemCreate(&handle, reserve_size, &prop, 0)); - - // reserve virtual address space (if not already reserved) - if (g_cuda_pool_addr[device] == 0) { - CU_CHECK(cuMemAddressReserve(&g_cuda_pool_addr[device], CUDA_POOL_VMM_MAX_SIZE, 0, 0, 0)); - } - - // map at the end of the pool - CU_CHECK(cuMemMap(g_cuda_pool_addr[device] + g_cuda_pool_size[device], reserve_size, 0, handle, 0)); - - // the memory allocation handle is no longer needed after mapping - CU_CHECK(cuMemRelease(handle)); - - // set access - CUmemAccessDesc access = {}; - access.location.type = CU_MEM_LOCATION_TYPE_DEVICE; - access.location.id = device; - access.flags = CU_MEM_ACCESS_FLAGS_PROT_READWRITE; - CU_CHECK(cuMemSetAccess(g_cuda_pool_addr[device] + g_cuda_pool_size[device], reserve_size, &access, 1)); - - // add to the pool - g_cuda_pool_size[device] += reserve_size; - - //printf("cuda pool[%d]: size increased to %llu MB (reserved %llu MB)\n", - // id, (unsigned long long) (g_cuda_pool_size[id]/1024/1024), - // (unsigned long long) (reserve_size/1024/1024)); - } - - GGML_ASSERT(g_cuda_pool_addr[device] != 0); - - void * ptr = (void *) (g_cuda_pool_addr[device] + g_cuda_pool_used[device]); - *actual_size = size; - g_cuda_pool_used[device] += size; - -#ifdef DEBUG_CUDA_MALLOC - printf("cuda pool[%d]: allocated %llu bytes at %llx\n", device, (unsigned long long) size, ptr); -#endif - - return ptr; -} - -static void ggml_cuda_pool_free_vmm(int device, void * ptr, size_t size) { - scoped_spin_lock lock(g_cuda_pool_lock); - -#ifdef DEBUG_CUDA_MALLOC - printf("cuda pool[%d]: freed %llu bytes at %llx\n", device, (unsigned long long) size, ptr); -#endif - - g_cuda_pool_used[device] -= size; - - // all deallocations must be in reverse order of the allocations - GGML_ASSERT(ptr == (void *) (g_cuda_pool_addr[device] + g_cuda_pool_used[device])); -} - -static void * ggml_cuda_pool_malloc(int device, size_t size, size_t * actual_size) { - if (g_device_caps[device].vmm) { - return ggml_cuda_pool_malloc_vmm(device, size, actual_size); - } else { - return ggml_cuda_pool_malloc_leg(device, size, actual_size); - } -} - -static void ggml_cuda_pool_free(int device, void * ptr, size_t size) { - if (g_device_caps[device].vmm) { - ggml_cuda_pool_free_vmm(device, ptr, size); - } else { - ggml_cuda_pool_free_leg(device, ptr, size); - } -} -#else -#define ggml_cuda_pool_malloc ggml_cuda_pool_malloc_leg -#define ggml_cuda_pool_free ggml_cuda_pool_free_leg -#endif // !defined(GGML_USE_HIPBLAS) - -template -struct cuda_pool_alloc { - int device = -1; - T * ptr = nullptr; - size_t actual_size = 0; - - // size is in number of elements - T * alloc(size_t size) { - GGML_ASSERT(ptr == nullptr); - CUDA_CHECK(cudaGetDevice(&device)); - ptr = (T *) ggml_cuda_pool_malloc(device, size * sizeof(T), &this->actual_size); - return ptr; - } - - cuda_pool_alloc(size_t size) { - alloc(size); - } - - ~cuda_pool_alloc() { - if (ptr != nullptr) { - ggml_cuda_pool_free(device, ptr, actual_size); - } - } - - T * get() { - return ptr; - } - - cuda_pool_alloc() = default; - cuda_pool_alloc(const cuda_pool_alloc &) = delete; - cuda_pool_alloc(cuda_pool_alloc &&) = delete; - cuda_pool_alloc& operator=(const cuda_pool_alloc &) = delete; - cuda_pool_alloc& operator=(cuda_pool_alloc &&) = delete; -}; - -static bool g_cublas_loaded = false; - -static void ggml_init_cublas() { - static bool initialized = false; - - if (!initialized) { - -#ifdef __HIP_PLATFORM_AMD__ - // Workaround for a rocBLAS bug when using multiple graphics cards: - // https://github.com/ROCmSoftwarePlatform/rocBLAS/issues/1346 - rocblas_initialize(); - CUDA_CHECK(cudaDeviceSynchronize()); -#endif - - if (cudaGetDeviceCount(&g_device_count) != cudaSuccess) { - initialized = true; - g_cublas_loaded = false; - fprintf(stderr, "%s: no " GGML_CUDA_NAME " devices found, " GGML_CUDA_NAME " will be disabled\n", __func__); - return; - } - - GGML_ASSERT(g_device_count <= GGML_CUDA_MAX_DEVICES); - int64_t total_vram = 0; -#if defined(GGML_CUDA_FORCE_MMQ) - fprintf(stderr, "%s: GGML_CUDA_FORCE_MMQ: yes\n", __func__); -#else - fprintf(stderr, "%s: GGML_CUDA_FORCE_MMQ: no\n", __func__); -#endif -#if defined(CUDA_USE_TENSOR_CORES) - fprintf(stderr, "%s: CUDA_USE_TENSOR_CORES: yes\n", __func__); -#else - fprintf(stderr, "%s: CUDA_USE_TENSOR_CORES: no\n", __func__); -#endif - fprintf(stderr, "%s: found %d " GGML_CUDA_NAME " devices:\n", __func__, g_device_count); - for (int id = 0; id < g_device_count; ++id) { - int device_vmm = 0; - -#if !defined(GGML_USE_HIPBLAS) - CUdevice device; - CU_CHECK(cuDeviceGet(&device, id)); - CU_CHECK(cuDeviceGetAttribute(&device_vmm, CU_DEVICE_ATTRIBUTE_VIRTUAL_MEMORY_MANAGEMENT_SUPPORTED, device)); - - if (device_vmm) { - CUmemAllocationProp alloc_prop = {}; - alloc_prop.type = CU_MEM_ALLOCATION_TYPE_PINNED; - alloc_prop.location.type = CU_MEM_LOCATION_TYPE_DEVICE; - alloc_prop.location.id = id; - CU_CHECK(cuMemGetAllocationGranularity(&g_device_caps[id].vmm_granularity, &alloc_prop, CU_MEM_ALLOC_GRANULARITY_RECOMMENDED)); - } -#endif // !defined(GGML_USE_HIPBLAS) - g_device_caps[id].vmm = !!device_vmm; - - cudaDeviceProp prop; - CUDA_CHECK(cudaGetDeviceProperties(&prop, id)); - fprintf(stderr, " Device %d: %s, compute capability %d.%d, VMM: %s\n", id, prop.name, prop.major, prop.minor, device_vmm ? "yes" : "no"); - - g_default_tensor_split[id] = total_vram; - total_vram += prop.totalGlobalMem; - -#if defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__) - g_device_caps[id].cc = 100*prop.major + 10*prop.minor + CC_OFFSET_AMD; -#else - g_device_caps[id].cc = 100*prop.major + 10*prop.minor; -#endif // defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__) - g_device_caps[id].smpb = prop.sharedMemPerBlock; - } - for (int id = 0; id < g_device_count; ++id) { - g_default_tensor_split[id] /= total_vram; - } - - for (int id = 0; id < g_device_count; ++id) { - ggml_cuda_set_device(id); - - // create cuda streams - for (int is = 0; is < MAX_STREAMS; ++is) { - CUDA_CHECK(cudaStreamCreateWithFlags(&g_cudaStreams[id][is], cudaStreamNonBlocking)); - } - - // create cublas handle - CUBLAS_CHECK(cublasCreate(&g_cublas_handles[id])); - CUBLAS_CHECK(cublasSetMathMode(g_cublas_handles[id], CUBLAS_TF32_TENSOR_OP_MATH)); - } - - // configure logging to stdout - // CUBLAS_CHECK(cublasLoggerConfigure(1, 1, 0, nullptr)); - - initialized = true; - g_cublas_loaded = true; - } -} - -static void * ggml_cuda_host_malloc(size_t size) { - if (getenv("GGML_CUDA_NO_PINNED") != nullptr) { - return nullptr; - } - - void * ptr = nullptr; - cudaError_t err = cudaMallocHost((void **) &ptr, size); - if (err != cudaSuccess) { - // clear the error - cudaGetLastError(); - fprintf(stderr, "%s: warning: failed to allocate %.2f MiB of pinned memory: %s\n", __func__, - size/1024.0/1024.0, cudaGetErrorString(err)); - return nullptr; - } - - return ptr; -} - -static void ggml_cuda_host_free(void * ptr) { - CUDA_CHECK(cudaFreeHost(ptr)); -} - static cudaError_t ggml_cuda_cpy_tensor_2d( void * dst, const struct ggml_tensor * src, int64_t i3, int64_t i2, int64_t i1_low, int64_t i1_high, cudaStream_t stream) { - cudaMemcpyKind kind; - char * src_ptr; - if (src->backend == GGML_BACKEND_TYPE_CPU) { - kind = cudaMemcpyHostToDevice; - src_ptr = (char *) src->data; - } else if (src->backend == GGML_BACKEND_TYPE_GPU || src->backend == GGML_BACKEND_TYPE_GPU_SPLIT) { - GGML_ASSERT(src->backend != GGML_BACKEND_TYPE_GPU_SPLIT || (i1_low == 0 && i1_high == src->ne[1])); - kind = cudaMemcpyDeviceToDevice; - ggml_tensor_extra_gpu * extra = (ggml_tensor_extra_gpu *) src->extra; - int id; - CUDA_CHECK(cudaGetDevice(&id)); - src_ptr = (char *) extra->data_device[id]; - } else { - GGML_ASSERT(false); - } + GGML_ASSERT(ggml_backend_buffer_is_cuda(src->buffer)); + char * src_ptr = (char *) src->data; char * dst_ptr = (char *) dst; const int64_t ne0 = src->ne[0]; @@ -7934,25 +8691,30 @@ static cudaError_t ggml_cuda_cpy_tensor_2d( const char * x = src_ptr + i1_low*nb1 + i2*nb2 + i3*nb3; if (nb0 == ts && nb1 == ts*ne0/bs) { - return cudaMemcpyAsync(dst_ptr, x, i1_diff*nb1, kind, stream); + return cudaMemcpyAsync(dst_ptr, x, i1_diff*nb1, cudaMemcpyDeviceToDevice, stream); } else if (nb0 == ts) { - return cudaMemcpy2DAsync(dst_ptr, ts*ne0/bs, x, nb1, ts*ne0/bs, i1_diff, kind, stream); + return cudaMemcpy2DAsync(dst_ptr, ts*ne0/bs, x, nb1, ts*ne0/bs, i1_diff, cudaMemcpyDeviceToDevice, stream); } else { for (int64_t i1 = 0; i1 < i1_diff; i1++) { const void * rx = (const void *) ((const char *) x + i1*nb1); void * rd = (void *) (dst_ptr + i1*ts*ne0/bs); // pretend the row is a matrix with cols=1 - cudaError_t r = cudaMemcpy2DAsync(rd, ts/bs, rx, nb0, ts/bs, ne0, kind, stream); - if (r != cudaSuccess) return r; + cudaError_t r = cudaMemcpy2DAsync(rd, ts/bs, rx, nb0, ts/bs, ne0, cudaMemcpyDeviceToDevice, stream); + if (r != cudaSuccess) { + return r; + } } return cudaSuccess; } } static void ggml_cuda_op_get_rows( + ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, const float * src0_d, const float * src1_d, float * dst_d, cudaStream_t stream) { + GGML_UNUSED(ctx); + GGML_ASSERT(src1->type == GGML_TYPE_I32); GGML_ASSERT(dst->type == GGML_TYPE_F32); @@ -7994,9 +8756,12 @@ static void ggml_cuda_op_get_rows( template static void ggml_cuda_op_bin_bcast( + ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, const float * src0_dd, const float * src1_dd, float * dst_dd, cudaStream_t main_stream) { + GGML_UNUSED(ctx); + GGML_ASSERT(src1->type == GGML_TYPE_F32); if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { @@ -8013,26 +8778,31 @@ static void ggml_cuda_op_bin_bcast( } static void ggml_cuda_op_repeat( + ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, const float * src0_d, const float * src1_d, float * dst_d, cudaStream_t main_stream) { - ggml_cuda_op_bin_bcast>(dst, src0, dst, nullptr, src0_d, dst_d, main_stream); + ggml_cuda_op_bin_bcast>(ctx, dst, src0, dst, nullptr, src0_d, dst_d, main_stream); - (void) src1; - (void) src1_d; + GGML_UNUSED(src1); + GGML_UNUSED(src1_d); } static void ggml_cuda_op_add( + ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, const float * src0_dd, const float * src1_dd, float * dst_dd, cudaStream_t main_stream) { - ggml_cuda_op_bin_bcast>(src0, src1, dst, src0_dd, src1_dd, dst_dd, main_stream); + ggml_cuda_op_bin_bcast>(ctx, src0, src1, dst, src0_dd, src1_dd, dst_dd, main_stream); } static void ggml_cuda_op_acc( + ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, const float * src0_dd, const float * src1_dd, float * dst_dd, cudaStream_t main_stream) { + GGML_UNUSED(ctx); + GGML_ASSERT(src0->type == GGML_TYPE_F32); GGML_ASSERT(src1->type == GGML_TYPE_F32); GGML_ASSERT( dst->type == GGML_TYPE_F32); @@ -8045,125 +8815,135 @@ static void ggml_cuda_op_acc( acc_f32_cuda(src0_dd, src1_dd, dst_dd, ggml_nelements(dst), src1->ne[0], src1->ne[1], src1->ne[2], nb1, nb2, offset, main_stream); - (void) dst; + GGML_UNUSED(dst); } static void ggml_cuda_op_mul( + ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, const float * src0_dd, const float * src1_dd, float * dst_dd, cudaStream_t main_stream) { - ggml_cuda_op_bin_bcast>(src0, src1, dst, src0_dd, src1_dd, dst_dd, main_stream); + ggml_cuda_op_bin_bcast>(ctx, src0, src1, dst, src0_dd, src1_dd, dst_dd, main_stream); } static void ggml_cuda_op_div( + ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, const float * src0_dd, const float * src1_dd, float * dst_dd, cudaStream_t main_stream) { - ggml_cuda_op_bin_bcast>(src0, src1, dst, src0_dd, src1_dd, dst_dd, main_stream); + ggml_cuda_op_bin_bcast>(ctx, src0, src1, dst, src0_dd, src1_dd, dst_dd, main_stream); } static void ggml_cuda_op_gelu( + ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, const float * src0_dd, const float * src1_dd, float * dst_dd, cudaStream_t main_stream) { - + GGML_UNUSED(ctx); GGML_ASSERT(src0->type == GGML_TYPE_F32); GGML_ASSERT( dst->type == GGML_TYPE_F32); gelu_f32_cuda(src0_dd, dst_dd, ggml_nelements(src0), main_stream); - (void) src1; - (void) dst; - (void) src1_dd; + GGML_UNUSED(src1); + GGML_UNUSED(dst); + GGML_UNUSED(src1_dd); } static void ggml_cuda_op_silu( + ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, const float * src0_dd, const float * src1_dd, float * dst_dd, cudaStream_t main_stream) { - + GGML_UNUSED(ctx); GGML_ASSERT(src0->type == GGML_TYPE_F32); GGML_ASSERT( dst->type == GGML_TYPE_F32); silu_f32_cuda(src0_dd, dst_dd, ggml_nelements(src0), main_stream); - (void) src1; - (void) dst; - (void) src1_dd; + GGML_UNUSED(src1); + GGML_UNUSED(dst); + GGML_UNUSED(src1_dd); } static void ggml_cuda_op_gelu_quick( + ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, const float * src0_dd, const float * src1_dd, float * dst_dd, cudaStream_t main_stream) { - + GGML_UNUSED(ctx); GGML_ASSERT(src0->type == GGML_TYPE_F32); GGML_ASSERT( dst->type == GGML_TYPE_F32); gelu_quick_f32_cuda(src0_dd, dst_dd, ggml_nelements(src0), main_stream); - (void) src1; - (void) dst; - (void) src1_dd; + GGML_UNUSED(src1); + GGML_UNUSED(dst); + GGML_UNUSED(src1_dd); } static void ggml_cuda_op_tanh( + ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, const float * src0_dd, const float * src1_dd, float * dst_dd, cudaStream_t main_stream) { - + GGML_UNUSED(ctx); GGML_ASSERT(src0->type == GGML_TYPE_F32); GGML_ASSERT( dst->type == GGML_TYPE_F32); tanh_f32_cuda(src0_dd, dst_dd, ggml_nelements(src0), main_stream); - (void) src1; - (void) dst; - (void) src1_dd; + GGML_UNUSED(src1); + GGML_UNUSED(dst); + GGML_UNUSED(src1_dd); } static void ggml_cuda_op_relu( + ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, const float * src0_dd, const float * src1_dd, float * dst_dd, cudaStream_t main_stream) { - + GGML_UNUSED(ctx); GGML_ASSERT(src0->type == GGML_TYPE_F32); GGML_ASSERT( dst->type == GGML_TYPE_F32); relu_f32_cuda(src0_dd, dst_dd, ggml_nelements(src0), main_stream); - (void) src1; - (void) dst; - (void) src1_dd; + GGML_UNUSED(src1); + GGML_UNUSED(dst); + GGML_UNUSED(src1_dd); } static void ggml_cuda_op_hardsigmoid( + ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, const float * src0_dd, const float * src1_dd, float * dst_dd, cudaStream_t main_stream) { - + GGML_UNUSED(ctx); GGML_ASSERT(src0->type == GGML_TYPE_F32); GGML_ASSERT( dst->type == GGML_TYPE_F32); hardsigmoid_f32_cuda(src0_dd, dst_dd, ggml_nelements(src0), main_stream); - (void) src1; - (void) dst; - (void) src1_dd; + GGML_UNUSED(src1); + GGML_UNUSED(dst); + GGML_UNUSED(src1_dd); } static void ggml_cuda_op_hardswish( + ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, const float * src0_dd, const float * src1_dd, float * dst_dd, cudaStream_t main_stream) { - + GGML_UNUSED(ctx); GGML_ASSERT(src0->type == GGML_TYPE_F32); GGML_ASSERT( dst->type == GGML_TYPE_F32); hardswish_f32_cuda(src0_dd, dst_dd, ggml_nelements(src0), main_stream); - (void) src1; - (void) dst; - (void) src1_dd; + GGML_UNUSED(src1); + GGML_UNUSED(dst); + GGML_UNUSED(src1_dd); } static void ggml_cuda_op_leaky_relu( + ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, const float * src0_dd, const float * src1_dd, float * dst_dd, cudaStream_t main_stream) { - + GGML_UNUSED(ctx); GGML_ASSERT(src0->type == GGML_TYPE_F32); GGML_ASSERT( dst->type == GGML_TYPE_F32); @@ -8172,29 +8952,31 @@ static void ggml_cuda_op_leaky_relu( leaky_relu_f32_cuda(src0_dd, dst_dd, ggml_nelements(src0), negative_slope, main_stream); - (void) src1; - (void) dst; - (void) src1_dd; + GGML_UNUSED(src1); + GGML_UNUSED(dst); + GGML_UNUSED(src1_dd); } static void ggml_cuda_op_sqr( + ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, const float * src0_dd, const float * src1_dd, float * dst_dd, cudaStream_t main_stream) { - + GGML_UNUSED(ctx); GGML_ASSERT(src0->type == GGML_TYPE_F32); GGML_ASSERT( dst->type == GGML_TYPE_F32); sqr_f32_cuda(src0_dd, dst_dd, ggml_nelements(src0), main_stream); - (void) src1; - (void) dst; - (void) src1_dd; + GGML_UNUSED(src1); + GGML_UNUSED(dst); + GGML_UNUSED(src1_dd); } static void ggml_cuda_op_norm( + ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, const float * src0_dd, const float * src1_dd, float * dst_dd, cudaStream_t main_stream) { - + GGML_UNUSED(ctx); GGML_ASSERT(src0->type == GGML_TYPE_F32); GGML_ASSERT( dst->type == GGML_TYPE_F32); @@ -8206,15 +8988,16 @@ static void ggml_cuda_op_norm( norm_f32_cuda(src0_dd, dst_dd, ne00, nrows, eps, main_stream); - (void) src1; - (void) dst; - (void) src1_dd; + GGML_UNUSED(src1); + GGML_UNUSED(dst); + GGML_UNUSED(src1_dd); } static void ggml_cuda_op_group_norm( + ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, const float * src0_dd, const float * src1_dd, float * dst_dd, cudaStream_t main_stream) { - + GGML_UNUSED(ctx); GGML_ASSERT(src0->type == GGML_TYPE_F32); GGML_ASSERT( dst->type == GGML_TYPE_F32); @@ -8222,15 +9005,16 @@ static void ggml_cuda_op_group_norm( int group_size = src0->ne[0] * src0->ne[1] * ((src0->ne[2] + num_groups - 1) / num_groups); group_norm_f32_cuda(src0_dd, dst_dd, num_groups * src0->ne[3], group_size, ggml_nelements(src0), main_stream); - (void) src1; - (void) dst; - (void) src1_dd; + GGML_UNUSED(src1); + GGML_UNUSED(dst); + GGML_UNUSED(src1_dd); } static void ggml_cuda_op_concat( + ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, const float * src0_dd, const float * src1_dd, float * dst_dd, cudaStream_t main_stream) { - + GGML_UNUSED(ctx); GGML_ASSERT(src0->type == GGML_TYPE_F32); GGML_ASSERT(src1->type == GGML_TYPE_F32); GGML_ASSERT(dst->type == GGML_TYPE_F32); @@ -8239,14 +9023,15 @@ static void ggml_cuda_op_concat( concat_f32_cuda(src0_dd + i3 * (src0->nb[3] / 4), src1_dd + i3 * (src1->nb[3] / 4), dst_dd + i3 * (dst->nb[3] / 4), dst->ne[0], dst->ne[1], dst->ne[2], src0->ne[2], main_stream); } - (void) src1; - (void) dst; + GGML_UNUSED(src1); + GGML_UNUSED(dst); } static void ggml_cuda_op_upscale( + ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, const float * src0_dd, const float * src1_dd, float * dst_dd, cudaStream_t main_stream) { - + GGML_UNUSED(ctx); GGML_ASSERT(src0->type == GGML_TYPE_F32); GGML_ASSERT(dst->type == GGML_TYPE_F32); GGML_ASSERT(src0->ne[3] == 1 && dst->ne[3] == 1); // just 3D tensors @@ -8255,15 +9040,16 @@ static void ggml_cuda_op_upscale( upscale_f32_cuda(src0_dd, dst_dd, src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3], scale_factor, main_stream); - (void) src1; - (void) dst; - (void) src1_dd; + GGML_UNUSED(src1); + GGML_UNUSED(dst); + GGML_UNUSED(src1_dd); } static void ggml_cuda_op_pad( + ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, const float * src0_dd, const float * src1_dd, float * dst_dd, cudaStream_t main_stream) { - + GGML_UNUSED(ctx); GGML_ASSERT(src0->type == GGML_TYPE_F32); GGML_ASSERT(dst->type == GGML_TYPE_F32); GGML_ASSERT(src0->ne[3] == 1 && dst->ne[3] == 1); // just 3D tensors @@ -8272,15 +9058,16 @@ static void ggml_cuda_op_pad( src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3], dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], main_stream); - (void) src1; - (void) dst; - (void) src1_dd; + GGML_UNUSED(src1); + GGML_UNUSED(dst); + GGML_UNUSED(src1_dd); } static void ggml_cuda_op_arange( + ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, const float * src0_dd, const float * src1_dd, float * dst_dd, cudaStream_t main_stream) { - + GGML_UNUSED(ctx); GGML_ASSERT(dst->type == GGML_TYPE_F32); float start; @@ -8295,16 +9082,17 @@ static void ggml_cuda_op_arange( arange_f32_cuda(dst_dd, dst->ne[0], start, step, main_stream); - (void) src0; - (void) src1; - (void) src0_dd; - (void) src1_dd; + GGML_UNUSED(src0); + GGML_UNUSED(src1); + GGML_UNUSED(src0_dd); + GGML_UNUSED(src1_dd); } static void ggml_cuda_op_timestep_embedding( + ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, const float * src0_dd, const float * src1_dd, float * dst_dd, cudaStream_t main_stream) { - + GGML_UNUSED(ctx); GGML_ASSERT(src0->type == GGML_TYPE_F32); GGML_ASSERT(dst->type == GGML_TYPE_F32); @@ -8313,15 +9101,16 @@ static void ggml_cuda_op_timestep_embedding( timestep_embedding_f32_cuda(src0_dd, dst_dd, src0->ne[0], dst->nb[1], dim, max_period, main_stream); - (void) src1; - (void) dst; - (void) src1_dd; + GGML_UNUSED(src1); + GGML_UNUSED(dst); + GGML_UNUSED(src1_dd); } static void ggml_cuda_op_rms_norm( + ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, const float * src0_dd, const float * src1_dd, float * dst_dd, cudaStream_t main_stream) { - + GGML_UNUSED(ctx); GGML_ASSERT(src0->type == GGML_TYPE_F32); GGML_ASSERT( dst->type == GGML_TYPE_F32); @@ -8333,12 +9122,13 @@ static void ggml_cuda_op_rms_norm( rms_norm_f32_cuda(src0_dd, dst_dd, ne00, nrows, eps, main_stream); - (void) src1; - (void) dst; - (void) src1_dd; + GGML_UNUSED(src1); + GGML_UNUSED(dst); + GGML_UNUSED(src1_dd); } static void ggml_cuda_op_mul_mat_q( + ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, const char * src0_dd_i, const float * src1_ddf_i, const char * src1_ddq_i, float * dst_dd_i, const int64_t row_low, const int64_t row_high, const int64_t src1_ncols, const int64_t src1_padded_row_size, cudaStream_t stream) { @@ -8352,12 +9142,11 @@ static void ggml_cuda_op_mul_mat_q( const int64_t row_diff = row_high - row_low; - int id; - CUDA_CHECK(cudaGetDevice(&id)); + int id = ggml_cuda_get_device(); // the main device has a larger memory buffer to hold the results from all GPUs // nrows_dst == nrows of the matrix that the kernel writes into - const int64_t nrows_dst = dst->backend == GGML_BACKEND_TYPE_GPU && id == g_main_device ? ne0 : row_diff; + const int64_t nrows_dst = id == ctx.device ? ne0 : row_diff; switch (src0->type) { case GGML_TYPE_Q4_0: @@ -8395,104 +9184,13 @@ static void ggml_cuda_op_mul_mat_q( break; } - (void) src1; - (void) dst; - (void) src1_ddf_i; -} - -static int64_t get_row_rounding(ggml_type type, const std::array & tensor_split) { - int64_t min_compute_capability = INT_MAX; - int64_t max_compute_capability = INT_MIN; - for (int id = 0; id < g_device_count; ++id) { - if (tensor_split[id] < (id + 1 < g_device_count ? tensor_split[id + 1] : 1.0f)) { - if (min_compute_capability > g_device_caps[id].cc) { - min_compute_capability = g_device_caps[id].cc; - } - if (max_compute_capability < g_device_caps[id].cc) { - max_compute_capability = g_device_caps[id].cc; - } - } - } - -#if defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__) - switch(type) { - case GGML_TYPE_Q4_0: - case GGML_TYPE_Q4_1: - case GGML_TYPE_Q5_0: - case GGML_TYPE_Q5_1: - case GGML_TYPE_Q8_0: - return max_compute_capability >= CC_RDNA2 ? 128 : 64; - case GGML_TYPE_F16: - case GGML_TYPE_F32: - return 1; - case GGML_TYPE_Q2_K: - return max_compute_capability >= CC_RDNA2 ? 128 : 32; - case GGML_TYPE_Q3_K: - return min_compute_capability < CC_RDNA2 ? 128 : 64; - case GGML_TYPE_Q4_K: - case GGML_TYPE_Q5_K: - case GGML_TYPE_Q6_K: - case GGML_TYPE_IQ2_XXS: - case GGML_TYPE_IQ2_XS: - case GGML_TYPE_IQ2_S: - case GGML_TYPE_IQ3_XXS: - case GGML_TYPE_IQ1_S: - case GGML_TYPE_IQ4_NL: - case GGML_TYPE_IQ4_XS: - case GGML_TYPE_IQ3_S: - return max_compute_capability >= CC_RDNA2 ? 128 : 64; - default: - GGML_ASSERT(false); - } -#else - switch(type) { - case GGML_TYPE_Q4_0: - case GGML_TYPE_Q4_1: - return max_compute_capability >= CC_VOLTA ? 128 : 64; - case GGML_TYPE_Q5_0: - case GGML_TYPE_Q5_1: - case GGML_TYPE_Q8_0: - return 64; - case GGML_TYPE_F16: - case GGML_TYPE_F32: - return 1; - case GGML_TYPE_Q2_K: - case GGML_TYPE_Q3_K: - case GGML_TYPE_Q4_K: - case GGML_TYPE_Q5_K: - case GGML_TYPE_IQ2_XXS: - case GGML_TYPE_IQ2_XS: - case GGML_TYPE_IQ2_S: - case GGML_TYPE_IQ3_XXS: - case GGML_TYPE_IQ1_S: - case GGML_TYPE_IQ4_NL: - case GGML_TYPE_IQ4_XS: - case GGML_TYPE_IQ3_S: - return max_compute_capability >= CC_VOLTA ? 128 : 64; - case GGML_TYPE_Q6_K: - return 64; - default: - GGML_ASSERT(false); - } -#endif // defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__) -} - -static void get_row_split(int64_t * row_low, int64_t * row_high, const ggml_tensor * tensor, const std::array & tensor_split, int id) { - const int64_t nrows = ggml_nrows(tensor); - const int64_t rounding = get_row_rounding(tensor->type, tensor_split); - - *row_low = id == 0 ? 0 : nrows*tensor_split[id]; - *row_low -= *row_low % rounding; - - if (id == g_device_count - 1) { - *row_high = nrows; - } else { - *row_high = nrows*tensor_split[id + 1]; - *row_high -= *row_high % rounding; - } + GGML_UNUSED(src1); + GGML_UNUSED(dst); + GGML_UNUSED(src1_ddf_i); } static void ggml_cuda_op_mul_mat_vec_q( + ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, const char * src0_dd_i, const float * src1_ddf_i, const char * src1_ddq_i, float * dst_dd_i, const int64_t row_low, const int64_t row_high, const int64_t src1_ncols, const int64_t src1_padded_row_size, cudaStream_t stream) { @@ -8510,7 +9208,7 @@ static void ggml_cuda_op_mul_mat_vec_q( // the main device has a larger memory buffer to hold the results from all GPUs // nrows_dst == nrows of the matrix that the kernel writes into - const int64_t nrows_dst = dst->backend == GGML_BACKEND_TYPE_GPU && id == g_main_device ? ne0 : row_diff; + const int64_t nrows_dst = id == ctx.device ? ne0 : row_diff; switch (src0->type) { case GGML_TYPE_Q4_0: @@ -8590,18 +9288,19 @@ static void ggml_cuda_op_mul_mat_vec_q( break; } - (void) src1; - (void) dst; - (void) src1_ddf_i; - (void) src1_ncols; - (void) src1_padded_row_size; + GGML_UNUSED(src1); + GGML_UNUSED(dst); + GGML_UNUSED(src1_ddf_i); + GGML_UNUSED(src1_ncols); + GGML_UNUSED(src1_padded_row_size); } static void ggml_cuda_op_dequantize_mul_mat_vec( + ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, const char * src0_dd_i, const float * src1_ddf_i, const char * src1_ddq_i, float * dst_dd_i, const int64_t row_low, const int64_t row_high, const int64_t src1_ncols, const int64_t src1_padded_row_size, cudaStream_t stream) { - + GGML_UNUSED(ctx); const int64_t ne00 = src0->ne[0]; const int64_t row_diff = row_high - row_low; @@ -8666,14 +9365,15 @@ static void ggml_cuda_op_dequantize_mul_mat_vec( break; } - (void) src1; - (void) dst; - (void) src1_ddq_i; - (void) src1_ncols; - (void) src1_padded_row_size; + GGML_UNUSED(src1); + GGML_UNUSED(dst); + GGML_UNUSED(src1_ddq_i); + GGML_UNUSED(src1_ncols); + GGML_UNUSED(src1_padded_row_size); } static void ggml_cuda_op_mul_mat_cublas( + ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, const char * src0_dd_i, const float * src1_ddf_i, const char * src1_ddq_i, float * dst_dd_i, const int64_t row_low, const int64_t row_high, const int64_t src1_ncols, const int64_t src1_padded_row_size, cudaStream_t stream) { @@ -8689,19 +9389,17 @@ static void ggml_cuda_op_mul_mat_cublas( const int64_t row_diff = row_high - row_low; - int id; - CUDA_CHECK(cudaGetDevice(&id)); + int id = ggml_cuda_get_device(); // the main device has a larger memory buffer to hold the results from all GPUs // ldc == nrows of the matrix that cuBLAS writes into - int ldc = dst->backend == GGML_BACKEND_TYPE_GPU && id == g_main_device ? ne0 : row_diff; + int ldc = id == ctx.device ? ne0 : row_diff; - const int compute_capability = g_device_caps[id].cc; + const int compute_capability = get_cuda_global_info().devices[id].cc; if (compute_capability >= CC_VOLTA && (src0->type == GGML_TYPE_F16 || ggml_is_quantized(src0->type)) && ggml_is_contiguous(src0) && row_diff == src0->ne[1] && dst->op_params[0] == GGML_PREC_DEFAULT) { - //printf("this branch\n"); // convert src0 and src1 to fp16, multiply as fp16, convert dst to fp32 - cuda_pool_alloc src0_as_f16; + ggml_cuda_pool_alloc src0_as_f16(ctx.pool()); if (src0->type != GGML_TYPE_F16) { const to_fp16_cuda_t to_fp16_cuda = ggml_get_to_fp16_cuda(src0->type); GGML_ASSERT(to_fp16_cuda != nullptr); @@ -8711,7 +9409,7 @@ static void ggml_cuda_op_mul_mat_cublas( } const half * src0_ptr = src0->type == GGML_TYPE_F16 ? (const half *) src0_dd_i : src0_as_f16.get(); - cuda_pool_alloc src1_as_f16; + ggml_cuda_pool_alloc src1_as_f16(ctx.pool()); if (src1->type != GGML_TYPE_F16) { const to_fp16_cuda_t to_fp16_cuda = ggml_get_to_fp16_cuda(src1->type); GGML_ASSERT(to_fp16_cuda != nullptr); @@ -8720,14 +9418,14 @@ static void ggml_cuda_op_mul_mat_cublas( to_fp16_cuda(src1_ddf_i, src1_as_f16.get(), ne, stream); } const half * src1_ptr = src1->type == GGML_TYPE_F16 ? (const half *) src1_ddf_i : src1_as_f16.get(); - cuda_pool_alloc dst_f16(row_diff*src1_ncols); + ggml_cuda_pool_alloc dst_f16(ctx.pool(), row_diff*src1_ncols); const half alpha_f16 = 1.0f; const half beta_f16 = 0.0f; - CUBLAS_CHECK(cublasSetStream(g_cublas_handles[id], stream)); + CUBLAS_CHECK(cublasSetStream(ctx.cublas_handle(id), stream)); CUBLAS_CHECK( - cublasGemmEx(g_cublas_handles[id], CUBLAS_OP_T, CUBLAS_OP_N, + cublasGemmEx(ctx.cublas_handle(id), CUBLAS_OP_T, CUBLAS_OP_N, row_diff, src1_ncols, ne10, &alpha_f16, src0_ptr, CUDA_R_16F, ne00, src1_ptr, CUDA_R_16F, ne10, @@ -8738,8 +9436,8 @@ static void ggml_cuda_op_mul_mat_cublas( const to_fp32_cuda_t to_fp32_cuda = ggml_get_to_fp32_cuda(GGML_TYPE_F16); to_fp32_cuda(dst_f16.get(), dst_dd_i, row_diff*src1_ncols, stream); } else { - cuda_pool_alloc src0_ddq_as_f32; - cuda_pool_alloc src1_ddq_as_f32; + ggml_cuda_pool_alloc src0_ddq_as_f32(ctx.pool(id)); + ggml_cuda_pool_alloc src1_ddq_as_f32(ctx.pool(id)); if (src0->type != GGML_TYPE_F32) { const to_fp32_cuda_t to_fp32_cuda = ggml_get_to_fp32_cuda(src0->type); @@ -8760,24 +9458,25 @@ static void ggml_cuda_op_mul_mat_cublas( const float alpha = 1.0f; const float beta = 0.0f; - CUBLAS_CHECK(cublasSetStream(g_cublas_handles[id], stream)); + CUBLAS_CHECK(cublasSetStream(ctx.cublas_handle(id), stream)); CUBLAS_CHECK( - cublasSgemm(g_cublas_handles[id], CUBLAS_OP_T, CUBLAS_OP_N, + cublasSgemm(ctx.cublas_handle(id), CUBLAS_OP_T, CUBLAS_OP_N, row_diff, src1_ncols, ne10, &alpha, src0_ddf_i, ne00, src1_ddf1_i, ne10, &beta, dst_dd_i, ldc)); } - (void) dst; - (void) src1_ddq_i; - (void) src1_padded_row_size; + GGML_UNUSED(dst); + GGML_UNUSED(src1_ddq_i); + GGML_UNUSED(src1_padded_row_size); } static void ggml_cuda_op_rope( + ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, const float * src0_dd, const float * src1_dd, float * dst_dd, cudaStream_t main_stream) { - + GGML_UNUSED(ctx); GGML_ASSERT(src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16); GGML_ASSERT( dst->type == GGML_TYPE_F32 || dst->type == GGML_TYPE_F16); GGML_ASSERT(src0->type == dst->type); @@ -8849,15 +9548,16 @@ static void ggml_cuda_op_rope( } } - (void) src1; - (void) dst; - (void) src1_dd; + GGML_UNUSED(src1); + GGML_UNUSED(dst); + GGML_UNUSED(src1_dd); } static void ggml_cuda_op_alibi( + ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, const float * src0_dd, const float * src1_dd, float * dst_dd, cudaStream_t main_stream) { - + GGML_UNUSED(ctx); GGML_ASSERT(src0->type == GGML_TYPE_F32); GGML_ASSERT( dst->type == GGML_TYPE_F32); @@ -8881,14 +9581,15 @@ static void ggml_cuda_op_alibi( alibi_f32_cuda(src0_dd, dst_dd, ne00, nrows, ne01, n_heads_log2_floor, m0, m1, main_stream); - (void) src1; - (void) src1_dd; + GGML_UNUSED(src1); + GGML_UNUSED(src1_dd); } static void ggml_cuda_op_pool2d( + ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, const float * src0_dd, const float * src1_dd, float * dst_dd, cudaStream_t main_stream) { - + GGML_UNUSED(ctx); GGML_ASSERT(src0->type == GGML_TYPE_F32); GGML_ASSERT( dst->type == GGML_TYPE_F32); @@ -8914,14 +9615,15 @@ static void ggml_cuda_op_pool2d( dim3 block_nums(num_blocks); pool2d_nchw_kernel<<>>(IH, IW, OH, OW, k1, k0, s1, s0, p1, p0, parallel_elements, src0_dd, dst_dd, op); - (void) src1; - (void) src1_dd; + GGML_UNUSED(src1); + GGML_UNUSED(src1_dd); } static void ggml_cuda_op_im2col( + ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, const float * src0_dd, const float * src1_dd, float * dst_dd, cudaStream_t main_stream) { - + GGML_UNUSED(ctx); GGML_ASSERT(src0->type == GGML_TYPE_F16); GGML_ASSERT(src1->type == GGML_TYPE_F32); GGML_ASSERT( dst->type == GGML_TYPE_F16 || dst->type == GGML_TYPE_F32); @@ -8955,14 +9657,15 @@ static void ggml_cuda_op_im2col( im2col_cuda(src1_dd, (float*) dst_dd, IW, IH, OW, OH, KW, KH, IC, batch, batch_offset, delta_offset, s0, s1, p0, p1, d0, d1, main_stream); } - (void) src0; - (void) src0_dd; + GGML_UNUSED(src0); + GGML_UNUSED(src0_dd); } static void ggml_cuda_op_sum_rows( + ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, const float * src0_dd, const float * src1_dd, float * dst_dd, cudaStream_t main_stream) { - + GGML_UNUSED(ctx); GGML_ASSERT(src0->type == GGML_TYPE_F32); GGML_ASSERT( dst->type == GGML_TYPE_F32); @@ -8971,15 +9674,16 @@ static void ggml_cuda_op_sum_rows( sum_rows_f32_cuda(src0_dd, dst_dd, ncols, nrows, main_stream); - (void) src1; - (void) dst; - (void) src1_dd; + GGML_UNUSED(src1); + GGML_UNUSED(dst); + GGML_UNUSED(src1_dd); } static void ggml_cuda_op_argsort( + ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, const float * src0_dd, const float * src1_dd, float * dst_dd, cudaStream_t main_stream) { - + GGML_UNUSED(ctx); GGML_ASSERT(src0->type == GGML_TYPE_F32); GGML_ASSERT( dst->type == GGML_TYPE_I32); @@ -8990,15 +9694,16 @@ static void ggml_cuda_op_argsort( argsort_f32_i32_cuda(src0_dd, (int *)dst_dd, ncols, nrows, order, main_stream); - (void) src1; - (void) dst; - (void) src1_dd; + GGML_UNUSED(src1); + GGML_UNUSED(dst); + GGML_UNUSED(src1_dd); } static void ggml_cuda_op_diag_mask_inf( + ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, const float * src0_dd, const float * src1_dd, float * dst_dd, cudaStream_t main_stream) { - + GGML_UNUSED(ctx); GGML_ASSERT(src0->type == GGML_TYPE_F32); GGML_ASSERT( dst->type == GGML_TYPE_F32); @@ -9010,15 +9715,16 @@ static void ggml_cuda_op_diag_mask_inf( diag_mask_inf_f32_cuda(src0_dd, dst_dd, ne00, nrows0, ne01, n_past, main_stream); - (void) src1; - (void) dst; - (void) src1_dd; + GGML_UNUSED(src1); + GGML_UNUSED(dst); + GGML_UNUSED(src1_dd); } static void ggml_cuda_op_soft_max( + ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, const float * src0_dd, const float * src1_dd, float * dst_dd, cudaStream_t main_stream) { - + GGML_UNUSED(ctx); GGML_ASSERT(src0->type == GGML_TYPE_F32); GGML_ASSERT( dst->type == GGML_TYPE_F32); @@ -9041,17 +9747,17 @@ static void ggml_cuda_op_soft_max( const bool use_src2 = src2 != nullptr; if (use_src2) { - ggml_tensor_extra_gpu * src2_extra = (ggml_tensor_extra_gpu *) src2->extra; - src2_dd = (float *) src2_extra->data_device[g_main_device]; + src2_dd = (float *)src2->data; } soft_max_f32_cuda(src0_dd, src1 ? src1_dd : nullptr, src2_dd, dst_dd, ne00, nrows_x, nrows_y, scale, max_bias, main_stream); } static void ggml_cuda_op_scale( + ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, const float * src0_dd, const float * src1_dd, float * dst_dd, cudaStream_t main_stream) { - + GGML_UNUSED(ctx); GGML_ASSERT(src0->type == GGML_TYPE_F32); GGML_ASSERT( dst->type == GGML_TYPE_F32); @@ -9061,15 +9767,16 @@ static void ggml_cuda_op_scale( scale_f32_cuda(src0_dd, dst_dd, scale, ggml_nelements(src0), main_stream); CUDA_CHECK(cudaGetLastError()); - (void) src1; - (void) dst; - (void) src1_dd; + GGML_UNUSED(src1); + GGML_UNUSED(dst); + GGML_UNUSED(src1_dd); } static void ggml_cuda_op_clamp( + ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, const float * src0_dd, const float * src1_dd, float * dst_dd, cudaStream_t main_stream) { - + GGML_UNUSED(ctx); GGML_ASSERT(src0->type == GGML_TYPE_F32); GGML_ASSERT( dst->type == GGML_TYPE_F32); @@ -9081,45 +9788,30 @@ static void ggml_cuda_op_clamp( clamp_f32_cuda(src0_dd, dst_dd, min, max, ggml_nelements(src0), main_stream); CUDA_CHECK(cudaGetLastError()); - (void) src1; - (void) dst; - (void) src1_dd; + GGML_UNUSED(src1); + GGML_UNUSED(dst); + GGML_UNUSED(src1_dd); } -static void ggml_cuda_op_flatten(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, const ggml_cuda_op_flatten_t op) { - const int64_t nrows0 = ggml_nrows(src0); - - const bool use_src1 = src1 != nullptr; - const int64_t nrows1 = use_src1 ? ggml_nrows(src1) : 1; - - GGML_ASSERT(!use_src1 || src1->backend != GGML_BACKEND_TYPE_GPU_SPLIT); - GGML_ASSERT( dst->backend != GGML_BACKEND_TYPE_GPU_SPLIT); - - ggml_tensor_extra_gpu * src0_extra = (ggml_tensor_extra_gpu *) src0->extra; - ggml_tensor_extra_gpu * src1_extra = use_src1 ? (ggml_tensor_extra_gpu *) src1->extra : nullptr; - ggml_tensor_extra_gpu * dst_extra = (ggml_tensor_extra_gpu *) dst->extra; +// TODO: remove this function +static void ggml_cuda_op_flatten(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, const ggml_cuda_op_flatten_t op) { + GGML_ASSERT(!src0 || ggml_backend_buffer_is_cuda(src0->buffer)); + GGML_ASSERT(!src1 || ggml_backend_buffer_is_cuda(src1->buffer)); + GGML_ASSERT( ggml_backend_buffer_is_cuda(dst->buffer)); // dd = data device - float * src0_ddf = nullptr; - float * src1_ddf = nullptr; - float * dst_ddf = nullptr; + float * src0_ddf = src0 ? (float *) src0->data : nullptr; + float * src1_ddf = src1 ? (float *) src1->data : nullptr; + float * dst_ddf = (float *) dst->data; - ggml_cuda_set_device(g_main_device); - cudaStream_t main_stream = g_cudaStreams[g_main_device][0]; - - src0_ddf = (float *) src0_extra->data_device[g_main_device]; - - if (use_src1) { - src1_ddf = (float *) src1_extra->data_device[g_main_device]; - } - dst_ddf = (float *) dst_extra->data_device[g_main_device]; + ggml_cuda_set_device(ctx.device); // do the computation - op(src0, src1, dst, src0_ddf, src1_ddf, dst_ddf, main_stream); + op(ctx, src0, src1, dst, src0_ddf, src1_ddf, dst_ddf, ctx.stream()); CUDA_CHECK(cudaGetLastError()); } -static void ggml_cuda_set_peer_access(const int n_tokens) { +static void ggml_cuda_set_peer_access(const int n_tokens, int main_device) { static bool peer_access_enabled = false; const bool enable_peer_access = n_tokens <= GGML_CUDA_PEER_MAX_BATCH_SIZE; @@ -9129,19 +9821,19 @@ static void ggml_cuda_set_peer_access(const int n_tokens) { } #ifdef NDEBUG - for (int id = 0; id < g_device_count; ++id) { + for (int id = 0; id < ggml_backend_cuda_get_device_count(); ++id) { ggml_cuda_set_device(id); CUDA_CHECK(cudaDeviceSynchronize()); } - for (int id = 0; id < g_device_count; ++id) { + for (int id = 0; id < ggml_backend_cuda_get_device_count(); ++id) { ggml_cuda_set_device(id); - for (int id_other = 0; id_other < g_device_count; ++id_other) { + for (int id_other = 0; id_other < ggml_backend_cuda_get_device_count(); ++id_other) { if (id == id_other) { continue; } - if (id != g_main_device && id_other != g_main_device) { + if (id != main_device && id_other != main_device) { continue; } @@ -9165,14 +9857,12 @@ static void ggml_cuda_set_peer_access(const int n_tokens) { #endif // NDEBUG peer_access_enabled = enable_peer_access; + + GGML_UNUSED(main_device); } -// FIXME: move this somewhere else -struct ggml_backend_cuda_split_buffer_type_context { - std::array tensor_split; -}; - static void ggml_cuda_op_mul_mat( + ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, ggml_cuda_op_mul_mat_t op, const bool convert_src1_to_q8_1) { @@ -9195,8 +9885,11 @@ static void ggml_cuda_op_mul_mat( const int nb2 = dst->nb[2]; const int nb3 = dst->nb[3]; - GGML_ASSERT(dst->backend != GGML_BACKEND_TYPE_GPU_SPLIT); - GGML_ASSERT(src1->backend != GGML_BACKEND_TYPE_GPU_SPLIT); + GGML_ASSERT(ggml_backend_buffer_is_cuda(dst->buffer)); + GGML_ASSERT(ggml_backend_buffer_is_cuda(src1->buffer)); + ggml_backend_cuda_buffer_context * src1_ctx = (ggml_backend_cuda_buffer_context *) src1->buffer->context; + ggml_backend_cuda_buffer_context * dst_ctx = (ggml_backend_cuda_buffer_context *) dst->buffer->context; + GGML_ASSERT(src1->type == GGML_TYPE_F32 || (src1->ne[2] == 1 && src1->ne[3] == 1)); GGML_ASSERT(ne12 >= ne02 && ne12 % ne02 == 0); @@ -9208,33 +9901,30 @@ static void ggml_cuda_op_mul_mat( const size_t q8_1_ts = sizeof(block_q8_1); const size_t q8_1_bs = QK8_1; - ggml_tensor_extra_gpu * src0_extra = (ggml_tensor_extra_gpu *) src0->extra; - ggml_tensor_extra_gpu * src1_extra = (ggml_tensor_extra_gpu *) src1->extra; - ggml_tensor_extra_gpu * dst_extra = (ggml_tensor_extra_gpu *) dst->extra; - const bool src0_is_contiguous = ggml_is_contiguous(src0); const bool src1_is_contiguous = ggml_is_contiguous(src1); const int64_t src1_padded_col_size = GGML_PAD(ne10, MATRIX_ROW_PADDING); - const bool split = src0->backend == GGML_BACKEND_TYPE_GPU_SPLIT; + const bool split = ggml_backend_buffer_is_cuda_split(src0->buffer); GGML_ASSERT(!(split && ne02 > 1)); GGML_ASSERT(!(split && ne03 > 1)); GGML_ASSERT(!(split && ne02 < ne12)); + ggml_tensor_extra_gpu * src0_extra = split ? (ggml_tensor_extra_gpu *) src0->extra : nullptr; + + std::array tensor_split; if (split) { - // TODO: check that src0->buffer->buft is a split buffer type, replace GGML_BACKEND_TYPE_GPU_SPLIT check - // GGML_ASSERT(src0->buffer != nullptr && src0->buffer->buft == ...); ggml_backend_cuda_split_buffer_type_context * buft_ctx = (ggml_backend_cuda_split_buffer_type_context *) src0->buffer->buft->context; tensor_split = buft_ctx->tensor_split; } struct dev_data { - cuda_pool_alloc src0_dd_alloc; - cuda_pool_alloc src1_ddf_alloc; - cuda_pool_alloc src1_ddq_alloc; - cuda_pool_alloc dst_dd_alloc; + ggml_cuda_pool_alloc src0_dd_alloc; + ggml_cuda_pool_alloc src1_ddf_alloc; + ggml_cuda_pool_alloc src1_ddq_alloc; + ggml_cuda_pool_alloc dst_dd_alloc; char * src0_dd = nullptr; float * src1_ddf = nullptr; // float @@ -9249,7 +9939,7 @@ static void ggml_cuda_op_mul_mat( int used_devices = 0; - for (int id = 0; id < g_device_count; ++id) { + for (int id = 0; id < ggml_backend_cuda_get_device_count(); ++id) { // by default, use all rows dev[id].row_low = 0; dev[id].row_high = ne01; @@ -9266,7 +9956,7 @@ static void ggml_cuda_op_mul_mat( } } - if (id != g_device_count - 1) { + if (id != ggml_backend_cuda_get_device_count() - 1) { dev[id].row_high = ne01*tensor_split[id + 1]; if (dev[id].row_high < ne01) { dev[id].row_high -= dev[id].row_high % rounding; @@ -9275,33 +9965,33 @@ static void ggml_cuda_op_mul_mat( } } - for (int id = 0; id < g_device_count; ++id) { - if ((!split && id != g_main_device) || dev[id].row_low == dev[id].row_high) { + for (int id = 0; id < ggml_backend_cuda_get_device_count(); ++id) { + if ((!split && id != ctx.device) || dev[id].row_low == dev[id].row_high) { continue; } used_devices++; - const bool src1_on_device = id == g_main_device; // TODO: check from buffer - const bool dst_on_device = id == g_main_device; + const bool src1_on_device = id == src1_ctx->device; + const bool dst_on_device = id == dst_ctx->device; ggml_cuda_set_device(id); - cudaStream_t stream = g_cudaStreams[id][0]; + cudaStream_t stream = ctx.stream(id, 0); if (src0_is_contiguous) { - dev[id].src0_dd = (char *) src0_extra->data_device[id]; + dev[id].src0_dd = split ? (char *) src0_extra->data_device[id] : (char *) src0->data; } else { - dev[id].src0_dd = dev[id].src0_dd_alloc.alloc(ggml_nbytes(src0)); + dev[id].src0_dd = dev[id].src0_dd_alloc.alloc(ctx.pool(id), ggml_nbytes(src0)); } if (src1_on_device && src1_is_contiguous) { - dev[id].src1_ddf = (float *) src1_extra->data_device[id]; + dev[id].src1_ddf = (float *) src1->data; } else { - dev[id].src1_ddf = dev[id].src1_ddf_alloc.alloc(ggml_nelements(src1)); + dev[id].src1_ddf = dev[id].src1_ddf_alloc.alloc(ctx.pool(id), ggml_nelements(src1)); } if (convert_src1_to_q8_1) { - dev[id].src1_ddq = dev[id].src1_ddq_alloc.alloc(nrows1*src1_padded_col_size*q8_1_ts/q8_1_bs); + dev[id].src1_ddq = dev[id].src1_ddq_alloc.alloc(ctx.pool(id), nrows1*src1_padded_col_size*q8_1_ts/q8_1_bs); if (src1_on_device && src1_is_contiguous) { quantize_row_q8_1_cuda(dev[id].src1_ddf, dev[id].src1_ddq, ne10, nrows1, src1_padded_col_size, stream); @@ -9310,40 +10000,40 @@ static void ggml_cuda_op_mul_mat( } if (dst_on_device) { - dev[id].dst_dd = (float *) dst_extra->data_device[id]; + dev[id].dst_dd = (float *) dst->data; } else { const size_t size_dst_ddf = split ? (dev[id].row_high - dev[id].row_low)*ne1 : ggml_nelements(dst); - dev[id].dst_dd = dev[id].dst_dd_alloc.alloc(size_dst_ddf); + dev[id].dst_dd = dev[id].dst_dd_alloc.alloc(ctx.pool(id), size_dst_ddf); } } // if multiple devices are used they need to wait for the main device // here an event is recorded that signals that the main device has finished calculating the input data if (split && used_devices > 1) { - ggml_cuda_set_device(g_main_device); - CUDA_CHECK(cudaEventRecord(src0_extra->events[g_main_device][0], g_cudaStreams[g_main_device][0])); + ggml_cuda_set_device(ctx.device); + CUDA_CHECK(cudaEventRecord(src0_extra->events[ctx.device][0], ctx.stream())); } const int64_t src1_col_stride = split && used_devices > 1 ? MUL_MAT_SRC1_COL_STRIDE : ne11; for (int64_t src1_col_0 = 0; src1_col_0 < ne11; src1_col_0 += src1_col_stride) { - const int64_t is = split ? (src1_col_0/src1_col_stride) % MAX_STREAMS : 0; + const int64_t is = split ? (src1_col_0/src1_col_stride) % GGML_CUDA_MAX_STREAMS : 0; const int64_t src1_ncols = src1_col_0 + src1_col_stride > ne11 ? ne11 - src1_col_0 : src1_col_stride; - for (int id = 0; id < g_device_count; ++id) { - if ((!split && id != g_main_device) || dev[id].row_low == dev[id].row_high) { + for (int id = 0; id < ggml_backend_cuda_get_device_count(); ++id) { + if ((!split && id != ctx.device) || dev[id].row_low == dev[id].row_high) { continue; } - const bool src1_on_device = id == g_main_device; // TODO: check from buffer - const bool dst_on_device = id == g_main_device; + const bool src1_on_device = id == src1_ctx->device; + const bool dst_on_device = id == dst_ctx->device; const int64_t row_diff = dev[id].row_high - dev[id].row_low; ggml_cuda_set_device(id); - cudaStream_t stream = g_cudaStreams[id][is]; + cudaStream_t stream = ctx.stream(id, is); // wait for main GPU data if necessary - if (split && (id != g_main_device || is != 0)) { - CUDA_CHECK(cudaStreamWaitEvent(stream, src0_extra->events[g_main_device][0], 0)); + if (split && (id != ctx.device || is != 0)) { + CUDA_CHECK(cudaStreamWaitEvent(stream, src0_extra->events[ctx.device][0], 0)); } for (int64_t i0 = 0; i0 < ne13*ne12; ++i0) { @@ -9360,21 +10050,21 @@ static void ggml_cuda_op_mul_mat( // the main device memory buffer can be on VRAM scratch, with space for all partial results // in that case an offset on dst_ddf_i is needed - if (id == g_main_device) { + if (id == ctx.device) { dst_dd_i += dev[id].row_low; // offset is 0 if no tensor split } // copy src0, src1 to device if necessary if (src1_is_contiguous) { - if (id != g_main_device) { + if (id != ctx.device) { if (convert_src1_to_q8_1) { - char * src1_ddq_i_source = dev[g_main_device].src1_ddq + src1_ddq_i_offset; - CUDA_CHECK(cudaMemcpyPeerAsync(src1_ddq_i, id, src1_ddq_i_source, g_main_device, + char * src1_ddq_i_source = dev[ctx.device].src1_ddq + src1_ddq_i_offset; + CUDA_CHECK(cudaMemcpyPeerAsync(src1_ddq_i, id, src1_ddq_i_source, ctx.device, src1_ncols*src1_padded_col_size*q8_1_ts/q8_1_bs, stream)); } else { - float * src1_ddf_i_source = (float *) src1_extra->data_device[g_main_device]; + float * src1_ddf_i_source = (float *) src1->data; src1_ddf_i_source += (i0*ne11 + src1_col_0) * ne10; - CUDA_CHECK(cudaMemcpyPeerAsync(src1_ddf_i, id, src1_ddf_i_source, g_main_device, + CUDA_CHECK(cudaMemcpyPeerAsync(src1_ddf_i, id, src1_ddf_i_source, ctx.device, src1_ncols*ne10*sizeof(float), stream)); } } @@ -9395,13 +10085,13 @@ static void ggml_cuda_op_mul_mat( } // do the computation - op(src0, src1, dst, src0_dd_i, src1_ddf_i, src1_ddq_i, dst_dd_i, + op(ctx, src0, src1, dst, src0_dd_i, src1_ddf_i, src1_ddq_i, dst_dd_i, dev[id].row_low, dev[id].row_high, src1_ncols, src1_padded_col_size, stream); CUDA_CHECK(cudaGetLastError()); // copy dst to host or other device if necessary if (!dst_on_device) { - void * dst_off_device = dst_extra->data_device[g_main_device]; + void * dst_off_device = dst->data; if (split) { // src0 = weight matrix is saved as a transposed matrix for better memory layout. // dst is NOT transposed. @@ -9414,7 +10104,7 @@ static void ggml_cuda_op_mul_mat( #if !defined(GGML_USE_HIPBLAS) // cudaMemcpy2DAsync may fail with copies between vmm pools of different devices cudaMemcpy3DPeerParms p = {}; - p.dstDevice = g_main_device; + p.dstDevice = ctx.device; p.dstPtr = make_cudaPitchedPtr(dhf_dst_i, ne0*sizeof(float), row_diff, src1_ncols); p.srcDevice = id; p.srcPtr = make_cudaPitchedPtr(dst_dd_i, row_diff*sizeof(float), row_diff, src1_ncols); @@ -9436,7 +10126,7 @@ static void ggml_cuda_op_mul_mat( } // add event for the main device to wait on until other device is done - if (split && (id != g_main_device || is != 0)) { + if (split && (id != ctx.device || is != 0)) { CUDA_CHECK(cudaEventRecord(src0_extra->events[id][is], stream)); } } @@ -9444,130 +10134,116 @@ static void ggml_cuda_op_mul_mat( } // main device waits for all other devices to be finished - if (split && g_device_count > 1) { + if (split && ggml_backend_cuda_get_device_count() > 1) { int64_t is_max = (ne11 + MUL_MAT_SRC1_COL_STRIDE - 1) / MUL_MAT_SRC1_COL_STRIDE; - is_max = is_max <= MAX_STREAMS ? is_max : MAX_STREAMS; + is_max = is_max <= GGML_CUDA_MAX_STREAMS ? is_max : GGML_CUDA_MAX_STREAMS; - ggml_cuda_set_device(g_main_device); - for (int id = 0; id < g_device_count; ++id) { + ggml_cuda_set_device(ctx.device); + for (int id = 0; id < ggml_backend_cuda_get_device_count(); ++id) { if (dev[id].row_low == dev[id].row_high) { continue; } for (int64_t is = 0; is < is_max; ++is) { - CUDA_CHECK(cudaStreamWaitEvent(g_cudaStreams[g_main_device][0], src0_extra->events[id][is], 0)); + CUDA_CHECK(cudaStreamWaitEvent(ctx.stream(), src0_extra->events[id][is], 0)); } } } } -static void ggml_cuda_repeat(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { - ggml_cuda_op_flatten(src0, src1, dst, ggml_cuda_op_repeat); +static void ggml_cuda_repeat(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + ggml_cuda_op_flatten(ctx, src0, src1, dst, ggml_cuda_op_repeat); } -static void ggml_cuda_get_rows(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { - ggml_cuda_op_flatten(src0, src1, dst, ggml_cuda_op_get_rows); +static void ggml_cuda_get_rows(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + ggml_cuda_op_flatten(ctx, src0, src1, dst, ggml_cuda_op_get_rows); } -static void ggml_cuda_add(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { - ggml_cuda_op_flatten(src0, src1, dst, ggml_cuda_op_add); +static void ggml_cuda_add(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + ggml_cuda_op_flatten(ctx, src0, src1, dst, ggml_cuda_op_add); } -static void ggml_cuda_acc(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { - ggml_cuda_op_flatten(src0, src1, dst, ggml_cuda_op_acc); +static void ggml_cuda_acc(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + ggml_cuda_op_flatten(ctx, src0, src1, dst, ggml_cuda_op_acc); } -static void ggml_cuda_mul(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { - ggml_cuda_op_flatten(src0, src1, dst, ggml_cuda_op_mul); +static void ggml_cuda_mul(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + ggml_cuda_op_flatten(ctx, src0, src1, dst, ggml_cuda_op_mul); } -static void ggml_cuda_div(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { - ggml_cuda_op_flatten(src0, src1, dst, ggml_cuda_op_div); +static void ggml_cuda_div(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + ggml_cuda_op_flatten(ctx, src0, src1, dst, ggml_cuda_op_div); } -static void ggml_cuda_gelu(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { - ggml_cuda_op_flatten(src0, src1, dst, ggml_cuda_op_gelu); +static void ggml_cuda_gelu(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + ggml_cuda_op_flatten(ctx, src0, src1, dst, ggml_cuda_op_gelu); } -static void ggml_cuda_silu(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { - ggml_cuda_op_flatten(src0, src1, dst, ggml_cuda_op_silu); +static void ggml_cuda_silu(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + ggml_cuda_op_flatten(ctx, src0, src1, dst, ggml_cuda_op_silu); } -static void ggml_cuda_gelu_quick(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { - ggml_cuda_op_flatten(src0, src1, dst, ggml_cuda_op_gelu_quick); +static void ggml_cuda_gelu_quick(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + ggml_cuda_op_flatten(ctx, src0, src1, dst, ggml_cuda_op_gelu_quick); } -static void ggml_cuda_tanh(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { - ggml_cuda_op_flatten(src0, src1, dst, ggml_cuda_op_tanh); +static void ggml_cuda_tanh(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + ggml_cuda_op_flatten(ctx, src0, src1, dst, ggml_cuda_op_tanh); } -static void ggml_cuda_relu(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { - ggml_cuda_op_flatten(src0, src1, dst, ggml_cuda_op_relu); +static void ggml_cuda_relu(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + ggml_cuda_op_flatten(ctx, src0, src1, dst, ggml_cuda_op_relu); } -static void ggml_cuda_hardsigmoid(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { - ggml_cuda_op_flatten(src0, src1, dst, ggml_cuda_op_hardsigmoid); +static void ggml_cuda_hardsigmoid(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + ggml_cuda_op_flatten(ctx, src0, src1, dst, ggml_cuda_op_hardsigmoid); } -static void ggml_cuda_hardswish(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { - ggml_cuda_op_flatten(src0, src1, dst, ggml_cuda_op_hardswish); +static void ggml_cuda_hardswish(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + ggml_cuda_op_flatten(ctx, src0, src1, dst, ggml_cuda_op_hardswish); } -static void ggml_cuda_leaky_relu(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { - ggml_cuda_op_flatten(src0, src1, dst, ggml_cuda_op_leaky_relu); +static void ggml_cuda_leaky_relu(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + ggml_cuda_op_flatten(ctx, src0, src1, dst, ggml_cuda_op_leaky_relu); } -static void ggml_cuda_sqr(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { - ggml_cuda_op_flatten(src0, src1, dst, ggml_cuda_op_sqr); +static void ggml_cuda_sqr(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + ggml_cuda_op_flatten(ctx, src0, src1, dst, ggml_cuda_op_sqr); } -static void ggml_cuda_norm(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { - ggml_cuda_op_flatten(src0, src1, dst, ggml_cuda_op_norm); +static void ggml_cuda_norm(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + ggml_cuda_op_flatten(ctx, src0, src1, dst, ggml_cuda_op_norm); } -static void ggml_cuda_group_norm(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { - ggml_cuda_op_flatten(src0, src1, dst, ggml_cuda_op_group_norm); +static void ggml_cuda_group_norm(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + ggml_cuda_op_flatten(ctx, src0, src1, dst, ggml_cuda_op_group_norm); } -static void ggml_cuda_concat(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { - ggml_cuda_op_flatten(src0, src1, dst, ggml_cuda_op_concat); +static void ggml_cuda_concat(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + ggml_cuda_op_flatten(ctx, src0, src1, dst, ggml_cuda_op_concat); } -static void ggml_cuda_upscale(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { - ggml_cuda_op_flatten(src0, src1, dst, ggml_cuda_op_upscale); +static void ggml_cuda_upscale(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + ggml_cuda_op_flatten(ctx, src0, src1, dst, ggml_cuda_op_upscale); } -static void ggml_cuda_pad(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { - ggml_cuda_op_flatten(src0, src1, dst, ggml_cuda_op_pad); +static void ggml_cuda_pad(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + ggml_cuda_op_flatten(ctx, src0, src1, dst, ggml_cuda_op_pad); } -static void ggml_cuda_arange(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { - ggml_tensor_extra_gpu * dst_extra = (ggml_tensor_extra_gpu *) dst->extra; - - // dd = data device - float * src0_ddf = nullptr; - float * src1_ddf = nullptr; - float * dst_ddf = nullptr; - - ggml_cuda_set_device(g_main_device); - cudaStream_t main_stream = g_cudaStreams[g_main_device][0]; - - dst_ddf = (float *) dst_extra->data_device[g_main_device]; - - // do the computation - ggml_cuda_op_arange(src0, src1, dst, src0_ddf, src1_ddf, dst_ddf, main_stream); - CUDA_CHECK(cudaGetLastError()); +static void ggml_cuda_arange(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + ggml_cuda_op_flatten(ctx, src0, src1, dst, ggml_cuda_op_arange); } -static void ggml_cuda_timestep_embedding(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { - ggml_cuda_op_flatten(src0, src1, dst, ggml_cuda_op_timestep_embedding); +static void ggml_cuda_timestep_embedding(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + ggml_cuda_op_flatten(ctx, src0, src1, dst, ggml_cuda_op_timestep_embedding); } -static void ggml_cuda_rms_norm(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { - ggml_cuda_op_flatten(src0, src1, dst, ggml_cuda_op_rms_norm); +static void ggml_cuda_rms_norm(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + ggml_cuda_op_flatten(ctx, src0, src1, dst, ggml_cuda_op_rms_norm); } -static void ggml_cuda_mul_mat_vec_p021(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst){ +static void ggml_cuda_mul_mat_vec_p021(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst){ GGML_ASSERT(ggml_is_permuted(src0) && ggml_is_permuted(src1)); - GGML_ASSERT(src0->backend != GGML_BACKEND_TYPE_GPU_SPLIT); + GGML_ASSERT(ggml_backend_buffer_is_cuda(src0->buffer)); GGML_ASSERT(src0->nb[0] <= src0->nb[1] && src0->nb[2] <= src0->nb[3]); // 0213 permutation GGML_ASSERT(src1->nb[0] <= src1->nb[1] && src1->nb[2] <= src1->nb[3]); // 0213 permutation GGML_ASSERT(src0->type == GGML_TYPE_F16); @@ -9579,26 +10255,21 @@ static void ggml_cuda_mul_mat_vec_p021(const ggml_tensor * src0, const ggml_tens const int64_t ne12 = src1->ne[2]; - ggml_cuda_set_device(g_main_device); - cudaStream_t main_stream = g_cudaStreams[g_main_device][0]; + ggml_cuda_set_device(ctx.device); + cudaStream_t main_stream = ctx.stream(); - ggml_tensor_extra_gpu * src0_extra = (ggml_tensor_extra_gpu *) src0->extra; - void * src0_ddq = src0_extra->data_device[g_main_device]; - - ggml_tensor_extra_gpu * src1_extra = (ggml_tensor_extra_gpu *) src1->extra; - float * src1_ddf = (float *) src1_extra->data_device[g_main_device]; - - ggml_tensor_extra_gpu * dst_extra = (ggml_tensor_extra_gpu *) dst->extra; - float * dst_ddf = (float *) dst_extra->data_device[g_main_device]; + void * src0_ddq = src0->data; + float * src1_ddf = (float *) src1->data; + float * dst_ddf = (float *) dst->data; ggml_mul_mat_p021_f16_f32_cuda(src0_ddq, src1_ddf, dst_ddf, ne00, ne01, ne02, ne12, main_stream); } -static void ggml_cuda_mul_mat_vec_nc(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst){ +static void ggml_cuda_mul_mat_vec_nc(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst){ GGML_ASSERT(!ggml_is_transposed(src0)); GGML_ASSERT(!ggml_is_transposed(src1)); GGML_ASSERT(!ggml_is_permuted(src0)); - GGML_ASSERT(src0->backend != GGML_BACKEND_TYPE_GPU_SPLIT); + GGML_ASSERT(ggml_backend_buffer_is_cuda(src0->buffer)); GGML_ASSERT(src0->type == GGML_TYPE_F16); GGML_ASSERT(src1->type == GGML_TYPE_F32); @@ -9611,17 +10282,12 @@ static void ggml_cuda_mul_mat_vec_nc(const ggml_tensor * src0, const ggml_tensor const int64_t ne12 = src1->ne[2]; - ggml_cuda_set_device(g_main_device); - cudaStream_t main_stream = g_cudaStreams[g_main_device][0]; + ggml_cuda_set_device(ctx.device); + cudaStream_t main_stream = ctx.stream(); - ggml_tensor_extra_gpu * src0_extra = (ggml_tensor_extra_gpu *) src0->extra; - void * src0_ddq = src0_extra->data_device[g_main_device]; - - ggml_tensor_extra_gpu * src1_extra = (ggml_tensor_extra_gpu *) src1->extra; - float * src1_ddf = (float *) src1_extra->data_device[g_main_device]; - - ggml_tensor_extra_gpu * dst_extra = (ggml_tensor_extra_gpu *) dst->extra; - float * dst_ddf = (float *) dst_extra->data_device[g_main_device]; + void * src0_ddq = src0->data; + float * src1_ddf = (float *) src1->data; + float * dst_ddf = (float *) dst->data; const int64_t row_stride_x = nb01 / sizeof(half); const int64_t channel_stride_x = nb02 / sizeof(half); @@ -9653,34 +10319,29 @@ static __global__ void k_compute_batched_ptrs( ptrs_dst[0*ne23 + i12 + i13*ne12] = ( char *) dst + i12*nbd2 + i13*nbd3; } -static void ggml_cuda_mul_mat_batched_cublas(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { +static void ggml_cuda_mul_mat_batched_cublas(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { GGML_ASSERT(!ggml_is_transposed(src0)); GGML_ASSERT(!ggml_is_transposed(src1)); - GGML_ASSERT(src0->backend != GGML_BACKEND_TYPE_GPU_SPLIT); + GGML_ASSERT(ggml_backend_buffer_is_cuda(src0->buffer)); GGML_ASSERT(src0->type == GGML_TYPE_F16); GGML_TENSOR_BINARY_OP_LOCALS const int64_t ne_dst = ggml_nelements(dst); - ggml_cuda_set_device(g_main_device); - cudaStream_t main_stream = g_cudaStreams[g_main_device][0]; + ggml_cuda_set_device(ctx.device); + cudaStream_t main_stream = ctx.stream(); - CUBLAS_CHECK(cublasSetStream(g_cublas_handles[g_main_device], main_stream)); + CUBLAS_CHECK(cublasSetStream(ctx.cublas_handle(), main_stream)); - ggml_tensor_extra_gpu * src0_extra = (ggml_tensor_extra_gpu *) src0->extra; - void * src0_ddq = src0_extra->data_device[g_main_device]; + void * src0_ddq = src0->data; half * src0_f16 = (half *) src0_ddq; - - ggml_tensor_extra_gpu * src1_extra = (ggml_tensor_extra_gpu *) src1->extra; - float * src1_ddf = (float *) src1_extra->data_device[g_main_device]; - - ggml_tensor_extra_gpu * dst_extra = (ggml_tensor_extra_gpu *) dst->extra; - float * dst_ddf = (float *) dst_extra->data_device[g_main_device]; + float * src1_ddf = (float *) src1->data; + float * dst_ddf = (float *) dst->data; // convert src1 to fp16 - cuda_pool_alloc src1_f16_alloc; + ggml_cuda_pool_alloc src1_f16_alloc(ctx.pool()); if (src1->type != GGML_TYPE_F16) { const to_fp16_cuda_t to_fp16_cuda = ggml_get_to_fp16_cuda(src1->type); const int64_t ne_src1 = ggml_nelements(src1); @@ -9690,7 +10351,7 @@ static void ggml_cuda_mul_mat_batched_cublas(const ggml_tensor * src0, const ggm } half * src1_f16 = src1->type == GGML_TYPE_F16 ? (half *) src1_ddf : src1_f16_alloc.get(); - cuda_pool_alloc dst_f16; + ggml_cuda_pool_alloc dst_f16(ctx.pool()); char * dst_t; cublasComputeType_t cu_compute_type = CUBLAS_COMPUTE_16F; @@ -9755,7 +10416,7 @@ static void ggml_cuda_mul_mat_batched_cublas(const ggml_tensor * src0, const ggm // there is no broadcast and src0, src1 are contiguous across dims 2, 3 // use cublasGemmStridedBatchedEx CUBLAS_CHECK( - cublasGemmStridedBatchedEx(g_cublas_handles[g_main_device], CUBLAS_OP_T, CUBLAS_OP_N, + cublasGemmStridedBatchedEx(ctx.cublas_handle(), CUBLAS_OP_T, CUBLAS_OP_N, ne01, ne11, ne10, alpha, (const char *) src0_f16, CUDA_R_16F, nb01/nb00, nb02/nb00, // strideA (const char *) src1_f16, CUDA_R_16F, nb11/nb10, nb12/nb10, // strideB @@ -9767,8 +10428,8 @@ static void ggml_cuda_mul_mat_batched_cublas(const ggml_tensor * src0, const ggm // use cublasGemmBatchedEx const int ne23 = ne12*ne13; - cuda_pool_alloc ptrs_src(2*ne23); - cuda_pool_alloc< void *> ptrs_dst(1*ne23); + ggml_cuda_pool_alloc ptrs_src(ctx.pool(), 2*ne23); + ggml_cuda_pool_alloc< void *> ptrs_dst(ctx.pool(), 1*ne23); dim3 block_dims(ne13, ne12); k_compute_batched_ptrs<<<1, block_dims, 0, main_stream>>>( @@ -9784,7 +10445,7 @@ static void ggml_cuda_mul_mat_batched_cublas(const ggml_tensor * src0, const ggm CUDA_CHECK(cudaGetLastError()); CUBLAS_CHECK( - cublasGemmBatchedEx(g_cublas_handles[g_main_device], CUBLAS_OP_T, CUBLAS_OP_N, + cublasGemmBatchedEx(ctx.cublas_handle(), CUBLAS_OP_T, CUBLAS_OP_N, ne01, ne11, ne10, alpha, (const void **) (ptrs_src.get() + 0*ne23), CUDA_R_16F, nb01/nb00, (const void **) (ptrs_src.get() + 1*ne23), CUDA_R_16F, nb11/nb10, @@ -9801,8 +10462,8 @@ static void ggml_cuda_mul_mat_batched_cublas(const ggml_tensor * src0, const ggm } } -static void ggml_cuda_mul_mat(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { - const bool split = src0->backend == GGML_BACKEND_TYPE_GPU_SPLIT; +static void ggml_cuda_mul_mat(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + const bool split = ggml_backend_buffer_is_cuda_split(src0->buffer); int64_t min_compute_capability = INT_MAX; @@ -9810,22 +10471,22 @@ static void ggml_cuda_mul_mat(const ggml_tensor * src0, const ggml_tensor * src1 if (split) { ggml_backend_cuda_split_buffer_type_context * buft_ctx = (ggml_backend_cuda_split_buffer_type_context *) src0->buffer->buft->context; auto & tensor_split = buft_ctx->tensor_split; - for (int id = 0; id < g_device_count; ++id) { + for (int id = 0; id < ggml_backend_cuda_get_device_count(); ++id) { // skip devices that are not going to do any work: - if (tensor_split[id] >= (id + 1 < g_device_count ? tensor_split[id + 1] : 1.0f)) { + if (tensor_split[id] >= (id + 1 < ggml_backend_cuda_get_device_count() ? tensor_split[id + 1] : 1.0f)) { continue; } - if (min_compute_capability > g_device_caps[id].cc) { - min_compute_capability = g_device_caps[id].cc; + if (min_compute_capability > get_cuda_global_info().devices[id].cc) { + min_compute_capability = get_cuda_global_info().devices[id].cc; } - if (g_device_caps[id].cc == 610) { + if (get_cuda_global_info().devices[id].cc == 610) { any_pascal_with_slow_fp16 = true; } } } else { - min_compute_capability = g_device_caps[g_main_device].cc; - any_pascal_with_slow_fp16 = g_device_caps[g_main_device].cc == 610; + min_compute_capability = get_cuda_global_info().devices[ctx.device].cc; + any_pascal_with_slow_fp16 = get_cuda_global_info().devices[ctx.device].cc == 610; } // check data types and tensor shapes for custom matrix multiplication kernels: @@ -9880,21 +10541,21 @@ static void ggml_cuda_mul_mat(const ggml_tensor * src0, const ggml_tensor * src1 if (!split && !fp16_performance_good && src0->type == GGML_TYPE_F16 && ggml_is_permuted(src0) && ggml_is_permuted(src1) && src1->ne[1] == 1) { // KQ single-batch - ggml_cuda_mul_mat_vec_p021(src0, src1, dst); + ggml_cuda_mul_mat_vec_p021(ctx, src0, src1, dst); } else if (!split && !fp16_performance_good && src0->type == GGML_TYPE_F16 && !ggml_is_contiguous(src0) && !ggml_is_transposed(src1) && src1->ne[1] == 1) { // KQV single-batch - ggml_cuda_mul_mat_vec_nc(src0, src1, dst); + ggml_cuda_mul_mat_vec_nc(ctx, src0, src1, dst); } else if (!split && fp16_performance_good && src0->type == GGML_TYPE_F16 && !ggml_is_transposed(src0) && !ggml_is_transposed(src1) && src1->ne[2]*src1->ne[3] > 1) { // KQ + KQV multi-batch - ggml_cuda_mul_mat_batched_cublas(src0, src1, dst); + ggml_cuda_mul_mat_batched_cublas(ctx, src0, src1, dst); } else if (use_dequantize_mul_mat_vec) { - ggml_cuda_op_mul_mat(src0, src1, dst, ggml_cuda_op_dequantize_mul_mat_vec, false); + ggml_cuda_op_mul_mat(ctx, src0, src1, dst, ggml_cuda_op_dequantize_mul_mat_vec, false); } else if (use_mul_mat_vec_q) { - ggml_cuda_op_mul_mat(src0, src1, dst, ggml_cuda_op_mul_mat_vec_q, true); + ggml_cuda_op_mul_mat(ctx, src0, src1, dst, ggml_cuda_op_mul_mat_vec_q, true); } else if (use_mul_mat_q) { - ggml_cuda_op_mul_mat(src0, src1, dst, ggml_cuda_op_mul_mat_q, true); + ggml_cuda_op_mul_mat(ctx, src0, src1, dst, ggml_cuda_op_mul_mat_q, true); } else { - ggml_cuda_op_mul_mat(src0, src1, dst, ggml_cuda_op_mul_mat_cublas, false); + ggml_cuda_op_mul_mat(ctx, src0, src1, dst, ggml_cuda_op_mul_mat_cublas, false); } } @@ -10079,12 +10740,12 @@ static void ggml_cuda_mul_mat_id_cublas(ggml_tensor * dst) { } #endif -static void ggml_cuda_mul_mat_id(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { +static void ggml_cuda_mul_mat_id(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { #if 0 ggml_cuda_mul_mat_id_cublas(dst); // TODO: mmq/mmv support #endif - cudaStream_t stream = g_cudaStreams[g_main_device][0]; + cudaStream_t stream = ctx.stream(); const size_t nb11 = src1->nb[1]; const size_t nb1 = dst->nb[1]; @@ -10094,27 +10755,15 @@ static void ggml_cuda_mul_mat_id(const ggml_tensor * src0, const ggml_tensor * s const int32_t n_as = ((int32_t *) dst->op_params)[1]; std::vector ids_host(ggml_nbytes(ids)); - const char * ids_dev = (const char *)((const ggml_tensor_extra_gpu *)ids->extra)->data_device[g_main_device]; + const char * ids_dev = (const char *) ids->data; CUDA_CHECK(cudaMemcpyAsync(ids_host.data(), ids_dev, ggml_nbytes(ids), cudaMemcpyDeviceToHost, stream)); CUDA_CHECK(cudaStreamSynchronize(stream)); - const ggml_tensor_extra_gpu * src1_extra = (const ggml_tensor_extra_gpu *) src1->extra; - const ggml_tensor_extra_gpu * dst_extra = (const ggml_tensor_extra_gpu *) dst->extra; - - ggml_tensor_extra_gpu src1_row_extra; - ggml_tensor_extra_gpu dst_row_extra; - ggml_tensor src1_row = *src1; ggml_tensor dst_row = *dst; - src1_row.backend = GGML_BACKEND_TYPE_GPU; - dst_row.backend = GGML_BACKEND_TYPE_GPU; - - src1_row.extra = &src1_row_extra; - dst_row.extra = &dst_row_extra; - - char * src1_original = (char *) src1_extra->data_device[g_main_device]; - char * dst_original = (char *) dst_extra->data_device[g_main_device]; + char * src1_original = (char *) src1->data; + char * dst_original = (char *) dst->data; if (src1->ne[1] == 1) { for (int64_t i01 = 0; i01 < ids->ne[1]; i01++) { @@ -10124,20 +10773,17 @@ static void ggml_cuda_mul_mat_id(const ggml_tensor * src0, const ggml_tensor * s const struct ggml_tensor * src0_row = dst->src[row_id + 2]; - src1_row_extra.data_device[g_main_device] = src1_original + i01*src1->nb[1]; - src1_row.data = (char *) src1->data + i01*src1->nb[1]; // TODO why is this set? + src1_row.data = src1_original + i01*src1->nb[1]; + dst_row.data = dst_original + i01*dst->nb[1]; - dst_row_extra.data_device[g_main_device] = dst_original + i01*dst->nb[1]; - dst_row.data = (char *) dst->data + i01*dst->nb[1]; // TODO why is this set? - - ggml_cuda_mul_mat(src0_row, &src1_row, &dst_row); + ggml_cuda_mul_mat(ctx, src0_row, &src1_row, &dst_row); } } else { - cuda_pool_alloc src1_contiguous(sizeof(float)*ggml_nelements(src1)); - cuda_pool_alloc dst_contiguous(sizeof(float)*ggml_nelements(dst)); + ggml_cuda_pool_alloc src1_contiguous(ctx.pool(), sizeof(float)*ggml_nelements(src1)); + ggml_cuda_pool_alloc dst_contiguous(ctx.pool(), sizeof(float)*ggml_nelements(dst)); - src1_row_extra.data_device[g_main_device] = src1_contiguous.get(); - dst_row_extra.data_device[g_main_device] = dst_contiguous.get(); + src1_row.data = src1_contiguous.get(); + dst_row.data = dst_contiguous.get(); for (int32_t row_id = 0; row_id < n_as; ++row_id) { const struct ggml_tensor * src0_row = dst->src[row_id + 2]; @@ -10172,7 +10818,7 @@ static void ggml_cuda_mul_mat_id(const ggml_tensor * src0, const ggml_tensor * s dst_row.nb[2] = num_src1_rows*nb1; dst_row.nb[3] = num_src1_rows*nb1; - ggml_cuda_mul_mat(src0_row, &src1_row, &dst_row); + ggml_cuda_mul_mat(ctx, src0_row, &src1_row, &dst_row); num_src1_rows = 0; for (int64_t i01 = 0; i01 < ids->ne[1]; i01++) { @@ -10192,20 +10838,19 @@ static void ggml_cuda_mul_mat_id(const ggml_tensor * src0, const ggml_tensor * s } } -static void ggml_cuda_scale(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { - ggml_cuda_op_flatten(src0, src1, dst, ggml_cuda_op_scale); +static void ggml_cuda_scale(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + ggml_cuda_op_flatten(ctx, src0, src1, dst, ggml_cuda_op_scale); } -static void ggml_cuda_clamp(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { - ggml_cuda_op_flatten(src0, src1, dst, ggml_cuda_op_clamp); +static void ggml_cuda_clamp(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + ggml_cuda_op_flatten(ctx, src0, src1, dst, ggml_cuda_op_clamp); } -static void ggml_cuda_cpy(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { +static void ggml_cuda_cpy(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { const int64_t ne = ggml_nelements(src0); GGML_ASSERT(ne == ggml_nelements(src1)); - GGML_ASSERT(src0->backend == GGML_BACKEND_TYPE_GPU); - GGML_ASSERT(src1->backend == GGML_BACKEND_TYPE_GPU); + GGML_ASSERT(ggml_backend_buffer_is_cuda(src0->buffer)); GGML_ASSERT(ggml_nbytes(src0) <= INT_MAX); GGML_ASSERT(ggml_nbytes(src1) <= INT_MAX); @@ -10232,14 +10877,11 @@ static void ggml_cuda_cpy(const ggml_tensor * src0, const ggml_tensor * src1, gg const int64_t nb12 = src1->nb[2]; const int64_t nb13 = src1->nb[3]; - ggml_cuda_set_device(g_main_device); - cudaStream_t main_stream = g_cudaStreams[g_main_device][0]; + ggml_cuda_set_device(ctx.device); + cudaStream_t main_stream = ctx.stream(); - const ggml_tensor_extra_gpu * src0_extra = (ggml_tensor_extra_gpu *) src0->extra; - const ggml_tensor_extra_gpu * src1_extra = (ggml_tensor_extra_gpu *) src1->extra; - - char * src0_ddc = (char *) src0_extra->data_device[g_main_device]; - char * src1_ddc = (char *) src1_extra->data_device[g_main_device]; + char * src0_ddc = (char *) src0->data; + char * src1_ddc = (char *) src1->data; if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_F32) { ggml_cpy_f32_f32_cuda (src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream); @@ -10261,87 +10903,61 @@ static void ggml_cuda_cpy(const ggml_tensor * src0, const ggml_tensor * src1, gg GGML_ASSERT(false); } - (void) dst; + GGML_UNUSED(dst); } -static void ggml_cuda_dup(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { +static void ggml_cuda_dup(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { // TODO: why do we pass dst as src1 here? - ggml_cuda_cpy(src0, dst, nullptr); - (void) src1; + ggml_cuda_cpy(ctx, src0, dst, nullptr); + GGML_UNUSED(src1); } -static void ggml_cuda_diag_mask_inf(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { - ggml_cuda_op_flatten(src0, src1, dst, ggml_cuda_op_diag_mask_inf); +static void ggml_cuda_diag_mask_inf(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + ggml_cuda_op_flatten(ctx, src0, src1, dst, ggml_cuda_op_diag_mask_inf); } -static void ggml_cuda_soft_max(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { - ggml_cuda_op_flatten(src0, src1, dst, ggml_cuda_op_soft_max); +static void ggml_cuda_soft_max(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + ggml_cuda_op_flatten(ctx, src0, src1, dst, ggml_cuda_op_soft_max); } -static void ggml_cuda_rope(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { +static void ggml_cuda_rope(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { GGML_ASSERT(ggml_is_contiguous(src0)); // TODO: this restriction is temporary until non-cont support is implemented - ggml_cuda_op_flatten(src0, src1, dst, ggml_cuda_op_rope); + ggml_cuda_op_flatten(ctx, src0, src1, dst, ggml_cuda_op_rope); } -static void ggml_cuda_alibi(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { - ggml_cuda_op_flatten(src0, src1, dst, ggml_cuda_op_alibi); +static void ggml_cuda_alibi(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + ggml_cuda_op_flatten(ctx, src0, src1, dst, ggml_cuda_op_alibi); } -static void ggml_cuda_pool2d(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { - ggml_cuda_op_flatten(src0, src1, dst, ggml_cuda_op_pool2d); +static void ggml_cuda_pool2d(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + ggml_cuda_op_flatten(ctx, src0, src1, dst, ggml_cuda_op_pool2d); } -static void ggml_cuda_im2col(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { - ggml_cuda_op_flatten(src0, src1, dst, ggml_cuda_op_im2col); +static void ggml_cuda_im2col(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + ggml_cuda_op_flatten(ctx, src0, src1, dst, ggml_cuda_op_im2col); } -static void ggml_cuda_sum_rows(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { +static void ggml_cuda_sum_rows(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { GGML_ASSERT(ggml_is_contiguous(src0)); - ggml_cuda_op_flatten(src0, src1, dst, ggml_cuda_op_sum_rows); + ggml_cuda_op_flatten(ctx, src0, src1, dst, ggml_cuda_op_sum_rows); } -static void ggml_cuda_argsort(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { +static void ggml_cuda_argsort(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { GGML_ASSERT(ggml_is_contiguous(src0)); - ggml_cuda_op_flatten(src0, src1, dst, ggml_cuda_op_argsort); + ggml_cuda_op_flatten(ctx, src0, src1, dst, ggml_cuda_op_argsort); } -static void ggml_cuda_nop(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { - (void) src0; - (void) src1; - (void) dst; +static void ggml_cuda_nop(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + GGML_UNUSED(ctx); + GGML_UNUSED(src0); + GGML_UNUSED(src1); + GGML_UNUSED(dst); } -static size_t ggml_nbytes_split(const struct ggml_tensor * tensor, int nrows_split) { - static_assert(GGML_MAX_DIMS == 4, "GGML_MAX_DIMS is not 4 - update this function"); - - return nrows_split*ggml_row_size(tensor->type, tensor->ne[0]); -} - -static void ggml_cuda_set_main_device(const int main_device) { - if (main_device >= g_device_count) { - fprintf(stderr, "warning: cannot set main_device=%d because there are only %d devices. Using device %d instead.\n", - main_device, g_device_count, g_main_device); - return; - } - - if (g_main_device != main_device && g_device_count > 1) { - g_main_device = main_device; - //cudaDeviceProp prop; - //CUDA_CHECK(cudaGetDeviceProperties(&prop, g_main_device)); - //fprintf(stderr, "%s: using device %d (%s) as main device\n", __func__, g_main_device, prop.name); - } -} - -static bool ggml_cuda_compute_forward(struct ggml_tensor * tensor) { - if (!g_cublas_loaded) return false; - - if (tensor->op == GGML_OP_MUL_MAT) { - if (tensor->src[0]->ne[3] != tensor->src[1]->ne[3]) { -#ifndef NDEBUG - fprintf(stderr, "%s: cannot compute %s: src0->ne[3] = %" PRId64 ", src1->ne[3] = %" PRId64 " - fallback to CPU\n", __func__, tensor->name, tensor->src[0]->ne[3], tensor->src[1]->ne[3]); -#endif - return false; - } +static bool ggml_cuda_compute_forward(ggml_backend_cuda_context & ctx, struct ggml_tensor * tensor) { + // FIXME: where should this be? + if (tensor->src[0] != nullptr && ggml_backend_buffer_is_cuda_split(tensor->src[0]->buffer)) { + ggml_cuda_set_peer_access(tensor->src[1]->ne[1], ctx.device); } ggml_cuda_func_t func; @@ -10423,7 +11039,12 @@ static bool ggml_cuda_compute_forward(struct ggml_tensor * tensor) { func = ggml_cuda_rms_norm; break; case GGML_OP_MUL_MAT: - func = ggml_cuda_mul_mat; + if (tensor->src[0]->ne[3] != tensor->src[1]->ne[3]) { + fprintf(stderr, "%s: cannot compute %s: src0->ne[3] = %" PRId64 ", src1->ne[3] = %" PRId64 " - fallback to CPU\n", __func__, tensor->name, tensor->src[0]->ne[3], tensor->src[1]->ne[3]); + return false; + } else { + func = ggml_cuda_mul_mat; + } break; case GGML_OP_MUL_MAT_ID: func = ggml_cuda_mul_mat_id; @@ -10478,640 +11099,13 @@ static bool ggml_cuda_compute_forward(struct ggml_tensor * tensor) { return false; } - if (tensor->src[0] != nullptr && tensor->src[0]->backend == GGML_BACKEND_TYPE_GPU_SPLIT) { - ggml_cuda_set_peer_access(tensor->src[1]->ne[1]); - } - - func(tensor->src[0], tensor->src[1], tensor); + func(ctx, tensor->src[0], tensor->src[1], tensor); return true; } -static int ggml_cuda_get_device_count() { - int device_count; - if (cudaGetDeviceCount(&device_count) != cudaSuccess) { - return 0; - } - return device_count; -} - -static void ggml_cuda_get_device_description(int device, char * description, size_t description_size) { - cudaDeviceProp prop; - CUDA_CHECK(cudaGetDeviceProperties(&prop, device)); - snprintf(description, description_size, "%s", prop.name); -} //////////////////////////////////////////////////////////////////////////////// -// backend interface - -#define UNUSED GGML_UNUSED - -struct ggml_backend_cuda_context { - explicit ggml_backend_cuda_context(int device) : - device(device), - name(GGML_CUDA_NAME + std::to_string(device)) { - } - - ~ggml_backend_cuda_context() { - if (copy_event != nullptr) { - CUDA_CHECK(cudaEventDestroy(copy_event)); - } - } - - int device; - std::string name; - cudaEvent_t copy_event = nullptr; -}; - -// cuda buffer - -struct ggml_backend_cuda_buffer_context { - int device; - void * dev_ptr = nullptr; - ggml_tensor_extra_gpu * temp_tensor_extras = nullptr; - size_t temp_tensor_extra_index = 0; - std::string name; - - ggml_backend_cuda_buffer_context(int device, void * dev_ptr) : - device(device), dev_ptr(dev_ptr), - name(GGML_CUDA_NAME + std::to_string(device)) { - } - - ~ggml_backend_cuda_buffer_context() { - delete[] temp_tensor_extras; - } - - ggml_tensor_extra_gpu * ggml_cuda_alloc_temp_tensor_extra() { - // TODO: remove GGML_CUDA_MAX_NODES, allocate dynamically and reuse in backend_buffer_reset - if (temp_tensor_extras == nullptr) { - temp_tensor_extras = new ggml_tensor_extra_gpu[GGML_CUDA_MAX_NODES]; - } - - size_t alloc_index = temp_tensor_extra_index; - temp_tensor_extra_index = (temp_tensor_extra_index + 1) % GGML_CUDA_MAX_NODES; - ggml_tensor_extra_gpu * extra = &temp_tensor_extras[alloc_index]; - memset(extra, 0, sizeof(*extra)); - - return extra; - } -}; - -GGML_CALL static const char * ggml_backend_cuda_buffer_get_name(ggml_backend_buffer_t buffer) { - ggml_backend_cuda_buffer_context * ctx = (ggml_backend_cuda_buffer_context *)buffer->context; - return ctx->name.c_str(); -} - -GGML_CALL static bool ggml_backend_buffer_is_cuda(ggml_backend_buffer_t buffer) { - return buffer->iface.get_name == ggml_backend_cuda_buffer_get_name; -} - -GGML_CALL static void ggml_backend_cuda_buffer_free_buffer(ggml_backend_buffer_t buffer) { - ggml_backend_cuda_buffer_context * ctx = (ggml_backend_cuda_buffer_context *)buffer->context; - CUDA_CHECK(cudaFree(ctx->dev_ptr)); - delete ctx; -} - -GGML_CALL static void * ggml_backend_cuda_buffer_get_base(ggml_backend_buffer_t buffer) { - ggml_backend_cuda_buffer_context * ctx = (ggml_backend_cuda_buffer_context *)buffer->context; - return ctx->dev_ptr; -} - -GGML_CALL static void ggml_backend_cuda_buffer_init_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor) { - ggml_backend_cuda_buffer_context * ctx = (ggml_backend_cuda_buffer_context *)buffer->context; - - if (tensor->view_src != NULL && tensor->view_offs == 0) { - assert(tensor->view_src->buffer->buft == buffer->buft); - tensor->backend = tensor->view_src->backend; - tensor->extra = tensor->view_src->extra; - return; - } - - ggml_tensor_extra_gpu * extra = ctx->ggml_cuda_alloc_temp_tensor_extra(); - - extra->data_device[ctx->device] = tensor->data; - - tensor->backend = GGML_BACKEND_TYPE_GPU; - tensor->extra = extra; - - if (ggml_is_quantized(tensor->type)) { - // initialize padding to 0 to avoid possible NaN values - size_t original_size = ggml_nbytes(tensor); - size_t padded_size = ggml_backend_buft_get_alloc_size(buffer->buft, tensor); - - if (padded_size > original_size && tensor->view_src == nullptr) { - ggml_cuda_set_device(ctx->device); - CUDA_CHECK(cudaMemset((char *)tensor->data + original_size, 0, padded_size - original_size)); - } - } -} - -GGML_CALL static void ggml_backend_cuda_buffer_set_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor, const void * data, size_t offset, size_t size) { - GGML_ASSERT(tensor->backend == GGML_BACKEND_TYPE_GPU); - - ggml_backend_cuda_buffer_context * ctx = (ggml_backend_cuda_buffer_context *)buffer->context; - - ggml_cuda_set_device(ctx->device); - CUDA_CHECK(cudaMemcpyAsync((char *)tensor->data + offset, data, size, cudaMemcpyHostToDevice, cudaStreamPerThread)); - CUDA_CHECK(cudaStreamSynchronize(cudaStreamPerThread)); -} - -GGML_CALL static void ggml_backend_cuda_buffer_get_tensor(ggml_backend_buffer_t buffer, const ggml_tensor * tensor, void * data, size_t offset, size_t size) { - GGML_ASSERT(tensor->backend == GGML_BACKEND_TYPE_GPU); - - ggml_backend_cuda_buffer_context * ctx = (ggml_backend_cuda_buffer_context *)buffer->context; - - ggml_cuda_set_device(ctx->device); - CUDA_CHECK(cudaMemcpyAsync(data, (const char *)tensor->data + offset, size, cudaMemcpyDeviceToHost, cudaStreamPerThread)); - CUDA_CHECK(cudaStreamSynchronize(cudaStreamPerThread)); -} - -GGML_CALL static bool ggml_backend_cuda_buffer_cpy_tensor(ggml_backend_buffer_t buffer, const ggml_tensor * src, ggml_tensor * dst) { - if (ggml_backend_buffer_is_cuda(src->buffer)) { - ggml_backend_cuda_buffer_context * src_ctx = (ggml_backend_cuda_buffer_context *)src->buffer->context; - ggml_backend_cuda_buffer_context * dst_ctx = (ggml_backend_cuda_buffer_context *)dst->buffer->context; - if (src_ctx->device == dst_ctx->device) { - CUDA_CHECK(cudaMemcpyAsync(dst->data, src->data, ggml_nbytes(src), cudaMemcpyDeviceToDevice, cudaStreamPerThread)); - } else { - CUDA_CHECK(cudaMemcpyPeerAsync(dst->data, dst_ctx->device, src->data, src_ctx->device, ggml_nbytes(src), cudaStreamPerThread)); - } - CUDA_CHECK(cudaStreamSynchronize(cudaStreamPerThread)); - return true; - } - return false; - - UNUSED(buffer); -} - -GGML_CALL static void ggml_backend_cuda_buffer_clear(ggml_backend_buffer_t buffer, uint8_t value) { - ggml_backend_cuda_buffer_context * ctx = (ggml_backend_cuda_buffer_context *)buffer->context; - - ggml_cuda_set_device(ctx->device); - CUDA_CHECK(cudaDeviceSynchronize()); - CUDA_CHECK(cudaMemset(ctx->dev_ptr, value, buffer->size)); - CUDA_CHECK(cudaDeviceSynchronize()); -} - -static ggml_backend_buffer_i ggml_backend_cuda_buffer_interface = { - /* .get_name = */ ggml_backend_cuda_buffer_get_name, - /* .free_buffer = */ ggml_backend_cuda_buffer_free_buffer, - /* .get_base = */ ggml_backend_cuda_buffer_get_base, - /* .init_tensor = */ ggml_backend_cuda_buffer_init_tensor, - /* .set_tensor = */ ggml_backend_cuda_buffer_set_tensor, - /* .get_tensor = */ ggml_backend_cuda_buffer_get_tensor, - /* .cpy_tensor = */ ggml_backend_cuda_buffer_cpy_tensor, - /* .clear = */ ggml_backend_cuda_buffer_clear, - /* .reset = */ NULL, -}; - -// cuda buffer type -struct ggml_backend_cuda_buffer_type_context { - int device; - std::string name; -}; - -GGML_CALL static const char * ggml_backend_cuda_buffer_type_name(ggml_backend_buffer_type_t buft) { - ggml_backend_cuda_buffer_type_context * ctx = (ggml_backend_cuda_buffer_type_context *)buft->context; - - return ctx->name.c_str(); -} - -GGML_CALL static ggml_backend_buffer_t ggml_backend_cuda_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size) { - ggml_backend_cuda_buffer_type_context * buft_ctx = (ggml_backend_cuda_buffer_type_context *)buft->context; - - ggml_cuda_set_device(buft_ctx->device); - - size = std::max(size, (size_t)1); // cudaMalloc returns null for size 0 - - void * dev_ptr; - cudaError_t err = cudaMalloc(&dev_ptr, size); - if (err != cudaSuccess) { - fprintf(stderr, "%s: allocating %.2f MiB on device %d: cudaMalloc failed: %s\n", __func__, size/1024.0/1024.0, buft_ctx->device, cudaGetErrorString(err)); - return nullptr; - } - - ggml_backend_cuda_buffer_context * ctx = new ggml_backend_cuda_buffer_context(buft_ctx->device, dev_ptr); - - return ggml_backend_buffer_init(buft, ggml_backend_cuda_buffer_interface, ctx, size); -} - -GGML_CALL static size_t ggml_backend_cuda_buffer_type_get_alignment(ggml_backend_buffer_type_t buft) { - return 128; - - UNUSED(buft); -} - -GGML_CALL static size_t ggml_backend_cuda_buffer_type_get_alloc_size(ggml_backend_buffer_type_t buft, const ggml_tensor * tensor) { - size_t size = ggml_nbytes(tensor); - int64_t ne0 = tensor->ne[0]; - - if (ggml_is_quantized(tensor->type)) { - if (ne0 % MATRIX_ROW_PADDING != 0) { - size += ggml_row_size(tensor->type, MATRIX_ROW_PADDING - ne0 % MATRIX_ROW_PADDING); - } - } - - return size; - - UNUSED(buft); -} - -GGML_CALL static bool ggml_backend_cuda_buffer_type_supports_backend(ggml_backend_buffer_type_t buft, ggml_backend_t backend) { - if (!ggml_backend_is_cuda(backend)) { - return false; - } - - ggml_backend_cuda_buffer_type_context * buft_ctx = (ggml_backend_cuda_buffer_type_context *)buft->context; - ggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *)backend->context; - - return buft_ctx->device == cuda_ctx->device; -} - -static ggml_backend_buffer_type_i ggml_backend_cuda_buffer_type_interface = { - /* .get_name = */ ggml_backend_cuda_buffer_type_name, - /* .alloc_buffer = */ ggml_backend_cuda_buffer_type_alloc_buffer, - /* .get_alignment = */ ggml_backend_cuda_buffer_type_get_alignment, - /* .get_max_size = */ NULL, // defaults to SIZE_MAX - /* .get_alloc_size = */ ggml_backend_cuda_buffer_type_get_alloc_size, - /* .supports_backend = */ ggml_backend_cuda_buffer_type_supports_backend, - /* .is_host = */ NULL, -}; - -GGML_CALL ggml_backend_buffer_type_t ggml_backend_cuda_buffer_type(int device) { - ggml_init_cublas(); - - // FIXME: this is not thread safe - if (device >= ggml_backend_cuda_get_device_count()) { - return nullptr; - } - - static ggml_backend_buffer_type ggml_backend_cuda_buffer_types[GGML_CUDA_MAX_DEVICES]; - - static bool ggml_backend_cuda_buffer_type_initialized = false; - - if (!ggml_backend_cuda_buffer_type_initialized) { - for (int i = 0; i < GGML_CUDA_MAX_DEVICES; i++) { - ggml_backend_cuda_buffer_types[i] = { - /* .iface = */ ggml_backend_cuda_buffer_type_interface, - /* .context = */ new ggml_backend_cuda_buffer_type_context{i, GGML_CUDA_NAME + std::to_string(i)}, - }; - } - ggml_backend_cuda_buffer_type_initialized = true; - } - - return &ggml_backend_cuda_buffer_types[device]; -} - -// cuda split buffer - -struct ggml_backend_cuda_split_buffer_context { - ~ggml_backend_cuda_split_buffer_context() { - for (ggml_tensor_extra_gpu * extra : tensor_extras) { - for (int id = 0; id < g_device_count; ++id) { - for (int64_t is = 0; is < MAX_STREAMS; ++is) { - if (extra->events[id][is] != nullptr) { - CUDA_CHECK(cudaEventDestroy(extra->events[id][is])); - } - } - if (extra->data_device[id] != nullptr) { - CUDA_CHECK(cudaFree(extra->data_device[id])); - } - } - delete extra; - } - } - - std::vector tensor_extras; -}; - -GGML_CALL static const char * ggml_backend_cuda_split_buffer_get_name(ggml_backend_buffer_t buffer) { - return GGML_CUDA_NAME "_Split"; - - UNUSED(buffer); -} - -static bool ggml_backend_buffer_is_cuda_split(ggml_backend_buffer_t buffer) { - return buffer->iface.get_name == ggml_backend_cuda_split_buffer_get_name; - UNUSED(ggml_backend_buffer_is_cuda_split); // only used in debug builds currently, avoid unused function warning in release builds -} - -GGML_CALL static void ggml_backend_cuda_split_buffer_free_buffer(ggml_backend_buffer_t buffer) { - ggml_backend_cuda_split_buffer_context * ctx = (ggml_backend_cuda_split_buffer_context *)buffer->context; - delete ctx; -} - -GGML_CALL static void * ggml_backend_cuda_split_buffer_get_base(ggml_backend_buffer_t buffer) { - // the pointers are stored in the tensor extras, this is just a dummy address and never dereferenced - return (void *)0x1000; - - UNUSED(buffer); -} - -GGML_CALL static void ggml_backend_cuda_split_buffer_init_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor) { - GGML_ASSERT(tensor->view_src == nullptr); // views of split tensors are not supported - - ggml_backend_cuda_split_buffer_context * ctx = (ggml_backend_cuda_split_buffer_context *)buffer->context; - ggml_backend_cuda_split_buffer_type_context * buft_ctx = (ggml_backend_cuda_split_buffer_type_context *)buffer->buft->context; - - const int64_t ne0 = tensor->ne[0]; - - ggml_tensor_extra_gpu * extra = new ggml_tensor_extra_gpu{}; - - ctx->tensor_extras.push_back(extra); - - for (int id = 0; id < g_device_count; ++id) { - int64_t row_low, row_high; - get_row_split(&row_low, &row_high, tensor, buft_ctx->tensor_split, id); - - int64_t nrows_split = row_high - row_low; - if (nrows_split == 0) { - continue; - } - - size_t size = ggml_nbytes_split(tensor, nrows_split); - const size_t original_size = size; - - // pad last row to a multiple of 512 elements to avoid out-of-bounds memory accesses - if (ne0 % MATRIX_ROW_PADDING != 0) { - size += ggml_row_size(tensor->type, MATRIX_ROW_PADDING - ne0 % MATRIX_ROW_PADDING); - } - - // FIXME: do not crash if cudaMalloc fails - // currently, init_tensor cannot fail, it needs to be fixed in ggml-backend first - ggml_cuda_set_device(id); - char * buf; - CUDA_CHECK(cudaMalloc(&buf, size)); - - // set padding to 0 to avoid possible NaN values - if (size > original_size) { - CUDA_CHECK(cudaMemset(buf + original_size, 0, size - original_size)); - } - - extra->data_device[id] = buf; - - for (int64_t is = 0; is < MAX_STREAMS; ++is) { - CUDA_CHECK(cudaEventCreateWithFlags(&extra->events[id][is], cudaEventDisableTiming)); - } - } - tensor->backend = GGML_BACKEND_TYPE_GPU_SPLIT; - tensor->extra = extra; -} - -GGML_CALL static void ggml_backend_cuda_split_buffer_set_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor, const void * data, size_t offset, size_t size) { - // split tensors must always be set in their entirety at once - GGML_ASSERT(offset == 0); - GGML_ASSERT(size == ggml_nbytes(tensor)); - - ggml_backend_cuda_split_buffer_type_context * buft_ctx = (ggml_backend_cuda_split_buffer_type_context *)buffer->buft->context; - - const int64_t ne0 = tensor->ne[0]; - const size_t nb1 = tensor->nb[1]; - ggml_tensor_extra_gpu * extra = (ggml_tensor_extra_gpu *)tensor->extra; - - for (int id = 0; id < g_device_count; ++id) { - int64_t row_low, row_high; - get_row_split(&row_low, &row_high, tensor, buft_ctx->tensor_split, id); - - int64_t nrows_split = row_high - row_low; - if (nrows_split == 0) { - continue; - } - - const size_t offset_split = row_low*nb1; - size_t size = ggml_nbytes_split(tensor, nrows_split); - const size_t original_size = size; - - // pad last row to a multiple of 512 elements to avoid out-of-bounds memory accesses - if (ne0 % MATRIX_ROW_PADDING != 0) { - size += ggml_row_size(tensor->type, MATRIX_ROW_PADDING - ne0 % MATRIX_ROW_PADDING); - } - - const char * buf_host = (const char *)data + offset_split; - CUDA_CHECK(cudaMemcpyAsync(extra->data_device[id], buf_host, original_size, cudaMemcpyHostToDevice, cudaStreamPerThread)); - } - - for (int id = 0; id < g_device_count; ++id) { - CUDA_CHECK(cudaStreamSynchronize(cudaStreamPerThread)); - } -} - -GGML_CALL static void ggml_backend_cuda_split_buffer_get_tensor(ggml_backend_buffer_t buffer, const ggml_tensor * tensor, void * data, size_t offset, size_t size) { - // split tensors must always be set in their entirety at once - GGML_ASSERT(offset == 0); - GGML_ASSERT(size == ggml_nbytes(tensor)); - - ggml_backend_cuda_split_buffer_type_context * buft_ctx = (ggml_backend_cuda_split_buffer_type_context *)buffer->buft->context; - - const int64_t ne0 = tensor->ne[0]; - const size_t nb1 = tensor->nb[1]; - ggml_tensor_extra_gpu * extra = (ggml_tensor_extra_gpu *)tensor->extra; - - for (int id = 0; id < g_device_count; ++id) { - int64_t row_low, row_high; - get_row_split(&row_low, &row_high, tensor, buft_ctx->tensor_split, id); - - int64_t nrows_split = row_high - row_low; - if (nrows_split == 0) { - continue; - } - - const size_t offset_split = row_low*nb1; - size_t size = ggml_nbytes_split(tensor, nrows_split); - const size_t original_size = size; - - // pad last row to a multiple of 512 elements to avoid out-of-bounds memory accesses - if (ne0 % MATRIX_ROW_PADDING != 0) { - size += ggml_row_size(tensor->type, MATRIX_ROW_PADDING - ne0 % MATRIX_ROW_PADDING); - } - - char * buf_host = (char *)data + offset_split; - CUDA_CHECK(cudaMemcpyAsync(buf_host, extra->data_device[id], original_size, cudaMemcpyDeviceToHost, cudaStreamPerThread)); - } - - for (int id = 0; id < g_device_count; ++id) { - CUDA_CHECK(cudaStreamSynchronize(cudaStreamPerThread)); - } -} - -GGML_CALL static void ggml_backend_cuda_split_buffer_clear(ggml_backend_buffer_t buffer, uint8_t value) { - UNUSED(buffer); - UNUSED(value); -} - -static struct ggml_backend_buffer_i ggml_backend_cuda_split_buffer_interface = { - /* .get_name = */ ggml_backend_cuda_split_buffer_get_name, - /* .free_buffer = */ ggml_backend_cuda_split_buffer_free_buffer, - /* .get_base = */ ggml_backend_cuda_split_buffer_get_base, - /* .init_tensor = */ ggml_backend_cuda_split_buffer_init_tensor, - /* .set_tensor = */ ggml_backend_cuda_split_buffer_set_tensor, - /* .get_tensor = */ ggml_backend_cuda_split_buffer_get_tensor, - /* .cpy_tensor = */ NULL, - /* .clear = */ ggml_backend_cuda_split_buffer_clear, - /* .reset = */ NULL, -}; - -// cuda split buffer type - -GGML_CALL static const char * ggml_backend_cuda_split_buffer_type_name(ggml_backend_buffer_type_t buft) { - return GGML_CUDA_NAME "_Split"; - - UNUSED(buft); -} - -GGML_CALL static ggml_backend_buffer_t ggml_backend_cuda_split_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size) { - // since we don't know the exact split after rounding, we cannot allocate the device buffers at this point - // instead, we allocate them for each tensor separately in init_tensor - // however, the size still represents the maximum cumulative size of all the device buffers after the tensors are allocated, - // as returned by get_alloc_size. this limit is enforced during tensor allocation by ggml-alloc, so it must be correct. - ggml_backend_cuda_split_buffer_context * ctx = new ggml_backend_cuda_split_buffer_context(); - - return ggml_backend_buffer_init(buft, ggml_backend_cuda_split_buffer_interface, ctx, size); -} - -GGML_CALL static size_t ggml_backend_cuda_split_buffer_type_get_alignment(ggml_backend_buffer_type_t buft) { - return 128; - - UNUSED(buft); -} - -GGML_CALL static size_t ggml_backend_cuda_split_buffer_type_get_alloc_size(ggml_backend_buffer_type_t buft, const ggml_tensor * tensor) { - ggml_backend_cuda_split_buffer_type_context * ctx = (ggml_backend_cuda_split_buffer_type_context *)buft->context; - - size_t total_size = 0; - - const int64_t ne0 = tensor->ne[0]; - - for (int id = 0; id < g_device_count; ++id) { - int64_t row_low, row_high; - get_row_split(&row_low, &row_high, tensor, ctx->tensor_split, id); - - int64_t nrows_split = row_high - row_low; - if (nrows_split == 0) { - continue; - } - - total_size += ggml_nbytes_split(tensor, nrows_split); - - // pad last row to a multiple of 512 elements to avoid out-of-bounds memory accesses - if (ne0 % MATRIX_ROW_PADDING != 0) { - total_size += ggml_row_size(tensor->type, MATRIX_ROW_PADDING - ne0 % MATRIX_ROW_PADDING); - } - } - - return total_size; -} - -GGML_CALL static bool ggml_backend_cuda_split_buffer_type_supports_backend(ggml_backend_buffer_type_t buft, ggml_backend_t backend) { - return ggml_backend_is_cuda(backend); - - UNUSED(buft); -} - -GGML_CALL static bool ggml_backend_cuda_split_buffer_type_is_host(ggml_backend_buffer_type_t buft) { - return false; - - UNUSED(buft); -} - -static ggml_backend_buffer_type_i ggml_backend_cuda_split_buffer_type_interface = { - /* .get_name = */ ggml_backend_cuda_split_buffer_type_name, - /* .alloc_buffer = */ ggml_backend_cuda_split_buffer_type_alloc_buffer, - /* .get_alignment = */ ggml_backend_cuda_split_buffer_type_get_alignment, - /* .get_max_size = */ NULL, // defaults to SIZE_MAX - /* .get_alloc_size = */ ggml_backend_cuda_split_buffer_type_get_alloc_size, - /* .supports_backend = */ ggml_backend_cuda_split_buffer_type_supports_backend, - /* .is_host = */ ggml_backend_cuda_split_buffer_type_is_host, -}; - -GGML_CALL ggml_backend_buffer_type_t ggml_backend_cuda_split_buffer_type(const float * tensor_split) { - ggml_init_cublas(); - - // FIXME: this is not thread safe - static std::map, struct ggml_backend_buffer_type> buft_map; - - std::array tensor_split_arr = {}; - - bool all_zero = tensor_split == nullptr || std::all_of(tensor_split, tensor_split + GGML_CUDA_MAX_DEVICES, [](float x) { return x == 0.0f; }); - if (all_zero) { - tensor_split_arr = g_default_tensor_split; - } else { - float split_sum = 0.0f; - for (int i = 0; i < g_device_count; ++i) { - tensor_split_arr[i] = split_sum; - split_sum += tensor_split[i]; - } - for (int i = 0; i < g_device_count; ++i) { - tensor_split_arr[i] /= split_sum; - } - } - - auto it = buft_map.find(tensor_split_arr); - if (it != buft_map.end()) { - return &it->second; - } - - struct ggml_backend_buffer_type buft { - /* .iface = */ ggml_backend_cuda_split_buffer_type_interface, - /* .context = */ new ggml_backend_cuda_split_buffer_type_context{tensor_split_arr}, - }; - - auto result = buft_map.emplace(tensor_split_arr, buft); - return &result.first->second; -} - -// host buffer type - -GGML_CALL static const char * ggml_backend_cuda_host_buffer_type_name(ggml_backend_buffer_type_t buft) { - return GGML_CUDA_NAME "_Host"; - - UNUSED(buft); -} - -GGML_CALL static const char * ggml_backend_cuda_host_buffer_name(ggml_backend_buffer_t buffer) { - return GGML_CUDA_NAME "_Host"; - - UNUSED(buffer); -} - -GGML_CALL static void ggml_backend_cuda_host_buffer_free_buffer(ggml_backend_buffer_t buffer) { - ggml_cuda_host_free(buffer->context); -} - -GGML_CALL static ggml_backend_buffer_t ggml_backend_cuda_host_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size) { - void * ptr = ggml_cuda_host_malloc(size); - - if (ptr == nullptr) { - // fallback to cpu buffer - return ggml_backend_buft_alloc_buffer(ggml_backend_cpu_buffer_type(), size); - } - - ggml_backend_buffer_t buffer = ggml_backend_cpu_buffer_from_ptr(ptr, size); - buffer->buft = buft; - buffer->iface.get_name = ggml_backend_cuda_host_buffer_name; - buffer->iface.free_buffer = ggml_backend_cuda_host_buffer_free_buffer; - - return buffer; -} - -GGML_CALL ggml_backend_buffer_type_t ggml_backend_cuda_host_buffer_type() { - static struct ggml_backend_buffer_type ggml_backend_cuda_buffer_type_host = { - /* .iface = */ { - /* .get_name = */ ggml_backend_cuda_host_buffer_type_name, - /* .alloc_buffer = */ ggml_backend_cuda_host_buffer_type_alloc_buffer, - /* .get_alignment = */ ggml_backend_cpu_buffer_type()->iface.get_alignment, - /* .get_max_size = */ NULL, // defaults to SIZE_MAX - /* .get_alloc_size = */ ggml_backend_cpu_buffer_type()->iface.get_alloc_size, - /* .supports_backend = */ ggml_backend_cpu_buffer_type()->iface.supports_backend, - /* .is_host = */ ggml_backend_cpu_buffer_type()->iface.is_host, - }, - /* .context = */ nullptr, - }; - - return &ggml_backend_cuda_buffer_type_host; -} - -//static bool ggml_backend_buffer_is_cuda_host(ggml_backend_buffer_t buffer) { -// return buffer->buft->iface.get_name == ggml_backend_cuda_host_buffer_type_name; -//} // backend @@ -11139,9 +11133,8 @@ GGML_CALL static void ggml_backend_cuda_set_tensor_async(ggml_backend_t backend, ggml_backend_buffer_t buf = tensor->view_src ? tensor->view_src->buffer : tensor->buffer; GGML_ASSERT(buf->buft == ggml_backend_cuda_buffer_type(cuda_ctx->device) && "unsupported buffer type"); - GGML_ASSERT(tensor->backend == GGML_BACKEND_TYPE_GPU); - CUDA_CHECK(cudaMemcpyAsync((char *)tensor->data + offset, data, size, cudaMemcpyHostToDevice, g_cudaStreams[cuda_ctx->device][0])); + CUDA_CHECK(cudaMemcpyAsync((char *)tensor->data + offset, data, size, cudaMemcpyHostToDevice, cuda_ctx->stream())); } GGML_CALL static void ggml_backend_cuda_get_tensor_async(ggml_backend_t backend, const ggml_tensor * tensor, void * data, size_t offset, size_t size) { @@ -11149,9 +11142,8 @@ GGML_CALL static void ggml_backend_cuda_get_tensor_async(ggml_backend_t backend, ggml_backend_buffer_t buf = tensor->view_src ? tensor->view_src->buffer : tensor->buffer; GGML_ASSERT(buf->buft == ggml_backend_cuda_buffer_type(cuda_ctx->device) && "unsupported buffer type"); - GGML_ASSERT(tensor->backend == GGML_BACKEND_TYPE_GPU); - CUDA_CHECK(cudaMemcpyAsync(data, (const char *)tensor->data + offset, size, cudaMemcpyDeviceToHost, g_cudaStreams[cuda_ctx->device][0])); + CUDA_CHECK(cudaMemcpyAsync(data, (const char *)tensor->data + offset, size, cudaMemcpyDeviceToHost, cuda_ctx->stream())); } GGML_CALL static bool ggml_backend_cuda_cpy_tensor_async(ggml_backend_t backend_src, ggml_backend_t backend_dst, const ggml_tensor * src, ggml_tensor * dst) { @@ -11186,19 +11178,19 @@ GGML_CALL static bool ggml_backend_cuda_cpy_tensor_async(ggml_backend_t backend_ // copy on src stream if (cuda_ctx_src->device == cuda_ctx_dst->device) { - CUDA_CHECK(cudaMemcpyAsync(dst->data, src->data, ggml_nbytes(dst), cudaMemcpyDeviceToDevice, g_cudaStreams[cuda_ctx_dst->device][0])); + CUDA_CHECK(cudaMemcpyAsync(dst->data, src->data, ggml_nbytes(dst), cudaMemcpyDeviceToDevice, cuda_ctx_dst->stream())); } else { - CUDA_CHECK(cudaMemcpyPeerAsync(dst->data, cuda_ctx_dst->device, src->data, cuda_ctx_src->device, ggml_nbytes(dst), g_cudaStreams[cuda_ctx_src->device][0])); + CUDA_CHECK(cudaMemcpyPeerAsync(dst->data, cuda_ctx_dst->device, src->data, cuda_ctx_src->device, ggml_nbytes(dst), cuda_ctx_src->stream())); } // record event on src stream - CUDA_CHECK(cudaEventRecord(cuda_ctx_src->copy_event, g_cudaStreams[cuda_ctx_src->device][0])); + CUDA_CHECK(cudaEventRecord(cuda_ctx_src->copy_event, cuda_ctx_src->stream())); // wait on dst stream for the copy to complete - CUDA_CHECK(cudaStreamWaitEvent(g_cudaStreams[cuda_ctx_dst->device][0], cuda_ctx_src->copy_event, 0)); + CUDA_CHECK(cudaStreamWaitEvent(cuda_ctx_dst->stream(), cuda_ctx_src->copy_event, 0)); } else { // src and dst are on the same backend - CUDA_CHECK(cudaMemcpyAsync(dst->data, src->data, ggml_nbytes(dst), cudaMemcpyDeviceToDevice, g_cudaStreams[cuda_ctx_dst->device][0])); + CUDA_CHECK(cudaMemcpyAsync(dst->data, src->data, ggml_nbytes(dst), cudaMemcpyDeviceToDevice, cuda_ctx_dst->stream())); } return true; } @@ -11206,15 +11198,15 @@ GGML_CALL static bool ggml_backend_cuda_cpy_tensor_async(ggml_backend_t backend_ GGML_CALL static void ggml_backend_cuda_synchronize(ggml_backend_t backend) { ggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *)backend->context; - CUDA_CHECK(cudaStreamSynchronize(g_cudaStreams[cuda_ctx->device][0])); + CUDA_CHECK(cudaStreamSynchronize(cuda_ctx->stream())); - UNUSED(backend); + GGML_UNUSED(backend); } GGML_CALL static enum ggml_status ggml_backend_cuda_graph_compute(ggml_backend_t backend, ggml_cgraph * cgraph) { ggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *)backend->context; - ggml_cuda_set_main_device(cuda_ctx->device); + ggml_cuda_set_device(cuda_ctx->device); for (int i = 0; i < cgraph->n_nodes; i++) { ggml_tensor * node = cgraph->nodes[i]; @@ -11224,20 +11216,15 @@ GGML_CALL static enum ggml_status ggml_backend_cuda_graph_compute(ggml_backend_t } #ifndef NDEBUG - assert(node->backend == GGML_BACKEND_TYPE_GPU || node->backend == GGML_BACKEND_TYPE_GPU_SPLIT); assert(node->buffer->buft == ggml_backend_cuda_buffer_type(cuda_ctx->device)); - assert(node->extra != nullptr); - for (int j = 0; j < GGML_MAX_SRC; j++) { if (node->src[j] != nullptr) { - assert(node->src[j]->backend == GGML_BACKEND_TYPE_GPU || node->src[j]->backend == GGML_BACKEND_TYPE_GPU_SPLIT); assert(node->src[j]->buffer->buft == ggml_backend_cuda_buffer_type(cuda_ctx->device) || ggml_backend_buffer_is_cuda_split(node->src[j]->buffer)); - assert(node->src[j]->extra != nullptr); } } #endif - bool ok = ggml_cuda_compute_forward(node); + bool ok = ggml_cuda_compute_forward(*cuda_ctx, node); if (!ok) { fprintf(stderr, "%s: error: op not supported %s (%s)\n", __func__, node->name, ggml_op_name(node->op)); } @@ -11371,7 +11358,7 @@ GGML_CALL static bool ggml_backend_cuda_supports_op(ggml_backend_t backend, cons return false; } - UNUSED(backend); + GGML_UNUSED(backend); } GGML_CALL static bool ggml_backend_cuda_offload_op(ggml_backend_t backend, const ggml_tensor * op) { @@ -11379,7 +11366,7 @@ GGML_CALL static bool ggml_backend_cuda_offload_op(ggml_backend_t backend, const return op->ne[1] >= min_batch_size && op->op != GGML_OP_GET_ROWS; - UNUSED(backend); + GGML_UNUSED(backend); } static ggml_backend_event_t ggml_backend_cuda_event_new(ggml_backend_t backend) { @@ -11405,14 +11392,14 @@ static void ggml_backend_cuda_event_free(ggml_backend_event_t event) { static void ggml_backend_cuda_event_record(ggml_backend_event_t event) { ggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *)event->backend->context; - CUDA_CHECK(cudaEventRecord((cudaEvent_t)event->context, g_cudaStreams[cuda_ctx->device][0])); + CUDA_CHECK(cudaEventRecord((cudaEvent_t)event->context, cuda_ctx->stream())); } static void ggml_backend_cuda_event_wait(ggml_backend_t backend, ggml_backend_event_t event) { ggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *)backend->context; if (ggml_backend_is_cuda(event->backend)) { - CUDA_CHECK(cudaStreamWaitEvent(g_cudaStreams[cuda_ctx->device][0], (cudaEvent_t)event->context, 0)); + CUDA_CHECK(cudaStreamWaitEvent(cuda_ctx->stream(), (cudaEvent_t)event->context, 0)); } else { #if 0 // untested @@ -11421,7 +11408,7 @@ static void ggml_backend_cuda_event_wait(ggml_backend_t backend, ggml_backend_ev ggml_backend_event_synchronize(event); }; - CUDA_CHECK(cudaLaunchHostFunc(g_cudaStreams[cuda_ctx->device][0], wait_fn, event)); + CUDA_CHECK(cudaLaunchHostFunc(cuda_ctx->stream(), wait_fn, event)); #endif GGML_ASSERT(false); } @@ -11458,16 +11445,11 @@ static ggml_guid_t ggml_backend_cuda_guid() { } GGML_CALL ggml_backend_t ggml_backend_cuda_init(int device) { - ggml_init_cublas(); - - if (device < 0 || device >= ggml_cuda_get_device_count()) { + if (device < 0 || device >= ggml_backend_cuda_get_device_count()) { fprintf(stderr, "%s: error: invalid device %d\n", __func__, device); return nullptr; } - // not strictly necessary, but it may reduce the overhead of the first graph_compute - ggml_cuda_set_main_device(device); - ggml_backend_cuda_context * ctx = new ggml_backend_cuda_context(device); if (ctx == nullptr) { fprintf(stderr, "%s: error: failed to allocate context\n", __func__); @@ -11488,11 +11470,13 @@ GGML_CALL bool ggml_backend_is_cuda(ggml_backend_t backend) { } GGML_CALL int ggml_backend_cuda_get_device_count() { - return ggml_cuda_get_device_count(); + return get_cuda_global_info().device_count; } GGML_CALL void ggml_backend_cuda_get_device_description(int device, char * description, size_t description_size) { - ggml_cuda_get_device_description(device, description, description_size); + cudaDeviceProp prop; + CUDA_CHECK(cudaGetDeviceProperties(&prop, device)); + snprintf(description, description_size, "%s", prop.name); } GGML_CALL void ggml_backend_cuda_get_device_memory(int device, size_t * free, size_t * total) { @@ -11531,13 +11515,13 @@ GGML_CALL static ggml_backend_t ggml_backend_reg_cuda_init(const char * params, ggml_backend_t cuda_backend = ggml_backend_cuda_init((int) (intptr_t) user_data); return cuda_backend; - UNUSED(params); + GGML_UNUSED(params); } extern "C" GGML_CALL int ggml_backend_cuda_reg_devices(); GGML_CALL int ggml_backend_cuda_reg_devices() { - int device_count = ggml_cuda_get_device_count(); + int device_count = ggml_backend_cuda_get_device_count(); //int device_count = 1; // DEBUG: some tools require delaying CUDA initialization for (int i = 0; i < device_count; i++) { char name[128]; From 272935b281fee5c683e3d6d1eb580b84553cf503 Mon Sep 17 00:00:00 2001 From: Ziang Wu <97337387+ZiangWu-77@users.noreply.github.com> Date: Wed, 20 Mar 2024 23:02:32 +0800 Subject: [PATCH 672/812] llava : add MobileVLM_V2 backup (#6175) * Add MobileVLM_V2 backup * Update MobileVLM-README.md * Update examples/llava/MobileVLM-README.md Co-authored-by: Georgi Gerganov * Update examples/llava/convert-image-encoder-to-gguf.py Co-authored-by: Georgi Gerganov * clip : fix whitespace * fix deifinition mistake in clip.cpp --------- Co-authored-by: Georgi Gerganov --- examples/llava/MobileVLM-README.md | 14 +++++- examples/llava/clip.cpp | 50 ++++++++++++++++++- .../llava/convert-image-encoder-to-gguf.py | 9 ++-- 3 files changed, 67 insertions(+), 6 deletions(-) diff --git a/examples/llava/MobileVLM-README.md b/examples/llava/MobileVLM-README.md index 9eba791da..c1f361d17 100644 --- a/examples/llava/MobileVLM-README.md +++ b/examples/llava/MobileVLM-README.md @@ -1,11 +1,13 @@ # MobileVLM -Currently this implementation supports [MobileVLM-v1.7](https://huggingface.co/mtgv/MobileVLM-1.7B) variants. +Currently this implementation supports [MobileVLM-1.7B](https://huggingface.co/mtgv/MobileVLM-1.7B) / [MobileVLM_V2-1.7B](https://huggingface.co/mtgv/MobileVLM_V2-1.7B) variants. for more information, please go to [Meituan-AutoML/MobileVLM](https://github.com/Meituan-AutoML/MobileVLM) The implementation is based on llava, and is compatible with llava and mobileVLM. The usage is basically same as llava. +Notice: The overall process of model inference for both **MobilVLM** and **MobilVLM_V2** models is the same, but the process of model conversion is a little different. Therefore, using MobiVLM as an example, the different conversion step will be shown. + ## Usage Build with cmake or run `make llava-cli` to build it. @@ -34,7 +36,7 @@ git clone https://huggingface.co/openai/clip-vit-large-patch14-336 python ./examples/llava/llava-surgery.py -m path/to/MobileVLM-1.7B ``` -3. Use `convert-image-encoder-to-gguf.py` with `--projector-type ldp` to convert the LLaVA image encoder to GGUF: +3. Use `convert-image-encoder-to-gguf.py` with `--projector-type ldp` (for **V2** the arg is `--projector-type ldpv2`) to convert the LLaVA image encoder to GGUF: ```sh python ./examples/llava/convert-image-encoder-to-gguf \ @@ -44,6 +46,14 @@ python ./examples/llava/convert-image-encoder-to-gguf \ --projector-type ldp ``` +```sh +python ./examples/llava/convert-image-encoder-to-gguf \ + -m path/to/clip-vit-large-patch14-336 \ + --llava-projector path/to/MobileVLM-1.7B_V2/llava.projector \ + --output-dir path/to/MobileVLM-1.7B_V2 \ + --projector-type ldpv2 +``` + 4. Use `convert.py` to convert the LLaMA part of LLaVA to GGUF: ```sh diff --git a/examples/llava/clip.cpp b/examples/llava/clip.cpp index 690bca2eb..48caafa87 100644 --- a/examples/llava/clip.cpp +++ b/examples/llava/clip.cpp @@ -119,6 +119,7 @@ static std::string format(const char * fmt, ...) { #define TN_LLAVA_PROJ "mm.%d.%s" #define TN_MVLM_PROJ_MLP "mm.model.mlp.%d.%s" #define TN_MVLM_PROJ_BLOCK "mm.model.mb_block.%d.block.%d.%s" +#define TN_MVLM_PROJ_PEG "mm.model.peg.%d.%s" #define TN_IMAGE_NEWLINE "model.image_newline" @@ -126,12 +127,14 @@ enum projector_type { PROJECTOR_TYPE_MLP, PROJECTOR_TYPE_MLP_NORM, PROJECTOR_TYPE_LDP, + PROJECTOR_TYPE_LDPV2, PROJECTOR_TYPE_UNKNOWN, }; static std::map PROJECTOR_TYPE_NAMES = { { PROJECTOR_TYPE_MLP, "mlp" }, { PROJECTOR_TYPE_LDP, "ldp" }, + { PROJECTOR_TYPE_LDPV2, "ldpv2"}, }; @@ -475,6 +478,14 @@ struct clip_vision_model { struct ggml_tensor * mm_model_block_2_block_2_0_w; struct ggml_tensor * mm_model_block_2_block_2_1_w; struct ggml_tensor * mm_model_block_2_block_2_1_b; + + // MobileVLM_V2 projection + struct ggml_tensor * mm_model_mlp_0_w; + struct ggml_tensor * mm_model_mlp_0_b; + struct ggml_tensor * mm_model_mlp_2_w; + struct ggml_tensor * mm_model_mlp_2_b; + struct ggml_tensor * mm_model_peg_0_w; + struct ggml_tensor * mm_model_peg_0_b; }; struct clip_ctx { @@ -807,6 +818,29 @@ static ggml_cgraph * clip_image_build_graph(clip_ctx * ctx, const clip_image_f32 } embeddings = block_1; } + else if (ctx->proj_type == PROJECTOR_TYPE_LDPV2) + { + int n_patch = 24; + struct ggml_tensor * mlp_0 = ggml_mul_mat(ctx0, model.mm_model_mlp_0_w, embeddings); + mlp_0 = ggml_add(ctx0, mlp_0, model.mm_model_mlp_0_b); + mlp_0 = ggml_gelu(ctx0, mlp_0); + struct ggml_tensor * mlp_2 = ggml_mul_mat(ctx0, model.mm_model_mlp_2_w, mlp_0); + mlp_2 = ggml_add(ctx0, mlp_2, model.mm_model_mlp_2_b); + // mlp_2 ne = [2048, 576, 1, 1] + // // AVG Pool Layer 2*2, strides = 2 + mlp_2 = ggml_cont(ctx0, ggml_permute(ctx0, mlp_2, 1, 0, 2, 3)); + // mlp_2 ne = [576, 2048, 1, 1] + mlp_2 = ggml_reshape_4d(ctx0, mlp_2, n_patch, n_patch, mlp_2->ne[1], mlp_2->ne[2]); + // mlp_2 ne [24, 24, 2048, 1] + mlp_2 = ggml_pool_2d(ctx0, mlp_2, GGML_OP_POOL_AVG, 2, 2, 2, 2, 0, 0); + // weight ne = [3, 3, 2048, 1] + struct ggml_tensor * peg_0 = ggml_conv_depthwise_2d(ctx0, model.mm_model_peg_0_w, mlp_2, 1, 1, 1, 1, 1, 1); + peg_0 = ggml_add(ctx0, peg_0, mlp_2); + peg_0 = ggml_cont(ctx0, ggml_permute(ctx0, peg_0, 1, 2, 0, 3)); + peg_0 = ggml_add(ctx0, peg_0, model.mm_model_peg_0_b); + peg_0 = ggml_reshape_3d(ctx0, peg_0, peg_0->ne[0], peg_0->ne[1] * peg_0->ne[2], peg_0->ne[3]); + embeddings = peg_0; + } else { GGML_ASSERT(false); } @@ -1177,7 +1211,18 @@ struct clip_ctx * clip_model_load(const char * fname, const int verbosity = 1) { vision_model.mm_model_block_2_block_2_0_w = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_BLOCK, 2, 2, "0.weight")); vision_model.mm_model_block_2_block_2_1_w = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_BLOCK, 2, 2, "1.weight")); vision_model.mm_model_block_2_block_2_1_b = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_BLOCK, 2, 2, "1.bias")); - } else { + } + else if (new_clip->proj_type == PROJECTOR_TYPE_LDPV2) + { + // MobilVLM_V2 projection + vision_model.mm_model_mlp_0_w = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_MLP, 0, "weight")); + vision_model.mm_model_mlp_0_b = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_MLP, 0, "bias")); + vision_model.mm_model_mlp_2_w = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_MLP, 2, "weight")); + vision_model.mm_model_mlp_2_b = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_MLP, 2, "bias")); + vision_model.mm_model_peg_0_w = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_PEG, 0, "weight")); + vision_model.mm_model_peg_0_b = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_PEG, 0, "bias")); + } + else { std::string proj_type = PROJECTOR_TYPE_NAMES[new_clip->proj_type]; throw std::runtime_error(format("%s: don't support projector with: %s currently\n", __func__, proj_type.c_str())); } @@ -1966,6 +2011,9 @@ int clip_n_mmproj_embd(const struct clip_ctx * ctx) { if (ctx->proj_type == PROJECTOR_TYPE_LDP) { return ctx->vision_model.mm_model_block_1_block_2_1_b->ne[0]; } + if (ctx->proj_type == PROJECTOR_TYPE_LDPV2) { + return ctx->vision_model.mm_model_peg_0_b->ne[0]; + } if (ctx->proj_type == PROJECTOR_TYPE_MLP) { return ctx->vision_model.mm_2_b->ne[0]; } diff --git a/examples/llava/convert-image-encoder-to-gguf.py b/examples/llava/convert-image-encoder-to-gguf.py index c69f89ac2..b00bf7c6d 100644 --- a/examples/llava/convert-image-encoder-to-gguf.py +++ b/examples/llava/convert-image-encoder-to-gguf.py @@ -1,6 +1,7 @@ import argparse import os import json +import re import torch import numpy as np @@ -38,9 +39,11 @@ def should_skip_tensor(name: str, has_text: bool, has_vision: bool, has_llava: b def get_tensor_name(name: str) -> str: if "projection" in name: return name - if "mm_projector" in name: - return name.replace("model.mm_projector", "mm") + name = name.replace("model.mm_projector", "mm") + name = re.sub(r'mm\.mlp\.mlp', 'mm.model.mlp', name, count=1) + name = re.sub(r'mm\.peg\.peg', 'mm.model.peg', name, count=1) + return name return name.replace("text_model", "t").replace("vision_model", "v").replace("encoder.layers", "blk").replace("embeddings.", "").replace("_proj", "").replace("self_attn.", "attn_").replace("layer_norm", "ln").replace("layernorm", "ln").replace("mlp.fc1", "ffn_down").replace("mlp.fc2", "ffn_up").replace("embedding", "embd").replace("final", "post").replace("layrnorm", "ln") @@ -83,7 +86,7 @@ ap.add_argument("--clip-model-is-vision", action="store_true", required=False, ap.add_argument("--clip-model-is-openclip", action="store_true", required=False, help="The clip model is from openclip (for ViT-SO400M type))") ap.add_argument("--llava-projector", help="Path to llava.projector file. If specified, save an image encoder for LLaVA models.") -ap.add_argument("--projector-type", help="Type of projector. Possible values: mlp, ldp", choices=["mlp", "ldp"], default="mlp") +ap.add_argument("--projector-type", help="Type of projector. Possible values: mlp, ldp, ldpv2", choices=["mlp", "ldp", "ldpv2"], default="mlp") ap.add_argument("-o", "--output-dir", help="Directory to save GGUF files. Default is the original model directory", default=None) # Example --image_mean 0.48145466 0.4578275 0.40821073 --image_std 0.26862954 0.26130258 0.27577711 # Example --image_mean 0.5 0.5 0.5 --image_std 0.5 0.5 0.5 From f9c7ba34476ffc4f13ae2cdb1aec493a16eb8d47 Mon Sep 17 00:00:00 2001 From: Ziang Wu <97337387+ZiangWu-77@users.noreply.github.com> Date: Wed, 20 Mar 2024 23:29:51 +0800 Subject: [PATCH 673/812] llava : update MobileVLM-README.md (#6180) --- examples/llava/MobileVLM-README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/llava/MobileVLM-README.md b/examples/llava/MobileVLM-README.md index c1f361d17..4d5fef020 100644 --- a/examples/llava/MobileVLM-README.md +++ b/examples/llava/MobileVLM-README.md @@ -6,7 +6,7 @@ for more information, please go to [Meituan-AutoML/MobileVLM](https://github.com The implementation is based on llava, and is compatible with llava and mobileVLM. The usage is basically same as llava. -Notice: The overall process of model inference for both **MobilVLM** and **MobilVLM_V2** models is the same, but the process of model conversion is a little different. Therefore, using MobiVLM as an example, the different conversion step will be shown. +Notice: The overall process of model inference for both **MobileVLM** and **MobileVLM_V2** models is the same, but the process of model conversion is a little different. Therefore, using MobiVLM as an example, the different conversion step will be shown. ## Usage Build with cmake or run `make llava-cli` to build it. From 1c51f98adcbad40e3c41f0a6ffadeb723190b417 Mon Sep 17 00:00:00 2001 From: slaren Date: Wed, 20 Mar 2024 21:03:26 +0100 Subject: [PATCH 674/812] cuda : print the returned error when CUDA initialization fails (#6185) --- ggml-cuda.cu | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ggml-cuda.cu b/ggml-cuda.cu index e70b79c30..24af9f671 100644 --- a/ggml-cuda.cu +++ b/ggml-cuda.cu @@ -294,8 +294,9 @@ static ggml_cuda_device_info ggml_cuda_init() { ggml_cuda_device_info info = {}; - if (cudaGetDeviceCount(&info.device_count) != cudaSuccess) { - fprintf(stderr, "%s: no " GGML_CUDA_NAME " devices found, " GGML_CUDA_NAME " will be disabled\n", __func__); + cudaError_t err = cudaGetDeviceCount(&info.device_count); + if (err != cudaSuccess) { + fprintf(stderr, "%s: failed to initialize " GGML_CUDA_NAME ": %s\n", __func__, cudaGetErrorString(err)); return info; } From 42e21c68826f2e56b9592dccd9f3c43895b6890d Mon Sep 17 00:00:00 2001 From: slaren Date: Thu, 21 Mar 2024 01:47:46 +0100 Subject: [PATCH 675/812] cuda : fix conflict with std::swap (#6186) --- ggml-cuda.cu | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/ggml-cuda.cu b/ggml-cuda.cu index 24af9f671..27816a7be 100644 --- a/ggml-cuda.cu +++ b/ggml-cuda.cu @@ -370,12 +370,6 @@ struct ggml_cuda_pool { virtual void * alloc(size_t size, size_t * actual_size) = 0; virtual void free(void * ptr, size_t size) = 0; - - ggml_cuda_pool() = default; - ggml_cuda_pool(const ggml_cuda_pool &) = delete; - ggml_cuda_pool(ggml_cuda_pool &&) = delete; - ggml_cuda_pool& operator=(const ggml_cuda_pool &) = delete; - ggml_cuda_pool& operator=(ggml_cuda_pool &&) = delete; }; struct ggml_cuda_pool_leg : public ggml_cuda_pool { @@ -6969,7 +6963,7 @@ static __global__ void k_sum_rows_f32(const float * x, float * dst, const int nc } template -static inline __device__ void swap(T & a, T & b) { +static inline __device__ void ggml_cuda_swap(T & a, T & b) { T tmp = a; a = b; b = tmp; @@ -6998,11 +6992,11 @@ static __global__ void k_argsort_f32_i32(const float * x, int * dst, const int n if (ixj > col) { if ((col & k) == 0) { if (order == GGML_SORT_ORDER_ASC ? x_row[dst_row[col]] > x_row[dst_row[ixj]] : x_row[dst_row[col]] < x_row[dst_row[ixj]]) { - swap(dst_row[col], dst_row[ixj]); + ggml_cuda_swap(dst_row[col], dst_row[ixj]); } } else { if (order == GGML_SORT_ORDER_ASC ? x_row[dst_row[col]] < x_row[dst_row[ixj]] : x_row[dst_row[col]] > x_row[dst_row[ixj]]) { - swap(dst_row[col], dst_row[ixj]); + ggml_cuda_swap(dst_row[col], dst_row[ixj]); } } } From c5b8595e3f4f4ed319ef71c9c9d868d1b7a27626 Mon Sep 17 00:00:00 2001 From: AidanBeltonS <87009434+AidanBeltonS@users.noreply.github.com> Date: Thu, 21 Mar 2024 06:10:52 +0000 Subject: [PATCH 676/812] Add nvidia and amd backends (#6157) --- ggml-sycl.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/ggml-sycl.cpp b/ggml-sycl.cpp index d51f23b41..cc9ee0762 100644 --- a/ggml-sycl.cpp +++ b/ggml-sycl.cpp @@ -977,8 +977,10 @@ namespace dpct static int convert_backend_index(std::string & backend) { if (backend == "ext_oneapi_level_zero:gpu") return 0; if (backend == "opencl:gpu") return 1; - if (backend == "opencl:cpu") return 2; - if (backend == "opencl:acc") return 3; + if (backend == "ext_oneapi_cuda:gpu") return 2; + if (backend == "ext_oneapi_hip:gpu") return 3; + if (backend == "opencl:cpu") return 4; + if (backend == "opencl:acc") return 5; printf("convert_backend_index: can't handle backend=%s\n", backend.c_str()); GGML_ASSERT(false); } From 76aa30a26353f597e4fbe3cf776772ae812af89a Mon Sep 17 00:00:00 2001 From: Kawrakow <48489457+ikawrakow@users.noreply.github.com> Date: Thu, 21 Mar 2024 08:27:57 +0100 Subject: [PATCH 677/812] Add ability to use Q5_0, Q5_1, and IQ4_NL for quantized K cache (#6183) * k_cache: be able to use Q5_0 * k_cache: be able to use Q5_1 on CODA * k_cache: be able to use Q5_0 on Metal * k_cache: be able to use Q5_1 on Metal * k_cache: be able to use IQ4_NL - just CUDA for now * k_cache: be able to use IQ4_NL on Metal * k_cache: add newly added supported types to llama-bench and CUDA supports_op --------- Co-authored-by: Iwan Kawrakow --- common/common.cpp | 3 + examples/llama-bench/llama-bench.cpp | 3 + ggml-cuda.cu | 165 ++++++++++++++++++ ggml-metal.m | 28 ++-- ggml-metal.metal | 240 ++++++++++++++++++++++++++- 5 files changed, 424 insertions(+), 15 deletions(-) diff --git a/common/common.cpp b/common/common.cpp index 5f10718ec..192182d05 100644 --- a/common/common.cpp +++ b/common/common.cpp @@ -1590,6 +1590,9 @@ static ggml_type kv_cache_type_from_str(const std::string & s) { if (s == "q4_1") { return GGML_TYPE_Q4_1; } + if (s == "iq4_nl") { + return GGML_TYPE_IQ4_NL; + } if (s == "q5_0") { return GGML_TYPE_Q5_0; } diff --git a/examples/llama-bench/llama-bench.cpp b/examples/llama-bench/llama-bench.cpp index 4cb230804..82413b79d 100644 --- a/examples/llama-bench/llama-bench.cpp +++ b/examples/llama-bench/llama-bench.cpp @@ -249,6 +249,9 @@ static ggml_type ggml_type_from_name(const std::string & s) { if (s == "q5_1") { return GGML_TYPE_Q5_1; } + if (s == "iq4_nl") { + return GGML_TYPE_IQ4_NL; + } return GGML_TYPE_COUNT; } diff --git a/ggml-cuda.cu b/ggml-cuda.cu index 27816a7be..280839ea4 100644 --- a/ggml-cuda.cu +++ b/ggml-cuda.cu @@ -6757,6 +6757,123 @@ static __device__ void cpy_blck_f32_q4_1(const char * cxi, char * cdsti) { } } +static __device__ void cpy_blck_f32_q5_0(const char * cxi, char * cdsti) { + const float * xi = (const float *) cxi; + block_q5_0 * dsti = (block_q5_0 *) cdsti; + + float amax = 0.0f; + float vmax = 0.0f; + + for (int j = 0; j < QK5_0; ++j) { + const float v = xi[j]; + if (amax < fabsf(v)) { + amax = fabsf(v); + vmax = v; + } + } + + const float d = vmax / -16; + const float id = d ? 1.0f/d : 0.0f; + + dsti->d = d; + + uint32_t qh = 0; + for (int j = 0; j < QK5_0/2; ++j) { + const float x0 = xi[0 + j]*id; + const float x1 = xi[QK5_0/2 + j]*id; + + const uint8_t xi0 = min(31, (int8_t)(x0 + 16.5f)); + const uint8_t xi1 = min(31, (int8_t)(x1 + 16.5f)); + + dsti->qs[j] = (xi0 & 0xf) | ((xi1 & 0xf) << 4); + qh |= ((xi0 & 0x10u) >> 4) << (j + 0); + qh |= ((xi1 & 0x10u) >> 4) << (j + QK5_0/2); + } + memcpy(dsti->qh, &qh, sizeof(qh)); +} + +static __device__ void cpy_blck_f32_q5_1(const char * cxi, char * cdsti) { + const float * xi = (const float *) cxi; + block_q5_1 * dsti = (block_q5_1 *) cdsti; + + float min = xi[0]; + float max = xi[0]; + + for (int j = 1; j < QK5_1; ++j) { + const float v = xi[j]; + min = v < min ? v : min; + max = v > max ? v : max; + } + + const float d = (max - min) / 31; + const float id = d ? 1.0f/d : 0.0f; + + dsti->dm.x = d; + dsti->dm.y = min; + + uint32_t qh = 0; + for (int j = 0; j < QK5_1/2; ++j) { + const float x0 = (xi[0 + j] - min)*id; + const float x1 = (xi[QK5_1/2 + j] - min)*id; + + const uint8_t xi0 = (uint8_t)(x0 + 0.5f); + const uint8_t xi1 = (uint8_t)(x1 + 0.5f); + + dsti->qs[j] = (xi0 & 0xf) | ((xi1 & 0xf) << 4); + qh |= ((xi0 & 0x10u) >> 4) << (j + 0); + qh |= ((xi1 & 0x10u) >> 4) << (j + QK5_1/2); + } + memcpy(dsti->qh, &qh, sizeof(qh)); +} + +static __device__ __forceinline__ int best_index_int8(int n, const int8_t * val, float x) { + if (x <= val[0]) return 0; + if (x >= val[n-1]) return n-1; + int ml = 0, mu = n-1; + while (mu-ml > 1) { + int mav = (ml+mu)/2; + if (x < val[mav]) mu = mav; else ml = mav; + } + return x - val[mu-1] < val[mu] - x ? mu-1 : mu; +} + +static __device__ void cpy_blck_f32_iq4_nl(const char * cxi, char * cdsti) { + const float * xi = (const float *) cxi; + block_iq4_nl * dsti = (block_iq4_nl *) cdsti; + + float amax = 0.0f; + float vmax = 0.0f; + + for (int j = 0; j < QK4_NL; ++j) { + const float v = xi[j]; + if (amax < fabsf(v)) { + amax = fabsf(v); + vmax = v; + } + } + + float d = vmax / kvalues_iq4nl[0]; + const float id = d ? 1.0f/d : 0.0f; + + float sumqx = 0, sumq2 = 0; + for (int j = 0; j < QK4_NL/2; ++j) { + const float x0 = xi[0 + j]*id; + const float x1 = xi[QK4_NL/2 + j]*id; + const uint8_t xi0 = best_index_int8(16, kvalues_iq4nl, x0); + const uint8_t xi1 = best_index_int8(16, kvalues_iq4nl, x1); + dsti->qs[j] = xi0 | (xi1 << 4); + const float v0 = kvalues_iq4nl[xi0]; + const float v1 = kvalues_iq4nl[xi1]; + const float w0 = xi[0 + j]*xi[0 + j]; + const float w1 = xi[QK4_NL/2 + j]*xi[QK4_NL/2 + j]; + sumqx += w0*v0*xi[j] + w1*v1*xi[QK4_NL/2 + j]; + sumq2 += w0*v0*v0 + w1*v1*v1; + } + + dsti->d = sumq2 > 0 ? sumqx/sumq2 : d; +} + + template static __global__ void cpy_f32_q(const char * cx, char * cdst, const int ne, const int ne00, const int ne01, const int ne02, const int nb00, const int nb01, const int nb02, @@ -8490,6 +8607,39 @@ static void ggml_cpy_f32_q4_1_cuda( (cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13); } +static void ggml_cpy_f32_q5_0_cuda( + const char * cx, char * cdst, const int ne, + const int ne00, const int ne01, const int ne02, const int nb00, const int nb01, const int nb02, + const int nb03, const int ne10, const int ne11, const int ne12, const int nb10, const int nb11, const int nb12, const int nb13, cudaStream_t stream) { + + GGML_ASSERT(ne % QK5_0 == 0); + const int num_blocks = ne / QK5_0; + cpy_f32_q<<>> + (cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13); +} + +static void ggml_cpy_f32_q5_1_cuda( + const char * cx, char * cdst, const int ne, + const int ne00, const int ne01, const int ne02, const int nb00, const int nb01, const int nb02, + const int nb03, const int ne10, const int ne11, const int ne12, const int nb10, const int nb11, const int nb12, const int nb13, cudaStream_t stream) { + + GGML_ASSERT(ne % QK5_1 == 0); + const int num_blocks = ne / QK5_1; + cpy_f32_q<<>> + (cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13); +} + +static void ggml_cpy_f32_iq4_nl_cuda( + const char * cx, char * cdst, const int ne, + const int ne00, const int ne01, const int ne02, const int nb00, const int nb01, const int nb02, + const int nb03, const int ne10, const int ne11, const int ne12, const int nb10, const int nb11, const int nb12, const int nb13, cudaStream_t stream) { + + GGML_ASSERT(ne % QK4_NL == 0); + const int num_blocks = ne / QK4_NL; + cpy_f32_q<<>> + (cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13); +} + static void ggml_cpy_f16_f16_cuda( const char * cx, char * cdst, const int ne, const int ne00, const int ne01, const int ne02, const int nb00, const int nb01, const int nb02, @@ -10888,6 +11038,12 @@ static void ggml_cuda_cpy(ggml_backend_cuda_context & ctx, const ggml_tensor * s ggml_cpy_f32_q4_0_cuda(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream); } else if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_Q4_1) { ggml_cpy_f32_q4_1_cuda(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream); + } else if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_Q5_0) { + ggml_cpy_f32_q5_0_cuda(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream); + } else if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_IQ4_NL) { + ggml_cpy_f32_iq4_nl_cuda(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream); + } else if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_Q5_1) { + ggml_cpy_f32_q5_1_cuda(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream); } else if (src0->type == GGML_TYPE_F16 && src1->type == GGML_TYPE_F16) { ggml_cpy_f16_f16_cuda (src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream); } else if (src0->type == GGML_TYPE_F16 && src1->type == GGML_TYPE_F32) { @@ -11304,6 +11460,15 @@ GGML_CALL static bool ggml_backend_cuda_supports_op(ggml_backend_t backend, cons if (src0_type == GGML_TYPE_F32 && src1_type == GGML_TYPE_Q4_1) { return true; } + if (src0_type == GGML_TYPE_F32 && src1_type == GGML_TYPE_Q5_0) { + return true; + } + if (src0_type == GGML_TYPE_F32 && src1_type == GGML_TYPE_Q5_1) { + return true; + } + if (src0_type == GGML_TYPE_F32 && src1_type == GGML_TYPE_IQ4_NL) { + return true; + } if (src0_type == GGML_TYPE_F16 && src1_type == GGML_TYPE_F16) { return true; } diff --git a/ggml-metal.m b/ggml-metal.m index 109e5fe6b..54d1f1d5a 100644 --- a/ggml-metal.m +++ b/ggml-metal.m @@ -173,8 +173,9 @@ enum ggml_metal_kernel_type { GGML_METAL_KERNEL_TYPE_CPY_F32_Q8_0, GGML_METAL_KERNEL_TYPE_CPY_F32_Q4_0, GGML_METAL_KERNEL_TYPE_CPY_F32_Q4_1, - //GGML_METAL_KERNEL_TYPE_CPY_F32_Q5_0, - //GGML_METAL_KERNEL_TYPE_CPY_F32_Q5_1, + GGML_METAL_KERNEL_TYPE_CPY_F32_Q5_0, + GGML_METAL_KERNEL_TYPE_CPY_F32_Q5_1, + GGML_METAL_KERNEL_TYPE_CPY_F32_IQ4_NL, GGML_METAL_KERNEL_TYPE_CPY_F16_F16, GGML_METAL_KERNEL_TYPE_CPY_F16_F32, GGML_METAL_KERNEL_TYPE_CONCAT, @@ -598,8 +599,9 @@ static struct ggml_metal_context * ggml_metal_init(int n_cb) { GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_CPY_F32_Q8_0, cpy_f32_q8_0, true); GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_CPY_F32_Q4_0, cpy_f32_q4_0, true); GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_CPY_F32_Q4_1, cpy_f32_q4_1, true); - //GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_CPY_F32_Q5_0, cpy_f32_q5_0, true); - //GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_CPY_F32_Q5_1, cpy_f32_q5_1, true); + GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_CPY_F32_Q5_0, cpy_f32_q5_0, true); + GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_CPY_F32_Q5_1, cpy_f32_q5_1, true); + GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_CPY_F32_IQ4_NL, cpy_f32_iq4_nl, true); GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_CPY_F16_F16, cpy_f16_f16, true); GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_CPY_F16_F32, cpy_f16_f32, true); GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_CONCAT, concat, true); @@ -739,6 +741,9 @@ static bool ggml_metal_supports_op(const struct ggml_metal_context * ctx, const case GGML_TYPE_Q8_0: case GGML_TYPE_Q4_0: case GGML_TYPE_Q4_1: + case GGML_TYPE_Q5_0: + case GGML_TYPE_Q5_1: + case GGML_TYPE_IQ4_NL: return true; default: return false; @@ -2431,13 +2436,14 @@ static enum ggml_status ggml_metal_graph_compute( GGML_ASSERT(ne0 % ggml_blck_size(dst->type) == 0); switch (dstt) { - case GGML_TYPE_F16: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_CPY_F32_F16].pipeline; break; - case GGML_TYPE_F32: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_CPY_F32_F32].pipeline; break; - case GGML_TYPE_Q8_0: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_CPY_F32_Q8_0].pipeline; break; - case GGML_TYPE_Q4_0: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_CPY_F32_Q4_0].pipeline; break; - case GGML_TYPE_Q4_1: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_CPY_F32_Q4_1].pipeline; break; - //case GGML_TYPE_Q5_0: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_CPY_F32_Q5_0].pipeline; break; - //case GGML_TYPE_Q5_1: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_CPY_F32_Q5_1].pipeline; break; + case GGML_TYPE_F16: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_CPY_F32_F16].pipeline; break; + case GGML_TYPE_F32: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_CPY_F32_F32].pipeline; break; + case GGML_TYPE_Q8_0: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_CPY_F32_Q8_0].pipeline; break; + case GGML_TYPE_Q4_0: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_CPY_F32_Q4_0].pipeline; break; + case GGML_TYPE_Q4_1: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_CPY_F32_Q4_1].pipeline; break; + case GGML_TYPE_Q5_0: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_CPY_F32_Q5_0].pipeline; break; + case GGML_TYPE_Q5_1: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_CPY_F32_Q5_1].pipeline; break; + case GGML_TYPE_IQ4_NL: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_CPY_F32_IQ4_NL].pipeline; break; default: GGML_ASSERT(false && "not implemented"); }; } break; diff --git a/ggml-metal.metal b/ggml-metal.metal index 63de56325..5552c8b28 100644 --- a/ggml-metal.metal +++ b/ggml-metal.metal @@ -2388,6 +2388,242 @@ kernel void kernel_cpy_f32_q4_1( } } +kernel void kernel_cpy_f32_q5_0( + device const float * src0, + device void * dst, + constant int64_t & ne00, + constant int64_t & ne01, + constant int64_t & ne02, + constant int64_t & ne03, + constant uint64_t & nb00, + constant uint64_t & nb01, + constant uint64_t & nb02, + constant uint64_t & nb03, + constant int64_t & ne0, + constant int64_t & ne1, + constant int64_t & ne2, + constant int64_t & ne3, + constant uint64_t & nb0, + constant uint64_t & nb1, + constant uint64_t & nb2, + constant uint64_t & nb3, + uint3 tgpig[[threadgroup_position_in_grid]], + uint3 tpitg[[thread_position_in_threadgroup]], + uint3 ntg[[threads_per_threadgroup]]) { + const int64_t i03 = tgpig[2]; + const int64_t i02 = tgpig[1]; + const int64_t i01 = tgpig[0]; + + const int64_t n = i03*ne02*ne01*ne00 + i02*ne01*ne00 + i01*ne00; + + const int64_t i3 = n / (ne2*ne1*ne0); + const int64_t i2 = (n - i3*ne2*ne1*ne0) / (ne1*ne0); + const int64_t i1 = (n - i3*ne2*ne1*ne0 - i2*ne1*ne0) / ne0; + const int64_t i0 = (n - i3*ne2*ne1*ne0 - i2*ne1*ne0 - i1*ne0)/QK5_0; + + device block_q5_0 * dst_data = (device block_q5_0 *) ((device char *) dst + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0); + + for (int64_t i00 = tpitg.x*QK5_0; i00 < ne00; i00 += ntg.x*QK5_0) { + device const float * src = (device float *)((device char *) src0 + i03*nb03 + i02*nb02 + i01*nb01 + i00*nb00); + + float amax = 0.0f; // absolute max + float max = 0.0f; + + for (int j = 0; j < QK5_0; j++) { + const float v = src[j]; + if (amax < fabs(v)) { + amax = fabs(v); + max = v; + } + } + + const float d = max / -16; + const float id = d ? 1.0f/d : 0.0f; + + dst_data[i00/QK5_0].d = d; + + uint32_t qh = 0; + for (int j = 0; j < QK5_0/2; ++j) { + const float x0 = src[0 + j]*id; + const float x1 = src[QK5_0/2 + j]*id; + + const uint8_t xi0 = MIN(31, (int8_t)(x0 + 16.5f)); + const uint8_t xi1 = MIN(31, (int8_t)(x1 + 16.5f)); + + dst_data[i00/QK5_0].qs[j] = (xi0 & 0xf) | ((xi1 & 0xf) << 4); + qh |= ((xi0 & 0x10u) >> 4) << (j + 0); + qh |= ((xi1 & 0x10u) >> 4) << (j + QK5_0/2); + } + thread const uint8_t * qh8 = (thread const uint8_t *)&qh; + for (int j = 0; j < 4; ++j) { + dst_data[i00/QK5_0].qh[j] = qh8[j]; + } + } +} + +kernel void kernel_cpy_f32_q5_1( + device const float * src0, + device void * dst, + constant int64_t & ne00, + constant int64_t & ne01, + constant int64_t & ne02, + constant int64_t & ne03, + constant uint64_t & nb00, + constant uint64_t & nb01, + constant uint64_t & nb02, + constant uint64_t & nb03, + constant int64_t & ne0, + constant int64_t & ne1, + constant int64_t & ne2, + constant int64_t & ne3, + constant uint64_t & nb0, + constant uint64_t & nb1, + constant uint64_t & nb2, + constant uint64_t & nb3, + uint3 tgpig[[threadgroup_position_in_grid]], + uint3 tpitg[[thread_position_in_threadgroup]], + uint3 ntg[[threads_per_threadgroup]]) { + const int64_t i03 = tgpig[2]; + const int64_t i02 = tgpig[1]; + const int64_t i01 = tgpig[0]; + + const int64_t n = i03*ne02*ne01*ne00 + i02*ne01*ne00 + i01*ne00; + + const int64_t i3 = n / (ne2*ne1*ne0); + const int64_t i2 = (n - i3*ne2*ne1*ne0) / (ne1*ne0); + const int64_t i1 = (n - i3*ne2*ne1*ne0 - i2*ne1*ne0) / ne0; + const int64_t i0 = (n - i3*ne2*ne1*ne0 - i2*ne1*ne0 - i1*ne0)/QK5_1; + + device block_q5_1 * dst_data = (device block_q5_1 *) ((device char *) dst + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0); + + for (int64_t i00 = tpitg.x*QK5_1; i00 < ne00; i00 += ntg.x*QK5_1) { + device const float * src = (device float *)((device char *) src0 + i03*nb03 + i02*nb02 + i01*nb01 + i00*nb00); + + float max = src[0]; + float min = src[0]; + + for (int j = 1; j < QK5_1; j++) { + const float v = src[j]; + min = v < min ? v : min; + max = v > max ? v : max; + } + + const float d = (max - min) / 31; + const float id = d ? 1.0f/d : 0.0f; + + dst_data[i00/QK5_1].d = d; + dst_data[i00/QK5_1].m = min; + + uint32_t qh = 0; + for (int j = 0; j < QK5_1/2; ++j) { + const float x0 = (src[0 + j] - min)*id; + const float x1 = (src[QK5_1/2 + j] - min)*id; + + const uint8_t xi0 = (uint8_t)(x0 + 0.5f); + const uint8_t xi1 = (uint8_t)(x1 + 0.5f); + + dst_data[i00/QK5_1].qs[j] = (xi0 & 0xf) | ((xi1 & 0xf) << 4); + qh |= ((xi0 & 0x10u) >> 4) << (j + 0); + qh |= ((xi1 & 0x10u) >> 4) << (j + QK5_1/2); + } + thread const uint8_t * qh8 = (thread const uint8_t *)&qh; + for (int j = 0; j < 4; ++j) { + dst_data[i00/QK5_1].qh[j] = qh8[j]; + } + } +} + +static inline int best_index_int8(int n, constant float * val, float x) { + if (x <= val[0]) return 0; + if (x >= val[n-1]) return n-1; + int ml = 0, mu = n-1; + while (mu-ml > 1) { + int mav = (ml+mu)/2; + if (x < val[mav]) mu = mav; else ml = mav; + } + return x - val[mu-1] < val[mu] - x ? mu-1 : mu; +} + +constexpr constant static float kvalues_iq4nl_f[16] = { + -127.f, -104.f, -83.f, -65.f, -49.f, -35.f, -22.f, -10.f, 1.f, 13.f, 25.f, 38.f, 53.f, 69.f, 89.f, 113.f +}; + +kernel void kernel_cpy_f32_iq4_nl( + device const float * src0, + device void * dst, + constant int64_t & ne00, + constant int64_t & ne01, + constant int64_t & ne02, + constant int64_t & ne03, + constant uint64_t & nb00, + constant uint64_t & nb01, + constant uint64_t & nb02, + constant uint64_t & nb03, + constant int64_t & ne0, + constant int64_t & ne1, + constant int64_t & ne2, + constant int64_t & ne3, + constant uint64_t & nb0, + constant uint64_t & nb1, + constant uint64_t & nb2, + constant uint64_t & nb3, + uint3 tgpig[[threadgroup_position_in_grid]], + uint3 tpitg[[thread_position_in_threadgroup]], + uint3 ntg[[threads_per_threadgroup]]) { + const int64_t i03 = tgpig[2]; + const int64_t i02 = tgpig[1]; + const int64_t i01 = tgpig[0]; + + const int64_t n = i03*ne02*ne01*ne00 + i02*ne01*ne00 + i01*ne00; + + const int64_t i3 = n / (ne2*ne1*ne0); + const int64_t i2 = (n - i3*ne2*ne1*ne0) / (ne1*ne0); + const int64_t i1 = (n - i3*ne2*ne1*ne0 - i2*ne1*ne0) / ne0; + const int64_t i0 = (n - i3*ne2*ne1*ne0 - i2*ne1*ne0 - i1*ne0)/QK4_NL; + + device block_iq4_nl * dst_data = (device block_iq4_nl *) ((device char *) dst + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0); + + for (int64_t i00 = tpitg.x*QK4_NL; i00 < ne00; i00 += ntg.x*QK4_NL) { + device const float * src = (device float *)((device char *) src0 + i03*nb03 + i02*nb02 + i01*nb01 + i00*nb00); + + float amax = 0.0f; // absolute max + float max = 0.0f; + + for (int j = 0; j < QK4_0; j++) { + const float v = src[j]; + if (amax < fabs(v)) { + amax = fabs(v); + max = v; + } + } + + const float d = max / kvalues_iq4nl_f[0]; + const float id = d ? 1.0f/d : 0.0f; + + float sumqx = 0, sumq2 = 0; + for (int j = 0; j < QK4_NL/2; ++j) { + const float x0 = src[0 + j]*id; + const float x1 = src[QK4_NL/2 + j]*id; + + const uint8_t xi0 = best_index_int8(16, kvalues_iq4nl_f, x0); + const uint8_t xi1 = best_index_int8(16, kvalues_iq4nl_f, x1); + + dst_data[i00/QK4_NL].qs[j] = xi0 | (xi1 << 4); + + const float v0 = kvalues_iq4nl_f[xi0]; + const float v1 = kvalues_iq4nl_f[xi1]; + const float w0 = src[0 + j]*src[0 + j]; + const float w1 = src[QK4_NL/2 + j]*src[QK4_NL/2 + j]; + sumqx += w0*v0*src[j] + w1*v1*src[QK4_NL/2 + j]; + sumq2 += w0*v0*v0 + w1*v1*v1; + + } + + dst_data[i00/QK4_NL].d = sumq2 > 0 ? sumqx/sumq2 : d; + + } +} + kernel void kernel_concat( device const char * src0, device const char * src1, @@ -4220,10 +4456,6 @@ void kernel_mul_mv_iq1_s_f32_impl( } } -constexpr constant static float kvalues_iq4nl_f[16] = { - -127.f, -104.f, -83.f, -65.f, -49.f, -35.f, -22.f, -10.f, 1.f, 13.f, 25.f, 38.f, 53.f, 69.f, 89.f, 113.f -}; - void kernel_mul_mv_iq4_nl_f32_impl( device const void * src0, device const float * src1, From 5e43ba87429d85acbde97d16b2f48d9de992cc80 Mon Sep 17 00:00:00 2001 From: Vaibhav Srivastav Date: Thu, 21 Mar 2024 10:13:12 +0100 Subject: [PATCH 678/812] build : add mac pre-build binaries (#6182) * Initial commit - add mac prebuilds. * forward contribution credits for building the workflow. * minor : remove trailing whitespaces --------- Co-authored-by: Nicolas Patry Co-authored-by: Georgi Gerganov --- .github/workflows/build.yml | 114 ++++++++++++++++++++++++++++++++++++ 1 file changed, 114 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 992c34a03..334a48824 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -21,6 +21,118 @@ env: GGML_N_THREADS: 1 jobs: + macOS-latest-cmake-arm64: + runs-on: macos-14 + + steps: + - name: Clone + id: checkout + uses: actions/checkout@v3 + + - name: Dependencies + id: depends + continue-on-error: true + run: | + brew update + + - name: Build + id: cmake_build + run: | + sysctl -a + mkdir build + cd build + cmake -DLLAMA_FATAL_WARNINGS=ON -DLLAMA_METAL_EMBED_LIBRARY=ON .. + cmake --build . --config Release -j $(sysctl -n hw.logicalcpu) + + - name: Test + id: cmake_test + run: | + cd build + ctest -L main --verbose --timeout 900 + + - name: Determine tag name + id: tag + shell: bash + run: | + BUILD_NUMBER="$(git rev-list --count HEAD)" + SHORT_HASH="$(git rev-parse --short=7 HEAD)" + if [[ "${{ env.BRANCH_NAME }}" == "master" ]]; then + echo "name=b${BUILD_NUMBER}" >> $GITHUB_OUTPUT + else + SAFE_NAME=$(echo "${{ env.BRANCH_NAME }}" | tr '/' '-') + echo "name=${SAFE_NAME}-b${BUILD_NUMBER}-${SHORT_HASH}" >> $GITHUB_OUTPUT + fi + + - name: Pack artifacts + id: pack_artifacts + if: ${{ ( github.event_name == 'push' && github.ref == 'refs/heads/master' ) || github.event.inputs.create_release == 'true' }} + run: | + cp LICENSE ./build/bin/ + zip -r llama-${{ steps.tag.outputs.name }}-bin-macos-arm64.zip ./build/bin/* + + - name: Upload artifacts + if: ${{ ( github.event_name == 'push' && github.ref == 'refs/heads/master' ) || github.event.inputs.create_release == 'true' }} + uses: actions/upload-artifact@v3 + with: + path: | + llama-${{ steps.tag.outputs.name }}-bin-macos-arm64.zip + + macOS-latest-cmake-x64: + runs-on: macos-latest + + steps: + - name: Clone + id: checkout + uses: actions/checkout@v3 + + - name: Dependencies + id: depends + continue-on-error: true + run: | + brew update + + - name: Build + id: cmake_build + run: | + sysctl -a + mkdir build + cd build + cmake -DLLAMA_FATAL_WARNINGS=ON -DLLAMA_METAL_EMBED_LIBRARY=ON .. + cmake --build . --config Release -j $(sysctl -n hw.logicalcpu) + + - name: Test + id: cmake_test + run: | + cd build + ctest -L main --verbose --timeout 900 + + - name: Determine tag name + id: tag + shell: bash + run: | + BUILD_NUMBER="$(git rev-list --count HEAD)" + SHORT_HASH="$(git rev-parse --short=7 HEAD)" + if [[ "${{ env.BRANCH_NAME }}" == "master" ]]; then + echo "name=b${BUILD_NUMBER}" >> $GITHUB_OUTPUT + else + SAFE_NAME=$(echo "${{ env.BRANCH_NAME }}" | tr '/' '-') + echo "name=${SAFE_NAME}-b${BUILD_NUMBER}-${SHORT_HASH}" >> $GITHUB_OUTPUT + fi + + - name: Pack artifacts + id: pack_artifacts + if: ${{ ( github.event_name == 'push' && github.ref == 'refs/heads/master' ) || github.event.inputs.create_release == 'true' }} + run: | + cp LICENSE ./build/bin/ + zip -r llama-${{ steps.tag.outputs.name }}-bin-macos-x64.zip ./build/bin/* + + - name: Upload artifacts + if: ${{ ( github.event_name == 'push' && github.ref == 'refs/heads/master' ) || github.event.inputs.create_release == 'true' }} + uses: actions/upload-artifact@v3 + with: + path: | + llama-${{ steps.tag.outputs.name }}-bin-macos-x64.zip + ubuntu-focal-make: runs-on: ubuntu-20.04 @@ -748,6 +860,8 @@ jobs: - macOS-latest-cmake - windows-latest-cmake - windows-latest-cmake-cublas + - macOS-latest-cmake-arm64 + - macOS-latest-cmake-x64 steps: - name: Clone From 1943c0198125a0da1a200390e82cf461f9080d99 Mon Sep 17 00:00:00 2001 From: Vaibhav Srivastav Date: Thu, 21 Mar 2024 10:30:40 +0100 Subject: [PATCH 679/812] ci : fix indentation error (#6195) --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 334a48824..0f68e4cc3 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -121,7 +121,7 @@ jobs: - name: Pack artifacts id: pack_artifacts - if: ${{ ( github.event_name == 'push' && github.ref == 'refs/heads/master' ) || github.event.inputs.create_release == 'true' }} + if: ${{ ( github.event_name == 'push' && github.ref == 'refs/heads/master' ) || github.event.inputs.create_release == 'true' }} run: | cp LICENSE ./build/bin/ zip -r llama-${{ steps.tag.outputs.name }}-bin-macos-x64.zip ./build/bin/* From 5b7b0ac8dfdd800c0fd0dc69b69991e8cb19fb46 Mon Sep 17 00:00:00 2001 From: Olivier Chafik Date: Thu, 21 Mar 2024 11:50:43 +0000 Subject: [PATCH 680/812] json-schema-to-grammar improvements (+ added to server) (#5978) * json: fix arrays (disallow `[,1]`) * json: support tuple types (`[number, string]`) * json: support additionalProperties (`{[k: string]: [string,number][]}`) * json: support required / optional properties * json: add support for pattern * json: resolve $ref (and support https schema urls) * json: fix $ref resolution * join: support union types (mostly for nullable types I think) * json: support allOf + nested anyOf * json: support any (`{}` or `{type: object}`) * json: fix merge * json: temp fix for escapes * json: spaces in output and unrestricted output spaces * json: add typings * json:fix typo * Create ts-type-to-grammar.sh * json: fix _format_literal (json.dumps already escapes quotes) * json: merge lit sequences and handle negatives {"type": "string", "pattern": "^({\"question\": \"[^\"]+\", \"response\": \"[^\"]+\"}\\n)+$"} * json: handle pattern repetitions * Update json-schema-to-grammar.mjs * Create regex-to-grammar.py * json: extract repeated regexp patterns to subrule * Update json-schema-to-grammar.py * Update json-schema-to-grammar.py * Update json-schema-to-grammar.py * json: handle schema from pydantic Optional fields * Update json-schema-to-grammar.py * Update json-schema-to-grammar.py * Update ts-type-to-grammar.sh * Update ts-type-to-grammar.sh * json: simplify nullable fields handling * json: accept duplicate identical rules * json: revert space to 1 at most * json: reuse regexp pattern subrules * json: handle uuid string format * json: fix literal escapes * json: add --allow-fetch * json: simplify range escapes * json: support negative ranges in patterns * Delete commit.txt * json: custom regex parser, adds dot support & JS-portable * json: rm trailing spaces * Update json-schema-to-grammar.mjs * json: updated server & chat `( cd examples/server && ./deps.sh )` * json: port fixes from mjs to python * Update ts-type-to-grammar.sh * json: support prefixItems alongside array items * json: add date format + fix uuid * json: add date, time, date-time formats * json: preserve order of props from TS defs * json: port schema converter to C++, wire in ./server * json: nits * Update json-schema-to-grammar.cpp * Update json-schema-to-grammar.cpp * Update json-schema-to-grammar.cpp * json: fix mjs implementation + align outputs * Update json-schema-to-grammar.mjs.hpp * json: test C++, JS & Python versions * json: nits + regen deps * json: cleanup test * json: revert from c++17 to 11 * json: nit fixes * json: dirty include for test * json: fix zig build * json: pass static command to std::system in tests (fixed temp files) * json: fix top-level $refs * json: don't use c++20 designated initializers * nit * json: basic support for reserved names `{number:{number:{root:number}}}` * Revamp test cmake to allow args (WORKING_DIRECTORY needed for JSON test) * json: re-ran server deps.sh * json: simplify test * json: support mix of additional props & required/optional * json: add tests for some expected failures * json: fix type=const in c++, add failure expectations for non-str const&enum * json: test (& simplify output of) empty schema * json: check parsing in test + fix value & string refs * json: add server tests for OAI JSON response_format * json: test/fix top-level anyOf * json: improve grammar parsing failures * json: test/fix additional props corner cases * json: fix string patterns (was missing quotes) * json: ws nit * json: fix json handling in server when there's no response_format * json: catch schema conversion errors in server * json: don't complain about unknown format type in server if unset * json: cleaner build of test * json: create examples/json-schema-pydantic-example.py * json: fix date pattern * json: move json.hpp & json-schema-to-grammar.{cpp,h} to common * json: indent 4 spaces * json: fix naming of top-level c++ function (+ drop unused one) * json: avoid using namespace std * json: fix zig build * Update server.feature * json: iostream -> fprintf * json: space before & refs for consistency * json: nits --- .gitignore | 1 + Makefile | 12 +- build.zig | 3 +- common/CMakeLists.txt | 3 + common/json-schema-to-grammar.cpp | 725 +++ common/json-schema-to-grammar.h | 4 + {examples/server => common}/json.hpp | 0 examples/json-schema-pydantic-example.py | 74 + examples/json-schema-to-grammar.py | 512 +- examples/regex-to-grammar.py | 20 + examples/server/CMakeLists.txt | 8 +- examples/server/chat.mjs | 5 +- examples/server/completion.js.hpp | 2 +- examples/server/index.html.hpp | 4592 +++++++++-------- examples/server/index.js.hpp | 1680 +++--- .../server/json-schema-to-grammar.mjs.hpp | 1942 +++++-- examples/server/public/index.html | 14 +- examples/server/public/index.js | 2 +- .../server/public/json-schema-to-grammar.mjs | 528 +- examples/server/server.cpp | 14 +- .../server/tests/features/security.feature | 16 + examples/server/tests/features/server.feature | 16 + examples/server/tests/features/steps/steps.py | 17 + examples/server/utils.hpp | 13 +- examples/ts-type-to-grammar.sh | 28 + tests/CMakeLists.txt | 113 +- tests/run-json-schema-to-grammar.mjs | 10 + tests/test-json-schema-to-grammar.cpp | 824 +++ 28 files changed, 7607 insertions(+), 3571 deletions(-) create mode 100644 common/json-schema-to-grammar.cpp create mode 100644 common/json-schema-to-grammar.h rename {examples/server => common}/json.hpp (100%) create mode 100644 examples/json-schema-pydantic-example.py create mode 100644 examples/regex-to-grammar.py create mode 100755 examples/ts-type-to-grammar.sh create mode 100644 tests/run-json-schema-to-grammar.mjs create mode 100755 tests/test-json-schema-to-grammar.cpp diff --git a/.gitignore b/.gitignore index fbeb9223c..51aa84222 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,7 @@ *.gcda *.dot *.bat +*.tmp *.metallib *.etag *.lastModified diff --git a/Makefile b/Makefile index 1daad45ed..9b72e1dbd 100644 --- a/Makefile +++ b/Makefile @@ -9,7 +9,8 @@ TEST_TARGETS = \ tests/test-llama-grammar tests/test-grammar-parser tests/test-double-float tests/test-grad0 tests/test-opt \ tests/test-quantize-fns tests/test-quantize-perf tests/test-sampling tests/test-tokenizer-0-llama \ tests/test-tokenizer-0-falcon tests/test-tokenizer-1-llama tests/test-tokenizer-1-bpe tests/test-rope \ - tests/test-backend-ops tests/test-model-load-cancel tests/test-autorelease + tests/test-backend-ops tests/test-model-load-cancel tests/test-autorelease \ + tests/test-json-schema-to-grammar # Code coverage output files COV_TARGETS = *.gcno tests/*.gcno *.gcda tests/*.gcda *.gcov tests/*.gcov lcov-report gcovr-report @@ -666,6 +667,9 @@ console.o: common/console.cpp common/console.h grammar-parser.o: common/grammar-parser.cpp common/grammar-parser.h $(CXX) $(CXXFLAGS) -c $< -o $@ +json-schema-to-grammar.o: common/json-schema-to-grammar.cpp common/json-schema-to-grammar.h + $(CXX) $(CXXFLAGS) -c $< -o $@ + train.o: common/train.cpp common/train.h $(CXX) $(CXXFLAGS) -c $< -o $@ @@ -745,7 +749,7 @@ save-load-state: examples/save-load-state/save-load-state.cpp ggml.o llama.o $(C $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) -server: examples/server/server.cpp examples/server/utils.hpp examples/server/httplib.h examples/server/json.hpp examples/server/index.html.hpp examples/server/index.js.hpp examples/server/completion.js.hpp common/stb_image.h ggml.o llama.o $(COMMON_DEPS) grammar-parser.o $(OBJS) +server: examples/server/server.cpp examples/server/utils.hpp examples/server/httplib.h common/json.hpp examples/server/index.html.hpp examples/server/index.js.hpp examples/server/completion.js.hpp json-schema-to-grammar.o common/stb_image.h ggml.o llama.o $(COMMON_DEPS) grammar-parser.o $(OBJS) $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) $(CXX) $(CXXFLAGS) $(filter-out %.h %.hpp $<,$^) -Iexamples/server $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) $(LWINSOCK2) @@ -865,6 +869,10 @@ tests/test-double-float: tests/test-double-float.cpp ggml.o $(OBJS) $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) +tests/test-json-schema-to-grammar: tests/test-json-schema-to-grammar.cpp json-schema-to-grammar.o ggml.o llama.o grammar-parser.o $(OBJS) + $(CXX) $(CXXFLAGS) -Iexamples/server -c $< -o $(call GET_OBJ_FILE, $<) + $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) + tests/test-grad0: tests/test-grad0.cpp ggml.o $(OBJS) $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) diff --git a/build.zig b/build.zig index 90609359b..a1de7083a 100644 --- a/build.zig +++ b/build.zig @@ -122,6 +122,7 @@ pub fn build(b: *std.build.Builder) !void { const console = make.obj("console", "common/console.cpp"); const sampling = make.obj("sampling", "common/sampling.cpp"); const grammar_parser = make.obj("grammar-parser", "common/grammar-parser.cpp"); + const json_schema_to_grammar = make.obj("json-schema-to-grammar", "common/json-schema-to-grammar.cpp"); const train = make.obj("train", "common/train.cpp"); const clip = make.obj("clip", "examples/llava/clip.cpp"); const llava = make.obj("llava", "examples/llava/llava.cpp"); @@ -133,7 +134,7 @@ pub fn build(b: *std.build.Builder) !void { _ = make.exe("finetune", "examples/finetune/finetune.cpp", &.{ ggml, ggml_alloc, ggml_backend, ggml_quants, llama, unicode, common, buildinfo, train }); _ = make.exe("train-text-from-scratch", "examples/train-text-from-scratch/train-text-from-scratch.cpp", &.{ ggml, ggml_alloc, ggml_backend, ggml_quants, llama, unicode, common, buildinfo, train }); - const server = make.exe("server", "examples/server/server.cpp", &.{ ggml, ggml_alloc, ggml_backend, ggml_quants, llama, unicode, common, buildinfo, sampling, grammar_parser, clip, llava }); + const server = make.exe("server", "examples/server/server.cpp", &.{ ggml, ggml_alloc, ggml_backend, ggml_quants, llama, unicode, common, buildinfo, sampling, grammar_parser, json_schema_to_grammar, clip, llava }); if (server.target.isWindows()) { server.linkSystemLibrary("ws2_32"); } diff --git a/common/CMakeLists.txt b/common/CMakeLists.txt index af2629a46..10951693a 100644 --- a/common/CMakeLists.txt +++ b/common/CMakeLists.txt @@ -47,6 +47,8 @@ if (BUILD_SHARED_LIBS) set_target_properties(${TARGET} PROPERTIES POSITION_INDEPENDENT_CODE ON) endif() +set(TARGET json-schema-to-grammar) +add_library(${TARGET} OBJECT json-schema-to-grammar.cpp json-schema-to-grammar.h) set(TARGET common) @@ -60,6 +62,7 @@ add_library(${TARGET} STATIC console.cpp grammar-parser.h grammar-parser.cpp + json.hpp train.h train.cpp ) diff --git a/common/json-schema-to-grammar.cpp b/common/json-schema-to-grammar.cpp new file mode 100644 index 000000000..dc6d30ef1 --- /dev/null +++ b/common/json-schema-to-grammar.cpp @@ -0,0 +1,725 @@ +#include "json-schema-to-grammar.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using json = nlohmann::json; + +const std::string SPACE_RULE = "\" \"?"; + +std::unordered_map PRIMITIVE_RULES = { + {"boolean", "(\"true\" | \"false\") space"}, + {"number", "(\"-\"? ([0-9] | [1-9] [0-9]*)) (\".\" [0-9]+)? ([eE] [-+]? [0-9]+)? space"}, + {"integer", "(\"-\"? ([0-9] | [1-9] [0-9]*)) space"}, + {"value", "object | array | string | number | boolean"}, + {"object", "\"{\" space ( string \":\" space value (\",\" space string \":\" space value)* )? \"}\" space"}, + {"array", "\"[\" space ( value (\",\" space value)* )? \"]\" space"}, + {"uuid", "\"\\\"\" [0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F] " + "\"-\" [0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F] " + "\"-\" [0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F] " + "\"-\" [0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F] " + "\"-\" [0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F] \"\\\"\" space"}, + {"string", " \"\\\"\" (\n" + " [^\"\\\\] |\n" + " \"\\\\\" ([\"\\\\/bfnrt] | \"u\" [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F])\n" + " )* \"\\\"\" space"}, + {"null", "\"null\" space"} +}; +std::vector OBJECT_RULE_NAMES = {"object", "array", "string", "number", "boolean", "null", "value"}; + +std::unordered_map DATE_RULES = { + {"date", "[0-9] [0-9] [0-9] [0-9] \"-\" ( \"0\" [1-9] | \"1\" [0-2] ) \"-\" ( \"0\" [1-9] | [1-2] [0-9] | \"3\" [0-1] )"}, + {"time", "([01] [0-9] | \"2\" [0-3]) \":\" [0-5] [0-9] \":\" [0-5] [0-9] ( \".\" [0-9] [0-9] [0-9] )? ( \"Z\" | ( \"+\" | \"-\" ) ( [01] [0-9] | \"2\" [0-3] ) \":\" [0-5] [0-9] )"}, + {"date-time", "date \"T\" time"}, + {"date-string", "\"\\\"\" date \"\\\"\" space"}, + {"time-string", "\"\\\"\" time \"\\\"\" space"}, + {"date-time-string", "\"\\\"\" date-time \"\\\"\" space"} +}; + +static bool is_reserved_name(const std::string & name) { + static std::unordered_set RESERVED_NAMES; + if (RESERVED_NAMES.empty()) { + RESERVED_NAMES.insert("root"); + for (const auto &p : PRIMITIVE_RULES) RESERVED_NAMES.insert(p.first); + for (const auto &p : DATE_RULES) RESERVED_NAMES.insert(p.first); + } + return RESERVED_NAMES.find(name) != RESERVED_NAMES.end(); +} + +std::regex INVALID_RULE_CHARS_RE("[^a-zA-Z0-9-]+"); +std::regex GRAMMAR_LITERAL_ESCAPE_RE("[\r\n\"]"); +std::regex GRAMMAR_RANGE_LITERAL_ESCAPE_RE("[\r\n\"\\]\\-\\\\]"); +std::unordered_map GRAMMAR_LITERAL_ESCAPES = { + {'\r', "\\r"}, {'\n', "\\n"}, {'"', "\\\""}, {'-', "\\-"}, {']', "\\]"} +}; + +std::unordered_set NON_LITERAL_SET = {'|', '.', '(', ')', '[', ']', '{', '}', '*', '+', '?'}; +std::unordered_set ESCAPED_IN_REGEXPS_BUT_NOT_IN_LITERALS = {'[', ']', '(', ')', '|', '{', '}', '*', '+', '?'}; + +template +std::string join(Iterator begin, Iterator end, const std::string & separator) { + std::ostringstream result; + if (begin != end) { + result << *begin; + for (Iterator it = begin + 1; it != end; ++it) { + result << separator << *it; + } + } + return result.str(); +} + +static std::vector split(const std::string & str, const std::string & delimiter) { + std::vector tokens; + size_t start = 0; + size_t end = str.find(delimiter); + + while (end != std::string::npos) { + tokens.push_back(str.substr(start, end - start)); + start = end + delimiter.length(); + end = str.find(delimiter, start); + } + + tokens.push_back(str.substr(start)); + + return tokens; +} + +static std::string repeat(const std::string & str, size_t n) { + if (n == 0) { + return ""; + } + + std::string result; + result.reserve(str.length() * n); + + for (size_t i = 0; i < n; ++i) { + result += str; + } + + return result; +} + +static std::string replacePattern(const std::string & input, const std::regex & regex, const std::function & replacement) { + std::smatch match; + std::string result; + + std::string::const_iterator searchStart(input.cbegin()); + std::string::const_iterator searchEnd(input.cend()); + + while (std::regex_search(searchStart, searchEnd, match, regex)) { + result.append(searchStart, searchStart + match.position()); + result.append(replacement(match)); + searchStart = match.suffix().first; + } + + result.append(searchStart, searchEnd); + + return result; +} + +static std::string format_literal(const std::string & literal) { + std::string escaped = replacePattern(json(literal).dump(), GRAMMAR_LITERAL_ESCAPE_RE, [&](const std::smatch & match) { + char c = match.str()[0]; + return GRAMMAR_LITERAL_ESCAPES.at(c); + }); + return "\"" + escaped + "\""; +} + + +class SchemaConverter { +private: + std::function _fetch_json; + bool _dotall; + std::map _rules; + std::unordered_map _refs; + std::unordered_set _refs_being_resolved; + std::vector _errors; + std::vector _warnings; + + std::string _add_rule(const std::string & name, const std::string & rule) { + std::string esc_name = regex_replace(name, INVALID_RULE_CHARS_RE, "-"); + if (_rules.find(esc_name) == _rules.end() || _rules[esc_name] == rule) { + _rules[esc_name] = rule; + return esc_name; + } else { + int i = 0; + while (_rules.find(esc_name + std::to_string(i)) != _rules.end() && _rules[esc_name + std::to_string(i)] != rule) { + i++; + } + std::string key = esc_name + std::to_string(i); + _rules[key] = rule; + return key; + } + } + + std::string _generate_union_rule(const std::string & name, const std::vector & alt_schemas) { + std::vector rules; + for (size_t i = 0; i < alt_schemas.size(); i++) { + rules.push_back(visit(alt_schemas[i], name + (name.empty() ? "alternative-" : "-") + std::to_string(i))); + } + return join(rules.begin(), rules.end(), " | "); + } + + std::string _visit_pattern(const std::string & pattern, const std::string & name) { + if (!(pattern.front() == '^' && pattern.back() == '$')) { + _errors.push_back("Pattern must start with '^' and end with '$'"); + return ""; + } + std::string sub_pattern = pattern.substr(1, pattern.length() - 2); + std::unordered_map sub_rule_ids; + + size_t i = 0; + size_t length = sub_pattern.length(); + + using literal_or_rule = std::pair; + auto to_rule = [&](const literal_or_rule & ls) { + auto is_literal = ls.second; + auto s = ls.first; + return is_literal ? "\"" + s + "\"" : s; + }; + std::function transform = [&]() -> literal_or_rule { + size_t start = i; + std::vector seq; + + auto get_dot = [&]() { + std::string rule; + if (_dotall) { + rule = "[\\U00000000-\\U0010FFFF]"; + } else { + rule = "[\\U00000000-\\x09\\x0B\\x0C\\x0E-\\U0010FFFF]"; + } + return _add_rule("dot", rule); + }; + + // Joins the sequence, merging consecutive literals together. + auto join_seq = [&]() { + std::vector ret; + + std::string literal; + auto flush_literal = [&]() { + if (literal.empty()) { + return false; + } + ret.push_back(std::make_pair(literal, true)); + literal.clear(); + return true; + }; + + for (const auto & item : seq) { + auto is_literal = item.second; + if (is_literal) { + literal += item.first; + } else { + flush_literal(); + ret.push_back(item); + } + } + flush_literal(); + + std::vector results; + for (const auto & item : ret) { + results.push_back(to_rule(item)); + } + return std::make_pair(join(results.begin(), results.end(), " "), false); + }; + + while (i < length) { + char c = sub_pattern[i]; + if (c == '.') { + seq.push_back(std::make_pair(get_dot(), false)); + i++; + } else if (c == '(') { + i++; + if (i < length) { + if (sub_pattern[i] == '?') { + _warnings.push_back("Unsupported pattern syntax"); + } + } + seq.push_back(std::make_pair("(" + to_rule(transform()) + ")", false)); + } else if (c == ')') { + i++; + if (start > 0 && sub_pattern[start - 1] != '(') { + _errors.push_back("Unbalanced parentheses"); + } + return join_seq(); + } else if (c == '[') { + std::string square_brackets = std::string(1, c); + i++; + while (i < length && sub_pattern[i] != ']') { + if (sub_pattern[i] == '\\') { + square_brackets += sub_pattern.substr(i, 2); + i += 2; + } else { + square_brackets += sub_pattern[i]; + i++; + } + } + if (i >= length) { + _errors.push_back("Unbalanced square brackets"); + } + square_brackets += ']'; + i++; + seq.push_back(std::make_pair(square_brackets, false)); + } else if (c == '|') { + seq.push_back(std::make_pair("|", false)); + i++; + } else if (c == '*' || c == '+' || c == '?') { + seq.back() = std::make_pair(to_rule(seq.back()) + c, false); + i++; + } else if (c == '{') { + std::string curly_brackets = std::string(1, c); + i++; + while (i < length && sub_pattern[i] != '}') { + curly_brackets += sub_pattern[i]; + i++; + } + if (i >= length) { + _errors.push_back("Unbalanced curly brackets"); + } + curly_brackets += '}'; + i++; + auto nums = split(curly_brackets.substr(1, curly_brackets.length() - 2), ","); + int min_times = 0; + int max_times = std::numeric_limits::max(); + try { + if (nums.size() == 1) { + min_times = max_times = std::stoi(nums[0]); + } else if (nums.size() != 2) { + _errors.push_back("Wrong number of values in curly brackets"); + } else { + if (!nums[0].empty()) { + min_times = std::stoi(nums[0]); + } + if (!nums[1].empty()) { + max_times = std::stoi(nums[1]); + } + } + } catch (const std::invalid_argument & e) { + _errors.push_back("Invalid number in curly brackets"); + return std::make_pair("", false); + } + auto &last = seq.back(); + auto &sub = last.first; + auto sub_is_literal = last.second; + + if (min_times == 0 && max_times == std::numeric_limits::max()) { + sub += "*"; + } else if (min_times == 0 && max_times == 1) { + sub += "?"; + } else if (min_times == 1 && max_times == std::numeric_limits::max()) { + sub += "+"; + } else { + if (!sub_is_literal) { + std::string & sub_id = sub_rule_ids[sub]; + if (sub_id.empty()) { + sub_id = _add_rule(name + "-" + std::to_string(sub_rule_ids.size()), sub); + } + sub = sub_id; + } + std::string result; + if (sub_is_literal && min_times > 0) { + result = "\"" + repeat(sub.substr(1, sub.length() - 2), min_times) + "\""; + } else { + for (int j = 0; j < min_times; j++) { + if (j > 0) { + result += " "; + } + result += sub; + } + } + if (min_times > 0 && min_times < max_times) { + result += " "; + } + if (max_times == std::numeric_limits::max()) { + result += sub + "*"; + } else { + for (int j = min_times; j < max_times; j++) { + if (j > min_times) { + result += " "; + } + result += sub + "?"; + } + } + seq.back().first = result; + seq.back().second = false; + } + } else { + std::string literal; + auto is_non_literal = [&](char c) { + return NON_LITERAL_SET.find(c) != NON_LITERAL_SET.end(); + }; + while (i < length) { + if (sub_pattern[i] == '\\' && i < length - 1) { + char next = sub_pattern[i + 1]; + if (ESCAPED_IN_REGEXPS_BUT_NOT_IN_LITERALS.find(next) != ESCAPED_IN_REGEXPS_BUT_NOT_IN_LITERALS.end()) { + i++; + literal += sub_pattern[i]; + i++; + } else { + literal += sub_pattern.substr(i, 2); + i += 2; + } + } else if (sub_pattern[i] == '"') { + literal += "\\\""; + i++; + } else if (!is_non_literal(sub_pattern[i]) && + (i == length - 1 || literal.empty() || sub_pattern[i + 1] == '.' || !is_non_literal(sub_pattern[i + 1]))) { + literal += sub_pattern[i]; + i++; + } else { + break; + } + } + if (!literal.empty()) { + seq.push_back(std::make_pair(literal, true)); + } + } + } + return join_seq(); + }; + return _add_rule(name, "\"\\\"\" " + to_rule(transform()) + " \"\\\"\" space"); + } + + std::string _resolve_ref(const std::string & ref) { + std::string ref_name = ref.substr(ref.find_last_of('/') + 1); + if (_rules.find(ref_name) == _rules.end() && _refs_being_resolved.find(ref) == _refs_being_resolved.end()) { + _refs_being_resolved.insert(ref); + json resolved = _refs[ref]; + ref_name = visit(resolved, ref_name); + _refs_being_resolved.erase(ref); + } + return ref_name; + } + + std::string _build_object_rule( + const std::vector> & properties, + const std::unordered_set & required, + const std::string & name, + const json & additional_properties) + { + std::vector required_props; + std::vector optional_props; + std::unordered_map prop_kv_rule_names; + for (const auto & kv : properties) { + const auto &prop_name = kv.first; + const auto &prop_schema = kv.second; + + std::string prop_rule_name = visit(prop_schema, name + (name.empty() ? "" : "-") + prop_name); + prop_kv_rule_names[prop_name] = _add_rule( + name + (name.empty() ? "" : "-") + prop_name + "-kv", + format_literal(prop_name) + " space \":\" space " + prop_rule_name + ); + if (required.find(prop_name) != required.end()) { + required_props.push_back(prop_name); + } else { + optional_props.push_back(prop_name); + } + } + if (additional_properties.is_object() || (additional_properties.is_boolean() && additional_properties.get())) { + std::string sub_name = name + (name.empty() ? "" : "-") + "additional"; + std::string value_rule = visit(additional_properties.is_object() ? additional_properties : json::object(), sub_name + "-value"); + std::string kv_rule = _add_rule(sub_name + "-kv", _add_rule("string", PRIMITIVE_RULES.at("string")) + " \":\" space " + value_rule); + prop_kv_rule_names["*"] = kv_rule; + optional_props.push_back("*"); + } + + std::string rule = "\"{\" space "; + for (size_t i = 0; i < required_props.size(); i++) { + if (i > 0) { + rule += " \",\" space "; + } + rule += prop_kv_rule_names[required_props[i]]; + } + + if (!optional_props.empty()) { + rule += " ("; + if (!required_props.empty()) { + rule += " \",\" space ( "; + } + + std::function &, bool)> get_recursive_refs = [&](const std::vector & ks, bool first_is_optional) { + std::string res; + if (ks.empty()) { + return res; + } + std::string k = ks[0]; + std::string kv_rule_name = prop_kv_rule_names[k]; + if (k == "*") { + res = _add_rule( + name + (name.empty() ? "" : "-") + "additional-kvs", + kv_rule_name + " ( \",\" space " + kv_rule_name + " )*" + ); + } else if (first_is_optional) { + res = "( \",\" space " + kv_rule_name + " )?"; + } else { + res = kv_rule_name; + } + if (ks.size() > 1) { + res += " " + _add_rule( + name + (name.empty() ? "" : "-") + k + "-rest", + get_recursive_refs(std::vector(ks.begin() + 1, ks.end()), true) + ); + } + return res; + }; + + for (size_t i = 0; i < optional_props.size(); i++) { + if (i > 0) { + rule += " | "; + } + rule += get_recursive_refs(std::vector(optional_props.begin() + i, optional_props.end()), false); + } + if (!required_props.empty()) { + rule += " )"; + } + rule += " )?"; + } + + rule += " \"}\" space"; + + return rule; + } + +public: + SchemaConverter( + const std::function & fetch_json, + bool dotall) + : _fetch_json(fetch_json), _dotall(dotall) + { + _rules["space"] = SPACE_RULE; + } + + void resolve_refs(nlohmann::json & schema, const std::string & url) { + /* + * Resolves all $ref fields in the given schema, fetching any remote schemas, + * replacing each $ref with absolute reference URL and populates _refs with the + * respective referenced (sub)schema dictionaries. + */ + std::function visit_refs = [&](json & n) { + if (n.is_array()) { + for (auto & x : n) { + visit_refs(x); + } + } else if (n.is_object()) { + if (n.contains("$ref")) { + std::string ref = n["$ref"]; + if (_refs.find(ref) == _refs.end()) { + json target; + if (ref.find("https://") == 0) { + std::string base_url = ref.substr(0, ref.find('#')); + auto it = _refs.find(base_url); + if (it != _refs.end()) { + target = it->second; + } else { + // Fetch the referenced schema and resolve its refs + auto referenced = _fetch_json(ref); + resolve_refs(referenced, base_url); + _refs[base_url] = referenced; + } + if (ref.find('#') == std::string::npos || ref.substr(ref.find('#') + 1).empty()) { + return; + } + } else if (ref.find("#/") == 0) { + target = schema; + n["$ref"] = url + ref; + ref = url + ref; + } else { + _errors.push_back("Unsupported ref: " + ref); + return; + } + std::string pointer = ref.substr(ref.find('#') + 1); + std::vector tokens = split(pointer, "/"); + for (size_t i = 1; i < tokens.size(); ++i) { + std::string sel = tokens[i]; + if (target.is_null() || !target.contains(sel)) { + _errors.push_back("Error resolving ref " + ref + ": " + sel + " not in " + target.dump()); + return; + } + target = target[sel]; + } + _refs[ref] = target; + } + } else { + for (auto & kv : n.items()) { + visit_refs(kv.value()); + } + } + } + }; + + visit_refs(schema); + } + + std::string _generate_constant_rule(const json & value) { + if (!value.is_string()) { + _errors.push_back("Only std::string constants are supported, got " + value.dump()); + return ""; + } + return format_literal(value.get()); + } + + std::string visit(const json & schema, const std::string & name) { + json schema_type = schema.contains("type") ? schema["type"] : json(); + std::string schema_format = schema.contains("format") ? schema["format"].get() : ""; + std::string rule_name = is_reserved_name(name) ? name + "-" : name.empty() ? "root" : name; + + if (schema.contains("$ref")) { + return _add_rule(rule_name, _resolve_ref(schema["$ref"])); + } else if (schema.contains("oneOf") || schema.contains("anyOf")) { + std::vector alt_schemas = schema.contains("oneOf") ? schema["oneOf"].get>() : schema["anyOf"].get>(); + return _add_rule(rule_name, _generate_union_rule(name, alt_schemas)); + } else if (schema_type.is_array()) { + std::vector schema_types; + for (const auto & t : schema_type) { + schema_types.push_back({{"type", t}}); + } + return _add_rule(rule_name, _generate_union_rule(name, schema_types)); + } else if (schema.contains("const")) { + return _add_rule(rule_name, _generate_constant_rule(schema["const"])); + } else if (schema.contains("enum")) { + std::vector enum_values; + for (const auto & v : schema["enum"]) { + enum_values.push_back(_generate_constant_rule(v)); + } + return _add_rule(rule_name, join(enum_values.begin(), enum_values.end(), " | ")); + } else if ((schema_type.is_null() || schema_type == "object") + && (schema.contains("properties") || + (schema.contains("additionalProperties") && schema["additionalProperties"] != true))) { + std::unordered_set required; + if (schema.contains("required") && schema["required"].is_array()) { + for (const auto & item : schema["required"]) { + if (item.is_string()) { + required.insert(item.get()); + } + } + } + std::vector> properties; + if (schema.contains("properties")) { + for (const auto & prop : schema["properties"].items()) { + properties.emplace_back(prop.key(), prop.value()); + } + } + return _add_rule(rule_name, + _build_object_rule( + properties, required, name, + schema.contains("additionalProperties") ? schema["additionalProperties"] : json())); + } else if ((schema_type.is_null() || schema_type == "object") && schema.contains("allOf")) { + std::unordered_set required; + std::vector> properties; + std::string hybrid_name = name; + std::function add_component = [&](const json & comp_schema, bool is_required) { + if (comp_schema.contains("$ref")) { + add_component(_refs[comp_schema["$ref"]], is_required); + } else if (comp_schema.contains("properties")) { + for (const auto & prop : comp_schema["properties"].items()) { + properties.emplace_back(prop.key(), prop.value()); + if (is_required) { + required.insert(prop.key()); + } + } + } else { + // todo warning + } + }; + for (auto & t : schema["allOf"]) { + if (t.contains("anyOf")) { + for (auto & tt : t["anyOf"]) { + add_component(tt, false); + } + } else { + add_component(t, true); + } + } + return _add_rule(rule_name, _build_object_rule(properties, required, hybrid_name, json())); + } else if ((schema_type.is_null() || schema_type == "array") && (schema.contains("items") || schema.contains("prefixItems"))) { + json items = schema.contains("items") ? schema["items"] : schema["prefixItems"]; + if (items.is_array()) { + std::string rule = "\"[\" space "; + for (size_t i = 0; i < items.size(); i++) { + if (i > 0) { + rule += " \",\" space "; + } + rule += visit(items[i], name + (name.empty() ? "" : "-") + "tuple-" + std::to_string(i)); + } + rule += " \"]\" space"; + return _add_rule(rule_name, rule); + } else { + std::string item_rule_name = visit(items, name + (name.empty() ? "" : "-") + "item"); + std::string list_item_operator = "( \",\" space " + item_rule_name + " )"; + std::string successive_items; + int min_items = schema.contains("minItems") ? schema["minItems"].get() : 0; + json max_items_json = schema.contains("maxItems") ? schema["maxItems"] : json(); + int max_items = max_items_json.is_number_integer() ? max_items_json.get() : -1; + if (min_items > 0) { + successive_items += repeat(list_item_operator, min_items - 1); + min_items--; + } + if (max_items >= 0 && max_items > min_items) { + successive_items += repeat(list_item_operator + "?", max_items - min_items - 1); + } else { + successive_items += list_item_operator + "*"; + } + std::string rule; + if (min_items == 0) { + rule = "\"[\" space ( " + item_rule_name + " " + successive_items + " )? \"]\" space"; + } else { + rule = "\"[\" space " + item_rule_name + " " + successive_items + " \"]\" space"; + } + return _add_rule(rule_name, rule); + } + } else if ((schema_type.is_null() || schema_type == "string") && schema.contains("pattern")) { + return _visit_pattern(schema["pattern"], rule_name); + } else if ((schema_type.is_null() || schema_type == "string") && std::regex_match(schema_format, std::regex("^uuid[1-5]?$"))) { + return _add_rule(rule_name == "root" ? "root" : schema_format, PRIMITIVE_RULES.at("uuid")); + } else if ((schema_type.is_null() || schema_type == "string") && DATE_RULES.find(schema_format) != DATE_RULES.end()) { + for (const auto & kv : DATE_RULES) { + _add_rule(kv.first, kv.second); + } + return schema_format + "-string"; + } else if (schema.empty() || schema_type == "object") { + for (const auto & n : OBJECT_RULE_NAMES) { + _add_rule(n, PRIMITIVE_RULES.at(n)); + } + return _add_rule(rule_name, "object"); + } else { + if (!schema_type.is_string() || PRIMITIVE_RULES.find(schema_type.get()) == PRIMITIVE_RULES.end()) { + _errors.push_back("Unrecognized schema: " + schema.dump()); + return ""; + } + // TODO: support minimum, maximum, exclusiveMinimum, exclusiveMaximum at least for zero + return _add_rule(rule_name == "root" ? "root" : schema_type.get(), PRIMITIVE_RULES.at(schema_type.get())); + } + } + + void check_errors() { + if (!_errors.empty()) { + throw std::runtime_error("JSON schema conversion failed:\n" + join(_errors.begin(), _errors.end(), "\n")); + } + if (!_warnings.empty()) { + fprintf(stderr, "WARNING: JSON schema conversion was incomplete: %s\n", join(_warnings.begin(), _warnings.end(), "; ").c_str()); + } + } + + std::string format_grammar() { + std::stringstream ss; + for (const auto & kv : _rules) { + ss << kv.first << " ::= " << kv.second << std::endl; + } + return ss.str(); + } +}; + +std::string json_schema_to_grammar(const json & schema) { + SchemaConverter converter([](const std::string &) { return json::object(); }, /* dotall= */ false); + auto copy = schema; + converter.resolve_refs(copy, "input"); + converter.visit(copy, ""); + converter.check_errors(); + return converter.format_grammar(); +} diff --git a/common/json-schema-to-grammar.h b/common/json-schema-to-grammar.h new file mode 100644 index 000000000..2e4c6ade3 --- /dev/null +++ b/common/json-schema-to-grammar.h @@ -0,0 +1,4 @@ +#pragma once +#include "json.hpp" + +std::string json_schema_to_grammar(const nlohmann::json& schema); diff --git a/examples/server/json.hpp b/common/json.hpp similarity index 100% rename from examples/server/json.hpp rename to common/json.hpp diff --git a/examples/json-schema-pydantic-example.py b/examples/json-schema-pydantic-example.py new file mode 100644 index 000000000..69ebfd409 --- /dev/null +++ b/examples/json-schema-pydantic-example.py @@ -0,0 +1,74 @@ +# Usage: +#! ./server -m some-model.gguf & +#! pip install pydantic +#! python json-schema-pydantic-example.py + +from pydantic import BaseModel, TypeAdapter +from annotated_types import MinLen +from typing import Annotated, List, Optional +import json, requests + +if True: + + def create_completion(*, response_model=None, endpoint="http://localhost:8080/v1/chat/completions", messages, **kwargs): + ''' + Creates a chat completion using an OpenAI-compatible endpoint w/ JSON schema support + (llama.cpp server, llama-cpp-python, Anyscale / Together...) + + The response_model param takes a type (+ supports Pydantic) and behaves just as w/ Instructor (see below) + ''' + if response_model: + type_adapter = TypeAdapter(response_model) + schema = type_adapter.json_schema() + messages = [{ + "role": "system", + "content": f"You respond in JSON format with the following schema: {json.dumps(schema, indent=2)}" + }] + messages + response_format={"type": "json_object", "schema": schema} + + data = requests.post(endpoint, headers={"Content-Type": "application/json"}, + json=dict(messages=messages, response_format=response_format, **kwargs)).json() + if 'error' in data: + raise Exception(data['error']['message']) + + content = data["choices"][0]["message"]["content"] + return type_adapter.validate_json(content) if type_adapter else content + +else: + + # This alternative branch uses Instructor + OpenAI client lib. + # Instructor support streamed iterable responses, retry & more. + # (see https://python.useinstructor.com/) + #! pip install instructor openai + import instructor, openai + client = instructor.patch( + openai.OpenAI(api_key="123", base_url="http://localhost:8080"), + mode=instructor.Mode.JSON_SCHEMA) + create_completion = client.chat.completions.create + + +if __name__ == '__main__': + + class QAPair(BaseModel): + question: str + concise_answer: str + justification: str + + class PyramidalSummary(BaseModel): + title: str + summary: str + question_answers: Annotated[List[QAPair], MinLen(2)] + sub_sections: Optional[Annotated[List['PyramidalSummary'], MinLen(2)]] + + print("# Summary\n", create_completion( + model="...", + response_model=PyramidalSummary, + messages=[{ + "role": "user", + "content": f""" + You are a highly efficient corporate document summarizer. + Create a pyramidal summary of an imaginary internal document about our company processes + (starting high-level, going down to each sub sections). + Keep questions short, and answers even shorter (trivia / quizz style). + """ + }])) diff --git a/examples/json-schema-to-grammar.py b/examples/json-schema-to-grammar.py index 6a977f031..9eead557f 100755 --- a/examples/json-schema-to-grammar.py +++ b/examples/json-schema-to-grammar.py @@ -1,8 +1,10 @@ #!/usr/bin/env python3 import argparse +import itertools import json import re import sys +from typing import Any, Dict, List, Set, Tuple, Union # whitespace is constrained to a single space char to prevent model "running away" in # whitespace. Also maybe improves generation quality? @@ -12,22 +14,50 @@ PRIMITIVE_RULES = { 'boolean': '("true" | "false") space', 'number': '("-"? ([0-9] | [1-9] [0-9]*)) ("." [0-9]+)? ([eE] [-+]? [0-9]+)? space', 'integer': '("-"? ([0-9] | [1-9] [0-9]*)) space', + 'value' : 'object | array | string | number | boolean', + 'object' : '"{" space ( string ":" space value ("," space string ":" space value)* )? "}" space', + 'array' : '"[" space ( value ("," space value)* )? "]" space', + 'uuid' : '"\\"" ' + ' "-" '.join('[0-9a-fA-F]' * n for n in [8, 4, 4, 4, 12]) + ' "\\"" space', 'string': r''' "\"" ( [^"\\] | "\\" (["\\/bfnrt] | "u" [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F]) - )* "\"" space ''', + )* "\"" space''', 'null': '"null" space', } +OBJECT_RULE_NAMES = ['object', 'array', 'string', 'number', 'boolean', 'null', 'value'] + +# TODO: support "uri", "email" string formats +DATE_RULES = { + 'date' : '[0-9] [0-9] [0-9] [0-9] "-" ( "0" [1-9] | "1" [0-2] ) "-" ( \"0\" [1-9] | [1-2] [0-9] | "3" [0-1] )', + 'time' : '([01] [0-9] | "2" [0-3]) ":" [0-5] [0-9] ":" [0-5] [0-9] ( "." [0-9] [0-9] [0-9] )? ( "Z" | ( "+" | "-" ) ( [01] [0-9] | "2" [0-3] ) ":" [0-5] [0-9] )', + 'date-time': 'date "T" time', + 'date-string': '"\\"" date "\\"" space', + 'time-string': '"\\"" time "\\"" space', + 'date-time-string': '"\\"" date-time "\\"" space', +} + +RESERVED_NAMES = set(["root", *PRIMITIVE_RULES.keys(), *DATE_RULES.keys()]) INVALID_RULE_CHARS_RE = re.compile(r'[^a-zA-Z0-9-]+') GRAMMAR_LITERAL_ESCAPE_RE = re.compile(r'[\r\n"]') -GRAMMAR_LITERAL_ESCAPES = {'\r': '\\r', '\n': '\\n', '"': '\\"'} +GRAMMAR_RANGE_LITERAL_ESCAPE_RE = re.compile(r'[\r\n"\]\-\\]') +GRAMMAR_LITERAL_ESCAPES = {'\r': '\\r', '\n': '\\n', '"': '\\"', '-': '\\-', ']': '\\]'} +NON_LITERAL_SET = set('|.()[]{}*+?') +ESCAPED_IN_REGEXPS_BUT_NOT_IN_LITERALS = set('[]()|{}*+?') + +DATE_PATTERN = '[0-9]{4}-(0[1-9]|1[0-2])-([0-2][0-9]|3[0-1])' +TIME_PATTERN = '([01][0-9]|2[0-3])(:[0-5][0-9]){2}(\\.[0-9]{1,3})?(Z|[+-](([01][0-9]|2[0-3]):[0-5][0-9]))' # Cap millisecond precision w/ 3 digits class SchemaConverter: - def __init__(self, prop_order): + def __init__(self, *, prop_order, allow_fetch, dotall, raw_pattern): self._prop_order = prop_order + self._allow_fetch = allow_fetch + self._dotall = dotall + self._raw_pattern = raw_pattern self._rules = {'space': SPACE_RULE} + self._refs = {} + self._refs_being_resolved = set() def _format_literal(self, literal): escaped = GRAMMAR_LITERAL_ESCAPE_RE.sub( @@ -41,78 +71,421 @@ class SchemaConverter: key = esc_name else: i = 0 - while f'{esc_name}{i}' in self._rules: + while f'{esc_name}{i}' in self._rules and self._rules[f'{esc_name}{i}'] != rule: i += 1 key = f'{esc_name}{i}' self._rules[key] = rule return key + def resolve_refs(self, schema: dict, url: str): + ''' + Resolves all $ref fields in the given schema, fetching any remote schemas, + replacing $ref with absolute reference URL and populating self._refs with the + respective referenced (sub)schema dictionaries. + ''' + def visit(n: dict): + if isinstance(n, list): + return [visit(x) for x in n] + elif isinstance(n, dict): + ref = n.get('$ref') + if ref is not None and ref not in self._refs: + if ref.startswith('https://'): + assert self._allow_fetch, 'Fetching remote schemas is not allowed (use --allow-fetch for force)' + import requests + + frag_split = ref.split('#') + base_url = frag_split[0] + + target = self._refs.get(base_url) + if target is None: + target = self.resolve_refs(requests.get(ref).json(), base_url) + self._refs[base_url] = target + + if len(frag_split) == 1 or frag_split[-1] == '': + return target + elif ref.startswith('#/'): + target = schema + ref = f'{url}{ref}' + n['$ref'] = ref + else: + raise ValueError(f'Unsupported ref {ref}') + + for sel in ref.split('#')[-1].split('/')[1:]: + assert target is not None and sel in target, f'Error resolving ref {ref}: {sel} not in {target}' + target = target[sel] + + self._refs[ref] = target + else: + for v in n.values(): + visit(v) + + return n + return visit(schema) + + def _generate_union_rule(self, name, alt_schemas): + return ' | '.join(( + self.visit(alt_schema, f'{name}{"-" if name else "alternative-"}{i}') + for i, alt_schema in enumerate(alt_schemas) + )) + + def _visit_pattern(self, pattern, name): + ''' + Transforms a regular expression pattern into a GBNF rule. + + Input: https://json-schema.org/understanding-json-schema/reference/regular_expressions + Output: https://github.com/ggerganov/llama.cpp/blob/master/grammars/README.md + + Unsupported features: negative/positive lookaheads, greedy/non-greedy modifiers. + + Mostly a 1:1 translation, except for {x} / {x,} / {x,y} quantifiers for which + we define sub-rules to keep the output lean. + ''' + + assert pattern.startswith('^') and pattern.endswith('$'), 'Pattern must start with "^" and end with "$"' + pattern = pattern[1:-1] + sub_rule_ids = {} + + i = 0 + length = len(pattern) + + def to_rule(s: Tuple[str, bool]) -> str: + (txt, is_literal) = s + return "\"" + txt + "\"" if is_literal else txt + + def transform() -> Tuple[str, bool]: + ''' + Parse a unit at index i (advancing it), and return its string representation + whether it's a literal. + ''' + nonlocal i + nonlocal pattern + nonlocal sub_rule_ids + + start = i + # For each component of this sequence, store its string representation and whether it's a literal. + # We only need a flat structure here to apply repetition operators to the last item, and + # to merge literals at the and (we're parsing grouped ( sequences ) recursively and don't treat '|' specially + # (GBNF's syntax is luckily very close to regular expressions!) + seq: list[Tuple[str, bool]] = [] + + def get_dot(): + if self._dotall: + rule = '[\\U00000000-\\U0010FFFF]' + else: + # Accept any character... except \n and \r line break chars (\x0A and \xOD) + rule = '[\\U00000000-\\x09\\x0B\\x0C\\x0E-\\U0010FFFF]' + return self._add_rule(f'dot', rule) + + def join_seq(): + nonlocal seq + ret = [] + for is_literal, g in itertools.groupby(seq, lambda x: x[1]): + if is_literal: + ret.append((''.join(x[0] for x in g), True)) + else: + ret.extend(g) + if len(ret) == 1: + return ret[0] + return (' '.join(to_rule(x) for x in seq), False) + + while i < length: + c = pattern[i] + if c == '.': + seq.append((get_dot(), False)) + i += 1 + elif c == '(': + i += 1 + if i < length: + assert pattern[i] != '?', f'Unsupported pattern syntax "{pattern[i]}" at index {i} of /{pattern}/' + seq.append((f'({to_rule(transform())})', False)) + elif c == ')': + i += 1 + assert start > 0 and pattern[start-1] == '(', f'Unbalanced parentheses; start = {start}, i = {i}, pattern = {pattern}' + return join_seq() + elif c == '[': + square_brackets = c + i += 1 + while i < length and pattern[i] != ']': + if pattern[i] == '\\': + square_brackets += pattern[i:i+2] + i += 2 + else: + square_brackets += pattern[i] + i += 1 + assert i < length, f'Unbalanced square brackets; start = {start}, i = {i}, pattern = {pattern}' + square_brackets += ']' + i += 1 + seq.append((square_brackets, False)) + elif c == '|': + seq.append(('|', False)) + i += 1 + elif c in ('*', '+', '?'): + seq[-1] = (to_rule(seq[-1]) + c, False) + i += 1 + elif c == '{': + curly_brackets = c + i += 1 + while i < length and pattern[i] != '}': + curly_brackets += pattern[i] + i += 1 + assert i < length, f'Unbalanced curly brackets; start = {start}, i = {i}, pattern = {pattern}' + curly_brackets += '}' + i += 1 + nums = [s.strip() for s in curly_brackets[1:-1].split(',')] + min_times = 0 + max_times = None + try: + if len(nums) == 1: + min_times = int(nums[0]) + max_times = min_times + else: + assert len(nums) == 2 + min_times = int(nums[0]) if nums[0] else 0 + max_times = int(nums[1]) if nums[1] else None + except ValueError: + raise ValueError(f'Invalid quantifier {curly_brackets} in /{pattern}/') + + (sub, sub_is_literal) = seq[-1] + + if min_times == 0 and max_times is None: + seq[-1] = (f'{sub}*', False) + elif min_times == 0 and max_times == 1: + seq[-1] = (f'{sub}?', False) + elif min_times == 1 and max_times is None: + seq[-1] = (f'{sub}+', False) + else: + if not sub_is_literal: + id = sub_rule_ids.get(sub) + if id is None: + id = self._add_rule(f'{name}-{len(sub_rule_ids) + 1}', sub) + sub_rule_ids[sub] = id + sub = id + + seq[-1] = ( + ' '.join( + ([f'"{sub[1:-1] * min_times}"'] if sub_is_literal else [sub] * min_times) + + ([f'{sub}?'] * (max_times - min_times) if max_times is not None else [f'{sub}*'])), + False + ) + else: + literal = '' + while i < length: + if pattern[i] == '\\' and i < length - 1: + next = pattern[i + 1] + if next in ESCAPED_IN_REGEXPS_BUT_NOT_IN_LITERALS: + i += 1 + literal += pattern[i] + i += 1 + else: + literal += pattern[i:i+2] + i += 2 + elif pattern[i] == '"' and not self._raw_pattern: + literal += '\\"' + i += 1 + elif pattern[i] not in NON_LITERAL_SET and \ + (i == length - 1 or literal == '' or pattern[i+1] == '.' or pattern[i+1] not in NON_LITERAL_SET): + literal += pattern[i] + i += 1 + else: + break + if literal: + seq.append((literal, True)) + + return join_seq() + + return self._add_rule( + name, + to_rule(transform()) if self._raw_pattern \ + else "\"\\\"\" " + to_rule(transform()) + " \"\\\"\" space") + + + def _resolve_ref(self, ref): + ref_name = ref.split('/')[-1] + if ref_name not in self._rules and ref not in self._refs_being_resolved: + self._refs_being_resolved.add(ref) + resolved = self._refs[ref] + ref_name = self.visit(resolved, ref_name) + self._refs_being_resolved.remove(ref) + return ref_name + + def _generate_constant_rule(self, value): + assert isinstance(value, str), f'Only string constants are supported, got {value}' + return self._format_literal(value) + def visit(self, schema, name): schema_type = schema.get('type') - rule_name = name or 'root' + schema_format = schema.get('format') + rule_name = name + '-' if name in RESERVED_NAMES else name or 'root' - if 'oneOf' in schema or 'anyOf' in schema: - rule = ' | '.join(( - self.visit(alt_schema, f'{name}{"-" if name else ""}{i}') - for i, alt_schema in enumerate(schema.get('oneOf') or schema['anyOf']) - )) - return self._add_rule(rule_name, rule) + if (ref := schema.get('$ref')) is not None: + return self._add_rule(rule_name, self._resolve_ref(ref)) + + elif 'oneOf' in schema or 'anyOf' in schema: + return self._add_rule(rule_name, self._generate_union_rule(name, schema.get('oneOf') or schema['anyOf'])) + + elif isinstance(schema_type, list): + return self._add_rule(rule_name, self._generate_union_rule(name, [{'type': t} for t in schema_type])) elif 'const' in schema: - return self._add_rule(rule_name, self._format_literal(schema['const'])) + return self._add_rule(rule_name, self._generate_constant_rule(schema['const'])) elif 'enum' in schema: - rule = ' | '.join((self._format_literal(v) for v in schema['enum'])) + rule = ' | '.join((self._generate_constant_rule(v) for v in schema['enum'])) return self._add_rule(rule_name, rule) - elif schema_type == 'object' and 'properties' in schema: - # TODO: `required` keyword - prop_order = self._prop_order - prop_pairs = sorted( - schema['properties'].items(), - # sort by position in prop_order (if specified) then by key - key=lambda kv: (prop_order.get(kv[0], len(prop_order)), kv[0]), + elif schema_type in (None, 'object') and \ + ('properties' in schema or \ + ('additionalProperties' in schema and schema['additionalProperties'] is not True)): + required = set(schema.get('required', [])) + properties = list(schema.get('properties', {}).items()) + return self._add_rule(rule_name, self._build_object_rule(properties, required, name, schema.get('additionalProperties'))) + + elif schema_type in (None, 'object') and 'allOf' in schema: + required = set() + properties = [] + hybrid_name = name + def add_component(comp_schema, is_required): + if (ref := comp_schema.get('$ref')) is not None: + comp_schema = self._refs[ref] + + if 'properties' in comp_schema: + for prop_name, prop_schema in comp_schema['properties'].items(): + properties.append((prop_name, prop_schema)) + if is_required: + required.add(prop_name) + + for t in schema['allOf']: + if 'anyOf' in t: + for tt in t['anyOf']: + add_component(tt, is_required=False) + else: + add_component(t, is_required=True) + + return self._add_rule(rule_name, self._build_object_rule(properties, required, hybrid_name, additional_properties=[])) + + elif schema_type in (None, 'array') and ('items' in schema or 'prefixItems' in schema): + items = schema.get('items') or schema['prefixItems'] + if isinstance(items, list): + return self._add_rule( + rule_name, + '"[" space ' + + ' "," space '.join( + self.visit(item, f'{name}{"-" if name else ""}tuple-{i}') + for i, item in enumerate(items)) + + ' "]" space') + else: + item_rule_name = self.visit(items, f'{name}{"-" if name else ""}item') + list_item_operator = f'( "," space {item_rule_name} )' + successive_items = "" + min_items = schema.get("minItems", 0) + max_items = schema.get("maxItems") + if min_items > 0: + successive_items = list_item_operator * (min_items - 1) + min_items -= 1 + if max_items is not None and max_items > min_items: + successive_items += (list_item_operator + "?") * (max_items - min_items - 1) + else: + successive_items += list_item_operator + "*" + if min_items == 0: + rule = f'"[" space ( {item_rule_name} {successive_items} )? "]" space' + else: + rule = f'"[" space {item_rule_name} {successive_items} "]" space' + return self._add_rule(rule_name, rule) + + elif schema_type in (None, 'string') and 'pattern' in schema: + return self._visit_pattern(schema['pattern'], rule_name) + + elif schema_type in (None, 'string') and re.match(r'^uuid[1-5]?$', schema_format or ''): + return self._add_rule( + 'root' if rule_name == 'root' else schema_format, + PRIMITIVE_RULES['uuid'] ) - rule = '"{" space' - for i, (prop_name, prop_schema) in enumerate(prop_pairs): - prop_rule_name = self.visit(prop_schema, f'{name}{"-" if name else ""}{prop_name}') - if i > 0: - rule += ' "," space' - rule += fr' {self._format_literal(prop_name)} space ":" space {prop_rule_name}' - rule += ' "}" space' + elif schema_type in (None, 'string') and schema_format in DATE_RULES: + for t, r in DATE_RULES.items(): + self._add_rule(t, r) + return schema_format + '-string' - return self._add_rule(rule_name, rule) - - elif schema_type == 'array' and 'items' in schema: - # TODO `prefixItems` keyword - item_rule_name = self.visit(schema['items'], f'{name}{"-" if name else ""}item') - list_item_operator = f'("," space {item_rule_name})' - successive_items = "" - min_items = schema.get("minItems", 0) - if min_items > 0: - first_item = f"({item_rule_name})" - successive_items = list_item_operator * (min_items - 1) - min_items -= 1 - else: - first_item = f"({item_rule_name})?" - max_items = schema.get("maxItems") - if max_items is not None and max_items > min_items: - successive_items += (list_item_operator + "?") * (max_items - min_items - 1) - else: - successive_items += list_item_operator + "*" - rule = f'"[" space {first_item} {successive_items} "]" space' - return self._add_rule(rule_name, rule) + elif (schema_type == 'object') or (len(schema) == 0): + for n in OBJECT_RULE_NAMES: + self._add_rule(n, PRIMITIVE_RULES[n]) + return self._add_rule(rule_name, 'object') else: assert schema_type in PRIMITIVE_RULES, f'Unrecognized schema: {schema}' + # TODO: support minimum, maximum, exclusiveMinimum, exclusiveMaximum at least for zero return self._add_rule( 'root' if rule_name == 'root' else schema_type, PRIMITIVE_RULES[schema_type] ) + def _build_object_rule(self, properties: List[Tuple[str, Any]], required: Set[str], name: str, additional_properties: Union[bool, Any]): + prop_order = self._prop_order + # sort by position in prop_order (if specified) then by original order + sorted_props = [kv[0] for _, kv in sorted(enumerate(properties), key=lambda ikv: (prop_order.get(ikv[1][0], len(prop_order)), ikv[0]))] + + prop_kv_rule_names = {} + for prop_name, prop_schema in properties: + prop_rule_name = self.visit(prop_schema, f'{name}{"-" if name else ""}{prop_name}') + prop_kv_rule_names[prop_name] = self._add_rule( + f'{name}{"-" if name else ""}{prop_name}-kv', + fr'{self._format_literal(prop_name)} space ":" space {prop_rule_name}' + ) + required_props = [k for k in sorted_props if k in required] + optional_props = [k for k in sorted_props if k not in required] + + if additional_properties == True or isinstance(additional_properties, dict): + sub_name = f'{name}{"-" if name else ""}additional' + value_rule = self.visit({} if additional_properties == True else additional_properties, f'{sub_name}-value') + prop_kv_rule_names["*"] = self._add_rule( + f'{sub_name}-kv', + self._add_rule('string', PRIMITIVE_RULES['string']) + f' ":" space {value_rule}' + ) + optional_props.append("*") + + rule = '"{" space ' + rule += ' "," space '.join(prop_kv_rule_names[k] for k in required_props) + + if optional_props: + rule += ' (' + if required_props: + rule += ' "," space ( ' + + def get_recursive_refs(ks, first_is_optional): + [k, *rest] = ks + kv_rule_name = prop_kv_rule_names[k] + if k == '*': + res = self._add_rule( + f'{name}{"-" if name else ""}additional-kvs', + f'{kv_rule_name} ( "," space ' + kv_rule_name + ' )*' + ) + elif first_is_optional: + res = f'( "," space {kv_rule_name} )?' + else: + res = kv_rule_name + if len(rest) > 0: + res += ' ' + self._add_rule( + f'{name}{"-" if name else ""}{k}-rest', + get_recursive_refs(rest, first_is_optional=True) + ) + return res + + rule += ' | '.join( + get_recursive_refs(optional_props[i:], first_is_optional=False) + for i in range(len(optional_props)) + ) + if required_props: + rule += ' )' + rule += ' )?' + + rule += ' "}" space' + + return rule + def format_grammar(self): - return '\n'.join((f'{name} ::= {rule}' for name, rule in self._rules.items())) + return '\n'.join( + f'{name} ::= {rule}' + for name, rule in sorted(self._rules.items(), key=lambda kv: kv[0]) + ) def main(args_in = None): @@ -129,16 +502,47 @@ def main(args_in = None): type=lambda s: s.split(','), help=''' comma-separated property names defining the order of precedence for object properties; - properties not specified here are given lower precedence than those that are, and are - sorted alphabetically + properties not specified here are given lower precedence than those that are, and + are kept in their original order from the schema. Required properties are always + given precedence over optional properties. ''' ) + parser.add_argument( + '--allow-fetch', + action='store_true', + default=False, + help='Whether to allow fetching referenced schemas over HTTPS') + parser.add_argument( + '--dotall', + action='store_true', + default=False, + help='Whether to treat dot (".") as matching all chars including line breaks in regular expression patterns') + parser.add_argument( + '--raw-pattern', + action='store_true', + default=False, + help='Treats string patterns as raw patterns w/o quotes (or quote escapes)') + parser.add_argument('schema', help='file containing JSON schema ("-" for stdin)') args = parser.parse_args(args_in) - schema = json.load(sys.stdin if args.schema == '-' else open(args.schema)) - prop_order = {name: idx for idx, name in enumerate(args.prop_order)} - converter = SchemaConverter(prop_order) + if args.schema.startswith('https://'): + url = args.schema + import requests + schema = requests.get(url).json() + elif args.schema == '-': + url = 'stdin' + schema = json.load(sys.stdin) + else: + url = f'file://{args.schema}' + with open(args.schema) as f: + schema = json.load(f) + converter = SchemaConverter( + prop_order={name: idx for idx, name in enumerate(args.prop_order)}, + allow_fetch=args.allow_fetch, + dotall=args.dotall, + raw_pattern=args.raw_pattern) + schema = converter.resolve_refs(schema, url) converter.visit(schema, '') print(converter.format_grammar()) diff --git a/examples/regex-to-grammar.py b/examples/regex-to-grammar.py new file mode 100644 index 000000000..c98118e95 --- /dev/null +++ b/examples/regex-to-grammar.py @@ -0,0 +1,20 @@ +import json, subprocess, sys, os + +assert len(sys.argv) >= 2 +[_, pattern, *rest] = sys.argv + +print(subprocess.check_output( + [ + "python", + os.path.join( + os.path.dirname(os.path.realpath(__file__)), + "json-schema-to-grammar.py"), + *rest, + "-", + "--raw-pattern", + ], + text=True, + input=json.dumps({ + "type": "string", + "pattern": pattern, + }, indent=2))) diff --git a/examples/server/CMakeLists.txt b/examples/server/CMakeLists.txt index f94de1e99..d2ee47d01 100644 --- a/examples/server/CMakeLists.txt +++ b/examples/server/CMakeLists.txt @@ -2,12 +2,16 @@ set(TARGET server) option(LLAMA_SERVER_VERBOSE "Build verbose logging option for Server" ON) option(LLAMA_SERVER_SSL "Build SSL support for the server" OFF) include_directories(${CMAKE_CURRENT_SOURCE_DIR}) -add_executable(${TARGET} server.cpp utils.hpp json.hpp httplib.h) +add_executable(${TARGET} + server.cpp + utils.hpp + httplib.h +) install(TARGETS ${TARGET} RUNTIME) target_compile_definitions(${TARGET} PRIVATE SERVER_VERBOSE=$ ) -target_link_libraries(${TARGET} PRIVATE common ${CMAKE_THREAD_LIBS_INIT}) +target_link_libraries(${TARGET} PRIVATE common json-schema-to-grammar ${CMAKE_THREAD_LIBS_INIT}) if (LLAMA_SERVER_SSL) find_package(OpenSSL REQUIRED) target_link_libraries(${TARGET} PRIVATE OpenSSL::SSL OpenSSL::Crypto) diff --git a/examples/server/chat.mjs b/examples/server/chat.mjs index 219ebb51a..a79c8a3cd 100644 --- a/examples/server/chat.mjs +++ b/examples/server/chat.mjs @@ -26,8 +26,9 @@ const propOrder = grammarJsonSchemaPropOrder let grammar = null if (grammarJsonSchemaFile) { - const schema = JSON.parse(readFileSync(grammarJsonSchemaFile, 'utf-8')) - const converter = new SchemaConverter(propOrder) + let schema = JSON.parse(readFileSync(grammarJsonSchemaFile, 'utf-8')) + const converter = new SchemaConverter({prop_order: propOrder, allow_fetch: true}) + schema = await converter.resolveRefs(schema, grammarJsonSchemaFile) converter.visit(schema, '') grammar = converter.formatGrammar() } diff --git a/examples/server/completion.js.hpp b/examples/server/completion.js.hpp index 10734091e..f8bb4b0f2 100644 --- a/examples/server/completion.js.hpp +++ b/examples/server/completion.js.hpp @@ -483,4 +483,4 @@ unsigned char completion_js[] = { 0x20, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x73, 0x65, 0x74, 0x74, 0x69, 0x6e, 0x67, 0x73, 0x3b, 0x0a, 0x7d, 0x0a }; -unsigned int completion_js_len = 5796; +size_t completion_js_len = 5796; diff --git a/examples/server/index.html.hpp b/examples/server/index.html.hpp index 20551520e..ae8c88980 100644 --- a/examples/server/index.html.hpp +++ b/examples/server/index.html.hpp @@ -417,2375 +417,2405 @@ unsigned char index_html[] = { 0x20, 0x30, 0x2c, 0x20, 0x2f, 0x2f, 0x20, 0x6e, 0x6f, 0x20, 0x63, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x70, 0x72, 0x6f, 0x62, 0x61, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x69, 0x65, 0x73, 0x2c, 0x0a, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x6d, 0x61, 0x67, 0x65, 0x5f, - 0x64, 0x61, 0x74, 0x61, 0x3a, 0x20, 0x5b, 0x5d, 0x2c, 0x0a, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x63, 0x61, 0x63, 0x68, 0x65, 0x5f, 0x70, 0x72, - 0x6f, 0x6d, 0x70, 0x74, 0x3a, 0x20, 0x74, 0x72, 0x75, 0x65, 0x2c, 0x0a, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x61, 0x70, 0x69, 0x5f, 0x6b, 0x65, - 0x79, 0x3a, 0x20, 0x27, 0x27, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x29, - 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x2f, 0x2a, 0x20, 0x53, 0x54, 0x41, - 0x52, 0x54, 0x3a, 0x20, 0x53, 0x75, 0x70, 0x70, 0x6f, 0x72, 0x74, 0x20, - 0x66, 0x6f, 0x72, 0x20, 0x73, 0x74, 0x6f, 0x72, 0x69, 0x6e, 0x67, 0x20, - 0x70, 0x72, 0x6f, 0x6d, 0x70, 0x74, 0x20, 0x74, 0x65, 0x6d, 0x70, 0x6c, - 0x61, 0x74, 0x65, 0x73, 0x20, 0x61, 0x6e, 0x64, 0x20, 0x70, 0x61, 0x72, - 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x73, 0x20, 0x69, 0x6e, 0x20, 0x62, - 0x72, 0x6f, 0x77, 0x73, 0x65, 0x72, 0x73, 0x20, 0x4c, 0x6f, 0x63, 0x61, - 0x6c, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x20, 0x2a, 0x2f, 0x0a, - 0x0a, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x6c, - 0x6f, 0x63, 0x61, 0x6c, 0x5f, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, - 0x5f, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x4b, 0x65, 0x79, 0x20, - 0x3d, 0x20, 0x22, 0x6c, 0x6c, 0x61, 0x6d, 0x61, 0x63, 0x70, 0x70, 0x5f, - 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x6c, 0x6f, 0x63, 0x61, 0x6c, - 0x5f, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x22, 0x3b, 0x0a, 0x0a, - 0x20, 0x20, 0x20, 0x20, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, - 0x20, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x5f, 0x73, 0x74, 0x6f, 0x72, 0x61, - 0x67, 0x65, 0x5f, 0x73, 0x65, 0x74, 0x44, 0x61, 0x74, 0x61, 0x46, 0x72, - 0x6f, 0x6d, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x28, 0x74, 0x61, 0x67, - 0x2c, 0x20, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x29, 0x20, 0x7b, - 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x6c, 0x6f, 0x63, 0x61, 0x6c, - 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x2e, 0x73, 0x65, 0x74, 0x49, - 0x74, 0x65, 0x6d, 0x28, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x5f, 0x73, 0x74, - 0x6f, 0x72, 0x61, 0x67, 0x65, 0x5f, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, - 0x65, 0x4b, 0x65, 0x79, 0x20, 0x2b, 0x20, 0x27, 0x2f, 0x27, 0x20, 0x2b, - 0x20, 0x74, 0x61, 0x67, 0x2c, 0x20, 0x4a, 0x53, 0x4f, 0x4e, 0x2e, 0x73, - 0x74, 0x72, 0x69, 0x6e, 0x67, 0x69, 0x66, 0x79, 0x28, 0x63, 0x6f, 0x6e, - 0x74, 0x65, 0x6e, 0x74, 0x29, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, - 0x7d, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x66, 0x75, 0x6e, 0x63, 0x74, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x6d, 0x69, 0x6e, 0x5f, 0x6b, 0x65, + 0x65, 0x70, 0x3a, 0x20, 0x30, 0x2c, 0x20, 0x2f, 0x2f, 0x20, 0x6d, 0x69, + 0x6e, 0x20, 0x70, 0x72, 0x6f, 0x62, 0x73, 0x20, 0x66, 0x72, 0x6f, 0x6d, + 0x20, 0x65, 0x61, 0x63, 0x68, 0x20, 0x73, 0x61, 0x6d, 0x70, 0x6c, 0x65, + 0x72, 0x2c, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x6d, 0x61, + 0x67, 0x65, 0x5f, 0x64, 0x61, 0x74, 0x61, 0x3a, 0x20, 0x5b, 0x5d, 0x2c, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x61, 0x63, 0x68, 0x65, + 0x5f, 0x70, 0x72, 0x6f, 0x6d, 0x70, 0x74, 0x3a, 0x20, 0x74, 0x72, 0x75, + 0x65, 0x2c, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x61, 0x70, 0x69, + 0x5f, 0x6b, 0x65, 0x79, 0x3a, 0x20, 0x27, 0x27, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x7d, 0x29, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x2f, 0x2a, 0x20, + 0x53, 0x54, 0x41, 0x52, 0x54, 0x3a, 0x20, 0x53, 0x75, 0x70, 0x70, 0x6f, + 0x72, 0x74, 0x20, 0x66, 0x6f, 0x72, 0x20, 0x73, 0x74, 0x6f, 0x72, 0x69, + 0x6e, 0x67, 0x20, 0x70, 0x72, 0x6f, 0x6d, 0x70, 0x74, 0x20, 0x74, 0x65, + 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x73, 0x20, 0x61, 0x6e, 0x64, 0x20, + 0x70, 0x61, 0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x73, 0x20, 0x69, + 0x6e, 0x20, 0x62, 0x72, 0x6f, 0x77, 0x73, 0x65, 0x72, 0x73, 0x20, 0x4c, + 0x6f, 0x63, 0x61, 0x6c, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x20, + 0x2a, 0x2f, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, + 0x74, 0x20, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x5f, 0x73, 0x74, 0x6f, 0x72, + 0x61, 0x67, 0x65, 0x5f, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x4b, + 0x65, 0x79, 0x20, 0x3d, 0x20, 0x22, 0x6c, 0x6c, 0x61, 0x6d, 0x61, 0x63, + 0x70, 0x70, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x6c, 0x6f, + 0x63, 0x61, 0x6c, 0x5f, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x22, + 0x3b, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x5f, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x5f, 0x73, 0x65, 0x74, 0x44, 0x61, 0x74, - 0x61, 0x46, 0x72, 0x6f, 0x6d, 0x52, 0x61, 0x77, 0x54, 0x65, 0x78, 0x74, - 0x28, 0x74, 0x61, 0x67, 0x2c, 0x20, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, - 0x74, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x6c, - 0x6f, 0x63, 0x61, 0x6c, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x2e, - 0x73, 0x65, 0x74, 0x49, 0x74, 0x65, 0x6d, 0x28, 0x6c, 0x6f, 0x63, 0x61, - 0x6c, 0x5f, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x5f, 0x73, 0x74, - 0x6f, 0x72, 0x61, 0x67, 0x65, 0x4b, 0x65, 0x79, 0x20, 0x2b, 0x20, 0x27, - 0x2f, 0x27, 0x20, 0x2b, 0x20, 0x74, 0x61, 0x67, 0x2c, 0x20, 0x63, 0x6f, - 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, - 0x7d, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x66, 0x75, 0x6e, 0x63, 0x74, - 0x69, 0x6f, 0x6e, 0x20, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x5f, 0x73, 0x74, - 0x6f, 0x72, 0x61, 0x67, 0x65, 0x5f, 0x67, 0x65, 0x74, 0x44, 0x61, 0x74, - 0x61, 0x41, 0x73, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x28, 0x74, 0x61, - 0x67, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, - 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x69, 0x74, 0x65, 0x6d, 0x20, 0x3d, 0x20, - 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, - 0x2e, 0x67, 0x65, 0x74, 0x49, 0x74, 0x65, 0x6d, 0x28, 0x6c, 0x6f, 0x63, - 0x61, 0x6c, 0x5f, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x5f, 0x73, - 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x4b, 0x65, 0x79, 0x20, 0x2b, 0x20, - 0x27, 0x2f, 0x27, 0x20, 0x2b, 0x20, 0x74, 0x61, 0x67, 0x29, 0x3b, 0x0a, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, 0x21, 0x69, - 0x74, 0x65, 0x6d, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x6e, 0x75, - 0x6c, 0x6c, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x20, - 0x65, 0x6c, 0x73, 0x65, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x4a, 0x53, - 0x4f, 0x4e, 0x2e, 0x70, 0x61, 0x72, 0x73, 0x65, 0x28, 0x69, 0x74, 0x65, - 0x6d, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, - 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x66, - 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x6c, 0x6f, 0x63, 0x61, - 0x6c, 0x5f, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x5f, 0x67, 0x65, - 0x74, 0x44, 0x61, 0x74, 0x61, 0x41, 0x73, 0x52, 0x61, 0x77, 0x54, 0x65, - 0x78, 0x74, 0x28, 0x74, 0x61, 0x67, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x69, 0x74, - 0x65, 0x6d, 0x20, 0x3d, 0x20, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x53, 0x74, - 0x6f, 0x72, 0x61, 0x67, 0x65, 0x2e, 0x67, 0x65, 0x74, 0x49, 0x74, 0x65, - 0x6d, 0x28, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x5f, 0x73, 0x74, 0x6f, 0x72, - 0x61, 0x67, 0x65, 0x5f, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x4b, - 0x65, 0x79, 0x20, 0x2b, 0x20, 0x27, 0x2f, 0x27, 0x20, 0x2b, 0x20, 0x74, - 0x61, 0x67, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, - 0x66, 0x20, 0x28, 0x21, 0x69, 0x74, 0x65, 0x6d, 0x29, 0x20, 0x7b, 0x0a, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, - 0x72, 0x6e, 0x20, 0x6e, 0x75, 0x6c, 0x6c, 0x3b, 0x0a, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x7d, 0x20, 0x65, 0x6c, 0x73, 0x65, 0x20, 0x7b, 0x0a, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, - 0x72, 0x6e, 0x20, 0x69, 0x74, 0x65, 0x6d, 0x3b, 0x0a, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x0a, - 0x20, 0x20, 0x20, 0x20, 0x2f, 0x2f, 0x20, 0x63, 0x72, 0x65, 0x61, 0x74, - 0x65, 0x20, 0x61, 0x20, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, - 0x72, 0x20, 0x66, 0x6f, 0x72, 0x20, 0x75, 0x73, 0x65, 0x72, 0x20, 0x74, - 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x73, 0x20, 0x61, 0x6e, 0x64, - 0x20, 0x73, 0x65, 0x74, 0x74, 0x69, 0x6e, 0x67, 0x73, 0x0a, 0x0a, 0x20, - 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x73, 0x61, 0x76, + 0x61, 0x46, 0x72, 0x6f, 0x6d, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x28, + 0x74, 0x61, 0x67, 0x2c, 0x20, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, + 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x6c, 0x6f, + 0x63, 0x61, 0x6c, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x2e, 0x73, + 0x65, 0x74, 0x49, 0x74, 0x65, 0x6d, 0x28, 0x6c, 0x6f, 0x63, 0x61, 0x6c, + 0x5f, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x5f, 0x73, 0x74, 0x6f, + 0x72, 0x61, 0x67, 0x65, 0x4b, 0x65, 0x79, 0x20, 0x2b, 0x20, 0x27, 0x2f, + 0x27, 0x20, 0x2b, 0x20, 0x74, 0x61, 0x67, 0x2c, 0x20, 0x4a, 0x53, 0x4f, + 0x4e, 0x2e, 0x73, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x69, 0x66, 0x79, 0x28, + 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x29, 0x29, 0x3b, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x66, 0x75, + 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x6c, 0x6f, 0x63, 0x61, 0x6c, + 0x5f, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x5f, 0x73, 0x65, 0x74, + 0x44, 0x61, 0x74, 0x61, 0x46, 0x72, 0x6f, 0x6d, 0x52, 0x61, 0x77, 0x54, + 0x65, 0x78, 0x74, 0x28, 0x74, 0x61, 0x67, 0x2c, 0x20, 0x63, 0x6f, 0x6e, + 0x74, 0x65, 0x6e, 0x74, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x53, 0x74, 0x6f, 0x72, 0x61, + 0x67, 0x65, 0x2e, 0x73, 0x65, 0x74, 0x49, 0x74, 0x65, 0x6d, 0x28, 0x6c, + 0x6f, 0x63, 0x61, 0x6c, 0x5f, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, + 0x5f, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x4b, 0x65, 0x79, 0x20, + 0x2b, 0x20, 0x27, 0x2f, 0x27, 0x20, 0x2b, 0x20, 0x74, 0x61, 0x67, 0x2c, + 0x20, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x29, 0x3b, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x66, 0x75, + 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x6c, 0x6f, 0x63, 0x61, 0x6c, + 0x5f, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x5f, 0x67, 0x65, 0x74, + 0x44, 0x61, 0x74, 0x61, 0x41, 0x73, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, + 0x28, 0x74, 0x61, 0x67, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x69, 0x74, 0x65, 0x6d, + 0x20, 0x3d, 0x20, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x53, 0x74, 0x6f, 0x72, + 0x61, 0x67, 0x65, 0x2e, 0x67, 0x65, 0x74, 0x49, 0x74, 0x65, 0x6d, 0x28, + 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x5f, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, + 0x65, 0x5f, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x4b, 0x65, 0x79, + 0x20, 0x2b, 0x20, 0x27, 0x2f, 0x27, 0x20, 0x2b, 0x20, 0x74, 0x61, 0x67, + 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, + 0x28, 0x21, 0x69, 0x74, 0x65, 0x6d, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, + 0x20, 0x6e, 0x75, 0x6c, 0x6c, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x7d, 0x20, 0x65, 0x6c, 0x73, 0x65, 0x20, 0x7b, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, + 0x20, 0x4a, 0x53, 0x4f, 0x4e, 0x2e, 0x70, 0x61, 0x72, 0x73, 0x65, 0x28, + 0x69, 0x74, 0x65, 0x6d, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x6c, + 0x6f, 0x63, 0x61, 0x6c, 0x5f, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, + 0x5f, 0x67, 0x65, 0x74, 0x44, 0x61, 0x74, 0x61, 0x41, 0x73, 0x52, 0x61, + 0x77, 0x54, 0x65, 0x78, 0x74, 0x28, 0x74, 0x61, 0x67, 0x29, 0x20, 0x7b, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, + 0x20, 0x69, 0x74, 0x65, 0x6d, 0x20, 0x3d, 0x20, 0x6c, 0x6f, 0x63, 0x61, + 0x6c, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x2e, 0x67, 0x65, 0x74, + 0x49, 0x74, 0x65, 0x6d, 0x28, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x5f, 0x73, + 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x5f, 0x73, 0x74, 0x6f, 0x72, 0x61, + 0x67, 0x65, 0x4b, 0x65, 0x79, 0x20, 0x2b, 0x20, 0x27, 0x2f, 0x27, 0x20, + 0x2b, 0x20, 0x74, 0x61, 0x67, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, 0x21, 0x69, 0x74, 0x65, 0x6d, 0x29, + 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, + 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x6e, 0x75, 0x6c, 0x6c, 0x3b, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x20, 0x65, 0x6c, 0x73, 0x65, + 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, + 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x69, 0x74, 0x65, 0x6d, 0x3b, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x7d, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x2f, 0x2f, 0x20, 0x63, 0x72, + 0x65, 0x61, 0x74, 0x65, 0x20, 0x61, 0x20, 0x63, 0x6f, 0x6e, 0x74, 0x61, + 0x69, 0x6e, 0x65, 0x72, 0x20, 0x66, 0x6f, 0x72, 0x20, 0x75, 0x73, 0x65, + 0x72, 0x20, 0x74, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x73, 0x20, + 0x61, 0x6e, 0x64, 0x20, 0x73, 0x65, 0x74, 0x74, 0x69, 0x6e, 0x67, 0x73, + 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, + 0x73, 0x61, 0x76, 0x65, 0x64, 0x55, 0x73, 0x65, 0x72, 0x54, 0x65, 0x6d, + 0x70, 0x6c, 0x61, 0x74, 0x65, 0x73, 0x20, 0x3d, 0x20, 0x73, 0x69, 0x67, + 0x6e, 0x61, 0x6c, 0x28, 0x7b, 0x7d, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x73, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x65, 0x64, 0x55, 0x73, 0x65, 0x72, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, - 0x74, 0x65, 0x73, 0x20, 0x3d, 0x20, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x6c, - 0x28, 0x7b, 0x7d, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, - 0x73, 0x74, 0x20, 0x73, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x65, 0x64, 0x55, - 0x73, 0x65, 0x72, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x20, - 0x3d, 0x20, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x28, 0x7b, 0x20, 0x6e, - 0x61, 0x6d, 0x65, 0x3a, 0x20, 0x27, 0x27, 0x2c, 0x20, 0x74, 0x65, 0x6d, - 0x70, 0x6c, 0x61, 0x74, 0x65, 0x3a, 0x20, 0x7b, 0x20, 0x73, 0x65, 0x73, - 0x73, 0x69, 0x6f, 0x6e, 0x3a, 0x20, 0x7b, 0x7d, 0x2c, 0x20, 0x70, 0x61, - 0x72, 0x61, 0x6d, 0x73, 0x3a, 0x20, 0x7b, 0x7d, 0x20, 0x7d, 0x20, 0x7d, - 0x29, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x2f, 0x2f, 0x20, 0x6c, 0x65, - 0x74, 0x27, 0x73, 0x20, 0x69, 0x6d, 0x70, 0x6f, 0x72, 0x74, 0x20, 0x6c, - 0x6f, 0x63, 0x61, 0x6c, 0x6c, 0x79, 0x20, 0x73, 0x61, 0x76, 0x65, 0x64, - 0x20, 0x74, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x73, 0x20, 0x61, - 0x6e, 0x64, 0x20, 0x73, 0x65, 0x74, 0x74, 0x69, 0x6e, 0x67, 0x73, 0x20, - 0x69, 0x66, 0x20, 0x74, 0x68, 0x65, 0x72, 0x65, 0x20, 0x61, 0x72, 0x65, - 0x20, 0x61, 0x6e, 0x79, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x2f, 0x2f, 0x20, - 0x75, 0x73, 0x65, 0x72, 0x20, 0x74, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, - 0x65, 0x73, 0x20, 0x61, 0x6e, 0x64, 0x20, 0x73, 0x65, 0x74, 0x74, 0x69, - 0x6e, 0x67, 0x73, 0x20, 0x61, 0x72, 0x65, 0x20, 0x73, 0x74, 0x6f, 0x72, - 0x65, 0x64, 0x20, 0x69, 0x6e, 0x20, 0x6f, 0x6e, 0x65, 0x20, 0x6f, 0x62, - 0x6a, 0x65, 0x63, 0x74, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x2f, 0x2f, 0x20, - 0x69, 0x6e, 0x20, 0x66, 0x6f, 0x72, 0x6d, 0x20, 0x6f, 0x66, 0x20, 0x7b, - 0x20, 0x22, 0x74, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x6e, 0x61, - 0x6d, 0x65, 0x22, 0x3a, 0x20, 0x22, 0x74, 0x65, 0x6d, 0x70, 0x6c, 0x61, - 0x74, 0x65, 0x64, 0x61, 0x74, 0x61, 0x22, 0x20, 0x7d, 0x20, 0x61, 0x6e, - 0x64, 0x20, 0x7b, 0x20, 0x22, 0x73, 0x65, 0x74, 0x74, 0x69, 0x6e, 0x67, - 0x73, 0x74, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x6e, 0x61, 0x6d, - 0x65, 0x22, 0x3a, 0x22, 0x73, 0x65, 0x74, 0x74, 0x69, 0x6e, 0x67, 0x73, - 0x64, 0x61, 0x74, 0x61, 0x22, 0x20, 0x7d, 0x0a, 0x0a, 0x20, 0x20, 0x20, - 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x6f, 0x6c, 0x65, 0x2e, 0x6c, 0x6f, 0x67, - 0x28, 0x27, 0x49, 0x6d, 0x70, 0x6f, 0x72, 0x74, 0x69, 0x6e, 0x67, 0x20, - 0x73, 0x61, 0x76, 0x65, 0x64, 0x20, 0x74, 0x65, 0x6d, 0x70, 0x6c, 0x61, - 0x74, 0x65, 0x73, 0x27, 0x29, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x6c, - 0x65, 0x74, 0x20, 0x69, 0x6d, 0x70, 0x6f, 0x72, 0x74, 0x65, 0x64, 0x54, - 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x73, 0x20, 0x3d, 0x20, 0x6c, - 0x6f, 0x63, 0x61, 0x6c, 0x5f, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, - 0x5f, 0x67, 0x65, 0x74, 0x44, 0x61, 0x74, 0x61, 0x41, 0x73, 0x4f, 0x62, - 0x6a, 0x65, 0x63, 0x74, 0x28, 0x27, 0x75, 0x73, 0x65, 0x72, 0x5f, 0x74, - 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x73, 0x27, 0x29, 0x0a, 0x0a, - 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, 0x69, 0x6d, 0x70, 0x6f, + 0x74, 0x65, 0x20, 0x3d, 0x20, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x28, + 0x7b, 0x20, 0x6e, 0x61, 0x6d, 0x65, 0x3a, 0x20, 0x27, 0x27, 0x2c, 0x20, + 0x74, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x3a, 0x20, 0x7b, 0x20, + 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x3a, 0x20, 0x7b, 0x7d, 0x2c, + 0x20, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x3a, 0x20, 0x7b, 0x7d, 0x20, + 0x7d, 0x20, 0x7d, 0x29, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x2f, 0x2f, + 0x20, 0x6c, 0x65, 0x74, 0x27, 0x73, 0x20, 0x69, 0x6d, 0x70, 0x6f, 0x72, + 0x74, 0x20, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x6c, 0x79, 0x20, 0x73, 0x61, + 0x76, 0x65, 0x64, 0x20, 0x74, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, + 0x73, 0x20, 0x61, 0x6e, 0x64, 0x20, 0x73, 0x65, 0x74, 0x74, 0x69, 0x6e, + 0x67, 0x73, 0x20, 0x69, 0x66, 0x20, 0x74, 0x68, 0x65, 0x72, 0x65, 0x20, + 0x61, 0x72, 0x65, 0x20, 0x61, 0x6e, 0x79, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x2f, 0x2f, 0x20, 0x75, 0x73, 0x65, 0x72, 0x20, 0x74, 0x65, 0x6d, 0x70, + 0x6c, 0x61, 0x74, 0x65, 0x73, 0x20, 0x61, 0x6e, 0x64, 0x20, 0x73, 0x65, + 0x74, 0x74, 0x69, 0x6e, 0x67, 0x73, 0x20, 0x61, 0x72, 0x65, 0x20, 0x73, + 0x74, 0x6f, 0x72, 0x65, 0x64, 0x20, 0x69, 0x6e, 0x20, 0x6f, 0x6e, 0x65, + 0x20, 0x6f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x2f, 0x2f, 0x20, 0x69, 0x6e, 0x20, 0x66, 0x6f, 0x72, 0x6d, 0x20, 0x6f, + 0x66, 0x20, 0x7b, 0x20, 0x22, 0x74, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, + 0x65, 0x6e, 0x61, 0x6d, 0x65, 0x22, 0x3a, 0x20, 0x22, 0x74, 0x65, 0x6d, + 0x70, 0x6c, 0x61, 0x74, 0x65, 0x64, 0x61, 0x74, 0x61, 0x22, 0x20, 0x7d, + 0x20, 0x61, 0x6e, 0x64, 0x20, 0x7b, 0x20, 0x22, 0x73, 0x65, 0x74, 0x74, + 0x69, 0x6e, 0x67, 0x73, 0x74, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, + 0x6e, 0x61, 0x6d, 0x65, 0x22, 0x3a, 0x22, 0x73, 0x65, 0x74, 0x74, 0x69, + 0x6e, 0x67, 0x73, 0x64, 0x61, 0x74, 0x61, 0x22, 0x20, 0x7d, 0x0a, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x6f, 0x6c, 0x65, 0x2e, + 0x6c, 0x6f, 0x67, 0x28, 0x27, 0x49, 0x6d, 0x70, 0x6f, 0x72, 0x74, 0x69, + 0x6e, 0x67, 0x20, 0x73, 0x61, 0x76, 0x65, 0x64, 0x20, 0x74, 0x65, 0x6d, + 0x70, 0x6c, 0x61, 0x74, 0x65, 0x73, 0x27, 0x29, 0x0a, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x6c, 0x65, 0x74, 0x20, 0x69, 0x6d, 0x70, 0x6f, 0x72, 0x74, + 0x65, 0x64, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x73, 0x20, + 0x3d, 0x20, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x5f, 0x73, 0x74, 0x6f, 0x72, + 0x61, 0x67, 0x65, 0x5f, 0x67, 0x65, 0x74, 0x44, 0x61, 0x74, 0x61, 0x41, + 0x73, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x28, 0x27, 0x75, 0x73, 0x65, + 0x72, 0x5f, 0x74, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x73, 0x27, + 0x29, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, 0x69, + 0x6d, 0x70, 0x6f, 0x72, 0x74, 0x65, 0x64, 0x54, 0x65, 0x6d, 0x70, 0x6c, + 0x61, 0x74, 0x65, 0x73, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x2f, 0x2f, 0x20, 0x73, 0x61, 0x76, 0x65, 0x64, 0x20, 0x74, + 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x73, 0x20, 0x77, 0x65, 0x72, + 0x65, 0x20, 0x73, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x66, 0x75, 0x6c, + 0x6c, 0x79, 0x20, 0x69, 0x6d, 0x70, 0x6f, 0x72, 0x74, 0x65, 0x64, 0x2e, + 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, + 0x6f, 0x6c, 0x65, 0x2e, 0x6c, 0x6f, 0x67, 0x28, 0x27, 0x50, 0x72, 0x6f, + 0x63, 0x65, 0x73, 0x73, 0x69, 0x6e, 0x67, 0x20, 0x73, 0x61, 0x76, 0x65, + 0x64, 0x20, 0x74, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x73, 0x20, + 0x61, 0x6e, 0x64, 0x20, 0x75, 0x70, 0x64, 0x61, 0x74, 0x69, 0x6e, 0x67, + 0x20, 0x64, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x20, 0x74, 0x65, 0x6d, + 0x70, 0x6c, 0x61, 0x74, 0x65, 0x27, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x2e, 0x76, 0x61, 0x6c, + 0x75, 0x65, 0x20, 0x3d, 0x20, 0x7b, 0x20, 0x2e, 0x2e, 0x2e, 0x70, 0x61, + 0x72, 0x61, 0x6d, 0x73, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x2c, 0x20, + 0x69, 0x6d, 0x61, 0x67, 0x65, 0x5f, 0x64, 0x61, 0x74, 0x61, 0x3a, 0x20, + 0x5b, 0x5d, 0x20, 0x7d, 0x3b, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x2f, 0x2f, 0x63, 0x6f, 0x6e, 0x73, 0x6f, 0x6c, 0x65, 0x2e, 0x6c, + 0x6f, 0x67, 0x28, 0x69, 0x6d, 0x70, 0x6f, 0x72, 0x74, 0x65, 0x64, 0x54, + 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x73, 0x29, 0x3b, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x73, 0x61, 0x76, 0x65, 0x64, 0x55, 0x73, + 0x65, 0x72, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x73, 0x2e, + 0x76, 0x61, 0x6c, 0x75, 0x65, 0x20, 0x3d, 0x20, 0x69, 0x6d, 0x70, 0x6f, 0x72, 0x74, 0x65, 0x64, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, - 0x73, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x2f, - 0x2f, 0x20, 0x73, 0x61, 0x76, 0x65, 0x64, 0x20, 0x74, 0x65, 0x6d, 0x70, - 0x6c, 0x61, 0x74, 0x65, 0x73, 0x20, 0x77, 0x65, 0x72, 0x65, 0x20, 0x73, - 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x66, 0x75, 0x6c, 0x6c, 0x79, 0x20, - 0x69, 0x6d, 0x70, 0x6f, 0x72, 0x74, 0x65, 0x64, 0x2e, 0x0a, 0x0a, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x6f, 0x6c, 0x65, - 0x2e, 0x6c, 0x6f, 0x67, 0x28, 0x27, 0x50, 0x72, 0x6f, 0x63, 0x65, 0x73, - 0x73, 0x69, 0x6e, 0x67, 0x20, 0x73, 0x61, 0x76, 0x65, 0x64, 0x20, 0x74, - 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x73, 0x20, 0x61, 0x6e, 0x64, - 0x20, 0x75, 0x70, 0x64, 0x61, 0x74, 0x69, 0x6e, 0x67, 0x20, 0x64, 0x65, - 0x66, 0x61, 0x75, 0x6c, 0x74, 0x20, 0x74, 0x65, 0x6d, 0x70, 0x6c, 0x61, - 0x74, 0x65, 0x27, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x70, - 0x61, 0x72, 0x61, 0x6d, 0x73, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x20, - 0x3d, 0x20, 0x7b, 0x20, 0x2e, 0x2e, 0x2e, 0x70, 0x61, 0x72, 0x61, 0x6d, - 0x73, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x2c, 0x20, 0x69, 0x6d, 0x61, - 0x67, 0x65, 0x5f, 0x64, 0x61, 0x74, 0x61, 0x3a, 0x20, 0x5b, 0x5d, 0x20, - 0x7d, 0x3b, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x2f, 0x2f, - 0x63, 0x6f, 0x6e, 0x73, 0x6f, 0x6c, 0x65, 0x2e, 0x6c, 0x6f, 0x67, 0x28, - 0x69, 0x6d, 0x70, 0x6f, 0x72, 0x74, 0x65, 0x64, 0x54, 0x65, 0x6d, 0x70, - 0x6c, 0x61, 0x74, 0x65, 0x73, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x73, 0x61, 0x76, 0x65, 0x64, 0x55, 0x73, 0x65, 0x72, 0x54, + 0x73, 0x3b, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x2f, 0x2f, + 0x6f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x20, 0x64, 0x65, 0x66, + 0x61, 0x75, 0x6c, 0x74, 0x20, 0x74, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, + 0x65, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x73, 0x61, 0x76, 0x65, + 0x64, 0x55, 0x73, 0x65, 0x72, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, + 0x65, 0x73, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x2e, 0x64, 0x65, 0x66, + 0x61, 0x75, 0x6c, 0x74, 0x20, 0x3d, 0x20, 0x7b, 0x20, 0x73, 0x65, 0x73, + 0x73, 0x69, 0x6f, 0x6e, 0x3a, 0x20, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, + 0x6e, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x2c, 0x20, 0x70, 0x61, 0x72, + 0x61, 0x6d, 0x73, 0x3a, 0x20, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x2e, + 0x76, 0x61, 0x6c, 0x75, 0x65, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x5f, 0x73, 0x74, 0x6f, 0x72, + 0x61, 0x67, 0x65, 0x5f, 0x73, 0x65, 0x74, 0x44, 0x61, 0x74, 0x61, 0x46, + 0x72, 0x6f, 0x6d, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x28, 0x27, 0x75, + 0x73, 0x65, 0x72, 0x5f, 0x74, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, + 0x73, 0x27, 0x2c, 0x20, 0x73, 0x61, 0x76, 0x65, 0x64, 0x55, 0x73, 0x65, + 0x72, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x73, 0x2e, 0x76, + 0x61, 0x6c, 0x75, 0x65, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x20, + 0x65, 0x6c, 0x73, 0x65, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x2f, 0x2f, 0x20, 0x6e, 0x6f, 0x20, 0x73, 0x61, 0x76, 0x65, 0x64, + 0x20, 0x74, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x73, 0x20, 0x64, + 0x65, 0x74, 0x65, 0x63, 0x74, 0x65, 0x64, 0x2e, 0x0a, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x6f, 0x6c, 0x65, 0x2e, + 0x6c, 0x6f, 0x67, 0x28, 0x27, 0x49, 0x6e, 0x69, 0x74, 0x69, 0x61, 0x6c, + 0x69, 0x7a, 0x69, 0x6e, 0x67, 0x20, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x53, + 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x20, 0x61, 0x6e, 0x64, 0x20, 0x73, + 0x61, 0x76, 0x69, 0x6e, 0x67, 0x20, 0x64, 0x65, 0x66, 0x61, 0x75, 0x6c, + 0x74, 0x20, 0x74, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x27, 0x29, + 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x73, 0x61, 0x76, 0x65, + 0x64, 0x55, 0x73, 0x65, 0x72, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, + 0x65, 0x73, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x20, 0x3d, 0x20, 0x7b, + 0x20, 0x22, 0x64, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x22, 0x3a, 0x20, + 0x7b, 0x20, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x3a, 0x20, 0x73, + 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, + 0x2c, 0x20, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x3a, 0x20, 0x70, 0x61, + 0x72, 0x61, 0x6d, 0x73, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x20, 0x7d, + 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x6c, 0x6f, 0x63, + 0x61, 0x6c, 0x5f, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x5f, 0x73, + 0x65, 0x74, 0x44, 0x61, 0x74, 0x61, 0x46, 0x72, 0x6f, 0x6d, 0x4f, 0x62, + 0x6a, 0x65, 0x63, 0x74, 0x28, 0x27, 0x75, 0x73, 0x65, 0x72, 0x5f, 0x74, + 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x73, 0x27, 0x2c, 0x20, 0x73, + 0x61, 0x76, 0x65, 0x64, 0x55, 0x73, 0x65, 0x72, 0x54, 0x65, 0x6d, 0x70, + 0x6c, 0x61, 0x74, 0x65, 0x73, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x29, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x75, 0x73, 0x65, + 0x72, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x52, 0x65, 0x73, + 0x65, 0x74, 0x54, 0x6f, 0x44, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x28, + 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, + 0x6e, 0x73, 0x6f, 0x6c, 0x65, 0x2e, 0x6c, 0x6f, 0x67, 0x28, 0x27, 0x52, + 0x65, 0x73, 0x65, 0x74, 0x74, 0x69, 0x6e, 0x67, 0x20, 0x74, 0x65, 0x6d, + 0x70, 0x6c, 0x61, 0x74, 0x65, 0x20, 0x74, 0x6f, 0x20, 0x64, 0x65, 0x66, + 0x61, 0x75, 0x6c, 0x74, 0x27, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x73, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x65, 0x64, 0x55, 0x73, 0x65, + 0x72, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x2e, 0x76, 0x61, + 0x6c, 0x75, 0x65, 0x2e, 0x6e, 0x61, 0x6d, 0x65, 0x20, 0x3d, 0x20, 0x27, + 0x64, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x27, 0x3b, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x73, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x65, 0x64, + 0x55, 0x73, 0x65, 0x72, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, + 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x20, + 0x3d, 0x20, 0x73, 0x61, 0x76, 0x65, 0x64, 0x55, 0x73, 0x65, 0x72, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x73, 0x2e, 0x76, 0x61, 0x6c, - 0x75, 0x65, 0x20, 0x3d, 0x20, 0x69, 0x6d, 0x70, 0x6f, 0x72, 0x74, 0x65, - 0x64, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x73, 0x3b, 0x0a, - 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x2f, 0x2f, 0x6f, 0x76, 0x65, - 0x72, 0x72, 0x69, 0x64, 0x65, 0x20, 0x64, 0x65, 0x66, 0x61, 0x75, 0x6c, - 0x74, 0x20, 0x74, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x0a, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x73, 0x61, 0x76, 0x65, 0x64, 0x55, 0x73, - 0x65, 0x72, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x73, 0x2e, - 0x76, 0x61, 0x6c, 0x75, 0x65, 0x2e, 0x64, 0x65, 0x66, 0x61, 0x75, 0x6c, - 0x74, 0x20, 0x3d, 0x20, 0x7b, 0x20, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, - 0x6e, 0x3a, 0x20, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x2e, 0x76, - 0x61, 0x6c, 0x75, 0x65, 0x2c, 0x20, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, - 0x3a, 0x20, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x2e, 0x76, 0x61, 0x6c, - 0x75, 0x65, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x6c, - 0x6f, 0x63, 0x61, 0x6c, 0x5f, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, - 0x5f, 0x73, 0x65, 0x74, 0x44, 0x61, 0x74, 0x61, 0x46, 0x72, 0x6f, 0x6d, + 0x75, 0x65, 0x5b, 0x27, 0x64, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x27, + 0x5d, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x75, + 0x73, 0x65, 0x72, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x41, + 0x70, 0x70, 0x6c, 0x79, 0x28, 0x74, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x2e, + 0x76, 0x61, 0x6c, 0x75, 0x65, 0x20, 0x3d, 0x20, 0x74, 0x2e, 0x64, 0x61, + 0x74, 0x61, 0x2e, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x3b, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, + 0x6e, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x20, 0x3d, 0x20, 0x7b, 0x20, + 0x2e, 0x2e, 0x2e, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x2e, 0x76, + 0x61, 0x6c, 0x75, 0x65, 0x2c, 0x20, 0x69, 0x6d, 0x61, 0x67, 0x65, 0x5f, + 0x73, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x65, 0x64, 0x3a, 0x20, 0x27, 0x27, + 0x20, 0x7d, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x70, 0x61, + 0x72, 0x61, 0x6d, 0x73, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x20, 0x3d, + 0x20, 0x74, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x70, 0x61, 0x72, 0x61, + 0x6d, 0x73, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x70, 0x61, + 0x72, 0x61, 0x6d, 0x73, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x20, 0x3d, + 0x20, 0x7b, 0x20, 0x2e, 0x2e, 0x2e, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, + 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x2c, 0x20, 0x69, 0x6d, 0x61, 0x67, + 0x65, 0x5f, 0x64, 0x61, 0x74, 0x61, 0x3a, 0x20, 0x5b, 0x5d, 0x20, 0x7d, + 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x75, 0x73, + 0x65, 0x72, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x52, 0x65, + 0x73, 0x65, 0x74, 0x54, 0x6f, 0x44, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, + 0x41, 0x6e, 0x64, 0x41, 0x70, 0x70, 0x6c, 0x79, 0x28, 0x29, 0x20, 0x7b, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x75, 0x73, 0x65, 0x72, 0x54, + 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x52, 0x65, 0x73, 0x65, 0x74, + 0x54, 0x6f, 0x44, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x28, 0x29, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x75, 0x73, 0x65, 0x72, 0x54, 0x65, + 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x41, 0x70, 0x70, 0x6c, 0x79, 0x28, + 0x73, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x65, 0x64, 0x55, 0x73, 0x65, 0x72, + 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x2e, 0x76, 0x61, 0x6c, + 0x75, 0x65, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, + 0x75, 0x73, 0x65, 0x72, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, + 0x4c, 0x6f, 0x61, 0x64, 0x41, 0x6e, 0x64, 0x41, 0x70, 0x70, 0x6c, 0x79, + 0x41, 0x75, 0x74, 0x6f, 0x73, 0x61, 0x76, 0x65, 0x64, 0x28, 0x29, 0x20, + 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x2f, 0x2f, 0x20, 0x67, + 0x65, 0x74, 0x20, 0x61, 0x75, 0x74, 0x6f, 0x73, 0x61, 0x76, 0x65, 0x64, + 0x20, 0x6c, 0x61, 0x73, 0x74, 0x20, 0x75, 0x73, 0x65, 0x64, 0x20, 0x74, + 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x6c, 0x65, 0x74, 0x20, 0x6c, 0x61, 0x73, 0x74, 0x55, 0x73, + 0x65, 0x64, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x20, 0x3d, + 0x20, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x5f, 0x73, 0x74, 0x6f, 0x72, 0x61, + 0x67, 0x65, 0x5f, 0x67, 0x65, 0x74, 0x44, 0x61, 0x74, 0x61, 0x41, 0x73, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x28, 0x27, 0x75, 0x73, 0x65, 0x72, - 0x5f, 0x74, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x73, 0x27, 0x2c, - 0x20, 0x73, 0x61, 0x76, 0x65, 0x64, 0x55, 0x73, 0x65, 0x72, 0x54, 0x65, - 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x73, 0x2e, 0x76, 0x61, 0x6c, 0x75, - 0x65, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x20, 0x65, 0x6c, 0x73, - 0x65, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x2f, 0x2f, - 0x20, 0x6e, 0x6f, 0x20, 0x73, 0x61, 0x76, 0x65, 0x64, 0x20, 0x74, 0x65, - 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x73, 0x20, 0x64, 0x65, 0x74, 0x65, - 0x63, 0x74, 0x65, 0x64, 0x2e, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x6f, 0x6c, 0x65, 0x2e, 0x6c, 0x6f, 0x67, - 0x28, 0x27, 0x49, 0x6e, 0x69, 0x74, 0x69, 0x61, 0x6c, 0x69, 0x7a, 0x69, - 0x6e, 0x67, 0x20, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x53, 0x74, 0x6f, 0x72, - 0x61, 0x67, 0x65, 0x20, 0x61, 0x6e, 0x64, 0x20, 0x73, 0x61, 0x76, 0x69, - 0x6e, 0x67, 0x20, 0x64, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x20, 0x74, - 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x27, 0x29, 0x0a, 0x0a, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x73, 0x61, 0x76, 0x65, 0x64, 0x55, 0x73, - 0x65, 0x72, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x73, 0x2e, - 0x76, 0x61, 0x6c, 0x75, 0x65, 0x20, 0x3d, 0x20, 0x7b, 0x20, 0x22, 0x64, - 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x22, 0x3a, 0x20, 0x7b, 0x20, 0x73, - 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x3a, 0x20, 0x73, 0x65, 0x73, 0x73, - 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x2c, 0x20, 0x70, - 0x61, 0x72, 0x61, 0x6d, 0x73, 0x3a, 0x20, 0x70, 0x61, 0x72, 0x61, 0x6d, - 0x73, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x20, 0x7d, 0x20, 0x7d, 0x0a, + 0x5f, 0x74, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x73, 0x5f, 0x6c, + 0x61, 0x73, 0x74, 0x27, 0x29, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x69, 0x66, 0x20, 0x28, 0x6c, 0x61, 0x73, 0x74, 0x55, 0x73, 0x65, + 0x64, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x29, 0x20, 0x7b, + 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, + 0x6e, 0x73, 0x6f, 0x6c, 0x65, 0x2e, 0x6c, 0x6f, 0x67, 0x28, 0x27, 0x41, + 0x75, 0x74, 0x6f, 0x73, 0x61, 0x76, 0x65, 0x64, 0x20, 0x74, 0x65, 0x6d, + 0x70, 0x6c, 0x61, 0x74, 0x65, 0x20, 0x66, 0x6f, 0x75, 0x6e, 0x64, 0x2c, + 0x20, 0x72, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x69, 0x6e, 0x67, 0x27, 0x29, + 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x73, 0x65, + 0x6c, 0x65, 0x63, 0x74, 0x65, 0x64, 0x55, 0x73, 0x65, 0x72, 0x54, 0x65, + 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, + 0x20, 0x3d, 0x20, 0x6c, 0x61, 0x73, 0x74, 0x55, 0x73, 0x65, 0x64, 0x54, + 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x65, 0x6c, + 0x73, 0x65, 0x20, 0x7b, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x6f, 0x6c, 0x65, 0x2e, 0x6c, 0x6f, + 0x67, 0x28, 0x27, 0x4e, 0x6f, 0x20, 0x61, 0x75, 0x74, 0x6f, 0x73, 0x61, + 0x76, 0x65, 0x64, 0x20, 0x74, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, + 0x20, 0x66, 0x6f, 0x75, 0x6e, 0x64, 0x2c, 0x20, 0x75, 0x73, 0x69, 0x6e, + 0x67, 0x20, 0x64, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x20, 0x74, 0x65, + 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x27, 0x29, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x2f, 0x2f, 0x20, 0x6e, 0x6f, 0x20, 0x61, + 0x75, 0x74, 0x6f, 0x73, 0x61, 0x76, 0x65, 0x64, 0x20, 0x6c, 0x61, 0x73, + 0x74, 0x20, 0x75, 0x73, 0x65, 0x64, 0x20, 0x74, 0x65, 0x6d, 0x70, 0x6c, + 0x61, 0x74, 0x65, 0x20, 0x77, 0x61, 0x73, 0x20, 0x66, 0x6f, 0x75, 0x6e, + 0x64, 0x2c, 0x20, 0x73, 0x6f, 0x20, 0x6c, 0x6f, 0x61, 0x64, 0x20, 0x66, + 0x72, 0x6f, 0x6d, 0x20, 0x64, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x2e, + 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x75, 0x73, + 0x65, 0x72, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x52, 0x65, + 0x73, 0x65, 0x74, 0x54, 0x6f, 0x44, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, + 0x28, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x6f, 0x6c, + 0x65, 0x2e, 0x6c, 0x6f, 0x67, 0x28, 0x27, 0x41, 0x70, 0x70, 0x6c, 0x79, + 0x69, 0x6e, 0x67, 0x20, 0x74, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, + 0x27, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x2f, 0x2f, 0x20, + 0x61, 0x6e, 0x64, 0x20, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x20, 0x69, + 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x20, 0x64, 0x61, 0x74, 0x61, + 0x20, 0x66, 0x72, 0x6f, 0x6d, 0x20, 0x74, 0x65, 0x6d, 0x70, 0x6c, 0x61, + 0x74, 0x65, 0x73, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x75, + 0x73, 0x65, 0x72, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x41, + 0x70, 0x70, 0x6c, 0x79, 0x28, 0x73, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x65, + 0x64, 0x55, 0x73, 0x65, 0x72, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, + 0x65, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x29, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x7d, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x2f, 0x2f, 0x63, 0x6f, + 0x6e, 0x73, 0x6f, 0x6c, 0x65, 0x2e, 0x6c, 0x6f, 0x67, 0x28, 0x73, 0x61, + 0x76, 0x65, 0x64, 0x55, 0x73, 0x65, 0x72, 0x54, 0x65, 0x6d, 0x70, 0x6c, + 0x61, 0x74, 0x65, 0x73, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x29, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x2f, 0x2f, 0x63, 0x6f, 0x6e, 0x73, 0x6f, 0x6c, + 0x65, 0x2e, 0x6c, 0x6f, 0x67, 0x28, 0x73, 0x65, 0x6c, 0x65, 0x63, 0x74, + 0x65, 0x64, 0x55, 0x73, 0x65, 0x72, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, + 0x74, 0x65, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x29, 0x0a, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, + 0x75, 0x73, 0x65, 0x72, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, + 0x41, 0x75, 0x74, 0x6f, 0x73, 0x61, 0x76, 0x65, 0x28, 0x29, 0x20, 0x7b, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x6f, + 0x6c, 0x65, 0x2e, 0x6c, 0x6f, 0x67, 0x28, 0x27, 0x54, 0x65, 0x6d, 0x70, + 0x6c, 0x61, 0x74, 0x65, 0x20, 0x41, 0x75, 0x74, 0x6f, 0x73, 0x61, 0x76, + 0x65, 0x2e, 0x2e, 0x2e, 0x27, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x69, 0x66, 0x20, 0x28, 0x73, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x65, + 0x64, 0x55, 0x73, 0x65, 0x72, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, + 0x65, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x2e, 0x6e, 0x61, 0x6d, 0x65, + 0x20, 0x3d, 0x3d, 0x20, 0x27, 0x64, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, + 0x27, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x2f, 0x2f, 0x20, 0x77, 0x65, 0x20, 0x64, 0x6f, 0x6e, 0x27, 0x74, + 0x20, 0x77, 0x61, 0x6e, 0x74, 0x20, 0x74, 0x6f, 0x20, 0x73, 0x61, 0x76, + 0x65, 0x20, 0x6f, 0x76, 0x65, 0x72, 0x20, 0x64, 0x65, 0x66, 0x61, 0x75, + 0x6c, 0x74, 0x20, 0x74, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x2c, + 0x20, 0x73, 0x6f, 0x20, 0x6c, 0x65, 0x74, 0x27, 0x73, 0x20, 0x63, 0x72, + 0x65, 0x61, 0x74, 0x65, 0x20, 0x61, 0x20, 0x6e, 0x65, 0x77, 0x20, 0x6f, + 0x6e, 0x65, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x6c, + 0x65, 0x74, 0x20, 0x6e, 0x65, 0x77, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, + 0x74, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x20, 0x3d, 0x20, 0x27, 0x55, 0x73, + 0x65, 0x72, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x2d, 0x27, + 0x20, 0x2b, 0x20, 0x44, 0x61, 0x74, 0x65, 0x2e, 0x6e, 0x6f, 0x77, 0x28, + 0x29, 0x2e, 0x74, 0x6f, 0x53, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x28, 0x29, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x6c, 0x65, 0x74, + 0x20, 0x6e, 0x65, 0x77, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, + 0x20, 0x3d, 0x20, 0x7b, 0x20, 0x27, 0x6e, 0x61, 0x6d, 0x65, 0x27, 0x3a, + 0x20, 0x6e, 0x65, 0x77, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, + 0x4e, 0x61, 0x6d, 0x65, 0x2c, 0x20, 0x27, 0x64, 0x61, 0x74, 0x61, 0x27, + 0x3a, 0x20, 0x7b, 0x20, 0x27, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, + 0x27, 0x3a, 0x20, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x2e, 0x76, + 0x61, 0x6c, 0x75, 0x65, 0x2c, 0x20, 0x27, 0x70, 0x61, 0x72, 0x61, 0x6d, + 0x73, 0x27, 0x3a, 0x20, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x2e, 0x76, + 0x61, 0x6c, 0x75, 0x65, 0x20, 0x7d, 0x20, 0x7d, 0x0a, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x6f, 0x6c, + 0x65, 0x2e, 0x6c, 0x6f, 0x67, 0x28, 0x27, 0x53, 0x61, 0x76, 0x69, 0x6e, + 0x67, 0x20, 0x61, 0x73, 0x20, 0x27, 0x20, 0x2b, 0x20, 0x6e, 0x65, 0x77, + 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x4e, 0x61, 0x6d, 0x65, + 0x29, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x2f, + 0x2f, 0x20, 0x73, 0x61, 0x76, 0x65, 0x20, 0x69, 0x6e, 0x20, 0x74, 0x68, + 0x65, 0x20, 0x61, 0x75, 0x74, 0x6f, 0x73, 0x61, 0x76, 0x65, 0x20, 0x73, + 0x6c, 0x6f, 0x74, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x5f, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, + 0x65, 0x5f, 0x73, 0x65, 0x74, 0x44, 0x61, 0x74, 0x61, 0x46, 0x72, 0x6f, + 0x6d, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x28, 0x27, 0x75, 0x73, 0x65, + 0x72, 0x5f, 0x74, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x73, 0x5f, + 0x6c, 0x61, 0x73, 0x74, 0x27, 0x2c, 0x20, 0x6e, 0x65, 0x77, 0x54, 0x65, + 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x29, 0x0a, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x2f, 0x2f, 0x20, 0x61, 0x6e, 0x64, 0x20, + 0x6c, 0x6f, 0x61, 0x64, 0x20, 0x69, 0x74, 0x20, 0x62, 0x61, 0x63, 0x6b, + 0x20, 0x61, 0x6e, 0x64, 0x20, 0x61, 0x70, 0x70, 0x6c, 0x79, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x75, 0x73, 0x65, 0x72, 0x54, + 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x4c, 0x6f, 0x61, 0x64, 0x41, + 0x6e, 0x64, 0x41, 0x70, 0x70, 0x6c, 0x79, 0x41, 0x75, 0x74, 0x6f, 0x73, + 0x61, 0x76, 0x65, 0x64, 0x28, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x7d, 0x20, 0x65, 0x6c, 0x73, 0x65, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x5f, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x5f, 0x73, 0x65, 0x74, 0x44, 0x61, 0x74, 0x61, 0x46, 0x72, 0x6f, 0x6d, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x28, 0x27, 0x75, 0x73, 0x65, 0x72, 0x5f, 0x74, 0x65, 0x6d, 0x70, - 0x6c, 0x61, 0x74, 0x65, 0x73, 0x27, 0x2c, 0x20, 0x73, 0x61, 0x76, 0x65, - 0x64, 0x55, 0x73, 0x65, 0x72, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, - 0x65, 0x73, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x29, 0x0a, 0x20, 0x20, - 0x20, 0x20, 0x7d, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x66, 0x75, 0x6e, - 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x75, 0x73, 0x65, 0x72, 0x54, 0x65, - 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x52, 0x65, 0x73, 0x65, 0x74, 0x54, - 0x6f, 0x44, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x28, 0x29, 0x20, 0x7b, - 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x6f, - 0x6c, 0x65, 0x2e, 0x6c, 0x6f, 0x67, 0x28, 0x27, 0x52, 0x65, 0x73, 0x65, - 0x74, 0x74, 0x69, 0x6e, 0x67, 0x20, 0x74, 0x65, 0x6d, 0x70, 0x6c, 0x61, - 0x74, 0x65, 0x20, 0x74, 0x6f, 0x20, 0x64, 0x65, 0x66, 0x61, 0x75, 0x6c, - 0x74, 0x27, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x73, 0x65, - 0x6c, 0x65, 0x63, 0x74, 0x65, 0x64, 0x55, 0x73, 0x65, 0x72, 0x54, 0x65, - 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, - 0x2e, 0x6e, 0x61, 0x6d, 0x65, 0x20, 0x3d, 0x20, 0x27, 0x64, 0x65, 0x66, - 0x61, 0x75, 0x6c, 0x74, 0x27, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x73, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x65, 0x64, 0x55, 0x73, 0x65, - 0x72, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x2e, 0x76, 0x61, - 0x6c, 0x75, 0x65, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x20, 0x3d, 0x20, 0x73, - 0x61, 0x76, 0x65, 0x64, 0x55, 0x73, 0x65, 0x72, 0x54, 0x65, 0x6d, 0x70, - 0x6c, 0x61, 0x74, 0x65, 0x73, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x5b, - 0x27, 0x64, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x27, 0x5d, 0x3b, 0x0a, - 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x66, - 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x75, 0x73, 0x65, 0x72, - 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x41, 0x70, 0x70, 0x6c, - 0x79, 0x28, 0x74, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x6c, 0x61, 0x74, 0x65, 0x73, 0x5f, 0x6c, 0x61, 0x73, 0x74, 0x27, 0x2c, + 0x20, 0x7b, 0x20, 0x27, 0x6e, 0x61, 0x6d, 0x65, 0x27, 0x3a, 0x20, 0x73, + 0x65, 0x6c, 0x65, 0x63, 0x74, 0x65, 0x64, 0x55, 0x73, 0x65, 0x72, 0x54, + 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x2e, 0x76, 0x61, 0x6c, 0x75, + 0x65, 0x2e, 0x6e, 0x61, 0x6d, 0x65, 0x2c, 0x20, 0x27, 0x64, 0x61, 0x74, + 0x61, 0x27, 0x3a, 0x20, 0x7b, 0x20, 0x27, 0x73, 0x65, 0x73, 0x73, 0x69, + 0x6f, 0x6e, 0x27, 0x3a, 0x20, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, + 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x2c, 0x20, 0x27, 0x70, 0x61, 0x72, + 0x61, 0x6d, 0x73, 0x27, 0x3a, 0x20, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, + 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x20, 0x7d, 0x20, 0x7d, 0x29, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x7d, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x6f, + 0x6c, 0x65, 0x2e, 0x6c, 0x6f, 0x67, 0x28, 0x27, 0x43, 0x68, 0x65, 0x63, + 0x6b, 0x69, 0x6e, 0x67, 0x20, 0x66, 0x6f, 0x72, 0x20, 0x61, 0x75, 0x74, + 0x6f, 0x73, 0x61, 0x76, 0x65, 0x64, 0x20, 0x6c, 0x61, 0x73, 0x74, 0x20, + 0x75, 0x73, 0x65, 0x64, 0x20, 0x74, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, + 0x65, 0x27, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x75, 0x73, 0x65, 0x72, + 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x4c, 0x6f, 0x61, 0x64, + 0x41, 0x6e, 0x64, 0x41, 0x70, 0x70, 0x6c, 0x79, 0x41, 0x75, 0x74, 0x6f, + 0x73, 0x61, 0x76, 0x65, 0x64, 0x28, 0x29, 0x0a, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x2f, 0x2a, 0x20, 0x45, 0x4e, 0x44, 0x3a, 0x20, 0x53, 0x75, 0x70, + 0x70, 0x6f, 0x72, 0x74, 0x20, 0x66, 0x6f, 0x72, 0x20, 0x73, 0x74, 0x6f, + 0x72, 0x69, 0x6e, 0x67, 0x20, 0x70, 0x72, 0x6f, 0x6d, 0x70, 0x74, 0x20, + 0x74, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x73, 0x20, 0x61, 0x6e, + 0x64, 0x20, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x73, + 0x20, 0x69, 0x6e, 0x20, 0x62, 0x72, 0x6f, 0x77, 0x73, 0x65, 0x72, 0x73, + 0x20, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, + 0x65, 0x20, 0x2a, 0x2f, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, + 0x6e, 0x73, 0x74, 0x20, 0x6c, 0x6c, 0x61, 0x6d, 0x61, 0x53, 0x74, 0x61, + 0x74, 0x73, 0x20, 0x3d, 0x20, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x28, + 0x6e, 0x75, 0x6c, 0x6c, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, + 0x6e, 0x73, 0x74, 0x20, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, + 0x65, 0x72, 0x20, 0x3d, 0x20, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x28, + 0x6e, 0x75, 0x6c, 0x6c, 0x29, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x2f, + 0x2f, 0x20, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x74, 0x6c, 0x79, 0x20, + 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6e, 0x67, 0x20, 0x61, + 0x20, 0x63, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x69, 0x6f, 0x6e, 0x3f, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x67, + 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6e, 0x67, 0x20, 0x3d, 0x20, + 0x63, 0x6f, 0x6d, 0x70, 0x75, 0x74, 0x65, 0x64, 0x28, 0x28, 0x29, 0x20, + 0x3d, 0x3e, 0x20, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, + 0x72, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x20, 0x21, 0x3d, 0x20, 0x6e, + 0x75, 0x6c, 0x6c, 0x29, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x2f, 0x2f, + 0x20, 0x68, 0x61, 0x73, 0x20, 0x74, 0x68, 0x65, 0x20, 0x75, 0x73, 0x65, + 0x72, 0x20, 0x73, 0x74, 0x61, 0x72, 0x74, 0x65, 0x64, 0x20, 0x61, 0x20, + 0x63, 0x68, 0x61, 0x74, 0x3f, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, + 0x6e, 0x73, 0x74, 0x20, 0x63, 0x68, 0x61, 0x74, 0x53, 0x74, 0x61, 0x72, + 0x74, 0x65, 0x64, 0x20, 0x3d, 0x20, 0x63, 0x6f, 0x6d, 0x70, 0x75, 0x74, + 0x65, 0x64, 0x28, 0x28, 0x29, 0x20, 0x3d, 0x3e, 0x20, 0x73, 0x65, 0x73, + 0x73, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x2e, 0x74, + 0x72, 0x61, 0x6e, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x2e, 0x6c, 0x65, + 0x6e, 0x67, 0x74, 0x68, 0x20, 0x3e, 0x20, 0x30, 0x29, 0x0a, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x74, 0x72, 0x61, + 0x6e, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x55, 0x70, 0x64, 0x61, 0x74, + 0x65, 0x20, 0x3d, 0x20, 0x28, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x63, 0x72, + 0x69, 0x70, 0x74, 0x29, 0x20, 0x3d, 0x3e, 0x20, 0x7b, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x2e, + 0x76, 0x61, 0x6c, 0x75, 0x65, 0x20, 0x3d, 0x20, 0x7b, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x2e, 0x2e, 0x2e, 0x73, 0x65, 0x73, + 0x73, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x2c, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x74, 0x72, 0x61, 0x6e, + 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x2f, 0x2f, 0x20, 0x73, 0x69, 0x6d, 0x70, 0x6c, 0x65, 0x20, + 0x74, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x20, 0x72, 0x65, 0x70, + 0x6c, 0x61, 0x63, 0x65, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, + 0x73, 0x74, 0x20, 0x74, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x20, + 0x3d, 0x20, 0x28, 0x73, 0x74, 0x72, 0x2c, 0x20, 0x65, 0x78, 0x74, 0x72, + 0x61, 0x53, 0x65, 0x74, 0x74, 0x69, 0x6e, 0x67, 0x73, 0x29, 0x20, 0x3d, + 0x3e, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x6c, 0x65, + 0x74, 0x20, 0x73, 0x65, 0x74, 0x74, 0x69, 0x6e, 0x67, 0x73, 0x20, 0x3d, 0x20, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x61, 0x6c, - 0x75, 0x65, 0x20, 0x3d, 0x20, 0x74, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x2e, - 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x3b, 0x0a, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x2e, 0x76, - 0x61, 0x6c, 0x75, 0x65, 0x20, 0x3d, 0x20, 0x7b, 0x20, 0x2e, 0x2e, 0x2e, - 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x61, 0x6c, 0x75, - 0x65, 0x2c, 0x20, 0x69, 0x6d, 0x61, 0x67, 0x65, 0x5f, 0x73, 0x65, 0x6c, - 0x65, 0x63, 0x74, 0x65, 0x64, 0x3a, 0x20, 0x27, 0x27, 0x20, 0x7d, 0x3b, - 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x70, 0x61, 0x72, 0x61, 0x6d, - 0x73, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x20, 0x3d, 0x20, 0x74, 0x2e, - 0x64, 0x61, 0x74, 0x61, 0x2e, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x3b, - 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x70, 0x61, 0x72, 0x61, 0x6d, - 0x73, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x20, 0x3d, 0x20, 0x7b, 0x20, - 0x2e, 0x2e, 0x2e, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x2e, 0x76, 0x61, - 0x6c, 0x75, 0x65, 0x2c, 0x20, 0x69, 0x6d, 0x61, 0x67, 0x65, 0x5f, 0x64, - 0x61, 0x74, 0x61, 0x3a, 0x20, 0x5b, 0x5d, 0x20, 0x7d, 0x3b, 0x0a, 0x20, - 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x66, 0x75, - 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x75, 0x73, 0x65, 0x72, 0x54, - 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x52, 0x65, 0x73, 0x65, 0x74, - 0x54, 0x6f, 0x44, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x41, 0x6e, 0x64, - 0x41, 0x70, 0x70, 0x6c, 0x79, 0x28, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x75, 0x73, 0x65, 0x72, 0x54, 0x65, 0x6d, 0x70, - 0x6c, 0x61, 0x74, 0x65, 0x52, 0x65, 0x73, 0x65, 0x74, 0x54, 0x6f, 0x44, - 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x28, 0x29, 0x0a, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x75, 0x73, 0x65, 0x72, 0x54, 0x65, 0x6d, 0x70, 0x6c, - 0x61, 0x74, 0x65, 0x41, 0x70, 0x70, 0x6c, 0x79, 0x28, 0x73, 0x65, 0x6c, - 0x65, 0x63, 0x74, 0x65, 0x64, 0x55, 0x73, 0x65, 0x72, 0x54, 0x65, 0x6d, - 0x70, 0x6c, 0x61, 0x74, 0x65, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x29, - 0x0a, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, - 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x75, 0x73, 0x65, - 0x72, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x4c, 0x6f, 0x61, - 0x64, 0x41, 0x6e, 0x64, 0x41, 0x70, 0x70, 0x6c, 0x79, 0x41, 0x75, 0x74, - 0x6f, 0x73, 0x61, 0x76, 0x65, 0x64, 0x28, 0x29, 0x20, 0x7b, 0x0a, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x2f, 0x2f, 0x20, 0x67, 0x65, 0x74, 0x20, - 0x61, 0x75, 0x74, 0x6f, 0x73, 0x61, 0x76, 0x65, 0x64, 0x20, 0x6c, 0x61, - 0x73, 0x74, 0x20, 0x75, 0x73, 0x65, 0x64, 0x20, 0x74, 0x65, 0x6d, 0x70, - 0x6c, 0x61, 0x74, 0x65, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x6c, - 0x65, 0x74, 0x20, 0x6c, 0x61, 0x73, 0x74, 0x55, 0x73, 0x65, 0x64, 0x54, - 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x20, 0x3d, 0x20, 0x6c, 0x6f, - 0x63, 0x61, 0x6c, 0x5f, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x5f, - 0x67, 0x65, 0x74, 0x44, 0x61, 0x74, 0x61, 0x41, 0x73, 0x4f, 0x62, 0x6a, - 0x65, 0x63, 0x74, 0x28, 0x27, 0x75, 0x73, 0x65, 0x72, 0x5f, 0x74, 0x65, - 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x73, 0x5f, 0x6c, 0x61, 0x73, 0x74, - 0x27, 0x29, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, - 0x20, 0x28, 0x6c, 0x61, 0x73, 0x74, 0x55, 0x73, 0x65, 0x64, 0x54, 0x65, - 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x29, 0x20, 0x7b, 0x0a, 0x0a, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x6f, - 0x6c, 0x65, 0x2e, 0x6c, 0x6f, 0x67, 0x28, 0x27, 0x41, 0x75, 0x74, 0x6f, - 0x73, 0x61, 0x76, 0x65, 0x64, 0x20, 0x74, 0x65, 0x6d, 0x70, 0x6c, 0x61, - 0x74, 0x65, 0x20, 0x66, 0x6f, 0x75, 0x6e, 0x64, 0x2c, 0x20, 0x72, 0x65, - 0x73, 0x74, 0x6f, 0x72, 0x69, 0x6e, 0x67, 0x27, 0x29, 0x0a, 0x0a, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x73, 0x65, 0x6c, 0x65, 0x63, - 0x74, 0x65, 0x64, 0x55, 0x73, 0x65, 0x72, 0x54, 0x65, 0x6d, 0x70, 0x6c, - 0x61, 0x74, 0x65, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x20, 0x3d, 0x20, - 0x6c, 0x61, 0x73, 0x74, 0x55, 0x73, 0x65, 0x64, 0x54, 0x65, 0x6d, 0x70, - 0x6c, 0x61, 0x74, 0x65, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, - 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x65, 0x6c, 0x73, 0x65, 0x20, - 0x7b, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, - 0x6f, 0x6e, 0x73, 0x6f, 0x6c, 0x65, 0x2e, 0x6c, 0x6f, 0x67, 0x28, 0x27, - 0x4e, 0x6f, 0x20, 0x61, 0x75, 0x74, 0x6f, 0x73, 0x61, 0x76, 0x65, 0x64, - 0x20, 0x74, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x20, 0x66, 0x6f, - 0x75, 0x6e, 0x64, 0x2c, 0x20, 0x75, 0x73, 0x69, 0x6e, 0x67, 0x20, 0x64, - 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x20, 0x74, 0x65, 0x6d, 0x70, 0x6c, - 0x61, 0x74, 0x65, 0x27, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x2f, 0x2f, 0x20, 0x6e, 0x6f, 0x20, 0x61, 0x75, 0x74, 0x6f, - 0x73, 0x61, 0x76, 0x65, 0x64, 0x20, 0x6c, 0x61, 0x73, 0x74, 0x20, 0x75, - 0x73, 0x65, 0x64, 0x20, 0x74, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, - 0x20, 0x77, 0x61, 0x73, 0x20, 0x66, 0x6f, 0x75, 0x6e, 0x64, 0x2c, 0x20, - 0x73, 0x6f, 0x20, 0x6c, 0x6f, 0x61, 0x64, 0x20, 0x66, 0x72, 0x6f, 0x6d, - 0x20, 0x64, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x2e, 0x0a, 0x0a, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x75, 0x73, 0x65, 0x72, 0x54, - 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x52, 0x65, 0x73, 0x65, 0x74, - 0x54, 0x6f, 0x44, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x28, 0x29, 0x0a, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x0a, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x6f, 0x6c, 0x65, 0x2e, 0x6c, - 0x6f, 0x67, 0x28, 0x27, 0x41, 0x70, 0x70, 0x6c, 0x79, 0x69, 0x6e, 0x67, - 0x20, 0x74, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x27, 0x29, 0x0a, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x2f, 0x2f, 0x20, 0x61, 0x6e, 0x64, - 0x20, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x20, 0x69, 0x6e, 0x74, 0x65, - 0x72, 0x6e, 0x61, 0x6c, 0x20, 0x64, 0x61, 0x74, 0x61, 0x20, 0x66, 0x72, - 0x6f, 0x6d, 0x20, 0x74, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x73, - 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x75, 0x73, 0x65, 0x72, - 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x41, 0x70, 0x70, 0x6c, - 0x79, 0x28, 0x73, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x65, 0x64, 0x55, 0x73, - 0x65, 0x72, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x2e, 0x76, - 0x61, 0x6c, 0x75, 0x65, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, - 0x0a, 0x20, 0x20, 0x20, 0x20, 0x2f, 0x2f, 0x63, 0x6f, 0x6e, 0x73, 0x6f, - 0x6c, 0x65, 0x2e, 0x6c, 0x6f, 0x67, 0x28, 0x73, 0x61, 0x76, 0x65, 0x64, - 0x55, 0x73, 0x65, 0x72, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, - 0x73, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x29, 0x0a, 0x20, 0x20, 0x20, - 0x20, 0x2f, 0x2f, 0x63, 0x6f, 0x6e, 0x73, 0x6f, 0x6c, 0x65, 0x2e, 0x6c, - 0x6f, 0x67, 0x28, 0x73, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x65, 0x64, 0x55, - 0x73, 0x65, 0x72, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x2e, - 0x76, 0x61, 0x6c, 0x75, 0x65, 0x29, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, - 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x75, 0x73, 0x65, - 0x72, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x41, 0x75, 0x74, - 0x6f, 0x73, 0x61, 0x76, 0x65, 0x28, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x6f, 0x6c, 0x65, 0x2e, - 0x6c, 0x6f, 0x67, 0x28, 0x27, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, - 0x65, 0x20, 0x41, 0x75, 0x74, 0x6f, 0x73, 0x61, 0x76, 0x65, 0x2e, 0x2e, - 0x2e, 0x27, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, - 0x20, 0x28, 0x73, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x65, 0x64, 0x55, 0x73, - 0x65, 0x72, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x2e, 0x76, - 0x61, 0x6c, 0x75, 0x65, 0x2e, 0x6e, 0x61, 0x6d, 0x65, 0x20, 0x3d, 0x3d, - 0x20, 0x27, 0x64, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x27, 0x29, 0x20, - 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x2f, 0x2f, - 0x20, 0x77, 0x65, 0x20, 0x64, 0x6f, 0x6e, 0x27, 0x74, 0x20, 0x77, 0x61, - 0x6e, 0x74, 0x20, 0x74, 0x6f, 0x20, 0x73, 0x61, 0x76, 0x65, 0x20, 0x6f, - 0x76, 0x65, 0x72, 0x20, 0x64, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x20, - 0x74, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x2c, 0x20, 0x73, 0x6f, - 0x20, 0x6c, 0x65, 0x74, 0x27, 0x73, 0x20, 0x63, 0x72, 0x65, 0x61, 0x74, - 0x65, 0x20, 0x61, 0x20, 0x6e, 0x65, 0x77, 0x20, 0x6f, 0x6e, 0x65, 0x0a, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x6c, 0x65, 0x74, 0x20, - 0x6e, 0x65, 0x77, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x4e, - 0x61, 0x6d, 0x65, 0x20, 0x3d, 0x20, 0x27, 0x55, 0x73, 0x65, 0x72, 0x54, - 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x2d, 0x27, 0x20, 0x2b, 0x20, - 0x44, 0x61, 0x74, 0x65, 0x2e, 0x6e, 0x6f, 0x77, 0x28, 0x29, 0x2e, 0x74, - 0x6f, 0x53, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x28, 0x29, 0x0a, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x6c, 0x65, 0x74, 0x20, 0x6e, 0x65, - 0x77, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x20, 0x3d, 0x20, - 0x7b, 0x20, 0x27, 0x6e, 0x61, 0x6d, 0x65, 0x27, 0x3a, 0x20, 0x6e, 0x65, - 0x77, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x4e, 0x61, 0x6d, - 0x65, 0x2c, 0x20, 0x27, 0x64, 0x61, 0x74, 0x61, 0x27, 0x3a, 0x20, 0x7b, - 0x20, 0x27, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x27, 0x3a, 0x20, - 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x61, 0x6c, 0x75, - 0x65, 0x2c, 0x20, 0x27, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x27, 0x3a, - 0x20, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x2e, 0x76, 0x61, 0x6c, 0x75, - 0x65, 0x20, 0x7d, 0x20, 0x7d, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x6f, 0x6c, 0x65, 0x2e, 0x6c, - 0x6f, 0x67, 0x28, 0x27, 0x53, 0x61, 0x76, 0x69, 0x6e, 0x67, 0x20, 0x61, - 0x73, 0x20, 0x27, 0x20, 0x2b, 0x20, 0x6e, 0x65, 0x77, 0x54, 0x65, 0x6d, - 0x70, 0x6c, 0x61, 0x74, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x29, 0x0a, 0x0a, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x2f, 0x2f, 0x20, 0x73, - 0x61, 0x76, 0x65, 0x20, 0x69, 0x6e, 0x20, 0x74, 0x68, 0x65, 0x20, 0x61, - 0x75, 0x74, 0x6f, 0x73, 0x61, 0x76, 0x65, 0x20, 0x73, 0x6c, 0x6f, 0x74, - 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x6c, 0x6f, 0x63, - 0x61, 0x6c, 0x5f, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x5f, 0x73, - 0x65, 0x74, 0x44, 0x61, 0x74, 0x61, 0x46, 0x72, 0x6f, 0x6d, 0x4f, 0x62, - 0x6a, 0x65, 0x63, 0x74, 0x28, 0x27, 0x75, 0x73, 0x65, 0x72, 0x5f, 0x74, - 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x73, 0x5f, 0x6c, 0x61, 0x73, - 0x74, 0x27, 0x2c, 0x20, 0x6e, 0x65, 0x77, 0x54, 0x65, 0x6d, 0x70, 0x6c, - 0x61, 0x74, 0x65, 0x29, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x2f, 0x2f, 0x20, 0x61, 0x6e, 0x64, 0x20, 0x6c, 0x6f, 0x61, - 0x64, 0x20, 0x69, 0x74, 0x20, 0x62, 0x61, 0x63, 0x6b, 0x20, 0x61, 0x6e, - 0x64, 0x20, 0x61, 0x70, 0x70, 0x6c, 0x79, 0x0a, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x75, 0x73, 0x65, 0x72, 0x54, 0x65, 0x6d, 0x70, - 0x6c, 0x61, 0x74, 0x65, 0x4c, 0x6f, 0x61, 0x64, 0x41, 0x6e, 0x64, 0x41, - 0x70, 0x70, 0x6c, 0x79, 0x41, 0x75, 0x74, 0x6f, 0x73, 0x61, 0x76, 0x65, - 0x64, 0x28, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x20, - 0x65, 0x6c, 0x73, 0x65, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x5f, 0x73, 0x74, 0x6f, - 0x72, 0x61, 0x67, 0x65, 0x5f, 0x73, 0x65, 0x74, 0x44, 0x61, 0x74, 0x61, - 0x46, 0x72, 0x6f, 0x6d, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x28, 0x27, - 0x75, 0x73, 0x65, 0x72, 0x5f, 0x74, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, - 0x65, 0x73, 0x5f, 0x6c, 0x61, 0x73, 0x74, 0x27, 0x2c, 0x20, 0x7b, 0x20, - 0x27, 0x6e, 0x61, 0x6d, 0x65, 0x27, 0x3a, 0x20, 0x73, 0x65, 0x6c, 0x65, - 0x63, 0x74, 0x65, 0x64, 0x55, 0x73, 0x65, 0x72, 0x54, 0x65, 0x6d, 0x70, - 0x6c, 0x61, 0x74, 0x65, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x2e, 0x6e, - 0x61, 0x6d, 0x65, 0x2c, 0x20, 0x27, 0x64, 0x61, 0x74, 0x61, 0x27, 0x3a, - 0x20, 0x7b, 0x20, 0x27, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x27, - 0x3a, 0x20, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x61, - 0x6c, 0x75, 0x65, 0x2c, 0x20, 0x27, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, - 0x27, 0x3a, 0x20, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x2e, 0x76, 0x61, - 0x6c, 0x75, 0x65, 0x20, 0x7d, 0x20, 0x7d, 0x29, 0x0a, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x0a, - 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x6f, 0x6c, 0x65, 0x2e, - 0x6c, 0x6f, 0x67, 0x28, 0x27, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x69, 0x6e, - 0x67, 0x20, 0x66, 0x6f, 0x72, 0x20, 0x61, 0x75, 0x74, 0x6f, 0x73, 0x61, - 0x76, 0x65, 0x64, 0x20, 0x6c, 0x61, 0x73, 0x74, 0x20, 0x75, 0x73, 0x65, - 0x64, 0x20, 0x74, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x27, 0x29, - 0x0a, 0x20, 0x20, 0x20, 0x20, 0x75, 0x73, 0x65, 0x72, 0x54, 0x65, 0x6d, - 0x70, 0x6c, 0x61, 0x74, 0x65, 0x4c, 0x6f, 0x61, 0x64, 0x41, 0x6e, 0x64, - 0x41, 0x70, 0x70, 0x6c, 0x79, 0x41, 0x75, 0x74, 0x6f, 0x73, 0x61, 0x76, - 0x65, 0x64, 0x28, 0x29, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x2f, 0x2a, - 0x20, 0x45, 0x4e, 0x44, 0x3a, 0x20, 0x53, 0x75, 0x70, 0x70, 0x6f, 0x72, - 0x74, 0x20, 0x66, 0x6f, 0x72, 0x20, 0x73, 0x74, 0x6f, 0x72, 0x69, 0x6e, - 0x67, 0x20, 0x70, 0x72, 0x6f, 0x6d, 0x70, 0x74, 0x20, 0x74, 0x65, 0x6d, - 0x70, 0x6c, 0x61, 0x74, 0x65, 0x73, 0x20, 0x61, 0x6e, 0x64, 0x20, 0x70, - 0x61, 0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x73, 0x20, 0x69, 0x6e, - 0x20, 0x62, 0x72, 0x6f, 0x77, 0x73, 0x65, 0x72, 0x73, 0x20, 0x4c, 0x6f, - 0x63, 0x61, 0x6c, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x20, 0x2a, - 0x2f, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, - 0x20, 0x6c, 0x6c, 0x61, 0x6d, 0x61, 0x53, 0x74, 0x61, 0x74, 0x73, 0x20, - 0x3d, 0x20, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x28, 0x6e, 0x75, 0x6c, - 0x6c, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, - 0x20, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x20, - 0x3d, 0x20, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x28, 0x6e, 0x75, 0x6c, - 0x6c, 0x29, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x2f, 0x2f, 0x20, 0x63, - 0x75, 0x72, 0x72, 0x65, 0x6e, 0x74, 0x6c, 0x79, 0x20, 0x67, 0x65, 0x6e, - 0x65, 0x72, 0x61, 0x74, 0x69, 0x6e, 0x67, 0x20, 0x61, 0x20, 0x63, 0x6f, - 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x69, 0x6f, 0x6e, 0x3f, 0x0a, 0x20, 0x20, - 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x67, 0x65, 0x6e, 0x65, - 0x72, 0x61, 0x74, 0x69, 0x6e, 0x67, 0x20, 0x3d, 0x20, 0x63, 0x6f, 0x6d, - 0x70, 0x75, 0x74, 0x65, 0x64, 0x28, 0x28, 0x29, 0x20, 0x3d, 0x3e, 0x20, - 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x2e, 0x76, - 0x61, 0x6c, 0x75, 0x65, 0x20, 0x21, 0x3d, 0x20, 0x6e, 0x75, 0x6c, 0x6c, - 0x29, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x2f, 0x2f, 0x20, 0x68, 0x61, - 0x73, 0x20, 0x74, 0x68, 0x65, 0x20, 0x75, 0x73, 0x65, 0x72, 0x20, 0x73, - 0x74, 0x61, 0x72, 0x74, 0x65, 0x64, 0x20, 0x61, 0x20, 0x63, 0x68, 0x61, - 0x74, 0x3f, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, - 0x20, 0x63, 0x68, 0x61, 0x74, 0x53, 0x74, 0x61, 0x72, 0x74, 0x65, 0x64, - 0x20, 0x3d, 0x20, 0x63, 0x6f, 0x6d, 0x70, 0x75, 0x74, 0x65, 0x64, 0x28, - 0x28, 0x29, 0x20, 0x3d, 0x3e, 0x20, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, - 0x6e, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x2e, 0x74, 0x72, 0x61, 0x6e, - 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x2e, 0x6c, 0x65, 0x6e, 0x67, 0x74, - 0x68, 0x20, 0x3e, 0x20, 0x30, 0x29, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, - 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x63, - 0x72, 0x69, 0x70, 0x74, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x20, 0x3d, - 0x20, 0x28, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, - 0x29, 0x20, 0x3d, 0x3e, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x61, 0x6c, - 0x75, 0x65, 0x20, 0x3d, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x2e, 0x2e, 0x2e, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, - 0x6e, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x2c, 0x0a, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x63, 0x72, - 0x69, 0x70, 0x74, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, - 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x2f, - 0x2f, 0x20, 0x73, 0x69, 0x6d, 0x70, 0x6c, 0x65, 0x20, 0x74, 0x65, 0x6d, - 0x70, 0x6c, 0x61, 0x74, 0x65, 0x20, 0x72, 0x65, 0x70, 0x6c, 0x61, 0x63, - 0x65, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, - 0x74, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x20, 0x3d, 0x20, 0x28, - 0x73, 0x74, 0x72, 0x2c, 0x20, 0x65, 0x78, 0x74, 0x72, 0x61, 0x53, 0x65, - 0x74, 0x74, 0x69, 0x6e, 0x67, 0x73, 0x29, 0x20, 0x3d, 0x3e, 0x20, 0x7b, - 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x6c, 0x65, 0x74, 0x20, 0x73, - 0x65, 0x74, 0x74, 0x69, 0x6e, 0x67, 0x73, 0x20, 0x3d, 0x20, 0x73, 0x65, - 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3b, - 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, 0x65, - 0x78, 0x74, 0x72, 0x61, 0x53, 0x65, 0x74, 0x74, 0x69, 0x6e, 0x67, 0x73, - 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x73, 0x65, 0x74, 0x74, 0x69, 0x6e, 0x67, 0x73, 0x20, 0x3d, 0x20, 0x7b, - 0x20, 0x2e, 0x2e, 0x2e, 0x73, 0x65, 0x74, 0x74, 0x69, 0x6e, 0x67, 0x73, - 0x2c, 0x20, 0x2e, 0x2e, 0x2e, 0x65, 0x78, 0x74, 0x72, 0x61, 0x53, 0x65, - 0x74, 0x74, 0x69, 0x6e, 0x67, 0x73, 0x20, 0x7d, 0x3b, 0x0a, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x53, 0x74, 0x72, 0x69, 0x6e, - 0x67, 0x28, 0x73, 0x74, 0x72, 0x29, 0x2e, 0x72, 0x65, 0x70, 0x6c, 0x61, - 0x63, 0x65, 0x41, 0x6c, 0x6c, 0x28, 0x2f, 0x5c, 0x7b, 0x5c, 0x7b, 0x28, - 0x2e, 0x2a, 0x3f, 0x29, 0x5c, 0x7d, 0x5c, 0x7d, 0x2f, 0x67, 0x2c, 0x20, - 0x28, 0x5f, 0x2c, 0x20, 0x6b, 0x65, 0x79, 0x29, 0x20, 0x3d, 0x3e, 0x20, - 0x74, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x28, 0x73, 0x65, 0x74, - 0x74, 0x69, 0x6e, 0x67, 0x73, 0x5b, 0x6b, 0x65, 0x79, 0x5d, 0x29, 0x29, - 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x0a, 0x20, 0x20, 0x20, - 0x20, 0x61, 0x73, 0x79, 0x6e, 0x63, 0x20, 0x66, 0x75, 0x6e, 0x63, 0x74, - 0x69, 0x6f, 0x6e, 0x20, 0x72, 0x75, 0x6e, 0x4c, 0x6c, 0x61, 0x6d, 0x61, - 0x28, 0x70, 0x72, 0x6f, 0x6d, 0x70, 0x74, 0x2c, 0x20, 0x6c, 0x6c, 0x61, - 0x6d, 0x61, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x2c, 0x20, 0x63, 0x68, - 0x61, 0x72, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, - 0x74, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x20, 0x3d, 0x20, - 0x5b, 0x5d, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, - 0x6e, 0x73, 0x74, 0x20, 0x68, 0x69, 0x73, 0x74, 0x6f, 0x72, 0x79, 0x20, - 0x3d, 0x20, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x61, - 0x6c, 0x75, 0x65, 0x2e, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x63, 0x72, 0x69, - 0x70, 0x74, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, - 0x20, 0x28, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, - 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x74, 0x68, 0x72, 0x6f, 0x77, 0x20, - 0x6e, 0x65, 0x77, 0x20, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x28, 0x22, 0x61, - 0x6c, 0x72, 0x65, 0x61, 0x64, 0x79, 0x20, 0x72, 0x75, 0x6e, 0x6e, 0x69, - 0x6e, 0x67, 0x22, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x74, - 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, - 0x20, 0x3d, 0x20, 0x6e, 0x65, 0x77, 0x20, 0x41, 0x62, 0x6f, 0x72, 0x74, - 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x28, 0x29, - 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x66, 0x6f, 0x72, 0x20, - 0x61, 0x77, 0x61, 0x69, 0x74, 0x20, 0x28, 0x63, 0x6f, 0x6e, 0x73, 0x74, - 0x20, 0x63, 0x68, 0x75, 0x6e, 0x6b, 0x20, 0x6f, 0x66, 0x20, 0x6c, 0x6c, + 0x75, 0x65, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, + 0x20, 0x28, 0x65, 0x78, 0x74, 0x72, 0x61, 0x53, 0x65, 0x74, 0x74, 0x69, + 0x6e, 0x67, 0x73, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x73, 0x65, 0x74, 0x74, 0x69, 0x6e, 0x67, 0x73, 0x20, + 0x3d, 0x20, 0x7b, 0x20, 0x2e, 0x2e, 0x2e, 0x73, 0x65, 0x74, 0x74, 0x69, + 0x6e, 0x67, 0x73, 0x2c, 0x20, 0x2e, 0x2e, 0x2e, 0x65, 0x78, 0x74, 0x72, + 0x61, 0x53, 0x65, 0x74, 0x74, 0x69, 0x6e, 0x67, 0x73, 0x20, 0x7d, 0x3b, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x53, 0x74, + 0x72, 0x69, 0x6e, 0x67, 0x28, 0x73, 0x74, 0x72, 0x29, 0x2e, 0x72, 0x65, + 0x70, 0x6c, 0x61, 0x63, 0x65, 0x41, 0x6c, 0x6c, 0x28, 0x2f, 0x5c, 0x7b, + 0x5c, 0x7b, 0x28, 0x2e, 0x2a, 0x3f, 0x29, 0x5c, 0x7d, 0x5c, 0x7d, 0x2f, + 0x67, 0x2c, 0x20, 0x28, 0x5f, 0x2c, 0x20, 0x6b, 0x65, 0x79, 0x29, 0x20, + 0x3d, 0x3e, 0x20, 0x74, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x28, + 0x73, 0x65, 0x74, 0x74, 0x69, 0x6e, 0x67, 0x73, 0x5b, 0x6b, 0x65, 0x79, + 0x5d, 0x29, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x61, 0x73, 0x79, 0x6e, 0x63, 0x20, 0x66, 0x75, + 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x72, 0x75, 0x6e, 0x4c, 0x6c, 0x61, 0x6d, 0x61, 0x28, 0x70, 0x72, 0x6f, 0x6d, 0x70, 0x74, 0x2c, 0x20, 0x6c, 0x6c, 0x61, 0x6d, 0x61, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x2c, - 0x20, 0x7b, 0x20, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, - 0x72, 0x3a, 0x20, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, - 0x72, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x20, 0x7d, 0x29, 0x29, 0x20, - 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, - 0x6e, 0x73, 0x74, 0x20, 0x64, 0x61, 0x74, 0x61, 0x20, 0x3d, 0x20, 0x63, - 0x68, 0x75, 0x6e, 0x6b, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x3b, 0x0a, 0x0a, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, - 0x64, 0x61, 0x74, 0x61, 0x2e, 0x73, 0x74, 0x6f, 0x70, 0x29, 0x20, 0x7b, - 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x77, - 0x68, 0x69, 0x6c, 0x65, 0x20, 0x28, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x75, 0x72, 0x72, 0x65, - 0x6e, 0x74, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x2e, 0x6c, - 0x65, 0x6e, 0x67, 0x74, 0x68, 0x20, 0x3e, 0x20, 0x30, 0x20, 0x26, 0x26, - 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x74, 0x4d, 0x65, 0x73, 0x73, - 0x61, 0x67, 0x65, 0x73, 0x5b, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x74, - 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x2e, 0x6c, 0x65, 0x6e, - 0x67, 0x74, 0x68, 0x20, 0x2d, 0x20, 0x31, 0x5d, 0x2e, 0x63, 0x6f, 0x6e, - 0x74, 0x65, 0x6e, 0x74, 0x2e, 0x6d, 0x61, 0x74, 0x63, 0x68, 0x28, 0x2f, - 0x5c, 0x6e, 0x24, 0x2f, 0x29, 0x20, 0x21, 0x3d, 0x20, 0x6e, 0x75, 0x6c, - 0x6c, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x63, 0x68, 0x61, 0x72, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x63, 0x75, 0x72, + 0x72, 0x65, 0x6e, 0x74, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, + 0x20, 0x3d, 0x20, 0x5b, 0x5d, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x68, 0x69, 0x73, 0x74, 0x6f, + 0x72, 0x79, 0x20, 0x3d, 0x20, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, + 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x2e, 0x74, 0x72, 0x61, 0x6e, 0x73, + 0x63, 0x72, 0x69, 0x70, 0x74, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x69, 0x66, 0x20, 0x28, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, + 0x6c, 0x65, 0x72, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x29, 0x20, 0x7b, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x74, 0x68, 0x72, + 0x6f, 0x77, 0x20, 0x6e, 0x65, 0x77, 0x20, 0x45, 0x72, 0x72, 0x6f, 0x72, + 0x28, 0x22, 0x61, 0x6c, 0x72, 0x65, 0x61, 0x64, 0x79, 0x20, 0x72, 0x75, + 0x6e, 0x6e, 0x69, 0x6e, 0x67, 0x22, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, + 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x2e, 0x76, 0x61, + 0x6c, 0x75, 0x65, 0x20, 0x3d, 0x20, 0x6e, 0x65, 0x77, 0x20, 0x41, 0x62, + 0x6f, 0x72, 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, + 0x72, 0x28, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x66, + 0x6f, 0x72, 0x20, 0x61, 0x77, 0x61, 0x69, 0x74, 0x20, 0x28, 0x63, 0x6f, + 0x6e, 0x73, 0x74, 0x20, 0x63, 0x68, 0x75, 0x6e, 0x6b, 0x20, 0x6f, 0x66, + 0x20, 0x6c, 0x6c, 0x61, 0x6d, 0x61, 0x28, 0x70, 0x72, 0x6f, 0x6d, 0x70, + 0x74, 0x2c, 0x20, 0x6c, 0x6c, 0x61, 0x6d, 0x61, 0x50, 0x61, 0x72, 0x61, + 0x6d, 0x73, 0x2c, 0x20, 0x7b, 0x20, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, + 0x6c, 0x6c, 0x65, 0x72, 0x3a, 0x20, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, + 0x6c, 0x6c, 0x65, 0x72, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x20, 0x7d, + 0x29, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x64, 0x61, 0x74, 0x61, 0x20, + 0x3d, 0x20, 0x63, 0x68, 0x75, 0x6e, 0x6b, 0x2e, 0x64, 0x61, 0x74, 0x61, + 0x3b, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, + 0x66, 0x20, 0x28, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x73, 0x74, 0x6f, 0x70, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x77, 0x68, 0x69, 0x6c, 0x65, 0x20, 0x28, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x75, + 0x72, 0x72, 0x65, 0x6e, 0x74, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, + 0x73, 0x2e, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x20, 0x3e, 0x20, 0x30, + 0x20, 0x26, 0x26, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x74, 0x4d, - 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x2e, 0x70, 0x6f, 0x70, 0x28, - 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x5b, 0x63, 0x75, 0x72, 0x72, + 0x65, 0x6e, 0x74, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x2e, + 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x20, 0x2d, 0x20, 0x31, 0x5d, 0x2e, + 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x2e, 0x6d, 0x61, 0x74, 0x63, + 0x68, 0x28, 0x2f, 0x5c, 0x6e, 0x24, 0x2f, 0x29, 0x20, 0x21, 0x3d, 0x20, + 0x6e, 0x75, 0x6c, 0x6c, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x75, 0x72, 0x72, 0x65, + 0x6e, 0x74, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x2e, 0x70, + 0x6f, 0x70, 0x28, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x63, 0x72, 0x69, + 0x70, 0x74, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x28, 0x5b, 0x2e, 0x2e, + 0x2e, 0x68, 0x69, 0x73, 0x74, 0x6f, 0x72, 0x79, 0x2c, 0x20, 0x5b, 0x63, + 0x68, 0x61, 0x72, 0x2c, 0x20, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x74, + 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x5d, 0x5d, 0x29, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, + 0x6e, 0x73, 0x6f, 0x6c, 0x65, 0x2e, 0x6c, 0x6f, 0x67, 0x28, 0x22, 0x43, + 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x66, 0x69, + 0x6e, 0x69, 0x73, 0x68, 0x65, 0x64, 0x3a, 0x20, 0x27, 0x22, 0x2c, 0x20, + 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x74, 0x4d, 0x65, 0x73, 0x73, 0x61, + 0x67, 0x65, 0x73, 0x2e, 0x6d, 0x61, 0x70, 0x28, 0x6d, 0x73, 0x67, 0x20, + 0x3d, 0x3e, 0x20, 0x6d, 0x73, 0x67, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x65, + 0x6e, 0x74, 0x29, 0x2e, 0x6a, 0x6f, 0x69, 0x6e, 0x28, 0x27, 0x27, 0x29, + 0x2c, 0x20, 0x22, 0x27, 0x2c, 0x20, 0x73, 0x75, 0x6d, 0x6d, 0x61, 0x72, + 0x79, 0x3a, 0x20, 0x22, 0x2c, 0x20, 0x64, 0x61, 0x74, 0x61, 0x29, 0x3b, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x20, 0x65, + 0x6c, 0x73, 0x65, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x74, 0x4d, + 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x2e, 0x70, 0x75, 0x73, 0x68, + 0x28, 0x64, 0x61, 0x74, 0x61, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x73, 0x6c, 0x6f, 0x74, 0x5f, 0x69, + 0x64, 0x20, 0x3d, 0x20, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x73, 0x6c, 0x6f, + 0x74, 0x5f, 0x69, 0x64, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, 0x73, 0x65, 0x6c, 0x65, + 0x63, 0x74, 0x65, 0x64, 0x5f, 0x69, 0x6d, 0x61, 0x67, 0x65, 0x20, 0x26, + 0x26, 0x20, 0x21, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x6d, 0x75, 0x6c, 0x74, + 0x69, 0x6d, 0x6f, 0x64, 0x61, 0x6c, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x61, 0x6c, + 0x65, 0x72, 0x74, 0x28, 0x22, 0x54, 0x68, 0x65, 0x20, 0x73, 0x65, 0x72, + 0x76, 0x65, 0x72, 0x20, 0x77, 0x61, 0x73, 0x20, 0x6e, 0x6f, 0x74, 0x20, + 0x63, 0x6f, 0x6d, 0x70, 0x69, 0x6c, 0x65, 0x64, 0x20, 0x66, 0x6f, 0x72, + 0x20, 0x6d, 0x75, 0x6c, 0x74, 0x69, 0x6d, 0x6f, 0x64, 0x61, 0x6c, 0x20, + 0x6f, 0x72, 0x20, 0x74, 0x68, 0x65, 0x20, 0x6d, 0x6f, 0x64, 0x65, 0x6c, + 0x20, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x20, 0x63, + 0x61, 0x6e, 0x27, 0x74, 0x20, 0x62, 0x65, 0x20, 0x6c, 0x6f, 0x61, 0x64, + 0x65, 0x64, 0x2e, 0x22, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, + 0x6e, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x28, 0x5b, 0x2e, 0x2e, 0x2e, 0x68, 0x69, 0x73, 0x74, 0x6f, 0x72, 0x79, 0x2c, 0x20, 0x5b, 0x63, 0x68, 0x61, 0x72, 0x2c, 0x20, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x74, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x5d, 0x5d, 0x29, 0x0a, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x6f, - 0x6c, 0x65, 0x2e, 0x6c, 0x6f, 0x67, 0x28, 0x22, 0x43, 0x6f, 0x6d, 0x70, - 0x6c, 0x65, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x66, 0x69, 0x6e, 0x69, 0x73, - 0x68, 0x65, 0x64, 0x3a, 0x20, 0x27, 0x22, 0x2c, 0x20, 0x63, 0x75, 0x72, - 0x72, 0x65, 0x6e, 0x74, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, - 0x2e, 0x6d, 0x61, 0x70, 0x28, 0x6d, 0x73, 0x67, 0x20, 0x3d, 0x3e, 0x20, - 0x6d, 0x73, 0x67, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x29, - 0x2e, 0x6a, 0x6f, 0x69, 0x6e, 0x28, 0x27, 0x27, 0x29, 0x2c, 0x20, 0x22, - 0x27, 0x2c, 0x20, 0x73, 0x75, 0x6d, 0x6d, 0x61, 0x72, 0x79, 0x3a, 0x20, - 0x22, 0x2c, 0x20, 0x64, 0x61, 0x74, 0x61, 0x29, 0x3b, 0x0a, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x20, 0x65, 0x6c, 0x73, 0x65, - 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x74, 0x4d, 0x65, 0x73, 0x73, - 0x61, 0x67, 0x65, 0x73, 0x2e, 0x70, 0x75, 0x73, 0x68, 0x28, 0x64, 0x61, - 0x74, 0x61, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x73, 0x6c, 0x6f, 0x74, 0x5f, 0x69, 0x64, 0x20, 0x3d, - 0x20, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x73, 0x6c, 0x6f, 0x74, 0x5f, 0x69, - 0x64, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x69, 0x66, 0x20, 0x28, 0x73, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x65, - 0x64, 0x5f, 0x69, 0x6d, 0x61, 0x67, 0x65, 0x20, 0x26, 0x26, 0x20, 0x21, - 0x64, 0x61, 0x74, 0x61, 0x2e, 0x6d, 0x75, 0x6c, 0x74, 0x69, 0x6d, 0x6f, - 0x64, 0x61, 0x6c, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x61, 0x6c, 0x65, 0x72, 0x74, - 0x28, 0x22, 0x54, 0x68, 0x65, 0x20, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, - 0x20, 0x77, 0x61, 0x73, 0x20, 0x6e, 0x6f, 0x74, 0x20, 0x63, 0x6f, 0x6d, - 0x70, 0x69, 0x6c, 0x65, 0x64, 0x20, 0x66, 0x6f, 0x72, 0x20, 0x6d, 0x75, - 0x6c, 0x74, 0x69, 0x6d, 0x6f, 0x64, 0x61, 0x6c, 0x20, 0x6f, 0x72, 0x20, - 0x74, 0x68, 0x65, 0x20, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x20, 0x70, 0x72, - 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x20, 0x63, 0x61, 0x6e, 0x27, - 0x74, 0x20, 0x62, 0x65, 0x20, 0x6c, 0x6f, 0x61, 0x64, 0x65, 0x64, 0x2e, - 0x22, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x3b, 0x0a, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x74, 0x72, - 0x61, 0x6e, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x55, 0x70, 0x64, 0x61, - 0x74, 0x65, 0x28, 0x5b, 0x2e, 0x2e, 0x2e, 0x68, 0x69, 0x73, 0x74, 0x6f, - 0x72, 0x79, 0x2c, 0x20, 0x5b, 0x63, 0x68, 0x61, 0x72, 0x2c, 0x20, 0x63, - 0x75, 0x72, 0x72, 0x65, 0x6e, 0x74, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, - 0x65, 0x73, 0x5d, 0x5d, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x7d, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x69, 0x66, 0x20, 0x28, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x74, 0x69, - 0x6d, 0x69, 0x6e, 0x67, 0x73, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x6c, 0x6c, 0x61, 0x6d, 0x61, - 0x53, 0x74, 0x61, 0x74, 0x73, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x20, - 0x3d, 0x20, 0x64, 0x61, 0x74, 0x61, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x7d, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, - 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x2e, 0x76, 0x61, 0x6c, 0x75, - 0x65, 0x20, 0x3d, 0x20, 0x6e, 0x75, 0x6c, 0x6c, 0x3b, 0x0a, 0x20, 0x20, - 0x20, 0x20, 0x7d, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x2f, 0x2f, 0x20, - 0x73, 0x65, 0x6e, 0x64, 0x20, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, - 0x20, 0x74, 0x6f, 0x20, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x0a, 0x20, - 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x63, 0x68, 0x61, - 0x74, 0x20, 0x3d, 0x20, 0x61, 0x73, 0x79, 0x6e, 0x63, 0x20, 0x28, 0x6d, - 0x73, 0x67, 0x29, 0x20, 0x3d, 0x3e, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, 0x63, 0x6f, 0x6e, 0x74, 0x72, - 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x29, - 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, - 0x6f, 0x6e, 0x73, 0x6f, 0x6c, 0x65, 0x2e, 0x6c, 0x6f, 0x67, 0x28, 0x27, - 0x61, 0x6c, 0x72, 0x65, 0x61, 0x64, 0x79, 0x20, 0x72, 0x75, 0x6e, 0x6e, - 0x69, 0x6e, 0x67, 0x2e, 0x2e, 0x2e, 0x27, 0x29, 0x3b, 0x0a, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, - 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x0a, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x63, 0x72, - 0x69, 0x70, 0x74, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x28, 0x5b, 0x2e, - 0x2e, 0x2e, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x61, - 0x6c, 0x75, 0x65, 0x2e, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x63, 0x72, 0x69, - 0x70, 0x74, 0x2c, 0x20, 0x5b, 0x22, 0x7b, 0x7b, 0x75, 0x73, 0x65, 0x72, - 0x7d, 0x7d, 0x22, 0x2c, 0x20, 0x6d, 0x73, 0x67, 0x5d, 0x5d, 0x29, 0x0a, - 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x6c, 0x65, 0x74, 0x20, 0x70, - 0x72, 0x6f, 0x6d, 0x70, 0x74, 0x20, 0x3d, 0x20, 0x74, 0x65, 0x6d, 0x70, - 0x6c, 0x61, 0x74, 0x65, 0x28, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, - 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x2e, 0x74, 0x65, 0x6d, 0x70, 0x6c, - 0x61, 0x74, 0x65, 0x2c, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x3a, 0x20, - 0x6d, 0x73, 0x67, 0x2c, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x68, 0x69, 0x73, 0x74, 0x6f, 0x72, 0x79, 0x3a, 0x20, 0x73, 0x65, - 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x2e, - 0x74, 0x72, 0x61, 0x6e, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x2e, 0x66, - 0x6c, 0x61, 0x74, 0x4d, 0x61, 0x70, 0x28, 0x0a, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x28, 0x5b, 0x6e, 0x61, 0x6d, 0x65, - 0x2c, 0x20, 0x64, 0x61, 0x74, 0x61, 0x5d, 0x29, 0x20, 0x3d, 0x3e, 0x0a, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x74, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x28, 0x0a, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x61, 0x6c, 0x75, - 0x65, 0x2e, 0x68, 0x69, 0x73, 0x74, 0x6f, 0x72, 0x79, 0x54, 0x65, 0x6d, - 0x70, 0x6c, 0x61, 0x74, 0x65, 0x2c, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7b, 0x0a, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x6e, 0x61, 0x6d, 0x65, 0x2c, 0x0a, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x3a, 0x20, 0x41, 0x72, - 0x72, 0x61, 0x79, 0x2e, 0x69, 0x73, 0x41, 0x72, 0x72, 0x61, 0x79, 0x28, - 0x64, 0x61, 0x74, 0x61, 0x29, 0x20, 0x3f, 0x0a, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x6d, 0x61, 0x70, 0x28, 0x6d, - 0x73, 0x67, 0x20, 0x3d, 0x3e, 0x20, 0x6d, 0x73, 0x67, 0x2e, 0x63, 0x6f, - 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x29, 0x2e, 0x6a, 0x6f, 0x69, 0x6e, 0x28, - 0x27, 0x27, 0x29, 0x2e, 0x72, 0x65, 0x70, 0x6c, 0x61, 0x63, 0x65, 0x28, - 0x2f, 0x5e, 0x5c, 0x73, 0x2f, 0x2c, 0x20, 0x27, 0x27, 0x29, 0x20, 0x3a, - 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x64, 0x61, 0x74, 0x61, 0x2c, - 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x29, 0x2e, 0x6a, 0x6f, 0x69, 0x6e, 0x28, 0x22, 0x5c, - 0x6e, 0x22, 0x29, 0x2c, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, - 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, - 0x28, 0x73, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x65, 0x64, 0x5f, 0x69, 0x6d, - 0x61, 0x67, 0x65, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x70, 0x72, 0x6f, 0x6d, 0x70, 0x74, 0x20, 0x3d, 0x20, - 0x60, 0x41, 0x20, 0x63, 0x68, 0x61, 0x74, 0x20, 0x62, 0x65, 0x74, 0x77, - 0x65, 0x65, 0x6e, 0x20, 0x61, 0x20, 0x63, 0x75, 0x72, 0x69, 0x6f, 0x75, - 0x73, 0x20, 0x68, 0x75, 0x6d, 0x61, 0x6e, 0x20, 0x61, 0x6e, 0x64, 0x20, - 0x61, 0x6e, 0x20, 0x61, 0x72, 0x74, 0x69, 0x66, 0x69, 0x63, 0x69, 0x61, - 0x6c, 0x20, 0x69, 0x6e, 0x74, 0x65, 0x6c, 0x6c, 0x69, 0x67, 0x65, 0x6e, - 0x63, 0x65, 0x20, 0x61, 0x73, 0x73, 0x69, 0x73, 0x74, 0x61, 0x6e, 0x74, - 0x2e, 0x20, 0x54, 0x68, 0x65, 0x20, 0x61, 0x73, 0x73, 0x69, 0x73, 0x74, - 0x61, 0x6e, 0x74, 0x20, 0x67, 0x69, 0x76, 0x65, 0x73, 0x20, 0x68, 0x65, - 0x6c, 0x70, 0x66, 0x75, 0x6c, 0x2c, 0x20, 0x64, 0x65, 0x74, 0x61, 0x69, - 0x6c, 0x65, 0x64, 0x2c, 0x20, 0x61, 0x6e, 0x64, 0x20, 0x70, 0x6f, 0x6c, - 0x69, 0x74, 0x65, 0x20, 0x61, 0x6e, 0x73, 0x77, 0x65, 0x72, 0x73, 0x20, - 0x74, 0x6f, 0x20, 0x74, 0x68, 0x65, 0x20, 0x68, 0x75, 0x6d, 0x61, 0x6e, - 0x27, 0x73, 0x20, 0x71, 0x75, 0x65, 0x73, 0x74, 0x69, 0x6f, 0x6e, 0x73, - 0x2e, 0x5c, 0x6e, 0x55, 0x53, 0x45, 0x52, 0x3a, 0x5b, 0x69, 0x6d, 0x67, - 0x2d, 0x31, 0x30, 0x5d, 0x24, 0x7b, 0x6d, 0x73, 0x67, 0x7d, 0x5c, 0x6e, - 0x41, 0x53, 0x53, 0x49, 0x53, 0x54, 0x41, 0x4e, 0x54, 0x3a, 0x60, 0x3b, - 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x61, 0x77, 0x61, 0x69, 0x74, 0x20, 0x72, 0x75, 0x6e, - 0x4c, 0x6c, 0x61, 0x6d, 0x61, 0x28, 0x70, 0x72, 0x6f, 0x6d, 0x70, 0x74, - 0x2c, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x2e, 0x2e, 0x2e, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x2e, 0x76, 0x61, - 0x6c, 0x75, 0x65, 0x2c, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x73, 0x6c, 0x6f, 0x74, 0x5f, 0x69, 0x64, 0x3a, 0x20, 0x73, 0x6c, - 0x6f, 0x74, 0x5f, 0x69, 0x64, 0x2c, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x73, 0x74, 0x6f, 0x70, 0x3a, 0x20, 0x5b, 0x22, 0x3c, - 0x2f, 0x73, 0x3e, 0x22, 0x2c, 0x20, 0x74, 0x65, 0x6d, 0x70, 0x6c, 0x61, - 0x74, 0x65, 0x28, 0x22, 0x7b, 0x7b, 0x63, 0x68, 0x61, 0x72, 0x7d, 0x7d, - 0x3a, 0x22, 0x29, 0x2c, 0x20, 0x74, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, - 0x65, 0x28, 0x22, 0x7b, 0x7b, 0x75, 0x73, 0x65, 0x72, 0x7d, 0x7d, 0x3a, - 0x22, 0x29, 0x5d, 0x2c, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, - 0x2c, 0x20, 0x22, 0x7b, 0x7b, 0x63, 0x68, 0x61, 0x72, 0x7d, 0x7d, 0x22, - 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x0a, 0x20, 0x20, - 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x72, 0x75, 0x6e, 0x43, - 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x3d, 0x20, - 0x28, 0x29, 0x20, 0x3d, 0x3e, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, - 0x6c, 0x6c, 0x65, 0x72, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x29, 0x20, - 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, - 0x6e, 0x73, 0x6f, 0x6c, 0x65, 0x2e, 0x6c, 0x6f, 0x67, 0x28, 0x27, 0x61, - 0x6c, 0x72, 0x65, 0x61, 0x64, 0x79, 0x20, 0x72, 0x75, 0x6e, 0x6e, 0x69, - 0x6e, 0x67, 0x2e, 0x2e, 0x2e, 0x27, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x3b, - 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x7b, 0x20, 0x70, - 0x72, 0x6f, 0x6d, 0x70, 0x74, 0x20, 0x7d, 0x20, 0x3d, 0x20, 0x73, 0x65, - 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3b, - 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x74, 0x72, 0x61, 0x6e, 0x73, - 0x63, 0x72, 0x69, 0x70, 0x74, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x28, - 0x5b, 0x2e, 0x2e, 0x2e, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x2e, - 0x76, 0x61, 0x6c, 0x75, 0x65, 0x2e, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x63, - 0x72, 0x69, 0x70, 0x74, 0x2c, 0x20, 0x5b, 0x22, 0x22, 0x2c, 0x20, 0x70, - 0x72, 0x6f, 0x6d, 0x70, 0x74, 0x5d, 0x5d, 0x29, 0x3b, 0x0a, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x72, 0x75, 0x6e, 0x4c, 0x6c, 0x61, 0x6d, 0x61, - 0x28, 0x70, 0x72, 0x6f, 0x6d, 0x70, 0x74, 0x2c, 0x20, 0x7b, 0x0a, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x2e, 0x2e, 0x2e, 0x70, 0x61, - 0x72, 0x61, 0x6d, 0x73, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x2c, 0x0a, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x73, 0x6c, 0x6f, 0x74, - 0x5f, 0x69, 0x64, 0x3a, 0x20, 0x73, 0x6c, 0x6f, 0x74, 0x5f, 0x69, 0x64, - 0x2c, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x73, 0x74, - 0x6f, 0x70, 0x3a, 0x20, 0x5b, 0x5d, 0x2c, 0x0a, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x7d, 0x2c, 0x20, 0x22, 0x22, 0x29, 0x2e, 0x66, 0x69, 0x6e, - 0x61, 0x6c, 0x6c, 0x79, 0x28, 0x28, 0x29, 0x20, 0x3d, 0x3e, 0x20, 0x7b, - 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x73, 0x65, 0x73, - 0x73, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x2e, 0x70, - 0x72, 0x6f, 0x6d, 0x70, 0x74, 0x20, 0x3d, 0x20, 0x73, 0x65, 0x73, 0x73, - 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x2e, 0x74, 0x72, - 0x61, 0x6e, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x2e, 0x6d, 0x61, 0x70, - 0x28, 0x28, 0x5b, 0x5f, 0x2c, 0x20, 0x64, 0x61, 0x74, 0x61, 0x5d, 0x29, - 0x20, 0x3d, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x41, 0x72, 0x72, 0x61, 0x79, 0x2e, 0x69, 0x73, 0x41, 0x72, - 0x72, 0x61, 0x79, 0x28, 0x64, 0x61, 0x74, 0x61, 0x29, 0x20, 0x3f, 0x20, - 0x64, 0x61, 0x74, 0x61, 0x2e, 0x6d, 0x61, 0x70, 0x28, 0x6d, 0x73, 0x67, - 0x20, 0x3d, 0x3e, 0x20, 0x6d, 0x73, 0x67, 0x2e, 0x63, 0x6f, 0x6e, 0x74, - 0x65, 0x6e, 0x74, 0x29, 0x2e, 0x6a, 0x6f, 0x69, 0x6e, 0x28, 0x27, 0x27, - 0x29, 0x20, 0x3a, 0x20, 0x64, 0x61, 0x74, 0x61, 0x0a, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x29, 0x2e, 0x6a, 0x6f, 0x69, 0x6e, 0x28, - 0x27, 0x27, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, 0x64, 0x61, 0x74, 0x61, + 0x2e, 0x74, 0x69, 0x6d, 0x69, 0x6e, 0x67, 0x73, 0x29, 0x20, 0x7b, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x6c, 0x6c, + 0x61, 0x6d, 0x61, 0x53, 0x74, 0x61, 0x74, 0x73, 0x2e, 0x76, 0x61, 0x6c, + 0x75, 0x65, 0x20, 0x3d, 0x20, 0x64, 0x61, 0x74, 0x61, 0x3b, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x2e, 0x76, + 0x61, 0x6c, 0x75, 0x65, 0x20, 0x3d, 0x20, 0x6e, 0x75, 0x6c, 0x6c, 0x3b, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x2f, 0x2f, 0x20, 0x73, 0x65, 0x6e, 0x64, 0x20, 0x6d, 0x65, 0x73, 0x73, + 0x61, 0x67, 0x65, 0x20, 0x74, 0x6f, 0x20, 0x73, 0x65, 0x72, 0x76, 0x65, + 0x72, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, + 0x63, 0x68, 0x61, 0x74, 0x20, 0x3d, 0x20, 0x61, 0x73, 0x79, 0x6e, 0x63, + 0x20, 0x28, 0x6d, 0x73, 0x67, 0x29, 0x20, 0x3d, 0x3e, 0x20, 0x7b, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, 0x63, 0x6f, + 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x2e, 0x76, 0x61, 0x6c, + 0x75, 0x65, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x6f, 0x6c, 0x65, 0x2e, 0x6c, 0x6f, + 0x67, 0x28, 0x27, 0x61, 0x6c, 0x72, 0x65, 0x61, 0x64, 0x79, 0x20, 0x72, + 0x75, 0x6e, 0x6e, 0x69, 0x6e, 0x67, 0x2e, 0x2e, 0x2e, 0x27, 0x29, 0x3b, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, 0x74, + 0x75, 0x72, 0x6e, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, + 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x74, 0x72, 0x61, 0x6e, + 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, + 0x28, 0x5b, 0x2e, 0x2e, 0x2e, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, + 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x2e, 0x74, 0x72, 0x61, 0x6e, 0x73, + 0x63, 0x72, 0x69, 0x70, 0x74, 0x2c, 0x20, 0x5b, 0x22, 0x7b, 0x7b, 0x75, + 0x73, 0x65, 0x72, 0x7d, 0x7d, 0x22, 0x2c, 0x20, 0x6d, 0x73, 0x67, 0x5d, + 0x5d, 0x29, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x6c, 0x65, + 0x74, 0x20, 0x70, 0x72, 0x6f, 0x6d, 0x70, 0x74, 0x20, 0x3d, 0x20, 0x74, + 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x28, 0x73, 0x65, 0x73, 0x73, + 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x2e, 0x74, 0x65, + 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x2c, 0x20, 0x7b, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, + 0x65, 0x3a, 0x20, 0x6d, 0x73, 0x67, 0x2c, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x68, 0x69, 0x73, 0x74, 0x6f, 0x72, 0x79, 0x3a, 0x20, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x2e, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x63, 0x72, 0x69, 0x70, - 0x74, 0x20, 0x3d, 0x20, 0x5b, 0x5d, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x7d, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x0a, - 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x73, 0x74, - 0x6f, 0x70, 0x20, 0x3d, 0x20, 0x28, 0x65, 0x29, 0x20, 0x3d, 0x3e, 0x20, - 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x65, 0x2e, 0x70, 0x72, - 0x65, 0x76, 0x65, 0x6e, 0x74, 0x44, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, - 0x28, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, - 0x20, 0x28, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, - 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, - 0x6c, 0x6c, 0x65, 0x72, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x2e, 0x61, - 0x62, 0x6f, 0x72, 0x74, 0x28, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, - 0x65, 0x72, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x20, 0x3d, 0x20, 0x6e, - 0x75, 0x6c, 0x6c, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, - 0x0a, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, - 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x72, 0x65, 0x73, 0x65, 0x74, 0x20, - 0x3d, 0x20, 0x28, 0x65, 0x29, 0x20, 0x3d, 0x3e, 0x20, 0x7b, 0x0a, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x73, 0x74, 0x6f, 0x70, 0x28, 0x65, 0x29, - 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x74, 0x72, 0x61, 0x6e, - 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, - 0x28, 0x5b, 0x5d, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, - 0x0a, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x75, - 0x70, 0x6c, 0x6f, 0x61, 0x64, 0x49, 0x6d, 0x61, 0x67, 0x65, 0x20, 0x3d, - 0x20, 0x28, 0x65, 0x29, 0x20, 0x3d, 0x3e, 0x20, 0x7b, 0x0a, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x65, 0x2e, 0x70, 0x72, 0x65, 0x76, 0x65, 0x6e, - 0x74, 0x44, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x28, 0x29, 0x3b, 0x0a, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x64, 0x6f, 0x63, 0x75, 0x6d, 0x65, - 0x6e, 0x74, 0x2e, 0x67, 0x65, 0x74, 0x45, 0x6c, 0x65, 0x6d, 0x65, 0x6e, - 0x74, 0x42, 0x79, 0x49, 0x64, 0x28, 0x22, 0x66, 0x69, 0x6c, 0x65, 0x49, - 0x6e, 0x70, 0x75, 0x74, 0x22, 0x29, 0x2e, 0x63, 0x6c, 0x69, 0x63, 0x6b, - 0x28, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x64, 0x6f, - 0x63, 0x75, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x67, 0x65, 0x74, 0x45, 0x6c, - 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x42, 0x79, 0x49, 0x64, 0x28, 0x22, 0x66, - 0x69, 0x6c, 0x65, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x22, 0x29, 0x2e, 0x61, - 0x64, 0x64, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x4c, 0x69, 0x73, 0x74, 0x65, - 0x6e, 0x65, 0x72, 0x28, 0x22, 0x63, 0x68, 0x61, 0x6e, 0x67, 0x65, 0x22, - 0x2c, 0x20, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x28, - 0x65, 0x76, 0x65, 0x6e, 0x74, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x73, - 0x65, 0x6c, 0x65, 0x63, 0x74, 0x65, 0x64, 0x46, 0x69, 0x6c, 0x65, 0x20, - 0x3d, 0x20, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x2e, 0x74, 0x61, 0x72, 0x67, - 0x65, 0x74, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x73, 0x5b, 0x30, 0x5d, 0x3b, - 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, - 0x28, 0x73, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x65, 0x64, 0x46, 0x69, 0x6c, - 0x65, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x72, 0x65, 0x61, - 0x64, 0x65, 0x72, 0x20, 0x3d, 0x20, 0x6e, 0x65, 0x77, 0x20, 0x46, 0x69, - 0x6c, 0x65, 0x52, 0x65, 0x61, 0x64, 0x65, 0x72, 0x28, 0x29, 0x3b, 0x0a, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, - 0x61, 0x64, 0x65, 0x72, 0x2e, 0x6f, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x20, - 0x3d, 0x20, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x28, - 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x69, 0x6d, - 0x61, 0x67, 0x65, 0x5f, 0x64, 0x61, 0x74, 0x61, 0x20, 0x3d, 0x20, 0x72, - 0x65, 0x61, 0x64, 0x65, 0x72, 0x2e, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, - 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x61, - 0x6c, 0x75, 0x65, 0x20, 0x3d, 0x20, 0x7b, 0x20, 0x2e, 0x2e, 0x2e, 0x73, - 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, - 0x2c, 0x20, 0x69, 0x6d, 0x61, 0x67, 0x65, 0x5f, 0x73, 0x65, 0x6c, 0x65, - 0x63, 0x74, 0x65, 0x64, 0x3a, 0x20, 0x69, 0x6d, 0x61, 0x67, 0x65, 0x5f, - 0x64, 0x61, 0x74, 0x61, 0x20, 0x7d, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x70, 0x61, 0x72, 0x61, - 0x6d, 0x73, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x20, 0x3d, 0x20, 0x7b, + 0x74, 0x2e, 0x66, 0x6c, 0x61, 0x74, 0x4d, 0x61, 0x70, 0x28, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x28, 0x5b, 0x6e, + 0x61, 0x6d, 0x65, 0x2c, 0x20, 0x64, 0x61, 0x74, 0x61, 0x5d, 0x29, 0x20, + 0x3d, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x74, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x28, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x2e, 0x76, + 0x61, 0x6c, 0x75, 0x65, 0x2e, 0x68, 0x69, 0x73, 0x74, 0x6f, 0x72, 0x79, + 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x2c, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x6e, 0x61, 0x6d, 0x65, 0x2c, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x3a, + 0x20, 0x41, 0x72, 0x72, 0x61, 0x79, 0x2e, 0x69, 0x73, 0x41, 0x72, 0x72, + 0x61, 0x79, 0x28, 0x64, 0x61, 0x74, 0x61, 0x29, 0x20, 0x3f, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x6d, 0x61, + 0x70, 0x28, 0x6d, 0x73, 0x67, 0x20, 0x3d, 0x3e, 0x20, 0x6d, 0x73, 0x67, + 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x29, 0x2e, 0x6a, 0x6f, + 0x69, 0x6e, 0x28, 0x27, 0x27, 0x29, 0x2e, 0x72, 0x65, 0x70, 0x6c, 0x61, + 0x63, 0x65, 0x28, 0x2f, 0x5e, 0x5c, 0x73, 0x2f, 0x2c, 0x20, 0x27, 0x27, + 0x29, 0x20, 0x3a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x64, 0x61, + 0x74, 0x61, 0x2c, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x29, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x29, 0x2e, 0x6a, 0x6f, 0x69, 0x6e, + 0x28, 0x22, 0x5c, 0x6e, 0x22, 0x29, 0x2c, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x7d, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x69, 0x66, 0x20, 0x28, 0x73, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x65, 0x64, + 0x5f, 0x69, 0x6d, 0x61, 0x67, 0x65, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x70, 0x72, 0x6f, 0x6d, 0x70, 0x74, + 0x20, 0x3d, 0x20, 0x60, 0x41, 0x20, 0x63, 0x68, 0x61, 0x74, 0x20, 0x62, + 0x65, 0x74, 0x77, 0x65, 0x65, 0x6e, 0x20, 0x61, 0x20, 0x63, 0x75, 0x72, + 0x69, 0x6f, 0x75, 0x73, 0x20, 0x68, 0x75, 0x6d, 0x61, 0x6e, 0x20, 0x61, + 0x6e, 0x64, 0x20, 0x61, 0x6e, 0x20, 0x61, 0x72, 0x74, 0x69, 0x66, 0x69, + 0x63, 0x69, 0x61, 0x6c, 0x20, 0x69, 0x6e, 0x74, 0x65, 0x6c, 0x6c, 0x69, + 0x67, 0x65, 0x6e, 0x63, 0x65, 0x20, 0x61, 0x73, 0x73, 0x69, 0x73, 0x74, + 0x61, 0x6e, 0x74, 0x2e, 0x20, 0x54, 0x68, 0x65, 0x20, 0x61, 0x73, 0x73, + 0x69, 0x73, 0x74, 0x61, 0x6e, 0x74, 0x20, 0x67, 0x69, 0x76, 0x65, 0x73, + 0x20, 0x68, 0x65, 0x6c, 0x70, 0x66, 0x75, 0x6c, 0x2c, 0x20, 0x64, 0x65, + 0x74, 0x61, 0x69, 0x6c, 0x65, 0x64, 0x2c, 0x20, 0x61, 0x6e, 0x64, 0x20, + 0x70, 0x6f, 0x6c, 0x69, 0x74, 0x65, 0x20, 0x61, 0x6e, 0x73, 0x77, 0x65, + 0x72, 0x73, 0x20, 0x74, 0x6f, 0x20, 0x74, 0x68, 0x65, 0x20, 0x68, 0x75, + 0x6d, 0x61, 0x6e, 0x27, 0x73, 0x20, 0x71, 0x75, 0x65, 0x73, 0x74, 0x69, + 0x6f, 0x6e, 0x73, 0x2e, 0x5c, 0x6e, 0x55, 0x53, 0x45, 0x52, 0x3a, 0x5b, + 0x69, 0x6d, 0x67, 0x2d, 0x31, 0x30, 0x5d, 0x24, 0x7b, 0x6d, 0x73, 0x67, + 0x7d, 0x5c, 0x6e, 0x41, 0x53, 0x53, 0x49, 0x53, 0x54, 0x41, 0x4e, 0x54, + 0x3a, 0x60, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x61, 0x77, 0x61, 0x69, 0x74, 0x20, + 0x72, 0x75, 0x6e, 0x4c, 0x6c, 0x61, 0x6d, 0x61, 0x28, 0x70, 0x72, 0x6f, + 0x6d, 0x70, 0x74, 0x2c, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x2e, 0x2e, 0x2e, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, - 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x2c, 0x20, 0x69, 0x6d, 0x61, 0x67, - 0x65, 0x5f, 0x64, 0x61, 0x74, 0x61, 0x3a, 0x20, 0x5b, 0x0a, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x7b, 0x20, 0x64, 0x61, 0x74, 0x61, 0x3a, 0x20, 0x69, 0x6d, - 0x61, 0x67, 0x65, 0x5f, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x72, 0x65, 0x70, - 0x6c, 0x61, 0x63, 0x65, 0x28, 0x2f, 0x64, 0x61, 0x74, 0x61, 0x3a, 0x69, - 0x6d, 0x61, 0x67, 0x65, 0x5c, 0x2f, 0x5b, 0x5e, 0x3b, 0x5d, 0x2b, 0x3b, - 0x62, 0x61, 0x73, 0x65, 0x36, 0x34, 0x2c, 0x2f, 0x2c, 0x20, 0x27, 0x27, - 0x29, 0x2c, 0x20, 0x69, 0x64, 0x3a, 0x20, 0x31, 0x30, 0x20, 0x7d, 0x5d, - 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x7d, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x73, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x65, 0x64, 0x5f, 0x69, - 0x6d, 0x61, 0x67, 0x65, 0x20, 0x3d, 0x20, 0x74, 0x72, 0x75, 0x65, 0x3b, - 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, - 0x65, 0x61, 0x64, 0x65, 0x72, 0x2e, 0x72, 0x65, 0x61, 0x64, 0x41, 0x73, - 0x44, 0x61, 0x74, 0x61, 0x55, 0x52, 0x4c, 0x28, 0x73, 0x65, 0x6c, 0x65, - 0x63, 0x74, 0x65, 0x64, 0x46, 0x69, 0x6c, 0x65, 0x29, 0x3b, 0x0a, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x7d, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x7d, - 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, - 0x6f, 0x6e, 0x20, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x49, 0x6e, - 0x70, 0x75, 0x74, 0x28, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x6d, 0x65, 0x73, 0x73, - 0x61, 0x67, 0x65, 0x20, 0x3d, 0x20, 0x75, 0x73, 0x65, 0x53, 0x69, 0x67, - 0x6e, 0x61, 0x6c, 0x28, 0x22, 0x22, 0x29, 0x0a, 0x0a, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x73, 0x75, 0x62, - 0x6d, 0x69, 0x74, 0x20, 0x3d, 0x20, 0x28, 0x65, 0x29, 0x20, 0x3d, 0x3e, - 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x73, - 0x74, 0x6f, 0x70, 0x28, 0x65, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x63, 0x68, 0x61, 0x74, 0x28, 0x6d, 0x65, 0x73, - 0x73, 0x61, 0x67, 0x65, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x29, 0x3b, - 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x6d, 0x65, 0x73, - 0x73, 0x61, 0x67, 0x65, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x20, 0x3d, - 0x20, 0x22, 0x22, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, - 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, - 0x74, 0x20, 0x65, 0x6e, 0x74, 0x65, 0x72, 0x53, 0x75, 0x62, 0x6d, 0x69, - 0x74, 0x73, 0x20, 0x3d, 0x20, 0x28, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x29, - 0x20, 0x3d, 0x3e, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x2e, - 0x77, 0x68, 0x69, 0x63, 0x68, 0x20, 0x3d, 0x3d, 0x3d, 0x20, 0x31, 0x33, - 0x20, 0x26, 0x26, 0x20, 0x21, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x2e, 0x73, - 0x68, 0x69, 0x66, 0x74, 0x4b, 0x65, 0x79, 0x29, 0x20, 0x7b, 0x0a, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x73, 0x75, 0x62, - 0x6d, 0x69, 0x74, 0x28, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x29, 0x3b, 0x0a, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x68, 0x74, 0x6d, 0x6c, - 0x60, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x66, - 0x6f, 0x72, 0x6d, 0x20, 0x6f, 0x6e, 0x73, 0x75, 0x62, 0x6d, 0x69, 0x74, - 0x3d, 0x24, 0x7b, 0x73, 0x75, 0x62, 0x6d, 0x69, 0x74, 0x7d, 0x3e, 0x0a, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x64, - 0x69, 0x76, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x3c, 0x74, 0x65, 0x78, 0x74, 0x61, 0x72, 0x65, - 0x61, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6c, 0x61, 0x73, 0x73, 0x4e, 0x61, - 0x6d, 0x65, 0x3d, 0x24, 0x7b, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, - 0x69, 0x6e, 0x67, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x20, 0x3f, 0x20, - 0x22, 0x6c, 0x6f, 0x61, 0x64, 0x69, 0x6e, 0x67, 0x22, 0x20, 0x3a, 0x20, - 0x6e, 0x75, 0x6c, 0x6c, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x6f, 0x6e, 0x69, - 0x6e, 0x70, 0x75, 0x74, 0x3d, 0x24, 0x7b, 0x28, 0x65, 0x29, 0x20, 0x3d, - 0x3e, 0x20, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x2e, 0x76, 0x61, - 0x6c, 0x75, 0x65, 0x20, 0x3d, 0x20, 0x65, 0x2e, 0x74, 0x61, 0x72, 0x67, - 0x65, 0x74, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x7d, 0x0a, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x6f, 0x6e, 0x6b, 0x65, 0x79, 0x70, 0x72, 0x65, 0x73, 0x73, 0x3d, - 0x24, 0x7b, 0x65, 0x6e, 0x74, 0x65, 0x72, 0x53, 0x75, 0x62, 0x6d, 0x69, - 0x74, 0x73, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x70, 0x6c, 0x61, 0x63, 0x65, - 0x68, 0x6f, 0x6c, 0x64, 0x65, 0x72, 0x3d, 0x22, 0x53, 0x61, 0x79, 0x20, - 0x73, 0x6f, 0x6d, 0x65, 0x74, 0x68, 0x69, 0x6e, 0x67, 0x2e, 0x2e, 0x2e, - 0x22, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, 0x6f, 0x77, 0x73, 0x3d, 0x32, 0x0a, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x74, 0x79, 0x70, 0x65, 0x3d, 0x22, 0x74, 0x65, 0x78, - 0x74, 0x22, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3d, - 0x22, 0x24, 0x7b, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x7d, 0x22, - 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x2f, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x3c, 0x2f, 0x64, 0x69, 0x76, 0x3e, 0x0a, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x64, 0x69, 0x76, 0x20, - 0x63, 0x6c, 0x61, 0x73, 0x73, 0x3d, 0x22, 0x72, 0x69, 0x67, 0x68, 0x74, - 0x22, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x3c, 0x62, 0x75, 0x74, 0x74, 0x6f, 0x6e, 0x20, 0x74, - 0x79, 0x70, 0x65, 0x3d, 0x22, 0x73, 0x75, 0x62, 0x6d, 0x69, 0x74, 0x22, - 0x20, 0x64, 0x69, 0x73, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x3d, 0x24, 0x7b, - 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x76, - 0x61, 0x6c, 0x75, 0x65, 0x7d, 0x3e, 0x53, 0x65, 0x6e, 0x64, 0x3c, 0x2f, - 0x62, 0x75, 0x74, 0x74, 0x6f, 0x6e, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x62, 0x75, 0x74, - 0x74, 0x6f, 0x6e, 0x20, 0x6f, 0x6e, 0x63, 0x6c, 0x69, 0x63, 0x6b, 0x3d, - 0x24, 0x7b, 0x75, 0x70, 0x6c, 0x6f, 0x61, 0x64, 0x49, 0x6d, 0x61, 0x67, - 0x65, 0x7d, 0x3e, 0x55, 0x70, 0x6c, 0x6f, 0x61, 0x64, 0x20, 0x49, 0x6d, - 0x61, 0x67, 0x65, 0x3c, 0x2f, 0x62, 0x75, 0x74, 0x74, 0x6f, 0x6e, 0x3e, - 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x3c, 0x62, 0x75, 0x74, 0x74, 0x6f, 0x6e, 0x20, 0x6f, 0x6e, 0x63, - 0x6c, 0x69, 0x63, 0x6b, 0x3d, 0x24, 0x7b, 0x73, 0x74, 0x6f, 0x70, 0x7d, - 0x20, 0x64, 0x69, 0x73, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x3d, 0x24, 0x7b, - 0x21, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6e, 0x67, 0x2e, - 0x76, 0x61, 0x6c, 0x75, 0x65, 0x7d, 0x3e, 0x53, 0x74, 0x6f, 0x70, 0x3c, - 0x2f, 0x62, 0x75, 0x74, 0x74, 0x6f, 0x6e, 0x3e, 0x0a, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x62, 0x75, - 0x74, 0x74, 0x6f, 0x6e, 0x20, 0x6f, 0x6e, 0x63, 0x6c, 0x69, 0x63, 0x6b, - 0x3d, 0x24, 0x7b, 0x72, 0x65, 0x73, 0x65, 0x74, 0x7d, 0x3e, 0x52, 0x65, - 0x73, 0x65, 0x74, 0x3c, 0x2f, 0x62, 0x75, 0x74, 0x74, 0x6f, 0x6e, 0x3e, - 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, - 0x2f, 0x64, 0x69, 0x76, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x3c, 0x2f, 0x66, 0x6f, 0x72, 0x6d, 0x3e, 0x0a, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x60, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, - 0x0a, 0x20, 0x20, 0x20, 0x20, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, - 0x6e, 0x20, 0x43, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x69, 0x6f, 0x6e, - 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x73, 0x28, 0x29, 0x20, 0x7b, - 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, - 0x20, 0x73, 0x75, 0x62, 0x6d, 0x69, 0x74, 0x20, 0x3d, 0x20, 0x28, 0x65, - 0x29, 0x20, 0x3d, 0x3e, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x73, 0x74, 0x6f, 0x70, 0x28, 0x65, 0x29, 0x3b, 0x0a, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, 0x75, 0x6e, 0x43, - 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x29, 0x3b, - 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x68, 0x74, - 0x6d, 0x6c, 0x60, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x3c, 0x64, 0x69, 0x76, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x3c, 0x62, 0x75, 0x74, 0x74, 0x6f, 0x6e, 0x20, - 0x6f, 0x6e, 0x63, 0x6c, 0x69, 0x63, 0x6b, 0x3d, 0x24, 0x7b, 0x73, 0x75, - 0x62, 0x6d, 0x69, 0x74, 0x7d, 0x20, 0x74, 0x79, 0x70, 0x65, 0x3d, 0x22, - 0x62, 0x75, 0x74, 0x74, 0x6f, 0x6e, 0x22, 0x20, 0x64, 0x69, 0x73, 0x61, - 0x62, 0x6c, 0x65, 0x64, 0x3d, 0x24, 0x7b, 0x67, 0x65, 0x6e, 0x65, 0x72, - 0x61, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x7d, - 0x3e, 0x53, 0x74, 0x61, 0x72, 0x74, 0x3c, 0x2f, 0x62, 0x75, 0x74, 0x74, - 0x6f, 0x6e, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x3c, 0x62, 0x75, 0x74, 0x74, 0x6f, 0x6e, 0x20, 0x6f, 0x6e, - 0x63, 0x6c, 0x69, 0x63, 0x6b, 0x3d, 0x24, 0x7b, 0x73, 0x74, 0x6f, 0x70, - 0x7d, 0x20, 0x64, 0x69, 0x73, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x3d, 0x24, - 0x7b, 0x21, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6e, 0x67, - 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x7d, 0x3e, 0x53, 0x74, 0x6f, 0x70, - 0x3c, 0x2f, 0x62, 0x75, 0x74, 0x74, 0x6f, 0x6e, 0x3e, 0x0a, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x62, 0x75, 0x74, - 0x74, 0x6f, 0x6e, 0x20, 0x6f, 0x6e, 0x63, 0x6c, 0x69, 0x63, 0x6b, 0x3d, - 0x24, 0x7b, 0x72, 0x65, 0x73, 0x65, 0x74, 0x7d, 0x3e, 0x52, 0x65, 0x73, - 0x65, 0x74, 0x3c, 0x2f, 0x62, 0x75, 0x74, 0x74, 0x6f, 0x6e, 0x3e, 0x0a, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x2f, 0x64, 0x69, - 0x76, 0x3e, 0x60, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x0a, - 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x43, 0x68, - 0x61, 0x74, 0x4c, 0x6f, 0x67, 0x20, 0x3d, 0x20, 0x28, 0x70, 0x72, 0x6f, - 0x70, 0x73, 0x29, 0x20, 0x3d, 0x3e, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x6d, 0x65, 0x73, - 0x73, 0x61, 0x67, 0x65, 0x73, 0x20, 0x3d, 0x20, 0x73, 0x65, 0x73, 0x73, - 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x2e, 0x74, 0x72, - 0x61, 0x6e, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x3b, 0x0a, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x63, 0x6f, - 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x20, 0x3d, 0x20, 0x75, 0x73, - 0x65, 0x52, 0x65, 0x66, 0x28, 0x6e, 0x75, 0x6c, 0x6c, 0x29, 0x0a, 0x0a, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x75, 0x73, 0x65, 0x45, 0x66, 0x66, - 0x65, 0x63, 0x74, 0x28, 0x28, 0x29, 0x20, 0x3d, 0x3e, 0x20, 0x7b, 0x0a, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x2f, 0x2f, 0x20, 0x73, - 0x63, 0x72, 0x6f, 0x6c, 0x6c, 0x20, 0x74, 0x6f, 0x20, 0x62, 0x6f, 0x74, - 0x74, 0x6f, 0x6d, 0x20, 0x28, 0x69, 0x66, 0x20, 0x6e, 0x65, 0x65, 0x64, - 0x65, 0x64, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x70, 0x61, 0x72, 0x65, 0x6e, 0x74, - 0x20, 0x3d, 0x20, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, - 0x2e, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x74, 0x2e, 0x70, 0x61, 0x72, - 0x65, 0x6e, 0x74, 0x45, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x3b, 0x0a, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, - 0x70, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x20, 0x26, 0x26, 0x20, 0x70, 0x61, - 0x72, 0x65, 0x6e, 0x74, 0x2e, 0x73, 0x63, 0x72, 0x6f, 0x6c, 0x6c, 0x48, - 0x65, 0x69, 0x67, 0x68, 0x74, 0x20, 0x3c, 0x3d, 0x20, 0x70, 0x61, 0x72, - 0x65, 0x6e, 0x74, 0x2e, 0x73, 0x63, 0x72, 0x6f, 0x6c, 0x6c, 0x54, 0x6f, - 0x70, 0x20, 0x2b, 0x20, 0x70, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x2e, 0x6f, - 0x66, 0x66, 0x73, 0x65, 0x74, 0x48, 0x65, 0x69, 0x67, 0x68, 0x74, 0x20, - 0x2b, 0x20, 0x33, 0x30, 0x30, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x70, 0x61, 0x72, 0x65, 0x6e, - 0x74, 0x2e, 0x73, 0x63, 0x72, 0x6f, 0x6c, 0x6c, 0x54, 0x6f, 0x28, 0x30, - 0x2c, 0x20, 0x70, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x2e, 0x73, 0x63, 0x72, - 0x6f, 0x6c, 0x6c, 0x48, 0x65, 0x69, 0x67, 0x68, 0x74, 0x29, 0x0a, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x7d, 0x2c, 0x20, 0x5b, 0x6d, 0x65, 0x73, 0x73, 0x61, - 0x67, 0x65, 0x73, 0x5d, 0x29, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x69, 0x73, 0x43, 0x6f, 0x6d, - 0x70, 0x6c, 0x65, 0x74, 0x69, 0x6f, 0x6e, 0x4d, 0x6f, 0x64, 0x65, 0x20, - 0x3d, 0x20, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x61, - 0x6c, 0x75, 0x65, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x20, 0x3d, 0x3d, 0x3d, - 0x20, 0x27, 0x63, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x69, 0x6f, 0x6e, - 0x27, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, - 0x74, 0x20, 0x63, 0x68, 0x61, 0x74, 0x4c, 0x69, 0x6e, 0x65, 0x20, 0x3d, - 0x20, 0x28, 0x5b, 0x75, 0x73, 0x65, 0x72, 0x2c, 0x20, 0x64, 0x61, 0x74, - 0x61, 0x5d, 0x2c, 0x20, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x29, 0x20, 0x3d, + 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x2c, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x73, 0x6c, 0x6f, 0x74, 0x5f, 0x69, 0x64, 0x3a, + 0x20, 0x73, 0x6c, 0x6f, 0x74, 0x5f, 0x69, 0x64, 0x2c, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x73, 0x74, 0x6f, 0x70, 0x3a, 0x20, + 0x5b, 0x22, 0x3c, 0x2f, 0x73, 0x3e, 0x22, 0x2c, 0x20, 0x74, 0x65, 0x6d, + 0x70, 0x6c, 0x61, 0x74, 0x65, 0x28, 0x22, 0x7b, 0x7b, 0x63, 0x68, 0x61, + 0x72, 0x7d, 0x7d, 0x3a, 0x22, 0x29, 0x2c, 0x20, 0x74, 0x65, 0x6d, 0x70, + 0x6c, 0x61, 0x74, 0x65, 0x28, 0x22, 0x7b, 0x7b, 0x75, 0x73, 0x65, 0x72, + 0x7d, 0x7d, 0x3a, 0x22, 0x29, 0x5d, 0x2c, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x7d, 0x2c, 0x20, 0x22, 0x7b, 0x7b, 0x63, 0x68, 0x61, 0x72, + 0x7d, 0x7d, 0x22, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x72, + 0x75, 0x6e, 0x43, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x69, 0x6f, 0x6e, + 0x20, 0x3d, 0x20, 0x28, 0x29, 0x20, 0x3d, 0x3e, 0x20, 0x7b, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, 0x63, 0x6f, 0x6e, + 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x2e, 0x76, 0x61, 0x6c, 0x75, + 0x65, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x6f, 0x6c, 0x65, 0x2e, 0x6c, 0x6f, 0x67, + 0x28, 0x27, 0x61, 0x6c, 0x72, 0x65, 0x61, 0x64, 0x79, 0x20, 0x72, 0x75, + 0x6e, 0x6e, 0x69, 0x6e, 0x67, 0x2e, 0x2e, 0x2e, 0x27, 0x29, 0x3b, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, + 0x72, 0x6e, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, + 0x7b, 0x20, 0x70, 0x72, 0x6f, 0x6d, 0x70, 0x74, 0x20, 0x7d, 0x20, 0x3d, + 0x20, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x61, 0x6c, + 0x75, 0x65, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x74, 0x72, + 0x61, 0x6e, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x55, 0x70, 0x64, 0x61, + 0x74, 0x65, 0x28, 0x5b, 0x2e, 0x2e, 0x2e, 0x73, 0x65, 0x73, 0x73, 0x69, + 0x6f, 0x6e, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x2e, 0x74, 0x72, 0x61, + 0x6e, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x2c, 0x20, 0x5b, 0x22, 0x22, + 0x2c, 0x20, 0x70, 0x72, 0x6f, 0x6d, 0x70, 0x74, 0x5d, 0x5d, 0x29, 0x3b, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, 0x75, 0x6e, 0x4c, 0x6c, + 0x61, 0x6d, 0x61, 0x28, 0x70, 0x72, 0x6f, 0x6d, 0x70, 0x74, 0x2c, 0x20, + 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x2e, 0x2e, + 0x2e, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x2e, 0x76, 0x61, 0x6c, 0x75, + 0x65, 0x2c, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x73, + 0x6c, 0x6f, 0x74, 0x5f, 0x69, 0x64, 0x3a, 0x20, 0x73, 0x6c, 0x6f, 0x74, + 0x5f, 0x69, 0x64, 0x2c, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x73, 0x74, 0x6f, 0x70, 0x3a, 0x20, 0x5b, 0x5d, 0x2c, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x2c, 0x20, 0x22, 0x22, 0x29, 0x2e, + 0x66, 0x69, 0x6e, 0x61, 0x6c, 0x6c, 0x79, 0x28, 0x28, 0x29, 0x20, 0x3d, 0x3e, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x6c, 0x65, 0x74, 0x20, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x0a, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, - 0x74, 0x20, 0x69, 0x73, 0x41, 0x72, 0x72, 0x61, 0x79, 0x4d, 0x65, 0x73, - 0x73, 0x61, 0x67, 0x65, 0x20, 0x3d, 0x20, 0x41, 0x72, 0x72, 0x61, 0x79, - 0x2e, 0x69, 0x73, 0x41, 0x72, 0x72, 0x61, 0x79, 0x28, 0x64, 0x61, 0x74, - 0x61, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, - 0x66, 0x20, 0x28, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x2e, 0x76, 0x61, - 0x6c, 0x75, 0x65, 0x2e, 0x6e, 0x5f, 0x70, 0x72, 0x6f, 0x62, 0x73, 0x20, - 0x3e, 0x20, 0x30, 0x20, 0x26, 0x26, 0x20, 0x69, 0x73, 0x41, 0x72, 0x72, - 0x61, 0x79, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x29, 0x20, 0x7b, - 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x6d, - 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x20, 0x3d, 0x20, 0x68, 0x74, 0x6d, - 0x6c, 0x60, 0x3c, 0x24, 0x7b, 0x50, 0x72, 0x6f, 0x62, 0x61, 0x62, 0x69, - 0x6c, 0x69, 0x74, 0x69, 0x65, 0x73, 0x7d, 0x20, 0x64, 0x61, 0x74, 0x61, - 0x3d, 0x24, 0x7b, 0x64, 0x61, 0x74, 0x61, 0x7d, 0x20, 0x2f, 0x3e, 0x60, - 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x20, 0x65, - 0x6c, 0x73, 0x65, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x74, 0x65, - 0x78, 0x74, 0x20, 0x3d, 0x20, 0x69, 0x73, 0x41, 0x72, 0x72, 0x61, 0x79, - 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x20, 0x3f, 0x0a, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x64, 0x61, - 0x74, 0x61, 0x2e, 0x6d, 0x61, 0x70, 0x28, 0x6d, 0x73, 0x67, 0x20, 0x3d, - 0x3e, 0x20, 0x6d, 0x73, 0x67, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, - 0x74, 0x29, 0x2e, 0x6a, 0x6f, 0x69, 0x6e, 0x28, 0x27, 0x27, 0x29, 0x2e, - 0x72, 0x65, 0x70, 0x6c, 0x61, 0x63, 0x65, 0x28, 0x2f, 0x5e, 0x5c, 0x73, - 0x2b, 0x2f, 0x2c, 0x20, 0x27, 0x27, 0x29, 0x20, 0x3a, 0x0a, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x64, 0x61, - 0x74, 0x61, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x20, 0x3d, 0x20, - 0x69, 0x73, 0x43, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x69, 0x6f, 0x6e, - 0x4d, 0x6f, 0x64, 0x65, 0x20, 0x3f, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x74, 0x65, 0x78, 0x74, 0x20, - 0x3a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x68, 0x74, 0x6d, 0x6c, 0x60, 0x3c, 0x24, 0x7b, 0x4d, 0x61, - 0x72, 0x6b, 0x64, 0x6f, 0x77, 0x6e, 0x69, 0x73, 0x68, 0x7d, 0x20, 0x74, - 0x65, 0x78, 0x74, 0x3d, 0x24, 0x7b, 0x74, 0x65, 0x6d, 0x70, 0x6c, 0x61, - 0x74, 0x65, 0x28, 0x74, 0x65, 0x78, 0x74, 0x29, 0x7d, 0x20, 0x2f, 0x3e, - 0x60, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, - 0x75, 0x73, 0x65, 0x72, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, - 0x20, 0x68, 0x74, 0x6d, 0x6c, 0x60, 0x3c, 0x70, 0x20, 0x6b, 0x65, 0x79, - 0x3d, 0x24, 0x7b, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x7d, 0x3e, 0x3c, 0x73, - 0x74, 0x72, 0x6f, 0x6e, 0x67, 0x3e, 0x24, 0x7b, 0x74, 0x65, 0x6d, 0x70, - 0x6c, 0x61, 0x74, 0x65, 0x28, 0x75, 0x73, 0x65, 0x72, 0x29, 0x7d, 0x3a, - 0x3c, 0x2f, 0x73, 0x74, 0x72, 0x6f, 0x6e, 0x67, 0x3e, 0x20, 0x24, 0x7b, - 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x7d, 0x3c, 0x2f, 0x70, 0x3e, - 0x60, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x20, - 0x65, 0x6c, 0x73, 0x65, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, - 0x69, 0x73, 0x43, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x69, 0x6f, 0x6e, - 0x4d, 0x6f, 0x64, 0x65, 0x20, 0x3f, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x68, 0x74, 0x6d, 0x6c, 0x60, - 0x3c, 0x73, 0x70, 0x61, 0x6e, 0x20, 0x6b, 0x65, 0x79, 0x3d, 0x24, 0x7b, - 0x69, 0x6e, 0x64, 0x65, 0x78, 0x7d, 0x3e, 0x24, 0x7b, 0x6d, 0x65, 0x73, - 0x73, 0x61, 0x67, 0x65, 0x7d, 0x3c, 0x2f, 0x73, 0x70, 0x61, 0x6e, 0x3e, - 0x60, 0x20, 0x3a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x68, 0x74, 0x6d, 0x6c, 0x60, 0x3c, 0x70, 0x20, - 0x6b, 0x65, 0x79, 0x3d, 0x24, 0x7b, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x7d, - 0x3e, 0x24, 0x7b, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x7d, 0x3c, - 0x2f, 0x70, 0x3e, 0x60, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x3b, 0x0a, - 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, - 0x20, 0x68, 0x61, 0x6e, 0x64, 0x6c, 0x65, 0x43, 0x6f, 0x6d, 0x70, 0x6c, - 0x65, 0x74, 0x69, 0x6f, 0x6e, 0x45, 0x64, 0x69, 0x74, 0x20, 0x3d, 0x20, - 0x28, 0x65, 0x29, 0x20, 0x3d, 0x3e, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, - 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x6d, 0x70, - 0x74, 0x20, 0x3d, 0x20, 0x65, 0x2e, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, - 0x2e, 0x69, 0x6e, 0x6e, 0x65, 0x72, 0x54, 0x65, 0x78, 0x74, 0x3b, 0x0a, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x73, 0x65, 0x73, 0x73, - 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x2e, 0x74, 0x72, - 0x61, 0x6e, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x20, 0x3d, 0x20, 0x5b, - 0x5d, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x0a, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, - 0x20, 0x68, 0x74, 0x6d, 0x6c, 0x60, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x3c, 0x64, 0x69, 0x76, 0x20, 0x69, 0x64, 0x3d, 0x22, - 0x63, 0x68, 0x61, 0x74, 0x22, 0x20, 0x72, 0x65, 0x66, 0x3d, 0x24, 0x7b, - 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x7d, 0x20, 0x6b, - 0x65, 0x79, 0x3d, 0x24, 0x7b, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, - 0x73, 0x2e, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x7d, 0x3e, 0x0a, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x69, 0x6d, - 0x67, 0x20, 0x73, 0x74, 0x79, 0x6c, 0x65, 0x3d, 0x22, 0x77, 0x69, 0x64, - 0x74, 0x68, 0x3a, 0x20, 0x36, 0x30, 0x25, 0x3b, 0x24, 0x7b, 0x21, 0x73, + 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x61, 0x6c, 0x75, + 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x6d, 0x70, 0x74, 0x20, 0x3d, 0x20, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, - 0x2e, 0x69, 0x6d, 0x61, 0x67, 0x65, 0x5f, 0x73, 0x65, 0x6c, 0x65, 0x63, - 0x74, 0x65, 0x64, 0x20, 0x3f, 0x20, 0x60, 0x64, 0x69, 0x73, 0x70, 0x6c, - 0x61, 0x79, 0x3a, 0x20, 0x6e, 0x6f, 0x6e, 0x65, 0x3b, 0x60, 0x20, 0x3a, - 0x20, 0x60, 0x60, 0x7d, 0x22, 0x20, 0x73, 0x72, 0x63, 0x3d, 0x22, 0x24, - 0x7b, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x61, 0x6c, - 0x75, 0x65, 0x2e, 0x69, 0x6d, 0x61, 0x67, 0x65, 0x5f, 0x73, 0x65, 0x6c, - 0x65, 0x63, 0x74, 0x65, 0x64, 0x7d, 0x22, 0x2f, 0x3e, 0x0a, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x73, 0x70, 0x61, - 0x6e, 0x20, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x65, 0x64, 0x69, - 0x74, 0x61, 0x62, 0x6c, 0x65, 0x3d, 0x24, 0x7b, 0x69, 0x73, 0x43, 0x6f, - 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x69, 0x6f, 0x6e, 0x4d, 0x6f, 0x64, 0x65, - 0x7d, 0x20, 0x72, 0x65, 0x66, 0x3d, 0x24, 0x7b, 0x63, 0x6f, 0x6e, 0x74, - 0x61, 0x69, 0x6e, 0x65, 0x72, 0x7d, 0x20, 0x6f, 0x6e, 0x69, 0x6e, 0x70, - 0x75, 0x74, 0x3d, 0x24, 0x7b, 0x68, 0x61, 0x6e, 0x64, 0x6c, 0x65, 0x43, - 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x69, 0x6f, 0x6e, 0x45, 0x64, 0x69, - 0x74, 0x7d, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x24, 0x7b, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, - 0x65, 0x73, 0x2e, 0x66, 0x6c, 0x61, 0x74, 0x4d, 0x61, 0x70, 0x28, 0x63, - 0x68, 0x61, 0x74, 0x4c, 0x69, 0x6e, 0x65, 0x29, 0x7d, 0x0a, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x2f, 0x73, 0x70, - 0x61, 0x6e, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x3c, 0x2f, 0x64, 0x69, 0x76, 0x3e, 0x60, 0x3b, 0x0a, 0x20, 0x20, 0x20, - 0x20, 0x7d, 0x3b, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, - 0x73, 0x74, 0x20, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x46, 0x6f, 0x72, - 0x6d, 0x20, 0x3d, 0x20, 0x28, 0x70, 0x72, 0x6f, 0x70, 0x73, 0x29, 0x20, - 0x3d, 0x3e, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, - 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x53, - 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x20, 0x3d, 0x20, 0x28, 0x65, 0x6c, - 0x29, 0x20, 0x3d, 0x3e, 0x20, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, + 0x2e, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x2e, + 0x6d, 0x61, 0x70, 0x28, 0x28, 0x5b, 0x5f, 0x2c, 0x20, 0x64, 0x61, 0x74, + 0x61, 0x5d, 0x29, 0x20, 0x3d, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x41, 0x72, 0x72, 0x61, 0x79, 0x2e, 0x69, + 0x73, 0x41, 0x72, 0x72, 0x61, 0x79, 0x28, 0x64, 0x61, 0x74, 0x61, 0x29, + 0x20, 0x3f, 0x20, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x6d, 0x61, 0x70, 0x28, + 0x6d, 0x73, 0x67, 0x20, 0x3d, 0x3e, 0x20, 0x6d, 0x73, 0x67, 0x2e, 0x63, + 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x29, 0x2e, 0x6a, 0x6f, 0x69, 0x6e, + 0x28, 0x27, 0x27, 0x29, 0x20, 0x3a, 0x20, 0x64, 0x61, 0x74, 0x61, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x29, 0x2e, 0x6a, 0x6f, + 0x69, 0x6e, 0x28, 0x27, 0x27, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x2e, + 0x76, 0x61, 0x6c, 0x75, 0x65, 0x2e, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x63, + 0x72, 0x69, 0x70, 0x74, 0x20, 0x3d, 0x20, 0x5b, 0x5d, 0x3b, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x7d, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, + 0x20, 0x73, 0x74, 0x6f, 0x70, 0x20, 0x3d, 0x20, 0x28, 0x65, 0x29, 0x20, + 0x3d, 0x3e, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x65, + 0x2e, 0x70, 0x72, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x44, 0x65, 0x66, 0x61, + 0x75, 0x6c, 0x74, 0x28, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x69, 0x66, 0x20, 0x28, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, + 0x6c, 0x65, 0x72, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x29, 0x20, 0x7b, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, + 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x2e, 0x76, 0x61, 0x6c, 0x75, + 0x65, 0x2e, 0x61, 0x62, 0x6f, 0x72, 0x74, 0x28, 0x29, 0x3b, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x74, 0x72, + 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x20, + 0x3d, 0x20, 0x6e, 0x75, 0x6c, 0x6c, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x72, 0x65, 0x73, + 0x65, 0x74, 0x20, 0x3d, 0x20, 0x28, 0x65, 0x29, 0x20, 0x3d, 0x3e, 0x20, + 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x73, 0x74, 0x6f, 0x70, + 0x28, 0x65, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x74, + 0x72, 0x61, 0x6e, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x55, 0x70, 0x64, + 0x61, 0x74, 0x65, 0x28, 0x5b, 0x5d, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x7d, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, + 0x74, 0x20, 0x75, 0x70, 0x6c, 0x6f, 0x61, 0x64, 0x49, 0x6d, 0x61, 0x67, + 0x65, 0x20, 0x3d, 0x20, 0x28, 0x65, 0x29, 0x20, 0x3d, 0x3e, 0x20, 0x7b, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x65, 0x2e, 0x70, 0x72, 0x65, + 0x76, 0x65, 0x6e, 0x74, 0x44, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x28, + 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x64, 0x6f, 0x63, + 0x75, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x67, 0x65, 0x74, 0x45, 0x6c, 0x65, + 0x6d, 0x65, 0x6e, 0x74, 0x42, 0x79, 0x49, 0x64, 0x28, 0x22, 0x66, 0x69, + 0x6c, 0x65, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x22, 0x29, 0x2e, 0x63, 0x6c, + 0x69, 0x63, 0x6b, 0x28, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x64, 0x6f, 0x63, 0x75, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x67, 0x65, + 0x74, 0x45, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x42, 0x79, 0x49, 0x64, + 0x28, 0x22, 0x66, 0x69, 0x6c, 0x65, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x22, + 0x29, 0x2e, 0x61, 0x64, 0x64, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x4c, 0x69, + 0x73, 0x74, 0x65, 0x6e, 0x65, 0x72, 0x28, 0x22, 0x63, 0x68, 0x61, 0x6e, + 0x67, 0x65, 0x22, 0x2c, 0x20, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, + 0x6e, 0x20, 0x28, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x29, 0x20, 0x7b, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, + 0x74, 0x20, 0x73, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x65, 0x64, 0x46, 0x69, + 0x6c, 0x65, 0x20, 0x3d, 0x20, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x2e, 0x74, + 0x61, 0x72, 0x67, 0x65, 0x74, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x73, 0x5b, + 0x30, 0x5d, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x69, 0x66, 0x20, 0x28, 0x73, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x65, 0x64, + 0x46, 0x69, 0x6c, 0x65, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, + 0x72, 0x65, 0x61, 0x64, 0x65, 0x72, 0x20, 0x3d, 0x20, 0x6e, 0x65, 0x77, + 0x20, 0x46, 0x69, 0x6c, 0x65, 0x52, 0x65, 0x61, 0x64, 0x65, 0x72, 0x28, + 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x72, 0x65, 0x61, 0x64, 0x65, 0x72, 0x2e, 0x6f, 0x6e, 0x6c, 0x6f, + 0x61, 0x64, 0x20, 0x3d, 0x20, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, + 0x6e, 0x20, 0x28, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, + 0x20, 0x69, 0x6d, 0x61, 0x67, 0x65, 0x5f, 0x64, 0x61, 0x74, 0x61, 0x20, + 0x3d, 0x20, 0x72, 0x65, 0x61, 0x64, 0x65, 0x72, 0x2e, 0x72, 0x65, 0x73, + 0x75, 0x6c, 0x74, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x20, 0x3d, 0x20, 0x7b, 0x20, 0x2e, 0x2e, 0x2e, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x61, + 0x6c, 0x75, 0x65, 0x2c, 0x20, 0x69, 0x6d, 0x61, 0x67, 0x65, 0x5f, 0x73, + 0x65, 0x6c, 0x65, 0x63, 0x74, 0x65, 0x64, 0x3a, 0x20, 0x69, 0x6d, 0x61, + 0x67, 0x65, 0x5f, 0x64, 0x61, 0x74, 0x61, 0x20, 0x7d, 0x3b, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x70, + 0x61, 0x72, 0x61, 0x6d, 0x73, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x20, + 0x3d, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x2e, 0x2e, 0x2e, 0x70, 0x61, 0x72, + 0x61, 0x6d, 0x73, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x2c, 0x20, 0x69, + 0x6d, 0x61, 0x67, 0x65, 0x5f, 0x64, 0x61, 0x74, 0x61, 0x3a, 0x20, 0x5b, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x7b, 0x20, 0x64, 0x61, 0x74, 0x61, 0x3a, + 0x20, 0x69, 0x6d, 0x61, 0x67, 0x65, 0x5f, 0x64, 0x61, 0x74, 0x61, 0x2e, + 0x72, 0x65, 0x70, 0x6c, 0x61, 0x63, 0x65, 0x28, 0x2f, 0x64, 0x61, 0x74, + 0x61, 0x3a, 0x69, 0x6d, 0x61, 0x67, 0x65, 0x5c, 0x2f, 0x5b, 0x5e, 0x3b, + 0x5d, 0x2b, 0x3b, 0x62, 0x61, 0x73, 0x65, 0x36, 0x34, 0x2c, 0x2f, 0x2c, + 0x20, 0x27, 0x27, 0x29, 0x2c, 0x20, 0x69, 0x64, 0x3a, 0x20, 0x31, 0x30, + 0x20, 0x7d, 0x5d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x7d, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x73, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x65, + 0x64, 0x5f, 0x69, 0x6d, 0x61, 0x67, 0x65, 0x20, 0x3d, 0x20, 0x74, 0x72, + 0x75, 0x65, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x72, 0x65, 0x61, 0x64, 0x65, 0x72, 0x2e, 0x72, 0x65, 0x61, + 0x64, 0x41, 0x73, 0x44, 0x61, 0x74, 0x61, 0x55, 0x52, 0x4c, 0x28, 0x73, + 0x65, 0x6c, 0x65, 0x63, 0x74, 0x65, 0x64, 0x46, 0x69, 0x6c, 0x65, 0x29, + 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x29, 0x3b, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x7d, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x66, 0x75, 0x6e, + 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, + 0x65, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x28, 0x29, 0x20, 0x7b, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x6d, + 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x20, 0x3d, 0x20, 0x75, 0x73, 0x65, + 0x53, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x28, 0x22, 0x22, 0x29, 0x0a, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, + 0x73, 0x75, 0x62, 0x6d, 0x69, 0x74, 0x20, 0x3d, 0x20, 0x28, 0x65, 0x29, + 0x20, 0x3d, 0x3e, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x73, 0x74, 0x6f, 0x70, 0x28, 0x65, 0x29, 0x3b, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x68, 0x61, 0x74, 0x28, + 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x2e, 0x76, 0x61, 0x6c, 0x75, + 0x65, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x2e, 0x76, 0x61, 0x6c, 0x75, + 0x65, 0x20, 0x3d, 0x20, 0x22, 0x22, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x7d, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, + 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x65, 0x6e, 0x74, 0x65, 0x72, 0x53, 0x75, + 0x62, 0x6d, 0x69, 0x74, 0x73, 0x20, 0x3d, 0x20, 0x28, 0x65, 0x76, 0x65, + 0x6e, 0x74, 0x29, 0x20, 0x3d, 0x3e, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, 0x65, 0x76, 0x65, + 0x6e, 0x74, 0x2e, 0x77, 0x68, 0x69, 0x63, 0x68, 0x20, 0x3d, 0x3d, 0x3d, + 0x20, 0x31, 0x33, 0x20, 0x26, 0x26, 0x20, 0x21, 0x65, 0x76, 0x65, 0x6e, + 0x74, 0x2e, 0x73, 0x68, 0x69, 0x66, 0x74, 0x4b, 0x65, 0x79, 0x29, 0x20, + 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x73, 0x75, 0x62, 0x6d, 0x69, 0x74, 0x28, 0x65, 0x76, 0x65, 0x6e, 0x74, + 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x68, + 0x74, 0x6d, 0x6c, 0x60, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x3c, 0x66, 0x6f, 0x72, 0x6d, 0x20, 0x6f, 0x6e, 0x73, 0x75, 0x62, + 0x6d, 0x69, 0x74, 0x3d, 0x24, 0x7b, 0x73, 0x75, 0x62, 0x6d, 0x69, 0x74, + 0x7d, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x3c, 0x64, 0x69, 0x76, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x74, 0x65, 0x78, 0x74, + 0x61, 0x72, 0x65, 0x61, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6c, 0x61, 0x73, + 0x73, 0x4e, 0x61, 0x6d, 0x65, 0x3d, 0x24, 0x7b, 0x67, 0x65, 0x6e, 0x65, + 0x72, 0x61, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, + 0x20, 0x3f, 0x20, 0x22, 0x6c, 0x6f, 0x61, 0x64, 0x69, 0x6e, 0x67, 0x22, + 0x20, 0x3a, 0x20, 0x6e, 0x75, 0x6c, 0x6c, 0x7d, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x6f, 0x6e, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x3d, 0x24, 0x7b, 0x28, 0x65, + 0x29, 0x20, 0x3d, 0x3e, 0x20, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, + 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x20, 0x3d, 0x20, 0x65, 0x2e, 0x74, + 0x61, 0x72, 0x67, 0x65, 0x74, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x7d, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x6f, 0x6e, 0x6b, 0x65, 0x79, 0x70, 0x72, 0x65, + 0x73, 0x73, 0x3d, 0x24, 0x7b, 0x65, 0x6e, 0x74, 0x65, 0x72, 0x53, 0x75, + 0x62, 0x6d, 0x69, 0x74, 0x73, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x70, 0x6c, + 0x61, 0x63, 0x65, 0x68, 0x6f, 0x6c, 0x64, 0x65, 0x72, 0x3d, 0x22, 0x53, + 0x61, 0x79, 0x20, 0x73, 0x6f, 0x6d, 0x65, 0x74, 0x68, 0x69, 0x6e, 0x67, + 0x2e, 0x2e, 0x2e, 0x22, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, 0x6f, 0x77, 0x73, + 0x3d, 0x32, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x74, 0x79, 0x70, 0x65, 0x3d, 0x22, + 0x74, 0x65, 0x78, 0x74, 0x22, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x76, 0x61, 0x6c, + 0x75, 0x65, 0x3d, 0x22, 0x24, 0x7b, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, + 0x65, 0x7d, 0x22, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x2f, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x2f, 0x64, 0x69, 0x76, 0x3e, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x64, + 0x69, 0x76, 0x20, 0x63, 0x6c, 0x61, 0x73, 0x73, 0x3d, 0x22, 0x72, 0x69, + 0x67, 0x68, 0x74, 0x22, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x62, 0x75, 0x74, 0x74, 0x6f, + 0x6e, 0x20, 0x74, 0x79, 0x70, 0x65, 0x3d, 0x22, 0x73, 0x75, 0x62, 0x6d, + 0x69, 0x74, 0x22, 0x20, 0x64, 0x69, 0x73, 0x61, 0x62, 0x6c, 0x65, 0x64, + 0x3d, 0x24, 0x7b, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6e, + 0x67, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x7d, 0x3e, 0x53, 0x65, 0x6e, + 0x64, 0x3c, 0x2f, 0x62, 0x75, 0x74, 0x74, 0x6f, 0x6e, 0x3e, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, + 0x62, 0x75, 0x74, 0x74, 0x6f, 0x6e, 0x20, 0x6f, 0x6e, 0x63, 0x6c, 0x69, + 0x63, 0x6b, 0x3d, 0x24, 0x7b, 0x75, 0x70, 0x6c, 0x6f, 0x61, 0x64, 0x49, + 0x6d, 0x61, 0x67, 0x65, 0x7d, 0x3e, 0x55, 0x70, 0x6c, 0x6f, 0x61, 0x64, + 0x20, 0x49, 0x6d, 0x61, 0x67, 0x65, 0x3c, 0x2f, 0x62, 0x75, 0x74, 0x74, + 0x6f, 0x6e, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x3c, 0x62, 0x75, 0x74, 0x74, 0x6f, 0x6e, 0x20, + 0x6f, 0x6e, 0x63, 0x6c, 0x69, 0x63, 0x6b, 0x3d, 0x24, 0x7b, 0x73, 0x74, + 0x6f, 0x70, 0x7d, 0x20, 0x64, 0x69, 0x73, 0x61, 0x62, 0x6c, 0x65, 0x64, + 0x3d, 0x24, 0x7b, 0x21, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x69, + 0x6e, 0x67, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x7d, 0x3e, 0x53, 0x74, + 0x6f, 0x70, 0x3c, 0x2f, 0x62, 0x75, 0x74, 0x74, 0x6f, 0x6e, 0x3e, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x3c, 0x62, 0x75, 0x74, 0x74, 0x6f, 0x6e, 0x20, 0x6f, 0x6e, 0x63, 0x6c, + 0x69, 0x63, 0x6b, 0x3d, 0x24, 0x7b, 0x72, 0x65, 0x73, 0x65, 0x74, 0x7d, + 0x3e, 0x52, 0x65, 0x73, 0x65, 0x74, 0x3c, 0x2f, 0x62, 0x75, 0x74, 0x74, + 0x6f, 0x6e, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x3c, 0x2f, 0x64, 0x69, 0x76, 0x3e, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x2f, 0x66, 0x6f, 0x72, 0x6d, 0x3e, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x60, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x7d, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x66, 0x75, 0x6e, 0x63, + 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x43, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, + 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x73, 0x28, + 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, + 0x6e, 0x73, 0x74, 0x20, 0x73, 0x75, 0x62, 0x6d, 0x69, 0x74, 0x20, 0x3d, + 0x20, 0x28, 0x65, 0x29, 0x20, 0x3d, 0x3e, 0x20, 0x7b, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x73, 0x74, 0x6f, 0x70, 0x28, 0x65, + 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, + 0x75, 0x6e, 0x43, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x69, 0x6f, 0x6e, + 0x28, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, + 0x20, 0x68, 0x74, 0x6d, 0x6c, 0x60, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x3c, 0x64, 0x69, 0x76, 0x3e, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x62, 0x75, 0x74, 0x74, + 0x6f, 0x6e, 0x20, 0x6f, 0x6e, 0x63, 0x6c, 0x69, 0x63, 0x6b, 0x3d, 0x24, + 0x7b, 0x73, 0x75, 0x62, 0x6d, 0x69, 0x74, 0x7d, 0x20, 0x74, 0x79, 0x70, + 0x65, 0x3d, 0x22, 0x62, 0x75, 0x74, 0x74, 0x6f, 0x6e, 0x22, 0x20, 0x64, + 0x69, 0x73, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x3d, 0x24, 0x7b, 0x67, 0x65, + 0x6e, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x76, 0x61, 0x6c, + 0x75, 0x65, 0x7d, 0x3e, 0x53, 0x74, 0x61, 0x72, 0x74, 0x3c, 0x2f, 0x62, + 0x75, 0x74, 0x74, 0x6f, 0x6e, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x62, 0x75, 0x74, 0x74, 0x6f, 0x6e, + 0x20, 0x6f, 0x6e, 0x63, 0x6c, 0x69, 0x63, 0x6b, 0x3d, 0x24, 0x7b, 0x73, + 0x74, 0x6f, 0x70, 0x7d, 0x20, 0x64, 0x69, 0x73, 0x61, 0x62, 0x6c, 0x65, + 0x64, 0x3d, 0x24, 0x7b, 0x21, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, + 0x69, 0x6e, 0x67, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x7d, 0x3e, 0x53, + 0x74, 0x6f, 0x70, 0x3c, 0x2f, 0x62, 0x75, 0x74, 0x74, 0x6f, 0x6e, 0x3e, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, + 0x62, 0x75, 0x74, 0x74, 0x6f, 0x6e, 0x20, 0x6f, 0x6e, 0x63, 0x6c, 0x69, + 0x63, 0x6b, 0x3d, 0x24, 0x7b, 0x72, 0x65, 0x73, 0x65, 0x74, 0x7d, 0x3e, + 0x52, 0x65, 0x73, 0x65, 0x74, 0x3c, 0x2f, 0x62, 0x75, 0x74, 0x74, 0x6f, + 0x6e, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, + 0x2f, 0x64, 0x69, 0x76, 0x3e, 0x60, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x7d, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, + 0x20, 0x43, 0x68, 0x61, 0x74, 0x4c, 0x6f, 0x67, 0x20, 0x3d, 0x20, 0x28, + 0x70, 0x72, 0x6f, 0x70, 0x73, 0x29, 0x20, 0x3d, 0x3e, 0x20, 0x7b, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, + 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x20, 0x3d, 0x20, 0x73, + 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, + 0x2e, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x3b, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, + 0x20, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x20, 0x3d, + 0x20, 0x75, 0x73, 0x65, 0x52, 0x65, 0x66, 0x28, 0x6e, 0x75, 0x6c, 0x6c, + 0x29, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x75, 0x73, 0x65, + 0x45, 0x66, 0x66, 0x65, 0x63, 0x74, 0x28, 0x28, 0x29, 0x20, 0x3d, 0x3e, + 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x2f, + 0x2f, 0x20, 0x73, 0x63, 0x72, 0x6f, 0x6c, 0x6c, 0x20, 0x74, 0x6f, 0x20, + 0x62, 0x6f, 0x74, 0x74, 0x6f, 0x6d, 0x20, 0x28, 0x69, 0x66, 0x20, 0x6e, + 0x65, 0x65, 0x64, 0x65, 0x64, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x70, 0x61, 0x72, + 0x65, 0x6e, 0x74, 0x20, 0x3d, 0x20, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, + 0x6e, 0x65, 0x72, 0x2e, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x74, 0x2e, + 0x70, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x45, 0x6c, 0x65, 0x6d, 0x65, 0x6e, + 0x74, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, + 0x66, 0x20, 0x28, 0x70, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x20, 0x26, 0x26, + 0x20, 0x70, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x2e, 0x73, 0x63, 0x72, 0x6f, + 0x6c, 0x6c, 0x48, 0x65, 0x69, 0x67, 0x68, 0x74, 0x20, 0x3c, 0x3d, 0x20, + 0x70, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x2e, 0x73, 0x63, 0x72, 0x6f, 0x6c, + 0x6c, 0x54, 0x6f, 0x70, 0x20, 0x2b, 0x20, 0x70, 0x61, 0x72, 0x65, 0x6e, + 0x74, 0x2e, 0x6f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x48, 0x65, 0x69, 0x67, + 0x68, 0x74, 0x20, 0x2b, 0x20, 0x33, 0x30, 0x30, 0x29, 0x20, 0x7b, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x70, 0x61, + 0x72, 0x65, 0x6e, 0x74, 0x2e, 0x73, 0x63, 0x72, 0x6f, 0x6c, 0x6c, 0x54, + 0x6f, 0x28, 0x30, 0x2c, 0x20, 0x70, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x2e, + 0x73, 0x63, 0x72, 0x6f, 0x6c, 0x6c, 0x48, 0x65, 0x69, 0x67, 0x68, 0x74, + 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x2c, 0x20, 0x5b, 0x6d, 0x65, + 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x5d, 0x29, 0x0a, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x69, 0x73, + 0x43, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x69, 0x6f, 0x6e, 0x4d, 0x6f, + 0x64, 0x65, 0x20, 0x3d, 0x20, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, + 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x20, + 0x3d, 0x3d, 0x3d, 0x20, 0x27, 0x63, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, + 0x69, 0x6f, 0x6e, 0x27, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, + 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x63, 0x68, 0x61, 0x74, 0x4c, 0x69, 0x6e, + 0x65, 0x20, 0x3d, 0x20, 0x28, 0x5b, 0x75, 0x73, 0x65, 0x72, 0x2c, 0x20, + 0x64, 0x61, 0x74, 0x61, 0x5d, 0x2c, 0x20, 0x69, 0x6e, 0x64, 0x65, 0x78, + 0x29, 0x20, 0x3d, 0x3e, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x6c, 0x65, 0x74, 0x20, 0x6d, 0x65, 0x73, 0x73, 0x61, + 0x67, 0x65, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, + 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x69, 0x73, 0x41, 0x72, 0x72, 0x61, 0x79, + 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x20, 0x3d, 0x20, 0x41, 0x72, + 0x72, 0x61, 0x79, 0x2e, 0x69, 0x73, 0x41, 0x72, 0x72, 0x61, 0x79, 0x28, + 0x64, 0x61, 0x74, 0x61, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, + 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x2e, 0x6e, 0x5f, 0x70, 0x72, 0x6f, + 0x62, 0x73, 0x20, 0x3e, 0x20, 0x30, 0x20, 0x26, 0x26, 0x20, 0x69, 0x73, + 0x41, 0x72, 0x72, 0x61, 0x79, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, + 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x20, 0x3d, 0x20, + 0x68, 0x74, 0x6d, 0x6c, 0x60, 0x3c, 0x24, 0x7b, 0x50, 0x72, 0x6f, 0x62, + 0x61, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x69, 0x65, 0x73, 0x7d, 0x20, 0x64, + 0x61, 0x74, 0x61, 0x3d, 0x24, 0x7b, 0x64, 0x61, 0x74, 0x61, 0x7d, 0x20, + 0x2f, 0x3e, 0x60, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x7d, 0x20, 0x65, 0x6c, 0x73, 0x65, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, + 0x20, 0x74, 0x65, 0x78, 0x74, 0x20, 0x3d, 0x20, 0x69, 0x73, 0x41, 0x72, + 0x72, 0x61, 0x79, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x20, 0x3f, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x6d, 0x61, 0x70, 0x28, 0x6d, 0x73, + 0x67, 0x20, 0x3d, 0x3e, 0x20, 0x6d, 0x73, 0x67, 0x2e, 0x63, 0x6f, 0x6e, + 0x74, 0x65, 0x6e, 0x74, 0x29, 0x2e, 0x6a, 0x6f, 0x69, 0x6e, 0x28, 0x27, + 0x27, 0x29, 0x2e, 0x72, 0x65, 0x70, 0x6c, 0x61, 0x63, 0x65, 0x28, 0x2f, + 0x5e, 0x5c, 0x73, 0x2b, 0x2f, 0x2c, 0x20, 0x27, 0x27, 0x29, 0x20, 0x3a, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x64, 0x61, 0x74, 0x61, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, + 0x20, 0x3d, 0x20, 0x69, 0x73, 0x43, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, + 0x69, 0x6f, 0x6e, 0x4d, 0x6f, 0x64, 0x65, 0x20, 0x3f, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x74, 0x65, + 0x78, 0x74, 0x20, 0x3a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x68, 0x74, 0x6d, 0x6c, 0x60, 0x3c, 0x24, + 0x7b, 0x4d, 0x61, 0x72, 0x6b, 0x64, 0x6f, 0x77, 0x6e, 0x69, 0x73, 0x68, + 0x7d, 0x20, 0x74, 0x65, 0x78, 0x74, 0x3d, 0x24, 0x7b, 0x74, 0x65, 0x6d, + 0x70, 0x6c, 0x61, 0x74, 0x65, 0x28, 0x74, 0x65, 0x78, 0x74, 0x29, 0x7d, + 0x20, 0x2f, 0x3e, 0x60, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, + 0x66, 0x20, 0x28, 0x75, 0x73, 0x65, 0x72, 0x29, 0x20, 0x7b, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, 0x74, + 0x75, 0x72, 0x6e, 0x20, 0x68, 0x74, 0x6d, 0x6c, 0x60, 0x3c, 0x70, 0x20, + 0x6b, 0x65, 0x79, 0x3d, 0x24, 0x7b, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x7d, + 0x3e, 0x3c, 0x73, 0x74, 0x72, 0x6f, 0x6e, 0x67, 0x3e, 0x24, 0x7b, 0x74, + 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x28, 0x75, 0x73, 0x65, 0x72, + 0x29, 0x7d, 0x3a, 0x3c, 0x2f, 0x73, 0x74, 0x72, 0x6f, 0x6e, 0x67, 0x3e, + 0x20, 0x24, 0x7b, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x7d, 0x3c, + 0x2f, 0x70, 0x3e, 0x60, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x7d, 0x20, 0x65, 0x6c, 0x73, 0x65, 0x20, 0x7b, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, + 0x72, 0x6e, 0x20, 0x69, 0x73, 0x43, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, + 0x69, 0x6f, 0x6e, 0x4d, 0x6f, 0x64, 0x65, 0x20, 0x3f, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x68, 0x74, + 0x6d, 0x6c, 0x60, 0x3c, 0x73, 0x70, 0x61, 0x6e, 0x20, 0x6b, 0x65, 0x79, + 0x3d, 0x24, 0x7b, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x7d, 0x3e, 0x24, 0x7b, + 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x7d, 0x3c, 0x2f, 0x73, 0x70, + 0x61, 0x6e, 0x3e, 0x60, 0x20, 0x3a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x68, 0x74, 0x6d, 0x6c, 0x60, + 0x3c, 0x70, 0x20, 0x6b, 0x65, 0x79, 0x3d, 0x24, 0x7b, 0x69, 0x6e, 0x64, + 0x65, 0x78, 0x7d, 0x3e, 0x24, 0x7b, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, + 0x65, 0x7d, 0x3c, 0x2f, 0x70, 0x3e, 0x60, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x7d, 0x3b, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, + 0x6e, 0x73, 0x74, 0x20, 0x68, 0x61, 0x6e, 0x64, 0x6c, 0x65, 0x43, 0x6f, + 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x69, 0x6f, 0x6e, 0x45, 0x64, 0x69, 0x74, + 0x20, 0x3d, 0x20, 0x28, 0x65, 0x29, 0x20, 0x3d, 0x3e, 0x20, 0x7b, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x73, 0x65, 0x73, 0x73, + 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x2e, 0x70, 0x72, + 0x6f, 0x6d, 0x70, 0x74, 0x20, 0x3d, 0x20, 0x65, 0x2e, 0x74, 0x61, 0x72, + 0x67, 0x65, 0x74, 0x2e, 0x69, 0x6e, 0x6e, 0x65, 0x72, 0x54, 0x65, 0x78, + 0x74, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x73, + 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, + 0x2e, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x20, + 0x3d, 0x20, 0x5b, 0x5d, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x7d, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, 0x74, + 0x75, 0x72, 0x6e, 0x20, 0x68, 0x74, 0x6d, 0x6c, 0x60, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x64, 0x69, 0x76, 0x20, 0x69, + 0x64, 0x3d, 0x22, 0x63, 0x68, 0x61, 0x74, 0x22, 0x20, 0x72, 0x65, 0x66, + 0x3d, 0x24, 0x7b, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, + 0x7d, 0x20, 0x6b, 0x65, 0x79, 0x3d, 0x24, 0x7b, 0x6d, 0x65, 0x73, 0x73, + 0x61, 0x67, 0x65, 0x73, 0x2e, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x7d, + 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x3c, 0x69, 0x6d, 0x67, 0x20, 0x73, 0x74, 0x79, 0x6c, 0x65, 0x3d, 0x22, + 0x77, 0x69, 0x64, 0x74, 0x68, 0x3a, 0x20, 0x36, 0x30, 0x25, 0x3b, 0x24, + 0x7b, 0x21, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x61, + 0x6c, 0x75, 0x65, 0x2e, 0x69, 0x6d, 0x61, 0x67, 0x65, 0x5f, 0x73, 0x65, + 0x6c, 0x65, 0x63, 0x74, 0x65, 0x64, 0x20, 0x3f, 0x20, 0x60, 0x64, 0x69, + 0x73, 0x70, 0x6c, 0x61, 0x79, 0x3a, 0x20, 0x6e, 0x6f, 0x6e, 0x65, 0x3b, + 0x60, 0x20, 0x3a, 0x20, 0x60, 0x60, 0x7d, 0x22, 0x20, 0x73, 0x72, 0x63, + 0x3d, 0x22, 0x24, 0x7b, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x2e, + 0x76, 0x61, 0x6c, 0x75, 0x65, 0x2e, 0x69, 0x6d, 0x61, 0x67, 0x65, 0x5f, + 0x73, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x65, 0x64, 0x7d, 0x22, 0x2f, 0x3e, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, + 0x73, 0x70, 0x61, 0x6e, 0x20, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, + 0x65, 0x64, 0x69, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x3d, 0x24, 0x7b, 0x69, + 0x73, 0x43, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x69, 0x6f, 0x6e, 0x4d, + 0x6f, 0x64, 0x65, 0x7d, 0x20, 0x72, 0x65, 0x66, 0x3d, 0x24, 0x7b, 0x63, + 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x7d, 0x20, 0x6f, 0x6e, + 0x69, 0x6e, 0x70, 0x75, 0x74, 0x3d, 0x24, 0x7b, 0x68, 0x61, 0x6e, 0x64, + 0x6c, 0x65, 0x43, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x69, 0x6f, 0x6e, + 0x45, 0x64, 0x69, 0x74, 0x7d, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x24, 0x7b, 0x6d, 0x65, 0x73, + 0x73, 0x61, 0x67, 0x65, 0x73, 0x2e, 0x66, 0x6c, 0x61, 0x74, 0x4d, 0x61, + 0x70, 0x28, 0x63, 0x68, 0x61, 0x74, 0x4c, 0x69, 0x6e, 0x65, 0x29, 0x7d, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, + 0x2f, 0x73, 0x70, 0x61, 0x6e, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x3c, 0x2f, 0x64, 0x69, 0x76, 0x3e, 0x60, 0x3b, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x7d, 0x3b, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, + 0x46, 0x6f, 0x72, 0x6d, 0x20, 0x3d, 0x20, 0x28, 0x70, 0x72, 0x6f, 0x70, + 0x73, 0x29, 0x20, 0x3d, 0x3e, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x75, 0x70, 0x64, 0x61, + 0x74, 0x65, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x20, 0x3d, 0x20, + 0x28, 0x65, 0x6c, 0x29, 0x20, 0x3d, 0x3e, 0x20, 0x73, 0x65, 0x73, 0x73, + 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x20, 0x3d, 0x20, + 0x7b, 0x20, 0x2e, 0x2e, 0x2e, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, + 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x2c, 0x20, 0x5b, 0x65, 0x6c, 0x2e, + 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x2e, 0x6e, 0x61, 0x6d, 0x65, 0x5d, + 0x3a, 0x20, 0x65, 0x6c, 0x2e, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x2e, + 0x76, 0x61, 0x6c, 0x75, 0x65, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x75, 0x70, 0x64, 0x61, + 0x74, 0x65, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x20, 0x3d, 0x20, 0x28, + 0x65, 0x6c, 0x29, 0x20, 0x3d, 0x3e, 0x20, 0x70, 0x61, 0x72, 0x61, 0x6d, + 0x73, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x20, 0x3d, 0x20, 0x7b, 0x20, + 0x2e, 0x2e, 0x2e, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x2c, 0x20, 0x5b, 0x65, 0x6c, 0x2e, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x2e, 0x6e, 0x61, 0x6d, 0x65, 0x5d, 0x3a, 0x20, 0x65, 0x6c, 0x2e, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x50, - 0x61, 0x72, 0x61, 0x6d, 0x73, 0x20, 0x3d, 0x20, 0x28, 0x65, 0x6c, 0x29, - 0x20, 0x3d, 0x3e, 0x20, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x2e, 0x76, - 0x61, 0x6c, 0x75, 0x65, 0x20, 0x3d, 0x20, 0x7b, 0x20, 0x2e, 0x2e, 0x2e, - 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, - 0x2c, 0x20, 0x5b, 0x65, 0x6c, 0x2e, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, - 0x2e, 0x6e, 0x61, 0x6d, 0x65, 0x5d, 0x3a, 0x20, 0x65, 0x6c, 0x2e, 0x74, - 0x61, 0x72, 0x67, 0x65, 0x74, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x20, - 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, - 0x74, 0x20, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x50, 0x61, 0x72, 0x61, - 0x6d, 0x73, 0x46, 0x6c, 0x6f, 0x61, 0x74, 0x20, 0x3d, 0x20, 0x28, 0x65, - 0x6c, 0x29, 0x20, 0x3d, 0x3e, 0x20, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, - 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x20, 0x3d, 0x20, 0x7b, 0x20, 0x2e, - 0x2e, 0x2e, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x2e, 0x76, 0x61, 0x6c, - 0x75, 0x65, 0x2c, 0x20, 0x5b, 0x65, 0x6c, 0x2e, 0x74, 0x61, 0x72, 0x67, - 0x65, 0x74, 0x2e, 0x6e, 0x61, 0x6d, 0x65, 0x5d, 0x3a, 0x20, 0x70, 0x61, - 0x72, 0x73, 0x65, 0x46, 0x6c, 0x6f, 0x61, 0x74, 0x28, 0x65, 0x6c, 0x2e, - 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, - 0x29, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, - 0x6e, 0x73, 0x74, 0x20, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x50, 0x61, - 0x72, 0x61, 0x6d, 0x73, 0x49, 0x6e, 0x74, 0x20, 0x3d, 0x20, 0x28, 0x65, - 0x6c, 0x29, 0x20, 0x3d, 0x3e, 0x20, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, - 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x20, 0x3d, 0x20, 0x7b, 0x20, 0x2e, - 0x2e, 0x2e, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x2e, 0x76, 0x61, 0x6c, - 0x75, 0x65, 0x2c, 0x20, 0x5b, 0x65, 0x6c, 0x2e, 0x74, 0x61, 0x72, 0x67, - 0x65, 0x74, 0x2e, 0x6e, 0x61, 0x6d, 0x65, 0x5d, 0x3a, 0x20, 0x4d, 0x61, - 0x74, 0x68, 0x2e, 0x66, 0x6c, 0x6f, 0x6f, 0x72, 0x28, 0x70, 0x61, 0x72, - 0x73, 0x65, 0x46, 0x6c, 0x6f, 0x61, 0x74, 0x28, 0x65, 0x6c, 0x2e, 0x74, - 0x61, 0x72, 0x67, 0x65, 0x74, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x29, - 0x29, 0x20, 0x7d, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, - 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x67, 0x72, 0x61, 0x6d, 0x6d, 0x61, 0x72, - 0x4a, 0x73, 0x6f, 0x6e, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x50, 0x72, - 0x6f, 0x70, 0x4f, 0x72, 0x64, 0x65, 0x72, 0x20, 0x3d, 0x20, 0x73, 0x69, - 0x67, 0x6e, 0x61, 0x6c, 0x28, 0x27, 0x27, 0x29, 0x0a, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x75, 0x70, 0x64, - 0x61, 0x74, 0x65, 0x47, 0x72, 0x61, 0x6d, 0x6d, 0x61, 0x72, 0x4a, 0x73, - 0x6f, 0x6e, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x50, 0x72, 0x6f, 0x70, - 0x4f, 0x72, 0x64, 0x65, 0x72, 0x20, 0x3d, 0x20, 0x28, 0x65, 0x6c, 0x29, - 0x20, 0x3d, 0x3e, 0x20, 0x67, 0x72, 0x61, 0x6d, 0x6d, 0x61, 0x72, 0x4a, - 0x73, 0x6f, 0x6e, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x50, 0x72, 0x6f, - 0x70, 0x4f, 0x72, 0x64, 0x65, 0x72, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, - 0x20, 0x3d, 0x20, 0x65, 0x6c, 0x2e, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, - 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x63, 0x6f, 0x6e, 0x76, 0x65, - 0x72, 0x74, 0x4a, 0x53, 0x4f, 0x4e, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, - 0x47, 0x72, 0x61, 0x6d, 0x6d, 0x61, 0x72, 0x20, 0x3d, 0x20, 0x28, 0x29, - 0x20, 0x3d, 0x3e, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x74, 0x72, 0x79, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x61, 0x72, 0x61, 0x6d, 0x73, 0x46, 0x6c, 0x6f, 0x61, 0x74, 0x20, 0x3d, + 0x20, 0x28, 0x65, 0x6c, 0x29, 0x20, 0x3d, 0x3e, 0x20, 0x70, 0x61, 0x72, + 0x61, 0x6d, 0x73, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x20, 0x3d, 0x20, + 0x7b, 0x20, 0x2e, 0x2e, 0x2e, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x2e, + 0x76, 0x61, 0x6c, 0x75, 0x65, 0x2c, 0x20, 0x5b, 0x65, 0x6c, 0x2e, 0x74, + 0x61, 0x72, 0x67, 0x65, 0x74, 0x2e, 0x6e, 0x61, 0x6d, 0x65, 0x5d, 0x3a, + 0x20, 0x70, 0x61, 0x72, 0x73, 0x65, 0x46, 0x6c, 0x6f, 0x61, 0x74, 0x28, + 0x65, 0x6c, 0x2e, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x2e, 0x76, 0x61, + 0x6c, 0x75, 0x65, 0x29, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x75, 0x70, 0x64, 0x61, 0x74, + 0x65, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x49, 0x6e, 0x74, 0x20, 0x3d, + 0x20, 0x28, 0x65, 0x6c, 0x29, 0x20, 0x3d, 0x3e, 0x20, 0x70, 0x61, 0x72, + 0x61, 0x6d, 0x73, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x20, 0x3d, 0x20, + 0x7b, 0x20, 0x2e, 0x2e, 0x2e, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x2e, + 0x76, 0x61, 0x6c, 0x75, 0x65, 0x2c, 0x20, 0x5b, 0x65, 0x6c, 0x2e, 0x74, + 0x61, 0x72, 0x67, 0x65, 0x74, 0x2e, 0x6e, 0x61, 0x6d, 0x65, 0x5d, 0x3a, + 0x20, 0x4d, 0x61, 0x74, 0x68, 0x2e, 0x66, 0x6c, 0x6f, 0x6f, 0x72, 0x28, + 0x70, 0x61, 0x72, 0x73, 0x65, 0x46, 0x6c, 0x6f, 0x61, 0x74, 0x28, 0x65, + 0x6c, 0x2e, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x2e, 0x76, 0x61, 0x6c, + 0x75, 0x65, 0x29, 0x29, 0x20, 0x7d, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x67, 0x72, 0x61, 0x6d, + 0x6d, 0x61, 0x72, 0x4a, 0x73, 0x6f, 0x6e, 0x53, 0x63, 0x68, 0x65, 0x6d, + 0x61, 0x50, 0x72, 0x6f, 0x70, 0x4f, 0x72, 0x64, 0x65, 0x72, 0x20, 0x3d, + 0x20, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x28, 0x27, 0x27, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, - 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x20, 0x3d, 0x20, 0x4a, 0x53, 0x4f, - 0x4e, 0x2e, 0x70, 0x61, 0x72, 0x73, 0x65, 0x28, 0x70, 0x61, 0x72, 0x61, - 0x6d, 0x73, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x2e, 0x67, 0x72, 0x61, - 0x6d, 0x6d, 0x61, 0x72, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x63, 0x6f, - 0x6e, 0x76, 0x65, 0x72, 0x74, 0x65, 0x72, 0x20, 0x3d, 0x20, 0x6e, 0x65, - 0x77, 0x20, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x43, 0x6f, 0x6e, 0x76, - 0x65, 0x72, 0x74, 0x65, 0x72, 0x28, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x67, 0x72, 0x61, 0x6d, 0x6d, + 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x47, 0x72, 0x61, 0x6d, 0x6d, 0x61, + 0x72, 0x4a, 0x73, 0x6f, 0x6e, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x50, + 0x72, 0x6f, 0x70, 0x4f, 0x72, 0x64, 0x65, 0x72, 0x20, 0x3d, 0x20, 0x28, + 0x65, 0x6c, 0x29, 0x20, 0x3d, 0x3e, 0x20, 0x67, 0x72, 0x61, 0x6d, 0x6d, 0x61, 0x72, 0x4a, 0x73, 0x6f, 0x6e, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x50, 0x72, 0x6f, 0x70, 0x4f, 0x72, 0x64, 0x65, 0x72, 0x2e, 0x76, 0x61, - 0x6c, 0x75, 0x65, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x2e, 0x73, 0x70, 0x6c, 0x69, 0x74, - 0x28, 0x27, 0x2c, 0x27, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x2e, 0x72, 0x65, 0x64, - 0x75, 0x63, 0x65, 0x28, 0x28, 0x61, 0x63, 0x63, 0x2c, 0x20, 0x63, 0x75, - 0x72, 0x2c, 0x20, 0x69, 0x29, 0x20, 0x3d, 0x3e, 0x20, 0x28, 0x7b, 0x20, - 0x2e, 0x2e, 0x2e, 0x61, 0x63, 0x63, 0x2c, 0x20, 0x5b, 0x63, 0x75, 0x72, - 0x2e, 0x74, 0x72, 0x69, 0x6d, 0x28, 0x29, 0x5d, 0x3a, 0x20, 0x69, 0x20, - 0x7d, 0x29, 0x2c, 0x20, 0x7b, 0x7d, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x76, 0x65, 0x72, - 0x74, 0x65, 0x72, 0x2e, 0x76, 0x69, 0x73, 0x69, 0x74, 0x28, 0x73, 0x63, - 0x68, 0x65, 0x6d, 0x61, 0x2c, 0x20, 0x27, 0x27, 0x29, 0x0a, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x70, 0x61, 0x72, 0x61, - 0x6d, 0x73, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x20, 0x3d, 0x20, 0x7b, - 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x2e, 0x2e, 0x2e, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x2e, 0x76, - 0x61, 0x6c, 0x75, 0x65, 0x2c, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x67, 0x72, 0x61, 0x6d, 0x6d, 0x61, - 0x72, 0x3a, 0x20, 0x63, 0x6f, 0x6e, 0x76, 0x65, 0x72, 0x74, 0x65, 0x72, - 0x2e, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x47, 0x72, 0x61, 0x6d, 0x6d, - 0x61, 0x72, 0x28, 0x29, 0x2c, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x7d, 0x20, 0x63, 0x61, 0x74, 0x63, 0x68, 0x20, 0x28, 0x65, - 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x61, 0x6c, 0x65, 0x72, 0x74, 0x28, 0x60, 0x43, 0x6f, 0x6e, - 0x76, 0x65, 0x72, 0x74, 0x20, 0x66, 0x61, 0x69, 0x6c, 0x65, 0x64, 0x3a, - 0x20, 0x24, 0x7b, 0x65, 0x2e, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, - 0x7d, 0x60, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x0a, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x46, - 0x6c, 0x6f, 0x61, 0x74, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x20, 0x3d, 0x20, - 0x28, 0x7b, 0x20, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x2c, 0x20, 0x6d, 0x61, - 0x78, 0x2c, 0x20, 0x6d, 0x69, 0x6e, 0x2c, 0x20, 0x6e, 0x61, 0x6d, 0x65, - 0x2c, 0x20, 0x73, 0x74, 0x65, 0x70, 0x2c, 0x20, 0x76, 0x61, 0x6c, 0x75, - 0x65, 0x20, 0x7d, 0x29, 0x20, 0x3d, 0x3e, 0x20, 0x7b, 0x0a, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, + 0x6c, 0x75, 0x65, 0x20, 0x3d, 0x20, 0x65, 0x6c, 0x2e, 0x74, 0x61, 0x72, + 0x67, 0x65, 0x74, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x63, 0x6f, + 0x6e, 0x76, 0x65, 0x72, 0x74, 0x4a, 0x53, 0x4f, 0x4e, 0x53, 0x63, 0x68, + 0x65, 0x6d, 0x61, 0x47, 0x72, 0x61, 0x6d, 0x6d, 0x61, 0x72, 0x20, 0x3d, + 0x20, 0x61, 0x73, 0x79, 0x6e, 0x63, 0x20, 0x28, 0x29, 0x20, 0x3d, 0x3e, + 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x74, + 0x72, 0x79, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x6c, 0x65, 0x74, 0x20, 0x73, 0x63, 0x68, 0x65, 0x6d, + 0x61, 0x20, 0x3d, 0x20, 0x4a, 0x53, 0x4f, 0x4e, 0x2e, 0x70, 0x61, 0x72, + 0x73, 0x65, 0x28, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x2e, 0x76, 0x61, + 0x6c, 0x75, 0x65, 0x2e, 0x67, 0x72, 0x61, 0x6d, 0x6d, 0x61, 0x72, 0x29, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, + 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x63, 0x6f, 0x6e, 0x76, 0x65, 0x72, 0x74, + 0x65, 0x72, 0x20, 0x3d, 0x20, 0x6e, 0x65, 0x77, 0x20, 0x53, 0x63, 0x68, + 0x65, 0x6d, 0x61, 0x43, 0x6f, 0x6e, 0x76, 0x65, 0x72, 0x74, 0x65, 0x72, + 0x28, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x70, 0x72, 0x6f, 0x70, 0x5f, 0x6f, 0x72, 0x64, 0x65, + 0x72, 0x3a, 0x20, 0x67, 0x72, 0x61, 0x6d, 0x6d, 0x61, 0x72, 0x4a, 0x73, + 0x6f, 0x6e, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x50, 0x72, 0x6f, 0x70, + 0x4f, 0x72, 0x64, 0x65, 0x72, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x2e, 0x73, 0x70, 0x6c, 0x69, 0x74, 0x28, 0x27, 0x2c, 0x27, + 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x2e, 0x72, 0x65, 0x64, 0x75, 0x63, 0x65, 0x28, + 0x28, 0x61, 0x63, 0x63, 0x2c, 0x20, 0x63, 0x75, 0x72, 0x2c, 0x20, 0x69, + 0x29, 0x20, 0x3d, 0x3e, 0x20, 0x28, 0x7b, 0x20, 0x2e, 0x2e, 0x2e, 0x61, + 0x63, 0x63, 0x2c, 0x20, 0x5b, 0x63, 0x75, 0x72, 0x2e, 0x74, 0x72, 0x69, + 0x6d, 0x28, 0x29, 0x5d, 0x3a, 0x20, 0x69, 0x20, 0x7d, 0x29, 0x2c, 0x20, + 0x7b, 0x7d, 0x29, 0x2c, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x61, 0x6c, 0x6c, 0x6f, 0x77, 0x5f, 0x66, + 0x65, 0x74, 0x63, 0x68, 0x3a, 0x20, 0x74, 0x72, 0x75, 0x65, 0x2c, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x29, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x73, + 0x63, 0x68, 0x65, 0x6d, 0x61, 0x20, 0x3d, 0x20, 0x61, 0x77, 0x61, 0x69, + 0x74, 0x20, 0x63, 0x6f, 0x6e, 0x76, 0x65, 0x72, 0x74, 0x65, 0x72, 0x2e, + 0x72, 0x65, 0x73, 0x6f, 0x6c, 0x76, 0x65, 0x52, 0x65, 0x66, 0x73, 0x28, + 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2c, 0x20, 0x27, 0x69, 0x6e, 0x70, + 0x75, 0x74, 0x27, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x76, 0x65, 0x72, 0x74, 0x65, 0x72, + 0x2e, 0x76, 0x69, 0x73, 0x69, 0x74, 0x28, 0x73, 0x63, 0x68, 0x65, 0x6d, + 0x61, 0x2c, 0x20, 0x27, 0x27, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x2e, + 0x76, 0x61, 0x6c, 0x75, 0x65, 0x20, 0x3d, 0x20, 0x7b, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x2e, 0x2e, + 0x2e, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x2e, 0x76, 0x61, 0x6c, 0x75, + 0x65, 0x2c, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x67, 0x72, 0x61, 0x6d, 0x6d, 0x61, 0x72, 0x3a, 0x20, + 0x63, 0x6f, 0x6e, 0x76, 0x65, 0x72, 0x74, 0x65, 0x72, 0x2e, 0x66, 0x6f, + 0x72, 0x6d, 0x61, 0x74, 0x47, 0x72, 0x61, 0x6d, 0x6d, 0x61, 0x72, 0x28, + 0x29, 0x2c, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, + 0x20, 0x63, 0x61, 0x74, 0x63, 0x68, 0x20, 0x28, 0x65, 0x29, 0x20, 0x7b, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x61, + 0x6c, 0x65, 0x72, 0x74, 0x28, 0x60, 0x43, 0x6f, 0x6e, 0x76, 0x65, 0x72, + 0x74, 0x20, 0x66, 0x61, 0x69, 0x6c, 0x65, 0x64, 0x3a, 0x20, 0x24, 0x7b, + 0x65, 0x2e, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x7d, 0x60, 0x29, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x46, 0x6c, 0x6f, 0x61, + 0x74, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x20, 0x3d, 0x20, 0x28, 0x7b, 0x20, + 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x2c, 0x20, 0x6d, 0x61, 0x78, 0x2c, 0x20, + 0x6d, 0x69, 0x6e, 0x2c, 0x20, 0x6e, 0x61, 0x6d, 0x65, 0x2c, 0x20, 0x73, + 0x74, 0x65, 0x70, 0x2c, 0x20, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x20, 0x7d, + 0x29, 0x20, 0x3d, 0x3e, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x68, 0x74, + 0x6d, 0x6c, 0x60, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x3c, 0x64, 0x69, 0x76, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x6c, 0x61, 0x62, + 0x65, 0x6c, 0x20, 0x66, 0x6f, 0x72, 0x3d, 0x22, 0x24, 0x7b, 0x6e, 0x61, + 0x6d, 0x65, 0x7d, 0x22, 0x3e, 0x24, 0x7b, 0x6c, 0x61, 0x62, 0x65, 0x6c, + 0x7d, 0x3c, 0x2f, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x3e, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x69, + 0x6e, 0x70, 0x75, 0x74, 0x20, 0x74, 0x79, 0x70, 0x65, 0x3d, 0x22, 0x72, + 0x61, 0x6e, 0x67, 0x65, 0x22, 0x20, 0x69, 0x64, 0x3d, 0x22, 0x24, 0x7b, + 0x6e, 0x61, 0x6d, 0x65, 0x7d, 0x22, 0x20, 0x6d, 0x69, 0x6e, 0x3d, 0x22, + 0x24, 0x7b, 0x6d, 0x69, 0x6e, 0x7d, 0x22, 0x20, 0x6d, 0x61, 0x78, 0x3d, + 0x22, 0x24, 0x7b, 0x6d, 0x61, 0x78, 0x7d, 0x22, 0x20, 0x73, 0x74, 0x65, + 0x70, 0x3d, 0x22, 0x24, 0x7b, 0x73, 0x74, 0x65, 0x70, 0x7d, 0x22, 0x20, + 0x6e, 0x61, 0x6d, 0x65, 0x3d, 0x22, 0x24, 0x7b, 0x6e, 0x61, 0x6d, 0x65, + 0x7d, 0x22, 0x20, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3d, 0x22, 0x24, 0x7b, + 0x76, 0x61, 0x6c, 0x75, 0x65, 0x7d, 0x22, 0x20, 0x6f, 0x6e, 0x69, 0x6e, + 0x70, 0x75, 0x74, 0x3d, 0x24, 0x7b, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, + 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x46, 0x6c, 0x6f, 0x61, 0x74, 0x7d, + 0x20, 0x2f, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x3c, 0x73, 0x70, 0x61, 0x6e, 0x3e, 0x24, 0x7b, + 0x76, 0x61, 0x6c, 0x75, 0x65, 0x7d, 0x3c, 0x2f, 0x73, 0x70, 0x61, 0x6e, + 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x3c, 0x2f, 0x64, 0x69, 0x76, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x60, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, + 0x3b, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, + 0x73, 0x74, 0x20, 0x49, 0x6e, 0x74, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x20, + 0x3d, 0x20, 0x28, 0x7b, 0x20, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x2c, 0x20, + 0x6d, 0x61, 0x78, 0x2c, 0x20, 0x6d, 0x69, 0x6e, 0x2c, 0x20, 0x6e, 0x61, + 0x6d, 0x65, 0x2c, 0x20, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x20, 0x7d, 0x29, + 0x20, 0x3d, 0x3e, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x68, 0x74, 0x6d, + 0x6c, 0x60, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x3c, 0x64, 0x69, 0x76, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x6c, 0x61, 0x62, 0x65, + 0x6c, 0x20, 0x66, 0x6f, 0x72, 0x3d, 0x22, 0x24, 0x7b, 0x6e, 0x61, 0x6d, + 0x65, 0x7d, 0x22, 0x3e, 0x24, 0x7b, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x7d, + 0x3c, 0x2f, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x3e, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x69, 0x6e, + 0x70, 0x75, 0x74, 0x20, 0x74, 0x79, 0x70, 0x65, 0x3d, 0x22, 0x72, 0x61, + 0x6e, 0x67, 0x65, 0x22, 0x20, 0x69, 0x64, 0x3d, 0x22, 0x24, 0x7b, 0x6e, + 0x61, 0x6d, 0x65, 0x7d, 0x22, 0x20, 0x6d, 0x69, 0x6e, 0x3d, 0x22, 0x24, + 0x7b, 0x6d, 0x69, 0x6e, 0x7d, 0x22, 0x20, 0x6d, 0x61, 0x78, 0x3d, 0x22, + 0x24, 0x7b, 0x6d, 0x61, 0x78, 0x7d, 0x22, 0x20, 0x6e, 0x61, 0x6d, 0x65, + 0x3d, 0x22, 0x24, 0x7b, 0x6e, 0x61, 0x6d, 0x65, 0x7d, 0x22, 0x20, 0x76, + 0x61, 0x6c, 0x75, 0x65, 0x3d, 0x22, 0x24, 0x7b, 0x76, 0x61, 0x6c, 0x75, + 0x65, 0x7d, 0x22, 0x20, 0x6f, 0x6e, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x3d, + 0x24, 0x7b, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x50, 0x61, 0x72, 0x61, + 0x6d, 0x73, 0x49, 0x6e, 0x74, 0x7d, 0x20, 0x2f, 0x3e, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x73, + 0x70, 0x61, 0x6e, 0x3e, 0x24, 0x7b, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x7d, + 0x3c, 0x2f, 0x73, 0x70, 0x61, 0x6e, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x2f, 0x64, 0x69, 0x76, 0x3e, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x60, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x3b, 0x0a, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x75, 0x73, 0x65, + 0x72, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x52, 0x65, 0x73, + 0x65, 0x74, 0x20, 0x3d, 0x20, 0x28, 0x65, 0x29, 0x20, 0x3d, 0x3e, 0x20, + 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x65, 0x2e, + 0x70, 0x72, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x44, 0x65, 0x66, 0x61, 0x75, + 0x6c, 0x74, 0x28, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x75, 0x73, 0x65, 0x72, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, + 0x74, 0x65, 0x52, 0x65, 0x73, 0x65, 0x74, 0x54, 0x6f, 0x44, 0x65, 0x66, + 0x61, 0x75, 0x6c, 0x74, 0x41, 0x6e, 0x64, 0x41, 0x70, 0x70, 0x6c, 0x79, + 0x28, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, + 0x55, 0x73, 0x65, 0x72, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, + 0x52, 0x65, 0x73, 0x65, 0x74, 0x42, 0x75, 0x74, 0x74, 0x6f, 0x6e, 0x20, + 0x3d, 0x20, 0x28, 0x29, 0x20, 0x3d, 0x3e, 0x20, 0x7b, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, 0x73, 0x65, + 0x6c, 0x65, 0x63, 0x74, 0x65, 0x64, 0x55, 0x73, 0x65, 0x72, 0x54, 0x65, + 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, + 0x2e, 0x6e, 0x61, 0x6d, 0x65, 0x20, 0x3d, 0x3d, 0x20, 0x27, 0x64, 0x65, + 0x66, 0x61, 0x75, 0x6c, 0x74, 0x27, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, + 0x72, 0x6e, 0x20, 0x68, 0x74, 0x6d, 0x6c, 0x60, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x62, 0x75, + 0x74, 0x74, 0x6f, 0x6e, 0x20, 0x64, 0x69, 0x73, 0x61, 0x62, 0x6c, 0x65, + 0x64, 0x3e, 0x55, 0x73, 0x69, 0x6e, 0x67, 0x20, 0x64, 0x65, 0x66, 0x61, + 0x75, 0x6c, 0x74, 0x20, 0x74, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, + 0x3c, 0x2f, 0x62, 0x75, 0x74, 0x74, 0x6f, 0x6e, 0x3e, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x60, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, + 0x68, 0x74, 0x6d, 0x6c, 0x60, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x3c, 0x62, 0x75, 0x74, 0x74, 0x6f, 0x6e, 0x20, + 0x6f, 0x6e, 0x63, 0x6c, 0x69, 0x63, 0x6b, 0x3d, 0x24, 0x7b, 0x75, 0x73, + 0x65, 0x72, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x52, 0x65, + 0x73, 0x65, 0x74, 0x7d, 0x3e, 0x52, 0x65, 0x73, 0x65, 0x74, 0x20, 0x61, + 0x6c, 0x6c, 0x20, 0x74, 0x6f, 0x20, 0x64, 0x65, 0x66, 0x61, 0x75, 0x6c, + 0x74, 0x3c, 0x2f, 0x62, 0x75, 0x74, 0x74, 0x6f, 0x6e, 0x3e, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x60, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x7d, 0x3b, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x75, 0x73, 0x65, 0x45, 0x66, 0x66, 0x65, 0x63, 0x74, 0x28, 0x28, + 0x29, 0x20, 0x3d, 0x3e, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x2f, 0x2f, 0x20, 0x61, 0x75, 0x74, 0x6f, 0x73, 0x61, + 0x76, 0x65, 0x20, 0x74, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x20, + 0x6f, 0x6e, 0x20, 0x65, 0x76, 0x65, 0x72, 0x79, 0x20, 0x63, 0x68, 0x61, + 0x6e, 0x67, 0x65, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x75, 0x73, 0x65, 0x72, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, + 0x41, 0x75, 0x74, 0x6f, 0x73, 0x61, 0x76, 0x65, 0x28, 0x29, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x2c, 0x20, 0x5b, 0x73, 0x65, 0x73, + 0x73, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x2c, 0x20, + 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, + 0x5d, 0x29, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, + 0x6e, 0x73, 0x74, 0x20, 0x47, 0x72, 0x61, 0x6d, 0x6d, 0x61, 0x72, 0x43, + 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x20, 0x3d, 0x20, 0x28, 0x29, 0x20, + 0x3d, 0x3e, 0x20, 0x28, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x68, 0x74, 0x6d, 0x6c, 0x60, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x64, 0x69, 0x76, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, - 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x20, 0x66, 0x6f, 0x72, 0x3d, 0x22, 0x24, - 0x7b, 0x6e, 0x61, 0x6d, 0x65, 0x7d, 0x22, 0x3e, 0x24, 0x7b, 0x6c, 0x61, - 0x62, 0x65, 0x6c, 0x7d, 0x3c, 0x2f, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x3e, + 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x20, 0x66, 0x6f, 0x72, 0x3d, 0x22, 0x74, + 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x22, 0x3e, 0x47, 0x72, 0x61, + 0x6d, 0x6d, 0x61, 0x72, 0x3c, 0x2f, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x3c, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x20, 0x74, 0x79, 0x70, 0x65, - 0x3d, 0x22, 0x72, 0x61, 0x6e, 0x67, 0x65, 0x22, 0x20, 0x69, 0x64, 0x3d, - 0x22, 0x24, 0x7b, 0x6e, 0x61, 0x6d, 0x65, 0x7d, 0x22, 0x20, 0x6d, 0x69, - 0x6e, 0x3d, 0x22, 0x24, 0x7b, 0x6d, 0x69, 0x6e, 0x7d, 0x22, 0x20, 0x6d, - 0x61, 0x78, 0x3d, 0x22, 0x24, 0x7b, 0x6d, 0x61, 0x78, 0x7d, 0x22, 0x20, - 0x73, 0x74, 0x65, 0x70, 0x3d, 0x22, 0x24, 0x7b, 0x73, 0x74, 0x65, 0x70, - 0x7d, 0x22, 0x20, 0x6e, 0x61, 0x6d, 0x65, 0x3d, 0x22, 0x24, 0x7b, 0x6e, - 0x61, 0x6d, 0x65, 0x7d, 0x22, 0x20, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3d, - 0x22, 0x24, 0x7b, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x7d, 0x22, 0x20, 0x6f, - 0x6e, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x3d, 0x24, 0x7b, 0x75, 0x70, 0x64, - 0x61, 0x74, 0x65, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x46, 0x6c, 0x6f, - 0x61, 0x74, 0x7d, 0x20, 0x2f, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x73, 0x70, 0x61, 0x6e, - 0x3e, 0x24, 0x7b, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x7d, 0x3c, 0x2f, 0x73, - 0x70, 0x61, 0x6e, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x3c, 0x2f, 0x64, 0x69, 0x76, 0x3e, 0x0a, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x60, 0x0a, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x7d, 0x3b, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x49, 0x6e, 0x74, 0x46, 0x69, 0x65, - 0x6c, 0x64, 0x20, 0x3d, 0x20, 0x28, 0x7b, 0x20, 0x6c, 0x61, 0x62, 0x65, - 0x6c, 0x2c, 0x20, 0x6d, 0x61, 0x78, 0x2c, 0x20, 0x6d, 0x69, 0x6e, 0x2c, - 0x20, 0x6e, 0x61, 0x6d, 0x65, 0x2c, 0x20, 0x76, 0x61, 0x6c, 0x75, 0x65, - 0x20, 0x7d, 0x29, 0x20, 0x3d, 0x3e, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, - 0x68, 0x74, 0x6d, 0x6c, 0x60, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x3c, 0x64, 0x69, 0x76, 0x3e, 0x0a, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x6c, - 0x61, 0x62, 0x65, 0x6c, 0x20, 0x66, 0x6f, 0x72, 0x3d, 0x22, 0x24, 0x7b, - 0x6e, 0x61, 0x6d, 0x65, 0x7d, 0x22, 0x3e, 0x24, 0x7b, 0x6c, 0x61, 0x62, - 0x65, 0x6c, 0x7d, 0x3c, 0x2f, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x3e, 0x0a, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x3c, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x20, 0x74, 0x79, 0x70, 0x65, 0x3d, - 0x22, 0x72, 0x61, 0x6e, 0x67, 0x65, 0x22, 0x20, 0x69, 0x64, 0x3d, 0x22, - 0x24, 0x7b, 0x6e, 0x61, 0x6d, 0x65, 0x7d, 0x22, 0x20, 0x6d, 0x69, 0x6e, - 0x3d, 0x22, 0x24, 0x7b, 0x6d, 0x69, 0x6e, 0x7d, 0x22, 0x20, 0x6d, 0x61, - 0x78, 0x3d, 0x22, 0x24, 0x7b, 0x6d, 0x61, 0x78, 0x7d, 0x22, 0x20, 0x6e, - 0x61, 0x6d, 0x65, 0x3d, 0x22, 0x24, 0x7b, 0x6e, 0x61, 0x6d, 0x65, 0x7d, - 0x22, 0x20, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3d, 0x22, 0x24, 0x7b, 0x76, - 0x61, 0x6c, 0x75, 0x65, 0x7d, 0x22, 0x20, 0x6f, 0x6e, 0x69, 0x6e, 0x70, - 0x75, 0x74, 0x3d, 0x24, 0x7b, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x50, - 0x61, 0x72, 0x61, 0x6d, 0x73, 0x49, 0x6e, 0x74, 0x7d, 0x20, 0x2f, 0x3e, - 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x3c, 0x73, 0x70, 0x61, 0x6e, 0x3e, 0x24, 0x7b, 0x76, 0x61, 0x6c, - 0x75, 0x65, 0x7d, 0x3c, 0x2f, 0x73, 0x70, 0x61, 0x6e, 0x3e, 0x0a, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x2f, 0x64, - 0x69, 0x76, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x60, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x3b, 0x0a, 0x0a, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, - 0x75, 0x73, 0x65, 0x72, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, - 0x52, 0x65, 0x73, 0x65, 0x74, 0x20, 0x3d, 0x20, 0x28, 0x65, 0x29, 0x20, - 0x3d, 0x3e, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x65, 0x2e, 0x70, 0x72, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x44, 0x65, - 0x66, 0x61, 0x75, 0x6c, 0x74, 0x28, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x75, 0x73, 0x65, 0x72, 0x54, 0x65, 0x6d, - 0x70, 0x6c, 0x61, 0x74, 0x65, 0x52, 0x65, 0x73, 0x65, 0x74, 0x54, 0x6f, - 0x44, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x41, 0x6e, 0x64, 0x41, 0x70, - 0x70, 0x6c, 0x79, 0x28, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x7d, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, - 0x73, 0x74, 0x20, 0x55, 0x73, 0x65, 0x72, 0x54, 0x65, 0x6d, 0x70, 0x6c, - 0x61, 0x74, 0x65, 0x52, 0x65, 0x73, 0x65, 0x74, 0x42, 0x75, 0x74, 0x74, - 0x6f, 0x6e, 0x20, 0x3d, 0x20, 0x28, 0x29, 0x20, 0x3d, 0x3e, 0x20, 0x7b, - 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, - 0x28, 0x73, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x65, 0x64, 0x55, 0x73, 0x65, - 0x72, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x2e, 0x76, 0x61, - 0x6c, 0x75, 0x65, 0x2e, 0x6e, 0x61, 0x6d, 0x65, 0x20, 0x3d, 0x3d, 0x20, - 0x27, 0x64, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x27, 0x29, 0x20, 0x7b, - 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, - 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x68, 0x74, 0x6d, 0x6c, 0x60, 0x0a, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x3c, 0x62, 0x75, 0x74, 0x74, 0x6f, 0x6e, 0x20, 0x64, 0x69, 0x73, 0x61, - 0x62, 0x6c, 0x65, 0x64, 0x3e, 0x55, 0x73, 0x69, 0x6e, 0x67, 0x20, 0x64, - 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x20, 0x74, 0x65, 0x6d, 0x70, 0x6c, - 0x61, 0x74, 0x65, 0x3c, 0x2f, 0x62, 0x75, 0x74, 0x74, 0x6f, 0x6e, 0x3e, - 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x60, - 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x0a, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, - 0x72, 0x6e, 0x20, 0x68, 0x74, 0x6d, 0x6c, 0x60, 0x0a, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x62, 0x75, 0x74, 0x74, - 0x6f, 0x6e, 0x20, 0x6f, 0x6e, 0x63, 0x6c, 0x69, 0x63, 0x6b, 0x3d, 0x24, - 0x7b, 0x75, 0x73, 0x65, 0x72, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, - 0x65, 0x52, 0x65, 0x73, 0x65, 0x74, 0x7d, 0x3e, 0x52, 0x65, 0x73, 0x65, - 0x74, 0x20, 0x61, 0x6c, 0x6c, 0x20, 0x74, 0x6f, 0x20, 0x64, 0x65, 0x66, - 0x61, 0x75, 0x6c, 0x74, 0x3c, 0x2f, 0x62, 0x75, 0x74, 0x74, 0x6f, 0x6e, - 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x60, 0x0a, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x3b, 0x0a, 0x0a, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x75, 0x73, 0x65, 0x45, 0x66, 0x66, 0x65, 0x63, - 0x74, 0x28, 0x28, 0x29, 0x20, 0x3d, 0x3e, 0x20, 0x7b, 0x0a, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x2f, 0x2f, 0x20, 0x61, 0x75, 0x74, - 0x6f, 0x73, 0x61, 0x76, 0x65, 0x20, 0x74, 0x65, 0x6d, 0x70, 0x6c, 0x61, - 0x74, 0x65, 0x20, 0x6f, 0x6e, 0x20, 0x65, 0x76, 0x65, 0x72, 0x79, 0x20, - 0x63, 0x68, 0x61, 0x6e, 0x67, 0x65, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x75, 0x73, 0x65, 0x72, 0x54, 0x65, 0x6d, 0x70, 0x6c, - 0x61, 0x74, 0x65, 0x41, 0x75, 0x74, 0x6f, 0x73, 0x61, 0x76, 0x65, 0x28, - 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x2c, 0x20, 0x5b, - 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x61, 0x6c, 0x75, - 0x65, 0x2c, 0x20, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x2e, 0x76, 0x61, - 0x6c, 0x75, 0x65, 0x5d, 0x29, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x47, 0x72, 0x61, 0x6d, 0x6d, - 0x61, 0x72, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x20, 0x3d, 0x20, - 0x28, 0x29, 0x20, 0x3d, 0x3e, 0x20, 0x28, 0x0a, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x68, 0x74, 0x6d, 0x6c, 0x60, 0x0a, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x64, 0x69, 0x76, - 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x3c, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x20, 0x66, 0x6f, 0x72, - 0x3d, 0x22, 0x74, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x22, 0x3e, - 0x47, 0x72, 0x61, 0x6d, 0x6d, 0x61, 0x72, 0x3c, 0x2f, 0x6c, 0x61, 0x62, - 0x65, 0x6c, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x3c, 0x74, 0x65, 0x78, 0x74, 0x61, 0x72, 0x65, - 0x61, 0x20, 0x69, 0x64, 0x3d, 0x22, 0x67, 0x72, 0x61, 0x6d, 0x6d, 0x61, - 0x72, 0x22, 0x20, 0x6e, 0x61, 0x6d, 0x65, 0x3d, 0x22, 0x67, 0x72, 0x61, - 0x6d, 0x6d, 0x61, 0x72, 0x22, 0x20, 0x70, 0x6c, 0x61, 0x63, 0x65, 0x68, - 0x6f, 0x6c, 0x64, 0x65, 0x72, 0x3d, 0x22, 0x55, 0x73, 0x65, 0x20, 0x67, - 0x62, 0x6e, 0x66, 0x20, 0x6f, 0x72, 0x20, 0x4a, 0x53, 0x4f, 0x4e, 0x20, - 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2b, 0x63, 0x6f, 0x6e, 0x76, 0x65, - 0x72, 0x74, 0x22, 0x20, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3d, 0x22, 0x24, - 0x7b, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x2e, 0x76, 0x61, 0x6c, 0x75, - 0x65, 0x2e, 0x67, 0x72, 0x61, 0x6d, 0x6d, 0x61, 0x72, 0x7d, 0x22, 0x20, - 0x72, 0x6f, 0x77, 0x73, 0x3d, 0x34, 0x20, 0x6f, 0x6e, 0x69, 0x6e, 0x70, - 0x75, 0x74, 0x3d, 0x24, 0x7b, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x50, - 0x61, 0x72, 0x61, 0x6d, 0x73, 0x7d, 0x2f, 0x3e, 0x0a, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x69, 0x6e, - 0x70, 0x75, 0x74, 0x20, 0x74, 0x79, 0x70, 0x65, 0x3d, 0x22, 0x74, 0x65, - 0x78, 0x74, 0x22, 0x20, 0x6e, 0x61, 0x6d, 0x65, 0x3d, 0x22, 0x70, 0x72, - 0x6f, 0x70, 0x2d, 0x6f, 0x72, 0x64, 0x65, 0x72, 0x22, 0x20, 0x70, 0x6c, - 0x61, 0x63, 0x65, 0x68, 0x6f, 0x6c, 0x64, 0x65, 0x72, 0x3d, 0x22, 0x6f, - 0x72, 0x64, 0x65, 0x72, 0x3a, 0x20, 0x70, 0x72, 0x6f, 0x70, 0x31, 0x2c, - 0x70, 0x72, 0x6f, 0x70, 0x32, 0x2c, 0x70, 0x72, 0x6f, 0x70, 0x33, 0x22, - 0x20, 0x6f, 0x6e, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x3d, 0x24, 0x7b, 0x75, - 0x70, 0x64, 0x61, 0x74, 0x65, 0x47, 0x72, 0x61, 0x6d, 0x6d, 0x61, 0x72, - 0x4a, 0x73, 0x6f, 0x6e, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x50, 0x72, - 0x6f, 0x70, 0x4f, 0x72, 0x64, 0x65, 0x72, 0x7d, 0x20, 0x2f, 0x3e, 0x0a, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x3c, 0x62, 0x75, 0x74, 0x74, 0x6f, 0x6e, 0x20, 0x74, 0x79, 0x70, 0x65, - 0x3d, 0x22, 0x62, 0x75, 0x74, 0x74, 0x6f, 0x6e, 0x22, 0x20, 0x6f, 0x6e, - 0x63, 0x6c, 0x69, 0x63, 0x6b, 0x3d, 0x24, 0x7b, 0x63, 0x6f, 0x6e, 0x76, - 0x65, 0x72, 0x74, 0x4a, 0x53, 0x4f, 0x4e, 0x53, 0x63, 0x68, 0x65, 0x6d, - 0x61, 0x47, 0x72, 0x61, 0x6d, 0x6d, 0x61, 0x72, 0x7d, 0x3e, 0x43, 0x6f, - 0x6e, 0x76, 0x65, 0x72, 0x74, 0x20, 0x4a, 0x53, 0x4f, 0x4e, 0x20, 0x53, - 0x63, 0x68, 0x65, 0x6d, 0x61, 0x3c, 0x2f, 0x62, 0x75, 0x74, 0x74, 0x6f, - 0x6e, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x3c, 0x2f, 0x64, 0x69, 0x76, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x60, 0x0a, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x29, 0x3b, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x50, 0x72, 0x6f, 0x6d, 0x70, 0x74, - 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x46, 0x69, 0x65, 0x6c, 0x64, - 0x53, 0x65, 0x74, 0x20, 0x3d, 0x20, 0x28, 0x29, 0x20, 0x3d, 0x3e, 0x20, - 0x28, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x68, 0x74, - 0x6d, 0x6c, 0x60, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x3c, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x65, 0x74, 0x3e, 0x0a, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x64, 0x69, - 0x76, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x3c, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x20, 0x68, 0x74, - 0x6d, 0x6c, 0x46, 0x6f, 0x72, 0x3d, 0x22, 0x70, 0x72, 0x6f, 0x6d, 0x70, - 0x74, 0x22, 0x3e, 0x50, 0x72, 0x6f, 0x6d, 0x70, 0x74, 0x3c, 0x2f, 0x6c, - 0x61, 0x62, 0x65, 0x6c, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x74, 0x65, 0x78, 0x74, 0x61, - 0x72, 0x65, 0x61, 0x20, 0x74, 0x79, 0x70, 0x65, 0x3d, 0x22, 0x74, 0x65, - 0x78, 0x74, 0x22, 0x20, 0x6e, 0x61, 0x6d, 0x65, 0x3d, 0x22, 0x70, 0x72, - 0x6f, 0x6d, 0x70, 0x74, 0x22, 0x20, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3d, - 0x22, 0x24, 0x7b, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x2e, 0x76, - 0x61, 0x6c, 0x75, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x6d, 0x70, 0x74, 0x7d, - 0x22, 0x20, 0x6f, 0x6e, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x3d, 0x24, 0x7b, - 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, - 0x6e, 0x7d, 0x2f, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x3c, 0x2f, 0x64, 0x69, 0x76, 0x3e, 0x0a, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x2f, 0x66, 0x69, 0x65, 0x6c, - 0x64, 0x73, 0x65, 0x74, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x60, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x29, 0x3b, - 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, - 0x74, 0x20, 0x43, 0x68, 0x61, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, - 0x46, 0x6f, 0x72, 0x6d, 0x20, 0x3d, 0x20, 0x28, 0x29, 0x20, 0x3d, 0x3e, - 0x20, 0x28, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x68, - 0x74, 0x6d, 0x6c, 0x60, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x24, 0x7b, 0x50, 0x72, 0x6f, 0x6d, 0x70, 0x74, 0x43, - 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x53, - 0x65, 0x74, 0x28, 0x29, 0x7d, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x73, - 0x65, 0x74, 0x20, 0x63, 0x6c, 0x61, 0x73, 0x73, 0x3d, 0x22, 0x74, 0x77, - 0x6f, 0x22, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x3c, 0x64, 0x69, 0x76, 0x3e, 0x0a, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x3c, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x20, 0x66, 0x6f, 0x72, 0x3d, 0x22, - 0x75, 0x73, 0x65, 0x72, 0x22, 0x3e, 0x55, 0x73, 0x65, 0x72, 0x20, 0x6e, - 0x61, 0x6d, 0x65, 0x3c, 0x2f, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x3e, 0x0a, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x3c, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x20, 0x74, 0x79, 0x70, - 0x65, 0x3d, 0x22, 0x74, 0x65, 0x78, 0x74, 0x22, 0x20, 0x6e, 0x61, 0x6d, - 0x65, 0x3d, 0x22, 0x75, 0x73, 0x65, 0x72, 0x22, 0x20, 0x76, 0x61, 0x6c, - 0x75, 0x65, 0x3d, 0x22, 0x24, 0x7b, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, - 0x6e, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x2e, 0x75, 0x73, 0x65, 0x72, - 0x7d, 0x22, 0x20, 0x6f, 0x6e, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x3d, 0x24, - 0x7b, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x53, 0x65, 0x73, 0x73, 0x69, - 0x6f, 0x6e, 0x7d, 0x20, 0x2f, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x2f, 0x64, 0x69, 0x76, - 0x3e, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x3c, 0x64, 0x69, 0x76, 0x3e, 0x0a, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, - 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x20, 0x66, 0x6f, 0x72, 0x3d, 0x22, 0x62, - 0x6f, 0x74, 0x22, 0x3e, 0x42, 0x6f, 0x74, 0x20, 0x6e, 0x61, 0x6d, 0x65, - 0x3c, 0x2f, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x3e, 0x0a, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, - 0x69, 0x6e, 0x70, 0x75, 0x74, 0x20, 0x74, 0x79, 0x70, 0x65, 0x3d, 0x22, - 0x74, 0x65, 0x78, 0x74, 0x22, 0x20, 0x6e, 0x61, 0x6d, 0x65, 0x3d, 0x22, - 0x63, 0x68, 0x61, 0x72, 0x22, 0x20, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3d, - 0x22, 0x24, 0x7b, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x2e, 0x76, - 0x61, 0x6c, 0x75, 0x65, 0x2e, 0x63, 0x68, 0x61, 0x72, 0x7d, 0x22, 0x20, - 0x6f, 0x6e, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x3d, 0x24, 0x7b, 0x75, 0x70, - 0x64, 0x61, 0x74, 0x65, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x7d, - 0x20, 0x2f, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x3c, 0x2f, 0x64, 0x69, 0x76, 0x3e, 0x0a, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x2f, 0x66, - 0x69, 0x65, 0x6c, 0x64, 0x73, 0x65, 0x74, 0x3e, 0x0a, 0x0a, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x66, 0x69, 0x65, - 0x6c, 0x64, 0x73, 0x65, 0x74, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x64, 0x69, 0x76, 0x3e, - 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x3c, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x20, 0x66, 0x6f, - 0x72, 0x3d, 0x22, 0x74, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x22, - 0x3e, 0x50, 0x72, 0x6f, 0x6d, 0x70, 0x74, 0x20, 0x74, 0x65, 0x6d, 0x70, - 0x6c, 0x61, 0x74, 0x65, 0x3c, 0x2f, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x3e, - 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x3c, 0x74, 0x65, 0x78, 0x74, 0x61, 0x72, 0x65, 0x61, - 0x20, 0x69, 0x64, 0x3d, 0x22, 0x74, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, - 0x65, 0x22, 0x20, 0x6e, 0x61, 0x6d, 0x65, 0x3d, 0x22, 0x74, 0x65, 0x6d, - 0x70, 0x6c, 0x61, 0x74, 0x65, 0x22, 0x20, 0x76, 0x61, 0x6c, 0x75, 0x65, - 0x3d, 0x22, 0x24, 0x7b, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x2e, - 0x76, 0x61, 0x6c, 0x75, 0x65, 0x2e, 0x74, 0x65, 0x6d, 0x70, 0x6c, 0x61, - 0x74, 0x65, 0x7d, 0x22, 0x20, 0x72, 0x6f, 0x77, 0x73, 0x3d, 0x34, 0x20, - 0x6f, 0x6e, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x3d, 0x24, 0x7b, 0x75, 0x70, - 0x64, 0x61, 0x74, 0x65, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x7d, - 0x2f, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x3c, 0x2f, 0x64, 0x69, 0x76, 0x3e, 0x0a, 0x0a, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, - 0x64, 0x69, 0x76, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x6c, 0x61, 0x62, 0x65, - 0x6c, 0x20, 0x66, 0x6f, 0x72, 0x3d, 0x22, 0x74, 0x65, 0x6d, 0x70, 0x6c, - 0x61, 0x74, 0x65, 0x22, 0x3e, 0x43, 0x68, 0x61, 0x74, 0x20, 0x68, 0x69, - 0x73, 0x74, 0x6f, 0x72, 0x79, 0x20, 0x74, 0x65, 0x6d, 0x70, 0x6c, 0x61, - 0x74, 0x65, 0x3c, 0x2f, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x3e, 0x0a, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x74, 0x65, 0x78, 0x74, 0x61, 0x72, 0x65, 0x61, 0x20, 0x69, - 0x64, 0x3d, 0x22, 0x74, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x22, - 0x20, 0x6e, 0x61, 0x6d, 0x65, 0x3d, 0x22, 0x68, 0x69, 0x73, 0x74, 0x6f, - 0x72, 0x79, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x22, 0x20, - 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3d, 0x22, 0x24, 0x7b, 0x73, 0x65, 0x73, - 0x73, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x2e, 0x68, - 0x69, 0x73, 0x74, 0x6f, 0x72, 0x79, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, - 0x74, 0x65, 0x7d, 0x22, 0x20, 0x72, 0x6f, 0x77, 0x73, 0x3d, 0x31, 0x20, - 0x6f, 0x6e, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x3d, 0x24, 0x7b, 0x75, 0x70, - 0x64, 0x61, 0x74, 0x65, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x7d, - 0x2f, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x3c, 0x2f, 0x64, 0x69, 0x76, 0x3e, 0x0a, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x24, 0x7b, - 0x47, 0x72, 0x61, 0x6d, 0x6d, 0x61, 0x72, 0x43, 0x6f, 0x6e, 0x74, 0x72, - 0x6f, 0x6c, 0x28, 0x29, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x3c, 0x2f, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x73, - 0x65, 0x74, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x60, 0x0a, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x29, 0x3b, 0x0a, 0x0a, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x43, 0x6f, - 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, - 0x69, 0x67, 0x46, 0x6f, 0x72, 0x6d, 0x20, 0x3d, 0x20, 0x28, 0x29, 0x20, - 0x3d, 0x3e, 0x20, 0x28, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x68, 0x74, 0x6d, 0x6c, 0x60, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x24, 0x7b, 0x50, 0x72, 0x6f, 0x6d, 0x70, - 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x46, 0x69, 0x65, 0x6c, - 0x64, 0x53, 0x65, 0x74, 0x28, 0x29, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x66, 0x69, 0x65, 0x6c, 0x64, - 0x73, 0x65, 0x74, 0x3e, 0x24, 0x7b, 0x47, 0x72, 0x61, 0x6d, 0x6d, 0x61, - 0x72, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x28, 0x29, 0x7d, 0x3c, - 0x2f, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x65, 0x74, 0x3e, 0x0a, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x60, 0x0a, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x29, 0x3b, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x68, 0x74, 0x6d, 0x6c, - 0x60, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x66, - 0x6f, 0x72, 0x6d, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x3c, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x65, 0x74, - 0x20, 0x63, 0x6c, 0x61, 0x73, 0x73, 0x3d, 0x22, 0x74, 0x77, 0x6f, 0x22, - 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x3c, 0x24, 0x7b, 0x55, 0x73, 0x65, 0x72, 0x54, 0x65, 0x6d, - 0x70, 0x6c, 0x61, 0x74, 0x65, 0x52, 0x65, 0x73, 0x65, 0x74, 0x42, 0x75, - 0x74, 0x74, 0x6f, 0x6e, 0x7d, 0x2f, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x64, 0x69, 0x76, - 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x3c, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x20, 0x63, - 0x6c, 0x61, 0x73, 0x73, 0x3d, 0x22, 0x73, 0x6c, 0x69, 0x6d, 0x22, 0x3e, - 0x3c, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x20, 0x74, 0x79, 0x70, 0x65, 0x3d, - 0x22, 0x72, 0x61, 0x64, 0x69, 0x6f, 0x22, 0x20, 0x6e, 0x61, 0x6d, 0x65, - 0x3d, 0x22, 0x74, 0x79, 0x70, 0x65, 0x22, 0x20, 0x76, 0x61, 0x6c, 0x75, - 0x65, 0x3d, 0x22, 0x63, 0x68, 0x61, 0x74, 0x22, 0x20, 0x63, 0x68, 0x65, - 0x63, 0x6b, 0x65, 0x64, 0x3d, 0x24, 0x7b, 0x73, 0x65, 0x73, 0x73, 0x69, - 0x6f, 0x6e, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x2e, 0x74, 0x79, 0x70, - 0x65, 0x20, 0x3d, 0x3d, 0x3d, 0x20, 0x22, 0x63, 0x68, 0x61, 0x74, 0x22, - 0x7d, 0x20, 0x6f, 0x6e, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x3d, 0x24, 0x7b, - 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, - 0x6e, 0x7d, 0x20, 0x2f, 0x3e, 0x20, 0x43, 0x68, 0x61, 0x74, 0x3c, 0x2f, - 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x6c, 0x61, - 0x62, 0x65, 0x6c, 0x20, 0x63, 0x6c, 0x61, 0x73, 0x73, 0x3d, 0x22, 0x73, - 0x6c, 0x69, 0x6d, 0x22, 0x3e, 0x3c, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x20, - 0x74, 0x79, 0x70, 0x65, 0x3d, 0x22, 0x72, 0x61, 0x64, 0x69, 0x6f, 0x22, - 0x20, 0x6e, 0x61, 0x6d, 0x65, 0x3d, 0x22, 0x74, 0x79, 0x70, 0x65, 0x22, - 0x20, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3d, 0x22, 0x63, 0x6f, 0x6d, 0x70, - 0x6c, 0x65, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x20, 0x63, 0x68, 0x65, 0x63, - 0x6b, 0x65, 0x64, 0x3d, 0x24, 0x7b, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, - 0x6e, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x2e, 0x74, 0x79, 0x70, 0x65, - 0x20, 0x3d, 0x3d, 0x3d, 0x20, 0x22, 0x63, 0x6f, 0x6d, 0x70, 0x6c, 0x65, - 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x7d, 0x20, 0x6f, 0x6e, 0x69, 0x6e, 0x70, - 0x75, 0x74, 0x3d, 0x24, 0x7b, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x53, - 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x7d, 0x20, 0x2f, 0x3e, 0x20, 0x43, - 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x69, 0x6f, 0x6e, 0x3c, 0x2f, 0x6c, - 0x61, 0x62, 0x65, 0x6c, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x2f, 0x64, 0x69, 0x76, 0x3e, - 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, - 0x2f, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x65, 0x74, 0x3e, 0x0a, 0x0a, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x24, 0x7b, + 0x64, 0x3d, 0x22, 0x67, 0x72, 0x61, 0x6d, 0x6d, 0x61, 0x72, 0x22, 0x20, + 0x6e, 0x61, 0x6d, 0x65, 0x3d, 0x22, 0x67, 0x72, 0x61, 0x6d, 0x6d, 0x61, + 0x72, 0x22, 0x20, 0x70, 0x6c, 0x61, 0x63, 0x65, 0x68, 0x6f, 0x6c, 0x64, + 0x65, 0x72, 0x3d, 0x22, 0x55, 0x73, 0x65, 0x20, 0x67, 0x62, 0x6e, 0x66, + 0x20, 0x6f, 0x72, 0x20, 0x4a, 0x53, 0x4f, 0x4e, 0x20, 0x53, 0x63, 0x68, + 0x65, 0x6d, 0x61, 0x2b, 0x63, 0x6f, 0x6e, 0x76, 0x65, 0x72, 0x74, 0x22, + 0x20, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3d, 0x22, 0x24, 0x7b, 0x70, 0x61, + 0x72, 0x61, 0x6d, 0x73, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x2e, 0x67, + 0x72, 0x61, 0x6d, 0x6d, 0x61, 0x72, 0x7d, 0x22, 0x20, 0x72, 0x6f, 0x77, + 0x73, 0x3d, 0x34, 0x20, 0x6f, 0x6e, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x3d, + 0x24, 0x7b, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x50, 0x61, 0x72, 0x61, + 0x6d, 0x73, 0x7d, 0x2f, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x69, 0x6e, 0x70, 0x75, 0x74, + 0x20, 0x74, 0x79, 0x70, 0x65, 0x3d, 0x22, 0x74, 0x65, 0x78, 0x74, 0x22, + 0x20, 0x6e, 0x61, 0x6d, 0x65, 0x3d, 0x22, 0x70, 0x72, 0x6f, 0x70, 0x2d, + 0x6f, 0x72, 0x64, 0x65, 0x72, 0x22, 0x20, 0x70, 0x6c, 0x61, 0x63, 0x65, + 0x68, 0x6f, 0x6c, 0x64, 0x65, 0x72, 0x3d, 0x22, 0x6f, 0x72, 0x64, 0x65, + 0x72, 0x3a, 0x20, 0x70, 0x72, 0x6f, 0x70, 0x31, 0x2c, 0x70, 0x72, 0x6f, + 0x70, 0x32, 0x2c, 0x70, 0x72, 0x6f, 0x70, 0x33, 0x22, 0x20, 0x6f, 0x6e, + 0x69, 0x6e, 0x70, 0x75, 0x74, 0x3d, 0x24, 0x7b, 0x75, 0x70, 0x64, 0x61, + 0x74, 0x65, 0x47, 0x72, 0x61, 0x6d, 0x6d, 0x61, 0x72, 0x4a, 0x73, 0x6f, + 0x6e, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x50, 0x72, 0x6f, 0x70, 0x4f, + 0x72, 0x64, 0x65, 0x72, 0x7d, 0x20, 0x2f, 0x3e, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x62, 0x75, + 0x74, 0x74, 0x6f, 0x6e, 0x20, 0x74, 0x79, 0x70, 0x65, 0x3d, 0x22, 0x62, + 0x75, 0x74, 0x74, 0x6f, 0x6e, 0x22, 0x20, 0x6f, 0x6e, 0x63, 0x6c, 0x69, + 0x63, 0x6b, 0x3d, 0x24, 0x7b, 0x63, 0x6f, 0x6e, 0x76, 0x65, 0x72, 0x74, + 0x4a, 0x53, 0x4f, 0x4e, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x47, 0x72, + 0x61, 0x6d, 0x6d, 0x61, 0x72, 0x7d, 0x3e, 0x43, 0x6f, 0x6e, 0x76, 0x65, + 0x72, 0x74, 0x20, 0x4a, 0x53, 0x4f, 0x4e, 0x20, 0x53, 0x63, 0x68, 0x65, + 0x6d, 0x61, 0x3c, 0x2f, 0x62, 0x75, 0x74, 0x74, 0x6f, 0x6e, 0x3e, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x2f, + 0x64, 0x69, 0x76, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x60, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x29, + 0x3b, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, + 0x73, 0x74, 0x20, 0x50, 0x72, 0x6f, 0x6d, 0x70, 0x74, 0x43, 0x6f, 0x6e, + 0x74, 0x72, 0x6f, 0x6c, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x53, 0x65, 0x74, + 0x20, 0x3d, 0x20, 0x28, 0x29, 0x20, 0x3d, 0x3e, 0x20, 0x28, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x68, 0x74, 0x6d, 0x6c, 0x60, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x66, 0x69, + 0x65, 0x6c, 0x64, 0x73, 0x65, 0x74, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x64, 0x69, 0x76, 0x3e, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x3c, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x20, 0x68, 0x74, 0x6d, 0x6c, 0x46, + 0x6f, 0x72, 0x3d, 0x22, 0x70, 0x72, 0x6f, 0x6d, 0x70, 0x74, 0x22, 0x3e, + 0x50, 0x72, 0x6f, 0x6d, 0x70, 0x74, 0x3c, 0x2f, 0x6c, 0x61, 0x62, 0x65, + 0x6c, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x3c, 0x74, 0x65, 0x78, 0x74, 0x61, 0x72, 0x65, 0x61, + 0x20, 0x74, 0x79, 0x70, 0x65, 0x3d, 0x22, 0x74, 0x65, 0x78, 0x74, 0x22, + 0x20, 0x6e, 0x61, 0x6d, 0x65, 0x3d, 0x22, 0x70, 0x72, 0x6f, 0x6d, 0x70, + 0x74, 0x22, 0x20, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3d, 0x22, 0x24, 0x7b, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x61, 0x6c, 0x75, - 0x65, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x20, 0x3d, 0x3d, 0x3d, 0x20, 0x27, - 0x63, 0x68, 0x61, 0x74, 0x27, 0x20, 0x3f, 0x20, 0x43, 0x68, 0x61, 0x74, - 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x46, 0x6f, 0x72, 0x6d, 0x28, 0x29, - 0x20, 0x3a, 0x20, 0x43, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x69, 0x6f, - 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x46, 0x6f, 0x72, 0x6d, 0x28, + 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x6d, 0x70, 0x74, 0x7d, 0x22, 0x20, 0x6f, + 0x6e, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x3d, 0x24, 0x7b, 0x75, 0x70, 0x64, + 0x61, 0x74, 0x65, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x7d, 0x2f, + 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x3c, 0x2f, 0x64, 0x69, 0x76, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x3c, 0x2f, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x65, + 0x74, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x60, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x29, 0x3b, 0x0a, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x43, + 0x68, 0x61, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x46, 0x6f, 0x72, + 0x6d, 0x20, 0x3d, 0x20, 0x28, 0x29, 0x20, 0x3d, 0x3e, 0x20, 0x28, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x68, 0x74, 0x6d, 0x6c, + 0x60, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x24, 0x7b, 0x50, 0x72, 0x6f, 0x6d, 0x70, 0x74, 0x43, 0x6f, 0x6e, 0x74, + 0x72, 0x6f, 0x6c, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x53, 0x65, 0x74, 0x28, 0x29, 0x7d, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x65, 0x74, 0x20, 0x63, 0x6c, 0x61, 0x73, 0x73, 0x3d, 0x22, 0x74, 0x77, 0x6f, 0x22, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x24, 0x7b, 0x49, 0x6e, 0x74, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x28, - 0x7b, 0x20, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x3a, 0x20, 0x22, 0x50, 0x72, - 0x65, 0x64, 0x69, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x22, 0x2c, 0x20, - 0x6d, 0x61, 0x78, 0x3a, 0x20, 0x32, 0x30, 0x34, 0x38, 0x2c, 0x20, 0x6d, - 0x69, 0x6e, 0x3a, 0x20, 0x2d, 0x31, 0x2c, 0x20, 0x6e, 0x61, 0x6d, 0x65, - 0x3a, 0x20, 0x22, 0x6e, 0x5f, 0x70, 0x72, 0x65, 0x64, 0x69, 0x63, 0x74, - 0x22, 0x2c, 0x20, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x20, 0x70, 0x61, - 0x72, 0x61, 0x6d, 0x73, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x2e, 0x6e, - 0x5f, 0x70, 0x72, 0x65, 0x64, 0x69, 0x63, 0x74, 0x20, 0x7d, 0x29, 0x7d, + 0x20, 0x3c, 0x64, 0x69, 0x76, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x6c, 0x61, + 0x62, 0x65, 0x6c, 0x20, 0x66, 0x6f, 0x72, 0x3d, 0x22, 0x75, 0x73, 0x65, + 0x72, 0x22, 0x3e, 0x55, 0x73, 0x65, 0x72, 0x20, 0x6e, 0x61, 0x6d, 0x65, + 0x3c, 0x2f, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x3e, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, + 0x69, 0x6e, 0x70, 0x75, 0x74, 0x20, 0x74, 0x79, 0x70, 0x65, 0x3d, 0x22, + 0x74, 0x65, 0x78, 0x74, 0x22, 0x20, 0x6e, 0x61, 0x6d, 0x65, 0x3d, 0x22, + 0x75, 0x73, 0x65, 0x72, 0x22, 0x20, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3d, + 0x22, 0x24, 0x7b, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x2e, 0x76, + 0x61, 0x6c, 0x75, 0x65, 0x2e, 0x75, 0x73, 0x65, 0x72, 0x7d, 0x22, 0x20, + 0x6f, 0x6e, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x3d, 0x24, 0x7b, 0x75, 0x70, + 0x64, 0x61, 0x74, 0x65, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x7d, + 0x20, 0x2f, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x3c, 0x2f, 0x64, 0x69, 0x76, 0x3e, 0x0a, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x3c, 0x64, 0x69, 0x76, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x6c, 0x61, 0x62, + 0x65, 0x6c, 0x20, 0x66, 0x6f, 0x72, 0x3d, 0x22, 0x62, 0x6f, 0x74, 0x22, + 0x3e, 0x42, 0x6f, 0x74, 0x20, 0x6e, 0x61, 0x6d, 0x65, 0x3c, 0x2f, 0x6c, + 0x61, 0x62, 0x65, 0x6c, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x69, 0x6e, 0x70, + 0x75, 0x74, 0x20, 0x74, 0x79, 0x70, 0x65, 0x3d, 0x22, 0x74, 0x65, 0x78, + 0x74, 0x22, 0x20, 0x6e, 0x61, 0x6d, 0x65, 0x3d, 0x22, 0x63, 0x68, 0x61, + 0x72, 0x22, 0x20, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3d, 0x22, 0x24, 0x7b, + 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x61, 0x6c, 0x75, + 0x65, 0x2e, 0x63, 0x68, 0x61, 0x72, 0x7d, 0x22, 0x20, 0x6f, 0x6e, 0x69, + 0x6e, 0x70, 0x75, 0x74, 0x3d, 0x24, 0x7b, 0x75, 0x70, 0x64, 0x61, 0x74, + 0x65, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x7d, 0x20, 0x2f, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x24, 0x7b, 0x46, 0x6c, 0x6f, 0x61, 0x74, 0x46, 0x69, 0x65, 0x6c, - 0x64, 0x28, 0x7b, 0x20, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x3a, 0x20, 0x22, - 0x54, 0x65, 0x6d, 0x70, 0x65, 0x72, 0x61, 0x74, 0x75, 0x72, 0x65, 0x22, - 0x2c, 0x20, 0x6d, 0x61, 0x78, 0x3a, 0x20, 0x32, 0x2e, 0x30, 0x2c, 0x20, - 0x6d, 0x69, 0x6e, 0x3a, 0x20, 0x30, 0x2e, 0x30, 0x2c, 0x20, 0x6e, 0x61, - 0x6d, 0x65, 0x3a, 0x20, 0x22, 0x74, 0x65, 0x6d, 0x70, 0x65, 0x72, 0x61, - 0x74, 0x75, 0x72, 0x65, 0x22, 0x2c, 0x20, 0x73, 0x74, 0x65, 0x70, 0x3a, - 0x20, 0x30, 0x2e, 0x30, 0x31, 0x2c, 0x20, 0x76, 0x61, 0x6c, 0x75, 0x65, - 0x3a, 0x20, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x2e, 0x76, 0x61, 0x6c, - 0x75, 0x65, 0x2e, 0x74, 0x65, 0x6d, 0x70, 0x65, 0x72, 0x61, 0x74, 0x75, - 0x72, 0x65, 0x20, 0x7d, 0x29, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x24, 0x7b, 0x46, 0x6c, 0x6f, - 0x61, 0x74, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x28, 0x7b, 0x20, 0x6c, 0x61, - 0x62, 0x65, 0x6c, 0x3a, 0x20, 0x22, 0x50, 0x65, 0x6e, 0x61, 0x6c, 0x69, - 0x7a, 0x65, 0x20, 0x72, 0x65, 0x70, 0x65, 0x61, 0x74, 0x20, 0x73, 0x65, - 0x71, 0x75, 0x65, 0x6e, 0x63, 0x65, 0x22, 0x2c, 0x20, 0x6d, 0x61, 0x78, - 0x3a, 0x20, 0x32, 0x2e, 0x30, 0x2c, 0x20, 0x6d, 0x69, 0x6e, 0x3a, 0x20, - 0x30, 0x2e, 0x30, 0x2c, 0x20, 0x6e, 0x61, 0x6d, 0x65, 0x3a, 0x20, 0x22, - 0x72, 0x65, 0x70, 0x65, 0x61, 0x74, 0x5f, 0x70, 0x65, 0x6e, 0x61, 0x6c, - 0x74, 0x79, 0x22, 0x2c, 0x20, 0x73, 0x74, 0x65, 0x70, 0x3a, 0x20, 0x30, - 0x2e, 0x30, 0x31, 0x2c, 0x20, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x20, - 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, - 0x2e, 0x72, 0x65, 0x70, 0x65, 0x61, 0x74, 0x5f, 0x70, 0x65, 0x6e, 0x61, - 0x6c, 0x74, 0x79, 0x20, 0x7d, 0x29, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x24, 0x7b, 0x49, 0x6e, - 0x74, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x28, 0x7b, 0x20, 0x6c, 0x61, 0x62, - 0x65, 0x6c, 0x3a, 0x20, 0x22, 0x43, 0x6f, 0x6e, 0x73, 0x69, 0x64, 0x65, - 0x72, 0x20, 0x4e, 0x20, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x73, 0x20, 0x66, - 0x6f, 0x72, 0x20, 0x70, 0x65, 0x6e, 0x61, 0x6c, 0x69, 0x7a, 0x65, 0x22, - 0x2c, 0x20, 0x6d, 0x61, 0x78, 0x3a, 0x20, 0x32, 0x30, 0x34, 0x38, 0x2c, - 0x20, 0x6d, 0x69, 0x6e, 0x3a, 0x20, 0x30, 0x2c, 0x20, 0x6e, 0x61, 0x6d, - 0x65, 0x3a, 0x20, 0x22, 0x72, 0x65, 0x70, 0x65, 0x61, 0x74, 0x5f, 0x6c, - 0x61, 0x73, 0x74, 0x5f, 0x6e, 0x22, 0x2c, 0x20, 0x76, 0x61, 0x6c, 0x75, - 0x65, 0x3a, 0x20, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x2e, 0x76, 0x61, - 0x6c, 0x75, 0x65, 0x2e, 0x72, 0x65, 0x70, 0x65, 0x61, 0x74, 0x5f, 0x6c, - 0x61, 0x73, 0x74, 0x5f, 0x6e, 0x20, 0x7d, 0x29, 0x7d, 0x0a, 0x20, 0x20, + 0x20, 0x3c, 0x2f, 0x64, 0x69, 0x76, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x2f, 0x66, 0x69, 0x65, 0x6c, + 0x64, 0x73, 0x65, 0x74, 0x3e, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x73, + 0x65, 0x74, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x3c, 0x64, 0x69, 0x76, 0x3e, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x3c, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x20, 0x66, 0x6f, 0x72, 0x3d, 0x22, + 0x74, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x22, 0x3e, 0x50, 0x72, + 0x6f, 0x6d, 0x70, 0x74, 0x20, 0x74, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, + 0x65, 0x3c, 0x2f, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x3e, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x3c, 0x74, 0x65, 0x78, 0x74, 0x61, 0x72, 0x65, 0x61, 0x20, 0x69, 0x64, + 0x3d, 0x22, 0x74, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x22, 0x20, + 0x6e, 0x61, 0x6d, 0x65, 0x3d, 0x22, 0x74, 0x65, 0x6d, 0x70, 0x6c, 0x61, + 0x74, 0x65, 0x22, 0x20, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3d, 0x22, 0x24, + 0x7b, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x61, 0x6c, + 0x75, 0x65, 0x2e, 0x74, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x7d, + 0x22, 0x20, 0x72, 0x6f, 0x77, 0x73, 0x3d, 0x34, 0x20, 0x6f, 0x6e, 0x69, + 0x6e, 0x70, 0x75, 0x74, 0x3d, 0x24, 0x7b, 0x75, 0x70, 0x64, 0x61, 0x74, + 0x65, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x7d, 0x2f, 0x3e, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x3c, 0x2f, 0x64, 0x69, 0x76, 0x3e, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x64, 0x69, 0x76, + 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x3c, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x20, 0x66, + 0x6f, 0x72, 0x3d, 0x22, 0x74, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, + 0x22, 0x3e, 0x43, 0x68, 0x61, 0x74, 0x20, 0x68, 0x69, 0x73, 0x74, 0x6f, + 0x72, 0x79, 0x20, 0x74, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x3c, + 0x2f, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x74, + 0x65, 0x78, 0x74, 0x61, 0x72, 0x65, 0x61, 0x20, 0x69, 0x64, 0x3d, 0x22, + 0x74, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x22, 0x20, 0x6e, 0x61, + 0x6d, 0x65, 0x3d, 0x22, 0x68, 0x69, 0x73, 0x74, 0x6f, 0x72, 0x79, 0x54, + 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x22, 0x20, 0x76, 0x61, 0x6c, + 0x75, 0x65, 0x3d, 0x22, 0x24, 0x7b, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, + 0x6e, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x2e, 0x68, 0x69, 0x73, 0x74, + 0x6f, 0x72, 0x79, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x7d, + 0x22, 0x20, 0x72, 0x6f, 0x77, 0x73, 0x3d, 0x31, 0x20, 0x6f, 0x6e, 0x69, + 0x6e, 0x70, 0x75, 0x74, 0x3d, 0x24, 0x7b, 0x75, 0x70, 0x64, 0x61, 0x74, + 0x65, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x7d, 0x2f, 0x3e, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x3c, 0x2f, 0x64, 0x69, 0x76, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x24, 0x7b, 0x47, 0x72, 0x61, + 0x6d, 0x6d, 0x61, 0x72, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x28, + 0x29, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x3c, 0x2f, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x65, 0x74, 0x3e, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x60, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x29, 0x3b, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x43, 0x6f, 0x6d, 0x70, 0x6c, + 0x65, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x46, + 0x6f, 0x72, 0x6d, 0x20, 0x3d, 0x20, 0x28, 0x29, 0x20, 0x3d, 0x3e, 0x20, + 0x28, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x68, 0x74, + 0x6d, 0x6c, 0x60, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x24, 0x7b, 0x50, 0x72, 0x6f, 0x6d, 0x70, 0x74, 0x43, 0x6f, + 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x53, 0x65, + 0x74, 0x28, 0x29, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x3c, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x65, 0x74, + 0x3e, 0x24, 0x7b, 0x47, 0x72, 0x61, 0x6d, 0x6d, 0x61, 0x72, 0x43, 0x6f, + 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x28, 0x29, 0x7d, 0x3c, 0x2f, 0x66, 0x69, + 0x65, 0x6c, 0x64, 0x73, 0x65, 0x74, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x60, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x29, 0x3b, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, + 0x74, 0x75, 0x72, 0x6e, 0x20, 0x68, 0x74, 0x6d, 0x6c, 0x60, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x66, 0x6f, 0x72, 0x6d, + 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x3c, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x65, 0x74, 0x20, 0x63, 0x6c, + 0x61, 0x73, 0x73, 0x3d, 0x22, 0x74, 0x77, 0x6f, 0x22, 0x3e, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, + 0x24, 0x7b, 0x55, 0x73, 0x65, 0x72, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, + 0x74, 0x65, 0x52, 0x65, 0x73, 0x65, 0x74, 0x42, 0x75, 0x74, 0x74, 0x6f, + 0x6e, 0x7d, 0x2f, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x64, 0x69, 0x76, 0x3e, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x3c, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x20, 0x63, 0x6c, 0x61, 0x73, + 0x73, 0x3d, 0x22, 0x73, 0x6c, 0x69, 0x6d, 0x22, 0x3e, 0x3c, 0x69, 0x6e, + 0x70, 0x75, 0x74, 0x20, 0x74, 0x79, 0x70, 0x65, 0x3d, 0x22, 0x72, 0x61, + 0x64, 0x69, 0x6f, 0x22, 0x20, 0x6e, 0x61, 0x6d, 0x65, 0x3d, 0x22, 0x74, + 0x79, 0x70, 0x65, 0x22, 0x20, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3d, 0x22, + 0x63, 0x68, 0x61, 0x74, 0x22, 0x20, 0x63, 0x68, 0x65, 0x63, 0x6b, 0x65, + 0x64, 0x3d, 0x24, 0x7b, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x2e, + 0x76, 0x61, 0x6c, 0x75, 0x65, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x20, 0x3d, + 0x3d, 0x3d, 0x20, 0x22, 0x63, 0x68, 0x61, 0x74, 0x22, 0x7d, 0x20, 0x6f, + 0x6e, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x3d, 0x24, 0x7b, 0x75, 0x70, 0x64, + 0x61, 0x74, 0x65, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x7d, 0x20, + 0x2f, 0x3e, 0x20, 0x43, 0x68, 0x61, 0x74, 0x3c, 0x2f, 0x6c, 0x61, 0x62, + 0x65, 0x6c, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x6c, 0x61, 0x62, 0x65, 0x6c, + 0x20, 0x63, 0x6c, 0x61, 0x73, 0x73, 0x3d, 0x22, 0x73, 0x6c, 0x69, 0x6d, + 0x22, 0x3e, 0x3c, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x20, 0x74, 0x79, 0x70, + 0x65, 0x3d, 0x22, 0x72, 0x61, 0x64, 0x69, 0x6f, 0x22, 0x20, 0x6e, 0x61, + 0x6d, 0x65, 0x3d, 0x22, 0x74, 0x79, 0x70, 0x65, 0x22, 0x20, 0x76, 0x61, + 0x6c, 0x75, 0x65, 0x3d, 0x22, 0x63, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, + 0x69, 0x6f, 0x6e, 0x22, 0x20, 0x63, 0x68, 0x65, 0x63, 0x6b, 0x65, 0x64, + 0x3d, 0x24, 0x7b, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x2e, 0x76, + 0x61, 0x6c, 0x75, 0x65, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x20, 0x3d, 0x3d, + 0x3d, 0x20, 0x22, 0x63, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x69, 0x6f, + 0x6e, 0x22, 0x7d, 0x20, 0x6f, 0x6e, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x3d, + 0x24, 0x7b, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x53, 0x65, 0x73, 0x73, + 0x69, 0x6f, 0x6e, 0x7d, 0x20, 0x2f, 0x3e, 0x20, 0x43, 0x6f, 0x6d, 0x70, + 0x6c, 0x65, 0x74, 0x69, 0x6f, 0x6e, 0x3c, 0x2f, 0x6c, 0x61, 0x62, 0x65, + 0x6c, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x3c, 0x2f, 0x64, 0x69, 0x76, 0x3e, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x2f, 0x66, 0x69, + 0x65, 0x6c, 0x64, 0x73, 0x65, 0x74, 0x3e, 0x0a, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x24, 0x7b, 0x73, 0x65, 0x73, + 0x73, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x2e, 0x74, + 0x79, 0x70, 0x65, 0x20, 0x3d, 0x3d, 0x3d, 0x20, 0x27, 0x63, 0x68, 0x61, + 0x74, 0x27, 0x20, 0x3f, 0x20, 0x43, 0x68, 0x61, 0x74, 0x43, 0x6f, 0x6e, + 0x66, 0x69, 0x67, 0x46, 0x6f, 0x72, 0x6d, 0x28, 0x29, 0x20, 0x3a, 0x20, + 0x43, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, + 0x6e, 0x66, 0x69, 0x67, 0x46, 0x6f, 0x72, 0x6d, 0x28, 0x29, 0x7d, 0x0a, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, + 0x66, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x65, 0x74, 0x20, 0x63, 0x6c, 0x61, + 0x73, 0x73, 0x3d, 0x22, 0x74, 0x77, 0x6f, 0x22, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x24, 0x7b, 0x49, 0x6e, 0x74, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x28, 0x7b, 0x20, 0x6c, - 0x61, 0x62, 0x65, 0x6c, 0x3a, 0x20, 0x22, 0x54, 0x6f, 0x70, 0x2d, 0x4b, - 0x20, 0x73, 0x61, 0x6d, 0x70, 0x6c, 0x69, 0x6e, 0x67, 0x22, 0x2c, 0x20, - 0x6d, 0x61, 0x78, 0x3a, 0x20, 0x31, 0x30, 0x30, 0x2c, 0x20, 0x6d, 0x69, - 0x6e, 0x3a, 0x20, 0x2d, 0x31, 0x2c, 0x20, 0x6e, 0x61, 0x6d, 0x65, 0x3a, - 0x20, 0x22, 0x74, 0x6f, 0x70, 0x5f, 0x6b, 0x22, 0x2c, 0x20, 0x76, 0x61, - 0x6c, 0x75, 0x65, 0x3a, 0x20, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x2e, - 0x76, 0x61, 0x6c, 0x75, 0x65, 0x2e, 0x74, 0x6f, 0x70, 0x5f, 0x6b, 0x20, + 0x61, 0x62, 0x65, 0x6c, 0x3a, 0x20, 0x22, 0x50, 0x72, 0x65, 0x64, 0x69, + 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x22, 0x2c, 0x20, 0x6d, 0x61, 0x78, + 0x3a, 0x20, 0x32, 0x30, 0x34, 0x38, 0x2c, 0x20, 0x6d, 0x69, 0x6e, 0x3a, + 0x20, 0x2d, 0x31, 0x2c, 0x20, 0x6e, 0x61, 0x6d, 0x65, 0x3a, 0x20, 0x22, + 0x6e, 0x5f, 0x70, 0x72, 0x65, 0x64, 0x69, 0x63, 0x74, 0x22, 0x2c, 0x20, + 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x20, 0x70, 0x61, 0x72, 0x61, 0x6d, + 0x73, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x2e, 0x6e, 0x5f, 0x70, 0x72, + 0x65, 0x64, 0x69, 0x63, 0x74, 0x20, 0x7d, 0x29, 0x7d, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x24, 0x7b, + 0x46, 0x6c, 0x6f, 0x61, 0x74, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x28, 0x7b, + 0x20, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x3a, 0x20, 0x22, 0x54, 0x65, 0x6d, + 0x70, 0x65, 0x72, 0x61, 0x74, 0x75, 0x72, 0x65, 0x22, 0x2c, 0x20, 0x6d, + 0x61, 0x78, 0x3a, 0x20, 0x32, 0x2e, 0x30, 0x2c, 0x20, 0x6d, 0x69, 0x6e, + 0x3a, 0x20, 0x30, 0x2e, 0x30, 0x2c, 0x20, 0x6e, 0x61, 0x6d, 0x65, 0x3a, + 0x20, 0x22, 0x74, 0x65, 0x6d, 0x70, 0x65, 0x72, 0x61, 0x74, 0x75, 0x72, + 0x65, 0x22, 0x2c, 0x20, 0x73, 0x74, 0x65, 0x70, 0x3a, 0x20, 0x30, 0x2e, + 0x30, 0x31, 0x2c, 0x20, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x20, 0x70, + 0x61, 0x72, 0x61, 0x6d, 0x73, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x2e, + 0x74, 0x65, 0x6d, 0x70, 0x65, 0x72, 0x61, 0x74, 0x75, 0x72, 0x65, 0x20, 0x7d, 0x29, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x24, 0x7b, 0x46, 0x6c, 0x6f, 0x61, 0x74, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x28, 0x7b, 0x20, 0x6c, 0x61, 0x62, 0x65, 0x6c, - 0x3a, 0x20, 0x22, 0x54, 0x6f, 0x70, 0x2d, 0x50, 0x20, 0x73, 0x61, 0x6d, - 0x70, 0x6c, 0x69, 0x6e, 0x67, 0x22, 0x2c, 0x20, 0x6d, 0x61, 0x78, 0x3a, - 0x20, 0x31, 0x2e, 0x30, 0x2c, 0x20, 0x6d, 0x69, 0x6e, 0x3a, 0x20, 0x30, - 0x2e, 0x30, 0x2c, 0x20, 0x6e, 0x61, 0x6d, 0x65, 0x3a, 0x20, 0x22, 0x74, - 0x6f, 0x70, 0x5f, 0x70, 0x22, 0x2c, 0x20, 0x73, 0x74, 0x65, 0x70, 0x3a, - 0x20, 0x30, 0x2e, 0x30, 0x31, 0x2c, 0x20, 0x76, 0x61, 0x6c, 0x75, 0x65, + 0x3a, 0x20, 0x22, 0x50, 0x65, 0x6e, 0x61, 0x6c, 0x69, 0x7a, 0x65, 0x20, + 0x72, 0x65, 0x70, 0x65, 0x61, 0x74, 0x20, 0x73, 0x65, 0x71, 0x75, 0x65, + 0x6e, 0x63, 0x65, 0x22, 0x2c, 0x20, 0x6d, 0x61, 0x78, 0x3a, 0x20, 0x32, + 0x2e, 0x30, 0x2c, 0x20, 0x6d, 0x69, 0x6e, 0x3a, 0x20, 0x30, 0x2e, 0x30, + 0x2c, 0x20, 0x6e, 0x61, 0x6d, 0x65, 0x3a, 0x20, 0x22, 0x72, 0x65, 0x70, + 0x65, 0x61, 0x74, 0x5f, 0x70, 0x65, 0x6e, 0x61, 0x6c, 0x74, 0x79, 0x22, + 0x2c, 0x20, 0x73, 0x74, 0x65, 0x70, 0x3a, 0x20, 0x30, 0x2e, 0x30, 0x31, + 0x2c, 0x20, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x20, 0x70, 0x61, 0x72, + 0x61, 0x6d, 0x73, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x2e, 0x72, 0x65, + 0x70, 0x65, 0x61, 0x74, 0x5f, 0x70, 0x65, 0x6e, 0x61, 0x6c, 0x74, 0x79, + 0x20, 0x7d, 0x29, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x24, 0x7b, 0x49, 0x6e, 0x74, 0x46, 0x69, + 0x65, 0x6c, 0x64, 0x28, 0x7b, 0x20, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x3a, + 0x20, 0x22, 0x43, 0x6f, 0x6e, 0x73, 0x69, 0x64, 0x65, 0x72, 0x20, 0x4e, + 0x20, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x73, 0x20, 0x66, 0x6f, 0x72, 0x20, + 0x70, 0x65, 0x6e, 0x61, 0x6c, 0x69, 0x7a, 0x65, 0x22, 0x2c, 0x20, 0x6d, + 0x61, 0x78, 0x3a, 0x20, 0x32, 0x30, 0x34, 0x38, 0x2c, 0x20, 0x6d, 0x69, + 0x6e, 0x3a, 0x20, 0x30, 0x2c, 0x20, 0x6e, 0x61, 0x6d, 0x65, 0x3a, 0x20, + 0x22, 0x72, 0x65, 0x70, 0x65, 0x61, 0x74, 0x5f, 0x6c, 0x61, 0x73, 0x74, + 0x5f, 0x6e, 0x22, 0x2c, 0x20, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x20, + 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, + 0x2e, 0x72, 0x65, 0x70, 0x65, 0x61, 0x74, 0x5f, 0x6c, 0x61, 0x73, 0x74, + 0x5f, 0x6e, 0x20, 0x7d, 0x29, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x24, 0x7b, 0x49, 0x6e, 0x74, + 0x46, 0x69, 0x65, 0x6c, 0x64, 0x28, 0x7b, 0x20, 0x6c, 0x61, 0x62, 0x65, + 0x6c, 0x3a, 0x20, 0x22, 0x54, 0x6f, 0x70, 0x2d, 0x4b, 0x20, 0x73, 0x61, + 0x6d, 0x70, 0x6c, 0x69, 0x6e, 0x67, 0x22, 0x2c, 0x20, 0x6d, 0x61, 0x78, + 0x3a, 0x20, 0x31, 0x30, 0x30, 0x2c, 0x20, 0x6d, 0x69, 0x6e, 0x3a, 0x20, + 0x2d, 0x31, 0x2c, 0x20, 0x6e, 0x61, 0x6d, 0x65, 0x3a, 0x20, 0x22, 0x74, + 0x6f, 0x70, 0x5f, 0x6b, 0x22, 0x2c, 0x20, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x20, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x2e, 0x76, 0x61, 0x6c, - 0x75, 0x65, 0x2e, 0x74, 0x6f, 0x70, 0x5f, 0x70, 0x20, 0x7d, 0x29, 0x7d, + 0x75, 0x65, 0x2e, 0x74, 0x6f, 0x70, 0x5f, 0x6b, 0x20, 0x7d, 0x29, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x24, 0x7b, 0x46, 0x6c, 0x6f, 0x61, 0x74, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x28, 0x7b, 0x20, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x3a, 0x20, 0x22, - 0x4d, 0x69, 0x6e, 0x2d, 0x50, 0x20, 0x73, 0x61, 0x6d, 0x70, 0x6c, 0x69, + 0x54, 0x6f, 0x70, 0x2d, 0x50, 0x20, 0x73, 0x61, 0x6d, 0x70, 0x6c, 0x69, 0x6e, 0x67, 0x22, 0x2c, 0x20, 0x6d, 0x61, 0x78, 0x3a, 0x20, 0x31, 0x2e, 0x30, 0x2c, 0x20, 0x6d, 0x69, 0x6e, 0x3a, 0x20, 0x30, 0x2e, 0x30, 0x2c, - 0x20, 0x6e, 0x61, 0x6d, 0x65, 0x3a, 0x20, 0x22, 0x6d, 0x69, 0x6e, 0x5f, + 0x20, 0x6e, 0x61, 0x6d, 0x65, 0x3a, 0x20, 0x22, 0x74, 0x6f, 0x70, 0x5f, 0x70, 0x22, 0x2c, 0x20, 0x73, 0x74, 0x65, 0x70, 0x3a, 0x20, 0x30, 0x2e, 0x30, 0x31, 0x2c, 0x20, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x20, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x2e, - 0x6d, 0x69, 0x6e, 0x5f, 0x70, 0x20, 0x7d, 0x29, 0x7d, 0x0a, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x2f, 0x66, 0x69, - 0x65, 0x6c, 0x64, 0x73, 0x65, 0x74, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x64, 0x65, 0x74, 0x61, 0x69, - 0x6c, 0x73, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x3c, 0x73, 0x75, 0x6d, 0x6d, 0x61, 0x72, 0x79, - 0x3e, 0x4d, 0x6f, 0x72, 0x65, 0x20, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, - 0x73, 0x3c, 0x2f, 0x73, 0x75, 0x6d, 0x6d, 0x61, 0x72, 0x79, 0x3e, 0x0a, + 0x74, 0x6f, 0x70, 0x5f, 0x70, 0x20, 0x7d, 0x29, 0x7d, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x24, 0x7b, + 0x46, 0x6c, 0x6f, 0x61, 0x74, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x28, 0x7b, + 0x20, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x3a, 0x20, 0x22, 0x4d, 0x69, 0x6e, + 0x2d, 0x50, 0x20, 0x73, 0x61, 0x6d, 0x70, 0x6c, 0x69, 0x6e, 0x67, 0x22, + 0x2c, 0x20, 0x6d, 0x61, 0x78, 0x3a, 0x20, 0x31, 0x2e, 0x30, 0x2c, 0x20, + 0x6d, 0x69, 0x6e, 0x3a, 0x20, 0x30, 0x2e, 0x30, 0x2c, 0x20, 0x6e, 0x61, + 0x6d, 0x65, 0x3a, 0x20, 0x22, 0x6d, 0x69, 0x6e, 0x5f, 0x70, 0x22, 0x2c, + 0x20, 0x73, 0x74, 0x65, 0x70, 0x3a, 0x20, 0x30, 0x2e, 0x30, 0x31, 0x2c, + 0x20, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x20, 0x70, 0x61, 0x72, 0x61, + 0x6d, 0x73, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x2e, 0x6d, 0x69, 0x6e, + 0x5f, 0x70, 0x20, 0x7d, 0x29, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x2f, 0x66, 0x69, 0x65, 0x6c, 0x64, + 0x73, 0x65, 0x74, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x3c, 0x64, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x73, 0x3e, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x3c, 0x73, 0x75, 0x6d, 0x6d, 0x61, 0x72, 0x79, 0x3e, 0x4d, 0x6f, + 0x72, 0x65, 0x20, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x3c, 0x2f, + 0x73, 0x75, 0x6d, 0x6d, 0x61, 0x72, 0x79, 0x3e, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x66, 0x69, + 0x65, 0x6c, 0x64, 0x73, 0x65, 0x74, 0x20, 0x63, 0x6c, 0x61, 0x73, 0x73, + 0x3d, 0x22, 0x74, 0x77, 0x6f, 0x22, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x24, 0x7b, + 0x46, 0x6c, 0x6f, 0x61, 0x74, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x28, 0x7b, + 0x20, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x3a, 0x20, 0x22, 0x54, 0x46, 0x53, + 0x2d, 0x5a, 0x22, 0x2c, 0x20, 0x6d, 0x61, 0x78, 0x3a, 0x20, 0x31, 0x2e, + 0x30, 0x2c, 0x20, 0x6d, 0x69, 0x6e, 0x3a, 0x20, 0x30, 0x2e, 0x30, 0x2c, + 0x20, 0x6e, 0x61, 0x6d, 0x65, 0x3a, 0x20, 0x22, 0x74, 0x66, 0x73, 0x5f, + 0x7a, 0x22, 0x2c, 0x20, 0x73, 0x74, 0x65, 0x70, 0x3a, 0x20, 0x30, 0x2e, + 0x30, 0x31, 0x2c, 0x20, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x20, 0x70, + 0x61, 0x72, 0x61, 0x6d, 0x73, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x2e, + 0x74, 0x66, 0x73, 0x5f, 0x7a, 0x20, 0x7d, 0x29, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x3c, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x65, 0x74, 0x20, 0x63, 0x6c, - 0x61, 0x73, 0x73, 0x3d, 0x22, 0x74, 0x77, 0x6f, 0x22, 0x3e, 0x0a, 0x20, + 0x24, 0x7b, 0x46, 0x6c, 0x6f, 0x61, 0x74, 0x46, 0x69, 0x65, 0x6c, 0x64, + 0x28, 0x7b, 0x20, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x3a, 0x20, 0x22, 0x54, + 0x79, 0x70, 0x69, 0x63, 0x61, 0x6c, 0x20, 0x50, 0x22, 0x2c, 0x20, 0x6d, + 0x61, 0x78, 0x3a, 0x20, 0x31, 0x2e, 0x30, 0x2c, 0x20, 0x6d, 0x69, 0x6e, + 0x3a, 0x20, 0x30, 0x2e, 0x30, 0x2c, 0x20, 0x6e, 0x61, 0x6d, 0x65, 0x3a, + 0x20, 0x22, 0x74, 0x79, 0x70, 0x69, 0x63, 0x61, 0x6c, 0x5f, 0x70, 0x22, + 0x2c, 0x20, 0x73, 0x74, 0x65, 0x70, 0x3a, 0x20, 0x30, 0x2e, 0x30, 0x31, + 0x2c, 0x20, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x20, 0x70, 0x61, 0x72, + 0x61, 0x6d, 0x73, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x2e, 0x74, 0x79, + 0x70, 0x69, 0x63, 0x61, 0x6c, 0x5f, 0x70, 0x20, 0x7d, 0x29, 0x7d, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x24, 0x7b, 0x46, 0x6c, 0x6f, 0x61, 0x74, 0x46, 0x69, 0x65, + 0x6c, 0x64, 0x28, 0x7b, 0x20, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x3a, 0x20, + 0x22, 0x50, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x20, 0x70, 0x65, + 0x6e, 0x61, 0x6c, 0x74, 0x79, 0x22, 0x2c, 0x20, 0x6d, 0x61, 0x78, 0x3a, + 0x20, 0x31, 0x2e, 0x30, 0x2c, 0x20, 0x6d, 0x69, 0x6e, 0x3a, 0x20, 0x30, + 0x2e, 0x30, 0x2c, 0x20, 0x6e, 0x61, 0x6d, 0x65, 0x3a, 0x20, 0x22, 0x70, + 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x5f, 0x70, 0x65, 0x6e, 0x61, + 0x6c, 0x74, 0x79, 0x22, 0x2c, 0x20, 0x73, 0x74, 0x65, 0x70, 0x3a, 0x20, + 0x30, 0x2e, 0x30, 0x31, 0x2c, 0x20, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, + 0x20, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x2e, 0x76, 0x61, 0x6c, 0x75, + 0x65, 0x2e, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x5f, 0x70, + 0x65, 0x6e, 0x61, 0x6c, 0x74, 0x79, 0x20, 0x7d, 0x29, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x24, 0x7b, 0x46, 0x6c, 0x6f, 0x61, 0x74, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x28, 0x7b, 0x20, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x3a, 0x20, 0x22, - 0x54, 0x46, 0x53, 0x2d, 0x5a, 0x22, 0x2c, 0x20, 0x6d, 0x61, 0x78, 0x3a, + 0x46, 0x72, 0x65, 0x71, 0x75, 0x65, 0x6e, 0x63, 0x79, 0x20, 0x70, 0x65, + 0x6e, 0x61, 0x6c, 0x74, 0x79, 0x22, 0x2c, 0x20, 0x6d, 0x61, 0x78, 0x3a, 0x20, 0x31, 0x2e, 0x30, 0x2c, 0x20, 0x6d, 0x69, 0x6e, 0x3a, 0x20, 0x30, - 0x2e, 0x30, 0x2c, 0x20, 0x6e, 0x61, 0x6d, 0x65, 0x3a, 0x20, 0x22, 0x74, - 0x66, 0x73, 0x5f, 0x7a, 0x22, 0x2c, 0x20, 0x73, 0x74, 0x65, 0x70, 0x3a, + 0x2e, 0x30, 0x2c, 0x20, 0x6e, 0x61, 0x6d, 0x65, 0x3a, 0x20, 0x22, 0x66, + 0x72, 0x65, 0x71, 0x75, 0x65, 0x6e, 0x63, 0x79, 0x5f, 0x70, 0x65, 0x6e, + 0x61, 0x6c, 0x74, 0x79, 0x22, 0x2c, 0x20, 0x73, 0x74, 0x65, 0x70, 0x3a, 0x20, 0x30, 0x2e, 0x30, 0x31, 0x2c, 0x20, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x20, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x2e, 0x76, 0x61, 0x6c, - 0x75, 0x65, 0x2e, 0x74, 0x66, 0x73, 0x5f, 0x7a, 0x20, 0x7d, 0x29, 0x7d, + 0x75, 0x65, 0x2e, 0x66, 0x72, 0x65, 0x71, 0x75, 0x65, 0x6e, 0x63, 0x79, + 0x5f, 0x70, 0x65, 0x6e, 0x61, 0x6c, 0x74, 0x79, 0x20, 0x7d, 0x29, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x24, 0x7b, 0x46, 0x6c, 0x6f, 0x61, 0x74, 0x46, 0x69, - 0x65, 0x6c, 0x64, 0x28, 0x7b, 0x20, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x3a, - 0x20, 0x22, 0x54, 0x79, 0x70, 0x69, 0x63, 0x61, 0x6c, 0x20, 0x50, 0x22, - 0x2c, 0x20, 0x6d, 0x61, 0x78, 0x3a, 0x20, 0x31, 0x2e, 0x30, 0x2c, 0x20, - 0x6d, 0x69, 0x6e, 0x3a, 0x20, 0x30, 0x2e, 0x30, 0x2c, 0x20, 0x6e, 0x61, - 0x6d, 0x65, 0x3a, 0x20, 0x22, 0x74, 0x79, 0x70, 0x69, 0x63, 0x61, 0x6c, - 0x5f, 0x70, 0x22, 0x2c, 0x20, 0x73, 0x74, 0x65, 0x70, 0x3a, 0x20, 0x30, - 0x2e, 0x30, 0x31, 0x2c, 0x20, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x20, + 0x20, 0x3c, 0x2f, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x65, 0x74, 0x3e, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x3c, 0x68, 0x72, 0x20, 0x2f, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x66, 0x69, 0x65, + 0x6c, 0x64, 0x73, 0x65, 0x74, 0x20, 0x63, 0x6c, 0x61, 0x73, 0x73, 0x3d, + 0x22, 0x74, 0x68, 0x72, 0x65, 0x65, 0x22, 0x3e, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, + 0x64, 0x69, 0x76, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x6c, 0x61, + 0x62, 0x65, 0x6c, 0x3e, 0x3c, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x20, 0x74, + 0x79, 0x70, 0x65, 0x3d, 0x22, 0x72, 0x61, 0x64, 0x69, 0x6f, 0x22, 0x20, + 0x6e, 0x61, 0x6d, 0x65, 0x3d, 0x22, 0x6d, 0x69, 0x72, 0x6f, 0x73, 0x74, + 0x61, 0x74, 0x22, 0x20, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3d, 0x22, 0x30, + 0x22, 0x20, 0x63, 0x68, 0x65, 0x63, 0x6b, 0x65, 0x64, 0x3d, 0x24, 0x7b, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, - 0x2e, 0x74, 0x79, 0x70, 0x69, 0x63, 0x61, 0x6c, 0x5f, 0x70, 0x20, 0x7d, - 0x29, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x24, 0x7b, 0x46, 0x6c, 0x6f, 0x61, 0x74, - 0x46, 0x69, 0x65, 0x6c, 0x64, 0x28, 0x7b, 0x20, 0x6c, 0x61, 0x62, 0x65, - 0x6c, 0x3a, 0x20, 0x22, 0x50, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, - 0x20, 0x70, 0x65, 0x6e, 0x61, 0x6c, 0x74, 0x79, 0x22, 0x2c, 0x20, 0x6d, - 0x61, 0x78, 0x3a, 0x20, 0x31, 0x2e, 0x30, 0x2c, 0x20, 0x6d, 0x69, 0x6e, - 0x3a, 0x20, 0x30, 0x2e, 0x30, 0x2c, 0x20, 0x6e, 0x61, 0x6d, 0x65, 0x3a, - 0x20, 0x22, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x5f, 0x70, - 0x65, 0x6e, 0x61, 0x6c, 0x74, 0x79, 0x22, 0x2c, 0x20, 0x73, 0x74, 0x65, - 0x70, 0x3a, 0x20, 0x30, 0x2e, 0x30, 0x31, 0x2c, 0x20, 0x76, 0x61, 0x6c, - 0x75, 0x65, 0x3a, 0x20, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x2e, 0x76, - 0x61, 0x6c, 0x75, 0x65, 0x2e, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, - 0x65, 0x5f, 0x70, 0x65, 0x6e, 0x61, 0x6c, 0x74, 0x79, 0x20, 0x7d, 0x29, - 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x24, 0x7b, 0x46, 0x6c, 0x6f, 0x61, 0x74, 0x46, - 0x69, 0x65, 0x6c, 0x64, 0x28, 0x7b, 0x20, 0x6c, 0x61, 0x62, 0x65, 0x6c, - 0x3a, 0x20, 0x22, 0x46, 0x72, 0x65, 0x71, 0x75, 0x65, 0x6e, 0x63, 0x79, - 0x20, 0x70, 0x65, 0x6e, 0x61, 0x6c, 0x74, 0x79, 0x22, 0x2c, 0x20, 0x6d, - 0x61, 0x78, 0x3a, 0x20, 0x31, 0x2e, 0x30, 0x2c, 0x20, 0x6d, 0x69, 0x6e, - 0x3a, 0x20, 0x30, 0x2e, 0x30, 0x2c, 0x20, 0x6e, 0x61, 0x6d, 0x65, 0x3a, - 0x20, 0x22, 0x66, 0x72, 0x65, 0x71, 0x75, 0x65, 0x6e, 0x63, 0x79, 0x5f, - 0x70, 0x65, 0x6e, 0x61, 0x6c, 0x74, 0x79, 0x22, 0x2c, 0x20, 0x73, 0x74, - 0x65, 0x70, 0x3a, 0x20, 0x30, 0x2e, 0x30, 0x31, 0x2c, 0x20, 0x76, 0x61, - 0x6c, 0x75, 0x65, 0x3a, 0x20, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x2e, - 0x76, 0x61, 0x6c, 0x75, 0x65, 0x2e, 0x66, 0x72, 0x65, 0x71, 0x75, 0x65, - 0x6e, 0x63, 0x79, 0x5f, 0x70, 0x65, 0x6e, 0x61, 0x6c, 0x74, 0x79, 0x20, + 0x2e, 0x6d, 0x69, 0x72, 0x6f, 0x73, 0x74, 0x61, 0x74, 0x20, 0x3d, 0x3d, + 0x20, 0x30, 0x7d, 0x20, 0x6f, 0x6e, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x3d, + 0x24, 0x7b, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x50, 0x61, 0x72, 0x61, + 0x6d, 0x73, 0x49, 0x6e, 0x74, 0x7d, 0x20, 0x2f, 0x3e, 0x20, 0x6e, 0x6f, + 0x20, 0x4d, 0x69, 0x72, 0x6f, 0x73, 0x74, 0x61, 0x74, 0x3c, 0x2f, 0x6c, + 0x61, 0x62, 0x65, 0x6c, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x6c, + 0x61, 0x62, 0x65, 0x6c, 0x3e, 0x3c, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x20, + 0x74, 0x79, 0x70, 0x65, 0x3d, 0x22, 0x72, 0x61, 0x64, 0x69, 0x6f, 0x22, + 0x20, 0x6e, 0x61, 0x6d, 0x65, 0x3d, 0x22, 0x6d, 0x69, 0x72, 0x6f, 0x73, + 0x74, 0x61, 0x74, 0x22, 0x20, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3d, 0x22, + 0x31, 0x22, 0x20, 0x63, 0x68, 0x65, 0x63, 0x6b, 0x65, 0x64, 0x3d, 0x24, + 0x7b, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x2e, 0x76, 0x61, 0x6c, 0x75, + 0x65, 0x2e, 0x6d, 0x69, 0x72, 0x6f, 0x73, 0x74, 0x61, 0x74, 0x20, 0x3d, + 0x3d, 0x20, 0x31, 0x7d, 0x20, 0x6f, 0x6e, 0x69, 0x6e, 0x70, 0x75, 0x74, + 0x3d, 0x24, 0x7b, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x50, 0x61, 0x72, + 0x61, 0x6d, 0x73, 0x49, 0x6e, 0x74, 0x7d, 0x20, 0x2f, 0x3e, 0x20, 0x4d, + 0x69, 0x72, 0x6f, 0x73, 0x74, 0x61, 0x74, 0x20, 0x76, 0x31, 0x3c, 0x2f, + 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, + 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x3e, 0x3c, 0x69, 0x6e, 0x70, 0x75, 0x74, + 0x20, 0x74, 0x79, 0x70, 0x65, 0x3d, 0x22, 0x72, 0x61, 0x64, 0x69, 0x6f, + 0x22, 0x20, 0x6e, 0x61, 0x6d, 0x65, 0x3d, 0x22, 0x6d, 0x69, 0x72, 0x6f, + 0x73, 0x74, 0x61, 0x74, 0x22, 0x20, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3d, + 0x22, 0x32, 0x22, 0x20, 0x63, 0x68, 0x65, 0x63, 0x6b, 0x65, 0x64, 0x3d, + 0x24, 0x7b, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x2e, 0x76, 0x61, 0x6c, + 0x75, 0x65, 0x2e, 0x6d, 0x69, 0x72, 0x6f, 0x73, 0x74, 0x61, 0x74, 0x20, + 0x3d, 0x3d, 0x20, 0x32, 0x7d, 0x20, 0x6f, 0x6e, 0x69, 0x6e, 0x70, 0x75, + 0x74, 0x3d, 0x24, 0x7b, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x50, 0x61, + 0x72, 0x61, 0x6d, 0x73, 0x49, 0x6e, 0x74, 0x7d, 0x20, 0x2f, 0x3e, 0x20, + 0x4d, 0x69, 0x72, 0x6f, 0x73, 0x74, 0x61, 0x74, 0x20, 0x76, 0x32, 0x3c, + 0x2f, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x2f, + 0x64, 0x69, 0x76, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x24, 0x7b, 0x46, 0x6c, 0x6f, + 0x61, 0x74, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x28, 0x7b, 0x20, 0x6c, 0x61, + 0x62, 0x65, 0x6c, 0x3a, 0x20, 0x22, 0x4d, 0x69, 0x72, 0x6f, 0x73, 0x74, + 0x61, 0x74, 0x20, 0x74, 0x61, 0x75, 0x22, 0x2c, 0x20, 0x6d, 0x61, 0x78, + 0x3a, 0x20, 0x31, 0x30, 0x2e, 0x30, 0x2c, 0x20, 0x6d, 0x69, 0x6e, 0x3a, + 0x20, 0x30, 0x2e, 0x30, 0x2c, 0x20, 0x6e, 0x61, 0x6d, 0x65, 0x3a, 0x20, + 0x22, 0x6d, 0x69, 0x72, 0x6f, 0x73, 0x74, 0x61, 0x74, 0x5f, 0x74, 0x61, + 0x75, 0x22, 0x2c, 0x20, 0x73, 0x74, 0x65, 0x70, 0x3a, 0x20, 0x30, 0x2e, + 0x30, 0x31, 0x2c, 0x20, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x20, 0x70, + 0x61, 0x72, 0x61, 0x6d, 0x73, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x2e, + 0x6d, 0x69, 0x72, 0x6f, 0x73, 0x74, 0x61, 0x74, 0x5f, 0x74, 0x61, 0x75, + 0x20, 0x7d, 0x29, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x24, 0x7b, 0x46, 0x6c, 0x6f, + 0x61, 0x74, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x28, 0x7b, 0x20, 0x6c, 0x61, + 0x62, 0x65, 0x6c, 0x3a, 0x20, 0x22, 0x4d, 0x69, 0x72, 0x6f, 0x73, 0x74, + 0x61, 0x74, 0x20, 0x65, 0x74, 0x61, 0x22, 0x2c, 0x20, 0x6d, 0x61, 0x78, + 0x3a, 0x20, 0x31, 0x2e, 0x30, 0x2c, 0x20, 0x6d, 0x69, 0x6e, 0x3a, 0x20, + 0x30, 0x2e, 0x30, 0x2c, 0x20, 0x6e, 0x61, 0x6d, 0x65, 0x3a, 0x20, 0x22, + 0x6d, 0x69, 0x72, 0x6f, 0x73, 0x74, 0x61, 0x74, 0x5f, 0x65, 0x74, 0x61, + 0x22, 0x2c, 0x20, 0x73, 0x74, 0x65, 0x70, 0x3a, 0x20, 0x30, 0x2e, 0x30, + 0x31, 0x2c, 0x20, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x20, 0x70, 0x61, + 0x72, 0x61, 0x6d, 0x73, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x2e, 0x6d, + 0x69, 0x72, 0x6f, 0x73, 0x74, 0x61, 0x74, 0x5f, 0x65, 0x74, 0x61, 0x20, 0x7d, 0x29, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x2f, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x65, 0x74, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x3c, 0x68, 0x72, 0x20, 0x2f, 0x3e, 0x0a, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, - 0x66, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x65, 0x74, 0x20, 0x63, 0x6c, 0x61, - 0x73, 0x73, 0x3d, 0x22, 0x74, 0x68, 0x72, 0x65, 0x65, 0x22, 0x3e, 0x0a, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x3c, 0x64, 0x69, 0x76, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x3c, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x3e, 0x3c, 0x69, 0x6e, 0x70, 0x75, - 0x74, 0x20, 0x74, 0x79, 0x70, 0x65, 0x3d, 0x22, 0x72, 0x61, 0x64, 0x69, - 0x6f, 0x22, 0x20, 0x6e, 0x61, 0x6d, 0x65, 0x3d, 0x22, 0x6d, 0x69, 0x72, - 0x6f, 0x73, 0x74, 0x61, 0x74, 0x22, 0x20, 0x76, 0x61, 0x6c, 0x75, 0x65, - 0x3d, 0x22, 0x30, 0x22, 0x20, 0x63, 0x68, 0x65, 0x63, 0x6b, 0x65, 0x64, - 0x3d, 0x24, 0x7b, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x2e, 0x76, 0x61, - 0x6c, 0x75, 0x65, 0x2e, 0x6d, 0x69, 0x72, 0x6f, 0x73, 0x74, 0x61, 0x74, - 0x20, 0x3d, 0x3d, 0x20, 0x30, 0x7d, 0x20, 0x6f, 0x6e, 0x69, 0x6e, 0x70, - 0x75, 0x74, 0x3d, 0x24, 0x7b, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x50, - 0x61, 0x72, 0x61, 0x6d, 0x73, 0x49, 0x6e, 0x74, 0x7d, 0x20, 0x2f, 0x3e, - 0x20, 0x6e, 0x6f, 0x20, 0x4d, 0x69, 0x72, 0x6f, 0x73, 0x74, 0x61, 0x74, - 0x3c, 0x2f, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x3e, 0x0a, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x3c, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x3e, 0x3c, 0x69, 0x6e, 0x70, - 0x75, 0x74, 0x20, 0x74, 0x79, 0x70, 0x65, 0x3d, 0x22, 0x72, 0x61, 0x64, - 0x69, 0x6f, 0x22, 0x20, 0x6e, 0x61, 0x6d, 0x65, 0x3d, 0x22, 0x6d, 0x69, - 0x72, 0x6f, 0x73, 0x74, 0x61, 0x74, 0x22, 0x20, 0x76, 0x61, 0x6c, 0x75, - 0x65, 0x3d, 0x22, 0x31, 0x22, 0x20, 0x63, 0x68, 0x65, 0x63, 0x6b, 0x65, - 0x64, 0x3d, 0x24, 0x7b, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x2e, 0x76, - 0x61, 0x6c, 0x75, 0x65, 0x2e, 0x6d, 0x69, 0x72, 0x6f, 0x73, 0x74, 0x61, - 0x74, 0x20, 0x3d, 0x3d, 0x20, 0x31, 0x7d, 0x20, 0x6f, 0x6e, 0x69, 0x6e, - 0x70, 0x75, 0x74, 0x3d, 0x24, 0x7b, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, - 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x49, 0x6e, 0x74, 0x7d, 0x20, 0x2f, - 0x3e, 0x20, 0x4d, 0x69, 0x72, 0x6f, 0x73, 0x74, 0x61, 0x74, 0x20, 0x76, - 0x31, 0x3c, 0x2f, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x3e, 0x0a, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x3c, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x3e, 0x3c, 0x69, 0x6e, - 0x70, 0x75, 0x74, 0x20, 0x74, 0x79, 0x70, 0x65, 0x3d, 0x22, 0x72, 0x61, - 0x64, 0x69, 0x6f, 0x22, 0x20, 0x6e, 0x61, 0x6d, 0x65, 0x3d, 0x22, 0x6d, - 0x69, 0x72, 0x6f, 0x73, 0x74, 0x61, 0x74, 0x22, 0x20, 0x76, 0x61, 0x6c, - 0x75, 0x65, 0x3d, 0x22, 0x32, 0x22, 0x20, 0x63, 0x68, 0x65, 0x63, 0x6b, - 0x65, 0x64, 0x3d, 0x24, 0x7b, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x2e, - 0x76, 0x61, 0x6c, 0x75, 0x65, 0x2e, 0x6d, 0x69, 0x72, 0x6f, 0x73, 0x74, - 0x61, 0x74, 0x20, 0x3d, 0x3d, 0x20, 0x32, 0x7d, 0x20, 0x6f, 0x6e, 0x69, - 0x6e, 0x70, 0x75, 0x74, 0x3d, 0x24, 0x7b, 0x75, 0x70, 0x64, 0x61, 0x74, - 0x65, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x49, 0x6e, 0x74, 0x7d, 0x20, - 0x2f, 0x3e, 0x20, 0x4d, 0x69, 0x72, 0x6f, 0x73, 0x74, 0x61, 0x74, 0x20, - 0x76, 0x32, 0x3c, 0x2f, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x3e, 0x0a, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x3c, 0x2f, 0x64, 0x69, 0x76, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x24, 0x7b, - 0x46, 0x6c, 0x6f, 0x61, 0x74, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x28, 0x7b, - 0x20, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x3a, 0x20, 0x22, 0x4d, 0x69, 0x72, - 0x6f, 0x73, 0x74, 0x61, 0x74, 0x20, 0x74, 0x61, 0x75, 0x22, 0x2c, 0x20, - 0x6d, 0x61, 0x78, 0x3a, 0x20, 0x31, 0x30, 0x2e, 0x30, 0x2c, 0x20, 0x6d, - 0x69, 0x6e, 0x3a, 0x20, 0x30, 0x2e, 0x30, 0x2c, 0x20, 0x6e, 0x61, 0x6d, - 0x65, 0x3a, 0x20, 0x22, 0x6d, 0x69, 0x72, 0x6f, 0x73, 0x74, 0x61, 0x74, - 0x5f, 0x74, 0x61, 0x75, 0x22, 0x2c, 0x20, 0x73, 0x74, 0x65, 0x70, 0x3a, - 0x20, 0x30, 0x2e, 0x30, 0x31, 0x2c, 0x20, 0x76, 0x61, 0x6c, 0x75, 0x65, - 0x3a, 0x20, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x2e, 0x76, 0x61, 0x6c, - 0x75, 0x65, 0x2e, 0x6d, 0x69, 0x72, 0x6f, 0x73, 0x74, 0x61, 0x74, 0x5f, - 0x74, 0x61, 0x75, 0x20, 0x7d, 0x29, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x24, 0x7b, - 0x46, 0x6c, 0x6f, 0x61, 0x74, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x28, 0x7b, - 0x20, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x3a, 0x20, 0x22, 0x4d, 0x69, 0x72, - 0x6f, 0x73, 0x74, 0x61, 0x74, 0x20, 0x65, 0x74, 0x61, 0x22, 0x2c, 0x20, - 0x6d, 0x61, 0x78, 0x3a, 0x20, 0x31, 0x2e, 0x30, 0x2c, 0x20, 0x6d, 0x69, - 0x6e, 0x3a, 0x20, 0x30, 0x2e, 0x30, 0x2c, 0x20, 0x6e, 0x61, 0x6d, 0x65, - 0x3a, 0x20, 0x22, 0x6d, 0x69, 0x72, 0x6f, 0x73, 0x74, 0x61, 0x74, 0x5f, - 0x65, 0x74, 0x61, 0x22, 0x2c, 0x20, 0x73, 0x74, 0x65, 0x70, 0x3a, 0x20, - 0x30, 0x2e, 0x30, 0x31, 0x2c, 0x20, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, - 0x20, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x2e, 0x76, 0x61, 0x6c, 0x75, - 0x65, 0x2e, 0x6d, 0x69, 0x72, 0x6f, 0x73, 0x74, 0x61, 0x74, 0x5f, 0x65, - 0x74, 0x61, 0x20, 0x7d, 0x29, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x2f, 0x66, 0x69, 0x65, - 0x6c, 0x64, 0x73, 0x65, 0x74, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x66, 0x69, 0x65, 0x6c, - 0x64, 0x73, 0x65, 0x74, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x24, 0x7b, 0x49, 0x6e, - 0x74, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x28, 0x7b, 0x20, 0x6c, 0x61, 0x62, - 0x65, 0x6c, 0x3a, 0x20, 0x22, 0x53, 0x68, 0x6f, 0x77, 0x20, 0x50, 0x72, - 0x6f, 0x62, 0x61, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x69, 0x65, 0x73, 0x22, - 0x2c, 0x20, 0x6d, 0x61, 0x78, 0x3a, 0x20, 0x31, 0x30, 0x2c, 0x20, 0x6d, - 0x69, 0x6e, 0x3a, 0x20, 0x30, 0x2c, 0x20, 0x6e, 0x61, 0x6d, 0x65, 0x3a, - 0x20, 0x22, 0x6e, 0x5f, 0x70, 0x72, 0x6f, 0x62, 0x73, 0x22, 0x2c, 0x20, - 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x20, 0x70, 0x61, 0x72, 0x61, 0x6d, - 0x73, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x2e, 0x6e, 0x5f, 0x70, 0x72, - 0x6f, 0x62, 0x73, 0x20, 0x7d, 0x29, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x2f, 0x66, 0x69, - 0x65, 0x6c, 0x64, 0x73, 0x65, 0x74, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x66, 0x69, 0x65, - 0x6c, 0x64, 0x73, 0x65, 0x74, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x6c, 0x61, - 0x62, 0x65, 0x6c, 0x20, 0x66, 0x6f, 0x72, 0x3d, 0x22, 0x61, 0x70, 0x69, - 0x5f, 0x6b, 0x65, 0x79, 0x22, 0x3e, 0x41, 0x50, 0x49, 0x20, 0x4b, 0x65, - 0x79, 0x3c, 0x2f, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x3e, 0x0a, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x3c, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x20, 0x74, 0x79, 0x70, 0x65, 0x3d, - 0x22, 0x74, 0x65, 0x78, 0x74, 0x22, 0x20, 0x6e, 0x61, 0x6d, 0x65, 0x3d, - 0x22, 0x61, 0x70, 0x69, 0x5f, 0x6b, 0x65, 0x79, 0x22, 0x20, 0x76, 0x61, - 0x6c, 0x75, 0x65, 0x3d, 0x22, 0x24, 0x7b, 0x70, 0x61, 0x72, 0x61, 0x6d, - 0x73, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x2e, 0x61, 0x70, 0x69, 0x5f, - 0x6b, 0x65, 0x79, 0x7d, 0x22, 0x20, 0x70, 0x6c, 0x61, 0x63, 0x65, 0x68, - 0x6f, 0x6c, 0x64, 0x65, 0x72, 0x3d, 0x22, 0x45, 0x6e, 0x74, 0x65, 0x72, - 0x20, 0x41, 0x50, 0x49, 0x20, 0x6b, 0x65, 0x79, 0x22, 0x20, 0x6f, 0x6e, - 0x69, 0x6e, 0x70, 0x75, 0x74, 0x3d, 0x24, 0x7b, 0x75, 0x70, 0x64, 0x61, - 0x74, 0x65, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x7d, 0x20, 0x2f, 0x3e, + 0x20, 0x20, 0x20, 0x20, 0x3c, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x65, + 0x74, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x24, 0x7b, 0x49, 0x6e, 0x74, 0x46, 0x69, + 0x65, 0x6c, 0x64, 0x28, 0x7b, 0x20, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x3a, + 0x20, 0x22, 0x53, 0x68, 0x6f, 0x77, 0x20, 0x50, 0x72, 0x6f, 0x62, 0x61, + 0x62, 0x69, 0x6c, 0x69, 0x74, 0x69, 0x65, 0x73, 0x22, 0x2c, 0x20, 0x6d, + 0x61, 0x78, 0x3a, 0x20, 0x31, 0x30, 0x2c, 0x20, 0x6d, 0x69, 0x6e, 0x3a, + 0x20, 0x30, 0x2c, 0x20, 0x6e, 0x61, 0x6d, 0x65, 0x3a, 0x20, 0x22, 0x6e, + 0x5f, 0x70, 0x72, 0x6f, 0x62, 0x73, 0x22, 0x2c, 0x20, 0x76, 0x61, 0x6c, + 0x75, 0x65, 0x3a, 0x20, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x2e, 0x76, + 0x61, 0x6c, 0x75, 0x65, 0x2e, 0x6e, 0x5f, 0x70, 0x72, 0x6f, 0x62, 0x73, + 0x20, 0x7d, 0x29, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x2f, 0x66, 0x69, 0x65, 0x6c, 0x64, + 0x73, 0x65, 0x74, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x73, + 0x65, 0x74, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x24, 0x7b, 0x49, 0x6e, 0x74, 0x46, + 0x69, 0x65, 0x6c, 0x64, 0x28, 0x7b, 0x20, 0x6c, 0x61, 0x62, 0x65, 0x6c, + 0x3a, 0x20, 0x22, 0x4d, 0x69, 0x6e, 0x20, 0x50, 0x72, 0x6f, 0x62, 0x61, + 0x62, 0x69, 0x6c, 0x69, 0x74, 0x69, 0x65, 0x73, 0x20, 0x66, 0x72, 0x6f, + 0x6d, 0x20, 0x65, 0x61, 0x63, 0x68, 0x20, 0x53, 0x61, 0x6d, 0x70, 0x6c, + 0x65, 0x72, 0x22, 0x2c, 0x20, 0x6d, 0x61, 0x78, 0x3a, 0x20, 0x31, 0x30, + 0x2c, 0x20, 0x6d, 0x69, 0x6e, 0x3a, 0x20, 0x30, 0x2c, 0x20, 0x6e, 0x61, + 0x6d, 0x65, 0x3a, 0x20, 0x22, 0x6d, 0x69, 0x6e, 0x5f, 0x6b, 0x65, 0x65, + 0x70, 0x22, 0x2c, 0x20, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x20, 0x70, + 0x61, 0x72, 0x61, 0x6d, 0x73, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x2e, + 0x6d, 0x69, 0x6e, 0x5f, 0x6b, 0x65, 0x65, 0x70, 0x20, 0x7d, 0x29, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x2f, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x65, 0x74, 0x3e, - 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, - 0x2f, 0x64, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x73, 0x3e, 0x0a, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x2f, 0x66, 0x6f, 0x72, 0x6d, - 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x60, 0x0a, 0x20, 0x20, - 0x20, 0x20, 0x7d, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, - 0x73, 0x74, 0x20, 0x70, 0x72, 0x6f, 0x62, 0x43, 0x6f, 0x6c, 0x6f, 0x72, - 0x20, 0x3d, 0x20, 0x28, 0x70, 0x29, 0x20, 0x3d, 0x3e, 0x20, 0x7b, 0x0a, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, - 0x72, 0x20, 0x3d, 0x20, 0x4d, 0x61, 0x74, 0x68, 0x2e, 0x66, 0x6c, 0x6f, - 0x6f, 0x72, 0x28, 0x31, 0x39, 0x32, 0x20, 0x2a, 0x20, 0x28, 0x31, 0x20, - 0x2d, 0x20, 0x70, 0x29, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x67, 0x20, 0x3d, 0x20, 0x4d, - 0x61, 0x74, 0x68, 0x2e, 0x66, 0x6c, 0x6f, 0x6f, 0x72, 0x28, 0x31, 0x39, - 0x32, 0x20, 0x2a, 0x20, 0x70, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x60, 0x72, 0x67, - 0x62, 0x61, 0x28, 0x24, 0x7b, 0x72, 0x7d, 0x2c, 0x24, 0x7b, 0x67, 0x7d, - 0x2c, 0x30, 0x2c, 0x30, 0x2e, 0x33, 0x29, 0x60, 0x3b, 0x0a, 0x20, 0x20, - 0x20, 0x20, 0x7d, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, - 0x73, 0x74, 0x20, 0x50, 0x72, 0x6f, 0x62, 0x61, 0x62, 0x69, 0x6c, 0x69, - 0x74, 0x69, 0x65, 0x73, 0x20, 0x3d, 0x20, 0x28, 0x70, 0x61, 0x72, 0x61, - 0x6d, 0x73, 0x29, 0x20, 0x3d, 0x3e, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x70, 0x61, - 0x72, 0x61, 0x6d, 0x73, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x6d, 0x61, - 0x70, 0x28, 0x6d, 0x73, 0x67, 0x20, 0x3d, 0x3e, 0x20, 0x7b, 0x0a, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, - 0x20, 0x7b, 0x20, 0x63, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x69, 0x6f, - 0x6e, 0x5f, 0x70, 0x72, 0x6f, 0x62, 0x61, 0x62, 0x69, 0x6c, 0x69, 0x74, - 0x69, 0x65, 0x73, 0x20, 0x7d, 0x20, 0x3d, 0x20, 0x6d, 0x73, 0x67, 0x3b, - 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, - 0x28, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x21, 0x63, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x69, 0x6f, 0x6e, 0x5f, - 0x70, 0x72, 0x6f, 0x62, 0x61, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x69, 0x65, - 0x73, 0x20, 0x7c, 0x7c, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x69, 0x6f, - 0x6e, 0x5f, 0x70, 0x72, 0x6f, 0x62, 0x61, 0x62, 0x69, 0x6c, 0x69, 0x74, - 0x69, 0x65, 0x73, 0x2e, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x20, 0x3d, - 0x3d, 0x3d, 0x20, 0x30, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x29, 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x6d, 0x73, - 0x67, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x0a, 0x0a, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, 0x63, - 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x70, 0x72, - 0x6f, 0x62, 0x61, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x69, 0x65, 0x73, 0x2e, - 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x20, 0x3e, 0x20, 0x31, 0x29, 0x20, - 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x2f, 0x2f, 0x20, 0x4e, 0x6f, 0x74, 0x20, 0x66, 0x6f, 0x72, 0x20, 0x62, - 0x79, 0x74, 0x65, 0x20, 0x70, 0x61, 0x69, 0x72, 0x0a, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, 0x63, - 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x70, 0x72, - 0x6f, 0x62, 0x61, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x69, 0x65, 0x73, 0x5b, - 0x30, 0x5d, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x2e, 0x73, - 0x74, 0x61, 0x72, 0x74, 0x73, 0x57, 0x69, 0x74, 0x68, 0x28, 0x27, 0x62, - 0x79, 0x74, 0x65, 0x3a, 0x20, 0x5c, 0x5c, 0x27, 0x29, 0x29, 0x20, 0x72, - 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x6d, 0x73, 0x67, 0x2e, 0x63, 0x6f, - 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x73, - 0x70, 0x6c, 0x69, 0x74, 0x44, 0x61, 0x74, 0x61, 0x20, 0x3d, 0x20, 0x63, - 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x70, 0x72, - 0x6f, 0x62, 0x61, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x69, 0x65, 0x73, 0x2e, - 0x6d, 0x61, 0x70, 0x28, 0x70, 0x72, 0x6f, 0x62, 0x20, 0x3d, 0x3e, 0x20, - 0x28, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x3a, 0x20, - 0x70, 0x72, 0x6f, 0x62, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, - 0x2c, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x63, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x69, 0x6f, 0x6e, - 0x5f, 0x70, 0x72, 0x6f, 0x62, 0x61, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x69, - 0x65, 0x73, 0x3a, 0x20, 0x5b, 0x70, 0x72, 0x6f, 0x62, 0x5d, 0x0a, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x29, 0x29, - 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, - 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x68, 0x74, 0x6d, 0x6c, 0x60, 0x3c, - 0x24, 0x7b, 0x50, 0x72, 0x6f, 0x62, 0x61, 0x62, 0x69, 0x6c, 0x69, 0x74, - 0x69, 0x65, 0x73, 0x7d, 0x20, 0x64, 0x61, 0x74, 0x61, 0x3d, 0x24, 0x7b, - 0x73, 0x70, 0x6c, 0x69, 0x74, 0x44, 0x61, 0x74, 0x61, 0x7d, 0x20, 0x2f, - 0x3e, 0x60, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, - 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, - 0x6e, 0x73, 0x74, 0x20, 0x7b, 0x20, 0x70, 0x72, 0x6f, 0x62, 0x73, 0x2c, - 0x20, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x20, 0x7d, 0x20, 0x3d, - 0x20, 0x63, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x69, 0x6f, 0x6e, 0x5f, - 0x70, 0x72, 0x6f, 0x62, 0x61, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x69, 0x65, - 0x73, 0x5b, 0x30, 0x5d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x66, 0x6f, 0x75, 0x6e, 0x64, - 0x20, 0x3d, 0x20, 0x70, 0x72, 0x6f, 0x62, 0x73, 0x2e, 0x66, 0x69, 0x6e, - 0x64, 0x28, 0x70, 0x20, 0x3d, 0x3e, 0x20, 0x70, 0x2e, 0x74, 0x6f, 0x6b, - 0x5f, 0x73, 0x74, 0x72, 0x20, 0x3d, 0x3d, 0x3d, 0x20, 0x6d, 0x73, 0x67, - 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x29, 0x0a, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, - 0x70, 0x43, 0x6f, 0x6c, 0x6f, 0x72, 0x20, 0x3d, 0x20, 0x66, 0x6f, 0x75, - 0x6e, 0x64, 0x20, 0x3f, 0x20, 0x70, 0x72, 0x6f, 0x62, 0x43, 0x6f, 0x6c, - 0x6f, 0x72, 0x28, 0x66, 0x6f, 0x75, 0x6e, 0x64, 0x2e, 0x70, 0x72, 0x6f, - 0x62, 0x29, 0x20, 0x3a, 0x20, 0x27, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x70, - 0x61, 0x72, 0x65, 0x6e, 0x74, 0x27, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x70, 0x6f, - 0x70, 0x6f, 0x76, 0x65, 0x72, 0x43, 0x68, 0x69, 0x6c, 0x64, 0x72, 0x65, - 0x6e, 0x20, 0x3d, 0x20, 0x68, 0x74, 0x6d, 0x6c, 0x60, 0x0a, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x64, 0x69, 0x76, - 0x20, 0x63, 0x6c, 0x61, 0x73, 0x73, 0x3d, 0x22, 0x70, 0x72, 0x6f, 0x62, - 0x2d, 0x73, 0x65, 0x74, 0x22, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x24, 0x7b, 0x70, 0x72, 0x6f, - 0x62, 0x73, 0x2e, 0x6d, 0x61, 0x70, 0x28, 0x28, 0x70, 0x2c, 0x20, 0x69, - 0x6e, 0x64, 0x65, 0x78, 0x29, 0x20, 0x3d, 0x3e, 0x20, 0x7b, 0x0a, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, 0x74, - 0x75, 0x72, 0x6e, 0x20, 0x68, 0x74, 0x6d, 0x6c, 0x60, 0x0a, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x3c, 0x64, 0x69, 0x76, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x6b, 0x65, 0x79, 0x3d, 0x24, 0x7b, 0x69, 0x6e, 0x64, 0x65, 0x78, - 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x74, 0x69, 0x74, 0x6c, - 0x65, 0x3d, 0x24, 0x7b, 0x60, 0x70, 0x72, 0x6f, 0x62, 0x3a, 0x20, 0x24, - 0x7b, 0x70, 0x2e, 0x70, 0x72, 0x6f, 0x62, 0x7d, 0x60, 0x7d, 0x0a, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x73, 0x74, 0x79, 0x6c, 0x65, 0x3d, 0x24, - 0x7b, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x70, 0x61, 0x64, 0x64, 0x69, 0x6e, 0x67, - 0x3a, 0x20, 0x27, 0x30, 0x2e, 0x33, 0x65, 0x6d, 0x27, 0x2c, 0x0a, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x62, 0x61, 0x63, 0x6b, 0x67, 0x72, 0x6f, 0x75, 0x6e, 0x64, 0x43, - 0x6f, 0x6c, 0x6f, 0x72, 0x3a, 0x20, 0x70, 0x2e, 0x74, 0x6f, 0x6b, 0x5f, - 0x73, 0x74, 0x72, 0x20, 0x3d, 0x3d, 0x3d, 0x20, 0x63, 0x6f, 0x6e, 0x74, - 0x65, 0x6e, 0x74, 0x20, 0x3f, 0x20, 0x70, 0x72, 0x6f, 0x62, 0x43, 0x6f, - 0x6c, 0x6f, 0x72, 0x28, 0x70, 0x2e, 0x70, 0x72, 0x6f, 0x62, 0x29, 0x20, - 0x3a, 0x20, 0x27, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x70, 0x61, 0x72, 0x65, - 0x6e, 0x74, 0x27, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x7d, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x73, 0x70, 0x61, 0x6e, - 0x3e, 0x24, 0x7b, 0x70, 0x2e, 0x74, 0x6f, 0x6b, 0x5f, 0x73, 0x74, 0x72, - 0x7d, 0x3a, 0x20, 0x3c, 0x2f, 0x73, 0x70, 0x61, 0x6e, 0x3e, 0x0a, 0x20, + 0x20, 0x3c, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x65, 0x74, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x73, 0x70, 0x61, 0x6e, 0x3e, 0x24, - 0x7b, 0x4d, 0x61, 0x74, 0x68, 0x2e, 0x66, 0x6c, 0x6f, 0x6f, 0x72, 0x28, - 0x70, 0x2e, 0x70, 0x72, 0x6f, 0x62, 0x20, 0x2a, 0x20, 0x31, 0x30, 0x30, - 0x29, 0x7d, 0x25, 0x3c, 0x2f, 0x73, 0x70, 0x61, 0x6e, 0x3e, 0x0a, 0x20, + 0x20, 0x20, 0x3c, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x20, 0x66, 0x6f, 0x72, + 0x3d, 0x22, 0x61, 0x70, 0x69, 0x5f, 0x6b, 0x65, 0x79, 0x22, 0x3e, 0x41, + 0x50, 0x49, 0x20, 0x4b, 0x65, 0x79, 0x3c, 0x2f, 0x6c, 0x61, 0x62, 0x65, + 0x6c, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x20, + 0x74, 0x79, 0x70, 0x65, 0x3d, 0x22, 0x74, 0x65, 0x78, 0x74, 0x22, 0x20, + 0x6e, 0x61, 0x6d, 0x65, 0x3d, 0x22, 0x61, 0x70, 0x69, 0x5f, 0x6b, 0x65, + 0x79, 0x22, 0x20, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3d, 0x22, 0x24, 0x7b, + 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, + 0x2e, 0x61, 0x70, 0x69, 0x5f, 0x6b, 0x65, 0x79, 0x7d, 0x22, 0x20, 0x70, + 0x6c, 0x61, 0x63, 0x65, 0x68, 0x6f, 0x6c, 0x64, 0x65, 0x72, 0x3d, 0x22, + 0x45, 0x6e, 0x74, 0x65, 0x72, 0x20, 0x41, 0x50, 0x49, 0x20, 0x6b, 0x65, + 0x79, 0x22, 0x20, 0x6f, 0x6e, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x3d, 0x24, + 0x7b, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x50, 0x61, 0x72, 0x61, 0x6d, + 0x73, 0x7d, 0x20, 0x2f, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x2f, 0x66, 0x69, 0x65, 0x6c, + 0x64, 0x73, 0x65, 0x74, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x3c, 0x2f, 0x64, 0x65, 0x74, 0x61, 0x69, 0x6c, + 0x73, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, + 0x2f, 0x66, 0x6f, 0x72, 0x6d, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x60, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x70, 0x72, 0x6f, 0x62, + 0x43, 0x6f, 0x6c, 0x6f, 0x72, 0x20, 0x3d, 0x20, 0x28, 0x70, 0x29, 0x20, + 0x3d, 0x3e, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, + 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x72, 0x20, 0x3d, 0x20, 0x4d, 0x61, 0x74, + 0x68, 0x2e, 0x66, 0x6c, 0x6f, 0x6f, 0x72, 0x28, 0x31, 0x39, 0x32, 0x20, + 0x2a, 0x20, 0x28, 0x31, 0x20, 0x2d, 0x20, 0x70, 0x29, 0x29, 0x3b, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, + 0x67, 0x20, 0x3d, 0x20, 0x4d, 0x61, 0x74, 0x68, 0x2e, 0x66, 0x6c, 0x6f, + 0x6f, 0x72, 0x28, 0x31, 0x39, 0x32, 0x20, 0x2a, 0x20, 0x70, 0x29, 0x3b, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, + 0x6e, 0x20, 0x60, 0x72, 0x67, 0x62, 0x61, 0x28, 0x24, 0x7b, 0x72, 0x7d, + 0x2c, 0x24, 0x7b, 0x67, 0x7d, 0x2c, 0x30, 0x2c, 0x30, 0x2e, 0x33, 0x29, + 0x60, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x50, 0x72, 0x6f, 0x62, + 0x61, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x69, 0x65, 0x73, 0x20, 0x3d, 0x20, + 0x28, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x29, 0x20, 0x3d, 0x3e, 0x20, + 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, + 0x72, 0x6e, 0x20, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x2e, 0x64, 0x61, + 0x74, 0x61, 0x2e, 0x6d, 0x61, 0x70, 0x28, 0x6d, 0x73, 0x67, 0x20, 0x3d, + 0x3e, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x7b, 0x20, 0x63, 0x6f, 0x6d, 0x70, + 0x6c, 0x65, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x70, 0x72, 0x6f, 0x62, 0x61, + 0x62, 0x69, 0x6c, 0x69, 0x74, 0x69, 0x65, 0x73, 0x20, 0x7d, 0x20, 0x3d, + 0x20, 0x6d, 0x73, 0x67, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x21, 0x63, 0x6f, 0x6d, 0x70, 0x6c, 0x65, + 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x70, 0x72, 0x6f, 0x62, 0x61, 0x62, 0x69, + 0x6c, 0x69, 0x74, 0x69, 0x65, 0x73, 0x20, 0x7c, 0x7c, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6d, 0x70, + 0x6c, 0x65, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x70, 0x72, 0x6f, 0x62, 0x61, + 0x62, 0x69, 0x6c, 0x69, 0x74, 0x69, 0x65, 0x73, 0x2e, 0x6c, 0x65, 0x6e, + 0x67, 0x74, 0x68, 0x20, 0x3d, 0x3d, 0x3d, 0x20, 0x30, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x29, 0x20, 0x72, 0x65, 0x74, 0x75, + 0x72, 0x6e, 0x20, 0x6d, 0x73, 0x67, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x65, + 0x6e, 0x74, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x69, 0x66, 0x20, 0x28, 0x63, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x69, + 0x6f, 0x6e, 0x5f, 0x70, 0x72, 0x6f, 0x62, 0x61, 0x62, 0x69, 0x6c, 0x69, + 0x74, 0x69, 0x65, 0x73, 0x2e, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x20, + 0x3e, 0x20, 0x31, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x2f, 0x2f, 0x20, 0x4e, 0x6f, 0x74, 0x20, + 0x66, 0x6f, 0x72, 0x20, 0x62, 0x79, 0x74, 0x65, 0x20, 0x70, 0x61, 0x69, + 0x72, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x69, 0x66, 0x20, 0x28, 0x63, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x69, + 0x6f, 0x6e, 0x5f, 0x70, 0x72, 0x6f, 0x62, 0x61, 0x62, 0x69, 0x6c, 0x69, + 0x74, 0x69, 0x65, 0x73, 0x5b, 0x30, 0x5d, 0x2e, 0x63, 0x6f, 0x6e, 0x74, + 0x65, 0x6e, 0x74, 0x2e, 0x73, 0x74, 0x61, 0x72, 0x74, 0x73, 0x57, 0x69, + 0x74, 0x68, 0x28, 0x27, 0x62, 0x79, 0x74, 0x65, 0x3a, 0x20, 0x5c, 0x5c, + 0x27, 0x29, 0x29, 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x6d, + 0x73, 0x67, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x0a, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, + 0x6e, 0x73, 0x74, 0x20, 0x73, 0x70, 0x6c, 0x69, 0x74, 0x44, 0x61, 0x74, + 0x61, 0x20, 0x3d, 0x20, 0x63, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x69, + 0x6f, 0x6e, 0x5f, 0x70, 0x72, 0x6f, 0x62, 0x61, 0x62, 0x69, 0x6c, 0x69, + 0x74, 0x69, 0x65, 0x73, 0x2e, 0x6d, 0x61, 0x70, 0x28, 0x70, 0x72, 0x6f, + 0x62, 0x20, 0x3d, 0x3e, 0x20, 0x28, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x74, + 0x65, 0x6e, 0x74, 0x3a, 0x20, 0x70, 0x72, 0x6f, 0x62, 0x2e, 0x63, 0x6f, + 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x2c, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6d, 0x70, 0x6c, + 0x65, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x70, 0x72, 0x6f, 0x62, 0x61, 0x62, + 0x69, 0x6c, 0x69, 0x74, 0x69, 0x65, 0x73, 0x3a, 0x20, 0x5b, 0x70, 0x72, + 0x6f, 0x62, 0x5d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x7d, 0x29, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x68, + 0x74, 0x6d, 0x6c, 0x60, 0x3c, 0x24, 0x7b, 0x50, 0x72, 0x6f, 0x62, 0x61, + 0x62, 0x69, 0x6c, 0x69, 0x74, 0x69, 0x65, 0x73, 0x7d, 0x20, 0x64, 0x61, + 0x74, 0x61, 0x3d, 0x24, 0x7b, 0x73, 0x70, 0x6c, 0x69, 0x74, 0x44, 0x61, + 0x74, 0x61, 0x7d, 0x20, 0x2f, 0x3e, 0x60, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x7b, 0x20, 0x70, + 0x72, 0x6f, 0x62, 0x73, 0x2c, 0x20, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, + 0x74, 0x20, 0x7d, 0x20, 0x3d, 0x20, 0x63, 0x6f, 0x6d, 0x70, 0x6c, 0x65, + 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x70, 0x72, 0x6f, 0x62, 0x61, 0x62, 0x69, + 0x6c, 0x69, 0x74, 0x69, 0x65, 0x73, 0x5b, 0x30, 0x5d, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, + 0x66, 0x6f, 0x75, 0x6e, 0x64, 0x20, 0x3d, 0x20, 0x70, 0x72, 0x6f, 0x62, + 0x73, 0x2e, 0x66, 0x69, 0x6e, 0x64, 0x28, 0x70, 0x20, 0x3d, 0x3e, 0x20, + 0x70, 0x2e, 0x74, 0x6f, 0x6b, 0x5f, 0x73, 0x74, 0x72, 0x20, 0x3d, 0x3d, + 0x3d, 0x20, 0x6d, 0x73, 0x67, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, + 0x74, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, + 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x70, 0x43, 0x6f, 0x6c, 0x6f, 0x72, 0x20, + 0x3d, 0x20, 0x66, 0x6f, 0x75, 0x6e, 0x64, 0x20, 0x3f, 0x20, 0x70, 0x72, + 0x6f, 0x62, 0x43, 0x6f, 0x6c, 0x6f, 0x72, 0x28, 0x66, 0x6f, 0x75, 0x6e, + 0x64, 0x2e, 0x70, 0x72, 0x6f, 0x62, 0x29, 0x20, 0x3a, 0x20, 0x27, 0x74, + 0x72, 0x61, 0x6e, 0x73, 0x70, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x27, 0x0a, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, + 0x73, 0x74, 0x20, 0x70, 0x6f, 0x70, 0x6f, 0x76, 0x65, 0x72, 0x43, 0x68, + 0x69, 0x6c, 0x64, 0x72, 0x65, 0x6e, 0x20, 0x3d, 0x20, 0x68, 0x74, 0x6d, + 0x6c, 0x60, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x3c, 0x64, 0x69, 0x76, 0x20, 0x63, 0x6c, 0x61, 0x73, 0x73, 0x3d, + 0x22, 0x70, 0x72, 0x6f, 0x62, 0x2d, 0x73, 0x65, 0x74, 0x22, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x3c, 0x2f, 0x64, 0x69, 0x76, 0x3e, 0x0a, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x60, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x29, - 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x3c, 0x2f, 0x64, 0x69, 0x76, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x60, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x24, 0x7b, 0x70, 0x72, 0x6f, 0x62, 0x73, 0x2e, 0x6d, 0x61, 0x70, 0x28, + 0x28, 0x70, 0x2c, 0x20, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x29, 0x20, 0x3d, + 0x3e, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x68, 0x74, 0x6d, 0x6c, 0x60, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x3c, 0x24, 0x7b, 0x50, 0x6f, 0x70, 0x6f, 0x76, 0x65, 0x72, 0x7d, - 0x20, 0x73, 0x74, 0x79, 0x6c, 0x65, 0x3d, 0x24, 0x7b, 0x7b, 0x20, 0x62, - 0x61, 0x63, 0x6b, 0x67, 0x72, 0x6f, 0x75, 0x6e, 0x64, 0x43, 0x6f, 0x6c, - 0x6f, 0x72, 0x3a, 0x20, 0x70, 0x43, 0x6f, 0x6c, 0x6f, 0x72, 0x20, 0x7d, - 0x7d, 0x20, 0x70, 0x6f, 0x70, 0x6f, 0x76, 0x65, 0x72, 0x43, 0x68, 0x69, - 0x6c, 0x64, 0x72, 0x65, 0x6e, 0x3d, 0x24, 0x7b, 0x70, 0x6f, 0x70, 0x6f, - 0x76, 0x65, 0x72, 0x43, 0x68, 0x69, 0x6c, 0x64, 0x72, 0x65, 0x6e, 0x7d, - 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x24, 0x7b, 0x6d, 0x73, 0x67, 0x2e, 0x63, 0x6f, 0x6e, 0x74, - 0x65, 0x6e, 0x74, 0x2e, 0x6d, 0x61, 0x74, 0x63, 0x68, 0x28, 0x2f, 0x5c, - 0x6e, 0x2f, 0x67, 0x69, 0x6d, 0x29, 0x20, 0x3f, 0x20, 0x68, 0x74, 0x6d, - 0x6c, 0x60, 0x3c, 0x62, 0x72, 0x20, 0x2f, 0x3e, 0x60, 0x20, 0x3a, 0x20, - 0x6d, 0x73, 0x67, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x7d, - 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, - 0x2f, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x60, - 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x29, 0x3b, 0x0a, 0x20, - 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x2f, 0x2f, - 0x20, 0x70, 0x6f, 0x6f, 0x72, 0x20, 0x6d, 0x61, 0x6e, 0x73, 0x20, 0x6d, - 0x61, 0x72, 0x6b, 0x64, 0x6f, 0x77, 0x6e, 0x20, 0x72, 0x65, 0x70, 0x6c, - 0x61, 0x63, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x0a, 0x20, 0x20, 0x20, 0x20, - 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x4d, 0x61, 0x72, 0x6b, 0x64, 0x6f, - 0x77, 0x6e, 0x69, 0x73, 0x68, 0x20, 0x3d, 0x20, 0x28, 0x70, 0x61, 0x72, - 0x61, 0x6d, 0x73, 0x29, 0x20, 0x3d, 0x3e, 0x20, 0x7b, 0x0a, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x6d, 0x64, - 0x20, 0x3d, 0x20, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x2e, 0x74, 0x65, - 0x78, 0x74, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x2e, - 0x72, 0x65, 0x70, 0x6c, 0x61, 0x63, 0x65, 0x28, 0x2f, 0x26, 0x2f, 0x67, - 0x2c, 0x20, 0x27, 0x26, 0x61, 0x6d, 0x70, 0x3b, 0x27, 0x29, 0x0a, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x2e, 0x72, 0x65, 0x70, 0x6c, - 0x61, 0x63, 0x65, 0x28, 0x2f, 0x3c, 0x2f, 0x67, 0x2c, 0x20, 0x27, 0x26, - 0x6c, 0x74, 0x3b, 0x27, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x2e, 0x72, 0x65, 0x70, 0x6c, 0x61, 0x63, 0x65, 0x28, 0x2f, - 0x3e, 0x2f, 0x67, 0x2c, 0x20, 0x27, 0x26, 0x67, 0x74, 0x3b, 0x27, 0x29, - 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x2e, 0x72, 0x65, - 0x70, 0x6c, 0x61, 0x63, 0x65, 0x28, 0x2f, 0x5e, 0x23, 0x7b, 0x31, 0x2c, - 0x36, 0x7d, 0x20, 0x28, 0x2e, 0x2a, 0x29, 0x24, 0x2f, 0x67, 0x69, 0x6d, - 0x2c, 0x20, 0x27, 0x3c, 0x68, 0x33, 0x3e, 0x24, 0x31, 0x3c, 0x2f, 0x68, - 0x33, 0x3e, 0x27, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x2e, 0x72, 0x65, 0x70, 0x6c, 0x61, 0x63, 0x65, 0x28, 0x2f, 0x5c, - 0x2a, 0x5c, 0x2a, 0x28, 0x2e, 0x2a, 0x3f, 0x29, 0x5c, 0x2a, 0x5c, 0x2a, - 0x2f, 0x67, 0x2c, 0x20, 0x27, 0x3c, 0x73, 0x74, 0x72, 0x6f, 0x6e, 0x67, - 0x3e, 0x24, 0x31, 0x3c, 0x2f, 0x73, 0x74, 0x72, 0x6f, 0x6e, 0x67, 0x3e, - 0x27, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x2e, - 0x72, 0x65, 0x70, 0x6c, 0x61, 0x63, 0x65, 0x28, 0x2f, 0x5f, 0x5f, 0x28, - 0x2e, 0x2a, 0x3f, 0x29, 0x5f, 0x5f, 0x2f, 0x67, 0x2c, 0x20, 0x27, 0x3c, - 0x73, 0x74, 0x72, 0x6f, 0x6e, 0x67, 0x3e, 0x24, 0x31, 0x3c, 0x2f, 0x73, - 0x74, 0x72, 0x6f, 0x6e, 0x67, 0x3e, 0x27, 0x29, 0x0a, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x2e, 0x72, 0x65, 0x70, 0x6c, 0x61, 0x63, - 0x65, 0x28, 0x2f, 0x5c, 0x2a, 0x28, 0x2e, 0x2a, 0x3f, 0x29, 0x5c, 0x2a, - 0x2f, 0x67, 0x2c, 0x20, 0x27, 0x3c, 0x65, 0x6d, 0x3e, 0x24, 0x31, 0x3c, - 0x2f, 0x65, 0x6d, 0x3e, 0x27, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x2e, 0x72, 0x65, 0x70, 0x6c, 0x61, 0x63, 0x65, 0x28, - 0x2f, 0x5f, 0x28, 0x2e, 0x2a, 0x3f, 0x29, 0x5f, 0x2f, 0x67, 0x2c, 0x20, - 0x27, 0x3c, 0x65, 0x6d, 0x3e, 0x24, 0x31, 0x3c, 0x2f, 0x65, 0x6d, 0x3e, - 0x27, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x2e, - 0x72, 0x65, 0x70, 0x6c, 0x61, 0x63, 0x65, 0x28, 0x2f, 0x60, 0x60, 0x60, - 0x2e, 0x2a, 0x3f, 0x5c, 0x6e, 0x28, 0x5b, 0x5c, 0x73, 0x5c, 0x53, 0x5d, - 0x2a, 0x3f, 0x29, 0x60, 0x60, 0x60, 0x2f, 0x67, 0x2c, 0x20, 0x27, 0x3c, - 0x70, 0x72, 0x65, 0x3e, 0x3c, 0x63, 0x6f, 0x64, 0x65, 0x3e, 0x24, 0x31, - 0x3c, 0x2f, 0x63, 0x6f, 0x64, 0x65, 0x3e, 0x3c, 0x2f, 0x70, 0x72, 0x65, - 0x3e, 0x27, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x2e, 0x72, 0x65, 0x70, 0x6c, 0x61, 0x63, 0x65, 0x28, 0x2f, 0x60, 0x28, - 0x2e, 0x2a, 0x3f, 0x29, 0x60, 0x2f, 0x67, 0x2c, 0x20, 0x27, 0x3c, 0x63, - 0x6f, 0x64, 0x65, 0x3e, 0x24, 0x31, 0x3c, 0x2f, 0x63, 0x6f, 0x64, 0x65, - 0x3e, 0x27, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x2e, 0x72, 0x65, 0x70, 0x6c, 0x61, 0x63, 0x65, 0x28, 0x2f, 0x5c, 0x6e, - 0x2f, 0x67, 0x69, 0x6d, 0x2c, 0x20, 0x27, 0x3c, 0x62, 0x72, 0x20, 0x2f, - 0x3e, 0x27, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, - 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x68, 0x74, 0x6d, 0x6c, 0x60, 0x3c, - 0x73, 0x70, 0x61, 0x6e, 0x20, 0x64, 0x61, 0x6e, 0x67, 0x65, 0x72, 0x6f, - 0x75, 0x73, 0x6c, 0x79, 0x53, 0x65, 0x74, 0x49, 0x6e, 0x6e, 0x65, 0x72, - 0x48, 0x54, 0x4d, 0x4c, 0x3d, 0x24, 0x7b, 0x7b, 0x20, 0x5f, 0x5f, 0x68, - 0x74, 0x6d, 0x6c, 0x3a, 0x20, 0x6d, 0x64, 0x20, 0x7d, 0x7d, 0x20, 0x2f, - 0x3e, 0x60, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x3b, 0x0a, 0x0a, - 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x4d, 0x6f, - 0x64, 0x65, 0x6c, 0x47, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, - 0x6e, 0x49, 0x6e, 0x66, 0x6f, 0x20, 0x3d, 0x20, 0x28, 0x70, 0x61, 0x72, - 0x61, 0x6d, 0x73, 0x29, 0x20, 0x3d, 0x3e, 0x20, 0x7b, 0x0a, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, 0x21, 0x6c, 0x6c, 0x61, - 0x6d, 0x61, 0x53, 0x74, 0x61, 0x74, 0x73, 0x2e, 0x76, 0x61, 0x6c, 0x75, - 0x65, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x68, 0x74, 0x6d, 0x6c, - 0x60, 0x3c, 0x73, 0x70, 0x61, 0x6e, 0x2f, 0x3e, 0x60, 0x0a, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x68, 0x74, 0x6d, 0x6c, 0x60, - 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x73, 0x70, - 0x61, 0x6e, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x24, 0x7b, 0x6c, 0x6c, 0x61, 0x6d, 0x61, 0x53, 0x74, 0x61, - 0x74, 0x73, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x2e, 0x74, 0x6f, 0x6b, - 0x65, 0x6e, 0x73, 0x5f, 0x70, 0x72, 0x65, 0x64, 0x69, 0x63, 0x74, 0x65, - 0x64, 0x7d, 0x20, 0x70, 0x72, 0x65, 0x64, 0x69, 0x63, 0x74, 0x65, 0x64, - 0x2c, 0x20, 0x24, 0x7b, 0x6c, 0x6c, 0x61, 0x6d, 0x61, 0x53, 0x74, 0x61, - 0x74, 0x73, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x2e, 0x74, 0x6f, 0x6b, - 0x65, 0x6e, 0x73, 0x5f, 0x63, 0x61, 0x63, 0x68, 0x65, 0x64, 0x7d, 0x20, - 0x63, 0x61, 0x63, 0x68, 0x65, 0x64, 0x2c, 0x20, 0x24, 0x7b, 0x6c, 0x6c, - 0x61, 0x6d, 0x61, 0x53, 0x74, 0x61, 0x74, 0x73, 0x2e, 0x76, 0x61, 0x6c, - 0x75, 0x65, 0x2e, 0x74, 0x69, 0x6d, 0x69, 0x6e, 0x67, 0x73, 0x2e, 0x70, - 0x72, 0x65, 0x64, 0x69, 0x63, 0x74, 0x65, 0x64, 0x5f, 0x70, 0x65, 0x72, - 0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x5f, 0x6d, 0x73, 0x2e, 0x74, 0x6f, - 0x46, 0x69, 0x78, 0x65, 0x64, 0x28, 0x29, 0x7d, 0x6d, 0x73, 0x20, 0x70, - 0x65, 0x72, 0x20, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x2c, 0x20, 0x24, 0x7b, - 0x6c, 0x6c, 0x61, 0x6d, 0x61, 0x53, 0x74, 0x61, 0x74, 0x73, 0x2e, 0x76, - 0x61, 0x6c, 0x75, 0x65, 0x2e, 0x74, 0x69, 0x6d, 0x69, 0x6e, 0x67, 0x73, - 0x2e, 0x70, 0x72, 0x65, 0x64, 0x69, 0x63, 0x74, 0x65, 0x64, 0x5f, 0x70, - 0x65, 0x72, 0x5f, 0x73, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x2e, 0x74, 0x6f, - 0x46, 0x69, 0x78, 0x65, 0x64, 0x28, 0x32, 0x29, 0x7d, 0x20, 0x74, 0x6f, - 0x6b, 0x65, 0x6e, 0x73, 0x20, 0x70, 0x65, 0x72, 0x20, 0x73, 0x65, 0x63, - 0x6f, 0x6e, 0x64, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x3c, 0x2f, 0x73, 0x70, 0x61, 0x6e, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x60, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x0a, 0x20, - 0x20, 0x20, 0x20, 0x2f, 0x2f, 0x20, 0x73, 0x69, 0x6d, 0x70, 0x6c, 0x65, - 0x20, 0x70, 0x6f, 0x70, 0x6f, 0x76, 0x65, 0x72, 0x20, 0x69, 0x6d, 0x70, - 0x6c, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, - 0x50, 0x6f, 0x70, 0x6f, 0x76, 0x65, 0x72, 0x20, 0x3d, 0x20, 0x28, 0x70, - 0x72, 0x6f, 0x70, 0x73, 0x29, 0x20, 0x3d, 0x3e, 0x20, 0x7b, 0x0a, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x69, - 0x73, 0x4f, 0x70, 0x65, 0x6e, 0x20, 0x3d, 0x20, 0x75, 0x73, 0x65, 0x53, - 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x28, 0x66, 0x61, 0x6c, 0x73, 0x65, 0x29, - 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, - 0x74, 0x20, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x3d, - 0x20, 0x75, 0x73, 0x65, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x28, 0x7b, - 0x20, 0x74, 0x6f, 0x70, 0x3a, 0x20, 0x27, 0x30, 0x70, 0x78, 0x27, 0x2c, - 0x20, 0x6c, 0x65, 0x66, 0x74, 0x3a, 0x20, 0x27, 0x30, 0x70, 0x78, 0x27, - 0x20, 0x7d, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, - 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x62, 0x75, 0x74, 0x74, 0x6f, 0x6e, 0x52, - 0x65, 0x66, 0x20, 0x3d, 0x20, 0x75, 0x73, 0x65, 0x52, 0x65, 0x66, 0x28, - 0x6e, 0x75, 0x6c, 0x6c, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x70, 0x6f, 0x70, 0x6f, 0x76, - 0x65, 0x72, 0x52, 0x65, 0x66, 0x20, 0x3d, 0x20, 0x75, 0x73, 0x65, 0x52, - 0x65, 0x66, 0x28, 0x6e, 0x75, 0x6c, 0x6c, 0x29, 0x3b, 0x0a, 0x0a, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x74, - 0x6f, 0x67, 0x67, 0x6c, 0x65, 0x50, 0x6f, 0x70, 0x6f, 0x76, 0x65, 0x72, - 0x20, 0x3d, 0x20, 0x28, 0x29, 0x20, 0x3d, 0x3e, 0x20, 0x7b, 0x0a, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, 0x62, - 0x75, 0x74, 0x74, 0x6f, 0x6e, 0x52, 0x65, 0x66, 0x2e, 0x63, 0x75, 0x72, - 0x72, 0x65, 0x6e, 0x74, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, - 0x72, 0x65, 0x63, 0x74, 0x20, 0x3d, 0x20, 0x62, 0x75, 0x74, 0x74, 0x6f, - 0x6e, 0x52, 0x65, 0x66, 0x2e, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x74, - 0x2e, 0x67, 0x65, 0x74, 0x42, 0x6f, 0x75, 0x6e, 0x64, 0x69, 0x6e, 0x67, - 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x63, 0x74, 0x28, 0x29, - 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x70, 0x6f, 0x73, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x61, 0x6c, - 0x75, 0x65, 0x20, 0x3d, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x74, 0x6f, 0x70, 0x3a, 0x20, - 0x60, 0x24, 0x7b, 0x72, 0x65, 0x63, 0x74, 0x2e, 0x62, 0x6f, 0x74, 0x74, - 0x6f, 0x6d, 0x20, 0x2b, 0x20, 0x77, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x2e, - 0x73, 0x63, 0x72, 0x6f, 0x6c, 0x6c, 0x59, 0x7d, 0x70, 0x78, 0x60, 0x2c, - 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x6c, 0x65, 0x66, 0x74, 0x3a, 0x20, 0x60, 0x24, 0x7b, 0x72, 0x65, - 0x63, 0x74, 0x2e, 0x6c, 0x65, 0x66, 0x74, 0x20, 0x2b, 0x20, 0x77, 0x69, - 0x6e, 0x64, 0x6f, 0x77, 0x2e, 0x73, 0x63, 0x72, 0x6f, 0x6c, 0x6c, 0x58, - 0x7d, 0x70, 0x78, 0x60, 0x2c, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x7d, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x69, 0x73, 0x4f, 0x70, 0x65, 0x6e, 0x2e, 0x76, 0x61, 0x6c, 0x75, - 0x65, 0x20, 0x3d, 0x20, 0x21, 0x69, 0x73, 0x4f, 0x70, 0x65, 0x6e, 0x2e, - 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x7d, 0x3b, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, - 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x68, 0x61, 0x6e, 0x64, 0x6c, 0x65, 0x43, - 0x6c, 0x69, 0x63, 0x6b, 0x4f, 0x75, 0x74, 0x73, 0x69, 0x64, 0x65, 0x20, - 0x3d, 0x20, 0x28, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x29, 0x20, 0x3d, 0x3e, - 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, - 0x66, 0x20, 0x28, 0x70, 0x6f, 0x70, 0x6f, 0x76, 0x65, 0x72, 0x52, 0x65, - 0x66, 0x2e, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x74, 0x20, 0x26, 0x26, - 0x20, 0x21, 0x70, 0x6f, 0x70, 0x6f, 0x76, 0x65, 0x72, 0x52, 0x65, 0x66, - 0x2e, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x74, 0x2e, 0x63, 0x6f, 0x6e, - 0x74, 0x61, 0x69, 0x6e, 0x73, 0x28, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x2e, - 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x29, 0x20, 0x26, 0x26, 0x20, 0x21, - 0x62, 0x75, 0x74, 0x74, 0x6f, 0x6e, 0x52, 0x65, 0x66, 0x2e, 0x63, 0x75, - 0x72, 0x72, 0x65, 0x6e, 0x74, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, - 0x6e, 0x73, 0x28, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x2e, 0x74, 0x61, 0x72, - 0x67, 0x65, 0x74, 0x29, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x73, 0x4f, 0x70, 0x65, 0x6e, - 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x20, 0x3d, 0x20, 0x66, 0x61, 0x6c, - 0x73, 0x65, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x3b, 0x0a, 0x0a, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x75, 0x73, 0x65, 0x45, 0x66, 0x66, - 0x65, 0x63, 0x74, 0x28, 0x28, 0x29, 0x20, 0x3d, 0x3e, 0x20, 0x7b, 0x0a, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x64, 0x6f, 0x63, 0x75, - 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x61, 0x64, 0x64, 0x45, 0x76, 0x65, 0x6e, - 0x74, 0x4c, 0x69, 0x73, 0x74, 0x65, 0x6e, 0x65, 0x72, 0x28, 0x27, 0x6d, - 0x6f, 0x75, 0x73, 0x65, 0x64, 0x6f, 0x77, 0x6e, 0x27, 0x2c, 0x20, 0x68, - 0x61, 0x6e, 0x64, 0x6c, 0x65, 0x43, 0x6c, 0x69, 0x63, 0x6b, 0x4f, 0x75, - 0x74, 0x73, 0x69, 0x64, 0x65, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x28, - 0x29, 0x20, 0x3d, 0x3e, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x64, 0x6f, 0x63, 0x75, 0x6d, 0x65, 0x6e, - 0x74, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x45, 0x76, 0x65, 0x6e, - 0x74, 0x4c, 0x69, 0x73, 0x74, 0x65, 0x6e, 0x65, 0x72, 0x28, 0x27, 0x6d, - 0x6f, 0x75, 0x73, 0x65, 0x64, 0x6f, 0x77, 0x6e, 0x27, 0x2c, 0x20, 0x68, - 0x61, 0x6e, 0x64, 0x6c, 0x65, 0x43, 0x6c, 0x69, 0x63, 0x6b, 0x4f, 0x75, - 0x74, 0x73, 0x69, 0x64, 0x65, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x7d, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x7d, 0x2c, 0x20, 0x5b, 0x5d, 0x29, 0x3b, 0x0a, 0x0a, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x68, - 0x74, 0x6d, 0x6c, 0x60, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x3c, 0x73, 0x70, 0x61, 0x6e, 0x20, 0x73, 0x74, 0x79, 0x6c, 0x65, - 0x3d, 0x24, 0x7b, 0x70, 0x72, 0x6f, 0x70, 0x73, 0x2e, 0x73, 0x74, 0x79, - 0x6c, 0x65, 0x7d, 0x20, 0x72, 0x65, 0x66, 0x3d, 0x24, 0x7b, 0x62, 0x75, - 0x74, 0x74, 0x6f, 0x6e, 0x52, 0x65, 0x66, 0x7d, 0x20, 0x6f, 0x6e, 0x43, - 0x6c, 0x69, 0x63, 0x6b, 0x3d, 0x24, 0x7b, 0x74, 0x6f, 0x67, 0x67, 0x6c, - 0x65, 0x50, 0x6f, 0x70, 0x6f, 0x76, 0x65, 0x72, 0x7d, 0x3e, 0x24, 0x7b, - 0x70, 0x72, 0x6f, 0x70, 0x73, 0x2e, 0x63, 0x68, 0x69, 0x6c, 0x64, 0x72, - 0x65, 0x6e, 0x7d, 0x3c, 0x2f, 0x73, 0x70, 0x61, 0x6e, 0x3e, 0x0a, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x24, 0x7b, 0x69, 0x73, 0x4f, - 0x70, 0x65, 0x6e, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x20, 0x26, 0x26, - 0x20, 0x68, 0x74, 0x6d, 0x6c, 0x60, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x24, 0x7b, 0x50, 0x6f, 0x72, 0x74, - 0x61, 0x6c, 0x7d, 0x20, 0x69, 0x6e, 0x74, 0x6f, 0x3d, 0x22, 0x23, 0x70, - 0x6f, 0x72, 0x74, 0x61, 0x6c, 0x22, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x64, 0x69, 0x76, - 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x72, 0x65, 0x66, 0x3d, 0x24, 0x7b, 0x70, 0x6f, 0x70, - 0x6f, 0x76, 0x65, 0x72, 0x52, 0x65, 0x66, 0x7d, 0x0a, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, - 0x6c, 0x61, 0x73, 0x73, 0x3d, 0x22, 0x70, 0x6f, 0x70, 0x6f, 0x76, 0x65, - 0x72, 0x2d, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x22, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x64, 0x69, 0x76, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x73, 0x74, 0x79, 0x6c, 0x65, 0x3d, 0x24, 0x7b, 0x7b, 0x0a, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x74, - 0x6f, 0x70, 0x3a, 0x20, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x69, 0x6f, 0x6e, - 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x2e, 0x74, 0x6f, 0x70, 0x2c, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x6b, 0x65, 0x79, 0x3d, 0x24, 0x7b, + 0x69, 0x6e, 0x64, 0x65, 0x78, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x74, 0x69, 0x74, 0x6c, 0x65, 0x3d, 0x24, 0x7b, 0x60, 0x70, 0x72, + 0x6f, 0x62, 0x3a, 0x20, 0x24, 0x7b, 0x70, 0x2e, 0x70, 0x72, 0x6f, 0x62, + 0x7d, 0x60, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x73, 0x74, + 0x79, 0x6c, 0x65, 0x3d, 0x24, 0x7b, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x70, 0x61, + 0x64, 0x64, 0x69, 0x6e, 0x67, 0x3a, 0x20, 0x27, 0x30, 0x2e, 0x33, 0x65, + 0x6d, 0x27, 0x2c, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x62, 0x61, 0x63, 0x6b, 0x67, 0x72, + 0x6f, 0x75, 0x6e, 0x64, 0x43, 0x6f, 0x6c, 0x6f, 0x72, 0x3a, 0x20, 0x70, + 0x2e, 0x74, 0x6f, 0x6b, 0x5f, 0x73, 0x74, 0x72, 0x20, 0x3d, 0x3d, 0x3d, + 0x20, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x20, 0x3f, 0x20, 0x70, + 0x72, 0x6f, 0x62, 0x43, 0x6f, 0x6c, 0x6f, 0x72, 0x28, 0x70, 0x2e, 0x70, + 0x72, 0x6f, 0x62, 0x29, 0x20, 0x3a, 0x20, 0x27, 0x74, 0x72, 0x61, 0x6e, + 0x73, 0x70, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x27, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x6c, 0x65, 0x66, 0x74, 0x3a, 0x20, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x69, - 0x6f, 0x6e, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x2e, 0x6c, 0x65, 0x66, - 0x74, 0x2c, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x7d, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x24, 0x7b, 0x70, 0x72, - 0x6f, 0x70, 0x73, 0x2e, 0x70, 0x6f, 0x70, 0x6f, 0x76, 0x65, 0x72, 0x43, - 0x68, 0x69, 0x6c, 0x64, 0x72, 0x65, 0x6e, 0x7d, 0x0a, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x2f, 0x64, - 0x69, 0x76, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x3c, 0x2f, 0x24, 0x7b, 0x50, 0x6f, 0x72, 0x74, 0x61, 0x6c, - 0x7d, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x60, - 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x60, 0x3b, 0x0a, 0x20, - 0x20, 0x20, 0x20, 0x7d, 0x3b, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x2f, - 0x2f, 0x20, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x3a, 0x20, 0x70, 0x72, - 0x65, 0x61, 0x63, 0x74, 0x2d, 0x70, 0x6f, 0x72, 0x74, 0x61, 0x6c, 0x20, - 0x28, 0x68, 0x74, 0x74, 0x70, 0x73, 0x3a, 0x2f, 0x2f, 0x67, 0x69, 0x74, - 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x64, 0x65, 0x76, 0x65, - 0x6c, 0x6f, 0x70, 0x69, 0x74, 0x2f, 0x70, 0x72, 0x65, 0x61, 0x63, 0x74, - 0x2d, 0x70, 0x6f, 0x72, 0x74, 0x61, 0x6c, 0x2f, 0x62, 0x6c, 0x6f, 0x62, - 0x2f, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x2f, 0x73, 0x72, 0x63, 0x2f, - 0x70, 0x72, 0x65, 0x61, 0x63, 0x74, 0x2d, 0x70, 0x6f, 0x72, 0x74, 0x61, - 0x6c, 0x2e, 0x6a, 0x73, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x2f, 0x2a, - 0x2a, 0x20, 0x52, 0x65, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x20, 0x72, - 0x65, 0x6e, 0x64, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x20, 0x6f, 0x66, 0x20, - 0x64, 0x65, 0x73, 0x63, 0x65, 0x6e, 0x64, 0x61, 0x6e, 0x74, 0x73, 0x20, - 0x69, 0x6e, 0x74, 0x6f, 0x20, 0x74, 0x68, 0x65, 0x20, 0x67, 0x69, 0x76, - 0x65, 0x6e, 0x20, 0x43, 0x53, 0x53, 0x20, 0x73, 0x65, 0x6c, 0x65, 0x63, - 0x74, 0x6f, 0x72, 0x20, 0x2a, 0x2f, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x63, - 0x6c, 0x61, 0x73, 0x73, 0x20, 0x50, 0x6f, 0x72, 0x74, 0x61, 0x6c, 0x20, - 0x65, 0x78, 0x74, 0x65, 0x6e, 0x64, 0x73, 0x20, 0x43, 0x6f, 0x6d, 0x70, - 0x6f, 0x6e, 0x65, 0x6e, 0x74, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x63, 0x6f, 0x6d, 0x70, 0x6f, 0x6e, 0x65, 0x6e, 0x74, 0x44, - 0x69, 0x64, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x28, 0x70, 0x72, 0x6f, - 0x70, 0x73, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x66, 0x6f, 0x72, 0x20, 0x28, 0x6c, 0x65, 0x74, 0x20, 0x69, - 0x20, 0x69, 0x6e, 0x20, 0x70, 0x72, 0x6f, 0x70, 0x73, 0x29, 0x20, 0x7b, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x3c, 0x73, 0x70, 0x61, 0x6e, 0x3e, 0x24, 0x7b, 0x70, 0x2e, 0x74, 0x6f, + 0x6b, 0x5f, 0x73, 0x74, 0x72, 0x7d, 0x3a, 0x20, 0x3c, 0x2f, 0x73, 0x70, + 0x61, 0x6e, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x73, + 0x70, 0x61, 0x6e, 0x3e, 0x24, 0x7b, 0x4d, 0x61, 0x74, 0x68, 0x2e, 0x66, + 0x6c, 0x6f, 0x6f, 0x72, 0x28, 0x70, 0x2e, 0x70, 0x72, 0x6f, 0x62, 0x20, + 0x2a, 0x20, 0x31, 0x30, 0x30, 0x29, 0x7d, 0x25, 0x3c, 0x2f, 0x73, 0x70, + 0x61, 0x6e, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x2f, 0x64, 0x69, + 0x76, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x60, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x7d, 0x29, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x2f, 0x64, 0x69, 0x76, 0x3e, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x60, 0x0a, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, + 0x6e, 0x20, 0x68, 0x74, 0x6d, 0x6c, 0x60, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x24, 0x7b, 0x50, 0x6f, 0x70, + 0x6f, 0x76, 0x65, 0x72, 0x7d, 0x20, 0x73, 0x74, 0x79, 0x6c, 0x65, 0x3d, + 0x24, 0x7b, 0x7b, 0x20, 0x62, 0x61, 0x63, 0x6b, 0x67, 0x72, 0x6f, 0x75, + 0x6e, 0x64, 0x43, 0x6f, 0x6c, 0x6f, 0x72, 0x3a, 0x20, 0x70, 0x43, 0x6f, + 0x6c, 0x6f, 0x72, 0x20, 0x7d, 0x7d, 0x20, 0x70, 0x6f, 0x70, 0x6f, 0x76, + 0x65, 0x72, 0x43, 0x68, 0x69, 0x6c, 0x64, 0x72, 0x65, 0x6e, 0x3d, 0x24, + 0x7b, 0x70, 0x6f, 0x70, 0x6f, 0x76, 0x65, 0x72, 0x43, 0x68, 0x69, 0x6c, + 0x64, 0x72, 0x65, 0x6e, 0x7d, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x24, 0x7b, 0x6d, 0x73, 0x67, + 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x2e, 0x6d, 0x61, 0x74, + 0x63, 0x68, 0x28, 0x2f, 0x5c, 0x6e, 0x2f, 0x67, 0x69, 0x6d, 0x29, 0x20, + 0x3f, 0x20, 0x68, 0x74, 0x6d, 0x6c, 0x60, 0x3c, 0x62, 0x72, 0x20, 0x2f, + 0x3e, 0x60, 0x20, 0x3a, 0x20, 0x6d, 0x73, 0x67, 0x2e, 0x63, 0x6f, 0x6e, + 0x74, 0x65, 0x6e, 0x74, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x3c, 0x2f, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x60, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x7d, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x2f, 0x2f, 0x20, 0x70, 0x6f, 0x6f, 0x72, 0x20, 0x6d, + 0x61, 0x6e, 0x73, 0x20, 0x6d, 0x61, 0x72, 0x6b, 0x64, 0x6f, 0x77, 0x6e, + 0x20, 0x72, 0x65, 0x70, 0x6c, 0x61, 0x63, 0x65, 0x6d, 0x65, 0x6e, 0x74, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x4d, + 0x61, 0x72, 0x6b, 0x64, 0x6f, 0x77, 0x6e, 0x69, 0x73, 0x68, 0x20, 0x3d, + 0x20, 0x28, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x29, 0x20, 0x3d, 0x3e, + 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, + 0x73, 0x74, 0x20, 0x6d, 0x64, 0x20, 0x3d, 0x20, 0x70, 0x61, 0x72, 0x61, + 0x6d, 0x73, 0x2e, 0x74, 0x65, 0x78, 0x74, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x2e, 0x72, 0x65, 0x70, 0x6c, 0x61, 0x63, 0x65, + 0x28, 0x2f, 0x26, 0x2f, 0x67, 0x2c, 0x20, 0x27, 0x26, 0x61, 0x6d, 0x70, + 0x3b, 0x27, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x2e, 0x72, 0x65, 0x70, 0x6c, 0x61, 0x63, 0x65, 0x28, 0x2f, 0x3c, 0x2f, + 0x67, 0x2c, 0x20, 0x27, 0x26, 0x6c, 0x74, 0x3b, 0x27, 0x29, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x2e, 0x72, 0x65, 0x70, 0x6c, + 0x61, 0x63, 0x65, 0x28, 0x2f, 0x3e, 0x2f, 0x67, 0x2c, 0x20, 0x27, 0x26, + 0x67, 0x74, 0x3b, 0x27, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x2e, 0x72, 0x65, 0x70, 0x6c, 0x61, 0x63, 0x65, 0x28, 0x2f, + 0x5e, 0x23, 0x7b, 0x31, 0x2c, 0x36, 0x7d, 0x20, 0x28, 0x2e, 0x2a, 0x29, + 0x24, 0x2f, 0x67, 0x69, 0x6d, 0x2c, 0x20, 0x27, 0x3c, 0x68, 0x33, 0x3e, + 0x24, 0x31, 0x3c, 0x2f, 0x68, 0x33, 0x3e, 0x27, 0x29, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x2e, 0x72, 0x65, 0x70, 0x6c, 0x61, + 0x63, 0x65, 0x28, 0x2f, 0x5c, 0x2a, 0x5c, 0x2a, 0x28, 0x2e, 0x2a, 0x3f, + 0x29, 0x5c, 0x2a, 0x5c, 0x2a, 0x2f, 0x67, 0x2c, 0x20, 0x27, 0x3c, 0x73, + 0x74, 0x72, 0x6f, 0x6e, 0x67, 0x3e, 0x24, 0x31, 0x3c, 0x2f, 0x73, 0x74, + 0x72, 0x6f, 0x6e, 0x67, 0x3e, 0x27, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x2e, 0x72, 0x65, 0x70, 0x6c, 0x61, 0x63, 0x65, + 0x28, 0x2f, 0x5f, 0x5f, 0x28, 0x2e, 0x2a, 0x3f, 0x29, 0x5f, 0x5f, 0x2f, + 0x67, 0x2c, 0x20, 0x27, 0x3c, 0x73, 0x74, 0x72, 0x6f, 0x6e, 0x67, 0x3e, + 0x24, 0x31, 0x3c, 0x2f, 0x73, 0x74, 0x72, 0x6f, 0x6e, 0x67, 0x3e, 0x27, + 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x2e, 0x72, + 0x65, 0x70, 0x6c, 0x61, 0x63, 0x65, 0x28, 0x2f, 0x5c, 0x2a, 0x28, 0x2e, + 0x2a, 0x3f, 0x29, 0x5c, 0x2a, 0x2f, 0x67, 0x2c, 0x20, 0x27, 0x3c, 0x65, + 0x6d, 0x3e, 0x24, 0x31, 0x3c, 0x2f, 0x65, 0x6d, 0x3e, 0x27, 0x29, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x2e, 0x72, 0x65, 0x70, + 0x6c, 0x61, 0x63, 0x65, 0x28, 0x2f, 0x5f, 0x28, 0x2e, 0x2a, 0x3f, 0x29, + 0x5f, 0x2f, 0x67, 0x2c, 0x20, 0x27, 0x3c, 0x65, 0x6d, 0x3e, 0x24, 0x31, + 0x3c, 0x2f, 0x65, 0x6d, 0x3e, 0x27, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x2e, 0x72, 0x65, 0x70, 0x6c, 0x61, 0x63, 0x65, + 0x28, 0x2f, 0x60, 0x60, 0x60, 0x2e, 0x2a, 0x3f, 0x5c, 0x6e, 0x28, 0x5b, + 0x5c, 0x73, 0x5c, 0x53, 0x5d, 0x2a, 0x3f, 0x29, 0x60, 0x60, 0x60, 0x2f, + 0x67, 0x2c, 0x20, 0x27, 0x3c, 0x70, 0x72, 0x65, 0x3e, 0x3c, 0x63, 0x6f, + 0x64, 0x65, 0x3e, 0x24, 0x31, 0x3c, 0x2f, 0x63, 0x6f, 0x64, 0x65, 0x3e, + 0x3c, 0x2f, 0x70, 0x72, 0x65, 0x3e, 0x27, 0x29, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x2e, 0x72, 0x65, 0x70, 0x6c, 0x61, 0x63, + 0x65, 0x28, 0x2f, 0x60, 0x28, 0x2e, 0x2a, 0x3f, 0x29, 0x60, 0x2f, 0x67, + 0x2c, 0x20, 0x27, 0x3c, 0x63, 0x6f, 0x64, 0x65, 0x3e, 0x24, 0x31, 0x3c, + 0x2f, 0x63, 0x6f, 0x64, 0x65, 0x3e, 0x27, 0x29, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x2e, 0x72, 0x65, 0x70, 0x6c, 0x61, 0x63, + 0x65, 0x28, 0x2f, 0x5c, 0x6e, 0x2f, 0x67, 0x69, 0x6d, 0x2c, 0x20, 0x27, + 0x3c, 0x62, 0x72, 0x20, 0x2f, 0x3e, 0x27, 0x29, 0x3b, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x68, + 0x74, 0x6d, 0x6c, 0x60, 0x3c, 0x73, 0x70, 0x61, 0x6e, 0x20, 0x64, 0x61, + 0x6e, 0x67, 0x65, 0x72, 0x6f, 0x75, 0x73, 0x6c, 0x79, 0x53, 0x65, 0x74, + 0x49, 0x6e, 0x6e, 0x65, 0x72, 0x48, 0x54, 0x4d, 0x4c, 0x3d, 0x24, 0x7b, + 0x7b, 0x20, 0x5f, 0x5f, 0x68, 0x74, 0x6d, 0x6c, 0x3a, 0x20, 0x6d, 0x64, + 0x20, 0x7d, 0x7d, 0x20, 0x2f, 0x3e, 0x60, 0x3b, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x7d, 0x3b, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, + 0x73, 0x74, 0x20, 0x4d, 0x6f, 0x64, 0x65, 0x6c, 0x47, 0x65, 0x6e, 0x65, + 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, 0x6f, 0x20, 0x3d, + 0x20, 0x28, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x29, 0x20, 0x3d, 0x3e, + 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, + 0x28, 0x21, 0x6c, 0x6c, 0x61, 0x6d, 0x61, 0x53, 0x74, 0x61, 0x74, 0x73, + 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, + 0x20, 0x68, 0x74, 0x6d, 0x6c, 0x60, 0x3c, 0x73, 0x70, 0x61, 0x6e, 0x2f, + 0x3e, 0x60, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, + 0x68, 0x74, 0x6d, 0x6c, 0x60, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x3c, 0x73, 0x70, 0x61, 0x6e, 0x3e, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x24, 0x7b, 0x6c, 0x6c, 0x61, + 0x6d, 0x61, 0x53, 0x74, 0x61, 0x74, 0x73, 0x2e, 0x76, 0x61, 0x6c, 0x75, + 0x65, 0x2e, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x73, 0x5f, 0x70, 0x72, 0x65, + 0x64, 0x69, 0x63, 0x74, 0x65, 0x64, 0x7d, 0x20, 0x70, 0x72, 0x65, 0x64, + 0x69, 0x63, 0x74, 0x65, 0x64, 0x2c, 0x20, 0x24, 0x7b, 0x6c, 0x6c, 0x61, + 0x6d, 0x61, 0x53, 0x74, 0x61, 0x74, 0x73, 0x2e, 0x76, 0x61, 0x6c, 0x75, + 0x65, 0x2e, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x73, 0x5f, 0x63, 0x61, 0x63, + 0x68, 0x65, 0x64, 0x7d, 0x20, 0x63, 0x61, 0x63, 0x68, 0x65, 0x64, 0x2c, + 0x20, 0x24, 0x7b, 0x6c, 0x6c, 0x61, 0x6d, 0x61, 0x53, 0x74, 0x61, 0x74, + 0x73, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x2e, 0x74, 0x69, 0x6d, 0x69, + 0x6e, 0x67, 0x73, 0x2e, 0x70, 0x72, 0x65, 0x64, 0x69, 0x63, 0x74, 0x65, + 0x64, 0x5f, 0x70, 0x65, 0x72, 0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x5f, + 0x6d, 0x73, 0x2e, 0x74, 0x6f, 0x46, 0x69, 0x78, 0x65, 0x64, 0x28, 0x29, + 0x7d, 0x6d, 0x73, 0x20, 0x70, 0x65, 0x72, 0x20, 0x74, 0x6f, 0x6b, 0x65, + 0x6e, 0x2c, 0x20, 0x24, 0x7b, 0x6c, 0x6c, 0x61, 0x6d, 0x61, 0x53, 0x74, + 0x61, 0x74, 0x73, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x2e, 0x74, 0x69, + 0x6d, 0x69, 0x6e, 0x67, 0x73, 0x2e, 0x70, 0x72, 0x65, 0x64, 0x69, 0x63, + 0x74, 0x65, 0x64, 0x5f, 0x70, 0x65, 0x72, 0x5f, 0x73, 0x65, 0x63, 0x6f, + 0x6e, 0x64, 0x2e, 0x74, 0x6f, 0x46, 0x69, 0x78, 0x65, 0x64, 0x28, 0x32, + 0x29, 0x7d, 0x20, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x73, 0x20, 0x70, 0x65, + 0x72, 0x20, 0x73, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x2f, 0x73, 0x70, 0x61, 0x6e, 0x3e, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x60, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x7d, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x2f, 0x2f, 0x20, 0x73, + 0x69, 0x6d, 0x70, 0x6c, 0x65, 0x20, 0x70, 0x6f, 0x70, 0x6f, 0x76, 0x65, + 0x72, 0x20, 0x69, 0x6d, 0x70, 0x6c, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x63, + 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x50, 0x6f, 0x70, 0x6f, 0x76, 0x65, 0x72, + 0x20, 0x3d, 0x20, 0x28, 0x70, 0x72, 0x6f, 0x70, 0x73, 0x29, 0x20, 0x3d, + 0x3e, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, + 0x6e, 0x73, 0x74, 0x20, 0x69, 0x73, 0x4f, 0x70, 0x65, 0x6e, 0x20, 0x3d, + 0x20, 0x75, 0x73, 0x65, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x28, 0x66, + 0x61, 0x6c, 0x73, 0x65, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x70, 0x6f, 0x73, 0x69, 0x74, + 0x69, 0x6f, 0x6e, 0x20, 0x3d, 0x20, 0x75, 0x73, 0x65, 0x53, 0x69, 0x67, + 0x6e, 0x61, 0x6c, 0x28, 0x7b, 0x20, 0x74, 0x6f, 0x70, 0x3a, 0x20, 0x27, + 0x30, 0x70, 0x78, 0x27, 0x2c, 0x20, 0x6c, 0x65, 0x66, 0x74, 0x3a, 0x20, + 0x27, 0x30, 0x70, 0x78, 0x27, 0x20, 0x7d, 0x29, 0x3b, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x62, 0x75, + 0x74, 0x74, 0x6f, 0x6e, 0x52, 0x65, 0x66, 0x20, 0x3d, 0x20, 0x75, 0x73, + 0x65, 0x52, 0x65, 0x66, 0x28, 0x6e, 0x75, 0x6c, 0x6c, 0x29, 0x3b, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, + 0x70, 0x6f, 0x70, 0x6f, 0x76, 0x65, 0x72, 0x52, 0x65, 0x66, 0x20, 0x3d, + 0x20, 0x75, 0x73, 0x65, 0x52, 0x65, 0x66, 0x28, 0x6e, 0x75, 0x6c, 0x6c, + 0x29, 0x3b, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, + 0x6e, 0x73, 0x74, 0x20, 0x74, 0x6f, 0x67, 0x67, 0x6c, 0x65, 0x50, 0x6f, + 0x70, 0x6f, 0x76, 0x65, 0x72, 0x20, 0x3d, 0x20, 0x28, 0x29, 0x20, 0x3d, + 0x3e, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x69, 0x66, 0x20, 0x28, 0x62, 0x75, 0x74, 0x74, 0x6f, 0x6e, 0x52, 0x65, + 0x66, 0x2e, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x74, 0x29, 0x20, 0x7b, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, + 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x72, 0x65, 0x63, 0x74, 0x20, 0x3d, 0x20, + 0x62, 0x75, 0x74, 0x74, 0x6f, 0x6e, 0x52, 0x65, 0x66, 0x2e, 0x63, 0x75, + 0x72, 0x72, 0x65, 0x6e, 0x74, 0x2e, 0x67, 0x65, 0x74, 0x42, 0x6f, 0x75, + 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x52, + 0x65, 0x63, 0x74, 0x28, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x69, 0x6f, + 0x6e, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x20, 0x3d, 0x20, 0x7b, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x74, 0x6f, 0x70, 0x3a, 0x20, 0x60, 0x24, 0x7b, 0x72, 0x65, 0x63, 0x74, + 0x2e, 0x62, 0x6f, 0x74, 0x74, 0x6f, 0x6d, 0x20, 0x2b, 0x20, 0x77, 0x69, + 0x6e, 0x64, 0x6f, 0x77, 0x2e, 0x73, 0x63, 0x72, 0x6f, 0x6c, 0x6c, 0x59, + 0x7d, 0x70, 0x78, 0x60, 0x2c, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x6c, 0x65, 0x66, 0x74, 0x3a, 0x20, + 0x60, 0x24, 0x7b, 0x72, 0x65, 0x63, 0x74, 0x2e, 0x6c, 0x65, 0x66, 0x74, + 0x20, 0x2b, 0x20, 0x77, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x2e, 0x73, 0x63, + 0x72, 0x6f, 0x6c, 0x6c, 0x58, 0x7d, 0x70, 0x78, 0x60, 0x2c, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x3b, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x73, 0x4f, 0x70, 0x65, 0x6e, + 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x20, 0x3d, 0x20, 0x21, 0x69, 0x73, + 0x4f, 0x70, 0x65, 0x6e, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3b, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x3b, 0x0a, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x68, 0x61, + 0x6e, 0x64, 0x6c, 0x65, 0x43, 0x6c, 0x69, 0x63, 0x6b, 0x4f, 0x75, 0x74, + 0x73, 0x69, 0x64, 0x65, 0x20, 0x3d, 0x20, 0x28, 0x65, 0x76, 0x65, 0x6e, + 0x74, 0x29, 0x20, 0x3d, 0x3e, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, 0x70, 0x6f, 0x70, 0x6f, + 0x76, 0x65, 0x72, 0x52, 0x65, 0x66, 0x2e, 0x63, 0x75, 0x72, 0x72, 0x65, + 0x6e, 0x74, 0x20, 0x26, 0x26, 0x20, 0x21, 0x70, 0x6f, 0x70, 0x6f, 0x76, + 0x65, 0x72, 0x52, 0x65, 0x66, 0x2e, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, + 0x74, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x73, 0x28, 0x65, + 0x76, 0x65, 0x6e, 0x74, 0x2e, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x29, + 0x20, 0x26, 0x26, 0x20, 0x21, 0x62, 0x75, 0x74, 0x74, 0x6f, 0x6e, 0x52, + 0x65, 0x66, 0x2e, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x74, 0x2e, 0x63, + 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x73, 0x28, 0x65, 0x76, 0x65, 0x6e, + 0x74, 0x2e, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x29, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, - 0x66, 0x20, 0x28, 0x70, 0x72, 0x6f, 0x70, 0x73, 0x5b, 0x69, 0x5d, 0x20, - 0x21, 0x3d, 0x3d, 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x70, 0x72, 0x6f, - 0x70, 0x73, 0x5b, 0x69, 0x5d, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, 0x74, - 0x75, 0x72, 0x6e, 0x20, 0x73, 0x65, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x6f, - 0x75, 0x74, 0x28, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x72, 0x65, 0x6e, 0x64, - 0x65, 0x72, 0x4c, 0x61, 0x79, 0x65, 0x72, 0x29, 0x3b, 0x0a, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x7d, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, - 0x6f, 0x6d, 0x70, 0x6f, 0x6e, 0x65, 0x6e, 0x74, 0x44, 0x69, 0x64, 0x4d, - 0x6f, 0x75, 0x6e, 0x74, 0x28, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x69, 0x73, - 0x4d, 0x6f, 0x75, 0x6e, 0x74, 0x65, 0x64, 0x20, 0x3d, 0x20, 0x74, 0x72, - 0x75, 0x65, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x74, 0x68, 0x69, 0x73, 0x2e, 0x72, 0x65, 0x6e, 0x64, 0x65, 0x72, 0x4c, - 0x61, 0x79, 0x65, 0x72, 0x20, 0x3d, 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, - 0x72, 0x65, 0x6e, 0x64, 0x65, 0x72, 0x4c, 0x61, 0x79, 0x65, 0x72, 0x2e, - 0x62, 0x69, 0x6e, 0x64, 0x28, 0x74, 0x68, 0x69, 0x73, 0x29, 0x3b, 0x0a, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x74, 0x68, 0x69, 0x73, - 0x2e, 0x72, 0x65, 0x6e, 0x64, 0x65, 0x72, 0x4c, 0x61, 0x79, 0x65, 0x72, - 0x28, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, - 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6d, 0x70, 0x6f, - 0x6e, 0x65, 0x6e, 0x74, 0x57, 0x69, 0x6c, 0x6c, 0x55, 0x6e, 0x6d, 0x6f, - 0x75, 0x6e, 0x74, 0x28, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x72, 0x65, 0x6e, - 0x64, 0x65, 0x72, 0x4c, 0x61, 0x79, 0x65, 0x72, 0x28, 0x66, 0x61, 0x6c, - 0x73, 0x65, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x69, 0x73, 0x4d, 0x6f, 0x75, 0x6e, - 0x74, 0x65, 0x64, 0x20, 0x3d, 0x20, 0x66, 0x61, 0x6c, 0x73, 0x65, 0x3b, - 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, - 0x28, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, - 0x20, 0x26, 0x26, 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x72, 0x65, 0x6d, - 0x6f, 0x74, 0x65, 0x2e, 0x70, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x4e, 0x6f, - 0x64, 0x65, 0x29, 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x72, 0x65, 0x6d, - 0x6f, 0x74, 0x65, 0x2e, 0x70, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x4e, 0x6f, - 0x64, 0x65, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x68, 0x69, - 0x6c, 0x64, 0x28, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x72, 0x65, 0x6d, 0x6f, - 0x74, 0x65, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, - 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x66, 0x69, 0x6e, 0x64, - 0x4e, 0x6f, 0x64, 0x65, 0x28, 0x6e, 0x6f, 0x64, 0x65, 0x29, 0x20, 0x7b, + 0x73, 0x4f, 0x70, 0x65, 0x6e, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x20, + 0x3d, 0x20, 0x66, 0x61, 0x6c, 0x73, 0x65, 0x3b, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x7d, 0x3b, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x75, + 0x73, 0x65, 0x45, 0x66, 0x66, 0x65, 0x63, 0x74, 0x28, 0x28, 0x29, 0x20, + 0x3d, 0x3e, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x64, 0x6f, 0x63, 0x75, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x61, 0x64, + 0x64, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x4c, 0x69, 0x73, 0x74, 0x65, 0x6e, + 0x65, 0x72, 0x28, 0x27, 0x6d, 0x6f, 0x75, 0x73, 0x65, 0x64, 0x6f, 0x77, + 0x6e, 0x27, 0x2c, 0x20, 0x68, 0x61, 0x6e, 0x64, 0x6c, 0x65, 0x43, 0x6c, + 0x69, 0x63, 0x6b, 0x4f, 0x75, 0x74, 0x73, 0x69, 0x64, 0x65, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, 0x74, - 0x75, 0x72, 0x6e, 0x20, 0x74, 0x79, 0x70, 0x65, 0x6f, 0x66, 0x20, 0x6e, - 0x6f, 0x64, 0x65, 0x20, 0x3d, 0x3d, 0x3d, 0x20, 0x27, 0x73, 0x74, 0x72, - 0x69, 0x6e, 0x67, 0x27, 0x20, 0x3f, 0x20, 0x64, 0x6f, 0x63, 0x75, 0x6d, - 0x65, 0x6e, 0x74, 0x2e, 0x71, 0x75, 0x65, 0x72, 0x79, 0x53, 0x65, 0x6c, - 0x65, 0x63, 0x74, 0x6f, 0x72, 0x28, 0x6e, 0x6f, 0x64, 0x65, 0x29, 0x20, - 0x3a, 0x20, 0x6e, 0x6f, 0x64, 0x65, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x7d, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, - 0x65, 0x6e, 0x64, 0x65, 0x72, 0x4c, 0x61, 0x79, 0x65, 0x72, 0x28, 0x73, - 0x68, 0x6f, 0x77, 0x20, 0x3d, 0x20, 0x74, 0x72, 0x75, 0x65, 0x29, 0x20, - 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, - 0x20, 0x28, 0x21, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x69, 0x73, 0x4d, 0x6f, - 0x75, 0x6e, 0x74, 0x65, 0x64, 0x29, 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, - 0x6e, 0x3b, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x2f, 0x2f, 0x20, 0x63, 0x6c, 0x65, 0x61, 0x6e, 0x20, 0x75, 0x70, 0x20, - 0x6f, 0x6c, 0x64, 0x20, 0x6e, 0x6f, 0x64, 0x65, 0x20, 0x69, 0x66, 0x20, - 0x6d, 0x6f, 0x76, 0x69, 0x6e, 0x67, 0x20, 0x62, 0x61, 0x73, 0x65, 0x73, - 0x3a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, - 0x20, 0x28, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x70, 0x73, - 0x2e, 0x69, 0x6e, 0x74, 0x6f, 0x20, 0x21, 0x3d, 0x3d, 0x20, 0x74, 0x68, - 0x69, 0x73, 0x2e, 0x69, 0x6e, 0x74, 0x6f, 0x50, 0x6f, 0x69, 0x6e, 0x74, - 0x65, 0x72, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x69, 0x6e, 0x74, - 0x6f, 0x50, 0x6f, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x20, 0x3d, 0x20, 0x74, - 0x68, 0x69, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x70, 0x73, 0x2e, 0x69, 0x6e, - 0x74, 0x6f, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x69, - 0x6e, 0x74, 0x6f, 0x20, 0x26, 0x26, 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, - 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x74, 0x68, - 0x69, 0x73, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x20, 0x3d, 0x20, - 0x72, 0x65, 0x6e, 0x64, 0x65, 0x72, 0x28, 0x68, 0x74, 0x6d, 0x6c, 0x60, - 0x3c, 0x24, 0x7b, 0x50, 0x6f, 0x72, 0x74, 0x61, 0x6c, 0x50, 0x72, 0x6f, - 0x78, 0x79, 0x7d, 0x20, 0x2f, 0x3e, 0x60, 0x2c, 0x20, 0x74, 0x68, 0x69, - 0x73, 0x2e, 0x69, 0x6e, 0x74, 0x6f, 0x2c, 0x20, 0x74, 0x68, 0x69, 0x73, - 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x29, 0x3b, 0x0a, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x74, 0x68, 0x69, 0x73, - 0x2e, 0x69, 0x6e, 0x74, 0x6f, 0x20, 0x3d, 0x20, 0x74, 0x68, 0x69, 0x73, - 0x2e, 0x66, 0x69, 0x6e, 0x64, 0x4e, 0x6f, 0x64, 0x65, 0x28, 0x74, 0x68, - 0x69, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x70, 0x73, 0x2e, 0x69, 0x6e, 0x74, - 0x6f, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x7d, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x74, - 0x68, 0x69, 0x73, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x20, 0x3d, - 0x20, 0x72, 0x65, 0x6e, 0x64, 0x65, 0x72, 0x28, 0x68, 0x74, 0x6d, 0x6c, - 0x60, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x3c, 0x24, 0x7b, 0x50, 0x6f, 0x72, 0x74, 0x61, 0x6c, 0x50, 0x72, 0x6f, - 0x78, 0x79, 0x7d, 0x20, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x3d, - 0x24, 0x7b, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x65, - 0x78, 0x74, 0x7d, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x24, 0x7b, 0x73, 0x68, 0x6f, 0x77, 0x20, - 0x26, 0x26, 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x70, - 0x73, 0x2e, 0x63, 0x68, 0x69, 0x6c, 0x64, 0x72, 0x65, 0x6e, 0x20, 0x7c, - 0x7c, 0x20, 0x6e, 0x75, 0x6c, 0x6c, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x2f, 0x24, 0x7b, 0x50, 0x6f, - 0x72, 0x74, 0x61, 0x6c, 0x50, 0x72, 0x6f, 0x78, 0x79, 0x7d, 0x3e, 0x0a, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x60, 0x2c, 0x20, 0x74, - 0x68, 0x69, 0x73, 0x2e, 0x69, 0x6e, 0x74, 0x6f, 0x2c, 0x20, 0x74, 0x68, - 0x69, 0x73, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x29, 0x3b, 0x0a, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x0a, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x72, 0x65, 0x6e, 0x64, 0x65, 0x72, 0x28, 0x29, 0x20, - 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, - 0x74, 0x75, 0x72, 0x6e, 0x20, 0x6e, 0x75, 0x6c, 0x6c, 0x3b, 0x0a, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x7d, - 0x0a, 0x20, 0x20, 0x20, 0x20, 0x2f, 0x2f, 0x20, 0x68, 0x69, 0x67, 0x68, - 0x2d, 0x6f, 0x72, 0x64, 0x65, 0x72, 0x20, 0x63, 0x6f, 0x6d, 0x70, 0x6f, - 0x6e, 0x65, 0x6e, 0x74, 0x20, 0x74, 0x68, 0x61, 0x74, 0x20, 0x72, 0x65, - 0x6e, 0x64, 0x65, 0x72, 0x73, 0x20, 0x69, 0x74, 0x73, 0x20, 0x66, 0x69, - 0x72, 0x73, 0x74, 0x20, 0x63, 0x68, 0x69, 0x6c, 0x64, 0x20, 0x69, 0x66, - 0x20, 0x69, 0x74, 0x20, 0x65, 0x78, 0x69, 0x73, 0x74, 0x73, 0x2e, 0x0a, - 0x20, 0x20, 0x20, 0x20, 0x2f, 0x2f, 0x20, 0x75, 0x73, 0x65, 0x64, 0x20, - 0x61, 0x73, 0x20, 0x61, 0x20, 0x63, 0x6f, 0x6e, 0x64, 0x69, 0x74, 0x69, - 0x6f, 0x6e, 0x61, 0x6c, 0x20, 0x72, 0x65, 0x6e, 0x64, 0x65, 0x72, 0x69, - 0x6e, 0x67, 0x20, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x2e, 0x0a, 0x20, 0x20, - 0x20, 0x20, 0x63, 0x6c, 0x61, 0x73, 0x73, 0x20, 0x50, 0x6f, 0x72, 0x74, - 0x61, 0x6c, 0x50, 0x72, 0x6f, 0x78, 0x79, 0x20, 0x65, 0x78, 0x74, 0x65, - 0x6e, 0x64, 0x73, 0x20, 0x43, 0x6f, 0x6d, 0x70, 0x6f, 0x6e, 0x65, 0x6e, - 0x74, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x67, 0x65, - 0x74, 0x43, 0x68, 0x69, 0x6c, 0x64, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, - 0x74, 0x28, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x74, 0x68, 0x69, - 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x70, 0x73, 0x2e, 0x63, 0x6f, 0x6e, 0x74, - 0x65, 0x78, 0x74, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, - 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, 0x6e, 0x64, 0x65, - 0x72, 0x28, 0x7b, 0x20, 0x63, 0x68, 0x69, 0x6c, 0x64, 0x72, 0x65, 0x6e, - 0x20, 0x7d, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x63, 0x68, 0x69, - 0x6c, 0x64, 0x72, 0x65, 0x6e, 0x20, 0x7c, 0x7c, 0x20, 0x6e, 0x75, 0x6c, - 0x6c, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, - 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x66, 0x75, - 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x41, 0x70, 0x70, 0x28, 0x70, - 0x72, 0x6f, 0x70, 0x73, 0x29, 0x20, 0x7b, 0x0a, 0x0a, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x68, 0x74, - 0x6d, 0x6c, 0x60, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x3c, 0x64, 0x69, 0x76, 0x20, 0x63, 0x6c, 0x61, 0x73, 0x73, 0x3d, 0x22, - 0x6d, 0x6f, 0x64, 0x65, 0x2d, 0x24, 0x7b, 0x73, 0x65, 0x73, 0x73, 0x69, - 0x6f, 0x6e, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x2e, 0x74, 0x79, 0x70, - 0x65, 0x7d, 0x22, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x3c, 0x68, 0x65, 0x61, 0x64, 0x65, 0x72, 0x3e, 0x0a, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x3c, 0x68, 0x31, 0x3e, 0x6c, 0x6c, 0x61, 0x6d, 0x61, 0x2e, 0x63, 0x70, - 0x70, 0x3c, 0x2f, 0x68, 0x31, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x2f, 0x68, 0x65, 0x61, 0x64, 0x65, - 0x72, 0x3e, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x3c, 0x6d, 0x61, 0x69, 0x6e, 0x20, 0x69, 0x64, 0x3d, 0x22, - 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x22, 0x3e, 0x0a, 0x20, 0x20, + 0x75, 0x72, 0x6e, 0x20, 0x28, 0x29, 0x20, 0x3d, 0x3e, 0x20, 0x7b, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x64, 0x6f, + 0x63, 0x75, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x76, + 0x65, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x4c, 0x69, 0x73, 0x74, 0x65, 0x6e, + 0x65, 0x72, 0x28, 0x27, 0x6d, 0x6f, 0x75, 0x73, 0x65, 0x64, 0x6f, 0x77, + 0x6e, 0x27, 0x2c, 0x20, 0x68, 0x61, 0x6e, 0x64, 0x6c, 0x65, 0x43, 0x6c, + 0x69, 0x63, 0x6b, 0x4f, 0x75, 0x74, 0x73, 0x69, 0x64, 0x65, 0x29, 0x3b, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x3b, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x2c, 0x20, 0x5b, 0x5d, 0x29, + 0x3b, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, 0x74, + 0x75, 0x72, 0x6e, 0x20, 0x68, 0x74, 0x6d, 0x6c, 0x60, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x73, 0x70, 0x61, 0x6e, 0x20, + 0x73, 0x74, 0x79, 0x6c, 0x65, 0x3d, 0x24, 0x7b, 0x70, 0x72, 0x6f, 0x70, + 0x73, 0x2e, 0x73, 0x74, 0x79, 0x6c, 0x65, 0x7d, 0x20, 0x72, 0x65, 0x66, + 0x3d, 0x24, 0x7b, 0x62, 0x75, 0x74, 0x74, 0x6f, 0x6e, 0x52, 0x65, 0x66, + 0x7d, 0x20, 0x6f, 0x6e, 0x43, 0x6c, 0x69, 0x63, 0x6b, 0x3d, 0x24, 0x7b, + 0x74, 0x6f, 0x67, 0x67, 0x6c, 0x65, 0x50, 0x6f, 0x70, 0x6f, 0x76, 0x65, + 0x72, 0x7d, 0x3e, 0x24, 0x7b, 0x70, 0x72, 0x6f, 0x70, 0x73, 0x2e, 0x63, + 0x68, 0x69, 0x6c, 0x64, 0x72, 0x65, 0x6e, 0x7d, 0x3c, 0x2f, 0x73, 0x70, + 0x61, 0x6e, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x24, 0x7b, 0x69, 0x73, 0x4f, 0x70, 0x65, 0x6e, 0x2e, 0x76, 0x61, 0x6c, + 0x75, 0x65, 0x20, 0x26, 0x26, 0x20, 0x68, 0x74, 0x6d, 0x6c, 0x60, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x24, - 0x7b, 0x63, 0x68, 0x61, 0x74, 0x53, 0x74, 0x61, 0x72, 0x74, 0x65, 0x64, - 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x20, 0x3f, 0x20, 0x43, 0x68, 0x61, - 0x74, 0x4c, 0x6f, 0x67, 0x20, 0x3a, 0x20, 0x43, 0x6f, 0x6e, 0x66, 0x69, - 0x67, 0x46, 0x6f, 0x72, 0x6d, 0x7d, 0x20, 0x2f, 0x3e, 0x0a, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x2f, 0x6d, 0x61, - 0x69, 0x6e, 0x3e, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x3c, 0x73, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, - 0x69, 0x64, 0x3d, 0x22, 0x77, 0x72, 0x69, 0x74, 0x65, 0x22, 0x3e, 0x0a, + 0x7b, 0x50, 0x6f, 0x72, 0x74, 0x61, 0x6c, 0x7d, 0x20, 0x69, 0x6e, 0x74, + 0x6f, 0x3d, 0x22, 0x23, 0x70, 0x6f, 0x72, 0x74, 0x61, 0x6c, 0x22, 0x3e, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x3c, 0x64, 0x69, 0x76, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, 0x66, 0x3d, + 0x24, 0x7b, 0x70, 0x6f, 0x70, 0x6f, 0x76, 0x65, 0x72, 0x52, 0x65, 0x66, + 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x63, 0x6c, 0x61, 0x73, 0x73, 0x3d, 0x22, 0x70, + 0x6f, 0x70, 0x6f, 0x76, 0x65, 0x72, 0x2d, 0x63, 0x6f, 0x6e, 0x74, 0x65, + 0x6e, 0x74, 0x22, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x73, 0x74, 0x79, 0x6c, 0x65, 0x3d, + 0x24, 0x7b, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x74, 0x6f, 0x70, 0x3a, 0x20, 0x70, 0x6f, 0x73, + 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x2e, + 0x74, 0x6f, 0x70, 0x2c, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x6c, 0x65, 0x66, 0x74, 0x3a, 0x20, 0x70, + 0x6f, 0x73, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x61, 0x6c, 0x75, + 0x65, 0x2e, 0x6c, 0x65, 0x66, 0x74, 0x2c, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x7d, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x3c, 0x24, 0x7b, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x2e, 0x76, - 0x61, 0x6c, 0x75, 0x65, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x20, 0x3d, 0x3d, - 0x3d, 0x20, 0x27, 0x63, 0x68, 0x61, 0x74, 0x27, 0x20, 0x3f, 0x20, 0x4d, - 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x20, - 0x3a, 0x20, 0x43, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x69, 0x6f, 0x6e, - 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x73, 0x7d, 0x20, 0x2f, 0x3e, + 0x20, 0x24, 0x7b, 0x70, 0x72, 0x6f, 0x70, 0x73, 0x2e, 0x70, 0x6f, 0x70, + 0x6f, 0x76, 0x65, 0x72, 0x43, 0x68, 0x69, 0x6c, 0x64, 0x72, 0x65, 0x6e, + 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x3c, 0x2f, 0x64, 0x69, 0x76, 0x3e, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x2f, 0x24, 0x7b, 0x50, + 0x6f, 0x72, 0x74, 0x61, 0x6c, 0x7d, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x60, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x60, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x3b, 0x0a, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x2f, 0x2f, 0x20, 0x53, 0x6f, 0x75, 0x72, 0x63, + 0x65, 0x3a, 0x20, 0x70, 0x72, 0x65, 0x61, 0x63, 0x74, 0x2d, 0x70, 0x6f, + 0x72, 0x74, 0x61, 0x6c, 0x20, 0x28, 0x68, 0x74, 0x74, 0x70, 0x73, 0x3a, + 0x2f, 0x2f, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, + 0x2f, 0x64, 0x65, 0x76, 0x65, 0x6c, 0x6f, 0x70, 0x69, 0x74, 0x2f, 0x70, + 0x72, 0x65, 0x61, 0x63, 0x74, 0x2d, 0x70, 0x6f, 0x72, 0x74, 0x61, 0x6c, + 0x2f, 0x62, 0x6c, 0x6f, 0x62, 0x2f, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, + 0x2f, 0x73, 0x72, 0x63, 0x2f, 0x70, 0x72, 0x65, 0x61, 0x63, 0x74, 0x2d, + 0x70, 0x6f, 0x72, 0x74, 0x61, 0x6c, 0x2e, 0x6a, 0x73, 0x29, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x2f, 0x2a, 0x2a, 0x20, 0x52, 0x65, 0x64, 0x69, 0x72, + 0x65, 0x63, 0x74, 0x20, 0x72, 0x65, 0x6e, 0x64, 0x65, 0x72, 0x69, 0x6e, + 0x67, 0x20, 0x6f, 0x66, 0x20, 0x64, 0x65, 0x73, 0x63, 0x65, 0x6e, 0x64, + 0x61, 0x6e, 0x74, 0x73, 0x20, 0x69, 0x6e, 0x74, 0x6f, 0x20, 0x74, 0x68, + 0x65, 0x20, 0x67, 0x69, 0x76, 0x65, 0x6e, 0x20, 0x43, 0x53, 0x53, 0x20, + 0x73, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x20, 0x2a, 0x2f, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x63, 0x6c, 0x61, 0x73, 0x73, 0x20, 0x50, 0x6f, + 0x72, 0x74, 0x61, 0x6c, 0x20, 0x65, 0x78, 0x74, 0x65, 0x6e, 0x64, 0x73, + 0x20, 0x43, 0x6f, 0x6d, 0x70, 0x6f, 0x6e, 0x65, 0x6e, 0x74, 0x20, 0x7b, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6d, 0x70, 0x6f, + 0x6e, 0x65, 0x6e, 0x74, 0x44, 0x69, 0x64, 0x55, 0x70, 0x64, 0x61, 0x74, + 0x65, 0x28, 0x70, 0x72, 0x6f, 0x70, 0x73, 0x29, 0x20, 0x7b, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x66, 0x6f, 0x72, 0x20, 0x28, + 0x6c, 0x65, 0x74, 0x20, 0x69, 0x20, 0x69, 0x6e, 0x20, 0x70, 0x72, 0x6f, + 0x70, 0x73, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, 0x70, 0x72, 0x6f, 0x70, + 0x73, 0x5b, 0x69, 0x5d, 0x20, 0x21, 0x3d, 0x3d, 0x20, 0x74, 0x68, 0x69, + 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x70, 0x73, 0x5b, 0x69, 0x5d, 0x29, 0x20, + 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x73, 0x65, 0x74, + 0x54, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x28, 0x74, 0x68, 0x69, 0x73, + 0x2e, 0x72, 0x65, 0x6e, 0x64, 0x65, 0x72, 0x4c, 0x61, 0x79, 0x65, 0x72, + 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6d, 0x70, 0x6f, 0x6e, 0x65, 0x6e, + 0x74, 0x44, 0x69, 0x64, 0x4d, 0x6f, 0x75, 0x6e, 0x74, 0x28, 0x29, 0x20, + 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x74, 0x68, + 0x69, 0x73, 0x2e, 0x69, 0x73, 0x4d, 0x6f, 0x75, 0x6e, 0x74, 0x65, 0x64, + 0x20, 0x3d, 0x20, 0x74, 0x72, 0x75, 0x65, 0x3b, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x72, 0x65, + 0x6e, 0x64, 0x65, 0x72, 0x4c, 0x61, 0x79, 0x65, 0x72, 0x20, 0x3d, 0x20, + 0x74, 0x68, 0x69, 0x73, 0x2e, 0x72, 0x65, 0x6e, 0x64, 0x65, 0x72, 0x4c, + 0x61, 0x79, 0x65, 0x72, 0x2e, 0x62, 0x69, 0x6e, 0x64, 0x28, 0x74, 0x68, + 0x69, 0x73, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x72, 0x65, 0x6e, 0x64, 0x65, 0x72, + 0x4c, 0x61, 0x79, 0x65, 0x72, 0x28, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x63, 0x6f, 0x6d, 0x70, 0x6f, 0x6e, 0x65, 0x6e, 0x74, 0x57, 0x69, 0x6c, + 0x6c, 0x55, 0x6e, 0x6d, 0x6f, 0x75, 0x6e, 0x74, 0x28, 0x29, 0x20, 0x7b, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x74, 0x68, 0x69, + 0x73, 0x2e, 0x72, 0x65, 0x6e, 0x64, 0x65, 0x72, 0x4c, 0x61, 0x79, 0x65, + 0x72, 0x28, 0x66, 0x61, 0x6c, 0x73, 0x65, 0x29, 0x3b, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x69, + 0x73, 0x4d, 0x6f, 0x75, 0x6e, 0x74, 0x65, 0x64, 0x20, 0x3d, 0x20, 0x66, + 0x61, 0x6c, 0x73, 0x65, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x72, + 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x20, 0x26, 0x26, 0x20, 0x74, 0x68, 0x69, + 0x73, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x2e, 0x70, 0x61, 0x72, + 0x65, 0x6e, 0x74, 0x4e, 0x6f, 0x64, 0x65, 0x29, 0x20, 0x74, 0x68, 0x69, + 0x73, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x2e, 0x70, 0x61, 0x72, + 0x65, 0x6e, 0x74, 0x4e, 0x6f, 0x64, 0x65, 0x2e, 0x72, 0x65, 0x6d, 0x6f, + 0x76, 0x65, 0x43, 0x68, 0x69, 0x6c, 0x64, 0x28, 0x74, 0x68, 0x69, 0x73, + 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x29, 0x3b, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x66, 0x69, 0x6e, 0x64, 0x4e, 0x6f, 0x64, 0x65, 0x28, 0x6e, 0x6f, + 0x64, 0x65, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x74, 0x79, 0x70, + 0x65, 0x6f, 0x66, 0x20, 0x6e, 0x6f, 0x64, 0x65, 0x20, 0x3d, 0x3d, 0x3d, + 0x20, 0x27, 0x73, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x27, 0x20, 0x3f, 0x20, + 0x64, 0x6f, 0x63, 0x75, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x71, 0x75, 0x65, + 0x72, 0x79, 0x53, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x28, 0x6e, + 0x6f, 0x64, 0x65, 0x29, 0x20, 0x3a, 0x20, 0x6e, 0x6f, 0x64, 0x65, 0x3b, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, 0x6e, 0x64, 0x65, 0x72, 0x4c, 0x61, + 0x79, 0x65, 0x72, 0x28, 0x73, 0x68, 0x6f, 0x77, 0x20, 0x3d, 0x20, 0x74, + 0x72, 0x75, 0x65, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, 0x21, 0x74, 0x68, 0x69, 0x73, + 0x2e, 0x69, 0x73, 0x4d, 0x6f, 0x75, 0x6e, 0x74, 0x65, 0x64, 0x29, 0x20, + 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x3b, 0x0a, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x2f, 0x2f, 0x20, 0x63, 0x6c, 0x65, 0x61, + 0x6e, 0x20, 0x75, 0x70, 0x20, 0x6f, 0x6c, 0x64, 0x20, 0x6e, 0x6f, 0x64, + 0x65, 0x20, 0x69, 0x66, 0x20, 0x6d, 0x6f, 0x76, 0x69, 0x6e, 0x67, 0x20, + 0x62, 0x61, 0x73, 0x65, 0x73, 0x3a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, 0x74, 0x68, 0x69, 0x73, 0x2e, + 0x70, 0x72, 0x6f, 0x70, 0x73, 0x2e, 0x69, 0x6e, 0x74, 0x6f, 0x20, 0x21, + 0x3d, 0x3d, 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x69, 0x6e, 0x74, 0x6f, + 0x50, 0x6f, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x29, 0x20, 0x7b, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x74, 0x68, 0x69, + 0x73, 0x2e, 0x69, 0x6e, 0x74, 0x6f, 0x50, 0x6f, 0x69, 0x6e, 0x74, 0x65, + 0x72, 0x20, 0x3d, 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x70, 0x72, 0x6f, + 0x70, 0x73, 0x2e, 0x69, 0x6e, 0x74, 0x6f, 0x3b, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, 0x74, + 0x68, 0x69, 0x73, 0x2e, 0x69, 0x6e, 0x74, 0x6f, 0x20, 0x26, 0x26, 0x20, + 0x74, 0x68, 0x69, 0x73, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x29, + 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x72, 0x65, 0x6d, 0x6f, + 0x74, 0x65, 0x20, 0x3d, 0x20, 0x72, 0x65, 0x6e, 0x64, 0x65, 0x72, 0x28, + 0x68, 0x74, 0x6d, 0x6c, 0x60, 0x3c, 0x24, 0x7b, 0x50, 0x6f, 0x72, 0x74, + 0x61, 0x6c, 0x50, 0x72, 0x6f, 0x78, 0x79, 0x7d, 0x20, 0x2f, 0x3e, 0x60, + 0x2c, 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x69, 0x6e, 0x74, 0x6f, 0x2c, + 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, + 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x69, 0x6e, 0x74, 0x6f, 0x20, 0x3d, + 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x66, 0x69, 0x6e, 0x64, 0x4e, 0x6f, + 0x64, 0x65, 0x28, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x70, + 0x73, 0x2e, 0x69, 0x6e, 0x74, 0x6f, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x72, 0x65, 0x6d, + 0x6f, 0x74, 0x65, 0x20, 0x3d, 0x20, 0x72, 0x65, 0x6e, 0x64, 0x65, 0x72, + 0x28, 0x68, 0x74, 0x6d, 0x6c, 0x60, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x24, 0x7b, 0x50, 0x6f, 0x72, 0x74, + 0x61, 0x6c, 0x50, 0x72, 0x6f, 0x78, 0x79, 0x7d, 0x20, 0x63, 0x6f, 0x6e, + 0x74, 0x65, 0x78, 0x74, 0x3d, 0x24, 0x7b, 0x74, 0x68, 0x69, 0x73, 0x2e, + 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x7d, 0x3e, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x24, 0x7b, + 0x73, 0x68, 0x6f, 0x77, 0x20, 0x26, 0x26, 0x20, 0x74, 0x68, 0x69, 0x73, + 0x2e, 0x70, 0x72, 0x6f, 0x70, 0x73, 0x2e, 0x63, 0x68, 0x69, 0x6c, 0x64, + 0x72, 0x65, 0x6e, 0x20, 0x7c, 0x7c, 0x20, 0x6e, 0x75, 0x6c, 0x6c, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, - 0x2f, 0x73, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x3e, 0x0a, 0x0a, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x66, 0x6f, - 0x6f, 0x74, 0x65, 0x72, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x70, 0x3e, 0x3c, 0x24, 0x7b, - 0x4d, 0x6f, 0x64, 0x65, 0x6c, 0x47, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, - 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, 0x6f, 0x7d, 0x20, 0x2f, 0x3e, 0x3c, - 0x2f, 0x70, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x3c, 0x70, 0x3e, 0x50, 0x6f, 0x77, 0x65, 0x72, - 0x65, 0x64, 0x20, 0x62, 0x79, 0x20, 0x3c, 0x61, 0x20, 0x68, 0x72, 0x65, + 0x2f, 0x24, 0x7b, 0x50, 0x6f, 0x72, 0x74, 0x61, 0x6c, 0x50, 0x72, 0x6f, + 0x78, 0x79, 0x7d, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x60, 0x2c, 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x69, 0x6e, 0x74, + 0x6f, 0x2c, 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x72, 0x65, 0x6d, 0x6f, + 0x74, 0x65, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, + 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, 0x6e, 0x64, + 0x65, 0x72, 0x28, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x6e, 0x75, + 0x6c, 0x6c, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x2f, 0x2f, + 0x20, 0x68, 0x69, 0x67, 0x68, 0x2d, 0x6f, 0x72, 0x64, 0x65, 0x72, 0x20, + 0x63, 0x6f, 0x6d, 0x70, 0x6f, 0x6e, 0x65, 0x6e, 0x74, 0x20, 0x74, 0x68, + 0x61, 0x74, 0x20, 0x72, 0x65, 0x6e, 0x64, 0x65, 0x72, 0x73, 0x20, 0x69, + 0x74, 0x73, 0x20, 0x66, 0x69, 0x72, 0x73, 0x74, 0x20, 0x63, 0x68, 0x69, + 0x6c, 0x64, 0x20, 0x69, 0x66, 0x20, 0x69, 0x74, 0x20, 0x65, 0x78, 0x69, + 0x73, 0x74, 0x73, 0x2e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x2f, 0x2f, 0x20, + 0x75, 0x73, 0x65, 0x64, 0x20, 0x61, 0x73, 0x20, 0x61, 0x20, 0x63, 0x6f, + 0x6e, 0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x61, 0x6c, 0x20, 0x72, 0x65, + 0x6e, 0x64, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x20, 0x70, 0x72, 0x6f, 0x78, + 0x79, 0x2e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6c, 0x61, 0x73, 0x73, + 0x20, 0x50, 0x6f, 0x72, 0x74, 0x61, 0x6c, 0x50, 0x72, 0x6f, 0x78, 0x79, + 0x20, 0x65, 0x78, 0x74, 0x65, 0x6e, 0x64, 0x73, 0x20, 0x43, 0x6f, 0x6d, + 0x70, 0x6f, 0x6e, 0x65, 0x6e, 0x74, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x67, 0x65, 0x74, 0x43, 0x68, 0x69, 0x6c, 0x64, 0x43, + 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x28, 0x29, 0x20, 0x7b, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, + 0x6e, 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x70, 0x73, + 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x3b, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x72, 0x65, 0x6e, 0x64, 0x65, 0x72, 0x28, 0x7b, 0x20, 0x63, 0x68, 0x69, + 0x6c, 0x64, 0x72, 0x65, 0x6e, 0x20, 0x7d, 0x29, 0x20, 0x7b, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, + 0x6e, 0x20, 0x63, 0x68, 0x69, 0x6c, 0x64, 0x72, 0x65, 0x6e, 0x20, 0x7c, + 0x7c, 0x20, 0x6e, 0x75, 0x6c, 0x6c, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, + 0x41, 0x70, 0x70, 0x28, 0x70, 0x72, 0x6f, 0x70, 0x73, 0x29, 0x20, 0x7b, + 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, + 0x72, 0x6e, 0x20, 0x68, 0x74, 0x6d, 0x6c, 0x60, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x64, 0x69, 0x76, 0x20, 0x63, 0x6c, + 0x61, 0x73, 0x73, 0x3d, 0x22, 0x6d, 0x6f, 0x64, 0x65, 0x2d, 0x24, 0x7b, + 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x61, 0x6c, 0x75, + 0x65, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x7d, 0x22, 0x3e, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x68, 0x65, 0x61, + 0x64, 0x65, 0x72, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x68, 0x31, 0x3e, 0x6c, 0x6c, 0x61, + 0x6d, 0x61, 0x2e, 0x63, 0x70, 0x70, 0x3c, 0x2f, 0x68, 0x31, 0x3e, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x2f, + 0x68, 0x65, 0x61, 0x64, 0x65, 0x72, 0x3e, 0x0a, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x6d, 0x61, 0x69, 0x6e, + 0x20, 0x69, 0x64, 0x3d, 0x22, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, + 0x22, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x3c, 0x24, 0x7b, 0x63, 0x68, 0x61, 0x74, 0x53, 0x74, + 0x61, 0x72, 0x74, 0x65, 0x64, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x20, + 0x3f, 0x20, 0x43, 0x68, 0x61, 0x74, 0x4c, 0x6f, 0x67, 0x20, 0x3a, 0x20, + 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x46, 0x6f, 0x72, 0x6d, 0x7d, 0x20, + 0x2f, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x3c, 0x2f, 0x6d, 0x61, 0x69, 0x6e, 0x3e, 0x0a, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x73, 0x65, 0x63, + 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x69, 0x64, 0x3d, 0x22, 0x77, 0x72, 0x69, + 0x74, 0x65, 0x22, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x24, 0x7b, 0x73, 0x65, 0x73, 0x73, + 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x2e, 0x74, 0x79, + 0x70, 0x65, 0x20, 0x3d, 0x3d, 0x3d, 0x20, 0x27, 0x63, 0x68, 0x61, 0x74, + 0x27, 0x20, 0x3f, 0x20, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x49, + 0x6e, 0x70, 0x75, 0x74, 0x20, 0x3a, 0x20, 0x43, 0x6f, 0x6d, 0x70, 0x6c, + 0x65, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, + 0x73, 0x7d, 0x20, 0x2f, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x3c, 0x2f, 0x73, 0x65, 0x63, 0x74, 0x69, 0x6f, + 0x6e, 0x3e, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x3c, 0x66, 0x6f, 0x6f, 0x74, 0x65, 0x72, 0x3e, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, + 0x70, 0x3e, 0x3c, 0x24, 0x7b, 0x4d, 0x6f, 0x64, 0x65, 0x6c, 0x47, 0x65, + 0x6e, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, 0x6f, + 0x7d, 0x20, 0x2f, 0x3e, 0x3c, 0x2f, 0x70, 0x3e, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x70, 0x3e, + 0x50, 0x6f, 0x77, 0x65, 0x72, 0x65, 0x64, 0x20, 0x62, 0x79, 0x20, 0x3c, + 0x61, 0x20, 0x68, 0x72, 0x65, 0x66, 0x3d, 0x22, 0x68, 0x74, 0x74, 0x70, + 0x73, 0x3a, 0x2f, 0x2f, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, + 0x6f, 0x6d, 0x2f, 0x67, 0x67, 0x65, 0x72, 0x67, 0x61, 0x6e, 0x6f, 0x76, + 0x2f, 0x6c, 0x6c, 0x61, 0x6d, 0x61, 0x2e, 0x63, 0x70, 0x70, 0x22, 0x3e, + 0x6c, 0x6c, 0x61, 0x6d, 0x61, 0x2e, 0x63, 0x70, 0x70, 0x3c, 0x2f, 0x61, + 0x3e, 0x20, 0x61, 0x6e, 0x64, 0x20, 0x3c, 0x61, 0x20, 0x68, 0x72, 0x65, 0x66, 0x3d, 0x22, 0x68, 0x74, 0x74, 0x70, 0x73, 0x3a, 0x2f, 0x2f, 0x67, - 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x67, 0x67, - 0x65, 0x72, 0x67, 0x61, 0x6e, 0x6f, 0x76, 0x2f, 0x6c, 0x6c, 0x61, 0x6d, - 0x61, 0x2e, 0x63, 0x70, 0x70, 0x22, 0x3e, 0x6c, 0x6c, 0x61, 0x6d, 0x61, - 0x2e, 0x63, 0x70, 0x70, 0x3c, 0x2f, 0x61, 0x3e, 0x20, 0x61, 0x6e, 0x64, - 0x20, 0x3c, 0x61, 0x20, 0x68, 0x72, 0x65, 0x66, 0x3d, 0x22, 0x68, 0x74, - 0x74, 0x70, 0x73, 0x3a, 0x2f, 0x2f, 0x67, 0x67, 0x6d, 0x6c, 0x2e, 0x61, - 0x69, 0x22, 0x3e, 0x67, 0x67, 0x6d, 0x6c, 0x2e, 0x61, 0x69, 0x3c, 0x2f, - 0x61, 0x3e, 0x2e, 0x3c, 0x2f, 0x70, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x2f, 0x66, 0x6f, 0x6f, 0x74, - 0x65, 0x72, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x3c, 0x2f, 0x64, 0x69, 0x76, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x60, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x0a, 0x20, - 0x20, 0x20, 0x20, 0x72, 0x65, 0x6e, 0x64, 0x65, 0x72, 0x28, 0x68, 0x28, - 0x41, 0x70, 0x70, 0x29, 0x2c, 0x20, 0x64, 0x6f, 0x63, 0x75, 0x6d, 0x65, - 0x6e, 0x74, 0x2e, 0x71, 0x75, 0x65, 0x72, 0x79, 0x53, 0x65, 0x6c, 0x65, - 0x63, 0x74, 0x6f, 0x72, 0x28, 0x27, 0x23, 0x63, 0x6f, 0x6e, 0x74, 0x61, - 0x69, 0x6e, 0x65, 0x72, 0x27, 0x29, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x3c, - 0x2f, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x3e, 0x0a, 0x3c, 0x2f, 0x68, - 0x65, 0x61, 0x64, 0x3e, 0x0a, 0x0a, 0x3c, 0x62, 0x6f, 0x64, 0x79, 0x3e, - 0x0a, 0x20, 0x20, 0x3c, 0x64, 0x69, 0x76, 0x20, 0x69, 0x64, 0x3d, 0x22, - 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x22, 0x3e, 0x0a, - 0x20, 0x20, 0x20, 0x20, 0x3c, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x20, 0x74, - 0x79, 0x70, 0x65, 0x3d, 0x22, 0x66, 0x69, 0x6c, 0x65, 0x22, 0x20, 0x69, - 0x64, 0x3d, 0x22, 0x66, 0x69, 0x6c, 0x65, 0x49, 0x6e, 0x70, 0x75, 0x74, - 0x22, 0x20, 0x61, 0x63, 0x63, 0x65, 0x70, 0x74, 0x3d, 0x22, 0x69, 0x6d, - 0x61, 0x67, 0x65, 0x2f, 0x2a, 0x22, 0x20, 0x73, 0x74, 0x79, 0x6c, 0x65, - 0x3d, 0x22, 0x64, 0x69, 0x73, 0x70, 0x6c, 0x61, 0x79, 0x3a, 0x20, 0x6e, - 0x6f, 0x6e, 0x65, 0x3b, 0x22, 0x3e, 0x0a, 0x20, 0x20, 0x3c, 0x2f, 0x64, - 0x69, 0x76, 0x3e, 0x0a, 0x20, 0x20, 0x3c, 0x64, 0x69, 0x76, 0x20, 0x69, - 0x64, 0x3d, 0x22, 0x70, 0x6f, 0x72, 0x74, 0x61, 0x6c, 0x22, 0x3e, 0x3c, - 0x2f, 0x64, 0x69, 0x76, 0x3e, 0x0a, 0x3c, 0x2f, 0x62, 0x6f, 0x64, 0x79, - 0x3e, 0x0a, 0x0a, 0x3c, 0x2f, 0x68, 0x74, 0x6d, 0x6c, 0x3e, 0x0a, 0x0a + 0x67, 0x6d, 0x6c, 0x2e, 0x61, 0x69, 0x22, 0x3e, 0x67, 0x67, 0x6d, 0x6c, + 0x2e, 0x61, 0x69, 0x3c, 0x2f, 0x61, 0x3e, 0x2e, 0x3c, 0x2f, 0x70, 0x3e, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, + 0x2f, 0x66, 0x6f, 0x6f, 0x74, 0x65, 0x72, 0x3e, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x2f, 0x64, 0x69, 0x76, 0x3e, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x60, 0x3b, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x7d, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, 0x6e, 0x64, + 0x65, 0x72, 0x28, 0x68, 0x28, 0x41, 0x70, 0x70, 0x29, 0x2c, 0x20, 0x64, + 0x6f, 0x63, 0x75, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x71, 0x75, 0x65, 0x72, + 0x79, 0x53, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x28, 0x27, 0x23, + 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x27, 0x29, 0x29, + 0x3b, 0x0a, 0x20, 0x20, 0x3c, 0x2f, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, + 0x3e, 0x0a, 0x3c, 0x2f, 0x68, 0x65, 0x61, 0x64, 0x3e, 0x0a, 0x0a, 0x3c, + 0x62, 0x6f, 0x64, 0x79, 0x3e, 0x0a, 0x20, 0x20, 0x3c, 0x64, 0x69, 0x76, + 0x20, 0x69, 0x64, 0x3d, 0x22, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, + 0x65, 0x72, 0x22, 0x3e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x3c, 0x69, 0x6e, + 0x70, 0x75, 0x74, 0x20, 0x74, 0x79, 0x70, 0x65, 0x3d, 0x22, 0x66, 0x69, + 0x6c, 0x65, 0x22, 0x20, 0x69, 0x64, 0x3d, 0x22, 0x66, 0x69, 0x6c, 0x65, + 0x49, 0x6e, 0x70, 0x75, 0x74, 0x22, 0x20, 0x61, 0x63, 0x63, 0x65, 0x70, + 0x74, 0x3d, 0x22, 0x69, 0x6d, 0x61, 0x67, 0x65, 0x2f, 0x2a, 0x22, 0x20, + 0x73, 0x74, 0x79, 0x6c, 0x65, 0x3d, 0x22, 0x64, 0x69, 0x73, 0x70, 0x6c, + 0x61, 0x79, 0x3a, 0x20, 0x6e, 0x6f, 0x6e, 0x65, 0x3b, 0x22, 0x3e, 0x0a, + 0x20, 0x20, 0x3c, 0x2f, 0x64, 0x69, 0x76, 0x3e, 0x0a, 0x20, 0x20, 0x3c, + 0x64, 0x69, 0x76, 0x20, 0x69, 0x64, 0x3d, 0x22, 0x70, 0x6f, 0x72, 0x74, + 0x61, 0x6c, 0x22, 0x3e, 0x3c, 0x2f, 0x64, 0x69, 0x76, 0x3e, 0x0a, 0x3c, + 0x2f, 0x62, 0x6f, 0x64, 0x79, 0x3e, 0x0a, 0x0a, 0x3c, 0x2f, 0x68, 0x74, + 0x6d, 0x6c, 0x3e, 0x0a, 0x0a }; -unsigned int index_html_len = 33456; +size_t index_html_len = 33809; diff --git a/examples/server/index.js.hpp b/examples/server/index.js.hpp index e09b3c8c5..24d0b33fa 100644 --- a/examples/server/index.js.hpp +++ b/examples/server/index.js.hpp @@ -12,7 +12,7 @@ unsigned char index_js[] = { 0x74, 0x2c, 0x6e, 0x3d, 0x21, 0x31, 0x3b, 0x77, 0x68, 0x69, 0x6c, 0x65, 0x28, 0x76, 0x6f, 0x69, 0x64, 0x20, 0x30, 0x21, 0x3d, 0x3d, 0x6f, 0x29, 0x7b, 0x6c, 0x65, 0x74, 0x20, 0x5f, 0x3d, 0x6f, 0x3b, 0x6f, 0x3d, 0x76, - 0x6f, 0x69, 0x64, 0x20, 0x30, 0x3b, 0x73, 0x2b, 0x2b, 0x3b, 0x77, 0x68, + 0x6f, 0x69, 0x64, 0x20, 0x30, 0x3b, 0x6c, 0x2b, 0x2b, 0x3b, 0x77, 0x68, 0x69, 0x6c, 0x65, 0x28, 0x76, 0x6f, 0x69, 0x64, 0x20, 0x30, 0x21, 0x3d, 0x3d, 0x5f, 0x29, 0x7b, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x69, 0x3d, 0x5f, 0x2e, 0x6f, 0x3b, 0x5f, 0x2e, 0x6f, 0x3d, 0x76, 0x6f, 0x69, 0x64, @@ -21,7 +21,7 @@ unsigned char index_js[] = { 0x70, 0x28, 0x5f, 0x29, 0x29, 0x74, 0x72, 0x79, 0x7b, 0x5f, 0x2e, 0x63, 0x28, 0x29, 0x7d, 0x63, 0x61, 0x74, 0x63, 0x68, 0x28, 0x65, 0x29, 0x7b, 0x69, 0x66, 0x28, 0x21, 0x6e, 0x29, 0x7b, 0x74, 0x3d, 0x65, 0x3b, 0x6e, - 0x3d, 0x21, 0x30, 0x7d, 0x7d, 0x5f, 0x3d, 0x69, 0x7d, 0x7d, 0x73, 0x3d, + 0x3d, 0x21, 0x30, 0x7d, 0x7d, 0x5f, 0x3d, 0x69, 0x7d, 0x7d, 0x6c, 0x3d, 0x30, 0x3b, 0x66, 0x2d, 0x2d, 0x3b, 0x69, 0x66, 0x28, 0x6e, 0x29, 0x74, 0x68, 0x72, 0x6f, 0x77, 0x20, 0x74, 0x7d, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x5f, 0x28, 0x74, 0x29, 0x7b, 0x69, 0x66, 0x28, @@ -37,8 +37,8 @@ unsigned char index_js[] = { 0x30, 0x3b, 0x72, 0x2b, 0x2b, 0x3b, 0x74, 0x72, 0x79, 0x7b, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x74, 0x28, 0x29, 0x7d, 0x66, 0x69, 0x6e, 0x61, 0x6c, 0x6c, 0x79, 0x7b, 0x72, 0x2d, 0x2d, 0x3b, 0x69, 0x3d, 0x6e, - 0x7d, 0x7d, 0x6c, 0x65, 0x74, 0x20, 0x66, 0x3d, 0x30, 0x2c, 0x73, 0x3d, - 0x30, 0x2c, 0x6c, 0x3d, 0x30, 0x3b, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, + 0x7d, 0x7d, 0x6c, 0x65, 0x74, 0x20, 0x66, 0x3d, 0x30, 0x2c, 0x6c, 0x3d, + 0x30, 0x2c, 0x73, 0x3d, 0x30, 0x3b, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x63, 0x28, 0x74, 0x29, 0x7b, 0x69, 0x66, 0x28, 0x76, 0x6f, 0x69, 0x64, 0x20, 0x30, 0x3d, 0x3d, 0x3d, 0x69, 0x29, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x3b, 0x6c, 0x65, 0x74, 0x20, 0x6e, 0x3d, 0x74, @@ -141,9 +141,9 @@ unsigned char index_js[] = { 0x20, 0x73, 0x69, 0x64, 0x65, 0x2d, 0x65, 0x66, 0x66, 0x65, 0x63, 0x74, 0x73, 0x22, 0x29, 0x7d, 0x28, 0x29, 0x3b, 0x69, 0x66, 0x28, 0x6e, 0x21, 0x3d, 0x3d, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x76, 0x29, 0x7b, 0x69, 0x66, - 0x28, 0x73, 0x3e, 0x31, 0x30, 0x30, 0x29, 0x74, 0x28, 0x29, 0x3b, 0x74, + 0x28, 0x6c, 0x3e, 0x31, 0x30, 0x30, 0x29, 0x74, 0x28, 0x29, 0x3b, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x76, 0x3d, 0x6e, 0x3b, 0x74, 0x68, 0x69, 0x73, - 0x2e, 0x69, 0x2b, 0x2b, 0x3b, 0x6c, 0x2b, 0x2b, 0x3b, 0x66, 0x2b, 0x2b, + 0x2e, 0x69, 0x2b, 0x2b, 0x3b, 0x73, 0x2b, 0x2b, 0x3b, 0x66, 0x2b, 0x2b, 0x3b, 0x74, 0x72, 0x79, 0x7b, 0x66, 0x6f, 0x72, 0x28, 0x6c, 0x65, 0x74, 0x20, 0x74, 0x3d, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x74, 0x3b, 0x76, 0x6f, 0x69, 0x64, 0x20, 0x30, 0x21, 0x3d, 0x3d, 0x74, 0x3b, 0x74, 0x3d, 0x74, @@ -191,7 +191,7 @@ unsigned char index_js[] = { 0x6f, 0x69, 0x64, 0x20, 0x30, 0x29, 0x3b, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x78, 0x3d, 0x74, 0x3b, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x73, 0x3d, 0x76, 0x6f, 0x69, 0x64, 0x20, 0x30, 0x3b, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x67, - 0x3d, 0x6c, 0x2d, 0x31, 0x3b, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x66, 0x3d, + 0x3d, 0x73, 0x2d, 0x31, 0x3b, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x66, 0x3d, 0x34, 0x7d, 0x28, 0x79, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x74, 0x79, 0x70, 0x65, 0x3d, 0x6e, 0x65, 0x77, 0x20, 0x68, 0x29, 0x2e, 0x68, 0x3d, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x29, 0x7b, 0x74, @@ -201,9 +201,9 @@ unsigned char index_js[] = { 0x3d, 0x3d, 0x28, 0x33, 0x36, 0x26, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x66, 0x29, 0x29, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x21, 0x30, 0x3b, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x66, 0x26, 0x3d, 0x2d, 0x35, 0x3b, 0x69, 0x66, - 0x28, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x67, 0x3d, 0x3d, 0x3d, 0x6c, 0x29, + 0x28, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x67, 0x3d, 0x3d, 0x3d, 0x73, 0x29, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x21, 0x30, 0x3b, 0x74, 0x68, 0x69, - 0x73, 0x2e, 0x67, 0x3d, 0x6c, 0x3b, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x66, + 0x73, 0x2e, 0x67, 0x3d, 0x73, 0x3b, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x66, 0x7c, 0x3d, 0x31, 0x3b, 0x69, 0x66, 0x28, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x69, 0x3e, 0x30, 0x26, 0x26, 0x21, 0x70, 0x28, 0x74, 0x68, 0x69, 0x73, 0x29, 0x29, 0x7b, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x66, 0x26, 0x3d, 0x2d, @@ -437,321 +437,340 @@ unsigned char index_js[] = { 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x3d, 0x3d, 0x74, 0x79, 0x70, 0x65, 0x6f, 0x66, 0x20, 0x74, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x3f, 0x71, 0x28, 0x74, 0x29, 0x3a, 0x6e, 0x75, 0x6c, 0x6c, 0x7d, 0x66, 0x75, 0x6e, 0x63, 0x74, - 0x69, 0x6f, 0x6e, 0x20, 0x42, 0x28, 0x74, 0x29, 0x7b, 0x76, 0x61, 0x72, - 0x20, 0x6e, 0x2c, 0x65, 0x3b, 0x69, 0x66, 0x28, 0x6e, 0x75, 0x6c, 0x6c, - 0x21, 0x3d, 0x28, 0x74, 0x3d, 0x74, 0x2e, 0x5f, 0x5f, 0x29, 0x26, 0x26, - 0x6e, 0x75, 0x6c, 0x6c, 0x21, 0x3d, 0x74, 0x2e, 0x5f, 0x5f, 0x63, 0x29, - 0x7b, 0x66, 0x6f, 0x72, 0x28, 0x74, 0x2e, 0x5f, 0x5f, 0x65, 0x3d, 0x74, - 0x2e, 0x5f, 0x5f, 0x63, 0x2e, 0x62, 0x61, 0x73, 0x65, 0x3d, 0x6e, 0x75, - 0x6c, 0x6c, 0x2c, 0x6e, 0x3d, 0x30, 0x3b, 0x6e, 0x3c, 0x74, 0x2e, 0x5f, - 0x5f, 0x6b, 0x2e, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x3b, 0x6e, 0x2b, - 0x2b, 0x29, 0x69, 0x66, 0x28, 0x6e, 0x75, 0x6c, 0x6c, 0x21, 0x3d, 0x28, - 0x65, 0x3d, 0x74, 0x2e, 0x5f, 0x5f, 0x6b, 0x5b, 0x6e, 0x5d, 0x29, 0x26, - 0x26, 0x6e, 0x75, 0x6c, 0x6c, 0x21, 0x3d, 0x65, 0x2e, 0x5f, 0x5f, 0x65, - 0x29, 0x7b, 0x74, 0x2e, 0x5f, 0x5f, 0x65, 0x3d, 0x74, 0x2e, 0x5f, 0x5f, - 0x63, 0x2e, 0x62, 0x61, 0x73, 0x65, 0x3d, 0x65, 0x2e, 0x5f, 0x5f, 0x65, - 0x3b, 0x62, 0x72, 0x65, 0x61, 0x6b, 0x7d, 0x72, 0x65, 0x74, 0x75, 0x72, - 0x6e, 0x20, 0x42, 0x28, 0x74, 0x29, 0x7d, 0x7d, 0x66, 0x75, 0x6e, 0x63, - 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x47, 0x28, 0x74, 0x29, 0x7b, 0x28, 0x21, - 0x74, 0x2e, 0x5f, 0x5f, 0x64, 0x26, 0x26, 0x28, 0x74, 0x2e, 0x5f, 0x5f, - 0x64, 0x3d, 0x21, 0x30, 0x29, 0x26, 0x26, 0x48, 0x2e, 0x70, 0x75, 0x73, - 0x68, 0x28, 0x74, 0x29, 0x26, 0x26, 0x21, 0x7a, 0x2e, 0x5f, 0x5f, 0x72, - 0x2b, 0x2b, 0x7c, 0x7c, 0x50, 0x21, 0x3d, 0x3d, 0x43, 0x2e, 0x64, 0x65, - 0x62, 0x6f, 0x75, 0x6e, 0x63, 0x65, 0x52, 0x65, 0x6e, 0x64, 0x65, 0x72, - 0x69, 0x6e, 0x67, 0x29, 0x26, 0x26, 0x28, 0x28, 0x50, 0x3d, 0x43, 0x2e, - 0x64, 0x65, 0x62, 0x6f, 0x75, 0x6e, 0x63, 0x65, 0x52, 0x65, 0x6e, 0x64, - 0x65, 0x72, 0x69, 0x6e, 0x67, 0x29, 0x7c, 0x7c, 0x4e, 0x29, 0x28, 0x7a, - 0x29, 0x7d, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x7a, - 0x28, 0x29, 0x7b, 0x76, 0x61, 0x72, 0x20, 0x74, 0x2c, 0x6e, 0x2c, 0x65, - 0x2c, 0x5f, 0x2c, 0x69, 0x2c, 0x6f, 0x2c, 0x72, 0x2c, 0x75, 0x2c, 0x66, - 0x3b, 0x66, 0x6f, 0x72, 0x28, 0x48, 0x2e, 0x73, 0x6f, 0x72, 0x74, 0x28, - 0x24, 0x29, 0x3b, 0x74, 0x3d, 0x48, 0x2e, 0x73, 0x68, 0x69, 0x66, 0x74, - 0x28, 0x29, 0x3b, 0x29, 0x74, 0x2e, 0x5f, 0x5f, 0x64, 0x26, 0x26, 0x28, - 0x6e, 0x3d, 0x48, 0x2e, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x2c, 0x5f, - 0x3d, 0x76, 0x6f, 0x69, 0x64, 0x20, 0x30, 0x2c, 0x6f, 0x3d, 0x28, 0x69, - 0x3d, 0x28, 0x65, 0x3d, 0x74, 0x29, 0x2e, 0x5f, 0x5f, 0x76, 0x29, 0x2e, - 0x5f, 0x5f, 0x65, 0x2c, 0x75, 0x3d, 0x5b, 0x5d, 0x2c, 0x66, 0x3d, 0x5b, - 0x5d, 0x2c, 0x28, 0x72, 0x3d, 0x65, 0x2e, 0x5f, 0x5f, 0x50, 0x29, 0x26, - 0x26, 0x28, 0x28, 0x5f, 0x3d, 0x4d, 0x28, 0x7b, 0x7d, 0x2c, 0x69, 0x29, - 0x29, 0x2e, 0x5f, 0x5f, 0x76, 0x3d, 0x69, 0x2e, 0x5f, 0x5f, 0x76, 0x2b, - 0x31, 0x2c, 0x43, 0x2e, 0x76, 0x6e, 0x6f, 0x64, 0x65, 0x26, 0x26, 0x43, - 0x2e, 0x76, 0x6e, 0x6f, 0x64, 0x65, 0x28, 0x5f, 0x29, 0x2c, 0x5f, 0x74, - 0x28, 0x72, 0x2c, 0x5f, 0x2c, 0x69, 0x2c, 0x65, 0x2e, 0x5f, 0x5f, 0x6e, - 0x2c, 0x76, 0x6f, 0x69, 0x64, 0x20, 0x30, 0x21, 0x3d, 0x3d, 0x72, 0x2e, - 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x53, 0x56, 0x47, 0x45, 0x6c, 0x65, 0x6d, - 0x65, 0x6e, 0x74, 0x2c, 0x33, 0x32, 0x26, 0x69, 0x2e, 0x5f, 0x5f, 0x75, - 0x3f, 0x5b, 0x6f, 0x5d, 0x3a, 0x6e, 0x75, 0x6c, 0x6c, 0x2c, 0x75, 0x2c, - 0x6e, 0x75, 0x6c, 0x6c, 0x3d, 0x3d, 0x6f, 0x3f, 0x71, 0x28, 0x69, 0x29, - 0x3a, 0x6f, 0x2c, 0x21, 0x21, 0x28, 0x33, 0x32, 0x26, 0x69, 0x2e, 0x5f, - 0x5f, 0x75, 0x29, 0x2c, 0x66, 0x29, 0x2c, 0x5f, 0x2e, 0x5f, 0x5f, 0x2e, - 0x5f, 0x5f, 0x6b, 0x5b, 0x5f, 0x2e, 0x5f, 0x5f, 0x69, 0x5d, 0x3d, 0x5f, - 0x2c, 0x69, 0x74, 0x28, 0x75, 0x2c, 0x5f, 0x2c, 0x66, 0x29, 0x2c, 0x5f, - 0x2e, 0x5f, 0x5f, 0x65, 0x21, 0x3d, 0x6f, 0x26, 0x26, 0x42, 0x28, 0x5f, - 0x29, 0x29, 0x2c, 0x48, 0x2e, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x3e, - 0x6e, 0x26, 0x26, 0x48, 0x2e, 0x73, 0x6f, 0x72, 0x74, 0x28, 0x24, 0x29, - 0x29, 0x3b, 0x7a, 0x2e, 0x5f, 0x5f, 0x72, 0x3d, 0x30, 0x7d, 0x66, 0x75, - 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x4a, 0x28, 0x74, 0x2c, 0x6e, - 0x2c, 0x65, 0x2c, 0x5f, 0x2c, 0x69, 0x2c, 0x6f, 0x2c, 0x72, 0x2c, 0x75, - 0x2c, 0x66, 0x2c, 0x73, 0x2c, 0x6c, 0x29, 0x7b, 0x76, 0x61, 0x72, 0x20, - 0x63, 0x2c, 0x68, 0x2c, 0x61, 0x2c, 0x70, 0x2c, 0x64, 0x2c, 0x76, 0x3d, - 0x5f, 0x26, 0x26, 0x5f, 0x2e, 0x5f, 0x5f, 0x6b, 0x7c, 0x7c, 0x56, 0x2c, - 0x79, 0x3d, 0x6e, 0x2e, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x3b, 0x66, - 0x6f, 0x72, 0x28, 0x65, 0x2e, 0x5f, 0x5f, 0x64, 0x3d, 0x66, 0x2c, 0x4b, - 0x28, 0x65, 0x2c, 0x6e, 0x2c, 0x76, 0x29, 0x2c, 0x66, 0x3d, 0x65, 0x2e, - 0x5f, 0x5f, 0x64, 0x2c, 0x63, 0x3d, 0x30, 0x3b, 0x63, 0x3c, 0x79, 0x3b, - 0x63, 0x2b, 0x2b, 0x29, 0x6e, 0x75, 0x6c, 0x6c, 0x21, 0x3d, 0x28, 0x61, - 0x3d, 0x65, 0x2e, 0x5f, 0x5f, 0x6b, 0x5b, 0x63, 0x5d, 0x29, 0x26, 0x26, - 0x22, 0x62, 0x6f, 0x6f, 0x6c, 0x65, 0x61, 0x6e, 0x22, 0x21, 0x3d, 0x74, - 0x79, 0x70, 0x65, 0x6f, 0x66, 0x20, 0x61, 0x26, 0x26, 0x22, 0x66, 0x75, - 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x21, 0x3d, 0x74, 0x79, 0x70, - 0x65, 0x6f, 0x66, 0x20, 0x61, 0x26, 0x26, 0x28, 0x68, 0x3d, 0x2d, 0x31, - 0x3d, 0x3d, 0x3d, 0x61, 0x2e, 0x5f, 0x5f, 0x69, 0x3f, 0x54, 0x3a, 0x76, - 0x5b, 0x61, 0x2e, 0x5f, 0x5f, 0x69, 0x5d, 0x7c, 0x7c, 0x54, 0x2c, 0x61, - 0x2e, 0x5f, 0x5f, 0x69, 0x3d, 0x63, 0x2c, 0x5f, 0x74, 0x28, 0x74, 0x2c, - 0x61, 0x2c, 0x68, 0x2c, 0x69, 0x2c, 0x6f, 0x2c, 0x72, 0x2c, 0x75, 0x2c, - 0x66, 0x2c, 0x73, 0x2c, 0x6c, 0x29, 0x2c, 0x70, 0x3d, 0x61, 0x2e, 0x5f, - 0x5f, 0x65, 0x2c, 0x61, 0x2e, 0x72, 0x65, 0x66, 0x26, 0x26, 0x68, 0x2e, - 0x72, 0x65, 0x66, 0x21, 0x3d, 0x61, 0x2e, 0x72, 0x65, 0x66, 0x26, 0x26, - 0x28, 0x68, 0x2e, 0x72, 0x65, 0x66, 0x26, 0x26, 0x72, 0x74, 0x28, 0x68, - 0x2e, 0x72, 0x65, 0x66, 0x2c, 0x6e, 0x75, 0x6c, 0x6c, 0x2c, 0x61, 0x29, - 0x2c, 0x6c, 0x2e, 0x70, 0x75, 0x73, 0x68, 0x28, 0x61, 0x2e, 0x72, 0x65, - 0x66, 0x2c, 0x61, 0x2e, 0x5f, 0x5f, 0x63, 0x7c, 0x7c, 0x70, 0x2c, 0x61, - 0x29, 0x29, 0x2c, 0x6e, 0x75, 0x6c, 0x6c, 0x3d, 0x3d, 0x64, 0x26, 0x26, - 0x6e, 0x75, 0x6c, 0x6c, 0x21, 0x3d, 0x70, 0x26, 0x26, 0x28, 0x64, 0x3d, - 0x70, 0x29, 0x2c, 0x36, 0x35, 0x35, 0x33, 0x36, 0x26, 0x61, 0x2e, 0x5f, - 0x5f, 0x75, 0x7c, 0x7c, 0x68, 0x2e, 0x5f, 0x5f, 0x6b, 0x3d, 0x3d, 0x3d, - 0x61, 0x2e, 0x5f, 0x5f, 0x6b, 0x3f, 0x66, 0x3d, 0x51, 0x28, 0x61, 0x2c, - 0x66, 0x2c, 0x74, 0x29, 0x3a, 0x22, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, - 0x6f, 0x6e, 0x22, 0x3d, 0x3d, 0x74, 0x79, 0x70, 0x65, 0x6f, 0x66, 0x20, - 0x61, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x26, 0x26, 0x76, 0x6f, 0x69, 0x64, - 0x20, 0x30, 0x21, 0x3d, 0x3d, 0x61, 0x2e, 0x5f, 0x5f, 0x64, 0x3f, 0x66, - 0x3d, 0x61, 0x2e, 0x5f, 0x5f, 0x64, 0x3a, 0x70, 0x26, 0x26, 0x28, 0x66, - 0x3d, 0x70, 0x2e, 0x6e, 0x65, 0x78, 0x74, 0x53, 0x69, 0x62, 0x6c, 0x69, - 0x6e, 0x67, 0x29, 0x2c, 0x61, 0x2e, 0x5f, 0x5f, 0x64, 0x3d, 0x76, 0x6f, - 0x69, 0x64, 0x20, 0x30, 0x2c, 0x61, 0x2e, 0x5f, 0x5f, 0x75, 0x26, 0x3d, - 0x2d, 0x31, 0x39, 0x36, 0x36, 0x30, 0x39, 0x29, 0x3b, 0x65, 0x2e, 0x5f, - 0x5f, 0x64, 0x3d, 0x66, 0x2c, 0x65, 0x2e, 0x5f, 0x5f, 0x65, 0x3d, 0x64, - 0x7d, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x4b, 0x28, - 0x74, 0x2c, 0x6e, 0x2c, 0x65, 0x29, 0x7b, 0x76, 0x61, 0x72, 0x20, 0x5f, - 0x2c, 0x69, 0x2c, 0x6f, 0x2c, 0x72, 0x2c, 0x75, 0x2c, 0x66, 0x3d, 0x6e, - 0x2e, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x2c, 0x73, 0x3d, 0x65, 0x2e, - 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x2c, 0x6c, 0x3d, 0x73, 0x2c, 0x63, - 0x3d, 0x30, 0x3b, 0x66, 0x6f, 0x72, 0x28, 0x74, 0x2e, 0x5f, 0x5f, 0x6b, - 0x3d, 0x5b, 0x5d, 0x2c, 0x5f, 0x3d, 0x30, 0x3b, 0x5f, 0x3c, 0x66, 0x3b, - 0x5f, 0x2b, 0x2b, 0x29, 0x6e, 0x75, 0x6c, 0x6c, 0x21, 0x3d, 0x28, 0x69, - 0x3d, 0x74, 0x2e, 0x5f, 0x5f, 0x6b, 0x5b, 0x5f, 0x5d, 0x3d, 0x6e, 0x75, - 0x6c, 0x6c, 0x3d, 0x3d, 0x28, 0x69, 0x3d, 0x6e, 0x5b, 0x5f, 0x5d, 0x29, - 0x7c, 0x7c, 0x22, 0x62, 0x6f, 0x6f, 0x6c, 0x65, 0x61, 0x6e, 0x22, 0x3d, - 0x3d, 0x74, 0x79, 0x70, 0x65, 0x6f, 0x66, 0x20, 0x69, 0x7c, 0x7c, 0x22, - 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x3d, 0x3d, 0x74, - 0x79, 0x70, 0x65, 0x6f, 0x66, 0x20, 0x69, 0x3f, 0x6e, 0x75, 0x6c, 0x6c, - 0x3a, 0x22, 0x73, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x22, 0x3d, 0x3d, 0x74, - 0x79, 0x70, 0x65, 0x6f, 0x66, 0x20, 0x69, 0x7c, 0x7c, 0x22, 0x6e, 0x75, - 0x6d, 0x62, 0x65, 0x72, 0x22, 0x3d, 0x3d, 0x74, 0x79, 0x70, 0x65, 0x6f, - 0x66, 0x20, 0x69, 0x7c, 0x7c, 0x22, 0x62, 0x69, 0x67, 0x69, 0x6e, 0x74, - 0x22, 0x3d, 0x3d, 0x74, 0x79, 0x70, 0x65, 0x6f, 0x66, 0x20, 0x69, 0x7c, - 0x7c, 0x69, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x72, 0x75, 0x63, 0x74, - 0x6f, 0x72, 0x3d, 0x3d, 0x53, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x3f, 0x4f, - 0x28, 0x6e, 0x75, 0x6c, 0x6c, 0x2c, 0x69, 0x2c, 0x6e, 0x75, 0x6c, 0x6c, - 0x2c, 0x6e, 0x75, 0x6c, 0x6c, 0x2c, 0x69, 0x29, 0x3a, 0x46, 0x28, 0x69, - 0x29, 0x3f, 0x4f, 0x28, 0x6a, 0x2c, 0x7b, 0x63, 0x68, 0x69, 0x6c, 0x64, - 0x72, 0x65, 0x6e, 0x3a, 0x69, 0x7d, 0x2c, 0x6e, 0x75, 0x6c, 0x6c, 0x2c, - 0x6e, 0x75, 0x6c, 0x6c, 0x2c, 0x6e, 0x75, 0x6c, 0x6c, 0x29, 0x3a, 0x76, - 0x6f, 0x69, 0x64, 0x20, 0x30, 0x3d, 0x3d, 0x3d, 0x69, 0x2e, 0x63, 0x6f, - 0x6e, 0x73, 0x74, 0x72, 0x75, 0x63, 0x74, 0x6f, 0x72, 0x26, 0x26, 0x69, - 0x2e, 0x5f, 0x5f, 0x62, 0x3e, 0x30, 0x3f, 0x4f, 0x28, 0x69, 0x2e, 0x74, - 0x79, 0x70, 0x65, 0x2c, 0x69, 0x2e, 0x70, 0x72, 0x6f, 0x70, 0x73, 0x2c, - 0x69, 0x2e, 0x6b, 0x65, 0x79, 0x2c, 0x69, 0x2e, 0x72, 0x65, 0x66, 0x3f, - 0x69, 0x2e, 0x72, 0x65, 0x66, 0x3a, 0x6e, 0x75, 0x6c, 0x6c, 0x2c, 0x69, - 0x2e, 0x5f, 0x5f, 0x76, 0x29, 0x3a, 0x69, 0x29, 0x3f, 0x28, 0x69, 0x2e, - 0x5f, 0x5f, 0x3d, 0x74, 0x2c, 0x69, 0x2e, 0x5f, 0x5f, 0x62, 0x3d, 0x74, - 0x2e, 0x5f, 0x5f, 0x62, 0x2b, 0x31, 0x2c, 0x75, 0x3d, 0x59, 0x28, 0x69, - 0x2c, 0x65, 0x2c, 0x72, 0x3d, 0x5f, 0x2b, 0x63, 0x2c, 0x6c, 0x29, 0x2c, - 0x69, 0x2e, 0x5f, 0x5f, 0x69, 0x3d, 0x75, 0x2c, 0x6f, 0x3d, 0x6e, 0x75, - 0x6c, 0x6c, 0x2c, 0x2d, 0x31, 0x21, 0x3d, 0x3d, 0x75, 0x26, 0x26, 0x28, - 0x6c, 0x2d, 0x2d, 0x2c, 0x28, 0x6f, 0x3d, 0x65, 0x5b, 0x75, 0x5d, 0x29, - 0x26, 0x26, 0x28, 0x6f, 0x2e, 0x5f, 0x5f, 0x75, 0x7c, 0x3d, 0x31, 0x33, - 0x31, 0x30, 0x37, 0x32, 0x29, 0x29, 0x2c, 0x6e, 0x75, 0x6c, 0x6c, 0x3d, - 0x3d, 0x6f, 0x7c, 0x7c, 0x6e, 0x75, 0x6c, 0x6c, 0x3d, 0x3d, 0x3d, 0x6f, - 0x2e, 0x5f, 0x5f, 0x76, 0x3f, 0x28, 0x2d, 0x31, 0x3d, 0x3d, 0x75, 0x26, - 0x26, 0x63, 0x2d, 0x2d, 0x2c, 0x22, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, - 0x6f, 0x6e, 0x22, 0x21, 0x3d, 0x74, 0x79, 0x70, 0x65, 0x6f, 0x66, 0x20, - 0x69, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x26, 0x26, 0x28, 0x69, 0x2e, 0x5f, - 0x5f, 0x75, 0x7c, 0x3d, 0x36, 0x35, 0x35, 0x33, 0x36, 0x29, 0x29, 0x3a, - 0x75, 0x21, 0x3d, 0x3d, 0x72, 0x26, 0x26, 0x28, 0x75, 0x3d, 0x3d, 0x3d, - 0x72, 0x2b, 0x31, 0x3f, 0x63, 0x2b, 0x2b, 0x3a, 0x75, 0x3e, 0x72, 0x3f, - 0x6c, 0x3e, 0x66, 0x2d, 0x72, 0x3f, 0x63, 0x2b, 0x3d, 0x75, 0x2d, 0x72, - 0x3a, 0x63, 0x2d, 0x2d, 0x3a, 0x63, 0x3d, 0x75, 0x3c, 0x72, 0x26, 0x26, - 0x75, 0x3d, 0x3d, 0x72, 0x2d, 0x31, 0x3f, 0x75, 0x2d, 0x72, 0x3a, 0x30, - 0x2c, 0x75, 0x21, 0x3d, 0x3d, 0x5f, 0x2b, 0x63, 0x26, 0x26, 0x28, 0x69, + 0x69, 0x6f, 0x6e, 0x20, 0x42, 0x28, 0x74, 0x2c, 0x6e, 0x2c, 0x65, 0x29, + 0x7b, 0x76, 0x61, 0x72, 0x20, 0x5f, 0x2c, 0x69, 0x3d, 0x74, 0x2e, 0x5f, + 0x5f, 0x76, 0x2c, 0x6f, 0x3d, 0x69, 0x2e, 0x5f, 0x5f, 0x65, 0x2c, 0x72, + 0x3d, 0x74, 0x2e, 0x5f, 0x5f, 0x50, 0x3b, 0x69, 0x66, 0x28, 0x72, 0x29, + 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x28, 0x5f, 0x3d, 0x4d, 0x28, 0x7b, + 0x7d, 0x2c, 0x69, 0x29, 0x29, 0x2e, 0x5f, 0x5f, 0x76, 0x3d, 0x69, 0x2e, + 0x5f, 0x5f, 0x76, 0x2b, 0x31, 0x2c, 0x43, 0x2e, 0x76, 0x6e, 0x6f, 0x64, + 0x65, 0x26, 0x26, 0x43, 0x2e, 0x76, 0x6e, 0x6f, 0x64, 0x65, 0x28, 0x5f, + 0x29, 0x2c, 0x69, 0x74, 0x28, 0x72, 0x2c, 0x5f, 0x2c, 0x69, 0x2c, 0x74, + 0x2e, 0x5f, 0x5f, 0x6e, 0x2c, 0x76, 0x6f, 0x69, 0x64, 0x20, 0x30, 0x21, + 0x3d, 0x3d, 0x72, 0x2e, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x53, 0x56, 0x47, + 0x45, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2c, 0x33, 0x32, 0x26, 0x69, + 0x2e, 0x5f, 0x5f, 0x75, 0x3f, 0x5b, 0x6f, 0x5d, 0x3a, 0x6e, 0x75, 0x6c, + 0x6c, 0x2c, 0x6e, 0x2c, 0x6e, 0x75, 0x6c, 0x6c, 0x3d, 0x3d, 0x6f, 0x3f, + 0x71, 0x28, 0x69, 0x29, 0x3a, 0x6f, 0x2c, 0x21, 0x21, 0x28, 0x33, 0x32, + 0x26, 0x69, 0x2e, 0x5f, 0x5f, 0x75, 0x29, 0x2c, 0x65, 0x29, 0x2c, 0x5f, + 0x2e, 0x5f, 0x5f, 0x76, 0x3d, 0x69, 0x2e, 0x5f, 0x5f, 0x76, 0x2c, 0x5f, + 0x2e, 0x5f, 0x5f, 0x2e, 0x5f, 0x5f, 0x6b, 0x5b, 0x5f, 0x2e, 0x5f, 0x5f, + 0x69, 0x5d, 0x3d, 0x5f, 0x2c, 0x5f, 0x2e, 0x5f, 0x5f, 0x64, 0x3d, 0x76, + 0x6f, 0x69, 0x64, 0x20, 0x30, 0x2c, 0x5f, 0x2e, 0x5f, 0x5f, 0x65, 0x21, + 0x3d, 0x6f, 0x26, 0x26, 0x47, 0x28, 0x5f, 0x29, 0x2c, 0x5f, 0x7d, 0x66, + 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x47, 0x28, 0x74, 0x29, + 0x7b, 0x76, 0x61, 0x72, 0x20, 0x6e, 0x2c, 0x65, 0x3b, 0x69, 0x66, 0x28, + 0x6e, 0x75, 0x6c, 0x6c, 0x21, 0x3d, 0x28, 0x74, 0x3d, 0x74, 0x2e, 0x5f, + 0x5f, 0x29, 0x26, 0x26, 0x6e, 0x75, 0x6c, 0x6c, 0x21, 0x3d, 0x74, 0x2e, + 0x5f, 0x5f, 0x63, 0x29, 0x7b, 0x66, 0x6f, 0x72, 0x28, 0x74, 0x2e, 0x5f, + 0x5f, 0x65, 0x3d, 0x74, 0x2e, 0x5f, 0x5f, 0x63, 0x2e, 0x62, 0x61, 0x73, + 0x65, 0x3d, 0x6e, 0x75, 0x6c, 0x6c, 0x2c, 0x6e, 0x3d, 0x30, 0x3b, 0x6e, + 0x3c, 0x74, 0x2e, 0x5f, 0x5f, 0x6b, 0x2e, 0x6c, 0x65, 0x6e, 0x67, 0x74, + 0x68, 0x3b, 0x6e, 0x2b, 0x2b, 0x29, 0x69, 0x66, 0x28, 0x6e, 0x75, 0x6c, + 0x6c, 0x21, 0x3d, 0x28, 0x65, 0x3d, 0x74, 0x2e, 0x5f, 0x5f, 0x6b, 0x5b, + 0x6e, 0x5d, 0x29, 0x26, 0x26, 0x6e, 0x75, 0x6c, 0x6c, 0x21, 0x3d, 0x65, + 0x2e, 0x5f, 0x5f, 0x65, 0x29, 0x7b, 0x74, 0x2e, 0x5f, 0x5f, 0x65, 0x3d, + 0x74, 0x2e, 0x5f, 0x5f, 0x63, 0x2e, 0x62, 0x61, 0x73, 0x65, 0x3d, 0x65, + 0x2e, 0x5f, 0x5f, 0x65, 0x3b, 0x62, 0x72, 0x65, 0x61, 0x6b, 0x7d, 0x72, + 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x47, 0x28, 0x74, 0x29, 0x7d, 0x7d, + 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x7a, 0x28, 0x74, + 0x29, 0x7b, 0x28, 0x21, 0x74, 0x2e, 0x5f, 0x5f, 0x64, 0x26, 0x26, 0x28, + 0x74, 0x2e, 0x5f, 0x5f, 0x64, 0x3d, 0x21, 0x30, 0x29, 0x26, 0x26, 0x48, + 0x2e, 0x70, 0x75, 0x73, 0x68, 0x28, 0x74, 0x29, 0x26, 0x26, 0x21, 0x4a, + 0x2e, 0x5f, 0x5f, 0x72, 0x2b, 0x2b, 0x7c, 0x7c, 0x50, 0x21, 0x3d, 0x3d, + 0x43, 0x2e, 0x64, 0x65, 0x62, 0x6f, 0x75, 0x6e, 0x63, 0x65, 0x52, 0x65, + 0x6e, 0x64, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x29, 0x26, 0x26, 0x28, 0x28, + 0x50, 0x3d, 0x43, 0x2e, 0x64, 0x65, 0x62, 0x6f, 0x75, 0x6e, 0x63, 0x65, + 0x52, 0x65, 0x6e, 0x64, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x29, 0x7c, 0x7c, + 0x4e, 0x29, 0x28, 0x4a, 0x29, 0x7d, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, + 0x6f, 0x6e, 0x20, 0x4a, 0x28, 0x29, 0x7b, 0x76, 0x61, 0x72, 0x20, 0x74, + 0x2c, 0x6e, 0x2c, 0x65, 0x2c, 0x5f, 0x3d, 0x5b, 0x5d, 0x2c, 0x69, 0x3d, + 0x5b, 0x5d, 0x3b, 0x66, 0x6f, 0x72, 0x28, 0x48, 0x2e, 0x73, 0x6f, 0x72, + 0x74, 0x28, 0x24, 0x29, 0x3b, 0x74, 0x3d, 0x48, 0x2e, 0x73, 0x68, 0x69, + 0x66, 0x74, 0x28, 0x29, 0x3b, 0x29, 0x74, 0x2e, 0x5f, 0x5f, 0x64, 0x26, + 0x26, 0x28, 0x65, 0x3d, 0x48, 0x2e, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, + 0x2c, 0x6e, 0x3d, 0x42, 0x28, 0x74, 0x2c, 0x5f, 0x2c, 0x69, 0x29, 0x7c, + 0x7c, 0x6e, 0x2c, 0x30, 0x3d, 0x3d, 0x3d, 0x65, 0x7c, 0x7c, 0x48, 0x2e, + 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x3e, 0x65, 0x3f, 0x28, 0x6f, 0x74, + 0x28, 0x5f, 0x2c, 0x6e, 0x2c, 0x69, 0x29, 0x2c, 0x69, 0x2e, 0x6c, 0x65, + 0x6e, 0x67, 0x74, 0x68, 0x3d, 0x5f, 0x2e, 0x6c, 0x65, 0x6e, 0x67, 0x74, + 0x68, 0x3d, 0x30, 0x2c, 0x6e, 0x3d, 0x76, 0x6f, 0x69, 0x64, 0x20, 0x30, + 0x2c, 0x48, 0x2e, 0x73, 0x6f, 0x72, 0x74, 0x28, 0x24, 0x29, 0x29, 0x3a, + 0x6e, 0x26, 0x26, 0x43, 0x2e, 0x5f, 0x5f, 0x63, 0x26, 0x26, 0x43, 0x2e, + 0x5f, 0x5f, 0x63, 0x28, 0x6e, 0x2c, 0x56, 0x29, 0x29, 0x3b, 0x6e, 0x26, + 0x26, 0x6f, 0x74, 0x28, 0x5f, 0x2c, 0x6e, 0x2c, 0x69, 0x29, 0x2c, 0x4a, + 0x2e, 0x5f, 0x5f, 0x72, 0x3d, 0x30, 0x7d, 0x66, 0x75, 0x6e, 0x63, 0x74, + 0x69, 0x6f, 0x6e, 0x20, 0x4b, 0x28, 0x74, 0x2c, 0x6e, 0x2c, 0x65, 0x2c, + 0x5f, 0x2c, 0x69, 0x2c, 0x6f, 0x2c, 0x72, 0x2c, 0x75, 0x2c, 0x66, 0x2c, + 0x6c, 0x2c, 0x73, 0x29, 0x7b, 0x76, 0x61, 0x72, 0x20, 0x63, 0x2c, 0x68, + 0x2c, 0x61, 0x2c, 0x70, 0x2c, 0x64, 0x2c, 0x76, 0x3d, 0x5f, 0x26, 0x26, + 0x5f, 0x2e, 0x5f, 0x5f, 0x6b, 0x7c, 0x7c, 0x56, 0x2c, 0x79, 0x3d, 0x6e, + 0x2e, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x3b, 0x66, 0x6f, 0x72, 0x28, + 0x65, 0x2e, 0x5f, 0x5f, 0x64, 0x3d, 0x66, 0x2c, 0x51, 0x28, 0x65, 0x2c, + 0x6e, 0x2c, 0x76, 0x29, 0x2c, 0x66, 0x3d, 0x65, 0x2e, 0x5f, 0x5f, 0x64, + 0x2c, 0x63, 0x3d, 0x30, 0x3b, 0x63, 0x3c, 0x79, 0x3b, 0x63, 0x2b, 0x2b, + 0x29, 0x6e, 0x75, 0x6c, 0x6c, 0x21, 0x3d, 0x28, 0x61, 0x3d, 0x65, 0x2e, + 0x5f, 0x5f, 0x6b, 0x5b, 0x63, 0x5d, 0x29, 0x26, 0x26, 0x22, 0x62, 0x6f, + 0x6f, 0x6c, 0x65, 0x61, 0x6e, 0x22, 0x21, 0x3d, 0x74, 0x79, 0x70, 0x65, + 0x6f, 0x66, 0x20, 0x61, 0x26, 0x26, 0x22, 0x66, 0x75, 0x6e, 0x63, 0x74, + 0x69, 0x6f, 0x6e, 0x22, 0x21, 0x3d, 0x74, 0x79, 0x70, 0x65, 0x6f, 0x66, + 0x20, 0x61, 0x26, 0x26, 0x28, 0x68, 0x3d, 0x2d, 0x31, 0x3d, 0x3d, 0x3d, + 0x61, 0x2e, 0x5f, 0x5f, 0x69, 0x3f, 0x54, 0x3a, 0x76, 0x5b, 0x61, 0x2e, + 0x5f, 0x5f, 0x69, 0x5d, 0x7c, 0x7c, 0x54, 0x2c, 0x61, 0x2e, 0x5f, 0x5f, + 0x69, 0x3d, 0x63, 0x2c, 0x69, 0x74, 0x28, 0x74, 0x2c, 0x61, 0x2c, 0x68, + 0x2c, 0x69, 0x2c, 0x6f, 0x2c, 0x72, 0x2c, 0x75, 0x2c, 0x66, 0x2c, 0x6c, + 0x2c, 0x73, 0x29, 0x2c, 0x70, 0x3d, 0x61, 0x2e, 0x5f, 0x5f, 0x65, 0x2c, + 0x61, 0x2e, 0x72, 0x65, 0x66, 0x26, 0x26, 0x68, 0x2e, 0x72, 0x65, 0x66, + 0x21, 0x3d, 0x61, 0x2e, 0x72, 0x65, 0x66, 0x26, 0x26, 0x28, 0x68, 0x2e, + 0x72, 0x65, 0x66, 0x26, 0x26, 0x75, 0x74, 0x28, 0x68, 0x2e, 0x72, 0x65, + 0x66, 0x2c, 0x6e, 0x75, 0x6c, 0x6c, 0x2c, 0x61, 0x29, 0x2c, 0x73, 0x2e, + 0x70, 0x75, 0x73, 0x68, 0x28, 0x61, 0x2e, 0x72, 0x65, 0x66, 0x2c, 0x61, + 0x2e, 0x5f, 0x5f, 0x63, 0x7c, 0x7c, 0x70, 0x2c, 0x61, 0x29, 0x29, 0x2c, + 0x6e, 0x75, 0x6c, 0x6c, 0x3d, 0x3d, 0x64, 0x26, 0x26, 0x6e, 0x75, 0x6c, + 0x6c, 0x21, 0x3d, 0x70, 0x26, 0x26, 0x28, 0x64, 0x3d, 0x70, 0x29, 0x2c, + 0x36, 0x35, 0x35, 0x33, 0x36, 0x26, 0x61, 0x2e, 0x5f, 0x5f, 0x75, 0x7c, + 0x7c, 0x68, 0x2e, 0x5f, 0x5f, 0x6b, 0x3d, 0x3d, 0x3d, 0x61, 0x2e, 0x5f, + 0x5f, 0x6b, 0x3f, 0x66, 0x3d, 0x58, 0x28, 0x61, 0x2c, 0x66, 0x2c, 0x74, + 0x29, 0x3a, 0x22, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x22, + 0x3d, 0x3d, 0x74, 0x79, 0x70, 0x65, 0x6f, 0x66, 0x20, 0x61, 0x2e, 0x74, + 0x79, 0x70, 0x65, 0x26, 0x26, 0x76, 0x6f, 0x69, 0x64, 0x20, 0x30, 0x21, + 0x3d, 0x3d, 0x61, 0x2e, 0x5f, 0x5f, 0x64, 0x3f, 0x66, 0x3d, 0x61, 0x2e, + 0x5f, 0x5f, 0x64, 0x3a, 0x70, 0x26, 0x26, 0x28, 0x66, 0x3d, 0x70, 0x2e, + 0x6e, 0x65, 0x78, 0x74, 0x53, 0x69, 0x62, 0x6c, 0x69, 0x6e, 0x67, 0x29, + 0x2c, 0x61, 0x2e, 0x5f, 0x5f, 0x64, 0x3d, 0x76, 0x6f, 0x69, 0x64, 0x20, + 0x30, 0x2c, 0x61, 0x2e, 0x5f, 0x5f, 0x75, 0x26, 0x3d, 0x2d, 0x31, 0x39, + 0x36, 0x36, 0x30, 0x39, 0x29, 0x3b, 0x65, 0x2e, 0x5f, 0x5f, 0x64, 0x3d, + 0x66, 0x2c, 0x65, 0x2e, 0x5f, 0x5f, 0x65, 0x3d, 0x64, 0x7d, 0x66, 0x75, + 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x51, 0x28, 0x74, 0x2c, 0x6e, + 0x2c, 0x65, 0x29, 0x7b, 0x76, 0x61, 0x72, 0x20, 0x5f, 0x2c, 0x69, 0x2c, + 0x6f, 0x2c, 0x72, 0x2c, 0x75, 0x2c, 0x66, 0x3d, 0x6e, 0x2e, 0x6c, 0x65, + 0x6e, 0x67, 0x74, 0x68, 0x2c, 0x6c, 0x3d, 0x65, 0x2e, 0x6c, 0x65, 0x6e, + 0x67, 0x74, 0x68, 0x2c, 0x73, 0x3d, 0x6c, 0x2c, 0x63, 0x3d, 0x30, 0x3b, + 0x66, 0x6f, 0x72, 0x28, 0x74, 0x2e, 0x5f, 0x5f, 0x6b, 0x3d, 0x5b, 0x5d, + 0x2c, 0x5f, 0x3d, 0x30, 0x3b, 0x5f, 0x3c, 0x66, 0x3b, 0x5f, 0x2b, 0x2b, + 0x29, 0x72, 0x3d, 0x5f, 0x2b, 0x63, 0x2c, 0x6e, 0x75, 0x6c, 0x6c, 0x21, + 0x3d, 0x28, 0x69, 0x3d, 0x74, 0x2e, 0x5f, 0x5f, 0x6b, 0x5b, 0x5f, 0x5d, + 0x3d, 0x6e, 0x75, 0x6c, 0x6c, 0x3d, 0x3d, 0x28, 0x69, 0x3d, 0x6e, 0x5b, + 0x5f, 0x5d, 0x29, 0x7c, 0x7c, 0x22, 0x62, 0x6f, 0x6f, 0x6c, 0x65, 0x61, + 0x6e, 0x22, 0x3d, 0x3d, 0x74, 0x79, 0x70, 0x65, 0x6f, 0x66, 0x20, 0x69, + 0x7c, 0x7c, 0x22, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x22, + 0x3d, 0x3d, 0x74, 0x79, 0x70, 0x65, 0x6f, 0x66, 0x20, 0x69, 0x3f, 0x6e, + 0x75, 0x6c, 0x6c, 0x3a, 0x22, 0x73, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x22, + 0x3d, 0x3d, 0x74, 0x79, 0x70, 0x65, 0x6f, 0x66, 0x20, 0x69, 0x7c, 0x7c, + 0x22, 0x6e, 0x75, 0x6d, 0x62, 0x65, 0x72, 0x22, 0x3d, 0x3d, 0x74, 0x79, + 0x70, 0x65, 0x6f, 0x66, 0x20, 0x69, 0x7c, 0x7c, 0x22, 0x62, 0x69, 0x67, + 0x69, 0x6e, 0x74, 0x22, 0x3d, 0x3d, 0x74, 0x79, 0x70, 0x65, 0x6f, 0x66, + 0x20, 0x69, 0x7c, 0x7c, 0x69, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x72, + 0x75, 0x63, 0x74, 0x6f, 0x72, 0x3d, 0x3d, 0x53, 0x74, 0x72, 0x69, 0x6e, + 0x67, 0x3f, 0x4f, 0x28, 0x6e, 0x75, 0x6c, 0x6c, 0x2c, 0x69, 0x2c, 0x6e, + 0x75, 0x6c, 0x6c, 0x2c, 0x6e, 0x75, 0x6c, 0x6c, 0x2c, 0x6e, 0x75, 0x6c, + 0x6c, 0x29, 0x3a, 0x46, 0x28, 0x69, 0x29, 0x3f, 0x4f, 0x28, 0x6a, 0x2c, + 0x7b, 0x63, 0x68, 0x69, 0x6c, 0x64, 0x72, 0x65, 0x6e, 0x3a, 0x69, 0x7d, + 0x2c, 0x6e, 0x75, 0x6c, 0x6c, 0x2c, 0x6e, 0x75, 0x6c, 0x6c, 0x2c, 0x6e, + 0x75, 0x6c, 0x6c, 0x29, 0x3a, 0x76, 0x6f, 0x69, 0x64, 0x20, 0x30, 0x3d, + 0x3d, 0x3d, 0x69, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x72, 0x75, 0x63, + 0x74, 0x6f, 0x72, 0x26, 0x26, 0x69, 0x2e, 0x5f, 0x5f, 0x62, 0x3e, 0x30, + 0x3f, 0x4f, 0x28, 0x69, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x2c, 0x69, 0x2e, + 0x70, 0x72, 0x6f, 0x70, 0x73, 0x2c, 0x69, 0x2e, 0x6b, 0x65, 0x79, 0x2c, + 0x69, 0x2e, 0x72, 0x65, 0x66, 0x3f, 0x69, 0x2e, 0x72, 0x65, 0x66, 0x3a, + 0x6e, 0x75, 0x6c, 0x6c, 0x2c, 0x69, 0x2e, 0x5f, 0x5f, 0x76, 0x29, 0x3a, + 0x69, 0x29, 0x3f, 0x28, 0x69, 0x2e, 0x5f, 0x5f, 0x3d, 0x74, 0x2c, 0x69, + 0x2e, 0x5f, 0x5f, 0x62, 0x3d, 0x74, 0x2e, 0x5f, 0x5f, 0x62, 0x2b, 0x31, + 0x2c, 0x75, 0x3d, 0x5a, 0x28, 0x69, 0x2c, 0x65, 0x2c, 0x72, 0x2c, 0x73, + 0x29, 0x2c, 0x69, 0x2e, 0x5f, 0x5f, 0x69, 0x3d, 0x75, 0x2c, 0x6f, 0x3d, + 0x6e, 0x75, 0x6c, 0x6c, 0x2c, 0x2d, 0x31, 0x21, 0x3d, 0x3d, 0x75, 0x26, + 0x26, 0x28, 0x73, 0x2d, 0x2d, 0x2c, 0x28, 0x6f, 0x3d, 0x65, 0x5b, 0x75, + 0x5d, 0x29, 0x26, 0x26, 0x28, 0x6f, 0x2e, 0x5f, 0x5f, 0x75, 0x7c, 0x3d, + 0x31, 0x33, 0x31, 0x30, 0x37, 0x32, 0x29, 0x29, 0x2c, 0x6e, 0x75, 0x6c, + 0x6c, 0x3d, 0x3d, 0x6f, 0x7c, 0x7c, 0x6e, 0x75, 0x6c, 0x6c, 0x3d, 0x3d, + 0x3d, 0x6f, 0x2e, 0x5f, 0x5f, 0x76, 0x3f, 0x28, 0x2d, 0x31, 0x3d, 0x3d, + 0x75, 0x26, 0x26, 0x63, 0x2d, 0x2d, 0x2c, 0x22, 0x66, 0x75, 0x6e, 0x63, + 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x21, 0x3d, 0x74, 0x79, 0x70, 0x65, 0x6f, + 0x66, 0x20, 0x69, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x26, 0x26, 0x28, 0x69, 0x2e, 0x5f, 0x5f, 0x75, 0x7c, 0x3d, 0x36, 0x35, 0x35, 0x33, 0x36, 0x29, - 0x29, 0x29, 0x3a, 0x28, 0x6f, 0x3d, 0x65, 0x5b, 0x5f, 0x5d, 0x29, 0x26, - 0x26, 0x6e, 0x75, 0x6c, 0x6c, 0x3d, 0x3d, 0x6f, 0x2e, 0x6b, 0x65, 0x79, - 0x26, 0x26, 0x6f, 0x2e, 0x5f, 0x5f, 0x65, 0x26, 0x26, 0x28, 0x6f, 0x2e, - 0x5f, 0x5f, 0x65, 0x3d, 0x3d, 0x74, 0x2e, 0x5f, 0x5f, 0x64, 0x26, 0x26, - 0x28, 0x74, 0x2e, 0x5f, 0x5f, 0x64, 0x3d, 0x71, 0x28, 0x6f, 0x29, 0x29, - 0x2c, 0x75, 0x74, 0x28, 0x6f, 0x2c, 0x6f, 0x2c, 0x21, 0x31, 0x29, 0x2c, - 0x65, 0x5b, 0x5f, 0x5d, 0x3d, 0x6e, 0x75, 0x6c, 0x6c, 0x2c, 0x6c, 0x2d, - 0x2d, 0x29, 0x3b, 0x69, 0x66, 0x28, 0x6c, 0x29, 0x66, 0x6f, 0x72, 0x28, - 0x5f, 0x3d, 0x30, 0x3b, 0x5f, 0x3c, 0x73, 0x3b, 0x5f, 0x2b, 0x2b, 0x29, - 0x6e, 0x75, 0x6c, 0x6c, 0x21, 0x3d, 0x28, 0x6f, 0x3d, 0x65, 0x5b, 0x5f, - 0x5d, 0x29, 0x26, 0x26, 0x30, 0x3d, 0x3d, 0x28, 0x31, 0x33, 0x31, 0x30, - 0x37, 0x32, 0x26, 0x6f, 0x2e, 0x5f, 0x5f, 0x75, 0x29, 0x26, 0x26, 0x28, - 0x6f, 0x2e, 0x5f, 0x5f, 0x65, 0x3d, 0x3d, 0x74, 0x2e, 0x5f, 0x5f, 0x64, - 0x26, 0x26, 0x28, 0x74, 0x2e, 0x5f, 0x5f, 0x64, 0x3d, 0x71, 0x28, 0x6f, - 0x29, 0x29, 0x2c, 0x75, 0x74, 0x28, 0x6f, 0x2c, 0x6f, 0x29, 0x29, 0x7d, - 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x51, 0x28, 0x74, - 0x2c, 0x6e, 0x2c, 0x65, 0x29, 0x7b, 0x76, 0x61, 0x72, 0x20, 0x5f, 0x2c, - 0x69, 0x3b, 0x69, 0x66, 0x28, 0x22, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, - 0x6f, 0x6e, 0x22, 0x3d, 0x3d, 0x74, 0x79, 0x70, 0x65, 0x6f, 0x66, 0x20, - 0x74, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x29, 0x7b, 0x66, 0x6f, 0x72, 0x28, - 0x5f, 0x3d, 0x74, 0x2e, 0x5f, 0x5f, 0x6b, 0x2c, 0x69, 0x3d, 0x30, 0x3b, - 0x5f, 0x26, 0x26, 0x69, 0x3c, 0x5f, 0x2e, 0x6c, 0x65, 0x6e, 0x67, 0x74, - 0x68, 0x3b, 0x69, 0x2b, 0x2b, 0x29, 0x5f, 0x5b, 0x69, 0x5d, 0x26, 0x26, - 0x28, 0x5f, 0x5b, 0x69, 0x5d, 0x2e, 0x5f, 0x5f, 0x3d, 0x74, 0x2c, 0x6e, - 0x3d, 0x51, 0x28, 0x5f, 0x5b, 0x69, 0x5d, 0x2c, 0x6e, 0x2c, 0x65, 0x29, - 0x29, 0x3b, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x6e, 0x7d, 0x72, - 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x74, 0x2e, 0x5f, 0x5f, 0x65, 0x21, - 0x3d, 0x6e, 0x26, 0x26, 0x28, 0x65, 0x2e, 0x69, 0x6e, 0x73, 0x65, 0x72, - 0x74, 0x42, 0x65, 0x66, 0x6f, 0x72, 0x65, 0x28, 0x74, 0x2e, 0x5f, 0x5f, - 0x65, 0x2c, 0x6e, 0x7c, 0x7c, 0x6e, 0x75, 0x6c, 0x6c, 0x29, 0x2c, 0x6e, - 0x3d, 0x74, 0x2e, 0x5f, 0x5f, 0x65, 0x29, 0x2c, 0x6e, 0x26, 0x26, 0x6e, - 0x2e, 0x6e, 0x65, 0x78, 0x74, 0x53, 0x69, 0x62, 0x6c, 0x69, 0x6e, 0x67, + 0x29, 0x3a, 0x75, 0x21, 0x3d, 0x3d, 0x72, 0x26, 0x26, 0x28, 0x75, 0x3d, + 0x3d, 0x3d, 0x72, 0x2b, 0x31, 0x3f, 0x63, 0x2b, 0x2b, 0x3a, 0x75, 0x3e, + 0x72, 0x3f, 0x73, 0x3e, 0x66, 0x2d, 0x72, 0x3f, 0x63, 0x2b, 0x3d, 0x75, + 0x2d, 0x72, 0x3a, 0x63, 0x2d, 0x2d, 0x3a, 0x75, 0x3c, 0x72, 0x3f, 0x75, + 0x3d, 0x3d, 0x72, 0x2d, 0x31, 0x26, 0x26, 0x28, 0x63, 0x3d, 0x75, 0x2d, + 0x72, 0x29, 0x3a, 0x63, 0x3d, 0x30, 0x2c, 0x75, 0x21, 0x3d, 0x3d, 0x5f, + 0x2b, 0x63, 0x26, 0x26, 0x28, 0x69, 0x2e, 0x5f, 0x5f, 0x75, 0x7c, 0x3d, + 0x36, 0x35, 0x35, 0x33, 0x36, 0x29, 0x29, 0x29, 0x3a, 0x28, 0x6f, 0x3d, + 0x65, 0x5b, 0x72, 0x5d, 0x29, 0x26, 0x26, 0x6e, 0x75, 0x6c, 0x6c, 0x3d, + 0x3d, 0x6f, 0x2e, 0x6b, 0x65, 0x79, 0x26, 0x26, 0x6f, 0x2e, 0x5f, 0x5f, + 0x65, 0x26, 0x26, 0x30, 0x3d, 0x3d, 0x28, 0x31, 0x33, 0x31, 0x30, 0x37, + 0x32, 0x26, 0x6f, 0x2e, 0x5f, 0x5f, 0x75, 0x29, 0x26, 0x26, 0x28, 0x6f, + 0x2e, 0x5f, 0x5f, 0x65, 0x3d, 0x3d, 0x74, 0x2e, 0x5f, 0x5f, 0x64, 0x26, + 0x26, 0x28, 0x74, 0x2e, 0x5f, 0x5f, 0x64, 0x3d, 0x71, 0x28, 0x6f, 0x29, + 0x29, 0x2c, 0x66, 0x74, 0x28, 0x6f, 0x2c, 0x6f, 0x2c, 0x21, 0x31, 0x29, + 0x2c, 0x65, 0x5b, 0x72, 0x5d, 0x3d, 0x6e, 0x75, 0x6c, 0x6c, 0x2c, 0x73, + 0x2d, 0x2d, 0x29, 0x3b, 0x69, 0x66, 0x28, 0x73, 0x29, 0x66, 0x6f, 0x72, + 0x28, 0x5f, 0x3d, 0x30, 0x3b, 0x5f, 0x3c, 0x6c, 0x3b, 0x5f, 0x2b, 0x2b, + 0x29, 0x6e, 0x75, 0x6c, 0x6c, 0x21, 0x3d, 0x28, 0x6f, 0x3d, 0x65, 0x5b, + 0x5f, 0x5d, 0x29, 0x26, 0x26, 0x30, 0x3d, 0x3d, 0x28, 0x31, 0x33, 0x31, + 0x30, 0x37, 0x32, 0x26, 0x6f, 0x2e, 0x5f, 0x5f, 0x75, 0x29, 0x26, 0x26, + 0x28, 0x6f, 0x2e, 0x5f, 0x5f, 0x65, 0x3d, 0x3d, 0x74, 0x2e, 0x5f, 0x5f, + 0x64, 0x26, 0x26, 0x28, 0x74, 0x2e, 0x5f, 0x5f, 0x64, 0x3d, 0x71, 0x28, + 0x6f, 0x29, 0x29, 0x2c, 0x66, 0x74, 0x28, 0x6f, 0x2c, 0x6f, 0x29, 0x29, 0x7d, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x58, 0x28, - 0x74, 0x2c, 0x6e, 0x29, 0x7b, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, - 0x6e, 0x3d, 0x6e, 0x7c, 0x7c, 0x5b, 0x5d, 0x2c, 0x6e, 0x75, 0x6c, 0x6c, - 0x3d, 0x3d, 0x74, 0x7c, 0x7c, 0x22, 0x62, 0x6f, 0x6f, 0x6c, 0x65, 0x61, - 0x6e, 0x22, 0x3d, 0x3d, 0x74, 0x79, 0x70, 0x65, 0x6f, 0x66, 0x20, 0x74, - 0x7c, 0x7c, 0x28, 0x46, 0x28, 0x74, 0x29, 0x3f, 0x74, 0x2e, 0x73, 0x6f, - 0x6d, 0x65, 0x28, 0x28, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, - 0x28, 0x74, 0x29, 0x7b, 0x58, 0x28, 0x74, 0x2c, 0x6e, 0x29, 0x7d, 0x29, - 0x29, 0x3a, 0x6e, 0x2e, 0x70, 0x75, 0x73, 0x68, 0x28, 0x74, 0x29, 0x29, - 0x2c, 0x6e, 0x7d, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, - 0x59, 0x28, 0x74, 0x2c, 0x6e, 0x2c, 0x65, 0x2c, 0x5f, 0x29, 0x7b, 0x76, - 0x61, 0x72, 0x20, 0x69, 0x3d, 0x74, 0x2e, 0x6b, 0x65, 0x79, 0x2c, 0x6f, - 0x3d, 0x74, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x2c, 0x72, 0x3d, 0x65, 0x2d, - 0x31, 0x2c, 0x75, 0x3d, 0x65, 0x2b, 0x31, 0x2c, 0x66, 0x3d, 0x6e, 0x5b, - 0x65, 0x5d, 0x3b, 0x69, 0x66, 0x28, 0x6e, 0x75, 0x6c, 0x6c, 0x3d, 0x3d, - 0x3d, 0x66, 0x7c, 0x7c, 0x66, 0x26, 0x26, 0x69, 0x3d, 0x3d, 0x66, 0x2e, - 0x6b, 0x65, 0x79, 0x26, 0x26, 0x6f, 0x3d, 0x3d, 0x3d, 0x66, 0x2e, 0x74, - 0x79, 0x70, 0x65, 0x29, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x65, - 0x3b, 0x69, 0x66, 0x28, 0x5f, 0x3e, 0x28, 0x6e, 0x75, 0x6c, 0x6c, 0x21, - 0x3d, 0x66, 0x26, 0x26, 0x30, 0x3d, 0x3d, 0x28, 0x31, 0x33, 0x31, 0x30, - 0x37, 0x32, 0x26, 0x66, 0x2e, 0x5f, 0x5f, 0x75, 0x29, 0x3f, 0x31, 0x3a, - 0x30, 0x29, 0x29, 0x66, 0x6f, 0x72, 0x28, 0x3b, 0x72, 0x3e, 0x3d, 0x30, - 0x7c, 0x7c, 0x75, 0x3c, 0x6e, 0x2e, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, - 0x3b, 0x29, 0x7b, 0x69, 0x66, 0x28, 0x72, 0x3e, 0x3d, 0x30, 0x29, 0x7b, - 0x69, 0x66, 0x28, 0x28, 0x66, 0x3d, 0x6e, 0x5b, 0x72, 0x5d, 0x29, 0x26, - 0x26, 0x30, 0x3d, 0x3d, 0x28, 0x31, 0x33, 0x31, 0x30, 0x37, 0x32, 0x26, - 0x66, 0x2e, 0x5f, 0x5f, 0x75, 0x29, 0x26, 0x26, 0x69, 0x3d, 0x3d, 0x66, - 0x2e, 0x6b, 0x65, 0x79, 0x26, 0x26, 0x6f, 0x3d, 0x3d, 0x3d, 0x66, 0x2e, - 0x74, 0x79, 0x70, 0x65, 0x29, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, - 0x72, 0x3b, 0x72, 0x2d, 0x2d, 0x7d, 0x69, 0x66, 0x28, 0x75, 0x3c, 0x6e, - 0x2e, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x29, 0x7b, 0x69, 0x66, 0x28, - 0x28, 0x66, 0x3d, 0x6e, 0x5b, 0x75, 0x5d, 0x29, 0x26, 0x26, 0x30, 0x3d, - 0x3d, 0x28, 0x31, 0x33, 0x31, 0x30, 0x37, 0x32, 0x26, 0x66, 0x2e, 0x5f, - 0x5f, 0x75, 0x29, 0x26, 0x26, 0x69, 0x3d, 0x3d, 0x66, 0x2e, 0x6b, 0x65, - 0x79, 0x26, 0x26, 0x6f, 0x3d, 0x3d, 0x3d, 0x66, 0x2e, 0x74, 0x79, 0x70, - 0x65, 0x29, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x75, 0x3b, 0x75, - 0x2b, 0x2b, 0x7d, 0x7d, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x2d, 0x31, - 0x7d, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x5a, 0x28, - 0x74, 0x2c, 0x6e, 0x2c, 0x65, 0x29, 0x7b, 0x22, 0x2d, 0x22, 0x3d, 0x3d, - 0x3d, 0x6e, 0x5b, 0x30, 0x5d, 0x3f, 0x74, 0x2e, 0x73, 0x65, 0x74, 0x50, - 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x79, 0x28, 0x6e, 0x2c, 0x6e, 0x75, - 0x6c, 0x6c, 0x3d, 0x3d, 0x65, 0x3f, 0x22, 0x22, 0x3a, 0x65, 0x29, 0x3a, - 0x74, 0x5b, 0x6e, 0x5d, 0x3d, 0x6e, 0x75, 0x6c, 0x6c, 0x3d, 0x3d, 0x65, - 0x3f, 0x22, 0x22, 0x3a, 0x22, 0x6e, 0x75, 0x6d, 0x62, 0x65, 0x72, 0x22, - 0x21, 0x3d, 0x74, 0x79, 0x70, 0x65, 0x6f, 0x66, 0x20, 0x65, 0x7c, 0x7c, - 0x41, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x28, 0x6e, 0x29, 0x3f, 0x65, 0x3a, - 0x65, 0x2b, 0x22, 0x70, 0x78, 0x22, 0x7d, 0x66, 0x75, 0x6e, 0x63, 0x74, - 0x69, 0x6f, 0x6e, 0x20, 0x74, 0x74, 0x28, 0x74, 0x2c, 0x6e, 0x2c, 0x65, - 0x2c, 0x5f, 0x2c, 0x69, 0x29, 0x7b, 0x76, 0x61, 0x72, 0x20, 0x6f, 0x3b, - 0x74, 0x3a, 0x69, 0x66, 0x28, 0x22, 0x73, 0x74, 0x79, 0x6c, 0x65, 0x22, - 0x3d, 0x3d, 0x3d, 0x6e, 0x29, 0x69, 0x66, 0x28, 0x22, 0x73, 0x74, 0x72, - 0x69, 0x6e, 0x67, 0x22, 0x3d, 0x3d, 0x74, 0x79, 0x70, 0x65, 0x6f, 0x66, - 0x20, 0x65, 0x29, 0x74, 0x2e, 0x73, 0x74, 0x79, 0x6c, 0x65, 0x2e, 0x63, - 0x73, 0x73, 0x54, 0x65, 0x78, 0x74, 0x3d, 0x65, 0x3b, 0x65, 0x6c, 0x73, - 0x65, 0x7b, 0x69, 0x66, 0x28, 0x22, 0x73, 0x74, 0x72, 0x69, 0x6e, 0x67, - 0x22, 0x3d, 0x3d, 0x74, 0x79, 0x70, 0x65, 0x6f, 0x66, 0x20, 0x5f, 0x26, - 0x26, 0x28, 0x74, 0x2e, 0x73, 0x74, 0x79, 0x6c, 0x65, 0x2e, 0x63, 0x73, - 0x73, 0x54, 0x65, 0x78, 0x74, 0x3d, 0x5f, 0x3d, 0x22, 0x22, 0x29, 0x2c, - 0x5f, 0x29, 0x66, 0x6f, 0x72, 0x28, 0x6e, 0x20, 0x69, 0x6e, 0x20, 0x5f, - 0x29, 0x65, 0x26, 0x26, 0x6e, 0x20, 0x69, 0x6e, 0x20, 0x65, 0x7c, 0x7c, - 0x5a, 0x28, 0x74, 0x2e, 0x73, 0x74, 0x79, 0x6c, 0x65, 0x2c, 0x6e, 0x2c, + 0x74, 0x2c, 0x6e, 0x2c, 0x65, 0x29, 0x7b, 0x76, 0x61, 0x72, 0x20, 0x5f, + 0x2c, 0x69, 0x3b, 0x69, 0x66, 0x28, 0x22, 0x66, 0x75, 0x6e, 0x63, 0x74, + 0x69, 0x6f, 0x6e, 0x22, 0x3d, 0x3d, 0x74, 0x79, 0x70, 0x65, 0x6f, 0x66, + 0x20, 0x74, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x29, 0x7b, 0x66, 0x6f, 0x72, + 0x28, 0x5f, 0x3d, 0x74, 0x2e, 0x5f, 0x5f, 0x6b, 0x2c, 0x69, 0x3d, 0x30, + 0x3b, 0x5f, 0x26, 0x26, 0x69, 0x3c, 0x5f, 0x2e, 0x6c, 0x65, 0x6e, 0x67, + 0x74, 0x68, 0x3b, 0x69, 0x2b, 0x2b, 0x29, 0x5f, 0x5b, 0x69, 0x5d, 0x26, + 0x26, 0x28, 0x5f, 0x5b, 0x69, 0x5d, 0x2e, 0x5f, 0x5f, 0x3d, 0x74, 0x2c, + 0x6e, 0x3d, 0x58, 0x28, 0x5f, 0x5b, 0x69, 0x5d, 0x2c, 0x6e, 0x2c, 0x65, + 0x29, 0x29, 0x3b, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x6e, 0x7d, + 0x74, 0x2e, 0x5f, 0x5f, 0x65, 0x21, 0x3d, 0x6e, 0x26, 0x26, 0x28, 0x65, + 0x2e, 0x69, 0x6e, 0x73, 0x65, 0x72, 0x74, 0x42, 0x65, 0x66, 0x6f, 0x72, + 0x65, 0x28, 0x74, 0x2e, 0x5f, 0x5f, 0x65, 0x2c, 0x6e, 0x7c, 0x7c, 0x6e, + 0x75, 0x6c, 0x6c, 0x29, 0x2c, 0x6e, 0x3d, 0x74, 0x2e, 0x5f, 0x5f, 0x65, + 0x29, 0x3b, 0x64, 0x6f, 0x7b, 0x6e, 0x3d, 0x6e, 0x26, 0x26, 0x6e, 0x2e, + 0x6e, 0x65, 0x78, 0x74, 0x53, 0x69, 0x62, 0x6c, 0x69, 0x6e, 0x67, 0x7d, + 0x77, 0x68, 0x69, 0x6c, 0x65, 0x28, 0x6e, 0x75, 0x6c, 0x6c, 0x21, 0x3d, + 0x6e, 0x26, 0x26, 0x38, 0x3d, 0x3d, 0x3d, 0x6e, 0x2e, 0x6e, 0x6f, 0x64, + 0x65, 0x54, 0x79, 0x70, 0x65, 0x29, 0x3b, 0x72, 0x65, 0x74, 0x75, 0x72, + 0x6e, 0x20, 0x6e, 0x7d, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, + 0x20, 0x59, 0x28, 0x74, 0x2c, 0x6e, 0x29, 0x7b, 0x72, 0x65, 0x74, 0x75, + 0x72, 0x6e, 0x20, 0x6e, 0x3d, 0x6e, 0x7c, 0x7c, 0x5b, 0x5d, 0x2c, 0x6e, + 0x75, 0x6c, 0x6c, 0x3d, 0x3d, 0x74, 0x7c, 0x7c, 0x22, 0x62, 0x6f, 0x6f, + 0x6c, 0x65, 0x61, 0x6e, 0x22, 0x3d, 0x3d, 0x74, 0x79, 0x70, 0x65, 0x6f, + 0x66, 0x20, 0x74, 0x7c, 0x7c, 0x28, 0x46, 0x28, 0x74, 0x29, 0x3f, 0x74, + 0x2e, 0x73, 0x6f, 0x6d, 0x65, 0x28, 0x28, 0x66, 0x75, 0x6e, 0x63, 0x74, + 0x69, 0x6f, 0x6e, 0x28, 0x74, 0x29, 0x7b, 0x59, 0x28, 0x74, 0x2c, 0x6e, + 0x29, 0x7d, 0x29, 0x29, 0x3a, 0x6e, 0x2e, 0x70, 0x75, 0x73, 0x68, 0x28, + 0x74, 0x29, 0x29, 0x2c, 0x6e, 0x7d, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, + 0x6f, 0x6e, 0x20, 0x5a, 0x28, 0x74, 0x2c, 0x6e, 0x2c, 0x65, 0x2c, 0x5f, + 0x29, 0x7b, 0x76, 0x61, 0x72, 0x20, 0x69, 0x3d, 0x74, 0x2e, 0x6b, 0x65, + 0x79, 0x2c, 0x6f, 0x3d, 0x74, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x2c, 0x72, + 0x3d, 0x65, 0x2d, 0x31, 0x2c, 0x75, 0x3d, 0x65, 0x2b, 0x31, 0x2c, 0x66, + 0x3d, 0x6e, 0x5b, 0x65, 0x5d, 0x3b, 0x69, 0x66, 0x28, 0x6e, 0x75, 0x6c, + 0x6c, 0x3d, 0x3d, 0x3d, 0x66, 0x7c, 0x7c, 0x66, 0x26, 0x26, 0x69, 0x3d, + 0x3d, 0x66, 0x2e, 0x6b, 0x65, 0x79, 0x26, 0x26, 0x6f, 0x3d, 0x3d, 0x3d, + 0x66, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x26, 0x26, 0x30, 0x3d, 0x3d, 0x28, + 0x31, 0x33, 0x31, 0x30, 0x37, 0x32, 0x26, 0x66, 0x2e, 0x5f, 0x5f, 0x75, + 0x29, 0x29, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x65, 0x3b, 0x69, + 0x66, 0x28, 0x5f, 0x3e, 0x28, 0x6e, 0x75, 0x6c, 0x6c, 0x21, 0x3d, 0x66, + 0x26, 0x26, 0x30, 0x3d, 0x3d, 0x28, 0x31, 0x33, 0x31, 0x30, 0x37, 0x32, + 0x26, 0x66, 0x2e, 0x5f, 0x5f, 0x75, 0x29, 0x3f, 0x31, 0x3a, 0x30, 0x29, + 0x29, 0x66, 0x6f, 0x72, 0x28, 0x3b, 0x72, 0x3e, 0x3d, 0x30, 0x7c, 0x7c, + 0x75, 0x3c, 0x6e, 0x2e, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x3b, 0x29, + 0x7b, 0x69, 0x66, 0x28, 0x72, 0x3e, 0x3d, 0x30, 0x29, 0x7b, 0x69, 0x66, + 0x28, 0x28, 0x66, 0x3d, 0x6e, 0x5b, 0x72, 0x5d, 0x29, 0x26, 0x26, 0x30, + 0x3d, 0x3d, 0x28, 0x31, 0x33, 0x31, 0x30, 0x37, 0x32, 0x26, 0x66, 0x2e, + 0x5f, 0x5f, 0x75, 0x29, 0x26, 0x26, 0x69, 0x3d, 0x3d, 0x66, 0x2e, 0x6b, + 0x65, 0x79, 0x26, 0x26, 0x6f, 0x3d, 0x3d, 0x3d, 0x66, 0x2e, 0x74, 0x79, + 0x70, 0x65, 0x29, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x72, 0x3b, + 0x72, 0x2d, 0x2d, 0x7d, 0x69, 0x66, 0x28, 0x75, 0x3c, 0x6e, 0x2e, 0x6c, + 0x65, 0x6e, 0x67, 0x74, 0x68, 0x29, 0x7b, 0x69, 0x66, 0x28, 0x28, 0x66, + 0x3d, 0x6e, 0x5b, 0x75, 0x5d, 0x29, 0x26, 0x26, 0x30, 0x3d, 0x3d, 0x28, + 0x31, 0x33, 0x31, 0x30, 0x37, 0x32, 0x26, 0x66, 0x2e, 0x5f, 0x5f, 0x75, + 0x29, 0x26, 0x26, 0x69, 0x3d, 0x3d, 0x66, 0x2e, 0x6b, 0x65, 0x79, 0x26, + 0x26, 0x6f, 0x3d, 0x3d, 0x3d, 0x66, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x29, + 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x75, 0x3b, 0x75, 0x2b, 0x2b, + 0x7d, 0x7d, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x2d, 0x31, 0x7d, 0x66, + 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x74, 0x74, 0x28, 0x74, + 0x2c, 0x6e, 0x2c, 0x65, 0x29, 0x7b, 0x22, 0x2d, 0x22, 0x3d, 0x3d, 0x3d, + 0x6e, 0x5b, 0x30, 0x5d, 0x3f, 0x74, 0x2e, 0x73, 0x65, 0x74, 0x50, 0x72, + 0x6f, 0x70, 0x65, 0x72, 0x74, 0x79, 0x28, 0x6e, 0x2c, 0x6e, 0x75, 0x6c, + 0x6c, 0x3d, 0x3d, 0x65, 0x3f, 0x22, 0x22, 0x3a, 0x65, 0x29, 0x3a, 0x74, + 0x5b, 0x6e, 0x5d, 0x3d, 0x6e, 0x75, 0x6c, 0x6c, 0x3d, 0x3d, 0x65, 0x3f, + 0x22, 0x22, 0x3a, 0x22, 0x6e, 0x75, 0x6d, 0x62, 0x65, 0x72, 0x22, 0x21, + 0x3d, 0x74, 0x79, 0x70, 0x65, 0x6f, 0x66, 0x20, 0x65, 0x7c, 0x7c, 0x41, + 0x2e, 0x74, 0x65, 0x73, 0x74, 0x28, 0x6e, 0x29, 0x3f, 0x65, 0x3a, 0x65, + 0x2b, 0x22, 0x70, 0x78, 0x22, 0x7d, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, + 0x6f, 0x6e, 0x20, 0x6e, 0x74, 0x28, 0x74, 0x2c, 0x6e, 0x2c, 0x65, 0x2c, + 0x5f, 0x2c, 0x69, 0x29, 0x7b, 0x76, 0x61, 0x72, 0x20, 0x6f, 0x3b, 0x74, + 0x3a, 0x69, 0x66, 0x28, 0x22, 0x73, 0x74, 0x79, 0x6c, 0x65, 0x22, 0x3d, + 0x3d, 0x3d, 0x6e, 0x29, 0x69, 0x66, 0x28, 0x22, 0x73, 0x74, 0x72, 0x69, + 0x6e, 0x67, 0x22, 0x3d, 0x3d, 0x74, 0x79, 0x70, 0x65, 0x6f, 0x66, 0x20, + 0x65, 0x29, 0x74, 0x2e, 0x73, 0x74, 0x79, 0x6c, 0x65, 0x2e, 0x63, 0x73, + 0x73, 0x54, 0x65, 0x78, 0x74, 0x3d, 0x65, 0x3b, 0x65, 0x6c, 0x73, 0x65, + 0x7b, 0x69, 0x66, 0x28, 0x22, 0x73, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x22, + 0x3d, 0x3d, 0x74, 0x79, 0x70, 0x65, 0x6f, 0x66, 0x20, 0x5f, 0x26, 0x26, + 0x28, 0x74, 0x2e, 0x73, 0x74, 0x79, 0x6c, 0x65, 0x2e, 0x63, 0x73, 0x73, + 0x54, 0x65, 0x78, 0x74, 0x3d, 0x5f, 0x3d, 0x22, 0x22, 0x29, 0x2c, 0x5f, + 0x29, 0x66, 0x6f, 0x72, 0x28, 0x6e, 0x20, 0x69, 0x6e, 0x20, 0x5f, 0x29, + 0x65, 0x26, 0x26, 0x6e, 0x20, 0x69, 0x6e, 0x20, 0x65, 0x7c, 0x7c, 0x74, + 0x74, 0x28, 0x74, 0x2e, 0x73, 0x74, 0x79, 0x6c, 0x65, 0x2c, 0x6e, 0x2c, 0x22, 0x22, 0x29, 0x3b, 0x69, 0x66, 0x28, 0x65, 0x29, 0x66, 0x6f, 0x72, 0x28, 0x6e, 0x20, 0x69, 0x6e, 0x20, 0x65, 0x29, 0x5f, 0x26, 0x26, 0x65, 0x5b, 0x6e, 0x5d, 0x3d, 0x3d, 0x3d, 0x5f, 0x5b, 0x6e, 0x5d, 0x7c, 0x7c, - 0x5a, 0x28, 0x74, 0x2e, 0x73, 0x74, 0x79, 0x6c, 0x65, 0x2c, 0x6e, 0x2c, - 0x65, 0x5b, 0x6e, 0x5d, 0x29, 0x7d, 0x65, 0x6c, 0x73, 0x65, 0x20, 0x69, - 0x66, 0x28, 0x22, 0x6f, 0x22, 0x3d, 0x3d, 0x3d, 0x6e, 0x5b, 0x30, 0x5d, - 0x26, 0x26, 0x22, 0x6e, 0x22, 0x3d, 0x3d, 0x3d, 0x6e, 0x5b, 0x31, 0x5d, - 0x29, 0x6f, 0x3d, 0x6e, 0x21, 0x3d, 0x3d, 0x28, 0x6e, 0x3d, 0x6e, 0x2e, - 0x72, 0x65, 0x70, 0x6c, 0x61, 0x63, 0x65, 0x28, 0x2f, 0x28, 0x50, 0x6f, - 0x69, 0x6e, 0x74, 0x65, 0x72, 0x43, 0x61, 0x70, 0x74, 0x75, 0x72, 0x65, - 0x29, 0x24, 0x7c, 0x43, 0x61, 0x70, 0x74, 0x75, 0x72, 0x65, 0x24, 0x2f, - 0x2c, 0x22, 0x24, 0x31, 0x22, 0x29, 0x29, 0x2c, 0x6e, 0x3d, 0x6e, 0x2e, - 0x74, 0x6f, 0x4c, 0x6f, 0x77, 0x65, 0x72, 0x43, 0x61, 0x73, 0x65, 0x28, - 0x29, 0x69, 0x6e, 0x20, 0x74, 0x3f, 0x6e, 0x2e, 0x74, 0x6f, 0x4c, 0x6f, - 0x77, 0x65, 0x72, 0x43, 0x61, 0x73, 0x65, 0x28, 0x29, 0x2e, 0x73, 0x6c, - 0x69, 0x63, 0x65, 0x28, 0x32, 0x29, 0x3a, 0x6e, 0x2e, 0x73, 0x6c, 0x69, - 0x63, 0x65, 0x28, 0x32, 0x29, 0x2c, 0x74, 0x2e, 0x6c, 0x7c, 0x7c, 0x28, - 0x74, 0x2e, 0x6c, 0x3d, 0x7b, 0x7d, 0x29, 0x2c, 0x74, 0x2e, 0x6c, 0x5b, - 0x6e, 0x2b, 0x6f, 0x5d, 0x3d, 0x65, 0x2c, 0x65, 0x3f, 0x5f, 0x3f, 0x65, - 0x2e, 0x75, 0x3d, 0x5f, 0x2e, 0x75, 0x3a, 0x28, 0x65, 0x2e, 0x75, 0x3d, - 0x44, 0x61, 0x74, 0x65, 0x2e, 0x6e, 0x6f, 0x77, 0x28, 0x29, 0x2c, 0x74, - 0x2e, 0x61, 0x64, 0x64, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x4c, 0x69, 0x73, - 0x74, 0x65, 0x6e, 0x65, 0x72, 0x28, 0x6e, 0x2c, 0x6f, 0x3f, 0x65, 0x74, - 0x3a, 0x6e, 0x74, 0x2c, 0x6f, 0x29, 0x29, 0x3a, 0x74, 0x2e, 0x72, 0x65, - 0x6d, 0x6f, 0x76, 0x65, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x4c, 0x69, 0x73, - 0x74, 0x65, 0x6e, 0x65, 0x72, 0x28, 0x6e, 0x2c, 0x6f, 0x3f, 0x65, 0x74, - 0x3a, 0x6e, 0x74, 0x2c, 0x6f, 0x29, 0x3b, 0x65, 0x6c, 0x73, 0x65, 0x7b, - 0x69, 0x66, 0x28, 0x69, 0x29, 0x6e, 0x3d, 0x6e, 0x2e, 0x72, 0x65, 0x70, - 0x6c, 0x61, 0x63, 0x65, 0x28, 0x2f, 0x78, 0x6c, 0x69, 0x6e, 0x6b, 0x28, - 0x48, 0x7c, 0x3a, 0x68, 0x29, 0x2f, 0x2c, 0x22, 0x68, 0x22, 0x29, 0x2e, - 0x72, 0x65, 0x70, 0x6c, 0x61, 0x63, 0x65, 0x28, 0x2f, 0x73, 0x4e, 0x61, - 0x6d, 0x65, 0x24, 0x2f, 0x2c, 0x22, 0x73, 0x22, 0x29, 0x3b, 0x65, 0x6c, - 0x73, 0x65, 0x20, 0x69, 0x66, 0x28, 0x22, 0x77, 0x69, 0x64, 0x74, 0x68, - 0x22, 0x21, 0x3d, 0x3d, 0x6e, 0x26, 0x26, 0x22, 0x68, 0x65, 0x69, 0x67, - 0x68, 0x74, 0x22, 0x21, 0x3d, 0x3d, 0x6e, 0x26, 0x26, 0x22, 0x68, 0x72, - 0x65, 0x66, 0x22, 0x21, 0x3d, 0x3d, 0x6e, 0x26, 0x26, 0x22, 0x6c, 0x69, - 0x73, 0x74, 0x22, 0x21, 0x3d, 0x3d, 0x6e, 0x26, 0x26, 0x22, 0x66, 0x6f, - 0x72, 0x6d, 0x22, 0x21, 0x3d, 0x3d, 0x6e, 0x26, 0x26, 0x22, 0x74, 0x61, - 0x62, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x22, 0x21, 0x3d, 0x3d, 0x6e, 0x26, - 0x26, 0x22, 0x64, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x22, 0x21, - 0x3d, 0x3d, 0x6e, 0x26, 0x26, 0x22, 0x72, 0x6f, 0x77, 0x53, 0x70, 0x61, - 0x6e, 0x22, 0x21, 0x3d, 0x3d, 0x6e, 0x26, 0x26, 0x22, 0x63, 0x6f, 0x6c, - 0x53, 0x70, 0x61, 0x6e, 0x22, 0x21, 0x3d, 0x3d, 0x6e, 0x26, 0x26, 0x22, - 0x72, 0x6f, 0x6c, 0x65, 0x22, 0x21, 0x3d, 0x3d, 0x6e, 0x26, 0x26, 0x6e, - 0x20, 0x69, 0x6e, 0x20, 0x74, 0x29, 0x74, 0x72, 0x79, 0x7b, 0x74, 0x5b, - 0x6e, 0x5d, 0x3d, 0x6e, 0x75, 0x6c, 0x6c, 0x3d, 0x3d, 0x65, 0x3f, 0x22, - 0x22, 0x3a, 0x65, 0x3b, 0x62, 0x72, 0x65, 0x61, 0x6b, 0x20, 0x74, 0x7d, - 0x63, 0x61, 0x74, 0x63, 0x68, 0x28, 0x74, 0x29, 0x7b, 0x7d, 0x22, 0x66, - 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x3d, 0x3d, 0x74, 0x79, - 0x70, 0x65, 0x6f, 0x66, 0x20, 0x65, 0x7c, 0x7c, 0x28, 0x6e, 0x75, 0x6c, - 0x6c, 0x3d, 0x3d, 0x65, 0x7c, 0x7c, 0x21, 0x31, 0x3d, 0x3d, 0x3d, 0x65, - 0x26, 0x26, 0x22, 0x2d, 0x22, 0x21, 0x3d, 0x3d, 0x6e, 0x5b, 0x34, 0x5d, - 0x3f, 0x74, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x41, 0x74, 0x74, - 0x72, 0x69, 0x62, 0x75, 0x74, 0x65, 0x28, 0x6e, 0x29, 0x3a, 0x74, 0x2e, - 0x73, 0x65, 0x74, 0x41, 0x74, 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, 0x65, - 0x28, 0x6e, 0x2c, 0x65, 0x29, 0x29, 0x7d, 0x7d, 0x66, 0x75, 0x6e, 0x63, - 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x6e, 0x74, 0x28, 0x74, 0x29, 0x7b, 0x76, - 0x61, 0x72, 0x20, 0x6e, 0x3d, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x6c, 0x5b, - 0x74, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x2b, 0x21, 0x31, 0x5d, 0x3b, 0x69, - 0x66, 0x28, 0x74, 0x2e, 0x74, 0x29, 0x7b, 0x69, 0x66, 0x28, 0x74, 0x2e, - 0x74, 0x3c, 0x3d, 0x6e, 0x2e, 0x75, 0x29, 0x72, 0x65, 0x74, 0x75, 0x72, - 0x6e, 0x7d, 0x65, 0x6c, 0x73, 0x65, 0x20, 0x74, 0x2e, 0x74, 0x3d, 0x44, - 0x61, 0x74, 0x65, 0x2e, 0x6e, 0x6f, 0x77, 0x28, 0x29, 0x3b, 0x72, 0x65, - 0x74, 0x75, 0x72, 0x6e, 0x20, 0x6e, 0x28, 0x43, 0x2e, 0x65, 0x76, 0x65, - 0x6e, 0x74, 0x3f, 0x43, 0x2e, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x28, 0x74, - 0x29, 0x3a, 0x74, 0x29, 0x7d, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, - 0x6e, 0x20, 0x65, 0x74, 0x28, 0x74, 0x29, 0x7b, 0x72, 0x65, 0x74, 0x75, + 0x74, 0x74, 0x28, 0x74, 0x2e, 0x73, 0x74, 0x79, 0x6c, 0x65, 0x2c, 0x6e, + 0x2c, 0x65, 0x5b, 0x6e, 0x5d, 0x29, 0x7d, 0x65, 0x6c, 0x73, 0x65, 0x20, + 0x69, 0x66, 0x28, 0x22, 0x6f, 0x22, 0x3d, 0x3d, 0x3d, 0x6e, 0x5b, 0x30, + 0x5d, 0x26, 0x26, 0x22, 0x6e, 0x22, 0x3d, 0x3d, 0x3d, 0x6e, 0x5b, 0x31, + 0x5d, 0x29, 0x6f, 0x3d, 0x6e, 0x21, 0x3d, 0x3d, 0x28, 0x6e, 0x3d, 0x6e, + 0x2e, 0x72, 0x65, 0x70, 0x6c, 0x61, 0x63, 0x65, 0x28, 0x2f, 0x28, 0x50, + 0x6f, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x43, 0x61, 0x70, 0x74, 0x75, 0x72, + 0x65, 0x29, 0x24, 0x7c, 0x43, 0x61, 0x70, 0x74, 0x75, 0x72, 0x65, 0x24, + 0x2f, 0x69, 0x2c, 0x22, 0x24, 0x31, 0x22, 0x29, 0x29, 0x2c, 0x6e, 0x3d, + 0x6e, 0x2e, 0x74, 0x6f, 0x4c, 0x6f, 0x77, 0x65, 0x72, 0x43, 0x61, 0x73, + 0x65, 0x28, 0x29, 0x69, 0x6e, 0x20, 0x74, 0x3f, 0x6e, 0x2e, 0x74, 0x6f, + 0x4c, 0x6f, 0x77, 0x65, 0x72, 0x43, 0x61, 0x73, 0x65, 0x28, 0x29, 0x2e, + 0x73, 0x6c, 0x69, 0x63, 0x65, 0x28, 0x32, 0x29, 0x3a, 0x6e, 0x2e, 0x73, + 0x6c, 0x69, 0x63, 0x65, 0x28, 0x32, 0x29, 0x2c, 0x74, 0x2e, 0x6c, 0x7c, + 0x7c, 0x28, 0x74, 0x2e, 0x6c, 0x3d, 0x7b, 0x7d, 0x29, 0x2c, 0x74, 0x2e, + 0x6c, 0x5b, 0x6e, 0x2b, 0x6f, 0x5d, 0x3d, 0x65, 0x2c, 0x65, 0x3f, 0x5f, + 0x3f, 0x65, 0x2e, 0x75, 0x3d, 0x5f, 0x2e, 0x75, 0x3a, 0x28, 0x65, 0x2e, + 0x75, 0x3d, 0x44, 0x61, 0x74, 0x65, 0x2e, 0x6e, 0x6f, 0x77, 0x28, 0x29, + 0x2c, 0x74, 0x2e, 0x61, 0x64, 0x64, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x4c, + 0x69, 0x73, 0x74, 0x65, 0x6e, 0x65, 0x72, 0x28, 0x6e, 0x2c, 0x6f, 0x3f, + 0x5f, 0x74, 0x3a, 0x65, 0x74, 0x2c, 0x6f, 0x29, 0x29, 0x3a, 0x74, 0x2e, + 0x72, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x4c, + 0x69, 0x73, 0x74, 0x65, 0x6e, 0x65, 0x72, 0x28, 0x6e, 0x2c, 0x6f, 0x3f, + 0x5f, 0x74, 0x3a, 0x65, 0x74, 0x2c, 0x6f, 0x29, 0x3b, 0x65, 0x6c, 0x73, + 0x65, 0x7b, 0x69, 0x66, 0x28, 0x69, 0x29, 0x6e, 0x3d, 0x6e, 0x2e, 0x72, + 0x65, 0x70, 0x6c, 0x61, 0x63, 0x65, 0x28, 0x2f, 0x78, 0x6c, 0x69, 0x6e, + 0x6b, 0x28, 0x48, 0x7c, 0x3a, 0x68, 0x29, 0x2f, 0x2c, 0x22, 0x68, 0x22, + 0x29, 0x2e, 0x72, 0x65, 0x70, 0x6c, 0x61, 0x63, 0x65, 0x28, 0x2f, 0x73, + 0x4e, 0x61, 0x6d, 0x65, 0x24, 0x2f, 0x2c, 0x22, 0x73, 0x22, 0x29, 0x3b, + 0x65, 0x6c, 0x73, 0x65, 0x20, 0x69, 0x66, 0x28, 0x22, 0x77, 0x69, 0x64, + 0x74, 0x68, 0x22, 0x21, 0x3d, 0x3d, 0x6e, 0x26, 0x26, 0x22, 0x68, 0x65, + 0x69, 0x67, 0x68, 0x74, 0x22, 0x21, 0x3d, 0x3d, 0x6e, 0x26, 0x26, 0x22, + 0x68, 0x72, 0x65, 0x66, 0x22, 0x21, 0x3d, 0x3d, 0x6e, 0x26, 0x26, 0x22, + 0x6c, 0x69, 0x73, 0x74, 0x22, 0x21, 0x3d, 0x3d, 0x6e, 0x26, 0x26, 0x22, + 0x66, 0x6f, 0x72, 0x6d, 0x22, 0x21, 0x3d, 0x3d, 0x6e, 0x26, 0x26, 0x22, + 0x74, 0x61, 0x62, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x22, 0x21, 0x3d, 0x3d, + 0x6e, 0x26, 0x26, 0x22, 0x64, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, + 0x22, 0x21, 0x3d, 0x3d, 0x6e, 0x26, 0x26, 0x22, 0x72, 0x6f, 0x77, 0x53, + 0x70, 0x61, 0x6e, 0x22, 0x21, 0x3d, 0x3d, 0x6e, 0x26, 0x26, 0x22, 0x63, + 0x6f, 0x6c, 0x53, 0x70, 0x61, 0x6e, 0x22, 0x21, 0x3d, 0x3d, 0x6e, 0x26, + 0x26, 0x22, 0x72, 0x6f, 0x6c, 0x65, 0x22, 0x21, 0x3d, 0x3d, 0x6e, 0x26, + 0x26, 0x6e, 0x20, 0x69, 0x6e, 0x20, 0x74, 0x29, 0x74, 0x72, 0x79, 0x7b, + 0x74, 0x5b, 0x6e, 0x5d, 0x3d, 0x6e, 0x75, 0x6c, 0x6c, 0x3d, 0x3d, 0x65, + 0x3f, 0x22, 0x22, 0x3a, 0x65, 0x3b, 0x62, 0x72, 0x65, 0x61, 0x6b, 0x20, + 0x74, 0x7d, 0x63, 0x61, 0x74, 0x63, 0x68, 0x28, 0x74, 0x29, 0x7b, 0x7d, + 0x22, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x3d, 0x3d, + 0x74, 0x79, 0x70, 0x65, 0x6f, 0x66, 0x20, 0x65, 0x7c, 0x7c, 0x28, 0x6e, + 0x75, 0x6c, 0x6c, 0x3d, 0x3d, 0x65, 0x7c, 0x7c, 0x21, 0x31, 0x3d, 0x3d, + 0x3d, 0x65, 0x26, 0x26, 0x22, 0x2d, 0x22, 0x21, 0x3d, 0x3d, 0x6e, 0x5b, + 0x34, 0x5d, 0x3f, 0x74, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x41, + 0x74, 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, 0x65, 0x28, 0x6e, 0x29, 0x3a, + 0x74, 0x2e, 0x73, 0x65, 0x74, 0x41, 0x74, 0x74, 0x72, 0x69, 0x62, 0x75, + 0x74, 0x65, 0x28, 0x6e, 0x2c, 0x65, 0x29, 0x29, 0x7d, 0x7d, 0x66, 0x75, + 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x65, 0x74, 0x28, 0x74, 0x29, + 0x7b, 0x69, 0x66, 0x28, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x6c, 0x29, 0x7b, + 0x76, 0x61, 0x72, 0x20, 0x6e, 0x3d, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x6c, + 0x5b, 0x74, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x2b, 0x21, 0x31, 0x5d, 0x3b, + 0x69, 0x66, 0x28, 0x74, 0x2e, 0x74, 0x29, 0x7b, 0x69, 0x66, 0x28, 0x74, + 0x2e, 0x74, 0x3c, 0x3d, 0x6e, 0x2e, 0x75, 0x29, 0x72, 0x65, 0x74, 0x75, + 0x72, 0x6e, 0x7d, 0x65, 0x6c, 0x73, 0x65, 0x20, 0x74, 0x2e, 0x74, 0x3d, + 0x44, 0x61, 0x74, 0x65, 0x2e, 0x6e, 0x6f, 0x77, 0x28, 0x29, 0x3b, 0x72, + 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x6e, 0x28, 0x43, 0x2e, 0x65, 0x76, + 0x65, 0x6e, 0x74, 0x3f, 0x43, 0x2e, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x28, + 0x74, 0x29, 0x3a, 0x74, 0x29, 0x7d, 0x7d, 0x66, 0x75, 0x6e, 0x63, 0x74, + 0x69, 0x6f, 0x6e, 0x20, 0x5f, 0x74, 0x28, 0x74, 0x29, 0x7b, 0x69, 0x66, + 0x28, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x6c, 0x29, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x6c, 0x5b, 0x74, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x2b, 0x21, 0x30, 0x5d, 0x28, 0x43, 0x2e, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x3f, 0x43, 0x2e, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x28, 0x74, 0x29, 0x3a, 0x74, 0x29, 0x7d, 0x66, 0x75, 0x6e, 0x63, 0x74, - 0x69, 0x6f, 0x6e, 0x20, 0x5f, 0x74, 0x28, 0x74, 0x2c, 0x6e, 0x2c, 0x65, + 0x69, 0x6f, 0x6e, 0x20, 0x69, 0x74, 0x28, 0x74, 0x2c, 0x6e, 0x2c, 0x65, 0x2c, 0x5f, 0x2c, 0x69, 0x2c, 0x6f, 0x2c, 0x72, 0x2c, 0x75, 0x2c, 0x66, - 0x2c, 0x73, 0x29, 0x7b, 0x76, 0x61, 0x72, 0x20, 0x6c, 0x2c, 0x63, 0x2c, + 0x2c, 0x6c, 0x29, 0x7b, 0x76, 0x61, 0x72, 0x20, 0x73, 0x2c, 0x63, 0x2c, 0x68, 0x2c, 0x61, 0x2c, 0x70, 0x2c, 0x64, 0x2c, 0x76, 0x2c, 0x79, 0x2c, 0x6d, 0x2c, 0x67, 0x2c, 0x62, 0x2c, 0x6b, 0x2c, 0x53, 0x2c, 0x77, 0x2c, 0x78, 0x2c, 0x45, 0x3d, 0x6e, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x3b, 0x69, @@ -762,16 +781,16 @@ unsigned char index_js[] = { 0x28, 0x66, 0x3d, 0x21, 0x21, 0x28, 0x33, 0x32, 0x26, 0x65, 0x2e, 0x5f, 0x5f, 0x75, 0x29, 0x2c, 0x6f, 0x3d, 0x5b, 0x75, 0x3d, 0x6e, 0x2e, 0x5f, 0x5f, 0x65, 0x3d, 0x65, 0x2e, 0x5f, 0x5f, 0x65, 0x5d, 0x29, 0x2c, 0x28, - 0x6c, 0x3d, 0x43, 0x2e, 0x5f, 0x5f, 0x62, 0x29, 0x26, 0x26, 0x6c, 0x28, + 0x73, 0x3d, 0x43, 0x2e, 0x5f, 0x5f, 0x62, 0x29, 0x26, 0x26, 0x73, 0x28, 0x6e, 0x29, 0x3b, 0x74, 0x3a, 0x69, 0x66, 0x28, 0x22, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x3d, 0x3d, 0x74, 0x79, 0x70, 0x65, 0x6f, 0x66, 0x20, 0x45, 0x29, 0x74, 0x72, 0x79, 0x7b, 0x69, 0x66, 0x28, 0x79, 0x3d, 0x6e, 0x2e, 0x70, 0x72, 0x6f, 0x70, 0x73, 0x2c, 0x6d, 0x3d, - 0x28, 0x6c, 0x3d, 0x45, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, - 0x54, 0x79, 0x70, 0x65, 0x29, 0x26, 0x26, 0x5f, 0x5b, 0x6c, 0x2e, 0x5f, - 0x5f, 0x63, 0x5d, 0x2c, 0x67, 0x3d, 0x6c, 0x3f, 0x6d, 0x3f, 0x6d, 0x2e, + 0x28, 0x73, 0x3d, 0x45, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, + 0x54, 0x79, 0x70, 0x65, 0x29, 0x26, 0x26, 0x5f, 0x5b, 0x73, 0x2e, 0x5f, + 0x5f, 0x63, 0x5d, 0x2c, 0x67, 0x3d, 0x73, 0x3f, 0x6d, 0x3f, 0x6d, 0x2e, 0x70, 0x72, 0x6f, 0x70, 0x73, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, - 0x6c, 0x2e, 0x5f, 0x5f, 0x3a, 0x5f, 0x2c, 0x65, 0x2e, 0x5f, 0x5f, 0x63, + 0x73, 0x2e, 0x5f, 0x5f, 0x3a, 0x5f, 0x2c, 0x65, 0x2e, 0x5f, 0x5f, 0x63, 0x3f, 0x76, 0x3d, 0x28, 0x63, 0x3d, 0x6e, 0x2e, 0x5f, 0x5f, 0x63, 0x3d, 0x65, 0x2e, 0x5f, 0x5f, 0x63, 0x29, 0x2e, 0x5f, 0x5f, 0x3d, 0x63, 0x2e, 0x5f, 0x5f, 0x45, 0x3a, 0x28, 0x22, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x74, @@ -782,7 +801,7 @@ unsigned char index_js[] = { 0x28, 0x6e, 0x2e, 0x5f, 0x5f, 0x63, 0x3d, 0x63, 0x3d, 0x6e, 0x65, 0x77, 0x20, 0x49, 0x28, 0x79, 0x2c, 0x67, 0x29, 0x2c, 0x63, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x72, 0x75, 0x63, 0x74, 0x6f, 0x72, 0x3d, 0x45, 0x2c, - 0x63, 0x2e, 0x72, 0x65, 0x6e, 0x64, 0x65, 0x72, 0x3d, 0x66, 0x74, 0x29, + 0x63, 0x2e, 0x72, 0x65, 0x6e, 0x64, 0x65, 0x72, 0x3d, 0x6c, 0x74, 0x29, 0x2c, 0x6d, 0x26, 0x26, 0x6d, 0x2e, 0x73, 0x75, 0x62, 0x28, 0x63, 0x29, 0x2c, 0x63, 0x2e, 0x70, 0x72, 0x6f, 0x70, 0x73, 0x3d, 0x79, 0x2c, 0x63, 0x2e, 0x73, 0x74, 0x61, 0x74, 0x65, 0x7c, 0x7c, 0x28, 0x63, 0x2e, 0x73, @@ -875,7 +894,7 @@ unsigned char index_js[] = { 0x7b, 0x66, 0x6f, 0x72, 0x28, 0x63, 0x2e, 0x73, 0x74, 0x61, 0x74, 0x65, 0x3d, 0x63, 0x2e, 0x5f, 0x5f, 0x73, 0x2c, 0x63, 0x2e, 0x5f, 0x5f, 0x64, 0x3d, 0x21, 0x31, 0x2c, 0x6b, 0x26, 0x26, 0x6b, 0x28, 0x6e, 0x29, 0x2c, - 0x6c, 0x3d, 0x63, 0x2e, 0x72, 0x65, 0x6e, 0x64, 0x65, 0x72, 0x28, 0x63, + 0x73, 0x3d, 0x63, 0x2e, 0x72, 0x65, 0x6e, 0x64, 0x65, 0x72, 0x28, 0x63, 0x2e, 0x70, 0x72, 0x6f, 0x70, 0x73, 0x2c, 0x63, 0x2e, 0x73, 0x74, 0x61, 0x74, 0x65, 0x2c, 0x63, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x29, 0x2c, 0x77, 0x3d, 0x30, 0x3b, 0x77, 0x3c, 0x63, 0x2e, 0x5f, 0x73, @@ -884,7 +903,7 @@ unsigned char index_js[] = { 0x63, 0x2e, 0x5f, 0x73, 0x62, 0x5b, 0x77, 0x5d, 0x29, 0x3b, 0x63, 0x2e, 0x5f, 0x73, 0x62, 0x3d, 0x5b, 0x5d, 0x7d, 0x65, 0x6c, 0x73, 0x65, 0x20, 0x64, 0x6f, 0x7b, 0x63, 0x2e, 0x5f, 0x5f, 0x64, 0x3d, 0x21, 0x31, 0x2c, - 0x6b, 0x26, 0x26, 0x6b, 0x28, 0x6e, 0x29, 0x2c, 0x6c, 0x3d, 0x63, 0x2e, + 0x6b, 0x26, 0x26, 0x6b, 0x28, 0x6e, 0x29, 0x2c, 0x73, 0x3d, 0x63, 0x2e, 0x72, 0x65, 0x6e, 0x64, 0x65, 0x72, 0x28, 0x63, 0x2e, 0x70, 0x72, 0x6f, 0x70, 0x73, 0x2c, 0x63, 0x2e, 0x73, 0x74, 0x61, 0x74, 0x65, 0x2c, 0x63, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x29, 0x2c, 0x63, 0x2e, @@ -902,14 +921,14 @@ unsigned char index_js[] = { 0x6f, 0x72, 0x65, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x7c, 0x7c, 0x28, 0x64, 0x3d, 0x63, 0x2e, 0x67, 0x65, 0x74, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x42, 0x65, 0x66, 0x6f, 0x72, 0x65, 0x55, 0x70, 0x64, - 0x61, 0x74, 0x65, 0x28, 0x61, 0x2c, 0x70, 0x29, 0x29, 0x2c, 0x4a, 0x28, + 0x61, 0x74, 0x65, 0x28, 0x61, 0x2c, 0x70, 0x29, 0x29, 0x2c, 0x4b, 0x28, 0x74, 0x2c, 0x46, 0x28, 0x78, 0x3d, 0x6e, 0x75, 0x6c, 0x6c, 0x21, 0x3d, - 0x6c, 0x26, 0x26, 0x6c, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x3d, 0x3d, 0x3d, - 0x6a, 0x26, 0x26, 0x6e, 0x75, 0x6c, 0x6c, 0x3d, 0x3d, 0x6c, 0x2e, 0x6b, - 0x65, 0x79, 0x3f, 0x6c, 0x2e, 0x70, 0x72, 0x6f, 0x70, 0x73, 0x2e, 0x63, - 0x68, 0x69, 0x6c, 0x64, 0x72, 0x65, 0x6e, 0x3a, 0x6c, 0x29, 0x3f, 0x78, + 0x73, 0x26, 0x26, 0x73, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x3d, 0x3d, 0x3d, + 0x6a, 0x26, 0x26, 0x6e, 0x75, 0x6c, 0x6c, 0x3d, 0x3d, 0x73, 0x2e, 0x6b, + 0x65, 0x79, 0x3f, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x70, 0x73, 0x2e, 0x63, + 0x68, 0x69, 0x6c, 0x64, 0x72, 0x65, 0x6e, 0x3a, 0x73, 0x29, 0x3f, 0x78, 0x3a, 0x5b, 0x78, 0x5d, 0x2c, 0x6e, 0x2c, 0x65, 0x2c, 0x5f, 0x2c, 0x69, - 0x2c, 0x6f, 0x2c, 0x72, 0x2c, 0x75, 0x2c, 0x66, 0x2c, 0x73, 0x29, 0x2c, + 0x2c, 0x6f, 0x2c, 0x72, 0x2c, 0x75, 0x2c, 0x66, 0x2c, 0x6c, 0x29, 0x2c, 0x63, 0x2e, 0x62, 0x61, 0x73, 0x65, 0x3d, 0x6e, 0x2e, 0x5f, 0x5f, 0x65, 0x2c, 0x6e, 0x2e, 0x5f, 0x5f, 0x75, 0x26, 0x3d, 0x2d, 0x31, 0x36, 0x31, 0x2c, 0x63, 0x2e, 0x5f, 0x5f, 0x68, 0x2e, 0x6c, 0x65, 0x6e, 0x67, 0x74, @@ -930,198 +949,198 @@ unsigned char index_js[] = { 0x5f, 0x76, 0x3d, 0x3d, 0x3d, 0x65, 0x2e, 0x5f, 0x5f, 0x76, 0x3f, 0x28, 0x6e, 0x2e, 0x5f, 0x5f, 0x6b, 0x3d, 0x65, 0x2e, 0x5f, 0x5f, 0x6b, 0x2c, 0x6e, 0x2e, 0x5f, 0x5f, 0x65, 0x3d, 0x65, 0x2e, 0x5f, 0x5f, 0x65, 0x29, - 0x3a, 0x6e, 0x2e, 0x5f, 0x5f, 0x65, 0x3d, 0x6f, 0x74, 0x28, 0x65, 0x2e, + 0x3a, 0x6e, 0x2e, 0x5f, 0x5f, 0x65, 0x3d, 0x72, 0x74, 0x28, 0x65, 0x2e, 0x5f, 0x5f, 0x65, 0x2c, 0x6e, 0x2c, 0x65, 0x2c, 0x5f, 0x2c, 0x69, 0x2c, - 0x6f, 0x2c, 0x72, 0x2c, 0x66, 0x2c, 0x73, 0x29, 0x3b, 0x28, 0x6c, 0x3d, - 0x43, 0x2e, 0x64, 0x69, 0x66, 0x66, 0x65, 0x64, 0x29, 0x26, 0x26, 0x6c, + 0x6f, 0x2c, 0x72, 0x2c, 0x66, 0x2c, 0x6c, 0x29, 0x3b, 0x28, 0x73, 0x3d, + 0x43, 0x2e, 0x64, 0x69, 0x66, 0x66, 0x65, 0x64, 0x29, 0x26, 0x26, 0x73, 0x28, 0x6e, 0x29, 0x7d, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, - 0x20, 0x69, 0x74, 0x28, 0x74, 0x2c, 0x6e, 0x2c, 0x65, 0x29, 0x7b, 0x6e, - 0x2e, 0x5f, 0x5f, 0x64, 0x3d, 0x76, 0x6f, 0x69, 0x64, 0x20, 0x30, 0x3b, - 0x66, 0x6f, 0x72, 0x28, 0x76, 0x61, 0x72, 0x20, 0x5f, 0x3d, 0x30, 0x3b, - 0x5f, 0x3c, 0x65, 0x2e, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x3b, 0x5f, - 0x2b, 0x2b, 0x29, 0x72, 0x74, 0x28, 0x65, 0x5b, 0x5f, 0x5d, 0x2c, 0x65, - 0x5b, 0x2b, 0x2b, 0x5f, 0x5d, 0x2c, 0x65, 0x5b, 0x2b, 0x2b, 0x5f, 0x5d, - 0x29, 0x3b, 0x43, 0x2e, 0x5f, 0x5f, 0x63, 0x26, 0x26, 0x43, 0x2e, 0x5f, - 0x5f, 0x63, 0x28, 0x6e, 0x2c, 0x74, 0x29, 0x2c, 0x74, 0x2e, 0x73, 0x6f, - 0x6d, 0x65, 0x28, 0x28, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, - 0x28, 0x6e, 0x29, 0x7b, 0x74, 0x72, 0x79, 0x7b, 0x74, 0x3d, 0x6e, 0x2e, - 0x5f, 0x5f, 0x68, 0x2c, 0x6e, 0x2e, 0x5f, 0x5f, 0x68, 0x3d, 0x5b, 0x5d, - 0x2c, 0x74, 0x2e, 0x73, 0x6f, 0x6d, 0x65, 0x28, 0x28, 0x66, 0x75, 0x6e, - 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x74, 0x29, 0x7b, 0x74, 0x2e, 0x63, - 0x61, 0x6c, 0x6c, 0x28, 0x6e, 0x29, 0x7d, 0x29, 0x29, 0x7d, 0x63, 0x61, - 0x74, 0x63, 0x68, 0x28, 0x74, 0x29, 0x7b, 0x43, 0x2e, 0x5f, 0x5f, 0x65, - 0x28, 0x74, 0x2c, 0x6e, 0x2e, 0x5f, 0x5f, 0x76, 0x29, 0x7d, 0x7d, 0x29, - 0x29, 0x7d, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x6f, - 0x74, 0x28, 0x74, 0x2c, 0x6e, 0x2c, 0x65, 0x2c, 0x5f, 0x2c, 0x69, 0x2c, - 0x6f, 0x2c, 0x72, 0x2c, 0x75, 0x2c, 0x66, 0x29, 0x7b, 0x76, 0x61, 0x72, - 0x20, 0x73, 0x2c, 0x6c, 0x2c, 0x63, 0x2c, 0x68, 0x2c, 0x61, 0x2c, 0x70, - 0x2c, 0x64, 0x2c, 0x76, 0x3d, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x70, 0x73, - 0x2c, 0x79, 0x3d, 0x6e, 0x2e, 0x70, 0x72, 0x6f, 0x70, 0x73, 0x2c, 0x6d, - 0x3d, 0x6e, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x3b, 0x69, 0x66, 0x28, 0x22, - 0x73, 0x76, 0x67, 0x22, 0x3d, 0x3d, 0x3d, 0x6d, 0x26, 0x26, 0x28, 0x69, - 0x3d, 0x21, 0x30, 0x29, 0x2c, 0x6e, 0x75, 0x6c, 0x6c, 0x21, 0x3d, 0x6f, - 0x29, 0x66, 0x6f, 0x72, 0x28, 0x73, 0x3d, 0x30, 0x3b, 0x73, 0x3c, 0x6f, - 0x2e, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x3b, 0x73, 0x2b, 0x2b, 0x29, - 0x69, 0x66, 0x28, 0x28, 0x61, 0x3d, 0x6f, 0x5b, 0x73, 0x5d, 0x29, 0x26, - 0x26, 0x22, 0x73, 0x65, 0x74, 0x41, 0x74, 0x74, 0x72, 0x69, 0x62, 0x75, - 0x74, 0x65, 0x22, 0x69, 0x6e, 0x20, 0x61, 0x3d, 0x3d, 0x21, 0x21, 0x6d, - 0x26, 0x26, 0x28, 0x6d, 0x3f, 0x61, 0x2e, 0x6c, 0x6f, 0x63, 0x61, 0x6c, - 0x4e, 0x61, 0x6d, 0x65, 0x3d, 0x3d, 0x3d, 0x6d, 0x3a, 0x33, 0x3d, 0x3d, - 0x3d, 0x61, 0x2e, 0x6e, 0x6f, 0x64, 0x65, 0x54, 0x79, 0x70, 0x65, 0x29, - 0x29, 0x7b, 0x74, 0x3d, 0x61, 0x2c, 0x6f, 0x5b, 0x73, 0x5d, 0x3d, 0x6e, - 0x75, 0x6c, 0x6c, 0x3b, 0x62, 0x72, 0x65, 0x61, 0x6b, 0x7d, 0x69, 0x66, - 0x28, 0x6e, 0x75, 0x6c, 0x6c, 0x3d, 0x3d, 0x74, 0x29, 0x7b, 0x69, 0x66, - 0x28, 0x6e, 0x75, 0x6c, 0x6c, 0x3d, 0x3d, 0x3d, 0x6d, 0x29, 0x72, 0x65, - 0x74, 0x75, 0x72, 0x6e, 0x20, 0x64, 0x6f, 0x63, 0x75, 0x6d, 0x65, 0x6e, - 0x74, 0x2e, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x54, 0x65, 0x78, 0x74, - 0x4e, 0x6f, 0x64, 0x65, 0x28, 0x79, 0x29, 0x3b, 0x74, 0x3d, 0x69, 0x3f, - 0x64, 0x6f, 0x63, 0x75, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x63, 0x72, 0x65, - 0x61, 0x74, 0x65, 0x45, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x4e, 0x53, - 0x28, 0x22, 0x68, 0x74, 0x74, 0x70, 0x3a, 0x2f, 0x2f, 0x77, 0x77, 0x77, - 0x2e, 0x77, 0x33, 0x2e, 0x6f, 0x72, 0x67, 0x2f, 0x32, 0x30, 0x30, 0x30, - 0x2f, 0x73, 0x76, 0x67, 0x22, 0x2c, 0x6d, 0x29, 0x3a, 0x64, 0x6f, 0x63, - 0x75, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, - 0x45, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x28, 0x6d, 0x2c, 0x79, 0x2e, - 0x69, 0x73, 0x26, 0x26, 0x79, 0x29, 0x2c, 0x6f, 0x3d, 0x6e, 0x75, 0x6c, - 0x6c, 0x2c, 0x75, 0x3d, 0x21, 0x31, 0x7d, 0x69, 0x66, 0x28, 0x6e, 0x75, - 0x6c, 0x6c, 0x3d, 0x3d, 0x3d, 0x6d, 0x29, 0x76, 0x3d, 0x3d, 0x3d, 0x79, - 0x7c, 0x7c, 0x75, 0x26, 0x26, 0x74, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x3d, - 0x3d, 0x3d, 0x79, 0x7c, 0x7c, 0x28, 0x74, 0x2e, 0x64, 0x61, 0x74, 0x61, - 0x3d, 0x79, 0x29, 0x3b, 0x65, 0x6c, 0x73, 0x65, 0x7b, 0x69, 0x66, 0x28, - 0x6f, 0x3d, 0x6f, 0x26, 0x26, 0x78, 0x2e, 0x63, 0x61, 0x6c, 0x6c, 0x28, - 0x74, 0x2e, 0x63, 0x68, 0x69, 0x6c, 0x64, 0x4e, 0x6f, 0x64, 0x65, 0x73, - 0x29, 0x2c, 0x76, 0x3d, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x70, 0x73, 0x7c, - 0x7c, 0x54, 0x2c, 0x21, 0x75, 0x26, 0x26, 0x6e, 0x75, 0x6c, 0x6c, 0x21, - 0x3d, 0x6f, 0x29, 0x66, 0x6f, 0x72, 0x28, 0x76, 0x3d, 0x7b, 0x7d, 0x2c, - 0x73, 0x3d, 0x30, 0x3b, 0x73, 0x3c, 0x74, 0x2e, 0x61, 0x74, 0x74, 0x72, - 0x69, 0x62, 0x75, 0x74, 0x65, 0x73, 0x2e, 0x6c, 0x65, 0x6e, 0x67, 0x74, - 0x68, 0x3b, 0x73, 0x2b, 0x2b, 0x29, 0x76, 0x5b, 0x28, 0x61, 0x3d, 0x74, - 0x2e, 0x61, 0x74, 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, 0x65, 0x73, 0x5b, - 0x73, 0x5d, 0x29, 0x2e, 0x6e, 0x61, 0x6d, 0x65, 0x5d, 0x3d, 0x61, 0x2e, - 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3b, 0x66, 0x6f, 0x72, 0x28, 0x73, 0x20, - 0x69, 0x6e, 0x20, 0x76, 0x29, 0x61, 0x3d, 0x76, 0x5b, 0x73, 0x5d, 0x2c, - 0x22, 0x63, 0x68, 0x69, 0x6c, 0x64, 0x72, 0x65, 0x6e, 0x22, 0x3d, 0x3d, - 0x73, 0x7c, 0x7c, 0x28, 0x22, 0x64, 0x61, 0x6e, 0x67, 0x65, 0x72, 0x6f, - 0x75, 0x73, 0x6c, 0x79, 0x53, 0x65, 0x74, 0x49, 0x6e, 0x6e, 0x65, 0x72, - 0x48, 0x54, 0x4d, 0x4c, 0x22, 0x3d, 0x3d, 0x73, 0x3f, 0x63, 0x3d, 0x61, - 0x3a, 0x22, 0x6b, 0x65, 0x79, 0x22, 0x3d, 0x3d, 0x3d, 0x73, 0x7c, 0x7c, - 0x73, 0x20, 0x69, 0x6e, 0x20, 0x79, 0x7c, 0x7c, 0x74, 0x74, 0x28, 0x74, - 0x2c, 0x73, 0x2c, 0x6e, 0x75, 0x6c, 0x6c, 0x2c, 0x61, 0x2c, 0x69, 0x29, - 0x29, 0x3b, 0x66, 0x6f, 0x72, 0x28, 0x73, 0x20, 0x69, 0x6e, 0x20, 0x79, - 0x29, 0x61, 0x3d, 0x79, 0x5b, 0x73, 0x5d, 0x2c, 0x22, 0x63, 0x68, 0x69, - 0x6c, 0x64, 0x72, 0x65, 0x6e, 0x22, 0x3d, 0x3d, 0x73, 0x3f, 0x68, 0x3d, - 0x61, 0x3a, 0x22, 0x64, 0x61, 0x6e, 0x67, 0x65, 0x72, 0x6f, 0x75, 0x73, - 0x6c, 0x79, 0x53, 0x65, 0x74, 0x49, 0x6e, 0x6e, 0x65, 0x72, 0x48, 0x54, - 0x4d, 0x4c, 0x22, 0x3d, 0x3d, 0x73, 0x3f, 0x6c, 0x3d, 0x61, 0x3a, 0x22, - 0x76, 0x61, 0x6c, 0x75, 0x65, 0x22, 0x3d, 0x3d, 0x73, 0x3f, 0x70, 0x3d, - 0x61, 0x3a, 0x22, 0x63, 0x68, 0x65, 0x63, 0x6b, 0x65, 0x64, 0x22, 0x3d, - 0x3d, 0x73, 0x3f, 0x64, 0x3d, 0x61, 0x3a, 0x22, 0x6b, 0x65, 0x79, 0x22, - 0x3d, 0x3d, 0x3d, 0x73, 0x7c, 0x7c, 0x75, 0x26, 0x26, 0x22, 0x66, 0x75, - 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x21, 0x3d, 0x74, 0x79, 0x70, - 0x65, 0x6f, 0x66, 0x20, 0x61, 0x7c, 0x7c, 0x76, 0x5b, 0x73, 0x5d, 0x3d, - 0x3d, 0x3d, 0x61, 0x7c, 0x7c, 0x74, 0x74, 0x28, 0x74, 0x2c, 0x73, 0x2c, - 0x61, 0x2c, 0x76, 0x5b, 0x73, 0x5d, 0x2c, 0x69, 0x29, 0x3b, 0x69, 0x66, - 0x28, 0x6c, 0x29, 0x75, 0x7c, 0x7c, 0x63, 0x26, 0x26, 0x28, 0x6c, 0x2e, - 0x5f, 0x5f, 0x68, 0x74, 0x6d, 0x6c, 0x3d, 0x3d, 0x3d, 0x63, 0x2e, 0x5f, - 0x5f, 0x68, 0x74, 0x6d, 0x6c, 0x7c, 0x7c, 0x6c, 0x2e, 0x5f, 0x5f, 0x68, - 0x74, 0x6d, 0x6c, 0x3d, 0x3d, 0x3d, 0x74, 0x2e, 0x69, 0x6e, 0x6e, 0x65, - 0x72, 0x48, 0x54, 0x4d, 0x4c, 0x29, 0x7c, 0x7c, 0x28, 0x74, 0x2e, 0x69, - 0x6e, 0x6e, 0x65, 0x72, 0x48, 0x54, 0x4d, 0x4c, 0x3d, 0x6c, 0x2e, 0x5f, - 0x5f, 0x68, 0x74, 0x6d, 0x6c, 0x29, 0x2c, 0x6e, 0x2e, 0x5f, 0x5f, 0x6b, - 0x3d, 0x5b, 0x5d, 0x3b, 0x65, 0x6c, 0x73, 0x65, 0x20, 0x69, 0x66, 0x28, - 0x63, 0x26, 0x26, 0x28, 0x74, 0x2e, 0x69, 0x6e, 0x6e, 0x65, 0x72, 0x48, - 0x54, 0x4d, 0x4c, 0x3d, 0x22, 0x22, 0x29, 0x2c, 0x4a, 0x28, 0x74, 0x2c, - 0x46, 0x28, 0x68, 0x29, 0x3f, 0x68, 0x3a, 0x5b, 0x68, 0x5d, 0x2c, 0x6e, - 0x2c, 0x65, 0x2c, 0x5f, 0x2c, 0x69, 0x26, 0x26, 0x22, 0x66, 0x6f, 0x72, - 0x65, 0x69, 0x67, 0x6e, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x22, 0x21, - 0x3d, 0x3d, 0x6d, 0x2c, 0x6f, 0x2c, 0x72, 0x2c, 0x6f, 0x3f, 0x6f, 0x5b, - 0x30, 0x5d, 0x3a, 0x65, 0x2e, 0x5f, 0x5f, 0x6b, 0x26, 0x26, 0x71, 0x28, - 0x65, 0x2c, 0x30, 0x29, 0x2c, 0x75, 0x2c, 0x66, 0x29, 0x2c, 0x6e, 0x75, - 0x6c, 0x6c, 0x21, 0x3d, 0x6f, 0x29, 0x66, 0x6f, 0x72, 0x28, 0x73, 0x3d, - 0x6f, 0x2e, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x3b, 0x73, 0x2d, 0x2d, - 0x3b, 0x29, 0x6e, 0x75, 0x6c, 0x6c, 0x21, 0x3d, 0x6f, 0x5b, 0x73, 0x5d, - 0x26, 0x26, 0x57, 0x28, 0x6f, 0x5b, 0x73, 0x5d, 0x29, 0x3b, 0x75, 0x7c, - 0x7c, 0x28, 0x73, 0x3d, 0x22, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x22, 0x2c, - 0x76, 0x6f, 0x69, 0x64, 0x20, 0x30, 0x21, 0x3d, 0x3d, 0x70, 0x26, 0x26, - 0x28, 0x70, 0x21, 0x3d, 0x3d, 0x74, 0x5b, 0x73, 0x5d, 0x7c, 0x7c, 0x22, - 0x70, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x22, 0x3d, 0x3d, 0x3d, - 0x6d, 0x26, 0x26, 0x21, 0x70, 0x7c, 0x7c, 0x22, 0x6f, 0x70, 0x74, 0x69, - 0x6f, 0x6e, 0x22, 0x3d, 0x3d, 0x3d, 0x6d, 0x26, 0x26, 0x70, 0x21, 0x3d, - 0x3d, 0x76, 0x5b, 0x73, 0x5d, 0x29, 0x26, 0x26, 0x74, 0x74, 0x28, 0x74, - 0x2c, 0x73, 0x2c, 0x70, 0x2c, 0x76, 0x5b, 0x73, 0x5d, 0x2c, 0x21, 0x31, - 0x29, 0x2c, 0x73, 0x3d, 0x22, 0x63, 0x68, 0x65, 0x63, 0x6b, 0x65, 0x64, - 0x22, 0x2c, 0x76, 0x6f, 0x69, 0x64, 0x20, 0x30, 0x21, 0x3d, 0x3d, 0x64, - 0x26, 0x26, 0x64, 0x21, 0x3d, 0x3d, 0x74, 0x5b, 0x73, 0x5d, 0x26, 0x26, - 0x74, 0x74, 0x28, 0x74, 0x2c, 0x73, 0x2c, 0x64, 0x2c, 0x76, 0x5b, 0x73, - 0x5d, 0x2c, 0x21, 0x31, 0x29, 0x29, 0x7d, 0x72, 0x65, 0x74, 0x75, 0x72, - 0x6e, 0x20, 0x74, 0x7d, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, - 0x20, 0x72, 0x74, 0x28, 0x74, 0x2c, 0x6e, 0x2c, 0x65, 0x29, 0x7b, 0x74, - 0x72, 0x79, 0x7b, 0x22, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, - 0x22, 0x3d, 0x3d, 0x74, 0x79, 0x70, 0x65, 0x6f, 0x66, 0x20, 0x74, 0x3f, - 0x74, 0x28, 0x6e, 0x29, 0x3a, 0x74, 0x2e, 0x63, 0x75, 0x72, 0x72, 0x65, - 0x6e, 0x74, 0x3d, 0x6e, 0x7d, 0x63, 0x61, 0x74, 0x63, 0x68, 0x28, 0x74, - 0x29, 0x7b, 0x43, 0x2e, 0x5f, 0x5f, 0x65, 0x28, 0x74, 0x2c, 0x65, 0x29, - 0x7d, 0x7d, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x75, - 0x74, 0x28, 0x74, 0x2c, 0x6e, 0x2c, 0x65, 0x29, 0x7b, 0x76, 0x61, 0x72, - 0x20, 0x5f, 0x2c, 0x69, 0x3b, 0x69, 0x66, 0x28, 0x43, 0x2e, 0x75, 0x6e, - 0x6d, 0x6f, 0x75, 0x6e, 0x74, 0x26, 0x26, 0x43, 0x2e, 0x75, 0x6e, 0x6d, - 0x6f, 0x75, 0x6e, 0x74, 0x28, 0x74, 0x29, 0x2c, 0x28, 0x5f, 0x3d, 0x74, - 0x2e, 0x72, 0x65, 0x66, 0x29, 0x26, 0x26, 0x28, 0x5f, 0x2e, 0x63, 0x75, - 0x72, 0x72, 0x65, 0x6e, 0x74, 0x26, 0x26, 0x5f, 0x2e, 0x63, 0x75, 0x72, - 0x72, 0x65, 0x6e, 0x74, 0x21, 0x3d, 0x3d, 0x74, 0x2e, 0x5f, 0x5f, 0x65, - 0x7c, 0x7c, 0x72, 0x74, 0x28, 0x5f, 0x2c, 0x6e, 0x75, 0x6c, 0x6c, 0x2c, - 0x6e, 0x29, 0x29, 0x2c, 0x6e, 0x75, 0x6c, 0x6c, 0x21, 0x3d, 0x28, 0x5f, - 0x3d, 0x74, 0x2e, 0x5f, 0x5f, 0x63, 0x29, 0x29, 0x7b, 0x69, 0x66, 0x28, - 0x5f, 0x2e, 0x63, 0x6f, 0x6d, 0x70, 0x6f, 0x6e, 0x65, 0x6e, 0x74, 0x57, - 0x69, 0x6c, 0x6c, 0x55, 0x6e, 0x6d, 0x6f, 0x75, 0x6e, 0x74, 0x29, 0x74, - 0x72, 0x79, 0x7b, 0x5f, 0x2e, 0x63, 0x6f, 0x6d, 0x70, 0x6f, 0x6e, 0x65, - 0x6e, 0x74, 0x57, 0x69, 0x6c, 0x6c, 0x55, 0x6e, 0x6d, 0x6f, 0x75, 0x6e, - 0x74, 0x28, 0x29, 0x7d, 0x63, 0x61, 0x74, 0x63, 0x68, 0x28, 0x74, 0x29, - 0x7b, 0x43, 0x2e, 0x5f, 0x5f, 0x65, 0x28, 0x74, 0x2c, 0x6e, 0x29, 0x7d, - 0x5f, 0x2e, 0x62, 0x61, 0x73, 0x65, 0x3d, 0x5f, 0x2e, 0x5f, 0x5f, 0x50, - 0x3d, 0x6e, 0x75, 0x6c, 0x6c, 0x2c, 0x74, 0x2e, 0x5f, 0x5f, 0x63, 0x3d, - 0x76, 0x6f, 0x69, 0x64, 0x20, 0x30, 0x7d, 0x69, 0x66, 0x28, 0x5f, 0x3d, - 0x74, 0x2e, 0x5f, 0x5f, 0x6b, 0x29, 0x66, 0x6f, 0x72, 0x28, 0x69, 0x3d, - 0x30, 0x3b, 0x69, 0x3c, 0x5f, 0x2e, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, - 0x3b, 0x69, 0x2b, 0x2b, 0x29, 0x5f, 0x5b, 0x69, 0x5d, 0x26, 0x26, 0x75, - 0x74, 0x28, 0x5f, 0x5b, 0x69, 0x5d, 0x2c, 0x6e, 0x2c, 0x65, 0x7c, 0x7c, - 0x22, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x21, 0x3d, - 0x74, 0x79, 0x70, 0x65, 0x6f, 0x66, 0x20, 0x74, 0x2e, 0x74, 0x79, 0x70, - 0x65, 0x29, 0x3b, 0x65, 0x7c, 0x7c, 0x6e, 0x75, 0x6c, 0x6c, 0x3d, 0x3d, - 0x74, 0x2e, 0x5f, 0x5f, 0x65, 0x7c, 0x7c, 0x57, 0x28, 0x74, 0x2e, 0x5f, - 0x5f, 0x65, 0x29, 0x2c, 0x74, 0x2e, 0x5f, 0x5f, 0x3d, 0x74, 0x2e, 0x5f, - 0x5f, 0x65, 0x3d, 0x74, 0x2e, 0x5f, 0x5f, 0x64, 0x3d, 0x76, 0x6f, 0x69, - 0x64, 0x20, 0x30, 0x7d, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, - 0x20, 0x66, 0x74, 0x28, 0x74, 0x2c, 0x6e, 0x2c, 0x65, 0x29, 0x7b, 0x72, - 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x63, - 0x6f, 0x6e, 0x73, 0x74, 0x72, 0x75, 0x63, 0x74, 0x6f, 0x72, 0x28, 0x74, - 0x2c, 0x65, 0x29, 0x7d, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, - 0x20, 0x73, 0x74, 0x28, 0x74, 0x2c, 0x6e, 0x2c, 0x65, 0x29, 0x7b, 0x76, - 0x61, 0x72, 0x20, 0x5f, 0x2c, 0x69, 0x2c, 0x6f, 0x2c, 0x72, 0x3b, 0x43, - 0x2e, 0x5f, 0x5f, 0x26, 0x26, 0x43, 0x2e, 0x5f, 0x5f, 0x28, 0x74, 0x2c, - 0x6e, 0x29, 0x2c, 0x69, 0x3d, 0x28, 0x5f, 0x3d, 0x22, 0x66, 0x75, 0x6e, - 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x3d, 0x3d, 0x74, 0x79, 0x70, 0x65, - 0x6f, 0x66, 0x20, 0x65, 0x29, 0x3f, 0x6e, 0x75, 0x6c, 0x6c, 0x3a, 0x65, - 0x26, 0x26, 0x65, 0x2e, 0x5f, 0x5f, 0x6b, 0x7c, 0x7c, 0x6e, 0x2e, 0x5f, - 0x5f, 0x6b, 0x2c, 0x6f, 0x3d, 0x5b, 0x5d, 0x2c, 0x72, 0x3d, 0x5b, 0x5d, - 0x2c, 0x5f, 0x74, 0x28, 0x6e, 0x2c, 0x74, 0x3d, 0x28, 0x21, 0x5f, 0x26, - 0x26, 0x65, 0x7c, 0x7c, 0x6e, 0x29, 0x2e, 0x5f, 0x5f, 0x6b, 0x3d, 0x4c, - 0x28, 0x6a, 0x2c, 0x6e, 0x75, 0x6c, 0x6c, 0x2c, 0x5b, 0x74, 0x5d, 0x29, - 0x2c, 0x69, 0x7c, 0x7c, 0x54, 0x2c, 0x54, 0x2c, 0x76, 0x6f, 0x69, 0x64, - 0x20, 0x30, 0x21, 0x3d, 0x3d, 0x6e, 0x2e, 0x6f, 0x77, 0x6e, 0x65, 0x72, - 0x53, 0x56, 0x47, 0x45, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2c, 0x21, - 0x5f, 0x26, 0x26, 0x65, 0x3f, 0x5b, 0x65, 0x5d, 0x3a, 0x69, 0x3f, 0x6e, - 0x75, 0x6c, 0x6c, 0x3a, 0x6e, 0x2e, 0x66, 0x69, 0x72, 0x73, 0x74, 0x43, - 0x68, 0x69, 0x6c, 0x64, 0x3f, 0x78, 0x2e, 0x63, 0x61, 0x6c, 0x6c, 0x28, - 0x6e, 0x2e, 0x63, 0x68, 0x69, 0x6c, 0x64, 0x4e, 0x6f, 0x64, 0x65, 0x73, - 0x29, 0x3a, 0x6e, 0x75, 0x6c, 0x6c, 0x2c, 0x6f, 0x2c, 0x21, 0x5f, 0x26, - 0x26, 0x65, 0x3f, 0x65, 0x3a, 0x69, 0x3f, 0x69, 0x2e, 0x5f, 0x5f, 0x65, - 0x3a, 0x6e, 0x2e, 0x66, 0x69, 0x72, 0x73, 0x74, 0x43, 0x68, 0x69, 0x6c, - 0x64, 0x2c, 0x5f, 0x2c, 0x72, 0x29, 0x2c, 0x69, 0x74, 0x28, 0x6f, 0x2c, + 0x20, 0x6f, 0x74, 0x28, 0x74, 0x2c, 0x6e, 0x2c, 0x65, 0x29, 0x7b, 0x66, + 0x6f, 0x72, 0x28, 0x76, 0x61, 0x72, 0x20, 0x5f, 0x3d, 0x30, 0x3b, 0x5f, + 0x3c, 0x65, 0x2e, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x3b, 0x5f, 0x2b, + 0x2b, 0x29, 0x75, 0x74, 0x28, 0x65, 0x5b, 0x5f, 0x5d, 0x2c, 0x65, 0x5b, + 0x2b, 0x2b, 0x5f, 0x5d, 0x2c, 0x65, 0x5b, 0x2b, 0x2b, 0x5f, 0x5d, 0x29, + 0x3b, 0x43, 0x2e, 0x5f, 0x5f, 0x63, 0x26, 0x26, 0x43, 0x2e, 0x5f, 0x5f, + 0x63, 0x28, 0x6e, 0x2c, 0x74, 0x29, 0x2c, 0x74, 0x2e, 0x73, 0x6f, 0x6d, + 0x65, 0x28, 0x28, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x28, + 0x6e, 0x29, 0x7b, 0x74, 0x72, 0x79, 0x7b, 0x74, 0x3d, 0x6e, 0x2e, 0x5f, + 0x5f, 0x68, 0x2c, 0x6e, 0x2e, 0x5f, 0x5f, 0x68, 0x3d, 0x5b, 0x5d, 0x2c, + 0x74, 0x2e, 0x73, 0x6f, 0x6d, 0x65, 0x28, 0x28, 0x66, 0x75, 0x6e, 0x63, + 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x74, 0x29, 0x7b, 0x74, 0x2e, 0x63, 0x61, + 0x6c, 0x6c, 0x28, 0x6e, 0x29, 0x7d, 0x29, 0x29, 0x7d, 0x63, 0x61, 0x74, + 0x63, 0x68, 0x28, 0x74, 0x29, 0x7b, 0x43, 0x2e, 0x5f, 0x5f, 0x65, 0x28, + 0x74, 0x2c, 0x6e, 0x2e, 0x5f, 0x5f, 0x76, 0x29, 0x7d, 0x7d, 0x29, 0x29, + 0x7d, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x72, 0x74, + 0x28, 0x74, 0x2c, 0x6e, 0x2c, 0x65, 0x2c, 0x5f, 0x2c, 0x69, 0x2c, 0x6f, + 0x2c, 0x72, 0x2c, 0x75, 0x2c, 0x66, 0x29, 0x7b, 0x76, 0x61, 0x72, 0x20, + 0x6c, 0x2c, 0x73, 0x2c, 0x63, 0x2c, 0x68, 0x2c, 0x61, 0x2c, 0x70, 0x2c, + 0x64, 0x2c, 0x76, 0x3d, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x70, 0x73, 0x2c, + 0x79, 0x3d, 0x6e, 0x2e, 0x70, 0x72, 0x6f, 0x70, 0x73, 0x2c, 0x6d, 0x3d, + 0x6e, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x3b, 0x69, 0x66, 0x28, 0x22, 0x73, + 0x76, 0x67, 0x22, 0x3d, 0x3d, 0x3d, 0x6d, 0x26, 0x26, 0x28, 0x69, 0x3d, + 0x21, 0x30, 0x29, 0x2c, 0x6e, 0x75, 0x6c, 0x6c, 0x21, 0x3d, 0x6f, 0x29, + 0x66, 0x6f, 0x72, 0x28, 0x6c, 0x3d, 0x30, 0x3b, 0x6c, 0x3c, 0x6f, 0x2e, + 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x3b, 0x6c, 0x2b, 0x2b, 0x29, 0x69, + 0x66, 0x28, 0x28, 0x61, 0x3d, 0x6f, 0x5b, 0x6c, 0x5d, 0x29, 0x26, 0x26, + 0x22, 0x73, 0x65, 0x74, 0x41, 0x74, 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, + 0x65, 0x22, 0x69, 0x6e, 0x20, 0x61, 0x3d, 0x3d, 0x21, 0x21, 0x6d, 0x26, + 0x26, 0x28, 0x6d, 0x3f, 0x61, 0x2e, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x4e, + 0x61, 0x6d, 0x65, 0x3d, 0x3d, 0x3d, 0x6d, 0x3a, 0x33, 0x3d, 0x3d, 0x3d, + 0x61, 0x2e, 0x6e, 0x6f, 0x64, 0x65, 0x54, 0x79, 0x70, 0x65, 0x29, 0x29, + 0x7b, 0x74, 0x3d, 0x61, 0x2c, 0x6f, 0x5b, 0x6c, 0x5d, 0x3d, 0x6e, 0x75, + 0x6c, 0x6c, 0x3b, 0x62, 0x72, 0x65, 0x61, 0x6b, 0x7d, 0x69, 0x66, 0x28, + 0x6e, 0x75, 0x6c, 0x6c, 0x3d, 0x3d, 0x74, 0x29, 0x7b, 0x69, 0x66, 0x28, + 0x6e, 0x75, 0x6c, 0x6c, 0x3d, 0x3d, 0x3d, 0x6d, 0x29, 0x72, 0x65, 0x74, + 0x75, 0x72, 0x6e, 0x20, 0x64, 0x6f, 0x63, 0x75, 0x6d, 0x65, 0x6e, 0x74, + 0x2e, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x54, 0x65, 0x78, 0x74, 0x4e, + 0x6f, 0x64, 0x65, 0x28, 0x79, 0x29, 0x3b, 0x74, 0x3d, 0x69, 0x3f, 0x64, + 0x6f, 0x63, 0x75, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x63, 0x72, 0x65, 0x61, + 0x74, 0x65, 0x45, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x4e, 0x53, 0x28, + 0x22, 0x68, 0x74, 0x74, 0x70, 0x3a, 0x2f, 0x2f, 0x77, 0x77, 0x77, 0x2e, + 0x77, 0x33, 0x2e, 0x6f, 0x72, 0x67, 0x2f, 0x32, 0x30, 0x30, 0x30, 0x2f, + 0x73, 0x76, 0x67, 0x22, 0x2c, 0x6d, 0x29, 0x3a, 0x64, 0x6f, 0x63, 0x75, + 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x45, + 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x28, 0x6d, 0x2c, 0x79, 0x2e, 0x69, + 0x73, 0x26, 0x26, 0x79, 0x29, 0x2c, 0x6f, 0x3d, 0x6e, 0x75, 0x6c, 0x6c, + 0x2c, 0x75, 0x3d, 0x21, 0x31, 0x7d, 0x69, 0x66, 0x28, 0x6e, 0x75, 0x6c, + 0x6c, 0x3d, 0x3d, 0x3d, 0x6d, 0x29, 0x76, 0x3d, 0x3d, 0x3d, 0x79, 0x7c, + 0x7c, 0x75, 0x26, 0x26, 0x74, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x3d, 0x3d, + 0x3d, 0x79, 0x7c, 0x7c, 0x28, 0x74, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x3d, + 0x79, 0x29, 0x3b, 0x65, 0x6c, 0x73, 0x65, 0x7b, 0x69, 0x66, 0x28, 0x6f, + 0x3d, 0x6f, 0x26, 0x26, 0x78, 0x2e, 0x63, 0x61, 0x6c, 0x6c, 0x28, 0x74, + 0x2e, 0x63, 0x68, 0x69, 0x6c, 0x64, 0x4e, 0x6f, 0x64, 0x65, 0x73, 0x29, + 0x2c, 0x76, 0x3d, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x70, 0x73, 0x7c, 0x7c, + 0x54, 0x2c, 0x21, 0x75, 0x26, 0x26, 0x6e, 0x75, 0x6c, 0x6c, 0x21, 0x3d, + 0x6f, 0x29, 0x66, 0x6f, 0x72, 0x28, 0x76, 0x3d, 0x7b, 0x7d, 0x2c, 0x6c, + 0x3d, 0x30, 0x3b, 0x6c, 0x3c, 0x74, 0x2e, 0x61, 0x74, 0x74, 0x72, 0x69, + 0x62, 0x75, 0x74, 0x65, 0x73, 0x2e, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, + 0x3b, 0x6c, 0x2b, 0x2b, 0x29, 0x76, 0x5b, 0x28, 0x61, 0x3d, 0x74, 0x2e, + 0x61, 0x74, 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, 0x65, 0x73, 0x5b, 0x6c, + 0x5d, 0x29, 0x2e, 0x6e, 0x61, 0x6d, 0x65, 0x5d, 0x3d, 0x61, 0x2e, 0x76, + 0x61, 0x6c, 0x75, 0x65, 0x3b, 0x66, 0x6f, 0x72, 0x28, 0x6c, 0x20, 0x69, + 0x6e, 0x20, 0x76, 0x29, 0x61, 0x3d, 0x76, 0x5b, 0x6c, 0x5d, 0x2c, 0x22, + 0x63, 0x68, 0x69, 0x6c, 0x64, 0x72, 0x65, 0x6e, 0x22, 0x3d, 0x3d, 0x6c, + 0x7c, 0x7c, 0x28, 0x22, 0x64, 0x61, 0x6e, 0x67, 0x65, 0x72, 0x6f, 0x75, + 0x73, 0x6c, 0x79, 0x53, 0x65, 0x74, 0x49, 0x6e, 0x6e, 0x65, 0x72, 0x48, + 0x54, 0x4d, 0x4c, 0x22, 0x3d, 0x3d, 0x6c, 0x3f, 0x63, 0x3d, 0x61, 0x3a, + 0x22, 0x6b, 0x65, 0x79, 0x22, 0x3d, 0x3d, 0x3d, 0x6c, 0x7c, 0x7c, 0x6c, + 0x20, 0x69, 0x6e, 0x20, 0x79, 0x7c, 0x7c, 0x6e, 0x74, 0x28, 0x74, 0x2c, + 0x6c, 0x2c, 0x6e, 0x75, 0x6c, 0x6c, 0x2c, 0x61, 0x2c, 0x69, 0x29, 0x29, + 0x3b, 0x66, 0x6f, 0x72, 0x28, 0x6c, 0x20, 0x69, 0x6e, 0x20, 0x79, 0x29, + 0x61, 0x3d, 0x79, 0x5b, 0x6c, 0x5d, 0x2c, 0x22, 0x63, 0x68, 0x69, 0x6c, + 0x64, 0x72, 0x65, 0x6e, 0x22, 0x3d, 0x3d, 0x6c, 0x3f, 0x68, 0x3d, 0x61, + 0x3a, 0x22, 0x64, 0x61, 0x6e, 0x67, 0x65, 0x72, 0x6f, 0x75, 0x73, 0x6c, + 0x79, 0x53, 0x65, 0x74, 0x49, 0x6e, 0x6e, 0x65, 0x72, 0x48, 0x54, 0x4d, + 0x4c, 0x22, 0x3d, 0x3d, 0x6c, 0x3f, 0x73, 0x3d, 0x61, 0x3a, 0x22, 0x76, + 0x61, 0x6c, 0x75, 0x65, 0x22, 0x3d, 0x3d, 0x6c, 0x3f, 0x70, 0x3d, 0x61, + 0x3a, 0x22, 0x63, 0x68, 0x65, 0x63, 0x6b, 0x65, 0x64, 0x22, 0x3d, 0x3d, + 0x6c, 0x3f, 0x64, 0x3d, 0x61, 0x3a, 0x22, 0x6b, 0x65, 0x79, 0x22, 0x3d, + 0x3d, 0x3d, 0x6c, 0x7c, 0x7c, 0x75, 0x26, 0x26, 0x22, 0x66, 0x75, 0x6e, + 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x21, 0x3d, 0x74, 0x79, 0x70, 0x65, + 0x6f, 0x66, 0x20, 0x61, 0x7c, 0x7c, 0x76, 0x5b, 0x6c, 0x5d, 0x3d, 0x3d, + 0x3d, 0x61, 0x7c, 0x7c, 0x6e, 0x74, 0x28, 0x74, 0x2c, 0x6c, 0x2c, 0x61, + 0x2c, 0x76, 0x5b, 0x6c, 0x5d, 0x2c, 0x69, 0x29, 0x3b, 0x69, 0x66, 0x28, + 0x73, 0x29, 0x75, 0x7c, 0x7c, 0x63, 0x26, 0x26, 0x28, 0x73, 0x2e, 0x5f, + 0x5f, 0x68, 0x74, 0x6d, 0x6c, 0x3d, 0x3d, 0x3d, 0x63, 0x2e, 0x5f, 0x5f, + 0x68, 0x74, 0x6d, 0x6c, 0x7c, 0x7c, 0x73, 0x2e, 0x5f, 0x5f, 0x68, 0x74, + 0x6d, 0x6c, 0x3d, 0x3d, 0x3d, 0x74, 0x2e, 0x69, 0x6e, 0x6e, 0x65, 0x72, + 0x48, 0x54, 0x4d, 0x4c, 0x29, 0x7c, 0x7c, 0x28, 0x74, 0x2e, 0x69, 0x6e, + 0x6e, 0x65, 0x72, 0x48, 0x54, 0x4d, 0x4c, 0x3d, 0x73, 0x2e, 0x5f, 0x5f, + 0x68, 0x74, 0x6d, 0x6c, 0x29, 0x2c, 0x6e, 0x2e, 0x5f, 0x5f, 0x6b, 0x3d, + 0x5b, 0x5d, 0x3b, 0x65, 0x6c, 0x73, 0x65, 0x20, 0x69, 0x66, 0x28, 0x63, + 0x26, 0x26, 0x28, 0x74, 0x2e, 0x69, 0x6e, 0x6e, 0x65, 0x72, 0x48, 0x54, + 0x4d, 0x4c, 0x3d, 0x22, 0x22, 0x29, 0x2c, 0x4b, 0x28, 0x74, 0x2c, 0x46, + 0x28, 0x68, 0x29, 0x3f, 0x68, 0x3a, 0x5b, 0x68, 0x5d, 0x2c, 0x6e, 0x2c, + 0x65, 0x2c, 0x5f, 0x2c, 0x69, 0x26, 0x26, 0x22, 0x66, 0x6f, 0x72, 0x65, + 0x69, 0x67, 0x6e, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x22, 0x21, 0x3d, + 0x3d, 0x6d, 0x2c, 0x6f, 0x2c, 0x72, 0x2c, 0x6f, 0x3f, 0x6f, 0x5b, 0x30, + 0x5d, 0x3a, 0x65, 0x2e, 0x5f, 0x5f, 0x6b, 0x26, 0x26, 0x71, 0x28, 0x65, + 0x2c, 0x30, 0x29, 0x2c, 0x75, 0x2c, 0x66, 0x29, 0x2c, 0x6e, 0x75, 0x6c, + 0x6c, 0x21, 0x3d, 0x6f, 0x29, 0x66, 0x6f, 0x72, 0x28, 0x6c, 0x3d, 0x6f, + 0x2e, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x3b, 0x6c, 0x2d, 0x2d, 0x3b, + 0x29, 0x6e, 0x75, 0x6c, 0x6c, 0x21, 0x3d, 0x6f, 0x5b, 0x6c, 0x5d, 0x26, + 0x26, 0x57, 0x28, 0x6f, 0x5b, 0x6c, 0x5d, 0x29, 0x3b, 0x75, 0x7c, 0x7c, + 0x28, 0x6c, 0x3d, 0x22, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x22, 0x2c, 0x76, + 0x6f, 0x69, 0x64, 0x20, 0x30, 0x21, 0x3d, 0x3d, 0x70, 0x26, 0x26, 0x28, + 0x70, 0x21, 0x3d, 0x3d, 0x74, 0x5b, 0x6c, 0x5d, 0x7c, 0x7c, 0x22, 0x70, + 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x22, 0x3d, 0x3d, 0x3d, 0x6d, + 0x26, 0x26, 0x21, 0x70, 0x7c, 0x7c, 0x22, 0x6f, 0x70, 0x74, 0x69, 0x6f, + 0x6e, 0x22, 0x3d, 0x3d, 0x3d, 0x6d, 0x26, 0x26, 0x70, 0x21, 0x3d, 0x3d, + 0x76, 0x5b, 0x6c, 0x5d, 0x29, 0x26, 0x26, 0x6e, 0x74, 0x28, 0x74, 0x2c, + 0x6c, 0x2c, 0x70, 0x2c, 0x76, 0x5b, 0x6c, 0x5d, 0x2c, 0x21, 0x31, 0x29, + 0x2c, 0x6c, 0x3d, 0x22, 0x63, 0x68, 0x65, 0x63, 0x6b, 0x65, 0x64, 0x22, + 0x2c, 0x76, 0x6f, 0x69, 0x64, 0x20, 0x30, 0x21, 0x3d, 0x3d, 0x64, 0x26, + 0x26, 0x64, 0x21, 0x3d, 0x3d, 0x74, 0x5b, 0x6c, 0x5d, 0x26, 0x26, 0x6e, + 0x74, 0x28, 0x74, 0x2c, 0x6c, 0x2c, 0x64, 0x2c, 0x76, 0x5b, 0x6c, 0x5d, + 0x2c, 0x21, 0x31, 0x29, 0x29, 0x7d, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, + 0x20, 0x74, 0x7d, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, + 0x75, 0x74, 0x28, 0x74, 0x2c, 0x6e, 0x2c, 0x65, 0x29, 0x7b, 0x74, 0x72, + 0x79, 0x7b, 0x22, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x22, + 0x3d, 0x3d, 0x74, 0x79, 0x70, 0x65, 0x6f, 0x66, 0x20, 0x74, 0x3f, 0x74, + 0x28, 0x6e, 0x29, 0x3a, 0x74, 0x2e, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, + 0x74, 0x3d, 0x6e, 0x7d, 0x63, 0x61, 0x74, 0x63, 0x68, 0x28, 0x74, 0x29, + 0x7b, 0x43, 0x2e, 0x5f, 0x5f, 0x65, 0x28, 0x74, 0x2c, 0x65, 0x29, 0x7d, + 0x7d, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x66, 0x74, + 0x28, 0x74, 0x2c, 0x6e, 0x2c, 0x65, 0x29, 0x7b, 0x76, 0x61, 0x72, 0x20, + 0x5f, 0x2c, 0x69, 0x3b, 0x69, 0x66, 0x28, 0x43, 0x2e, 0x75, 0x6e, 0x6d, + 0x6f, 0x75, 0x6e, 0x74, 0x26, 0x26, 0x43, 0x2e, 0x75, 0x6e, 0x6d, 0x6f, + 0x75, 0x6e, 0x74, 0x28, 0x74, 0x29, 0x2c, 0x28, 0x5f, 0x3d, 0x74, 0x2e, + 0x72, 0x65, 0x66, 0x29, 0x26, 0x26, 0x28, 0x5f, 0x2e, 0x63, 0x75, 0x72, + 0x72, 0x65, 0x6e, 0x74, 0x26, 0x26, 0x5f, 0x2e, 0x63, 0x75, 0x72, 0x72, + 0x65, 0x6e, 0x74, 0x21, 0x3d, 0x3d, 0x74, 0x2e, 0x5f, 0x5f, 0x65, 0x7c, + 0x7c, 0x75, 0x74, 0x28, 0x5f, 0x2c, 0x6e, 0x75, 0x6c, 0x6c, 0x2c, 0x6e, + 0x29, 0x29, 0x2c, 0x6e, 0x75, 0x6c, 0x6c, 0x21, 0x3d, 0x28, 0x5f, 0x3d, + 0x74, 0x2e, 0x5f, 0x5f, 0x63, 0x29, 0x29, 0x7b, 0x69, 0x66, 0x28, 0x5f, + 0x2e, 0x63, 0x6f, 0x6d, 0x70, 0x6f, 0x6e, 0x65, 0x6e, 0x74, 0x57, 0x69, + 0x6c, 0x6c, 0x55, 0x6e, 0x6d, 0x6f, 0x75, 0x6e, 0x74, 0x29, 0x74, 0x72, + 0x79, 0x7b, 0x5f, 0x2e, 0x63, 0x6f, 0x6d, 0x70, 0x6f, 0x6e, 0x65, 0x6e, + 0x74, 0x57, 0x69, 0x6c, 0x6c, 0x55, 0x6e, 0x6d, 0x6f, 0x75, 0x6e, 0x74, + 0x28, 0x29, 0x7d, 0x63, 0x61, 0x74, 0x63, 0x68, 0x28, 0x74, 0x29, 0x7b, + 0x43, 0x2e, 0x5f, 0x5f, 0x65, 0x28, 0x74, 0x2c, 0x6e, 0x29, 0x7d, 0x5f, + 0x2e, 0x62, 0x61, 0x73, 0x65, 0x3d, 0x5f, 0x2e, 0x5f, 0x5f, 0x50, 0x3d, + 0x6e, 0x75, 0x6c, 0x6c, 0x2c, 0x74, 0x2e, 0x5f, 0x5f, 0x63, 0x3d, 0x76, + 0x6f, 0x69, 0x64, 0x20, 0x30, 0x7d, 0x69, 0x66, 0x28, 0x5f, 0x3d, 0x74, + 0x2e, 0x5f, 0x5f, 0x6b, 0x29, 0x66, 0x6f, 0x72, 0x28, 0x69, 0x3d, 0x30, + 0x3b, 0x69, 0x3c, 0x5f, 0x2e, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x3b, + 0x69, 0x2b, 0x2b, 0x29, 0x5f, 0x5b, 0x69, 0x5d, 0x26, 0x26, 0x66, 0x74, + 0x28, 0x5f, 0x5b, 0x69, 0x5d, 0x2c, 0x6e, 0x2c, 0x65, 0x7c, 0x7c, 0x22, + 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x21, 0x3d, 0x74, + 0x79, 0x70, 0x65, 0x6f, 0x66, 0x20, 0x74, 0x2e, 0x74, 0x79, 0x70, 0x65, + 0x29, 0x3b, 0x65, 0x7c, 0x7c, 0x6e, 0x75, 0x6c, 0x6c, 0x3d, 0x3d, 0x74, + 0x2e, 0x5f, 0x5f, 0x65, 0x7c, 0x7c, 0x57, 0x28, 0x74, 0x2e, 0x5f, 0x5f, + 0x65, 0x29, 0x2c, 0x74, 0x2e, 0x5f, 0x5f, 0x3d, 0x74, 0x2e, 0x5f, 0x5f, + 0x65, 0x3d, 0x74, 0x2e, 0x5f, 0x5f, 0x64, 0x3d, 0x76, 0x6f, 0x69, 0x64, + 0x20, 0x30, 0x7d, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, + 0x6c, 0x74, 0x28, 0x74, 0x2c, 0x6e, 0x2c, 0x65, 0x29, 0x7b, 0x72, 0x65, + 0x74, 0x75, 0x72, 0x6e, 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x63, 0x6f, + 0x6e, 0x73, 0x74, 0x72, 0x75, 0x63, 0x74, 0x6f, 0x72, 0x28, 0x74, 0x2c, + 0x65, 0x29, 0x7d, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, + 0x73, 0x74, 0x28, 0x74, 0x2c, 0x6e, 0x2c, 0x65, 0x29, 0x7b, 0x76, 0x61, + 0x72, 0x20, 0x5f, 0x2c, 0x69, 0x2c, 0x6f, 0x2c, 0x72, 0x3b, 0x43, 0x2e, + 0x5f, 0x5f, 0x26, 0x26, 0x43, 0x2e, 0x5f, 0x5f, 0x28, 0x74, 0x2c, 0x6e, + 0x29, 0x2c, 0x69, 0x3d, 0x28, 0x5f, 0x3d, 0x22, 0x66, 0x75, 0x6e, 0x63, + 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x3d, 0x3d, 0x74, 0x79, 0x70, 0x65, 0x6f, + 0x66, 0x20, 0x65, 0x29, 0x3f, 0x6e, 0x75, 0x6c, 0x6c, 0x3a, 0x65, 0x26, + 0x26, 0x65, 0x2e, 0x5f, 0x5f, 0x6b, 0x7c, 0x7c, 0x6e, 0x2e, 0x5f, 0x5f, + 0x6b, 0x2c, 0x6f, 0x3d, 0x5b, 0x5d, 0x2c, 0x72, 0x3d, 0x5b, 0x5d, 0x2c, + 0x69, 0x74, 0x28, 0x6e, 0x2c, 0x74, 0x3d, 0x28, 0x21, 0x5f, 0x26, 0x26, + 0x65, 0x7c, 0x7c, 0x6e, 0x29, 0x2e, 0x5f, 0x5f, 0x6b, 0x3d, 0x4c, 0x28, + 0x6a, 0x2c, 0x6e, 0x75, 0x6c, 0x6c, 0x2c, 0x5b, 0x74, 0x5d, 0x29, 0x2c, + 0x69, 0x7c, 0x7c, 0x54, 0x2c, 0x54, 0x2c, 0x76, 0x6f, 0x69, 0x64, 0x20, + 0x30, 0x21, 0x3d, 0x3d, 0x6e, 0x2e, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x53, + 0x56, 0x47, 0x45, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2c, 0x21, 0x5f, + 0x26, 0x26, 0x65, 0x3f, 0x5b, 0x65, 0x5d, 0x3a, 0x69, 0x3f, 0x6e, 0x75, + 0x6c, 0x6c, 0x3a, 0x6e, 0x2e, 0x66, 0x69, 0x72, 0x73, 0x74, 0x43, 0x68, + 0x69, 0x6c, 0x64, 0x3f, 0x78, 0x2e, 0x63, 0x61, 0x6c, 0x6c, 0x28, 0x6e, + 0x2e, 0x63, 0x68, 0x69, 0x6c, 0x64, 0x4e, 0x6f, 0x64, 0x65, 0x73, 0x29, + 0x3a, 0x6e, 0x75, 0x6c, 0x6c, 0x2c, 0x6f, 0x2c, 0x21, 0x5f, 0x26, 0x26, + 0x65, 0x3f, 0x65, 0x3a, 0x69, 0x3f, 0x69, 0x2e, 0x5f, 0x5f, 0x65, 0x3a, + 0x6e, 0x2e, 0x66, 0x69, 0x72, 0x73, 0x74, 0x43, 0x68, 0x69, 0x6c, 0x64, + 0x2c, 0x5f, 0x2c, 0x72, 0x29, 0x2c, 0x74, 0x2e, 0x5f, 0x5f, 0x64, 0x3d, + 0x76, 0x6f, 0x69, 0x64, 0x20, 0x30, 0x2c, 0x6f, 0x74, 0x28, 0x6f, 0x2c, 0x74, 0x2c, 0x72, 0x29, 0x7d, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, - 0x6e, 0x20, 0x6c, 0x74, 0x28, 0x74, 0x2c, 0x6e, 0x29, 0x7b, 0x73, 0x74, - 0x28, 0x74, 0x2c, 0x6e, 0x2c, 0x6c, 0x74, 0x29, 0x7d, 0x66, 0x75, 0x6e, - 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x63, 0x74, 0x28, 0x74, 0x2c, 0x6e, + 0x6e, 0x20, 0x63, 0x74, 0x28, 0x74, 0x2c, 0x6e, 0x29, 0x7b, 0x73, 0x74, + 0x28, 0x74, 0x2c, 0x6e, 0x2c, 0x63, 0x74, 0x29, 0x7d, 0x66, 0x75, 0x6e, + 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x68, 0x74, 0x28, 0x74, 0x2c, 0x6e, 0x2c, 0x65, 0x29, 0x7b, 0x76, 0x61, 0x72, 0x20, 0x5f, 0x2c, 0x69, 0x2c, 0x6f, 0x2c, 0x72, 0x2c, 0x75, 0x3d, 0x4d, 0x28, 0x7b, 0x7d, 0x2c, 0x74, 0x2e, 0x70, 0x72, 0x6f, 0x70, 0x73, 0x29, 0x3b, 0x66, 0x6f, 0x72, 0x28, @@ -1146,7 +1165,7 @@ unsigned char index_js[] = { 0x74, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x2c, 0x75, 0x2c, 0x5f, 0x7c, 0x7c, 0x74, 0x2e, 0x6b, 0x65, 0x79, 0x2c, 0x69, 0x7c, 0x7c, 0x74, 0x2e, 0x72, 0x65, 0x66, 0x2c, 0x6e, 0x75, 0x6c, 0x6c, 0x29, 0x7d, 0x66, 0x75, 0x6e, - 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x68, 0x74, 0x28, 0x74, 0x2c, 0x6e, + 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x61, 0x74, 0x28, 0x74, 0x2c, 0x6e, 0x29, 0x7b, 0x76, 0x61, 0x72, 0x20, 0x65, 0x3d, 0x7b, 0x5f, 0x5f, 0x63, 0x3a, 0x6e, 0x3d, 0x22, 0x5f, 0x5f, 0x63, 0x43, 0x22, 0x2b, 0x44, 0x2b, 0x2b, 0x2c, 0x5f, 0x5f, 0x3a, 0x74, 0x2c, 0x43, 0x6f, 0x6e, 0x73, 0x75, @@ -1171,7 +1190,7 @@ unsigned char index_js[] = { 0x21, 0x3d, 0x3d, 0x74, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x26, 0x26, 0x65, 0x2e, 0x73, 0x6f, 0x6d, 0x65, 0x28, 0x28, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x74, 0x29, 0x7b, 0x74, 0x2e, 0x5f, 0x5f, - 0x65, 0x3d, 0x21, 0x30, 0x2c, 0x47, 0x28, 0x74, 0x29, 0x7d, 0x29, 0x29, + 0x65, 0x3d, 0x21, 0x30, 0x2c, 0x7a, 0x28, 0x74, 0x29, 0x7d, 0x29, 0x29, 0x7d, 0x2c, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x73, 0x75, 0x62, 0x3d, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x74, 0x29, 0x7b, 0x65, 0x2e, 0x70, 0x75, 0x73, 0x68, 0x28, 0x74, 0x29, 0x3b, 0x76, 0x61, 0x72, @@ -1235,14 +1254,14 @@ unsigned char index_js[] = { 0x6e, 0x75, 0x6c, 0x6c, 0x21, 0x3d, 0x74, 0x26, 0x26, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x5f, 0x5f, 0x76, 0x26, 0x26, 0x28, 0x6e, 0x26, 0x26, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x5f, 0x73, 0x62, 0x2e, 0x70, 0x75, 0x73, 0x68, - 0x28, 0x6e, 0x29, 0x2c, 0x47, 0x28, 0x74, 0x68, 0x69, 0x73, 0x29, 0x29, + 0x28, 0x6e, 0x29, 0x2c, 0x7a, 0x28, 0x74, 0x68, 0x69, 0x73, 0x29, 0x29, 0x7d, 0x2c, 0x49, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x74, 0x79, 0x70, 0x65, 0x2e, 0x66, 0x6f, 0x72, 0x63, 0x65, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x3d, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x74, 0x29, 0x7b, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x5f, 0x5f, 0x76, 0x26, 0x26, 0x28, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x5f, 0x5f, 0x65, 0x3d, 0x21, 0x30, 0x2c, 0x74, 0x26, 0x26, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x5f, 0x5f, 0x68, - 0x2e, 0x70, 0x75, 0x73, 0x68, 0x28, 0x74, 0x29, 0x2c, 0x47, 0x28, 0x74, + 0x2e, 0x70, 0x75, 0x73, 0x68, 0x28, 0x74, 0x29, 0x2c, 0x7a, 0x28, 0x74, 0x68, 0x69, 0x73, 0x29, 0x29, 0x7d, 0x2c, 0x49, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x74, 0x79, 0x70, 0x65, 0x2e, 0x72, 0x65, 0x6e, 0x64, 0x65, 0x72, 0x3d, 0x6a, 0x2c, 0x48, 0x3d, 0x5b, 0x5d, 0x2c, 0x4e, 0x3d, 0x22, @@ -1256,113 +1275,115 @@ unsigned char index_js[] = { 0x24, 0x3d, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x74, 0x2c, 0x6e, 0x29, 0x7b, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x74, 0x2e, 0x5f, 0x5f, 0x76, 0x2e, 0x5f, 0x5f, 0x62, 0x2d, 0x6e, 0x2e, 0x5f, - 0x5f, 0x76, 0x2e, 0x5f, 0x5f, 0x62, 0x7d, 0x2c, 0x7a, 0x2e, 0x5f, 0x5f, + 0x5f, 0x76, 0x2e, 0x5f, 0x5f, 0x62, 0x7d, 0x2c, 0x4a, 0x2e, 0x5f, 0x5f, 0x72, 0x3d, 0x30, 0x2c, 0x44, 0x3d, 0x30, 0x3b, 0x76, 0x61, 0x72, 0x20, - 0x61, 0x74, 0x2c, 0x70, 0x74, 0x2c, 0x64, 0x74, 0x2c, 0x76, 0x74, 0x2c, - 0x79, 0x74, 0x3d, 0x30, 0x2c, 0x6d, 0x74, 0x3d, 0x5b, 0x5d, 0x2c, 0x67, - 0x74, 0x3d, 0x5b, 0x5d, 0x2c, 0x62, 0x74, 0x3d, 0x43, 0x2e, 0x5f, 0x5f, - 0x62, 0x2c, 0x6b, 0x74, 0x3d, 0x43, 0x2e, 0x5f, 0x5f, 0x72, 0x2c, 0x53, - 0x74, 0x3d, 0x43, 0x2e, 0x64, 0x69, 0x66, 0x66, 0x65, 0x64, 0x2c, 0x77, - 0x74, 0x3d, 0x43, 0x2e, 0x5f, 0x5f, 0x63, 0x2c, 0x78, 0x74, 0x3d, 0x43, - 0x2e, 0x75, 0x6e, 0x6d, 0x6f, 0x75, 0x6e, 0x74, 0x3b, 0x66, 0x75, 0x6e, - 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x43, 0x74, 0x28, 0x74, 0x2c, 0x6e, - 0x29, 0x7b, 0x43, 0x2e, 0x5f, 0x5f, 0x68, 0x26, 0x26, 0x43, 0x2e, 0x5f, - 0x5f, 0x68, 0x28, 0x70, 0x74, 0x2c, 0x74, 0x2c, 0x79, 0x74, 0x7c, 0x7c, - 0x6e, 0x29, 0x2c, 0x79, 0x74, 0x3d, 0x30, 0x3b, 0x76, 0x61, 0x72, 0x20, - 0x65, 0x3d, 0x70, 0x74, 0x2e, 0x5f, 0x5f, 0x48, 0x7c, 0x7c, 0x28, 0x70, - 0x74, 0x2e, 0x5f, 0x5f, 0x48, 0x3d, 0x7b, 0x5f, 0x5f, 0x3a, 0x5b, 0x5d, - 0x2c, 0x5f, 0x5f, 0x68, 0x3a, 0x5b, 0x5d, 0x7d, 0x29, 0x3b, 0x72, 0x65, - 0x74, 0x75, 0x72, 0x6e, 0x20, 0x74, 0x3e, 0x3d, 0x65, 0x2e, 0x5f, 0x5f, - 0x2e, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x26, 0x26, 0x65, 0x2e, 0x5f, - 0x5f, 0x2e, 0x70, 0x75, 0x73, 0x68, 0x28, 0x7b, 0x5f, 0x5f, 0x56, 0x3a, - 0x67, 0x74, 0x7d, 0x29, 0x2c, 0x65, 0x2e, 0x5f, 0x5f, 0x5b, 0x74, 0x5d, - 0x7d, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x45, 0x74, - 0x28, 0x74, 0x29, 0x7b, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x79, - 0x74, 0x3d, 0x31, 0x2c, 0x55, 0x74, 0x28, 0x71, 0x74, 0x2c, 0x74, 0x29, - 0x7d, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x55, 0x74, - 0x28, 0x74, 0x2c, 0x6e, 0x2c, 0x65, 0x29, 0x7b, 0x76, 0x61, 0x72, 0x20, - 0x5f, 0x3d, 0x43, 0x74, 0x28, 0x61, 0x74, 0x2b, 0x2b, 0x2c, 0x32, 0x29, - 0x3b, 0x69, 0x66, 0x28, 0x5f, 0x2e, 0x74, 0x3d, 0x74, 0x2c, 0x21, 0x5f, - 0x2e, 0x5f, 0x5f, 0x63, 0x26, 0x26, 0x28, 0x5f, 0x2e, 0x5f, 0x5f, 0x3d, - 0x5b, 0x65, 0x3f, 0x65, 0x28, 0x6e, 0x29, 0x3a, 0x71, 0x74, 0x28, 0x76, - 0x6f, 0x69, 0x64, 0x20, 0x30, 0x2c, 0x6e, 0x29, 0x2c, 0x66, 0x75, 0x6e, - 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x74, 0x29, 0x7b, 0x76, 0x61, 0x72, - 0x20, 0x6e, 0x3d, 0x5f, 0x2e, 0x5f, 0x5f, 0x4e, 0x3f, 0x5f, 0x2e, 0x5f, - 0x5f, 0x4e, 0x5b, 0x30, 0x5d, 0x3a, 0x5f, 0x2e, 0x5f, 0x5f, 0x5b, 0x30, - 0x5d, 0x2c, 0x65, 0x3d, 0x5f, 0x2e, 0x74, 0x28, 0x6e, 0x2c, 0x74, 0x29, - 0x3b, 0x6e, 0x21, 0x3d, 0x3d, 0x65, 0x26, 0x26, 0x28, 0x5f, 0x2e, 0x5f, - 0x5f, 0x4e, 0x3d, 0x5b, 0x65, 0x2c, 0x5f, 0x2e, 0x5f, 0x5f, 0x5b, 0x31, - 0x5d, 0x5d, 0x2c, 0x5f, 0x2e, 0x5f, 0x5f, 0x63, 0x2e, 0x73, 0x65, 0x74, - 0x53, 0x74, 0x61, 0x74, 0x65, 0x28, 0x7b, 0x7d, 0x29, 0x29, 0x7d, 0x5d, - 0x2c, 0x5f, 0x2e, 0x5f, 0x5f, 0x63, 0x3d, 0x70, 0x74, 0x2c, 0x21, 0x70, - 0x74, 0x2e, 0x75, 0x29, 0x29, 0x7b, 0x76, 0x61, 0x72, 0x20, 0x69, 0x3d, - 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x74, 0x2c, 0x6e, - 0x2c, 0x65, 0x29, 0x7b, 0x69, 0x66, 0x28, 0x21, 0x5f, 0x2e, 0x5f, 0x5f, - 0x63, 0x2e, 0x5f, 0x5f, 0x48, 0x29, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, - 0x21, 0x30, 0x3b, 0x76, 0x61, 0x72, 0x20, 0x69, 0x3d, 0x5f, 0x2e, 0x5f, - 0x5f, 0x63, 0x2e, 0x5f, 0x5f, 0x48, 0x2e, 0x5f, 0x5f, 0x2e, 0x66, 0x69, - 0x6c, 0x74, 0x65, 0x72, 0x28, 0x28, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, - 0x6f, 0x6e, 0x28, 0x74, 0x29, 0x7b, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, - 0x20, 0x74, 0x2e, 0x5f, 0x5f, 0x63, 0x7d, 0x29, 0x29, 0x3b, 0x69, 0x66, - 0x28, 0x69, 0x2e, 0x65, 0x76, 0x65, 0x72, 0x79, 0x28, 0x28, 0x66, 0x75, - 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x74, 0x29, 0x7b, 0x72, 0x65, - 0x74, 0x75, 0x72, 0x6e, 0x21, 0x74, 0x2e, 0x5f, 0x5f, 0x4e, 0x7d, 0x29, - 0x29, 0x29, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x21, 0x6f, 0x7c, 0x7c, - 0x6f, 0x2e, 0x63, 0x61, 0x6c, 0x6c, 0x28, 0x74, 0x68, 0x69, 0x73, 0x2c, - 0x74, 0x2c, 0x6e, 0x2c, 0x65, 0x29, 0x3b, 0x76, 0x61, 0x72, 0x20, 0x72, - 0x3d, 0x21, 0x31, 0x3b, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x69, - 0x2e, 0x66, 0x6f, 0x72, 0x45, 0x61, 0x63, 0x68, 0x28, 0x28, 0x66, 0x75, - 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x74, 0x29, 0x7b, 0x69, 0x66, - 0x28, 0x74, 0x2e, 0x5f, 0x5f, 0x4e, 0x29, 0x7b, 0x76, 0x61, 0x72, 0x20, - 0x6e, 0x3d, 0x74, 0x2e, 0x5f, 0x5f, 0x5b, 0x30, 0x5d, 0x3b, 0x74, 0x2e, - 0x5f, 0x5f, 0x3d, 0x74, 0x2e, 0x5f, 0x5f, 0x4e, 0x2c, 0x74, 0x2e, 0x5f, - 0x5f, 0x4e, 0x3d, 0x76, 0x6f, 0x69, 0x64, 0x20, 0x30, 0x2c, 0x6e, 0x21, - 0x3d, 0x3d, 0x74, 0x2e, 0x5f, 0x5f, 0x5b, 0x30, 0x5d, 0x26, 0x26, 0x28, - 0x72, 0x3d, 0x21, 0x30, 0x29, 0x7d, 0x7d, 0x29, 0x29, 0x2c, 0x21, 0x28, - 0x21, 0x72, 0x26, 0x26, 0x5f, 0x2e, 0x5f, 0x5f, 0x63, 0x2e, 0x70, 0x72, - 0x6f, 0x70, 0x73, 0x3d, 0x3d, 0x3d, 0x74, 0x29, 0x26, 0x26, 0x28, 0x21, - 0x6f, 0x7c, 0x7c, 0x6f, 0x2e, 0x63, 0x61, 0x6c, 0x6c, 0x28, 0x74, 0x68, - 0x69, 0x73, 0x2c, 0x74, 0x2c, 0x6e, 0x2c, 0x65, 0x29, 0x29, 0x7d, 0x3b, - 0x70, 0x74, 0x2e, 0x75, 0x3d, 0x21, 0x30, 0x3b, 0x76, 0x61, 0x72, 0x20, - 0x6f, 0x3d, 0x70, 0x74, 0x2e, 0x73, 0x68, 0x6f, 0x75, 0x6c, 0x64, 0x43, - 0x6f, 0x6d, 0x70, 0x6f, 0x6e, 0x65, 0x6e, 0x74, 0x55, 0x70, 0x64, 0x61, - 0x74, 0x65, 0x2c, 0x72, 0x3d, 0x70, 0x74, 0x2e, 0x63, 0x6f, 0x6d, 0x70, - 0x6f, 0x6e, 0x65, 0x6e, 0x74, 0x57, 0x69, 0x6c, 0x6c, 0x55, 0x70, 0x64, - 0x61, 0x74, 0x65, 0x3b, 0x70, 0x74, 0x2e, 0x63, 0x6f, 0x6d, 0x70, 0x6f, - 0x6e, 0x65, 0x6e, 0x74, 0x57, 0x69, 0x6c, 0x6c, 0x55, 0x70, 0x64, 0x61, - 0x74, 0x65, 0x3d, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x28, - 0x74, 0x2c, 0x6e, 0x2c, 0x65, 0x29, 0x7b, 0x69, 0x66, 0x28, 0x74, 0x68, - 0x69, 0x73, 0x2e, 0x5f, 0x5f, 0x65, 0x29, 0x7b, 0x76, 0x61, 0x72, 0x20, - 0x5f, 0x3d, 0x6f, 0x3b, 0x6f, 0x3d, 0x76, 0x6f, 0x69, 0x64, 0x20, 0x30, - 0x2c, 0x69, 0x28, 0x74, 0x2c, 0x6e, 0x2c, 0x65, 0x29, 0x2c, 0x6f, 0x3d, - 0x5f, 0x7d, 0x72, 0x26, 0x26, 0x72, 0x2e, 0x63, 0x61, 0x6c, 0x6c, 0x28, - 0x74, 0x68, 0x69, 0x73, 0x2c, 0x74, 0x2c, 0x6e, 0x2c, 0x65, 0x29, 0x7d, - 0x2c, 0x70, 0x74, 0x2e, 0x73, 0x68, 0x6f, 0x75, 0x6c, 0x64, 0x43, 0x6f, - 0x6d, 0x70, 0x6f, 0x6e, 0x65, 0x6e, 0x74, 0x55, 0x70, 0x64, 0x61, 0x74, - 0x65, 0x3d, 0x69, 0x7d, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x5f, - 0x2e, 0x5f, 0x5f, 0x4e, 0x7c, 0x7c, 0x5f, 0x2e, 0x5f, 0x5f, 0x7d, 0x66, - 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x48, 0x74, 0x28, 0x74, - 0x2c, 0x6e, 0x29, 0x7b, 0x76, 0x61, 0x72, 0x20, 0x65, 0x3d, 0x43, 0x74, - 0x28, 0x61, 0x74, 0x2b, 0x2b, 0x2c, 0x33, 0x29, 0x3b, 0x21, 0x43, 0x2e, - 0x5f, 0x5f, 0x73, 0x26, 0x26, 0x49, 0x74, 0x28, 0x65, 0x2e, 0x5f, 0x5f, - 0x48, 0x2c, 0x6e, 0x29, 0x26, 0x26, 0x28, 0x65, 0x2e, 0x5f, 0x5f, 0x3d, - 0x74, 0x2c, 0x65, 0x2e, 0x69, 0x3d, 0x6e, 0x2c, 0x70, 0x74, 0x2e, 0x5f, - 0x5f, 0x48, 0x2e, 0x5f, 0x5f, 0x68, 0x2e, 0x70, 0x75, 0x73, 0x68, 0x28, - 0x65, 0x29, 0x29, 0x7d, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, - 0x20, 0x50, 0x74, 0x28, 0x74, 0x2c, 0x6e, 0x29, 0x7b, 0x76, 0x61, 0x72, - 0x20, 0x65, 0x3d, 0x43, 0x74, 0x28, 0x61, 0x74, 0x2b, 0x2b, 0x2c, 0x34, - 0x29, 0x3b, 0x21, 0x43, 0x2e, 0x5f, 0x5f, 0x73, 0x26, 0x26, 0x49, 0x74, + 0x70, 0x74, 0x2c, 0x64, 0x74, 0x2c, 0x76, 0x74, 0x2c, 0x79, 0x74, 0x2c, + 0x6d, 0x74, 0x3d, 0x30, 0x2c, 0x67, 0x74, 0x3d, 0x5b, 0x5d, 0x2c, 0x62, + 0x74, 0x3d, 0x5b, 0x5d, 0x2c, 0x6b, 0x74, 0x3d, 0x43, 0x2c, 0x53, 0x74, + 0x3d, 0x6b, 0x74, 0x2e, 0x5f, 0x5f, 0x62, 0x2c, 0x77, 0x74, 0x3d, 0x6b, + 0x74, 0x2e, 0x5f, 0x5f, 0x72, 0x2c, 0x78, 0x74, 0x3d, 0x6b, 0x74, 0x2e, + 0x64, 0x69, 0x66, 0x66, 0x65, 0x64, 0x2c, 0x43, 0x74, 0x3d, 0x6b, 0x74, + 0x2e, 0x5f, 0x5f, 0x63, 0x2c, 0x45, 0x74, 0x3d, 0x6b, 0x74, 0x2e, 0x75, + 0x6e, 0x6d, 0x6f, 0x75, 0x6e, 0x74, 0x2c, 0x55, 0x74, 0x3d, 0x6b, 0x74, + 0x2e, 0x5f, 0x5f, 0x3b, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, + 0x20, 0x48, 0x74, 0x28, 0x74, 0x2c, 0x6e, 0x29, 0x7b, 0x6b, 0x74, 0x2e, + 0x5f, 0x5f, 0x68, 0x26, 0x26, 0x6b, 0x74, 0x2e, 0x5f, 0x5f, 0x68, 0x28, + 0x64, 0x74, 0x2c, 0x74, 0x2c, 0x6d, 0x74, 0x7c, 0x7c, 0x6e, 0x29, 0x2c, + 0x6d, 0x74, 0x3d, 0x30, 0x3b, 0x76, 0x61, 0x72, 0x20, 0x65, 0x3d, 0x64, + 0x74, 0x2e, 0x5f, 0x5f, 0x48, 0x7c, 0x7c, 0x28, 0x64, 0x74, 0x2e, 0x5f, + 0x5f, 0x48, 0x3d, 0x7b, 0x5f, 0x5f, 0x3a, 0x5b, 0x5d, 0x2c, 0x5f, 0x5f, + 0x68, 0x3a, 0x5b, 0x5d, 0x7d, 0x29, 0x3b, 0x72, 0x65, 0x74, 0x75, 0x72, + 0x6e, 0x20, 0x74, 0x3e, 0x3d, 0x65, 0x2e, 0x5f, 0x5f, 0x2e, 0x6c, 0x65, + 0x6e, 0x67, 0x74, 0x68, 0x26, 0x26, 0x65, 0x2e, 0x5f, 0x5f, 0x2e, 0x70, + 0x75, 0x73, 0x68, 0x28, 0x7b, 0x5f, 0x5f, 0x56, 0x3a, 0x62, 0x74, 0x7d, + 0x29, 0x2c, 0x65, 0x2e, 0x5f, 0x5f, 0x5b, 0x74, 0x5d, 0x7d, 0x66, 0x75, + 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x50, 0x74, 0x28, 0x74, 0x29, + 0x7b, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x6d, 0x74, 0x3d, 0x31, + 0x2c, 0x4e, 0x74, 0x28, 0x7a, 0x74, 0x2c, 0x74, 0x29, 0x7d, 0x66, 0x75, + 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x4e, 0x74, 0x28, 0x74, 0x2c, + 0x6e, 0x2c, 0x65, 0x29, 0x7b, 0x76, 0x61, 0x72, 0x20, 0x5f, 0x3d, 0x48, + 0x74, 0x28, 0x70, 0x74, 0x2b, 0x2b, 0x2c, 0x32, 0x29, 0x3b, 0x69, 0x66, + 0x28, 0x5f, 0x2e, 0x74, 0x3d, 0x74, 0x2c, 0x21, 0x5f, 0x2e, 0x5f, 0x5f, + 0x63, 0x26, 0x26, 0x28, 0x5f, 0x2e, 0x5f, 0x5f, 0x3d, 0x5b, 0x65, 0x3f, + 0x65, 0x28, 0x6e, 0x29, 0x3a, 0x7a, 0x74, 0x28, 0x76, 0x6f, 0x69, 0x64, + 0x20, 0x30, 0x2c, 0x6e, 0x29, 0x2c, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, + 0x6f, 0x6e, 0x28, 0x74, 0x29, 0x7b, 0x76, 0x61, 0x72, 0x20, 0x6e, 0x3d, + 0x5f, 0x2e, 0x5f, 0x5f, 0x4e, 0x3f, 0x5f, 0x2e, 0x5f, 0x5f, 0x4e, 0x5b, + 0x30, 0x5d, 0x3a, 0x5f, 0x2e, 0x5f, 0x5f, 0x5b, 0x30, 0x5d, 0x2c, 0x65, + 0x3d, 0x5f, 0x2e, 0x74, 0x28, 0x6e, 0x2c, 0x74, 0x29, 0x3b, 0x6e, 0x21, + 0x3d, 0x3d, 0x65, 0x26, 0x26, 0x28, 0x5f, 0x2e, 0x5f, 0x5f, 0x4e, 0x3d, + 0x5b, 0x65, 0x2c, 0x5f, 0x2e, 0x5f, 0x5f, 0x5b, 0x31, 0x5d, 0x5d, 0x2c, + 0x5f, 0x2e, 0x5f, 0x5f, 0x63, 0x2e, 0x73, 0x65, 0x74, 0x53, 0x74, 0x61, + 0x74, 0x65, 0x28, 0x7b, 0x7d, 0x29, 0x29, 0x7d, 0x5d, 0x2c, 0x5f, 0x2e, + 0x5f, 0x5f, 0x63, 0x3d, 0x64, 0x74, 0x2c, 0x21, 0x64, 0x74, 0x2e, 0x75, + 0x29, 0x29, 0x7b, 0x76, 0x61, 0x72, 0x20, 0x69, 0x3d, 0x66, 0x75, 0x6e, + 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x74, 0x2c, 0x6e, 0x2c, 0x65, 0x29, + 0x7b, 0x69, 0x66, 0x28, 0x21, 0x5f, 0x2e, 0x5f, 0x5f, 0x63, 0x2e, 0x5f, + 0x5f, 0x48, 0x29, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x21, 0x30, 0x3b, + 0x76, 0x61, 0x72, 0x20, 0x69, 0x3d, 0x5f, 0x2e, 0x5f, 0x5f, 0x63, 0x2e, + 0x5f, 0x5f, 0x48, 0x2e, 0x5f, 0x5f, 0x2e, 0x66, 0x69, 0x6c, 0x74, 0x65, + 0x72, 0x28, 0x28, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x28, + 0x74, 0x29, 0x7b, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x21, 0x21, 0x74, + 0x2e, 0x5f, 0x5f, 0x63, 0x7d, 0x29, 0x29, 0x3b, 0x69, 0x66, 0x28, 0x69, + 0x2e, 0x65, 0x76, 0x65, 0x72, 0x79, 0x28, 0x28, 0x66, 0x75, 0x6e, 0x63, + 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x74, 0x29, 0x7b, 0x72, 0x65, 0x74, 0x75, + 0x72, 0x6e, 0x21, 0x74, 0x2e, 0x5f, 0x5f, 0x4e, 0x7d, 0x29, 0x29, 0x29, + 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x21, 0x6f, 0x7c, 0x7c, 0x6f, 0x2e, + 0x63, 0x61, 0x6c, 0x6c, 0x28, 0x74, 0x68, 0x69, 0x73, 0x2c, 0x74, 0x2c, + 0x6e, 0x2c, 0x65, 0x29, 0x3b, 0x76, 0x61, 0x72, 0x20, 0x72, 0x3d, 0x21, + 0x31, 0x3b, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x69, 0x2e, 0x66, + 0x6f, 0x72, 0x45, 0x61, 0x63, 0x68, 0x28, 0x28, 0x66, 0x75, 0x6e, 0x63, + 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x74, 0x29, 0x7b, 0x69, 0x66, 0x28, 0x74, + 0x2e, 0x5f, 0x5f, 0x4e, 0x29, 0x7b, 0x76, 0x61, 0x72, 0x20, 0x6e, 0x3d, + 0x74, 0x2e, 0x5f, 0x5f, 0x5b, 0x30, 0x5d, 0x3b, 0x74, 0x2e, 0x5f, 0x5f, + 0x3d, 0x74, 0x2e, 0x5f, 0x5f, 0x4e, 0x2c, 0x74, 0x2e, 0x5f, 0x5f, 0x4e, + 0x3d, 0x76, 0x6f, 0x69, 0x64, 0x20, 0x30, 0x2c, 0x6e, 0x21, 0x3d, 0x3d, + 0x74, 0x2e, 0x5f, 0x5f, 0x5b, 0x30, 0x5d, 0x26, 0x26, 0x28, 0x72, 0x3d, + 0x21, 0x30, 0x29, 0x7d, 0x7d, 0x29, 0x29, 0x2c, 0x21, 0x28, 0x21, 0x72, + 0x26, 0x26, 0x5f, 0x2e, 0x5f, 0x5f, 0x63, 0x2e, 0x70, 0x72, 0x6f, 0x70, + 0x73, 0x3d, 0x3d, 0x3d, 0x74, 0x29, 0x26, 0x26, 0x28, 0x21, 0x6f, 0x7c, + 0x7c, 0x6f, 0x2e, 0x63, 0x61, 0x6c, 0x6c, 0x28, 0x74, 0x68, 0x69, 0x73, + 0x2c, 0x74, 0x2c, 0x6e, 0x2c, 0x65, 0x29, 0x29, 0x7d, 0x3b, 0x64, 0x74, + 0x2e, 0x75, 0x3d, 0x21, 0x30, 0x3b, 0x76, 0x61, 0x72, 0x20, 0x6f, 0x3d, + 0x64, 0x74, 0x2e, 0x73, 0x68, 0x6f, 0x75, 0x6c, 0x64, 0x43, 0x6f, 0x6d, + 0x70, 0x6f, 0x6e, 0x65, 0x6e, 0x74, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, + 0x2c, 0x72, 0x3d, 0x64, 0x74, 0x2e, 0x63, 0x6f, 0x6d, 0x70, 0x6f, 0x6e, + 0x65, 0x6e, 0x74, 0x57, 0x69, 0x6c, 0x6c, 0x55, 0x70, 0x64, 0x61, 0x74, + 0x65, 0x3b, 0x64, 0x74, 0x2e, 0x63, 0x6f, 0x6d, 0x70, 0x6f, 0x6e, 0x65, + 0x6e, 0x74, 0x57, 0x69, 0x6c, 0x6c, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, + 0x3d, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x74, 0x2c, + 0x6e, 0x2c, 0x65, 0x29, 0x7b, 0x69, 0x66, 0x28, 0x74, 0x68, 0x69, 0x73, + 0x2e, 0x5f, 0x5f, 0x65, 0x29, 0x7b, 0x76, 0x61, 0x72, 0x20, 0x5f, 0x3d, + 0x6f, 0x3b, 0x6f, 0x3d, 0x76, 0x6f, 0x69, 0x64, 0x20, 0x30, 0x2c, 0x69, + 0x28, 0x74, 0x2c, 0x6e, 0x2c, 0x65, 0x29, 0x2c, 0x6f, 0x3d, 0x5f, 0x7d, + 0x72, 0x26, 0x26, 0x72, 0x2e, 0x63, 0x61, 0x6c, 0x6c, 0x28, 0x74, 0x68, + 0x69, 0x73, 0x2c, 0x74, 0x2c, 0x6e, 0x2c, 0x65, 0x29, 0x7d, 0x2c, 0x64, + 0x74, 0x2e, 0x73, 0x68, 0x6f, 0x75, 0x6c, 0x64, 0x43, 0x6f, 0x6d, 0x70, + 0x6f, 0x6e, 0x65, 0x6e, 0x74, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x3d, + 0x69, 0x7d, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x5f, 0x2e, 0x5f, + 0x5f, 0x4e, 0x7c, 0x7c, 0x5f, 0x2e, 0x5f, 0x5f, 0x7d, 0x66, 0x75, 0x6e, + 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x24, 0x74, 0x28, 0x74, 0x2c, 0x6e, + 0x29, 0x7b, 0x76, 0x61, 0x72, 0x20, 0x65, 0x3d, 0x48, 0x74, 0x28, 0x70, + 0x74, 0x2b, 0x2b, 0x2c, 0x33, 0x29, 0x3b, 0x21, 0x6b, 0x74, 0x2e, 0x5f, + 0x5f, 0x73, 0x26, 0x26, 0x47, 0x74, 0x28, 0x65, 0x2e, 0x5f, 0x5f, 0x48, + 0x2c, 0x6e, 0x29, 0x26, 0x26, 0x28, 0x65, 0x2e, 0x5f, 0x5f, 0x3d, 0x74, + 0x2c, 0x65, 0x2e, 0x69, 0x3d, 0x6e, 0x2c, 0x64, 0x74, 0x2e, 0x5f, 0x5f, + 0x48, 0x2e, 0x5f, 0x5f, 0x68, 0x2e, 0x70, 0x75, 0x73, 0x68, 0x28, 0x65, + 0x29, 0x29, 0x7d, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, + 0x44, 0x74, 0x28, 0x74, 0x2c, 0x6e, 0x29, 0x7b, 0x76, 0x61, 0x72, 0x20, + 0x65, 0x3d, 0x48, 0x74, 0x28, 0x70, 0x74, 0x2b, 0x2b, 0x2c, 0x34, 0x29, + 0x3b, 0x21, 0x6b, 0x74, 0x2e, 0x5f, 0x5f, 0x73, 0x26, 0x26, 0x47, 0x74, 0x28, 0x65, 0x2e, 0x5f, 0x5f, 0x48, 0x2c, 0x6e, 0x29, 0x26, 0x26, 0x28, 0x65, 0x2e, 0x5f, 0x5f, 0x3d, 0x74, 0x2c, 0x65, 0x2e, 0x69, 0x3d, 0x6e, - 0x2c, 0x70, 0x74, 0x2e, 0x5f, 0x5f, 0x68, 0x2e, 0x70, 0x75, 0x73, 0x68, + 0x2c, 0x64, 0x74, 0x2e, 0x5f, 0x5f, 0x68, 0x2e, 0x70, 0x75, 0x73, 0x68, 0x28, 0x65, 0x29, 0x29, 0x7d, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, - 0x6e, 0x20, 0x4e, 0x74, 0x28, 0x74, 0x29, 0x7b, 0x72, 0x65, 0x74, 0x75, - 0x72, 0x6e, 0x20, 0x79, 0x74, 0x3d, 0x35, 0x2c, 0x44, 0x74, 0x28, 0x28, + 0x6e, 0x20, 0x54, 0x74, 0x28, 0x74, 0x29, 0x7b, 0x72, 0x65, 0x74, 0x75, + 0x72, 0x6e, 0x20, 0x6d, 0x74, 0x3d, 0x35, 0x2c, 0x41, 0x74, 0x28, 0x28, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x29, 0x7b, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x7b, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x74, 0x3a, 0x74, 0x7d, 0x7d, 0x29, 0x2c, 0x5b, 0x5d, 0x29, 0x7d, 0x66, - 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x24, 0x74, 0x28, 0x74, - 0x2c, 0x6e, 0x2c, 0x65, 0x29, 0x7b, 0x79, 0x74, 0x3d, 0x36, 0x2c, 0x50, + 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x56, 0x74, 0x28, 0x74, + 0x2c, 0x6e, 0x2c, 0x65, 0x29, 0x7b, 0x6d, 0x74, 0x3d, 0x36, 0x2c, 0x44, 0x74, 0x28, 0x28, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x29, 0x7b, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x22, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x3d, 0x3d, 0x74, 0x79, 0x70, 0x65, @@ -1377,197 +1398,204 @@ unsigned char index_js[] = { 0x64, 0x20, 0x30, 0x7d, 0x29, 0x2c, 0x6e, 0x75, 0x6c, 0x6c, 0x3d, 0x3d, 0x65, 0x3f, 0x65, 0x3a, 0x65, 0x2e, 0x63, 0x6f, 0x6e, 0x63, 0x61, 0x74, 0x28, 0x74, 0x29, 0x29, 0x7d, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, - 0x6e, 0x20, 0x44, 0x74, 0x28, 0x74, 0x2c, 0x6e, 0x29, 0x7b, 0x76, 0x61, - 0x72, 0x20, 0x65, 0x3d, 0x43, 0x74, 0x28, 0x61, 0x74, 0x2b, 0x2b, 0x2c, - 0x37, 0x29, 0x3b, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x49, 0x74, + 0x6e, 0x20, 0x41, 0x74, 0x28, 0x74, 0x2c, 0x6e, 0x29, 0x7b, 0x76, 0x61, + 0x72, 0x20, 0x65, 0x3d, 0x48, 0x74, 0x28, 0x70, 0x74, 0x2b, 0x2b, 0x2c, + 0x37, 0x29, 0x3b, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x47, 0x74, 0x28, 0x65, 0x2e, 0x5f, 0x5f, 0x48, 0x2c, 0x6e, 0x29, 0x3f, 0x28, 0x65, 0x2e, 0x5f, 0x5f, 0x56, 0x3d, 0x74, 0x28, 0x29, 0x2c, 0x65, 0x2e, 0x69, 0x3d, 0x6e, 0x2c, 0x65, 0x2e, 0x5f, 0x5f, 0x68, 0x3d, 0x74, 0x2c, 0x65, 0x2e, 0x5f, 0x5f, 0x56, 0x29, 0x3a, 0x65, 0x2e, 0x5f, 0x5f, 0x7d, 0x66, - 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x54, 0x74, 0x28, 0x74, - 0x2c, 0x6e, 0x29, 0x7b, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x79, - 0x74, 0x3d, 0x38, 0x2c, 0x44, 0x74, 0x28, 0x28, 0x66, 0x75, 0x6e, 0x63, + 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x46, 0x74, 0x28, 0x74, + 0x2c, 0x6e, 0x29, 0x7b, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x6d, + 0x74, 0x3d, 0x38, 0x2c, 0x41, 0x74, 0x28, 0x28, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x29, 0x7b, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x74, 0x7d, 0x29, 0x2c, 0x6e, 0x29, 0x7d, 0x66, 0x75, 0x6e, - 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x56, 0x74, 0x28, 0x74, 0x29, 0x7b, - 0x76, 0x61, 0x72, 0x20, 0x6e, 0x3d, 0x70, 0x74, 0x2e, 0x63, 0x6f, 0x6e, + 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x4d, 0x74, 0x28, 0x74, 0x29, 0x7b, + 0x76, 0x61, 0x72, 0x20, 0x6e, 0x3d, 0x64, 0x74, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x5b, 0x74, 0x2e, 0x5f, 0x5f, 0x63, 0x5d, 0x2c, - 0x65, 0x3d, 0x43, 0x74, 0x28, 0x61, 0x74, 0x2b, 0x2b, 0x2c, 0x39, 0x29, + 0x65, 0x3d, 0x48, 0x74, 0x28, 0x70, 0x74, 0x2b, 0x2b, 0x2c, 0x39, 0x29, 0x3b, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x65, 0x2e, 0x63, 0x3d, 0x74, 0x2c, 0x6e, 0x3f, 0x28, 0x6e, 0x75, 0x6c, 0x6c, 0x3d, 0x3d, 0x65, 0x2e, 0x5f, 0x5f, 0x26, 0x26, 0x28, 0x65, 0x2e, 0x5f, 0x5f, 0x3d, 0x21, - 0x30, 0x2c, 0x6e, 0x2e, 0x73, 0x75, 0x62, 0x28, 0x70, 0x74, 0x29, 0x29, + 0x30, 0x2c, 0x6e, 0x2e, 0x73, 0x75, 0x62, 0x28, 0x64, 0x74, 0x29, 0x29, 0x2c, 0x6e, 0x2e, 0x70, 0x72, 0x6f, 0x70, 0x73, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x29, 0x3a, 0x74, 0x2e, 0x5f, 0x5f, 0x7d, 0x66, 0x75, 0x6e, - 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x41, 0x74, 0x28, 0x74, 0x2c, 0x6e, - 0x29, 0x7b, 0x43, 0x2e, 0x75, 0x73, 0x65, 0x44, 0x65, 0x62, 0x75, 0x67, - 0x56, 0x61, 0x6c, 0x75, 0x65, 0x26, 0x26, 0x43, 0x2e, 0x75, 0x73, 0x65, - 0x44, 0x65, 0x62, 0x75, 0x67, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x28, 0x6e, - 0x3f, 0x6e, 0x28, 0x74, 0x29, 0x3a, 0x74, 0x29, 0x7d, 0x66, 0x75, 0x6e, - 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x46, 0x74, 0x28, 0x74, 0x29, 0x7b, - 0x76, 0x61, 0x72, 0x20, 0x6e, 0x3d, 0x43, 0x74, 0x28, 0x61, 0x74, 0x2b, - 0x2b, 0x2c, 0x31, 0x30, 0x29, 0x2c, 0x65, 0x3d, 0x45, 0x74, 0x28, 0x29, - 0x3b, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x6e, 0x2e, 0x5f, 0x5f, - 0x3d, 0x74, 0x2c, 0x70, 0x74, 0x2e, 0x63, 0x6f, 0x6d, 0x70, 0x6f, 0x6e, - 0x65, 0x6e, 0x74, 0x44, 0x69, 0x64, 0x43, 0x61, 0x74, 0x63, 0x68, 0x7c, - 0x7c, 0x28, 0x70, 0x74, 0x2e, 0x63, 0x6f, 0x6d, 0x70, 0x6f, 0x6e, 0x65, - 0x6e, 0x74, 0x44, 0x69, 0x64, 0x43, 0x61, 0x74, 0x63, 0x68, 0x3d, 0x66, - 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x74, 0x2c, 0x5f, 0x29, - 0x7b, 0x6e, 0x2e, 0x5f, 0x5f, 0x26, 0x26, 0x6e, 0x2e, 0x5f, 0x5f, 0x28, - 0x74, 0x2c, 0x5f, 0x29, 0x2c, 0x65, 0x5b, 0x31, 0x5d, 0x28, 0x74, 0x29, - 0x7d, 0x29, 0x2c, 0x5b, 0x65, 0x5b, 0x30, 0x5d, 0x2c, 0x66, 0x75, 0x6e, - 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x29, 0x7b, 0x65, 0x5b, 0x31, 0x5d, - 0x28, 0x76, 0x6f, 0x69, 0x64, 0x20, 0x30, 0x29, 0x7d, 0x5d, 0x7d, 0x66, - 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x4d, 0x74, 0x28, 0x29, - 0x7b, 0x76, 0x61, 0x72, 0x20, 0x74, 0x3d, 0x43, 0x74, 0x28, 0x61, 0x74, - 0x2b, 0x2b, 0x2c, 0x31, 0x31, 0x29, 0x3b, 0x69, 0x66, 0x28, 0x21, 0x74, - 0x2e, 0x5f, 0x5f, 0x29, 0x7b, 0x66, 0x6f, 0x72, 0x28, 0x76, 0x61, 0x72, - 0x20, 0x6e, 0x3d, 0x70, 0x74, 0x2e, 0x5f, 0x5f, 0x76, 0x3b, 0x6e, 0x75, - 0x6c, 0x6c, 0x21, 0x3d, 0x3d, 0x6e, 0x26, 0x26, 0x21, 0x6e, 0x2e, 0x5f, - 0x5f, 0x6d, 0x26, 0x26, 0x6e, 0x75, 0x6c, 0x6c, 0x21, 0x3d, 0x3d, 0x6e, - 0x2e, 0x5f, 0x5f, 0x3b, 0x29, 0x6e, 0x3d, 0x6e, 0x2e, 0x5f, 0x5f, 0x3b, - 0x76, 0x61, 0x72, 0x20, 0x65, 0x3d, 0x6e, 0x2e, 0x5f, 0x5f, 0x6d, 0x7c, - 0x7c, 0x28, 0x6e, 0x2e, 0x5f, 0x5f, 0x6d, 0x3d, 0x5b, 0x30, 0x2c, 0x30, - 0x5d, 0x29, 0x3b, 0x74, 0x2e, 0x5f, 0x5f, 0x3d, 0x22, 0x50, 0x22, 0x2b, - 0x65, 0x5b, 0x30, 0x5d, 0x2b, 0x22, 0x2d, 0x22, 0x2b, 0x65, 0x5b, 0x31, - 0x5d, 0x2b, 0x2b, 0x7d, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x74, - 0x2e, 0x5f, 0x5f, 0x7d, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, - 0x20, 0x57, 0x74, 0x28, 0x29, 0x7b, 0x66, 0x6f, 0x72, 0x28, 0x76, 0x61, - 0x72, 0x20, 0x74, 0x3b, 0x74, 0x3d, 0x6d, 0x74, 0x2e, 0x73, 0x68, 0x69, - 0x66, 0x74, 0x28, 0x29, 0x3b, 0x29, 0x69, 0x66, 0x28, 0x74, 0x2e, 0x5f, - 0x5f, 0x50, 0x26, 0x26, 0x74, 0x2e, 0x5f, 0x5f, 0x48, 0x29, 0x74, 0x72, - 0x79, 0x7b, 0x74, 0x2e, 0x5f, 0x5f, 0x48, 0x2e, 0x5f, 0x5f, 0x68, 0x2e, - 0x66, 0x6f, 0x72, 0x45, 0x61, 0x63, 0x68, 0x28, 0x52, 0x74, 0x29, 0x2c, - 0x74, 0x2e, 0x5f, 0x5f, 0x48, 0x2e, 0x5f, 0x5f, 0x68, 0x2e, 0x66, 0x6f, - 0x72, 0x45, 0x61, 0x63, 0x68, 0x28, 0x6a, 0x74, 0x29, 0x2c, 0x74, 0x2e, - 0x5f, 0x5f, 0x48, 0x2e, 0x5f, 0x5f, 0x68, 0x3d, 0x5b, 0x5d, 0x7d, 0x63, - 0x61, 0x74, 0x63, 0x68, 0x28, 0x75, 0x29, 0x7b, 0x74, 0x2e, 0x5f, 0x5f, - 0x48, 0x2e, 0x5f, 0x5f, 0x68, 0x3d, 0x5b, 0x5d, 0x2c, 0x43, 0x2e, 0x5f, - 0x5f, 0x65, 0x28, 0x75, 0x2c, 0x74, 0x2e, 0x5f, 0x5f, 0x76, 0x29, 0x7d, - 0x7d, 0x43, 0x2e, 0x5f, 0x5f, 0x62, 0x3d, 0x66, 0x75, 0x6e, 0x63, 0x74, - 0x69, 0x6f, 0x6e, 0x28, 0x74, 0x29, 0x7b, 0x70, 0x74, 0x3d, 0x6e, 0x75, - 0x6c, 0x6c, 0x2c, 0x62, 0x74, 0x26, 0x26, 0x62, 0x74, 0x28, 0x74, 0x29, - 0x7d, 0x2c, 0x43, 0x2e, 0x5f, 0x5f, 0x72, 0x3d, 0x66, 0x75, 0x6e, 0x63, - 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x74, 0x29, 0x7b, 0x6b, 0x74, 0x26, 0x26, - 0x6b, 0x74, 0x28, 0x74, 0x29, 0x2c, 0x61, 0x74, 0x3d, 0x30, 0x3b, 0x76, - 0x61, 0x72, 0x20, 0x6e, 0x3d, 0x28, 0x70, 0x74, 0x3d, 0x74, 0x2e, 0x5f, - 0x5f, 0x63, 0x29, 0x2e, 0x5f, 0x5f, 0x48, 0x3b, 0x6e, 0x26, 0x26, 0x28, - 0x64, 0x74, 0x3d, 0x3d, 0x3d, 0x70, 0x74, 0x3f, 0x28, 0x6e, 0x2e, 0x5f, - 0x5f, 0x68, 0x3d, 0x5b, 0x5d, 0x2c, 0x70, 0x74, 0x2e, 0x5f, 0x5f, 0x68, - 0x3d, 0x5b, 0x5d, 0x2c, 0x6e, 0x2e, 0x5f, 0x5f, 0x2e, 0x66, 0x6f, 0x72, - 0x45, 0x61, 0x63, 0x68, 0x28, 0x28, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, - 0x6f, 0x6e, 0x28, 0x74, 0x29, 0x7b, 0x74, 0x2e, 0x5f, 0x5f, 0x4e, 0x26, - 0x26, 0x28, 0x74, 0x2e, 0x5f, 0x5f, 0x3d, 0x74, 0x2e, 0x5f, 0x5f, 0x4e, - 0x29, 0x2c, 0x74, 0x2e, 0x5f, 0x5f, 0x56, 0x3d, 0x67, 0x74, 0x2c, 0x74, - 0x2e, 0x5f, 0x5f, 0x4e, 0x3d, 0x74, 0x2e, 0x69, 0x3d, 0x76, 0x6f, 0x69, - 0x64, 0x20, 0x30, 0x7d, 0x29, 0x29, 0x29, 0x3a, 0x28, 0x6e, 0x2e, 0x5f, - 0x5f, 0x68, 0x2e, 0x66, 0x6f, 0x72, 0x45, 0x61, 0x63, 0x68, 0x28, 0x52, - 0x74, 0x29, 0x2c, 0x6e, 0x2e, 0x5f, 0x5f, 0x68, 0x2e, 0x66, 0x6f, 0x72, - 0x45, 0x61, 0x63, 0x68, 0x28, 0x6a, 0x74, 0x29, 0x2c, 0x6e, 0x2e, 0x5f, - 0x5f, 0x68, 0x3d, 0x5b, 0x5d, 0x2c, 0x61, 0x74, 0x3d, 0x30, 0x29, 0x29, - 0x2c, 0x64, 0x74, 0x3d, 0x70, 0x74, 0x7d, 0x2c, 0x43, 0x2e, 0x64, 0x69, - 0x66, 0x66, 0x65, 0x64, 0x3d, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, - 0x6e, 0x28, 0x74, 0x29, 0x7b, 0x53, 0x74, 0x26, 0x26, 0x53, 0x74, 0x28, - 0x74, 0x29, 0x3b, 0x76, 0x61, 0x72, 0x20, 0x6e, 0x3d, 0x74, 0x2e, 0x5f, - 0x5f, 0x63, 0x3b, 0x6e, 0x26, 0x26, 0x6e, 0x2e, 0x5f, 0x5f, 0x48, 0x26, - 0x26, 0x28, 0x6e, 0x2e, 0x5f, 0x5f, 0x48, 0x2e, 0x5f, 0x5f, 0x68, 0x2e, - 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x26, 0x26, 0x28, 0x31, 0x21, 0x3d, - 0x3d, 0x6d, 0x74, 0x2e, 0x70, 0x75, 0x73, 0x68, 0x28, 0x6e, 0x29, 0x26, - 0x26, 0x76, 0x74, 0x3d, 0x3d, 0x3d, 0x43, 0x2e, 0x72, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x41, 0x6e, 0x69, 0x6d, 0x61, 0x74, 0x69, 0x6f, 0x6e, - 0x46, 0x72, 0x61, 0x6d, 0x65, 0x7c, 0x7c, 0x28, 0x28, 0x76, 0x74, 0x3d, - 0x43, 0x2e, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x41, 0x6e, 0x69, - 0x6d, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x46, 0x72, 0x61, 0x6d, 0x65, 0x29, - 0x7c, 0x7c, 0x4f, 0x74, 0x29, 0x28, 0x57, 0x74, 0x29, 0x29, 0x2c, 0x6e, - 0x2e, 0x5f, 0x5f, 0x48, 0x2e, 0x5f, 0x5f, 0x2e, 0x66, 0x6f, 0x72, 0x45, - 0x61, 0x63, 0x68, 0x28, 0x28, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, - 0x6e, 0x28, 0x74, 0x29, 0x7b, 0x74, 0x2e, 0x69, 0x26, 0x26, 0x28, 0x74, - 0x2e, 0x5f, 0x5f, 0x48, 0x3d, 0x74, 0x2e, 0x69, 0x29, 0x2c, 0x74, 0x2e, - 0x5f, 0x5f, 0x56, 0x21, 0x3d, 0x3d, 0x67, 0x74, 0x26, 0x26, 0x28, 0x74, - 0x2e, 0x5f, 0x5f, 0x3d, 0x74, 0x2e, 0x5f, 0x5f, 0x56, 0x29, 0x2c, 0x74, - 0x2e, 0x69, 0x3d, 0x76, 0x6f, 0x69, 0x64, 0x20, 0x30, 0x2c, 0x74, 0x2e, - 0x5f, 0x5f, 0x56, 0x3d, 0x67, 0x74, 0x7d, 0x29, 0x29, 0x29, 0x2c, 0x64, - 0x74, 0x3d, 0x70, 0x74, 0x3d, 0x6e, 0x75, 0x6c, 0x6c, 0x7d, 0x2c, 0x43, - 0x2e, 0x5f, 0x5f, 0x63, 0x3d, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, - 0x6e, 0x28, 0x74, 0x2c, 0x6e, 0x29, 0x7b, 0x6e, 0x2e, 0x73, 0x6f, 0x6d, - 0x65, 0x28, 0x28, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x28, - 0x74, 0x29, 0x7b, 0x74, 0x72, 0x79, 0x7b, 0x74, 0x2e, 0x5f, 0x5f, 0x68, - 0x2e, 0x66, 0x6f, 0x72, 0x45, 0x61, 0x63, 0x68, 0x28, 0x52, 0x74, 0x29, - 0x2c, 0x74, 0x2e, 0x5f, 0x5f, 0x68, 0x3d, 0x74, 0x2e, 0x5f, 0x5f, 0x68, - 0x2e, 0x66, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x28, 0x28, 0x66, 0x75, 0x6e, - 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x74, 0x29, 0x7b, 0x72, 0x65, 0x74, - 0x75, 0x72, 0x6e, 0x21, 0x74, 0x2e, 0x5f, 0x5f, 0x7c, 0x7c, 0x6a, 0x74, - 0x28, 0x74, 0x29, 0x7d, 0x29, 0x29, 0x7d, 0x63, 0x61, 0x74, 0x63, 0x68, - 0x28, 0x6c, 0x29, 0x7b, 0x6e, 0x2e, 0x73, 0x6f, 0x6d, 0x65, 0x28, 0x28, - 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x74, 0x29, 0x7b, - 0x74, 0x2e, 0x5f, 0x5f, 0x68, 0x26, 0x26, 0x28, 0x74, 0x2e, 0x5f, 0x5f, - 0x68, 0x3d, 0x5b, 0x5d, 0x29, 0x7d, 0x29, 0x29, 0x2c, 0x6e, 0x3d, 0x5b, - 0x5d, 0x2c, 0x43, 0x2e, 0x5f, 0x5f, 0x65, 0x28, 0x6c, 0x2c, 0x74, 0x2e, - 0x5f, 0x5f, 0x76, 0x29, 0x7d, 0x7d, 0x29, 0x29, 0x2c, 0x77, 0x74, 0x26, - 0x26, 0x77, 0x74, 0x28, 0x74, 0x2c, 0x6e, 0x29, 0x7d, 0x2c, 0x43, 0x2e, - 0x75, 0x6e, 0x6d, 0x6f, 0x75, 0x6e, 0x74, 0x3d, 0x66, 0x75, 0x6e, 0x63, - 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x74, 0x29, 0x7b, 0x78, 0x74, 0x26, 0x26, - 0x78, 0x74, 0x28, 0x74, 0x29, 0x3b, 0x76, 0x61, 0x72, 0x20, 0x6e, 0x2c, - 0x65, 0x3d, 0x74, 0x2e, 0x5f, 0x5f, 0x63, 0x3b, 0x65, 0x26, 0x26, 0x65, - 0x2e, 0x5f, 0x5f, 0x48, 0x26, 0x26, 0x28, 0x65, 0x2e, 0x5f, 0x5f, 0x48, + 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x57, 0x74, 0x28, 0x74, 0x2c, 0x6e, + 0x29, 0x7b, 0x6b, 0x74, 0x2e, 0x75, 0x73, 0x65, 0x44, 0x65, 0x62, 0x75, + 0x67, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x26, 0x26, 0x6b, 0x74, 0x2e, 0x75, + 0x73, 0x65, 0x44, 0x65, 0x62, 0x75, 0x67, 0x56, 0x61, 0x6c, 0x75, 0x65, + 0x28, 0x6e, 0x3f, 0x6e, 0x28, 0x74, 0x29, 0x3a, 0x74, 0x29, 0x7d, 0x66, + 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x4c, 0x74, 0x28, 0x74, + 0x29, 0x7b, 0x76, 0x61, 0x72, 0x20, 0x6e, 0x3d, 0x48, 0x74, 0x28, 0x70, + 0x74, 0x2b, 0x2b, 0x2c, 0x31, 0x30, 0x29, 0x2c, 0x65, 0x3d, 0x50, 0x74, + 0x28, 0x29, 0x3b, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x6e, 0x2e, + 0x5f, 0x5f, 0x3d, 0x74, 0x2c, 0x64, 0x74, 0x2e, 0x63, 0x6f, 0x6d, 0x70, + 0x6f, 0x6e, 0x65, 0x6e, 0x74, 0x44, 0x69, 0x64, 0x43, 0x61, 0x74, 0x63, + 0x68, 0x7c, 0x7c, 0x28, 0x64, 0x74, 0x2e, 0x63, 0x6f, 0x6d, 0x70, 0x6f, + 0x6e, 0x65, 0x6e, 0x74, 0x44, 0x69, 0x64, 0x43, 0x61, 0x74, 0x63, 0x68, + 0x3d, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x74, 0x2c, + 0x5f, 0x29, 0x7b, 0x6e, 0x2e, 0x5f, 0x5f, 0x26, 0x26, 0x6e, 0x2e, 0x5f, + 0x5f, 0x28, 0x74, 0x2c, 0x5f, 0x29, 0x2c, 0x65, 0x5b, 0x31, 0x5d, 0x28, + 0x74, 0x29, 0x7d, 0x29, 0x2c, 0x5b, 0x65, 0x5b, 0x30, 0x5d, 0x2c, 0x66, + 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x29, 0x7b, 0x65, 0x5b, + 0x31, 0x5d, 0x28, 0x76, 0x6f, 0x69, 0x64, 0x20, 0x30, 0x29, 0x7d, 0x5d, + 0x7d, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x4f, 0x74, + 0x28, 0x29, 0x7b, 0x76, 0x61, 0x72, 0x20, 0x74, 0x3d, 0x48, 0x74, 0x28, + 0x70, 0x74, 0x2b, 0x2b, 0x2c, 0x31, 0x31, 0x29, 0x3b, 0x69, 0x66, 0x28, + 0x21, 0x74, 0x2e, 0x5f, 0x5f, 0x29, 0x7b, 0x66, 0x6f, 0x72, 0x28, 0x76, + 0x61, 0x72, 0x20, 0x6e, 0x3d, 0x64, 0x74, 0x2e, 0x5f, 0x5f, 0x76, 0x3b, + 0x6e, 0x75, 0x6c, 0x6c, 0x21, 0x3d, 0x3d, 0x6e, 0x26, 0x26, 0x21, 0x6e, + 0x2e, 0x5f, 0x5f, 0x6d, 0x26, 0x26, 0x6e, 0x75, 0x6c, 0x6c, 0x21, 0x3d, + 0x3d, 0x6e, 0x2e, 0x5f, 0x5f, 0x3b, 0x29, 0x6e, 0x3d, 0x6e, 0x2e, 0x5f, + 0x5f, 0x3b, 0x76, 0x61, 0x72, 0x20, 0x65, 0x3d, 0x6e, 0x2e, 0x5f, 0x5f, + 0x6d, 0x7c, 0x7c, 0x28, 0x6e, 0x2e, 0x5f, 0x5f, 0x6d, 0x3d, 0x5b, 0x30, + 0x2c, 0x30, 0x5d, 0x29, 0x3b, 0x74, 0x2e, 0x5f, 0x5f, 0x3d, 0x22, 0x50, + 0x22, 0x2b, 0x65, 0x5b, 0x30, 0x5d, 0x2b, 0x22, 0x2d, 0x22, 0x2b, 0x65, + 0x5b, 0x31, 0x5d, 0x2b, 0x2b, 0x7d, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, + 0x20, 0x74, 0x2e, 0x5f, 0x5f, 0x7d, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, + 0x6f, 0x6e, 0x20, 0x52, 0x74, 0x28, 0x29, 0x7b, 0x66, 0x6f, 0x72, 0x28, + 0x76, 0x61, 0x72, 0x20, 0x74, 0x3b, 0x74, 0x3d, 0x67, 0x74, 0x2e, 0x73, + 0x68, 0x69, 0x66, 0x74, 0x28, 0x29, 0x3b, 0x29, 0x69, 0x66, 0x28, 0x74, + 0x2e, 0x5f, 0x5f, 0x50, 0x26, 0x26, 0x74, 0x2e, 0x5f, 0x5f, 0x48, 0x29, + 0x74, 0x72, 0x79, 0x7b, 0x74, 0x2e, 0x5f, 0x5f, 0x48, 0x2e, 0x5f, 0x5f, + 0x68, 0x2e, 0x66, 0x6f, 0x72, 0x45, 0x61, 0x63, 0x68, 0x28, 0x71, 0x74, + 0x29, 0x2c, 0x74, 0x2e, 0x5f, 0x5f, 0x48, 0x2e, 0x5f, 0x5f, 0x68, 0x2e, + 0x66, 0x6f, 0x72, 0x45, 0x61, 0x63, 0x68, 0x28, 0x42, 0x74, 0x29, 0x2c, + 0x74, 0x2e, 0x5f, 0x5f, 0x48, 0x2e, 0x5f, 0x5f, 0x68, 0x3d, 0x5b, 0x5d, + 0x7d, 0x63, 0x61, 0x74, 0x63, 0x68, 0x28, 0x6e, 0x29, 0x7b, 0x74, 0x2e, + 0x5f, 0x5f, 0x48, 0x2e, 0x5f, 0x5f, 0x68, 0x3d, 0x5b, 0x5d, 0x2c, 0x6b, + 0x74, 0x2e, 0x5f, 0x5f, 0x65, 0x28, 0x6e, 0x2c, 0x74, 0x2e, 0x5f, 0x5f, + 0x76, 0x29, 0x7d, 0x7d, 0x6b, 0x74, 0x2e, 0x5f, 0x5f, 0x62, 0x3d, 0x66, + 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x74, 0x29, 0x7b, 0x64, + 0x74, 0x3d, 0x6e, 0x75, 0x6c, 0x6c, 0x2c, 0x53, 0x74, 0x26, 0x26, 0x53, + 0x74, 0x28, 0x74, 0x29, 0x7d, 0x2c, 0x6b, 0x74, 0x2e, 0x5f, 0x5f, 0x3d, + 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x74, 0x2c, 0x6e, + 0x29, 0x7b, 0x74, 0x26, 0x26, 0x6e, 0x2e, 0x5f, 0x5f, 0x6b, 0x26, 0x26, + 0x6e, 0x2e, 0x5f, 0x5f, 0x6b, 0x2e, 0x5f, 0x5f, 0x6d, 0x26, 0x26, 0x28, + 0x74, 0x2e, 0x5f, 0x5f, 0x6d, 0x3d, 0x6e, 0x2e, 0x5f, 0x5f, 0x6b, 0x2e, + 0x5f, 0x5f, 0x6d, 0x29, 0x2c, 0x55, 0x74, 0x26, 0x26, 0x55, 0x74, 0x28, + 0x74, 0x2c, 0x6e, 0x29, 0x7d, 0x2c, 0x6b, 0x74, 0x2e, 0x5f, 0x5f, 0x72, + 0x3d, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x74, 0x29, + 0x7b, 0x77, 0x74, 0x26, 0x26, 0x77, 0x74, 0x28, 0x74, 0x29, 0x2c, 0x70, + 0x74, 0x3d, 0x30, 0x3b, 0x76, 0x61, 0x72, 0x20, 0x6e, 0x3d, 0x28, 0x64, + 0x74, 0x3d, 0x74, 0x2e, 0x5f, 0x5f, 0x63, 0x29, 0x2e, 0x5f, 0x5f, 0x48, + 0x3b, 0x6e, 0x26, 0x26, 0x28, 0x76, 0x74, 0x3d, 0x3d, 0x3d, 0x64, 0x74, + 0x3f, 0x28, 0x6e, 0x2e, 0x5f, 0x5f, 0x68, 0x3d, 0x5b, 0x5d, 0x2c, 0x64, + 0x74, 0x2e, 0x5f, 0x5f, 0x68, 0x3d, 0x5b, 0x5d, 0x2c, 0x6e, 0x2e, 0x5f, + 0x5f, 0x2e, 0x66, 0x6f, 0x72, 0x45, 0x61, 0x63, 0x68, 0x28, 0x28, 0x66, + 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x74, 0x29, 0x7b, 0x74, + 0x2e, 0x5f, 0x5f, 0x4e, 0x26, 0x26, 0x28, 0x74, 0x2e, 0x5f, 0x5f, 0x3d, + 0x74, 0x2e, 0x5f, 0x5f, 0x4e, 0x29, 0x2c, 0x74, 0x2e, 0x5f, 0x5f, 0x56, + 0x3d, 0x62, 0x74, 0x2c, 0x74, 0x2e, 0x5f, 0x5f, 0x4e, 0x3d, 0x74, 0x2e, + 0x69, 0x3d, 0x76, 0x6f, 0x69, 0x64, 0x20, 0x30, 0x7d, 0x29, 0x29, 0x29, + 0x3a, 0x28, 0x6e, 0x2e, 0x5f, 0x5f, 0x68, 0x2e, 0x66, 0x6f, 0x72, 0x45, + 0x61, 0x63, 0x68, 0x28, 0x71, 0x74, 0x29, 0x2c, 0x6e, 0x2e, 0x5f, 0x5f, + 0x68, 0x2e, 0x66, 0x6f, 0x72, 0x45, 0x61, 0x63, 0x68, 0x28, 0x42, 0x74, + 0x29, 0x2c, 0x6e, 0x2e, 0x5f, 0x5f, 0x68, 0x3d, 0x5b, 0x5d, 0x2c, 0x70, + 0x74, 0x3d, 0x30, 0x29, 0x29, 0x2c, 0x76, 0x74, 0x3d, 0x64, 0x74, 0x7d, + 0x2c, 0x6b, 0x74, 0x2e, 0x64, 0x69, 0x66, 0x66, 0x65, 0x64, 0x3d, 0x66, + 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x74, 0x29, 0x7b, 0x78, + 0x74, 0x26, 0x26, 0x78, 0x74, 0x28, 0x74, 0x29, 0x3b, 0x76, 0x61, 0x72, + 0x20, 0x6e, 0x3d, 0x74, 0x2e, 0x5f, 0x5f, 0x63, 0x3b, 0x6e, 0x26, 0x26, + 0x6e, 0x2e, 0x5f, 0x5f, 0x48, 0x26, 0x26, 0x28, 0x6e, 0x2e, 0x5f, 0x5f, + 0x48, 0x2e, 0x5f, 0x5f, 0x68, 0x2e, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, + 0x26, 0x26, 0x28, 0x31, 0x21, 0x3d, 0x3d, 0x67, 0x74, 0x2e, 0x70, 0x75, + 0x73, 0x68, 0x28, 0x6e, 0x29, 0x26, 0x26, 0x79, 0x74, 0x3d, 0x3d, 0x3d, + 0x6b, 0x74, 0x2e, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x41, 0x6e, + 0x69, 0x6d, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x46, 0x72, 0x61, 0x6d, 0x65, + 0x7c, 0x7c, 0x28, 0x28, 0x79, 0x74, 0x3d, 0x6b, 0x74, 0x2e, 0x72, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x41, 0x6e, 0x69, 0x6d, 0x61, 0x74, 0x69, + 0x6f, 0x6e, 0x46, 0x72, 0x61, 0x6d, 0x65, 0x29, 0x7c, 0x7c, 0x49, 0x74, + 0x29, 0x28, 0x52, 0x74, 0x29, 0x29, 0x2c, 0x6e, 0x2e, 0x5f, 0x5f, 0x48, 0x2e, 0x5f, 0x5f, 0x2e, 0x66, 0x6f, 0x72, 0x45, 0x61, 0x63, 0x68, 0x28, 0x28, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x74, 0x29, - 0x7b, 0x74, 0x72, 0x79, 0x7b, 0x52, 0x74, 0x28, 0x74, 0x29, 0x7d, 0x63, - 0x61, 0x74, 0x63, 0x68, 0x28, 0x74, 0x29, 0x7b, 0x6e, 0x3d, 0x74, 0x7d, - 0x7d, 0x29, 0x29, 0x2c, 0x65, 0x2e, 0x5f, 0x5f, 0x48, 0x3d, 0x76, 0x6f, - 0x69, 0x64, 0x20, 0x30, 0x2c, 0x6e, 0x26, 0x26, 0x43, 0x2e, 0x5f, 0x5f, + 0x7b, 0x74, 0x2e, 0x69, 0x26, 0x26, 0x28, 0x74, 0x2e, 0x5f, 0x5f, 0x48, + 0x3d, 0x74, 0x2e, 0x69, 0x29, 0x2c, 0x74, 0x2e, 0x5f, 0x5f, 0x56, 0x21, + 0x3d, 0x3d, 0x62, 0x74, 0x26, 0x26, 0x28, 0x74, 0x2e, 0x5f, 0x5f, 0x3d, + 0x74, 0x2e, 0x5f, 0x5f, 0x56, 0x29, 0x2c, 0x74, 0x2e, 0x69, 0x3d, 0x76, + 0x6f, 0x69, 0x64, 0x20, 0x30, 0x2c, 0x74, 0x2e, 0x5f, 0x5f, 0x56, 0x3d, + 0x62, 0x74, 0x7d, 0x29, 0x29, 0x29, 0x2c, 0x76, 0x74, 0x3d, 0x64, 0x74, + 0x3d, 0x6e, 0x75, 0x6c, 0x6c, 0x7d, 0x2c, 0x6b, 0x74, 0x2e, 0x5f, 0x5f, + 0x63, 0x3d, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x74, + 0x2c, 0x6e, 0x29, 0x7b, 0x6e, 0x2e, 0x73, 0x6f, 0x6d, 0x65, 0x28, 0x28, + 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x74, 0x29, 0x7b, + 0x74, 0x72, 0x79, 0x7b, 0x74, 0x2e, 0x5f, 0x5f, 0x68, 0x2e, 0x66, 0x6f, + 0x72, 0x45, 0x61, 0x63, 0x68, 0x28, 0x71, 0x74, 0x29, 0x2c, 0x74, 0x2e, + 0x5f, 0x5f, 0x68, 0x3d, 0x74, 0x2e, 0x5f, 0x5f, 0x68, 0x2e, 0x66, 0x69, + 0x6c, 0x74, 0x65, 0x72, 0x28, 0x28, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, + 0x6f, 0x6e, 0x28, 0x74, 0x29, 0x7b, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, + 0x21, 0x74, 0x2e, 0x5f, 0x5f, 0x7c, 0x7c, 0x42, 0x74, 0x28, 0x74, 0x29, + 0x7d, 0x29, 0x29, 0x7d, 0x63, 0x61, 0x74, 0x63, 0x68, 0x28, 0x75, 0x29, + 0x7b, 0x6e, 0x2e, 0x73, 0x6f, 0x6d, 0x65, 0x28, 0x28, 0x66, 0x75, 0x6e, + 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x74, 0x29, 0x7b, 0x74, 0x2e, 0x5f, + 0x5f, 0x68, 0x26, 0x26, 0x28, 0x74, 0x2e, 0x5f, 0x5f, 0x68, 0x3d, 0x5b, + 0x5d, 0x29, 0x7d, 0x29, 0x29, 0x2c, 0x6e, 0x3d, 0x5b, 0x5d, 0x2c, 0x6b, + 0x74, 0x2e, 0x5f, 0x5f, 0x65, 0x28, 0x75, 0x2c, 0x74, 0x2e, 0x5f, 0x5f, + 0x76, 0x29, 0x7d, 0x7d, 0x29, 0x29, 0x2c, 0x43, 0x74, 0x26, 0x26, 0x43, + 0x74, 0x28, 0x74, 0x2c, 0x6e, 0x29, 0x7d, 0x2c, 0x6b, 0x74, 0x2e, 0x75, + 0x6e, 0x6d, 0x6f, 0x75, 0x6e, 0x74, 0x3d, 0x66, 0x75, 0x6e, 0x63, 0x74, + 0x69, 0x6f, 0x6e, 0x28, 0x74, 0x29, 0x7b, 0x45, 0x74, 0x26, 0x26, 0x45, + 0x74, 0x28, 0x74, 0x29, 0x3b, 0x76, 0x61, 0x72, 0x20, 0x6e, 0x2c, 0x65, + 0x3d, 0x74, 0x2e, 0x5f, 0x5f, 0x63, 0x3b, 0x65, 0x26, 0x26, 0x65, 0x2e, + 0x5f, 0x5f, 0x48, 0x26, 0x26, 0x28, 0x65, 0x2e, 0x5f, 0x5f, 0x48, 0x2e, + 0x5f, 0x5f, 0x2e, 0x66, 0x6f, 0x72, 0x45, 0x61, 0x63, 0x68, 0x28, 0x28, + 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x74, 0x29, 0x7b, + 0x74, 0x72, 0x79, 0x7b, 0x71, 0x74, 0x28, 0x74, 0x29, 0x7d, 0x63, 0x61, + 0x74, 0x63, 0x68, 0x28, 0x74, 0x29, 0x7b, 0x6e, 0x3d, 0x74, 0x7d, 0x7d, + 0x29, 0x29, 0x2c, 0x65, 0x2e, 0x5f, 0x5f, 0x48, 0x3d, 0x76, 0x6f, 0x69, + 0x64, 0x20, 0x30, 0x2c, 0x6e, 0x26, 0x26, 0x6b, 0x74, 0x2e, 0x5f, 0x5f, 0x65, 0x28, 0x6e, 0x2c, 0x65, 0x2e, 0x5f, 0x5f, 0x76, 0x29, 0x29, 0x7d, - 0x3b, 0x76, 0x61, 0x72, 0x20, 0x4c, 0x74, 0x3d, 0x22, 0x66, 0x75, 0x6e, + 0x3b, 0x76, 0x61, 0x72, 0x20, 0x6a, 0x74, 0x3d, 0x22, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x3d, 0x3d, 0x74, 0x79, 0x70, 0x65, 0x6f, 0x66, 0x20, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x41, 0x6e, 0x69, 0x6d, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x46, 0x72, 0x61, 0x6d, 0x65, - 0x3b, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x4f, 0x74, + 0x3b, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x49, 0x74, 0x28, 0x74, 0x29, 0x7b, 0x76, 0x61, 0x72, 0x20, 0x6e, 0x2c, 0x65, 0x3d, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x29, 0x7b, 0x63, 0x6c, 0x65, 0x61, 0x72, 0x54, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x28, - 0x5f, 0x29, 0x2c, 0x4c, 0x74, 0x26, 0x26, 0x63, 0x61, 0x6e, 0x63, 0x65, + 0x5f, 0x29, 0x2c, 0x6a, 0x74, 0x26, 0x26, 0x63, 0x61, 0x6e, 0x63, 0x65, 0x6c, 0x41, 0x6e, 0x69, 0x6d, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x46, 0x72, 0x61, 0x6d, 0x65, 0x28, 0x6e, 0x29, 0x2c, 0x73, 0x65, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x28, 0x74, 0x29, 0x7d, 0x2c, 0x5f, 0x3d, 0x73, 0x65, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x28, 0x65, - 0x2c, 0x31, 0x30, 0x30, 0x29, 0x3b, 0x4c, 0x74, 0x26, 0x26, 0x28, 0x6e, + 0x2c, 0x31, 0x30, 0x30, 0x29, 0x3b, 0x6a, 0x74, 0x26, 0x26, 0x28, 0x6e, 0x3d, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x41, 0x6e, 0x69, 0x6d, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x46, 0x72, 0x61, 0x6d, 0x65, 0x28, 0x65, 0x29, 0x29, 0x7d, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, - 0x52, 0x74, 0x28, 0x74, 0x29, 0x7b, 0x76, 0x61, 0x72, 0x20, 0x6e, 0x3d, - 0x70, 0x74, 0x2c, 0x65, 0x3d, 0x74, 0x2e, 0x5f, 0x5f, 0x63, 0x3b, 0x22, + 0x71, 0x74, 0x28, 0x74, 0x29, 0x7b, 0x76, 0x61, 0x72, 0x20, 0x6e, 0x3d, + 0x64, 0x74, 0x2c, 0x65, 0x3d, 0x74, 0x2e, 0x5f, 0x5f, 0x63, 0x3b, 0x22, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x3d, 0x3d, 0x74, 0x79, 0x70, 0x65, 0x6f, 0x66, 0x20, 0x65, 0x26, 0x26, 0x28, 0x74, 0x2e, 0x5f, 0x5f, 0x63, 0x3d, 0x76, 0x6f, 0x69, 0x64, 0x20, 0x30, 0x2c, 0x65, - 0x28, 0x29, 0x29, 0x2c, 0x70, 0x74, 0x3d, 0x6e, 0x7d, 0x66, 0x75, 0x6e, - 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x6a, 0x74, 0x28, 0x74, 0x29, 0x7b, - 0x76, 0x61, 0x72, 0x20, 0x6e, 0x3d, 0x70, 0x74, 0x3b, 0x74, 0x2e, 0x5f, - 0x5f, 0x63, 0x3d, 0x74, 0x2e, 0x5f, 0x5f, 0x28, 0x29, 0x2c, 0x70, 0x74, + 0x28, 0x29, 0x29, 0x2c, 0x64, 0x74, 0x3d, 0x6e, 0x7d, 0x66, 0x75, 0x6e, + 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x42, 0x74, 0x28, 0x74, 0x29, 0x7b, + 0x76, 0x61, 0x72, 0x20, 0x6e, 0x3d, 0x64, 0x74, 0x3b, 0x74, 0x2e, 0x5f, + 0x5f, 0x63, 0x3d, 0x74, 0x2e, 0x5f, 0x5f, 0x28, 0x29, 0x2c, 0x64, 0x74, 0x3d, 0x6e, 0x7d, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, - 0x49, 0x74, 0x28, 0x74, 0x2c, 0x6e, 0x29, 0x7b, 0x72, 0x65, 0x74, 0x75, + 0x47, 0x74, 0x28, 0x74, 0x2c, 0x6e, 0x29, 0x7b, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x21, 0x74, 0x7c, 0x7c, 0x74, 0x2e, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x21, 0x3d, 0x3d, 0x6e, 0x2e, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x7c, 0x7c, 0x6e, 0x2e, 0x73, 0x6f, 0x6d, 0x65, 0x28, 0x28, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x6e, 0x2c, 0x65, 0x29, 0x7b, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x6e, 0x21, 0x3d, 0x3d, 0x74, 0x5b, 0x65, 0x5d, 0x7d, 0x29, 0x29, 0x7d, 0x66, 0x75, 0x6e, 0x63, - 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x71, 0x74, 0x28, 0x74, 0x2c, 0x6e, 0x29, + 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x7a, 0x74, 0x28, 0x74, 0x2c, 0x6e, 0x29, 0x7b, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x22, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x3d, 0x3d, 0x74, 0x79, 0x70, 0x65, 0x6f, 0x66, 0x20, 0x6e, 0x3f, 0x6e, 0x28, 0x74, 0x29, 0x3a, 0x6e, 0x7d, 0x66, - 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x42, 0x74, 0x28, 0x74, + 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x4a, 0x74, 0x28, 0x74, 0x2c, 0x6e, 0x29, 0x7b, 0x43, 0x5b, 0x74, 0x5d, 0x3d, 0x6e, 0x2e, 0x62, 0x69, 0x6e, 0x64, 0x28, 0x6e, 0x75, 0x6c, 0x6c, 0x2c, 0x43, 0x5b, 0x74, 0x5d, 0x7c, 0x7c, 0x28, 0x28, 0x29, 0x3d, 0x3e, 0x7b, 0x7d, 0x29, 0x29, - 0x7d, 0x6c, 0x65, 0x74, 0x20, 0x47, 0x74, 0x2c, 0x7a, 0x74, 0x3b, 0x66, - 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x4a, 0x74, 0x28, 0x74, - 0x29, 0x7b, 0x69, 0x66, 0x28, 0x7a, 0x74, 0x29, 0x7a, 0x74, 0x28, 0x29, - 0x3b, 0x7a, 0x74, 0x3d, 0x74, 0x26, 0x26, 0x74, 0x2e, 0x53, 0x28, 0x29, - 0x7d, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x4b, 0x74, + 0x7d, 0x6c, 0x65, 0x74, 0x20, 0x4b, 0x74, 0x2c, 0x51, 0x74, 0x3b, 0x66, + 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x58, 0x74, 0x28, 0x74, + 0x29, 0x7b, 0x69, 0x66, 0x28, 0x51, 0x74, 0x29, 0x51, 0x74, 0x28, 0x29, + 0x3b, 0x51, 0x74, 0x3d, 0x74, 0x26, 0x26, 0x74, 0x2e, 0x53, 0x28, 0x29, + 0x7d, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x59, 0x74, 0x28, 0x7b, 0x64, 0x61, 0x74, 0x61, 0x3a, 0x74, 0x7d, 0x29, 0x7b, 0x63, - 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x6e, 0x3d, 0x58, 0x74, 0x28, 0x74, 0x29, + 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x6e, 0x3d, 0x74, 0x6e, 0x28, 0x74, 0x29, 0x3b, 0x6e, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3d, 0x74, 0x3b, 0x63, - 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x65, 0x3d, 0x44, 0x74, 0x28, 0x28, 0x29, + 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x65, 0x3d, 0x41, 0x74, 0x28, 0x28, 0x29, 0x3d, 0x3e, 0x7b, 0x6c, 0x65, 0x74, 0x20, 0x74, 0x3d, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x5f, 0x5f, 0x76, 0x3b, 0x77, 0x68, 0x69, 0x6c, 0x65, 0x28, 0x74, 0x3d, 0x74, 0x2e, 0x5f, 0x5f, 0x29, 0x69, 0x66, 0x28, 0x74, 0x2e, @@ -1592,7 +1620,7 @@ unsigned char index_js[] = { 0x74, 0x3f, 0x30, 0x3a, 0x21, 0x30, 0x3d, 0x3d, 0x3d, 0x74, 0x3f, 0x22, 0x22, 0x3a, 0x74, 0x7c, 0x7c, 0x22, 0x22, 0x7d, 0x29, 0x7d, 0x2c, 0x5b, 0x5d, 0x29, 0x3b, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x65, 0x2e, - 0x76, 0x61, 0x6c, 0x75, 0x65, 0x7d, 0x4b, 0x74, 0x2e, 0x64, 0x69, 0x73, + 0x76, 0x61, 0x6c, 0x75, 0x65, 0x7d, 0x59, 0x74, 0x2e, 0x64, 0x69, 0x73, 0x70, 0x6c, 0x61, 0x79, 0x4e, 0x61, 0x6d, 0x65, 0x3d, 0x22, 0x5f, 0x73, 0x74, 0x22, 0x3b, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x2e, 0x64, 0x65, 0x66, 0x69, 0x6e, 0x65, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, @@ -1603,14 +1631,14 @@ unsigned char index_js[] = { 0x75, 0x65, 0x3a, 0x76, 0x6f, 0x69, 0x64, 0x20, 0x30, 0x7d, 0x2c, 0x74, 0x79, 0x70, 0x65, 0x3a, 0x7b, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x61, 0x62, 0x6c, 0x65, 0x3a, 0x21, 0x30, 0x2c, 0x76, 0x61, 0x6c, - 0x75, 0x65, 0x3a, 0x4b, 0x74, 0x7d, 0x2c, 0x70, 0x72, 0x6f, 0x70, 0x73, + 0x75, 0x65, 0x3a, 0x59, 0x74, 0x7d, 0x2c, 0x70, 0x72, 0x6f, 0x70, 0x73, 0x3a, 0x7b, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x61, 0x62, 0x6c, 0x65, 0x3a, 0x21, 0x30, 0x2c, 0x67, 0x65, 0x74, 0x28, 0x29, 0x7b, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x7b, 0x64, 0x61, 0x74, 0x61, 0x3a, 0x74, 0x68, 0x69, 0x73, 0x7d, 0x7d, 0x7d, 0x2c, 0x5f, 0x5f, 0x62, 0x3a, 0x7b, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x61, 0x62, 0x6c, 0x65, 0x3a, 0x21, 0x30, 0x2c, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x31, - 0x7d, 0x7d, 0x29, 0x3b, 0x42, 0x74, 0x28, 0x22, 0x5f, 0x5f, 0x62, 0x22, + 0x7d, 0x7d, 0x29, 0x3b, 0x4a, 0x74, 0x28, 0x22, 0x5f, 0x5f, 0x62, 0x22, 0x2c, 0x28, 0x74, 0x2c, 0x6e, 0x29, 0x3d, 0x3e, 0x7b, 0x69, 0x66, 0x28, 0x22, 0x73, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x22, 0x3d, 0x3d, 0x74, 0x79, 0x70, 0x65, 0x6f, 0x66, 0x20, 0x6e, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x29, @@ -1625,8 +1653,8 @@ unsigned char index_js[] = { 0x6e, 0x2e, 0x5f, 0x5f, 0x6e, 0x70, 0x3d, 0x74, 0x3d, 0x7b, 0x7d, 0x3b, 0x74, 0x5b, 0x5f, 0x5d, 0x3d, 0x69, 0x3b, 0x65, 0x5b, 0x5f, 0x5d, 0x3d, 0x69, 0x2e, 0x70, 0x65, 0x65, 0x6b, 0x28, 0x29, 0x7d, 0x7d, 0x7d, 0x74, - 0x28, 0x6e, 0x29, 0x7d, 0x29, 0x3b, 0x42, 0x74, 0x28, 0x22, 0x5f, 0x5f, - 0x72, 0x22, 0x2c, 0x28, 0x74, 0x2c, 0x6e, 0x29, 0x3d, 0x3e, 0x7b, 0x4a, + 0x28, 0x6e, 0x29, 0x7d, 0x29, 0x3b, 0x4a, 0x74, 0x28, 0x22, 0x5f, 0x5f, + 0x72, 0x22, 0x2c, 0x28, 0x74, 0x2c, 0x6e, 0x29, 0x3d, 0x3e, 0x7b, 0x58, 0x74, 0x28, 0x29, 0x3b, 0x6c, 0x65, 0x74, 0x20, 0x65, 0x2c, 0x5f, 0x3d, 0x6e, 0x2e, 0x5f, 0x5f, 0x63, 0x3b, 0x69, 0x66, 0x28, 0x5f, 0x29, 0x7b, 0x5f, 0x2e, 0x5f, 0x5f, 0x24, 0x66, 0x26, 0x3d, 0x2d, 0x32, 0x3b, 0x65, @@ -1640,14 +1668,14 @@ unsigned char index_js[] = { 0x5f, 0x5f, 0x24, 0x66, 0x7c, 0x3d, 0x31, 0x3b, 0x5f, 0x2e, 0x73, 0x65, 0x74, 0x53, 0x74, 0x61, 0x74, 0x65, 0x28, 0x7b, 0x7d, 0x29, 0x7d, 0x3b, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x6e, 0x7d, 0x28, 0x29, 0x7d, - 0x47, 0x74, 0x3d, 0x5f, 0x3b, 0x4a, 0x74, 0x28, 0x65, 0x29, 0x3b, 0x74, - 0x28, 0x6e, 0x29, 0x7d, 0x29, 0x3b, 0x42, 0x74, 0x28, 0x22, 0x5f, 0x5f, + 0x4b, 0x74, 0x3d, 0x5f, 0x3b, 0x58, 0x74, 0x28, 0x65, 0x29, 0x3b, 0x74, + 0x28, 0x6e, 0x29, 0x7d, 0x29, 0x3b, 0x4a, 0x74, 0x28, 0x22, 0x5f, 0x5f, 0x65, 0x22, 0x2c, 0x28, 0x74, 0x2c, 0x6e, 0x2c, 0x65, 0x2c, 0x5f, 0x29, - 0x3d, 0x3e, 0x7b, 0x4a, 0x74, 0x28, 0x29, 0x3b, 0x47, 0x74, 0x3d, 0x76, + 0x3d, 0x3e, 0x7b, 0x58, 0x74, 0x28, 0x29, 0x3b, 0x4b, 0x74, 0x3d, 0x76, 0x6f, 0x69, 0x64, 0x20, 0x30, 0x3b, 0x74, 0x28, 0x6e, 0x2c, 0x65, 0x2c, - 0x5f, 0x29, 0x7d, 0x29, 0x3b, 0x42, 0x74, 0x28, 0x22, 0x64, 0x69, 0x66, + 0x5f, 0x29, 0x7d, 0x29, 0x3b, 0x4a, 0x74, 0x28, 0x22, 0x64, 0x69, 0x66, 0x66, 0x65, 0x64, 0x22, 0x2c, 0x28, 0x74, 0x2c, 0x6e, 0x29, 0x3d, 0x3e, - 0x7b, 0x4a, 0x74, 0x28, 0x29, 0x3b, 0x47, 0x74, 0x3d, 0x76, 0x6f, 0x69, + 0x7b, 0x58, 0x74, 0x28, 0x29, 0x3b, 0x4b, 0x74, 0x3d, 0x76, 0x6f, 0x69, 0x64, 0x20, 0x30, 0x3b, 0x6c, 0x65, 0x74, 0x20, 0x65, 0x3b, 0x69, 0x66, 0x28, 0x22, 0x73, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x22, 0x3d, 0x3d, 0x74, 0x79, 0x70, 0x65, 0x6f, 0x66, 0x20, 0x6e, 0x2e, 0x74, 0x79, 0x70, 0x65, @@ -1667,11 +1695,11 @@ unsigned char index_js[] = { 0x6e, 0x20, 0x74, 0x29, 0x7b, 0x6c, 0x65, 0x74, 0x20, 0x6f, 0x3d, 0x6e, 0x5b, 0x69, 0x5d, 0x2c, 0x72, 0x3d, 0x74, 0x5b, 0x69, 0x5d, 0x3b, 0x69, 0x66, 0x28, 0x76, 0x6f, 0x69, 0x64, 0x20, 0x30, 0x3d, 0x3d, 0x3d, 0x6f, - 0x29, 0x7b, 0x6f, 0x3d, 0x51, 0x74, 0x28, 0x65, 0x2c, 0x69, 0x2c, 0x72, + 0x29, 0x7b, 0x6f, 0x3d, 0x5a, 0x74, 0x28, 0x65, 0x2c, 0x69, 0x2c, 0x72, 0x2c, 0x5f, 0x29, 0x3b, 0x6e, 0x5b, 0x69, 0x5d, 0x3d, 0x6f, 0x7d, 0x65, 0x6c, 0x73, 0x65, 0x20, 0x6f, 0x2e, 0x6f, 0x28, 0x72, 0x2c, 0x5f, 0x29, 0x7d, 0x7d, 0x7d, 0x74, 0x28, 0x6e, 0x29, 0x7d, 0x29, 0x3b, 0x66, 0x75, - 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x51, 0x74, 0x28, 0x74, 0x2c, + 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x5a, 0x74, 0x28, 0x74, 0x2c, 0x6e, 0x2c, 0x65, 0x2c, 0x5f, 0x29, 0x7b, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x69, 0x3d, 0x6e, 0x20, 0x69, 0x6e, 0x20, 0x74, 0x26, 0x26, 0x76, 0x6f, 0x69, 0x64, 0x20, 0x30, 0x3d, 0x3d, 0x3d, 0x74, 0x2e, 0x6f, 0x77, @@ -1689,7 +1717,7 @@ unsigned char index_js[] = { 0x74, 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, 0x65, 0x28, 0x6e, 0x2c, 0x65, 0x29, 0x3b, 0x65, 0x6c, 0x73, 0x65, 0x20, 0x74, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x41, 0x74, 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, 0x65, - 0x28, 0x6e, 0x29, 0x7d, 0x7d, 0x29, 0x7d, 0x7d, 0x42, 0x74, 0x28, 0x22, + 0x28, 0x6e, 0x29, 0x7d, 0x7d, 0x29, 0x7d, 0x7d, 0x4a, 0x74, 0x28, 0x22, 0x75, 0x6e, 0x6d, 0x6f, 0x75, 0x6e, 0x74, 0x22, 0x2c, 0x28, 0x74, 0x2c, 0x6e, 0x29, 0x3d, 0x3e, 0x7b, 0x69, 0x66, 0x28, 0x22, 0x73, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x22, 0x3d, 0x3d, 0x74, 0x79, 0x70, 0x65, 0x6f, 0x66, @@ -1707,7 +1735,7 @@ unsigned char index_js[] = { 0x24, 0x75, 0x3b, 0x69, 0x66, 0x28, 0x6e, 0x29, 0x7b, 0x74, 0x2e, 0x5f, 0x5f, 0x24, 0x75, 0x3d, 0x76, 0x6f, 0x69, 0x64, 0x20, 0x30, 0x3b, 0x6e, 0x2e, 0x64, 0x28, 0x29, 0x7d, 0x7d, 0x7d, 0x74, 0x28, 0x6e, 0x29, 0x7d, - 0x29, 0x3b, 0x42, 0x74, 0x28, 0x22, 0x5f, 0x5f, 0x68, 0x22, 0x2c, 0x28, + 0x29, 0x3b, 0x4a, 0x74, 0x28, 0x22, 0x5f, 0x5f, 0x68, 0x22, 0x2c, 0x28, 0x74, 0x2c, 0x6e, 0x2c, 0x65, 0x2c, 0x5f, 0x29, 0x3d, 0x3e, 0x7b, 0x69, 0x66, 0x28, 0x5f, 0x3c, 0x33, 0x7c, 0x7c, 0x39, 0x3d, 0x3d, 0x3d, 0x5f, 0x29, 0x6e, 0x2e, 0x5f, 0x5f, 0x24, 0x66, 0x7c, 0x3d, 0x32, 0x3b, 0x74, @@ -1735,23 +1763,23 @@ unsigned char index_js[] = { 0x6f, 0x70, 0x73, 0x29, 0x69, 0x66, 0x28, 0x21, 0x28, 0x5f, 0x20, 0x69, 0x6e, 0x20, 0x74, 0x29, 0x29, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x21, 0x30, 0x3b, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x21, 0x31, 0x7d, 0x3b, - 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x58, 0x74, 0x28, - 0x74, 0x29, 0x7b, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x44, 0x74, + 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x74, 0x6e, 0x28, + 0x74, 0x29, 0x7b, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x41, 0x74, 0x28, 0x28, 0x29, 0x3d, 0x3e, 0x61, 0x28, 0x74, 0x29, 0x2c, 0x5b, 0x5d, - 0x29, 0x7d, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x59, - 0x74, 0x28, 0x74, 0x29, 0x7b, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x6e, - 0x3d, 0x4e, 0x74, 0x28, 0x74, 0x29, 0x3b, 0x6e, 0x2e, 0x63, 0x75, 0x72, - 0x72, 0x65, 0x6e, 0x74, 0x3d, 0x74, 0x3b, 0x47, 0x74, 0x2e, 0x5f, 0x5f, + 0x29, 0x7d, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x6e, + 0x6e, 0x28, 0x74, 0x29, 0x7b, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x6e, + 0x3d, 0x54, 0x74, 0x28, 0x74, 0x29, 0x3b, 0x6e, 0x2e, 0x63, 0x75, 0x72, + 0x72, 0x65, 0x6e, 0x74, 0x3d, 0x74, 0x3b, 0x4b, 0x74, 0x2e, 0x5f, 0x5f, 0x24, 0x66, 0x7c, 0x3d, 0x34, 0x3b, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, - 0x20, 0x44, 0x74, 0x28, 0x28, 0x29, 0x3d, 0x3e, 0x6d, 0x28, 0x28, 0x29, + 0x20, 0x41, 0x74, 0x28, 0x28, 0x29, 0x3d, 0x3e, 0x6d, 0x28, 0x28, 0x29, 0x3d, 0x3e, 0x6e, 0x2e, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x74, 0x28, 0x29, 0x29, 0x2c, 0x5b, 0x5d, 0x29, 0x7d, 0x66, 0x75, 0x6e, 0x63, 0x74, - 0x69, 0x6f, 0x6e, 0x20, 0x5a, 0x74, 0x28, 0x74, 0x29, 0x7b, 0x63, 0x6f, - 0x6e, 0x73, 0x74, 0x20, 0x6e, 0x3d, 0x4e, 0x74, 0x28, 0x74, 0x29, 0x3b, + 0x69, 0x6f, 0x6e, 0x20, 0x65, 0x6e, 0x28, 0x74, 0x29, 0x7b, 0x63, 0x6f, + 0x6e, 0x73, 0x74, 0x20, 0x6e, 0x3d, 0x54, 0x74, 0x28, 0x74, 0x29, 0x3b, 0x6e, 0x2e, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x74, 0x3d, 0x74, 0x3b, - 0x48, 0x74, 0x28, 0x28, 0x29, 0x3d, 0x3e, 0x77, 0x28, 0x28, 0x29, 0x3d, + 0x24, 0x74, 0x28, 0x28, 0x29, 0x3d, 0x3e, 0x77, 0x28, 0x28, 0x29, 0x3d, 0x3e, 0x6e, 0x2e, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x74, 0x28, 0x29, - 0x29, 0x2c, 0x5b, 0x5d, 0x29, 0x7d, 0x76, 0x61, 0x72, 0x20, 0x74, 0x6e, + 0x29, 0x2c, 0x5b, 0x5d, 0x29, 0x7d, 0x76, 0x61, 0x72, 0x20, 0x5f, 0x6e, 0x3d, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x74, 0x2c, 0x6e, 0x2c, 0x65, 0x2c, 0x5f, 0x29, 0x7b, 0x76, 0x61, 0x72, 0x20, 0x69, 0x3b, 0x6e, 0x5b, 0x30, 0x5d, 0x3d, 0x30, 0x3b, 0x66, 0x6f, 0x72, 0x28, @@ -1771,21 +1799,21 @@ unsigned char index_js[] = { 0x3a, 0x36, 0x3d, 0x3d, 0x3d, 0x72, 0x3f, 0x5f, 0x5b, 0x31, 0x5d, 0x5b, 0x6e, 0x5b, 0x2b, 0x2b, 0x6f, 0x5d, 0x5d, 0x2b, 0x3d, 0x75, 0x2b, 0x22, 0x22, 0x3a, 0x72, 0x3f, 0x28, 0x69, 0x3d, 0x74, 0x2e, 0x61, 0x70, 0x70, - 0x6c, 0x79, 0x28, 0x75, 0x2c, 0x74, 0x6e, 0x28, 0x74, 0x2c, 0x75, 0x2c, + 0x6c, 0x79, 0x28, 0x75, 0x2c, 0x5f, 0x6e, 0x28, 0x74, 0x2c, 0x75, 0x2c, 0x65, 0x2c, 0x5b, 0x22, 0x22, 0x2c, 0x6e, 0x75, 0x6c, 0x6c, 0x5d, 0x29, 0x29, 0x2c, 0x5f, 0x2e, 0x70, 0x75, 0x73, 0x68, 0x28, 0x69, 0x29, 0x2c, 0x75, 0x5b, 0x30, 0x5d, 0x3f, 0x6e, 0x5b, 0x30, 0x5d, 0x7c, 0x3d, 0x32, 0x3a, 0x28, 0x6e, 0x5b, 0x6f, 0x2d, 0x32, 0x5d, 0x3d, 0x30, 0x2c, 0x6e, 0x5b, 0x6f, 0x5d, 0x3d, 0x69, 0x29, 0x29, 0x3a, 0x5f, 0x2e, 0x70, 0x75, 0x73, 0x68, 0x28, 0x75, 0x29, 0x7d, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, - 0x20, 0x5f, 0x7d, 0x2c, 0x6e, 0x6e, 0x3d, 0x6e, 0x65, 0x77, 0x20, 0x4d, + 0x20, 0x5f, 0x7d, 0x2c, 0x6f, 0x6e, 0x3d, 0x6e, 0x65, 0x77, 0x20, 0x4d, 0x61, 0x70, 0x3b, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, - 0x65, 0x6e, 0x28, 0x74, 0x29, 0x7b, 0x76, 0x61, 0x72, 0x20, 0x6e, 0x3d, - 0x6e, 0x6e, 0x2e, 0x67, 0x65, 0x74, 0x28, 0x74, 0x68, 0x69, 0x73, 0x29, + 0x72, 0x6e, 0x28, 0x74, 0x29, 0x7b, 0x76, 0x61, 0x72, 0x20, 0x6e, 0x3d, + 0x6f, 0x6e, 0x2e, 0x67, 0x65, 0x74, 0x28, 0x74, 0x68, 0x69, 0x73, 0x29, 0x3b, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x6e, 0x7c, 0x7c, 0x28, - 0x6e, 0x3d, 0x6e, 0x65, 0x77, 0x20, 0x4d, 0x61, 0x70, 0x2c, 0x6e, 0x6e, + 0x6e, 0x3d, 0x6e, 0x65, 0x77, 0x20, 0x4d, 0x61, 0x70, 0x2c, 0x6f, 0x6e, 0x2e, 0x73, 0x65, 0x74, 0x28, 0x74, 0x68, 0x69, 0x73, 0x2c, 0x6e, 0x29, - 0x29, 0x2c, 0x28, 0x6e, 0x3d, 0x74, 0x6e, 0x28, 0x74, 0x68, 0x69, 0x73, + 0x29, 0x2c, 0x28, 0x6e, 0x3d, 0x5f, 0x6e, 0x28, 0x74, 0x68, 0x69, 0x73, 0x2c, 0x6e, 0x2e, 0x67, 0x65, 0x74, 0x28, 0x74, 0x29, 0x7c, 0x7c, 0x28, 0x6e, 0x2e, 0x73, 0x65, 0x74, 0x28, 0x74, 0x2c, 0x6e, 0x3d, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x74, 0x29, 0x7b, 0x66, 0x6f, @@ -1816,10 +1844,10 @@ unsigned char index_js[] = { 0x30, 0x3b, 0x66, 0x3c, 0x74, 0x2e, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x3b, 0x66, 0x2b, 0x2b, 0x29, 0x7b, 0x66, 0x26, 0x26, 0x28, 0x31, 0x3d, 0x3d, 0x3d, 0x5f, 0x26, 0x26, 0x75, 0x28, 0x29, 0x2c, 0x75, 0x28, 0x66, - 0x29, 0x29, 0x3b, 0x66, 0x6f, 0x72, 0x28, 0x76, 0x61, 0x72, 0x20, 0x73, - 0x3d, 0x30, 0x3b, 0x73, 0x3c, 0x74, 0x5b, 0x66, 0x5d, 0x2e, 0x6c, 0x65, - 0x6e, 0x67, 0x74, 0x68, 0x3b, 0x73, 0x2b, 0x2b, 0x29, 0x6e, 0x3d, 0x74, - 0x5b, 0x66, 0x5d, 0x5b, 0x73, 0x5d, 0x2c, 0x31, 0x3d, 0x3d, 0x3d, 0x5f, + 0x29, 0x29, 0x3b, 0x66, 0x6f, 0x72, 0x28, 0x76, 0x61, 0x72, 0x20, 0x6c, + 0x3d, 0x30, 0x3b, 0x6c, 0x3c, 0x74, 0x5b, 0x66, 0x5d, 0x2e, 0x6c, 0x65, + 0x6e, 0x67, 0x74, 0x68, 0x3b, 0x6c, 0x2b, 0x2b, 0x29, 0x6e, 0x3d, 0x74, + 0x5b, 0x66, 0x5d, 0x5b, 0x6c, 0x5d, 0x2c, 0x31, 0x3d, 0x3d, 0x3d, 0x5f, 0x3f, 0x22, 0x3c, 0x22, 0x3d, 0x3d, 0x3d, 0x6e, 0x3f, 0x28, 0x75, 0x28, 0x29, 0x2c, 0x72, 0x3d, 0x5b, 0x72, 0x5d, 0x2c, 0x5f, 0x3d, 0x33, 0x29, 0x3a, 0x69, 0x2b, 0x3d, 0x6e, 0x3a, 0x34, 0x3d, 0x3d, 0x3d, 0x5f, 0x3f, @@ -1835,7 +1863,7 @@ unsigned char index_js[] = { 0x5f, 0x3d, 0x35, 0x2c, 0x65, 0x3d, 0x69, 0x2c, 0x69, 0x3d, 0x22, 0x22, 0x29, 0x3a, 0x22, 0x2f, 0x22, 0x3d, 0x3d, 0x3d, 0x6e, 0x26, 0x26, 0x28, 0x5f, 0x3c, 0x35, 0x7c, 0x7c, 0x22, 0x3e, 0x22, 0x3d, 0x3d, 0x3d, 0x74, - 0x5b, 0x66, 0x5d, 0x5b, 0x73, 0x2b, 0x31, 0x5d, 0x29, 0x3f, 0x28, 0x75, + 0x5b, 0x66, 0x5d, 0x5b, 0x6c, 0x2b, 0x31, 0x5d, 0x29, 0x3f, 0x28, 0x75, 0x28, 0x29, 0x2c, 0x33, 0x3d, 0x3d, 0x3d, 0x5f, 0x26, 0x26, 0x28, 0x72, 0x3d, 0x72, 0x5b, 0x30, 0x5d, 0x29, 0x2c, 0x5f, 0x3d, 0x72, 0x2c, 0x28, 0x72, 0x3d, 0x72, 0x5b, 0x30, 0x5d, 0x29, 0x2e, 0x70, 0x75, 0x73, 0x68, @@ -1851,53 +1879,53 @@ unsigned char index_js[] = { 0x2c, 0x72, 0x7d, 0x28, 0x74, 0x29, 0x29, 0x2c, 0x6e, 0x29, 0x2c, 0x61, 0x72, 0x67, 0x75, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x2c, 0x5b, 0x5d, 0x29, 0x29, 0x2e, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x3e, 0x31, 0x3f, 0x6e, - 0x3a, 0x6e, 0x5b, 0x30, 0x5d, 0x7d, 0x76, 0x61, 0x72, 0x20, 0x5f, 0x6e, - 0x3d, 0x65, 0x6e, 0x2e, 0x62, 0x69, 0x6e, 0x64, 0x28, 0x4c, 0x29, 0x3b, + 0x3a, 0x6e, 0x5b, 0x30, 0x5d, 0x7d, 0x76, 0x61, 0x72, 0x20, 0x75, 0x6e, + 0x3d, 0x72, 0x6e, 0x2e, 0x62, 0x69, 0x6e, 0x64, 0x28, 0x4c, 0x29, 0x3b, 0x65, 0x78, 0x70, 0x6f, 0x72, 0x74, 0x7b, 0x49, 0x20, 0x61, 0x73, 0x20, 0x43, 0x6f, 0x6d, 0x70, 0x6f, 0x6e, 0x65, 0x6e, 0x74, 0x2c, 0x6a, 0x20, 0x61, 0x73, 0x20, 0x46, 0x72, 0x61, 0x67, 0x6d, 0x65, 0x6e, 0x74, 0x2c, 0x68, 0x20, 0x61, 0x73, 0x20, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x2c, - 0x5f, 0x20, 0x61, 0x73, 0x20, 0x62, 0x61, 0x74, 0x63, 0x68, 0x2c, 0x63, + 0x5f, 0x20, 0x61, 0x73, 0x20, 0x62, 0x61, 0x74, 0x63, 0x68, 0x2c, 0x68, 0x74, 0x20, 0x61, 0x73, 0x20, 0x63, 0x6c, 0x6f, 0x6e, 0x65, 0x45, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2c, 0x6d, 0x20, 0x61, 0x73, 0x20, 0x63, - 0x6f, 0x6d, 0x70, 0x75, 0x74, 0x65, 0x64, 0x2c, 0x68, 0x74, 0x20, 0x61, + 0x6f, 0x6d, 0x70, 0x75, 0x74, 0x65, 0x64, 0x2c, 0x61, 0x74, 0x20, 0x61, 0x73, 0x20, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x2c, 0x4c, 0x20, 0x61, 0x73, 0x20, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x45, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2c, 0x52, 0x20, 0x61, 0x73, 0x20, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x52, 0x65, 0x66, 0x2c, 0x77, 0x20, 0x61, 0x73, 0x20, 0x65, 0x66, 0x66, 0x65, 0x63, - 0x74, 0x2c, 0x4c, 0x20, 0x61, 0x73, 0x20, 0x68, 0x2c, 0x5f, 0x6e, 0x20, - 0x61, 0x73, 0x20, 0x68, 0x74, 0x6d, 0x6c, 0x2c, 0x6c, 0x74, 0x20, 0x61, + 0x74, 0x2c, 0x4c, 0x20, 0x61, 0x73, 0x20, 0x68, 0x2c, 0x75, 0x6e, 0x20, + 0x61, 0x73, 0x20, 0x68, 0x74, 0x6d, 0x6c, 0x2c, 0x63, 0x74, 0x20, 0x61, 0x73, 0x20, 0x68, 0x79, 0x64, 0x72, 0x61, 0x74, 0x65, 0x2c, 0x55, 0x20, 0x61, 0x73, 0x20, 0x69, 0x73, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x45, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2c, 0x43, 0x20, 0x61, 0x73, 0x20, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x2c, 0x73, 0x74, 0x20, 0x61, 0x73, 0x20, 0x72, 0x65, 0x6e, 0x64, 0x65, 0x72, 0x2c, 0x61, 0x20, 0x61, 0x73, - 0x20, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x2c, 0x58, 0x20, 0x61, 0x73, + 0x20, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x2c, 0x59, 0x20, 0x61, 0x73, 0x20, 0x74, 0x6f, 0x43, 0x68, 0x69, 0x6c, 0x64, 0x41, 0x72, 0x72, 0x61, 0x79, 0x2c, 0x75, 0x20, 0x61, 0x73, 0x20, 0x75, 0x6e, 0x74, 0x72, 0x61, - 0x63, 0x6b, 0x65, 0x64, 0x2c, 0x54, 0x74, 0x20, 0x61, 0x73, 0x20, 0x75, - 0x73, 0x65, 0x43, 0x61, 0x6c, 0x6c, 0x62, 0x61, 0x63, 0x6b, 0x2c, 0x59, - 0x74, 0x20, 0x61, 0x73, 0x20, 0x75, 0x73, 0x65, 0x43, 0x6f, 0x6d, 0x70, - 0x75, 0x74, 0x65, 0x64, 0x2c, 0x56, 0x74, 0x20, 0x61, 0x73, 0x20, 0x75, - 0x73, 0x65, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x2c, 0x41, 0x74, + 0x63, 0x6b, 0x65, 0x64, 0x2c, 0x46, 0x74, 0x20, 0x61, 0x73, 0x20, 0x75, + 0x73, 0x65, 0x43, 0x61, 0x6c, 0x6c, 0x62, 0x61, 0x63, 0x6b, 0x2c, 0x6e, + 0x6e, 0x20, 0x61, 0x73, 0x20, 0x75, 0x73, 0x65, 0x43, 0x6f, 0x6d, 0x70, + 0x75, 0x74, 0x65, 0x64, 0x2c, 0x4d, 0x74, 0x20, 0x61, 0x73, 0x20, 0x75, + 0x73, 0x65, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x2c, 0x57, 0x74, 0x20, 0x61, 0x73, 0x20, 0x75, 0x73, 0x65, 0x44, 0x65, 0x62, 0x75, 0x67, - 0x56, 0x61, 0x6c, 0x75, 0x65, 0x2c, 0x48, 0x74, 0x20, 0x61, 0x73, 0x20, - 0x75, 0x73, 0x65, 0x45, 0x66, 0x66, 0x65, 0x63, 0x74, 0x2c, 0x46, 0x74, + 0x56, 0x61, 0x6c, 0x75, 0x65, 0x2c, 0x24, 0x74, 0x20, 0x61, 0x73, 0x20, + 0x75, 0x73, 0x65, 0x45, 0x66, 0x66, 0x65, 0x63, 0x74, 0x2c, 0x4c, 0x74, 0x20, 0x61, 0x73, 0x20, 0x75, 0x73, 0x65, 0x45, 0x72, 0x72, 0x6f, 0x72, - 0x42, 0x6f, 0x75, 0x6e, 0x64, 0x61, 0x72, 0x79, 0x2c, 0x4d, 0x74, 0x20, - 0x61, 0x73, 0x20, 0x75, 0x73, 0x65, 0x49, 0x64, 0x2c, 0x24, 0x74, 0x20, + 0x42, 0x6f, 0x75, 0x6e, 0x64, 0x61, 0x72, 0x79, 0x2c, 0x4f, 0x74, 0x20, + 0x61, 0x73, 0x20, 0x75, 0x73, 0x65, 0x49, 0x64, 0x2c, 0x56, 0x74, 0x20, 0x61, 0x73, 0x20, 0x75, 0x73, 0x65, 0x49, 0x6d, 0x70, 0x65, 0x72, 0x61, - 0x74, 0x69, 0x76, 0x65, 0x48, 0x61, 0x6e, 0x64, 0x6c, 0x65, 0x2c, 0x50, + 0x74, 0x69, 0x76, 0x65, 0x48, 0x61, 0x6e, 0x64, 0x6c, 0x65, 0x2c, 0x44, 0x74, 0x20, 0x61, 0x73, 0x20, 0x75, 0x73, 0x65, 0x4c, 0x61, 0x79, 0x6f, - 0x75, 0x74, 0x45, 0x66, 0x66, 0x65, 0x63, 0x74, 0x2c, 0x44, 0x74, 0x20, - 0x61, 0x73, 0x20, 0x75, 0x73, 0x65, 0x4d, 0x65, 0x6d, 0x6f, 0x2c, 0x55, + 0x75, 0x74, 0x45, 0x66, 0x66, 0x65, 0x63, 0x74, 0x2c, 0x41, 0x74, 0x20, + 0x61, 0x73, 0x20, 0x75, 0x73, 0x65, 0x4d, 0x65, 0x6d, 0x6f, 0x2c, 0x4e, 0x74, 0x20, 0x61, 0x73, 0x20, 0x75, 0x73, 0x65, 0x52, 0x65, 0x64, 0x75, - 0x63, 0x65, 0x72, 0x2c, 0x4e, 0x74, 0x20, 0x61, 0x73, 0x20, 0x75, 0x73, - 0x65, 0x52, 0x65, 0x66, 0x2c, 0x58, 0x74, 0x20, 0x61, 0x73, 0x20, 0x75, - 0x73, 0x65, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x2c, 0x5a, 0x74, 0x20, + 0x63, 0x65, 0x72, 0x2c, 0x54, 0x74, 0x20, 0x61, 0x73, 0x20, 0x75, 0x73, + 0x65, 0x52, 0x65, 0x66, 0x2c, 0x74, 0x6e, 0x20, 0x61, 0x73, 0x20, 0x75, + 0x73, 0x65, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x2c, 0x65, 0x6e, 0x20, 0x61, 0x73, 0x20, 0x75, 0x73, 0x65, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x6c, - 0x45, 0x66, 0x66, 0x65, 0x63, 0x74, 0x2c, 0x45, 0x74, 0x20, 0x61, 0x73, + 0x45, 0x66, 0x66, 0x65, 0x63, 0x74, 0x2c, 0x50, 0x74, 0x20, 0x61, 0x73, 0x20, 0x75, 0x73, 0x65, 0x53, 0x74, 0x61, 0x74, 0x65, 0x7d, 0x3b, 0x0a }; -unsigned int index_js_len = 22800; +size_t index_js_len = 23136; diff --git a/examples/server/json-schema-to-grammar.mjs.hpp b/examples/server/json-schema-to-grammar.mjs.hpp index 0a05c369d..72fb6b80f 100644 --- a/examples/server/json-schema-to-grammar.mjs.hpp +++ b/examples/server/json-schema-to-grammar.mjs.hpp @@ -1,311 +1,1667 @@ unsigned char json_schema_to_grammar_mjs[] = { - 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x53, 0x50, 0x41, 0x43, 0x45, 0x5f, - 0x52, 0x55, 0x4c, 0x45, 0x20, 0x3d, 0x20, 0x27, 0x22, 0x20, 0x22, 0x3f, - 0x27, 0x3b, 0x0a, 0x0a, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x50, 0x52, - 0x49, 0x4d, 0x49, 0x54, 0x49, 0x56, 0x45, 0x5f, 0x52, 0x55, 0x4c, 0x45, - 0x53, 0x20, 0x3d, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x62, 0x6f, 0x6f, 0x6c, - 0x65, 0x61, 0x6e, 0x3a, 0x20, 0x27, 0x28, 0x22, 0x74, 0x72, 0x75, 0x65, - 0x22, 0x20, 0x7c, 0x20, 0x22, 0x66, 0x61, 0x6c, 0x73, 0x65, 0x22, 0x29, - 0x20, 0x73, 0x70, 0x61, 0x63, 0x65, 0x27, 0x2c, 0x0a, 0x20, 0x20, 0x6e, - 0x75, 0x6d, 0x62, 0x65, 0x72, 0x3a, 0x20, 0x27, 0x28, 0x22, 0x2d, 0x22, - 0x3f, 0x20, 0x28, 0x5b, 0x30, 0x2d, 0x39, 0x5d, 0x20, 0x7c, 0x20, 0x5b, - 0x31, 0x2d, 0x39, 0x5d, 0x20, 0x5b, 0x30, 0x2d, 0x39, 0x5d, 0x2a, 0x29, - 0x29, 0x20, 0x28, 0x22, 0x2e, 0x22, 0x20, 0x5b, 0x30, 0x2d, 0x39, 0x5d, - 0x2b, 0x29, 0x3f, 0x20, 0x28, 0x5b, 0x65, 0x45, 0x5d, 0x20, 0x5b, 0x2d, - 0x2b, 0x5d, 0x3f, 0x20, 0x5b, 0x30, 0x2d, 0x39, 0x5d, 0x2b, 0x29, 0x3f, - 0x20, 0x73, 0x70, 0x61, 0x63, 0x65, 0x27, 0x2c, 0x0a, 0x20, 0x20, 0x69, - 0x6e, 0x74, 0x65, 0x67, 0x65, 0x72, 0x3a, 0x20, 0x27, 0x28, 0x22, 0x2d, + 0x2f, 0x2f, 0x20, 0x57, 0x41, 0x52, 0x4e, 0x49, 0x4e, 0x47, 0x3a, 0x20, + 0x54, 0x68, 0x69, 0x73, 0x20, 0x66, 0x69, 0x6c, 0x65, 0x20, 0x77, 0x61, + 0x73, 0x20, 0x70, 0x6f, 0x72, 0x74, 0x65, 0x64, 0x20, 0x66, 0x72, 0x6f, + 0x6d, 0x20, 0x6a, 0x73, 0x6f, 0x6e, 0x2d, 0x73, 0x63, 0x68, 0x65, 0x6d, + 0x61, 0x2d, 0x74, 0x6f, 0x2d, 0x67, 0x72, 0x61, 0x6d, 0x6d, 0x61, 0x72, + 0x2e, 0x70, 0x79, 0x2c, 0x20, 0x70, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x20, + 0x66, 0x69, 0x78, 0x20, 0x62, 0x75, 0x67, 0x73, 0x20, 0x2f, 0x20, 0x61, + 0x64, 0x64, 0x20, 0x66, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x20, + 0x74, 0x68, 0x65, 0x72, 0x65, 0x20, 0x66, 0x69, 0x72, 0x73, 0x74, 0x2e, + 0x0a, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x53, 0x50, 0x41, 0x43, 0x45, + 0x5f, 0x52, 0x55, 0x4c, 0x45, 0x20, 0x3d, 0x20, 0x27, 0x22, 0x20, 0x22, + 0x3f, 0x27, 0x3b, 0x0a, 0x0a, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x50, + 0x52, 0x49, 0x4d, 0x49, 0x54, 0x49, 0x56, 0x45, 0x5f, 0x52, 0x55, 0x4c, + 0x45, 0x53, 0x20, 0x3d, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x62, 0x6f, 0x6f, + 0x6c, 0x65, 0x61, 0x6e, 0x3a, 0x20, 0x27, 0x28, 0x22, 0x74, 0x72, 0x75, + 0x65, 0x22, 0x20, 0x7c, 0x20, 0x22, 0x66, 0x61, 0x6c, 0x73, 0x65, 0x22, + 0x29, 0x20, 0x73, 0x70, 0x61, 0x63, 0x65, 0x27, 0x2c, 0x0a, 0x20, 0x20, + 0x6e, 0x75, 0x6d, 0x62, 0x65, 0x72, 0x3a, 0x20, 0x27, 0x28, 0x22, 0x2d, 0x22, 0x3f, 0x20, 0x28, 0x5b, 0x30, 0x2d, 0x39, 0x5d, 0x20, 0x7c, 0x20, 0x5b, 0x31, 0x2d, 0x39, 0x5d, 0x20, 0x5b, 0x30, 0x2d, 0x39, 0x5d, 0x2a, - 0x29, 0x29, 0x20, 0x73, 0x70, 0x61, 0x63, 0x65, 0x27, 0x2c, 0x0a, 0x20, - 0x20, 0x73, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x3a, 0x20, 0x60, 0x20, 0x22, - 0x5c, 0x5c, 0x22, 0x22, 0x20, 0x28, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x5b, 0x5e, 0x22, 0x5c, 0x5c, 0x5c, 0x5c, 0x5d, 0x20, - 0x7c, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x22, 0x5c, - 0x5c, 0x5c, 0x5c, 0x22, 0x20, 0x28, 0x5b, 0x22, 0x5c, 0x5c, 0x5c, 0x5c, - 0x2f, 0x62, 0x66, 0x6e, 0x72, 0x74, 0x5d, 0x20, 0x7c, 0x20, 0x22, 0x75, - 0x22, 0x20, 0x5b, 0x30, 0x2d, 0x39, 0x61, 0x2d, 0x66, 0x41, 0x2d, 0x46, - 0x5d, 0x20, 0x5b, 0x30, 0x2d, 0x39, 0x61, 0x2d, 0x66, 0x41, 0x2d, 0x46, - 0x5d, 0x20, 0x5b, 0x30, 0x2d, 0x39, 0x61, 0x2d, 0x66, 0x41, 0x2d, 0x46, - 0x5d, 0x20, 0x5b, 0x30, 0x2d, 0x39, 0x61, 0x2d, 0x66, 0x41, 0x2d, 0x46, - 0x5d, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x29, 0x2a, 0x20, - 0x22, 0x5c, 0x5c, 0x22, 0x22, 0x20, 0x73, 0x70, 0x61, 0x63, 0x65, 0x60, - 0x2c, 0x0a, 0x20, 0x20, 0x6e, 0x75, 0x6c, 0x6c, 0x3a, 0x20, 0x27, 0x22, - 0x6e, 0x75, 0x6c, 0x6c, 0x22, 0x20, 0x73, 0x70, 0x61, 0x63, 0x65, 0x27, - 0x2c, 0x0a, 0x7d, 0x3b, 0x0a, 0x0a, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, - 0x49, 0x4e, 0x56, 0x41, 0x4c, 0x49, 0x44, 0x5f, 0x52, 0x55, 0x4c, 0x45, - 0x5f, 0x43, 0x48, 0x41, 0x52, 0x53, 0x5f, 0x52, 0x45, 0x20, 0x3d, 0x20, - 0x2f, 0x5b, 0x5e, 0x5c, 0x64, 0x41, 0x2d, 0x5a, 0x61, 0x2d, 0x7a, 0x2d, - 0x5d, 0x2b, 0x2f, 0x67, 0x3b, 0x0a, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, + 0x29, 0x29, 0x20, 0x28, 0x22, 0x2e, 0x22, 0x20, 0x5b, 0x30, 0x2d, 0x39, + 0x5d, 0x2b, 0x29, 0x3f, 0x20, 0x28, 0x5b, 0x65, 0x45, 0x5d, 0x20, 0x5b, + 0x2d, 0x2b, 0x5d, 0x3f, 0x20, 0x5b, 0x30, 0x2d, 0x39, 0x5d, 0x2b, 0x29, + 0x3f, 0x20, 0x73, 0x70, 0x61, 0x63, 0x65, 0x27, 0x2c, 0x0a, 0x20, 0x20, + 0x69, 0x6e, 0x74, 0x65, 0x67, 0x65, 0x72, 0x3a, 0x20, 0x27, 0x28, 0x22, + 0x2d, 0x22, 0x3f, 0x20, 0x28, 0x5b, 0x30, 0x2d, 0x39, 0x5d, 0x20, 0x7c, + 0x20, 0x5b, 0x31, 0x2d, 0x39, 0x5d, 0x20, 0x5b, 0x30, 0x2d, 0x39, 0x5d, + 0x2a, 0x29, 0x29, 0x20, 0x73, 0x70, 0x61, 0x63, 0x65, 0x27, 0x2c, 0x0a, + 0x20, 0x20, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x20, 0x27, 0x6f, 0x62, + 0x6a, 0x65, 0x63, 0x74, 0x20, 0x7c, 0x20, 0x61, 0x72, 0x72, 0x61, 0x79, + 0x20, 0x7c, 0x20, 0x73, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x20, 0x7c, 0x20, + 0x6e, 0x75, 0x6d, 0x62, 0x65, 0x72, 0x20, 0x7c, 0x20, 0x62, 0x6f, 0x6f, + 0x6c, 0x65, 0x61, 0x6e, 0x27, 0x2c, 0x0a, 0x20, 0x20, 0x6f, 0x62, 0x6a, + 0x65, 0x63, 0x74, 0x3a, 0x20, 0x27, 0x22, 0x7b, 0x22, 0x20, 0x73, 0x70, + 0x61, 0x63, 0x65, 0x20, 0x28, 0x20, 0x73, 0x74, 0x72, 0x69, 0x6e, 0x67, + 0x20, 0x22, 0x3a, 0x22, 0x20, 0x73, 0x70, 0x61, 0x63, 0x65, 0x20, 0x76, + 0x61, 0x6c, 0x75, 0x65, 0x20, 0x28, 0x22, 0x2c, 0x22, 0x20, 0x73, 0x70, + 0x61, 0x63, 0x65, 0x20, 0x73, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x20, 0x22, + 0x3a, 0x22, 0x20, 0x73, 0x70, 0x61, 0x63, 0x65, 0x20, 0x76, 0x61, 0x6c, + 0x75, 0x65, 0x29, 0x2a, 0x20, 0x29, 0x3f, 0x20, 0x22, 0x7d, 0x22, 0x20, + 0x73, 0x70, 0x61, 0x63, 0x65, 0x27, 0x2c, 0x0a, 0x20, 0x20, 0x61, 0x72, + 0x72, 0x61, 0x79, 0x3a, 0x20, 0x27, 0x22, 0x5b, 0x22, 0x20, 0x73, 0x70, + 0x61, 0x63, 0x65, 0x20, 0x28, 0x20, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x20, + 0x28, 0x22, 0x2c, 0x22, 0x20, 0x73, 0x70, 0x61, 0x63, 0x65, 0x20, 0x76, + 0x61, 0x6c, 0x75, 0x65, 0x29, 0x2a, 0x20, 0x29, 0x3f, 0x20, 0x22, 0x5d, + 0x22, 0x20, 0x73, 0x70, 0x61, 0x63, 0x65, 0x27, 0x2c, 0x0a, 0x20, 0x20, + 0x75, 0x75, 0x69, 0x64, 0x3a, 0x20, 0x27, 0x22, 0x5c, 0x5c, 0x22, 0x22, + 0x20, 0x27, 0x20, 0x2b, 0x20, 0x5b, 0x38, 0x2c, 0x20, 0x34, 0x2c, 0x20, + 0x34, 0x2c, 0x20, 0x34, 0x2c, 0x20, 0x31, 0x32, 0x5d, 0x2e, 0x6d, 0x61, + 0x70, 0x28, 0x6e, 0x20, 0x3d, 0x3e, 0x20, 0x5b, 0x2e, 0x2e, 0x2e, 0x6e, + 0x65, 0x77, 0x20, 0x41, 0x72, 0x72, 0x61, 0x79, 0x28, 0x6e, 0x29, 0x5d, + 0x2e, 0x6d, 0x61, 0x70, 0x28, 0x5f, 0x20, 0x3d, 0x3e, 0x20, 0x27, 0x5b, + 0x30, 0x2d, 0x39, 0x61, 0x2d, 0x66, 0x41, 0x2d, 0x46, 0x5d, 0x27, 0x29, + 0x2e, 0x6a, 0x6f, 0x69, 0x6e, 0x28, 0x27, 0x27, 0x29, 0x29, 0x2e, 0x6a, + 0x6f, 0x69, 0x6e, 0x28, 0x27, 0x20, 0x22, 0x2d, 0x22, 0x20, 0x27, 0x29, + 0x20, 0x2b, 0x20, 0x27, 0x20, 0x22, 0x5c, 0x5c, 0x22, 0x22, 0x20, 0x73, + 0x70, 0x61, 0x63, 0x65, 0x27, 0x2c, 0x0a, 0x20, 0x20, 0x73, 0x74, 0x72, + 0x69, 0x6e, 0x67, 0x3a, 0x20, 0x60, 0x20, 0x22, 0x5c, 0x5c, 0x22, 0x22, + 0x20, 0x28, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x5b, + 0x5e, 0x22, 0x5c, 0x5c, 0x5c, 0x5c, 0x5d, 0x20, 0x7c, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x22, 0x5c, 0x5c, 0x5c, 0x5c, 0x22, + 0x20, 0x28, 0x5b, 0x22, 0x5c, 0x5c, 0x5c, 0x5c, 0x2f, 0x62, 0x66, 0x6e, + 0x72, 0x74, 0x5d, 0x20, 0x7c, 0x20, 0x22, 0x75, 0x22, 0x20, 0x5b, 0x30, + 0x2d, 0x39, 0x61, 0x2d, 0x66, 0x41, 0x2d, 0x46, 0x5d, 0x20, 0x5b, 0x30, + 0x2d, 0x39, 0x61, 0x2d, 0x66, 0x41, 0x2d, 0x46, 0x5d, 0x20, 0x5b, 0x30, + 0x2d, 0x39, 0x61, 0x2d, 0x66, 0x41, 0x2d, 0x46, 0x5d, 0x20, 0x5b, 0x30, + 0x2d, 0x39, 0x61, 0x2d, 0x66, 0x41, 0x2d, 0x46, 0x5d, 0x29, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x29, 0x2a, 0x20, 0x22, 0x5c, 0x5c, 0x22, + 0x22, 0x20, 0x73, 0x70, 0x61, 0x63, 0x65, 0x60, 0x2c, 0x0a, 0x20, 0x20, + 0x6e, 0x75, 0x6c, 0x6c, 0x3a, 0x20, 0x27, 0x22, 0x6e, 0x75, 0x6c, 0x6c, + 0x22, 0x20, 0x73, 0x70, 0x61, 0x63, 0x65, 0x27, 0x2c, 0x0a, 0x7d, 0x3b, + 0x0a, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x4f, 0x42, 0x4a, 0x45, 0x43, + 0x54, 0x5f, 0x52, 0x55, 0x4c, 0x45, 0x5f, 0x4e, 0x41, 0x4d, 0x45, 0x53, + 0x20, 0x3d, 0x20, 0x5b, 0x27, 0x6f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x27, + 0x2c, 0x20, 0x27, 0x61, 0x72, 0x72, 0x61, 0x79, 0x27, 0x2c, 0x20, 0x27, + 0x73, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x27, 0x2c, 0x20, 0x27, 0x6e, 0x75, + 0x6d, 0x62, 0x65, 0x72, 0x27, 0x2c, 0x20, 0x27, 0x62, 0x6f, 0x6f, 0x6c, + 0x65, 0x61, 0x6e, 0x27, 0x2c, 0x20, 0x27, 0x6e, 0x75, 0x6c, 0x6c, 0x27, + 0x2c, 0x20, 0x27, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x27, 0x5d, 0x3b, 0x0a, + 0x0a, 0x2f, 0x2f, 0x20, 0x54, 0x4f, 0x44, 0x4f, 0x3a, 0x20, 0x73, 0x75, + 0x70, 0x70, 0x6f, 0x72, 0x74, 0x20, 0x22, 0x75, 0x72, 0x69, 0x22, 0x2c, + 0x20, 0x22, 0x65, 0x6d, 0x61, 0x69, 0x6c, 0x22, 0x20, 0x73, 0x74, 0x72, + 0x69, 0x6e, 0x67, 0x20, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x73, 0x0a, + 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x44, 0x41, 0x54, 0x45, 0x5f, 0x52, + 0x55, 0x4c, 0x45, 0x53, 0x20, 0x3d, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x27, 0x64, 0x61, 0x74, 0x65, 0x27, 0x20, 0x20, 0x20, 0x3a, 0x20, + 0x27, 0x5b, 0x30, 0x2d, 0x39, 0x5d, 0x20, 0x5b, 0x30, 0x2d, 0x39, 0x5d, + 0x20, 0x5b, 0x30, 0x2d, 0x39, 0x5d, 0x20, 0x5b, 0x30, 0x2d, 0x39, 0x5d, + 0x20, 0x22, 0x2d, 0x22, 0x20, 0x28, 0x20, 0x22, 0x30, 0x22, 0x20, 0x5b, + 0x31, 0x2d, 0x39, 0x5d, 0x20, 0x7c, 0x20, 0x22, 0x31, 0x22, 0x20, 0x5b, + 0x30, 0x2d, 0x32, 0x5d, 0x20, 0x29, 0x20, 0x22, 0x2d, 0x22, 0x20, 0x28, + 0x20, 0x5c, 0x22, 0x30, 0x5c, 0x22, 0x20, 0x5b, 0x31, 0x2d, 0x39, 0x5d, + 0x20, 0x7c, 0x20, 0x5b, 0x31, 0x2d, 0x32, 0x5d, 0x20, 0x5b, 0x30, 0x2d, + 0x39, 0x5d, 0x20, 0x7c, 0x20, 0x22, 0x33, 0x22, 0x20, 0x5b, 0x30, 0x2d, + 0x31, 0x5d, 0x20, 0x29, 0x27, 0x2c, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x27, + 0x74, 0x69, 0x6d, 0x65, 0x27, 0x20, 0x20, 0x20, 0x3a, 0x20, 0x27, 0x28, + 0x5b, 0x30, 0x31, 0x5d, 0x20, 0x5b, 0x30, 0x2d, 0x39, 0x5d, 0x20, 0x7c, + 0x20, 0x22, 0x32, 0x22, 0x20, 0x5b, 0x30, 0x2d, 0x33, 0x5d, 0x29, 0x20, + 0x22, 0x3a, 0x22, 0x20, 0x5b, 0x30, 0x2d, 0x35, 0x5d, 0x20, 0x5b, 0x30, + 0x2d, 0x39, 0x5d, 0x20, 0x22, 0x3a, 0x22, 0x20, 0x5b, 0x30, 0x2d, 0x35, + 0x5d, 0x20, 0x5b, 0x30, 0x2d, 0x39, 0x5d, 0x20, 0x28, 0x20, 0x22, 0x2e, + 0x22, 0x20, 0x5b, 0x30, 0x2d, 0x39, 0x5d, 0x20, 0x5b, 0x30, 0x2d, 0x39, + 0x5d, 0x20, 0x5b, 0x30, 0x2d, 0x39, 0x5d, 0x20, 0x29, 0x3f, 0x20, 0x28, + 0x20, 0x22, 0x5a, 0x22, 0x20, 0x7c, 0x20, 0x28, 0x20, 0x22, 0x2b, 0x22, + 0x20, 0x7c, 0x20, 0x22, 0x2d, 0x22, 0x20, 0x29, 0x20, 0x28, 0x20, 0x5b, + 0x30, 0x31, 0x5d, 0x20, 0x5b, 0x30, 0x2d, 0x39, 0x5d, 0x20, 0x7c, 0x20, + 0x22, 0x32, 0x22, 0x20, 0x5b, 0x30, 0x2d, 0x33, 0x5d, 0x20, 0x29, 0x20, + 0x22, 0x3a, 0x22, 0x20, 0x5b, 0x30, 0x2d, 0x35, 0x5d, 0x20, 0x5b, 0x30, + 0x2d, 0x39, 0x5d, 0x20, 0x29, 0x27, 0x2c, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x27, 0x64, 0x61, 0x74, 0x65, 0x2d, 0x74, 0x69, 0x6d, 0x65, 0x27, 0x3a, + 0x20, 0x27, 0x64, 0x61, 0x74, 0x65, 0x20, 0x22, 0x54, 0x22, 0x20, 0x74, + 0x69, 0x6d, 0x65, 0x27, 0x2c, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x27, 0x64, + 0x61, 0x74, 0x65, 0x2d, 0x73, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x27, 0x3a, + 0x20, 0x27, 0x22, 0x5c, 0x5c, 0x22, 0x22, 0x20, 0x64, 0x61, 0x74, 0x65, + 0x20, 0x22, 0x5c, 0x5c, 0x22, 0x22, 0x20, 0x73, 0x70, 0x61, 0x63, 0x65, + 0x27, 0x2c, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x27, 0x74, 0x69, 0x6d, 0x65, + 0x2d, 0x73, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x27, 0x3a, 0x20, 0x27, 0x22, + 0x5c, 0x5c, 0x22, 0x22, 0x20, 0x74, 0x69, 0x6d, 0x65, 0x20, 0x22, 0x5c, + 0x5c, 0x22, 0x22, 0x20, 0x73, 0x70, 0x61, 0x63, 0x65, 0x27, 0x2c, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x27, 0x64, 0x61, 0x74, 0x65, 0x2d, 0x74, 0x69, + 0x6d, 0x65, 0x2d, 0x73, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x27, 0x3a, 0x20, + 0x27, 0x22, 0x5c, 0x5c, 0x22, 0x22, 0x20, 0x64, 0x61, 0x74, 0x65, 0x2d, + 0x74, 0x69, 0x6d, 0x65, 0x20, 0x22, 0x5c, 0x5c, 0x22, 0x22, 0x20, 0x73, + 0x70, 0x61, 0x63, 0x65, 0x27, 0x2c, 0x0a, 0x7d, 0x3b, 0x0a, 0x0a, 0x63, + 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x52, 0x45, 0x53, 0x45, 0x52, 0x56, 0x45, + 0x44, 0x5f, 0x4e, 0x41, 0x4d, 0x45, 0x53, 0x20, 0x3d, 0x20, 0x7b, 0x27, + 0x72, 0x6f, 0x6f, 0x74, 0x27, 0x3a, 0x20, 0x74, 0x72, 0x75, 0x65, 0x2c, + 0x20, 0x2e, 0x2e, 0x2e, 0x50, 0x52, 0x49, 0x4d, 0x49, 0x54, 0x49, 0x56, + 0x45, 0x5f, 0x52, 0x55, 0x4c, 0x45, 0x53, 0x2c, 0x20, 0x2e, 0x2e, 0x2e, + 0x44, 0x41, 0x54, 0x45, 0x5f, 0x52, 0x55, 0x4c, 0x45, 0x53, 0x7d, 0x3b, + 0x0a, 0x0a, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x49, 0x4e, 0x56, 0x41, + 0x4c, 0x49, 0x44, 0x5f, 0x52, 0x55, 0x4c, 0x45, 0x5f, 0x43, 0x48, 0x41, + 0x52, 0x53, 0x5f, 0x52, 0x45, 0x20, 0x3d, 0x20, 0x2f, 0x5b, 0x5e, 0x5c, + 0x64, 0x41, 0x2d, 0x5a, 0x61, 0x2d, 0x7a, 0x2d, 0x5d, 0x2b, 0x2f, 0x67, + 0x3b, 0x0a, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x47, 0x52, 0x41, 0x4d, + 0x4d, 0x41, 0x52, 0x5f, 0x4c, 0x49, 0x54, 0x45, 0x52, 0x41, 0x4c, 0x5f, + 0x45, 0x53, 0x43, 0x41, 0x50, 0x45, 0x5f, 0x52, 0x45, 0x20, 0x3d, 0x20, + 0x2f, 0x5b, 0x5c, 0x6e, 0x5c, 0x72, 0x22, 0x5d, 0x2f, 0x67, 0x3b, 0x0a, + 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x47, 0x52, 0x41, 0x4d, 0x4d, 0x41, + 0x52, 0x5f, 0x52, 0x41, 0x4e, 0x47, 0x45, 0x5f, 0x4c, 0x49, 0x54, 0x45, + 0x52, 0x41, 0x4c, 0x5f, 0x45, 0x53, 0x43, 0x41, 0x50, 0x45, 0x5f, 0x52, + 0x45, 0x20, 0x3d, 0x20, 0x2f, 0x5b, 0x5c, 0x6e, 0x5c, 0x72, 0x22, 0x5c, + 0x5d, 0x5c, 0x2d, 0x5c, 0x5c, 0x5d, 0x2f, 0x67, 0x3b, 0x0a, 0x63, 0x6f, + 0x6e, 0x73, 0x74, 0x20, 0x47, 0x52, 0x41, 0x4d, 0x4d, 0x41, 0x52, 0x5f, + 0x4c, 0x49, 0x54, 0x45, 0x52, 0x41, 0x4c, 0x5f, 0x45, 0x53, 0x43, 0x41, + 0x50, 0x45, 0x53, 0x20, 0x3d, 0x20, 0x7b, 0x20, 0x27, 0x5c, 0x72, 0x27, + 0x3a, 0x20, 0x27, 0x5c, 0x5c, 0x72, 0x27, 0x2c, 0x20, 0x27, 0x5c, 0x6e, + 0x27, 0x3a, 0x20, 0x27, 0x5c, 0x5c, 0x6e, 0x27, 0x2c, 0x20, 0x27, 0x22, + 0x27, 0x3a, 0x20, 0x27, 0x5c, 0x5c, 0x22, 0x27, 0x2c, 0x20, 0x27, 0x2d, + 0x27, 0x3a, 0x20, 0x27, 0x5c, 0x5c, 0x2d, 0x27, 0x2c, 0x20, 0x27, 0x5d, + 0x27, 0x3a, 0x20, 0x27, 0x5c, 0x5c, 0x5d, 0x27, 0x20, 0x7d, 0x3b, 0x0a, + 0x0a, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x4e, 0x4f, 0x4e, 0x5f, 0x4c, + 0x49, 0x54, 0x45, 0x52, 0x41, 0x4c, 0x5f, 0x53, 0x45, 0x54, 0x20, 0x3d, + 0x20, 0x6e, 0x65, 0x77, 0x20, 0x53, 0x65, 0x74, 0x28, 0x27, 0x7c, 0x2e, + 0x28, 0x29, 0x5b, 0x5d, 0x7b, 0x7d, 0x2a, 0x2b, 0x3f, 0x27, 0x29, 0x3b, + 0x0a, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x45, 0x53, 0x43, 0x41, 0x50, + 0x45, 0x44, 0x5f, 0x49, 0x4e, 0x5f, 0x52, 0x45, 0x47, 0x45, 0x58, 0x50, + 0x53, 0x5f, 0x42, 0x55, 0x54, 0x5f, 0x4e, 0x4f, 0x54, 0x5f, 0x49, 0x4e, + 0x5f, 0x4c, 0x49, 0x54, 0x45, 0x52, 0x41, 0x4c, 0x53, 0x20, 0x3d, 0x20, + 0x6e, 0x65, 0x77, 0x20, 0x53, 0x65, 0x74, 0x28, 0x27, 0x5b, 0x5d, 0x28, + 0x29, 0x7c, 0x7b, 0x7d, 0x2a, 0x2b, 0x3f, 0x27, 0x29, 0x3b, 0x0a, 0x0a, + 0x65, 0x78, 0x70, 0x6f, 0x72, 0x74, 0x20, 0x63, 0x6c, 0x61, 0x73, 0x73, + 0x20, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x43, 0x6f, 0x6e, 0x76, 0x65, + 0x72, 0x74, 0x65, 0x72, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x63, 0x6f, 0x6e, + 0x73, 0x74, 0x72, 0x75, 0x63, 0x74, 0x6f, 0x72, 0x28, 0x6f, 0x70, 0x74, + 0x69, 0x6f, 0x6e, 0x73, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x74, 0x68, 0x69, 0x73, 0x2e, 0x5f, 0x70, 0x72, 0x6f, 0x70, 0x4f, 0x72, + 0x64, 0x65, 0x72, 0x20, 0x3d, 0x20, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, + 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x70, 0x5f, 0x6f, 0x72, 0x64, 0x65, 0x72, + 0x20, 0x7c, 0x7c, 0x20, 0x7b, 0x7d, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x74, 0x68, 0x69, 0x73, 0x2e, 0x5f, 0x61, 0x6c, 0x6c, 0x6f, 0x77, 0x46, + 0x65, 0x74, 0x63, 0x68, 0x20, 0x3d, 0x20, 0x6f, 0x70, 0x74, 0x69, 0x6f, + 0x6e, 0x73, 0x2e, 0x61, 0x6c, 0x6c, 0x6f, 0x77, 0x5f, 0x66, 0x65, 0x74, + 0x63, 0x68, 0x20, 0x7c, 0x7c, 0x20, 0x66, 0x61, 0x6c, 0x73, 0x65, 0x3b, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x5f, 0x64, + 0x6f, 0x74, 0x61, 0x6c, 0x6c, 0x20, 0x3d, 0x20, 0x6f, 0x70, 0x74, 0x69, + 0x6f, 0x6e, 0x73, 0x2e, 0x64, 0x6f, 0x74, 0x61, 0x6c, 0x6c, 0x20, 0x7c, + 0x7c, 0x20, 0x66, 0x61, 0x6c, 0x73, 0x65, 0x3b, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x5f, 0x72, 0x75, 0x6c, 0x65, 0x73, + 0x20, 0x3d, 0x20, 0x7b, 0x27, 0x73, 0x70, 0x61, 0x63, 0x65, 0x27, 0x3a, + 0x20, 0x53, 0x50, 0x41, 0x43, 0x45, 0x5f, 0x52, 0x55, 0x4c, 0x45, 0x7d, + 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x5f, + 0x72, 0x65, 0x66, 0x73, 0x20, 0x3d, 0x20, 0x7b, 0x7d, 0x3b, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x5f, 0x72, 0x65, 0x66, + 0x73, 0x42, 0x65, 0x69, 0x6e, 0x67, 0x52, 0x65, 0x73, 0x6f, 0x6c, 0x76, + 0x65, 0x64, 0x20, 0x3d, 0x20, 0x6e, 0x65, 0x77, 0x20, 0x53, 0x65, 0x74, + 0x28, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x7d, 0x0a, 0x0a, 0x20, 0x20, 0x5f, + 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x4c, 0x69, 0x74, 0x65, 0x72, 0x61, + 0x6c, 0x28, 0x6c, 0x69, 0x74, 0x65, 0x72, 0x61, 0x6c, 0x29, 0x20, 0x7b, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x65, + 0x73, 0x63, 0x61, 0x70, 0x65, 0x64, 0x20, 0x3d, 0x20, 0x4a, 0x53, 0x4f, + 0x4e, 0x2e, 0x73, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x69, 0x66, 0x79, 0x28, + 0x6c, 0x69, 0x74, 0x65, 0x72, 0x61, 0x6c, 0x29, 0x2e, 0x72, 0x65, 0x70, + 0x6c, 0x61, 0x63, 0x65, 0x28, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x47, 0x52, 0x41, 0x4d, 0x4d, 0x41, 0x52, 0x5f, 0x4c, 0x49, 0x54, 0x45, 0x52, 0x41, 0x4c, 0x5f, 0x45, 0x53, 0x43, 0x41, 0x50, 0x45, 0x5f, 0x52, - 0x45, 0x20, 0x3d, 0x20, 0x2f, 0x5b, 0x5c, 0x6e, 0x5c, 0x72, 0x22, 0x5d, - 0x2f, 0x67, 0x3b, 0x0a, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x47, 0x52, - 0x41, 0x4d, 0x4d, 0x41, 0x52, 0x5f, 0x4c, 0x49, 0x54, 0x45, 0x52, 0x41, - 0x4c, 0x5f, 0x45, 0x53, 0x43, 0x41, 0x50, 0x45, 0x53, 0x20, 0x3d, 0x20, - 0x7b, 0x27, 0x5c, 0x72, 0x27, 0x3a, 0x20, 0x27, 0x5c, 0x5c, 0x72, 0x27, - 0x2c, 0x20, 0x27, 0x5c, 0x6e, 0x27, 0x3a, 0x20, 0x27, 0x5c, 0x5c, 0x6e, - 0x27, 0x2c, 0x20, 0x27, 0x22, 0x27, 0x3a, 0x20, 0x27, 0x5c, 0x5c, 0x22, - 0x27, 0x7d, 0x3b, 0x0a, 0x0a, 0x65, 0x78, 0x70, 0x6f, 0x72, 0x74, 0x20, - 0x63, 0x6c, 0x61, 0x73, 0x73, 0x20, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, - 0x43, 0x6f, 0x6e, 0x76, 0x65, 0x72, 0x74, 0x65, 0x72, 0x20, 0x7b, 0x0a, - 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x72, 0x75, 0x63, 0x74, 0x6f, - 0x72, 0x28, 0x70, 0x72, 0x6f, 0x70, 0x4f, 0x72, 0x64, 0x65, 0x72, 0x29, - 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, - 0x5f, 0x70, 0x72, 0x6f, 0x70, 0x4f, 0x72, 0x64, 0x65, 0x72, 0x20, 0x3d, - 0x20, 0x70, 0x72, 0x6f, 0x70, 0x4f, 0x72, 0x64, 0x65, 0x72, 0x20, 0x7c, - 0x7c, 0x20, 0x7b, 0x7d, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x74, 0x68, - 0x69, 0x73, 0x2e, 0x5f, 0x72, 0x75, 0x6c, 0x65, 0x73, 0x20, 0x3d, 0x20, - 0x6e, 0x65, 0x77, 0x20, 0x4d, 0x61, 0x70, 0x28, 0x29, 0x3b, 0x0a, 0x20, - 0x20, 0x20, 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x5f, 0x72, 0x75, 0x6c, - 0x65, 0x73, 0x2e, 0x73, 0x65, 0x74, 0x28, 0x27, 0x73, 0x70, 0x61, 0x63, - 0x65, 0x27, 0x2c, 0x20, 0x53, 0x50, 0x41, 0x43, 0x45, 0x5f, 0x52, 0x55, - 0x4c, 0x45, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x7d, 0x0a, 0x0a, 0x20, 0x20, - 0x5f, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x4c, 0x69, 0x74, 0x65, 0x72, - 0x61, 0x6c, 0x28, 0x6c, 0x69, 0x74, 0x65, 0x72, 0x61, 0x6c, 0x29, 0x20, - 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, - 0x65, 0x73, 0x63, 0x61, 0x70, 0x65, 0x64, 0x20, 0x3d, 0x20, 0x4a, 0x53, - 0x4f, 0x4e, 0x2e, 0x73, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x69, 0x66, 0x79, - 0x28, 0x6c, 0x69, 0x74, 0x65, 0x72, 0x61, 0x6c, 0x29, 0x2e, 0x72, 0x65, - 0x70, 0x6c, 0x61, 0x63, 0x65, 0x28, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x45, 0x2c, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x6d, 0x20, 0x3d, + 0x3e, 0x20, 0x47, 0x52, 0x41, 0x4d, 0x4d, 0x41, 0x52, 0x5f, 0x4c, 0x49, + 0x54, 0x45, 0x52, 0x41, 0x4c, 0x5f, 0x45, 0x53, 0x43, 0x41, 0x50, 0x45, + 0x53, 0x5b, 0x6d, 0x5d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x29, 0x3b, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x60, + 0x22, 0x24, 0x7b, 0x65, 0x73, 0x63, 0x61, 0x70, 0x65, 0x64, 0x7d, 0x22, + 0x60, 0x3b, 0x0a, 0x20, 0x20, 0x7d, 0x0a, 0x0a, 0x20, 0x20, 0x5f, 0x66, + 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x52, 0x61, 0x6e, 0x67, 0x65, 0x43, 0x68, + 0x61, 0x72, 0x28, 0x6c, 0x69, 0x74, 0x65, 0x72, 0x61, 0x6c, 0x29, 0x20, + 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, + 0x20, 0x4a, 0x53, 0x4f, 0x4e, 0x2e, 0x73, 0x74, 0x72, 0x69, 0x6e, 0x67, + 0x69, 0x66, 0x79, 0x28, 0x6c, 0x69, 0x74, 0x65, 0x72, 0x61, 0x6c, 0x29, + 0x2e, 0x73, 0x6c, 0x69, 0x63, 0x65, 0x28, 0x31, 0x2c, 0x20, 0x2d, 0x31, + 0x29, 0x2e, 0x72, 0x65, 0x70, 0x6c, 0x61, 0x63, 0x65, 0x28, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x47, 0x52, 0x41, 0x4d, 0x4d, 0x41, 0x52, + 0x5f, 0x52, 0x41, 0x4e, 0x47, 0x45, 0x5f, 0x4c, 0x49, 0x54, 0x45, 0x52, + 0x41, 0x4c, 0x5f, 0x45, 0x53, 0x43, 0x41, 0x50, 0x45, 0x5f, 0x52, 0x45, + 0x2c, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x6d, 0x20, 0x3d, 0x3e, 0x20, 0x47, 0x52, 0x41, 0x4d, 0x4d, 0x41, 0x52, 0x5f, 0x4c, 0x49, 0x54, - 0x45, 0x52, 0x41, 0x4c, 0x5f, 0x45, 0x53, 0x43, 0x41, 0x50, 0x45, 0x5f, - 0x52, 0x45, 0x2c, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x6d, 0x20, - 0x3d, 0x3e, 0x20, 0x47, 0x52, 0x41, 0x4d, 0x4d, 0x41, 0x52, 0x5f, 0x4c, - 0x49, 0x54, 0x45, 0x52, 0x41, 0x4c, 0x5f, 0x45, 0x53, 0x43, 0x41, 0x50, - 0x45, 0x53, 0x5b, 0x6d, 0x5d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x29, 0x3b, - 0x0a, 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, - 0x60, 0x22, 0x24, 0x7b, 0x65, 0x73, 0x63, 0x61, 0x70, 0x65, 0x64, 0x7d, - 0x22, 0x60, 0x3b, 0x0a, 0x20, 0x20, 0x7d, 0x0a, 0x0a, 0x20, 0x20, 0x5f, - 0x61, 0x64, 0x64, 0x52, 0x75, 0x6c, 0x65, 0x28, 0x6e, 0x61, 0x6d, 0x65, - 0x2c, 0x20, 0x72, 0x75, 0x6c, 0x65, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, - 0x20, 0x20, 0x6c, 0x65, 0x74, 0x20, 0x65, 0x73, 0x63, 0x4e, 0x61, 0x6d, - 0x65, 0x20, 0x3d, 0x20, 0x6e, 0x61, 0x6d, 0x65, 0x2e, 0x72, 0x65, 0x70, - 0x6c, 0x61, 0x63, 0x65, 0x28, 0x49, 0x4e, 0x56, 0x41, 0x4c, 0x49, 0x44, - 0x5f, 0x52, 0x55, 0x4c, 0x45, 0x5f, 0x43, 0x48, 0x41, 0x52, 0x53, 0x5f, - 0x52, 0x45, 0x2c, 0x20, 0x27, 0x2d, 0x27, 0x29, 0x3b, 0x0a, 0x20, 0x20, - 0x20, 0x20, 0x6c, 0x65, 0x74, 0x20, 0x6b, 0x65, 0x79, 0x20, 0x3d, 0x20, - 0x65, 0x73, 0x63, 0x4e, 0x61, 0x6d, 0x65, 0x3b, 0x0a, 0x0a, 0x20, 0x20, - 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x5f, - 0x72, 0x75, 0x6c, 0x65, 0x73, 0x2e, 0x68, 0x61, 0x73, 0x28, 0x65, 0x73, - 0x63, 0x4e, 0x61, 0x6d, 0x65, 0x29, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, + 0x45, 0x52, 0x41, 0x4c, 0x5f, 0x45, 0x53, 0x43, 0x41, 0x50, 0x45, 0x53, + 0x5b, 0x6d, 0x5d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x29, 0x3b, 0x0a, 0x20, + 0x20, 0x7d, 0x0a, 0x0a, 0x20, 0x20, 0x5f, 0x61, 0x64, 0x64, 0x52, 0x75, + 0x6c, 0x65, 0x28, 0x6e, 0x61, 0x6d, 0x65, 0x2c, 0x20, 0x72, 0x75, 0x6c, + 0x65, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x6c, 0x65, 0x74, + 0x20, 0x65, 0x73, 0x63, 0x4e, 0x61, 0x6d, 0x65, 0x20, 0x3d, 0x20, 0x6e, + 0x61, 0x6d, 0x65, 0x2e, 0x72, 0x65, 0x70, 0x6c, 0x61, 0x63, 0x65, 0x28, + 0x49, 0x4e, 0x56, 0x41, 0x4c, 0x49, 0x44, 0x5f, 0x52, 0x55, 0x4c, 0x45, + 0x5f, 0x43, 0x48, 0x41, 0x52, 0x53, 0x5f, 0x52, 0x45, 0x2c, 0x20, 0x27, + 0x2d, 0x27, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x6c, 0x65, 0x74, + 0x20, 0x6b, 0x65, 0x79, 0x20, 0x3d, 0x20, 0x65, 0x73, 0x63, 0x4e, 0x61, + 0x6d, 0x65, 0x3b, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, + 0x28, 0x65, 0x73, 0x63, 0x4e, 0x61, 0x6d, 0x65, 0x20, 0x69, 0x6e, 0x20, + 0x74, 0x68, 0x69, 0x73, 0x2e, 0x5f, 0x72, 0x75, 0x6c, 0x65, 0x73, 0x29, + 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, + 0x28, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x5f, 0x72, 0x75, 0x6c, 0x65, 0x73, + 0x5b, 0x65, 0x73, 0x63, 0x4e, 0x61, 0x6d, 0x65, 0x5d, 0x20, 0x3d, 0x3d, + 0x3d, 0x20, 0x72, 0x75, 0x6c, 0x65, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, + 0x20, 0x6b, 0x65, 0x79, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x7d, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x6c, 0x65, 0x74, + 0x20, 0x69, 0x20, 0x3d, 0x20, 0x30, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x77, 0x68, 0x69, 0x6c, 0x65, 0x20, 0x28, 0x28, 0x60, 0x24, + 0x7b, 0x65, 0x73, 0x63, 0x4e, 0x61, 0x6d, 0x65, 0x7d, 0x24, 0x7b, 0x69, + 0x7d, 0x60, 0x20, 0x69, 0x6e, 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x5f, + 0x72, 0x75, 0x6c, 0x65, 0x73, 0x29, 0x20, 0x26, 0x26, 0x20, 0x28, 0x74, + 0x68, 0x69, 0x73, 0x2e, 0x5f, 0x72, 0x75, 0x6c, 0x65, 0x73, 0x5b, 0x60, + 0x24, 0x7b, 0x65, 0x73, 0x63, 0x4e, 0x61, 0x6d, 0x65, 0x7d, 0x24, 0x7b, + 0x69, 0x7d, 0x60, 0x5d, 0x20, 0x21, 0x3d, 0x3d, 0x20, 0x72, 0x75, 0x6c, + 0x65, 0x29, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x69, 0x20, 0x2b, 0x3d, 0x20, 0x31, 0x3b, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x6b, 0x65, 0x79, 0x20, 0x3d, 0x20, 0x60, 0x24, 0x7b, 0x65, 0x73, 0x63, + 0x4e, 0x61, 0x6d, 0x65, 0x7d, 0x24, 0x7b, 0x69, 0x7d, 0x60, 0x3b, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x74, + 0x68, 0x69, 0x73, 0x2e, 0x5f, 0x72, 0x75, 0x6c, 0x65, 0x73, 0x5b, 0x6b, + 0x65, 0x79, 0x5d, 0x20, 0x3d, 0x20, 0x72, 0x75, 0x6c, 0x65, 0x3b, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x6b, + 0x65, 0x79, 0x3b, 0x0a, 0x20, 0x20, 0x7d, 0x0a, 0x0a, 0x20, 0x20, 0x61, + 0x73, 0x79, 0x6e, 0x63, 0x20, 0x72, 0x65, 0x73, 0x6f, 0x6c, 0x76, 0x65, + 0x52, 0x65, 0x66, 0x73, 0x28, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2c, + 0x20, 0x75, 0x72, 0x6c, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x76, 0x69, 0x73, 0x69, 0x74, 0x20, + 0x3d, 0x20, 0x61, 0x73, 0x79, 0x6e, 0x63, 0x20, 0x28, 0x6e, 0x29, 0x20, + 0x3d, 0x3e, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, + 0x66, 0x20, 0x28, 0x41, 0x72, 0x72, 0x61, 0x79, 0x2e, 0x69, 0x73, 0x41, + 0x72, 0x72, 0x61, 0x79, 0x28, 0x6e, 0x29, 0x29, 0x20, 0x7b, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, + 0x6e, 0x20, 0x50, 0x72, 0x6f, 0x6d, 0x69, 0x73, 0x65, 0x2e, 0x61, 0x6c, + 0x6c, 0x28, 0x6e, 0x2e, 0x6d, 0x61, 0x70, 0x28, 0x76, 0x69, 0x73, 0x69, + 0x74, 0x29, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, + 0x20, 0x65, 0x6c, 0x73, 0x65, 0x20, 0x69, 0x66, 0x20, 0x28, 0x74, 0x79, + 0x70, 0x65, 0x6f, 0x66, 0x20, 0x6e, 0x20, 0x3d, 0x3d, 0x3d, 0x20, 0x27, + 0x6f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x27, 0x20, 0x26, 0x26, 0x20, 0x6e, + 0x20, 0x21, 0x3d, 0x3d, 0x20, 0x6e, 0x75, 0x6c, 0x6c, 0x29, 0x20, 0x7b, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x6c, 0x65, 0x74, + 0x20, 0x72, 0x65, 0x66, 0x20, 0x3d, 0x20, 0x6e, 0x2e, 0x24, 0x72, 0x65, + 0x66, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x6c, + 0x65, 0x74, 0x20, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x3b, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, 0x72, + 0x65, 0x66, 0x20, 0x21, 0x3d, 0x3d, 0x20, 0x75, 0x6e, 0x64, 0x65, 0x66, + 0x69, 0x6e, 0x65, 0x64, 0x20, 0x26, 0x26, 0x20, 0x21, 0x74, 0x68, 0x69, + 0x73, 0x2e, 0x5f, 0x72, 0x65, 0x66, 0x73, 0x5b, 0x72, 0x65, 0x66, 0x5d, + 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, 0x72, 0x65, 0x66, 0x2e, 0x73, 0x74, + 0x61, 0x72, 0x74, 0x73, 0x57, 0x69, 0x74, 0x68, 0x28, 0x27, 0x68, 0x74, + 0x74, 0x70, 0x73, 0x3a, 0x2f, 0x2f, 0x27, 0x29, 0x29, 0x20, 0x7b, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x69, 0x66, 0x20, 0x28, 0x21, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x5f, 0x61, + 0x6c, 0x6c, 0x6f, 0x77, 0x46, 0x65, 0x74, 0x63, 0x68, 0x29, 0x20, 0x7b, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x74, 0x68, 0x72, 0x6f, 0x77, 0x20, 0x6e, 0x65, 0x77, + 0x20, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x28, 0x27, 0x46, 0x65, 0x74, 0x63, + 0x68, 0x69, 0x6e, 0x67, 0x20, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x20, + 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x73, 0x20, 0x69, 0x73, 0x20, 0x6e, + 0x6f, 0x74, 0x20, 0x61, 0x6c, 0x6c, 0x6f, 0x77, 0x65, 0x64, 0x20, 0x28, + 0x75, 0x73, 0x65, 0x20, 0x2d, 0x2d, 0x61, 0x6c, 0x6c, 0x6f, 0x77, 0x2d, + 0x66, 0x65, 0x74, 0x63, 0x68, 0x20, 0x66, 0x6f, 0x72, 0x20, 0x66, 0x6f, + 0x72, 0x63, 0x65, 0x29, 0x27, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, + 0x6e, 0x73, 0x74, 0x20, 0x66, 0x65, 0x74, 0x63, 0x68, 0x20, 0x3d, 0x20, + 0x28, 0x61, 0x77, 0x61, 0x69, 0x74, 0x20, 0x69, 0x6d, 0x70, 0x6f, 0x72, + 0x74, 0x28, 0x27, 0x6e, 0x6f, 0x64, 0x65, 0x2d, 0x66, 0x65, 0x74, 0x63, + 0x68, 0x27, 0x29, 0x29, 0x2e, 0x64, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, + 0x3b, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x66, 0x72, 0x61, + 0x67, 0x53, 0x70, 0x6c, 0x69, 0x74, 0x20, 0x3d, 0x20, 0x72, 0x65, 0x66, + 0x2e, 0x73, 0x70, 0x6c, 0x69, 0x74, 0x28, 0x27, 0x23, 0x27, 0x29, 0x3b, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x62, 0x61, 0x73, 0x65, 0x55, + 0x72, 0x6c, 0x20, 0x3d, 0x20, 0x66, 0x72, 0x61, 0x67, 0x53, 0x70, 0x6c, + 0x69, 0x74, 0x5b, 0x30, 0x5d, 0x3b, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x74, 0x61, 0x72, 0x67, + 0x65, 0x74, 0x20, 0x3d, 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x5f, 0x72, + 0x65, 0x66, 0x73, 0x5b, 0x62, 0x61, 0x73, 0x65, 0x55, 0x72, 0x6c, 0x5d, + 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, 0x21, 0x74, 0x61, 0x72, 0x67, 0x65, + 0x74, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x74, 0x61, 0x72, 0x67, 0x65, + 0x74, 0x20, 0x3d, 0x20, 0x61, 0x77, 0x61, 0x69, 0x74, 0x20, 0x74, 0x68, + 0x69, 0x73, 0x2e, 0x72, 0x65, 0x73, 0x6f, 0x6c, 0x76, 0x65, 0x52, 0x65, + 0x66, 0x73, 0x28, 0x61, 0x77, 0x61, 0x69, 0x74, 0x20, 0x66, 0x65, 0x74, + 0x63, 0x68, 0x28, 0x72, 0x65, 0x66, 0x29, 0x2e, 0x74, 0x68, 0x65, 0x6e, + 0x28, 0x72, 0x65, 0x73, 0x20, 0x3d, 0x3e, 0x20, 0x72, 0x65, 0x73, 0x2e, + 0x6a, 0x73, 0x6f, 0x6e, 0x28, 0x29, 0x29, 0x2c, 0x20, 0x62, 0x61, 0x73, + 0x65, 0x55, 0x72, 0x6c, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x74, 0x68, 0x69, + 0x73, 0x2e, 0x5f, 0x72, 0x65, 0x66, 0x73, 0x5b, 0x62, 0x61, 0x73, 0x65, + 0x55, 0x72, 0x6c, 0x5d, 0x20, 0x3d, 0x20, 0x74, 0x61, 0x72, 0x67, 0x65, + 0x74, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, 0x66, 0x72, + 0x61, 0x67, 0x53, 0x70, 0x6c, 0x69, 0x74, 0x2e, 0x6c, 0x65, 0x6e, 0x67, + 0x74, 0x68, 0x20, 0x3d, 0x3d, 0x3d, 0x20, 0x31, 0x20, 0x7c, 0x7c, 0x20, + 0x66, 0x72, 0x61, 0x67, 0x53, 0x70, 0x6c, 0x69, 0x74, 0x5b, 0x66, 0x72, + 0x61, 0x67, 0x53, 0x70, 0x6c, 0x69, 0x74, 0x2e, 0x6c, 0x65, 0x6e, 0x67, + 0x74, 0x68, 0x20, 0x2d, 0x20, 0x31, 0x5d, 0x20, 0x3d, 0x3d, 0x3d, 0x20, + 0x27, 0x27, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, + 0x72, 0x6e, 0x20, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x3b, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, + 0x20, 0x65, 0x6c, 0x73, 0x65, 0x20, 0x69, 0x66, 0x20, 0x28, 0x72, 0x65, + 0x66, 0x2e, 0x73, 0x74, 0x61, 0x72, 0x74, 0x73, 0x57, 0x69, 0x74, 0x68, + 0x28, 0x27, 0x23, 0x2f, 0x27, 0x29, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x74, 0x61, + 0x72, 0x67, 0x65, 0x74, 0x20, 0x3d, 0x20, 0x73, 0x63, 0x68, 0x65, 0x6d, + 0x61, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x72, 0x65, 0x66, 0x20, 0x3d, 0x20, 0x60, 0x24, 0x7b, + 0x75, 0x72, 0x6c, 0x7d, 0x24, 0x7b, 0x72, 0x65, 0x66, 0x7d, 0x60, 0x3b, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x6e, 0x2e, 0x24, 0x72, 0x65, 0x66, 0x20, 0x3d, 0x20, 0x72, 0x65, + 0x66, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x7d, 0x20, 0x65, 0x6c, 0x73, 0x65, 0x20, 0x7b, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x74, 0x68, + 0x72, 0x6f, 0x77, 0x20, 0x6e, 0x65, 0x77, 0x20, 0x45, 0x72, 0x72, 0x6f, + 0x72, 0x28, 0x60, 0x55, 0x6e, 0x73, 0x75, 0x70, 0x70, 0x6f, 0x72, 0x74, + 0x65, 0x64, 0x20, 0x72, 0x65, 0x66, 0x20, 0x24, 0x7b, 0x72, 0x65, 0x66, + 0x7d, 0x60, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x73, 0x65, + 0x6c, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x73, 0x20, 0x3d, 0x20, 0x72, 0x65, + 0x66, 0x2e, 0x73, 0x70, 0x6c, 0x69, 0x74, 0x28, 0x27, 0x23, 0x27, 0x29, + 0x5b, 0x31, 0x5d, 0x2e, 0x73, 0x70, 0x6c, 0x69, 0x74, 0x28, 0x27, 0x2f, + 0x27, 0x29, 0x2e, 0x73, 0x6c, 0x69, 0x63, 0x65, 0x28, 0x31, 0x29, 0x3b, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x66, + 0x6f, 0x72, 0x20, 0x28, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x73, 0x65, + 0x6c, 0x20, 0x6f, 0x66, 0x20, 0x73, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x6f, + 0x72, 0x73, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, 0x21, 0x74, + 0x61, 0x72, 0x67, 0x65, 0x74, 0x20, 0x7c, 0x7c, 0x20, 0x21, 0x28, 0x73, + 0x65, 0x6c, 0x20, 0x69, 0x6e, 0x20, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, + 0x29, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x74, 0x68, 0x72, 0x6f, 0x77, + 0x20, 0x6e, 0x65, 0x77, 0x20, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x28, 0x60, + 0x45, 0x72, 0x72, 0x6f, 0x72, 0x20, 0x72, 0x65, 0x73, 0x6f, 0x6c, 0x76, + 0x69, 0x6e, 0x67, 0x20, 0x72, 0x65, 0x66, 0x20, 0x24, 0x7b, 0x72, 0x65, + 0x66, 0x7d, 0x3a, 0x20, 0x24, 0x7b, 0x73, 0x65, 0x6c, 0x7d, 0x20, 0x6e, + 0x6f, 0x74, 0x20, 0x69, 0x6e, 0x20, 0x24, 0x7b, 0x4a, 0x53, 0x4f, 0x4e, + 0x2e, 0x73, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x69, 0x66, 0x79, 0x28, 0x74, + 0x61, 0x72, 0x67, 0x65, 0x74, 0x29, 0x7d, 0x60, 0x29, 0x3b, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x20, 0x3d, 0x20, 0x74, 0x61, + 0x72, 0x67, 0x65, 0x74, 0x5b, 0x73, 0x65, 0x6c, 0x5d, 0x3b, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x74, 0x68, + 0x69, 0x73, 0x2e, 0x5f, 0x72, 0x65, 0x66, 0x73, 0x5b, 0x72, 0x65, 0x66, + 0x5d, 0x20, 0x3d, 0x20, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x3b, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x20, 0x65, 0x6c, + 0x73, 0x65, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x61, 0x77, 0x61, 0x69, 0x74, 0x20, 0x50, 0x72, 0x6f, + 0x6d, 0x69, 0x73, 0x65, 0x2e, 0x61, 0x6c, 0x6c, 0x28, 0x4f, 0x62, 0x6a, + 0x65, 0x63, 0x74, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x73, 0x28, 0x6e, + 0x29, 0x2e, 0x6d, 0x61, 0x70, 0x28, 0x76, 0x69, 0x73, 0x69, 0x74, 0x29, + 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x6e, + 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x3b, 0x0a, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x76, 0x69, 0x73, + 0x69, 0x74, 0x28, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x29, 0x3b, 0x0a, + 0x20, 0x20, 0x7d, 0x0a, 0x0a, 0x20, 0x20, 0x5f, 0x67, 0x65, 0x6e, 0x65, + 0x72, 0x61, 0x74, 0x65, 0x55, 0x6e, 0x69, 0x6f, 0x6e, 0x52, 0x75, 0x6c, + 0x65, 0x28, 0x6e, 0x61, 0x6d, 0x65, 0x2c, 0x20, 0x61, 0x6c, 0x74, 0x53, + 0x63, 0x68, 0x65, 0x6d, 0x61, 0x73, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x61, 0x6c, 0x74, + 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x73, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x2e, 0x6d, 0x61, 0x70, 0x28, 0x28, 0x61, 0x6c, 0x74, 0x53, + 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2c, 0x20, 0x69, 0x29, 0x20, 0x3d, 0x3e, + 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x76, 0x69, 0x73, 0x69, 0x74, 0x28, + 0x61, 0x6c, 0x74, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2c, 0x20, 0x60, + 0x24, 0x7b, 0x6e, 0x61, 0x6d, 0x65, 0x20, 0x3f, 0x3f, 0x20, 0x27, 0x27, + 0x7d, 0x24, 0x7b, 0x6e, 0x61, 0x6d, 0x65, 0x20, 0x3f, 0x20, 0x27, 0x2d, + 0x27, 0x20, 0x3a, 0x20, 0x27, 0x61, 0x6c, 0x74, 0x65, 0x72, 0x6e, 0x61, + 0x74, 0x69, 0x76, 0x65, 0x2d, 0x27, 0x7d, 0x24, 0x7b, 0x69, 0x7d, 0x60, + 0x29, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x2e, 0x6a, 0x6f, + 0x69, 0x6e, 0x28, 0x27, 0x20, 0x7c, 0x20, 0x27, 0x29, 0x3b, 0x0a, 0x20, + 0x20, 0x7d, 0x0a, 0x0a, 0x20, 0x20, 0x5f, 0x76, 0x69, 0x73, 0x69, 0x74, + 0x50, 0x61, 0x74, 0x74, 0x65, 0x72, 0x6e, 0x28, 0x70, 0x61, 0x74, 0x74, + 0x65, 0x72, 0x6e, 0x2c, 0x20, 0x6e, 0x61, 0x6d, 0x65, 0x29, 0x20, 0x7b, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, 0x21, 0x70, 0x61, + 0x74, 0x74, 0x65, 0x72, 0x6e, 0x2e, 0x73, 0x74, 0x61, 0x72, 0x74, 0x73, + 0x57, 0x69, 0x74, 0x68, 0x28, 0x27, 0x5e, 0x27, 0x29, 0x20, 0x7c, 0x7c, + 0x20, 0x21, 0x70, 0x61, 0x74, 0x74, 0x65, 0x72, 0x6e, 0x2e, 0x65, 0x6e, + 0x64, 0x73, 0x57, 0x69, 0x74, 0x68, 0x28, 0x27, 0x24, 0x27, 0x29, 0x29, + 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x74, 0x68, 0x72, + 0x6f, 0x77, 0x20, 0x6e, 0x65, 0x77, 0x20, 0x45, 0x72, 0x72, 0x6f, 0x72, + 0x28, 0x27, 0x50, 0x61, 0x74, 0x74, 0x65, 0x72, 0x6e, 0x20, 0x6d, 0x75, + 0x73, 0x74, 0x20, 0x73, 0x74, 0x61, 0x72, 0x74, 0x20, 0x77, 0x69, 0x74, + 0x68, 0x20, 0x22, 0x5e, 0x22, 0x20, 0x61, 0x6e, 0x64, 0x20, 0x65, 0x6e, + 0x64, 0x20, 0x77, 0x69, 0x74, 0x68, 0x20, 0x22, 0x24, 0x22, 0x27, 0x29, + 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x70, 0x61, 0x74, 0x74, 0x65, 0x72, 0x6e, 0x20, 0x3d, 0x20, 0x70, 0x61, + 0x74, 0x74, 0x65, 0x72, 0x6e, 0x2e, 0x73, 0x6c, 0x69, 0x63, 0x65, 0x28, + 0x31, 0x2c, 0x20, 0x2d, 0x31, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x73, 0x75, 0x62, 0x52, 0x75, 0x6c, + 0x65, 0x49, 0x64, 0x73, 0x20, 0x3d, 0x20, 0x7b, 0x7d, 0x3b, 0x0a, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x6c, 0x65, 0x74, 0x20, 0x69, 0x20, 0x3d, 0x20, + 0x30, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, + 0x20, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x20, 0x3d, 0x20, 0x70, 0x61, + 0x74, 0x74, 0x65, 0x72, 0x6e, 0x2e, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, + 0x3b, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, + 0x20, 0x67, 0x65, 0x74, 0x44, 0x6f, 0x74, 0x20, 0x3d, 0x20, 0x28, 0x29, + 0x20, 0x3d, 0x3e, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x6c, 0x65, 0x74, 0x20, 0x72, 0x75, 0x6c, 0x65, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, 0x74, 0x68, 0x69, 0x73, - 0x2e, 0x5f, 0x72, 0x75, 0x6c, 0x65, 0x73, 0x2e, 0x67, 0x65, 0x74, 0x28, - 0x65, 0x73, 0x63, 0x4e, 0x61, 0x6d, 0x65, 0x29, 0x20, 0x3d, 0x3d, 0x3d, - 0x20, 0x72, 0x75, 0x6c, 0x65, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, + 0x2e, 0x5f, 0x64, 0x6f, 0x74, 0x61, 0x6c, 0x6c, 0x29, 0x20, 0x7b, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, 0x75, 0x6c, 0x65, + 0x20, 0x3d, 0x20, 0x27, 0x5b, 0x5c, 0x5c, 0x55, 0x30, 0x30, 0x30, 0x30, + 0x30, 0x30, 0x30, 0x30, 0x2d, 0x5c, 0x5c, 0x55, 0x30, 0x30, 0x31, 0x30, + 0x46, 0x46, 0x46, 0x46, 0x5d, 0x27, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x7d, 0x20, 0x65, 0x6c, 0x73, 0x65, 0x20, 0x7b, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x2f, 0x2f, 0x20, 0x41, 0x63, + 0x63, 0x65, 0x70, 0x74, 0x20, 0x61, 0x6e, 0x79, 0x20, 0x63, 0x68, 0x61, + 0x72, 0x61, 0x63, 0x74, 0x65, 0x72, 0x2e, 0x2e, 0x2e, 0x20, 0x65, 0x78, + 0x63, 0x65, 0x70, 0x74, 0x20, 0x5c, 0x6e, 0x20, 0x61, 0x6e, 0x64, 0x20, + 0x5c, 0x72, 0x20, 0x6c, 0x69, 0x6e, 0x65, 0x20, 0x62, 0x72, 0x65, 0x61, + 0x6b, 0x20, 0x63, 0x68, 0x61, 0x72, 0x73, 0x20, 0x28, 0x5c, 0x78, 0x30, + 0x41, 0x20, 0x61, 0x6e, 0x64, 0x20, 0x5c, 0x78, 0x4f, 0x44, 0x29, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, 0x75, 0x6c, 0x65, + 0x20, 0x3d, 0x20, 0x27, 0x5b, 0x5c, 0x5c, 0x55, 0x30, 0x30, 0x30, 0x30, + 0x30, 0x30, 0x30, 0x30, 0x2d, 0x5c, 0x5c, 0x78, 0x30, 0x39, 0x5c, 0x5c, + 0x78, 0x30, 0x42, 0x5c, 0x5c, 0x78, 0x30, 0x43, 0x5c, 0x5c, 0x78, 0x30, + 0x45, 0x2d, 0x5c, 0x5c, 0x55, 0x30, 0x30, 0x31, 0x30, 0x46, 0x46, 0x46, + 0x46, 0x5d, 0x27, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, + 0x6e, 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x5f, 0x61, 0x64, 0x64, 0x52, + 0x75, 0x6c, 0x65, 0x28, 0x27, 0x64, 0x6f, 0x74, 0x27, 0x2c, 0x20, 0x72, + 0x75, 0x6c, 0x65, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x3b, + 0x0a, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, + 0x20, 0x74, 0x6f, 0x52, 0x75, 0x6c, 0x65, 0x20, 0x3d, 0x20, 0x28, 0x5b, + 0x73, 0x2c, 0x20, 0x69, 0x73, 0x4c, 0x69, 0x74, 0x65, 0x72, 0x61, 0x6c, + 0x5d, 0x29, 0x20, 0x3d, 0x3e, 0x20, 0x69, 0x73, 0x4c, 0x69, 0x74, 0x65, + 0x72, 0x61, 0x6c, 0x20, 0x3f, 0x20, 0x22, 0x5c, 0x22, 0x22, 0x20, 0x2b, + 0x20, 0x73, 0x20, 0x2b, 0x20, 0x22, 0x5c, 0x22, 0x22, 0x20, 0x3a, 0x20, + 0x73, 0x3b, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, + 0x74, 0x20, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x20, + 0x3d, 0x20, 0x28, 0x29, 0x20, 0x3d, 0x3e, 0x20, 0x7b, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x73, 0x74, + 0x61, 0x72, 0x74, 0x20, 0x3d, 0x20, 0x69, 0x3b, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x2f, 0x2f, 0x20, 0x46, 0x6f, 0x72, 0x20, 0x65, 0x61, + 0x63, 0x68, 0x20, 0x63, 0x6f, 0x6d, 0x70, 0x6f, 0x6e, 0x65, 0x6e, 0x74, + 0x20, 0x6f, 0x66, 0x20, 0x74, 0x68, 0x69, 0x73, 0x20, 0x73, 0x65, 0x71, + 0x75, 0x65, 0x6e, 0x63, 0x65, 0x2c, 0x20, 0x73, 0x74, 0x6f, 0x72, 0x65, + 0x20, 0x69, 0x74, 0x73, 0x20, 0x73, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x20, + 0x72, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x74, 0x61, 0x74, 0x69, + 0x6f, 0x6e, 0x20, 0x61, 0x6e, 0x64, 0x20, 0x77, 0x68, 0x65, 0x74, 0x68, + 0x65, 0x72, 0x20, 0x69, 0x74, 0x27, 0x73, 0x20, 0x61, 0x20, 0x6c, 0x69, + 0x74, 0x65, 0x72, 0x61, 0x6c, 0x2e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x2f, 0x2f, 0x20, 0x57, 0x65, 0x20, 0x6f, 0x6e, 0x6c, 0x79, 0x20, + 0x6e, 0x65, 0x65, 0x64, 0x20, 0x61, 0x20, 0x66, 0x6c, 0x61, 0x74, 0x20, + 0x73, 0x74, 0x72, 0x75, 0x63, 0x74, 0x75, 0x72, 0x65, 0x20, 0x68, 0x65, + 0x72, 0x65, 0x20, 0x74, 0x6f, 0x20, 0x61, 0x70, 0x70, 0x6c, 0x79, 0x20, + 0x72, 0x65, 0x70, 0x65, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x6f, + 0x70, 0x65, 0x72, 0x61, 0x74, 0x6f, 0x72, 0x73, 0x20, 0x74, 0x6f, 0x20, + 0x74, 0x68, 0x65, 0x20, 0x6c, 0x61, 0x73, 0x74, 0x20, 0x69, 0x74, 0x65, + 0x6d, 0x2c, 0x20, 0x61, 0x6e, 0x64, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x2f, 0x2f, 0x20, 0x74, 0x6f, 0x20, 0x6d, 0x65, 0x72, 0x67, 0x65, + 0x20, 0x6c, 0x69, 0x74, 0x65, 0x72, 0x61, 0x6c, 0x73, 0x20, 0x61, 0x74, + 0x20, 0x74, 0x68, 0x65, 0x20, 0x61, 0x6e, 0x64, 0x20, 0x28, 0x77, 0x65, + 0x27, 0x72, 0x65, 0x20, 0x70, 0x61, 0x72, 0x73, 0x69, 0x6e, 0x67, 0x20, + 0x67, 0x72, 0x6f, 0x75, 0x70, 0x65, 0x64, 0x20, 0x28, 0x20, 0x73, 0x65, + 0x71, 0x75, 0x65, 0x6e, 0x63, 0x65, 0x73, 0x20, 0x29, 0x20, 0x72, 0x65, + 0x63, 0x75, 0x72, 0x73, 0x69, 0x76, 0x65, 0x6c, 0x79, 0x20, 0x61, 0x6e, + 0x64, 0x20, 0x64, 0x6f, 0x6e, 0x27, 0x74, 0x20, 0x74, 0x72, 0x65, 0x61, + 0x74, 0x20, 0x27, 0x7c, 0x27, 0x20, 0x73, 0x70, 0x65, 0x63, 0x69, 0x61, + 0x6c, 0x6c, 0x79, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x2f, 0x2f, + 0x20, 0x28, 0x47, 0x42, 0x4e, 0x46, 0x27, 0x73, 0x20, 0x73, 0x79, 0x6e, + 0x74, 0x61, 0x78, 0x20, 0x69, 0x73, 0x20, 0x6c, 0x75, 0x63, 0x6b, 0x69, + 0x6c, 0x79, 0x20, 0x76, 0x65, 0x72, 0x79, 0x20, 0x63, 0x6c, 0x6f, 0x73, + 0x65, 0x20, 0x74, 0x6f, 0x20, 0x72, 0x65, 0x67, 0x75, 0x6c, 0x61, 0x72, + 0x20, 0x65, 0x78, 0x70, 0x72, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x73, + 0x21, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, + 0x73, 0x74, 0x20, 0x73, 0x65, 0x71, 0x20, 0x3d, 0x20, 0x5b, 0x5d, 0x3b, + 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, + 0x74, 0x20, 0x6a, 0x6f, 0x69, 0x6e, 0x53, 0x65, 0x71, 0x20, 0x3d, 0x20, + 0x28, 0x29, 0x20, 0x3d, 0x3e, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x72, 0x65, + 0x74, 0x20, 0x3d, 0x20, 0x5b, 0x5d, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x66, 0x6f, 0x72, 0x20, 0x28, 0x63, 0x6f, 0x6e, + 0x73, 0x74, 0x20, 0x5b, 0x69, 0x73, 0x4c, 0x69, 0x74, 0x65, 0x72, 0x61, + 0x6c, 0x2c, 0x20, 0x67, 0x5d, 0x20, 0x6f, 0x66, 0x20, 0x67, 0x72, 0x6f, + 0x75, 0x70, 0x42, 0x79, 0x28, 0x73, 0x65, 0x71, 0x2c, 0x20, 0x78, 0x20, + 0x3d, 0x3e, 0x20, 0x78, 0x5b, 0x31, 0x5d, 0x29, 0x29, 0x20, 0x7b, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, + 0x20, 0x28, 0x69, 0x73, 0x4c, 0x69, 0x74, 0x65, 0x72, 0x61, 0x6c, 0x29, + 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x72, 0x65, 0x74, 0x2e, 0x70, 0x75, 0x73, 0x68, 0x28, + 0x5b, 0x5b, 0x2e, 0x2e, 0x2e, 0x67, 0x5d, 0x2e, 0x6d, 0x61, 0x70, 0x28, + 0x78, 0x20, 0x3d, 0x3e, 0x20, 0x78, 0x5b, 0x30, 0x5d, 0x29, 0x2e, 0x6a, + 0x6f, 0x69, 0x6e, 0x28, 0x27, 0x27, 0x29, 0x2c, 0x20, 0x74, 0x72, 0x75, + 0x65, 0x5d, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x7d, 0x20, 0x65, 0x6c, 0x73, 0x65, 0x20, 0x7b, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x72, 0x65, 0x74, 0x2e, 0x70, 0x75, 0x73, 0x68, 0x28, 0x2e, 0x2e, 0x2e, + 0x67, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, + 0x20, 0x28, 0x72, 0x65, 0x74, 0x2e, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, + 0x20, 0x3d, 0x3d, 0x3d, 0x20, 0x31, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, + 0x72, 0x6e, 0x20, 0x72, 0x65, 0x74, 0x5b, 0x30, 0x5d, 0x3b, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, - 0x6b, 0x65, 0x79, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, - 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x6c, 0x65, 0x74, 0x20, - 0x69, 0x20, 0x3d, 0x20, 0x30, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x77, 0x68, 0x69, 0x6c, 0x65, 0x20, 0x28, 0x74, 0x68, 0x69, 0x73, - 0x2e, 0x5f, 0x72, 0x75, 0x6c, 0x65, 0x73, 0x2e, 0x68, 0x61, 0x73, 0x28, - 0x60, 0x24, 0x7b, 0x65, 0x73, 0x63, 0x4e, 0x61, 0x6d, 0x65, 0x7d, 0x24, - 0x7b, 0x69, 0x7d, 0x60, 0x29, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, + 0x5b, 0x72, 0x65, 0x74, 0x2e, 0x6d, 0x61, 0x70, 0x28, 0x78, 0x20, 0x3d, + 0x3e, 0x20, 0x74, 0x6f, 0x52, 0x75, 0x6c, 0x65, 0x28, 0x78, 0x29, 0x29, + 0x2e, 0x6a, 0x6f, 0x69, 0x6e, 0x28, 0x27, 0x20, 0x27, 0x29, 0x2c, 0x20, + 0x66, 0x61, 0x6c, 0x73, 0x65, 0x5d, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x7d, 0x3b, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x77, 0x68, 0x69, 0x6c, 0x65, 0x20, 0x28, 0x69, 0x20, 0x3c, 0x20, 0x6c, + 0x65, 0x6e, 0x67, 0x74, 0x68, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x63, + 0x20, 0x3d, 0x20, 0x70, 0x61, 0x74, 0x74, 0x65, 0x72, 0x6e, 0x5b, 0x69, + 0x5d, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, + 0x66, 0x20, 0x28, 0x63, 0x20, 0x3d, 0x3d, 0x3d, 0x20, 0x27, 0x2e, 0x27, + 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x73, 0x65, 0x71, 0x2e, 0x70, 0x75, 0x73, 0x68, 0x28, 0x5b, + 0x67, 0x65, 0x74, 0x44, 0x6f, 0x74, 0x28, 0x29, 0x2c, 0x20, 0x66, 0x61, + 0x6c, 0x73, 0x65, 0x5d, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x20, 0x2b, 0x3d, 0x20, 0x31, 0x3b, - 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x6b, 0x65, 0x79, 0x20, 0x3d, 0x20, 0x60, 0x24, 0x7b, - 0x65, 0x73, 0x63, 0x4e, 0x61, 0x6d, 0x65, 0x7d, 0x24, 0x7b, 0x69, 0x7d, - 0x60, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x0a, 0x20, 0x20, - 0x20, 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x5f, 0x72, 0x75, 0x6c, 0x65, - 0x73, 0x2e, 0x73, 0x65, 0x74, 0x28, 0x6b, 0x65, 0x79, 0x2c, 0x20, 0x72, - 0x75, 0x6c, 0x65, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, - 0x74, 0x75, 0x72, 0x6e, 0x20, 0x6b, 0x65, 0x79, 0x3b, 0x0a, 0x20, 0x20, - 0x7d, 0x0a, 0x0a, 0x20, 0x20, 0x76, 0x69, 0x73, 0x69, 0x74, 0x28, 0x73, - 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2c, 0x20, 0x6e, 0x61, 0x6d, 0x65, 0x29, - 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, - 0x20, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x54, 0x79, 0x70, 0x65, 0x20, - 0x3d, 0x20, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2e, 0x74, 0x79, 0x70, - 0x65, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, - 0x20, 0x72, 0x75, 0x6c, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x20, 0x3d, 0x20, - 0x6e, 0x61, 0x6d, 0x65, 0x20, 0x7c, 0x7c, 0x20, 0x27, 0x72, 0x6f, 0x6f, - 0x74, 0x27, 0x3b, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, - 0x28, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2e, 0x6f, 0x6e, 0x65, 0x4f, - 0x66, 0x20, 0x7c, 0x7c, 0x20, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2e, - 0x61, 0x6e, 0x79, 0x4f, 0x66, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x72, 0x75, 0x6c, - 0x65, 0x20, 0x3d, 0x20, 0x28, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2e, - 0x6f, 0x6e, 0x65, 0x4f, 0x66, 0x20, 0x7c, 0x7c, 0x20, 0x73, 0x63, 0x68, - 0x65, 0x6d, 0x61, 0x2e, 0x61, 0x6e, 0x79, 0x4f, 0x66, 0x29, 0x2e, 0x6d, - 0x61, 0x70, 0x28, 0x28, 0x61, 0x6c, 0x74, 0x53, 0x63, 0x68, 0x65, 0x6d, - 0x61, 0x2c, 0x20, 0x69, 0x29, 0x20, 0x3d, 0x3e, 0x0a, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x76, 0x69, - 0x73, 0x69, 0x74, 0x28, 0x61, 0x6c, 0x74, 0x53, 0x63, 0x68, 0x65, 0x6d, - 0x61, 0x2c, 0x20, 0x60, 0x24, 0x7b, 0x6e, 0x61, 0x6d, 0x65, 0x7d, 0x24, - 0x7b, 0x6e, 0x61, 0x6d, 0x65, 0x20, 0x3f, 0x20, 0x22, 0x2d, 0x22, 0x20, - 0x3a, 0x20, 0x22, 0x22, 0x7d, 0x24, 0x7b, 0x69, 0x7d, 0x60, 0x29, 0x0a, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x29, 0x2e, 0x6a, 0x6f, 0x69, 0x6e, - 0x28, 0x27, 0x20, 0x7c, 0x20, 0x27, 0x29, 0x3b, 0x0a, 0x0a, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x74, - 0x68, 0x69, 0x73, 0x2e, 0x5f, 0x61, 0x64, 0x64, 0x52, 0x75, 0x6c, 0x65, - 0x28, 0x72, 0x75, 0x6c, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x2c, 0x20, 0x72, - 0x75, 0x6c, 0x65, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x20, - 0x65, 0x6c, 0x73, 0x65, 0x20, 0x69, 0x66, 0x20, 0x28, 0x27, 0x63, 0x6f, - 0x6e, 0x73, 0x74, 0x27, 0x20, 0x69, 0x6e, 0x20, 0x73, 0x63, 0x68, 0x65, - 0x6d, 0x61, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x20, 0x65, + 0x6c, 0x73, 0x65, 0x20, 0x69, 0x66, 0x20, 0x28, 0x63, 0x20, 0x3d, 0x3d, + 0x3d, 0x20, 0x27, 0x28, 0x27, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x20, 0x2b, 0x3d, 0x20, + 0x31, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x69, 0x66, 0x20, 0x28, 0x69, 0x20, 0x3c, 0x20, 0x6c, 0x65, 0x6e, + 0x67, 0x74, 0x68, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, 0x70, + 0x61, 0x74, 0x74, 0x65, 0x72, 0x6e, 0x5b, 0x69, 0x5d, 0x20, 0x3d, 0x3d, + 0x3d, 0x20, 0x27, 0x3f, 0x27, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x74, + 0x68, 0x72, 0x6f, 0x77, 0x20, 0x6e, 0x65, 0x77, 0x20, 0x45, 0x72, 0x72, + 0x6f, 0x72, 0x28, 0x60, 0x55, 0x6e, 0x73, 0x75, 0x70, 0x70, 0x6f, 0x72, + 0x74, 0x65, 0x64, 0x20, 0x70, 0x61, 0x74, 0x74, 0x65, 0x72, 0x6e, 0x20, + 0x73, 0x79, 0x6e, 0x74, 0x61, 0x78, 0x20, 0x22, 0x24, 0x7b, 0x70, 0x61, + 0x74, 0x74, 0x65, 0x72, 0x6e, 0x5b, 0x69, 0x5d, 0x7d, 0x22, 0x20, 0x61, + 0x74, 0x20, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x20, 0x24, 0x7b, 0x69, 0x7d, + 0x20, 0x6f, 0x66, 0x20, 0x2f, 0x24, 0x7b, 0x70, 0x61, 0x74, 0x74, 0x65, + 0x72, 0x6e, 0x7d, 0x2f, 0x60, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x73, 0x65, 0x71, 0x2e, + 0x70, 0x75, 0x73, 0x68, 0x28, 0x5b, 0x60, 0x28, 0x24, 0x7b, 0x74, 0x6f, + 0x52, 0x75, 0x6c, 0x65, 0x28, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, + 0x72, 0x6d, 0x28, 0x29, 0x29, 0x7d, 0x29, 0x60, 0x2c, 0x20, 0x66, 0x61, + 0x6c, 0x73, 0x65, 0x5d, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x7d, 0x20, 0x65, 0x6c, 0x73, 0x65, 0x20, 0x69, 0x66, + 0x20, 0x28, 0x63, 0x20, 0x3d, 0x3d, 0x3d, 0x20, 0x27, 0x29, 0x27, 0x29, + 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x69, 0x20, 0x2b, 0x3d, 0x20, 0x31, 0x3b, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, 0x73, + 0x74, 0x61, 0x72, 0x74, 0x20, 0x3c, 0x3d, 0x20, 0x30, 0x20, 0x7c, 0x7c, + 0x20, 0x70, 0x61, 0x74, 0x74, 0x65, 0x72, 0x6e, 0x5b, 0x73, 0x74, 0x61, + 0x72, 0x74, 0x20, 0x2d, 0x20, 0x31, 0x5d, 0x20, 0x21, 0x3d, 0x3d, 0x20, + 0x27, 0x28, 0x27, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x74, 0x68, 0x72, 0x6f, 0x77, + 0x20, 0x6e, 0x65, 0x77, 0x20, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x28, 0x60, + 0x55, 0x6e, 0x62, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, 0x64, 0x20, 0x70, + 0x61, 0x72, 0x65, 0x6e, 0x74, 0x68, 0x65, 0x73, 0x65, 0x73, 0x3b, 0x20, + 0x73, 0x74, 0x61, 0x72, 0x74, 0x20, 0x3d, 0x20, 0x24, 0x7b, 0x73, 0x74, + 0x61, 0x72, 0x74, 0x7d, 0x2c, 0x20, 0x69, 0x20, 0x3d, 0x20, 0x24, 0x7b, + 0x69, 0x7d, 0x2c, 0x20, 0x70, 0x61, 0x74, 0x74, 0x65, 0x72, 0x6e, 0x20, + 0x3d, 0x20, 0x24, 0x7b, 0x70, 0x61, 0x74, 0x74, 0x65, 0x72, 0x6e, 0x7d, + 0x60, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x6a, 0x6f, 0x69, + 0x6e, 0x53, 0x65, 0x71, 0x28, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x7d, 0x20, 0x65, 0x6c, 0x73, 0x65, 0x20, 0x69, + 0x66, 0x20, 0x28, 0x63, 0x20, 0x3d, 0x3d, 0x3d, 0x20, 0x27, 0x5b, 0x27, + 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x6c, 0x65, 0x74, 0x20, 0x73, 0x71, 0x75, 0x61, 0x72, 0x65, + 0x42, 0x72, 0x61, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x20, 0x3d, 0x20, 0x63, + 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x69, 0x20, 0x2b, 0x3d, 0x20, 0x31, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x77, 0x68, 0x69, 0x6c, 0x65, 0x20, + 0x28, 0x69, 0x20, 0x3c, 0x20, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x20, + 0x26, 0x26, 0x20, 0x70, 0x61, 0x74, 0x74, 0x65, 0x72, 0x6e, 0x5b, 0x69, + 0x5d, 0x20, 0x21, 0x3d, 0x3d, 0x20, 0x27, 0x5d, 0x27, 0x29, 0x20, 0x7b, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x69, 0x66, 0x20, 0x28, 0x70, 0x61, 0x74, 0x74, 0x65, 0x72, 0x6e, + 0x5b, 0x69, 0x5d, 0x20, 0x3d, 0x3d, 0x3d, 0x20, 0x27, 0x5c, 0x5c, 0x27, + 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x73, 0x71, 0x75, 0x61, 0x72, 0x65, + 0x42, 0x72, 0x61, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x20, 0x2b, 0x3d, 0x20, + 0x70, 0x61, 0x74, 0x74, 0x65, 0x72, 0x6e, 0x2e, 0x73, 0x6c, 0x69, 0x63, + 0x65, 0x28, 0x69, 0x2c, 0x20, 0x69, 0x20, 0x2b, 0x20, 0x32, 0x29, 0x3b, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x69, 0x20, 0x2b, 0x3d, 0x20, 0x32, 0x3b, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, + 0x20, 0x65, 0x6c, 0x73, 0x65, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x73, 0x71, + 0x75, 0x61, 0x72, 0x65, 0x42, 0x72, 0x61, 0x63, 0x6b, 0x65, 0x74, 0x73, + 0x20, 0x2b, 0x3d, 0x20, 0x70, 0x61, 0x74, 0x74, 0x65, 0x72, 0x6e, 0x5b, + 0x69, 0x5d, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x20, 0x2b, 0x3d, 0x20, 0x31, + 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, 0x69, 0x20, 0x3e, 0x3d, 0x20, 0x6c, + 0x65, 0x6e, 0x67, 0x74, 0x68, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x74, 0x68, 0x72, + 0x6f, 0x77, 0x20, 0x6e, 0x65, 0x77, 0x20, 0x45, 0x72, 0x72, 0x6f, 0x72, + 0x28, 0x60, 0x55, 0x6e, 0x62, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, 0x64, + 0x20, 0x73, 0x71, 0x75, 0x61, 0x72, 0x65, 0x20, 0x62, 0x72, 0x61, 0x63, + 0x6b, 0x65, 0x74, 0x73, 0x3b, 0x20, 0x73, 0x74, 0x61, 0x72, 0x74, 0x20, + 0x3d, 0x20, 0x24, 0x7b, 0x73, 0x74, 0x61, 0x72, 0x74, 0x7d, 0x2c, 0x20, + 0x69, 0x20, 0x3d, 0x20, 0x24, 0x7b, 0x69, 0x7d, 0x2c, 0x20, 0x70, 0x61, + 0x74, 0x74, 0x65, 0x72, 0x6e, 0x20, 0x3d, 0x20, 0x24, 0x7b, 0x70, 0x61, + 0x74, 0x74, 0x65, 0x72, 0x6e, 0x7d, 0x60, 0x29, 0x3b, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x73, 0x71, 0x75, 0x61, + 0x72, 0x65, 0x42, 0x72, 0x61, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x20, 0x2b, + 0x3d, 0x20, 0x27, 0x5d, 0x27, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x20, 0x2b, 0x3d, 0x20, 0x31, 0x3b, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x73, + 0x65, 0x71, 0x2e, 0x70, 0x75, 0x73, 0x68, 0x28, 0x5b, 0x73, 0x71, 0x75, + 0x61, 0x72, 0x65, 0x42, 0x72, 0x61, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x2c, + 0x20, 0x66, 0x61, 0x6c, 0x73, 0x65, 0x5d, 0x29, 0x3b, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x20, 0x65, 0x6c, 0x73, 0x65, + 0x20, 0x69, 0x66, 0x20, 0x28, 0x63, 0x20, 0x3d, 0x3d, 0x3d, 0x20, 0x27, + 0x7c, 0x27, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x73, 0x65, 0x71, 0x2e, 0x70, 0x75, 0x73, 0x68, + 0x28, 0x5b, 0x27, 0x7c, 0x27, 0x2c, 0x20, 0x66, 0x61, 0x6c, 0x73, 0x65, + 0x5d, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x69, 0x20, 0x2b, 0x3d, 0x20, 0x31, 0x3b, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x20, 0x65, 0x6c, 0x73, 0x65, + 0x20, 0x69, 0x66, 0x20, 0x28, 0x63, 0x20, 0x3d, 0x3d, 0x3d, 0x20, 0x27, + 0x2a, 0x27, 0x20, 0x7c, 0x7c, 0x20, 0x63, 0x20, 0x3d, 0x3d, 0x3d, 0x20, + 0x27, 0x2b, 0x27, 0x20, 0x7c, 0x7c, 0x20, 0x63, 0x20, 0x3d, 0x3d, 0x3d, + 0x20, 0x27, 0x3f, 0x27, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x73, 0x65, 0x71, 0x5b, 0x73, 0x65, + 0x71, 0x2e, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x20, 0x2d, 0x20, 0x31, + 0x5d, 0x20, 0x3d, 0x20, 0x5b, 0x74, 0x6f, 0x52, 0x75, 0x6c, 0x65, 0x28, + 0x73, 0x65, 0x71, 0x5b, 0x73, 0x65, 0x71, 0x2e, 0x6c, 0x65, 0x6e, 0x67, + 0x74, 0x68, 0x20, 0x2d, 0x20, 0x31, 0x5d, 0x29, 0x20, 0x2b, 0x20, 0x63, + 0x2c, 0x20, 0x66, 0x61, 0x6c, 0x73, 0x65, 0x5d, 0x3b, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x20, 0x2b, 0x3d, + 0x20, 0x31, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x7d, 0x20, 0x65, 0x6c, 0x73, 0x65, 0x20, 0x69, 0x66, 0x20, 0x28, 0x63, + 0x20, 0x3d, 0x3d, 0x3d, 0x20, 0x27, 0x7b, 0x27, 0x29, 0x20, 0x7b, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x6c, 0x65, + 0x74, 0x20, 0x63, 0x75, 0x72, 0x6c, 0x79, 0x42, 0x72, 0x61, 0x63, 0x6b, + 0x65, 0x74, 0x73, 0x20, 0x3d, 0x20, 0x63, 0x3b, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x20, 0x2b, 0x3d, 0x20, + 0x31, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x77, 0x68, 0x69, 0x6c, 0x65, 0x20, 0x28, 0x69, 0x20, 0x3c, 0x20, + 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x20, 0x26, 0x26, 0x20, 0x70, 0x61, + 0x74, 0x74, 0x65, 0x72, 0x6e, 0x5b, 0x69, 0x5d, 0x20, 0x21, 0x3d, 0x3d, + 0x20, 0x27, 0x7d, 0x27, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x75, 0x72, 0x6c, + 0x79, 0x42, 0x72, 0x61, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x20, 0x2b, 0x3d, + 0x20, 0x70, 0x61, 0x74, 0x74, 0x65, 0x72, 0x6e, 0x5b, 0x69, 0x5d, 0x3b, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x69, 0x20, 0x2b, 0x3d, 0x20, 0x31, 0x3b, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, 0x69, + 0x20, 0x3e, 0x3d, 0x20, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x29, 0x20, + 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x74, 0x68, 0x72, 0x6f, 0x77, 0x20, 0x6e, 0x65, 0x77, 0x20, + 0x45, 0x72, 0x72, 0x6f, 0x72, 0x28, 0x60, 0x55, 0x6e, 0x62, 0x61, 0x6c, + 0x61, 0x6e, 0x63, 0x65, 0x64, 0x20, 0x63, 0x75, 0x72, 0x6c, 0x79, 0x20, + 0x62, 0x72, 0x61, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x3b, 0x20, 0x73, 0x74, + 0x61, 0x72, 0x74, 0x20, 0x3d, 0x20, 0x24, 0x7b, 0x73, 0x74, 0x61, 0x72, + 0x74, 0x7d, 0x2c, 0x20, 0x69, 0x20, 0x3d, 0x20, 0x24, 0x7b, 0x69, 0x7d, + 0x2c, 0x20, 0x70, 0x61, 0x74, 0x74, 0x65, 0x72, 0x6e, 0x20, 0x3d, 0x20, + 0x24, 0x7b, 0x70, 0x61, 0x74, 0x74, 0x65, 0x72, 0x6e, 0x7d, 0x60, 0x29, + 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x63, 0x75, 0x72, 0x6c, 0x79, 0x42, 0x72, 0x61, 0x63, 0x6b, 0x65, 0x74, + 0x73, 0x20, 0x2b, 0x3d, 0x20, 0x27, 0x7d, 0x27, 0x3b, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x20, 0x2b, 0x3d, + 0x20, 0x31, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x6e, 0x75, 0x6d, 0x73, + 0x20, 0x3d, 0x20, 0x63, 0x75, 0x72, 0x6c, 0x79, 0x42, 0x72, 0x61, 0x63, + 0x6b, 0x65, 0x74, 0x73, 0x2e, 0x73, 0x6c, 0x69, 0x63, 0x65, 0x28, 0x31, + 0x2c, 0x20, 0x2d, 0x31, 0x29, 0x2e, 0x73, 0x70, 0x6c, 0x69, 0x74, 0x28, + 0x27, 0x2c, 0x27, 0x29, 0x2e, 0x6d, 0x61, 0x70, 0x28, 0x73, 0x20, 0x3d, + 0x3e, 0x20, 0x73, 0x2e, 0x74, 0x72, 0x69, 0x6d, 0x28, 0x29, 0x29, 0x3b, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x6c, + 0x65, 0x74, 0x20, 0x6d, 0x69, 0x6e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x2c, + 0x20, 0x6d, 0x61, 0x78, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x3b, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, + 0x28, 0x6e, 0x75, 0x6d, 0x73, 0x2e, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, + 0x20, 0x3d, 0x3d, 0x3d, 0x20, 0x31, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x6d, 0x69, + 0x6e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x20, 0x3d, 0x20, 0x70, 0x61, 0x72, + 0x73, 0x65, 0x49, 0x6e, 0x74, 0x28, 0x6e, 0x75, 0x6d, 0x73, 0x5b, 0x30, + 0x5d, 0x2c, 0x20, 0x31, 0x30, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x6d, 0x61, 0x78, 0x54, + 0x69, 0x6d, 0x65, 0x73, 0x20, 0x3d, 0x20, 0x6d, 0x69, 0x6e, 0x54, 0x69, + 0x6d, 0x65, 0x73, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x7d, 0x20, 0x65, 0x6c, 0x73, 0x65, 0x20, 0x7b, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x69, 0x66, 0x20, 0x28, 0x6e, 0x75, 0x6d, 0x73, 0x2e, 0x6c, 0x65, 0x6e, + 0x67, 0x74, 0x68, 0x20, 0x21, 0x3d, 0x3d, 0x20, 0x32, 0x29, 0x20, 0x7b, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x74, 0x68, 0x72, 0x6f, 0x77, 0x20, 0x6e, 0x65, 0x77, + 0x20, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x28, 0x60, 0x49, 0x6e, 0x76, 0x61, + 0x6c, 0x69, 0x64, 0x20, 0x71, 0x75, 0x61, 0x6e, 0x74, 0x69, 0x66, 0x69, + 0x65, 0x72, 0x20, 0x24, 0x7b, 0x63, 0x75, 0x72, 0x6c, 0x79, 0x42, 0x72, + 0x61, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x7d, 0x60, 0x29, 0x3b, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x6d, 0x69, 0x6e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x20, 0x3d, 0x20, + 0x6e, 0x75, 0x6d, 0x73, 0x5b, 0x30, 0x5d, 0x20, 0x3f, 0x20, 0x70, 0x61, + 0x72, 0x73, 0x65, 0x49, 0x6e, 0x74, 0x28, 0x6e, 0x75, 0x6d, 0x73, 0x5b, + 0x30, 0x5d, 0x2c, 0x20, 0x31, 0x30, 0x29, 0x20, 0x3a, 0x20, 0x30, 0x3b, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x6d, 0x61, 0x78, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x20, 0x3d, 0x20, + 0x6e, 0x75, 0x6d, 0x73, 0x5b, 0x31, 0x5d, 0x20, 0x3f, 0x20, 0x70, 0x61, + 0x72, 0x73, 0x65, 0x49, 0x6e, 0x74, 0x28, 0x6e, 0x75, 0x6d, 0x73, 0x5b, + 0x31, 0x5d, 0x2c, 0x20, 0x31, 0x30, 0x29, 0x20, 0x3a, 0x20, 0x49, 0x6e, + 0x66, 0x69, 0x6e, 0x69, 0x74, 0x79, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x6c, 0x65, 0x74, 0x20, 0x5b, + 0x73, 0x75, 0x62, 0x2c, 0x20, 0x73, 0x75, 0x62, 0x49, 0x73, 0x4c, 0x69, + 0x74, 0x65, 0x72, 0x61, 0x6c, 0x5d, 0x20, 0x3d, 0x20, 0x73, 0x65, 0x71, + 0x5b, 0x73, 0x65, 0x71, 0x2e, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x20, + 0x2d, 0x20, 0x31, 0x5d, 0x3b, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, 0x6d, 0x69, 0x6e, + 0x54, 0x69, 0x6d, 0x65, 0x73, 0x20, 0x3d, 0x3d, 0x3d, 0x20, 0x30, 0x20, + 0x26, 0x26, 0x20, 0x6d, 0x61, 0x78, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x20, + 0x3d, 0x3d, 0x3d, 0x20, 0x49, 0x6e, 0x66, 0x69, 0x6e, 0x69, 0x74, 0x79, + 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x73, 0x65, 0x71, 0x5b, 0x73, 0x65, 0x71, 0x2e, + 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x20, 0x2d, 0x20, 0x31, 0x5d, 0x20, + 0x3d, 0x20, 0x5b, 0x60, 0x24, 0x7b, 0x73, 0x75, 0x62, 0x7d, 0x2a, 0x60, + 0x2c, 0x20, 0x66, 0x61, 0x6c, 0x73, 0x65, 0x5d, 0x3b, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x20, 0x65, 0x6c, + 0x73, 0x65, 0x20, 0x69, 0x66, 0x20, 0x28, 0x6d, 0x69, 0x6e, 0x54, 0x69, + 0x6d, 0x65, 0x73, 0x20, 0x3d, 0x3d, 0x3d, 0x20, 0x30, 0x20, 0x26, 0x26, + 0x20, 0x6d, 0x61, 0x78, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x20, 0x3d, 0x3d, + 0x3d, 0x20, 0x31, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x73, 0x65, 0x71, 0x5b, 0x73, + 0x65, 0x71, 0x2e, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x20, 0x2d, 0x20, + 0x31, 0x5d, 0x20, 0x3d, 0x20, 0x5b, 0x60, 0x24, 0x7b, 0x73, 0x75, 0x62, + 0x7d, 0x3f, 0x60, 0x2c, 0x20, 0x66, 0x61, 0x6c, 0x73, 0x65, 0x5d, 0x3b, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, + 0x20, 0x65, 0x6c, 0x73, 0x65, 0x20, 0x69, 0x66, 0x20, 0x28, 0x6d, 0x69, + 0x6e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x20, 0x3d, 0x3d, 0x3d, 0x20, 0x31, + 0x20, 0x26, 0x26, 0x20, 0x6d, 0x61, 0x78, 0x54, 0x69, 0x6d, 0x65, 0x73, + 0x20, 0x3d, 0x3d, 0x3d, 0x20, 0x49, 0x6e, 0x66, 0x69, 0x6e, 0x69, 0x74, + 0x79, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x73, 0x65, 0x71, 0x5b, 0x73, 0x65, 0x71, + 0x2e, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x20, 0x2d, 0x20, 0x31, 0x5d, + 0x20, 0x3d, 0x20, 0x5b, 0x60, 0x24, 0x7b, 0x73, 0x75, 0x62, 0x7d, 0x2b, + 0x60, 0x2c, 0x20, 0x66, 0x61, 0x6c, 0x73, 0x65, 0x5d, 0x3b, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x20, 0x65, + 0x6c, 0x73, 0x65, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, 0x21, 0x73, + 0x75, 0x62, 0x49, 0x73, 0x4c, 0x69, 0x74, 0x65, 0x72, 0x61, 0x6c, 0x29, + 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x6c, 0x65, 0x74, 0x20, 0x69, 0x64, 0x20, + 0x3d, 0x20, 0x73, 0x75, 0x62, 0x52, 0x75, 0x6c, 0x65, 0x49, 0x64, 0x73, + 0x5b, 0x73, 0x75, 0x62, 0x5d, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, + 0x28, 0x69, 0x64, 0x20, 0x3d, 0x3d, 0x3d, 0x20, 0x75, 0x6e, 0x64, 0x65, + 0x66, 0x69, 0x6e, 0x65, 0x64, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x69, 0x64, 0x20, 0x3d, 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x5f, + 0x61, 0x64, 0x64, 0x52, 0x75, 0x6c, 0x65, 0x28, 0x60, 0x24, 0x7b, 0x6e, + 0x61, 0x6d, 0x65, 0x7d, 0x2d, 0x24, 0x7b, 0x4f, 0x62, 0x6a, 0x65, 0x63, + 0x74, 0x2e, 0x6b, 0x65, 0x79, 0x73, 0x28, 0x73, 0x75, 0x62, 0x52, 0x75, + 0x6c, 0x65, 0x49, 0x64, 0x73, 0x29, 0x2e, 0x6c, 0x65, 0x6e, 0x67, 0x74, + 0x68, 0x20, 0x2b, 0x20, 0x31, 0x7d, 0x60, 0x2c, 0x20, 0x73, 0x75, 0x62, + 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x73, 0x75, 0x62, 0x52, 0x75, + 0x6c, 0x65, 0x49, 0x64, 0x73, 0x5b, 0x73, 0x75, 0x62, 0x5d, 0x20, 0x3d, + 0x20, 0x69, 0x64, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x73, + 0x75, 0x62, 0x20, 0x3d, 0x20, 0x69, 0x64, 0x3b, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x72, 0x65, 0x70, 0x65, 0x61, 0x74, + 0x65, 0x64, 0x53, 0x75, 0x62, 0x20, 0x3d, 0x20, 0x41, 0x72, 0x72, 0x61, + 0x79, 0x2e, 0x66, 0x72, 0x6f, 0x6d, 0x28, 0x7b, 0x20, 0x6c, 0x65, 0x6e, + 0x67, 0x74, 0x68, 0x3a, 0x20, 0x6d, 0x69, 0x6e, 0x54, 0x69, 0x6d, 0x65, + 0x73, 0x20, 0x7d, 0x2c, 0x20, 0x28, 0x29, 0x20, 0x3d, 0x3e, 0x20, 0x73, + 0x75, 0x62, 0x49, 0x73, 0x4c, 0x69, 0x74, 0x65, 0x72, 0x61, 0x6c, 0x20, + 0x3f, 0x20, 0x60, 0x22, 0x24, 0x7b, 0x73, 0x75, 0x62, 0x2e, 0x73, 0x6c, + 0x69, 0x63, 0x65, 0x28, 0x31, 0x2c, 0x20, 0x2d, 0x31, 0x29, 0x2e, 0x72, + 0x65, 0x70, 0x65, 0x61, 0x74, 0x28, 0x6d, 0x69, 0x6e, 0x54, 0x69, 0x6d, + 0x65, 0x73, 0x29, 0x7d, 0x22, 0x60, 0x20, 0x3a, 0x20, 0x73, 0x75, 0x62, + 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x6f, 0x70, 0x74, + 0x69, 0x6f, 0x6e, 0x61, 0x6c, 0x53, 0x75, 0x62, 0x20, 0x3d, 0x20, 0x6d, + 0x61, 0x78, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x20, 0x21, 0x3d, 0x3d, 0x20, + 0x75, 0x6e, 0x64, 0x65, 0x66, 0x69, 0x6e, 0x65, 0x64, 0x20, 0x3f, 0x20, + 0x41, 0x72, 0x72, 0x61, 0x79, 0x2e, 0x66, 0x72, 0x6f, 0x6d, 0x28, 0x7b, + 0x20, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x3a, 0x20, 0x6d, 0x61, 0x78, + 0x54, 0x69, 0x6d, 0x65, 0x73, 0x20, 0x2d, 0x20, 0x6d, 0x69, 0x6e, 0x54, + 0x69, 0x6d, 0x65, 0x73, 0x20, 0x7d, 0x2c, 0x20, 0x28, 0x29, 0x20, 0x3d, + 0x3e, 0x20, 0x60, 0x24, 0x7b, 0x73, 0x75, 0x62, 0x7d, 0x3f, 0x60, 0x29, + 0x20, 0x3a, 0x20, 0x5b, 0x60, 0x24, 0x7b, 0x73, 0x75, 0x62, 0x7d, 0x2a, + 0x60, 0x5d, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x73, 0x65, 0x71, 0x5b, 0x73, 0x65, 0x71, 0x2e, + 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x20, 0x2d, 0x20, 0x31, 0x5d, 0x20, + 0x3d, 0x20, 0x5b, 0x72, 0x65, 0x70, 0x65, 0x61, 0x74, 0x65, 0x64, 0x53, + 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6e, 0x63, 0x61, 0x74, 0x28, 0x6f, 0x70, + 0x74, 0x69, 0x6f, 0x6e, 0x61, 0x6c, 0x53, 0x75, 0x62, 0x29, 0x2e, 0x6a, + 0x6f, 0x69, 0x6e, 0x28, 0x27, 0x20, 0x27, 0x29, 0x2c, 0x20, 0x66, 0x61, + 0x6c, 0x73, 0x65, 0x5d, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x7d, 0x20, 0x65, 0x6c, 0x73, 0x65, 0x20, 0x7b, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x6c, 0x65, 0x74, + 0x20, 0x6c, 0x69, 0x74, 0x65, 0x72, 0x61, 0x6c, 0x20, 0x3d, 0x20, 0x27, + 0x27, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x77, 0x68, 0x69, 0x6c, 0x65, 0x20, 0x28, 0x69, 0x20, 0x3c, 0x20, + 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, + 0x20, 0x28, 0x70, 0x61, 0x74, 0x74, 0x65, 0x72, 0x6e, 0x5b, 0x69, 0x5d, + 0x20, 0x3d, 0x3d, 0x3d, 0x20, 0x27, 0x5c, 0x5c, 0x27, 0x20, 0x26, 0x26, + 0x20, 0x69, 0x20, 0x3c, 0x20, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x20, + 0x2d, 0x20, 0x31, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, + 0x73, 0x74, 0x20, 0x6e, 0x65, 0x78, 0x74, 0x20, 0x3d, 0x20, 0x70, 0x61, + 0x74, 0x74, 0x65, 0x72, 0x6e, 0x5b, 0x69, 0x20, 0x2b, 0x20, 0x31, 0x5d, + 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, 0x45, 0x53, 0x43, 0x41, + 0x50, 0x45, 0x44, 0x5f, 0x49, 0x4e, 0x5f, 0x52, 0x45, 0x47, 0x45, 0x58, + 0x50, 0x53, 0x5f, 0x42, 0x55, 0x54, 0x5f, 0x4e, 0x4f, 0x54, 0x5f, 0x49, + 0x4e, 0x5f, 0x4c, 0x49, 0x54, 0x45, 0x52, 0x41, 0x4c, 0x53, 0x2e, 0x68, + 0x61, 0x73, 0x28, 0x6e, 0x65, 0x78, 0x74, 0x29, 0x29, 0x20, 0x7b, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x69, 0x20, 0x2b, 0x3d, 0x20, 0x31, 0x3b, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x6c, 0x69, 0x74, 0x65, 0x72, 0x61, 0x6c, 0x20, + 0x2b, 0x3d, 0x20, 0x70, 0x61, 0x74, 0x74, 0x65, 0x72, 0x6e, 0x5b, 0x69, + 0x5d, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x20, 0x2b, 0x3d, 0x20, + 0x31, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x20, 0x65, 0x6c, 0x73, 0x65, 0x20, + 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x6c, 0x69, 0x74, 0x65, 0x72, 0x61, + 0x6c, 0x20, 0x2b, 0x3d, 0x20, 0x70, 0x61, 0x74, 0x74, 0x65, 0x72, 0x6e, + 0x2e, 0x73, 0x6c, 0x69, 0x63, 0x65, 0x28, 0x69, 0x2c, 0x20, 0x69, 0x20, + 0x2b, 0x20, 0x32, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x20, + 0x2b, 0x3d, 0x20, 0x32, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x20, + 0x65, 0x6c, 0x73, 0x65, 0x20, 0x69, 0x66, 0x20, 0x28, 0x70, 0x61, 0x74, + 0x74, 0x65, 0x72, 0x6e, 0x5b, 0x69, 0x5d, 0x20, 0x3d, 0x3d, 0x3d, 0x20, + 0x27, 0x22, 0x27, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x6c, 0x69, 0x74, + 0x65, 0x72, 0x61, 0x6c, 0x20, 0x2b, 0x3d, 0x20, 0x27, 0x5c, 0x5c, 0x22, + 0x27, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x20, 0x2b, 0x3d, 0x20, 0x31, 0x3b, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x7d, 0x20, 0x65, 0x6c, 0x73, 0x65, 0x20, 0x69, 0x66, 0x20, 0x28, + 0x21, 0x4e, 0x4f, 0x4e, 0x5f, 0x4c, 0x49, 0x54, 0x45, 0x52, 0x41, 0x4c, + 0x5f, 0x53, 0x45, 0x54, 0x2e, 0x68, 0x61, 0x73, 0x28, 0x70, 0x61, 0x74, + 0x74, 0x65, 0x72, 0x6e, 0x5b, 0x69, 0x5d, 0x29, 0x20, 0x26, 0x26, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x28, 0x69, 0x20, 0x3d, 0x3d, 0x3d, 0x20, 0x6c, + 0x65, 0x6e, 0x67, 0x74, 0x68, 0x20, 0x2d, 0x20, 0x31, 0x20, 0x7c, 0x7c, + 0x20, 0x6c, 0x69, 0x74, 0x65, 0x72, 0x61, 0x6c, 0x20, 0x3d, 0x3d, 0x3d, + 0x20, 0x27, 0x27, 0x20, 0x7c, 0x7c, 0x20, 0x70, 0x61, 0x74, 0x74, 0x65, + 0x72, 0x6e, 0x5b, 0x69, 0x20, 0x2b, 0x20, 0x31, 0x5d, 0x20, 0x3d, 0x3d, + 0x3d, 0x20, 0x27, 0x2e, 0x27, 0x20, 0x7c, 0x7c, 0x20, 0x21, 0x4e, 0x4f, + 0x4e, 0x5f, 0x4c, 0x49, 0x54, 0x45, 0x52, 0x41, 0x4c, 0x5f, 0x53, 0x45, + 0x54, 0x2e, 0x68, 0x61, 0x73, 0x28, 0x70, 0x61, 0x74, 0x74, 0x65, 0x72, + 0x6e, 0x5b, 0x69, 0x2b, 0x31, 0x5d, 0x29, 0x29, 0x29, 0x20, 0x7b, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x6c, 0x69, 0x74, 0x65, 0x72, 0x61, 0x6c, 0x20, 0x2b, 0x3d, + 0x20, 0x70, 0x61, 0x74, 0x74, 0x65, 0x72, 0x6e, 0x5b, 0x69, 0x5d, 0x3b, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x69, 0x20, 0x2b, 0x3d, 0x20, 0x31, 0x3b, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, + 0x20, 0x65, 0x6c, 0x73, 0x65, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x62, 0x72, + 0x65, 0x61, 0x6b, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, 0x6c, 0x69, 0x74, + 0x65, 0x72, 0x61, 0x6c, 0x20, 0x21, 0x3d, 0x3d, 0x20, 0x27, 0x27, 0x29, + 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x73, 0x65, 0x71, 0x2e, 0x70, 0x75, 0x73, 0x68, 0x28, + 0x5b, 0x6c, 0x69, 0x74, 0x65, 0x72, 0x61, 0x6c, 0x2c, 0x20, 0x74, 0x72, + 0x75, 0x65, 0x5d, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, + 0x6e, 0x20, 0x6a, 0x6f, 0x69, 0x6e, 0x53, 0x65, 0x71, 0x28, 0x29, 0x3b, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x3b, 0x0a, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x74, 0x68, 0x69, 0x73, + 0x2e, 0x5f, 0x61, 0x64, 0x64, 0x52, 0x75, 0x6c, 0x65, 0x28, 0x6e, 0x61, + 0x6d, 0x65, 0x2c, 0x20, 0x22, 0x5c, 0x22, 0x5c, 0x5c, 0x5c, 0x22, 0x5c, + 0x22, 0x20, 0x22, 0x20, 0x2b, 0x20, 0x74, 0x6f, 0x52, 0x75, 0x6c, 0x65, + 0x28, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x28, 0x29, + 0x29, 0x20, 0x2b, 0x20, 0x22, 0x20, 0x5c, 0x22, 0x5c, 0x5c, 0x5c, 0x22, + 0x5c, 0x22, 0x20, 0x73, 0x70, 0x61, 0x63, 0x65, 0x22, 0x29, 0x0a, 0x20, + 0x20, 0x7d, 0x0a, 0x0a, 0x20, 0x20, 0x5f, 0x72, 0x65, 0x73, 0x6f, 0x6c, + 0x76, 0x65, 0x52, 0x65, 0x66, 0x28, 0x72, 0x65, 0x66, 0x29, 0x20, 0x7b, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x6c, 0x65, 0x74, 0x20, 0x72, 0x65, 0x66, + 0x4e, 0x61, 0x6d, 0x65, 0x20, 0x3d, 0x20, 0x72, 0x65, 0x66, 0x2e, 0x73, + 0x70, 0x6c, 0x69, 0x74, 0x28, 0x27, 0x2f, 0x27, 0x29, 0x2e, 0x70, 0x6f, + 0x70, 0x28, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, + 0x28, 0x21, 0x28, 0x72, 0x65, 0x66, 0x4e, 0x61, 0x6d, 0x65, 0x20, 0x69, + 0x6e, 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x5f, 0x72, 0x75, 0x6c, 0x65, + 0x73, 0x29, 0x20, 0x26, 0x26, 0x20, 0x21, 0x74, 0x68, 0x69, 0x73, 0x2e, + 0x5f, 0x72, 0x65, 0x66, 0x73, 0x42, 0x65, 0x69, 0x6e, 0x67, 0x52, 0x65, + 0x73, 0x6f, 0x6c, 0x76, 0x65, 0x64, 0x2e, 0x68, 0x61, 0x73, 0x28, 0x72, + 0x65, 0x66, 0x29, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x5f, 0x72, 0x65, 0x66, 0x73, 0x42, + 0x65, 0x69, 0x6e, 0x67, 0x52, 0x65, 0x73, 0x6f, 0x6c, 0x76, 0x65, 0x64, + 0x2e, 0x61, 0x64, 0x64, 0x28, 0x72, 0x65, 0x66, 0x29, 0x3b, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x72, + 0x65, 0x73, 0x6f, 0x6c, 0x76, 0x65, 0x64, 0x20, 0x3d, 0x20, 0x74, 0x68, + 0x69, 0x73, 0x2e, 0x5f, 0x72, 0x65, 0x66, 0x73, 0x5b, 0x72, 0x65, 0x66, + 0x5d, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, 0x66, + 0x4e, 0x61, 0x6d, 0x65, 0x20, 0x3d, 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, + 0x76, 0x69, 0x73, 0x69, 0x74, 0x28, 0x72, 0x65, 0x73, 0x6f, 0x6c, 0x76, + 0x65, 0x64, 0x2c, 0x20, 0x72, 0x65, 0x66, 0x4e, 0x61, 0x6d, 0x65, 0x29, + 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x74, 0x68, 0x69, 0x73, + 0x2e, 0x5f, 0x72, 0x65, 0x66, 0x73, 0x42, 0x65, 0x69, 0x6e, 0x67, 0x52, + 0x65, 0x73, 0x6f, 0x6c, 0x76, 0x65, 0x64, 0x2e, 0x64, 0x65, 0x6c, 0x65, + 0x74, 0x65, 0x28, 0x72, 0x65, 0x66, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, + 0x6e, 0x20, 0x72, 0x65, 0x66, 0x4e, 0x61, 0x6d, 0x65, 0x3b, 0x0a, 0x20, + 0x20, 0x7d, 0x0a, 0x0a, 0x20, 0x20, 0x5f, 0x67, 0x65, 0x6e, 0x65, 0x72, + 0x61, 0x74, 0x65, 0x43, 0x6f, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x74, 0x52, + 0x75, 0x6c, 0x65, 0x28, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x29, 0x20, 0x7b, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, 0x74, 0x79, 0x70, + 0x65, 0x6f, 0x66, 0x20, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x20, 0x21, 0x3d, + 0x3d, 0x20, 0x27, 0x73, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x27, 0x29, 0x20, + 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x74, 0x68, 0x72, 0x6f, + 0x77, 0x20, 0x6e, 0x65, 0x77, 0x20, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x28, + 0x27, 0x4f, 0x6e, 0x6c, 0x79, 0x20, 0x73, 0x74, 0x72, 0x69, 0x6e, 0x67, + 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x74, 0x73, 0x20, 0x61, + 0x72, 0x65, 0x20, 0x73, 0x75, 0x70, 0x70, 0x6f, 0x72, 0x74, 0x65, 0x64, + 0x2c, 0x20, 0x67, 0x6f, 0x74, 0x20, 0x27, 0x20, 0x2b, 0x20, 0x4a, 0x53, + 0x4f, 0x4e, 0x2e, 0x73, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x69, 0x66, 0x79, + 0x28, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x29, 0x29, 0x3b, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, + 0x72, 0x6e, 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x5f, 0x66, 0x6f, 0x72, + 0x6d, 0x61, 0x74, 0x4c, 0x69, 0x74, 0x65, 0x72, 0x61, 0x6c, 0x28, 0x76, + 0x61, 0x6c, 0x75, 0x65, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x7d, 0x0a, 0x0a, + 0x20, 0x20, 0x76, 0x69, 0x73, 0x69, 0x74, 0x28, 0x73, 0x63, 0x68, 0x65, + 0x6d, 0x61, 0x2c, 0x20, 0x6e, 0x61, 0x6d, 0x65, 0x29, 0x20, 0x7b, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x73, 0x63, + 0x68, 0x65, 0x6d, 0x61, 0x54, 0x79, 0x70, 0x65, 0x20, 0x3d, 0x20, 0x73, + 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x3b, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x73, 0x63, + 0x68, 0x65, 0x6d, 0x61, 0x46, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x20, 0x3d, + 0x20, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2e, 0x66, 0x6f, 0x72, 0x6d, + 0x61, 0x74, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, + 0x74, 0x20, 0x72, 0x75, 0x6c, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x20, 0x3d, + 0x20, 0x6e, 0x61, 0x6d, 0x65, 0x20, 0x69, 0x6e, 0x20, 0x52, 0x45, 0x53, + 0x45, 0x52, 0x56, 0x45, 0x44, 0x5f, 0x4e, 0x41, 0x4d, 0x45, 0x53, 0x20, + 0x3f, 0x20, 0x6e, 0x61, 0x6d, 0x65, 0x20, 0x2b, 0x20, 0x27, 0x2d, 0x27, + 0x20, 0x3a, 0x20, 0x6e, 0x61, 0x6d, 0x65, 0x20, 0x3d, 0x3d, 0x20, 0x27, + 0x27, 0x20, 0x3f, 0x20, 0x27, 0x72, 0x6f, 0x6f, 0x74, 0x27, 0x20, 0x3a, + 0x20, 0x6e, 0x61, 0x6d, 0x65, 0x3b, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x72, 0x65, 0x66, 0x20, 0x3d, 0x20, + 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2e, 0x24, 0x72, 0x65, 0x66, 0x3b, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, 0x72, 0x65, 0x66, + 0x20, 0x21, 0x3d, 0x3d, 0x20, 0x75, 0x6e, 0x64, 0x65, 0x66, 0x69, 0x6e, + 0x65, 0x64, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x5f, 0x61, 0x64, 0x64, 0x52, 0x75, 0x6c, 0x65, 0x28, 0x72, 0x75, 0x6c, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x2c, 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, - 0x5f, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x4c, 0x69, 0x74, 0x65, 0x72, - 0x61, 0x6c, 0x28, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2e, 0x63, 0x6f, - 0x6e, 0x73, 0x74, 0x29, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x7d, - 0x20, 0x65, 0x6c, 0x73, 0x65, 0x20, 0x69, 0x66, 0x20, 0x28, 0x27, 0x65, - 0x6e, 0x75, 0x6d, 0x27, 0x20, 0x69, 0x6e, 0x20, 0x73, 0x63, 0x68, 0x65, - 0x6d, 0x61, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x72, 0x75, 0x6c, 0x65, 0x20, 0x3d, - 0x20, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2e, 0x65, 0x6e, 0x75, 0x6d, - 0x2e, 0x6d, 0x61, 0x70, 0x28, 0x76, 0x20, 0x3d, 0x3e, 0x20, 0x74, 0x68, - 0x69, 0x73, 0x2e, 0x5f, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x4c, 0x69, - 0x74, 0x65, 0x72, 0x61, 0x6c, 0x28, 0x76, 0x29, 0x29, 0x2e, 0x6a, 0x6f, - 0x69, 0x6e, 0x28, 0x27, 0x20, 0x7c, 0x20, 0x27, 0x29, 0x3b, 0x0a, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, - 0x74, 0x68, 0x69, 0x73, 0x2e, 0x5f, 0x61, 0x64, 0x64, 0x52, 0x75, 0x6c, - 0x65, 0x28, 0x72, 0x75, 0x6c, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x2c, 0x20, - 0x72, 0x75, 0x6c, 0x65, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x7d, + 0x5f, 0x72, 0x65, 0x73, 0x6f, 0x6c, 0x76, 0x65, 0x52, 0x65, 0x66, 0x28, + 0x72, 0x65, 0x66, 0x29, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x20, 0x65, 0x6c, 0x73, 0x65, 0x20, 0x69, 0x66, 0x20, 0x28, 0x73, 0x63, - 0x68, 0x65, 0x6d, 0x61, 0x54, 0x79, 0x70, 0x65, 0x20, 0x3d, 0x3d, 0x3d, - 0x20, 0x27, 0x6f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x27, 0x20, 0x26, 0x26, - 0x20, 0x27, 0x70, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, - 0x27, 0x20, 0x69, 0x6e, 0x20, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x29, - 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x2f, 0x2f, 0x20, - 0x54, 0x4f, 0x44, 0x4f, 0x3a, 0x20, 0x60, 0x72, 0x65, 0x71, 0x75, 0x69, - 0x72, 0x65, 0x64, 0x60, 0x20, 0x6b, 0x65, 0x79, 0x77, 0x6f, 0x72, 0x64, - 0x20, 0x28, 0x66, 0x72, 0x6f, 0x6d, 0x20, 0x70, 0x79, 0x74, 0x68, 0x6f, - 0x6e, 0x20, 0x69, 0x6d, 0x70, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x61, - 0x74, 0x69, 0x6f, 0x6e, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x68, 0x65, 0x6d, 0x61, 0x2e, 0x6f, 0x6e, 0x65, 0x4f, 0x66, 0x20, 0x7c, + 0x7c, 0x20, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2e, 0x61, 0x6e, 0x79, + 0x4f, 0x66, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, + 0x5f, 0x61, 0x64, 0x64, 0x52, 0x75, 0x6c, 0x65, 0x28, 0x72, 0x75, 0x6c, + 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x2c, 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, + 0x5f, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x55, 0x6e, 0x69, + 0x6f, 0x6e, 0x52, 0x75, 0x6c, 0x65, 0x28, 0x6e, 0x61, 0x6d, 0x65, 0x2c, + 0x20, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2e, 0x6f, 0x6e, 0x65, 0x4f, + 0x66, 0x20, 0x7c, 0x7c, 0x20, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2e, + 0x61, 0x6e, 0x79, 0x4f, 0x66, 0x29, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x7d, 0x20, 0x65, 0x6c, 0x73, 0x65, 0x20, 0x69, 0x66, 0x20, 0x28, + 0x41, 0x72, 0x72, 0x61, 0x79, 0x2e, 0x69, 0x73, 0x41, 0x72, 0x72, 0x61, + 0x79, 0x28, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x54, 0x79, 0x70, 0x65, + 0x29, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, + 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x5f, + 0x61, 0x64, 0x64, 0x52, 0x75, 0x6c, 0x65, 0x28, 0x72, 0x75, 0x6c, 0x65, + 0x4e, 0x61, 0x6d, 0x65, 0x2c, 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x5f, + 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x55, 0x6e, 0x69, 0x6f, + 0x6e, 0x52, 0x75, 0x6c, 0x65, 0x28, 0x6e, 0x61, 0x6d, 0x65, 0x2c, 0x20, + 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x54, 0x79, 0x70, 0x65, 0x2e, 0x6d, + 0x61, 0x70, 0x28, 0x74, 0x20, 0x3d, 0x3e, 0x20, 0x28, 0x7b, 0x20, 0x74, + 0x79, 0x70, 0x65, 0x3a, 0x20, 0x74, 0x20, 0x7d, 0x29, 0x29, 0x29, 0x29, + 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x20, 0x65, 0x6c, 0x73, 0x65, + 0x20, 0x69, 0x66, 0x20, 0x28, 0x27, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x27, + 0x20, 0x69, 0x6e, 0x20, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x29, 0x20, + 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, + 0x74, 0x79, 0x70, 0x65, 0x6f, 0x66, 0x20, 0x73, 0x63, 0x68, 0x65, 0x6d, + 0x61, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x21, 0x3d, 0x3d, 0x20, + 0x27, 0x73, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x27, 0x29, 0x20, 0x7b, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x74, 0x68, 0x72, 0x6f, + 0x77, 0x20, 0x6e, 0x65, 0x77, 0x20, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x28, + 0x27, 0x4f, 0x6e, 0x6c, 0x79, 0x20, 0x73, 0x74, 0x72, 0x69, 0x6e, 0x67, + 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x74, 0x73, 0x20, 0x61, + 0x72, 0x65, 0x20, 0x73, 0x75, 0x70, 0x70, 0x6f, 0x72, 0x74, 0x65, 0x64, + 0x2c, 0x20, 0x67, 0x6f, 0x74, 0x20, 0x27, 0x20, 0x2b, 0x20, 0x4a, 0x53, + 0x4f, 0x4e, 0x2e, 0x73, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x69, 0x66, 0x79, + 0x28, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2e, 0x63, 0x6f, 0x6e, 0x73, + 0x74, 0x29, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, + 0x6e, 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x5f, 0x61, 0x64, 0x64, 0x52, + 0x75, 0x6c, 0x65, 0x28, 0x72, 0x75, 0x6c, 0x65, 0x4e, 0x61, 0x6d, 0x65, + 0x2c, 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x5f, 0x67, 0x65, 0x6e, 0x65, + 0x72, 0x61, 0x74, 0x65, 0x43, 0x6f, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x74, + 0x52, 0x75, 0x6c, 0x65, 0x28, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2e, + 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x29, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x7d, 0x20, 0x65, 0x6c, 0x73, 0x65, 0x20, 0x69, 0x66, 0x20, 0x28, + 0x27, 0x65, 0x6e, 0x75, 0x6d, 0x27, 0x20, 0x69, 0x6e, 0x20, 0x73, 0x63, + 0x68, 0x65, 0x6d, 0x61, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x72, 0x75, 0x6c, 0x65, + 0x20, 0x3d, 0x20, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2e, 0x65, 0x6e, + 0x75, 0x6d, 0x2e, 0x6d, 0x61, 0x70, 0x28, 0x76, 0x20, 0x3d, 0x3e, 0x20, + 0x74, 0x68, 0x69, 0x73, 0x2e, 0x5f, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, + 0x74, 0x65, 0x43, 0x6f, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x74, 0x52, 0x75, + 0x6c, 0x65, 0x28, 0x76, 0x29, 0x29, 0x2e, 0x6a, 0x6f, 0x69, 0x6e, 0x28, + 0x27, 0x20, 0x7c, 0x20, 0x27, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x74, 0x68, 0x69, + 0x73, 0x2e, 0x5f, 0x61, 0x64, 0x64, 0x52, 0x75, 0x6c, 0x65, 0x28, 0x72, + 0x75, 0x6c, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x2c, 0x20, 0x72, 0x75, 0x6c, + 0x65, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x20, 0x65, 0x6c, + 0x73, 0x65, 0x20, 0x69, 0x66, 0x20, 0x28, 0x28, 0x73, 0x63, 0x68, 0x65, + 0x6d, 0x61, 0x54, 0x79, 0x70, 0x65, 0x20, 0x3d, 0x3d, 0x3d, 0x20, 0x75, + 0x6e, 0x64, 0x65, 0x66, 0x69, 0x6e, 0x65, 0x64, 0x20, 0x7c, 0x7c, 0x20, + 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x54, 0x79, 0x70, 0x65, 0x20, 0x3d, + 0x3d, 0x3d, 0x20, 0x27, 0x6f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x27, 0x29, + 0x20, 0x26, 0x26, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x28, 0x27, 0x70, 0x72, 0x6f, + 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x27, 0x20, 0x69, 0x6e, 0x20, + 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x20, 0x7c, 0x7c, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x28, 0x27, 0x61, 0x64, 0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e, + 0x61, 0x6c, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, + 0x27, 0x20, 0x69, 0x6e, 0x20, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x20, + 0x26, 0x26, 0x20, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2e, 0x61, 0x64, + 0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x61, 0x6c, 0x50, 0x72, 0x6f, 0x70, + 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x20, 0x21, 0x3d, 0x3d, 0x20, 0x74, + 0x72, 0x75, 0x65, 0x29, 0x29, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x72, 0x65, 0x71, + 0x75, 0x69, 0x72, 0x65, 0x64, 0x20, 0x3d, 0x20, 0x6e, 0x65, 0x77, 0x20, + 0x53, 0x65, 0x74, 0x28, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2e, 0x72, + 0x65, 0x71, 0x75, 0x69, 0x72, 0x65, 0x64, 0x20, 0x7c, 0x7c, 0x20, 0x5b, + 0x5d, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, + 0x6e, 0x73, 0x74, 0x20, 0x70, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, + 0x65, 0x73, 0x20, 0x3d, 0x20, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x2e, + 0x65, 0x6e, 0x74, 0x72, 0x69, 0x65, 0x73, 0x28, 0x73, 0x63, 0x68, 0x65, + 0x6d, 0x61, 0x2e, 0x70, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, + 0x73, 0x20, 0x3f, 0x3f, 0x20, 0x7b, 0x7d, 0x29, 0x3b, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x74, + 0x68, 0x69, 0x73, 0x2e, 0x5f, 0x61, 0x64, 0x64, 0x52, 0x75, 0x6c, 0x65, + 0x28, 0x72, 0x75, 0x6c, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x2c, 0x20, 0x74, + 0x68, 0x69, 0x73, 0x2e, 0x5f, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x4f, 0x62, + 0x6a, 0x65, 0x63, 0x74, 0x52, 0x75, 0x6c, 0x65, 0x28, 0x70, 0x72, 0x6f, + 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x2c, 0x20, 0x72, 0x65, 0x71, + 0x75, 0x69, 0x72, 0x65, 0x64, 0x2c, 0x20, 0x6e, 0x61, 0x6d, 0x65, 0x2c, + 0x20, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2e, 0x61, 0x64, 0x64, 0x69, + 0x74, 0x69, 0x6f, 0x6e, 0x61, 0x6c, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, + 0x74, 0x69, 0x65, 0x73, 0x29, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x7d, 0x20, 0x65, 0x6c, 0x73, 0x65, 0x20, 0x69, 0x66, 0x20, 0x28, 0x28, + 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x54, 0x79, 0x70, 0x65, 0x20, 0x3d, + 0x3d, 0x3d, 0x20, 0x75, 0x6e, 0x64, 0x65, 0x66, 0x69, 0x6e, 0x65, 0x64, + 0x20, 0x7c, 0x7c, 0x20, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x54, 0x79, + 0x70, 0x65, 0x20, 0x3d, 0x3d, 0x3d, 0x20, 0x27, 0x6f, 0x62, 0x6a, 0x65, + 0x63, 0x74, 0x27, 0x29, 0x20, 0x26, 0x26, 0x20, 0x27, 0x61, 0x6c, 0x6c, + 0x4f, 0x66, 0x27, 0x20, 0x69, 0x6e, 0x20, 0x73, 0x63, 0x68, 0x65, 0x6d, + 0x61, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, + 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x72, 0x65, 0x71, 0x75, 0x69, 0x72, 0x65, + 0x64, 0x20, 0x3d, 0x20, 0x6e, 0x65, 0x77, 0x20, 0x53, 0x65, 0x74, 0x28, + 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, + 0x73, 0x74, 0x20, 0x70, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, + 0x73, 0x20, 0x3d, 0x20, 0x5b, 0x5d, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x61, 0x64, 0x64, 0x43, + 0x6f, 0x6d, 0x70, 0x6f, 0x6e, 0x65, 0x6e, 0x74, 0x20, 0x3d, 0x20, 0x28, + 0x63, 0x6f, 0x6d, 0x70, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2c, 0x20, + 0x69, 0x73, 0x52, 0x65, 0x71, 0x75, 0x69, 0x72, 0x65, 0x64, 0x29, 0x20, + 0x3d, 0x3e, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x72, 0x65, 0x66, 0x20, 0x3d, + 0x20, 0x63, 0x6f, 0x6d, 0x70, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2e, + 0x24, 0x72, 0x65, 0x66, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, 0x72, 0x65, 0x66, 0x20, 0x21, 0x3d, + 0x3d, 0x20, 0x75, 0x6e, 0x64, 0x65, 0x66, 0x69, 0x6e, 0x65, 0x64, 0x29, + 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x63, 0x6f, 0x6d, 0x70, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x20, + 0x3d, 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x5f, 0x72, 0x65, 0x66, 0x73, + 0x5b, 0x72, 0x65, 0x66, 0x5d, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, 0x27, 0x70, 0x72, 0x6f, 0x70, 0x65, + 0x72, 0x74, 0x69, 0x65, 0x73, 0x27, 0x20, 0x69, 0x6e, 0x20, 0x63, 0x6f, + 0x6d, 0x70, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x29, 0x20, 0x7b, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x66, 0x6f, + 0x72, 0x20, 0x28, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x5b, 0x70, 0x72, + 0x6f, 0x70, 0x4e, 0x61, 0x6d, 0x65, 0x2c, 0x20, 0x70, 0x72, 0x6f, 0x70, + 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x5d, 0x20, 0x6f, 0x66, 0x20, 0x4f, + 0x62, 0x6a, 0x65, 0x63, 0x74, 0x2e, 0x65, 0x6e, 0x74, 0x72, 0x69, 0x65, + 0x73, 0x28, 0x63, 0x6f, 0x6d, 0x70, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, + 0x2e, 0x70, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x29, + 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x70, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, + 0x65, 0x73, 0x2e, 0x70, 0x75, 0x73, 0x68, 0x28, 0x5b, 0x70, 0x72, 0x6f, + 0x70, 0x4e, 0x61, 0x6d, 0x65, 0x2c, 0x20, 0x70, 0x72, 0x6f, 0x70, 0x53, + 0x63, 0x68, 0x65, 0x6d, 0x61, 0x5d, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, + 0x28, 0x69, 0x73, 0x52, 0x65, 0x71, 0x75, 0x69, 0x72, 0x65, 0x64, 0x29, + 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, 0x71, 0x75, 0x69, 0x72, 0x65, + 0x64, 0x2e, 0x61, 0x64, 0x64, 0x28, 0x70, 0x72, 0x6f, 0x70, 0x4e, 0x61, + 0x6d, 0x65, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, + 0x3b, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x66, 0x6f, 0x72, + 0x20, 0x28, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x74, 0x20, 0x6f, 0x66, + 0x20, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2e, 0x61, 0x6c, 0x6c, 0x4f, + 0x66, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x69, 0x66, 0x20, 0x28, 0x27, 0x61, 0x6e, 0x79, 0x4f, 0x66, 0x27, + 0x20, 0x69, 0x6e, 0x20, 0x74, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x66, 0x6f, 0x72, 0x20, 0x28, + 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x74, 0x74, 0x20, 0x6f, 0x66, 0x20, + 0x74, 0x2e, 0x61, 0x6e, 0x79, 0x4f, 0x66, 0x29, 0x20, 0x7b, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x61, + 0x64, 0x64, 0x43, 0x6f, 0x6d, 0x70, 0x6f, 0x6e, 0x65, 0x6e, 0x74, 0x28, + 0x74, 0x74, 0x2c, 0x20, 0x66, 0x61, 0x6c, 0x73, 0x65, 0x29, 0x3b, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x20, 0x65, 0x6c, + 0x73, 0x65, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x61, 0x64, 0x64, 0x43, 0x6f, 0x6d, 0x70, 0x6f, 0x6e, + 0x65, 0x6e, 0x74, 0x28, 0x74, 0x2c, 0x20, 0x74, 0x72, 0x75, 0x65, 0x29, + 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x74, 0x68, + 0x69, 0x73, 0x2e, 0x5f, 0x61, 0x64, 0x64, 0x52, 0x75, 0x6c, 0x65, 0x28, + 0x72, 0x75, 0x6c, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x2c, 0x20, 0x74, 0x68, + 0x69, 0x73, 0x2e, 0x5f, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x4f, 0x62, 0x6a, + 0x65, 0x63, 0x74, 0x52, 0x75, 0x6c, 0x65, 0x28, 0x70, 0x72, 0x6f, 0x70, + 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x2c, 0x20, 0x72, 0x65, 0x71, 0x75, + 0x69, 0x72, 0x65, 0x64, 0x2c, 0x20, 0x6e, 0x61, 0x6d, 0x65, 0x2c, 0x20, + 0x2f, 0x2a, 0x20, 0x61, 0x64, 0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x61, + 0x6c, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x3d, + 0x20, 0x2a, 0x2f, 0x20, 0x66, 0x61, 0x6c, 0x73, 0x65, 0x29, 0x29, 0x3b, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x20, 0x65, 0x6c, 0x73, 0x65, 0x20, + 0x69, 0x66, 0x20, 0x28, 0x28, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x54, + 0x79, 0x70, 0x65, 0x20, 0x3d, 0x3d, 0x3d, 0x20, 0x75, 0x6e, 0x64, 0x65, + 0x66, 0x69, 0x6e, 0x65, 0x64, 0x20, 0x7c, 0x7c, 0x20, 0x73, 0x63, 0x68, + 0x65, 0x6d, 0x61, 0x54, 0x79, 0x70, 0x65, 0x20, 0x3d, 0x3d, 0x3d, 0x20, + 0x27, 0x61, 0x72, 0x72, 0x61, 0x79, 0x27, 0x29, 0x20, 0x26, 0x26, 0x20, + 0x28, 0x27, 0x69, 0x74, 0x65, 0x6d, 0x73, 0x27, 0x20, 0x69, 0x6e, 0x20, + 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x20, 0x7c, 0x7c, 0x20, 0x27, 0x70, + 0x72, 0x65, 0x66, 0x69, 0x78, 0x49, 0x74, 0x65, 0x6d, 0x73, 0x27, 0x20, + 0x69, 0x6e, 0x20, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x29, 0x29, 0x20, + 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, + 0x74, 0x20, 0x69, 0x74, 0x65, 0x6d, 0x73, 0x20, 0x3d, 0x20, 0x73, 0x63, + 0x68, 0x65, 0x6d, 0x61, 0x2e, 0x69, 0x74, 0x65, 0x6d, 0x73, 0x20, 0x3f, + 0x3f, 0x20, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2e, 0x70, 0x72, 0x65, + 0x66, 0x69, 0x78, 0x49, 0x74, 0x65, 0x6d, 0x73, 0x3b, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, 0x41, 0x72, 0x72, 0x61, + 0x79, 0x2e, 0x69, 0x73, 0x41, 0x72, 0x72, 0x61, 0x79, 0x28, 0x69, 0x74, + 0x65, 0x6d, 0x73, 0x29, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x74, + 0x68, 0x69, 0x73, 0x2e, 0x5f, 0x61, 0x64, 0x64, 0x52, 0x75, 0x6c, 0x65, + 0x28, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x72, 0x75, 0x6c, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x2c, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x27, 0x22, 0x5b, 0x22, + 0x20, 0x73, 0x70, 0x61, 0x63, 0x65, 0x20, 0x27, 0x20, 0x2b, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, + 0x74, 0x65, 0x6d, 0x73, 0x2e, 0x6d, 0x61, 0x70, 0x28, 0x28, 0x69, 0x74, + 0x65, 0x6d, 0x2c, 0x20, 0x69, 0x29, 0x20, 0x3d, 0x3e, 0x20, 0x74, 0x68, + 0x69, 0x73, 0x2e, 0x76, 0x69, 0x73, 0x69, 0x74, 0x28, 0x69, 0x74, 0x65, + 0x6d, 0x2c, 0x20, 0x60, 0x24, 0x7b, 0x6e, 0x61, 0x6d, 0x65, 0x20, 0x3f, + 0x3f, 0x20, 0x27, 0x27, 0x7d, 0x24, 0x7b, 0x6e, 0x61, 0x6d, 0x65, 0x20, + 0x3f, 0x20, 0x27, 0x2d, 0x27, 0x20, 0x3a, 0x20, 0x27, 0x27, 0x7d, 0x74, + 0x75, 0x70, 0x6c, 0x65, 0x2d, 0x24, 0x7b, 0x69, 0x7d, 0x60, 0x29, 0x29, + 0x2e, 0x6a, 0x6f, 0x69, 0x6e, 0x28, 0x27, 0x20, 0x22, 0x2c, 0x22, 0x20, + 0x73, 0x70, 0x61, 0x63, 0x65, 0x20, 0x27, 0x29, 0x20, 0x2b, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x27, + 0x20, 0x22, 0x5d, 0x22, 0x20, 0x73, 0x70, 0x61, 0x63, 0x65, 0x27, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x29, 0x3b, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x20, 0x65, 0x6c, 0x73, 0x65, 0x20, + 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, + 0x6e, 0x73, 0x74, 0x20, 0x69, 0x74, 0x65, 0x6d, 0x52, 0x75, 0x6c, 0x65, + 0x4e, 0x61, 0x6d, 0x65, 0x20, 0x3d, 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, + 0x76, 0x69, 0x73, 0x69, 0x74, 0x28, 0x69, 0x74, 0x65, 0x6d, 0x73, 0x2c, + 0x20, 0x60, 0x24, 0x7b, 0x6e, 0x61, 0x6d, 0x65, 0x20, 0x3f, 0x3f, 0x20, + 0x27, 0x27, 0x7d, 0x24, 0x7b, 0x6e, 0x61, 0x6d, 0x65, 0x20, 0x3f, 0x20, + 0x27, 0x2d, 0x27, 0x20, 0x3a, 0x20, 0x27, 0x27, 0x7d, 0x69, 0x74, 0x65, + 0x6d, 0x60, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x6c, 0x69, 0x73, 0x74, 0x49, + 0x74, 0x65, 0x6d, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x6f, 0x72, 0x20, + 0x3d, 0x20, 0x60, 0x28, 0x20, 0x22, 0x2c, 0x22, 0x20, 0x73, 0x70, 0x61, + 0x63, 0x65, 0x20, 0x24, 0x7b, 0x69, 0x74, 0x65, 0x6d, 0x52, 0x75, 0x6c, + 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x7d, 0x20, 0x29, 0x60, 0x3b, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x6c, 0x65, 0x74, 0x20, 0x73, + 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x69, 0x76, 0x65, 0x49, 0x74, 0x65, + 0x6d, 0x73, 0x20, 0x3d, 0x20, 0x27, 0x27, 0x3b, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x6c, 0x65, 0x74, 0x20, 0x6d, 0x69, 0x6e, + 0x49, 0x74, 0x65, 0x6d, 0x73, 0x20, 0x3d, 0x20, 0x73, 0x63, 0x68, 0x65, + 0x6d, 0x61, 0x2e, 0x6d, 0x69, 0x6e, 0x49, 0x74, 0x65, 0x6d, 0x73, 0x20, + 0x7c, 0x7c, 0x20, 0x30, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x6d, 0x61, 0x78, 0x49, + 0x74, 0x65, 0x6d, 0x73, 0x20, 0x3d, 0x20, 0x73, 0x63, 0x68, 0x65, 0x6d, + 0x61, 0x2e, 0x6d, 0x61, 0x78, 0x49, 0x74, 0x65, 0x6d, 0x73, 0x3b, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, + 0x6d, 0x69, 0x6e, 0x49, 0x74, 0x65, 0x6d, 0x73, 0x20, 0x3e, 0x20, 0x30, + 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x73, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x69, 0x76, 0x65, + 0x49, 0x74, 0x65, 0x6d, 0x73, 0x20, 0x3d, 0x20, 0x6c, 0x69, 0x73, 0x74, + 0x49, 0x74, 0x65, 0x6d, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x6f, 0x72, + 0x2e, 0x72, 0x65, 0x70, 0x65, 0x61, 0x74, 0x28, 0x6d, 0x69, 0x6e, 0x49, + 0x74, 0x65, 0x6d, 0x73, 0x20, 0x2d, 0x20, 0x31, 0x29, 0x3b, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x6d, 0x69, 0x6e, + 0x49, 0x74, 0x65, 0x6d, 0x73, 0x2d, 0x2d, 0x3b, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, 0x6d, 0x61, 0x78, 0x49, 0x74, + 0x65, 0x6d, 0x73, 0x20, 0x21, 0x3d, 0x3d, 0x20, 0x75, 0x6e, 0x64, 0x65, + 0x66, 0x69, 0x6e, 0x65, 0x64, 0x20, 0x26, 0x26, 0x20, 0x6d, 0x61, 0x78, + 0x49, 0x74, 0x65, 0x6d, 0x73, 0x20, 0x3e, 0x20, 0x6d, 0x69, 0x6e, 0x49, + 0x74, 0x65, 0x6d, 0x73, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x73, 0x75, 0x63, 0x63, 0x65, 0x73, + 0x73, 0x69, 0x76, 0x65, 0x49, 0x74, 0x65, 0x6d, 0x73, 0x20, 0x2b, 0x3d, + 0x20, 0x60, 0x24, 0x7b, 0x6c, 0x69, 0x73, 0x74, 0x49, 0x74, 0x65, 0x6d, + 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x6f, 0x72, 0x7d, 0x3f, 0x60, 0x2e, + 0x72, 0x65, 0x70, 0x65, 0x61, 0x74, 0x28, 0x6d, 0x61, 0x78, 0x49, 0x74, + 0x65, 0x6d, 0x73, 0x20, 0x2d, 0x20, 0x6d, 0x69, 0x6e, 0x49, 0x74, 0x65, + 0x6d, 0x73, 0x20, 0x2d, 0x20, 0x31, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x20, 0x65, 0x6c, 0x73, 0x65, 0x20, + 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x73, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x69, 0x76, 0x65, 0x49, 0x74, + 0x65, 0x6d, 0x73, 0x20, 0x2b, 0x3d, 0x20, 0x60, 0x24, 0x7b, 0x6c, 0x69, + 0x73, 0x74, 0x49, 0x74, 0x65, 0x6d, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, + 0x6f, 0x72, 0x7d, 0x2a, 0x60, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x72, 0x75, 0x6c, 0x65, 0x20, + 0x3d, 0x20, 0x6d, 0x69, 0x6e, 0x49, 0x74, 0x65, 0x6d, 0x73, 0x20, 0x3d, + 0x3d, 0x3d, 0x20, 0x30, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x3f, 0x20, 0x60, 0x22, 0x5b, 0x22, 0x20, 0x73, 0x70, + 0x61, 0x63, 0x65, 0x20, 0x28, 0x20, 0x24, 0x7b, 0x69, 0x74, 0x65, 0x6d, + 0x52, 0x75, 0x6c, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x7d, 0x20, 0x24, 0x7b, + 0x73, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x69, 0x76, 0x65, 0x49, 0x74, + 0x65, 0x6d, 0x73, 0x7d, 0x20, 0x29, 0x3f, 0x20, 0x22, 0x5d, 0x22, 0x20, + 0x73, 0x70, 0x61, 0x63, 0x65, 0x60, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x3a, 0x20, 0x60, 0x22, 0x5b, 0x22, 0x20, + 0x73, 0x70, 0x61, 0x63, 0x65, 0x20, 0x24, 0x7b, 0x69, 0x74, 0x65, 0x6d, + 0x52, 0x75, 0x6c, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x7d, 0x20, 0x24, 0x7b, + 0x73, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x69, 0x76, 0x65, 0x49, 0x74, + 0x65, 0x6d, 0x73, 0x7d, 0x20, 0x22, 0x5d, 0x22, 0x20, 0x73, 0x70, 0x61, + 0x63, 0x65, 0x60, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x74, 0x68, 0x69, 0x73, + 0x2e, 0x5f, 0x61, 0x64, 0x64, 0x52, 0x75, 0x6c, 0x65, 0x28, 0x72, 0x75, + 0x6c, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x2c, 0x20, 0x72, 0x75, 0x6c, 0x65, + 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x7d, 0x20, 0x65, 0x6c, 0x73, 0x65, 0x20, 0x69, 0x66, + 0x20, 0x28, 0x28, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x54, 0x79, 0x70, + 0x65, 0x20, 0x3d, 0x3d, 0x3d, 0x20, 0x75, 0x6e, 0x64, 0x65, 0x66, 0x69, + 0x6e, 0x65, 0x64, 0x20, 0x7c, 0x7c, 0x20, 0x73, 0x63, 0x68, 0x65, 0x6d, + 0x61, 0x54, 0x79, 0x70, 0x65, 0x20, 0x3d, 0x3d, 0x3d, 0x20, 0x27, 0x73, + 0x74, 0x72, 0x69, 0x6e, 0x67, 0x27, 0x29, 0x20, 0x26, 0x26, 0x20, 0x27, + 0x70, 0x61, 0x74, 0x74, 0x65, 0x72, 0x6e, 0x27, 0x20, 0x69, 0x6e, 0x20, + 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x74, + 0x68, 0x69, 0x73, 0x2e, 0x5f, 0x76, 0x69, 0x73, 0x69, 0x74, 0x50, 0x61, + 0x74, 0x74, 0x65, 0x72, 0x6e, 0x28, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, + 0x2e, 0x70, 0x61, 0x74, 0x74, 0x65, 0x72, 0x6e, 0x2c, 0x20, 0x72, 0x75, + 0x6c, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x7d, 0x20, 0x65, 0x6c, 0x73, 0x65, 0x20, 0x69, 0x66, 0x20, 0x28, + 0x28, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x54, 0x79, 0x70, 0x65, 0x20, + 0x3d, 0x3d, 0x3d, 0x20, 0x75, 0x6e, 0x64, 0x65, 0x66, 0x69, 0x6e, 0x65, + 0x64, 0x20, 0x7c, 0x7c, 0x20, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x54, + 0x79, 0x70, 0x65, 0x20, 0x3d, 0x3d, 0x3d, 0x20, 0x27, 0x73, 0x74, 0x72, + 0x69, 0x6e, 0x67, 0x27, 0x29, 0x20, 0x26, 0x26, 0x20, 0x2f, 0x5e, 0x75, + 0x75, 0x69, 0x64, 0x5b, 0x31, 0x2d, 0x35, 0x5d, 0x3f, 0x24, 0x2f, 0x2e, + 0x74, 0x65, 0x73, 0x74, 0x28, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2e, + 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x20, 0x7c, 0x7c, 0x20, 0x27, 0x27, + 0x29, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, + 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x5f, + 0x61, 0x64, 0x64, 0x52, 0x75, 0x6c, 0x65, 0x28, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, 0x75, 0x6c, 0x65, 0x4e, + 0x61, 0x6d, 0x65, 0x20, 0x3d, 0x3d, 0x3d, 0x20, 0x27, 0x72, 0x6f, 0x6f, + 0x74, 0x27, 0x20, 0x3f, 0x20, 0x27, 0x72, 0x6f, 0x6f, 0x74, 0x27, 0x20, + 0x3a, 0x20, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x46, 0x6f, 0x72, 0x6d, + 0x61, 0x74, 0x2c, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x50, 0x52, 0x49, 0x4d, 0x49, 0x54, 0x49, 0x56, 0x45, 0x5f, + 0x52, 0x55, 0x4c, 0x45, 0x53, 0x5b, 0x27, 0x75, 0x75, 0x69, 0x64, 0x27, + 0x5d, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x20, 0x65, 0x6c, 0x73, + 0x65, 0x20, 0x69, 0x66, 0x20, 0x28, 0x28, 0x73, 0x63, 0x68, 0x65, 0x6d, + 0x61, 0x54, 0x79, 0x70, 0x65, 0x20, 0x3d, 0x3d, 0x3d, 0x20, 0x75, 0x6e, + 0x64, 0x65, 0x66, 0x69, 0x6e, 0x65, 0x64, 0x20, 0x7c, 0x7c, 0x20, 0x73, + 0x63, 0x68, 0x65, 0x6d, 0x61, 0x54, 0x79, 0x70, 0x65, 0x20, 0x3d, 0x3d, + 0x3d, 0x20, 0x27, 0x73, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x27, 0x29, 0x20, + 0x26, 0x26, 0x20, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2e, 0x66, 0x6f, + 0x72, 0x6d, 0x61, 0x74, 0x20, 0x69, 0x6e, 0x20, 0x44, 0x41, 0x54, 0x45, + 0x5f, 0x52, 0x55, 0x4c, 0x45, 0x53, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x66, 0x6f, 0x72, 0x20, 0x28, 0x63, 0x6f, 0x6e, + 0x73, 0x74, 0x20, 0x5b, 0x74, 0x2c, 0x20, 0x72, 0x5d, 0x20, 0x6f, 0x66, + 0x20, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x2e, 0x65, 0x6e, 0x74, 0x72, + 0x69, 0x65, 0x73, 0x28, 0x44, 0x41, 0x54, 0x45, 0x5f, 0x52, 0x55, 0x4c, + 0x45, 0x53, 0x29, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x5f, 0x61, 0x64, 0x64, + 0x52, 0x75, 0x6c, 0x65, 0x28, 0x74, 0x2c, 0x20, 0x72, 0x29, 0x3b, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x73, 0x63, 0x68, + 0x65, 0x6d, 0x61, 0x46, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x20, 0x2b, 0x20, + 0x27, 0x2d, 0x73, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x27, 0x3b, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x7d, 0x20, 0x65, 0x6c, 0x73, 0x65, 0x20, 0x69, 0x66, + 0x20, 0x28, 0x28, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x54, 0x79, 0x70, + 0x65, 0x20, 0x3d, 0x3d, 0x3d, 0x20, 0x27, 0x6f, 0x62, 0x6a, 0x65, 0x63, + 0x74, 0x27, 0x29, 0x20, 0x7c, 0x7c, 0x20, 0x28, 0x4f, 0x62, 0x6a, 0x65, + 0x63, 0x74, 0x2e, 0x6b, 0x65, 0x79, 0x73, 0x28, 0x73, 0x63, 0x68, 0x65, + 0x6d, 0x61, 0x29, 0x2e, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x20, 0x3d, + 0x3d, 0x3d, 0x20, 0x30, 0x29, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x66, 0x6f, 0x72, 0x20, 0x28, 0x63, 0x6f, 0x6e, 0x73, + 0x74, 0x20, 0x6e, 0x20, 0x6f, 0x66, 0x20, 0x4f, 0x42, 0x4a, 0x45, 0x43, + 0x54, 0x5f, 0x52, 0x55, 0x4c, 0x45, 0x5f, 0x4e, 0x41, 0x4d, 0x45, 0x53, + 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x74, 0x68, 0x69, 0x73, 0x2e, 0x5f, 0x61, 0x64, 0x64, 0x52, 0x75, 0x6c, + 0x65, 0x28, 0x6e, 0x2c, 0x20, 0x50, 0x52, 0x49, 0x4d, 0x49, 0x54, 0x49, + 0x56, 0x45, 0x5f, 0x52, 0x55, 0x4c, 0x45, 0x53, 0x5b, 0x6e, 0x5d, 0x29, + 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x74, + 0x68, 0x69, 0x73, 0x2e, 0x5f, 0x61, 0x64, 0x64, 0x52, 0x75, 0x6c, 0x65, + 0x28, 0x72, 0x75, 0x6c, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x2c, 0x20, 0x27, + 0x6f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x27, 0x29, 0x3b, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x7d, 0x20, 0x65, 0x6c, 0x73, 0x65, 0x20, 0x7b, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, 0x21, 0x28, 0x73, + 0x63, 0x68, 0x65, 0x6d, 0x61, 0x54, 0x79, 0x70, 0x65, 0x20, 0x69, 0x6e, + 0x20, 0x50, 0x52, 0x49, 0x4d, 0x49, 0x54, 0x49, 0x56, 0x45, 0x5f, 0x52, + 0x55, 0x4c, 0x45, 0x53, 0x29, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x74, 0x68, 0x72, 0x6f, 0x77, 0x20, 0x6e, + 0x65, 0x77, 0x20, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x28, 0x60, 0x55, 0x6e, + 0x72, 0x65, 0x63, 0x6f, 0x67, 0x6e, 0x69, 0x7a, 0x65, 0x64, 0x20, 0x73, + 0x63, 0x68, 0x65, 0x6d, 0x61, 0x3a, 0x20, 0x24, 0x7b, 0x4a, 0x53, 0x4f, + 0x4e, 0x2e, 0x73, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x69, 0x66, 0x79, 0x28, + 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x29, 0x7d, 0x60, 0x29, 0x3b, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x2f, 0x2f, 0x20, 0x54, 0x4f, 0x44, 0x4f, 0x3a, 0x20, 0x73, + 0x75, 0x70, 0x70, 0x6f, 0x72, 0x74, 0x20, 0x6d, 0x69, 0x6e, 0x69, 0x6d, + 0x75, 0x6d, 0x2c, 0x20, 0x6d, 0x61, 0x78, 0x69, 0x6d, 0x75, 0x6d, 0x2c, + 0x20, 0x65, 0x78, 0x63, 0x6c, 0x75, 0x73, 0x69, 0x76, 0x65, 0x4d, 0x69, + 0x6e, 0x69, 0x6d, 0x75, 0x6d, 0x2c, 0x20, 0x65, 0x78, 0x63, 0x6c, 0x75, + 0x73, 0x69, 0x76, 0x65, 0x4d, 0x61, 0x78, 0x69, 0x6d, 0x75, 0x6d, 0x20, + 0x61, 0x74, 0x20, 0x6c, 0x65, 0x61, 0x73, 0x74, 0x20, 0x66, 0x6f, 0x72, + 0x20, 0x7a, 0x65, 0x72, 0x6f, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, + 0x5f, 0x61, 0x64, 0x64, 0x52, 0x75, 0x6c, 0x65, 0x28, 0x72, 0x75, 0x6c, + 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x20, 0x3d, 0x3d, 0x3d, 0x20, 0x27, 0x72, + 0x6f, 0x6f, 0x74, 0x27, 0x20, 0x3f, 0x20, 0x27, 0x72, 0x6f, 0x6f, 0x74, + 0x27, 0x20, 0x3a, 0x20, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x54, 0x79, + 0x70, 0x65, 0x2c, 0x20, 0x50, 0x52, 0x49, 0x4d, 0x49, 0x54, 0x49, 0x56, + 0x45, 0x5f, 0x52, 0x55, 0x4c, 0x45, 0x53, 0x5b, 0x73, 0x63, 0x68, 0x65, + 0x6d, 0x61, 0x54, 0x79, 0x70, 0x65, 0x5d, 0x29, 0x3b, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x7d, 0x0a, 0x0a, 0x20, 0x20, 0x5f, + 0x62, 0x75, 0x69, 0x6c, 0x64, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x52, + 0x75, 0x6c, 0x65, 0x28, 0x70, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, + 0x65, 0x73, 0x2c, 0x20, 0x72, 0x65, 0x71, 0x75, 0x69, 0x72, 0x65, 0x64, + 0x2c, 0x20, 0x6e, 0x61, 0x6d, 0x65, 0x2c, 0x20, 0x61, 0x64, 0x64, 0x69, + 0x74, 0x69, 0x6f, 0x6e, 0x61, 0x6c, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, + 0x74, 0x69, 0x65, 0x73, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x70, 0x72, 0x6f, 0x70, 0x4f, 0x72, 0x64, 0x65, 0x72, 0x20, 0x3d, 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x5f, 0x70, 0x72, 0x6f, 0x70, 0x4f, 0x72, 0x64, 0x65, 0x72, 0x3b, 0x0a, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x70, - 0x72, 0x6f, 0x70, 0x50, 0x61, 0x69, 0x72, 0x73, 0x20, 0x3d, 0x20, 0x4f, - 0x62, 0x6a, 0x65, 0x63, 0x74, 0x2e, 0x65, 0x6e, 0x74, 0x72, 0x69, 0x65, - 0x73, 0x28, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2e, 0x70, 0x72, 0x6f, - 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x29, 0x2e, 0x73, 0x6f, 0x72, - 0x74, 0x28, 0x28, 0x61, 0x2c, 0x20, 0x62, 0x29, 0x20, 0x3d, 0x3e, 0x20, - 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x2f, 0x2f, - 0x20, 0x73, 0x6f, 0x72, 0x74, 0x20, 0x62, 0x79, 0x20, 0x70, 0x6f, 0x73, - 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x69, 0x6e, 0x20, 0x70, 0x72, 0x6f, - 0x70, 0x5f, 0x6f, 0x72, 0x64, 0x65, 0x72, 0x20, 0x28, 0x69, 0x66, 0x20, - 0x73, 0x70, 0x65, 0x63, 0x69, 0x66, 0x69, 0x65, 0x64, 0x29, 0x20, 0x74, - 0x68, 0x65, 0x6e, 0x20, 0x62, 0x79, 0x20, 0x6b, 0x65, 0x79, 0x0a, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, - 0x20, 0x6f, 0x72, 0x64, 0x65, 0x72, 0x41, 0x20, 0x3d, 0x20, 0x74, 0x79, - 0x70, 0x65, 0x6f, 0x66, 0x20, 0x70, 0x72, 0x6f, 0x70, 0x4f, 0x72, 0x64, - 0x65, 0x72, 0x5b, 0x61, 0x5b, 0x30, 0x5d, 0x5d, 0x20, 0x3d, 0x3d, 0x3d, - 0x20, 0x27, 0x6e, 0x75, 0x6d, 0x62, 0x65, 0x72, 0x27, 0x20, 0x3f, 0x20, - 0x70, 0x72, 0x6f, 0x70, 0x4f, 0x72, 0x64, 0x65, 0x72, 0x5b, 0x61, 0x5b, - 0x30, 0x5d, 0x5d, 0x20, 0x3a, 0x20, 0x49, 0x6e, 0x66, 0x69, 0x6e, 0x69, - 0x74, 0x79, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x6f, 0x72, 0x64, 0x65, 0x72, 0x42, - 0x20, 0x3d, 0x20, 0x74, 0x79, 0x70, 0x65, 0x6f, 0x66, 0x20, 0x70, 0x72, - 0x6f, 0x70, 0x4f, 0x72, 0x64, 0x65, 0x72, 0x5b, 0x62, 0x5b, 0x30, 0x5d, - 0x5d, 0x20, 0x3d, 0x3d, 0x3d, 0x20, 0x27, 0x6e, 0x75, 0x6d, 0x62, 0x65, - 0x72, 0x27, 0x20, 0x3f, 0x20, 0x70, 0x72, 0x6f, 0x70, 0x4f, 0x72, 0x64, - 0x65, 0x72, 0x5b, 0x62, 0x5b, 0x30, 0x5d, 0x5d, 0x20, 0x3a, 0x20, 0x49, + 0x20, 0x20, 0x20, 0x2f, 0x2f, 0x20, 0x73, 0x6f, 0x72, 0x74, 0x20, 0x62, + 0x79, 0x20, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x69, + 0x6e, 0x20, 0x70, 0x72, 0x6f, 0x70, 0x5f, 0x6f, 0x72, 0x64, 0x65, 0x72, + 0x20, 0x28, 0x69, 0x66, 0x20, 0x73, 0x70, 0x65, 0x63, 0x69, 0x66, 0x69, + 0x65, 0x64, 0x29, 0x20, 0x74, 0x68, 0x65, 0x6e, 0x20, 0x62, 0x79, 0x20, + 0x6f, 0x72, 0x69, 0x67, 0x69, 0x6e, 0x61, 0x6c, 0x20, 0x6f, 0x72, 0x64, + 0x65, 0x72, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, + 0x20, 0x73, 0x6f, 0x72, 0x74, 0x65, 0x64, 0x50, 0x72, 0x6f, 0x70, 0x73, + 0x20, 0x3d, 0x20, 0x70, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, + 0x73, 0x2e, 0x6d, 0x61, 0x70, 0x28, 0x28, 0x5b, 0x6b, 0x5d, 0x29, 0x20, + 0x3d, 0x3e, 0x20, 0x6b, 0x29, 0x2e, 0x73, 0x6f, 0x72, 0x74, 0x28, 0x28, + 0x61, 0x2c, 0x20, 0x62, 0x29, 0x20, 0x3d, 0x3e, 0x20, 0x7b, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x6f, + 0x72, 0x64, 0x65, 0x72, 0x41, 0x20, 0x3d, 0x20, 0x70, 0x72, 0x6f, 0x70, + 0x4f, 0x72, 0x64, 0x65, 0x72, 0x5b, 0x61, 0x5d, 0x20, 0x7c, 0x7c, 0x20, + 0x49, 0x6e, 0x66, 0x69, 0x6e, 0x69, 0x74, 0x79, 0x3b, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x6f, 0x72, + 0x64, 0x65, 0x72, 0x42, 0x20, 0x3d, 0x20, 0x70, 0x72, 0x6f, 0x70, 0x4f, + 0x72, 0x64, 0x65, 0x72, 0x5b, 0x62, 0x5d, 0x20, 0x7c, 0x7c, 0x20, 0x49, 0x6e, 0x66, 0x69, 0x6e, 0x69, 0x74, 0x79, 0x3b, 0x0a, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, - 0x6f, 0x72, 0x64, 0x65, 0x72, 0x41, 0x20, 0x2d, 0x20, 0x6f, 0x72, 0x64, - 0x65, 0x72, 0x42, 0x20, 0x7c, 0x7c, 0x20, 0x61, 0x5b, 0x30, 0x5d, 0x2e, - 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x65, 0x43, 0x6f, 0x6d, 0x70, 0x61, 0x72, - 0x65, 0x28, 0x62, 0x5b, 0x30, 0x5d, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x7d, 0x29, 0x3b, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x6c, 0x65, 0x74, 0x20, 0x72, 0x75, 0x6c, 0x65, 0x20, 0x3d, - 0x20, 0x27, 0x22, 0x7b, 0x22, 0x20, 0x73, 0x70, 0x61, 0x63, 0x65, 0x27, - 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x70, 0x72, 0x6f, 0x70, - 0x50, 0x61, 0x69, 0x72, 0x73, 0x2e, 0x66, 0x6f, 0x72, 0x45, 0x61, 0x63, - 0x68, 0x28, 0x28, 0x5b, 0x70, 0x72, 0x6f, 0x70, 0x4e, 0x61, 0x6d, 0x65, - 0x2c, 0x20, 0x70, 0x72, 0x6f, 0x70, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, - 0x5d, 0x2c, 0x20, 0x69, 0x29, 0x20, 0x3d, 0x3e, 0x20, 0x7b, 0x0a, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, - 0x20, 0x70, 0x72, 0x6f, 0x70, 0x52, 0x75, 0x6c, 0x65, 0x4e, 0x61, 0x6d, - 0x65, 0x20, 0x3d, 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x76, 0x69, 0x73, - 0x69, 0x74, 0x28, 0x70, 0x72, 0x6f, 0x70, 0x53, 0x63, 0x68, 0x65, 0x6d, - 0x61, 0x2c, 0x20, 0x60, 0x24, 0x7b, 0x6e, 0x61, 0x6d, 0x65, 0x7d, 0x24, - 0x7b, 0x6e, 0x61, 0x6d, 0x65, 0x20, 0x3f, 0x20, 0x22, 0x2d, 0x22, 0x20, - 0x3a, 0x20, 0x22, 0x22, 0x7d, 0x24, 0x7b, 0x70, 0x72, 0x6f, 0x70, 0x4e, - 0x61, 0x6d, 0x65, 0x7d, 0x60, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, 0x69, 0x20, 0x3e, 0x20, - 0x30, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x72, 0x75, 0x6c, 0x65, 0x20, 0x2b, 0x3d, 0x20, 0x27, - 0x20, 0x22, 0x2c, 0x22, 0x20, 0x73, 0x70, 0x61, 0x63, 0x65, 0x27, 0x3b, - 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, 0x75, 0x6c, 0x65, 0x20, - 0x2b, 0x3d, 0x20, 0x60, 0x20, 0x24, 0x7b, 0x74, 0x68, 0x69, 0x73, 0x2e, - 0x5f, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x4c, 0x69, 0x74, 0x65, 0x72, - 0x61, 0x6c, 0x28, 0x70, 0x72, 0x6f, 0x70, 0x4e, 0x61, 0x6d, 0x65, 0x29, - 0x7d, 0x20, 0x73, 0x70, 0x61, 0x63, 0x65, 0x20, 0x22, 0x3a, 0x22, 0x20, - 0x73, 0x70, 0x61, 0x63, 0x65, 0x20, 0x24, 0x7b, 0x70, 0x72, 0x6f, 0x70, - 0x52, 0x75, 0x6c, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x7d, 0x60, 0x3b, 0x0a, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x29, 0x3b, 0x0a, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x72, 0x75, 0x6c, 0x65, 0x20, 0x2b, 0x3d, 0x20, - 0x27, 0x20, 0x22, 0x7d, 0x22, 0x20, 0x73, 0x70, 0x61, 0x63, 0x65, 0x27, - 0x3b, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, 0x74, - 0x75, 0x72, 0x6e, 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x5f, 0x61, 0x64, - 0x64, 0x52, 0x75, 0x6c, 0x65, 0x28, 0x72, 0x75, 0x6c, 0x65, 0x4e, 0x61, - 0x6d, 0x65, 0x2c, 0x20, 0x72, 0x75, 0x6c, 0x65, 0x29, 0x3b, 0x0a, 0x20, - 0x20, 0x20, 0x20, 0x7d, 0x20, 0x65, 0x6c, 0x73, 0x65, 0x20, 0x69, 0x66, - 0x20, 0x28, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x54, 0x79, 0x70, 0x65, - 0x20, 0x3d, 0x3d, 0x3d, 0x20, 0x27, 0x61, 0x72, 0x72, 0x61, 0x79, 0x27, - 0x20, 0x26, 0x26, 0x20, 0x27, 0x69, 0x74, 0x65, 0x6d, 0x73, 0x27, 0x20, - 0x69, 0x6e, 0x20, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x29, 0x20, 0x7b, - 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x2f, 0x2f, 0x20, 0x54, 0x4f, - 0x44, 0x4f, 0x20, 0x60, 0x70, 0x72, 0x65, 0x66, 0x69, 0x78, 0x49, 0x74, - 0x65, 0x6d, 0x73, 0x60, 0x20, 0x6b, 0x65, 0x79, 0x77, 0x6f, 0x72, 0x64, - 0x20, 0x28, 0x66, 0x72, 0x6f, 0x6d, 0x20, 0x70, 0x79, 0x74, 0x68, 0x6f, - 0x6e, 0x20, 0x69, 0x6d, 0x70, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x61, - 0x74, 0x69, 0x6f, 0x6e, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x69, 0x74, 0x65, 0x6d, 0x52, 0x75, - 0x6c, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x20, 0x3d, 0x20, 0x74, 0x68, 0x69, - 0x73, 0x2e, 0x76, 0x69, 0x73, 0x69, 0x74, 0x28, 0x73, 0x63, 0x68, 0x65, - 0x6d, 0x61, 0x2e, 0x69, 0x74, 0x65, 0x6d, 0x73, 0x2c, 0x20, 0x60, 0x24, - 0x7b, 0x6e, 0x61, 0x6d, 0x65, 0x7d, 0x24, 0x7b, 0x6e, 0x61, 0x6d, 0x65, - 0x20, 0x3f, 0x20, 0x22, 0x2d, 0x22, 0x20, 0x3a, 0x20, 0x22, 0x22, 0x7d, - 0x69, 0x74, 0x65, 0x6d, 0x60, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x72, 0x75, 0x6c, 0x65, - 0x20, 0x3d, 0x20, 0x60, 0x22, 0x5b, 0x22, 0x20, 0x73, 0x70, 0x61, 0x63, - 0x65, 0x20, 0x28, 0x24, 0x7b, 0x69, 0x74, 0x65, 0x6d, 0x52, 0x75, 0x6c, - 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x7d, 0x20, 0x28, 0x22, 0x2c, 0x22, 0x20, - 0x73, 0x70, 0x61, 0x63, 0x65, 0x20, 0x24, 0x7b, 0x69, 0x74, 0x65, 0x6d, - 0x52, 0x75, 0x6c, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x7d, 0x29, 0x2a, 0x29, - 0x3f, 0x20, 0x22, 0x5d, 0x22, 0x20, 0x73, 0x70, 0x61, 0x63, 0x65, 0x60, - 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, - 0x72, 0x6e, 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x5f, 0x61, 0x64, 0x64, - 0x52, 0x75, 0x6c, 0x65, 0x28, 0x72, 0x75, 0x6c, 0x65, 0x4e, 0x61, 0x6d, - 0x65, 0x2c, 0x20, 0x72, 0x75, 0x6c, 0x65, 0x29, 0x3b, 0x0a, 0x20, 0x20, - 0x20, 0x20, 0x7d, 0x20, 0x65, 0x6c, 0x73, 0x65, 0x20, 0x7b, 0x0a, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, 0x21, 0x50, 0x52, - 0x49, 0x4d, 0x49, 0x54, 0x49, 0x56, 0x45, 0x5f, 0x52, 0x55, 0x4c, 0x45, - 0x53, 0x5b, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x54, 0x79, 0x70, 0x65, - 0x5d, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x74, 0x68, 0x72, 0x6f, 0x77, 0x20, 0x6e, 0x65, 0x77, 0x20, 0x45, - 0x72, 0x72, 0x6f, 0x72, 0x28, 0x60, 0x55, 0x6e, 0x72, 0x65, 0x63, 0x6f, - 0x67, 0x6e, 0x69, 0x7a, 0x65, 0x64, 0x20, 0x73, 0x63, 0x68, 0x65, 0x6d, - 0x61, 0x3a, 0x20, 0x24, 0x7b, 0x4a, 0x53, 0x4f, 0x4e, 0x2e, 0x73, 0x74, - 0x72, 0x69, 0x6e, 0x67, 0x69, 0x66, 0x79, 0x28, 0x73, 0x63, 0x68, 0x65, - 0x6d, 0x61, 0x29, 0x7d, 0x60, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, - 0x74, 0x75, 0x72, 0x6e, 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x5f, 0x61, - 0x64, 0x64, 0x52, 0x75, 0x6c, 0x65, 0x28, 0x0a, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x72, 0x75, 0x6c, 0x65, 0x4e, 0x61, 0x6d, 0x65, - 0x20, 0x3d, 0x3d, 0x3d, 0x20, 0x27, 0x72, 0x6f, 0x6f, 0x74, 0x27, 0x20, - 0x3f, 0x20, 0x27, 0x72, 0x6f, 0x6f, 0x74, 0x27, 0x20, 0x3a, 0x20, 0x73, - 0x63, 0x68, 0x65, 0x6d, 0x61, 0x54, 0x79, 0x70, 0x65, 0x2c, 0x0a, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x50, 0x52, 0x49, 0x4d, 0x49, - 0x54, 0x49, 0x56, 0x45, 0x5f, 0x52, 0x55, 0x4c, 0x45, 0x53, 0x5b, 0x73, - 0x63, 0x68, 0x65, 0x6d, 0x61, 0x54, 0x79, 0x70, 0x65, 0x5d, 0x0a, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, - 0x7d, 0x0a, 0x20, 0x20, 0x7d, 0x0a, 0x0a, 0x20, 0x20, 0x66, 0x6f, 0x72, - 0x6d, 0x61, 0x74, 0x47, 0x72, 0x61, 0x6d, 0x6d, 0x61, 0x72, 0x28, 0x29, - 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x6c, 0x65, 0x74, 0x20, 0x67, - 0x72, 0x61, 0x6d, 0x6d, 0x61, 0x72, 0x20, 0x3d, 0x20, 0x27, 0x27, 0x3b, - 0x0a, 0x20, 0x20, 0x20, 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x5f, 0x72, - 0x75, 0x6c, 0x65, 0x73, 0x2e, 0x66, 0x6f, 0x72, 0x45, 0x61, 0x63, 0x68, - 0x28, 0x28, 0x72, 0x75, 0x6c, 0x65, 0x2c, 0x20, 0x6e, 0x61, 0x6d, 0x65, - 0x29, 0x20, 0x3d, 0x3e, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x67, 0x72, 0x61, 0x6d, 0x6d, 0x61, 0x72, 0x20, 0x2b, 0x3d, 0x20, - 0x60, 0x24, 0x7b, 0x6e, 0x61, 0x6d, 0x65, 0x7d, 0x20, 0x3a, 0x3a, 0x3d, - 0x20, 0x24, 0x7b, 0x72, 0x75, 0x6c, 0x65, 0x7d, 0x5c, 0x6e, 0x60, 0x3b, - 0x0a, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, - 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x67, 0x72, 0x61, 0x6d, - 0x6d, 0x61, 0x72, 0x3b, 0x0a, 0x20, 0x20, 0x7d, 0x0a, 0x7d, 0x0a + 0x20, 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x6f, 0x72, + 0x64, 0x65, 0x72, 0x41, 0x20, 0x2d, 0x20, 0x6f, 0x72, 0x64, 0x65, 0x72, + 0x42, 0x20, 0x7c, 0x7c, 0x20, 0x70, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, + 0x69, 0x65, 0x73, 0x2e, 0x66, 0x69, 0x6e, 0x64, 0x49, 0x6e, 0x64, 0x65, + 0x78, 0x28, 0x28, 0x5b, 0x6b, 0x5d, 0x29, 0x20, 0x3d, 0x3e, 0x20, 0x6b, + 0x20, 0x3d, 0x3d, 0x3d, 0x20, 0x61, 0x29, 0x20, 0x2d, 0x20, 0x70, 0x72, + 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x2e, 0x66, 0x69, 0x6e, + 0x64, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x28, 0x28, 0x5b, 0x6b, 0x5d, 0x29, + 0x20, 0x3d, 0x3e, 0x20, 0x6b, 0x20, 0x3d, 0x3d, 0x3d, 0x20, 0x62, 0x29, + 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x29, 0x3b, 0x0a, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x70, 0x72, 0x6f, + 0x70, 0x4b, 0x76, 0x52, 0x75, 0x6c, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x73, + 0x20, 0x3d, 0x20, 0x7b, 0x7d, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x66, + 0x6f, 0x72, 0x20, 0x28, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x5b, 0x70, + 0x72, 0x6f, 0x70, 0x4e, 0x61, 0x6d, 0x65, 0x2c, 0x20, 0x70, 0x72, 0x6f, + 0x70, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x5d, 0x20, 0x6f, 0x66, 0x20, + 0x70, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x29, 0x20, + 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, + 0x74, 0x20, 0x70, 0x72, 0x6f, 0x70, 0x52, 0x75, 0x6c, 0x65, 0x4e, 0x61, + 0x6d, 0x65, 0x20, 0x3d, 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x76, 0x69, + 0x73, 0x69, 0x74, 0x28, 0x70, 0x72, 0x6f, 0x70, 0x53, 0x63, 0x68, 0x65, + 0x6d, 0x61, 0x2c, 0x20, 0x60, 0x24, 0x7b, 0x6e, 0x61, 0x6d, 0x65, 0x20, + 0x3f, 0x3f, 0x20, 0x27, 0x27, 0x7d, 0x24, 0x7b, 0x6e, 0x61, 0x6d, 0x65, + 0x20, 0x3f, 0x20, 0x27, 0x2d, 0x27, 0x20, 0x3a, 0x20, 0x27, 0x27, 0x7d, + 0x24, 0x7b, 0x70, 0x72, 0x6f, 0x70, 0x4e, 0x61, 0x6d, 0x65, 0x7d, 0x60, + 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x70, 0x72, 0x6f, + 0x70, 0x4b, 0x76, 0x52, 0x75, 0x6c, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x73, + 0x5b, 0x70, 0x72, 0x6f, 0x70, 0x4e, 0x61, 0x6d, 0x65, 0x5d, 0x20, 0x3d, + 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x5f, 0x61, 0x64, 0x64, 0x52, 0x75, + 0x6c, 0x65, 0x28, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x60, 0x24, 0x7b, 0x6e, 0x61, 0x6d, 0x65, 0x20, 0x3f, 0x3f, 0x20, 0x27, + 0x27, 0x7d, 0x24, 0x7b, 0x6e, 0x61, 0x6d, 0x65, 0x20, 0x3f, 0x20, 0x27, + 0x2d, 0x27, 0x20, 0x3a, 0x20, 0x27, 0x27, 0x7d, 0x24, 0x7b, 0x70, 0x72, + 0x6f, 0x70, 0x4e, 0x61, 0x6d, 0x65, 0x7d, 0x2d, 0x6b, 0x76, 0x60, 0x2c, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x60, 0x24, 0x7b, + 0x74, 0x68, 0x69, 0x73, 0x2e, 0x5f, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74, + 0x4c, 0x69, 0x74, 0x65, 0x72, 0x61, 0x6c, 0x28, 0x70, 0x72, 0x6f, 0x70, + 0x4e, 0x61, 0x6d, 0x65, 0x29, 0x7d, 0x20, 0x73, 0x70, 0x61, 0x63, 0x65, + 0x20, 0x22, 0x3a, 0x22, 0x20, 0x73, 0x70, 0x61, 0x63, 0x65, 0x20, 0x24, + 0x7b, 0x70, 0x72, 0x6f, 0x70, 0x52, 0x75, 0x6c, 0x65, 0x4e, 0x61, 0x6d, + 0x65, 0x7d, 0x60, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x29, 0x3b, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x63, + 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x72, 0x65, 0x71, 0x75, 0x69, 0x72, 0x65, + 0x64, 0x50, 0x72, 0x6f, 0x70, 0x73, 0x20, 0x3d, 0x20, 0x73, 0x6f, 0x72, + 0x74, 0x65, 0x64, 0x50, 0x72, 0x6f, 0x70, 0x73, 0x2e, 0x66, 0x69, 0x6c, + 0x74, 0x65, 0x72, 0x28, 0x6b, 0x20, 0x3d, 0x3e, 0x20, 0x72, 0x65, 0x71, + 0x75, 0x69, 0x72, 0x65, 0x64, 0x2e, 0x68, 0x61, 0x73, 0x28, 0x6b, 0x29, + 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, + 0x20, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x61, 0x6c, 0x50, 0x72, 0x6f, + 0x70, 0x73, 0x20, 0x3d, 0x20, 0x73, 0x6f, 0x72, 0x74, 0x65, 0x64, 0x50, + 0x72, 0x6f, 0x70, 0x73, 0x2e, 0x66, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x28, + 0x6b, 0x20, 0x3d, 0x3e, 0x20, 0x21, 0x72, 0x65, 0x71, 0x75, 0x69, 0x72, + 0x65, 0x64, 0x2e, 0x68, 0x61, 0x73, 0x28, 0x6b, 0x29, 0x29, 0x3b, 0x0a, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, 0x74, 0x79, 0x70, + 0x65, 0x6f, 0x66, 0x20, 0x61, 0x64, 0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e, + 0x61, 0x6c, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, + 0x20, 0x3d, 0x3d, 0x3d, 0x20, 0x27, 0x6f, 0x62, 0x6a, 0x65, 0x63, 0x74, + 0x27, 0x20, 0x7c, 0x7c, 0x20, 0x61, 0x64, 0x64, 0x69, 0x74, 0x69, 0x6f, + 0x6e, 0x61, 0x6c, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, + 0x73, 0x20, 0x3d, 0x3d, 0x3d, 0x20, 0x74, 0x72, 0x75, 0x65, 0x29, 0x20, + 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, + 0x74, 0x20, 0x73, 0x75, 0x62, 0x4e, 0x61, 0x6d, 0x65, 0x20, 0x3d, 0x20, + 0x60, 0x24, 0x7b, 0x6e, 0x61, 0x6d, 0x65, 0x20, 0x3f, 0x3f, 0x20, 0x27, + 0x27, 0x7d, 0x24, 0x7b, 0x6e, 0x61, 0x6d, 0x65, 0x20, 0x3f, 0x20, 0x27, + 0x2d, 0x27, 0x20, 0x3a, 0x20, 0x27, 0x27, 0x7d, 0x61, 0x64, 0x64, 0x69, + 0x74, 0x69, 0x6f, 0x6e, 0x61, 0x6c, 0x60, 0x3b, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x76, 0x61, 0x6c, + 0x75, 0x65, 0x52, 0x75, 0x6c, 0x65, 0x20, 0x3d, 0x20, 0x74, 0x68, 0x69, + 0x73, 0x2e, 0x76, 0x69, 0x73, 0x69, 0x74, 0x28, 0x61, 0x64, 0x64, 0x69, + 0x74, 0x69, 0x6f, 0x6e, 0x61, 0x6c, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, + 0x74, 0x69, 0x65, 0x73, 0x20, 0x3d, 0x3d, 0x3d, 0x20, 0x74, 0x72, 0x75, + 0x65, 0x20, 0x3f, 0x20, 0x7b, 0x7d, 0x20, 0x3a, 0x20, 0x61, 0x64, 0x64, + 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x61, 0x6c, 0x50, 0x72, 0x6f, 0x70, 0x65, + 0x72, 0x74, 0x69, 0x65, 0x73, 0x2c, 0x20, 0x60, 0x24, 0x7b, 0x73, 0x75, + 0x62, 0x4e, 0x61, 0x6d, 0x65, 0x7d, 0x2d, 0x76, 0x61, 0x6c, 0x75, 0x65, + 0x60, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x70, 0x72, + 0x6f, 0x70, 0x4b, 0x76, 0x52, 0x75, 0x6c, 0x65, 0x4e, 0x61, 0x6d, 0x65, + 0x73, 0x5b, 0x27, 0x2a, 0x27, 0x5d, 0x20, 0x3d, 0x20, 0x74, 0x68, 0x69, + 0x73, 0x2e, 0x5f, 0x61, 0x64, 0x64, 0x52, 0x75, 0x6c, 0x65, 0x28, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x60, 0x24, 0x7b, 0x73, + 0x75, 0x62, 0x4e, 0x61, 0x6d, 0x65, 0x7d, 0x2d, 0x6b, 0x76, 0x60, 0x2c, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x60, 0x24, 0x7b, + 0x74, 0x68, 0x69, 0x73, 0x2e, 0x5f, 0x61, 0x64, 0x64, 0x52, 0x75, 0x6c, + 0x65, 0x28, 0x27, 0x73, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x27, 0x2c, 0x20, + 0x50, 0x52, 0x49, 0x4d, 0x49, 0x54, 0x49, 0x56, 0x45, 0x5f, 0x52, 0x55, + 0x4c, 0x45, 0x53, 0x5b, 0x27, 0x73, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x27, + 0x5d, 0x29, 0x7d, 0x20, 0x22, 0x3a, 0x22, 0x20, 0x73, 0x70, 0x61, 0x63, + 0x65, 0x20, 0x24, 0x7b, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x75, 0x6c, + 0x65, 0x7d, 0x60, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x61, 0x6c, 0x50, 0x72, 0x6f, 0x70, + 0x73, 0x2e, 0x70, 0x75, 0x73, 0x68, 0x28, 0x27, 0x2a, 0x27, 0x29, 0x3b, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x6c, 0x65, 0x74, 0x20, 0x72, 0x75, 0x6c, 0x65, 0x20, 0x3d, 0x20, 0x27, + 0x22, 0x7b, 0x22, 0x20, 0x73, 0x70, 0x61, 0x63, 0x65, 0x20, 0x27, 0x3b, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x72, 0x75, 0x6c, 0x65, 0x20, 0x2b, 0x3d, + 0x20, 0x72, 0x65, 0x71, 0x75, 0x69, 0x72, 0x65, 0x64, 0x50, 0x72, 0x6f, + 0x70, 0x73, 0x2e, 0x6d, 0x61, 0x70, 0x28, 0x6b, 0x20, 0x3d, 0x3e, 0x20, + 0x70, 0x72, 0x6f, 0x70, 0x4b, 0x76, 0x52, 0x75, 0x6c, 0x65, 0x4e, 0x61, + 0x6d, 0x65, 0x73, 0x5b, 0x6b, 0x5d, 0x29, 0x2e, 0x6a, 0x6f, 0x69, 0x6e, + 0x28, 0x27, 0x20, 0x22, 0x2c, 0x22, 0x20, 0x73, 0x70, 0x61, 0x63, 0x65, + 0x20, 0x27, 0x29, 0x3b, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, + 0x20, 0x28, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x61, 0x6c, 0x50, 0x72, + 0x6f, 0x70, 0x73, 0x2e, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x20, 0x3e, + 0x20, 0x30, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x72, 0x75, 0x6c, 0x65, 0x20, 0x2b, 0x3d, 0x20, 0x27, 0x20, 0x28, 0x27, + 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, + 0x72, 0x65, 0x71, 0x75, 0x69, 0x72, 0x65, 0x64, 0x50, 0x72, 0x6f, 0x70, + 0x73, 0x2e, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x20, 0x3e, 0x20, 0x30, + 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x72, 0x75, 0x6c, 0x65, 0x20, 0x2b, 0x3d, 0x20, 0x27, 0x20, 0x22, 0x2c, + 0x22, 0x20, 0x73, 0x70, 0x61, 0x63, 0x65, 0x20, 0x28, 0x20, 0x27, 0x3b, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x67, 0x65, + 0x74, 0x52, 0x65, 0x63, 0x75, 0x72, 0x73, 0x69, 0x76, 0x65, 0x52, 0x65, + 0x66, 0x73, 0x20, 0x3d, 0x20, 0x28, 0x6b, 0x73, 0x2c, 0x20, 0x66, 0x69, + 0x72, 0x73, 0x74, 0x49, 0x73, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x61, + 0x6c, 0x29, 0x20, 0x3d, 0x3e, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x5b, 0x6b, + 0x2c, 0x20, 0x2e, 0x2e, 0x2e, 0x72, 0x65, 0x73, 0x74, 0x5d, 0x20, 0x3d, + 0x20, 0x6b, 0x73, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x6b, 0x76, 0x52, 0x75, 0x6c, + 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x20, 0x3d, 0x20, 0x70, 0x72, 0x6f, 0x70, + 0x4b, 0x76, 0x52, 0x75, 0x6c, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x73, 0x5b, + 0x6b, 0x5d, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x6c, 0x65, 0x74, 0x20, 0x72, 0x65, 0x73, 0x3b, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, 0x6b, 0x20, 0x3d, + 0x3d, 0x3d, 0x20, 0x27, 0x2a, 0x27, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, + 0x73, 0x20, 0x3d, 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x5f, 0x61, 0x64, + 0x64, 0x52, 0x75, 0x6c, 0x65, 0x28, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x60, + 0x24, 0x7b, 0x6e, 0x61, 0x6d, 0x65, 0x20, 0x3f, 0x3f, 0x20, 0x27, 0x27, + 0x7d, 0x24, 0x7b, 0x6e, 0x61, 0x6d, 0x65, 0x20, 0x3f, 0x20, 0x27, 0x2d, + 0x27, 0x20, 0x3a, 0x20, 0x27, 0x27, 0x7d, 0x61, 0x64, 0x64, 0x69, 0x74, + 0x69, 0x6f, 0x6e, 0x61, 0x6c, 0x2d, 0x6b, 0x76, 0x73, 0x60, 0x2c, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x60, 0x24, 0x7b, 0x6b, 0x76, 0x52, 0x75, 0x6c, + 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x7d, 0x20, 0x28, 0x20, 0x22, 0x2c, 0x22, + 0x20, 0x73, 0x70, 0x61, 0x63, 0x65, 0x20, 0x60, 0x20, 0x2b, 0x20, 0x6b, + 0x76, 0x52, 0x75, 0x6c, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x20, 0x2b, 0x20, + 0x60, 0x20, 0x29, 0x2a, 0x60, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x7d, 0x20, 0x65, 0x6c, 0x73, 0x65, 0x20, 0x69, + 0x66, 0x20, 0x28, 0x66, 0x69, 0x72, 0x73, 0x74, 0x49, 0x73, 0x4f, 0x70, + 0x74, 0x69, 0x6f, 0x6e, 0x61, 0x6c, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, 0x73, 0x20, + 0x3d, 0x20, 0x60, 0x28, 0x20, 0x22, 0x2c, 0x22, 0x20, 0x73, 0x70, 0x61, + 0x63, 0x65, 0x20, 0x24, 0x7b, 0x6b, 0x76, 0x52, 0x75, 0x6c, 0x65, 0x4e, + 0x61, 0x6d, 0x65, 0x7d, 0x20, 0x29, 0x3f, 0x60, 0x3b, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x20, 0x65, 0x6c, 0x73, 0x65, + 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x72, 0x65, 0x73, 0x20, 0x3d, 0x20, 0x6b, 0x76, 0x52, 0x75, 0x6c, + 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x69, 0x66, 0x20, 0x28, 0x72, 0x65, 0x73, 0x74, 0x2e, 0x6c, 0x65, + 0x6e, 0x67, 0x74, 0x68, 0x20, 0x3e, 0x20, 0x30, 0x29, 0x20, 0x7b, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, + 0x73, 0x20, 0x2b, 0x3d, 0x20, 0x27, 0x20, 0x27, 0x20, 0x2b, 0x20, 0x74, + 0x68, 0x69, 0x73, 0x2e, 0x5f, 0x61, 0x64, 0x64, 0x52, 0x75, 0x6c, 0x65, + 0x28, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x60, 0x24, 0x7b, 0x6e, 0x61, 0x6d, 0x65, 0x20, 0x3f, 0x3f, + 0x20, 0x27, 0x27, 0x7d, 0x24, 0x7b, 0x6e, 0x61, 0x6d, 0x65, 0x20, 0x3f, + 0x20, 0x27, 0x2d, 0x27, 0x20, 0x3a, 0x20, 0x27, 0x27, 0x7d, 0x24, 0x7b, + 0x6b, 0x7d, 0x2d, 0x72, 0x65, 0x73, 0x74, 0x60, 0x2c, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x67, 0x65, + 0x74, 0x52, 0x65, 0x63, 0x75, 0x72, 0x73, 0x69, 0x76, 0x65, 0x52, 0x65, + 0x66, 0x73, 0x28, 0x72, 0x65, 0x73, 0x74, 0x2c, 0x20, 0x74, 0x72, 0x75, + 0x65, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, + 0x74, 0x75, 0x72, 0x6e, 0x20, 0x72, 0x65, 0x73, 0x3b, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x7d, 0x3b, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x72, 0x75, 0x6c, 0x65, 0x20, 0x2b, 0x3d, 0x20, 0x6f, 0x70, + 0x74, 0x69, 0x6f, 0x6e, 0x61, 0x6c, 0x50, 0x72, 0x6f, 0x70, 0x73, 0x2e, + 0x6d, 0x61, 0x70, 0x28, 0x28, 0x5f, 0x2c, 0x20, 0x69, 0x29, 0x20, 0x3d, + 0x3e, 0x20, 0x67, 0x65, 0x74, 0x52, 0x65, 0x63, 0x75, 0x72, 0x73, 0x69, + 0x76, 0x65, 0x52, 0x65, 0x66, 0x73, 0x28, 0x6f, 0x70, 0x74, 0x69, 0x6f, + 0x6e, 0x61, 0x6c, 0x50, 0x72, 0x6f, 0x70, 0x73, 0x2e, 0x73, 0x6c, 0x69, + 0x63, 0x65, 0x28, 0x69, 0x29, 0x2c, 0x20, 0x66, 0x61, 0x6c, 0x73, 0x65, + 0x29, 0x29, 0x2e, 0x6a, 0x6f, 0x69, 0x6e, 0x28, 0x27, 0x20, 0x7c, 0x20, + 0x27, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, + 0x20, 0x28, 0x72, 0x65, 0x71, 0x75, 0x69, 0x72, 0x65, 0x64, 0x50, 0x72, + 0x6f, 0x70, 0x73, 0x2e, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x20, 0x3e, + 0x20, 0x30, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x72, 0x75, 0x6c, 0x65, 0x20, 0x2b, 0x3d, 0x20, 0x27, 0x20, + 0x29, 0x27, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, 0x75, 0x6c, 0x65, 0x20, 0x2b, + 0x3d, 0x20, 0x27, 0x20, 0x29, 0x3f, 0x27, 0x3b, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x7d, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x72, 0x75, 0x6c, 0x65, + 0x20, 0x2b, 0x3d, 0x20, 0x27, 0x20, 0x22, 0x7d, 0x22, 0x20, 0x73, 0x70, + 0x61, 0x63, 0x65, 0x27, 0x3b, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x72, + 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x72, 0x75, 0x6c, 0x65, 0x3b, 0x0a, + 0x20, 0x20, 0x7d, 0x0a, 0x0a, 0x20, 0x20, 0x66, 0x6f, 0x72, 0x6d, 0x61, + 0x74, 0x47, 0x72, 0x61, 0x6d, 0x6d, 0x61, 0x72, 0x28, 0x29, 0x20, 0x7b, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x6c, 0x65, 0x74, 0x20, 0x67, 0x72, 0x61, + 0x6d, 0x6d, 0x61, 0x72, 0x20, 0x3d, 0x20, 0x27, 0x27, 0x3b, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x66, 0x6f, 0x72, 0x20, 0x28, 0x63, 0x6f, 0x6e, 0x73, + 0x74, 0x20, 0x5b, 0x6e, 0x61, 0x6d, 0x65, 0x2c, 0x20, 0x72, 0x75, 0x6c, + 0x65, 0x5d, 0x20, 0x6f, 0x66, 0x20, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, + 0x2e, 0x65, 0x6e, 0x74, 0x72, 0x69, 0x65, 0x73, 0x28, 0x74, 0x68, 0x69, + 0x73, 0x2e, 0x5f, 0x72, 0x75, 0x6c, 0x65, 0x73, 0x29, 0x2e, 0x73, 0x6f, + 0x72, 0x74, 0x28, 0x28, 0x5b, 0x61, 0x5d, 0x2c, 0x20, 0x5b, 0x62, 0x5d, + 0x29, 0x20, 0x3d, 0x3e, 0x20, 0x61, 0x2e, 0x6c, 0x6f, 0x63, 0x61, 0x6c, + 0x65, 0x43, 0x6f, 0x6d, 0x70, 0x61, 0x72, 0x65, 0x28, 0x62, 0x29, 0x29, + 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x67, 0x72, + 0x61, 0x6d, 0x6d, 0x61, 0x72, 0x20, 0x2b, 0x3d, 0x20, 0x60, 0x24, 0x7b, + 0x6e, 0x61, 0x6d, 0x65, 0x7d, 0x20, 0x3a, 0x3a, 0x3d, 0x20, 0x24, 0x7b, + 0x72, 0x75, 0x6c, 0x65, 0x7d, 0x5c, 0x6e, 0x60, 0x3b, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, + 0x72, 0x6e, 0x20, 0x67, 0x72, 0x61, 0x6d, 0x6d, 0x61, 0x72, 0x3b, 0x0a, + 0x20, 0x20, 0x7d, 0x0a, 0x7d, 0x0a, 0x0a, 0x2f, 0x2f, 0x20, 0x48, 0x65, + 0x6c, 0x70, 0x65, 0x72, 0x20, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, + 0x6e, 0x20, 0x74, 0x6f, 0x20, 0x67, 0x72, 0x6f, 0x75, 0x70, 0x20, 0x65, + 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x20, 0x62, 0x79, 0x20, 0x61, + 0x20, 0x6b, 0x65, 0x79, 0x20, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, + 0x6e, 0x0a, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x2a, 0x20, + 0x67, 0x72, 0x6f, 0x75, 0x70, 0x42, 0x79, 0x28, 0x69, 0x74, 0x65, 0x72, + 0x61, 0x62, 0x6c, 0x65, 0x2c, 0x20, 0x6b, 0x65, 0x79, 0x46, 0x6e, 0x29, + 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x6c, 0x65, 0x74, 0x20, 0x6c, 0x61, 0x73, + 0x74, 0x4b, 0x65, 0x79, 0x20, 0x3d, 0x20, 0x6e, 0x75, 0x6c, 0x6c, 0x3b, + 0x0a, 0x20, 0x20, 0x6c, 0x65, 0x74, 0x20, 0x67, 0x72, 0x6f, 0x75, 0x70, + 0x20, 0x3d, 0x20, 0x5b, 0x5d, 0x3b, 0x0a, 0x20, 0x20, 0x66, 0x6f, 0x72, + 0x20, 0x28, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x65, 0x6c, 0x65, 0x6d, + 0x65, 0x6e, 0x74, 0x20, 0x6f, 0x66, 0x20, 0x69, 0x74, 0x65, 0x72, 0x61, + 0x62, 0x6c, 0x65, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x63, + 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x6b, 0x65, 0x79, 0x20, 0x3d, 0x20, 0x6b, + 0x65, 0x79, 0x46, 0x6e, 0x28, 0x65, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, + 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, 0x6c, + 0x61, 0x73, 0x74, 0x4b, 0x65, 0x79, 0x20, 0x21, 0x3d, 0x3d, 0x20, 0x6e, + 0x75, 0x6c, 0x6c, 0x20, 0x26, 0x26, 0x20, 0x6b, 0x65, 0x79, 0x20, 0x21, + 0x3d, 0x3d, 0x20, 0x6c, 0x61, 0x73, 0x74, 0x4b, 0x65, 0x79, 0x29, 0x20, + 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x79, 0x69, 0x65, 0x6c, + 0x64, 0x20, 0x5b, 0x6c, 0x61, 0x73, 0x74, 0x4b, 0x65, 0x79, 0x2c, 0x20, + 0x67, 0x72, 0x6f, 0x75, 0x70, 0x5d, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x67, 0x72, 0x6f, 0x75, 0x70, 0x20, 0x3d, 0x20, 0x5b, 0x5d, + 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x67, 0x72, 0x6f, 0x75, 0x70, 0x2e, 0x70, 0x75, 0x73, 0x68, 0x28, 0x65, + 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x6c, 0x61, 0x73, 0x74, 0x4b, 0x65, 0x79, 0x20, 0x3d, 0x20, 0x6b, + 0x65, 0x79, 0x3b, 0x0a, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x69, 0x66, + 0x20, 0x28, 0x67, 0x72, 0x6f, 0x75, 0x70, 0x2e, 0x6c, 0x65, 0x6e, 0x67, + 0x74, 0x68, 0x20, 0x3e, 0x20, 0x30, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x79, 0x69, 0x65, 0x6c, 0x64, 0x20, 0x5b, 0x6c, 0x61, 0x73, + 0x74, 0x4b, 0x65, 0x79, 0x2c, 0x20, 0x67, 0x72, 0x6f, 0x75, 0x70, 0x5d, + 0x3b, 0x0a, 0x20, 0x20, 0x7d, 0x0a, 0x7d, 0x0a }; -unsigned int json_schema_to_grammar_mjs_len = 3695; +size_t json_schema_to_grammar_mjs_len = 19964; diff --git a/examples/server/public/index.html b/examples/server/public/index.html index 84038ddce..bbc5c2f68 100644 --- a/examples/server/public/index.html +++ b/examples/server/public/index.html @@ -630,14 +630,16 @@ const grammarJsonSchemaPropOrder = signal('') const updateGrammarJsonSchemaPropOrder = (el) => grammarJsonSchemaPropOrder.value = el.target.value - const convertJSONSchemaGrammar = () => { + const convertJSONSchemaGrammar = async () => { try { - const schema = JSON.parse(params.value.grammar) - const converter = new SchemaConverter( - grammarJsonSchemaPropOrder.value + let schema = JSON.parse(params.value.grammar) + const converter = new SchemaConverter({ + prop_order: grammarJsonSchemaPropOrder.value .split(',') - .reduce((acc, cur, i) => ({ ...acc, [cur.trim()]: i }), {}) - ) + .reduce((acc, cur, i) => ({ ...acc, [cur.trim()]: i }), {}), + allow_fetch: true, + }) + schema = await converter.resolveRefs(schema, 'input') converter.visit(schema, '') params.value = { ...params.value, diff --git a/examples/server/public/index.js b/examples/server/public/index.js index 9db5a9d9f..aa3ffb5e0 100644 --- a/examples/server/public/index.js +++ b/examples/server/public/index.js @@ -1 +1 @@ -function t(){throw new Error("Cycle detected")}function n(){if(u>1){u--;return}let t,n=!1;while(void 0!==_){let i=_;_=void 0;f++;while(void 0!==i){const _=i.o;i.o=void 0;i.f&=-3;if(!(8&i.f)&&a(i))try{i.c()}catch(e){if(!n){t=e;n=!0}}i=_}}f=0;u--;if(n)throw t}function e(t){if(u>0)return t();u++;try{return t()}finally{n()}}let i,_,o=0;function r(t){if(o>0)return t();const n=i;i=void 0;o++;try{return t()}finally{o--;i=n}}let u=0,f=0,l=0;function s(t){if(void 0===i)return;let n=t.n;if(void 0===n||n.t!==i){n={i:0,S:t,p:i.s,n:void 0,t:i,e:void 0,x:void 0,r:n};if(void 0!==i.s)i.s.n=n;i.s=n;t.n=n;if(32&i.f)t.S(n);return n}else if(-1===n.i){n.i=0;if(void 0!==n.n){n.n.p=n.p;if(void 0!==n.p)n.p.n=n.n;n.p=i.s;n.n=void 0;i.s.n=n;i.s=n}return n}}function c(t){this.v=t;this.i=0;this.n=void 0;this.t=void 0}c.prototype.h=function(){return!0};c.prototype.S=function(t){if(this.t!==t&&void 0===t.e){t.x=this.t;if(void 0!==this.t)this.t.e=t;this.t=t}};c.prototype.U=function(t){if(void 0!==this.t){const n=t.e,e=t.x;if(void 0!==n){n.x=e;t.e=void 0}if(void 0!==e){e.e=n;t.x=void 0}if(t===this.t)this.t=e}};c.prototype.subscribe=function(t){const n=this;return S((function(){const e=n.value,i=32&this.f;this.f&=-33;try{t(e)}finally{this.f|=i}}))};c.prototype.valueOf=function(){return this.value};c.prototype.toString=function(){return this.value+""};c.prototype.toJSON=function(){return this.value};c.prototype.peek=function(){return this.v};Object.defineProperty(c.prototype,"value",{get(){const t=s(this);if(void 0!==t)t.i=this.i;return this.v},set(e){if(i instanceof v)!function(){throw new Error("Computed cannot have side-effects")}();if(e!==this.v){if(f>100)t();this.v=e;this.i++;l++;u++;try{for(let t=this.t;void 0!==t;t=t.x)t.t.N()}finally{n()}}}});function h(t){return new c(t)}function a(t){for(let n=t.s;void 0!==n;n=n.n)if(n.S.i!==n.i||!n.S.h()||n.S.i!==n.i)return!0;return!1}function p(t){for(let n=t.s;void 0!==n;n=n.n){const e=n.S.n;if(void 0!==e)n.r=e;n.S.n=n;n.i=-1;if(void 0===n.n){t.s=n;break}}}function d(t){let n,e=t.s;while(void 0!==e){const t=e.p;if(-1===e.i){e.S.U(e);if(void 0!==t)t.n=e.n;if(void 0!==e.n)e.n.p=t}else n=e;e.S.n=e.r;if(void 0!==e.r)e.r=void 0;e=t}t.s=n}function v(t){c.call(this,void 0);this.x=t;this.s=void 0;this.g=l-1;this.f=4}(v.prototype=new c).h=function(){this.f&=-3;if(1&this.f)return!1;if(32==(36&this.f))return!0;this.f&=-5;if(this.g===l)return!0;this.g=l;this.f|=1;if(this.i>0&&!a(this)){this.f&=-2;return!0}const t=i;try{p(this);i=this;const t=this.x();if(16&this.f||this.v!==t||0===this.i){this.v=t;this.f&=-17;this.i++}}catch(t){this.v=t;this.f|=16;this.i++}i=t;d(this);this.f&=-2;return!0};v.prototype.S=function(t){if(void 0===this.t){this.f|=36;for(let t=this.s;void 0!==t;t=t.n)t.S.S(t)}c.prototype.S.call(this,t)};v.prototype.U=function(t){if(void 0!==this.t){c.prototype.U.call(this,t);if(void 0===this.t){this.f&=-33;for(let t=this.s;void 0!==t;t=t.n)t.S.U(t)}}};v.prototype.N=function(){if(!(2&this.f)){this.f|=6;for(let t=this.t;void 0!==t;t=t.x)t.t.N()}};v.prototype.peek=function(){if(!this.h())t();if(16&this.f)throw this.v;return this.v};Object.defineProperty(v.prototype,"value",{get(){if(1&this.f)t();const n=s(this);this.h();if(void 0!==n)n.i=this.i;if(16&this.f)throw this.v;return this.v}});function y(t){return new v(t)}function m(t){const e=t.u;t.u=void 0;if("function"==typeof e){u++;const _=i;i=void 0;try{e()}catch(n){t.f&=-2;t.f|=8;g(t);throw n}finally{i=_;n()}}}function g(t){for(let n=t.s;void 0!==n;n=n.n)n.S.U(n);t.x=void 0;t.s=void 0;m(t)}function b(t){if(i!==this)throw new Error("Out-of-order effect");d(this);i=t;this.f&=-2;if(8&this.f)g(this);n()}function k(t){this.x=t;this.u=void 0;this.s=void 0;this.o=void 0;this.f=32}k.prototype.c=function(){const t=this.S();try{if(8&this.f)return;if(void 0===this.x)return;const n=this.x();if("function"==typeof n)this.u=n}finally{t()}};k.prototype.S=function(){if(1&this.f)t();this.f|=1;this.f&=-9;m(this);p(this);u++;const n=i;i=this;return b.bind(this,n)};k.prototype.N=function(){if(!(2&this.f)){this.f|=2;this.o=_;_=this}};k.prototype.d=function(){this.f|=8;if(!(1&this.f))g(this)};function S(t){const n=new k(t);try{n.c()}catch(t){n.d();throw t}return n.d.bind(n)}var x,w,C,E,U,H,N,P,$,D={},T=[],V=/acit|ex(?:s|g|n|p|$)|rph|grid|ows|mnc|ntw|ine[ch]|zoo|^ord|itera/i,A=Array.isArray;function F(t,n){for(var e in n)t[e]=n[e];return t}function M(t){var n=t.parentNode;n&&n.removeChild(t)}function W(t,n,e){var i,_,o,r={};for(o in n)"key"==o?i=n[o]:"ref"==o?_=n[o]:r[o]=n[o];if(arguments.length>2&&(r.children=arguments.length>3?x.call(arguments,2):e),"function"==typeof t&&null!=t.defaultProps)for(o in t.defaultProps)void 0===r[o]&&(r[o]=t.defaultProps[o]);return O(t,r,i,_,null)}function O(t,n,e,i,_){var o={type:t,props:n,key:e,ref:i,__k:null,__:null,__b:0,__e:null,__d:void 0,__c:null,__h:null,constructor:void 0,__v:null==_?++C:_};return null==_&&null!=w.vnode&&w.vnode(o),o}function L(){return{current:null}}function R(t){return t.children}function I(t,n){this.props=t,this.context=n}function j(t,n){if(null==n)return t.__?j(t.__,t.__.__k.indexOf(t)+1):null;for(var e;nn&&U.sort(P));G.__r=0}function z(t,n,e,i,_,o,r,u,f,l,s){var c,h,a,p,d,v,y,m,g,b,k=0,S=i&&i.__k||T,x=S.length,w=x,C=n.length;for(e.__k=[],c=0;c0?O(p.type,p.props,p.key,p.ref?p.ref:null,p.__v):p)&&(p.__=e,p.__b=e.__b+1,-1===(m=X(p,S,y=c+k,w))?a=D:(a=S[m]||D,S[m]=void 0,w--),it(t,p,a,_,o,r,u,f,l,s),d=p.__e,(h=p.ref)&&a.ref!=h&&(a.ref&&rt(a.ref,null,p),s.push(h,p.__c||d,p)),null!=d&&(null==v&&(v=d),b=!(g=a===D||null===a.__v)&&m===y,g?-1==m&&k--:m!==y&&(m===y+1?(k++,b=!0):m>y?w>C-y?(k+=m-y,b=!0):k--:k=m(null!=f?1:0))for(;r>=0||u=0){if((f=n[r])&&_==f.key&&o===f.type)return r;r--}if(u2&&(u.children=arguments.length>3?x.call(arguments,2):e),O(t.type,u,i||t.key,_||t.ref,null)}function ht(t,n){var e={__c:n="__cC"+$++,__:t,Consumer:function(t,n){return t.children(n)},Provider:function(t){var e,i;return this.getChildContext||(e=[],(i={})[n]=this,this.getChildContext=function(){return i},this.shouldComponentUpdate=function(t){this.props.value!==t.value&&e.some((function(t){t.__e=!0,q(t)}))},this.sub=function(t){e.push(t);var n=t.componentWillUnmount;t.componentWillUnmount=function(){e.splice(e.indexOf(t),1),n&&n.call(t)}}),t.children}};return e.Provider.__=e.Consumer.contextType=e}x=T.slice,w={__e:function(t,n,e,i){for(var _,o,r;n=n.__;)if((_=n.__c)&&!_.__)try{if((o=_.constructor)&&null!=o.getDerivedStateFromError&&(_.setState(o.getDerivedStateFromError(t)),r=_.__d),null!=_.componentDidCatch&&(_.componentDidCatch(t,i||{}),r=_.__d),r)return _.__E=_}catch(n){t=n}throw t}},C=0,E=function(t){return null!=t&&void 0===t.constructor},I.prototype.setState=function(t,n){var e;e=null!=this.__s&&this.__s!==this.state?this.__s:this.__s=F({},this.state),"function"==typeof t&&(t=t(F({},e),this.props)),t&&F(e,t),null!=t&&this.__v&&(n&&this._sb.push(n),q(this))},I.prototype.forceUpdate=function(t){this.__v&&(this.__e=!0,t&&this.__h.push(t),q(this))},I.prototype.render=R,U=[],N="function"==typeof Promise?Promise.prototype.then.bind(Promise.resolve()):setTimeout,P=function(t,n){return t.__v.__b-n.__v.__b},G.__r=0,$=0;var at,pt,dt,vt,yt=0,mt=[],gt=[],bt=w.__b,kt=w.__r,St=w.diffed,xt=w.__c,wt=w.unmount;function Ct(t,n){w.__h&&w.__h(pt,t,yt||n),yt=0;var e=pt.__H||(pt.__H={__:[],__h:[]});return t>=e.__.length&&e.__.push({__V:gt}),e.__[t]}function Et(t){return yt=1,Ut(Bt,t)}function Ut(t,n,e){var i=Ct(at++,2);if(i.t=t,!i.__c&&(i.__=[e?e(n):Bt(void 0,n),function(t){var n=i.__N?i.__N[0]:i.__[0],e=i.t(n,t);n!==e&&(i.__N=[e,i.__[1]],i.__c.setState({}))}],i.__c=pt,!pt.u)){var _=function(t,n,e){if(!i.__c.__H)return!0;var _=i.__c.__H.__.filter((function(t){return t.__c}));if(_.every((function(t){return!t.__N})))return!o||o.call(this,t,n,e);var r=!1;return _.forEach((function(t){if(t.__N){var n=t.__[0];t.__=t.__N,t.__N=void 0,n!==t.__[0]&&(r=!0)}})),!(!r&&i.__c.props===t)&&(!o||o.call(this,t,n,e))};pt.u=!0;var o=pt.shouldComponentUpdate,r=pt.componentWillUpdate;pt.componentWillUpdate=function(t,n,e){if(this.__e){var i=o;o=void 0,_(t,n,e),o=i}r&&r.call(this,t,n,e)},pt.shouldComponentUpdate=_}return i.__N||i.__}function Ht(t,n){var e=Ct(at++,3);!w.__s&&jt(e.__H,n)&&(e.__=t,e.i=n,pt.__H.__h.push(e))}function Nt(t,n){var e=Ct(at++,4);!w.__s&&jt(e.__H,n)&&(e.__=t,e.i=n,pt.__h.push(e))}function Pt(t){return yt=5,Dt((function(){return{current:t}}),[])}function $t(t,n,e){yt=6,Nt((function(){return"function"==typeof t?(t(n()),function(){return t(null)}):t?(t.current=n(),function(){return t.current=null}):void 0}),null==e?e:e.concat(t))}function Dt(t,n){var e=Ct(at++,7);return jt(e.__H,n)?(e.__V=t(),e.i=n,e.__h=t,e.__V):e.__}function Tt(t,n){return yt=8,Dt((function(){return t}),n)}function Vt(t){var n=pt.context[t.__c],e=Ct(at++,9);return e.c=t,n?(null==e.__&&(e.__=!0,n.sub(pt)),n.props.value):t.__}function At(t,n){w.useDebugValue&&w.useDebugValue(n?n(t):t)}function Ft(t){var n=Ct(at++,10),e=Et();return n.__=t,pt.componentDidCatch||(pt.componentDidCatch=function(t,i){n.__&&n.__(t,i),e[1](t)}),[e[0],function(){e[1](void 0)}]}function Mt(){var t=Ct(at++,11);if(!t.__){for(var n=pt.__v;null!==n&&!n.__m&&null!==n.__;)n=n.__;var e=n.__m||(n.__m=[0,0]);t.__="P"+e[0]+"-"+e[1]++}return t.__}function Wt(){for(var t;t=mt.shift();)if(t.__P&&t.__H)try{t.__H.__h.forEach(Rt),t.__H.__h.forEach(It),t.__H.__h=[]}catch(u){t.__H.__h=[],w.__e(u,t.__v)}}w.__b=function(t){pt=null,bt&&bt(t)},w.__r=function(t){kt&&kt(t),at=0;var n=(pt=t.__c).__H;n&&(dt===pt?(n.__h=[],pt.__h=[],n.__.forEach((function(t){t.__N&&(t.__=t.__N),t.__V=gt,t.__N=t.i=void 0}))):(n.__h.forEach(Rt),n.__h.forEach(It),n.__h=[],at=0)),dt=pt},w.diffed=function(t){St&&St(t);var n=t.__c;n&&n.__H&&(n.__H.__h.length&&(1!==mt.push(n)&&vt===w.requestAnimationFrame||((vt=w.requestAnimationFrame)||Lt)(Wt)),n.__H.__.forEach((function(t){t.i&&(t.__H=t.i),t.__V!==gt&&(t.__=t.__V),t.i=void 0,t.__V=gt}))),dt=pt=null},w.__c=function(t,n){n.some((function(t){try{t.__h.forEach(Rt),t.__h=t.__h.filter((function(t){return!t.__||It(t)}))}catch(s){n.some((function(t){t.__h&&(t.__h=[])})),n=[],w.__e(s,t.__v)}})),xt&&xt(t,n)},w.unmount=function(t){wt&&wt(t);var n,e=t.__c;e&&e.__H&&(e.__H.__.forEach((function(t){try{Rt(t)}catch(t){n=t}})),e.__H=void 0,n&&w.__e(n,e.__v))};var Ot="function"==typeof requestAnimationFrame;function Lt(t){var n,e=function(){clearTimeout(i),Ot&&cancelAnimationFrame(n),setTimeout(t)},i=setTimeout(e,100);Ot&&(n=requestAnimationFrame(e))}function Rt(t){var n=pt,e=t.__c;"function"==typeof e&&(t.__c=void 0,e()),pt=n}function It(t){var n=pt;t.__c=t.__(),pt=n}function jt(t,n){return!t||t.length!==n.length||n.some((function(n,e){return n!==t[e]}))}function Bt(t,n){return"function"==typeof n?n(t):n}function qt(t,n){w[t]=n.bind(null,w[t]||(()=>{}))}let Gt,zt;function Jt(t){if(zt)zt();zt=t&&t.S()}function Kt({data:t}){const n=Xt(t);n.value=t;const e=Dt(()=>{let t=this.__v;while(t=t.__)if(t.__c){t.__c.__$f|=4;break}this.__$u.c=()=>{var t;if(!E(e.peek())&&3===(null==(t=this.base)?void 0:t.nodeType))this.base.data=e.peek();else{this.__$f|=1;this.setState({})}};return y(()=>{let t=n.value.value;return 0===t?0:!0===t?"":t||""})},[]);return e.value}Kt.displayName="_st";Object.defineProperties(c.prototype,{constructor:{configurable:!0,value:void 0},type:{configurable:!0,value:Kt},props:{configurable:!0,get(){return{data:this}}},__b:{configurable:!0,value:1}});qt("__b",(t,n)=>{if("string"==typeof n.type){let t,e=n.props;for(let i in e){if("children"===i)continue;let _=e[i];if(_ instanceof c){if(!t)n.__np=t={};t[i]=_;e[i]=_.peek()}}}t(n)});qt("__r",(t,n)=>{Jt();let e,i=n.__c;if(i){i.__$f&=-2;e=i.__$u;if(void 0===e)i.__$u=e=function(t){let n;S((function(){n=this}));n.c=()=>{i.__$f|=1;i.setState({})};return n}()}Gt=i;Jt(e);t(n)});qt("__e",(t,n,e,i)=>{Jt();Gt=void 0;t(n,e,i)});qt("diffed",(t,n)=>{Jt();Gt=void 0;let e;if("string"==typeof n.type&&(e=n.__e)){let t=n.__np,i=n.props;if(t){let n=e.U;if(n)for(let e in n){let i=n[e];if(void 0!==i&&!(e in t)){i.d();n[e]=void 0}}else{n={};e.U=n}for(let _ in t){let o=n[_],r=t[_];if(void 0===o){o=Qt(e,_,r,i);n[_]=o}else o.o(r,i)}}}t(n)});function Qt(t,n,e,i){const _=n in t&&void 0===t.ownerSVGElement,o=h(e);return{o:(t,n)=>{o.value=t;i=n},d:S(()=>{const e=o.value.value;if(i[n]!==e){i[n]=e;if(_)t[n]=e;else if(e)t.setAttribute(n,e);else t.removeAttribute(n)}})}}qt("unmount",(t,n)=>{if("string"==typeof n.type){let t=n.__e;if(t){const n=t.U;if(n){t.U=void 0;for(let t in n){let e=n[t];if(e)e.d()}}}}else{let t=n.__c;if(t){const n=t.__$u;if(n){t.__$u=void 0;n.d()}}}t(n)});qt("__h",(t,n,e,i)=>{if(i<3||9===i)n.__$f|=2;t(n,e,i)});I.prototype.shouldComponentUpdate=function(t,n){const e=this.__$u;if(!(e&&void 0!==e.s||4&this.__$f))return!0;if(3&this.__$f)return!0;for(let i in n)return!0;for(let i in t)if("__source"!==i&&t[i]!==this.props[i])return!0;for(let i in this.props)if(!(i in t))return!0;return!1};function Xt(t){return Dt(()=>h(t),[])}function Yt(t){const n=Pt(t);n.current=t;Gt.__$f|=4;return Dt(()=>y(()=>n.current()),[])}function Zt(t){const n=Pt(t);n.current=t;Ht(()=>S(()=>n.current()),[])}var tn=function(t,n,e,i){var _;n[0]=0;for(var o=1;o=5&&((_||!t&&5===i)&&(r.push(i,0,_,e),i=6),t&&(r.push(i,t,0,e),i=6)),_=""},f=0;f"===n?(i=1,_=""):_=n+_[0]:o?n===o?o="":_+=n:'"'===n||"'"===n?o=n:">"===n?(u(),i=1):i&&("="===n?(i=5,e=_,_=""):"/"===n&&(i<5||">"===t[f][l+1])?(u(),3===i&&(r=r[0]),i=r,(r=r[0]).push(2,0,i),i=0):" "===n||"\t"===n||"\n"===n||"\r"===n?(u(),i=2):_+=n),3===i&&"!--"===_&&(i=4,r=r[0])}return u(),r}(t)),n),arguments,[])).length>1?n:n[0]}var _n=en.bind(W);export{I as Component,R as Fragment,c as Signal,e as batch,ct as cloneElement,y as computed,ht as createContext,W as createElement,L as createRef,S as effect,W as h,_n as html,st as hydrate,E as isValidElement,w as options,lt as render,h as signal,K as toChildArray,r as untracked,Tt as useCallback,Yt as useComputed,Vt as useContext,At as useDebugValue,Ht as useEffect,Ft as useErrorBoundary,Mt as useId,$t as useImperativeHandle,Nt as useLayoutEffect,Dt as useMemo,Ut as useReducer,Pt as useRef,Xt as useSignal,Zt as useSignalEffect,Et as useState}; +function t(){throw new Error("Cycle detected")}const n=Symbol.for("preact-signals");function e(){if(f>1){f--;return}let t,n=!1;while(void 0!==o){let _=o;o=void 0;l++;while(void 0!==_){const i=_.o;_.o=void 0;_.f&=-3;if(!(8&_.f)&&p(_))try{_.c()}catch(e){if(!n){t=e;n=!0}}_=i}}l=0;f--;if(n)throw t}function _(t){if(f>0)return t();f++;try{return t()}finally{e()}}let i,o,r=0;function u(t){if(r>0)return t();const n=i;i=void 0;r++;try{return t()}finally{r--;i=n}}let f=0,l=0,s=0;function c(t){if(void 0===i)return;let n=t.n;if(void 0===n||n.t!==i){n={i:0,S:t,p:i.s,n:void 0,t:i,e:void 0,x:void 0,r:n};if(void 0!==i.s)i.s.n=n;i.s=n;t.n=n;if(32&i.f)t.S(n);return n}else if(-1===n.i){n.i=0;if(void 0!==n.n){n.n.p=n.p;if(void 0!==n.p)n.p.n=n.n;n.p=i.s;n.n=void 0;i.s.n=n;i.s=n}return n}}function h(t){this.v=t;this.i=0;this.n=void 0;this.t=void 0}h.prototype.brand=n;h.prototype.h=function(){return!0};h.prototype.S=function(t){if(this.t!==t&&void 0===t.e){t.x=this.t;if(void 0!==this.t)this.t.e=t;this.t=t}};h.prototype.U=function(t){if(void 0!==this.t){const n=t.e,e=t.x;if(void 0!==n){n.x=e;t.e=void 0}if(void 0!==e){e.e=n;t.x=void 0}if(t===this.t)this.t=e}};h.prototype.subscribe=function(t){const n=this;return w((function(){const e=n.value,_=32&this.f;this.f&=-33;try{t(e)}finally{this.f|=_}}))};h.prototype.valueOf=function(){return this.value};h.prototype.toString=function(){return this.value+""};h.prototype.toJSON=function(){return this.value};h.prototype.peek=function(){return this.v};Object.defineProperty(h.prototype,"value",{get(){const t=c(this);if(void 0!==t)t.i=this.i;return this.v},set(n){if(i instanceof y)!function(){throw new Error("Computed cannot have side-effects")}();if(n!==this.v){if(l>100)t();this.v=n;this.i++;s++;f++;try{for(let t=this.t;void 0!==t;t=t.x)t.t.N()}finally{e()}}}});function a(t){return new h(t)}function p(t){for(let n=t.s;void 0!==n;n=n.n)if(n.S.i!==n.i||!n.S.h()||n.S.i!==n.i)return!0;return!1}function d(t){for(let n=t.s;void 0!==n;n=n.n){const e=n.S.n;if(void 0!==e)n.r=e;n.S.n=n;n.i=-1;if(void 0===n.n){t.s=n;break}}}function v(t){let n,e=t.s;while(void 0!==e){const t=e.p;if(-1===e.i){e.S.U(e);if(void 0!==t)t.n=e.n;if(void 0!==e.n)e.n.p=t}else n=e;e.S.n=e.r;if(void 0!==e.r)e.r=void 0;e=t}t.s=n}function y(t){h.call(this,void 0);this.x=t;this.s=void 0;this.g=s-1;this.f=4}(y.prototype=new h).h=function(){this.f&=-3;if(1&this.f)return!1;if(32==(36&this.f))return!0;this.f&=-5;if(this.g===s)return!0;this.g=s;this.f|=1;if(this.i>0&&!p(this)){this.f&=-2;return!0}const t=i;try{d(this);i=this;const t=this.x();if(16&this.f||this.v!==t||0===this.i){this.v=t;this.f&=-17;this.i++}}catch(t){this.v=t;this.f|=16;this.i++}i=t;v(this);this.f&=-2;return!0};y.prototype.S=function(t){if(void 0===this.t){this.f|=36;for(let t=this.s;void 0!==t;t=t.n)t.S.S(t)}h.prototype.S.call(this,t)};y.prototype.U=function(t){if(void 0!==this.t){h.prototype.U.call(this,t);if(void 0===this.t){this.f&=-33;for(let t=this.s;void 0!==t;t=t.n)t.S.U(t)}}};y.prototype.N=function(){if(!(2&this.f)){this.f|=6;for(let t=this.t;void 0!==t;t=t.x)t.t.N()}};y.prototype.peek=function(){if(!this.h())t();if(16&this.f)throw this.v;return this.v};Object.defineProperty(y.prototype,"value",{get(){if(1&this.f)t();const n=c(this);this.h();if(void 0!==n)n.i=this.i;if(16&this.f)throw this.v;return this.v}});function m(t){return new y(t)}function g(t){const n=t.u;t.u=void 0;if("function"==typeof n){f++;const _=i;i=void 0;try{n()}catch(n){t.f&=-2;t.f|=8;b(t);throw n}finally{i=_;e()}}}function b(t){for(let n=t.s;void 0!==n;n=n.n)n.S.U(n);t.x=void 0;t.s=void 0;g(t)}function k(t){if(i!==this)throw new Error("Out-of-order effect");v(this);i=t;this.f&=-2;if(8&this.f)b(this);e()}function S(t){this.x=t;this.u=void 0;this.s=void 0;this.o=void 0;this.f=32}S.prototype.c=function(){const t=this.S();try{if(8&this.f)return;if(void 0===this.x)return;const n=this.x();if("function"==typeof n)this.u=n}finally{t()}};S.prototype.S=function(){if(1&this.f)t();this.f|=1;this.f&=-9;g(this);d(this);f++;const n=i;i=this;return k.bind(this,n)};S.prototype.N=function(){if(!(2&this.f)){this.f|=2;this.o=o;o=this}};S.prototype.d=function(){this.f|=8;if(!(1&this.f))b(this)};function w(t){const n=new S(t);try{n.c()}catch(t){n.d();throw t}return n.d.bind(n)}var x,C,E,U,H,P,N,$,D,T={},V=[],A=/acit|ex(?:s|g|n|p|$)|rph|grid|ows|mnc|ntw|ine[ch]|zoo|^ord|itera/i,F=Array.isArray;function M(t,n){for(var e in n)t[e]=n[e];return t}function W(t){var n=t.parentNode;n&&n.removeChild(t)}function L(t,n,e){var _,i,o,r={};for(o in n)"key"==o?_=n[o]:"ref"==o?i=n[o]:r[o]=n[o];if(arguments.length>2&&(r.children=arguments.length>3?x.call(arguments,2):e),"function"==typeof t&&null!=t.defaultProps)for(o in t.defaultProps)void 0===r[o]&&(r[o]=t.defaultProps[o]);return O(t,r,_,i,null)}function O(t,n,e,_,i){var o={type:t,props:n,key:e,ref:_,__k:null,__:null,__b:0,__e:null,__d:void 0,__c:null,constructor:void 0,__v:null==i?++E:i,__i:-1,__u:0};return null==i&&null!=C.vnode&&C.vnode(o),o}function R(){return{current:null}}function j(t){return t.children}function I(t,n){this.props=t,this.context=n}function q(t,n){if(null==n)return t.__?q(t.__,t.__i+1):null;for(var e;ne?(ot(_,n,i),i.length=_.length=0,n=void 0,H.sort($)):n&&C.__c&&C.__c(n,V));n&&ot(_,n,i),J.__r=0}function K(t,n,e,_,i,o,r,u,f,l,s){var c,h,a,p,d,v=_&&_.__k||V,y=n.length;for(e.__d=f,Q(e,n,v),f=e.__d,c=0;c0?O(i.type,i.props,i.key,i.ref?i.ref:null,i.__v):i)?(i.__=t,i.__b=t.__b+1,u=Z(i,e,r,s),i.__i=u,o=null,-1!==u&&(s--,(o=e[u])&&(o.__u|=131072)),null==o||null===o.__v?(-1==u&&c--,"function"!=typeof i.type&&(i.__u|=65536)):u!==r&&(u===r+1?c++:u>r?s>f-r?c+=u-r:c--:u(null!=f&&0==(131072&f.__u)?1:0))for(;r>=0||u=0){if((f=n[r])&&0==(131072&f.__u)&&i==f.key&&o===f.type)return r;r--}if(u2&&(u.children=arguments.length>3?x.call(arguments,2):e),O(t.type,u,_||t.key,i||t.ref,null)}function at(t,n){var e={__c:n="__cC"+D++,__:t,Consumer:function(t,n){return t.children(n)},Provider:function(t){var e,_;return this.getChildContext||(e=[],(_={})[n]=this,this.getChildContext=function(){return _},this.shouldComponentUpdate=function(t){this.props.value!==t.value&&e.some((function(t){t.__e=!0,z(t)}))},this.sub=function(t){e.push(t);var n=t.componentWillUnmount;t.componentWillUnmount=function(){e.splice(e.indexOf(t),1),n&&n.call(t)}}),t.children}};return e.Provider.__=e.Consumer.contextType=e}x=V.slice,C={__e:function(t,n,e,_){for(var i,o,r;n=n.__;)if((i=n.__c)&&!i.__)try{if((o=i.constructor)&&null!=o.getDerivedStateFromError&&(i.setState(o.getDerivedStateFromError(t)),r=i.__d),null!=i.componentDidCatch&&(i.componentDidCatch(t,_||{}),r=i.__d),r)return i.__E=i}catch(n){t=n}throw t}},E=0,U=function(t){return null!=t&&null==t.constructor},I.prototype.setState=function(t,n){var e;e=null!=this.__s&&this.__s!==this.state?this.__s:this.__s=M({},this.state),"function"==typeof t&&(t=t(M({},e),this.props)),t&&M(e,t),null!=t&&this.__v&&(n&&this._sb.push(n),z(this))},I.prototype.forceUpdate=function(t){this.__v&&(this.__e=!0,t&&this.__h.push(t),z(this))},I.prototype.render=j,H=[],N="function"==typeof Promise?Promise.prototype.then.bind(Promise.resolve()):setTimeout,$=function(t,n){return t.__v.__b-n.__v.__b},J.__r=0,D=0;var pt,dt,vt,yt,mt=0,gt=[],bt=[],kt=C,St=kt.__b,wt=kt.__r,xt=kt.diffed,Ct=kt.__c,Et=kt.unmount,Ut=kt.__;function Ht(t,n){kt.__h&&kt.__h(dt,t,mt||n),mt=0;var e=dt.__H||(dt.__H={__:[],__h:[]});return t>=e.__.length&&e.__.push({__V:bt}),e.__[t]}function Pt(t){return mt=1,Nt(zt,t)}function Nt(t,n,e){var _=Ht(pt++,2);if(_.t=t,!_.__c&&(_.__=[e?e(n):zt(void 0,n),function(t){var n=_.__N?_.__N[0]:_.__[0],e=_.t(n,t);n!==e&&(_.__N=[e,_.__[1]],_.__c.setState({}))}],_.__c=dt,!dt.u)){var i=function(t,n,e){if(!_.__c.__H)return!0;var i=_.__c.__H.__.filter((function(t){return!!t.__c}));if(i.every((function(t){return!t.__N})))return!o||o.call(this,t,n,e);var r=!1;return i.forEach((function(t){if(t.__N){var n=t.__[0];t.__=t.__N,t.__N=void 0,n!==t.__[0]&&(r=!0)}})),!(!r&&_.__c.props===t)&&(!o||o.call(this,t,n,e))};dt.u=!0;var o=dt.shouldComponentUpdate,r=dt.componentWillUpdate;dt.componentWillUpdate=function(t,n,e){if(this.__e){var _=o;o=void 0,i(t,n,e),o=_}r&&r.call(this,t,n,e)},dt.shouldComponentUpdate=i}return _.__N||_.__}function $t(t,n){var e=Ht(pt++,3);!kt.__s&&Gt(e.__H,n)&&(e.__=t,e.i=n,dt.__H.__h.push(e))}function Dt(t,n){var e=Ht(pt++,4);!kt.__s&&Gt(e.__H,n)&&(e.__=t,e.i=n,dt.__h.push(e))}function Tt(t){return mt=5,At((function(){return{current:t}}),[])}function Vt(t,n,e){mt=6,Dt((function(){return"function"==typeof t?(t(n()),function(){return t(null)}):t?(t.current=n(),function(){return t.current=null}):void 0}),null==e?e:e.concat(t))}function At(t,n){var e=Ht(pt++,7);return Gt(e.__H,n)?(e.__V=t(),e.i=n,e.__h=t,e.__V):e.__}function Ft(t,n){return mt=8,At((function(){return t}),n)}function Mt(t){var n=dt.context[t.__c],e=Ht(pt++,9);return e.c=t,n?(null==e.__&&(e.__=!0,n.sub(dt)),n.props.value):t.__}function Wt(t,n){kt.useDebugValue&&kt.useDebugValue(n?n(t):t)}function Lt(t){var n=Ht(pt++,10),e=Pt();return n.__=t,dt.componentDidCatch||(dt.componentDidCatch=function(t,_){n.__&&n.__(t,_),e[1](t)}),[e[0],function(){e[1](void 0)}]}function Ot(){var t=Ht(pt++,11);if(!t.__){for(var n=dt.__v;null!==n&&!n.__m&&null!==n.__;)n=n.__;var e=n.__m||(n.__m=[0,0]);t.__="P"+e[0]+"-"+e[1]++}return t.__}function Rt(){for(var t;t=gt.shift();)if(t.__P&&t.__H)try{t.__H.__h.forEach(qt),t.__H.__h.forEach(Bt),t.__H.__h=[]}catch(n){t.__H.__h=[],kt.__e(n,t.__v)}}kt.__b=function(t){dt=null,St&&St(t)},kt.__=function(t,n){t&&n.__k&&n.__k.__m&&(t.__m=n.__k.__m),Ut&&Ut(t,n)},kt.__r=function(t){wt&&wt(t),pt=0;var n=(dt=t.__c).__H;n&&(vt===dt?(n.__h=[],dt.__h=[],n.__.forEach((function(t){t.__N&&(t.__=t.__N),t.__V=bt,t.__N=t.i=void 0}))):(n.__h.forEach(qt),n.__h.forEach(Bt),n.__h=[],pt=0)),vt=dt},kt.diffed=function(t){xt&&xt(t);var n=t.__c;n&&n.__H&&(n.__H.__h.length&&(1!==gt.push(n)&&yt===kt.requestAnimationFrame||((yt=kt.requestAnimationFrame)||It)(Rt)),n.__H.__.forEach((function(t){t.i&&(t.__H=t.i),t.__V!==bt&&(t.__=t.__V),t.i=void 0,t.__V=bt}))),vt=dt=null},kt.__c=function(t,n){n.some((function(t){try{t.__h.forEach(qt),t.__h=t.__h.filter((function(t){return!t.__||Bt(t)}))}catch(u){n.some((function(t){t.__h&&(t.__h=[])})),n=[],kt.__e(u,t.__v)}})),Ct&&Ct(t,n)},kt.unmount=function(t){Et&&Et(t);var n,e=t.__c;e&&e.__H&&(e.__H.__.forEach((function(t){try{qt(t)}catch(t){n=t}})),e.__H=void 0,n&&kt.__e(n,e.__v))};var jt="function"==typeof requestAnimationFrame;function It(t){var n,e=function(){clearTimeout(_),jt&&cancelAnimationFrame(n),setTimeout(t)},_=setTimeout(e,100);jt&&(n=requestAnimationFrame(e))}function qt(t){var n=dt,e=t.__c;"function"==typeof e&&(t.__c=void 0,e()),dt=n}function Bt(t){var n=dt;t.__c=t.__(),dt=n}function Gt(t,n){return!t||t.length!==n.length||n.some((function(n,e){return n!==t[e]}))}function zt(t,n){return"function"==typeof n?n(t):n}function Jt(t,n){C[t]=n.bind(null,C[t]||(()=>{}))}let Kt,Qt;function Xt(t){if(Qt)Qt();Qt=t&&t.S()}function Yt({data:t}){const n=tn(t);n.value=t;const e=At(()=>{let t=this.__v;while(t=t.__)if(t.__c){t.__c.__$f|=4;break}this.__$u.c=()=>{var t;if(!U(e.peek())&&3===(null==(t=this.base)?void 0:t.nodeType))this.base.data=e.peek();else{this.__$f|=1;this.setState({})}};return m(()=>{let t=n.value.value;return 0===t?0:!0===t?"":t||""})},[]);return e.value}Yt.displayName="_st";Object.defineProperties(h.prototype,{constructor:{configurable:!0,value:void 0},type:{configurable:!0,value:Yt},props:{configurable:!0,get(){return{data:this}}},__b:{configurable:!0,value:1}});Jt("__b",(t,n)=>{if("string"==typeof n.type){let t,e=n.props;for(let _ in e){if("children"===_)continue;let i=e[_];if(i instanceof h){if(!t)n.__np=t={};t[_]=i;e[_]=i.peek()}}}t(n)});Jt("__r",(t,n)=>{Xt();let e,_=n.__c;if(_){_.__$f&=-2;e=_.__$u;if(void 0===e)_.__$u=e=function(t){let n;w((function(){n=this}));n.c=()=>{_.__$f|=1;_.setState({})};return n}()}Kt=_;Xt(e);t(n)});Jt("__e",(t,n,e,_)=>{Xt();Kt=void 0;t(n,e,_)});Jt("diffed",(t,n)=>{Xt();Kt=void 0;let e;if("string"==typeof n.type&&(e=n.__e)){let t=n.__np,_=n.props;if(t){let n=e.U;if(n)for(let e in n){let _=n[e];if(void 0!==_&&!(e in t)){_.d();n[e]=void 0}}else{n={};e.U=n}for(let i in t){let o=n[i],r=t[i];if(void 0===o){o=Zt(e,i,r,_);n[i]=o}else o.o(r,_)}}}t(n)});function Zt(t,n,e,_){const i=n in t&&void 0===t.ownerSVGElement,o=a(e);return{o:(t,n)=>{o.value=t;_=n},d:w(()=>{const e=o.value.value;if(_[n]!==e){_[n]=e;if(i)t[n]=e;else if(e)t.setAttribute(n,e);else t.removeAttribute(n)}})}}Jt("unmount",(t,n)=>{if("string"==typeof n.type){let t=n.__e;if(t){const n=t.U;if(n){t.U=void 0;for(let t in n){let e=n[t];if(e)e.d()}}}}else{let t=n.__c;if(t){const n=t.__$u;if(n){t.__$u=void 0;n.d()}}}t(n)});Jt("__h",(t,n,e,_)=>{if(_<3||9===_)n.__$f|=2;t(n,e,_)});I.prototype.shouldComponentUpdate=function(t,n){const e=this.__$u;if(!(e&&void 0!==e.s||4&this.__$f))return!0;if(3&this.__$f)return!0;for(let _ in n)return!0;for(let _ in t)if("__source"!==_&&t[_]!==this.props[_])return!0;for(let _ in this.props)if(!(_ in t))return!0;return!1};function tn(t){return At(()=>a(t),[])}function nn(t){const n=Tt(t);n.current=t;Kt.__$f|=4;return At(()=>m(()=>n.current()),[])}function en(t){const n=Tt(t);n.current=t;$t(()=>w(()=>n.current()),[])}var _n=function(t,n,e,_){var i;n[0]=0;for(var o=1;o=5&&((i||!t&&5===_)&&(r.push(_,0,i,e),_=6),t&&(r.push(_,t,0,e),_=6)),i=""},f=0;f"===n?(_=1,i=""):i=n+i[0]:o?n===o?o="":i+=n:'"'===n||"'"===n?o=n:">"===n?(u(),_=1):_&&("="===n?(_=5,e=i,i=""):"/"===n&&(_<5||">"===t[f][l+1])?(u(),3===_&&(r=r[0]),_=r,(r=r[0]).push(2,0,_),_=0):" "===n||"\t"===n||"\n"===n||"\r"===n?(u(),_=2):i+=n),3===_&&"!--"===i&&(_=4,r=r[0])}return u(),r}(t)),n),arguments,[])).length>1?n:n[0]}var un=rn.bind(L);export{I as Component,j as Fragment,h as Signal,_ as batch,ht as cloneElement,m as computed,at as createContext,L as createElement,R as createRef,w as effect,L as h,un as html,ct as hydrate,U as isValidElement,C as options,st as render,a as signal,Y as toChildArray,u as untracked,Ft as useCallback,nn as useComputed,Mt as useContext,Wt as useDebugValue,$t as useEffect,Lt as useErrorBoundary,Ot as useId,Vt as useImperativeHandle,Dt as useLayoutEffect,At as useMemo,Nt as useReducer,Tt as useRef,tn as useSignal,en as useSignalEffect,Pt as useState}; diff --git a/examples/server/public/json-schema-to-grammar.mjs b/examples/server/public/json-schema-to-grammar.mjs index 3f1b255c2..baa6761c8 100644 --- a/examples/server/public/json-schema-to-grammar.mjs +++ b/examples/server/public/json-schema-to-grammar.mjs @@ -1,25 +1,50 @@ +// WARNING: This file was ported from json-schema-to-grammar.py, please fix bugs / add features there first. const SPACE_RULE = '" "?'; const PRIMITIVE_RULES = { boolean: '("true" | "false") space', number: '("-"? ([0-9] | [1-9] [0-9]*)) ("." [0-9]+)? ([eE] [-+]? [0-9]+)? space', integer: '("-"? ([0-9] | [1-9] [0-9]*)) space', + value: 'object | array | string | number | boolean', + object: '"{" space ( string ":" space value ("," space string ":" space value)* )? "}" space', + array: '"[" space ( value ("," space value)* )? "]" space', + uuid: '"\\"" ' + [8, 4, 4, 4, 12].map(n => [...new Array(n)].map(_ => '[0-9a-fA-F]').join('')).join(' "-" ') + ' "\\"" space', string: ` "\\"" ( [^"\\\\] | "\\\\" (["\\\\/bfnrt] | "u" [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F]) )* "\\"" space`, null: '"null" space', }; +const OBJECT_RULE_NAMES = ['object', 'array', 'string', 'number', 'boolean', 'null', 'value']; + +// TODO: support "uri", "email" string formats +const DATE_RULES = { + 'date' : '[0-9] [0-9] [0-9] [0-9] "-" ( "0" [1-9] | "1" [0-2] ) "-" ( \"0\" [1-9] | [1-2] [0-9] | "3" [0-1] )', + 'time' : '([01] [0-9] | "2" [0-3]) ":" [0-5] [0-9] ":" [0-5] [0-9] ( "." [0-9] [0-9] [0-9] )? ( "Z" | ( "+" | "-" ) ( [01] [0-9] | "2" [0-3] ) ":" [0-5] [0-9] )', + 'date-time': 'date "T" time', + 'date-string': '"\\"" date "\\"" space', + 'time-string': '"\\"" time "\\"" space', + 'date-time-string': '"\\"" date-time "\\"" space', +}; + +const RESERVED_NAMES = {'root': true, ...PRIMITIVE_RULES, ...DATE_RULES}; const INVALID_RULE_CHARS_RE = /[^\dA-Za-z-]+/g; const GRAMMAR_LITERAL_ESCAPE_RE = /[\n\r"]/g; -const GRAMMAR_LITERAL_ESCAPES = {'\r': '\\r', '\n': '\\n', '"': '\\"'}; +const GRAMMAR_RANGE_LITERAL_ESCAPE_RE = /[\n\r"\]\-\\]/g; +const GRAMMAR_LITERAL_ESCAPES = { '\r': '\\r', '\n': '\\n', '"': '\\"', '-': '\\-', ']': '\\]' }; + +const NON_LITERAL_SET = new Set('|.()[]{}*+?'); +const ESCAPED_IN_REGEXPS_BUT_NOT_IN_LITERALS = new Set('[]()|{}*+?'); export class SchemaConverter { - constructor(propOrder) { - this._propOrder = propOrder || {}; - this._rules = new Map(); - this._rules.set('space', SPACE_RULE); + constructor(options) { + this._propOrder = options.prop_order || {}; + this._allowFetch = options.allow_fetch || false; + this._dotall = options.dotall || false; + this._rules = {'space': SPACE_RULE}; + this._refs = {}; + this._refsBeingResolved = new Set(); } _formatLiteral(literal) { @@ -30,83 +55,490 @@ export class SchemaConverter { return `"${escaped}"`; } + _formatRangeChar(literal) { + return JSON.stringify(literal).slice(1, -1).replace( + GRAMMAR_RANGE_LITERAL_ESCAPE_RE, + m => GRAMMAR_LITERAL_ESCAPES[m] + ); + } + _addRule(name, rule) { let escName = name.replace(INVALID_RULE_CHARS_RE, '-'); let key = escName; - if (this._rules.has(escName)) { - if (this._rules.get(escName) === rule) { + if (escName in this._rules) { + if (this._rules[escName] === rule) { return key; } let i = 0; - while (this._rules.has(`${escName}${i}`)) { + while ((`${escName}${i}` in this._rules) && (this._rules[`${escName}${i}`] !== rule)) { i += 1; } key = `${escName}${i}`; } - this._rules.set(key, rule); + this._rules[key] = rule; return key; } + async resolveRefs(schema, url) { + const visit = async (n) => { + if (Array.isArray(n)) { + return Promise.all(n.map(visit)); + } else if (typeof n === 'object' && n !== null) { + let ref = n.$ref; + let target; + if (ref !== undefined && !this._refs[ref]) { + if (ref.startsWith('https://')) { + if (!this._allowFetch) { + throw new Error('Fetching remote schemas is not allowed (use --allow-fetch for force)'); + } + const fetch = (await import('node-fetch')).default; + + const fragSplit = ref.split('#'); + const baseUrl = fragSplit[0]; + + target = this._refs[baseUrl]; + if (!target) { + target = await this.resolveRefs(await fetch(ref).then(res => res.json()), baseUrl); + this._refs[baseUrl] = target; + } + + if (fragSplit.length === 1 || fragSplit[fragSplit.length - 1] === '') { + return target; + } + } else if (ref.startsWith('#/')) { + target = schema; + ref = `${url}${ref}`; + n.$ref = ref; + } else { + throw new Error(`Unsupported ref ${ref}`); + } + + const selectors = ref.split('#')[1].split('/').slice(1); + for (const sel of selectors) { + if (!target || !(sel in target)) { + throw new Error(`Error resolving ref ${ref}: ${sel} not in ${JSON.stringify(target)}`); + } + target = target[sel]; + } + + this._refs[ref] = target; + } else { + await Promise.all(Object.values(n).map(visit)); + } + } + + return n; + }; + + return visit(schema); + } + + _generateUnionRule(name, altSchemas) { + return altSchemas + .map((altSchema, i) => this.visit(altSchema, `${name ?? ''}${name ? '-' : 'alternative-'}${i}`)) + .join(' | '); + } + + _visitPattern(pattern, name) { + if (!pattern.startsWith('^') || !pattern.endsWith('$')) { + throw new Error('Pattern must start with "^" and end with "$"'); + } + pattern = pattern.slice(1, -1); + const subRuleIds = {}; + + let i = 0; + const length = pattern.length; + + const getDot = () => { + let rule; + if (this._dotall) { + rule = '[\\U00000000-\\U0010FFFF]'; + } else { + // Accept any character... except \n and \r line break chars (\x0A and \xOD) + rule = '[\\U00000000-\\x09\\x0B\\x0C\\x0E-\\U0010FFFF]'; + } + return this._addRule('dot', rule); + }; + + + const toRule = ([s, isLiteral]) => isLiteral ? "\"" + s + "\"" : s; + + const transform = () => { + const start = i; + // For each component of this sequence, store its string representation and whether it's a literal. + // We only need a flat structure here to apply repetition operators to the last item, and + // to merge literals at the and (we're parsing grouped ( sequences ) recursively and don't treat '|' specially + // (GBNF's syntax is luckily very close to regular expressions!) + const seq = []; + + const joinSeq = () => { + const ret = []; + for (const [isLiteral, g] of groupBy(seq, x => x[1])) { + if (isLiteral) { + ret.push([[...g].map(x => x[0]).join(''), true]); + } else { + ret.push(...g); + } + } + if (ret.length === 1) { + return ret[0]; + } + return [ret.map(x => toRule(x)).join(' '), false]; + }; + + while (i < length) { + const c = pattern[i]; + if (c === '.') { + seq.push([getDot(), false]); + i += 1; + } else if (c === '(') { + i += 1; + if (i < length) { + if (pattern[i] === '?') { + throw new Error(`Unsupported pattern syntax "${pattern[i]}" at index ${i} of /${pattern}/`); + } + } + seq.push([`(${toRule(transform())})`, false]); + } else if (c === ')') { + i += 1; + if (start <= 0 || pattern[start - 1] !== '(') { + throw new Error(`Unbalanced parentheses; start = ${start}, i = ${i}, pattern = ${pattern}`); + } + return joinSeq(); + } else if (c === '[') { + let squareBrackets = c; + i += 1; + while (i < length && pattern[i] !== ']') { + if (pattern[i] === '\\') { + squareBrackets += pattern.slice(i, i + 2); + i += 2; + } else { + squareBrackets += pattern[i]; + i += 1; + } + } + if (i >= length) { + throw new Error(`Unbalanced square brackets; start = ${start}, i = ${i}, pattern = ${pattern}`); + } + squareBrackets += ']'; + i += 1; + seq.push([squareBrackets, false]); + } else if (c === '|') { + seq.push(['|', false]); + i += 1; + } else if (c === '*' || c === '+' || c === '?') { + seq[seq.length - 1] = [toRule(seq[seq.length - 1]) + c, false]; + i += 1; + } else if (c === '{') { + let curlyBrackets = c; + i += 1; + while (i < length && pattern[i] !== '}') { + curlyBrackets += pattern[i]; + i += 1; + } + if (i >= length) { + throw new Error(`Unbalanced curly brackets; start = ${start}, i = ${i}, pattern = ${pattern}`); + } + curlyBrackets += '}'; + i += 1; + const nums = curlyBrackets.slice(1, -1).split(',').map(s => s.trim()); + let minTimes, maxTimes; + if (nums.length === 1) { + minTimes = parseInt(nums[0], 10); + maxTimes = minTimes; + } else { + if (nums.length !== 2) { + throw new Error(`Invalid quantifier ${curlyBrackets}`); + } + minTimes = nums[0] ? parseInt(nums[0], 10) : 0; + maxTimes = nums[1] ? parseInt(nums[1], 10) : Infinity; + } + + let [sub, subIsLiteral] = seq[seq.length - 1]; + + if (minTimes === 0 && maxTimes === Infinity) { + seq[seq.length - 1] = [`${sub}*`, false]; + } else if (minTimes === 0 && maxTimes === 1) { + seq[seq.length - 1] = [`${sub}?`, false]; + } else if (minTimes === 1 && maxTimes === Infinity) { + seq[seq.length - 1] = [`${sub}+`, false]; + } else { + if (!subIsLiteral) { + let id = subRuleIds[sub]; + if (id === undefined) { + id = this._addRule(`${name}-${Object.keys(subRuleIds).length + 1}`, sub); + subRuleIds[sub] = id; + } + sub = id; + } + + const repeatedSub = Array.from({ length: minTimes }, () => subIsLiteral ? `"${sub.slice(1, -1).repeat(minTimes)}"` : sub); + const optionalSub = maxTimes !== undefined ? Array.from({ length: maxTimes - minTimes }, () => `${sub}?`) : [`${sub}*`]; + seq[seq.length - 1] = [repeatedSub.concat(optionalSub).join(' '), false]; + } + } else { + let literal = ''; + while (i < length) { + if (pattern[i] === '\\' && i < length - 1) { + const next = pattern[i + 1]; + if (ESCAPED_IN_REGEXPS_BUT_NOT_IN_LITERALS.has(next)) { + i += 1; + literal += pattern[i]; + i += 1; + } else { + literal += pattern.slice(i, i + 2); + i += 2; + } + } else if (pattern[i] === '"') { + literal += '\\"'; + i += 1; + } else if (!NON_LITERAL_SET.has(pattern[i]) && + (i === length - 1 || literal === '' || pattern[i + 1] === '.' || !NON_LITERAL_SET.has(pattern[i+1]))) { + literal += pattern[i]; + i += 1; + } else { + break; + } + } + if (literal !== '') { + seq.push([literal, true]); + } + } + } + + return joinSeq(); + }; + + return this._addRule(name, "\"\\\"\" " + toRule(transform()) + " \"\\\"\" space") + } + + _resolveRef(ref) { + let refName = ref.split('/').pop(); + if (!(refName in this._rules) && !this._refsBeingResolved.has(ref)) { + this._refsBeingResolved.add(ref); + const resolved = this._refs[ref]; + refName = this.visit(resolved, refName); + this._refsBeingResolved.delete(ref); + } + return refName; + } + + _generateConstantRule(value) { + if (typeof value !== 'string') { + throw new Error('Only string constants are supported, got ' + JSON.stringify(value)); + } + return this._formatLiteral(value); + } + visit(schema, name) { const schemaType = schema.type; - const ruleName = name || 'root'; + const schemaFormat = schema.format; + const ruleName = name in RESERVED_NAMES ? name + '-' : name == '' ? 'root' : name; - if (schema.oneOf || schema.anyOf) { - const rule = (schema.oneOf || schema.anyOf).map((altSchema, i) => - this.visit(altSchema, `${name}${name ? "-" : ""}${i}`) - ).join(' | '); - - return this._addRule(ruleName, rule); + const ref = schema.$ref; + if (ref !== undefined) { + return this._addRule(ruleName, this._resolveRef(ref)); + } else if (schema.oneOf || schema.anyOf) { + return this._addRule(ruleName, this._generateUnionRule(name, schema.oneOf || schema.anyOf)); + } else if (Array.isArray(schemaType)) { + return this._addRule(ruleName, this._generateUnionRule(name, schemaType.map(t => ({ type: t })))); } else if ('const' in schema) { - return this._addRule(ruleName, this._formatLiteral(schema.const)); + if (typeof schema.const !== 'string') { + throw new Error('Only string constants are supported, got ' + JSON.stringify(schema.const)); + } + return this._addRule(ruleName, this._generateConstantRule(schema.const)); } else if ('enum' in schema) { - const rule = schema.enum.map(v => this._formatLiteral(v)).join(' | '); + const rule = schema.enum.map(v => this._generateConstantRule(v)).join(' | '); return this._addRule(ruleName, rule); - } else if (schemaType === 'object' && 'properties' in schema) { - // TODO: `required` keyword (from python implementation) - const propOrder = this._propOrder; - const propPairs = Object.entries(schema.properties).sort((a, b) => { - // sort by position in prop_order (if specified) then by key - const orderA = typeof propOrder[a[0]] === 'number' ? propOrder[a[0]] : Infinity; - const orderB = typeof propOrder[b[0]] === 'number' ? propOrder[b[0]] : Infinity; - return orderA - orderB || a[0].localeCompare(b[0]); - }); - - let rule = '"{" space'; - propPairs.forEach(([propName, propSchema], i) => { - const propRuleName = this.visit(propSchema, `${name}${name ? "-" : ""}${propName}`); - if (i > 0) { - rule += ' "," space'; + } else if ((schemaType === undefined || schemaType === 'object') && + ('properties' in schema || + ('additionalProperties' in schema && schema.additionalProperties !== true))) { + const required = new Set(schema.required || []); + const properties = Object.entries(schema.properties ?? {}); + return this._addRule(ruleName, this._buildObjectRule(properties, required, name, schema.additionalProperties)); + } else if ((schemaType === undefined || schemaType === 'object') && 'allOf' in schema) { + const required = new Set(); + const properties = []; + const addComponent = (compSchema, isRequired) => { + const ref = compSchema.$ref; + if (ref !== undefined) { + compSchema = this._refs[ref]; } - rule += ` ${this._formatLiteral(propName)} space ":" space ${propRuleName}`; - }); - rule += ' "}" space'; - return this._addRule(ruleName, rule); - } else if (schemaType === 'array' && 'items' in schema) { - // TODO `prefixItems` keyword (from python implementation) - const itemRuleName = this.visit(schema.items, `${name}${name ? "-" : ""}item`); - const rule = `"[" space (${itemRuleName} ("," space ${itemRuleName})*)? "]" space`; - return this._addRule(ruleName, rule); + if ('properties' in compSchema) { + for (const [propName, propSchema] of Object.entries(compSchema.properties)) { + properties.push([propName, propSchema]); + if (isRequired) { + required.add(propName); + } + } + } + }; + + for (const t of schema.allOf) { + if ('anyOf' in t) { + for (const tt of t.anyOf) { + addComponent(tt, false); + } + } else { + addComponent(t, true); + } + } + + return this._addRule(ruleName, this._buildObjectRule(properties, required, name, /* additionalProperties= */ false)); + } else if ((schemaType === undefined || schemaType === 'array') && ('items' in schema || 'prefixItems' in schema)) { + const items = schema.items ?? schema.prefixItems; + if (Array.isArray(items)) { + return this._addRule( + ruleName, + '"[" space ' + + items.map((item, i) => this.visit(item, `${name ?? ''}${name ? '-' : ''}tuple-${i}`)).join(' "," space ') + + ' "]" space' + ); + } else { + const itemRuleName = this.visit(items, `${name ?? ''}${name ? '-' : ''}item`); + const listItemOperator = `( "," space ${itemRuleName} )`; + let successiveItems = ''; + let minItems = schema.minItems || 0; + const maxItems = schema.maxItems; + if (minItems > 0) { + successiveItems = listItemOperator.repeat(minItems - 1); + minItems--; + } + if (maxItems !== undefined && maxItems > minItems) { + successiveItems += `${listItemOperator}?`.repeat(maxItems - minItems - 1); + } else { + successiveItems += `${listItemOperator}*`; + } + const rule = minItems === 0 + ? `"[" space ( ${itemRuleName} ${successiveItems} )? "]" space` + : `"[" space ${itemRuleName} ${successiveItems} "]" space`; + return this._addRule(ruleName, rule); + } + } else if ((schemaType === undefined || schemaType === 'string') && 'pattern' in schema) { + return this._visitPattern(schema.pattern, ruleName); + } else if ((schemaType === undefined || schemaType === 'string') && /^uuid[1-5]?$/.test(schema.format || '')) { + return this._addRule( + ruleName === 'root' ? 'root' : schemaFormat, + PRIMITIVE_RULES['uuid']) + } else if ((schemaType === undefined || schemaType === 'string') && schema.format in DATE_RULES) { + for (const [t, r] of Object.entries(DATE_RULES)) { + this._addRule(t, r); + } + return schemaFormat + '-string'; + } else if ((schemaType === 'object') || (Object.keys(schema).length === 0)) { + for (const n of OBJECT_RULE_NAMES) { + this._addRule(n, PRIMITIVE_RULES[n]); + } + return this._addRule(ruleName, 'object'); } else { - if (!PRIMITIVE_RULES[schemaType]) { + if (!(schemaType in PRIMITIVE_RULES)) { throw new Error(`Unrecognized schema: ${JSON.stringify(schema)}`); } - return this._addRule( - ruleName === 'root' ? 'root' : schemaType, - PRIMITIVE_RULES[schemaType] + // TODO: support minimum, maximum, exclusiveMinimum, exclusiveMaximum at least for zero + return this._addRule(ruleName === 'root' ? 'root' : schemaType, PRIMITIVE_RULES[schemaType]); + } + } + + _buildObjectRule(properties, required, name, additionalProperties) { + const propOrder = this._propOrder; + // sort by position in prop_order (if specified) then by original order + const sortedProps = properties.map(([k]) => k).sort((a, b) => { + const orderA = propOrder[a] || Infinity; + const orderB = propOrder[b] || Infinity; + return orderA - orderB || properties.findIndex(([k]) => k === a) - properties.findIndex(([k]) => k === b); + }); + + const propKvRuleNames = {}; + for (const [propName, propSchema] of properties) { + const propRuleName = this.visit(propSchema, `${name ?? ''}${name ? '-' : ''}${propName}`); + propKvRuleNames[propName] = this._addRule( + `${name ?? ''}${name ? '-' : ''}${propName}-kv`, + `${this._formatLiteral(propName)} space ":" space ${propRuleName}` ); } + const requiredProps = sortedProps.filter(k => required.has(k)); + const optionalProps = sortedProps.filter(k => !required.has(k)); + + if (typeof additionalProperties === 'object' || additionalProperties === true) { + const subName = `${name ?? ''}${name ? '-' : ''}additional`; + const valueRule = this.visit(additionalProperties === true ? {} : additionalProperties, `${subName}-value`); + propKvRuleNames['*'] = this._addRule( + `${subName}-kv`, + `${this._addRule('string', PRIMITIVE_RULES['string'])} ":" space ${valueRule}`); + optionalProps.push('*'); + } + + let rule = '"{" space '; + rule += requiredProps.map(k => propKvRuleNames[k]).join(' "," space '); + + if (optionalProps.length > 0) { + rule += ' ('; + if (requiredProps.length > 0) { + rule += ' "," space ( '; + } + + const getRecursiveRefs = (ks, firstIsOptional) => { + const [k, ...rest] = ks; + const kvRuleName = propKvRuleNames[k]; + let res; + if (k === '*') { + res = this._addRule( + `${name ?? ''}${name ? '-' : ''}additional-kvs`, + `${kvRuleName} ( "," space ` + kvRuleName + ` )*` + ) + } else if (firstIsOptional) { + res = `( "," space ${kvRuleName} )?`; + } else { + res = kvRuleName; + } + if (rest.length > 0) { + res += ' ' + this._addRule( + `${name ?? ''}${name ? '-' : ''}${k}-rest`, + getRecursiveRefs(rest, true) + ); + } + return res; + }; + + rule += optionalProps.map((_, i) => getRecursiveRefs(optionalProps.slice(i), false)).join(' | '); + if (requiredProps.length > 0) { + rule += ' )'; + } + rule += ' )?'; + } + + rule += ' "}" space'; + + return rule; } formatGrammar() { let grammar = ''; - this._rules.forEach((rule, name) => { + for (const [name, rule] of Object.entries(this._rules).sort(([a], [b]) => a.localeCompare(b))) { grammar += `${name} ::= ${rule}\n`; - }); + } return grammar; } } + +// Helper function to group elements by a key function +function* groupBy(iterable, keyFn) { + let lastKey = null; + let group = []; + for (const element of iterable) { + const key = keyFn(element); + if (lastKey !== null && key !== lastKey) { + yield [lastKey, group]; + group = []; + } + group.push(element); + lastKey = key; + } + if (group.length > 0) { + yield [lastKey, group]; + } +} diff --git a/examples/server/server.cpp b/examples/server/server.cpp index d2a8e541d..cf075d6c4 100644 --- a/examples/server/server.cpp +++ b/examples/server/server.cpp @@ -1,6 +1,7 @@ #include "utils.hpp" #include "common.h" +#include "json-schema-to-grammar.h" #include "llama.h" #include "grammar-parser.h" @@ -178,6 +179,7 @@ struct server_slot { llama_token sampled; struct llama_sampling_params sparams; llama_sampling_context * ctx_sampling = nullptr; + json json_schema; int32_t ga_i = 0; // group-attention state int32_t ga_n = 1; // group-attention factor @@ -845,7 +847,17 @@ struct server_context { slot.sparams.penalize_nl = json_value(data, "penalize_nl", default_sparams.penalize_nl); slot.params.n_keep = json_value(data, "n_keep", slot.params.n_keep); slot.params.seed = json_value(data, "seed", default_params.seed); - slot.sparams.grammar = json_value(data, "grammar", default_sparams.grammar); + if (data.contains("json_schema") && !data.contains("grammar")) { + try { + auto schema = json_value(data, "json_schema", json::object()); + slot.sparams.grammar = json_schema_to_grammar(schema); + } catch (const std::exception & e) { + send_error(task, std::string("\"json_schema\": ") + e.what(), ERROR_TYPE_INVALID_REQUEST); + return false; + } + } else { + slot.sparams.grammar = json_value(data, "grammar", default_sparams.grammar); + } slot.sparams.n_probs = json_value(data, "n_probs", default_sparams.n_probs); slot.sparams.min_keep = json_value(data, "min_keep", default_sparams.min_keep); diff --git a/examples/server/tests/features/security.feature b/examples/server/tests/features/security.feature index 1d6aa40ea..eb82e7aca 100644 --- a/examples/server/tests/features/security.feature +++ b/examples/server/tests/features/security.feature @@ -37,6 +37,22 @@ Feature: Security | llama.cpp | no | | hackme | raised | + Scenario Outline: OAI Compatibility (invalid response formats) + Given a system prompt test + And a user prompt test + And a response format + And a model test + And 2 max tokens to predict + And streaming is disabled + Given an OAI compatible chat completions request with raised api error + + Examples: Prompts + | response_format | + | {"type": "sound"} | + | {"type": "json_object", "schema": 123} | + | {"type": "json_object", "schema": {"type": 123}} | + | {"type": "json_object", "schema": {"type": "hiccup"}} | + Scenario Outline: CORS Options Given a user api key llama.cpp diff --git a/examples/server/tests/features/server.feature b/examples/server/tests/features/server.feature index 45a988db6..a2e0e5b35 100644 --- a/examples/server/tests/features/server.feature +++ b/examples/server/tests/features/server.feature @@ -70,6 +70,22 @@ Feature: llama.cpp server | codellama70b | You are a coding assistant. | Write the fibonacci function in c++. | 128 | (thanks\|happy\|bird\|Annabyear)+ | -1 | 64 | enabled | | + Scenario Outline: OAI Compatibility w/ response format + Given a model test + And a system prompt test + And a user prompt test + And a response format + And 10 max tokens to predict + Given an OAI compatible chat completions request with no api error + Then tokens are predicted matching + + Examples: Prompts + | response_format | n_predicted | re_content | + | {"type": "json_object", "schema": {"const": "42"}} | 5 | "42" | + | {"type": "json_object", "schema": {"items": [{"type": "integer"}]}} | 10 | \[ -300 \] | + | {"type": "json_object"} | 10 | \{ " Jacky. | + + Scenario: Tokenize / Detokenize When tokenizing: """ diff --git a/examples/server/tests/features/steps/steps.py b/examples/server/tests/features/steps/steps.py index 42cdd8985..03f55f659 100644 --- a/examples/server/tests/features/steps/steps.py +++ b/examples/server/tests/features/steps/steps.py @@ -59,6 +59,7 @@ def step_server_config(context, server_fqdn, server_port): context.seed = None context.server_seed = None context.user_api_key = None + context.response_format = None context.tasks_result = [] context.concurrent_tasks = [] @@ -269,6 +270,11 @@ def step_max_tokens(context, max_tokens): context.n_predict = max_tokens +@step('a response format {response_format}') +def step_response_format(context, response_format): + context.response_format = json.loads(response_format) + + @step('streaming is {enable_streaming}') def step_streaming(context, enable_streaming): context.enable_streaming = enable_streaming == 'enabled' @@ -384,6 +390,9 @@ async def step_oai_chat_completions(context, api_error): enable_streaming=context.enable_streaming if hasattr(context, 'enable_streaming') else None, + response_format=context.response_format + if hasattr(context, 'response_format') else None, + seed=await completions_seed(context), user_api_key=context.user_api_key @@ -443,6 +452,8 @@ async def step_oai_chat_completions(context): if hasattr(context, 'n_predict') else None, enable_streaming=context.enable_streaming if hasattr(context, 'enable_streaming') else None, + response_format=context.response_format + if hasattr(context, 'response_format') else None, seed=await completions_seed(context), user_api_key=context.user_api_key if hasattr(context, 'user_api_key') else None) @@ -463,6 +474,8 @@ async def step_oai_chat_completions(context): if hasattr(context, 'n_predict') else None, enable_streaming=context.enable_streaming if hasattr(context, 'enable_streaming') else None, + response_format=context.response_format + if hasattr(context, 'response_format') else None, seed=context.seed if hasattr(context, 'seed') else context.server_seed @@ -745,6 +758,7 @@ async def oai_chat_completions(user_prompt, model=None, n_predict=None, enable_streaming=None, + response_format=None, seed=None, user_api_key=None, expect_api_error=None): @@ -770,6 +784,8 @@ async def oai_chat_completions(user_prompt, "stream": enable_streaming, "seed": seed } + if response_format is not None: + payload['response_format'] = response_format completion_response = { 'content': '', 'timings': { @@ -830,6 +846,7 @@ async def oai_chat_completions(user_prompt, model=model, max_tokens=n_predict, stream=enable_streaming, + response_format=payload.get('response_format'), seed=seed ) except openai.error.AuthenticationError as e: diff --git a/examples/server/utils.hpp b/examples/server/utils.hpp index cc5fce691..ff146893d 100644 --- a/examples/server/utils.hpp +++ b/examples/server/utils.hpp @@ -373,10 +373,21 @@ static json oaicompat_completion_params_parse( llama_params["tfs_z"] = json_value(body, "tfs_z", default_sparams.tfs_z); llama_params["n_keep"] = json_value(body, "n_keep", 0); - if (body.count("grammar") != 0) { + if (body.contains("grammar")) { llama_params["grammar"] = json_value(body, "grammar", json::object()); } + if (body.contains("response_format")) { + auto response_format = json_value(body, "response_format", json::object()); + if (response_format.contains("type")) { + if (response_format["type"] == "json_object") { + llama_params["json_schema"] = json_value(response_format, "schema", json::object()); + } else { + throw std::runtime_error("response_format type not supported: " + response_format["type"].dump()); + } + } + } + // Handle 'stop' field if (body.contains("stop") && body["stop"].is_string()) { llama_params["stop"] = json::array({body["stop"].get()}); diff --git a/examples/ts-type-to-grammar.sh b/examples/ts-type-to-grammar.sh new file mode 100755 index 000000000..a7ad00173 --- /dev/null +++ b/examples/ts-type-to-grammar.sh @@ -0,0 +1,28 @@ +#!/bin/bash +# +# ./examples/ts-type-to-grammar.sh "{a:string,b:string,c?:string}" +# python examples/json-schema-to-grammar.py https://json.schemastore.org/tsconfig.json +# +set -euo pipefail + +readonly type="$1" + +# Create a temporary directory +TMPDIR="" +trap 'rm -fR "$TMPDIR"' EXIT +TMPDIR=$(mktemp -d) + +DTS_FILE="$TMPDIR/type.d.ts" +SCHEMA_FILE="$TMPDIR/schema.json" + +echo "export type MyType = $type" > "$DTS_FILE" + +# This is a fork of typescript-json-schema, actively maintained as of March 2024: +# https://github.com/vega/ts-json-schema-generator +npx ts-json-schema-generator --unstable --no-top-ref --path "$DTS_FILE" --type MyType -e none > "$SCHEMA_FILE" + +# Alternative, not actively maintained as of March 2024: +# https://github.com/YousefED/typescript-json-schema +# npx typescript-json-schema --defaultProps --required "$DTS_FILE" MyType | tee "$SCHEMA_FILE" >&2 + +./examples/json-schema-to-grammar.py "$SCHEMA_FILE" diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 10326d531..a43439aed 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -1,66 +1,75 @@ -function(llama_build_executable source) - get_filename_component(TEST_TARGET ${source} NAME_WE) +# Builds and runs a test source file. +# Optional args: +# - NAME: name of the executable & test target (defaults to the source file name without extension) +# - LABEL: label for the test (defaults to main) +# - ARGS: arguments to pass to the test executable +# - WORKING_DIRECTORY +function(llama_test source) + include(CMakeParseArguments) + set(options) + set(oneValueArgs NAME LABEL WORKING_DIRECTORY) + set(multiValueArgs ARGS) + cmake_parse_arguments(LLAMA_TEST "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) + + if (NOT DEFINED LLAMA_TEST_LABEL) + set(LLAMA_TEST_LABEL "main") + endif() + if (NOT DEFINED LLAMA_TEST_WORKING_DIRECTORY) + set(LLAMA_TEST_WORKING_DIRECTORY .) + endif() + if (DEFINED LLAMA_TEST_NAME) + set(TEST_TARGET ${LLAMA_TEST_NAME}) + else() + get_filename_component(TEST_TARGET ${source} NAME_WE) + endif() + add_executable(${TEST_TARGET} ${source} get-model.cpp) install(TARGETS ${TEST_TARGET} RUNTIME) - target_link_libraries(${TEST_TARGET} PRIVATE common) + target_link_libraries(${TEST_TARGET} PRIVATE common json-schema-to-grammar) + add_test( + NAME ${TEST_TARGET} + WORKING_DIRECTORY ${LLAMA_TEST_WORKING_DIRECTORY} + COMMAND $ + ${LLAMA_TEST_ARGS}) + + set_property(TEST ${TEST_TARGET} PROPERTY LABELS ${LLAMA_TEST_LABEL}) endfunction() -function(llama_test_executable name source) - get_filename_component(TEST_TARGET ${source} NAME_WE) - add_test(NAME ${name} COMMAND $ ${ARGN}) - set_property(TEST ${name} PROPERTY LABELS "main") -endfunction() +# llama_test(test-double-float.cpp) # SLOW +llama_test(test-quantize-fns.cpp) +llama_test(test-quantize-perf.cpp) +llama_test(test-sampling.cpp) +llama_test(test-chat-template.cpp) -function(llama_build_and_test_executable source) - llama_build_and_test_executable_with_label(${source} "main") -endfunction() +llama_test(test-tokenizer-0-llama.cpp NAME test-tokenizer-0-llama ARGS ${CMAKE_CURRENT_SOURCE_DIR}/../models/ggml-vocab-llama.gguf) +llama_test(test-tokenizer-0-falcon.cpp NAME test-tokenizer-0-falcon ARGS ${CMAKE_CURRENT_SOURCE_DIR}/../models/ggml-vocab-falcon.gguf) -function(llama_build_and_test_executable_with_label source label) - get_filename_component(TEST_TARGET ${source} NAME_WE) - add_executable(${TEST_TARGET} ${source} get-model.cpp) - install(TARGETS ${TEST_TARGET} RUNTIME) - target_link_libraries(${TEST_TARGET} PRIVATE common) - add_test(NAME ${TEST_TARGET} COMMAND $ ${ARGN}) - set_property(TEST ${TEST_TARGET} PROPERTY LABELS ${label}) -endfunction() +llama_test(test-tokenizer-1-llama.cpp NAME test-tokenizer-1-llama ARGS ${CMAKE_CURRENT_SOURCE_DIR}/../models/ggml-vocab-llama.gguf) +llama_test(test-tokenizer-1-llama.cpp NAME test-tokenizer-1-baichuan ARGS ${CMAKE_CURRENT_SOURCE_DIR}/../models/ggml-vocab-baichuan.gguf) -# llama_build_and_test_executable(test-double-float.cpp) # SLOW -llama_build_and_test_executable(test-quantize-fns.cpp) -llama_build_and_test_executable(test-quantize-perf.cpp) -llama_build_and_test_executable(test-sampling.cpp) -llama_build_and_test_executable(test-chat-template.cpp) +llama_test(test-tokenizer-1-bpe.cpp NAME test-tokenizer-1-falcon ARGS ${CMAKE_CURRENT_SOURCE_DIR}/../models/ggml-vocab-falcon.gguf) +llama_test(test-tokenizer-1-bpe.cpp NAME test-tokenizer-1-aquila ARGS ${CMAKE_CURRENT_SOURCE_DIR}/../models/ggml-vocab-aquila.gguf) +llama_test(test-tokenizer-1-bpe.cpp NAME test-tokenizer-1-mpt ARGS ${CMAKE_CURRENT_SOURCE_DIR}/../models/ggml-vocab-mpt.gguf) +llama_test(test-tokenizer-1-bpe.cpp NAME test-tokenizer-1-stablelm-3b-4e1t ARGS ${CMAKE_CURRENT_SOURCE_DIR}/../models/ggml-vocab-stablelm-3b-4e1t.gguf) +llama_test(test-tokenizer-1-bpe.cpp NAME test-tokenizer-1-gpt-neox ARGS ${CMAKE_CURRENT_SOURCE_DIR}/../models/ggml-vocab-gpt-neox.gguf) +llama_test(test-tokenizer-1-bpe.cpp NAME test-tokenizer-1-refact ARGS ${CMAKE_CURRENT_SOURCE_DIR}/../models/ggml-vocab-refact.gguf) +llama_test(test-tokenizer-1-bpe.cpp NAME test-tokenizer-1-starcoder ARGS ${CMAKE_CURRENT_SOURCE_DIR}/../models/ggml-vocab-starcoder.gguf) +llama_test(test-tokenizer-1-bpe.cpp NAME test-tokenizer-1-gpt2 ARGS ${CMAKE_CURRENT_SOURCE_DIR}/../models/ggml-vocab-gpt2.gguf) +#llama_test(test-tokenizer-1-bpe.cpp NAME test-tokenizer-1-bloom ARGS ${CMAKE_CURRENT_SOURCE_DIR}/../models/ggml-vocab-bloom.gguf) # BIG -llama_build_executable(test-tokenizer-0-llama.cpp) -llama_test_executable (test-tokenizer-0-llama test-tokenizer-0-llama.cpp ${CMAKE_CURRENT_SOURCE_DIR}/../models/ggml-vocab-llama.gguf) +llama_test(test-grammar-parser.cpp) +llama_test(test-llama-grammar.cpp) +llama_test(test-grad0.cpp) +# llama_test(test-opt.cpp) # SLOW +llama_test(test-backend-ops.cpp) -llama_build_executable(test-tokenizer-0-falcon.cpp) -llama_test_executable (test-tokenizer-0-falcon test-tokenizer-0-falcon.cpp ${CMAKE_CURRENT_SOURCE_DIR}/../models/ggml-vocab-falcon.gguf) +llama_test(test-rope.cpp) -llama_build_executable(test-tokenizer-1-llama.cpp) -llama_test_executable (test-tokenizer-1-llama test-tokenizer-1-llama.cpp ${CMAKE_CURRENT_SOURCE_DIR}/../models/ggml-vocab-llama.gguf) -llama_test_executable (test-tokenizer-1-baichuan test-tokenizer-1-llama.cpp ${CMAKE_CURRENT_SOURCE_DIR}/../models/ggml-vocab-baichuan.gguf) +llama_test(test-model-load-cancel.cpp LABEL "model") +llama_test(test-autorelease.cpp LABEL "model") -llama_build_executable(test-tokenizer-1-bpe.cpp) -llama_test_executable (test-tokenizer-1-falcon test-tokenizer-1-bpe.cpp ${CMAKE_CURRENT_SOURCE_DIR}/../models/ggml-vocab-falcon.gguf) -llama_test_executable (test-tokenizer-1-aquila test-tokenizer-1-bpe.cpp ${CMAKE_CURRENT_SOURCE_DIR}/../models/ggml-vocab-aquila.gguf) -llama_test_executable (test-tokenizer-1-mpt test-tokenizer-1-bpe.cpp ${CMAKE_CURRENT_SOURCE_DIR}/../models/ggml-vocab-mpt.gguf) -llama_test_executable (test-tokenizer-1-stablelm-3b-4e1t test-tokenizer-1-bpe.cpp ${CMAKE_CURRENT_SOURCE_DIR}/../models/ggml-vocab-stablelm-3b-4e1t.gguf) -llama_test_executable (test-tokenizer-1-gpt-neox test-tokenizer-1-bpe.cpp ${CMAKE_CURRENT_SOURCE_DIR}/../models/ggml-vocab-gpt-neox.gguf) -llama_test_executable (test-tokenizer-1-refact test-tokenizer-1-bpe.cpp ${CMAKE_CURRENT_SOURCE_DIR}/../models/ggml-vocab-refact.gguf) -llama_test_executable (test-tokenizer-1-starcoder test-tokenizer-1-bpe.cpp ${CMAKE_CURRENT_SOURCE_DIR}/../models/ggml-vocab-starcoder.gguf) -llama_test_executable (test-tokenizer-1-gpt2 test-tokenizer-1-bpe.cpp ${CMAKE_CURRENT_SOURCE_DIR}/../models/ggml-vocab-gpt2.gguf) -# llama_test_executable (test-tokenizer-1-bloom test-tokenizer-1-bpe.cpp ${CMAKE_CURRENT_SOURCE_DIR}/../models/ggml-vocab-bloom.gguf) # BIG - -llama_build_and_test_executable(test-grammar-parser.cpp) -llama_build_and_test_executable(test-llama-grammar.cpp) -llama_build_and_test_executable(test-grad0.cpp) -# llama_build_and_test_executable(test-opt.cpp) # SLOW -llama_build_and_test_executable(test-backend-ops.cpp) - -llama_build_and_test_executable(test-rope.cpp) - -llama_build_and_test_executable_with_label(test-model-load-cancel.cpp "model") -llama_build_and_test_executable_with_label(test-autorelease.cpp "model") +llama_test(test-json-schema-to-grammar.cpp WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/..) +target_include_directories(test-json-schema-to-grammar PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/../examples/server) # dummy executable - not installed get_filename_component(TEST_TARGET test-c.c NAME_WE) diff --git a/tests/run-json-schema-to-grammar.mjs b/tests/run-json-schema-to-grammar.mjs new file mode 100644 index 000000000..71bf62ed3 --- /dev/null +++ b/tests/run-json-schema-to-grammar.mjs @@ -0,0 +1,10 @@ +import { readFileSync } from "fs" +import { SchemaConverter } from "../examples/server/public/json-schema-to-grammar.mjs" + +const [, , file] = process.argv +const url = `file://${file}` +let schema = JSON.parse(readFileSync(file, "utf8")); +const converter = new SchemaConverter({}) +schema = await converter.resolveRefs(schema, url) +converter.visit(schema, '') +console.log(converter.formatGrammar()) diff --git a/tests/test-json-schema-to-grammar.cpp b/tests/test-json-schema-to-grammar.cpp new file mode 100755 index 000000000..a13bb9d7f --- /dev/null +++ b/tests/test-json-schema-to-grammar.cpp @@ -0,0 +1,824 @@ +#ifdef NDEBUG +#undef NDEBUG +#endif + +#include +#include +#include + +#include "json-schema-to-grammar.h" +#include "grammar-parser.h" + +static std::string trim(const std::string & source) { + std::string s(source); + s.erase(0,s.find_first_not_of(" \n\r\t")); + s.erase(s.find_last_not_of(" \n\r\t")+1); + return std::regex_replace(s, std::regex("(^|\n)[ \t]+"), "$1"); +} + +enum TestCaseStatus { + SUCCESS, + FAILURE +}; + +struct TestCase { + TestCaseStatus expected_status; + std::string name; + std::string schema; + std::string expected_grammar; + + void _print_failure_header() const { + fprintf(stderr, "#\n# Test '%s' failed.\n#\n%s\n", name.c_str(), schema.c_str()); + } + void verify(const std::string & actual_grammar) const { + if (trim(actual_grammar) != trim(expected_grammar)) { + _print_failure_header(); + fprintf(stderr, "# EXPECTED:\n%s\n# ACTUAL:\n%s\n", expected_grammar.c_str(), actual_grammar.c_str()); + assert(false); + } + } + void verify_expectation_parseable() const { + try { + auto state = grammar_parser::parse(expected_grammar.c_str()); + if (state.symbol_ids.find("root") == state.symbol_ids.end()) { + throw std::runtime_error("Grammar failed to parse:\n" + expected_grammar); + } + } catch (const std::runtime_error & ex) { + _print_failure_header(); + fprintf(stderr, "# GRAMMAR ERROR: %s\n", ex.what()); + assert(false); + } + } + void verify_status(TestCaseStatus status) const { + if (status != expected_status) { + _print_failure_header(); + fprintf(stderr, "# EXPECTED STATUS: %s\n", expected_status == SUCCESS ? "SUCCESS" : "FAILURE"); + fprintf(stderr, "# ACTUAL STATUS: %s\n", status == SUCCESS ? "SUCCESS" : "FAILURE"); + assert(false); + } + } +}; + +static void write(const std::string & file, const std::string & content) { + std::ofstream f; + f.open(file.c_str()); + f << content.c_str(); + f.close(); +} + +static std::string read(const std::string & file) { + std::ostringstream actuals; + actuals << std::ifstream(file.c_str()).rdbuf(); + return actuals.str(); +} + +static void test_all(const std::string & lang, std::function runner) { + fprintf(stderr, "#\n# Testing JSON schema conversion (%s)\n#\n", lang.c_str()); + auto test = [&](const TestCase & tc) { + fprintf(stderr, "- %s%s\n", tc.name.c_str(), tc.expected_status == FAILURE ? " (failure expected)" : ""); + runner(tc); + }; + + test({ + FAILURE, + "unknown type", + R"""({ + "type": "kaboom" + })""", + "" + }); + + test({ + FAILURE, + "invalid type type", + R"""({ + "type": 123 + })""", + "" + }); + + test({ + SUCCESS, + "empty schema (object)", + "{}", + R"""( + array ::= "[" space ( value ("," space value)* )? "]" space + boolean ::= ("true" | "false") space + null ::= "null" space + number ::= ("-"? ([0-9] | [1-9] [0-9]*)) ("." [0-9]+)? ([eE] [-+]? [0-9]+)? space + object ::= "{" space ( string ":" space value ("," space string ":" space value)* )? "}" space + root ::= object + space ::= " "? + string ::= "\"" ( + [^"\\] | + "\\" (["\\/bfnrt] | "u" [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F]) + )* "\"" space + value ::= object | array | string | number | boolean + )""" + }); + + test({ + SUCCESS, + "exotic formats", + R"""({ + "items": [ + { "format": "date" }, + { "format": "uuid" }, + { "format": "time" }, + { "format": "date-time" } + ] + })""", + R"""( + date ::= [0-9] [0-9] [0-9] [0-9] "-" ( "0" [1-9] | "1" [0-2] ) "-" ( "0" [1-9] | [1-2] [0-9] | "3" [0-1] ) + date-string ::= "\"" date "\"" space + date-time ::= date "T" time + date-time-string ::= "\"" date-time "\"" space + root ::= "[" space date-string "," space uuid "," space time-string "," space date-time-string "]" space + space ::= " "? + time ::= ([01] [0-9] | "2" [0-3]) ":" [0-5] [0-9] ":" [0-5] [0-9] ( "." [0-9] [0-9] [0-9] )? ( "Z" | ( "+" | "-" ) ( [01] [0-9] | "2" [0-3] ) ":" [0-5] [0-9] ) + time-string ::= "\"" time "\"" space + uuid ::= "\"" [0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F] "-" [0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F] "-" [0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F] "-" [0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F] "-" [0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F] "\"" space + )""" + }); + + test({ + SUCCESS, + "string", + R"""({ + "type": "string" + })""", + R"""( + root ::= "\"" ( + [^"\\] | + "\\" (["\\/bfnrt] | "u" [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F]) + )* "\"" space + space ::= " "? + )""" + }); + + test({ + SUCCESS, + "boolean", + R"""({ + "type": "boolean" + })""", + R"""( + root ::= ("true" | "false") space + space ::= " "? + )""" + }); + + test({ + SUCCESS, + "integer", + R"""({ + "type": "integer" + })""", + R"""( + root ::= ("-"? ([0-9] | [1-9] [0-9]*)) space + space ::= " "? + )""" + }); + + test({ + SUCCESS, + "string const", + R"""({ + "const": "foo" + })""", + R"""( + root ::= "\"foo\"" + space ::= " "? + )""" + }); + + test({ + FAILURE, + "non-string const", + R"""({ + "const": 123 + })""", + "" + }); + + test({ + FAILURE, + "non-string enum", + R"""({ + "enum": [123] + })""", + "" + }); + + test({ + SUCCESS, + "tuple1", + R"""({ + "prefixItems": [{ "type": "string" }] + })""", + R"""( + root ::= "[" space string "]" space + space ::= " "? + string ::= "\"" ( + [^"\\] | + "\\" (["\\/bfnrt] | "u" [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F]) + )* "\"" space + )""" + }); + + test({ + SUCCESS, + "tuple2", + R"""({ + "prefixItems": [{ "type": "string" }, { "type": "number" }] + })""", + R"""( + number ::= ("-"? ([0-9] | [1-9] [0-9]*)) ("." [0-9]+)? ([eE] [-+]? [0-9]+)? space + root ::= "[" space string "," space number "]" space + space ::= " "? + string ::= "\"" ( + [^"\\] | + "\\" (["\\/bfnrt] | "u" [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F]) + )* "\"" space + )""" + }); + + test({ + SUCCESS, + "number", + R"""({ + "type": "number" + })""", + R"""( + root ::= ("-"? ([0-9] | [1-9] [0-9]*)) ("." [0-9]+)? ([eE] [-+]? [0-9]+)? space + space ::= " "? + )""" + }); + + test({ + SUCCESS, + "minItems", + R"""({ + "items": { + "type": "boolean" + }, + "minItems": 2 + })""", + R"""( + boolean ::= ("true" | "false") space + root ::= "[" space boolean ( "," space boolean )( "," space boolean )* "]" space + space ::= " "? + )""" + }); + + test({ + SUCCESS, + "maxItems 1", + R"""({ + "items": { + "type": "boolean" + }, + "maxItems": 1 + })""", + R"""( + boolean ::= ("true" | "false") space + root ::= "[" space ( boolean )? "]" space + space ::= " "? + )""" + }); + + test({ + SUCCESS, + "maxItems 2", + R"""({ + "items": { + "type": "boolean" + }, + "maxItems": 2 + })""", + R"""( + boolean ::= ("true" | "false") space + root ::= "[" space ( boolean ( "," space boolean )? )? "]" space + space ::= " "? + )""" + }); + + test({ + SUCCESS, + "min + maxItems", + R"""({ + "items": { + "type": ["number", "integer"] + }, + "minItems": 3, + "maxItems": 5 + })""", + R"""( + integer ::= ("-"? ([0-9] | [1-9] [0-9]*)) space + item ::= number | integer + number ::= ("-"? ([0-9] | [1-9] [0-9]*)) ("." [0-9]+)? ([eE] [-+]? [0-9]+)? space + root ::= "[" space item ( "," space item )( "," space item )( "," space item )?( "," space item )? "]" space + space ::= " "? + )""" + }); + + test({ + SUCCESS, + "simple regexp", + R"""({ + "type": "string", + "pattern": "^abc?d*efg+(hij)?kl$" + })""", + R"""( + root ::= "\"" "ab" "c"? "d"* "ef" "g"+ ("hij")? "kl" "\"" space + space ::= " "? + )""" + }); + + test({ + SUCCESS, + "regexp escapes", + R"""({ + "type": "string", + "pattern": "^\\[\\]\\{\\}\\(\\)\\|\\+\\*\\?$" + })""", + R"""( + root ::= "\"" "[]{}()|+*?" "\"" space + space ::= " "? + )""" + }); + + test({ + SUCCESS, + "regexp quote", + R"""({ + "type": "string", + "pattern": "^\"$" + })""", + R"""( + root ::= "\"" "\"" "\"" space + space ::= " "? + )""" + }); + + test({ + SUCCESS, + "regexp", + R"""({ + "type": "string", + "pattern": "^(\\([0-9]{1,3}\\))?[0-9]{3}-[0-9]{4} and...$" + })""", + R"""( + dot ::= [\U00000000-\x09\x0B\x0C\x0E-\U0010FFFF] + root ::= "\"" ("(" root-1 root-1? root-1? ")")? root-1 root-1 root-1 "-" root-1 root-1 root-1 root-1 " and" dot dot dot "\"" space + root-1 ::= [0-9] + space ::= " "? + )""" + }); + + test({ + SUCCESS, + "required props", + R"""({ + "type": "object", + "properties": { + "a": { + "type": "string" + }, + "b": { + "type": "string" + } + }, + "required": [ + "a", + "b" + ], + "additionalProperties": false, + "definitions": {} + })""", + R"""( + a-kv ::= "\"a\"" space ":" space string + b-kv ::= "\"b\"" space ":" space string + root ::= "{" space a-kv "," space b-kv "}" space + space ::= " "? + string ::= "\"" ( + [^"\\] | + "\\" (["\\/bfnrt] | "u" [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F]) + )* "\"" space + )""" + }); + + test({ + SUCCESS, + "1 optional prop", + R"""({ + "properties": { + "a": { + "type": "string" + } + }, + "additionalProperties": false + })""", + R"""( + a-kv ::= "\"a\"" space ":" space string + root ::= "{" space (a-kv )? "}" space + space ::= " "? + string ::= "\"" ( + [^"\\] | + "\\" (["\\/bfnrt] | "u" [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F]) + )* "\"" space + )""" + }); + + test({ + SUCCESS, + "N optional props", + R"""({ + "properties": { + "a": {"type": "string"}, + "b": {"type": "string"}, + "c": {"type": "string"} + }, + "additionalProperties": false + })""", + R"""( + a-kv ::= "\"a\"" space ":" space string + a-rest ::= ( "," space b-kv )? b-rest + b-kv ::= "\"b\"" space ":" space string + b-rest ::= ( "," space c-kv )? + c-kv ::= "\"c\"" space ":" space string + root ::= "{" space (a-kv a-rest | b-kv b-rest | c-kv )? "}" space + space ::= " "? + string ::= "\"" ( + [^"\\] | + "\\" (["\\/bfnrt] | "u" [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F]) + )* "\"" space + )""" + }); + + test({ + SUCCESS, + "required + optional props", + R"""({ + "properties": { + "a": {"type": "string"}, + "b": {"type": "string"}, + "c": {"type": "string"}, + "d": {"type": "string"} + }, + "required": ["a", "b"], + "additionalProperties": false + })""", + R"""( + a-kv ::= "\"a\"" space ":" space string + b-kv ::= "\"b\"" space ":" space string + c-kv ::= "\"c\"" space ":" space string + c-rest ::= ( "," space d-kv )? + d-kv ::= "\"d\"" space ":" space string + root ::= "{" space a-kv "," space b-kv ( "," space ( c-kv c-rest | d-kv ) )? "}" space + space ::= " "? + string ::= "\"" ( + [^"\\] | + "\\" (["\\/bfnrt] | "u" [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F]) + )* "\"" space + )""" + }); + + test({ + SUCCESS, + "additional props", + R"""({ + "type": "object", + "additionalProperties": {"type": "array", "items": {"type": "number"}} + })""", + R"""( + additional-kv ::= string ":" space additional-value + additional-kvs ::= additional-kv ( "," space additional-kv )* + additional-value ::= "[" space ( number ( "," space number )* )? "]" space + number ::= ("-"? ([0-9] | [1-9] [0-9]*)) ("." [0-9]+)? ([eE] [-+]? [0-9]+)? space + root ::= "{" space (additional-kvs )? "}" space + space ::= " "? + string ::= "\"" ( + [^"\\] | + "\\" (["\\/bfnrt] | "u" [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F]) + )* "\"" space + )""" + }); + + test({ + SUCCESS, + "additional props (true)", + R"""({ + "type": "object", + "additionalProperties": true + })""", + R"""( + array ::= "[" space ( value ("," space value)* )? "]" space + boolean ::= ("true" | "false") space + null ::= "null" space + number ::= ("-"? ([0-9] | [1-9] [0-9]*)) ("." [0-9]+)? ([eE] [-+]? [0-9]+)? space + object ::= "{" space ( string ":" space value ("," space string ":" space value)* )? "}" space + root ::= object + space ::= " "? + string ::= "\"" ( + [^"\\] | + "\\" (["\\/bfnrt] | "u" [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F]) + )* "\"" space + value ::= object | array | string | number | boolean + )""" + }); + + test({ + SUCCESS, + "additional props (implicit)", + R"""({ + "type": "object" + })""", + R"""( + array ::= "[" space ( value ("," space value)* )? "]" space + boolean ::= ("true" | "false") space + null ::= "null" space + number ::= ("-"? ([0-9] | [1-9] [0-9]*)) ("." [0-9]+)? ([eE] [-+]? [0-9]+)? space + object ::= "{" space ( string ":" space value ("," space string ":" space value)* )? "}" space + root ::= object + space ::= " "? + string ::= "\"" ( + [^"\\] | + "\\" (["\\/bfnrt] | "u" [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F]) + )* "\"" space + value ::= object | array | string | number | boolean + )""" + }); + + test({ + SUCCESS, + "empty w/o additional props", + R"""({ + "type": "object", + "additionalProperties": false + })""", + R"""( + root ::= "{" space "}" space + space ::= " "? + )""" + }); + + test({ + SUCCESS, + "required + additional props", + R"""({ + "type": "object", + "properties": { + "a": {"type": "number"} + }, + "required": ["a"], + "additionalProperties": {"type": "string"} + })""", + R"""( + a-kv ::= "\"a\"" space ":" space number + additional-kv ::= string ":" space string + additional-kvs ::= additional-kv ( "," space additional-kv )* + number ::= ("-"? ([0-9] | [1-9] [0-9]*)) ("." [0-9]+)? ([eE] [-+]? [0-9]+)? space + root ::= "{" space a-kv ( "," space ( additional-kvs ) )? "}" space + space ::= " "? + string ::= "\"" ( + [^"\\] | + "\\" (["\\/bfnrt] | "u" [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F]) + )* "\"" space + )""" + }); + + test({ + SUCCESS, + "optional + additional props", + R"""({ + "type": "object", + "properties": { + "a": {"type": "number"} + }, + "additionalProperties": {"type": "number"} + })""", + R"""( + a-kv ::= "\"a\"" space ":" space number + a-rest ::= additional-kvs + additional-kv ::= string ":" space number + additional-kvs ::= additional-kv ( "," space additional-kv )* + number ::= ("-"? ([0-9] | [1-9] [0-9]*)) ("." [0-9]+)? ([eE] [-+]? [0-9]+)? space + root ::= "{" space (a-kv a-rest | additional-kvs )? "}" space + space ::= " "? + string ::= "\"" ( + [^"\\] | + "\\" (["\\/bfnrt] | "u" [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F]) + )* "\"" space + )""" + }); + + test({ + SUCCESS, + "required + optional + additional props", + R"""({ + "type": "object", + "properties": { + "a": {"type": "number"}, + "b": {"type": "number"} + }, + "required": ["a"], + "additionalProperties": {"type": "number"} + })""", + R"""( + a-kv ::= "\"a\"" space ":" space number + additional-kv ::= string ":" space number + additional-kvs ::= additional-kv ( "," space additional-kv )* + b-kv ::= "\"b\"" space ":" space number + b-rest ::= additional-kvs + number ::= ("-"? ([0-9] | [1-9] [0-9]*)) ("." [0-9]+)? ([eE] [-+]? [0-9]+)? space + root ::= "{" space a-kv ( "," space ( b-kv b-rest | additional-kvs ) )? "}" space + space ::= " "? + string ::= "\"" ( + [^"\\] | + "\\" (["\\/bfnrt] | "u" [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F]) + )* "\"" space + )""" + }); + + test({ + SUCCESS, + "top-level $ref", + R"""({ + "$ref": "#/definitions/MyType", + "definitions": { + "MyType": { + "type": "object", + "properties": { + "a": { + "type": "string" + } + }, + "required": [ + "a" + ], + "additionalProperties": false + } + } + })""", + R"""( + MyType ::= "{" space MyType-a-kv "}" space + MyType-a-kv ::= "\"a\"" space ":" space string + root ::= MyType + space ::= " "? + string ::= "\"" ( + [^"\\] | + "\\" (["\\/bfnrt] | "u" [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F]) + )* "\"" space + )""" + }); + + test({ + SUCCESS, + "anyOf", + R"""({ + "anyOf": [ + {"$ref": "#/definitions/foo"}, + {"$ref": "#/definitions/bar"} + ], + "definitions": { + "foo": { + "properties": {"a": {"type": "number"}} + }, + "bar": { + "properties": {"b": {"type": "number"}} + } + }, + "type": "object" + })""", + R"""( + alternative-0 ::= foo + alternative-1 ::= bar + bar ::= "{" space (bar-b-kv )? "}" space + bar-b-kv ::= "\"b\"" space ":" space number + foo ::= "{" space (foo-a-kv )? "}" space + foo-a-kv ::= "\"a\"" space ":" space number + number ::= ("-"? ([0-9] | [1-9] [0-9]*)) ("." [0-9]+)? ([eE] [-+]? [0-9]+)? space + root ::= alternative-0 | alternative-1 + space ::= " "? + )""" + }); + + test({ + SUCCESS, + "mix of allOf, anyOf and $ref (similar to https://json.schemastore.org/tsconfig.json)", + R"""({ + "allOf": [ + {"$ref": "#/definitions/foo"}, + {"$ref": "#/definitions/bar"}, + { + "anyOf": [ + {"$ref": "#/definitions/baz"}, + {"$ref": "#/definitions/bam"} + ] + } + ], + "definitions": { + "foo": { + "properties": {"a": {"type": "number"}} + }, + "bar": { + "properties": {"b": {"type": "number"}} + }, + "bam": { + "properties": {"c": {"type": "number"}} + }, + "baz": { + "properties": {"d": {"type": "number"}} + } + }, + "type": "object" + })""", + R"""( + a-kv ::= "\"a\"" space ":" space number + b-kv ::= "\"b\"" space ":" space number + c-kv ::= "\"c\"" space ":" space number + d-kv ::= "\"d\"" space ":" space number + d-rest ::= ( "," space c-kv )? + number ::= ("-"? ([0-9] | [1-9] [0-9]*)) ("." [0-9]+)? ([eE] [-+]? [0-9]+)? space + root ::= "{" space a-kv "," space b-kv ( "," space ( d-kv d-rest | c-kv ) )? "}" space + space ::= " "? + )""" + }); + + test({ + SUCCESS, + "conflicting names", + R"""({ + "type": "object", + "properties": { + "number": { + "type": "object", + "properties": { + "number": { + "type": "object", + "properties": { + "root": { + "type": "number" + } + }, + "required": [ + "root" + ], + "additionalProperties": false + } + }, + "required": [ + "number" + ], + "additionalProperties": false + } + }, + "required": [ + "number" + ], + "additionalProperties": false, + "definitions": {} + })""", + R"""( + number ::= ("-"? ([0-9] | [1-9] [0-9]*)) ("." [0-9]+)? ([eE] [-+]? [0-9]+)? space + number- ::= "{" space number-number-kv "}" space + number-kv ::= "\"number\"" space ":" space number- + number-number ::= "{" space number-number-root-kv "}" space + number-number-kv ::= "\"number\"" space ":" space number-number + number-number-root-kv ::= "\"root\"" space ":" space number + root ::= "{" space number-kv "}" space + space ::= " "? + )""" + }); +} + +int main() { + test_all("C++", [](const TestCase & tc) { + try { + tc.verify(json_schema_to_grammar(nlohmann::json::parse(tc.schema))); + tc.verify_status(SUCCESS); + } catch (const std::runtime_error & ex) { + fprintf(stderr, "Error: %s\n", ex.what()); + tc.verify_status(FAILURE); + } + }); + test_all("Python", [](const TestCase & tc) { + write("test-json-schema-input.tmp", tc.schema); + tc.verify_status(std::system( + "python ./examples/json-schema-to-grammar.py test-json-schema-input.tmp > test-grammar-output.tmp") == 0 ? SUCCESS : FAILURE); + tc.verify(read("test-grammar-output.tmp")); + }); + test_all("JavaScript", [](const TestCase & tc) { + write("test-json-schema-input.tmp", tc.schema); + tc.verify_status(std::system( + "node ./tests/run-json-schema-to-grammar.mjs test-json-schema-input.tmp > test-grammar-output.tmp") == 0 ? SUCCESS : FAILURE); + tc.verify(read("test-grammar-output.tmp")); + }); + + test_all("Check Expectations Validity", [](const TestCase & tc) { + if (tc.expected_status == SUCCESS) { + tc.verify_expectation_parseable(); + } + }); +} From cfd3be76e37dab92c846d75a2421178f20db4a11 Mon Sep 17 00:00:00 2001 From: Kawrakow <48489457+ikawrakow@users.noreply.github.com> Date: Thu, 21 Mar 2024 13:59:38 +0100 Subject: [PATCH 681/812] ggml : same IQ4_NL quantization for CPU/CUDA/Metal (#6196) * Make quantize_row_iq4_nl do the same thing is quantization on CUDA * Make quantize_row_iq4_nl do the same thing is quantization on CUDA This time for real. backend-ops tests pass. * Now fix test-quantize-fns --------- Co-authored-by: Iwan Kawrakow --- ggml-quants.c | 38 +++++++++++++++++++++++++------------- 1 file changed, 25 insertions(+), 13 deletions(-) diff --git a/ggml-quants.c b/ggml-quants.c index 109dd6660..2eaca0593 100644 --- a/ggml-quants.c +++ b/ggml-quants.c @@ -11705,9 +11705,8 @@ static void quantize_row_iq4_nl_impl(const int super_block_size, const int block ggml_fp16_t * dh, uint8_t * q4, uint16_t * scales_h, uint8_t * scales_l, float * scales, float * weight, uint8_t * L, const int8_t * values, - const float * quant_weights) { - - const int ntry = 7; + const float * quant_weights, + const int ntry) { float sigma2 = 0; for (int j = 0; j < super_block_size; ++j) sigma2 += x[j]*x[j]; @@ -11719,6 +11718,7 @@ static void quantize_row_iq4_nl_impl(const int super_block_size, const int block float max_scale = 0, amax_scale = 0; for (int ib = 0; ib < super_block_size/block_size; ++ib) { const float * xb = x + ib*block_size; + uint8_t * Lb = L + ib*block_size; if (quant_weights) { const float * qw = quant_weights + ib*block_size; for (int j = 0; j < block_size; ++j) weight[j] = qw[j] * sqrtf(sigma2 + xb[j]*xb[j]); @@ -11736,12 +11736,13 @@ static void quantize_row_iq4_nl_impl(const int super_block_size, const int block scales[ib] = 0; continue; } - float d = -max/values[0]; + float d = ntry > 0 ? -max/values[0] : max/values[0]; float id = 1/d; float sumqx = 0, sumq2 = 0; for (int j = 0; j < block_size; ++j) { float al = id*xb[j]; int l = best_index_int8(16, values, al); + Lb[j] = l; float q = values[l]; float w = weight[j]; sumqx += w*q*xb[j]; @@ -11796,9 +11797,11 @@ static void quantize_row_iq4_nl_impl(const int super_block_size, const int block } } else { dh[0] = GGML_FP32_TO_FP16(scales[0]); - float id = scales[0] ? 1/scales[0] : 0; - for (int j = 0; j < super_block_size; ++j) { - L[j] = best_index_int8(16, values, id*x[j]); + if (ntry > 0) { + float id = scales[0] ? 1/scales[0] : 0; + for (int j = 0; j < super_block_size; ++j) { + L[j] = best_index_int8(16, values, id*x[j]); + } } } @@ -11823,7 +11826,7 @@ size_t quantize_iq4_nl(const float * restrict src, void * restrict dst, int nrow for (int ibl = 0; ibl < nblock; ++ibl) { const float * qw = quant_weights ? quant_weights + QK4_NL*ibl : NULL; quantize_row_iq4_nl_impl(QK4_NL, 32, src + QK4_NL*ibl, &iq4[ibl].d, iq4[ibl].qs, &unused_h, unused_l, - &scale, weight, L, kvalues_iq4nl, qw); + &scale, weight, L, kvalues_iq4nl, qw, 7); } src += n_per_row; qrow += nblock*sizeof(block_iq4_nl); @@ -11832,14 +11835,23 @@ size_t quantize_iq4_nl(const float * restrict src, void * restrict dst, int nrow } void quantize_row_iq4_nl(const float * restrict x, void * restrict vy, int k) { - assert(k % QK4_NL == 0); - block_iq4_nl * restrict y = vy; - quantize_row_iq4_nl_reference(x, y, k); + GGML_ASSERT(k%QK4_NL == 0); + int nblock = k/QK4_NL; + uint8_t L[QK4_NL]; + float weight[QK4_NL]; + uint16_t unused_h; + uint8_t * unused_l = NULL; + float scale; + block_iq4_nl * iq4 = (block_iq4_nl *)vy; + for (int ibl = 0; ibl < nblock; ++ibl) { + quantize_row_iq4_nl_impl(QK4_NL, 32, x + QK4_NL*ibl, &iq4[ibl].d, iq4[ibl].qs, &unused_h, unused_l, + &scale, weight, L, kvalues_iq4nl, NULL, -1); + } } void quantize_row_iq4_nl_reference(const float * restrict x, block_iq4_nl * restrict y, int k) { assert(k % QK4_NL == 0); - quantize_iq4_nl(x, y, 1, k, NULL); + quantize_row_iq4_nl(x, y, k); } size_t quantize_iq4_xs(const float * restrict src, void * restrict dst, int nrow, int n_per_row, const float * quant_weights) { @@ -11857,7 +11869,7 @@ size_t quantize_iq4_xs(const float * restrict src, void * restrict dst, int nrow for (int ibl = 0; ibl < nblock; ++ibl) { const float * qw = quant_weights ? quant_weights + QK_K*ibl : NULL; quantize_row_iq4_nl_impl(QK_K, 32, src + QK_K*ibl, &iq4[ibl].d, iq4[ibl].qs, &iq4[ibl].scales_h, iq4[ibl].scales_l, - scales, weight, L, kvalues_iq4nl, qw); + scales, weight, L, kvalues_iq4nl, qw, 7); } src += n_per_row; qrow += nblock*sizeof(block_iq4_xs); From 03a8f8fafec2e21009be9fe7297d92e5d4538964 Mon Sep 17 00:00:00 2001 From: slaren Date: Thu, 21 Mar 2024 13:59:53 +0100 Subject: [PATCH 682/812] cuda : fix LLAMA_CUDA_F16 build (#6197) --- ggml-cuda.cu | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ggml-cuda.cu b/ggml-cuda.cu index 280839ea4..04c6f5d07 100644 --- a/ggml-cuda.cu +++ b/ggml-cuda.cu @@ -9453,7 +9453,7 @@ static void ggml_cuda_op_dequantize_mul_mat_vec( // on some GPUs it is faster to convert src1 to half and to use half precision intrinsics #ifdef GGML_CUDA_F16 - cuda_pool_alloc src1_dfloat_a; + ggml_cuda_pool_alloc src1_dfloat_a(ctx.pool()); half * src1_dfloat = nullptr; // dfloat == half bool src1_convert_f16 = From 924ce1dce7fdf54894b462d03e7b608a6e574c32 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Thu, 21 Mar 2024 16:20:05 +0200 Subject: [PATCH 683/812] tests : disable system() calls (#6198) ggml-ci --- tests/test-json-schema-to-grammar.cpp | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/tests/test-json-schema-to-grammar.cpp b/tests/test-json-schema-to-grammar.cpp index a13bb9d7f..3c40e7563 100755 --- a/tests/test-json-schema-to-grammar.cpp +++ b/tests/test-json-schema-to-grammar.cpp @@ -803,18 +803,18 @@ int main() { tc.verify_status(FAILURE); } }); - test_all("Python", [](const TestCase & tc) { - write("test-json-schema-input.tmp", tc.schema); - tc.verify_status(std::system( - "python ./examples/json-schema-to-grammar.py test-json-schema-input.tmp > test-grammar-output.tmp") == 0 ? SUCCESS : FAILURE); - tc.verify(read("test-grammar-output.tmp")); - }); - test_all("JavaScript", [](const TestCase & tc) { - write("test-json-schema-input.tmp", tc.schema); - tc.verify_status(std::system( - "node ./tests/run-json-schema-to-grammar.mjs test-json-schema-input.tmp > test-grammar-output.tmp") == 0 ? SUCCESS : FAILURE); - tc.verify(read("test-grammar-output.tmp")); - }); + //test_all("Python", [](const TestCase & tc) { + // write("test-json-schema-input.tmp", tc.schema); + // tc.verify_status(std::system( + // "python ./examples/json-schema-to-grammar.py test-json-schema-input.tmp > test-grammar-output.tmp") == 0 ? SUCCESS : FAILURE); + // tc.verify(read("test-grammar-output.tmp")); + //}); + //test_all("JavaScript", [](const TestCase & tc) { + // write("test-json-schema-input.tmp", tc.schema); + // tc.verify_status(std::system( + // "node ./tests/run-json-schema-to-grammar.mjs test-json-schema-input.tmp > test-grammar-output.tmp") == 0 ? SUCCESS : FAILURE); + // tc.verify(read("test-grammar-output.tmp")); + //}); test_all("Check Expectations Validity", [](const TestCase & tc) { if (tc.expected_status == SUCCESS) { From f372c49ccdc561ab96fb3c7d2b7cbc0f89a4b359 Mon Sep 17 00:00:00 2001 From: semidark Date: Thu, 21 Mar 2024 11:52:35 -0600 Subject: [PATCH 684/812] Corrected typo to wrong file (#6199) The stated file `./devops/main-server.Dockerfile` does not exist. I figure that `.devops/server-intel.Dockerfile` was meant. --- README-sycl.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README-sycl.md b/README-sycl.md index 32adfda47..501b9d481 100644 --- a/README-sycl.md +++ b/README-sycl.md @@ -115,7 +115,7 @@ You can choose between **F16** and **F32** build. F16 is faster for long-prompt # Or, for F32: docker build -t llama-cpp-sycl -f .devops/main-intel.Dockerfile . -# Note: you can also use the ".devops/main-server.Dockerfile", which compiles the "server" example +# Note: you can also use the ".devops/server-intel.Dockerfile", which compiles the "server" example ``` ### Run From d0a71233fbf8ade8ef06ad8e6b81d1d7b254895f Mon Sep 17 00:00:00 2001 From: slaren Date: Thu, 21 Mar 2024 19:54:28 +0100 Subject: [PATCH 685/812] cuda : disable host register by default (#6206) --- ggml-cuda.cu | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/ggml-cuda.cu b/ggml-cuda.cu index 04c6f5d07..14f409eb1 100644 --- a/ggml-cuda.cu +++ b/ggml-cuda.cu @@ -11646,7 +11646,7 @@ GGML_CALL void ggml_backend_cuda_get_device_memory(int device, size_t * free, si } GGML_CALL bool ggml_backend_cuda_register_host_buffer(void * buffer, size_t size) { - if (getenv("GGML_CUDA_NO_PINNED") != nullptr) { + if (getenv("GGML_CUDA_REGISTER_HOST") == nullptr) { return false; } @@ -11663,6 +11663,10 @@ GGML_CALL bool ggml_backend_cuda_register_host_buffer(void * buffer, size_t size } GGML_CALL void ggml_backend_cuda_unregister_host_buffer(void * buffer) { + if (getenv("GGML_CUDA_REGISTER_HOST") == nullptr) { + return; + } + cudaError_t err = cudaHostUnregister(buffer); if (err != cudaSuccess) { // clear the error From be07a03217376c53b1d95e5f658adbbbb2831555 Mon Sep 17 00:00:00 2001 From: Jan Boon Date: Fri, 22 Mar 2024 06:41:24 +0800 Subject: [PATCH 686/812] server : update readme doc from `slot_id` to `id_slot` (#6213) --- examples/server/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/server/README.md b/examples/server/README.md index 755e1d538..355601ff4 100644 --- a/examples/server/README.md +++ b/examples/server/README.md @@ -260,7 +260,7 @@ node index.js `image_data`: An array of objects to hold base64-encoded image `data` and its `id`s to be reference in `prompt`. You can determine the place of the image in the prompt as in the following: `USER:[img-12]Describe the image in detail.\nASSISTANT:`. In this case, `[img-12]` will be replaced by the embeddings of the image with id `12` in the following `image_data` array: `{..., "image_data": [{"data": "", "id": 12}]}`. Use `image_data` only with multimodal models, e.g., LLaVA. - `slot_id`: Assign the completion task to an specific slot. If is -1 the task will be assigned to a Idle slot (default: -1) + `id_slot`: Assign the completion task to an specific slot. If is -1 the task will be assigned to a Idle slot (default: -1) `cache_prompt`: Re-use previously cached prompt from the last request if possible. This may prevent re-caching the prompt from scratch. (default: false) From fa046eafbc70bf97dcf39843af0323f19a8c9ac3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?DAN=E2=84=A2?= Date: Thu, 21 Mar 2024 21:32:42 -0400 Subject: [PATCH 687/812] Fix params underscore convert to dash. (#6203) * Fix params underscore convert to dash. * Update common/common.cpp --------- Co-authored-by: slaren --- common/common.cpp | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/common/common.cpp b/common/common.cpp index 192182d05..11bf554d2 100644 --- a/common/common.cpp +++ b/common/common.cpp @@ -154,8 +154,7 @@ bool gpt_params_parse(int argc, char ** argv, gpt_params & params) { return result; } -static bool gpt_params_find_arg(int argc, char ** argv, gpt_params & params, int & i, bool & invalid_param) { - std::string arg = argv[i]; +static bool gpt_params_find_arg(int argc, char ** argv, const std::string & arg, gpt_params & params, int & i, bool & invalid_param) { llama_sampling_params& sparams = params.sparams; if (arg == "-s" || arg == "--seed") { @@ -1201,7 +1200,7 @@ bool gpt_params_parse_ex(int argc, char ** argv, gpt_params & params) { std::replace(arg.begin(), arg.end(), '_', '-'); } - if (!gpt_params_find_arg(argc, argv, params, i, invalid_param)) { + if (!gpt_params_find_arg(argc, argv, arg, params, i, invalid_param)) { throw std::invalid_argument("error: unknown argument: " + arg); } } From 59c17f02de8fdf7b084d6100b875b7e2bc07a83b Mon Sep 17 00:00:00 2001 From: Neo Zhang Jianyu Date: Fri, 22 Mar 2024 15:19:37 +0800 Subject: [PATCH 688/812] add blog link (#6222) --- README-sycl.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README-sycl.md b/README-sycl.md index 501b9d481..cbf14f2da 100644 --- a/README-sycl.md +++ b/README-sycl.md @@ -29,6 +29,7 @@ For Intel CPU, recommend to use llama.cpp for X86 (Intel MKL building). ## News - 2024.3 + - A blog is published: **Run LLM on all Intel GPUs Using llama.cpp**: [intel.com](https://www.intel.com/content/www/us/en/developer/articles/technical/run-llm-on-all-gpus-using-llama-cpp-artical.html) or [medium.com](https://medium.com/@jianyu_neo/run-llm-on-all-intel-gpus-using-llama-cpp-fd2e2dcbd9bd). - New base line is ready: [tag b2437](https://github.com/ggerganov/llama.cpp/tree/b2437). - Support multiple cards: **--split-mode**: [none|layer]; not support [row], it's on developing. - Support to assign main GPU by **--main-gpu**, replace $GGML_SYCL_DEVICE. From 95d576b48ebf582b112d1c9cf4eed7142fa4e464 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Fri, 22 Mar 2024 09:36:03 +0200 Subject: [PATCH 689/812] metal : pad n_ctx by 32 (#6177) * metal : require ne00 >= 128 for mat-mat kernels ggml-ci * llama : pad n_ctx by 32 ggml-ci --- common/common.cpp | 2 +- examples/batched/batched.cpp | 4 +++- llama.cpp | 3 +++ tests/test-backend-ops.cpp | 7 +++++++ 4 files changed, 14 insertions(+), 2 deletions(-) diff --git a/common/common.cpp b/common/common.cpp index 11bf554d2..cc230c9ff 100644 --- a/common/common.cpp +++ b/common/common.cpp @@ -101,7 +101,7 @@ int32_t get_num_physical_cores() { return n_threads > 0 ? (n_threads <= 4 ? n_threads : n_threads / 2) : 4; } -void process_escapes(std::string& input) { +void process_escapes(std::string & input) { std::size_t input_len = input.length(); std::size_t output_idx = 0; diff --git a/examples/batched/batched.cpp b/examples/batched/batched.cpp index ee1f8f1bf..7aaf63ceb 100644 --- a/examples/batched/batched.cpp +++ b/examples/batched/batched.cpp @@ -48,6 +48,8 @@ int main(int argc, char ** argv) { params.prompt = "Hello my name is"; } + process_escapes(params.prompt); + // init LLM llama_backend_init(); @@ -78,7 +80,7 @@ int main(int argc, char ** argv) { llama_context_params ctx_params = llama_context_default_params(); ctx_params.seed = 1234; - ctx_params.n_ctx = n_kv_req; + ctx_params.n_ctx = n_kv_req; ctx_params.n_batch = std::max(n_len, n_parallel); ctx_params.n_seq_max = n_parallel; ctx_params.n_threads = params.n_threads; diff --git a/llama.cpp b/llama.cpp index 1a9fe0c4d..9de4a8602 100644 --- a/llama.cpp +++ b/llama.cpp @@ -13044,6 +13044,9 @@ struct llama_context * llama_new_context_with_model( cparams.rope_freq_base = params.rope_freq_base == 0.0f ? hparams.rope_freq_base_train : params.rope_freq_base; cparams.rope_freq_scale = params.rope_freq_scale == 0.0f ? hparams.rope_freq_scale_train : params.rope_freq_scale; + // this is necessary due to kv_self.n being padded later during inference + cparams.n_ctx = GGML_PAD(cparams.n_ctx, 32); + // with causal attention, the batch size is limited by the context size cparams.n_batch = hparams.causal_attn ? std::min(cparams.n_ctx, params.n_batch) : params.n_batch; cparams.n_ubatch = std::min(cparams.n_batch, params.n_ubatch == 0 ? params.n_batch : params.n_ubatch); diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index c2916c3e4..1998e1cbc 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -2091,6 +2091,13 @@ static bool test_backend(ggml_backend_t backend, test_mode mode, const char * op } } + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F16, GGML_TYPE_F32, 64, 2, 128, { 8, 1}, {1, 1})); + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F16, GGML_TYPE_F32, 83, 2, 128, { 8, 1}, {4, 1})); + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F16, GGML_TYPE_F32, 64, 2, 64, { 8, 1}, {4, 1})); + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F16, GGML_TYPE_F32, 83, 2, 64, { 8, 1}, {4, 1})); + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F16, GGML_TYPE_F32, 64, 45, 128, { 8, 1}, {4, 1})); + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F16, GGML_TYPE_F32, 128, 45, 64, { 8, 1}, {4, 1})); + for (ggml_type type_a : all_types) { for (ggml_type type_b : {GGML_TYPE_F32 /*, GGML_TYPE_F16 */}) { for (int n_mats : {2, 4, 8}) { From b2075fd6a578f5685060df4baa90ae9e48e98c70 Mon Sep 17 00:00:00 2001 From: Vaibhav Srivastav Date: Fri, 22 Mar 2024 08:53:43 +0100 Subject: [PATCH 690/812] ci : add CURL flag for the mac builds (#6214) --- .github/workflows/build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 0f68e4cc3..b8d66a837 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -41,7 +41,7 @@ jobs: sysctl -a mkdir build cd build - cmake -DLLAMA_FATAL_WARNINGS=ON -DLLAMA_METAL_EMBED_LIBRARY=ON .. + cmake -DLLAMA_FATAL_WARNINGS=ON -DLLAMA_METAL_EMBED_LIBRARY=ON -DLLAMA_CURL=ON .. cmake --build . --config Release -j $(sysctl -n hw.logicalcpu) - name: Test @@ -97,7 +97,7 @@ jobs: sysctl -a mkdir build cd build - cmake -DLLAMA_FATAL_WARNINGS=ON -DLLAMA_METAL_EMBED_LIBRARY=ON .. + cmake -DLLAMA_FATAL_WARNINGS=ON -DLLAMA_METAL_EMBED_LIBRARY=ON -DLLAMA_CURL=ON .. cmake --build . --config Release -j $(sysctl -n hw.logicalcpu) - name: Test From b3e94f26bab8e3b5338b5902e5af5bb01894cb4a Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Fri, 22 Mar 2024 11:35:53 +0200 Subject: [PATCH 691/812] metal : proper assert for mat-mat memory alignment (#6225) * metal : proper assert for mat-mat memory alignment ggml-ci * readme : add notice about the bug fix * metal : fix the fix ggml-ci --- README.md | 1 + ggml-metal.m | 16 ++++++++++++++++ ggml-metal.metal | 48 ++++++++++++++++++++++++------------------------ 3 files changed, 41 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index c2f3342f0..aa41b9e76 100644 --- a/README.md +++ b/README.md @@ -17,6 +17,7 @@ Inference of Meta's [LLaMA](https://arxiv.org/abs/2302.13971) model (and others) ### Hot topics +- Fix major bug in Metal batched inference https://github.com/ggerganov/llama.cpp/pull/6225 - Multi-GPU pipeline parallelizm support https://github.com/ggerganov/llama.cpp/pull/6017 - Looking for contributions to add Deepseek support: https://github.com/ggerganov/llama.cpp/issues/5981 - Quantization blind testing: https://github.com/ggerganov/llama.cpp/discussions/5962 diff --git a/ggml-metal.m b/ggml-metal.m index 54d1f1d5a..416b24532 100644 --- a/ggml-metal.m +++ b/ggml-metal.m @@ -1392,6 +1392,14 @@ static enum ggml_status ggml_metal_graph_compute( (ne11 > ne11_mm_min || (ggml_is_quantized(src0t) && ne12 > 1))) { //printf("matrix: ne00 = %6d, ne01 = %6d, ne02 = %6d, ne11 = %6d, ne12 = %6d\n", ne00, ne01, ne02, ne11, ne12); + // some Metal matrix data types require aligned pointers + // ref: https://developer.apple.com/metal/Metal-Shading-Language-Specification.pdf (Table 2.5) + switch (src0->type) { + case GGML_TYPE_F32: GGML_ASSERT(nb01 % 16 == 0); break; + case GGML_TYPE_F16: GGML_ASSERT(nb01 % 8 == 0); break; + default: break; + } + id pipeline = nil; switch (src0->type) { @@ -1706,6 +1714,14 @@ static enum ggml_status ggml_metal_graph_compute( ne20 % 32 == 0 && ne20 >= 64 && ne11 > ne11_mm_min) { + // some Metal matrix data types require aligned pointers + // ref: https://developer.apple.com/metal/Metal-Shading-Language-Specification.pdf (Table 2.5) + switch (src0->type) { + case GGML_TYPE_F32: GGML_ASSERT(nb01 % 16 == 0); break; + case GGML_TYPE_F16: GGML_ASSERT(nb01 % 8 == 0); break; + default: break; + } + id pipeline = nil; switch (src2->type) { diff --git a/ggml-metal.metal b/ggml-metal.metal index 5552c8b28..748f0acef 100644 --- a/ggml-metal.metal +++ b/ggml-metal.metal @@ -5760,18 +5760,18 @@ typedef void (mat_mm_t)( threadgroup uchar *, uint3, uint, uint); -template [[host_name("kernel_mul_mm_f32_f32")]] kernel mat_mm_t kernel_mul_mm; -template [[host_name("kernel_mul_mm_f16_f32")]] kernel mat_mm_t kernel_mul_mm; -template [[host_name("kernel_mul_mm_q4_0_f32")]] kernel mat_mm_t kernel_mul_mm; -template [[host_name("kernel_mul_mm_q4_1_f32")]] kernel mat_mm_t kernel_mul_mm; -template [[host_name("kernel_mul_mm_q5_0_f32")]] kernel mat_mm_t kernel_mul_mm; -template [[host_name("kernel_mul_mm_q5_1_f32")]] kernel mat_mm_t kernel_mul_mm; -template [[host_name("kernel_mul_mm_q8_0_f32")]] kernel mat_mm_t kernel_mul_mm; -template [[host_name("kernel_mul_mm_q2_K_f32")]] kernel mat_mm_t kernel_mul_mm; -template [[host_name("kernel_mul_mm_q3_K_f32")]] kernel mat_mm_t kernel_mul_mm; -template [[host_name("kernel_mul_mm_q4_K_f32")]] kernel mat_mm_t kernel_mul_mm; -template [[host_name("kernel_mul_mm_q5_K_f32")]] kernel mat_mm_t kernel_mul_mm; -template [[host_name("kernel_mul_mm_q6_K_f32")]] kernel mat_mm_t kernel_mul_mm; +template [[host_name("kernel_mul_mm_f32_f32")]] kernel mat_mm_t kernel_mul_mm; +template [[host_name("kernel_mul_mm_f16_f32")]] kernel mat_mm_t kernel_mul_mm; +template [[host_name("kernel_mul_mm_q4_0_f32")]] kernel mat_mm_t kernel_mul_mm; +template [[host_name("kernel_mul_mm_q4_1_f32")]] kernel mat_mm_t kernel_mul_mm; +template [[host_name("kernel_mul_mm_q5_0_f32")]] kernel mat_mm_t kernel_mul_mm; +template [[host_name("kernel_mul_mm_q5_1_f32")]] kernel mat_mm_t kernel_mul_mm; +template [[host_name("kernel_mul_mm_q8_0_f32")]] kernel mat_mm_t kernel_mul_mm; +template [[host_name("kernel_mul_mm_q2_K_f32")]] kernel mat_mm_t kernel_mul_mm; +template [[host_name("kernel_mul_mm_q3_K_f32")]] kernel mat_mm_t kernel_mul_mm; +template [[host_name("kernel_mul_mm_q4_K_f32")]] kernel mat_mm_t kernel_mul_mm; +template [[host_name("kernel_mul_mm_q5_K_f32")]] kernel mat_mm_t kernel_mul_mm; +template [[host_name("kernel_mul_mm_q6_K_f32")]] kernel mat_mm_t kernel_mul_mm; template [[host_name("kernel_mul_mm_iq2_xxs_f32")]] kernel mat_mm_t kernel_mul_mm; template [[host_name("kernel_mul_mm_iq2_xs_f32")]] kernel mat_mm_t kernel_mul_mm; template [[host_name("kernel_mul_mm_iq3_xxs_f32")]] kernel mat_mm_t kernel_mul_mm; @@ -5820,18 +5820,18 @@ typedef void (mat_mm_id_t)( threadgroup uchar *, uint3, uint, uint); -template [[host_name("kernel_mul_mm_id_f32_f32")]] kernel mat_mm_id_t kernel_mul_mm_id; -template [[host_name("kernel_mul_mm_id_f16_f32")]] kernel mat_mm_id_t kernel_mul_mm_id; -template [[host_name("kernel_mul_mm_id_q4_0_f32")]] kernel mat_mm_id_t kernel_mul_mm_id; -template [[host_name("kernel_mul_mm_id_q4_1_f32")]] kernel mat_mm_id_t kernel_mul_mm_id; -template [[host_name("kernel_mul_mm_id_q5_0_f32")]] kernel mat_mm_id_t kernel_mul_mm_id; -template [[host_name("kernel_mul_mm_id_q5_1_f32")]] kernel mat_mm_id_t kernel_mul_mm_id; -template [[host_name("kernel_mul_mm_id_q8_0_f32")]] kernel mat_mm_id_t kernel_mul_mm_id; -template [[host_name("kernel_mul_mm_id_q2_K_f32")]] kernel mat_mm_id_t kernel_mul_mm_id; -template [[host_name("kernel_mul_mm_id_q3_K_f32")]] kernel mat_mm_id_t kernel_mul_mm_id; -template [[host_name("kernel_mul_mm_id_q4_K_f32")]] kernel mat_mm_id_t kernel_mul_mm_id; -template [[host_name("kernel_mul_mm_id_q5_K_f32")]] kernel mat_mm_id_t kernel_mul_mm_id; -template [[host_name("kernel_mul_mm_id_q6_K_f32")]] kernel mat_mm_id_t kernel_mul_mm_id; +template [[host_name("kernel_mul_mm_id_f32_f32")]] kernel mat_mm_id_t kernel_mul_mm_id; +template [[host_name("kernel_mul_mm_id_f16_f32")]] kernel mat_mm_id_t kernel_mul_mm_id; +template [[host_name("kernel_mul_mm_id_q4_0_f32")]] kernel mat_mm_id_t kernel_mul_mm_id; +template [[host_name("kernel_mul_mm_id_q4_1_f32")]] kernel mat_mm_id_t kernel_mul_mm_id; +template [[host_name("kernel_mul_mm_id_q5_0_f32")]] kernel mat_mm_id_t kernel_mul_mm_id; +template [[host_name("kernel_mul_mm_id_q5_1_f32")]] kernel mat_mm_id_t kernel_mul_mm_id; +template [[host_name("kernel_mul_mm_id_q8_0_f32")]] kernel mat_mm_id_t kernel_mul_mm_id; +template [[host_name("kernel_mul_mm_id_q2_K_f32")]] kernel mat_mm_id_t kernel_mul_mm_id; +template [[host_name("kernel_mul_mm_id_q3_K_f32")]] kernel mat_mm_id_t kernel_mul_mm_id; +template [[host_name("kernel_mul_mm_id_q4_K_f32")]] kernel mat_mm_id_t kernel_mul_mm_id; +template [[host_name("kernel_mul_mm_id_q5_K_f32")]] kernel mat_mm_id_t kernel_mul_mm_id; +template [[host_name("kernel_mul_mm_id_q6_K_f32")]] kernel mat_mm_id_t kernel_mul_mm_id; template [[host_name("kernel_mul_mm_id_iq2_xxs_f32")]] kernel mat_mm_id_t kernel_mul_mm_id; template [[host_name("kernel_mul_mm_id_iq2_xs_f32")]] kernel mat_mm_id_t kernel_mul_mm_id; template [[host_name("kernel_mul_mm_id_iq3_xxs_f32")]] kernel mat_mm_id_t kernel_mul_mm_id; From 68e210b3543e0cc71268bee0920441747679ee13 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Fri, 22 Mar 2024 13:08:28 +0200 Subject: [PATCH 692/812] server : enable continuous batching by default (#6231) --- common/common.h | 2 +- examples/server/server.cpp | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/common/common.h b/common/common.h index 8dd8a3edc..31fd401b6 100644 --- a/common/common.h +++ b/common/common.h @@ -139,7 +139,7 @@ struct gpt_params { bool interactive_first = false; // wait for user input immediately bool multiline_input = false; // reverse the usage of `\` bool simple_io = false; // improves compatibility with subprocesses and limited consoles - bool cont_batching = false; // insert new sequences for decoding on-the-fly + bool cont_batching = true; // insert new sequences for decoding on-the-fly bool input_prefix_bos = false; // prefix BOS to user inputs, preceding input_prefix bool ignore_eos = false; // ignore generated EOS tokens diff --git a/examples/server/server.cpp b/examples/server/server.cpp index cf075d6c4..c918fabe4 100644 --- a/examples/server/server.cpp +++ b/examples/server/server.cpp @@ -1758,7 +1758,7 @@ struct server_context { } // process in chunks of params.n_batch - int32_t n_batch = llama_n_batch(ctx); + int32_t n_batch = llama_n_batch(ctx); int32_t n_ubatch = llama_n_ubatch(ctx); // next, batch any pending prompts without exceeding n_batch @@ -2225,7 +2225,7 @@ static void server_print_usage(const char * argv0, const gpt_params & params, co printf(" -to N, --timeout N server read/write timeout in seconds (default: %d)\n", sparams.read_timeout); printf(" --embeddings enable embedding vector output (default: %s)\n", params.embedding ? "enabled" : "disabled"); printf(" -np N, --parallel N number of slots for process requests (default: %d)\n", params.n_parallel); - printf(" -cb, --cont-batching enable continuous batching (a.k.a dynamic batching) (default: disabled)\n"); + printf(" -cb, --cont-batching enable continuous batching (a.k.a dynamic batching) (default: enabled)\n"); printf(" -spf FNAME, --system-prompt-file FNAME\n"); printf(" set a file to load a system prompt (initial prompt of all slots), this is useful for chat applications.\n"); printf(" -ctk TYPE, --cache-type-k TYPE\n"); From 6b8bb3a31d260a01020497c5f01025c34222fcb9 Mon Sep 17 00:00:00 2001 From: Jan Boon Date: Fri, 22 Mar 2024 19:12:05 +0800 Subject: [PATCH 693/812] server : fix n_keep always showing as 0 in response (#6211) --- examples/server/server.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/server/server.cpp b/examples/server/server.cpp index c918fabe4..421fa6d41 100644 --- a/examples/server/server.cpp +++ b/examples/server/server.cpp @@ -1247,7 +1247,7 @@ struct server_context { {"penalize_nl", slot.sparams.penalize_nl}, {"stop", slot.params.antiprompt}, {"n_predict", slot.params.n_predict}, // TODO: fix duplicate key n_predict - {"n_keep", params.n_keep}, + {"n_keep", slot.params.n_keep}, {"ignore_eos", ignore_eos}, {"stream", slot.params.stream}, {"logit_bias", slot.sparams.logit_bias}, From 29ab270e65975785cdca3243a3de71ccebc1252a Mon Sep 17 00:00:00 2001 From: Xiaoyi Chen Date: Fri, 22 Mar 2024 04:29:49 -0700 Subject: [PATCH 694/812] readme : add RecurseChat to the list of UIs (#6219) --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index aa41b9e76..368489caa 100644 --- a/README.md +++ b/README.md @@ -166,6 +166,7 @@ Unless otherwise noted these projects are open-source with permissive licensing: - [cztomsik/ava](https://github.com/cztomsik/ava) (MIT) - [ptsochantaris/emeltal](https://github.com/ptsochantaris/emeltal) - [pythops/tenere](https://github.com/pythops/tenere) (AGPL) +- [RecurseChat](https://recurse.chat/) (proprietary) - [semperai/amica](https://github.com/semperai/amica) - [withcatai/catai](https://github.com/withcatai/catai) - [Mobile-Artificial-Intelligence/maid](https://github.com/Mobile-Artificial-Intelligence/maid) (MIT) From 2f0e81e053b41ca28e73a841e7bdbf9820baaa57 Mon Sep 17 00:00:00 2001 From: slaren Date: Fri, 22 Mar 2024 14:05:31 +0100 Subject: [PATCH 695/812] cuda : add LLAMA_CUDA_NO_PEER_COPY to workaround broken ROCm p2p copy (#6208) * cuda : add LLAMA_CUDA_NO_PEER_COPY to workaround broken ROCm p2p copy * add LLAMA_CUDA_NO_PEER_COPY to HIP build --- CMakeLists.txt | 8 ++++++++ Makefile | 9 ++++++--- ggml-cuda.cu | 28 ++++++++++++++++++++-------- 3 files changed, 34 insertions(+), 11 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index fc4cff28f..3333ee1c9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -99,6 +99,7 @@ option(LLAMA_CUDA_F16 "llama: use 16 bit floats for some set(LLAMA_CUDA_KQUANTS_ITER "2" CACHE STRING "llama: iters./thread per block for Q2_K/Q6_K") set(LLAMA_CUDA_PEER_MAX_BATCH_SIZE "128" CACHE STRING "llama: max. batch size for using peer access") +option(LLAMA_CUDA_NO_PEER_COPY "llama: do not use peer to peer copies" OFF) option(LLAMA_CURL "llama: use libcurl to download model from an URL" OFF) option(LLAMA_HIPBLAS "llama: use hipBLAS" OFF) option(LLAMA_HIP_UMA "llama: use HIP unified memory architecture" OFF) @@ -387,6 +388,9 @@ if (LLAMA_CUBLAS) endif() add_compile_definitions(K_QUANTS_PER_ITERATION=${LLAMA_CUDA_KQUANTS_ITER}) add_compile_definitions(GGML_CUDA_PEER_MAX_BATCH_SIZE=${LLAMA_CUDA_PEER_MAX_BATCH_SIZE}) + if (LLAMA_CUDA_NO_PEER_COPY) + add_compile_definitions(GGML_CUDA_NO_PEER_COPY) + endif() if (LLAMA_STATIC) if (WIN32) @@ -531,6 +535,10 @@ if (LLAMA_HIPBLAS) add_compile_definitions(GGML_CUDA_FORCE_MMQ) endif() + if (LLAMA_CUDA_NO_PEER_COPY) + add_compile_definitions(GGML_CUDA_NO_PEER_COPY) + endif() + add_compile_definitions(GGML_CUDA_DMMV_X=${LLAMA_CUDA_DMMV_X}) add_compile_definitions(GGML_CUDA_MMV_Y=${LLAMA_CUDA_MMV_Y}) add_compile_definitions(K_QUANTS_PER_ITERATION=${LLAMA_CUDA_KQUANTS_ITER}) diff --git a/Makefile b/Makefile index 9b72e1dbd..fa112e708 100644 --- a/Makefile +++ b/Makefile @@ -452,9 +452,9 @@ ifdef LLAMA_CUDA_PEER_MAX_BATCH_SIZE else MK_NVCCFLAGS += -DGGML_CUDA_PEER_MAX_BATCH_SIZE=128 endif # LLAMA_CUDA_PEER_MAX_BATCH_SIZE -#ifdef LLAMA_CUDA_CUBLAS -# MK_NVCCFLAGS += -DGGML_CUDA_CUBLAS -#endif # LLAMA_CUDA_CUBLAS +ifdef LLAMA_CUDA_NO_PEER_COPY + MK_NVCCFLAGS += -DGGML_CUDA_NO_PEER_COPY +endif # LLAMA_CUDA_NO_PEER_COPY ifdef LLAMA_CUDA_CCBIN MK_NVCCFLAGS += -ccbin $(LLAMA_CUDA_CCBIN) endif @@ -535,6 +535,9 @@ endif # LLAMA_HIP_UMA ifdef LLAMA_CUDA_FORCE_DMMV HIPFLAGS += -DGGML_CUDA_FORCE_DMMV endif # LLAMA_CUDA_FORCE_DMMV +ifdef LLAMA_CUDA_NO_PEER_COPY + HIPFLAGS += -DGGML_CUDA_NO_PEER_COPY +endif # LLAMA_CUDA_NO_PEER_COPY OBJS += ggml-cuda.o ggml-cuda.o: ggml-cuda.cu ggml-cuda.h $(HIPCC) $(CXXFLAGS) $(HIPFLAGS) -x hip -c -o $@ $< diff --git a/ggml-cuda.cu b/ggml-cuda.cu index 14f409eb1..adf930478 100644 --- a/ggml-cuda.cu +++ b/ggml-cuda.cu @@ -771,7 +771,11 @@ GGML_CALL static bool ggml_backend_cuda_buffer_cpy_tensor(ggml_backend_buffer_t if (src_ctx->device == dst_ctx->device) { CUDA_CHECK(cudaMemcpyAsync(dst->data, src->data, ggml_nbytes(src), cudaMemcpyDeviceToDevice, cudaStreamPerThread)); } else { +#ifdef GGML_CUDA_NO_PEER_COPY + return false; +#else CUDA_CHECK(cudaMemcpyPeerAsync(dst->data, dst_ctx->device, src->data, src_ctx->device, ggml_nbytes(src), cudaStreamPerThread)); +#endif } CUDA_CHECK(cudaStreamSynchronize(cudaStreamPerThread)); return true; @@ -11322,19 +11326,23 @@ GGML_CALL static bool ggml_backend_cuda_cpy_tensor_async(ggml_backend_t backend_ GGML_ASSERT(cuda_ctx_src->device == buf_ctx_src->device); GGML_ASSERT(cuda_ctx_dst->device == buf_ctx_dst->device); + // copy on src stream + if (cuda_ctx_src->device == cuda_ctx_dst->device) { + CUDA_CHECK(cudaMemcpyAsync(dst->data, src->data, ggml_nbytes(dst), cudaMemcpyDeviceToDevice, cuda_ctx_dst->stream())); + } else { +#ifdef GGML_CUDA_NO_PEER_COPY + return false; +#else + CUDA_CHECK(cudaMemcpyPeerAsync(dst->data, cuda_ctx_dst->device, src->data, cuda_ctx_src->device, ggml_nbytes(dst), cuda_ctx_src->stream())); +#endif + } + + // record event on src stream if (!cuda_ctx_src->copy_event) { ggml_cuda_set_device(cuda_ctx_src->device); CUDA_CHECK(cudaEventCreateWithFlags(&cuda_ctx_src->copy_event, cudaEventDisableTiming)); } - // copy on src stream - if (cuda_ctx_src->device == cuda_ctx_dst->device) { - CUDA_CHECK(cudaMemcpyAsync(dst->data, src->data, ggml_nbytes(dst), cudaMemcpyDeviceToDevice, cuda_ctx_dst->stream())); - } else { - CUDA_CHECK(cudaMemcpyPeerAsync(dst->data, cuda_ctx_dst->device, src->data, cuda_ctx_src->device, ggml_nbytes(dst), cuda_ctx_src->stream())); - } - - // record event on src stream CUDA_CHECK(cudaEventRecord(cuda_ctx_src->copy_event, cuda_ctx_src->stream())); // wait on dst stream for the copy to complete @@ -11530,6 +11538,9 @@ GGML_CALL static bool ggml_backend_cuda_offload_op(ggml_backend_t backend, const } static ggml_backend_event_t ggml_backend_cuda_event_new(ggml_backend_t backend) { +#ifdef GGML_CUDA_NO_PEER_COPY + return nullptr; +#else ggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *)backend->context; ggml_cuda_set_device(cuda_ctx->device); @@ -11541,6 +11552,7 @@ static ggml_backend_event_t ggml_backend_cuda_event_new(ggml_backend_t backend) /* .backend = */ backend, /* .context = */ event, }; +#endif } static void ggml_backend_cuda_event_free(ggml_backend_event_t event) { From 72114edf068a9b2f54d3b09e9a198f611be397e8 Mon Sep 17 00:00:00 2001 From: Olivier Chafik Date: Fri, 22 Mar 2024 13:07:44 +0000 Subject: [PATCH 696/812] json-schema-to-grammar : fix order of props + non-str const/enum (#6232) * json: ordered json in server/schema converter to respect orig order * json: ws nits * json: support non-string const / enums --- common/json-schema-to-grammar.cpp | 16 +- common/json-schema-to-grammar.h | 2 +- examples/json-schema-to-grammar.py | 7 +- .../server/json-schema-to-grammar.mjs.hpp | 2825 ++++++++--------- .../server/public/json-schema-to-grammar.mjs | 12 +- examples/server/server.cpp | 2 +- examples/server/utils.hpp | 2 +- tests/test-json-schema-to-grammar.cpp | 101 +- 8 files changed, 1469 insertions(+), 1498 deletions(-) diff --git a/common/json-schema-to-grammar.cpp b/common/json-schema-to-grammar.cpp index dc6d30ef1..0e4680346 100644 --- a/common/json-schema-to-grammar.cpp +++ b/common/json-schema-to-grammar.cpp @@ -9,7 +9,7 @@ #include #include -using json = nlohmann::json; +using json = nlohmann::ordered_json; const std::string SPACE_RULE = "\" \"?"; @@ -124,7 +124,7 @@ static std::string replacePattern(const std::string & input, const std::regex & } static std::string format_literal(const std::string & literal) { - std::string escaped = replacePattern(json(literal).dump(), GRAMMAR_LITERAL_ESCAPE_RE, [&](const std::smatch & match) { + std::string escaped = replacePattern(literal, GRAMMAR_LITERAL_ESCAPE_RE, [&](const std::smatch & match) { char c = match.str()[0]; return GRAMMAR_LITERAL_ESCAPES.at(c); }); @@ -137,7 +137,7 @@ private: std::function _fetch_json; bool _dotall; std::map _rules; - std::unordered_map _refs; + std::unordered_map _refs; std::unordered_set _refs_being_resolved; std::vector _errors; std::vector _warnings; @@ -413,7 +413,7 @@ private: std::string prop_rule_name = visit(prop_schema, name + (name.empty() ? "" : "-") + prop_name); prop_kv_rule_names[prop_name] = _add_rule( name + (name.empty() ? "" : "-") + prop_name + "-kv", - format_literal(prop_name) + " space \":\" space " + prop_rule_name + format_literal(json(prop_name).dump()) + " space \":\" space " + prop_rule_name ); if (required.find(prop_name) != required.end()) { required_props.push_back(prop_name); @@ -495,7 +495,7 @@ public: _rules["space"] = SPACE_RULE; } - void resolve_refs(nlohmann::json & schema, const std::string & url) { + void resolve_refs(json & schema, const std::string & url) { /* * Resolves all $ref fields in the given schema, fetching any remote schemas, * replacing each $ref with absolute reference URL and populates _refs with the @@ -557,11 +557,7 @@ public: } std::string _generate_constant_rule(const json & value) { - if (!value.is_string()) { - _errors.push_back("Only std::string constants are supported, got " + value.dump()); - return ""; - } - return format_literal(value.get()); + return format_literal(value.dump()); } std::string visit(const json & schema, const std::string & name) { diff --git a/common/json-schema-to-grammar.h b/common/json-schema-to-grammar.h index 2e4c6ade3..e1abed303 100644 --- a/common/json-schema-to-grammar.h +++ b/common/json-schema-to-grammar.h @@ -1,4 +1,4 @@ #pragma once #include "json.hpp" -std::string json_schema_to_grammar(const nlohmann::json& schema); +std::string json_schema_to_grammar(const nlohmann::ordered_json& schema); diff --git a/examples/json-schema-to-grammar.py b/examples/json-schema-to-grammar.py index 9eead557f..91dd734cc 100755 --- a/examples/json-schema-to-grammar.py +++ b/examples/json-schema-to-grammar.py @@ -61,7 +61,7 @@ class SchemaConverter: def _format_literal(self, literal): escaped = GRAMMAR_LITERAL_ESCAPE_RE.sub( - lambda m: GRAMMAR_LITERAL_ESCAPES.get(m.group(0)), json.dumps(literal) + lambda m: GRAMMAR_LITERAL_ESCAPES.get(m.group(0)), literal ) return f'"{escaped}"' @@ -308,8 +308,7 @@ class SchemaConverter: return ref_name def _generate_constant_rule(self, value): - assert isinstance(value, str), f'Only string constants are supported, got {value}' - return self._format_literal(value) + return self._format_literal(json.dumps(value)) def visit(self, schema, name): schema_type = schema.get('type') @@ -428,7 +427,7 @@ class SchemaConverter: prop_rule_name = self.visit(prop_schema, f'{name}{"-" if name else ""}{prop_name}') prop_kv_rule_names[prop_name] = self._add_rule( f'{name}{"-" if name else ""}{prop_name}-kv', - fr'{self._format_literal(prop_name)} space ":" space {prop_rule_name}' + fr'{self._format_literal(json.dumps(prop_name))} space ":" space {prop_rule_name}' ) required_props = [k for k in sorted_props if k in required] optional_props = [k for k in sorted_props if k not in required] diff --git a/examples/server/json-schema-to-grammar.mjs.hpp b/examples/server/json-schema-to-grammar.mjs.hpp index 72fb6b80f..ef1d5dcf3 100644 --- a/examples/server/json-schema-to-grammar.mjs.hpp +++ b/examples/server/json-schema-to-grammar.mjs.hpp @@ -189,1479 +189,1456 @@ unsigned char json_schema_to_grammar_mjs[] = { 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x4c, 0x69, 0x74, 0x65, 0x72, 0x61, 0x6c, 0x28, 0x6c, 0x69, 0x74, 0x65, 0x72, 0x61, 0x6c, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x65, - 0x73, 0x63, 0x61, 0x70, 0x65, 0x64, 0x20, 0x3d, 0x20, 0x4a, 0x53, 0x4f, + 0x73, 0x63, 0x61, 0x70, 0x65, 0x64, 0x20, 0x3d, 0x20, 0x6c, 0x69, 0x74, + 0x65, 0x72, 0x61, 0x6c, 0x2e, 0x72, 0x65, 0x70, 0x6c, 0x61, 0x63, 0x65, + 0x28, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x47, 0x52, 0x41, 0x4d, + 0x4d, 0x41, 0x52, 0x5f, 0x4c, 0x49, 0x54, 0x45, 0x52, 0x41, 0x4c, 0x5f, + 0x45, 0x53, 0x43, 0x41, 0x50, 0x45, 0x5f, 0x52, 0x45, 0x2c, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x6d, 0x20, 0x3d, 0x3e, 0x20, 0x47, 0x52, + 0x41, 0x4d, 0x4d, 0x41, 0x52, 0x5f, 0x4c, 0x49, 0x54, 0x45, 0x52, 0x41, + 0x4c, 0x5f, 0x45, 0x53, 0x43, 0x41, 0x50, 0x45, 0x53, 0x5b, 0x6d, 0x5d, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x60, 0x22, 0x24, 0x7b, 0x65, + 0x73, 0x63, 0x61, 0x70, 0x65, 0x64, 0x7d, 0x22, 0x60, 0x3b, 0x0a, 0x20, + 0x20, 0x7d, 0x0a, 0x0a, 0x20, 0x20, 0x5f, 0x66, 0x6f, 0x72, 0x6d, 0x61, + 0x74, 0x52, 0x61, 0x6e, 0x67, 0x65, 0x43, 0x68, 0x61, 0x72, 0x28, 0x6c, + 0x69, 0x74, 0x65, 0x72, 0x61, 0x6c, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x4a, 0x53, 0x4f, 0x4e, 0x2e, 0x73, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x69, 0x66, 0x79, 0x28, - 0x6c, 0x69, 0x74, 0x65, 0x72, 0x61, 0x6c, 0x29, 0x2e, 0x72, 0x65, 0x70, - 0x6c, 0x61, 0x63, 0x65, 0x28, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x47, 0x52, 0x41, 0x4d, 0x4d, 0x41, 0x52, 0x5f, 0x4c, 0x49, 0x54, 0x45, - 0x52, 0x41, 0x4c, 0x5f, 0x45, 0x53, 0x43, 0x41, 0x50, 0x45, 0x5f, 0x52, - 0x45, 0x2c, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x6d, 0x20, 0x3d, - 0x3e, 0x20, 0x47, 0x52, 0x41, 0x4d, 0x4d, 0x41, 0x52, 0x5f, 0x4c, 0x49, - 0x54, 0x45, 0x52, 0x41, 0x4c, 0x5f, 0x45, 0x53, 0x43, 0x41, 0x50, 0x45, - 0x53, 0x5b, 0x6d, 0x5d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x29, 0x3b, 0x0a, - 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x60, - 0x22, 0x24, 0x7b, 0x65, 0x73, 0x63, 0x61, 0x70, 0x65, 0x64, 0x7d, 0x22, - 0x60, 0x3b, 0x0a, 0x20, 0x20, 0x7d, 0x0a, 0x0a, 0x20, 0x20, 0x5f, 0x66, - 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x52, 0x61, 0x6e, 0x67, 0x65, 0x43, 0x68, - 0x61, 0x72, 0x28, 0x6c, 0x69, 0x74, 0x65, 0x72, 0x61, 0x6c, 0x29, 0x20, - 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, - 0x20, 0x4a, 0x53, 0x4f, 0x4e, 0x2e, 0x73, 0x74, 0x72, 0x69, 0x6e, 0x67, - 0x69, 0x66, 0x79, 0x28, 0x6c, 0x69, 0x74, 0x65, 0x72, 0x61, 0x6c, 0x29, - 0x2e, 0x73, 0x6c, 0x69, 0x63, 0x65, 0x28, 0x31, 0x2c, 0x20, 0x2d, 0x31, - 0x29, 0x2e, 0x72, 0x65, 0x70, 0x6c, 0x61, 0x63, 0x65, 0x28, 0x0a, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x47, 0x52, 0x41, 0x4d, 0x4d, 0x41, 0x52, - 0x5f, 0x52, 0x41, 0x4e, 0x47, 0x45, 0x5f, 0x4c, 0x49, 0x54, 0x45, 0x52, - 0x41, 0x4c, 0x5f, 0x45, 0x53, 0x43, 0x41, 0x50, 0x45, 0x5f, 0x52, 0x45, - 0x2c, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x6d, 0x20, 0x3d, 0x3e, - 0x20, 0x47, 0x52, 0x41, 0x4d, 0x4d, 0x41, 0x52, 0x5f, 0x4c, 0x49, 0x54, - 0x45, 0x52, 0x41, 0x4c, 0x5f, 0x45, 0x53, 0x43, 0x41, 0x50, 0x45, 0x53, - 0x5b, 0x6d, 0x5d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x29, 0x3b, 0x0a, 0x20, - 0x20, 0x7d, 0x0a, 0x0a, 0x20, 0x20, 0x5f, 0x61, 0x64, 0x64, 0x52, 0x75, - 0x6c, 0x65, 0x28, 0x6e, 0x61, 0x6d, 0x65, 0x2c, 0x20, 0x72, 0x75, 0x6c, - 0x65, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x6c, 0x65, 0x74, - 0x20, 0x65, 0x73, 0x63, 0x4e, 0x61, 0x6d, 0x65, 0x20, 0x3d, 0x20, 0x6e, - 0x61, 0x6d, 0x65, 0x2e, 0x72, 0x65, 0x70, 0x6c, 0x61, 0x63, 0x65, 0x28, - 0x49, 0x4e, 0x56, 0x41, 0x4c, 0x49, 0x44, 0x5f, 0x52, 0x55, 0x4c, 0x45, - 0x5f, 0x43, 0x48, 0x41, 0x52, 0x53, 0x5f, 0x52, 0x45, 0x2c, 0x20, 0x27, - 0x2d, 0x27, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x6c, 0x65, 0x74, - 0x20, 0x6b, 0x65, 0x79, 0x20, 0x3d, 0x20, 0x65, 0x73, 0x63, 0x4e, 0x61, - 0x6d, 0x65, 0x3b, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, - 0x28, 0x65, 0x73, 0x63, 0x4e, 0x61, 0x6d, 0x65, 0x20, 0x69, 0x6e, 0x20, - 0x74, 0x68, 0x69, 0x73, 0x2e, 0x5f, 0x72, 0x75, 0x6c, 0x65, 0x73, 0x29, - 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, - 0x28, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x5f, 0x72, 0x75, 0x6c, 0x65, 0x73, - 0x5b, 0x65, 0x73, 0x63, 0x4e, 0x61, 0x6d, 0x65, 0x5d, 0x20, 0x3d, 0x3d, - 0x3d, 0x20, 0x72, 0x75, 0x6c, 0x65, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, - 0x20, 0x6b, 0x65, 0x79, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x7d, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x6c, 0x65, 0x74, - 0x20, 0x69, 0x20, 0x3d, 0x20, 0x30, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x77, 0x68, 0x69, 0x6c, 0x65, 0x20, 0x28, 0x28, 0x60, 0x24, - 0x7b, 0x65, 0x73, 0x63, 0x4e, 0x61, 0x6d, 0x65, 0x7d, 0x24, 0x7b, 0x69, - 0x7d, 0x60, 0x20, 0x69, 0x6e, 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x5f, - 0x72, 0x75, 0x6c, 0x65, 0x73, 0x29, 0x20, 0x26, 0x26, 0x20, 0x28, 0x74, - 0x68, 0x69, 0x73, 0x2e, 0x5f, 0x72, 0x75, 0x6c, 0x65, 0x73, 0x5b, 0x60, - 0x24, 0x7b, 0x65, 0x73, 0x63, 0x4e, 0x61, 0x6d, 0x65, 0x7d, 0x24, 0x7b, - 0x69, 0x7d, 0x60, 0x5d, 0x20, 0x21, 0x3d, 0x3d, 0x20, 0x72, 0x75, 0x6c, - 0x65, 0x29, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x69, 0x20, 0x2b, 0x3d, 0x20, 0x31, 0x3b, 0x0a, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x6b, 0x65, 0x79, 0x20, 0x3d, 0x20, 0x60, 0x24, 0x7b, 0x65, 0x73, 0x63, - 0x4e, 0x61, 0x6d, 0x65, 0x7d, 0x24, 0x7b, 0x69, 0x7d, 0x60, 0x3b, 0x0a, - 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x74, - 0x68, 0x69, 0x73, 0x2e, 0x5f, 0x72, 0x75, 0x6c, 0x65, 0x73, 0x5b, 0x6b, - 0x65, 0x79, 0x5d, 0x20, 0x3d, 0x20, 0x72, 0x75, 0x6c, 0x65, 0x3b, 0x0a, - 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x6b, - 0x65, 0x79, 0x3b, 0x0a, 0x20, 0x20, 0x7d, 0x0a, 0x0a, 0x20, 0x20, 0x61, - 0x73, 0x79, 0x6e, 0x63, 0x20, 0x72, 0x65, 0x73, 0x6f, 0x6c, 0x76, 0x65, - 0x52, 0x65, 0x66, 0x73, 0x28, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2c, - 0x20, 0x75, 0x72, 0x6c, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, - 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x76, 0x69, 0x73, 0x69, 0x74, 0x20, - 0x3d, 0x20, 0x61, 0x73, 0x79, 0x6e, 0x63, 0x20, 0x28, 0x6e, 0x29, 0x20, - 0x3d, 0x3e, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, - 0x66, 0x20, 0x28, 0x41, 0x72, 0x72, 0x61, 0x79, 0x2e, 0x69, 0x73, 0x41, - 0x72, 0x72, 0x61, 0x79, 0x28, 0x6e, 0x29, 0x29, 0x20, 0x7b, 0x0a, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, - 0x6e, 0x20, 0x50, 0x72, 0x6f, 0x6d, 0x69, 0x73, 0x65, 0x2e, 0x61, 0x6c, - 0x6c, 0x28, 0x6e, 0x2e, 0x6d, 0x61, 0x70, 0x28, 0x76, 0x69, 0x73, 0x69, - 0x74, 0x29, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, - 0x20, 0x65, 0x6c, 0x73, 0x65, 0x20, 0x69, 0x66, 0x20, 0x28, 0x74, 0x79, - 0x70, 0x65, 0x6f, 0x66, 0x20, 0x6e, 0x20, 0x3d, 0x3d, 0x3d, 0x20, 0x27, - 0x6f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x27, 0x20, 0x26, 0x26, 0x20, 0x6e, - 0x20, 0x21, 0x3d, 0x3d, 0x20, 0x6e, 0x75, 0x6c, 0x6c, 0x29, 0x20, 0x7b, - 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x6c, 0x65, 0x74, - 0x20, 0x72, 0x65, 0x66, 0x20, 0x3d, 0x20, 0x6e, 0x2e, 0x24, 0x72, 0x65, - 0x66, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x6c, - 0x65, 0x74, 0x20, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x3b, 0x0a, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, 0x72, - 0x65, 0x66, 0x20, 0x21, 0x3d, 0x3d, 0x20, 0x75, 0x6e, 0x64, 0x65, 0x66, - 0x69, 0x6e, 0x65, 0x64, 0x20, 0x26, 0x26, 0x20, 0x21, 0x74, 0x68, 0x69, - 0x73, 0x2e, 0x5f, 0x72, 0x65, 0x66, 0x73, 0x5b, 0x72, 0x65, 0x66, 0x5d, - 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, 0x72, 0x65, 0x66, 0x2e, 0x73, 0x74, - 0x61, 0x72, 0x74, 0x73, 0x57, 0x69, 0x74, 0x68, 0x28, 0x27, 0x68, 0x74, - 0x74, 0x70, 0x73, 0x3a, 0x2f, 0x2f, 0x27, 0x29, 0x29, 0x20, 0x7b, 0x0a, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x69, 0x66, 0x20, 0x28, 0x21, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x5f, 0x61, - 0x6c, 0x6c, 0x6f, 0x77, 0x46, 0x65, 0x74, 0x63, 0x68, 0x29, 0x20, 0x7b, - 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x74, 0x68, 0x72, 0x6f, 0x77, 0x20, 0x6e, 0x65, 0x77, - 0x20, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x28, 0x27, 0x46, 0x65, 0x74, 0x63, - 0x68, 0x69, 0x6e, 0x67, 0x20, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x20, - 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x73, 0x20, 0x69, 0x73, 0x20, 0x6e, - 0x6f, 0x74, 0x20, 0x61, 0x6c, 0x6c, 0x6f, 0x77, 0x65, 0x64, 0x20, 0x28, - 0x75, 0x73, 0x65, 0x20, 0x2d, 0x2d, 0x61, 0x6c, 0x6c, 0x6f, 0x77, 0x2d, - 0x66, 0x65, 0x74, 0x63, 0x68, 0x20, 0x66, 0x6f, 0x72, 0x20, 0x66, 0x6f, - 0x72, 0x63, 0x65, 0x29, 0x27, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, - 0x6e, 0x73, 0x74, 0x20, 0x66, 0x65, 0x74, 0x63, 0x68, 0x20, 0x3d, 0x20, - 0x28, 0x61, 0x77, 0x61, 0x69, 0x74, 0x20, 0x69, 0x6d, 0x70, 0x6f, 0x72, - 0x74, 0x28, 0x27, 0x6e, 0x6f, 0x64, 0x65, 0x2d, 0x66, 0x65, 0x74, 0x63, - 0x68, 0x27, 0x29, 0x29, 0x2e, 0x64, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, - 0x3b, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x66, 0x72, 0x61, - 0x67, 0x53, 0x70, 0x6c, 0x69, 0x74, 0x20, 0x3d, 0x20, 0x72, 0x65, 0x66, - 0x2e, 0x73, 0x70, 0x6c, 0x69, 0x74, 0x28, 0x27, 0x23, 0x27, 0x29, 0x3b, - 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x62, 0x61, 0x73, 0x65, 0x55, - 0x72, 0x6c, 0x20, 0x3d, 0x20, 0x66, 0x72, 0x61, 0x67, 0x53, 0x70, 0x6c, - 0x69, 0x74, 0x5b, 0x30, 0x5d, 0x3b, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x74, 0x61, 0x72, 0x67, - 0x65, 0x74, 0x20, 0x3d, 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x5f, 0x72, - 0x65, 0x66, 0x73, 0x5b, 0x62, 0x61, 0x73, 0x65, 0x55, 0x72, 0x6c, 0x5d, - 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, 0x21, 0x74, 0x61, 0x72, 0x67, 0x65, - 0x74, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x74, 0x61, 0x72, 0x67, 0x65, - 0x74, 0x20, 0x3d, 0x20, 0x61, 0x77, 0x61, 0x69, 0x74, 0x20, 0x74, 0x68, - 0x69, 0x73, 0x2e, 0x72, 0x65, 0x73, 0x6f, 0x6c, 0x76, 0x65, 0x52, 0x65, - 0x66, 0x73, 0x28, 0x61, 0x77, 0x61, 0x69, 0x74, 0x20, 0x66, 0x65, 0x74, - 0x63, 0x68, 0x28, 0x72, 0x65, 0x66, 0x29, 0x2e, 0x74, 0x68, 0x65, 0x6e, - 0x28, 0x72, 0x65, 0x73, 0x20, 0x3d, 0x3e, 0x20, 0x72, 0x65, 0x73, 0x2e, - 0x6a, 0x73, 0x6f, 0x6e, 0x28, 0x29, 0x29, 0x2c, 0x20, 0x62, 0x61, 0x73, - 0x65, 0x55, 0x72, 0x6c, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x74, 0x68, 0x69, - 0x73, 0x2e, 0x5f, 0x72, 0x65, 0x66, 0x73, 0x5b, 0x62, 0x61, 0x73, 0x65, - 0x55, 0x72, 0x6c, 0x5d, 0x20, 0x3d, 0x20, 0x74, 0x61, 0x72, 0x67, 0x65, - 0x74, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, 0x66, 0x72, - 0x61, 0x67, 0x53, 0x70, 0x6c, 0x69, 0x74, 0x2e, 0x6c, 0x65, 0x6e, 0x67, - 0x74, 0x68, 0x20, 0x3d, 0x3d, 0x3d, 0x20, 0x31, 0x20, 0x7c, 0x7c, 0x20, - 0x66, 0x72, 0x61, 0x67, 0x53, 0x70, 0x6c, 0x69, 0x74, 0x5b, 0x66, 0x72, - 0x61, 0x67, 0x53, 0x70, 0x6c, 0x69, 0x74, 0x2e, 0x6c, 0x65, 0x6e, 0x67, - 0x74, 0x68, 0x20, 0x2d, 0x20, 0x31, 0x5d, 0x20, 0x3d, 0x3d, 0x3d, 0x20, - 0x27, 0x27, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, - 0x72, 0x6e, 0x20, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x3b, 0x0a, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, - 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, - 0x20, 0x65, 0x6c, 0x73, 0x65, 0x20, 0x69, 0x66, 0x20, 0x28, 0x72, 0x65, - 0x66, 0x2e, 0x73, 0x74, 0x61, 0x72, 0x74, 0x73, 0x57, 0x69, 0x74, 0x68, - 0x28, 0x27, 0x23, 0x2f, 0x27, 0x29, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x74, 0x61, - 0x72, 0x67, 0x65, 0x74, 0x20, 0x3d, 0x20, 0x73, 0x63, 0x68, 0x65, 0x6d, - 0x61, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x72, 0x65, 0x66, 0x20, 0x3d, 0x20, 0x60, 0x24, 0x7b, - 0x75, 0x72, 0x6c, 0x7d, 0x24, 0x7b, 0x72, 0x65, 0x66, 0x7d, 0x60, 0x3b, - 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x6e, 0x2e, 0x24, 0x72, 0x65, 0x66, 0x20, 0x3d, 0x20, 0x72, 0x65, - 0x66, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x7d, 0x20, 0x65, 0x6c, 0x73, 0x65, 0x20, 0x7b, 0x0a, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x74, 0x68, - 0x72, 0x6f, 0x77, 0x20, 0x6e, 0x65, 0x77, 0x20, 0x45, 0x72, 0x72, 0x6f, - 0x72, 0x28, 0x60, 0x55, 0x6e, 0x73, 0x75, 0x70, 0x70, 0x6f, 0x72, 0x74, - 0x65, 0x64, 0x20, 0x72, 0x65, 0x66, 0x20, 0x24, 0x7b, 0x72, 0x65, 0x66, - 0x7d, 0x60, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x73, 0x65, - 0x6c, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x73, 0x20, 0x3d, 0x20, 0x72, 0x65, - 0x66, 0x2e, 0x73, 0x70, 0x6c, 0x69, 0x74, 0x28, 0x27, 0x23, 0x27, 0x29, - 0x5b, 0x31, 0x5d, 0x2e, 0x73, 0x70, 0x6c, 0x69, 0x74, 0x28, 0x27, 0x2f, - 0x27, 0x29, 0x2e, 0x73, 0x6c, 0x69, 0x63, 0x65, 0x28, 0x31, 0x29, 0x3b, - 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x66, - 0x6f, 0x72, 0x20, 0x28, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x73, 0x65, - 0x6c, 0x20, 0x6f, 0x66, 0x20, 0x73, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x6f, - 0x72, 0x73, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, 0x21, 0x74, - 0x61, 0x72, 0x67, 0x65, 0x74, 0x20, 0x7c, 0x7c, 0x20, 0x21, 0x28, 0x73, - 0x65, 0x6c, 0x20, 0x69, 0x6e, 0x20, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, - 0x29, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x74, 0x68, 0x72, 0x6f, 0x77, - 0x20, 0x6e, 0x65, 0x77, 0x20, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x28, 0x60, - 0x45, 0x72, 0x72, 0x6f, 0x72, 0x20, 0x72, 0x65, 0x73, 0x6f, 0x6c, 0x76, - 0x69, 0x6e, 0x67, 0x20, 0x72, 0x65, 0x66, 0x20, 0x24, 0x7b, 0x72, 0x65, - 0x66, 0x7d, 0x3a, 0x20, 0x24, 0x7b, 0x73, 0x65, 0x6c, 0x7d, 0x20, 0x6e, - 0x6f, 0x74, 0x20, 0x69, 0x6e, 0x20, 0x24, 0x7b, 0x4a, 0x53, 0x4f, 0x4e, - 0x2e, 0x73, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x69, 0x66, 0x79, 0x28, 0x74, - 0x61, 0x72, 0x67, 0x65, 0x74, 0x29, 0x7d, 0x60, 0x29, 0x3b, 0x0a, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, - 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x20, 0x3d, 0x20, 0x74, 0x61, - 0x72, 0x67, 0x65, 0x74, 0x5b, 0x73, 0x65, 0x6c, 0x5d, 0x3b, 0x0a, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x0a, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x74, 0x68, - 0x69, 0x73, 0x2e, 0x5f, 0x72, 0x65, 0x66, 0x73, 0x5b, 0x72, 0x65, 0x66, - 0x5d, 0x20, 0x3d, 0x20, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x3b, 0x0a, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x20, 0x65, 0x6c, - 0x73, 0x65, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x61, 0x77, 0x61, 0x69, 0x74, 0x20, 0x50, 0x72, 0x6f, - 0x6d, 0x69, 0x73, 0x65, 0x2e, 0x61, 0x6c, 0x6c, 0x28, 0x4f, 0x62, 0x6a, - 0x65, 0x63, 0x74, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x73, 0x28, 0x6e, - 0x29, 0x2e, 0x6d, 0x61, 0x70, 0x28, 0x76, 0x69, 0x73, 0x69, 0x74, 0x29, - 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, - 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x0a, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x6e, - 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x3b, 0x0a, 0x0a, 0x20, 0x20, - 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x76, 0x69, 0x73, - 0x69, 0x74, 0x28, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x29, 0x3b, 0x0a, - 0x20, 0x20, 0x7d, 0x0a, 0x0a, 0x20, 0x20, 0x5f, 0x67, 0x65, 0x6e, 0x65, - 0x72, 0x61, 0x74, 0x65, 0x55, 0x6e, 0x69, 0x6f, 0x6e, 0x52, 0x75, 0x6c, - 0x65, 0x28, 0x6e, 0x61, 0x6d, 0x65, 0x2c, 0x20, 0x61, 0x6c, 0x74, 0x53, - 0x63, 0x68, 0x65, 0x6d, 0x61, 0x73, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, - 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x61, 0x6c, 0x74, - 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x73, 0x0a, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x2e, 0x6d, 0x61, 0x70, 0x28, 0x28, 0x61, 0x6c, 0x74, 0x53, - 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2c, 0x20, 0x69, 0x29, 0x20, 0x3d, 0x3e, - 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x76, 0x69, 0x73, 0x69, 0x74, 0x28, - 0x61, 0x6c, 0x74, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2c, 0x20, 0x60, - 0x24, 0x7b, 0x6e, 0x61, 0x6d, 0x65, 0x20, 0x3f, 0x3f, 0x20, 0x27, 0x27, - 0x7d, 0x24, 0x7b, 0x6e, 0x61, 0x6d, 0x65, 0x20, 0x3f, 0x20, 0x27, 0x2d, - 0x27, 0x20, 0x3a, 0x20, 0x27, 0x61, 0x6c, 0x74, 0x65, 0x72, 0x6e, 0x61, - 0x74, 0x69, 0x76, 0x65, 0x2d, 0x27, 0x7d, 0x24, 0x7b, 0x69, 0x7d, 0x60, - 0x29, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x2e, 0x6a, 0x6f, - 0x69, 0x6e, 0x28, 0x27, 0x20, 0x7c, 0x20, 0x27, 0x29, 0x3b, 0x0a, 0x20, - 0x20, 0x7d, 0x0a, 0x0a, 0x20, 0x20, 0x5f, 0x76, 0x69, 0x73, 0x69, 0x74, - 0x50, 0x61, 0x74, 0x74, 0x65, 0x72, 0x6e, 0x28, 0x70, 0x61, 0x74, 0x74, - 0x65, 0x72, 0x6e, 0x2c, 0x20, 0x6e, 0x61, 0x6d, 0x65, 0x29, 0x20, 0x7b, - 0x0a, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, 0x21, 0x70, 0x61, - 0x74, 0x74, 0x65, 0x72, 0x6e, 0x2e, 0x73, 0x74, 0x61, 0x72, 0x74, 0x73, - 0x57, 0x69, 0x74, 0x68, 0x28, 0x27, 0x5e, 0x27, 0x29, 0x20, 0x7c, 0x7c, - 0x20, 0x21, 0x70, 0x61, 0x74, 0x74, 0x65, 0x72, 0x6e, 0x2e, 0x65, 0x6e, - 0x64, 0x73, 0x57, 0x69, 0x74, 0x68, 0x28, 0x27, 0x24, 0x27, 0x29, 0x29, - 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x74, 0x68, 0x72, - 0x6f, 0x77, 0x20, 0x6e, 0x65, 0x77, 0x20, 0x45, 0x72, 0x72, 0x6f, 0x72, - 0x28, 0x27, 0x50, 0x61, 0x74, 0x74, 0x65, 0x72, 0x6e, 0x20, 0x6d, 0x75, - 0x73, 0x74, 0x20, 0x73, 0x74, 0x61, 0x72, 0x74, 0x20, 0x77, 0x69, 0x74, - 0x68, 0x20, 0x22, 0x5e, 0x22, 0x20, 0x61, 0x6e, 0x64, 0x20, 0x65, 0x6e, - 0x64, 0x20, 0x77, 0x69, 0x74, 0x68, 0x20, 0x22, 0x24, 0x22, 0x27, 0x29, - 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, - 0x70, 0x61, 0x74, 0x74, 0x65, 0x72, 0x6e, 0x20, 0x3d, 0x20, 0x70, 0x61, - 0x74, 0x74, 0x65, 0x72, 0x6e, 0x2e, 0x73, 0x6c, 0x69, 0x63, 0x65, 0x28, - 0x31, 0x2c, 0x20, 0x2d, 0x31, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, - 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x73, 0x75, 0x62, 0x52, 0x75, 0x6c, - 0x65, 0x49, 0x64, 0x73, 0x20, 0x3d, 0x20, 0x7b, 0x7d, 0x3b, 0x0a, 0x0a, - 0x20, 0x20, 0x20, 0x20, 0x6c, 0x65, 0x74, 0x20, 0x69, 0x20, 0x3d, 0x20, - 0x30, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, - 0x20, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x20, 0x3d, 0x20, 0x70, 0x61, - 0x74, 0x74, 0x65, 0x72, 0x6e, 0x2e, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, - 0x3b, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, - 0x20, 0x67, 0x65, 0x74, 0x44, 0x6f, 0x74, 0x20, 0x3d, 0x20, 0x28, 0x29, - 0x20, 0x3d, 0x3e, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x6c, 0x65, 0x74, 0x20, 0x72, 0x75, 0x6c, 0x65, 0x3b, 0x0a, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, 0x74, 0x68, 0x69, 0x73, - 0x2e, 0x5f, 0x64, 0x6f, 0x74, 0x61, 0x6c, 0x6c, 0x29, 0x20, 0x7b, 0x0a, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, 0x75, 0x6c, 0x65, - 0x20, 0x3d, 0x20, 0x27, 0x5b, 0x5c, 0x5c, 0x55, 0x30, 0x30, 0x30, 0x30, - 0x30, 0x30, 0x30, 0x30, 0x2d, 0x5c, 0x5c, 0x55, 0x30, 0x30, 0x31, 0x30, - 0x46, 0x46, 0x46, 0x46, 0x5d, 0x27, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x7d, 0x20, 0x65, 0x6c, 0x73, 0x65, 0x20, 0x7b, 0x0a, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x2f, 0x2f, 0x20, 0x41, 0x63, - 0x63, 0x65, 0x70, 0x74, 0x20, 0x61, 0x6e, 0x79, 0x20, 0x63, 0x68, 0x61, - 0x72, 0x61, 0x63, 0x74, 0x65, 0x72, 0x2e, 0x2e, 0x2e, 0x20, 0x65, 0x78, - 0x63, 0x65, 0x70, 0x74, 0x20, 0x5c, 0x6e, 0x20, 0x61, 0x6e, 0x64, 0x20, - 0x5c, 0x72, 0x20, 0x6c, 0x69, 0x6e, 0x65, 0x20, 0x62, 0x72, 0x65, 0x61, - 0x6b, 0x20, 0x63, 0x68, 0x61, 0x72, 0x73, 0x20, 0x28, 0x5c, 0x78, 0x30, - 0x41, 0x20, 0x61, 0x6e, 0x64, 0x20, 0x5c, 0x78, 0x4f, 0x44, 0x29, 0x0a, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, 0x75, 0x6c, 0x65, - 0x20, 0x3d, 0x20, 0x27, 0x5b, 0x5c, 0x5c, 0x55, 0x30, 0x30, 0x30, 0x30, - 0x30, 0x30, 0x30, 0x30, 0x2d, 0x5c, 0x5c, 0x78, 0x30, 0x39, 0x5c, 0x5c, - 0x78, 0x30, 0x42, 0x5c, 0x5c, 0x78, 0x30, 0x43, 0x5c, 0x5c, 0x78, 0x30, - 0x45, 0x2d, 0x5c, 0x5c, 0x55, 0x30, 0x30, 0x31, 0x30, 0x46, 0x46, 0x46, - 0x46, 0x5d, 0x27, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, - 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, - 0x6e, 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x5f, 0x61, 0x64, 0x64, 0x52, - 0x75, 0x6c, 0x65, 0x28, 0x27, 0x64, 0x6f, 0x74, 0x27, 0x2c, 0x20, 0x72, - 0x75, 0x6c, 0x65, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x3b, - 0x0a, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, - 0x20, 0x74, 0x6f, 0x52, 0x75, 0x6c, 0x65, 0x20, 0x3d, 0x20, 0x28, 0x5b, - 0x73, 0x2c, 0x20, 0x69, 0x73, 0x4c, 0x69, 0x74, 0x65, 0x72, 0x61, 0x6c, - 0x5d, 0x29, 0x20, 0x3d, 0x3e, 0x20, 0x69, 0x73, 0x4c, 0x69, 0x74, 0x65, - 0x72, 0x61, 0x6c, 0x20, 0x3f, 0x20, 0x22, 0x5c, 0x22, 0x22, 0x20, 0x2b, - 0x20, 0x73, 0x20, 0x2b, 0x20, 0x22, 0x5c, 0x22, 0x22, 0x20, 0x3a, 0x20, - 0x73, 0x3b, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, - 0x74, 0x20, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x20, - 0x3d, 0x20, 0x28, 0x29, 0x20, 0x3d, 0x3e, 0x20, 0x7b, 0x0a, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x73, 0x74, - 0x61, 0x72, 0x74, 0x20, 0x3d, 0x20, 0x69, 0x3b, 0x0a, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x2f, 0x2f, 0x20, 0x46, 0x6f, 0x72, 0x20, 0x65, 0x61, - 0x63, 0x68, 0x20, 0x63, 0x6f, 0x6d, 0x70, 0x6f, 0x6e, 0x65, 0x6e, 0x74, - 0x20, 0x6f, 0x66, 0x20, 0x74, 0x68, 0x69, 0x73, 0x20, 0x73, 0x65, 0x71, - 0x75, 0x65, 0x6e, 0x63, 0x65, 0x2c, 0x20, 0x73, 0x74, 0x6f, 0x72, 0x65, - 0x20, 0x69, 0x74, 0x73, 0x20, 0x73, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x20, - 0x72, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x74, 0x61, 0x74, 0x69, - 0x6f, 0x6e, 0x20, 0x61, 0x6e, 0x64, 0x20, 0x77, 0x68, 0x65, 0x74, 0x68, - 0x65, 0x72, 0x20, 0x69, 0x74, 0x27, 0x73, 0x20, 0x61, 0x20, 0x6c, 0x69, - 0x74, 0x65, 0x72, 0x61, 0x6c, 0x2e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x2f, 0x2f, 0x20, 0x57, 0x65, 0x20, 0x6f, 0x6e, 0x6c, 0x79, 0x20, - 0x6e, 0x65, 0x65, 0x64, 0x20, 0x61, 0x20, 0x66, 0x6c, 0x61, 0x74, 0x20, - 0x73, 0x74, 0x72, 0x75, 0x63, 0x74, 0x75, 0x72, 0x65, 0x20, 0x68, 0x65, - 0x72, 0x65, 0x20, 0x74, 0x6f, 0x20, 0x61, 0x70, 0x70, 0x6c, 0x79, 0x20, - 0x72, 0x65, 0x70, 0x65, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x6f, - 0x70, 0x65, 0x72, 0x61, 0x74, 0x6f, 0x72, 0x73, 0x20, 0x74, 0x6f, 0x20, - 0x74, 0x68, 0x65, 0x20, 0x6c, 0x61, 0x73, 0x74, 0x20, 0x69, 0x74, 0x65, - 0x6d, 0x2c, 0x20, 0x61, 0x6e, 0x64, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x2f, 0x2f, 0x20, 0x74, 0x6f, 0x20, 0x6d, 0x65, 0x72, 0x67, 0x65, - 0x20, 0x6c, 0x69, 0x74, 0x65, 0x72, 0x61, 0x6c, 0x73, 0x20, 0x61, 0x74, - 0x20, 0x74, 0x68, 0x65, 0x20, 0x61, 0x6e, 0x64, 0x20, 0x28, 0x77, 0x65, - 0x27, 0x72, 0x65, 0x20, 0x70, 0x61, 0x72, 0x73, 0x69, 0x6e, 0x67, 0x20, - 0x67, 0x72, 0x6f, 0x75, 0x70, 0x65, 0x64, 0x20, 0x28, 0x20, 0x73, 0x65, - 0x71, 0x75, 0x65, 0x6e, 0x63, 0x65, 0x73, 0x20, 0x29, 0x20, 0x72, 0x65, - 0x63, 0x75, 0x72, 0x73, 0x69, 0x76, 0x65, 0x6c, 0x79, 0x20, 0x61, 0x6e, - 0x64, 0x20, 0x64, 0x6f, 0x6e, 0x27, 0x74, 0x20, 0x74, 0x72, 0x65, 0x61, - 0x74, 0x20, 0x27, 0x7c, 0x27, 0x20, 0x73, 0x70, 0x65, 0x63, 0x69, 0x61, - 0x6c, 0x6c, 0x79, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x2f, 0x2f, - 0x20, 0x28, 0x47, 0x42, 0x4e, 0x46, 0x27, 0x73, 0x20, 0x73, 0x79, 0x6e, - 0x74, 0x61, 0x78, 0x20, 0x69, 0x73, 0x20, 0x6c, 0x75, 0x63, 0x6b, 0x69, - 0x6c, 0x79, 0x20, 0x76, 0x65, 0x72, 0x79, 0x20, 0x63, 0x6c, 0x6f, 0x73, - 0x65, 0x20, 0x74, 0x6f, 0x20, 0x72, 0x65, 0x67, 0x75, 0x6c, 0x61, 0x72, - 0x20, 0x65, 0x78, 0x70, 0x72, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x73, - 0x21, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, - 0x73, 0x74, 0x20, 0x73, 0x65, 0x71, 0x20, 0x3d, 0x20, 0x5b, 0x5d, 0x3b, - 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, - 0x74, 0x20, 0x6a, 0x6f, 0x69, 0x6e, 0x53, 0x65, 0x71, 0x20, 0x3d, 0x20, - 0x28, 0x29, 0x20, 0x3d, 0x3e, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x72, 0x65, - 0x74, 0x20, 0x3d, 0x20, 0x5b, 0x5d, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x66, 0x6f, 0x72, 0x20, 0x28, 0x63, 0x6f, 0x6e, - 0x73, 0x74, 0x20, 0x5b, 0x69, 0x73, 0x4c, 0x69, 0x74, 0x65, 0x72, 0x61, - 0x6c, 0x2c, 0x20, 0x67, 0x5d, 0x20, 0x6f, 0x66, 0x20, 0x67, 0x72, 0x6f, - 0x75, 0x70, 0x42, 0x79, 0x28, 0x73, 0x65, 0x71, 0x2c, 0x20, 0x78, 0x20, - 0x3d, 0x3e, 0x20, 0x78, 0x5b, 0x31, 0x5d, 0x29, 0x29, 0x20, 0x7b, 0x0a, + 0x6c, 0x69, 0x74, 0x65, 0x72, 0x61, 0x6c, 0x29, 0x2e, 0x73, 0x6c, 0x69, + 0x63, 0x65, 0x28, 0x31, 0x2c, 0x20, 0x2d, 0x31, 0x29, 0x2e, 0x72, 0x65, + 0x70, 0x6c, 0x61, 0x63, 0x65, 0x28, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x47, 0x52, 0x41, 0x4d, 0x4d, 0x41, 0x52, 0x5f, 0x52, 0x41, 0x4e, + 0x47, 0x45, 0x5f, 0x4c, 0x49, 0x54, 0x45, 0x52, 0x41, 0x4c, 0x5f, 0x45, + 0x53, 0x43, 0x41, 0x50, 0x45, 0x5f, 0x52, 0x45, 0x2c, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x6d, 0x20, 0x3d, 0x3e, 0x20, 0x47, 0x52, 0x41, + 0x4d, 0x4d, 0x41, 0x52, 0x5f, 0x4c, 0x49, 0x54, 0x45, 0x52, 0x41, 0x4c, + 0x5f, 0x45, 0x53, 0x43, 0x41, 0x50, 0x45, 0x53, 0x5b, 0x6d, 0x5d, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x7d, 0x0a, 0x0a, + 0x20, 0x20, 0x5f, 0x61, 0x64, 0x64, 0x52, 0x75, 0x6c, 0x65, 0x28, 0x6e, + 0x61, 0x6d, 0x65, 0x2c, 0x20, 0x72, 0x75, 0x6c, 0x65, 0x29, 0x20, 0x7b, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x6c, 0x65, 0x74, 0x20, 0x65, 0x73, 0x63, + 0x4e, 0x61, 0x6d, 0x65, 0x20, 0x3d, 0x20, 0x6e, 0x61, 0x6d, 0x65, 0x2e, + 0x72, 0x65, 0x70, 0x6c, 0x61, 0x63, 0x65, 0x28, 0x49, 0x4e, 0x56, 0x41, + 0x4c, 0x49, 0x44, 0x5f, 0x52, 0x55, 0x4c, 0x45, 0x5f, 0x43, 0x48, 0x41, + 0x52, 0x53, 0x5f, 0x52, 0x45, 0x2c, 0x20, 0x27, 0x2d, 0x27, 0x29, 0x3b, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x6c, 0x65, 0x74, 0x20, 0x6b, 0x65, 0x79, + 0x20, 0x3d, 0x20, 0x65, 0x73, 0x63, 0x4e, 0x61, 0x6d, 0x65, 0x3b, 0x0a, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, 0x65, 0x73, 0x63, + 0x4e, 0x61, 0x6d, 0x65, 0x20, 0x69, 0x6e, 0x20, 0x74, 0x68, 0x69, 0x73, + 0x2e, 0x5f, 0x72, 0x75, 0x6c, 0x65, 0x73, 0x29, 0x20, 0x7b, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, 0x74, 0x68, 0x69, + 0x73, 0x2e, 0x5f, 0x72, 0x75, 0x6c, 0x65, 0x73, 0x5b, 0x65, 0x73, 0x63, + 0x4e, 0x61, 0x6d, 0x65, 0x5d, 0x20, 0x3d, 0x3d, 0x3d, 0x20, 0x72, 0x75, + 0x6c, 0x65, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x6b, 0x65, 0x79, + 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x6c, 0x65, 0x74, 0x20, 0x69, 0x20, 0x3d, + 0x20, 0x30, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x77, 0x68, + 0x69, 0x6c, 0x65, 0x20, 0x28, 0x28, 0x60, 0x24, 0x7b, 0x65, 0x73, 0x63, + 0x4e, 0x61, 0x6d, 0x65, 0x7d, 0x24, 0x7b, 0x69, 0x7d, 0x60, 0x20, 0x69, + 0x6e, 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x5f, 0x72, 0x75, 0x6c, 0x65, + 0x73, 0x29, 0x20, 0x26, 0x26, 0x20, 0x28, 0x74, 0x68, 0x69, 0x73, 0x2e, + 0x5f, 0x72, 0x75, 0x6c, 0x65, 0x73, 0x5b, 0x60, 0x24, 0x7b, 0x65, 0x73, + 0x63, 0x4e, 0x61, 0x6d, 0x65, 0x7d, 0x24, 0x7b, 0x69, 0x7d, 0x60, 0x5d, + 0x20, 0x21, 0x3d, 0x3d, 0x20, 0x72, 0x75, 0x6c, 0x65, 0x29, 0x29, 0x20, + 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x20, + 0x2b, 0x3d, 0x20, 0x31, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x6b, 0x65, 0x79, 0x20, + 0x3d, 0x20, 0x60, 0x24, 0x7b, 0x65, 0x73, 0x63, 0x4e, 0x61, 0x6d, 0x65, + 0x7d, 0x24, 0x7b, 0x69, 0x7d, 0x60, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x7d, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, + 0x5f, 0x72, 0x75, 0x6c, 0x65, 0x73, 0x5b, 0x6b, 0x65, 0x79, 0x5d, 0x20, + 0x3d, 0x20, 0x72, 0x75, 0x6c, 0x65, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x6b, 0x65, 0x79, 0x3b, 0x0a, + 0x20, 0x20, 0x7d, 0x0a, 0x0a, 0x20, 0x20, 0x61, 0x73, 0x79, 0x6e, 0x63, + 0x20, 0x72, 0x65, 0x73, 0x6f, 0x6c, 0x76, 0x65, 0x52, 0x65, 0x66, 0x73, + 0x28, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2c, 0x20, 0x75, 0x72, 0x6c, + 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, + 0x74, 0x20, 0x76, 0x69, 0x73, 0x69, 0x74, 0x20, 0x3d, 0x20, 0x61, 0x73, + 0x79, 0x6e, 0x63, 0x20, 0x28, 0x6e, 0x29, 0x20, 0x3d, 0x3e, 0x20, 0x7b, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, 0x41, + 0x72, 0x72, 0x61, 0x79, 0x2e, 0x69, 0x73, 0x41, 0x72, 0x72, 0x61, 0x79, + 0x28, 0x6e, 0x29, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x50, 0x72, + 0x6f, 0x6d, 0x69, 0x73, 0x65, 0x2e, 0x61, 0x6c, 0x6c, 0x28, 0x6e, 0x2e, + 0x6d, 0x61, 0x70, 0x28, 0x76, 0x69, 0x73, 0x69, 0x74, 0x29, 0x29, 0x3b, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x20, 0x65, 0x6c, 0x73, + 0x65, 0x20, 0x69, 0x66, 0x20, 0x28, 0x74, 0x79, 0x70, 0x65, 0x6f, 0x66, + 0x20, 0x6e, 0x20, 0x3d, 0x3d, 0x3d, 0x20, 0x27, 0x6f, 0x62, 0x6a, 0x65, + 0x63, 0x74, 0x27, 0x20, 0x26, 0x26, 0x20, 0x6e, 0x20, 0x21, 0x3d, 0x3d, + 0x20, 0x6e, 0x75, 0x6c, 0x6c, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x6c, 0x65, 0x74, 0x20, 0x72, 0x65, 0x66, + 0x20, 0x3d, 0x20, 0x6e, 0x2e, 0x24, 0x72, 0x65, 0x66, 0x3b, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x6c, 0x65, 0x74, 0x20, 0x74, + 0x61, 0x72, 0x67, 0x65, 0x74, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, 0x72, 0x65, 0x66, 0x20, 0x21, + 0x3d, 0x3d, 0x20, 0x75, 0x6e, 0x64, 0x65, 0x66, 0x69, 0x6e, 0x65, 0x64, + 0x20, 0x26, 0x26, 0x20, 0x21, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x5f, 0x72, + 0x65, 0x66, 0x73, 0x5b, 0x72, 0x65, 0x66, 0x5d, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, - 0x20, 0x28, 0x69, 0x73, 0x4c, 0x69, 0x74, 0x65, 0x72, 0x61, 0x6c, 0x29, - 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x72, 0x65, 0x74, 0x2e, 0x70, 0x75, 0x73, 0x68, 0x28, - 0x5b, 0x5b, 0x2e, 0x2e, 0x2e, 0x67, 0x5d, 0x2e, 0x6d, 0x61, 0x70, 0x28, - 0x78, 0x20, 0x3d, 0x3e, 0x20, 0x78, 0x5b, 0x30, 0x5d, 0x29, 0x2e, 0x6a, - 0x6f, 0x69, 0x6e, 0x28, 0x27, 0x27, 0x29, 0x2c, 0x20, 0x74, 0x72, 0x75, - 0x65, 0x5d, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x7d, 0x20, 0x65, 0x6c, 0x73, 0x65, 0x20, 0x7b, 0x0a, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x72, 0x65, 0x74, 0x2e, 0x70, 0x75, 0x73, 0x68, 0x28, 0x2e, 0x2e, 0x2e, - 0x67, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, - 0x20, 0x28, 0x72, 0x65, 0x74, 0x2e, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, - 0x20, 0x3d, 0x3d, 0x3d, 0x20, 0x31, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, - 0x72, 0x6e, 0x20, 0x72, 0x65, 0x74, 0x5b, 0x30, 0x5d, 0x3b, 0x0a, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, - 0x5b, 0x72, 0x65, 0x74, 0x2e, 0x6d, 0x61, 0x70, 0x28, 0x78, 0x20, 0x3d, - 0x3e, 0x20, 0x74, 0x6f, 0x52, 0x75, 0x6c, 0x65, 0x28, 0x78, 0x29, 0x29, - 0x2e, 0x6a, 0x6f, 0x69, 0x6e, 0x28, 0x27, 0x20, 0x27, 0x29, 0x2c, 0x20, - 0x66, 0x61, 0x6c, 0x73, 0x65, 0x5d, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x7d, 0x3b, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x77, 0x68, 0x69, 0x6c, 0x65, 0x20, 0x28, 0x69, 0x20, 0x3c, 0x20, 0x6c, - 0x65, 0x6e, 0x67, 0x74, 0x68, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x63, - 0x20, 0x3d, 0x20, 0x70, 0x61, 0x74, 0x74, 0x65, 0x72, 0x6e, 0x5b, 0x69, - 0x5d, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, - 0x66, 0x20, 0x28, 0x63, 0x20, 0x3d, 0x3d, 0x3d, 0x20, 0x27, 0x2e, 0x27, - 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x73, 0x65, 0x71, 0x2e, 0x70, 0x75, 0x73, 0x68, 0x28, 0x5b, - 0x67, 0x65, 0x74, 0x44, 0x6f, 0x74, 0x28, 0x29, 0x2c, 0x20, 0x66, 0x61, - 0x6c, 0x73, 0x65, 0x5d, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x20, 0x2b, 0x3d, 0x20, 0x31, 0x3b, - 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x20, 0x65, - 0x6c, 0x73, 0x65, 0x20, 0x69, 0x66, 0x20, 0x28, 0x63, 0x20, 0x3d, 0x3d, - 0x3d, 0x20, 0x27, 0x28, 0x27, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x20, 0x2b, 0x3d, 0x20, - 0x31, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x69, 0x66, 0x20, 0x28, 0x69, 0x20, 0x3c, 0x20, 0x6c, 0x65, 0x6e, - 0x67, 0x74, 0x68, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, 0x70, - 0x61, 0x74, 0x74, 0x65, 0x72, 0x6e, 0x5b, 0x69, 0x5d, 0x20, 0x3d, 0x3d, - 0x3d, 0x20, 0x27, 0x3f, 0x27, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x28, 0x72, 0x65, 0x66, 0x2e, 0x73, 0x74, 0x61, 0x72, 0x74, 0x73, + 0x57, 0x69, 0x74, 0x68, 0x28, 0x27, 0x68, 0x74, 0x74, 0x70, 0x73, 0x3a, + 0x2f, 0x2f, 0x27, 0x29, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, + 0x21, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x5f, 0x61, 0x6c, 0x6c, 0x6f, 0x77, + 0x46, 0x65, 0x74, 0x63, 0x68, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x74, 0x68, 0x72, 0x6f, 0x77, 0x20, 0x6e, 0x65, 0x77, 0x20, 0x45, 0x72, 0x72, - 0x6f, 0x72, 0x28, 0x60, 0x55, 0x6e, 0x73, 0x75, 0x70, 0x70, 0x6f, 0x72, - 0x74, 0x65, 0x64, 0x20, 0x70, 0x61, 0x74, 0x74, 0x65, 0x72, 0x6e, 0x20, - 0x73, 0x79, 0x6e, 0x74, 0x61, 0x78, 0x20, 0x22, 0x24, 0x7b, 0x70, 0x61, - 0x74, 0x74, 0x65, 0x72, 0x6e, 0x5b, 0x69, 0x5d, 0x7d, 0x22, 0x20, 0x61, - 0x74, 0x20, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x20, 0x24, 0x7b, 0x69, 0x7d, - 0x20, 0x6f, 0x66, 0x20, 0x2f, 0x24, 0x7b, 0x70, 0x61, 0x74, 0x74, 0x65, - 0x72, 0x6e, 0x7d, 0x2f, 0x60, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x6f, 0x72, 0x28, 0x27, 0x46, 0x65, 0x74, 0x63, 0x68, 0x69, 0x6e, 0x67, + 0x20, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x20, 0x73, 0x63, 0x68, 0x65, + 0x6d, 0x61, 0x73, 0x20, 0x69, 0x73, 0x20, 0x6e, 0x6f, 0x74, 0x20, 0x61, + 0x6c, 0x6c, 0x6f, 0x77, 0x65, 0x64, 0x20, 0x28, 0x75, 0x73, 0x65, 0x20, + 0x2d, 0x2d, 0x61, 0x6c, 0x6c, 0x6f, 0x77, 0x2d, 0x66, 0x65, 0x74, 0x63, + 0x68, 0x20, 0x66, 0x6f, 0x72, 0x20, 0x66, 0x6f, 0x72, 0x63, 0x65, 0x29, + 0x27, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, + 0x66, 0x65, 0x74, 0x63, 0x68, 0x20, 0x3d, 0x20, 0x28, 0x61, 0x77, 0x61, + 0x69, 0x74, 0x20, 0x69, 0x6d, 0x70, 0x6f, 0x72, 0x74, 0x28, 0x27, 0x6e, + 0x6f, 0x64, 0x65, 0x2d, 0x66, 0x65, 0x74, 0x63, 0x68, 0x27, 0x29, 0x29, + 0x2e, 0x64, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x3b, 0x0a, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, + 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x66, 0x72, 0x61, 0x67, 0x53, 0x70, 0x6c, + 0x69, 0x74, 0x20, 0x3d, 0x20, 0x72, 0x65, 0x66, 0x2e, 0x73, 0x70, 0x6c, + 0x69, 0x74, 0x28, 0x27, 0x23, 0x27, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, + 0x73, 0x74, 0x20, 0x62, 0x61, 0x73, 0x65, 0x55, 0x72, 0x6c, 0x20, 0x3d, + 0x20, 0x66, 0x72, 0x61, 0x67, 0x53, 0x70, 0x6c, 0x69, 0x74, 0x5b, 0x30, + 0x5d, 0x3b, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x20, 0x3d, + 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x5f, 0x72, 0x65, 0x66, 0x73, 0x5b, + 0x62, 0x61, 0x73, 0x65, 0x55, 0x72, 0x6c, 0x5d, 0x3b, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, + 0x20, 0x28, 0x21, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x29, 0x20, 0x7b, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x20, 0x3d, 0x20, + 0x61, 0x77, 0x61, 0x69, 0x74, 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x72, + 0x65, 0x73, 0x6f, 0x6c, 0x76, 0x65, 0x52, 0x65, 0x66, 0x73, 0x28, 0x61, + 0x77, 0x61, 0x69, 0x74, 0x20, 0x66, 0x65, 0x74, 0x63, 0x68, 0x28, 0x72, + 0x65, 0x66, 0x29, 0x2e, 0x74, 0x68, 0x65, 0x6e, 0x28, 0x72, 0x65, 0x73, + 0x20, 0x3d, 0x3e, 0x20, 0x72, 0x65, 0x73, 0x2e, 0x6a, 0x73, 0x6f, 0x6e, + 0x28, 0x29, 0x29, 0x2c, 0x20, 0x62, 0x61, 0x73, 0x65, 0x55, 0x72, 0x6c, + 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x5f, 0x72, + 0x65, 0x66, 0x73, 0x5b, 0x62, 0x61, 0x73, 0x65, 0x55, 0x72, 0x6c, 0x5d, + 0x20, 0x3d, 0x20, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x3b, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, + 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, 0x66, 0x72, 0x61, 0x67, 0x53, 0x70, + 0x6c, 0x69, 0x74, 0x2e, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x20, 0x3d, + 0x3d, 0x3d, 0x20, 0x31, 0x20, 0x7c, 0x7c, 0x20, 0x66, 0x72, 0x61, 0x67, + 0x53, 0x70, 0x6c, 0x69, 0x74, 0x5b, 0x66, 0x72, 0x61, 0x67, 0x53, 0x70, + 0x6c, 0x69, 0x74, 0x2e, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x20, 0x2d, + 0x20, 0x31, 0x5d, 0x20, 0x3d, 0x3d, 0x3d, 0x20, 0x27, 0x27, 0x29, 0x20, + 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x74, + 0x61, 0x72, 0x67, 0x65, 0x74, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x20, 0x65, 0x6c, 0x73, + 0x65, 0x20, 0x69, 0x66, 0x20, 0x28, 0x72, 0x65, 0x66, 0x2e, 0x73, 0x74, + 0x61, 0x72, 0x74, 0x73, 0x57, 0x69, 0x74, 0x68, 0x28, 0x27, 0x23, 0x2f, + 0x27, 0x29, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, + 0x20, 0x3d, 0x20, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x3b, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, + 0x65, 0x66, 0x20, 0x3d, 0x20, 0x60, 0x24, 0x7b, 0x75, 0x72, 0x6c, 0x7d, + 0x24, 0x7b, 0x72, 0x65, 0x66, 0x7d, 0x60, 0x3b, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x6e, 0x2e, 0x24, + 0x72, 0x65, 0x66, 0x20, 0x3d, 0x20, 0x72, 0x65, 0x66, 0x3b, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x20, 0x65, + 0x6c, 0x73, 0x65, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x74, 0x68, 0x72, 0x6f, 0x77, 0x20, + 0x6e, 0x65, 0x77, 0x20, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x28, 0x60, 0x55, + 0x6e, 0x73, 0x75, 0x70, 0x70, 0x6f, 0x72, 0x74, 0x65, 0x64, 0x20, 0x72, + 0x65, 0x66, 0x20, 0x24, 0x7b, 0x72, 0x65, 0x66, 0x7d, 0x60, 0x29, 0x3b, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, + 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x73, 0x65, 0x6c, 0x65, 0x63, 0x74, + 0x6f, 0x72, 0x73, 0x20, 0x3d, 0x20, 0x72, 0x65, 0x66, 0x2e, 0x73, 0x70, + 0x6c, 0x69, 0x74, 0x28, 0x27, 0x23, 0x27, 0x29, 0x5b, 0x31, 0x5d, 0x2e, + 0x73, 0x70, 0x6c, 0x69, 0x74, 0x28, 0x27, 0x2f, 0x27, 0x29, 0x2e, 0x73, + 0x6c, 0x69, 0x63, 0x65, 0x28, 0x31, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x66, 0x6f, 0x72, 0x20, 0x28, + 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x73, 0x65, 0x6c, 0x20, 0x6f, 0x66, + 0x20, 0x73, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x73, 0x29, 0x20, + 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, 0x21, 0x74, 0x61, 0x72, 0x67, 0x65, + 0x74, 0x20, 0x7c, 0x7c, 0x20, 0x21, 0x28, 0x73, 0x65, 0x6c, 0x20, 0x69, + 0x6e, 0x20, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x29, 0x29, 0x20, 0x7b, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x74, 0x68, 0x72, 0x6f, 0x77, 0x20, 0x6e, 0x65, 0x77, + 0x20, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x28, 0x60, 0x45, 0x72, 0x72, 0x6f, + 0x72, 0x20, 0x72, 0x65, 0x73, 0x6f, 0x6c, 0x76, 0x69, 0x6e, 0x67, 0x20, + 0x72, 0x65, 0x66, 0x20, 0x24, 0x7b, 0x72, 0x65, 0x66, 0x7d, 0x3a, 0x20, + 0x24, 0x7b, 0x73, 0x65, 0x6c, 0x7d, 0x20, 0x6e, 0x6f, 0x74, 0x20, 0x69, + 0x6e, 0x20, 0x24, 0x7b, 0x4a, 0x53, 0x4f, 0x4e, 0x2e, 0x73, 0x74, 0x72, + 0x69, 0x6e, 0x67, 0x69, 0x66, 0x79, 0x28, 0x74, 0x61, 0x72, 0x67, 0x65, + 0x74, 0x29, 0x7d, 0x60, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x74, 0x61, 0x72, + 0x67, 0x65, 0x74, 0x20, 0x3d, 0x20, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, + 0x5b, 0x73, 0x65, 0x6c, 0x5d, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x5f, + 0x72, 0x65, 0x66, 0x73, 0x5b, 0x72, 0x65, 0x66, 0x5d, 0x20, 0x3d, 0x20, + 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x7d, 0x20, 0x65, 0x6c, 0x73, 0x65, 0x20, 0x7b, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x61, + 0x77, 0x61, 0x69, 0x74, 0x20, 0x50, 0x72, 0x6f, 0x6d, 0x69, 0x73, 0x65, + 0x2e, 0x61, 0x6c, 0x6c, 0x28, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x2e, + 0x76, 0x61, 0x6c, 0x75, 0x65, 0x73, 0x28, 0x6e, 0x29, 0x2e, 0x6d, 0x61, + 0x70, 0x28, 0x76, 0x69, 0x73, 0x69, 0x74, 0x29, 0x29, 0x3b, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x6e, 0x3b, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x7d, 0x3b, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, + 0x74, 0x75, 0x72, 0x6e, 0x20, 0x76, 0x69, 0x73, 0x69, 0x74, 0x28, 0x73, + 0x63, 0x68, 0x65, 0x6d, 0x61, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x7d, 0x0a, + 0x0a, 0x20, 0x20, 0x5f, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, + 0x55, 0x6e, 0x69, 0x6f, 0x6e, 0x52, 0x75, 0x6c, 0x65, 0x28, 0x6e, 0x61, + 0x6d, 0x65, 0x2c, 0x20, 0x61, 0x6c, 0x74, 0x53, 0x63, 0x68, 0x65, 0x6d, + 0x61, 0x73, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, + 0x74, 0x75, 0x72, 0x6e, 0x20, 0x61, 0x6c, 0x74, 0x53, 0x63, 0x68, 0x65, + 0x6d, 0x61, 0x73, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x2e, 0x6d, + 0x61, 0x70, 0x28, 0x28, 0x61, 0x6c, 0x74, 0x53, 0x63, 0x68, 0x65, 0x6d, + 0x61, 0x2c, 0x20, 0x69, 0x29, 0x20, 0x3d, 0x3e, 0x20, 0x74, 0x68, 0x69, + 0x73, 0x2e, 0x76, 0x69, 0x73, 0x69, 0x74, 0x28, 0x61, 0x6c, 0x74, 0x53, + 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2c, 0x20, 0x60, 0x24, 0x7b, 0x6e, 0x61, + 0x6d, 0x65, 0x20, 0x3f, 0x3f, 0x20, 0x27, 0x27, 0x7d, 0x24, 0x7b, 0x6e, + 0x61, 0x6d, 0x65, 0x20, 0x3f, 0x20, 0x27, 0x2d, 0x27, 0x20, 0x3a, 0x20, + 0x27, 0x61, 0x6c, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x74, 0x69, 0x76, 0x65, + 0x2d, 0x27, 0x7d, 0x24, 0x7b, 0x69, 0x7d, 0x60, 0x29, 0x29, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x2e, 0x6a, 0x6f, 0x69, 0x6e, 0x28, 0x27, + 0x20, 0x7c, 0x20, 0x27, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x7d, 0x0a, 0x0a, + 0x20, 0x20, 0x5f, 0x76, 0x69, 0x73, 0x69, 0x74, 0x50, 0x61, 0x74, 0x74, + 0x65, 0x72, 0x6e, 0x28, 0x70, 0x61, 0x74, 0x74, 0x65, 0x72, 0x6e, 0x2c, + 0x20, 0x6e, 0x61, 0x6d, 0x65, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x69, 0x66, 0x20, 0x28, 0x21, 0x70, 0x61, 0x74, 0x74, 0x65, 0x72, + 0x6e, 0x2e, 0x73, 0x74, 0x61, 0x72, 0x74, 0x73, 0x57, 0x69, 0x74, 0x68, + 0x28, 0x27, 0x5e, 0x27, 0x29, 0x20, 0x7c, 0x7c, 0x20, 0x21, 0x70, 0x61, + 0x74, 0x74, 0x65, 0x72, 0x6e, 0x2e, 0x65, 0x6e, 0x64, 0x73, 0x57, 0x69, + 0x74, 0x68, 0x28, 0x27, 0x24, 0x27, 0x29, 0x29, 0x20, 0x7b, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x74, 0x68, 0x72, 0x6f, 0x77, 0x20, 0x6e, + 0x65, 0x77, 0x20, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x28, 0x27, 0x50, 0x61, + 0x74, 0x74, 0x65, 0x72, 0x6e, 0x20, 0x6d, 0x75, 0x73, 0x74, 0x20, 0x73, + 0x74, 0x61, 0x72, 0x74, 0x20, 0x77, 0x69, 0x74, 0x68, 0x20, 0x22, 0x5e, + 0x22, 0x20, 0x61, 0x6e, 0x64, 0x20, 0x65, 0x6e, 0x64, 0x20, 0x77, 0x69, + 0x74, 0x68, 0x20, 0x22, 0x24, 0x22, 0x27, 0x29, 0x3b, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x70, 0x61, 0x74, 0x74, + 0x65, 0x72, 0x6e, 0x20, 0x3d, 0x20, 0x70, 0x61, 0x74, 0x74, 0x65, 0x72, + 0x6e, 0x2e, 0x73, 0x6c, 0x69, 0x63, 0x65, 0x28, 0x31, 0x2c, 0x20, 0x2d, + 0x31, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, + 0x74, 0x20, 0x73, 0x75, 0x62, 0x52, 0x75, 0x6c, 0x65, 0x49, 0x64, 0x73, + 0x20, 0x3d, 0x20, 0x7b, 0x7d, 0x3b, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x6c, 0x65, 0x74, 0x20, 0x69, 0x20, 0x3d, 0x20, 0x30, 0x3b, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x6c, 0x65, 0x6e, + 0x67, 0x74, 0x68, 0x20, 0x3d, 0x20, 0x70, 0x61, 0x74, 0x74, 0x65, 0x72, + 0x6e, 0x2e, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x3b, 0x0a, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x67, 0x65, 0x74, + 0x44, 0x6f, 0x74, 0x20, 0x3d, 0x20, 0x28, 0x29, 0x20, 0x3d, 0x3e, 0x20, + 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x6c, 0x65, 0x74, 0x20, + 0x72, 0x75, 0x6c, 0x65, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x69, 0x66, 0x20, 0x28, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x5f, 0x64, 0x6f, + 0x74, 0x61, 0x6c, 0x6c, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x72, 0x75, 0x6c, 0x65, 0x20, 0x3d, 0x20, 0x27, + 0x5b, 0x5c, 0x5c, 0x55, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, + 0x2d, 0x5c, 0x5c, 0x55, 0x30, 0x30, 0x31, 0x30, 0x46, 0x46, 0x46, 0x46, + 0x5d, 0x27, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x20, + 0x65, 0x6c, 0x73, 0x65, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x2f, 0x2f, 0x20, 0x41, 0x63, 0x63, 0x65, 0x70, 0x74, + 0x20, 0x61, 0x6e, 0x79, 0x20, 0x63, 0x68, 0x61, 0x72, 0x61, 0x63, 0x74, + 0x65, 0x72, 0x2e, 0x2e, 0x2e, 0x20, 0x65, 0x78, 0x63, 0x65, 0x70, 0x74, + 0x20, 0x5c, 0x6e, 0x20, 0x61, 0x6e, 0x64, 0x20, 0x5c, 0x72, 0x20, 0x6c, + 0x69, 0x6e, 0x65, 0x20, 0x62, 0x72, 0x65, 0x61, 0x6b, 0x20, 0x63, 0x68, + 0x61, 0x72, 0x73, 0x20, 0x28, 0x5c, 0x78, 0x30, 0x41, 0x20, 0x61, 0x6e, + 0x64, 0x20, 0x5c, 0x78, 0x4f, 0x44, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x72, 0x75, 0x6c, 0x65, 0x20, 0x3d, 0x20, 0x27, + 0x5b, 0x5c, 0x5c, 0x55, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, + 0x2d, 0x5c, 0x5c, 0x78, 0x30, 0x39, 0x5c, 0x5c, 0x78, 0x30, 0x42, 0x5c, + 0x5c, 0x78, 0x30, 0x43, 0x5c, 0x5c, 0x78, 0x30, 0x45, 0x2d, 0x5c, 0x5c, + 0x55, 0x30, 0x30, 0x31, 0x30, 0x46, 0x46, 0x46, 0x46, 0x5d, 0x27, 0x3b, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x74, 0x68, + 0x69, 0x73, 0x2e, 0x5f, 0x61, 0x64, 0x64, 0x52, 0x75, 0x6c, 0x65, 0x28, + 0x27, 0x64, 0x6f, 0x74, 0x27, 0x2c, 0x20, 0x72, 0x75, 0x6c, 0x65, 0x29, + 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x3b, 0x0a, 0x0a, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x74, 0x6f, 0x52, + 0x75, 0x6c, 0x65, 0x20, 0x3d, 0x20, 0x28, 0x5b, 0x73, 0x2c, 0x20, 0x69, + 0x73, 0x4c, 0x69, 0x74, 0x65, 0x72, 0x61, 0x6c, 0x5d, 0x29, 0x20, 0x3d, + 0x3e, 0x20, 0x69, 0x73, 0x4c, 0x69, 0x74, 0x65, 0x72, 0x61, 0x6c, 0x20, + 0x3f, 0x20, 0x22, 0x5c, 0x22, 0x22, 0x20, 0x2b, 0x20, 0x73, 0x20, 0x2b, + 0x20, 0x22, 0x5c, 0x22, 0x22, 0x20, 0x3a, 0x20, 0x73, 0x3b, 0x0a, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x74, 0x72, + 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x20, 0x3d, 0x20, 0x28, 0x29, + 0x20, 0x3d, 0x3e, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x73, 0x74, 0x61, 0x72, 0x74, 0x20, + 0x3d, 0x20, 0x69, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x2f, + 0x2f, 0x20, 0x46, 0x6f, 0x72, 0x20, 0x65, 0x61, 0x63, 0x68, 0x20, 0x63, + 0x6f, 0x6d, 0x70, 0x6f, 0x6e, 0x65, 0x6e, 0x74, 0x20, 0x6f, 0x66, 0x20, + 0x74, 0x68, 0x69, 0x73, 0x20, 0x73, 0x65, 0x71, 0x75, 0x65, 0x6e, 0x63, + 0x65, 0x2c, 0x20, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x20, 0x69, 0x74, 0x73, + 0x20, 0x73, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x20, 0x72, 0x65, 0x70, 0x72, + 0x65, 0x73, 0x65, 0x6e, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x61, + 0x6e, 0x64, 0x20, 0x77, 0x68, 0x65, 0x74, 0x68, 0x65, 0x72, 0x20, 0x69, + 0x74, 0x27, 0x73, 0x20, 0x61, 0x20, 0x6c, 0x69, 0x74, 0x65, 0x72, 0x61, + 0x6c, 0x2e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x2f, 0x2f, 0x20, + 0x57, 0x65, 0x20, 0x6f, 0x6e, 0x6c, 0x79, 0x20, 0x6e, 0x65, 0x65, 0x64, + 0x20, 0x61, 0x20, 0x66, 0x6c, 0x61, 0x74, 0x20, 0x73, 0x74, 0x72, 0x75, + 0x63, 0x74, 0x75, 0x72, 0x65, 0x20, 0x68, 0x65, 0x72, 0x65, 0x20, 0x74, + 0x6f, 0x20, 0x61, 0x70, 0x70, 0x6c, 0x79, 0x20, 0x72, 0x65, 0x70, 0x65, + 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x6f, 0x70, 0x65, 0x72, 0x61, + 0x74, 0x6f, 0x72, 0x73, 0x20, 0x74, 0x6f, 0x20, 0x74, 0x68, 0x65, 0x20, + 0x6c, 0x61, 0x73, 0x74, 0x20, 0x69, 0x74, 0x65, 0x6d, 0x2c, 0x20, 0x61, + 0x6e, 0x64, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x2f, 0x2f, 0x20, + 0x74, 0x6f, 0x20, 0x6d, 0x65, 0x72, 0x67, 0x65, 0x20, 0x6c, 0x69, 0x74, + 0x65, 0x72, 0x61, 0x6c, 0x73, 0x20, 0x61, 0x74, 0x20, 0x74, 0x68, 0x65, + 0x20, 0x61, 0x6e, 0x64, 0x20, 0x28, 0x77, 0x65, 0x27, 0x72, 0x65, 0x20, + 0x70, 0x61, 0x72, 0x73, 0x69, 0x6e, 0x67, 0x20, 0x67, 0x72, 0x6f, 0x75, + 0x70, 0x65, 0x64, 0x20, 0x28, 0x20, 0x73, 0x65, 0x71, 0x75, 0x65, 0x6e, + 0x63, 0x65, 0x73, 0x20, 0x29, 0x20, 0x72, 0x65, 0x63, 0x75, 0x72, 0x73, + 0x69, 0x76, 0x65, 0x6c, 0x79, 0x20, 0x61, 0x6e, 0x64, 0x20, 0x64, 0x6f, + 0x6e, 0x27, 0x74, 0x20, 0x74, 0x72, 0x65, 0x61, 0x74, 0x20, 0x27, 0x7c, + 0x27, 0x20, 0x73, 0x70, 0x65, 0x63, 0x69, 0x61, 0x6c, 0x6c, 0x79, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x2f, 0x2f, 0x20, 0x28, 0x47, 0x42, + 0x4e, 0x46, 0x27, 0x73, 0x20, 0x73, 0x79, 0x6e, 0x74, 0x61, 0x78, 0x20, + 0x69, 0x73, 0x20, 0x6c, 0x75, 0x63, 0x6b, 0x69, 0x6c, 0x79, 0x20, 0x76, + 0x65, 0x72, 0x79, 0x20, 0x63, 0x6c, 0x6f, 0x73, 0x65, 0x20, 0x74, 0x6f, + 0x20, 0x72, 0x65, 0x67, 0x75, 0x6c, 0x61, 0x72, 0x20, 0x65, 0x78, 0x70, + 0x72, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x73, 0x21, 0x29, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x73, + 0x65, 0x71, 0x20, 0x3d, 0x20, 0x5b, 0x5d, 0x3b, 0x0a, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x6a, 0x6f, + 0x69, 0x6e, 0x53, 0x65, 0x71, 0x20, 0x3d, 0x20, 0x28, 0x29, 0x20, 0x3d, + 0x3e, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x72, 0x65, 0x74, 0x20, 0x3d, 0x20, + 0x5b, 0x5d, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x66, 0x6f, 0x72, 0x20, 0x28, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x5b, + 0x69, 0x73, 0x4c, 0x69, 0x74, 0x65, 0x72, 0x61, 0x6c, 0x2c, 0x20, 0x67, + 0x5d, 0x20, 0x6f, 0x66, 0x20, 0x67, 0x72, 0x6f, 0x75, 0x70, 0x42, 0x79, + 0x28, 0x73, 0x65, 0x71, 0x2c, 0x20, 0x78, 0x20, 0x3d, 0x3e, 0x20, 0x78, + 0x5b, 0x31, 0x5d, 0x29, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, 0x69, 0x73, + 0x4c, 0x69, 0x74, 0x65, 0x72, 0x61, 0x6c, 0x29, 0x20, 0x7b, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, + 0x65, 0x74, 0x2e, 0x70, 0x75, 0x73, 0x68, 0x28, 0x5b, 0x5b, 0x2e, 0x2e, + 0x2e, 0x67, 0x5d, 0x2e, 0x6d, 0x61, 0x70, 0x28, 0x78, 0x20, 0x3d, 0x3e, + 0x20, 0x78, 0x5b, 0x30, 0x5d, 0x29, 0x2e, 0x6a, 0x6f, 0x69, 0x6e, 0x28, + 0x27, 0x27, 0x29, 0x2c, 0x20, 0x74, 0x72, 0x75, 0x65, 0x5d, 0x29, 0x3b, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, + 0x20, 0x65, 0x6c, 0x73, 0x65, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, 0x74, 0x2e, + 0x70, 0x75, 0x73, 0x68, 0x28, 0x2e, 0x2e, 0x2e, 0x67, 0x29, 0x3b, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x73, 0x65, 0x71, 0x2e, - 0x70, 0x75, 0x73, 0x68, 0x28, 0x5b, 0x60, 0x28, 0x24, 0x7b, 0x74, 0x6f, - 0x52, 0x75, 0x6c, 0x65, 0x28, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, - 0x72, 0x6d, 0x28, 0x29, 0x29, 0x7d, 0x29, 0x60, 0x2c, 0x20, 0x66, 0x61, - 0x6c, 0x73, 0x65, 0x5d, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x7d, 0x20, 0x65, 0x6c, 0x73, 0x65, 0x20, 0x69, 0x66, - 0x20, 0x28, 0x63, 0x20, 0x3d, 0x3d, 0x3d, 0x20, 0x27, 0x29, 0x27, 0x29, - 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, 0x72, 0x65, + 0x74, 0x2e, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x20, 0x3d, 0x3d, 0x3d, + 0x20, 0x31, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x72, + 0x65, 0x74, 0x5b, 0x30, 0x5d, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x5b, 0x72, 0x65, 0x74, + 0x2e, 0x6d, 0x61, 0x70, 0x28, 0x78, 0x20, 0x3d, 0x3e, 0x20, 0x74, 0x6f, + 0x52, 0x75, 0x6c, 0x65, 0x28, 0x78, 0x29, 0x29, 0x2e, 0x6a, 0x6f, 0x69, + 0x6e, 0x28, 0x27, 0x20, 0x27, 0x29, 0x2c, 0x20, 0x66, 0x61, 0x6c, 0x73, + 0x65, 0x5d, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x3b, + 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x77, 0x68, 0x69, 0x6c, + 0x65, 0x20, 0x28, 0x69, 0x20, 0x3c, 0x20, 0x6c, 0x65, 0x6e, 0x67, 0x74, + 0x68, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x63, 0x20, 0x3d, 0x20, 0x70, + 0x61, 0x74, 0x74, 0x65, 0x72, 0x6e, 0x5b, 0x69, 0x5d, 0x3b, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, 0x63, + 0x20, 0x3d, 0x3d, 0x3d, 0x20, 0x27, 0x2e, 0x27, 0x29, 0x20, 0x7b, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x73, 0x65, + 0x71, 0x2e, 0x70, 0x75, 0x73, 0x68, 0x28, 0x5b, 0x67, 0x65, 0x74, 0x44, + 0x6f, 0x74, 0x28, 0x29, 0x2c, 0x20, 0x66, 0x61, 0x6c, 0x73, 0x65, 0x5d, + 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x20, 0x2b, 0x3d, 0x20, 0x31, 0x3b, 0x0a, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, 0x73, - 0x74, 0x61, 0x72, 0x74, 0x20, 0x3c, 0x3d, 0x20, 0x30, 0x20, 0x7c, 0x7c, - 0x20, 0x70, 0x61, 0x74, 0x74, 0x65, 0x72, 0x6e, 0x5b, 0x73, 0x74, 0x61, - 0x72, 0x74, 0x20, 0x2d, 0x20, 0x31, 0x5d, 0x20, 0x21, 0x3d, 0x3d, 0x20, - 0x27, 0x28, 0x27, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x20, 0x65, 0x6c, 0x73, 0x65, 0x20, + 0x69, 0x66, 0x20, 0x28, 0x63, 0x20, 0x3d, 0x3d, 0x3d, 0x20, 0x27, 0x28, + 0x27, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x69, 0x20, 0x2b, 0x3d, 0x20, 0x31, 0x3b, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, + 0x28, 0x69, 0x20, 0x3c, 0x20, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x29, + 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, 0x70, 0x61, 0x74, 0x74, 0x65, + 0x72, 0x6e, 0x5b, 0x69, 0x5d, 0x20, 0x3d, 0x3d, 0x3d, 0x20, 0x27, 0x3f, + 0x27, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x74, 0x68, 0x72, 0x6f, 0x77, 0x20, 0x6e, 0x65, 0x77, 0x20, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x28, 0x60, - 0x55, 0x6e, 0x62, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, 0x64, 0x20, 0x70, - 0x61, 0x72, 0x65, 0x6e, 0x74, 0x68, 0x65, 0x73, 0x65, 0x73, 0x3b, 0x20, - 0x73, 0x74, 0x61, 0x72, 0x74, 0x20, 0x3d, 0x20, 0x24, 0x7b, 0x73, 0x74, - 0x61, 0x72, 0x74, 0x7d, 0x2c, 0x20, 0x69, 0x20, 0x3d, 0x20, 0x24, 0x7b, - 0x69, 0x7d, 0x2c, 0x20, 0x70, 0x61, 0x74, 0x74, 0x65, 0x72, 0x6e, 0x20, - 0x3d, 0x20, 0x24, 0x7b, 0x70, 0x61, 0x74, 0x74, 0x65, 0x72, 0x6e, 0x7d, + 0x55, 0x6e, 0x73, 0x75, 0x70, 0x70, 0x6f, 0x72, 0x74, 0x65, 0x64, 0x20, + 0x70, 0x61, 0x74, 0x74, 0x65, 0x72, 0x6e, 0x20, 0x73, 0x79, 0x6e, 0x74, + 0x61, 0x78, 0x20, 0x22, 0x24, 0x7b, 0x70, 0x61, 0x74, 0x74, 0x65, 0x72, + 0x6e, 0x5b, 0x69, 0x5d, 0x7d, 0x22, 0x20, 0x61, 0x74, 0x20, 0x69, 0x6e, + 0x64, 0x65, 0x78, 0x20, 0x24, 0x7b, 0x69, 0x7d, 0x20, 0x6f, 0x66, 0x20, + 0x2f, 0x24, 0x7b, 0x70, 0x61, 0x74, 0x74, 0x65, 0x72, 0x6e, 0x7d, 0x2f, 0x60, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x6a, 0x6f, 0x69, - 0x6e, 0x53, 0x65, 0x71, 0x28, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x7d, 0x20, 0x65, 0x6c, 0x73, 0x65, 0x20, 0x69, - 0x66, 0x20, 0x28, 0x63, 0x20, 0x3d, 0x3d, 0x3d, 0x20, 0x27, 0x5b, 0x27, - 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x6c, 0x65, 0x74, 0x20, 0x73, 0x71, 0x75, 0x61, 0x72, 0x65, - 0x42, 0x72, 0x61, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x20, 0x3d, 0x20, 0x63, - 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x69, 0x20, 0x2b, 0x3d, 0x20, 0x31, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x77, 0x68, 0x69, 0x6c, 0x65, 0x20, - 0x28, 0x69, 0x20, 0x3c, 0x20, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x20, - 0x26, 0x26, 0x20, 0x70, 0x61, 0x74, 0x74, 0x65, 0x72, 0x6e, 0x5b, 0x69, - 0x5d, 0x20, 0x21, 0x3d, 0x3d, 0x20, 0x27, 0x5d, 0x27, 0x29, 0x20, 0x7b, - 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x69, 0x66, 0x20, 0x28, 0x70, 0x61, 0x74, 0x74, 0x65, 0x72, 0x6e, - 0x5b, 0x69, 0x5d, 0x20, 0x3d, 0x3d, 0x3d, 0x20, 0x27, 0x5c, 0x5c, 0x27, - 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x73, 0x65, 0x71, 0x2e, 0x70, 0x75, 0x73, 0x68, + 0x28, 0x5b, 0x60, 0x28, 0x24, 0x7b, 0x74, 0x6f, 0x52, 0x75, 0x6c, 0x65, + 0x28, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x28, 0x29, + 0x29, 0x7d, 0x29, 0x60, 0x2c, 0x20, 0x66, 0x61, 0x6c, 0x73, 0x65, 0x5d, + 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, + 0x20, 0x65, 0x6c, 0x73, 0x65, 0x20, 0x69, 0x66, 0x20, 0x28, 0x63, 0x20, + 0x3d, 0x3d, 0x3d, 0x20, 0x27, 0x29, 0x27, 0x29, 0x20, 0x7b, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x20, 0x2b, + 0x3d, 0x20, 0x31, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, 0x73, 0x74, 0x61, 0x72, 0x74, + 0x20, 0x3c, 0x3d, 0x20, 0x30, 0x20, 0x7c, 0x7c, 0x20, 0x70, 0x61, 0x74, + 0x74, 0x65, 0x72, 0x6e, 0x5b, 0x73, 0x74, 0x61, 0x72, 0x74, 0x20, 0x2d, + 0x20, 0x31, 0x5d, 0x20, 0x21, 0x3d, 0x3d, 0x20, 0x27, 0x28, 0x27, 0x29, + 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x74, 0x68, 0x72, 0x6f, 0x77, 0x20, 0x6e, 0x65, 0x77, + 0x20, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x28, 0x60, 0x55, 0x6e, 0x62, 0x61, + 0x6c, 0x61, 0x6e, 0x63, 0x65, 0x64, 0x20, 0x70, 0x61, 0x72, 0x65, 0x6e, + 0x74, 0x68, 0x65, 0x73, 0x65, 0x73, 0x3b, 0x20, 0x73, 0x74, 0x61, 0x72, + 0x74, 0x20, 0x3d, 0x20, 0x24, 0x7b, 0x73, 0x74, 0x61, 0x72, 0x74, 0x7d, + 0x2c, 0x20, 0x69, 0x20, 0x3d, 0x20, 0x24, 0x7b, 0x69, 0x7d, 0x2c, 0x20, + 0x70, 0x61, 0x74, 0x74, 0x65, 0x72, 0x6e, 0x20, 0x3d, 0x20, 0x24, 0x7b, + 0x70, 0x61, 0x74, 0x74, 0x65, 0x72, 0x6e, 0x7d, 0x60, 0x29, 0x3b, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, + 0x74, 0x75, 0x72, 0x6e, 0x20, 0x6a, 0x6f, 0x69, 0x6e, 0x53, 0x65, 0x71, + 0x28, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x7d, 0x20, 0x65, 0x6c, 0x73, 0x65, 0x20, 0x69, 0x66, 0x20, 0x28, 0x63, + 0x20, 0x3d, 0x3d, 0x3d, 0x20, 0x27, 0x5b, 0x27, 0x29, 0x20, 0x7b, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x6c, 0x65, + 0x74, 0x20, 0x73, 0x71, 0x75, 0x61, 0x72, 0x65, 0x42, 0x72, 0x61, 0x63, + 0x6b, 0x65, 0x74, 0x73, 0x20, 0x3d, 0x20, 0x63, 0x3b, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x20, 0x2b, 0x3d, + 0x20, 0x31, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x77, 0x68, 0x69, 0x6c, 0x65, 0x20, 0x28, 0x69, 0x20, 0x3c, + 0x20, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x20, 0x26, 0x26, 0x20, 0x70, + 0x61, 0x74, 0x74, 0x65, 0x72, 0x6e, 0x5b, 0x69, 0x5d, 0x20, 0x21, 0x3d, + 0x3d, 0x20, 0x27, 0x5d, 0x27, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, + 0x28, 0x70, 0x61, 0x74, 0x74, 0x65, 0x72, 0x6e, 0x5b, 0x69, 0x5d, 0x20, + 0x3d, 0x3d, 0x3d, 0x20, 0x27, 0x5c, 0x5c, 0x27, 0x29, 0x20, 0x7b, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x73, 0x71, 0x75, 0x61, 0x72, 0x65, 0x42, 0x72, 0x61, 0x63, + 0x6b, 0x65, 0x74, 0x73, 0x20, 0x2b, 0x3d, 0x20, 0x70, 0x61, 0x74, 0x74, + 0x65, 0x72, 0x6e, 0x2e, 0x73, 0x6c, 0x69, 0x63, 0x65, 0x28, 0x69, 0x2c, + 0x20, 0x69, 0x20, 0x2b, 0x20, 0x32, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, + 0x20, 0x2b, 0x3d, 0x20, 0x32, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x20, 0x65, 0x6c, 0x73, + 0x65, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x73, 0x71, 0x75, 0x61, 0x72, 0x65, 0x42, 0x72, 0x61, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x20, 0x2b, 0x3d, 0x20, - 0x70, 0x61, 0x74, 0x74, 0x65, 0x72, 0x6e, 0x2e, 0x73, 0x6c, 0x69, 0x63, - 0x65, 0x28, 0x69, 0x2c, 0x20, 0x69, 0x20, 0x2b, 0x20, 0x32, 0x29, 0x3b, - 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x69, 0x20, 0x2b, 0x3d, 0x20, 0x32, 0x3b, 0x0a, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, - 0x20, 0x65, 0x6c, 0x73, 0x65, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x73, 0x71, - 0x75, 0x61, 0x72, 0x65, 0x42, 0x72, 0x61, 0x63, 0x6b, 0x65, 0x74, 0x73, - 0x20, 0x2b, 0x3d, 0x20, 0x70, 0x61, 0x74, 0x74, 0x65, 0x72, 0x6e, 0x5b, - 0x69, 0x5d, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x20, 0x2b, 0x3d, 0x20, 0x31, - 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, 0x69, 0x20, 0x3e, 0x3d, 0x20, 0x6c, - 0x65, 0x6e, 0x67, 0x74, 0x68, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x74, 0x68, 0x72, - 0x6f, 0x77, 0x20, 0x6e, 0x65, 0x77, 0x20, 0x45, 0x72, 0x72, 0x6f, 0x72, - 0x28, 0x60, 0x55, 0x6e, 0x62, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, 0x64, - 0x20, 0x73, 0x71, 0x75, 0x61, 0x72, 0x65, 0x20, 0x62, 0x72, 0x61, 0x63, + 0x70, 0x61, 0x74, 0x74, 0x65, 0x72, 0x6e, 0x5b, 0x69, 0x5d, 0x3b, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x69, 0x20, 0x2b, 0x3d, 0x20, 0x31, 0x3b, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, + 0x20, 0x28, 0x69, 0x20, 0x3e, 0x3d, 0x20, 0x6c, 0x65, 0x6e, 0x67, 0x74, + 0x68, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x74, 0x68, 0x72, 0x6f, 0x77, 0x20, 0x6e, + 0x65, 0x77, 0x20, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x28, 0x60, 0x55, 0x6e, + 0x62, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, 0x64, 0x20, 0x73, 0x71, 0x75, + 0x61, 0x72, 0x65, 0x20, 0x62, 0x72, 0x61, 0x63, 0x6b, 0x65, 0x74, 0x73, + 0x3b, 0x20, 0x73, 0x74, 0x61, 0x72, 0x74, 0x20, 0x3d, 0x20, 0x24, 0x7b, + 0x73, 0x74, 0x61, 0x72, 0x74, 0x7d, 0x2c, 0x20, 0x69, 0x20, 0x3d, 0x20, + 0x24, 0x7b, 0x69, 0x7d, 0x2c, 0x20, 0x70, 0x61, 0x74, 0x74, 0x65, 0x72, + 0x6e, 0x20, 0x3d, 0x20, 0x24, 0x7b, 0x70, 0x61, 0x74, 0x74, 0x65, 0x72, + 0x6e, 0x7d, 0x60, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x73, 0x71, 0x75, 0x61, 0x72, 0x65, 0x42, 0x72, + 0x61, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x20, 0x2b, 0x3d, 0x20, 0x27, 0x5d, + 0x27, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x69, 0x20, 0x2b, 0x3d, 0x20, 0x31, 0x3b, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x73, 0x65, 0x71, 0x2e, 0x70, + 0x75, 0x73, 0x68, 0x28, 0x5b, 0x73, 0x71, 0x75, 0x61, 0x72, 0x65, 0x42, + 0x72, 0x61, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x2c, 0x20, 0x66, 0x61, 0x6c, + 0x73, 0x65, 0x5d, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x7d, 0x20, 0x65, 0x6c, 0x73, 0x65, 0x20, 0x69, 0x66, 0x20, + 0x28, 0x63, 0x20, 0x3d, 0x3d, 0x3d, 0x20, 0x27, 0x7c, 0x27, 0x29, 0x20, + 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x73, 0x65, 0x71, 0x2e, 0x70, 0x75, 0x73, 0x68, 0x28, 0x5b, 0x27, 0x7c, + 0x27, 0x2c, 0x20, 0x66, 0x61, 0x6c, 0x73, 0x65, 0x5d, 0x29, 0x3b, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x20, + 0x2b, 0x3d, 0x20, 0x31, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x7d, 0x20, 0x65, 0x6c, 0x73, 0x65, 0x20, 0x69, 0x66, 0x20, + 0x28, 0x63, 0x20, 0x3d, 0x3d, 0x3d, 0x20, 0x27, 0x2a, 0x27, 0x20, 0x7c, + 0x7c, 0x20, 0x63, 0x20, 0x3d, 0x3d, 0x3d, 0x20, 0x27, 0x2b, 0x27, 0x20, + 0x7c, 0x7c, 0x20, 0x63, 0x20, 0x3d, 0x3d, 0x3d, 0x20, 0x27, 0x3f, 0x27, + 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x73, 0x65, 0x71, 0x5b, 0x73, 0x65, 0x71, 0x2e, 0x6c, 0x65, + 0x6e, 0x67, 0x74, 0x68, 0x20, 0x2d, 0x20, 0x31, 0x5d, 0x20, 0x3d, 0x20, + 0x5b, 0x74, 0x6f, 0x52, 0x75, 0x6c, 0x65, 0x28, 0x73, 0x65, 0x71, 0x5b, + 0x73, 0x65, 0x71, 0x2e, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x20, 0x2d, + 0x20, 0x31, 0x5d, 0x29, 0x20, 0x2b, 0x20, 0x63, 0x2c, 0x20, 0x66, 0x61, + 0x6c, 0x73, 0x65, 0x5d, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x69, 0x20, 0x2b, 0x3d, 0x20, 0x31, 0x3b, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x20, 0x65, 0x6c, + 0x73, 0x65, 0x20, 0x69, 0x66, 0x20, 0x28, 0x63, 0x20, 0x3d, 0x3d, 0x3d, + 0x20, 0x27, 0x7b, 0x27, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x6c, 0x65, 0x74, 0x20, 0x63, 0x75, + 0x72, 0x6c, 0x79, 0x42, 0x72, 0x61, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x20, + 0x3d, 0x20, 0x63, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x69, 0x20, 0x2b, 0x3d, 0x20, 0x31, 0x3b, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x77, 0x68, 0x69, + 0x6c, 0x65, 0x20, 0x28, 0x69, 0x20, 0x3c, 0x20, 0x6c, 0x65, 0x6e, 0x67, + 0x74, 0x68, 0x20, 0x26, 0x26, 0x20, 0x70, 0x61, 0x74, 0x74, 0x65, 0x72, + 0x6e, 0x5b, 0x69, 0x5d, 0x20, 0x21, 0x3d, 0x3d, 0x20, 0x27, 0x7d, 0x27, + 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x63, 0x75, 0x72, 0x6c, 0x79, 0x42, 0x72, 0x61, + 0x63, 0x6b, 0x65, 0x74, 0x73, 0x20, 0x2b, 0x3d, 0x20, 0x70, 0x61, 0x74, + 0x74, 0x65, 0x72, 0x6e, 0x5b, 0x69, 0x5d, 0x3b, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x20, 0x2b, + 0x3d, 0x20, 0x31, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, 0x69, 0x20, 0x3e, 0x3d, 0x20, + 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x74, 0x68, + 0x72, 0x6f, 0x77, 0x20, 0x6e, 0x65, 0x77, 0x20, 0x45, 0x72, 0x72, 0x6f, + 0x72, 0x28, 0x60, 0x55, 0x6e, 0x62, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, + 0x64, 0x20, 0x63, 0x75, 0x72, 0x6c, 0x79, 0x20, 0x62, 0x72, 0x61, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x3b, 0x20, 0x73, 0x74, 0x61, 0x72, 0x74, 0x20, 0x3d, 0x20, 0x24, 0x7b, 0x73, 0x74, 0x61, 0x72, 0x74, 0x7d, 0x2c, 0x20, 0x69, 0x20, 0x3d, 0x20, 0x24, 0x7b, 0x69, 0x7d, 0x2c, 0x20, 0x70, 0x61, 0x74, 0x74, 0x65, 0x72, 0x6e, 0x20, 0x3d, 0x20, 0x24, 0x7b, 0x70, 0x61, 0x74, 0x74, 0x65, 0x72, 0x6e, 0x7d, 0x60, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x73, 0x71, 0x75, 0x61, - 0x72, 0x65, 0x42, 0x72, 0x61, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x20, 0x2b, - 0x3d, 0x20, 0x27, 0x5d, 0x27, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x20, 0x2b, 0x3d, 0x20, 0x31, 0x3b, - 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x73, - 0x65, 0x71, 0x2e, 0x70, 0x75, 0x73, 0x68, 0x28, 0x5b, 0x73, 0x71, 0x75, - 0x61, 0x72, 0x65, 0x42, 0x72, 0x61, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x2c, - 0x20, 0x66, 0x61, 0x6c, 0x73, 0x65, 0x5d, 0x29, 0x3b, 0x0a, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x20, 0x65, 0x6c, 0x73, 0x65, - 0x20, 0x69, 0x66, 0x20, 0x28, 0x63, 0x20, 0x3d, 0x3d, 0x3d, 0x20, 0x27, - 0x7c, 0x27, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x73, 0x65, 0x71, 0x2e, 0x70, 0x75, 0x73, 0x68, - 0x28, 0x5b, 0x27, 0x7c, 0x27, 0x2c, 0x20, 0x66, 0x61, 0x6c, 0x73, 0x65, - 0x5d, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x69, 0x20, 0x2b, 0x3d, 0x20, 0x31, 0x3b, 0x0a, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x20, 0x65, 0x6c, 0x73, 0x65, - 0x20, 0x69, 0x66, 0x20, 0x28, 0x63, 0x20, 0x3d, 0x3d, 0x3d, 0x20, 0x27, - 0x2a, 0x27, 0x20, 0x7c, 0x7c, 0x20, 0x63, 0x20, 0x3d, 0x3d, 0x3d, 0x20, - 0x27, 0x2b, 0x27, 0x20, 0x7c, 0x7c, 0x20, 0x63, 0x20, 0x3d, 0x3d, 0x3d, - 0x20, 0x27, 0x3f, 0x27, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x73, 0x65, 0x71, 0x5b, 0x73, 0x65, - 0x71, 0x2e, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x20, 0x2d, 0x20, 0x31, - 0x5d, 0x20, 0x3d, 0x20, 0x5b, 0x74, 0x6f, 0x52, 0x75, 0x6c, 0x65, 0x28, - 0x73, 0x65, 0x71, 0x5b, 0x73, 0x65, 0x71, 0x2e, 0x6c, 0x65, 0x6e, 0x67, - 0x74, 0x68, 0x20, 0x2d, 0x20, 0x31, 0x5d, 0x29, 0x20, 0x2b, 0x20, 0x63, - 0x2c, 0x20, 0x66, 0x61, 0x6c, 0x73, 0x65, 0x5d, 0x3b, 0x0a, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x20, 0x2b, 0x3d, - 0x20, 0x31, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x7d, 0x20, 0x65, 0x6c, 0x73, 0x65, 0x20, 0x69, 0x66, 0x20, 0x28, 0x63, - 0x20, 0x3d, 0x3d, 0x3d, 0x20, 0x27, 0x7b, 0x27, 0x29, 0x20, 0x7b, 0x0a, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x6c, 0x65, - 0x74, 0x20, 0x63, 0x75, 0x72, 0x6c, 0x79, 0x42, 0x72, 0x61, 0x63, 0x6b, - 0x65, 0x74, 0x73, 0x20, 0x3d, 0x20, 0x63, 0x3b, 0x0a, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x20, 0x2b, 0x3d, 0x20, - 0x31, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x77, 0x68, 0x69, 0x6c, 0x65, 0x20, 0x28, 0x69, 0x20, 0x3c, 0x20, - 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x20, 0x26, 0x26, 0x20, 0x70, 0x61, - 0x74, 0x74, 0x65, 0x72, 0x6e, 0x5b, 0x69, 0x5d, 0x20, 0x21, 0x3d, 0x3d, - 0x20, 0x27, 0x7d, 0x27, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x75, 0x72, 0x6c, 0x79, 0x42, 0x72, 0x61, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x20, 0x2b, 0x3d, - 0x20, 0x70, 0x61, 0x74, 0x74, 0x65, 0x72, 0x6e, 0x5b, 0x69, 0x5d, 0x3b, - 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x69, 0x20, 0x2b, 0x3d, 0x20, 0x31, 0x3b, 0x0a, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, 0x69, - 0x20, 0x3e, 0x3d, 0x20, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x29, 0x20, - 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x74, 0x68, 0x72, 0x6f, 0x77, 0x20, 0x6e, 0x65, 0x77, 0x20, - 0x45, 0x72, 0x72, 0x6f, 0x72, 0x28, 0x60, 0x55, 0x6e, 0x62, 0x61, 0x6c, - 0x61, 0x6e, 0x63, 0x65, 0x64, 0x20, 0x63, 0x75, 0x72, 0x6c, 0x79, 0x20, - 0x62, 0x72, 0x61, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x3b, 0x20, 0x73, 0x74, - 0x61, 0x72, 0x74, 0x20, 0x3d, 0x20, 0x24, 0x7b, 0x73, 0x74, 0x61, 0x72, - 0x74, 0x7d, 0x2c, 0x20, 0x69, 0x20, 0x3d, 0x20, 0x24, 0x7b, 0x69, 0x7d, - 0x2c, 0x20, 0x70, 0x61, 0x74, 0x74, 0x65, 0x72, 0x6e, 0x20, 0x3d, 0x20, - 0x24, 0x7b, 0x70, 0x61, 0x74, 0x74, 0x65, 0x72, 0x6e, 0x7d, 0x60, 0x29, - 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x63, 0x75, 0x72, 0x6c, 0x79, 0x42, 0x72, 0x61, 0x63, 0x6b, 0x65, 0x74, - 0x73, 0x20, 0x2b, 0x3d, 0x20, 0x27, 0x7d, 0x27, 0x3b, 0x0a, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x20, 0x2b, 0x3d, - 0x20, 0x31, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x6e, 0x75, 0x6d, 0x73, - 0x20, 0x3d, 0x20, 0x63, 0x75, 0x72, 0x6c, 0x79, 0x42, 0x72, 0x61, 0x63, - 0x6b, 0x65, 0x74, 0x73, 0x2e, 0x73, 0x6c, 0x69, 0x63, 0x65, 0x28, 0x31, - 0x2c, 0x20, 0x2d, 0x31, 0x29, 0x2e, 0x73, 0x70, 0x6c, 0x69, 0x74, 0x28, - 0x27, 0x2c, 0x27, 0x29, 0x2e, 0x6d, 0x61, 0x70, 0x28, 0x73, 0x20, 0x3d, - 0x3e, 0x20, 0x73, 0x2e, 0x74, 0x72, 0x69, 0x6d, 0x28, 0x29, 0x29, 0x3b, - 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x6c, - 0x65, 0x74, 0x20, 0x6d, 0x69, 0x6e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x2c, - 0x20, 0x6d, 0x61, 0x78, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x3b, 0x0a, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, - 0x28, 0x6e, 0x75, 0x6d, 0x73, 0x2e, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, - 0x20, 0x3d, 0x3d, 0x3d, 0x20, 0x31, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x6d, 0x69, - 0x6e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x20, 0x3d, 0x20, 0x70, 0x61, 0x72, - 0x73, 0x65, 0x49, 0x6e, 0x74, 0x28, 0x6e, 0x75, 0x6d, 0x73, 0x5b, 0x30, - 0x5d, 0x2c, 0x20, 0x31, 0x30, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x6d, 0x61, 0x78, 0x54, - 0x69, 0x6d, 0x65, 0x73, 0x20, 0x3d, 0x20, 0x6d, 0x69, 0x6e, 0x54, 0x69, - 0x6d, 0x65, 0x73, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x7d, 0x20, 0x65, 0x6c, 0x73, 0x65, 0x20, 0x7b, 0x0a, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x69, 0x66, 0x20, 0x28, 0x6e, 0x75, 0x6d, 0x73, 0x2e, 0x6c, 0x65, 0x6e, - 0x67, 0x74, 0x68, 0x20, 0x21, 0x3d, 0x3d, 0x20, 0x32, 0x29, 0x20, 0x7b, - 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x74, 0x68, 0x72, 0x6f, 0x77, 0x20, 0x6e, 0x65, 0x77, - 0x20, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x28, 0x60, 0x49, 0x6e, 0x76, 0x61, - 0x6c, 0x69, 0x64, 0x20, 0x71, 0x75, 0x61, 0x6e, 0x74, 0x69, 0x66, 0x69, - 0x65, 0x72, 0x20, 0x24, 0x7b, 0x63, 0x75, 0x72, 0x6c, 0x79, 0x42, 0x72, - 0x61, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x7d, 0x60, 0x29, 0x3b, 0x0a, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, - 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x6d, 0x69, 0x6e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x20, 0x3d, 0x20, - 0x6e, 0x75, 0x6d, 0x73, 0x5b, 0x30, 0x5d, 0x20, 0x3f, 0x20, 0x70, 0x61, - 0x72, 0x73, 0x65, 0x49, 0x6e, 0x74, 0x28, 0x6e, 0x75, 0x6d, 0x73, 0x5b, - 0x30, 0x5d, 0x2c, 0x20, 0x31, 0x30, 0x29, 0x20, 0x3a, 0x20, 0x30, 0x3b, - 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x6d, 0x61, 0x78, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x20, 0x3d, 0x20, - 0x6e, 0x75, 0x6d, 0x73, 0x5b, 0x31, 0x5d, 0x20, 0x3f, 0x20, 0x70, 0x61, - 0x72, 0x73, 0x65, 0x49, 0x6e, 0x74, 0x28, 0x6e, 0x75, 0x6d, 0x73, 0x5b, - 0x31, 0x5d, 0x2c, 0x20, 0x31, 0x30, 0x29, 0x20, 0x3a, 0x20, 0x49, 0x6e, - 0x66, 0x69, 0x6e, 0x69, 0x74, 0x79, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x0a, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x6c, 0x65, 0x74, 0x20, 0x5b, - 0x73, 0x75, 0x62, 0x2c, 0x20, 0x73, 0x75, 0x62, 0x49, 0x73, 0x4c, 0x69, - 0x74, 0x65, 0x72, 0x61, 0x6c, 0x5d, 0x20, 0x3d, 0x20, 0x73, 0x65, 0x71, - 0x5b, 0x73, 0x65, 0x71, 0x2e, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x20, - 0x2d, 0x20, 0x31, 0x5d, 0x3b, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, 0x6d, 0x69, 0x6e, - 0x54, 0x69, 0x6d, 0x65, 0x73, 0x20, 0x3d, 0x3d, 0x3d, 0x20, 0x30, 0x20, - 0x26, 0x26, 0x20, 0x6d, 0x61, 0x78, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x20, - 0x3d, 0x3d, 0x3d, 0x20, 0x49, 0x6e, 0x66, 0x69, 0x6e, 0x69, 0x74, 0x79, - 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x73, 0x65, 0x71, 0x5b, 0x73, 0x65, 0x71, 0x2e, - 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x20, 0x2d, 0x20, 0x31, 0x5d, 0x20, - 0x3d, 0x20, 0x5b, 0x60, 0x24, 0x7b, 0x73, 0x75, 0x62, 0x7d, 0x2a, 0x60, - 0x2c, 0x20, 0x66, 0x61, 0x6c, 0x73, 0x65, 0x5d, 0x3b, 0x0a, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x20, 0x65, 0x6c, - 0x73, 0x65, 0x20, 0x69, 0x66, 0x20, 0x28, 0x6d, 0x69, 0x6e, 0x54, 0x69, - 0x6d, 0x65, 0x73, 0x20, 0x3d, 0x3d, 0x3d, 0x20, 0x30, 0x20, 0x26, 0x26, - 0x20, 0x6d, 0x61, 0x78, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x20, 0x3d, 0x3d, - 0x3d, 0x20, 0x31, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x73, 0x65, 0x71, 0x5b, 0x73, - 0x65, 0x71, 0x2e, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x20, 0x2d, 0x20, - 0x31, 0x5d, 0x20, 0x3d, 0x20, 0x5b, 0x60, 0x24, 0x7b, 0x73, 0x75, 0x62, - 0x7d, 0x3f, 0x60, 0x2c, 0x20, 0x66, 0x61, 0x6c, 0x73, 0x65, 0x5d, 0x3b, - 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, - 0x20, 0x65, 0x6c, 0x73, 0x65, 0x20, 0x69, 0x66, 0x20, 0x28, 0x6d, 0x69, - 0x6e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x20, 0x3d, 0x3d, 0x3d, 0x20, 0x31, - 0x20, 0x26, 0x26, 0x20, 0x6d, 0x61, 0x78, 0x54, 0x69, 0x6d, 0x65, 0x73, - 0x20, 0x3d, 0x3d, 0x3d, 0x20, 0x49, 0x6e, 0x66, 0x69, 0x6e, 0x69, 0x74, - 0x79, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x73, 0x65, 0x71, 0x5b, 0x73, 0x65, 0x71, - 0x2e, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x20, 0x2d, 0x20, 0x31, 0x5d, - 0x20, 0x3d, 0x20, 0x5b, 0x60, 0x24, 0x7b, 0x73, 0x75, 0x62, 0x7d, 0x2b, - 0x60, 0x2c, 0x20, 0x66, 0x61, 0x6c, 0x73, 0x65, 0x5d, 0x3b, 0x0a, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x20, 0x65, - 0x6c, 0x73, 0x65, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, 0x21, 0x73, - 0x75, 0x62, 0x49, 0x73, 0x4c, 0x69, 0x74, 0x65, 0x72, 0x61, 0x6c, 0x29, - 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x6c, 0x65, 0x74, 0x20, 0x69, 0x64, 0x20, - 0x3d, 0x20, 0x73, 0x75, 0x62, 0x52, 0x75, 0x6c, 0x65, 0x49, 0x64, 0x73, - 0x5b, 0x73, 0x75, 0x62, 0x5d, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, - 0x28, 0x69, 0x64, 0x20, 0x3d, 0x3d, 0x3d, 0x20, 0x75, 0x6e, 0x64, 0x65, - 0x66, 0x69, 0x6e, 0x65, 0x64, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x69, 0x64, 0x20, 0x3d, 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x5f, - 0x61, 0x64, 0x64, 0x52, 0x75, 0x6c, 0x65, 0x28, 0x60, 0x24, 0x7b, 0x6e, - 0x61, 0x6d, 0x65, 0x7d, 0x2d, 0x24, 0x7b, 0x4f, 0x62, 0x6a, 0x65, 0x63, - 0x74, 0x2e, 0x6b, 0x65, 0x79, 0x73, 0x28, 0x73, 0x75, 0x62, 0x52, 0x75, - 0x6c, 0x65, 0x49, 0x64, 0x73, 0x29, 0x2e, 0x6c, 0x65, 0x6e, 0x67, 0x74, - 0x68, 0x20, 0x2b, 0x20, 0x31, 0x7d, 0x60, 0x2c, 0x20, 0x73, 0x75, 0x62, - 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x73, 0x75, 0x62, 0x52, 0x75, - 0x6c, 0x65, 0x49, 0x64, 0x73, 0x5b, 0x73, 0x75, 0x62, 0x5d, 0x20, 0x3d, - 0x20, 0x69, 0x64, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x73, - 0x75, 0x62, 0x20, 0x3d, 0x20, 0x69, 0x64, 0x3b, 0x0a, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x0a, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x72, 0x65, 0x70, 0x65, 0x61, 0x74, - 0x65, 0x64, 0x53, 0x75, 0x62, 0x20, 0x3d, 0x20, 0x41, 0x72, 0x72, 0x61, - 0x79, 0x2e, 0x66, 0x72, 0x6f, 0x6d, 0x28, 0x7b, 0x20, 0x6c, 0x65, 0x6e, - 0x67, 0x74, 0x68, 0x3a, 0x20, 0x6d, 0x69, 0x6e, 0x54, 0x69, 0x6d, 0x65, - 0x73, 0x20, 0x7d, 0x2c, 0x20, 0x28, 0x29, 0x20, 0x3d, 0x3e, 0x20, 0x73, - 0x75, 0x62, 0x49, 0x73, 0x4c, 0x69, 0x74, 0x65, 0x72, 0x61, 0x6c, 0x20, - 0x3f, 0x20, 0x60, 0x22, 0x24, 0x7b, 0x73, 0x75, 0x62, 0x2e, 0x73, 0x6c, - 0x69, 0x63, 0x65, 0x28, 0x31, 0x2c, 0x20, 0x2d, 0x31, 0x29, 0x2e, 0x72, - 0x65, 0x70, 0x65, 0x61, 0x74, 0x28, 0x6d, 0x69, 0x6e, 0x54, 0x69, 0x6d, - 0x65, 0x73, 0x29, 0x7d, 0x22, 0x60, 0x20, 0x3a, 0x20, 0x73, 0x75, 0x62, - 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x6f, 0x70, 0x74, - 0x69, 0x6f, 0x6e, 0x61, 0x6c, 0x53, 0x75, 0x62, 0x20, 0x3d, 0x20, 0x6d, - 0x61, 0x78, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x20, 0x21, 0x3d, 0x3d, 0x20, - 0x75, 0x6e, 0x64, 0x65, 0x66, 0x69, 0x6e, 0x65, 0x64, 0x20, 0x3f, 0x20, - 0x41, 0x72, 0x72, 0x61, 0x79, 0x2e, 0x66, 0x72, 0x6f, 0x6d, 0x28, 0x7b, - 0x20, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x3a, 0x20, 0x6d, 0x61, 0x78, - 0x54, 0x69, 0x6d, 0x65, 0x73, 0x20, 0x2d, 0x20, 0x6d, 0x69, 0x6e, 0x54, - 0x69, 0x6d, 0x65, 0x73, 0x20, 0x7d, 0x2c, 0x20, 0x28, 0x29, 0x20, 0x3d, - 0x3e, 0x20, 0x60, 0x24, 0x7b, 0x73, 0x75, 0x62, 0x7d, 0x3f, 0x60, 0x29, - 0x20, 0x3a, 0x20, 0x5b, 0x60, 0x24, 0x7b, 0x73, 0x75, 0x62, 0x7d, 0x2a, - 0x60, 0x5d, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x73, 0x65, 0x71, 0x5b, 0x73, 0x65, 0x71, 0x2e, - 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x20, 0x2d, 0x20, 0x31, 0x5d, 0x20, - 0x3d, 0x20, 0x5b, 0x72, 0x65, 0x70, 0x65, 0x61, 0x74, 0x65, 0x64, 0x53, - 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6e, 0x63, 0x61, 0x74, 0x28, 0x6f, 0x70, - 0x74, 0x69, 0x6f, 0x6e, 0x61, 0x6c, 0x53, 0x75, 0x62, 0x29, 0x2e, 0x6a, - 0x6f, 0x69, 0x6e, 0x28, 0x27, 0x20, 0x27, 0x29, 0x2c, 0x20, 0x66, 0x61, - 0x6c, 0x73, 0x65, 0x5d, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x7d, 0x20, 0x65, 0x6c, 0x73, 0x65, 0x20, 0x7b, 0x0a, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x6c, 0x65, 0x74, - 0x20, 0x6c, 0x69, 0x74, 0x65, 0x72, 0x61, 0x6c, 0x20, 0x3d, 0x20, 0x27, - 0x27, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x77, 0x68, 0x69, 0x6c, 0x65, 0x20, 0x28, 0x69, 0x20, 0x3c, 0x20, - 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, - 0x20, 0x28, 0x70, 0x61, 0x74, 0x74, 0x65, 0x72, 0x6e, 0x5b, 0x69, 0x5d, - 0x20, 0x3d, 0x3d, 0x3d, 0x20, 0x27, 0x5c, 0x5c, 0x27, 0x20, 0x26, 0x26, - 0x20, 0x69, 0x20, 0x3c, 0x20, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x20, - 0x2d, 0x20, 0x31, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, - 0x73, 0x74, 0x20, 0x6e, 0x65, 0x78, 0x74, 0x20, 0x3d, 0x20, 0x70, 0x61, - 0x74, 0x74, 0x65, 0x72, 0x6e, 0x5b, 0x69, 0x20, 0x2b, 0x20, 0x31, 0x5d, - 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, 0x45, 0x53, 0x43, 0x41, - 0x50, 0x45, 0x44, 0x5f, 0x49, 0x4e, 0x5f, 0x52, 0x45, 0x47, 0x45, 0x58, - 0x50, 0x53, 0x5f, 0x42, 0x55, 0x54, 0x5f, 0x4e, 0x4f, 0x54, 0x5f, 0x49, - 0x4e, 0x5f, 0x4c, 0x49, 0x54, 0x45, 0x52, 0x41, 0x4c, 0x53, 0x2e, 0x68, - 0x61, 0x73, 0x28, 0x6e, 0x65, 0x78, 0x74, 0x29, 0x29, 0x20, 0x7b, 0x0a, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x27, 0x7d, 0x27, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x20, 0x2b, 0x3d, 0x20, 0x31, 0x3b, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, + 0x6e, 0x73, 0x74, 0x20, 0x6e, 0x75, 0x6d, 0x73, 0x20, 0x3d, 0x20, 0x63, + 0x75, 0x72, 0x6c, 0x79, 0x42, 0x72, 0x61, 0x63, 0x6b, 0x65, 0x74, 0x73, + 0x2e, 0x73, 0x6c, 0x69, 0x63, 0x65, 0x28, 0x31, 0x2c, 0x20, 0x2d, 0x31, + 0x29, 0x2e, 0x73, 0x70, 0x6c, 0x69, 0x74, 0x28, 0x27, 0x2c, 0x27, 0x29, + 0x2e, 0x6d, 0x61, 0x70, 0x28, 0x73, 0x20, 0x3d, 0x3e, 0x20, 0x73, 0x2e, + 0x74, 0x72, 0x69, 0x6d, 0x28, 0x29, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x6c, 0x65, 0x74, 0x20, 0x6d, + 0x69, 0x6e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x2c, 0x20, 0x6d, 0x61, 0x78, + 0x54, 0x69, 0x6d, 0x65, 0x73, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, 0x6e, 0x75, 0x6d, + 0x73, 0x2e, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x20, 0x3d, 0x3d, 0x3d, + 0x20, 0x31, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x6d, 0x69, 0x6e, 0x54, 0x69, 0x6d, + 0x65, 0x73, 0x20, 0x3d, 0x20, 0x70, 0x61, 0x72, 0x73, 0x65, 0x49, 0x6e, + 0x74, 0x28, 0x6e, 0x75, 0x6d, 0x73, 0x5b, 0x30, 0x5d, 0x2c, 0x20, 0x31, + 0x30, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x6d, 0x61, 0x78, 0x54, 0x69, 0x6d, 0x65, 0x73, + 0x20, 0x3d, 0x20, 0x6d, 0x69, 0x6e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x3b, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, + 0x20, 0x65, 0x6c, 0x73, 0x65, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, + 0x6e, 0x75, 0x6d, 0x73, 0x2e, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x20, + 0x21, 0x3d, 0x3d, 0x20, 0x32, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x74, + 0x68, 0x72, 0x6f, 0x77, 0x20, 0x6e, 0x65, 0x77, 0x20, 0x45, 0x72, 0x72, + 0x6f, 0x72, 0x28, 0x60, 0x49, 0x6e, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x20, + 0x71, 0x75, 0x61, 0x6e, 0x74, 0x69, 0x66, 0x69, 0x65, 0x72, 0x20, 0x24, + 0x7b, 0x63, 0x75, 0x72, 0x6c, 0x79, 0x42, 0x72, 0x61, 0x63, 0x6b, 0x65, + 0x74, 0x73, 0x7d, 0x60, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x6d, 0x69, 0x6e, + 0x54, 0x69, 0x6d, 0x65, 0x73, 0x20, 0x3d, 0x20, 0x6e, 0x75, 0x6d, 0x73, + 0x5b, 0x30, 0x5d, 0x20, 0x3f, 0x20, 0x70, 0x61, 0x72, 0x73, 0x65, 0x49, + 0x6e, 0x74, 0x28, 0x6e, 0x75, 0x6d, 0x73, 0x5b, 0x30, 0x5d, 0x2c, 0x20, + 0x31, 0x30, 0x29, 0x20, 0x3a, 0x20, 0x30, 0x3b, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x6d, 0x61, 0x78, + 0x54, 0x69, 0x6d, 0x65, 0x73, 0x20, 0x3d, 0x20, 0x6e, 0x75, 0x6d, 0x73, + 0x5b, 0x31, 0x5d, 0x20, 0x3f, 0x20, 0x70, 0x61, 0x72, 0x73, 0x65, 0x49, + 0x6e, 0x74, 0x28, 0x6e, 0x75, 0x6d, 0x73, 0x5b, 0x31, 0x5d, 0x2c, 0x20, + 0x31, 0x30, 0x29, 0x20, 0x3a, 0x20, 0x49, 0x6e, 0x66, 0x69, 0x6e, 0x69, + 0x74, 0x79, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x7d, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x6c, 0x65, 0x74, 0x20, 0x5b, 0x73, 0x75, 0x62, 0x2c, + 0x20, 0x73, 0x75, 0x62, 0x49, 0x73, 0x4c, 0x69, 0x74, 0x65, 0x72, 0x61, + 0x6c, 0x5d, 0x20, 0x3d, 0x20, 0x73, 0x65, 0x71, 0x5b, 0x73, 0x65, 0x71, + 0x2e, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x20, 0x2d, 0x20, 0x31, 0x5d, + 0x3b, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x69, 0x66, 0x20, 0x28, 0x6d, 0x69, 0x6e, 0x54, 0x69, 0x6d, 0x65, + 0x73, 0x20, 0x3d, 0x3d, 0x3d, 0x20, 0x30, 0x20, 0x26, 0x26, 0x20, 0x6d, + 0x61, 0x78, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x20, 0x3d, 0x3d, 0x3d, 0x20, + 0x49, 0x6e, 0x66, 0x69, 0x6e, 0x69, 0x74, 0x79, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x6c, 0x69, 0x74, 0x65, 0x72, 0x61, 0x6c, 0x20, - 0x2b, 0x3d, 0x20, 0x70, 0x61, 0x74, 0x74, 0x65, 0x72, 0x6e, 0x5b, 0x69, - 0x5d, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x20, 0x2b, 0x3d, 0x20, - 0x31, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x73, 0x65, 0x71, 0x5b, 0x73, 0x65, 0x71, 0x2e, 0x6c, 0x65, 0x6e, 0x67, + 0x74, 0x68, 0x20, 0x2d, 0x20, 0x31, 0x5d, 0x20, 0x3d, 0x20, 0x5b, 0x60, + 0x24, 0x7b, 0x73, 0x75, 0x62, 0x7d, 0x2a, 0x60, 0x2c, 0x20, 0x66, 0x61, + 0x6c, 0x73, 0x65, 0x5d, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x7d, 0x20, 0x65, 0x6c, 0x73, 0x65, 0x20, 0x69, + 0x66, 0x20, 0x28, 0x6d, 0x69, 0x6e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x20, + 0x3d, 0x3d, 0x3d, 0x20, 0x30, 0x20, 0x26, 0x26, 0x20, 0x6d, 0x61, 0x78, + 0x54, 0x69, 0x6d, 0x65, 0x73, 0x20, 0x3d, 0x3d, 0x3d, 0x20, 0x31, 0x29, + 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x73, 0x65, 0x71, 0x5b, 0x73, 0x65, 0x71, 0x2e, 0x6c, + 0x65, 0x6e, 0x67, 0x74, 0x68, 0x20, 0x2d, 0x20, 0x31, 0x5d, 0x20, 0x3d, + 0x20, 0x5b, 0x60, 0x24, 0x7b, 0x73, 0x75, 0x62, 0x7d, 0x3f, 0x60, 0x2c, + 0x20, 0x66, 0x61, 0x6c, 0x73, 0x65, 0x5d, 0x3b, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x20, 0x65, 0x6c, 0x73, + 0x65, 0x20, 0x69, 0x66, 0x20, 0x28, 0x6d, 0x69, 0x6e, 0x54, 0x69, 0x6d, + 0x65, 0x73, 0x20, 0x3d, 0x3d, 0x3d, 0x20, 0x31, 0x20, 0x26, 0x26, 0x20, + 0x6d, 0x61, 0x78, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x20, 0x3d, 0x3d, 0x3d, + 0x20, 0x49, 0x6e, 0x66, 0x69, 0x6e, 0x69, 0x74, 0x79, 0x29, 0x20, 0x7b, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x73, 0x65, 0x71, 0x5b, 0x73, 0x65, 0x71, 0x2e, 0x6c, 0x65, 0x6e, + 0x67, 0x74, 0x68, 0x20, 0x2d, 0x20, 0x31, 0x5d, 0x20, 0x3d, 0x20, 0x5b, + 0x60, 0x24, 0x7b, 0x73, 0x75, 0x62, 0x7d, 0x2b, 0x60, 0x2c, 0x20, 0x66, + 0x61, 0x6c, 0x73, 0x65, 0x5d, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x20, 0x65, 0x6c, 0x73, 0x65, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x6c, 0x69, 0x74, 0x65, 0x72, 0x61, - 0x6c, 0x20, 0x2b, 0x3d, 0x20, 0x70, 0x61, 0x74, 0x74, 0x65, 0x72, 0x6e, - 0x2e, 0x73, 0x6c, 0x69, 0x63, 0x65, 0x28, 0x69, 0x2c, 0x20, 0x69, 0x20, - 0x2b, 0x20, 0x32, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x20, - 0x2b, 0x3d, 0x20, 0x32, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x20, - 0x65, 0x6c, 0x73, 0x65, 0x20, 0x69, 0x66, 0x20, 0x28, 0x70, 0x61, 0x74, - 0x74, 0x65, 0x72, 0x6e, 0x5b, 0x69, 0x5d, 0x20, 0x3d, 0x3d, 0x3d, 0x20, - 0x27, 0x22, 0x27, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x6c, 0x69, 0x74, - 0x65, 0x72, 0x61, 0x6c, 0x20, 0x2b, 0x3d, 0x20, 0x27, 0x5c, 0x5c, 0x22, - 0x27, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x20, 0x2b, 0x3d, 0x20, 0x31, 0x3b, - 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x7d, 0x20, 0x65, 0x6c, 0x73, 0x65, 0x20, 0x69, 0x66, 0x20, 0x28, - 0x21, 0x4e, 0x4f, 0x4e, 0x5f, 0x4c, 0x49, 0x54, 0x45, 0x52, 0x41, 0x4c, - 0x5f, 0x53, 0x45, 0x54, 0x2e, 0x68, 0x61, 0x73, 0x28, 0x70, 0x61, 0x74, - 0x74, 0x65, 0x72, 0x6e, 0x5b, 0x69, 0x5d, 0x29, 0x20, 0x26, 0x26, 0x0a, + 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, 0x21, 0x73, 0x75, 0x62, 0x49, 0x73, + 0x4c, 0x69, 0x74, 0x65, 0x72, 0x61, 0x6c, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x28, 0x69, 0x20, 0x3d, 0x3d, 0x3d, 0x20, 0x6c, - 0x65, 0x6e, 0x67, 0x74, 0x68, 0x20, 0x2d, 0x20, 0x31, 0x20, 0x7c, 0x7c, - 0x20, 0x6c, 0x69, 0x74, 0x65, 0x72, 0x61, 0x6c, 0x20, 0x3d, 0x3d, 0x3d, - 0x20, 0x27, 0x27, 0x20, 0x7c, 0x7c, 0x20, 0x70, 0x61, 0x74, 0x74, 0x65, - 0x72, 0x6e, 0x5b, 0x69, 0x20, 0x2b, 0x20, 0x31, 0x5d, 0x20, 0x3d, 0x3d, - 0x3d, 0x20, 0x27, 0x2e, 0x27, 0x20, 0x7c, 0x7c, 0x20, 0x21, 0x4e, 0x4f, - 0x4e, 0x5f, 0x4c, 0x49, 0x54, 0x45, 0x52, 0x41, 0x4c, 0x5f, 0x53, 0x45, - 0x54, 0x2e, 0x68, 0x61, 0x73, 0x28, 0x70, 0x61, 0x74, 0x74, 0x65, 0x72, - 0x6e, 0x5b, 0x69, 0x2b, 0x31, 0x5d, 0x29, 0x29, 0x29, 0x20, 0x7b, 0x0a, + 0x20, 0x6c, 0x65, 0x74, 0x20, 0x69, 0x64, 0x20, 0x3d, 0x20, 0x73, 0x75, + 0x62, 0x52, 0x75, 0x6c, 0x65, 0x49, 0x64, 0x73, 0x5b, 0x73, 0x75, 0x62, + 0x5d, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, 0x69, 0x64, 0x20, + 0x3d, 0x3d, 0x3d, 0x20, 0x75, 0x6e, 0x64, 0x65, 0x66, 0x69, 0x6e, 0x65, + 0x64, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x64, 0x20, + 0x3d, 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x5f, 0x61, 0x64, 0x64, 0x52, + 0x75, 0x6c, 0x65, 0x28, 0x60, 0x24, 0x7b, 0x6e, 0x61, 0x6d, 0x65, 0x7d, + 0x2d, 0x24, 0x7b, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x2e, 0x6b, 0x65, + 0x79, 0x73, 0x28, 0x73, 0x75, 0x62, 0x52, 0x75, 0x6c, 0x65, 0x49, 0x64, + 0x73, 0x29, 0x2e, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x20, 0x2b, 0x20, + 0x31, 0x7d, 0x60, 0x2c, 0x20, 0x73, 0x75, 0x62, 0x29, 0x3b, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x73, 0x75, 0x62, 0x52, 0x75, 0x6c, 0x65, 0x49, 0x64, + 0x73, 0x5b, 0x73, 0x75, 0x62, 0x5d, 0x20, 0x3d, 0x20, 0x69, 0x64, 0x3b, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x73, 0x75, 0x62, 0x20, 0x3d, + 0x20, 0x69, 0x64, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, + 0x74, 0x20, 0x72, 0x65, 0x70, 0x65, 0x61, 0x74, 0x65, 0x64, 0x53, 0x75, + 0x62, 0x20, 0x3d, 0x20, 0x41, 0x72, 0x72, 0x61, 0x79, 0x2e, 0x66, 0x72, + 0x6f, 0x6d, 0x28, 0x7b, 0x20, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x3a, + 0x20, 0x6d, 0x69, 0x6e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x20, 0x7d, 0x2c, + 0x20, 0x28, 0x29, 0x20, 0x3d, 0x3e, 0x20, 0x73, 0x75, 0x62, 0x49, 0x73, + 0x4c, 0x69, 0x74, 0x65, 0x72, 0x61, 0x6c, 0x20, 0x3f, 0x20, 0x60, 0x22, + 0x24, 0x7b, 0x73, 0x75, 0x62, 0x2e, 0x73, 0x6c, 0x69, 0x63, 0x65, 0x28, + 0x31, 0x2c, 0x20, 0x2d, 0x31, 0x29, 0x2e, 0x72, 0x65, 0x70, 0x65, 0x61, + 0x74, 0x28, 0x6d, 0x69, 0x6e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x29, 0x7d, + 0x22, 0x60, 0x20, 0x3a, 0x20, 0x73, 0x75, 0x62, 0x29, 0x3b, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, + 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x61, + 0x6c, 0x53, 0x75, 0x62, 0x20, 0x3d, 0x20, 0x6d, 0x61, 0x78, 0x54, 0x69, + 0x6d, 0x65, 0x73, 0x20, 0x21, 0x3d, 0x3d, 0x20, 0x75, 0x6e, 0x64, 0x65, + 0x66, 0x69, 0x6e, 0x65, 0x64, 0x20, 0x3f, 0x20, 0x41, 0x72, 0x72, 0x61, + 0x79, 0x2e, 0x66, 0x72, 0x6f, 0x6d, 0x28, 0x7b, 0x20, 0x6c, 0x65, 0x6e, + 0x67, 0x74, 0x68, 0x3a, 0x20, 0x6d, 0x61, 0x78, 0x54, 0x69, 0x6d, 0x65, + 0x73, 0x20, 0x2d, 0x20, 0x6d, 0x69, 0x6e, 0x54, 0x69, 0x6d, 0x65, 0x73, + 0x20, 0x7d, 0x2c, 0x20, 0x28, 0x29, 0x20, 0x3d, 0x3e, 0x20, 0x60, 0x24, + 0x7b, 0x73, 0x75, 0x62, 0x7d, 0x3f, 0x60, 0x29, 0x20, 0x3a, 0x20, 0x5b, + 0x60, 0x24, 0x7b, 0x73, 0x75, 0x62, 0x7d, 0x2a, 0x60, 0x5d, 0x3b, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x73, 0x65, 0x71, 0x5b, 0x73, 0x65, 0x71, 0x2e, 0x6c, 0x65, 0x6e, 0x67, + 0x74, 0x68, 0x20, 0x2d, 0x20, 0x31, 0x5d, 0x20, 0x3d, 0x20, 0x5b, 0x72, + 0x65, 0x70, 0x65, 0x61, 0x74, 0x65, 0x64, 0x53, 0x75, 0x62, 0x2e, 0x63, + 0x6f, 0x6e, 0x63, 0x61, 0x74, 0x28, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, + 0x61, 0x6c, 0x53, 0x75, 0x62, 0x29, 0x2e, 0x6a, 0x6f, 0x69, 0x6e, 0x28, + 0x27, 0x20, 0x27, 0x29, 0x2c, 0x20, 0x66, 0x61, 0x6c, 0x73, 0x65, 0x5d, + 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x20, + 0x65, 0x6c, 0x73, 0x65, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x6c, 0x65, 0x74, 0x20, 0x6c, 0x69, 0x74, + 0x65, 0x72, 0x61, 0x6c, 0x20, 0x3d, 0x20, 0x27, 0x27, 0x3b, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x77, 0x68, 0x69, + 0x6c, 0x65, 0x20, 0x28, 0x69, 0x20, 0x3c, 0x20, 0x6c, 0x65, 0x6e, 0x67, + 0x74, 0x68, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, 0x70, 0x61, + 0x74, 0x74, 0x65, 0x72, 0x6e, 0x5b, 0x69, 0x5d, 0x20, 0x3d, 0x3d, 0x3d, + 0x20, 0x27, 0x5c, 0x5c, 0x27, 0x20, 0x26, 0x26, 0x20, 0x69, 0x20, 0x3c, + 0x20, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x20, 0x2d, 0x20, 0x31, 0x29, + 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x6e, + 0x65, 0x78, 0x74, 0x20, 0x3d, 0x20, 0x70, 0x61, 0x74, 0x74, 0x65, 0x72, + 0x6e, 0x5b, 0x69, 0x20, 0x2b, 0x20, 0x31, 0x5d, 0x3b, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x69, 0x66, 0x20, 0x28, 0x45, 0x53, 0x43, 0x41, 0x50, 0x45, 0x44, 0x5f, + 0x49, 0x4e, 0x5f, 0x52, 0x45, 0x47, 0x45, 0x58, 0x50, 0x53, 0x5f, 0x42, + 0x55, 0x54, 0x5f, 0x4e, 0x4f, 0x54, 0x5f, 0x49, 0x4e, 0x5f, 0x4c, 0x49, + 0x54, 0x45, 0x52, 0x41, 0x4c, 0x53, 0x2e, 0x68, 0x61, 0x73, 0x28, 0x6e, + 0x65, 0x78, 0x74, 0x29, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x69, 0x20, 0x2b, 0x3d, 0x20, 0x31, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x6c, 0x69, 0x74, 0x65, 0x72, 0x61, 0x6c, 0x20, 0x2b, 0x3d, 0x20, 0x70, + 0x61, 0x74, 0x74, 0x65, 0x72, 0x6e, 0x5b, 0x69, 0x5d, 0x3b, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x69, 0x20, 0x2b, 0x3d, 0x20, 0x31, 0x3b, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x7d, 0x20, 0x65, 0x6c, 0x73, 0x65, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x6c, 0x69, 0x74, 0x65, 0x72, 0x61, 0x6c, 0x20, 0x2b, 0x3d, - 0x20, 0x70, 0x61, 0x74, 0x74, 0x65, 0x72, 0x6e, 0x5b, 0x69, 0x5d, 0x3b, - 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x69, 0x20, 0x2b, 0x3d, 0x20, 0x31, 0x3b, 0x0a, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, - 0x20, 0x65, 0x6c, 0x73, 0x65, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x62, 0x72, - 0x65, 0x61, 0x6b, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, 0x6c, 0x69, 0x74, - 0x65, 0x72, 0x61, 0x6c, 0x20, 0x21, 0x3d, 0x3d, 0x20, 0x27, 0x27, 0x29, - 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x73, 0x65, 0x71, 0x2e, 0x70, 0x75, 0x73, 0x68, 0x28, - 0x5b, 0x6c, 0x69, 0x74, 0x65, 0x72, 0x61, 0x6c, 0x2c, 0x20, 0x74, 0x72, - 0x75, 0x65, 0x5d, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x70, 0x61, 0x74, 0x74, 0x65, 0x72, 0x6e, 0x2e, 0x73, 0x6c, 0x69, + 0x63, 0x65, 0x28, 0x69, 0x2c, 0x20, 0x69, 0x20, 0x2b, 0x20, 0x32, 0x29, + 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x20, 0x2b, 0x3d, 0x20, 0x32, + 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, - 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, - 0x6e, 0x20, 0x6a, 0x6f, 0x69, 0x6e, 0x53, 0x65, 0x71, 0x28, 0x29, 0x3b, - 0x0a, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x3b, 0x0a, 0x0a, 0x20, 0x20, 0x20, - 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x74, 0x68, 0x69, 0x73, - 0x2e, 0x5f, 0x61, 0x64, 0x64, 0x52, 0x75, 0x6c, 0x65, 0x28, 0x6e, 0x61, - 0x6d, 0x65, 0x2c, 0x20, 0x22, 0x5c, 0x22, 0x5c, 0x5c, 0x5c, 0x22, 0x5c, - 0x22, 0x20, 0x22, 0x20, 0x2b, 0x20, 0x74, 0x6f, 0x52, 0x75, 0x6c, 0x65, - 0x28, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x28, 0x29, - 0x29, 0x20, 0x2b, 0x20, 0x22, 0x20, 0x5c, 0x22, 0x5c, 0x5c, 0x5c, 0x22, - 0x5c, 0x22, 0x20, 0x73, 0x70, 0x61, 0x63, 0x65, 0x22, 0x29, 0x0a, 0x20, - 0x20, 0x7d, 0x0a, 0x0a, 0x20, 0x20, 0x5f, 0x72, 0x65, 0x73, 0x6f, 0x6c, - 0x76, 0x65, 0x52, 0x65, 0x66, 0x28, 0x72, 0x65, 0x66, 0x29, 0x20, 0x7b, - 0x0a, 0x20, 0x20, 0x20, 0x20, 0x6c, 0x65, 0x74, 0x20, 0x72, 0x65, 0x66, - 0x4e, 0x61, 0x6d, 0x65, 0x20, 0x3d, 0x20, 0x72, 0x65, 0x66, 0x2e, 0x73, - 0x70, 0x6c, 0x69, 0x74, 0x28, 0x27, 0x2f, 0x27, 0x29, 0x2e, 0x70, 0x6f, - 0x70, 0x28, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, - 0x28, 0x21, 0x28, 0x72, 0x65, 0x66, 0x4e, 0x61, 0x6d, 0x65, 0x20, 0x69, - 0x6e, 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x5f, 0x72, 0x75, 0x6c, 0x65, - 0x73, 0x29, 0x20, 0x26, 0x26, 0x20, 0x21, 0x74, 0x68, 0x69, 0x73, 0x2e, - 0x5f, 0x72, 0x65, 0x66, 0x73, 0x42, 0x65, 0x69, 0x6e, 0x67, 0x52, 0x65, - 0x73, 0x6f, 0x6c, 0x76, 0x65, 0x64, 0x2e, 0x68, 0x61, 0x73, 0x28, 0x72, - 0x65, 0x66, 0x29, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x5f, 0x72, 0x65, 0x66, 0x73, 0x42, - 0x65, 0x69, 0x6e, 0x67, 0x52, 0x65, 0x73, 0x6f, 0x6c, 0x76, 0x65, 0x64, - 0x2e, 0x61, 0x64, 0x64, 0x28, 0x72, 0x65, 0x66, 0x29, 0x3b, 0x0a, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x72, - 0x65, 0x73, 0x6f, 0x6c, 0x76, 0x65, 0x64, 0x20, 0x3d, 0x20, 0x74, 0x68, - 0x69, 0x73, 0x2e, 0x5f, 0x72, 0x65, 0x66, 0x73, 0x5b, 0x72, 0x65, 0x66, - 0x5d, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, 0x66, - 0x4e, 0x61, 0x6d, 0x65, 0x20, 0x3d, 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, - 0x76, 0x69, 0x73, 0x69, 0x74, 0x28, 0x72, 0x65, 0x73, 0x6f, 0x6c, 0x76, - 0x65, 0x64, 0x2c, 0x20, 0x72, 0x65, 0x66, 0x4e, 0x61, 0x6d, 0x65, 0x29, - 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x74, 0x68, 0x69, 0x73, - 0x2e, 0x5f, 0x72, 0x65, 0x66, 0x73, 0x42, 0x65, 0x69, 0x6e, 0x67, 0x52, - 0x65, 0x73, 0x6f, 0x6c, 0x76, 0x65, 0x64, 0x2e, 0x64, 0x65, 0x6c, 0x65, - 0x74, 0x65, 0x28, 0x72, 0x65, 0x66, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, - 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, - 0x6e, 0x20, 0x72, 0x65, 0x66, 0x4e, 0x61, 0x6d, 0x65, 0x3b, 0x0a, 0x20, - 0x20, 0x7d, 0x0a, 0x0a, 0x20, 0x20, 0x5f, 0x67, 0x65, 0x6e, 0x65, 0x72, - 0x61, 0x74, 0x65, 0x43, 0x6f, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x74, 0x52, - 0x75, 0x6c, 0x65, 0x28, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x29, 0x20, 0x7b, - 0x0a, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, 0x74, 0x79, 0x70, - 0x65, 0x6f, 0x66, 0x20, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x20, 0x21, 0x3d, - 0x3d, 0x20, 0x27, 0x73, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x27, 0x29, 0x20, - 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x74, 0x68, 0x72, 0x6f, - 0x77, 0x20, 0x6e, 0x65, 0x77, 0x20, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x28, - 0x27, 0x4f, 0x6e, 0x6c, 0x79, 0x20, 0x73, 0x74, 0x72, 0x69, 0x6e, 0x67, - 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x74, 0x73, 0x20, 0x61, - 0x72, 0x65, 0x20, 0x73, 0x75, 0x70, 0x70, 0x6f, 0x72, 0x74, 0x65, 0x64, - 0x2c, 0x20, 0x67, 0x6f, 0x74, 0x20, 0x27, 0x20, 0x2b, 0x20, 0x4a, 0x53, - 0x4f, 0x4e, 0x2e, 0x73, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x69, 0x66, 0x79, - 0x28, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x29, 0x29, 0x3b, 0x0a, 0x20, 0x20, - 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, - 0x72, 0x6e, 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x5f, 0x66, 0x6f, 0x72, - 0x6d, 0x61, 0x74, 0x4c, 0x69, 0x74, 0x65, 0x72, 0x61, 0x6c, 0x28, 0x76, - 0x61, 0x6c, 0x75, 0x65, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x7d, 0x0a, 0x0a, - 0x20, 0x20, 0x76, 0x69, 0x73, 0x69, 0x74, 0x28, 0x73, 0x63, 0x68, 0x65, - 0x6d, 0x61, 0x2c, 0x20, 0x6e, 0x61, 0x6d, 0x65, 0x29, 0x20, 0x7b, 0x0a, - 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x73, 0x63, - 0x68, 0x65, 0x6d, 0x61, 0x54, 0x79, 0x70, 0x65, 0x20, 0x3d, 0x20, 0x73, - 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x3b, 0x0a, - 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x73, 0x63, - 0x68, 0x65, 0x6d, 0x61, 0x46, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x20, 0x3d, - 0x20, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2e, 0x66, 0x6f, 0x72, 0x6d, - 0x61, 0x74, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, - 0x74, 0x20, 0x72, 0x75, 0x6c, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x20, 0x3d, - 0x20, 0x6e, 0x61, 0x6d, 0x65, 0x20, 0x69, 0x6e, 0x20, 0x52, 0x45, 0x53, - 0x45, 0x52, 0x56, 0x45, 0x44, 0x5f, 0x4e, 0x41, 0x4d, 0x45, 0x53, 0x20, - 0x3f, 0x20, 0x6e, 0x61, 0x6d, 0x65, 0x20, 0x2b, 0x20, 0x27, 0x2d, 0x27, - 0x20, 0x3a, 0x20, 0x6e, 0x61, 0x6d, 0x65, 0x20, 0x3d, 0x3d, 0x20, 0x27, - 0x27, 0x20, 0x3f, 0x20, 0x27, 0x72, 0x6f, 0x6f, 0x74, 0x27, 0x20, 0x3a, - 0x20, 0x6e, 0x61, 0x6d, 0x65, 0x3b, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, - 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x72, 0x65, 0x66, 0x20, 0x3d, 0x20, - 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2e, 0x24, 0x72, 0x65, 0x66, 0x3b, - 0x0a, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, 0x72, 0x65, 0x66, - 0x20, 0x21, 0x3d, 0x3d, 0x20, 0x75, 0x6e, 0x64, 0x65, 0x66, 0x69, 0x6e, - 0x65, 0x64, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, - 0x5f, 0x61, 0x64, 0x64, 0x52, 0x75, 0x6c, 0x65, 0x28, 0x72, 0x75, 0x6c, - 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x2c, 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, - 0x5f, 0x72, 0x65, 0x73, 0x6f, 0x6c, 0x76, 0x65, 0x52, 0x65, 0x66, 0x28, - 0x72, 0x65, 0x66, 0x29, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x7d, - 0x20, 0x65, 0x6c, 0x73, 0x65, 0x20, 0x69, 0x66, 0x20, 0x28, 0x73, 0x63, - 0x68, 0x65, 0x6d, 0x61, 0x2e, 0x6f, 0x6e, 0x65, 0x4f, 0x66, 0x20, 0x7c, - 0x7c, 0x20, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2e, 0x61, 0x6e, 0x79, - 0x4f, 0x66, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, - 0x5f, 0x61, 0x64, 0x64, 0x52, 0x75, 0x6c, 0x65, 0x28, 0x72, 0x75, 0x6c, - 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x2c, 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, - 0x5f, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x55, 0x6e, 0x69, - 0x6f, 0x6e, 0x52, 0x75, 0x6c, 0x65, 0x28, 0x6e, 0x61, 0x6d, 0x65, 0x2c, - 0x20, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2e, 0x6f, 0x6e, 0x65, 0x4f, - 0x66, 0x20, 0x7c, 0x7c, 0x20, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2e, - 0x61, 0x6e, 0x79, 0x4f, 0x66, 0x29, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, - 0x20, 0x7d, 0x20, 0x65, 0x6c, 0x73, 0x65, 0x20, 0x69, 0x66, 0x20, 0x28, - 0x41, 0x72, 0x72, 0x61, 0x79, 0x2e, 0x69, 0x73, 0x41, 0x72, 0x72, 0x61, - 0x79, 0x28, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x54, 0x79, 0x70, 0x65, - 0x29, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, - 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x5f, - 0x61, 0x64, 0x64, 0x52, 0x75, 0x6c, 0x65, 0x28, 0x72, 0x75, 0x6c, 0x65, - 0x4e, 0x61, 0x6d, 0x65, 0x2c, 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x5f, - 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x55, 0x6e, 0x69, 0x6f, - 0x6e, 0x52, 0x75, 0x6c, 0x65, 0x28, 0x6e, 0x61, 0x6d, 0x65, 0x2c, 0x20, - 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x54, 0x79, 0x70, 0x65, 0x2e, 0x6d, - 0x61, 0x70, 0x28, 0x74, 0x20, 0x3d, 0x3e, 0x20, 0x28, 0x7b, 0x20, 0x74, - 0x79, 0x70, 0x65, 0x3a, 0x20, 0x74, 0x20, 0x7d, 0x29, 0x29, 0x29, 0x29, - 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x20, 0x65, 0x6c, 0x73, 0x65, - 0x20, 0x69, 0x66, 0x20, 0x28, 0x27, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x27, - 0x20, 0x69, 0x6e, 0x20, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x29, 0x20, - 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, - 0x74, 0x79, 0x70, 0x65, 0x6f, 0x66, 0x20, 0x73, 0x63, 0x68, 0x65, 0x6d, - 0x61, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x21, 0x3d, 0x3d, 0x20, - 0x27, 0x73, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x27, 0x29, 0x20, 0x7b, 0x0a, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x74, 0x68, 0x72, 0x6f, - 0x77, 0x20, 0x6e, 0x65, 0x77, 0x20, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x28, - 0x27, 0x4f, 0x6e, 0x6c, 0x79, 0x20, 0x73, 0x74, 0x72, 0x69, 0x6e, 0x67, - 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x74, 0x73, 0x20, 0x61, - 0x72, 0x65, 0x20, 0x73, 0x75, 0x70, 0x70, 0x6f, 0x72, 0x74, 0x65, 0x64, - 0x2c, 0x20, 0x67, 0x6f, 0x74, 0x20, 0x27, 0x20, 0x2b, 0x20, 0x4a, 0x53, - 0x4f, 0x4e, 0x2e, 0x73, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x69, 0x66, 0x79, - 0x28, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2e, 0x63, 0x6f, 0x6e, 0x73, - 0x74, 0x29, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, - 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, - 0x6e, 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x5f, 0x61, 0x64, 0x64, 0x52, - 0x75, 0x6c, 0x65, 0x28, 0x72, 0x75, 0x6c, 0x65, 0x4e, 0x61, 0x6d, 0x65, - 0x2c, 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x5f, 0x67, 0x65, 0x6e, 0x65, - 0x72, 0x61, 0x74, 0x65, 0x43, 0x6f, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x74, - 0x52, 0x75, 0x6c, 0x65, 0x28, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2e, - 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x29, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, - 0x20, 0x7d, 0x20, 0x65, 0x6c, 0x73, 0x65, 0x20, 0x69, 0x66, 0x20, 0x28, - 0x27, 0x65, 0x6e, 0x75, 0x6d, 0x27, 0x20, 0x69, 0x6e, 0x20, 0x73, 0x63, - 0x68, 0x65, 0x6d, 0x61, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x72, 0x75, 0x6c, 0x65, - 0x20, 0x3d, 0x20, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2e, 0x65, 0x6e, - 0x75, 0x6d, 0x2e, 0x6d, 0x61, 0x70, 0x28, 0x76, 0x20, 0x3d, 0x3e, 0x20, - 0x74, 0x68, 0x69, 0x73, 0x2e, 0x5f, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, - 0x74, 0x65, 0x43, 0x6f, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x74, 0x52, 0x75, - 0x6c, 0x65, 0x28, 0x76, 0x29, 0x29, 0x2e, 0x6a, 0x6f, 0x69, 0x6e, 0x28, - 0x27, 0x20, 0x7c, 0x20, 0x27, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x74, 0x68, 0x69, - 0x73, 0x2e, 0x5f, 0x61, 0x64, 0x64, 0x52, 0x75, 0x6c, 0x65, 0x28, 0x72, - 0x75, 0x6c, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x2c, 0x20, 0x72, 0x75, 0x6c, - 0x65, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x20, 0x65, 0x6c, - 0x73, 0x65, 0x20, 0x69, 0x66, 0x20, 0x28, 0x28, 0x73, 0x63, 0x68, 0x65, - 0x6d, 0x61, 0x54, 0x79, 0x70, 0x65, 0x20, 0x3d, 0x3d, 0x3d, 0x20, 0x75, - 0x6e, 0x64, 0x65, 0x66, 0x69, 0x6e, 0x65, 0x64, 0x20, 0x7c, 0x7c, 0x20, - 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x54, 0x79, 0x70, 0x65, 0x20, 0x3d, - 0x3d, 0x3d, 0x20, 0x27, 0x6f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x27, 0x29, - 0x20, 0x26, 0x26, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x28, 0x27, 0x70, 0x72, 0x6f, - 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x27, 0x20, 0x69, 0x6e, 0x20, - 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x20, 0x7c, 0x7c, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x20, 0x65, 0x6c, 0x73, 0x65, + 0x20, 0x69, 0x66, 0x20, 0x28, 0x70, 0x61, 0x74, 0x74, 0x65, 0x72, 0x6e, + 0x5b, 0x69, 0x5d, 0x20, 0x3d, 0x3d, 0x3d, 0x20, 0x27, 0x22, 0x27, 0x29, + 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x6c, 0x69, 0x74, 0x65, 0x72, 0x61, 0x6c, + 0x20, 0x2b, 0x3d, 0x20, 0x27, 0x5c, 0x5c, 0x22, 0x27, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x28, 0x27, 0x61, 0x64, 0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e, - 0x61, 0x6c, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, - 0x27, 0x20, 0x69, 0x6e, 0x20, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x20, - 0x26, 0x26, 0x20, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2e, 0x61, 0x64, - 0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x61, 0x6c, 0x50, 0x72, 0x6f, 0x70, - 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x20, 0x21, 0x3d, 0x3d, 0x20, 0x74, - 0x72, 0x75, 0x65, 0x29, 0x29, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x72, 0x65, 0x71, - 0x75, 0x69, 0x72, 0x65, 0x64, 0x20, 0x3d, 0x20, 0x6e, 0x65, 0x77, 0x20, - 0x53, 0x65, 0x74, 0x28, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2e, 0x72, - 0x65, 0x71, 0x75, 0x69, 0x72, 0x65, 0x64, 0x20, 0x7c, 0x7c, 0x20, 0x5b, - 0x5d, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, - 0x6e, 0x73, 0x74, 0x20, 0x70, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, - 0x65, 0x73, 0x20, 0x3d, 0x20, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x2e, - 0x65, 0x6e, 0x74, 0x72, 0x69, 0x65, 0x73, 0x28, 0x73, 0x63, 0x68, 0x65, - 0x6d, 0x61, 0x2e, 0x70, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, - 0x73, 0x20, 0x3f, 0x3f, 0x20, 0x7b, 0x7d, 0x29, 0x3b, 0x0a, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x74, - 0x68, 0x69, 0x73, 0x2e, 0x5f, 0x61, 0x64, 0x64, 0x52, 0x75, 0x6c, 0x65, - 0x28, 0x72, 0x75, 0x6c, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x2c, 0x20, 0x74, - 0x68, 0x69, 0x73, 0x2e, 0x5f, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x4f, 0x62, - 0x6a, 0x65, 0x63, 0x74, 0x52, 0x75, 0x6c, 0x65, 0x28, 0x70, 0x72, 0x6f, - 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x2c, 0x20, 0x72, 0x65, 0x71, - 0x75, 0x69, 0x72, 0x65, 0x64, 0x2c, 0x20, 0x6e, 0x61, 0x6d, 0x65, 0x2c, - 0x20, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2e, 0x61, 0x64, 0x64, 0x69, - 0x74, 0x69, 0x6f, 0x6e, 0x61, 0x6c, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, - 0x74, 0x69, 0x65, 0x73, 0x29, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, - 0x7d, 0x20, 0x65, 0x6c, 0x73, 0x65, 0x20, 0x69, 0x66, 0x20, 0x28, 0x28, - 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x54, 0x79, 0x70, 0x65, 0x20, 0x3d, - 0x3d, 0x3d, 0x20, 0x75, 0x6e, 0x64, 0x65, 0x66, 0x69, 0x6e, 0x65, 0x64, - 0x20, 0x7c, 0x7c, 0x20, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x54, 0x79, - 0x70, 0x65, 0x20, 0x3d, 0x3d, 0x3d, 0x20, 0x27, 0x6f, 0x62, 0x6a, 0x65, - 0x63, 0x74, 0x27, 0x29, 0x20, 0x26, 0x26, 0x20, 0x27, 0x61, 0x6c, 0x6c, - 0x4f, 0x66, 0x27, 0x20, 0x69, 0x6e, 0x20, 0x73, 0x63, 0x68, 0x65, 0x6d, - 0x61, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, - 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x72, 0x65, 0x71, 0x75, 0x69, 0x72, 0x65, - 0x64, 0x20, 0x3d, 0x20, 0x6e, 0x65, 0x77, 0x20, 0x53, 0x65, 0x74, 0x28, - 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, - 0x73, 0x74, 0x20, 0x70, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, - 0x73, 0x20, 0x3d, 0x20, 0x5b, 0x5d, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x61, 0x64, 0x64, 0x43, - 0x6f, 0x6d, 0x70, 0x6f, 0x6e, 0x65, 0x6e, 0x74, 0x20, 0x3d, 0x20, 0x28, - 0x63, 0x6f, 0x6d, 0x70, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2c, 0x20, - 0x69, 0x73, 0x52, 0x65, 0x71, 0x75, 0x69, 0x72, 0x65, 0x64, 0x29, 0x20, - 0x3d, 0x3e, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x72, 0x65, 0x66, 0x20, 0x3d, - 0x20, 0x63, 0x6f, 0x6d, 0x70, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2e, - 0x24, 0x72, 0x65, 0x66, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x69, 0x20, 0x2b, 0x3d, 0x20, 0x31, 0x3b, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x20, 0x65, + 0x6c, 0x73, 0x65, 0x20, 0x69, 0x66, 0x20, 0x28, 0x21, 0x4e, 0x4f, 0x4e, + 0x5f, 0x4c, 0x49, 0x54, 0x45, 0x52, 0x41, 0x4c, 0x5f, 0x53, 0x45, 0x54, + 0x2e, 0x68, 0x61, 0x73, 0x28, 0x70, 0x61, 0x74, 0x74, 0x65, 0x72, 0x6e, + 0x5b, 0x69, 0x5d, 0x29, 0x20, 0x26, 0x26, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x28, 0x69, 0x20, 0x3d, 0x3d, 0x3d, 0x20, 0x6c, 0x65, 0x6e, 0x67, 0x74, + 0x68, 0x20, 0x2d, 0x20, 0x31, 0x20, 0x7c, 0x7c, 0x20, 0x6c, 0x69, 0x74, + 0x65, 0x72, 0x61, 0x6c, 0x20, 0x3d, 0x3d, 0x3d, 0x20, 0x27, 0x27, 0x20, + 0x7c, 0x7c, 0x20, 0x70, 0x61, 0x74, 0x74, 0x65, 0x72, 0x6e, 0x5b, 0x69, + 0x20, 0x2b, 0x20, 0x31, 0x5d, 0x20, 0x3d, 0x3d, 0x3d, 0x20, 0x27, 0x2e, + 0x27, 0x20, 0x7c, 0x7c, 0x20, 0x21, 0x4e, 0x4f, 0x4e, 0x5f, 0x4c, 0x49, + 0x54, 0x45, 0x52, 0x41, 0x4c, 0x5f, 0x53, 0x45, 0x54, 0x2e, 0x68, 0x61, + 0x73, 0x28, 0x70, 0x61, 0x74, 0x74, 0x65, 0x72, 0x6e, 0x5b, 0x69, 0x2b, + 0x31, 0x5d, 0x29, 0x29, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x6c, 0x69, + 0x74, 0x65, 0x72, 0x61, 0x6c, 0x20, 0x2b, 0x3d, 0x20, 0x70, 0x61, 0x74, + 0x74, 0x65, 0x72, 0x6e, 0x5b, 0x69, 0x5d, 0x3b, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, + 0x20, 0x2b, 0x3d, 0x20, 0x31, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x20, 0x65, 0x6c, 0x73, + 0x65, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x62, 0x72, 0x65, 0x61, 0x6b, 0x3b, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x69, 0x66, 0x20, 0x28, 0x6c, 0x69, 0x74, 0x65, 0x72, 0x61, 0x6c, + 0x20, 0x21, 0x3d, 0x3d, 0x20, 0x27, 0x27, 0x29, 0x20, 0x7b, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x73, + 0x65, 0x71, 0x2e, 0x70, 0x75, 0x73, 0x68, 0x28, 0x5b, 0x6c, 0x69, 0x74, + 0x65, 0x72, 0x61, 0x6c, 0x2c, 0x20, 0x74, 0x72, 0x75, 0x65, 0x5d, 0x29, + 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x6a, 0x6f, + 0x69, 0x6e, 0x53, 0x65, 0x71, 0x28, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x7d, 0x3b, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, 0x74, + 0x75, 0x72, 0x6e, 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x5f, 0x61, 0x64, + 0x64, 0x52, 0x75, 0x6c, 0x65, 0x28, 0x6e, 0x61, 0x6d, 0x65, 0x2c, 0x20, + 0x22, 0x5c, 0x22, 0x5c, 0x5c, 0x5c, 0x22, 0x5c, 0x22, 0x20, 0x22, 0x20, + 0x2b, 0x20, 0x74, 0x6f, 0x52, 0x75, 0x6c, 0x65, 0x28, 0x74, 0x72, 0x61, + 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x28, 0x29, 0x29, 0x20, 0x2b, 0x20, + 0x22, 0x20, 0x5c, 0x22, 0x5c, 0x5c, 0x5c, 0x22, 0x5c, 0x22, 0x20, 0x73, + 0x70, 0x61, 0x63, 0x65, 0x22, 0x29, 0x0a, 0x20, 0x20, 0x7d, 0x0a, 0x0a, + 0x20, 0x20, 0x5f, 0x72, 0x65, 0x73, 0x6f, 0x6c, 0x76, 0x65, 0x52, 0x65, + 0x66, 0x28, 0x72, 0x65, 0x66, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x6c, 0x65, 0x74, 0x20, 0x72, 0x65, 0x66, 0x4e, 0x61, 0x6d, 0x65, + 0x20, 0x3d, 0x20, 0x72, 0x65, 0x66, 0x2e, 0x73, 0x70, 0x6c, 0x69, 0x74, + 0x28, 0x27, 0x2f, 0x27, 0x29, 0x2e, 0x70, 0x6f, 0x70, 0x28, 0x29, 0x3b, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, 0x21, 0x28, 0x72, + 0x65, 0x66, 0x4e, 0x61, 0x6d, 0x65, 0x20, 0x69, 0x6e, 0x20, 0x74, 0x68, + 0x69, 0x73, 0x2e, 0x5f, 0x72, 0x75, 0x6c, 0x65, 0x73, 0x29, 0x20, 0x26, + 0x26, 0x20, 0x21, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x5f, 0x72, 0x65, 0x66, + 0x73, 0x42, 0x65, 0x69, 0x6e, 0x67, 0x52, 0x65, 0x73, 0x6f, 0x6c, 0x76, + 0x65, 0x64, 0x2e, 0x68, 0x61, 0x73, 0x28, 0x72, 0x65, 0x66, 0x29, 0x29, + 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x74, 0x68, 0x69, + 0x73, 0x2e, 0x5f, 0x72, 0x65, 0x66, 0x73, 0x42, 0x65, 0x69, 0x6e, 0x67, + 0x52, 0x65, 0x73, 0x6f, 0x6c, 0x76, 0x65, 0x64, 0x2e, 0x61, 0x64, 0x64, + 0x28, 0x72, 0x65, 0x66, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x72, 0x65, 0x73, 0x6f, 0x6c, + 0x76, 0x65, 0x64, 0x20, 0x3d, 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x5f, + 0x72, 0x65, 0x66, 0x73, 0x5b, 0x72, 0x65, 0x66, 0x5d, 0x3b, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, 0x66, 0x4e, 0x61, 0x6d, 0x65, + 0x20, 0x3d, 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x76, 0x69, 0x73, 0x69, + 0x74, 0x28, 0x72, 0x65, 0x73, 0x6f, 0x6c, 0x76, 0x65, 0x64, 0x2c, 0x20, + 0x72, 0x65, 0x66, 0x4e, 0x61, 0x6d, 0x65, 0x29, 0x3b, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x5f, 0x72, 0x65, + 0x66, 0x73, 0x42, 0x65, 0x69, 0x6e, 0x67, 0x52, 0x65, 0x73, 0x6f, 0x6c, + 0x76, 0x65, 0x64, 0x2e, 0x64, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x28, 0x72, + 0x65, 0x66, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x72, 0x65, + 0x66, 0x4e, 0x61, 0x6d, 0x65, 0x3b, 0x0a, 0x20, 0x20, 0x7d, 0x0a, 0x0a, + 0x20, 0x20, 0x5f, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x43, + 0x6f, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x74, 0x52, 0x75, 0x6c, 0x65, 0x28, + 0x76, 0x61, 0x6c, 0x75, 0x65, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x74, 0x68, 0x69, 0x73, + 0x2e, 0x5f, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x4c, 0x69, 0x74, 0x65, + 0x72, 0x61, 0x6c, 0x28, 0x4a, 0x53, 0x4f, 0x4e, 0x2e, 0x73, 0x74, 0x72, + 0x69, 0x6e, 0x67, 0x69, 0x66, 0x79, 0x28, 0x76, 0x61, 0x6c, 0x75, 0x65, + 0x29, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x7d, 0x0a, 0x0a, 0x20, 0x20, 0x76, + 0x69, 0x73, 0x69, 0x74, 0x28, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2c, + 0x20, 0x6e, 0x61, 0x6d, 0x65, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x73, 0x63, 0x68, 0x65, 0x6d, + 0x61, 0x54, 0x79, 0x70, 0x65, 0x20, 0x3d, 0x20, 0x73, 0x63, 0x68, 0x65, + 0x6d, 0x61, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x3b, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x73, 0x63, 0x68, 0x65, 0x6d, + 0x61, 0x46, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x20, 0x3d, 0x20, 0x73, 0x63, + 0x68, 0x65, 0x6d, 0x61, 0x2e, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x3b, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x72, + 0x75, 0x6c, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x20, 0x3d, 0x20, 0x6e, 0x61, + 0x6d, 0x65, 0x20, 0x69, 0x6e, 0x20, 0x52, 0x45, 0x53, 0x45, 0x52, 0x56, + 0x45, 0x44, 0x5f, 0x4e, 0x41, 0x4d, 0x45, 0x53, 0x20, 0x3f, 0x20, 0x6e, + 0x61, 0x6d, 0x65, 0x20, 0x2b, 0x20, 0x27, 0x2d, 0x27, 0x20, 0x3a, 0x20, + 0x6e, 0x61, 0x6d, 0x65, 0x20, 0x3d, 0x3d, 0x20, 0x27, 0x27, 0x20, 0x3f, + 0x20, 0x27, 0x72, 0x6f, 0x6f, 0x74, 0x27, 0x20, 0x3a, 0x20, 0x6e, 0x61, + 0x6d, 0x65, 0x3b, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, + 0x73, 0x74, 0x20, 0x72, 0x65, 0x66, 0x20, 0x3d, 0x20, 0x73, 0x63, 0x68, + 0x65, 0x6d, 0x61, 0x2e, 0x24, 0x72, 0x65, 0x66, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, 0x72, 0x65, 0x66, 0x20, 0x21, 0x3d, 0x3d, 0x20, 0x75, 0x6e, 0x64, 0x65, 0x66, 0x69, 0x6e, 0x65, 0x64, 0x29, - 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x63, 0x6f, 0x6d, 0x70, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x20, - 0x3d, 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x5f, 0x72, 0x65, 0x66, 0x73, - 0x5b, 0x72, 0x65, 0x66, 0x5d, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, 0x27, 0x70, 0x72, 0x6f, 0x70, 0x65, - 0x72, 0x74, 0x69, 0x65, 0x73, 0x27, 0x20, 0x69, 0x6e, 0x20, 0x63, 0x6f, - 0x6d, 0x70, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x29, 0x20, 0x7b, 0x0a, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x66, 0x6f, - 0x72, 0x20, 0x28, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x5b, 0x70, 0x72, - 0x6f, 0x70, 0x4e, 0x61, 0x6d, 0x65, 0x2c, 0x20, 0x70, 0x72, 0x6f, 0x70, - 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x5d, 0x20, 0x6f, 0x66, 0x20, 0x4f, - 0x62, 0x6a, 0x65, 0x63, 0x74, 0x2e, 0x65, 0x6e, 0x74, 0x72, 0x69, 0x65, - 0x73, 0x28, 0x63, 0x6f, 0x6d, 0x70, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, - 0x2e, 0x70, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x29, - 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x70, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, - 0x65, 0x73, 0x2e, 0x70, 0x75, 0x73, 0x68, 0x28, 0x5b, 0x70, 0x72, 0x6f, - 0x70, 0x4e, 0x61, 0x6d, 0x65, 0x2c, 0x20, 0x70, 0x72, 0x6f, 0x70, 0x53, - 0x63, 0x68, 0x65, 0x6d, 0x61, 0x5d, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, - 0x28, 0x69, 0x73, 0x52, 0x65, 0x71, 0x75, 0x69, 0x72, 0x65, 0x64, 0x29, - 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, 0x71, 0x75, 0x69, 0x72, 0x65, - 0x64, 0x2e, 0x61, 0x64, 0x64, 0x28, 0x70, 0x72, 0x6f, 0x70, 0x4e, 0x61, - 0x6d, 0x65, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, - 0x3b, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x66, 0x6f, 0x72, - 0x20, 0x28, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x74, 0x20, 0x6f, 0x66, - 0x20, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2e, 0x61, 0x6c, 0x6c, 0x4f, - 0x66, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x69, 0x66, 0x20, 0x28, 0x27, 0x61, 0x6e, 0x79, 0x4f, 0x66, 0x27, - 0x20, 0x69, 0x6e, 0x20, 0x74, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x66, 0x6f, 0x72, 0x20, 0x28, - 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x74, 0x74, 0x20, 0x6f, 0x66, 0x20, - 0x74, 0x2e, 0x61, 0x6e, 0x79, 0x4f, 0x66, 0x29, 0x20, 0x7b, 0x0a, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x61, - 0x64, 0x64, 0x43, 0x6f, 0x6d, 0x70, 0x6f, 0x6e, 0x65, 0x6e, 0x74, 0x28, - 0x74, 0x74, 0x2c, 0x20, 0x66, 0x61, 0x6c, 0x73, 0x65, 0x29, 0x3b, 0x0a, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x20, 0x65, 0x6c, - 0x73, 0x65, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x61, 0x64, 0x64, 0x43, 0x6f, 0x6d, 0x70, 0x6f, 0x6e, - 0x65, 0x6e, 0x74, 0x28, 0x74, 0x2c, 0x20, 0x74, 0x72, 0x75, 0x65, 0x29, - 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x0a, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x74, 0x68, - 0x69, 0x73, 0x2e, 0x5f, 0x61, 0x64, 0x64, 0x52, 0x75, 0x6c, 0x65, 0x28, - 0x72, 0x75, 0x6c, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x2c, 0x20, 0x74, 0x68, - 0x69, 0x73, 0x2e, 0x5f, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x4f, 0x62, 0x6a, - 0x65, 0x63, 0x74, 0x52, 0x75, 0x6c, 0x65, 0x28, 0x70, 0x72, 0x6f, 0x70, - 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x2c, 0x20, 0x72, 0x65, 0x71, 0x75, - 0x69, 0x72, 0x65, 0x64, 0x2c, 0x20, 0x6e, 0x61, 0x6d, 0x65, 0x2c, 0x20, - 0x2f, 0x2a, 0x20, 0x61, 0x64, 0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x61, - 0x6c, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x3d, - 0x20, 0x2a, 0x2f, 0x20, 0x66, 0x61, 0x6c, 0x73, 0x65, 0x29, 0x29, 0x3b, - 0x0a, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x20, 0x65, 0x6c, 0x73, 0x65, 0x20, - 0x69, 0x66, 0x20, 0x28, 0x28, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x54, - 0x79, 0x70, 0x65, 0x20, 0x3d, 0x3d, 0x3d, 0x20, 0x75, 0x6e, 0x64, 0x65, - 0x66, 0x69, 0x6e, 0x65, 0x64, 0x20, 0x7c, 0x7c, 0x20, 0x73, 0x63, 0x68, - 0x65, 0x6d, 0x61, 0x54, 0x79, 0x70, 0x65, 0x20, 0x3d, 0x3d, 0x3d, 0x20, - 0x27, 0x61, 0x72, 0x72, 0x61, 0x79, 0x27, 0x29, 0x20, 0x26, 0x26, 0x20, - 0x28, 0x27, 0x69, 0x74, 0x65, 0x6d, 0x73, 0x27, 0x20, 0x69, 0x6e, 0x20, - 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x20, 0x7c, 0x7c, 0x20, 0x27, 0x70, - 0x72, 0x65, 0x66, 0x69, 0x78, 0x49, 0x74, 0x65, 0x6d, 0x73, 0x27, 0x20, - 0x69, 0x6e, 0x20, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x29, 0x29, 0x20, - 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, - 0x74, 0x20, 0x69, 0x74, 0x65, 0x6d, 0x73, 0x20, 0x3d, 0x20, 0x73, 0x63, - 0x68, 0x65, 0x6d, 0x61, 0x2e, 0x69, 0x74, 0x65, 0x6d, 0x73, 0x20, 0x3f, - 0x3f, 0x20, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2e, 0x70, 0x72, 0x65, - 0x66, 0x69, 0x78, 0x49, 0x74, 0x65, 0x6d, 0x73, 0x3b, 0x0a, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, 0x41, 0x72, 0x72, 0x61, - 0x79, 0x2e, 0x69, 0x73, 0x41, 0x72, 0x72, 0x61, 0x79, 0x28, 0x69, 0x74, - 0x65, 0x6d, 0x73, 0x29, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x74, - 0x68, 0x69, 0x73, 0x2e, 0x5f, 0x61, 0x64, 0x64, 0x52, 0x75, 0x6c, 0x65, - 0x28, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x72, 0x75, 0x6c, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x2c, 0x0a, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x27, 0x22, 0x5b, 0x22, - 0x20, 0x73, 0x70, 0x61, 0x63, 0x65, 0x20, 0x27, 0x20, 0x2b, 0x0a, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, - 0x74, 0x65, 0x6d, 0x73, 0x2e, 0x6d, 0x61, 0x70, 0x28, 0x28, 0x69, 0x74, - 0x65, 0x6d, 0x2c, 0x20, 0x69, 0x29, 0x20, 0x3d, 0x3e, 0x20, 0x74, 0x68, - 0x69, 0x73, 0x2e, 0x76, 0x69, 0x73, 0x69, 0x74, 0x28, 0x69, 0x74, 0x65, - 0x6d, 0x2c, 0x20, 0x60, 0x24, 0x7b, 0x6e, 0x61, 0x6d, 0x65, 0x20, 0x3f, - 0x3f, 0x20, 0x27, 0x27, 0x7d, 0x24, 0x7b, 0x6e, 0x61, 0x6d, 0x65, 0x20, - 0x3f, 0x20, 0x27, 0x2d, 0x27, 0x20, 0x3a, 0x20, 0x27, 0x27, 0x7d, 0x74, - 0x75, 0x70, 0x6c, 0x65, 0x2d, 0x24, 0x7b, 0x69, 0x7d, 0x60, 0x29, 0x29, - 0x2e, 0x6a, 0x6f, 0x69, 0x6e, 0x28, 0x27, 0x20, 0x22, 0x2c, 0x22, 0x20, - 0x73, 0x70, 0x61, 0x63, 0x65, 0x20, 0x27, 0x29, 0x20, 0x2b, 0x0a, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x27, - 0x20, 0x22, 0x5d, 0x22, 0x20, 0x73, 0x70, 0x61, 0x63, 0x65, 0x27, 0x0a, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x29, 0x3b, 0x0a, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x20, 0x65, 0x6c, 0x73, 0x65, 0x20, - 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, - 0x6e, 0x73, 0x74, 0x20, 0x69, 0x74, 0x65, 0x6d, 0x52, 0x75, 0x6c, 0x65, - 0x4e, 0x61, 0x6d, 0x65, 0x20, 0x3d, 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, - 0x76, 0x69, 0x73, 0x69, 0x74, 0x28, 0x69, 0x74, 0x65, 0x6d, 0x73, 0x2c, - 0x20, 0x60, 0x24, 0x7b, 0x6e, 0x61, 0x6d, 0x65, 0x20, 0x3f, 0x3f, 0x20, - 0x27, 0x27, 0x7d, 0x24, 0x7b, 0x6e, 0x61, 0x6d, 0x65, 0x20, 0x3f, 0x20, - 0x27, 0x2d, 0x27, 0x20, 0x3a, 0x20, 0x27, 0x27, 0x7d, 0x69, 0x74, 0x65, - 0x6d, 0x60, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x6c, 0x69, 0x73, 0x74, 0x49, - 0x74, 0x65, 0x6d, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x6f, 0x72, 0x20, - 0x3d, 0x20, 0x60, 0x28, 0x20, 0x22, 0x2c, 0x22, 0x20, 0x73, 0x70, 0x61, - 0x63, 0x65, 0x20, 0x24, 0x7b, 0x69, 0x74, 0x65, 0x6d, 0x52, 0x75, 0x6c, - 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x7d, 0x20, 0x29, 0x60, 0x3b, 0x0a, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x6c, 0x65, 0x74, 0x20, 0x73, - 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x69, 0x76, 0x65, 0x49, 0x74, 0x65, - 0x6d, 0x73, 0x20, 0x3d, 0x20, 0x27, 0x27, 0x3b, 0x0a, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x6c, 0x65, 0x74, 0x20, 0x6d, 0x69, 0x6e, - 0x49, 0x74, 0x65, 0x6d, 0x73, 0x20, 0x3d, 0x20, 0x73, 0x63, 0x68, 0x65, - 0x6d, 0x61, 0x2e, 0x6d, 0x69, 0x6e, 0x49, 0x74, 0x65, 0x6d, 0x73, 0x20, - 0x7c, 0x7c, 0x20, 0x30, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x6d, 0x61, 0x78, 0x49, - 0x74, 0x65, 0x6d, 0x73, 0x20, 0x3d, 0x20, 0x73, 0x63, 0x68, 0x65, 0x6d, - 0x61, 0x2e, 0x6d, 0x61, 0x78, 0x49, 0x74, 0x65, 0x6d, 0x73, 0x3b, 0x0a, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, - 0x6d, 0x69, 0x6e, 0x49, 0x74, 0x65, 0x6d, 0x73, 0x20, 0x3e, 0x20, 0x30, - 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x73, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x69, 0x76, 0x65, - 0x49, 0x74, 0x65, 0x6d, 0x73, 0x20, 0x3d, 0x20, 0x6c, 0x69, 0x73, 0x74, - 0x49, 0x74, 0x65, 0x6d, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x6f, 0x72, - 0x2e, 0x72, 0x65, 0x70, 0x65, 0x61, 0x74, 0x28, 0x6d, 0x69, 0x6e, 0x49, - 0x74, 0x65, 0x6d, 0x73, 0x20, 0x2d, 0x20, 0x31, 0x29, 0x3b, 0x0a, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x6d, 0x69, 0x6e, - 0x49, 0x74, 0x65, 0x6d, 0x73, 0x2d, 0x2d, 0x3b, 0x0a, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, 0x6d, 0x61, 0x78, 0x49, 0x74, - 0x65, 0x6d, 0x73, 0x20, 0x21, 0x3d, 0x3d, 0x20, 0x75, 0x6e, 0x64, 0x65, - 0x66, 0x69, 0x6e, 0x65, 0x64, 0x20, 0x26, 0x26, 0x20, 0x6d, 0x61, 0x78, - 0x49, 0x74, 0x65, 0x6d, 0x73, 0x20, 0x3e, 0x20, 0x6d, 0x69, 0x6e, 0x49, - 0x74, 0x65, 0x6d, 0x73, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x73, 0x75, 0x63, 0x63, 0x65, 0x73, - 0x73, 0x69, 0x76, 0x65, 0x49, 0x74, 0x65, 0x6d, 0x73, 0x20, 0x2b, 0x3d, - 0x20, 0x60, 0x24, 0x7b, 0x6c, 0x69, 0x73, 0x74, 0x49, 0x74, 0x65, 0x6d, - 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x6f, 0x72, 0x7d, 0x3f, 0x60, 0x2e, - 0x72, 0x65, 0x70, 0x65, 0x61, 0x74, 0x28, 0x6d, 0x61, 0x78, 0x49, 0x74, - 0x65, 0x6d, 0x73, 0x20, 0x2d, 0x20, 0x6d, 0x69, 0x6e, 0x49, 0x74, 0x65, - 0x6d, 0x73, 0x20, 0x2d, 0x20, 0x31, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x20, 0x65, 0x6c, 0x73, 0x65, 0x20, + 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, 0x74, + 0x75, 0x72, 0x6e, 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x5f, 0x61, 0x64, + 0x64, 0x52, 0x75, 0x6c, 0x65, 0x28, 0x72, 0x75, 0x6c, 0x65, 0x4e, 0x61, + 0x6d, 0x65, 0x2c, 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x5f, 0x72, 0x65, + 0x73, 0x6f, 0x6c, 0x76, 0x65, 0x52, 0x65, 0x66, 0x28, 0x72, 0x65, 0x66, + 0x29, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x20, 0x65, 0x6c, + 0x73, 0x65, 0x20, 0x69, 0x66, 0x20, 0x28, 0x73, 0x63, 0x68, 0x65, 0x6d, + 0x61, 0x2e, 0x6f, 0x6e, 0x65, 0x4f, 0x66, 0x20, 0x7c, 0x7c, 0x20, 0x73, + 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2e, 0x61, 0x6e, 0x79, 0x4f, 0x66, 0x29, + 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, 0x74, + 0x75, 0x72, 0x6e, 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x5f, 0x61, 0x64, + 0x64, 0x52, 0x75, 0x6c, 0x65, 0x28, 0x72, 0x75, 0x6c, 0x65, 0x4e, 0x61, + 0x6d, 0x65, 0x2c, 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x5f, 0x67, 0x65, + 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x55, 0x6e, 0x69, 0x6f, 0x6e, 0x52, + 0x75, 0x6c, 0x65, 0x28, 0x6e, 0x61, 0x6d, 0x65, 0x2c, 0x20, 0x73, 0x63, + 0x68, 0x65, 0x6d, 0x61, 0x2e, 0x6f, 0x6e, 0x65, 0x4f, 0x66, 0x20, 0x7c, + 0x7c, 0x20, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2e, 0x61, 0x6e, 0x79, + 0x4f, 0x66, 0x29, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x20, + 0x65, 0x6c, 0x73, 0x65, 0x20, 0x69, 0x66, 0x20, 0x28, 0x41, 0x72, 0x72, + 0x61, 0x79, 0x2e, 0x69, 0x73, 0x41, 0x72, 0x72, 0x61, 0x79, 0x28, 0x73, + 0x63, 0x68, 0x65, 0x6d, 0x61, 0x54, 0x79, 0x70, 0x65, 0x29, 0x29, 0x20, + 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, + 0x72, 0x6e, 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x5f, 0x61, 0x64, 0x64, + 0x52, 0x75, 0x6c, 0x65, 0x28, 0x72, 0x75, 0x6c, 0x65, 0x4e, 0x61, 0x6d, + 0x65, 0x2c, 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x5f, 0x67, 0x65, 0x6e, + 0x65, 0x72, 0x61, 0x74, 0x65, 0x55, 0x6e, 0x69, 0x6f, 0x6e, 0x52, 0x75, + 0x6c, 0x65, 0x28, 0x6e, 0x61, 0x6d, 0x65, 0x2c, 0x20, 0x73, 0x63, 0x68, + 0x65, 0x6d, 0x61, 0x54, 0x79, 0x70, 0x65, 0x2e, 0x6d, 0x61, 0x70, 0x28, + 0x74, 0x20, 0x3d, 0x3e, 0x20, 0x28, 0x7b, 0x20, 0x74, 0x79, 0x70, 0x65, + 0x3a, 0x20, 0x74, 0x20, 0x7d, 0x29, 0x29, 0x29, 0x29, 0x3b, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x7d, 0x20, 0x65, 0x6c, 0x73, 0x65, 0x20, 0x69, 0x66, + 0x20, 0x28, 0x27, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x27, 0x20, 0x69, 0x6e, + 0x20, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x29, 0x20, 0x7b, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, + 0x74, 0x68, 0x69, 0x73, 0x2e, 0x5f, 0x61, 0x64, 0x64, 0x52, 0x75, 0x6c, + 0x65, 0x28, 0x72, 0x75, 0x6c, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x2c, 0x20, + 0x74, 0x68, 0x69, 0x73, 0x2e, 0x5f, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, + 0x74, 0x65, 0x43, 0x6f, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x74, 0x52, 0x75, + 0x6c, 0x65, 0x28, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2e, 0x63, 0x6f, + 0x6e, 0x73, 0x74, 0x29, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x7d, + 0x20, 0x65, 0x6c, 0x73, 0x65, 0x20, 0x69, 0x66, 0x20, 0x28, 0x27, 0x65, + 0x6e, 0x75, 0x6d, 0x27, 0x20, 0x69, 0x6e, 0x20, 0x73, 0x63, 0x68, 0x65, + 0x6d, 0x61, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x72, 0x75, 0x6c, 0x65, 0x20, 0x3d, + 0x20, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2e, 0x65, 0x6e, 0x75, 0x6d, + 0x2e, 0x6d, 0x61, 0x70, 0x28, 0x76, 0x20, 0x3d, 0x3e, 0x20, 0x74, 0x68, + 0x69, 0x73, 0x2e, 0x5f, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, + 0x43, 0x6f, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x74, 0x52, 0x75, 0x6c, 0x65, + 0x28, 0x76, 0x29, 0x29, 0x2e, 0x6a, 0x6f, 0x69, 0x6e, 0x28, 0x27, 0x20, + 0x7c, 0x20, 0x27, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, + 0x5f, 0x61, 0x64, 0x64, 0x52, 0x75, 0x6c, 0x65, 0x28, 0x72, 0x75, 0x6c, + 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x2c, 0x20, 0x72, 0x75, 0x6c, 0x65, 0x29, + 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x20, 0x65, 0x6c, 0x73, 0x65, + 0x20, 0x69, 0x66, 0x20, 0x28, 0x28, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, + 0x54, 0x79, 0x70, 0x65, 0x20, 0x3d, 0x3d, 0x3d, 0x20, 0x75, 0x6e, 0x64, + 0x65, 0x66, 0x69, 0x6e, 0x65, 0x64, 0x20, 0x7c, 0x7c, 0x20, 0x73, 0x63, + 0x68, 0x65, 0x6d, 0x61, 0x54, 0x79, 0x70, 0x65, 0x20, 0x3d, 0x3d, 0x3d, + 0x20, 0x27, 0x6f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x27, 0x29, 0x20, 0x26, + 0x26, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x28, 0x27, 0x70, 0x72, 0x6f, 0x70, 0x65, + 0x72, 0x74, 0x69, 0x65, 0x73, 0x27, 0x20, 0x69, 0x6e, 0x20, 0x73, 0x63, + 0x68, 0x65, 0x6d, 0x61, 0x20, 0x7c, 0x7c, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x28, 0x27, 0x61, 0x64, 0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x61, 0x6c, + 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x27, 0x20, + 0x69, 0x6e, 0x20, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x20, 0x26, 0x26, + 0x20, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2e, 0x61, 0x64, 0x64, 0x69, + 0x74, 0x69, 0x6f, 0x6e, 0x61, 0x6c, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, + 0x74, 0x69, 0x65, 0x73, 0x20, 0x21, 0x3d, 0x3d, 0x20, 0x74, 0x72, 0x75, + 0x65, 0x29, 0x29, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x72, 0x65, 0x71, 0x75, 0x69, + 0x72, 0x65, 0x64, 0x20, 0x3d, 0x20, 0x6e, 0x65, 0x77, 0x20, 0x53, 0x65, + 0x74, 0x28, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2e, 0x72, 0x65, 0x71, + 0x75, 0x69, 0x72, 0x65, 0x64, 0x20, 0x7c, 0x7c, 0x20, 0x5b, 0x5d, 0x29, + 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, + 0x74, 0x20, 0x70, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, + 0x20, 0x3d, 0x20, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x2e, 0x65, 0x6e, + 0x74, 0x72, 0x69, 0x65, 0x73, 0x28, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, + 0x2e, 0x70, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x20, + 0x3f, 0x3f, 0x20, 0x7b, 0x7d, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x74, 0x68, 0x69, + 0x73, 0x2e, 0x5f, 0x61, 0x64, 0x64, 0x52, 0x75, 0x6c, 0x65, 0x28, 0x72, + 0x75, 0x6c, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x2c, 0x20, 0x74, 0x68, 0x69, + 0x73, 0x2e, 0x5f, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x4f, 0x62, 0x6a, 0x65, + 0x63, 0x74, 0x52, 0x75, 0x6c, 0x65, 0x28, 0x70, 0x72, 0x6f, 0x70, 0x65, + 0x72, 0x74, 0x69, 0x65, 0x73, 0x2c, 0x20, 0x72, 0x65, 0x71, 0x75, 0x69, + 0x72, 0x65, 0x64, 0x2c, 0x20, 0x6e, 0x61, 0x6d, 0x65, 0x2c, 0x20, 0x73, + 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2e, 0x61, 0x64, 0x64, 0x69, 0x74, 0x69, + 0x6f, 0x6e, 0x61, 0x6c, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, + 0x65, 0x73, 0x29, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x20, + 0x65, 0x6c, 0x73, 0x65, 0x20, 0x69, 0x66, 0x20, 0x28, 0x28, 0x73, 0x63, + 0x68, 0x65, 0x6d, 0x61, 0x54, 0x79, 0x70, 0x65, 0x20, 0x3d, 0x3d, 0x3d, + 0x20, 0x75, 0x6e, 0x64, 0x65, 0x66, 0x69, 0x6e, 0x65, 0x64, 0x20, 0x7c, + 0x7c, 0x20, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x54, 0x79, 0x70, 0x65, + 0x20, 0x3d, 0x3d, 0x3d, 0x20, 0x27, 0x6f, 0x62, 0x6a, 0x65, 0x63, 0x74, + 0x27, 0x29, 0x20, 0x26, 0x26, 0x20, 0x27, 0x61, 0x6c, 0x6c, 0x4f, 0x66, + 0x27, 0x20, 0x69, 0x6e, 0x20, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x29, + 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, + 0x73, 0x74, 0x20, 0x72, 0x65, 0x71, 0x75, 0x69, 0x72, 0x65, 0x64, 0x20, + 0x3d, 0x20, 0x6e, 0x65, 0x77, 0x20, 0x53, 0x65, 0x74, 0x28, 0x29, 0x3b, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, + 0x20, 0x70, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x20, + 0x3d, 0x20, 0x5b, 0x5d, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x61, 0x64, 0x64, 0x43, 0x6f, 0x6d, + 0x70, 0x6f, 0x6e, 0x65, 0x6e, 0x74, 0x20, 0x3d, 0x20, 0x28, 0x63, 0x6f, + 0x6d, 0x70, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2c, 0x20, 0x69, 0x73, + 0x52, 0x65, 0x71, 0x75, 0x69, 0x72, 0x65, 0x64, 0x29, 0x20, 0x3d, 0x3e, + 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, + 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x72, 0x65, 0x66, 0x20, 0x3d, 0x20, 0x63, + 0x6f, 0x6d, 0x70, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2e, 0x24, 0x72, + 0x65, 0x66, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x69, 0x66, 0x20, 0x28, 0x72, 0x65, 0x66, 0x20, 0x21, 0x3d, 0x3d, 0x20, + 0x75, 0x6e, 0x64, 0x65, 0x66, 0x69, 0x6e, 0x65, 0x64, 0x29, 0x20, 0x7b, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, + 0x6f, 0x6d, 0x70, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x20, 0x3d, 0x20, + 0x74, 0x68, 0x69, 0x73, 0x2e, 0x5f, 0x72, 0x65, 0x66, 0x73, 0x5b, 0x72, + 0x65, 0x66, 0x5d, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x7d, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x69, 0x66, 0x20, 0x28, 0x27, 0x70, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, + 0x69, 0x65, 0x73, 0x27, 0x20, 0x69, 0x6e, 0x20, 0x63, 0x6f, 0x6d, 0x70, + 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x66, 0x6f, 0x72, 0x20, + 0x28, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x5b, 0x70, 0x72, 0x6f, 0x70, + 0x4e, 0x61, 0x6d, 0x65, 0x2c, 0x20, 0x70, 0x72, 0x6f, 0x70, 0x53, 0x63, + 0x68, 0x65, 0x6d, 0x61, 0x5d, 0x20, 0x6f, 0x66, 0x20, 0x4f, 0x62, 0x6a, + 0x65, 0x63, 0x74, 0x2e, 0x65, 0x6e, 0x74, 0x72, 0x69, 0x65, 0x73, 0x28, + 0x63, 0x6f, 0x6d, 0x70, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2e, 0x70, + 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x29, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x73, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x69, 0x76, 0x65, 0x49, 0x74, - 0x65, 0x6d, 0x73, 0x20, 0x2b, 0x3d, 0x20, 0x60, 0x24, 0x7b, 0x6c, 0x69, - 0x73, 0x74, 0x49, 0x74, 0x65, 0x6d, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, - 0x6f, 0x72, 0x7d, 0x2a, 0x60, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x70, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, + 0x2e, 0x70, 0x75, 0x73, 0x68, 0x28, 0x5b, 0x70, 0x72, 0x6f, 0x70, 0x4e, + 0x61, 0x6d, 0x65, 0x2c, 0x20, 0x70, 0x72, 0x6f, 0x70, 0x53, 0x63, 0x68, + 0x65, 0x6d, 0x61, 0x5d, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, 0x69, + 0x73, 0x52, 0x65, 0x71, 0x75, 0x69, 0x72, 0x65, 0x64, 0x29, 0x20, 0x7b, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x72, 0x65, 0x71, 0x75, 0x69, 0x72, 0x65, 0x64, 0x2e, + 0x61, 0x64, 0x64, 0x28, 0x70, 0x72, 0x6f, 0x70, 0x4e, 0x61, 0x6d, 0x65, + 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x72, 0x75, 0x6c, 0x65, 0x20, - 0x3d, 0x20, 0x6d, 0x69, 0x6e, 0x49, 0x74, 0x65, 0x6d, 0x73, 0x20, 0x3d, - 0x3d, 0x3d, 0x20, 0x30, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x3f, 0x20, 0x60, 0x22, 0x5b, 0x22, 0x20, 0x73, 0x70, - 0x61, 0x63, 0x65, 0x20, 0x28, 0x20, 0x24, 0x7b, 0x69, 0x74, 0x65, 0x6d, - 0x52, 0x75, 0x6c, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x7d, 0x20, 0x24, 0x7b, - 0x73, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x69, 0x76, 0x65, 0x49, 0x74, - 0x65, 0x6d, 0x73, 0x7d, 0x20, 0x29, 0x3f, 0x20, 0x22, 0x5d, 0x22, 0x20, - 0x73, 0x70, 0x61, 0x63, 0x65, 0x60, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x3a, 0x20, 0x60, 0x22, 0x5b, 0x22, 0x20, - 0x73, 0x70, 0x61, 0x63, 0x65, 0x20, 0x24, 0x7b, 0x69, 0x74, 0x65, 0x6d, - 0x52, 0x75, 0x6c, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x7d, 0x20, 0x24, 0x7b, - 0x73, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x69, 0x76, 0x65, 0x49, 0x74, - 0x65, 0x6d, 0x73, 0x7d, 0x20, 0x22, 0x5d, 0x22, 0x20, 0x73, 0x70, 0x61, - 0x63, 0x65, 0x60, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x3b, 0x0a, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x66, 0x6f, 0x72, 0x20, 0x28, + 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x74, 0x20, 0x6f, 0x66, 0x20, 0x73, + 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2e, 0x61, 0x6c, 0x6c, 0x4f, 0x66, 0x29, + 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, + 0x66, 0x20, 0x28, 0x27, 0x61, 0x6e, 0x79, 0x4f, 0x66, 0x27, 0x20, 0x69, + 0x6e, 0x20, 0x74, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x66, 0x6f, 0x72, 0x20, 0x28, 0x63, 0x6f, + 0x6e, 0x73, 0x74, 0x20, 0x74, 0x74, 0x20, 0x6f, 0x66, 0x20, 0x74, 0x2e, + 0x61, 0x6e, 0x79, 0x4f, 0x66, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x61, 0x64, 0x64, + 0x43, 0x6f, 0x6d, 0x70, 0x6f, 0x6e, 0x65, 0x6e, 0x74, 0x28, 0x74, 0x74, + 0x2c, 0x20, 0x66, 0x61, 0x6c, 0x73, 0x65, 0x29, 0x3b, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x20, 0x65, 0x6c, 0x73, 0x65, + 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x61, 0x64, 0x64, 0x43, 0x6f, 0x6d, 0x70, 0x6f, 0x6e, 0x65, 0x6e, + 0x74, 0x28, 0x74, 0x2c, 0x20, 0x74, 0x72, 0x75, 0x65, 0x29, 0x3b, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x5f, 0x61, 0x64, 0x64, 0x52, 0x75, 0x6c, 0x65, 0x28, 0x72, 0x75, - 0x6c, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x2c, 0x20, 0x72, 0x75, 0x6c, 0x65, - 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, + 0x6c, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x2c, 0x20, 0x74, 0x68, 0x69, 0x73, + 0x2e, 0x5f, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x4f, 0x62, 0x6a, 0x65, 0x63, + 0x74, 0x52, 0x75, 0x6c, 0x65, 0x28, 0x70, 0x72, 0x6f, 0x70, 0x65, 0x72, + 0x74, 0x69, 0x65, 0x73, 0x2c, 0x20, 0x72, 0x65, 0x71, 0x75, 0x69, 0x72, + 0x65, 0x64, 0x2c, 0x20, 0x6e, 0x61, 0x6d, 0x65, 0x2c, 0x20, 0x2f, 0x2a, + 0x20, 0x61, 0x64, 0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x61, 0x6c, 0x50, + 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x3d, 0x20, 0x2a, + 0x2f, 0x20, 0x66, 0x61, 0x6c, 0x73, 0x65, 0x29, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x20, 0x65, 0x6c, 0x73, 0x65, 0x20, 0x69, 0x66, 0x20, 0x28, 0x28, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x54, 0x79, 0x70, 0x65, 0x20, 0x3d, 0x3d, 0x3d, 0x20, 0x75, 0x6e, 0x64, 0x65, 0x66, 0x69, 0x6e, 0x65, 0x64, 0x20, 0x7c, 0x7c, 0x20, 0x73, 0x63, 0x68, 0x65, 0x6d, - 0x61, 0x54, 0x79, 0x70, 0x65, 0x20, 0x3d, 0x3d, 0x3d, 0x20, 0x27, 0x73, - 0x74, 0x72, 0x69, 0x6e, 0x67, 0x27, 0x29, 0x20, 0x26, 0x26, 0x20, 0x27, - 0x70, 0x61, 0x74, 0x74, 0x65, 0x72, 0x6e, 0x27, 0x20, 0x69, 0x6e, 0x20, - 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x74, - 0x68, 0x69, 0x73, 0x2e, 0x5f, 0x76, 0x69, 0x73, 0x69, 0x74, 0x50, 0x61, - 0x74, 0x74, 0x65, 0x72, 0x6e, 0x28, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, - 0x2e, 0x70, 0x61, 0x74, 0x74, 0x65, 0x72, 0x6e, 0x2c, 0x20, 0x72, 0x75, - 0x6c, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, + 0x61, 0x54, 0x79, 0x70, 0x65, 0x20, 0x3d, 0x3d, 0x3d, 0x20, 0x27, 0x61, + 0x72, 0x72, 0x61, 0x79, 0x27, 0x29, 0x20, 0x26, 0x26, 0x20, 0x28, 0x27, + 0x69, 0x74, 0x65, 0x6d, 0x73, 0x27, 0x20, 0x69, 0x6e, 0x20, 0x73, 0x63, + 0x68, 0x65, 0x6d, 0x61, 0x20, 0x7c, 0x7c, 0x20, 0x27, 0x70, 0x72, 0x65, + 0x66, 0x69, 0x78, 0x49, 0x74, 0x65, 0x6d, 0x73, 0x27, 0x20, 0x69, 0x6e, + 0x20, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x29, 0x29, 0x20, 0x7b, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, + 0x69, 0x74, 0x65, 0x6d, 0x73, 0x20, 0x3d, 0x20, 0x73, 0x63, 0x68, 0x65, + 0x6d, 0x61, 0x2e, 0x69, 0x74, 0x65, 0x6d, 0x73, 0x20, 0x3f, 0x3f, 0x20, + 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2e, 0x70, 0x72, 0x65, 0x66, 0x69, + 0x78, 0x49, 0x74, 0x65, 0x6d, 0x73, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, 0x41, 0x72, 0x72, 0x61, 0x79, 0x2e, + 0x69, 0x73, 0x41, 0x72, 0x72, 0x61, 0x79, 0x28, 0x69, 0x74, 0x65, 0x6d, + 0x73, 0x29, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x74, 0x68, 0x69, + 0x73, 0x2e, 0x5f, 0x61, 0x64, 0x64, 0x52, 0x75, 0x6c, 0x65, 0x28, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, 0x75, + 0x6c, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x2c, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x27, 0x22, 0x5b, 0x22, 0x20, 0x73, + 0x70, 0x61, 0x63, 0x65, 0x20, 0x27, 0x20, 0x2b, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x74, 0x65, + 0x6d, 0x73, 0x2e, 0x6d, 0x61, 0x70, 0x28, 0x28, 0x69, 0x74, 0x65, 0x6d, + 0x2c, 0x20, 0x69, 0x29, 0x20, 0x3d, 0x3e, 0x20, 0x74, 0x68, 0x69, 0x73, + 0x2e, 0x76, 0x69, 0x73, 0x69, 0x74, 0x28, 0x69, 0x74, 0x65, 0x6d, 0x2c, + 0x20, 0x60, 0x24, 0x7b, 0x6e, 0x61, 0x6d, 0x65, 0x20, 0x3f, 0x3f, 0x20, + 0x27, 0x27, 0x7d, 0x24, 0x7b, 0x6e, 0x61, 0x6d, 0x65, 0x20, 0x3f, 0x20, + 0x27, 0x2d, 0x27, 0x20, 0x3a, 0x20, 0x27, 0x27, 0x7d, 0x74, 0x75, 0x70, + 0x6c, 0x65, 0x2d, 0x24, 0x7b, 0x69, 0x7d, 0x60, 0x29, 0x29, 0x2e, 0x6a, + 0x6f, 0x69, 0x6e, 0x28, 0x27, 0x20, 0x22, 0x2c, 0x22, 0x20, 0x73, 0x70, + 0x61, 0x63, 0x65, 0x20, 0x27, 0x29, 0x20, 0x2b, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x27, 0x20, 0x22, + 0x5d, 0x22, 0x20, 0x73, 0x70, 0x61, 0x63, 0x65, 0x27, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x7d, 0x20, 0x65, 0x6c, 0x73, 0x65, 0x20, 0x7b, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, + 0x74, 0x20, 0x69, 0x74, 0x65, 0x6d, 0x52, 0x75, 0x6c, 0x65, 0x4e, 0x61, + 0x6d, 0x65, 0x20, 0x3d, 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x76, 0x69, + 0x73, 0x69, 0x74, 0x28, 0x69, 0x74, 0x65, 0x6d, 0x73, 0x2c, 0x20, 0x60, + 0x24, 0x7b, 0x6e, 0x61, 0x6d, 0x65, 0x20, 0x3f, 0x3f, 0x20, 0x27, 0x27, + 0x7d, 0x24, 0x7b, 0x6e, 0x61, 0x6d, 0x65, 0x20, 0x3f, 0x20, 0x27, 0x2d, + 0x27, 0x20, 0x3a, 0x20, 0x27, 0x27, 0x7d, 0x69, 0x74, 0x65, 0x6d, 0x60, + 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, + 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x6c, 0x69, 0x73, 0x74, 0x49, 0x74, 0x65, + 0x6d, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x6f, 0x72, 0x20, 0x3d, 0x20, + 0x60, 0x28, 0x20, 0x22, 0x2c, 0x22, 0x20, 0x73, 0x70, 0x61, 0x63, 0x65, + 0x20, 0x24, 0x7b, 0x69, 0x74, 0x65, 0x6d, 0x52, 0x75, 0x6c, 0x65, 0x4e, + 0x61, 0x6d, 0x65, 0x7d, 0x20, 0x29, 0x60, 0x3b, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x6c, 0x65, 0x74, 0x20, 0x73, 0x75, 0x63, + 0x63, 0x65, 0x73, 0x73, 0x69, 0x76, 0x65, 0x49, 0x74, 0x65, 0x6d, 0x73, + 0x20, 0x3d, 0x20, 0x27, 0x27, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x6c, 0x65, 0x74, 0x20, 0x6d, 0x69, 0x6e, 0x49, 0x74, + 0x65, 0x6d, 0x73, 0x20, 0x3d, 0x20, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, + 0x2e, 0x6d, 0x69, 0x6e, 0x49, 0x74, 0x65, 0x6d, 0x73, 0x20, 0x7c, 0x7c, + 0x20, 0x30, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x6d, 0x61, 0x78, 0x49, 0x74, 0x65, + 0x6d, 0x73, 0x20, 0x3d, 0x20, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2e, + 0x6d, 0x61, 0x78, 0x49, 0x74, 0x65, 0x6d, 0x73, 0x3b, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, 0x6d, 0x69, + 0x6e, 0x49, 0x74, 0x65, 0x6d, 0x73, 0x20, 0x3e, 0x20, 0x30, 0x29, 0x20, + 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x73, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x69, 0x76, 0x65, 0x49, 0x74, + 0x65, 0x6d, 0x73, 0x20, 0x3d, 0x20, 0x6c, 0x69, 0x73, 0x74, 0x49, 0x74, + 0x65, 0x6d, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x6f, 0x72, 0x2e, 0x72, + 0x65, 0x70, 0x65, 0x61, 0x74, 0x28, 0x6d, 0x69, 0x6e, 0x49, 0x74, 0x65, + 0x6d, 0x73, 0x20, 0x2d, 0x20, 0x31, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x6d, 0x69, 0x6e, 0x49, 0x74, + 0x65, 0x6d, 0x73, 0x2d, 0x2d, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x69, 0x66, 0x20, 0x28, 0x6d, 0x61, 0x78, 0x49, 0x74, 0x65, 0x6d, + 0x73, 0x20, 0x21, 0x3d, 0x3d, 0x20, 0x75, 0x6e, 0x64, 0x65, 0x66, 0x69, + 0x6e, 0x65, 0x64, 0x20, 0x26, 0x26, 0x20, 0x6d, 0x61, 0x78, 0x49, 0x74, + 0x65, 0x6d, 0x73, 0x20, 0x3e, 0x20, 0x6d, 0x69, 0x6e, 0x49, 0x74, 0x65, + 0x6d, 0x73, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x73, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x69, + 0x76, 0x65, 0x49, 0x74, 0x65, 0x6d, 0x73, 0x20, 0x2b, 0x3d, 0x20, 0x60, + 0x24, 0x7b, 0x6c, 0x69, 0x73, 0x74, 0x49, 0x74, 0x65, 0x6d, 0x4f, 0x70, + 0x65, 0x72, 0x61, 0x74, 0x6f, 0x72, 0x7d, 0x3f, 0x60, 0x2e, 0x72, 0x65, + 0x70, 0x65, 0x61, 0x74, 0x28, 0x6d, 0x61, 0x78, 0x49, 0x74, 0x65, 0x6d, + 0x73, 0x20, 0x2d, 0x20, 0x6d, 0x69, 0x6e, 0x49, 0x74, 0x65, 0x6d, 0x73, + 0x20, 0x2d, 0x20, 0x31, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x7d, 0x20, 0x65, 0x6c, 0x73, 0x65, 0x20, 0x7b, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x73, 0x75, + 0x63, 0x63, 0x65, 0x73, 0x73, 0x69, 0x76, 0x65, 0x49, 0x74, 0x65, 0x6d, + 0x73, 0x20, 0x2b, 0x3d, 0x20, 0x60, 0x24, 0x7b, 0x6c, 0x69, 0x73, 0x74, + 0x49, 0x74, 0x65, 0x6d, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x6f, 0x72, + 0x7d, 0x2a, 0x60, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, + 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x72, 0x75, 0x6c, 0x65, 0x20, 0x3d, 0x20, + 0x6d, 0x69, 0x6e, 0x49, 0x74, 0x65, 0x6d, 0x73, 0x20, 0x3d, 0x3d, 0x3d, + 0x20, 0x30, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x3f, 0x20, 0x60, 0x22, 0x5b, 0x22, 0x20, 0x73, 0x70, 0x61, 0x63, + 0x65, 0x20, 0x28, 0x20, 0x24, 0x7b, 0x69, 0x74, 0x65, 0x6d, 0x52, 0x75, + 0x6c, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x7d, 0x20, 0x24, 0x7b, 0x73, 0x75, + 0x63, 0x63, 0x65, 0x73, 0x73, 0x69, 0x76, 0x65, 0x49, 0x74, 0x65, 0x6d, + 0x73, 0x7d, 0x20, 0x29, 0x3f, 0x20, 0x22, 0x5d, 0x22, 0x20, 0x73, 0x70, + 0x61, 0x63, 0x65, 0x60, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x3a, 0x20, 0x60, 0x22, 0x5b, 0x22, 0x20, 0x73, 0x70, + 0x61, 0x63, 0x65, 0x20, 0x24, 0x7b, 0x69, 0x74, 0x65, 0x6d, 0x52, 0x75, + 0x6c, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x7d, 0x20, 0x24, 0x7b, 0x73, 0x75, + 0x63, 0x63, 0x65, 0x73, 0x73, 0x69, 0x76, 0x65, 0x49, 0x74, 0x65, 0x6d, + 0x73, 0x7d, 0x20, 0x22, 0x5d, 0x22, 0x20, 0x73, 0x70, 0x61, 0x63, 0x65, + 0x60, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, + 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x5f, + 0x61, 0x64, 0x64, 0x52, 0x75, 0x6c, 0x65, 0x28, 0x72, 0x75, 0x6c, 0x65, + 0x4e, 0x61, 0x6d, 0x65, 0x2c, 0x20, 0x72, 0x75, 0x6c, 0x65, 0x29, 0x3b, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x20, 0x65, 0x6c, 0x73, 0x65, 0x20, 0x69, 0x66, 0x20, 0x28, 0x28, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x54, 0x79, 0x70, 0x65, 0x20, 0x3d, 0x3d, 0x3d, 0x20, 0x75, 0x6e, 0x64, 0x65, 0x66, 0x69, 0x6e, 0x65, 0x64, 0x20, 0x7c, 0x7c, 0x20, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x54, 0x79, 0x70, 0x65, 0x20, 0x3d, 0x3d, 0x3d, 0x20, 0x27, 0x73, 0x74, 0x72, - 0x69, 0x6e, 0x67, 0x27, 0x29, 0x20, 0x26, 0x26, 0x20, 0x2f, 0x5e, 0x75, - 0x75, 0x69, 0x64, 0x5b, 0x31, 0x2d, 0x35, 0x5d, 0x3f, 0x24, 0x2f, 0x2e, - 0x74, 0x65, 0x73, 0x74, 0x28, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2e, - 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x20, 0x7c, 0x7c, 0x20, 0x27, 0x27, - 0x29, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, - 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x5f, - 0x61, 0x64, 0x64, 0x52, 0x75, 0x6c, 0x65, 0x28, 0x0a, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, 0x75, 0x6c, 0x65, 0x4e, + 0x69, 0x6e, 0x67, 0x27, 0x29, 0x20, 0x26, 0x26, 0x20, 0x27, 0x70, 0x61, + 0x74, 0x74, 0x65, 0x72, 0x6e, 0x27, 0x20, 0x69, 0x6e, 0x20, 0x73, 0x63, + 0x68, 0x65, 0x6d, 0x61, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x74, 0x68, 0x69, + 0x73, 0x2e, 0x5f, 0x76, 0x69, 0x73, 0x69, 0x74, 0x50, 0x61, 0x74, 0x74, + 0x65, 0x72, 0x6e, 0x28, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2e, 0x70, + 0x61, 0x74, 0x74, 0x65, 0x72, 0x6e, 0x2c, 0x20, 0x72, 0x75, 0x6c, 0x65, + 0x4e, 0x61, 0x6d, 0x65, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x7d, + 0x20, 0x65, 0x6c, 0x73, 0x65, 0x20, 0x69, 0x66, 0x20, 0x28, 0x28, 0x73, + 0x63, 0x68, 0x65, 0x6d, 0x61, 0x54, 0x79, 0x70, 0x65, 0x20, 0x3d, 0x3d, + 0x3d, 0x20, 0x75, 0x6e, 0x64, 0x65, 0x66, 0x69, 0x6e, 0x65, 0x64, 0x20, + 0x7c, 0x7c, 0x20, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x54, 0x79, 0x70, + 0x65, 0x20, 0x3d, 0x3d, 0x3d, 0x20, 0x27, 0x73, 0x74, 0x72, 0x69, 0x6e, + 0x67, 0x27, 0x29, 0x20, 0x26, 0x26, 0x20, 0x2f, 0x5e, 0x75, 0x75, 0x69, + 0x64, 0x5b, 0x31, 0x2d, 0x35, 0x5d, 0x3f, 0x24, 0x2f, 0x2e, 0x74, 0x65, + 0x73, 0x74, 0x28, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2e, 0x66, 0x6f, + 0x72, 0x6d, 0x61, 0x74, 0x20, 0x7c, 0x7c, 0x20, 0x27, 0x27, 0x29, 0x29, + 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, 0x74, + 0x75, 0x72, 0x6e, 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x5f, 0x61, 0x64, + 0x64, 0x52, 0x75, 0x6c, 0x65, 0x28, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, 0x75, 0x6c, 0x65, 0x4e, 0x61, 0x6d, + 0x65, 0x20, 0x3d, 0x3d, 0x3d, 0x20, 0x27, 0x72, 0x6f, 0x6f, 0x74, 0x27, + 0x20, 0x3f, 0x20, 0x27, 0x72, 0x6f, 0x6f, 0x74, 0x27, 0x20, 0x3a, 0x20, + 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x46, 0x6f, 0x72, 0x6d, 0x61, 0x74, + 0x2c, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x50, 0x52, 0x49, 0x4d, 0x49, 0x54, 0x49, 0x56, 0x45, 0x5f, 0x52, 0x55, + 0x4c, 0x45, 0x53, 0x5b, 0x27, 0x75, 0x75, 0x69, 0x64, 0x27, 0x5d, 0x29, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x20, 0x65, 0x6c, 0x73, 0x65, 0x20, + 0x69, 0x66, 0x20, 0x28, 0x28, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x54, + 0x79, 0x70, 0x65, 0x20, 0x3d, 0x3d, 0x3d, 0x20, 0x75, 0x6e, 0x64, 0x65, + 0x66, 0x69, 0x6e, 0x65, 0x64, 0x20, 0x7c, 0x7c, 0x20, 0x73, 0x63, 0x68, + 0x65, 0x6d, 0x61, 0x54, 0x79, 0x70, 0x65, 0x20, 0x3d, 0x3d, 0x3d, 0x20, + 0x27, 0x73, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x27, 0x29, 0x20, 0x26, 0x26, + 0x20, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2e, 0x66, 0x6f, 0x72, 0x6d, + 0x61, 0x74, 0x20, 0x69, 0x6e, 0x20, 0x44, 0x41, 0x54, 0x45, 0x5f, 0x52, + 0x55, 0x4c, 0x45, 0x53, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x66, 0x6f, 0x72, 0x20, 0x28, 0x63, 0x6f, 0x6e, 0x73, 0x74, + 0x20, 0x5b, 0x74, 0x2c, 0x20, 0x72, 0x5d, 0x20, 0x6f, 0x66, 0x20, 0x4f, + 0x62, 0x6a, 0x65, 0x63, 0x74, 0x2e, 0x65, 0x6e, 0x74, 0x72, 0x69, 0x65, + 0x73, 0x28, 0x44, 0x41, 0x54, 0x45, 0x5f, 0x52, 0x55, 0x4c, 0x45, 0x53, + 0x29, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x5f, 0x61, 0x64, 0x64, 0x52, 0x75, + 0x6c, 0x65, 0x28, 0x74, 0x2c, 0x20, 0x72, 0x29, 0x3b, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x73, 0x63, 0x68, 0x65, 0x6d, + 0x61, 0x46, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x20, 0x2b, 0x20, 0x27, 0x2d, + 0x73, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x27, 0x3b, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x7d, 0x20, 0x65, 0x6c, 0x73, 0x65, 0x20, 0x69, 0x66, 0x20, 0x28, + 0x28, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x54, 0x79, 0x70, 0x65, 0x20, + 0x3d, 0x3d, 0x3d, 0x20, 0x27, 0x6f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x27, + 0x29, 0x20, 0x7c, 0x7c, 0x20, 0x28, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, + 0x2e, 0x6b, 0x65, 0x79, 0x73, 0x28, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, + 0x29, 0x2e, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x20, 0x3d, 0x3d, 0x3d, + 0x20, 0x30, 0x29, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x66, 0x6f, 0x72, 0x20, 0x28, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, + 0x6e, 0x20, 0x6f, 0x66, 0x20, 0x4f, 0x42, 0x4a, 0x45, 0x43, 0x54, 0x5f, + 0x52, 0x55, 0x4c, 0x45, 0x5f, 0x4e, 0x41, 0x4d, 0x45, 0x53, 0x29, 0x20, + 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x74, 0x68, + 0x69, 0x73, 0x2e, 0x5f, 0x61, 0x64, 0x64, 0x52, 0x75, 0x6c, 0x65, 0x28, + 0x6e, 0x2c, 0x20, 0x50, 0x52, 0x49, 0x4d, 0x49, 0x54, 0x49, 0x56, 0x45, + 0x5f, 0x52, 0x55, 0x4c, 0x45, 0x53, 0x5b, 0x6e, 0x5d, 0x29, 0x3b, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x74, 0x68, 0x69, + 0x73, 0x2e, 0x5f, 0x61, 0x64, 0x64, 0x52, 0x75, 0x6c, 0x65, 0x28, 0x72, + 0x75, 0x6c, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x2c, 0x20, 0x27, 0x6f, 0x62, + 0x6a, 0x65, 0x63, 0x74, 0x27, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x7d, 0x20, 0x65, 0x6c, 0x73, 0x65, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, 0x21, 0x28, 0x73, 0x63, 0x68, + 0x65, 0x6d, 0x61, 0x54, 0x79, 0x70, 0x65, 0x20, 0x69, 0x6e, 0x20, 0x50, + 0x52, 0x49, 0x4d, 0x49, 0x54, 0x49, 0x56, 0x45, 0x5f, 0x52, 0x55, 0x4c, + 0x45, 0x53, 0x29, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x74, 0x68, 0x72, 0x6f, 0x77, 0x20, 0x6e, 0x65, 0x77, + 0x20, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x28, 0x60, 0x55, 0x6e, 0x72, 0x65, + 0x63, 0x6f, 0x67, 0x6e, 0x69, 0x7a, 0x65, 0x64, 0x20, 0x73, 0x63, 0x68, + 0x65, 0x6d, 0x61, 0x3a, 0x20, 0x24, 0x7b, 0x4a, 0x53, 0x4f, 0x4e, 0x2e, + 0x73, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x69, 0x66, 0x79, 0x28, 0x73, 0x63, + 0x68, 0x65, 0x6d, 0x61, 0x29, 0x7d, 0x60, 0x29, 0x3b, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x2f, 0x2f, 0x20, 0x54, 0x4f, 0x44, 0x4f, 0x3a, 0x20, 0x73, 0x75, 0x70, + 0x70, 0x6f, 0x72, 0x74, 0x20, 0x6d, 0x69, 0x6e, 0x69, 0x6d, 0x75, 0x6d, + 0x2c, 0x20, 0x6d, 0x61, 0x78, 0x69, 0x6d, 0x75, 0x6d, 0x2c, 0x20, 0x65, + 0x78, 0x63, 0x6c, 0x75, 0x73, 0x69, 0x76, 0x65, 0x4d, 0x69, 0x6e, 0x69, + 0x6d, 0x75, 0x6d, 0x2c, 0x20, 0x65, 0x78, 0x63, 0x6c, 0x75, 0x73, 0x69, + 0x76, 0x65, 0x4d, 0x61, 0x78, 0x69, 0x6d, 0x75, 0x6d, 0x20, 0x61, 0x74, + 0x20, 0x6c, 0x65, 0x61, 0x73, 0x74, 0x20, 0x66, 0x6f, 0x72, 0x20, 0x7a, + 0x65, 0x72, 0x6f, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, + 0x74, 0x75, 0x72, 0x6e, 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x5f, 0x61, + 0x64, 0x64, 0x52, 0x75, 0x6c, 0x65, 0x28, 0x72, 0x75, 0x6c, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x20, 0x3d, 0x3d, 0x3d, 0x20, 0x27, 0x72, 0x6f, 0x6f, 0x74, 0x27, 0x20, 0x3f, 0x20, 0x27, 0x72, 0x6f, 0x6f, 0x74, 0x27, 0x20, - 0x3a, 0x20, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x46, 0x6f, 0x72, 0x6d, - 0x61, 0x74, 0x2c, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x50, 0x52, 0x49, 0x4d, 0x49, 0x54, 0x49, 0x56, 0x45, 0x5f, - 0x52, 0x55, 0x4c, 0x45, 0x53, 0x5b, 0x27, 0x75, 0x75, 0x69, 0x64, 0x27, - 0x5d, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x20, 0x65, 0x6c, 0x73, - 0x65, 0x20, 0x69, 0x66, 0x20, 0x28, 0x28, 0x73, 0x63, 0x68, 0x65, 0x6d, - 0x61, 0x54, 0x79, 0x70, 0x65, 0x20, 0x3d, 0x3d, 0x3d, 0x20, 0x75, 0x6e, - 0x64, 0x65, 0x66, 0x69, 0x6e, 0x65, 0x64, 0x20, 0x7c, 0x7c, 0x20, 0x73, - 0x63, 0x68, 0x65, 0x6d, 0x61, 0x54, 0x79, 0x70, 0x65, 0x20, 0x3d, 0x3d, - 0x3d, 0x20, 0x27, 0x73, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x27, 0x29, 0x20, - 0x26, 0x26, 0x20, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2e, 0x66, 0x6f, - 0x72, 0x6d, 0x61, 0x74, 0x20, 0x69, 0x6e, 0x20, 0x44, 0x41, 0x54, 0x45, - 0x5f, 0x52, 0x55, 0x4c, 0x45, 0x53, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x66, 0x6f, 0x72, 0x20, 0x28, 0x63, 0x6f, 0x6e, - 0x73, 0x74, 0x20, 0x5b, 0x74, 0x2c, 0x20, 0x72, 0x5d, 0x20, 0x6f, 0x66, - 0x20, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x2e, 0x65, 0x6e, 0x74, 0x72, - 0x69, 0x65, 0x73, 0x28, 0x44, 0x41, 0x54, 0x45, 0x5f, 0x52, 0x55, 0x4c, - 0x45, 0x53, 0x29, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x5f, 0x61, 0x64, 0x64, - 0x52, 0x75, 0x6c, 0x65, 0x28, 0x74, 0x2c, 0x20, 0x72, 0x29, 0x3b, 0x0a, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x73, 0x63, 0x68, - 0x65, 0x6d, 0x61, 0x46, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x20, 0x2b, 0x20, - 0x27, 0x2d, 0x73, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x27, 0x3b, 0x0a, 0x20, - 0x20, 0x20, 0x20, 0x7d, 0x20, 0x65, 0x6c, 0x73, 0x65, 0x20, 0x69, 0x66, - 0x20, 0x28, 0x28, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x54, 0x79, 0x70, - 0x65, 0x20, 0x3d, 0x3d, 0x3d, 0x20, 0x27, 0x6f, 0x62, 0x6a, 0x65, 0x63, - 0x74, 0x27, 0x29, 0x20, 0x7c, 0x7c, 0x20, 0x28, 0x4f, 0x62, 0x6a, 0x65, - 0x63, 0x74, 0x2e, 0x6b, 0x65, 0x79, 0x73, 0x28, 0x73, 0x63, 0x68, 0x65, - 0x6d, 0x61, 0x29, 0x2e, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x20, 0x3d, - 0x3d, 0x3d, 0x20, 0x30, 0x29, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x66, 0x6f, 0x72, 0x20, 0x28, 0x63, 0x6f, 0x6e, 0x73, - 0x74, 0x20, 0x6e, 0x20, 0x6f, 0x66, 0x20, 0x4f, 0x42, 0x4a, 0x45, 0x43, - 0x54, 0x5f, 0x52, 0x55, 0x4c, 0x45, 0x5f, 0x4e, 0x41, 0x4d, 0x45, 0x53, - 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x74, 0x68, 0x69, 0x73, 0x2e, 0x5f, 0x61, 0x64, 0x64, 0x52, 0x75, 0x6c, - 0x65, 0x28, 0x6e, 0x2c, 0x20, 0x50, 0x52, 0x49, 0x4d, 0x49, 0x54, 0x49, - 0x56, 0x45, 0x5f, 0x52, 0x55, 0x4c, 0x45, 0x53, 0x5b, 0x6e, 0x5d, 0x29, - 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x74, - 0x68, 0x69, 0x73, 0x2e, 0x5f, 0x61, 0x64, 0x64, 0x52, 0x75, 0x6c, 0x65, - 0x28, 0x72, 0x75, 0x6c, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x2c, 0x20, 0x27, - 0x6f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x27, 0x29, 0x3b, 0x0a, 0x20, 0x20, - 0x20, 0x20, 0x7d, 0x20, 0x65, 0x6c, 0x73, 0x65, 0x20, 0x7b, 0x0a, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, 0x21, 0x28, 0x73, - 0x63, 0x68, 0x65, 0x6d, 0x61, 0x54, 0x79, 0x70, 0x65, 0x20, 0x69, 0x6e, - 0x20, 0x50, 0x52, 0x49, 0x4d, 0x49, 0x54, 0x49, 0x56, 0x45, 0x5f, 0x52, - 0x55, 0x4c, 0x45, 0x53, 0x29, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x74, 0x68, 0x72, 0x6f, 0x77, 0x20, 0x6e, - 0x65, 0x77, 0x20, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x28, 0x60, 0x55, 0x6e, - 0x72, 0x65, 0x63, 0x6f, 0x67, 0x6e, 0x69, 0x7a, 0x65, 0x64, 0x20, 0x73, - 0x63, 0x68, 0x65, 0x6d, 0x61, 0x3a, 0x20, 0x24, 0x7b, 0x4a, 0x53, 0x4f, - 0x4e, 0x2e, 0x73, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x69, 0x66, 0x79, 0x28, - 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x29, 0x7d, 0x60, 0x29, 0x3b, 0x0a, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x2f, 0x2f, 0x20, 0x54, 0x4f, 0x44, 0x4f, 0x3a, 0x20, 0x73, - 0x75, 0x70, 0x70, 0x6f, 0x72, 0x74, 0x20, 0x6d, 0x69, 0x6e, 0x69, 0x6d, - 0x75, 0x6d, 0x2c, 0x20, 0x6d, 0x61, 0x78, 0x69, 0x6d, 0x75, 0x6d, 0x2c, - 0x20, 0x65, 0x78, 0x63, 0x6c, 0x75, 0x73, 0x69, 0x76, 0x65, 0x4d, 0x69, - 0x6e, 0x69, 0x6d, 0x75, 0x6d, 0x2c, 0x20, 0x65, 0x78, 0x63, 0x6c, 0x75, - 0x73, 0x69, 0x76, 0x65, 0x4d, 0x61, 0x78, 0x69, 0x6d, 0x75, 0x6d, 0x20, - 0x61, 0x74, 0x20, 0x6c, 0x65, 0x61, 0x73, 0x74, 0x20, 0x66, 0x6f, 0x72, - 0x20, 0x7a, 0x65, 0x72, 0x6f, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, - 0x5f, 0x61, 0x64, 0x64, 0x52, 0x75, 0x6c, 0x65, 0x28, 0x72, 0x75, 0x6c, - 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x20, 0x3d, 0x3d, 0x3d, 0x20, 0x27, 0x72, - 0x6f, 0x6f, 0x74, 0x27, 0x20, 0x3f, 0x20, 0x27, 0x72, 0x6f, 0x6f, 0x74, - 0x27, 0x20, 0x3a, 0x20, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x54, 0x79, - 0x70, 0x65, 0x2c, 0x20, 0x50, 0x52, 0x49, 0x4d, 0x49, 0x54, 0x49, 0x56, - 0x45, 0x5f, 0x52, 0x55, 0x4c, 0x45, 0x53, 0x5b, 0x73, 0x63, 0x68, 0x65, - 0x6d, 0x61, 0x54, 0x79, 0x70, 0x65, 0x5d, 0x29, 0x3b, 0x0a, 0x20, 0x20, - 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x7d, 0x0a, 0x0a, 0x20, 0x20, 0x5f, - 0x62, 0x75, 0x69, 0x6c, 0x64, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x52, - 0x75, 0x6c, 0x65, 0x28, 0x70, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, - 0x65, 0x73, 0x2c, 0x20, 0x72, 0x65, 0x71, 0x75, 0x69, 0x72, 0x65, 0x64, - 0x2c, 0x20, 0x6e, 0x61, 0x6d, 0x65, 0x2c, 0x20, 0x61, 0x64, 0x64, 0x69, - 0x74, 0x69, 0x6f, 0x6e, 0x61, 0x6c, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, - 0x74, 0x69, 0x65, 0x73, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, - 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x70, 0x72, 0x6f, 0x70, 0x4f, 0x72, - 0x64, 0x65, 0x72, 0x20, 0x3d, 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x5f, - 0x70, 0x72, 0x6f, 0x70, 0x4f, 0x72, 0x64, 0x65, 0x72, 0x3b, 0x0a, 0x20, - 0x20, 0x20, 0x20, 0x2f, 0x2f, 0x20, 0x73, 0x6f, 0x72, 0x74, 0x20, 0x62, - 0x79, 0x20, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x69, - 0x6e, 0x20, 0x70, 0x72, 0x6f, 0x70, 0x5f, 0x6f, 0x72, 0x64, 0x65, 0x72, - 0x20, 0x28, 0x69, 0x66, 0x20, 0x73, 0x70, 0x65, 0x63, 0x69, 0x66, 0x69, - 0x65, 0x64, 0x29, 0x20, 0x74, 0x68, 0x65, 0x6e, 0x20, 0x62, 0x79, 0x20, - 0x6f, 0x72, 0x69, 0x67, 0x69, 0x6e, 0x61, 0x6c, 0x20, 0x6f, 0x72, 0x64, - 0x65, 0x72, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, - 0x20, 0x73, 0x6f, 0x72, 0x74, 0x65, 0x64, 0x50, 0x72, 0x6f, 0x70, 0x73, - 0x20, 0x3d, 0x20, 0x70, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, - 0x73, 0x2e, 0x6d, 0x61, 0x70, 0x28, 0x28, 0x5b, 0x6b, 0x5d, 0x29, 0x20, - 0x3d, 0x3e, 0x20, 0x6b, 0x29, 0x2e, 0x73, 0x6f, 0x72, 0x74, 0x28, 0x28, - 0x61, 0x2c, 0x20, 0x62, 0x29, 0x20, 0x3d, 0x3e, 0x20, 0x7b, 0x0a, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x6f, - 0x72, 0x64, 0x65, 0x72, 0x41, 0x20, 0x3d, 0x20, 0x70, 0x72, 0x6f, 0x70, - 0x4f, 0x72, 0x64, 0x65, 0x72, 0x5b, 0x61, 0x5d, 0x20, 0x7c, 0x7c, 0x20, - 0x49, 0x6e, 0x66, 0x69, 0x6e, 0x69, 0x74, 0x79, 0x3b, 0x0a, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x6f, 0x72, - 0x64, 0x65, 0x72, 0x42, 0x20, 0x3d, 0x20, 0x70, 0x72, 0x6f, 0x70, 0x4f, - 0x72, 0x64, 0x65, 0x72, 0x5b, 0x62, 0x5d, 0x20, 0x7c, 0x7c, 0x20, 0x49, - 0x6e, 0x66, 0x69, 0x6e, 0x69, 0x74, 0x79, 0x3b, 0x0a, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x6f, 0x72, - 0x64, 0x65, 0x72, 0x41, 0x20, 0x2d, 0x20, 0x6f, 0x72, 0x64, 0x65, 0x72, - 0x42, 0x20, 0x7c, 0x7c, 0x20, 0x70, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, - 0x69, 0x65, 0x73, 0x2e, 0x66, 0x69, 0x6e, 0x64, 0x49, 0x6e, 0x64, 0x65, - 0x78, 0x28, 0x28, 0x5b, 0x6b, 0x5d, 0x29, 0x20, 0x3d, 0x3e, 0x20, 0x6b, - 0x20, 0x3d, 0x3d, 0x3d, 0x20, 0x61, 0x29, 0x20, 0x2d, 0x20, 0x70, 0x72, - 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x2e, 0x66, 0x69, 0x6e, - 0x64, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x28, 0x28, 0x5b, 0x6b, 0x5d, 0x29, - 0x20, 0x3d, 0x3e, 0x20, 0x6b, 0x20, 0x3d, 0x3d, 0x3d, 0x20, 0x62, 0x29, - 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x29, 0x3b, 0x0a, 0x0a, 0x20, - 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x70, 0x72, 0x6f, - 0x70, 0x4b, 0x76, 0x52, 0x75, 0x6c, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x73, - 0x20, 0x3d, 0x20, 0x7b, 0x7d, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x66, - 0x6f, 0x72, 0x20, 0x28, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x5b, 0x70, - 0x72, 0x6f, 0x70, 0x4e, 0x61, 0x6d, 0x65, 0x2c, 0x20, 0x70, 0x72, 0x6f, - 0x70, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x5d, 0x20, 0x6f, 0x66, 0x20, - 0x70, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x29, 0x20, - 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, - 0x74, 0x20, 0x70, 0x72, 0x6f, 0x70, 0x52, 0x75, 0x6c, 0x65, 0x4e, 0x61, - 0x6d, 0x65, 0x20, 0x3d, 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x76, 0x69, - 0x73, 0x69, 0x74, 0x28, 0x70, 0x72, 0x6f, 0x70, 0x53, 0x63, 0x68, 0x65, - 0x6d, 0x61, 0x2c, 0x20, 0x60, 0x24, 0x7b, 0x6e, 0x61, 0x6d, 0x65, 0x20, - 0x3f, 0x3f, 0x20, 0x27, 0x27, 0x7d, 0x24, 0x7b, 0x6e, 0x61, 0x6d, 0x65, - 0x20, 0x3f, 0x20, 0x27, 0x2d, 0x27, 0x20, 0x3a, 0x20, 0x27, 0x27, 0x7d, - 0x24, 0x7b, 0x70, 0x72, 0x6f, 0x70, 0x4e, 0x61, 0x6d, 0x65, 0x7d, 0x60, - 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x70, 0x72, 0x6f, - 0x70, 0x4b, 0x76, 0x52, 0x75, 0x6c, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x73, - 0x5b, 0x70, 0x72, 0x6f, 0x70, 0x4e, 0x61, 0x6d, 0x65, 0x5d, 0x20, 0x3d, - 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x5f, 0x61, 0x64, 0x64, 0x52, 0x75, - 0x6c, 0x65, 0x28, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x60, 0x24, 0x7b, 0x6e, 0x61, 0x6d, 0x65, 0x20, 0x3f, 0x3f, 0x20, 0x27, - 0x27, 0x7d, 0x24, 0x7b, 0x6e, 0x61, 0x6d, 0x65, 0x20, 0x3f, 0x20, 0x27, - 0x2d, 0x27, 0x20, 0x3a, 0x20, 0x27, 0x27, 0x7d, 0x24, 0x7b, 0x70, 0x72, - 0x6f, 0x70, 0x4e, 0x61, 0x6d, 0x65, 0x7d, 0x2d, 0x6b, 0x76, 0x60, 0x2c, - 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x60, 0x24, 0x7b, - 0x74, 0x68, 0x69, 0x73, 0x2e, 0x5f, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74, - 0x4c, 0x69, 0x74, 0x65, 0x72, 0x61, 0x6c, 0x28, 0x70, 0x72, 0x6f, 0x70, - 0x4e, 0x61, 0x6d, 0x65, 0x29, 0x7d, 0x20, 0x73, 0x70, 0x61, 0x63, 0x65, - 0x20, 0x22, 0x3a, 0x22, 0x20, 0x73, 0x70, 0x61, 0x63, 0x65, 0x20, 0x24, - 0x7b, 0x70, 0x72, 0x6f, 0x70, 0x52, 0x75, 0x6c, 0x65, 0x4e, 0x61, 0x6d, - 0x65, 0x7d, 0x60, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x29, 0x3b, - 0x0a, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x63, - 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x72, 0x65, 0x71, 0x75, 0x69, 0x72, 0x65, - 0x64, 0x50, 0x72, 0x6f, 0x70, 0x73, 0x20, 0x3d, 0x20, 0x73, 0x6f, 0x72, - 0x74, 0x65, 0x64, 0x50, 0x72, 0x6f, 0x70, 0x73, 0x2e, 0x66, 0x69, 0x6c, - 0x74, 0x65, 0x72, 0x28, 0x6b, 0x20, 0x3d, 0x3e, 0x20, 0x72, 0x65, 0x71, - 0x75, 0x69, 0x72, 0x65, 0x64, 0x2e, 0x68, 0x61, 0x73, 0x28, 0x6b, 0x29, - 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, - 0x20, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x61, 0x6c, 0x50, 0x72, 0x6f, - 0x70, 0x73, 0x20, 0x3d, 0x20, 0x73, 0x6f, 0x72, 0x74, 0x65, 0x64, 0x50, - 0x72, 0x6f, 0x70, 0x73, 0x2e, 0x66, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x28, - 0x6b, 0x20, 0x3d, 0x3e, 0x20, 0x21, 0x72, 0x65, 0x71, 0x75, 0x69, 0x72, - 0x65, 0x64, 0x2e, 0x68, 0x61, 0x73, 0x28, 0x6b, 0x29, 0x29, 0x3b, 0x0a, - 0x0a, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, 0x74, 0x79, 0x70, - 0x65, 0x6f, 0x66, 0x20, 0x61, 0x64, 0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e, - 0x61, 0x6c, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, - 0x20, 0x3d, 0x3d, 0x3d, 0x20, 0x27, 0x6f, 0x62, 0x6a, 0x65, 0x63, 0x74, - 0x27, 0x20, 0x7c, 0x7c, 0x20, 0x61, 0x64, 0x64, 0x69, 0x74, 0x69, 0x6f, - 0x6e, 0x61, 0x6c, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, - 0x73, 0x20, 0x3d, 0x3d, 0x3d, 0x20, 0x74, 0x72, 0x75, 0x65, 0x29, 0x20, - 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, - 0x74, 0x20, 0x73, 0x75, 0x62, 0x4e, 0x61, 0x6d, 0x65, 0x20, 0x3d, 0x20, - 0x60, 0x24, 0x7b, 0x6e, 0x61, 0x6d, 0x65, 0x20, 0x3f, 0x3f, 0x20, 0x27, - 0x27, 0x7d, 0x24, 0x7b, 0x6e, 0x61, 0x6d, 0x65, 0x20, 0x3f, 0x20, 0x27, - 0x2d, 0x27, 0x20, 0x3a, 0x20, 0x27, 0x27, 0x7d, 0x61, 0x64, 0x64, 0x69, - 0x74, 0x69, 0x6f, 0x6e, 0x61, 0x6c, 0x60, 0x3b, 0x0a, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x76, 0x61, 0x6c, - 0x75, 0x65, 0x52, 0x75, 0x6c, 0x65, 0x20, 0x3d, 0x20, 0x74, 0x68, 0x69, - 0x73, 0x2e, 0x76, 0x69, 0x73, 0x69, 0x74, 0x28, 0x61, 0x64, 0x64, 0x69, - 0x74, 0x69, 0x6f, 0x6e, 0x61, 0x6c, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, - 0x74, 0x69, 0x65, 0x73, 0x20, 0x3d, 0x3d, 0x3d, 0x20, 0x74, 0x72, 0x75, - 0x65, 0x20, 0x3f, 0x20, 0x7b, 0x7d, 0x20, 0x3a, 0x20, 0x61, 0x64, 0x64, - 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x61, 0x6c, 0x50, 0x72, 0x6f, 0x70, 0x65, - 0x72, 0x74, 0x69, 0x65, 0x73, 0x2c, 0x20, 0x60, 0x24, 0x7b, 0x73, 0x75, - 0x62, 0x4e, 0x61, 0x6d, 0x65, 0x7d, 0x2d, 0x76, 0x61, 0x6c, 0x75, 0x65, - 0x60, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x70, 0x72, - 0x6f, 0x70, 0x4b, 0x76, 0x52, 0x75, 0x6c, 0x65, 0x4e, 0x61, 0x6d, 0x65, - 0x73, 0x5b, 0x27, 0x2a, 0x27, 0x5d, 0x20, 0x3d, 0x20, 0x74, 0x68, 0x69, - 0x73, 0x2e, 0x5f, 0x61, 0x64, 0x64, 0x52, 0x75, 0x6c, 0x65, 0x28, 0x0a, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x60, 0x24, 0x7b, 0x73, - 0x75, 0x62, 0x4e, 0x61, 0x6d, 0x65, 0x7d, 0x2d, 0x6b, 0x76, 0x60, 0x2c, - 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x60, 0x24, 0x7b, - 0x74, 0x68, 0x69, 0x73, 0x2e, 0x5f, 0x61, 0x64, 0x64, 0x52, 0x75, 0x6c, - 0x65, 0x28, 0x27, 0x73, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x27, 0x2c, 0x20, - 0x50, 0x52, 0x49, 0x4d, 0x49, 0x54, 0x49, 0x56, 0x45, 0x5f, 0x52, 0x55, - 0x4c, 0x45, 0x53, 0x5b, 0x27, 0x73, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x27, - 0x5d, 0x29, 0x7d, 0x20, 0x22, 0x3a, 0x22, 0x20, 0x73, 0x70, 0x61, 0x63, - 0x65, 0x20, 0x24, 0x7b, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x75, 0x6c, - 0x65, 0x7d, 0x60, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x61, 0x6c, 0x50, 0x72, 0x6f, 0x70, - 0x73, 0x2e, 0x70, 0x75, 0x73, 0x68, 0x28, 0x27, 0x2a, 0x27, 0x29, 0x3b, - 0x0a, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, - 0x6c, 0x65, 0x74, 0x20, 0x72, 0x75, 0x6c, 0x65, 0x20, 0x3d, 0x20, 0x27, - 0x22, 0x7b, 0x22, 0x20, 0x73, 0x70, 0x61, 0x63, 0x65, 0x20, 0x27, 0x3b, - 0x0a, 0x20, 0x20, 0x20, 0x20, 0x72, 0x75, 0x6c, 0x65, 0x20, 0x2b, 0x3d, - 0x20, 0x72, 0x65, 0x71, 0x75, 0x69, 0x72, 0x65, 0x64, 0x50, 0x72, 0x6f, - 0x70, 0x73, 0x2e, 0x6d, 0x61, 0x70, 0x28, 0x6b, 0x20, 0x3d, 0x3e, 0x20, - 0x70, 0x72, 0x6f, 0x70, 0x4b, 0x76, 0x52, 0x75, 0x6c, 0x65, 0x4e, 0x61, - 0x6d, 0x65, 0x73, 0x5b, 0x6b, 0x5d, 0x29, 0x2e, 0x6a, 0x6f, 0x69, 0x6e, - 0x28, 0x27, 0x20, 0x22, 0x2c, 0x22, 0x20, 0x73, 0x70, 0x61, 0x63, 0x65, - 0x20, 0x27, 0x29, 0x3b, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, - 0x20, 0x28, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x61, 0x6c, 0x50, 0x72, - 0x6f, 0x70, 0x73, 0x2e, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x20, 0x3e, - 0x20, 0x30, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x72, 0x75, 0x6c, 0x65, 0x20, 0x2b, 0x3d, 0x20, 0x27, 0x20, 0x28, 0x27, - 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, - 0x72, 0x65, 0x71, 0x75, 0x69, 0x72, 0x65, 0x64, 0x50, 0x72, 0x6f, 0x70, - 0x73, 0x2e, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x20, 0x3e, 0x20, 0x30, - 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x72, 0x75, 0x6c, 0x65, 0x20, 0x2b, 0x3d, 0x20, 0x27, 0x20, 0x22, 0x2c, - 0x22, 0x20, 0x73, 0x70, 0x61, 0x63, 0x65, 0x20, 0x28, 0x20, 0x27, 0x3b, - 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x0a, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x67, 0x65, - 0x74, 0x52, 0x65, 0x63, 0x75, 0x72, 0x73, 0x69, 0x76, 0x65, 0x52, 0x65, - 0x66, 0x73, 0x20, 0x3d, 0x20, 0x28, 0x6b, 0x73, 0x2c, 0x20, 0x66, 0x69, - 0x72, 0x73, 0x74, 0x49, 0x73, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x61, - 0x6c, 0x29, 0x20, 0x3d, 0x3e, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x5b, 0x6b, - 0x2c, 0x20, 0x2e, 0x2e, 0x2e, 0x72, 0x65, 0x73, 0x74, 0x5d, 0x20, 0x3d, - 0x20, 0x6b, 0x73, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x6b, 0x76, 0x52, 0x75, 0x6c, - 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x20, 0x3d, 0x20, 0x70, 0x72, 0x6f, 0x70, - 0x4b, 0x76, 0x52, 0x75, 0x6c, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x73, 0x5b, - 0x6b, 0x5d, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x6c, 0x65, 0x74, 0x20, 0x72, 0x65, 0x73, 0x3b, 0x0a, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, 0x6b, 0x20, 0x3d, - 0x3d, 0x3d, 0x20, 0x27, 0x2a, 0x27, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, - 0x73, 0x20, 0x3d, 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x5f, 0x61, 0x64, - 0x64, 0x52, 0x75, 0x6c, 0x65, 0x28, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x60, - 0x24, 0x7b, 0x6e, 0x61, 0x6d, 0x65, 0x20, 0x3f, 0x3f, 0x20, 0x27, 0x27, - 0x7d, 0x24, 0x7b, 0x6e, 0x61, 0x6d, 0x65, 0x20, 0x3f, 0x20, 0x27, 0x2d, - 0x27, 0x20, 0x3a, 0x20, 0x27, 0x27, 0x7d, 0x61, 0x64, 0x64, 0x69, 0x74, - 0x69, 0x6f, 0x6e, 0x61, 0x6c, 0x2d, 0x6b, 0x76, 0x73, 0x60, 0x2c, 0x0a, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x60, 0x24, 0x7b, 0x6b, 0x76, 0x52, 0x75, 0x6c, - 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x7d, 0x20, 0x28, 0x20, 0x22, 0x2c, 0x22, - 0x20, 0x73, 0x70, 0x61, 0x63, 0x65, 0x20, 0x60, 0x20, 0x2b, 0x20, 0x6b, - 0x76, 0x52, 0x75, 0x6c, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x20, 0x2b, 0x20, - 0x60, 0x20, 0x29, 0x2a, 0x60, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x7d, 0x20, 0x65, 0x6c, 0x73, 0x65, 0x20, 0x69, - 0x66, 0x20, 0x28, 0x66, 0x69, 0x72, 0x73, 0x74, 0x49, 0x73, 0x4f, 0x70, - 0x74, 0x69, 0x6f, 0x6e, 0x61, 0x6c, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, 0x73, 0x20, - 0x3d, 0x20, 0x60, 0x28, 0x20, 0x22, 0x2c, 0x22, 0x20, 0x73, 0x70, 0x61, - 0x63, 0x65, 0x20, 0x24, 0x7b, 0x6b, 0x76, 0x52, 0x75, 0x6c, 0x65, 0x4e, - 0x61, 0x6d, 0x65, 0x7d, 0x20, 0x29, 0x3f, 0x60, 0x3b, 0x0a, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x20, 0x65, 0x6c, 0x73, 0x65, - 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x72, 0x65, 0x73, 0x20, 0x3d, 0x20, 0x6b, 0x76, 0x52, 0x75, 0x6c, - 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x69, 0x66, 0x20, 0x28, 0x72, 0x65, 0x73, 0x74, 0x2e, 0x6c, 0x65, - 0x6e, 0x67, 0x74, 0x68, 0x20, 0x3e, 0x20, 0x30, 0x29, 0x20, 0x7b, 0x0a, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, - 0x73, 0x20, 0x2b, 0x3d, 0x20, 0x27, 0x20, 0x27, 0x20, 0x2b, 0x20, 0x74, - 0x68, 0x69, 0x73, 0x2e, 0x5f, 0x61, 0x64, 0x64, 0x52, 0x75, 0x6c, 0x65, - 0x28, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x60, 0x24, 0x7b, 0x6e, 0x61, 0x6d, 0x65, 0x20, 0x3f, 0x3f, + 0x3a, 0x20, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x54, 0x79, 0x70, 0x65, + 0x2c, 0x20, 0x50, 0x52, 0x49, 0x4d, 0x49, 0x54, 0x49, 0x56, 0x45, 0x5f, + 0x52, 0x55, 0x4c, 0x45, 0x53, 0x5b, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, + 0x54, 0x79, 0x70, 0x65, 0x5d, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x7d, 0x0a, 0x20, 0x20, 0x7d, 0x0a, 0x0a, 0x20, 0x20, 0x5f, 0x62, 0x75, + 0x69, 0x6c, 0x64, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x52, 0x75, 0x6c, + 0x65, 0x28, 0x70, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, + 0x2c, 0x20, 0x72, 0x65, 0x71, 0x75, 0x69, 0x72, 0x65, 0x64, 0x2c, 0x20, + 0x6e, 0x61, 0x6d, 0x65, 0x2c, 0x20, 0x61, 0x64, 0x64, 0x69, 0x74, 0x69, + 0x6f, 0x6e, 0x61, 0x6c, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, + 0x65, 0x73, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, + 0x6e, 0x73, 0x74, 0x20, 0x70, 0x72, 0x6f, 0x70, 0x4f, 0x72, 0x64, 0x65, + 0x72, 0x20, 0x3d, 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x5f, 0x70, 0x72, + 0x6f, 0x70, 0x4f, 0x72, 0x64, 0x65, 0x72, 0x3b, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x2f, 0x2f, 0x20, 0x73, 0x6f, 0x72, 0x74, 0x20, 0x62, 0x79, 0x20, + 0x70, 0x6f, 0x73, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x69, 0x6e, 0x20, + 0x70, 0x72, 0x6f, 0x70, 0x5f, 0x6f, 0x72, 0x64, 0x65, 0x72, 0x20, 0x28, + 0x69, 0x66, 0x20, 0x73, 0x70, 0x65, 0x63, 0x69, 0x66, 0x69, 0x65, 0x64, + 0x29, 0x20, 0x74, 0x68, 0x65, 0x6e, 0x20, 0x62, 0x79, 0x20, 0x6f, 0x72, + 0x69, 0x67, 0x69, 0x6e, 0x61, 0x6c, 0x20, 0x6f, 0x72, 0x64, 0x65, 0x72, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x73, + 0x6f, 0x72, 0x74, 0x65, 0x64, 0x50, 0x72, 0x6f, 0x70, 0x73, 0x20, 0x3d, + 0x20, 0x70, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x2e, + 0x6d, 0x61, 0x70, 0x28, 0x28, 0x5b, 0x6b, 0x5d, 0x29, 0x20, 0x3d, 0x3e, + 0x20, 0x6b, 0x29, 0x2e, 0x73, 0x6f, 0x72, 0x74, 0x28, 0x28, 0x61, 0x2c, + 0x20, 0x62, 0x29, 0x20, 0x3d, 0x3e, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x6f, 0x72, 0x64, + 0x65, 0x72, 0x41, 0x20, 0x3d, 0x20, 0x70, 0x72, 0x6f, 0x70, 0x4f, 0x72, + 0x64, 0x65, 0x72, 0x5b, 0x61, 0x5d, 0x20, 0x7c, 0x7c, 0x20, 0x49, 0x6e, + 0x66, 0x69, 0x6e, 0x69, 0x74, 0x79, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x6f, 0x72, 0x64, 0x65, + 0x72, 0x42, 0x20, 0x3d, 0x20, 0x70, 0x72, 0x6f, 0x70, 0x4f, 0x72, 0x64, + 0x65, 0x72, 0x5b, 0x62, 0x5d, 0x20, 0x7c, 0x7c, 0x20, 0x49, 0x6e, 0x66, + 0x69, 0x6e, 0x69, 0x74, 0x79, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x6f, 0x72, 0x64, 0x65, + 0x72, 0x41, 0x20, 0x2d, 0x20, 0x6f, 0x72, 0x64, 0x65, 0x72, 0x42, 0x20, + 0x7c, 0x7c, 0x20, 0x70, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, + 0x73, 0x2e, 0x66, 0x69, 0x6e, 0x64, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x28, + 0x28, 0x5b, 0x6b, 0x5d, 0x29, 0x20, 0x3d, 0x3e, 0x20, 0x6b, 0x20, 0x3d, + 0x3d, 0x3d, 0x20, 0x61, 0x29, 0x20, 0x2d, 0x20, 0x70, 0x72, 0x6f, 0x70, + 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x2e, 0x66, 0x69, 0x6e, 0x64, 0x49, + 0x6e, 0x64, 0x65, 0x78, 0x28, 0x28, 0x5b, 0x6b, 0x5d, 0x29, 0x20, 0x3d, + 0x3e, 0x20, 0x6b, 0x20, 0x3d, 0x3d, 0x3d, 0x20, 0x62, 0x29, 0x3b, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x7d, 0x29, 0x3b, 0x0a, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x70, 0x72, 0x6f, 0x70, 0x4b, + 0x76, 0x52, 0x75, 0x6c, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x73, 0x20, 0x3d, + 0x20, 0x7b, 0x7d, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x66, 0x6f, 0x72, + 0x20, 0x28, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x5b, 0x70, 0x72, 0x6f, + 0x70, 0x4e, 0x61, 0x6d, 0x65, 0x2c, 0x20, 0x70, 0x72, 0x6f, 0x70, 0x53, + 0x63, 0x68, 0x65, 0x6d, 0x61, 0x5d, 0x20, 0x6f, 0x66, 0x20, 0x70, 0x72, + 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x29, 0x20, 0x7b, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, + 0x70, 0x72, 0x6f, 0x70, 0x52, 0x75, 0x6c, 0x65, 0x4e, 0x61, 0x6d, 0x65, + 0x20, 0x3d, 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x76, 0x69, 0x73, 0x69, + 0x74, 0x28, 0x70, 0x72, 0x6f, 0x70, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, + 0x2c, 0x20, 0x60, 0x24, 0x7b, 0x6e, 0x61, 0x6d, 0x65, 0x20, 0x3f, 0x3f, 0x20, 0x27, 0x27, 0x7d, 0x24, 0x7b, 0x6e, 0x61, 0x6d, 0x65, 0x20, 0x3f, 0x20, 0x27, 0x2d, 0x27, 0x20, 0x3a, 0x20, 0x27, 0x27, 0x7d, 0x24, 0x7b, - 0x6b, 0x7d, 0x2d, 0x72, 0x65, 0x73, 0x74, 0x60, 0x2c, 0x0a, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x67, 0x65, - 0x74, 0x52, 0x65, 0x63, 0x75, 0x72, 0x73, 0x69, 0x76, 0x65, 0x52, 0x65, - 0x66, 0x73, 0x28, 0x72, 0x65, 0x73, 0x74, 0x2c, 0x20, 0x74, 0x72, 0x75, - 0x65, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, - 0x74, 0x75, 0x72, 0x6e, 0x20, 0x72, 0x65, 0x73, 0x3b, 0x0a, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x7d, 0x3b, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x72, 0x75, 0x6c, 0x65, 0x20, 0x2b, 0x3d, 0x20, 0x6f, 0x70, - 0x74, 0x69, 0x6f, 0x6e, 0x61, 0x6c, 0x50, 0x72, 0x6f, 0x70, 0x73, 0x2e, - 0x6d, 0x61, 0x70, 0x28, 0x28, 0x5f, 0x2c, 0x20, 0x69, 0x29, 0x20, 0x3d, - 0x3e, 0x20, 0x67, 0x65, 0x74, 0x52, 0x65, 0x63, 0x75, 0x72, 0x73, 0x69, - 0x76, 0x65, 0x52, 0x65, 0x66, 0x73, 0x28, 0x6f, 0x70, 0x74, 0x69, 0x6f, - 0x6e, 0x61, 0x6c, 0x50, 0x72, 0x6f, 0x70, 0x73, 0x2e, 0x73, 0x6c, 0x69, - 0x63, 0x65, 0x28, 0x69, 0x29, 0x2c, 0x20, 0x66, 0x61, 0x6c, 0x73, 0x65, - 0x29, 0x29, 0x2e, 0x6a, 0x6f, 0x69, 0x6e, 0x28, 0x27, 0x20, 0x7c, 0x20, - 0x27, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, + 0x70, 0x72, 0x6f, 0x70, 0x4e, 0x61, 0x6d, 0x65, 0x7d, 0x60, 0x29, 0x3b, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x70, 0x72, 0x6f, 0x70, 0x4b, + 0x76, 0x52, 0x75, 0x6c, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x73, 0x5b, 0x70, + 0x72, 0x6f, 0x70, 0x4e, 0x61, 0x6d, 0x65, 0x5d, 0x20, 0x3d, 0x20, 0x74, + 0x68, 0x69, 0x73, 0x2e, 0x5f, 0x61, 0x64, 0x64, 0x52, 0x75, 0x6c, 0x65, + 0x28, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x60, 0x24, + 0x7b, 0x6e, 0x61, 0x6d, 0x65, 0x20, 0x3f, 0x3f, 0x20, 0x27, 0x27, 0x7d, + 0x24, 0x7b, 0x6e, 0x61, 0x6d, 0x65, 0x20, 0x3f, 0x20, 0x27, 0x2d, 0x27, + 0x20, 0x3a, 0x20, 0x27, 0x27, 0x7d, 0x24, 0x7b, 0x70, 0x72, 0x6f, 0x70, + 0x4e, 0x61, 0x6d, 0x65, 0x7d, 0x2d, 0x6b, 0x76, 0x60, 0x2c, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x60, 0x24, 0x7b, 0x74, 0x68, + 0x69, 0x73, 0x2e, 0x5f, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x4c, 0x69, + 0x74, 0x65, 0x72, 0x61, 0x6c, 0x28, 0x4a, 0x53, 0x4f, 0x4e, 0x2e, 0x73, + 0x74, 0x72, 0x69, 0x6e, 0x67, 0x69, 0x66, 0x79, 0x28, 0x70, 0x72, 0x6f, + 0x70, 0x4e, 0x61, 0x6d, 0x65, 0x29, 0x29, 0x7d, 0x20, 0x73, 0x70, 0x61, + 0x63, 0x65, 0x20, 0x22, 0x3a, 0x22, 0x20, 0x73, 0x70, 0x61, 0x63, 0x65, + 0x20, 0x24, 0x7b, 0x70, 0x72, 0x6f, 0x70, 0x52, 0x75, 0x6c, 0x65, 0x4e, + 0x61, 0x6d, 0x65, 0x7d, 0x60, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x72, 0x65, 0x71, 0x75, 0x69, + 0x72, 0x65, 0x64, 0x50, 0x72, 0x6f, 0x70, 0x73, 0x20, 0x3d, 0x20, 0x73, + 0x6f, 0x72, 0x74, 0x65, 0x64, 0x50, 0x72, 0x6f, 0x70, 0x73, 0x2e, 0x66, + 0x69, 0x6c, 0x74, 0x65, 0x72, 0x28, 0x6b, 0x20, 0x3d, 0x3e, 0x20, 0x72, + 0x65, 0x71, 0x75, 0x69, 0x72, 0x65, 0x64, 0x2e, 0x68, 0x61, 0x73, 0x28, + 0x6b, 0x29, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, + 0x73, 0x74, 0x20, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x61, 0x6c, 0x50, + 0x72, 0x6f, 0x70, 0x73, 0x20, 0x3d, 0x20, 0x73, 0x6f, 0x72, 0x74, 0x65, + 0x64, 0x50, 0x72, 0x6f, 0x70, 0x73, 0x2e, 0x66, 0x69, 0x6c, 0x74, 0x65, + 0x72, 0x28, 0x6b, 0x20, 0x3d, 0x3e, 0x20, 0x21, 0x72, 0x65, 0x71, 0x75, + 0x69, 0x72, 0x65, 0x64, 0x2e, 0x68, 0x61, 0x73, 0x28, 0x6b, 0x29, 0x29, + 0x3b, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, 0x74, + 0x79, 0x70, 0x65, 0x6f, 0x66, 0x20, 0x61, 0x64, 0x64, 0x69, 0x74, 0x69, + 0x6f, 0x6e, 0x61, 0x6c, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, + 0x65, 0x73, 0x20, 0x3d, 0x3d, 0x3d, 0x20, 0x27, 0x6f, 0x62, 0x6a, 0x65, + 0x63, 0x74, 0x27, 0x20, 0x7c, 0x7c, 0x20, 0x61, 0x64, 0x64, 0x69, 0x74, + 0x69, 0x6f, 0x6e, 0x61, 0x6c, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, + 0x69, 0x65, 0x73, 0x20, 0x3d, 0x3d, 0x3d, 0x20, 0x74, 0x72, 0x75, 0x65, + 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, + 0x6e, 0x73, 0x74, 0x20, 0x73, 0x75, 0x62, 0x4e, 0x61, 0x6d, 0x65, 0x20, + 0x3d, 0x20, 0x60, 0x24, 0x7b, 0x6e, 0x61, 0x6d, 0x65, 0x20, 0x3f, 0x3f, + 0x20, 0x27, 0x27, 0x7d, 0x24, 0x7b, 0x6e, 0x61, 0x6d, 0x65, 0x20, 0x3f, + 0x20, 0x27, 0x2d, 0x27, 0x20, 0x3a, 0x20, 0x27, 0x27, 0x7d, 0x61, 0x64, + 0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x61, 0x6c, 0x60, 0x3b, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x76, + 0x61, 0x6c, 0x75, 0x65, 0x52, 0x75, 0x6c, 0x65, 0x20, 0x3d, 0x20, 0x74, + 0x68, 0x69, 0x73, 0x2e, 0x76, 0x69, 0x73, 0x69, 0x74, 0x28, 0x61, 0x64, + 0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x61, 0x6c, 0x50, 0x72, 0x6f, 0x70, + 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x20, 0x3d, 0x3d, 0x3d, 0x20, 0x74, + 0x72, 0x75, 0x65, 0x20, 0x3f, 0x20, 0x7b, 0x7d, 0x20, 0x3a, 0x20, 0x61, + 0x64, 0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x61, 0x6c, 0x50, 0x72, 0x6f, + 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x2c, 0x20, 0x60, 0x24, 0x7b, + 0x73, 0x75, 0x62, 0x4e, 0x61, 0x6d, 0x65, 0x7d, 0x2d, 0x76, 0x61, 0x6c, + 0x75, 0x65, 0x60, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x70, 0x72, 0x6f, 0x70, 0x4b, 0x76, 0x52, 0x75, 0x6c, 0x65, 0x4e, 0x61, + 0x6d, 0x65, 0x73, 0x5b, 0x27, 0x2a, 0x27, 0x5d, 0x20, 0x3d, 0x20, 0x74, + 0x68, 0x69, 0x73, 0x2e, 0x5f, 0x61, 0x64, 0x64, 0x52, 0x75, 0x6c, 0x65, + 0x28, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x60, 0x24, + 0x7b, 0x73, 0x75, 0x62, 0x4e, 0x61, 0x6d, 0x65, 0x7d, 0x2d, 0x6b, 0x76, + 0x60, 0x2c, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x60, + 0x24, 0x7b, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x5f, 0x61, 0x64, 0x64, 0x52, + 0x75, 0x6c, 0x65, 0x28, 0x27, 0x73, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x27, + 0x2c, 0x20, 0x50, 0x52, 0x49, 0x4d, 0x49, 0x54, 0x49, 0x56, 0x45, 0x5f, + 0x52, 0x55, 0x4c, 0x45, 0x53, 0x5b, 0x27, 0x73, 0x74, 0x72, 0x69, 0x6e, + 0x67, 0x27, 0x5d, 0x29, 0x7d, 0x20, 0x22, 0x3a, 0x22, 0x20, 0x73, 0x70, + 0x61, 0x63, 0x65, 0x20, 0x24, 0x7b, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x52, + 0x75, 0x6c, 0x65, 0x7d, 0x60, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x61, 0x6c, 0x50, 0x72, + 0x6f, 0x70, 0x73, 0x2e, 0x70, 0x75, 0x73, 0x68, 0x28, 0x27, 0x2a, 0x27, + 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x6c, 0x65, 0x74, 0x20, 0x72, 0x75, 0x6c, 0x65, 0x20, 0x3d, + 0x20, 0x27, 0x22, 0x7b, 0x22, 0x20, 0x73, 0x70, 0x61, 0x63, 0x65, 0x20, + 0x27, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x72, 0x75, 0x6c, 0x65, 0x20, + 0x2b, 0x3d, 0x20, 0x72, 0x65, 0x71, 0x75, 0x69, 0x72, 0x65, 0x64, 0x50, + 0x72, 0x6f, 0x70, 0x73, 0x2e, 0x6d, 0x61, 0x70, 0x28, 0x6b, 0x20, 0x3d, + 0x3e, 0x20, 0x70, 0x72, 0x6f, 0x70, 0x4b, 0x76, 0x52, 0x75, 0x6c, 0x65, + 0x4e, 0x61, 0x6d, 0x65, 0x73, 0x5b, 0x6b, 0x5d, 0x29, 0x2e, 0x6a, 0x6f, + 0x69, 0x6e, 0x28, 0x27, 0x20, 0x22, 0x2c, 0x22, 0x20, 0x73, 0x70, 0x61, + 0x63, 0x65, 0x20, 0x27, 0x29, 0x3b, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x69, 0x66, 0x20, 0x28, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x61, 0x6c, + 0x50, 0x72, 0x6f, 0x70, 0x73, 0x2e, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, + 0x20, 0x3e, 0x20, 0x30, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x72, 0x75, 0x6c, 0x65, 0x20, 0x2b, 0x3d, 0x20, 0x27, 0x20, + 0x28, 0x27, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, 0x72, 0x65, 0x71, 0x75, 0x69, 0x72, 0x65, 0x64, 0x50, 0x72, 0x6f, 0x70, 0x73, 0x2e, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x20, 0x3e, 0x20, 0x30, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, 0x75, 0x6c, 0x65, 0x20, 0x2b, 0x3d, 0x20, 0x27, 0x20, - 0x29, 0x27, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, 0x75, 0x6c, 0x65, 0x20, 0x2b, - 0x3d, 0x20, 0x27, 0x20, 0x29, 0x3f, 0x27, 0x3b, 0x0a, 0x20, 0x20, 0x20, - 0x20, 0x7d, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x72, 0x75, 0x6c, 0x65, - 0x20, 0x2b, 0x3d, 0x20, 0x27, 0x20, 0x22, 0x7d, 0x22, 0x20, 0x73, 0x70, - 0x61, 0x63, 0x65, 0x27, 0x3b, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x72, - 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x72, 0x75, 0x6c, 0x65, 0x3b, 0x0a, - 0x20, 0x20, 0x7d, 0x0a, 0x0a, 0x20, 0x20, 0x66, 0x6f, 0x72, 0x6d, 0x61, - 0x74, 0x47, 0x72, 0x61, 0x6d, 0x6d, 0x61, 0x72, 0x28, 0x29, 0x20, 0x7b, - 0x0a, 0x20, 0x20, 0x20, 0x20, 0x6c, 0x65, 0x74, 0x20, 0x67, 0x72, 0x61, - 0x6d, 0x6d, 0x61, 0x72, 0x20, 0x3d, 0x20, 0x27, 0x27, 0x3b, 0x0a, 0x20, - 0x20, 0x20, 0x20, 0x66, 0x6f, 0x72, 0x20, 0x28, 0x63, 0x6f, 0x6e, 0x73, - 0x74, 0x20, 0x5b, 0x6e, 0x61, 0x6d, 0x65, 0x2c, 0x20, 0x72, 0x75, 0x6c, - 0x65, 0x5d, 0x20, 0x6f, 0x66, 0x20, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, - 0x2e, 0x65, 0x6e, 0x74, 0x72, 0x69, 0x65, 0x73, 0x28, 0x74, 0x68, 0x69, - 0x73, 0x2e, 0x5f, 0x72, 0x75, 0x6c, 0x65, 0x73, 0x29, 0x2e, 0x73, 0x6f, - 0x72, 0x74, 0x28, 0x28, 0x5b, 0x61, 0x5d, 0x2c, 0x20, 0x5b, 0x62, 0x5d, - 0x29, 0x20, 0x3d, 0x3e, 0x20, 0x61, 0x2e, 0x6c, 0x6f, 0x63, 0x61, 0x6c, - 0x65, 0x43, 0x6f, 0x6d, 0x70, 0x61, 0x72, 0x65, 0x28, 0x62, 0x29, 0x29, - 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x67, 0x72, - 0x61, 0x6d, 0x6d, 0x61, 0x72, 0x20, 0x2b, 0x3d, 0x20, 0x60, 0x24, 0x7b, - 0x6e, 0x61, 0x6d, 0x65, 0x7d, 0x20, 0x3a, 0x3a, 0x3d, 0x20, 0x24, 0x7b, - 0x72, 0x75, 0x6c, 0x65, 0x7d, 0x5c, 0x6e, 0x60, 0x3b, 0x0a, 0x20, 0x20, - 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, 0x74, 0x75, - 0x72, 0x6e, 0x20, 0x67, 0x72, 0x61, 0x6d, 0x6d, 0x61, 0x72, 0x3b, 0x0a, - 0x20, 0x20, 0x7d, 0x0a, 0x7d, 0x0a, 0x0a, 0x2f, 0x2f, 0x20, 0x48, 0x65, - 0x6c, 0x70, 0x65, 0x72, 0x20, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, - 0x6e, 0x20, 0x74, 0x6f, 0x20, 0x67, 0x72, 0x6f, 0x75, 0x70, 0x20, 0x65, - 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x20, 0x62, 0x79, 0x20, 0x61, - 0x20, 0x6b, 0x65, 0x79, 0x20, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, - 0x6e, 0x0a, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x2a, 0x20, - 0x67, 0x72, 0x6f, 0x75, 0x70, 0x42, 0x79, 0x28, 0x69, 0x74, 0x65, 0x72, - 0x61, 0x62, 0x6c, 0x65, 0x2c, 0x20, 0x6b, 0x65, 0x79, 0x46, 0x6e, 0x29, - 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x6c, 0x65, 0x74, 0x20, 0x6c, 0x61, 0x73, - 0x74, 0x4b, 0x65, 0x79, 0x20, 0x3d, 0x20, 0x6e, 0x75, 0x6c, 0x6c, 0x3b, - 0x0a, 0x20, 0x20, 0x6c, 0x65, 0x74, 0x20, 0x67, 0x72, 0x6f, 0x75, 0x70, - 0x20, 0x3d, 0x20, 0x5b, 0x5d, 0x3b, 0x0a, 0x20, 0x20, 0x66, 0x6f, 0x72, - 0x20, 0x28, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x65, 0x6c, 0x65, 0x6d, - 0x65, 0x6e, 0x74, 0x20, 0x6f, 0x66, 0x20, 0x69, 0x74, 0x65, 0x72, 0x61, - 0x62, 0x6c, 0x65, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x63, - 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x6b, 0x65, 0x79, 0x20, 0x3d, 0x20, 0x6b, - 0x65, 0x79, 0x46, 0x6e, 0x28, 0x65, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, - 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, 0x6c, - 0x61, 0x73, 0x74, 0x4b, 0x65, 0x79, 0x20, 0x21, 0x3d, 0x3d, 0x20, 0x6e, - 0x75, 0x6c, 0x6c, 0x20, 0x26, 0x26, 0x20, 0x6b, 0x65, 0x79, 0x20, 0x21, - 0x3d, 0x3d, 0x20, 0x6c, 0x61, 0x73, 0x74, 0x4b, 0x65, 0x79, 0x29, 0x20, - 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x79, 0x69, 0x65, 0x6c, - 0x64, 0x20, 0x5b, 0x6c, 0x61, 0x73, 0x74, 0x4b, 0x65, 0x79, 0x2c, 0x20, - 0x67, 0x72, 0x6f, 0x75, 0x70, 0x5d, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x67, 0x72, 0x6f, 0x75, 0x70, 0x20, 0x3d, 0x20, 0x5b, 0x5d, - 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, - 0x67, 0x72, 0x6f, 0x75, 0x70, 0x2e, 0x70, 0x75, 0x73, 0x68, 0x28, 0x65, - 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, - 0x20, 0x6c, 0x61, 0x73, 0x74, 0x4b, 0x65, 0x79, 0x20, 0x3d, 0x20, 0x6b, - 0x65, 0x79, 0x3b, 0x0a, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x69, 0x66, - 0x20, 0x28, 0x67, 0x72, 0x6f, 0x75, 0x70, 0x2e, 0x6c, 0x65, 0x6e, 0x67, - 0x74, 0x68, 0x20, 0x3e, 0x20, 0x30, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, - 0x20, 0x20, 0x79, 0x69, 0x65, 0x6c, 0x64, 0x20, 0x5b, 0x6c, 0x61, 0x73, - 0x74, 0x4b, 0x65, 0x79, 0x2c, 0x20, 0x67, 0x72, 0x6f, 0x75, 0x70, 0x5d, - 0x3b, 0x0a, 0x20, 0x20, 0x7d, 0x0a, 0x7d, 0x0a + 0x22, 0x2c, 0x22, 0x20, 0x73, 0x70, 0x61, 0x63, 0x65, 0x20, 0x28, 0x20, + 0x27, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, + 0x67, 0x65, 0x74, 0x52, 0x65, 0x63, 0x75, 0x72, 0x73, 0x69, 0x76, 0x65, + 0x52, 0x65, 0x66, 0x73, 0x20, 0x3d, 0x20, 0x28, 0x6b, 0x73, 0x2c, 0x20, + 0x66, 0x69, 0x72, 0x73, 0x74, 0x49, 0x73, 0x4f, 0x70, 0x74, 0x69, 0x6f, + 0x6e, 0x61, 0x6c, 0x29, 0x20, 0x3d, 0x3e, 0x20, 0x7b, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, + 0x5b, 0x6b, 0x2c, 0x20, 0x2e, 0x2e, 0x2e, 0x72, 0x65, 0x73, 0x74, 0x5d, + 0x20, 0x3d, 0x20, 0x6b, 0x73, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x6b, 0x76, 0x52, + 0x75, 0x6c, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x20, 0x3d, 0x20, 0x70, 0x72, + 0x6f, 0x70, 0x4b, 0x76, 0x52, 0x75, 0x6c, 0x65, 0x4e, 0x61, 0x6d, 0x65, + 0x73, 0x5b, 0x6b, 0x5d, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x6c, 0x65, 0x74, 0x20, 0x72, 0x65, 0x73, 0x3b, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, 0x6b, + 0x20, 0x3d, 0x3d, 0x3d, 0x20, 0x27, 0x2a, 0x27, 0x29, 0x20, 0x7b, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x72, 0x65, 0x73, 0x20, 0x3d, 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x5f, + 0x61, 0x64, 0x64, 0x52, 0x75, 0x6c, 0x65, 0x28, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x60, 0x24, 0x7b, 0x6e, 0x61, 0x6d, 0x65, 0x20, 0x3f, 0x3f, 0x20, + 0x27, 0x27, 0x7d, 0x24, 0x7b, 0x6e, 0x61, 0x6d, 0x65, 0x20, 0x3f, 0x20, + 0x27, 0x2d, 0x27, 0x20, 0x3a, 0x20, 0x27, 0x27, 0x7d, 0x61, 0x64, 0x64, + 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x61, 0x6c, 0x2d, 0x6b, 0x76, 0x73, 0x60, + 0x2c, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x60, 0x24, 0x7b, 0x6b, 0x76, 0x52, + 0x75, 0x6c, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x7d, 0x20, 0x28, 0x20, 0x22, + 0x2c, 0x22, 0x20, 0x73, 0x70, 0x61, 0x63, 0x65, 0x20, 0x60, 0x20, 0x2b, + 0x20, 0x6b, 0x76, 0x52, 0x75, 0x6c, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x20, + 0x2b, 0x20, 0x60, 0x20, 0x29, 0x2a, 0x60, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x29, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x20, 0x65, 0x6c, 0x73, 0x65, + 0x20, 0x69, 0x66, 0x20, 0x28, 0x66, 0x69, 0x72, 0x73, 0x74, 0x49, 0x73, + 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x61, 0x6c, 0x29, 0x20, 0x7b, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, + 0x73, 0x20, 0x3d, 0x20, 0x60, 0x28, 0x20, 0x22, 0x2c, 0x22, 0x20, 0x73, + 0x70, 0x61, 0x63, 0x65, 0x20, 0x24, 0x7b, 0x6b, 0x76, 0x52, 0x75, 0x6c, + 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x7d, 0x20, 0x29, 0x3f, 0x60, 0x3b, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x20, 0x65, 0x6c, + 0x73, 0x65, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x72, 0x65, 0x73, 0x20, 0x3d, 0x20, 0x6b, 0x76, 0x52, + 0x75, 0x6c, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x3b, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, 0x72, 0x65, 0x73, 0x74, 0x2e, + 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x20, 0x3e, 0x20, 0x30, 0x29, 0x20, + 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x72, 0x65, 0x73, 0x20, 0x2b, 0x3d, 0x20, 0x27, 0x20, 0x27, 0x20, 0x2b, + 0x20, 0x74, 0x68, 0x69, 0x73, 0x2e, 0x5f, 0x61, 0x64, 0x64, 0x52, 0x75, + 0x6c, 0x65, 0x28, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x60, 0x24, 0x7b, 0x6e, 0x61, 0x6d, 0x65, 0x20, + 0x3f, 0x3f, 0x20, 0x27, 0x27, 0x7d, 0x24, 0x7b, 0x6e, 0x61, 0x6d, 0x65, + 0x20, 0x3f, 0x20, 0x27, 0x2d, 0x27, 0x20, 0x3a, 0x20, 0x27, 0x27, 0x7d, + 0x24, 0x7b, 0x6b, 0x7d, 0x2d, 0x72, 0x65, 0x73, 0x74, 0x60, 0x2c, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x67, 0x65, 0x74, 0x52, 0x65, 0x63, 0x75, 0x72, 0x73, 0x69, 0x76, 0x65, + 0x52, 0x65, 0x66, 0x73, 0x28, 0x72, 0x65, 0x73, 0x74, 0x2c, 0x20, 0x74, + 0x72, 0x75, 0x65, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x72, 0x65, 0x73, 0x3b, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x3b, 0x0a, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x72, 0x75, 0x6c, 0x65, 0x20, 0x2b, 0x3d, 0x20, + 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x61, 0x6c, 0x50, 0x72, 0x6f, 0x70, + 0x73, 0x2e, 0x6d, 0x61, 0x70, 0x28, 0x28, 0x5f, 0x2c, 0x20, 0x69, 0x29, + 0x20, 0x3d, 0x3e, 0x20, 0x67, 0x65, 0x74, 0x52, 0x65, 0x63, 0x75, 0x72, + 0x73, 0x69, 0x76, 0x65, 0x52, 0x65, 0x66, 0x73, 0x28, 0x6f, 0x70, 0x74, + 0x69, 0x6f, 0x6e, 0x61, 0x6c, 0x50, 0x72, 0x6f, 0x70, 0x73, 0x2e, 0x73, + 0x6c, 0x69, 0x63, 0x65, 0x28, 0x69, 0x29, 0x2c, 0x20, 0x66, 0x61, 0x6c, + 0x73, 0x65, 0x29, 0x29, 0x2e, 0x6a, 0x6f, 0x69, 0x6e, 0x28, 0x27, 0x20, + 0x7c, 0x20, 0x27, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x69, 0x66, 0x20, 0x28, 0x72, 0x65, 0x71, 0x75, 0x69, 0x72, 0x65, 0x64, + 0x50, 0x72, 0x6f, 0x70, 0x73, 0x2e, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, + 0x20, 0x3e, 0x20, 0x30, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x72, 0x75, 0x6c, 0x65, 0x20, 0x2b, 0x3d, 0x20, + 0x27, 0x20, 0x29, 0x27, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x72, 0x75, 0x6c, 0x65, + 0x20, 0x2b, 0x3d, 0x20, 0x27, 0x20, 0x29, 0x3f, 0x27, 0x3b, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x72, 0x75, + 0x6c, 0x65, 0x20, 0x2b, 0x3d, 0x20, 0x27, 0x20, 0x22, 0x7d, 0x22, 0x20, + 0x73, 0x70, 0x61, 0x63, 0x65, 0x27, 0x3b, 0x0a, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x72, 0x75, 0x6c, 0x65, + 0x3b, 0x0a, 0x20, 0x20, 0x7d, 0x0a, 0x0a, 0x20, 0x20, 0x66, 0x6f, 0x72, + 0x6d, 0x61, 0x74, 0x47, 0x72, 0x61, 0x6d, 0x6d, 0x61, 0x72, 0x28, 0x29, + 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x6c, 0x65, 0x74, 0x20, 0x67, + 0x72, 0x61, 0x6d, 0x6d, 0x61, 0x72, 0x20, 0x3d, 0x20, 0x27, 0x27, 0x3b, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x66, 0x6f, 0x72, 0x20, 0x28, 0x63, 0x6f, + 0x6e, 0x73, 0x74, 0x20, 0x5b, 0x6e, 0x61, 0x6d, 0x65, 0x2c, 0x20, 0x72, + 0x75, 0x6c, 0x65, 0x5d, 0x20, 0x6f, 0x66, 0x20, 0x4f, 0x62, 0x6a, 0x65, + 0x63, 0x74, 0x2e, 0x65, 0x6e, 0x74, 0x72, 0x69, 0x65, 0x73, 0x28, 0x74, + 0x68, 0x69, 0x73, 0x2e, 0x5f, 0x72, 0x75, 0x6c, 0x65, 0x73, 0x29, 0x2e, + 0x73, 0x6f, 0x72, 0x74, 0x28, 0x28, 0x5b, 0x61, 0x5d, 0x2c, 0x20, 0x5b, + 0x62, 0x5d, 0x29, 0x20, 0x3d, 0x3e, 0x20, 0x61, 0x2e, 0x6c, 0x6f, 0x63, + 0x61, 0x6c, 0x65, 0x43, 0x6f, 0x6d, 0x70, 0x61, 0x72, 0x65, 0x28, 0x62, + 0x29, 0x29, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x67, 0x72, 0x61, 0x6d, 0x6d, 0x61, 0x72, 0x20, 0x2b, 0x3d, 0x20, 0x60, + 0x24, 0x7b, 0x6e, 0x61, 0x6d, 0x65, 0x7d, 0x20, 0x3a, 0x3a, 0x3d, 0x20, + 0x24, 0x7b, 0x72, 0x75, 0x6c, 0x65, 0x7d, 0x5c, 0x6e, 0x60, 0x3b, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, + 0x74, 0x75, 0x72, 0x6e, 0x20, 0x67, 0x72, 0x61, 0x6d, 0x6d, 0x61, 0x72, + 0x3b, 0x0a, 0x20, 0x20, 0x7d, 0x0a, 0x7d, 0x0a, 0x0a, 0x2f, 0x2f, 0x20, + 0x48, 0x65, 0x6c, 0x70, 0x65, 0x72, 0x20, 0x66, 0x75, 0x6e, 0x63, 0x74, + 0x69, 0x6f, 0x6e, 0x20, 0x74, 0x6f, 0x20, 0x67, 0x72, 0x6f, 0x75, 0x70, + 0x20, 0x65, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x20, 0x62, 0x79, + 0x20, 0x61, 0x20, 0x6b, 0x65, 0x79, 0x20, 0x66, 0x75, 0x6e, 0x63, 0x74, + 0x69, 0x6f, 0x6e, 0x0a, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, + 0x2a, 0x20, 0x67, 0x72, 0x6f, 0x75, 0x70, 0x42, 0x79, 0x28, 0x69, 0x74, + 0x65, 0x72, 0x61, 0x62, 0x6c, 0x65, 0x2c, 0x20, 0x6b, 0x65, 0x79, 0x46, + 0x6e, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x6c, 0x65, 0x74, 0x20, 0x6c, + 0x61, 0x73, 0x74, 0x4b, 0x65, 0x79, 0x20, 0x3d, 0x20, 0x6e, 0x75, 0x6c, + 0x6c, 0x3b, 0x0a, 0x20, 0x20, 0x6c, 0x65, 0x74, 0x20, 0x67, 0x72, 0x6f, + 0x75, 0x70, 0x20, 0x3d, 0x20, 0x5b, 0x5d, 0x3b, 0x0a, 0x20, 0x20, 0x66, + 0x6f, 0x72, 0x20, 0x28, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x65, 0x6c, + 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x20, 0x6f, 0x66, 0x20, 0x69, 0x74, 0x65, + 0x72, 0x61, 0x62, 0x6c, 0x65, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, + 0x20, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x20, 0x6b, 0x65, 0x79, 0x20, 0x3d, + 0x20, 0x6b, 0x65, 0x79, 0x46, 0x6e, 0x28, 0x65, 0x6c, 0x65, 0x6d, 0x65, + 0x6e, 0x74, 0x29, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, + 0x28, 0x6c, 0x61, 0x73, 0x74, 0x4b, 0x65, 0x79, 0x20, 0x21, 0x3d, 0x3d, + 0x20, 0x6e, 0x75, 0x6c, 0x6c, 0x20, 0x26, 0x26, 0x20, 0x6b, 0x65, 0x79, + 0x20, 0x21, 0x3d, 0x3d, 0x20, 0x6c, 0x61, 0x73, 0x74, 0x4b, 0x65, 0x79, + 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x79, 0x69, + 0x65, 0x6c, 0x64, 0x20, 0x5b, 0x6c, 0x61, 0x73, 0x74, 0x4b, 0x65, 0x79, + 0x2c, 0x20, 0x67, 0x72, 0x6f, 0x75, 0x70, 0x5d, 0x3b, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x67, 0x72, 0x6f, 0x75, 0x70, 0x20, 0x3d, 0x20, + 0x5b, 0x5d, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, + 0x20, 0x20, 0x67, 0x72, 0x6f, 0x75, 0x70, 0x2e, 0x70, 0x75, 0x73, 0x68, + 0x28, 0x65, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x29, 0x3b, 0x0a, 0x20, + 0x20, 0x20, 0x20, 0x6c, 0x61, 0x73, 0x74, 0x4b, 0x65, 0x79, 0x20, 0x3d, + 0x20, 0x6b, 0x65, 0x79, 0x3b, 0x0a, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, + 0x69, 0x66, 0x20, 0x28, 0x67, 0x72, 0x6f, 0x75, 0x70, 0x2e, 0x6c, 0x65, + 0x6e, 0x67, 0x74, 0x68, 0x20, 0x3e, 0x20, 0x30, 0x29, 0x20, 0x7b, 0x0a, + 0x20, 0x20, 0x20, 0x20, 0x79, 0x69, 0x65, 0x6c, 0x64, 0x20, 0x5b, 0x6c, + 0x61, 0x73, 0x74, 0x4b, 0x65, 0x79, 0x2c, 0x20, 0x67, 0x72, 0x6f, 0x75, + 0x70, 0x5d, 0x3b, 0x0a, 0x20, 0x20, 0x7d, 0x0a, 0x7d, 0x0a }; -size_t json_schema_to_grammar_mjs_len = 19964; +unsigned int json_schema_to_grammar_mjs_len = 19690; diff --git a/examples/server/public/json-schema-to-grammar.mjs b/examples/server/public/json-schema-to-grammar.mjs index baa6761c8..17bd60fde 100644 --- a/examples/server/public/json-schema-to-grammar.mjs +++ b/examples/server/public/json-schema-to-grammar.mjs @@ -48,7 +48,7 @@ export class SchemaConverter { } _formatLiteral(literal) { - const escaped = JSON.stringify(literal).replace( + const escaped = literal.replace( GRAMMAR_LITERAL_ESCAPE_RE, m => GRAMMAR_LITERAL_ESCAPES[m] ); @@ -327,10 +327,7 @@ export class SchemaConverter { } _generateConstantRule(value) { - if (typeof value !== 'string') { - throw new Error('Only string constants are supported, got ' + JSON.stringify(value)); - } - return this._formatLiteral(value); + return this._formatLiteral(JSON.stringify(value)); } visit(schema, name) { @@ -346,9 +343,6 @@ export class SchemaConverter { } else if (Array.isArray(schemaType)) { return this._addRule(ruleName, this._generateUnionRule(name, schemaType.map(t => ({ type: t })))); } else if ('const' in schema) { - if (typeof schema.const !== 'string') { - throw new Error('Only string constants are supported, got ' + JSON.stringify(schema.const)); - } return this._addRule(ruleName, this._generateConstantRule(schema.const)); } else if ('enum' in schema) { const rule = schema.enum.map(v => this._generateConstantRule(v)).join(' | '); @@ -457,7 +451,7 @@ export class SchemaConverter { const propRuleName = this.visit(propSchema, `${name ?? ''}${name ? '-' : ''}${propName}`); propKvRuleNames[propName] = this._addRule( `${name ?? ''}${name ? '-' : ''}${propName}-kv`, - `${this._formatLiteral(propName)} space ":" space ${propRuleName}` + `${this._formatLiteral(JSON.stringify(propName))} space ":" space ${propRuleName}` ); } const requiredProps = sortedProps.filter(k => required.has(k)); diff --git a/examples/server/server.cpp b/examples/server/server.cpp index 421fa6d41..27bd2dd70 100644 --- a/examples/server/server.cpp +++ b/examples/server/server.cpp @@ -30,7 +30,7 @@ #include #include -using json = nlohmann::json; +using json = nlohmann::ordered_json; bool server_verbose = false; bool server_log_json = true; diff --git a/examples/server/utils.hpp b/examples/server/utils.hpp index ff146893d..d7eef556a 100644 --- a/examples/server/utils.hpp +++ b/examples/server/utils.hpp @@ -12,7 +12,7 @@ #define DEFAULT_OAICOMPAT_MODEL "gpt-3.5-turbo-0613" -using json = nlohmann::json; +using json = nlohmann::ordered_json; // https://community.openai.com/t/openai-chat-list-of-error-codes-and-types/357791/11 enum error_type { diff --git a/tests/test-json-schema-to-grammar.cpp b/tests/test-json-schema-to-grammar.cpp index 3c40e7563..8aa7c37d8 100755 --- a/tests/test-json-schema-to-grammar.cpp +++ b/tests/test-json-schema-to-grammar.cpp @@ -90,7 +90,7 @@ static void test_all(const std::string & lang, std::function Date: Fri, 22 Mar 2024 13:09:07 +0000 Subject: [PATCH 697/812] tests : conditional python & node json schema tests (#6207) * json: only attempt python & node schema conversion tests if their bins are present Tests introduced in https://github.com/ggerganov/llama.cpp/pull/5978 disabled in https://github.com/ggerganov/llama.cpp/pull/6198 * json: orange warnings when tests skipped * json: ensure py/js schema conv tested on ubuntu-focal-make * json: print env vars in test --- .github/workflows/build.yml | 11 ++++++++ tests/test-json-schema-to-grammar.cpp | 37 ++++++++++++++++++--------- 2 files changed, 36 insertions(+), 12 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b8d66a837..469d81165 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -135,6 +135,9 @@ jobs: ubuntu-focal-make: runs-on: ubuntu-20.04 + env: + LLAMA_NODE_AVAILABLE: true + LLAMA_PYTHON_AVAILABLE: true steps: - name: Clone @@ -147,6 +150,14 @@ jobs: sudo apt-get update sudo apt-get install build-essential gcc-8 + - uses: actions/setup-node@v4 + with: + node-version: "20" + + - uses: actions/setup-python@v4 + with: + python-version: "3.11" + - name: Build id: make_build env: diff --git a/tests/test-json-schema-to-grammar.cpp b/tests/test-json-schema-to-grammar.cpp index 8aa7c37d8..4d110cf5a 100755 --- a/tests/test-json-schema-to-grammar.cpp +++ b/tests/test-json-schema-to-grammar.cpp @@ -799,6 +799,9 @@ static void test_all(const std::string & lang, std::function test-grammar-output.tmp") == 0 ? SUCCESS : FAILURE); - // tc.verify(read("test-grammar-output.tmp")); - //}); - //test_all("JavaScript", [](const TestCase & tc) { - // write("test-json-schema-input.tmp", tc.schema); - // tc.verify_status(std::system( - // "node ./tests/run-json-schema-to-grammar.mjs test-json-schema-input.tmp > test-grammar-output.tmp") == 0 ? SUCCESS : FAILURE); - // tc.verify(read("test-grammar-output.tmp")); - //}); + + if (getenv("LLAMA_PYTHON_AVAILABLE") || (std::system("python --version") == 0)) { + test_all("Python", [](const TestCase & tc) { + write("test-json-schema-input.tmp", tc.schema); + tc.verify_status(std::system( + "python ./examples/json-schema-to-grammar.py test-json-schema-input.tmp > test-grammar-output.tmp") == 0 ? SUCCESS : FAILURE); + tc.verify(read("test-grammar-output.tmp")); + }); + } else { + fprintf(stderr, "\033[33mWARNING: Python not found, skipping Python JSON schema -> grammar tests.\n\033[0m"); + } + + if (getenv("LLAMA_NODE_AVAILABLE") || (std::system("node --version") == 0)) { + test_all("JavaScript", [](const TestCase & tc) { + write("test-json-schema-input.tmp", tc.schema); + tc.verify_status(std::system( + "node ./tests/run-json-schema-to-grammar.mjs test-json-schema-input.tmp > test-grammar-output.tmp") == 0 ? SUCCESS : FAILURE); + tc.verify(read("test-grammar-output.tmp")); + }); + } else { + fprintf(stderr, "\033[33mWARNING: Node not found, skipping JavaScript JSON schema -> grammar tests.\n\033[0m"); + } test_all("Check Expectations Validity", [](const TestCase & tc) { if (tc.expected_status == SUCCESS) { From e80f06d2a194be62ab5b1cd7ef7c7a5b241dd4fb Mon Sep 17 00:00:00 2001 From: Nexesenex <124105151+Nexesenex@users.noreply.github.com> Date: Fri, 22 Mar 2024 14:32:02 +0100 Subject: [PATCH 698/812] llama : correction of the attn.v.weight quantization for IQ3_XS (#6209) IQ3_XS was not mentioned, IQ3_S and IQ3_M were present twice. That PR corrects this in the manner which was probably intended initially. --- llama.cpp | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/llama.cpp b/llama.cpp index 9de4a8602..91bd6b8d0 100644 --- a/llama.cpp +++ b/llama.cpp @@ -12027,13 +12027,7 @@ static ggml_type llama_tensor_get_type(quantize_state_internal & qs, ggml_type n else if (ftype == LLAMA_FTYPE_MOSTLY_IQ3_XXS) { new_type = qs.model.hparams.n_gqa() >= 4 ? GGML_TYPE_Q4_K : !qs.has_imatrix ? GGML_TYPE_IQ3_S : GGML_TYPE_IQ3_XXS; } - else if (ftype == LLAMA_FTYPE_MOSTLY_IQ3_S && qs.model.hparams.n_gqa() >= 4) { - new_type = GGML_TYPE_Q4_K; - } - else if (ftype == LLAMA_FTYPE_MOSTLY_IQ3_M) { - new_type = GGML_TYPE_Q4_K; - } - else if (ftype == LLAMA_FTYPE_MOSTLY_IQ3_S && qs.model.hparams.n_gqa() >= 4) { + else if ((ftype == LLAMA_FTYPE_MOSTLY_IQ3_XS || ftype == LLAMA_FTYPE_MOSTLY_IQ3_S) && qs.model.hparams.n_gqa() >= 4) { new_type = GGML_TYPE_Q4_K; } else if (ftype == LLAMA_FTYPE_MOSTLY_IQ3_M) { From 80bd33bc2c4be352697dc8473339f25e1085d117 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Fri, 22 Mar 2024 15:33:38 +0200 Subject: [PATCH 699/812] common : add HF arg helpers (#6234) * common : add HF arg helpers * common : remove defaults --- common/common.cpp | 93 ++++++++++++++++++++++++++++++++++++++--------- common/common.h | 10 +++-- 2 files changed, 82 insertions(+), 21 deletions(-) diff --git a/common/common.cpp b/common/common.cpp index cc230c9ff..0cc4859f1 100644 --- a/common/common.cpp +++ b/common/common.cpp @@ -647,14 +647,6 @@ static bool gpt_params_find_arg(int argc, char ** argv, const std::string & arg, params.model = argv[i]; return true; } - if (arg == "-mu" || arg == "--model-url") { - if (++i >= argc) { - invalid_param = true; - return true; - } - params.model_url = argv[i]; - return true; - } if (arg == "-md" || arg == "--model-draft") { if (++i >= argc) { invalid_param = true; @@ -671,6 +663,30 @@ static bool gpt_params_find_arg(int argc, char ** argv, const std::string & arg, params.model_alias = argv[i]; return true; } + if (arg == "-mu" || arg == "--model-url") { + if (++i >= argc) { + invalid_param = true; + return true; + } + params.model_url = argv[i]; + return true; + } + if (arg == "-hfr" || arg == "--hf-repo") { + if (++i >= argc) { + invalid_param = true; + return true; + } + params.hf_repo = argv[i]; + return true; + } + if (arg == "-hff" || arg == "--hf-file") { + if (++i >= argc) { + invalid_param = true; + return true; + } + params.hf_file = argv[i]; + return true; + } if (arg == "--lora") { if (++i >= argc) { invalid_param = true; @@ -1403,10 +1419,14 @@ void gpt_print_usage(int /*argc*/, char ** argv, const gpt_params & params) { printf(" layer range to apply the control vector(s) to, start and end inclusive\n"); printf(" -m FNAME, --model FNAME\n"); printf(" model path (default: %s)\n", params.model.c_str()); - printf(" -mu MODEL_URL, --model-url MODEL_URL\n"); - printf(" model download url (default: %s)\n", params.model_url.c_str()); printf(" -md FNAME, --model-draft FNAME\n"); - printf(" draft model for speculative decoding\n"); + printf(" draft model for speculative decoding (default: unused)\n"); + printf(" -mu MODEL_URL, --model-url MODEL_URL\n"); + printf(" model download url (default: unused)\n"); + printf(" -hfr REPO, --hf-repo REPO\n"); + printf(" Hugging Face model repository (default: unused)\n"); + printf(" -hff FILE, --hf-file FILE\n"); + printf(" Hugging Face model file (default: unused)\n"); printf(" -ld LOGDIR, --logdir LOGDIR\n"); printf(" path under which to save YAML logs (no logging if unset)\n"); printf(" --override-kv KEY=TYPE:VALUE\n"); @@ -1655,8 +1675,10 @@ void llama_batch_add( #ifdef LLAMA_USE_CURL -struct llama_model * llama_load_model_from_url(const char * model_url, const char * path_model, - struct llama_model_params params) { +struct llama_model * llama_load_model_from_url( + const char * model_url, + const char * path_model, + const struct llama_model_params & params) { // Basic validation of the model_url if (!model_url || strlen(model_url) == 0) { fprintf(stderr, "%s: invalid model_url\n", __func__); @@ -1850,25 +1872,62 @@ struct llama_model * llama_load_model_from_url(const char * model_url, const cha return llama_load_model_from_file(path_model, params); } +struct llama_model * llama_load_model_from_hf( + const char * repo, + const char * model, + const char * path_model, + const struct llama_model_params & params) { + // construct hugging face model url: + // + // --repo ggml-org/models --file tinyllama-1.1b/ggml-model-f16.gguf + // https://huggingface.co/ggml-org/models/resolve/main/tinyllama-1.1b/ggml-model-f16.gguf + // + // --repo TheBloke/Mixtral-8x7B-v0.1-GGUF --file mixtral-8x7b-v0.1.Q4_K_M.gguf + // https://huggingface.co/TheBloke/Mixtral-8x7B-v0.1-GGUF/resolve/main/mixtral-8x7b-v0.1.Q4_K_M.gguf + // + + std::string model_url = "https://huggingface.co/"; + model_url += repo; + model_url += "/resolve/main/"; + model_url += model; + + return llama_load_model_from_url(model_url.c_str(), path_model, params); +} + #else -struct llama_model * llama_load_model_from_url(const char * /*model_url*/, const char * /*path_model*/, - struct llama_model_params /*params*/) { +struct llama_model * llama_load_model_from_url( + const char * /*model_url*/, + const char * /*path_model*/, + const struct llama_model_params & /*params*/) { fprintf(stderr, "%s: llama.cpp built without libcurl, downloading from an url not supported.\n", __func__); return nullptr; } +struct llama_model * llama_load_model_from_hf( + const char * /*repo*/, + const char * /*model*/, + const char * /*path_model*/, + const struct llama_model_params & /*params*/) { + fprintf(stderr, "%s: llama.cpp built without libcurl, downloading from Hugging Face not supported.\n", __func__); + return nullptr; +} + #endif // LLAMA_USE_CURL std::tuple llama_init_from_gpt_params(gpt_params & params) { auto mparams = llama_model_params_from_gpt_params(params); llama_model * model = nullptr; - if (!params.model_url.empty()) { + + if (!params.hf_repo.empty() && !params.hf_file.empty()) { + model = llama_load_model_from_hf(params.hf_repo.c_str(), params.hf_file.c_str(), params.model.c_str(), mparams); + } else if (!params.model_url.empty()) { model = llama_load_model_from_url(params.model_url.c_str(), params.model.c_str(), mparams); } else { model = llama_load_model_from_file(params.model.c_str(), mparams); } + if (model == NULL) { fprintf(stderr, "%s: error: failed to load model '%s'\n", __func__, params.model.c_str()); return std::make_tuple(nullptr, nullptr); @@ -1908,7 +1967,7 @@ std::tuple llama_init_from_gpt_par } for (unsigned int i = 0; i < params.lora_adapter.size(); ++i) { - const std::string& lora_adapter = std::get<0>(params.lora_adapter[i]); + const std::string & lora_adapter = std::get<0>(params.lora_adapter[i]); float lora_scale = std::get<1>(params.lora_adapter[i]); int err = llama_model_apply_lora_from_file(model, lora_adapter.c_str(), diff --git a/common/common.h b/common/common.h index 31fd401b6..d827d4df7 100644 --- a/common/common.h +++ b/common/common.h @@ -89,9 +89,11 @@ struct gpt_params { struct llama_sampling_params sparams; std::string model = "models/7B/ggml-model-f16.gguf"; // model path - std::string model_url = ""; // model url to download - std::string model_draft = ""; // draft model for speculative decoding + std::string model_draft = ""; // draft model for speculative decoding std::string model_alias = "unknown"; // model alias + std::string model_url = ""; // model url to download + std::string hf_repo = ""; // HF repo + std::string hf_file = ""; // HF file std::string prompt = ""; std::string prompt_file = ""; // store the external prompt file name std::string path_prompt_cache = ""; // path to file for saving/loading prompt eval state @@ -192,8 +194,8 @@ std::tuple llama_init_from_gpt_par struct llama_model_params llama_model_params_from_gpt_params (const gpt_params & params); struct llama_context_params llama_context_params_from_gpt_params(const gpt_params & params); -struct llama_model * llama_load_model_from_url(const char * model_url, const char * path_model, - struct llama_model_params params); +struct llama_model * llama_load_model_from_url(const char * model_url, const char * path_model, const struct llama_model_params & params); +struct llama_model * llama_load_model_from_hf(const char * repo, const char * file, const char * path_model, const struct llama_model_params & params); // Batch utils From ee804f6223777019cf921e0d99cc24669313ab98 Mon Sep 17 00:00:00 2001 From: Minsoo Cheong <54794500+mscheong01@users.noreply.github.com> Date: Sat, 23 Mar 2024 02:15:06 +0900 Subject: [PATCH 700/812] ci: apply concurrency limit for github workflows (#6243) --- .github/workflows/build.yml | 4 ++++ .github/workflows/code-coverage.yml | 4 ++++ .github/workflows/docker.yml | 4 ++++ .github/workflows/editorconfig.yml | 4 ++++ .github/workflows/nix-ci-aarch64.yml | 4 ++++ .github/workflows/nix-ci.yml | 4 ++++ .github/workflows/python-check-requirements.yml | 4 ++++ .github/workflows/python-lint.yml | 4 ++++ .github/workflows/server.yml | 4 ++++ .github/workflows/zig-build.yml | 4 ++++ 10 files changed, 40 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 469d81165..7711bd8d8 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -15,6 +15,10 @@ on: types: [opened, synchronize, reopened] paths: ['**/CMakeLists.txt', '**/Makefile', '**/*.h', '**/*.hpp', '**/*.c', '**/*.cpp', '**/*.cu', '**/*.swift', '**/*.m'] +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + env: BRANCH_NAME: ${{ github.head_ref || github.ref_name }} GGML_NLOOP: 3 diff --git a/.github/workflows/code-coverage.yml b/.github/workflows/code-coverage.yml index 392db8a08..4112518bb 100644 --- a/.github/workflows/code-coverage.yml +++ b/.github/workflows/code-coverage.yml @@ -5,6 +5,10 @@ env: GGML_NLOOP: 3 GGML_N_THREADS: 1 +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + jobs: run: runs-on: ubuntu-20.04 diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 94f9161fc..9591bfc2a 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -15,6 +15,10 @@ on: branches: - master +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + jobs: push_to_registry: name: Push Docker image to Docker Hub diff --git a/.github/workflows/editorconfig.yml b/.github/workflows/editorconfig.yml index 0e0993cd4..7b2a00c90 100644 --- a/.github/workflows/editorconfig.yml +++ b/.github/workflows/editorconfig.yml @@ -14,6 +14,10 @@ on: branches: - master +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + jobs: editorconfig: runs-on: ubuntu-latest diff --git a/.github/workflows/nix-ci-aarch64.yml b/.github/workflows/nix-ci-aarch64.yml index 8d0a3fd7f..109a793ea 100644 --- a/.github/workflows/nix-ci-aarch64.yml +++ b/.github/workflows/nix-ci-aarch64.yml @@ -17,6 +17,10 @@ on: types: [opened, synchronize, reopened] paths: ['**/*.nix', 'flake.lock'] +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + jobs: nix-build-aarch64: runs-on: ubuntu-latest diff --git a/.github/workflows/nix-ci.yml b/.github/workflows/nix-ci.yml index 01c5a9d5a..8b5b99c8f 100644 --- a/.github/workflows/nix-ci.yml +++ b/.github/workflows/nix-ci.yml @@ -8,6 +8,10 @@ on: pull_request: types: [opened, synchronize, reopened] +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + jobs: nix-eval: strategy: diff --git a/.github/workflows/python-check-requirements.yml b/.github/workflows/python-check-requirements.yml index b82205992..4092b12fa 100644 --- a/.github/workflows/python-check-requirements.yml +++ b/.github/workflows/python-check-requirements.yml @@ -16,6 +16,10 @@ on: - 'requirements.txt' - 'requirements/*.txt' +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + jobs: python-check-requirements: runs-on: ubuntu-latest diff --git a/.github/workflows/python-lint.yml b/.github/workflows/python-lint.yml index ea0a05ea1..4bdd79c4a 100644 --- a/.github/workflows/python-lint.yml +++ b/.github/workflows/python-lint.yml @@ -2,6 +2,10 @@ name: flake8 Lint on: [push, pull_request] +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + jobs: flake8-lint: runs-on: ubuntu-latest diff --git a/.github/workflows/server.yml b/.github/workflows/server.yml index 65ca7d9ca..b74dc5e21 100644 --- a/.github/workflows/server.yml +++ b/.github/workflows/server.yml @@ -18,6 +18,10 @@ on: schedule: - cron: '0 0 * * *' +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + jobs: server: runs-on: ubuntu-latest diff --git a/.github/workflows/zig-build.yml b/.github/workflows/zig-build.yml index 68a698ab9..cb43954eb 100644 --- a/.github/workflows/zig-build.yml +++ b/.github/workflows/zig-build.yml @@ -6,6 +6,10 @@ on: branches: - master +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + jobs: build: strategy: From dba1af612926cbd4ebe2d876277af1e3305177e0 Mon Sep 17 00:00:00 2001 From: Pierrick Hymbert Date: Fri, 22 Mar 2024 19:00:01 +0100 Subject: [PATCH 701/812] llama_model_loader: support multiple split/shard GGUFs (#6187) * split: support in llama_model_loader * avoid copying the entire vector Co-authored-by: slaren * split: move llama_tensor_offset to llama_model_loader * llama_model_loader: PR feedbacks: - use only one gguf_context for metadata only - store all ggml_context in a vector as the files and mappings - store all weights in a vector along with the source tensor - rename ctx_gguf to meta - rename ctx_meta to contexts * avoid copying the entire vector * Simplify this by making these optional, switch some layer creation tensor optional Co-authored-by: Georgi Gerganov * Handle optional tensors Co-authored-by: Georgi Gerganov * llama_model_loader: fail if backend cannot allocate buffer * fix mmap buffer management * llama_model_loader: map file to backend buffer if the allocation succeeds only * llama_model_loader: only map tensors included in the context * llama_model_loader: minor, use same variable name for consistency, fix spacing in types cast * llama_model_loader: fail if any of backend buffer cannot be allocated * spacing Co-authored-by: slaren * fix loop over pointer Co-authored-by: slaren * llama_model_loader: if n_tensors declared not equals to loaded tensors in split, throw an exception instead of asserting * llama_model_loader: ensure mappings vector has the expected size * llama_model_loader: use at instead of operator[] if this should never add to the map. * llama_model_loader: immediately add the backend buffer to the model buffers in order to free them if an error occurs in the next allocation. Reserve the expected size. * llama_model_loader: be sure the model mappings has enough capacity before allocating backend buffer * llama_model_loader: fix map -> unordered map * llama_split_prefix: use a clearer version, not pass split path len but dest max len. Co-authored-by: Xuan Son Nguyen * llama : minor ggml-ci * llama : introduce some typedef helpers * docs: add model shard in hot topic * llama_model_loader: put mapping in a unique_ptr from the moment it is allocated Co-authored-by: slaren * fix llama_split_prefix --------- Co-authored-by: slaren Co-authored-by: Georgi Gerganov Co-authored-by: Xuan Son Nguyen --- README.md | 1 + examples/gguf-split/gguf-split.cpp | 151 ++++----- llama.cpp | 474 ++++++++++++++++++++--------- llama.h | 10 + 4 files changed, 412 insertions(+), 224 deletions(-) diff --git a/README.md b/README.md index 368489caa..f9cf19616 100644 --- a/README.md +++ b/README.md @@ -22,6 +22,7 @@ Inference of Meta's [LLaMA](https://arxiv.org/abs/2302.13971) model (and others) - Looking for contributions to add Deepseek support: https://github.com/ggerganov/llama.cpp/issues/5981 - Quantization blind testing: https://github.com/ggerganov/llama.cpp/discussions/5962 - Initial Mamba support has been added: https://github.com/ggerganov/llama.cpp/pull/5328 +- Support loading sharded model, using `gguf-split` CLI https://github.com/ggerganov/llama.cpp/pull/6187 ---- diff --git a/examples/gguf-split/gguf-split.cpp b/examples/gguf-split/gguf-split.cpp index 8e12e6493..f703588e1 100644 --- a/examples/gguf-split/gguf-split.cpp +++ b/examples/gguf-split/gguf-split.cpp @@ -1,31 +1,34 @@ #include "llama.h" -#include "ggml.h" #include "common.h" #include #include -#include #include #include -#include #include #include #include -#include #include +#include +#include + +#if defined(_WIN32) + #include + #ifndef PATH_MAX + #define PATH_MAX MAX_PATH + #endif + #include +#endif enum split_operation : uint8_t { SPLIT_OP_SPLIT, SPLIT_OP_MERGE, }; -static const char * const LLM_KV_GENERAL_SPLIT_I_SPLIT = "general.split"; -static const char * const LLM_KV_GENERAL_SPLIT_N_SPLIT = "general.split_count"; - -static const int SPLIT_FILENAME_MAX = 256; - -static const char * const SPLIT_FILENAME_FORMAT = "%s-%05d-of-%05d.gguf"; +static const char * const LLM_KV_SPLIT_NO = "split.no"; +static const char * const LLM_KV_SPLIT_COUNT = "split.count"; +static const char * const LLM_KV_SPLIT_TENSORS_COUNT = "split.tensors.count"; struct split_params { split_operation operation = SPLIT_OP_SPLIT; @@ -116,13 +119,13 @@ static bool split_params_parse(int argc, const char ** argv, split_params & para try { if (!split_params_parse_ex(argc, argv, params)) { split_print_usage(argv[0]); - exit(1); + exit(EXIT_FAILURE); } } catch (const std::invalid_argument & ex) { fprintf(stderr, "%s\n", ex.what()); split_print_usage(argv[0]); - exit(1); + exit(EXIT_FAILURE); } return result; } @@ -134,12 +137,6 @@ static void zeros(std::ofstream & file, size_t n) { } } -static std::string split_file_name(const std::string & path, int i_split, int n_split) { - char f_split[SPLIT_FILENAME_MAX] = {0}; - snprintf(f_split, sizeof(f_split), SPLIT_FILENAME_FORMAT, path.c_str(), i_split + 1, n_split); - return std::string(f_split); -} - struct split_strategy { const split_params params; std::ifstream & f_input; @@ -180,8 +177,9 @@ struct split_strategy { if (i_split == 0) { gguf_set_kv(ctx_out, ctx_gguf); } - gguf_set_val_u8(ctx_out, LLM_KV_GENERAL_SPLIT_I_SPLIT, i_split); - gguf_set_val_u8(ctx_out, LLM_KV_GENERAL_SPLIT_N_SPLIT, n_split); + gguf_set_val_u16(ctx_out, LLM_KV_SPLIT_NO, i_split); + gguf_set_val_u16(ctx_out, LLM_KV_SPLIT_COUNT, n_split); + gguf_set_val_i32(ctx_out, LLM_KV_SPLIT_TENSORS_COUNT, n_tensors); // populate the original tensors, so we get an initial metadata for (int i = i_split * params.n_split_tensors; i < n_tensors && i < (i_split + 1) * params.n_split_tensors; ++i) { @@ -189,10 +187,11 @@ struct split_strategy { gguf_add_tensor(ctx_out, meta); } - auto split_name = split_file_name(params.output, i_split, n_split); + char split_path[PATH_MAX] = {0}; + llama_split_path(split_path, sizeof(split_path), params.output.c_str(), i_split, n_split); - fprintf(stderr, "%s: %s ...", __func__, split_name.c_str()); - fout = std::ofstream(split_name, std::ios::binary); + fprintf(stderr, "%s: %s ...", __func__, split_path); + fout = std::ofstream(split_path, std::ios::binary); fout.exceptions(std::ofstream::failbit); // fail fast on write errors auto meta_size = gguf_get_meta_size(ctx_out); @@ -250,19 +249,23 @@ static void gguf_split(const split_params & split_params) { std::ifstream f_input(split_params.input.c_str(), std::ios::binary); if (!f_input.is_open()) { fprintf(stderr, "%s: failed to open input GGUF from %s\n", __func__, split_params.input.c_str()); - exit(1); + exit(EXIT_FAILURE); } auto * ctx_gguf = gguf_init_from_file(split_params.input.c_str(), params); if (!ctx_gguf) { fprintf(stderr, "%s: failed to load input GGUF from %s\n", __func__, split_params.input.c_str()); - exit(1); + exit(EXIT_FAILURE); } split_strategy strategy(split_params, f_input, ctx_gguf, ctx_meta); + + char first_split_path[PATH_MAX] = {0}; + llama_split_path(first_split_path, sizeof(first_split_path), + split_params.output.c_str(), strategy.i_split, strategy.n_split); fprintf(stderr, "%s: %s -> %s (%d tensors per file)\n", __func__, split_params.input.c_str(), - split_file_name(split_params.output, strategy.i_split, strategy.n_split).c_str(), + first_split_path, split_params.n_split_tensors); strategy.split_start(); @@ -298,7 +301,9 @@ static void gguf_merge(const split_params & split_params) { std::vector ctx_metas; std::vector ctx_ggufs; - std::string split_prefix; + char split_path[PATH_MAX] = {0}; + strncpy(split_path, split_params.input.c_str(), sizeof(split_path) - 1); + char split_prefix[PATH_MAX] = {0}; // First pass to find KV and tensors metadata for (int i_split = 0; i_split < n_split; i_split++) { @@ -309,89 +314,66 @@ static void gguf_merge(const split_params & split_params) { /*.ctx = */ &ctx_meta, }; - auto split_name = split_params.input; if (i_split > 0) { - split_name = split_file_name(split_prefix, i_split, n_split); + llama_split_path(split_path, sizeof(split_path), split_prefix, i_split, n_split); } - fprintf(stderr, "%s: reading metadata %s ...", __func__, split_name.c_str()); + fprintf(stderr, "%s: reading metadata %s ...", __func__, split_path); - auto * ctx_gguf = gguf_init_from_file(split_name.c_str(), params); + auto * ctx_gguf = gguf_init_from_file(split_path, params); if (!ctx_gguf) { fprintf(stderr, "\n%s: failed to load input GGUF from %s\n", __func__, split_params.input.c_str()); - exit(1); + exit(EXIT_FAILURE); } ctx_ggufs.push_back(ctx_gguf); ctx_metas.push_back(ctx_meta); if (i_split == 0) { - auto key_n_split = gguf_find_key(ctx_gguf, LLM_KV_GENERAL_SPLIT_N_SPLIT); + auto key_n_split = gguf_find_key(ctx_gguf, LLM_KV_SPLIT_COUNT); if (key_n_split < 0) { fprintf(stderr, "\n%s: input file does not contain %s metadata\n", __func__, - LLM_KV_GENERAL_SPLIT_N_SPLIT); + LLM_KV_SPLIT_COUNT); gguf_free(ctx_gguf); + ggml_free(ctx_meta); gguf_free(ctx_out); fout.close(); - exit(1); + exit(EXIT_FAILURE); } - n_split = gguf_get_val_u8(ctx_gguf, key_n_split); + n_split = gguf_get_val_u16(ctx_gguf, key_n_split); if (n_split < 1) { fprintf(stderr, "\n%s: input file does not contain a valid split count %d\n", __func__, n_split); gguf_free(ctx_gguf); + ggml_free(ctx_meta); gguf_free(ctx_out); fout.close(); - exit(1); + exit(EXIT_FAILURE); + } + + // Verify the file naming and extract split_prefix + if (!llama_split_prefix(split_prefix, sizeof (split_prefix), split_path, i_split, n_split)) { + fprintf(stderr, "\n%s: unexpected input file name: %s" + " i_split=%d" + " n_split=%d\n", __func__, + split_path, i_split, n_split); + gguf_free(ctx_gguf); + ggml_free(ctx_meta); + gguf_free(ctx_out); + fout.close(); + exit(EXIT_FAILURE); } // Do not trigger merge if we try to merge again the output - gguf_set_val_u8(ctx_out, LLM_KV_GENERAL_SPLIT_N_SPLIT, 0); + gguf_set_val_u16(ctx_gguf, LLM_KV_SPLIT_COUNT, 0); // Set metadata from the first split gguf_set_kv(ctx_out, ctx_gguf); } - // Verify the file naming - { - int i_split_file = 0; - int n_split_file = 0; - const char * i_split_format = "-00000-of-00000.gguf"; - - if (split_name.size() < strlen(i_split_format)) { - fprintf(stderr, "\n%s: unexpected input file name: %s\n", __func__, split_params.input.c_str()); - for (auto * _ctx_gguf : ctx_ggufs) { - gguf_free(_ctx_gguf); - } - gguf_free(ctx_out); - fout.close(); - exit(1); - } - - split_prefix = split_name.substr(0, split_name.size() - strlen(i_split_format)); - - const char * split_name_c_str = split_name.c_str(); - int n_part = sscanf(&split_name_c_str[0] + split_prefix.size(), "-%d-of-%d", &i_split_file, &n_split_file); - - if (n_part != 2 || i_split_file - 1 != i_split || n_split_file != n_split) { - fprintf(stderr, "\n%s: unexpected input file name: %s" - " i_split=%d i_split_file=%d" - " n_split=%d n_split_file=%d\n", __func__, - split_params.input.c_str(), - i_split, i_split_file, - n_split, n_split_file); - for (auto * _ctx_gguf : ctx_ggufs) { - gguf_free(_ctx_gguf); - } - gguf_free(ctx_out); - fout.close(); - exit(1); - } - } - auto n_tensors = gguf_get_n_tensors(ctx_gguf); for (int i_tensor = 0; i_tensor < n_tensors; i_tensor++) { const char * t_name = gguf_get_tensor_name(ctx_gguf, i_tensor); @@ -411,18 +393,19 @@ static void gguf_merge(const split_params & split_params) { // Write tensors data for (int i_split = 0; i_split < n_split; i_split++) { - auto split_name = split_file_name(split_prefix, i_split, n_split); - std::ifstream f_input(split_name.c_str(), std::ios::binary); + llama_split_path(split_path, sizeof(split_path), split_prefix, i_split, n_split); + std::ifstream f_input(split_path, std::ios::binary); if (!f_input.is_open()) { - fprintf(stderr, "%s: failed to open input GGUF from %s\n", __func__, split_name.c_str()); - for (auto * _ctx_gguf : ctx_ggufs) { - gguf_free(_ctx_gguf); + fprintf(stderr, "%s: failed to open input GGUF from %s\n", __func__, split_path); + for (uint32_t i = 0; i < ctx_ggufs.size(); i++) { + gguf_free(ctx_ggufs[i]); + ggml_free(ctx_metas[i]); } gguf_free(ctx_out); fout.close(); - exit(1); + exit(EXIT_FAILURE); } - fprintf(stderr, "%s: writing tensors %s ...", __func__, split_name.c_str()); + fprintf(stderr, "%s: writing tensors %s ...", __func__, split_path); auto * ctx_gguf = ctx_ggufs[i_split]; auto * ctx_meta = ctx_metas[i_split]; @@ -481,8 +464,8 @@ int main(int argc, const char ** argv) { break; case SPLIT_OP_MERGE: gguf_merge(params); break; - default:split_print_usage(argv[0]); - exit(1); + default: split_print_usage(argv[0]); + exit(EXIT_FAILURE); } return 0; diff --git a/llama.cpp b/llama.cpp index 91bd6b8d0..aa6c89246 100644 --- a/llama.cpp +++ b/llama.cpp @@ -52,6 +52,9 @@ #define NOMINMAX #endif #include + #ifndef PATH_MAX + #define PATH_MAX MAX_PATH + #endif #include #endif @@ -290,6 +293,10 @@ enum llm_kv { LLM_KV_ROPE_SCALING_ORIG_CTX_LEN, LLM_KV_ROPE_SCALING_FINETUNED, + LLM_KV_SPLIT_NO, + LLM_KV_SPLIT_COUNT, + LLM_KV_SPLIT_TENSORS_COUNT, + LLM_KV_SSM_INNER_SIZE, LLM_KV_SSM_CONV_KERNEL, LLM_KV_SSM_STATE_SIZE, @@ -355,6 +362,10 @@ static const std::map LLM_KV_NAMES = { { LLM_KV_ROPE_SCALING_ORIG_CTX_LEN, "%s.rope.scaling.original_context_length" }, { LLM_KV_ROPE_SCALING_FINETUNED, "%s.rope.scaling.finetuned" }, + { LLM_KV_SPLIT_NO, "split.no" }, + { LLM_KV_SPLIT_COUNT, "split.count" }, + { LLM_KV_SPLIT_TENSORS_COUNT, "split.tensors.count" }, + { LLM_KV_SSM_CONV_KERNEL, "%s.ssm.conv_kernel" }, { LLM_KV_SSM_INNER_SIZE, "%s.ssm.inner_size" }, { LLM_KV_SSM_STATE_SIZE, "%s.ssm.state_size" }, @@ -1099,6 +1110,7 @@ struct llama_file { } } }; +using llama_files = std::vector>; struct llama_mmap { void * addr; @@ -1299,6 +1311,7 @@ struct llama_mmap { } #endif }; +using llama_mmaps = std::vector>; // Represents some region of memory being locked using mlock or VirtualLock; // will automatically unlock on destruction. @@ -1448,6 +1461,7 @@ struct llama_mlock { static void raw_unlock(const void * addr, size_t len) {} #endif }; +using llama_mlocks = std::vector>; static std::string llama_token_to_piece(const struct llama_context * ctx, llama_token token) { std::vector result(8, 0); @@ -2023,12 +2037,12 @@ struct llama_model { // the model memory buffers for the tensor data std::vector bufs; - // model memory mapped file - std::unique_ptr mapping; + // model memory mapped files + llama_mmaps mappings; // objects representing data potentially being locked in memory - std::vector> mlock_bufs; - llama_mlock mlock_mmap; + llama_mlocks mlock_bufs; + llama_mlocks mlock_mmaps; // for quantize-stats only std::vector> tensors_by_name; @@ -2792,6 +2806,8 @@ namespace GGUFMeta { }; } +using llama_buf_map = std::unordered_map; + struct llama_model_loader { int n_kv = 0; int n_tensors = 0; @@ -2802,54 +2818,133 @@ struct llama_model_loader { bool use_mmap = false; - llama_file file; + llama_files files; llama_ftype ftype; llama_fver fver; - std::unique_ptr mapping; + llama_mmaps mappings; + + // Holds information on a model weights + struct llama_tensor_weights { + uint16_t idx; // source file index + size_t offs; // tensor data offset in the original file + + ggml_tensor * tensor; + + llama_tensor_weights(uint16_t idx, const char * name, const struct gguf_context * gguf_ctx, ggml_tensor * tensor) : idx(idx), tensor(tensor) { + const int tensor_idx = gguf_find_tensor(gguf_ctx, name); + offs = gguf_get_data_offset(gguf_ctx) + gguf_get_tensor_offset(gguf_ctx, tensor_idx); + } + }; + std::vector weights; + std::unordered_map kv_overrides; - struct gguf_context * ctx_gguf = NULL; - struct ggml_context * ctx_meta = NULL; + struct gguf_context * meta = NULL; + std::vector contexts; std::string arch_name; LLM_KV llm_kv = LLM_KV(LLM_ARCH_UNKNOWN); - llama_model_loader(const std::string & fname, bool use_mmap, const struct llama_model_kv_override * param_overrides_p) : file(fname.c_str(), "rb") { + llama_model_loader(const std::string & fname, bool use_mmap, const struct llama_model_kv_override * param_overrides_p) { int trace = 0; if (getenv("LLAMA_TRACE")) { trace = atoi(getenv("LLAMA_TRACE")); } - struct gguf_init_params params = { - /*.no_alloc = */ true, - /*.ctx = */ &ctx_meta, - }; - if (param_overrides_p != nullptr) { for (const struct llama_model_kv_override *p = param_overrides_p; p->key[0] != 0; p++) { kv_overrides.insert({std::string(p->key), *p}); } } - ctx_gguf = gguf_init_from_file(fname.c_str(), params); - if (!ctx_gguf) { + struct ggml_context * ctx = NULL; + struct gguf_init_params params = { + /*.no_alloc = */ true, + /*.ctx = */ &ctx, + }; + + meta = gguf_init_from_file(fname.c_str(), params); + if (!meta) { throw std::runtime_error(format("%s: failed to load model from %s\n", __func__, fname.c_str())); } get_key(llm_kv(LLM_KV_GENERAL_ARCHITECTURE), arch_name, false); llm_kv = LLM_KV(llm_arch_from_string(arch_name)); - n_kv = gguf_get_n_kv(ctx_gguf); - n_tensors = gguf_get_n_tensors(ctx_gguf); + // Save tensors data offset of the main file. + // For subsidiary files, `meta` tensor data offset must not be used, + // so we build a unified tensors index for weights. + for (ggml_tensor * cur = ggml_get_first_tensor(ctx); cur; cur = ggml_get_next_tensor(ctx, cur)) { + weights.emplace_back(llama_tensor_weights(0, cur->name, meta, cur)); + } + files.emplace_back(new llama_file(fname.c_str(), "rb")); + contexts.emplace_back(ctx); - fver = (enum llama_fver ) gguf_get_version(ctx_gguf); + uint16_t n_split = 0; + get_key(llm_kv(LLM_KV_SPLIT_COUNT), n_split, false); - for (int i = 0; i < n_tensors; i++) { - const char * name = gguf_get_tensor_name(ctx_gguf, i); - struct ggml_tensor * t = ggml_get_tensor(ctx_meta, name); - n_elements += ggml_nelements(t); - n_bytes += ggml_nbytes(t); + // Load additional GGML contexts + if (n_split > 1) { + uint16_t idx = 0; + get_key(llm_kv(LLM_KV_SPLIT_NO), idx); + if (idx != 0) { + throw std::runtime_error(format("illegal split file: %d, model must be loaded with the first split", idx)); + } + + char split_prefix[PATH_MAX] = {0}; + if (!llama_split_prefix(split_prefix, sizeof(split_prefix), fname.c_str(), idx, n_split)) { + throw std::runtime_error(format("invalid split file: %s", fname.c_str())); + } + + if (trace > 0) { + LLAMA_LOG_INFO("%s: loading additional %d GGUFs\n", __func__, n_split); + } + + char split_path[PATH_MAX] = {0}; + for (idx = 1; idx < n_split; idx++) { + llama_split_path(split_path, sizeof(split_path), split_prefix, idx, n_split); + + struct gguf_init_params split_params = { + /*.no_alloc = */ true, + /*.ctx = */ &ctx, + }; + struct gguf_context * ctx_gguf = gguf_init_from_file(split_path, split_params); + if (!ctx_gguf) { + throw std::runtime_error(format("%s: failed to load GGUF split from %s\n", __func__, split_path)); + } + + // Save tensors data offset info of the shard. + for (ggml_tensor * cur = ggml_get_first_tensor(ctx); cur; cur = ggml_get_next_tensor(ctx, cur)) { + weights.emplace_back(llama_tensor_weights(idx, cur->name, ctx_gguf, cur)); + } + files.emplace_back(new llama_file(split_path, "rb")); + contexts.emplace_back(ctx); + + gguf_free(ctx_gguf); + } + + get_key(llm_kv(LLM_KV_SPLIT_TENSORS_COUNT), n_tensors); + + // sanity check + { + const int n_tensors_loaded = (int) weights.size(); + if (n_tensors != n_tensors_loaded) { + throw std::runtime_error(format("corrupted model: %d tensors expected but %d found", n_tensors, n_tensors_loaded)); + } + } + + LLAMA_LOG_INFO("%s: additional %d GGUFs metadata loaded.\n", __func__, n_split); + } + + n_kv = gguf_get_n_kv(meta); + n_tensors = weights.size(); + + fver = (enum llama_fver) gguf_get_version(meta); + + for (auto & w : weights) { + n_elements += ggml_nelements(w.tensor); + n_bytes += ggml_nbytes(w.tensor); } LLAMA_LOG_INFO("%s: loaded meta data with %d key-value pairs and %d tensors from %s (version %s)\n", @@ -2864,7 +2959,8 @@ struct llama_model_loader { enum ggml_type type_max = GGML_TYPE_F32; for (int i = 0; i < n_tensors; i++) { - enum ggml_type type = gguf_get_tensor_type(ctx_gguf, i); + const ggml_tensor * tensor = weights.at(i).tensor; + enum ggml_type type = tensor->type; n_type[type]++; @@ -2874,8 +2970,8 @@ struct llama_model_loader { } if (trace > 0) { - struct ggml_tensor * meta = ggml_get_tensor(ctx_meta, gguf_get_tensor_name(ctx_gguf, i)); - LLAMA_LOG_INFO("%s: - tensor %4d: %32s %-8s [ %s ]\n", __func__, i, ggml_get_name(meta), ggml_type_name(type), llama_format_tensor_shape(meta).c_str()); + const uint16_t sid = weights.at(i).idx; + LLAMA_LOG_INFO("%s: - tensor %4d, split %2d: %32s %-8s [ %s ]\n", __func__, i, sid, ggml_get_name(tensor), ggml_type_name(type), llama_format_tensor_shape(tensor).c_str()); } } @@ -2911,22 +3007,23 @@ struct llama_model_loader { ftype = (llama_ftype) (ftype | LLAMA_FTYPE_GUESSED); { - const int kid = gguf_find_key(ctx_gguf, "general.file_type"); + const int kid = gguf_find_key(meta, "general.file_type"); if (kid >= 0) { - ftype = (llama_ftype) gguf_get_val_u32(ctx_gguf, kid); + ftype = (llama_ftype) gguf_get_val_u32(meta, kid); } } LLAMA_LOG_INFO("%s: Dumping metadata keys/values. Note: KV overrides do not apply in this output.\n", __func__); + for (int i = 0; i < n_kv; i++) { - const char * name = gguf_get_key(ctx_gguf, i); - const enum gguf_type type = gguf_get_kv_type(ctx_gguf, i); + const char * name = gguf_get_key(meta, i); + const enum gguf_type type = gguf_get_kv_type(meta, i); const std::string type_name = type == GGUF_TYPE_ARRAY - ? format("%s[%s,%d]", gguf_type_name(type), gguf_type_name(gguf_get_arr_type(ctx_gguf, i)), gguf_get_arr_n(ctx_gguf, i)) + ? format("%s[%s,%d]", gguf_type_name(type), gguf_type_name(gguf_get_arr_type(meta, i)), gguf_get_arr_n(meta, i)) : gguf_type_name(type); - std::string value = gguf_kv_to_str(ctx_gguf, i); + std::string value = gguf_kv_to_str(meta, i); const size_t MAX_VALUE_LEN = 40; if (value.size() > MAX_VALUE_LEN) { value = format("%s...", value.substr(0, MAX_VALUE_LEN - 3).c_str()); @@ -2955,18 +3052,18 @@ struct llama_model_loader { } ~llama_model_loader() { - if (ctx_gguf) { - gguf_free(ctx_gguf); + if (meta) { + gguf_free(meta); } - if (ctx_meta) { - ggml_free(ctx_meta); + for (auto * ctx : contexts) { + ggml_free(ctx); } } template typename std::enable_if::value, bool>::type get_arr_n(const std::string & key, T & result, const bool required = true) { - const int kid = gguf_find_key(ctx_gguf, key.c_str()); + const int kid = gguf_find_key(meta, key.c_str()); if (kid < 0) { if (required) { @@ -2976,7 +3073,7 @@ struct llama_model_loader { } struct GGUFMeta::ArrayInfo arr_info = - GGUFMeta::GKV::get_kv(ctx_gguf, kid); + GGUFMeta::GKV::get_kv(meta, kid); result = arr_info.length; @@ -2996,7 +3093,7 @@ struct llama_model_loader { const struct llama_model_kv_override * override = it != kv_overrides.end() ? &it->second : nullptr; - const bool found = GGUFMeta::GKV::set(ctx_gguf, key, result, override); + const bool found = GGUFMeta::GKV::set(meta, key, result, override); if (required && !found) { throw std::runtime_error(format("key not found in model: %s", key.c_str())); @@ -3019,20 +3116,33 @@ struct llama_model_loader { } const char * get_tensor_name(int i) const { - return gguf_get_tensor_name(ctx_gguf, i); + return weights.at(i).tensor->name; + } + + const llama_tensor_weights & get_weights(const char * name) const { + for (const auto & weight : weights) { + if (strcmp(name, weight.tensor->name) == 0) { + return weight; + } + } + throw std::runtime_error(format("tensor %s not found", name)); } struct ggml_tensor * get_tensor_meta(const char * name) const { - return ggml_get_tensor(ctx_meta, name); + try { + return get_weights(name).tensor; + } catch (const std::runtime_error & e) { + return NULL; + } } struct ggml_tensor * get_tensor_meta(int i) const { return get_tensor_meta(get_tensor_name(i)); } - struct ggml_tensor * create_tensor_for(struct ggml_context * ctx, struct ggml_tensor * meta) { - struct ggml_tensor * tensor = ggml_dup_tensor(ctx, meta); - ggml_set_name(tensor, ggml_get_name(meta)); + struct ggml_tensor * create_tensor_for(struct ggml_context * ctx, const struct ggml_tensor * cur) { + struct ggml_tensor * tensor = ggml_dup_tensor(ctx, cur); + ggml_set_name(tensor, ggml_get_name(cur)); n_created++; @@ -3040,7 +3150,7 @@ struct llama_model_loader { } struct ggml_tensor * create_tensor(struct ggml_context * ctx, const std::string & name, const std::vector & ne, bool required = true) { - struct ggml_tensor * cur = ggml_get_tensor(ctx_meta, name.c_str()); + const struct ggml_tensor * cur = get_tensor_meta(name.c_str()); if (cur == NULL) { if (!required) { @@ -3075,76 +3185,79 @@ struct llama_model_loader { } } - size_t file_offset(const char * name) const { - const int idx = gguf_find_tensor(ctx_gguf, name); - - if (idx < 0) { - throw std::runtime_error(format("%s: tensor '%s' not found in the file", __func__, name)); - } - - return gguf_get_data_offset(ctx_gguf) + gguf_get_tensor_offset(ctx_gguf, idx); - } - - void init_mapping(bool prefetch = true, llama_mlock * lmlock = nullptr) { - // prefetch the whole file - all the data is needed anyway + void init_mappings(bool prefetch = true, llama_mlocks * mlock_mmaps = nullptr) { if (use_mmap) { - mapping.reset(new llama_mmap(&file, prefetch ? -1 : 0, ggml_is_numa())); + mappings.reserve(files.size()); + mmaps_used.reserve(files.size()); + for (const auto & file : files) { + std::unique_ptr mapping(new llama_mmap(file.get(), prefetch ? -1 : 0, ggml_is_numa())); + mmaps_used.emplace_back(std::make_pair(mapping->size, 0)); + if (mlock_mmaps) { + std::unique_ptr mlock_mmap(new llama_mlock()); + mlock_mmap->init(mapping->addr); + mlock_mmaps->emplace_back(std::move(mlock_mmap)); + } + mappings.emplace_back(std::move(mapping)); + } } // compute the total size of all tensors for progress reporting - for (int i = 0; i < gguf_get_n_tensors(ctx_gguf); i++) { - struct ggml_tensor * cur = ggml_get_tensor(ctx_meta, gguf_get_tensor_name(ctx_gguf, i)); - size_data += ggml_nbytes(cur); - } - - if (use_mmap && mapping) { - if (lmlock) { - lmlock->init(mapping->addr); - } - mmap_used_first = mapping->size; + for (auto & w : weights) { + size_data += ggml_nbytes(w.tensor); } } - void get_mapping_range(size_t * first, size_t * last, ggml_context * ctx) const { - GGML_ASSERT(mapping); + void get_mapping_range(size_t * first, size_t * last, void ** addr, int idx, ggml_context * ctx) const { + GGML_ASSERT(!mappings.empty()); + const auto & mapping = mappings.at(idx); *first = mapping->size; *last = 0; + *addr = mapping->addr; for (ggml_tensor * tensor = ggml_get_first_tensor(ctx); tensor; tensor = ggml_get_next_tensor(ctx, tensor)) { - const size_t offs = file_offset(ggml_get_name(tensor)); - *first = std::min(*first, offs); - *last = std::max(*last, offs + ggml_nbytes(tensor)); + const auto & w = get_weights(ggml_get_name(tensor)); + if (w.idx != idx) { + continue; + } + *first = std::min(*first, w.offs); + *last = std::max(*last, w.offs + ggml_nbytes(tensor)); } } // for backwards compatibility, does not support ggml-backend void load_data_for(struct ggml_tensor * cur) const { - const size_t offs = file_offset(ggml_get_name(cur)); + const auto & w = get_weights(ggml_get_name(cur)); - if (use_mmap && mapping) { + if (use_mmap) { + const auto & mapping = mappings.at(w.idx); if (cur->data == nullptr) { - cur->data = (uint8_t *)mapping->addr + offs; + cur->data = (uint8_t *)mapping->addr + w.offs; } else { - memcpy(cur->data, (uint8_t *)mapping->addr + offs, ggml_nbytes(cur)); + memcpy(cur->data, (uint8_t *)mapping->addr + w.offs, ggml_nbytes(cur)); } } else { GGML_ASSERT(cur->data != nullptr); - file.seek(offs, SEEK_SET); - file.read_raw(cur->data, ggml_nbytes(cur)); + GGML_ASSERT(w.idx < files.size()); + const auto & file = files.at(w.idx); + file->seek(w.offs, SEEK_SET); + file->read_raw(cur->data, ggml_nbytes(cur)); } } size_t size_done = 0; size_t size_data = 0; - size_t mmap_used_first = -1; - size_t mmap_used_last = 0; + std::vector> mmaps_used; // Returns false if cancelled by progress_callback - bool load_all_data(struct ggml_context * ctx, llama_progress_callback progress_callback, void * progress_callback_user_data, ggml_backend_buffer_t buf_mmap, llama_mlock * lmlock) { - GGML_ASSERT(size_data != 0 && "call init_mapping() first"); + bool load_all_data( + struct ggml_context * ctx, + llama_buf_map & bufs_mmap, + llama_mlocks * lmlocks, + llama_progress_callback progress_callback, + void * progress_callback_user_data) { + GGML_ASSERT(size_data != 0 && "call init_mappings() first"); std::vector> read_buf; - for (struct ggml_tensor * cur = ggml_get_first_tensor(ctx); cur != NULL; cur = ggml_get_next_tensor(ctx, cur)) { if (progress_callback) { if (!progress_callback((float) size_done / size_data, progress_callback_user_data)) { @@ -3152,41 +3265,57 @@ struct llama_model_loader { } } - const size_t offs = file_offset(ggml_get_name(cur)); + const auto & w = get_weights(ggml_get_name(cur)); + size_t n_size = ggml_nbytes(cur); - if (use_mmap && mapping) { + if (use_mmap) { + const auto & mapping = mappings.at(w.idx); + ggml_backend_buffer_t buf_mmap = nullptr; + if (bufs_mmap.count(w.idx)) { + buf_mmap = bufs_mmap.at(w.idx); + } + GGML_ASSERT(buf_mmap || cur->data); // either we have a buffer to allocate the tensor in, or it is already allocated if (buf_mmap && cur->data == nullptr) { - ggml_backend_tensor_alloc(buf_mmap, cur, (uint8_t *) mapping->addr + offs); - if (lmlock) { - lmlock->grow_to(offs + ggml_nbytes(cur)); + ggml_backend_tensor_alloc(buf_mmap, cur, (uint8_t *) mapping->addr + w.offs); + if (lmlocks) { + const auto & lmlock = lmlocks->at(w.idx); + lmlock->grow_to(w.offs + ggml_nbytes(cur)); } - mmap_used_first = std::min(mmap_used_first, offs); - mmap_used_last = std::max(mmap_used_last, offs + ggml_nbytes(cur)); + + auto & mmap_used = mmaps_used[w.idx]; + mmap_used.first = std::min(mmap_used.first, w.offs); + mmap_used.second = std::max(mmap_used.second, w.offs + n_size); } else { - ggml_backend_tensor_set(cur, (uint8_t *) mapping->addr + offs, 0, ggml_nbytes(cur)); + ggml_backend_tensor_set(cur, (uint8_t *) mapping->addr + w.offs, 0, n_size); } } else { + GGML_ASSERT(w.idx < files.size()); + const auto & file = files.at(w.idx); if (ggml_backend_buffer_is_host(cur->buffer)) { - file.seek(offs, SEEK_SET); - file.read_raw(cur->data, ggml_nbytes(cur)); + file->seek(w.offs, SEEK_SET); + file->read_raw(cur->data, ggml_nbytes(cur)); } else { read_buf.resize(ggml_nbytes(cur)); - file.seek(offs, SEEK_SET); - file.read_raw(read_buf.data(), ggml_nbytes(cur)); - ggml_backend_tensor_set(cur, read_buf.data(), 0, ggml_nbytes(cur)); + file->seek(w.offs, SEEK_SET); + file->read_raw(read_buf.data(), ggml_nbytes(cur)); + ggml_backend_tensor_set(cur, read_buf.data(), 0, n_size); } } - size_done += ggml_nbytes(cur); + size_done += n_size; } // check if this is the last call and do final cleanup if (size_done >= size_data) { // unmap offloaded tensors and metadata - if (use_mmap && mapping) { - mapping->unmap_fragment(0, mmap_used_first); - if (mmap_used_last != 0) { - mapping->unmap_fragment(mmap_used_last, mapping->size); + if (use_mmap) { + for (uint32_t idx = 0; idx < mappings.size(); idx++) { + const auto & mmap_used = mmaps_used.at(idx); + auto & mapping = mappings.at(idx); + mapping->unmap_fragment(0, mmap_used.first); + if (mmap_used.second != 0) { + mapping->unmap_fragment(mmap_used.second, mapping->size); + } } } if (progress_callback) { @@ -3319,7 +3448,7 @@ static void llm_load_hparams( llama_model_loader & ml, llama_model & model) { auto & hparams = model.hparams; - const gguf_context * ctx = ml.ctx_gguf; + const gguf_context * ctx = ml.meta; // get metadata as string for (int i = 0; i < gguf_get_n_kv(ctx); i++) { @@ -3709,7 +3838,7 @@ static void llm_load_vocab( llama_model & model) { auto & vocab = model.vocab; - struct gguf_context * ctx = ml.ctx_gguf; + struct gguf_context * ctx = ml.meta; const auto kv = LLM_KV(model.arch); @@ -4319,10 +4448,8 @@ static bool llm_load_tensors( layer.attn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}); layer.attn_norm_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM, "bias", i), {n_embd}); - if (gguf_find_tensor(ml.ctx_gguf, tn(LLM_TENSOR_ATTN_NORM_2, "weight", i).c_str()) >= 0) { - layer.attn_norm_2 = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM_2, "weight", i), {n_embd}); - layer.attn_norm_2_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM_2, "bias", i), {n_embd}); - } + layer.attn_norm_2 = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM_2, "weight", i), {n_embd}, false); + layer.attn_norm_2_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM_2, "bias", i), {n_embd}, false); layer.wqkv = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_QKV, "weight", i), {n_embd, n_embd + 2*n_embd_gqa}); layer.wo = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd}); @@ -5024,56 +5151,97 @@ static bool llm_load_tensors( ml.done_getting_tensors(); - ml.init_mapping(true, use_mlock ? &model.mlock_mmap : nullptr); + ml.init_mappings(true, &model.mlock_mmaps); + model.mappings.reserve(ml.mappings.size()); // create the backend buffers - std::vector> ctx_bufs; + std::vector> ctx_bufs; + ctx_bufs.reserve(ctx_map.size()); + + // Ensure we have enough capacity for the maximum backend buffer we will potentially create + size_t n_max_backend_buffer = ctx_map.size() * ml.files.size(); + model.bufs.reserve(n_max_backend_buffer); for (auto & it : ctx_map) { ggml_backend_buffer_type_t buft = it.first; - ggml_context * ctx = it.second; - ggml_backend_buffer_t buf = nullptr; + ggml_context * ctx = it.second; + + llama_buf_map bufs; + bufs.reserve(n_max_backend_buffer); // only the mmap region containing the tensors in the model is mapped to the backend buffer // this is important for metal with apple silicon: if the entire model could be mapped to a metal buffer, then we could just use metal for all layers // this allows using partial offloading when the model size exceeds the metal buffer size, but not the RAM size if (ml.use_mmap && buft == llama_default_buffer_type_cpu(true)) { - size_t first, last; - ml.get_mapping_range(&first, &last, ctx); - buf = ggml_backend_cpu_buffer_from_ptr((char *) ml.mapping->addr + first, last - first); + for (uint32_t idx = 0; idx < ml.files.size(); idx++) { + void * addr = nullptr; + size_t first, last; + ml.get_mapping_range(&first, &last, &addr, idx, ctx); + if (first >= last) { + continue; + } + ggml_backend_buffer_t buf = ggml_backend_cpu_buffer_from_ptr((char *) addr + first, last - first); + if (buf == nullptr) { + throw std::runtime_error("unable to allocate backend CPU buffer"); + } + model.bufs.push_back(buf); + bufs.emplace(idx, buf); #ifdef GGML_USE_CUBLAS - if (n_layer >= n_gpu_layers) { - ggml_backend_cuda_register_host_buffer( + if (n_layer >= n_gpu_layers) { + ggml_backend_cuda_register_host_buffer( ggml_backend_buffer_get_base(buf), ggml_backend_buffer_get_size(buf)); - } + } #endif + } } #ifdef GGML_USE_METAL else if (ml.use_mmap && buft == ggml_backend_metal_buffer_type()) { - const size_t max_size = ggml_get_max_tensor_size(ctx); - size_t first, last; - ml.get_mapping_range(&first, &last, ctx); - buf = ggml_backend_metal_buffer_from_ptr((char *) ml.mapping->addr + first, last - first, max_size); + for (uint32_t idx = 0; idx < ml.files.size(); idx++) { + const size_t max_size = ggml_get_max_tensor_size(ctx); + void * addr = nullptr; + size_t first, last; + ml.get_mapping_range(&first, &last, &addr, idx, ctx); + if (first >= last) { + continue; + } + ggml_backend_buffer_t buf = ggml_backend_metal_buffer_from_ptr((char *) addr + first, last - first, max_size); + if (buf == nullptr) { + throw std::runtime_error("unable to allocate backend metal buffer"); + } + model.bufs.push_back(buf); + bufs.emplace(idx, buf); + } } #endif else { - buf = ggml_backend_alloc_ctx_tensors_from_buft(ctx, buft); - if (buf != nullptr && use_mlock && ggml_backend_buffer_is_host(buf)) { + ggml_backend_buffer_t buf = ggml_backend_alloc_ctx_tensors_from_buft(ctx, buft); + if (buf == nullptr) { + throw std::runtime_error("unable to allocate backend buffer"); + } + model.bufs.push_back(buf); + if (use_mlock && ggml_backend_buffer_is_host(buf)) { model.mlock_bufs.emplace_back(new llama_mlock); auto & mlock_buf = model.mlock_bufs.back(); mlock_buf->init (ggml_backend_buffer_get_base(buf)); mlock_buf->grow_to(ggml_backend_buffer_get_size(buf)); } + for (uint32_t idx = 0; idx < ml.files.size(); idx++) { + bufs.emplace(idx, buf); + } } - if (buf == nullptr) { + + if (bufs.empty()) { throw std::runtime_error("failed to allocate buffer"); } - // indicate that this buffer contains weights - // this is used by ggml_backend_sched to improve op scheduling -> ops that use a weight are preferably scheduled to the backend that contains the weight - ggml_backend_buffer_set_usage(buf, GGML_BACKEND_BUFFER_USAGE_WEIGHTS); - model.bufs.push_back(buf); - ctx_bufs.emplace_back(ctx, buf); + + for (auto & buf : bufs) { + // indicate that this buffer contains weights + // this is used by ggml_backend_sched to improve op scheduling -> ops that use a weight are preferably scheduled to the backend that contains the weight + ggml_backend_buffer_set_usage(buf.second, GGML_BACKEND_BUFFER_USAGE_WEIGHTS); + } + + ctx_bufs.emplace_back(ctx, bufs); } if (llama_supports_gpu_offload()) { @@ -5105,13 +5273,15 @@ static bool llm_load_tensors( // load tensor data for (auto & it : ctx_bufs) { ggml_context * ctx = it.first; - ggml_backend_buffer_t buf = it.second; - if (!ml.load_all_data(ctx, progress_callback, progress_callback_user_data, buf, use_mlock ? &model.mlock_mmap : NULL)) { + auto & bufs = it.second; + if (!ml.load_all_data(ctx, bufs, use_mlock ? &model.mlock_mmaps : NULL, progress_callback, progress_callback_user_data)) { return false; } } - model.mapping = std::move(ml.mapping); + for (auto & mapping : ml.mappings) { + model.mappings.emplace_back(std::move(mapping)); + } // loading time will be recalculate after the first eval, so // we take page faults deferred by mmap() into consideration @@ -12302,7 +12472,7 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s #endif llama_model_loader ml(fname_inp, use_mmap, NULL); - ml.init_mapping(false); // no prefetching? + ml.init_mappings(false); // no prefetching? llama_model model; llm_load_arch(ml, model); @@ -12326,12 +12496,12 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s struct gguf_context * ctx_out = gguf_init_empty(); // copy the KV pairs from the input file - gguf_set_kv (ctx_out, ml.ctx_gguf); + gguf_set_kv (ctx_out, ml.meta); gguf_set_val_u32(ctx_out, "general.quantization_version", GGML_QNT_VERSION); gguf_set_val_u32(ctx_out, "general.file_type", ftype); for (int i = 0; i < ml.n_tensors; ++i) { - struct ggml_tensor * meta = ml.get_tensor_meta(i); + const struct ggml_tensor * meta = ml.get_tensor_meta(i); const std::string name = ggml_get_name(meta); @@ -12371,7 +12541,7 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s // populate the original tensors so we get an initial meta data for (int i = 0; i < ml.n_tensors; ++i) { - struct ggml_tensor * meta = ml.get_tensor_meta(i); + const struct ggml_tensor * meta = ml.get_tensor_meta(i); gguf_add_tensor(ctx_out, meta); } @@ -12576,7 +12746,7 @@ static int llama_apply_lora_from_file_internal( if (path_base_model) { LLAMA_LOG_INFO("%s: loading base model from '%s'\n", __func__, path_base_model); ml.reset(new llama_model_loader(path_base_model, /*use_mmap*/ true, /*kv_overrides*/ nullptr)); - ml->init_mapping(/*prefetch*/ false); // no prefetching + ml->init_mappings(/*prefetch*/ false); // no prefetching } struct tensor_meta { @@ -12697,7 +12867,7 @@ static int llama_apply_lora_from_file_internal( ggml_tensor * base_t; if (ml) { - if (gguf_find_tensor(ml->ctx_gguf, base_name.c_str()) < 0) { + if (!ml->get_tensor_meta(base_name.c_str())) { LLAMA_LOG_ERROR("%s: error: tensor '%s' not found in base model\n", __func__, base_name.c_str()); return 1; } @@ -14645,6 +14815,30 @@ LLAMA_API int32_t llama_chat_apply_template( return res; } +LLAMA_API int llama_split_path(char * split_path, size_t maxlen, const char * path_prefix, int split_no, int split_count) { + static const char * const SPLIT_PATH_FORMAT = "%s-%05d-of-%05d.gguf"; + if (snprintf(split_path, maxlen, SPLIT_PATH_FORMAT, path_prefix, split_no + 1, split_count)) { + return strlen(split_path); + } + return 0; +} + +int llama_split_prefix(char * dest, size_t maxlen, const char * split_path, int split_no, int split_count) { + std::string str_split_path(split_path); + char postfix[32]; + snprintf(postfix, 32, "-%05d-of-%05d.gguf", split_no + 1, split_count); + std::string str_postfix(postfix); + + // check if dest ends with postfix + int size_prefix = str_split_path.size() - str_postfix.size(); + if (size_prefix > 0 && str_split_path.find(str_postfix, size_prefix) != std::string::npos) { + snprintf(dest, std::min((size_t) size_prefix, maxlen), "%s", split_path); + return size_prefix; + } + + return 0; +} + struct llama_timings llama_get_timings(struct llama_context * ctx) { struct llama_timings result = { /*.t_start_ms =*/ 1e-3 * ctx->t_start_us, diff --git a/llama.h b/llama.h index 40dcf54e3..7e8ac4b62 100644 --- a/llama.h +++ b/llama.h @@ -960,6 +960,16 @@ extern "C" { int32_t n_past, int32_t n_predict); + /// @details Build a split GGUF final path for this chunk. + /// llama_split_path(split_path, sizeof(split_path), "/models/ggml-model-q4_0", 2, 4) => split_path = "/models/ggml-model-q4_0-00002-of-00004.gguf" + // Returns the split_path length. + LLAMA_API int llama_split_path(char * split_path, size_t maxlen, const char * path_prefix, int split_no, int split_count); + + /// @details Extract the path prefix from the split_path if and only if the split_no and split_count match. + /// llama_split_prefix(split_prefix, 64, "/models/ggml-model-q4_0-00002-of-00004.gguf", 2, 4) => split_prefix = "/models/ggml-model-q4_0" + // Returns the split_prefix length. + LLAMA_API int llama_split_prefix(char * split_prefix, size_t maxlen, const char * split_path, int split_no, int split_count); + // Performance information LLAMA_API struct llama_timings llama_get_timings(struct llama_context * ctx); From 1d0331c12a2f2a6296b471232bd4e66fbf06e6a1 Mon Sep 17 00:00:00 2001 From: Kawrakow <48489457+ikawrakow@users.noreply.github.com> Date: Fri, 22 Mar 2024 19:47:14 +0100 Subject: [PATCH 702/812] quantize: options for output and token embedding tensors qtype (#6239) * quantize: be able to specify the output tensor type * quantize: be able to specify the token embedding tensor type --------- Co-authored-by: Iwan Kawrakow --- examples/quantize/quantize.cpp | 24 +++++++++++++++++ llama.cpp | 47 ++++++++++++++++++++-------------- llama.h | 16 +++++++----- 3 files changed, 61 insertions(+), 26 deletions(-) diff --git a/examples/quantize/quantize.cpp b/examples/quantize/quantize.cpp index 7662ec80c..79e60ea7b 100644 --- a/examples/quantize/quantize.cpp +++ b/examples/quantize/quantize.cpp @@ -189,6 +189,18 @@ static void prepare_imatrix(const std::string& imatrix_file, } } +static ggml_type parse_ggml_type(const char * arg) { + ggml_type result = GGML_TYPE_COUNT; + for (int j = 0; j < GGML_TYPE_COUNT; ++j) { + auto type = ggml_type(j); + const auto * name = ggml_type_name(type); + if (name && strcmp(arg, name) == 0) { + result = type; break; + } + } + return result; +} + int main(int argc, char ** argv) { if (argc < 3) { usage(argv[0]); @@ -203,6 +215,18 @@ int main(int argc, char ** argv) { for (; arg_idx < argc && strncmp(argv[arg_idx], "--", 2) == 0; arg_idx++) { if (strcmp(argv[arg_idx], "--leave-output-tensor") == 0) { params.quantize_output_tensor = false; + } else if (strcmp(argv[arg_idx], "--output-tensor-type") == 0) { + if (arg_idx < argc-1) { + params.output_tensor_type = parse_ggml_type(argv[++arg_idx]); + } else { + usage(argv[0]); + } + } else if (strcmp(argv[arg_idx], "--token-embedding-type") == 0) { + if (arg_idx < argc-1) { + params.token_embedding_type = parse_ggml_type(argv[++arg_idx]); + } else { + usage(argv[0]); + } } else if (strcmp(argv[arg_idx], "--allow-requantize") == 0) { params.allow_requantize = true; } else if (strcmp(argv[arg_idx], "--pure") == 0) { diff --git a/llama.cpp b/llama.cpp index aa6c89246..eedca802b 100644 --- a/llama.cpp +++ b/llama.cpp @@ -12141,27 +12141,34 @@ static ggml_type llama_tensor_get_type(quantize_state_internal & qs, ggml_type n // for arches that share the same tensor between the token embeddings and the output, we quantize the token embeddings // with the quantization of the output tensor if (name == tn(LLM_TENSOR_OUTPUT, "weight") || (!qs.has_output && name == tn(LLM_TENSOR_TOKEN_EMBD, "weight"))) { - int nx = tensor->ne[0]; - if (arch == LLM_ARCH_FALCON || nx % QK_K != 0) { - new_type = GGML_TYPE_Q8_0; - } - else if (ftype == LLAMA_FTYPE_MOSTLY_IQ2_XXS || ftype == LLAMA_FTYPE_MOSTLY_IQ2_XS || ftype == LLAMA_FTYPE_MOSTLY_IQ3_XXS || - ftype == LLAMA_FTYPE_MOSTLY_IQ1_S || ftype == LLAMA_FTYPE_MOSTLY_IQ2_S || ftype == LLAMA_FTYPE_MOSTLY_IQ2_M) { - new_type = GGML_TYPE_Q5_K; - } - else if (new_type != GGML_TYPE_Q8_0) { - new_type = GGML_TYPE_Q6_K; + if (qs.params->output_tensor_type < GGML_TYPE_COUNT) { + new_type = qs.params->output_tensor_type; + } else { + int nx = tensor->ne[0]; + if (arch == LLM_ARCH_FALCON || nx % QK_K != 0) { + new_type = GGML_TYPE_Q8_0; + } + else if (ftype == LLAMA_FTYPE_MOSTLY_IQ2_XXS || ftype == LLAMA_FTYPE_MOSTLY_IQ2_XS || ftype == LLAMA_FTYPE_MOSTLY_IQ3_XXS || + ftype == LLAMA_FTYPE_MOSTLY_IQ1_S || ftype == LLAMA_FTYPE_MOSTLY_IQ2_S || ftype == LLAMA_FTYPE_MOSTLY_IQ2_M) { + new_type = GGML_TYPE_Q5_K; + } + else if (new_type != GGML_TYPE_Q8_0) { + new_type = GGML_TYPE_Q6_K; + } } } else if (name == "token_embd.weight") { - if (ftype == LLAMA_FTYPE_MOSTLY_IQ2_XXS || ftype == LLAMA_FTYPE_MOSTLY_IQ2_XS || - ftype == LLAMA_FTYPE_MOSTLY_IQ1_S) { - new_type = GGML_TYPE_Q2_K; - } - else if (ftype == LLAMA_FTYPE_MOSTLY_IQ2_S || ftype == LLAMA_FTYPE_MOSTLY_IQ2_M) { - new_type = GGML_TYPE_IQ3_S; - } - else if (ftype == LLAMA_FTYPE_MOSTLY_IQ3_XXS) { - new_type = GGML_TYPE_IQ3_S; + if (qs.params->token_embedding_type < GGML_TYPE_COUNT) { + new_type = qs.params->token_embedding_type; + } else { + if (ftype == LLAMA_FTYPE_MOSTLY_IQ2_XXS || ftype == LLAMA_FTYPE_MOSTLY_IQ2_XS || ftype == LLAMA_FTYPE_MOSTLY_IQ1_S) { + new_type = GGML_TYPE_Q2_K; + } + else if (ftype == LLAMA_FTYPE_MOSTLY_IQ2_S || ftype == LLAMA_FTYPE_MOSTLY_IQ2_M) { + new_type = GGML_TYPE_IQ3_S; + } + else if (ftype == LLAMA_FTYPE_MOSTLY_IQ3_XXS) { + new_type = GGML_TYPE_IQ3_S; + } } } else if (ftype == LLAMA_FTYPE_MOSTLY_IQ2_XXS || ftype == LLAMA_FTYPE_MOSTLY_IQ2_XS || ftype == LLAMA_FTYPE_MOSTLY_IQ1_S || ftype == LLAMA_FTYPE_MOSTLY_IQ2_S || ftype == LLAMA_FTYPE_MOSTLY_IQ2_M) { @@ -13051,6 +13058,8 @@ struct llama_model_quantize_params llama_model_quantize_default_params() { struct llama_model_quantize_params result = { /*.nthread =*/ 0, /*.ftype =*/ LLAMA_FTYPE_MOSTLY_Q5_1, + /*.output_tensor_type =*/ GGML_TYPE_COUNT, + /*.token_embedding_type =*/ GGML_TYPE_COUNT, /*.allow_requantize =*/ false, /*.quantize_output_tensor =*/ true, /*.only_copy =*/ false, diff --git a/llama.h b/llama.h index 7e8ac4b62..74f0e56de 100644 --- a/llama.h +++ b/llama.h @@ -275,13 +275,15 @@ extern "C" { // model quantization parameters typedef struct llama_model_quantize_params { - int32_t nthread; // number of threads to use for quantizing, if <=0 will use std::thread::hardware_concurrency() - enum llama_ftype ftype; // quantize to this llama_ftype - bool allow_requantize; // allow quantizing non-f32/f16 tensors - bool quantize_output_tensor; // quantize output.weight - bool only_copy; // only copy tensors - ftype, allow_requantize and quantize_output_tensor are ignored - bool pure; // quantize all tensors to the default type - void * imatrix; // pointer to importance matrix data + int32_t nthread; // number of threads to use for quantizing, if <=0 will use std::thread::hardware_concurrency() + enum llama_ftype ftype; // quantize to this llama_ftype + enum ggml_type output_tensor_type; // output tensor type + enum ggml_type token_embedding_type; // itoken embeddings tensor type + bool allow_requantize; // allow quantizing non-f32/f16 tensors + bool quantize_output_tensor; // quantize output.weight + bool only_copy; // only copy tensors - ftype, allow_requantize and quantize_output_tensor are ignored + bool pure; // quantize all tensors to the default type + void * imatrix; // pointer to importance matrix data } llama_model_quantize_params; // grammar types From 92397d87a45a09b5449d845a64856f177cd7a920 Mon Sep 17 00:00:00 2001 From: fraxy-v <65565042+fraxy-v@users.noreply.github.com> Date: Fri, 22 Mar 2024 20:49:06 +0200 Subject: [PATCH 703/812] convert-llama2c-to-ggml : enable conversion of GQA models (#6237) * convert-llama2c-to-ggml: enable conversion of multiqueries, #5608 * add test in build action * Update build.yml * Update build.yml * Update build.yml * gg patch --- .github/workflows/build.yml | 11 + examples/convert-llama2c-to-ggml/README.md | 2 + .../convert-llama2c-to-ggml.cpp | 387 ++++++++---------- 3 files changed, 193 insertions(+), 207 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 7711bd8d8..bf42df8fe 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -225,6 +225,17 @@ jobs: cd build ctest -L main --verbose --timeout 900 + - name: Test llama2c conversion + id: llama2c_test + run: | + cd build + echo "Fetch tokenizer" + wget https://huggingface.co/karpathy/tinyllamas/resolve/main/stories260K/tok512.bin + echo "Fetch llama2c model" + wget https://huggingface.co/karpathy/tinyllamas/resolve/main/stories260K/stories260K.bin + ./bin/convert-llama2c-to-ggml --copy-vocab-from-model ./tok512.bin --llama2c-model stories260K.bin --llama2c-output-model stories260K.gguf + ./bin/main -m stories260K.gguf -p "One day, Lily met a Shoggoth" -n 500 -c 256 + # ubuntu-latest-cmake-sanitizer: # runs-on: ubuntu-latest # diff --git a/examples/convert-llama2c-to-ggml/README.md b/examples/convert-llama2c-to-ggml/README.md index 0f37d295b..6da2d7e18 100644 --- a/examples/convert-llama2c-to-ggml/README.md +++ b/examples/convert-llama2c-to-ggml/README.md @@ -21,6 +21,8 @@ An example command using a model from [karpathy/tinyllamas](https://huggingface. `$ ./convert-llama2c-to-ggml --copy-vocab-from-model llama-2-7b-chat.gguf.q2_K.bin --llama2c-model stories42M.bin --llama2c-output-model stories42M.gguf.bin` +Note: The vocabulary for `stories260K.bin` should be its own tokenizer `tok512.bin` found in [karpathy/tinyllamas/stories260K](https://huggingface.co/karpathy/tinyllamas/tree/main/stories260K). + Now you can use the model with a command like: `$ ./main -m stories42M.gguf.bin -p "One day, Lily met a Shoggoth" -n 500 -c 256` diff --git a/examples/convert-llama2c-to-ggml/convert-llama2c-to-ggml.cpp b/examples/convert-llama2c-to-ggml/convert-llama2c-to-ggml.cpp index 8209dcb64..6b5c66530 100644 --- a/examples/convert-llama2c-to-ggml/convert-llama2c-to-ggml.cpp +++ b/examples/convert-llama2c-to-ggml/convert-llama2c-to-ggml.cpp @@ -1,6 +1,7 @@ #include "ggml.h" #include "llama.h" #include "common.h" +#include "log.h" #include #include @@ -78,111 +79,101 @@ typedef struct { struct TransformerWeights { // token embedding table - float* token_embedding_table; // (vocab_size, dim) + std::vector token_embedding_table; // (vocab_size, dim) // weights for rmsnorms - float* rms_att_weight; // (layer, dim) rmsnorm weights - float* rms_ffn_weight; // (layer, dim) + std::vector rms_att_weight; // (layer, dim) rmsnorm weights + std::vector rms_ffn_weight; // (layer, dim) // weights for matmuls - float* wq; // (layer, dim, dim) - float* wk; // (layer, dim, dim) - float* wv; // (layer, dim, dim) - float* wo; // (layer, dim, dim) + std::vector wq; // (layer, dim, dim) + std::vector wk; // (layer, dim, dim) + std::vector wv; // (layer, dim, dim) + std::vector wo; // (layer, dim, dim) // weights for ffn - float* w1; // (layer, hidden_dim, dim) - float* w2; // (layer, dim, hidden_dim) - float* w3; // (layer, hidden_dim, dim) + std::vector w1; // (layer, hidden_dim, dim) + std::vector w2; // (layer, dim, hidden_dim) + std::vector w3; // (layer, hidden_dim, dim) // final rmsnorm - float* rms_final_weight; // (dim,) + std::vector rms_final_weight; // (dim,) // freq_cis for RoPE relatively positional embeddings - // float* freq_cis_real; // (seq_len, dim/2) - // float* freq_cis_imag; // (seq_len, dim/2) + // std::vector freq_cis_real; // (seq_len, dim/2) + // std::vector freq_cis_imag; // (seq_len, dim/2) // (optional) classifier weights for the logits, on the last layer - float* wcls; - - ~TransformerWeights() { - delete[] token_embedding_table; - delete[] rms_att_weight; - delete[] rms_ffn_weight; - delete[] wq; - delete[] wk; - delete[] wv; - delete[] wo; - delete[] w1; - delete[] w2; - delete[] w3; - delete[] rms_final_weight; - delete[] wcls; - } + std::vector wcls; }; -static void malloc_weights(TransformerWeights* w, Config* p, bool shared_weights) { - // we calloc instead of malloc to keep valgrind happy - w->token_embedding_table = new float[p->vocab_size * p->dim](); - printf("[%s:AK] Allocating [%d] x [%d] = [%d] float space for w->token_embedding_table\n",__func__,p->vocab_size , p->dim, p->vocab_size * p->dim); +static void alloc_weights(TransformerWeights * w, const Config * p, bool shared_weights) { + const int n_multiqueries = p->n_kv_heads <= 0 || p->n_kv_heads >= p->n_heads ? 1 : p->n_heads / p->n_kv_heads; + try { + w->token_embedding_table.resize(p->vocab_size * p->dim); + LOG("%s: Allocating [%d] x [%d] = [%d] float space for w->token_embedding_table\n",__func__,p->vocab_size , p->dim, p->vocab_size * p->dim); - w->rms_att_weight = new float[p->n_layers * p->dim](); - printf("[%s:AK] Allocating [%d] x [%d] = [%d] float space for w->rms_att_weight\n",__func__,p->n_layers, p->dim, p->n_layers * p->dim); + w->rms_att_weight.resize(p->n_layers * p->dim); + LOG("%s: Allocating [%d] x [%d] = [%d] float space for w->rms_att_weight\n",__func__,p->n_layers, p->dim, p->n_layers * p->dim); - w->rms_ffn_weight = new float[p->n_layers * p->dim](); - printf("[%s:AK] Allocating [%d] x [%d] = [%d] float space for w->rms_ffn_weight\n",__func__,p->n_layers , p->dim, p->n_layers * p->dim); + w->rms_ffn_weight.resize(p->n_layers * p->dim); + LOG("%s: Allocating [%d] x [%d] = [%d] float space for w->rms_ffn_weight\n",__func__,p->n_layers , p->dim, p->n_layers * p->dim); - w->wq = new float[p->n_layers * p->dim * p->dim](); - printf("[%s:AK] Allocating [%d] x [%d] x [%d] = [%d] float space for w->wq\n",__func__,p->n_layers, p->dim, p->dim, p->n_layers * p->dim * p->dim); + w->wq.resize(p->n_layers * p->dim * p->dim); + LOG("%s: Allocating [%d] x [%d] x [%d] = [%d] float space for w->wq\n",__func__,p->n_layers, p->dim, p->dim, p->n_layers * p->dim * p->dim); - w->wk = new float[p->n_layers * p->dim * p->dim](); - printf("[%s:AK] Allocating [%d] x [%d] x [%d] = [%d] float space for w->wk\n",__func__,p->n_layers, p->dim, p->dim, p->n_layers * p->dim * p->dim); + w->wk.resize(p->n_layers * p->dim * p->dim / n_multiqueries); + LOG("%s: Allocating [%d] x [%d] x [%d] = [%d] float space for w->wk\n",__func__,p->n_layers, p->dim, p->dim / n_multiqueries, p->n_layers * p->dim * p->dim / n_multiqueries); - w->wv = new float[p->n_layers * p->dim * p->dim](); - printf("[%s:AK] Allocating [%d] x [%d] x [%d] = [%d] float space for w->wv\n",__func__, p->n_layers, p->dim, p->dim, p->n_layers * p->dim * p->dim); + w->wv.resize(p->n_layers * p->dim * p->dim / n_multiqueries); + LOG("%s: Allocating [%d] x [%d] x [%d] = [%d] float space for w->wv\n",__func__, p->n_layers, p->dim, p->dim / n_multiqueries, p->n_layers * p->dim * p->dim / n_multiqueries); - w->wo = new float[p->n_layers * p->dim * p->dim](); - printf("[%s:AK] Allocating [%d] x [%d] x [%d] = [%d] float space for w->wo\n",__func__,p->n_layers, p->dim, p->dim, p->n_layers * p->dim * p->dim); + w->wo.resize(p->n_layers * p->dim * p->dim); + LOG("%s: Allocating [%d] x [%d] x [%d] = [%d] float space for w->wo\n",__func__,p->n_layers, p->dim, p->dim, p->n_layers * p->dim * p->dim); - w->w1 = new float[p->n_layers * p->hidden_dim * p->dim](); - printf("[%s:AK] Allocating [%d] x [%d] x [%d] = [%d] float space for w->w1\n",__func__,p->n_layers, p->hidden_dim, p->dim, p->n_layers * p->hidden_dim * p->dim); + w->w1.resize(p->n_layers * p->hidden_dim * p->dim); + LOG("%s: Allocating [%d] x [%d] x [%d] = [%d] float space for w->w1\n",__func__,p->n_layers, p->hidden_dim, p->dim, p->n_layers * p->hidden_dim * p->dim); - w->w2 = new float[p->n_layers * p->hidden_dim * p->dim](); - printf("[%s:AK] Allocating [%d] x [%d] x [%d] = [%d] float space for w->w2\n",__func__,p->n_layers, p->dim, p->hidden_dim, p->n_layers * p->hidden_dim * p->dim); + w->w2.resize(p->n_layers * p->hidden_dim * p->dim); + LOG("%s: Allocating [%d] x [%d] x [%d] = [%d] float space for w->w2\n",__func__,p->n_layers, p->dim, p->hidden_dim, p->n_layers * p->hidden_dim * p->dim); - w->w3 = new float[p->n_layers * p->hidden_dim * p->dim](); - printf("[%s:AK] Allocating [%d] x [%d] x [%d] = [%d] float space for w->w3\n",__func__,p->n_layers, p->hidden_dim, p->dim, p->n_layers * p->hidden_dim * p->dim); + w->w3.resize(p->n_layers * p->hidden_dim * p->dim); + LOG("%s: Allocating [%d] x [%d] x [%d] = [%d] float space for w->w3\n",__func__,p->n_layers, p->hidden_dim, p->dim, p->n_layers * p->hidden_dim * p->dim); - w->rms_final_weight = new float[p->dim](); - printf("[%s:AK] Allocating [%d] float space for w->rms_final_weight\n",__func__,p->dim); + w->rms_final_weight.resize(p->dim); + LOG("%s: Allocating [%d] float space for w->rms_final_weight\n",__func__,p->dim); - if (shared_weights) { - w->wcls = NULL; - } else { - w->wcls = new float[p->vocab_size * p->dim](); - printf("[%s:AK] Allocating [%d] x [%d] = [%d] float space for w->wcls\n",__func__,p->vocab_size , p->dim, p->vocab_size * p->dim); + if (shared_weights) { + w->wcls = {}; + } else { + w->wcls.resize(p->vocab_size * p->dim); + LOG("%s: Allocating [%d] x [%d] = [%d] float space for w->wcls\n",__func__,p->vocab_size , p->dim, p->vocab_size * p->dim); + } + } + catch (std::length_error &) { + die("Invalid configuration. Failed to allocate memory for weights"); } } -static int checkpoint_init_weights(TransformerWeights *w, Config* p, FILE* f, bool shared_weights) { - if (fread(w->token_embedding_table, sizeof(float), p->vocab_size * p->dim, f) != static_cast(p->vocab_size * p->dim)) return 1; - if (fread(w->rms_att_weight, sizeof(float), p->n_layers * p->dim, f) != static_cast(p->n_layers * p->dim)) return 1; - if (fread(w->wq, sizeof(float), p->n_layers * p->dim * p->dim, f) != static_cast(p->n_layers * p->dim * p->dim)) return 1; - if (fread(w->wk, sizeof(float), p->n_layers * p->dim * p->dim, f) != static_cast(p->n_layers * p->dim * p->dim)) return 1; - if (fread(w->wv, sizeof(float), p->n_layers * p->dim * p->dim, f) != static_cast(p->n_layers * p->dim * p->dim)) return 1; - if (fread(w->wo, sizeof(float), p->n_layers * p->dim * p->dim, f) != static_cast(p->n_layers * p->dim * p->dim)) return 1; - if (fread(w->rms_ffn_weight, sizeof(float), p->n_layers * p->dim, f) != static_cast(p->n_layers * p->dim)) return 1; - if (fread(w->w1, sizeof(float), p->n_layers * p->dim * p->hidden_dim, f) != static_cast(p->n_layers * p->dim * p->hidden_dim)) return 1; - if (fread(w->w2, sizeof(float), p->n_layers * p->hidden_dim * p->dim, f) != static_cast(p->n_layers * p->hidden_dim * p->dim)) return 1; - if (fread(w->w3, sizeof(float), p->n_layers * p->dim * p->hidden_dim, f) != static_cast(p->n_layers * p->dim * p->hidden_dim)) return 1; - if (fread(w->rms_final_weight, sizeof(float), p->dim, f) != static_cast(p->dim)) return 1; +static int checkpoint_init_weights(TransformerWeights * w, const Config * p, FILE * f, bool shared_weights) { + if (fread(w->token_embedding_table.data(), sizeof(float), w->token_embedding_table.size(), f) != w->token_embedding_table.size()) return 1; + if (fread(w->rms_att_weight.data(), sizeof(float), w->rms_att_weight.size(), f) != w->rms_att_weight.size()) return 1; + if (fread(w->wq.data(), sizeof(float), w->wq.size(), f) != w->wq.size()) return 1; + if (fread(w->wk.data(), sizeof(float), w->wk.size(), f) != w->wk.size()) return 1; + if (fread(w->wv.data(), sizeof(float), w->wv.size(), f) != w->wv.size()) return 1; + if (fread(w->wo.data(), sizeof(float), w->wo.size(), f) != w->wo.size()) return 1; + if (fread(w->rms_ffn_weight.data(), sizeof(float), w->rms_ffn_weight.size(), f) != w->rms_ffn_weight.size()) return 1; + if (fread(w->w1.data(), sizeof(float), w->w1.size(), f) != w->w1.size()) return 1; + if (fread(w->w2.data(), sizeof(float), w->w2.size(), f) != w->w2.size()) return 1; + if (fread(w->w3.data(), sizeof(float), w->w3.size(), f) != w->w3.size()) return 1; + if (fread(w->rms_final_weight.data(), sizeof(float), w->rms_final_weight.size(), f) != w->rms_final_weight.size()) return 1; // Skip freq_cis_real & freq_cis_imag int head_size = p->dim / p->n_heads; fseek(f, p->seq_len * head_size * sizeof(float), SEEK_CUR); - if (!shared_weights && fread(w->wcls, sizeof(float), p->vocab_size * p->dim, f) != static_cast(p->vocab_size * p->dim)) return 1; + if (!shared_weights && fread(w->wcls.data(), sizeof(float), w->wcls.size(), f) != w->wcls.size()) return 1; // Check we didn't forget to read anything auto curr = ftell(f); fseek(f, 0, SEEK_END); auto end = ftell(f); if (curr != end) { - printf("Error: failed to read the checkpoint file to the end (curr = %ld, end = %ld)\n", curr, end); + LOG("%s: Error: failed to read the checkpoint file to the end (curr = %ld, end = %ld)\n", __func__, curr, end); return 1; } @@ -190,20 +181,20 @@ static int checkpoint_init_weights(TransformerWeights *w, Config* p, FILE* f, bo } static void print_sample_weights(TransformerWeights *w){ - printf("----- Quick print of first of the weight vales of all the variables\n"); - printf("%f\n", w->token_embedding_table[0]); - printf("%f\n", w->rms_att_weight[0]); - printf("%f\n", w->rms_ffn_weight[0]); + LOG("----- Quick print of first of the weight vales of all the variables\n"); + LOG("%f\n", w->token_embedding_table[0]); + LOG("%f\n", w->rms_att_weight[0]); + LOG("%f\n", w->rms_ffn_weight[0]); - printf("%f\n", w->wq[0]); - printf("%f\n", w->wk[0]); - printf("%f\n", w->wv[0]); - printf("%f\n", w->wo[0]); - printf("%f\n", w->w1[0]); - printf("%f\n", w->w2[0]); - printf("%f\n", w->w3[0]); - printf("%f\n", w->rms_att_weight[0]); - if (w->wcls) printf("%f\n", w->wcls[0]); + LOG("%f\n", w->wq[0]); + LOG("%f\n", w->wk[0]); + LOG("%f\n", w->wv[0]); + LOG("%f\n", w->wo[0]); + LOG("%f\n", w->w1[0]); + LOG("%f\n", w->w2[0]); + LOG("%f\n", w->w3[0]); + LOG("%f\n", w->rms_att_weight[0]); + if (!w->wcls.empty()) LOG("%f\n", w->wcls[0]); } //////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -225,14 +216,16 @@ struct llama_vocab { }; struct my_llama_hparams { - uint32_t n_vocab = 32000; - uint32_t n_ctx = 512; // this is provided as user input? - uint32_t n_embd = 4096; - uint32_t n_ff = 11008; - uint32_t n_mult = 4; - uint32_t n_head = 32; - uint32_t n_layer = 32; - uint32_t n_rot = 64; + uint32_t n_vocab = 32000; + uint32_t n_ctx = 512; // this is provided as user input? + uint32_t n_embd = 4096; + uint32_t n_ff = 11008; + uint32_t n_mult = 4; + uint32_t n_head = 32; + uint32_t n_head_kv = 32; + uint32_t n_layer = 32; + uint32_t n_rot = 64; + bool operator!=(const my_llama_hparams& other) const { return memcmp(this, &other, sizeof(my_llama_hparams)); } @@ -325,14 +318,30 @@ struct train_params { }; static void print_params(struct my_llama_hparams * params) { - printf("%s: n_vocab: %u\n", __func__, params->n_vocab); - printf("%s: n_ctx: %u\n", __func__, params->n_ctx); - printf("%s: n_embd: %u\n", __func__, params->n_embd); - printf("%s: n_mult: %u\n", __func__, params->n_mult); - printf("%s: n_head: %u\n", __func__, params->n_head); - printf("%s: n_ff: %u\n", __func__, params->n_ff); - printf("%s: n_layer: %u\n", __func__, params->n_layer); - printf("%s: n_rot: %u\n", __func__, params->n_rot); + LOG("%s: n_vocab: %u\n", __func__, params->n_vocab); + LOG("%s: n_ctx: %u\n", __func__, params->n_ctx); + LOG("%s: n_embd: %u\n", __func__, params->n_embd); + LOG("%s: n_mult: %u\n", __func__, params->n_mult); + LOG("%s: n_head: %u\n", __func__, params->n_head); + LOG("%s: n_head_kv: %u\n", __func__, params->n_head_kv); + LOG("%s: n_ff: %u\n", __func__, params->n_ff); + LOG("%s: n_layer: %u\n", __func__, params->n_layer); + LOG("%s: n_rot: %u\n", __func__, params->n_rot); +} + +static void print_tensor_info(const struct ggml_context * ctx) { + for (auto t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) { + LOG("%s: Allocating ", __func__); + int64_t total = 1; + int i = 0; + for (; i < ggml_n_dims(t); ++i) { + if (i > 0) LOG("x "); + LOG("[%" PRId64 "] ", t->ne[i]); + total *= t->ne[i]; + } + if (i > 1) LOG("= [%" PRId64 "] ", total); + LOG("float space for %s\n", ggml_get_name(t)); + } } static void init_model(struct my_llama_model * model) { @@ -342,6 +351,8 @@ static void init_model(struct my_llama_model * model) { const uint32_t n_layer = hparams.n_layer; const uint32_t n_vocab = hparams.n_vocab; + const uint32_t n_multiqueries = hparams.n_head_kv <= 0 || hparams.n_head_kv >= hparams.n_head ? 1 : hparams.n_head / hparams.n_head_kv; + const uint32_t n_ff = hparams.n_ff; struct ggml_context * ctx = model->ctx; @@ -350,25 +361,8 @@ static void init_model(struct my_llama_model * model) { model->train_tokens = 0; model->tok_embeddings = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, n_embd, n_vocab); - printf("[%s:GG] Allocating [%u] x [%u] = [%u] float space for model->tok_embeddings\n",__func__,n_embd , n_vocab, n_embd * n_vocab); - model->norm = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd); - printf("[%s:GG] Allocating [%u] float space for model->norm\n",__func__,n_embd); - model->output = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, n_embd, n_vocab); - printf("[%s:GG] Allocating [%u] x[%u] = [%u] float space for model->output\n",__func__,n_embd, n_vocab, n_embd * n_vocab); - - // printing the per-layer allocations here so we dont print in the for loop. - printf("[%s:GG] Allocating [%u] x[%u] = [%u] float space for layer.wq for [%u] layers\n",__func__, n_embd, n_embd, n_embd * n_embd, n_layer); - printf("[%s:GG] Allocating [%u] x[%u] = [%u] float space for layer.wk for [%u] layers\n",__func__, n_embd, n_embd, n_embd * n_embd, n_layer); - printf("[%s:GG] Allocating [%u] x[%u] = [%u] float space for layer.wv for [%u] layers\n",__func__, n_embd, n_embd, n_embd * n_embd, n_layer); - printf("[%s:GG] Allocating [%u] x[%u] = [%u] float space for layer.wo for [%u] layers\n",__func__, n_embd, n_embd, n_embd * n_embd, n_layer); - - printf("[%s:GG] Allocating [%u] float space for layer.ffn_norm for [%u] layers\n",__func__,n_embd, n_layer); - - printf("[%s:GG] Allocating [%u] x[%u] = [%u] float space for layer.w1 for [%u] layers\n",__func__, n_ff, n_embd, n_embd * n_ff, n_layer); - printf("[%s:GG] Allocating [%u] x[%u] = [%u] float space for layer.w2 for [%u] layers\n",__func__, n_embd, n_ff, n_ff * n_embd, n_layer); - printf("[%s:GG] Allocating [%u] x[%u] = [%u] float space for layer.w3 for [%u] layers\n",__func__, n_ff, n_embd, n_embd * n_ff, n_layer); ggml_set_name(model->tok_embeddings, "tok_embeddings.weight"); ggml_set_name(model->norm, "norm.weight"); @@ -383,8 +377,8 @@ static void init_model(struct my_llama_model * model) { layer.attention_norm = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd); layer.wq = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, n_embd, n_embd); - layer.wk = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, n_embd, n_embd); - layer.wv = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, n_embd, n_embd); + layer.wk = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, n_embd, n_embd / n_multiqueries); + layer.wv = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, n_embd, n_embd / n_multiqueries); layer.wo = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, n_embd, n_embd); layer.ffn_norm = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd); @@ -406,6 +400,8 @@ static void init_model(struct my_llama_model * model) { ggml_format_name(layer.w2, "%s.feed_forward.w2.weight", layers_i.c_str()); ggml_format_name(layer.w3, "%s.feed_forward.w3.weight", layers_i.c_str()); } + + print_tensor_info(ctx); } static float get_f32_2d(struct ggml_tensor * tensor, int64_t i0, int64_t i1) { @@ -421,9 +417,9 @@ static int32_t get_i32_2d(struct ggml_tensor * tensor, int64_t i0, int64_t i1) { static void print_row(struct ggml_tensor * probs, int i) { for (int k = 0; k < probs->ne[0]; ++k) { float p = get_f32_2d(probs, k, i); - printf(" %f", p); + LOG(" %f", p); } - printf("\n"); + LOG("\n"); } static void print_matrix(struct ggml_tensor * probs) { @@ -431,33 +427,12 @@ static void print_matrix(struct ggml_tensor * probs) { for (int i = 0; i < probs->ne[1]; ++i) { for (int k = 0; k < probs->ne[0]; ++k) { float p = get_f32_2d(probs, k, i); - printf(" %.2f", p); + LOG(" %.2f", p); } - printf("\n"); + LOG("\n"); } } -#ifdef __GNUC__ -#ifdef __MINGW32__ -__attribute__((format(gnu_printf, 1, 2))) -#else -__attribute__((format(printf, 1, 2))) -#endif -#endif -static std::string format(const char * fmt, ...) { - va_list ap, ap2; - va_start(ap, fmt); - va_copy(ap2, ap); - int size = vsnprintf(NULL, 0, fmt, ap); - GGML_ASSERT(size >= 0 && size < INT_MAX); - std::vector buf(size + 1); - int size2 = vsnprintf(buf.data(), size + 1, fmt, ap2); - GGML_ASSERT(size2 == size); - va_end(ap2); - va_end(ap); - return std::string(buf.data(), size); -} - struct llama_file { // use FILE * so we don't have to re-open the file to mmap FILE * fp; @@ -549,8 +524,9 @@ static std::string llama_escape_whitespaces(const std::string & text) { return out.str(); } -static void load_vocab(const char *filename, Config *config, struct llama_vocab *vocab) { +static void load_vocab(const char * filename, const Config * config, struct llama_vocab * vocab) { if (is_ggml_file(filename)) { + LOG("%s: Loading vocabulary from gguf file %s\n", __func__, filename); struct ggml_context * ctx_data = NULL; struct gguf_init_params params = { @@ -578,6 +554,9 @@ static void load_vocab(const char *filename, Config *config, struct llama_vocab const int * toktypes = (const int * ) gguf_get_arr_data(ctx, toktype_idx); const uint32_t n_vocab = gguf_get_arr_n(ctx, token_idx); + if (n_vocab != static_cast(config->vocab_size)) { + die_fmt("vocab size mismatch: (gguf) %u != (llama2c) %d", n_vocab, config->vocab_size); + } vocab->id_to_token.resize(n_vocab); @@ -595,7 +574,7 @@ static void load_vocab(const char *filename, Config *config, struct llama_vocab gguf_free(ctx); } else { // assume llama2.c vocabulary - printf("Assuming llama2.c vocabulary since %s is not a gguf file\n", filename); + LOG("%s: Assuming llama2.c vocabulary since %s is not a gguf file\n", __func__, filename); llama_file file(filename, "rb"); if (!file.fp) { die_fmt("%s: %s", strerror(errno), filename); @@ -638,38 +617,15 @@ static void load_vocab(const char *filename, Config *config, struct llama_vocab } static void convert_weights_ak_to_gg(struct ggml_tensor * gg_weights, const float * karpathy_weights) { - int ct; - switch (ggml_n_dims(gg_weights)) { - case 1: - ct = 0; - for (int i0 = 0; i0 < gg_weights->ne[0]; i0++){ - float * ptr = (float *) ((char *) gg_weights->data + i0*gg_weights->nb[0]); - *ptr = karpathy_weights[ct]; - ct++; - } - break; - case 2: - ct = 0; - for (int i1 = 0; i1 < gg_weights->ne[1]; i1++) { - for (int i0 = 0; i0 < gg_weights->ne[0]; i0++) { - float * ptr = (float *) ((char *) gg_weights->data + i0*gg_weights->nb[0] + i1*gg_weights->nb[1]); - *ptr = karpathy_weights[ct]; - ct++; - } - } - break; - case 3: - ct = 0; - for (int i2 = 0; i2 < gg_weights->ne[2]; i2++) { - for (int i1 = 0; i1 < gg_weights->ne[1]; i1++) { - for (int i0 = 0; i0 < gg_weights->ne[0]; i0++) { - float * ptr = (float *) ((char *) gg_weights->data + i0*gg_weights->nb[0] + i1*gg_weights->nb[1] + i2*gg_weights->nb[2]); - *ptr = karpathy_weights[ct]; - ct++; - } - } - } - break; + int size = 1; + for (int dim = 0; dim < ggml_n_dims(gg_weights); ++dim) { + size *= gg_weights->ne[dim]; + } + for (int ct = 0; ct < size; ++ct) { + int64_t i0 = 0; int64_t i1 = 0; + int64_t i2 = 0; int64_t i3 = 0; + ggml_unravel_index(gg_weights, ct, &i0, &i1, &i2, &i3); + ggml_set_f32_nd(gg_weights, i0, i1, i2, i3, karpathy_weights[ct]); } } @@ -679,16 +635,18 @@ static void save_as_llama_model( // convert AK weights into GG weights one by one. // w->token_embedding_table -> model->tok_embeddings // float* -> struct ggml_tensor - convert_weights_ak_to_gg(model->tok_embeddings, w->token_embedding_table); - convert_weights_ak_to_gg(model->output, w->wcls ? w->wcls : w->token_embedding_table); + convert_weights_ak_to_gg(model->tok_embeddings, w->token_embedding_table.data()); + convert_weights_ak_to_gg(model->output, !w->wcls.empty() ? w->wcls.data() : w->token_embedding_table.data()); - convert_weights_ak_to_gg(model->norm, w->rms_final_weight); + convert_weights_ak_to_gg(model->norm, w->rms_final_weight.data()); //print_row(model->norm, 0); // for rms-att-weight int row_length = model->hparams.n_embd; int n_ff = model->hparams.n_ff; + const uint32_t n_multiqueries = model->hparams.n_head_kv <= 0 || model->hparams.n_head_kv >= model->hparams.n_head ? 1 : model->hparams.n_head / model->hparams.n_head_kv; + for (uint32_t i = 0; i < model->hparams.n_layer; ++i){ auto & layer = model->layers[i]; // 1d @@ -697,9 +655,10 @@ static void save_as_llama_model( // from 3d matrix layer x dim x dim to 2d matrix dim x dim convert_weights_ak_to_gg(layer.wq , &w->wq[i*row_length*row_length]); - convert_weights_ak_to_gg(layer.wk , &w->wk[i*row_length*row_length]); - convert_weights_ak_to_gg(layer.wv , &w->wv[i*row_length*row_length]); convert_weights_ak_to_gg(layer.wo , &w->wo[i*row_length*row_length]); + // from 3d matrix layer x dim x dim to 2d matrix dim x dim / n_multiqueries + convert_weights_ak_to_gg(layer.wk , &w->wk[i*row_length*row_length/n_multiqueries]); + convert_weights_ak_to_gg(layer.wv , &w->wv[i*row_length*row_length/n_multiqueries]); convert_weights_ak_to_gg(layer.w1 , &w->w1[i*row_length*n_ff]); convert_weights_ak_to_gg(layer.w2 , &w->w2[i*n_ff*row_length]); @@ -736,8 +695,8 @@ static void save_as_llama_model( gguf_set_val_u32(ctx, KV_EMBEDDING_LENGTH, model->hparams.n_embd); gguf_set_val_u32(ctx, KV_FEED_FORWARD_LENGTH, model->hparams.n_ff); gguf_set_val_u32(ctx, KV_ATTENTION_HEAD_COUNT, model->hparams.n_head); - // n_head_kv is optional, default to n_head - // gguf_set_val_u32(ctx, KV_ATTENTION_HEAD_COUNT_KV, ...); + gguf_set_val_u32(ctx, KV_ATTENTION_HEAD_COUNT, model->hparams.n_head); + gguf_set_val_u32(ctx, KV_ATTENTION_HEAD_COUNT_KV, model->hparams.n_head_kv); gguf_set_val_u32(ctx, KV_BLOCK_COUNT, model->hparams.n_layer); gguf_set_val_u32(ctx, KV_ROPE_DIMENSION_COUNT, model->hparams.n_rot); gguf_set_val_f32(ctx, KV_ATTENTION_LAYERNORM_RMS_EPS, 1e-5f); @@ -789,12 +748,12 @@ static void save_as_llama_model( static struct train_params get_default_train_params() { struct train_params params; - params.fn_vocab_model = "models/7B/ggml-model-f16.gguf"; + params.fn_vocab_model = "models/7B/ggml-model-f16.gguf"; params.fn_llama2c_output_model = "ak_llama_model.bin"; - params.fn_train_data = "shakespeare.txt"; - params.fn_checkpoint_in = "checkpoint.bin"; - params.fn_checkpoint_out = "checkpoint.bin"; - params.fn_model_out = "ggml-checkpoint-f32.bin"; + params.fn_train_data = "shakespeare.txt"; + params.fn_checkpoint_in = "checkpoint.bin"; + params.fn_checkpoint_out = "checkpoint.bin"; + params.fn_model_out = "ggml-checkpoint-f32.bin"; params.seed = -1; @@ -829,8 +788,8 @@ static struct train_params get_default_train_params() { params.adam_alpha = 1e-3f; params.adam_decay = 1e-3f; - params.mem_model_gb = 2; - params.mem_compute_gb = 24; + params.mem_model_gb = 2; + params.mem_compute_gb = 24; params.mem_compute0_gb = 8; params.mem_compute1_gb = 2; @@ -916,19 +875,30 @@ int main(int argc, char ** argv) { if (!params_parse(argc, argv, ¶ms)) { return 1; } + log_set_target(stdout); Config config; TransformerWeights weights = {}; { - FILE *file = fopen(params.fn_llama2c_model, "rb"); - if (!file) { printf("Unable to open the checkpoint file %s!\n", params.fn_llama2c_model); return 1; } + LOG("%s: Loading llama2c model from %s\n", __func__, params.fn_llama2c_model); + FILE *file = fopen(params.fn_llama2c_model, "r"); + if (!file) { + LOG("%s: Unable to open the checkpoint file %s!\n", __func__, params.fn_llama2c_model); + return 1; + } // read in the config header - if(fread(&config, sizeof(Config), 1, file) != 1) { return 1; } + if (fread(&config, sizeof(Config), 1, file) != 1) { + LOG("%s: Unable to read llama2c config from %s!\n",__func__,params.fn_llama2c_model); + return 1; + } auto shared_weights = config.vocab_size > 0; config.vocab_size = abs(config.vocab_size); // read in the Transformer weights - malloc_weights(&weights, &config, shared_weights); - if(checkpoint_init_weights(&weights, &config, file, shared_weights)) { return 1; } + alloc_weights(&weights, &config, shared_weights); + if (checkpoint_init_weights(&weights, &config, file, shared_weights)) { + LOG("%s: Unable to initialize transformer weights from %s!",__func__,params.fn_llama2c_model); + return 1; + } fclose(file); } @@ -936,15 +906,18 @@ int main(int argc, char ** argv) { load_vocab(params.fn_vocab_model, &config, &vocab); struct my_llama_model model; - model.hparams.n_vocab = config.vocab_size; //llama_n_vocab(lctx); - model.hparams.n_ctx = params.n_ctx; - model.hparams.n_embd = config.dim; //params.n_embd; - model.hparams.n_ff = config.hidden_dim; - model.hparams.n_mult = 32;//params.n_mult; - model.hparams.n_head = config.n_heads; //params.n_head; - model.hparams.n_layer = config.n_layers; //params.n_layer; - model.hparams.n_rot = std::min((uint32_t)params.n_rotmax, model.hparams.n_embd / model.hparams.n_head); + model.hparams.n_vocab = config.vocab_size; //llama_n_vocab(lctx); + model.hparams.n_ctx = params.n_ctx; + model.hparams.n_embd = config.dim; //params.n_embd; + model.hparams.n_ff = config.hidden_dim; + model.hparams.n_mult = 32;//params.n_mult; + model.hparams.n_head = config.n_heads; //params.n_head; + model.hparams.n_head_kv = config.n_kv_heads; + model.hparams.n_layer = config.n_layers; //params.n_layer; + model.hparams.n_rot = std::min((uint32_t)params.n_rotmax, model.hparams.n_embd / model.hparams.n_head); + print_params(&model.hparams); + struct ggml_init_params lcparams; lcparams.mem_size = 1024ll*1024ll*1024ll*((size_t) params.mem_model_gb); lcparams.mem_buffer = NULL; @@ -956,7 +929,7 @@ int main(int argc, char ** argv) { model.name = basename(params.fn_llama2c_model); save_as_llama_model(&vocab, &model, &weights, params.fn_llama2c_output_model); - printf("Saving llama.c model file %s in ggml format at %s\n", params.fn_llama2c_model, params.fn_llama2c_output_model); + LOG("%s: Saving llama.c model file %s in ggml format at %s\n", __func__, params.fn_llama2c_model, params.fn_llama2c_output_model); ggml_free(model.ctx); return 0; From 56a00f0a2f48a85376f48b5ce77699df781631ae Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Fri, 22 Mar 2024 21:10:39 +0200 Subject: [PATCH 704/812] common : default --hf-file to --model (#6234) --- common/common.cpp | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/common/common.cpp b/common/common.cpp index 0cc4859f1..de6eb960a 100644 --- a/common/common.cpp +++ b/common/common.cpp @@ -1220,9 +1220,11 @@ bool gpt_params_parse_ex(int argc, char ** argv, gpt_params & params) { throw std::invalid_argument("error: unknown argument: " + arg); } } + if (invalid_param) { throw std::invalid_argument("error: invalid parameter for argument: " + arg); } + if (params.prompt_cache_all && (params.interactive || params.interactive_first || params.instruct)) { @@ -1230,6 +1232,11 @@ bool gpt_params_parse_ex(int argc, char ** argv, gpt_params & params) { throw std::invalid_argument("error: --prompt-cache-all not supported in interactive mode yet\n"); } + // short-hand to avoid specifying --hf-file -> default it to --model + if (!params.hf_repo.empty() && params.hf_file.empty()) { + params.hf_file = params.model; + } + if (params.escape) { process_escapes(params.prompt); process_escapes(params.input_prefix); From 50ccaf5eacb50a2ca378a4ef0dc7aeb45fead652 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20G=C3=A4=C3=9Fler?= Date: Sat, 23 Mar 2024 01:24:36 +0100 Subject: [PATCH 705/812] lookup: complement data from context with general text statistics (#5479) * lookup: evaluation tools, use corpus/previous gens * fixup! lookup: evaluation tools, use corpus/previous gens * fixup! lookup: evaluation tools, use corpus/previous gens * fixup! lookup: evaluation tools, use corpus/previous gens * fixup! lookup: evaluation tools, use corpus/previous gens --- .gitignore | 3 + Makefile | 13 +- common/CMakeLists.txt | 2 + common/common.cpp | 20 +++ common/common.h | 28 +-- common/ngram-cache.cpp | 280 ++++++++++++++++++++++++++++++ common/ngram-cache.h | 94 ++++++++++ examples/lookup/CMakeLists.txt | 18 ++ examples/lookup/lookup-create.cpp | 43 +++++ examples/lookup/lookup-merge.cpp | 47 +++++ examples/lookup/lookup-stats.cpp | 163 +++++++++++++++++ examples/lookup/lookup.cpp | 116 ++++++++----- scripts/get-wikitext-103.sh | 10 ++ 13 files changed, 774 insertions(+), 63 deletions(-) create mode 100644 common/ngram-cache.cpp create mode 100644 common/ngram-cache.h create mode 100644 examples/lookup/lookup-create.cpp create mode 100644 examples/lookup/lookup-merge.cpp create mode 100644 examples/lookup/lookup-stats.cpp create mode 100755 scripts/get-wikitext-103.sh diff --git a/.gitignore b/.gitignore index 51aa84222..27562f6d7 100644 --- a/.gitignore +++ b/.gitignore @@ -58,6 +58,9 @@ models-mnt /llava-cli /lookahead /lookup +/lookup-create +/lookup-merge +/lookup-stats /main /metal /passkey diff --git a/Makefile b/Makefile index fa112e708..b8b261aba 100644 --- a/Makefile +++ b/Makefile @@ -676,6 +676,9 @@ json-schema-to-grammar.o: common/json-schema-to-grammar.cpp common/json-schema-t train.o: common/train.cpp common/train.h $(CXX) $(CXXFLAGS) -c $< -o $@ +ngram-cache.o: common/ngram-cache.cpp common/ngram-cache.h + $(CXX) $(CXXFLAGS) -c $< -o $@ + libllama.so: llama.o ggml.o $(OBJS) $(CXX) $(CXXFLAGS) -shared -fPIC -o $@ $^ $(LDFLAGS) @@ -683,7 +686,7 @@ libllama.a: llama.o ggml.o $(OBJS) $(COMMON_DEPS) ar rcs libllama.a llama.o ggml.o $(OBJS) $(COMMON_DEPS) clean: - rm -vrf *.o tests/*.o *.so *.a *.dll benchmark-matmult common/build-info.cpp *.dot $(COV_TARGETS) $(BUILD_TARGETS) $(TEST_TARGETS) + rm -vrf *.o tests/*.o *.so *.a *.dll benchmark-matmult lookup-create lookup-merge lookup-stats common/build-info.cpp *.dot $(COV_TARGETS) $(BUILD_TARGETS) $(TEST_TARGETS) find examples pocs -type f -name "*.o" -delete # @@ -813,9 +816,15 @@ lookahead: examples/lookahead/lookahead.cpp ggml.o llama.o $(COMMON_DEPS) $(OBJS $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) -lookup: examples/lookup/lookup.cpp ggml.o llama.o $(COMMON_DEPS) $(OBJS) +lookup: examples/lookup/lookup.cpp ggml.o llama.o ngram-cache.o $(COMMON_DEPS) $(OBJS) $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) + $(CXX) $(CXXFLAGS) -c examples/lookup/lookup-create.cpp -o $(call GET_OBJ_FILE, examples/lookup/lookup-create.cpp) + $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, examples/lookup/lookup-create.cpp) -o lookup-create $(LDFLAGS) + $(CXX) $(CXXFLAGS) -c examples/lookup/lookup-merge.cpp -o $(call GET_OBJ_FILE, examples/lookup/lookup-merge.cpp) + $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, examples/lookup/lookup-merge.cpp) -o lookup-merge $(LDFLAGS) + $(CXX) $(CXXFLAGS) -c examples/lookup/lookup-stats.cpp -o $(call GET_OBJ_FILE, examples/lookup/lookup-stats.cpp) + $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, examples/lookup/lookup-stats.cpp) -o lookup-stats $(LDFLAGS) passkey: examples/passkey/passkey.cpp ggml.o llama.o $(COMMON_DEPS) $(OBJS) $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) diff --git a/common/CMakeLists.txt b/common/CMakeLists.txt index 10951693a..1d840e5f7 100644 --- a/common/CMakeLists.txt +++ b/common/CMakeLists.txt @@ -65,6 +65,8 @@ add_library(${TARGET} STATIC json.hpp train.h train.cpp + ngram-cache.h + ngram-cache.cpp ) if (BUILD_SHARED_LIBS) diff --git a/common/common.cpp b/common/common.cpp index de6eb960a..69c2d5bf7 100644 --- a/common/common.cpp +++ b/common/common.cpp @@ -963,6 +963,22 @@ static bool gpt_params_find_arg(int argc, char ** argv, const std::string & arg, } return true; } + if (arg == "-lcs" || arg == "--lookup-cache-static") { + if (++i >= argc) { + invalid_param = true; + return true; + } + params.lookup_cache_static = argv[i]; + return true; + } + if (arg == "-lcd" || arg == "--lookup-cache-dynamic") { + if (++i >= argc) { + invalid_param = true; + return true; + } + params.lookup_cache_dynamic = argv[i]; + return true; + } if (arg == "--save-all-logits" || arg == "--kl-divergence-base") { if (++i >= argc) { invalid_param = true; @@ -1436,6 +1452,10 @@ void gpt_print_usage(int /*argc*/, char ** argv, const gpt_params & params) { printf(" Hugging Face model file (default: unused)\n"); printf(" -ld LOGDIR, --logdir LOGDIR\n"); printf(" path under which to save YAML logs (no logging if unset)\n"); + printf(" -lcs FNAME, --lookup-cache-static FNAME\n"); + printf(" path to static lookup cache to use for lookup decoding (not updated by generation)\n"); + printf(" -lcd FNAME, --lookup-cache-dynamic FNAME\n"); + printf(" path to dynamic lookup cache to use for lookup decoding (updated by generation)\n"); printf(" --override-kv KEY=TYPE:VALUE\n"); printf(" advanced option to override model metadata by key. may be specified multiple times.\n"); printf(" types: int, float, bool. example: --override-kv tokenizer.ggml.add_bos_token=bool:false\n"); diff --git a/common/common.h b/common/common.h index d827d4df7..afa4cf6d7 100644 --- a/common/common.h +++ b/common/common.h @@ -88,20 +88,22 @@ struct gpt_params { // // sampling parameters struct llama_sampling_params sparams; - std::string model = "models/7B/ggml-model-f16.gguf"; // model path - std::string model_draft = ""; // draft model for speculative decoding - std::string model_alias = "unknown"; // model alias - std::string model_url = ""; // model url to download - std::string hf_repo = ""; // HF repo - std::string hf_file = ""; // HF file - std::string prompt = ""; - std::string prompt_file = ""; // store the external prompt file name - std::string path_prompt_cache = ""; // path to file for saving/loading prompt eval state - std::string input_prefix = ""; // string to prefix user inputs with - std::string input_suffix = ""; // string to suffix user inputs with + std::string model = "models/7B/ggml-model-f16.gguf"; // model path + std::string model_draft = ""; // draft model for speculative decoding + std::string model_alias = "unknown"; // model alias + std::string model_url = ""; // model url to download + std::string hf_repo = ""; // HF repo + std::string hf_file = ""; // HF file + std::string prompt = ""; + std::string prompt_file = ""; // store the external prompt file name + std::string path_prompt_cache = ""; // path to file for saving/loading prompt eval state + std::string input_prefix = ""; // string to prefix user inputs with + std::string input_suffix = ""; // string to suffix user inputs with std::vector antiprompt; // string upon seeing which more user input is prompted - std::string logdir = ""; // directory in which to save YAML log files - std::string logits_file = ""; // file for saving *all* logits + std::string logdir = ""; // directory in which to save YAML log files + std::string lookup_cache_static = ""; // path of static ngram cache file for lookup decoding + std::string lookup_cache_dynamic = ""; // path of dynamic ngram cache file for lookup decoding + std::string logits_file = ""; // file for saving *all* logits std::vector kv_overrides; diff --git a/common/ngram-cache.cpp b/common/ngram-cache.cpp new file mode 100644 index 000000000..20703d306 --- /dev/null +++ b/common/ngram-cache.cpp @@ -0,0 +1,280 @@ +#include "ngram-cache.h" +#include "log.h" + +#include + +void llama_ngram_cache_update(llama_ngram_cache & ngram_cache, int ngram_min, int ngram_max, + std::vector & inp, int nnew, bool print_progress) { + const int64_t t_start_ms = ggml_time_ms(); + const int64_t inp_size = inp.size(); + + const int64_t n_todo = inp_size * (ngram_max - ngram_min + 1); + int64_t n_done = 0; + + for (int64_t ngram_size = ngram_min; ngram_size <= ngram_max; ++ngram_size) { + const int64_t i_start = std::max(inp_size - nnew, ngram_size); + for (int64_t i = i_start; i < inp_size; ++i) { + const int64_t ngram_start = i - ngram_size; + llama_ngram ngram(&inp[ngram_start], ngram_size); + const llama_token token = inp[i]; + + llama_ngram_cache::iterator part_it = ngram_cache.find(ngram); + if (part_it == ngram_cache.end()) { + llama_ngram_cache_part part; + part.emplace(token, 1); + ngram_cache.emplace(ngram, part); + } else { + llama_ngram_cache_part::iterator token_count_it = part_it->second.find(token); + if (token_count_it == part_it->second.end()) { + part_it->second.emplace(token, 1); + } else { + token_count_it->second++; + } + } + ++n_done; + + if (print_progress && n_done % 10000000 == 0) { + const int64_t t_now_ms = ggml_time_ms(); + const int64_t eta_ms = (inp_size*(ngram_max-ngram_min+1) - n_done) * (t_now_ms - t_start_ms) / n_done; + const int64_t eta_min = eta_ms / (60*1000); + const int64_t eta_s = (eta_ms - 60*1000*eta_min) / 1000; + + fprintf(stderr, "%s: %" PRId64 "/%" PRId64 " done, ETA: %02" PRId64 ":%02" PRId64 "\n", __func__, n_done, n_todo, eta_min, eta_s); + } + } + } +} + +// Helper function to get a token from the combined, speculative sequence of inp and draft. +static llama_token get_token(const std::vector & inp, const std::vector & draft, const size_t i) { + return i < inp.size() ? inp[i] : draft[1 + i - inp.size()]; +} + +// If sample size or percentage are below these thresholds the draft is aborted early: +constexpr int draft_min_sample_size_lax[LLAMA_NGRAM_MAX] = { 2, 2, 1, 1}; +constexpr int draft_min_percent_lax[LLAMA_NGRAM_MAX] = {66, 50, 50, 50}; +constexpr int draft_min_sample_size_strict[LLAMA_NGRAM_MAX] = { 4, 3, 2, 2}; +constexpr int draft_min_percent_strict[LLAMA_NGRAM_MAX] = {75, 66, 66, 66}; + +// Helper function that tries to draft a token from only the static ngram cache: +static llama_token try_draft(llama_ngram_cache & nc_static, const llama_ngram ngram_static) { + llama_ngram_cache::iterator part_static_it = nc_static.find(ngram_static); + if (part_static_it == nc_static.end()) { + return -1; + } + const llama_ngram_cache_part part_static = part_static_it->second; + + int max_count_static = 0; + int sum_count_static = 0; + llama_token max_token = -1; + + for (std::pair token_count_static : part_static) { + const llama_token token = token_count_static.first; + const int32_t count_static = token_count_static.second; + + if (count_static > max_count_static) { + max_token = token; + max_count_static = count_static; + } + sum_count_static += count_static; + } + + if (sum_count_static < draft_min_sample_size_lax[LLAMA_NGRAM_STATIC-1]) { + return -1; + } + if (100*max_count_static < draft_min_percent_lax[LLAMA_NGRAM_STATIC-1]*sum_count_static) { + return -1; + } + return max_token; +} + +// Try to draft a token from primary cache (context/dynamic), validate with static cache: +static llama_token try_draft( + llama_ngram_cache & nc_primary, const std::vector & ngrams_primary, llama_ngram_cache_part & part_static, + const int * min_sample_size, const int * min_percent) { + + llama_token drafted_token = -1; + + for (int i = ngrams_primary.size()-1; i >= 0 && drafted_token == -1; --i) { + const llama_ngram ngram_primary = ngrams_primary[i]; + + llama_ngram_cache::iterator part_primary_it = nc_primary.find(ngram_primary); + if (part_primary_it == nc_primary.end()) { + continue; + } + const llama_ngram_cache_part part_primary = part_primary_it->second; + + int max_count_primary = 0; + int max_count_static = 0; + int sum_count_primary = 0; + llama_token max_token = -1; + + for (std::pair token_count_primary : part_primary) { + const llama_token token = token_count_primary.first; + + llama_ngram_cache_part::iterator token_count_static_it = part_static.find(token); + + const int32_t count_primary = token_count_primary.second; + const int32_t count_static = token_count_static_it != part_static.end() ? 100*token_count_static_it->second : 1; + + if (count_primary*count_static > max_count_primary*max_count_static) { + max_token = token; + max_count_primary = count_primary; + max_count_static = count_static; + } + sum_count_primary += count_primary; + } + + if (sum_count_primary < min_sample_size[i]) { + continue; + } + if (100*max_count_primary < min_percent[i]*sum_count_primary) { + continue;; + } + drafted_token = max_token; + } + + return drafted_token; +} + +void llama_ngram_cache_draft( + std::vector & inp, std::vector & draft, int n_draft, int ngram_min, int ngram_max, + llama_ngram_cache & nc_context, llama_ngram_cache & nc_dynamic, llama_ngram_cache & nc_static +) { + GGML_ASSERT(draft.size() == 1); + const int inp_size = inp.size(); + + if (inp_size < LLAMA_NGRAM_STATIC) { + return; + } + + while ((int) draft.size()-1 < n_draft) { + llama_token drafted_token = -1; + + const int ngram_start_static = inp_size-LLAMA_NGRAM_STATIC + draft.size()-1; + llama_ngram ngram_static; + for (int j = ngram_start_static; j < ngram_start_static + LLAMA_NGRAM_STATIC; ++j) { + ngram_static.tokens[j-ngram_start_static] = get_token(inp, draft, j); + } + llama_ngram_cache::iterator part_static_it = nc_static.find(ngram_static); + llama_ngram_cache_part part_static; + if (part_static_it != nc_static.end()) { + part_static = part_static_it->second; + } + + // cd = context + dynamic + std::vector ngrams_cd; + for (int ngram_size_cd = ngram_min; ngram_size_cd <= ngram_max; ++ngram_size_cd) { + const int ngram_start_cd = inp_size-ngram_size_cd + draft.size()-1; + llama_ngram ngram_cd; + for (int j = ngram_start_cd; j < ngram_start_cd + ngram_size_cd; ++j) { + ngram_cd.tokens[j-ngram_start_cd] = get_token(inp, draft, j); + } + ngrams_cd.push_back(ngram_cd); + } + if (drafted_token == -1) { + drafted_token = try_draft(nc_context, ngrams_cd, part_static, draft_min_sample_size_lax, draft_min_percent_lax); + } + if (drafted_token == -1) { + drafted_token = try_draft(nc_dynamic, ngrams_cd, part_static, draft_min_sample_size_strict, draft_min_percent_strict); + } + if (drafted_token == -1) { + drafted_token = try_draft(nc_static, ngram_static); + } + + if (drafted_token == -1) { + break; + } + + LOG(" - draft candidate: token=%d\n", drafted_token); + draft.push_back(drafted_token); + } +} + +void llama_ngram_cache_save(llama_ngram_cache & ngram_cache, std::string & filename) { + std::ofstream file_out(filename, std::ios::binary); + for (std::pair item : ngram_cache) { + const llama_ngram ngram = item.first; + llama_ngram_cache_part token_counts = item.second; + GGML_ASSERT(!token_counts.empty()); + const int32_t ntokens = token_counts.size(); + GGML_ASSERT(ntokens > 0); + + file_out.write(reinterpret_cast(&ngram), sizeof(llama_ngram)); + file_out.write(reinterpret_cast(&ntokens), sizeof(int32_t)); + for (std::pair item2 : token_counts) { + const llama_token token = item2.first; + const int32_t count = item2.second; + GGML_ASSERT(count > 0); + + file_out.write(reinterpret_cast(&token), sizeof(llama_token)); + file_out.write(reinterpret_cast(&count), sizeof(int32_t)); + } + } + +} + +llama_ngram_cache llama_ngram_cache_load(std::string & filename) { + std::ifstream hashmap_file(filename, std::ios::binary); + if (!hashmap_file) { + throw std::ifstream::failure("Unable to open file " + filename); + } + llama_ngram_cache ngram_cache; + + llama_ngram ngram; + int32_t ntokens; + llama_token token; + int32_t count; + + char * ngramc = reinterpret_cast(&ngram); + char * ntokensc = reinterpret_cast(&ntokens); + char * tokenc = reinterpret_cast(&token); + char * countc = reinterpret_cast(&count); + while(hashmap_file.read(ngramc, sizeof(llama_ngram))) { + GGML_ASSERT(!hashmap_file.eof()); + GGML_ASSERT(hashmap_file.read(ntokensc, sizeof(int32_t))); + GGML_ASSERT(ntokens > 0); + llama_ngram_cache_part token_counts; + + for (int i = 0; i < ntokens; ++i) { + GGML_ASSERT(!hashmap_file.eof()); + GGML_ASSERT(hashmap_file.read(tokenc, sizeof(llama_token))); + GGML_ASSERT(!hashmap_file.eof()); + GGML_ASSERT(hashmap_file.read(countc, sizeof(int32_t))); + GGML_ASSERT(count > 0); + token_counts.emplace(token, count); + } + + ngram_cache.emplace(ngram, token_counts); + } + GGML_ASSERT(hashmap_file.eof()); + + return ngram_cache; +} + +void llama_ngram_cache_merge(llama_ngram_cache & ngram_cache_target, llama_ngram_cache & ngram_cache_add) { + for (std::pair ngram_part : ngram_cache_add) { + const llama_ngram ngram = ngram_part.first; + llama_ngram_cache_part part = ngram_part.second; + + llama_ngram_cache::iterator part_merged_it = ngram_cache_target.find(ngram); + if (part_merged_it == ngram_cache_target.end()) { + ngram_cache_target.emplace(ngram, part); + continue; + } + + for (std::pair token_count : part) { + const llama_token token = token_count.first; + const int32_t count = token_count.second; + GGML_ASSERT(count > 0); + + llama_ngram_cache_part::iterator token_count_merged_it = part_merged_it->second.find(token); + if (token_count_merged_it == part_merged_it->second.end()) { + part_merged_it->second.emplace(token, count); + continue; + } + + token_count_merged_it->second += count; + } + } +} diff --git a/common/ngram-cache.h b/common/ngram-cache.h new file mode 100644 index 000000000..e4fa4cbd1 --- /dev/null +++ b/common/ngram-cache.h @@ -0,0 +1,94 @@ +#pragma once + +#include "llama.h" + +#include +#include +#include + +#define LLAMA_NGRAM_MIN 1 +#define LLAMA_NGRAM_MAX 4 +#define LLAMA_NGRAM_STATIC 2 + +// Data structures to map n-grams to empirical token probabilities: + +struct llama_ngram { + llama_token tokens[LLAMA_NGRAM_MAX]; + + llama_ngram() { + for (int i = 0; i < LLAMA_NGRAM_MAX; ++i) { + tokens[i] = -1; + } + } + + llama_ngram(const llama_token * input, const int ngram_size) { + for (int i = 0; i < LLAMA_NGRAM_MAX; ++i) { + tokens[i] = i < ngram_size ? input[i] : -1; + } + } + + bool operator==(const llama_ngram & other) const { + for (int i = 0; i < LLAMA_NGRAM_MAX; ++i) { + if (tokens[i] != other.tokens[i]) { + return false; + } + } + return true; + } +}; + +struct llama_ngram_hash_function { + size_t operator()(const llama_ngram & ngram) const { + size_t hash = 0; + for (int i = 0; i < LLAMA_NGRAM_MAX; ++i) { + hash ^= std::hash{}(ngram.tokens[i]); + } + return hash; + } +}; + +// token -> number of times token has been seen +typedef std::unordered_map llama_ngram_cache_part; + +// n-gram -> empirical distribution of following tokens +typedef std::unordered_map llama_ngram_cache; + + +// Update an ngram cache with tokens. +// ngram_cache: the cache to modify. +// ngram_min/ngram_max: the min/max size of the ngrams to extract from inp_data. +// inp_data: the token sequence with which to update ngram_cache. +// nnew: how many new tokens have been appended to inp_data since the last call to this function. +// print_progress: whether to print progress to stderr. +// +// In order to get correct results inp_data can ONLY BE APPENDED TO. +// Changes in the middle need a complete rebuild. +void llama_ngram_cache_update( + llama_ngram_cache & ngram_cache, int ngram_min, int ngram_max, std::vector & inp_data, int nnew, bool print_progress); + +// Try to draft tokens from ngram caches. +// inp: the tokens generated so far. +// draft: the token sequence to draft. Expected to initially contain the previously sampled token. +// n_draft: maximum number of tokens to add to draft. +// ngram_min/gram_max: the min/max size of the ngrams in nc_context and nc_dynamic. +// nc_context: ngram cache based on current context. +// nc_dynamic: ngram cache based on previous user generations. +// nc_static: ngram cache generated from a large text corpus, used for validation. +void llama_ngram_cache_draft( + std::vector & inp, std::vector & draft, int n_draft, int ngram_min, int ngram_max, + llama_ngram_cache & nc_context, llama_ngram_cache & nc_dynamic, llama_ngram_cache & nc_static); + +// Save an ngram cache to a file. +// ngram_cache: the ngram cache to save. +// filename: the path under which to save the ngram cache. +void llama_ngram_cache_save(llama_ngram_cache & ngram_cache, std::string & filename); + +// Load an ngram cache saved with llama_ngram_cache_save. +// filename: the path from which to load the ngram cache. +// returns: an ngram cache containing the information saved to filename. +llama_ngram_cache llama_ngram_cache_load(std::string & filename); + +// Merge two ngram caches. +// ngram_cache_target: the ngram cache to which to add the information from ngram_cache_add. +// ngram_cache_add: the ngram cache to add to ngram_cache_target. +void llama_ngram_cache_merge(llama_ngram_cache & ngram_cache_target, llama_ngram_cache & ngram_cache_add); diff --git a/examples/lookup/CMakeLists.txt b/examples/lookup/CMakeLists.txt index c060b8f56..b91633f63 100644 --- a/examples/lookup/CMakeLists.txt +++ b/examples/lookup/CMakeLists.txt @@ -3,3 +3,21 @@ add_executable(${TARGET} lookup.cpp) install(TARGETS ${TARGET} RUNTIME) target_link_libraries(${TARGET} PRIVATE common llama ${CMAKE_THREAD_LIBS_INIT}) target_compile_features(${TARGET} PRIVATE cxx_std_11) + +set(TARGET lookup-create) +add_executable(${TARGET} lookup-create.cpp) +install(TARGETS ${TARGET} RUNTIME) +target_link_libraries(${TARGET} PRIVATE common llama ${CMAKE_THREAD_LIBS_INIT}) +target_compile_features(${TARGET} PRIVATE cxx_std_11) + +set(TARGET lookup-merge) +add_executable(${TARGET} lookup-merge.cpp) +install(TARGETS ${TARGET} RUNTIME) +target_link_libraries(${TARGET} PRIVATE common llama ${CMAKE_THREAD_LIBS_INIT}) +target_compile_features(${TARGET} PRIVATE cxx_std_11) + +set(TARGET lookup-stats) +add_executable(${TARGET} lookup-stats.cpp) +install(TARGETS ${TARGET} RUNTIME) +target_link_libraries(${TARGET} PRIVATE common llama ${CMAKE_THREAD_LIBS_INIT}) +target_compile_features(${TARGET} PRIVATE cxx_std_11) diff --git a/examples/lookup/lookup-create.cpp b/examples/lookup/lookup-create.cpp new file mode 100644 index 000000000..46a6bed07 --- /dev/null +++ b/examples/lookup/lookup-create.cpp @@ -0,0 +1,43 @@ +#include "ggml.h" +#include "llama.h" +#include "common.h" +#include "ngram-cache.h" + +#include +#include +#include +#include +#include +#include + +int main(int argc, char ** argv){ + gpt_params params; + + if (!gpt_params_parse(argc, argv, params)) { + return 1; + } + // init llama.cpp + llama_backend_init(); + llama_numa_init(params.numa); + + llama_model * model = NULL; + llama_context * ctx = NULL; + + // load the model + std::tie(model, ctx) = llama_init_from_gpt_params(params); + GGML_ASSERT(model != nullptr); + + // tokenize the prompt + const bool add_bos = llama_should_add_bos_token(model); + + std::vector inp; + inp = ::llama_tokenize(ctx, params.prompt, add_bos, true); + fprintf(stderr, "%s: tokenization done\n", __func__); + + + llama_ngram_cache ngram_cache; + llama_ngram_cache_update(ngram_cache, LLAMA_NGRAM_STATIC, LLAMA_NGRAM_STATIC, inp, inp.size(), true); + fprintf(stderr, "%s: hashing done, writing file to %s\n", __func__, params.lookup_cache_static.c_str()); + + llama_ngram_cache_save(ngram_cache, params.lookup_cache_static); +} diff --git a/examples/lookup/lookup-merge.cpp b/examples/lookup/lookup-merge.cpp new file mode 100644 index 000000000..07c93eb8d --- /dev/null +++ b/examples/lookup/lookup-merge.cpp @@ -0,0 +1,47 @@ +#include "ggml.h" +#include "llama.h" +#include "common.h" +#include "ngram-cache.h" + +#include +#include +#include +#include +#include +#include +#include + +static void print_usage() { + fprintf(stderr, "Merges multiple lookup cache files into a single one.\n"); + fprintf(stderr, "Usage: lookup-merge [--help] lookup_part_1.bin lookup_part_2.bin ... lookup_merged.bin\n"); +} + +int main(int argc, char ** argv){ + if (argc < 3) { + print_usage(); + exit(1); + } + + std::vector args; + args.resize(argc-1); + for (int i = 0; i < argc-1; ++i) { + args[i] = argv[i+1]; + if (args[i] == "-h" || args[i] == "--help") { + print_usage(); + exit(0); + } + } + + fprintf(stderr, "lookup-merge: loading file %s\n", args[0].c_str()); + llama_ngram_cache ngram_cache_merged = llama_ngram_cache_load(args[0]); + + for (size_t i = 1; i < args.size()-1; ++i) { + fprintf(stderr, "lookup-merge: loading file %s\n", args[i].c_str()); + llama_ngram_cache ngram_cache = llama_ngram_cache_load(args[i]); + + llama_ngram_cache_merge(ngram_cache_merged, ngram_cache); + } + + fprintf(stderr, "lookup-merge: saving file %s\n", args.back().c_str()); + llama_ngram_cache_save(ngram_cache_merged, args.back()); +} diff --git a/examples/lookup/lookup-stats.cpp b/examples/lookup/lookup-stats.cpp new file mode 100644 index 000000000..31f227773 --- /dev/null +++ b/examples/lookup/lookup-stats.cpp @@ -0,0 +1,163 @@ +#include "ggml.h" +#include "common.h" +#include "llama.h" +#include "log.h" +#include "ngram-cache.h" + +#include +#include +#include +#include +#include +#include +#include + +int main(int argc, char ** argv){ + gpt_params params; + + if (!gpt_params_parse(argc, argv, params)) { + return 1; + } + + const int n_draft = params.n_draft; + + // init llama.cpp + llama_backend_init(); + llama_numa_init(params.numa); + + llama_model * model = NULL; + llama_context * ctx = NULL; + + // load the model + std::tie(model, ctx) = llama_init_from_gpt_params(params); + llama_set_rng_seed(ctx, params.seed); + GGML_ASSERT(llama_n_vocab(model) < (1 << 16)); + + // tokenize the prompt + const bool add_bos = llama_should_add_bos_token(model); + LOG("add_bos tgt: %d\n", add_bos); + + std::vector inp; + inp = ::llama_tokenize(ctx, params.prompt, add_bos, true); + + llama_ngram_cache ngram_cache_context; + llama_ngram_cache ngram_cache_dynamic; + llama_ngram_cache ngram_cache_static; + int64_t t_draft_flat_us = 0; + int64_t t_draft_us = 0; + + { + const int64_t t_start_draft_us = ggml_time_us(); + + if (!params.lookup_cache_static.empty()) { + try { + ngram_cache_static = llama_ngram_cache_load(params.lookup_cache_static); + } catch (std::ifstream::failure const &) { + fprintf(stderr, "error: failed to open static lookup cache: %s", params.lookup_cache_static.c_str()); + exit(1); + } + } + + if (!params.lookup_cache_dynamic.empty()) { + try { + ngram_cache_dynamic = llama_ngram_cache_load(params.lookup_cache_dynamic); + } catch (std::ifstream::failure const &) {} // if the file does not exist it will simply be created at the end of the program + } + + t_draft_flat_us += ggml_time_us() - t_start_draft_us; + } + + const int n_input = inp.size(); + const int n_ctx = params.n_ctx; + + int n_drafted = 0; + int n_accept = 0; + + const int64_t t_start_ms = ggml_time_ms(); + + // Iterate over input tokens in chunks of size n_ctx. + // Each chunk is treated as if a sequential generation but with pre-determined tokens to ensure reproducibility. + for (int i_start = 0; i_start + n_ctx < n_input; i_start += n_ctx) { + const std::vector inp_slice(inp.begin() + i_start, inp.begin() + i_start + n_ctx); + std::vector pseudo_output; + pseudo_output.push_back(inp_slice[0]); + + while ((int) pseudo_output.size() < n_ctx) { + // Simulate drafting and decoding from draft: + std::vector draft; + draft.push_back(pseudo_output.back()); + + { + const int64_t t_start_draft_us = ggml_time_us(); + llama_ngram_cache_draft(pseudo_output, draft, n_draft, LLAMA_NGRAM_MIN, LLAMA_NGRAM_MAX, ngram_cache_context, ngram_cache_dynamic, ngram_cache_static); + t_draft_us += ggml_time_us() - t_start_draft_us; + } + + n_drafted += draft.size() - 1; + + for (size_t j = 1; j < draft.size() && (int) pseudo_output.size() < n_ctx; ++j) { + const llama_token ground_truth = inp_slice[pseudo_output.size()]; + const llama_token drafted = draft[j]; + + if (ground_truth != drafted) { + break; + } + + ++n_accept; + pseudo_output.push_back(ground_truth); + + { + const int64_t t_start_draft_us = ggml_time_us(); + llama_ngram_cache_update(ngram_cache_context, LLAMA_NGRAM_MIN, LLAMA_NGRAM_MAX, pseudo_output, 1, false); + t_draft_us += ggml_time_us() - t_start_draft_us; + } + } + + // After each simulated batch decoding simulate the sampling of a single token: + if ((int) pseudo_output.size() < n_ctx) { + pseudo_output.push_back(inp_slice[pseudo_output.size()]); + { + const int64_t t_start_draft_us = ggml_time_us(); + llama_ngram_cache_update(ngram_cache_context, LLAMA_NGRAM_MIN, LLAMA_NGRAM_MAX, pseudo_output, 1, false); + t_draft_us += ggml_time_us() - t_start_draft_us; + } + } + + draft.erase(draft.begin()); + + } + if (i_start > 0 && i_start / 100000 != (i_start - n_ctx) / 100000) { + const int64_t t_now_ms = ggml_time_ms(); + const int64_t eta_ms = (n_input - i_start) * (t_now_ms - t_start_ms) / i_start; + const int64_t eta_min = eta_ms / (60*1000); + const int64_t eta_s = (eta_ms - 60*1000*eta_min) / 1000; + + LOG_TEE("lookup-stats: %d/%d done, ETA: %02" PRId64 ":%02" PRId64 "\n", i_start, n_input, eta_min, eta_s); + } + + // After each chunk, update the dynamic ngram cache with the context ngram cache: + llama_ngram_cache_merge(ngram_cache_dynamic, ngram_cache_context); + ngram_cache_context.clear(); + } + + LOG_TEE("\n"); + + LOG_TEE("\n"); + LOG_TEE("n_draft = %d\n", n_draft); + LOG_TEE("n_predict = %d\n", n_input - n_input % n_ctx); + LOG_TEE("n_drafted = %d\n", n_drafted); + LOG_TEE("t_draft_flat = %.2f ms\n", t_draft_flat_us*1e-3); + LOG_TEE("t_draft = %.2f ms, %.2f us per token, %.2f tokens per second\n", + t_draft_us*1e-3, 1.0f*t_draft_us/n_drafted, n_drafted/(1e-6*t_draft_us)); + LOG_TEE("n_accept = %d\n", n_accept); + LOG_TEE("accept = %.3f%%\n", 100.0f * n_accept / n_drafted); + + llama_free(ctx); + llama_free_model(model); + + llama_backend_free(); + + fprintf(stderr, "\n\n"); + + return 0; +} diff --git a/examples/lookup/lookup.cpp b/examples/lookup/lookup.cpp index b53fae110..2e8c35de3 100644 --- a/examples/lookup/lookup.cpp +++ b/examples/lookup/lookup.cpp @@ -1,12 +1,15 @@ -#include "common.h" #include "ggml.h" #include "llama.h" +#include "common.h" +#include "ngram-cache.h" #include #include #include +#include #include #include +#include int main(int argc, char ** argv){ gpt_params params; @@ -15,11 +18,7 @@ int main(int argc, char ** argv){ return 1; } - // max/min n-grams size to search for in prompt - const int ngram_max = 4; - const int ngram_min = 1; - - // length of the candidate / draft sequence, if match is found + // max. number of additional tokens to draft if match is found const int n_draft = params.n_draft; const bool dump_kv_cache = params.dump_kv_cache; @@ -39,6 +38,8 @@ int main(int argc, char ** argv){ // load the model std::tie(model, ctx) = llama_init_from_gpt_params(params); + llama_set_rng_seed(ctx, params.seed); + GGML_ASSERT(llama_n_vocab(model) < (1 << 16)); // tokenize the prompt const bool add_bos = llama_should_add_bos_token(model); @@ -47,6 +48,35 @@ int main(int argc, char ** argv){ std::vector inp; inp = ::llama_tokenize(ctx, params.prompt, add_bos, true); + llama_ngram_cache ngram_cache_context; + llama_ngram_cache ngram_cache_dynamic; + llama_ngram_cache ngram_cache_static; + int64_t t_draft_flat_us = 0; + int64_t t_draft_us = 0; + + { + // Fill up context ngram cache with tokens from user input: + const int64_t t_start_draft_us = ggml_time_us(); + llama_ngram_cache_update(ngram_cache_context, LLAMA_NGRAM_MIN, LLAMA_NGRAM_MAX, inp, inp.size(), false); + + if (!params.lookup_cache_static.empty()) { + try { + ngram_cache_static = llama_ngram_cache_load(params.lookup_cache_static); + } catch (std::ifstream::failure const &) { + fprintf(stderr, "error: failed to open static lookup cache: %s", params.lookup_cache_static.c_str()); + exit(1); + } + } + + if (!params.lookup_cache_dynamic.empty()) { + try { + ngram_cache_dynamic = llama_ngram_cache_load(params.lookup_cache_dynamic); + } catch (std::ifstream::failure const &) {} // if the file does not exist it will simply be created at the end of the program + } + + t_draft_flat_us += ggml_time_us() - t_start_draft_us; + } + const int max_context_size = llama_n_ctx(ctx); const int max_tokens_list_size = max_context_size - 4; @@ -76,8 +106,6 @@ int main(int argc, char ** argv){ int n_drafted = 0; int n_accept = 0; - int64_t t_draft_us = 0; - int n_past = inp.size(); bool has_eos = false; @@ -129,6 +157,12 @@ int main(int argc, char ** argv){ ++n_past; ++i_dft; inp.push_back(id); + { + // Update context ngram cache with the newly accepted token: + const int64_t t_start_draft_us = ggml_time_us(); + llama_ngram_cache_update(ngram_cache_context, LLAMA_NGRAM_MIN, LLAMA_NGRAM_MAX, inp, 1, false); + t_draft_us += ggml_time_us() - t_start_draft_us; + } if (params.use_color) { // color accepted draft token @@ -149,6 +183,12 @@ int main(int argc, char ** argv){ draft.clear(); draft.push_back(id); inp.push_back(id); + { + // Update context ngram cache with the newly accepted token: + const int64_t t_start_draft_us = ggml_time_us(); + llama_ngram_cache_update(ngram_cache_context, LLAMA_NGRAM_MIN, LLAMA_NGRAM_MAX, inp, 1, false); + t_draft_us += ggml_time_us() - t_start_draft_us; + } break; } @@ -163,44 +203,19 @@ int main(int argc, char ** argv){ llama_batch_clear(batch_tgt); llama_batch_add(batch_tgt, draft[0], n_past, { 0 }, true); - // generate n_pred tokens through prompt lookup - auto prompt_lookup = [&]() -> void { - const int inp_size = inp.size(); - for (int ngram_size = ngram_max ; ngram_size > ngram_min; --ngram_size){ - const llama_token * ngram = &inp[inp_size - ngram_size]; - - for (int i = 0; i <= (int) inp_size - (ngram_size * 2); ++i) { - bool match = true; - for (int j = 0; j < ngram_size; ++j) { - if (inp[i + j] != ngram[j]) { - match = false; - break; - } - } - - if (match) { - const int startIdx = i + ngram_size; - const int endIdx = startIdx + n_draft; - if (endIdx < inp_size) { - for (int j = startIdx; j < endIdx; ++j) { - LOG(" - draft candidate %d: %d\n", j, inp[j]); - draft.push_back(inp[j]); - llama_batch_add(batch_tgt, inp[j], n_past + (j - startIdx) + 1, { 0 }, true); - ++n_drafted; - } - return; - } - } - } - } - return; - }; - + // Draft already contains a single token sampled from the model: + GGML_ASSERT(draft.size() == 1); + GGML_ASSERT(draft[0] == inp.back()); const int64_t t_start_draft_us = ggml_time_us(); - prompt_lookup(); + llama_ngram_cache_draft(inp, draft, n_draft, LLAMA_NGRAM_MIN, LLAMA_NGRAM_MAX, ngram_cache_context, ngram_cache_dynamic, ngram_cache_static); + + for (size_t i = 1; i < draft.size(); ++i) { + llama_batch_add(batch_tgt, draft[i], n_past + i, { 0 }, true); + } t_draft_us += ggml_time_us() - t_start_draft_us; + n_drafted += draft.size() - 1; llama_decode(ctx, batch_tgt); ++n_past; @@ -210,19 +225,24 @@ int main(int argc, char ** argv){ auto t_dec_end = ggml_time_us(); + // Update dynamic ngram cache with context ngram cache and save it to disk: + llama_ngram_cache_merge(ngram_cache_dynamic, ngram_cache_context); + llama_ngram_cache_save(ngram_cache_dynamic, params.lookup_cache_dynamic); + LOG_TEE("\n\n"); LOG_TEE("encoded %4d tokens in %8.3f seconds, speed: %8.3f t/s\n", n_input, (t_enc_end - t_enc_start) / 1e6f, inp.size() / ((t_enc_end - t_enc_start) / 1e6f)); LOG_TEE("decoded %4d tokens in %8.3f seconds, speed: %8.3f t/s\n", n_predict, (t_dec_end - t_dec_start) / 1e6f, n_predict / ((t_dec_end - t_dec_start) / 1e6f)); LOG_TEE("\n"); - LOG_TEE("n_draft = %d\n", n_draft); - LOG_TEE("n_predict = %d\n", n_predict); - LOG_TEE("n_drafted = %d\n", n_drafted); - LOG_TEE("t_draft = %.2f ms, %.2f us per token, %.2f tokens per second\n", + LOG_TEE("n_draft = %d\n", n_draft); + LOG_TEE("n_predict = %d\n", n_predict); + LOG_TEE("n_drafted = %d\n", n_drafted); + LOG_TEE("t_draft_flat = %.2f ms\n", t_draft_flat_us*1e-3); + LOG_TEE("t_draft = %.2f ms, %.2f us per token, %.2f tokens per second\n", t_draft_us*1e-3, 1.0f*t_draft_us/n_drafted, n_drafted/(1e-6*t_draft_us)); - LOG_TEE("n_accept = %d\n", n_accept); - LOG_TEE("accept = %.3f%%\n", 100.0f * n_accept / n_drafted); + LOG_TEE("n_accept = %d\n", n_accept); + LOG_TEE("accept = %.3f%%\n", 100.0f * n_accept / n_drafted); LOG_TEE("\ntarget:\n"); llama_print_timings(ctx); diff --git a/scripts/get-wikitext-103.sh b/scripts/get-wikitext-103.sh new file mode 100755 index 000000000..880dd5cbe --- /dev/null +++ b/scripts/get-wikitext-103.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +wget https://s3.amazonaws.com/research.metamind.io/wikitext/wikitext-103-raw-v1.zip + +echo "Usage:" +echo "" +echo " ./perplexity -m model.gguf -f wiki.test.raw [other params]" +echo "" + +exit 0 From 1b26aebe4de4f048ac99996efd8a2c9af150904d Mon Sep 17 00:00:00 2001 From: Pierrick Hymbert Date: Sat, 23 Mar 2024 13:18:45 +0100 Subject: [PATCH 706/812] server: flush stdout after logging in both text and json layout (#6253) --- examples/server/utils.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/server/utils.hpp b/examples/server/utils.hpp index d7eef556a..8f20ff614 100644 --- a/examples/server/utils.hpp +++ b/examples/server/utils.hpp @@ -95,8 +95,8 @@ static inline void server_log(const char *level, const char *function, int line, const std::string str = ss.str(); printf("%.*s\n", (int)str.size(), str.data()); - fflush(stdout); } + fflush(stdout); } // From 21cad01b6e6e1a96f99391f95e8ea8ae25c8288e Mon Sep 17 00:00:00 2001 From: Pierrick Hymbert Date: Sat, 23 Mar 2024 17:18:13 +0100 Subject: [PATCH 707/812] split: add gguf-split in the make build target (#6262) --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index b8b261aba..4f260cc3d 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ # Define the default target now so that it is always the first target BUILD_TARGETS = \ main quantize quantize-stats perplexity imatrix embedding vdot q8dot train-text-from-scratch convert-llama2c-to-ggml \ - simple batched batched-bench save-load-state server gguf llama-bench libllava.a llava-cli baby-llama beam-search \ + simple batched batched-bench save-load-state server gguf gguf-split llama-bench libllava.a llava-cli baby-llama beam-search \ speculative infill tokenize benchmark-matmult parallel finetune export-lora lookahead lookup passkey gritlm tests/test-c.o # Binaries only useful for tests From 476b0251b27fb64c575507024a671e639d675594 Mon Sep 17 00:00:00 2001 From: Julius Arkenberg Date: Sat, 23 Mar 2024 17:41:53 +0100 Subject: [PATCH 708/812] llama : add grok-1 support (#6204) * Add support for Grok model architecture * Revert convert-hf-to-gguf to default options * Fixed f_norm_rms_eps bug * Fix whitespaces * llama : fix grok rope type * llama : minor --------- Co-authored-by: Georgi Gerganov --- convert-hf-to-gguf.py | 26 +++ gguf-py/gguf/constants.py | 24 +++ gguf-py/gguf/tensor_mapping.py | 55 +++--- llama.cpp | 299 +++++++++++++++++++++++++++++++++ 4 files changed, 384 insertions(+), 20 deletions(-) diff --git a/convert-hf-to-gguf.py b/convert-hf-to-gguf.py index 1e49d56c1..723ea18e3 100755 --- a/convert-hf-to-gguf.py +++ b/convert-hf-to-gguf.py @@ -93,31 +93,42 @@ class Model(ABC): if (n_ctx := self.find_hparam(["max_position_embeddings", "n_ctx"], optional=True)) is not None: self.gguf_writer.add_context_length(n_ctx) + print(f"gguf: context length = {n_ctx}") n_embd = self.find_hparam(["hidden_size", "n_embd"]) self.gguf_writer.add_embedding_length(n_embd) + print(f"gguf: embedding length = {n_embd}") if (n_ff := self.find_hparam(["intermediate_size", "n_inner"], optional=True)) is not None: self.gguf_writer.add_feed_forward_length(n_ff) + print(f"gguf: feed forward length = {n_ff}") n_head = self.find_hparam(["num_attention_heads", "n_head"]) self.gguf_writer.add_head_count(n_head) + print(f"gguf: head count = {n_head}") if (n_head_kv := self.hparams.get("num_key_value_heads")) is not None: self.gguf_writer.add_head_count_kv(n_head_kv) + print(f"gguf: key-value head count = {n_head_kv}") if (rope_theta := self.hparams.get("rope_theta")) is not None: self.gguf_writer.add_rope_freq_base(rope_theta) + print(f"gguf: rope theta = {rope_theta}") if (f_rms_eps := self.hparams.get("rms_norm_eps")) is not None: self.gguf_writer.add_layer_norm_rms_eps(f_rms_eps) + print(f"gguf: rms norm epsilon = {f_rms_eps}") if (f_norm_eps := self.find_hparam(["layer_norm_eps", "layer_norm_epsilon", "norm_epsilon"], optional=True)) is not None: self.gguf_writer.add_layer_norm_eps(f_norm_eps) + print(f"gguf: layer norm epsilon = {f_norm_eps}") if (n_experts := self.hparams.get("num_local_experts")) is not None: self.gguf_writer.add_expert_count(n_experts) + print(f"gguf: expert count = {n_experts}") if (n_experts_used := self.hparams.get("num_experts_per_tok")) is not None: self.gguf_writer.add_expert_used_count(n_experts_used) + print(f"gguf: experts used count = {n_experts_used}") self.gguf_writer.add_file_type(self.ftype) + print(f"gguf: file type = {self.ftype}") def write_tensors(self): block_count = self.hparams.get("n_layers", self.hparams.get("num_hidden_layers", self.hparams.get("n_layer"))) @@ -1051,6 +1062,21 @@ class MixtralModel(Model): self._set_vocab_sentencepiece() +@Model.register("GrokForCausalLM") +class GrokModel(Model): + model_arch = gguf.MODEL_ARCH.GROK + + def set_vocab(self): + self._set_vocab_sentencepiece() + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + def set_gguf_parameters(self): + super().set_gguf_parameters() + self.gguf_writer.add_name("Grok") + + @Model.register("MiniCPMForCausalLM") class MiniCPMModel(Model): model_arch = gguf.MODEL_ARCH.MINICPM diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py index 4a4facb06..e47896e2a 100644 --- a/gguf-py/gguf/constants.py +++ b/gguf-py/gguf/constants.py @@ -100,6 +100,7 @@ class MODEL_ARCH(IntEnum): LLAMA = auto() FALCON = auto() BAICHUAN = auto() + GROK = auto() GPT2 = auto() GPTJ = auto() GPTNEOX = auto() @@ -167,6 +168,7 @@ MODEL_ARCH_NAMES: dict[MODEL_ARCH, str] = { MODEL_ARCH.LLAMA: "llama", MODEL_ARCH.FALCON: "falcon", MODEL_ARCH.BAICHUAN: "baichuan", + MODEL_ARCH.GROK: "grok", MODEL_ARCH.GPT2: "gpt2", MODEL_ARCH.GPTJ: "gptj", MODEL_ARCH.GPTNEOX: "gptneox", @@ -251,6 +253,28 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = { MODEL_TENSOR.FFN_DOWN_EXP, MODEL_TENSOR.FFN_UP_EXP, ], + MODEL_ARCH.GROK: [ + MODEL_TENSOR.TOKEN_EMBD, + MODEL_TENSOR.OUTPUT_NORM, + MODEL_TENSOR.OUTPUT, + MODEL_TENSOR.ROPE_FREQS, + MODEL_TENSOR.ATTN_NORM, + MODEL_TENSOR.ATTN_Q, + MODEL_TENSOR.ATTN_K, + MODEL_TENSOR.ATTN_V, + MODEL_TENSOR.ATTN_OUT, + MODEL_TENSOR.ATTN_ROT_EMBD, + MODEL_TENSOR.ATTN_OUT_NORM, + MODEL_TENSOR.FFN_GATE_INP, + MODEL_TENSOR.FFN_NORM, + MODEL_TENSOR.FFN_GATE, + MODEL_TENSOR.FFN_DOWN, + MODEL_TENSOR.FFN_UP, + MODEL_TENSOR.FFN_GATE_EXP, + MODEL_TENSOR.FFN_DOWN_EXP, + MODEL_TENSOR.FFN_UP_EXP, + MODEL_TENSOR.LAYER_OUT_NORM, + ], MODEL_ARCH.GPTNEOX: [ MODEL_TENSOR.TOKEN_EMBD, MODEL_TENSOR.OUTPUT_NORM, diff --git a/gguf-py/gguf/tensor_mapping.py b/gguf-py/gguf/tensor_mapping.py index ed89955d8..11fd34b8b 100644 --- a/gguf-py/gguf/tensor_mapping.py +++ b/gguf-py/gguf/tensor_mapping.py @@ -23,6 +23,7 @@ class TensorNameMap: "model.embedding", # mamba-qbert "backbone.embedding", # mamba "backbone.embeddings", # mamba-hf + "transformer.in_out_embed", # Grok ), # Token type embeddings @@ -66,6 +67,7 @@ class TensorNameMap: "lm_head.ln", # phi2 "model.norm_f", # mamba-qbert "backbone.norm_f", # mamba + "transformer.rms_norm", # Grok ), # Rope frequencies @@ -93,6 +95,7 @@ class TensorNameMap: "model.layers.{bid}.attention_norm", # internlm2 "model.layers.{bid}.norm", # mamba-qbert "backbone.layers.{bid}.norm", # mamba + "transformer.decoder_layer.{bid}.rms_norm", # Grok ), # Attention norm 2 @@ -116,32 +119,35 @@ class TensorNameMap: # Attention query MODEL_TENSOR.ATTN_Q: ( - "model.layers.{bid}.self_attn.q_proj", # llama-hf - "layers.{bid}.attention.wq", # llama-pth - "encoder.layer.{bid}.attention.self.query", # bert - "transformer.h.{bid}.attn.q_proj", # gpt-j - "model.layers.layers.{bid}.self_attn.q_proj", # plamo - "model.layers.{bid}.attention.wq" # internlm2 + "model.layers.{bid}.self_attn.q_proj", # llama-hf + "layers.{bid}.attention.wq", # llama-pth + "encoder.layer.{bid}.attention.self.query", # bert + "transformer.h.{bid}.attn.q_proj", # gpt-j + "model.layers.layers.{bid}.self_attn.q_proj", # plamo + "model.layers.{bid}.attention.wq", # internlm2 + "transformer.decoder_layer.{bid}.multi_head_attention.query" # Grok ), # Attention key MODEL_TENSOR.ATTN_K: ( - "model.layers.{bid}.self_attn.k_proj", # llama-hf - "layers.{bid}.attention.wk", # llama-pth - "encoder.layer.{bid}.attention.self.key", # bert - "transformer.h.{bid}.attn.k_proj", # gpt-j - "model.layers.layers.{bid}.self_attn.k_proj", # plamo - "model.layers.{bid}.attention.wk" # internlm2 + "model.layers.{bid}.self_attn.k_proj", # llama-hf + "layers.{bid}.attention.wk", # llama-pth + "encoder.layer.{bid}.attention.self.key", # bert + "transformer.h.{bid}.attn.k_proj", # gpt-j + "model.layers.layers.{bid}.self_attn.k_proj", # plamo + "model.layers.{bid}.attention.wk", # internlm2 + "transformer.decoder_layer.{bid}.multi_head_attention.key" # Grok ), # Attention value MODEL_TENSOR.ATTN_V: ( - "model.layers.{bid}.self_attn.v_proj", # llama-hf - "layers.{bid}.attention.wv", # llama-pth - "encoder.layer.{bid}.attention.self.value", # bert - "transformer.h.{bid}.attn.v_proj", # gpt-j - "model.layers.layers.{bid}.self_attn.v_proj", # plamo - "model.layers.{bid}.attention.wv" # internlm2 + "model.layers.{bid}.self_attn.v_proj", # llama-hf + "layers.{bid}.attention.wv", # llama-pth + "encoder.layer.{bid}.attention.self.value", # bert + "transformer.h.{bid}.attn.v_proj", # gpt-j + "model.layers.layers.{bid}.self_attn.v_proj", # plamo + "model.layers.{bid}.attention.wv", # internlm2 + "transformer.decoder_layer.{bid}.multi_head_attention.value" # Grok ), # Attention output @@ -162,12 +168,14 @@ class TensorNameMap: "model.layers.layers.{bid}.self_attn.o_proj", # plamo "model.layers.{bid}.attention.wo", # internlm2 "encoder.layers.{bid}.attn.out_proj", # nomic-bert + "transformer.decoder_layer.{bid}.multi_head_attention.linear"# Grok ), # Attention output norm MODEL_TENSOR.ATTN_OUT_NORM: ( "encoder.layer.{bid}.attention.output.LayerNorm", # bert "encoder.layers.{bid}.norm1", # nomic-bert + "transformer.decoder_layer.{bid}.rms_norm_1", # Grok ), # Rotary embeddings @@ -190,11 +198,13 @@ class TensorNameMap: "model.layers.{bid}.ln2", # yi "h.{bid}.ln_2", # gpt2 "model.layers.{bid}.ffn_norm", # internlm2 + "transformer.decoder_layer.{bid}.rms_norm_2", # Grok ), MODEL_TENSOR.FFN_GATE_INP: ( "layers.{bid}.feed_forward.gate", # mixtral "model.layers.{bid}.block_sparse_moe.gate", # mixtral + "transformer.decoder_layer.{bid}.router" # Grok ), # Feed-forward up @@ -223,6 +233,7 @@ class TensorNameMap: MODEL_TENSOR.FFN_UP_EXP: ( "layers.{bid}.feed_forward.experts.{xid}.w3", # mixtral "model.layers.{bid}.block_sparse_moe.experts.{xid}.w3", # mixtral + "transformer.decoder_layer.{bid}.moe.{xid}.linear_v", # Grok ), # AWQ-activation gate @@ -243,6 +254,7 @@ class TensorNameMap: MODEL_TENSOR.FFN_GATE_EXP: ( "layers.{bid}.feed_forward.experts.{xid}.w1", # mixtral "model.layers.{bid}.block_sparse_moe.experts.{xid}.w1", # mixtral + "transformer.decoder_layer.{bid}.moe.{xid}.linear" # Grok ), # Feed-forward down @@ -270,6 +282,8 @@ class TensorNameMap: MODEL_TENSOR.FFN_DOWN_EXP: ( "layers.{bid}.feed_forward.experts.{xid}.w2", # mixtral "model.layers.{bid}.block_sparse_moe.experts.{xid}.w2", # mixtral + "transformer.decoder_layer.{bid}.moe.{xid}.linear_1", # Grok + ), MODEL_TENSOR.ATTN_Q_NORM: ( @@ -287,8 +301,9 @@ class TensorNameMap: ), MODEL_TENSOR.LAYER_OUT_NORM: ( - "encoder.layer.{bid}.output.LayerNorm", # bert - "encoder.layers.{bid}.norm2", # nomic-bert + "encoder.layer.{bid}.output.LayerNorm", # bert + "encoder.layers.{bid}.norm2", # nomic-bert + "transformer.decoder_layer.{bid}.rms_norm_3", # Grok ), MODEL_TENSOR.SSM_IN: ( diff --git a/llama.cpp b/llama.cpp index eedca802b..4e08be18d 100644 --- a/llama.cpp +++ b/llama.cpp @@ -195,6 +195,7 @@ enum llm_arch { LLM_ARCH_LLAMA, LLM_ARCH_FALCON, LLM_ARCH_BAICHUAN, + LLM_ARCH_GROK, LLM_ARCH_GPT2, LLM_ARCH_GPTJ, LLM_ARCH_GPTNEOX, @@ -224,6 +225,7 @@ enum llm_arch { static const std::map LLM_ARCH_NAMES = { { LLM_ARCH_LLAMA, "llama" }, { LLM_ARCH_FALCON, "falcon" }, + { LLM_ARCH_GROK, "grok" }, { LLM_ARCH_GPT2, "gpt2" }, { LLM_ARCH_GPTJ, "gptj" }, { LLM_ARCH_GPTNEOX, "gptneox" }, @@ -494,6 +496,28 @@ static const std::map> LLM_TENSOR_NA { LLM_TENSOR_FFN_UP, "blk.%d.ffn_up" }, }, }, + { + LLM_ARCH_GROK, + { + { LLM_TENSOR_TOKEN_EMBD, "token_embd" }, + { LLM_TENSOR_OUTPUT_NORM, "output_norm" }, + { LLM_TENSOR_OUTPUT, "output" }, + { LLM_TENSOR_ROPE_FREQS, "rope_freqs" }, + { LLM_TENSOR_ATTN_NORM, "blk.%d.attn_norm" }, + { LLM_TENSOR_ATTN_Q, "blk.%d.attn_q" }, + { LLM_TENSOR_ATTN_K, "blk.%d.attn_k" }, + { LLM_TENSOR_ATTN_V, "blk.%d.attn_v" }, + { LLM_TENSOR_ATTN_OUT, "blk.%d.attn_output" }, + { LLM_TENSOR_ATTN_ROT_EMBD, "blk.%d.attn_rot_embd" }, + { LLM_TENSOR_FFN_GATE_INP, "blk.%d.ffn_gate_inp" }, + { LLM_TENSOR_FFN_NORM, "blk.%d.ffn_norm" }, + { LLM_TENSOR_FFN_GATE_EXP, "blk.%d.ffn_gate.%d" }, + { LLM_TENSOR_FFN_DOWN_EXP, "blk.%d.ffn_down.%d" }, + { LLM_TENSOR_FFN_UP_EXP, "blk.%d.ffn_up.%d" }, + { LLM_TENSOR_LAYER_OUT_NORM, "blk.%d.layer_output_norm" }, + { LLM_TENSOR_ATTN_OUT_NORM, "blk.%d.attn_output_norm" }, + }, + }, { LLM_ARCH_GPT2, { @@ -1635,6 +1659,7 @@ enum e_model { MODEL_40B, MODEL_65B, MODEL_70B, + MODEL_314B, MODEL_SMALL, MODEL_MEDIUM, MODEL_LARGE, @@ -3419,6 +3444,7 @@ static const char * llama_model_type_name(e_model type) { case MODEL_40B: return "40B"; case MODEL_65B: return "65B"; case MODEL_70B: return "70B"; + case MODEL_314B: return "314B"; case MODEL_SMALL: return "0.1B"; case MODEL_MEDIUM: return "0.4B"; case MODEL_LARGE: return "0.8B"; @@ -3557,6 +3583,15 @@ static void llm_load_hparams( default: model.type = e_model::MODEL_UNKNOWN; } } break; + case LLM_ARCH_GROK: + { + ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps); + + switch (hparams.n_layer) { + case 64: model.type = e_model::MODEL_314B; break; + default: model.type = e_model::MODEL_UNKNOWN; + } + } break; case LLM_ARCH_FALCON: { ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps); @@ -4394,6 +4429,54 @@ static bool llm_load_tensors( } } } break; + case LLM_ARCH_GROK: + { + model.tok_embd = ml.create_tensor(ctx_input, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}); + + // output + { + model.output_norm = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}); + model.output = ml.create_tensor(ctx_output_split, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, false); + // if output is NULL, init from the input tok embed + if (model.output == NULL) { + model.output = ml.create_tensor(ctx_output, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}); + ml.n_created--; // artificial tensor + ml.size_data += ggml_nbytes(model.output); + } + } + + for (int i = 0; i < n_layer; ++i) { + ggml_context * ctx_layer = ctx_for_layer(i); + ggml_context * ctx_split = ctx_for_layer_split(i); + + auto & layer = model.layers[i]; + + layer.attn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}); + + layer.wq = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_embd}); + layer.wk = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_K, "weight", i), {n_embd, n_embd_gqa}); + layer.wv = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_V, "weight", i), {n_embd, n_embd_gqa}); + layer.wo = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd}); + + layer.attn_out_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_OUT_NORM, "weight", i), {n_embd}); + + layer.ffn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}); + + layer.ffn_gate_inp = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_GATE_INP, "weight", i), {n_embd}); + + GGML_ASSERT(hparams.n_expert > 0); + GGML_ASSERT(hparams.n_expert_used > 0); + + // MoE branch + for (uint32_t x = 0; x < hparams.n_expert; ++x) { + layer.ffn_gate_exp[x] = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_GATE_EXP, "weight", i, x), {n_embd, n_ff}); + layer.ffn_down_exp[x] = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_DOWN_EXP, "weight", i, x), { n_ff, n_embd}); + layer.ffn_up_exp[x] = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_UP_EXP, "weight", i, x), {n_embd, n_ff}); + } + + layer.layer_out_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_LAYER_OUT_NORM, "weight", i), {n_embd}); + } + } break; case LLM_ARCH_BAICHUAN: { model.tok_embd = ml.create_tensor(ctx_input, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}); @@ -5621,6 +5704,20 @@ static struct ggml_tensor * llm_build_kqv( ggml_mul_mat_set_prec(kq, GGML_PREC_F32); } + if (model.arch == LLM_ARCH_GROK) { + // need to do the following: + // multiply by attn_output_multiplyer of 0.08838834764831845 + // and then : + // kq = 30 * tanh(kq / 30) + // before the softmax below + + //try from phi2 + //ggml_mul_mat_set_prec(kq, GGML_PREC_F32); + + kq = ggml_tanh(ctx, ggml_scale(ctx, kq, 0.08838834764831845f/30.0f)); + kq = ggml_scale(ctx, kq, 30); + } + #if defined(GGML_USE_KOMPUTE) #pragma message("TODO: ALiBi support in ggml_soft_max_ext is not implemented for Kompute") #pragma message(" Falling back to ggml_alibi(). Will become an error in Mar 2024") @@ -6395,6 +6492,203 @@ struct llm_build_context { return gf; } + struct ggml_cgraph * build_grok() { + struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, LLAMA_MAX_NODES, false); + + const int64_t n_embd_head = hparams.n_embd_head_v; + GGML_ASSERT(n_embd_head == hparams.n_embd_head_k); + GGML_ASSERT(n_embd_head == hparams.n_rot); + + struct ggml_tensor * cur; + struct ggml_tensor * inpL; + + inpL = llm_build_inp_embd(ctx0, lctx, hparams, batch, model.tok_embd, cb); + + // multiply by embedding_multiplier_scale of 78.38367176906169 + inpL = ggml_scale(ctx0, inpL, 78.38367176906169f); + + // inp_pos - contains the positions + struct ggml_tensor * inp_pos = build_inp_pos(); + + // KQ_mask (mask for 1 head, it will be broadcasted to all heads) + struct ggml_tensor * KQ_mask = build_inp_KQ_mask(); + + for (int il = 0; il < n_layer; ++il) { + struct ggml_tensor * inpSA = inpL; + + // norm + cur = llm_build_norm(ctx0, inpL, hparams, + model.layers[il].attn_norm, NULL, + LLM_NORM_RMS, cb, il); + cb(cur, "attn_norm", il); + + + // self-attention + { + // compute Q and K and RoPE them + struct ggml_tensor * Qcur = ggml_mul_mat(ctx0, model.layers[il].wq, cur); + cb(Qcur, "Qcur", il); + if (model.layers[il].bq) { + Qcur = ggml_add(ctx0, Qcur, model.layers[il].bq); + cb(Qcur, "Qcur", il); + } + + struct ggml_tensor * Kcur = ggml_mul_mat(ctx0, model.layers[il].wk, cur); + cb(Kcur, "Kcur", il); + if (model.layers[il].bk) { + Kcur = ggml_add(ctx0, Kcur, model.layers[il].bk); + cb(Kcur, "Kcur", il); + } + + struct ggml_tensor * Vcur = ggml_mul_mat(ctx0, model.layers[il].wv, cur); + cb(Vcur, "Vcur", il); + if (model.layers[il].bv) { + Vcur = ggml_add(ctx0, Vcur, model.layers[il].bv); + cb(Vcur, "Vcur", il); + } + + Qcur = ggml_rope_custom( + ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, + n_rot, rope_type, 0, n_orig_ctx, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow + ); + cb(Qcur, "Qcur", il); + + Kcur = ggml_rope_custom( + ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, + n_rot, rope_type, 0, n_orig_ctx, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow + ); + cb(Kcur, "Kcur", il); + + cur = llm_build_kv(ctx0, model, hparams, kv_self, gf, + model.layers[il].wo, model.layers[il].bo, + Kcur, Vcur, Qcur, KQ_mask, nullptr, n_ctx, n_tokens, kv_head, n_kv, 1.0f, cb, il); + } + + // Grok + // if attn_out_norm is present then apply it before adding the input + if (model.layers[il].attn_out_norm) { + cur = llm_build_norm(ctx0, cur, hparams, + model.layers[il].attn_out_norm, NULL, + LLM_NORM_RMS, cb, il); + cb(cur, "attn_out_norm", il); + } + + struct ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA); + cb(ffn_inp, "ffn_inp", il); + + // feed-forward network + // MoE branch + cur = llm_build_norm(ctx0, ffn_inp, hparams, + model.layers[il].ffn_norm, NULL, + LLM_NORM_RMS, cb, il); + cb(cur, "ffn_norm", il); + + ggml_tensor * logits = ggml_mul_mat(ctx0, model.layers[il].ffn_gate_inp, cur); // [n_tokens, num_experts] + cb(logits, "ffn_moe_logits", il); + + ggml_tensor * probs = ggml_soft_max(ctx0, logits); // [n_tokens, num_experts] + cb(probs, "ffn_moe_probs", il); + + // select experts + ggml_tensor * selected_experts = ggml_top_k(ctx0, probs, n_expert_used); // [n_tokens, num_experts_per_tok] + cb(selected_experts->src[0], "ffn_moe_argsort", il); + + ggml_tensor * weights = ggml_get_rows(ctx0, + ggml_reshape_3d(ctx0, probs, 1, n_expert, n_tokens), selected_experts); + cb(weights, "ffn_moe_weights", il); + + weights = ggml_reshape_2d(ctx0, weights, n_expert_used, n_tokens); // [n_tokens, num_experts_per_tok] + + ggml_tensor * weights_sum = ggml_sum_rows(ctx0, weights); + cb(weights_sum, "ffn_moe_weights_sum", il); + + weights = ggml_div(ctx0, weights, weights_sum); // [n_tokens, num_experts_per_tok] + cb(weights, "ffn_moe_weights_norm", il); + + // compute expert outputs + ggml_tensor * moe_out = nullptr; + + for (int i = 0; i < n_expert_used; ++i) { + ggml_tensor * cur_expert; + + ggml_tensor * cur_up = ggml_mul_mat_id(ctx0, model.layers[il].ffn_up_exp, n_expert, selected_experts, i, cur); + cb(cur_up, "ffn_moe_up", il); + + ggml_tensor * cur_gate = ggml_mul_mat_id(ctx0, model.layers[il].ffn_gate_exp, n_expert, selected_experts, i, cur); + cb(cur_gate, "ffn_moe_gate", il); + + //GeLU + cur_gate = ggml_gelu(ctx0, cur_gate); + cb(cur_gate, "ffn_moe_gelu", il); + + cur_expert = ggml_mul(ctx0, cur_up, cur_gate); // [n_tokens, n_embd] + cb(cur_expert, "ffn_moe_gate_par", il); + + cur_expert = ggml_mul_mat_id(ctx0, model.layers[il].ffn_down_exp, n_expert, selected_experts, i, cur_expert); // [n_tokens, n_embd] + cb(cur_expert, "ffn_moe_down", il); + + cur_expert = ggml_mul(ctx0, cur_expert, + ggml_view_2d(ctx0, weights, 1, n_tokens, weights->nb[1], i*weights->nb[0])); + cb(cur_expert, "ffn_moe_weighted", il); + + if (i == 0) { + moe_out = cur_expert; + } else { + moe_out = ggml_add(ctx0, moe_out, cur_expert); + cb(moe_out, "ffn_moe_out", il); + } + } + + cur = moe_out; + + // Grok + // if layer_out_norm is present then apply it before adding the input + // Idea: maybe ffn_out_norm is a better name + if (model.layers[il].layer_out_norm) { + cur = llm_build_norm(ctx0, cur, hparams, + model.layers[il].layer_out_norm, NULL, + LLM_NORM_RMS, cb, il); + cb(cur, "layer_out_norm", il); + } + + + cur = ggml_add(ctx0, cur, ffn_inp); + cb(cur, "ffn_out", il); + + ggml_tensor * layer_dir = lctx.cvec.tensor_for(il); + if (layer_dir != nullptr) { + cur = ggml_add(ctx0, cur, layer_dir); + } + cb(cur, "l_out", il); + + // input for next layer + inpL = cur; + } + + cur = inpL; + + cur = llm_build_norm(ctx0, cur, hparams, + model.output_norm, NULL, + LLM_NORM_RMS, cb, -1); + cb(cur, "result_norm", -1); + + // lm_head + cur = ggml_mul_mat(ctx0, model.output, cur); + + // Grok + // multiply logits by output_multiplier_scale of 0.5773502691896257 + + cur = ggml_scale(ctx0, cur, 0.5773502691896257f); + + cb(cur, "result_output", -1); + + ggml_build_forward_expand(gf, cur); + + return gf; + } + struct ggml_cgraph * build_starcoder() { struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, LLAMA_MAX_NODES, false); @@ -8818,6 +9112,10 @@ static struct ggml_cgraph * llama_build_graph( { result = llm.build_falcon(); } break; + case LLM_ARCH_GROK: + { + result = llm.build_grok(); + } break; case LLM_ARCH_STARCODER: { result = llm.build_starcoder(); @@ -13561,6 +13859,7 @@ enum llama_rope_type llama_rope_type(const struct llama_model * model) { // the pairs of head values are offset by n_rot/2 case LLM_ARCH_FALCON: + case LLM_ARCH_GROK: case LLM_ARCH_PERSIMMON: case LLM_ARCH_BERT: case LLM_ARCH_NOMIC_BERT: From 1997577d5e121568ae39f538021733ccd4278c23 Mon Sep 17 00:00:00 2001 From: Pierrick Hymbert Date: Sat, 23 Mar 2024 18:00:38 +0100 Subject: [PATCH 709/812] server: docs: `--threads` and `--threads`, `--ubatch-size`, `--log-disable` (#6254) --- examples/server/README.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/examples/server/README.md b/examples/server/README.md index 355601ff4..8aa4eac9d 100644 --- a/examples/server/README.md +++ b/examples/server/README.md @@ -16,8 +16,8 @@ The project is under active development, and we are [looking for feedback and co **Command line options:** -- `--threads N`, `-t N`: Set the number of threads to use during generation. -- `-tb N, --threads-batch N`: Set the number of threads to use during batch and prompt processing. If not specified, the number of threads will be set to the number of threads used for generation. +- `--threads N`, `-t N`: Set the number of threads to use during generation. Not used if model layers are offloaded to GPU. The server is using batching, this parameter is used only if one token is to be processed on CPU backend. +- `-tb N, --threads-batch N`: Set the number of threads to use during batch and prompt processing. If not specified, the number of threads will be set to the number of threads used for generation. Not used if model layers are offloaded to GPU. - `--threads-http N`: number of threads in the http server pool to process requests (default: `max(std::thread::hardware_concurrency() - 1, --parallel N + 2)`) - `-m FNAME`, `--model FNAME`: Specify the path to the LLaMA model file (e.g., `models/7B/ggml-model.gguf`). - `-mu MODEL_URL --model-url MODEL_URL`: Specify a remote http url to download the file (e.g https://huggingface.co/ggml-org/models/resolve/main/phi-2/ggml-model-q4_0.gguf). @@ -26,7 +26,8 @@ The project is under active development, and we are [looking for feedback and co - `-ngl N`, `--n-gpu-layers N`: When compiled with appropriate support (currently CLBlast or cuBLAS), this option allows offloading some layers to the GPU for computation. Generally results in increased performance. - `-mg i, --main-gpu i`: When using multiple GPUs this option controls which GPU is used for small tensors for which the overhead of splitting the computation across all GPUs is not worthwhile. The GPU in question will use slightly more VRAM to store a scratch buffer for temporary results. By default GPU 0 is used. Requires cuBLAS. - `-ts SPLIT, --tensor-split SPLIT`: When using multiple GPUs this option controls how large tensors should be split across all GPUs. `SPLIT` is a comma-separated list of non-negative values that assigns the proportion of data that each GPU should get in order. For example, "3,2" will assign 60% of the data to GPU 0 and 40% to GPU 1. By default the data is split in proportion to VRAM but this may not be optimal for performance. Requires cuBLAS. -- `-b N`, `--batch-size N`: Set the batch size for prompt processing. Default: `512`. +- `-b N`, `--batch-size N`: Set the batch size for prompt processing. Default: `2048`. +- `-ub N`, `--ubatch-size N`: physical maximum batch size. Default: `512`. - `--memory-f32`: Use 32-bit floats instead of 16-bit floats for memory key+value. Not recommended. - `--mlock`: Lock the model in memory, preventing it from being swapped out when memory-mapped. - `--no-mmap`: Do not memory-map the model. By default, models are mapped into memory, which allows the system to load only the necessary parts of the model as needed. @@ -57,7 +58,7 @@ see https://github.com/ggerganov/llama.cpp/issues/1437 - `--slots-endpoint-disable`: To disable slots state monitoring endpoint. Slots state may contain user data, prompts included. - `--metrics`: enable prometheus `/metrics` compatible endpoint (default: disabled) - `--chat-template JINJA_TEMPLATE`: Set custom jinja chat template. This parameter accepts a string, not a file name (default: template taken from model's metadata). We only support [some pre-defined templates](https://github.com/ggerganov/llama.cpp/wiki/Templates-supported-by-llama_chat_apply_template) -- `--log-disable`: Output logs to stdout only, default: enabled. +- `--log-disable`: Output logs to stdout only, not to `llama.log`. default: enabled. - `--log-format FORMAT`: Define the log output to FORMAT: json or text (default: json) **If compiled with `LLAMA_SERVER_SSL=ON`** From f482bb2e4920e544651fb832f2e0bcb4d2ff69ab Mon Sep 17 00:00:00 2001 From: Pierrick Hymbert Date: Sat, 23 Mar 2024 18:07:00 +0100 Subject: [PATCH 710/812] common: llama_load_model_from_url split support (#6192) * llama: llama_split_prefix fix strncpy does not include string termination common: llama_load_model_from_url: - fix header name case sensitive - support downloading additional split in parallel - hide password in url * common: EOL EOF * common: remove redundant LLAMA_CURL_MAX_PATH_LENGTH definition * common: change max url max length * common: minor comment * server: support HF URL options * llama: llama_model_loader fix log * common: use a constant for max url length * common: clean up curl if file cannot be loaded in gguf * server: tests: add split tests, and HF options params * common: move llama_download_hide_password_in_url inside llama_download_file as a lambda * server: tests: enable back Release test on PR * spacing Co-authored-by: Georgi Gerganov * spacing Co-authored-by: Georgi Gerganov * spacing Co-authored-by: Georgi Gerganov --------- Co-authored-by: Georgi Gerganov --- .github/workflows/server.yml | 1 - common/common.cpp | 205 ++++++++++++++---- common/common.h | 7 + examples/gguf-split/gguf-split.cpp | 4 - examples/server/README.md | 4 +- examples/server/server.cpp | 18 +- .../server/tests/features/parallel.feature | 3 +- examples/server/tests/features/server.feature | 4 +- examples/server/tests/features/steps/steps.py | 13 +- llama.cpp | 4 +- 10 files changed, 200 insertions(+), 63 deletions(-) diff --git a/.github/workflows/server.yml b/.github/workflows/server.yml index b74dc5e21..f07d25536 100644 --- a/.github/workflows/server.yml +++ b/.github/workflows/server.yml @@ -35,7 +35,6 @@ jobs: include: - build_type: Release sanitizer: "" - disabled_on_pr: true fail-fast: false # While -DLLAMA_SANITIZE_THREAD=ON is broken container: diff --git a/common/common.cpp b/common/common.cpp index 69c2d5bf7..fb80d4bf7 100644 --- a/common/common.cpp +++ b/common/common.cpp @@ -39,6 +39,9 @@ #endif #if defined(LLAMA_USE_CURL) #include +#include +#include +#include #endif #if defined(_MSC_VER) @@ -61,7 +64,7 @@ #else #include #endif -#define LLAMA_CURL_MAX_PATH_LENGTH PATH_MAX +#define LLAMA_CURL_MAX_URL_LENGTH 2084 // Maximum URL Length in Chrome: 2083 #define LLAMA_CURL_MAX_HEADER_LENGTH 256 #endif // LLAMA_USE_CURL @@ -1702,27 +1705,13 @@ void llama_batch_add( #ifdef LLAMA_USE_CURL -struct llama_model * llama_load_model_from_url( - const char * model_url, - const char * path_model, - const struct llama_model_params & params) { - // Basic validation of the model_url - if (!model_url || strlen(model_url) == 0) { - fprintf(stderr, "%s: invalid model_url\n", __func__); - return NULL; - } - - // Initialize libcurl globally - auto curl = curl_easy_init(); - - if (!curl) { - fprintf(stderr, "%s: error initializing libcurl\n", __func__); - return NULL; - } +static bool llama_download_file(CURL * curl, const char * url, const char * path) { + bool force_download = false; // Set the URL, allow to follow http redirection - curl_easy_setopt(curl, CURLOPT_URL, model_url); + curl_easy_setopt(curl, CURLOPT_URL, url); curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L); + #if defined(_WIN32) // CURLSSLOPT_NATIVE_CA tells libcurl to use standard certificate store of // operating system. Currently implemented under MS-Windows. @@ -1731,16 +1720,16 @@ struct llama_model * llama_load_model_from_url( // Check if the file already exists locally struct stat model_file_info; - auto file_exists = (stat(path_model, &model_file_info) == 0); + auto file_exists = (stat(path, &model_file_info) == 0); // If the file exists, check for ${path_model}.etag or ${path_model}.lastModified files char etag[LLAMA_CURL_MAX_HEADER_LENGTH] = {0}; - char etag_path[LLAMA_CURL_MAX_PATH_LENGTH] = {0}; - snprintf(etag_path, sizeof(etag_path), "%s.etag", path_model); + char etag_path[PATH_MAX] = {0}; + snprintf(etag_path, sizeof(etag_path), "%s.etag", path); char last_modified[LLAMA_CURL_MAX_HEADER_LENGTH] = {0}; - char last_modified_path[LLAMA_CURL_MAX_PATH_LENGTH] = {0}; - snprintf(last_modified_path, sizeof(last_modified_path), "%s.lastModified", path_model); + char last_modified_path[PATH_MAX] = {0}; + snprintf(last_modified_path, sizeof(last_modified_path), "%s.lastModified", path); if (file_exists) { auto * f_etag = fopen(etag_path, "r"); @@ -1748,7 +1737,7 @@ struct llama_model * llama_load_model_from_url( if (!fgets(etag, sizeof(etag), f_etag)) { fprintf(stderr, "%s: unable to read file %s\n", __func__, etag_path); } else { - fprintf(stderr, "%s: previous model file found %s: %s\n", __func__, etag_path, etag); + fprintf(stderr, "%s: previous file found %s: %s\n", __func__, etag_path, etag); } fclose(f_etag); } @@ -1758,7 +1747,7 @@ struct llama_model * llama_load_model_from_url( if (!fgets(last_modified, sizeof(last_modified), f_last_modified)) { fprintf(stderr, "%s: unable to read file %s\n", __func__, last_modified_path); } else { - fprintf(stderr, "%s: previous model file found %s: %s\n", __func__, last_modified_path, + fprintf(stderr, "%s: previous file found %s: %s\n", __func__, last_modified_path, last_modified); } fclose(f_last_modified); @@ -1776,6 +1765,11 @@ struct llama_model * llama_load_model_from_url( auto header_callback = [](char * buffer, size_t /*size*/, size_t n_items, void * userdata) -> size_t { llama_load_model_from_url_headers *headers = (llama_load_model_from_url_headers *) userdata; + // Convert header field name to lowercase + for (size_t i = 0; i < n_items && buffer[i] != ':'; ++i) { + buffer[i] = tolower(buffer[i]); + } + const char * etag_prefix = "etag: "; if (strncmp(buffer, etag_prefix, strlen(etag_prefix)) == 0) { strncpy(headers->etag, buffer + strlen(etag_prefix), n_items - strlen(etag_prefix) - 2); // Remove CRLF @@ -1798,7 +1792,7 @@ struct llama_model * llama_load_model_from_url( if (res != CURLE_OK) { curl_easy_cleanup(curl); fprintf(stderr, "%s: curl_easy_perform() failed: %s\n", __func__, curl_easy_strerror(res)); - return NULL; + return false; } long http_code = 0; @@ -1806,30 +1800,34 @@ struct llama_model * llama_load_model_from_url( if (http_code != 200) { // HEAD not supported, we don't know if the file has changed // force trigger downloading - file_exists = false; + force_download = true; fprintf(stderr, "%s: HEAD invalid http status code received: %ld\n", __func__, http_code); } } // If the ETag or the Last-Modified headers are different: trigger a new download - if (!file_exists || strcmp(etag, headers.etag) != 0 || strcmp(last_modified, headers.last_modified) != 0) { - char path_model_temporary[LLAMA_CURL_MAX_PATH_LENGTH] = {0}; - snprintf(path_model_temporary, sizeof(path_model_temporary), "%s.downloadInProgress", path_model); + bool should_download = !file_exists + || force_download + || (strlen(headers.etag) > 0 && strcmp(etag, headers.etag) != 0) + || (strlen(headers.last_modified) > 0 && strcmp(last_modified, headers.last_modified) != 0); + if (should_download) { + char path_temporary[PATH_MAX] = {0}; + snprintf(path_temporary, sizeof(path_temporary), "%s.downloadInProgress", path); if (file_exists) { - fprintf(stderr, "%s: deleting previous downloaded model file: %s\n", __func__, path_model); - if (remove(path_model) != 0) { + fprintf(stderr, "%s: deleting previous downloaded file: %s\n", __func__, path); + if (remove(path) != 0) { curl_easy_cleanup(curl); - fprintf(stderr, "%s: unable to delete file: %s\n", __func__, path_model); - return NULL; + fprintf(stderr, "%s: unable to delete file: %s\n", __func__, path); + return false; } } // Set the output file - auto * outfile = fopen(path_model_temporary, "wb"); + auto * outfile = fopen(path_temporary, "wb"); if (!outfile) { curl_easy_cleanup(curl); - fprintf(stderr, "%s: error opening local file for writing: %s\n", __func__, path_model); - return NULL; + fprintf(stderr, "%s: error opening local file for writing: %s\n", __func__, path); + return false; } typedef size_t(*CURLOPT_WRITEFUNCTION_PTR)(void * data, size_t size, size_t nmemb, void * fd); @@ -1843,15 +1841,30 @@ struct llama_model * llama_load_model_from_url( // display download progress curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 0L); + // helper function to hide password in URL + auto llama_download_hide_password_in_url = [](const std::string & url) -> std::string { + std::size_t protocol_pos = url.find("://"); + if (protocol_pos == std::string::npos) { + return url; // Malformed URL + } + + std::size_t at_pos = url.find('@', protocol_pos + 3); + if (at_pos == std::string::npos) { + return url; // No password in URL + } + + return url.substr(0, protocol_pos + 3) + "********" + url.substr(at_pos); + }; + // start the download - fprintf(stderr, "%s: downloading model from %s to %s (server_etag:%s, server_last_modified:%s)...\n", __func__, - model_url, path_model, headers.etag, headers.last_modified); + fprintf(stderr, "%s: downloading from %s to %s (server_etag:%s, server_last_modified:%s)...\n", __func__, + llama_download_hide_password_in_url(url).c_str(), path, headers.etag, headers.last_modified); auto res = curl_easy_perform(curl); if (res != CURLE_OK) { fclose(outfile); curl_easy_cleanup(curl); fprintf(stderr, "%s: curl_easy_perform() failed: %s\n", __func__, curl_easy_strerror(res)); - return NULL; + return false; } long http_code = 0; @@ -1860,7 +1873,7 @@ struct llama_model * llama_load_model_from_url( fclose(outfile); curl_easy_cleanup(curl); fprintf(stderr, "%s: invalid http status code received: %ld\n", __func__, http_code); - return NULL; + return false; } // Clean up @@ -1872,7 +1885,7 @@ struct llama_model * llama_load_model_from_url( if (etag_file) { fputs(headers.etag, etag_file); fclose(etag_file); - fprintf(stderr, "%s: model etag saved %s: %s\n", __func__, etag_path, headers.etag); + fprintf(stderr, "%s: file etag saved %s: %s\n", __func__, etag_path, headers.etag); } } @@ -1882,20 +1895,118 @@ struct llama_model * llama_load_model_from_url( if (last_modified_file) { fputs(headers.last_modified, last_modified_file); fclose(last_modified_file); - fprintf(stderr, "%s: model last modified saved %s: %s\n", __func__, last_modified_path, + fprintf(stderr, "%s: file last modified saved %s: %s\n", __func__, last_modified_path, headers.last_modified); } } - if (rename(path_model_temporary, path_model) != 0) { + if (rename(path_temporary, path) != 0) { curl_easy_cleanup(curl); - fprintf(stderr, "%s: unable to rename file: %s to %s\n", __func__, path_model_temporary, path_model); - return NULL; + fprintf(stderr, "%s: unable to rename file: %s to %s\n", __func__, path_temporary, path); + return false; } } + return true; +} + +struct llama_model * llama_load_model_from_url( + const char * model_url, + const char * path_model, + const struct llama_model_params & params) { + // Basic validation of the model_url + if (!model_url || strlen(model_url) == 0) { + fprintf(stderr, "%s: invalid model_url\n", __func__); + return NULL; + } + + // Initialize libcurl + auto * curl = curl_easy_init(); + + if (!curl) { + fprintf(stderr, "%s: error initializing libcurl\n", __func__); + return NULL; + } + + if (!curl) { + fprintf(stderr, "%s: error initializing libcurl\n", __func__); + return NULL; + } + + if (!llama_download_file(curl, model_url, path_model)) { + return NULL; + } + + // check for additional GGUFs split to download + int n_split = 0; + { + struct gguf_init_params gguf_params = { + /*.no_alloc = */ true, + /*.ctx = */ NULL, + }; + auto * ctx_gguf = gguf_init_from_file(path_model, gguf_params); + if (!ctx_gguf) { + fprintf(stderr, "\n%s: failed to load input GGUF from %s\n", __func__, path_model); + curl_easy_cleanup(curl); + return NULL; + } + + auto key_n_split = gguf_find_key(ctx_gguf, LLM_KV_SPLIT_COUNT); + if (key_n_split >= 0) { + n_split = gguf_get_val_u16(ctx_gguf, key_n_split); + } + + gguf_free(ctx_gguf); + } + curl_easy_cleanup(curl); + if (n_split > 1) { + char split_prefix[PATH_MAX] = {0}; + char split_url_prefix[LLAMA_CURL_MAX_URL_LENGTH] = {0}; + + // Verify the first split file format + // and extract split URL and PATH prefixes + { + if (!llama_split_prefix(split_prefix, sizeof(split_prefix), path_model, 0, n_split)) { + fprintf(stderr, "\n%s: unexpected model file name: %s" + " n_split=%d\n", __func__, path_model, n_split); + return NULL; + } + + if (!llama_split_prefix(split_url_prefix, sizeof(split_url_prefix), model_url, 0, n_split)) { + fprintf(stderr, "\n%s: unexpected model url: %s" + " n_split=%d\n", __func__, model_url, n_split); + return NULL; + } + } + + // Prepare download in parallel + std::vector> futures_download; + for (int idx = 1; idx < n_split; idx++) { + futures_download.push_back(std::async(std::launch::async, [&split_prefix, &split_url_prefix, &n_split](int download_idx) -> bool { + char split_path[PATH_MAX] = {0}; + llama_split_path(split_path, sizeof(split_path), split_prefix, download_idx, n_split); + + char split_url[LLAMA_CURL_MAX_URL_LENGTH] = {0}; + llama_split_path(split_url, sizeof(split_url), split_url_prefix, download_idx, n_split); + + auto * curl = curl_easy_init(); + bool res = llama_download_file(curl, split_url, split_path); + curl_easy_cleanup(curl); + + return res; + }, idx)); + } + + // Wait for all downloads to complete + for (auto & f : futures_download) { + if (!f.get()) { + return NULL; + } + } + } + return llama_load_model_from_file(path_model, params); } diff --git a/common/common.h b/common/common.h index afa4cf6d7..a223eceaa 100644 --- a/common/common.h +++ b/common/common.h @@ -306,3 +306,10 @@ struct llama_control_vector_load_info { // Load control vectors, scale each by strength, and add them together. // On error, returns {-1, empty} llama_control_vector_data llama_control_vector_load(const std::vector & load_infos); + +// +// Split utils +// +static const char * const LLM_KV_SPLIT_NO = "split.no"; +static const char * const LLM_KV_SPLIT_COUNT = "split.count"; +static const char * const LLM_KV_SPLIT_TENSORS_COUNT = "split.tensors.count"; diff --git a/examples/gguf-split/gguf-split.cpp b/examples/gguf-split/gguf-split.cpp index f703588e1..b1af59992 100644 --- a/examples/gguf-split/gguf-split.cpp +++ b/examples/gguf-split/gguf-split.cpp @@ -26,10 +26,6 @@ enum split_operation : uint8_t { SPLIT_OP_MERGE, }; -static const char * const LLM_KV_SPLIT_NO = "split.no"; -static const char * const LLM_KV_SPLIT_COUNT = "split.count"; -static const char * const LLM_KV_SPLIT_TENSORS_COUNT = "split.tensors.count"; - struct split_params { split_operation operation = SPLIT_OP_SPLIT; int n_split_tensors = 128; diff --git a/examples/server/README.md b/examples/server/README.md index 8aa4eac9d..dfea2b905 100644 --- a/examples/server/README.md +++ b/examples/server/README.md @@ -20,7 +20,9 @@ The project is under active development, and we are [looking for feedback and co - `-tb N, --threads-batch N`: Set the number of threads to use during batch and prompt processing. If not specified, the number of threads will be set to the number of threads used for generation. Not used if model layers are offloaded to GPU. - `--threads-http N`: number of threads in the http server pool to process requests (default: `max(std::thread::hardware_concurrency() - 1, --parallel N + 2)`) - `-m FNAME`, `--model FNAME`: Specify the path to the LLaMA model file (e.g., `models/7B/ggml-model.gguf`). -- `-mu MODEL_URL --model-url MODEL_URL`: Specify a remote http url to download the file (e.g https://huggingface.co/ggml-org/models/resolve/main/phi-2/ggml-model-q4_0.gguf). +- `-mu MODEL_URL --model-url MODEL_URL`: Specify a remote http url to download the file (default: unused). +- `-hfr REPO, --hf-repo REPO`: Hugging Face model repository (default: unused). +- `-hff FILE, --hf-file FILE`: Hugging Face model file (default: unused). - `-a ALIAS`, `--alias ALIAS`: Set an alias for the model. The alias will be returned in API responses. - `-c N`, `--ctx-size N`: Set the size of the prompt context. The default is 512, but LLaMA models were built with a context of 2048, which will provide better results for longer input/inference. The size may differ in other models, for example, baichuan models were build with a context of 4096. - `-ngl N`, `--n-gpu-layers N`: When compiled with appropriate support (currently CLBlast or cuBLAS), this option allows offloading some layers to the GPU for computation. Generally results in increased performance. diff --git a/examples/server/server.cpp b/examples/server/server.cpp index 27bd2dd70..b02c2546e 100644 --- a/examples/server/server.cpp +++ b/examples/server/server.cpp @@ -2208,7 +2208,11 @@ static void server_print_usage(const char * argv0, const gpt_params & params, co printf(" -m FNAME, --model FNAME\n"); printf(" model path (default: %s)\n", params.model.c_str()); printf(" -mu MODEL_URL, --model-url MODEL_URL\n"); - printf(" model download url (default: %s)\n", params.model_url.c_str()); + printf(" model download url (default: unused)\n"); + printf(" -hfr REPO, --hf-repo REPO\n"); + printf(" Hugging Face model repository (default: unused)\n"); + printf(" -hff FILE, --hf-file FILE\n"); + printf(" Hugging Face model file (default: unused)\n"); printf(" -a ALIAS, --alias ALIAS\n"); printf(" set an alias for the model, will be added as `model` field in completion response\n"); printf(" --lora FNAME apply LoRA adapter (implies --no-mmap)\n"); @@ -2337,6 +2341,18 @@ static void server_params_parse(int argc, char ** argv, server_params & sparams, break; } params.model_url = argv[i]; + } else if (arg == "-hfr" || arg == "--hf-repo") { + if (++i >= argc) { + invalid_param = true; + break; + } + params.hf_repo = argv[i]; + } else if (arg == "-hff" || arg == "--hf-file") { + if (++i >= argc) { + invalid_param = true; + break; + } + params.hf_file = argv[i]; } else if (arg == "-a" || arg == "--alias") { if (++i >= argc) { invalid_param = true; diff --git a/examples/server/tests/features/parallel.feature b/examples/server/tests/features/parallel.feature index a66fed626..6cd306a2b 100644 --- a/examples/server/tests/features/parallel.feature +++ b/examples/server/tests/features/parallel.feature @@ -4,7 +4,8 @@ Feature: Parallel Background: Server startup Given a server listening on localhost:8080 - And a model file tinyllamas/stories260K.gguf from HF repo ggml-org/models + And a model file tinyllamas/split/stories15M-00001-of-00003.gguf from HF repo ggml-org/models + And a model file test-model-00001-of-00003.gguf And 42 as server seed And 128 as batch size And 256 KV cache size diff --git a/examples/server/tests/features/server.feature b/examples/server/tests/features/server.feature index a2e0e5b35..646a4e49d 100644 --- a/examples/server/tests/features/server.feature +++ b/examples/server/tests/features/server.feature @@ -4,8 +4,8 @@ Feature: llama.cpp server Background: Server startup Given a server listening on localhost:8080 - And a model url https://huggingface.co/ggml-org/models/resolve/main/tinyllamas/stories260K.gguf - And a model file stories260K.gguf + And a model file tinyllamas/stories260K.gguf from HF repo ggml-org/models + And a model file test-model.gguf And a model alias tinyllama-2 And 42 as server seed # KV Cache corresponds to the total amount of tokens diff --git a/examples/server/tests/features/steps/steps.py b/examples/server/tests/features/steps/steps.py index 03f55f659..86c3339dc 100644 --- a/examples/server/tests/features/steps/steps.py +++ b/examples/server/tests/features/steps/steps.py @@ -16,7 +16,6 @@ import numpy as np import openai from behave import step from behave.api.async_step import async_run_until_complete -from huggingface_hub import hf_hub_download from prometheus_client import parser @@ -39,6 +38,8 @@ def step_server_config(context, server_fqdn, server_port): context.model_alias = None context.model_file = None + context.model_hf_repo = None + context.model_hf_file = None context.model_url = None context.n_batch = None context.n_ubatch = None @@ -68,9 +69,9 @@ def step_server_config(context, server_fqdn, server_port): @step('a model file {hf_file} from HF repo {hf_repo}') def step_download_hf_model(context, hf_file, hf_repo): - context.model_file = hf_hub_download(repo_id=hf_repo, filename=hf_file) - if context.debug: - print(f"model file: {context.model_file}") + context.model_hf_repo = hf_repo + context.model_hf_file = hf_file + context.model_file = os.path.basename(hf_file) @step('a model file {model_file}') @@ -1079,6 +1080,10 @@ def start_server_background(context): server_args.extend(['--model', context.model_file]) if context.model_url: server_args.extend(['--model-url', context.model_url]) + if context.model_hf_repo: + server_args.extend(['--hf-repo', context.model_hf_repo]) + if context.model_hf_file: + server_args.extend(['--hf-file', context.model_hf_file]) if context.n_batch: server_args.extend(['--batch-size', context.n_batch]) if context.n_ubatch: diff --git a/llama.cpp b/llama.cpp index 4e08be18d..b385ae360 100644 --- a/llama.cpp +++ b/llama.cpp @@ -2959,7 +2959,7 @@ struct llama_model_loader { } } - LLAMA_LOG_INFO("%s: additional %d GGUFs metadata loaded.\n", __func__, n_split); + LLAMA_LOG_INFO("%s: additional %d GGUFs metadata loaded.\n", __func__, n_split - 1); } n_kv = gguf_get_n_kv(meta); @@ -15140,7 +15140,7 @@ int llama_split_prefix(char * dest, size_t maxlen, const char * split_path, int // check if dest ends with postfix int size_prefix = str_split_path.size() - str_postfix.size(); if (size_prefix > 0 && str_split_path.find(str_postfix, size_prefix) != std::string::npos) { - snprintf(dest, std::min((size_t) size_prefix, maxlen), "%s", split_path); + snprintf(dest, std::min((size_t) size_prefix + 1, maxlen), "%s", split_path); return size_prefix; } From 95562175f83a49755ff6fd3bad09409417c8e6f9 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Sat, 23 Mar 2024 21:35:23 +0200 Subject: [PATCH 711/812] gitignore : gguf-split --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 27562f6d7..072945180 100644 --- a/.gitignore +++ b/.gitignore @@ -50,6 +50,7 @@ models-mnt /embedding /gguf /gguf-llama-simple +/gguf-split /gritlm /imatrix /infill From 94d1b3b4119209efcdd08df0dceaecbd1fe7f85c Mon Sep 17 00:00:00 2001 From: Jared Van Bortel Date: Sat, 23 Mar 2024 18:48:02 -0400 Subject: [PATCH 712/812] use _wfopen instead of fopen on Windows (#6248) also fix missing #defines before windows.h, and BPE LF token on MSVC --- ggml.c | 66 ++++++++++++++++++++++++++++++++++++++++++++++++++----- ggml.h | 8 +++++-- llama.cpp | 4 ++-- 3 files changed, 69 insertions(+), 9 deletions(-) diff --git a/ggml.c b/ggml.c index 1d5854960..54365b7ae 100644 --- a/ggml.c +++ b/ggml.c @@ -3,6 +3,7 @@ #include "ggml-impl.h" #include "ggml-quants.h" +#include "ggml.h" #if defined(_MSC_VER) || defined(__MINGW32__) #include // using malloc.h with MSC/MINGW @@ -43,6 +44,10 @@ #if defined(_WIN32) +#define WIN32_LEAN_AND_MEAN +#ifndef NOMINMAX + #define NOMINMAX +#endif #include typedef volatile LONG atomic_int; @@ -430,6 +435,57 @@ int64_t ggml_cycles_per_ms(void) { #define ggml_perf_cycles_per_ms() 0 #endif +// +// cross-platform UTF-8 file paths +// + +#ifdef _WIN32 +static wchar_t * ggml_mbstowcs(const char * mbs) { + int wlen = MultiByteToWideChar(CP_UTF8, 0, mbs, -1, NULL, 0); + if (!wlen) { + errno = EINVAL; + return NULL; + } + + wchar_t * wbuf = GGML_MALLOC(wlen * sizeof(wchar_t)); + wlen = MultiByteToWideChar(CP_UTF8, 0, mbs, -1, wbuf, wlen); + if (!wlen) { + GGML_FREE(wbuf); + errno = EINVAL; + return NULL; + } + + return wbuf; +} +#endif + +FILE * ggml_fopen(const char * fname, const char * mode) { +#ifdef _WIN32 + FILE * file = NULL; + + // convert fname (UTF-8) + wchar_t * wfname = ggml_mbstowcs(fname); + if (wfname) { + // convert mode (ANSI) + wchar_t * wmode = GGML_MALLOC(strlen(mode) + 1); + wchar_t * wmode_p = wmode; + do { + *wmode_p++ = (wchar_t)*mode; + } while (*mode++); + + // open file + file = _wfopen(wfname, wmode); + + GGML_FREE(wfname); + GGML_FREE(wmode); + } + + return file; +#else + return fopen(fname, mode); +#endif +} + // // cache line // @@ -18739,7 +18795,7 @@ void ggml_graph_export(const struct ggml_cgraph * cgraph, const char * fname) { // write binary data { - FILE * fout = fopen(fname, "wb"); + FILE * fout = ggml_fopen(fname, "wb"); if (!fout) { fprintf(stderr, "%s: failed to open %s\n", __func__, fname); @@ -18877,7 +18933,7 @@ struct ggml_cgraph * ggml_graph_import(const char * fname, struct ggml_context * // read file into data { - FILE * fin = fopen(fname, "rb"); + FILE * fin = ggml_fopen(fname, "rb"); if (!fin) { fprintf(stderr, "%s: failed to open %s\n", __func__, fname); return result; @@ -19213,7 +19269,7 @@ static void ggml_graph_dump_dot_leaf_edge(FILE * fp, struct ggml_tensor * node, void ggml_graph_dump_dot(const struct ggml_cgraph * gb, const struct ggml_cgraph * gf, const char * filename) { char color[16]; - FILE * fp = fopen(filename, "w"); + FILE * fp = ggml_fopen(filename, "w"); GGML_ASSERT(fp); fprintf(fp, "digraph G {\n"); @@ -20531,7 +20587,7 @@ struct gguf_context * gguf_init_empty(void) { } struct gguf_context * gguf_init_from_file(const char * fname, struct gguf_init_params params) { - FILE * file = fopen(fname, "rb"); + FILE * file = ggml_fopen(fname, "rb"); if (!file) { return NULL; } @@ -21486,7 +21542,7 @@ static void gguf_write_to_buf(const struct gguf_context * ctx, struct gguf_buf * } void gguf_write_to_file(const struct gguf_context * ctx, const char * fname, bool only_meta) { - FILE * file = fopen(fname, "wb"); + FILE * file = ggml_fopen(fname, "wb"); if (!file) { GGML_ASSERT(false && "failed to open file for writing"); } diff --git a/ggml.h b/ggml.h index c937d4a53..0a5af7205 100644 --- a/ggml.h +++ b/ggml.h @@ -214,9 +214,10 @@ # define GGML_ATTRIBUTE_FORMAT(...) __attribute__((format(printf, __VA_ARGS__))) #endif -#include -#include #include +#include +#include +#include #define GGML_FILE_MAGIC 0x67676d6c // "ggml" #define GGML_FILE_VERSION 1 @@ -708,6 +709,9 @@ extern "C" { GGML_API void ggml_print_backtrace(void); + // accepts a UTF-8 path, even on Windows + GGML_API FILE * ggml_fopen(const char * fname, const char * mode); + GGML_API void ggml_numa_init(enum ggml_numa_strategy numa); // call once for better performance on NUMA systems GGML_API bool ggml_is_numa(void); // true if init detected that system has >1 NUMA node diff --git a/llama.cpp b/llama.cpp index b385ae360..9614cdb17 100644 --- a/llama.cpp +++ b/llama.cpp @@ -1065,7 +1065,7 @@ struct llama_file { size_t size; llama_file(const char * fname, const char * mode) { - fp = std::fopen(fname, mode); + fp = ggml_fopen(fname, mode); if (fp == NULL) { throw std::runtime_error(format("failed to open %s: %s", fname, strerror(errno))); } @@ -4006,7 +4006,7 @@ static void llm_load_vocab( } else if (vocab.type == LLAMA_VOCAB_TYPE_WPM) { vocab.linefeed_id = vocab.special_pad_id; } else { - const std::vector ids = llama_tokenize_internal(vocab, "\u010A", false); + const std::vector ids = llama_tokenize_internal(vocab, "\xC4\x8A", false); // U+010A GGML_ASSERT(!ids.empty() && "model vocab missing newline token"); vocab.linefeed_id = ids[0]; } From d03224ac9840351023ff8abcf4aa0542258a53df Mon Sep 17 00:00:00 2001 From: Neo Zhang Jianyu Date: Sun, 24 Mar 2024 09:44:01 +0800 Subject: [PATCH 713/812] Support build win release for SYCL (#6241) * support release win * fix value * fix value * fix value * fix error * fix error * fix format --- .github/workflows/build.yml | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index bf42df8fe..0e7643bba 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -800,6 +800,7 @@ jobs: windows-latest-cmake-sycl: runs-on: windows-latest + defaults: run: shell: bash @@ -808,7 +809,6 @@ jobs: WINDOWS_BASEKIT_URL: https://registrationcenter-download.intel.com/akdlm/IRC_NAS/62641e01-1e8d-4ace-91d6-ae03f7f8a71f/w_BaseKit_p_2024.0.0.49563_offline.exe WINDOWS_DPCPP_MKL: intel.oneapi.win.cpp-dpcpp-common:intel.oneapi.win.mkl.devel - steps: - name: Clone id: checkout @@ -823,6 +823,32 @@ jobs: id: cmake_build run: examples/sycl/win-build-sycl.bat + - name: Determine tag name + id: tag + shell: bash + run: | + BUILD_NUMBER="$(git rev-list --count HEAD)" + SHORT_HASH="$(git rev-parse --short=7 HEAD)" + if [[ "${{ env.BRANCH_NAME }}" == "master" ]]; then + echo "name=b${BUILD_NUMBER}" >> $GITHUB_OUTPUT + else + SAFE_NAME=$(echo "${{ env.BRANCH_NAME }}" | tr '/' '-') + echo "name=${SAFE_NAME}-b${BUILD_NUMBER}-${SHORT_HASH}" >> $GITHUB_OUTPUT + fi + + - name: Pack artifacts + id: pack_artifacts + if: ${{ ( github.event_name == 'push' && github.ref == 'refs/heads/master' ) || github.event.inputs.create_release == 'true' }} + run: | + 7z a llama-${{ steps.tag.outputs.name }}-bin-win-sycl-x64.zip .\build\bin\* + + - name: Upload artifacts + if: ${{ ( github.event_name == 'push' && github.ref == 'refs/heads/master' ) || github.event.inputs.create_release == 'true' }} + uses: actions/upload-artifact@v3 + with: + path: | + llama-${{ steps.tag.outputs.name }}-bin-win-sycl-x64.zip + ios-xcode-build: runs-on: macos-latest From ddf65685105a39a57b1e7f80c3aa502a6313af24 Mon Sep 17 00:00:00 2001 From: "Meng, Hengyu" Date: Sun, 24 Mar 2024 12:04:25 +0800 Subject: [PATCH 714/812] [SYCL] offload op (#6217) * remove no USM methods * leave the schedule to ggml_backend_sched entirely --- ggml-sycl.cpp | 293 ++++---------------------------------------------- ggml-sycl.h | 16 ++- ggml.c | 10 -- llama.cpp | 36 +++---- 4 files changed, 51 insertions(+), 304 deletions(-) diff --git a/ggml-sycl.cpp b/ggml-sycl.cpp index cc9ee0762..fc4d2964c 100644 --- a/ggml-sycl.cpp +++ b/ggml-sycl.cpp @@ -740,11 +740,7 @@ namespace dpct sycl::queue &default_queue() { -#ifdef DPCT_USM_LEVEL_NONE - return out_of_order_queue(); -#else return in_order_queue(); -#endif // DPCT_USM_LEVEL_NONE } void queues_wait_and_throw() @@ -763,11 +759,7 @@ namespace dpct sycl::queue *create_queue(bool enable_exception_handler = false) { -#ifdef DPCT_USM_LEVEL_NONE - return create_out_of_order_queue(enable_exception_handler); -#else return create_in_order_queue(enable_exception_handler); -#endif // DPCT_USM_LEVEL_NONE } sycl::queue *create_queue(sycl::context context, sycl::device device, @@ -1075,11 +1067,6 @@ namespace dpct static pointer_access_attribute get_pointer_attribute(sycl::queue &q, const void *ptr) { -#ifdef DPCT_USM_LEVEL_NONE - return mem_mgr::instance().is_device_ptr(ptr) - ? pointer_access_attribute::device_only - : pointer_access_attribute::host_only; -#else switch (sycl::get_pointer_type(ptr, q.get_context())) { case sycl::usm::alloc::unknown: @@ -1090,7 +1077,6 @@ namespace dpct case sycl::usm::alloc::host: return pointer_access_attribute::host_device; } -#endif } template @@ -1273,11 +1259,7 @@ namespace dpct static inline void *dpct_malloc(size_t size, sycl::queue &q) { -#ifdef DPCT_USM_LEVEL_NONE - return mem_mgr::instance().mem_alloc(size * sizeof(byte_t)); -#else return sycl::malloc_device(size, q.get_device(), q.get_context()); -#endif // DPCT_USM_LEVEL_NONE } #define PITCH_DEFAULT_ALIGN(x) (((x) + 31) & ~(0x1F)) @@ -1301,25 +1283,7 @@ namespace dpct static inline sycl::event dpct_memset(sycl::queue &q, void *dev_ptr, valueT value, size_t size) { -#ifdef DPCT_USM_LEVEL_NONE - auto &mm = mem_mgr::instance(); - assert(mm.is_device_ptr(dev_ptr)); - auto alloc = mm.translate_ptr(dev_ptr); - size_t offset = (valueT *)dev_ptr - (valueT *)alloc.alloc_ptr; - - return q.submit([&](sycl::handler &cgh) - { - auto r = sycl::range<1>(size); - auto o = sycl::id<1>(offset); - auto new_buffer = alloc.buffer.reinterpret( - sycl::range<1>(alloc.size / sizeof(valueT))); - sycl::accessor - acc(new_buffer, cgh, r, o); - cgh.fill(acc, value); }); -#else return q.fill(dev_ptr, value, size); -#endif // DPCT_USM_LEVEL_NONE } /** @@ -1413,72 +1377,8 @@ namespace dpct { if (!size) return sycl::event{}; -#ifdef DPCT_USM_LEVEL_NONE - auto &mm = mem_mgr::instance(); - auto real_direction = deduce_memcpy_direction(q, to_ptr, from_ptr, direction); - - switch (real_direction) - { - case host_to_host: - return q.submit([&](sycl::handler &cgh) - { - cgh.depends_on(dep_events); - cgh.host_task([=] { std::memcpy(to_ptr, from_ptr, size); }); }); - case host_to_device: - { - auto alloc = mm.translate_ptr(to_ptr); - size_t offset = (byte_t *)to_ptr - alloc.alloc_ptr; - return q.submit([&](sycl::handler &cgh) - { - cgh.depends_on(dep_events); - auto r = sycl::range<1>(size); - auto o = sycl::id<1>(offset); - sycl::accessor - acc(alloc.buffer, cgh, r, o); - cgh.copy(from_ptr, acc); }); - } - case device_to_host: - { - auto alloc = mm.translate_ptr(from_ptr); - size_t offset = (byte_t *)from_ptr - alloc.alloc_ptr; - return q.submit([&](sycl::handler &cgh) - { - cgh.depends_on(dep_events); - auto r = sycl::range<1>(size); - auto o = sycl::id<1>(offset); - sycl::accessor - acc(alloc.buffer, cgh, r, o); - cgh.copy(acc, to_ptr); }); - } - case device_to_device: - { - auto to_alloc = mm.translate_ptr(to_ptr); - auto from_alloc = mm.translate_ptr(from_ptr); - size_t to_offset = (byte_t *)to_ptr - to_alloc.alloc_ptr; - size_t from_offset = (byte_t *)from_ptr - from_alloc.alloc_ptr; - return q.submit([&](sycl::handler &cgh) - { - cgh.depends_on(dep_events); - auto r = sycl::range<1>(size); - auto to_o = sycl::id<1>(to_offset); - auto from_o = sycl::id<1>(from_offset); - sycl::accessor - to_acc(to_alloc.buffer, cgh, r, to_o); - sycl::accessor - from_acc(from_alloc.buffer, cgh, r, from_o); - cgh.copy(from_acc, to_acc); }); - } - default: - throw std::runtime_error("dpct_memcpy: invalid direction value"); - } -#else return q.memcpy(to_ptr, from_ptr, size, dep_events); GGML_UNUSED(direction); -#endif // DPCT_USM_LEVEL_NONE } // Get actual copy range and make sure it will not exceed range. @@ -1618,45 +1518,15 @@ namespace dpct break; } case device_to_device: -#ifdef DPCT_USM_LEVEL_NONE - { - auto &mm = mem_mgr::instance(); - auto to_alloc = mm.translate_ptr(to_surface); - auto from_alloc = mm.translate_ptr(from_surface); - size_t to_offset = (byte_t *)to_surface - to_alloc.alloc_ptr; - size_t from_offset = (byte_t *)from_surface - from_alloc.alloc_ptr; - event_list.push_back(q.submit([&](sycl::handler &cgh) - { - cgh.depends_on(dep_events); - auto to_o = sycl::id<1>(to_offset); - auto from_o = sycl::id<1>(from_offset); - sycl::accessor - to_acc(to_alloc.buffer, cgh, - get_copy_range(size, to_slice, to_range.get(0)), to_o); - sycl::accessor - from_acc(from_alloc.buffer, cgh, - get_copy_range(size, from_slice, from_range.get(0)), from_o); - cgh.parallel_for( - size, - [=](sycl::id<3> id) { - to_acc[get_offset(id, to_slice, to_range.get(0))] = - from_acc[get_offset(id, from_slice, from_range.get(0))]; - }); })); - } -#else - event_list.push_back(q.submit([&](sycl::handler &cgh) - { - cgh.depends_on(dep_events); - cgh.parallel_for( - size, - [=](sycl::id<3> id) { - to_surface[get_offset(id, to_slice, to_range.get(0))] = - from_surface[get_offset(id, from_slice, from_range.get(0))]; - }); })); -#endif - break; + event_list.push_back(q.submit([&](sycl::handler &cgh){ + cgh.depends_on(dep_events); + cgh.parallel_for( + size, + [=](sycl::id<3> id) { + to_surface[get_offset(id, to_slice, to_range.get(0))] = + from_surface[get_offset(id, from_slice, from_range.get(0))]; + }); })); + break; default: throw std::runtime_error("dpct_memcpy: invalid direction value"); } @@ -1754,11 +1624,7 @@ namespace dpct { if (ptr) { -#ifdef DPCT_USM_LEVEL_NONE - detail::mem_mgr::instance().mem_free(ptr); -#else sycl::free(ptr, q.get_context()); -#endif // DPCT_USM_LEVEL_NONE } } @@ -1766,11 +1632,7 @@ namespace dpct inline auto get_memory(const void *x) { T *new_x = reinterpret_cast(const_cast(x)); -#ifdef DPCT_USM_LEVEL_NONE - return dpct::get_buffer>(new_x); -#else return new_x; -#endif } template @@ -2222,72 +2084,8 @@ namespace dpct { if (!size) return sycl::event{}; -#ifdef DPCT_USM_LEVEL_NONE - auto &mm = mem_mgr::instance(); - auto real_direction = deduce_memcpy_direction(q, to_ptr, from_ptr, direction); - - switch (real_direction) - { - case host_to_host: - return q.submit([&](sycl::handler &cgh) - { - cgh.depends_on(dep_events); - cgh.host_task([=] { std::memcpy(to_ptr, from_ptr, size); }); }); - case host_to_device: - { - auto alloc = mm.translate_ptr(to_ptr); - size_t offset = (byte_t *)to_ptr - alloc.alloc_ptr; - return q.submit([&](sycl::handler &cgh) - { - cgh.depends_on(dep_events); - auto r = sycl::range<1>(size); - auto o = sycl::id<1>(offset); - sycl::accessor - acc(alloc.buffer, cgh, r, o); - cgh.copy(from_ptr, acc); }); - } - case device_to_host: - { - auto alloc = mm.translate_ptr(from_ptr); - size_t offset = (byte_t *)from_ptr - alloc.alloc_ptr; - return q.submit([&](sycl::handler &cgh) - { - cgh.depends_on(dep_events); - auto r = sycl::range<1>(size); - auto o = sycl::id<1>(offset); - sycl::accessor - acc(alloc.buffer, cgh, r, o); - cgh.copy(acc, to_ptr); }); - } - case device_to_device: - { - auto to_alloc = mm.translate_ptr(to_ptr); - auto from_alloc = mm.translate_ptr(from_ptr); - size_t to_offset = (byte_t *)to_ptr - to_alloc.alloc_ptr; - size_t from_offset = (byte_t *)from_ptr - from_alloc.alloc_ptr; - return q.submit([&](sycl::handler &cgh) - { - cgh.depends_on(dep_events); - auto r = sycl::range<1>(size); - auto to_o = sycl::id<1>(to_offset); - auto from_o = sycl::id<1>(from_offset); - sycl::accessor - to_acc(to_alloc.buffer, cgh, r, to_o); - sycl::accessor - from_acc(from_alloc.buffer, cgh, r, from_o); - cgh.copy(from_acc, to_acc); }); - } - default: - throw std::runtime_error("dpct_memcpy: invalid direction value"); - } -#else return q.memcpy(to_ptr, from_ptr, size, dep_events); GGML_UNUSED(direction); -#endif // DPCT_USM_LEVEL_NONE } // Get actual copy range and make sure it will not exceed range. @@ -2427,34 +2225,6 @@ namespace dpct break; } case device_to_device: -#ifdef DPCT_USM_LEVEL_NONE - { - auto &mm = mem_mgr::instance(); - auto to_alloc = mm.translate_ptr(to_surface); - auto from_alloc = mm.translate_ptr(from_surface); - size_t to_offset = (byte_t *)to_surface - to_alloc.alloc_ptr; - size_t from_offset = (byte_t *)from_surface - from_alloc.alloc_ptr; - event_list.push_back(q.submit([&](sycl::handler &cgh) - { - cgh.depends_on(dep_events); - auto to_o = sycl::id<1>(to_offset); - auto from_o = sycl::id<1>(from_offset); - sycl::accessor - to_acc(to_alloc.buffer, cgh, - get_copy_range(size, to_slice, to_range.get(0)), to_o); - sycl::accessor - from_acc(from_alloc.buffer, cgh, - get_copy_range(size, from_slice, from_range.get(0)), from_o); - cgh.parallel_for( - size, - [=](sycl::id<3> id) { - to_acc[get_offset(id, to_slice, to_range.get(0))] = - from_acc[get_offset(id, from_slice, from_range.get(0))]; - }); })); - } -#else event_list.push_back(q.submit([&](sycl::handler &cgh) { cgh.depends_on(dep_events); @@ -2464,7 +2234,6 @@ namespace dpct to_surface[get_offset(id, to_slice, to_range.get(0))] = from_surface[get_offset(id, from_slice, from_range.get(0))]; }); })); -#endif break; default: throw std::runtime_error("dpct_memcpy: invalid direction value"); @@ -2655,9 +2424,6 @@ namespace dpct void *c[], library_data_t c_type, int ldc, int batch_size, library_data_t scaling_type) { -#ifdef DPCT_USM_LEVEL_NONE - throw std::runtime_error("this API is unsupported when USM level is none"); -#else if (scaling_type == library_data_t::real_float && c_type == library_data_t::complex_float) { @@ -2792,7 +2558,6 @@ namespace dpct default: throw std::runtime_error("the combination of data type is unsupported"); } -#endif } /// Computes a batch of matrix-matrix product with general matrices. @@ -3131,24 +2896,9 @@ namespace dpct template typename std::enable_if::type &operator[](size_t index) { init(); - #ifdef DPCT_USM_LEVEL_NONE - return dpct::get_buffer::type>( - _device_ptr) - .template get_access()[index]; - #else return _device_ptr[index]; - #endif // DPCT_USM_LEVEL_NONE } - #ifdef DPCT_USM_LEVEL_NONE - /// Get sycl::accessor for the device memory object when usm is not used. - accessor_t get_access(sycl::handler &cgh) { - return get_buffer(_device_ptr) - .template reinterpret(_range) - .template get_access::mode, - detail::memory_traits::target>(cgh); - } - #else /// Get dpct::accessor with dimension info for the device memory object /// when usm is used and dimension is greater than 1. template @@ -3156,7 +2906,6 @@ namespace dpct get_access(sycl::handler &cgh) { return dpct_accessor_t((T *)_device_ptr, _range); } - #endif // DPCT_USM_LEVEL_NONE private: device_memory(value_t *memory_ptr, size_t size) @@ -3201,15 +2950,6 @@ namespace dpct /// Default constructor device_memory() : base(1) {} - - #ifdef DPCT_USM_LEVEL_NONE - /// Get sycl::accessor for the device memory object when usm is not used. - accessor_t get_access(sycl::handler &cgh) { - auto buf = get_buffer(base::get_ptr()) - .template reinterpret(sycl::range<1>(1)); - return accessor_t(buf, cgh); - } - #endif // DPCT_USM_LEVEL_NONE }; } // namespace detail @@ -13181,7 +12921,7 @@ int get_work_group_size(int user_device_id) { return prop.get_max_work_group_size(); } -void ggml_init_sycl() try { +static void ggml_init_sycl() try { static bool initialized = false; if (!initialized) { @@ -16677,6 +16417,7 @@ static ggml_backend_buffer_type_i ggml_backend_sycl_buffer_type_interface = { }; ggml_backend_buffer_type_t ggml_backend_sycl_buffer_type(int device_index) { + ggml_init_sycl(); if (device_index>=g_device_count or device_index<0) { printf("ggml_backend_sycl_buffer_type error: device_index:%d is out of range [0, %d], miss to call ggml_backend_sycl_set_single_device()\n", device_index, g_device_count-1); @@ -17046,6 +16787,7 @@ static ggml_backend_buffer_type_i ggml_backend_sycl_split_buffer_type_interface }; GGML_CALL ggml_backend_buffer_type_t ggml_backend_sycl_split_buffer_type(const float * tensor_split) { + ggml_init_sycl(); // FIXME: this is not thread safe static std::map, struct ggml_backend_buffer_type> buft_map; @@ -17379,6 +17121,13 @@ GGML_CALL static bool ggml_backend_sycl_supports_op(ggml_backend_t backend, cons UNUSED(backend); } +GGML_CALL static bool ggml_backend_sycl_offload_op(ggml_backend_t backend, const ggml_tensor * op) { + const int min_batch_size = 32; + return op->ne[1] >= min_batch_size && op->op != GGML_OP_GET_ROWS; + GGML_UNUSED(backend); +} + + static ggml_backend_i ggml_backend_sycl_interface = { /* .get_name = */ ggml_backend_sycl_name, /* .free = */ ggml_backend_sycl_free, @@ -17392,7 +17141,7 @@ static ggml_backend_i ggml_backend_sycl_interface = { /* .graph_plan_compute = */ NULL, /* .graph_compute = */ ggml_backend_sycl_graph_compute, /* .supports_op = */ ggml_backend_sycl_supports_op, - /* .offload_op = */ NULL, + /* .offload_op = */ ggml_backend_sycl_offload_op, /* .event_new = */ NULL, /* .event_free = */ NULL, /* .event_record = */ NULL, @@ -17406,7 +17155,7 @@ static ggml_guid_t ggml_backend_sycl_guid() { } GGML_CALL ggml_backend_t ggml_backend_sycl_init(int device) { - ggml_init_sycl(); // TODO: remove from ggml.c + ggml_init_sycl(); check_allow_gpu_index(device); diff --git a/ggml-sycl.h b/ggml-sycl.h index 1c9d52115..a9f776fc1 100644 --- a/ggml-sycl.h +++ b/ggml-sycl.h @@ -16,16 +16,22 @@ extern "C" { #define GGML_SYCL_MAX_DEVICES 48 #define GGML_SYCL_NAME "SYCL" -GGML_API void ggml_init_sycl(void); -GGML_API bool ggml_sycl_compute_forward(struct ggml_compute_params * params, struct ggml_tensor * tensor); +// backend API GGML_API ggml_backend_t ggml_backend_sycl_init(int device); + +// devide buffer GGML_API ggml_backend_buffer_type_t ggml_backend_sycl_buffer_type(int device); + +// split tensor buffer that splits matrices by rows across multiple devices +GGML_API GGML_CALL ggml_backend_buffer_type_t ggml_backend_sycl_split_buffer_type(const float * tensor_split); + +// pinned host buffer for use with the CPU backend for faster copies between CPU and GPU GGML_API ggml_backend_buffer_type_t ggml_backend_sycl_host_buffer_type(void); + GGML_API void ggml_backend_sycl_print_sycl_devices(void); GGML_API GGML_CALL void ggml_sycl_get_gpu_list(int *id_list, int max_len); GGML_API GGML_CALL void ggml_sycl_get_device_description(int device, char *description, size_t description_size); GGML_API GGML_CALL int ggml_backend_sycl_get_device_count(); -GGML_API GGML_CALL ggml_backend_buffer_type_t ggml_backend_sycl_split_buffer_type(const float * tensor_split); GGML_API GGML_CALL void ggml_backend_sycl_get_device_memory(int device, size_t *free, size_t *total); GGML_API GGML_CALL int ggml_backend_sycl_get_device_index(int device_id); @@ -34,6 +40,10 @@ GGML_API GGML_CALL int ggml_backend_sycl_get_device_index(int device_id); GGML_API GGML_CALL int ggml_backend_sycl_get_device_id(int device_index); GGML_API GGML_CALL void ggml_backend_sycl_set_single_device_mode(int main_gpu_id); GGML_API GGML_CALL void ggml_backend_sycl_set_mul_device_mode(); + +// SYCL doesn't support registering host memory, keep here for reference +// GGML_API GGML_CALL bool ggml_backend_sycl_register_host_buffer(void * buffer, size_t size); +// GGML_API GGML_CALL void ggml_backend_sycl_unregister_host_buffer(void * buffer); #ifdef __cplusplus } #endif diff --git a/ggml.c b/ggml.c index 54365b7ae..18f10a3dc 100644 --- a/ggml.c +++ b/ggml.c @@ -291,8 +291,6 @@ inline static void * ggml_calloc(size_t num, size_t size) { #include "ggml-opencl.h" #elif defined(GGML_USE_VULKAN) #include "ggml-vulkan.h" -#elif defined(GGML_USE_SYCL) -#include "ggml-sycl.h" #endif // floating point type used to accumulate sums @@ -2698,8 +2696,6 @@ struct ggml_context * ggml_init(struct ggml_init_params params) { ggml_cl_init(); #elif defined(GGML_USE_VULKAN) ggml_vk_init_cpu_assist(); -#elif defined(GGML_USE_SYCL) - ggml_init_sycl(); #endif ggml_setup_op_has_task_pass(); @@ -16115,12 +16111,6 @@ static void ggml_compute_forward(struct ggml_compute_params * params, struct ggm GGML_ASSERT(tensor->src[1] == NULL || tensor->src[1]->backend == GGML_BACKEND_TYPE_CPU); #endif // GGML_USE_VULKAN -#ifdef GGML_USE_SYCL - bool skip_cpu = ggml_sycl_compute_forward(params, tensor); - if (skip_cpu) { - return; - } -#endif // GGML_USE_SYCL switch (tensor->op) { case GGML_OP_DUP: { diff --git a/llama.cpp b/llama.cpp index 9614cdb17..61587cb7a 100644 --- a/llama.cpp +++ b/llama.cpp @@ -13632,30 +13632,28 @@ struct llama_context * llama_new_context_with_model( } } #elif defined(GGML_USE_SYCL) - if (model->n_gpu_layers > 0) { - // with split_mode LLAMA_SPLIT_MODE_NONE or LLAMA_SPLIT_MODE_ROW, only the main GPU backend is used - if (model->split_mode == LLAMA_SPLIT_MODE_NONE || model->split_mode == LLAMA_SPLIT_MODE_ROW) { - ggml_backend_t backend = ggml_backend_sycl_init(model->main_gpu); + // with split_mode LLAMA_SPLIT_MODE_NONE or LLAMA_SPLIT_MODE_ROW, only the main GPU backend is used + if (model->split_mode == LLAMA_SPLIT_MODE_NONE || model->split_mode == LLAMA_SPLIT_MODE_ROW) { + ggml_backend_t backend = ggml_backend_sycl_init(model->main_gpu); + if (backend == nullptr) { + int main_gpu_id = ggml_backend_sycl_get_device_id(model->main_gpu); + LLAMA_LOG_ERROR("%s: failed to initialize SYCL%d (index %d) backend\n", __func__, main_gpu_id, model->main_gpu); + llama_free(ctx); + return nullptr; + } + ctx->backends.push_back(backend); + } else { + // LLAMA_SPLIT_LAYER requires a backend for each GPU + for (int i = 0; i < ggml_backend_sycl_get_device_count(); ++i) { + ggml_backend_t backend = ggml_backend_sycl_init(i); if (backend == nullptr) { - int main_gpu_id = ggml_backend_sycl_get_device_id(model->main_gpu); - LLAMA_LOG_ERROR("%s: failed to initialize SYCL%d (index %d) backend\n", __func__, main_gpu_id, model->main_gpu); + int id_list[GGML_SYCL_MAX_DEVICES]; + ggml_sycl_get_gpu_list(id_list, GGML_SYCL_MAX_DEVICES); + LLAMA_LOG_ERROR("%s: failed to initialize SYCL%d (index %d) backend\n", __func__, id_list[i], i); llama_free(ctx); return nullptr; } ctx->backends.push_back(backend); - } else { - // LLAMA_SPLIT_LAYER requires a backend for each GPU - for (int i = 0; i < ggml_backend_sycl_get_device_count(); ++i) { - ggml_backend_t backend = ggml_backend_sycl_init(i); - if (backend == nullptr) { - int id_list[GGML_SYCL_MAX_DEVICES]; - ggml_sycl_get_gpu_list(id_list, GGML_SYCL_MAX_DEVICES); - LLAMA_LOG_ERROR("%s: failed to initialize SYCL%d (index %d) backend\n", __func__, id_list[i], i); - llama_free(ctx); - return nullptr; - } - ctx->backends.push_back(backend); - } } } #elif defined(GGML_USE_KOMPUTE) From 586e7bc561be88e929a9afca7e67d8ead00c53bd Mon Sep 17 00:00:00 2001 From: Minsoo Cheong <54794500+mscheong01@users.noreply.github.com> Date: Sun, 24 Mar 2024 17:54:07 +0900 Subject: [PATCH 715/812] sampling : deduplicated code for probability distribution access (#6240) * sampling: remove duplicated code for probability distribution access * free original_logits * fix original_logits allocation * fixes based on review @cebtenzzre * change function name to `llama_sampling_prepare` --- common/sampling.cpp | 93 ++++++--------------------- common/sampling.h | 8 ++- examples/speculative/speculative.cpp | 3 +- retrieval | Bin 0 -> 1637080 bytes 4 files changed, 28 insertions(+), 76 deletions(-) create mode 100755 retrieval diff --git a/common/sampling.cpp b/common/sampling.cpp index 5a5450982..45d68b26c 100644 --- a/common/sampling.cpp +++ b/common/sampling.cpp @@ -168,77 +168,20 @@ static llama_token llama_sampling_sample_impl( bool is_resampling) { // Add a parameter to indicate if we are resampling const llama_sampling_params & params = ctx_sampling->params; - const int n_vocab = llama_n_vocab(llama_get_model(ctx_main)); - const float temp = params.temp; - const int32_t penalty_last_n = params.penalty_last_n < 0 ? params.n_prev : params.penalty_last_n; - const float penalty_repeat = params.penalty_repeat; - const float penalty_freq = params.penalty_freq; - const float penalty_present = params.penalty_present; const int mirostat = params.mirostat; const float mirostat_tau = params.mirostat_tau; const float mirostat_eta = params.mirostat_eta; - const bool penalize_nl = params.penalize_nl; - - auto & prev = ctx_sampling->prev; - auto & cur = ctx_sampling->cur; + std::vector original_logits; + auto cur_p = llama_sampling_prepare(ctx_sampling, ctx_main, ctx_cfg, idx, !is_resampling, &original_logits); + if (!is_resampling) { + GGML_ASSERT(!original_logits.empty()); + } llama_token id = 0; - // Get a pointer to the logits float * logits = llama_get_logits_ith(ctx_main, idx); - // Declare original_logits at the beginning of the function scope - std::vector original_logits; - - if (!is_resampling) { - // Only make a copy of the original logits if we are not in the resampling phase, not sure if I actually have to do this. - original_logits = std::vector(logits, logits + llama_n_vocab(llama_get_model(ctx_main))); - } - - // apply params.logit_bias map - for (auto it = params.logit_bias.begin(); it != params.logit_bias.end(); it++) { - logits[it->first] += it->second; - } - - if (ctx_cfg) { - float * logits_guidance = llama_get_logits_ith(ctx_cfg, idx); - llama_sample_apply_guidance(ctx_main, logits, logits_guidance, params.cfg_scale); - } - - cur.clear(); - - for (llama_token token_id = 0; token_id < n_vocab; token_id++) { - cur.emplace_back(llama_token_data{token_id, logits[token_id], 0.0f}); - } - - llama_token_data_array cur_p = { cur.data(), cur.size(), false }; - - // apply penalties - const auto& penalty_tokens = params.use_penalty_prompt_tokens ? params.penalty_prompt_tokens : prev; - const int penalty_tokens_used_size = std::min((int)penalty_tokens.size(), penalty_last_n); - if (penalty_tokens_used_size) { - const float nl_logit = logits[llama_token_nl(llama_get_model(ctx_main))]; - - llama_sample_repetition_penalties(ctx_main, &cur_p, - penalty_tokens.data() + penalty_tokens.size() - penalty_tokens_used_size, - penalty_tokens_used_size, penalty_repeat, penalty_freq, penalty_present); - - if (!penalize_nl) { - for (size_t idx = 0; idx < cur_p.size; idx++) { - if (cur_p.data[idx].id == llama_token_nl(llama_get_model(ctx_main))) { - cur_p.data[idx].logit = nl_logit; - break; - } - } - } - } - - // If we are in the resampling phase, apply grammar checks before sampling logic - if (is_resampling && ctx_sampling->grammar != NULL) { - llama_sample_grammar(ctx_main, &cur_p, ctx_sampling->grammar); - } - if (temp < 0.0) { // greedy sampling, with probs llama_sample_softmax(ctx_main, &cur_p); @@ -302,11 +245,13 @@ static llama_token llama_sampling_sample_impl( return id; } -static llama_token_data_array llama_sample_probability_distribution_impl( +static llama_token_data_array llama_sampling_prepare_impl( struct llama_sampling_context * ctx_sampling, struct llama_context * ctx_main, struct llama_context * ctx_cfg, - const int idx) { + const int idx, + bool apply_grammar, + std::vector * original_logits) { const llama_sampling_params & params = ctx_sampling->params; const int n_vocab = llama_n_vocab(llama_get_model(ctx_main)); @@ -315,6 +260,7 @@ static llama_token_data_array llama_sample_probability_distribution_impl( const float penalty_repeat = params.penalty_repeat; const float penalty_freq = params.penalty_freq; const float penalty_present = params.penalty_present; + const bool penalize_nl = params.penalize_nl; auto & prev = ctx_sampling->prev; @@ -323,8 +269,10 @@ static llama_token_data_array llama_sample_probability_distribution_impl( // Get a pointer to the logits float * logits = llama_get_logits_ith(ctx_main, idx); - // Declare original_logits at the beginning of the function scope - std::vector original_logits; + if (apply_grammar && original_logits != NULL) { + // Only make a copy of the original logits if we are not applying grammar checks, not sure if I actually have to do this. + *original_logits = {logits, logits + llama_n_vocab(llama_get_model(ctx_main))}; + } // apply params.logit_bias map for (auto it = params.logit_bias.begin(); it != params.logit_bias.end(); it++) { @@ -364,12 +312,11 @@ static llama_token_data_array llama_sample_probability_distribution_impl( } } - // apply grammar checks - if (ctx_sampling->grammar != NULL) { + // apply grammar checks before sampling logic + if (apply_grammar && ctx_sampling->grammar != NULL) { llama_sample_grammar(ctx_main, &cur_p, ctx_sampling->grammar); } - llama_sample_softmax(ctx_main, &cur_p); return cur_p; } @@ -382,12 +329,14 @@ llama_token llama_sampling_sample( return llama_sampling_sample_impl(ctx_sampling, ctx_main, ctx_cfg, idx, false); } -llama_token_data_array llama_sampling_probability_distribution( +llama_token_data_array llama_sampling_prepare( struct llama_sampling_context * ctx_sampling, struct llama_context * ctx_main, struct llama_context * ctx_cfg, - const int idx) { - return llama_sample_probability_distribution_impl(ctx_sampling,ctx_main, ctx_cfg, idx); + const int idx, + bool apply_grammar, + std::vector * original_logits) { + return llama_sampling_prepare_impl(ctx_sampling,ctx_main, ctx_cfg, idx, apply_grammar, original_logits); } void llama_sampling_accept( diff --git a/common/sampling.h b/common/sampling.h index 79a998be8..56ed991b8 100644 --- a/common/sampling.h +++ b/common/sampling.h @@ -131,12 +131,14 @@ llama_token llama_sampling_sample( struct llama_context * ctx_cfg, int idx = 0); -// returns the probability that token of given id will be sampled -llama_token_data_array llama_sampling_probability_distribution( +// Prepares and adjusts the set of token candidates for sampling based on penalties, biases, and sampling parameters. +llama_token_data_array llama_sampling_prepare( struct llama_sampling_context * ctx_sampling, struct llama_context * ctx_main, struct llama_context * ctx_cfg, - int idx = 0); + int idx = 0, + bool apply_grammar = true, + std::vector * original_logits = nullptr); void llama_sampling_accept( struct llama_sampling_context * ctx_sampling, diff --git a/examples/speculative/speculative.cpp b/examples/speculative/speculative.cpp index e991b8846..8b31b678a 100644 --- a/examples/speculative/speculative.cpp +++ b/examples/speculative/speculative.cpp @@ -219,7 +219,8 @@ int main(int argc, char ** argv) { if (params.sparams.temp > 0) { // stochastic verification - llama_token_data_array dist_tgt = llama_sampling_probability_distribution(ctx_sampling, ctx_tgt, NULL, drafts[s_keep].i_batch_tgt[i_dft]); + llama_token_data_array dist_tgt = llama_sampling_prepare(ctx_sampling, ctx_tgt, NULL, drafts[s_keep].i_batch_tgt[i_dft], true, NULL); + llama_sample_softmax(ctx_tgt, &dist_tgt); float p_tgt = 0, p_dft = 0; // GGML_ASSERT(dist_tgt.size() == dist_dft.size()); diff --git a/retrieval b/retrieval new file mode 100755 index 0000000000000000000000000000000000000000..dd31789f89c3a41fb84b88c08444eb8e4a9dbfbe GIT binary patch literal 1637080 zcmeFad3==B_4t3EnFOB62AHs8l3~%AAfm+$4P_<)4U1@zR&8xVKl4ZBAB3<-}^jE^2}tCptYahKi|Aw zH}|>E+~wSJ&pr2?bI-lc%ctJ?G)XB<@#i88l?PtjRqT}~Bb`T_pMP!es;OU}I{O;q zb;LiTaKxLg38^>amU`w)y==vT`F*|eT zya7fgNlU*v{^C?|$1@N4`O8;)XWsITSJCzPmJTqGbAiXqB57xRx>X-2Gyd}Pm&{u* zzq8Wl`fj-0tS_^VnP1Y*`cxPA3Ktiy$Y1j9Z!f=P1z<+kx3bu*Pq*5aw6nfJ)_Xh3 z&!4|!@$yatM%Pzt)wlm{b5tcAXBUnUQ{VCx%L*6YYW)^n-_?Cn4g9S*VP=rD^Un#Z zywA_Sb^iSPTUO4$<+c_1D+=aC*Z0gPW_=6inYBndx(;Xk)_BZcSg?Hg;(|rDMAvsg zewqPaKdX?Woqy4FTlFovWkpD;>oOjbt@^IBkS1y8U)TD+v-o?3i*L>^T>R}NWQ>My z>wFWw&kM6f{*>UHV1Dj3v#y(C{LXwbMl@jMw&$CbW@;)4GS~VO z7>6aDRMLHY&3(x%f7B=C?LQ&!XY2%VSGtRYulXJmKA+W3NjvLvRAoF;-?yEF7v10f z4_p1U$W78c*JsRIgH9F}E??25zTcSj+4HPF>8SbyZ5@9WofIxAxb+qp+UWX*MAawh z-o|w$8O(Y*no(-6)z4y!1CciV0%jyIO0b8?o)0z?=&rskW^tqbsD9eN7_n8~@>>=z zyXD&kf0A+S%(BITuhe8GO482ypto3fGsZz`jD~N>(`J9KwVIH$vp!zOs_*846$KW& zk?@70>XUR-eVv`*DL=oXs+jfdvFgjS`YY+4>zlV6v~~PN_g6h*_P6vHc>tzGmOq1|QHJT-^y@bbZ24%=Lffv{^HzPMKzM6Qu%Plb>|dGtg-A z^;2eTQjfE)i{KFEzzkFk=PEV9N;jN4P`w{8*XSXX+4J3j>fvM_ZWyHY-sVz?JlbQu zhc#G23-jP-BD6RgAfNNE_1ZzIKF05e11ggsvt|5umMuyEe^n^!HonY8>3CkPxL zo%_qXwiZu1w)510d9Ii4&-)VZG6`mt0O(3Fo{ipWpSrX7%}IDZ`X}`y{oY<#NJyG6 zmmt5H^IaASO^v^9)$$d$EE+$vaNe?lWvj+txs282drOuTEgv78KmV46w=64Iam)GN zw%?Bb+AYhL7cz;*2OYmtQ`h<{Ae{5G9_qhx$#)h*)YKNM{@n4MzgzX+K->ZUsIP*O z9^t?GmK6mHV^tj-Z~R1UvEZ)aZSVEYC|uD))%rh9PQ*5dHuA`+pet z{}}^&-TANSx*Cv?td=$?H774YJ*sQN%Ju!3Wy%w1li!OyYKtn)D3@oY&P`W=L+Qrz z-cjkgIwxICQfEmzKTa*(5vS%fyIooN#cJt1>hq}LSK$k3p2F4lXDO{bu~?PA?e^5n zjbC3oFZt=6aVxzkYNdD97-BE+mBcy3w-V1Mo}zuZc53|O+ABS(_Nrv9cDm*&-&~@O zX`i;wnXKt8nVOoDrF+VB&GUjM&Qq2-Ca^}}3rtl{>hbDWNqhSo8@93WU138cHemy< zMBp=FTUYB10T&@NZph*S&-&UNp5CoOZwpj0UX0oH;e{qymS5F=~wyR(*6JoONCxtZNC+`?bX9$zPM9sbh_Uw+)lN zNNC@n4xC=Cg-_?I@+P+`->Zsem)sb)RbN{?`{>v}xTx~(S6?{!+Ye^E^4AZF<~M&Z zeeI0O8K-AIu;BFcvP{2Td3tp)OdZCU#;R+v)Rk8{yII@5OzLZC`{}Di3!eO7$BKg= zlnpsz)pu3Q`fj>0E=>W4zLgP&~=21!|E!YEQ4Oua@%KC+@AK32KWzW@epFQy2Ltudm8!b8CUNxgPbt z+L>AIjW3=(lK!uyT`6Pd&xlt`8y_@r*1k{h#?x)%sHxy?d zbt|*JIo|T8>HF!`Ibq6}_{r4{oV?x@{_q-i`1I;H`lFQ(ObWEEp#81CA6fpPTZIi+ z=8P==HDz~eYS(gpTmCg~`F7XdN!1B~w(zjXwi?~3HPw=r-%fnmzMDGC_WO@2f5>Qm z3hm{FPhJlkxtaloJ~|lot<@_%zMQbW)>En0CR9p)^@pwZX@NGydwG|l0(CDJr~~?= z!EnOEx#47fOB@|&YyYNtEW|kU1OFF*%l;2#l)n`0SNRfiy&pJvFJ2w!KP1rh?!^9; z*oD%%<}0p~e3GZ%x_*`KdS0rirOajmG<-mxm{3{j^;B;6CK_-UH$~& zwOXa_DXz=(G*raOLqAWe*9+EZp1|+C4`=rCzFzPxHKf4co4Vj)O?9O`7cvV&tx`v6 z+qT)w8QYiXWA;`cZWuf@PiVi%U9N7NYveJ~1<8@=ra>KP{n|iXM!dGP3Er#E3Dixa zyr18F$$FV$&PY4vJ9or8d!E|mODL}>Z_mSvK6Y=_p(z>1Ea+xBW2*b1L2HfBp$X>O zu>xpIXKu(d^LUQl-+V^moTEd7^4{iSi17xG_{{mwq|T_e?6wyHN3q?mRgb)zYrnH# z_mNhAEH(J8jhl>{)Y2UK)O?I_>Sv78FxJdVs&#l$31jf*xqEDUoj&E>s=j3PZ**=r zL-VxQyhvy6c)+Fbz_H-62wY~3OQ@U%E+t=qi*<7Z^A((inD5h}Gu`uig{*fag44`E zTLN%SW3Een8EX%7B!T|qkzevfY*Mf?|jGF(`J;n_@<1jU&fYv z_14(>1xKWV*7*7bkMcWp3!eRgd(zP`wC{5tXc^mg(a}-*qED81!l!Oc2)E`Z!sC+Q zb;cz50U`_2}-0MwMSb zwRHA_M}N6&<)J%gxBlXn%RK$hOHb&3e)^A@I};CIP=5Q(m9sDZ_&3Wc6V;a0q<@m6 zw)}+nfn>F1`JoHSi(TiY>&P6{w_o&RqQ{@!NLqO^(>3?G>6*uz?)B*zO82I(07m7@ zN}tPbvhM%z)cS*&MVG7zkWcde0=VS&z)*hIeMX+S{H}uD^ljE}N`9kkvGG1L{m1g2 z^xOrPtl8Q}m2b-l94q;_eU9DNHT-Trui`lXUeQv@wZ3Ju3mv~LA6cy^xR9RRTC2qxK?`p$n)O`9&Ms$8o^6g<(lZx)I@unKI$tpUi@mF zjHA}S{4KX$GeZkZWXvw|$E)(`C0hApO*>Y=Sjkvi$XerH8@PWgZO@>dgrwrxK6vp7 zSD+0ocvqTNl{fYYv=#HLlc)V)d@E|%l18_yw$ZKCvLlf;&{aJ9giE#AdH1reR)1NQ z4}_0L=Wlcmtii6ZwV66awrWPUI+_!h*yIk@7JuoE?8d|#;tkn%eCf{YAGlPwG)GO` zcqL)Fn%I1BYHMkhYAg9tNp_=4?aD|Bd{EDv1ukf@c~A4D%mMoChkpIguRU)Y-Dx$a zSI>ghuZHGlLi;nA1JlEwESVNQIx25b%h;@k9r1xs-;zq_mYb^d|4E`D)OKw}1J17rZOO&_mwWs|Fb9 z?RMImYNY4d>8dm%ooT1NgUs^jcKN|(c{}YLYL>UtRm05kcG^3_EPu9L{%o_no%Uv! z;W_dg9z1%E6 z#4dk@S>8^2Cz<8#bk@}xr|=P<{O@gHoZt2DE`o%XuT@^-o^&Ma@Iy?xB`gY5Es z&GL5I>oLpQ>8b>?yq)$YndJxC<&(|wcG}y|EN`c)OnS|<)84bp@@aN?lU}#mY41R@ zyq&Hx={3_%dxx0iQ|7SD-r;6>J6&bcYo?v{rkmx@vdf$Fy4_BDy=HkkU1id1 zrk(bVHp>sN%bWDN-A;SYHOt%SDwAF_?X)+`EZ^TQZ_?{_JMA53mbcSYCcSzo-`ux& z_BwbtJoMEy#6I}O?ZkP+D~UtI-z6?3#_pI_O}vb_f%rD!W?~HUX}VTCdm*upcmZ)9 zF*d`r5b?K(ONp^Zr&SYUpGa#Uo=eBM=&dBh>&D~U^qrw~^Y=MXm#2Z)=Azf7#h70=El_7PuB zoJWi;GA%@W32`a$Ma0#_m=Mz%h%X>+CO)57M;AVi*hkFzmX=5CBMuQ`pG+$y9!*?L zJd(J9IFq=UIGtGUQ#@O2B0l0_#CgPGzX=f!A}%FPBd#Vsi@1TfKXEfLCeJjTam0R_ z<|9rZ&Lh@|L&SZEONrx%tBGC24a6SSdeLXmOJ4CKD@-o*T^3?34k0VdWqn@v)i`vI zV%Cjj+z^I zr4>)I^TvEXB`{Hb*QO{VR&l28#O92lLnR^uzd*e@?+b?p>cYTUg#5mvy?wXczQn$G zU^8{5S43wj0**v2v239hsJj>ad^0-KQ|RQ)$g-O$zlb3E^x~m`ww)KIR0_^EBmc{L zdAEOUa^+6XixrFNg5f3gx#6V)18oIk0&N%eQ3qb4oRqCy+plu}#r-SCU7}U))RM|F zR8os7-ds_n<+ttt=7qrf;>YbB@EW+HKMjIM`jf!#HS|ZP&1y$~bitvf8#wl<{MOHs zAJ>al;|Uu5Y2W8M@X3;RWS>6BKz)&gbmRvQ@=XHrzpMPDJHF;4cU;YfZgLZwRQY+lf7hK@GoJVY!eb@t z!c`mBhqKp>s7zXuR5^Z4zshUZ^slTfQROcUK+i>n-Mn^0<=fa1MyyM!%-0fI8X3<` zDW0-vZqExd?h3T7;dkWTV^!omlB(LSM{eBc9#rZ-e2tAIS~BHz?5d z&N;}8-%^h?pkEE3Kl;c4*ct+D?_(?K&p00&7HE@q?3AU2t_)1fOHmgM2e%^#GDkbh z6xT@^$$!DR{*`({LxtFPgifYQ0N&^QiIs2plPmKWSCLcE0p>_O&G#`+z^};0AG56Me1X1Yx~8@$DjKCO?A z3x@^;>hyRuAT&r}6PXUZPJ?dqpx>*Y<13Njry^s!4B379^0oIDX~3Bkow&kuw$p)r^;i&e&j*Tfd39h z{eSDR{@gHi8smDO1%ITjqrs1@@%c#_WAqx)1_#t7BsqrGMe z_d(WrC;p$L{nF`yiKXb5#b@oERD8!B*`-!>Jb}-QP&;nSWZGzUF?5!SJ!_z>Y9O^I|O(dfuRXq`VesZk};9C zhx9e)6UP0HwqHQ5wA*|-3jR4`)XMJKEOykfqn}krjym>q6x?&Bsy*G+fxlT=c=4M1 zwYa!h!i#pM1nz%3QSEvGp76Fi>4p6X1IylJAM8EWnI`s&n&R)+`BKp2NAJ3mYPPRB z|4I1tWBb4-dP2yYL718MJl>eu>AuGRY7vGctWv(1`M+jqU^POjOzYWy132&3Ki;UCU6ic@-T zBePFu8_G4gR=K*i@rPC8S9WOw{OkZ<#VK)}{QyU?`hn9=M?X5+uyAMhgEsa!+c5E- zSo01zF9F{r;Jel;W8+tF+XQZ9ojbicm-&^$JPX3(pD|?3k59l$;V(x=X^Wh5PY-0K zi+u7~cz~>GlUh@PH6JU@kmhE<7Lv4+y~nR4_B$hUe(0;3DBiXTk>rCJ#J7 zhX?rK0e*Nun#BVK+B{$p`>9bpU@|;FhX*90m;1=Sjrrf)F@FX!|I@n6|J{!H|2VW~ z*U`fKm$`4$!`y$UV-8vMM9qC?J=-1i>|pK>QkSJ)X3YPA%>O~m|FoK4&SnqIG5;TM zlu1Od)hQ!(#ci~Gh(5?%6#n%#Wkc`)_N@#aCNv=R`?!Us0N2g)oyu+!oLkMv24Ie$J6ACUf*(Z*)lP|2=Y zwX}hZ{KBuSHUxjdpWZWgl&3}bfbf}P@RUsWKqh>kKVwh@Zg;?I-gl?eG_Sf~CGu?mdGQb z_m0J%ME194u(w)>4rA;aA)hB9hj?gr9Bme%qwVm#P_YfYsf002WM3*7eRL;0YrU4# z(uAy^#J*HA^7syTnB)ucPUM|x=D+R^&-4n=A4R`dJlz zA3Ecp{#@`Ro?Y-#)3-p{_pJ#$kXRiIkEPxP;46-P=T*g3y6Jaldt#+4`MHYVcF%Ar zBX*Ju4{`=DTJ^0_sEZj0ZCI%K}g!UYj{Asx}L%)+-i`oePu{N(F^`EL&R-b82rzodZ%Ck;I2j!6Uc z(7<+uJZs?E`=mVvpOpr@77Y|Y15%$M*P3;%vFmiwLHu*x4%|8EApXzZ4m=8t+H}DC z4xIi3oW{Xx-X(s7@HF8yLNpzeejYkFJFW{INPjX5vpR5N_m6iSeH&Al)zKfGJNm$L z2i$KvVeEw$czfUlg5zjjAp305yufSmg1o*CUa&DPCLIK!6>IKx^z*aQz&RETXuxjO zW%!&q$G?L|JXesd9@hJ*hx_+auY14Mkw^H5%`5&~u%^rJ&_qE8&wy_^%7(%30O;al z;wK0%6Alxi=>mJjXXg(my3&R42ODmKH?Y_3NXP7-kuR#xQjfDAC!8?5YK@UR~}u* zaq94bzxg?;T;#qZ=!s&}o_Px0?T)zeW^8i>?3H_UUwI=ov5?qlhbVM7Vr)R6k;IwA z$bR~0)n#u!dLLbU48Mb&SlS(gUEU2niND|);%Zl*?GG-!teE!2R`k1T z=*y?=7n{BYk$WZLXA$dtbi2~t31rtxW(L}v{k83d{pqjRyu_xZU#sd~1D?a^Qz8$G zUxia9jwf&NOP+A7c4dF0yWh?f_wTfI#eK;c;NFHk<3-tbh#Yq}{Sy6Jbp2>MTQhqS zBKsF61zV>hKaNFSEc$su<=8~E`#5rs*xtr6ZZipsl3%Q-txc-DIzjEe_bl`P!mpBF zs(5dAO69LSYWMfi18yT6OMa!|!vE=CIe>S9cb{8bQi9H4a0l?yiK2l!$+!1aPx#>1@U=y6 zw{00>Td{4cvFiV>RloG@0Jf-e;A8y>qiFBPq+{7Z^B9v8j6pah*ebe)7oFc87awCX zjgWv$KfTIRIbBz~(VN?f2s4rQH-{4|clK4gH!~)pqurOhzv89Gk}FeqCwRY)G5J0| zn(yJG`QDh9D$wmOeFIyGm-Xa6@VFR!E+$R^pVD>-_>{e%An`UWxkV#3ZI^CijvNM0 zHZHHAuj2P+<5Kcz^i$@NGv9^eYXq0a4O|Y6z@=@c>aLBERvUs(X{Q-lcn3W81E+1r zks$~jw4bXTXwBEKf4R^VJN=#ozarmO;ZrAg%>$p3_L2^PM|rNt#>0Mz@m#>O%n2Xs zyYLcgFZ5B_3-$Xvm3~_%WSllWlpEfZ(&^V&rSlHjl`*pY8a3X@9C6ZbwOj4Fl>U^$ zmu=ennOj|Sn7u)J?i_B7J$Jq`X2037YptNGJgg~ zxA#f=#nPTVp8spLXP3SH({|t)6q83jN#4l$(dql$n$H}u{J&pjeCh~7|36~f`Vwox zr*dUK)y+Os9Jb1M#;gxx))&5}IjBdV}{pQK&;5(DeaXgQ4^fQjh==SGR=Sz&^xQWS?k1&qg;2FUr<^p&u zqx??v`XctK-w?eX`X~dh&(qF&c#TGS6ZjSXRN zjjw5AKUl{7UoLftjIlj8?C~uEhaU@Fbn?b6vJQWWU6nnt*}aXi?4>$ss)2obHZNy8 z+p@>@TC1&7|8n_T-*Ed+o~!DHlfN;yK6`HNUD?g&0{^Y5u9Whj>r`Df=N1mThsPZI zNz^5N5JFdT4|>AOuUBo&gAM<|EzZ3(FS1}Gb(~(E$Jk!Q7+=X)Pi4%fu=mIwRj#@B zD1AK$ZPY?1HxPbEC?+f*{9|U^)^quN1;2O9^D|iEWNoN-@KHZ=emX(kYcg+{p9=_* zE|EFQ++RR2ez)dpcWc_|kGjlDW8AhdZi18E*0gH+BWs$>N$2>{-mX|{N|V)|l#@B# z-J10$@2lj?rAl7>QirWsDN7aT2-Wv&;<{r$XdEKQf*zXV~+@{%sm^ojphFcVT1? zd!E?uJiHSg*WFzCQa9uD>?db7SDLMHI@7r_$ZAi@^)wDipK}~OJ!#H;d#?Q919PrC z#29>5ouq}jjD2|g_ypFTbdpB$cBgj;EZzNfraDOj^*i)IgP+~R7=?P#2XzbfSUSmP z#m6r#eE7})A3ymIFg^k?@bQO_d#4W@_E`9c zrK?8k@jm9U?SnFsXIq{bVc{lZ;pUPa^!WkgwdIx0dD3-lY~LWdvm053^OCFMIj5?X z+weroqGD&TVaqJ8&u(;=WcxTn)Qq2;=nUd-`VeZU`b3;##Yv2YyS59xy~v)Xw7A7HDt z`XFOkz2RHpLlq(S)f+xkraoZn0o#tbw-yOp_-Q$Pk$iFvOY|>fMFSRaDCr>Skm&g? z^nLUzn+Br&kSc(u!2wUSU1SIC{(un8iw>g)hOmeD(G5kn&O^7>`V!c1%i3+dz4^wqW_ zJOWM@)5mM*>nq7GSM>jFze@I|JNhX$gL-WKVlyZq7Mnqk_zkUJi$;7B`@#uqe4>*? z_eF3dH0;FFIToIDeA7GE#MpQm{yFh#n#FHM1EWTkHs?z7!T8ybtQeCO5dlTkHI(g znS4X+1Fs{;oDF`8r^R)dGd3^m&c^y1+86pSZ>@~n(+Zt>b4r5>I&CX!pYi5E! z2)Q>Q#<=_xoSf0PSn$Uhm(Plq=HrgJ zA<`aHe>%;Qu1<@{kiR$-U#{hyP)`tc!t{v(6MR7W`iXUT$*Y zWkBzEx%NMdm&;<{6h?$xQI~q{rq{rYD)iw(&mwZdZM$w{btQnpWQosWRw3tNgt!BKYASY zV6!_TeqzBNi#|TCjh%yZ;Trfk;c}t> zYVhq?I-RT?53!GHuN`8uu-A>Y_q)!?d&OSbi*A=dTVmtVv2h*7MiT9tf0%j$=%}(j z6f&-|KE$HY+um_+{W_sLxV?hTd>9LCtAwb!8D*5RK6o6w_eWzJ+B&iIUDa%<$67=8X28&_|1Bx7R39g9B1 z797hj#hzZs`*gD=c1F{#l%2x_~|0>Ac%?er{Ooj=E)c+yqbIyvyzd#0i$&aZ?JmNWxpc zQsL%bcG?@G?fbufuhZt(3~qa~IsTUX_L?O&M^E&5>NBu8iqAnc<8?n_-W2YTnxpE( zpSIoYI&kV%H#W*RY?blYEV0q9#zzQyWi$>$jH#S2DdF7PNqmPw;7#@ss<<1V0el&I z3OpajFQ)-Kgt2>zO_v)}=ZIZ@GIJEVcBC*$@;9u4JFyKuhQrq z(%}ze)AOz1QvB(J$F;qd;MX~G-OO4(%&_?mw)P5!M9j0VQ${C$<6k_SvmclxK6aaK z54CP~VPD3->1x7y@#BLw?gf@v1o4v;KfikikS0i4{QNc#Bux;Xs1p1q7t_!6;vdDc z_!}<9xA2A9}fw$h)ZCM|j z{*VdG^Jss_67X1vKjZ@DSI0e%&|M+=@dD___J@@7EV90femq~+8kf4rDNp^J{>fU? zW&IahZ7(tv-?d1Sy6ic1zg3s4@v=@%1HWPPb5qZ@*RIGtsJkd*ukYJC_Tf5h|F)jJ z8QguBFp~Bjhekh(Z(}q)iXUTWbbWXMdrdz43w3-B3*k}59YelgSnxTWbJ2d@%b94Q zTh1xM>mvOP?KM4GKVe=PzJVpio}20KJ_uWgz5eCVXQwZsy$1dj`&e&lpvbY}!{@B~ zYp2^Uw#VQ?t8S<7;cEPiqkRuUj7@L8hv|;7arz#{UUz!;J-mSNn?iUd$MiESXB_{@ z`Y~!tZg>pmH)Tw@n=|5__#@t9yo;+Doi-&bc{VuG^80$y-N{`(%a7G>`LW7a8+)|) zu@=dGBV#RNynykS@s_a`AI0v*Hq_VLTNK?S+NSmDJ0@>giEP6<)nnIg2;7%9skqp+ZK{a0$y0CcFrTpHVpYY?fFZ9uDw|(S%qx@87dCq>w znrHJgQ}>N6C(0h>@4@ZcEwSV~t!5N@brb8hlLwRoLj*4YhA~$Avd1hkGpC)poNejJ zme}3?)^};Ei|$fAzT885)!<9o>&eFVZOYrcr&G7?%6r5=b2MQLK?A?h{vJ;LV)%Q+ zpt-^|F6bW(pI#Y4&JkHd;b0*F>qQR1E*fzMboIw7X`0DCtmw$VYvrS=ye`^%EoP5OP4ur(Cg+i#O+baJLkBi zD|JJ!UH$$330_|zJn#kLwx9>xmc+nqQw-b&d%&%|=0xB(YR$3nTh$GIFFHf~mj9dh z-2-kPB3xnMxAR`QBgl!7chf1(vpc>~5@QW}6B~kU2R*`>UR#ckzTOI)QL@9MV)s0S zZOYJJxEs4PC%(0rb)yu!Ku>bS97msfk|Tadzhtd--VJr!y2mnNKEmUi_vjrFJBplH zU6)pwy4@X?GiR~*Y?ZO@oAQgTHyxea9bfIkS=4yWOvTB1)yWH_?S7H>>>WYx;Vkik zX)n6Np>+ef?=W`zPPxPRU6KZYAtoQ{3k-c~c8}s)9*#BRDEuf|COX2|IrIPnhJn() zY5$6tw`TiY=WMmnzMO%J!`=~(-J=iokG|MJIRCa9KP%2gMc^PqwdMEcdo+e!%3ZUb z=XpGT2(GsAJd$UzZ9T>FA?#A(50|e77H#MG4Od!Gu@R>h%_V+{bijyHi!zC~xl)T1 z@kf>|>vC+bVk14=;XgLmILrIgTkQ|Fp_hM*PJpf`x`aWWv!TysOP6Ry_LB36&B$4D z9=-X36dMx*Glo7vuNly5@!m)c6@;t8a zygf(fb0_8Hx#4l-JD)twSyRG3e$V2!#(Q!WDZW+0{fjkolS;BPd2T8TE|Tw;s5q@Q zAhFzQ$X&xCdt*!6<#mgm6lq5*rEfM5&V0keSZIFcguija-t0^{AD7WphA1)YS3|L1 zO=H~hD03C#cqQXGmGf;=@W(c7Sl?%kts=}PTupfCsC(-n!bw5`>2$)E2&1fd5!+TD z`+U32BX0nH;T_*&Y_K7Aj7mw4hY-s+ zGrm3xpM$_>i9_$Sb^ePS`0V6Av30s0Y<8VIfcYoB;?pd3eTBx(|&_WXP@iZ8i*E z#5NnbuNY~YO^?D$PkV~bfCI6U)HBXsBpyyT$B_4t)3$sLJiKMq<;25tXMl&lQO-#p zJ>lU&2ObpP2-1$I@hVgucr?w#T6@ zxd%ed*(`)-IPEdp#2&+5-45i0Fzd~E*1dT_-t9zYsD&Tf_vXnSp6mnrxUOq=&|cfcF$m&x4_+nDDP%iS9CEO$i6x&JESEbiWz_VdKbX`G==Wp6=j zN|S}Z!f);UM6WzkR^HdcXA6Oqd;Bjh1TL{vacA$GC5(;k%}n3S?|a~zV*8YJ=T2kZ z4K(Dg0>)bQ1*=)xEj#KK(F;Dr7FTV_UDdKavo2R#a#yt_cQK~pobzs$RhPWq3l5tT z;me{^Ky$L!c$sk~!rp6)+|!@zpt;7^JmC*tN(fsr{}!ihrQYC6vac6;&gAH*9KID~ z^50+4?;jIxB-}w*NI0C$x1NX-3)sN`2`(`E3C!r^uZ%K%J zsKK}0F?_QW`6BJpXXcyr!M{4!HEBcS2Jy`jyHad9FLob7j`Kqs;LV+Oe)DX{>sC8r zuXoZ&5i}w)ODS~H(>*&jY(1SXdWtq8aS&UcleJZ3m-Y>vx^lZ)P1IHC?9zSJ%aq19 z#u!@{e8^24M;uS+BZ0EApThSxB6aC~?gQ^#5x?l234QqHRPWzoi{Mx1a(9TeZkHH( z5_iDZ@)2oU=Ba`=*fNjs2}5oopUhXW1vSA7qW9b6o{|Q5QkbxbP(+wX7)MAU$a-I$ z8<_YL=C+KR$jUOmL&p3z$F2~(M(w>B^ZOHQZZdBA5M|ttB=ft}n%||?{N^Ln9c$|K zRz2-4E+vEF;d+Xf3L9f<+9KMaQ?*h4;Htlt&V?U1XB%nJ-@PweDE6ILb&)*Vz zXTp_>-n(LI$2Y~=n&AtC{e<5T{&I!Lz@2m@eihLBXQHh|=$3NR2Z2yFW_R=!FJP zubv5xW`L{d;A|SYdLG}gy9!)h2~MX%ds7y*=bGQM+XubX5*{R!5Q+$w8}w(&N-=bW zPJD~aD!%nn(GkR#t(r5$Iy7K^AEp2t&+o9Wn0NBs3O*dNwZQ{E#*P=VctFTn<3kn? z2w6Pf#6iv9JXrm%d0(Ja53rUG1}8VEx{!?%_T*x%(_Yr;)2lmm2V;G<^rJ`U>yHSZ zUT)gzL^gDuCy$^l(KqQM{yw{H8BO}_Wi5^FS>nU5n_mxQOw9n>!L-ck5! z$l1^)-oDw&N(RaQs$xUC-B?r_`Zje2e1zQg>NL4(x%9$vZn0?SNDLIQsh^WVdb6N ze^$zR)xaGvA>vZvs^NjUCsG4-f58_`>_Zy~&)=SLonD_+wqSkW0lwRCKztRBy8ShL z`@77;I~~78@d1(cgz$<z^x7Xk&jT_)ALy9Lg*mPy{kQnP&fiY)#{+3QU_|ysGp|z>R zvFXj+2QX;Ou%D=7`p-)%;!g(;8tX(1+FQ$7&>ii)#u=sVXzyiybJE_|&w%y@8?-0$ zBo2Prc|P7rPt_aFZnf8t63ZSZcOqn=t}HzrHr5aUcmFsebmKI#i^x+ z9cdPH2LBLx%OgGk9UHbs=%Y~X0>@4)-)PC( z_t->u%svgVoS9>M@5+T=iyPkzxeHfdPP1Sxg$K*N z?#7gyu&d+uw3?)jxghxm)ja5c`C(}DE`rqcNJrmNYh1t}cBGpLQwjHjXPLKySo5?F zeK9q{=V-+d6HZ$n!xr$q7hjpa*pY2tnMnIzWciyZZ_Cx&@t-&CJCW;y;Un#kT}Sl~ z)QuhzsI%nEI$MSme>CoL>sIC!e>C}Kn$3SE^X&Wi#)>{RF9PqL$((A$7kG&V(wpx5Bf*+%aH%z?{Y7_rj7imt3Thuj&o%ckiJ9i;=jzBu7>a% zda&|&)45;zr{bSu-o^;7l zz}c0AFA3gqBkx{{#Fu&ZQnhvWlE}y+FY5Ip%l!4D$`+iT6P{i_+PD`cR6nNdt)F%M z_S-k+i+D zmuJ9dp34z`z-U~?g70GBJ9MecIqg8pSPk9QwaETfTxMow`a9@vdFW7nLINT4GUgI< zM_0a|HsebndrLwKBOSDG7O<3BxJr09VH|qV@8m9L(Fp$Q`_$ zcVWj__KqdsNaRpCFQoGw0R5%}|3u?#&k*ZuPo%wlKKX4KK>f)RPJ1~4IbaAjv`D+_ zsav0|78)K(yN9s5z6c)vo6sE|WQ~o+Llg44>;sAqNHiXF?geWG2hTb15Mr$pe-rUP zk^VH(cOU2V8UzR6qMA4wHzCovVw`*O(@*g$@d-{MZ19mj8X4ph-;7KCln}n?Fxf#$Z3c zEg`haJ!+)iz(?6jd>=aQ8^F+n-gF;r$a~SJC@Bbh)5FRJ|gm5?EHp15l zQwXtiXOSgEhpfVmDdS=5p!MKW=BuG!8n(;MbAW{zah*1 zSkS&`Ej4&&SN{_6#q}YpB;a@d4svuY`@MX;ycE&Pix>OP3+6Ani;e=llD?*@B0_0 zRK9KO?ebkUzNwgmOq+r&@CC}eXYB31Z0zl7{ML^)5@|#BcH5b61$Ns}z!?dwJq?Q(t{u%b-hoRwk+Pw3kcsUl8Bf0FNfKPA7D7G6|9JX!H=SK@qy zUPt^0{doi4*2J881o(wF$$Y!ddS4*V)_X6{=U>BEvZkgGzl8qx?kw(}Bz}o_7V#s* ze4AqTeZ(&@R~F0f#6|F{BF4A~{$~4c`+-sCn|Edn6`e(d}$^;KsZ2ng79;~?Sz{Nvj~?FMiP<; zCuLs6mbG57^<$B>Y@Q}KF?2@8ry2X0$P^8RUJ0K=4qJe(FgCe?Xpe_a@ixK_wUv#3pG#K#XN_#BtxgTM4s0T8qRk{kyl;e zSo&oy_&V`d{2H!q5*Gdb{s z&LsG7%eNLRy8$#HvPTHIng(7(<_-~yy~t@7nCFnc`_Ui8TGe2{9if{_>AbcN${EXxKrUh){OJBlPX_?--s+!0v{It zpoPfdx3Q*+@6Ai_^IFeK6&bGNmJ)dFWAMS+6yz%S@MKqt;WP9oJn#|PKCpHG{Cq%* z_za0WUJl>e0^d`#r@pKzWq)R}%hiI_V9p)z+6&j|l?pz4VZz>ueE4=1e72nWGuOJP zTeWye7gP6Fq+a&$6Re1>5GvLwU3ows5)f(McgZk+&QzyKU_e-og z+cjnI+_}_wjjTQLJ3RP8z7ui*LC)4|q;=%44DccRHxIs+f$Uep^Ju=QWcvdh0Z-WT zUi}ty;PeH8SHB{@K*z9oiCkL^yv@VVn}DsEJIcf#sF^#;s)3=IJIc_{UQv^qqBWZAd@tb_7oO z-p`HDZ!TdRVK6}>w7@(5NO+16CWw7jDbbi!c^fS~O-S->ldvANu(Xhw5kES;sbXaUTjFb2c zPP2HC%p2R*AwIaaA9w-t#JQi@gbX5bzWCi4JdCuA_s!jWpKKHT6yD*m(;M^YY-^lL z8Rwq-rlh|OjB|*w65lTNha-1bz!2)<+P-hxJTi_^>`&ai2Z0_%c|qf1Y__&j+C& znKO4WnQeZRviz$x<}n$G82<0N!m&AL$vyw%os_^Pe%@Kwv*mcUyLomT^I zHFO^R-5mCA3_pL{&nKF=`5?z`00N$d9c=g76>AZhyGKfV58r^c zFNryhrf;R-N_d{&N$id`j_#qYX#5Np`>{2af*TpjQfn+rqsG$ikL`nG-**o^8D|b; zKTd-`bZ2X_Wi64NWQ~{efp;;cRq&3V8Dl!wu#?)dz&C(b#=hF}`Lx%lcd>KXvVice zYT%Y{D~c@8V95dv)|d+KYG53McV%=Z4=}cu0lUa*=Jzs9IBY&P(HdJPACo#|Z1E>{ z@Uh5pV;y6=7k}bE;7=@c)1AC9#2VW_h)xxAZ8v3A!Jptx##r!XjyUL!)&YADQua0UK%Z8rPv`(U_pVOf5Ls84y6idL z$(uXv`|^zf@f#4I0kLzhWBfyeG0?*=nd{=0Wz4G{@!h#EY4c67Id5~|d_S^8pE&b7 zrW=et54Gl->8HCMIAxq1JmOK2z0b06Ztw~AG)jer*b_GR1pY_{pEy^wIq@|x*TlKv zY@zM}N7$lAM#&C3KDu&;SSQ|0cW;1^`f zg?8q`IAp)Rj7ujU92UU`hXT8drDI+j7frbA@j3oi=*3aDf{zNE(c>fS$Q-%C z7@wgQkBzLWk-F?TlF44sMaEu`?0-b=( zyIX_Ac5BPrRoJ=hx#E<$o3Lp+W$pq=J7n%A>>h@!AvO+cUrcw%!D8pIZFC|Nk0fp5 z>=*P?))5)EJn$mxQcq_RD#(APGH?_9ejOZiJvU;0AK9sIX!NOLU(lg%1nFBfwug-b z(J$(`GpCxq%O0VmgQUf#Bl`-PJY%~s_A1!B4oMl=Z;)rn&vO$tGNEf(qih`9LZ3nw zO?0j^k>^cbp&!2`AX`L!PiZT@zgqg2d|UC~pNUTm`w61+dL25R-Odc!nL7)a)G0sQ zkXZf?>}1ih%X75#&V(n}GdK6;oxZ?ABeE~=A;=n#fM3g4LMHw!p^r;u`#jalk~s6B z^z+i^4$Vye>IHFIll83h%wT5vY@R*7%=GO;{prbpG3m&Q2Q-xT@O5P!PbJK8w6 zRyEC=UfM@}#iijplQ}Xy@M30qflH0EWd?g+|D6fXE|j&R8F;4Ydn^1K6+U8~nyi&` zF?*?FQwHRmYGkcVh*RY=JPj3d!`NW3VQcN}bD+KZrEDJMJd}Hja${F6XkE|UHO+&! z4O>t7o2~MB_^Bp%_VQgNRhcjJ@M-%Tow01>ZUCv{^y;VZ#pnC&p978`g7bXZ+zy;? zF^@*B%xpcqy5899%>+jK`FrHE+p%G8Hu@Th=qu}q0c#$x3av_C@2Bj@mA+Pi*RC&m ze$by+6F47MsB?!-Q~&ZAnqF3fZ`gUOidwf}bt-=?$(|EU}OKCe*1s<0?Y3=R11KcmB7M@@3W!wkSrtpR) z@-3vC?cZn9<;m;!8F+B|LNK1Y#5Z9t`RCH6obmYZnx~C?cHId=Q`GJJP3pcASj0}> zlh5YI?7y61trFi4p?Q%rg;(KU3Y_kmNAL|7`BnT74*|FM%(d}uy<08a&z#@$&wyXy zqwyo)3#6%CBIAFIpQ`R2o~Ua>6BV|2^m5}YP%Cvd_EF&x@2@?WExtXz`QGk?+#nFx;6K@0DZH^VQ|eTkJ=izPPOG;ANl@ld5@fv`;Ex= zgXR42Z=yD4Ic9)|0W=>t)Sf&tBg?_N*5C zLismd&OPOqt2%uA>+<=={0w-22fCULO~^OTiV6!`jqjcn`dj^8RWAH82*20}{FANx z3FIG3{>{L>xo|=2BI2EeS*_E_zaIV~`8TnqKF|DKKn(xhDe*@eVqR$1A% zkh0a36W+r4!nECeRQXdby~PEb8`1wpuKG^vy!Q58bMMXx-~TCkKmGd%nIW3zoCV&_ zAzV(FMQC@&!8a0+a}(J+PvUG*GJGcm-LM~b6}!vvt9rC!ug^WK)o0*RExp&H8GGI0 z?=16rA7=tnE!dU=+xn+{Em_Qsjo{$oRm)q$vyjbzuTu_=l;N-cGv5tlJuju-YSn~R znO}0&wAfW#S4MvfUk2VGcPuS)YwUrmnl9}LE+{{e_5>GVzZXC0S%MGBo}@mbyr#`* zyA;`ZylOk)*0ThkPxVp9PPqHmj61F7JlaPckoh>|#@kwlv$uN%bq!@+>enHkuospw zPFvdWM>S^{bFzqarD$TG@{LOMU$82)((g$q%hWPn^HQ$l4{DcB|5Js2*I>?;=rzyZ zUTCdp!Bg8;s5x>@B+K(B&SDR)oD3~$jB%A)rAgU>)!mm3tcqRs_g2}aHQkrpdwcA% z9H=+iZ7S}e?7G-xKd{OM@9w^A=AE(2o*k#sCeYts-=lVY0{$B>4YY~R{~^w7%3jC- z^dLF^5lR4$`=jxA-Ko9Z)H~9u_jswZ-r>|c%&7O4Ug}kM$A?K;&l9@5rOqpiWc;H~lU*!3RNRN6@Tap(z=bNb`M$9Qgx9?zUp zYJKeT@NomK<8{$+rPhf5W>2^cToSv^M}SFid);rk*7^5d>fAmdcAY=A>iqStu65qm zOP%V%*md4%)tUOZS*N-_@H=y!m^Akr&N27c|MO$lyO?@?^gr}7=NvKT-ru=1KVu4eoKM8@#Gz}+DD zz@3L3Wuy2g;m=^-KO20W`ZM`<1NH4!(Rt?w=@P8}9|4s5wH|l@BLTIVre<_Fm+k$@R{rD8fdRYH^@}J8)=x>_gTY#Sy z@48!4RX4onMQG-7@Hgy7YL`JPz$fG50lv4ugWUg;M69~3;~P4se7KU z5SU6P0+R)&4O7!)!D4B zy~PPn5$6RL0Z&=t{)&aik}H$;;bWPwzv90A_?bM7Pno{I;^p$>%J1|1E1uu^L2~8Q zJj*#lopNh{`vJl;gf|J@;Vf`tY`PKNkKSkU$NS7Vte-o>ug3@O_Y)H42kNF1x3BLT zZeh-x#~QbAm2XcTacGr)PsoU~_8jUr%YS6>HGX?eeZYC?o_JLwGI}hzC>I$`=G*a& zk$iPNbF-)V=3)noRo{54zU9ar(pT5@=6r3k z;u#Zy;j6F632)-;>(L!8e>CaYSYNI08+X#Qxt`m! z{{(ph|MG-g)Gd7xokR4E6YR0}H0GIi#qP@(Yu+8d*(pnjd?x3Yhu56N_p*Dcw>N$= zVV<*IQ;(7FuI=cJpZ3B}wu}9WG45yZ7p+C1LrdD$WrRLupF(I)WS}eHGw-r4bnqhj z*TipvyDj9CKAOB}M9mTMg}|@KY2ERl8yN?&dxkcccR%3=utm-qzx|iav&JHC`7RL| z$^6co*bklKo<|)u(9-)8kmYIb*I!Y)pEY2N-Jfqq^(ROf@AYbz(1MeO1rJW0eeUqs zWB;cfa3}i|PTUE62d`7RhDuu!?o_*lFZ#e=GjlGJ`y3iK+>`Aa4BoDaoP!#9oc&Iq z--7RI&D?{_z%RTz{@geuHthR)fPF+1?0LZ3e6_i*q`o)3(1Q$ql(E%#uaKiXUQ>=f zI?9xzoomiQ1LLc@$Pmy)( zrH%yXBVUV64~MABIYvScRadHAsl+|eLnHpOvH03I44igds~c#?IX|TT%X7_nEwpFz zG~xd;pI)`r_U^`XZ#R7PX``=Q+pmsl|3=zeHQ8)m`r6t45QA?C48tvcX!1yNp6>)M znX|q`boe);b@&@k1=wedg%|P5kUgQsIVO)Wbw<<96M)t-4H+i2<~3`q<|2m+48ns} zSz{#l+i(4zd7a4ECQOpY4_q>i0qS|y$TPHMue4?Gcj3im-VE|dp7-G+PdIU9^Sn^l(4ZbTD((u)juiQmpm&v`ZlbF zJtu<>WnWC{kpAU3#|0kt);&{LgM;CBe-j9ovYzYsmM5-Fsr;C8aCwVTD^oe&CUda? z`*BOEYHP99>Ne_=J&2J47wvzawb@*&&2{K6++QU46Z&@2wUfv7EPraoex40?ngw@` z;L?J-1zoL_`m4_qe}VorM}L-U++}eTolDN?9UW}O(-tLG`sv5bE>D^8J&`qRTY|tp zw*&T6#%)B6+_xpLm;Af1TQu+{_GYQ?4ucMc*5KolCVOjhkqL5WL)IA41@5)l?QRbC zjIY#|%+cmJ54Y=o*ApH|{hf0ywPuCYMxM;4qtWx}>&CuStodZ>1j9_7z@S}IhIz#5 zn~aI9#q1FneJe(Wm$7T=mm3zG3eEl#T~=uH1#2unZ$I~1^lZxl_eQ}WyzY3K+WiJR zz{_*h@a}CH#og%fuh5SPls!1qTv_ zPWE0Do6kLD!TCU%xmMb6I@d#64jpR4X?zRRd534@8rIK(@f$tpV7<|z{BDdtbHKKP zk7As}XX$cl4u6mFk$EU%^l$M9Yn~-Yv)h*;#1f*$ncM_x5$7+Y4=tb1{S)Lm_T z56O^K*{72`i%fsh$Z`BH>ge!=Vm@~AnxQqzSV#W-J`3Nn)@iHH!Uv?^vWK_-?5cPPj#wJWGb&Vy+ zxvC)cTd8X-LDKc-lBTY)1mkzoQkS0~Hn;k7NTcI+S6}^@$ohhkcGOo7pLNz3lr($b z-PIQy60(CEeE-#O~s(uO9uvSznNJ1bm#a)|Yo*Up=-I zXMI6QJK^KJwX?pUq`!x)_^ft={yfAH^p_S{Up?piob?4sJL^;Z+vofMo?!CF;WeX> z5pO!(K3HrcVzXb({&j)nPwdpe!Us&AV5~EiO~0~F5A&b=<7h|DP#XNE!$&!CO?n_k zy@uX|%-6|VBFo%O8L>Z#Zr->-d{&0n4E==f&my-rT6(@yum2{zS>$DryW2i($5oNL zB^>spQH$E=+(LaK7YPp$dBb>ye`dby3ExP$7Rm{ZZilDz#5?nWO>_n)p781IR;Ry? z{5GGIw&x-@O`yG(X|L4s185yDHgq*dd?Ms7VeyG5?NeMQeh~}kZz+5DI(`wJujEYh zbF3ou>OkAzzT$5(#Nd%fv(#3-Z*g6QHn?aUJ~$ruVW+=IYD+PGKg0g+-dgScdh2sh z_it5+oy73>zXpGaS@=y{jsL_<{3vGN!#W)w*0}QX&WTIQp!}9b{M4Gb|94E>*5E(f zTgMV|c|U{qeR=;?o(l>GLD@G5&k+&{ODKOMA(t?Yu#$E*6Sfm@q}bX( z=+5uhIJ&6${|_rKKkJr(~l-@`fK1qlrm)A#H6V_>B4zd(I+nu6o>MT#Tmg1LK%b1{N#~bewviqiZwDbvi#{08*?=kag zBX~}<@+6aIxRJ*rd4UnJMbB3(gOnGt0bpPMwpk$CnP9 zjE~t-_qMOImKAyF*GJ5&p3aY!o)hT6ODlL8yJ}_Ydc!yKIz!%1#6L{%@+0t);WqK| z1Y@usd=zsgxYHIEdFJS@R}5d5HV^fM@o5u38PD;x2UGD|lKG&BL$mQ?!Dm}|kUmqj zg?>mHpWD#mq#q@1ohJ!d=Sf1=d6JNIo@D-P(T{8&wX~Ys@%w9>jg9YXk?&W?`>*l- z>yhsl%lohM{#%jnL-PJxyq_2O9$%gZ8|U%<+mY{YlK0={{esB%+_V2+;{x6NpF^=|O6`wWnV;hL?)sud;Rd^hCS{3a_o)`ZrC(o+lU9`_v4t46xCr|78 zgNgXUPUjr;bbMhK;tRVOU)ZZxEyPFC(^7QdW_)2cvmcw-;S0;Y`kXv`VTBj_w7}yY zd^#s9wbVagTZMmsDl^(%%Qx-OqTGo z;|D7Ke${=rFGg{13+*k0E(D+N+%EVWWZ-i;_!N0pe2eqxv*7b1;E;CyL^}gDPgz}# zR(`pANZHp2O|;wK(#oXmG;l2Q>>H!wwm$W#^8ebU>gr}|;opAC75?va?(kFV;=%%t z?LRK}t+nO{S%aQ6{D#}{uRp!o6+XQ|m3!Eu$470K&mRa+2OrYDz@L|`$`i$&q$QRm zGk!_*<^N;u%;Te~&c=W4On{jzkdTCIm}C-`Ob`(TAqr(CfU+nm#FbVOVrxPK#ibPy zk`P)G0?KII65j%%l^IBtSE!~nC1AB+tW{LhYF`7`b|wQN0+LZMzwdKrCK*B!waf3* zKknz=d+xbsdCs$)=RD^*65qhd*jTZ}TepvLxL;6Cj}hB6@9LiO9Q$aGz}fG2Eqj{+ z;~pk-5?(!na>7R{Cu>|jZJE&;Uan&1McKY6pv>@miw%FKKKo`@MHZyIgvQlP)^}b~fho5VwALeR_4I+HgW2~wTAF4UiP8*zQ zG0JBi&pKI-YDlw(wWaMewxwM`91w6Q)aTm2x1g*+-{(LX2Uvr<1)3LH^ZTU6xr$D} z1Mr6q^||g!pC)`FrlIqReq)A4b)QaTRva*jXU%ONaSrx9mF7qQ=2m4=o>+R0S?USATnGczU5-)~$O~}T#XUIjYm<5~>v)gK_)9CN}nPD}>=s-2- zJL`5?y{*7i`e0geR~tNZn;HL>Y}N>^H^kS?`;ppObgG@P^w0IAJgIy7(1^MNz_lx{ z5S=0Il(L%F=Q=I;qsqAaE;B6d+x*L`7^g|ZLmsoWJUML{;~--YfW>C!CjPy%%t3(( zdm;CZBmcv`b+eD9t-oU~bh^}QatsDme*soEZcKBSfR*4_8vfCV6_vF)$iq$Wy-B<; zVD7OAzBg#e5`6zf4#nv<&lYG>{$bx$$;n-1$$OA<*v+ntM{a<0?u(Iq9t35??9I}yN_=-Bq~F74 zIKxMRpRDhU%pr~@{|Domi@(e3xTn#iA^u~CF}Rnj;hMuC)dMV%eX>$tLZ9mb+op46aMwWXLUKB6z*T!FD~k!LE*EZy!Arb)iurZL&5vl z67$xFlhgXB(RUiElDp46~&vx%I*F+1R!g#Kt^)R(?|+c-9S! z({^Tf=QLHGo&PHGJN;uKb^{rYw4E1v@7#5EUS-o9Yk5wy|3fE&ce*W& zv=PV*p-aJ)QuLY)z#*JI&J4s?~B z%$;Xa$K}+q8rqh9G8WPl*f)78& zZ7-c9AH|(S2bAMW_8F9| zxT#I-F85QGB1Twc-`tnd@CVv%s7&6+o&=c>7ySVK)knn?{eW^S=87%ijM!OzTSUBT zhfzb{J2P%&P#0VP9_DfVfXl)4q9v^6UhWbno3ket_I8#p@vVI_k97C9wV}RE>lq2r+x=Q_dt8z%5 zc;2NPJul;#9?Sexo3ziCq{asSF;kvN^SlJzFWw1GyCc-;_7wJq8X5ao<;#M9xf7JH zv>)En;Is)FPW*>U+cqKlD~;?Oy4s#I4S6Hw8EChEA6J~~J@$mOP)^Ha<&ZsZiNjTm zkKi@FF0odpN0)9{yZUlWy_-VD;}J{o~;-Y_9NN%wE$XP%6T6c#Qgo+K-|&O zpH!x4DJ96PJP&1A>F33y%`?X9X`%}R|BvARNBLLwftmXG_kq0*9qa99A6VD+I%%(@ ziEbtBPLs1IfJa9^Ew-FZoR7kpFCBe4I84<@oP=N>KjFP2KX8%!$3nip&v&V#HyFrR z6@As-F;?oUeizAJdin2+F$NwfA>*CKbFlB>_)P@I=@{czgnkt|HjDKAKCvB6nhGV=QCxk!l+! z?K9o5w9Rxivgb!?vBrNdKr{NkC!b_aKKGl(K(P^gGz)p*chG{ zIvz`VXk*#m`B&gcJ@M4DgEOwBydbUY{(JlO@yPa8$`>AP0@hZ_7dn}q1m40=dOFwH zlR0@|Fz&nX+cWn<`|S+wOvOA`_JtZ;hSRJ`QIE`l?9@?mDqvrN2g_MG(4%j^o3%y-@={Umce;aB@7E1%Tk zg-(lUuf(5~IiSc|@$FuOY?c_uTadd88|=>8c20IKMz<(I&W=Vl$QLVEqCbsS(fXIqYNAVWHP6^Go5+9=qG&!I^1)Wll9KG zEFZ*x8*#2;zDax!{rhyjmni2TzVAyNsDGDzclqQI8a9i(Djw+YWGd_xa;D25*A4QG z@zlSSpZRTeu--kCdtIR3$NlBbi*vX)_hHUJ+ZX!N zvVTO{b7B6pD+2i@NASI1@drl(d=e*Ct=i_^w`-Qg>*U5_mY~Dkx+c!y+1%F;I~jxL{9|BV7#JTJ zgJ=C?V7?-dCS&lde+7z zf$ul*~sFyRpKAU849v&6w-xCQA_l2D_vX*wG#QAxK;D&$fzkk^$Ldsl9 znM>i`e~w{;l)ean3HY`;^S)gMkpTchhGHoQ2O~--h=Qfrc8ZYUP^GB zm@Kj)-gOgcEm{7!N|o>!H$29@5xtFmxT{q|@l!UdCv%{+rNZDSCry05Bvw$7{FixV zTA`%PIWtW!fAX1WH%PuhgF}OV1T_vd%fQe_aOlQ}rM|GJ!TxX7~RYyZP^%RQ>}_`hUQxyZTO$r)xVuCr#nYE3h*VUrWZ^ zD`|7IGt&M}n!qj_Stf9>{%DZjmia}%mI;hxY^GsTc?et#()~>AT{6aB0yEYS2MT^C zF%DMB%IaYpnx&mx#^E*6vI66f)x$V6%70zP;U)PmFb-KgjKg!?{O6MY0^^X?!#Hf{ z=D#Q9zrZ+T^)L<(ck|x^q@{Nmhbl?yG7c+A6WNtMs%c`&sIrOa7jnOl`#A37xL?Bk z67D0ok0_fM4qcsuzPr-!9a0WFHf@ADYC|8|eeXSOyT57QCVA0~blczvbQrNkNo?j@ zmn>*Y!|t#iU5{~Qy&&9MfDX6-{yq;{ZYDi~ytgvGYl!6{afi|W^z~%1iAY|t=VoaU z5yEev|Ir0sOMDv@*UG!ZmC|Vzo-Z`p{8>BYVe1sVZV9#m^q@9mi!OI8i4p_HFXxLA z9oh6nkZmhi07d5?W`on7nCXM9!(~Hyc-SJ=0c<2kS={~=kAXe6USDIvc0k{bCR^(xqTg* zO2<2Q7?Z}YHyYkP_$u>i>?Te4)cwYo*tU^04c|E7$3dURk9YgyK&#b$9_qKJ#JT=T zd1-2pSHor~b?)T-*aOTDvENo9Lp;b3Szj;bdywAkVpF3%KfTs&6w5T+;*WKFV+U}g7JFZG&U)yn?7%0yq=-09!r#AH;=t%treV12Pe^QpSM!C=+cbc^Ol>F9ecj5-s8!RCh`=s zep~(7eeV=UnX9d=*^2+Qtk=mr&Q#9&3gam9c~f94Z)Yssmx)bA`P5jo?VkB!D~Z$h zZ~qTx4C-srA1LdI$;vl~H4X9ENJ6%X57Ha7d9~OzMZ-9}MwW#+qLvS? zJ3Ka)GhCAET+~0FOU`RKG+c9b^lzT%K_-{Kk138TfW=8<>?eiy64>uPT~Jf1UI_r0=Lhk3Q~rQB|G#ukO?XZfimISW6tt!9sV=f4)vaUhwHtoK{NERaebj}lnqf()6WhCq`JbdM046i=@i4t?clM9- zulXGzUo@9q7GrK=%%!Z#6YY6t!jyR&`Km{z)NO{Bc`~QTT9Geps><o%6b&OTmV8?;MetkOU72?MrJJYlzZ?JAP1o>=Cs@`yu z&HB`9Hs^tv%D3^icIQFnqEG)m*ZC=W&ojFwJO6eaF%DnX^|sOJ#I~;z9S+*}yik?w z+KQioW}aEKU_slk4~yD1z3I=F&$?Jo`O?W}qF(dcx~v(!9vw|${UBFLT3=X|yqPks zTKLRW3rgB9`>?pp^q#-W7HrNvm3dLH%n$u#rjxg)GUrN}e+{&g`9)7)x0Qu^=lm#fI%Q@PVA7yjZi9rl-D_**GE8+;;; z>E7Tt>_4W*q1zh@?+wS>^VnywHwf1P@OhBDOPd1lvk>1&e6oXZ+RyvoV4eUxHV_vo zRnh|eE?^GR-lcDLA88@&w=j1V86kDWS_}^Ny~OZ|Qj1%>{hVK}>+3vqq>uBP5z6Nw zzUG(481^|vyNc-ZN#g+5DPw;Zaii9jToK`q@x$6xFvd@i&+O-1nyy7Y213XdAvX?VE2f+^$ZVlsH!poe=>ETZIIQo+w;dDPc)oD(TukE*ZA=NP5r2$;zP2`0U$dikio89&bPjscZ1ktA(W9%QI0|INzr56 zw3D>w)zgC=?R=NC_juoaz;8eDf{P+s6G=5!)b71s$GE?U zyf$^4K9?<3!m$D$$m^eQ*Oq>79 za-K7IZry>ch`ry3UQ|xjdwlz=!xpwYXj~|;`H*&r{1qEXJNohr^l#C(%u9aOmQ@aq z$37)?q?xSQzfPTzudb_|ft+}K!HBkV@xN`w=l0gG6UJp6#N#Adv1h52Y1c(8lHU2T=vL7dzb zu-R|G=M(s_Zs=?2r#O#0*7YdmAEWJ?Ds;c#GjOSZGW7BNV3Y%VaCCPz)2Z8D+w@pU zSlqs3&JL&OgL0-*6t-rgLwwsL_E~p$rRzq!)3CQ9L#GyO5@ zbzJ&5%~9TA2Z?tLw2<_^qz85EWAMgi{6j>iAim)9 zCkI&^Z5uU5GP=V@ye}<`5kqKvD)X396qxtWfS>96?l92;>#m88kqySiOH6&F~8SwCDxnI_yz6Uc69Wj+4vb7qsNK=>Tcpj zUyJ|o?ttBIGje?o<;nlz2cJ)QBj{_ZF@iV?5&9hQT;`TDnNx~yv6MHD_3#78ztH!ktmjtGu3S@phv;ydA_IB*ubG{-=GS*% z|86)|C_1ae{Y-Feps&R%rsT-n#k4HB?sfDni8C{;xTvj|^~hq@IXt>OoHb>EPhy6n zhJOUdg_dlzTWnNkz;Hr_qdFvy27Sm`A(P0zf&6k7?*i667c5=S<{6e|^j(w&v}D{FO)Bx9hlbbEabhungkPGQO`C zIuKYKR7-Te`(C=k1nk}8l<#G}F}C{LekS-Mw489>n%VC!tiMC&WgCY$TA?SM7W)tN z^RrPy9VvoSDMS4@bxWornzp7BqeRweXHjRc4B4X@+MXNuCUoBleoaT8{lsAQ;xCjp z4L+Cz-#Ez_wZEX{xWU~sV6!&UF_wR=6;pDuuoKBTjo1W3$IQdpk`;SN*D(`Z=sHFj zl$C{Vmz?Y6=ShMSiLSwcbnzXrju3w@{Ew&yU%8`O&{=dCw+#ughsL{x^z;v7UV<(p zazS{Q@MiWgc>jdXBz-U!;P-}(lL21%W2Gjz?qdGa(lNfVuC}DG8j1TLu}E@5@-5s$ z?D3`yV#x`=leVY04wF~v$!Go`<8X*~>0{Tvn32hoXvZ3CEfV|A9)d4N1OG^@68X>X z?-o(_Hh40vkFDVte(=Hrs?np5XDWQ}n8SbzHzMo6iO-iV)p6qUCm$XvGVt@9jYA#q zqt8$NeyD@Br_c8-7~+t%SkIrQIGMBPvP$-9?9=5x{MA4mx#NdAG8tF-uH!0whlUqd1GAaT7Z1U! z4if`;6?8NU9{7otsPnDWto5Pu92K4=ZNHW8Yxur{KDK~AC48sf`uEoZw09g^qLe+8 z{Z}5LHU0xuYxi9|)G;3U@D+WU%NR}I0+w%`+&5H*p~Ud?Ku1~7QK`s#_FHA?baZDM zc2u7mzcS+Gbo5JXIG@S*@{KwCS}Sn$c!ASDg6lJWzBpI%T<8CveN|`t|9^pRz5PE^ z^33)BPdt({{(m#@t+)R#kv!}RG$AvJ=(9eCz(8c5^r12Ey|*&5B~PKhjLL4x*b?~O zTNz1`=N5k%o4P5ZF7Um#GE9=E#9zklZpv5{_}*I?Cle{-Hh&rJZpv69W$?YXGX5@Q z-03gF1Wt8@X_1t{_uk5QPs;ci{tSoObWDM)|#S*IgZcKoUC0q_5Arm9Tw^kea;CV={h$!z4QunvdhuUW}>6bKv$d2 zykHu(T%&Hwb-ZV+DWq)i$x+z0%lfIY_@^|O@n44@a{jLRJbgNQ(9P;})%M5tdOrQt z7gdo>Uznr#mX5EYT`ke`TumOk;gtS-4bM9br=~L|=CSy{NEr<~C#v#_3HtL5d>^V# z6N}y@s zrXAt5!#p%l*A>#1KBuan3-gZxY1^qkscd4^IS=l2#&7w;s^~+YUhyUQ@;U=>!pCE2 z7;)n@#|id?1@~rb0h4-cB4d>LiG;M>pnwsOLxGS!ed1>Z0FmSr>Ki z{4y8cLpy~Zi%rWsK#kqhH@;eYev4BBdJ=1tCNuU}E!lB9`NfwfpFL5F@TqQEWApu) z_5aVs*0|N~eEpfp&OhNV(md4W+Xj4V#P8%LwKWS_9*qujX#D2Y8J7dtGBlN zr`zzDw1K^2zeWa~(GL7`_tg7g@&+)Gb`-G|6YT2&Y~bzz__uYT@ePw?y{6M|GU&Th zbW>S___Ukjl_Sd%?>}F>{9;FNtRjH3K33&@9ij&Y>)A^^rgD5nN|m#HGBM@A!~fX< zzOXJ5#MAw)?XoALTU@ExMcxIy<6pJpS=4);6_V$U-t#PxJU{I{&+U@u-cDSfNxyr; z^_$5rzU4aZF87yRd2N|K&+m>lu9xyxe0#oGl27*9L1$9l3FbG!@^V9He2U~-_3imC zmV6IDTMND&pNj%CwH4eM4V@jC+C6Q_`lsN*nfbmSjeRl*{9g3m)u!J{o9_7UYSSaq zrl0ks;3oWXL9Dhzl9=Yh=jgAiwMZ&z}#-kb2}q z)DNQSdct$bCwc0{ZxLA{K8#&u$!=YiL{$$JSgrW)f|ZgsvETO$xza_}>T;zAICYh^ zJ;CYw$lQN-nT*?xa~SlrKOe|NsgoIUn-sLd6YI|~ z|GEjgFP8rYw>7cGvi|;y>zn%8xnENM6y*hNio}ZBI}Cs9;JcjDgRNip+qjb$vyy(# z?}%I@4*HWx?jB<{cNY9tYLS5#uA&0?7@|O{G4)R&xwD3 zXM*bOh^dl0d!I2VVy-tI_?(_{O|nZuv*P1dtyJ+f;*MJKsrFK5YJ@|%T+FVr{> z%&_p=_P&^BRh}_^?XkAE+g~xi{rAU`oCh9GcFG(_d}}0zmV0AAM=0E#;DZ$(chUFv zKd5|7vW{w2V_V-B`=sA)yYosdZ+evfxj>$=v5^0$CeQF?dEPKf%lrDq$yRp;asP93 zPqo}xHu2c6rdmA_T7%bh=|~UzgFH*MhQk+68|k@V`pA}S+iCI1YPpg8U#mtZwsC30 z2yEQ=a?OqgHZI1&TVZyHjr-OQ$G72|74rao#Geso@WIi!&Nn&p{VjWB-Hd=ui8*Z^ z{-?e*tmP~umP(p2eX*>ooO;_>BW*hzY};Jgb{_5d`%J56xYpqL3D0x6XJ0YW`-52{ zJO73 zTf>0m15%#1*cy&3i%zMx(x#zNDMzp$2y6sKpI5LRd@1nPqP=D8%kkWSf1+lqZyndi zVLSTZgw_j`gIh-pA|>C7e$e$#`7LH!GKk%RM~K zDfr#;k*qJLxbA~K51h2IR^Yes{*5@@)B0hf3GNH|TKU?q?3eQyHqPE;DSGl-qECxV zJyETE`Wy<~+D-eqo4bmA_Xsx0uJUtr(umaF=B?pztlb(Dv)d}PhO~Za@v%8KI!A!h zvUe$hc%CN%v~V2S6h0|-gU!s@ORVXs_=d;0feXIjCCA=j-TDHR7t7r9z?^=LgI5BR zKMPEfUHR-UbYFQ{&h89tY}qXG0mD0Y5`!}Uw_Qi^Kl0<3ZnGEv!ySgf-hAxE$L_tm zP1bM3E?%7NUmJ*H?#({Mqkn?V#4qB*@WkqT$`$ypq-{kfm%n2vDSBCjMW&^fWW6kV z$<`TCy!*+YXiTYggU`pZ)wWyrM*E|6m@|G=W&&kg8@&Al@D~3%!{^s;%%yJ$=_YcyU(AZ(oyP==M zr2!h_e(EveZGFV~#o+tBe5>a2?!Xqv-E&a+rpq1vBQ#tL4SS&BuZ?lpIvtiM-|-z< z!#cigMs@=Cy(ZvzBESdrH3)E^347Gb_ukc36^IYNDi9z3BK+8C-_d4pMAiZl7%$I3 zn{WJ{tZIME(^%JG=EZ91g)hq*z$XU(8mZe5TkQd#+%Kv|p#@WbN3=eO&35Qj&IsC^ zCH=@ubWR(3b;rB=xmaiWoPIg(9d_ML_U~@9T6*Wp!FEpvN4stFV>X|LjqQP?Ed6|> z{qvNsZ){i%zD2W5(Ef2?cMQ50U0v?7#;?;qa4{17Osda~TCrudofs6fMdI$r zy1Z30HHR}UZpzl<#u&q^ZBuP8E1M-%O;s<;T2?u0T{X829w_)e0(_Tt3BFpXS8UxH zmx0U3H6*^YzT%A`vz~fm=&T{}yS9|`yq;&1UDih9yhh3@c-Jz^_HO#DbT0L7`Xcqg z`ODWC2A6oD)t&G%-F_aDzG|Gxt6pwA3mu;`w8a>E%+au(CLN!ja=jIf#7Cmo|YM5s?8d(>defW3e zHIGbfxln6#Q;&&q-RKW$Cu_QetQmll;p5LsMK5bKkGH0ZZW2KpGjB6`89u?D>y&TL z3fBLw#BUthVBND3x^HM5p*5n@H6EF3Z#+E5-WUVDjG?}{{aBx;ACqWPI(-^HFsx=j zbkW2&d2ank&Wcenl2_V`zeS1rMO(vq4|oM1oTix~-0vuTkGqHVW|gak;Qv|tKZMJK zZm7}jE6`zEfd{(a(PqYM9yqm&J2Gf5V>e3t@8mu{aCct^FTD~!b8~5ZX3VZFmX9rq zazD;kRQPe`qLm*HSv2&Xf?2-`OT5W)PeK3T+^^z2rf=d+xxCNj{Z8KR;=PdfyLo?< z_xE{U$@}BHtE_@lm30f(kGX!rg>O~X9bC6_E#@lZx|{1xuDiHy=DLULr(8cv1vigc zpm)8U(2)3|q(d9iG;{NwaC5ZfY397+7RFKD=ke~Y@Z(Vg{xiap zWL=!xOL`7>Y_LAjy2up1dRqZ?hg ziMdg4x^U6!szVok9@|8aKeltWP7vps19DLI8HM7L_)+wz7YB5hu6ptBX=iYJd((@{ z%-N|j77CuK!DF>|H!Qb+*9VwyrE@lS7&1_JW-Gio9A5zseA)XBer-{El1=sI6p3pz zNsB0ngjVfKz!~sS{6e%cAvf~uBxAnJsRd_lNM3EfPES>w871Xkbl*jjZ?y$x4>LF zvM^@Xxq`E~;9{XEGj*kD2)qKE%o3iF$u%T(MvU4jvWKpt6s982UroirdwDYqHX^&&QL~oI3r^# zvRUBOGE8g4Hu}73+&R(xR2cTrvWe}vhMe|;s-Y5n(p{%~Ytg4-`l+k)?UvM*Or_87 zOz0o(+pudecS}Q`F}=Dj8M?TdeG3hugSv;WOFoR;{|vl!YW5v) zGWd1VR?^A_*d8i3MmLX8s<}jquJ%5M|Ed<%T*^OQz8|35&rp3F$+0p43)P~uC(di~&L_h#hL z^5qYkNpFQ#Tc1_F8Ak2{6lXqjkC3~mx^+~Ta|W?#TGOz%Y@Xx1lKlBxlgV>4R}pz6 zri}@h?w=g)JX)t3j#P7|PDMZGrYxJw(2F@Y{3Op_8GzltJI}EIcb)WHV#NzM5(; z%4{XhJU|0ZC&bg8i+v|m`jA#)iTW9XoBxL(Upa6y@C0J z@FEXB)UDJVoJ$C=n+mTRZZEC>6dq@MbJ(n@@VnvmU0V+DuJIn^d9T1%yW@Gn?@DM} zF0j0HK-oZ<7gn0o80J@J@IT3y4g4qB)KTGi9`>JrySlG}XLFv@5LH?q#(ZYjIi>aI zPu;a8_v2xUE)5%e)9kRgo92W?-cQyKfMvX|Dg@fv3l z@}$xj7;ErQ##&&X%Q$E1yw#W{V_nWzX94>^;rDVV0E?_D;yf86U2iO>+MPN zx2LOY53w1%sg>mH6!jm!yVPeU(W z$aNr7<^~z5`v)*?@DHIS_Whhm%kT~v!+(i?z&CXMQPDmB5SrF$8JmP|8<6^-WxMu& z6EdKijOel}Gqhh$uxp=8$mBUwTgP+L#Ed9g;9J>5J8{1BZ+*%p3Qf<@*c;0wHiK7~ zkLdJ?ycN0>8WsD878W+k4Q+ZN;2CGor>%b28yT~PzmYlXeC!XS;T6NqnFQVHJX6=f zgl;y4(oL|B(2wv)J9Lww34B$WCBQ3%*B-hK`50nHlX)6_#Bj>+iRnMKV|D&TjTs z)<0Kwv74oTN4uH3NOjoFp8j3ueBl`99Cq}*7rU98zDwMVuC}uz=An~hei`WdwIQ~% zt+ZEcXFHkasj&WOP0abE|C5Zdx-DI7Xa8XC)?_|6)q5R#ebLv&ezqUn%bZ|JGen;Y zJ!Phf?M!6sk?Lv4>dTPT?5BnHg0?lWwTT=?#>NHH+}NXo>85-&CKQ*&rWKS+B9HoT z_2s$@nKXKOY5lM_htHb&=J~TOLne)$zH7@6-Y4@ODwqE7?vPm<-_4v=!DXOcDYNk1 z%teI(yH{^=Ck%U9-*a|tIe%(t{RLO-+7gs2HOP@iLgdPCcz==i*FL^r(O%@t2fTmC z`#(OuaM3Z|zvBIC-V2c{g~*vgUE4te{E!>+9ACc!7T14|g z+OZj*RB1~!^Uy>7x~o4s2!3l()$4(GIhPu5NzI2}$vUj?tI$5BhxTcmzfWt>!KFW$ z^k=me-_e(FMPG)~7v$}~qc2V9V67LMIkU!)&f3{llr7<*|yI$||#!=}3TTwoBP zBC8KE7NSFh>Lmi>U!a@40>2mCkh$oVUjXx+ytnXQb62o!VguMB`l|2(%f~|(4TbJTLw6TIcR_t|4)1wF zhXH-@Zr*>&dyuXlmt>ced703OqMIAaQT}dV=sLiKX1a`AD8@dxOx9{GoDFI zX(j*kAYbCY62?LJ&>`R=^CH$_T_Qh%vPf|Ld1ON$=0YNS1m~Y;ej;`vA-WV=s@lT@%c{h z_pT6peuVc&c@NHiUgZ5H-h*=LL*DSUx!`jyG>{8^ z=Yrq4&_pgYkc&*pMONh^r{JMFz8`w-8s}#4z1kSryoT`+n+rU;x*qs_2JaMJxsh>N z$u-{;mPULH9p?k_%-``|c)fFearVm<#S31}P0UP{XBpF=P7tJtLySWq`t3^i&dt#) zu3U+JTZn%9W1fGGep`rsyE5Ju_*OQt5dBvFhJGt_Qi5)~noH+N0sVHy5%4WUznzSJ zy9#)FU&wP#p`J?_+uIrIJFc+R-wt0}3}3qgyWgE}44-xT)Jdf4>wCI>D|G)l`*?)z zZv_rs;2<<#1so2+`>U{V$lUN|blk#dLt0^o%(xys>)UJ_4@Y)y+YlUY1*gqIZ-U$4 zdMIvp;Qr6r^SZ7hYNbe#;npU-3VuV>=JUaj)PBV^9E& zuL#WHR)FILfjL|SI9>pbuL#WHR$yZ+2+ZLsdcpApU2t6Ha0>!Bo=ZJb7}uXL&bLoJ zdk*&#=5V(&hZET*`jX&yYfm^1tqYD@gw6x=xPlNIzX2S70~?6U+qQ5Qy?izI0k-ti z!x6SdvHOXwDC=kKW43|QC#wBtuj$Y8u6DYgvgMfALoWW1KNQ~-?5(*!9ELxOKR&sa zH21G;Ia%a6_L1M8VRkUzlaeo%J-zJ5ByE%NTJaSJYW{6J#c1Oeau{%;uZ*~K*af*H_ zbNOk;_~tb9(1Wy9Y`acuykfKXu61{{KTgN~<-!Kp)z&HfbkN5S|7fgvh3ndI*0pP} z$7<>VIe$yw=YLP@c<*}FjKqOB9z~pwFV1(21%{P|6EDdeMsyqY*AJ{TeEE_&BCJMF zCyqS(^Ycn?#GX#yvdC|;ypvpPUO$_;$%1s!1Q)KHte+3XzwVd1o(EGpLoHrU*`?Rw z!wTLjoB!Mhm0mV`{hXfvJFDHa`A*s_<6dMq-qB`k7z2yw!^glvPY<+NvG1^cs*xDb zWwSSP&*$#sF7>Xt#$RtyH|;gulQuBza8g8M64oUPOvR`B)NKav)(HNFfyaHo z=f3Py>_>b&&9Og%_^a}to&OT&+}mWbeU(2<<+b74D}KHBfUlJ^8qBQ24o-L0)kTyw z6h(fNY+93&GHs|D`=zmu>+k)PFLuDI$wdeHr0gD|#vV6@xlYjM?esZ^K5sJkI{J*i z&%jNFlP|5K&w6^G&-3Pnj?-T{#>s|0DYlot{u4=2ROGji*{A=e^IHaPGWS(8t-S&?7W%a0`xNPI%4r-{}o(F3*dmXN9uIr zgDve^MS0T#?fP`1&8hZ`ua>=c7Gj`@-@5LreS@)PAmz%KM*R!rWPuOFt#F05S0Z9yJJ?wBWG~-8oMNnzbAozm->54%d$O1BZ{1|9 zS=r0?<2M^??(60Iz9M7IUA=t2oA0;sotVfE9U4W9ls@L_5@gY8^b*mV53}A?)@OOW zrO47>aixbPE-GgZp3AeO*?AY;q>yJxo6mbijJ9yPSQetEu-eG$V6m&O3TXpPxck)6p%`(JfR{W@_0`bze}wN+Q-}C=Dn# zv2tEB*39Nw%XKeTdxh-((1V z=|5sGxWaD?-B__VTd_ymL>v&)Db89uCHuYY?1lHA0p~^+F|A2(c(_}yPIRPQuNJrR zJmV-aw_{e-rkyWmKvmY3>uu>h_R*KmP4 zX^V-&L(DhjY9HZ;%S}PJOxAR`4CKA+I(YjC?6}k?I+D~ki}bH}p7)^Ewk|{Y9v_Gw z_CSRnX`;)*_h6fzp-u5Bq58yd^h+a;8#!k-`uBL0!K;Z|I;OHOcrtO-Ui14Kzi{6E*DtK$Engt>y_@7a z`zAccwEvxcjX%vmnlFVobi{r&^)Xc^aBh~IPx(L8lRlescIboHrdBZyGnV9XJ^*%! zl=2*JT6s<@af7Vq!1sxD!`71KBk@*w&rI<=jm+6rGYxe>i0NTi9yw%hwhbl4pp={bRX1)seaw| z+hRJju)j;)dY+@6GG!mLz|2OwiW1c}@i`jYmpG@;M$V21J(hg%`SC<`k@$W|{!(P; z0c6WHM&>!!4X!(oDL51BdW80^M>j7bO)(!2#gW0(=N>NW=gL36;E9Z{%n<~3KjRcx|SKnJ{e$-T{_Mb+nGOzr(gFwyAAoo zmsulY5!0F95Wv|ZUGsyJEyN}&m-3@K^XCTIbFgcElRBDD+@ias{Qhdpe%c>|6>;Wj z%1-nfw}@|>I1eP)w_sb}3X~NOyn=L9AbRbJGil~E@`ch&TL7lA*T>DdOlcO*WkSz2 zTLW>Epb6mY3ulaH?2mKE8K~G_9+G&i#0>T3j+HYFlUzx}aN@v~ZK7+uO&v|_!`sbz zX%K(T3E)r5AkLEN$2x%2HC8R&g^5-Csc;Q+it#_sX1uu_R$b+|n*8+b^Y z_tOqH?KG=z9uoR~YBYPPpkL4X_#acJe9Q8-ZGs!J@2J&`i&D9g{BuWrIvE=b=`A|sZJH*cZ2VY#@JZY zISvD|XeIv!`y;WoUZzjT(u`hMb?{{V5#QJwk*hn1MRoxEDgvHTc4)f5 z^CY~y(x5I<2hZdOiIjPQGDC4&5nDz26RdXtc|vhJlwU-t&mWSuPoi%bndr8cu#avr z{mcN@=}&8#tnm;pFGFpcgf5&9EW%@nS8U;(xL_@`IUL(~w!9NhEMI6O1s|+*wJqXV z_K$OJg2Y>|O84j6!kLG17UDW^(M(zdc{daHc{*vOq*bLMf9R*qBN^u>3pI!EN#jv$ z)Qn>#IIHvi=#jd7tz;aDt20_L@6dG=Mot$HcCP7P4oM<&{0w{psPPKQoi#*mVI@|UM{3{gyh-2pyJAns$FD@=_`)sno zd4%}b9l8-QOLY9cexzgX>GlL4a1`9@OI_XRE9-wt-0wbNz4Orz&U5tPe<|lUd1aix z1Q!}9KeTNf`ql728=^-zwvtEJkE$8u%B~=UuNDYx?}bSaTj% zBG+Nw-{;!NCGGhwX>++&a*3>bjQg!zO`MzD-9CJ=bE!yd6MQd%c6PIuUUa&6^e$Pu zZ-R#2rLVHzO}?)qM%@bRBytY&qZ#paP2hzM`feuAZuBLQl^ft)P2fj9=T5vDOdpC2 zh1R6J^{3mn?GESyLVx_{bDwJ8rpMkz*AclOaTXJiDecsE02|02{!1^ur_Dy3zvumB z`UGxbcgm+Nn@P*39Gk?OqfKt)q~4D}dD8yv;IFh_Z$CzwM?to}*&NqkaGmxX@W5yw1M>~4b3s=5fC z@f)5$>8rLX*8XHJ#Q5m4W+M5@Bz{O=_Vs2EW03EDJ~0SB5$9Uer5riOLF{j#u@~fQ z@?dNw@bayW^Ayml^!efw!yO(!eLQc89`3k`GSLTS_f*Cd@TaSt(L@{qvFS7;$Bxri z@pFn_h7UcsG{~YHJL#Xy+x|@1=xSrc&u8B*kiIW}mV4aj+<*o#Z9pbjb53{GZY7c|53CdK&iOeNVGPA}c5|2va*j^)y6u^`c* zZpd$wG8Hir^m`opLF7F#a8D21sVB5L?=)UH+TzQwI8`UPcHVjHHYNt{`rHD?E}q%4iJs z7X|NE3-4U`epiD&cpfBN?+W8~Ro+{6F#zfT#=*L8SA*-i)h;^6ZB0VuR`A;Y-1t5GPxkN5F8y26P5(-J>0j~x zr~b|TC;N9vm;PPTP5&(g=Trp??typ4ySxTJ{ps^7c&D86CHlDV&Y!_MPr?s{ckYIF zChEL%JiIg3)y#h~S1i+cXT0lgyuVN0eY3{bi48=`eqYLlcUA^?XJvqQx@Vc{_5}K8 zN>b<;XVRZ1^Htra<=??SUkLQ?`|_8E`7cy9)brkzf372~C;mAd`K0sDDCO(gzkjEE zSo%-)Z*G_V-Q7+9Zt10e#owWSJ>zHocgD}jf&P78{LJLPQ2d<8dsqA%C1v-3pH+eW znW7YW<9`l6BLe;VzWC|Y&cx3?l+zVIPm-r6{PYC*%RiyNgvMz!<#n%{Z3yUQ@6d+t zFW=)`FYuq}e)$32>`%Pw{kyhPH`@{DpV*;9H!BkU(l4=k5bH484TIPljQ`I-)gZcF zBYFLLUYu(y&tk9A?UgY-(`RpIPJM}=(J}m#PN1i@1?)VgAKINsA-tBj4trB*!~O+J z;kAjbwbU!H5c}y;cl?iPb8e`a7Ok zt4B^8Ko0bD7SkaW7I%X3dgAjZfUnF0B+jGoV$=0@XJQC^WnC?qe(hXvFYrxpji)^V zUs*G|5%{JVlRDtL1o-9z;A;ZD$vS*52EKz_iTo#Ltoq@b;JSeKB=RQCy|@m09PmwW z4UxWffp1p7Bz?_(w;`bezGi_lFy7Pw;{+YX@nLGrZ9RJmeK})?oih>J_g%lR(Z@JfgtS9%6Mmvn4sF`Uy5SDPAg}nxcI8z; zSrZ(y_xUGSFY9hS$|h;&M#iQ$8py)tm!d4{+$lV-b!M;K=qx=jna~vgTl5 z?kw$JNvuT1Cq`&i&Zo2zcmF>6E3ixfS1XC1r`z+Gw`cJ!`^c|0B{Q-j+YDNl)uf?AC+9H0&ah&raI^bxFI6v=0Mm*w~0&6X$xCH1Qb= z?Zb+1+8_IKFpc@V-f7cmG|?3;G>V;A_{2rT$sMRvoakV4 zNf(_@U@Wke|9*-s+(cRx?^fDn#XcJj?6Zp(w6)%w)dnw(`2@Q-XXxa~Khl1wTjIZp z|6p9Z#Mnx3?QJ*Ktf4JE(YMU+M}tE*3(tT@P41P}{b#KuVxv&l6B0`E-^o%qo zGM&zUI0^fLwk)Y`B>fRykk2~xZ#c_K@bpsB66fRuXmyY)hj*ciL50Lo0Uu?pTJjlK z=lp;+|28AnnZzEDh1gIkMGh)_EZL(f??t@Jdc!8*g}tF8b`t&?4I7{{!{T%1NLh`P zvxPGyA}@i~A0TgLm~%~mdRzW;x+Xg}5953w&QUDFUJ=E*)C0s6dXu%w)j3g|`ATd; z@L%Er{W=*xv(f$axZY>yD{GK^lIKsHjl9MDM(q{-g6BiX*%0A%a^@}ZGOA6iHOiWW ztaEru^Z(alV676kg#$P1)!}v7*kG(y_JhDrtn9(FQn#0VB0R4We{FLnvQt01sb`+h z-FlIU@e0~ygjb90Uq5dQJ{`|EzLdcj;D;ml^A<1@{w!yOC9?k94$qRg#?hUuC$zVZ zLoPieb|lj^=DOG%nb(~iFS$25^2v@)RB@&{2skcgIt>f zZ4#NWls3%^$c+88$w!;Ah@GOhslZ$p|8|(fE{$n7M0lsetBZi8hnPMB<8#Tkx*)u6 z4RBr!oLA%1EAiJ?Yvxt57il5iwpfTi$-XlD2K6$EbbdXR^^Pg5cTC1lHW%BG9Y5LE z^z{o_Tk5*5B)XS7U>DnouaxDb1aRBrYWW7cJ7@Cg>vyVhb|n2u=e-3wFhK)q=gx^O z#_|dK>AUa+@dbuQ)QGR)0el^0O-cM-{!TfPW`nQ%MV?7(xAP_)9JNwp3*zzM5L{%L4TCeAbR^;PrX*DWADz8M=t9LrXf(+f0mQ6|`tB%6dtj znRhwN$Vd}>&C2_CDLbLJvWbCwR@tM!d)cb@vVD}z8OgMp_Ui4nfA_LIj7^U)fp-MU z7Fd{oiOj{1M<}0rupXawujrX3;5`@|d2)E9BcA*A191+~5w6Wl(dYVd&f2ASsG8!Z zY)%<7foE}Gtdyzfta187FKybxns8TGN}Bl9ZNjHY{OYPh#m(P*39Pw+F6D!AwE~3w6^C0+Y;)zM{kFjc}&;uA?@g( z#gOsn4R^){>+Ocd!ILxKBlx_Bx+bwMB0hxK_z=!x4K0`Pe*LCgr}N(8wu|uF5&YSW z9?=`#($8&!Lh3dJ`qUKQFG`uJ#ZO7js{8$E_QA*)pqEPI&nGvtfaaB&LdgC5BOOPa^+FycctUK^s^`45=zH{Yx-C~ z$M_(R5kK83>YPxZ^mA!Lai?nO(zZ|>m;b~kP0A}Z;;RQ=3B^&X?0xEn<`wXC1=?pj z-M;-0aFce(Sob#0B1fL=tzQ>+(olD}>yb0oyJ=6;vG#4h2$ZWh=e1VaLtCmC?{NI7 z*wg;D$7pV555cO7*l(~Ozj*clHQ)m@PR4AyIWXS9PR6^6SmY6m{i$IQb&tM3#VN71 zE}@L$#_(q8qX|FK-ufx{JU^tL-Q%;I=@9PLvnQ}p6$k;S^Ue1VV@&;!vlDlAE2AG5Pl`)BDjy1;o~FxD>znFJ&du9 zF&-W013p6AKhx(L)6gp}L%+BbJ!2~R#uW6W$?S>DU5Fl{?=5OCRXGB~Z)*#B=+PF& z^SidCf0wrWa)+_zf4N@cYJWxccNmz(OagY|&jTE}u|C}<`tcM2&bhou<>TYo^DRU!b9uMel!r%9!&ppn6cL8_N zwM%#o>d);)O|Nq+?Nt0L^Rn-IPY?Xp^&TuInBLQSF#TV>n{FLEQ05yFBf$)kFq@^u_CO@%c_}Mm1w5s)$Ai$Q?49a6mtQt4eVRm zNbHdRmObVr?5{4suWkl>BN*poGjUGDPuDK{sRMh;p%p!LLb_6o`kH=Gias7*bcydY z9+6XK9f+Jdt7ANPZ{ghvY&ymx&OXZ)XPcGIWeC7uyN5krHnm7`PY;P9F5|HPUMlA% z_cSIGdm0n=daVHuZe={`xyq}ERecY!Q>kML_gwDJJW*QzEYHvI{1*4uxJ#TDiG3mQ zT_lFfj{~u+C06w<#HKz!5X)L(TVKHY!+}`V61(~l-meYBvX)ra*LTFS?i#!L7Gk>m zm@63Dx-)jQKbCc8Z0ps)uAWQAa6kJ+UY9YXZxXAfV+@Dd?wikLu^k|W@_|%|Ar~6k zT0{3x#IFvHtMIeqSF&F%z|&-2ug@*@il=R`q8;{?#^Lxz3kjU9H%Tf_cSOYVG6CYmHxbR$hF3JMvD4|Fm+=(5m#Z zt=g)y^1l<3znHwQe@p&9hUBjzZ+3joWwWodqb-ux+Q*;Q#q)I9)U}`3=sNQ6AdBcBitcl?Y`n;`AQ`^LDV!eub26Lu7F>WV~)E1jK*NDAF zYiHaL>8NJzdHIs!drHia)%jZ5zWwx5{Hwlze*~Y|gV6E3JgW0~Omye-r7@k)2ST2= z4(v={7Tfv!O~`X;Tu6Gz^M!*t)1MA`{yOCO2MHm0k~*JnO6hzq8r=DO=Q*9vAEb6Z zPfPE7&dTh3&KcVITr<4$dAhap`G(P*&+mpjAHJY7{rWMT&##W{e7@tN&gUf|&!r*H zJI8hA`EgF?^X(zeOGBPFUfh{y#3h~2M<;bYAF_8ozdfb%dDYa;=RucsKHnVjTs^Hb z{bb1VqUoLK>u0c^#PQKLpZQ+MyII*9tXkRy(g)^* zd6HlFhS(m3Mr;5Tq1)obpo-0G19}0OM z@)H#o7E;ea?4wd<8FgLKXso%CtB~twT!w>>{`vNO#P?%Nk19-_=f8_EeR(*+Psy)@Mt#Nbk;b zRNj`M>~R`}ZJ&Kk>&Rmv&5KWaDQBqdgui(QvxkayucC}nE!=A%trcJQl4Wsv`!;ZP zn|BEN(dd)@o%j95e!y`y@VOc|T?M?Z1a7mKYhA${Cd?6ei;8m&Qe!3|^EdR<@>T6?R~X86Ywwr5B3)=Ue8`# zCw!Z(MhtaN|o5~tqGrd~-N5<(G?fXRQ z%bx$szIxVpW1BL}`KqZ9TH_q~vMDOBT+_y~j&RhAUXv!VOSzXC2bE;~&i>U*p4(@# z2a>smfy>Ax>-_nw7Y`{_bx&6$w!OyM?`T8c=F!WO>b6!Sv~8tt7d~L~{bRMwx3%Ji zw)NoCMYB|$8M)Fjll87J0%(i#C-w6`wi<8H|Ji}hD|6($(0sCTyj zn;K9z=Dz5fU+t^J-sldG-$~+p^BnAn6-jN@WwGFK-@4B(Q$8#7CiuMn2sWz_ zeAeLcz0r=$n_SRv>j>iKL%R~E{usD^AaaT`e2dL0ap@wphzQn?u3pWVMAf{PfWr&4 z$cTBGsbt31%H%`vhIQH??~%#M_iJ$ODe&wdcq6nd{3G9R^;dSD-RvnvFF#rhyxi~z zc^<(tbY9Ym?PdpkiiI~!qmDQ&qWLT6<9#l{Me(!tVe|a0F{$lq#{LxhdF2c_H+ZZ# zS7kRit!aaaT@&kC(q^m?T5?ZjZzelc3b)TN`dB5QSikpZmJ*+-gd zYO4JMu=t10FAVUDDPJjrpJ$}c(LU;2Y}=wnmAgl3p_UJ_opQ z1NDsI3ghzgjWYN|Bz$6B?Lq1bgC9&XMtFU(HMKR=ml<8*Pcsk`AiAb@Lcb^B85OkS z|B?6R@lnrX8c#jDVkBq9KJqg zq*cHi4+fWV;MY%+pZx*+_15{J@EVUV-?-)c6yw(OYYU9h)6k9<;2Pf9(fB9ooPw7+ zui#vG*#d57Z!I|Q`}gkB?nt9ca3Ve+AGU|~5V-2&KMwafG29@QqwZo*UB1*=eDqBzVM$^Jrn;7dS-%e^A+Bz zk)gl`*XNvxje3W8%lRzkfjsz@ig4=!VANX#uXA`+3altQfPVpVM5F z6edwO?JflTIp9C{qn|p z>1P7{e5R_Her>#Bg?%p_dODJOBSK%O+RVKd`CkpVqPf6F(cu7oS}Ml)=@htU&(Sra z8`$%)yEG@!XJ9Xn_tbsPz6)R5kI*mm!#_VOs_wPt$6J--NMw%{|NN+yljM-P581nq z?^kdP${~@Q2kA&q2Du*@B%?CKl>8tWWFRug;rV^cZkO9y`@&r7C1^zA{19`q=9%?A z`}`Pd8TfV>+2bI!4vs=c*C2Ofys!B0MgMj-kNq0?-%CrMwEw67tf3ADcKK4tEt6b^ zEssd%`EQdoSj(WSu@)G}cYRsj*Z@7;Fn!^Zq0_H@Qu4+F^qW^e{Sxyp}ao zO^_CN(A&)Mn_R1wM@yc3iqzpAI@aa^EzpDt``T?y#-DuQ4~5ng%TIr5!T$)s3~*G7 z++qO>sr63bhVZ_7k{!` z(qlYc@sTq4jbby&uj{BDf-YKYT$@dt%Jio7-*xxjQMG5aF`}7oOO|bXN$}JfGuj}x?&Wzb!HHk4_(7b^&#g!hjmU*h>ywPCpDkIH0<|;TxN&GGvsWF~I z4-Jm7uRX@3+B1w%F@FYQETblkh0Yo@0y_{hcg5m~png zGRB_obIiAX6VGd<@A};V4LW&|I<53uGQqX1uVnijxkl+F`QgPL@&o6R9|m!hoKS&m zuwr4P*|2G(B{@MdLJ~5`L`P&}S3t&>?z$Bjx`uVQ5&kYYzmxI!W&2dtP~%IeUCbK3 z$ofgXmt2oLZ`VqH+Lh#0JiBE3piHltAd4B>*4(-F*d){2zae|nBFjsrC}3<`7@Oqh z4A!{dhlZ_lm;BTZ^N}3I>jn|ap*=Ar|Fkj>bAbD6E91=%f&Cn3bYl)Np%yTh!(Jgd zj_Ag5^MP&Tp_|Sve9i;x*D=?#!36{Vo5Rt|dzC>AZ031Xrm?EOE3Gl7yYN6HurV|u z(mRkCSj!RLXaO6WfQK{SOB}c%S+WV3_0r!K`2A*{Ke=g)g?_fG{gu(yFs}Cj8|Az^ zlh)4p8gS~+SY_*7yDsU;rgZ3pf%c<-Vav*$|>&!3ySEswP^={+-ylzn0&k%{qBRuHq(0e&r-QEvaf-DCWsBAUH~g>$@b2k}dJt~6g0b@^iH z-(5y8?<6C}d!OO*e%o+(E&5~E%4WZ_Bgpy3knzue`=0|h)N%N?97me>ByoL$wE8$wQT2K1B5&Cqm zO@I5Kqa~O%&=>KWPH6Buj7>Px30#N<6C30`PfY9<=2P~TGtm9%Hb3nPEl%`G|4BRJ zqGpl!C3Y6ZAbX2=&JM-~ty(1-E#0Ks*~ixeJjxccm;b8fis*DQzsdGeNsPb7)>-H0 z`J&VIZ;Vs2g6MQ6aBwSQmpoE2!%v%YIR^ER#_3;KKj`yd_^IggRnX^s(C6U#iFU=_ z6ln_Q{`xxhseQo6pL_G%j3>W-9sM}?b?YcN6yLGe5PqsPR4iOIW9+N-3&3J-`DIva zoO&4+D>!cf2EPyN%_MG4FxUtTR!Ej&omK#YmC&el*BhlVteNz$Sa?i5`{_holVYkq z61Bx__z`Uu{2T&))=us&^^Z5IR9QP^%enBg^Bz@u{@M8hbec*+k!zV5P%gwgEu(x-+vyUw^J#^Z&dL(k;X7bZ) z{hBeU3ohC66zjb1ydPF0OZ~9g zIMENQt=KCx4(YX8)nWzZJIlYIvK8Ad4dG}JrH8A_~P zl5ffczdUe)_1nNay+S|oD#?4O|^`<539}tFXNK-4RCpdohmsgHGqFNlTG`>5usMad_WqWXJ~kD8Bu5eXNGYY@(0Bz8yE5 z-k&lLN9jwjfAJBC@S9j@V*;=gTbmzF-RIc9-M{`C(^8)=lO%CUI zc*m4W^LyPk*+-FQ?fF&z)%Ru0vE6_A`7V1`X^uD9bIks9M!bKXkuiv!Nb0GlG!HdK z*!G(|KP*;ojAoq$htCj4BfN=-%J0f8B6eqi%e)ggZE)2AokIq>S&fcqRrY@_(g3Vf0FZ!UfVc3Z%|z0TAU{AgRp@!mnetz_Yo zLyfNQ^UiNCHC9~(d=HM#f3EEYH?@Q#ye-lbQzN{JHEwbF^}+L73bcRuD(Qo*r}Hi0 z9D0&hvQ|StFI;=QAIHvbaagUV4NLmG<*jaNKhJYHk2WX$>C)^zBr2Y*cKYh$h89vXiOe@8iP zs=t|<=%YPG{g`M+eKB>>skv_BZwoXwkGS7O%$XaTY)jo3>+9A>`H|AM!f1cPgE+zH0Ox@##1J@PrkFhcGPY8q|ZO%G!}ds zW-Qo@E$$roaU;V-(PklXz-_-e+s z5tvAa#!A=u*8ogPem_P357WOU`g?@DNy*9i$jP?vW$xAU^}fP@tSlH0${Bflb1{7i zcSb6YjXA%!bD!3X@2o+l5st||ZQBl+-*rztVXb8?UgDc4`Q~xv>H>PmMd<4neD@Tz zwkiST zCO3k%i8hY*B5gKp7;PCXh1Tr~CEg^4xQSTeC*p{s=tVpQu_p84iTMs$b(3OER6qWo zsS7;MD7~f>zYlSL?##U@&um?JM{D}p@1{E$Z6z4c31a;K3&UP~l&4PRx@j#B-k zQTjXN_USRgRpfQ9_aMK|v~x-#eAQ3zOcHt(b}+@FWOAKLY=k`~uB*StH9SZ*(gK~6 zBbm`n13j7YuO5f_>6jID_M`O1{`=Z&p-j@XJ;*7GC$PwmiykD$@* z*!n&i$Ps?~2=#c|b2*Y{(++)pWB|u>qiZPhu&)<11Rm^TOclM^zmppNk<6j==?{q~ zuEl@+xF^E zVM{prqtnya5{V3r+(}MNSF_7q(p+L}Jv-5DiO;B>sP>XN!rE4ya;>iZ=t)bwu=#n# zQ1-3(CA9GHf55XQP`@;GPgN?i#Pri{Gco#iRli4;7`wht>fV*)S<{#1y3v;QoEe)H zVYXriZtjP?>u`G?gP&w0htxo4^)1DJY-Nnvt44fId>C8P>@H|Z9q~~u>x-(W+c28Tea5qZge`((j`cw^FYke)>nn{jX5_74&7H!?a;omo}j{cl*C6=7z`yUQZP8~8Q zI<=l}pJW`$!?x&qHhoau^>TQ!L2P^(`;BS;&M7_i8>?|)<7Ph2P8@5E%}O+z`z4?| z;b-Dm>EF#A;Ugq#NT-*7RrC3E>)1q%?kk}?IZ6I_tHqOuw+8mdW*gVSzh-B0y@xrN zVdtNDe3|zerF)sjnW6~}Xuv&2>FxkcI7FOHCfAL?%}i(l_F`Z4-Q>8203a4JbeSO^TF)i<_dfWx^9ElJYLq@+)OQzmz`nW9~xoa?N{c{Uo3ezyD4q$ zmBxJ8c*@{s2Z06YN*`?Uq^{uIs~7Y#e}XM(HM+@a^0QX+{AOnejT$2Gy`MOHH#Rug zzZhO8`m+K%s{gxvz2Kg`|8F^RL(e|zz5P?7%ne40ck_WKtskJnK2{dRHV!!-g zc9h+B`AIehc;q5<#wF;CFD0^XV;XzMC5#%MnQ-&?!i3S|%M-?oUmKEFzb=ZscbF^T zpm@+{D?8~&T>1w0&;{fej1Ai&w zdS`(4O@^*D4TW9>Xy0O;2WVfW&Y^vJX2(^W3oiJ78RtD|-}WutrHW5&%!MvpGs@by zWu*1sbw$>!n?_mv^NXxu#wcqbw6R+@arEHhLyX}K?0vC_^U27OTI1Ty0s6)Zf1}ik53_K7Wa`a^KbCg`vAi!4%e&MFjjS}B6%V|= zB;^qEv-nDU;LO=#;%(Ppi+pCIe~wBx_G6BAF-Oaw^@@84ht`)dN28(XFQ;E`opB|U zJji)9bU)K@tg22OVTsm=#>;-j`+V#+Rcb~xR!xJZANz5n`KPG8Rp;j$=5M&(L`=oG z`B7#ybM+Q9zH5H0xosh_p3tpb$iBNxPwH;I`yRA^H*+F;)p6*gY(cxRckO1csom%o z$BeMXLx!V6wY8)xhB=mARR(&6zXufm4@__&ORPPwtJK%Wb-eCu{^W(4sj#qvhcEIu20b};(FRVW>wzB>=x8>J=a!)~h#=(s7AqTG< zUv@Ba{LKgZk6(AN-}oOM95DXVgIVL>J2-H>_GHr@Zj0EH?In)dzilz+?{d`MZA&;m z$}zZa+gfaR>tdE{+-*2l9bW`}gqKy*Kh?RUCcG~n8L5Ub3swf1epvBU`C(;84_L9~ zf`B}53$P(sK=9JWd$NF;Cc(?f5mrxFS^G0TtUL&8>;hI60xQ4o0W0z!eKOy130{8p z6?oaT(vw=wT0aQ9l+TZ5j-9|q3N0R=aX76Ht*>{N;c5g2I+g_B#Ra_dT*HIBBMf-i z1P%tDAI&f^lzTl#6@PJo;1xL!ZoWvb7Dn*qT4dSLYb6r8NR z;->oQMAkmVs9&GvsNanJRxnTw49IsS{0PFpBJ>^Ek^ivCwL0_X?A6HeRoW=pJ+y~t zi)gE98~Htl<6w@Tyy04%$FZAZ3+>08f1h?gZ9MHT?S0ytv~Zs3E>a#zBsy;tx^FZ( za16R|EP8z$wQfRIWue#4lst@HzZ@CXuhYMu@^shAJ6dd=-ZQg$hYYTr|P_b^C8XOT7WyMBmFphl?xi=BT#_$Mq1?hWzv1?Z>=$ksr5gmzL=k|T* z3Lm0Byy}RFJcO>W4_!e#RJ#15?-p58`JQy~m4)mj9MHp84kIrIU49Gi`+(Ng5NfS) z8nZi!!mT6d`t{_?WIrqYEB_Zir9Kq3Z=P#TGGyy*Y)0ol9M66>>{)#{e9^b2qt`d1 z?`yB*X7qH$htvRr*|z>1X--GxDMKd=)>vAEPG|dM8*a9`S>x)JbFGe|F?<)D6`8j= z=8sj>jC?-(P+!v%{oATB}@!mLFC%=aKw%(82beMM^Dl|Hd z5Yr)lb1nGlD?%U6*6`!kjH6t_a>13Z4>=GxCeJyMj8O)dHiEQ=3yzpOue-V9* zYbPr&=tkfU_*~AKy88!oRqjm(-g9i3ix{UH*n5Tb3d&uJziE^{0K7}~YCzs{a4nb* z>Yj^7l7kBD@&1kiodfSWZ@Qjy;9cj7bq>7iJd^XD^4ePV=Deh<^4ld{bp$zC0ll@F z^XRwob0mY+qOU#zy!n7PVs37q6}>N|k^9Q+dYF3+j@XKIZ>&tIbT}*C|IE*Kyd32# zUPL^}LvJDr0+U0xBMWlOVExKyh0t;ra*HFxIX(ow^O-BOWB^C;9lk@{veTFUDSPup zzhCtV_L|O(5oTiS-l_wfpCOmwiG^|Iar~5Xu&V^+lHVTjq^j0^pUPM>xiZQeN!+Sp z?nV-$mF#lR!fEl|>s`r>=`Q3Fe4>gePj>Wa+&sT`Dm7C}h7aM$H_rZZZ*v5(h99Hr zN#`RK)%*Odz0GOPaO9jM??Cw05%g1M_K~zD1C05K2V0EXv7hySi+3esKe>KLa;oZR zcK&l8Gnsz;44LSE-P(tKg?oPo|2R)hL%-O{`YV%mZtRy(I6f<(X#ADvjQ!CWow0fK zAxTR%%9kiO?Bc)a4s>(kF6D3VP#>@byIdFb*Um4Tav=53$>wR`Uijo@eP&E{n>paq zAgvYiJsA9a+Tm$z!_N2g43DYWcu@u;|y|fM8v^ta1?p=Zp2fK>ks}g5KmgC8Q2^`16pw>UScDMT1PV1Dx?@N&Fv~KPp zDdyPBWb?|*M0+1c7jW&@FH>v(=)*qRV@>2MY-XLbCOcS@eAc8VO{#`3_oPXgT=%3& z)xEz=lQOvu(xj#Y&I2@QvCacDDO2YGnzSSS%j;pF*9U1*6Tby%QZ?s6npDR5KSGlR z?Fh)i+b_|iU;J&F6!KST(%N7CpQTAoWZ(ZPO;UXeXLdi^?-ry@1<)qdZ2d=Q(^qw! zAEWauL+5#eqw0e6gLe+V&YT7R9Eh#?N^H+pe|+`${@9;)2lSbp=rhLx`phoQ!)9j2 zNC$eC^8p_Zv;Vh&>tAvGCE5pk25Fya6{$w-)ySYJv=G`4kV*d+t)8}n)=K-BwiG$` zVOmh8eUN*%(Q>#)K>zC7X*bY5q-~*9)2`w7f1-ucx_3E|aYK;>!;lHXkqzC*h#q9z z2y&&Js|MVLJb?VxoM8-C?V^1Fns&&64+`3|2b!kZ50YD}p(}gg=kiI3hGla7J~S=p zld4WLN*~4^T!HORa^`&G%oo?EuX+RY{DJHNhnrY`H=geZ+XORDIan_cDTF<I+p#kby^*~(Z#2wyhs#^Z{5!GF?4<7PzC{JrueTLh zJ6KoE<7B?M$1qAi#^0~~JY%kp zHS0%WdxS45&;3ZW;m>J@XH|3F7yjh^d45g>Jmi_m2z&<-J?yFZ&u_sV?sLkf8c{Lx zjc4Ff5f#PYk8+smH5<9r`1Z|E%B{fhvkM>NyhKwIw)62md^QS z$rgT|)d=s2T99hqx**N0pdO&&fZN|;ZGp$V$Qvi&(|f5MD|zE8{FD`rXljeZczrg{ ziZzc7#b=foX|^Jl^iw@w>{xbPLEhu$zajiL6FX^Ez+QR=yK0vmqlFEWScD722w&OC z9%1&m?xhf$p#LP)9>Yc(ltb*ePUHdcylQl%yP+dLqE^eRt=+??@Xa0cr-F5?yw`Z6 zZrBR);*9;oQSL{s-@kl_+icGCm`?nHet)~ZlU6&wMc>tV65s4?e^2qh--doXN?S%7 zMZ1cYOzU=O-}x}`AsoDLgC8F71bQ+r5`VaJ)lg_kGrGdqD~#c8a3feBz>iZU{I{5A z#_{Y}-Z2K88jZi%W$UQfz*0J}u{e&L=RW@P0?w~SR%+f*U}du&E%4uH@cZabimVI$ zndh`3EB>xg)&T0tUj==-_EeE|J?pRcsNO>ZI!p;R2Js{1N$kbOUxQD#d7#tUE0}P_ zG&Zxx=$=30+XGI{#D`i<C3<(x?7J90>Gvt&rT zc}ccGhI{$}Kb&L?2{(P{50OJs`A?|XZ%BygmvNGjarz?TB-k>JVB&LpSCVti;)6X6 z44fmL?}8mS7fT*TT&=|#og$vN9e6m-@5flH3-*6mx2LE%d|;II8EdCKdXFNfVuQEk z)?mJd>RAe2*0Qdek4A9l5Vj*PN9mmJGbid>8~sIJdhVnvrX-f}U5(yiphv{L6k*0; zua4uq*TTN0V_~T2To`ZeT41C;YQ&ZNdO%#Ni{E}c+GWP&C!3Gr4__7HGJB0mHCK_Z z(d2fSMo4e-FgEEX#+HE%%c1y{>#;qLHnyoQC-lj-WdX0$5R3L@vSyFb-z;D*nsyG} z*6otbCIYyL1b(7`qiEnM2H3-Hv&*$gaN>r3E>AXwUxGu{7}`%hl;(BAF4p-(9y}9OCId$hw|gX1o}RnG2F^nZedI%;6gFD;J4QV{ZwUF zKr_LGO5iz@^S(dJpD~d2x^qFed49hi2O=IUveK>^W!2$N&4o7BE$!2p3vH~!o|6kM zD0fVHLME|M4bHy4AgPs(Ncq+Ug~mrqJD|8c;Qa?|}jsRYwI zB+VQ?gm@3+_e;JgSHKtbU3^hX@3nnVTOA&+yCBs(13gq+!+G?#3-mjm*qjUwJL%=>vd@hPs}zN4||cS+Qo1eeZ)7c#a2>kIfnB=}(p zKQuq!h33YGEi#A~^5H}?jxCHMhPhb9Tnzn>aPt)OAoiDu_;Y?$W#G4Q#JpRTxjn=@ z!dy()My@mS;9Tf5jcb$43?mKPNr3P6F%L8Lj0nR#<%(;(8eiSc@5Y&m%ZhV4ZTuM) zmTKNU#%bfv_UHrbg-!kweu^E$;$_S%+txILIB_@lGc$9WVx`g@F}CkUW7Rl)_=2Y~ zp69@ucW4`E->2PA`&)R_6L)Od`~PA5Dd>Sej<4WP7x;4?9uhJ2zZZYng+E1?@rT4B zOS!)Pz4(*-H}U5!8-G&3pB{c(@TVVs+7Oy-A3f!gWAKzv@MoSsAJC6KMSqDu?ck4O z6X6d?{eJ;}lK&3;Nim20E&NIT+xU|UzU0voXkWbUT7721FX1=TR3zRz7g=XIvd*5^ zFUvaANhG$e0Q$bu-UDEIKz5lCkX^)sBhh7Nfmeq(uY*o0Mk@!}JrjCa@lW`V;ltv^ zYrbW4xUunx=Cl)^e2O|;9mHE4CSJ*K<1>VR4kM;Sc017l<^L8tBE5&;p^JI1a&SK) zChbR#*v6IAJXbDY9Cq*9X@_~A>U15UcGaiIhEok^<79lDQ|Z5KFN$M6LVnh#$RJ0F zKg|yzJ}ZitkUwV3%KT#{wKMxsLo;jE6kaUvm*oC43lC%&A>I($ zqEY_-PQh=5K4SM*d4F-_T@G_bLd!kGY(|pv=a1*q{wTN6$6h`9>83yaXCisVpQjmo zW)XJy;5g$>5VLFdr-g4$VT@D1W{mlav9QM& zHLv^rr@_N^zdZH-Iy`h%&&s3~(#mOnH$2ea|55NjfBq^wd}WMZ2M?2hhwi1axr89k zg(BAx&vSP;bb$Rc=3#d!a<0mTW_X|(Psb4R${yx3bc7BlenoZ_>BZC@?vhL?*)fyr zNMzE%$d%HetEmYe1sxfJY`I2qGS|`QS@qa18ZP+)$MMave0vOi7)@VpMz$Pf(KlPq z8iJnHApMBigwe>BIxpjVCU)Us?B;66`!X?dhWWWs_Wq-Z{h`okhpWb0zu?6E`H?IeTKNnN!}^B$l%= zJ=0&qS^d#|mlx36FT&GQzi90b^DiEbln*fKJDcd+;Z)<#havhhXV)3PgM9o_C zlyPw$`PJ3*?cyi-Rx>iG`(pQ-?;scFSIpbCf&WgKdBZl1rQ6xtCwt^?8AAr^K8|(` z%}x7{*S^drRF7~EYo5-0W?;i{5BM^_$Rv)aC%@>aH|O(>J*>BJ)ELV19oUyj_gV!O zYt)f#6`ZdlKW968(WetNH9nrwy+t`)-~ zUS>Fq1wF^~503$!G8MV)?-~O%y^tExnb}i&jG^p*e+<-64vc|)O)v9t^&vJc|GBv7 z^PamMKc7J?s1Xv|m~R+f&9CfYnvY?`a2kO*S%k04Y0tOvHhNLg$RDGs9EEb4;?brs zuc7=$^}&vlW1$>W%{}Wt?p-7?CXwC-gZfkaPiuCXwNqZHD2Rn(i;L?_HU;%>Lh%Y4`KY<@@@tp1;z5-oBr>bE9$tm)Xzr zfBScGVffGe+$*==mz7pzzvrvJ>-$%xDaWwfexCnb>YvWlFa4)q+B*CBul}yzveMRT zZ1(g0(xzVezRvy6P7VAvX9+mVJU?=_o4u35n48$zsl-+V;aM@aqCs`ws=bC$tl>lR z{cDJfVz~dt8dCS|A6vt+ziSPbG1sH1C;lFF#8prHHjcBo{zh2xcT|sj4%e%>{t4Hr zKmJp$w{YFb^#-oDaxIw8%zl*iAZ;40oHmCxoAzDW{j~4V9-}=@dw@2V_5|(8v^m5e zRAUQx=UtT(l(=F#I|$r#z3xOM;W+&_LZCc zb;{BMwO6n&D}PY^&4b=57uPqf&{C{HGd2v_`_5tmY#vxZjVL!ZcgD#$US`d{!(_mSj%Dvn0=japdK$;|Bz=D(HsxBr8zq5o+9hpeLiDP}-9 zrdKfM!CK})nIPN!<+)UCmtZ|*a*6zPH=tdD{SPH4GS&vK<%OpPBtu`pm(!A#cTz z*WAwYl8|fT<=ORu23!Fxh=(R5KpT2P zPZIG}hOX)dJsAuANS{H?d*aach8w@m*&4Iz8S1^~&Yk{bG5OK?)O#1LX&z?`S3KT0 zS_F9?W2-%;dmJ{~DBeMPD0)wox87}5jO8A3^?kI5qyJDqw= ze5Nwge2ChC3D{lJk()lf!RUGk8}Mf%jIJZl#@2YFtL<*1YbSnE;Yu`3@?aA_pa6#1f!`;r%>Gb_*fbL0}$70^DU!Y%n5{PMn0D_uG$>uXKruS%u{UAHsLwn-}2 z`XDf|89iV1N}9lP$&ts2M^eqcZbzt1%gT{=Uc+9heB5P5nzzzOwR64Y6KK1e@gpy_ z-6z^d&1~$Oc1}C7C9)kTw_Ub`21j4p*CCv25-t);WB-mXN4R(d+$}(NP#smlO3n4g zaK*8Wr40fu#%;joH^SIF7&sY6^8!B)z2Pwj>mG3OZQ8TI#l|=Knk%m39&qv<+K;0D zRQ16dX=dASWAk6eX5IxX-w8~Y0^1XT@jKvix8qCp=UEO0eu(4j*m129 zjsPqPwl2d`FJxM0tsmCHd%#lM*T9my_MI#9pS)q9@uc*|%dlj81(srgCFm78iC{@^ z5J!xgV5y-8EY%Z#bR0Rn9+*s? z4Rc1I*QEItL%&2L1y8LuJn=j5gkRc*r%d4KDd?EScpbL3I3uxfDEh%Tn({uoz^iv$ ziJceIjjmrV?cM3eHzVOhoxd(aS3|zl?ua19kNOQy;p+!Zr=0Zr=GsR%;4jhmmU&M& zIE+r(8tJxr0lzItqiy-19sR!D5$gSvKDQRZv+(P;78&rPP;cv)!1WlT>mvLs5Bv}g zWB?!PuYA_E+2jM|8C~tzEIvbaR&9zIZjWg#P3-jWofFp-+w~(pn>Nnc01l2MXT61a zQ$G*G+qB*%CK_Gx=RDn?+)DC&TllVOK7TgNW3|vH)uX6={z;oxZaDq~?+Nk#vi)%@ zhZd#5BZ`q!9YWQj+|IM2S0$?;DvKV4c2 zENLts438&XG~R5do{DOVXn%O=ZGs!+``b0#sXf1)|Bs-q%nObr?>T2IHF5p6Q_)D( zl(~pL*9o0G2R&Uxt&=6xIx)iY>YcGmHl9P?E)%Wfx4Zt_{bpYysj(9~!`=9`eoHgP zFZ%KEY1I=v( zrVatC+yCGn-|-t9$c!M}jb=KKg(u@8Aeie#hezLMW$bZE!ip7O?bj8 zB$wfJY8D=+PQ@Ij3tM=cSI=x)dS_=IzP)^4Lw&5v{hsyx>|W-k_eNL|D@JJ2f`oLvW=&6vMlc!H=YPsmR?h+pRgTGN31 zCwB}moKos zUiW(8|A4o~z;7+!c@Hw%@@u%?JHP9j@K;at@2j$_Q_SXX7@Pl6kGzk5Ork&c(yx2y z-`()nyYQule<`j+d4*&9px+^@WpZx4bDdScX@qreHM&K)0c<6Aj_r?r0gt=!gCc7L zJhK|u$wuy+ha4u^^ZY=A8ZW&&RZ~>{b!T#I`_iV6Ro?ZwWgx>*^>^^OG# z59Aznv0aaUkMlb0e0qoYdKeXhRr2jG%F&78p|#YTR{vk!@`M$MKYcQEtPy{Dx5maX*#BJ44{^^&?w7_<$FYIy z3XZaA=5l0@zUOinM=?!uiZ2P8rQ>9yt1%$Ayvp%b;^pg@mq&=>+Rwfn=>^m;iifY= zXOI)yr*jE9zho8F`tkeHJk%gu^lwe@Jn_N~Xz3B?v0n!6RqMCOh1l|o_=&%cRRXvd z4&J+K*J2-OQVo1;b)waZpBL_lzsSyA=_H4edGp)x`eMiHgB>r?u7f67cLVdYjM`$7 zTd39MTMT`XEVJHqC$&Re__)%%>lsf2G4s;niLF~Dc_kD2abhxXOnsLEoudhVUa<9Z-j9dd>l}4n`U|wwbk-ByM_s4mC zyP~Y7m7$iGnpb`Jc?V&k*FU?y=o}GQ4XV0;&f>Vu6-hgY9NPEmc;ZpN1(*x?&n z8m_l;sdKQ=_dV-p_V^@AMS5SNZ}qO^&JOAnijN7-CgH0YM*gbmrk&%M4%{AQ9aJj_ zLE4N$t{4JL`)m^%u|<)h?IJgxzq}ZX*@mmP$?17@HqGQkh_u!zYW7 zQ&)1FKog7~88-2JIrQ-}Y`E?CY(7PHT8lq+9sby(>;-!4pZ>BY$YFAwj*#nAj=ozB zuIU;4*PD+3-&)gH{Ll|z<0UuQ`zz#O>8fRlJp*T_;QM9HY@C%Z>YX~*>Z7!MwDq)S zY28Z|a~1_piH5htz++*UWdsM~q&HlClr zd&cv=Tj5c+;FI&md#Og4JHVT&mCJ@6Xxlz@UMd>SHv+#m5g(lmPizKn)9Jr-K=Ig0 z&Kp@HcgUix;-$^$)SiLn%T^f8{j1=)(AthV;6eBl%YI6=1D~|6hWBQ`1N*`IiZ&Ek zqd4{_7esh98h_vz^uWFx6F9!kzD~;>ag8@&JGzeRM^DUu?gQ%3{SJ9F2YzXC{>2=J zbu8X#+4M9Oy#P8OeWh*07+a4x>MgWZI6}M^h^cM~_uI{mT^MEUMvwm!y1+b_)2rHY zFLG>#o;1*Z8#Y;}0TeQOunve(A&X~XL@QeZ5uz#(przx&cJV7+tjgFQ>4CER| zjIRt>{R1$k`guLyc?4hJU%qqU-*~6%(mRziE*^gZ8usjU@#fE%m)9H~?{G(|H;OtX zZt9feP+v}XE*`&`+NCS#SFmrBT`|73%=N9{b3u8ic>-EKg!-us4!;cQ;N6lzPcXO2 zrIP-V0gm%hgJZqvkXmDn1$1A7lU9E~HuLkIMI1nn@qGXwmC-`O(gzX$sb zP5rP(zcYc8%l%Hf)bC9ANHntKEy$89kR>aIMVl4K(24~pAjjgy%|+IrXlkM?=xd61iB?<% zzUG3%55v0@_v7GR(67G(T9NvTA}g9_UM0r88`x8RMI6t>@XSZD@i=3Bi?QPjh2~t1 ztlNw4M}4E0xq|O@JFvB3FTDEoB5Nr0lA!`4eRRQ{xERUFjrmm4(3hc_}?gsC3XfGgVR`AS)*IZUN`6yGN0g_YZ<|LX` z0~1Ym+*N6Fs1v0+;EFrwM9;ebKU@U7%Qx;Juc=nBUyzx0=fAU;cQ(FZTOnmP+ho^C@!)@=aZRFf` zUU6lLBfg6FcH$d3&m3LE7jpqRG9`S+jjAP~`V@;fZ*sZnjn1{R4i8*p$81C}SBHy; z*%-L}Tj88HW%*+^4reX;mhKt&gdKtVj*x^|O_%QX{mz9yx>kSsR8hSnr0=XFNjyic zSN_F}Sxu?izm-WWTmHvu$2&hDZ<=_v{P4v$CR59CF+B2Od4Y9-xt4Ej&$;e5Q_d|s zuy*G6FRkMYGT}}HLt-B};y3q@| zaRqcE9vLA4S)n&FLn5+65;8|JzSXc*LnL!RC$e*m;TND~TfiUrJETvR4MJ}O*QYbT z7BZrE|Bf8)f$P)Zoh``7x|hklZQ#~)kXf%4Sq}j_&Eyf({&OSIT^^_AUIeh*@{wWHw_j^@oOW2Jh8kV6%XKx3vSsCt z(*@Q@{1k1k6k><$uMw<2kH*%mO#I)r@U)B~SZ(Dad=yx`1uf#_f2foODIucpvMVji2&7r9t?P~nT zTBkL@RQ=M@&S%giYoLGK*e!R0$Fh?u=4Au=&WUp3(ddf@`9k+vp?ksmnI86I@3nJd zf8rHhR-6Ly3IjRb#&s{^7ZmSs2iJ4Bek>60@Hp2$<@%FAyu;79HnPjuM|~Qt=YHy6 z-EaN!e(L`H)i3Y2UW~j}%^a(5-Od!B;;>qr5slA8MtCEjl?BAp2KQ3W0ah0yvl)ST zw9v;z`xSem@n`WY{KbmM#b0<;Dzyw%6Va`>+s4VrN&S!&WoIDAy73|O>k8m0SF{z_ zaUvfUUZ0URm3j+>ZsoW7l^*8rH2O<`C*&nZ*wSm3uceAhkH0JOWbEg=R*&LtadSYNsS986Y z>wo21F(g~K{w>#i!jr$Vo9o|m?T;Z*{D|U76kDPg62*^vm!{Yf#gHg|MDZkwEm7>r z_i0bk=BBM+&dZRaeStY&+uB|F4r7xoMQ{?F-yobiz}M{Lj5OtSe~f>PALF5u3D9T!)kUGJ za-h%I(C6iYjp0`XX!9XxZd0yNdM&hBwPA!;Wn2$}E@vY@wL()Si&jIw2SJym-?c*b zbT5;8El+u@o;szUKHtfAOZomp`f&&SxgGj^8}xYs^m+Wl^w*}(&Cr{m?&yHdH&5jH zPCuQlAIPjU6@ z5Asnnv~aIu>c1Rv7#-Hc5mxI^>?QFAG&HgE1o1iTcN<-YfRB23)aS&DufwM#(1*1|U&!f|(IT2DNeZYDU4gkCleEVd-)xuqXD;=PGk9%`#4Qrq*c&Tsqy9-ou@ z7xYxWXWFqmi8f9B3^;w8|4yQolWK=5XH)bv9UfZ`ET2TyX#^+EzcSYL-6_U|y?Ws( zvZYcx0y=v>urHA>tI(Q+O=XQ>hxoQ-;7*Y4{#D#zKF%>0UCiA!j_+_h?Hgyk$rzL$ zcrMqCpNPF3;J=a&F0u~iN{E-?ImN?hFW9jwRZ|@ujj?IpsMEh^uaW?+{pEOj*Dhqd zvJt8N-&e<60UTM38=7kGTXGKjnT>yp`{LdHeP)AWm&|o8_wqg_!M$TIg7d1CE;toF zG%-g7%!TBEKVW|l%+xT}B>28yX0Ic)u?4x|>Ce9ZTne;Vup~IT_~|&t&R!4lx6&WY z?>YGJd6(8exd-rJ>b?jLTLf>DiAe_D+9Tn?g0EM?i7_$~&Ql{$`n=ZeT)M^+fn4ri zL#>s;n4bnd&-q-|=>UATp*JnJ-{H10c}EcD+|0FnrQ-FHY3w8O@;&y<48q>?%>UmD zd#Qo72*O{mrhcXPcmVz)fWPJXE^Fc8yYd<5zyr$$M4Q4LKMzl>4f5`uxgO##{}Xt5 zCa`h{Kk;_z@C(Nk|MA!GdjqgCvBtG}8tq2f0GcoNE$FxdI_`vSx}cvS(9uxnX&7`o z9DkN$Rrk{H{lPe`nn}iR)xNgnY}VK@i5R44KP}JWdhxx~LkQAxWKtVW@8MpMmV1$rO)@l~+uLn1Yxp*17dh_!J^(@FCc*W?NiyiV= zbbudl6#id?_nZmm-L5#>UVmZfXiIi(*&RFFT~O?YX!9i zr9-qmW%!>dLRWYJn8QzdCW}~_+BPXGmNBvkzTFgA^HbY*|85Z*x%gdhv0Yf8u|pTXJ>HFpz#T(Z~4C)U-nT_o-(*obkVLaoYYof2ppYr?a?-o8b4q{Hn(f{|6De`B`+m?yG zC|^~^j2pM*NJpf(nZr!v)-?E<^nZHPx9Gh3D|;{5Xu|^E)x7IlEj*WVrhBvU zDnw(U`yIDp6Bhl}x}1SNw?H%cAd}q)9ZrUBzfCQ^iJZR!E#CrtP>yLWGyvUqRm=K9 z>-_17_WARPZpjTYPUUs@7-je-A zvW|S|Em_6ZwvQgQ->LV#ef%+GY7c%O-dFDTEs2&MMxKx^QiI%A121{|__aKDwRM*M z59WQDz)ip8oHS=nzcfdvC%q86N=cu7X*IyGbOynoAD-^AkH5hV^Y_5hXVBJHf9qOZ zcS*kX=VS+IsD6K$-$e_x@5vnOu#=FpwMVV!rGNjxQufzy`$Q{?u~~^$dPFOMPuXf@ zix3SIZLA00?_oR<$oaYGIJ)14j34|hNFN{kFVM#a|9?auZ5s#l(Z?KiI}&``iQ8;- zL^f7{$G1R>w9jTPN1Gmfg*Ik=g*M9mBii@?_#~QOuNg4s$blv~Jn5Oh+}J*uX^#6t zVsm6C0XDBCj_5kcN~h;oo4_Xv*!9O~2l6rbkL)GA;f2;;p=qCChmQuvF6($-g+b~9 z@Vo4Uf;rVgob2lBD?v6?tamf`W7j4EpSpp=0&E2~O$PtutB_qqxSI#=I>571YKp(Q z!L@oP?FCx*F4?`~k;fB|$9p5YCL+ToAiI z=u+h9iTIShZ1Y}xBlH_PNp>l=5#(LoThYqAG*9!()LzcDx1;7N5;^$>>Jur4r3SfJ zc8gZ#hdrGmq3O4~qu)=NN!(Ivq|w!%nw!l@Mpp!K^>X%Ni9+sO&YqlZj?Dx4jthGx zYps22wz=5n7&)>P8Mh5ta>!7l^sRp{vTFK6+sl#nsMj8&=ZG`2pN}=VzS%zixjpF1 z&E4VFFjq=rHogrv`tlxlZ1WB1s{GcL6=umFbTZuNIyn?O!IlWCW1`WO0{v)NKgNpC z`XcL!FRNChbTIpz=Uavb&UvnRAnPz9a6N+G(24E;UBmep`k3m!7OZGpT9K>Y$FHP4 zBW_Z^lQc(ZkuA&@JhQ@0%}RrLaT{sA4}m8qxYCk4%JLNvv)@0`-bbh^y$~^d=g@>)w$`T!I zLk1G9l>hrAzP)wA;eZX7n9)GqY*|_bbF%^)Wu+tDjsX*Y6W^cgFnrRpg7%xM@Ga_l z>%G@nFS!yrk5!PLLB6YOCVP=j+O)w+&f7h+R+)b@9{4BQI1{tQ{%EUzh3Gxv%f~A6yLJtV1g>=8v_AU@bj?yxX!g z*>?7^cJ;J1tY1F;)f%*8Lp+5o`v`K;QRJf0$jbjWb2j7fMaZWi`*uDy9X~9vpK=eF z)cP-G{ofDpl-h>Jt+t`#tl}X~)5knlP|vRz->>`^@;O8?ZyyqurM=yPu-DG~$?hea z8FDmmoy6Sq)&$S#q~GwiJ)D-Ik~mv=w#*iKg#ikl;ayIM_(()|51)Vq#WN! zIr?FF_B`5L+V|6r6Q?Ghi1u68x+y>WI#*cZd)Pl-rY>_hu}MdXQCkKsze{|~GB-Kf z;Ih68&e*v88f)oh9z4+INamz@;3!M7r{cj{{t7)U-s`8QgD%n2h%eF8Waz14>?b>t zeb0c4iY1V4;dW6|F+e*FXr|F;itStcI@&M@+Hf{Ye*HxDzP`)e`}%LuhxXrmJ$;Z| zAzQM45A0-{J}3?$DDw*U#rqZeu$}q%-FnyR6SSSQi;uhw-*&*io$zsLm`w;lh6u$s zgH3N&_k#(ZSwYOV>R{eVZSNTNkH{k?>15KHl*vXECW~9o zx5fV`rEQC`)ybN5bF6eE`kF?OONc+Hl31B)og=fTCTAJvQP}>Ji`a%OMzuehitsyD z66b?&>7hQE^ukZgK^ld)lxMv`C^rw&oi0^NH zf}`|h>I=%h-2~6mUU;ED?|$L1$GG-QY*Wjb?{tqbdmZ{p{`IP-O&!0L$OEs70M!yD~^BnWu`~OO8A8-si|q zb-Bar`hZ=;i=S7#6YtoA{%K&dX`&xTkP{X$-qqw1Ou{d#xF!eVR9uXFH?}UWbp)Rs z@Y=fE+4eZQa%KDJgDuiO&K=A_9^=%QY6e~&Ly3Ht0XXmxccK_0*&h@K*}z);wRNw- zMyqjrPJb`b57{CttrtA*{PVyX^K93Cp9Wr(PqgI_^(h(Op890*Zpo|pS>RE5JT=i6 zKX7t}z9`N^@!W#h)5L{dfCsumjBBR@Ct=7H%Q?Djo$U6`9OgHNxl-*L1DY;b1Dm(C zHfqrwP0@?)aK@-sKyT`YDDPHvT9az)4!BmgQU9ieeQfQ#C16SOV)u8ce@^XsJMfG$ z=1u%x^CF#5W6TDYTj2k)&77Y&(Q2+6Z@H;;8ieH>`sM~+XVA|9